diff --git a/.binder/postBuild b/.binder/postBuild old mode 100644 new mode 100755 index c33605a68456c..00e8d39b93549 --- a/.binder/postBuild +++ b/.binder/postBuild @@ -6,9 +6,9 @@ set -e # inside a git checkout of the scikit-learn/scikit-learn repo. This script is # generating notebooks from the scikit-learn python examples. -if [[ ! -f /.dockerenv ]]; then - echo "This script was written for repo2docker and is supposed to run inside a docker container." - echo "Exiting because this script can delete data if run outside of a docker container." +if [[ -z "${REPO_DIR}" ]]; then + echo "This script was written for repo2docker and the REPO_DIR environment variable is supposed to be set." + echo "Exiting because this script can delete data if run outside of a repo2docker context." exit 1 fi @@ -23,7 +23,7 @@ find . -delete GENERATED_NOTEBOOKS_DIR=.generated-notebooks cp -r $TMP_CONTENT_DIR/examples $GENERATED_NOTEBOOKS_DIR -find $GENERATED_NOTEBOOKS_DIR -name '*.py' -exec sphx_glr_python_to_jupyter.py '{}' + +find $GENERATED_NOTEBOOKS_DIR -name '*.py' -exec sphinx_gallery_py2jupyter '{}' + NON_NOTEBOOKS=$(find $GENERATED_NOTEBOOKS_DIR -type f | grep -v '\.ipynb') rm -f $NON_NOTEBOOKS diff --git a/.binder/requirements.txt b/.binder/requirements.txt index 9ecc5c6fba79c..bd2b70f5f43b0 100644 --- a/.binder/requirements.txt +++ b/.binder/requirements.txt @@ -1,8 +1,10 @@ ---extra-index https://pypi.anaconda.org/scipy-wheels-nightly/simple scikit-learn +--find-links https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/scikit-learn --pre matplotlib scikit-image pandas +seaborn +Pillow sphinx-gallery scikit-learn - +polars diff --git a/.binder/runtime.txt b/.binder/runtime.txt new file mode 100644 index 0000000000000..8fdd90711cf30 --- /dev/null +++ b/.binder/runtime.txt @@ -0,0 +1 @@ +python-3.9 diff --git a/.circleci/artifact_path b/.circleci/artifact_path deleted file mode 100644 index 82181e4f2a5d1..0000000000000 --- a/.circleci/artifact_path +++ /dev/null @@ -1 +0,0 @@ -0/doc/_changed.html diff --git a/.circleci/config.yml b/.circleci/config.yml index bc4acd8a35fcb..bd4914056fe10 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,26 +1,38 @@ -version: 2 +version: 2.1 jobs: + lint: + docker: + - image: cimg/python:3.10.16 + steps: + - checkout + - run: + name: dependencies + command: | + source build_tools/shared.sh + # Include pytest compatibility with mypy + pip install pytest $(get_dep ruff min) $(get_dep mypy min) cython-lint + - run: + name: linting + command: ./build_tools/linting.sh + doc-min-dependencies: docker: - - image: circleci/python:3.7.3-stretch + - image: cimg/base:current-22.04 environment: - - OMP_NUM_THREADS: 2 - MKL_NUM_THREADS: 2 + - OPENBLAS_NUM_THREADS: 2 - CONDA_ENV_NAME: testenv - - PYTHON_VERSION: 3.6 - - NUMPY_VERSION: 'min' - - SCIPY_VERSION: 'min' - - MATPLOTLIB_VERSION: 'min' - - CYTHON_VERSION: 'min' - - SCIKIT_IMAGE_VERSION: 'min' - - SPHINX_VERSION: 'min' - - PANDAS_VERSION: 'min' + - LOCK_FILE: build_tools/circle/doc_min_dependencies_linux-64_conda.lock + # Do not fail if the documentation build generates warnings with minimum + # dependencies as long as we can avoid raising warnings with more recent + # versions of the same dependencies. + - SKLEARN_WARNINGS_AS_ERRORS: '0' steps: - checkout - run: ./build_tools/circle/checkout_merge_commit.sh - restore_cache: - key: v1-datasets-{{ .Branch }} + key: v1-doc-min-deps-datasets-{{ .Branch }} - restore_cache: keys: - doc-min-deps-ccache-{{ .Branch }} @@ -32,7 +44,7 @@ jobs: - ~/.ccache - ~/.cache/pip - save_cache: - key: v1-datasets-{{ .Branch }} + key: v1-doc-min-deps-datasets-{{ .Branch }} paths: - ~/scikit_learn_data - store_artifacts: @@ -44,24 +56,20 @@ jobs: doc: docker: - - image: circleci/python:3.7.3-stretch + - image: cimg/base:current-22.04 environment: - - OMP_NUM_THREADS: 2 - MKL_NUM_THREADS: 2 + - OPENBLAS_NUM_THREADS: 2 - CONDA_ENV_NAME: testenv - - PYTHON_VERSION: 3 - - NUMPY_VERSION: 'latest' - - SCIPY_VERSION: 'latest' - - MATPLOTLIB_VERSION: 'latest' - - CYTHON_VERSION: 'latest' - - SCIKIT_IMAGE_VERSION: 'latest' - - SPHINX_VERSION: 'min' - - PANDAS_VERSION: 'latest' + - LOCK_FILE: build_tools/circle/doc_linux-64_conda.lock + # Make sure that we fail if the documentation build generates warnings with + # recent versions of the dependencies. + - SKLEARN_WARNINGS_AS_ERRORS: '1' steps: - checkout - run: ./build_tools/circle/checkout_merge_commit.sh - restore_cache: - key: v1-datasets-{{ .Branch }} + key: v1-doc-datasets-{{ .Branch }} - restore_cache: keys: - doc-ccache-{{ .Branch }} @@ -73,7 +81,7 @@ jobs: - ~/.ccache - ~/.cache/pip - save_cache: - key: v1-datasets-{{ .Branch }} + key: v1-doc-datasets-{{ .Branch }} paths: - ~/scikit_learn_data - store_artifacts: @@ -88,43 +96,9 @@ jobs: root: doc/_build/html paths: . - lint: - docker: - - image: circleci/python:3.6 - steps: - - checkout - - run: ./build_tools/circle/checkout_merge_commit.sh - - run: - name: dependencies - command: sudo pip install flake8 - - run: - name: linting - command: ./build_tools/circle/linting.sh - - pypy3: - docker: - - image: condaforge/miniforge3 - environment: - # Avoid the interactive dialog when installing tzdata - - DEBIAN_FRONTEND: noninteractive - steps: - - restore_cache: - keys: - - pypy3-ccache-{{ .Branch }} - - pypy3-ccache - - run: apt-get -yq update && apt-get -yq install git ssh - - checkout - - run: conda init bash && source ~/.bashrc - - run: ./build_tools/circle/build_test_pypy.sh - - save_cache: - key: pypy3-ccache-{{ .Branch }}-{{ .BuildNum }} - paths: - - ~/.ccache - - ~/.cache/pip - deploy: docker: - - image: circleci/python:3.6 + - image: cimg/base:current-22.04 steps: - checkout - run: ./build_tools/circle/checkout_merge_commit.sh @@ -133,7 +107,7 @@ jobs: - attach_workspace: at: doc/_build/html - run: ls -ltrh doc/_build/html/stable - - deploy: + - run: command: | if [[ "${CIRCLE_BRANCH}" =~ ^main$|^[0-9]+\.[0-9]+\.X$ ]]; then bash build_tools/circle/push_doc.sh doc/_build/html/stable @@ -150,21 +124,6 @@ workflows: - doc-min-dependencies: requires: - lint - - pypy3: - filters: - branches: - only: - - 0.20.X - deploy: requires: - doc - pypy: - triggers: - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - main - jobs: - - pypy3 diff --git a/.codecov.yml b/.codecov.yml index d430925ea7508..f4ecd6e7d8fee 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -28,4 +28,6 @@ codecov: ignore: - "sklearn/externals" - "sklearn/_build_utils" -- "**/setup.py" +- "sklearn/__check_build" +- "sklearn/_min_dependencies.py" +- "**/conftest.py" diff --git a/.coveragerc b/.coveragerc index a8601458a0b07..0d5f02b3edafc 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,9 +1,11 @@ [run] -branch = True +# Use statement coverage rather than branch coverage because +# COVERAGE_CORE=sysmon can make branch coverage slower rather than faster. See +# https://github.com/nedbat/coveragepy/issues/1812 for more details. +branch = False source = sklearn parallel = True omit = */sklearn/externals/* */sklearn/_build_utils/* */benchmarks/* - **/setup.py diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000000000..77fb878ee8fe7 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,48 @@ +# Since git version 2.23, git-blame has a feature to ignore +# certain commits. +# +# This file contains a list of commits that are not likely what +# you are looking for in `git blame`. You can set this file as +# a default ignore file for blame by running the following +# command. +# +# $ git config blame.ignoreRevsFile .git-blame-ignore-revs + +# PR 18948: Migrate code style to Black +82df48934eba1df9a1ed3be98aaace8eada59e6e + +# PR 20294: Use target_version >= 3.7 in Black +351ace7935a4ea685171cc6d174890f08facd561 + +# PR 20412: Use experimental_string_processing=true in Black +3ae7c7615343bbd36acece57825d8b0d70fd9da4 + +# PR 20502: Runs Black on examples +70a185ae59b4362633d18b0d0083abb1b6f7370c + +# PR 22474: Update to Black 22.1.0 +1fc86b6aacd89da44a3b4e8abf7c3e2ba4336ffe + +# PR 22983: Update to Black 22.3.0 +d4aad64b1eb2e42e76f49db2ccfbe4b4660d092b + +# PR 26110: Update black to 23.3.0 +893d5accaf9d16f447645e704f85a216187564f7 + +# PR 26649: Add isort and ruff rules +42173fdb34b5aded79664e045cada719dfbe39dc + +# PR 28802: Update black to 24.3.0 +c4c546355667b070edd5c892b206aa4a97af9a0b + +# PR 30694: Enforce ruff rules (RUF) +fe7c4176828af5231f526e76683fb9bdb9ea0367 + +# PR 30695: Apply ruff/flake8-implicit-str-concat rules (ISC) +5cdbbf15e3fade7cc2462ef66dc4ea0f37f390e3 + +# PR 31015: black -> ruff format +ff78e258ccf11068e2b3a433c51517ae56234f88 + +# PR 31226: Enforce ruff/pygrep-hooks rules +b98dc797c480b1b9495f918e201d45ee07f29feb diff --git a/.gitattributes b/.gitattributes index 163f2a4fe2030..f45e0f29ccfa2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,6 @@ -/doc/whats_new.rst merge=union +.* export-ignore +asv_benchmarks export-ignore +azure-pipelines.yml export-ignore +benchmarks export-ignore +build_tools export-ignore +maint_tools export-ignore diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index 102ebd0770535..0000000000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,71 +0,0 @@ ---- -name: Bug report -about: Create a report to help us reproduce and correct the bug -title: '' -labels: 'Bug: triage' -assignees: '' - ---- - - - -#### Describe the bug - - -#### Steps/Code to Reproduce - - -``` -Sample code to reproduce the problem -``` - -#### Expected Results - - -#### Actual Results - - -#### Versions - - - - diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000000000..bc8e5b5ff70d1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,95 @@ +name: Bug Report +description: Create a report to help us reproduce and correct the bug +labels: ['Bug', 'Needs Triage'] + +body: +- type: markdown + attributes: + value: > + #### Before submitting a bug, please make sure the issue hasn't been already + addressed by searching through [the past issues](https://github.com/scikit-learn/scikit-learn/issues). +- type: textarea + attributes: + label: Describe the bug + description: > + A clear and concise description of what the bug is. + validations: + required: true +- type: textarea + attributes: + label: Steps/Code to Reproduce + description: | + Please add a [minimal code example](https://scikit-learn.org/dev/developers/minimal_reproducer.html) that can reproduce the error when running it. Be as succinct as possible, **do not depend on external data files**: instead you can generate synthetic data using `numpy.random`, [sklearn.datasets.make_regression](https://scikit-learn.org/stable/modules/generated/sklearn.datasets.make_regression.html), [sklearn.datasets.make_classification](https://scikit-learn.org/stable/modules/generated/sklearn.datasets.make_classification.html) or a few lines of Python code. Example: + + ```python + from sklearn.feature_extraction.text import CountVectorizer + from sklearn.decomposition import LatentDirichletAllocation + docs = ["Help I have a bug" for i in range(1000)] + vectorizer = CountVectorizer(input=docs, analyzer='word') + lda_features = vectorizer.fit_transform(docs) + lda_model = LatentDirichletAllocation( + n_topics=10, + learning_method='online', + evaluate_every=10, + n_jobs=4, + ) + model = lda_model.fit(lda_features) + ``` + + If the code is too long, feel free to put it in a public gist and link it in the issue: https://gist.github.com. + + In short, **we are going to copy-paste your code** to run it and we expect to get the same result as you. + + We acknowledge that crafting a [minimal reproducible code example](https://scikit-learn.org/dev/developers/minimal_reproducer.html) requires some effort on your side but it really helps the maintainers quickly reproduce the problem and analyze its cause without any ambiguity. Ambiguous bug reports tend to be slower to fix because they will require more effort and back and forth discussion between the maintainers and the reporter to pin-point the precise conditions necessary to reproduce the problem. + placeholder: | + ``` + Sample code to reproduce the problem + ``` + validations: + required: true +- type: textarea + attributes: + label: Expected Results + description: > + Please paste or describe the expected results. + placeholder: > + Example: No error is thrown. + validations: + required: true +- type: textarea + attributes: + label: Actual Results + description: | + Please paste or describe the results you observe instead of the expected results. If you observe an error, please paste the error message including the **full traceback** of the exception. For instance the code above raises the following exception: + + ```python-traceback + --------------------------------------------------------------------------- + TypeError Traceback (most recent call last) + in + 4 vectorizer = CountVectorizer(input=docs, analyzer='word') + 5 lda_features = vectorizer.fit_transform(docs) + ----> 6 lda_model = LatentDirichletAllocation( + 7 n_topics=10, + 8 learning_method='online', + + TypeError: __init__() got an unexpected keyword argument 'n_topics' + ``` + placeholder: > + Please paste or specifically describe the actual output or traceback. + validations: + required: true +- type: textarea + attributes: + label: Versions + render: shell + description: | + Please run the following and paste the output below. + ```python + import sklearn; sklearn.show_versions() + ``` + validations: + required: true +- type: markdown + attributes: + value: > + Thanks for contributing 🎉! diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 7d39c399ca81b..0ebed8c85161b 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,14 +1,17 @@ blank_issues_enabled: false contact_links: - name: Discussions - url: https://github.com/scikit-learn/scikit-learn/discussions + url: https://github.com/scikit-learn/scikit-learn/discussions/new about: Ask questions and discuss with other scikit-learn community members - - name: Stack overflow + - name: Stack Overflow url: https://stackoverflow.com/questions/tagged/scikit-learn - about: Please ask and answer usage questions on stackoverflow - - name: Mailing list + about: Please ask and answer usage questions on Stack Overflow + - name: Mailing list url: https://mail.python.org/mailman/listinfo/scikit-learn about: General discussions and announcements on the mailing list - - name: Gitter - url: https://gitter.im/scikit-learn/scikit-learn - about: Users and developers can sometimes be found on the gitter channel + - name: Discord server + url: https://discord.gg/h9qyrK8Jc8 + about: Developers and users can be found on the Discord server + - name: Blank issue + url: https://github.com/scikit-learn/scikit-learn/issues/new?template=BLANK_ISSUE + about: Please note that GitHub Discussions should be used in most cases instead diff --git a/.github/ISSUE_TEMPLATE/doc_improvement.md b/.github/ISSUE_TEMPLATE/doc_improvement.md deleted file mode 100644 index 4c2906bb18418..0000000000000 --- a/.github/ISSUE_TEMPLATE/doc_improvement.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: Documentation improvement -about: Create a report to help us improve the documentation. Alternatively you can just open a pull request with the suggested change. -title: '' -labels: Documentation -assignees: '' - ---- - -#### Describe the issue linked to the documentation - - - -#### Suggest a potential alternative/fix - - diff --git a/.github/ISSUE_TEMPLATE/doc_improvement.yml b/.github/ISSUE_TEMPLATE/doc_improvement.yml new file mode 100644 index 0000000000000..48d0c3de89103 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/doc_improvement.yml @@ -0,0 +1,17 @@ +name: Documentation improvement +description: Create a report to help us improve the documentation. Alternatively you can just open a pull request with the suggested change. +labels: [Documentation, 'Needs Triage'] + +body: +- type: textarea + attributes: + label: Describe the issue linked to the documentation + description: > + Tell us about the confusion introduced in the documentation. + validations: + required: true +- type: textarea + attributes: + label: Suggest a potential alternative/fix + description: > + Tell us how we could improve the documentation in this regard. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index b2ff110d69a04..0000000000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -name: Feature request -about: Suggest a new algorithm, enhancement to an existing algorithm, etc. -title: '' -labels: New Feature -assignees: '' - ---- - - - -#### Describe the workflow you want to enable - -#### Describe your proposed solution - -#### Describe alternatives you've considered, if relevant - -#### Additional context diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000000000..51a2cdd94920d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,25 @@ +name: Feature request +description: Suggest a new algorithm, enhancement to an existing algorithm, etc. +labels: ['New Feature', 'Needs Triage'] + +body: +- type: markdown + attributes: + value: > + #### If you want to propose a new algorithm, please refer first to the [scikit-learn inclusion criterion](https://scikit-learn.org/stable/faq.html#what-are-the-inclusion-criteria-for-new-algorithms). +- type: textarea + attributes: + label: Describe the workflow you want to enable + validations: + required: true +- type: textarea + attributes: + label: Describe your proposed solution + validations: + required: true +- type: textarea + attributes: + label: Describe alternatives you've considered, if relevant +- type: textarea + attributes: + label: Additional context diff --git a/.github/ISSUE_TEMPLATE/other_template.md b/.github/ISSUE_TEMPLATE/other_template.md deleted file mode 100644 index d46ae9e50b18f..0000000000000 --- a/.github/ISSUE_TEMPLATE/other_template.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -name: Other -about: For all other issues to reach the community... -title: '' -labels: '' -assignees: '' - ---- - - diff --git a/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md similarity index 92% rename from PULL_REQUEST_TEMPLATE.md rename to .github/PULL_REQUEST_TEMPLATE.md index 8528d5386b58a..f59f9bc2fbcd7 100644 --- a/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -26,7 +26,7 @@ review, either the pull request needs some benchmarking, tinkering, convincing, etc. or more likely the reviewers are simply busy. In either case, we ask for your understanding during the review process. For more information, see our FAQ on this topic: -http://scikit-learn.org/dev/faq.html#why-is-my-pull-request-not-getting-any-attention. +https://scikit-learn.org/dev/faq.html#why-is-my-pull-request-not-getting-any-attention. Thanks for contributing! --> diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000000..7ac17eb0442ad --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,21 @@ +version: 2 +updates: + # Maintain dependencies for GitHub Actions as recommended in SPEC8: + # https://github.com/scientific-python/specs/pull/325 + # At the time of writing, release critical workflows such as + # pypa/gh-action-pypi-publish should use hash-based versioning for security + # reasons. This strategy may be generalized to all other github actions + # in the future. + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + groups: + actions: + patterns: + - "*" + labels: + - "Build / CI" + - "dependencies" + reviewers: + - "scikit-learn/core-devs" diff --git a/.github/labeler-file-extensions.yml b/.github/labeler-file-extensions.yml new file mode 100644 index 0000000000000..63fcfcacfeb17 --- /dev/null +++ b/.github/labeler-file-extensions.yml @@ -0,0 +1,8 @@ +cython: +- sklearn/**/*.pyx +- sklearn/**/*.pxd +- sklearn/**/*.pxi +# Tempita templates +- sklearn/**/*.pyx.tp +- sklearn/**/*.pxd.tp +- sklearn/**/*.pxi.tp diff --git a/.github/scripts/label_title_regex.py b/.github/scripts/label_title_regex.py index 26fc02b3aef38..9a689b8db09b4 100644 --- a/.github/scripts/label_title_regex.py +++ b/.github/scripts/label_title_regex.py @@ -1,25 +1,25 @@ """Labels PRs based on title. Must be run in a github action with the pull_request_target event.""" -from ghapi.all import context_github -from ghapi.all import GhApi -from ghapi.all import user_repo -from ghapi.all import github_token + +import json +import os import re -owner, repo = user_repo() -pull_request = context_github.event.pull_request -title = pull_request.title +from github import Github + +context_dict = json.loads(os.getenv("CONTEXT_GITHUB")) + +repo = context_dict["repository"] +g = Github(context_dict["token"]) +repo = g.get_repo(repo) +pr_number = context_dict["event"]["number"] +issue = repo.get_issue(number=pr_number) +title = issue.title + -regex_to_labels = [ - (r"\bDOC\b", "Documentation"), - (r"\bCI\b", "Build / CI") -] +regex_to_labels = [(r"\bDOC\b", "Documentation"), (r"\bCI\b", "Build / CI")] -labels_to_add = [ - label for regex, label in regex_to_labels - if re.search(regex, title) -] +labels_to_add = [label for regex, label in regex_to_labels if re.search(regex, title)] if labels_to_add: - api = GhApi(owner=owner, repo=repo, token=github_token()) - api.issues.add_labels(pull_request.number, labels=labels_to_add) + issue.add_to_labels(*labels_to_add) diff --git a/.github/workflows/arm-unit-tests.yml b/.github/workflows/arm-unit-tests.yml new file mode 100644 index 0000000000000..e7636d55d7945 --- /dev/null +++ b/.github/workflows/arm-unit-tests.yml @@ -0,0 +1,54 @@ +name: Unit test for ARM +permissions: + contents: read + +on: + push: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + if: github.repository == 'scikit-learn/scikit-learn' + + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + - name: Install linters + run: | + source build_tools/shared.sh + # Include pytest compatibility with mypy + pip install pytest $(get_dep ruff min) $(get_dep mypy min) cython-lint + - name: Run linters + run: ./build_tools/linting.sh + - name: Run Meson OpenMP checks + run: | + pip install ninja meson scipy + python build_tools/check-meson-openmp-dependencies.py + + run-unit-tests: + name: Run unit tests + runs-on: ubuntu-24.04-arm + if: github.repository == 'scikit-learn/scikit-learn' + needs: [lint] + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: mamba-org/setup-micromamba@v2 + with: + environment-file: build_tools/github/pymin_conda_forge_arm_linux-aarch64_conda.lock + environment-name: ci + cache-environment: true + + - name: Build and run tests + shell: bash -el {0} + run: bash build_tools/github/build_test_arm.sh diff --git a/.github/workflows/artifact-redirector.yml b/.github/workflows/artifact-redirector.yml new file mode 100644 index 0000000000000..690cacefda935 --- /dev/null +++ b/.github/workflows/artifact-redirector.yml @@ -0,0 +1,24 @@ +name: CircleCI artifacts redirector +on: [status] + +# Restrict the permissions granted to the use of secrets.GITHUB_TOKEN in this +# github actions workflow: +# https://docs.github.com/en/actions/security-guides/automatic-token-authentication +permissions: + statuses: write + +jobs: + circleci_artifacts_redirector_job: + runs-on: ubuntu-latest + # For testing this action on a fork, remove the "github.repository =="" condition. + if: "github.repository == 'scikit-learn/scikit-learn' && github.event.context == 'ci/circleci: doc'" + name: Run CircleCI artifacts redirector + steps: + - name: GitHub Action step + uses: scientific-python/circleci-artifacts-redirector-action@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + api-token: ${{ secrets.CIRCLECI_TOKEN }} + artifact-path: 0/doc/_changed.html + circleci-jobs: doc + job-title: Check the rendered docs here! diff --git a/.github/workflows/assign.yml b/.github/workflows/assign.yml index 4b067135bbfdb..a69b60ee0f0a0 100644 --- a/.github/workflows/assign.yml +++ b/.github/workflows/assign.yml @@ -4,15 +4,27 @@ on: issue_comment: types: created +# Restrict the permissions granted to the use of secrets.GITHUB_TOKEN in this +# github actions workflow: +# https://docs.github.com/en/actions/security-guides/automatic-token-authentication +permissions: + issues: write + jobs: one: runs-on: ubuntu-latest + # Note that string comparisons is not case sensitive. if: >- - (github.event.comment.body == 'take' || - github.event.comment.body == 'Take') - && !github.event.issue.assignee + startsWith(github.event.comment.body, '/take') + && !github.event.issue.assignee steps: - run: | + # Using REST API directly because assigning through gh has some severe limitations. For more details, see + # https://github.com/scikit-learn/scikit-learn/issues/29395#issuecomment-2206776963 echo "Assigning issue ${{ github.event.issue.number }} to ${{ github.event.comment.user.login }}" - curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"assignees": ["${{ github.event.comment.user.login }}"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees - curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -X "DELETE" https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/labels/help%20wanted + curl -H "Authorization: token $GH_TOKEN" -d '{"assignees": ["${{ github.event.comment.user.login }}"]}' \ + https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees + gh issue edit $ISSUE --remove-label "help wanted" + env: + GH_TOKEN: ${{ github.token }} + ISSUE: ${{ github.event.issue.html_url }} diff --git a/.github/workflows/check-changelog.yml b/.github/workflows/check-changelog.yml index 753f473354131..00e6a81f8cd0b 100644 --- a/.github/workflows/check-changelog.yml +++ b/.github/workflows/check-changelog.yml @@ -1,54 +1,36 @@ name: Check Changelog +permissions: + contents: read + # This check makes sure that the changelog is properly updated # when a PR introduces a change in a test file. # To bypass this check, label the PR with "No Changelog Needed". on: pull_request: - types: [opened, edited, labeled, unlabeled, synchronize] + types: [opened, synchronize, labeled, unlabeled] jobs: check: + name: A reviewer will let you know if it is required or can be bypassed runs-on: ubuntu-latest - if: ${{ contains(github.event.pull_request.labels.*.name, 'No Changelog Needed') == 0 }} steps: - - name: Get PR number and milestone - run: | - echo "PR_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_ENV - echo "TAGGED_MILESTONE=${{ github.event.pull_request.milestone.title }}" >> $GITHUB_ENV - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: '0' - - name: Check the changelog + - name: Check if tests have changed + id: tests_changed run: | set -xe changed_files=$(git diff --name-only origin/main) # Changelog should be updated only if tests have been modified - if [[ ! "$changed_files" =~ tests ]] + if [[ "$changed_files" =~ tests ]] then - exit 0 - fi - all_changelogs=$(cat ./doc/whats_new/v*.rst) - if [[ "$all_changelogs" =~ :pr:\`$PR_NUMBER\` ]] - then - echo "Changelog has been updated." - # If the pull request is milestoned check the correspondent changelog - if exist -f ./doc/whats_new/v${TAGGED_MILESTONE:0:4}.rst - then - expected_changelog=$(cat ./doc/whats_new/v${TAGGED_MILESTONE:0:4}.rst) - if [[ "$expected_changelog" =~ :pr:\`$PR_NUMBER\` ]] - then - echo "Changelog and milestone correspond." - else - echo "Changelog and milestone do not correspond." - echo "If you see this error make sure that the tagged milestone for the PR" - echo "and the changelog name properly match." - exit 1 - fi - fi - else - echo "Changelog entry is missing." - echo "If you see this error and there is already a changelog entry then make sure that" - echo "the PR number is correct. If no changelog entry is required for this PR," - echo "label the PR with 'No Changelog Needed' to bypass this check." - exit 1 + echo "check_changelog=true" >> $GITHUB_OUTPUT fi + + - name: Check changelog entry + if: steps.tests_changed.outputs.check_changelog == 'true' + uses: scientific-python/action-towncrier-changelog@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BOT_USERNAME: changelog-bot diff --git a/.github/workflows/check-manifest.yml b/.github/workflows/check-manifest.yml deleted file mode 100644 index b9ce183890e72..0000000000000 --- a/.github/workflows/check-manifest.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: "Check Manifest" - -on: - schedule: - - cron: '0 0 * * *' - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: '3.9' - - name: Install dependencies - # scipy and cython are required to build sdist - run: | - python -m pip install --upgrade pip - pip install check-manifest scipy cython - - run: | - check-manifest -v diff --git a/.github/workflows/check-sdist.yml b/.github/workflows/check-sdist.yml new file mode 100644 index 0000000000000..d97236dae1e40 --- /dev/null +++ b/.github/workflows/check-sdist.yml @@ -0,0 +1,35 @@ +name: "Check sdist" +permissions: + contents: read + +on: + schedule: + - cron: '0 0 * * *' + +jobs: + check-sdist: + # Don't run on forks + if: github.repository == 'scikit-learn/scikit-learn' + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Install dependencies + # scipy and cython are required to build sdist + run: | + python -m pip install --upgrade pip + pip install check-sdist + - run: | + check-sdist --inject-junk + + update-tracker: + uses: ./.github/workflows/update_tracking_issue.yml + if: ${{ always() }} + needs: [check-sdist] + with: + job_status: ${{ needs.check-sdist.result }} + secrets: + BOT_GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000000000..58b8fbf5c4ce7 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,73 @@ +name: "CodeQL" + +on: + push: + branches: [ "main", "*.X" ] + pull_request: + branches: [ "main", "*.X" ] + schedule: + - cron: '0 6 * * 1' + +jobs: + analyze: + name: Analyze + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners + # Consider using larger runners for possible analysis time improvements. + runs-on: 'ubuntu-latest' + timeout-minutes: 360 + permissions: + # required for all workflows + security-events: write + + # only required for workflows in private repositories + actions: read + contents: read + + strategy: + fail-fast: false + matrix: + language: [ 'javascript-typescript', 'python', 'actions' ] + # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ] + # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v3 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/cuda-ci.yml b/.github/workflows/cuda-ci.yml new file mode 100644 index 0000000000000..028ff06903e8a --- /dev/null +++ b/.github/workflows/cuda-ci.yml @@ -0,0 +1,78 @@ +name: CUDA GPU +permissions: + contents: read + +# Only run this workflow when a Pull Request is labeled with the +# 'CUDA CI' label. +on: + pull_request: + types: + - labeled + +jobs: + build_wheel: + if: contains(github.event.pull_request.labels.*.name, 'CUDA CI') + runs-on: "ubuntu-latest" + name: Build wheel for Pull Request + steps: + - uses: actions/checkout@v4 + + - name: Build wheels + uses: pypa/cibuildwheel@faf86a6ed7efa889faf6996aa23820831055001a + env: + CIBW_BUILD: cp313-manylinux_x86_64 + CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 + CIBW_BUILD_VERBOSITY: 1 + CIBW_ARCHS: x86_64 + + - uses: actions/upload-artifact@v4 + with: + name: cibw-wheels + path: ./wheelhouse/*.whl + + tests: + if: contains(github.event.pull_request.labels.*.name, 'CUDA CI') + needs: [build_wheel] + runs-on: + group: cuda-gpu-runner-group + # Set this high enough so that the tests can comforatble run. We set a + # timeout to make abusing this workflow less attractive. + timeout-minutes: 20 + name: Run Array API unit tests + steps: + - uses: actions/download-artifact@v4 + with: + pattern: cibw-wheels + path: ~/dist + + - uses: actions/setup-python@v5 + with: + # XXX: The 3.12.4 release of Python on GitHub Actions is corrupted: + # https://github.com/actions/setup-python/issues/886 + python-version: '3.12.3' + - name: Checkout main repository + uses: actions/checkout@v4 + - name: Cache conda environment + id: cache-conda + uses: actions/cache@v4 + with: + path: ~/conda + key: ${{ runner.os }}-build-${{ hashFiles('build_tools/github/create_gpu_environment.sh') }}-${{ hashFiles('build_tools/github/pylatest_conda_forge_cuda_array-api_linux-64_conda.lock') }} + - name: Install miniforge + if: ${{ steps.cache-conda.outputs.cache-hit != 'true' }} + run: bash build_tools/github/create_gpu_environment.sh + - name: Install scikit-learn + run: | + source "${HOME}/conda/etc/profile.d/conda.sh" + conda activate sklearn + pip install ~/dist/cibw-wheels/$(ls ~/dist/cibw-wheels) + + - name: Run array API tests + run: | + source "${HOME}/conda/etc/profile.d/conda.sh" + conda activate sklearn + python -c "import sklearn; sklearn.show_versions()" + + SCIPY_ARRAY_API=1 pytest --pyargs sklearn -k 'array_api' -v + # Run in /home/runner to not load sklearn from the checkout repo + working-directory: /home/runner diff --git a/.github/workflows/cuda-label-remover.yml b/.github/workflows/cuda-label-remover.yml new file mode 100644 index 0000000000000..bb87f5419b662 --- /dev/null +++ b/.github/workflows/cuda-label-remover.yml @@ -0,0 +1,23 @@ +name: Remove "CUDA CI" Label + +# This workflow removes the "CUDA CI" label that triggers the actual +# CUDA CI. It is separate so that we can use the `pull_request_target` +# trigger which has a API token with write access. +on: + pull_request_target: + types: + - labeled + +# In order to remove the "CUDA CI" label we need to have write permissions for PRs +permissions: + pull-requests: write + +jobs: + label-remover: + if: contains(github.event.pull_request.labels.*.name, 'CUDA CI') + name: Remove "CUDA CI" Label + runs-on: ubuntu-24.04 + steps: + - uses: actions-ecosystem/action-remove-labels@v1 + with: + labels: CUDA CI diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml new file mode 100644 index 0000000000000..6ed68496de8b2 --- /dev/null +++ b/.github/workflows/emscripten.yml @@ -0,0 +1,108 @@ +name: Test Emscripten/Pyodide build + +on: + schedule: + # Nightly build at 3:42 A.M. + - cron: "42 3 */1 * *" + push: + branches: + - main + # Release branches + - "[0-9]+.[0-9]+.X" + pull_request: + branches: + - main + - "[0-9]+.[0-9]+.X" + # Manual run + workflow_dispatch: + +env: + FORCE_COLOR: 3 + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + check_build_trigger: + name: Check build trigger + runs-on: ubuntu-latest + if: github.repository == 'scikit-learn/scikit-learn' + outputs: + build: ${{ steps.check_build_trigger.outputs.build }} + steps: + - name: Checkout scikit-learn + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + + - id: check_build_trigger + name: Check build trigger + shell: bash + run: | + set -e + set -x + + COMMIT_MSG=$(git log --no-merges -1 --oneline) + + # The commit marker "[pyodide]" will trigger the build when required + if [[ "$GITHUB_EVENT_NAME" == schedule || + "$GITHUB_EVENT_NAME" == workflow_dispatch || + "$COMMIT_MSG" =~ \[pyodide\] ]]; then + echo "build=true" >> $GITHUB_OUTPUT + fi + + build_wasm_wheel: + name: Build WASM wheel + runs-on: ubuntu-latest + needs: check_build_trigger + if: needs.check_build_trigger.outputs.build + steps: + - name: Checkout scikit-learn + uses: actions/checkout@v4 + with: + persist-credentials: false + + - uses: pypa/cibuildwheel@faf86a6ed7efa889faf6996aa23820831055001a + env: + CIBW_PLATFORM: pyodide + SKLEARN_SKIP_OPENMP_TEST: "true" + SKLEARN_SKIP_NETWORK_TESTS: 1 + # Temporary work-around to avoid joblib 1.5.0 until there is a joblib + # release with https://github.com/joblib/joblib/pull/1721 + CIBW_TEST_REQUIRES: "pytest pandas joblib!=1.5.0" + # -s pytest argument is needed to avoid an issue in pytest output capturing with Pyodide + CIBW_TEST_COMMAND: "python -m pytest -svra --pyargs sklearn --durations 20 --showlocals" + + - name: Upload wheel artifact + uses: actions/upload-artifact@v4 + with: + name: pyodide_wheel + path: ./wheelhouse/*.whl + if-no-files-found: error + + # Push to https://anaconda.org/scientific-python-nightly-wheels/scikit-learn + # WARNING: this job will overwrite any existing WASM wheels. + upload-wheels: + name: Upload scikit-learn WASM wheels to Anaconda.org + runs-on: ubuntu-latest + permissions: {} + environment: upload_anaconda + needs: [build_wasm_wheel] + if: github.repository == 'scikit-learn/scikit-learn' && github.event_name != 'pull_request' + steps: + - name: Download wheel artifact + uses: actions/download-artifact@v4 + with: + path: wheelhouse/ + merge-multiple: true + + - name: Push to Anaconda PyPI index + uses: scientific-python/upload-nightly-action@b36e8c0c10dbcfd2e05bf95f17ef8c14fd708dbf # 0.6.2 + with: + artifacts_path: wheelhouse/ + anaconda_nightly_upload_token: ${{ secrets.SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN }} diff --git a/.github/workflows/label-blank-issue.yml b/.github/workflows/label-blank-issue.yml new file mode 100644 index 0000000000000..7c00984d1169f --- /dev/null +++ b/.github/workflows/label-blank-issue.yml @@ -0,0 +1,16 @@ +name: Labels Blank issues +permissions: + issues: write + +on: + issues: + types: [opened] + +jobs: + label-blank-issues: + runs-on: ubuntu-latest + steps: + - uses: andymckay/labeler@1.0.4 + with: + add-labels: "Needs Triage" + ignore-if-labeled: true diff --git a/.github/workflows/labeler-module.yml b/.github/workflows/labeler-module.yml index 3a9ed8d364f79..468d3282903f2 100644 --- a/.github/workflows/labeler-module.yml +++ b/.github/workflows/labeler-module.yml @@ -1,14 +1,33 @@ name: "Pull Request Labeler" -on: pull_request_target +on: + pull_request_target: + types: [opened] + +# Restrict the permissions granted to the use of secrets.GITHUB_TOKEN in this +# github actions workflow: +# https://docs.github.com/en/actions/security-guides/automatic-token-authentication +permissions: + contents: read + pull-requests: write jobs: triage: runs-on: ubuntu-latest steps: - - uses: thomasjpfan/labeler@v2.5.0 + - uses: thomasjpfan/labeler@v2.5.1 continue-on-error: true if: github.repository == 'scikit-learn/scikit-learn' with: repo-token: "${{ secrets.GITHUB_TOKEN }}" max-labels: "3" configuration-path: ".github/labeler-module.yml" + + triage_file_extensions: + runs-on: ubuntu-latest + steps: + - uses: thomasjpfan/labeler@v2.5.1 + continue-on-error: true + if: github.repository == 'scikit-learn/scikit-learn' + with: + repo-token: "${{ secrets.GITHUB_TOKEN }}" + configuration-path: ".github/labeler-file-extensions.yml" diff --git a/.github/workflows/labeler-title-regex.yml b/.github/workflows/labeler-title-regex.yml index e3c0812029d1b..8b127925cbdae 100644 --- a/.github/workflows/labeler-title-regex.yml +++ b/.github/workflows/labeler-title-regex.yml @@ -3,17 +3,24 @@ on: pull_request_target: types: [opened, edited] +# Restrict the permissions granted to the use of secrets.GITHUB_TOKEN in this +# github actions workflow: +# https://docs.github.com/en/actions/security-guides/automatic-token-authentication +permissions: + contents: read + pull-requests: write + jobs: labeler: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: '3.9' - - name: Install ghapi - run: pip install -Uq ghapi + - name: Install PyGithub + run: pip install -Uq PyGithub - name: Label pull request run: python .github/scripts/label_title_regex.py env: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000000000..f8075e779c56b --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,103 @@ +# This linter job on GH actions is used to trigger the commenter bot +# in bot-lint-comment.yml file. It stores the output of the linter to be used +# by the commenter bot. +name: linter + +on: + - pull_request_target + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref }} + cancel-in-progress: true + +jobs: + lint: + runs-on: ubuntu-latest + + # setting any permission will set everything else to none for GITHUB_TOKEN + permissions: + pull-requests: none + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.11 + + - name: Install dependencies + run: | + curl https://raw.githubusercontent.com/${{ github.repository }}/main/build_tools/shared.sh --retry 5 -o ./build_tools/shared.sh + source build_tools/shared.sh + # Include pytest compatibility with mypy + pip install pytest $(get_dep ruff min) $(get_dep mypy min) cython-lint + # we save the versions of the linters to be used in the error message later. + python -c "from importlib.metadata import version; print(f\"ruff={version('ruff')}\")" >> /tmp/versions.txt + python -c "from importlib.metadata import version; print(f\"mypy={version('mypy')}\")" >> /tmp/versions.txt + python -c "from importlib.metadata import version; print(f\"cython-lint={version('cython-lint')}\")" >> /tmp/versions.txt + + - name: Run linting + id: lint-script + # We download the linting script from main, since this workflow is run + # from main itself. + run: | + curl https://raw.githubusercontent.com/${{ github.repository }}/main/build_tools/linting.sh --retry 5 -o ./build_tools/linting.sh + set +e + ./build_tools/linting.sh &> /tmp/linting_output.txt + cat /tmp/linting_output.txt + + - name: Upload Artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: lint-log + path: | + /tmp/linting_output.txt + /tmp/versions.txt + retention-days: 1 + + comment: + needs: lint + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + + # We need these permissions to be able to post / update comments + permissions: + pull-requests: write + issues: write + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.11 + + - name: Install dependencies + run: python -m pip install requests + + - name: Download artifact + id: download-artifact + uses: actions/download-artifact@v4 + with: + name: lint-log + + - name: Print log + run: cat linting_output.txt + + - name: Process Comments + id: process-comments + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + BRANCH_SHA: ${{ github.event.pull_request.head.sha }} + RUN_ID: ${{ github.run_id }} + LOG_FILE: linting_output.txt + VERSIONS_FILE: versions.txt + run: python ./build_tools/get_comment.py diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index 677188a3567b3..ad24ea805eb8a 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -13,9 +13,13 @@ on: jobs: publish: runs-on: ubuntu-latest + environment: publish_pypi + permissions: + # IMPORTANT: this permission is mandatory for trusted publishing + id-token: write steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: '3.8' - name: Install dependencies @@ -35,15 +39,13 @@ jobs: run: | python build_tools/github/check_wheels.py - name: Publish package to TestPyPI - uses: pypa/gh-action-pypi-publish@v1.4.1 + uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # v1.12.4 with: - user: __token__ - password: ${{ secrets.TEST_PYPI_TOKEN }} - repository_url: https://test.pypi.org/legacy/ + repository-url: https://test.pypi.org/legacy/ + print-hash: true if: ${{ github.event.inputs.pypi_repo == 'testpypi' }} - name: Publish package to PyPI - uses: pypa/gh-action-pypi-publish@v1.4.1 - with: - user: __token__ - password: ${{ secrets.PYPI_TOKEN }} + uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # v1.12.4 if: ${{ github.event.inputs.pypi_repo == 'pypi' }} + with: + print-hash: true diff --git a/.github/workflows/twitter.yml b/.github/workflows/twitter.yml deleted file mode 100644 index 96b32ec902efa..0000000000000 --- a/.github/workflows/twitter.yml +++ /dev/null @@ -1,26 +0,0 @@ -# Tweet the URL of a commit on @sklearn_commits whenever a push event -# happens on the main branch -name: Twitter Push Notification - - -on: - push: - branches: - - main - - -jobs: - tweet: - name: Twitter Notification - runs-on: ubuntu-latest - steps: - - name: Tweet URL of last commit as @sklearn_commits - if: github.repository == 'scikit-learn/scikit-learn' - uses: docker://thomasjpfan/twitter-action:0.3 - with: - args: "-message \"https://github.com/scikit-learn/scikit-learn/commit/${{ github.sha }}\"" - env: - TWITTER_CONSUMER_KEY: ${{ secrets.TWITTER_CONSUMER_KEY }} - TWITTER_CONSUMER_SECRET: ${{ secrets.TWITTER_CONSUMER_SECRET }} - TWITTER_ACCESS_TOKEN: ${{ secrets.TWITTER_ACCESS_TOKEN }} - TWITTER_ACCESS_SECRET: ${{ secrets.TWITTER_ACCESS_SECRET }} diff --git a/.github/workflows/unassign.yml b/.github/workflows/unassign.yml index 0f4e78478b810..94a50d49839d6 100644 --- a/.github/workflows/unassign.yml +++ b/.github/workflows/unassign.yml @@ -4,6 +4,12 @@ on: issues: types: unassigned +# Restrict the permissions granted to the use of secrets.GITHUB_TOKEN in this +# github actions workflow: +# https://docs.github.com/en/actions/security-guides/automatic-token-authentication +permissions: + issues: write + jobs: one: runs-on: ubuntu-latest @@ -12,4 +18,7 @@ jobs: if: github.event.issue.state == 'open' run: | echo "Marking issue ${{ github.event.issue.number }} as help wanted" - curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"labels": ["help wanted"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/labels + gh issue edit $ISSUE --add-label "help wanted" + env: + GH_TOKEN: ${{ github.token }} + ISSUE: ${{ github.event.issue.html_url }} diff --git a/.github/workflows/update-lock-files.yml b/.github/workflows/update-lock-files.yml new file mode 100644 index 0000000000000..3d67bd9f70701 --- /dev/null +++ b/.github/workflows/update-lock-files.yml @@ -0,0 +1,88 @@ +# Workflow to update lock files +name: Update lock files +permissions: + contents: read + +on: + workflow_dispatch: + schedule: + - cron: '0 5 * * 1' + +jobs: + update_lock_files: + if: github.repository == 'scikit-learn/scikit-learn' + runs-on: ubuntu-latest + + strategy: + # Ensure that each build will continue even if one build in the matrix fails + fail-fast: false + matrix: + include: + - name: main + update_script_args: "--select-tag main-ci" + additional_commit_message: "[doc build]" + - name: scipy-dev + update_script_args: "--select-tag scipy-dev" + additional_commit_message: "[scipy-dev]" + - name: free-threaded + update_script_args: "--select-tag free-threaded" + additional_commit_message: "[free-threaded]" + - name: array-api + update_script_args: "--select-tag cuda" + + steps: + - uses: actions/checkout@v4 + - name: Generate lock files + run: | + source build_tools/shared.sh + source $CONDA/bin/activate + conda update -n base --all + conda install -n base conda conda-libmamba-solver -y + conda config --set solver libmamba + conda install -c conda-forge "$(get_dep conda-lock min)" -y + + python build_tools/update_environments_and_lock_files.py ${{ matrix.update_script_args }} + + - name: Create Pull Request + id: cpr + uses: peter-evans/create-pull-request@v7 + with: + token: ${{ secrets.BOT_GITHUB_TOKEN }} + push-to-fork: scikit-learn-bot/scikit-learn + commit-message: Update CI lock files ${{ matrix.additional_commit_message }} + committer: "Lock file bot " + author: "Lock file bot " + delete-branch: true + branch: auto-update-lock-files-${{ matrix.name }} + title: ":lock: :robot: CI Update lock files for ${{ matrix.name }} CI build(s) :lock: :robot:" + body: | + Update lock files. + + ### Note + If the CI tasks fail, create a new branch based on this PR and add the required fixes to that branch. + + # The CUDA workflow needs to be triggered explicitly as it uses an expensive runner + - name: Trigger additional tests + if: steps.cpr.outputs.pull-request-number != '' && matrix.name == 'array-api' + env: + GH_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} + PR_NUMBER: ${{steps.cpr.outputs.pull-request-number}} + run: | + curl -L \ + -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $GH_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos/scikit-learn/scikit-learn/issues/$PR_NUMBER/labels \ + -d '{"labels":["CUDA CI"]}' + + - name: Check Pull Request + if: steps.cpr.outputs.pull-request-number != '' + run: | + echo "### :rocket: Pull-Request Summary" >> ${GITHUB_STEP_SUMMARY} + echo "" >> ${GITHUB_STEP_SUMMARY} + echo "The following lock files pull-request has been auto-generated:" + echo "- **PR** #${{ steps.cpr.outputs.pull-request-number }}" >> ${GITHUB_STEP_SUMMARY} + echo "- **URL** ${{ steps.cpr.outputs.pull-request-url }}" >> ${GITHUB_STEP_SUMMARY} + echo "- **Operation** [${{ steps.cpr.outputs.pull-request-operation }}]" >> ${GITHUB_STEP_SUMMARY} + echo "- **SHA** ${{ steps.cpr.outputs.pull-request-head-sha }}" >> ${GITHUB_STEP_SUMMARY} diff --git a/.github/workflows/update_tracking_issue.yml b/.github/workflows/update_tracking_issue.yml new file mode 100644 index 0000000000000..54db3f50bc43b --- /dev/null +++ b/.github/workflows/update_tracking_issue.yml @@ -0,0 +1,51 @@ +# For workflows to use this workflow include the following: +# +# update-tracker: +# uses: ./.github/workflows/update_tracking_issue.yml +# if: ${{ always() }} +# needs: [JOB_NAME] +# with: +# job_status: ${{ needs.JOB_NAME.result }} +# secrets: +# BOT_GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} +# Where JOB_NAME is contains the status of the job you are interested in + +name: "Update tracking issue" +permissions: + contents: read + +on: + workflow_call: + inputs: + job_status: + required: true + type: string + secrets: + BOT_GITHUB_TOKEN: + required: true + +jobs: + update_tracking_issue: + runs-on: ubuntu-latest + if: github.repository == 'scikit-learn/scikit-learn' && github.event_name == 'schedule' + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.9' + - name: Update tracking issue on GitHub + run: | + set -ex + if [[ ${{ inputs.job_status }} == "success" ]]; then + TESTS_PASSED=true + else + TESTS_PASSED=false + fi + + pip install defusedxml PyGithub + python maint_tools/update_tracking_issue.py \ + ${{ secrets.BOT_GITHUB_TOKEN }} \ + "$GITHUB_WORKFLOW" \ + "$GITHUB_REPOSITORY" \ + https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID \ + --tests-passed $TESTS_PASSED diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index ffddf9ef88db3..33e8897c147f7 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -1,5 +1,7 @@ # Workflow to build and test wheels name: Wheel builder +permissions: + contents: read on: schedule: @@ -17,6 +19,10 @@ on: # Manual run workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: # Check whether to build the wheels and the source tarball check_build_trigger: @@ -28,7 +34,7 @@ jobs: steps: - name: Checkout scikit-learn - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} @@ -36,10 +42,15 @@ jobs: name: Check build trigger run: bash build_tools/github/check_build_trigger.sh - # Build the wheels for Linux, Windows and macOS for Python 3.6 and newer + # Build the wheels for Linux, Windows and macOS for Python 3.9 and newer build_wheels: name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}-${{ matrix.manylinux_image }} runs-on: ${{ matrix.os }} + + # For conda-incubator/setup-miniconda to work + defaults: + run: + shell: bash -el {0} needs: check_build_trigger if: needs.check_build_trigger.outputs.build @@ -47,69 +58,157 @@ jobs: # Ensure that a wheel builder finishes even if another fails fail-fast: false matrix: - os: [windows-latest, ubuntu-latest, macos-latest] - python: [36, 37, 38, 39] - bitness: [32, 64] - manylinux_image: [manylinux1, manylinux2010] include: - # Run 32 and 64 bit version in parallel for Linux and Windows + # Window 64 bit + - os: windows-latest + python: 310 + platform_id: win_amd64 + - os: windows-latest + python: 311 + platform_id: win_amd64 + - os: windows-latest + python: 312 + platform_id: win_amd64 - os: windows-latest - bitness: 64 + python: 313 platform_id: win_amd64 - os: windows-latest - bitness: 32 - platform_id: win32 + python: 313t + platform_id: win_amd64 + free_threaded_support: True + + # Linux 64 bit manylinux2014 + - os: ubuntu-latest + python: 310 + platform_id: manylinux_x86_64 + manylinux_image: manylinux2014 + - os: ubuntu-latest + python: 311 + platform_id: manylinux_x86_64 + manylinux_image: manylinux2014 + - os: ubuntu-latest + python: 312 + platform_id: manylinux_x86_64 + manylinux_image: manylinux2014 - os: ubuntu-latest - bitness: 64 + python: 313 platform_id: manylinux_x86_64 + manylinux_image: manylinux2014 - os: ubuntu-latest - bitness: 32 - platform_id: manylinux_i686 - - os: macos-latest - bitness: 64 + python: 313t + platform_id: manylinux_x86_64 + manylinux_image: manylinux2014 + free_threaded_support: True + + # # Linux 64 bit manylinux2014 + - os: ubuntu-24.04-arm + python: 310 + platform_id: manylinux_aarch64 + manylinux_image: manylinux2014 + - os: ubuntu-24.04-arm + python: 311 + platform_id: manylinux_aarch64 + manylinux_image: manylinux2014 + - os: ubuntu-24.04-arm + python: 312 + platform_id: manylinux_aarch64 + manylinux_image: manylinux2014 + - os: ubuntu-24.04-arm + python: 313 + platform_id: manylinux_aarch64 + manylinux_image: manylinux2014 + + # MacOS x86_64 + - os: macos-13 + python: 310 platform_id: macosx_x86_64 - exclude: - - os: macos-latest - bitness: 32 - # Remove manylinux1 from the windows and osx build matrix since - # manylinux_image is not used for these platforms - - os: windows-latest - manylinux_image: manylinux1 - - os: macos-latest - manylinux_image: manylinux1 + - os: macos-13 + python: 311 + platform_id: macosx_x86_64 + - os: macos-13 + python: 312 + platform_id: macosx_x86_64 + - os: macos-13 + python: 313 + platform_id: macosx_x86_64 + - os: macos-13 + python: 313t + platform_id: macosx_x86_64 + free_threaded_support: True + + # MacOS arm64 + - os: macos-14 + python: 310 + platform_id: macosx_arm64 + - os: macos-14 + python: 311 + platform_id: macosx_arm64 + - os: macos-14 + python: 312 + platform_id: macosx_arm64 + - os: macos-14 + python: 313 + platform_id: macosx_arm64 + - os: macos-14 + python: 313t + platform_id: macosx_arm64 + free_threaded_support: True steps: - name: Checkout scikit-learn - uses: actions/checkout@v1 + uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 + with: + python-version: "3.11" # update once build dependencies are available + + - uses: conda-incubator/setup-miniconda@v3 + if: ${{ startsWith(matrix.platform_id, 'macosx') }} - name: Build and test wheels env: - CONFTEST_PATH: ${{ github.workspace }}/conftest.py - CONFTEST_NAME: conftest.py - CIBW_ENVIRONMENT: OMP_NUM_THREADS=2 - OPENBLAS_NUM_THREADS=2 - SKLEARN_SKIP_NETWORK_TESTS=1 - SKLEARN_BUILD_PARALLEL=3 - MACOSX_DEPLOYMENT_TARGET=10.13 + CIBW_PRERELEASE_PYTHONS: ${{ matrix.prerelease_pythons }} + CIBW_FREE_THREADED_SUPPORT: ${{ matrix.free_threaded_support }} + CIBW_ENVIRONMENT: SKLEARN_SKIP_NETWORK_TESTS=1 CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} + CIBW_ARCHS: all CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} - CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} - CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} - CIBW_TEST_REQUIRES: pytest pandas threadpoolctl - CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh - CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }} + # Needed on Windows CI to compile with Visual Studio compiler + # otherwise Meson detects a MINGW64 platform and use MINGW64 + # toolchain + CIBW_CONFIG_SETTINGS_WINDOWS: "setup-args=--vsenv" + CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} + CIBW_BEFORE_BUILD: bash {project}/build_tools/wheels/cibw_before_build.sh {project} + CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} + CIBW_ENVIRONMENT_PASS_LINUX: RUNNER_OS + CIBW_TEST_REQUIRES: pytest pandas + # On Windows, we use a custom Docker image and CIBW_TEST_REQUIRES_WINDOWS + # does not make sense because it would install dependencies in the host + # rather than inside the Docker image + CIBW_TEST_REQUIRES_WINDOWS: "" + CIBW_TEST_COMMAND: bash {project}/build_tools/wheels/test_wheels.sh {project} + CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} {project} + CIBW_BUILD_VERBOSITY: 1 - run: bash build_tools/github/build_wheels.sh + run: bash build_tools/wheels/build_wheels.sh - name: Store artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: + name: cibw-wheels-cp${{ matrix.python }}-${{ matrix.platform_id }} path: wheelhouse/*.whl + update-tracker: + uses: ./.github/workflows/update_tracking_issue.yml + if: ${{ always() }} + needs: [build_wheels] + with: + job_status: ${{ needs.build_wheels.result }} + secrets: + BOT_GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} + # Build the source distribution under Linux build_sdist: name: Source distribution @@ -119,52 +218,55 @@ jobs: steps: - name: Checkout scikit-learn - uses: actions/checkout@v1 + uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 + with: + python-version: "3.12" - name: Build source distribution run: bash build_tools/github/build_source.sh - env: - SKLEARN_BUILD_PARALLEL: 3 - name: Test source distribution run: bash build_tools/github/test_source.sh env: - OMP_NUM_THREADS: 2 - OPENBLAS_NUM_THREADS: 2 SKLEARN_SKIP_NETWORK_TESTS: 1 - name: Store artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: + name: cibw-sdist path: dist/*.tar.gz # Upload the wheels and the source distribution upload_anaconda: name: Upload to Anaconda runs-on: ubuntu-latest + environment: upload_anaconda needs: [build_wheels, build_sdist] # The artifacts cannot be uploaded on PRs if: github.event_name != 'pull_request' steps: - name: Checkout scikit-learn - uses: actions/checkout@v1 + uses: actions/checkout@v4 - name: Download artifacts - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: + pattern: cibw-* path: dist + merge-multiple: true - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 - name: Upload artifacts env: # Secret variables need to be mapped to environment variables explicitly SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN }} SCIKIT_LEARN_STAGING_UPLOAD_TOKEN: ${{ secrets.SCIKIT_LEARN_STAGING_UPLOAD_TOKEN }} + ARTIFACTS_PATH: dist # Force a replacement if the remote file already exists run: bash build_tools/github/upload_anaconda.sh diff --git a/.gitignore b/.gitignore index 3ebd8e2bb1699..7e00b8802bd01 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -*.pyc +*.pyc* *.so *.pyd *~ @@ -13,12 +13,21 @@ sklearn/**/*.html dist/ MANIFEST +doc/sg_execution_times.rst doc/_build/ +doc/api/*.rst doc/auto_examples/ +doc/css/* +!doc/css/.gitkeep doc/modules/generated/ doc/datasets/generated/ +doc/developers/maintainer.rst +doc/index.rst doc/min_dependency_table.rst doc/min_dependency_substitutions.rst +# release notes generated by towncrier +doc/whats_new/notes-towncrier.rst + *.pdf pip-log.txt scikit_learn.egg-info/ @@ -53,11 +62,15 @@ nips2010_pdf/ examples/cluster/joblib reuters/ benchmarks/bench_covertype_data/ +benchmarks/HIGGS.csv.gz +bench_pca_solvers.csv *.prefs .pydevproject .idea .vscode +# used by pyenv +.python-version *.c *.cpp @@ -75,7 +88,11 @@ _configtest.o.d # Used by mypy .mypy_cache/ -# files generated from a template -sklearn/utils/_seq_dataset.pyx -sklearn/utils/_seq_dataset.pxd -sklearn/linear_model/_sag_fast.pyx +# virtualenv from advanced installation guide +sklearn-env/ + +# Default JupyterLite content +jupyterlite_contents + +# file recognised by vscode IDEs containing env variables +.env diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2ebdc1346b183..48871d2a4abed 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,20 +1,33 @@ +exclude: '^(.git/|sklearn/externals/|asv_benchmarks/env/)' repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.3.0 + rev: v5.0.0 hooks: - id: check-yaml - id: end-of-file-fixer - id: trailing-whitespace -- repo: https://gitlab.com/pycqa/flake8 - rev: 3.7.8 +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.11.7 hooks: - - id: flake8 - types: [file, python] - # only check for unused imports for now, as long as - # the code is not fully PEP8 compatible - args: [--select=F401] + - id: ruff + args: ["--fix", "--output-format=full"] + - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.782 + rev: v1.15.0 hooks: - id: mypy files: sklearn/ + additional_dependencies: [pytest==6.2.4] +- repo: https://github.com/MarcoGorelli/cython-lint + rev: v0.16.6 + hooks: + # TODO: add the double-quote-cython-strings hook when it's usability has improved: + # possibility to pass a directory and use it as a check instead of auto-formatter. + - id: cython-lint +- repo: https://github.com/pre-commit/mirrors-prettier + rev: v2.7.1 + hooks: + - id: prettier + files: ^doc/scss/|^doc/js/scripts/ + exclude: ^doc/js/scripts/vendor/ + types_or: ["scss", "javascript"] diff --git a/.spin/cmds.py b/.spin/cmds.py new file mode 100644 index 0000000000000..954749b8005c2 --- /dev/null +++ b/.spin/cmds.py @@ -0,0 +1,29 @@ +import shutil +import sys + +import click +from spin.cmds import util + + +@click.command() +def clean(): + """đŸĒĨ Clean build folder. + + Very rarely needed since meson-python recompiles as needed when sklearn is + imported. + + One known use case where "spin clean" is useful: avoid compilation errors + when switching from numpy<2 to numpy>=2 in the same conda environment or + virtualenv. + """ + util.run([sys.executable, "-m", "pip", "uninstall", "scikit-learn", "-y"]) + default_meson_build_dir = ( + f"build/cp{sys.version_info.major}{sys.version_info.minor}" + ) + click.secho( + f"removing default Meson build dir: {default_meson_build_dir}", + bold=True, + fg="bright_blue", + ) + + shutil.rmtree(default_meson_build_dir, ignore_errors=True) diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 1e6ed78d28ac2..0000000000000 --- a/.travis.yml +++ /dev/null @@ -1,90 +0,0 @@ -# Make it explicit that we favor the -# new container-based Travis workers -language: python -dist: xenial - -cache: - apt: true - directories: - - $HOME/.cache/pip - - $HOME/.ccache - -env: - global: - - CPU_COUNT=3 - - TEST_DIR=/tmp/sklearn # Test directory for continuous integration jobs - - PYTEST_VERSION=latest - - OMP_NUM_THREADS=2 - - OPENBLAS_NUM_THREADS=2 - - SKLEARN_BUILD_PARALLEL=3 - - SKLEARN_SKIP_NETWORK_TESTS=1 - # Custom environment variables for the ARM wheel builder - - CIBW_BUILD_VERBOSITY=1 - - CIBW_TEST_REQUIRES="pytest pytest-xdist threadpoolctl" - - CIBW_TEST_COMMAND="bash {project}/build_tools/travis/test_wheels.sh" - - CIBW_ENVIRONMENT="CPU_COUNT=2 - OMP_NUM_THREADS=2 - OPENBLAS_NUM_THREADS=2 - SKLEARN_BUILD_PARALLEL=3 - SKLEARN_SKIP_NETWORK_TESTS=1" - -jobs: - include: - # Manual trigger of linux/arm64 tests in PR without triggering the full - # wheel building process for all the Python versions. - - python: 3.9 - os: linux - arch: arm64 - if: commit_message =~ /\[arm64\]/ - env: - - CPU_COUNT=4 - - # Linux environments to build the scikit-learn wheels for the ARM64 - # architecture and Python 3.6 and newer. This is used both at release time - # with the manual trigger in the commit message in the release branch and as - # a scheduled task to build the weekly dev build on the main branch. The - # weekly frequency is meant to avoid depleting the Travis CI credits too - # fast. - - python: 3.6 - os: linux - arch: arm64 - if: type = cron or commit_message =~ /\[cd build\]/ - env: - - BUILD_WHEEL=true - - CIBW_BUILD=cp36-manylinux_aarch64 - - - python: 3.7 - os: linux - arch: arm64 - if: type = cron or commit_message =~ /\[cd build\]/ - env: - - BUILD_WHEEL=true - - CIBW_BUILD=cp37-manylinux_aarch64 - - - python: 3.8 - os: linux - arch: arm64 - if: type = cron or commit_message =~ /\[cd build\]/ - env: - - BUILD_WHEEL=true - - CIBW_BUILD=cp38-manylinux_aarch64 - - - python: 3.9 - os: linux - arch: arm64 - if: type = cron or commit_message =~ /\[cd build\]/ - env: - - BUILD_WHEEL=true - - CIBW_BUILD=cp39-manylinux_aarch64 - -install: source build_tools/travis/install.sh || travis_terminate 1 -script: source build_tools/travis/script.sh || travis_terminate 1 -after_success: source build_tools/travis/after_success.sh || travis_terminate 1 - -notifications: - webhooks: - urls: - - https://webhooks.gitter.im/e/4ffabb4df010b70cd624 - on_success: change - on_failure: always - on_start: never diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000000000..c3e367c124f81 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,48 @@ +cff-version: 1.2.0 +title: scikit-learn +type: software +authors: + - name: "The scikit-learn developers" +message: "If you use scikit-learn in a scientific publication, we would appreciate citations to the following paper:" +preferred-citation: + type: article + title: "Scikit-learn: Machine Learning in Python" + authors: + - family-names: "Pedregosa" + given-names: "Fabian" + - family-names: "Varoquaux" + given-names: "GaÃĢl" + - family-names: "Gramfort" + given-names: "Alexandre" + - family-names: "Michel" + given-names: "Vincent" + - family-names: "Thirion" + given-names: "Bertrand" + - family-names: "Grisel" + given-names: "Olivier" + - family-names: "Blondel" + given-names: "Mathieu" + - family-names: "Prettenhofer" + given-names: "Peter" + - family-names: "Weiss" + given-names: "Ron" + - family-names: "Dubourg" + given-names: "Vincent" + - family-names: "Vanderplas" + given-names: "Jake" + - family-names: "Passos" + given-names: "Alexandre" + - family-names: "Cournapeau" + given-names: "David" + - family-names: "Brucher" + given-names: "Matthieu" + - family-names: "Perrot" + given-names: "Matthieu" + - family-names: "Duchesnay" + given-names: "Édouard" + journal: "Journal of Machine Learning Research" + volume: 12 + start: 2825 + end: 2830 + year: 2011 + url: "https://jmlr.csail.mit.edu/papers/v12/pedregosa11a.html" diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 23016563a5f6e..b4e1709e67c3f 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -13,4 +13,3 @@ all priceless contributions. We abide by the principles of openness, respect, and consideration of others of the Python Software Foundation: https://www.python.org/psf/codeofconduct/ - diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f6f65883c65b2..92a673462e3a6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,7 +18,7 @@ Documentation can be found under the But there are many other ways to help. In particular answering queries on the [issue tracker](https://github.com/scikit-learn/scikit-learn/issues), investigating bugs, and [reviewing other developers' pull -requests](http://scikit-learn.org/dev/developers/contributing.html#code-review-guidelines) +requests](https://scikit-learn.org/dev/developers/contributing.html#code-review-guidelines) are very valuable contributions that decrease the burden on the project maintainers. @@ -30,8 +30,8 @@ link to it from your website, or simply star it in GitHub to say "I use it". Quick links ----------- -* [Submitting a bug report or feature request](http://scikit-learn.org/dev/developers/contributing.html#submitting-a-bug-report-or-a-feature-request) -* [Contributing code](http://scikit-learn.org/dev/developers/contributing.html#contributing-code) +* [Submitting a bug report or feature request](https://scikit-learn.org/dev/developers/contributing.html#submitting-a-bug-report-or-a-feature-request) +* [Contributing code](https://scikit-learn.org/dev/developers/contributing.html#contributing-code) * [Coding guidelines](https://scikit-learn.org/dev/developers/develop.html#coding-guidelines) * [Tips to read current code](https://scikit-learn.org/dev/developers/contributing.html#reading-the-existing-code-base) diff --git a/COPYING b/COPYING index 558c4c1245615..e1cd01d584578 100644 --- a/COPYING +++ b/COPYING @@ -1,6 +1,6 @@ BSD 3-Clause License -Copyright (c) 2007-2020 The scikit-learn developers. +Copyright (c) 2007-2024 The scikit-learn developers. All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 5a4b582bd9886..0000000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,32 +0,0 @@ -include *.rst -recursive-include doc * -recursive-include examples * -recursive-include sklearn *.c *.h *.pyx *.pxd *.pxi *.tp -recursive-include sklearn/datasets *.csv *.csv.gz *.rst *.jpg *.txt *.arff.gz *.json.gz -include COPYING -include README.rst -include pyproject.toml -include sklearn/externals/README -include sklearn/svm/src/liblinear/COPYRIGHT -include sklearn/svm/src/libsvm/LIBSVM_CHANGES -include conftest.py -include Makefile -include MANIFEST.in -include .coveragerc - -# exclude from sdist -recursive-exclude asv_benchmarks * -recursive-exclude benchmarks * -recursive-exclude build_tools * -recursive-exclude maint_tools * -recursive-exclude benchmarks * -recursive-exclude .binder * -recursive-exclude .circleci * -exclude .codecov.yml -exclude .mailmap -exclude .pre-commit-config.yaml -exclude azure-pipelines.yml -exclude lgtm.yml -exclude CODE_OF_CONDUCT.md -exclude CONTRIBUTING.md -exclude PULL_REQUEST_TEMPLATE.md diff --git a/Makefile b/Makefile index 112b1e68188a0..eb6ec39edcbdc 100644 --- a/Makefile +++ b/Makefile @@ -1,68 +1,27 @@ # simple makefile to simplify repetitive build env management tasks under posix -# caution: testing won't work on windows, see README - PYTHON ?= python -CYTHON ?= cython -PYTEST ?= pytest -CTAGS ?= ctags - -# skip doctests on 32bit python -BITS := $(shell python -c 'import struct; print(8 * struct.calcsize("P"))') - -all: clean inplace test - -clean-ctags: - rm -f tags - -clean: clean-ctags - $(PYTHON) setup.py clean - rm -rf dist - -in: inplace # just a shortcut -inplace: - $(PYTHON) setup.py build_ext -i - -test-code: in - $(PYTEST) --showlocals -v sklearn --durations=20 -test-sphinxext: - $(PYTEST) --showlocals -v doc/sphinxext/ -test-doc: -ifeq ($(BITS),64) - $(PYTEST) $(shell find doc -name '*.rst' | sort) -endif -test-code-parallel: in - $(PYTEST) -n auto --showlocals -v sklearn --durations=20 - -test-coverage: - rm -rf coverage .coverage - $(PYTEST) sklearn --showlocals -v --cov=sklearn --cov-report=html:coverage -test-coverage-parallel: - rm -rf coverage .coverage .coverage.* - $(PYTEST) sklearn -n auto --showlocals -v --cov=sklearn --cov-report=html:coverage - -test: test-code test-sphinxext test-doc - -trailing-spaces: - find sklearn -name "*.py" -exec perl -pi -e 's/[ \t]*$$//' {} \; +DEFAULT_MESON_BUILD_DIR = build/cp$(shell python -c 'import sys; print(f"{sys.version_info.major}{sys.version_info.minor}")' ) -cython: - python setup.py build_src +all: + @echo "Please use 'make ' where is one of" + @echo " dev build scikit-learn with Meson" + @echo " clean clean scikit-learn Meson build. Very rarely needed," + @echo " since meson-python recompiles on import." -ctags: - # make tags for symbol based navigation in emacs and vim - # Install with: sudo apt-get install exuberant-ctags - $(CTAGS) --python-kinds=-i -R sklearn +.PHONY: all -doc: inplace - $(MAKE) -C doc html +dev: dev-meson -doc-noplot: inplace - $(MAKE) -C doc html-noplot +dev-meson: + pip install --verbose --no-build-isolation --editable . --config-settings editable-verbose=true -code-analysis: - flake8 sklearn | grep -v __init__ | grep -v external - pylint -E -i y sklearn/ -d E1103,E0611,E1101 +clean: clean-meson -flake8-diff: - git diff upstream/main -u -- "*.py" | flake8 --diff +clean-meson: + pip uninstall -y scikit-learn + # It seems in some cases removing the folder avoids weird compilation + # errors (e.g. when switching from numpy>=2 to numpy<2). For some + # reason ninja clean -C $(DEFAULT_MESON_BUILD_DIR) is not + # enough. + rm -rf $(DEFAULT_MESON_BUILD_DIR) diff --git a/README.rst b/README.rst index 68f9ffee17d03..5885bce67baa7 100644 --- a/README.rst +++ b/README.rst @@ -1,43 +1,47 @@ .. -*- mode: rst -*- -|Azure|_ |Travis|_ |Codecov|_ |CircleCI|_ |Nightly wheels|_ |PythonVersion|_ |PyPi|_ |DOI|_ +|Azure| |Codecov| |CircleCI| |Nightly wheels| |Ruff| |PythonVersion| |PyPi| |DOI| |Benchmark| .. |Azure| image:: https://dev.azure.com/scikit-learn/scikit-learn/_apis/build/status/scikit-learn.scikit-learn?branchName=main -.. _Azure: https://dev.azure.com/scikit-learn/scikit-learn/_build/latest?definitionId=1&branchName=main + :target: https://dev.azure.com/scikit-learn/scikit-learn/_build/latest?definitionId=1&branchName=main -.. |Travis| image:: https://api.travis-ci.com/scikit-learn/scikit-learn.svg?branch=main -.. _Travis: https://travis-ci.com/scikit-learn/scikit-learn +.. |CircleCI| image:: https://circleci.com/gh/scikit-learn/scikit-learn/tree/main.svg?style=shield + :target: https://circleci.com/gh/scikit-learn/scikit-learn -.. |Codecov| image:: https://codecov.io/github/scikit-learn/scikit-learn/badge.svg?branch=main&service=github -.. _Codecov: https://codecov.io/github/scikit-learn/scikit-learn?branch=main +.. |Codecov| image:: https://codecov.io/gh/scikit-learn/scikit-learn/branch/main/graph/badge.svg?token=Pk8G9gg3y9 + :target: https://codecov.io/gh/scikit-learn/scikit-learn -.. |CircleCI| image:: https://circleci.com/gh/scikit-learn/scikit-learn/tree/main.svg?style=shield&circle-token=:circle-token -.. _CircleCI: https://circleci.com/gh/scikit-learn/scikit-learn +.. |Nightly wheels| image:: https://github.com/scikit-learn/scikit-learn/actions/workflows/wheels.yml/badge.svg?event=schedule + :target: https://github.com/scikit-learn/scikit-learn/actions?query=workflow%3A%22Wheel+builder%22+event%3Aschedule -.. |Nightly wheels| image:: https://github.com/scikit-learn/scikit-learn/workflows/Wheel%20builder/badge.svg?event=schedule -.. _`Nightly wheels`: https://github.com/scikit-learn/scikit-learn/actions?query=workflow%3A%22Wheel+builder%22+event%3Aschedule +.. |Ruff| image:: https://img.shields.io/badge/code%20style-ruff-000000.svg + :target: https://github.com/astral-sh/ruff -.. |PythonVersion| image:: https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-blue -.. _PythonVersion: https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-blue +.. |PythonVersion| image:: https://img.shields.io/pypi/pyversions/scikit-learn.svg + :target: https://pypi.org/project/scikit-learn/ -.. |PyPi| image:: https://badge.fury.io/py/scikit-learn.svg -.. _PyPi: https://badge.fury.io/py/scikit-learn +.. |PyPi| image:: https://img.shields.io/pypi/v/scikit-learn + :target: https://pypi.org/project/scikit-learn .. |DOI| image:: https://zenodo.org/badge/21369/scikit-learn/scikit-learn.svg -.. _DOI: https://zenodo.org/badge/latestdoi/21369/scikit-learn/scikit-learn - -.. |PythonMinVersion| replace:: 3.6 -.. |NumPyMinVersion| replace:: 1.13.3 -.. |SciPyMinVersion| replace:: 0.19.1 -.. |JoblibMinVersion| replace:: 0.11 -.. |ThreadpoolctlMinVersion| replace:: 2.0.0 -.. |MatplotlibMinVersion| replace:: 2.1.1 -.. |Scikit-ImageMinVersion| replace:: 0.13 -.. |PandasMinVersion| replace:: 0.25.0 + :target: https://zenodo.org/badge/latestdoi/21369/scikit-learn/scikit-learn + +.. |Benchmark| image:: https://img.shields.io/badge/Benchmarked%20by-asv-blue + :target: https://scikit-learn.org/scikit-learn-benchmarks + +.. |PythonMinVersion| replace:: 3.10 +.. |NumPyMinVersion| replace:: 1.22.0 +.. |SciPyMinVersion| replace:: 1.8.0 +.. |JoblibMinVersion| replace:: 1.2.0 +.. |ThreadpoolctlMinVersion| replace:: 3.1.0 +.. |MatplotlibMinVersion| replace:: 3.5.0 +.. |Scikit-ImageMinVersion| replace:: 0.19.0 +.. |PandasMinVersion| replace:: 1.4.0 .. |SeabornMinVersion| replace:: 0.9.0 -.. |PytestMinVersion| replace:: 5.0.1 +.. |PytestMinVersion| replace:: 7.1.2 +.. |PlotlyMinVersion| replace:: 5.14.0 -.. image:: doc/logos/scikit-learn-logo.png +.. image:: https://raw.githubusercontent.com/scikit-learn/scikit-learn/main/doc/logos/scikit-learn-logo.png :target: https://scikit-learn.org/ **scikit-learn** is a Python module for machine learning built on top of @@ -68,21 +72,18 @@ scikit-learn requires: ======= -**Scikit-learn 0.20 was the last version to support Python 2.7 and Python 3.4.** -scikit-learn 0.23 and later require Python 3.6 or newer. - Scikit-learn plotting capabilities (i.e., functions start with ``plot_`` and -classes end with "Display") require Matplotlib (>= |MatplotlibMinVersion|). +classes end with ``Display``) require Matplotlib (>= |MatplotlibMinVersion|). For running the examples Matplotlib >= |MatplotlibMinVersion| is required. A few examples require scikit-image >= |Scikit-ImageMinVersion|, a few examples require pandas >= |PandasMinVersion|, some examples require seaborn >= -|SeabornMinVersion|. +|SeabornMinVersion| and plotly >= |PlotlyMinVersion|. User installation ~~~~~~~~~~~~~~~~~ -If you already have a working installation of numpy and scipy, -the easiest way to install scikit-learn is using ``pip`` :: +If you already have a working installation of NumPy and SciPy, +the easiest way to install scikit-learn is using ``pip``:: pip install -U scikit-learn @@ -137,7 +138,7 @@ directory (you will need to have ``pytest`` >= |PyTestMinVersion| installed):: pytest sklearn -See the web page https://scikit-learn.org/dev/developers/advanced_installation.html#testing +See the web page https://scikit-learn.org/dev/developers/contributing.html#testing-and-improving-test-coverage for more information. Random number generation can be controlled during testing by setting @@ -175,12 +176,36 @@ Documentation Communication ~~~~~~~~~~~~~ -- Mailing list: https://mail.python.org/mailman/listinfo/scikit-learn -- IRC channel: ``#scikit-learn`` at ``webchat.freenode.net`` -- Gitter: https://gitter.im/scikit-learn/scikit-learn -- Twitter: https://twitter.com/scikit_learn -- Stack Overflow: https://stackoverflow.com/questions/tagged/scikit-learn -- Website: https://scikit-learn.org +Main Channels +^^^^^^^^^^^^^ + +- **Website**: https://scikit-learn.org +- **Blog**: https://blog.scikit-learn.org +- **Mailing list**: https://mail.python.org/mailman/listinfo/scikit-learn + +Developer & Support +^^^^^^^^^^^^^^^^^^^^^^ + +- **GitHub Discussions**: https://github.com/scikit-learn/scikit-learn/discussions +- **Stack Overflow**: https://stackoverflow.com/questions/tagged/scikit-learn +- **Discord**: https://discord.gg/h9qyrK8Jc8 + +Social Media Platforms +^^^^^^^^^^^^^^^^^^^^^^ + +- **LinkedIn**: https://www.linkedin.com/company/scikit-learn +- **YouTube**: https://www.youtube.com/channel/UCJosFjYm0ZYVUARxuOZqnnw/playlists +- **Facebook**: https://www.facebook.com/scikitlearnofficial/ +- **Instagram**: https://www.instagram.com/scikitlearnofficial/ +- **TikTok**: https://www.tiktok.com/@scikit.learn +- **Bluesky**: https://bsky.app/profile/scikit-learn.org +- **Mastodon**: https://mastodon.social/@sklearn@fosstodon.org + +Resources +^^^^^^^^^ + +- **Calendar**: https://blog.scikit-learn.org/calendar/ +- **Logos & Branding**: https://github.com/scikit-learn/scikit-learn/tree/main/doc/logos Citation ~~~~~~~~ diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000000..56c3e982be28a --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,23 @@ +# Security Policy + +## Supported Versions + +| Version | Supported | +| ------------- | ------------------ | +| 1.7.0 | :white_check_mark: | +| < 1.7.0 | :x: | + +## Reporting a Vulnerability + +Please report security vulnerabilities by opening a new [GitHub security +advisory](https://github.com/scikit-learn/scikit-learn/security/advisories/new). + +You can also send an email to `security@scikit-learn.org`, which is an alias to +a subset of the scikit-learn maintainers' team. + +If the security vulnerability is accepted, a patch will be crafted privately +in order to prepare a dedicated bugfix release as timely as possible (depending +on the complexity of the fix). + +In addition to the options above, you can also report security vulnerabilities +to [tidelift](https://tidelift.com/security). diff --git a/asv_benchmarks/asv.conf.json b/asv_benchmarks/asv.conf.json index c6ba1de71c2d2..3b16389139c0c 100644 --- a/asv_benchmarks/asv.conf.json +++ b/asv_benchmarks/asv.conf.json @@ -7,31 +7,21 @@ "project": "scikit-learn", // The project's homepage - "project_url": "scikit-learn.org/", + "project_url": "https://scikit-learn.org/", // The URL or local path of the source code repository for the // project being benchmarked "repo": "..", - // The Python project's subdirectory in your repo. If missing or - // the empty string, the project is assumed to be located at the root - // of the repository. - // "repo_subdir": "", - // Customizable commands for building, installing, and // uninstalling the project. See asv.conf.json documentation. - // - // "install_command": ["python -mpip install {wheel_file}"], - // "uninstall_command": ["return-code=any python -mpip uninstall -y {project}"], - // "build_command": [ - // "python setup.py build", - // "PIP_NO_BUILD_ISOLATION=false python -mpip wheel --no-deps --no-index -w {build_cache_dir} {build_dir}" - // ], + "install_command": ["python -mpip install {wheel_file}"], + "uninstall_command": ["return-code=any python -mpip uninstall -y {project}"], + "build_command": ["python -m build --wheel -o {build_cache_dir} {build_dir}"], - // List of branches to benchmark. If not provided, defaults to "master + // List of branches to benchmark. If not provided, defaults to "main" // (for git) or "default" (for mercurial). "branches": ["main"], - // "branches": ["default"], // for mercurial // The DVCS being used. If not set, it will be automatically // determined from "repo" by looking at the protocol in the URL @@ -50,19 +40,19 @@ // defaults to 10 min //"install_timeout": 600, + // timeout in seconds all benchmarks, can be overridden per benchmark + // defaults to 1 min + //"default_benchmark_timeout": 60, + // the base URL to show a commit for the project. "show_commit_url": "https://github.com/scikit-learn/scikit-learn/commit/", - // The Pythons you'd like to test against. If not provided, defaults + // The Pythons you'd like to test against. If not provided, defaults // to the current version of Python used to run `asv`. - // "pythons": ["3.6"], + // "pythons": ["3.12"], - // The list of conda channel names to be searched for benchmark - // dependency packages in the specified order - // "conda_channels": ["conda-forge", "defaults"] - - // The matrix of dependencies to test. Each key is the name of a - // package (in PyPI) and the values are version numbers. An empty + // The matrix of dependencies to test. Each key is the name of a + // package (in PyPI) and the values are version numbers. An empty // list or empty string indicates to just test against the default // (latest) version. null indicates that the package is to not be // installed. If the package to be tested is only available from @@ -71,12 +61,17 @@ // pip (with all the conda available packages installed first, // followed by the pip installed packages). // + // The versions of the dependencies should be bumped in a dedicated commit + // to easily identify regressions/improvements due to code changes from + // those due to dependency changes. + // "matrix": { - "numpy": [], - "scipy": [], - "cython": [], - "joblib": [], - "threadpoolctl": [] + "numpy": ["2.0.0"], + "scipy": ["1.14.0"], + "cython": ["3.0.10"], + "joblib": ["1.3.2"], + "threadpoolctl": ["3.2.0"], + "pandas": ["2.2.2"] }, // Combinations of libraries/python versions can be excluded/included @@ -106,10 +101,10 @@ // ], // // "include": [ - // // additional env for python2.7 - // {"python": "2.7", "numpy": "1.8"}, + // // additional env for python3.12 + // {"python": "3.12", "numpy": "1.26"}, // // additional env if run on windows+conda - // {"platform": "win32", "environment_type": "conda", "python": "2.7", "libpython": ""}, + // {"sys_platform": "win32", "environment_type": "conda", "python": "3.12", "libpython": ""}, // ], // The directory (relative to the current directory) that benchmarks are @@ -131,10 +126,10 @@ // The number of characters to retain in the commit hashes. // "hash_length": 8, - // `asv` will cache results of the recent builds in each + // `asv` will cache wheels of the recent builds in each // environment, making them faster to install next time. This is - // the number of builds to keep, per environment. - // "build_cache_size": 2, + // number of builds to keep, per environment. + // "build_cache_size": 0 // The commits after which the regression search in `asv publish` // should start looking for regressions. Dictionary whose keys are @@ -147,16 +142,5 @@ // "regressions_first_commits": { // "some_benchmark": "352cdf", // Consider regressions only after this commit // "another_benchmark": null, // Skip regression detection altogether - // }, - - // The thresholds for relative change in results, after which `asv - // publish` starts reporting regressions. Dictionary of the same - // form as in ``regressions_first_commits``, with values - // indicating the thresholds. If multiple entries match, the - // maximum is taken. If no entry matches, the default is 5%. - // - // "regressions_thresholds": { - // "some_benchmark": 0.01, // Threshold of 1% - // "another_benchmark": 0.5, // Threshold of 50% - // }, + // } } diff --git a/asv_benchmarks/benchmarks/cluster.py b/asv_benchmarks/benchmarks/cluster.py index 7e92f8cb6ddd2..457a15dd938e9 100644 --- a/asv_benchmarks/benchmarks/cluster.py +++ b/asv_benchmarks/benchmarks/cluster.py @@ -1,7 +1,7 @@ from sklearn.cluster import KMeans, MiniBatchKMeans from .common import Benchmark, Estimator, Predictor, Transformer -from .datasets import _blobs_dataset, _20newsgroups_highdim_dataset +from .datasets import _20newsgroups_highdim_dataset, _blobs_dataset from .utils import neg_mean_inertia @@ -10,8 +10,8 @@ class KMeansBenchmark(Predictor, Transformer, Estimator, Benchmark): Benchmarks for KMeans. """ - param_names = ['representation', 'algorithm', 'init'] - params = (['dense', 'sparse'], ['full', 'elkan'], ['random', 'k-means++']) + param_names = ["representation", "algorithm", "init"] + params = (["dense", "sparse"], ["lloyd", "elkan"], ["random", "k-means++"]) def setup_cache(self): super().setup_cache() @@ -19,7 +19,7 @@ def setup_cache(self): def make_data(self, params): representation, algorithm, init = params - if representation == 'sparse': + if representation == "sparse": data = _20newsgroups_highdim_dataset(n_samples=8000) else: data = _blobs_dataset(n_clusters=20) @@ -29,27 +29,29 @@ def make_data(self, params): def make_estimator(self, params): representation, algorithm, init = params - max_iter = 30 if representation == 'sparse' else 100 + max_iter = 30 if representation == "sparse" else 100 - estimator = KMeans(n_clusters=20, - algorithm=algorithm, - init=init, - n_init=1, - max_iter=max_iter, - tol=-1, - random_state=0) + estimator = KMeans( + n_clusters=20, + algorithm=algorithm, + init=init, + n_init=1, + max_iter=max_iter, + tol=0, + random_state=0, + ) return estimator def make_scorers(self): - self.train_scorer = ( - lambda _, __: neg_mean_inertia(self.X, - self.estimator.predict(self.X), - self.estimator.cluster_centers_)) - self.test_scorer = ( - lambda _, __: neg_mean_inertia(self.X_val, - self.estimator.predict(self.X_val), - self.estimator.cluster_centers_)) + self.train_scorer = lambda _, __: neg_mean_inertia( + self.X, self.estimator.predict(self.X), self.estimator.cluster_centers_ + ) + self.test_scorer = lambda _, __: neg_mean_inertia( + self.X_val, + self.estimator.predict(self.X_val), + self.estimator.cluster_centers_, + ) class MiniBatchKMeansBenchmark(Predictor, Transformer, Estimator, Benchmark): @@ -57,8 +59,8 @@ class MiniBatchKMeansBenchmark(Predictor, Transformer, Estimator, Benchmark): Benchmarks for MiniBatchKMeans. """ - param_names = ['representation', 'init'] - params = (['dense', 'sparse'], ['random', 'k-means++']) + param_names = ["representation", "init"] + params = (["dense", "sparse"], ["random", "k-means++"]) def setup_cache(self): super().setup_cache() @@ -66,7 +68,7 @@ def setup_cache(self): def make_data(self, params): representation, init = params - if representation == 'sparse': + if representation == "sparse": data = _20newsgroups_highdim_dataset() else: data = _blobs_dataset(n_clusters=20) @@ -76,25 +78,27 @@ def make_data(self, params): def make_estimator(self, params): representation, init = params - max_iter = 5 if representation == 'sparse' else 2 + max_iter = 5 if representation == "sparse" else 2 - estimator = MiniBatchKMeans(n_clusters=20, - init=init, - n_init=1, - max_iter=max_iter, - batch_size=1000, - max_no_improvement=None, - compute_labels=False, - random_state=0) + estimator = MiniBatchKMeans( + n_clusters=20, + init=init, + n_init=1, + max_iter=max_iter, + batch_size=1000, + max_no_improvement=None, + compute_labels=False, + random_state=0, + ) return estimator def make_scorers(self): - self.train_scorer = ( - lambda _, __: neg_mean_inertia(self.X, - self.estimator.predict(self.X), - self.estimator.cluster_centers_)) - self.test_scorer = ( - lambda _, __: neg_mean_inertia(self.X_val, - self.estimator.predict(self.X_val), - self.estimator.cluster_centers_)) + self.train_scorer = lambda _, __: neg_mean_inertia( + self.X, self.estimator.predict(self.X), self.estimator.cluster_centers_ + ) + self.test_scorer = lambda _, __: neg_mean_inertia( + self.X_val, + self.estimator.predict(self.X_val), + self.estimator.cluster_centers_, + ) diff --git a/asv_benchmarks/benchmarks/common.py b/asv_benchmarks/benchmarks/common.py index 70760dc47a9b7..c12da551010f6 100644 --- a/asv_benchmarks/benchmarks/common.py +++ b/asv_benchmarks/benchmarks/common.py @@ -1,11 +1,11 @@ -import os +import itertools import json -import timeit +import os import pickle -import itertools +import timeit from abc import ABC, abstractmethod -from pathlib import Path from multiprocessing import cpu_count +from pathlib import Path import numpy as np @@ -14,86 +14,102 @@ def get_from_config(): """Get benchmarks configuration from the config.json file""" current_path = Path(__file__).resolve().parent - config_path = current_path / 'config.json' - with open(config_path, 'r') as config_file: - config_file = ''.join(line for line in config_file - if line and '//' not in line) + config_path = current_path / "config.json" + with open(config_path, "r") as config_file: + config_file = "".join(line for line in config_file if line and "//" not in line) config = json.loads(config_file) - profile = os.getenv('SKLBENCH_PROFILE', config['profile']) + profile = os.getenv("SKLBENCH_PROFILE", config["profile"]) - n_jobs_vals_env = os.getenv('SKLBENCH_NJOBS') + n_jobs_vals_env = os.getenv("SKLBENCH_NJOBS") if n_jobs_vals_env: - n_jobs_vals = eval(n_jobs_vals_env) + n_jobs_vals = json.loads(n_jobs_vals_env) else: - n_jobs_vals = config['n_jobs_vals'] + n_jobs_vals = config["n_jobs_vals"] if not n_jobs_vals: n_jobs_vals = list(range(1, 1 + cpu_count())) - cache_path = current_path / 'cache' + cache_path = current_path / "cache" cache_path.mkdir(exist_ok=True) - (cache_path / 'estimators').mkdir(exist_ok=True) - (cache_path / 'tmp').mkdir(exist_ok=True) + (cache_path / "estimators").mkdir(exist_ok=True) + (cache_path / "tmp").mkdir(exist_ok=True) - save_estimators = os.getenv('SKLBENCH_SAVE_ESTIMATORS', - config['save_estimators']) - save_dir = os.getenv('ASV_COMMIT', 'new')[:8] + save_estimators = os.getenv("SKLBENCH_SAVE_ESTIMATORS", config["save_estimators"]) + save_dir = os.getenv("ASV_COMMIT", "new")[:8] if save_estimators: - (cache_path / 'estimators' / save_dir).mkdir(exist_ok=True) + (cache_path / "estimators" / save_dir).mkdir(exist_ok=True) - base_commit = os.getenv('SKLBENCH_BASE_COMMIT', config['base_commit']) + base_commit = os.getenv("SKLBENCH_BASE_COMMIT", config["base_commit"]) - bench_predict = os.getenv('SKLBENCH_PREDICT', config['bench_predict']) - bench_transform = os.getenv('SKLBENCH_TRANSFORM', - config['bench_transform']) + bench_predict = os.getenv("SKLBENCH_PREDICT", config["bench_predict"]) + bench_transform = os.getenv("SKLBENCH_TRANSFORM", config["bench_transform"]) - return (profile, n_jobs_vals, save_estimators, save_dir, base_commit, - bench_predict, bench_transform) + return ( + profile, + n_jobs_vals, + save_estimators, + save_dir, + base_commit, + bench_predict, + bench_transform, + ) def get_estimator_path(benchmark, directory, params, save=False): """Get path of pickled fitted estimator""" - path = Path(__file__).resolve().parent / 'cache' - path = (path / 'estimators' / directory) if save else (path / 'tmp') + path = Path(__file__).resolve().parent / "cache" + path = (path / "estimators" / directory) if save else (path / "tmp") - filename = (benchmark.__class__.__name__ - + '_estimator_' + '_'.join(list(map(str, params))) + '.pkl') + filename = ( + benchmark.__class__.__name__ + + "_estimator_" + + "_".join(list(map(str, params))) + + ".pkl" + ) return path / filename def clear_tmp(): """Clean the tmp directory""" - path = Path(__file__).resolve().parent / 'cache' / 'tmp' + path = Path(__file__).resolve().parent / "cache" / "tmp" for child in path.iterdir(): child.unlink() class Benchmark(ABC): """Abstract base class for all the benchmarks""" + timer = timeit.default_timer # wall time processes = 1 timeout = 500 - (profile, n_jobs_vals, save_estimators, save_dir, base_commit, - bench_predict, bench_transform) = get_from_config() - - if profile == 'fast': + ( + profile, + n_jobs_vals, + save_estimators, + save_dir, + base_commit, + bench_predict, + bench_transform, + ) = get_from_config() + + if profile == "fast": warmup_time = 0 repeat = 1 number = 1 min_run_count = 1 - data_size = 'small' - elif profile == 'regular': + data_size = "small" + elif profile == "regular": warmup_time = 1 repeat = (3, 100, 30) - data_size = 'small' - elif profile == 'large_scale': + data_size = "small" + elif profile == "large_scale": warmup_time = 1 repeat = 3 number = 1 - data_size = 'large' + data_size = "large" @property @abstractmethod @@ -103,6 +119,7 @@ def params(self): class Estimator(ABC): """Abstract base class for all benchmarks of estimators""" + @abstractmethod def make_data(self, params): """Return the dataset for a combination of parameters""" @@ -112,8 +129,7 @@ def make_data(self, params): @abstractmethod def make_estimator(self, params): - """Return an instance of the estimator for a combination of parameters - """ + """Return an instance of the estimator for a combination of parameters""" pass def skip(self, params): @@ -137,9 +153,10 @@ def setup_cache(self): estimator.fit(X, y) - est_path = get_estimator_path(self, Benchmark.save_dir, - params, Benchmark.save_estimators) - with est_path.open(mode='wb') as f: + est_path = get_estimator_path( + self, Benchmark.save_dir, params, Benchmark.save_estimators + ) + with est_path.open(mode="wb") as f: pickle.dump(estimator, f) def setup(self, *params): @@ -152,9 +169,10 @@ def setup(self, *params): self.X, self.X_val, self.y, self.y_val = self.make_data(params) - est_path = get_estimator_path(self, Benchmark.save_dir, - params, Benchmark.save_estimators) - with est_path.open(mode='rb') as f: + est_path = get_estimator_path( + self, Benchmark.save_dir, params, Benchmark.save_estimators + ) + with est_path.open(mode="rb") as f: self.estimator = pickle.load(f) self.make_scorers() @@ -166,14 +184,14 @@ def peakmem_fit(self, *args): self.estimator.fit(self.X, self.y) def track_train_score(self, *args): - if hasattr(self.estimator, 'predict'): + if hasattr(self.estimator, "predict"): y_pred = self.estimator.predict(self.X) else: y_pred = None return float(self.train_scorer(self.y, y_pred)) def track_test_score(self, *args): - if hasattr(self.estimator, 'predict'): + if hasattr(self.estimator, "predict"): y_val_pred = self.estimator.predict(self.X_val) else: y_val_pred = None @@ -182,7 +200,9 @@ def track_test_score(self, *args): class Predictor(ABC): """Abstract base class for benchmarks of estimators implementing predict""" + if Benchmark.bench_predict: + def time_predict(self, *args): self.estimator.predict(self.X) @@ -190,10 +210,10 @@ def peakmem_predict(self, *args): self.estimator.predict(self.X) if Benchmark.base_commit is not None: + def track_same_prediction(self, *args): - est_path = get_estimator_path(self, Benchmark.base_commit, - args, True) - with est_path.open(mode='rb') as f: + est_path = get_estimator_path(self, Benchmark.base_commit, args, True) + with est_path.open(mode="rb") as f: estimator_base = pickle.load(f) y_val_pred_base = estimator_base.predict(self.X_val) @@ -208,9 +228,10 @@ def params(self): class Transformer(ABC): - """Abstract base class for benchmarks of estimators implementing transform - """ + """Abstract base class for benchmarks of estimators implementing transform""" + if Benchmark.bench_transform: + def time_transform(self, *args): self.estimator.transform(self.X) @@ -218,10 +239,10 @@ def peakmem_transform(self, *args): self.estimator.transform(self.X) if Benchmark.base_commit is not None: + def track_same_transform(self, *args): - est_path = get_estimator_path(self, Benchmark.base_commit, - args, True) - with est_path.open(mode='rb') as f: + est_path = get_estimator_path(self, Benchmark.base_commit, args, True) + with est_path.open(mode="rb") as f: estimator_base = pickle.load(f) X_val_t_base = estimator_base.transform(self.X_val) diff --git a/asv_benchmarks/benchmarks/config.json b/asv_benchmarks/benchmarks/config.json index f50827cdbd7b7..b5a10b930e60b 100644 --- a/asv_benchmarks/benchmarks/config.json +++ b/asv_benchmarks/benchmarks/config.json @@ -9,7 +9,7 @@ // Can be overridden by environment variable SKLBENCH_PROFILE. "profile": "regular", - // List of values of n_jobs to use for estimators which accept this + // List of values of n_jobs to use for estimators which accept this // parameter (-1 means all cores). An empty list means all values from 1 to // the maximum number of available cores. // Can be overridden by environment variable SKLBENCH_NJOBS. diff --git a/asv_benchmarks/benchmarks/datasets.py b/asv_benchmarks/benchmarks/datasets.py index b00d5888fd2b2..bbf5029062448 100644 --- a/asv_benchmarks/benchmarks/datasets.py +++ b/asv_benchmarks/benchmarks/datasets.py @@ -1,25 +1,32 @@ +from pathlib import Path + import numpy as np import scipy.sparse as sp from joblib import Memory -from pathlib import Path +from sklearn.datasets import ( + fetch_20newsgroups, + fetch_olivetti_faces, + fetch_openml, + load_digits, + make_blobs, + make_classification, + make_regression, +) from sklearn.decomposition import TruncatedSVD -from sklearn.datasets import (make_blobs, fetch_20newsgroups, - fetch_openml, load_digits, make_regression, - make_classification, fetch_olivetti_faces) -from sklearn.preprocessing import MaxAbsScaler, StandardScaler from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.model_selection import train_test_split +from sklearn.preprocessing import MaxAbsScaler, StandardScaler # memory location for caching datasets -M = Memory(location=str(Path(__file__).resolve().parent / 'cache')) +M = Memory(location=str(Path(__file__).resolve().parent / "cache")) @M.cache -def _blobs_dataset(n_samples=500000, n_features=3, n_clusters=100, - dtype=np.float32): - X, _ = make_blobs(n_samples=n_samples, n_features=n_features, - centers=n_clusters, random_state=0) +def _blobs_dataset(n_samples=500000, n_features=3, n_clusters=100, dtype=np.float32): + X, _ = make_blobs( + n_samples=n_samples, n_features=n_features, centers=n_clusters, random_state=0 + ) X = X.astype(dtype, copy=False) X, X_val = train_test_split(X, test_size=0.1, random_state=0) @@ -27,8 +34,7 @@ def _blobs_dataset(n_samples=500000, n_features=3, n_clusters=100, @M.cache -def _20newsgroups_highdim_dataset(n_samples=None, ngrams=(1, 1), - dtype=np.float32): +def _20newsgroups_highdim_dataset(n_samples=None, ngrams=(1, 1), dtype=np.float32): newsgroups = fetch_20newsgroups(random_state=0) vectorizer = TfidfVectorizer(ngram_range=ngrams, dtype=dtype) X = vectorizer.fit_transform(newsgroups.data[:n_samples]) @@ -39,8 +45,7 @@ def _20newsgroups_highdim_dataset(n_samples=None, ngrams=(1, 1), @M.cache -def _20newsgroups_lowdim_dataset(n_components=100, ngrams=(1, 1), - dtype=np.float32): +def _20newsgroups_lowdim_dataset(n_components=100, ngrams=(1, 1), dtype=np.float32): newsgroups = fetch_20newsgroups() vectorizer = TfidfVectorizer(ngram_range=ngrams) X = vectorizer.fit_transform(newsgroups.data) @@ -55,8 +60,7 @@ def _20newsgroups_lowdim_dataset(n_components=100, ngrams=(1, 1), @M.cache def _mnist_dataset(dtype=np.float32): - X, y = fetch_openml('mnist_784', version=1, return_X_y=True, - as_frame=False) + X, y = fetch_openml("mnist_784", version=1, return_X_y=True, as_frame=False) X = X.astype(dtype, copy=False) X = MaxAbsScaler().fit_transform(X) @@ -77,11 +81,14 @@ def _digits_dataset(n_samples=None, dtype=np.float32): @M.cache -def _synth_regression_dataset(n_samples=100000, n_features=100, - dtype=np.float32): - X, y = make_regression(n_samples=n_samples, n_features=n_features, - n_informative=n_features // 10, noise=50, - random_state=0) +def _synth_regression_dataset(n_samples=100000, n_features=100, dtype=np.float32): + X, y = make_regression( + n_samples=n_samples, + n_features=n_features, + n_informative=n_features // 10, + noise=50, + random_state=0, + ) X = X.astype(dtype, copy=False) X = StandardScaler().fit_transform(X) @@ -90,10 +97,12 @@ def _synth_regression_dataset(n_samples=100000, n_features=100, @M.cache -def _synth_regression_sparse_dataset(n_samples=10000, n_features=10000, - density=0.01, dtype=np.float32): - X = sp.random(m=n_samples, n=n_features, density=density, format='csr', - random_state=0) +def _synth_regression_sparse_dataset( + n_samples=10000, n_features=10000, density=0.01, dtype=np.float32 +): + X = sp.random( + m=n_samples, n=n_features, density=density, format="csr", random_state=0 + ) X.data = np.random.RandomState(0).randn(X.getnnz()) X = X.astype(dtype, copy=False) coefs = sp.random(m=n_features, n=1, density=0.5, random_state=0) @@ -106,11 +115,17 @@ def _synth_regression_sparse_dataset(n_samples=10000, n_features=10000, @M.cache -def _synth_classification_dataset(n_samples=1000, n_features=10000, - n_classes=2, dtype=np.float32): - X, y = make_classification(n_samples=n_samples, n_features=n_features, - n_classes=n_classes, random_state=0, - n_informative=n_features, n_redundant=0) +def _synth_classification_dataset( + n_samples=1000, n_features=10000, n_classes=2, dtype=np.float32 +): + X, y = make_classification( + n_samples=n_samples, + n_features=n_features, + n_classes=n_classes, + random_state=0, + n_informative=n_features, + n_redundant=0, + ) X = X.astype(dtype, copy=False) X = StandardScaler().fit_transform(X) @@ -133,14 +148,21 @@ def _olivetti_faces_dataset(): @M.cache -def _random_dataset(n_samples=1000, n_features=1000, - representation='dense', dtype=np.float32): - if representation == 'dense': +def _random_dataset( + n_samples=1000, n_features=1000, representation="dense", dtype=np.float32 +): + if representation == "dense": X = np.random.RandomState(0).random_sample((n_samples, n_features)) X = X.astype(dtype, copy=False) else: - X = sp.random(n_samples, n_features, density=0.05, format='csr', - dtype=dtype, random_state=0) + X = sp.random( + n_samples, + n_features, + density=0.05, + format="csr", + dtype=dtype, + random_state=0, + ) X, X_val = train_test_split(X, test_size=0.1, random_state=0) return X, X_val, None, None diff --git a/asv_benchmarks/benchmarks/decomposition.py b/asv_benchmarks/benchmarks/decomposition.py index ea23b6d0d4c82..0a7bb7ad07f3e 100644 --- a/asv_benchmarks/benchmarks/decomposition.py +++ b/asv_benchmarks/benchmarks/decomposition.py @@ -1,9 +1,8 @@ -from sklearn.decomposition import (PCA, DictionaryLearning, - MiniBatchDictionaryLearning) +from sklearn.decomposition import PCA, DictionaryLearning, MiniBatchDictionaryLearning from .common import Benchmark, Estimator, Transformer -from .datasets import _olivetti_faces_dataset, _mnist_dataset -from .utils import make_pca_scorers, make_dict_learning_scorers +from .datasets import _mnist_dataset, _olivetti_faces_dataset +from .utils import make_dict_learning_scorers, make_pca_scorers class PCABenchmark(Transformer, Estimator, Benchmark): @@ -11,8 +10,8 @@ class PCABenchmark(Transformer, Estimator, Benchmark): Benchmarks for PCA. """ - param_names = ['svd_solver'] - params = (['full', 'arpack', 'randomized'],) + param_names = ["svd_solver"] + params = (["full", "arpack", "randomized"],) def setup_cache(self): super().setup_cache() @@ -21,11 +20,9 @@ def make_data(self, params): return _mnist_dataset() def make_estimator(self, params): - svd_solver, = params + (svd_solver,) = params - estimator = PCA(n_components=32, - svd_solver=svd_solver, - random_state=0) + estimator = PCA(n_components=32, svd_solver=svd_solver, random_state=0) return estimator @@ -38,8 +35,8 @@ class DictionaryLearningBenchmark(Transformer, Estimator, Benchmark): Benchmarks for DictionaryLearning. """ - param_names = ['fit_algorithm', 'n_jobs'] - params = (['lars', 'cd'], Benchmark.n_jobs_vals) + param_names = ["fit_algorithm", "n_jobs"] + params = (["lars", "cd"], Benchmark.n_jobs_vals) def setup_cache(self): super().setup_cache() @@ -50,13 +47,16 @@ def make_data(self, params): def make_estimator(self, params): fit_algorithm, n_jobs = params - estimator = DictionaryLearning(n_components=15, - fit_algorithm=fit_algorithm, - alpha=0.1, - max_iter=20, - tol=1e-16, - random_state=0, - n_jobs=n_jobs) + estimator = DictionaryLearning( + n_components=15, + fit_algorithm=fit_algorithm, + alpha=0.1, + transform_alpha=1, + max_iter=20, + tol=1e-16, + random_state=0, + n_jobs=n_jobs, + ) return estimator @@ -69,8 +69,8 @@ class MiniBatchDictionaryLearningBenchmark(Transformer, Estimator, Benchmark): Benchmarks for MiniBatchDictionaryLearning """ - param_names = ['fit_algorithm', 'n_jobs'] - params = (['lars', 'cd'], Benchmark.n_jobs_vals) + param_names = ["fit_algorithm", "n_jobs"] + params = (["lars", "cd"], Benchmark.n_jobs_vals) def setup_cache(self): super().setup_cache() @@ -81,12 +81,14 @@ def make_data(self, params): def make_estimator(self, params): fit_algorithm, n_jobs = params - estimator = MiniBatchDictionaryLearning(n_components=15, - fit_algorithm=fit_algorithm, - alpha=0.1, - batch_size=3, - random_state=0, - n_jobs=n_jobs) + estimator = MiniBatchDictionaryLearning( + n_components=15, + fit_algorithm=fit_algorithm, + alpha=0.1, + batch_size=3, + random_state=0, + n_jobs=n_jobs, + ) return estimator diff --git a/asv_benchmarks/benchmarks/ensemble.py b/asv_benchmarks/benchmarks/ensemble.py index c46ac07c84475..c336d1e5f8805 100644 --- a/asv_benchmarks/benchmarks/ensemble.py +++ b/asv_benchmarks/benchmarks/ensemble.py @@ -1,12 +1,15 @@ -from sklearn.experimental import enable_hist_gradient_boosting # noqa -from sklearn.ensemble import (RandomForestClassifier, - GradientBoostingClassifier, - HistGradientBoostingClassifier) +from sklearn.ensemble import ( + GradientBoostingClassifier, + HistGradientBoostingClassifier, + RandomForestClassifier, +) from .common import Benchmark, Estimator, Predictor -from .datasets import (_20newsgroups_highdim_dataset, - _20newsgroups_lowdim_dataset, - _synth_classification_dataset) +from .datasets import ( + _20newsgroups_highdim_dataset, + _20newsgroups_lowdim_dataset, + _synth_classification_dataset, +) from .utils import make_gen_classif_scorers @@ -15,8 +18,8 @@ class RandomForestClassifierBenchmark(Predictor, Estimator, Benchmark): Benchmarks for RandomForestClassifier. """ - param_names = ['representation', 'n_jobs'] - params = (['dense', 'sparse'], Benchmark.n_jobs_vals) + param_names = ["representation", "n_jobs"] + params = (["dense", "sparse"], Benchmark.n_jobs_vals) def setup_cache(self): super().setup_cache() @@ -24,7 +27,7 @@ def setup_cache(self): def make_data(self, params): representation, n_jobs = params - if representation == 'sparse': + if representation == "sparse": data = _20newsgroups_highdim_dataset() else: data = _20newsgroups_lowdim_dataset() @@ -34,13 +37,15 @@ def make_data(self, params): def make_estimator(self, params): representation, n_jobs = params - n_estimators = 500 if Benchmark.data_size == 'large' else 100 + n_estimators = 500 if Benchmark.data_size == "large" else 100 - estimator = RandomForestClassifier(n_estimators=n_estimators, - min_samples_split=10, - max_features='log2', - n_jobs=n_jobs, - random_state=0) + estimator = RandomForestClassifier( + n_estimators=n_estimators, + min_samples_split=10, + max_features="log2", + n_jobs=n_jobs, + random_state=0, + ) return estimator @@ -53,16 +58,16 @@ class GradientBoostingClassifierBenchmark(Predictor, Estimator, Benchmark): Benchmarks for GradientBoostingClassifier. """ - param_names = ['representation'] - params = (['dense', 'sparse'],) + param_names = ["representation"] + params = (["dense", "sparse"],) def setup_cache(self): super().setup_cache() def make_data(self, params): - representation, = params + (representation,) = params - if representation == 'sparse': + if representation == "sparse": data = _20newsgroups_highdim_dataset() else: data = _20newsgroups_lowdim_dataset() @@ -70,14 +75,16 @@ def make_data(self, params): return data def make_estimator(self, params): - representation, = params + (representation,) = params - n_estimators = 100 if Benchmark.data_size == 'large' else 10 + n_estimators = 100 if Benchmark.data_size == "large" else 10 - estimator = GradientBoostingClassifier(n_estimators=n_estimators, - max_features='log2', - subsample=0.5, - random_state=0) + estimator = GradientBoostingClassifier( + n_estimators=n_estimators, + max_features="log2", + subsample=0.5, + random_state=0, + ) return estimator @@ -97,17 +104,16 @@ def setup_cache(self): super().setup_cache() def make_data(self, params): - data = _synth_classification_dataset(n_samples=10000, - n_features=100, - n_classes=5) + data = _synth_classification_dataset( + n_samples=10000, n_features=100, n_classes=5 + ) return data def make_estimator(self, params): - estimator = HistGradientBoostingClassifier(max_iter=100, - max_leaf_nodes=15, - early_stopping=False, - random_state=0) + estimator = HistGradientBoostingClassifier( + max_iter=100, max_leaf_nodes=15, early_stopping=False, random_state=0 + ) return estimator diff --git a/asv_benchmarks/benchmarks/linear_model.py b/asv_benchmarks/benchmarks/linear_model.py index e8f41a97a80cd..24153895611df 100644 --- a/asv_benchmarks/benchmarks/linear_model.py +++ b/asv_benchmarks/benchmarks/linear_model.py @@ -1,11 +1,19 @@ -from sklearn.linear_model import (LogisticRegression, Ridge, ElasticNet, Lasso, - LinearRegression, SGDRegressor) +from sklearn.linear_model import ( + ElasticNet, + Lasso, + LinearRegression, + LogisticRegression, + Ridge, + SGDRegressor, +) from .common import Benchmark, Estimator, Predictor -from .datasets import (_20newsgroups_highdim_dataset, - _20newsgroups_lowdim_dataset, - _synth_regression_dataset, - _synth_regression_sparse_dataset) +from .datasets import ( + _20newsgroups_highdim_dataset, + _20newsgroups_lowdim_dataset, + _synth_regression_dataset, + _synth_regression_sparse_dataset, +) from .utils import make_gen_classif_scorers, make_gen_reg_scorers @@ -14,8 +22,8 @@ class LogisticRegressionBenchmark(Predictor, Estimator, Benchmark): Benchmarks for LogisticRegression. """ - param_names = ['representation', 'solver', 'n_jobs'] - params = (['dense', 'sparse'], ['lbfgs', 'saga'], Benchmark.n_jobs_vals) + param_names = ["representation", "solver", "n_jobs"] + params = (["dense", "sparse"], ["lbfgs", "saga"], Benchmark.n_jobs_vals) def setup_cache(self): super().setup_cache() @@ -23,13 +31,13 @@ def setup_cache(self): def make_data(self, params): representation, solver, n_jobs = params - if Benchmark.data_size == 'large': - if representation == 'sparse': + if Benchmark.data_size == "large": + if representation == "sparse": data = _20newsgroups_highdim_dataset(n_samples=10000) else: data = _20newsgroups_lowdim_dataset(n_components=1e3) else: - if representation == 'sparse': + if representation == "sparse": data = _20newsgroups_highdim_dataset(n_samples=2500) else: data = _20newsgroups_lowdim_dataset() @@ -39,14 +47,15 @@ def make_data(self, params): def make_estimator(self, params): representation, solver, n_jobs = params - penalty = 'l2' if solver == 'lbfgs' else 'l1' + penalty = "l2" if solver == "lbfgs" else "l1" - estimator = LogisticRegression(solver=solver, - penalty=penalty, - multi_class='multinomial', - tol=0.01, - n_jobs=n_jobs, - random_state=0) + estimator = LogisticRegression( + solver=solver, + penalty=penalty, + tol=0.01, + n_jobs=n_jobs, + random_state=0, + ) return estimator @@ -59,9 +68,11 @@ class RidgeBenchmark(Predictor, Estimator, Benchmark): Benchmarks for Ridge. """ - param_names = ['representation', 'solver'] - params = (['dense', 'sparse'], - ['auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag', 'saga']) + param_names = ["representation", "solver"] + params = ( + ["dense", "sparse"], + ["auto", "svd", "cholesky", "lsqr", "sparse_cg", "sag", "saga"], + ) def setup_cache(self): super().setup_cache() @@ -69,21 +80,19 @@ def setup_cache(self): def make_data(self, params): representation, solver = params - if representation == 'dense': + if representation == "dense": data = _synth_regression_dataset(n_samples=500000, n_features=100) else: - data = _synth_regression_sparse_dataset(n_samples=100000, - n_features=10000, - density=0.005) + data = _synth_regression_sparse_dataset( + n_samples=100000, n_features=10000, density=0.005 + ) return data def make_estimator(self, params): representation, solver = params - estimator = Ridge(solver=solver, - fit_intercept=False, - random_state=0) + estimator = Ridge(solver=solver, fit_intercept=False, random_state=0) return estimator @@ -93,31 +102,31 @@ def make_scorers(self): def skip(self, params): representation, solver = params - if representation == 'sparse' and solver == 'svd': + if representation == "sparse" and solver == "svd": return True return False class LinearRegressionBenchmark(Predictor, Estimator, Benchmark): """ - Benchmarks for Linear Reagression. + Benchmarks for Linear Regression. """ - param_names = ['representation'] - params = (['dense', 'sparse'],) + param_names = ["representation"] + params = (["dense", "sparse"],) def setup_cache(self): super().setup_cache() def make_data(self, params): - representation, = params + (representation,) = params - if representation == 'dense': + if representation == "dense": data = _synth_regression_dataset(n_samples=1000000, n_features=100) else: - data = _synth_regression_sparse_dataset(n_samples=10000, - n_features=100000, - density=0.01) + data = _synth_regression_sparse_dataset( + n_samples=10000, n_features=100000, density=0.01 + ) return data @@ -135,28 +144,30 @@ class SGDRegressorBenchmark(Predictor, Estimator, Benchmark): Benchmark for SGD """ - param_names = ['representation'] - params = (['dense', 'sparse'],) + param_names = ["representation"] + params = (["dense", "sparse"],) def setup_cache(self): super().setup_cache() def make_data(self, params): - representation, = params + (representation,) = params - if representation == 'dense': + if representation == "dense": data = _synth_regression_dataset(n_samples=100000, n_features=200) else: - data = _synth_regression_sparse_dataset(n_samples=100000, - n_features=1000, - density=0.01) + data = _synth_regression_sparse_dataset( + n_samples=100000, n_features=1000, density=0.01 + ) return data def make_estimator(self, params): - estimator = SGDRegressor(max_iter=1000, - tol=1e-16, - random_state=0) + (representation,) = params + + max_iter = 60 if representation == "dense" else 300 + + estimator = SGDRegressor(max_iter=max_iter, tol=None, random_state=0) return estimator @@ -169,8 +180,8 @@ class ElasticNetBenchmark(Predictor, Estimator, Benchmark): Benchmarks for ElasticNet. """ - param_names = ['representation', 'precompute'] - params = (['dense', 'sparse'], [True, False]) + param_names = ["representation", "precompute"] + params = (["dense", "sparse"], [True, False]) def setup_cache(self): super().setup_cache() @@ -178,21 +189,19 @@ def setup_cache(self): def make_data(self, params): representation, precompute = params - if representation == 'dense': + if representation == "dense": data = _synth_regression_dataset(n_samples=1000000, n_features=100) else: - data = _synth_regression_sparse_dataset(n_samples=50000, - n_features=5000, - density=0.01) + data = _synth_regression_sparse_dataset( + n_samples=50000, n_features=5000, density=0.01 + ) return data def make_estimator(self, params): representation, precompute = params - estimator = ElasticNet(precompute=precompute, - alpha=0.001, - random_state=0) + estimator = ElasticNet(precompute=precompute, alpha=0.001, random_state=0) return estimator @@ -202,7 +211,7 @@ def make_scorers(self): def skip(self, params): representation, precompute = params - if representation == 'sparse' and precompute is False: + if representation == "sparse" and precompute is False: return True return False @@ -212,8 +221,8 @@ class LassoBenchmark(Predictor, Estimator, Benchmark): Benchmarks for Lasso. """ - param_names = ['representation', 'precompute'] - params = (['dense', 'sparse'], [True, False]) + param_names = ["representation", "precompute"] + params = (["dense", "sparse"], [True, False]) def setup_cache(self): super().setup_cache() @@ -221,21 +230,19 @@ def setup_cache(self): def make_data(self, params): representation, precompute = params - if representation == 'dense': + if representation == "dense": data = _synth_regression_dataset(n_samples=1000000, n_features=100) else: - data = _synth_regression_sparse_dataset(n_samples=50000, - n_features=5000, - density=0.01) + data = _synth_regression_sparse_dataset( + n_samples=50000, n_features=5000, density=0.01 + ) return data def make_estimator(self, params): representation, precompute = params - estimator = Lasso(precompute=precompute, - alpha=0.001, - random_state=0) + estimator = Lasso(precompute=precompute, alpha=0.001, random_state=0) return estimator @@ -245,6 +252,6 @@ def make_scorers(self): def skip(self, params): representation, precompute = params - if representation == 'sparse' and precompute is False: + if representation == "sparse" and precompute is False: return True return False diff --git a/asv_benchmarks/benchmarks/manifold.py b/asv_benchmarks/benchmarks/manifold.py index 26197dc8bbc31..c32f3e061dc33 100644 --- a/asv_benchmarks/benchmarks/manifold.py +++ b/asv_benchmarks/benchmarks/manifold.py @@ -9,21 +9,21 @@ class TSNEBenchmark(Estimator, Benchmark): Benchmarks for t-SNE. """ - param_names = ['method'] - params = (['exact', 'barnes_hut'],) + param_names = ["method"] + params = (["exact", "barnes_hut"],) def setup_cache(self): super().setup_cache() def make_data(self, params): - method, = params + (method,) = params - n_samples = 500 if method == 'exact' else None + n_samples = 500 if method == "exact" else None return _digits_dataset(n_samples=n_samples) def make_estimator(self, params): - method, = params + (method,) = params estimator = TSNE(random_state=0, method=method) diff --git a/asv_benchmarks/benchmarks/metrics.py b/asv_benchmarks/benchmarks/metrics.py index 4a84cf1941a8f..597e5dc789f6c 100644 --- a/asv_benchmarks/benchmarks/metrics.py +++ b/asv_benchmarks/benchmarks/metrics.py @@ -9,34 +9,34 @@ class PairwiseDistancesBenchmark(Benchmark): Benchmarks for pairwise distances. """ - param_names = ['representation', 'metric', 'n_jobs'] - params = (['dense', 'sparse'], - ['cosine', 'euclidean', 'manhattan', 'correlation'], - Benchmark.n_jobs_vals) + param_names = ["representation", "metric", "n_jobs"] + params = ( + ["dense", "sparse"], + ["cosine", "euclidean", "manhattan", "correlation"], + Benchmark.n_jobs_vals, + ) def setup(self, *params): representation, metric, n_jobs = params - if representation == 'sparse' and metric == 'correlation': + if representation == "sparse" and metric == "correlation": raise NotImplementedError - if Benchmark.data_size == 'large': - if metric in ('manhattan', 'correlation'): + if Benchmark.data_size == "large": + if metric in ("manhattan", "correlation"): n_samples = 8000 else: n_samples = 24000 else: - if metric in ('manhattan', 'correlation'): + if metric in ("manhattan", "correlation"): n_samples = 4000 else: n_samples = 12000 - data = _random_dataset(n_samples=n_samples, - representation=representation) + data = _random_dataset(n_samples=n_samples, representation=representation) self.X, self.X_val, self.y, self.y_val = data - self.pdist_params = {'metric': metric, - 'n_jobs': n_jobs} + self.pdist_params = {"metric": metric, "n_jobs": n_jobs} def time_pairwise_distances(self, *args): pairwise_distances(self.X, **self.pdist_params) diff --git a/asv_benchmarks/benchmarks/model_selection.py b/asv_benchmarks/benchmarks/model_selection.py index 4e7058ffc2262..335ffe498adaa 100644 --- a/asv_benchmarks/benchmarks/model_selection.py +++ b/asv_benchmarks/benchmarks/model_selection.py @@ -13,23 +13,20 @@ class CrossValidationBenchmark(Benchmark): timeout = 20000 - param_names = ['n_jobs'] + param_names = ["n_jobs"] params = (Benchmark.n_jobs_vals,) def setup(self, *params): - n_jobs, = params + (n_jobs,) = params data = _synth_classification_dataset(n_samples=50000, n_features=100) self.X, self.X_val, self.y, self.y_val = data - self.clf = RandomForestClassifier(n_estimators=50, - max_depth=10, - random_state=0) + self.clf = RandomForestClassifier(n_estimators=50, max_depth=10, random_state=0) - cv = 16 if Benchmark.data_size == 'large' else 4 + cv = 16 if Benchmark.data_size == "large" else 4 - self.cv_params = {'n_jobs': n_jobs, - 'cv': cv} + self.cv_params = {"n_jobs": n_jobs, "cv": cv} def time_crossval(self, *args): cross_val_score(self.clf, self.X, self.y, **self.cv_params) @@ -38,8 +35,7 @@ def peakmem_crossval(self, *args): cross_val_score(self.clf, self.X, self.y, **self.cv_params) def track_crossval(self, *args): - return float(cross_val_score(self.clf, self.X, - self.y, **self.cv_params).mean()) + return float(cross_val_score(self.clf, self.X, self.y, **self.cv_params).mean()) class GridSearchBenchmark(Predictor, Estimator, Benchmark): @@ -49,7 +45,7 @@ class GridSearchBenchmark(Predictor, Estimator, Benchmark): timeout = 20000 - param_names = ['n_jobs'] + param_names = ["n_jobs"] params = (Benchmark.n_jobs_vals,) def setup_cache(self): @@ -61,11 +57,11 @@ def make_data(self, params): return data def make_estimator(self, params): - n_jobs, = params + (n_jobs,) = params clf = RandomForestClassifier(random_state=0) - if Benchmark.data_size == 'large': + if Benchmark.data_size == "large": n_estimators_list = [10, 25, 50, 100, 500] max_depth_list = [5, 10, None] max_features_list = [0.1, 0.4, 0.8, 1.0] @@ -74,9 +70,11 @@ def make_estimator(self, params): max_depth_list = [5, 10] max_features_list = [0.1, 0.4, 0.8] - param_grid = {'n_estimators': n_estimators_list, - 'max_depth': max_depth_list, - 'max_features': max_features_list} + param_grid = { + "n_estimators": n_estimators_list, + "max_depth": max_depth_list, + "max_features": max_features_list, + } estimator = GridSearchCV(clf, param_grid, n_jobs=n_jobs, cv=4) diff --git a/asv_benchmarks/benchmarks/neighbors.py b/asv_benchmarks/benchmarks/neighbors.py index 2be6cc2f09364..b0bf6aba1d85b 100644 --- a/asv_benchmarks/benchmarks/neighbors.py +++ b/asv_benchmarks/benchmarks/neighbors.py @@ -10,10 +10,8 @@ class KNeighborsClassifierBenchmark(Predictor, Estimator, Benchmark): Benchmarks for KNeighborsClassifier. """ - param_names = ['algorithm', 'dimension', 'n_jobs'] - params = (['brute', 'kd_tree', 'ball_tree'], - ['low', 'high'], - Benchmark.n_jobs_vals) + param_names = ["algorithm", "dimension", "n_jobs"] + params = (["brute", "kd_tree", "ball_tree"], ["low", "high"], Benchmark.n_jobs_vals) def setup_cache(self): super().setup_cache() @@ -21,10 +19,10 @@ def setup_cache(self): def make_data(self, params): algorithm, dimension, n_jobs = params - if Benchmark.data_size == 'large': - n_components = 40 if dimension == 'low' else 200 + if Benchmark.data_size == "large": + n_components = 40 if dimension == "low" else 200 else: - n_components = 10 if dimension == 'low' else 50 + n_components = 10 if dimension == "low" else 50 data = _20newsgroups_lowdim_dataset(n_components=n_components) @@ -33,8 +31,7 @@ def make_data(self, params): def make_estimator(self, params): algorithm, dimension, n_jobs = params - estimator = KNeighborsClassifier(algorithm=algorithm, - n_jobs=n_jobs) + estimator = KNeighborsClassifier(algorithm=algorithm, n_jobs=n_jobs) return estimator diff --git a/asv_benchmarks/benchmarks/svm.py b/asv_benchmarks/benchmarks/svm.py index bbcc7a27edecf..36d3066484ee5 100644 --- a/asv_benchmarks/benchmarks/svm.py +++ b/asv_benchmarks/benchmarks/svm.py @@ -8,8 +8,8 @@ class SVCBenchmark(Predictor, Estimator, Benchmark): """Benchmarks for SVC.""" - param_names = ['kernel'] - params = (['linear', 'poly', 'rbf', 'sigmoid'],) + param_names = ["kernel"] + params = (["linear", "poly", "rbf", "sigmoid"],) def setup_cache(self): super().setup_cache() @@ -18,13 +18,11 @@ def make_data(self, params): return _synth_classification_dataset() def make_estimator(self, params): - kernel, = params + (kernel,) = params - estimator = SVC(max_iter=100, - tol=1e-16, - kernel=kernel, - random_state=0, - gamma='scale') + estimator = SVC( + max_iter=100, tol=1e-16, kernel=kernel, random_state=0, gamma="scale" + ) return estimator diff --git a/asv_benchmarks/benchmarks/utils.py b/asv_benchmarks/benchmarks/utils.py index 6a3073a634169..fca30579e529b 100644 --- a/asv_benchmarks/benchmarks/utils.py +++ b/asv_benchmarks/benchmarks/utils.py @@ -4,7 +4,7 @@ def neg_mean_inertia(X, labels, centers): - return - (np.asarray(X - centers[labels])**2).sum(axis=1).mean() + return -(np.asarray(X - centers[labels]) ** 2).sum(axis=1).mean() def make_gen_classif_scorers(caller): @@ -18,18 +18,22 @@ def make_gen_reg_scorers(caller): def neg_mean_data_error(X, U, V): - return - np.sqrt(((X - U.dot(V))**2).mean()) + return -np.sqrt(((X - U.dot(V)) ** 2).mean()) def make_dict_learning_scorers(caller): caller.train_scorer = lambda _, __: ( - neg_mean_data_error(caller.X, - caller.estimator.transform(caller.X), - caller.estimator.components_)) + neg_mean_data_error( + caller.X, caller.estimator.transform(caller.X), caller.estimator.components_ + ) + ) caller.test_scorer = lambda _, __: ( - neg_mean_data_error(caller.X_val, - caller.estimator.transform(caller.X_val), - caller.estimator.components_)) + neg_mean_data_error( + caller.X_val, + caller.estimator.transform(caller.X_val), + caller.estimator.components_, + ) + ) def explained_variance_ratio(Xt, X): @@ -37,8 +41,7 @@ def explained_variance_ratio(Xt, X): def make_pca_scorers(caller): - caller.train_scorer = ( - lambda _, __: caller.estimator.explained_variance_ratio_.sum()) + caller.train_scorer = lambda _, __: caller.estimator.explained_variance_ratio_.sum() caller.test_scorer = lambda _, __: ( - explained_variance_ratio(caller.estimator.transform(caller.X_val), - caller.X_val)) + explained_variance_ratio(caller.estimator.transform(caller.X_val), caller.X_val) + ) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3cd2b5bb4cd9f..a36daf39b50db 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -11,20 +11,9 @@ jobs: - job: git_commit displayName: Get Git Commit pool: - vmImage: ubuntu-18.04 + vmImage: ubuntu-24.04 steps: - - bash: | - set -ex - if [[ $BUILD_REASON == "PullRequest" ]]; then - # By default pull requests use refs/pull/PULL_ID/merge as the source branch - # which has a "Merge ID into ID" as a commit message. The latest commit - # message is the second to last commit - COMMIT_ID=$(echo $BUILD_SOURCEVERSIONMESSAGE | awk '{print $2}') - message=$(git log $COMMIT_ID -1 --pretty=%B) - else - message=$BUILD_SOURCEVERSIONMESSAGE - fi - echo "##vso[task.setvariable variable=message;isOutput=true]$message" + - bash: python build_tools/azure/get_commit_message.py name: commit displayName: Get source version message @@ -38,25 +27,29 @@ jobs: ) displayName: Linting pool: - vmImage: ubuntu-18.04 + vmImage: ubuntu-24.04 steps: - task: UsePythonVersion@0 inputs: - versionSpec: '3.9' + versionSpec: '3.12' - bash: | - pip install flake8 mypy==0.782 + source build_tools/shared.sh + # Include pytest compatibility with mypy + pip install pytest $(get_dep ruff min) $(get_dep mypy min) cython-lint displayName: Install linters - bash: | - ./build_tools/circle/linting.sh - displayName: Run linting + ./build_tools/linting.sh + displayName: Run linters - bash: | - mypy sklearn/ - displayName: Run mypy + pip install ninja meson scipy + python build_tools/check-meson-openmp-dependencies.py + displayName: Run Meson OpenMP checks + - template: build_tools/azure/posix.yml parameters: name: Linux_Nightly - vmImage: ubuntu-18.04 + vmImage: ubuntu-22.04 dependsOn: [git_commit, linting] condition: | and( @@ -70,41 +63,37 @@ jobs: matrix: pylatest_pip_scipy_dev: DISTRIB: 'conda-pip-scipy-dev' - PYTHON_VERSION: '*' - CHECK_WARNINGS: 'true' + LOCK_FILE: './build_tools/azure/pylatest_pip_scipy_dev_linux-64_conda.lock' + SKLEARN_WARNINGS_AS_ERRORS: '1' CHECK_PYTEST_SOFT_DEPENDENCY: 'true' - TEST_DOCSTRINGS: 'true' - # Tests that require large downloads over the networks are skipped in CI. - # Here we make sure, that they are still run on a regular basis. - SKLEARN_SKIP_NETWORK_TESTS: '0' -# Check compilation with intel C++ compiler (ICC) - template: build_tools/azure/posix.yml + # CPython 3.13 free-threaded build parameters: - name: Linux_Nightly_ICC - vmImage: ubuntu-18.04 + name: Linux_free_threaded + vmImage: ubuntu-22.04 dependsOn: [git_commit, linting] condition: | and( succeeded(), not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')), or(eq(variables['Build.Reason'], 'Schedule'), - contains(dependencies['git_commit']['outputs']['commit.message'], '[icc-build]') + contains(dependencies['git_commit']['outputs']['commit.message'], '[free-threaded]' + ) ) ) matrix: - pylatest_conda_mkl: - DISTRIB: 'conda' - PYTHON_VERSION: '*' - BLAS: 'mkl' + pylatest_free_threaded: + DISTRIB: 'conda-free-threaded' + LOCK_FILE: './build_tools/azure/pylatest_free_threaded_linux-64_conda.lock' COVERAGE: 'false' - BUILD_WITH_ICC: 'true' + SKLEARN_FAULTHANDLER_TIMEOUT: '1800' # 30 * 60 seconds # Will run all the time regardless of linting outcome. - template: build_tools/azure/posix.yml parameters: name: Linux_Runs - vmImage: ubuntu-18.04 + vmImage: ubuntu-22.04 dependsOn: [git_commit] condition: | and( @@ -112,124 +101,171 @@ jobs: not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')) ) matrix: - pylatest_conda_mkl: + pylatest_conda_forge_mkl: DISTRIB: 'conda' - PYTHON_VERSION: '*' - BLAS: 'mkl' + LOCK_FILE: './build_tools/azure/pylatest_conda_forge_mkl_linux-64_conda.lock' COVERAGE: 'true' + SKLEARN_TESTS_GLOBAL_RANDOM_SEED: '42' # default global random seed + # Tests that require large downloads over the networks are skipped in CI. + # Here we make sure, that they are still run on a regular basis. + ${{ if eq(variables['Build.Reason'], 'Schedule') }}: + SKLEARN_SKIP_NETWORK_TESTS: '0' + SCIPY_ARRAY_API: '1' -- template: build_tools/azure/posix.yml +# Check compilation with Ubuntu 22.04 LTS (Jammy Jellyfish) and scipy from conda-forge +# By default the CI is sequential, where `Ubuntu_Jammy_Jellyfish` runs first and +# the others jobs are run only if `Ubuntu_Jammy_Jellyfish` succeeds. +# When "[azure parallel]" is in the commit message, `Ubuntu_Jammy_Jellyfish` will +# run in parallel with the rest of the jobs. On Azure, the job's name will be +# `Ubuntu_Jammy_Jellyfish_Parallel`. +- template: build_tools/azure/posix-all-parallel.yml parameters: - name: Linux - vmImage: ubuntu-18.04 - dependsOn: [linting, git_commit] + name: Ubuntu_Jammy_Jellyfish + vmImage: ubuntu-22.04 + dependsOn: [git_commit, linting] condition: | and( succeeded(), - not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')), - ne(variables['Build.Reason'], 'Schedule') + not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')) + ) + commitMessage: dependencies['git_commit']['outputs']['commit.message'] + matrix: + pymin_conda_forge_openblas_ubuntu_2204: + DISTRIB: 'conda' + LOCK_FILE: './build_tools/azure/pymin_conda_forge_openblas_ubuntu_2204_linux-64_conda.lock' + SKLEARN_WARNINGS_AS_ERRORS: '1' + COVERAGE: 'false' + SKLEARN_TESTS_GLOBAL_RANDOM_SEED: '0' # non-default seed + +- template: build_tools/azure/posix.yml + parameters: + name: Ubuntu_Atlas + vmImage: ubuntu-24.04 + dependsOn: [linting, git_commit, Ubuntu_Jammy_Jellyfish] + # Runs when dependencies succeeded or skipped + condition: | + and( + not(or(failed(), canceled())), + not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')) ) matrix: # Linux environment to test that scikit-learn can be built against - # versions of numpy, scipy with ATLAS that comes with Ubuntu Bionic 18.04 - # i.e. numpy 1.13.3 and scipy 0.19 - py36_ubuntu_atlas: + # versions of numpy, scipy with ATLAS that comes with Ubuntu 24.04 Noble Numbat + # i.e. numpy 1.26.4 and scipy 1.11.4 + ubuntu_atlas: DISTRIB: 'ubuntu' - PYTHON_VERSION: '3.6' - JOBLIB_VERSION: 'min' - PANDAS_VERSION: 'none' - THREADPOOLCTL_VERSION: 'min' - PYTEST_VERSION: 'min' - PYTEST_XDIST_VERSION: 'none' + LOCK_FILE: './build_tools/azure/ubuntu_atlas_lock.txt' COVERAGE: 'false' - # Linux + Python 3.6 build with OpenBLAS and without SITE_JOBLIB - py36_conda_openblas: + SKLEARN_TESTS_GLOBAL_RANDOM_SEED: '1' # non-default seed + +- template: build_tools/azure/posix.yml + parameters: + name: Linux + vmImage: ubuntu-22.04 + dependsOn: [linting, git_commit, Ubuntu_Jammy_Jellyfish] + # Runs when dependencies succeeded or skipped + condition: | + and( + not(or(failed(), canceled())), + not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')) + ) + matrix: + # Linux build with minimum supported version of dependencies + pymin_conda_forge_openblas_min_dependencies: DISTRIB: 'conda' - PYTHON_VERSION: '3.6' - BLAS: 'openblas' - NUMPY_VERSION: 'min' - SCIPY_VERSION: 'min' - MATPLOTLIB_VERSION: 'min' - # latest version of joblib available in conda for Python 3.6 - JOBLIB_VERSION: '0.13.2' - THREADPOOLCTL_VERSION: '2.0.0' - # temporary pin pytest due to unknown failure with pytest 5.4 and - # python 3.6 - PYTEST_VERSION: 'min' - PYTEST_XDIST_VERSION: 'none' - # Linux environment to test the latest available dependencies and MKL. + LOCK_FILE: './build_tools/azure/pymin_conda_forge_openblas_min_dependencies_linux-64_conda.lock' + # Enable debug Cython directives to capture IndexError exceptions in + # combination with the -Werror::pytest.PytestUnraisableExceptionWarning + # flag for pytest. + # https://github.com/scikit-learn/scikit-learn/pull/24438 + SKLEARN_ENABLE_DEBUG_CYTHON_DIRECTIVES: '1' + SKLEARN_RUN_FLOAT32_TESTS: '1' + SKLEARN_TESTS_GLOBAL_RANDOM_SEED: '2' # non-default seed + # Linux environment to test the latest available dependencies. # It runs tests requiring lightgbm, pandas and PyAMG. pylatest_pip_openblas_pandas: DISTRIB: 'conda-pip-latest' - PYTHON_VERSION: '3.9' - PANDAS_VERSION: 'none' + LOCK_FILE: './build_tools/azure/pylatest_pip_openblas_pandas_linux-64_conda.lock' CHECK_PYTEST_SOFT_DEPENDENCY: 'true' - TEST_DOCSTRINGS: 'true' - CHECK_WARNINGS: 'true' + SKLEARN_WARNINGS_AS_ERRORS: '1' + SKLEARN_TESTS_GLOBAL_RANDOM_SEED: '3' # non-default seed + # disable pytest-xdist to have 1 job where OpenMP and BLAS are not single + # threaded because by default the tests configuration (sklearn/conftest.py) + # makes sure that they are single threaded in each xdist subprocess. + PYTEST_XDIST_VERSION: 'none' + PIP_BUILD_ISOLATION: 'true' + SCIPY_ARRAY_API: '1' -- template: build_tools/azure/posix-32.yml +- template: build_tools/azure/posix-docker.yml parameters: - name: Linux32 - vmImage: ubuntu-18.04 - dependsOn: [linting, git_commit] + name: Linux_Docker + vmImage: ubuntu-24.04 + dependsOn: [linting, git_commit, Ubuntu_Jammy_Jellyfish] + # Runs when dependencies succeeded or skipped condition: | and( - succeeded(), - not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')), - ne(variables['Build.Reason'], 'Schedule') + not(or(failed(), canceled())), + not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')) ) matrix: - py36_ubuntu_atlas_32bit: - DISTRIB: 'ubuntu-32' - PYTHON_VERSION: '3.6' - JOBLIB_VERSION: 'min' + debian_32bit: + DOCKER_CONTAINER: 'i386/debian:trixie' + DISTRIB: 'debian-32' + COVERAGE: "true" + LOCK_FILE: './build_tools/azure/debian_32bit_lock.txt' # disable pytest xdist due to unknown bug with 32-bit container PYTEST_XDIST_VERSION: 'none' - # temporary pin pytest due to unknown failure with pytest 5.4 and - # python 3.6 - PYTEST_VERSION: 'min' - THREADPOOLCTL_VERSION: 'min' + SKLEARN_TESTS_GLOBAL_RANDOM_SEED: '4' # non-default seed - template: build_tools/azure/posix.yml parameters: name: macOS - vmImage: macOS-10.14 - dependsOn: [linting, git_commit] + vmImage: macOS-13 + dependsOn: [linting, git_commit, Ubuntu_Jammy_Jellyfish] + # Runs when dependencies succeeded or skipped condition: | and( - succeeded(), - not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')), - ne(variables['Build.Reason'], 'Schedule') + not(or(failed(), canceled())), + not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')) ) matrix: pylatest_conda_forge_mkl: DISTRIB: 'conda' - BLAS: 'mkl' - CONDA_CHANNEL: 'conda-forge' + LOCK_FILE: './build_tools/azure/pylatest_conda_forge_mkl_osx-64_conda.lock' + SKLEARN_TESTS_GLOBAL_RANDOM_SEED: '5' # non-default seed + SCIPY_ARRAY_API: '1' pylatest_conda_mkl_no_openmp: DISTRIB: 'conda' - BLAS: 'mkl' + LOCK_FILE: './build_tools/azure/pylatest_conda_mkl_no_openmp_osx-64_conda.lock' SKLEARN_TEST_NO_OPENMP: 'true' SKLEARN_SKIP_OPENMP_TEST: 'true' + SKLEARN_TESTS_GLOBAL_RANDOM_SEED: '6' # non-default seed - template: build_tools/azure/windows.yml parameters: name: Windows - vmImage: vs2017-win2016 - dependsOn: [linting, git_commit] + vmImage: windows-latest + dependsOn: [linting, git_commit, Ubuntu_Jammy_Jellyfish] + # Runs when dependencies succeeded or skipped condition: | and( - succeeded(), - not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')), - ne(variables['Build.Reason'], 'Schedule') + not(or(failed(), canceled())), + not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')) ) matrix: - py37_conda_mkl: - PYTHON_VERSION: '3.7' - CHECK_WARNINGS: 'true' - PYTHON_ARCH: '64' - PYTEST_VERSION: '*' - COVERAGE: 'true' - py36_pip_openblas_32bit: - PYTHON_VERSION: '3.6' - PYTHON_ARCH: '32' + pymin_conda_forge_mkl: + DISTRIB: 'conda' + LOCK_FILE: ./build_tools/azure/pymin_conda_forge_mkl_win-64_conda.lock + SKLEARN_WARNINGS_AS_ERRORS: '1' + # The Azure Windows runner is typically much slower than other CI + # runners due to the lack of compiler cache. Running the tests with + # coverage enabled make them run extra slower. Since very few parts of + # code should have windows-specific code branches, it should be enable + # to restrict the code coverage collection to the non-windows runners. + COVERAGE: 'false' + # Enable debug Cython directives to capture IndexError exceptions in + # combination with the -Werror::pytest.PytestUnraisableExceptionWarning + # flag for pytest. + # https://github.com/scikit-learn/scikit-learn/pull/24438 + SKLEARN_ENABLE_DEBUG_CYTHON_DIRECTIVES: '1' + SKLEARN_TESTS_GLOBAL_RANDOM_SEED: '7' # non-default seed diff --git a/benchmarks/bench_20newsgroups.py b/benchmarks/bench_20newsgroups.py index 8efc740e937da..a559bc59b5f8a 100644 --- a/benchmarks/bench_20newsgroups.py +++ b/benchmarks/bench_20newsgroups.py @@ -1,25 +1,24 @@ -from time import time import argparse -import numpy as np +from time import time -from sklearn.dummy import DummyClassifier +import numpy as np from sklearn.datasets import fetch_20newsgroups_vectorized -from sklearn.metrics import accuracy_score -from sklearn.utils.validation import check_array - -from sklearn.ensemble import RandomForestClassifier -from sklearn.ensemble import ExtraTreesClassifier -from sklearn.ensemble import AdaBoostClassifier +from sklearn.dummy import DummyClassifier +from sklearn.ensemble import ( + AdaBoostClassifier, + ExtraTreesClassifier, + RandomForestClassifier, +) from sklearn.linear_model import LogisticRegression +from sklearn.metrics import accuracy_score from sklearn.naive_bayes import MultinomialNB +from sklearn.utils.validation import check_array ESTIMATORS = { "dummy": DummyClassifier(), - "random_forest": RandomForestClassifier(max_features="sqrt", - min_samples_split=10), - "extra_trees": ExtraTreesClassifier(max_features="sqrt", - min_samples_split=10), + "random_forest": RandomForestClassifier(max_features="sqrt", min_samples_split=10), + "extra_trees": ExtraTreesClassifier(max_features="sqrt", min_samples_split=10), "logistic_regression": LogisticRegression(), "naive_bayes": MultinomialNB(), "adaboost": AdaBoostClassifier(n_estimators=10), @@ -30,34 +29,31 @@ # Data if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument('-e', '--estimators', nargs="+", required=True, - choices=ESTIMATORS) + parser.add_argument( + "-e", "--estimators", nargs="+", required=True, choices=ESTIMATORS + ) args = vars(parser.parse_args()) data_train = fetch_20newsgroups_vectorized(subset="train") data_test = fetch_20newsgroups_vectorized(subset="test") - X_train = check_array(data_train.data, dtype=np.float32, - accept_sparse="csc") + X_train = check_array(data_train.data, dtype=np.float32, accept_sparse="csc") X_test = check_array(data_test.data, dtype=np.float32, accept_sparse="csr") y_train = data_train.target y_test = data_test.target print("20 newsgroups") print("=============") - print("X_train.shape = {0}".format(X_train.shape)) - print("X_train.format = {0}".format(X_train.format)) - print("X_train.dtype = {0}".format(X_train.dtype)) - print("X_train density = {0}" - "".format(X_train.nnz / np.product(X_train.shape))) - print("y_train {0}".format(y_train.shape)) - print("X_test {0}".format(X_test.shape)) - print("X_test.format = {0}".format(X_test.format)) - print("X_test.dtype = {0}".format(X_test.dtype)) - print("y_test {0}".format(y_test.shape)) + print(f"X_train.shape = {X_train.shape}") + print(f"X_train.format = {X_train.format}") + print(f"X_train.dtype = {X_train.dtype}") + print(f"X_train density = {X_train.nnz / np.prod(X_train.shape)}") + print(f"y_train {y_train.shape}") + print(f"X_test {X_test.shape}") + print(f"X_test.format = {X_test.format}") + print(f"X_test.dtype = {X_test.dtype}") + print(f"y_test {y_test.shape}") print() - print("Classifier Training") print("===================") accuracy, train_time, test_time = {}, {}, {} @@ -82,13 +78,17 @@ print("Classification performance:") print("===========================") print() - print("%s %s %s %s" % ("Classifier ", "train-time", "test-time", - "Accuracy")) + print("%s %s %s %s" % ("Classifier ", "train-time", "test-time", "Accuracy")) print("-" * 44) for name in sorted(accuracy, key=accuracy.get): - print("%s %s %s %s" % (name.ljust(16), - ("%.4fs" % train_time[name]).center(10), - ("%.4fs" % test_time[name]).center(10), - ("%.4f" % accuracy[name]).center(10))) + print( + "%s %s %s %s" + % ( + name.ljust(16), + ("%.4fs" % train_time[name]).center(10), + ("%.4fs" % test_time[name]).center(10), + ("%.4f" % accuracy[name]).center(10), + ) + ) print() diff --git a/benchmarks/bench_covertype.py b/benchmarks/bench_covertype.py index b74f74bbbbb76..243cce03a632f 100644 --- a/benchmarks/bench_covertype.py +++ b/benchmarks/bench_covertype.py @@ -25,13 +25,13 @@ The same task has been used in a number of papers including: - * `"SVM Optimization: Inverse Dependence on Training Set Size" - `_ + * :doi:`"SVM Optimization: Inverse Dependence on Training Set Size" S. Shalev-Shwartz, N. Srebro - In Proceedings of ICML '08. + <10.1145/1390156.1390273>` - * `"Pegasos: Primal estimated sub-gradient solver for svm" - `_ + * :doi:`"Pegasos: Primal estimated sub-gradient solver for svm" S. Shalev-Shwartz, Y. Singer, N. Srebro - In Proceedings of ICML '07. + <10.1145/1273496.1273598>` * `"Training Linear SVMs in Linear Time" `_ @@ -41,42 +41,47 @@ """ -# Author: Peter Prettenhofer -# Arnaud Joly -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +import argparse import os from time import time -import argparse + import numpy as np from joblib import Memory from sklearn.datasets import fetch_covtype, get_data_home -from sklearn.svm import LinearSVC -from sklearn.linear_model import SGDClassifier, LogisticRegression +from sklearn.ensemble import ( + ExtraTreesClassifier, + GradientBoostingClassifier, + RandomForestClassifier, +) +from sklearn.linear_model import LogisticRegression, SGDClassifier +from sklearn.metrics import zero_one_loss from sklearn.naive_bayes import GaussianNB +from sklearn.svm import LinearSVC from sklearn.tree import DecisionTreeClassifier -from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier -from sklearn.ensemble import GradientBoostingClassifier -from sklearn.metrics import zero_one_loss from sklearn.utils import check_array # Memoize the data extraction and memory map the resulting # train / test splits in readonly mode -memory = Memory(os.path.join(get_data_home(), 'covertype_benchmark_data'), - mmap_mode='r') +memory = Memory( + os.path.join(get_data_home(), "covertype_benchmark_data"), mmap_mode="r" +) @memory.cache -def load_data(dtype=np.float32, order='C', random_state=13): +def load_data(dtype=np.float32, order="C", random_state=13): """Load the data, then cache and memmap the train/test split""" ###################################################################### # Load dataset print("Loading dataset...") - data = fetch_covtype(download_if_missing=True, shuffle=True, - random_state=random_state) - X = check_array(data['data'], dtype=dtype, order=order) - y = (data['target'] != 1).astype(int) + data = fetch_covtype( + download_if_missing=True, shuffle=True, random_state=random_state + ) + X = check_array(data["data"], dtype=dtype, order=order) + y = (data["target"] != 1).astype(int) # Create train-test split (as [Joachims, 2006]) print("Creating train-test split...") @@ -97,39 +102,59 @@ def load_data(dtype=np.float32, order='C', random_state=13): ESTIMATORS = { - 'GBRT': GradientBoostingClassifier(n_estimators=250), - 'ExtraTrees': ExtraTreesClassifier(n_estimators=20), - 'RandomForest': RandomForestClassifier(n_estimators=20), - 'CART': DecisionTreeClassifier(min_samples_split=5), - 'SGD': SGDClassifier(alpha=0.001), - 'GaussianNB': GaussianNB(), - 'liblinear': LinearSVC(loss="l2", penalty="l2", C=1000, dual=False, - tol=1e-3), - 'SAG': LogisticRegression(solver='sag', max_iter=2, C=1000) + "GBRT": GradientBoostingClassifier(n_estimators=250), + "ExtraTrees": ExtraTreesClassifier(n_estimators=20), + "RandomForest": RandomForestClassifier(n_estimators=20), + "CART": DecisionTreeClassifier(min_samples_split=5), + "SGD": SGDClassifier(alpha=0.001), + "GaussianNB": GaussianNB(), + "liblinear": LinearSVC(loss="l2", penalty="l2", C=1000, dual=False, tol=1e-3), + "SAG": LogisticRegression(solver="sag", max_iter=2, C=1000), } if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('--classifiers', nargs="+", - choices=ESTIMATORS, type=str, - default=['liblinear', 'GaussianNB', 'SGD', 'CART'], - help="list of classifiers to benchmark.") - parser.add_argument('--n-jobs', nargs="?", default=1, type=int, - help="Number of concurrently running workers for " - "models that support parallelism.") - parser.add_argument('--order', nargs="?", default="C", type=str, - choices=["F", "C"], - help="Allow to choose between fortran and C ordered " - "data") - parser.add_argument('--random-seed', nargs="?", default=13, type=int, - help="Common seed used by random number generator.") + parser.add_argument( + "--classifiers", + nargs="+", + choices=ESTIMATORS, + type=str, + default=["liblinear", "GaussianNB", "SGD", "CART"], + help="list of classifiers to benchmark.", + ) + parser.add_argument( + "--n-jobs", + nargs="?", + default=1, + type=int, + help=( + "Number of concurrently running workers for " + "models that support parallelism." + ), + ) + parser.add_argument( + "--order", + nargs="?", + default="C", + type=str, + choices=["F", "C"], + help="Allow to choose between fortran and C ordered data", + ) + parser.add_argument( + "--random-seed", + nargs="?", + default=13, + type=int, + help="Common seed used by random number generator.", + ) args = vars(parser.parse_args()) print(__doc__) X_train, X_test, y_train, y_test = load_data( - order=args["order"], random_state=args["random_seed"]) + order=args["order"], random_state=args["random_seed"] + ) print("") print("Dataset statistics:") @@ -137,14 +162,26 @@ def load_data(dtype=np.float32, order='C', random_state=13): print("%s %d" % ("number of features:".ljust(25), X_train.shape[1])) print("%s %d" % ("number of classes:".ljust(25), np.unique(y_train).size)) print("%s %s" % ("data type:".ljust(25), X_train.dtype)) - print("%s %d (pos=%d, neg=%d, size=%dMB)" - % ("number of train samples:".ljust(25), - X_train.shape[0], np.sum(y_train == 1), - np.sum(y_train == 0), int(X_train.nbytes / 1e6))) - print("%s %d (pos=%d, neg=%d, size=%dMB)" - % ("number of test samples:".ljust(25), - X_test.shape[0], np.sum(y_test == 1), - np.sum(y_test == 0), int(X_test.nbytes / 1e6))) + print( + "%s %d (pos=%d, neg=%d, size=%dMB)" + % ( + "number of train samples:".ljust(25), + X_train.shape[0], + np.sum(y_train == 1), + np.sum(y_train == 0), + int(X_train.nbytes / 1e6), + ) + ) + print( + "%s %d (pos=%d, neg=%d, size=%dMB)" + % ( + "number of test samples:".ljust(25), + X_test.shape[0], + np.sum(y_test == 1), + np.sum(y_test == 0), + int(X_test.nbytes / 1e6), + ) + ) print() print("Training Classifiers") @@ -155,9 +192,13 @@ def load_data(dtype=np.float32, order='C', random_state=13): estimator = ESTIMATORS[name] estimator_params = estimator.get_params() - estimator.set_params(**{p: args["random_seed"] - for p in estimator_params - if p.endswith("random_state")}) + estimator.set_params( + **{ + p: args["random_seed"] + for p in estimator_params + if p.endswith("random_state") + } + ) if "n_jobs" in estimator_params: estimator.set_params(n_jobs=args["n_jobs"]) @@ -177,13 +218,17 @@ def load_data(dtype=np.float32, order='C', random_state=13): print() print("Classification performance:") print("===========================") - print("%s %s %s %s" - % ("Classifier ", "train-time", "test-time", "error-rate")) + print("%s %s %s %s" % ("Classifier ", "train-time", "test-time", "error-rate")) print("-" * 44) for name in sorted(args["classifiers"], key=error.get): - print("%s %s %s %s" % (name.ljust(12), - ("%.4fs" % train_time[name]).center(10), - ("%.4fs" % test_time[name]).center(10), - ("%.4f" % error[name]).center(10))) + print( + "%s %s %s %s" + % ( + name.ljust(12), + ("%.4fs" % train_time[name]).center(10), + ("%.4fs" % test_time[name]).center(10), + ("%.4f" % error[name]).center(10), + ) + ) print() diff --git a/benchmarks/bench_feature_expansions.py b/benchmarks/bench_feature_expansions.py index 412ab28598c9b..b9d9efbdea4f1 100644 --- a/benchmarks/bench_feature_expansions.py +++ b/benchmarks/bench_feature_expansions.py @@ -1,8 +1,10 @@ +from time import time + import matplotlib.pyplot as plt import numpy as np import scipy.sparse as sparse + from sklearn.preprocessing import PolynomialFeatures -from time import time degree = 2 trials = 3 @@ -11,8 +13,9 @@ densities = np.array([0.01, 0.1, 1.0]) csr_times = {d: np.zeros(len(dimensionalities)) for d in densities} dense_times = {d: np.zeros(len(dimensionalities)) for d in densities} -transform = PolynomialFeatures(degree=degree, include_bias=False, - interaction_only=False) +transform = PolynomialFeatures( + degree=degree, include_bias=False, interaction_only=False +) for trial in range(trials): for density in densities: @@ -34,16 +37,22 @@ fig, axes = plt.subplots(nrows=len(densities), ncols=1, figsize=(8, 10)) for density, ax in zip(densities, axes): - - ax.plot(dimensionalities, csr_times[density] / trials, - label='csr', linestyle=csr_linestyle) - ax.plot(dimensionalities, dense_times[density] / trials, - label='dense', linestyle=dense_linestyle) - ax.set_title("density %0.2f, degree=%d, n_samples=%d" % - (density, degree, num_rows)) + ax.plot( + dimensionalities, + csr_times[density] / trials, + label="csr", + linestyle=csr_linestyle, + ) + ax.plot( + dimensionalities, + dense_times[density] / trials, + label="dense", + linestyle=dense_linestyle, + ) + ax.set_title("density %0.2f, degree=%d, n_samples=%d" % (density, degree, num_rows)) ax.legend() - ax.set_xlabel('Dimensionality') - ax.set_ylabel('Time (seconds)') + ax.set_xlabel("Dimensionality") + ax.set_ylabel("Time (seconds)") plt.tight_layout() plt.show() diff --git a/benchmarks/bench_glm.py b/benchmarks/bench_glm.py index afb9f0d3bb0f1..84cf31858afa7 100644 --- a/benchmarks/bench_glm.py +++ b/benchmarks/bench_glm.py @@ -4,13 +4,14 @@ Data comes from a random square matrix. """ + from datetime import datetime -import numpy as np -from sklearn import linear_model +import numpy as np -if __name__ == '__main__': +from sklearn import linear_model +if __name__ == "__main__": import matplotlib.pyplot as plt n_iter = 40 @@ -22,8 +23,7 @@ dimensions = 500 * np.arange(1, n_iter + 1) for i in range(n_iter): - - print('Iteration %s of %s' % (i, n_iter)) + print("Iteration %s of %s" % (i, n_iter)) n_samples, n_features = 10 * i + 3, 10 * i + 3 @@ -31,7 +31,7 @@ Y = np.random.randn(n_samples) start = datetime.now() - ridge = linear_model.Ridge(alpha=1.) + ridge = linear_model.Ridge(alpha=1.0) ridge.fit(X, Y) time_ridge[i] = (datetime.now() - start).total_seconds() @@ -45,13 +45,13 @@ lasso.fit(X, Y) time_lasso[i] = (datetime.now() - start).total_seconds() - plt.figure('scikit-learn GLM benchmark results') - plt.xlabel('Dimensions') - plt.ylabel('Time (s)') - plt.plot(dimensions, time_ridge, color='r') - plt.plot(dimensions, time_ols, color='g') - plt.plot(dimensions, time_lasso, color='b') + plt.figure("scikit-learn GLM benchmark results") + plt.xlabel("Dimensions") + plt.ylabel("Time (s)") + plt.plot(dimensions, time_ridge, color="r") + plt.plot(dimensions, time_ols, color="g") + plt.plot(dimensions, time_lasso, color="b") - plt.legend(['Ridge', 'OLS', 'LassoLars'], loc='upper left') - plt.axis('tight') + plt.legend(["Ridge", "OLS", "LassoLars"], loc="upper left") + plt.axis("tight") plt.show() diff --git a/benchmarks/bench_glmnet.py b/benchmarks/bench_glmnet.py index e8841cba46d57..1aaad99c10587 100644 --- a/benchmarks/bench_glmnet.py +++ b/benchmarks/bench_glmnet.py @@ -16,9 +16,12 @@ In both cases, only 10% of the features are informative. """ -import numpy as np + import gc from time import time + +import numpy as np + from sklearn.datasets import make_regression alpha = 0.1 @@ -35,7 +38,7 @@ def bench(factory, X, Y, X_test, Y_test, ref_coef): # start time tstart = time() clf = factory(alpha=alpha).fit(X, Y) - delta = (time() - tstart) + delta = time() - tstart # stop time print("duration: %0.3fs" % delta) @@ -44,11 +47,12 @@ def bench(factory, X, Y, X_test, Y_test, ref_coef): return delta -if __name__ == '__main__': - from glmnet.elastic_net import Lasso as GlmnetLasso - from sklearn.linear_model import Lasso as ScikitLasso +if __name__ == "__main__": # Delayed import of matplotlib.pyplot import matplotlib.pyplot as plt + from glmnet.elastic_net import Lasso as GlmnetLasso + + from sklearn.linear_model import Lasso as ScikitLasso scikit_results = [] glmnet_results = [] @@ -58,18 +62,22 @@ def bench(factory, X, Y, X_test, Y_test, ref_coef): n_informative = n_features / 10 n_test_samples = 1000 for i in range(1, n + 1): - print('==================') - print('Iteration %s of %s' % (i, n)) - print('==================') + print("==================") + print("Iteration %s of %s" % (i, n)) + print("==================") X, Y, coef_ = make_regression( - n_samples=(i * step) + n_test_samples, n_features=n_features, - noise=0.1, n_informative=n_informative, coef=True) + n_samples=(i * step) + n_test_samples, + n_features=n_features, + noise=0.1, + n_informative=n_informative, + coef=True, + ) X_test = X[-n_test_samples:] Y_test = Y[-n_test_samples:] - X = X[:(i * step)] - Y = Y[:(i * step)] + X = X[: (i * step)] + Y = Y[: (i * step)] print("benchmarking scikit-learn: ") scikit_results.append(bench(ScikitLasso, X, Y, X_test, Y_test, coef_)) @@ -78,12 +86,12 @@ def bench(factory, X, Y, X_test, Y_test, ref_coef): plt.clf() xx = range(0, n * step, step) - plt.title('Lasso regression on sample dataset (%d features)' % n_features) - plt.plot(xx, scikit_results, 'b-', label='scikit-learn') - plt.plot(xx, glmnet_results, 'r-', label='glmnet') + plt.title("Lasso regression on sample dataset (%d features)" % n_features) + plt.plot(xx, scikit_results, "b-", label="scikit-learn") + plt.plot(xx, glmnet_results, "r-", label="glmnet") plt.legend() - plt.xlabel('number of samples to classify') - plt.ylabel('Time (s)') + plt.xlabel("number of samples to classify") + plt.ylabel("Time (s)") plt.show() # now do a benchmark where the number of points is fixed @@ -96,15 +104,19 @@ def bench(factory, X, Y, X_test, Y_test, ref_coef): n_samples = 500 for i in range(1, n + 1): - print('==================') - print('Iteration %02d of %02d' % (i, n)) - print('==================') + print("==================") + print("Iteration %02d of %02d" % (i, n)) + print("==================") n_features = i * step n_informative = n_features / 10 X, Y, coef_ = make_regression( - n_samples=(i * step) + n_test_samples, n_features=n_features, - noise=0.1, n_informative=n_informative, coef=True) + n_samples=(i * step) + n_test_samples, + n_features=n_features, + noise=0.1, + n_informative=n_informative, + coef=True, + ) X_test = X[-n_test_samples:] Y_test = Y[-n_test_samples:] @@ -117,12 +129,12 @@ def bench(factory, X, Y, X_test, Y_test, ref_coef): glmnet_results.append(bench(GlmnetLasso, X, Y, X_test, Y_test, coef_)) xx = np.arange(100, 100 + n * step, step) - plt.figure('scikit-learn vs. glmnet benchmark results') - plt.title('Regression in high dimensional spaces (%d samples)' % n_samples) - plt.plot(xx, scikit_results, 'b-', label='scikit-learn') - plt.plot(xx, glmnet_results, 'r-', label='glmnet') + plt.figure("scikit-learn vs. glmnet benchmark results") + plt.title("Regression in high dimensional spaces (%d samples)" % n_samples) + plt.plot(xx, scikit_results, "b-", label="scikit-learn") + plt.plot(xx, glmnet_results, "r-", label="glmnet") plt.legend() - plt.xlabel('number of features') - plt.ylabel('Time (s)') - plt.axis('tight') + plt.xlabel("number of features") + plt.ylabel("Time (s)") + plt.axis("tight") plt.show() diff --git a/benchmarks/bench_hist_gradient_boosting.py b/benchmarks/bench_hist_gradient_boosting.py index 158b6fbb22d2b..c1dfffabe71c2 100644 --- a/benchmarks/bench_hist_gradient_boosting.py +++ b/benchmarks/bench_hist_gradient_boosting.py @@ -1,40 +1,48 @@ -from time import time import argparse +from time import time import matplotlib.pyplot as plt import numpy as np -from sklearn.model_selection import train_test_split -# To use this experimental feature, we need to explicitly ask for it: -from sklearn.experimental import enable_hist_gradient_boosting # noqa -from sklearn.ensemble import HistGradientBoostingRegressor -from sklearn.ensemble import HistGradientBoostingClassifier -from sklearn.datasets import make_classification -from sklearn.datasets import make_regression -from sklearn.ensemble._hist_gradient_boosting.utils import ( - get_equivalent_estimator) +from sklearn.datasets import make_classification, make_regression +from sklearn.ensemble import ( + HistGradientBoostingClassifier, + HistGradientBoostingRegressor, +) +from sklearn.ensemble._hist_gradient_boosting.utils import get_equivalent_estimator +from sklearn.model_selection import train_test_split parser = argparse.ArgumentParser() -parser.add_argument('--n-leaf-nodes', type=int, default=31) -parser.add_argument('--n-trees', type=int, default=10) -parser.add_argument('--lightgbm', action="store_true", default=False, - help='also plot lightgbm') -parser.add_argument('--xgboost', action="store_true", default=False, - help='also plot xgboost') -parser.add_argument('--catboost', action="store_true", default=False, - help='also plot catboost') -parser.add_argument('--learning-rate', type=float, default=.1) -parser.add_argument('--problem', type=str, default='classification', - choices=['classification', 'regression']) -parser.add_argument('--loss', type=str, default='default') -parser.add_argument('--missing-fraction', type=float, default=0) -parser.add_argument('--n-classes', type=int, default=2) -parser.add_argument('--n-samples-max', type=int, default=int(1e6)) -parser.add_argument('--n-features', type=int, default=20) -parser.add_argument('--max-bins', type=int, default=255) -parser.add_argument('--random-sample-weights', action="store_true", - default=False, - help="generate and use random sample weights") +parser.add_argument("--n-leaf-nodes", type=int, default=31) +parser.add_argument("--n-trees", type=int, default=10) +parser.add_argument( + "--lightgbm", action="store_true", default=False, help="also plot lightgbm" +) +parser.add_argument( + "--xgboost", action="store_true", default=False, help="also plot xgboost" +) +parser.add_argument( + "--catboost", action="store_true", default=False, help="also plot catboost" +) +parser.add_argument("--learning-rate", type=float, default=0.1) +parser.add_argument( + "--problem", + type=str, + default="classification", + choices=["classification", "regression"], +) +parser.add_argument("--loss", type=str, default="default") +parser.add_argument("--missing-fraction", type=float, default=0) +parser.add_argument("--n-classes", type=int, default=2) +parser.add_argument("--n-samples-max", type=int, default=int(1e6)) +parser.add_argument("--n-features", type=int, default=20) +parser.add_argument("--max-bins", type=int, default=255) +parser.add_argument( + "--random-sample-weights", + action="store_true", + default=False, + help="generate and use random sample weights", +) args = parser.parse_args() n_leaf_nodes = args.n_leaf_nodes @@ -44,24 +52,26 @@ def get_estimator_and_data(): - if args.problem == 'classification': - X, y = make_classification(args.n_samples_max * 2, - n_features=args.n_features, - n_classes=args.n_classes, - n_clusters_per_class=1, - n_informative=args.n_classes, - random_state=0) + if args.problem == "classification": + X, y = make_classification( + args.n_samples_max * 2, + n_features=args.n_features, + n_classes=args.n_classes, + n_clusters_per_class=1, + n_informative=args.n_classes, + random_state=0, + ) return X, y, HistGradientBoostingClassifier - elif args.problem == 'regression': - X, y = make_regression(args.n_samples_max * 2, - n_features=args.n_features, random_state=0) + elif args.problem == "regression": + X, y = make_regression( + args.n_samples_max * 2, n_features=args.n_features, random_state=0 + ) return X, y, HistGradientBoostingRegressor X, y, Estimator = get_estimator_and_data() if args.missing_fraction: - mask = np.random.binomial(1, args.missing_fraction, size=X.shape).astype( - bool) + mask = np.random.binomial(1, args.missing_fraction, size=X.shape).astype(bool) X[mask] = np.nan if args.random_sample_weights: @@ -70,12 +80,13 @@ def get_estimator_and_data(): sample_weight = None if sample_weight is not None: - (X_train_, X_test_, y_train_, y_test_, - sample_weight_train_, _) = train_test_split( - X, y, sample_weight, test_size=0.5, random_state=0) + (X_train_, X_test_, y_train_, y_test_, sample_weight_train_, _) = train_test_split( + X, y, sample_weight, test_size=0.5, random_state=0 + ) else: X_train_, X_test_, y_train_, y_test_ = train_test_split( - X, y, test_size=0.5, random_state=0) + X, y, test_size=0.5, random_state=0 + ) sample_weight_train_ = None @@ -90,27 +101,26 @@ def one_run(n_samples): sample_weight_train = None assert X_train.shape[0] == n_samples assert X_test.shape[0] == n_samples - print("Data size: %d samples train, %d samples test." - % (n_samples, n_samples)) + print("Data size: %d samples train, %d samples test." % (n_samples, n_samples)) print("Fitting a sklearn model...") tic = time() - est = Estimator(learning_rate=lr, - max_iter=n_trees, - max_bins=max_bins, - max_leaf_nodes=n_leaf_nodes, - early_stopping=False, - random_state=0, - verbose=0) + est = Estimator( + learning_rate=lr, + max_iter=n_trees, + max_bins=max_bins, + max_leaf_nodes=n_leaf_nodes, + early_stopping=False, + random_state=0, + verbose=0, + ) loss = args.loss - if args.problem == 'classification': - if loss == 'default': - # loss='auto' does not work with get_equivalent_estimator() - loss = 'binary_crossentropy' if args.n_classes == 2 else \ - 'categorical_crossentropy' + if args.problem == "classification": + if loss == "default": + loss = "log_loss" else: # regression - if loss == 'default': - loss = 'least_squares' + if loss == "default": + loss = "squared_error" est.set_params(loss=loss) est.fit(X_train, y_train, sample_weight=sample_weight_train) sklearn_fit_duration = time() - tic @@ -126,7 +136,9 @@ def one_run(n_samples): lightgbm_score_duration = None if args.lightgbm: print("Fitting a LightGBM model...") - lightgbm_est = get_equivalent_estimator(est, lib='lightgbm') + lightgbm_est = get_equivalent_estimator( + est, lib="lightgbm", n_classes=args.n_classes + ) tic = time() lightgbm_est.fit(X_train, y_train, sample_weight=sample_weight_train) @@ -143,7 +155,7 @@ def one_run(n_samples): xgb_score_duration = None if args.xgboost: print("Fitting an XGBoost model...") - xgb_est = get_equivalent_estimator(est, lib='xgboost') + xgb_est = get_equivalent_estimator(est, lib="xgboost", n_classes=args.n_classes) tic = time() xgb_est.fit(X_train, y_train, sample_weight=sample_weight_train) @@ -160,7 +172,9 @@ def one_run(n_samples): cat_score_duration = None if args.catboost: print("Fitting a CatBoost model...") - cat_est = get_equivalent_estimator(est, lib='catboost') + cat_est = get_equivalent_estimator( + est, lib="catboost", n_classes=args.n_classes + ) tic = time() cat_est.fit(X_train, y_train, sample_weight=sample_weight_train) @@ -172,15 +186,26 @@ def one_run(n_samples): print("fit duration: {:.3f}s,".format(cat_fit_duration)) print("score duration: {:.3f}s,".format(cat_score_duration)) - return (sklearn_score, sklearn_fit_duration, sklearn_score_duration, - lightgbm_score, lightgbm_fit_duration, lightgbm_score_duration, - xgb_score, xgb_fit_duration, xgb_score_duration, - cat_score, cat_fit_duration, cat_score_duration) + return ( + sklearn_score, + sklearn_fit_duration, + sklearn_score_duration, + lightgbm_score, + lightgbm_fit_duration, + lightgbm_score_duration, + xgb_score, + xgb_fit_duration, + xgb_score_duration, + cat_score, + cat_fit_duration, + cat_score_duration, + ) n_samples_list = [1000, 10000, 100000, 500000, 1000000, 5000000, 10000000] -n_samples_list = [n_samples for n_samples in n_samples_list - if n_samples <= args.n_samples_max] +n_samples_list = [ + n_samples for n_samples in n_samples_list if n_samples <= args.n_samples_max +] sklearn_scores = [] sklearn_fit_durations = [] @@ -196,67 +221,70 @@ def one_run(n_samples): cat_score_durations = [] for n_samples in n_samples_list: - (sklearn_score, - sklearn_fit_duration, - sklearn_score_duration, - lightgbm_score, - lightgbm_fit_duration, - lightgbm_score_duration, - xgb_score, - xgb_fit_duration, - xgb_score_duration, - cat_score, - cat_fit_duration, - cat_score_duration) = one_run(n_samples) + ( + sklearn_score, + sklearn_fit_duration, + sklearn_score_duration, + lightgbm_score, + lightgbm_fit_duration, + lightgbm_score_duration, + xgb_score, + xgb_fit_duration, + xgb_score_duration, + cat_score, + cat_fit_duration, + cat_score_duration, + ) = one_run(n_samples) for scores, score in ( - (sklearn_scores, sklearn_score), - (sklearn_fit_durations, sklearn_fit_duration), - (sklearn_score_durations, sklearn_score_duration), - (lightgbm_scores, lightgbm_score), - (lightgbm_fit_durations, lightgbm_fit_duration), - (lightgbm_score_durations, lightgbm_score_duration), - (xgb_scores, xgb_score), - (xgb_fit_durations, xgb_fit_duration), - (xgb_score_durations, xgb_score_duration), - (cat_scores, cat_score), - (cat_fit_durations, cat_fit_duration), - (cat_score_durations, cat_score_duration)): + (sklearn_scores, sklearn_score), + (sklearn_fit_durations, sklearn_fit_duration), + (sklearn_score_durations, sklearn_score_duration), + (lightgbm_scores, lightgbm_score), + (lightgbm_fit_durations, lightgbm_fit_duration), + (lightgbm_score_durations, lightgbm_score_duration), + (xgb_scores, xgb_score), + (xgb_fit_durations, xgb_fit_duration), + (xgb_score_durations, xgb_score_duration), + (cat_scores, cat_score), + (cat_fit_durations, cat_fit_duration), + (cat_score_durations, cat_score_duration), + ): scores.append(score) fig, axs = plt.subplots(3, sharex=True) -axs[0].plot(n_samples_list, sklearn_scores, label='sklearn') -axs[1].plot(n_samples_list, sklearn_fit_durations, label='sklearn') -axs[2].plot(n_samples_list, sklearn_score_durations, label='sklearn') +axs[0].plot(n_samples_list, sklearn_scores, label="sklearn") +axs[1].plot(n_samples_list, sklearn_fit_durations, label="sklearn") +axs[2].plot(n_samples_list, sklearn_score_durations, label="sklearn") if args.lightgbm: - axs[0].plot(n_samples_list, lightgbm_scores, label='lightgbm') - axs[1].plot(n_samples_list, lightgbm_fit_durations, label='lightgbm') - axs[2].plot(n_samples_list, lightgbm_score_durations, label='lightgbm') + axs[0].plot(n_samples_list, lightgbm_scores, label="lightgbm") + axs[1].plot(n_samples_list, lightgbm_fit_durations, label="lightgbm") + axs[2].plot(n_samples_list, lightgbm_score_durations, label="lightgbm") if args.xgboost: - axs[0].plot(n_samples_list, xgb_scores, label='XGBoost') - axs[1].plot(n_samples_list, xgb_fit_durations, label='XGBoost') - axs[2].plot(n_samples_list, xgb_score_durations, label='XGBoost') + axs[0].plot(n_samples_list, xgb_scores, label="XGBoost") + axs[1].plot(n_samples_list, xgb_fit_durations, label="XGBoost") + axs[2].plot(n_samples_list, xgb_score_durations, label="XGBoost") if args.catboost: - axs[0].plot(n_samples_list, cat_scores, label='CatBoost') - axs[1].plot(n_samples_list, cat_fit_durations, label='CatBoost') - axs[2].plot(n_samples_list, cat_score_durations, label='CatBoost') + axs[0].plot(n_samples_list, cat_scores, label="CatBoost") + axs[1].plot(n_samples_list, cat_fit_durations, label="CatBoost") + axs[2].plot(n_samples_list, cat_score_durations, label="CatBoost") for ax in axs: - ax.set_xscale('log') - ax.legend(loc='best') - ax.set_xlabel('n_samples') + ax.set_xscale("log") + ax.legend(loc="best") + ax.set_xlabel("n_samples") -axs[0].set_title('scores') -axs[1].set_title('fit duration (s)') -axs[2].set_title('score duration (s)') +axs[0].set_title("scores") +axs[1].set_title("fit duration (s)") +axs[2].set_title("score duration (s)") title = args.problem -if args.problem == 'classification': - title += ' n_classes = {}'.format(args.n_classes) +if args.problem == "classification": + title += " n_classes = {}".format(args.n_classes) fig.suptitle(title) diff --git a/benchmarks/bench_hist_gradient_boosting_adult.py b/benchmarks/bench_hist_gradient_boosting_adult.py index 5b47fcb3a6678..4d5ce48cded81 100644 --- a/benchmarks/bench_hist_gradient_boosting_adult.py +++ b/benchmarks/bench_hist_gradient_boosting_adult.py @@ -1,23 +1,25 @@ import argparse from time import time -from sklearn.model_selection import train_test_split +import numpy as np +import pandas as pd + +from sklearn.compose import make_column_selector, make_column_transformer from sklearn.datasets import fetch_openml -from sklearn.metrics import accuracy_score, roc_auc_score -from sklearn.experimental import enable_hist_gradient_boosting # noqa from sklearn.ensemble import HistGradientBoostingClassifier -from sklearn.ensemble._hist_gradient_boosting.utils import ( - get_equivalent_estimator) - +from sklearn.ensemble._hist_gradient_boosting.utils import get_equivalent_estimator +from sklearn.metrics import accuracy_score, roc_auc_score +from sklearn.model_selection import train_test_split +from sklearn.preprocessing import OrdinalEncoder parser = argparse.ArgumentParser() -parser.add_argument('--n-leaf-nodes', type=int, default=31) -parser.add_argument('--n-trees', type=int, default=100) -parser.add_argument('--lightgbm', action="store_true", default=False) -parser.add_argument('--learning-rate', type=float, default=.1) -parser.add_argument('--max-bins', type=int, default=255) -parser.add_argument('--no-predict', action="store_true", default=False) -parser.add_argument('--verbose', action="store_true", default=False) +parser.add_argument("--n-leaf-nodes", type=int, default=31) +parser.add_argument("--n-trees", type=int, default=100) +parser.add_argument("--lightgbm", action="store_true", default=False) +parser.add_argument("--learning-rate", type=float, default=0.1) +parser.add_argument("--max-bins", type=int, default=255) +parser.add_argument("--no-predict", action="store_true", default=False) +parser.add_argument("--verbose", action="store_true", default=False) args = parser.parse_args() n_leaf_nodes = args.n_leaf_nodes @@ -44,28 +46,37 @@ def predict(est, data_test, target_test): toc = time() roc_auc = roc_auc_score(target_test, predicted_proba_test[:, 1]) acc = accuracy_score(target_test, predicted_test) - print(f"predicted in {toc - tic:.3f}s, " - f"ROC AUC: {roc_auc:.4f}, ACC: {acc :.4f}") + print(f"predicted in {toc - tic:.3f}s, ROC AUC: {roc_auc:.4f}, ACC: {acc:.4f}") -data = fetch_openml(data_id=179, as_frame=False) # adult dataset +data = fetch_openml(data_id=179, as_frame=True) # adult dataset X, y = data.data, data.target +# Ordinal encode the categories to use the native support available in HGBDT +cat_columns = make_column_selector(dtype_include="category")(X) +preprocessing = make_column_transformer( + (OrdinalEncoder(), cat_columns), + remainder="passthrough", + verbose_feature_names_out=False, +) +X = pd.DataFrame( + preprocessing.fit_transform(X), + columns=preprocessing.get_feature_names_out(), +) + +n_classes = len(np.unique(y)) n_features = X.shape[1] -n_categorical_features = len(data.categories) +n_categorical_features = len(cat_columns) n_numerical_features = n_features - n_categorical_features print(f"Number of features: {n_features}") print(f"Number of categorical features: {n_categorical_features}") print(f"Number of numerical features: {n_numerical_features}") -X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, - random_state=0) +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0) -# Note: no need to use an OrdinalEncoder because categorical features are -# already clean -is_categorical = [name in data.categories for name in data.feature_names] +is_categorical = [True] * n_categorical_features + [False] * n_numerical_features est = HistGradientBoostingClassifier( - loss='binary_crossentropy', + loss="log_loss", learning_rate=lr, max_iter=n_trees, max_bins=max_bins, @@ -73,18 +84,17 @@ def predict(est, data_test, target_test): categorical_features=is_categorical, early_stopping=False, random_state=0, - verbose=verbose + verbose=verbose, ) -fit(est, X_train, y_train, 'sklearn') +fit(est, X_train, y_train, "sklearn") predict(est, X_test, y_test) if args.lightgbm: - est = get_equivalent_estimator(est, lib='lightgbm') + est = get_equivalent_estimator(est, lib="lightgbm", n_classes=n_classes) est.set_params(max_cat_to_onehot=1) # dont use OHE - categorical_features = [f_idx - for (f_idx, is_cat) in enumerate(is_categorical) - if is_cat] - fit(est, X_train, y_train, 'lightgbm', - categorical_feature=categorical_features) + categorical_features = [ + f_idx for (f_idx, is_cat) in enumerate(is_categorical) if is_cat + ] + fit(est, X_train, y_train, "lightgbm", categorical_feature=categorical_features) predict(est, X_test, y_test) diff --git a/benchmarks/bench_hist_gradient_boosting_categorical_only.py b/benchmarks/bench_hist_gradient_boosting_categorical_only.py index 6c69b32eff26f..1085bbc49f4f8 100644 --- a/benchmarks/bench_hist_gradient_boosting_categorical_only.py +++ b/benchmarks/bench_hist_gradient_boosting_categorical_only.py @@ -1,25 +1,22 @@ import argparse from time import time -from sklearn.preprocessing import KBinsDiscretizer from sklearn.datasets import make_classification -from sklearn.experimental import enable_hist_gradient_boosting # noqa from sklearn.ensemble import HistGradientBoostingClassifier -from sklearn.ensemble._hist_gradient_boosting.utils import ( - get_equivalent_estimator) - +from sklearn.ensemble._hist_gradient_boosting.utils import get_equivalent_estimator +from sklearn.preprocessing import KBinsDiscretizer parser = argparse.ArgumentParser() -parser.add_argument('--n-leaf-nodes', type=int, default=31) -parser.add_argument('--n-trees', type=int, default=100) -parser.add_argument('--n-features', type=int, default=20) -parser.add_argument('--n-cats', type=int, default=20) -parser.add_argument('--n-samples', type=int, default=10_000) -parser.add_argument('--lightgbm', action="store_true", default=False) -parser.add_argument('--learning-rate', type=float, default=.1) -parser.add_argument('--max-bins', type=int, default=255) -parser.add_argument('--no-predict', action="store_true", default=False) -parser.add_argument('--verbose', action="store_true", default=False) +parser.add_argument("--n-leaf-nodes", type=int, default=31) +parser.add_argument("--n-trees", type=int, default=100) +parser.add_argument("--n-features", type=int, default=20) +parser.add_argument("--n-cats", type=int, default=20) +parser.add_argument("--n-samples", type=int, default=10_000) +parser.add_argument("--lightgbm", action="store_true", default=False) +parser.add_argument("--learning-rate", type=float, default=0.1) +parser.add_argument("--max-bins", type=int, default=255) +parser.add_argument("--no-predict", action="store_true", default=False) +parser.add_argument("--verbose", action="store_true", default=False) args = parser.parse_args() n_leaf_nodes = args.n_leaf_nodes @@ -51,17 +48,16 @@ def predict(est, data_test): print(f"predicted in {toc - tic:.3f}s") -X, y = make_classification(n_samples=n_samples, n_features=n_features, - random_state=0) +X, y = make_classification(n_samples=n_samples, n_features=n_features, random_state=0) -X = KBinsDiscretizer(n_bins=n_categories, encode='ordinal').fit_transform(X) +X = KBinsDiscretizer(n_bins=n_categories, encode="ordinal").fit_transform(X) print(f"Number of features: {n_features}") print(f"Number of samples: {n_samples}") is_categorical = [True] * n_features est = HistGradientBoostingClassifier( - loss='binary_crossentropy', + loss="log_loss", learning_rate=lr, max_iter=n_trees, max_bins=max_bins, @@ -69,16 +65,15 @@ def predict(est, data_test): categorical_features=is_categorical, early_stopping=False, random_state=0, - verbose=verbose + verbose=verbose, ) -fit(est, X, y, 'sklearn') +fit(est, X, y, "sklearn") predict(est, X) if args.lightgbm: - est = get_equivalent_estimator(est, lib='lightgbm') + est = get_equivalent_estimator(est, lib="lightgbm", n_classes=2) est.set_params(max_cat_to_onehot=1) # dont use OHE categorical_features = list(range(n_features)) - fit(est, X, y, 'lightgbm', - categorical_feature=categorical_features) + fit(est, X, y, "lightgbm", categorical_feature=categorical_features) predict(est, X) diff --git a/benchmarks/bench_hist_gradient_boosting_higgsboson.py b/benchmarks/bench_hist_gradient_boosting_higgsboson.py index 2c74bb8818343..ceab576bc0a52 100644 --- a/benchmarks/bench_hist_gradient_boosting_higgsboson.py +++ b/benchmarks/bench_hist_gradient_boosting_higgsboson.py @@ -1,49 +1,48 @@ -from urllib.request import urlretrieve +import argparse import os from gzip import GzipFile from time import time -import argparse +from urllib.request import urlretrieve import numpy as np import pandas as pd from joblib import Memory -from sklearn.model_selection import train_test_split -from sklearn.metrics import accuracy_score, roc_auc_score -# To use this experimental feature, we need to explicitly ask for it: -from sklearn.experimental import enable_hist_gradient_boosting # noqa -from sklearn.ensemble import HistGradientBoostingClassifier -from sklearn.ensemble._hist_gradient_boosting.utils import ( - get_equivalent_estimator) +from sklearn.ensemble import HistGradientBoostingClassifier +from sklearn.ensemble._hist_gradient_boosting.utils import get_equivalent_estimator +from sklearn.metrics import accuracy_score, roc_auc_score +from sklearn.model_selection import train_test_split parser = argparse.ArgumentParser() -parser.add_argument('--n-leaf-nodes', type=int, default=31) -parser.add_argument('--n-trees', type=int, default=10) -parser.add_argument('--lightgbm', action="store_true", default=False) -parser.add_argument('--xgboost', action="store_true", default=False) -parser.add_argument('--catboost', action="store_true", default=False) -parser.add_argument('--learning-rate', type=float, default=1.) -parser.add_argument('--subsample', type=int, default=None) -parser.add_argument('--max-bins', type=int, default=255) -parser.add_argument('--no-predict', action="store_true", default=False) -parser.add_argument('--cache-loc', type=str, default='/tmp') +parser.add_argument("--n-leaf-nodes", type=int, default=31) +parser.add_argument("--n-trees", type=int, default=10) +parser.add_argument("--lightgbm", action="store_true", default=False) +parser.add_argument("--xgboost", action="store_true", default=False) +parser.add_argument("--catboost", action="store_true", default=False) +parser.add_argument("--learning-rate", type=float, default=1.0) +parser.add_argument("--subsample", type=int, default=None) +parser.add_argument("--max-bins", type=int, default=255) +parser.add_argument("--no-predict", action="store_true", default=False) +parser.add_argument("--cache-loc", type=str, default="/tmp") +parser.add_argument("--no-interactions", type=bool, default=False) +parser.add_argument("--max-features", type=float, default=1.0) args = parser.parse_args() HERE = os.path.dirname(__file__) -URL = ("https://archive.ics.uci.edu/ml/machine-learning-databases/00280/" - "HIGGS.csv.gz") -m = Memory(location=args.cache_loc, mmap_mode='r') +URL = "https://archive.ics.uci.edu/ml/machine-learning-databases/00280/HIGGS.csv.gz" +m = Memory(location=args.cache_loc, mmap_mode="r") n_leaf_nodes = args.n_leaf_nodes n_trees = args.n_trees subsample = args.subsample lr = args.learning_rate max_bins = args.max_bins +max_features = args.max_features @m.cache def load_data(): - filename = os.path.join(HERE, URL.rsplit('/', 1)[-1]) + filename = os.path.join(HERE, URL.rsplit("/", 1)[-1]) if not os.path.exists(filename): print(f"Downloading {URL} to {filename} (2.6 GB)...") urlretrieve(URL, filename) @@ -75,15 +74,16 @@ def predict(est, data_test, target_test): toc = time() roc_auc = roc_auc_score(target_test, predicted_proba_test[:, 1]) acc = accuracy_score(target_test, predicted_test) - print(f"predicted in {toc - tic:.3f}s, " - f"ROC AUC: {roc_auc:.4f}, ACC: {acc :.4f}") + print(f"predicted in {toc - tic:.3f}s, ROC AUC: {roc_auc:.4f}, ACC: {acc:.4f}") df = load_data() target = df.values[:, 0] data = np.ascontiguousarray(df.values[:, 1:]) data_train, data_test, target_train, target_test = train_test_split( - data, target, test_size=.2, random_state=0) + data, target, test_size=0.2, random_state=0 +) +n_classes = len(np.unique(target)) if subsample is not None: data_train, target_train = data_train[:subsample], target_train[:subsample] @@ -91,28 +91,37 @@ def predict(est, data_test, target_test): n_samples, n_features = data_train.shape print(f"Training set with {n_samples} records with {n_features} features.") -est = HistGradientBoostingClassifier(loss='binary_crossentropy', - learning_rate=lr, - max_iter=n_trees, - max_bins=max_bins, - max_leaf_nodes=n_leaf_nodes, - early_stopping=False, - random_state=0, - verbose=1) -fit(est, data_train, target_train, 'sklearn') +if args.no_interactions: + interaction_cst = [[i] for i in range(n_features)] +else: + interaction_cst = None + +est = HistGradientBoostingClassifier( + loss="log_loss", + learning_rate=lr, + max_iter=n_trees, + max_bins=max_bins, + max_leaf_nodes=n_leaf_nodes, + early_stopping=False, + random_state=0, + verbose=1, + interaction_cst=interaction_cst, + max_features=max_features, +) +fit(est, data_train, target_train, "sklearn") predict(est, data_test, target_test) if args.lightgbm: - est = get_equivalent_estimator(est, lib='lightgbm') - fit(est, data_train, target_train, 'lightgbm') + est = get_equivalent_estimator(est, lib="lightgbm", n_classes=n_classes) + fit(est, data_train, target_train, "lightgbm") predict(est, data_test, target_test) if args.xgboost: - est = get_equivalent_estimator(est, lib='xgboost') - fit(est, data_train, target_train, 'xgboost') + est = get_equivalent_estimator(est, lib="xgboost", n_classes=n_classes) + fit(est, data_train, target_train, "xgboost") predict(est, data_test, target_test) if args.catboost: - est = get_equivalent_estimator(est, lib='catboost') - fit(est, data_train, target_train, 'catboost') + est = get_equivalent_estimator(est, lib="catboost", n_classes=n_classes) + fit(est, data_train, target_train, "catboost") predict(est, data_test, target_test) diff --git a/benchmarks/bench_hist_gradient_boosting_threading.py b/benchmarks/bench_hist_gradient_boosting_threading.py index 3cc6afa3871c6..9acf65bdbaf6a 100644 --- a/benchmarks/bench_hist_gradient_boosting_threading.py +++ b/benchmarks/bench_hist_gradient_boosting_threading.py @@ -1,49 +1,59 @@ -from time import time import argparse import os from pprint import pprint +from time import time import numpy as np from threadpoolctl import threadpool_limits + import sklearn +from sklearn.datasets import make_classification, make_regression +from sklearn.ensemble import ( + HistGradientBoostingClassifier, + HistGradientBoostingRegressor, +) +from sklearn.ensemble._hist_gradient_boosting.utils import get_equivalent_estimator from sklearn.model_selection import train_test_split -# To use this experimental feature, we need to explicitly ask for it: -from sklearn.experimental import enable_hist_gradient_boosting # noqa -from sklearn.ensemble import HistGradientBoostingRegressor -from sklearn.ensemble import HistGradientBoostingClassifier -from sklearn.datasets import make_classification -from sklearn.datasets import make_regression -from sklearn.ensemble._hist_gradient_boosting.utils import ( - get_equivalent_estimator) - parser = argparse.ArgumentParser() -parser.add_argument('--n-leaf-nodes', type=int, default=31) -parser.add_argument('--n-trees', type=int, default=10) -parser.add_argument('--lightgbm', action="store_true", default=False, - help='also benchmark lightgbm') -parser.add_argument('--xgboost', action="store_true", default=False, - help='also benchmark xgboost') -parser.add_argument('--catboost', action="store_true", default=False, - help='also benchmark catboost') -parser.add_argument('--learning-rate', type=float, default=.1) -parser.add_argument('--problem', type=str, default='classification', - choices=['classification', 'regression']) -parser.add_argument('--loss', type=str, default='default') -parser.add_argument('--missing-fraction', type=float, default=0) -parser.add_argument('--n-classes', type=int, default=2) -parser.add_argument('--n-samples', type=int, default=int(1e6)) -parser.add_argument('--n-features', type=int, default=100) -parser.add_argument('--max-bins', type=int, default=255) - -parser.add_argument('--print-params', action="store_true", default=False) -parser.add_argument('--random-sample-weights', action="store_true", - default=False, - help="generate and use random sample weights") -parser.add_argument('--plot', action="store_true", default=False, - help='show a plot results') -parser.add_argument('--plot-filename', default=None, - help='filename to save the figure to disk') +parser.add_argument("--n-leaf-nodes", type=int, default=31) +parser.add_argument("--n-trees", type=int, default=10) +parser.add_argument( + "--lightgbm", action="store_true", default=False, help="also benchmark lightgbm" +) +parser.add_argument( + "--xgboost", action="store_true", default=False, help="also benchmark xgboost" +) +parser.add_argument( + "--catboost", action="store_true", default=False, help="also benchmark catboost" +) +parser.add_argument("--learning-rate", type=float, default=0.1) +parser.add_argument( + "--problem", + type=str, + default="classification", + choices=["classification", "regression"], +) +parser.add_argument("--loss", type=str, default="default") +parser.add_argument("--missing-fraction", type=float, default=0) +parser.add_argument("--n-classes", type=int, default=2) +parser.add_argument("--n-samples", type=int, default=int(1e6)) +parser.add_argument("--n-features", type=int, default=100) +parser.add_argument("--max-bins", type=int, default=255) + +parser.add_argument("--print-params", action="store_true", default=False) +parser.add_argument( + "--random-sample-weights", + action="store_true", + default=False, + help="generate and use random sample weights", +) +parser.add_argument( + "--plot", action="store_true", default=False, help="show a plot results" +) +parser.add_argument( + "--plot-filename", default=None, help="filename to save the figure to disk" +) args = parser.parse_args() n_samples = args.n_samples @@ -53,30 +63,31 @@ max_bins = args.max_bins -print("Data size: %d samples train, %d samples test." - % (n_samples, n_samples)) +print("Data size: %d samples train, %d samples test." % (n_samples, n_samples)) print(f"n_features: {args.n_features}") def get_estimator_and_data(): - if args.problem == 'classification': - X, y = make_classification(args.n_samples * 2, - n_features=args.n_features, - n_classes=args.n_classes, - n_clusters_per_class=1, - n_informative=args.n_features // 2, - random_state=0) + if args.problem == "classification": + X, y = make_classification( + args.n_samples * 2, + n_features=args.n_features, + n_classes=args.n_classes, + n_clusters_per_class=1, + n_informative=args.n_features // 2, + random_state=0, + ) return X, y, HistGradientBoostingClassifier - elif args.problem == 'regression': - X, y = make_regression(args.n_samples_max * 2, - n_features=args.n_features, random_state=0) + elif args.problem == "regression": + X, y = make_regression( + args.n_samples_max * 2, n_features=args.n_features, random_state=0 + ) return X, y, HistGradientBoostingRegressor X, y, Estimator = get_estimator_and_data() if args.missing_fraction: - mask = np.random.binomial(1, args.missing_fraction, size=X.shape).astype( - bool) + mask = np.random.binomial(1, args.missing_fraction, size=X.shape).astype(bool) X[mask] = np.nan if args.random_sample_weights: @@ -85,12 +96,13 @@ def get_estimator_and_data(): sample_weight = None if sample_weight is not None: - (X_train_, X_test_, y_train_, y_test_, - sample_weight_train_, _) = train_test_split( - X, y, sample_weight, test_size=0.5, random_state=0) + (X_train_, X_test_, y_train_, y_test_, sample_weight_train_, _) = train_test_split( + X, y, sample_weight, test_size=0.5, random_state=0 + ) else: X_train_, X_test_, y_train_, y_test_ = train_test_split( - X, y, test_size=0.5, random_state=0) + X, y, test_size=0.5, random_state=0 + ) sample_weight_train_ = None @@ -104,15 +116,14 @@ def get_estimator_and_data(): verbose=0, ) loss = args.loss -if args.problem == 'classification': - if loss == 'default': +if args.problem == "classification": + if loss == "default": # loss='auto' does not work with get_equivalent_estimator() - loss = 'binary_crossentropy' if args.n_classes == 2 else \ - 'categorical_crossentropy' + loss = "log_loss" else: # regression - if loss == 'default': - loss = 'least_squares' + if loss == "default": + loss = "squared_error" sklearn_est.set_params(loss=loss) @@ -123,7 +134,9 @@ def get_estimator_and_data(): for libname in ["lightgbm", "xgboost", "catboost"]: if getattr(args, libname): print(libname) - est = get_equivalent_estimator(sklearn_est, lib=libname) + est = get_equivalent_estimator( + sklearn_est, lib=libname, n_classes=args.n_classes + ) pprint(est.get_params()) @@ -157,7 +170,9 @@ def one_run(n_threads, n_samples): lightgbm_score_duration = None if args.lightgbm: print("Fitting a LightGBM model...") - lightgbm_est = get_equivalent_estimator(est, lib='lightgbm') + lightgbm_est = get_equivalent_estimator( + est, lib="lightgbm", n_classes=args.n_classes + ) lightgbm_est.set_params(num_threads=n_threads) tic = time() @@ -175,7 +190,7 @@ def one_run(n_threads, n_samples): xgb_score_duration = None if args.xgboost: print("Fitting an XGBoost model...") - xgb_est = get_equivalent_estimator(est, lib='xgboost') + xgb_est = get_equivalent_estimator(est, lib="xgboost", n_classes=args.n_classes) xgb_est.set_params(nthread=n_threads) tic = time() @@ -193,7 +208,9 @@ def one_run(n_threads, n_samples): cat_score_duration = None if args.catboost: print("Fitting a CatBoost model...") - cat_est = get_equivalent_estimator(est, lib='catboost') + cat_est = get_equivalent_estimator( + est, lib="catboost", n_classes=args.n_classes + ) cat_est.set_params(thread_count=n_threads) tic = time() @@ -206,14 +223,24 @@ def one_run(n_threads, n_samples): print("fit duration: {:.3f}s,".format(cat_fit_duration)) print("score duration: {:.3f}s,".format(cat_score_duration)) - return (sklearn_score, sklearn_fit_duration, sklearn_score_duration, - lightgbm_score, lightgbm_fit_duration, lightgbm_score_duration, - xgb_score, xgb_fit_duration, xgb_score_duration, - cat_score, cat_fit_duration, cat_score_duration) + return ( + sklearn_score, + sklearn_fit_duration, + sklearn_score_duration, + lightgbm_score, + lightgbm_fit_duration, + lightgbm_score_duration, + xgb_score, + xgb_fit_duration, + xgb_score_duration, + cat_score, + cat_fit_duration, + cat_score_duration, + ) max_threads = os.cpu_count() -n_threads_list = [2 ** i for i in range(8) if (2 ** i) < max_threads] +n_threads_list = [2**i for i in range(8) if (2**i) < max_threads] n_threads_list.append(max_threads) sklearn_scores = [] @@ -243,28 +270,29 @@ def one_run(n_threads, n_samples): xgb_score_duration, cat_score, cat_fit_duration, - cat_score_duration + cat_score_duration, ) = one_run(n_threads, n_samples) for scores, score in ( - (sklearn_scores, sklearn_score), - (sklearn_fit_durations, sklearn_fit_duration), - (sklearn_score_durations, sklearn_score_duration), - (lightgbm_scores, lightgbm_score), - (lightgbm_fit_durations, lightgbm_fit_duration), - (lightgbm_score_durations, lightgbm_score_duration), - (xgb_scores, xgb_score), - (xgb_fit_durations, xgb_fit_duration), - (xgb_score_durations, xgb_score_duration), - (cat_scores, cat_score), - (cat_fit_durations, cat_fit_duration), - (cat_score_durations, cat_score_duration)): + (sklearn_scores, sklearn_score), + (sklearn_fit_durations, sklearn_fit_duration), + (sklearn_score_durations, sklearn_score_duration), + (lightgbm_scores, lightgbm_score), + (lightgbm_fit_durations, lightgbm_fit_duration), + (lightgbm_score_durations, lightgbm_score_duration), + (xgb_scores, xgb_score), + (xgb_fit_durations, xgb_fit_duration), + (xgb_score_durations, xgb_score_duration), + (cat_scores, cat_score), + (cat_fit_durations, cat_fit_duration), + (cat_score_durations, cat_score_duration), + ): scores.append(score) if args.plot or args.plot_filename: - import matplotlib.pyplot as plt import matplotlib + import matplotlib.pyplot as plt fig, axs = plt.subplots(2, figsize=(12, 12)) @@ -274,37 +302,40 @@ def one_run(n_threads, n_samples): if args.lightgbm: import lightgbm - label = f'LightGBM {lightgbm.__version__}' + + label = f"LightGBM {lightgbm.__version__}" axs[0].plot(n_threads_list, lightgbm_fit_durations, label=label) axs[1].plot(n_threads_list, lightgbm_score_durations, label=label) if args.xgboost: import xgboost - label = f'XGBoost {xgboost.__version__}' + + label = f"XGBoost {xgboost.__version__}" axs[0].plot(n_threads_list, xgb_fit_durations, label=label) axs[1].plot(n_threads_list, xgb_score_durations, label=label) if args.catboost: import catboost - label = f'CatBoost {catboost.__version__}' + + label = f"CatBoost {catboost.__version__}" axs[0].plot(n_threads_list, cat_fit_durations, label=label) axs[1].plot(n_threads_list, cat_score_durations, label=label) for ax in axs: - ax.set_xscale('log') - ax.set_xlabel('n_threads') - ax.set_ylabel('duration (s)') + ax.set_xscale("log") + ax.set_xlabel("n_threads") + ax.set_ylabel("duration (s)") ax.set_ylim(0, None) ax.set_xticks(n_threads_list) ax.get_xaxis().set_major_formatter(matplotlib.ticker.ScalarFormatter()) - ax.legend(loc='best') + ax.legend(loc="best") - axs[0].set_title('fit duration (s)') - axs[1].set_title('score duration (s)') + axs[0].set_title("fit duration (s)") + axs[1].set_title("score duration (s)") title = args.problem - if args.problem == 'classification': - title += ' n_classes = {}'.format(args.n_classes) + if args.problem == "classification": + title += " n_classes = {}".format(args.n_classes) fig.suptitle(title) plt.tight_layout() diff --git a/benchmarks/bench_isolation_forest.py b/benchmarks/bench_isolation_forest.py index b673b5606473a..743911936dccc 100644 --- a/benchmarks/bench_isolation_forest.py +++ b/benchmarks/bench_isolation_forest.py @@ -17,12 +17,13 @@ """ from time import time -import numpy as np + import matplotlib.pyplot as plt +import numpy as np +from sklearn.datasets import fetch_covtype, fetch_kddcup99, fetch_openml from sklearn.ensemble import IsolationForest -from sklearn.metrics import roc_curve, auc -from sklearn.datasets import fetch_kddcup99, fetch_covtype, fetch_openml +from sklearn.metrics import auc, roc_curve from sklearn.preprocessing import LabelBinarizer from sklearn.utils import shuffle as sh @@ -48,34 +49,34 @@ def print_outlier_ratio(y): with_decision_function_histograms = False # datasets available = ['http', 'smtp', 'SA', 'SF', 'shuttle', 'forestcover'] -datasets = ['http', 'smtp', 'SA', 'SF', 'shuttle', 'forestcover'] +datasets = ["http", "smtp", "SA", "SF", "shuttle", "forestcover"] # Loop over all datasets for fitting and scoring the estimator: for dat in datasets: - # Loading and vectorizing the data: - print('====== %s ======' % dat) - print('--- Fetching data...') - if dat in ['http', 'smtp', 'SF', 'SA']: - dataset = fetch_kddcup99(subset=dat, shuffle=True, - percent10=True, random_state=random_state) + print("====== %s ======" % dat) + print("--- Fetching data...") + if dat in ["http", "smtp", "SF", "SA"]: + dataset = fetch_kddcup99( + subset=dat, shuffle=True, percent10=True, random_state=random_state + ) X = dataset.data y = dataset.target - if dat == 'shuttle': - dataset = fetch_openml('shuttle') + if dat == "shuttle": + dataset = fetch_openml("shuttle", as_frame=False) X = dataset.data - y = dataset.target + y = dataset.target.astype(np.int64) X, y = sh(X, y, random_state=random_state) # we remove data with label 4 # normal data are then those of class 1 - s = (y != 4) + s = y != 4 X = X[s, :] y = y[s] y = (y != 1).astype(int) - print('----- ') + print("----- ") - if dat == 'forestcover': + if dat == "forestcover": dataset = fetch_covtype(shuffle=True, random_state=random_state) X = dataset.data y = dataset.target @@ -87,26 +88,26 @@ def print_outlier_ratio(y): y = (y != 2).astype(int) print_outlier_ratio(y) - print('--- Vectorizing data...') + print("--- Vectorizing data...") - if dat == 'SF': + if dat == "SF": lb = LabelBinarizer() x1 = lb.fit_transform(X[:, 1].astype(str)) X = np.c_[X[:, :1], x1, X[:, 2:]] - y = (y != b'normal.').astype(int) + y = (y != b"normal.").astype(int) print_outlier_ratio(y) - if dat == 'SA': + if dat == "SA": lb = LabelBinarizer() x1 = lb.fit_transform(X[:, 1].astype(str)) x2 = lb.fit_transform(X[:, 2].astype(str)) x3 = lb.fit_transform(X[:, 3].astype(str)) X = np.c_[X[:, :1], x1, x2, x3, X[:, 4:]] - y = (y != b'normal.').astype(int) + y = (y != b"normal.").astype(int) print_outlier_ratio(y) - if dat in ('http', 'smtp'): - y = (y != b'normal.').astype(int) + if dat in ("http", "smtp"): + y = (y != b"normal.").astype(int) print_outlier_ratio(y) n_samples, n_features = X.shape @@ -118,32 +119,36 @@ def print_outlier_ratio(y): y_train = y[:n_samples_train] y_test = y[n_samples_train:] - print('--- Fitting the IsolationForest estimator...') + print("--- Fitting the IsolationForest estimator...") model = IsolationForest(n_jobs=-1, random_state=random_state) tstart = time() model.fit(X_train) fit_time = time() - tstart tstart = time() - scoring = - model.decision_function(X_test) # the lower, the more abnormal + scoring = -model.decision_function(X_test) # the lower, the more abnormal print("--- Preparing the plot elements...") if with_decision_function_histograms: fig, ax = plt.subplots(3, sharex=True, sharey=True) bins = np.linspace(-0.5, 0.5, 200) - ax[0].hist(scoring, bins, color='black') - ax[0].set_title('Decision function for %s dataset' % dat) - ax[1].hist(scoring[y_test == 0], bins, color='b', label='normal data') + ax[0].hist(scoring, bins, color="black") + ax[0].set_title("Decision function for %s dataset" % dat) + ax[1].hist(scoring[y_test == 0], bins, color="b", label="normal data") ax[1].legend(loc="lower right") - ax[2].hist(scoring[y_test == 1], bins, color='r', label='outliers') + ax[2].hist(scoring[y_test == 1], bins, color="r", label="outliers") ax[2].legend(loc="lower right") # Show ROC Curves predict_time = time() - tstart fpr, tpr, thresholds = roc_curve(y_test, scoring) auc_score = auc(fpr, tpr) - label = ('%s (AUC: %0.3f, train_time= %0.2fs, ' - 'test_time= %0.2fs)' % (dat, auc_score, fit_time, predict_time)) + label = "%s (AUC: %0.3f, train_time= %0.2fs, test_time= %0.2fs)" % ( + dat, + auc_score, + fit_time, + predict_time, + ) # Print AUC score and train/test time: print(label) ax_roc.plot(fpr, tpr, lw=1, label=label) @@ -151,9 +156,9 @@ def print_outlier_ratio(y): ax_roc.set_xlim([-0.05, 1.05]) ax_roc.set_ylim([-0.05, 1.05]) -ax_roc.set_xlabel('False Positive Rate') -ax_roc.set_ylabel('True Positive Rate') -ax_roc.set_title('Receiver operating characteristic (ROC) curves') +ax_roc.set_xlabel("False Positive Rate") +ax_roc.set_ylabel("True Positive Rate") +ax_roc.set_title("Receiver operating characteristic (ROC) curves") ax_roc.legend(loc="lower right") fig_roc.tight_layout() plt.show() diff --git a/benchmarks/bench_isolation_forest_predict.py b/benchmarks/bench_isolation_forest_predict.py new file mode 100644 index 0000000000000..f16e65cf19511 --- /dev/null +++ b/benchmarks/bench_isolation_forest_predict.py @@ -0,0 +1,213 @@ +""" +========================================== +IsolationForest prediction benchmark +========================================== +A test of IsolationForest on classical anomaly detection datasets. + +The benchmark is run as follows: +1. The dataset is randomly split into a training set and a test set, both +assumed to contain outliers. +2. Isolation Forest is trained on the training set fixed at 1000 samples. +3. The test samples are scored using the trained model at: + - 1000, 10000, 50000 samples + - 10, 100, 1000 features + - 0.01, 0.1, 0.5 contamination + - 1, 2, 3, 4 n_jobs + +We compare the prediction time at the very end. + +Here are instructions for running this benchmark to compare runtime against main branch: + +1. Build and run on a branch or main, e.g. for a branch named `pr`: + +```bash +python bench_isolation_forest_predict.py bench ~/bench_results pr +``` + +2. Plotting to compare two branches `pr` and `main`: + +```bash +python bench_isolation_forest_predict.py plot ~/bench_results pr main results_image.png +``` +""" + +import argparse +from collections import defaultdict +from pathlib import Path +from time import time + +import numpy as np +import pandas as pd +from joblib import parallel_config + +from sklearn.ensemble import IsolationForest + +print(__doc__) + + +def get_data( + n_samples_train, n_samples_test, n_features, contamination=0.1, random_state=0 +): + """Function based on code from: https://scikit-learn.org/stable/ + auto_examples/ensemble/plot_isolation_forest.html#sphx-glr-auto- + examples-ensemble-plot-isolation-forest-py + """ + rng = np.random.RandomState(random_state) + + X = 0.3 * rng.randn(n_samples_train, n_features) + X_train = np.r_[X + 2, X - 2] + + X = 0.3 * rng.randn(n_samples_test, n_features) + X_test = np.r_[X + 2, X - 2] + + n_outliers = int(np.floor(contamination * n_samples_test)) + X_outliers = rng.uniform(low=-4, high=4, size=(n_outliers, n_features)) + + outlier_idx = rng.choice(np.arange(0, n_samples_test), n_outliers, replace=False) + X_test[outlier_idx, :] = X_outliers + + return X_train, X_test + + +def plot(args): + import matplotlib.pyplot as plt + import seaborn as sns + + bench_results = Path(args.bench_results) + pr_name = args.pr_name + main_name = args.main_name + image_path = args.image_path + + results_path = Path(bench_results) + pr_path = results_path / f"{pr_name}.csv" + main_path = results_path / f"{main_name}.csv" + image_path = results_path / image_path + + df_pr = pd.read_csv(pr_path).assign(branch=pr_name) + df_main = pd.read_csv(main_path).assign(branch=main_name) + + # Merge the two datasets on the common columns + merged_data = pd.merge( + df_pr, + df_main, + on=["n_samples_test", "n_jobs"], + suffixes=("_pr", "_main"), + ) + + # Set up the plotting grid + sns.set(style="whitegrid", context="notebook", font_scale=1.5) + + # Create a figure with subplots + fig, axes = plt.subplots(1, 2, figsize=(18, 6), sharex=True, sharey=True) + + # Plot predict time as a function of n_samples_test with different n_jobs + print(merged_data["n_jobs"].unique()) + ax = axes[0] + sns.lineplot( + data=merged_data, + x="n_samples_test", + y="predict_time_pr", + hue="n_jobs", + style="n_jobs", + markers="o", + ax=ax, + legend="full", + ) + ax.set_title(f"Predict Time vs. n_samples_test - {pr_name} branch") + ax.set_ylabel("Predict Time (Seconds)") + ax.set_xlabel("n_samples_test") + + ax = axes[1] + sns.lineplot( + data=merged_data, + x="n_samples_test", + y="predict_time_main", + hue="n_jobs", + style="n_jobs", + markers="X", + dashes=True, + ax=ax, + legend=None, + ) + ax.set_title(f"Predict Time vs. n_samples_test - {main_name} branch") + ax.set_ylabel("Predict Time") + ax.set_xlabel("n_samples_test") + + # Adjust layout and display the plots + plt.tight_layout() + fig.savefig(image_path, bbox_inches="tight") + print(f"Saved image to {image_path}") + + +def bench(args): + results_dir = Path(args.bench_results) + branch = args.branch + random_state = 1 + + results = defaultdict(list) + + # Loop over all datasets for fitting and scoring the estimator: + n_samples_train = 1000 + for n_samples_test in [ + 1000, + 10000, + 50000, + ]: + for n_features in [10, 100, 1000]: + for contamination in [0.01, 0.1, 0.5]: + for n_jobs in [1, 2, 3, 4]: + X_train, X_test = get_data( + n_samples_train, + n_samples_test, + n_features, + contamination, + random_state, + ) + + print("--- Fitting the IsolationForest estimator...") + model = IsolationForest(n_jobs=-1, random_state=random_state) + tstart = time() + model.fit(X_train) + fit_time = time() - tstart + + # clearcache + for _ in range(1000): + 1 + 1 + with parallel_config("threading", n_jobs=n_jobs): + tstart = time() + model.decision_function(X_test) # the lower, the more abnormal + predict_time = time() - tstart + + results["predict_time"].append(predict_time) + results["fit_time"].append(fit_time) + results["n_samples_train"].append(n_samples_train) + results["n_samples_test"].append(n_samples_test) + results["n_features"].append(n_features) + results["contamination"].append(contamination) + results["n_jobs"].append(n_jobs) + + df = pd.DataFrame(results) + df.to_csv(results_dir / f"{branch}.csv", index=False) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + # parse arguments for benchmarking + subparsers = parser.add_subparsers() + bench_parser = subparsers.add_parser("bench") + bench_parser.add_argument("bench_results") + bench_parser.add_argument("branch") + bench_parser.set_defaults(func=bench) + + # parse arguments for plotting + plot_parser = subparsers.add_parser("plot") + plot_parser.add_argument("bench_results") + plot_parser.add_argument("pr_name") + plot_parser.add_argument("main_name") + plot_parser.add_argument("image_path") + plot_parser.set_defaults(func=plot) + + # enable the parser and run the relevant function + args = parser.parse_args() + args.func(args) diff --git a/benchmarks/bench_isotonic.py b/benchmarks/bench_isotonic.py index d1eacaa8d1758..be2ff6548cb92 100644 --- a/benchmarks/bench_isotonic.py +++ b/benchmarks/bench_isotonic.py @@ -10,18 +10,20 @@ This allows the scaling of the algorithm with the problem size to be visualized and understood. """ -import numpy as np + +import argparse import gc -from datetime import datetime -from sklearn.isotonic import isotonic_regression -from scipy.special import expit +from timeit import default_timer + import matplotlib.pyplot as plt -import argparse +import numpy as np +from scipy.special import expit + +from sklearn.isotonic import isotonic_regression def generate_perturbed_logarithm_dataset(size): - return (np.random.randint(-50, 50, size=size) + - 50. * np.log(1 + np.arange(size))) + return np.random.randint(-50, 50, size=size) + 50.0 * np.log(1 + np.arange(size)) def generate_logistic_dataset(size): @@ -31,15 +33,15 @@ def generate_logistic_dataset(size): def generate_pathological_dataset(size): # Triggers O(n^2) complexity on the original implementation. - return np.r_[np.arange(size), - np.arange(-(size - 1), size), - np.arange(-(size - 1), 1)] + return np.r_[ + np.arange(size), np.arange(-(size - 1), size), np.arange(-(size - 1), 1) + ] DATASET_GENERATORS = { - 'perturbed_logarithm': generate_perturbed_logarithm_dataset, - 'logistic': generate_logistic_dataset, - 'pathological': generate_pathological_dataset, + "perturbed_logarithm": generate_perturbed_logarithm_dataset, + "logistic": generate_logistic_dataset, + "pathological": generate_pathological_dataset, } @@ -50,39 +52,48 @@ def bench_isotonic_regression(Y): """ gc.collect() - tstart = datetime.now() + tstart = default_timer() isotonic_regression(Y) - return (datetime.now() - tstart).total_seconds() - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description="Isotonic Regression benchmark tool") - parser.add_argument('--seed', type=int, - help="RNG seed") - parser.add_argument('--iterations', type=int, required=True, - help="Number of iterations to average timings over " - "for each problem size") - parser.add_argument('--log_min_problem_size', type=int, required=True, - help="Base 10 logarithm of the minimum problem size") - parser.add_argument('--log_max_problem_size', type=int, required=True, - help="Base 10 logarithm of the maximum problem size") - parser.add_argument('--show_plot', action='store_true', - help="Plot timing output with matplotlib") - parser.add_argument('--dataset', choices=DATASET_GENERATORS.keys(), - required=True) + return default_timer() - tstart + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Isotonic Regression benchmark tool") + parser.add_argument("--seed", type=int, help="RNG seed") + parser.add_argument( + "--iterations", + type=int, + required=True, + help="Number of iterations to average timings over for each problem size", + ) + parser.add_argument( + "--log_min_problem_size", + type=int, + required=True, + help="Base 10 logarithm of the minimum problem size", + ) + parser.add_argument( + "--log_max_problem_size", + type=int, + required=True, + help="Base 10 logarithm of the maximum problem size", + ) + parser.add_argument( + "--show_plot", action="store_true", help="Plot timing output with matplotlib" + ) + parser.add_argument("--dataset", choices=DATASET_GENERATORS.keys(), required=True) args = parser.parse_args() np.random.seed(args.seed) timings = [] - for exponent in range(args.log_min_problem_size, - args.log_max_problem_size): - n = 10 ** exponent + for exponent in range(args.log_min_problem_size, args.log_max_problem_size): + n = 10**exponent Y = DATASET_GENERATORS[args.dataset](n) - time_per_iteration = \ - [bench_isotonic_regression(Y) for i in range(args.iterations)] + time_per_iteration = [ + bench_isotonic_regression(Y) for i in range(args.iterations) + ] timing = (n, np.mean(time_per_iteration)) timings.append(timing) @@ -93,8 +104,8 @@ def bench_isotonic_regression(Y): if args.show_plot: plt.plot(*zip(*timings)) plt.title("Average time taken running isotonic regression") - plt.xlabel('Number of observations') - plt.ylabel('Time (s)') - plt.axis('tight') + plt.xlabel("Number of observations") + plt.ylabel("Time (s)") + plt.axis("tight") plt.loglog() plt.show() diff --git a/benchmarks/bench_kernel_pca_solvers_time_vs_n_components.py b/benchmarks/bench_kernel_pca_solvers_time_vs_n_components.py new file mode 100644 index 0000000000000..a468f7b3e1abf --- /dev/null +++ b/benchmarks/bench_kernel_pca_solvers_time_vs_n_components.py @@ -0,0 +1,177 @@ +""" +============================================================= +Kernel PCA Solvers comparison benchmark: time vs n_components +============================================================= + +This benchmark shows that the approximate solvers provided in Kernel PCA can +help significantly improve its execution speed when an approximate solution +(small `n_components`) is acceptable. In many real-world datasets a few +hundreds of principal components are indeed sufficient enough to capture the +underlying distribution. + +Description: +------------ +A fixed number of training (default: 2000) and test (default: 1000) samples +with 2 features is generated using the `make_circles` helper method. + +KernelPCA models are trained on the training set with an increasing number of +principal components, between 1 and `max_n_compo` (default: 1999), with +`n_compo_grid_size` positions (default: 10). For each value of `n_components` +to try, KernelPCA models are trained for the various possible `eigen_solver` +values. The execution times are displayed in a plot at the end of the +experiment. + +What you can observe: +--------------------- +When the number of requested principal components is small, the dense solver +takes more time to complete, while the randomized method returns similar +results with shorter execution times. + +Going further: +-------------- +You can adjust `max_n_compo` and `n_compo_grid_size` if you wish to explore a +different range of values for `n_components`. + +You can also set `arpack_all=True` to activate arpack solver for large number +of components (this takes more time). +""" + +import time + +import matplotlib.pyplot as plt +import numpy as np +from numpy.testing import assert_array_almost_equal + +from sklearn.datasets import make_circles +from sklearn.decomposition import KernelPCA + +print(__doc__) + + +# 1- Design the Experiment +# ------------------------ +n_train, n_test = 2000, 1000 # the sample sizes to use +max_n_compo = 1999 # max n_components to try +n_compo_grid_size = 10 # nb of positions in the grid to try +# generate the grid +n_compo_range = [ + np.round(np.exp((x / (n_compo_grid_size - 1)) * np.log(max_n_compo))) + for x in range(0, n_compo_grid_size) +] + +n_iter = 3 # the number of times each experiment will be repeated +arpack_all = False # set to True if you wish to run arpack for all n_compo + + +# 2- Generate random data +# ----------------------- +n_features = 2 +X, y = make_circles( + n_samples=(n_train + n_test), factor=0.3, noise=0.05, random_state=0 +) +X_train, X_test = X[:n_train, :], X[n_train:, :] + + +# 3- Benchmark +# ------------ +# init +ref_time = np.empty((len(n_compo_range), n_iter)) * np.nan +a_time = np.empty((len(n_compo_range), n_iter)) * np.nan +r_time = np.empty((len(n_compo_range), n_iter)) * np.nan +# loop +for j, n_components in enumerate(n_compo_range): + n_components = int(n_components) + print("Performing kPCA with n_components = %i" % n_components) + + # A- reference (dense) + print(" - dense solver") + for i in range(n_iter): + start_time = time.perf_counter() + ref_pred = ( + KernelPCA(n_components, eigen_solver="dense").fit(X_train).transform(X_test) + ) + ref_time[j, i] = time.perf_counter() - start_time + + # B- arpack (for small number of components only, too slow otherwise) + if arpack_all or n_components < 100: + print(" - arpack solver") + for i in range(n_iter): + start_time = time.perf_counter() + a_pred = ( + KernelPCA(n_components, eigen_solver="arpack") + .fit(X_train) + .transform(X_test) + ) + a_time[j, i] = time.perf_counter() - start_time + # check that the result is still correct despite the approx + assert_array_almost_equal(np.abs(a_pred), np.abs(ref_pred)) + + # C- randomized + print(" - randomized solver") + for i in range(n_iter): + start_time = time.perf_counter() + r_pred = ( + KernelPCA(n_components, eigen_solver="randomized") + .fit(X_train) + .transform(X_test) + ) + r_time[j, i] = time.perf_counter() - start_time + # check that the result is still correct despite the approximation + assert_array_almost_equal(np.abs(r_pred), np.abs(ref_pred)) + +# Compute statistics for the 3 methods +avg_ref_time = ref_time.mean(axis=1) +std_ref_time = ref_time.std(axis=1) +avg_a_time = a_time.mean(axis=1) +std_a_time = a_time.std(axis=1) +avg_r_time = r_time.mean(axis=1) +std_r_time = r_time.std(axis=1) + + +# 4- Plots +# -------- +fig, ax = plt.subplots(figsize=(12, 8)) + +# Display 1 plot with error bars per method +ax.errorbar( + n_compo_range, + avg_ref_time, + yerr=std_ref_time, + marker="x", + linestyle="", + color="r", + label="full", +) +ax.errorbar( + n_compo_range, + avg_a_time, + yerr=std_a_time, + marker="x", + linestyle="", + color="g", + label="arpack", +) +ax.errorbar( + n_compo_range, + avg_r_time, + yerr=std_r_time, + marker="x", + linestyle="", + color="b", + label="randomized", +) +ax.legend(loc="upper left") + +# customize axes +ax.set_xscale("log") +ax.set_xlim(1, max(n_compo_range) * 1.1) +ax.set_ylabel("Execution time (s)") +ax.set_xlabel("n_components") + +ax.set_title( + "kPCA Execution time comparison on %i samples with %i " + "features, according to the choice of `eigen_solver`" + "" % (n_train, n_features) +) + +plt.show() diff --git a/benchmarks/bench_kernel_pca_solvers_time_vs_n_samples.py b/benchmarks/bench_kernel_pca_solvers_time_vs_n_samples.py new file mode 100644 index 0000000000000..cae74c6f442ff --- /dev/null +++ b/benchmarks/bench_kernel_pca_solvers_time_vs_n_samples.py @@ -0,0 +1,183 @@ +""" +========================================================== +Kernel PCA Solvers comparison benchmark: time vs n_samples +========================================================== + +This benchmark shows that the approximate solvers provided in Kernel PCA can +help significantly improve its execution speed when an approximate solution +(small `n_components`) is acceptable. In many real-world datasets the number of +samples is very large, but a few hundreds of principal components are +sufficient enough to capture the underlying distribution. + +Description: +------------ +An increasing number of examples is used to train a KernelPCA, between +`min_n_samples` (default: 101) and `max_n_samples` (default: 4000) with +`n_samples_grid_size` positions (default: 4). Samples have 2 features, and are +generated using `make_circles`. For each training sample size, KernelPCA models +are trained for the various possible `eigen_solver` values. All of them are +trained to obtain `n_components` principal components (default: 100). The +execution times are displayed in a plot at the end of the experiment. + +What you can observe: +--------------------- +When the number of samples provided gets large, the dense solver takes a lot +of time to complete, while the randomized method returns similar results in +much shorter execution times. + +Going further: +-------------- +You can increase `max_n_samples` and `nb_n_samples_to_try` if you wish to +explore a wider range of values for `n_samples`. + +You can also set `include_arpack=True` to add this other solver in the +experiments (much slower). + +Finally you can have a look at the second example of this series, "Kernel PCA +Solvers comparison benchmark: time vs n_components", where this time the number +of examples is fixed, and the desired number of components varies. +""" + +# Author: Sylvain MARIE, Schneider Electric + +import time + +import matplotlib.pyplot as plt +import numpy as np +from numpy.testing import assert_array_almost_equal + +from sklearn.datasets import make_circles +from sklearn.decomposition import KernelPCA + +print(__doc__) + + +# 1- Design the Experiment +# ------------------------ +min_n_samples, max_n_samples = 101, 4000 # min and max n_samples to try +n_samples_grid_size = 4 # nb of positions in the grid to try +# generate the grid +n_samples_range = [ + min_n_samples + + np.floor((x / (n_samples_grid_size - 1)) * (max_n_samples - min_n_samples)) + for x in range(0, n_samples_grid_size) +] + +n_components = 100 # the number of principal components we want to use +n_iter = 3 # the number of times each experiment will be repeated +include_arpack = False # set this to True to include arpack solver (slower) + + +# 2- Generate random data +# ----------------------- +n_features = 2 +X, y = make_circles(n_samples=max_n_samples, factor=0.3, noise=0.05, random_state=0) + + +# 3- Benchmark +# ------------ +# init +ref_time = np.empty((len(n_samples_range), n_iter)) * np.nan +a_time = np.empty((len(n_samples_range), n_iter)) * np.nan +r_time = np.empty((len(n_samples_range), n_iter)) * np.nan + +# loop +for j, n_samples in enumerate(n_samples_range): + n_samples = int(n_samples) + print("Performing kPCA with n_samples = %i" % n_samples) + + X_train = X[:n_samples, :] + X_test = X_train + + # A- reference (dense) + print(" - dense") + for i in range(n_iter): + start_time = time.perf_counter() + ref_pred = ( + KernelPCA(n_components, eigen_solver="dense").fit(X_train).transform(X_test) + ) + ref_time[j, i] = time.perf_counter() - start_time + + # B- arpack + if include_arpack: + print(" - arpack") + for i in range(n_iter): + start_time = time.perf_counter() + a_pred = ( + KernelPCA(n_components, eigen_solver="arpack") + .fit(X_train) + .transform(X_test) + ) + a_time[j, i] = time.perf_counter() - start_time + # check that the result is still correct despite the approx + assert_array_almost_equal(np.abs(a_pred), np.abs(ref_pred)) + + # C- randomized + print(" - randomized") + for i in range(n_iter): + start_time = time.perf_counter() + r_pred = ( + KernelPCA(n_components, eigen_solver="randomized") + .fit(X_train) + .transform(X_test) + ) + r_time[j, i] = time.perf_counter() - start_time + # check that the result is still correct despite the approximation + assert_array_almost_equal(np.abs(r_pred), np.abs(ref_pred)) + +# Compute statistics for the 3 methods +avg_ref_time = ref_time.mean(axis=1) +std_ref_time = ref_time.std(axis=1) +avg_a_time = a_time.mean(axis=1) +std_a_time = a_time.std(axis=1) +avg_r_time = r_time.mean(axis=1) +std_r_time = r_time.std(axis=1) + + +# 4- Plots +# -------- +fig, ax = plt.subplots(figsize=(12, 8)) + +# Display 1 plot with error bars per method +ax.errorbar( + n_samples_range, + avg_ref_time, + yerr=std_ref_time, + marker="x", + linestyle="", + color="r", + label="full", +) +if include_arpack: + ax.errorbar( + n_samples_range, + avg_a_time, + yerr=std_a_time, + marker="x", + linestyle="", + color="g", + label="arpack", + ) +ax.errorbar( + n_samples_range, + avg_r_time, + yerr=std_r_time, + marker="x", + linestyle="", + color="b", + label="randomized", +) +ax.legend(loc="upper left") + +# customize axes +ax.set_xlim(min(n_samples_range) * 0.9, max(n_samples_range) * 1.1) +ax.set_ylabel("Execution time (s)") +ax.set_xlabel("n_samples") + +ax.set_title( + "Execution time comparison of kPCA with %i components on samples " + "with %i features, according to the choice of `eigen_solver`" + "" % (n_components, n_features) +) + +plt.show() diff --git a/benchmarks/bench_lasso.py b/benchmarks/bench_lasso.py index 4a2c8bbe6e248..9bae570505a75 100644 --- a/benchmarks/bench_lasso.py +++ b/benchmarks/bench_lasso.py @@ -11,8 +11,10 @@ In both cases, only 10% of the features are informative. """ + import gc from time import time + import numpy as np from sklearn.datasets import make_regression @@ -27,29 +29,30 @@ def compute_bench(alpha, n_samples, n_features, precompute): for ns in n_samples: for nf in n_features: it += 1 - print('==================') - print('Iteration %s of %s' % (it, max(len(n_samples), - len(n_features)))) - print('==================') + print("==================") + print("Iteration %s of %s" % (it, max(len(n_samples), len(n_features)))) + print("==================") n_informative = nf // 10 - X, Y, coef_ = make_regression(n_samples=ns, n_features=nf, - n_informative=n_informative, - noise=0.1, coef=True) + X, Y, coef_ = make_regression( + n_samples=ns, + n_features=nf, + n_informative=n_informative, + noise=0.1, + coef=True, + ) - X /= np.sqrt(np.sum(X ** 2, axis=0)) # Normalize data + X /= np.sqrt(np.sum(X**2, axis=0)) # Normalize data gc.collect() print("- benchmarking Lasso") - clf = Lasso(alpha=alpha, fit_intercept=False, - precompute=precompute) + clf = Lasso(alpha=alpha, fit_intercept=False, precompute=precompute) tstart = time() clf.fit(X, Y) lasso_results.append(time() - tstart) gc.collect() print("- benchmarking LassoLars") - clf = LassoLars(alpha=alpha, fit_intercept=False, - normalize=False, precompute=precompute) + clf = LassoLars(alpha=alpha, fit_intercept=False, precompute=precompute) tstart = time() clf.fit(X, Y) lars_lasso_results.append(time() - tstart) @@ -57,40 +60,40 @@ def compute_bench(alpha, n_samples, n_features, precompute): return lasso_results, lars_lasso_results -if __name__ == '__main__': - from sklearn.linear_model import Lasso, LassoLars +if __name__ == "__main__": import matplotlib.pyplot as plt + from sklearn.linear_model import Lasso, LassoLars + alpha = 0.01 # regularization parameter n_features = 10 list_n_samples = np.linspace(100, 1000000, 5).astype(int) - lasso_results, lars_lasso_results = compute_bench(alpha, list_n_samples, - [n_features], precompute=True) + lasso_results, lars_lasso_results = compute_bench( + alpha, list_n_samples, [n_features], precompute=True + ) - plt.figure('scikit-learn LASSO benchmark results') + plt.figure("scikit-learn LASSO benchmark results") plt.subplot(211) - plt.plot(list_n_samples, lasso_results, 'b-', - label='Lasso') - plt.plot(list_n_samples, lars_lasso_results, 'r-', - label='LassoLars') - plt.title('precomputed Gram matrix, %d features, alpha=%s' % (n_features, - alpha)) - plt.legend(loc='upper left') - plt.xlabel('number of samples') - plt.ylabel('Time (s)') - plt.axis('tight') + plt.plot(list_n_samples, lasso_results, "b-", label="Lasso") + plt.plot(list_n_samples, lars_lasso_results, "r-", label="LassoLars") + plt.title("precomputed Gram matrix, %d features, alpha=%s" % (n_features, alpha)) + plt.legend(loc="upper left") + plt.xlabel("number of samples") + plt.ylabel("Time (s)") + plt.axis("tight") n_samples = 2000 list_n_features = np.linspace(500, 3000, 5).astype(int) - lasso_results, lars_lasso_results = compute_bench(alpha, [n_samples], - list_n_features, precompute=False) + lasso_results, lars_lasso_results = compute_bench( + alpha, [n_samples], list_n_features, precompute=False + ) plt.subplot(212) - plt.plot(list_n_features, lasso_results, 'b-', label='Lasso') - plt.plot(list_n_features, lars_lasso_results, 'r-', label='LassoLars') - plt.title('%d samples, alpha=%s' % (n_samples, alpha)) - plt.legend(loc='upper left') - plt.xlabel('number of features') - plt.ylabel('Time (s)') - plt.axis('tight') + plt.plot(list_n_features, lasso_results, "b-", label="Lasso") + plt.plot(list_n_features, lars_lasso_results, "r-", label="LassoLars") + plt.title("%d samples, alpha=%s" % (n_samples, alpha)) + plt.legend(loc="upper left") + plt.xlabel("number of features") + plt.ylabel("Time (s)") + plt.axis("tight") plt.show() diff --git a/benchmarks/bench_lof.py b/benchmarks/bench_lof.py index 288caf212e7af..2c9732fab901f 100644 --- a/benchmarks/bench_lof.py +++ b/benchmarks/bench_lof.py @@ -18,11 +18,13 @@ """ from time import time -import numpy as np + import matplotlib.pyplot as plt +import numpy as np + +from sklearn.datasets import fetch_covtype, fetch_kddcup99, fetch_openml +from sklearn.metrics import auc, roc_curve from sklearn.neighbors import LocalOutlierFactor -from sklearn.metrics import roc_curve, auc -from sklearn.datasets import fetch_kddcup99, fetch_covtype, fetch_openml from sklearn.preprocessing import LabelBinarizer print(__doc__) @@ -30,30 +32,31 @@ random_state = 2 # to control the random selection of anomalies in SA # datasets available: ['http', 'smtp', 'SA', 'SF', 'shuttle', 'forestcover'] -datasets = ['http', 'smtp', 'SA', 'SF', 'shuttle', 'forestcover'] +datasets = ["http", "smtp", "SA", "SF", "shuttle", "forestcover"] plt.figure() for dataset_name in datasets: # loading and vectorization - print('loading data') - if dataset_name in ['http', 'smtp', 'SA', 'SF']: - dataset = fetch_kddcup99(subset=dataset_name, percent10=True, - random_state=random_state) + print("loading data") + if dataset_name in ["http", "smtp", "SA", "SF"]: + dataset = fetch_kddcup99( + subset=dataset_name, percent10=True, random_state=random_state + ) X = dataset.data y = dataset.target - if dataset_name == 'shuttle': - dataset = fetch_openml('shuttle') + if dataset_name == "shuttle": + dataset = fetch_openml("shuttle", as_frame=False) X = dataset.data - y = dataset.target + y = dataset.target.astype(np.int64) # we remove data with label 4 # normal data are then those of class 1 - s = (y != 4) + s = y != 4 X = X[s, :] y = y[s] y = (y != 1).astype(int) - if dataset_name == 'forestcover': + if dataset_name == "forestcover": dataset = fetch_covtype() X = dataset.data y = dataset.target @@ -64,28 +67,28 @@ y = y[s] y = (y != 2).astype(int) - print('vectorizing data') + print("vectorizing data") - if dataset_name == 'SF': + if dataset_name == "SF": lb = LabelBinarizer() x1 = lb.fit_transform(X[:, 1].astype(str)) X = np.c_[X[:, :1], x1, X[:, 2:]] - y = (y != b'normal.').astype(int) + y = (y != b"normal.").astype(int) - if dataset_name == 'SA': + if dataset_name == "SA": lb = LabelBinarizer() x1 = lb.fit_transform(X[:, 1].astype(str)) x2 = lb.fit_transform(X[:, 2].astype(str)) x3 = lb.fit_transform(X[:, 3].astype(str)) X = np.c_[X[:, :1], x1, x2, x3, X[:, 4:]] - y = (y != b'normal.').astype(int) + y = (y != b"normal.").astype(int) - if dataset_name == 'http' or dataset_name == 'smtp': - y = (y != b'normal.').astype(int) + if dataset_name == "http" or dataset_name == "smtp": + y = (y != b"normal.").astype(int) X = X.astype(float) - print('LocalOutlierFactor processing...') + print("LocalOutlierFactor processing...") model = LocalOutlierFactor(n_neighbors=20) tstart = time() model.fit(X) @@ -93,14 +96,18 @@ scoring = -model.negative_outlier_factor_ # the lower, the more normal fpr, tpr, thresholds = roc_curve(y, scoring) AUC = auc(fpr, tpr) - plt.plot(fpr, tpr, lw=1, - label=('ROC for %s (area = %0.3f, train-time: %0.2fs)' - % (dataset_name, AUC, fit_time))) + plt.plot( + fpr, + tpr, + lw=1, + label="ROC for %s (area = %0.3f, train-time: %0.2fs)" + % (dataset_name, AUC, fit_time), + ) plt.xlim([-0.05, 1.05]) plt.ylim([-0.05, 1.05]) -plt.xlabel('False Positive Rate') -plt.ylabel('True Positive Rate') -plt.title('Receiver operating characteristic') +plt.xlabel("False Positive Rate") +plt.ylabel("True Positive Rate") +plt.title("Receiver operating characteristic") plt.legend(loc="lower right") plt.show() diff --git a/benchmarks/bench_mnist.py b/benchmarks/bench_mnist.py index 1ff76028739c6..5745a6d1e3882 100644 --- a/benchmarks/bench_mnist.py +++ b/benchmarks/bench_mnist.py @@ -6,7 +6,7 @@ Benchmark on the MNIST dataset. The dataset comprises 70,000 samples and 784 features. Here, we consider the task of predicting 10 classes - digits from 0 to 9 from their raw images. By contrast to the -covertype dataset, the feature space is homogenous. +covertype dataset, the feature space is homogeneous. Example of output : [..] @@ -26,45 +26,41 @@ dummy 0.00s 0.01s 0.8973 """ -# Author: Issam H. Laradji -# Arnaud Joly -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +import argparse import os from time import time -import argparse + import numpy as np from joblib import Memory -from sklearn.datasets import fetch_openml -from sklearn.datasets import get_data_home -from sklearn.ensemble import ExtraTreesClassifier -from sklearn.ensemble import RandomForestClassifier +from sklearn.datasets import fetch_openml, get_data_home from sklearn.dummy import DummyClassifier -from sklearn.kernel_approximation import Nystroem -from sklearn.kernel_approximation import RBFSampler +from sklearn.ensemble import ExtraTreesClassifier, RandomForestClassifier +from sklearn.kernel_approximation import Nystroem, RBFSampler +from sklearn.linear_model import LogisticRegression from sklearn.metrics import zero_one_loss +from sklearn.neural_network import MLPClassifier from sklearn.pipeline import make_pipeline from sklearn.svm import LinearSVC from sklearn.tree import DecisionTreeClassifier from sklearn.utils import check_array -from sklearn.linear_model import LogisticRegression -from sklearn.neural_network import MLPClassifier # Memoize the data extraction and memory map the resulting # train / test splits in readonly mode -memory = Memory(os.path.join(get_data_home(), 'mnist_benchmark_data'), - mmap_mode='r') +memory = Memory(os.path.join(get_data_home(), "mnist_benchmark_data"), mmap_mode="r") @memory.cache -def load_data(dtype=np.float32, order='F'): +def load_data(dtype=np.float32, order="F"): """Load the data, then cache and memmap the train/test split""" ###################################################################### # Load dataset print("Loading dataset...") - data = fetch_openml('mnist_784') - X = check_array(data['data'], dtype=dtype, order=order) + data = fetch_openml("mnist_784", as_frame=True) + X = check_array(data["data"], dtype=dtype, order=order) y = data["target"] # Normalize features @@ -83,43 +79,76 @@ def load_data(dtype=np.float32, order='F'): ESTIMATORS = { "dummy": DummyClassifier(), - 'CART': DecisionTreeClassifier(), - 'ExtraTrees': ExtraTreesClassifier(), - 'RandomForest': RandomForestClassifier(), - 'Nystroem-SVM': make_pipeline( - Nystroem(gamma=0.015, n_components=1000), LinearSVC(C=100)), - 'SampledRBF-SVM': make_pipeline( - RBFSampler(gamma=0.015, n_components=1000), LinearSVC(C=100)), - 'LogisticRegression-SAG': LogisticRegression(solver='sag', tol=1e-1, - C=1e4), - 'LogisticRegression-SAGA': LogisticRegression(solver='saga', tol=1e-1, - C=1e4), - 'MultilayerPerceptron': MLPClassifier( - hidden_layer_sizes=(100, 100), max_iter=400, alpha=1e-4, - solver='sgd', learning_rate_init=0.2, momentum=0.9, verbose=1, - tol=1e-4, random_state=1), - 'MLP-adam': MLPClassifier( - hidden_layer_sizes=(100, 100), max_iter=400, alpha=1e-4, - solver='adam', learning_rate_init=0.001, verbose=1, - tol=1e-4, random_state=1) + "CART": DecisionTreeClassifier(), + "ExtraTrees": ExtraTreesClassifier(), + "RandomForest": RandomForestClassifier(), + "Nystroem-SVM": make_pipeline( + Nystroem(gamma=0.015, n_components=1000), LinearSVC(C=100) + ), + "SampledRBF-SVM": make_pipeline( + RBFSampler(gamma=0.015, n_components=1000), LinearSVC(C=100) + ), + "LogisticRegression-SAG": LogisticRegression(solver="sag", tol=1e-1, C=1e4), + "LogisticRegression-SAGA": LogisticRegression(solver="saga", tol=1e-1, C=1e4), + "MultilayerPerceptron": MLPClassifier( + hidden_layer_sizes=(100, 100), + max_iter=400, + alpha=1e-4, + solver="sgd", + learning_rate_init=0.2, + momentum=0.9, + verbose=1, + tol=1e-4, + random_state=1, + ), + "MLP-adam": MLPClassifier( + hidden_layer_sizes=(100, 100), + max_iter=400, + alpha=1e-4, + solver="adam", + learning_rate_init=0.001, + verbose=1, + tol=1e-4, + random_state=1, + ), } if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('--classifiers', nargs="+", - choices=ESTIMATORS, type=str, - default=['ExtraTrees', 'Nystroem-SVM'], - help="list of classifiers to benchmark.") - parser.add_argument('--n-jobs', nargs="?", default=1, type=int, - help="Number of concurrently running workers for " - "models that support parallelism.") - parser.add_argument('--order', nargs="?", default="C", type=str, - choices=["F", "C"], - help="Allow to choose between fortran and C ordered " - "data") - parser.add_argument('--random-seed', nargs="?", default=0, type=int, - help="Common seed used by random number generator.") + parser.add_argument( + "--classifiers", + nargs="+", + choices=ESTIMATORS, + type=str, + default=["ExtraTrees", "Nystroem-SVM"], + help="list of classifiers to benchmark.", + ) + parser.add_argument( + "--n-jobs", + nargs="?", + default=1, + type=int, + help=( + "Number of concurrently running workers for " + "models that support parallelism." + ), + ) + parser.add_argument( + "--order", + nargs="?", + default="C", + type=str, + choices=["F", "C"], + help="Allow to choose between fortran and C ordered data", + ) + parser.add_argument( + "--random-seed", + nargs="?", + default=0, + type=int, + help="Common seed used by random number generator.", + ) args = vars(parser.parse_args()) print(__doc__) @@ -132,10 +161,22 @@ def load_data(dtype=np.float32, order='F'): print("%s %d" % ("number of features:".ljust(25), X_train.shape[1])) print("%s %d" % ("number of classes:".ljust(25), np.unique(y_train).size)) print("%s %s" % ("data type:".ljust(25), X_train.dtype)) - print("%s %d (size=%dMB)" % ("number of train samples:".ljust(25), - X_train.shape[0], int(X_train.nbytes / 1e6))) - print("%s %d (size=%dMB)" % ("number of test samples:".ljust(25), - X_test.shape[0], int(X_test.nbytes / 1e6))) + print( + "%s %d (size=%dMB)" + % ( + "number of train samples:".ljust(25), + X_train.shape[0], + int(X_train.nbytes / 1e6), + ) + ) + print( + "%s %d (size=%dMB)" + % ( + "number of test samples:".ljust(25), + X_test.shape[0], + int(X_test.nbytes / 1e6), + ) + ) print() print("Training Classifiers") @@ -146,9 +187,13 @@ def load_data(dtype=np.float32, order='F'): estimator = ESTIMATORS[name] estimator_params = estimator.get_params() - estimator.set_params(**{p: args["random_seed"] - for p in estimator_params - if p.endswith("random_state")}) + estimator.set_params( + **{ + p: args["random_seed"] + for p in estimator_params + if p.endswith("random_state") + } + ) if "n_jobs" in estimator_params: estimator.set_params(n_jobs=args["n_jobs"]) @@ -168,12 +213,17 @@ def load_data(dtype=np.float32, order='F'): print() print("Classification performance:") print("===========================") - print("{0: <24} {1: >10} {2: >11} {3: >12}" - "".format("Classifier ", "train-time", "test-time", "error-rate")) + print( + "{0: <24} {1: >10} {2: >11} {3: >12}".format( + "Classifier ", "train-time", "test-time", "error-rate" + ) + ) print("-" * 60) for name in sorted(args["classifiers"], key=error.get): - - print("{0: <23} {1: >10.2f}s {2: >10.2f}s {3: >12.4f}" - "".format(name, train_time[name], test_time[name], error[name])) + print( + "{0: <23} {1: >10.2f}s {2: >10.2f}s {3: >12.4f}".format( + name, train_time[name], test_time[name], error[name] + ) + ) print() diff --git a/benchmarks/bench_multilabel_metrics.py b/benchmarks/bench_multilabel_metrics.py index 36fc7cb3c47b8..1b8449a24da51 100755 --- a/benchmarks/bench_multilabel_metrics.py +++ b/benchmarks/bench_multilabel_metrics.py @@ -3,43 +3,50 @@ A comparison of multilabel target formats and metrics over them """ -from timeit import timeit -from functools import partial -import itertools import argparse +import itertools import sys +from functools import partial +from timeit import timeit import matplotlib.pyplot as plt -import scipy.sparse as sp import numpy as np +import scipy.sparse as sp from sklearn.datasets import make_multilabel_classification -from sklearn.metrics import (f1_score, accuracy_score, hamming_loss, - jaccard_similarity_score) +from sklearn.metrics import ( + accuracy_score, + f1_score, + hamming_loss, + jaccard_similarity_score, +) from sklearn.utils._testing import ignore_warnings - METRICS = { - 'f1': partial(f1_score, average='micro'), - 'f1-by-sample': partial(f1_score, average='samples'), - 'accuracy': accuracy_score, - 'hamming': hamming_loss, - 'jaccard': jaccard_similarity_score, + "f1": partial(f1_score, average="micro"), + "f1-by-sample": partial(f1_score, average="samples"), + "accuracy": accuracy_score, + "hamming": hamming_loss, + "jaccard": jaccard_similarity_score, } FORMATS = { - 'sequences': lambda y: [list(np.flatnonzero(s)) for s in y], - 'dense': lambda y: y, - 'csr': lambda y: sp.csr_matrix(y), - 'csc': lambda y: sp.csc_matrix(y), + "sequences": lambda y: [list(np.flatnonzero(s)) for s in y], + "dense": lambda y: y, + "csr": sp.csr_matrix, + "csc": sp.csc_matrix, } @ignore_warnings -def benchmark(metrics=tuple(v for k, v in sorted(METRICS.items())), - formats=tuple(v for k, v in sorted(FORMATS.items())), - samples=1000, classes=4, density=.2, - n_times=5): +def benchmark( + metrics=tuple(v for k, v in sorted(METRICS.items())), + formats=tuple(v for k, v in sorted(FORMATS.items())), + samples=1000, + classes=4, + density=0.2, + n_times=5, +): """Times metric calculations for a number of inputs Parameters @@ -73,16 +80,18 @@ def benchmark(metrics=tuple(v for k, v in sorted(METRICS.items())), classes = np.atleast_1d(classes) density = np.atleast_1d(density) formats = np.atleast_1d(formats) - out = np.zeros((len(metrics), len(formats), len(samples), len(classes), - len(density)), dtype=float) + out = np.zeros( + (len(metrics), len(formats), len(samples), len(classes), len(density)), + dtype=float, + ) it = itertools.product(samples, classes, density) for i, (s, c, d) in enumerate(it): - _, y_true = make_multilabel_classification(n_samples=s, n_features=1, - n_classes=c, n_labels=d * c, - random_state=42) - _, y_pred = make_multilabel_classification(n_samples=s, n_features=1, - n_classes=c, n_labels=d * c, - random_state=84) + _, y_true = make_multilabel_classification( + n_samples=s, n_features=1, n_classes=c, n_labels=d * c, random_state=42 + ) + _, y_pred = make_multilabel_classification( + n_samples=s, n_features=1, n_classes=c, n_labels=d * c, random_state=84 + ) for j, f in enumerate(formats): f_true = f(y_true) f_pred = f(y_pred) @@ -100,70 +109,95 @@ def _tabulate(results, metrics, formats): """ column_width = max(max(len(k) for k in formats) + 1, 8) first_width = max(len(k) for k in metrics) - head_fmt = ('{:<{fw}s}' + '{:>{cw}s}' * len(formats)) - row_fmt = ('{:<{fw}s}' + '{:>{cw}.3f}' * len(formats)) - print(head_fmt.format('Metric', *formats, - cw=column_width, fw=first_width)) + head_fmt = "{:<{fw}s}" + "{:>{cw}s}" * len(formats) + row_fmt = "{:<{fw}s}" + "{:>{cw}.3f}" * len(formats) + print(head_fmt.format("Metric", *formats, cw=column_width, fw=first_width)) for metric, row in zip(metrics, results[:, :, -1, -1, -1]): - print(row_fmt.format(metric, *row, - cw=column_width, fw=first_width)) - - -def _plot(results, metrics, formats, title, x_ticks, x_label, - format_markers=('x', '|', 'o', '+'), - metric_colors=('c', 'm', 'y', 'k', 'g', 'r', 'b')): + print(row_fmt.format(metric, *row, cw=column_width, fw=first_width)) + + +def _plot( + results, + metrics, + formats, + title, + x_ticks, + x_label, + format_markers=("x", "|", "o", "+"), + metric_colors=("c", "m", "y", "k", "g", "r", "b"), +): """ Plot the results by metric, format and some other variable given by x_label """ - fig = plt.figure('scikit-learn multilabel metrics benchmarks') + fig = plt.figure("scikit-learn multilabel metrics benchmarks") plt.title(title) ax = fig.add_subplot(111) for i, metric in enumerate(metrics): for j, format in enumerate(formats): - ax.plot(x_ticks, results[i, j].flat, - label='{}, {}'.format(metric, format), - marker=format_markers[j], - color=metric_colors[i % len(metric_colors)]) + ax.plot( + x_ticks, + results[i, j].flat, + label="{}, {}".format(metric, format), + marker=format_markers[j], + color=metric_colors[i % len(metric_colors)], + ) ax.set_xlabel(x_label) - ax.set_ylabel('Time (s)') + ax.set_ylabel("Time (s)") ax.legend() plt.show() if __name__ == "__main__": ap = argparse.ArgumentParser() - ap.add_argument('metrics', nargs='*', default=sorted(METRICS), - help='Specifies metrics to benchmark, defaults to all. ' - 'Choices are: {}'.format(sorted(METRICS))) - ap.add_argument('--formats', nargs='+', choices=sorted(FORMATS), - help='Specifies multilabel formats to benchmark ' - '(defaults to all).') - ap.add_argument('--samples', type=int, default=1000, - help='The number of samples to generate') - ap.add_argument('--classes', type=int, default=10, - help='The number of classes') - ap.add_argument('--density', type=float, default=.2, - help='The average density of labels per sample') - ap.add_argument('--plot', choices=['classes', 'density', 'samples'], - default=None, - help='Plot time with respect to this parameter varying ' - 'up to the specified value') - ap.add_argument('--n-steps', default=10, type=int, - help='Plot this many points for each metric') - ap.add_argument('--n-times', - default=5, type=int, - help="Time performance over n_times trials") + ap.add_argument( + "metrics", + nargs="*", + default=sorted(METRICS), + help="Specifies metrics to benchmark, defaults to all. Choices are: {}".format( + sorted(METRICS) + ), + ) + ap.add_argument( + "--formats", + nargs="+", + choices=sorted(FORMATS), + help="Specifies multilabel formats to benchmark (defaults to all).", + ) + ap.add_argument( + "--samples", type=int, default=1000, help="The number of samples to generate" + ) + ap.add_argument("--classes", type=int, default=10, help="The number of classes") + ap.add_argument( + "--density", + type=float, + default=0.2, + help="The average density of labels per sample", + ) + ap.add_argument( + "--plot", + choices=["classes", "density", "samples"], + default=None, + help=( + "Plot time with respect to this parameter varying up to the specified value" + ), + ) + ap.add_argument( + "--n-steps", default=10, type=int, help="Plot this many points for each metric" + ) + ap.add_argument( + "--n-times", default=5, type=int, help="Time performance over n_times trials" + ) args = ap.parse_args() if args.plot is not None: max_val = getattr(args, args.plot) - if args.plot in ('classes', 'samples'): + if args.plot in ("classes", "samples"): min_val = 2 else: min_val = 0 steps = np.linspace(min_val, max_val, num=args.n_steps + 1)[1:] - if args.plot in ('classes', 'samples'): + if args.plot in ("classes", "samples"): steps = np.unique(np.round(steps).astype(int)) setattr(args, args.plot, steps) @@ -172,17 +206,22 @@ def _plot(results, metrics, formats, title, x_ticks, x_label, if args.formats is None: args.formats = sorted(FORMATS) - results = benchmark([METRICS[k] for k in args.metrics], - [FORMATS[k] for k in args.formats], - args.samples, args.classes, args.density, - args.n_times) + results = benchmark( + [METRICS[k] for k in args.metrics], + [FORMATS[k] for k in args.formats], + args.samples, + args.classes, + args.density, + args.n_times, + ) _tabulate(results, args.metrics, args.formats) if args.plot is not None: - print('Displaying plot', file=sys.stderr) - title = ('Multilabel metrics with %s' % - ', '.join('{0}={1}'.format(field, getattr(args, field)) - for field in ['samples', 'classes', 'density'] - if args.plot != field)) + print("Displaying plot", file=sys.stderr) + title = "Multilabel metrics with %s" % ", ".join( + "{0}={1}".format(field, getattr(args, field)) + for field in ["samples", "classes", "density"] + if args.plot != field + ) _plot(results, args.metrics, args.formats, title, steps, args.plot) diff --git a/benchmarks/bench_online_ocsvm.py b/benchmarks/bench_online_ocsvm.py new file mode 100644 index 0000000000000..9f92150e079dd --- /dev/null +++ b/benchmarks/bench_online_ocsvm.py @@ -0,0 +1,294 @@ +""" +===================================== +SGDOneClassSVM benchmark +===================================== +This benchmark compares the :class:`SGDOneClassSVM` with :class:`OneClassSVM`. +The former is an online One-Class SVM implemented with a Stochastic Gradient +Descent (SGD). The latter is based on the LibSVM implementation. The +complexity of :class:`SGDOneClassSVM` is linear in the number of samples +whereas the one of :class:`OneClassSVM` is at best quadratic in the number of +samples. We here compare the performance in terms of AUC and training time on +classical anomaly detection datasets. + +The :class:`OneClassSVM` is applied with a Gaussian kernel and we therefore +use a kernel approximation prior to the application of :class:`SGDOneClassSVM`. +""" + +from time import time + +import matplotlib +import matplotlib.pyplot as plt +import numpy as np +from scipy.interpolate import interp1d + +from sklearn.datasets import fetch_covtype, fetch_kddcup99 +from sklearn.kernel_approximation import Nystroem +from sklearn.linear_model import SGDOneClassSVM +from sklearn.metrics import auc, roc_curve +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import LabelBinarizer, StandardScaler +from sklearn.svm import OneClassSVM +from sklearn.utils import shuffle + +font = {"weight": "normal", "size": 15} + +matplotlib.rc("font", **font) + +print(__doc__) + + +def print_outlier_ratio(y): + """ + Helper function to show the distinct value count of element in the target. + Useful indicator for the datasets used in bench_isolation_forest.py. + """ + uniq, cnt = np.unique(y, return_counts=True) + print("----- Target count values: ") + for u, c in zip(uniq, cnt): + print("------ %s -> %d occurrences" % (str(u), c)) + print("----- Outlier ratio: %.5f" % (np.min(cnt) / len(y))) + + +# for roc curve computation +n_axis = 1000 +x_axis = np.linspace(0, 1, n_axis) + +datasets = ["http", "smtp", "SA", "SF", "forestcover"] + +novelty_detection = False # if False, training set polluted by outliers + +random_states = [42] +nu = 0.05 + +results_libsvm = np.empty((len(datasets), n_axis + 5)) +results_online = np.empty((len(datasets), n_axis + 5)) + +for dat, dataset_name in enumerate(datasets): + print(dataset_name) + + # Loading datasets + if dataset_name in ["http", "smtp", "SA", "SF"]: + dataset = fetch_kddcup99( + subset=dataset_name, shuffle=False, percent10=False, random_state=88 + ) + X = dataset.data + y = dataset.target + + if dataset_name == "forestcover": + dataset = fetch_covtype(shuffle=False) + X = dataset.data + y = dataset.target + # normal data are those with attribute 2 + # abnormal those with attribute 4 + s = (y == 2) + (y == 4) + X = X[s, :] + y = y[s] + y = (y != 2).astype(int) + + # Vectorizing data + if dataset_name == "SF": + # Casting type of X (object) as string is needed for string categorical + # features to apply LabelBinarizer + lb = LabelBinarizer() + x1 = lb.fit_transform(X[:, 1].astype(str)) + X = np.c_[X[:, :1], x1, X[:, 2:]] + y = (y != b"normal.").astype(int) + + if dataset_name == "SA": + lb = LabelBinarizer() + # Casting type of X (object) as string is needed for string categorical + # features to apply LabelBinarizer + x1 = lb.fit_transform(X[:, 1].astype(str)) + x2 = lb.fit_transform(X[:, 2].astype(str)) + x3 = lb.fit_transform(X[:, 3].astype(str)) + X = np.c_[X[:, :1], x1, x2, x3, X[:, 4:]] + y = (y != b"normal.").astype(int) + + if dataset_name in ["http", "smtp"]: + y = (y != b"normal.").astype(int) + + print_outlier_ratio(y) + + n_samples, n_features = np.shape(X) + if dataset_name == "SA": # LibSVM too long with n_samples // 2 + n_samples_train = n_samples // 20 + else: + n_samples_train = n_samples // 2 + + n_samples_test = n_samples - n_samples_train + print("n_train: ", n_samples_train) + print("n_features: ", n_features) + + tpr_libsvm = np.zeros(n_axis) + tpr_online = np.zeros(n_axis) + fit_time_libsvm = 0 + fit_time_online = 0 + predict_time_libsvm = 0 + predict_time_online = 0 + + X = X.astype(float) + + gamma = 1 / n_features # OCSVM default parameter + + for random_state in random_states: + print("random state: %s" % random_state) + + X, y = shuffle(X, y, random_state=random_state) + X_train = X[:n_samples_train] + X_test = X[n_samples_train:] + y_train = y[:n_samples_train] + y_test = y[n_samples_train:] + + if novelty_detection: + X_train = X_train[y_train == 0] + y_train = y_train[y_train == 0] + + std = StandardScaler() + + print("----------- LibSVM OCSVM ------------") + ocsvm = OneClassSVM(kernel="rbf", gamma=gamma, nu=nu) + pipe_libsvm = make_pipeline(std, ocsvm) + + tstart = time() + pipe_libsvm.fit(X_train) + fit_time_libsvm += time() - tstart + + tstart = time() + # scoring such that the lower, the more normal + scoring = -pipe_libsvm.decision_function(X_test) + predict_time_libsvm += time() - tstart + fpr_libsvm_, tpr_libsvm_, _ = roc_curve(y_test, scoring) + + f_libsvm = interp1d(fpr_libsvm_, tpr_libsvm_) + tpr_libsvm += f_libsvm(x_axis) + + print("----------- Online OCSVM ------------") + nystroem = Nystroem(gamma=gamma, random_state=random_state) + online_ocsvm = SGDOneClassSVM(nu=nu, random_state=random_state) + pipe_online = make_pipeline(std, nystroem, online_ocsvm) + + tstart = time() + pipe_online.fit(X_train) + fit_time_online += time() - tstart + + tstart = time() + # scoring such that the lower, the more normal + scoring = -pipe_online.decision_function(X_test) + predict_time_online += time() - tstart + fpr_online_, tpr_online_, _ = roc_curve(y_test, scoring) + + f_online = interp1d(fpr_online_, tpr_online_) + tpr_online += f_online(x_axis) + + tpr_libsvm /= len(random_states) + tpr_libsvm[0] = 0.0 + fit_time_libsvm /= len(random_states) + predict_time_libsvm /= len(random_states) + auc_libsvm = auc(x_axis, tpr_libsvm) + + results_libsvm[dat] = [ + fit_time_libsvm, + predict_time_libsvm, + auc_libsvm, + n_samples_train, + n_features, + ] + list(tpr_libsvm) + + tpr_online /= len(random_states) + tpr_online[0] = 0.0 + fit_time_online /= len(random_states) + predict_time_online /= len(random_states) + auc_online = auc(x_axis, tpr_online) + + results_online[dat] = [ + fit_time_online, + predict_time_online, + auc_online, + n_samples_train, + n_features, + ] + list(tpr_libsvm) + + +# -------- Plotting bar charts ------------- +fit_time_libsvm_all = results_libsvm[:, 0] +predict_time_libsvm_all = results_libsvm[:, 1] +auc_libsvm_all = results_libsvm[:, 2] +n_train_all = results_libsvm[:, 3] +n_features_all = results_libsvm[:, 4] + +fit_time_online_all = results_online[:, 0] +predict_time_online_all = results_online[:, 1] +auc_online_all = results_online[:, 2] + + +width = 0.7 +ind = 2 * np.arange(len(datasets)) +x_tickslabels = [ + (name + "\n" + r"$n={:,d}$" + "\n" + r"$d={:d}$").format(int(n), int(d)) + for name, n, d in zip(datasets, n_train_all, n_features_all) +] + + +def autolabel_auc(rects, ax): + """Attach a text label above each bar displaying its height.""" + for rect in rects: + height = rect.get_height() + ax.text( + rect.get_x() + rect.get_width() / 2.0, + 1.05 * height, + "%.3f" % height, + ha="center", + va="bottom", + ) + + +def autolabel_time(rects, ax): + """Attach a text label above each bar displaying its height.""" + for rect in rects: + height = rect.get_height() + ax.text( + rect.get_x() + rect.get_width() / 2.0, + 1.05 * height, + "%.1f" % height, + ha="center", + va="bottom", + ) + + +fig, ax = plt.subplots(figsize=(15, 8)) +ax.set_ylabel("AUC") +ax.set_ylim((0, 1.3)) +rect_libsvm = ax.bar(ind, auc_libsvm_all, width=width, color="r") +rect_online = ax.bar(ind + width, auc_online_all, width=width, color="y") +ax.legend((rect_libsvm[0], rect_online[0]), ("LibSVM", "Online SVM")) +ax.set_xticks(ind + width / 2) +ax.set_xticklabels(x_tickslabels) +autolabel_auc(rect_libsvm, ax) +autolabel_auc(rect_online, ax) +plt.show() + + +fig, ax = plt.subplots(figsize=(15, 8)) +ax.set_ylabel("Training time (sec) - Log scale") +ax.set_yscale("log") +rect_libsvm = ax.bar(ind, fit_time_libsvm_all, color="r", width=width) +rect_online = ax.bar(ind + width, fit_time_online_all, color="y", width=width) +ax.legend((rect_libsvm[0], rect_online[0]), ("LibSVM", "Online SVM")) +ax.set_xticks(ind + width / 2) +ax.set_xticklabels(x_tickslabels) +autolabel_time(rect_libsvm, ax) +autolabel_time(rect_online, ax) +plt.show() + + +fig, ax = plt.subplots(figsize=(15, 8)) +ax.set_ylabel("Testing time (sec) - Log scale") +ax.set_yscale("log") +rect_libsvm = ax.bar(ind, predict_time_libsvm_all, color="r", width=width) +rect_online = ax.bar(ind + width, predict_time_online_all, color="y", width=width) +ax.legend((rect_libsvm[0], rect_online[0]), ("LibSVM", "Online SVM")) +ax.set_xticks(ind + width / 2) +ax.set_xticklabels(x_tickslabels) +autolabel_time(rect_libsvm, ax) +autolabel_time(rect_online, ax) +plt.show() diff --git a/benchmarks/bench_pca_solvers.py b/benchmarks/bench_pca_solvers.py new file mode 100644 index 0000000000000..337af3a42e900 --- /dev/null +++ b/benchmarks/bench_pca_solvers.py @@ -0,0 +1,165 @@ +# %% +# +# This benchmark compares the speed of PCA solvers on datasets of different +# sizes in order to determine the best solver to select by default via the +# "auto" heuristic. +# +# Note: we do not control for the accuracy of the solvers: we assume that all +# solvers yield transformed data with similar explained variance. This +# assumption is generally true, except for the randomized solver that might +# require more power iterations. +# +# We generate synthetic data with dimensions that are useful to plot: +# - time vs n_samples for a fixed n_features and, +# - time vs n_features for a fixed n_samples for a fixed n_features. +import itertools +from math import log10 +from time import perf_counter + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd + +from sklearn import config_context +from sklearn.decomposition import PCA + +REF_DIMS = [100, 1000, 10_000] +data_shapes = [] +for ref_dim in REF_DIMS: + data_shapes.extend([(ref_dim, 10**i) for i in range(1, 8 - int(log10(ref_dim)))]) + data_shapes.extend( + [(ref_dim, 3 * 10**i) for i in range(1, 8 - int(log10(ref_dim)))] + ) + data_shapes.extend([(10**i, ref_dim) for i in range(1, 8 - int(log10(ref_dim)))]) + data_shapes.extend( + [(3 * 10**i, ref_dim) for i in range(1, 8 - int(log10(ref_dim)))] + ) + +# Remove duplicates: +data_shapes = sorted(set(data_shapes)) + +print("Generating test datasets...") +rng = np.random.default_rng(0) +datasets = [rng.normal(size=shape) for shape in data_shapes] + + +# %% +def measure_one(data, n_components, solver, method_name="fit"): + print( + f"Benchmarking {solver=!r}, {n_components=}, {method_name=!r} on data with" + f" shape {data.shape}" + ) + pca = PCA(n_components=n_components, svd_solver=solver, random_state=0) + timings = [] + elapsed = 0 + method = getattr(pca, method_name) + with config_context(assume_finite=True): + while elapsed < 0.5: + tic = perf_counter() + method(data) + duration = perf_counter() - tic + timings.append(duration) + elapsed += duration + return np.median(timings) + + +SOLVERS = ["full", "covariance_eigh", "arpack", "randomized", "auto"] +measurements = [] +for data, n_components, method_name in itertools.product( + datasets, [2, 50], ["fit", "fit_transform"] +): + if n_components >= min(data.shape): + continue + for solver in SOLVERS: + if solver == "covariance_eigh" and data.shape[1] > 5000: + # Too much memory and too slow. + continue + if solver in ["arpack", "full"] and log10(data.size) > 7: + # Too slow, in particular for the full solver. + continue + time = measure_one(data, n_components, solver, method_name=method_name) + measurements.append( + { + "n_components": n_components, + "n_samples": data.shape[0], + "n_features": data.shape[1], + "time": time, + "solver": solver, + "method_name": method_name, + } + ) +measurements = pd.DataFrame(measurements) +measurements.to_csv("bench_pca_solvers.csv", index=False) + +# %% +all_method_names = measurements["method_name"].unique() +all_n_components = measurements["n_components"].unique() + +for method_name in all_method_names: + fig, axes = plt.subplots( + figsize=(16, 16), + nrows=len(REF_DIMS), + ncols=len(all_n_components), + sharey=True, + constrained_layout=True, + ) + fig.suptitle(f"Benchmarks for PCA.{method_name}, varying n_samples", fontsize=16) + + for row_idx, ref_dim in enumerate(REF_DIMS): + for n_components, ax in zip(all_n_components, axes[row_idx]): + for solver in SOLVERS: + if solver == "auto": + style_kwargs = dict(linewidth=2, color="black", style="--") + else: + style_kwargs = dict(style="o-") + ax.set( + title=f"n_components={n_components}, n_features={ref_dim}", + ylabel="time (s)", + ) + measurements.query( + "n_components == @n_components and n_features == @ref_dim" + " and solver == @solver and method_name == @method_name" + ).plot.line( + x="n_samples", + y="time", + label=solver, + logx=True, + logy=True, + ax=ax, + **style_kwargs, + ) +# %% +for method_name in all_method_names: + fig, axes = plt.subplots( + figsize=(16, 16), + nrows=len(REF_DIMS), + ncols=len(all_n_components), + sharey=True, + ) + fig.suptitle(f"Benchmarks for PCA.{method_name}, varying n_features", fontsize=16) + + for row_idx, ref_dim in enumerate(REF_DIMS): + for n_components, ax in zip(all_n_components, axes[row_idx]): + for solver in SOLVERS: + if solver == "auto": + style_kwargs = dict(linewidth=2, color="black", style="--") + else: + style_kwargs = dict(style="o-") + ax.set( + title=f"n_components={n_components}, n_samples={ref_dim}", + ylabel="time (s)", + ) + measurements.query( + "n_components == @n_components and n_samples == @ref_dim " + " and solver == @solver and method_name == @method_name" + ).plot.line( + x="n_features", + y="time", + label=solver, + logx=True, + logy=True, + ax=ax, + **style_kwargs, + ) + +# %% diff --git a/benchmarks/bench_plot_fastkmeans.py b/benchmarks/bench_plot_fastkmeans.py index dd37f159a5c91..d5a2d10fbf22d 100644 --- a/benchmarks/bench_plot_fastkmeans.py +++ b/benchmarks/bench_plot_fastkmeans.py @@ -8,7 +8,6 @@ def compute_bench(samples_range, features_range): - it = 0 results = defaultdict(lambda: []) chunk = 100 @@ -17,29 +16,29 @@ def compute_bench(samples_range, features_range): for n_samples in samples_range: for n_features in features_range: it += 1 - print('==============================') - print('Iteration %03d of %03d' % (it, max_it)) - print('==============================') + print("==============================") + print("Iteration %03d of %03d" % (it, max_it)) + print("==============================") print() data = nr.randint(-50, 51, (n_samples, n_features)) - print('K-Means') + print("K-Means") tstart = time() - kmeans = KMeans(init='k-means++', n_clusters=10).fit(data) + kmeans = KMeans(init="k-means++", n_clusters=10).fit(data) delta = time() - tstart print("Speed: %0.3fs" % delta) print("Inertia: %0.5f" % kmeans.inertia_) print() - results['kmeans_speed'].append(delta) - results['kmeans_quality'].append(kmeans.inertia_) + results["kmeans_speed"].append(delta) + results["kmeans_quality"].append(kmeans.inertia_) - print('Fast K-Means') + print("Fast K-Means") # let's prepare the data in small chunks - mbkmeans = MiniBatchKMeans(init='k-means++', - n_clusters=10, - batch_size=chunk) + mbkmeans = MiniBatchKMeans( + init="k-means++", n_clusters=10, batch_size=chunk + ) tstart = time() mbkmeans.fit(data) delta = time() - tstart @@ -48,8 +47,8 @@ def compute_bench(samples_range, features_range): print() print() - results['MiniBatchKMeans Speed'].append(delta) - results['MiniBatchKMeans Quality'].append(mbkmeans.inertia_) + results["MiniBatchKMeans Speed"].append(delta) + results["MiniBatchKMeans Quality"].append(mbkmeans.inertia_) return results @@ -57,8 +56,18 @@ def compute_bench(samples_range, features_range): def compute_bench_2(chunks): results = defaultdict(lambda: []) n_features = 50000 - means = np.array([[1, 1], [-1, -1], [1, -1], [-1, 1], - [0.5, 0.5], [0.75, -0.5], [-1, 0.75], [1, 0]]) + means = np.array( + [ + [1, 1], + [-1, -1], + [1, -1], + [-1, 1], + [0.5, 0.5], + [0.75, -0.5], + [-1, 0.75], + [1, 0], + ] + ) X = np.empty((0, 2)) for i in range(8): X = np.r_[X, means[i] + 0.8 * np.random.randn(n_features, 2)] @@ -66,16 +75,14 @@ def compute_bench_2(chunks): it = 0 for chunk in chunks: it += 1 - print('==============================') - print('Iteration %03d of %03d' % (it, max_it)) - print('==============================') + print("==============================") + print("Iteration %03d of %03d" % (it, max_it)) + print("==============================") print() - print('Fast K-Means') + print("Fast K-Means") tstart = time() - mbkmeans = MiniBatchKMeans(init='k-means++', - n_clusters=8, - batch_size=chunk) + mbkmeans = MiniBatchKMeans(init="k-means++", n_clusters=8, batch_size=chunk) mbkmeans.fit(X) delta = time() - tstart @@ -83,15 +90,15 @@ def compute_bench_2(chunks): print("Inertia: %0.3fs" % mbkmeans.inertia_) print() - results['MiniBatchKMeans Speed'].append(delta) - results['MiniBatchKMeans Quality'].append(mbkmeans.inertia_) + results["MiniBatchKMeans Speed"].append(delta) + results["MiniBatchKMeans Quality"].append(mbkmeans.inertia_) return results -if __name__ == '__main__': - from mpl_toolkits.mplot3d import axes3d # register the 3d projection +if __name__ == "__main__": import matplotlib.pyplot as plt + from mpl_toolkits.mplot3d import axes3d # register the 3d projection # noqa: F401 samples_range = np.linspace(50, 150, 5).astype(int) features_range = np.linspace(150, 50000, 5).astype(int) @@ -100,37 +107,35 @@ def compute_bench_2(chunks): results = compute_bench(samples_range, features_range) results_2 = compute_bench_2(chunks) - max_time = max([max(i) for i in [t for (label, t) in results.items() - if "speed" in label]]) - max_inertia = max([max(i) for i in [ - t for (label, t) in results.items() - if "speed" not in label]]) - - fig = plt.figure('scikit-learn K-Means benchmark results') - for c, (label, timings) in zip('brcy', - sorted(results.items())): - if 'speed' in label: - ax = fig.add_subplot(2, 2, 1, projection='3d') + max_time = max( + [max(i) for i in [t for (label, t) in results.items() if "speed" in label]] + ) + max_inertia = max( + [max(i) for i in [t for (label, t) in results.items() if "speed" not in label]] + ) + + fig = plt.figure("scikit-learn K-Means benchmark results") + for c, (label, timings) in zip("brcy", sorted(results.items())): + if "speed" in label: + ax = fig.add_subplot(2, 2, 1, projection="3d") ax.set_zlim3d(0.0, max_time * 1.1) else: - ax = fig.add_subplot(2, 2, 2, projection='3d') + ax = fig.add_subplot(2, 2, 2, projection="3d") ax.set_zlim3d(0.0, max_inertia * 1.1) X, Y = np.meshgrid(samples_range, features_range) - Z = np.asarray(timings).reshape(samples_range.shape[0], - features_range.shape[0]) + Z = np.asarray(timings).reshape(samples_range.shape[0], features_range.shape[0]) ax.plot_surface(X, Y, Z.T, cstride=1, rstride=1, color=c, alpha=0.5) - ax.set_xlabel('n_samples') - ax.set_ylabel('n_features') + ax.set_xlabel("n_samples") + ax.set_ylabel("n_features") i = 0 - for c, (label, timings) in zip('br', - sorted(results_2.items())): + for c, (label, timings) in zip("br", sorted(results_2.items())): i += 1 ax = fig.add_subplot(2, 2, i + 2) y = np.asarray(timings) ax.plot(chunks, y, color=c, alpha=0.8) - ax.set_xlabel('Chunks') + ax.set_xlabel("Chunks") ax.set_ylabel(label) plt.show() diff --git a/benchmarks/bench_plot_hierarchical.py b/benchmarks/bench_plot_hierarchical.py index 72c3f36616ff4..861a0ea0b5296 100644 --- a/benchmarks/bench_plot_hierarchical.py +++ b/benchmarks/bench_plot_hierarchical.py @@ -8,7 +8,6 @@ def compute_bench(samples_range, features_range): - it = 0 results = defaultdict(lambda: []) @@ -16,20 +15,17 @@ def compute_bench(samples_range, features_range): for n_samples in samples_range: for n_features in features_range: it += 1 - print('==============================') - print('Iteration %03d of %03d' % (it, max_it)) - print('n_samples %05d; n_features %02d' % (n_samples, n_features)) - print('==============================') + print("==============================") + print("Iteration %03d of %03d" % (it, max_it)) + print("n_samples %05d; n_features %02d" % (n_samples, n_features)) + print("==============================") print() data = nr.randint(-50, 51, (n_samples, n_features)) for linkage in ("single", "average", "complete", "ward"): print(linkage.capitalize()) tstart = time() - AgglomerativeClustering( - linkage=linkage, - n_clusters=10 - ).fit(data) + AgglomerativeClustering(linkage=linkage, n_clusters=10).fit(data) delta = time() - tstart print("Speed: %0.3fs" % delta) @@ -40,7 +36,7 @@ def compute_bench(samples_range, features_range): return results -if __name__ == '__main__': +if __name__ == "__main__": import matplotlib.pyplot as plt samples_range = np.linspace(1000, 15000, 8).astype(int) @@ -50,36 +46,32 @@ def compute_bench(samples_range, features_range): max_time = max([max(i) for i in [t for (label, t) in results.items()]]) - colors = plt.get_cmap('tab10')(np.linspace(0, 1, 10))[:4] + colors = plt.get_cmap("tab10")(np.linspace(0, 1, 10))[:4] lines = {linkage: None for linkage in results.keys()} fig, axs = plt.subplots(2, 2, sharex=True, sharey=True) - fig.suptitle( - 'Scikit-learn agglomerative clustering benchmark results', - fontsize=16 - ) - for c, (label, timings) in zip(colors, - sorted(results.items())): + fig.suptitle("Scikit-learn agglomerative clustering benchmark results", fontsize=16) + for c, (label, timings) in zip(colors, sorted(results.items())): timing_by_samples = np.asarray(timings).reshape( - samples_range.shape[0], - features_range.shape[0] + samples_range.shape[0], features_range.shape[0] ) for n in range(timing_by_samples.shape[1]): ax = axs.flatten()[n] - lines[label], = ax.plot( - samples_range, - timing_by_samples[:, n], - color=c, - label=label + (lines[label],) = ax.plot( + samples_range, timing_by_samples[:, n], color=c, label=label ) - ax.set_title('n_features = %d' % features_range[n]) + ax.set_title("n_features = %d" % features_range[n]) if n >= 2: - ax.set_xlabel('n_samples') + ax.set_xlabel("n_samples") if n % 2 == 0: - ax.set_ylabel('time (s)') + ax.set_ylabel("time (s)") fig.subplots_adjust(right=0.8) - fig.legend([lines[link] for link in sorted(results.keys())], - sorted(results.keys()), loc="center right", fontsize=8) + fig.legend( + [lines[link] for link in sorted(results.keys())], + sorted(results.keys()), + loc="center right", + fontsize=8, + ) plt.show() diff --git a/benchmarks/bench_plot_incremental_pca.py b/benchmarks/bench_plot_incremental_pca.py index 8579abcae3bed..49b87c8c7060a 100644 --- a/benchmarks/bench_plot_incremental_pca.py +++ b/benchmarks/bench_plot_incremental_pca.py @@ -7,17 +7,19 @@ """ -import numpy as np import gc -from time import time from collections import defaultdict +from time import time + import matplotlib.pyplot as plt +import numpy as np + from sklearn.datasets import fetch_lfw_people -from sklearn.decomposition import IncrementalPCA, PCA +from sklearn.decomposition import PCA, IncrementalPCA def plot_results(X, y, label): - plt.plot(X, y, label=label, marker='o') + plt.plot(X, y, label=label, marker="o") def benchmark(estimator, data): @@ -29,60 +31,68 @@ def benchmark(estimator, data): data_t = estimator.transform(data) data_r = estimator.inverse_transform(data_t) reconstruction_error = np.mean(np.abs(data - data_r)) - return {'time': training_time, 'error': reconstruction_error} + return {"time": training_time, "error": reconstruction_error} def plot_feature_times(all_times, batch_size, all_components, data): plt.figure() - plot_results(all_components, all_times['pca'], label="PCA") - plot_results(all_components, all_times['ipca'], - label="IncrementalPCA, bsize=%i" % batch_size) + plot_results(all_components, all_times["pca"], label="PCA") + plot_results( + all_components, all_times["ipca"], label="IncrementalPCA, bsize=%i" % batch_size + ) plt.legend(loc="upper left") - plt.suptitle("Algorithm runtime vs. n_components\n \ - LFW, size %i x %i" % data.shape) + plt.suptitle( + "Algorithm runtime vs. n_components\n LFW, size %i x %i" + % data.shape + ) plt.xlabel("Number of components (out of max %i)" % data.shape[1]) plt.ylabel("Time (seconds)") def plot_feature_errors(all_errors, batch_size, all_components, data): plt.figure() - plot_results(all_components, all_errors['pca'], label="PCA") - plot_results(all_components, all_errors['ipca'], - label="IncrementalPCA, bsize=%i" % batch_size) + plot_results(all_components, all_errors["pca"], label="PCA") + plot_results( + all_components, + all_errors["ipca"], + label="IncrementalPCA, bsize=%i" % batch_size, + ) plt.legend(loc="lower left") - plt.suptitle("Algorithm error vs. n_components\n" - "LFW, size %i x %i" % data.shape) + plt.suptitle("Algorithm error vs. n_components\nLFW, size %i x %i" % data.shape) plt.xlabel("Number of components (out of max %i)" % data.shape[1]) plt.ylabel("Mean absolute error") def plot_batch_times(all_times, n_features, all_batch_sizes, data): plt.figure() - plot_results(all_batch_sizes, all_times['pca'], label="PCA") - plot_results(all_batch_sizes, all_times['ipca'], label="IncrementalPCA") + plot_results(all_batch_sizes, all_times["pca"], label="PCA") + plot_results(all_batch_sizes, all_times["ipca"], label="IncrementalPCA") plt.legend(loc="lower left") - plt.suptitle("Algorithm runtime vs. batch_size for n_components %i\n \ - LFW, size %i x %i" % ( - n_features, data.shape[0], data.shape[1])) + plt.suptitle( + "Algorithm runtime vs. batch_size for n_components %i\n LFW," + " size %i x %i" % (n_features, data.shape[0], data.shape[1]) + ) plt.xlabel("Batch size") plt.ylabel("Time (seconds)") def plot_batch_errors(all_errors, n_features, all_batch_sizes, data): plt.figure() - plot_results(all_batch_sizes, all_errors['pca'], label="PCA") - plot_results(all_batch_sizes, all_errors['ipca'], label="IncrementalPCA") + plot_results(all_batch_sizes, all_errors["pca"], label="PCA") + plot_results(all_batch_sizes, all_errors["ipca"], label="IncrementalPCA") plt.legend(loc="lower left") - plt.suptitle("Algorithm error vs. batch_size for n_components %i\n \ - LFW, size %i x %i" % ( - n_features, data.shape[0], data.shape[1])) + plt.suptitle( + "Algorithm error vs. batch_size for n_components %i\n LFW," + " size %i x %i" % (n_features, data.shape[0], data.shape[1]) + ) plt.xlabel("Batch size") plt.ylabel("Mean absolute error") def fixed_batch_size_comparison(data): - all_features = [i.astype(int) for i in np.linspace(data.shape[1] // 10, - data.shape[1], num=5)] + all_features = [ + i.astype(int) for i in np.linspace(data.shape[1] // 10, data.shape[1], num=5) + ] batch_size = 1000 # Compare runtimes and error for fixed batch size all_times = defaultdict(list) @@ -90,53 +100,52 @@ def fixed_batch_size_comparison(data): for n_components in all_features: pca = PCA(n_components=n_components) ipca = IncrementalPCA(n_components=n_components, batch_size=batch_size) - results_dict = {k: benchmark(est, data) for k, est in [('pca', pca), - ('ipca', ipca)]} + results_dict = { + k: benchmark(est, data) for k, est in [("pca", pca), ("ipca", ipca)] + } for k in sorted(results_dict.keys()): - all_times[k].append(results_dict[k]['time']) - all_errors[k].append(results_dict[k]['error']) + all_times[k].append(results_dict[k]["time"]) + all_errors[k].append(results_dict[k]["error"]) plot_feature_times(all_times, batch_size, all_features, data) plot_feature_errors(all_errors, batch_size, all_features, data) def variable_batch_size_comparison(data): - batch_sizes = [i.astype(int) for i in np.linspace(data.shape[0] // 10, - data.shape[0], num=10)] + batch_sizes = [ + i.astype(int) for i in np.linspace(data.shape[0] // 10, data.shape[0], num=10) + ] - for n_components in [i.astype(int) for i in - np.linspace(data.shape[1] // 10, - data.shape[1], num=4)]: + for n_components in [ + i.astype(int) for i in np.linspace(data.shape[1] // 10, data.shape[1], num=4) + ]: all_times = defaultdict(list) all_errors = defaultdict(list) pca = PCA(n_components=n_components) - rpca = PCA(n_components=n_components, svd_solver='randomized', - random_state=1999) - results_dict = {k: benchmark(est, data) for k, est in [('pca', pca), - ('rpca', rpca)]} + rpca = PCA( + n_components=n_components, svd_solver="randomized", random_state=1999 + ) + results_dict = { + k: benchmark(est, data) for k, est in [("pca", pca), ("rpca", rpca)] + } # Create flat baselines to compare the variation over batch size - all_times['pca'].extend([results_dict['pca']['time']] * - len(batch_sizes)) - all_errors['pca'].extend([results_dict['pca']['error']] * - len(batch_sizes)) - all_times['rpca'].extend([results_dict['rpca']['time']] * - len(batch_sizes)) - all_errors['rpca'].extend([results_dict['rpca']['error']] * - len(batch_sizes)) + all_times["pca"].extend([results_dict["pca"]["time"]] * len(batch_sizes)) + all_errors["pca"].extend([results_dict["pca"]["error"]] * len(batch_sizes)) + all_times["rpca"].extend([results_dict["rpca"]["time"]] * len(batch_sizes)) + all_errors["rpca"].extend([results_dict["rpca"]["error"]] * len(batch_sizes)) for batch_size in batch_sizes: - ipca = IncrementalPCA(n_components=n_components, - batch_size=batch_size) - results_dict = {k: benchmark(est, data) for k, est in [('ipca', - ipca)]} - all_times['ipca'].append(results_dict['ipca']['time']) - all_errors['ipca'].append(results_dict['ipca']['error']) + ipca = IncrementalPCA(n_components=n_components, batch_size=batch_size) + results_dict = {k: benchmark(est, data) for k, est in [("ipca", ipca)]} + all_times["ipca"].append(results_dict["ipca"]["time"]) + all_errors["ipca"].append(results_dict["ipca"]["error"]) plot_batch_times(all_times, n_components, batch_sizes, data) plot_batch_errors(all_errors, n_components, batch_sizes, data) -faces = fetch_lfw_people(resize=.2, min_faces_per_person=5) + +faces = fetch_lfw_people(resize=0.2, min_faces_per_person=5) # limit dataset to 5000 people (don't care who they are!) X = faces.data[:5000] n_samples, h, w = faces.images.shape diff --git a/benchmarks/bench_plot_lasso_path.py b/benchmarks/bench_plot_lasso_path.py index a1530bb5c06c4..9acc1b4b35952 100644 --- a/benchmarks/bench_plot_lasso_path.py +++ b/benchmarks/bench_plot_lasso_path.py @@ -2,20 +2,19 @@ The input data is mostly low rank but is a fat infinite tail. """ -from collections import defaultdict + import gc import sys +from collections import defaultdict from time import time import numpy as np -from sklearn.linear_model import lars_path, lars_path_gram -from sklearn.linear_model import lasso_path from sklearn.datasets import make_regression +from sklearn.linear_model import lars_path, lars_path_gram, lasso_path def compute_bench(samples_range, features_range): - it = 0 results = defaultdict(lambda: []) @@ -24,65 +23,65 @@ def compute_bench(samples_range, features_range): for n_samples in samples_range: for n_features in features_range: it += 1 - print('====================') - print('Iteration %03d of %03d' % (it, max_it)) - print('====================') + print("====================") + print("Iteration %03d of %03d" % (it, max_it)) + print("====================") dataset_kwargs = { - 'n_samples': n_samples, - 'n_features': n_features, - 'n_informative': n_features // 10, - 'effective_rank': min(n_samples, n_features) / 10, - #'effective_rank': None, - 'bias': 0.0, + "n_samples": n_samples, + "n_features": n_features, + "n_informative": n_features // 10, + "effective_rank": min(n_samples, n_features) / 10, + # 'effective_rank': None, + "bias": 0.0, } print("n_samples: %d" % n_samples) print("n_features: %d" % n_features) X, y = make_regression(**dataset_kwargs) gc.collect() - print("benchmarking lars_path (with Gram):", end='') + print("benchmarking lars_path (with Gram):", end="") sys.stdout.flush() tstart = time() G = np.dot(X.T, X) # precomputed Gram matrix Xy = np.dot(X.T, y) - lars_path_gram(Xy=Xy, Gram=G, n_samples=y.size, method='lasso') + lars_path_gram(Xy=Xy, Gram=G, n_samples=y.size, method="lasso") delta = time() - tstart print("%0.3fs" % delta) - results['lars_path (with Gram)'].append(delta) + results["lars_path (with Gram)"].append(delta) gc.collect() - print("benchmarking lars_path (without Gram):", end='') + print("benchmarking lars_path (without Gram):", end="") sys.stdout.flush() tstart = time() - lars_path(X, y, method='lasso') + lars_path(X, y, method="lasso") delta = time() - tstart print("%0.3fs" % delta) - results['lars_path (without Gram)'].append(delta) + results["lars_path (without Gram)"].append(delta) gc.collect() - print("benchmarking lasso_path (with Gram):", end='') + print("benchmarking lasso_path (with Gram):", end="") sys.stdout.flush() tstart = time() lasso_path(X, y, precompute=True) delta = time() - tstart print("%0.3fs" % delta) - results['lasso_path (with Gram)'].append(delta) + results["lasso_path (with Gram)"].append(delta) gc.collect() - print("benchmarking lasso_path (without Gram):", end='') + print("benchmarking lasso_path (without Gram):", end="") sys.stdout.flush() tstart = time() lasso_path(X, y, precompute=False) delta = time() - tstart print("%0.3fs" % delta) - results['lasso_path (without Gram)'].append(delta) + results["lasso_path (without Gram)"].append(delta) return results -if __name__ == '__main__': - from mpl_toolkits.mplot3d import axes3d # register the 3d projection +if __name__ == "__main__": import matplotlib.pyplot as plt + from mpl_toolkits.mplot3d import axes3d # register the 3d projection # noqa: F401 samples_range = np.linspace(10, 2000, 5).astype(int) features_range = np.linspace(10, 2000, 5).astype(int) @@ -90,13 +89,12 @@ def compute_bench(samples_range, features_range): max_time = max(max(t) for t in results.values()) - fig = plt.figure('scikit-learn Lasso path benchmark results') + fig = plt.figure("scikit-learn Lasso path benchmark results") i = 1 - for c, (label, timings) in zip('bcry', sorted(results.items())): - ax = fig.add_subplot(2, 2, i, projection='3d') + for c, (label, timings) in zip("bcry", sorted(results.items())): + ax = fig.add_subplot(2, 2, i, projection="3d") X, Y = np.meshgrid(samples_range, features_range) - Z = np.asarray(timings).reshape(samples_range.shape[0], - features_range.shape[0]) + Z = np.asarray(timings).reshape(samples_range.shape[0], features_range.shape[0]) # plot the actual surface ax.plot_surface(X, Y, Z.T, cstride=1, rstride=1, color=c, alpha=0.8) @@ -105,9 +103,9 @@ def compute_bench(samples_range, features_range): # support legends (yet?) # ax.plot([1], [1], [1], color=c, label=label) - ax.set_xlabel('n_samples') - ax.set_ylabel('n_features') - ax.set_zlabel('Time (s)') + ax.set_xlabel("n_samples") + ax.set_ylabel("n_features") + ax.set_zlabel("Time (s)") ax.set_zlim3d(0.0, max_time * 1.1) ax.set_title(label) # ax.legend() diff --git a/benchmarks/bench_plot_neighbors.py b/benchmarks/bench_plot_neighbors.py index 85a8586af024c..2cedb19fb23c4 100644 --- a/benchmarks/bench_plot_neighbors.py +++ b/benchmarks/bench_plot_neighbors.py @@ -1,20 +1,21 @@ """ Plot the scaling of the nearest neighbors algorithms with k, D, and N """ + from time import time -import numpy as np import matplotlib.pyplot as plt +import numpy as np from matplotlib import ticker -from sklearn import neighbors, datasets +from sklearn import datasets, neighbors -def get_data(N, D, dataset='dense'): - if dataset == 'dense': +def get_data(N, D, dataset="dense"): + if dataset == "dense": np.random.seed(0) return np.random.random((N, D)) - elif dataset == 'digits': + elif dataset == "digits": X, _ = datasets.load_digits(return_X_y=True) i = np.argsort(X[0])[::-1] X = X[:, i] @@ -23,129 +24,121 @@ def get_data(N, D, dataset='dense'): raise ValueError("invalid dataset: %s" % dataset) -def barplot_neighbors(Nrange=2 ** np.arange(1, 11), - Drange=2 ** np.arange(7), - krange=2 ** np.arange(10), - N=1000, - D=64, - k=5, - leaf_size=30, - dataset='digits'): - algorithms = ('kd_tree', 'brute', 'ball_tree') - fiducial_values = {'N': N, - 'D': D, - 'k': k} - - #------------------------------------------------------------ +def barplot_neighbors( + Nrange=2 ** np.arange(1, 11), + Drange=2 ** np.arange(7), + krange=2 ** np.arange(10), + N=1000, + D=64, + k=5, + leaf_size=30, + dataset="digits", +): + algorithms = ("kd_tree", "brute", "ball_tree") + fiducial_values = {"N": N, "D": D, "k": k} + + # ------------------------------------------------------------ # varying N - N_results_build = {alg: np.zeros(len(Nrange)) - for alg in algorithms} - N_results_query = {alg: np.zeros(len(Nrange)) - for alg in algorithms} + N_results_build = {alg: np.zeros(len(Nrange)) for alg in algorithms} + N_results_query = {alg: np.zeros(len(Nrange)) for alg in algorithms} for i, NN in enumerate(Nrange): print("N = %i (%i out of %i)" % (NN, i + 1, len(Nrange))) X = get_data(NN, D, dataset) for algorithm in algorithms: - nbrs = neighbors.NearestNeighbors(n_neighbors=min(NN, k), - algorithm=algorithm, - leaf_size=leaf_size) + nbrs = neighbors.NearestNeighbors( + n_neighbors=min(NN, k), algorithm=algorithm, leaf_size=leaf_size + ) t0 = time() nbrs.fit(X) t1 = time() nbrs.kneighbors(X) t2 = time() - N_results_build[algorithm][i] = (t1 - t0) - N_results_query[algorithm][i] = (t2 - t1) + N_results_build[algorithm][i] = t1 - t0 + N_results_query[algorithm][i] = t2 - t1 - #------------------------------------------------------------ + # ------------------------------------------------------------ # varying D - D_results_build = {alg: np.zeros(len(Drange)) - for alg in algorithms} - D_results_query = {alg: np.zeros(len(Drange)) - for alg in algorithms} + D_results_build = {alg: np.zeros(len(Drange)) for alg in algorithms} + D_results_query = {alg: np.zeros(len(Drange)) for alg in algorithms} for i, DD in enumerate(Drange): print("D = %i (%i out of %i)" % (DD, i + 1, len(Drange))) X = get_data(N, DD, dataset) for algorithm in algorithms: - nbrs = neighbors.NearestNeighbors(n_neighbors=k, - algorithm=algorithm, - leaf_size=leaf_size) + nbrs = neighbors.NearestNeighbors( + n_neighbors=k, algorithm=algorithm, leaf_size=leaf_size + ) t0 = time() nbrs.fit(X) t1 = time() nbrs.kneighbors(X) t2 = time() - D_results_build[algorithm][i] = (t1 - t0) - D_results_query[algorithm][i] = (t2 - t1) + D_results_build[algorithm][i] = t1 - t0 + D_results_query[algorithm][i] = t2 - t1 - #------------------------------------------------------------ + # ------------------------------------------------------------ # varying k - k_results_build = {alg: np.zeros(len(krange)) - for alg in algorithms} - k_results_query = {alg: np.zeros(len(krange)) - for alg in algorithms} + k_results_build = {alg: np.zeros(len(krange)) for alg in algorithms} + k_results_query = {alg: np.zeros(len(krange)) for alg in algorithms} X = get_data(N, DD, dataset) for i, kk in enumerate(krange): print("k = %i (%i out of %i)" % (kk, i + 1, len(krange))) for algorithm in algorithms: - nbrs = neighbors.NearestNeighbors(n_neighbors=kk, - algorithm=algorithm, - leaf_size=leaf_size) + nbrs = neighbors.NearestNeighbors( + n_neighbors=kk, algorithm=algorithm, leaf_size=leaf_size + ) t0 = time() nbrs.fit(X) t1 = time() nbrs.kneighbors(X) t2 = time() - k_results_build[algorithm][i] = (t1 - t0) - k_results_query[algorithm][i] = (t2 - t1) + k_results_build[algorithm][i] = t1 - t0 + k_results_query[algorithm][i] = t2 - t1 plt.figure(figsize=(8, 11)) - for (sbplt, vals, quantity, - build_time, query_time) in [(311, Nrange, 'N', - N_results_build, - N_results_query), - (312, Drange, 'D', - D_results_build, - D_results_query), - (313, krange, 'k', - k_results_build, - k_results_query)]: - ax = plt.subplot(sbplt, yscale='log') + for sbplt, vals, quantity, build_time, query_time in [ + (311, Nrange, "N", N_results_build, N_results_query), + (312, Drange, "D", D_results_build, D_results_query), + (313, krange, "k", k_results_build, k_results_query), + ]: + ax = plt.subplot(sbplt, yscale="log") plt.grid(True) tick_vals = [] tick_labels = [] - bottom = 10 ** np.min([min(np.floor(np.log10(build_time[alg]))) - for alg in algorithms]) + bottom = 10 ** np.min( + [min(np.floor(np.log10(build_time[alg]))) for alg in algorithms] + ) for i, alg in enumerate(algorithms): xvals = 0.1 + i * (1 + len(vals)) + np.arange(len(vals)) width = 0.8 - c_bar = plt.bar(xvals, build_time[alg] - bottom, - width, bottom, color='r') - q_bar = plt.bar(xvals, query_time[alg], - width, build_time[alg], color='b') + c_bar = plt.bar(xvals, build_time[alg] - bottom, width, bottom, color="r") + q_bar = plt.bar(xvals, query_time[alg], width, build_time[alg], color="b") tick_vals += list(xvals + 0.5 * width) - tick_labels += ['%i' % val for val in vals] + tick_labels += ["%i" % val for val in vals] - plt.text((i + 0.02) / len(algorithms), 0.98, alg, - transform=ax.transAxes, - ha='left', - va='top', - bbox=dict(facecolor='w', edgecolor='w', alpha=0.5)) + plt.text( + (i + 0.02) / len(algorithms), + 0.98, + alg, + transform=ax.transAxes, + ha="left", + va="top", + bbox=dict(facecolor="w", edgecolor="w", alpha=0.5), + ) - plt.ylabel('Time (s)') + plt.ylabel("Time (s)") ax.xaxis.set_major_locator(ticker.FixedLocator(tick_vals)) ax.xaxis.set_major_formatter(ticker.FixedFormatter(tick_labels)) @@ -154,32 +147,45 @@ def barplot_neighbors(Nrange=2 ** np.arange(1, 11), label.set_rotation(-90) label.set_fontsize(10) - title_string = 'Varying %s' % quantity + title_string = "Varying %s" % quantity - descr_string = '' + descr_string = "" - for s in 'NDk': + for s in "NDk": if s == quantity: pass else: - descr_string += '%s = %i, ' % (s, fiducial_values[s]) + descr_string += "%s = %i, " % (s, fiducial_values[s]) descr_string = descr_string[:-2] - plt.text(1.01, 0.5, title_string, - transform=ax.transAxes, rotation=-90, - ha='left', va='center', fontsize=20) - - plt.text(0.99, 0.5, descr_string, - transform=ax.transAxes, rotation=-90, - ha='right', va='center') + plt.text( + 1.01, + 0.5, + title_string, + transform=ax.transAxes, + rotation=-90, + ha="left", + va="center", + fontsize=20, + ) + + plt.text( + 0.99, + 0.5, + descr_string, + transform=ax.transAxes, + rotation=-90, + ha="right", + va="center", + ) plt.gcf().suptitle("%s data set" % dataset.capitalize(), fontsize=16) - plt.figlegend((c_bar, q_bar), ('construction', 'N-point query'), - 'upper right') + plt.figlegend((c_bar, q_bar), ("construction", "N-point query"), "upper right") + -if __name__ == '__main__': - barplot_neighbors(dataset='digits') - barplot_neighbors(dataset='dense') +if __name__ == "__main__": + barplot_neighbors(dataset="digits") + barplot_neighbors(dataset="dense") plt.show() diff --git a/benchmarks/bench_plot_nmf.py b/benchmarks/bench_plot_nmf.py index 48f1dd1891392..76d1a6de8286c 100644 --- a/benchmarks/bench_plot_nmf.py +++ b/benchmarks/bench_plot_nmf.py @@ -1,34 +1,30 @@ """ Benchmarks of Non-Negative Matrix Factorization """ -# Authors: Tom Dupre la Tour (benchmark) -# Chih-Jen Linn (original projected gradient NMF implementation) -# Anthony Di Franco (projected gradient, Python and NumPy port) -# License: BSD 3 clause -from time import time +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import numbers import sys import warnings -import numbers +from time import time -import numpy as np import matplotlib.pyplot as plt -from joblib import Memory +import numpy as np import pandas +from joblib import Memory -from sklearn.utils._testing import ignore_warnings -from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.decomposition import NMF -from sklearn.decomposition._nmf import _initialize_nmf -from sklearn.decomposition._nmf import _beta_divergence -from sklearn.decomposition._nmf import _check_init +from sklearn.decomposition._nmf import _beta_divergence, _check_init, _initialize_nmf from sklearn.exceptions import ConvergenceWarning -from sklearn.utils.extmath import safe_sparse_dot, squared_norm +from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.utils import check_array +from sklearn.utils._testing import ignore_warnings +from sklearn.utils.extmath import safe_sparse_dot, squared_norm from sklearn.utils.validation import check_is_fitted, check_non_negative - -mem = Memory(cachedir='.', verbose=0) +mem = Memory(cachedir=".", verbose=0) ################### # Start of _PGNMF # @@ -41,13 +37,14 @@ def _norm(x): """Dot product-based Euclidean norm implementation - See: http://fseoane.net/blog/2011/computing-the-vector-norm/ + See: https://fa.bianp.net/blog/2011/computing-the-vector-norm/ """ return np.sqrt(squared_norm(x)) -def _nls_subproblem(X, W, H, tol, max_iter, alpha=0., l1_ratio=0., - sigma=0.01, beta=0.1): +def _nls_subproblem( + X, W, H, tol, max_iter, alpha=0.0, l1_ratio=0.0, sigma=0.01, beta=0.1 +): """Non-negative least square solver Solves a non-negative least squares subproblem using the projected gradient descent algorithm. @@ -104,7 +101,7 @@ def _nls_subproblem(X, W, H, tol, max_iter, alpha=0., l1_ratio=0., gamma = 1 for n_iter in range(1, max_iter + 1): grad = np.dot(WtW, H) - WtX - if alpha > 0 and l1_ratio == 1.: + if alpha > 0 and l1_ratio == 1.0: grad += alpha elif alpha > 0: grad += alpha * (l1_ratio + (1 - l1_ratio) * H) @@ -142,18 +139,14 @@ def _nls_subproblem(X, W, H, tol, max_iter, alpha=0., l1_ratio=0., Hp = Hn if n_iter == max_iter: - warnings.warn("Iteration limit reached in nls subproblem.", - ConvergenceWarning) + warnings.warn("Iteration limit reached in nls subproblem.", ConvergenceWarning) return H, grad, n_iter -def _fit_projected_gradient(X, W, H, tol, max_iter, nls_max_iter, alpha, - l1_ratio): - gradW = (np.dot(W, np.dot(H, H.T)) - - safe_sparse_dot(X, H.T, dense_output=True)) - gradH = (np.dot(np.dot(W.T, W), H) - - safe_sparse_dot(W.T, X, dense_output=True)) +def _fit_projected_gradient(X, W, H, tol, max_iter, nls_max_iter, alpha, l1_ratio): + gradW = np.dot(W, np.dot(H, H.T)) - safe_sparse_dot(X, H.T, dense_output=True) + gradH = np.dot(np.dot(W.T, W), H) - safe_sparse_dot(W.T, X, dense_output=True) init_grad = squared_norm(gradW) + squared_norm(gradH.T) # max(0.001, tol) to force alternating minimizations of W and H @@ -165,28 +158,31 @@ def _fit_projected_gradient(X, W, H, tol, max_iter, nls_max_iter, alpha, proj_grad_W = squared_norm(gradW * np.logical_or(gradW < 0, W > 0)) proj_grad_H = squared_norm(gradH * np.logical_or(gradH < 0, H > 0)) - if (proj_grad_W + proj_grad_H) / init_grad < tol ** 2: + if (proj_grad_W + proj_grad_H) / init_grad < tol**2: break # update W - Wt, gradWt, iterW = _nls_subproblem(X.T, H.T, W.T, tolW, nls_max_iter, - alpha=alpha, l1_ratio=l1_ratio) + Wt, gradWt, iterW = _nls_subproblem( + X.T, H.T, W.T, tolW, nls_max_iter, alpha=alpha, l1_ratio=l1_ratio + ) W, gradW = Wt.T, gradWt.T if iterW == 1: tolW = 0.1 * tolW # update H - H, gradH, iterH = _nls_subproblem(X, W, H, tolH, nls_max_iter, - alpha=alpha, l1_ratio=l1_ratio) + H, gradH, iterH = _nls_subproblem( + X, W, H, tolH, nls_max_iter, alpha=alpha, l1_ratio=l1_ratio + ) if iterH == 1: tolH = 0.1 * tolH - H[H == 0] = 0 # fix up negative zeros + H[H == 0] = 0 # fix up negative zeros if n_iter == max_iter: - Wt, _, _ = _nls_subproblem(X.T, H.T, W.T, tolW, nls_max_iter, - alpha=alpha, l1_ratio=l1_ratio) + Wt, _, _ = _nls_subproblem( + X.T, H.T, W.T, tolW, nls_max_iter, alpha=alpha, l1_ratio=l1_ratio + ) W = Wt.T return W, H, n_iter @@ -199,13 +195,30 @@ class _PGNMF(NMF): It may change or disappear without notice. """ - def __init__(self, n_components=None, solver='pg', init=None, - tol=1e-4, max_iter=200, random_state=None, - alpha=0., l1_ratio=0., nls_max_iter=10): + + def __init__( + self, + n_components=None, + solver="pg", + init=None, + tol=1e-4, + max_iter=200, + random_state=None, + alpha=0.0, + l1_ratio=0.0, + nls_max_iter=10, + ): super().__init__( - n_components=n_components, init=init, solver=solver, tol=tol, - max_iter=max_iter, random_state=random_state, alpha=alpha, - l1_ratio=l1_ratio) + n_components=n_components, + init=init, + solver=solver, + tol=tol, + max_iter=max_iter, + random_state=random_state, + alpha_W=alpha, + alpha_H=alpha, + l1_ratio=l1_ratio, + ) self.nls_max_iter = nls_max_iter def fit(self, X, y=None, **params): @@ -228,7 +241,7 @@ def fit_transform(self, X, y=None, W=None, H=None): return W def _fit_transform(self, X, y=None, W=None, H=None, update_H=True): - X = check_array(X, accept_sparse=('csr', 'csc')) + X = check_array(X, accept_sparse=("csr", "csc")) check_non_negative(X, "NMF (input X)") n_samples, n_features = X.shape @@ -236,47 +249,67 @@ def _fit_transform(self, X, y=None, W=None, H=None, update_H=True): if n_components is None: n_components = n_features - if (not isinstance(n_components, numbers.Integral) or - n_components <= 0): - raise ValueError("Number of components must be a positive integer;" - " got (n_components=%r)" % n_components) - if (not isinstance(self.max_iter, numbers.Integral) or - self.max_iter < 0): - raise ValueError("Maximum number of iterations must be a positive " - "integer; got (max_iter=%r)" % self.max_iter) + if not isinstance(n_components, numbers.Integral) or n_components <= 0: + raise ValueError( + "Number of components must be a positive integer; got (n_components=%r)" + % n_components + ) + if not isinstance(self.max_iter, numbers.Integral) or self.max_iter < 0: + raise ValueError( + "Maximum number of iterations must be a positive " + "integer; got (max_iter=%r)" % self.max_iter + ) if not isinstance(self.tol, numbers.Number) or self.tol < 0: - raise ValueError("Tolerance for stopping criteria must be " - "positive; got (tol=%r)" % self.tol) + raise ValueError( + "Tolerance for stopping criteria must be positive; got (tol=%r)" + % self.tol + ) # check W and H, or initialize them - if self.init == 'custom' and update_H: + if self.init == "custom" and update_H: _check_init(H, (n_components, n_features), "NMF (input H)") _check_init(W, (n_samples, n_components), "NMF (input W)") elif not update_H: _check_init(H, (n_components, n_features), "NMF (input H)") W = np.zeros((n_samples, n_components)) else: - W, H = _initialize_nmf(X, n_components, init=self.init, - random_state=self.random_state) + W, H = _initialize_nmf( + X, n_components, init=self.init, random_state=self.random_state + ) if update_H: # fit_transform W, H, n_iter = _fit_projected_gradient( - X, W, H, self.tol, self.max_iter, self.nls_max_iter, - self.alpha, self.l1_ratio) + X, + W, + H, + self.tol, + self.max_iter, + self.nls_max_iter, + self.alpha, + self.l1_ratio, + ) else: # transform - Wt, _, n_iter = _nls_subproblem(X.T, H.T, W.T, self.tol, - self.nls_max_iter, - alpha=self.alpha, - l1_ratio=self.l1_ratio) + Wt, _, n_iter = _nls_subproblem( + X.T, + H.T, + W.T, + self.tol, + self.nls_max_iter, + alpha=self.alpha, + l1_ratio=self.l1_ratio, + ) W = Wt.T if n_iter == self.max_iter and self.tol > 0: - warnings.warn("Maximum number of iteration %d reached. Increase it" - " to improve convergence." % self.max_iter, - ConvergenceWarning) + warnings.warn( + "Maximum number of iteration %d reached. Increase it" + " to improve convergence." % self.max_iter, + ConvergenceWarning, + ) return W, H, n_iter + ################# # End of _PGNMF # ################# @@ -287,22 +320,27 @@ def plot_results(results_df, plot_name): return None plt.figure(figsize=(16, 6)) - colors = 'bgr' - markers = 'ovs' + colors = "bgr" + markers = "ovs" ax = plt.subplot(1, 3, 1) - for i, init in enumerate(np.unique(results_df['init'])): + for i, init in enumerate(np.unique(results_df["init"])): plt.subplot(1, 3, i + 1, sharex=ax, sharey=ax) - for j, method in enumerate(np.unique(results_df['method'])): - mask = np.logical_and(results_df['init'] == init, - results_df['method'] == method) + for j, method in enumerate(np.unique(results_df["method"])): + mask = np.logical_and( + results_df["init"] == init, results_df["method"] == method + ) selected_items = results_df[mask] - plt.plot(selected_items['time'], selected_items['loss'], - color=colors[j % len(colors)], ls='-', - marker=markers[j % len(markers)], - label=method) + plt.plot( + selected_items["time"], + selected_items["loss"], + color=colors[j % len(colors)], + ls="-", + marker=markers[j % len(markers)], + label=method, + ) - plt.legend(loc=0, fontsize='x-small') + plt.legend(loc=0, fontsize="x-small") plt.xlabel("Time (s)") plt.ylabel("loss") plt.title("%s" % init) @@ -312,9 +350,10 @@ def plot_results(results_df, plot_name): @ignore_warnings(category=ConvergenceWarning) # use joblib to cache the results. # X_shape is specified in arguments for avoiding hashing X -@mem.cache(ignore=['X', 'W0', 'H0']) -def bench_one(name, X, W0, H0, X_shape, clf_type, clf_params, init, - n_components, random_state): +@mem.cache(ignore=["X", "W0", "H0"]) +def bench_one( + name, X, W0, H0, X_shape, clf_type, clf_params, init, n_components, random_state +): W = W0.copy() H = H0.copy() @@ -334,22 +373,22 @@ def run_bench(X, clfs, plot_name, n_components, tol, alpha, l1_ratio): results = [] for name, clf_type, iter_range, clf_params in clfs: print("Training %s:" % name) - for rs, init in enumerate(('nndsvd', 'nndsvdar', 'random')): + for rs, init in enumerate(("nndsvd", "nndsvdar", "random")): print(" %s %s: " % (init, " " * (8 - len(init))), end="") W, H = _initialize_nmf(X, n_components, init, 1e-6, rs) for max_iter in iter_range: - clf_params['alpha'] = alpha - clf_params['l1_ratio'] = l1_ratio - clf_params['max_iter'] = max_iter - clf_params['tol'] = tol - clf_params['random_state'] = rs - clf_params['init'] = 'custom' - clf_params['n_components'] = n_components - - this_loss, duration = bench_one(name, X, W, H, X.shape, - clf_type, clf_params, - init, n_components, rs) + clf_params["alpha"] = alpha + clf_params["l1_ratio"] = l1_ratio + clf_params["max_iter"] = max_iter + clf_params["tol"] = tol + clf_params["random_state"] = rs + clf_params["init"] = "custom" + clf_params["n_components"] = n_components + + this_loss, duration = bench_one( + name, X, W, H, X.shape, clf_type, clf_params, init, n_components, rs + ) init_name = "init='%s'" % init results.append((name, this_loss, duration, init_name)) @@ -359,8 +398,7 @@ def run_bench(X, clfs, plot_name, n_components, tol, alpha, l1_ratio): print(" ") # Use a panda dataframe to organize the results - results_df = pandas.DataFrame(results, - columns="method loss time init".split()) + results_df = pandas.DataFrame(results, columns="method loss time init".split()) print("Total time = %0.3f sec\n" % (time() - start)) # plot the results @@ -372,9 +410,11 @@ def load_20news(): print("Loading 20 newsgroups dataset") print("-----------------------------") from sklearn.datasets import fetch_20newsgroups - dataset = fetch_20newsgroups(shuffle=True, random_state=1, - remove=('headers', 'footers', 'quotes')) - vectorizer = TfidfVectorizer(max_df=0.95, min_df=2, stop_words='english') + + dataset = fetch_20newsgroups( + shuffle=True, random_state=1, remove=("headers", "footers", "quotes") + ) + vectorizer = TfidfVectorizer(max_df=0.95, min_df=2, stop_words="english") tfidf = vectorizer.fit_transform(dataset.data) return tfidf @@ -383,20 +423,22 @@ def load_faces(): print("Loading Olivetti face dataset") print("-----------------------------") from sklearn.datasets import fetch_olivetti_faces + faces = fetch_olivetti_faces(shuffle=True) return faces.data def build_clfs(cd_iters, pg_iters, mu_iters): - clfs = [("Coordinate Descent", NMF, cd_iters, {'solver': 'cd'}), - ("Projected Gradient", _PGNMF, pg_iters, {'solver': 'pg'}), - ("Multiplicative Update", NMF, mu_iters, {'solver': 'mu'}), - ] + clfs = [ + ("Coordinate Descent", NMF, cd_iters, {"solver": "cd"}), + ("Projected Gradient", _PGNMF, pg_iters, {"solver": "pg"}), + ("Multiplicative Update", NMF, mu_iters, {"solver": "mu"}), + ] return clfs -if __name__ == '__main__': - alpha = 0. +if __name__ == "__main__": + alpha = 0.0 l1_ratio = 0.5 n_components = 10 tol = 1e-15 @@ -417,6 +459,14 @@ def build_clfs(cd_iters, pg_iters, mu_iters): mu_iters = np.arange(1, 30) clfs = build_clfs(cd_iters, pg_iters, mu_iters) X_faces = load_faces() - run_bench(X_faces, clfs, plot_name, n_components, tol, alpha, l1_ratio,) + run_bench( + X_faces, + clfs, + plot_name, + n_components, + tol, + alpha, + l1_ratio, + ) plt.show() diff --git a/benchmarks/bench_plot_omp_lars.py b/benchmarks/bench_plot_omp_lars.py index bd10183565847..8a4bc9b1a34fe 100644 --- a/benchmarks/bench_plot_omp_lars.py +++ b/benchmarks/bench_plot_omp_lars.py @@ -3,18 +3,18 @@ The input data is mostly low rank but is a fat infinite tail. """ + import gc import sys from time import time import numpy as np -from sklearn.linear_model import lars_path, lars_path_gram, orthogonal_mp from sklearn.datasets import make_sparse_coded_signal +from sklearn.linear_model import lars_path, lars_path_gram, orthogonal_mp def compute_bench(samples_range, features_range): - it = 0 results = dict() @@ -27,10 +27,10 @@ def compute_bench(samples_range, features_range): for i_s, n_samples in enumerate(samples_range): for i_f, n_features in enumerate(features_range): it += 1 - n_informative = n_features / 10 - print('====================') - print('Iteration %03d of %03d' % (it, max_it)) - print('====================') + n_informative = n_features // 10 + print("====================") + print("Iteration %03d of %03d" % (it, max_it)) + print("====================") # dataset_kwargs = { # 'n_train_samples': n_samples, # 'n_test_samples': 2, @@ -41,31 +41,30 @@ def compute_bench(samples_range, features_range): # 'bias': 0.0, # } dataset_kwargs = { - 'n_samples': 1, - 'n_components': n_features, - 'n_features': n_samples, - 'n_nonzero_coefs': n_informative, - 'random_state': 0 + "n_samples": 1, + "n_components": n_features, + "n_features": n_samples, + "n_nonzero_coefs": n_informative, + "random_state": 0, } print("n_samples: %d" % n_samples) print("n_features: %d" % n_features) y, X, _ = make_sparse_coded_signal(**dataset_kwargs) - X = np.asfortranarray(X) + X = np.asfortranarray(X.T) gc.collect() - print("benchmarking lars_path (with Gram):", end='') + print("benchmarking lars_path (with Gram):", end="") sys.stdout.flush() tstart = time() G = np.dot(X.T, X) # precomputed Gram matrix Xy = np.dot(X.T, y) - lars_path_gram(Xy=Xy, Gram=G, n_samples=y.size, - max_iter=n_informative) + lars_path_gram(Xy=Xy, Gram=G, n_samples=y.size, max_iter=n_informative) delta = time() - tstart print("%0.3fs" % delta) lars_gram[i_f, i_s] = delta gc.collect() - print("benchmarking lars_path (without Gram):", end='') + print("benchmarking lars_path (without Gram):", end="") sys.stdout.flush() tstart = time() lars_path(X, y, Gram=None, max_iter=n_informative) @@ -74,49 +73,48 @@ def compute_bench(samples_range, features_range): lars[i_f, i_s] = delta gc.collect() - print("benchmarking orthogonal_mp (with Gram):", end='') + print("benchmarking orthogonal_mp (with Gram):", end="") sys.stdout.flush() tstart = time() - orthogonal_mp(X, y, precompute=True, - n_nonzero_coefs=n_informative) + orthogonal_mp(X, y, precompute=True, n_nonzero_coefs=n_informative) delta = time() - tstart print("%0.3fs" % delta) omp_gram[i_f, i_s] = delta gc.collect() - print("benchmarking orthogonal_mp (without Gram):", end='') + print("benchmarking orthogonal_mp (without Gram):", end="") sys.stdout.flush() tstart = time() - orthogonal_mp(X, y, precompute=False, - n_nonzero_coefs=n_informative) + orthogonal_mp(X, y, precompute=False, n_nonzero_coefs=n_informative) delta = time() - tstart print("%0.3fs" % delta) omp[i_f, i_s] = delta - results['time(LARS) / time(OMP)\n (w/ Gram)'] = (lars_gram / omp_gram) - results['time(LARS) / time(OMP)\n (w/o Gram)'] = (lars / omp) + results["time(LARS) / time(OMP)\n (w/ Gram)"] = lars_gram / omp_gram + results["time(LARS) / time(OMP)\n (w/o Gram)"] = lars / omp return results -if __name__ == '__main__': +if __name__ == "__main__": samples_range = np.linspace(1000, 5000, 5).astype(int) features_range = np.linspace(1000, 5000, 5).astype(int) results = compute_bench(samples_range, features_range) max_time = max(np.max(t) for t in results.values()) import matplotlib.pyplot as plt - fig = plt.figure('scikit-learn OMP vs. LARS benchmark results') + + fig = plt.figure("scikit-learn OMP vs. LARS benchmark results") for i, (label, timings) in enumerate(sorted(results.items())): - ax = fig.add_subplot(1, 2, i+1) + ax = fig.add_subplot(1, 2, i + 1) vmax = max(1 - timings.min(), -1 + timings.max()) plt.matshow(timings, fignum=False, vmin=1 - vmax, vmax=1 + vmax) - ax.set_xticklabels([''] + [str(each) for each in samples_range]) - ax.set_yticklabels([''] + [str(each) for each in features_range]) - plt.xlabel('n_samples') - plt.ylabel('n_features') + ax.set_xticklabels([""] + [str(each) for each in samples_range]) + ax.set_yticklabels([""] + [str(each) for each in features_range]) + plt.xlabel("n_samples") + plt.ylabel("n_features") plt.title(label) plt.subplots_adjust(0.1, 0.08, 0.96, 0.98, 0.4, 0.63) ax = plt.axes([0.1, 0.08, 0.8, 0.06]) - plt.colorbar(cax=ax, orientation='horizontal') + plt.colorbar(cax=ax, orientation="horizontal") plt.show() diff --git a/benchmarks/bench_plot_parallel_pairwise.py b/benchmarks/bench_plot_parallel_pairwise.py index 0fed06929bebc..5b7cf81f8fce4 100644 --- a/benchmarks/bench_plot_parallel_pairwise.py +++ b/benchmarks/bench_plot_parallel_pairwise.py @@ -1,12 +1,13 @@ -# Author: Mathieu Blondel -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import time import matplotlib.pyplot as plt +from sklearn.metrics.pairwise import pairwise_distances, pairwise_kernels from sklearn.utils import check_random_state -from sklearn.metrics.pairwise import pairwise_distances -from sklearn.metrics.pairwise import pairwise_kernels + def plot(func): random_state = check_random_state(0) @@ -25,12 +26,12 @@ def plot(func): func(X, n_jobs=-1) multi_core.append(time.time() - start) - plt.figure('scikit-learn parallel %s benchmark results' % func.__name__) + plt.figure("scikit-learn parallel %s benchmark results" % func.__name__) plt.plot(sample_sizes, one_core, label="one core") plt.plot(sample_sizes, multi_core, label="multi core") - plt.xlabel('n_samples') - plt.ylabel('Time (s)') - plt.title('Parallel %s' % func.__name__) + plt.xlabel("n_samples") + plt.ylabel("Time (s)") + plt.title("Parallel %s" % func.__name__) plt.legend() @@ -41,6 +42,7 @@ def euclidean_distances(X, n_jobs): def rbf_kernels(X, n_jobs): return pairwise_kernels(X, metric="rbf", n_jobs=n_jobs, gamma=0.1) + plot(euclidean_distances) plot(rbf_kernels) plt.show() diff --git a/benchmarks/bench_plot_polynomial_kernel_approximation.py b/benchmarks/bench_plot_polynomial_kernel_approximation.py index 2b7556f37320e..1e23e0a3c79ad 100644 --- a/benchmarks/bench_plot_polynomial_kernel_approximation.py +++ b/benchmarks/bench_plot_polynomial_kernel_approximation.py @@ -30,33 +30,34 @@ [1] Pham, N., & Pagh, R. (2013, August). Fast and scalable polynomial kernels via explicit feature maps. In Proceedings of the 19th ACM SIGKDD international conference on Knowledge discovery and data mining (pp. 239-247) -(http://chbrown.github.io/kdd-2013-usb/kdd/p239.pdf) +(https://chbrown.github.io/kdd-2013-usb/kdd/p239.pdf) [2] Charikar, M., Chen, K., & Farach-Colton, M. (2002, July). Finding frequent items in data streams. In International Colloquium on Automata, Languages, and Programming (pp. 693-703). Springer, Berlin, Heidelberg. -(http://www.vldb.org/pvldb/1/1454225.pdf) +(https://people.cs.rutgers.edu/~farach/pubs/FrequentStream.pdf) """ -# Author: Daniel Lopez-Sanchez -# License: BSD 3 clause + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause # Load data manipulation functions -from sklearn.datasets import load_digits -from sklearn.model_selection import train_test_split +# Will use this for timing results +from time import time # Some common libraries import matplotlib.pyplot as plt import numpy as np -# Will use this for timing results -from time import time - -# Import SVM classifiers and feature map approximation algorithms -from sklearn.svm import LinearSVC, SVC +from sklearn.datasets import load_digits from sklearn.kernel_approximation import Nystroem, PolynomialCountSketch +from sklearn.model_selection import train_test_split from sklearn.pipeline import Pipeline +# Import SVM classifiers and feature map approximation algorithms +from sklearn.svm import SVC, LinearSVC + # Split data in train and test sets X, y = load_digits()["data"], load_digits()["target"] X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.7) @@ -66,11 +67,11 @@ # Evaluate Linear SVM lsvm = LinearSVC().fit(X_train, y_train) -lsvm_score = 100*lsvm.score(X_test, y_test) +lsvm_score = 100 * lsvm.score(X_test, y_test) # Evaluate kernelized SVM -ksvm = SVC(kernel="poly", degree=2, gamma=1.).fit(X_train, y_train) -ksvm_score = 100*ksvm.score(X_test, y_test) +ksvm = SVC(kernel="poly", degree=2, gamma=1.0).fit(X_train, y_train) +ksvm_score = 100 * ksvm.score(X_test, y_test) # Evaluate PolynomialCountSketch + LinearSVM ps_svm_scores = [] @@ -80,11 +81,14 @@ for k in out_dims: score_avg = 0 for _ in range(n_runs): - ps_svm = Pipeline([("PS", PolynomialCountSketch(degree=2, - n_components=k)), - ("SVM", LinearSVC())]) + ps_svm = Pipeline( + [ + ("PS", PolynomialCountSketch(degree=2, n_components=k)), + ("SVM", LinearSVC()), + ] + ) score_avg += ps_svm.fit(X_train, y_train).score(X_test, y_test) - ps_svm_scores.append(100*score_avg/n_runs) + ps_svm_scores.append(100 * score_avg / n_runs) # Evaluate Nystroem + LinearSVM ny_svm_scores = [] @@ -93,23 +97,39 @@ for k in out_dims: score_avg = 0 for _ in range(n_runs): - ny_svm = Pipeline([("NY", Nystroem(kernel="poly", gamma=1., degree=2, - coef0=0, n_components=k)), - ("SVM", LinearSVC())]) + ny_svm = Pipeline( + [ + ( + "NY", + Nystroem( + kernel="poly", gamma=1.0, degree=2, coef0=0, n_components=k + ), + ), + ("SVM", LinearSVC()), + ] + ) score_avg += ny_svm.fit(X_train, y_train).score(X_test, y_test) - ny_svm_scores.append(100*score_avg/n_runs) + ny_svm_scores.append(100 * score_avg / n_runs) # Show results fig, ax = plt.subplots(figsize=(6, 4)) ax.set_title("Accuracy results") -ax.plot(out_dims, ps_svm_scores, label="PolynomialCountSketch + linear SVM", - c="orange") -ax.plot(out_dims, ny_svm_scores, label="Nystroem + linear SVM", - c="blue") -ax.plot([out_dims[0], out_dims[-1]], [lsvm_score, lsvm_score], - label="Linear SVM", c="black", dashes=[2, 2]) -ax.plot([out_dims[0], out_dims[-1]], [ksvm_score, ksvm_score], - label="Poly-kernel SVM", c="red", dashes=[2, 2]) +ax.plot(out_dims, ps_svm_scores, label="PolynomialCountSketch + linear SVM", c="orange") +ax.plot(out_dims, ny_svm_scores, label="Nystroem + linear SVM", c="blue") +ax.plot( + [out_dims[0], out_dims[-1]], + [lsvm_score, lsvm_score], + label="Linear SVM", + c="black", + dashes=[2, 2], +) +ax.plot( + [out_dims[0], out_dims[-1]], + [ksvm_score, ksvm_score], + label="Poly-kernel SVM", + c="red", + dashes=[2, 2], +) ax.legend() ax.set_xlabel("N_components for PolynomialCountSketch and Nystroem") ax.set_ylabel("Accuracy (%)") @@ -137,7 +157,7 @@ # This can take a while due to the inefficient training phase ny_svm_times = [] for k in out_dims: - ny = Nystroem(kernel="poly", gamma=1., degree=2, coef0=0, n_components=k) + ny = Nystroem(kernel="poly", gamma=1.0, degree=2, coef0=0, n_components=k) start = time() ny.fit_transform(fakeData, None) diff --git a/benchmarks/bench_plot_randomized_svd.py b/benchmarks/bench_plot_randomized_svd.py index f752eeb5e863b..e955be64cdee3 100644 --- a/benchmarks/bench_plot_randomized_svd.py +++ b/benchmarks/bench_plot_randomized_svd.py @@ -50,9 +50,10 @@ References ---------- -(1) Finding structure with randomness: Stochastic algorithms for constructing - approximate matrix decompositions - Halko, et al., 2009 https://arxiv.org/abs/0909.4061 +(1) :arxiv:`"Finding structure with randomness: + Stochastic algorithms for constructing approximate matrix decompositions." + <0909.4061>` + Halko, et al., (2009) (2) A randomized algorithm for the decomposition of matrices Per-Gunnar Martinsson, Vladimir Rokhlin and Mark Tygert @@ -62,31 +63,36 @@ A. Szlam et al. 2014 """ -# Author: Giorgio Patrini - -import numpy as np -import scipy as sp -import matplotlib.pyplot as plt +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import gc +import os.path import pickle -from time import time from collections import defaultdict -import os.path +from time import time -from sklearn.utils._arpack import _init_arpack_v0 +import matplotlib.pyplot as plt +import numpy as np +import scipy as sp + +from sklearn.datasets import ( + fetch_20newsgroups_vectorized, + fetch_lfw_people, + fetch_olivetti_faces, + fetch_openml, + fetch_rcv1, + make_low_rank_matrix, + make_sparse_uncorrelated, +) from sklearn.utils import gen_batches -from sklearn.utils.validation import check_random_state +from sklearn.utils._arpack import _init_arpack_v0 from sklearn.utils.extmath import randomized_svd -from sklearn.datasets import make_low_rank_matrix, make_sparse_uncorrelated -from sklearn.datasets import (fetch_lfw_people, - fetch_openml, - fetch_20newsgroups_vectorized, - fetch_olivetti_faces, - fetch_rcv1) +from sklearn.utils.validation import check_random_state try: import fbpca + fbpca_available = True except ImportError: fbpca_available = False @@ -103,7 +109,7 @@ # Determine when to switch to batch computation for matrix norms, # in case the reconstructed (dense) matrix is too large -MAX_MEMORY = int(2e9) +MAX_MEMORY = int(4e9) # The following datasets can be downloaded manually from: # CIFAR 10: https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz @@ -111,15 +117,24 @@ CIFAR_FOLDER = "./cifar-10-batches-py/" SVHN_FOLDER = "./SVHN/" -datasets = ['low rank matrix', 'lfw_people', 'olivetti_faces', '20newsgroups', - 'mnist_784', 'CIFAR', 'a3a', 'SVHN', 'uncorrelated matrix'] +datasets = [ + "low rank matrix", + "lfw_people", + "olivetti_faces", + "20newsgroups", + "mnist_784", + "CIFAR", + "a3a", + "SVHN", + "uncorrelated matrix", +] -big_sparse_datasets = ['big sparse matrix', 'rcv1'] +big_sparse_datasets = ["big sparse matrix", "rcv1"] def unpickle(file_name): - with open(file_name, 'rb') as fo: - return pickle.load(fo, encoding='latin1')["data"] + with open(file_name, "rb") as fo: + return pickle.load(fo, encoding="latin1")["data"] def handle_missing_dataset(file_folder): @@ -131,41 +146,45 @@ def handle_missing_dataset(file_folder): def get_data(dataset_name): print("Getting dataset: %s" % dataset_name) - if dataset_name == 'lfw_people': + if dataset_name == "lfw_people": X = fetch_lfw_people().data - elif dataset_name == '20newsgroups': + elif dataset_name == "20newsgroups": X = fetch_20newsgroups_vectorized().data[:, :100000] - elif dataset_name == 'olivetti_faces': + elif dataset_name == "olivetti_faces": X = fetch_olivetti_faces().data - elif dataset_name == 'rcv1': + elif dataset_name == "rcv1": X = fetch_rcv1().data - elif dataset_name == 'CIFAR': - if handle_missing_dataset(CIFAR_FOLDER) == "skip": + elif dataset_name == "CIFAR": + if handle_missing_dataset(CIFAR_FOLDER) == 0: return - X1 = [unpickle("%sdata_batch_%d" % (CIFAR_FOLDER, i + 1)) - for i in range(5)] + X1 = [unpickle("%sdata_batch_%d" % (CIFAR_FOLDER, i + 1)) for i in range(5)] X = np.vstack(X1) del X1 - elif dataset_name == 'SVHN': + elif dataset_name == "SVHN": if handle_missing_dataset(SVHN_FOLDER) == 0: return - X1 = sp.io.loadmat("%strain_32x32.mat" % SVHN_FOLDER)['X'] + X1 = sp.io.loadmat("%strain_32x32.mat" % SVHN_FOLDER)["X"] X2 = [X1[:, :, :, i].reshape(32 * 32 * 3) for i in range(X1.shape[3])] X = np.vstack(X2) del X1 del X2 - elif dataset_name == 'low rank matrix': - X = make_low_rank_matrix(n_samples=500, n_features=int(1e4), - effective_rank=100, tail_strength=.5, - random_state=random_state) - elif dataset_name == 'uncorrelated matrix': - X, _ = make_sparse_uncorrelated(n_samples=500, n_features=10000, - random_state=random_state) - elif dataset_name == 'big sparse matrix': + elif dataset_name == "low rank matrix": + X = make_low_rank_matrix( + n_samples=500, + n_features=int(1e4), + effective_rank=100, + tail_strength=0.5, + random_state=random_state, + ) + elif dataset_name == "uncorrelated matrix": + X, _ = make_sparse_uncorrelated( + n_samples=500, n_features=10000, random_state=random_state + ) + elif dataset_name == "big sparse matrix": sparsity = int(1e6) size = int(1e6) small_size = int(1e4) - data = np.random.normal(0, 1, int(sparsity/10)) + data = np.random.normal(0, 1, int(sparsity / 10)) data = np.repeat(data, 10) row = np.random.uniform(0, small_size, sparsity) col = np.random.uniform(0, small_size, sparsity) @@ -180,16 +199,22 @@ def get_data(dataset_name): def plot_time_vs_s(time, norm, point_labels, title): plt.figure() - colors = ['g', 'b', 'y'] + colors = ["g", "b", "y"] for i, l in enumerate(sorted(norm.keys())): if l != "fbpca": - plt.plot(time[l], norm[l], label=l, marker='o', c=colors.pop()) + plt.plot(time[l], norm[l], label=l, marker="o", c=colors.pop()) else: - plt.plot(time[l], norm[l], label=l, marker='^', c='red') + plt.plot(time[l], norm[l], label=l, marker="^", c="red") for label, x, y in zip(point_labels, list(time[l]), list(norm[l])): - plt.annotate(label, xy=(x, y), xytext=(0, -20), - textcoords='offset points', ha='right', va='bottom') + plt.annotate( + label, + xy=(x, y), + xytext=(0, -20), + textcoords="offset points", + ha="right", + va="bottom", + ) plt.legend(loc="upper right") plt.suptitle(title) plt.ylabel("norm discrepancy") @@ -201,21 +226,33 @@ def scatter_time_vs_s(time, norm, point_labels, title): size = 100 for i, l in enumerate(sorted(norm.keys())): if l != "fbpca": - plt.scatter(time[l], norm[l], label=l, marker='o', c='b', s=size) + plt.scatter(time[l], norm[l], label=l, marker="o", c="b", s=size) for label, x, y in zip(point_labels, list(time[l]), list(norm[l])): - plt.annotate(label, xy=(x, y), xytext=(0, -80), - textcoords='offset points', ha='right', - arrowprops=dict(arrowstyle="->", - connectionstyle="arc3"), - va='bottom', size=11, rotation=90) + plt.annotate( + label, + xy=(x, y), + xytext=(0, -80), + textcoords="offset points", + ha="right", + arrowprops=dict(arrowstyle="->", connectionstyle="arc3"), + va="bottom", + size=11, + rotation=90, + ) else: - plt.scatter(time[l], norm[l], label=l, marker='^', c='red', s=size) + plt.scatter(time[l], norm[l], label=l, marker="^", c="red", s=size) for label, x, y in zip(point_labels, list(time[l]), list(norm[l])): - plt.annotate(label, xy=(x, y), xytext=(0, 30), - textcoords='offset points', ha='right', - arrowprops=dict(arrowstyle="->", - connectionstyle="arc3"), - va='bottom', size=11, rotation=90) + plt.annotate( + label, + xy=(x, y), + xytext=(0, 30), + textcoords="offset points", + ha="right", + arrowprops=dict(arrowstyle="->", connectionstyle="arc3"), + va="bottom", + size=11, + rotation=90, + ) plt.legend(loc="best") plt.suptitle(title) @@ -226,32 +263,40 @@ def scatter_time_vs_s(time, norm, point_labels, title): def plot_power_iter_vs_s(power_iter, s, title): plt.figure() for l in sorted(s.keys()): - plt.plot(power_iter, s[l], label=l, marker='o') - plt.legend(loc="lower right", prop={'size': 10}) + plt.plot(power_iter, s[l], label=l, marker="o") + plt.legend(loc="lower right", prop={"size": 10}) plt.suptitle(title) plt.ylabel("norm discrepancy") plt.xlabel("n_iter") -def svd_timing(X, n_comps, n_iter, n_oversamples, - power_iteration_normalizer='auto', method=None): +def svd_timing( + X, n_comps, n_iter, n_oversamples, power_iteration_normalizer="auto", method=None +): """ Measure time for decomposition """ print("... running SVD ...") - if method is not 'fbpca': + if method != "fbpca": gc.collect() t0 = time() - U, mu, V = randomized_svd(X, n_comps, n_oversamples, n_iter, - power_iteration_normalizer, - random_state=random_state, transpose=False) + U, mu, V = randomized_svd( + X, + n_comps, + n_oversamples=n_oversamples, + n_iter=n_iter, + power_iteration_normalizer=power_iteration_normalizer, + random_state=random_state, + transpose=False, + ) call_time = time() - t0 else: gc.collect() t0 = time() # There is a different convention for l here - U, mu, V = fbpca.pca(X, n_comps, raw=True, n_iter=n_iter, - l=n_oversamples+n_comps) + U, mu, V = fbpca.pca( + X, n_comps, raw=True, n_iter=n_iter, l=n_oversamples + n_comps + ) call_time = time() - t0 return U, mu, V, call_time @@ -270,10 +315,7 @@ def norm_diff(A, norm=2, msg=True, random_state=None): if norm == 2: # s = sp.linalg.norm(A, ord=2) # slow v0 = _init_arpack_v0(min(A.shape), random_state) - value = sp.sparse.linalg.svds(A, - k=1, - return_singular_vectors=False, - v0=v0) + value = sp.sparse.linalg.svds(A, k=1, return_singular_vectors=False, v0=v0) else: if sp.sparse.issparse(A): value = sp.sparse.linalg.norm(A, ord=norm) @@ -283,36 +325,42 @@ def norm_diff(A, norm=2, msg=True, random_state=None): def scalable_frobenius_norm_discrepancy(X, U, s, V): - # if the input is not too big, just call scipy - if X.shape[0] * X.shape[1] < MAX_MEMORY: + if not sp.sparse.issparse(X) or ( + X.shape[0] * X.shape[1] * X.dtype.itemsize < MAX_MEMORY + ): + # if the input is not sparse or sparse but not too big, + # U.dot(np.diag(s).dot(V)) will fit in RAM A = X - U.dot(np.diag(s).dot(V)) - return norm_diff(A, norm='fro') + return norm_diff(A, norm="fro") print("... computing fro norm by batches...") batch_size = 1000 Vhat = np.diag(s).dot(V) - cum_norm = .0 + cum_norm = 0.0 for batch in gen_batches(X.shape[0], batch_size): M = X[batch, :] - U[batch, :].dot(Vhat) - cum_norm += norm_diff(M, norm='fro', msg=False) + cum_norm += norm_diff(M, norm="fro", msg=False) return np.sqrt(cum_norm) def bench_a(X, dataset_name, power_iter, n_oversamples, n_comps): - all_time = defaultdict(list) if enable_spectral_norm: all_spectral = defaultdict(list) X_spectral_norm = norm_diff(X, norm=2, msg=False, random_state=0) all_frobenius = defaultdict(list) - X_fro_norm = norm_diff(X, norm='fro', msg=False) + X_fro_norm = norm_diff(X, norm="fro", msg=False) for pi in power_iter: - for pm in ['none', 'LU', 'QR']: + for pm in ["none", "LU", "QR"]: print("n_iter = %d on sklearn - %s" % (pi, pm)) - U, s, V, time = svd_timing(X, n_comps, n_iter=pi, - power_iteration_normalizer=pm, - n_oversamples=n_oversamples) + U, s, V, time = svd_timing( + X, + n_comps, + n_iter=pi, + power_iteration_normalizer=pm, + n_oversamples=n_oversamples, + ) label = "sklearn - %s" % pm all_time[label].append(time) if enable_spectral_norm: @@ -325,10 +373,14 @@ def bench_a(X, dataset_name, power_iter, n_oversamples, n_comps): if fbpca_available: print("n_iter = %d on fbca" % (pi)) - U, s, V, time = svd_timing(X, n_comps, n_iter=pi, - power_iteration_normalizer=pm, - n_oversamples=n_oversamples, - method='fbpca') + U, s, V, time = svd_timing( + X, + n_comps, + n_iter=pi, + power_iteration_normalizer=pm, + n_oversamples=n_oversamples, + method="fbpca", + ) label = "fbpca" all_time[label].append(time) if enable_spectral_norm: @@ -347,10 +399,13 @@ def bench_a(X, dataset_name, power_iter, n_oversamples, n_comps): def bench_b(power_list): - n_samples, n_features = 1000, 10000 - data_params = {'n_samples': n_samples, 'n_features': n_features, - 'tail_strength': .7, 'random_state': random_state} + data_params = { + "n_samples": n_samples, + "n_features": n_features, + "tail_strength": 0.7, + "random_state": random_state, + } dataset_name = "low rank matrix %d x %d" % (n_samples, n_features) ranks = [10, 50, 100] @@ -361,19 +416,23 @@ def bench_b(power_list): X = make_low_rank_matrix(effective_rank=rank, **data_params) if enable_spectral_norm: X_spectral_norm = norm_diff(X, norm=2, msg=False, random_state=0) - X_fro_norm = norm_diff(X, norm='fro', msg=False) + X_fro_norm = norm_diff(X, norm="fro", msg=False) - for n_comp in [int(rank/2), rank, rank*2]: + for n_comp in [int(rank / 2), rank, rank * 2]: label = "rank=%d, n_comp=%d" % (rank, n_comp) print(label) for pi in power_list: - U, s, V, _ = svd_timing(X, n_comp, n_iter=pi, n_oversamples=2, - power_iteration_normalizer='LU') + U, s, V, _ = svd_timing( + X, + n_comp, + n_iter=pi, + n_oversamples=2, + power_iteration_normalizer="LU", + ) if enable_spectral_norm: A = U.dot(np.diag(s).dot(V)) all_spectral[label].append( - norm_diff(X - A, norm=2, random_state=0) / - X_spectral_norm + norm_diff(X - A, norm=2, random_state=0) / X_spectral_norm ) f = scalable_frobenius_norm_discrepancy(X, U, s, V) all_frobenius[label].append(f / X_fro_norm) @@ -398,14 +457,12 @@ def bench_c(datasets, n_comps): if enable_spectral_norm: X_spectral_norm = norm_diff(X, norm=2, msg=False, random_state=0) - X_fro_norm = norm_diff(X, norm='fro', msg=False) + X_fro_norm = norm_diff(X, norm="fro", msg=False) n_comps = np.minimum(n_comps, np.min(X.shape)) label = "sklearn" - print("%s %d x %d - %s" % - (dataset_name, X.shape[0], X.shape[1], label)) - U, s, V, time = svd_timing(X, n_comps, n_iter=2, n_oversamples=10, - method=label) + print("%s %d x %d - %s" % (dataset_name, X.shape[0], X.shape[1], label)) + U, s, V, time = svd_timing(X, n_comps, n_iter=2, n_oversamples=10, method=label) all_time[label].append(time) if enable_spectral_norm: @@ -418,10 +475,10 @@ def bench_c(datasets, n_comps): if fbpca_available: label = "fbpca" - print("%s %d x %d - %s" % - (dataset_name, X.shape[0], X.shape[1], label)) - U, s, V, time = svd_timing(X, n_comps, n_iter=2, n_oversamples=2, - method=label) + print("%s %d x %d - %s" % (dataset_name, X.shape[0], X.shape[1], label)) + U, s, V, time = svd_timing( + X, n_comps, n_iter=2, n_oversamples=2, method=label + ) all_time[label].append(time) if enable_spectral_norm: A = U.dot(np.diag(s).dot(V)) @@ -441,20 +498,27 @@ def bench_c(datasets, n_comps): scatter_time_vs_s(all_time, all_frobenius, datasets, title) -if __name__ == '__main__': +if __name__ == "__main__": random_state = check_random_state(1234) - power_iter = np.linspace(0, 6, 7, dtype=int) + power_iter = np.arange(0, 6) n_comps = 50 for dataset_name in datasets: X = get_data(dataset_name) if X is None: continue - print(" >>>>>> Benching sklearn and fbpca on %s %d x %d" % - (dataset_name, X.shape[0], X.shape[1])) - bench_a(X, dataset_name, power_iter, n_oversamples=2, - n_comps=np.minimum(n_comps, np.min(X.shape))) + print( + " >>>>>> Benching sklearn and fbpca on %s %d x %d" + % (dataset_name, X.shape[0], X.shape[1]) + ) + bench_a( + X, + dataset_name, + power_iter, + n_oversamples=2, + n_comps=np.minimum(n_comps, np.min(X.shape)), + ) print(" >>>>>> Benching on simulated low rank matrix with variable rank") bench_b(power_iter) diff --git a/benchmarks/bench_plot_svd.py b/benchmarks/bench_plot_svd.py index e58fb48c3051c..f93920cae5305 100644 --- a/benchmarks/bench_plot_svd.py +++ b/benchmarks/bench_plot_svd.py @@ -2,18 +2,19 @@ The data is mostly low rank but is a fat infinite tail. """ + import gc -from time import time -import numpy as np from collections import defaultdict +from time import time +import numpy as np from scipy.linalg import svd -from sklearn.utils.extmath import randomized_svd + from sklearn.datasets import make_low_rank_matrix +from sklearn.utils.extmath import randomized_svd def compute_bench(samples_range, features_range, n_iter=3, rank=50): - it = 0 results = defaultdict(lambda: []) @@ -22,61 +23,58 @@ def compute_bench(samples_range, features_range, n_iter=3, rank=50): for n_samples in samples_range: for n_features in features_range: it += 1 - print('====================') - print('Iteration %03d of %03d' % (it, max_it)) - print('====================') - X = make_low_rank_matrix(n_samples, n_features, - effective_rank=rank, - tail_strength=0.2) + print("====================") + print("Iteration %03d of %03d" % (it, max_it)) + print("====================") + X = make_low_rank_matrix( + n_samples, n_features, effective_rank=rank, tail_strength=0.2 + ) gc.collect() print("benchmarking scipy svd: ") tstart = time() svd(X, full_matrices=False) - results['scipy svd'].append(time() - tstart) + results["scipy svd"].append(time() - tstart) gc.collect() print("benchmarking scikit-learn randomized_svd: n_iter=0") tstart = time() randomized_svd(X, rank, n_iter=0) - results['scikit-learn randomized_svd (n_iter=0)'].append( - time() - tstart) + results["scikit-learn randomized_svd (n_iter=0)"].append(time() - tstart) gc.collect() - print("benchmarking scikit-learn randomized_svd: n_iter=%d " - % n_iter) + print("benchmarking scikit-learn randomized_svd: n_iter=%d " % n_iter) tstart = time() randomized_svd(X, rank, n_iter=n_iter) - results['scikit-learn randomized_svd (n_iter=%d)' - % n_iter].append(time() - tstart) + results["scikit-learn randomized_svd (n_iter=%d)" % n_iter].append( + time() - tstart + ) return results -if __name__ == '__main__': - from mpl_toolkits.mplot3d import axes3d # register the 3d projection +if __name__ == "__main__": import matplotlib.pyplot as plt + from mpl_toolkits.mplot3d import axes3d # register the 3d projection # noqa: F401 samples_range = np.linspace(2, 1000, 4).astype(int) features_range = np.linspace(2, 1000, 4).astype(int) results = compute_bench(samples_range, features_range) - label = 'scikit-learn singular value decomposition benchmark results' + label = "scikit-learn singular value decomposition benchmark results" fig = plt.figure(label) - ax = fig.gca(projection='3d') - for c, (label, timings) in zip('rbg', sorted(results.items())): + ax = fig.gca(projection="3d") + for c, (label, timings) in zip("rbg", sorted(results.items())): X, Y = np.meshgrid(samples_range, features_range) - Z = np.asarray(timings).reshape(samples_range.shape[0], - features_range.shape[0]) + Z = np.asarray(timings).reshape(samples_range.shape[0], features_range.shape[0]) # plot the actual surface - ax.plot_surface(X, Y, Z, rstride=8, cstride=8, alpha=0.3, - color=c) + ax.plot_surface(X, Y, Z, rstride=8, cstride=8, alpha=0.3, color=c) # dummy point plot to stick the legend to since surface plot do not # support legends (yet?) ax.plot([1], [1], [1], color=c, label=label) - ax.set_xlabel('n_samples') - ax.set_ylabel('n_features') - ax.set_zlabel('Time (s)') + ax.set_xlabel("n_samples") + ax.set_ylabel("n_features") + ax.set_zlabel("Time (s)") ax.legend() plt.show() diff --git a/benchmarks/bench_plot_ward.py b/benchmarks/bench_plot_ward.py index 01fe4f8f025aa..fe5cee201dff4 100644 --- a/benchmarks/bench_plot_ward.py +++ b/benchmarks/bench_plot_ward.py @@ -4,18 +4,17 @@ import time +import matplotlib.pyplot as plt import numpy as np from scipy.cluster import hierarchy -import matplotlib.pyplot as plt from sklearn.cluster import AgglomerativeClustering -ward = AgglomerativeClustering(n_clusters=3, linkage='ward') +ward = AgglomerativeClustering(n_clusters=3, linkage="ward") -n_samples = np.logspace(.5, 3, 9) +n_samples = np.logspace(0.5, 3, 9) n_features = np.logspace(1, 3.5, 7) -N_samples, N_features = np.meshgrid(n_samples, - n_features) +N_samples, N_features = np.meshgrid(n_samples, n_features) scikits_time = np.zeros(N_samples.shape) scipy_time = np.zeros(N_samples.shape) @@ -32,12 +31,18 @@ ratio = scikits_time / scipy_time plt.figure("scikit-learn Ward's method benchmark results") -plt.imshow(np.log(ratio), aspect='auto', origin="lower") +plt.imshow(np.log(ratio), aspect="auto", origin="lower") plt.colorbar() -plt.contour(ratio, levels=[1, ], colors='k') +plt.contour( + ratio, + levels=[ + 1, + ], + colors="k", +) plt.yticks(range(len(n_features)), n_features.astype(int)) -plt.ylabel('N features') +plt.ylabel("N features") plt.xticks(range(len(n_samples)), n_samples.astype(int)) -plt.xlabel('N samples') +plt.xlabel("N samples") plt.title("Scikit's time, in units of scipy time (log)") plt.show() diff --git a/benchmarks/bench_random_projections.py b/benchmarks/bench_random_projections.py index fb301d2ed0b00..6551de690994b 100644 --- a/benchmarks/bench_random_projections.py +++ b/benchmarks/bench_random_projections.py @@ -6,19 +6,22 @@ Benchmarks for random projections. """ + +import collections import gc -import sys import optparse +import sys from datetime import datetime -import collections import numpy as np import scipy.sparse as sp from sklearn import clone -from sklearn.random_projection import (SparseRandomProjection, - GaussianRandomProjection, - johnson_lindenstrauss_min_dim) +from sklearn.random_projection import ( + GaussianRandomProjection, + SparseRandomProjection, + johnson_lindenstrauss_min_dim, +) def type_auto_or_float(val): @@ -36,27 +39,27 @@ def type_auto_or_int(val): def compute_time(t_start, delta): - mu_second = 0.0 + 10 ** 6 # number of microseconds in a second + mu_second = 0.0 + 10**6 # number of microseconds in a second return delta.seconds + delta.microseconds / mu_second -def bench_scikit_transformer(X, transfomer): +def bench_scikit_transformer(X, transformer): gc.collect() - clf = clone(transfomer) + clf = clone(transformer) # start time t_start = datetime.now() clf.fit(X) - delta = (datetime.now() - t_start) + delta = datetime.now() - t_start # stop time time_to_fit = compute_time(t_start, delta) # start time t_start = datetime.now() clf.transform(X) - delta = (datetime.now() - t_start) + delta = datetime.now() - t_start # stop time time_to_transform = compute_time(t_start, delta) @@ -65,21 +68,30 @@ def bench_scikit_transformer(X, transfomer): # Make some random data with uniformly located non zero entries with # Gaussian distributed values -def make_sparse_random_data(n_samples, n_features, n_nonzeros, - random_state=None): +def make_sparse_random_data(n_samples, n_features, n_nonzeros, random_state=None): rng = np.random.RandomState(random_state) data_coo = sp.coo_matrix( - (rng.randn(n_nonzeros), - (rng.randint(n_samples, size=n_nonzeros), - rng.randint(n_features, size=n_nonzeros))), - shape=(n_samples, n_features)) + ( + rng.randn(n_nonzeros), + ( + rng.randint(n_samples, size=n_nonzeros), + rng.randint(n_features, size=n_nonzeros), + ), + ), + shape=(n_samples, n_features), + ) return data_coo.toarray(), data_coo.tocsr() def print_row(clf_type, time_fit, time_transform): - print("%s | %s | %s" % (clf_type.ljust(30), - ("%.4fs" % time_fit).center(12), - ("%.4fs" % time_transform).center(12))) + print( + "%s | %s | %s" + % ( + clf_type.ljust(30), + ("%.4fs" % time_fit).center(12), + ("%.4fs" % time_transform).center(12), + ) + ) if __name__ == "__main__": @@ -87,53 +99,89 @@ def print_row(clf_type, time_fit, time_transform): # Option parser ########################################################################### op = optparse.OptionParser() - op.add_option("--n-times", - dest="n_times", default=5, type=int, - help="Benchmark results are average over n_times experiments") - - op.add_option("--n-features", - dest="n_features", default=10 ** 4, type=int, - help="Number of features in the benchmarks") - - op.add_option("--n-components", - dest="n_components", default="auto", - help="Size of the random subspace." - " ('auto' or int > 0)") - - op.add_option("--ratio-nonzeros", - dest="ratio_nonzeros", default=10 ** -3, type=float, - help="Number of features in the benchmarks") - - op.add_option("--n-samples", - dest="n_samples", default=500, type=int, - help="Number of samples in the benchmarks") - - op.add_option("--random-seed", - dest="random_seed", default=13, type=int, - help="Seed used by the random number generators.") - - op.add_option("--density", - dest="density", default=1 / 3, - help="Density used by the sparse random projection." - " ('auto' or float (0.0, 1.0]") - - op.add_option("--eps", - dest="eps", default=0.5, type=float, - help="See the documentation of the underlying transformers.") - - op.add_option("--transformers", - dest="selected_transformers", - default='GaussianRandomProjection,SparseRandomProjection', - type=str, - help="Comma-separated list of transformer to benchmark. " - "Default: %default. Available: " - "GaussianRandomProjection,SparseRandomProjection") - - op.add_option("--dense", - dest="dense", - default=False, - action="store_true", - help="Set input space as a dense matrix.") + op.add_option( + "--n-times", + dest="n_times", + default=5, + type=int, + help="Benchmark results are average over n_times experiments", + ) + + op.add_option( + "--n-features", + dest="n_features", + default=10**4, + type=int, + help="Number of features in the benchmarks", + ) + + op.add_option( + "--n-components", + dest="n_components", + default="auto", + help="Size of the random subspace. ('auto' or int > 0)", + ) + + op.add_option( + "--ratio-nonzeros", + dest="ratio_nonzeros", + default=10**-3, + type=float, + help="Number of features in the benchmarks", + ) + + op.add_option( + "--n-samples", + dest="n_samples", + default=500, + type=int, + help="Number of samples in the benchmarks", + ) + + op.add_option( + "--random-seed", + dest="random_seed", + default=13, + type=int, + help="Seed used by the random number generators.", + ) + + op.add_option( + "--density", + dest="density", + default=1 / 3, + help=( + "Density used by the sparse random projection. ('auto' or float (0.0, 1.0]" + ), + ) + + op.add_option( + "--eps", + dest="eps", + default=0.5, + type=float, + help="See the documentation of the underlying transformers.", + ) + + op.add_option( + "--transformers", + dest="selected_transformers", + default="GaussianRandomProjection,SparseRandomProjection", + type=str, + help=( + "Comma-separated list of transformer to benchmark. " + "Default: %default. Available: " + "GaussianRandomProjection,SparseRandomProjection" + ), + ) + + op.add_option( + "--dense", + dest="dense", + default=False, + action="store_true", + help="Set input space as a dense matrix.", + ) (opts, args) = op.parse_args() if len(args) > 0: @@ -141,27 +189,28 @@ def print_row(clf_type, time_fit, time_transform): sys.exit(1) opts.n_components = type_auto_or_int(opts.n_components) opts.density = type_auto_or_float(opts.density) - selected_transformers = opts.selected_transformers.split(',') + selected_transformers = opts.selected_transformers.split(",") ########################################################################### # Generate dataset ########################################################################### n_nonzeros = int(opts.ratio_nonzeros * opts.n_features) - print('Dataset statics') + print("Dataset statistics") print("===========================") - print('n_samples \t= %s' % opts.n_samples) - print('n_features \t= %s' % opts.n_features) + print("n_samples \t= %s" % opts.n_samples) + print("n_features \t= %s" % opts.n_features) if opts.n_components == "auto": - print('n_components \t= %s (auto)' % - johnson_lindenstrauss_min_dim(n_samples=opts.n_samples, - eps=opts.eps)) + print( + "n_components \t= %s (auto)" + % johnson_lindenstrauss_min_dim(n_samples=opts.n_samples, eps=opts.eps) + ) else: - print('n_components \t= %s' % opts.n_components) - print('n_elements \t= %s' % (opts.n_features * opts.n_samples)) - print('n_nonzeros \t= %s per feature' % n_nonzeros) - print('ratio_nonzeros \t= %s' % opts.ratio_nonzeros) - print('') + print("n_components \t= %s" % opts.n_components) + print("n_elements \t= %s" % (opts.n_features * opts.n_samples)) + print("n_nonzeros \t= %s per feature" % n_nonzeros) + print("ratio_nonzeros \t= %s" % opts.ratio_nonzeros) + print("") ########################################################################### # Set transformer input @@ -172,10 +221,11 @@ def print_row(clf_type, time_fit, time_transform): # Set GaussianRandomProjection input gaussian_matrix_params = { "n_components": opts.n_components, - "random_state": opts.random_seed + "random_state": opts.random_seed, } - transformers["GaussianRandomProjection"] = \ - GaussianRandomProjection(**gaussian_matrix_params) + transformers["GaussianRandomProjection"] = GaussianRandomProjection( + **gaussian_matrix_params + ) ########################################################################### # Set SparseRandomProjection input @@ -186,8 +236,9 @@ def print_row(clf_type, time_fit, time_transform): "eps": opts.eps, } - transformers["SparseRandomProjection"] = \ - SparseRandomProjection(**sparse_matrix_params) + transformers["SparseRandomProjection"] = SparseRandomProjection( + **sparse_matrix_params + ) ########################################################################### # Perform benchmark @@ -195,13 +246,12 @@ def print_row(clf_type, time_fit, time_transform): time_fit = collections.defaultdict(list) time_transform = collections.defaultdict(list) - print('Benchmarks') + print("Benchmarks") print("===========================") print("Generate dataset benchmarks... ", end="") - X_dense, X_sparse = make_sparse_random_data(opts.n_samples, - opts.n_features, - n_nonzeros, - random_state=opts.random_seed) + X_dense, X_sparse = make_sparse_random_data( + opts.n_samples, opts.n_features, n_nonzeros, random_state=opts.random_seed + ) X = X_dense if opts.dense else X_sparse print("done") @@ -210,8 +260,9 @@ def print_row(clf_type, time_fit, time_transform): for iteration in range(opts.n_times): print("\titer %s..." % iteration, end="") - time_to_fit, time_to_transform = bench_scikit_transformer(X_dense, - transformers[name]) + time_to_fit, time_to_transform = bench_scikit_transformer( + X_dense, transformers[name] + ) time_fit[name].append(time_to_fit) time_transform[name].append(time_to_transform) print("done") @@ -224,27 +275,30 @@ def print_row(clf_type, time_fit, time_transform): print("Script arguments") print("===========================") arguments = vars(opts) - print("%s \t | %s " % ("Arguments".ljust(16), - "Value".center(12),)) + print( + "%s \t | %s " + % ( + "Arguments".ljust(16), + "Value".center(12), + ) + ) print(25 * "-" + ("|" + "-" * 14) * 1) for key, value in arguments.items(): - print("%s \t | %s " % (str(key).ljust(16), - str(value).strip().center(12))) + print("%s \t | %s " % (str(key).ljust(16), str(value).strip().center(12))) print("") print("Transformer performance:") print("===========================") print("Results are averaged over %s repetition(s)." % opts.n_times) print("") - print("%s | %s | %s" % ("Transformer".ljust(30), - "fit".center(12), - "transform".center(12))) + print( + "%s | %s | %s" + % ("Transformer".ljust(30), "fit".center(12), "transform".center(12)) + ) print(31 * "-" + ("|" + "-" * 14) * 2) for name in sorted(selected_transformers): - print_row(name, - np.mean(time_fit[name]), - np.mean(time_transform[name])) + print_row(name, np.mean(time_fit[name]), np.mean(time_transform[name])) print("") print("") diff --git a/benchmarks/bench_rcv1_logreg_convergence.py b/benchmarks/bench_rcv1_logreg_convergence.py index 051496c4483a2..27e730736a3de 100644 --- a/benchmarks/bench_rcv1_logreg_convergence.py +++ b/benchmarks/bench_rcv1_logreg_convergence.py @@ -1,16 +1,15 @@ -# Authors: Tom Dupre la Tour -# Olivier Grisel -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import matplotlib.pyplot as plt -from joblib import Memory -import numpy as np import gc import time -from sklearn.linear_model import (LogisticRegression, SGDClassifier) +import matplotlib.pyplot as plt +import numpy as np +from joblib import Memory + from sklearn.datasets import fetch_rcv1 +from sklearn.linear_model import LogisticRegression, SGDClassifier from sklearn.linear_model._sag import get_auto_step_size try: @@ -18,16 +17,16 @@ except ImportError: lightning_clf = None -m = Memory(cachedir='.', verbose=0) +m = Memory(cachedir=".", verbose=0) # compute logistic loss def get_loss(w, intercept, myX, myy, C): n_samples = myX.shape[0] w = w.ravel() - p = np.mean(np.log(1. + np.exp(-myy * (myX.dot(w) + intercept)))) - print("%f + %f" % (p, w.dot(w) / 2. / C / n_samples)) - p += w.dot(w) / 2. / C / n_samples + p = np.mean(np.log(1.0 + np.exp(-myy * (myX.dot(w) + intercept)))) + print("%f + %f" % (p, w.dot(w) / 2.0 / C / n_samples)) + p += w.dot(w) / 2.0 / C / n_samples return p @@ -39,7 +38,7 @@ def bench_one(name, clf_type, clf_params, n_iter): clf = clf_type(**clf_params) try: clf.set_params(max_iter=n_iter, random_state=42) - except: + except Exception: clf.set_params(n_iter=n_iter, random_state=42) st = time.time() @@ -48,13 +47,13 @@ def bench_one(name, clf_type, clf_params, n_iter): try: C = 1.0 / clf.alpha / n_samples - except: + except Exception: C = clf.C try: intercept = clf.intercept_ - except: - intercept = 0. + except Exception: + intercept = 0.0 train_loss = get_loss(clf.coef_, intercept, X, y, C) train_score = clf.score(X, y) @@ -65,8 +64,15 @@ def bench_one(name, clf_type, clf_params, n_iter): def bench(clfs): - for (name, clf, iter_range, train_losses, train_scores, - test_scores, durations) in clfs: + for ( + name, + clf, + iter_range, + train_losses, + train_scores, + test_scores, + durations, + ) in clfs: print("training %s" % name) clf_type = type(clf) clf_params = clf.get_params() @@ -75,7 +81,8 @@ def bench(clfs): gc.collect() train_loss, train_score, test_score, duration = bench_one( - name, clf_type, clf_params, n_iter) + name, clf_type, clf_params, n_iter + ) train_losses.append(train_loss) train_scores.append(train_score) @@ -94,8 +101,8 @@ def bench(clfs): def plot_train_losses(clfs): plt.figure() - for (name, _, _, train_losses, _, _, durations) in clfs: - plt.plot(durations, train_losses, '-o', label=name) + for name, _, _, train_losses, _, _, durations in clfs: + plt.plot(durations, train_losses, "-o", label=name) plt.legend(loc=0) plt.xlabel("seconds") plt.ylabel("train loss") @@ -103,8 +110,8 @@ def plot_train_losses(clfs): def plot_train_scores(clfs): plt.figure() - for (name, _, _, _, train_scores, _, durations) in clfs: - plt.plot(durations, train_scores, '-o', label=name) + for name, _, _, _, train_scores, _, durations in clfs: + plt.plot(durations, train_scores, "-o", label=name) plt.legend(loc=0) plt.xlabel("seconds") plt.ylabel("train score") @@ -113,8 +120,8 @@ def plot_train_scores(clfs): def plot_test_scores(clfs): plt.figure() - for (name, _, _, _, _, test_scores, durations) in clfs: - plt.plot(durations, test_scores, '-o', label=name) + for name, _, _, _, _, test_scores, durations in clfs: + plt.plot(durations, test_scores, "-o", label=name) plt.legend(loc=0) plt.xlabel("seconds") plt.ylabel("test score") @@ -124,16 +131,16 @@ def plot_test_scores(clfs): def plot_dloss(clfs): plt.figure() pobj_final = [] - for (name, _, _, train_losses, _, _, durations) in clfs: + for name, _, _, train_losses, _, _, durations in clfs: pobj_final.append(train_losses[-1]) indices = np.argsort(pobj_final) pobj_best = pobj_final[indices[0]] - for (name, _, _, train_losses, _, _, durations) in clfs: + for name, _, _, train_losses, _, _, durations in clfs: log_pobj = np.log(abs(np.array(train_losses) - pobj_best)) / np.log(10) - plt.plot(durations, log_pobj, '-o', label=name) + plt.plot(durations, log_pobj, "-o", label=name) plt.legend(loc=0) plt.xlabel("seconds") plt.ylabel("log(best - train_loss)") @@ -141,19 +148,20 @@ def plot_dloss(clfs): def get_max_squared_sum(X): """Get the maximum row-wise sum of squares""" - return np.sum(X ** 2, axis=1).max() + return np.sum(X**2, axis=1).max() + rcv1 = fetch_rcv1() X = rcv1.data n_samples, n_features = X.shape # consider the binary classification problem 'CCAT' vs the rest -ccat_idx = rcv1.target_names.tolist().index('CCAT') +ccat_idx = rcv1.target_names.tolist().index("CCAT") y = rcv1.target.tocsc()[:, ccat_idx].toarray().ravel().astype(np.float64) y[y == 0] = -1 # parameters -C = 1. +C = 1.0 fit_intercept = True tol = 1.0e-14 @@ -166,51 +174,116 @@ def get_max_squared_sum(X): sag_iter_range = list(range(1, 37, 3)) clfs = [ - ("LR-liblinear", - LogisticRegression(C=C, tol=tol, - solver="liblinear", fit_intercept=fit_intercept, - intercept_scaling=1), - liblinear_iter_range, [], [], [], []), - ("LR-liblinear-dual", - LogisticRegression(C=C, tol=tol, dual=True, - solver="liblinear", fit_intercept=fit_intercept, - intercept_scaling=1), - liblinear_dual_iter_range, [], [], [], []), - ("LR-SAG", - LogisticRegression(C=C, tol=tol, - solver="sag", fit_intercept=fit_intercept), - sag_iter_range, [], [], [], []), - ("LR-newton-cg", - LogisticRegression(C=C, tol=tol, solver="newton-cg", - fit_intercept=fit_intercept), - newton_iter_range, [], [], [], []), - ("LR-lbfgs", - LogisticRegression(C=C, tol=tol, - solver="lbfgs", fit_intercept=fit_intercept), - lbfgs_iter_range, [], [], [], []), - ("SGD", - SGDClassifier(alpha=1.0 / C / n_samples, penalty='l2', loss='log', - fit_intercept=fit_intercept, verbose=0), - sgd_iter_range, [], [], [], [])] + ( + "LR-liblinear", + LogisticRegression( + C=C, + tol=tol, + solver="liblinear", + fit_intercept=fit_intercept, + intercept_scaling=1, + ), + liblinear_iter_range, + [], + [], + [], + [], + ), + ( + "LR-liblinear-dual", + LogisticRegression( + C=C, + tol=tol, + dual=True, + solver="liblinear", + fit_intercept=fit_intercept, + intercept_scaling=1, + ), + liblinear_dual_iter_range, + [], + [], + [], + [], + ), + ( + "LR-SAG", + LogisticRegression(C=C, tol=tol, solver="sag", fit_intercept=fit_intercept), + sag_iter_range, + [], + [], + [], + [], + ), + ( + "LR-newton-cg", + LogisticRegression( + C=C, tol=tol, solver="newton-cg", fit_intercept=fit_intercept + ), + newton_iter_range, + [], + [], + [], + [], + ), + ( + "LR-lbfgs", + LogisticRegression(C=C, tol=tol, solver="lbfgs", fit_intercept=fit_intercept), + lbfgs_iter_range, + [], + [], + [], + [], + ), + ( + "SGD", + SGDClassifier( + alpha=1.0 / C / n_samples, + penalty="l2", + loss="log_loss", + fit_intercept=fit_intercept, + verbose=0, + ), + sgd_iter_range, + [], + [], + [], + [], + ), +] if lightning_clf is not None and not fit_intercept: - alpha = 1. / C / n_samples + alpha = 1.0 / C / n_samples # compute the same step_size than in LR-sag max_squared_sum = get_max_squared_sum(X) - step_size = get_auto_step_size(max_squared_sum, alpha, "log", - fit_intercept) + step_size = get_auto_step_size(max_squared_sum, alpha, "log", fit_intercept) clfs.append( - ("Lightning-SVRG", - lightning_clf.SVRGClassifier(alpha=alpha, eta=step_size, - tol=tol, loss="log"), - sag_iter_range, [], [], [], [])) + ( + "Lightning-SVRG", + lightning_clf.SVRGClassifier( + alpha=alpha, eta=step_size, tol=tol, loss="log" + ), + sag_iter_range, + [], + [], + [], + [], + ) + ) clfs.append( - ("Lightning-SAG", - lightning_clf.SAGClassifier(alpha=alpha, eta=step_size, - tol=tol, loss="log"), - sag_iter_range, [], [], [], [])) + ( + "Lightning-SAG", + lightning_clf.SAGClassifier( + alpha=alpha, eta=step_size, tol=tol, loss="log" + ), + sag_iter_range, + [], + [], + [], + [], + ) + ) # We keep only 200 features, to have a dense dataset, # and compare to lightning SAG, which seems incorrect in the sparse case. diff --git a/benchmarks/bench_saga.py b/benchmarks/bench_saga.py index 492527d7e4c67..97d4ba7b4b75b 100644 --- a/benchmarks/bench_saga.py +++ b/benchmarks/bench_saga.py @@ -3,45 +3,61 @@ Benchmarks of sklearn SAGA vs lightning SAGA vs Liblinear. Shows the gain in using multinomial logistic regression in term of learning time. """ + import json -import time import os +import time -from joblib import Parallel -from sklearn.utils.fixes import delayed import matplotlib.pyplot as plt import numpy as np -from sklearn.datasets import fetch_rcv1, load_iris, load_digits, \ - fetch_20newsgroups_vectorized +from sklearn.datasets import ( + fetch_20newsgroups_vectorized, + fetch_rcv1, + load_digits, + load_iris, +) from sklearn.linear_model import LogisticRegression from sklearn.metrics import log_loss from sklearn.model_selection import train_test_split +from sklearn.multiclass import OneVsRestClassifier from sklearn.preprocessing import LabelBinarizer, LabelEncoder from sklearn.utils.extmath import safe_sparse_dot, softmax - - -def fit_single(solver, X, y, penalty='l2', single_target=True, C=1, - max_iter=10, skip_slow=False, dtype=np.float64): - if skip_slow and solver == 'lightning' and penalty == 'l1': - print('skip_slowping l1 logistic regression with solver lightning.') +from sklearn.utils.parallel import Parallel, delayed + + +def fit_single( + solver, + X, + y, + penalty="l2", + single_target=True, + C=1, + max_iter=10, + skip_slow=False, + dtype=np.float64, +): + if skip_slow and solver == "lightning" and penalty == "l1": + print("skip_slowping l1 logistic regression with solver lightning.") return - print('Solving %s logistic regression with penalty %s, solver %s.' - % ('binary' if single_target else 'multinomial', - penalty, solver)) + print( + "Solving %s logistic regression with penalty %s, solver %s." + % ("binary" if single_target else "multinomial", penalty, solver) + ) - if solver == 'lightning': + if solver == "lightning": from lightning.classification import SAGAClassifier - if single_target or solver not in ['sag', 'saga']: - multi_class = 'ovr' + if single_target or solver not in ["sag", "saga"]: + multi_class = "ovr" else: - multi_class = 'multinomial' + multi_class = "multinomial" X = X.astype(dtype) y = y.astype(dtype) - X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42, - stratify=y) + X_train, X_test, y_train, y_test = train_test_split( + X, y, random_state=42, stratify=y + ) n_samples = X_train.shape[0] n_classes = np.unique(y_train).shape[0] test_scores = [1] @@ -49,32 +65,46 @@ def fit_single(solver, X, y, penalty='l2', single_target=True, C=1, accuracies = [1 / n_classes] times = [0] - if penalty == 'l2': - alpha = 1. / (C * n_samples) + if penalty == "l2": + alpha = 1.0 / (C * n_samples) beta = 0 lightning_penalty = None else: - alpha = 0. - beta = 1. / (C * n_samples) - lightning_penalty = 'l1' + alpha = 0.0 + beta = 1.0 / (C * n_samples) + lightning_penalty = "l1" for this_max_iter in range(1, max_iter + 1, 2): - print('[%s, %s, %s] Max iter: %s' % - ('binary' if single_target else 'multinomial', - penalty, solver, this_max_iter)) - if solver == 'lightning': - lr = SAGAClassifier(loss='log', alpha=alpha, beta=beta, - penalty=lightning_penalty, - tol=-1, max_iter=this_max_iter) + print( + "[%s, %s, %s] Max iter: %s" + % ( + "binary" if single_target else "multinomial", + penalty, + solver, + this_max_iter, + ) + ) + if solver == "lightning": + lr = SAGAClassifier( + loss="log", + alpha=alpha, + beta=beta, + penalty=lightning_penalty, + tol=-1, + max_iter=this_max_iter, + ) else: - lr = LogisticRegression(solver=solver, - multi_class=multi_class, - C=C, - penalty=penalty, - fit_intercept=False, tol=0, - max_iter=this_max_iter, - random_state=42, - ) + lr = LogisticRegression( + solver=solver, + C=C, + penalty=penalty, + fit_intercept=False, + tol=0, + max_iter=this_max_iter, + random_state=42, + ) + if multi_class == "ovr": + lr = OneVsRestClassifier(lr) # Makes cpu cache even for all fit calls X_train.max() @@ -84,15 +114,18 @@ def fit_single(solver, X, y, penalty='l2', single_target=True, C=1, train_time = time.clock() - t0 scores = [] - for (X, y) in [(X_train, y_train), (X_test, y_test)]: + for X, y in [(X_train, y_train), (X_test, y_test)]: try: y_pred = lr.predict_proba(X) except NotImplementedError: # Lightning predict_proba is not implemented for n_classes > 2 y_pred = _predict_proba(lr, X) + if isinstance(lr, OneVsRestClassifier): + coef = np.concatenate([est.coef_ for est in lr.estimators_]) + else: + coef = lr.coef_ score = log_loss(y, y_pred, normalize=False) / n_samples - score += (0.5 * alpha * np.sum(lr.coef_ ** 2) + - beta * np.sum(np.abs(lr.coef_))) + score += 0.5 * alpha * np.sum(coef**2) + beta * np.sum(np.abs(coef)) scores.append(score) train_score, test_score = tuple(scores) @@ -106,21 +139,29 @@ def fit_single(solver, X, y, penalty='l2', single_target=True, C=1, def _predict_proba(lr, X): + """Predict proba for lightning for n_classes >=3.""" pred = safe_sparse_dot(X, lr.coef_.T) if hasattr(lr, "intercept_"): pred += lr.intercept_ return softmax(pred) -def exp(solvers, penalty, single_target, - n_samples=30000, max_iter=20, - dataset='rcv1', n_jobs=1, skip_slow=False): +def exp( + solvers, + penalty, + single_target, + n_samples=30000, + max_iter=20, + dataset="rcv1", + n_jobs=1, + skip_slow=False, +): dtypes_mapping = { "float64": np.float64, "float32": np.float32, } - if dataset == 'rcv1': + if dataset == "rcv1": rcv1 = fetch_rcv1() lbin = LabelBinarizer() @@ -137,17 +178,17 @@ def exp(solvers, penalty, single_target, y_n[y <= 16] = 0 y = y_n - elif dataset == 'digits': + elif dataset == "digits": X, y = load_digits(return_X_y=True) if single_target: y_n = y.copy() y_n[y < 5] = 1 y_n[y >= 5] = 0 y = y_n - elif dataset == 'iris': + elif dataset == "iris": iris = load_iris() X, y = iris.data, iris.target - elif dataset == '20newspaper': + elif dataset == "20newspaper": ng = fetch_20newsgroups_vectorized() X = ng.data y = ng.target @@ -161,44 +202,55 @@ def exp(solvers, penalty, single_target, y = y[:n_samples] out = Parallel(n_jobs=n_jobs, mmap_mode=None)( - delayed(fit_single)(solver, X, y, - penalty=penalty, single_target=single_target, - dtype=dtype, - C=1, max_iter=max_iter, skip_slow=skip_slow) + delayed(fit_single)( + solver, + X, + y, + penalty=penalty, + single_target=single_target, + dtype=dtype, + C=1, + max_iter=max_iter, + skip_slow=skip_slow, + ) for solver in solvers - for dtype in dtypes_mapping.values()) + for dtype in dtypes_mapping.values() + ) res = [] idx = 0 for dtype_name in dtypes_mapping.keys(): for solver in solvers: - if not (skip_slow and - solver == 'lightning' and - penalty == 'l1'): + if not (skip_slow and solver == "lightning" and penalty == "l1"): lr, times, train_scores, test_scores, accuracies = out[idx] - this_res = dict(solver=solver, penalty=penalty, - dtype=dtype_name, - single_target=single_target, - times=times, train_scores=train_scores, - test_scores=test_scores, - accuracies=accuracies) + this_res = dict( + solver=solver, + penalty=penalty, + dtype=dtype_name, + single_target=single_target, + times=times, + train_scores=train_scores, + test_scores=test_scores, + accuracies=accuracies, + ) res.append(this_res) idx += 1 - with open('bench_saga.json', 'w+') as f: + with open("bench_saga.json", "w+") as f: json.dump(res, f) def plot(outname=None): import pandas as pd - with open('bench_saga.json', 'r') as f: + + with open("bench_saga.json", "r") as f: f = json.load(f) res = pd.DataFrame(f) - res.set_index(['single_target'], inplace=True) + res.set_index(["single_target"], inplace=True) - grouped = res.groupby(level=['single_target']) + grouped = res.groupby(level=["single_target"]) - colors = {'saga': 'C0', 'liblinear': 'C1', 'lightning': 'C2'} + colors = {"saga": "C0", "liblinear": "C1", "lightning": "C2"} linestyles = {"float32": "--", "float64": "-"} alpha = {"float64": 0.5, "float32": 1} @@ -207,93 +259,122 @@ def plot(outname=None): fig, axes = plt.subplots(figsize=(12, 4), ncols=4) ax = axes[0] - for scores, times, solver, dtype in zip(group['train_scores'], - group['times'], - group['solver'], - group["dtype"]): - ax.plot(times, scores, label="%s - %s" % (solver, dtype), - color=colors[solver], - alpha=alpha[dtype], - marker=".", - linestyle=linestyles[dtype]) - ax.axvline(times[-1], color=colors[solver], - alpha=alpha[dtype], - linestyle=linestyles[dtype]) - ax.set_xlabel('Time (s)') - ax.set_ylabel('Training objective (relative to min)') - ax.set_yscale('log') + for scores, times, solver, dtype in zip( + group["train_scores"], group["times"], group["solver"], group["dtype"] + ): + ax.plot( + times, + scores, + label="%s - %s" % (solver, dtype), + color=colors[solver], + alpha=alpha[dtype], + marker=".", + linestyle=linestyles[dtype], + ) + ax.axvline( + times[-1], + color=colors[solver], + alpha=alpha[dtype], + linestyle=linestyles[dtype], + ) + ax.set_xlabel("Time (s)") + ax.set_ylabel("Training objective (relative to min)") + ax.set_yscale("log") ax = axes[1] - for scores, times, solver, dtype in zip(group['test_scores'], - group['times'], - group['solver'], - group["dtype"]): - ax.plot(times, scores, label=solver, color=colors[solver], - linestyle=linestyles[dtype], - marker=".", - alpha=alpha[dtype]) - ax.axvline(times[-1], color=colors[solver], - alpha=alpha[dtype], - linestyle=linestyles[dtype]) - - ax.set_xlabel('Time (s)') - ax.set_ylabel('Test objective (relative to min)') - ax.set_yscale('log') + for scores, times, solver, dtype in zip( + group["test_scores"], group["times"], group["solver"], group["dtype"] + ): + ax.plot( + times, + scores, + label=solver, + color=colors[solver], + linestyle=linestyles[dtype], + marker=".", + alpha=alpha[dtype], + ) + ax.axvline( + times[-1], + color=colors[solver], + alpha=alpha[dtype], + linestyle=linestyles[dtype], + ) + + ax.set_xlabel("Time (s)") + ax.set_ylabel("Test objective (relative to min)") + ax.set_yscale("log") ax = axes[2] - for accuracy, times, solver, dtype in zip(group['accuracies'], - group['times'], - group['solver'], - group["dtype"]): - ax.plot(times, accuracy, label="%s - %s" % (solver, dtype), - alpha=alpha[dtype], - marker=".", - color=colors[solver], linestyle=linestyles[dtype]) - ax.axvline(times[-1], color=colors[solver], - alpha=alpha[dtype], - linestyle=linestyles[dtype]) - - ax.set_xlabel('Time (s)') - ax.set_ylabel('Test accuracy') + for accuracy, times, solver, dtype in zip( + group["accuracies"], group["times"], group["solver"], group["dtype"] + ): + ax.plot( + times, + accuracy, + label="%s - %s" % (solver, dtype), + alpha=alpha[dtype], + marker=".", + color=colors[solver], + linestyle=linestyles[dtype], + ) + ax.axvline( + times[-1], + color=colors[solver], + alpha=alpha[dtype], + linestyle=linestyles[dtype], + ) + + ax.set_xlabel("Time (s)") + ax.set_ylabel("Test accuracy") ax.legend() - name = 'single_target' if single_target else 'multi_target' - name += '_%s' % penalty + name = "single_target" if single_target else "multi_target" + name += "_%s" % penalty plt.suptitle(name) if outname is None: - outname = name + '.png' + outname = name + ".png" fig.tight_layout() fig.subplots_adjust(top=0.9) ax = axes[3] - for scores, times, solver, dtype in zip(group['train_scores'], - group['times'], - group['solver'], - group["dtype"]): - ax.plot(np.arange(len(scores)), - scores, label="%s - %s" % (solver, dtype), - marker=".", - alpha=alpha[dtype], - color=colors[solver], linestyle=linestyles[dtype]) + for scores, times, solver, dtype in zip( + group["train_scores"], group["times"], group["solver"], group["dtype"] + ): + ax.plot( + np.arange(len(scores)), + scores, + label="%s - %s" % (solver, dtype), + marker=".", + alpha=alpha[dtype], + color=colors[solver], + linestyle=linestyles[dtype], + ) ax.set_yscale("log") - ax.set_xlabel('# iterations') - ax.set_ylabel('Objective function') + ax.set_xlabel("# iterations") + ax.set_ylabel("Objective function") ax.legend() plt.savefig(outname) -if __name__ == '__main__': - solvers = ['saga', 'liblinear', 'lightning'] - penalties = ['l1', 'l2'] +if __name__ == "__main__": + solvers = ["saga", "liblinear", "lightning"] + penalties = ["l1", "l2"] n_samples = [100000, 300000, 500000, 800000, None] single_target = True for penalty in penalties: for n_sample in n_samples: - exp(solvers, penalty, single_target, - n_samples=n_sample, n_jobs=1, - dataset='rcv1', max_iter=10) + exp( + solvers, + penalty, + single_target, + n_samples=n_sample, + n_jobs=1, + dataset="rcv1", + max_iter=10, + ) if n_sample is not None: outname = "figures/saga_%s_%d.png" % (penalty, n_sample) else: diff --git a/benchmarks/bench_sample_without_replacement.py b/benchmarks/bench_sample_without_replacement.py index fcd41640843e7..39cf1a11ffed6 100644 --- a/benchmarks/bench_sample_without_replacement.py +++ b/benchmarks/bench_sample_without_replacement.py @@ -2,21 +2,22 @@ Benchmarks for sampling without replacement of integer. """ + import gc -import sys +import operator import optparse +import random +import sys from datetime import datetime -import operator import matplotlib.pyplot as plt import numpy as np -import random from sklearn.utils.random import sample_without_replacement def compute_time(t_start, delta): - mu_second = 0.0 + 10 ** 6 # number of microseconds in a second + mu_second = 0.0 + 10**6 # number of microseconds in a second return delta.seconds + delta.microseconds / mu_second @@ -26,38 +27,57 @@ def bench_sample(sampling, n_population, n_samples): # start time t_start = datetime.now() sampling(n_population, n_samples) - delta = (datetime.now() - t_start) + delta = datetime.now() - t_start # stop time time = compute_time(t_start, delta) return time + if __name__ == "__main__": ########################################################################### # Option parser ########################################################################### op = optparse.OptionParser() - op.add_option("--n-times", - dest="n_times", default=5, type=int, - help="Benchmark results are average over n_times experiments") - - op.add_option("--n-population", - dest="n_population", default=100000, type=int, - help="Size of the population to sample from.") - - op.add_option("--n-step", - dest="n_steps", default=5, type=int, - help="Number of step interval between 0 and n_population.") - - default_algorithms = "custom-tracking-selection,custom-auto," \ - "custom-reservoir-sampling,custom-pool,"\ - "python-core-sample,numpy-permutation" - - op.add_option("--algorithm", - dest="selected_algorithm", - default=default_algorithms, - type=str, - help="Comma-separated list of transformer to benchmark. " - "Default: %default. \nAvailable: %default") + op.add_option( + "--n-times", + dest="n_times", + default=5, + type=int, + help="Benchmark results are average over n_times experiments", + ) + + op.add_option( + "--n-population", + dest="n_population", + default=100000, + type=int, + help="Size of the population to sample from.", + ) + + op.add_option( + "--n-step", + dest="n_steps", + default=5, + type=int, + help="Number of step interval between 0 and n_population.", + ) + + default_algorithms = ( + "custom-tracking-selection,custom-auto," + "custom-reservoir-sampling,custom-pool," + "python-core-sample,numpy-permutation" + ) + + op.add_option( + "--algorithm", + dest="selected_algorithm", + default=default_algorithms, + type=str, + help=( + "Comma-separated list of transformer to benchmark. " + "Default: %default. \nAvailable: %default" + ), + ) # op.add_option("--random-seed", # dest="random_seed", default=13, type=int, @@ -68,11 +88,13 @@ def bench_sample(sampling, n_population, n_samples): op.error("this script takes no arguments.") sys.exit(1) - selected_algorithm = opts.selected_algorithm.split(',') + selected_algorithm = opts.selected_algorithm.split(",") for key in selected_algorithm: - if key not in default_algorithms.split(','): - raise ValueError("Unknown sampling algorithm \"%s\" not in (%s)." - % (key, default_algorithms)) + if key not in default_algorithms.split(","): + raise ValueError( + 'Unknown sampling algorithm "%s" not in (%s).' + % (key, default_algorithms) + ) ########################################################################### # List sampling algorithm @@ -84,66 +106,73 @@ def bench_sample(sampling, n_population, n_samples): ########################################################################### # Set Python core input - sampling_algorithm["python-core-sample"] = \ - lambda n_population, n_sample: \ - random.sample(range(n_population), n_sample) + sampling_algorithm["python-core-sample"] = ( + lambda n_population, n_sample: random.sample(range(n_population), n_sample) + ) ########################################################################### # Set custom automatic method selection - sampling_algorithm["custom-auto"] = \ - lambda n_population, n_samples, random_state=None: \ - sample_without_replacement(n_population, n_samples, method="auto", - random_state=random_state) + sampling_algorithm["custom-auto"] = ( + lambda n_population, n_samples, random_state=None: sample_without_replacement( + n_population, n_samples, method="auto", random_state=random_state + ) + ) ########################################################################### # Set custom tracking based method - sampling_algorithm["custom-tracking-selection"] = \ - lambda n_population, n_samples, random_state=None: \ - sample_without_replacement(n_population, - n_samples, - method="tracking_selection", - random_state=random_state) + sampling_algorithm["custom-tracking-selection"] = ( + lambda n_population, n_samples, random_state=None: sample_without_replacement( + n_population, + n_samples, + method="tracking_selection", + random_state=random_state, + ) + ) ########################################################################### # Set custom reservoir based method - sampling_algorithm["custom-reservoir-sampling"] = \ - lambda n_population, n_samples, random_state=None: \ - sample_without_replacement(n_population, - n_samples, - method="reservoir_sampling", - random_state=random_state) + sampling_algorithm["custom-reservoir-sampling"] = ( + lambda n_population, n_samples, random_state=None: sample_without_replacement( + n_population, + n_samples, + method="reservoir_sampling", + random_state=random_state, + ) + ) ########################################################################### # Set custom reservoir based method - sampling_algorithm["custom-pool"] = \ - lambda n_population, n_samples, random_state=None: \ - sample_without_replacement(n_population, - n_samples, - method="pool", - random_state=random_state) + sampling_algorithm["custom-pool"] = ( + lambda n_population, n_samples, random_state=None: sample_without_replacement( + n_population, n_samples, method="pool", random_state=random_state + ) + ) ########################################################################### # Numpy permutation based - sampling_algorithm["numpy-permutation"] = \ - lambda n_population, n_sample: \ - np.random.permutation(n_population)[:n_sample] + sampling_algorithm["numpy-permutation"] = ( + lambda n_population, n_sample: np.random.permutation(n_population)[:n_sample] + ) ########################################################################### # Remove unspecified algorithm - sampling_algorithm = {key: value - for key, value in sampling_algorithm.items() - if key in selected_algorithm} + sampling_algorithm = { + key: value + for key, value in sampling_algorithm.items() + if key in selected_algorithm + } ########################################################################### # Perform benchmark ########################################################################### time = {} - n_samples = np.linspace(start=0, stop=opts.n_population, - num=opts.n_steps).astype(int) + n_samples = np.linspace(start=0, stop=opts.n_population, num=opts.n_steps).astype( + int + ) ratio = n_samples / opts.n_population - print('Benchmarks') + print("Benchmarks") print("===========================") for name in sorted(sampling_algorithm): @@ -152,9 +181,9 @@ def bench_sample(sampling, n_population, n_samples): for step in range(opts.n_steps): for it in range(opts.n_times): - time[name][step, it] = bench_sample(sampling_algorithm[name], - opts.n_population, - n_samples[step]) + time[name][step, it] = bench_sample( + sampling_algorithm[name], opts.n_population, n_samples[step] + ) print("done") @@ -168,12 +197,16 @@ def bench_sample(sampling, n_population, n_samples): print("Script arguments") print("===========================") arguments = vars(opts) - print("%s \t | %s " % ("Arguments".ljust(16), - "Value".center(12),)) + print( + "%s \t | %s " + % ( + "Arguments".ljust(16), + "Value".center(12), + ) + ) print(25 * "-" + ("|" + "-" * 14) * 1) for key, value in arguments.items(): - print("%s \t | %s " % (str(key).ljust(16), - str(value).strip().center(12))) + print("%s \t | %s " % (str(key).ljust(16), str(value).strip().center(12))) print("") print("Sampling algorithm performance:") @@ -181,15 +214,14 @@ def bench_sample(sampling, n_population, n_samples): print("Results are averaged over %s repetition(s)." % opts.n_times) print("") - fig = plt.figure('scikit-learn sample w/o replacement benchmark results') - plt.title("n_population = %s, n_times = %s" % - (opts.n_population, opts.n_times)) + fig = plt.figure("scikit-learn sample w/o replacement benchmark results") + fig.suptitle("n_population = %s, n_times = %s" % (opts.n_population, opts.n_times)) ax = fig.add_subplot(111) for name in sampling_algorithm: ax.plot(ratio, time[name], label=name) - ax.set_xlabel('ratio of n_sample / n_population') - ax.set_ylabel('Time (s)') + ax.set_xlabel("ratio of n_sample / n_population") + ax.set_ylabel("Time (s)") ax.legend() # Sort legend labels diff --git a/benchmarks/bench_sgd_regression.py b/benchmarks/bench_sgd_regression.py index 1f5c6320b03e5..bd00615e3d5f9 100644 --- a/benchmarks/bench_sgd_regression.py +++ b/benchmarks/bench_sgd_regression.py @@ -1,16 +1,15 @@ -# Author: Peter Prettenhofer -# License: BSD 3 clause - -import numpy as np -import matplotlib.pyplot as plt +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import gc - from time import time -from sklearn.linear_model import Ridge, SGDRegressor, ElasticNet -from sklearn.metrics import mean_squared_error +import matplotlib.pyplot as plt +import numpy as np + from sklearn.datasets import make_regression +from sklearn.linear_model import ElasticNet, Ridge, SGDRegressor +from sklearn.metrics import mean_squared_error """ Benchmark for SGD regression @@ -35,8 +34,11 @@ for i, n_train in enumerate(list_n_samples): for j, n_features in enumerate(list_n_features): X, y, coef = make_regression( - n_samples=n_train + n_test, n_features=n_features, - noise=noise, coef=True) + n_samples=n_train + n_test, + n_features=n_features, + noise=noise, + coef=True, + ) X_train = X[:n_train] y_train = y[:n_train] @@ -70,34 +72,43 @@ clf = ElasticNet(alpha=alpha, l1_ratio=0.5, fit_intercept=False) tstart = time() clf.fit(X_train, y_train) - elnet_results[i, j, 0] = mean_squared_error(clf.predict(X_test), - y_test) + elnet_results[i, j, 0] = mean_squared_error(clf.predict(X_test), y_test) elnet_results[i, j, 1] = time() - tstart gc.collect() print("- benchmarking SGD") - clf = SGDRegressor(alpha=alpha / n_train, fit_intercept=False, - max_iter=max_iter, learning_rate="invscaling", - eta0=.01, power_t=0.25, tol=1e-3) + clf = SGDRegressor( + alpha=alpha / n_train, + fit_intercept=False, + max_iter=max_iter, + learning_rate="invscaling", + eta0=0.01, + power_t=0.25, + tol=1e-3, + ) tstart = time() clf.fit(X_train, y_train) - sgd_results[i, j, 0] = mean_squared_error(clf.predict(X_test), - y_test) + sgd_results[i, j, 0] = mean_squared_error(clf.predict(X_test), y_test) sgd_results[i, j, 1] = time() - tstart gc.collect() print("max_iter", max_iter) print("- benchmarking A-SGD") - clf = SGDRegressor(alpha=alpha / n_train, fit_intercept=False, - max_iter=max_iter, learning_rate="invscaling", - eta0=.002, power_t=0.05, tol=1e-3, - average=(max_iter * n_train // 2)) + clf = SGDRegressor( + alpha=alpha / n_train, + fit_intercept=False, + max_iter=max_iter, + learning_rate="invscaling", + eta0=0.002, + power_t=0.05, + tol=1e-3, + average=(max_iter * n_train // 2), + ) tstart = time() clf.fit(X_train, y_train) - asgd_results[i, j, 0] = mean_squared_error(clf.predict(X_test), - y_test) + asgd_results[i, j, 0] = mean_squared_error(clf.predict(X_test), y_test) asgd_results[i, j, 1] = time() - tstart gc.collect() @@ -105,25 +116,19 @@ clf = Ridge(alpha=alpha, fit_intercept=False) tstart = time() clf.fit(X_train, y_train) - ridge_results[i, j, 0] = mean_squared_error(clf.predict(X_test), - y_test) + ridge_results[i, j, 0] = mean_squared_error(clf.predict(X_test), y_test) ridge_results[i, j, 1] = time() - tstart # Plot results i = 0 m = len(list_n_features) - plt.figure('scikit-learn SGD regression benchmark results', - figsize=(5 * 2, 4 * m)) + plt.figure("scikit-learn SGD regression benchmark results", figsize=(5 * 2, 4 * m)) for j in range(m): plt.subplot(m, 2, i + 1) - plt.plot(list_n_samples, np.sqrt(elnet_results[:, j, 0]), - label="ElasticNet") - plt.plot(list_n_samples, np.sqrt(sgd_results[:, j, 0]), - label="SGDRegressor") - plt.plot(list_n_samples, np.sqrt(asgd_results[:, j, 0]), - label="A-SGDRegressor") - plt.plot(list_n_samples, np.sqrt(ridge_results[:, j, 0]), - label="Ridge") + plt.plot(list_n_samples, np.sqrt(elnet_results[:, j, 0]), label="ElasticNet") + plt.plot(list_n_samples, np.sqrt(sgd_results[:, j, 0]), label="SGDRegressor") + plt.plot(list_n_samples, np.sqrt(asgd_results[:, j, 0]), label="A-SGDRegressor") + plt.plot(list_n_samples, np.sqrt(ridge_results[:, j, 0]), label="Ridge") plt.legend(prop={"size": 10}) plt.xlabel("n_train") plt.ylabel("RMSE") @@ -131,20 +136,16 @@ i += 1 plt.subplot(m, 2, i + 1) - plt.plot(list_n_samples, np.sqrt(elnet_results[:, j, 1]), - label="ElasticNet") - plt.plot(list_n_samples, np.sqrt(sgd_results[:, j, 1]), - label="SGDRegressor") - plt.plot(list_n_samples, np.sqrt(asgd_results[:, j, 1]), - label="A-SGDRegressor") - plt.plot(list_n_samples, np.sqrt(ridge_results[:, j, 1]), - label="Ridge") + plt.plot(list_n_samples, np.sqrt(elnet_results[:, j, 1]), label="ElasticNet") + plt.plot(list_n_samples, np.sqrt(sgd_results[:, j, 1]), label="SGDRegressor") + plt.plot(list_n_samples, np.sqrt(asgd_results[:, j, 1]), label="A-SGDRegressor") + plt.plot(list_n_samples, np.sqrt(ridge_results[:, j, 1]), label="Ridge") plt.legend(prop={"size": 10}) plt.xlabel("n_train") plt.ylabel("Time [sec]") plt.title("Training time - %d features" % list_n_features[j]) i += 1 - plt.subplots_adjust(hspace=.30) + plt.subplots_adjust(hspace=0.30) plt.show() diff --git a/benchmarks/bench_sparsify.py b/benchmarks/bench_sparsify.py index be1f3bffe0181..1832ca40c6ddb 100644 --- a/benchmarks/bench_sparsify.py +++ b/benchmarks/bench_sparsify.py @@ -43,8 +43,9 @@ 60 300 381409 1271.4 97.1 clf.predict(X_test_sparse) """ -from scipy.sparse.csr import csr_matrix import numpy as np +from scipy.sparse import csr_matrix + from sklearn.linear_model import SGDRegressor from sklearn.metrics import r2_score @@ -54,16 +55,17 @@ def sparsity_ratio(X): return np.count_nonzero(X) / float(n_samples * n_features) + n_samples, n_features = 5000, 300 X = np.random.randn(n_samples, n_features) inds = np.arange(n_samples) np.random.shuffle(inds) -X[inds[int(n_features / 1.2):]] = 0 # sparsify input +X[inds[int(n_features / 1.2) :]] = 0 # sparsify input print("input data sparsity: %f" % sparsity_ratio(X)) coef = 3 * np.random.randn(n_features) inds = np.arange(n_features) np.random.shuffle(inds) -coef[inds[n_features // 2:]] = 0 # sparsify coef +coef[inds[n_features // 2 :]] = 0 # sparsify coef print("true coef sparsity: %f" % sparsity_ratio(coef)) y = np.dot(X, coef) @@ -72,13 +74,12 @@ def sparsity_ratio(X): # Split data in train set and test set n_samples = X.shape[0] -X_train, y_train = X[:n_samples // 2], y[:n_samples // 2] -X_test, y_test = X[n_samples // 2:], y[n_samples // 2:] +X_train, y_train = X[: n_samples // 2], y[: n_samples // 2] +X_test, y_test = X[n_samples // 2 :], y[n_samples // 2 :] print("test data sparsity: %f" % sparsity_ratio(X_test)) ############################################################################### -clf = SGDRegressor(penalty='l1', alpha=.2, max_iter=2000, - tol=None) +clf = SGDRegressor(penalty="l1", alpha=0.2, max_iter=2000, tol=None) clf.fit(X_train, y_train) print("model sparsity: %f" % sparsity_ratio(clf.coef_)) @@ -98,8 +99,9 @@ def score(y_test, y_pred, case): r2 = r2_score(y_test, y_pred) print("r^2 on test data (%s) : %f" % (case, r2)) -score(y_test, clf.predict(X_test), 'dense model') + +score(y_test, clf.predict(X_test), "dense model") benchmark_dense_predict() clf.sparsify() -score(y_test, clf.predict(X_test), 'sparse model') +score(y_test, clf.predict(X_test), "sparse model") benchmark_sparse_predict() diff --git a/benchmarks/bench_text_vectorizers.py b/benchmarks/bench_text_vectorizers.py index 96dbc04312291..2eab7071544f9 100644 --- a/benchmarks/bench_text_vectorizers.py +++ b/benchmarks/bench_text_vectorizers.py @@ -8,16 +8,20 @@ * psutil (optional, but recommended) """ -import timeit + import itertools +import timeit import numpy as np import pandas as pd from memory_profiler import memory_usage from sklearn.datasets import fetch_20newsgroups -from sklearn.feature_extraction.text import (CountVectorizer, TfidfVectorizer, - HashingVectorizer) +from sklearn.feature_extraction.text import ( + CountVectorizer, + HashingVectorizer, + TfidfVectorizer, +) n_repeat = 3 @@ -26,47 +30,45 @@ def run_vectorizer(Vectorizer, X, **params): def f(): vect = Vectorizer(**params) vect.fit_transform(X) + return f -text = fetch_20newsgroups(subset='train').data[:1000] +text = fetch_20newsgroups(subset="train").data[:1000] -print("="*80 + '\n#' + " Text vectorizers benchmark" + '\n' + '='*80 + '\n') -print("Using a subset of the 20 newsgroups dataset ({} documents)." - .format(len(text))) +print("=" * 80 + "\n#" + " Text vectorizers benchmark" + "\n" + "=" * 80 + "\n") +print("Using a subset of the 20 newsgroups dataset ({} documents).".format(len(text))) print("This benchmarks runs in ~1 min ...") res = [] for Vectorizer, (analyzer, ngram_range) in itertools.product( - [CountVectorizer, TfidfVectorizer, HashingVectorizer], - [('word', (1, 1)), - ('word', (1, 2)), - ('char', (4, 4)), - ('char_wb', (4, 4)) - ]): - - bench = {'vectorizer': Vectorizer.__name__} - params = {'analyzer': analyzer, 'ngram_range': ngram_range} + [CountVectorizer, TfidfVectorizer, HashingVectorizer], + [("word", (1, 1)), ("word", (1, 2)), ("char", (4, 4)), ("char_wb", (4, 4))], +): + bench = {"vectorizer": Vectorizer.__name__} + params = {"analyzer": analyzer, "ngram_range": ngram_range} bench.update(params) - dt = timeit.repeat(run_vectorizer(Vectorizer, text, **params), - number=1, - repeat=n_repeat) - bench['time'] = "{:.3f} (+-{:.3f})".format(np.mean(dt), np.std(dt)) + dt = timeit.repeat( + run_vectorizer(Vectorizer, text, **params), number=1, repeat=n_repeat + ) + bench["time"] = "{:.3f} (+-{:.3f})".format(np.mean(dt), np.std(dt)) mem_usage = memory_usage(run_vectorizer(Vectorizer, text, **params)) - bench['memory'] = "{:.1f}".format(np.max(mem_usage)) + bench["memory"] = "{:.1f}".format(np.max(mem_usage)) res.append(bench) -df = pd.DataFrame(res).set_index(['analyzer', 'ngram_range', 'vectorizer']) +df = pd.DataFrame(res).set_index(["analyzer", "ngram_range", "vectorizer"]) -print('\n========== Run time performance (sec) ===========\n') -print('Computing the mean and the standard deviation ' - 'of the run time over {} runs...\n'.format(n_repeat)) -print(df['time'].unstack(level=-1)) +print("\n========== Run time performance (sec) ===========\n") +print( + "Computing the mean and the standard deviation " + "of the run time over {} runs...\n".format(n_repeat) +) +print(df["time"].unstack(level=-1)) -print('\n=============== Memory usage (MB) ===============\n') -print(df['memory'].unstack(level=-1)) +print("\n=============== Memory usage (MB) ===============\n") +print(df["memory"].unstack(level=-1)) diff --git a/benchmarks/bench_tree.py b/benchmarks/bench_tree.py index 8a0af26d4c221..c522bcb39e994 100644 --- a/benchmarks/bench_tree.py +++ b/benchmarks/bench_tree.py @@ -13,16 +13,18 @@ training set, classify a sample and plot the time taken as a function of the number of dimensions. """ -import numpy as np -import matplotlib.pyplot as plt + import gc from datetime import datetime +import matplotlib.pyplot as plt +import numpy as np + # to store the results scikit_classifier_results = [] scikit_regressor_results = [] -mu_second = 0.0 + 10 ** 6 # number of microseconds in a second +mu_second = 0.0 + 10**6 # number of microseconds in a second def bench_scikit_tree_classifier(X, Y): @@ -36,11 +38,10 @@ def bench_scikit_tree_classifier(X, Y): tstart = datetime.now() clf = DecisionTreeClassifier() clf.fit(X, Y).predict(X) - delta = (datetime.now() - tstart) + delta = datetime.now() - tstart # stop time - scikit_classifier_results.append( - delta.seconds + delta.microseconds / mu_second) + scikit_classifier_results.append(delta.seconds + delta.microseconds / mu_second) def bench_scikit_tree_regressor(X, Y): @@ -54,18 +55,16 @@ def bench_scikit_tree_regressor(X, Y): tstart = datetime.now() clf = DecisionTreeRegressor() clf.fit(X, Y).predict(X) - delta = (datetime.now() - tstart) + delta = datetime.now() - tstart # stop time - scikit_regressor_results.append( - delta.seconds + delta.microseconds / mu_second) - + scikit_regressor_results.append(delta.seconds + delta.microseconds / mu_second) -if __name__ == '__main__': - print('============================================') - print('Warning: this is going to take a looong time') - print('============================================') +if __name__ == "__main__": + print("============================================") + print("Warning: this is going to take a looong time") + print("============================================") n = 10 step = 10000 @@ -73,9 +72,9 @@ def bench_scikit_tree_regressor(X, Y): dim = 10 n_classes = 10 for i in range(n): - print('============================================') - print('Entering iteration %s of %s' % (i, n)) - print('============================================') + print("============================================") + print("Entering iteration %s of %s" % (i, n)) + print("============================================") n_samples += step X = np.random.randn(n_samples, dim) Y = np.random.randint(0, n_classes, (n_samples,)) @@ -84,14 +83,14 @@ def bench_scikit_tree_regressor(X, Y): bench_scikit_tree_regressor(X, Y) xx = range(0, n * step, step) - plt.figure('scikit-learn tree benchmark results') + plt.figure("scikit-learn tree benchmark results") plt.subplot(211) - plt.title('Learning with varying number of samples') - plt.plot(xx, scikit_classifier_results, 'g-', label='classification') - plt.plot(xx, scikit_regressor_results, 'r-', label='regression') - plt.legend(loc='upper left') - plt.xlabel('number of samples') - plt.ylabel('Time (s)') + plt.title("Learning with varying number of samples") + plt.plot(xx, scikit_classifier_results, "g-", label="classification") + plt.plot(xx, scikit_regressor_results, "r-", label="regression") + plt.legend(loc="upper left") + plt.xlabel("number of samples") + plt.ylabel("Time (s)") scikit_classifier_results = [] scikit_regressor_results = [] @@ -102,9 +101,9 @@ def bench_scikit_tree_regressor(X, Y): dim = start_dim for i in range(0, n): - print('============================================') - print('Entering iteration %s of %s' % (i, n)) - print('============================================') + print("============================================") + print("Entering iteration %s of %s" % (i, n)) + print("============================================") dim += step X = np.random.randn(100, dim) Y = np.random.randint(0, n_classes, (100,)) @@ -114,11 +113,11 @@ def bench_scikit_tree_regressor(X, Y): xx = np.arange(start_dim, start_dim + n * step, step) plt.subplot(212) - plt.title('Learning in high dimensional spaces') - plt.plot(xx, scikit_classifier_results, 'g-', label='classification') - plt.plot(xx, scikit_regressor_results, 'r-', label='regression') - plt.legend(loc='upper left') - plt.xlabel('number of dimensions') - plt.ylabel('Time (s)') - plt.axis('tight') + plt.title("Learning in high dimensional spaces") + plt.plot(xx, scikit_classifier_results, "g-", label="classification") + plt.plot(xx, scikit_regressor_results, "r-", label="regression") + plt.legend(loc="upper left") + plt.xlabel("number of dimensions") + plt.ylabel("Time (s)") + plt.axis("tight") plt.show() diff --git a/benchmarks/bench_tsne_mnist.py b/benchmarks/bench_tsne_mnist.py index 1f1dc5143d177..8649c7a46b629 100644 --- a/benchmarks/bench_tsne_mnist.py +++ b/benchmarks/bench_tsne_mnist.py @@ -5,20 +5,21 @@ """ -# License: BSD 3 clause +# SPDX-License-Identifier: BSD-3-Clause +import argparse +import json import os import os.path as op from time import time + import numpy as np -import json -import argparse from joblib import Memory from sklearn.datasets import fetch_openml +from sklearn.decomposition import PCA from sklearn.manifold import TSNE from sklearn.neighbors import NearestNeighbors -from sklearn.decomposition import PCA from sklearn.utils import check_array from sklearn.utils import shuffle as _shuffle from sklearn.utils._openmp_helpers import _openmp_effective_n_threads @@ -28,17 +29,16 @@ os.mkdir(LOG_DIR) -memory = Memory(os.path.join(LOG_DIR, 'mnist_tsne_benchmark_data'), - mmap_mode='r') +memory = Memory(os.path.join(LOG_DIR, "mnist_tsne_benchmark_data"), mmap_mode="r") @memory.cache -def load_data(dtype=np.float32, order='C', shuffle=True, seed=0): +def load_data(dtype=np.float32, order="C", shuffle=True, seed=0): """Load the data, then cache and memmap the train/test split""" print("Loading dataset...") - data = fetch_openml('mnist_784') + data = fetch_openml("mnist_784", as_frame=True) - X = check_array(data['data'], dtype=dtype, order=order) + X = check_array(data["data"], dtype=dtype, order=order) y = data["target"] if shuffle: @@ -63,27 +63,43 @@ def tsne_fit_transform(model, data): def sanitize(filename): - return filename.replace("/", '-').replace(" ", "_") + return filename.replace("/", "-").replace(" ", "_") if __name__ == "__main__": - parser = argparse.ArgumentParser('Benchmark for t-SNE') - parser.add_argument('--order', type=str, default='C', - help='Order of the input data') - parser.add_argument('--perplexity', type=float, default=30) - parser.add_argument('--bhtsne', action='store_true', - help="if set and the reference bhtsne code is " - "correctly installed, run it in the benchmark.") - parser.add_argument('--all', action='store_true', - help="if set, run the benchmark with the whole MNIST." - "dataset. Note that it will take up to 1 hour.") - parser.add_argument('--profile', action='store_true', - help="if set, run the benchmark with a memory " - "profiler.") - parser.add_argument('--verbose', type=int, default=0) - parser.add_argument('--pca-components', type=int, default=50, - help="Number of principal components for " - "preprocessing.") + parser = argparse.ArgumentParser("Benchmark for t-SNE") + parser.add_argument( + "--order", type=str, default="C", help="Order of the input data" + ) + parser.add_argument("--perplexity", type=float, default=30) + parser.add_argument( + "--bhtsne", + action="store_true", + help=( + "if set and the reference bhtsne code is " + "correctly installed, run it in the benchmark." + ), + ) + parser.add_argument( + "--all", + action="store_true", + help=( + "if set, run the benchmark with the whole MNIST." + "dataset. Note that it will take up to 1 hour." + ), + ) + parser.add_argument( + "--profile", + action="store_true", + help="if set, run the benchmark with a memory profiler.", + ) + parser.add_argument("--verbose", type=int, default=0) + parser.add_argument( + "--pca-components", + type=int, + default=50, + help="Number of principal components for preprocessing.", + ) args = parser.parse_args() print("Used number of threads: {}".format(_openmp_effective_n_threads())) @@ -92,22 +108,30 @@ def sanitize(filename): if args.pca_components > 0: t0 = time() X = PCA(n_components=args.pca_components).fit_transform(X) - print("PCA preprocessing down to {} dimensions took {:0.3f}s" - .format(args.pca_components, time() - t0)) + print( + "PCA preprocessing down to {} dimensions took {:0.3f}s".format( + args.pca_components, time() - t0 + ) + ) methods = [] # Put TSNE in methods - tsne = TSNE(n_components=2, init='pca', perplexity=args.perplexity, - verbose=args.verbose, n_iter=1000) - methods.append(("sklearn TSNE", - lambda data: tsne_fit_transform(tsne, data))) + tsne = TSNE( + n_components=2, + init="pca", + perplexity=args.perplexity, + verbose=args.verbose, + n_iter=1000, + ) + methods.append(("sklearn TSNE", lambda data: tsne_fit_transform(tsne, data))) if args.bhtsne: try: from bhtsne.bhtsne import run_bh_tsne except ImportError as e: - raise ImportError("""\ + raise ImportError( + """\ If you want comparison with the reference implementation, build the binary from source (https://github.com/lvdmaaten/bhtsne) in the folder benchmarks/bhtsne and add an empty `__init__.py` file in the folder: @@ -117,24 +141,34 @@ def sanitize(filename): $ g++ sptree.cpp tsne.cpp tsne_main.cpp -o bh_tsne -O2 $ touch __init__.py $ cd .. -""") from e +""" + ) from e def bhtsne(X): """Wrapper for the reference lvdmaaten/bhtsne implementation.""" # PCA preprocessing is done elsewhere in the benchmark script n_iter = -1 # TODO find a way to report the number of iterations - return run_bh_tsne(X, use_pca=False, perplexity=args.perplexity, - verbose=args.verbose > 0), n_iter + return ( + run_bh_tsne( + X, + use_pca=False, + perplexity=args.perplexity, + verbose=args.verbose > 0, + ), + n_iter, + ) + methods.append(("lvdmaaten/bhtsne", bhtsne)) if args.profile: - try: from memory_profiler import profile except ImportError as e: - raise ImportError("To run the benchmark with `--profile`, you " - "need to install `memory_profiler`. Please " - "run `pip install memory_profiler`.") from e + raise ImportError( + "To run the benchmark with `--profile`, you " + "need to install `memory_profiler`. Please " + "run `pip install memory_profiler`." + ) from e methods = [(n, profile(m)) for n, m in methods] data_size = [100, 500, 1000, 5000, 10000] @@ -143,7 +177,7 @@ def bhtsne(X): results = [] basename = os.path.basename(os.path.splitext(__file__)[0]) - log_filename = os.path.join(LOG_DIR, basename + '.json') + log_filename = os.path.join(LOG_DIR, basename + ".json") for n in data_size: X_train = X[:n] y_train = y[:n] @@ -151,19 +185,24 @@ def bhtsne(X): for name, method in methods: print("Fitting {} on {} samples...".format(name, n)) t0 = time() - np.save(os.path.join(LOG_DIR, 'mnist_{}_{}.npy' - .format('original', n)), X_train) - np.save(os.path.join(LOG_DIR, 'mnist_{}_{}.npy' - .format('original_labels', n)), y_train) + np.save( + os.path.join(LOG_DIR, "mnist_{}_{}.npy".format("original", n)), X_train + ) + np.save( + os.path.join(LOG_DIR, "mnist_{}_{}.npy".format("original_labels", n)), + y_train, + ) X_embedded, n_iter = method(X_train) duration = time() - t0 precision_5 = nn_accuracy(X_train, X_embedded) - print("Fitting {} on {} samples took {:.3f}s in {:d} iterations, " - "nn accuracy: {:0.3f}".format( - name, n, duration, n_iter, precision_5)) + print( + "Fitting {} on {} samples took {:.3f}s in {:d} iterations, " + "nn accuracy: {:0.3f}".format(name, n, duration, n_iter, precision_5) + ) results.append(dict(method=name, duration=duration, n_samples=n)) - with open(log_filename, 'w', encoding='utf-8') as f: + with open(log_filename, "w", encoding="utf-8") as f: json.dump(results, f) method_name = sanitize(name) - np.save(op.join(LOG_DIR, 'mnist_{}_{}.npy'.format(method_name, n)), - X_embedded) + np.save( + op.join(LOG_DIR, "mnist_{}_{}.npy".format(method_name, n)), X_embedded + ) diff --git a/benchmarks/plot_tsne_mnist.py b/benchmarks/plot_tsne_mnist.py index 0ffd32b3de779..fff71eed0a26c 100644 --- a/benchmarks/plot_tsne_mnist.py +++ b/benchmarks/plot_tsne_mnist.py @@ -1,23 +1,26 @@ -import matplotlib.pyplot as plt -import numpy as np -import os.path as op - import argparse +import os.path as op +import matplotlib.pyplot as plt +import numpy as np LOG_DIR = "mnist_tsne_output" if __name__ == "__main__": - parser = argparse.ArgumentParser('Plot benchmark results for t-SNE') + parser = argparse.ArgumentParser("Plot benchmark results for t-SNE") parser.add_argument( - '--labels', type=str, - default=op.join(LOG_DIR, 'mnist_original_labels_10000.npy'), - help='1D integer numpy array for labels') + "--labels", + type=str, + default=op.join(LOG_DIR, "mnist_original_labels_10000.npy"), + help="1D integer numpy array for labels", + ) parser.add_argument( - '--embedding', type=str, - default=op.join(LOG_DIR, 'mnist_sklearn_TSNE_10000.npy'), - help='2D float numpy array for embedded data') + "--embedding", + type=str, + default=op.join(LOG_DIR, "mnist_sklearn_TSNE_10000.npy"), + help="2D float numpy array for embedded data", + ) args = parser.parse_args() X = np.load(args.embedding) @@ -26,5 +29,5 @@ for i in np.unique(y): mask = y == i plt.scatter(X[mask, 0], X[mask, 1], alpha=0.2, label=int(i)) - plt.legend(loc='best') + plt.legend(loc="best") plt.show() diff --git a/build_tools/azure/combine_coverage_reports.sh b/build_tools/azure/combine_coverage_reports.sh new file mode 100755 index 0000000000000..c3b90fdd4fcdb --- /dev/null +++ b/build_tools/azure/combine_coverage_reports.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e + +# Defines the show_installed_libraries and activate_environment functions. +source build_tools/shared.sh + +activate_environment + +# Combine all coverage files generated by subprocesses workers such +# such as pytest-xdist and joblib/loky: +pushd $TEST_DIR +coverage combine --append +coverage xml +popd + +# Copy the combined coverage file to the root of the repository: +cp $TEST_DIR/coverage.xml $BUILD_REPOSITORY_LOCALPATH diff --git a/build_tools/azure/debian_32bit_lock.txt b/build_tools/azure/debian_32bit_lock.txt new file mode 100644 index 0000000000000..c7b8cbceccacb --- /dev/null +++ b/build_tools/azure/debian_32bit_lock.txt @@ -0,0 +1,37 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --output-file=build_tools/azure/debian_32bit_lock.txt build_tools/azure/debian_32bit_requirements.txt +# +coverage[toml]==7.8.2 + # via pytest-cov +cython==3.1.1 + # via -r build_tools/azure/debian_32bit_requirements.txt +iniconfig==2.1.0 + # via pytest +joblib==1.5.1 + # via -r build_tools/azure/debian_32bit_requirements.txt +meson==1.8.1 + # via meson-python +meson-python==0.18.0 + # via -r build_tools/azure/debian_32bit_requirements.txt +ninja==1.11.1.4 + # via -r build_tools/azure/debian_32bit_requirements.txt +packaging==25.0 + # via + # meson-python + # pyproject-metadata + # pytest +pluggy==1.6.0 + # via pytest +pyproject-metadata==0.9.1 + # via meson-python +pytest==8.3.5 + # via + # -r build_tools/azure/debian_32bit_requirements.txt + # pytest-cov +pytest-cov==6.1.1 + # via -r build_tools/azure/debian_32bit_requirements.txt +threadpoolctl==3.6.0 + # via -r build_tools/azure/debian_32bit_requirements.txt diff --git a/build_tools/azure/debian_32bit_requirements.txt b/build_tools/azure/debian_32bit_requirements.txt new file mode 100644 index 0000000000000..6dcf67d11c58d --- /dev/null +++ b/build_tools/azure/debian_32bit_requirements.txt @@ -0,0 +1,10 @@ +# DO NOT EDIT: this file is generated from the specification found in the +# following script to centralize the configuration for CI builds: +# build_tools/update_environments_and_lock_files.py +cython +joblib +threadpoolctl +pytest +pytest-cov +ninja +meson-python diff --git a/build_tools/azure/get_commit_message.py b/build_tools/azure/get_commit_message.py new file mode 100644 index 0000000000000..0b1246b8d2724 --- /dev/null +++ b/build_tools/azure/get_commit_message.py @@ -0,0 +1,65 @@ +import argparse +import os +import subprocess + + +def get_commit_message(): + """Retrieve the commit message.""" + build_source_version_message = os.environ["BUILD_SOURCEVERSIONMESSAGE"] + + if os.environ["BUILD_REASON"] == "PullRequest": + # By default pull requests use refs/pull/PULL_ID/merge as the source branch + # which has a "Merge ID into ID" as a commit message. The latest commit + # message is the second to last commit + commit_id = build_source_version_message.split()[1] + git_cmd = ["git", "log", commit_id, "-1", "--pretty=%B"] + commit_message = subprocess.run( + git_cmd, capture_output=True, text=True + ).stdout.strip() + else: + commit_message = build_source_version_message + + # Sanitize the commit message to avoid introducing a vulnerability: a PR + # submitter could include the "##vso" special marker in their commit + # message to attempt to obfuscate the injection of arbitrary commands in + # the Azure pipeline. + # + # This can be a problem if the PR reviewers do not pay close enough + # attention to the full commit message prior to clicking the merge button + # and as a result make the inject code run in a protected branch with + # elevated access to CI secrets. On a protected branch, Azure + # already sanitizes `BUILD_SOURCEVERSIONMESSAGE`, but the message + # will still be sanitized here out of precaution. + commit_message = commit_message.replace("##vso", "..vso") + + return commit_message + + +def parsed_args(): + parser = argparse.ArgumentParser( + description=( + "Show commit message that triggered the build in Azure DevOps pipeline" + ) + ) + parser.add_argument( + "--only-show-message", + action="store_true", + default=False, + help=( + "Only print commit message. Useful for direct use in scripts rather than" + " setting output variable of the Azure job" + ), + ) + return parser.parse_args() + + +if __name__ == "__main__": + args = parsed_args() + commit_message = get_commit_message() + + if args.only_show_message: + print(commit_message) + else: + # set the environment variable to be propagated to other steps + print(f"##vso[task.setvariable variable=message;isOutput=true]{commit_message}") + print(f"commit message: {commit_message}") # helps debugging diff --git a/build_tools/azure/get_selected_tests.py b/build_tools/azure/get_selected_tests.py new file mode 100644 index 0000000000000..f453748f843c4 --- /dev/null +++ b/build_tools/azure/get_selected_tests.py @@ -0,0 +1,34 @@ +from get_commit_message import get_commit_message + + +def get_selected_tests(): + """Parse the commit message to check if pytest should run only specific tests. + + If so, selected tests will be run with SKLEARN_TESTS_GLOBAL_RANDOM_SEED="all". + + The commit message must take the form: + [all random seeds] + <test_name_1> + <test_name_2> + ... + """ + commit_message = get_commit_message() + + if "[all random seeds]" in commit_message: + selected_tests = commit_message.split("[all random seeds]")[1].strip() + selected_tests = selected_tests.replace("\n", " or ") + else: + selected_tests = "" + + return selected_tests + + +if __name__ == "__main__": + # set the environment variable to be propagated to other steps + selected_tests = get_selected_tests() + + if selected_tests: + print(f"##vso[task.setvariable variable=SELECTED_TESTS]'{selected_tests}'") + print(f"selected tests: {selected_tests}") # helps debugging + else: + print("no selected tests") diff --git a/build_tools/azure/install.cmd b/build_tools/azure/install.cmd deleted file mode 100644 index caf28261dcc9f..0000000000000 --- a/build_tools/azure/install.cmd +++ /dev/null @@ -1,45 +0,0 @@ -@rem https://github.com/numba/numba/blob/master/buildscripts/incremental/setup_conda_environment.cmd -@rem The cmd /C hack circumvents a regression where conda installs a conda.bat -@rem script in non-root environments. -set CONDA_INSTALL=cmd /C conda install -q -y -set PIP_INSTALL=pip install -q - -@echo on - -IF "%PYTHON_ARCH%"=="64" ( - @rem Deactivate any environment - call deactivate - @rem Clean up any left-over from a previous build - conda remove --all -q -y -n %VIRTUALENV% - conda create -n %VIRTUALENV% -q -y python=%PYTHON_VERSION% numpy scipy cython matplotlib wheel pillow joblib - - call activate %VIRTUALENV% - - pip install threadpoolctl - - IF "%PYTEST_VERSION%"=="*" ( - pip install pytest - ) else ( - pip install pytest==%PYTEST_VERSION% - ) -) else ( - pip install numpy scipy cython pytest wheel pillow joblib threadpoolctl -) - -IF "%PYTEST_XDIST%" == "true" ( - pip install pytest-xdist -) - -if "%COVERAGE%" == "true" ( - pip install coverage codecov pytest-cov -) -python --version -pip --version - -@rem Install the build and runtime dependencies of the project. -python setup.py bdist_wheel bdist_wininst -b doc\logos\scikit-learn-logo.bmp - -@rem Install the generated wheel package to test it -pip install --pre --no-index --find-links dist\ scikit-learn - -if %errorlevel% neq 0 exit /b %errorlevel% diff --git a/build_tools/azure/install.sh b/build_tools/azure/install.sh index fbe0c90a473ab..9ae67f8db5e29 100755 --- a/build_tools/azure/install.sh +++ b/build_tools/azure/install.sh @@ -3,167 +3,136 @@ set -e set -x +# defines the get_dep and show_installed_libraries functions +source build_tools/shared.sh + UNAMESTR=`uname` +CCACHE_LINKS_DIR="/tmp/ccache" -make_conda() { - TO_INSTALL="$@" - conda create -n $VIRTUALENV --yes $TO_INSTALL - source activate $VIRTUALENV +setup_ccache() { + CCACHE_BIN=`which ccache || echo ""` + if [[ "${CCACHE_BIN}" == "" ]]; then + echo "ccache not found, skipping..." + elif [[ -d "${CCACHE_LINKS_DIR}" ]]; then + echo "ccache already configured, skipping..." + else + echo "Setting up ccache with CCACHE_DIR=${CCACHE_DIR}" + mkdir ${CCACHE_LINKS_DIR} + which ccache + for name in gcc g++ cc c++ clang clang++ i686-linux-gnu-gcc i686-linux-gnu-c++ x86_64-linux-gnu-gcc x86_64-linux-gnu-c++ x86_64-apple-darwin13.4.0-clang x86_64-apple-darwin13.4.0-clang++; do + ln -s ${CCACHE_BIN} "${CCACHE_LINKS_DIR}/${name}" + done + export PATH="${CCACHE_LINKS_DIR}:${PATH}" + ccache -M 256M + + # Zeroing statistics so that ccache statistics are shown only for this build + ccache -z + fi } -setup_ccache() { - echo "Setting up ccache" - mkdir /tmp/ccache/ - which ccache - for name in gcc g++ cc c++ x86_64-linux-gnu-gcc x86_64-linux-gnu-c++; do - ln -s $(which ccache) "/tmp/ccache/${name}" +pre_python_environment_install() { + if [[ "$DISTRIB" == "ubuntu" ]]; then + sudo apt-get update + sudo apt-get install python3-scipy python3-matplotlib \ + libatlas3-base libatlas-base-dev python3-virtualenv ccache + + elif [[ "$DISTRIB" == "debian-32" ]]; then + apt-get update + apt-get install -y python3-dev python3-numpy python3-scipy \ + python3-matplotlib libopenblas-dev \ + python3-virtualenv python3-pandas ccache git + fi +} + +check_packages_dev_version() { + for package in $@; do + package_version=$(python -c "import $package; print($package.__version__)") + if [[ $package_version =~ "^[.0-9]+$" ]]; then + echo "$package is not a development version: $package_version" + exit 1 + fi done - export PATH="/tmp/ccache/:${PATH}" - ccache -M 256M } -# imports get_dep -source build_tools/shared.sh +python_environment_install_and_activate() { + if [[ "$DISTRIB" == "conda"* ]]; then + create_conda_environment_from_lock_file $VIRTUALENV $LOCK_FILE + activate_environment -if [[ "$DISTRIB" == "conda" ]]; then + elif [[ "$DISTRIB" == "ubuntu" || "$DISTRIB" == "debian-32" ]]; then + python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV + activate_environment + pip install -r "${LOCK_FILE}" - if [[ "$CONDA_CHANNEL" != "" ]]; then - TO_INSTALL="-c $CONDA_CHANNEL" - else - TO_INSTALL="" fi - TO_INSTALL="$TO_INSTALL python=$PYTHON_VERSION ccache pip blas[build=$BLAS]" - - TO_INSTALL="$TO_INSTALL $(get_dep numpy $NUMPY_VERSION)" - TO_INSTALL="$TO_INSTALL $(get_dep scipy $SCIPY_VERSION)" - TO_INSTALL="$TO_INSTALL $(get_dep cython $CYTHON_VERSION)" - TO_INSTALL="$TO_INSTALL $(get_dep joblib $JOBLIB_VERSION)" - TO_INSTALL="$TO_INSTALL $(get_dep pandas $PANDAS_VERSION)" - TO_INSTALL="$TO_INSTALL $(get_dep pyamg $PYAMG_VERSION)" - TO_INSTALL="$TO_INSTALL $(get_dep Pillow $PILLOW_VERSION)" - TO_INSTALL="$TO_INSTALL $(get_dep matplotlib $MATPLOTLIB_VERSION)" - - if [[ "$UNAMESTR" == "Darwin" ]]; then - if [[ "$SKLEARN_TEST_NO_OPENMP" != "true" ]]; then - # on macOS, install an OpenMP-enabled clang/llvm from conda-forge. - # TODO: Remove !=1.1.0 when the following is fixed: - # sklearn/svm/_libsvm.cpython-38-darwin.so, - # 2): Symbol not found: _svm_check_parameter error - TO_INSTALL="$TO_INSTALL compilers>=1.0.4,!=1.1.0 llvm-openmp" - fi - fi - make_conda $TO_INSTALL - setup_ccache + # Install additional packages on top of the lock-file in specific cases + if [[ "$DISTRIB" == "conda-pip-scipy-dev" ]]; then + echo "Installing development dependency wheels" + dev_anaconda_url=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple + dev_packages="numpy scipy pandas Cython" + pip install --pre --upgrade --timeout=60 --extra-index $dev_anaconda_url $dev_packages --only-binary :all: -elif [[ "$DISTRIB" == "ubuntu" ]]; then - sudo add-apt-repository --remove ppa:ubuntu-toolchain-r/test - sudo apt-get update - sudo apt-get install python3-scipy python3-matplotlib libatlas3-base libatlas-base-dev python3-virtualenv ccache - python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV - source $VIRTUALENV/bin/activate - setup_ccache - python -m pip install $(get_dep cython $CYTHON_VERSION) \ - $(get_dep joblib $JOBLIB_VERSION) + check_packages_dev_version $dev_packages -elif [[ "$DISTRIB" == "ubuntu-32" ]]; then - apt-get update - apt-get install -y python3-dev python3-scipy python3-matplotlib libatlas3-base libatlas-base-dev python3-virtualenv python3-pandas ccache + echo "Installing joblib from latest sources" + pip install https://github.com/joblib/joblib/archive/master.zip + echo "Installing pillow from latest sources" + pip install https://github.com/python-pillow/Pillow/archive/main.zip + fi +} - python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV - source $VIRTUALENV/bin/activate - setup_ccache - python -m pip install $(get_dep cython $CYTHON_VERSION) \ - $(get_dep joblib $JOBLIB_VERSION) - -elif [[ "$DISTRIB" == "conda-pip-latest" ]]; then - # Since conda main channel usually lacks behind on the latest releases, - # we use pypi to test against the latest releases of the dependencies. - # conda is still used as a convenient way to install Python and pip. - make_conda "ccache python=$PYTHON_VERSION" +scikit_learn_install() { setup_ccache - python -m pip install -U pip - - # Do not build scikit-image from source because it is an optional dependency - python -m pip install --only-binary :all: scikit-image || true - - python -m pip install pandas matplotlib pyamg - # do not install dependencies for lightgbm since it requires scikit-learn - # and install a version less than 3.0.0 until the issue #18316 is solved. - python -m pip install "lightgbm<3.0.0" --no-deps -elif [[ "$DISTRIB" == "conda-pip-scipy-dev" ]]; then - make_conda "ccache python=$PYTHON_VERSION" - python -m pip install -U pip - echo "Installing numpy and scipy master wheels" - dev_anaconda_url=https://pypi.anaconda.org/scipy-wheels-nightly/simple - pip install --pre --upgrade --timeout=60 --extra-index $dev_anaconda_url numpy pandas - - # issue with metadata in scipy dev builds https://github.com/scipy/scipy/issues/13196 - # --use-deprecated=legacy-resolver needs to be included - pip install --pre --upgrade --timeout=60 --extra-index $dev_anaconda_url scipy --use-deprecated=legacy-resolver - pip install --pre cython - setup_ccache - echo "Installing joblib master" - pip install https://github.com/joblib/joblib/archive/master.zip - echo "Installing pillow master" - pip install https://github.com/python-pillow/Pillow/archive/master.zip -fi - -python -m pip install $(get_dep threadpoolctl $THREADPOOLCTL_VERSION) \ - $(get_dep pytest $PYTEST_VERSION) \ - $(get_dep pytest-xdist $PYTEST_XDIST_VERSION) - -if [[ "$COVERAGE" == "true" ]]; then - python -m pip install codecov pytest-cov -fi - -if [[ "$PYTEST_XDIST_VERSION" != "none" ]]; then - python -m pip install pytest-xdist -fi - -if [[ "$TEST_DOCSTRINGS" == "true" ]]; then - # numpydoc requires sphinx - python -m pip install sphinx - python -m pip install numpydoc -fi - -python --version -python -c "import numpy; print('numpy %s' % numpy.__version__)" -python -c "import scipy; print('scipy %s' % scipy.__version__)" -python -c "\ -try: - import pandas - print('pandas %s' % pandas.__version__) -except ImportError: - print('pandas not installed') -" -# Set parallelism to 3 to overlap IO bound tasks with CPU bound tasks on CI -# workers with 2 cores when building the compiled extensions of scikit-learn. -export SKLEARN_BUILD_PARALLEL=3 - -python -m pip list -if [[ "$DISTRIB" == "conda-pip-latest" ]]; then - # Check that pip can automatically build scikit-learn with the build - # dependencies specified in pyproject.toml using an isolated build - # environment: - pip install --verbose --editable . -else - if [[ "$BUILD_WITH_ICC" == "true" ]]; then - wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB - sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB - rm GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB - sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main" - sudo apt-get update - sudo apt-get install intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic - source /opt/intel/oneapi/setvars.sh + show_installed_libraries + + if [[ "$UNAMESTR" == "Darwin" && "$SKLEARN_TEST_NO_OPENMP" == "true" ]]; then + # Without openmp, we use the system clang. Here we use /usr/bin/ar + # instead because llvm-ar errors + export AR=/usr/bin/ar + # Make sure omp.h is not present in the conda environment, so that + # using an unprotected "cimport openmp" will make this build fail. At + # the time of writing (2023-01-13), on OSX, blas (mkl or openblas) + # brings in openmp so that you end up having the omp.h include inside + # the conda environment. + find $CONDA_PREFIX -name omp.h -delete -print + # meson >= 1.5 detects OpenMP installed with brew and OpenMP may be installed + # with brew in CI runner. OpenMP was installed with brew in macOS-12 CI + # runners which doesn't seem to be the case in macOS-13 runners anymore, + # but we keep the next line just to be safe ... + brew uninstall --ignore-dependencies --force libomp + fi + + if [[ "$UNAMESTR" == "Linux" ]]; then + # FIXME: temporary fix to link against system libraries on linux + # https://github.com/scikit-learn/scikit-learn/issues/20640 + export LDFLAGS="$LDFLAGS -Wl,--sysroot=/" + fi - # The "build_clib" command is implicitly used to build "libsvm-skl". - # To compile with a different compiler, we also need to specify the - # compiler for this command - python setup.py build_ext --compiler=intelem -i build_clib --compiler=intelem + if [[ "$PIP_BUILD_ISOLATION" == "true" ]]; then + # Check that pip can automatically build scikit-learn with the build + # dependencies specified in pyproject.toml using an isolated build + # environment: + pip install --verbose . + else + if [[ "$UNAMESTR" == "MINGW64"* ]]; then + # Needed on Windows CI to compile with Visual Studio compiler + # otherwise Meson detects a MINGW64 platform and use MINGW64 + # toolchain + ADDITIONAL_PIP_OPTIONS='-Csetup-args=--vsenv' + fi + # Use the pre-installed build dependencies and build directly in the + # current environment. + pip install --verbose --no-build-isolation --editable . $ADDITIONAL_PIP_OPTIONS fi - # Use the pre-installed build dependencies and build directly in the - # current environment. - python setup.py develop -fi -ccache -s + + ccache -s || echo "ccache not installed, skipping ccache statistics" +} + +main() { + pre_python_environment_install + python_environment_install_and_activate + scikit_learn_install +} + +main diff --git a/build_tools/azure/install_setup_conda.sh b/build_tools/azure/install_setup_conda.sh new file mode 100755 index 0000000000000..d09a02cda5a9f --- /dev/null +++ b/build_tools/azure/install_setup_conda.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -e +set -x + +if [[ -z "${CONDA}" ]]; then + # In some runners (macOS-13 and macOS-14 in October 2024) conda is not + # installed so we install it ourselves + MINIFORGE_URL="https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" + wget ${MINIFORGE_URL} -O miniforge.sh + bash miniforge.sh -b -u -p $HOME/miniforge3 + CONDA="$HOME/miniforge3" +else + # In most runners (in October 2024) conda is installed, + # but in a system folder and we want it user writable + sudo chown -R $USER $CONDA +fi + +# Add conda to the PATH so that it can be used in further Azure CI steps. +# Need set +x for ##vso Azure magic otherwise it may add a quote in the PATH. +# For more details, see https://github.com/microsoft/azure-pipelines-tasks/issues/10331 +set +x +echo "##vso[task.prependpath]$CONDA/bin" +set -x diff --git a/build_tools/azure/posix-32.yml b/build_tools/azure/posix-32.yml deleted file mode 100644 index 5e4689a2505e5..0000000000000 --- a/build_tools/azure/posix-32.yml +++ /dev/null @@ -1,84 +0,0 @@ -parameters: - name: '' - vmImage: '' - matrix: [] - dependsOn: [] - condition: ne(variables['Build.Reason'], 'Schedule') - -jobs: -- job: ${{ parameters.name }} - dependsOn: ${{ parameters.dependsOn }} - condition: ${{ parameters.condition }} - pool: - vmImage: ${{ parameters.vmImage }} - variables: - TEST_DIR: '$(Agent.WorkFolder)/tmp_folder' - JUNITXML: 'test-data.xml' - OMP_NUM_THREADS: '2' - OPENBLAS_NUM_THREADS: '2' - SKLEARN_SKIP_NETWORK_TESTS: '1' - NUMPY_VERSION: 'latest' - SCIPY_VERSION: 'latest' - CYTHON_VERSION: 'latest' - JOBLIB_VERSION: 'latest' - PANDAS_VERSION: 'latest' - PYAMG_VERSION: 'latest' - PILLOW_VERSION: 'latest' - MATPLOTLIB_VERSION: 'latest' - PYTEST_VERSION: 'latest' - PYTEST_XDIST_VERSION: 'latest' - THREADPOOLCTL_VERSION: 'latest' - COVERAGE: 'false' - TEST_DOCSTRINGS: 'false' - strategy: - matrix: - ${{ insert }}: ${{ parameters.matrix }} - - steps: - # Container is detached and sleeping, allowing steps to run commmands - # in the container. The TEST_DIR is mapped allowing the host to access - # the JUNITXML file - - script: > - docker container run --rm - --volume $TEST_DIR:/temp_dir - --volume $PWD:/io - -w /io - --detach - --name skcontainer - -e DISTRIB=ubuntu-32 - -e TEST_DIR=/temp_dir - -e JUNITXML=$JUNITXML - -e VIRTUALENV=testvenv - -e NUMPY_VERSION=$NUMPY_VERSION - -e SCIPY_VERSION=$SCIPY_VERSION - -e CYTHON_VERSION=$CYTHON_VERSION - -e JOBLIB_VERSION=$JOBLIB_VERSION - -e PANDAS_VERSION=$PANDAS_VERSION - -e PYAMG_VERSION=$PYAMG_VERSION - -e PILLOW_VERSION=$PILLOW_VERSION - -e MATPLOTLIB_VERSION=$MATPLOTLIB_VERSION - -e PYTEST_VERSION=$PYTEST_VERSION - -e PYTEST_XDIST_VERSION=$PYTEST_XDIST_VERSION - -e THREADPOOLCTL_VERSION=$THREADPOOLCTL_VERSION - -e OMP_NUM_THREADS=$OMP_NUM_THREADS - -e OPENBLAS_NUM_THREADS=$OPENBLAS_NUM_THREADS - -e SKLEARN_SKIP_NETWORK_TESTS=$SKLEARN_SKIP_NETWORK_TESTS - i386/ubuntu:18.04 - sleep 1000000 - displayName: 'Start container' - - script: > - docker exec skcontainer ./build_tools/azure/install.sh - displayName: 'Install' - - script: > - docker exec skcontainer ./build_tools/azure/test_script.sh - displayName: 'Test Library' - - task: PublishTestResults@2 - inputs: - testResultsFiles: '$(TEST_DIR)/$(JUNITXML)' - testRunTitle: ${{ format('{0}-$(Agent.JobName)', parameters.name) }} - displayName: 'Publish Test Results' - condition: succeededOrFailed() - - script: > - docker container stop skcontainer - displayName: 'Stop container' - condition: always() diff --git a/build_tools/azure/posix-all-parallel.yml b/build_tools/azure/posix-all-parallel.yml new file mode 100644 index 0000000000000..45d2b4569110f --- /dev/null +++ b/build_tools/azure/posix-all-parallel.yml @@ -0,0 +1,50 @@ +# This configuration allows enables a job based on `posix.yml` to have two modes: +# +# 1. When `[azure parallel]` *is not* in the commit message, then this job will +# run first. If this job succeeds, then all dependent jobs can run. +# 2. When `[azure parallel]` *is* in the commit message, then this job will +# run with name `{{ parameters.name }}_Parallel` along with all other jobs. +# +# To enable this template, all dependent jobs should check if this job succeeded +# or skipped by using: +# dependsOn: in(dependencies[{{ parameters.name }}]['result'], 'Succeeded', 'Skipped') + +parameters: + name: '' + vmImage: '' + matrix: [] + dependsOn: [] + condition: '' + commitMessage: '' + +jobs: + +# When [azure parallel] *is not* in the commit message, this job will run +# first. +- template: posix.yml + parameters: + name: ${{ parameters.name }} + vmImage: ${{ parameters.vmImage }} + matrix: ${{ parameters.matrix }} + dependsOn: ${{ parameters.dependsOn }} + condition: | + and( + ${{ parameters.condition }}, + not(contains(${{ parameters.commitMessage }}, '[azure parallel]')) + ) + +# When [azure parallel] *is* in the commit message, this job and dependent +# jobs will run in parallel. Implementation-wise, the job above is skipped and +# this job, named ${{ parameters.name }}_Parallel, will run in parallel with +# the other jobs. +- template: posix.yml + parameters: + name: ${{ parameters.name }}_Parallel + vmImage: ${{ parameters.vmImage }} + matrix: ${{ parameters.matrix }} + dependsOn: ${{ parameters.dependsOn }} + condition: | + and( + ${{ parameters.condition }}, + contains(${{ parameters.commitMessage }}, '[azure parallel]') + ) diff --git a/build_tools/azure/posix-docker.yml b/build_tools/azure/posix-docker.yml new file mode 100644 index 0000000000000..49b0eb5f0f356 --- /dev/null +++ b/build_tools/azure/posix-docker.yml @@ -0,0 +1,134 @@ +parameters: + name: '' + vmImage: '' + matrix: [] + dependsOn: [] + condition: ne(variables['Build.Reason'], 'Schedule') + +jobs: +- job: ${{ parameters.name }} + dependsOn: ${{ parameters.dependsOn }} + condition: ${{ parameters.condition }} + timeoutInMinutes: 120 + pool: + vmImage: ${{ parameters.vmImage }} + variables: + VIRTUALENV: 'testvenv' + TEST_DIR: '$(Agent.WorkFolder)/tmp_folder' + JUNITXML: 'test-data.xml' + SKLEARN_SKIP_NETWORK_TESTS: '1' + PYTEST_XDIST_VERSION: 'latest' + COVERAGE: 'false' + # Set in azure-pipelines.yml + DISTRIB: '' + DOCKER_CONTAINER: '' + CREATE_ISSUE_ON_TRACKER: 'true' + CCACHE_DIR: $(Pipeline.Workspace)/ccache + CCACHE_COMPRESS: '1' + strategy: + matrix: + ${{ insert }}: ${{ parameters.matrix }} + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.9' + addToPath: false + name: pyTools + displayName: Select python version to run CI python scripts + - bash: $(pyTools.pythonLocation)/bin/python build_tools/azure/get_selected_tests.py + displayName: Check selected tests for all random seeds + condition: eq(variables['Build.Reason'], 'PullRequest') + - task: Cache@2 + inputs: + key: '"ccache-v1" | "$(Agent.JobName)" | "$(Build.BuildNumber)"' + restoreKeys: | + "ccache-v1" | "$(Agent.JobName)" + path: $(CCACHE_DIR) + displayName: ccache + continueOnError: true + - script: > + mkdir -p $CCACHE_DIR + # Container is detached and sleeping, allowing steps to run commands + # in the container. The TEST_DIR is mapped allowing the host to access + # the JUNITXML file + - script: > + docker container run --rm + --volume $TEST_DIR:/temp_dir + --volume $BUILD_REPOSITORY_LOCALPATH:/repo_localpath + --volume $PWD:/io + --volume $CCACHE_DIR:/ccache + -w /io + --detach + --name skcontainer + -e BUILD_SOURCESDIRECTORY=/io + -e TEST_DIR=/temp_dir + -e CCACHE_DIR=/ccache + -e BUILD_REPOSITORY_LOCALPATH=/repo_localpath + -e COVERAGE + -e DISTRIB + -e LOCK_FILE + -e JUNITXML + -e VIRTUALENV + -e PYTEST_XDIST_VERSION + -e SKLEARN_SKIP_NETWORK_TESTS + -e SELECTED_TESTS + -e CCACHE_COMPRESS + -e BUILD_SOURCEVERSIONMESSAGE + -e BUILD_REASON + $DOCKER_CONTAINER + sleep 1000000 + displayName: 'Start container' + - script: > + docker exec skcontainer ./build_tools/azure/install.sh + displayName: 'Install' + - script: > + docker exec skcontainer ./build_tools/azure/test_script.sh + displayName: 'Test Library' + - script: > + docker exec skcontainer ./build_tools/azure/combine_coverage_reports.sh + condition: and(succeeded(), eq(variables['COVERAGE'], 'true'), + eq(variables['SELECTED_TESTS'], '')) + displayName: 'Combine coverage' + - task: PublishTestResults@2 + inputs: + testResultsFiles: '$(TEST_DIR)/$(JUNITXML)' + testRunTitle: ${{ format('{0}-$(Agent.JobName)', parameters.name) }} + displayName: 'Publish Test Results' + condition: succeededOrFailed() + - script: > + docker container stop skcontainer + displayName: 'Stop container' + condition: always() + - bash: | + set -ex + if [[ $(BOT_GITHUB_TOKEN) == "" ]]; then + echo "GitHub Token is not set. Issue tracker will not be updated." + exit + fi + + LINK_TO_RUN="https://dev.azure.com/$BUILD_REPOSITORY_NAME/_build/results?buildId=$BUILD_BUILDID&view=logs&j=$SYSTEM_JOBID" + CI_NAME="$SYSTEM_JOBIDENTIFIER" + ISSUE_REPO="$BUILD_REPOSITORY_NAME" + + $(pyTools.pythonLocation)/bin/pip install defusedxml PyGithub + $(pyTools.pythonLocation)/bin/python maint_tools/update_tracking_issue.py \ + $(BOT_GITHUB_TOKEN) \ + $CI_NAME \ + $ISSUE_REPO \ + $LINK_TO_RUN \ + --junit-file $JUNIT_FILE \ + --auto-close false + displayName: 'Update issue tracker' + env: + JUNIT_FILE: $(TEST_DIR)/$(JUNITXML) + condition: and(succeededOrFailed(), eq(variables['CREATE_ISSUE_ON_TRACKER'], 'true'), + eq(variables['Build.Reason'], 'Schedule')) + - bash: bash build_tools/azure/upload_codecov.sh + condition: and(succeeded(), eq(variables['COVERAGE'], 'true'), + eq(variables['SELECTED_TESTS'], '')) + displayName: 'Upload To Codecov' + retryCountOnTaskFailure: 5 + env: + CODECOV_TOKEN: $(CODECOV_TOKEN) + JUNIT_FILE: $(TEST_DIR)/$(JUNITXML) diff --git a/build_tools/azure/posix.yml b/build_tools/azure/posix.yml index ae5726aab0b65..e0f504ba540db 100644 --- a/build_tools/azure/posix.yml +++ b/build_tools/azure/posix.yml @@ -9,46 +9,44 @@ jobs: - job: ${{ parameters.name }} dependsOn: ${{ parameters.dependsOn }} condition: ${{ parameters.condition }} + timeoutInMinutes: 120 pool: vmImage: ${{ parameters.vmImage }} variables: TEST_DIR: '$(Agent.WorkFolder)/tmp_folder' VIRTUALENV: 'testvenv' JUNITXML: 'test-data.xml' - OMP_NUM_THREADS: '2' - OPENBLAS_NUM_THREADS: '2' SKLEARN_SKIP_NETWORK_TESTS: '1' CCACHE_DIR: $(Pipeline.Workspace)/ccache CCACHE_COMPRESS: '1' - NUMPY_VERSION: 'latest' - SCIPY_VERSION: 'latest' - CYTHON_VERSION: 'latest' - JOBLIB_VERSION: 'latest' - PANDAS_VERSION: 'latest' - PYAMG_VERSION: 'latest' - PILLOW_VERSION: 'latest' - MATPLOTLIB_VERSION: 'latest' - PYTEST_VERSION: 'latest' PYTEST_XDIST_VERSION: 'latest' - THREADPOOLCTL_VERSION: 'latest' COVERAGE: 'true' - TEST_DOCSTRINGS: 'false' + CREATE_ISSUE_ON_TRACKER: 'true' strategy: matrix: ${{ insert }}: ${{ parameters.matrix }} steps: - - bash: echo "##vso[task.prependpath]$CONDA/bin" - displayName: Add conda to PATH - condition: startsWith(variables['DISTRIB'], 'conda') - - bash: sudo chown -R $USER $CONDA - displayName: Take ownership of conda installation + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.9' + addToPath: false + name: pyTools + displayName: Select python version to run CI python scripts + - bash: $(pyTools.pythonLocation)/bin/python build_tools/azure/get_selected_tests.py + displayName: Check selected tests for all random seeds + condition: eq(variables['Build.Reason'], 'PullRequest') + - bash: build_tools/azure/install_setup_conda.sh + displayName: Install conda if necessary and set it up condition: startsWith(variables['DISTRIB'], 'conda') - task: Cache@2 inputs: - key: '"$(Agent.JobName)"' + key: '"ccache-v1" | "$(Agent.JobName)" | "$(Build.BuildNumber)"' + restoreKeys: | + "ccache-v1" | "$(Agent.JobName)" path: $(CCACHE_DIR) displayName: ccache + continueOnError: true - script: | build_tools/azure/install.sh displayName: 'Install' @@ -58,19 +56,54 @@ jobs: - script: | build_tools/azure/test_docs.sh displayName: 'Test Docs' + condition: and(succeeded(), eq(variables['SELECTED_TESTS'], '')) - script: | build_tools/azure/test_pytest_soft_dependency.sh displayName: 'Test Soft Dependency' - condition: eq(variables['CHECK_PYTEST_SOFT_DEPENDENCY'], 'true') + condition: and(succeeded(), + eq(variables['CHECK_PYTEST_SOFT_DEPENDENCY'], 'true'), + eq(variables['SELECTED_TESTS'], '')) + - script: | + build_tools/azure/combine_coverage_reports.sh + condition: and(succeeded(), eq(variables['COVERAGE'], 'true'), + eq(variables['SELECTED_TESTS'], '')) + displayName: 'Combine coverage' - task: PublishTestResults@2 inputs: testResultsFiles: '$(TEST_DIR)/$(JUNITXML)' testRunTitle: ${{ format('{0}-$(Agent.JobName)', parameters.name) }} displayName: 'Publish Test Results' condition: succeededOrFailed() + - bash: | + set -ex + if [[ $(BOT_GITHUB_TOKEN) == "" ]]; then + echo "GitHub Token is not set. Issue tracker will not be updated." + exit + fi + + LINK_TO_RUN="https://dev.azure.com/$BUILD_REPOSITORY_NAME/_build/results?buildId=$BUILD_BUILDID&view=logs&j=$SYSTEM_JOBID" + CI_NAME="$SYSTEM_JOBIDENTIFIER" + ISSUE_REPO="$BUILD_REPOSITORY_NAME" + + $(pyTools.pythonLocation)/bin/pip install defusedxml PyGithub + $(pyTools.pythonLocation)/bin/python maint_tools/update_tracking_issue.py \ + $(BOT_GITHUB_TOKEN) \ + $CI_NAME \ + $ISSUE_REPO \ + $LINK_TO_RUN \ + --junit-file $JUNIT_FILE \ + --auto-close false + displayName: 'Update issue tracker' + env: + JUNIT_FILE: $(TEST_DIR)/$(JUNITXML) + condition: and(succeededOrFailed(), eq(variables['CREATE_ISSUE_ON_TRACKER'], 'true'), + eq(variables['Build.Reason'], 'Schedule')) - script: | build_tools/azure/upload_codecov.sh - condition: and(succeeded(), eq(variables['COVERAGE'], 'true')) + condition: and(succeeded(), eq(variables['COVERAGE'], 'true'), + eq(variables['SELECTED_TESTS'], '')) displayName: 'Upload To Codecov' + retryCountOnTaskFailure: 5 env: CODECOV_TOKEN: $(CODECOV_TOKEN) + JUNIT_FILE: $(TEST_DIR)/$(JUNITXML) diff --git a/build_tools/azure/pylatest_conda_forge_mkl_linux-64_conda.lock b/build_tools/azure/pylatest_conda_forge_mkl_linux-64_conda.lock new file mode 100644 index 0000000000000..e99219a40736d --- /dev/null +++ b/build_tools/azure/pylatest_conda_forge_mkl_linux-64_conda.lock @@ -0,0 +1,246 @@ +# Generated by conda-lock. +# platform: linux-64 +# input_hash: f524d159a11a0a80ead3448f16255169f24edde269f6b81e8e28453bc4f7fc53 +@EXPLICIT +https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2#0c96522c6bdaed4b1566d11387caaf45 +https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2#34893075a5c9e55cdafac56607368fc6 +https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2#4d59c254e01d9cde7957100457e2d5fb +https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-h77eed37_3.conda#49023d73832ef61042f6a237cb2687e7 +https://conda.anaconda.org/conda-forge/linux-64/libopentelemetry-cpp-headers-1.21.0-ha770c72_0.conda#11b1bed92c943d3b741e8a1e1a815ed1 +https://conda.anaconda.org/conda-forge/linux-64/mkl-include-2024.2.2-ha957f24_16.conda#42b0d14354b5910a9f41e29289914f6b +https://conda.anaconda.org/conda-forge/linux-64/nlohmann_json-3.12.0-h3f2d84a_0.conda#d76872d096d063e226482c99337209dc +https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-7_cp313.conda#e84b44e6300f1703cb25d29120c5b1d8 +https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda#4222072737ccff51314b5ece9c7d6f5a +https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.4.26-hbd8a1cb_0.conda#95db94f75ba080a22eb623590993167b +https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-0.tar.bz2#f766549260d6815b0c52253f1fb1bb29 +https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.43-h712a8e2_4.conda#01f8d123c96816249efd255a31ad7712 +https://conda.anaconda.org/conda-forge/linux-64/libglvnd-1.7.0-ha4b6fd6_2.conda#434ca7e50e40f4918ab701e3facd59a0 +https://conda.anaconda.org/conda-forge/linux-64/llvm-openmp-20.1.6-h024ca30_0.conda#e4ece7ed81e43ae97a3b58ac4230c3c5 +https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-3_kmp_llvm.conda#ee5c2118262e30b972bc0b4db8ef0ba5 +https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2#fee5683a3f04bd15cbd8318b096a27ab +https://conda.anaconda.org/conda-forge/linux-64/libegl-1.7.0-ha4b6fd6_2.conda#c151d5eb730e9b7480e6d48c0fc44048 +https://conda.anaconda.org/conda-forge/linux-64/libopengl-1.7.0-ha4b6fd6_2.conda#7df50d44d4a14d6c31a2c54f2cd92157 +https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.1.0-h767d61c_2.conda#ea8ac52380885ed41c1baa8f1d6d2b93 +https://conda.anaconda.org/conda-forge/linux-64/alsa-lib-1.2.14-hb9d3cd8_0.conda#76df83c2a9035c54df5d04ff81bcc02d +https://conda.anaconda.org/conda-forge/linux-64/aws-c-common-0.12.3-hb9d3cd8_0.conda#8448031a22c697fac3ed98d69e8a9160 +https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.34.5-hb9d3cd8_0.conda#f7f0d6cc2dc986d42ac2689ec88192be +https://conda.anaconda.org/conda-forge/linux-64/libbrotlicommon-1.1.0-hb9d3cd8_2.conda#41b599ed2b02abcfdd84302bff174b23 +https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.24-h86f0d12_0.conda#64f0c503da58ec25ebd359e4d990afa8 +https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.7.0-h5888daf_0.conda#db0bfbe7dd197b68ad5f30333bae6ce0 +https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.6-h2dba641_1.conda#ede4673863426c0883c0063d853bbd85 +https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-15.1.0-h69a702a_2.conda#ddca86c7040dd0e73b2b69bd7833d225 +https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-15.1.0-hcea5267_2.conda#01de444988ed960031dbe84cf4f9b1fc +https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.18-h4ce23a2_1.conda#e796ff8ddc598affdf7c173d6145f087 +https://conda.anaconda.org/conda-forge/linux-64/libjpeg-turbo-3.1.0-hb9d3cd8_0.conda#9fa334557db9f63da6c9285fd2a48638 +https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.1-hb9d3cd8_1.conda#a76fd702c93cd2dfd89eff30a5fd45a8 +https://conda.anaconda.org/conda-forge/linux-64/libmpdec-4.0.0-hb9d3cd8_0.conda#c7e925f37e3b40d893459e625f6a53f1 +https://conda.anaconda.org/conda-forge/linux-64/libntlm-1.8-hb9d3cd8_0.conda#7c7927b404672409d9917d49bff5f2d6 +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-15.1.0-h8f9b012_2.conda#1cb1c67961f6dd257eae9e9691b341aa +https://conda.anaconda.org/conda-forge/linux-64/libutf8proc-2.10.0-h202a827_0.conda#0f98f3e95272d118f7931b6bef69bfe5 +https://conda.anaconda.org/conda-forge/linux-64/libuv-1.51.0-hb9d3cd8_0.conda#1349c022c92c5efd3fd705a79a5804d8 +https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.5.0-h851e524_0.conda#63f790534398730f59e1b899c3644d4a +https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda#edb0dca6bc32e4f4789199455a1dbeb8 +https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda#47e340acb35de30501a76c7c799c41d7 +https://conda.anaconda.org/conda-forge/linux-64/openssl-3.5.0-h7b32b05_1.conda#de356753cfdbffcde5bb1e86e3aa6cd0 +https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-hb9d3cd8_1002.conda#b3c17d95b5a10c6e64a21fa17573e70e +https://conda.anaconda.org/conda-forge/linux-64/xorg-libice-1.1.2-hb9d3cd8_0.conda#fb901ff28063514abb6046c9ec2c4a45 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxau-1.0.12-hb9d3cd8_0.conda#f6ebe2cb3f82ba6c057dde5d9debe4f7 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdmcp-1.1.5-hb9d3cd8_0.conda#8035c64cb77ed555e3f150b7b3972480 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-cal-0.9.1-h5e3027f_0.conda#da0b556585013ad26b3c052b61205f74 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-compression-0.3.1-hafb2847_5.conda#e96cc668c0f9478f5771b37d57f90386 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-sdkutils-0.2.4-hafb2847_0.conda#65853df44b7e4029d978c50be888ed89 +https://conda.anaconda.org/conda-forge/linux-64/aws-checksums-0.2.7-hafb2847_1.conda#6d28d50637fac4f081a0903b4b33d56d +https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h4bc722e_7.conda#62ee74e96c5ebb0af99386de58cf9553 +https://conda.anaconda.org/conda-forge/linux-64/double-conversion-3.3.1-h5888daf_0.conda#bfd56492d8346d669010eccafe0ba058 +https://conda.anaconda.org/conda-forge/linux-64/gflags-2.2.2-h5888daf_1005.conda#d411fc29e338efb48c5fd4576d71d881 +https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.1-h166bdaf_0.tar.bz2#30186d27e2c9fa62b45fb1476b7200e3 +https://conda.anaconda.org/conda-forge/linux-64/lerc-4.0.0-h0aef613_1.conda#9344155d33912347b37f0ae6c410a835 +https://conda.anaconda.org/conda-forge/linux-64/libabseil-20250127.1-cxx17_hbbce691_0.conda#00290e549c5c8a32cc271020acc9ec6b +https://conda.anaconda.org/conda-forge/linux-64/libbrotlidec-1.1.0-hb9d3cd8_2.conda#9566f0bd264fbd463002e759b8a82401 +https://conda.anaconda.org/conda-forge/linux-64/libbrotlienc-1.1.0-hb9d3cd8_2.conda#06f70867945ea6a84d35836af780f1de +https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20250104-pl5321h7949ede_0.conda#c277e0a4d549b03ac1e9d6cbbe3d017b +https://conda.anaconda.org/conda-forge/linux-64/libev-4.33-hd590300_2.conda#172bf1cd1ff8629f2b1179945ed45055 +https://conda.anaconda.org/conda-forge/linux-64/libevent-2.1.12-hf998b51_1.conda#a1cfcc585f0c42bf8d5546bb1dfb668d +https://conda.anaconda.org/conda-forge/linux-64/libgfortran-15.1.0-h69a702a_2.conda#f92e6e0a3c0c0c85561ef61aa59d555d +https://conda.anaconda.org/conda-forge/linux-64/libpciaccess-0.18-hd590300_0.conda#48f4330bfcd959c3cfb704d424903c82 +https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.47-h943b412_0.conda#55199e2ae2c3651f6f9b2a447b47bdc9 +https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.50.0-hee588c1_0.conda#71888e92098d0f8c41b09a671ad289bc +https://conda.anaconda.org/conda-forge/linux-64/libssh2-1.11.1-hcf80075_0.conda#eecce068c7e4eddeb169591baac20ac4 +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-15.1.0-h4852527_2.conda#9d2072af184b5caa29492bf2344597bb +https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda#40b61aab5c7ba9ff276c41cfffe6b80b +https://conda.anaconda.org/conda-forge/linux-64/libxcb-1.17.0-h8a09558_0.conda#92ed62436b625154323d40d5f2f11dd7 +https://conda.anaconda.org/conda-forge/linux-64/lz4-c-1.10.0-h5888daf_1.conda#9de5350a85c4a20c685259b889aa6393 +https://conda.anaconda.org/conda-forge/linux-64/ninja-1.12.1-hff21bea_1.conda#2322531904f27501ee19847b87ba7c64 +https://conda.anaconda.org/conda-forge/linux-64/pixman-0.46.0-h29eaf8c_0.conda#d2f1c87d4416d1e7344cf92b1aaee1c4 +https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8c095d6_2.conda#283b96675859b20a825f8fa30f311446 +https://conda.anaconda.org/conda-forge/linux-64/s2n-1.5.19-h763c568_0.conda#1e0cc52ba44d40408e5fb7b4dbdcde8f +https://conda.anaconda.org/conda-forge/linux-64/sleef-3.8-h1b44611_0.conda#aec4dba5d4c2924730088753f6fa164b +https://conda.anaconda.org/conda-forge/linux-64/snappy-1.2.1-h8bd8927_1.conda#3b3e64af585eadfb52bb90b553db5edf +https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_hd72426e_102.conda#a0116df4f4ed05c303811a837d5b39d8 +https://conda.anaconda.org/conda-forge/linux-64/wayland-1.23.1-h3e06ad9_1.conda#a37843723437ba75f42c9270ffe800b1 +https://conda.anaconda.org/conda-forge/linux-64/zlib-1.3.1-hb9d3cd8_2.conda#c9f075ab2f33b3bbee9e62d4ad0a6cd8 +https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb8e6e7a_2.conda#6432cb5d4ac0046c3ac0a8a0f95842f9 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-io-0.19.1-h93b6419_1.conda#86689a4e294ac0e78b046f5989157f7a +https://conda.anaconda.org/conda-forge/linux-64/brotli-bin-1.1.0-hb9d3cd8_2.conda#c63b5e52939e795ba8d26e35d767a843 +https://conda.anaconda.org/conda-forge/linux-64/glog-0.7.1-hbabe93e_0.conda#ff862eebdfeb2fd048ae9dc92510baca +https://conda.anaconda.org/conda-forge/linux-64/gmp-6.3.0-hac33072_2.conda#c94a5994ef49749880a8139cf9afcbe1 +https://conda.anaconda.org/conda-forge/linux-64/graphite2-1.3.13-h59595ed_1003.conda#f87c7b7c2cb45f323ffbce941c78ab7c +https://conda.anaconda.org/conda-forge/linux-64/icu-75.1-he02047a_0.conda#8b189310083baabfb622af68fd9d3ae3 +https://conda.anaconda.org/conda-forge/linux-64/krb5-1.21.3-h659f571_0.conda#3f43953b7d3fb3aaa1d0d0723d91e368 +https://conda.anaconda.org/conda-forge/linux-64/libcrc32c-1.1.2-h9c3ff4c_0.tar.bz2#c965a5aa0d5c1c37ffc62dff36e28400 +https://conda.anaconda.org/conda-forge/linux-64/libdrm-2.4.124-hb9d3cd8_0.conda#8bc89311041d7fcb510238cf0848ccae +https://conda.anaconda.org/conda-forge/linux-64/libfreetype6-2.13.3-h48d6fc4_1.conda#3c255be50a506c50765a93a6644f32fe +https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-15.1.0-h69a702a_2.conda#a483a87b71e974bb75d1b9413d4436dd +https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.64.0-h161d5f1_0.conda#19e57602824042dfd0446292ef90488b +https://conda.anaconda.org/conda-forge/linux-64/libprotobuf-5.29.3-h501fc15_1.conda#edb86556cf4a0c133e7932a1597ff236 +https://conda.anaconda.org/conda-forge/linux-64/libre2-11-2024.07.02-hba17884_3.conda#545e93a513c10603327c76c15485e946 +https://conda.anaconda.org/conda-forge/linux-64/libthrift-0.21.0-h0e7cc3e_0.conda#dcb95c0a98ba9ff737f7ae482aef7833 +https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.7.0-hf01ce69_5.conda#e79a094918988bb1807462cd42c83962 +https://conda.anaconda.org/conda-forge/linux-64/pcre2-10.45-hc749103_0.conda#b90bece58b4c2bf25969b70f3be42d25 +https://conda.anaconda.org/conda-forge/linux-64/python-3.13.3-hf636f53_101_cp313.conda#10622e12d649154af0bd76bcf33a7c5c +https://conda.anaconda.org/conda-forge/linux-64/qhull-2020.2-h434a139_5.conda#353823361b1d27eb3960efb076dfcaf6 +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-0.4.1-hb711507_2.conda#8637c3e5821654d0edf97e2b0404b443 +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-keysyms-0.4.1-hb711507_0.conda#ad748ccca349aec3e91743e08b5e2b50 +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-renderutil-0.3.10-hb711507_0.conda#0e0cbe0564d03a99afd5fd7b362feecd +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-wm-0.4.2-hb711507_0.conda#608e0ef8256b81d04456e8d211eee3e8 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.6-he73a12e_0.conda#1c74ff8c35dcadf952a16f752ca5aa49 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.8.12-h4f16b4b_0.conda#db038ce880f100acc74dba10302b5630 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-event-stream-0.5.4-haaa725d_10.conda#ed15f12bd23f3861d61e3d71c0e639ee +https://conda.anaconda.org/conda-forge/linux-64/aws-c-http-0.10.1-hd7992d4_3.conda#8ee52f649777534cd21fc6905b83316d +https://conda.anaconda.org/conda-forge/linux-64/brotli-1.1.0-hb9d3cd8_2.conda#98514fe74548d768907ce7a13f680e8f +https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda#962b9857ee8e7018c22f2776ffa0b2d7 +https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.3-py313hd8ed1ab_101.conda#904a822cbd380adafb9070debf8579a8 +https://conda.anaconda.org/conda-forge/noarch/cycler-0.12.1-pyhd8ed1ab_1.conda#44600c4667a319d67dbe0681fc0bc833 +https://conda.anaconda.org/conda-forge/linux-64/cyrus-sasl-2.1.27-h54b06d7_7.conda#dce22f70b4e5a407ce88f2be046f4ceb +https://conda.anaconda.org/conda-forge/linux-64/cython-3.1.1-py313h5dec8f5_1.conda#f114755cdd37627732b1884b7b15d4b5 +https://conda.anaconda.org/conda-forge/noarch/execnet-2.1.1-pyhd8ed1ab_1.conda#a71efeae2c160f6789900ba2631a2c90 +https://conda.anaconda.org/conda-forge/noarch/filelock-3.18.0-pyhd8ed1ab_0.conda#4547b39256e296bb758166893e909a7c +https://conda.anaconda.org/conda-forge/noarch/fsspec-2025.5.1-pyhd8ed1ab_0.conda#2d2c9ef879a7e64e2dc657b09272c2b6 +https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_1.conda#6837f3eff7dcea42ecd714ce1ac2b108 +https://conda.anaconda.org/conda-forge/linux-64/kiwisolver-1.4.7-py313h33d0bda_0.conda#9862d13a5e466273d5a4738cffcb8d6c +https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.17-h717163a_0.conda#000e85703f0fd9594c81710dd5066471 +https://conda.anaconda.org/conda-forge/linux-64/libcups-2.3.3-h4637d8d_4.conda#d4529f4dff3057982a7617c7ac58fde3 +https://conda.anaconda.org/conda-forge/linux-64/libcurl-8.14.0-h332b0f4_0.conda#d1738cf06503218acee63669029fd8e8 +https://conda.anaconda.org/conda-forge/linux-64/libfreetype-2.13.3-ha770c72_1.conda#51f5be229d83ecd401fb369ab96ae669 +https://conda.anaconda.org/conda-forge/linux-64/libglib-2.84.2-h3618099_0.conda#072ab14a02164b7c0c089055368ff776 +https://conda.anaconda.org/conda-forge/linux-64/libglx-1.7.0-ha4b6fd6_2.conda#c8013e438185f33b13814c5c488acd5c +https://conda.anaconda.org/conda-forge/linux-64/libhiredis-1.0.2-h2cc385e_0.tar.bz2#b34907d3a81a3cd8095ee83d174c074a +https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.13.8-h4bc477f_0.conda#14dbe05b929e329dbaa6f2d0aa19466d +https://conda.anaconda.org/conda-forge/linux-64/markupsafe-3.0.2-py313h8060acc_1.conda#21b62c55924f01b6eef6827167b46acb +https://conda.anaconda.org/conda-forge/noarch/meson-1.8.1-pyhe01879c_0.conda#f3cccd9a6ce5331ae33f69ade5529162 +https://conda.anaconda.org/conda-forge/linux-64/mpfr-4.2.1-h90cbb55_3.conda#2eeb50cab6652538eee8fc0bc3340c81 +https://conda.anaconda.org/conda-forge/noarch/mpmath-1.3.0-pyhd8ed1ab_1.conda#3585aa87c43ab15b167b574cd73b057b +https://conda.anaconda.org/conda-forge/noarch/munkres-1.1.4-pyh9f0ad1d_0.tar.bz2#2ba8498c1018c1e9c61eb99b973dfe19 +https://conda.anaconda.org/conda-forge/noarch/networkx-3.4.2-pyh267e887_2.conda#fd40bf7f7f4bc4b647dc8512053d9873 +https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.3-h5fbd93e_0.conda#9e5816bc95d285c115a3ebc2f8563564 +https://conda.anaconda.org/conda-forge/linux-64/orc-2.1.2-h17f744e_0.conda#ef7f9897a244b2023a066c22a1089ce4 +https://conda.anaconda.org/conda-forge/noarch/packaging-25.0-pyh29332c3_1.conda#58335b26c38bf4a20f399384c33cbcf9 +https://conda.anaconda.org/conda-forge/noarch/pip-25.1.1-pyh145f28c_0.conda#01384ff1639c6330a0924791413b8714 +https://conda.anaconda.org/conda-forge/noarch/pluggy-1.6.0-pyhd8ed1ab_0.conda#7da7ccd349dbf6487a7778579d2bb971 +https://conda.anaconda.org/conda-forge/noarch/pybind11-global-2.13.6-pyh217bc35_3.conda#730a5284e26d6bdb73332dafb26aec82 +https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.2.3-pyhd8ed1ab_1.conda#513d3c262ee49b54a8fec85c5bc99764 +https://conda.anaconda.org/conda-forge/noarch/python-tzdata-2025.2-pyhd8ed1ab_0.conda#88476ae6ebd24f39261e0854ac244f33 +https://conda.anaconda.org/conda-forge/noarch/pytz-2025.2-pyhd8ed1ab_0.conda#bc8e3267d44011051f2eb14d22fb0960 +https://conda.anaconda.org/conda-forge/linux-64/re2-2024.07.02-h9925aae_3.conda#6f445fb139c356f903746b2b91bbe786 +https://conda.anaconda.org/conda-forge/noarch/setuptools-75.8.2-pyhff2d567_0.conda#9bddfdbf4e061821a1a443f93223be61 +https://conda.anaconda.org/conda-forge/noarch/six-1.17.0-pyhd8ed1ab_0.conda#a451d576819089b0d672f18768be0f65 +https://conda.anaconda.org/conda-forge/noarch/threadpoolctl-3.6.0-pyhecae5ae_0.conda#9d64911b31d57ca443e9f1e36b04385f +https://conda.anaconda.org/conda-forge/noarch/toml-0.10.2-pyhd8ed1ab_1.conda#b0dd904de08b7db706167240bf37b164 +https://conda.anaconda.org/conda-forge/noarch/tomli-2.2.1-pyhd8ed1ab_1.conda#ac944244f1fed2eb49bae07193ae8215 +https://conda.anaconda.org/conda-forge/linux-64/tornado-6.5.1-py313h536fd9c_0.conda#e9434a5155db25c38ade26f71a2f5a48 +https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.13.2-pyh29332c3_0.conda#83fc6ae00127671e301c9f44254c31b8 +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-image-0.4.0-hb711507_2.conda#a0901183f08b6c7107aab109733a3c91 +https://conda.anaconda.org/conda-forge/linux-64/xkeyboard-config-2.44-hb9d3cd8_0.conda#7c91bfc90672888259675ad2ad28af9c +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.6-hb9d3cd8_0.conda#febbab7d15033c913d53c7a2c102309d +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxfixes-6.0.1-hb9d3cd8_0.conda#4bdb303603e9821baf5fe5fdff1dc8f8 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.12-hb9d3cd8_0.conda#96d57aba173e878a2089d5638016dc5e +https://conda.anaconda.org/conda-forge/linux-64/aws-c-auth-0.9.0-h3318fae_10.conda#0966b2b633190f1f24a92ddb25559ff8 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-mqtt-0.13.0-h7b3935a_4.conda#d60f0cefef7f0dd076d8335781a676fd +https://conda.anaconda.org/conda-forge/linux-64/azure-core-cpp-1.14.0-h5cfcd09_0.conda#0a8838771cc2e985cd295e01ae83baf1 +https://conda.anaconda.org/conda-forge/linux-64/ccache-4.11.3-h80c52d3_0.conda#eb517c6a2b960c3ccb6f1db1005f063a +https://conda.anaconda.org/conda-forge/linux-64/coverage-7.8.2-py313h8060acc_0.conda#b278629953bd3424060870fca744de4a +https://conda.anaconda.org/conda-forge/linux-64/dbus-1.16.2-h3c4dab8_0.conda#679616eb5ad4e521c83da4650860aba7 +https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.0-pyhd8ed1ab_0.conda#72e42d28960d875c7654614f8b50939a +https://conda.anaconda.org/conda-forge/linux-64/fonttools-4.58.1-py313h8060acc_0.conda#f03a1dc39346922cb5cf2ee190ac9b95 +https://conda.anaconda.org/conda-forge/linux-64/freetype-2.13.3-ha770c72_1.conda#9ccd736d31e0c6e41f54e704e5312811 +https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.6-pyhd8ed1ab_0.conda#446bd6c8cb26050d528881df495ce646 +https://conda.anaconda.org/conda-forge/noarch/joblib-1.5.1-pyhd8ed1ab_0.conda#fb1c14694de51a476ce8636d92b6f42c +https://conda.anaconda.org/conda-forge/linux-64/libgl-1.7.0-ha4b6fd6_2.conda#928b8be80851f5d8ffb016f9c81dae7a +https://conda.anaconda.org/conda-forge/linux-64/libgrpc-1.71.0-h8e591d7_1.conda#c3cfd72cbb14113abee7bbd86f44ad69 +https://conda.anaconda.org/conda-forge/linux-64/libhwloc-2.11.2-default_h0d58e46_1001.conda#804ca9e91bcaea0824a341d55b1684f2 +https://conda.anaconda.org/conda-forge/linux-64/libllvm20-20.1.6-he9d0ab4_0.conda#bf8ccdd2c1c1a54a3fa25bb61f26460e +https://conda.anaconda.org/conda-forge/linux-64/libxkbcommon-1.10.0-h65c71a3_0.conda#fedf6bfe5d21d21d2b1785ec00a8889a +https://conda.anaconda.org/conda-forge/linux-64/libxslt-1.1.39-h76b75d6_0.conda#e71f31f8cfb0a91439f2086fc8aa0461 +https://conda.anaconda.org/conda-forge/linux-64/mpc-1.3.1-h24ddda3_1.conda#aa14b9a5196a6d8dd364164b7ce56acf +https://conda.anaconda.org/conda-forge/linux-64/openldap-2.6.10-he970967_0.conda#2e5bf4f1da39c0b32778561c3c4e5878 +https://conda.anaconda.org/conda-forge/linux-64/pillow-11.2.1-py313h8db990d_0.conda#91b00afee98d72d29dc3d1c1ab0008d7 +https://conda.anaconda.org/conda-forge/linux-64/prometheus-cpp-1.3.0-ha5d0236_0.conda#a83f6a2fdc079e643237887a37460668 +https://conda.anaconda.org/conda-forge/noarch/pybind11-2.13.6-pyhc790b64_3.conda#1594696beebf1ecb6d29a1136f859a74 +https://conda.anaconda.org/conda-forge/noarch/pyproject-metadata-0.9.1-pyhd8ed1ab_0.conda#22ae7c6ea81e0c8661ef32168dda929b +https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhff2d567_1.conda#5ba79d7c71f03c678c8ead841f347d6e +https://conda.anaconda.org/conda-forge/noarch/python-gil-3.13.3-h4df99d1_101.conda#82c2641f2f0f513f7d2d1b847a2588e3 +https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.13.2-h0e9735f_0.conda#568ed1300869dca0ba09fb750cda5dbb +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-cursor-0.1.5-hb9d3cd8_0.conda#eb44b3b6deb1cab08d72cb61686fe64c +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxcomposite-0.4.6-hb9d3cd8_2.conda#d3c295b50f092ab525ffe3c2aa4b7413 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxcursor-1.2.3-hb9d3cd8_0.conda#2ccd714aa2242315acaf0a67faea780b +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdamage-1.1.6-hb9d3cd8_0.conda#b5fcc7172d22516e1f965490e65e33a4 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxi-1.8.2-hb9d3cd8_0.conda#17dcc85db3c7886650b8908b183d6876 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrandr-1.5.4-hb9d3cd8_0.conda#2de7f99d6581a4a7adbff607b5c278ca +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxxf86vm-1.1.6-hb9d3cd8_0.conda#5efa5fa6243a622445fdfd72aee15efa +https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda#aaa2a381ccc56eac91d63b6c1240312f +https://conda.anaconda.org/conda-forge/linux-64/aws-c-s3-0.8.0-h365f71b_1.conda#eff3d619b784eb000e62a6fd7797d6a5 +https://conda.anaconda.org/conda-forge/linux-64/azure-identity-cpp-1.10.0-h113e628_0.conda#73f73f60854f325a55f1d31459f2ab73 +https://conda.anaconda.org/conda-forge/linux-64/azure-storage-common-cpp-12.8.0-h736e048_1.conda#13de36be8de3ae3f05ba127631599213 +https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.15.0-h7e30c49_1.conda#8f5b0b297b59e1ac160ad4beec99dbee +https://conda.anaconda.org/conda-forge/linux-64/gmpy2-2.2.1-py313h11186cd_0.conda#54d020e0eaacf1e99bfb2410b9aa2e5e +https://conda.anaconda.org/conda-forge/linux-64/libclang-cpp20.1-20.1.6-default_h1df26ce_0.conda#99ead3b974685e44df8b1e3953503cfc +https://conda.anaconda.org/conda-forge/linux-64/libclang13-20.1.6-default_he06ed0a_0.conda#cc6c469d9d7fc0ac106cef5f45d973a9 +https://conda.anaconda.org/conda-forge/linux-64/libgoogle-cloud-2.36.0-hc4361e1_1.conda#ae36e6296a8dd8e8a9a8375965bf6398 +https://conda.anaconda.org/conda-forge/linux-64/libopentelemetry-cpp-1.21.0-hd1b1c89_0.conda#4b25cd8720fd8d5319206e4f899f2707 +https://conda.anaconda.org/conda-forge/linux-64/libpq-17.5-h27ae623_0.conda#6458be24f09e1b034902ab44fe9de908 +https://conda.anaconda.org/conda-forge/noarch/meson-python-0.18.0-pyh70fd9c4_0.conda#576c04b9d9f8e45285fb4d9452c26133 +https://conda.anaconda.org/conda-forge/linux-64/optree-0.16.0-py313h33d0bda_0.conda#5c211bb056e1a3263a163ba21e3fbf73 +https://conda.anaconda.org/conda-forge/noarch/pytest-8.3.5-pyhd8ed1ab_0.conda#c3c9316209dec74a705a36797970c6be +https://conda.anaconda.org/conda-forge/linux-64/tbb-2021.13.0-hceb3a55_1.conda#ba7726b8df7b9d34ea80e82b097a4893 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxtst-1.2.5-hb9d3cd8_3.conda#7bbe9a0cc0df0ac5f5a8ad6d6a11af2f +https://conda.anaconda.org/conda-forge/linux-64/aws-crt-cpp-0.32.6-h96107e3_3.conda#c8e99a9c0b60e7695de83512e53b9eb3 +https://conda.anaconda.org/conda-forge/linux-64/azure-storage-blobs-cpp-12.13.0-h3cf044e_1.conda#7eb66060455c7a47d9dcdbfa9f46579b +https://conda.anaconda.org/conda-forge/linux-64/cairo-1.18.4-h3394656_0.conda#09262e66b19567aff4f592fb53b28760 +https://conda.anaconda.org/conda-forge/linux-64/libgoogle-cloud-storage-2.36.0-h0121fbd_1.conda#a0f7588c1f0a26d550e7bae4fb49427a +https://conda.anaconda.org/conda-forge/linux-64/mkl-2024.2.2-ha957f24_16.conda#1459379c79dda834673426504d52b319 +https://conda.anaconda.org/conda-forge/noarch/pytest-cov-6.1.1-pyhd8ed1ab_0.conda#1e35d8f975bc0e984a19819aa91c440a +https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.7.0-pyhd8ed1ab_0.conda#15353a2a0ea6dfefaa52fc5ab5b98f41 +https://conda.anaconda.org/conda-forge/noarch/sympy-1.14.0-pyh2585a3b_105.conda#8c09fac3785696e1c477156192d64b91 +https://conda.anaconda.org/conda-forge/linux-64/aws-sdk-cpp-1.11.510-hf282fd2_9.conda#1583a30af14c72e76034ca6e04abe64c +https://conda.anaconda.org/conda-forge/linux-64/azure-storage-files-datalake-cpp-12.12.0-ha633028_1.conda#7c1980f89dd41b097549782121a73490 +https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-11.2.1-h3beb420_0.conda#0e6e192d4b3d95708ad192d957cf3163 +https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-31_hfdb39a5_mkl.conda#bdf4a57254e8248222cb631db4393ff1 +https://conda.anaconda.org/conda-forge/linux-64/mkl-devel-2024.2.2-ha770c72_16.conda#140891ea14285fc634353b31e9e40a95 +https://conda.anaconda.org/conda-forge/linux-64/libarrow-20.0.0-h6b3f9de_5_cpu.conda#2f4c442e11ca9695ff779ffa709e0026 +https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-31_h372d94f_mkl.conda#2a06a6c16b45bd3d10002927ca204b67 +https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-31_hc41d3b0_mkl.conda#10d012ddd7cc1c7ff9093d4974a34e53 +https://conda.anaconda.org/conda-forge/linux-64/qt6-main-6.9.0-h0384650_3.conda#8aa69e15597a205fd6f81781fe62c232 +https://conda.anaconda.org/conda-forge/linux-64/libarrow-acero-20.0.0-hcb10f89_5_cpu.conda#2d29a8510e16df15bfb6c0b149c78c84 +https://conda.anaconda.org/conda-forge/linux-64/liblapacke-3.9.0-31_hbc6e62b_mkl.conda#562026e418363dc346ad5a9e18cce73c +https://conda.anaconda.org/conda-forge/linux-64/libparquet-20.0.0-h081d1f1_5_cpu.conda#6f60536e2136749dc1cb29da302c13e3 +https://conda.anaconda.org/conda-forge/linux-64/libtorch-2.7.0-cpu_mkl_hf6ddc5a_100.conda#6bdda0b10852c6d03b030bab7ec251f0 +https://conda.anaconda.org/conda-forge/linux-64/numpy-2.2.6-py313h17eae1a_0.conda#7a2d2f9adecd86ed5c29c2115354f615 +https://conda.anaconda.org/conda-forge/linux-64/pyarrow-core-20.0.0-py313he5f92c8_0_cpu.conda#2afdef63d9fbc2cd0e52f8e8f3472404 +https://conda.anaconda.org/conda-forge/linux-64/pyside6-6.9.0-py313h5f61773_0.conda#f51f25ec8fcbf777f8b186bb5deeed40 +https://conda.anaconda.org/conda-forge/noarch/array-api-strict-2.3.1-pyhd8ed1ab_0.conda#11107d0aeb8c590a34fee0894909816b +https://conda.anaconda.org/conda-forge/linux-64/blas-devel-3.9.0-31_hcf00494_mkl.conda#368c93bde87a67d24a74de15bf4c49fd +https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.3.2-py313h33d0bda_0.conda#5dc81fffe102f63045225007a33d6199 +https://conda.anaconda.org/conda-forge/linux-64/libarrow-dataset-20.0.0-hcb10f89_5_cpu.conda#d07f972af0508d56382c74846a386185 +https://conda.anaconda.org/conda-forge/linux-64/pandas-2.2.3-py313ha87cce1_3.conda#6248b529e537b1d4cb5ab3ef7f537795 +https://conda.anaconda.org/conda-forge/linux-64/polars-default-1.30.0-py39hfac2b71_0.conda#cd33cf1e631b4d766858c90e333b4832 +https://conda.anaconda.org/conda-forge/linux-64/pytorch-2.7.0-cpu_mkl_py313_hea9ba1b_100.conda#3c2ce6a304aa827f1e3cc21f7df9190d +https://conda.anaconda.org/conda-forge/linux-64/scipy-1.15.2-py313h86fcf2b_0.conda#ca68acd9febc86448eeed68d0c6c8643 +https://conda.anaconda.org/conda-forge/noarch/scipy-doctest-1.8.0-pyhe01879c_0.conda#5bc3f4bc1e027aa4ba6fdad1a84b5d3c +https://conda.anaconda.org/conda-forge/linux-64/blas-2.131-mkl.conda#9bb865b7e01104255ca54e61a58ded15 +https://conda.anaconda.org/conda-forge/linux-64/libarrow-substrait-20.0.0-h1bed206_5_cpu.conda#dbfd38071ac2e09f7761e9c8129c18c4 +https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.10.3-py313h129903b_0.conda#4f8816d006b1c155ec416bcf7ff6cee2 +https://conda.anaconda.org/conda-forge/linux-64/polars-1.30.0-default_h1443d73_0.conda#19698b29e8544d2dd615699826037039 +https://conda.anaconda.org/conda-forge/linux-64/pyamg-5.2.1-py313hf0ab243_1.conda#4c769bf3858f424cb2ecf952175ec600 +https://conda.anaconda.org/conda-forge/linux-64/pytorch-cpu-2.7.0-cpu_mkl_hc60beec_100.conda#20b3051f55ad823a27818dfa46a41c8f +https://conda.anaconda.org/conda-forge/linux-64/matplotlib-3.10.3-py313h78bf25f_0.conda#cc9324e614a297fdf23439d887d3513d +https://conda.anaconda.org/conda-forge/linux-64/pyarrow-20.0.0-py313h78bf25f_0.conda#6b8d388845ce750fe2ad8436669182f3 diff --git a/build_tools/azure/pylatest_conda_forge_mkl_linux-64_environment.yml b/build_tools/azure/pylatest_conda_forge_mkl_linux-64_environment.yml new file mode 100644 index 0000000000000..e804bf1ce8e31 --- /dev/null +++ b/build_tools/azure/pylatest_conda_forge_mkl_linux-64_environment.yml @@ -0,0 +1,31 @@ +# DO NOT EDIT: this file is generated from the specification found in the +# following script to centralize the configuration for CI builds: +# build_tools/update_environments_and_lock_files.py +channels: + - conda-forge +dependencies: + - python + - numpy + - blas[build=mkl] + - scipy + - cython + - joblib + - threadpoolctl + - matplotlib + - pandas + - pyamg + - pytest + - pytest-xdist + - pillow + - pip + - ninja + - meson-python + - pytest-cov + - coverage + - ccache + - pytorch + - pytorch-cpu + - polars + - pyarrow + - array-api-strict + - scipy-doctest diff --git a/build_tools/azure/pylatest_conda_forge_mkl_osx-64_conda.lock b/build_tools/azure/pylatest_conda_forge_mkl_osx-64_conda.lock new file mode 100644 index 0000000000000..edb27c0d3475d --- /dev/null +++ b/build_tools/azure/pylatest_conda_forge_mkl_osx-64_conda.lock @@ -0,0 +1,133 @@ +# Generated by conda-lock. +# platform: osx-64 +# input_hash: cee22335ff0a429180f2d8eeb31943f2646e3e653f1197f57ba6e39fc9659b05 +@EXPLICIT +https://conda.anaconda.org/conda-forge/noarch/libgfortran-devel_osx-64-13.3.0-h297be85_105.conda#c4967f8e797d0ffef3c5650fcdc2cdb5 +https://conda.anaconda.org/conda-forge/osx-64/mkl-include-2023.2.0-h6bab518_50500.conda#835abb8ded5e26f23ea6996259c7972e +https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-7_cp313.conda#e84b44e6300f1703cb25d29120c5b1d8 +https://conda.anaconda.org/conda-forge/osx-64/tbb-2021.10.0-h1c7c39f_2.conda#73434bcf87082942e938352afae9b0fa +https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda#4222072737ccff51314b5ece9c7d6f5a +https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-hfdf4475_7.conda#7ed4301d437b59045be7e051a0308211 +https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.4.26-hbd8a1cb_0.conda#95db94f75ba080a22eb623590993167b +https://conda.anaconda.org/conda-forge/osx-64/icu-75.1-h120a0e1_0.conda#d68d48a3060eb5abdc1cdc8e2a3a5966 +https://conda.anaconda.org/conda-forge/osx-64/libbrotlicommon-1.1.0-h00291cd_2.conda#58f2c4bdd56c46cc7451596e4ae68e0b +https://conda.anaconda.org/conda-forge/osx-64/libcxx-20.1.6-hf95d169_0.conda#460934df319a215557816480e9ea78cf +https://conda.anaconda.org/conda-forge/osx-64/libdeflate-1.24-hcc1b750_0.conda#f0a46c359722a3e84deb05cd4072d153 +https://conda.anaconda.org/conda-forge/osx-64/libexpat-2.7.0-h240833e_0.conda#026d0a1056ba2a3dbbea6d4b08188676 +https://conda.anaconda.org/conda-forge/osx-64/libffi-3.4.6-h281671d_1.conda#4ca9ea59839a9ca8df84170fab4ceb41 +https://conda.anaconda.org/conda-forge/osx-64/libiconv-1.18-h4b5e92a_1.conda#6283140d7b2b55b6b095af939b71b13f +https://conda.anaconda.org/conda-forge/osx-64/libjpeg-turbo-3.1.0-h6e16a3a_0.conda#87537967e6de2f885a9fcebd42b7cb10 +https://conda.anaconda.org/conda-forge/osx-64/liblzma-5.8.1-hd471939_1.conda#f87e8821e0e38a4140a7ed4f52530053 +https://conda.anaconda.org/conda-forge/osx-64/libmpdec-4.0.0-h6e16a3a_0.conda#18b81186a6adb43f000ad19ed7b70381 +https://conda.anaconda.org/conda-forge/osx-64/libwebp-base-1.5.0-h6cf52b4_0.conda#5e0cefc99a231ac46ba21e27ae44689f +https://conda.anaconda.org/conda-forge/osx-64/libzlib-1.3.1-hd23fc13_2.conda#003a54a4e32b02f7355b50a837e699da +https://conda.anaconda.org/conda-forge/osx-64/llvm-openmp-20.1.6-ha54dae1_0.conda#c55751d61e1f8be539e0e4beffad3e5a +https://conda.anaconda.org/conda-forge/osx-64/ncurses-6.5-h0622a9a_3.conda#ced34dd9929f491ca6dab6a2927aff25 +https://conda.anaconda.org/conda-forge/osx-64/pthread-stubs-0.4-h00291cd_1002.conda#8bcf980d2c6b17094961198284b8e862 +https://conda.anaconda.org/conda-forge/osx-64/xorg-libxau-1.0.12-h6e16a3a_0.conda#4cf40e60b444d56512a64f39d12c20bd +https://conda.anaconda.org/conda-forge/osx-64/xorg-libxdmcp-1.1.5-h00291cd_0.conda#9f438e1b6f4e73fd9e6d78bfe7c36743 +https://conda.anaconda.org/conda-forge/osx-64/gmp-6.3.0-hf036a51_2.conda#427101d13f19c4974552a4e5b072eef1 +https://conda.anaconda.org/conda-forge/osx-64/isl-0.26-imath32_h2e86a7b_101.conda#d06222822a9144918333346f145b68c6 +https://conda.anaconda.org/conda-forge/osx-64/lerc-4.0.0-hcca01a6_1.conda#21f765ced1a0ef4070df53cb425e1967 +https://conda.anaconda.org/conda-forge/osx-64/libbrotlidec-1.1.0-h00291cd_2.conda#34709a1f5df44e054c4a12ab536c5459 +https://conda.anaconda.org/conda-forge/osx-64/libbrotlienc-1.1.0-h00291cd_2.conda#691f0dcb36f1ae67f5c489f20ae987ea +https://conda.anaconda.org/conda-forge/osx-64/libcxx-devel-18.1.8-h7c275be_8.conda#a9513c41f070a9e2d5c370ba5d6c0c00 +https://conda.anaconda.org/conda-forge/osx-64/libgfortran5-14.2.0-h58528f3_105.conda#94560312ff3c78225bed62ab59854c31 +https://conda.anaconda.org/conda-forge/osx-64/libpng-1.6.47-h3c4a55f_0.conda#8461ab86d2cdb76d6e971aab225be73f +https://conda.anaconda.org/conda-forge/osx-64/libsqlite-3.50.0-hdb6dae5_0.conda#caf16742f7e16475603cd9981ef36195 +https://conda.anaconda.org/conda-forge/osx-64/libxcb-1.17.0-hf1f96e2_0.conda#bbeca862892e2898bdb45792a61c4afc +https://conda.anaconda.org/conda-forge/osx-64/libxml2-2.14.3-h8c082e5_0.conda#f886f309637a6ff2ff858b38b7395aa1 +https://conda.anaconda.org/conda-forge/osx-64/mkl-2023.2.0-h54c2260_50500.conda#0a342ccdc79e4fcd359245ac51941e7b +https://conda.anaconda.org/conda-forge/osx-64/ninja-1.12.1-hd6aca1a_1.conda#1cf196736676270fa876001901e4e1db +https://conda.anaconda.org/conda-forge/osx-64/openssl-3.5.0-hc426f3f_1.conda#919faa07b9647beb99a0e7404596a465 +https://conda.anaconda.org/conda-forge/osx-64/qhull-2020.2-h3c5361c_5.conda#dd1ea9ff27c93db7c01a7b7656bd4ad4 +https://conda.anaconda.org/conda-forge/osx-64/readline-8.2-h7cca4af_2.conda#342570f8e02f2f022147a7f841475784 +https://conda.anaconda.org/conda-forge/osx-64/tapi-1300.6.5-h390ca13_0.conda#c6ee25eb54accb3f1c8fc39203acfaf1 +https://conda.anaconda.org/conda-forge/osx-64/tk-8.6.13-hf689a15_2.conda#9864891a6946c2fe037c02fca7392ab4 +https://conda.anaconda.org/conda-forge/osx-64/zlib-1.3.1-hd23fc13_2.conda#c989e0295dcbdc08106fe5d9e935f0b9 +https://conda.anaconda.org/conda-forge/osx-64/zstd-1.5.7-h8210216_2.conda#cd60a4a5a8d6a476b30d8aa4bb49251a +https://conda.anaconda.org/conda-forge/osx-64/brotli-bin-1.1.0-h00291cd_2.conda#049933ecbf552479a12c7917f0a4ce59 +https://conda.anaconda.org/conda-forge/osx-64/libblas-3.9.0-20_osx64_mkl.conda#160fdc97a51d66d51dc782fb67d35205 +https://conda.anaconda.org/conda-forge/osx-64/libfreetype6-2.13.3-h40dfd5c_1.conda#c76e6f421a0e95c282142f820835e186 +https://conda.anaconda.org/conda-forge/osx-64/libgfortran-14.2.0-hef36b68_105.conda#6b27baf030f5d6603713c7e72d3f6b9a +https://conda.anaconda.org/conda-forge/osx-64/libllvm18-18.1.8-default_h3571c67_5.conda#01dd8559b569ad39b64fef0a61ded1e9 +https://conda.anaconda.org/conda-forge/osx-64/libtiff-4.7.0-h1167cee_5.conda#fc84af14a09e779f1d37ab1d16d5c4e2 +https://conda.anaconda.org/conda-forge/osx-64/mkl-devel-2023.2.0-h694c41f_50500.conda#1b4d0235ef253a1e19459351badf4f9f +https://conda.anaconda.org/conda-forge/osx-64/mpfr-4.2.1-haed47dc_3.conda#d511e58aaaabfc23136880d9956fa7a6 +https://conda.anaconda.org/conda-forge/osx-64/python-3.13.3-h534c281_101_cp313.conda#ebcc7c42561d8d8b01477020b63218c0 +https://conda.anaconda.org/conda-forge/osx-64/sigtool-0.1.3-h88f4db0_0.tar.bz2#fbfb84b9de9a6939cb165c02c69b1865 +https://conda.anaconda.org/conda-forge/osx-64/brotli-1.1.0-h00291cd_2.conda#2db0c38a7f2321c5bdaf32b181e832c7 +https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda#962b9857ee8e7018c22f2776ffa0b2d7 +https://conda.anaconda.org/conda-forge/noarch/cycler-0.12.1-pyhd8ed1ab_1.conda#44600c4667a319d67dbe0681fc0bc833 +https://conda.anaconda.org/conda-forge/osx-64/cython-3.1.1-py313h9efc8c2_1.conda#b94bca8fec5fbaa69375656928e05c1d +https://conda.anaconda.org/conda-forge/noarch/execnet-2.1.1-pyhd8ed1ab_1.conda#a71efeae2c160f6789900ba2631a2c90 +https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_1.conda#6837f3eff7dcea42ecd714ce1ac2b108 +https://conda.anaconda.org/conda-forge/osx-64/kiwisolver-1.4.7-py313h0c4e38b_0.conda#c37fceab459e104e77bb5456e219fc37 +https://conda.anaconda.org/conda-forge/osx-64/lcms2-2.17-h72f5680_0.conda#bf210d0c63f2afb9e414a858b79f0eaa +https://conda.anaconda.org/conda-forge/osx-64/ld64_osx-64-951.9-h33512f0_6.conda#6cd120f5c9dae65b858e1fad2b7959a0 +https://conda.anaconda.org/conda-forge/osx-64/libcblas-3.9.0-20_osx64_mkl.conda#51089a4865eb4aec2bc5c7468bd07f9f +https://conda.anaconda.org/conda-forge/osx-64/libclang-cpp18.1-18.1.8-default_h3571c67_10.conda#bf6753267e6f848f369c5bc2373dddd6 +https://conda.anaconda.org/conda-forge/osx-64/libfreetype-2.13.3-h694c41f_1.conda#07c8d3fbbe907f32014b121834b36dd5 +https://conda.anaconda.org/conda-forge/osx-64/libhiredis-1.0.2-h2beb688_0.tar.bz2#524282b2c46c9dedf051b3bc2ae05494 +https://conda.anaconda.org/conda-forge/osx-64/liblapack-3.9.0-20_osx64_mkl.conda#58f08e12ad487fac4a08f90ff0b87aec +https://conda.anaconda.org/conda-forge/osx-64/llvm-tools-18-18.1.8-default_h3571c67_5.conda#4391981e855468ced32ca1940b3d7613 +https://conda.anaconda.org/conda-forge/noarch/meson-1.8.1-pyhe01879c_0.conda#f3cccd9a6ce5331ae33f69ade5529162 +https://conda.anaconda.org/conda-forge/osx-64/mpc-1.3.1-h9d8efa1_1.conda#0520855aaae268ea413d6bc913f1384c +https://conda.anaconda.org/conda-forge/noarch/munkres-1.1.4-pyh9f0ad1d_0.tar.bz2#2ba8498c1018c1e9c61eb99b973dfe19 +https://conda.anaconda.org/conda-forge/osx-64/openjpeg-2.5.3-h7fd6d84_0.conda#025c711177fc3309228ca1a32374458d +https://conda.anaconda.org/conda-forge/noarch/packaging-25.0-pyh29332c3_1.conda#58335b26c38bf4a20f399384c33cbcf9 +https://conda.anaconda.org/conda-forge/noarch/pip-25.1.1-pyh145f28c_0.conda#01384ff1639c6330a0924791413b8714 +https://conda.anaconda.org/conda-forge/noarch/pluggy-1.6.0-pyhd8ed1ab_0.conda#7da7ccd349dbf6487a7778579d2bb971 +https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.2.3-pyhd8ed1ab_1.conda#513d3c262ee49b54a8fec85c5bc99764 +https://conda.anaconda.org/conda-forge/noarch/python-tzdata-2025.2-pyhd8ed1ab_0.conda#88476ae6ebd24f39261e0854ac244f33 +https://conda.anaconda.org/conda-forge/noarch/pytz-2025.2-pyhd8ed1ab_0.conda#bc8e3267d44011051f2eb14d22fb0960 +https://conda.anaconda.org/conda-forge/noarch/setuptools-80.9.0-pyhff2d567_0.conda#4de79c071274a53dcaf2a8c749d1499e +https://conda.anaconda.org/conda-forge/noarch/six-1.17.0-pyhd8ed1ab_0.conda#a451d576819089b0d672f18768be0f65 +https://conda.anaconda.org/conda-forge/noarch/threadpoolctl-3.6.0-pyhecae5ae_0.conda#9d64911b31d57ca443e9f1e36b04385f +https://conda.anaconda.org/conda-forge/noarch/toml-0.10.2-pyhd8ed1ab_1.conda#b0dd904de08b7db706167240bf37b164 +https://conda.anaconda.org/conda-forge/noarch/tomli-2.2.1-pyhd8ed1ab_1.conda#ac944244f1fed2eb49bae07193ae8215 +https://conda.anaconda.org/conda-forge/osx-64/tornado-6.5.1-py313h63b0ddb_0.conda#7554d07cbe64f41c73a403e99bccf3c6 +https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.13.2-pyh29332c3_0.conda#83fc6ae00127671e301c9f44254c31b8 +https://conda.anaconda.org/conda-forge/osx-64/ccache-4.11.3-h33566b8_0.conda#b65cad834bd6c1f660c101cca09430bf +https://conda.anaconda.org/conda-forge/osx-64/clang-18-18.1.8-default_h3571c67_10.conda#62e1cd0882dad47d6a6878ad037f7b9d +https://conda.anaconda.org/conda-forge/osx-64/coverage-7.8.2-py313h717bdf5_0.conda#73eb83ea3d00f06bf78e242cca5e8e44 +https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.0-pyhd8ed1ab_0.conda#72e42d28960d875c7654614f8b50939a +https://conda.anaconda.org/conda-forge/osx-64/fonttools-4.58.1-py313h717bdf5_0.conda#b08a911c3cc2cf9ff4f48f4e06bbc2dd +https://conda.anaconda.org/conda-forge/osx-64/freetype-2.13.3-h694c41f_1.conda#126dba1baf5030cb6f34533718924577 +https://conda.anaconda.org/conda-forge/osx-64/gfortran_impl_osx-64-13.3.0-hbf5bf67_105.conda#f56a107c8d1253346d01785ecece7977 +https://conda.anaconda.org/conda-forge/noarch/joblib-1.5.1-pyhd8ed1ab_0.conda#fb1c14694de51a476ce8636d92b6f42c +https://conda.anaconda.org/conda-forge/osx-64/ld64-951.9-h4e51db5_6.conda#45bf526d53b1bc95bc0b932a91a41576 +https://conda.anaconda.org/conda-forge/osx-64/liblapacke-3.9.0-20_osx64_mkl.conda#124ae8e384268a8da66f1d64114a1eda +https://conda.anaconda.org/conda-forge/osx-64/llvm-tools-18.1.8-default_h3571c67_5.conda#cc07ff74d2547da1f1452c42b67bafd6 +https://conda.anaconda.org/conda-forge/osx-64/numpy-2.2.6-py313hc518a0f_0.conda#7b80c7ace05b1b9d7ec6f55130776988 +https://conda.anaconda.org/conda-forge/osx-64/pillow-11.2.1-py313h0c4f865_0.conda#b4647eda8779d0e5d25cc8c9b124b303 +https://conda.anaconda.org/conda-forge/noarch/pyproject-metadata-0.9.1-pyhd8ed1ab_0.conda#22ae7c6ea81e0c8661ef32168dda929b +https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhff2d567_1.conda#5ba79d7c71f03c678c8ead841f347d6e +https://conda.anaconda.org/conda-forge/osx-64/blas-devel-3.9.0-20_osx64_mkl.conda#cc3260179093918b801e373c6e888e02 +https://conda.anaconda.org/conda-forge/osx-64/cctools_osx-64-1010.6-hd19c6af_6.conda#4694e9e497454a8ce5b9fb61e50d9c5d +https://conda.anaconda.org/conda-forge/osx-64/clang-18.1.8-default_h576c50e_10.conda#350a10c62423982b0c80a043b9921c00 +https://conda.anaconda.org/conda-forge/osx-64/contourpy-1.3.2-py313ha0b1807_0.conda#2c2d1f840df1c512b34e0537ef928169 +https://conda.anaconda.org/conda-forge/noarch/meson-python-0.18.0-pyh70fd9c4_0.conda#576c04b9d9f8e45285fb4d9452c26133 +https://conda.anaconda.org/conda-forge/osx-64/pandas-2.2.3-py313h2e7108f_3.conda#5c37fc7549913fc4895d7d2e097091ed +https://conda.anaconda.org/conda-forge/noarch/pytest-8.3.5-pyhd8ed1ab_0.conda#c3c9316209dec74a705a36797970c6be +https://conda.anaconda.org/conda-forge/osx-64/scipy-1.15.2-py313h7e69c36_0.conda#53c23f87aedf2d139d54c88894c8a07f +https://conda.anaconda.org/conda-forge/osx-64/blas-2.120-mkl.conda#b041a7677a412f3d925d8208936cb1e2 +https://conda.anaconda.org/conda-forge/osx-64/cctools-1010.6-ha66f10e_6.conda#a126dcde2752751ac781b67238f7fac4 +https://conda.anaconda.org/conda-forge/osx-64/clangxx-18.1.8-default_heb2e8d1_10.conda#c39251c90faf5ba495d9f9ef88d7563e +https://conda.anaconda.org/conda-forge/osx-64/matplotlib-base-3.10.3-py313he981572_0.conda#91c22969c0974f2f23470d517774d457 +https://conda.anaconda.org/conda-forge/osx-64/pyamg-5.2.1-py313h0322a6a_1.conda#4bda5182eeaef3d2017a2ec625802e1a +https://conda.anaconda.org/conda-forge/noarch/pytest-cov-6.1.1-pyhd8ed1ab_0.conda#1e35d8f975bc0e984a19819aa91c440a +https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.7.0-pyhd8ed1ab_0.conda#15353a2a0ea6dfefaa52fc5ab5b98f41 +https://conda.anaconda.org/conda-forge/noarch/compiler-rt_osx-64-18.1.8-hf2b8a54_1.conda#76f906e6bdc58976c5593f650290ae20 +https://conda.anaconda.org/conda-forge/osx-64/matplotlib-3.10.3-py313habf4b1d_0.conda#c1043254f405998ece984e5f66a10943 +https://conda.anaconda.org/conda-forge/osx-64/compiler-rt-18.1.8-h1020d70_1.conda#bc1714a1e73be18e411cff30dc1fe011 +https://conda.anaconda.org/conda-forge/osx-64/clang_impl_osx-64-18.1.8-h6a44ed1_25.conda#bfc995f8ab9e8c22ebf365844da3383d +https://conda.anaconda.org/conda-forge/osx-64/clang_osx-64-18.1.8-h7e5c614_25.conda#1fea06d9ced6b87fe63384443bc2efaf +https://conda.anaconda.org/conda-forge/osx-64/c-compiler-1.9.0-h09a7c41_0.conda#ab45badcb5d035d3bddfdbdd96e00967 +https://conda.anaconda.org/conda-forge/osx-64/clangxx_impl_osx-64-18.1.8-h4b7810f_25.conda#c03c94381d9ffbec45c98b800e7d3e86 +https://conda.anaconda.org/conda-forge/osx-64/gfortran_osx-64-13.3.0-h3223c34_1.conda#a6eeb1519091ac3239b88ee3914d6cb6 +https://conda.anaconda.org/conda-forge/osx-64/clangxx_osx-64-18.1.8-h7e5c614_25.conda#2e5c84e93a3519d77a0d8d9b3ea664fd +https://conda.anaconda.org/conda-forge/osx-64/gfortran-13.3.0-hcc3c99d_1.conda#e1177b9b139c6cf43250427819f2f07b +https://conda.anaconda.org/conda-forge/osx-64/cxx-compiler-1.9.0-h20888b2_0.conda#cd17d9bf9780b0db4ed31fb9958b167f +https://conda.anaconda.org/conda-forge/osx-64/fortran-compiler-1.9.0-h02557f8_0.conda#2cf645572d7ae534926093b6e9f3bdff +https://conda.anaconda.org/conda-forge/osx-64/compilers-1.9.0-h694c41f_0.conda#b84884262dcd1c2f56a9e1961fdd3326 diff --git a/build_tools/azure/pylatest_conda_forge_mkl_osx-64_environment.yml b/build_tools/azure/pylatest_conda_forge_mkl_osx-64_environment.yml new file mode 100644 index 0000000000000..ad177e4ed391b --- /dev/null +++ b/build_tools/azure/pylatest_conda_forge_mkl_osx-64_environment.yml @@ -0,0 +1,27 @@ +# DO NOT EDIT: this file is generated from the specification found in the +# following script to centralize the configuration for CI builds: +# build_tools/update_environments_and_lock_files.py +channels: + - conda-forge +dependencies: + - python + - numpy + - blas[build=mkl] + - scipy + - cython + - joblib + - threadpoolctl + - matplotlib + - pandas + - pyamg + - pytest + - pytest-xdist + - pillow + - pip + - ninja + - meson-python + - pytest-cov + - coverage + - ccache + - compilers + - llvm-openmp diff --git a/build_tools/azure/pylatest_conda_mkl_no_openmp_environment.yml b/build_tools/azure/pylatest_conda_mkl_no_openmp_environment.yml new file mode 100644 index 0000000000000..0c2eec344c26b --- /dev/null +++ b/build_tools/azure/pylatest_conda_mkl_no_openmp_environment.yml @@ -0,0 +1,28 @@ +# DO NOT EDIT: this file is generated from the specification found in the +# following script to centralize the configuration for CI builds: +# build_tools/update_environments_and_lock_files.py +channels: + - defaults +dependencies: + - python + - numpy + - blas[build=mkl] + - scipy<1.12 + - joblib + - matplotlib + - pandas + - pyamg + - pytest + - pytest-xdist + - pillow + - pip + - ninja + - pytest-cov + - coverage + - ccache + - pip + - pip: + - cython + - threadpoolctl + - meson-python + - meson diff --git a/build_tools/azure/pylatest_conda_mkl_no_openmp_osx-64_conda.lock b/build_tools/azure/pylatest_conda_mkl_no_openmp_osx-64_conda.lock new file mode 100644 index 0000000000000..7be311177e65f --- /dev/null +++ b/build_tools/azure/pylatest_conda_mkl_no_openmp_osx-64_conda.lock @@ -0,0 +1,82 @@ +# Generated by conda-lock. +# platform: osx-64 +# input_hash: cc639ea0beeaceb46e2ad729ba559d5d5e746b8f6ff522bc718109af6265069c +@EXPLICIT +https://repo.anaconda.com/pkgs/main/osx-64/blas-1.0-mkl.conda#cb2c87e85ac8e0ceae776d26d4214c8a +https://repo.anaconda.com/pkgs/main/osx-64/bzip2-1.0.8-h6c40b1e_6.conda#96224786021d0765ce05818fa3c59bdb +https://repo.anaconda.com/pkgs/main/osx-64/ca-certificates-2025.2.25-hecd8cb5_0.conda#12ab77db61795036e15a5b14929ad4a1 +https://repo.anaconda.com/pkgs/main/osx-64/jpeg-9e-h46256e1_3.conda#b1d9769eac428e11f5f922531a1da2e0 +https://repo.anaconda.com/pkgs/main/osx-64/libcxx-14.0.6-h9765a3e_0.conda#387757bb354ae9042370452cd0fb5627 +https://repo.anaconda.com/pkgs/main/osx-64/libdeflate-1.22-h46256e1_0.conda#7612fb79e5e76fcd16655c7d026f4a66 +https://repo.anaconda.com/pkgs/main/osx-64/libffi-3.4.4-hecd8cb5_1.conda#eb7f09ada4d95f1a26f483f1009d9286 +https://repo.anaconda.com/pkgs/main/osx-64/libwebp-base-1.3.2-h46256e1_1.conda#399c11b50e6e7a6969aca9a84ea416b7 +https://repo.anaconda.com/pkgs/main/osx-64/llvm-openmp-14.0.6-h0dcd299_0.conda#b5804d32b87dc61ca94561ade33d5f2d +https://repo.anaconda.com/pkgs/main/osx-64/ncurses-6.4-hcec6c5f_0.conda#0214d1ee980e217fabc695f1e40662aa +https://repo.anaconda.com/pkgs/main/noarch/tzdata-2025b-h04d1e81_0.conda#1d027393db3427ab22a02aa44a56f143 +https://repo.anaconda.com/pkgs/main/osx-64/xz-5.6.4-h46256e1_1.conda#ce989a528575ad332a650bb7c7f7e5d5 +https://repo.anaconda.com/pkgs/main/osx-64/zlib-1.2.13-h4b97444_1.conda#38e35f7c817fac0973034bfce6706ec2 +https://repo.anaconda.com/pkgs/main/osx-64/ccache-3.7.9-hf120daa_0.conda#a01515a32e721c51d631283f991bc8ea +https://repo.anaconda.com/pkgs/main/osx-64/expat-2.7.1-h6d0c2b6_0.conda#6cdc93776b7551083854e7f106a62720 +https://repo.anaconda.com/pkgs/main/osx-64/intel-openmp-2023.1.0-ha357a0b_43548.conda#ba8a89ffe593eb88e4c01334753c40c3 +https://repo.anaconda.com/pkgs/main/osx-64/lerc-4.0.0-h6d0c2b6_0.conda#824f87854c58df1525557c8639ce7f93 +https://repo.anaconda.com/pkgs/main/osx-64/libgfortran5-11.3.0-h9dfd629_28.conda#1fa1a27ee100b1918c3021dbfa3895a3 +https://repo.anaconda.com/pkgs/main/osx-64/libpng-1.6.39-h6c40b1e_0.conda#a3c824835f53ad27aeb86d2b55e47804 +https://repo.anaconda.com/pkgs/main/osx-64/lz4-c-1.9.4-hcec6c5f_1.conda#aee0efbb45220e1985533dbff48551f8 +https://repo.anaconda.com/pkgs/main/osx-64/ninja-base-1.12.1-h1962661_0.conda#9c0a94a811e88f182519d9309cf5f634 +https://repo.anaconda.com/pkgs/main/osx-64/openssl-3.0.16-h184c1cd_0.conda#8e3c130ef85c3260d535153b4d0fd63a +https://repo.anaconda.com/pkgs/main/osx-64/readline-8.2-hca72f7f_0.conda#971667436260e523f6f7355fdfa238bf +https://repo.anaconda.com/pkgs/main/osx-64/tbb-2021.8.0-ha357a0b_0.conda#fb48530a3eea681c11dafb95b3387c0f +https://repo.anaconda.com/pkgs/main/osx-64/tk-8.6.14-h4d00af3_0.conda#a2c03940c2ae54614301ec82e6a98d75 +https://repo.anaconda.com/pkgs/main/osx-64/freetype-2.13.3-h02243ff_0.conda#acf5e48106235eb200eecb79119c7ffc +https://repo.anaconda.com/pkgs/main/osx-64/libgfortran-5.0.0-11_3_0_hecd8cb5_28.conda#2eb13b680803f1064e53873ae0aaafb3 +https://repo.anaconda.com/pkgs/main/osx-64/mkl-2023.1.0-h8e150cf_43560.conda#85d0f3431dd5c6ae44f8725fdd3d3e59 +https://repo.anaconda.com/pkgs/main/osx-64/sqlite-3.45.3-h6c40b1e_0.conda#2edf909b937b3aad48322c9cb2e8f1a0 +https://repo.anaconda.com/pkgs/main/osx-64/zstd-1.5.6-h138b38a_0.conda#f4d15d7d0054d39e6a24fe8d7d1e37c5 +https://repo.anaconda.com/pkgs/main/osx-64/libtiff-4.7.0-h2dfa3ea_0.conda#82a118ce0139e2bf6f7a99c4cfbd4749 +https://repo.anaconda.com/pkgs/main/osx-64/python-3.12.9-hcd54a6c_0.conda#1bf9af06f3e476df1f72e8674a9224df +https://repo.anaconda.com/pkgs/main/osx-64/brotli-python-1.0.9-py312h6d0c2b6_9.conda#425936421fe402074163ac3ffe33a060 +https://repo.anaconda.com/pkgs/main/osx-64/coverage-7.6.9-py312h46256e1_0.conda#f8c1547bbf522a600ee795901240a7b0 +https://repo.anaconda.com/pkgs/main/noarch/cycler-0.11.0-pyhd3eb1b0_0.conda#f5e365d2cdb66d547eb8c3ab93843aab +https://repo.anaconda.com/pkgs/main/noarch/execnet-2.1.1-pyhd3eb1b0_0.conda#b3cb797432ee4657d5907b91a5dc65ad +https://repo.anaconda.com/pkgs/main/noarch/iniconfig-1.1.1-pyhd3eb1b0_0.tar.bz2#e40edff2c5708f342cef43c7f280c507 +https://repo.anaconda.com/pkgs/main/osx-64/joblib-1.4.2-py312hecd8cb5_0.conda#8ab03dfa447b4e0bfa0bd3d25930f3b6 +https://repo.anaconda.com/pkgs/main/osx-64/kiwisolver-1.4.8-py312h6d0c2b6_0.conda#060d4498fcc967a640829cb7e55c95f2 +https://repo.anaconda.com/pkgs/main/osx-64/lcms2-2.16-h31d93a5_1.conda#42450b66e91caf9ab0672a599e2a7bd0 +https://repo.anaconda.com/pkgs/main/osx-64/mkl-service-2.4.0-py312h46256e1_2.conda#04297cb766cabf38613ed6eb4eec85c3 +https://repo.anaconda.com/pkgs/main/osx-64/ninja-1.12.1-hecd8cb5_0.conda#ee3b660616ef0fbcbd0096a67c11c94b +https://repo.anaconda.com/pkgs/main/osx-64/openjpeg-2.5.2-h2d09ccc_1.conda#0f2e221843154b436b5982c695df627b +https://repo.anaconda.com/pkgs/main/osx-64/packaging-24.2-py312hecd8cb5_0.conda#76512e47c9c37443444ef0624769f620 +https://repo.anaconda.com/pkgs/main/osx-64/pluggy-1.5.0-py312hecd8cb5_0.conda#ca381e438f1dbd7986ac0fa0da70c9d8 +https://repo.anaconda.com/pkgs/main/osx-64/pyparsing-3.2.0-py312hecd8cb5_0.conda#e4086daaaed13f68cc8d5b9da7db73cc +https://repo.anaconda.com/pkgs/main/noarch/python-tzdata-2025.2-pyhd3eb1b0_0.conda#5ac858f05dbf9d3cdb04d53516901247 +https://repo.anaconda.com/pkgs/main/osx-64/pytz-2024.1-py312hecd8cb5_0.conda#2b28ec0e0d07f5c0c701f75200b1e8b6 +https://repo.anaconda.com/pkgs/main/osx-64/setuptools-78.1.1-py312hecd8cb5_0.conda#76b66b96a1564cb76011408c1eb8df3e +https://repo.anaconda.com/pkgs/main/osx-64/six-1.17.0-py312hecd8cb5_0.conda#aadd782bc06426887ae0835eedd98ceb +https://repo.anaconda.com/pkgs/main/noarch/toml-0.10.2-pyhd3eb1b0_0.conda#cda05f5f6d8509529d1a2743288d197a +https://repo.anaconda.com/pkgs/main/osx-64/tornado-6.5-py312h46256e1_0.conda#7e82973ed53e71854971e7b3922fad24 +https://repo.anaconda.com/pkgs/main/osx-64/unicodedata2-15.1.0-py312h46256e1_1.conda#4a7fd1dec7277c8ab71aa11aa08df86b +https://repo.anaconda.com/pkgs/main/osx-64/wheel-0.45.1-py312hecd8cb5_0.conda#fafb8687668467d8624d2ddd0909bce9 +https://repo.anaconda.com/pkgs/main/osx-64/fonttools-4.55.3-py312h46256e1_0.conda#f7680dd6b8b1c2f8aab17cf6630c6deb +https://repo.anaconda.com/pkgs/main/osx-64/numpy-base-1.26.4-py312h6f81483_0.conda#87f73efbf26ab2e2ea7c32481a71bd47 +https://repo.anaconda.com/pkgs/main/osx-64/pillow-11.1.0-py312h935ef2f_1.conda#c2f7a3f027cc93a3626d50b765b75dc5 +https://repo.anaconda.com/pkgs/main/noarch/pip-25.1-pyhc872135_2.conda#2778327d2a700153fefe0e69438b18e1 +https://repo.anaconda.com/pkgs/main/osx-64/pytest-8.3.4-py312hecd8cb5_0.conda#b15ee02022967632dfa1672669228bee +https://repo.anaconda.com/pkgs/main/osx-64/python-dateutil-2.9.0post0-py312hecd8cb5_2.conda#1047dde28f78127dd9f6121e882926dd +https://repo.anaconda.com/pkgs/main/osx-64/pytest-cov-6.0.0-py312hecd8cb5_0.conda#db697e319a4d1145363246a51eef0352 +https://repo.anaconda.com/pkgs/main/osx-64/pytest-xdist-3.6.1-py312hecd8cb5_0.conda#38df9520774ee82bf143218f1271f936 +https://repo.anaconda.com/pkgs/main/osx-64/bottleneck-1.4.2-py312ha2b695f_0.conda#7efb63b6a5b33829a3b2c7a3efcf53ce +https://repo.anaconda.com/pkgs/main/osx-64/contourpy-1.3.1-py312h1962661_0.conda#41499d3a415721b0514f0cccb8288cb1 +https://repo.anaconda.com/pkgs/main/osx-64/matplotlib-3.10.0-py312hecd8cb5_0.conda#2977e81a7775be7963daf49df981b6e0 +https://repo.anaconda.com/pkgs/main/osx-64/matplotlib-base-3.10.0-py312h919b35b_0.conda#afc11bf311f5921ca4674ebac9592cf8 +https://repo.anaconda.com/pkgs/main/osx-64/mkl_fft-1.3.8-py312h6c40b1e_0.conda#d59d01b940493f2b6a84aac922fd0c76 +https://repo.anaconda.com/pkgs/main/osx-64/mkl_random-1.2.4-py312ha357a0b_0.conda#c1ea9c8eee79a5af3399f3c31be0e9c6 +https://repo.anaconda.com/pkgs/main/osx-64/numpy-1.26.4-py312hac873b0_0.conda#3150bac1e382156f82a153229e1ebd06 +https://repo.anaconda.com/pkgs/main/osx-64/numexpr-2.8.7-py312hac873b0_0.conda#6303ba071636ef57fddf69eb6f440ec1 +https://repo.anaconda.com/pkgs/main/osx-64/scipy-1.11.4-py312h81688c2_0.conda#7d57b4c21a9261f97fa511e0940c5d93 +https://repo.anaconda.com/pkgs/main/osx-64/pandas-2.2.3-py312h6d0c2b6_0.conda#84ce5b8ec4a986d13a5df17811f556a2 +https://repo.anaconda.com/pkgs/main/osx-64/pyamg-5.2.1-py312h1962661_0.conda#58881950d4ce74c9302b56961f97a43c +# pip cython @ https://files.pythonhosted.org/packages/78/06/83ff82381319ff68ae46f9dd3024b1d5101997e81a8e955811525b6f934b/cython-3.1.1-cp312-cp312-macosx_10_13_x86_64.whl#sha256=9d7dc0e4d0cd491fac679a61e9ede348c64ca449f99a284f9a01851aa1dbc7f6 +# pip meson @ https://files.pythonhosted.org/packages/46/77/726b14be352aa6911e206ca7c4d95c5be49660604dfee0bfed0fc75823e5/meson-1.8.1-py3-none-any.whl#sha256=374bbf71247e629475fc10b0bd2ef66fc418c2d8f4890572f74de0f97d0d42da +# pip threadpoolctl @ https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl#sha256=43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb +# pip pyproject-metadata @ https://files.pythonhosted.org/packages/7e/b1/8e63033b259e0a4e40dd1ec4a9fee17718016845048b43a36ec67d62e6fe/pyproject_metadata-0.9.1-py3-none-any.whl#sha256=ee5efde548c3ed9b75a354fc319d5afd25e9585fa918a34f62f904cc731973ad +# pip meson-python @ https://files.pythonhosted.org/packages/28/58/66db620a8a7ccb32633de9f403fe49f1b63c68ca94e5c340ec5cceeb9821/meson_python-0.18.0-py3-none-any.whl#sha256=3b0fe051551cc238f5febb873247c0949cd60ded556efa130aa57021804868e2 diff --git a/build_tools/azure/pylatest_free_threaded_environment.yml b/build_tools/azure/pylatest_free_threaded_environment.yml new file mode 100644 index 0000000000000..8980bfce4adaf --- /dev/null +++ b/build_tools/azure/pylatest_free_threaded_environment.yml @@ -0,0 +1,18 @@ +# DO NOT EDIT: this file is generated from the specification found in the +# following script to centralize the configuration for CI builds: +# build_tools/update_environments_and_lock_files.py +channels: + - conda-forge +dependencies: + - python-freethreading + - numpy + - scipy + - cython + - joblib + - threadpoolctl + - pytest + - pytest-xdist + - ninja + - meson-python + - ccache + - pip diff --git a/build_tools/azure/pylatest_free_threaded_linux-64_conda.lock b/build_tools/azure/pylatest_free_threaded_linux-64_conda.lock new file mode 100644 index 0000000000000..40254398d3bb7 --- /dev/null +++ b/build_tools/azure/pylatest_free_threaded_linux-64_conda.lock @@ -0,0 +1,61 @@ +# Generated by conda-lock. +# platform: linux-64 +# input_hash: b76364b5635e8c36a0fc0777955b5664a336ba94ac96f3ade7aad842ab7e15c5 +@EXPLICIT +https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81 +https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-7_cp313t.conda#df81edcc11a1176315e8226acab83eec +https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda#4222072737ccff51314b5ece9c7d6f5a +https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.4.26-hbd8a1cb_0.conda#95db94f75ba080a22eb623590993167b +https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.43-h712a8e2_4.conda#01f8d123c96816249efd255a31ad7712 +https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.1.0-h767d61c_2.conda#fbe7d535ff9d3a168c148e07358cd5b1 +https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2#73aaf86a425cc6e73fcf236a5a46396d +https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.1.0-h767d61c_2.conda#ea8ac52380885ed41c1baa8f1d6d2b93 +https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.7.0-h5888daf_0.conda#db0bfbe7dd197b68ad5f30333bae6ce0 +https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.6-h2dba641_1.conda#ede4673863426c0883c0063d853bbd85 +https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-15.1.0-h69a702a_2.conda#ddca86c7040dd0e73b2b69bd7833d225 +https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-15.1.0-hcea5267_2.conda#01de444988ed960031dbe84cf4f9b1fc +https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.1-hb9d3cd8_1.conda#a76fd702c93cd2dfd89eff30a5fd45a8 +https://conda.anaconda.org/conda-forge/linux-64/libmpdec-4.0.0-hb9d3cd8_0.conda#c7e925f37e3b40d893459e625f6a53f1 +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-15.1.0-h8f9b012_2.conda#1cb1c67961f6dd257eae9e9691b341aa +https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda#edb0dca6bc32e4f4789199455a1dbeb8 +https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda#47e340acb35de30501a76c7c799c41d7 +https://conda.anaconda.org/conda-forge/linux-64/openssl-3.5.0-h7b32b05_1.conda#de356753cfdbffcde5bb1e86e3aa6cd0 +https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h4bc722e_7.conda#62ee74e96c5ebb0af99386de58cf9553 +https://conda.anaconda.org/conda-forge/linux-64/libgfortran-15.1.0-h69a702a_2.conda#f92e6e0a3c0c0c85561ef61aa59d555d +https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.50.0-hee588c1_0.conda#71888e92098d0f8c41b09a671ad289bc +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-15.1.0-h4852527_2.conda#9d2072af184b5caa29492bf2344597bb +https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda#40b61aab5c7ba9ff276c41cfffe6b80b +https://conda.anaconda.org/conda-forge/linux-64/ninja-1.12.1-hff21bea_1.conda#2322531904f27501ee19847b87ba7c64 +https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8c095d6_2.conda#283b96675859b20a825f8fa30f311446 +https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_hd72426e_102.conda#a0116df4f4ed05c303811a837d5b39d8 +https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb8e6e7a_2.conda#6432cb5d4ac0046c3ac0a8a0f95842f9 +https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-15.1.0-h69a702a_2.conda#a483a87b71e974bb75d1b9413d4436dd +https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.29-pthreads_h94d23a6_0.conda#0a4d0252248ef9a0f88f2ba8b8a08e12 +https://conda.anaconda.org/conda-forge/linux-64/python-3.13.3-h4724d56_1_cp313t.conda#8193603fe48ace3d8801cfb246f44491 +https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda#962b9857ee8e7018c22f2776ffa0b2d7 +https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.3-py313hd8ed1ab_1.conda#6ba9ba47b91b7758cb963d0f0eaf3422 +https://conda.anaconda.org/conda-forge/noarch/cython-3.1.1-pyh2c78169_101.conda#f085516359786c4fea51bf05227ef3d2 +https://conda.anaconda.org/conda-forge/noarch/execnet-2.1.1-pyhd8ed1ab_1.conda#a71efeae2c160f6789900ba2631a2c90 +https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_1.conda#6837f3eff7dcea42ecd714ce1ac2b108 +https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-31_h59b9bed_openblas.conda#728dbebd0f7a20337218beacffd37916 +https://conda.anaconda.org/conda-forge/linux-64/libhiredis-1.0.2-h2cc385e_0.tar.bz2#b34907d3a81a3cd8095ee83d174c074a +https://conda.anaconda.org/conda-forge/noarch/meson-1.8.1-pyhe01879c_0.conda#f3cccd9a6ce5331ae33f69ade5529162 +https://conda.anaconda.org/conda-forge/noarch/packaging-25.0-pyh29332c3_1.conda#58335b26c38bf4a20f399384c33cbcf9 +https://conda.anaconda.org/conda-forge/noarch/pip-25.1.1-pyh145f28c_0.conda#01384ff1639c6330a0924791413b8714 +https://conda.anaconda.org/conda-forge/noarch/pluggy-1.6.0-pyhd8ed1ab_0.conda#7da7ccd349dbf6487a7778579d2bb971 +https://conda.anaconda.org/conda-forge/noarch/setuptools-80.9.0-pyhff2d567_0.conda#4de79c071274a53dcaf2a8c749d1499e +https://conda.anaconda.org/conda-forge/noarch/threadpoolctl-3.6.0-pyhecae5ae_0.conda#9d64911b31d57ca443e9f1e36b04385f +https://conda.anaconda.org/conda-forge/noarch/tomli-2.2.1-pyhd8ed1ab_1.conda#ac944244f1fed2eb49bae07193ae8215 +https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.13.2-pyh29332c3_0.conda#83fc6ae00127671e301c9f44254c31b8 +https://conda.anaconda.org/conda-forge/linux-64/ccache-4.11.3-h80c52d3_0.conda#eb517c6a2b960c3ccb6f1db1005f063a +https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.0-pyhd8ed1ab_0.conda#72e42d28960d875c7654614f8b50939a +https://conda.anaconda.org/conda-forge/noarch/joblib-1.5.1-pyhd8ed1ab_0.conda#fb1c14694de51a476ce8636d92b6f42c +https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-31_he106b2a_openblas.conda#abb32c727da370c481a1c206f5159ce9 +https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-31_h7ac8fdf_openblas.conda#452b98eafe050ecff932f0ec832dd03f +https://conda.anaconda.org/conda-forge/noarch/pyproject-metadata-0.9.1-pyhd8ed1ab_0.conda#22ae7c6ea81e0c8661ef32168dda929b +https://conda.anaconda.org/conda-forge/noarch/python-freethreading-3.13.3-h92d6c8b_1.conda#4fa25290aec662a01642ba4b3c0ff5c1 +https://conda.anaconda.org/conda-forge/noarch/meson-python-0.18.0-pyh70fd9c4_0.conda#576c04b9d9f8e45285fb4d9452c26133 +https://conda.anaconda.org/conda-forge/linux-64/numpy-2.2.6-py313h103f029_0.conda#7ae0a483b2cbbdf15d8429eb38f74a9e +https://conda.anaconda.org/conda-forge/noarch/pytest-8.3.5-pyhd8ed1ab_0.conda#c3c9316209dec74a705a36797970c6be +https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.7.0-pyhd8ed1ab_0.conda#15353a2a0ea6dfefaa52fc5ab5b98f41 +https://conda.anaconda.org/conda-forge/linux-64/scipy-1.15.2-py313h7f7b39c_0.conda#65f0c403e4324062633e648933f20a2e diff --git a/build_tools/azure/pylatest_pip_openblas_pandas_environment.yml b/build_tools/azure/pylatest_pip_openblas_pandas_environment.yml new file mode 100644 index 0000000000000..6c3da4bb863b4 --- /dev/null +++ b/build_tools/azure/pylatest_pip_openblas_pandas_environment.yml @@ -0,0 +1,31 @@ +# DO NOT EDIT: this file is generated from the specification found in the +# following script to centralize the configuration for CI builds: +# build_tools/update_environments_and_lock_files.py +channels: + - defaults +dependencies: + - python + - ccache + - pip + - pip: + - numpy + - scipy + - cython + - joblib + - threadpoolctl + - matplotlib + - pandas + - pyamg + - pytest + - pytest-xdist + - pillow + - ninja + - meson-python + - pytest-cov + - coverage + - sphinx + - numpydoc + - lightgbm + - scikit-image + - array-api-strict + - scipy-doctest diff --git a/build_tools/azure/pylatest_pip_openblas_pandas_linux-64_conda.lock b/build_tools/azure/pylatest_pip_openblas_pandas_linux-64_conda.lock new file mode 100644 index 0000000000000..9861391b52e67 --- /dev/null +++ b/build_tools/azure/pylatest_pip_openblas_pandas_linux-64_conda.lock @@ -0,0 +1,91 @@ +# Generated by conda-lock. +# platform: linux-64 +# input_hash: 50f16a0198b6eb575a737fee25051b52a644d72f5fca26bd661651a85fcb6a07 +@EXPLICIT +https://repo.anaconda.com/pkgs/main/linux-64/_libgcc_mutex-0.1-main.conda#c3473ff8bdb3d124ed5ff11ec380d6f9 +https://repo.anaconda.com/pkgs/main/linux-64/ca-certificates-2025.2.25-h06a4308_0.conda#495015d24da8ad929e3ae2d18571016d +https://repo.anaconda.com/pkgs/main/linux-64/ld_impl_linux-64-2.40-h12ee557_0.conda#ee672b5f635340734f58d618b7bca024 +https://repo.anaconda.com/pkgs/main/linux-64/python_abi-3.13-0_cp313.conda#d4009c49dd2b54ffded7f1365b5f6505 +https://repo.anaconda.com/pkgs/main/noarch/tzdata-2025b-h04d1e81_0.conda#1d027393db3427ab22a02aa44a56f143 +https://repo.anaconda.com/pkgs/main/linux-64/libgomp-11.2.0-h1234567_1.conda#b372c0eea9b60732fdae4b817a63c8cd +https://repo.anaconda.com/pkgs/main/linux-64/libstdcxx-ng-11.2.0-h1234567_1.conda#57623d10a70e09e1d048c2b2b6f4e2dd +https://repo.anaconda.com/pkgs/main/linux-64/_openmp_mutex-5.1-1_gnu.conda#71d281e9c2192cb3fa425655a8defb85 +https://repo.anaconda.com/pkgs/main/linux-64/libgcc-ng-11.2.0-h1234567_1.conda#a87728dabf3151fb9cfa990bd2eb0464 +https://repo.anaconda.com/pkgs/main/linux-64/bzip2-1.0.8-h5eee18b_6.conda#f21a3ff51c1b271977f53ce956a69297 +https://repo.anaconda.com/pkgs/main/linux-64/expat-2.7.1-h6a678d5_0.conda#269942a9f3f943e2e5d8a2516a861f7c +https://repo.anaconda.com/pkgs/main/linux-64/libffi-3.4.4-h6a678d5_1.conda#70646cc713f0c43926cfdcfe9b695fe0 +https://repo.anaconda.com/pkgs/main/linux-64/libmpdec-4.0.0-h5eee18b_0.conda#feb10f42b1a7b523acbf85461be41a3e +https://repo.anaconda.com/pkgs/main/linux-64/libuuid-1.41.5-h5eee18b_0.conda#4a6a2354414c9080327274aa514e5299 +https://repo.anaconda.com/pkgs/main/linux-64/ncurses-6.4-h6a678d5_0.conda#5558eec6e2191741a92f832ea826251c +https://repo.anaconda.com/pkgs/main/linux-64/openssl-3.0.16-h5eee18b_0.conda#5875526739afa058cfa84da1fa7a2ef4 +https://repo.anaconda.com/pkgs/main/linux-64/xz-5.6.4-h5eee18b_1.conda#3581505fa450962d631bd82b8616350e +https://repo.anaconda.com/pkgs/main/linux-64/zlib-1.2.13-h5eee18b_1.conda#92e42d8310108b0a440fb2e60b2b2a25 +https://repo.anaconda.com/pkgs/main/linux-64/ccache-3.7.9-hfe4627d_0.conda#bef6fc681c273bb7bd0c67d1a591365e +https://repo.anaconda.com/pkgs/main/linux-64/readline-8.2-h5eee18b_0.conda#be42180685cce6e6b0329201d9f48efb +https://repo.anaconda.com/pkgs/main/linux-64/tk-8.6.14-h39e8969_0.conda#78dbc5e3c69143ebc037fc5d5b22e597 +https://repo.anaconda.com/pkgs/main/linux-64/sqlite-3.45.3-h5eee18b_0.conda#acf93d6aceb74d6110e20b44cc45939e +https://repo.anaconda.com/pkgs/main/linux-64/python-3.13.2-hf623796_100_cp313.conda#bf836f30ac4c16fd3d71c1aaa25da08c +https://repo.anaconda.com/pkgs/main/linux-64/setuptools-78.1.1-py313h06a4308_0.conda#8f8e1c1e3af9d2d371aaa0ee8316ae7c +https://repo.anaconda.com/pkgs/main/linux-64/wheel-0.45.1-py313h06a4308_0.conda#29057e876eedce0e37c2388c138a19f9 +https://repo.anaconda.com/pkgs/main/noarch/pip-25.1-pyhc872135_2.conda#2778327d2a700153fefe0e69438b18e1 +# pip alabaster @ https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl#sha256=fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b +# pip babel @ https://files.pythonhosted.org/packages/b7/b8/3fe70c75fe32afc4bb507f75563d39bc5642255d1d94f1f23604725780bf/babel-2.17.0-py3-none-any.whl#sha256=4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2 +# pip certifi @ https://files.pythonhosted.org/packages/4a/7e/3db2bd1b1f9e95f7cddca6d6e75e2f2bd9f51b1246e546d88addca0106bd/certifi-2025.4.26-py3-none-any.whl#sha256=30350364dfe371162649852c63336a15c70c6510c2ad5015b21c2345311805f3 +# pip charset-normalizer @ https://files.pythonhosted.org/packages/e2/28/ffc026b26f441fc67bd21ab7f03b313ab3fe46714a14b516f931abe1a2d8/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=6c9379d65defcab82d07b2a9dfbfc2e95bc8fe0ebb1b176a3190230a3ef0e07c +# pip coverage @ https://files.pythonhosted.org/packages/89/60/f5f50f61b6332451520e6cdc2401700c48310c64bc2dd34027a47d6ab4ca/coverage-7.8.2-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=dc67994df9bcd7e0150a47ef41278b9e0a0ea187caba72414b71dc590b99a108 +# pip cycler @ https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl#sha256=85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30 +# pip cython @ https://files.pythonhosted.org/packages/ca/90/9fe8b93fa239b4871252274892c232415f53d5af0859c4a6ac9b1cbf9950/cython-3.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=7da069ca769903c5dee56c5f7ab47b2b7b91030eee48912630db5f4f3ec5954a +# pip docutils @ https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl#sha256=dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2 +# pip execnet @ https://files.pythonhosted.org/packages/43/09/2aea36ff60d16dd8879bdb2f5b3ee0ba8d08cbbdcdfe870e695ce3784385/execnet-2.1.1-py3-none-any.whl#sha256=26dee51f1b80cebd6d0ca8e74dd8745419761d3bef34163928cbebbdc4749fdc +# pip fonttools @ https://files.pythonhosted.org/packages/c1/46/8b46469c6edac393de1c380c7ec61922d5440f25605dfca7849e5ffff295/fonttools-4.58.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=9b8860f8cd48b345bd1df1d7be650f600f69ee971ffe338c5bd5bcb6bdb3b92c +# pip idna @ https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl#sha256=946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 +# pip imagesize @ https://files.pythonhosted.org/packages/ff/62/85c4c919272577931d407be5ba5d71c20f0b616d31a0befe0ae45bb79abd/imagesize-1.4.1-py2.py3-none-any.whl#sha256=0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b +# pip iniconfig @ https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl#sha256=9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760 +# pip joblib @ https://files.pythonhosted.org/packages/7d/4f/1195bbac8e0c2acc5f740661631d8d750dc38d4a32b23ee5df3cde6f4e0d/joblib-1.5.1-py3-none-any.whl#sha256=4719a31f054c7d766948dcd83e9613686b27114f190f717cec7eaa2084f8a74a +# pip kiwisolver @ https://files.pythonhosted.org/packages/8f/e9/6a7d025d8da8c4931522922cd706105aa32b3291d1add8c5427cdcd66e63/kiwisolver-1.4.8-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=a5ce1e481a74b44dd5e92ff03ea0cb371ae7a0268318e202be06c8f04f4f1246 +# pip markupsafe @ https://files.pythonhosted.org/packages/0c/91/96cf928db8236f1bfab6ce15ad070dfdd02ed88261c2afafd4b43575e9e9/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396 +# pip meson @ https://files.pythonhosted.org/packages/46/77/726b14be352aa6911e206ca7c4d95c5be49660604dfee0bfed0fc75823e5/meson-1.8.1-py3-none-any.whl#sha256=374bbf71247e629475fc10b0bd2ef66fc418c2d8f4890572f74de0f97d0d42da +# pip networkx @ https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl#sha256=0030d386a9a06dee3565298b4a734b68589749a544acbb6c412dc9e2489ec6ec +# pip ninja @ https://files.pythonhosted.org/packages/eb/7a/455d2877fe6cf99886849c7f9755d897df32eaf3a0fba47b56e615f880f7/ninja-1.11.1.4-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.whl#sha256=096487995473320de7f65d622c3f1d16c3ad174797602218ca8c967f51ec38a0 +# pip numpy @ https://files.pythonhosted.org/packages/19/49/4df9123aafa7b539317bf6d342cb6d227e49f7a35b99c287a6109b13dd93/numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f +# pip packaging @ https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl#sha256=29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 +# pip pillow @ https://files.pythonhosted.org/packages/13/eb/2552ecebc0b887f539111c2cd241f538b8ff5891b8903dfe672e997529be/pillow-11.2.1-cp313-cp313-manylinux_2_28_x86_64.whl#sha256=ad275964d52e2243430472fc5d2c2334b4fc3ff9c16cb0a19254e25efa03a155 +# pip pluggy @ https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl#sha256=e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 +# pip pygments @ https://files.pythonhosted.org/packages/8a/0b/9fcc47d19c48b59121088dd6da2488a49d5f72dacf8262e2790a1d2c7d15/pygments-2.19.1-py3-none-any.whl#sha256=9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c +# pip pyparsing @ https://files.pythonhosted.org/packages/05/e7/df2285f3d08fee213f2d041540fa4fc9ca6c2d44cf36d3a035bf2a8d2bcc/pyparsing-3.2.3-py3-none-any.whl#sha256=a749938e02d6fd0b59b356ca504a24982314bb090c383e3cf201c95ef7e2bfcf +# pip pytz @ https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl#sha256=5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00 +# pip roman-numerals-py @ https://files.pythonhosted.org/packages/53/97/d2cbbaa10c9b826af0e10fdf836e1bf344d9f0abb873ebc34d1f49642d3f/roman_numerals_py-3.1.0-py3-none-any.whl#sha256=9da2ad2fb670bcf24e81070ceb3be72f6c11c440d73bd579fbeca1e9f330954c +# pip six @ https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl#sha256=4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 +# pip snowballstemmer @ https://files.pythonhosted.org/packages/c8/78/3565d011c61f5a43488987ee32b6f3f656e7f107ac2782dd57bdd7d91d9a/snowballstemmer-3.0.1-py3-none-any.whl#sha256=6cd7b3897da8d6c9ffb968a6781fa6532dce9c3618a4b127d920dab764a19064 +# pip sphinxcontrib-applehelp @ https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl#sha256=4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5 +# pip sphinxcontrib-devhelp @ https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl#sha256=aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2 +# pip sphinxcontrib-htmlhelp @ https://files.pythonhosted.org/packages/0a/7b/18a8c0bcec9182c05a0b3ec2a776bba4ead82750a55ff798e8d406dae604/sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl#sha256=166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8 +# pip sphinxcontrib-jsmath @ https://files.pythonhosted.org/packages/c2/42/4c8646762ee83602e3fb3fbe774c2fac12f317deb0b5dbeeedd2d3ba4b77/sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl#sha256=2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178 +# pip sphinxcontrib-qthelp @ https://files.pythonhosted.org/packages/27/83/859ecdd180cacc13b1f7e857abf8582a64552ea7a061057a6c716e790fce/sphinxcontrib_qthelp-2.0.0-py3-none-any.whl#sha256=b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb +# pip sphinxcontrib-serializinghtml @ https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl#sha256=6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331 +# pip tabulate @ https://files.pythonhosted.org/packages/40/44/4a5f08c96eb108af5cb50b41f76142f0afa346dfa99d5296fe7202a11854/tabulate-0.9.0-py3-none-any.whl#sha256=024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f +# pip threadpoolctl @ https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl#sha256=43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb +# pip tzdata @ https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl#sha256=1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 +# pip urllib3 @ https://files.pythonhosted.org/packages/6b/11/cc635220681e93a0183390e26485430ca2c7b5f9d33b15c74c2861cb8091/urllib3-2.4.0-py3-none-any.whl#sha256=4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813 +# pip array-api-strict @ https://files.pythonhosted.org/packages/fe/c7/a97e26083985b49a7a54006364348cf1c26e5523850b8522a39b02b19715/array_api_strict-2.3.1-py3-none-any.whl#sha256=0ca6988be1c82d2f05b6cd44bc7e14cb390555d1455deb50f431d6d0cf468ded +# pip contourpy @ https://files.pythonhosted.org/packages/c8/65/5245ce8c548a8422236c13ffcdcdada6a2a812c361e9e0c70548bb40b661/contourpy-1.3.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=434f0adf84911c924519d2b08fc10491dd282b20bdd3fa8f60fd816ea0b48841 +# pip imageio @ https://files.pythonhosted.org/packages/cb/bd/b394387b598ed84d8d0fa90611a90bee0adc2021820ad5729f7ced74a8e2/imageio-2.37.0-py3-none-any.whl#sha256=11efa15b87bc7871b61590326b2d635439acc321cf7f8ce996f812543ce10eed +# pip jinja2 @ https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl#sha256=85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 +# pip lazy-loader @ https://files.pythonhosted.org/packages/83/60/d497a310bde3f01cb805196ac61b7ad6dc5dcf8dce66634dc34364b20b4f/lazy_loader-0.4-py3-none-any.whl#sha256=342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc +# pip pyproject-metadata @ https://files.pythonhosted.org/packages/7e/b1/8e63033b259e0a4e40dd1ec4a9fee17718016845048b43a36ec67d62e6fe/pyproject_metadata-0.9.1-py3-none-any.whl#sha256=ee5efde548c3ed9b75a354fc319d5afd25e9585fa918a34f62f904cc731973ad +# pip pytest @ https://files.pythonhosted.org/packages/30/3d/64ad57c803f1fa1e963a7946b6e0fea4a70df53c1a7fed304586539c2bac/pytest-8.3.5-py3-none-any.whl#sha256=c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820 +# pip python-dateutil @ https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl#sha256=a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 +# pip requests @ https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl#sha256=70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 +# pip scipy @ https://files.pythonhosted.org/packages/b5/09/c5b6734a50ad4882432b6bb7c02baf757f5b2f256041da5df242e2d7e6b6/scipy-1.15.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=c9deabd6d547aee2c9a81dee6cc96c6d7e9a9b1953f74850c179f91fdc729cb7 +# pip tifffile @ https://files.pythonhosted.org/packages/4d/77/7f7dfcf2d847c1c1c63a2d4157c480eb4c74e4aa56e844008795ff01f86d/tifffile-2025.6.1-py3-none-any.whl#sha256=ff7163f1aaea519b769a2ac77c43be69e7d83e5b5d5d6a676497399de50535e5 +# pip lightgbm @ https://files.pythonhosted.org/packages/42/86/dabda8fbcb1b00bcfb0003c3776e8ade1aa7b413dff0a2c08f457dace22f/lightgbm-4.6.0-py3-none-manylinux_2_28_x86_64.whl#sha256=cb19b5afea55b5b61cbb2131095f50538bd608a00655f23ad5d25ae3e3bf1c8d +# pip matplotlib @ https://files.pythonhosted.org/packages/f5/64/41c4367bcaecbc03ef0d2a3ecee58a7065d0a36ae1aa817fe573a2da66d4/matplotlib-3.10.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=a80fcccbef63302c0efd78042ea3c2436104c5b1a4d3ae20f864593696364ac7 +# pip meson-python @ https://files.pythonhosted.org/packages/28/58/66db620a8a7ccb32633de9f403fe49f1b63c68ca94e5c340ec5cceeb9821/meson_python-0.18.0-py3-none-any.whl#sha256=3b0fe051551cc238f5febb873247c0949cd60ded556efa130aa57021804868e2 +# pip pandas @ https://files.pythonhosted.org/packages/e8/31/aa8da88ca0eadbabd0a639788a6da13bb2ff6edbbb9f29aa786450a30a91/pandas-2.2.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=f3a255b2c19987fbbe62a9dfd6cff7ff2aa9ccab3fc75218fd4b7530f01efa24 +# pip pyamg @ https://files.pythonhosted.org/packages/cd/a7/0df731cbfb09e73979a1a032fc7bc5be0eba617d798b998a0f887afe8ade/pyamg-5.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=6999b351ab969c79faacb81faa74c0fa9682feeff3954979212872a3ee40c298 +# pip pytest-cov @ https://files.pythonhosted.org/packages/28/d0/def53b4a790cfb21483016430ed828f64830dd981ebe1089971cd10cab25/pytest_cov-6.1.1-py3-none-any.whl#sha256=bddf29ed2d0ab6f4df17b4c55b0a657287db8684af9c42ea546b21b1041b3dde +# pip pytest-xdist @ https://files.pythonhosted.org/packages/0d/b2/0e802fde6f1c5b2f7ae7e9ad42b83fd4ecebac18a8a8c2f2f14e39dce6e1/pytest_xdist-3.7.0-py3-none-any.whl#sha256=7d3fbd255998265052435eb9daa4e99b62e6fb9cfb6efd1f858d4d8c0c7f0ca0 +# pip scikit-image @ https://files.pythonhosted.org/packages/cd/9b/c3da56a145f52cd61a68b8465d6a29d9503bc45bc993bb45e84371c97d94/scikit_image-0.25.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=b8abd3c805ce6944b941cfed0406d88faeb19bab3ed3d4b50187af55cf24d147 +# pip scipy-doctest @ https://files.pythonhosted.org/packages/c9/13/cd25d1875f3804b73fd4a4ae00e2c76e274e1e0608d79148cac251b644b1/scipy_doctest-1.8.0-py3-none-any.whl#sha256=5863208368c35486e143ce3283ab2f517a0d6b0c63d0d5f19f38a823fc82016f +# pip sphinx @ https://files.pythonhosted.org/packages/31/53/136e9eca6e0b9dc0e1962e2c908fbea2e5ac000c2a2fbd9a35797958c48b/sphinx-8.2.3-py3-none-any.whl#sha256=4405915165f13521d875a8c29c8970800a0141c14cc5416a38feca4ea5d9b9c3 +# pip numpydoc @ https://files.pythonhosted.org/packages/6c/45/56d99ba9366476cd8548527667f01869279cedb9e66b28eb4dfb27701679/numpydoc-1.8.0-py3-none-any.whl#sha256=72024c7fd5e17375dec3608a27c03303e8ad00c81292667955c6fea7a3ccf541 diff --git a/build_tools/azure/pylatest_pip_scipy_dev_environment.yml b/build_tools/azure/pylatest_pip_scipy_dev_environment.yml new file mode 100644 index 0000000000000..01709b79e3720 --- /dev/null +++ b/build_tools/azure/pylatest_pip_scipy_dev_environment.yml @@ -0,0 +1,22 @@ +# DO NOT EDIT: this file is generated from the specification found in the +# following script to centralize the configuration for CI builds: +# build_tools/update_environments_and_lock_files.py +channels: + - defaults +dependencies: + - python + - ccache + - pip + - pip: + - threadpoolctl + - pytest + - pytest-xdist + - pip + - ninja + - meson-python + - pytest-cov + - coverage + - pooch + - sphinx + - numpydoc + - python-dateutil diff --git a/build_tools/azure/pylatest_pip_scipy_dev_linux-64_conda.lock b/build_tools/azure/pylatest_pip_scipy_dev_linux-64_conda.lock new file mode 100644 index 0000000000000..a8fac4ea35b6c --- /dev/null +++ b/build_tools/azure/pylatest_pip_scipy_dev_linux-64_conda.lock @@ -0,0 +1,70 @@ +# Generated by conda-lock. +# platform: linux-64 +# input_hash: 7555819e95d879c5a5147e6431581e17ffc5d77e8a43b19c8a911821378d2521 +@EXPLICIT +https://repo.anaconda.com/pkgs/main/linux-64/_libgcc_mutex-0.1-main.conda#c3473ff8bdb3d124ed5ff11ec380d6f9 +https://repo.anaconda.com/pkgs/main/linux-64/ca-certificates-2025.2.25-h06a4308_0.conda#495015d24da8ad929e3ae2d18571016d +https://repo.anaconda.com/pkgs/main/linux-64/ld_impl_linux-64-2.40-h12ee557_0.conda#ee672b5f635340734f58d618b7bca024 +https://repo.anaconda.com/pkgs/main/linux-64/python_abi-3.13-0_cp313.conda#d4009c49dd2b54ffded7f1365b5f6505 +https://repo.anaconda.com/pkgs/main/noarch/tzdata-2025b-h04d1e81_0.conda#1d027393db3427ab22a02aa44a56f143 +https://repo.anaconda.com/pkgs/main/linux-64/libgomp-11.2.0-h1234567_1.conda#b372c0eea9b60732fdae4b817a63c8cd +https://repo.anaconda.com/pkgs/main/linux-64/libstdcxx-ng-11.2.0-h1234567_1.conda#57623d10a70e09e1d048c2b2b6f4e2dd +https://repo.anaconda.com/pkgs/main/linux-64/_openmp_mutex-5.1-1_gnu.conda#71d281e9c2192cb3fa425655a8defb85 +https://repo.anaconda.com/pkgs/main/linux-64/libgcc-ng-11.2.0-h1234567_1.conda#a87728dabf3151fb9cfa990bd2eb0464 +https://repo.anaconda.com/pkgs/main/linux-64/bzip2-1.0.8-h5eee18b_6.conda#f21a3ff51c1b271977f53ce956a69297 +https://repo.anaconda.com/pkgs/main/linux-64/expat-2.7.1-h6a678d5_0.conda#269942a9f3f943e2e5d8a2516a861f7c +https://repo.anaconda.com/pkgs/main/linux-64/libffi-3.4.4-h6a678d5_1.conda#70646cc713f0c43926cfdcfe9b695fe0 +https://repo.anaconda.com/pkgs/main/linux-64/libmpdec-4.0.0-h5eee18b_0.conda#feb10f42b1a7b523acbf85461be41a3e +https://repo.anaconda.com/pkgs/main/linux-64/libuuid-1.41.5-h5eee18b_0.conda#4a6a2354414c9080327274aa514e5299 +https://repo.anaconda.com/pkgs/main/linux-64/ncurses-6.4-h6a678d5_0.conda#5558eec6e2191741a92f832ea826251c +https://repo.anaconda.com/pkgs/main/linux-64/openssl-3.0.16-h5eee18b_0.conda#5875526739afa058cfa84da1fa7a2ef4 +https://repo.anaconda.com/pkgs/main/linux-64/xz-5.6.4-h5eee18b_1.conda#3581505fa450962d631bd82b8616350e +https://repo.anaconda.com/pkgs/main/linux-64/zlib-1.2.13-h5eee18b_1.conda#92e42d8310108b0a440fb2e60b2b2a25 +https://repo.anaconda.com/pkgs/main/linux-64/ccache-3.7.9-hfe4627d_0.conda#bef6fc681c273bb7bd0c67d1a591365e +https://repo.anaconda.com/pkgs/main/linux-64/readline-8.2-h5eee18b_0.conda#be42180685cce6e6b0329201d9f48efb +https://repo.anaconda.com/pkgs/main/linux-64/tk-8.6.14-h39e8969_0.conda#78dbc5e3c69143ebc037fc5d5b22e597 +https://repo.anaconda.com/pkgs/main/linux-64/sqlite-3.45.3-h5eee18b_0.conda#acf93d6aceb74d6110e20b44cc45939e +https://repo.anaconda.com/pkgs/main/linux-64/python-3.13.2-hf623796_100_cp313.conda#bf836f30ac4c16fd3d71c1aaa25da08c +https://repo.anaconda.com/pkgs/main/linux-64/setuptools-78.1.1-py313h06a4308_0.conda#8f8e1c1e3af9d2d371aaa0ee8316ae7c +https://repo.anaconda.com/pkgs/main/linux-64/wheel-0.45.1-py313h06a4308_0.conda#29057e876eedce0e37c2388c138a19f9 +https://repo.anaconda.com/pkgs/main/noarch/pip-25.1-pyhc872135_2.conda#2778327d2a700153fefe0e69438b18e1 +# pip alabaster @ https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl#sha256=fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b +# pip babel @ https://files.pythonhosted.org/packages/b7/b8/3fe70c75fe32afc4bb507f75563d39bc5642255d1d94f1f23604725780bf/babel-2.17.0-py3-none-any.whl#sha256=4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2 +# pip certifi @ https://files.pythonhosted.org/packages/4a/7e/3db2bd1b1f9e95f7cddca6d6e75e2f2bd9f51b1246e546d88addca0106bd/certifi-2025.4.26-py3-none-any.whl#sha256=30350364dfe371162649852c63336a15c70c6510c2ad5015b21c2345311805f3 +# pip charset-normalizer @ https://files.pythonhosted.org/packages/e2/28/ffc026b26f441fc67bd21ab7f03b313ab3fe46714a14b516f931abe1a2d8/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=6c9379d65defcab82d07b2a9dfbfc2e95bc8fe0ebb1b176a3190230a3ef0e07c +# pip coverage @ https://files.pythonhosted.org/packages/89/60/f5f50f61b6332451520e6cdc2401700c48310c64bc2dd34027a47d6ab4ca/coverage-7.8.2-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=dc67994df9bcd7e0150a47ef41278b9e0a0ea187caba72414b71dc590b99a108 +# pip docutils @ https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl#sha256=dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2 +# pip execnet @ https://files.pythonhosted.org/packages/43/09/2aea36ff60d16dd8879bdb2f5b3ee0ba8d08cbbdcdfe870e695ce3784385/execnet-2.1.1-py3-none-any.whl#sha256=26dee51f1b80cebd6d0ca8e74dd8745419761d3bef34163928cbebbdc4749fdc +# pip idna @ https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl#sha256=946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 +# pip imagesize @ https://files.pythonhosted.org/packages/ff/62/85c4c919272577931d407be5ba5d71c20f0b616d31a0befe0ae45bb79abd/imagesize-1.4.1-py2.py3-none-any.whl#sha256=0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b +# pip iniconfig @ https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl#sha256=9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760 +# pip markupsafe @ https://files.pythonhosted.org/packages/0c/91/96cf928db8236f1bfab6ce15ad070dfdd02ed88261c2afafd4b43575e9e9/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396 +# pip meson @ https://files.pythonhosted.org/packages/46/77/726b14be352aa6911e206ca7c4d95c5be49660604dfee0bfed0fc75823e5/meson-1.8.1-py3-none-any.whl#sha256=374bbf71247e629475fc10b0bd2ef66fc418c2d8f4890572f74de0f97d0d42da +# pip ninja @ https://files.pythonhosted.org/packages/eb/7a/455d2877fe6cf99886849c7f9755d897df32eaf3a0fba47b56e615f880f7/ninja-1.11.1.4-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.whl#sha256=096487995473320de7f65d622c3f1d16c3ad174797602218ca8c967f51ec38a0 +# pip packaging @ https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl#sha256=29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 +# pip platformdirs @ https://files.pythonhosted.org/packages/fe/39/979e8e21520d4e47a0bbe349e2713c0aac6f3d853d0e5b34d76206c439aa/platformdirs-4.3.8-py3-none-any.whl#sha256=ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4 +# pip pluggy @ https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl#sha256=e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 +# pip pygments @ https://files.pythonhosted.org/packages/8a/0b/9fcc47d19c48b59121088dd6da2488a49d5f72dacf8262e2790a1d2c7d15/pygments-2.19.1-py3-none-any.whl#sha256=9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c +# pip roman-numerals-py @ https://files.pythonhosted.org/packages/53/97/d2cbbaa10c9b826af0e10fdf836e1bf344d9f0abb873ebc34d1f49642d3f/roman_numerals_py-3.1.0-py3-none-any.whl#sha256=9da2ad2fb670bcf24e81070ceb3be72f6c11c440d73bd579fbeca1e9f330954c +# pip six @ https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl#sha256=4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 +# pip snowballstemmer @ https://files.pythonhosted.org/packages/c8/78/3565d011c61f5a43488987ee32b6f3f656e7f107ac2782dd57bdd7d91d9a/snowballstemmer-3.0.1-py3-none-any.whl#sha256=6cd7b3897da8d6c9ffb968a6781fa6532dce9c3618a4b127d920dab764a19064 +# pip sphinxcontrib-applehelp @ https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl#sha256=4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5 +# pip sphinxcontrib-devhelp @ https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl#sha256=aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2 +# pip sphinxcontrib-htmlhelp @ https://files.pythonhosted.org/packages/0a/7b/18a8c0bcec9182c05a0b3ec2a776bba4ead82750a55ff798e8d406dae604/sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl#sha256=166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8 +# pip sphinxcontrib-jsmath @ https://files.pythonhosted.org/packages/c2/42/4c8646762ee83602e3fb3fbe774c2fac12f317deb0b5dbeeedd2d3ba4b77/sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl#sha256=2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178 +# pip sphinxcontrib-qthelp @ https://files.pythonhosted.org/packages/27/83/859ecdd180cacc13b1f7e857abf8582a64552ea7a061057a6c716e790fce/sphinxcontrib_qthelp-2.0.0-py3-none-any.whl#sha256=b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb +# pip sphinxcontrib-serializinghtml @ https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl#sha256=6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331 +# pip tabulate @ https://files.pythonhosted.org/packages/40/44/4a5f08c96eb108af5cb50b41f76142f0afa346dfa99d5296fe7202a11854/tabulate-0.9.0-py3-none-any.whl#sha256=024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f +# pip threadpoolctl @ https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl#sha256=43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb +# pip urllib3 @ https://files.pythonhosted.org/packages/6b/11/cc635220681e93a0183390e26485430ca2c7b5f9d33b15c74c2861cb8091/urllib3-2.4.0-py3-none-any.whl#sha256=4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813 +# pip jinja2 @ https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl#sha256=85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 +# pip pyproject-metadata @ https://files.pythonhosted.org/packages/7e/b1/8e63033b259e0a4e40dd1ec4a9fee17718016845048b43a36ec67d62e6fe/pyproject_metadata-0.9.1-py3-none-any.whl#sha256=ee5efde548c3ed9b75a354fc319d5afd25e9585fa918a34f62f904cc731973ad +# pip pytest @ https://files.pythonhosted.org/packages/30/3d/64ad57c803f1fa1e963a7946b6e0fea4a70df53c1a7fed304586539c2bac/pytest-8.3.5-py3-none-any.whl#sha256=c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820 +# pip python-dateutil @ https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl#sha256=a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 +# pip requests @ https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl#sha256=70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 +# pip meson-python @ https://files.pythonhosted.org/packages/28/58/66db620a8a7ccb32633de9f403fe49f1b63c68ca94e5c340ec5cceeb9821/meson_python-0.18.0-py3-none-any.whl#sha256=3b0fe051551cc238f5febb873247c0949cd60ded556efa130aa57021804868e2 +# pip pooch @ https://files.pythonhosted.org/packages/a8/87/77cc11c7a9ea9fd05503def69e3d18605852cd0d4b0d3b8f15bbeb3ef1d1/pooch-1.8.2-py3-none-any.whl#sha256=3529a57096f7198778a5ceefd5ac3ef0e4d06a6ddaf9fc2d609b806f25302c47 +# pip pytest-cov @ https://files.pythonhosted.org/packages/28/d0/def53b4a790cfb21483016430ed828f64830dd981ebe1089971cd10cab25/pytest_cov-6.1.1-py3-none-any.whl#sha256=bddf29ed2d0ab6f4df17b4c55b0a657287db8684af9c42ea546b21b1041b3dde +# pip pytest-xdist @ https://files.pythonhosted.org/packages/0d/b2/0e802fde6f1c5b2f7ae7e9ad42b83fd4ecebac18a8a8c2f2f14e39dce6e1/pytest_xdist-3.7.0-py3-none-any.whl#sha256=7d3fbd255998265052435eb9daa4e99b62e6fb9cfb6efd1f858d4d8c0c7f0ca0 +# pip sphinx @ https://files.pythonhosted.org/packages/31/53/136e9eca6e0b9dc0e1962e2c908fbea2e5ac000c2a2fbd9a35797958c48b/sphinx-8.2.3-py3-none-any.whl#sha256=4405915165f13521d875a8c29c8970800a0141c14cc5416a38feca4ea5d9b9c3 +# pip numpydoc @ https://files.pythonhosted.org/packages/6c/45/56d99ba9366476cd8548527667f01869279cedb9e66b28eb4dfb27701679/numpydoc-1.8.0-py3-none-any.whl#sha256=72024c7fd5e17375dec3608a27c03303e8ad00c81292667955c6fea7a3ccf541 diff --git a/build_tools/azure/pymin_conda_forge_mkl_environment.yml b/build_tools/azure/pymin_conda_forge_mkl_environment.yml new file mode 100644 index 0000000000000..fe6ce91950e4a --- /dev/null +++ b/build_tools/azure/pymin_conda_forge_mkl_environment.yml @@ -0,0 +1,24 @@ +# DO NOT EDIT: this file is generated from the specification found in the +# following script to centralize the configuration for CI builds: +# build_tools/update_environments_and_lock_files.py +channels: + - conda-forge +dependencies: + - python=3.10 + - numpy + - blas[build=mkl] + - scipy + - cython + - joblib + - threadpoolctl + - matplotlib + - pytest + - pytest-xdist + - pillow + - pip + - ninja + - meson-python + - pytest-cov + - coverage + - wheel + - pip diff --git a/build_tools/azure/pymin_conda_forge_mkl_win-64_conda.lock b/build_tools/azure/pymin_conda_forge_mkl_win-64_conda.lock new file mode 100644 index 0000000000000..9e7e414a90156 --- /dev/null +++ b/build_tools/azure/pymin_conda_forge_mkl_win-64_conda.lock @@ -0,0 +1,118 @@ +# Generated by conda-lock. +# platform: win-64 +# input_hash: cc5e2a711eb32773dc46fe159e1c3fe14f4fd07565fc8d3dedf2d748d4f2f694 +@EXPLICIT +https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2#0c96522c6bdaed4b1566d11387caaf45 +https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2#34893075a5c9e55cdafac56607368fc6 +https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2#4d59c254e01d9cde7957100457e2d5fb +https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-h77eed37_3.conda#49023d73832ef61042f6a237cb2687e7 +https://conda.anaconda.org/conda-forge/win-64/intel-openmp-2024.2.1-h57928b3_1083.conda#2d89243bfb53652c182a7c73182cce4f +https://conda.anaconda.org/conda-forge/win-64/mkl-include-2024.2.2-h66d3029_15.conda#e2f516189b44b6e042199d13e7015361 +https://conda.anaconda.org/conda-forge/noarch/python_abi-3.10-7_cp310.conda#44e871cba2b162368476a84b8d040b6c +https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda#4222072737ccff51314b5ece9c7d6f5a +https://conda.anaconda.org/conda-forge/win-64/ucrt-10.0.22621.0-h57928b3_1.conda#6797b005cd0f439c4c5c9ac565783700 +https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.4.26-h4c7d964_0.conda#23c7fd5062b48d8294fc7f61bf157fba +https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-0.tar.bz2#f766549260d6815b0c52253f1fb1bb29 +https://conda.anaconda.org/conda-forge/win-64/libwinpthread-12.0.0.r4.gg4f2fc60ca-h57928b3_9.conda#08bfa5da6e242025304b206d152479ef +https://conda.anaconda.org/conda-forge/win-64/vc14_runtime-14.42.34438-hfd919c2_26.conda#91651a36d31aa20c7ba36299fb7068f4 +https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2#fee5683a3f04bd15cbd8318b096a27ab +https://conda.anaconda.org/conda-forge/win-64/libgomp-15.1.0-h1383e82_2.conda#5fbacaa9b41e294a6966602205b99747 +https://conda.anaconda.org/conda-forge/win-64/vc-14.3-h2b53caa_26.conda#d3f0381e38093bde620a8d85f266ae55 +https://conda.anaconda.org/conda-forge/win-64/_openmp_mutex-4.5-2_gnu.conda#37e16618af5c4851a3f3d66dd0e11141 +https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h2466b09_7.conda#276e7ffe9ffe39688abc665ef0f45596 +https://conda.anaconda.org/conda-forge/win-64/double-conversion-3.3.1-he0c23c2_0.conda#e9a1402439c18a4e3c7a52e4246e9e1c +https://conda.anaconda.org/conda-forge/win-64/graphite2-1.3.13-h63175ca_1003.conda#3194499ee7d1a67404a87d0eefdd92c6 +https://conda.anaconda.org/conda-forge/win-64/icu-75.1-he0c23c2_0.conda#8579b6bb8d18be7c0b27fb08adeeeb40 +https://conda.anaconda.org/conda-forge/win-64/lerc-4.0.0-h6470a55_1.conda#c1b81da6d29a14b542da14a36c9fbf3f +https://conda.anaconda.org/conda-forge/win-64/libbrotlicommon-1.1.0-h2466b09_2.conda#f7dc9a8f21d74eab46456df301da2972 +https://conda.anaconda.org/conda-forge/win-64/libdeflate-1.24-h76ddb4d_0.conda#08d988e266c6ae77e03d164b83786dc4 +https://conda.anaconda.org/conda-forge/win-64/libexpat-2.7.0-he0c23c2_0.conda#b6f5352fdb525662f4169a0431d2dd7a +https://conda.anaconda.org/conda-forge/win-64/libffi-3.4.6-h537db12_1.conda#85d8fa5e55ed8f93f874b3b23ed54ec6 +https://conda.anaconda.org/conda-forge/win-64/libiconv-1.18-h135ad9c_1.conda#21fc5dba2cbcd8e5e26ff976a312122c +https://conda.anaconda.org/conda-forge/win-64/libjpeg-turbo-3.1.0-h2466b09_0.conda#7c51d27540389de84852daa1cdb9c63c +https://conda.anaconda.org/conda-forge/win-64/liblzma-5.8.1-h2466b09_1.conda#14a1042c163181e143a7522dfb8ad6ab +https://conda.anaconda.org/conda-forge/win-64/libsqlite-3.50.0-h67fdade_0.conda#92b11b0b2120d563caa1629928122cee +https://conda.anaconda.org/conda-forge/win-64/libwebp-base-1.5.0-h3b0e114_0.conda#33f7313967072c6e6d8f865f5493c7ae +https://conda.anaconda.org/conda-forge/win-64/libzlib-1.3.1-h2466b09_2.conda#41fbfac52c601159df6c01f875de31b9 +https://conda.anaconda.org/conda-forge/win-64/ninja-1.12.1-hc790b64_1.conda#3974c522f3248d4a93e6940c463d2de7 +https://conda.anaconda.org/conda-forge/win-64/openssl-3.5.0-ha4e3fda_1.conda#72c07e46b6766bb057018a9a74861b89 +https://conda.anaconda.org/conda-forge/win-64/pixman-0.46.0-had0cd8c_0.conda#01617534ef71b5385ebba940a6d6150d +https://conda.anaconda.org/conda-forge/win-64/qhull-2020.2-hc790b64_5.conda#854fbdff64b572b5c0b470f334d34c11 +https://conda.anaconda.org/conda-forge/win-64/tk-8.6.13-h2c6b04d_2.conda#ebd0e761de9aa879a51d22cc721bd095 +https://conda.anaconda.org/conda-forge/win-64/krb5-1.21.3-hdf4eb48_0.conda#31aec030344e962fbd7dbbbbd68e60a9 +https://conda.anaconda.org/conda-forge/win-64/libbrotlidec-1.1.0-h2466b09_2.conda#9bae75ce723fa34e98e239d21d752a7e +https://conda.anaconda.org/conda-forge/win-64/libbrotlienc-1.1.0-h2466b09_2.conda#85741a24d97954a991e55e34bc55990b +https://conda.anaconda.org/conda-forge/win-64/libgcc-15.1.0-h1383e82_2.conda#9bedb24480136bfeb81ebc81d4285e70 +https://conda.anaconda.org/conda-forge/win-64/libintl-0.22.5-h5728263_3.conda#2cf0cf76cc15d360dfa2f17fd6cf9772 +https://conda.anaconda.org/conda-forge/win-64/libpng-1.6.47-h7a4582a_0.conda#ad620e92b82d2948bc019e029c574ebb +https://conda.anaconda.org/conda-forge/win-64/libxml2-2.13.8-h442d1da_0.conda#833c2dbc1a5020007b520b044c713ed3 +https://conda.anaconda.org/conda-forge/win-64/pcre2-10.45-h99c9b8b_0.conda#f4c483274001678e129f5cbaf3a8d765 +https://conda.anaconda.org/conda-forge/win-64/python-3.10.17-h8c5b53a_0_cpython.conda#0c59918f056ab2e9c7bb45970d32b2ea +https://conda.anaconda.org/conda-forge/win-64/zstd-1.5.7-hbeecb71_2.conda#21f56217d6125fb30c3c3f10c786d751 +https://conda.anaconda.org/conda-forge/win-64/brotli-bin-1.1.0-h2466b09_2.conda#d22534a9be5771fc58eb7564947f669d +https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda#962b9857ee8e7018c22f2776ffa0b2d7 +https://conda.anaconda.org/conda-forge/noarch/cycler-0.12.1-pyhd8ed1ab_1.conda#44600c4667a319d67dbe0681fc0bc833 +https://conda.anaconda.org/conda-forge/win-64/cython-3.1.1-py310h6bd2d47_1.conda#165131d296d24f798fa76a26694d4565 +https://conda.anaconda.org/conda-forge/noarch/execnet-2.1.1-pyhd8ed1ab_1.conda#a71efeae2c160f6789900ba2631a2c90 +https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_1.conda#6837f3eff7dcea42ecd714ce1ac2b108 +https://conda.anaconda.org/conda-forge/win-64/kiwisolver-1.4.7-py310hc19bc0b_0.conda#50d96539497fc7493cbe469fbb6b8b6e +https://conda.anaconda.org/conda-forge/win-64/libclang13-20.1.6-default_h6e92b77_0.conda#3920536319b052a9a49639e02fda2db7 +https://conda.anaconda.org/conda-forge/win-64/libfreetype6-2.13.3-h0b5ce68_1.conda#a84b7d1a13060a9372bea961a8131dbc +https://conda.anaconda.org/conda-forge/win-64/libglib-2.84.2-hbc94333_0.conda#fee05801cc5db97bec20a5e78fb3905b +https://conda.anaconda.org/conda-forge/win-64/libhwloc-2.11.2-default_ha69328c_1001.conda#b87a0ac5ab6495d8225db5dc72dd21cd +https://conda.anaconda.org/conda-forge/win-64/libtiff-4.7.0-h05922d8_5.conda#75370aba951b47ec3b5bfe689f1bcf7f +https://conda.anaconda.org/conda-forge/win-64/libxslt-1.1.39-h3df6e99_0.conda#279ee338c9b34871d578cb3c7aa68f70 +https://conda.anaconda.org/conda-forge/noarch/meson-1.8.1-pyhe01879c_0.conda#f3cccd9a6ce5331ae33f69ade5529162 +https://conda.anaconda.org/conda-forge/noarch/munkres-1.1.4-pyh9f0ad1d_0.tar.bz2#2ba8498c1018c1e9c61eb99b973dfe19 +https://conda.anaconda.org/conda-forge/noarch/packaging-25.0-pyh29332c3_1.conda#58335b26c38bf4a20f399384c33cbcf9 +https://conda.anaconda.org/conda-forge/noarch/pluggy-1.6.0-pyhd8ed1ab_0.conda#7da7ccd349dbf6487a7778579d2bb971 +https://conda.anaconda.org/conda-forge/win-64/pthread-stubs-0.4-h0e40799_1002.conda#3c8f2573569bb816483e5cf57efbbe29 +https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.2.3-pyhd8ed1ab_1.conda#513d3c262ee49b54a8fec85c5bc99764 +https://conda.anaconda.org/conda-forge/noarch/setuptools-80.9.0-pyhff2d567_0.conda#4de79c071274a53dcaf2a8c749d1499e +https://conda.anaconda.org/conda-forge/noarch/six-1.17.0-pyhd8ed1ab_0.conda#a451d576819089b0d672f18768be0f65 +https://conda.anaconda.org/conda-forge/noarch/threadpoolctl-3.6.0-pyhecae5ae_0.conda#9d64911b31d57ca443e9f1e36b04385f +https://conda.anaconda.org/conda-forge/noarch/toml-0.10.2-pyhd8ed1ab_1.conda#b0dd904de08b7db706167240bf37b164 +https://conda.anaconda.org/conda-forge/noarch/tomli-2.2.1-pyhd8ed1ab_1.conda#ac944244f1fed2eb49bae07193ae8215 +https://conda.anaconda.org/conda-forge/win-64/tornado-6.5.1-py310ha8f682b_0.conda#4c8f599990e386f3a0aba3f3bd8608da +https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.13.2-pyh29332c3_0.conda#83fc6ae00127671e301c9f44254c31b8 +https://conda.anaconda.org/conda-forge/win-64/unicodedata2-16.0.0-py310ha8f682b_0.conda#b28aead44c6e19a1fbba7752aa242b34 +https://conda.anaconda.org/conda-forge/noarch/wheel-0.45.1-pyhd8ed1ab_1.conda#75cb7132eb58d97896e173ef12ac9986 +https://conda.anaconda.org/conda-forge/win-64/xorg-libxau-1.0.12-h0e40799_0.conda#2ffbfae4548098297c033228256eb96e +https://conda.anaconda.org/conda-forge/win-64/xorg-libxdmcp-1.1.5-h0e40799_0.conda#8393c0f7e7870b4eb45553326f81f0ff +https://conda.anaconda.org/conda-forge/win-64/brotli-1.1.0-h2466b09_2.conda#378f1c9421775dfe644731cb121c8979 +https://conda.anaconda.org/conda-forge/win-64/coverage-7.8.2-py310h38315fa_0.conda#5e09090744ab0b70b2882bc415c0d5ad +https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.0-pyhd8ed1ab_0.conda#72e42d28960d875c7654614f8b50939a +https://conda.anaconda.org/conda-forge/noarch/joblib-1.5.1-pyhd8ed1ab_0.conda#fb1c14694de51a476ce8636d92b6f42c +https://conda.anaconda.org/conda-forge/win-64/lcms2-2.17-hbcf6048_0.conda#3538827f77b82a837fa681a4579e37a1 +https://conda.anaconda.org/conda-forge/win-64/libfreetype-2.13.3-h57928b3_1.conda#410ba2c8e7bdb278dfbb5d40220e39d2 +https://conda.anaconda.org/conda-forge/win-64/libxcb-1.17.0-h0e4246c_0.conda#a69bbf778a462da324489976c84cfc8c +https://conda.anaconda.org/conda-forge/win-64/openjpeg-2.5.3-h4d64b90_0.conda#fc050366dd0b8313eb797ed1ffef3a29 +https://conda.anaconda.org/conda-forge/noarch/pip-25.1.1-pyh8b19718_0.conda#32d0781ace05105cc99af55d36cbec7c +https://conda.anaconda.org/conda-forge/noarch/pyproject-metadata-0.9.1-pyhd8ed1ab_0.conda#22ae7c6ea81e0c8661ef32168dda929b +https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhff2d567_1.conda#5ba79d7c71f03c678c8ead841f347d6e +https://conda.anaconda.org/conda-forge/win-64/tbb-2021.13.0-h62715c5_1.conda#9190dd0a23d925f7602f9628b3aed511 +https://conda.anaconda.org/conda-forge/win-64/fonttools-4.58.1-py310h38315fa_0.conda#76a9c04ac1c23cee8b00733eb942f8e5 +https://conda.anaconda.org/conda-forge/win-64/freetype-2.13.3-h57928b3_1.conda#633504fe3f96031192e40e3e6c18ef06 +https://conda.anaconda.org/conda-forge/noarch/meson-python-0.18.0-pyh70fd9c4_0.conda#576c04b9d9f8e45285fb4d9452c26133 +https://conda.anaconda.org/conda-forge/win-64/mkl-2024.2.2-h66d3029_15.conda#302dff2807f2927b3e9e0d19d60121de +https://conda.anaconda.org/conda-forge/win-64/pillow-11.2.1-py310h9595edc_0.conda#33d0663d469cc146b5fc68587348f450 +https://conda.anaconda.org/conda-forge/noarch/pytest-8.3.5-pyhd8ed1ab_0.conda#c3c9316209dec74a705a36797970c6be +https://conda.anaconda.org/conda-forge/win-64/fontconfig-2.15.0-h765892d_1.conda#9bb0026a2131b09404c59c4290c697cd +https://conda.anaconda.org/conda-forge/win-64/libblas-3.9.0-31_h641d27c_mkl.conda#d05563c577fe2f37693a554b3f271e8f +https://conda.anaconda.org/conda-forge/win-64/mkl-devel-2024.2.2-h57928b3_15.conda#a85f53093da069c7c657f090e388f3ef +https://conda.anaconda.org/conda-forge/noarch/pytest-cov-6.1.1-pyhd8ed1ab_0.conda#1e35d8f975bc0e984a19819aa91c440a +https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.7.0-pyhd8ed1ab_0.conda#15353a2a0ea6dfefaa52fc5ab5b98f41 +https://conda.anaconda.org/conda-forge/win-64/cairo-1.18.4-h5782bbf_0.conda#20e32ced54300292aff690a69c5e7b97 +https://conda.anaconda.org/conda-forge/win-64/libcblas-3.9.0-31_h5e41251_mkl.conda#43c100b94ad2607382b0cf0f3a6b0bf3 +https://conda.anaconda.org/conda-forge/win-64/liblapack-3.9.0-31_h1aa476e_mkl.conda#40b47ee720a185289760960fc6185750 +https://conda.anaconda.org/conda-forge/win-64/harfbuzz-11.2.1-h8796e6f_0.conda#bccea58fbf7910ce868b084f27ffe8bd +https://conda.anaconda.org/conda-forge/win-64/liblapacke-3.9.0-31_h845c4fa_mkl.conda#003a2041cb07a7cf698f48dd26301273 +https://conda.anaconda.org/conda-forge/win-64/numpy-2.2.6-py310h4987827_0.conda#d2596785ac2cf5bab04e2ee9e5d04041 +https://conda.anaconda.org/conda-forge/win-64/blas-devel-3.9.0-31_hfb1a452_mkl.conda#0deeb3d9d6f0e56393c55ef382899010 +https://conda.anaconda.org/conda-forge/win-64/contourpy-1.3.2-py310hc19bc0b_0.conda#039416813b5290e7d100a05bb4326110 +https://conda.anaconda.org/conda-forge/win-64/qt6-main-6.9.0-h02ddd7d_3.conda#8aeebdf27e439648236c3eb856ce7777 +https://conda.anaconda.org/conda-forge/win-64/scipy-1.15.2-py310h15c175c_0.conda#81798168111d1021e3d815217c444418 +https://conda.anaconda.org/conda-forge/win-64/blas-2.131-mkl.conda#1842bfaa4e349875c47bde1d9871bda6 +https://conda.anaconda.org/conda-forge/win-64/matplotlib-base-3.10.3-py310h37e0a56_0.conda#de9ddae6f97b78860c256de480ea1a84 +https://conda.anaconda.org/conda-forge/win-64/pyside6-6.9.0-py310hc1b6536_0.conda#e90c8d8a817b5d63b7785d7d18c99ae0 +https://conda.anaconda.org/conda-forge/win-64/matplotlib-3.10.3-py310h5588dad_0.conda#103adee33db124a0263d0b4551e232e3 diff --git a/build_tools/azure/pymin_conda_forge_openblas_min_dependencies_environment.yml b/build_tools/azure/pymin_conda_forge_openblas_min_dependencies_environment.yml new file mode 100644 index 0000000000000..a179c55fed993 --- /dev/null +++ b/build_tools/azure/pymin_conda_forge_openblas_min_dependencies_environment.yml @@ -0,0 +1,26 @@ +# DO NOT EDIT: this file is generated from the specification found in the +# following script to centralize the configuration for CI builds: +# build_tools/update_environments_and_lock_files.py +channels: + - conda-forge +dependencies: + - python=3.10 + - numpy=1.22.0 # min + - blas[build=openblas] + - scipy=1.8.0 # min + - cython=3.0.10 # min + - joblib=1.2.0 # min + - threadpoolctl=3.1.0 # min + - matplotlib=3.5.0 # min + - pandas=1.4.0 # min + - pyamg=4.2.1 # min + - pytest + - pytest-xdist + - pillow + - pip + - ninja + - meson-python=0.16.0 # min + - pytest-cov + - coverage + - ccache + - polars=0.20.30 # min diff --git a/build_tools/azure/pymin_conda_forge_openblas_min_dependencies_linux-64_conda.lock b/build_tools/azure/pymin_conda_forge_openblas_min_dependencies_linux-64_conda.lock new file mode 100644 index 0000000000000..f55381fb64f3f --- /dev/null +++ b/build_tools/azure/pymin_conda_forge_openblas_min_dependencies_linux-64_conda.lock @@ -0,0 +1,189 @@ +# Generated by conda-lock. +# platform: linux-64 +# input_hash: 41111e5656d9d33f83f1160f643ec4ab314aa8e179923dbe1350c87b0ac2f400 +@EXPLICIT +https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2#0c96522c6bdaed4b1566d11387caaf45 +https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2#34893075a5c9e55cdafac56607368fc6 +https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2#4d59c254e01d9cde7957100457e2d5fb +https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-h77eed37_3.conda#49023d73832ef61042f6a237cb2687e7 +https://conda.anaconda.org/conda-forge/noarch/python_abi-3.10-7_cp310.conda#44e871cba2b162368476a84b8d040b6c +https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda#4222072737ccff51314b5ece9c7d6f5a +https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.4.26-hbd8a1cb_0.conda#95db94f75ba080a22eb623590993167b +https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-0.tar.bz2#f766549260d6815b0c52253f1fb1bb29 +https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.43-h712a8e2_4.conda#01f8d123c96816249efd255a31ad7712 +https://conda.anaconda.org/conda-forge/linux-64/libglvnd-1.7.0-ha4b6fd6_2.conda#434ca7e50e40f4918ab701e3facd59a0 +https://conda.anaconda.org/conda-forge/linux-64/llvm-openmp-20.1.6-h024ca30_0.conda#e4ece7ed81e43ae97a3b58ac4230c3c5 +https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-3_kmp_llvm.conda#ee5c2118262e30b972bc0b4db8ef0ba5 +https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2#fee5683a3f04bd15cbd8318b096a27ab +https://conda.anaconda.org/conda-forge/linux-64/libegl-1.7.0-ha4b6fd6_2.conda#c151d5eb730e9b7480e6d48c0fc44048 +https://conda.anaconda.org/conda-forge/linux-64/libopengl-1.7.0-ha4b6fd6_2.conda#7df50d44d4a14d6c31a2c54f2cd92157 +https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.1.0-h767d61c_2.conda#ea8ac52380885ed41c1baa8f1d6d2b93 +https://conda.anaconda.org/conda-forge/linux-64/alsa-lib-1.2.14-hb9d3cd8_0.conda#76df83c2a9035c54df5d04ff81bcc02d +https://conda.anaconda.org/conda-forge/linux-64/gettext-tools-0.24.1-h5888daf_0.conda#d54305672f0361c2f3886750e7165b5f +https://conda.anaconda.org/conda-forge/linux-64/libbrotlicommon-1.1.0-hb9d3cd8_2.conda#41b599ed2b02abcfdd84302bff174b23 +https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.24-h86f0d12_0.conda#64f0c503da58ec25ebd359e4d990afa8 +https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.7.0-h5888daf_0.conda#db0bfbe7dd197b68ad5f30333bae6ce0 +https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.6-h2dba641_1.conda#ede4673863426c0883c0063d853bbd85 +https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-15.1.0-h69a702a_2.conda#ddca86c7040dd0e73b2b69bd7833d225 +https://conda.anaconda.org/conda-forge/linux-64/libgettextpo-0.24.1-h5888daf_0.conda#2ee6d71b72f75d50581f2f68e965efdb +https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-15.1.0-hcea5267_2.conda#01de444988ed960031dbe84cf4f9b1fc +https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.18-h4ce23a2_1.conda#e796ff8ddc598affdf7c173d6145f087 +https://conda.anaconda.org/conda-forge/linux-64/libjpeg-turbo-3.1.0-hb9d3cd8_0.conda#9fa334557db9f63da6c9285fd2a48638 +https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.1-hb9d3cd8_1.conda#a76fd702c93cd2dfd89eff30a5fd45a8 +https://conda.anaconda.org/conda-forge/linux-64/libntlm-1.8-hb9d3cd8_0.conda#7c7927b404672409d9917d49bff5f2d6 +https://conda.anaconda.org/conda-forge/linux-64/libogg-1.3.5-hd0c01bc_1.conda#68e52064ed3897463c0e958ab5c8f91b +https://conda.anaconda.org/conda-forge/linux-64/libopus-1.5.2-hd0c01bc_0.conda#b64523fb87ac6f87f0790f324ad43046 +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-15.1.0-h8f9b012_2.conda#1cb1c67961f6dd257eae9e9691b341aa +https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.5.0-h851e524_0.conda#63f790534398730f59e1b899c3644d4a +https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda#edb0dca6bc32e4f4789199455a1dbeb8 +https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda#47e340acb35de30501a76c7c799c41d7 +https://conda.anaconda.org/conda-forge/linux-64/openssl-3.5.0-h7b32b05_1.conda#de356753cfdbffcde5bb1e86e3aa6cd0 +https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-hb9d3cd8_1002.conda#b3c17d95b5a10c6e64a21fa17573e70e +https://conda.anaconda.org/conda-forge/linux-64/xorg-libice-1.1.2-hb9d3cd8_0.conda#fb901ff28063514abb6046c9ec2c4a45 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxau-1.0.12-hb9d3cd8_0.conda#f6ebe2cb3f82ba6c057dde5d9debe4f7 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdmcp-1.1.5-hb9d3cd8_0.conda#8035c64cb77ed555e3f150b7b3972480 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxshmfence-1.3.3-hb9d3cd8_0.conda#9a809ce9f65460195777f2f2116bae02 +https://conda.anaconda.org/conda-forge/linux-64/attr-2.5.1-h166bdaf_1.tar.bz2#d9c69a24ad678ffce24c6543a0176b00 +https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h4bc722e_7.conda#62ee74e96c5ebb0af99386de58cf9553 +https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.1-h166bdaf_0.tar.bz2#30186d27e2c9fa62b45fb1476b7200e3 +https://conda.anaconda.org/conda-forge/linux-64/lame-3.100-h166bdaf_1003.tar.bz2#a8832b479f93521a9e7b5b743803be51 +https://conda.anaconda.org/conda-forge/linux-64/lerc-4.0.0-h0aef613_1.conda#9344155d33912347b37f0ae6c410a835 +https://conda.anaconda.org/conda-forge/linux-64/libasprintf-0.24.1-h8e693c7_0.conda#57566a81dd1e5aa3d98ac7582e8bfe03 +https://conda.anaconda.org/conda-forge/linux-64/libbrotlidec-1.1.0-hb9d3cd8_2.conda#9566f0bd264fbd463002e759b8a82401 +https://conda.anaconda.org/conda-forge/linux-64/libbrotlienc-1.1.0-hb9d3cd8_2.conda#06f70867945ea6a84d35836af780f1de +https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20250104-pl5321h7949ede_0.conda#c277e0a4d549b03ac1e9d6cbbe3d017b +https://conda.anaconda.org/conda-forge/linux-64/libevent-2.1.12-hf998b51_1.conda#a1cfcc585f0c42bf8d5546bb1dfb668d +https://conda.anaconda.org/conda-forge/linux-64/libgettextpo-devel-0.24.1-h5888daf_0.conda#8f04c7aae6a46503bc36d1ed5abc8c7c +https://conda.anaconda.org/conda-forge/linux-64/libgfortran-15.1.0-h69a702a_2.conda#f92e6e0a3c0c0c85561ef61aa59d555d +https://conda.anaconda.org/conda-forge/linux-64/libgpg-error-1.55-h3f2d84a_0.conda#2bd47db5807daade8500ed7ca4c512a4 +https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hd590300_0.conda#30fd6e37fe21f86f4bd26d6ee73eeec7 +https://conda.anaconda.org/conda-forge/linux-64/libpciaccess-0.18-hd590300_0.conda#48f4330bfcd959c3cfb704d424903c82 +https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.47-h943b412_0.conda#55199e2ae2c3651f6f9b2a447b47bdc9 +https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.50.0-hee588c1_0.conda#71888e92098d0f8c41b09a671ad289bc +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-15.1.0-h4852527_2.conda#9d2072af184b5caa29492bf2344597bb +https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda#40b61aab5c7ba9ff276c41cfffe6b80b +https://conda.anaconda.org/conda-forge/linux-64/libxcb-1.17.0-h8a09558_0.conda#92ed62436b625154323d40d5f2f11dd7 +https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda#5aa797f8787fe7a17d1b0821485b5adc +https://conda.anaconda.org/conda-forge/linux-64/lz4-c-1.10.0-h5888daf_1.conda#9de5350a85c4a20c685259b889aa6393 +https://conda.anaconda.org/conda-forge/linux-64/mpg123-1.32.9-hc50e24c_0.conda#c7f302fd11eeb0987a6a5e1f3aed6a21 +https://conda.anaconda.org/conda-forge/linux-64/ninja-1.12.1-hff21bea_1.conda#2322531904f27501ee19847b87ba7c64 +https://conda.anaconda.org/conda-forge/linux-64/nspr-4.36-h5888daf_0.conda#de9cd5bca9e4918527b9b72b6e2e1409 +https://conda.anaconda.org/conda-forge/linux-64/pixman-0.46.0-h29eaf8c_0.conda#d2f1c87d4416d1e7344cf92b1aaee1c4 +https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8c095d6_2.conda#283b96675859b20a825f8fa30f311446 +https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_hd72426e_102.conda#a0116df4f4ed05c303811a837d5b39d8 +https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb8e6e7a_2.conda#6432cb5d4ac0046c3ac0a8a0f95842f9 +https://conda.anaconda.org/conda-forge/linux-64/brotli-bin-1.1.0-hb9d3cd8_2.conda#c63b5e52939e795ba8d26e35d767a843 +https://conda.anaconda.org/conda-forge/linux-64/graphite2-1.3.13-h59595ed_1003.conda#f87c7b7c2cb45f323ffbce941c78ab7c +https://conda.anaconda.org/conda-forge/linux-64/icu-75.1-he02047a_0.conda#8b189310083baabfb622af68fd9d3ae3 +https://conda.anaconda.org/conda-forge/linux-64/krb5-1.21.3-h659f571_0.conda#3f43953b7d3fb3aaa1d0d0723d91e368 +https://conda.anaconda.org/conda-forge/linux-64/libasprintf-devel-0.24.1-h8e693c7_0.conda#8f66ed2e34507b7ae44afa31c3e4ec79 +https://conda.anaconda.org/conda-forge/linux-64/libcap-2.75-h39aace5_0.conda#c44c16d6976d2aebbd65894d7741e67e +https://conda.anaconda.org/conda-forge/linux-64/libdrm-2.4.124-hb9d3cd8_0.conda#8bc89311041d7fcb510238cf0848ccae +https://conda.anaconda.org/conda-forge/linux-64/libfreetype6-2.13.3-h48d6fc4_1.conda#3c255be50a506c50765a93a6644f32fe +https://conda.anaconda.org/conda-forge/linux-64/libgcrypt-lib-1.11.1-hb9d3cd8_0.conda#8504a291085c9fb809b66cabd5834307 +https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-15.1.0-h69a702a_2.conda#a483a87b71e974bb75d1b9413d4436dd +https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.7.0-hf01ce69_5.conda#e79a094918988bb1807462cd42c83962 +https://conda.anaconda.org/conda-forge/linux-64/libvorbis-1.3.7-h9c3ff4c_0.tar.bz2#309dec04b70a3cc0f1e84a4013683bc0 +https://conda.anaconda.org/conda-forge/linux-64/nss-3.112-h159eef7_0.conda#688a8bc02e57e6b741a040c84e931a7d +https://conda.anaconda.org/conda-forge/linux-64/pcre2-10.45-hc749103_0.conda#b90bece58b4c2bf25969b70f3be42d25 +https://conda.anaconda.org/conda-forge/linux-64/python-3.10.17-hd6af730_0_cpython.conda#7bb89638dae9ce1b8e051d0b721e83c2 +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-0.4.1-hb711507_2.conda#8637c3e5821654d0edf97e2b0404b443 +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-keysyms-0.4.1-hb711507_0.conda#ad748ccca349aec3e91743e08b5e2b50 +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-renderutil-0.3.10-hb711507_0.conda#0e0cbe0564d03a99afd5fd7b362feecd +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-wm-0.4.2-hb711507_0.conda#608e0ef8256b81d04456e8d211eee3e8 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.6-he73a12e_0.conda#1c74ff8c35dcadf952a16f752ca5aa49 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.8.12-h4f16b4b_0.conda#db038ce880f100acc74dba10302b5630 +https://conda.anaconda.org/conda-forge/linux-64/brotli-1.1.0-hb9d3cd8_2.conda#98514fe74548d768907ce7a13f680e8f +https://conda.anaconda.org/conda-forge/noarch/certifi-2025.4.26-pyhd8ed1ab_0.conda#c33eeaaa33f45031be34cda513df39b6 +https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda#962b9857ee8e7018c22f2776ffa0b2d7 +https://conda.anaconda.org/conda-forge/noarch/cycler-0.12.1-pyhd8ed1ab_1.conda#44600c4667a319d67dbe0681fc0bc833 +https://conda.anaconda.org/conda-forge/linux-64/cyrus-sasl-2.1.27-h54b06d7_7.conda#dce22f70b4e5a407ce88f2be046f4ceb +https://conda.anaconda.org/conda-forge/linux-64/cython-3.0.10-py310hc6cd4ac_0.conda#bd1d71ee240be36f1d85c86177d6964f +https://conda.anaconda.org/conda-forge/noarch/execnet-2.1.1-pyhd8ed1ab_1.conda#a71efeae2c160f6789900ba2631a2c90 +https://conda.anaconda.org/conda-forge/linux-64/gettext-0.24.1-h5888daf_0.conda#c63e7590d4d6f4c85721040ed8b12888 +https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_1.conda#6837f3eff7dcea42ecd714ce1ac2b108 +https://conda.anaconda.org/conda-forge/linux-64/kiwisolver-1.4.7-py310h3788b33_0.conda#4186d9b4d004b0fe0de6aa62496fb48a +https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.17-h717163a_0.conda#000e85703f0fd9594c81710dd5066471 +https://conda.anaconda.org/conda-forge/linux-64/libcups-2.3.3-h4637d8d_4.conda#d4529f4dff3057982a7617c7ac58fde3 +https://conda.anaconda.org/conda-forge/linux-64/libfreetype-2.13.3-ha770c72_1.conda#51f5be229d83ecd401fb369ab96ae669 +https://conda.anaconda.org/conda-forge/linux-64/libglib-2.84.2-h3618099_0.conda#072ab14a02164b7c0c089055368ff776 +https://conda.anaconda.org/conda-forge/linux-64/libglx-1.7.0-ha4b6fd6_2.conda#c8013e438185f33b13814c5c488acd5c +https://conda.anaconda.org/conda-forge/linux-64/libhiredis-1.0.2-h2cc385e_0.tar.bz2#b34907d3a81a3cd8095ee83d174c074a +https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.25-pthreads_h413a1c8_0.conda#d172b34a443b95f86089e8229ddc9a17 +https://conda.anaconda.org/conda-forge/linux-64/libsystemd0-257.4-h4e0b6ca_1.conda#04bcf3055e51f8dde6fab9672fb9fca0 +https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.13.8-h4bc477f_0.conda#14dbe05b929e329dbaa6f2d0aa19466d +https://conda.anaconda.org/conda-forge/noarch/meson-1.8.1-pyhe01879c_0.conda#f3cccd9a6ce5331ae33f69ade5529162 +https://conda.anaconda.org/conda-forge/noarch/munkres-1.1.4-pyh9f0ad1d_0.tar.bz2#2ba8498c1018c1e9c61eb99b973dfe19 +https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.3-h5fbd93e_0.conda#9e5816bc95d285c115a3ebc2f8563564 +https://conda.anaconda.org/conda-forge/noarch/packaging-25.0-pyh29332c3_1.conda#58335b26c38bf4a20f399384c33cbcf9 +https://conda.anaconda.org/conda-forge/noarch/pluggy-1.6.0-pyhd8ed1ab_0.conda#7da7ccd349dbf6487a7778579d2bb971 +https://conda.anaconda.org/conda-forge/noarch/ply-3.11-pyhd8ed1ab_3.conda#fd5062942bfa1b0bd5e0d2a4397b099e +https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.2.3-pyhd8ed1ab_1.conda#513d3c262ee49b54a8fec85c5bc99764 +https://conda.anaconda.org/conda-forge/noarch/pytz-2025.2-pyhd8ed1ab_0.conda#bc8e3267d44011051f2eb14d22fb0960 +https://conda.anaconda.org/conda-forge/noarch/setuptools-80.9.0-pyhff2d567_0.conda#4de79c071274a53dcaf2a8c749d1499e +https://conda.anaconda.org/conda-forge/noarch/six-1.17.0-pyhd8ed1ab_0.conda#a451d576819089b0d672f18768be0f65 +https://conda.anaconda.org/conda-forge/noarch/threadpoolctl-3.1.0-pyh8a188c0_0.tar.bz2#a2995ee828f65687ac5b1e71a2ab1e0c +https://conda.anaconda.org/conda-forge/noarch/toml-0.10.2-pyhd8ed1ab_1.conda#b0dd904de08b7db706167240bf37b164 +https://conda.anaconda.org/conda-forge/noarch/tomli-2.2.1-pyhd8ed1ab_1.conda#ac944244f1fed2eb49bae07193ae8215 +https://conda.anaconda.org/conda-forge/linux-64/tornado-6.5.1-py310ha75aee5_0.conda#6f3da1072c0c4d2a1beb1e84615f7c9c +https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.13.2-pyh29332c3_0.conda#83fc6ae00127671e301c9f44254c31b8 +https://conda.anaconda.org/conda-forge/linux-64/unicodedata2-16.0.0-py310ha75aee5_0.conda#1d7a4b9202cdd10d56ecdd7f6c347190 +https://conda.anaconda.org/conda-forge/noarch/wheel-0.45.1-pyhd8ed1ab_1.conda#75cb7132eb58d97896e173ef12ac9986 +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-image-0.4.0-hb711507_2.conda#a0901183f08b6c7107aab109733a3c91 +https://conda.anaconda.org/conda-forge/linux-64/xkeyboard-config-2.44-hb9d3cd8_0.conda#7c91bfc90672888259675ad2ad28af9c +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.6-hb9d3cd8_0.conda#febbab7d15033c913d53c7a2c102309d +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxfixes-6.0.1-hb9d3cd8_0.conda#4bdb303603e9821baf5fe5fdff1dc8f8 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.12-hb9d3cd8_0.conda#96d57aba173e878a2089d5638016dc5e +https://conda.anaconda.org/conda-forge/linux-64/ccache-4.11.3-h80c52d3_0.conda#eb517c6a2b960c3ccb6f1db1005f063a +https://conda.anaconda.org/conda-forge/linux-64/coverage-7.8.2-py310h89163eb_0.conda#5ca8ab35287adc83b2d1996e5c2ac14c +https://conda.anaconda.org/conda-forge/linux-64/dbus-1.16.2-h3c4dab8_0.conda#679616eb5ad4e521c83da4650860aba7 +https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.0-pyhd8ed1ab_0.conda#72e42d28960d875c7654614f8b50939a +https://conda.anaconda.org/conda-forge/linux-64/fonttools-4.58.1-py310h89163eb_0.conda#f4f46207c6defa5ea17b0299298ba849 +https://conda.anaconda.org/conda-forge/linux-64/freetype-2.13.3-ha770c72_1.conda#9ccd736d31e0c6e41f54e704e5312811 +https://conda.anaconda.org/conda-forge/linux-64/glib-tools-2.84.2-h4833e2c_0.conda#f2ec1facec64147850b7674633978050 +https://conda.anaconda.org/conda-forge/noarch/joblib-1.2.0-pyhd8ed1ab_0.tar.bz2#7583652522d71ad78ba536bba06940eb +https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-20_linux64_openblas.conda#2b7bb4f7562c8cf334fc2e20c2d28abc +https://conda.anaconda.org/conda-forge/linux-64/libflac-1.4.3-h59595ed_0.conda#ee48bf17cc83a00f59ca1494d5646869 +https://conda.anaconda.org/conda-forge/linux-64/libgl-1.7.0-ha4b6fd6_2.conda#928b8be80851f5d8ffb016f9c81dae7a +https://conda.anaconda.org/conda-forge/linux-64/libllvm20-20.1.6-he9d0ab4_0.conda#bf8ccdd2c1c1a54a3fa25bb61f26460e +https://conda.anaconda.org/conda-forge/linux-64/libxkbcommon-1.10.0-h65c71a3_0.conda#fedf6bfe5d21d21d2b1785ec00a8889a +https://conda.anaconda.org/conda-forge/linux-64/openblas-0.3.25-pthreads_h7a3da1a_0.conda#87661673941b5e702275fdf0fc095ad0 +https://conda.anaconda.org/conda-forge/linux-64/openldap-2.6.10-he970967_0.conda#2e5bf4f1da39c0b32778561c3c4e5878 +https://conda.anaconda.org/conda-forge/linux-64/pillow-11.2.1-py310h7e6dc6c_0.conda#5645a243d90adb50909b9edc209d84fe +https://conda.anaconda.org/conda-forge/noarch/pip-25.1.1-pyh8b19718_0.conda#32d0781ace05105cc99af55d36cbec7c +https://conda.anaconda.org/conda-forge/noarch/pyproject-metadata-0.9.1-pyhd8ed1ab_0.conda#22ae7c6ea81e0c8661ef32168dda929b +https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhff2d567_1.conda#5ba79d7c71f03c678c8ead841f347d6e +https://conda.anaconda.org/conda-forge/linux-64/sip-6.10.0-py310hf71b8c6_0.conda#2d7e4445be227e8210140b75725689ad +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxcomposite-0.4.6-hb9d3cd8_2.conda#d3c295b50f092ab525ffe3c2aa4b7413 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdamage-1.1.6-hb9d3cd8_0.conda#b5fcc7172d22516e1f965490e65e33a4 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxxf86vm-1.1.6-hb9d3cd8_0.conda#5efa5fa6243a622445fdfd72aee15efa +https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.15.0-h7e30c49_1.conda#8f5b0b297b59e1ac160ad4beec99dbee +https://conda.anaconda.org/conda-forge/linux-64/glib-2.84.2-h6287aef_0.conda#704648df3a01d4d24bc2c0466b718d63 +https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-20_linux64_openblas.conda#36d486d72ab64ffea932329a1d3729a3 +https://conda.anaconda.org/conda-forge/linux-64/libclang-cpp20.1-20.1.6-default_h1df26ce_0.conda#99ead3b974685e44df8b1e3953503cfc +https://conda.anaconda.org/conda-forge/linux-64/libclang13-20.1.6-default_he06ed0a_0.conda#cc6c469d9d7fc0ac106cef5f45d973a9 +https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-20_linux64_openblas.conda#6fabc51f5e647d09cc010c40061557e0 +https://conda.anaconda.org/conda-forge/linux-64/libpq-17.5-h27ae623_0.conda#6458be24f09e1b034902ab44fe9de908 +https://conda.anaconda.org/conda-forge/linux-64/libsndfile-1.2.2-hc60ed4a_1.conda#ef1910918dd895516a769ed36b5b3a4e +https://conda.anaconda.org/conda-forge/noarch/meson-python-0.16.0-pyh0c530f3_0.conda#e16f0dbf502da873be9f9adb0dc52547 +https://conda.anaconda.org/conda-forge/linux-64/pyqt5-sip-12.17.0-py310hf71b8c6_0.conda#012465861673a67a30bc8ca6284074f3 +https://conda.anaconda.org/conda-forge/noarch/pytest-8.3.5-pyhd8ed1ab_0.conda#c3c9316209dec74a705a36797970c6be +https://conda.anaconda.org/conda-forge/linux-64/cairo-1.18.4-h3394656_0.conda#09262e66b19567aff4f592fb53b28760 +https://conda.anaconda.org/conda-forge/linux-64/gstreamer-1.24.11-hc37bda9_0.conda#056d86cacf2b48c79c6a562a2486eb8c +https://conda.anaconda.org/conda-forge/linux-64/liblapacke-3.9.0-20_linux64_openblas.conda#05c5862c7dc25e65ba6c471d96429dae +https://conda.anaconda.org/conda-forge/linux-64/numpy-1.22.0-py310h454958d_1.tar.bz2#607c66f0cce2986515a8fe9e136b2b57 +https://conda.anaconda.org/conda-forge/linux-64/pulseaudio-client-17.0-hac146a9_1.conda#66b1fa9608d8836e25f9919159adc9c6 +https://conda.anaconda.org/conda-forge/noarch/pytest-cov-6.1.1-pyhd8ed1ab_0.conda#1e35d8f975bc0e984a19819aa91c440a +https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.7.0-pyhd8ed1ab_0.conda#15353a2a0ea6dfefaa52fc5ab5b98f41 +https://conda.anaconda.org/conda-forge/linux-64/blas-devel-3.9.0-20_linux64_openblas.conda#9932a1d4e9ecf2d35fb19475446e361e +https://conda.anaconda.org/conda-forge/linux-64/gst-plugins-base-1.24.11-h651a532_0.conda#d8d8894f8ced2c9be76dc9ad1ae531ce +https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-11.2.1-h3beb420_0.conda#0e6e192d4b3d95708ad192d957cf3163 +https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.5.0-py310h23f4a51_0.tar.bz2#9911225650b298776c8e8c083b5cacf1 +https://conda.anaconda.org/conda-forge/linux-64/pandas-1.4.0-py310hb5077e9_0.tar.bz2#43e920bc9856daa7d8d18fcbfb244c4e +https://conda.anaconda.org/conda-forge/linux-64/polars-0.20.30-py310h031f9ce_0.conda#0743f5db9f978b6df92d412935ff8371 +https://conda.anaconda.org/conda-forge/linux-64/scipy-1.8.0-py310hea5193d_1.tar.bz2#664d80ddeb51241629b3ada5ea926e4d +https://conda.anaconda.org/conda-forge/linux-64/blas-2.120-openblas.conda#c8f6916a81a340650078171b1d852574 +https://conda.anaconda.org/conda-forge/linux-64/pyamg-4.2.1-py310h7c3ba0c_0.tar.bz2#89f5a48e1f23b5cf3163a6094903d181 +https://conda.anaconda.org/conda-forge/linux-64/qt-main-5.15.15-hea1682b_4.conda#c054d7f22cc719e12c72d454b2328d6c +https://conda.anaconda.org/conda-forge/linux-64/pyqt-5.15.11-py310hf392a12_0.conda#65924d3e57be25342c76530d23d75f0f +https://conda.anaconda.org/conda-forge/linux-64/matplotlib-3.5.0-py310hff52083_0.tar.bz2#1b2f3b135d5d9c594b5e0e6150c03b7b diff --git a/build_tools/azure/pymin_conda_forge_openblas_ubuntu_2204_environment.yml b/build_tools/azure/pymin_conda_forge_openblas_ubuntu_2204_environment.yml new file mode 100644 index 0000000000000..267c149fd1c35 --- /dev/null +++ b/build_tools/azure/pymin_conda_forge_openblas_ubuntu_2204_environment.yml @@ -0,0 +1,24 @@ +# DO NOT EDIT: this file is generated from the specification found in the +# following script to centralize the configuration for CI builds: +# build_tools/update_environments_and_lock_files.py +channels: + - conda-forge +dependencies: + - python=3.10 + - numpy + - blas[build=openblas] + - scipy + - cython + - joblib + - threadpoolctl + - pandas + - pyamg + - pytest + - pytest-xdist + - pillow + - pip + - ninja + - meson-python + - sphinx + - numpydoc + - ccache diff --git a/build_tools/azure/pymin_conda_forge_openblas_ubuntu_2204_linux-64_conda.lock b/build_tools/azure/pymin_conda_forge_openblas_ubuntu_2204_linux-64_conda.lock new file mode 100644 index 0000000000000..08a8597ed4fae --- /dev/null +++ b/build_tools/azure/pymin_conda_forge_openblas_ubuntu_2204_linux-64_conda.lock @@ -0,0 +1,116 @@ +# Generated by conda-lock. +# platform: linux-64 +# input_hash: 26bb2530999c20f24bbab0f7b6e3545ad84d059a25027cb624997210afc23693 +@EXPLICIT +https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81 +https://conda.anaconda.org/conda-forge/noarch/python_abi-3.10-7_cp310.conda#44e871cba2b162368476a84b8d040b6c +https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda#4222072737ccff51314b5ece9c7d6f5a +https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.4.26-hbd8a1cb_0.conda#95db94f75ba080a22eb623590993167b +https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.43-h712a8e2_4.conda#01f8d123c96816249efd255a31ad7712 +https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.1.0-h767d61c_2.conda#fbe7d535ff9d3a168c148e07358cd5b1 +https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2#73aaf86a425cc6e73fcf236a5a46396d +https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.1.0-h767d61c_2.conda#ea8ac52380885ed41c1baa8f1d6d2b93 +https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.24-h86f0d12_0.conda#64f0c503da58ec25ebd359e4d990afa8 +https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.7.0-h5888daf_0.conda#db0bfbe7dd197b68ad5f30333bae6ce0 +https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.6-h2dba641_1.conda#ede4673863426c0883c0063d853bbd85 +https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-15.1.0-h69a702a_2.conda#ddca86c7040dd0e73b2b69bd7833d225 +https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-15.1.0-hcea5267_2.conda#01de444988ed960031dbe84cf4f9b1fc +https://conda.anaconda.org/conda-forge/linux-64/libjpeg-turbo-3.1.0-hb9d3cd8_0.conda#9fa334557db9f63da6c9285fd2a48638 +https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.1-hb9d3cd8_1.conda#a76fd702c93cd2dfd89eff30a5fd45a8 +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-15.1.0-h8f9b012_2.conda#1cb1c67961f6dd257eae9e9691b341aa +https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.5.0-h851e524_0.conda#63f790534398730f59e1b899c3644d4a +https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda#edb0dca6bc32e4f4789199455a1dbeb8 +https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda#47e340acb35de30501a76c7c799c41d7 +https://conda.anaconda.org/conda-forge/linux-64/openssl-3.5.0-h7b32b05_1.conda#de356753cfdbffcde5bb1e86e3aa6cd0 +https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-hb9d3cd8_1002.conda#b3c17d95b5a10c6e64a21fa17573e70e +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxau-1.0.12-hb9d3cd8_0.conda#f6ebe2cb3f82ba6c057dde5d9debe4f7 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdmcp-1.1.5-hb9d3cd8_0.conda#8035c64cb77ed555e3f150b7b3972480 +https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h4bc722e_7.conda#62ee74e96c5ebb0af99386de58cf9553 +https://conda.anaconda.org/conda-forge/linux-64/lerc-4.0.0-h0aef613_1.conda#9344155d33912347b37f0ae6c410a835 +https://conda.anaconda.org/conda-forge/linux-64/libgfortran-15.1.0-h69a702a_2.conda#f92e6e0a3c0c0c85561ef61aa59d555d +https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hd590300_0.conda#30fd6e37fe21f86f4bd26d6ee73eeec7 +https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.47-h943b412_0.conda#55199e2ae2c3651f6f9b2a447b47bdc9 +https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.50.0-hee588c1_0.conda#71888e92098d0f8c41b09a671ad289bc +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-15.1.0-h4852527_2.conda#9d2072af184b5caa29492bf2344597bb +https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda#40b61aab5c7ba9ff276c41cfffe6b80b +https://conda.anaconda.org/conda-forge/linux-64/libxcb-1.17.0-h8a09558_0.conda#92ed62436b625154323d40d5f2f11dd7 +https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda#5aa797f8787fe7a17d1b0821485b5adc +https://conda.anaconda.org/conda-forge/linux-64/ninja-1.12.1-hff21bea_1.conda#2322531904f27501ee19847b87ba7c64 +https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8c095d6_2.conda#283b96675859b20a825f8fa30f311446 +https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_hd72426e_102.conda#a0116df4f4ed05c303811a837d5b39d8 +https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb8e6e7a_2.conda#6432cb5d4ac0046c3ac0a8a0f95842f9 +https://conda.anaconda.org/conda-forge/linux-64/libfreetype6-2.13.3-h48d6fc4_1.conda#3c255be50a506c50765a93a6644f32fe +https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-15.1.0-h69a702a_2.conda#a483a87b71e974bb75d1b9413d4436dd +https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.29-pthreads_h94d23a6_0.conda#0a4d0252248ef9a0f88f2ba8b8a08e12 +https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.7.0-hf01ce69_5.conda#e79a094918988bb1807462cd42c83962 +https://conda.anaconda.org/conda-forge/linux-64/python-3.10.17-hd6af730_0_cpython.conda#7bb89638dae9ce1b8e051d0b721e83c2 +https://conda.anaconda.org/conda-forge/noarch/alabaster-1.0.0-pyhd8ed1ab_1.conda#1fd9696649f65fd6611fcdb4ffec738a +https://conda.anaconda.org/conda-forge/linux-64/brotli-python-1.1.0-py310hf71b8c6_2.conda#bf502c169c71e3c6ac0d6175addfacc2 +https://conda.anaconda.org/conda-forge/noarch/certifi-2025.4.26-pyhd8ed1ab_0.conda#c33eeaaa33f45031be34cda513df39b6 +https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.4.2-pyhd8ed1ab_0.conda#40fe4284b8b5835a9073a645139f35af +https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda#962b9857ee8e7018c22f2776ffa0b2d7 +https://conda.anaconda.org/conda-forge/linux-64/cython-3.1.1-py310had8cdd9_1.conda#4904cb1ba6e72940ff22a5235554532d +https://conda.anaconda.org/conda-forge/noarch/docutils-0.21.2-pyhd8ed1ab_1.conda#24c1ca34138ee57de72a943237cde4cc +https://conda.anaconda.org/conda-forge/noarch/execnet-2.1.1-pyhd8ed1ab_1.conda#a71efeae2c160f6789900ba2631a2c90 +https://conda.anaconda.org/conda-forge/noarch/hpack-4.1.0-pyhd8ed1ab_0.conda#0a802cb9888dd14eeefc611f05c40b6e +https://conda.anaconda.org/conda-forge/noarch/hyperframe-6.1.0-pyhd8ed1ab_0.conda#8e6923fc12f1fe8f8c4e5c9f343256ac +https://conda.anaconda.org/conda-forge/noarch/idna-3.10-pyhd8ed1ab_1.conda#39a4f67be3286c86d696df570b1201b7 +https://conda.anaconda.org/conda-forge/noarch/imagesize-1.4.1-pyhd8ed1ab_0.tar.bz2#7de5386c8fea29e76b303f37dde4c352 +https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_1.conda#6837f3eff7dcea42ecd714ce1ac2b108 +https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.17-h717163a_0.conda#000e85703f0fd9594c81710dd5066471 +https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-31_h59b9bed_openblas.conda#728dbebd0f7a20337218beacffd37916 +https://conda.anaconda.org/conda-forge/linux-64/libfreetype-2.13.3-ha770c72_1.conda#51f5be229d83ecd401fb369ab96ae669 +https://conda.anaconda.org/conda-forge/linux-64/libhiredis-1.0.2-h2cc385e_0.tar.bz2#b34907d3a81a3cd8095ee83d174c074a +https://conda.anaconda.org/conda-forge/linux-64/markupsafe-3.0.2-py310h89163eb_1.conda#8ce3f0332fd6de0d737e2911d329523f +https://conda.anaconda.org/conda-forge/noarch/meson-1.8.1-pyhe01879c_0.conda#f3cccd9a6ce5331ae33f69ade5529162 +https://conda.anaconda.org/conda-forge/linux-64/openblas-0.3.29-pthreads_h6ec200e_0.conda#7e4d48870b3258bea920d51b7f495a81 +https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.3-h5fbd93e_0.conda#9e5816bc95d285c115a3ebc2f8563564 +https://conda.anaconda.org/conda-forge/noarch/packaging-25.0-pyh29332c3_1.conda#58335b26c38bf4a20f399384c33cbcf9 +https://conda.anaconda.org/conda-forge/noarch/pluggy-1.6.0-pyhd8ed1ab_0.conda#7da7ccd349dbf6487a7778579d2bb971 +https://conda.anaconda.org/conda-forge/noarch/pycparser-2.22-pyh29332c3_1.conda#12c566707c80111f9799308d9e265aef +https://conda.anaconda.org/conda-forge/noarch/pygments-2.19.1-pyhd8ed1ab_0.conda#232fb4577b6687b2d503ef8e254270c9 +https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha55dd90_7.conda#461219d1a5bd61342293efa2c0c90eac +https://conda.anaconda.org/conda-forge/noarch/python-tzdata-2025.2-pyhd8ed1ab_0.conda#88476ae6ebd24f39261e0854ac244f33 +https://conda.anaconda.org/conda-forge/noarch/pytz-2025.2-pyhd8ed1ab_0.conda#bc8e3267d44011051f2eb14d22fb0960 +https://conda.anaconda.org/conda-forge/noarch/setuptools-80.9.0-pyhff2d567_0.conda#4de79c071274a53dcaf2a8c749d1499e +https://conda.anaconda.org/conda-forge/noarch/six-1.17.0-pyhd8ed1ab_0.conda#a451d576819089b0d672f18768be0f65 +https://conda.anaconda.org/conda-forge/noarch/snowballstemmer-3.0.1-pyhd8ed1ab_0.conda#755cf22df8693aa0d1aec1c123fa5863 +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-jsmath-1.0.1-pyhd8ed1ab_1.conda#fa839b5ff59e192f411ccc7dae6588bb +https://conda.anaconda.org/conda-forge/noarch/tabulate-0.9.0-pyhd8ed1ab_2.conda#959484a66b4b76befcddc4fa97c95567 +https://conda.anaconda.org/conda-forge/noarch/threadpoolctl-3.6.0-pyhecae5ae_0.conda#9d64911b31d57ca443e9f1e36b04385f +https://conda.anaconda.org/conda-forge/noarch/tomli-2.2.1-pyhd8ed1ab_1.conda#ac944244f1fed2eb49bae07193ae8215 +https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.13.2-pyh29332c3_0.conda#83fc6ae00127671e301c9f44254c31b8 +https://conda.anaconda.org/conda-forge/noarch/wheel-0.45.1-pyhd8ed1ab_1.conda#75cb7132eb58d97896e173ef12ac9986 +https://conda.anaconda.org/conda-forge/noarch/babel-2.17.0-pyhd8ed1ab_0.conda#0a01c169f0ab0f91b26e77a3301fbfe4 +https://conda.anaconda.org/conda-forge/linux-64/ccache-4.11.3-h80c52d3_0.conda#eb517c6a2b960c3ccb6f1db1005f063a +https://conda.anaconda.org/conda-forge/linux-64/cffi-1.17.1-py310h8deb56e_0.conda#1fc24a3196ad5ede2a68148be61894f4 +https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.0-pyhd8ed1ab_0.conda#72e42d28960d875c7654614f8b50939a +https://conda.anaconda.org/conda-forge/noarch/h2-4.2.0-pyhd8ed1ab_0.conda#b4754fb1bdcb70c8fd54f918301582c6 +https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.6-pyhd8ed1ab_0.conda#446bd6c8cb26050d528881df495ce646 +https://conda.anaconda.org/conda-forge/noarch/joblib-1.5.1-pyhd8ed1ab_0.conda#fb1c14694de51a476ce8636d92b6f42c +https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-31_he106b2a_openblas.conda#abb32c727da370c481a1c206f5159ce9 +https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-31_h7ac8fdf_openblas.conda#452b98eafe050ecff932f0ec832dd03f +https://conda.anaconda.org/conda-forge/linux-64/pillow-11.2.1-py310h7e6dc6c_0.conda#5645a243d90adb50909b9edc209d84fe +https://conda.anaconda.org/conda-forge/noarch/pip-25.1.1-pyh8b19718_0.conda#32d0781ace05105cc99af55d36cbec7c +https://conda.anaconda.org/conda-forge/noarch/pyproject-metadata-0.9.1-pyhd8ed1ab_0.conda#22ae7c6ea81e0c8661ef32168dda929b +https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhff2d567_1.conda#5ba79d7c71f03c678c8ead841f347d6e +https://conda.anaconda.org/conda-forge/linux-64/liblapacke-3.9.0-31_he2f377e_openblas.conda#7e5fff7d0db69be3a266f7e79a3bb0e2 +https://conda.anaconda.org/conda-forge/noarch/meson-python-0.18.0-pyh70fd9c4_0.conda#576c04b9d9f8e45285fb4d9452c26133 +https://conda.anaconda.org/conda-forge/linux-64/numpy-2.2.6-py310hefbff90_0.conda#b0cea2c364bf65cd19e023040eeab05d +https://conda.anaconda.org/conda-forge/noarch/pytest-8.3.5-pyhd8ed1ab_0.conda#c3c9316209dec74a705a36797970c6be +https://conda.anaconda.org/conda-forge/linux-64/zstandard-0.23.0-py310ha75aee5_2.conda#f9254b5b0193982416b91edcb4b2676f +https://conda.anaconda.org/conda-forge/linux-64/blas-devel-3.9.0-31_h1ea3ea9_openblas.conda#ba652ee0576396d4765e567f043c57f9 +https://conda.anaconda.org/conda-forge/linux-64/pandas-2.2.3-py310h5eaa309_3.conda#07697a584fab513ce895c4511f7a2403 +https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.7.0-pyhd8ed1ab_0.conda#15353a2a0ea6dfefaa52fc5ab5b98f41 +https://conda.anaconda.org/conda-forge/linux-64/scipy-1.15.2-py310h1d65ade_0.conda#8c29cd33b64b2eb78597fa28b5595c8d +https://conda.anaconda.org/conda-forge/noarch/urllib3-2.4.0-pyhd8ed1ab_0.conda#c1e349028e0052c4eea844e94f773065 +https://conda.anaconda.org/conda-forge/linux-64/blas-2.131-openblas.conda#38b2ec894c69bb4be0e66d2ef7fc60bf +https://conda.anaconda.org/conda-forge/linux-64/pyamg-5.2.1-py310ha2bacc8_1.conda#817d32861729e14f474249f1036291c4 +https://conda.anaconda.org/conda-forge/noarch/requests-2.32.3-pyhd8ed1ab_1.conda#a9b9368f3701a417eac9edbcae7cb737 +https://conda.anaconda.org/conda-forge/noarch/numpydoc-1.8.0-pyhd8ed1ab_1.conda#5af206d64d18d6c8dfb3122b4d9e643b +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-applehelp-2.0.0-pyhd8ed1ab_1.conda#16e3f039c0aa6446513e94ab18a8784b +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-devhelp-2.0.0-pyhd8ed1ab_1.conda#910f28a05c178feba832f842155cbfff +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-htmlhelp-2.1.0-pyhd8ed1ab_1.conda#e9fb3fe8a5b758b4aff187d434f94f03 +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-qthelp-2.0.0-pyhd8ed1ab_1.conda#00534ebcc0375929b45c3039b5ba7636 +https://conda.anaconda.org/conda-forge/noarch/sphinx-8.1.3-pyhd8ed1ab_1.conda#1a3281a0dc355c02b5506d87db2d78ac +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-serializinghtml-1.1.10-pyhd8ed1ab_1.conda#3bc61f7161d28137797e038263c04c54 diff --git a/build_tools/azure/test_docs.sh b/build_tools/azure/test_docs.sh index 18b3ccb148b5e..f3f824d5806b0 100755 --- a/build_tools/azure/test_docs.sh +++ b/build_tools/azure/test_docs.sh @@ -1,15 +1,21 @@ #!/bin/bash -set -e +set -ex -if [[ "$DISTRIB" =~ ^conda.* ]]; then - source activate $VIRTUALENV -elif [[ "$DISTRIB" == "ubuntu" ]]; then - source $VIRTUALENV/bin/activate -fi +source build_tools/shared.sh +activate_environment -if [[ "$BUILD_WITH_ICC" == "true" ]]; then - source /opt/intel/oneapi/setvars.sh +scipy_doctest_installed=$(python -c 'import scipy_doctest' && echo "True" || echo "False") +if [[ "$scipy_doctest_installed" == "True" ]]; then + doc_rst_files=$(find $PWD/doc -name '*.rst' | sort) + # Changing dir, as we do in build_tools/azure/test_script.sh, avoids an + # error when importing sklearn. Not sure why this happens ... I am going to + # wild guess that it has something to do with the bespoke way we set up + # conda with putting conda in the PATH and source activate, rather than + # source <conda_root>/etc/profile.d/conda.sh + conda activate. + cd $TEST_DIR + # with scipy-doctest, --doctest-modules only runs doctests (in contrary to + # vanilla pytest where it runs doctests on top of normal tests) + python -m pytest --doctest-modules --pyargs sklearn + python -m pytest --doctest-modules $doc_rst_files fi - -make test-doc diff --git a/build_tools/azure/test_script.cmd b/build_tools/azure/test_script.cmd deleted file mode 100644 index c5f8d0e33889c..0000000000000 --- a/build_tools/azure/test_script.cmd +++ /dev/null @@ -1,24 +0,0 @@ -@echo on - -@rem Only 64 bit uses conda and uses a python newer than 3.5 -IF "%PYTHON_ARCH%"=="64" ( - call activate %VIRTUALENV% -) - -mkdir %TMP_FOLDER% -cd %TMP_FOLDER% - -if "%PYTEST_XDIST%" == "true" ( - set PYTEST_ARGS=%PYTEST_ARGS% -n2 -) - -if "%CHECK_WARNINGS%" == "true" ( - REM numpy's 1.19.0's tostring() deprecation is ignored until scipy and joblib removes its usage - set PYTEST_ARGS=%PYTEST_ARGS% -Werror::DeprecationWarning -Werror::FutureWarning -Wignore:tostring:DeprecationWarning -) - -if "%COVERAGE%" == "true" ( - set PYTEST_ARGS=%PYTEST_ARGS% --cov sklearn -) - -pytest --junitxml=%JUNITXML% --showlocals --durations=20 %PYTEST_ARGS% --pyargs sklearn diff --git a/build_tools/azure/test_script.sh b/build_tools/azure/test_script.sh index 858d691b38216..eb4414283be2b 100755 --- a/build_tools/azure/test_script.sh +++ b/build_tools/azure/test_script.sh @@ -2,30 +2,44 @@ set -e -if [[ "$DISTRIB" =~ ^conda.* ]]; then - source activate $VIRTUALENV -elif [[ "$DISTRIB" == "ubuntu" ]] || [[ "$DISTRIB" == "ubuntu-32" ]]; then - source $VIRTUALENV/bin/activate +# Defines the show_installed_libraries and activate_environment functions. +source build_tools/shared.sh + +activate_environment + +if [[ "$BUILD_REASON" == "Schedule" ]]; then + # Enable global random seed randomization to discover seed-sensitive tests + # only on nightly builds. + # https://scikit-learn.org/stable/computing/parallelism.html#environment-variables + export SKLEARN_TESTS_GLOBAL_RANDOM_SEED=$(($RANDOM % 100)) + echo "To reproduce this test run, set the following environment variable:" + echo " SKLEARN_TESTS_GLOBAL_RANDOM_SEED=$SKLEARN_TESTS_GLOBAL_RANDOM_SEED", + echo "See: https://scikit-learn.org/dev/computing/parallelism.html#sklearn-tests-global-random-seed" + + # Enable global dtype fixture for all nightly builds to discover + # numerical-sensitive tests. + # https://scikit-learn.org/stable/computing/parallelism.html#environment-variables + export SKLEARN_RUN_FLOAT32_TESTS=1 fi -if [[ "$BUILD_WITH_ICC" == "true" ]]; then - source /opt/intel/oneapi/setvars.sh +COMMIT_MESSAGE=$(python build_tools/azure/get_commit_message.py --only-show-message) + +if [[ "$COMMIT_MESSAGE" =~ \[float32\] ]]; then + echo "float32 tests will be run due to commit message" + export SKLEARN_RUN_FLOAT32_TESTS=1 fi -python --version -python -c "import numpy; print('numpy %s' % numpy.__version__)" -python -c "import scipy; print('scipy %s' % scipy.__version__)" -python -c "\ -try: - import pandas - print('pandas %s' % pandas.__version__) -except ImportError: - print('pandas not installed') -" -python -c "import multiprocessing as mp; print('%d CPUs' % mp.cpu_count())" -pip list - -TEST_CMD="python -m pytest --showlocals --durations=20 --junitxml=$JUNITXML" +mkdir -p $TEST_DIR +cp pyproject.toml $TEST_DIR +cd $TEST_DIR + +python -c "import joblib; print(f'Number of cores (physical): \ +{joblib.cpu_count()} ({joblib.cpu_count(only_physical_cores=True)})')" +python -c "import sklearn; sklearn.show_versions()" + +show_installed_libraries + +TEST_CMD="python -m pytest --showlocals --durations=20 --junitxml=$JUNITXML -o junit_family=legacy" if [[ "$COVERAGE" == "true" ]]; then # Note: --cov-report= is used to disable to long text output report in the @@ -34,22 +48,43 @@ if [[ "$COVERAGE" == "true" ]]; then # report that otherwise hides the test failures and forces long scrolls in # the CI logs. export COVERAGE_PROCESS_START="$BUILD_SOURCESDIRECTORY/.coveragerc" - TEST_CMD="$TEST_CMD --cov-config=$COVERAGE_PROCESS_START --cov sklearn --cov-report=" -fi -if [[ -n "$CHECK_WARNINGS" ]]; then - # numpy's 1.19.0's tostring() deprecation is ignored until scipy and joblib removes its usage - TEST_CMD="$TEST_CMD -Werror::DeprecationWarning -Werror::FutureWarning -Wignore:tostring:DeprecationWarning" + # Use sys.monitoring to make coverage faster for Python >= 3.12 + HAS_SYSMON=$(python -c 'import sys; print(sys.version_info >= (3, 12))') + if [[ "$HAS_SYSMON" == "True" ]]; then + export COVERAGE_CORE=sysmon + fi + TEST_CMD="$TEST_CMD --cov-config='$COVERAGE_PROCESS_START' --cov sklearn --cov-report=" fi if [[ "$PYTEST_XDIST_VERSION" != "none" ]]; then - TEST_CMD="$TEST_CMD -n2" + XDIST_WORKERS=$(python -c "import joblib; print(joblib.cpu_count(only_physical_cores=True))") + TEST_CMD="$TEST_CMD -n$XDIST_WORKERS" fi -mkdir -p $TEST_DIR -cp setup.cfg $TEST_DIR -cd $TEST_DIR +if [[ -n "$SELECTED_TESTS" ]]; then + TEST_CMD="$TEST_CMD -k $SELECTED_TESTS" + + # Override to make selected tests run on all random seeds + export SKLEARN_TESTS_GLOBAL_RANDOM_SEED="all" +fi + +if which lscpu ; then + lscpu +else + echo "Could not inspect CPU architecture." +fi + +if [[ "$DISTRIB" == "conda-free-threaded" ]]; then + # Make sure that GIL is disabled even when importing extensions that have + # not declared free-threaded compatibility. This can be removed when numpy, + # scipy and scikit-learn extensions all have declared free-threaded + # compatibility. + export PYTHON_GIL=0 +fi + +TEST_CMD="$TEST_CMD --pyargs sklearn" set -x -$TEST_CMD --pyargs sklearn +eval "$TEST_CMD" set +x diff --git a/build_tools/azure/ubuntu_atlas_lock.txt b/build_tools/azure/ubuntu_atlas_lock.txt new file mode 100644 index 0000000000000..9c1faa23ab962 --- /dev/null +++ b/build_tools/azure/ubuntu_atlas_lock.txt @@ -0,0 +1,45 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=build_tools/azure/ubuntu_atlas_lock.txt build_tools/azure/ubuntu_atlas_requirements.txt +# +cython==3.0.10 + # via -r build_tools/azure/ubuntu_atlas_requirements.txt +exceptiongroup==1.3.0 + # via pytest +execnet==2.1.1 + # via pytest-xdist +iniconfig==2.1.0 + # via pytest +joblib==1.2.0 + # via -r build_tools/azure/ubuntu_atlas_requirements.txt +meson==1.8.1 + # via meson-python +meson-python==0.18.0 + # via -r build_tools/azure/ubuntu_atlas_requirements.txt +ninja==1.11.1.4 + # via -r build_tools/azure/ubuntu_atlas_requirements.txt +packaging==25.0 + # via + # meson-python + # pyproject-metadata + # pytest +pluggy==1.6.0 + # via pytest +pyproject-metadata==0.9.1 + # via meson-python +pytest==8.3.5 + # via + # -r build_tools/azure/ubuntu_atlas_requirements.txt + # pytest-xdist +pytest-xdist==3.7.0 + # via -r build_tools/azure/ubuntu_atlas_requirements.txt +threadpoolctl==3.1.0 + # via -r build_tools/azure/ubuntu_atlas_requirements.txt +tomli==2.2.1 + # via + # meson-python + # pytest +typing-extensions==4.13.2 + # via exceptiongroup diff --git a/build_tools/azure/ubuntu_atlas_requirements.txt b/build_tools/azure/ubuntu_atlas_requirements.txt new file mode 100644 index 0000000000000..dfb0cfebc54d1 --- /dev/null +++ b/build_tools/azure/ubuntu_atlas_requirements.txt @@ -0,0 +1,10 @@ +# DO NOT EDIT: this file is generated from the specification found in the +# following script to centralize the configuration for CI builds: +# build_tools/update_environments_and_lock_files.py +cython==3.0.10 # min +joblib==1.2.0 # min +threadpoolctl==3.1.0 # min +pytest +pytest-xdist +ninja +meson-python diff --git a/build_tools/azure/upload_codecov.cmd b/build_tools/azure/upload_codecov.cmd deleted file mode 100644 index 6150b75a1ea54..0000000000000 --- a/build_tools/azure/upload_codecov.cmd +++ /dev/null @@ -1,10 +0,0 @@ -@echo on - -@rem Only 64 bit uses conda -IF "%PYTHON_ARCH%"=="64" ( - call activate %VIRTUALENV% -) - -copy %TMP_FOLDER%\.coverage %BUILD_REPOSITORY_LOCALPATH% - -codecov --root %BUILD_REPOSITORY_LOCALPATH% -t %CODECOV_TOKEN% diff --git a/build_tools/azure/upload_codecov.sh b/build_tools/azure/upload_codecov.sh index 274106cb19f75..4c3db8fe8bbd6 100755 --- a/build_tools/azure/upload_codecov.sh +++ b/build_tools/azure/upload_codecov.sh @@ -2,15 +2,58 @@ set -e -# called when COVERAGE=="true" and DISTRIB=="conda" -export PATH=$HOME/miniconda3/bin:$PATH -source activate $VIRTUALENV +# Do not upload to codecov on forks +if [[ "$BUILD_REPOSITORY_NAME" != "scikit-learn/scikit-learn" ]]; then + exit 0 +fi -# Need to run codecov from a git checkout, so we copy .coverage -# from TEST_DIR where pytest has been run -pushd $TEST_DIR -coverage combine --append -popd -cp $TEST_DIR/.coverage $BUILD_REPOSITORY_LOCALPATH +# When we update the codecov uploader version, we need to update the checksums. +# The checksum for each codecov binary is available at +# https://cli.codecov.io e.g. for linux +# https://cli.codecov.io/v10.2.1/linux/codecov.SHA256SUM. -codecov --root $BUILD_REPOSITORY_LOCALPATH -t $CODECOV_TOKEN || echo "codecov upload failed" +# Instead of hardcoding a specific version and signature in this script, it +# would be possible to use the "latest" symlink URL but then we need to +# download both the codecov.SHA256SUM files each time and check the signatures +# with the codecov gpg key as well, see: +# https://docs.codecov.com/docs/codecov-uploader#integrity-checking-the-uploader +# However this approach would yield a larger number of downloads from +# codecov.io and keybase.io, therefore increasing the risk of running into +# network failures. +CODECOV_CLI_VERSION=10.2.1 +CODECOV_BASE_URL="https://cli.codecov.io/v$CODECOV_CLI_VERSION" + +# Check that the git repo is located at the expected location: +if [[ ! -d "$BUILD_REPOSITORY_LOCALPATH/.git" ]]; then + echo "Could not find the git checkout at $BUILD_REPOSITORY_LOCALPATH" + exit 1 +fi + +# Check that the combined coverage file exists at the expected location: +export COVERAGE_XML="$BUILD_REPOSITORY_LOCALPATH/coverage.xml" +if [[ ! -f "$COVERAGE_XML" ]]; then + echo "Could not find the combined coverage file at $COVERAGE_XML" + exit 1 +fi + +if [[ $OSTYPE == *"linux"* ]]; then + curl -Os "$CODECOV_BASE_URL/linux/codecov" + SHA256SUM="39dd112393680356daf701c07f375303aef5de62f06fc80b466b5c3571336014 codecov" + echo "$SHA256SUM" | shasum -a256 -c + chmod +x codecov + ./codecov upload-coverage -t ${CODECOV_TOKEN} -f coverage.xml -Z + ./codecov do-upload --disable-search --report-type test_results --file $JUNIT_FILE +elif [[ $OSTYPE == *"darwin"* ]]; then + curl -Os "$CODECOV_BASE_URL/macos/codecov" + SHA256SUM="01183f6367c7baff4947cce389eaa511b7a6d938e37ae579b08a86b51f769fd9 codecov" + echo "$SHA256SUM" | shasum -a256 -c + chmod +x codecov + ./codecov upload-coverage -t ${CODECOV_TOKEN} -f coverage.xml -Z + ./codecov do-upload --disable-search --report-type test_results --file $JUNIT_FILE +else + curl -Os "$CODECOV_BASE_URL/windows/codecov.exe" + SHA256SUM="e54e9520428701a510ef451001db56b56fb17f9b0484a266f184b73dd27b77e7 codecov.exe" + echo "$SHA256SUM" | sha256sum -c + ./codecov.exe upload-coverage -t ${CODECOV_TOKEN} -f coverage.xml -Z + ./codecov.exe do-upload --disable-search --report-type test_results --file $JUNIT_FILE +fi diff --git a/build_tools/azure/windows.yml b/build_tools/azure/windows.yml index 8a5edd4b93019..b3fcf130f9350 100644 --- a/build_tools/azure/windows.yml +++ b/build_tools/azure/windows.yml @@ -16,39 +16,71 @@ jobs: VIRTUALENV: 'testvenv' JUNITXML: 'test-data.xml' SKLEARN_SKIP_NETWORK_TESTS: '1' - PYTEST_VERSION: '5.2.1' - PYTEST_XDIST: 'true' - TMP_FOLDER: '$(Agent.WorkFolder)\tmp_folder' + PYTEST_XDIST_VERSION: 'latest' + TEST_DIR: '$(Agent.WorkFolder)/tmp_folder' + SHOW_SHORT_SUMMARY: 'false' strategy: matrix: ${{ insert }}: ${{ parameters.matrix }} steps: - - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" - displayName: Add conda to PATH for 64 bit Python - condition: eq(variables['PYTHON_ARCH'], '64') + - bash: python build_tools/azure/get_selected_tests.py + displayName: Check selected tests for all random seeds + condition: eq(variables['Build.Reason'], 'PullRequest') + - bash: echo "##vso[task.prependpath]$CONDA/Scripts" + displayName: Add conda to PATH + condition: startsWith(variables['DISTRIB'], 'conda') - task: UsePythonVersion@0 inputs: versionSpec: '$(PYTHON_VERSION)' addToPath: true architecture: 'x86' displayName: Use 32 bit System Python - condition: eq(variables['PYTHON_ARCH'], '32') - - script: | - build_tools\\azure\\install.cmd + condition: and(succeeded(), eq(variables['PYTHON_ARCH'], '32')) + - bash: ./build_tools/azure/install.sh displayName: 'Install' - - script: | - build_tools\\azure\\test_script.cmd + - bash: ./build_tools/azure/test_script.sh displayName: 'Test Library' - - script: | - build_tools\\azure\\upload_codecov.cmd - condition: and(succeeded(), eq(variables['COVERAGE'], 'true')) - displayName: 'Upload To Codecov' - env: - CODECOV_TOKEN: $(CODECOV_TOKEN) + - bash: ./build_tools/azure/combine_coverage_reports.sh + condition: and(succeeded(), eq(variables['COVERAGE'], 'true'), + eq(variables['SELECTED_TESTS'], '')) + displayName: 'Combine coverage' - task: PublishTestResults@2 inputs: - testResultsFiles: '$(TMP_FOLDER)\$(JUNITXML)' + testResultsFiles: '$(TEST_DIR)/$(JUNITXML)' testRunTitle: ${{ format('{0}-$(Agent.JobName)', parameters.name) }} displayName: 'Publish Test Results' condition: succeededOrFailed() + - bash: | + set -ex + if [[ $(BOT_GITHUB_TOKEN) == "" ]]; then + echo "GitHub Token is not set. Issue tracker will not be updated." + exit + fi + + LINK_TO_RUN="https://dev.azure.com/$BUILD_REPOSITORY_NAME/_build/results?buildId=$BUILD_BUILDID&view=logs&j=$SYSTEM_JOBID" + CI_NAME="$SYSTEM_JOBIDENTIFIER" + ISSUE_REPO="$BUILD_REPOSITORY_NAME" + + $(pyTools.pythonLocation)/bin/pip install defusedxml PyGithub + $(pyTools.pythonLocation)/bin/python maint_tools/update_tracking_issue.py \ + $(BOT_GITHUB_TOKEN) \ + $CI_NAME \ + $ISSUE_REPO \ + $LINK_TO_RUN \ + --junit-file $JUNIT_FILE \ + --auto-close false + displayName: 'Update issue tracker' + env: + JUNIT_FILE: $(TEST_DIR)/$(JUNITXML) + condition: and(succeededOrFailed(), eq(variables['CREATE_ISSUE_ON_TRACKER'], 'true'), + eq(variables['Build.Reason'], 'Schedule')) + - bash: ./build_tools/azure/upload_codecov.sh + condition: and(succeeded(), + eq(variables['COVERAGE'], 'true'), + eq(variables['SELECTED_TESTS'], '')) + displayName: 'Upload To Codecov' + retryCountOnTaskFailure: 5 + env: + CODECOV_TOKEN: $(CODECOV_TOKEN) + JUNIT_FILE: $(TEST_DIR)/$(JUNITXML) diff --git a/build_tools/check-meson-openmp-dependencies.py b/build_tools/check-meson-openmp-dependencies.py new file mode 100644 index 0000000000000..43a7426494160 --- /dev/null +++ b/build_tools/check-meson-openmp-dependencies.py @@ -0,0 +1,172 @@ +""" +Check that OpenMP dependencies are correctly defined in meson.build files. + +This is based on trying to make sure the the following two things match: +- the Cython files using OpenMP (based on a git grep regex) +- the Cython extension modules that are built with OpenMP compiler flags (based + on meson introspect json output) +""" + +import json +import re +import subprocess +from pathlib import Path + + +def has_source_openmp_flags(target_source): + return any("openmp" in arg for arg in target_source["parameters"]) + + +def has_openmp_flags(target): + """Return whether target sources use OpenMP flags. + + Make sure that both compiler and linker source use OpenMP. + Look at `get_meson_info` docstring to see what `target` looks like. + """ + target_sources = target["target_sources"] + + target_use_openmp_flags = any( + has_source_openmp_flags(target_source) for target_source in target_sources + ) + + if not target_use_openmp_flags: + return False + + # When the target use OpenMP we expect a compiler + linker source and we + # want to make sure that both the compiler and the linker use OpenMP + assert len(target_sources) == 2 + compiler_source, linker_source = target_sources + assert "compiler" in compiler_source + assert "linker" in linker_source + + compiler_use_openmp_flags = any( + "openmp" in arg for arg in compiler_source["parameters"] + ) + linker_use_openmp_flags = any( + "openmp" in arg for arg in linker_source["parameters"] + ) + + assert compiler_use_openmp_flags == linker_use_openmp_flags + return compiler_use_openmp_flags + + +def get_canonical_name_meson(target, build_path): + """Return a name based on generated shared library. + + The goal is to return a name that can be easily matched with the output + from `git_grep_info`. + + Look at `get_meson_info` docstring to see what `target` looks like. + """ + # Expect a list with one element with the name of the shared library + assert len(target["filename"]) == 1 + shared_library_path = Path(target["filename"][0]) + shared_library_relative_path = shared_library_path.relative_to( + build_path.absolute() + ) + # Needed on Windows to match git grep output + rel_path = shared_library_relative_path.as_posix() + # OS-specific naming of the shared library .cpython- on POSIX and + # something like .cp312- on Windows + pattern = r"\.(cpython|cp\d+)-.+" + return re.sub(pattern, "", str(rel_path)) + + +def get_canonical_name_git_grep(filename): + """Return name based on filename. + + The goal is to return a name that can easily be matched with the output + from `get_meson_info`. + """ + return re.sub(r"\.pyx(\.tp)?", "", filename) + + +def get_meson_info(): + """Return names of extension that use OpenMP based on meson introspect output. + + The meson introspect json info is a list of targets where a target is a dict + that looks like this (parts not used in this script are not shown for simplicity): + { + 'name': '_k_means_elkan.cpython-312-x86_64-linux-gnu', + 'filename': [ + '<meson_build_dir>/sklearn/cluster/_k_means_elkan.cpython-312-x86_64-linux-gnu.so' + ], + 'target_sources': [ + { + 'compiler': ['ccache', 'cc'], + 'parameters': [ + '-Wall', + '-std=c11', + '-fopenmp', + ... + ], + ... + }, + { + 'linker': ['cc'], + 'parameters': [ + '-shared', + '-fPIC', + '-fopenmp', + ... + ] + } + ] + } + """ + build_path = Path("build/introspect") + subprocess.check_call(["meson", "setup", build_path, "--reconfigure"]) + + json_out = subprocess.check_output( + ["meson", "introspect", build_path, "--targets"], text=True + ) + target_list = json.loads(json_out) + meson_targets = [target for target in target_list if has_openmp_flags(target)] + + return [get_canonical_name_meson(each, build_path) for each in meson_targets] + + +def get_git_grep_info(): + """Return names of extensions that use OpenMP based on git grep regex.""" + git_grep_filenames = subprocess.check_output( + ["git", "grep", "-lP", "cython.*parallel|_openmp_helpers"], text=True + ).splitlines() + git_grep_filenames = [f for f in git_grep_filenames if ".pyx" in f] + + return [get_canonical_name_git_grep(each) for each in git_grep_filenames] + + +def main(): + from_meson = set(get_meson_info()) + from_git_grep = set(get_git_grep_info()) + + only_in_git_grep = from_git_grep - from_meson + only_in_meson = from_meson - from_git_grep + + msg = "" + if only_in_git_grep: + only_in_git_grep_msg = "\n".join( + [f" {each}" for each in sorted(only_in_git_grep)] + ) + msg += ( + "Some Cython files use OpenMP," + " but their meson.build is missing the openmp_dep dependency:\n" + f"{only_in_git_grep_msg}\n\n" + ) + + if only_in_meson: + only_in_meson_msg = "\n".join([f" {each}" for each in sorted(only_in_meson)]) + msg += ( + "Some Cython files do not use OpenMP," + " you should remove openmp_dep from their meson.build:\n" + f"{only_in_meson_msg}\n\n" + ) + + if from_meson != from_git_grep: + raise ValueError( + f"Some issues have been found in Meson OpenMP dependencies:\n\n{msg}" + ) + + +if __name__ == "__main__": + main() diff --git a/build_tools/circle/build_doc.sh b/build_tools/circle/build_doc.sh index 37afb1841d368..e85f3ab15e617 100755 --- a/build_tools/circle/build_doc.sh +++ b/build_tools/circle/build_doc.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -x set -e +set -x # Decide what kind of documentation build to run, and run it. # @@ -17,6 +17,32 @@ set -e # If the inspection of the current commit fails for any reason, the default # behavior is to quick build the documentation. +# defines the get_dep and show_installed_libraries functions +source build_tools/shared.sh + +if [ -n "$GITHUB_ACTION" ] +then + # Map the variables from Github Action to CircleCI + CIRCLE_SHA1=$(git log -1 --pretty=format:%H) + + CIRCLE_JOB=$GITHUB_JOB + + if [ "$GITHUB_EVENT_NAME" == "pull_request" ] + then + CIRCLE_BRANCH=$GITHUB_HEAD_REF + CI_PULL_REQUEST=true + CI_TARGET_BRANCH=$GITHUB_BASE_REF + else + CIRCLE_BRANCH=$GITHUB_REF_NAME + fi +fi + +if [[ -n "$CI_PULL_REQUEST" && -z "$CI_TARGET_BRANCH" ]] +then + # Get the target branch name when using CircleCI + CI_TARGET_BRANCH=$(curl -s "https://api.github.com/repos/scikit-learn/scikit-learn/pulls/$CIRCLE_PR_NUMBER" | jq -r .base.ref) +fi + get_build_type() { if [ -z "$CIRCLE_SHA1" ] then @@ -130,83 +156,81 @@ else make_args=html fi -make_args="SPHINXOPTS=-T $make_args" # show full traceback on exception - # Installing required system packages to support the rendering of math # notation in the HTML documentation and to optimize the image files -sudo -E apt-get -yq update +sudo -E apt-get -yq update --allow-releaseinfo-change sudo -E apt-get -yq --no-install-suggests --no-install-recommends \ install dvipng gsfonts ccache zip optipng -# deactivate circleci virtualenv and setup a miniconda env instead +# deactivate circleci virtualenv and setup a conda env instead if [[ `type -t deactivate` ]]; then deactivate fi -MINICONDA_PATH=$HOME/miniconda -# Install dependencies with miniconda -wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ - -O miniconda.sh -chmod +x miniconda.sh && ./miniconda.sh -b -p $MINICONDA_PATH -export PATH="/usr/lib/ccache:$MINICONDA_PATH/bin:$PATH" +# Install Miniforge +MINIFORGE_URL="https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh" +curl -L --retry 10 $MINIFORGE_URL -o miniconda.sh +MINIFORGE_PATH=$HOME/miniforge3 +bash ./miniconda.sh -b -p $MINIFORGE_PATH +source $MINIFORGE_PATH/etc/profile.d/conda.sh +conda activate + +create_conda_environment_from_lock_file $CONDA_ENV_NAME $LOCK_FILE +conda activate $CONDA_ENV_NAME + +# Sets up ccache when using system compiler +export PATH="/usr/lib/ccache:$PATH" +# Sets up ccache when using conda-forge compilers (needs to be after conda +# activate which sets CC and CXX) +export CC="ccache $CC" +export CXX="ccache $CXX" ccache -M 512M export CCACHE_COMPRESS=1 +# Zeroing statistics so that ccache statistics are shown only for this build +ccache -z -# Old packages coming from the 'free' conda channel have been removed but we -# are using them for our min-dependencies doc generation. See -# https://www.anaconda.com/why-we-removed-the-free-channel-in-conda-4-7/ for -# more details. -if [[ "$CIRCLE_JOB" == "doc-min-dependencies" ]]; then - conda config --set restore_free_channel true -fi +show_installed_libraries -# imports get_dep -source build_tools/shared.sh +# Specify explicitly ninja -j argument because ninja does not handle cgroups v2 and +# use the same default rule as ninja (-j3 since we have 2 cores on CircleCI), see +# https://github.com/scikit-learn/scikit-learn/pull/30333 +pip install -e . --no-build-isolation --config-settings=compile-args="-j 3" -# packaging won't be needed once setuptools starts shipping packaging>=17.0 -conda create -n $CONDA_ENV_NAME --yes --quiet \ - python="${PYTHON_VERSION:-*}" \ - "$(get_dep numpy $NUMPY_VERSION)" \ - "$(get_dep scipy $SCIPY_VERSION)" \ - "$(get_dep cython $CYTHON_VERSION)" \ - "$(get_dep matplotlib $MATPLOTLIB_VERSION)" \ - "$(get_dep sphinx $SPHINX_VERSION)" \ - "$(get_dep scikit-image $SCIKIT_IMAGE_VERSION)" \ - "$(get_dep pandas $PANDAS_VERSION)" \ - joblib memory_profiler packaging seaborn pillow pytest coverage - -source activate testenv -pip install sphinx-gallery -pip install numpydoc -pip install sphinx-prompt - -# Set parallelism to 3 to overlap IO bound tasks with CPU bound tasks on CI -# workers with 2 cores when building the compiled extensions of scikit-learn. -export SKLEARN_BUILD_PARALLEL=3 -python setup.py develop +echo "ccache build summary:" +ccache -s export OMP_NUM_THREADS=1 +if [[ "$CIRCLE_BRANCH" == "main" || "$CI_TARGET_BRANCH" == "main" ]] +then + towncrier build --yes +fi + if [[ "$CIRCLE_BRANCH" =~ ^main$ && -z "$CI_PULL_REQUEST" ]] then # List available documentation versions if on main - python build_tools/circle/list_versions.py > doc/versions.rst + python build_tools/circle/list_versions.py --json doc/js/versions.json --rst doc/versions.rst fi + # The pipefail is requested to propagate exit code set -o pipefail && cd doc && make $make_args 2>&1 | tee ~/log.txt -# Insert the version warning for deployment -find _build/html/stable -name "*.html" | xargs sed -i '/<\/body>/ i \ -\ <script src="https://scikit-learn.org/versionwarning.js"></script>' - cd - set +o pipefail affected_doc_paths() { + scikit_learn_version=$(python -c 'import re; import sklearn; print(re.sub(r"(\d+\.\d+).+", r"\1", sklearn.__version__))') files=$(git diff --name-only origin/main...$CIRCLE_SHA1) - echo "$files" | grep ^doc/.*\.rst | sed 's/^doc\/\(.*\)\.rst$/\1.html/' + # use sed to replace files ending by .rst or .rst.template by .html + echo "$files" | grep -vP 'upcoming_changes/.*/\d+.*\.rst' | grep ^doc/.*\.rst | \ + sed 's/^doc\/\(.*\)\.rst$/\1.html/; s/^doc\/\(.*\)\.rst\.template$/\1.html/' + # replace towncrier fragment files by link to changelog. uniq is used + # because in some edge cases multiple fragments can be added and we want a + # single link to the changelog. + echo "$files" | grep -P 'upcoming_changes/.*/\d+.*\.rst' | sed "s@.*@whats_new/v${scikit_learn_version}.html@" | uniq + echo "$files" | grep ^examples/.*.py | sed 's/^\(.*\)\.py$/auto_\1.html/' sklearn_files=$(echo "$files" | grep '^sklearn/') if [ -n "$sklearn_files" ] @@ -244,7 +268,7 @@ then ( echo '<html><body><ul>' echo "$affected" | sed 's|.*|<li><a href="&">&</a> [<a href="https://scikit-learn.org/dev/&">dev</a>, <a href="https://scikit-learn.org/stable/&">stable</a>]</li>|' - echo '</ul><p>General: <a href="index.html">Home</a> | <a href="modules/classes.html">API Reference</a> | <a href="auto_examples/index.html">Examples</a></p>' + echo '</ul><p>General: <a href="index.html">Home</a> | <a href="api/index.html">API Reference</a> | <a href="auto_examples/index.html">Examples</a></p>' echo '<strong>Sphinx Warnings in affected files</strong><ul>' echo "$warnings" | sed 's/\/home\/circleci\/project\//<li>/g' echo '</ul></body></html>' diff --git a/build_tools/circle/build_test_pypy.sh b/build_tools/circle/build_test_pypy.sh deleted file mode 100755 index c1def8ce2670b..0000000000000 --- a/build_tools/circle/build_test_pypy.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -set -x -set -e - -# System build tools -apt-get -yq update -apt-get -yq install wget bzip2 build-essential ccache - -# Install pypy and all the scikit-learn dependencies from conda-forge. In -# particular, we want to install pypy compatible binary packages for numpy and -# scipy as it would be to costly to build those from source. -conda install -y mamba -mamba create -n pypy -y \ - pypy numpy scipy cython \ - joblib threadpoolctl pillow pytest \ - sphinx numpydoc docutils - -eval "$(conda shell.bash hook)" -conda activate pypy - -# Check that we are running PyPy instead of CPython in this environment. -python --version -which python -python -c "import platform; assert platform.python_implementation() == 'PyPy'" - -# Build and install scikit-learn in dev mode -ccache -M 512M -export CCACHE_COMPRESS=1 -export PATH=/usr/lib/ccache:$PATH -export LOKY_MAX_CPU_COUNT="2" -export OMP_NUM_THREADS="1" -# Set parallelism to 3 to overlap IO bound tasks with CPU bound tasks on CI -# workers with 2 cores when building the compiled extensions of scikit-learn. -export SKLEARN_BUILD_PARALLEL=3 -pip install --no-build-isolation -e . - -python -m pytest sklearn diff --git a/build_tools/circle/doc_environment.yml b/build_tools/circle/doc_environment.yml new file mode 100644 index 0000000000000..bc36e178de058 --- /dev/null +++ b/build_tools/circle/doc_environment.yml @@ -0,0 +1,44 @@ +# DO NOT EDIT: this file is generated from the specification found in the +# following script to centralize the configuration for CI builds: +# build_tools/update_environments_and_lock_files.py +channels: + - conda-forge +dependencies: + - python=3.10 + - numpy + - blas + - scipy + - cython + - joblib + - threadpoolctl + - matplotlib + - pandas + - pyamg + - pytest + - pytest-xdist + - pillow + - pip + - ninja + - meson-python + - scikit-image + - seaborn + - memory_profiler + - compilers + - sphinx + - sphinx-gallery + - sphinx-copybutton + - numpydoc + - sphinx-prompt + - plotly + - polars + - pooch + - sphinxext-opengraph + - sphinx-remove-toctrees + - sphinx-design + - pydata-sphinx-theme + - towncrier + - pip + - pip: + - jupyterlite-sphinx + - jupyterlite-pyodide-kernel + - sphinxcontrib-sass diff --git a/build_tools/circle/doc_linux-64_conda.lock b/build_tools/circle/doc_linux-64_conda.lock new file mode 100644 index 0000000000000..d19f830684796 --- /dev/null +++ b/build_tools/circle/doc_linux-64_conda.lock @@ -0,0 +1,329 @@ +# Generated by conda-lock. +# platform: linux-64 +# input_hash: 93cb6f7aa17dce662512650f1419e87eae56ed49163348847bf965697cd268bb +@EXPLICIT +https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81 +https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2#0c96522c6bdaed4b1566d11387caaf45 +https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2#34893075a5c9e55cdafac56607368fc6 +https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2#4d59c254e01d9cde7957100457e2d5fb +https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-h77eed37_3.conda#49023d73832ef61042f6a237cb2687e7 +https://conda.anaconda.org/conda-forge/noarch/kernel-headers_linux-64-3.10.0-he073ed8_18.conda#ad8527bf134a90e1c9ed35fa0b64318c +https://conda.anaconda.org/conda-forge/noarch/python_abi-3.10-7_cp310.conda#44e871cba2b162368476a84b8d040b6c +https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda#4222072737ccff51314b5ece9c7d6f5a +https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.4.26-hbd8a1cb_0.conda#95db94f75ba080a22eb623590993167b +https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-0.tar.bz2#f766549260d6815b0c52253f1fb1bb29 +https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.43-h712a8e2_4.conda#01f8d123c96816249efd255a31ad7712 +https://conda.anaconda.org/conda-forge/noarch/libgcc-devel_linux-64-13.3.0-hc03c837_102.conda#4c1d6961a6a54f602ae510d9bf31fa60 +https://conda.anaconda.org/conda-forge/linux-64/libglvnd-1.7.0-ha4b6fd6_2.conda#434ca7e50e40f4918ab701e3facd59a0 +https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.1.0-h767d61c_2.conda#fbe7d535ff9d3a168c148e07358cd5b1 +https://conda.anaconda.org/conda-forge/noarch/libstdcxx-devel_linux-64-13.3.0-hc03c837_102.conda#aa38de2738c5f4a72a880e3d31ffe8b4 +https://conda.anaconda.org/conda-forge/noarch/sysroot_linux-64-2.17-h0157908_18.conda#460eba7851277ec1fd80a1a24080787a +https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2#73aaf86a425cc6e73fcf236a5a46396d +https://conda.anaconda.org/conda-forge/linux-64/binutils_impl_linux-64-2.43-h4bf12b8_4.conda#ef67db625ad0d2dce398837102f875ed +https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2#fee5683a3f04bd15cbd8318b096a27ab +https://conda.anaconda.org/conda-forge/linux-64/libegl-1.7.0-ha4b6fd6_2.conda#c151d5eb730e9b7480e6d48c0fc44048 +https://conda.anaconda.org/conda-forge/linux-64/libopengl-1.7.0-ha4b6fd6_2.conda#7df50d44d4a14d6c31a2c54f2cd92157 +https://conda.anaconda.org/conda-forge/linux-64/binutils-2.43-h4852527_4.conda#29782348a527eda3ecfc673109d28e93 +https://conda.anaconda.org/conda-forge/linux-64/binutils_linux-64-2.43-h4852527_4.conda#c87e146f5b685672d4aa6b527c6d3b5e +https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.1.0-h767d61c_2.conda#ea8ac52380885ed41c1baa8f1d6d2b93 +https://conda.anaconda.org/conda-forge/linux-64/alsa-lib-1.2.14-hb9d3cd8_0.conda#76df83c2a9035c54df5d04ff81bcc02d +https://conda.anaconda.org/conda-forge/linux-64/libbrotlicommon-1.1.0-hb9d3cd8_2.conda#41b599ed2b02abcfdd84302bff174b23 +https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.24-h86f0d12_0.conda#64f0c503da58ec25ebd359e4d990afa8 +https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.7.0-h5888daf_0.conda#db0bfbe7dd197b68ad5f30333bae6ce0 +https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.6-h2dba641_1.conda#ede4673863426c0883c0063d853bbd85 +https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-15.1.0-h69a702a_2.conda#ddca86c7040dd0e73b2b69bd7833d225 +https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-15.1.0-hcea5267_2.conda#01de444988ed960031dbe84cf4f9b1fc +https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.18-h4ce23a2_1.conda#e796ff8ddc598affdf7c173d6145f087 +https://conda.anaconda.org/conda-forge/linux-64/libjpeg-turbo-3.1.0-hb9d3cd8_0.conda#9fa334557db9f63da6c9285fd2a48638 +https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.1-hb9d3cd8_1.conda#a76fd702c93cd2dfd89eff30a5fd45a8 +https://conda.anaconda.org/conda-forge/linux-64/libntlm-1.8-hb9d3cd8_0.conda#7c7927b404672409d9917d49bff5f2d6 +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-15.1.0-h8f9b012_2.conda#1cb1c67961f6dd257eae9e9691b341aa +https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.5.0-h851e524_0.conda#63f790534398730f59e1b899c3644d4a +https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda#edb0dca6bc32e4f4789199455a1dbeb8 +https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda#47e340acb35de30501a76c7c799c41d7 +https://conda.anaconda.org/conda-forge/linux-64/openssl-3.5.0-h7b32b05_1.conda#de356753cfdbffcde5bb1e86e3aa6cd0 +https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-hb9d3cd8_1002.conda#b3c17d95b5a10c6e64a21fa17573e70e +https://conda.anaconda.org/conda-forge/linux-64/rav1e-0.7.1-h8fae777_3.conda#2c42649888aac645608191ffdc80d13a +https://conda.anaconda.org/conda-forge/linux-64/xorg-libice-1.1.2-hb9d3cd8_0.conda#fb901ff28063514abb6046c9ec2c4a45 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxau-1.0.12-hb9d3cd8_0.conda#f6ebe2cb3f82ba6c057dde5d9debe4f7 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdmcp-1.1.5-hb9d3cd8_0.conda#8035c64cb77ed555e3f150b7b3972480 +https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h4bc722e_7.conda#62ee74e96c5ebb0af99386de58cf9553 +https://conda.anaconda.org/conda-forge/linux-64/dav1d-1.2.1-hd590300_0.conda#418c6ca5929a611cbd69204907a83995 +https://conda.anaconda.org/conda-forge/linux-64/double-conversion-3.3.1-h5888daf_0.conda#bfd56492d8346d669010eccafe0ba058 +https://conda.anaconda.org/conda-forge/linux-64/giflib-5.2.2-hd590300_0.conda#3bf7b9fd5a7136126e0234db4b87c8b6 +https://conda.anaconda.org/conda-forge/linux-64/jxrlib-1.1-hd590300_3.conda#5aeabe88534ea4169d4c49998f293d6c +https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.1-h166bdaf_0.tar.bz2#30186d27e2c9fa62b45fb1476b7200e3 +https://conda.anaconda.org/conda-forge/linux-64/lerc-4.0.0-h0aef613_1.conda#9344155d33912347b37f0ae6c410a835 +https://conda.anaconda.org/conda-forge/linux-64/libbrotlidec-1.1.0-hb9d3cd8_2.conda#9566f0bd264fbd463002e759b8a82401 +https://conda.anaconda.org/conda-forge/linux-64/libbrotlienc-1.1.0-hb9d3cd8_2.conda#06f70867945ea6a84d35836af780f1de +https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20250104-pl5321h7949ede_0.conda#c277e0a4d549b03ac1e9d6cbbe3d017b +https://conda.anaconda.org/conda-forge/linux-64/libgfortran-15.1.0-h69a702a_2.conda#f92e6e0a3c0c0c85561ef61aa59d555d +https://conda.anaconda.org/conda-forge/linux-64/libhwy-1.2.0-hf40a0c7_0.conda#2f433d593a66044c3f163cb25f0a09de +https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hd590300_0.conda#30fd6e37fe21f86f4bd26d6ee73eeec7 +https://conda.anaconda.org/conda-forge/linux-64/libpciaccess-0.18-hd590300_0.conda#48f4330bfcd959c3cfb704d424903c82 +https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.47-h943b412_0.conda#55199e2ae2c3651f6f9b2a447b47bdc9 +https://conda.anaconda.org/conda-forge/linux-64/libsanitizer-13.3.0-he8ea267_2.conda#2b6cdf7bb95d3d10ef4e38ce0bc95dba +https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.50.0-hee588c1_0.conda#71888e92098d0f8c41b09a671ad289bc +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-15.1.0-h4852527_2.conda#9d2072af184b5caa29492bf2344597bb +https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda#40b61aab5c7ba9ff276c41cfffe6b80b +https://conda.anaconda.org/conda-forge/linux-64/libxcb-1.17.0-h8a09558_0.conda#92ed62436b625154323d40d5f2f11dd7 +https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda#5aa797f8787fe7a17d1b0821485b5adc +https://conda.anaconda.org/conda-forge/linux-64/lz4-c-1.10.0-h5888daf_1.conda#9de5350a85c4a20c685259b889aa6393 +https://conda.anaconda.org/conda-forge/linux-64/ninja-1.12.1-hff21bea_1.conda#2322531904f27501ee19847b87ba7c64 +https://conda.anaconda.org/conda-forge/linux-64/pixman-0.46.0-h29eaf8c_0.conda#d2f1c87d4416d1e7344cf92b1aaee1c4 +https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8c095d6_2.conda#283b96675859b20a825f8fa30f311446 +https://conda.anaconda.org/conda-forge/linux-64/snappy-1.2.1-h8bd8927_1.conda#3b3e64af585eadfb52bb90b553db5edf +https://conda.anaconda.org/conda-forge/linux-64/svt-av1-3.0.2-h5888daf_0.conda#0096882bd623e6cc09e8bf920fc8fb47 +https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_hd72426e_102.conda#a0116df4f4ed05c303811a837d5b39d8 +https://conda.anaconda.org/conda-forge/linux-64/wayland-1.23.1-h3e06ad9_1.conda#a37843723437ba75f42c9270ffe800b1 +https://conda.anaconda.org/conda-forge/linux-64/zfp-1.0.1-h5888daf_2.conda#e0409515c467b87176b070bff5d9442e +https://conda.anaconda.org/conda-forge/linux-64/zlib-ng-2.2.4-h7955e40_0.conda#c8a816dbf59eb8ba6346a8f10014b302 +https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb8e6e7a_2.conda#6432cb5d4ac0046c3ac0a8a0f95842f9 +https://conda.anaconda.org/conda-forge/linux-64/aom-3.9.1-hac33072_0.conda#346722a0be40f6edc53f12640d301338 +https://conda.anaconda.org/conda-forge/linux-64/blosc-1.21.6-he440d0b_1.conda#2c2fae981fd2afd00812c92ac47d023d +https://conda.anaconda.org/conda-forge/linux-64/brotli-bin-1.1.0-hb9d3cd8_2.conda#c63b5e52939e795ba8d26e35d767a843 +https://conda.anaconda.org/conda-forge/linux-64/c-blosc2-2.17.1-h3122c55_0.conda#009d16d3c9ed3e70d58ed46dab1571d1 +https://conda.anaconda.org/conda-forge/linux-64/charls-2.4.2-h59595ed_0.conda#4336bd67920dd504cd8c6761d6a99645 +https://conda.anaconda.org/conda-forge/linux-64/gcc_impl_linux-64-13.3.0-h1e990d8_2.conda#f46cf0acdcb6019397d37df1e407ab91 +https://conda.anaconda.org/conda-forge/linux-64/graphite2-1.3.13-h59595ed_1003.conda#f87c7b7c2cb45f323ffbce941c78ab7c +https://conda.anaconda.org/conda-forge/linux-64/icu-75.1-he02047a_0.conda#8b189310083baabfb622af68fd9d3ae3 +https://conda.anaconda.org/conda-forge/linux-64/krb5-1.21.3-h659f571_0.conda#3f43953b7d3fb3aaa1d0d0723d91e368 +https://conda.anaconda.org/conda-forge/linux-64/libaec-1.1.3-h59595ed_0.conda#5e97e271911b8b2001a8b71860c32faa +https://conda.anaconda.org/conda-forge/linux-64/libdrm-2.4.124-hb9d3cd8_0.conda#8bc89311041d7fcb510238cf0848ccae +https://conda.anaconda.org/conda-forge/linux-64/libfreetype6-2.13.3-h48d6fc4_1.conda#3c255be50a506c50765a93a6644f32fe +https://conda.anaconda.org/conda-forge/linux-64/libjxl-0.11.1-h7b0646d_1.conda#959fc2b6c0df7883e070b3fe525219a5 +https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.29-pthreads_h94d23a6_0.conda#0a4d0252248ef9a0f88f2ba8b8a08e12 +https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.7.0-hf01ce69_5.conda#e79a094918988bb1807462cd42c83962 +https://conda.anaconda.org/conda-forge/linux-64/libzopfli-1.0.3-h9c3ff4c_0.tar.bz2#c66fe2d123249af7651ebde8984c51c2 +https://conda.anaconda.org/conda-forge/linux-64/pcre2-10.45-hc749103_0.conda#b90bece58b4c2bf25969b70f3be42d25 +https://conda.anaconda.org/conda-forge/linux-64/python-3.10.17-hd6af730_0_cpython.conda#7bb89638dae9ce1b8e051d0b721e83c2 +https://conda.anaconda.org/conda-forge/linux-64/qhull-2020.2-h434a139_5.conda#353823361b1d27eb3960efb076dfcaf6 +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-0.4.1-hb711507_2.conda#8637c3e5821654d0edf97e2b0404b443 +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-keysyms-0.4.1-hb711507_0.conda#ad748ccca349aec3e91743e08b5e2b50 +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-renderutil-0.3.10-hb711507_0.conda#0e0cbe0564d03a99afd5fd7b362feecd +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-wm-0.4.2-hb711507_0.conda#608e0ef8256b81d04456e8d211eee3e8 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.6-he73a12e_0.conda#1c74ff8c35dcadf952a16f752ca5aa49 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.8.12-h4f16b4b_0.conda#db038ce880f100acc74dba10302b5630 +https://conda.anaconda.org/conda-forge/noarch/alabaster-1.0.0-pyhd8ed1ab_1.conda#1fd9696649f65fd6611fcdb4ffec738a +https://conda.anaconda.org/conda-forge/linux-64/brotli-1.1.0-hb9d3cd8_2.conda#98514fe74548d768907ce7a13f680e8f +https://conda.anaconda.org/conda-forge/linux-64/brotli-python-1.1.0-py310hf71b8c6_2.conda#bf502c169c71e3c6ac0d6175addfacc2 +https://conda.anaconda.org/conda-forge/noarch/certifi-2025.4.26-pyhd8ed1ab_0.conda#c33eeaaa33f45031be34cda513df39b6 +https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.4.2-pyhd8ed1ab_0.conda#40fe4284b8b5835a9073a645139f35af +https://conda.anaconda.org/conda-forge/noarch/click-8.2.1-pyh707e725_0.conda#94b550b8d3a614dbd326af798c7dfb40 +https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda#962b9857ee8e7018c22f2776ffa0b2d7 +https://conda.anaconda.org/conda-forge/noarch/cpython-3.10.17-py310hd8ed1ab_0.conda#e2b81369f0473107784f8b7da8e6a8e9 +https://conda.anaconda.org/conda-forge/noarch/cycler-0.12.1-pyhd8ed1ab_1.conda#44600c4667a319d67dbe0681fc0bc833 +https://conda.anaconda.org/conda-forge/linux-64/cyrus-sasl-2.1.27-h54b06d7_7.conda#dce22f70b4e5a407ce88f2be046f4ceb +https://conda.anaconda.org/conda-forge/linux-64/cython-3.1.1-py310had8cdd9_1.conda#4904cb1ba6e72940ff22a5235554532d +https://conda.anaconda.org/conda-forge/noarch/docutils-0.21.2-pyhd8ed1ab_1.conda#24c1ca34138ee57de72a943237cde4cc +https://conda.anaconda.org/conda-forge/noarch/execnet-2.1.1-pyhd8ed1ab_1.conda#a71efeae2c160f6789900ba2631a2c90 +https://conda.anaconda.org/conda-forge/linux-64/gcc-13.3.0-h9576a4e_2.conda#d92e51bf4b6bdbfe45e5884fb0755afe +https://conda.anaconda.org/conda-forge/linux-64/gcc_linux-64-13.3.0-hc28eda2_10.conda#d151142bbafe5e68ec7fc065c5e6f80c +https://conda.anaconda.org/conda-forge/linux-64/gfortran_impl_linux-64-13.3.0-h84c1745_2.conda#4e21ed177b76537067736f20f54fee0a +https://conda.anaconda.org/conda-forge/linux-64/gxx_impl_linux-64-13.3.0-hae580e1_2.conda#b55f02540605c322a47719029f8404cc +https://conda.anaconda.org/conda-forge/noarch/hpack-4.1.0-pyhd8ed1ab_0.conda#0a802cb9888dd14eeefc611f05c40b6e +https://conda.anaconda.org/conda-forge/noarch/hyperframe-6.1.0-pyhd8ed1ab_0.conda#8e6923fc12f1fe8f8c4e5c9f343256ac +https://conda.anaconda.org/conda-forge/noarch/idna-3.10-pyhd8ed1ab_1.conda#39a4f67be3286c86d696df570b1201b7 +https://conda.anaconda.org/conda-forge/noarch/imagesize-1.4.1-pyhd8ed1ab_0.tar.bz2#7de5386c8fea29e76b303f37dde4c352 +https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_1.conda#6837f3eff7dcea42ecd714ce1ac2b108 +https://conda.anaconda.org/conda-forge/linux-64/kiwisolver-1.4.7-py310h3788b33_0.conda#4186d9b4d004b0fe0de6aa62496fb48a +https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.17-h717163a_0.conda#000e85703f0fd9594c81710dd5066471 +https://conda.anaconda.org/conda-forge/linux-64/libavif16-1.3.0-h766b0b6_0.conda#f17f2d0e5c9ad6b958547fd67b155771 +https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-31_h59b9bed_openblas.conda#728dbebd0f7a20337218beacffd37916 +https://conda.anaconda.org/conda-forge/linux-64/libcups-2.3.3-h4637d8d_4.conda#d4529f4dff3057982a7617c7ac58fde3 +https://conda.anaconda.org/conda-forge/linux-64/libfreetype-2.13.3-ha770c72_1.conda#51f5be229d83ecd401fb369ab96ae669 +https://conda.anaconda.org/conda-forge/linux-64/libglib-2.84.2-h3618099_0.conda#072ab14a02164b7c0c089055368ff776 +https://conda.anaconda.org/conda-forge/linux-64/libglx-1.7.0-ha4b6fd6_2.conda#c8013e438185f33b13814c5c488acd5c +https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.13.8-h4bc477f_0.conda#14dbe05b929e329dbaa6f2d0aa19466d +https://conda.anaconda.org/conda-forge/linux-64/markupsafe-3.0.2-py310h89163eb_1.conda#8ce3f0332fd6de0d737e2911d329523f +https://conda.anaconda.org/conda-forge/noarch/meson-1.8.1-pyhe01879c_0.conda#f3cccd9a6ce5331ae33f69ade5529162 +https://conda.anaconda.org/conda-forge/noarch/munkres-1.1.4-pyh9f0ad1d_0.tar.bz2#2ba8498c1018c1e9c61eb99b973dfe19 +https://conda.anaconda.org/conda-forge/noarch/narwhals-1.41.0-pyhe01879c_0.conda#580a340cc0f5eab2b18adb1b55e032e5 +https://conda.anaconda.org/conda-forge/noarch/networkx-3.4.2-pyh267e887_2.conda#fd40bf7f7f4bc4b647dc8512053d9873 +https://conda.anaconda.org/conda-forge/linux-64/openblas-0.3.29-pthreads_h6ec200e_0.conda#7e4d48870b3258bea920d51b7f495a81 +https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.3-h5fbd93e_0.conda#9e5816bc95d285c115a3ebc2f8563564 +https://conda.anaconda.org/conda-forge/noarch/packaging-25.0-pyh29332c3_1.conda#58335b26c38bf4a20f399384c33cbcf9 +https://conda.anaconda.org/conda-forge/noarch/platformdirs-4.3.8-pyhe01879c_0.conda#424844562f5d337077b445ec6b1398a7 +https://conda.anaconda.org/conda-forge/noarch/pluggy-1.6.0-pyhd8ed1ab_0.conda#7da7ccd349dbf6487a7778579d2bb971 +https://conda.anaconda.org/conda-forge/linux-64/psutil-7.0.0-py310ha75aee5_0.conda#da7d592394ff9084a23f62a1186451a2 +https://conda.anaconda.org/conda-forge/noarch/pycparser-2.22-pyh29332c3_1.conda#12c566707c80111f9799308d9e265aef +https://conda.anaconda.org/conda-forge/noarch/pygments-2.19.1-pyhd8ed1ab_0.conda#232fb4577b6687b2d503ef8e254270c9 +https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.2.3-pyhd8ed1ab_1.conda#513d3c262ee49b54a8fec85c5bc99764 +https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha55dd90_7.conda#461219d1a5bd61342293efa2c0c90eac +https://conda.anaconda.org/conda-forge/noarch/python-tzdata-2025.2-pyhd8ed1ab_0.conda#88476ae6ebd24f39261e0854ac244f33 +https://conda.anaconda.org/conda-forge/noarch/pytz-2025.2-pyhd8ed1ab_0.conda#bc8e3267d44011051f2eb14d22fb0960 +https://conda.anaconda.org/conda-forge/noarch/setuptools-80.9.0-pyhff2d567_0.conda#4de79c071274a53dcaf2a8c749d1499e +https://conda.anaconda.org/conda-forge/noarch/six-1.17.0-pyhd8ed1ab_0.conda#a451d576819089b0d672f18768be0f65 +https://conda.anaconda.org/conda-forge/noarch/snowballstemmer-3.0.1-pyhd8ed1ab_0.conda#755cf22df8693aa0d1aec1c123fa5863 +https://conda.anaconda.org/conda-forge/noarch/soupsieve-2.7-pyhd8ed1ab_0.conda#fb32097c717486aa34b38a9db57eb49e +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-jsmath-1.0.1-pyhd8ed1ab_1.conda#fa839b5ff59e192f411ccc7dae6588bb +https://conda.anaconda.org/conda-forge/noarch/tabulate-0.9.0-pyhd8ed1ab_2.conda#959484a66b4b76befcddc4fa97c95567 +https://conda.anaconda.org/conda-forge/noarch/threadpoolctl-3.6.0-pyhecae5ae_0.conda#9d64911b31d57ca443e9f1e36b04385f +https://conda.anaconda.org/conda-forge/noarch/tomli-2.2.1-pyhd8ed1ab_1.conda#ac944244f1fed2eb49bae07193ae8215 +https://conda.anaconda.org/conda-forge/linux-64/tornado-6.5.1-py310ha75aee5_0.conda#6f3da1072c0c4d2a1beb1e84615f7c9c +https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.13.2-pyh29332c3_0.conda#83fc6ae00127671e301c9f44254c31b8 +https://conda.anaconda.org/conda-forge/linux-64/unicodedata2-16.0.0-py310ha75aee5_0.conda#1d7a4b9202cdd10d56ecdd7f6c347190 +https://conda.anaconda.org/conda-forge/noarch/wheel-0.45.1-pyhd8ed1ab_1.conda#75cb7132eb58d97896e173ef12ac9986 +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-image-0.4.0-hb711507_2.conda#a0901183f08b6c7107aab109733a3c91 +https://conda.anaconda.org/conda-forge/linux-64/xkeyboard-config-2.44-hb9d3cd8_0.conda#7c91bfc90672888259675ad2ad28af9c +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.6-hb9d3cd8_0.conda#febbab7d15033c913d53c7a2c102309d +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxfixes-6.0.1-hb9d3cd8_0.conda#4bdb303603e9821baf5fe5fdff1dc8f8 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.12-hb9d3cd8_0.conda#96d57aba173e878a2089d5638016dc5e +https://conda.anaconda.org/conda-forge/noarch/zipp-3.22.0-pyhd8ed1ab_0.conda#234be740b00b8e41567e5b0ed95aaba9 +https://conda.anaconda.org/conda-forge/noarch/accessible-pygments-0.0.5-pyhd8ed1ab_1.conda#74ac5069774cdbc53910ec4d631a3999 +https://conda.anaconda.org/conda-forge/noarch/babel-2.17.0-pyhd8ed1ab_0.conda#0a01c169f0ab0f91b26e77a3301fbfe4 +https://conda.anaconda.org/conda-forge/linux-64/brunsli-0.1-h9c3ff4c_0.tar.bz2#c1ac6229d0bfd14f8354ff9ad2a26cad +https://conda.anaconda.org/conda-forge/linux-64/c-compiler-1.9.0-h2b85faf_0.conda#3cb814f83f1f71ac1985013697f80cc1 +https://conda.anaconda.org/conda-forge/linux-64/cffi-1.17.1-py310h8deb56e_0.conda#1fc24a3196ad5ede2a68148be61894f4 +https://conda.anaconda.org/conda-forge/linux-64/dbus-1.16.2-h3c4dab8_0.conda#679616eb5ad4e521c83da4650860aba7 +https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.0-pyhd8ed1ab_0.conda#72e42d28960d875c7654614f8b50939a +https://conda.anaconda.org/conda-forge/linux-64/fonttools-4.58.1-py310h89163eb_0.conda#f4f46207c6defa5ea17b0299298ba849 +https://conda.anaconda.org/conda-forge/linux-64/freetype-2.13.3-ha770c72_1.conda#9ccd736d31e0c6e41f54e704e5312811 +https://conda.anaconda.org/conda-forge/linux-64/gfortran-13.3.0-h9576a4e_2.conda#19e6d3c9cde10a0a9a170a684082588e +https://conda.anaconda.org/conda-forge/linux-64/gfortran_linux-64-13.3.0-hb919d3a_10.conda#7ce070e3329cd10bf79dbed562a21bd4 +https://conda.anaconda.org/conda-forge/linux-64/gxx-13.3.0-h9576a4e_2.conda#07e8df00b7cd3084ad3ef598ce32a71c +https://conda.anaconda.org/conda-forge/linux-64/gxx_linux-64-13.3.0-h6834431_10.conda#9a8ebde471cec5cc9c48f8682f434f92 +https://conda.anaconda.org/conda-forge/noarch/h2-4.2.0-pyhd8ed1ab_0.conda#b4754fb1bdcb70c8fd54f918301582c6 +https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-8.7.0-pyhe01879c_1.conda#63ccfdc3a3ce25b027b8767eb722fca8 +https://conda.anaconda.org/conda-forge/noarch/importlib_resources-6.5.2-pyhd8ed1ab_0.conda#c85c76dc67d75619a92f51dfbce06992 +https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.6-pyhd8ed1ab_0.conda#446bd6c8cb26050d528881df495ce646 +https://conda.anaconda.org/conda-forge/noarch/joblib-1.5.1-pyhd8ed1ab_0.conda#fb1c14694de51a476ce8636d92b6f42c +https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-31_he106b2a_openblas.conda#abb32c727da370c481a1c206f5159ce9 +https://conda.anaconda.org/conda-forge/linux-64/libgl-1.7.0-ha4b6fd6_2.conda#928b8be80851f5d8ffb016f9c81dae7a +https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-31_h7ac8fdf_openblas.conda#452b98eafe050ecff932f0ec832dd03f +https://conda.anaconda.org/conda-forge/linux-64/libllvm20-20.1.6-he9d0ab4_0.conda#bf8ccdd2c1c1a54a3fa25bb61f26460e +https://conda.anaconda.org/conda-forge/linux-64/libxkbcommon-1.10.0-h65c71a3_0.conda#fedf6bfe5d21d21d2b1785ec00a8889a +https://conda.anaconda.org/conda-forge/linux-64/libxslt-1.1.39-h76b75d6_0.conda#e71f31f8cfb0a91439f2086fc8aa0461 +https://conda.anaconda.org/conda-forge/noarch/memory_profiler-0.61.0-pyhd8ed1ab_1.conda#71abbefb6f3b95e1668cd5e0af3affb9 +https://conda.anaconda.org/conda-forge/linux-64/openldap-2.6.10-he970967_0.conda#2e5bf4f1da39c0b32778561c3c4e5878 +https://conda.anaconda.org/conda-forge/linux-64/pillow-11.2.1-py310h7e6dc6c_0.conda#5645a243d90adb50909b9edc209d84fe +https://conda.anaconda.org/conda-forge/noarch/pip-25.1.1-pyh8b19718_0.conda#32d0781ace05105cc99af55d36cbec7c +https://conda.anaconda.org/conda-forge/noarch/plotly-6.1.2-pyhd8ed1ab_0.conda#f547ee092ef42452ddaffdfa59ff4987 +https://conda.anaconda.org/conda-forge/noarch/pyproject-metadata-0.9.1-pyhd8ed1ab_0.conda#22ae7c6ea81e0c8661ef32168dda929b +https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhff2d567_1.conda#5ba79d7c71f03c678c8ead841f347d6e +https://conda.anaconda.org/conda-forge/noarch/python-gil-3.10.17-hd8ed1ab_0.conda#c856adbd93a57004e21cd26564f4f724 +https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.13.2-h0e9735f_0.conda#568ed1300869dca0ba09fb750cda5dbb +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-cursor-0.1.5-hb9d3cd8_0.conda#eb44b3b6deb1cab08d72cb61686fe64c +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxcomposite-0.4.6-hb9d3cd8_2.conda#d3c295b50f092ab525ffe3c2aa4b7413 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxcursor-1.2.3-hb9d3cd8_0.conda#2ccd714aa2242315acaf0a67faea780b +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdamage-1.1.6-hb9d3cd8_0.conda#b5fcc7172d22516e1f965490e65e33a4 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxi-1.8.2-hb9d3cd8_0.conda#17dcc85db3c7886650b8908b183d6876 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrandr-1.5.4-hb9d3cd8_0.conda#2de7f99d6581a4a7adbff607b5c278ca +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxxf86vm-1.1.6-hb9d3cd8_0.conda#5efa5fa6243a622445fdfd72aee15efa +https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda#aaa2a381ccc56eac91d63b6c1240312f +https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.13.4-pyha770c72_0.conda#9f07c4fc992adb2d6c30da7fab3959a7 +https://conda.anaconda.org/conda-forge/linux-64/cxx-compiler-1.9.0-h1a2810e_0.conda#1ce8b218d359d9ed0ab481f2a3f3c512 +https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.15.0-h7e30c49_1.conda#8f5b0b297b59e1ac160ad4beec99dbee +https://conda.anaconda.org/conda-forge/linux-64/fortran-compiler-1.9.0-h36df796_0.conda#cc0cf942201f9d3b0e9654ea02e12486 +https://conda.anaconda.org/conda-forge/noarch/importlib-resources-6.5.2-pyhd8ed1ab_0.conda#e376ea42e9ae40f3278b0f79c9bf9826 +https://conda.anaconda.org/conda-forge/noarch/lazy-loader-0.4-pyhd8ed1ab_2.conda#d10d9393680734a8febc4b362a4c94f2 +https://conda.anaconda.org/conda-forge/linux-64/libclang-cpp20.1-20.1.6-default_h1df26ce_0.conda#99ead3b974685e44df8b1e3953503cfc +https://conda.anaconda.org/conda-forge/linux-64/libclang13-20.1.6-default_he06ed0a_0.conda#cc6c469d9d7fc0ac106cef5f45d973a9 +https://conda.anaconda.org/conda-forge/linux-64/liblapacke-3.9.0-31_he2f377e_openblas.conda#7e5fff7d0db69be3a266f7e79a3bb0e2 +https://conda.anaconda.org/conda-forge/linux-64/libpq-17.5-h27ae623_0.conda#6458be24f09e1b034902ab44fe9de908 +https://conda.anaconda.org/conda-forge/noarch/meson-python-0.18.0-pyh70fd9c4_0.conda#576c04b9d9f8e45285fb4d9452c26133 +https://conda.anaconda.org/conda-forge/linux-64/numpy-2.2.6-py310hefbff90_0.conda#b0cea2c364bf65cd19e023040eeab05d +https://conda.anaconda.org/conda-forge/noarch/pytest-8.3.5-pyhd8ed1ab_0.conda#c3c9316209dec74a705a36797970c6be +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxtst-1.2.5-hb9d3cd8_3.conda#7bbe9a0cc0df0ac5f5a8ad6d6a11af2f +https://conda.anaconda.org/conda-forge/linux-64/zstandard-0.23.0-py310ha75aee5_2.conda#f9254b5b0193982416b91edcb4b2676f +https://conda.anaconda.org/conda-forge/linux-64/blas-devel-3.9.0-31_h1ea3ea9_openblas.conda#ba652ee0576396d4765e567f043c57f9 +https://conda.anaconda.org/conda-forge/linux-64/cairo-1.18.4-h3394656_0.conda#09262e66b19567aff4f592fb53b28760 +https://conda.anaconda.org/conda-forge/linux-64/compilers-1.9.0-ha770c72_0.conda#5859096e397aba423340d0bbbb11ec64 +https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.3.2-py310h3788b33_0.conda#b6420d29123c7c823de168f49ccdfe6a +https://conda.anaconda.org/conda-forge/linux-64/imagecodecs-2025.3.30-py310h481ba9f_0.conda#453c8da1b70f7b76b3884e18015bc568 +https://conda.anaconda.org/conda-forge/noarch/imageio-2.37.0-pyhfb79c49_0.conda#b5577bc2212219566578fd5af9993af6 +https://conda.anaconda.org/conda-forge/linux-64/pandas-2.2.3-py310h5eaa309_3.conda#07697a584fab513ce895c4511f7a2403 +https://conda.anaconda.org/conda-forge/noarch/patsy-1.0.1-pyhd8ed1ab_1.conda#ee23fabfd0a8c6b8d6f3729b47b2859d +https://conda.anaconda.org/conda-forge/linux-64/polars-default-1.30.0-py39hfac2b71_0.conda#cd33cf1e631b4d766858c90e333b4832 +https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.7.0-pyhd8ed1ab_0.conda#15353a2a0ea6dfefaa52fc5ab5b98f41 +https://conda.anaconda.org/conda-forge/linux-64/pywavelets-1.8.0-py310hf462985_0.conda#4c441eff2be2e65bd67765c5642051c5 +https://conda.anaconda.org/conda-forge/linux-64/scipy-1.15.2-py310h1d65ade_0.conda#8c29cd33b64b2eb78597fa28b5595c8d +https://conda.anaconda.org/conda-forge/noarch/towncrier-24.8.0-pyhd8ed1ab_1.conda#820b6a1ddf590fba253f8204f7200d82 +https://conda.anaconda.org/conda-forge/noarch/urllib3-2.4.0-pyhd8ed1ab_0.conda#c1e349028e0052c4eea844e94f773065 +https://conda.anaconda.org/conda-forge/linux-64/blas-2.131-openblas.conda#38b2ec894c69bb4be0e66d2ef7fc60bf +https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-11.2.1-h3beb420_0.conda#0e6e192d4b3d95708ad192d957cf3163 +https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.10.3-py310h68603db_0.conda#50084ca38bf28440e2762966bac143fc +https://conda.anaconda.org/conda-forge/linux-64/polars-1.30.0-default_h1443d73_0.conda#19698b29e8544d2dd615699826037039 +https://conda.anaconda.org/conda-forge/linux-64/pyamg-5.2.1-py310ha2bacc8_1.conda#817d32861729e14f474249f1036291c4 +https://conda.anaconda.org/conda-forge/noarch/requests-2.32.3-pyhd8ed1ab_1.conda#a9b9368f3701a417eac9edbcae7cb737 +https://conda.anaconda.org/conda-forge/linux-64/statsmodels-0.14.4-py310hf462985_0.conda#636d3c500d8a851e377360e88ec95372 +https://conda.anaconda.org/conda-forge/noarch/tifffile-2025.5.10-pyhd8ed1ab_0.conda#1fdb801f28bf4987294c49aaa314bf5e +https://conda.anaconda.org/conda-forge/noarch/pooch-1.8.2-pyhd8ed1ab_1.conda#b3e783e8e8ed7577cf0b6dee37d1fbac +https://conda.anaconda.org/conda-forge/linux-64/qt6-main-6.9.0-h0384650_3.conda#8aa69e15597a205fd6f81781fe62c232 +https://conda.anaconda.org/conda-forge/linux-64/scikit-image-0.25.2-py310h5eaa309_1.conda#ed21ab72d049ecdb60f829f04b4dca1c +https://conda.anaconda.org/conda-forge/noarch/seaborn-base-0.13.2-pyhd8ed1ab_3.conda#fd96da444e81f9e6fcaac38590f3dd42 +https://conda.anaconda.org/conda-forge/linux-64/pyside6-6.9.0-py310hfd10a26_0.conda#1610ccfe262ee519716bb69bd4395572 +https://conda.anaconda.org/conda-forge/noarch/seaborn-0.13.2-hd8ed1ab_3.conda#62afb877ca2c2b4b6f9ecb37320085b6 +https://conda.anaconda.org/conda-forge/linux-64/matplotlib-3.10.3-py310hff52083_0.conda#4162a00ddf1d805557aff34ddf113f46 +https://conda.anaconda.org/conda-forge/noarch/numpydoc-1.8.0-pyhd8ed1ab_1.conda#5af206d64d18d6c8dfb3122b4d9e643b +https://conda.anaconda.org/conda-forge/noarch/pydata-sphinx-theme-0.16.1-pyhd8ed1ab_0.conda#837aaf71ddf3b27acae0e7e9015eebc6 +https://conda.anaconda.org/conda-forge/noarch/sphinx-copybutton-0.5.2-pyhd8ed1ab_1.conda#bf22cb9c439572760316ce0748af3713 +https://conda.anaconda.org/conda-forge/noarch/sphinx-design-0.6.1-pyhd8ed1ab_2.conda#3e6c15d914b03f83fc96344f917e0838 +https://conda.anaconda.org/conda-forge/noarch/sphinx-gallery-0.19.0-pyhd8ed1ab_0.conda#3cfa26d23bd7987d84051879f202a855 +https://conda.anaconda.org/conda-forge/noarch/sphinx-prompt-1.4.0-pyhd8ed1ab_0.tar.bz2#88ee91e8679603f2a5bd036d52919cc2 +https://conda.anaconda.org/conda-forge/noarch/sphinx-remove-toctrees-1.0.0.post1-pyhd8ed1ab_1.conda#b275c865b753413caaa8548b9d44c024 +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-applehelp-2.0.0-pyhd8ed1ab_1.conda#16e3f039c0aa6446513e94ab18a8784b +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-devhelp-2.0.0-pyhd8ed1ab_1.conda#910f28a05c178feba832f842155cbfff +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-htmlhelp-2.1.0-pyhd8ed1ab_1.conda#e9fb3fe8a5b758b4aff187d434f94f03 +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-qthelp-2.0.0-pyhd8ed1ab_1.conda#00534ebcc0375929b45c3039b5ba7636 +https://conda.anaconda.org/conda-forge/noarch/sphinx-8.1.3-pyhd8ed1ab_1.conda#1a3281a0dc355c02b5506d87db2d78ac +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-serializinghtml-1.1.10-pyhd8ed1ab_1.conda#3bc61f7161d28137797e038263c04c54 +https://conda.anaconda.org/conda-forge/noarch/sphinxext-opengraph-0.9.1-pyhd8ed1ab_1.conda#79f5d05ad914baf152fb7f75073fe36d +# pip attrs @ https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl#sha256=427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3 +# pip cloudpickle @ https://files.pythonhosted.org/packages/7e/e8/64c37fadfc2816a7701fa8a6ed8d87327c7d54eacfbfb6edab14a2f2be75/cloudpickle-3.1.1-py3-none-any.whl#sha256=c8c5a44295039331ee9dad40ba100a9c7297b6f988e50e87ccdf3765a668350e +# pip defusedxml @ https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl#sha256=a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61 +# pip fastjsonschema @ https://files.pythonhosted.org/packages/90/2b/0817a2b257fe88725c25589d89aec060581aabf668707a8d03b2e9e0cb2a/fastjsonschema-2.21.1-py3-none-any.whl#sha256=c9e5b7e908310918cf494a434eeb31384dd84a98b57a30bcb1f535015b554667 +# pip fqdn @ https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl#sha256=3a179af3761e4df6eb2e026ff9e1a3033d3587bf980a0b1b2e1e5d08d7358014 +# pip json5 @ https://files.pythonhosted.org/packages/41/9f/3500910d5a98549e3098807493851eeef2b89cdd3032227558a104dfe926/json5-0.12.0-py3-none-any.whl#sha256=6d37aa6c08b0609f16e1ec5ff94697e2cbbfbad5ac112afa05794da9ab7810db +# pip jsonpointer @ https://files.pythonhosted.org/packages/71/92/5e77f98553e9e75130c78900d000368476aed74276eb8ae8796f65f00918/jsonpointer-3.0.0-py2.py3-none-any.whl#sha256=13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942 +# pip jupyterlab-pygments @ https://files.pythonhosted.org/packages/b1/dd/ead9d8ea85bf202d90cc513b533f9c363121c7792674f78e0d8a854b63b4/jupyterlab_pygments-0.3.0-py3-none-any.whl#sha256=841a89020971da1d8693f1a99997aefc5dc424bb1b251fd6322462a1b8842780 +# pip libsass @ https://files.pythonhosted.org/packages/fd/5a/eb5b62641df0459a3291fc206cf5bd669c0feed7814dded8edef4ade8512/libsass-0.23.0-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.whl#sha256=4a218406d605f325d234e4678bd57126a66a88841cb95bee2caeafdc6f138306 +# pip mdurl @ https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl#sha256=84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 +# pip overrides @ https://files.pythonhosted.org/packages/2c/ab/fc8290c6a4c722e5514d80f62b2dc4c4df1a68a41d1364e625c35990fcf3/overrides-7.7.0-py3-none-any.whl#sha256=c7ed9d062f78b8e4c1a7b70bd8796b35ead4d9f510227ef9c5dc7626c60d7e49 +# pip pandocfilters @ https://files.pythonhosted.org/packages/ef/af/4fbc8cab944db5d21b7e2a5b8e9211a03a79852b1157e2c102fcc61ac440/pandocfilters-1.5.1-py2.py3-none-any.whl#sha256=93be382804a9cdb0a7267585f157e5d1731bbe5545a85b268d6f5fe6232de2bc +# pip pkginfo @ https://files.pythonhosted.org/packages/fa/3d/f4f2ba829efb54b6cd2d91349c7463316a9cc55a43fc980447416c88540f/pkginfo-1.12.1.2-py3-none-any.whl#sha256=c783ac885519cab2c34927ccfa6bf64b5a704d7c69afaea583dd9b7afe969343 +# pip prometheus-client @ https://files.pythonhosted.org/packages/50/c7/cee159ba3d7192e84a4c166ec1752f44a5fa859ac0eeda2d73a1da65ab47/prometheus_client-0.22.0-py3-none-any.whl#sha256=c8951bbe64e62b96cd8e8f5d917279d1b9b91ab766793f33d4dce6c228558713 +# pip ptyprocess @ https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl#sha256=4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35 +# pip python-json-logger @ https://files.pythonhosted.org/packages/08/20/0f2523b9e50a8052bc6a8b732dfc8568abbdc42010aef03a2d750bdab3b2/python_json_logger-3.3.0-py3-none-any.whl#sha256=dd980fae8cffb24c13caf6e158d3d61c0d6d22342f932cb6e9deedab3d35eec7 +# pip pyyaml @ https://files.pythonhosted.org/packages/6b/4e/1523cb902fd98355e2e9ea5e5eb237cbc5f3ad5f3075fa65087aa0ecb669/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed +# pip rfc3986-validator @ https://files.pythonhosted.org/packages/9e/51/17023c0f8f1869d8806b979a2bffa3f861f26a3f1a66b094288323fba52f/rfc3986_validator-0.1.1-py2.py3-none-any.whl#sha256=2f235c432ef459970b4306369336b9d5dbdda31b510ca1e327636e01f528bfa9 +# pip rpds-py @ https://files.pythonhosted.org/packages/eb/76/66b523ffc84cf47db56efe13ae7cf368dee2bacdec9d89b9baca5e2e6301/rpds_py-0.25.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=0701942049095741a8aeb298a31b203e735d1c61f4423511d2b1a41dcd8a16da +# pip send2trash @ https://files.pythonhosted.org/packages/40/b0/4562db6223154aa4e22f939003cb92514c79f3d4dccca3444253fd17f902/Send2Trash-1.8.3-py3-none-any.whl#sha256=0c31227e0bd08961c7665474a3d1ef7193929fedda4233843689baa056be46c9 +# pip sniffio @ https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl#sha256=2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 +# pip traitlets @ https://files.pythonhosted.org/packages/00/c0/8f5d070730d7836adc9c9b6408dec68c6ced86b304a9b26a14df072a6e8c/traitlets-5.14.3-py3-none-any.whl#sha256=b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f +# pip types-python-dateutil @ https://files.pythonhosted.org/packages/c5/3f/b0e8db149896005adc938a1e7f371d6d7e9eca4053a29b108978ed15e0c2/types_python_dateutil-2.9.0.20250516-py3-none-any.whl#sha256=2b2b3f57f9c6a61fba26a9c0ffb9ea5681c9b83e69cd897c6b5f668d9c0cab93 +# pip uri-template @ https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl#sha256=a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363 +# pip webcolors @ https://files.pythonhosted.org/packages/60/e8/c0e05e4684d13459f93d312077a9a2efbe04d59c393bc2b8802248c908d4/webcolors-24.11.1-py3-none-any.whl#sha256=515291393b4cdf0eb19c155749a096f779f7d909f7cceea072791cb9095b92e9 +# pip webencodings @ https://files.pythonhosted.org/packages/f4/24/2a3e3df732393fed8b3ebf2ec078f05546de641fe1b667ee316ec1dcf3b7/webencodings-0.5.1-py2.py3-none-any.whl#sha256=a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 +# pip websocket-client @ https://files.pythonhosted.org/packages/5a/84/44687a29792a70e111c5c477230a72c4b957d88d16141199bf9acb7537a3/websocket_client-1.8.0-py3-none-any.whl#sha256=17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526 +# pip anyio @ https://files.pythonhosted.org/packages/a1/ee/48ca1a7c89ffec8b6a0c5d02b89c305671d5ffd8d3c94acf8b8c408575bb/anyio-4.9.0-py3-none-any.whl#sha256=9f76d541cad6e36af7beb62e978876f3b41e3e04f2c1fbf0884604c0a9c4d93c +# pip argon2-cffi-bindings @ https://files.pythonhosted.org/packages/ec/f7/378254e6dd7ae6f31fe40c8649eea7d4832a42243acaf0f1fff9083b2bed/argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=b746dba803a79238e925d9046a63aa26bf86ab2a2fe74ce6b009a1c3f5c8f2ae +# pip arrow @ https://files.pythonhosted.org/packages/f8/ed/e97229a566617f2ae958a6b13e7cc0f585470eac730a73e9e82c32a3cdd2/arrow-1.3.0-py3-none-any.whl#sha256=c728b120ebc00eb84e01882a6f5e7927a53960aa990ce7dd2b10f39005a67f80 +# pip doit @ https://files.pythonhosted.org/packages/44/83/a2960d2c975836daa629a73995134fd86520c101412578c57da3d2aa71ee/doit-0.36.0-py3-none-any.whl#sha256=ebc285f6666871b5300091c26eafdff3de968a6bd60ea35dd1e3fc6f2e32479a +# pip jupyter-core @ https://files.pythonhosted.org/packages/2f/57/6bffd4b20b88da3800c5d691e0337761576ee688eb01299eae865689d2df/jupyter_core-5.8.1-py3-none-any.whl#sha256=c28d268fc90fb53f1338ded2eb410704c5449a358406e8a948b75706e24863d0 +# pip markdown-it-py @ https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl#sha256=355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1 +# pip mistune @ https://files.pythonhosted.org/packages/01/4d/23c4e4f09da849e127e9f123241946c23c1e30f45a88366879e064211815/mistune-3.1.3-py3-none-any.whl#sha256=1a32314113cff28aa6432e99e522677c8587fd83e3d51c29b82a52409c842bd9 +# pip pyzmq @ https://files.pythonhosted.org/packages/c1/3e/2de5928cdadc2105e7c8f890cc5f404136b41ce5b6eae5902167f1d5641c/pyzmq-26.4.0-cp310-cp310-manylinux_2_28_x86_64.whl#sha256=7dacb06a9c83b007cc01e8e5277f94c95c453c5851aac5e83efe93e72226353f +# pip referencing @ https://files.pythonhosted.org/packages/c1/b1/3baf80dc6d2b7bc27a95a67752d0208e410351e3feb4eb78de5f77454d8d/referencing-0.36.2-py3-none-any.whl#sha256=e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0 +# pip rfc3339-validator @ https://files.pythonhosted.org/packages/7b/44/4e421b96b67b2daff264473f7465db72fbdf36a07e05494f50300cc7b0c6/rfc3339_validator-0.1.4-py2.py3-none-any.whl#sha256=24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa +# pip sphinxcontrib-sass @ https://files.pythonhosted.org/packages/3f/ec/194f2dbe55b3fe0941b43286c21abb49064d9d023abfb99305c79ad77cad/sphinxcontrib_sass-0.3.5-py2.py3-none-any.whl#sha256=850c83a36ed2d2059562504ccf496ca626c9c0bb89ec642a2d9c42105704bef6 +# pip terminado @ https://files.pythonhosted.org/packages/6a/9e/2064975477fdc887e47ad42157e214526dcad8f317a948dee17e1659a62f/terminado-0.18.1-py3-none-any.whl#sha256=a4468e1b37bb318f8a86514f65814e1afc977cf29b3992a4500d9dd305dcceb0 +# pip tinycss2 @ https://files.pythonhosted.org/packages/e6/34/ebdc18bae6aa14fbee1a08b63c015c72b64868ff7dae68808ab500c492e2/tinycss2-1.4.0-py3-none-any.whl#sha256=3a49cf47b7675da0b15d0c6e1df8df4ebd96e9394bb905a5775adb0d884c5289 +# pip argon2-cffi @ https://files.pythonhosted.org/packages/a4/6a/e8a041599e78b6b3752da48000b14c8d1e8a04ded09c88c714ba047f34f5/argon2_cffi-23.1.0-py3-none-any.whl#sha256=c670642b78ba29641818ab2e68bd4e6a78ba53b7eff7b4c3815ae16abf91c7ea +# pip bleach @ https://files.pythonhosted.org/packages/fc/55/96142937f66150805c25c4d0f31ee4132fd33497753400734f9dfdcbdc66/bleach-6.2.0-py3-none-any.whl#sha256=117d9c6097a7c3d22fd578fcd8d35ff1e125df6736f554da4e432fdd63f31e5e +# pip isoduration @ https://files.pythonhosted.org/packages/7b/55/e5326141505c5d5e34c5e0935d2908a74e4561eca44108fbfb9c13d2911a/isoduration-20.11.0-py3-none-any.whl#sha256=b2904c2a4228c3d44f409c8ae8e2370eb21a26f7ac2ec5446df141dde3452042 +# pip jsonschema-specifications @ https://files.pythonhosted.org/packages/01/0e/b27cdbaccf30b890c40ed1da9fd4a3593a5cf94dae54fb34f8a4b74fcd3f/jsonschema_specifications-2025.4.1-py3-none-any.whl#sha256=4653bffbd6584f7de83a67e0d620ef16900b390ddc7939d56684d6c81e33f1af +# pip jupyter-client @ https://files.pythonhosted.org/packages/11/85/b0394e0b6fcccd2c1eeefc230978a6f8cb0c5df1e4cd3e7625735a0d7d1e/jupyter_client-8.6.3-py3-none-any.whl#sha256=e8a19cc986cc45905ac3362915f410f3af85424b4c0905e94fa5f2cb08e8f23f +# pip jupyter-server-terminals @ https://files.pythonhosted.org/packages/07/2d/2b32cdbe8d2a602f697a649798554e4f072115438e92249624e532e8aca6/jupyter_server_terminals-0.5.3-py3-none-any.whl#sha256=41ee0d7dc0ebf2809c668e0fc726dfaf258fcd3e769568996ca731b6194ae9aa +# pip jupyterlite-core @ https://files.pythonhosted.org/packages/46/15/1d9160819d1e6e018d15de0e98b9297d0a09cfcfdc73add6e24ee3b2b83c/jupyterlite_core-0.5.1-py3-none-any.whl#sha256=76381619a632f06bf67fb47e5464af762ad8836df5ffe3d7e7ee0e316c1407ee +# pip mdit-py-plugins @ https://files.pythonhosted.org/packages/a7/f7/7782a043553ee469c1ff49cfa1cdace2d6bf99a1f333cf38676b3ddf30da/mdit_py_plugins-0.4.2-py3-none-any.whl#sha256=0c673c3f889399a33b95e88d2f0d111b4447bdfea7f237dab2d488f459835636 +# pip jsonschema @ https://files.pythonhosted.org/packages/a2/3d/023389198f69c722d039351050738d6755376c8fd343e91dc493ea485905/jsonschema-4.24.0-py3-none-any.whl#sha256=a462455f19f5faf404a7902952b6f0e3ce868f3ee09a359b05eca6673bd8412d +# pip jupyterlite-pyodide-kernel @ https://files.pythonhosted.org/packages/1b/b5/959a03ca011d1031abac03c18af9e767c18d6a9beb443eb106dda609748c/jupyterlite_pyodide_kernel-0.5.2-py3-none-any.whl#sha256=63ba6ce28d32f2cd19f636c40c153e171369a24189e11e2235457bd7000c5907 +# pip jupyter-events @ https://files.pythonhosted.org/packages/e2/48/577993f1f99c552f18a0428731a755e06171f9902fa118c379eb7c04ea22/jupyter_events-0.12.0-py3-none-any.whl#sha256=6464b2fa5ad10451c3d35fabc75eab39556ae1e2853ad0c0cc31b656731a97fb +# pip nbformat @ https://files.pythonhosted.org/packages/a9/82/0340caa499416c78e5d8f5f05947ae4bc3cba53c9f038ab6e9ed964e22f1/nbformat-5.10.4-py3-none-any.whl#sha256=3b48d6c8fbca4b299bf3982ea7db1af21580e4fec269ad087b9e81588891200b +# pip jupytext @ https://files.pythonhosted.org/packages/ed/f1/82ea8e783433707cafd9790099a2d19f113c22f32a31c8bb5abdc7a61dbb/jupytext-1.17.2-py3-none-any.whl#sha256=4f85dc43bb6a24b75491c5c434001ad5ef563932f68f15dd3e1c8ce12a4a426b +# pip nbclient @ https://files.pythonhosted.org/packages/34/6d/e7fa07f03a4a7b221d94b4d586edb754a9b0dc3c9e2c93353e9fa4e0d117/nbclient-0.10.2-py3-none-any.whl#sha256=4ffee11e788b4a27fabeb7955547e4318a5298f34342a4bfd01f2e1faaeadc3d +# pip nbconvert @ https://files.pythonhosted.org/packages/cc/9a/cd673b2f773a12c992f41309ef81b99da1690426bd2f96957a7ade0d3ed7/nbconvert-7.16.6-py3-none-any.whl#sha256=1375a7b67e0c2883678c48e506dc320febb57685e5ee67faa51b18a90f3a712b +# pip jupyter-server @ https://files.pythonhosted.org/packages/46/1f/5ebbced977171d09a7b0c08a285ff9a20aafb9c51bde07e52349ff1ddd71/jupyter_server-2.16.0-py3-none-any.whl#sha256=3d8db5be3bc64403b1c65b400a1d7f4647a5ce743f3b20dbdefe8ddb7b55af9e +# pip jupyterlab-server @ https://files.pythonhosted.org/packages/54/09/2032e7d15c544a0e3cd831c51d77a8ca57f7555b2e1b2922142eddb02a84/jupyterlab_server-2.27.3-py3-none-any.whl#sha256=e697488f66c3db49df675158a77b3b017520d772c6e1548c7d9bcc5df7944ee4 +# pip jupyterlite-sphinx @ https://files.pythonhosted.org/packages/b8/68/d35f70a5ae17b30da996c48138c2d655232c2ee839c881ef44587d75d0d3/jupyterlite_sphinx-0.20.1-py3-none-any.whl#sha256=6f477879e9793813b5ed554f08d87b2d949b68595ec5b7570332aa2d0fe0a8c1 diff --git a/build_tools/circle/doc_min_dependencies_environment.yml b/build_tools/circle/doc_min_dependencies_environment.yml new file mode 100644 index 0000000000000..1a93231019fbb --- /dev/null +++ b/build_tools/circle/doc_min_dependencies_environment.yml @@ -0,0 +1,42 @@ +# DO NOT EDIT: this file is generated from the specification found in the +# following script to centralize the configuration for CI builds: +# build_tools/update_environments_and_lock_files.py +channels: + - conda-forge +dependencies: + - python=3.10 + - numpy=1.22.0 # min + - blas + - scipy=1.8.0 # min + - cython=3.0.10 # min + - joblib + - threadpoolctl + - matplotlib=3.5.0 # min + - pandas=1.4.0 # min + - pyamg=4.2.1 # min + - pytest + - pytest-xdist + - pillow + - pip + - ninja + - meson-python + - scikit-image=0.19.0 # min + - seaborn + - memory_profiler + - compilers + - sphinx=7.3.7 # min + - sphinx-gallery=0.17.1 # min + - sphinx-copybutton=0.5.2 # min + - numpydoc=1.2.0 # min + - sphinx-prompt=1.4.0 # min + - plotly=5.14.0 # min + - polars=0.20.30 # min + - pooch=1.6.0 # min + - sphinx-remove-toctrees=1.0.0.post1 # min + - sphinx-design=0.6.0 # min + - pydata-sphinx-theme=0.15.3 # min + - towncrier=24.8.0 # min + - pip + - pip: + - sphinxext-opengraph==0.9.1 # min + - sphinxcontrib-sass==0.3.4 # min diff --git a/build_tools/circle/doc_min_dependencies_linux-64_conda.lock b/build_tools/circle/doc_min_dependencies_linux-64_conda.lock new file mode 100644 index 0000000000000..89d34a59f3b7b --- /dev/null +++ b/build_tools/circle/doc_min_dependencies_linux-64_conda.lock @@ -0,0 +1,296 @@ +# Generated by conda-lock. +# platform: linux-64 +# input_hash: cf86af2534e8e281654ed19bc893b468656b355b2b200b12321dbc61cce562db +@EXPLICIT +https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81 +https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2#0c96522c6bdaed4b1566d11387caaf45 +https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2#34893075a5c9e55cdafac56607368fc6 +https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2#4d59c254e01d9cde7957100457e2d5fb +https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-h77eed37_3.conda#49023d73832ef61042f6a237cb2687e7 +https://conda.anaconda.org/conda-forge/noarch/kernel-headers_linux-64-3.10.0-he073ed8_18.conda#ad8527bf134a90e1c9ed35fa0b64318c +https://conda.anaconda.org/conda-forge/noarch/python_abi-3.10-7_cp310.conda#44e871cba2b162368476a84b8d040b6c +https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda#4222072737ccff51314b5ece9c7d6f5a +https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.4.26-hbd8a1cb_0.conda#95db94f75ba080a22eb623590993167b +https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-0.tar.bz2#f766549260d6815b0c52253f1fb1bb29 +https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.43-h712a8e2_4.conda#01f8d123c96816249efd255a31ad7712 +https://conda.anaconda.org/conda-forge/noarch/libgcc-devel_linux-64-13.3.0-hc03c837_102.conda#4c1d6961a6a54f602ae510d9bf31fa60 +https://conda.anaconda.org/conda-forge/linux-64/libglvnd-1.7.0-ha4b6fd6_2.conda#434ca7e50e40f4918ab701e3facd59a0 +https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.1.0-h767d61c_2.conda#fbe7d535ff9d3a168c148e07358cd5b1 +https://conda.anaconda.org/conda-forge/noarch/libstdcxx-devel_linux-64-13.3.0-hc03c837_102.conda#aa38de2738c5f4a72a880e3d31ffe8b4 +https://conda.anaconda.org/conda-forge/noarch/sysroot_linux-64-2.17-h0157908_18.conda#460eba7851277ec1fd80a1a24080787a +https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2#73aaf86a425cc6e73fcf236a5a46396d +https://conda.anaconda.org/conda-forge/linux-64/binutils_impl_linux-64-2.43-h4bf12b8_4.conda#ef67db625ad0d2dce398837102f875ed +https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2#fee5683a3f04bd15cbd8318b096a27ab +https://conda.anaconda.org/conda-forge/linux-64/libegl-1.7.0-ha4b6fd6_2.conda#c151d5eb730e9b7480e6d48c0fc44048 +https://conda.anaconda.org/conda-forge/linux-64/libopengl-1.7.0-ha4b6fd6_2.conda#7df50d44d4a14d6c31a2c54f2cd92157 +https://conda.anaconda.org/conda-forge/linux-64/binutils-2.43-h4852527_4.conda#29782348a527eda3ecfc673109d28e93 +https://conda.anaconda.org/conda-forge/linux-64/binutils_linux-64-2.43-h4852527_4.conda#c87e146f5b685672d4aa6b527c6d3b5e +https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.1.0-h767d61c_2.conda#ea8ac52380885ed41c1baa8f1d6d2b93 +https://conda.anaconda.org/conda-forge/linux-64/alsa-lib-1.2.14-hb9d3cd8_0.conda#76df83c2a9035c54df5d04ff81bcc02d +https://conda.anaconda.org/conda-forge/linux-64/gettext-tools-0.24.1-h5888daf_0.conda#d54305672f0361c2f3886750e7165b5f +https://conda.anaconda.org/conda-forge/linux-64/libbrotlicommon-1.1.0-hb9d3cd8_2.conda#41b599ed2b02abcfdd84302bff174b23 +https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.24-h86f0d12_0.conda#64f0c503da58ec25ebd359e4d990afa8 +https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.7.0-h5888daf_0.conda#db0bfbe7dd197b68ad5f30333bae6ce0 +https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.6-h2dba641_1.conda#ede4673863426c0883c0063d853bbd85 +https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-15.1.0-h69a702a_2.conda#ddca86c7040dd0e73b2b69bd7833d225 +https://conda.anaconda.org/conda-forge/linux-64/libgettextpo-0.24.1-h5888daf_0.conda#2ee6d71b72f75d50581f2f68e965efdb +https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-15.1.0-hcea5267_2.conda#01de444988ed960031dbe84cf4f9b1fc +https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.18-h4ce23a2_1.conda#e796ff8ddc598affdf7c173d6145f087 +https://conda.anaconda.org/conda-forge/linux-64/libjpeg-turbo-3.1.0-hb9d3cd8_0.conda#9fa334557db9f63da6c9285fd2a48638 +https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.1-hb9d3cd8_1.conda#a76fd702c93cd2dfd89eff30a5fd45a8 +https://conda.anaconda.org/conda-forge/linux-64/libntlm-1.8-hb9d3cd8_0.conda#7c7927b404672409d9917d49bff5f2d6 +https://conda.anaconda.org/conda-forge/linux-64/libogg-1.3.5-hd0c01bc_1.conda#68e52064ed3897463c0e958ab5c8f91b +https://conda.anaconda.org/conda-forge/linux-64/libopus-1.5.2-hd0c01bc_0.conda#b64523fb87ac6f87f0790f324ad43046 +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-15.1.0-h8f9b012_2.conda#1cb1c67961f6dd257eae9e9691b341aa +https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.5.0-h851e524_0.conda#63f790534398730f59e1b899c3644d4a +https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda#edb0dca6bc32e4f4789199455a1dbeb8 +https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda#47e340acb35de30501a76c7c799c41d7 +https://conda.anaconda.org/conda-forge/linux-64/openssl-3.5.0-h7b32b05_1.conda#de356753cfdbffcde5bb1e86e3aa6cd0 +https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-hb9d3cd8_1002.conda#b3c17d95b5a10c6e64a21fa17573e70e +https://conda.anaconda.org/conda-forge/linux-64/rav1e-0.7.1-h8fae777_3.conda#2c42649888aac645608191ffdc80d13a +https://conda.anaconda.org/conda-forge/linux-64/xorg-libice-1.1.2-hb9d3cd8_0.conda#fb901ff28063514abb6046c9ec2c4a45 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxau-1.0.12-hb9d3cd8_0.conda#f6ebe2cb3f82ba6c057dde5d9debe4f7 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdmcp-1.1.5-hb9d3cd8_0.conda#8035c64cb77ed555e3f150b7b3972480 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxshmfence-1.3.3-hb9d3cd8_0.conda#9a809ce9f65460195777f2f2116bae02 +https://conda.anaconda.org/conda-forge/linux-64/attr-2.5.1-h166bdaf_1.tar.bz2#d9c69a24ad678ffce24c6543a0176b00 +https://conda.anaconda.org/conda-forge/linux-64/blis-0.9.0-h4ab18f5_2.conda#6f77ba1352b69c4a6f8a6d20def30e4e +https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h4bc722e_7.conda#62ee74e96c5ebb0af99386de58cf9553 +https://conda.anaconda.org/conda-forge/linux-64/dav1d-1.2.1-hd590300_0.conda#418c6ca5929a611cbd69204907a83995 +https://conda.anaconda.org/conda-forge/linux-64/giflib-5.2.2-hd590300_0.conda#3bf7b9fd5a7136126e0234db4b87c8b6 +https://conda.anaconda.org/conda-forge/linux-64/jxrlib-1.1-hd590300_3.conda#5aeabe88534ea4169d4c49998f293d6c +https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.1-h166bdaf_0.tar.bz2#30186d27e2c9fa62b45fb1476b7200e3 +https://conda.anaconda.org/conda-forge/linux-64/lame-3.100-h166bdaf_1003.tar.bz2#a8832b479f93521a9e7b5b743803be51 +https://conda.anaconda.org/conda-forge/linux-64/lerc-4.0.0-h0aef613_1.conda#9344155d33912347b37f0ae6c410a835 +https://conda.anaconda.org/conda-forge/linux-64/libasprintf-0.24.1-h8e693c7_0.conda#57566a81dd1e5aa3d98ac7582e8bfe03 +https://conda.anaconda.org/conda-forge/linux-64/libbrotlidec-1.1.0-hb9d3cd8_2.conda#9566f0bd264fbd463002e759b8a82401 +https://conda.anaconda.org/conda-forge/linux-64/libbrotlienc-1.1.0-hb9d3cd8_2.conda#06f70867945ea6a84d35836af780f1de +https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20250104-pl5321h7949ede_0.conda#c277e0a4d549b03ac1e9d6cbbe3d017b +https://conda.anaconda.org/conda-forge/linux-64/libevent-2.1.12-hf998b51_1.conda#a1cfcc585f0c42bf8d5546bb1dfb668d +https://conda.anaconda.org/conda-forge/linux-64/libgettextpo-devel-0.24.1-h5888daf_0.conda#8f04c7aae6a46503bc36d1ed5abc8c7c +https://conda.anaconda.org/conda-forge/linux-64/libgfortran-15.1.0-h69a702a_2.conda#f92e6e0a3c0c0c85561ef61aa59d555d +https://conda.anaconda.org/conda-forge/linux-64/libgpg-error-1.55-h3f2d84a_0.conda#2bd47db5807daade8500ed7ca4c512a4 +https://conda.anaconda.org/conda-forge/linux-64/libhwy-1.2.0-hf40a0c7_0.conda#2f433d593a66044c3f163cb25f0a09de +https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hd590300_0.conda#30fd6e37fe21f86f4bd26d6ee73eeec7 +https://conda.anaconda.org/conda-forge/linux-64/libpciaccess-0.18-hd590300_0.conda#48f4330bfcd959c3cfb704d424903c82 +https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.47-h943b412_0.conda#55199e2ae2c3651f6f9b2a447b47bdc9 +https://conda.anaconda.org/conda-forge/linux-64/libsanitizer-13.3.0-he8ea267_2.conda#2b6cdf7bb95d3d10ef4e38ce0bc95dba +https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.50.0-hee588c1_0.conda#71888e92098d0f8c41b09a671ad289bc +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-15.1.0-h4852527_2.conda#9d2072af184b5caa29492bf2344597bb +https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda#40b61aab5c7ba9ff276c41cfffe6b80b +https://conda.anaconda.org/conda-forge/linux-64/libxcb-1.17.0-h8a09558_0.conda#92ed62436b625154323d40d5f2f11dd7 +https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda#5aa797f8787fe7a17d1b0821485b5adc +https://conda.anaconda.org/conda-forge/linux-64/lz4-c-1.10.0-h5888daf_1.conda#9de5350a85c4a20c685259b889aa6393 +https://conda.anaconda.org/conda-forge/linux-64/mpg123-1.32.9-hc50e24c_0.conda#c7f302fd11eeb0987a6a5e1f3aed6a21 +https://conda.anaconda.org/conda-forge/linux-64/ninja-1.12.1-hff21bea_1.conda#2322531904f27501ee19847b87ba7c64 +https://conda.anaconda.org/conda-forge/linux-64/nspr-4.36-h5888daf_0.conda#de9cd5bca9e4918527b9b72b6e2e1409 +https://conda.anaconda.org/conda-forge/linux-64/pixman-0.46.0-h29eaf8c_0.conda#d2f1c87d4416d1e7344cf92b1aaee1c4 +https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8c095d6_2.conda#283b96675859b20a825f8fa30f311446 +https://conda.anaconda.org/conda-forge/linux-64/snappy-1.2.1-h8bd8927_1.conda#3b3e64af585eadfb52bb90b553db5edf +https://conda.anaconda.org/conda-forge/linux-64/svt-av1-3.0.2-h5888daf_0.conda#0096882bd623e6cc09e8bf920fc8fb47 +https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_hd72426e_102.conda#a0116df4f4ed05c303811a837d5b39d8 +https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h7f98852_2.tar.bz2#4cb3ad778ec2d5a7acbdf254eb1c42ae +https://conda.anaconda.org/conda-forge/linux-64/zfp-1.0.1-h5888daf_2.conda#e0409515c467b87176b070bff5d9442e +https://conda.anaconda.org/conda-forge/linux-64/zlib-ng-2.2.4-h7955e40_0.conda#c8a816dbf59eb8ba6346a8f10014b302 +https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb8e6e7a_2.conda#6432cb5d4ac0046c3ac0a8a0f95842f9 +https://conda.anaconda.org/conda-forge/linux-64/aom-3.9.1-hac33072_0.conda#346722a0be40f6edc53f12640d301338 +https://conda.anaconda.org/conda-forge/linux-64/blosc-1.21.6-he440d0b_1.conda#2c2fae981fd2afd00812c92ac47d023d +https://conda.anaconda.org/conda-forge/linux-64/brotli-bin-1.1.0-hb9d3cd8_2.conda#c63b5e52939e795ba8d26e35d767a843 +https://conda.anaconda.org/conda-forge/linux-64/c-blosc2-2.17.1-h3122c55_0.conda#009d16d3c9ed3e70d58ed46dab1571d1 +https://conda.anaconda.org/conda-forge/linux-64/charls-2.4.2-h59595ed_0.conda#4336bd67920dd504cd8c6761d6a99645 +https://conda.anaconda.org/conda-forge/linux-64/gcc_impl_linux-64-13.3.0-h1e990d8_2.conda#f46cf0acdcb6019397d37df1e407ab91 +https://conda.anaconda.org/conda-forge/linux-64/graphite2-1.3.13-h59595ed_1003.conda#f87c7b7c2cb45f323ffbce941c78ab7c +https://conda.anaconda.org/conda-forge/linux-64/icu-75.1-he02047a_0.conda#8b189310083baabfb622af68fd9d3ae3 +https://conda.anaconda.org/conda-forge/linux-64/krb5-1.21.3-h659f571_0.conda#3f43953b7d3fb3aaa1d0d0723d91e368 +https://conda.anaconda.org/conda-forge/linux-64/libaec-1.1.3-h59595ed_0.conda#5e97e271911b8b2001a8b71860c32faa +https://conda.anaconda.org/conda-forge/linux-64/libasprintf-devel-0.24.1-h8e693c7_0.conda#8f66ed2e34507b7ae44afa31c3e4ec79 +https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-31_h66dfbfd_blis.conda#612d513ce8103e41dbcb4d941a325027 +https://conda.anaconda.org/conda-forge/linux-64/libcap-2.75-h39aace5_0.conda#c44c16d6976d2aebbd65894d7741e67e +https://conda.anaconda.org/conda-forge/linux-64/libdrm-2.4.124-hb9d3cd8_0.conda#8bc89311041d7fcb510238cf0848ccae +https://conda.anaconda.org/conda-forge/linux-64/libfreetype6-2.13.3-h48d6fc4_1.conda#3c255be50a506c50765a93a6644f32fe +https://conda.anaconda.org/conda-forge/linux-64/libgcrypt-lib-1.11.1-hb9d3cd8_0.conda#8504a291085c9fb809b66cabd5834307 +https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-15.1.0-h69a702a_2.conda#a483a87b71e974bb75d1b9413d4436dd +https://conda.anaconda.org/conda-forge/linux-64/libjxl-0.11.1-h7b0646d_1.conda#959fc2b6c0df7883e070b3fe525219a5 +https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.7.0-hf01ce69_5.conda#e79a094918988bb1807462cd42c83962 +https://conda.anaconda.org/conda-forge/linux-64/libvorbis-1.3.7-h9c3ff4c_0.tar.bz2#309dec04b70a3cc0f1e84a4013683bc0 +https://conda.anaconda.org/conda-forge/linux-64/libzopfli-1.0.3-h9c3ff4c_0.tar.bz2#c66fe2d123249af7651ebde8984c51c2 +https://conda.anaconda.org/conda-forge/linux-64/nss-3.112-h159eef7_0.conda#688a8bc02e57e6b741a040c84e931a7d +https://conda.anaconda.org/conda-forge/linux-64/pcre2-10.45-hc749103_0.conda#b90bece58b4c2bf25969b70f3be42d25 +https://conda.anaconda.org/conda-forge/linux-64/python-3.10.17-hd6af730_0_cpython.conda#7bb89638dae9ce1b8e051d0b721e83c2 +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-0.4.1-hb711507_2.conda#8637c3e5821654d0edf97e2b0404b443 +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-keysyms-0.4.1-hb711507_0.conda#ad748ccca349aec3e91743e08b5e2b50 +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-renderutil-0.3.10-hb711507_0.conda#0e0cbe0564d03a99afd5fd7b362feecd +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-wm-0.4.2-hb711507_0.conda#608e0ef8256b81d04456e8d211eee3e8 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.6-he73a12e_0.conda#1c74ff8c35dcadf952a16f752ca5aa49 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.8.12-h4f16b4b_0.conda#db038ce880f100acc74dba10302b5630 +https://conda.anaconda.org/conda-forge/noarch/alabaster-0.7.16-pyhd8ed1ab_0.conda#def531a3ac77b7fb8c21d17bb5d0badb +https://conda.anaconda.org/conda-forge/noarch/appdirs-1.4.4-pyhd8ed1ab_1.conda#f4e90937bbfc3a4a92539545a37bb448 +https://conda.anaconda.org/conda-forge/linux-64/brotli-1.1.0-hb9d3cd8_2.conda#98514fe74548d768907ce7a13f680e8f +https://conda.anaconda.org/conda-forge/linux-64/brotli-python-1.1.0-py310hf71b8c6_2.conda#bf502c169c71e3c6ac0d6175addfacc2 +https://conda.anaconda.org/conda-forge/noarch/certifi-2025.4.26-pyhd8ed1ab_0.conda#c33eeaaa33f45031be34cda513df39b6 +https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.4.2-pyhd8ed1ab_0.conda#40fe4284b8b5835a9073a645139f35af +https://conda.anaconda.org/conda-forge/noarch/click-8.2.1-pyh707e725_0.conda#94b550b8d3a614dbd326af798c7dfb40 +https://conda.anaconda.org/conda-forge/noarch/cloudpickle-3.1.1-pyhd8ed1ab_0.conda#364ba6c9fb03886ac979b482f39ebb92 +https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda#962b9857ee8e7018c22f2776ffa0b2d7 +https://conda.anaconda.org/conda-forge/noarch/cycler-0.12.1-pyhd8ed1ab_1.conda#44600c4667a319d67dbe0681fc0bc833 +https://conda.anaconda.org/conda-forge/linux-64/cyrus-sasl-2.1.27-h54b06d7_7.conda#dce22f70b4e5a407ce88f2be046f4ceb +https://conda.anaconda.org/conda-forge/linux-64/cython-3.0.10-py310hc6cd4ac_0.conda#bd1d71ee240be36f1d85c86177d6964f +https://conda.anaconda.org/conda-forge/noarch/docutils-0.21.2-pyhd8ed1ab_1.conda#24c1ca34138ee57de72a943237cde4cc +https://conda.anaconda.org/conda-forge/noarch/execnet-2.1.1-pyhd8ed1ab_1.conda#a71efeae2c160f6789900ba2631a2c90 +https://conda.anaconda.org/conda-forge/noarch/fsspec-2025.5.1-pyhd8ed1ab_0.conda#2d2c9ef879a7e64e2dc657b09272c2b6 +https://conda.anaconda.org/conda-forge/linux-64/gcc-13.3.0-h9576a4e_2.conda#d92e51bf4b6bdbfe45e5884fb0755afe +https://conda.anaconda.org/conda-forge/linux-64/gcc_linux-64-13.3.0-hc28eda2_10.conda#d151142bbafe5e68ec7fc065c5e6f80c +https://conda.anaconda.org/conda-forge/linux-64/gettext-0.24.1-h5888daf_0.conda#c63e7590d4d6f4c85721040ed8b12888 +https://conda.anaconda.org/conda-forge/linux-64/gfortran_impl_linux-64-13.3.0-h84c1745_2.conda#4e21ed177b76537067736f20f54fee0a +https://conda.anaconda.org/conda-forge/linux-64/gxx_impl_linux-64-13.3.0-hae580e1_2.conda#b55f02540605c322a47719029f8404cc +https://conda.anaconda.org/conda-forge/noarch/hpack-4.1.0-pyhd8ed1ab_0.conda#0a802cb9888dd14eeefc611f05c40b6e +https://conda.anaconda.org/conda-forge/noarch/hyperframe-6.1.0-pyhd8ed1ab_0.conda#8e6923fc12f1fe8f8c4e5c9f343256ac +https://conda.anaconda.org/conda-forge/noarch/idna-3.10-pyhd8ed1ab_1.conda#39a4f67be3286c86d696df570b1201b7 +https://conda.anaconda.org/conda-forge/noarch/imagesize-1.4.1-pyhd8ed1ab_0.tar.bz2#7de5386c8fea29e76b303f37dde4c352 +https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_1.conda#6837f3eff7dcea42ecd714ce1ac2b108 +https://conda.anaconda.org/conda-forge/linux-64/kiwisolver-1.4.7-py310h3788b33_0.conda#4186d9b4d004b0fe0de6aa62496fb48a +https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.17-h717163a_0.conda#000e85703f0fd9594c81710dd5066471 +https://conda.anaconda.org/conda-forge/linux-64/libavif16-1.3.0-h766b0b6_0.conda#f17f2d0e5c9ad6b958547fd67b155771 +https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-31_hba4ea11_blis.conda#1ea7ae3db0fea0c5222388d841583c51 +https://conda.anaconda.org/conda-forge/linux-64/libcups-2.3.3-h4637d8d_4.conda#d4529f4dff3057982a7617c7ac58fde3 +https://conda.anaconda.org/conda-forge/linux-64/libfreetype-2.13.3-ha770c72_1.conda#51f5be229d83ecd401fb369ab96ae669 +https://conda.anaconda.org/conda-forge/linux-64/libglib-2.84.2-h3618099_0.conda#072ab14a02164b7c0c089055368ff776 +https://conda.anaconda.org/conda-forge/linux-64/libglx-1.7.0-ha4b6fd6_2.conda#c8013e438185f33b13814c5c488acd5c +https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-12_hd37a5e2_netlib.conda#4b181b55915cefcd35c8398c9274e629 +https://conda.anaconda.org/conda-forge/linux-64/libsystemd0-257.4-h4e0b6ca_1.conda#04bcf3055e51f8dde6fab9672fb9fca0 +https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.13.8-h4bc477f_0.conda#14dbe05b929e329dbaa6f2d0aa19466d +https://conda.anaconda.org/conda-forge/noarch/locket-1.0.0-pyhd8ed1ab_0.tar.bz2#91e27ef3d05cc772ce627e51cff111c4 +https://conda.anaconda.org/conda-forge/linux-64/markupsafe-3.0.2-py310h89163eb_1.conda#8ce3f0332fd6de0d737e2911d329523f +https://conda.anaconda.org/conda-forge/noarch/meson-1.8.1-pyhe01879c_0.conda#f3cccd9a6ce5331ae33f69ade5529162 +https://conda.anaconda.org/conda-forge/noarch/munkres-1.1.4-pyh9f0ad1d_0.tar.bz2#2ba8498c1018c1e9c61eb99b973dfe19 +https://conda.anaconda.org/conda-forge/noarch/networkx-3.2-pyhd8ed1ab_0.conda#cec8cc498664cc00a070676aa89e69a7 +https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.3-h5fbd93e_0.conda#9e5816bc95d285c115a3ebc2f8563564 +https://conda.anaconda.org/conda-forge/noarch/packaging-25.0-pyh29332c3_1.conda#58335b26c38bf4a20f399384c33cbcf9 +https://conda.anaconda.org/conda-forge/noarch/pluggy-1.6.0-pyhd8ed1ab_0.conda#7da7ccd349dbf6487a7778579d2bb971 +https://conda.anaconda.org/conda-forge/noarch/ply-3.11-pyhd8ed1ab_3.conda#fd5062942bfa1b0bd5e0d2a4397b099e +https://conda.anaconda.org/conda-forge/linux-64/psutil-7.0.0-py310ha75aee5_0.conda#da7d592394ff9084a23f62a1186451a2 +https://conda.anaconda.org/conda-forge/noarch/pycparser-2.22-pyh29332c3_1.conda#12c566707c80111f9799308d9e265aef +https://conda.anaconda.org/conda-forge/noarch/pygments-2.19.1-pyhd8ed1ab_0.conda#232fb4577b6687b2d503ef8e254270c9 +https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.2.3-pyhd8ed1ab_1.conda#513d3c262ee49b54a8fec85c5bc99764 +https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha55dd90_7.conda#461219d1a5bd61342293efa2c0c90eac +https://conda.anaconda.org/conda-forge/noarch/pytz-2025.2-pyhd8ed1ab_0.conda#bc8e3267d44011051f2eb14d22fb0960 +https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0.2-py310h89163eb_2.conda#fd343408e64cf1e273ab7c710da374db +https://conda.anaconda.org/conda-forge/noarch/setuptools-80.9.0-pyhff2d567_0.conda#4de79c071274a53dcaf2a8c749d1499e +https://conda.anaconda.org/conda-forge/noarch/six-1.17.0-pyhd8ed1ab_0.conda#a451d576819089b0d672f18768be0f65 +https://conda.anaconda.org/conda-forge/noarch/snowballstemmer-3.0.1-pyhd8ed1ab_0.conda#755cf22df8693aa0d1aec1c123fa5863 +https://conda.anaconda.org/conda-forge/noarch/soupsieve-2.7-pyhd8ed1ab_0.conda#fb32097c717486aa34b38a9db57eb49e +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-jsmath-1.0.1-pyhd8ed1ab_1.conda#fa839b5ff59e192f411ccc7dae6588bb +https://conda.anaconda.org/conda-forge/noarch/tenacity-9.1.2-pyhd8ed1ab_0.conda#5d99943f2ae3cc69e1ada12ce9d4d701 +https://conda.anaconda.org/conda-forge/noarch/threadpoolctl-3.6.0-pyhecae5ae_0.conda#9d64911b31d57ca443e9f1e36b04385f +https://conda.anaconda.org/conda-forge/noarch/toml-0.10.2-pyhd8ed1ab_1.conda#b0dd904de08b7db706167240bf37b164 +https://conda.anaconda.org/conda-forge/noarch/tomli-2.2.1-pyhd8ed1ab_1.conda#ac944244f1fed2eb49bae07193ae8215 +https://conda.anaconda.org/conda-forge/noarch/toolz-1.0.0-pyhd8ed1ab_1.conda#40d0ed782a8aaa16ef248e68c06c168d +https://conda.anaconda.org/conda-forge/linux-64/tornado-6.5.1-py310ha75aee5_0.conda#6f3da1072c0c4d2a1beb1e84615f7c9c +https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.13.2-pyh29332c3_0.conda#83fc6ae00127671e301c9f44254c31b8 +https://conda.anaconda.org/conda-forge/linux-64/unicodedata2-16.0.0-py310ha75aee5_0.conda#1d7a4b9202cdd10d56ecdd7f6c347190 +https://conda.anaconda.org/conda-forge/noarch/wheel-0.45.1-pyhd8ed1ab_1.conda#75cb7132eb58d97896e173ef12ac9986 +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-image-0.4.0-hb711507_2.conda#a0901183f08b6c7107aab109733a3c91 +https://conda.anaconda.org/conda-forge/linux-64/xkeyboard-config-2.44-hb9d3cd8_0.conda#7c91bfc90672888259675ad2ad28af9c +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.6-hb9d3cd8_0.conda#febbab7d15033c913d53c7a2c102309d +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxfixes-6.0.1-hb9d3cd8_0.conda#4bdb303603e9821baf5fe5fdff1dc8f8 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.12-hb9d3cd8_0.conda#96d57aba173e878a2089d5638016dc5e +https://conda.anaconda.org/conda-forge/noarch/zipp-3.22.0-pyhd8ed1ab_0.conda#234be740b00b8e41567e5b0ed95aaba9 +https://conda.anaconda.org/conda-forge/noarch/accessible-pygments-0.0.5-pyhd8ed1ab_1.conda#74ac5069774cdbc53910ec4d631a3999 +https://conda.anaconda.org/conda-forge/noarch/babel-2.17.0-pyhd8ed1ab_0.conda#0a01c169f0ab0f91b26e77a3301fbfe4 +https://conda.anaconda.org/conda-forge/linux-64/brunsli-0.1-h9c3ff4c_0.tar.bz2#c1ac6229d0bfd14f8354ff9ad2a26cad +https://conda.anaconda.org/conda-forge/linux-64/c-compiler-1.9.0-h2b85faf_0.conda#3cb814f83f1f71ac1985013697f80cc1 +https://conda.anaconda.org/conda-forge/linux-64/cffi-1.17.1-py310h8deb56e_0.conda#1fc24a3196ad5ede2a68148be61894f4 +https://conda.anaconda.org/conda-forge/linux-64/cytoolz-1.0.1-py310ha75aee5_0.conda#d0be1adaa04a03aed745f3d02afb59ce +https://conda.anaconda.org/conda-forge/linux-64/dbus-1.16.2-h3c4dab8_0.conda#679616eb5ad4e521c83da4650860aba7 +https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.0-pyhd8ed1ab_0.conda#72e42d28960d875c7654614f8b50939a +https://conda.anaconda.org/conda-forge/linux-64/fonttools-4.58.1-py310h89163eb_0.conda#f4f46207c6defa5ea17b0299298ba849 +https://conda.anaconda.org/conda-forge/linux-64/freetype-2.13.3-ha770c72_1.conda#9ccd736d31e0c6e41f54e704e5312811 +https://conda.anaconda.org/conda-forge/linux-64/gfortran-13.3.0-h9576a4e_2.conda#19e6d3c9cde10a0a9a170a684082588e +https://conda.anaconda.org/conda-forge/linux-64/gfortran_linux-64-13.3.0-hb919d3a_10.conda#7ce070e3329cd10bf79dbed562a21bd4 +https://conda.anaconda.org/conda-forge/linux-64/glib-tools-2.84.2-h4833e2c_0.conda#f2ec1facec64147850b7674633978050 +https://conda.anaconda.org/conda-forge/linux-64/gxx-13.3.0-h9576a4e_2.conda#07e8df00b7cd3084ad3ef598ce32a71c +https://conda.anaconda.org/conda-forge/linux-64/gxx_linux-64-13.3.0-h6834431_10.conda#9a8ebde471cec5cc9c48f8682f434f92 +https://conda.anaconda.org/conda-forge/noarch/h2-4.2.0-pyhd8ed1ab_0.conda#b4754fb1bdcb70c8fd54f918301582c6 +https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-8.7.0-pyhe01879c_1.conda#63ccfdc3a3ce25b027b8767eb722fca8 +https://conda.anaconda.org/conda-forge/noarch/importlib_resources-6.5.2-pyhd8ed1ab_0.conda#c85c76dc67d75619a92f51dfbce06992 +https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.6-pyhd8ed1ab_0.conda#446bd6c8cb26050d528881df495ce646 +https://conda.anaconda.org/conda-forge/noarch/joblib-1.5.1-pyhd8ed1ab_0.conda#fb1c14694de51a476ce8636d92b6f42c +https://conda.anaconda.org/conda-forge/linux-64/libflac-1.4.3-h59595ed_0.conda#ee48bf17cc83a00f59ca1494d5646869 +https://conda.anaconda.org/conda-forge/linux-64/libgl-1.7.0-ha4b6fd6_2.conda#928b8be80851f5d8ffb016f9c81dae7a +https://conda.anaconda.org/conda-forge/linux-64/liblapacke-3.9.0-12_hce4cc19_netlib.conda#bdcf65db13abdddba7af29592f93600b +https://conda.anaconda.org/conda-forge/linux-64/libllvm20-20.1.6-he9d0ab4_0.conda#bf8ccdd2c1c1a54a3fa25bb61f26460e +https://conda.anaconda.org/conda-forge/linux-64/libxkbcommon-1.10.0-h65c71a3_0.conda#fedf6bfe5d21d21d2b1785ec00a8889a +https://conda.anaconda.org/conda-forge/noarch/memory_profiler-0.61.0-pyhd8ed1ab_1.conda#71abbefb6f3b95e1668cd5e0af3affb9 +https://conda.anaconda.org/conda-forge/linux-64/numpy-1.22.0-py310h454958d_1.tar.bz2#607c66f0cce2986515a8fe9e136b2b57 +https://conda.anaconda.org/conda-forge/linux-64/openldap-2.6.10-he970967_0.conda#2e5bf4f1da39c0b32778561c3c4e5878 +https://conda.anaconda.org/conda-forge/noarch/partd-1.4.2-pyhd8ed1ab_0.conda#0badf9c54e24cecfb0ad2f99d680c163 +https://conda.anaconda.org/conda-forge/linux-64/pillow-11.2.1-py310h7e6dc6c_0.conda#5645a243d90adb50909b9edc209d84fe +https://conda.anaconda.org/conda-forge/noarch/pip-25.1.1-pyh8b19718_0.conda#32d0781ace05105cc99af55d36cbec7c +https://conda.anaconda.org/conda-forge/noarch/plotly-5.14.0-pyhd8ed1ab_0.conda#6a7bcc42ef58dd6cf3da9333ea102433 +https://conda.anaconda.org/conda-forge/noarch/pyproject-metadata-0.9.1-pyhd8ed1ab_0.conda#22ae7c6ea81e0c8661ef32168dda929b +https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhff2d567_1.conda#5ba79d7c71f03c678c8ead841f347d6e +https://conda.anaconda.org/conda-forge/linux-64/sip-6.10.0-py310hf71b8c6_0.conda#2d7e4445be227e8210140b75725689ad +https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.13.2-h0e9735f_0.conda#568ed1300869dca0ba09fb750cda5dbb +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxcomposite-0.4.6-hb9d3cd8_2.conda#d3c295b50f092ab525ffe3c2aa4b7413 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdamage-1.1.6-hb9d3cd8_0.conda#b5fcc7172d22516e1f965490e65e33a4 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxxf86vm-1.1.6-hb9d3cd8_0.conda#5efa5fa6243a622445fdfd72aee15efa +https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.13.4-pyha770c72_0.conda#9f07c4fc992adb2d6c30da7fab3959a7 +https://conda.anaconda.org/conda-forge/linux-64/blas-devel-3.9.0-31_hdec4247_blis.conda#1675e95a742c910204645f7b6d7e56dc +https://conda.anaconda.org/conda-forge/linux-64/cxx-compiler-1.9.0-h1a2810e_0.conda#1ce8b218d359d9ed0ab481f2a3f3c512 +https://conda.anaconda.org/conda-forge/noarch/dask-core-2025.5.1-pyhd8ed1ab_0.conda#8f0ef561cd615a17df3256742a3457c4 +https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.15.0-h7e30c49_1.conda#8f5b0b297b59e1ac160ad4beec99dbee +https://conda.anaconda.org/conda-forge/linux-64/fortran-compiler-1.9.0-h36df796_0.conda#cc0cf942201f9d3b0e9654ea02e12486 +https://conda.anaconda.org/conda-forge/linux-64/glib-2.84.2-h6287aef_0.conda#704648df3a01d4d24bc2c0466b718d63 +https://conda.anaconda.org/conda-forge/linux-64/imagecodecs-2025.3.30-py310h481ba9f_0.conda#453c8da1b70f7b76b3884e18015bc568 +https://conda.anaconda.org/conda-forge/noarch/imageio-2.37.0-pyhfb79c49_0.conda#b5577bc2212219566578fd5af9993af6 +https://conda.anaconda.org/conda-forge/noarch/importlib-resources-6.5.2-pyhd8ed1ab_0.conda#e376ea42e9ae40f3278b0f79c9bf9826 +https://conda.anaconda.org/conda-forge/linux-64/libclang-cpp20.1-20.1.6-default_h1df26ce_0.conda#99ead3b974685e44df8b1e3953503cfc +https://conda.anaconda.org/conda-forge/linux-64/libclang13-20.1.6-default_he06ed0a_0.conda#cc6c469d9d7fc0ac106cef5f45d973a9 +https://conda.anaconda.org/conda-forge/linux-64/libpq-17.5-h27ae623_0.conda#6458be24f09e1b034902ab44fe9de908 +https://conda.anaconda.org/conda-forge/linux-64/libsndfile-1.2.2-hc60ed4a_1.conda#ef1910918dd895516a769ed36b5b3a4e +https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.5.0-py310h23f4a51_0.tar.bz2#9911225650b298776c8e8c083b5cacf1 +https://conda.anaconda.org/conda-forge/noarch/meson-python-0.18.0-pyh70fd9c4_0.conda#576c04b9d9f8e45285fb4d9452c26133 +https://conda.anaconda.org/conda-forge/linux-64/pandas-1.4.0-py310hb5077e9_0.tar.bz2#43e920bc9856daa7d8d18fcbfb244c4e +https://conda.anaconda.org/conda-forge/noarch/patsy-1.0.1-pyhd8ed1ab_1.conda#ee23fabfd0a8c6b8d6f3729b47b2859d +https://conda.anaconda.org/conda-forge/linux-64/polars-0.20.30-py310h031f9ce_0.conda#0743f5db9f978b6df92d412935ff8371 +https://conda.anaconda.org/conda-forge/linux-64/pyqt5-sip-12.17.0-py310hf71b8c6_0.conda#012465861673a67a30bc8ca6284074f3 +https://conda.anaconda.org/conda-forge/noarch/pytest-8.3.5-pyhd8ed1ab_0.conda#c3c9316209dec74a705a36797970c6be +https://conda.anaconda.org/conda-forge/linux-64/pywavelets-1.6.0-py310h261611a_0.conda#04a405ee0bccb4de8d1ed0c87704f5f6 +https://conda.anaconda.org/conda-forge/linux-64/scipy-1.8.0-py310hea5193d_1.tar.bz2#664d80ddeb51241629b3ada5ea926e4d +https://conda.anaconda.org/conda-forge/linux-64/zstandard-0.23.0-py310ha75aee5_2.conda#f9254b5b0193982416b91edcb4b2676f +https://conda.anaconda.org/conda-forge/linux-64/blas-2.131-blis.conda#87829e6b9fe49a926280e100959b7d2b +https://conda.anaconda.org/conda-forge/linux-64/cairo-1.18.4-h3394656_0.conda#09262e66b19567aff4f592fb53b28760 +https://conda.anaconda.org/conda-forge/linux-64/compilers-1.9.0-ha770c72_0.conda#5859096e397aba423340d0bbbb11ec64 +https://conda.anaconda.org/conda-forge/linux-64/gstreamer-1.24.11-hc37bda9_0.conda#056d86cacf2b48c79c6a562a2486eb8c +https://conda.anaconda.org/conda-forge/linux-64/pulseaudio-client-17.0-hac146a9_1.conda#66b1fa9608d8836e25f9919159adc9c6 +https://conda.anaconda.org/conda-forge/linux-64/pyamg-4.2.1-py310h7c3ba0c_0.tar.bz2#89f5a48e1f23b5cf3163a6094903d181 +https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.7.0-pyhd8ed1ab_0.conda#15353a2a0ea6dfefaa52fc5ab5b98f41 +https://conda.anaconda.org/conda-forge/noarch/seaborn-base-0.13.2-pyhd8ed1ab_3.conda#fd96da444e81f9e6fcaac38590f3dd42 +https://conda.anaconda.org/conda-forge/linux-64/statsmodels-0.14.2-py310h261611a_0.conda#4b8508bab02b2aa2cef12eab4883f4a1 +https://conda.anaconda.org/conda-forge/noarch/tifffile-2025.5.10-pyhd8ed1ab_0.conda#1fdb801f28bf4987294c49aaa314bf5e +https://conda.anaconda.org/conda-forge/noarch/towncrier-24.8.0-pyhd8ed1ab_1.conda#820b6a1ddf590fba253f8204f7200d82 +https://conda.anaconda.org/conda-forge/noarch/urllib3-2.4.0-pyhd8ed1ab_0.conda#c1e349028e0052c4eea844e94f773065 +https://conda.anaconda.org/conda-forge/linux-64/gst-plugins-base-1.24.11-h651a532_0.conda#d8d8894f8ced2c9be76dc9ad1ae531ce +https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-11.2.1-h3beb420_0.conda#0e6e192d4b3d95708ad192d957cf3163 +https://conda.anaconda.org/conda-forge/noarch/requests-2.32.3-pyhd8ed1ab_1.conda#a9b9368f3701a417eac9edbcae7cb737 +https://conda.anaconda.org/conda-forge/linux-64/scikit-image-0.19.0-py310hb5077e9_0.tar.bz2#aa24b3a4aa979641ac3144405209cd89 +https://conda.anaconda.org/conda-forge/noarch/seaborn-0.13.2-hd8ed1ab_3.conda#62afb877ca2c2b4b6f9ecb37320085b6 +https://conda.anaconda.org/conda-forge/noarch/pooch-1.6.0-pyhd8ed1ab_0.tar.bz2#6429e1d1091c51f626b5dcfdd38bf429 +https://conda.anaconda.org/conda-forge/linux-64/qt-main-5.15.15-hea1682b_4.conda#c054d7f22cc719e12c72d454b2328d6c +https://conda.anaconda.org/conda-forge/linux-64/pyqt-5.15.11-py310hf392a12_0.conda#65924d3e57be25342c76530d23d75f0f +https://conda.anaconda.org/conda-forge/linux-64/matplotlib-3.5.0-py310hff52083_0.tar.bz2#1b2f3b135d5d9c594b5e0e6150c03b7b +https://conda.anaconda.org/conda-forge/noarch/numpydoc-1.2-pyhd8ed1ab_0.tar.bz2#025ad7ca2c7f65007ab6b6f5d93a56eb +https://conda.anaconda.org/conda-forge/noarch/pydata-sphinx-theme-0.15.3-pyhd8ed1ab_0.conda#55e445f4fcb07f2471fb0e1102d36488 +https://conda.anaconda.org/conda-forge/noarch/sphinx-copybutton-0.5.2-pyhd8ed1ab_1.conda#bf22cb9c439572760316ce0748af3713 +https://conda.anaconda.org/conda-forge/noarch/sphinx-design-0.6.0-pyhd8ed1ab_0.conda#b04f3c04e4f7939c6207dc0c0355f468 +https://conda.anaconda.org/conda-forge/noarch/sphinx-gallery-0.17.1-pyhd8ed1ab_0.conda#0adfccc6e7269a29a63c1c8ee3c6d8ba +https://conda.anaconda.org/conda-forge/noarch/sphinx-prompt-1.4.0-pyhd8ed1ab_0.tar.bz2#88ee91e8679603f2a5bd036d52919cc2 +https://conda.anaconda.org/conda-forge/noarch/sphinx-remove-toctrees-1.0.0.post1-pyhd8ed1ab_1.conda#b275c865b753413caaa8548b9d44c024 +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-applehelp-2.0.0-pyhd8ed1ab_1.conda#16e3f039c0aa6446513e94ab18a8784b +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-devhelp-2.0.0-pyhd8ed1ab_1.conda#910f28a05c178feba832f842155cbfff +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-htmlhelp-2.1.0-pyhd8ed1ab_1.conda#e9fb3fe8a5b758b4aff187d434f94f03 +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-qthelp-2.0.0-pyhd8ed1ab_1.conda#00534ebcc0375929b45c3039b5ba7636 +https://conda.anaconda.org/conda-forge/noarch/sphinx-7.3.7-pyhd8ed1ab_0.conda#7b1465205e28d75d2c0e1a868ee00a67 +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-serializinghtml-1.1.10-pyhd8ed1ab_1.conda#3bc61f7161d28137797e038263c04c54 +# pip libsass @ https://files.pythonhosted.org/packages/fd/5a/eb5b62641df0459a3291fc206cf5bd669c0feed7814dded8edef4ade8512/libsass-0.23.0-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.whl#sha256=4a218406d605f325d234e4678bd57126a66a88841cb95bee2caeafdc6f138306 +# pip sphinxcontrib-sass @ https://files.pythonhosted.org/packages/2e/87/7c2eb08e3ca1d6baae32c0a5e005330fe1cec93a36aa085e714c3b3a3c7d/sphinxcontrib_sass-0.3.4-py2.py3-none-any.whl#sha256=a0c79a44ae8b8935c02dc340ebe40c9e002c839331201c899dc93708970c355a +# pip sphinxext-opengraph @ https://files.pythonhosted.org/packages/92/0a/970b80b4fa1feeb6deb6f2e22d4cb14e388b27b315a1afdb9db930ff91a4/sphinxext_opengraph-0.9.1-py3-none-any.whl#sha256=b3b230cc6a5b5189139df937f0d9c7b23c7c204493b22646273687969dcb760e diff --git a/build_tools/circle/download_documentation.sh b/build_tools/circle/download_documentation.sh new file mode 100755 index 0000000000000..c2d6d09d0abb9 --- /dev/null +++ b/build_tools/circle/download_documentation.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -e +set -x + +wget $GITHUB_ARTIFACT_URL +mkdir -p doc/_build/html/stable +unzip doc*.zip -d doc/_build/html/stable diff --git a/build_tools/circle/linting.sh b/build_tools/circle/linting.sh deleted file mode 100755 index aebe42dfecc70..0000000000000 --- a/build_tools/circle/linting.sh +++ /dev/null @@ -1,179 +0,0 @@ -#!/bin/bash - -# This script is used in CircleCI to check that PRs do not add obvious -# flake8 violations. It relies on two things: -# - find common ancestor between branch and -# scikit-learn/scikit-learn remote -# - run flake8 --diff on the diff between the branch and the common -# ancestor -# -# Additional features: -# - the line numbers in Travis match the local branch on the PR -# author machine. -# - ./build_tools/circle/flake8_diff.sh can be run locally for quick -# turn-around - -set -e -# pipefail is necessary to propagate exit codes -set -o pipefail - -PROJECT=scikit-learn/scikit-learn -PROJECT_URL=https://github.com/$PROJECT.git - -# Find the remote with the project name (upstream in most cases) -REMOTE=$(git remote -v | grep $PROJECT | cut -f1 | head -1 || echo '') - -# Add a temporary remote if needed. For example this is necessary when -# Travis is configured to run in a fork. In this case 'origin' is the -# fork and not the reference repo we want to diff against. -if [[ -z "$REMOTE" ]]; then - TMP_REMOTE=tmp_reference_upstream - REMOTE=$TMP_REMOTE - git remote add $REMOTE $PROJECT_URL -fi - -echo "Remotes:" -echo '--------------------------------------------------------------------------------' -git remote --verbose - -# Travis does the git clone with a limited depth (50 at the time of -# writing). This may not be enough to find the common ancestor with -# $REMOTE/main so we unshallow the git checkout -if [[ -a .git/shallow ]]; then - echo -e '\nTrying to unshallow the repo:' - echo '--------------------------------------------------------------------------------' - git fetch --unshallow -fi - -if [[ "$TRAVIS" == "true" ]]; then - if [[ "$TRAVIS_PULL_REQUEST" == "false" ]] - then - # In main repo, using TRAVIS_COMMIT_RANGE to test the commits - # that were pushed into a branch - if [[ "$PROJECT" == "$TRAVIS_REPO_SLUG" ]]; then - if [[ -z "$TRAVIS_COMMIT_RANGE" ]]; then - echo "New branch, no commit range from Travis so passing this test by convention" - exit 0 - fi - COMMIT_RANGE=$TRAVIS_COMMIT_RANGE - fi - else - # We want to fetch the code as it is in the PR branch and not - # the result of the merge into main. This way line numbers - # reported by Travis will match with the local code. - LOCAL_BRANCH_REF=travis_pr_$TRAVIS_PULL_REQUEST - # In Travis the PR target is always origin - git fetch origin pull/$TRAVIS_PULL_REQUEST/head:refs/$LOCAL_BRANCH_REF - fi -fi - -# If not using the commit range from Travis we need to find the common -# ancestor between $LOCAL_BRANCH_REF and $REMOTE/main -if [[ -z "$COMMIT_RANGE" ]]; then - if [[ -z "$LOCAL_BRANCH_REF" ]]; then - LOCAL_BRANCH_REF=$(git rev-parse --abbrev-ref HEAD) - fi - echo -e "\nLast 2 commits in $LOCAL_BRANCH_REF:" - echo '--------------------------------------------------------------------------------' - git --no-pager log -2 $LOCAL_BRANCH_REF - - REMOTE_MAIN_REF="$REMOTE/main" - # Make sure that $REMOTE_MAIN_REF is a valid reference - echo -e "\nFetching $REMOTE_MAIN_REF" - echo '--------------------------------------------------------------------------------' - git fetch $REMOTE main:refs/remotes/$REMOTE_MAIN_REF - LOCAL_BRANCH_SHORT_HASH=$(git rev-parse --short $LOCAL_BRANCH_REF) - REMOTE_MAIN_SHORT_HASH=$(git rev-parse --short $REMOTE_MAIN_REF) - - COMMIT=$(git merge-base $LOCAL_BRANCH_REF $REMOTE_MAIN_REF) || \ - echo "No common ancestor found for $(git show $LOCAL_BRANCH_REF -q) and $(git show $REMOTE_MAIN_REF -q)" - - if [ -z "$COMMIT" ]; then - exit 1 - fi - - COMMIT_SHORT_HASH=$(git rev-parse --short $COMMIT) - - echo -e "\nCommon ancestor between $LOCAL_BRANCH_REF ($LOCAL_BRANCH_SHORT_HASH)"\ - "and $REMOTE_MAIN_REF ($REMOTE_MAIN_SHORT_HASH) is $COMMIT_SHORT_HASH:" - echo '--------------------------------------------------------------------------------' - git --no-pager show --no-patch $COMMIT_SHORT_HASH - - COMMIT_RANGE="$COMMIT_SHORT_HASH..$LOCAL_BRANCH_SHORT_HASH" - - if [[ -n "$TMP_REMOTE" ]]; then - git remote remove $TMP_REMOTE - fi - -else - echo "Got the commit range from Travis: $COMMIT_RANGE" -fi - -echo -e '\nRunning flake8 on the diff in the range' "$COMMIT_RANGE" \ - "($(git rev-list $COMMIT_RANGE | wc -l) commit(s)):" -echo '--------------------------------------------------------------------------------' - -# We ignore files from sklearn/externals. Unfortunately there is no -# way to do it with flake8 directly (the --exclude does not seem to -# work with --diff). We could use the exclude magic in the git pathspec -# ':!sklearn/externals' but it is only available on git 1.9 and Travis -# uses git 1.8. -# We need the following command to exit with 0 hence the echo in case -# there is no match -MODIFIED_FILES="$(git diff --name-only $COMMIT_RANGE | grep -v 'sklearn/externals' | \ - grep -v 'doc/sphinxext' || echo "no_match")" - -check_files() { - files="$1" - shift - options="$*" - if [ -n "$files" ]; then - # Conservative approach: diff without context (--unified=0) so that code - # that was not changed does not create failures - git diff --unified=0 $COMMIT_RANGE -- $files | flake8 --diff --show-source $options - fi -} - -if [[ "$MODIFIED_FILES" == "no_match" ]]; then - echo "No file outside sklearn/externals and doc/sphinxext has been modified" -else - check_files "$MODIFIED_FILES" - # check code for unused imports - flake8 --exclude=sklearn/externals/ --select=F401 sklearn/ examples/ -fi -echo -e "No problem detected by flake8\n" - -# For docstrings and warnings of deprecated attributes to be rendered -# properly, the property decorator must come before the deprecated decorator -# (else they are treated as functions) - -# do not error when grep -B1 "@property" finds nothing -set +e -bad_deprecation_property_order=`git grep -A 10 "@property" -- "*.py" | awk '/@property/,/def /' | grep -B1 "@deprecated"` - -if [ ! -z "$bad_deprecation_property_order" ] -then - echo "property decorator should come before deprecated decorator" - echo "found the following occurrencies:" - echo $bad_deprecation_property_order - exit 1 -fi - -# Check for default doctest directives ELLIPSIS and NORMALIZE_WHITESPACE - -doctest_directive="$(git grep -nw -E "# doctest\: \+(ELLIPSIS|NORMALIZE_WHITESPACE)")" - -if [ ! -z "$doctest_directive" ] -then - echo "ELLIPSIS and NORMALIZE_WHITESPACE doctest directives are enabled by default, but were found in:" - echo "$doctest_directive" - exit 1 -fi - -joblib_import="$(git grep -l -A 10 -E "joblib import.+delayed" -- "*.py" ":!sklearn/utils/_joblib.py" ":!sklearn/utils/fixes.py")" - -if [ ! -z "$joblib_import" ]; then - echo "Use from sklearn.utils.fixes import delayed instead of joblib delayed. The following files contains imports to joblib.delayed:" - echo "$joblib_import" - exit 1 -fi diff --git a/build_tools/circle/list_versions.py b/build_tools/circle/list_versions.py index 19bee5ae1cfc7..00526f062f200 100755 --- a/build_tools/circle/list_versions.py +++ b/build_tools/circle/list_versions.py @@ -1,19 +1,24 @@ #!/usr/bin/env python3 -# List all available versions of the documentation +# Write the available versions page (--rst) and the version switcher JSON (--json). +# Version switcher see: +# https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/version-dropdown.html +# https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/announcements.html#announcement-banners + +import argparse import json import re import sys - -from distutils.version import LooseVersion from urllib.request import urlopen +from sklearn.utils.fixes import parse_version + def json_urlread(url): try: - return json.loads(urlopen(url).read().decode('utf8')) + return json.loads(urlopen(url).read().decode("utf8")) except Exception: - print('Error reading', url, file=sys.stderr) + print("Error reading", url, file=sys.stderr) raise @@ -21,8 +26,7 @@ def human_readable_data_quantity(quantity, multiple=1024): # https://stackoverflow.com/questions/1094841/reusable-library-to-get-human-readable-version-of-file-size if quantity == 0: quantity = +0 - SUFFIXES = ["B"] + [i + {1000: "B", 1024: "iB"}[multiple] - for i in "KMGTPEZY"] + SUFFIXES = ["B"] + [i + {1000: "B", 1024: "iB"}[multiple] for i in "KMGTPEZY"] for suffix in SUFFIXES: if quantity < multiple or suffix == SUFFIXES[-1]: if suffix == SUFFIXES[0]: @@ -34,55 +38,60 @@ def human_readable_data_quantity(quantity, multiple=1024): def get_file_extension(version): - if 'dev' in version: - # The 'dev' branch should be explictly handled - return 'zip' + if "dev" in version: + # The 'dev' branch should be explicitly handled + return "zip" - current_version = LooseVersion(version) - min_zip_version = LooseVersion('0.24') + current_version = parse_version(version) + min_zip_version = parse_version("0.24") - return 'zip' if current_version >= min_zip_version else 'pdf' + return "zip" if current_version >= min_zip_version else "pdf" def get_file_size(version): - api_url = ROOT_URL + '%s/_downloads' % version + api_url = ROOT_URL + "%s/_downloads" % version for path_details in json_urlread(api_url): file_extension = get_file_extension(version) - file_path = f'scikit-learn-docs.{file_extension}' - if path_details['name'] == file_path: - return human_readable_data_quantity(path_details['size'], 1000) - - -print(':orphan:') -print() -heading = 'Available documentation for Scikit-learn' -print(heading) -print('=' * len(heading)) -print() -print('Web-based documentation is available for versions listed below:') -print() - -ROOT_URL = 'https://api.github.com/repos/scikit-learn/scikit-learn.github.io/contents/' # noqa -RAW_FMT = 'https://raw.githubusercontent.com/scikit-learn/scikit-learn.github.io/master/%s/index.html' # noqa + file_path = f"scikit-learn-docs.{file_extension}" + if path_details["name"] == file_path: + return human_readable_data_quantity(path_details["size"], 1000) + + +parser = argparse.ArgumentParser() +parser.add_argument("--rst", type=str, required=True) +parser.add_argument("--json", type=str, required=True) +args = parser.parse_args() + +heading = "Available documentation for scikit-learn" +json_content = [] +rst_content = [ + ":orphan:\n", + heading, + "=" * len(heading) + "\n", + "Web-based documentation is available for versions listed below:\n", +] + +ROOT_URL = "https://api.github.com/repos/scikit-learn/scikit-learn.github.io/contents/" +RAW_FMT = "https://raw.githubusercontent.com/scikit-learn/scikit-learn.github.io/master/%s/index.html" VERSION_RE = re.compile(r"scikit-learn ([\w\.\-]+) documentation") -NAMED_DIRS = ['dev', 'stable'] +NAMED_DIRS = ["dev", "stable"] # Gather data for each version directory, including symlinks dirs = {} symlinks = {} root_listing = json_urlread(ROOT_URL) for path_details in root_listing: - name = path_details['name'] + name = path_details["name"] if not (name[:1].isdigit() or name in NAMED_DIRS): continue - if path_details['type'] == 'dir': - html = urlopen(RAW_FMT % name).read().decode('utf8') + if path_details["type"] == "dir": + html = urlopen(RAW_FMT % name).read().decode("utf8") version_num = VERSION_RE.search(html).group(1) file_size = get_file_size(name) dirs[name] = (version_num, file_size) - if path_details['type'] == 'symlink': - symlinks[name] = json_urlread(path_details['_links']['self'])['target'] + if path_details["type"] == "symlink": + symlinks[name] = json_urlread(path_details["_links"]["self"])["target"] # Symlinks should have same data as target @@ -92,21 +101,42 @@ def get_file_size(version): # Output in order: dev, stable, decreasing other version seen = set() -for name in (NAMED_DIRS + - sorted((k for k in dirs if k[:1].isdigit()), - key=LooseVersion, reverse=True)): +for i, name in enumerate( + NAMED_DIRS + + sorted((k for k in dirs if k[:1].isdigit()), key=parse_version, reverse=True) +): version_num, file_size = dirs[name] if version_num in seen: # symlink came first continue else: seen.add(version_num) - name_display = '' if name[:1].isdigit() else ' (%s)' % name - path = 'https://scikit-learn.org/%s/' % name - out = ('* `Scikit-learn %s%s documentation <%s>`_' - % (version_num, name_display, path)) + + full_name = f"{version_num}" if name[:1].isdigit() else f"{version_num} ({name})" + path = f"https://scikit-learn.org/{name}/" + + # Update JSON for the version switcher; only keep the 8 latest versions to avoid + # overloading the version switcher dropdown + if i < 8: + info = {"name": full_name, "version": version_num, "url": path} + if name == "stable": + info["preferred"] = True + json_content.append(info) + + # Printout for the historical version page + out = f"* `scikit-learn {full_name} documentation <{path}>`_" if file_size is not None: file_extension = get_file_extension(version_num) - out += (f' (`{file_extension.upper()} {file_size} <{path}/' - f'_downloads/scikit-learn-docs.{file_extension}>`_)') - print(out) + out += ( + f" (`{file_extension.upper()} {file_size} <{path}/" + f"_downloads/scikit-learn-docs.{file_extension}>`_)" + ) + rst_content.append(out) + +with open(args.rst, "w", encoding="utf-8") as f: + f.write("\n".join(rst_content) + "\n") +print(f"Written {args.rst}") + +with open(args.json, "w", encoding="utf-8") as f: + json.dump(json_content, f, indent=2) +print(f"Written {args.json}") diff --git a/build_tools/circle/push_doc.sh b/build_tools/circle/push_doc.sh index 5b94211e4e30e..f959b8b65c85c 100755 --- a/build_tools/circle/push_doc.sh +++ b/build_tools/circle/push_doc.sh @@ -1,8 +1,8 @@ #!/bin/bash # This script is meant to be called in the "deploy" step defined in -# circle.yml. See https://circleci.com/docs/ for more details. +# .circleci/config.yml. See https://circleci.com/docs/ for more details. # The behavior of the script is controlled by environment variable defined -# in the circle.yml in the top level folder of the project. +# in the .circleci/config.yml file. set -ex @@ -56,7 +56,7 @@ then git rm -rf $dir/ && rm -rf $dir/ fi cp -R $GENERATED_DOC_DIR $dir -git config user.email "olivier.grisel+sklearn-ci@gmail.com" +git config user.email "ci@scikit-learn.org" git config user.name $USERNAME git config push.default matching git add -f $dir/ diff --git a/build_tools/codespell_ignore_words.txt b/build_tools/codespell_ignore_words.txt new file mode 100644 index 0000000000000..6b942a2eabe6d --- /dev/null +++ b/build_tools/codespell_ignore_words.txt @@ -0,0 +1,56 @@ +achin +aggresive +aline +ba +basf +boun +bre +bu +cach +chanel +complies +coo +copys +datas +deine +didi +feld +fo +fpr +fro +fwe +gool +hart +heping +hist +ines +inout +ist +jaques +lamas +linke +lod +mape +mis +mor +nd +nmae +ocur +pullrequest +repid +ro +ser +soler +suh +suprised +te +technic +teh +thi +usal +vie +vor +wan +whis +winn +yau diff --git a/build_tools/generate_authors_table.py b/build_tools/generate_authors_table.py index f8b1191d14d9b..6dcddda40af4d 100644 --- a/build_tools/generate_authors_table.py +++ b/build_tools/generate_authors_table.py @@ -6,27 +6,31 @@ The table should be updated for each new inclusion in the teams. Generating the table requires admin rights. """ -import sys -import requests + import getpass +import sys import time -from pathlib import Path from os import path +from pathlib import Path + +import requests -print("user:", file=sys.stderr) +print("Input user:", file=sys.stderr) user = input() -token = getpass.getpass("access token:\n") +token = getpass.getpass("Input access token:\n") auth = (user, token) -LOGO_URL = 'https://avatars2.githubusercontent.com/u/365630?v=4' +LOGO_URL = "https://avatars2.githubusercontent.com/u/365630?v=4" REPO_FOLDER = Path(path.abspath(__file__)).parent.parent def get(url): for sleep_time in [10, 30, 0]: reply = requests.get(url, auth=auth) - api_limit = ("message" in reply.json() - and "API rate limit exceeded" in reply.json()["message"]) + api_limit = ( + "message" in reply.json() + and "API rate limit exceeded" in reply.json()["message"] + ) if not api_limit: break print("API rate limit exceeded, waiting..") @@ -38,55 +42,113 @@ def get(url): def get_contributors(): """Get the list of contributor profiles. Require admin rights.""" - # get core devs and triage team + # get core devs and contributor experience team core_devs = [] - triage_team = [] - for team_id, lst in zip((11523, 3593183), (core_devs, triage_team)): + documentation_team = [] + contributor_experience_team = [] + comm_team = [] + core_devs_slug = "core-devs" + contributor_experience_team_slug = "contributor-experience-team" + comm_team_slug = "communication-team" + documentation_team_slug = "documentation-team" + + entry_point = "https://api.github.com/orgs/scikit-learn/" + + for team_slug, lst in zip( + ( + core_devs_slug, + contributor_experience_team_slug, + comm_team_slug, + documentation_team_slug, + ), + (core_devs, contributor_experience_team, comm_team, documentation_team), + ): + print(f"Retrieving {team_slug}\n") for page in [1, 2]: # 30 per page - reply = get( - f"https://api.github.com/teams/{team_id}/members?page={page}" - ) + reply = get(f"{entry_point}teams/{team_slug}/members?page={page}") lst.extend(reply.json()) # get members of scikit-learn on GitHub + print("Retrieving members\n") members = [] - for page in [1, 2]: # 30 per page - reply = get( - "https://api.github.com/orgs/scikit-learn/members?page=%d" % - (page, )) + for page in [1, 2, 3]: # 30 per page + reply = get(f"{entry_point}members?page={page}") members.extend(reply.json()) # keep only the logins - core_devs = set(c['login'] for c in core_devs) - triage_team = set(c['login'] for c in triage_team) - members = set(c['login'] for c in members) + core_devs = set(c["login"] for c in core_devs) + documentation_team = set(c["login"] for c in documentation_team) + contributor_experience_team = set(c["login"] for c in contributor_experience_team) + comm_team = set(c["login"] for c in comm_team) + members = set(c["login"] for c in members) # add missing contributors with GitHub accounts - members |= {'dubourg', 'mbrucher', 'thouis', 'jarrodmillman'} + members |= {"dubourg", "mbrucher", "thouis", "jarrodmillman"} # add missing contributors without GitHub accounts - members |= {'Angel Soler Gollonet'} + members |= {"Angel Soler Gollonet"} # remove CI bots - members -= {'sklearn-ci', 'sklearn-lgtm', 'sklearn-wheels'} - triage_team -= core_devs # remove ogrisel from triage_team + members -= {"sklearn-ci", "sklearn-wheels", "sklearn-lgtm"} + contributor_experience_team -= ( + core_devs # remove ogrisel from contributor_experience_team + ) + + emeritus = ( + members + - core_devs + - contributor_experience_team + - comm_team + - documentation_team + ) + + # hard coded + emeritus_contributor_experience_team = { + "cmarmo", + } + emeritus_comm_team = {"reshamas"} + + # Up-to-now, we can subtract the team emeritus from the original emeritus + emeritus -= emeritus_contributor_experience_team | emeritus_comm_team - emeritus = members - core_devs - triage_team + comm_team -= {"reshamas"} # in the comm team but not on the web page # get profiles from GitHub core_devs = [get_profile(login) for login in core_devs] emeritus = [get_profile(login) for login in emeritus] - triage_team = [get_profile(login) for login in triage_team] + contributor_experience_team = [ + get_profile(login) for login in contributor_experience_team + ] + emeritus_contributor_experience_team = [ + get_profile(login) for login in emeritus_contributor_experience_team + ] + comm_team = [get_profile(login) for login in comm_team] + emeritus_comm_team = [get_profile(login) for login in emeritus_comm_team] + documentation_team = [get_profile(login) for login in documentation_team] # sort by last name core_devs = sorted(core_devs, key=key) emeritus = sorted(emeritus, key=key) - triage_team = sorted(triage_team, key=key) - - return core_devs, emeritus, triage_team + contributor_experience_team = sorted(contributor_experience_team, key=key) + emeritus_contributor_experience_team = sorted( + emeritus_contributor_experience_team, key=key + ) + documentation_team = sorted(documentation_team, key=key) + comm_team = sorted(comm_team, key=key) + emeritus_comm_team = sorted(emeritus_comm_team, key=key) + + return ( + core_devs, + emeritus, + contributor_experience_team, + emeritus_contributor_experience_team, + comm_team, + emeritus_comm_team, + documentation_team, + ) def get_profile(login): """Get the GitHub profile from login""" - print("get profile for %s" % (login, )) + print("get profile for %s" % (login,)) try: profile = get("https://api.github.com/users/%s" % login).json() except requests.exceptions.HTTPError: @@ -97,11 +159,11 @@ def get_profile(login): # fix missing names missing_names = { - 'bthirion': 'Bertrand Thirion', - 'dubourg': 'Vincent Dubourg', - 'Duchesnay': 'Edouard Duchesnay', - 'Lars': 'Lars Buitinck', - 'MechCoder': 'Manoj Kumar', + "bthirion": "Bertrand Thirion", + "dubourg": "Vincent Dubourg", + "Duchesnay": "Edouard Duchesnay", + "Lars": "Lars Buitinck", + "MechCoder": "Manoj Kumar", } if profile["name"] in missing_names: profile["name"] = missing_names[profile["name"]] @@ -111,46 +173,83 @@ def get_profile(login): def key(profile): """Get a sorting key based on the lower case last name, then firstname""" - components = profile["name"].lower().split(' ') + components = profile["name"].lower().split(" ") return " ".join([components[-1]] + components[:-1]) def generate_table(contributors): lines = [ - (".. raw :: html\n"), - (" "), - ("
"), - (" "), + ".. raw :: html\n", + " ", + '
', + " ", ] for contributor in contributors: lines.append("
") lines.append( - "
" % - (contributor["html_url"], contributor["avatar_url"])) - lines.append("

%s

" % (contributor["name"], )) + "
" + % (contributor["html_url"], contributor["avatar_url"]) + ) + lines.append("

%s

" % (contributor["name"],)) lines.append("
") lines.append("
") - return '\n'.join(lines) + return "\n".join(lines) + "\n" def generate_list(contributors): lines = [] for contributor in contributors: - lines.append("- %s" % (contributor["name"], )) - return '\n'.join(lines) + lines.append("- %s" % (contributor["name"],)) + return "\n".join(lines) + "\n" if __name__ == "__main__": - - core_devs, emeritus, triage_team = get_contributors() - - with open(REPO_FOLDER / "doc" / "authors.rst", "w+") as rst_file: + ( + core_devs, + emeritus, + contributor_experience_team, + emeritus_contributor_experience_team, + comm_team, + emeritus_comm_team, + documentation_team, + ) = get_contributors() + + print("Generating rst files") + with open( + REPO_FOLDER / "doc" / "maintainers.rst", "w+", encoding="utf-8" + ) as rst_file: rst_file.write(generate_table(core_devs)) - with open(REPO_FOLDER / "doc" / "authors_emeritus.rst", "w+") as rst_file: + with open( + REPO_FOLDER / "doc" / "maintainers_emeritus.rst", "w+", encoding="utf-8" + ) as rst_file: rst_file.write(generate_list(emeritus)) - with open(REPO_FOLDER / "doc" / "triage_team.rst", "w+") as rst_file: - rst_file.write(generate_table(triage_team)) + with open( + REPO_FOLDER / "doc" / "contributor_experience_team.rst", "w+", encoding="utf-8" + ) as rst_file: + rst_file.write(generate_table(contributor_experience_team)) + + with open( + REPO_FOLDER / "doc" / "contributor_experience_team_emeritus.rst", + "w+", + encoding="utf-8", + ) as rst_file: + rst_file.write(generate_list(emeritus_contributor_experience_team)) + + with open( + REPO_FOLDER / "doc" / "communication_team.rst", "w+", encoding="utf-8" + ) as rst_file: + rst_file.write(generate_table(comm_team)) + + with open( + REPO_FOLDER / "doc" / "communication_team_emeritus.rst", "w+", encoding="utf-8" + ) as rst_file: + rst_file.write(generate_list(emeritus_comm_team)) + + with open( + REPO_FOLDER / "doc" / "documentation_team.rst", "w+", encoding="utf-8" + ) as rst_file: + rst_file.write(generate_table(documentation_team)) diff --git a/build_tools/get_comment.py b/build_tools/get_comment.py new file mode 100644 index 0000000000000..48ff14a058c9a --- /dev/null +++ b/build_tools/get_comment.py @@ -0,0 +1,351 @@ +# This script is used to generate a comment for a PR when linting issues are +# detected. It is used by the `Comment on failed linting` GitHub Action. +# This script fails if there are not comments to be posted. + +import os + +import requests + + +def get_versions(versions_file): + """Get the versions of the packages used in the linter job. + + Parameters + ---------- + versions_file : str + The path to the file that contains the versions of the packages. + + Returns + ------- + versions : dict + A dictionary with the versions of the packages. + """ + with open("versions.txt", "r") as f: + return dict(line.strip().split("=") for line in f) + + +def get_step_message(log, start, end, title, message, details): + """Get the message for a specific test. + + Parameters + ---------- + log : str + The log of the linting job. + + start : str + The string that marks the start of the test. + + end : str + The string that marks the end of the test. + + title : str + The title for this section. + + message : str + The message to be added at the beginning of the section. + + details : bool + Whether to add the details of each step. + + Returns + ------- + message : str + The message to be added to the comment. + """ + if end not in log: + return "" + res = ( + f"-----------------------------------------------\n### {title}\n\n{message}\n\n" + ) + if details: + res += ( + "
\n\n```\n" + + log[log.find(start) + len(start) + 1 : log.find(end) - 1] + + "\n```\n\n
\n\n" + ) + return res + + +def get_message(log_file, repo, pr_number, sha, run_id, details, versions): + with open(log_file, "r") as f: + log = f.read() + + sub_text = ( + "\n\n _Generated for commit:" + f" [{sha[:7]}](https://github.com/{repo}/pull/{pr_number}/commits/{sha}). " + "Link to the linter CI: [here]" + f"(https://github.com/{repo}/actions/runs/{run_id})_ " + ) + + if "### Linting completed ###" not in log: + return ( + "## ❌ Linting issues\n\n" + "There was an issue running the linter job. Please update with " + "`upstream/main` ([link](" + "https://scikit-learn.org/dev/developers/contributing.html" + "#how-to-contribute)) and push the changes. If you already have done " + "that, please send an empty commit with `git commit --allow-empty` " + "and push the changes to trigger the CI.\n\n" + sub_text + ) + + message = "" + + # ruff check + message += get_step_message( + log, + start="### Running the ruff linter ###", + end="Problems detected by ruff check", + title="`ruff check`", + message=( + "`ruff` detected issues. Please run " + "`ruff check --fix --output-format=full` locally, fix the remaining " + "issues, and push the changes. Here you can see the detected issues. Note " + f"that the installed `ruff` version is `ruff={versions['ruff']}`." + ), + details=details, + ) + + # ruff format + message += get_step_message( + log, + start="### Running the ruff formatter ###", + end="Problems detected by ruff format", + title="`ruff format`", + message=( + "`ruff` detected issues. Please run `ruff format` locally and push " + "the changes. Here you can see the detected issues. Note that the " + f"installed `ruff` version is `ruff={versions['ruff']}`." + ), + details=details, + ) + + # mypy + message += get_step_message( + log, + start="### Running mypy ###", + end="Problems detected by mypy", + title="`mypy`", + message=( + "`mypy` detected issues. Please fix them locally and push the changes. " + "Here you can see the detected issues. Note that the installed `mypy` " + f"version is `mypy={versions['mypy']}`." + ), + details=details, + ) + + # cython-lint + message += get_step_message( + log, + start="### Running cython-lint ###", + end="Problems detected by cython-lint", + title="`cython-lint`", + message=( + "`cython-lint` detected issues. Please fix them locally and push " + "the changes. Here you can see the detected issues. Note that the " + "installed `cython-lint` version is " + f"`cython-lint={versions['cython-lint']}`." + ), + details=details, + ) + + # deprecation order + message += get_step_message( + log, + start="### Checking for bad deprecation order ###", + end="Problems detected by deprecation order check", + title="Deprecation Order", + message=( + "Deprecation order check detected issues. Please fix them locally and " + "push the changes. Here you can see the detected issues." + ), + details=details, + ) + + # doctest directives + message += get_step_message( + log, + start="### Checking for default doctest directives ###", + end="Problems detected by doctest directive check", + title="Doctest Directives", + message=( + "doctest directive check detected issues. Please fix them locally and " + "push the changes. Here you can see the detected issues." + ), + details=details, + ) + + # joblib imports + message += get_step_message( + log, + start="### Checking for joblib imports ###", + end="Problems detected by joblib import check", + title="Joblib Imports", + message=( + "`joblib` import check detected issues. Please fix them locally and " + "push the changes. Here you can see the detected issues." + ), + details=details, + ) + + if not message: + # no issues detected, so this script "fails" + return ( + "## ✔ī¸ Linting Passed\n" + "All linting checks passed. Your pull request is in excellent shape! ☀ī¸" + + sub_text + ) + + if not details: + # This happens if posting the log fails, which happens if the log is too + # long. Typically, this happens if the PR branch hasn't been updated + # since we've introduced import sorting. + branch_not_updated = ( + "_Merging with `upstream/main` might fix / improve the issues if you " + "haven't done that since 21.06.2023._\n\n" + ) + else: + branch_not_updated = "" + + message = ( + "## ❌ Linting issues\n\n" + + branch_not_updated + + "This PR is introducing linting issues. Here's a summary of the issues. " + + "Note that you can avoid having linting issues by enabling `pre-commit` " + + "hooks. Instructions to enable them can be found [here](" + + "https://scikit-learn.org/dev/developers/contributing.html#how-to-contribute)" + + ".\n\n" + + "You can see the details of the linting issues under the `lint` job [here]" + + f"(https://github.com/{repo}/actions/runs/{run_id})\n\n" + + message + + sub_text + ) + + return message + + +def get_headers(token): + """Get the headers for the GitHub API.""" + return { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + } + + +def find_lint_bot_comments(repo, token, pr_number): + """Get the comment from the linting bot.""" + # repo is in the form of "org/repo" + # API doc: https://docs.github.com/en/rest/issues/comments?apiVersion=2022-11-28#list-issue-comments + response = requests.get( + f"https://api.github.com/repos/{repo}/issues/{pr_number}/comments", + headers=get_headers(token), + ) + response.raise_for_status() + all_comments = response.json() + + failed_comment = "❌ Linting issues" + success_comment = "✔ī¸ Linting Passed" + + # Find all comments that match the linting bot, and return the first one. + # There should always be only one such comment, or none, if the PR is + # just created. + comments = [ + comment + for comment in all_comments + if comment["user"]["login"] == "github-actions[bot]" + and (failed_comment in comment["body"] or success_comment in comment["body"]) + ] + + if len(all_comments) > 25 and not comments: + # By default the API returns the first 30 comments. If we can't find the + # comment created by the bot in those, then we raise and we skip creating + # a comment in the first place. + raise RuntimeError("Comment not found in the first 30 comments.") + + return comments[0] if comments else None + + +def create_or_update_comment(comment, message, repo, pr_number, token): + """Create a new comment or update existing one.""" + # repo is in the form of "org/repo" + if comment is not None: + print("updating existing comment") + # API doc: https://docs.github.com/en/rest/issues/comments?apiVersion=2022-11-28#update-an-issue-comment + response = requests.patch( + f"https://api.github.com/repos/{repo}/issues/comments/{comment['id']}", + headers=get_headers(token), + json={"body": message}, + ) + else: + print("creating new comment") + # API doc: https://docs.github.com/en/rest/issues/comments?apiVersion=2022-11-28#create-an-issue-comment + response = requests.post( + f"https://api.github.com/repos/{repo}/issues/{pr_number}/comments", + headers=get_headers(token), + json={"body": message}, + ) + + response.raise_for_status() + + +if __name__ == "__main__": + repo = os.environ["GITHUB_REPOSITORY"] + token = os.environ["GITHUB_TOKEN"] + pr_number = os.environ["PR_NUMBER"] + sha = os.environ["BRANCH_SHA"] + log_file = os.environ["LOG_FILE"] + run_id = os.environ["RUN_ID"] + versions_file = os.environ["VERSIONS_FILE"] + + versions = get_versions(versions_file) + + if not repo or not token or not pr_number or not log_file or not run_id: + raise ValueError( + "One of the following environment variables is not set: " + "GITHUB_REPOSITORY, GITHUB_TOKEN, PR_NUMBER, LOG_FILE, RUN_ID" + ) + + try: + comment = find_lint_bot_comments(repo, token, pr_number) + except RuntimeError: + print("Comment not found in the first 30 comments. Skipping!") + exit(0) + + try: + message = get_message( + log_file, + repo=repo, + pr_number=pr_number, + sha=sha, + run_id=run_id, + details=True, + versions=versions, + ) + create_or_update_comment( + comment=comment, + message=message, + repo=repo, + pr_number=pr_number, + token=token, + ) + print(message) + except requests.HTTPError: + # The above fails if the message is too long. In that case, we + # try again without the details. + message = get_message( + log_file, + repo=repo, + pr_number=pr_number, + sha=sha, + run_id=run_id, + details=False, + versions=versions, + ) + create_or_update_comment( + comment=comment, + message=message, + repo=repo, + pr_number=pr_number, + token=token, + ) + print(message) diff --git a/build_tools/github/Windows b/build_tools/github/Windows deleted file mode 100644 index 5ba35f790ca5e..0000000000000 --- a/build_tools/github/Windows +++ /dev/null @@ -1,15 +0,0 @@ -# Get the Python version of the base image from a build argument -ARG PYTHON_VERSION -FROM winamd64/python:$PYTHON_VERSION-windowsservercore - -ARG WHEEL_NAME -ARG CONFTEST_NAME -ARG CIBW_TEST_REQUIRES - -# Copy and install the Windows wheel -COPY $WHEEL_NAME $WHEEL_NAME -COPY $CONFTEST_NAME $CONFTEST_NAME -RUN pip install $env:WHEEL_NAME - -# Install the testing dependencies -RUN pip install $env:CIBW_TEST_REQUIRES.split(" ") diff --git a/build_tools/github/build_minimal_windows_image.sh b/build_tools/github/build_minimal_windows_image.sh old mode 100644 new mode 100755 index 67a6180952591..8cc9af937dfd9 --- a/build_tools/github/build_minimal_windows_image.sh +++ b/build_tools/github/build_minimal_windows_image.sh @@ -4,28 +4,48 @@ set -e set -x PYTHON_VERSION=$1 -BITNESS=$2 -if [[ "$PYTHON_VERSION" == "36" || "$BITNESS" == "32" ]]; then - # Python 3.6 and 32-bit architectures are not supported - # by the official Docker images: Tests will just be run - # on the host (instead of the minimal Docker container). - exit 0 -fi +FREE_THREADED_BUILD="$(python -c"import sysconfig; print(bool(sysconfig.get_config_var('Py_GIL_DISABLED')))")" + +if [[ $FREE_THREADED_BUILD == "False" ]]; then + # Prepare a minimal Windows environment without any developer runtime libraries + # installed to check that the scikit-learn wheel does not implicitly rely on + # external DLLs when running the tests. + TEMP_FOLDER="$HOME/AppData/Local/Temp" + WHEEL_PATH=$(ls -d $TEMP_FOLDER/**/*/repaired_wheel/*) + WHEEL_NAME=$(basename $WHEEL_PATH) + + cp $WHEEL_PATH $WHEEL_NAME -TEMP_FOLDER="$HOME/AppData/Local/Temp" -WHEEL_PATH=$(ls -d $TEMP_FOLDER/*/repaired_wheel/*) -WHEEL_NAME=$(basename $WHEEL_PATH) + # Dot the Python version for identifying the base Docker image + PYTHON_DOCKER_IMAGE_PART=$(echo ${PYTHON_VERSION:0:1}.${PYTHON_VERSION:1:2}) -cp $WHEEL_PATH $WHEEL_NAME + if [[ "$CIBW_PRERELEASE_PYTHONS" =~ [tT]rue ]]; then + PYTHON_DOCKER_IMAGE_PART="${PYTHON_DOCKER_IMAGE_PART}-rc" + fi -# Dot the Python version for identyfing the base Docker image -PYTHON_VERSION=$(echo ${PYTHON_VERSION:0:1}.${PYTHON_VERSION:1:2}) + # We could have all of the following logic in a Dockerfile but it's a lot + # easier to do it in bash rather than figure out how to do it in Powershell + # inside the Dockerfile ... + DOCKER_IMAGE="winamd64/python:${PYTHON_DOCKER_IMAGE_PART}-windowsservercore" + MNT_FOLDER="C:/mnt" + CONTAINER_ID=$(docker run -it -v "$(cygpath -w $PWD):$MNT_FOLDER" -d $DOCKER_IMAGE) -# Build a minimal Windows Docker image for testing the wheels -docker build --build-arg PYTHON_VERSION=$PYTHON_VERSION \ - --build-arg WHEEL_NAME=$WHEEL_NAME \ - --build-arg CONFTEST_NAME=$CONFTEST_NAME \ - --build-arg CIBW_TEST_REQUIRES="$CIBW_TEST_REQUIRES" \ - -f build_tools/github/Windows \ - -t scikit-learn/minimal-windows . + function exec_inside_container() { + docker exec $CONTAINER_ID powershell -Command $1 + } + + exec_inside_container "python -m pip install $MNT_FOLDER/$WHEEL_NAME" + exec_inside_container "python -m pip install $CIBW_TEST_REQUIRES" + + # Save container state to scikit-learn/minimal-windows image. On Windows the + # container needs to be stopped first. + docker stop $CONTAINER_ID + docker commit $CONTAINER_ID scikit-learn/minimal-windows +else + # This is too cumbersome to use a Docker image in the free-threaded case + # TODO When pandas has a release with a Windows free-threaded wheel we can + # replace the next line with + # python -m pip install CIBW_TEST_REQUIRES + python -m pip install pytest +fi diff --git a/build_tools/github/build_source.sh b/build_tools/github/build_source.sh old mode 100644 new mode 100755 index a4d9c7bd05387..ec53284012fa4 --- a/build_tools/github/build_source.sh +++ b/build_tools/github/build_source.sh @@ -11,10 +11,10 @@ python -m venv build_env source build_env/bin/activate python -m pip install numpy scipy cython -python -m pip install twine +python -m pip install twine build cd scikit-learn/scikit-learn -python setup.py sdist +python -m build --sdist # Check whether the source distribution will render correctly twine check dist/*.tar.gz diff --git a/build_tools/github/build_test_arm.sh b/build_tools/github/build_test_arm.sh new file mode 100755 index 0000000000000..db11fdc0e82f0 --- /dev/null +++ b/build_tools/github/build_test_arm.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +set -e +set -x + +UNAMESTR=`uname` +N_CORES=`nproc --all` + +# defines the get_dep and show_installed_libraries functions +source build_tools/shared.sh + +setup_ccache() { + echo "Setting up ccache" + mkdir /tmp/ccache/ + which ccache + for name in gcc g++ cc c++ x86_64-linux-gnu-gcc x86_64-linux-gnu-c++; do + ln -s $(which ccache) "/tmp/ccache/${name}" + done + export PATH="/tmp/ccache:${PATH}" + # Unset ccache limits + ccache -F 0 + ccache -M 0 +} + +setup_ccache + +python --version + +# Disable the build isolation and build in the tree so that the same folder can be +# cached between CI runs. +pip install --verbose --no-build-isolation . + +# Report cache usage +ccache -s --verbose + +micromamba list + +# Changing directory not to have module resolution use scikit-learn source +# directory but to the installed package. +cd /tmp +python -c "import sklearn; sklearn.show_versions()" +python -m threadpoolctl --import sklearn +# Test using as many workers as available cores +pytest --pyargs -n $N_CORES sklearn diff --git a/build_tools/github/build_wheels.sh b/build_tools/github/build_wheels.sh deleted file mode 100644 index 9b45481cbb978..0000000000000 --- a/build_tools/github/build_wheels.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -set -e -set -x - -# OpenMP is not present on macOS by default -if [[ "$RUNNER_OS" == "macOS" ]]; then - # Make sure to use a libomp version binary compatible with the oldest - # supported version of the macos SDK as libomp will be vendored into the - # scikit-learn wheels for macos. The list of bottles can be found at: - # https://formulae.brew.sh/api/formula/libomp.json. Currently, the oldest - # supported macos version is: High Sierra / 10.13. When upgrading this, be - # sure to update the MACOSX_DEPLOYMENT_TARGET environment variable in - # wheels.yml accordingly. - wget https://homebrew.bintray.com/bottles/libomp-11.0.0.high_sierra.bottle.tar.gz - brew install libomp-11.0.0.high_sierra.bottle.tar.gz - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - export CPPFLAGS="$CPPFLAGS -Xpreprocessor -fopenmp" - export CFLAGS="$CFLAGS -I/usr/local/opt/libomp/include" - export CXXFLAGS="$CXXFLAGS -I/usr/local/opt/libomp/include" - export LDFLAGS="$LDFLAGS -Wl,-rpath,/usr/local/opt/libomp/lib -L/usr/local/opt/libomp/lib -lomp" -fi - -# The version of the built dependencies are specified -# in the pyproject.toml file, while the tests are run -# against the most recent version of the dependencies - -python -m pip install cibuildwheel -python -m cibuildwheel --output-dir wheelhouse diff --git a/build_tools/github/check_build_trigger.sh b/build_tools/github/check_build_trigger.sh old mode 100644 new mode 100755 index b80fe4674864d..e6bc77b00e71f --- a/build_tools/github/check_build_trigger.sh +++ b/build_tools/github/check_build_trigger.sh @@ -7,6 +7,7 @@ COMMIT_MSG=$(git log --no-merges -1 --oneline) # The commit marker "[cd build]" will trigger the build when required if [[ "$GITHUB_EVENT_NAME" == schedule || + "$GITHUB_EVENT_NAME" == workflow_dispatch || "$COMMIT_MSG" =~ \[cd\ build\] ]]; then - echo "::set-output name=build::true" + echo "build=true" >> $GITHUB_OUTPUT fi diff --git a/build_tools/github/check_wheels.py b/build_tools/github/check_wheels.py index c213991394a6b..21c9a529b265b 100644 --- a/build_tools/github/check_wheels.py +++ b/build_tools/github/check_wheels.py @@ -1,41 +1,29 @@ """Checks that dist/* contains the number of wheels built from the .github/workflows/wheels.yml config.""" -import yaml -from pathlib import Path + import sys +from pathlib import Path + +import yaml gh_wheel_path = Path.cwd() / ".github" / "workflows" / "wheels.yml" -with gh_wheel_path.open('r') as f: +with gh_wheel_path.open("r") as f: wheel_config = yaml.safe_load(f) -build_matrix = wheel_config['jobs']['build_wheels']['strategy']['matrix'] -n_python_versions = len(build_matrix['python']) - -# For each python version we have: 7 wheels -# 1 osx wheel (x86_64) -# 4 linux wheel (i686 + x86_64) * (manylinux1 + manylinux2010) -# 2 windows wheel (win32 + wind_amd64) -n_wheels = 7 * n_python_versions +build_matrix = wheel_config["jobs"]["build_wheels"]["strategy"]["matrix"]["include"] +n_wheels = len(build_matrix) # plus one more for the sdist n_wheels += 1 -# aarch64 builds from travis -travis_config_path = Path.cwd() / ".travis.yml" -with travis_config_path.open('r') as f: - travis_config = yaml.safe_load(f) - -jobs = travis_config['jobs']['include'] -travis_builds = [j for j in jobs - if any("CIBW_BUILD" in env for env in j["env"])] -n_wheels += len(travis_builds) - -dist_files = list(Path("dist").glob('**/*')) +dist_files = list(Path("dist").glob("**/*")) n_dist_files = len(dist_files) if n_dist_files != n_wheels: - print(f"Expected {n_wheels} wheels in dist/* but " - f"got {n_dist_files} artifacts instead.") + print( + f"Expected {n_wheels} wheels in dist/* but " + f"got {n_dist_files} artifacts instead." + ) sys.exit(1) print(f"dist/* has the expected {n_wheels} wheels:") diff --git a/build_tools/github/create_gpu_environment.sh b/build_tools/github/create_gpu_environment.sh new file mode 100755 index 0000000000000..96a62d7678566 --- /dev/null +++ b/build_tools/github/create_gpu_environment.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -e +set -x + +curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" +bash Miniforge3-$(uname)-$(uname -m).sh -b -p "${HOME}/conda" +source "${HOME}/conda/etc/profile.d/conda.sh" + + +# defines the get_dep and show_installed_libraries functions +source build_tools/shared.sh +conda activate base + +CONDA_ENV_NAME=sklearn +LOCK_FILE=build_tools/github/pylatest_conda_forge_cuda_array-api_linux-64_conda.lock +create_conda_environment_from_lock_file $CONDA_ENV_NAME $LOCK_FILE + +conda activate $CONDA_ENV_NAME +conda list diff --git a/build_tools/github/pylatest_conda_forge_cuda_array-api_linux-64_conda.lock b/build_tools/github/pylatest_conda_forge_cuda_array-api_linux-64_conda.lock new file mode 100644 index 0000000000000..315164f96c77c --- /dev/null +++ b/build_tools/github/pylatest_conda_forge_cuda_array-api_linux-64_conda.lock @@ -0,0 +1,254 @@ +# Generated by conda-lock. +# platform: linux-64 +# input_hash: 0c167b26e12c284b769bf4d76bd3e604db266ed21c8f9e11e4bb737419ccdc93 +@EXPLICIT +https://conda.anaconda.org/conda-forge/noarch/cuda-version-11.8-h70ddcb2_3.conda#670f0e1593b8c1d84f57ad5fe5256799 +https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2#0c96522c6bdaed4b1566d11387caaf45 +https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2#34893075a5c9e55cdafac56607368fc6 +https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2#4d59c254e01d9cde7957100457e2d5fb +https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-h77eed37_3.conda#49023d73832ef61042f6a237cb2687e7 +https://conda.anaconda.org/conda-forge/noarch/kernel-headers_linux-64-3.10.0-he073ed8_18.conda#ad8527bf134a90e1c9ed35fa0b64318c +https://conda.anaconda.org/conda-forge/linux-64/libopentelemetry-cpp-headers-1.18.0-ha770c72_1.conda#4fb055f57404920a43b147031471e03b +https://conda.anaconda.org/conda-forge/linux-64/nlohmann_json-3.12.0-h3f2d84a_0.conda#d76872d096d063e226482c99337209dc +https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-7_cp313.conda#e84b44e6300f1703cb25d29120c5b1d8 +https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda#4222072737ccff51314b5ece9c7d6f5a +https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.4.26-hbd8a1cb_0.conda#95db94f75ba080a22eb623590993167b +https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-0.tar.bz2#f766549260d6815b0c52253f1fb1bb29 +https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.43-h712a8e2_4.conda#01f8d123c96816249efd255a31ad7712 +https://conda.anaconda.org/conda-forge/linux-64/libglvnd-1.7.0-ha4b6fd6_2.conda#434ca7e50e40f4918ab701e3facd59a0 +https://conda.anaconda.org/conda-forge/linux-64/llvm-openmp-20.1.6-h024ca30_0.conda#e4ece7ed81e43ae97a3b58ac4230c3c5 +https://conda.anaconda.org/conda-forge/noarch/sysroot_linux-64-2.17-h0157908_18.conda#460eba7851277ec1fd80a1a24080787a +https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-3_kmp_llvm.conda#ee5c2118262e30b972bc0b4db8ef0ba5 +https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2#fee5683a3f04bd15cbd8318b096a27ab +https://conda.anaconda.org/conda-forge/linux-64/libegl-1.7.0-ha4b6fd6_2.conda#c151d5eb730e9b7480e6d48c0fc44048 +https://conda.anaconda.org/conda-forge/linux-64/libopengl-1.7.0-ha4b6fd6_2.conda#7df50d44d4a14d6c31a2c54f2cd92157 +https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.1.0-h767d61c_2.conda#ea8ac52380885ed41c1baa8f1d6d2b93 +https://conda.anaconda.org/conda-forge/linux-64/alsa-lib-1.2.14-hb9d3cd8_0.conda#76df83c2a9035c54df5d04ff81bcc02d +https://conda.anaconda.org/conda-forge/linux-64/aws-c-common-0.12.0-hb9d3cd8_0.conda#f65c946f28f0518f41ced702f44c52b7 +https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.34.5-hb9d3cd8_0.conda#f7f0d6cc2dc986d42ac2689ec88192be +https://conda.anaconda.org/conda-forge/linux-64/libbrotlicommon-1.1.0-hb9d3cd8_2.conda#41b599ed2b02abcfdd84302bff174b23 +https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.24-h86f0d12_0.conda#64f0c503da58ec25ebd359e4d990afa8 +https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.7.0-h5888daf_0.conda#db0bfbe7dd197b68ad5f30333bae6ce0 +https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.6-h2dba641_1.conda#ede4673863426c0883c0063d853bbd85 +https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-15.1.0-h69a702a_2.conda#ddca86c7040dd0e73b2b69bd7833d225 +https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-15.1.0-hcea5267_2.conda#01de444988ed960031dbe84cf4f9b1fc +https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.18-h4ce23a2_1.conda#e796ff8ddc598affdf7c173d6145f087 +https://conda.anaconda.org/conda-forge/linux-64/libjpeg-turbo-3.1.0-hb9d3cd8_0.conda#9fa334557db9f63da6c9285fd2a48638 +https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.1-hb9d3cd8_1.conda#a76fd702c93cd2dfd89eff30a5fd45a8 +https://conda.anaconda.org/conda-forge/linux-64/libmpdec-4.0.0-hb9d3cd8_0.conda#c7e925f37e3b40d893459e625f6a53f1 +https://conda.anaconda.org/conda-forge/linux-64/libntlm-1.8-hb9d3cd8_0.conda#7c7927b404672409d9917d49bff5f2d6 +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-15.1.0-h8f9b012_2.conda#1cb1c67961f6dd257eae9e9691b341aa +https://conda.anaconda.org/conda-forge/linux-64/libutf8proc-2.10.0-h202a827_0.conda#0f98f3e95272d118f7931b6bef69bfe5 +https://conda.anaconda.org/conda-forge/linux-64/libuv-1.51.0-hb9d3cd8_0.conda#1349c022c92c5efd3fd705a79a5804d8 +https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.5.0-h851e524_0.conda#63f790534398730f59e1b899c3644d4a +https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda#edb0dca6bc32e4f4789199455a1dbeb8 +https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda#47e340acb35de30501a76c7c799c41d7 +https://conda.anaconda.org/conda-forge/linux-64/openssl-3.5.0-h7b32b05_1.conda#de356753cfdbffcde5bb1e86e3aa6cd0 +https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-hb9d3cd8_1002.conda#b3c17d95b5a10c6e64a21fa17573e70e +https://conda.anaconda.org/conda-forge/linux-64/xorg-libice-1.1.2-hb9d3cd8_0.conda#fb901ff28063514abb6046c9ec2c4a45 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxau-1.0.12-hb9d3cd8_0.conda#f6ebe2cb3f82ba6c057dde5d9debe4f7 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdmcp-1.1.5-hb9d3cd8_0.conda#8035c64cb77ed555e3f150b7b3972480 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-cal-0.8.7-h043a21b_0.conda#4fdf835d66ea197e693125c64fbd4482 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-compression-0.3.1-h3870646_2.conda#17ccde79d864e6183a83c5bbb8fff34d +https://conda.anaconda.org/conda-forge/linux-64/aws-c-sdkutils-0.2.3-h3870646_2.conda#06008b5ab42117c89c982aa2a32a5b25 +https://conda.anaconda.org/conda-forge/linux-64/aws-checksums-0.2.3-h3870646_2.conda#303d9e83e0518f1dcb66e90054635ca6 +https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h4bc722e_7.conda#62ee74e96c5ebb0af99386de58cf9553 +https://conda.anaconda.org/conda-forge/linux-64/double-conversion-3.3.1-h5888daf_0.conda#bfd56492d8346d669010eccafe0ba058 +https://conda.anaconda.org/conda-forge/linux-64/gflags-2.2.2-h5888daf_1005.conda#d411fc29e338efb48c5fd4576d71d881 +https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.1-h166bdaf_0.tar.bz2#30186d27e2c9fa62b45fb1476b7200e3 +https://conda.anaconda.org/conda-forge/linux-64/lerc-4.0.0-h0aef613_1.conda#9344155d33912347b37f0ae6c410a835 +https://conda.anaconda.org/conda-forge/linux-64/libabseil-20240722.0-cxx17_hbbce691_4.conda#488f260ccda0afaf08acb286db439c2f +https://conda.anaconda.org/conda-forge/linux-64/libbrotlidec-1.1.0-hb9d3cd8_2.conda#9566f0bd264fbd463002e759b8a82401 +https://conda.anaconda.org/conda-forge/linux-64/libbrotlienc-1.1.0-hb9d3cd8_2.conda#06f70867945ea6a84d35836af780f1de +https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20250104-pl5321h7949ede_0.conda#c277e0a4d549b03ac1e9d6cbbe3d017b +https://conda.anaconda.org/conda-forge/linux-64/libev-4.33-hd590300_2.conda#172bf1cd1ff8629f2b1179945ed45055 +https://conda.anaconda.org/conda-forge/linux-64/libevent-2.1.12-hf998b51_1.conda#a1cfcc585f0c42bf8d5546bb1dfb668d +https://conda.anaconda.org/conda-forge/linux-64/libgfortran-15.1.0-h69a702a_2.conda#f92e6e0a3c0c0c85561ef61aa59d555d +https://conda.anaconda.org/conda-forge/linux-64/libpciaccess-0.18-hd590300_0.conda#48f4330bfcd959c3cfb704d424903c82 +https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.47-h943b412_0.conda#55199e2ae2c3651f6f9b2a447b47bdc9 +https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.50.0-hee588c1_0.conda#71888e92098d0f8c41b09a671ad289bc +https://conda.anaconda.org/conda-forge/linux-64/libssh2-1.11.1-hcf80075_0.conda#eecce068c7e4eddeb169591baac20ac4 +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-15.1.0-h4852527_2.conda#9d2072af184b5caa29492bf2344597bb +https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda#40b61aab5c7ba9ff276c41cfffe6b80b +https://conda.anaconda.org/conda-forge/linux-64/libxcb-1.17.0-h8a09558_0.conda#92ed62436b625154323d40d5f2f11dd7 +https://conda.anaconda.org/conda-forge/linux-64/lz4-c-1.10.0-h5888daf_1.conda#9de5350a85c4a20c685259b889aa6393 +https://conda.anaconda.org/conda-forge/linux-64/ninja-1.12.1-hff21bea_1.conda#2322531904f27501ee19847b87ba7c64 +https://conda.anaconda.org/conda-forge/linux-64/pixman-0.46.0-h29eaf8c_0.conda#d2f1c87d4416d1e7344cf92b1aaee1c4 +https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8c095d6_2.conda#283b96675859b20a825f8fa30f311446 +https://conda.anaconda.org/conda-forge/linux-64/s2n-1.5.14-h6c98b2b_0.conda#efab4ad81ba5731b2fefa0ab4359e884 +https://conda.anaconda.org/conda-forge/linux-64/sleef-3.8-h1b44611_0.conda#aec4dba5d4c2924730088753f6fa164b +https://conda.anaconda.org/conda-forge/linux-64/snappy-1.2.1-h8bd8927_1.conda#3b3e64af585eadfb52bb90b553db5edf +https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_hd72426e_102.conda#a0116df4f4ed05c303811a837d5b39d8 +https://conda.anaconda.org/conda-forge/linux-64/wayland-1.23.1-h3e06ad9_1.conda#a37843723437ba75f42c9270ffe800b1 +https://conda.anaconda.org/conda-forge/linux-64/zlib-1.3.1-hb9d3cd8_2.conda#c9f075ab2f33b3bbee9e62d4ad0a6cd8 +https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb8e6e7a_2.conda#6432cb5d4ac0046c3ac0a8a0f95842f9 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-io-0.17.0-h3dad3f2_6.conda#3a127d28266cdc0da93384d1f59fe8df +https://conda.anaconda.org/conda-forge/linux-64/brotli-bin-1.1.0-hb9d3cd8_2.conda#c63b5e52939e795ba8d26e35d767a843 +https://conda.anaconda.org/conda-forge/linux-64/cudatoolkit-11.8.0-h4ba93d1_13.conda#eb43f5f1f16e2fad2eba22219c3e499b +https://conda.anaconda.org/conda-forge/linux-64/glog-0.7.1-hbabe93e_0.conda#ff862eebdfeb2fd048ae9dc92510baca +https://conda.anaconda.org/conda-forge/linux-64/gmp-6.3.0-hac33072_2.conda#c94a5994ef49749880a8139cf9afcbe1 +https://conda.anaconda.org/conda-forge/linux-64/graphite2-1.3.13-h59595ed_1003.conda#f87c7b7c2cb45f323ffbce941c78ab7c +https://conda.anaconda.org/conda-forge/linux-64/icu-75.1-he02047a_0.conda#8b189310083baabfb622af68fd9d3ae3 +https://conda.anaconda.org/conda-forge/linux-64/krb5-1.21.3-h659f571_0.conda#3f43953b7d3fb3aaa1d0d0723d91e368 +https://conda.anaconda.org/conda-forge/linux-64/libcrc32c-1.1.2-h9c3ff4c_0.tar.bz2#c965a5aa0d5c1c37ffc62dff36e28400 +https://conda.anaconda.org/conda-forge/linux-64/libdrm-2.4.124-hb9d3cd8_0.conda#8bc89311041d7fcb510238cf0848ccae +https://conda.anaconda.org/conda-forge/linux-64/libfreetype6-2.13.3-h48d6fc4_1.conda#3c255be50a506c50765a93a6644f32fe +https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-15.1.0-h69a702a_2.conda#a483a87b71e974bb75d1b9413d4436dd +https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.64.0-h161d5f1_0.conda#19e57602824042dfd0446292ef90488b +https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.29-pthreads_h94d23a6_0.conda#0a4d0252248ef9a0f88f2ba8b8a08e12 +https://conda.anaconda.org/conda-forge/linux-64/libprotobuf-5.28.3-h6128344_1.conda#d8703f1ffe5a06356f06467f1d0b9464 +https://conda.anaconda.org/conda-forge/linux-64/libre2-11-2024.07.02-hbbce691_2.conda#b2fede24428726dd867611664fb372e8 +https://conda.anaconda.org/conda-forge/linux-64/libthrift-0.21.0-h0e7cc3e_0.conda#dcb95c0a98ba9ff737f7ae482aef7833 +https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.7.0-hf01ce69_5.conda#e79a094918988bb1807462cd42c83962 +https://conda.anaconda.org/conda-forge/linux-64/nccl-2.26.6.1-h03a54cd_0.conda#323f9253803b715728e8d0c94c495f53 +https://conda.anaconda.org/conda-forge/linux-64/pcre2-10.45-hc749103_0.conda#b90bece58b4c2bf25969b70f3be42d25 +https://conda.anaconda.org/conda-forge/linux-64/python-3.13.3-hf636f53_101_cp313.conda#10622e12d649154af0bd76bcf33a7c5c +https://conda.anaconda.org/conda-forge/linux-64/qhull-2020.2-h434a139_5.conda#353823361b1d27eb3960efb076dfcaf6 +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-0.4.1-hb711507_2.conda#8637c3e5821654d0edf97e2b0404b443 +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-keysyms-0.4.1-hb711507_0.conda#ad748ccca349aec3e91743e08b5e2b50 +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-renderutil-0.3.10-hb711507_0.conda#0e0cbe0564d03a99afd5fd7b362feecd +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-wm-0.4.2-hb711507_0.conda#608e0ef8256b81d04456e8d211eee3e8 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.6-he73a12e_0.conda#1c74ff8c35dcadf952a16f752ca5aa49 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.8.12-h4f16b4b_0.conda#db038ce880f100acc74dba10302b5630 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-event-stream-0.5.4-h04a3f94_2.conda#81096a80f03fc2f0fb2a230f5d028643 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-http-0.9.4-hb9b18c6_4.conda#773c99d0dbe2b3704af165f97ff399e5 +https://conda.anaconda.org/conda-forge/linux-64/brotli-1.1.0-hb9d3cd8_2.conda#98514fe74548d768907ce7a13f680e8f +https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda#962b9857ee8e7018c22f2776ffa0b2d7 +https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.3-py313hd8ed1ab_101.conda#904a822cbd380adafb9070debf8579a8 +https://conda.anaconda.org/conda-forge/noarch/cycler-0.12.1-pyhd8ed1ab_1.conda#44600c4667a319d67dbe0681fc0bc833 +https://conda.anaconda.org/conda-forge/linux-64/cyrus-sasl-2.1.27-h54b06d7_7.conda#dce22f70b4e5a407ce88f2be046f4ceb +https://conda.anaconda.org/conda-forge/linux-64/cython-3.1.1-py313h5dec8f5_1.conda#f114755cdd37627732b1884b7b15d4b5 +https://conda.anaconda.org/conda-forge/noarch/execnet-2.1.1-pyhd8ed1ab_1.conda#a71efeae2c160f6789900ba2631a2c90 +https://conda.anaconda.org/conda-forge/linux-64/fastrlock-0.8.3-py313h9800cb9_1.conda#54dd71b3be2ed6ccc50f180347c901db +https://conda.anaconda.org/conda-forge/noarch/filelock-3.18.0-pyhd8ed1ab_0.conda#4547b39256e296bb758166893e909a7c +https://conda.anaconda.org/conda-forge/noarch/fsspec-2025.5.1-pyhd8ed1ab_0.conda#2d2c9ef879a7e64e2dc657b09272c2b6 +https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_1.conda#6837f3eff7dcea42ecd714ce1ac2b108 +https://conda.anaconda.org/conda-forge/linux-64/kiwisolver-1.4.7-py313h33d0bda_0.conda#9862d13a5e466273d5a4738cffcb8d6c +https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.17-h717163a_0.conda#000e85703f0fd9594c81710dd5066471 +https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-31_h59b9bed_openblas.conda#728dbebd0f7a20337218beacffd37916 +https://conda.anaconda.org/conda-forge/linux-64/libcudnn-9.10.1.4-h7d33bf5_0.conda#93fe78190bc6fe40d5e7a737c8065286 +https://conda.anaconda.org/conda-forge/linux-64/libcups-2.3.3-h4637d8d_4.conda#d4529f4dff3057982a7617c7ac58fde3 +https://conda.anaconda.org/conda-forge/linux-64/libcurl-8.14.0-h332b0f4_0.conda#d1738cf06503218acee63669029fd8e8 +https://conda.anaconda.org/conda-forge/linux-64/libfreetype-2.13.3-ha770c72_1.conda#51f5be229d83ecd401fb369ab96ae669 +https://conda.anaconda.org/conda-forge/linux-64/libglib-2.84.2-h3618099_0.conda#072ab14a02164b7c0c089055368ff776 +https://conda.anaconda.org/conda-forge/linux-64/libglx-1.7.0-ha4b6fd6_2.conda#c8013e438185f33b13814c5c488acd5c +https://conda.anaconda.org/conda-forge/linux-64/libhiredis-1.0.2-h2cc385e_0.tar.bz2#b34907d3a81a3cd8095ee83d174c074a +https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.13.8-h4bc477f_0.conda#14dbe05b929e329dbaa6f2d0aa19466d +https://conda.anaconda.org/conda-forge/linux-64/markupsafe-3.0.2-py313h8060acc_1.conda#21b62c55924f01b6eef6827167b46acb +https://conda.anaconda.org/conda-forge/noarch/meson-1.8.1-pyhe01879c_0.conda#f3cccd9a6ce5331ae33f69ade5529162 +https://conda.anaconda.org/conda-forge/linux-64/mpfr-4.2.1-h90cbb55_3.conda#2eeb50cab6652538eee8fc0bc3340c81 +https://conda.anaconda.org/conda-forge/noarch/mpmath-1.3.0-pyhd8ed1ab_1.conda#3585aa87c43ab15b167b574cd73b057b +https://conda.anaconda.org/conda-forge/noarch/munkres-1.1.4-pyh9f0ad1d_0.tar.bz2#2ba8498c1018c1e9c61eb99b973dfe19 +https://conda.anaconda.org/conda-forge/noarch/networkx-3.4.2-pyh267e887_2.conda#fd40bf7f7f4bc4b647dc8512053d9873 +https://conda.anaconda.org/conda-forge/linux-64/openblas-0.3.29-pthreads_h6ec200e_0.conda#7e4d48870b3258bea920d51b7f495a81 +https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.3-h5fbd93e_0.conda#9e5816bc95d285c115a3ebc2f8563564 +https://conda.anaconda.org/conda-forge/linux-64/orc-2.1.1-h2271f48_0.conda#67075ef2cb33079efee3abfe58127a3b +https://conda.anaconda.org/conda-forge/noarch/packaging-25.0-pyh29332c3_1.conda#58335b26c38bf4a20f399384c33cbcf9 +https://conda.anaconda.org/conda-forge/noarch/pip-25.1.1-pyh145f28c_0.conda#01384ff1639c6330a0924791413b8714 +https://conda.anaconda.org/conda-forge/noarch/pluggy-1.6.0-pyhd8ed1ab_0.conda#7da7ccd349dbf6487a7778579d2bb971 +https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.2.3-pyhd8ed1ab_1.conda#513d3c262ee49b54a8fec85c5bc99764 +https://conda.anaconda.org/conda-forge/noarch/python-tzdata-2025.2-pyhd8ed1ab_0.conda#88476ae6ebd24f39261e0854ac244f33 +https://conda.anaconda.org/conda-forge/noarch/pytz-2025.2-pyhd8ed1ab_0.conda#bc8e3267d44011051f2eb14d22fb0960 +https://conda.anaconda.org/conda-forge/linux-64/re2-2024.07.02-h9925aae_2.conda#e84ddf12bde691e8ec894b00ea829ddf +https://conda.anaconda.org/conda-forge/noarch/setuptools-80.9.0-pyhff2d567_0.conda#4de79c071274a53dcaf2a8c749d1499e +https://conda.anaconda.org/conda-forge/noarch/six-1.17.0-pyhd8ed1ab_0.conda#a451d576819089b0d672f18768be0f65 +https://conda.anaconda.org/conda-forge/noarch/threadpoolctl-3.6.0-pyhecae5ae_0.conda#9d64911b31d57ca443e9f1e36b04385f +https://conda.anaconda.org/conda-forge/noarch/toml-0.10.2-pyhd8ed1ab_1.conda#b0dd904de08b7db706167240bf37b164 +https://conda.anaconda.org/conda-forge/noarch/tomli-2.2.1-pyhd8ed1ab_1.conda#ac944244f1fed2eb49bae07193ae8215 +https://conda.anaconda.org/conda-forge/linux-64/tornado-6.5.1-py313h536fd9c_0.conda#e9434a5155db25c38ade26f71a2f5a48 +https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.13.2-pyh29332c3_0.conda#83fc6ae00127671e301c9f44254c31b8 +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-image-0.4.0-hb711507_2.conda#a0901183f08b6c7107aab109733a3c91 +https://conda.anaconda.org/conda-forge/linux-64/xkeyboard-config-2.44-hb9d3cd8_0.conda#7c91bfc90672888259675ad2ad28af9c +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.6-hb9d3cd8_0.conda#febbab7d15033c913d53c7a2c102309d +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxfixes-6.0.1-hb9d3cd8_0.conda#4bdb303603e9821baf5fe5fdff1dc8f8 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.12-hb9d3cd8_0.conda#96d57aba173e878a2089d5638016dc5e +https://conda.anaconda.org/conda-forge/linux-64/aws-c-auth-0.8.6-hd08a7f5_4.conda#f5a770ac1fd2cb34b21327fc513013a7 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-mqtt-0.12.2-h108da3e_2.conda#90e07c8bac8da6378ee1882ef0a9374a +https://conda.anaconda.org/conda-forge/linux-64/azure-core-cpp-1.14.0-h5cfcd09_0.conda#0a8838771cc2e985cd295e01ae83baf1 +https://conda.anaconda.org/conda-forge/linux-64/ccache-4.11.3-h80c52d3_0.conda#eb517c6a2b960c3ccb6f1db1005f063a +https://conda.anaconda.org/conda-forge/linux-64/coverage-7.8.2-py313h8060acc_0.conda#b278629953bd3424060870fca744de4a +https://conda.anaconda.org/conda-forge/linux-64/dbus-1.16.2-h3c4dab8_0.conda#679616eb5ad4e521c83da4650860aba7 +https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.0-pyhd8ed1ab_0.conda#72e42d28960d875c7654614f8b50939a +https://conda.anaconda.org/conda-forge/linux-64/fonttools-4.58.1-py313h8060acc_0.conda#f03a1dc39346922cb5cf2ee190ac9b95 +https://conda.anaconda.org/conda-forge/linux-64/freetype-2.13.3-ha770c72_1.conda#9ccd736d31e0c6e41f54e704e5312811 +https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.6-pyhd8ed1ab_0.conda#446bd6c8cb26050d528881df495ce646 +https://conda.anaconda.org/conda-forge/noarch/joblib-1.5.1-pyhd8ed1ab_0.conda#fb1c14694de51a476ce8636d92b6f42c +https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-31_he106b2a_openblas.conda#abb32c727da370c481a1c206f5159ce9 +https://conda.anaconda.org/conda-forge/linux-64/libcudnn-dev-9.10.1.4-h0fdc2d1_0.conda#a0c0b44d26a4710e6ea577fcddbe09d1 +https://conda.anaconda.org/conda-forge/linux-64/libgl-1.7.0-ha4b6fd6_2.conda#928b8be80851f5d8ffb016f9c81dae7a +https://conda.anaconda.org/conda-forge/linux-64/libgrpc-1.67.1-h25350d4_2.conda#bfcedaf5f9b003029cc6abe9431f66bf +https://conda.anaconda.org/conda-forge/linux-64/libhwloc-2.11.2-default_h0d58e46_1001.conda#804ca9e91bcaea0824a341d55b1684f2 +https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-31_h7ac8fdf_openblas.conda#452b98eafe050ecff932f0ec832dd03f +https://conda.anaconda.org/conda-forge/linux-64/libllvm20-20.1.6-he9d0ab4_0.conda#bf8ccdd2c1c1a54a3fa25bb61f26460e +https://conda.anaconda.org/conda-forge/linux-64/libxkbcommon-1.10.0-h65c71a3_0.conda#fedf6bfe5d21d21d2b1785ec00a8889a +https://conda.anaconda.org/conda-forge/linux-64/libxslt-1.1.39-h76b75d6_0.conda#e71f31f8cfb0a91439f2086fc8aa0461 +https://conda.anaconda.org/conda-forge/linux-64/mpc-1.3.1-h24ddda3_1.conda#aa14b9a5196a6d8dd364164b7ce56acf +https://conda.anaconda.org/conda-forge/linux-64/openldap-2.6.10-he970967_0.conda#2e5bf4f1da39c0b32778561c3c4e5878 +https://conda.anaconda.org/conda-forge/linux-64/pillow-11.2.1-py313h8db990d_0.conda#91b00afee98d72d29dc3d1c1ab0008d7 +https://conda.anaconda.org/conda-forge/linux-64/prometheus-cpp-1.3.0-ha5d0236_0.conda#a83f6a2fdc079e643237887a37460668 +https://conda.anaconda.org/conda-forge/noarch/pyproject-metadata-0.9.1-pyhd8ed1ab_0.conda#22ae7c6ea81e0c8661ef32168dda929b +https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhff2d567_1.conda#5ba79d7c71f03c678c8ead841f347d6e +https://conda.anaconda.org/conda-forge/noarch/python-gil-3.13.3-h4df99d1_101.conda#82c2641f2f0f513f7d2d1b847a2588e3 +https://conda.anaconda.org/conda-forge/linux-64/xcb-util-cursor-0.1.5-hb9d3cd8_0.conda#eb44b3b6deb1cab08d72cb61686fe64c +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxcomposite-0.4.6-hb9d3cd8_2.conda#d3c295b50f092ab525ffe3c2aa4b7413 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxcursor-1.2.3-hb9d3cd8_0.conda#2ccd714aa2242315acaf0a67faea780b +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdamage-1.1.6-hb9d3cd8_0.conda#b5fcc7172d22516e1f965490e65e33a4 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxi-1.8.2-hb9d3cd8_0.conda#17dcc85db3c7886650b8908b183d6876 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrandr-1.5.4-hb9d3cd8_0.conda#2de7f99d6581a4a7adbff607b5c278ca +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxxf86vm-1.1.6-hb9d3cd8_0.conda#5efa5fa6243a622445fdfd72aee15efa +https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda#aaa2a381ccc56eac91d63b6c1240312f +https://conda.anaconda.org/conda-forge/linux-64/aws-c-s3-0.7.13-h822ba82_2.conda#9cf2c3c13468f2209ee814be2c88655f +https://conda.anaconda.org/conda-forge/linux-64/azure-identity-cpp-1.10.0-h113e628_0.conda#73f73f60854f325a55f1d31459f2ab73 +https://conda.anaconda.org/conda-forge/linux-64/azure-storage-common-cpp-12.8.0-h736e048_1.conda#13de36be8de3ae3f05ba127631599213 +https://conda.anaconda.org/conda-forge/linux-64/cudnn-9.10.1.4-haad7af6_0.conda#8382d957333e0d3280dcbf5691516dc1 +https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.15.0-h7e30c49_1.conda#8f5b0b297b59e1ac160ad4beec99dbee +https://conda.anaconda.org/conda-forge/linux-64/gmpy2-2.2.1-py313h11186cd_0.conda#54d020e0eaacf1e99bfb2410b9aa2e5e +https://conda.anaconda.org/conda-forge/linux-64/libclang-cpp20.1-20.1.6-default_h1df26ce_0.conda#99ead3b974685e44df8b1e3953503cfc +https://conda.anaconda.org/conda-forge/linux-64/libclang13-20.1.6-default_he06ed0a_0.conda#cc6c469d9d7fc0ac106cef5f45d973a9 +https://conda.anaconda.org/conda-forge/linux-64/libgoogle-cloud-2.36.0-h2b5623c_0.conda#c96ca58ad3352a964bfcb85de6cd1496 +https://conda.anaconda.org/conda-forge/linux-64/liblapacke-3.9.0-31_he2f377e_openblas.conda#7e5fff7d0db69be3a266f7e79a3bb0e2 +https://conda.anaconda.org/conda-forge/linux-64/libmagma-2.9.0-h45b15fe_0.conda#703a1ab01e36111d8bb40bc7517e900b +https://conda.anaconda.org/conda-forge/linux-64/libopentelemetry-cpp-1.18.0-hfcad708_1.conda#1f5a5d66e77a39dc5bd639ec953705cf +https://conda.anaconda.org/conda-forge/linux-64/libpq-17.5-h27ae623_0.conda#6458be24f09e1b034902ab44fe9de908 +https://conda.anaconda.org/conda-forge/noarch/meson-python-0.18.0-pyh70fd9c4_0.conda#576c04b9d9f8e45285fb4d9452c26133 +https://conda.anaconda.org/conda-forge/linux-64/numpy-2.2.6-py313h17eae1a_0.conda#7a2d2f9adecd86ed5c29c2115354f615 +https://conda.anaconda.org/conda-forge/noarch/pytest-8.3.5-pyhd8ed1ab_0.conda#c3c9316209dec74a705a36797970c6be +https://conda.anaconda.org/conda-forge/linux-64/tbb-2021.13.0-hceb3a55_1.conda#ba7726b8df7b9d34ea80e82b097a4893 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxtst-1.2.5-hb9d3cd8_3.conda#7bbe9a0cc0df0ac5f5a8ad6d6a11af2f +https://conda.anaconda.org/conda-forge/noarch/array-api-strict-2.3.1-pyhd8ed1ab_0.conda#11107d0aeb8c590a34fee0894909816b +https://conda.anaconda.org/conda-forge/linux-64/aws-crt-cpp-0.31.0-h55f77e1_4.conda#0627af705ed70681f5bede31e72348e5 +https://conda.anaconda.org/conda-forge/linux-64/azure-storage-blobs-cpp-12.13.0-h3cf044e_1.conda#7eb66060455c7a47d9dcdbfa9f46579b +https://conda.anaconda.org/conda-forge/linux-64/blas-devel-3.9.0-31_h1ea3ea9_openblas.conda#ba652ee0576396d4765e567f043c57f9 +https://conda.anaconda.org/conda-forge/linux-64/cairo-1.18.4-h3394656_0.conda#09262e66b19567aff4f592fb53b28760 +https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.3.2-py313h33d0bda_0.conda#5dc81fffe102f63045225007a33d6199 +https://conda.anaconda.org/conda-forge/linux-64/cupy-core-13.4.1-py313hc2a895b_0.conda#46dd595e816b278b178e3bef8a6acf71 +https://conda.anaconda.org/conda-forge/linux-64/libgoogle-cloud-storage-2.36.0-h0121fbd_0.conda#fc5efe1833a4d709953964037985bb72 +https://conda.anaconda.org/conda-forge/linux-64/libmagma_sparse-2.9.0-h45b15fe_0.conda#beac0a5bbe0af75db6b16d3d8fd24f7e +https://conda.anaconda.org/conda-forge/linux-64/mkl-2024.2.2-ha957f24_16.conda#1459379c79dda834673426504d52b319 +https://conda.anaconda.org/conda-forge/linux-64/pandas-2.2.3-py313ha87cce1_3.conda#6248b529e537b1d4cb5ab3ef7f537795 +https://conda.anaconda.org/conda-forge/linux-64/polars-default-1.30.0-py39hfac2b71_0.conda#cd33cf1e631b4d766858c90e333b4832 +https://conda.anaconda.org/conda-forge/noarch/pytest-cov-6.1.1-pyhd8ed1ab_0.conda#1e35d8f975bc0e984a19819aa91c440a +https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.7.0-pyhd8ed1ab_0.conda#15353a2a0ea6dfefaa52fc5ab5b98f41 +https://conda.anaconda.org/conda-forge/linux-64/scipy-1.15.2-py313h86fcf2b_0.conda#ca68acd9febc86448eeed68d0c6c8643 +https://conda.anaconda.org/conda-forge/noarch/sympy-1.14.0-pyh2585a3b_105.conda#8c09fac3785696e1c477156192d64b91 +https://conda.anaconda.org/conda-forge/linux-64/aws-sdk-cpp-1.11.510-h37a5c72_3.conda#beb8577571033140c6897d257acc7724 +https://conda.anaconda.org/conda-forge/linux-64/azure-storage-files-datalake-cpp-12.12.0-ha633028_1.conda#7c1980f89dd41b097549782121a73490 +https://conda.anaconda.org/conda-forge/linux-64/blas-2.131-openblas.conda#38b2ec894c69bb4be0e66d2ef7fc60bf +https://conda.anaconda.org/conda-forge/linux-64/cupy-13.4.1-py313h66a2ee2_0.conda#784d6bd149ef2b5d9c733ea3dd4d15ad +https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-11.2.1-h3beb420_0.conda#0e6e192d4b3d95708ad192d957cf3163 +https://conda.anaconda.org/conda-forge/linux-64/libtorch-2.4.1-cuda118_mkl_hee7131c_306.conda#28b3b3da11973494ed0100aa50f47328 +https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.10.3-py313h129903b_0.conda#4f8816d006b1c155ec416bcf7ff6cee2 +https://conda.anaconda.org/conda-forge/linux-64/polars-1.30.0-default_h1443d73_0.conda#19698b29e8544d2dd615699826037039 +https://conda.anaconda.org/conda-forge/linux-64/pyamg-5.2.1-py313hf0ab243_1.conda#4c769bf3858f424cb2ecf952175ec600 +https://conda.anaconda.org/conda-forge/linux-64/libarrow-19.0.1-hc7b3859_3_cpu.conda#9ed3ded6da29dec8417f2e1db68798f2 +https://conda.anaconda.org/conda-forge/linux-64/pytorch-2.4.1-cuda118_mkl_py313_h909c4c2_306.conda#de6e45613bbdb51127e9ff483c31bf41 +https://conda.anaconda.org/conda-forge/linux-64/qt6-main-6.9.0-h0384650_3.conda#8aa69e15597a205fd6f81781fe62c232 +https://conda.anaconda.org/conda-forge/linux-64/libarrow-acero-19.0.1-hcb10f89_3_cpu.conda#8f8dc214d89e06933f1bc1dcd2310b9c +https://conda.anaconda.org/conda-forge/linux-64/libparquet-19.0.1-h081d1f1_3_cpu.conda#1d04307cdb1d8aeb5f55b047d5d403ea +https://conda.anaconda.org/conda-forge/linux-64/pyarrow-core-19.0.1-py313he5f92c8_0_cpu.conda#7d8649531c807b24295c8f9a0a396a78 +https://conda.anaconda.org/conda-forge/linux-64/pyside6-6.9.0-py313h5f61773_0.conda#f51f25ec8fcbf777f8b186bb5deeed40 +https://conda.anaconda.org/conda-forge/linux-64/pytorch-gpu-2.4.1-cuda118_mkl_hf8a3b2d_306.conda#b1802a39f1ca7ebed5f8c35755bffec1 +https://conda.anaconda.org/conda-forge/linux-64/libarrow-dataset-19.0.1-hcb10f89_3_cpu.conda#a28f04b6e68a1c76de76783108ad729d +https://conda.anaconda.org/conda-forge/linux-64/matplotlib-3.10.3-py313h78bf25f_0.conda#cc9324e614a297fdf23439d887d3513d +https://conda.anaconda.org/conda-forge/linux-64/libarrow-substrait-19.0.1-h08228c5_3_cpu.conda#a58e4763af8293deaac77b63bc7804d8 +https://conda.anaconda.org/conda-forge/linux-64/pyarrow-19.0.1-py313h78bf25f_0.conda#e8efe6998a383dd149787c83d3d6a92e diff --git a/build_tools/github/pylatest_conda_forge_cuda_array-api_linux-64_environment.yml b/build_tools/github/pylatest_conda_forge_cuda_array-api_linux-64_environment.yml new file mode 100644 index 0000000000000..bbfb91d24fd1a --- /dev/null +++ b/build_tools/github/pylatest_conda_forge_cuda_array-api_linux-64_environment.yml @@ -0,0 +1,32 @@ +# DO NOT EDIT: this file is generated from the specification found in the +# following script to centralize the configuration for CI builds: +# build_tools/update_environments_and_lock_files.py +channels: + - conda-forge + - pytorch + - nvidia +dependencies: + - python + - numpy + - blas + - scipy + - cython + - joblib + - threadpoolctl + - matplotlib + - pandas + - pyamg + - pytest + - pytest-xdist + - pillow + - pip + - ninja + - meson-python + - pytest-cov + - coverage + - ccache + - pytorch-gpu + - polars + - pyarrow + - cupy + - array-api-strict diff --git a/build_tools/github/pymin_conda_forge_arm_environment.yml b/build_tools/github/pymin_conda_forge_arm_environment.yml new file mode 100644 index 0000000000000..c65ab4aaecf14 --- /dev/null +++ b/build_tools/github/pymin_conda_forge_arm_environment.yml @@ -0,0 +1,22 @@ +# DO NOT EDIT: this file is generated from the specification found in the +# following script to centralize the configuration for CI builds: +# build_tools/update_environments_and_lock_files.py +channels: + - conda-forge +dependencies: + - python=3.10 + - numpy + - blas + - scipy + - cython + - joblib + - threadpoolctl + - matplotlib + - pytest + - pytest-xdist + - pillow + - pip + - ninja + - meson-python + - pip + - ccache diff --git a/build_tools/github/pymin_conda_forge_arm_linux-aarch64_conda.lock b/build_tools/github/pymin_conda_forge_arm_linux-aarch64_conda.lock new file mode 100644 index 0000000000000..fa19d32158855 --- /dev/null +++ b/build_tools/github/pymin_conda_forge_arm_linux-aarch64_conda.lock @@ -0,0 +1,160 @@ +# Generated by conda-lock. +# platform: linux-aarch64 +# input_hash: f12646c755adbf5f02f95c5d07e868bf1570777923e737bc27273eb1a5e40cd7 +@EXPLICIT +https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2#0c96522c6bdaed4b1566d11387caaf45 +https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2#34893075a5c9e55cdafac56607368fc6 +https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2#4d59c254e01d9cde7957100457e2d5fb +https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-h77eed37_3.conda#49023d73832ef61042f6a237cb2687e7 +https://conda.anaconda.org/conda-forge/linux-aarch64/ld_impl_linux-aarch64-2.43-h80caac9_4.conda#80c9ad5e05e91bb6c0967af3880c9742 +https://conda.anaconda.org/conda-forge/linux-aarch64/libglvnd-1.7.0-hd24410f_2.conda#9e115653741810778c9a915a2f8439e7 +https://conda.anaconda.org/conda-forge/linux-aarch64/libgomp-15.1.0-he277a41_2.conda#a28544b28961994eab37e1132a7dadcf +https://conda.anaconda.org/conda-forge/noarch/python_abi-3.10-7_cp310.conda#44e871cba2b162368476a84b8d040b6c +https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda#4222072737ccff51314b5ece9c7d6f5a +https://conda.anaconda.org/conda-forge/linux-aarch64/_openmp_mutex-4.5-2_gnu.tar.bz2#6168d71addc746e8f2b8d57dfd2edcea +https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.4.26-hbd8a1cb_0.conda#95db94f75ba080a22eb623590993167b +https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-0.tar.bz2#f766549260d6815b0c52253f1fb1bb29 +https://conda.anaconda.org/conda-forge/linux-aarch64/libegl-1.7.0-hd24410f_2.conda#cf105bce884e4ef8c8ccdca9fe6695e7 +https://conda.anaconda.org/conda-forge/linux-aarch64/libopengl-1.7.0-hd24410f_2.conda#cf9d12bfab305e48d095a4c79002c922 +https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2#fee5683a3f04bd15cbd8318b096a27ab +https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-15.1.0-he277a41_2.conda#224e999bbcad260d7bd4c0c27fdb99a4 +https://conda.anaconda.org/conda-forge/linux-aarch64/alsa-lib-1.2.14-h86ecc28_0.conda#a696b24c1b473ecc4774bcb5a6ac6337 +https://conda.anaconda.org/conda-forge/linux-aarch64/libbrotlicommon-1.1.0-h86ecc28_2.conda#3ee026955c688f551a9999840cff4c67 +https://conda.anaconda.org/conda-forge/linux-aarch64/libdeflate-1.24-he377734_0.conda#f0b3d6494663b3385bf87fc206d7451a +https://conda.anaconda.org/conda-forge/linux-aarch64/libexpat-2.7.0-h5ad3122_0.conda#d41a057e7968705dae8dcb7c8ba2c8dd +https://conda.anaconda.org/conda-forge/linux-aarch64/libffi-3.4.6-he21f813_1.conda#15a131f30cae36e9a655ca81fee9a285 +https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-ng-15.1.0-he9431aa_2.conda#d12a4b26073751bbc3db18de83ccba5f +https://conda.anaconda.org/conda-forge/linux-aarch64/libgfortran5-15.1.0-hbc25352_2.conda#4b5f4d119f9b28f254f82dbe56b2406f +https://conda.anaconda.org/conda-forge/linux-aarch64/libiconv-1.18-hc99b53d_1.conda#81541d85a45fbf4d0a29346176f1f21c +https://conda.anaconda.org/conda-forge/linux-aarch64/libjpeg-turbo-3.1.0-h86ecc28_0.conda#a689388210d502364b79e8b19e7fa2cb +https://conda.anaconda.org/conda-forge/linux-aarch64/liblzma-5.8.1-h86ecc28_1.conda#8ced9a547a29f7a71b7f15a4443ad1de +https://conda.anaconda.org/conda-forge/linux-aarch64/libstdcxx-15.1.0-h3f4de04_2.conda#6247ea6d1ecac20a9e98674342984726 +https://conda.anaconda.org/conda-forge/linux-aarch64/libwebp-base-1.5.0-h0886dbf_0.conda#95ef4a689b8cc1b7e18b53784d88f96b +https://conda.anaconda.org/conda-forge/linux-aarch64/libzlib-1.3.1-h86ecc28_2.conda#08aad7cbe9f5a6b460d0976076b6ae64 +https://conda.anaconda.org/conda-forge/linux-aarch64/ncurses-6.5-ha32ae93_3.conda#182afabe009dc78d8b73100255ee6868 +https://conda.anaconda.org/conda-forge/linux-aarch64/openssl-3.5.0-hd08dc88_1.conda#ee68fdc3a8723e9c58bdd2f10544658f +https://conda.anaconda.org/conda-forge/linux-aarch64/pthread-stubs-0.4-h86ecc28_1002.conda#bb5a90c93e3bac3d5690acf76b4a6386 +https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libice-1.1.2-h86ecc28_0.conda#c8d8ec3e00cd0fd8a231789b91a7c5b7 +https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxau-1.0.12-h86ecc28_0.conda#d5397424399a66d33c80b1f2345a36a6 +https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxdmcp-1.1.5-h57736b2_0.conda#25a5a7b797fe6e084e04ffe2db02fc62 +https://conda.anaconda.org/conda-forge/linux-aarch64/bzip2-1.0.8-h68df207_7.conda#56398c28220513b9ea13d7b450acfb20 +https://conda.anaconda.org/conda-forge/linux-aarch64/double-conversion-3.3.1-h5ad3122_0.conda#399959d889e1a73fc99f12ce480e77e1 +https://conda.anaconda.org/conda-forge/linux-aarch64/keyutils-1.6.1-h4e544f5_0.tar.bz2#1f24853e59c68892452ef94ddd8afd4b +https://conda.anaconda.org/conda-forge/linux-aarch64/lerc-4.0.0-hfdc4d58_1.conda#60dceb7e876f4d74a9cbd42bbbc6b9cf +https://conda.anaconda.org/conda-forge/linux-aarch64/libbrotlidec-1.1.0-h86ecc28_2.conda#e64d0f3b59c7c4047446b97a8624a72d +https://conda.anaconda.org/conda-forge/linux-aarch64/libbrotlienc-1.1.0-h86ecc28_2.conda#0e9bd365480c72b25c71a448257b537d +https://conda.anaconda.org/conda-forge/linux-aarch64/libedit-3.1.20250104-pl5321h976ea20_0.conda#fb640d776fc92b682a14e001980825b1 +https://conda.anaconda.org/conda-forge/linux-aarch64/libgfortran-15.1.0-he9431aa_2.conda#dc8675aa2658bb0d92cefbff83ce2db8 +https://conda.anaconda.org/conda-forge/linux-aarch64/libnsl-2.0.1-h31becfc_0.conda#c14f32510f694e3185704d89967ec422 +https://conda.anaconda.org/conda-forge/linux-aarch64/libntlm-1.4-hf897c2e_1002.tar.bz2#835c7c4137821de5c309f4266a51ba89 +https://conda.anaconda.org/conda-forge/linux-aarch64/libpciaccess-0.18-h31becfc_0.conda#6d48179630f00e8c9ad9e30879ce1e54 +https://conda.anaconda.org/conda-forge/linux-aarch64/libpng-1.6.47-hec79eb8_0.conda#c4b1ba0d7cef5002759d2f156722feee +https://conda.anaconda.org/conda-forge/linux-aarch64/libsqlite-3.50.0-h5eb1b54_0.conda#634a05a598cd4b3b852443f8e3b45003 +https://conda.anaconda.org/conda-forge/linux-aarch64/libstdcxx-ng-15.1.0-hf1166c9_2.conda#18e532d1a39ae9f78cc8988a034f1cae +https://conda.anaconda.org/conda-forge/linux-aarch64/libuuid-2.38.1-hb4cce97_0.conda#000e30b09db0b7c775b21695dff30969 +https://conda.anaconda.org/conda-forge/linux-aarch64/libxcb-1.17.0-h262b8f6_0.conda#cd14ee5cca2464a425b1dbfc24d90db2 +https://conda.anaconda.org/conda-forge/linux-aarch64/libxcrypt-4.4.36-h31becfc_1.conda#b4df5d7d4b63579d081fd3a4cf99740e +https://conda.anaconda.org/conda-forge/linux-aarch64/ninja-1.12.1-h17cf362_1.conda#885414635e2a65ed06f284f6d569cdff +https://conda.anaconda.org/conda-forge/linux-aarch64/pixman-0.46.0-h86a87f0_0.conda#1328d5bad76f7b31926ccd2a33e0d6ef +https://conda.anaconda.org/conda-forge/linux-aarch64/readline-8.2-h8382b9d_2.conda#c0f08fc2737967edde1a272d4bf41ed9 +https://conda.anaconda.org/conda-forge/linux-aarch64/tk-8.6.13-noxft_h5688188_102.conda#2562c9bfd1de3f9c590f0fe53858d85c +https://conda.anaconda.org/conda-forge/linux-aarch64/wayland-1.23.1-h698ed42_1.conda#229b00f81a229af79547a7e4776ccf6e +https://conda.anaconda.org/conda-forge/linux-aarch64/zstd-1.5.7-hbcf94c1_2.conda#5be90c5a3e4b43c53e38f50a85e11527 +https://conda.anaconda.org/conda-forge/linux-aarch64/brotli-bin-1.1.0-h86ecc28_2.conda#7d48b185fe1f722f8cda4539bb931f85 +https://conda.anaconda.org/conda-forge/linux-aarch64/graphite2-1.3.13-h2f0025b_1003.conda#f33009add6a08358bc12d114ceec1304 +https://conda.anaconda.org/conda-forge/linux-aarch64/icu-75.1-hf9b3779_0.conda#268203e8b983fddb6412b36f2024e75c +https://conda.anaconda.org/conda-forge/linux-aarch64/krb5-1.21.3-h50a48e9_0.conda#29c10432a2ca1472b53f299ffb2ffa37 +https://conda.anaconda.org/conda-forge/linux-aarch64/libdrm-2.4.124-h86ecc28_0.conda#a8058bcb6b4fa195aaa20452437c7727 +https://conda.anaconda.org/conda-forge/linux-aarch64/libfreetype6-2.13.3-he93130f_1.conda#51eae9012d75b8f7e4b0adfe61a83330 +https://conda.anaconda.org/conda-forge/linux-aarch64/libgfortran-ng-15.1.0-he9431aa_2.conda#55c5691e8b65612aaa0ef109cf645724 +https://conda.anaconda.org/conda-forge/linux-aarch64/libopenblas-0.3.29-pthreads_h9d3fd7e_0.conda#a99e2bfcb1ad6362544c71281eb617e9 +https://conda.anaconda.org/conda-forge/linux-aarch64/libtiff-4.7.0-h7c15681_5.conda#264a9aac20276b1784dac8c5f8d3704a +https://conda.anaconda.org/conda-forge/linux-aarch64/pcre2-10.45-hf4ec17f_0.conda#ad22a9a9497f7aedce73e0da53cd215f +https://conda.anaconda.org/conda-forge/linux-aarch64/python-3.10.17-h256493d_0_cpython.conda#c496213b6ede3c5a30ce1bf02bebf382 +https://conda.anaconda.org/conda-forge/linux-aarch64/qhull-2020.2-h70be974_5.conda#bb138086d938e2b64f5f364945793ebf +https://conda.anaconda.org/conda-forge/linux-aarch64/xcb-util-0.4.1-h5c728e9_2.conda#b4cf8ba6cff9cdf1249bcfe1314222b0 +https://conda.anaconda.org/conda-forge/linux-aarch64/xcb-util-keysyms-0.4.1-h5c728e9_0.conda#57ca8564599ddf8b633c4ea6afee6f3a +https://conda.anaconda.org/conda-forge/linux-aarch64/xcb-util-renderutil-0.3.10-h5c728e9_0.conda#7beeda4223c5484ef72d89fb66b7e8c1 +https://conda.anaconda.org/conda-forge/linux-aarch64/xcb-util-wm-0.4.2-h5c728e9_0.conda#f14dcda6894722e421da2b7dcffb0b78 +https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libsm-1.2.6-h0808dbd_0.conda#2d1409c50882819cb1af2de82e2b7208 +https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libx11-1.8.12-hca56bd8_0.conda#3df132f0048b9639bc091ef22937c111 +https://conda.anaconda.org/conda-forge/linux-aarch64/brotli-1.1.0-h86ecc28_2.conda#5094acc34eb173f74205c0b55f0dd4a4 +https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda#962b9857ee8e7018c22f2776ffa0b2d7 +https://conda.anaconda.org/conda-forge/noarch/cycler-0.12.1-pyhd8ed1ab_1.conda#44600c4667a319d67dbe0681fc0bc833 +https://conda.anaconda.org/conda-forge/linux-aarch64/cyrus-sasl-2.1.27-hf6b2984_7.conda#7a85d417c8acd7a5215c082c5b9219e5 +https://conda.anaconda.org/conda-forge/linux-aarch64/cython-3.1.1-py310hc86cfe9_1.conda#927cfc32db9db46c1804d977aaa11173 +https://conda.anaconda.org/conda-forge/noarch/execnet-2.1.1-pyhd8ed1ab_1.conda#a71efeae2c160f6789900ba2631a2c90 +https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_1.conda#6837f3eff7dcea42ecd714ce1ac2b108 +https://conda.anaconda.org/conda-forge/linux-aarch64/kiwisolver-1.4.7-py310h5d7f10c_0.conda#b86d594bf17c9ad7a291593368ae8ba7 +https://conda.anaconda.org/conda-forge/linux-aarch64/lcms2-2.17-hc88f144_0.conda#b87b1abd2542cf65a00ad2e2461a3083 +https://conda.anaconda.org/conda-forge/linux-aarch64/libblas-3.9.0-31_h1a9f1db_openblas.conda#48bd5bf15ccf3e409840be9caafc0ad5 +https://conda.anaconda.org/conda-forge/linux-aarch64/libcups-2.3.3-h405e4a8_4.conda#d42c670b0c96c1795fd859d5e0275a55 +https://conda.anaconda.org/conda-forge/linux-aarch64/libfreetype-2.13.3-h8af1aa0_1.conda#2d4a1c3dcabb80b4a56d5c34bdacea08 +https://conda.anaconda.org/conda-forge/linux-aarch64/libglib-2.84.2-hc022ef1_0.conda#51323eab8e9f049d001424828c4c25a4 +https://conda.anaconda.org/conda-forge/linux-aarch64/libglx-1.7.0-hd24410f_2.conda#1d4269e233636148696a67e2d30dad2a +https://conda.anaconda.org/conda-forge/linux-aarch64/libhiredis-1.0.2-h05efe27_0.tar.bz2#a87f068744fd20334cd41489eb163bee +https://conda.anaconda.org/conda-forge/linux-aarch64/libxml2-2.13.8-he060846_0.conda#c73dfe6886cc8d39a09c357a36f91fb2 +https://conda.anaconda.org/conda-forge/noarch/meson-1.8.1-pyhe01879c_0.conda#f3cccd9a6ce5331ae33f69ade5529162 +https://conda.anaconda.org/conda-forge/noarch/munkres-1.1.4-pyh9f0ad1d_0.tar.bz2#2ba8498c1018c1e9c61eb99b973dfe19 +https://conda.anaconda.org/conda-forge/linux-aarch64/openblas-0.3.29-pthreads_h3a8cbd8_0.conda#4ec5b6144709ced5e7933977675f61c6 +https://conda.anaconda.org/conda-forge/linux-aarch64/openjpeg-2.5.3-h3f56577_0.conda#04231368e4af50d11184b50e14250993 +https://conda.anaconda.org/conda-forge/noarch/packaging-25.0-pyh29332c3_1.conda#58335b26c38bf4a20f399384c33cbcf9 +https://conda.anaconda.org/conda-forge/noarch/pluggy-1.6.0-pyhd8ed1ab_0.conda#7da7ccd349dbf6487a7778579d2bb971 +https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.2.3-pyhd8ed1ab_1.conda#513d3c262ee49b54a8fec85c5bc99764 +https://conda.anaconda.org/conda-forge/noarch/setuptools-80.9.0-pyhff2d567_0.conda#4de79c071274a53dcaf2a8c749d1499e +https://conda.anaconda.org/conda-forge/noarch/six-1.17.0-pyhd8ed1ab_0.conda#a451d576819089b0d672f18768be0f65 +https://conda.anaconda.org/conda-forge/noarch/threadpoolctl-3.6.0-pyhecae5ae_0.conda#9d64911b31d57ca443e9f1e36b04385f +https://conda.anaconda.org/conda-forge/noarch/tomli-2.2.1-pyhd8ed1ab_1.conda#ac944244f1fed2eb49bae07193ae8215 +https://conda.anaconda.org/conda-forge/linux-aarch64/tornado-6.5.1-py310h78583b1_0.conda#e1e576b66cca7642b0a66310b675ea36 +https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.13.2-pyh29332c3_0.conda#83fc6ae00127671e301c9f44254c31b8 +https://conda.anaconda.org/conda-forge/linux-aarch64/unicodedata2-16.0.0-py310ha766c32_0.conda#2936ce19a675e162962f396c7b40b905 +https://conda.anaconda.org/conda-forge/noarch/wheel-0.45.1-pyhd8ed1ab_1.conda#75cb7132eb58d97896e173ef12ac9986 +https://conda.anaconda.org/conda-forge/linux-aarch64/xcb-util-image-0.4.0-h5c728e9_2.conda#b82e5c78dbbfa931980e8bfe83bce913 +https://conda.anaconda.org/conda-forge/linux-aarch64/xkeyboard-config-2.44-h86ecc28_0.conda#4d91bf5ccb5b31be8e070fda2ed13c50 +https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxext-1.3.6-h57736b2_0.conda#bd1e86dd8aa3afd78a4bfdb4ef918165 +https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxfixes-6.0.1-h57736b2_0.conda#78f8715c002cc66991d7c11e3cf66039 +https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxrender-0.9.12-h86ecc28_0.conda#ae2c2dd0e2d38d249887727db2af960e +https://conda.anaconda.org/conda-forge/linux-aarch64/ccache-4.11.3-h4889ad1_0.conda#e0b9e519da2bf0fb8c48381daf87a194 +https://conda.anaconda.org/conda-forge/linux-aarch64/dbus-1.16.2-heda779d_0.conda#9203b74bb1f3fa0d6f308094b3b44c1e +https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.0-pyhd8ed1ab_0.conda#72e42d28960d875c7654614f8b50939a +https://conda.anaconda.org/conda-forge/linux-aarch64/fonttools-4.58.1-py310heeae437_0.conda#837e7673572a3d0ecd6cf5a31dee2f35 +https://conda.anaconda.org/conda-forge/linux-aarch64/freetype-2.13.3-h8af1aa0_1.conda#71c4cbe1b384a8e7b56993394a435343 +https://conda.anaconda.org/conda-forge/noarch/joblib-1.5.1-pyhd8ed1ab_0.conda#fb1c14694de51a476ce8636d92b6f42c +https://conda.anaconda.org/conda-forge/linux-aarch64/libcblas-3.9.0-31_hab92f65_openblas.conda#6b81dbae56a519f1ec2f25e0ee2f4334 +https://conda.anaconda.org/conda-forge/linux-aarch64/libgl-1.7.0-hd24410f_2.conda#0d00176464ebb25af83d40736a2cd3bb +https://conda.anaconda.org/conda-forge/linux-aarch64/liblapack-3.9.0-31_h411afd4_openblas.conda#41dbff5eb805a75c120a7b7a1c744dc2 +https://conda.anaconda.org/conda-forge/linux-aarch64/libllvm20-20.1.6-h07bd352_0.conda#978603200db5e721247fdb529a6e7321 +https://conda.anaconda.org/conda-forge/linux-aarch64/libxkbcommon-1.10.0-hbab7b08_0.conda#36cd1db31e923c6068b7e0e6fce2cd7b +https://conda.anaconda.org/conda-forge/linux-aarch64/libxslt-1.1.39-h1cc9640_0.conda#13e1d3f9188e85c6d59a98651aced002 +https://conda.anaconda.org/conda-forge/linux-aarch64/openldap-2.6.10-h30c48ee_0.conda#48f31a61be512ec1929f4b4a9cedf4bd +https://conda.anaconda.org/conda-forge/linux-aarch64/pillow-11.2.1-py310h34c99de_0.conda#116816e9f034fcaeafcd878ef8b1e323 +https://conda.anaconda.org/conda-forge/noarch/pip-25.1.1-pyh8b19718_0.conda#32d0781ace05105cc99af55d36cbec7c +https://conda.anaconda.org/conda-forge/noarch/pyproject-metadata-0.9.1-pyhd8ed1ab_0.conda#22ae7c6ea81e0c8661ef32168dda929b +https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhff2d567_1.conda#5ba79d7c71f03c678c8ead841f347d6e +https://conda.anaconda.org/conda-forge/linux-aarch64/xcb-util-cursor-0.1.5-h86ecc28_0.conda#d6bb2038d26fa118d5cbc2761116f3e5 +https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxcomposite-0.4.6-h86ecc28_2.conda#86051eee0766c3542be24844a9c3cf36 +https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxcursor-1.2.3-h86ecc28_0.conda#f2054759c2203d12d0007005e1f1296d +https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxdamage-1.1.6-h86ecc28_0.conda#d5773c4e4d64428d7ddaa01f6f845dc7 +https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxi-1.8.2-h57736b2_0.conda#eeee3bdb31c6acde2b81ad1b8c287087 +https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxrandr-1.5.4-h86ecc28_0.conda#dd3e74283a082381aa3860312e3c721e +https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxxf86vm-1.1.6-h86ecc28_0.conda#d745faa2d7c15092652e40a22bb261ed +https://conda.anaconda.org/conda-forge/linux-aarch64/fontconfig-2.15.0-h8dda3cd_1.conda#112b71b6af28b47c624bcbeefeea685b +https://conda.anaconda.org/conda-forge/linux-aarch64/libclang-cpp20.1-20.1.6-default_h7d4303a_0.conda#688d99949628971e08e6e44ee8b68a28 +https://conda.anaconda.org/conda-forge/linux-aarch64/libclang13-20.1.6-default_h9e36cb9_0.conda#ad384e458f9b9c2d5b22a399786b226a +https://conda.anaconda.org/conda-forge/linux-aarch64/liblapacke-3.9.0-31_hc659ca5_openblas.conda#256bb281d78e5b8927ff13a1cde9f6f5 +https://conda.anaconda.org/conda-forge/linux-aarch64/libpq-17.5-hf590da8_0.conda#b5a01e5aa04651ccf5865c2d029affa3 +https://conda.anaconda.org/conda-forge/noarch/meson-python-0.18.0-pyh70fd9c4_0.conda#576c04b9d9f8e45285fb4d9452c26133 +https://conda.anaconda.org/conda-forge/linux-aarch64/numpy-2.2.6-py310h6e5608f_0.conda#9e9f1f279eb02c41bda162a42861adc0 +https://conda.anaconda.org/conda-forge/noarch/pytest-8.3.5-pyhd8ed1ab_0.conda#c3c9316209dec74a705a36797970c6be +https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxtst-1.2.5-h57736b2_3.conda#c05698071b5c8e0da82a282085845860 +https://conda.anaconda.org/conda-forge/linux-aarch64/blas-devel-3.9.0-31_h9678261_openblas.conda#a2cc143d7e25e52a915cb320e5b0d592 +https://conda.anaconda.org/conda-forge/linux-aarch64/cairo-1.18.4-h83712da_0.conda#cd55953a67ec727db5dc32b167201aa6 +https://conda.anaconda.org/conda-forge/linux-aarch64/contourpy-1.3.2-py310hf54e67a_0.conda#779694434d1f0a67c5260db76b7b7907 +https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.7.0-pyhd8ed1ab_0.conda#15353a2a0ea6dfefaa52fc5ab5b98f41 +https://conda.anaconda.org/conda-forge/linux-aarch64/scipy-1.15.2-py310hf37559f_0.conda#5c9b72f10d2118d943a5eaaf2f396891 +https://conda.anaconda.org/conda-forge/linux-aarch64/blas-2.131-openblas.conda#51c5f346e1ebee750f76066490059df9 +https://conda.anaconda.org/conda-forge/linux-aarch64/harfbuzz-11.2.1-h405b6a2_0.conda#b55680fc90e9747dc858e7ceb0abc2b2 +https://conda.anaconda.org/conda-forge/linux-aarch64/matplotlib-base-3.10.3-py310h2cc5e2d_0.conda#e29f4329f4f76cf14f74ed86dcc59bac +https://conda.anaconda.org/conda-forge/linux-aarch64/qt6-main-6.9.0-h13135bf_3.conda#f3d24ce6f388642e76f4917b5069c2e9 +https://conda.anaconda.org/conda-forge/linux-aarch64/pyside6-6.9.0-py310hee8ad4f_0.conda#68f556281ac23f1780381f00de99d66d +https://conda.anaconda.org/conda-forge/linux-aarch64/matplotlib-3.10.3-py310hbbe02a8_0.conda#08982f6ac753e962d59160b08839221b diff --git a/build_tools/github/repair_windows_wheels.sh b/build_tools/github/repair_windows_wheels.sh old mode 100644 new mode 100755 index de564fc177c89..8f51a34d4039b --- a/build_tools/github/repair_windows_wheels.sh +++ b/build_tools/github/repair_windows_wheels.sh @@ -5,12 +5,12 @@ set -x WHEEL=$1 DEST_DIR=$2 -BITNESS=$3 # By default, the Windows wheels are not repaired. # In this case, we need to vendor VCRUNTIME140.dll +pip install wheel wheel unpack "$WHEEL" WHEEL_DIRNAME=$(ls -d scikit_learn-*) -python build_tools/github/vendor.py "$WHEEL_DIRNAME" "$BITNESS" +python build_tools/github/vendor.py "$WHEEL_DIRNAME" wheel pack "$WHEEL_DIRNAME" -d "$DEST_DIR" rm -rf "$WHEEL_DIRNAME" diff --git a/build_tools/github/test_source.sh b/build_tools/github/test_source.sh old mode 100644 new mode 100755 index 3a65a657addec..c93d22a08e791 --- a/build_tools/github/test_source.sh +++ b/build_tools/github/test_source.sh @@ -13,7 +13,6 @@ python -m pip install pytest pandas # Run the tests on the installed source distribution mkdir tmp_for_test -cp scikit-learn/scikit-learn/conftest.py tmp_for_test cd tmp_for_test pytest --pyargs sklearn diff --git a/build_tools/github/test_wheels.sh b/build_tools/github/test_wheels.sh deleted file mode 100644 index 58a05b6182006..0000000000000 --- a/build_tools/github/test_wheels.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -e -set -x - -if [[ "$OSTYPE" != "linux-gnu" ]]; then - # The Linux test environment is run in a Docker container and - # it is not possible to copy the test configuration file (yet) - cp $CONFTEST_PATH $CONFTEST_NAME -fi - -pytest --pyargs sklearn - -# Test that there are no links to system libraries -python -m threadpoolctl -i sklearn diff --git a/build_tools/github/test_windows_wheels.sh b/build_tools/github/test_windows_wheels.sh old mode 100644 new mode 100755 index ff3f823be754b..c96ec4ad89d3e --- a/build_tools/github/test_windows_wheels.sh +++ b/build_tools/github/test_windows_wheels.sh @@ -4,21 +4,27 @@ set -e set -x PYTHON_VERSION=$1 -BITNESS=$2 +PROJECT_DIR=$2 -if [[ "$PYTHON_VERSION" == "36" || "$BITNESS" == "32" ]]; then - # For Python 3.6 and 32-bit architecture use the regular - # test command (outside of the minimal Docker container) - cp $CONFTEST_PATH $CONFTEST_NAME - pytest --pyargs sklearn - python -m threadpoolctl -i sklearn -else - docker container run -e SKLEARN_SKIP_NETWORK_TESTS=1 \ - -e OMP_NUM_THREADS=2 \ - -e OPENBLAS_NUM_THREADS=2 \ - --rm scikit-learn/minimal-windows \ - powershell -Command "pytest --pyargs sklearn" +python $PROJECT_DIR/build_tools/wheels/check_license.py + +FREE_THREADED_BUILD="$(python -c"import sysconfig; print(bool(sysconfig.get_config_var('Py_GIL_DISABLED')))")" - docker container run --rm scikit-learn/minimal-windows \ - powershell -Command "python -m threadpoolctl -i sklearn" +if [[ $FREE_THREADED_BUILD == "False" ]]; then + # Run the tests for the scikit-learn wheel in a minimal Windows environment + # without any developer runtime libraries installed to ensure that it does not + # implicitly rely on the presence of the DLLs of such runtime libraries. + docker container run \ + --rm scikit-learn/minimal-windows \ + powershell -Command "python -c 'import sklearn; sklearn.show_versions()'" + + docker container run \ + -e SKLEARN_SKIP_NETWORK_TESTS=1 \ + --rm scikit-learn/minimal-windows \ + powershell -Command "pytest --pyargs sklearn" +else + # This is too cumbersome to use a Docker image in the free-threaded case + export PYTHON_GIL=0 + python -c "import sklearn; sklearn.show_versions()" + pytest --pyargs sklearn fi diff --git a/build_tools/github/upload_anaconda.sh b/build_tools/github/upload_anaconda.sh old mode 100644 new mode 100755 index 7651576cf558e..b53f27b75e72b --- a/build_tools/github/upload_anaconda.sh +++ b/build_tools/github/upload_anaconda.sh @@ -3,16 +3,19 @@ set -e set -x -if [ "$GITHUB_EVENT_NAME" == "schedule" ]; then - ANACONDA_ORG="scipy-wheels-nightly" +if [[ "$GITHUB_EVENT_NAME" == "schedule" \ + || "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]]; then + ANACONDA_ORG="scientific-python-nightly-wheels" ANACONDA_TOKEN="$SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN" else ANACONDA_ORG="scikit-learn-wheels-staging" ANACONDA_TOKEN="$SCIKIT_LEARN_STAGING_UPLOAD_TOKEN" fi -pip install git+https://github.com/Anaconda-Server/anaconda-client +export PATH=$CONDA/bin:$PATH +conda create -n upload -y anaconda-client +source activate upload # Force a replacement if the remote file already exists -anaconda -t $ANACONDA_TOKEN upload --force -u $ANACONDA_ORG dist/artifact/* +anaconda -t $ANACONDA_TOKEN upload --force -u $ANACONDA_ORG $ARTIFACTS_PATH/* echo "Index: https://pypi.anaconda.org/$ANACONDA_ORG/simple" diff --git a/build_tools/github/vendor.py b/build_tools/github/vendor.py index 5b367f3fb4ecc..28b44be3c9aa9 100644 --- a/build_tools/github/vendor.py +++ b/build_tools/github/vendor.py @@ -1,9 +1,4 @@ -"""Embed vcomp140.dll, vcruntime140.dll and vcruntime140_1.dll. - -Note that vcruntime140_1.dll is only required (and available) -for 64-bit architectures. -""" - +"""Embed vcomp140.dll and msvcp140.dll.""" import os import os.path as op @@ -11,67 +6,32 @@ import sys import textwrap - TARGET_FOLDER = op.join("sklearn", ".libs") DISTRIBUTOR_INIT = op.join("sklearn", "_distributor_init.py") VCOMP140_SRC_PATH = "C:\\Windows\\System32\\vcomp140.dll" -VCRUNTIME140_SRC_PATH = "C:\\Windows\\System32\\vcruntime140.dll" -VCRUNTIME140_1_SRC_PATH = "C:\\Windows\\System32\\vcruntime140_1.dll" - - -def make_distributor_init_32_bits(distributor_init, - vcomp140_dll_filename, - vcruntime140_dll_filename): - """Create a _distributor_init.py file for 32-bit architectures. - - This file is imported first when importing the sklearn package - so as to pre-load the vendored vcomp140.dll and vcruntime140.dll. - """ - with open(distributor_init, "wt") as f: - f.write(textwrap.dedent(""" - '''Helper to preload vcomp140.dll and vcruntime140.dll to - prevent "not found" errors. - - Once vcomp140.dll and vcruntime140.dll are preloaded, the - namespace is made available to any subsequent vcomp140.dll - and vcruntime140.dll. This is created as part of the scripts - that build the wheel. - ''' +MSVCP140_SRC_PATH = "C:\\Windows\\System32\\msvcp140.dll" - import os - import os.path as op - from ctypes import WinDLL - - - if os.name == "nt": - # Load vcomp140.dll and vcruntime140.dll - libs_path = op.join(op.dirname(__file__), ".libs") - vcomp140_dll_filename = op.join(libs_path, "{0}") - vcruntime140_dll_filename = op.join(libs_path, "{1}") - WinDLL(op.abspath(vcomp140_dll_filename)) - WinDLL(op.abspath(vcruntime140_dll_filename)) - """.format(vcomp140_dll_filename, vcruntime140_dll_filename))) - - -def make_distributor_init_64_bits(distributor_init, - vcomp140_dll_filename, - vcruntime140_dll_filename, - vcruntime140_1_dll_filename): +def make_distributor_init_64_bits( + distributor_init, + vcomp140_dll_filename, + msvcp140_dll_filename, +): """Create a _distributor_init.py file for 64-bit architectures. This file is imported first when importing the sklearn package - so as to pre-load the vendored vcomp140.dll, vcruntime140.dll - and vcruntime140_1.dll. + so as to pre-load the vendored vcomp140.dll and msvcp140.dll. """ with open(distributor_init, "wt") as f: - f.write(textwrap.dedent(""" - '''Helper to preload vcomp140.dll, vcruntime140.dll and - vcruntime140_1.dll to prevent "not found" errors. + f.write( + textwrap.dedent( + """ + '''Helper to preload vcomp140.dll and msvcp140.dll to prevent + "not found" errors. - Once vcomp140.dll, vcruntime140.dll and vcruntime140_1.dll are + Once vcomp140.dll and msvcp140.dll are preloaded, the namespace is made available to any subsequent - vcomp140.dll, vcruntime140.dll and vcruntime140_1.dll. This is + vcomp140.dll and msvcp140.dll. This is created as part of the scripts that build the wheel. ''' @@ -82,36 +42,32 @@ def make_distributor_init_64_bits(distributor_init, if os.name == "nt": - # Load vcomp140.dll, vcruntime140.dll and vcruntime140_1.dll libs_path = op.join(op.dirname(__file__), ".libs") vcomp140_dll_filename = op.join(libs_path, "{0}") - vcruntime140_dll_filename = op.join(libs_path, "{1}") - vcruntime140_1_dll_filename = op.join(libs_path, "{2}") + msvcp140_dll_filename = op.join(libs_path, "{1}") WinDLL(op.abspath(vcomp140_dll_filename)) - WinDLL(op.abspath(vcruntime140_dll_filename)) - WinDLL(op.abspath(vcruntime140_1_dll_filename)) - """.format(vcomp140_dll_filename, - vcruntime140_dll_filename, - vcruntime140_1_dll_filename))) + WinDLL(op.abspath(msvcp140_dll_filename)) + """.format( + vcomp140_dll_filename, + msvcp140_dll_filename, + ) + ) + ) -def main(wheel_dirname, bitness): - """Embed vcomp140.dll, vcruntime140.dll and vcruntime140_1.dll.""" +def main(wheel_dirname): + """Embed vcomp140.dll and msvcp140.dll.""" if not op.exists(VCOMP140_SRC_PATH): raise ValueError(f"Could not find {VCOMP140_SRC_PATH}.") - if not op.exists(VCRUNTIME140_SRC_PATH): - raise ValueError(f"Could not find {VCRUNTIME140_SRC_PATH}.") - - if not op.exists(VCRUNTIME140_1_SRC_PATH) and bitness == "64": - raise ValueError(f"Could not find {VCRUNTIME140_1_SRC_PATH}.") + if not op.exists(MSVCP140_SRC_PATH): + raise ValueError(f"Could not find {MSVCP140_SRC_PATH}.") if not op.isdir(wheel_dirname): raise RuntimeError(f"Could not find {wheel_dirname} file.") vcomp140_dll_filename = op.basename(VCOMP140_SRC_PATH) - vcruntime140_dll_filename = op.basename(VCRUNTIME140_SRC_PATH) - vcruntime140_1_dll_filename = op.basename(VCRUNTIME140_1_SRC_PATH) + msvcp140_dll_filename = op.basename(MSVCP140_SRC_PATH) target_folder = op.join(wheel_dirname, TARGET_FOLDER) distributor_init = op.join(wheel_dirname, DISTRIBUTOR_INIT) @@ -123,26 +79,18 @@ def main(wheel_dirname, bitness): print(f"Copying {VCOMP140_SRC_PATH} to {target_folder}.") shutil.copy2(VCOMP140_SRC_PATH, target_folder) - print(f"Copying {VCRUNTIME140_SRC_PATH} to {target_folder}.") - shutil.copy2(VCRUNTIME140_SRC_PATH, target_folder) - - if bitness == "64": - print(f"Copying {VCRUNTIME140_1_SRC_PATH} to {target_folder}.") - shutil.copy2(VCRUNTIME140_1_SRC_PATH, target_folder) + print(f"Copying {MSVCP140_SRC_PATH} to {target_folder}.") + shutil.copy2(MSVCP140_SRC_PATH, target_folder) # Generate the _distributor_init file in the source tree print("Generating the '_distributor_init.py' file.") - if bitness == "32": - make_distributor_init_32_bits(distributor_init, - vcomp140_dll_filename, - vcruntime140_dll_filename) - else: - make_distributor_init_64_bits(distributor_init, - vcomp140_dll_filename, - vcruntime140_dll_filename, - vcruntime140_1_dll_filename) + make_distributor_init_64_bits( + distributor_init, + vcomp140_dll_filename, + msvcp140_dll_filename, + ) if __name__ == "__main__": - _, wheel_file, bitness = sys.argv - main(wheel_file, bitness) + _, wheel_file = sys.argv + main(wheel_file) diff --git a/build_tools/linting.sh b/build_tools/linting.sh new file mode 100755 index 0000000000000..34b37530e10ff --- /dev/null +++ b/build_tools/linting.sh @@ -0,0 +1,123 @@ +#!/bin/bash + +# Note that any change in this file, adding or removing steps or changing the +# printed messages, should be also reflected in the `get_comment.py` file. + +# This script shouldn't exit if a command / pipeline fails +set +e +# pipefail is necessary to propagate exit codes +set -o pipefail + +global_status=0 + +echo -e "### Running the ruff linter ###\n" +ruff check --output-format=full +status=$? +if [[ $status -eq 0 ]] +then + echo -e "No problem detected by the ruff linter\n" +else + echo -e "Problems detected by ruff check, please fix them\n" + global_status=1 +fi + +echo -e "### Running the ruff formatter ###\n" +ruff format --diff +status=$? +if [[ $status -eq 0 ]] +then + echo -e "No problem detected by the ruff formatter\n" +else + echo -e "Problems detected by ruff format, please run ruff format and commit the result\n" + global_status=1 +fi + +echo -e "### Running mypy ###\n" +mypy sklearn/ +status=$? +if [[ $status -eq 0 ]] +then + echo -e "No problem detected by mypy\n" +else + echo -e "Problems detected by mypy, please fix them\n" + global_status=1 +fi + +echo -e "### Running cython-lint ###\n" +cython-lint sklearn/ +status=$? +if [[ $status -eq 0 ]] +then + echo -e "No problem detected by cython-lint\n" +else + echo -e "Problems detected by cython-lint, please fix them\n" + global_status=1 +fi + +# For docstrings and warnings of deprecated attributes to be rendered +# properly, the `deprecated` decorator must come before the `property` decorator +# (else they are treated as functions) + +echo -e "### Checking for bad deprecation order ###\n" +bad_deprecation_property_order=`git grep -A 10 "@property" -- "*.py" | awk '/@property/,/def /' | grep -B1 "@deprecated"` + +if [ ! -z "$bad_deprecation_property_order" ] +then + echo "deprecated decorator should come before property decorator" + echo "found the following occurrences:" + echo $bad_deprecation_property_order + echo -e "\nProblems detected by deprecation order check\n" + global_status=1 +else + echo -e "No problems detected related to deprecation order\n" +fi + +# Check for default doctest directives ELLIPSIS and NORMALIZE_WHITESPACE + +echo -e "### Checking for default doctest directives ###\n" +doctest_directive="$(git grep -nw -E "# doctest\: \+(ELLIPSIS|NORMALIZE_WHITESPACE)")" + +if [ ! -z "$doctest_directive" ] +then + echo "ELLIPSIS and NORMALIZE_WHITESPACE doctest directives are enabled by default, but were found in:" + echo "$doctest_directive" + echo -e "\nProblems detected by doctest directive check\n" + global_status=1 +else + echo -e "No problems detected related to doctest directives\n" +fi + +# Check for joblib.delayed and joblib.Parallel imports +echo -e "### Checking for joblib imports ###\n" +joblib_status=0 +joblib_delayed_import="$(git grep -l -A 10 -E "joblib import.+delayed" -- "*.py" ":!sklearn/utils/parallel.py")" +if [ ! -z "$joblib_delayed_import" ]; then + echo "Use from sklearn.utils.parallel import delayed instead of joblib delayed. The following files contains imports to joblib.delayed:" + echo "$joblib_delayed_import" + joblib_status=1 +fi +joblib_Parallel_import="$(git grep -l -A 10 -E "joblib import.+Parallel" -- "*.py" ":!sklearn/utils/parallel.py")" +if [ ! -z "$joblib_Parallel_import" ]; then + echo "Use from sklearn.utils.parallel import Parallel instead of joblib Parallel. The following files contains imports to joblib.Parallel:" + echo "$joblib_Parallel_import" + joblib_status=1 +fi + +if [[ $joblib_status -eq 0 ]] +then + echo -e "No problems detected related to joblib imports\n" +else + echo -e "\nProblems detected by joblib import check\n" + global_status=1 +fi + +echo -e "### Linting completed ###\n" + +if [[ $global_status -eq 1 ]] +then + echo -e "Linting failed\n" + exit 1 +else + echo -e "Linting passed\n" + exit 0 +fi diff --git a/build_tools/shared.sh b/build_tools/shared.sh index 4aa260675b9be..3c6f238385506 100644 --- a/build_tools/shared.sh +++ b/build_tools/shared.sh @@ -5,7 +5,7 @@ get_dep() { # do not install with none echo elif [[ "${version%%[^0-9.]*}" ]]; then - # version number is explicity passed + # version number is explicitly passed echo "$package==$version" elif [[ "$version" == "latest" ]]; then # use latest @@ -14,3 +14,38 @@ get_dep() { echo "$package==$(python sklearn/_min_dependencies.py $package)" fi } + +show_installed_libraries(){ + # use conda list when inside a conda environment. conda list shows more + # info than pip list, e.g. whether OpenBLAS or MKL is installed as well as + # the version of OpenBLAS or MKL + if [[ -n "$CONDA_PREFIX" ]]; then + conda list + else + python -m pip list + fi +} + +activate_environment() { + if [[ "$DISTRIB" =~ ^conda.* ]]; then + source activate $VIRTUALENV + elif [[ "$DISTRIB" == "ubuntu" || "$DISTRIB" == "debian-32" ]]; then + source $VIRTUALENV/bin/activate + fi +} + +create_conda_environment_from_lock_file() { + ENV_NAME=$1 + LOCK_FILE=$2 + # Because we are using lock-files with the "explicit" format, conda can + # install them directly, provided the lock-file does not contain pip solved + # packages. For more details, see + # https://conda.github.io/conda-lock/output/#explicit-lockfile + lock_file_has_pip_packages=$(grep -q files.pythonhosted.org $LOCK_FILE && echo "true" || echo "false") + if [[ "$lock_file_has_pip_packages" == "false" ]]; then + conda create --name $ENV_NAME --file $LOCK_FILE + else + python -m pip install "$(get_dep conda-lock min)" + conda-lock install --name $ENV_NAME $LOCK_FILE + fi +} diff --git a/build_tools/travis/after_success.sh b/build_tools/travis/after_success.sh deleted file mode 100755 index 2123f7efafc22..0000000000000 --- a/build_tools/travis/after_success.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -# This script is meant to be called by the "after_success" step -# defined in ".travis.yml". In particular, we upload the wheels -# of the ARM64 architecture for the continuous deployment jobs. - -set -e - -# The wheels cannot be uploaded on PRs -if [[ $BUILD_WHEEL == true && $TRAVIS_EVENT_TYPE != pull_request ]]; then - # Nightly upload token and staging upload token are set in - # Travis settings (originally generated at Anaconda cloud) - if [[ $TRAVIS_EVENT_TYPE == cron ]]; then - ANACONDA_ORG="scipy-wheels-nightly" - ANACONDA_TOKEN="$SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN" - else - ANACONDA_ORG="scikit-learn-wheels-staging" - ANACONDA_TOKEN="$SCIKIT_LEARN_STAGING_UPLOAD_TOKEN" - fi - - pip install git+https://github.com/Anaconda-Server/anaconda-client - - # Force a replacement if the remote file already exists - anaconda -t $ANACONDA_TOKEN upload --force -u $ANACONDA_ORG wheelhouse/*.whl - echo "Index: https://pypi.anaconda.org/$ANACONDA_ORG/simple" -fi diff --git a/build_tools/travis/install.sh b/build_tools/travis/install.sh deleted file mode 100644 index 1e8e2963711ef..0000000000000 --- a/build_tools/travis/install.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -# This script is meant to be called by the "install" step -# defined in the ".travis.yml" file. In particular, it is -# important that we call to the right installation script. - -set -e - -if [[ $BUILD_WHEEL == true ]]; then - source build_tools/travis/install_wheels.sh -else - source build_tools/travis/install_main.sh -fi diff --git a/build_tools/travis/install_main.sh b/build_tools/travis/install_main.sh deleted file mode 100755 index c0795139859bb..0000000000000 --- a/build_tools/travis/install_main.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -# Travis clone "scikit-learn/scikit-learn" repository into -# a local repository. We use a cached directory with three -# scikit-learn repositories (one for each matrix entry for -# non continuous deployment jobs) from which we pull local -# Travis repository. This allows us to keep build artifact -# for GCC + Cython, and gain time. - -set -e - -echo "CPU Arch: $TRAVIS_CPU_ARCH." - -# Import "get_dep" -source build_tools/shared.sh - -echo "List files from cached directories." -echo "pip:" -ls $HOME/.cache/pip - -export CC=/usr/lib/ccache/gcc -export CXX=/usr/lib/ccache/g++ - -# Useful for debugging how ccache is used -# export CCACHE_LOGFILE=/tmp/ccache.log - -# 60MB are (more or less) used by .ccache, when -# compiling from scratch at the time of writing -ccache --max-size 100M --show-stats - -# Deactivate the default virtual environment -# to setup a conda-based environment instead -deactivate - -MINICONDA_URL="https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-aarch64.sh" - -# Install Miniconda -wget $MINICONDA_URL -O miniconda.sh -MINICONDA_PATH=$HOME/miniconda -chmod +x miniconda.sh && ./miniconda.sh -b -p $MINICONDA_PATH -export PATH=$MINICONDA_PATH/bin:$PATH -conda update --yes conda - -# Create environment and install dependencies -conda create -n testenv --yes python=3.7 - -source activate testenv -conda install -y scipy numpy pandas cython -pip install joblib threadpoolctl - -pip install $(get_dep pytest $PYTEST_VERSION) pytest-xdist - -# Build scikit-learn in this script to collapse the -# verbose build output in the Travis output when it -# succeeds -python --version -python -c "import numpy; print(f'numpy {numpy.__version__}')" -python -c "import scipy; print(f'scipy {scipy.__version__}')" - -pip install -e . -python setup.py develop - -ccache --show-stats - -# Useful for debugging how ccache is used -# cat $CCACHE_LOGFILE diff --git a/build_tools/travis/install_wheels.sh b/build_tools/travis/install_wheels.sh deleted file mode 100644 index 4bb52f51f27f7..0000000000000 --- a/build_tools/travis/install_wheels.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -e - -python -m pip install cibuildwheel -python -m cibuildwheel --output-dir wheelhouse diff --git a/build_tools/travis/script.sh b/build_tools/travis/script.sh deleted file mode 100644 index 2b7aecb295d82..0000000000000 --- a/build_tools/travis/script.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -# This script is meant to be called by the "script" step defined -# in the ".travis.yml" file. While this step is forbidden by the -# continuous deployment jobs, we have to execute the scripts for -# testing the continuous integration jobs. - -set -e - -if [[ $BUILD_WHEEL != true ]]; then - # This trick will make Travis terminate the continuation of the pipeline - bash build_tools/travis/test_script.sh || travis_terminate 1 - bash build_tools/travis/test_docs.sh || travis_terminate 1 -fi diff --git a/build_tools/travis/test_docs.sh b/build_tools/travis/test_docs.sh deleted file mode 100755 index 4907dee1c9789..0000000000000 --- a/build_tools/travis/test_docs.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -e - -if [[ $TRAVIS_CPU_ARCH != arm64 ]]; then - # Faster run of the documentation tests - PYTEST="pytest -n $CPU_COUNT" make test-doc -fi diff --git a/build_tools/travis/test_script.sh b/build_tools/travis/test_script.sh deleted file mode 100755 index cb5a3dbfeed33..0000000000000 --- a/build_tools/travis/test_script.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -set -e - -python --version -python -c "import numpy; print(f'numpy {numpy.__version__}')" -python -c "import scipy; print(f'scipy {scipy.__version__}')" -python -c "\ -try: - import pandas - print(f'pandas {pandas.__version__}') -except ImportError: - pass -" -python -c "import joblib; print(f'{joblib.cpu_count()} CPUs')" -python -c "import platform; print(f'{platform.machine()}')" - -TEST_CMD="pytest --showlocals --durations=20 --pyargs" - -# Run the tests on the installed version -mkdir -p $TEST_DIR - -# Copy "setup.cfg" for the test settings -cp setup.cfg $TEST_DIR -cd $TEST_DIR - -if [[ $TRAVIS_CPU_ARCH == arm64 ]]; then - # Faster run of the source code tests - TEST_CMD="$TEST_CMD -n $CPU_COUNT" - - # Remove the option to test the docstring - sed -i -e 's/--doctest-modules//g' setup.cfg -fi - -if [[ -n $CHECK_WARNINGS ]]; then - TEST_CMD="$TEST_CMD -Werror::DeprecationWarning -Werror::FutureWarning" -fi - -$TEST_CMD sklearn diff --git a/build_tools/travis/test_wheels.sh b/build_tools/travis/test_wheels.sh deleted file mode 100644 index be2328e3d44d6..0000000000000 --- a/build_tools/travis/test_wheels.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set -e - -# Faster run of the source code tests -pytest -n $CPU_COUNT --pyargs sklearn - -# Test that there are no links to system libraries -python -m threadpoolctl -i sklearn diff --git a/build_tools/update_environments_and_lock_files.py b/build_tools/update_environments_and_lock_files.py new file mode 100644 index 0000000000000..5efd7f12cffd7 --- /dev/null +++ b/build_tools/update_environments_and_lock_files.py @@ -0,0 +1,778 @@ +"""Script to update CI environment files and associated lock files. + +To run it you need to be in the root folder of the scikit-learn repo: +python build_tools/update_environments_and_lock_files.py + +Two scenarios where this script can be useful: +- make sure that the latest versions of all the dependencies are used in the CI. + There is a scheduled workflow that does this, see + .github/workflows/update-lock-files.yml. This is still useful to run this + script when the automated PR fails and for example some packages need to + be pinned. You can add the pins to this script, run it, and open a PR with + the changes. +- bump minimum dependencies in sklearn/_min_dependencies.py. Running this + script will update both the CI environment files and associated lock files. + You can then open a PR with the changes. +- pin some packages to an older version by adding them to the + default_package_constraints variable. This is useful when regressions are + introduced in our dependencies, this has happened for example with pytest 7 + and coverage 6.3. + +Environments are conda environment.yml or pip requirements.txt. Lock files are +conda-lock lock files or pip-compile requirements.txt. + +pip requirements.txt are used when we install some dependencies (e.g. numpy and +scipy) with apt-get and the rest of the dependencies (e.g. pytest and joblib) +with pip. + +To run this script you need: +- conda +- conda-lock. The version should match the one used in the CI in + sklearn/_min_dependencies.py +- pip-tools + +To only update the environment and lock files for specific builds, you can use +the command line argument `--select-build` which will take a regex. For example, +to only update the documentation builds you can use: +`python build_tools/update_environments_and_lock_files.py --select-build doc` +""" + +import json +import logging +import re +import subprocess +import sys +from importlib.metadata import version +from pathlib import Path + +import click +from jinja2 import Environment +from packaging.version import Version + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +handler = logging.StreamHandler() +logger.addHandler(handler) + +TRACE = logging.DEBUG - 5 + + +common_dependencies_without_coverage = [ + "python", + "numpy", + "blas", + "scipy", + "cython", + "joblib", + "threadpoolctl", + "matplotlib", + "pandas", + "pyamg", + "pytest", + "pytest-xdist", + "pillow", + "pip", + "ninja", + "meson-python", +] + +common_dependencies = common_dependencies_without_coverage + [ + "pytest-cov", + "coverage", +] + +docstring_test_dependencies = ["sphinx", "numpydoc"] + +default_package_constraints = {} + + +def remove_from(alist, to_remove): + return [each for each in alist if each not in to_remove] + + +build_metadata_list = [ + { + "name": "pylatest_conda_forge_cuda_array-api_linux-64", + "type": "conda", + "tag": "cuda", + "folder": "build_tools/github", + "platform": "linux-64", + "channels": ["conda-forge", "pytorch", "nvidia"], + "conda_dependencies": common_dependencies + + [ + "ccache", + "pytorch-gpu", + "polars", + "pyarrow", + "cupy", + "array-api-strict", + ], + }, + { + "name": "pylatest_conda_forge_mkl_linux-64", + "type": "conda", + "tag": "main-ci", + "folder": "build_tools/azure", + "platform": "linux-64", + "channels": ["conda-forge"], + "conda_dependencies": common_dependencies + + [ + "ccache", + "pytorch", + "pytorch-cpu", + "polars", + "pyarrow", + "array-api-strict", + "scipy-doctest", + ], + "package_constraints": { + "blas": "[build=mkl]", + }, + }, + { + "name": "pylatest_conda_forge_mkl_osx-64", + "type": "conda", + "tag": "main-ci", + "folder": "build_tools/azure", + "platform": "osx-64", + "channels": ["conda-forge"], + "conda_dependencies": common_dependencies + + [ + "ccache", + "compilers", + "llvm-openmp", + ], + "package_constraints": { + "blas": "[build=mkl]", + }, + }, + { + "name": "pylatest_conda_mkl_no_openmp", + "type": "conda", + "tag": "main-ci", + "folder": "build_tools/azure", + "platform": "osx-64", + "channels": ["defaults"], + "conda_dependencies": remove_from( + common_dependencies, ["cython", "threadpoolctl", "meson-python"] + ) + + ["ccache"], + "package_constraints": { + "blas": "[build=mkl]", + # scipy 1.12.x crashes on this platform (https://github.com/scipy/scipy/pull/20086) + # TODO: release scipy constraint when 1.13 is available in the "default" + # channel. + "scipy": "<1.12", + }, + # TODO: put cython, threadpoolctl and meson-python back to conda + # dependencies when required version is available on the main channel + "pip_dependencies": ["cython", "threadpoolctl", "meson-python", "meson"], + }, + { + "name": "pymin_conda_forge_openblas_min_dependencies", + "type": "conda", + "tag": "main-ci", + "folder": "build_tools/azure", + "platform": "linux-64", + "channels": ["conda-forge"], + "conda_dependencies": common_dependencies + ["ccache", "polars"], + "package_constraints": { + "python": "3.10", + "blas": "[build=openblas]", + "numpy": "min", + "scipy": "min", + "matplotlib": "min", + "cython": "min", + "joblib": "min", + "threadpoolctl": "min", + "meson-python": "min", + "pandas": "min", + "polars": "min", + "pyamg": "min", + }, + }, + { + "name": "pymin_conda_forge_openblas_ubuntu_2204", + "type": "conda", + "tag": "main-ci", + "folder": "build_tools/azure", + "platform": "linux-64", + "channels": ["conda-forge"], + "conda_dependencies": ( + remove_from(common_dependencies_without_coverage, ["matplotlib"]) + + docstring_test_dependencies + + ["ccache"] + ), + "package_constraints": { + "python": "3.10", + "blas": "[build=openblas]", + }, + }, + { + "name": "pylatest_pip_openblas_pandas", + "type": "conda", + "tag": "main-ci", + "folder": "build_tools/azure", + "platform": "linux-64", + "channels": ["defaults"], + "conda_dependencies": ["python", "ccache"], + "pip_dependencies": ( + remove_from(common_dependencies, ["python", "blas", "pip"]) + + docstring_test_dependencies + # Test with some optional dependencies + + ["lightgbm", "scikit-image"] + # Test array API on CPU without PyTorch + + ["array-api-strict"] + # doctests dependencies + + ["scipy-doctest"] + ), + }, + { + "name": "pylatest_pip_scipy_dev", + "type": "conda", + "tag": "scipy-dev", + "folder": "build_tools/azure", + "platform": "linux-64", + "channels": ["defaults"], + "conda_dependencies": ["python", "ccache"], + "pip_dependencies": ( + remove_from( + common_dependencies, + [ + "python", + "blas", + "matplotlib", + "pyamg", + # all the dependencies below have a development version + # installed in the CI, so they can be removed from the + # environment.yml + "numpy", + "scipy", + "pandas", + "cython", + "joblib", + "pillow", + ], + ) + + ["pooch"] + + docstring_test_dependencies + # python-dateutil is a dependency of pandas and pandas is removed from + # the environment.yml. Adding python-dateutil so it is pinned + + ["python-dateutil"] + ), + }, + { + "name": "pylatest_free_threaded", + "type": "conda", + "tag": "free-threaded", + "folder": "build_tools/azure", + "platform": "linux-64", + "channels": ["conda-forge"], + "conda_dependencies": [ + "python-freethreading", + "numpy", + "scipy", + "cython", + "joblib", + "threadpoolctl", + "pytest", + "pytest-xdist", + "ninja", + "meson-python", + "ccache", + "pip", + ], + }, + { + "name": "pymin_conda_forge_mkl", + "type": "conda", + "tag": "main-ci", + "folder": "build_tools/azure", + "platform": "win-64", + "channels": ["conda-forge"], + "conda_dependencies": remove_from(common_dependencies, ["pandas", "pyamg"]) + + [ + "wheel", + "pip", + ], + "package_constraints": { + "python": "3.10", + "blas": "[build=mkl]", + }, + }, + { + "name": "doc_min_dependencies", + "type": "conda", + "tag": "main-ci", + "folder": "build_tools/circle", + "platform": "linux-64", + "channels": ["conda-forge"], + "conda_dependencies": common_dependencies_without_coverage + + [ + "scikit-image", + "seaborn", + "memory_profiler", + "compilers", + "sphinx", + "sphinx-gallery", + "sphinx-copybutton", + "numpydoc", + "sphinx-prompt", + "plotly", + "polars", + "pooch", + "sphinx-remove-toctrees", + "sphinx-design", + "pydata-sphinx-theme", + "towncrier", + ], + "pip_dependencies": [ + "sphinxext-opengraph", + "sphinxcontrib-sass", + ], + "package_constraints": { + "python": "3.10", + "numpy": "min", + "scipy": "min", + "matplotlib": "min", + "cython": "min", + "scikit-image": "min", + "sphinx": "min", + "pandas": "min", + "sphinx-gallery": "min", + "sphinx-copybutton": "min", + "numpydoc": "min", + "sphinx-prompt": "min", + "sphinxext-opengraph": "min", + "plotly": "min", + "polars": "min", + "pooch": "min", + "pyamg": "min", + "sphinx-design": "min", + "sphinxcontrib-sass": "min", + "sphinx-remove-toctrees": "min", + "pydata-sphinx-theme": "min", + "towncrier": "min", + }, + }, + { + "name": "doc", + "type": "conda", + "tag": "main-ci", + "folder": "build_tools/circle", + "platform": "linux-64", + "channels": ["conda-forge"], + "conda_dependencies": common_dependencies_without_coverage + + [ + "scikit-image", + "seaborn", + "memory_profiler", + "compilers", + "sphinx", + "sphinx-gallery", + "sphinx-copybutton", + "numpydoc", + "sphinx-prompt", + "plotly", + "polars", + "pooch", + "sphinxext-opengraph", + "sphinx-remove-toctrees", + "sphinx-design", + "pydata-sphinx-theme", + "towncrier", + ], + "pip_dependencies": [ + "jupyterlite-sphinx", + "jupyterlite-pyodide-kernel", + "sphinxcontrib-sass", + ], + "package_constraints": { + "python": "3.10", + }, + }, + { + "name": "pymin_conda_forge_arm", + "type": "conda", + "tag": "main-ci", + "folder": "build_tools/github", + "platform": "linux-aarch64", + "channels": ["conda-forge"], + "conda_dependencies": remove_from( + common_dependencies_without_coverage, ["pandas", "pyamg"] + ) + + ["pip", "ccache"], + "package_constraints": { + "python": "3.10", + }, + }, + { + "name": "debian_32bit", + "type": "pip", + "tag": "main-ci", + "folder": "build_tools/azure", + "pip_dependencies": [ + "cython", + "joblib", + "threadpoolctl", + "pytest", + "pytest-cov", + "ninja", + "meson-python", + ], + # Python version from the python3 APT package in the debian-32 docker + # image. + "python_version": "3.12.5", + }, + { + "name": "ubuntu_atlas", + "type": "pip", + "tag": "main-ci", + "folder": "build_tools/azure", + "pip_dependencies": [ + "cython", + "joblib", + "threadpoolctl", + "pytest", + "pytest-xdist", + "ninja", + "meson-python", + ], + "package_constraints": { + "joblib": "min", + "threadpoolctl": "min", + "cython": "min", + }, + "python_version": "3.10.4", + }, +] + + +def execute_command(command_list): + logger.debug(" ".join(command_list)) + proc = subprocess.Popen( + command_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + + out, err = proc.communicate() + out, err = out.decode(errors="replace"), err.decode(errors="replace") + + if proc.returncode != 0: + command_str = " ".join(command_list) + raise RuntimeError( + "Command exited with non-zero exit code.\n" + "Exit code: {}\n" + "Command:\n{}\n" + "stdout:\n{}\n" + "stderr:\n{}\n".format(proc.returncode, command_str, out, err) + ) + logger.log(TRACE, out) + return out + + +def get_package_with_constraint(package_name, build_metadata, uses_pip=False): + build_package_constraints = build_metadata.get("package_constraints") + if build_package_constraints is None: + constraint = None + else: + constraint = build_package_constraints.get(package_name) + + constraint = constraint or default_package_constraints.get(package_name) + + if constraint is None: + return package_name + + comment = "" + if constraint == "min": + constraint = execute_command( + [sys.executable, "sklearn/_min_dependencies.py", package_name] + ).strip() + comment = " # min" + + if re.match(r"\d[.\d]*", constraint): + equality = "==" if uses_pip else "=" + constraint = equality + constraint + + return f"{package_name}{constraint}{comment}" + + +environment = Environment(trim_blocks=True, lstrip_blocks=True) +environment.filters["get_package_with_constraint"] = get_package_with_constraint + + +def get_conda_environment_content(build_metadata): + template = environment.from_string( + """ +# DO NOT EDIT: this file is generated from the specification found in the +# following script to centralize the configuration for CI builds: +# build_tools/update_environments_and_lock_files.py +channels: + {% for channel in build_metadata['channels'] %} + - {{ channel }} + {% endfor %} +dependencies: + {% for conda_dep in build_metadata['conda_dependencies'] %} + - {{ conda_dep | get_package_with_constraint(build_metadata) }} + {% endfor %} + {% if build_metadata['pip_dependencies'] %} + - pip + - pip: + {% for pip_dep in build_metadata.get('pip_dependencies', []) %} + - {{ pip_dep | get_package_with_constraint(build_metadata, uses_pip=True) }} + {% endfor %} + {% endif %}""".strip() + ) + return template.render(build_metadata=build_metadata) + + +def write_conda_environment(build_metadata): + content = get_conda_environment_content(build_metadata) + build_name = build_metadata["name"] + folder_path = Path(build_metadata["folder"]) + output_path = folder_path / f"{build_name}_environment.yml" + logger.debug(output_path) + output_path.write_text(content) + + +def write_all_conda_environments(build_metadata_list): + for build_metadata in build_metadata_list: + write_conda_environment(build_metadata) + + +def conda_lock(environment_path, lock_file_path, platform): + execute_command( + [ + "conda-lock", + "lock", + "--mamba", + "--kind", + "explicit", + "--platform", + platform, + "--file", + str(environment_path), + "--filename-template", + str(lock_file_path), + ] + ) + + +def create_conda_lock_file(build_metadata): + build_name = build_metadata["name"] + folder_path = Path(build_metadata["folder"]) + environment_path = folder_path / f"{build_name}_environment.yml" + platform = build_metadata["platform"] + lock_file_basename = build_name + if not lock_file_basename.endswith(platform): + lock_file_basename = f"{lock_file_basename}_{platform}" + + lock_file_path = folder_path / f"{lock_file_basename}_conda.lock" + conda_lock(environment_path, lock_file_path, platform) + + +def write_all_conda_lock_files(build_metadata_list): + for build_metadata in build_metadata_list: + logger.info(f"# Locking dependencies for {build_metadata['name']}") + create_conda_lock_file(build_metadata) + + +def get_pip_requirements_content(build_metadata): + template = environment.from_string( + """ +# DO NOT EDIT: this file is generated from the specification found in the +# following script to centralize the configuration for CI builds: +# build_tools/update_environments_and_lock_files.py +{% for pip_dep in build_metadata['pip_dependencies'] %} +{{ pip_dep | get_package_with_constraint(build_metadata, uses_pip=True) }} +{% endfor %}""".strip() + ) + return template.render(build_metadata=build_metadata) + + +def write_pip_requirements(build_metadata): + build_name = build_metadata["name"] + content = get_pip_requirements_content(build_metadata) + folder_path = Path(build_metadata["folder"]) + output_path = folder_path / f"{build_name}_requirements.txt" + logger.debug(output_path) + output_path.write_text(content) + + +def write_all_pip_requirements(build_metadata_list): + for build_metadata in build_metadata_list: + write_pip_requirements(build_metadata) + + +def pip_compile(pip_compile_path, requirements_path, lock_file_path): + execute_command( + [ + str(pip_compile_path), + "--upgrade", + str(requirements_path), + "-o", + str(lock_file_path), + ] + ) + + +def write_pip_lock_file(build_metadata): + build_name = build_metadata["name"] + python_version = build_metadata["python_version"] + environment_name = f"pip-tools-python{python_version}" + # To make sure that the Python used to create the pip lock file is the same + # as the one used during the CI build where the lock file is used, we first + # create a conda environment with the correct Python version and + # pip-compile and run pip-compile in this environment + + execute_command( + [ + "conda", + "create", + "-c", + "conda-forge", + "-n", + f"pip-tools-python{python_version}", + f"python={python_version}", + "pip-tools", + "-y", + ] + ) + + json_output = execute_command(["conda", "info", "--json"]) + conda_info = json.loads(json_output) + environment_folder = next( + each for each in conda_info["envs"] if each.endswith(environment_name) + ) + environment_path = Path(environment_folder) + pip_compile_path = environment_path / "bin" / "pip-compile" + + folder_path = Path(build_metadata["folder"]) + requirement_path = folder_path / f"{build_name}_requirements.txt" + lock_file_path = folder_path / f"{build_name}_lock.txt" + pip_compile(pip_compile_path, requirement_path, lock_file_path) + + +def write_all_pip_lock_files(build_metadata_list): + for build_metadata in build_metadata_list: + logger.info(f"# Locking dependencies for {build_metadata['name']}") + write_pip_lock_file(build_metadata) + + +def check_conda_lock_version(): + # Check that the installed conda-lock version is consistent with _min_dependencies. + expected_conda_lock_version = execute_command( + [sys.executable, "sklearn/_min_dependencies.py", "conda-lock"] + ).strip() + + installed_conda_lock_version = version("conda-lock") + if installed_conda_lock_version != expected_conda_lock_version: + raise RuntimeError( + f"Expected conda-lock version: {expected_conda_lock_version}, got:" + f" {installed_conda_lock_version}" + ) + + +def check_conda_version(): + # Avoid issues with glibc (https://github.com/conda/conda-lock/issues/292) + # or osx (https://github.com/conda/conda-lock/issues/408) virtual package. + # The glibc one has been fixed in conda 23.1.0 and the osx has been fixed + # in conda 23.7.0. + conda_info_output = execute_command(["conda", "info", "--json"]) + + conda_info = json.loads(conda_info_output) + conda_version = Version(conda_info["conda_version"]) + + if Version("22.9.0") < conda_version < Version("23.7"): + raise RuntimeError( + f"conda version should be <= 22.9.0 or >= 23.7 got: {conda_version}" + ) + + +@click.command() +@click.option( + "--select-build", + default="", + help=( + "Regex to filter the builds we want to update environment and lock files. By" + " default all the builds are selected." + ), +) +@click.option( + "--skip-build", + default=None, + help="Regex to skip some builds from the builds selected by --select-build", +) +@click.option( + "--select-tag", + default=None, + help=( + "Tag to filter the builds, e.g. 'main-ci' or 'scipy-dev'. " + "This is an additional filtering on top of --select-build." + ), +) +@click.option( + "-v", + "--verbose", + is_flag=True, + help="Print commands executed by the script", +) +@click.option( + "-vv", + "--very-verbose", + is_flag=True, + help="Print output of commands executed by the script", +) +def main(select_build, skip_build, select_tag, verbose, very_verbose): + if verbose: + logger.setLevel(logging.DEBUG) + if very_verbose: + logger.setLevel(TRACE) + handler.setLevel(TRACE) + check_conda_lock_version() + check_conda_version() + + filtered_build_metadata_list = [ + each for each in build_metadata_list if re.search(select_build, each["name"]) + ] + if select_tag is not None: + filtered_build_metadata_list = [ + each for each in build_metadata_list if each["tag"] == select_tag + ] + if skip_build is not None: + filtered_build_metadata_list = [ + each + for each in filtered_build_metadata_list + if not re.search(skip_build, each["name"]) + ] + + selected_build_info = "\n".join( + f" - {each['name']}, type: {each['type']}, tag: {each['tag']}" + for each in filtered_build_metadata_list + ) + selected_build_message = ( + f"# {len(filtered_build_metadata_list)} selected builds\n{selected_build_info}" + ) + logger.info(selected_build_message) + + filtered_conda_build_metadata_list = [ + each for each in filtered_build_metadata_list if each["type"] == "conda" + ] + + if filtered_conda_build_metadata_list: + logger.info("# Writing conda environments") + write_all_conda_environments(filtered_conda_build_metadata_list) + logger.info("# Writing conda lock files") + write_all_conda_lock_files(filtered_conda_build_metadata_list) + + filtered_pip_build_metadata_list = [ + each for each in filtered_build_metadata_list if each["type"] == "pip" + ] + if filtered_pip_build_metadata_list: + logger.info("# Writing pip requirements") + write_all_pip_requirements(filtered_pip_build_metadata_list) + logger.info("# Writing pip lock files") + write_all_pip_lock_files(filtered_pip_build_metadata_list) + + +if __name__ == "__main__": + main() diff --git a/build_tools/wheels/LICENSE_linux.txt b/build_tools/wheels/LICENSE_linux.txt new file mode 100644 index 0000000000000..057656fcc789d --- /dev/null +++ b/build_tools/wheels/LICENSE_linux.txt @@ -0,0 +1,80 @@ +This binary distribution of scikit-learn also bundles the following software: + +---- + +Name: GCC runtime library +Files: scikit_learn.libs/libgomp*.so* +Availability: https://gcc.gnu.org/git/?p=gcc.git;a=tree;f=libgomp + +GCC RUNTIME LIBRARY EXCEPTION + +Version 3.1, 31 March 2009 + +Copyright (C) 2009 Free Software Foundation, Inc. + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + +This GCC Runtime Library Exception ("Exception") is an additional +permission under section 7 of the GNU General Public License, version +3 ("GPLv3"). It applies to a given file (the "Runtime Library") that +bears a notice placed by the copyright holder of the file stating that +the file is governed by GPLv3 along with this Exception. + +When you use GCC to compile a program, GCC may combine portions of +certain GCC header files and runtime libraries with the compiled +program. The purpose of this Exception is to allow compilation of +non-GPL (including proprietary) programs to use, in this way, the +header files and runtime libraries covered by this Exception. + +0. Definitions. + +A file is an "Independent Module" if it either requires the Runtime +Library for execution after a Compilation Process, or makes use of an +interface provided by the Runtime Library, but is not otherwise based +on the Runtime Library. + +"GCC" means a version of the GNU Compiler Collection, with or without +modifications, governed by version 3 (or a specified later version) of +the GNU General Public License (GPL) with the option of using any +subsequent versions published by the FSF. + +"GPL-compatible Software" is software whose conditions of propagation, +modification and use would permit combination with GCC in accord with +the license of GCC. + +"Target Code" refers to output from any compiler for a real or virtual +target processor architecture, in executable form or suitable for +input to an assembler, loader, linker and/or execution +phase. Notwithstanding that, Target Code does not include data in any +format that is used as a compiler intermediate representation, or used +for producing a compiler intermediate representation. + +The "Compilation Process" transforms code entirely represented in +non-intermediate languages designed for human-written code, and/or in +Java Virtual Machine byte code, into Target Code. Thus, for example, +use of source code generators and preprocessors need not be considered +part of the Compilation Process, since the Compilation Process can be +understood as starting with the output of the generators or +preprocessors. + +A Compilation Process is "Eligible" if it is done using GCC, alone or +with other GPL-compatible software, or if it is done without using any +work based on GCC. For example, using non-GPL-compatible Software to +optimize any GCC intermediate representations would not qualify as an +Eligible Compilation Process. + +1. Grant of Additional Permission. + +You have permission to propagate a work of Target Code formed by +combining the Runtime Library with Independent Modules, even if such +propagation would otherwise violate the terms of GPLv3, provided that +all Target Code was generated by Eligible Compilation Processes. You +may then convey such a combination under terms of your choice, +consistent with the licensing of the Independent Modules. + +2. No Weakening of GCC Copyleft. + +The availability of this Exception does not imply any general +presumption that third-party software is unaffected by the copyleft +requirements of the license of GCC. diff --git a/build_tools/wheels/LICENSE_macos.txt b/build_tools/wheels/LICENSE_macos.txt new file mode 100644 index 0000000000000..61a523f47663c --- /dev/null +++ b/build_tools/wheels/LICENSE_macos.txt @@ -0,0 +1,286 @@ +This binary distribution of scikit-learn also bundles the following software: + +---- + +Name: libomp runtime library +Files: sklearn/.dylibs/libomp.dylib +Availability: https://github.com/llvm/llvm-project + +============================================================================== +The LLVM Project is under the Apache License v2.0 with LLVM Exceptions: +============================================================================== + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + +============================================================================== +Software from third parties included in the LLVM Project: +============================================================================== +The LLVM Project contains third party software which is under different license +terms. All such code will be identified clearly using at least one of two +mechanisms: +1) It will be in a separate directory tree with its own `LICENSE.txt` or + `LICENSE` file at the top containing the specific license and restrictions + which apply to that software, or +2) It will contain specific license and restriction terms at the top of every + file. + +============================================================================== +Legacy LLVM License (https://llvm.org/docs/DeveloperPolicy.html#legacy): +============================================================================== +University of Illinois/NCSA +Open Source License + +Copyright (c) 2003-2019 University of Illinois at Urbana-Champaign. +All rights reserved. + +Developed by: + + LLVM Team + + University of Illinois at Urbana-Champaign + + http://llvm.org + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of the LLVM Team, University of Illinois at + Urbana-Champaign, nor the names of its contributors may be used to + endorse or promote products derived from this Software without specific + prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. diff --git a/build_tools/wheels/LICENSE_windows.txt b/build_tools/wheels/LICENSE_windows.txt new file mode 100644 index 0000000000000..9e98ad8defac2 --- /dev/null +++ b/build_tools/wheels/LICENSE_windows.txt @@ -0,0 +1,25 @@ +This binary distribution of scikit-learn also bundles the following software: + +---- + +Name: Microsoft Visual C++ Runtime Files +Files: sklearn\.libs\*.dll +Availability: https://learn.microsoft.com/en-us/visualstudio/releases/2015/2015-redistribution-vs + +Subject to the License Terms for the software, you may copy and distribute with your +program any of the files within the followng folder and its subfolders except as noted +below. You may not modify these files. + +C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\redist + +You may not distribute the contents of the following folders: + +C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\redist\debug_nonredist +C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\redist\onecore\debug_nonredist + +Subject to the License Terms for the software, you may copy and distribute the following +files with your program in your program’s application local folder or by deploying them +into the Global Assembly Cache (GAC): + +VC\atlmfc\lib\mfcmifc80.dll +VC\atlmfc\lib\amd64\mfcmifc80.dll diff --git a/build_tools/wheels/build_wheels.sh b/build_tools/wheels/build_wheels.sh new file mode 100755 index 0000000000000..02b05bc8a2795 --- /dev/null +++ b/build_tools/wheels/build_wheels.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +set -e +set -x + +# Set environment variables to make our wheel build easier to reproduce byte +# for byte from source. See https://reproducible-builds.org/. The long term +# motivation would be to be able to detect supply chain attacks. +# +# In particular we set SOURCE_DATE_EPOCH to the commit date of the last commit. +# +# XXX: setting those environment variables is not enough. See the following +# issue for more details on what remains to do: +# https://github.com/scikit-learn/scikit-learn/issues/28151 +export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) +export PYTHONHASHSEED=0 + +# OpenMP is not present on macOS by default +if [[ $(uname) == "Darwin" ]]; then + # Make sure to use a libomp version binary compatible with the oldest + # supported version of the macos SDK as libomp will be vendored into the + # scikit-learn wheels for macos. + + if [[ "$CIBW_BUILD" == *-macosx_arm64 ]]; then + if [[ $(uname -m) == "x86_64" ]]; then + # arm64 builds must cross compile because the CI instance is x86 + # This turns off the computation of the test program in + # sklearn/_build_utils/pre_build_helpers.py + export PYTHON_CROSSENV=1 + fi + # SciPy requires 12.0 on arm to prevent kernel panics + # https://github.com/scipy/scipy/issues/14688 + # We use the same deployment target to match SciPy. + export MACOSX_DEPLOYMENT_TARGET=12.0 + OPENMP_URL="https://anaconda.org/conda-forge/llvm-openmp/11.1.0/download/osx-arm64/llvm-openmp-11.1.0-hf3c4609_1.tar.bz2" + else + export MACOSX_DEPLOYMENT_TARGET=10.9 + OPENMP_URL="https://anaconda.org/conda-forge/llvm-openmp/11.1.0/download/osx-64/llvm-openmp-11.1.0-hda6cdc1_1.tar.bz2" + fi + + conda create -n build $OPENMP_URL + PREFIX="$HOME/miniconda3/envs/build" + + export CC=/usr/bin/clang + export CXX=/usr/bin/clang++ + export CPPFLAGS="$CPPFLAGS -Xpreprocessor -fopenmp" + export CFLAGS="$CFLAGS -I$PREFIX/include" + export CXXFLAGS="$CXXFLAGS -I$PREFIX/include" + export LDFLAGS="$LDFLAGS -Wl,-rpath,$PREFIX/lib -L$PREFIX/lib -lomp" +fi + +if [[ "$CIBW_FREE_THREADED_SUPPORT" =~ [tT]rue ]]; then + # Numpy, scipy, Cython only have free-threaded wheels on scientific-python-nightly-wheels + # TODO: remove this after CPython 3.13 is released (scheduled October 2024) + # and our dependencies have free-threaded wheels on PyPI + export CIBW_BUILD_FRONTEND='pip; args: --pre --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" --only-binary :all:' +fi + +# The version of the built dependencies are specified +# in the pyproject.toml file, while the tests are run +# against the most recent version of the dependencies + +python -m pip install cibuildwheel +python -m cibuildwheel --output-dir wheelhouse diff --git a/build_tools/wheels/check_license.py b/build_tools/wheels/check_license.py new file mode 100644 index 0000000000000..00fe4169be65d --- /dev/null +++ b/build_tools/wheels/check_license.py @@ -0,0 +1,30 @@ +"""Checks the bundled license is installed with the wheel.""" + +import platform +import site +from itertools import chain +from pathlib import Path + +site_packages = site.getsitepackages() + +site_packages_path = (Path(p) for p in site_packages) + +try: + distinfo_path = next( + chain( + s + for site_package in site_packages_path + for s in site_package.glob("scikit_learn-*.dist-info") + ) + ) +except StopIteration as e: + raise RuntimeError("Unable to find scikit-learn's dist-info") from e + +license_text = (distinfo_path / "COPYING").read_text() + +assert "Copyright (c)" in license_text + +assert ( + "This binary distribution of scikit-learn also bundles the following software" + in license_text +), f"Unable to find bundled license for {platform.system()}" diff --git a/build_tools/wheels/cibw_before_build.sh b/build_tools/wheels/cibw_before_build.sh new file mode 100755 index 0000000000000..4e4558db5a5bc --- /dev/null +++ b/build_tools/wheels/cibw_before_build.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -euxo pipefail + +PROJECT_DIR="$1" +LICENSE_FILE="$PROJECT_DIR/COPYING" + +echo "" >>"$LICENSE_FILE" +echo "----" >>"$LICENSE_FILE" +echo "" >>"$LICENSE_FILE" + +if [[ $RUNNER_OS == "Linux" ]]; then + cat $PROJECT_DIR/build_tools/wheels/LICENSE_linux.txt >>"$LICENSE_FILE" +elif [[ $RUNNER_OS == "macOS" ]]; then + cat $PROJECT_DIR/build_tools/wheels/LICENSE_macos.txt >>"$LICENSE_FILE" +elif [[ $RUNNER_OS == "Windows" ]]; then + cat $PROJECT_DIR/build_tools/wheels/LICENSE_windows.txt >>"$LICENSE_FILE" +fi diff --git a/build_tools/wheels/test_wheels.sh b/build_tools/wheels/test_wheels.sh new file mode 100755 index 0000000000000..1d6ee19bda8a8 --- /dev/null +++ b/build_tools/wheels/test_wheels.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -e +set -x + +PROJECT_DIR="$1" + +python $PROJECT_DIR/build_tools/wheels/check_license.py + +python -c "import joblib; print(f'Number of cores (physical): \ +{joblib.cpu_count()} ({joblib.cpu_count(only_physical_cores=True)})')" + +FREE_THREADED_BUILD="$(python -c"import sysconfig; print(bool(sysconfig.get_config_var('Py_GIL_DISABLED')))")" +if [[ $FREE_THREADED_BUILD == "True" ]]; then + # TODO: delete when importing numpy no longer enables the GIL + # setting to zero ensures the GIL is disabled while running the + # tests under free-threaded python + export PYTHON_GIL=0 +fi + +# Test that there are no links to system libraries in the +# threadpoolctl output section of the show_versions output: +python -c "import sklearn; sklearn.show_versions()" + +if pip show -qq pytest-xdist; then + XDIST_WORKERS=$(python -c "import joblib; print(joblib.cpu_count(only_physical_cores=True))") + pytest --pyargs sklearn -n $XDIST_WORKERS +else + pytest --pyargs sklearn +fi diff --git a/conftest.py b/conftest.py deleted file mode 100644 index aec49c03ae13d..0000000000000 --- a/conftest.py +++ /dev/null @@ -1,91 +0,0 @@ -# Even if empty this file is useful so that when running from the root folder -# ./sklearn is added to sys.path by pytest. See -# https://docs.pytest.org/en/latest/pythonpath.html for more details. For -# example, this allows to build extensions in place and run pytest -# doc/modules/clustering.rst and use sklearn from the local folder rather than -# the one from site-packages. - -import platform -import sys - -import pytest -from _pytest.doctest import DoctestItem - -from sklearn.utils import _IS_32BIT -from sklearn.externals import _pilutil -from sklearn._min_dependencies import PYTEST_MIN_VERSION -from sklearn.utils.fixes import np_version, parse_version - -if parse_version(pytest.__version__) < parse_version(PYTEST_MIN_VERSION): - raise ImportError('Your version of pytest is too old, you should have ' - 'at least pytest >= {} installed.' - .format(PYTEST_MIN_VERSION)) - - -def pytest_collection_modifyitems(config, items): - for item in items: - # FeatureHasher is not compatible with PyPy - if (item.name.endswith(('_hash.FeatureHasher', - 'text.HashingVectorizer')) - and platform.python_implementation() == 'PyPy'): - marker = pytest.mark.skip( - reason='FeatureHasher is not compatible with PyPy') - item.add_marker(marker) - # Known failure on with GradientBoostingClassifier on ARM64 - elif (item.name.endswith('GradientBoostingClassifier') - and platform.machine() == 'aarch64'): - - marker = pytest.mark.xfail( - reason=( - 'know failure. See ' - 'https://github.com/scikit-learn/scikit-learn/issues/17797' # noqa - ) - ) - item.add_marker(marker) - - # numpy changed the str/repr formatting of numpy arrays in 1.14. We want to - # run doctests only for numpy >= 1.14. - skip_doctests = False - try: - if np_version < parse_version('1.14'): - reason = 'doctests are only run for numpy >= 1.14' - skip_doctests = True - elif _IS_32BIT: - reason = ('doctest are only run when the default numpy int is ' - '64 bits.') - skip_doctests = True - elif sys.platform.startswith("win32"): - reason = ("doctests are not run for Windows because numpy arrays " - "repr is inconsistent across platforms.") - skip_doctests = True - except ImportError: - pass - - if skip_doctests: - skip_marker = pytest.mark.skip(reason=reason) - - for item in items: - if isinstance(item, DoctestItem): - item.add_marker(skip_marker) - elif not _pilutil.pillow_installed: - skip_marker = pytest.mark.skip(reason="pillow (or PIL) not installed!") - for item in items: - if item.name in [ - "sklearn.feature_extraction.image.PatchExtractor", - "sklearn.feature_extraction.image.extract_patches_2d"]: - item.add_marker(skip_marker) - - -def pytest_configure(config): - import sys - sys._is_pytest_session = True - # declare our custom markers to avoid PytestUnknownMarkWarning - config.addinivalue_line( - "markers", - "network: mark a test for execution if network available." - ) - - -def pytest_unconfigure(config): - import sys - del sys._is_pytest_session diff --git a/doc/Makefile b/doc/Makefile index 6146d11123017..1419bac49316d 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -2,18 +2,29 @@ # # You can set these variables from the command line. -SPHINXOPTS = -j auto +SPHINXOPTS ?= -T SPHINXBUILD ?= sphinx-build PAPER = BUILDDIR = _build + ifneq ($(EXAMPLES_PATTERN),) EXAMPLES_PATTERN_OPTS := -D sphinx_gallery_conf.filename_pattern="$(EXAMPLES_PATTERN)" endif +ifeq ($(CI), true) + # On CircleCI using -j2 does not seem to speed up the html-noplot build + SPHINX_NUMJOBS_NOPLOT_DEFAULT=1 +else ifeq ($(shell uname), Darwin) + # Avoid stalling issues on MacOS + SPHINX_NUMJOBS_NOPLOT_DEFAULT=1 +else + SPHINX_NUMJOBS_NOPLOT_DEFAULT=auto +endif + # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -T -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS)\ +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS)\ $(EXAMPLES_PATTERN_OPTS) . @@ -36,21 +47,40 @@ help: clean: -rm -rf $(BUILDDIR)/* + @echo "Removed $(BUILDDIR)/*" -rm -rf auto_examples/ + @echo "Removed auto_examples/" -rm -rf generated/* + @echo "Removed generated/" -rm -rf modules/generated/ - + @echo "Removed modules/generated/" + -rm -rf css/styles/ + @echo "Removed css/styles/" + -rm -rf api/*.rst + @echo "Removed api/*.rst" + +# Default to SPHINX_NUMJOBS=1 for full documentation build. Using +# SPHINX_NUMJOBS!=1 may actually slow down the build, or cause weird issues in +# the CI (job stalling or EOFError), see +# https://github.com/scikit-learn/scikit-learn/pull/25836 or +# https://github.com/scikit-learn/scikit-learn/pull/25809 +html: SPHINX_NUMJOBS ?= 1 html: + @echo $(ALLSPHINXOPTS) # These two lines make the build a bit more lengthy, and the # the embedding of images more robust rm -rf $(BUILDDIR)/html/_images #rm -rf _build/doctrees/ - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html/stable + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) -j$(SPHINX_NUMJOBS) $(BUILDDIR)/html/stable @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html/stable" +# Default to SPHINX_NUMJOBS=auto (except on MacOS and CI) since this makes +# html-noplot build faster +html-noplot: SPHINX_NUMJOBS ?= $(SPHINX_NUMJOBS_NOPLOT_DEFAULT) html-noplot: - $(SPHINXBUILD) -D plot_gallery=0 -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html/stable + $(SPHINXBUILD) -D plot_gallery=0 -b html $(ALLSPHINXOPTS) -j$(SPHINX_NUMJOBS) \ + $(BUILDDIR)/html/stable @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html/stable." diff --git a/doc/README.md b/doc/README.md index 18d4bde4f5862..537ed85006006 100644 --- a/doc/README.md +++ b/doc/README.md @@ -1,6 +1,6 @@ # Documentation for scikit-learn -This directory contains the full manual and web site as displayed at -http://scikit-learn.org. See -http://scikit-learn.org/dev/developers/contributing.html#documentation for -detailed information about the documentation. +This directory contains the full manual and website as displayed at +https://scikit-learn.org. See +https://scikit-learn.org/dev/developers/contributing.html#documentation for +detailed information about the documentation. diff --git a/doc/about.rst b/doc/about.rst index fdfe8241b8aec..4db39f9709e73 100644 --- a/doc/about.rst +++ b/doc/about.rst @@ -1,480 +1,512 @@ .. _about: +======== About us ======== History -------- +======= This project was started in 2007 as a Google Summer of Code project by -David Cournapeau. Later that year, Matthieu Brucher started work on -this project as part of his thesis. +David Cournapeau. Later that year, Matthieu Brucher started working on this project +as part of his thesis. In 2010 Fabian Pedregosa, Gael Varoquaux, Alexandre Gramfort and Vincent Michel of INRIA took leadership of the project and made the first public release, February the 1st 2010. Since then, several releases have appeared -following a ~ 3-month cycle, and a thriving international community has -been leading the development. +following an approximately 3-month cycle, and a thriving international +community has been leading the development. As a result, INRIA holds the +copyright over the work done by people who were employed by INRIA at the +time of the contribution. Governance ----------- -The decision making process and governance structure of scikit-learn is laid -out in the :ref:`governance document `. +========== + +The decision making process and governance structure of scikit-learn, like roles and responsibilities, is laid out in the :ref:`governance document `. + +.. The "author" anchors below is there to ensure that old html links (in + the form of "about.html#author" still work) + +.. _authors: + +The people behind scikit-learn +============================== + +Scikit-learn is a community project, developed by a large group of +people, all across the world. A few core contributor teams, listed below, have +central roles, however a more complete list of contributors can be found `on +github +`__. + +Active Core Contributors +------------------------ -Authors -------- +Maintainers Team +................ -The following people are currently core contributors to scikit-learn's development -and maintenance: +The following people are currently maintainers, in charge of +consolidating scikit-learn's development and maintenance: -.. include:: authors.rst +.. include:: maintainers.rst -Please do not email the authors directly to ask for assistance or report issues. -Instead, please see `What's the best way to ask questions about scikit-learn -`_ -in the FAQ. +.. note:: + + Please do not email the authors directly to ask for assistance or report issues. + Instead, please see `What's the best way to ask questions about scikit-learn + `_ + in the FAQ. .. seealso:: - :ref:`How you can contribute to the project ` + How you can :ref:`contribute to the project `. + +Documentation Team +.................. + +The following people help with documenting the project: + +.. include:: documentation_team.rst -Triage Team ------------ +Contributor Experience Team +........................... The following people are active contributors who also help with :ref:`triaging issues `, PRs, and general maintenance: -.. include:: triage_team.rst +.. include:: contributor_experience_team.rst -Emeritus Core Developers ------------------------- +Communication Team +.................. + +The following people help with :ref:`communication around scikit-learn +`. + +.. include:: communication_team.rst + +Emeritus Core Contributors +-------------------------- + +Emeritus Maintainers Team +......................... The following people have been active contributors in the past, but are no longer active in the project: -.. include:: authors_emeritus.rst +.. include:: maintainers_emeritus.rst + +Emeritus Communication Team +........................... + +The following people have been active in the communication team in the +past, but no longer have communication responsibilities: + +.. include:: communication_team_emeritus.rst +Emeritus Contributor Experience Team +.................................... + +The following people have been active in the contributor experience team in the +past: + +.. include:: contributor_experience_team_emeritus.rst .. _citing-scikit-learn: Citing scikit-learn -------------------- +=================== If you use scikit-learn in a scientific publication, we would appreciate citations to the following paper: - `Scikit-learn: Machine Learning in Python - `_, Pedregosa - *et al.*, JMLR 12, pp. 2825-2830, 2011. - - Bibtex entry:: - - @article{scikit-learn, - title={Scikit-learn: Machine Learning in {P}ython}, - author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. - and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. - and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and - Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, - journal={Journal of Machine Learning Research}, - volume={12}, - pages={2825--2830}, - year={2011} - } +`Scikit-learn: Machine Learning in Python +`_, Pedregosa +*et al.*, JMLR 12, pp. 2825-2830, 2011. + +Bibtex entry:: + + @article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} + } If you want to cite scikit-learn for its API or design, you may also want to consider the following paper: - `API design for machine learning software: experiences from the scikit-learn - project `_, Buitinck *et al.*, 2013. - - Bibtex entry:: - - @inproceedings{sklearn_api, - author = {Lars Buitinck and Gilles Louppe and Mathieu Blondel and - Fabian Pedregosa and Andreas Mueller and Olivier Grisel and - Vlad Niculae and Peter Prettenhofer and Alexandre Gramfort - and Jaques Grobler and Robert Layton and Jake VanderPlas and - Arnaud Joly and Brian Holt and Ga{\"{e}}l Varoquaux}, - title = {{API} design for machine learning software: experiences from the scikit-learn - project}, - booktitle = {ECML PKDD Workshop: Languages for Data Mining and Machine Learning}, - year = {2013}, - pages = {108--122}, - } +:arxiv:`API design for machine learning software: experiences from the scikit-learn +project <1309.0238>`, Buitinck *et al.*, 2013. + +Bibtex entry:: + + @inproceedings{sklearn_api, + author = {Lars Buitinck and Gilles Louppe and Mathieu Blondel and + Fabian Pedregosa and Andreas Mueller and Olivier Grisel and + Vlad Niculae and Peter Prettenhofer and Alexandre Gramfort + and Jaques Grobler and Robert Layton and Jake VanderPlas and + Arnaud Joly and Brian Holt and Ga{\"{e}}l Varoquaux}, + title = {{API} design for machine learning software: experiences from the scikit-learn + project}, + booktitle = {ECML PKDD Workshop: Languages for Data Mining and Machine Learning}, + year = {2013}, + pages = {108--122}, + } Artwork -------- +======= High quality PNG and SVG logos are available in the `doc/logos/ `_ source directory. .. image:: images/scikit-learn-logo-notext.png - :align: center + :align: center Funding -------- -Scikit-Learn is a community driven project, however institutional and private +======= + +Scikit-learn is a community driven project, however institutional and private grants help to assure its sustainability. The project would like to thank the following funders. ................................... -.. raw:: html +.. div:: sk-text-image-grid-small -
-
+ .. div:: text-box -The `Members `_ of -the `Scikit-Learn Consortium at Inria Foundation -`_ fund Olivier -Grisel, Guillaume Lemaitre, JÊrÊmie du Boisberranger and Chiara Marmo. + `:probabl. `_ employs Adrin Jalali, Arturo Amor, + François Goupil, Guillaume Lemaitre, JÊrÊmie du Boisberranger, Loïc Estève, + Olivier Grisel, and Stefanie Senger. -.. raw:: html + .. div:: image-box -
+ .. image:: images/probabl.png + :target: https://probabl.ai -.. |msn| image:: images/microsoft.png - :width: 100pt - :target: https://www.microsoft.com/ +.......... -.. |bcg| image:: images/bcg.png - :width: 100pt - :target: https://www.bcg.com/beyond-consulting/bcg-gamma/default.aspx +.. |chanel| image:: images/chanel.png + :target: https://www.chanel.com .. |axa| image:: images/axa.png - :width: 50pt - :target: https://www.axa.fr/ + :target: https://www.axa.fr/ .. |bnp| image:: images/bnp.png - :width: 150pt - :target: https://www.bnpparibascardif.com/ - -.. |fujitsu| image:: images/fujitsu.png - :width: 100pt - :target: https://www.fujitsu.com/global/ + :target: https://www.bnpparibascardif.com/ .. |dataiku| image:: images/dataiku.png - :width: 70pt - :target: https://www.dataiku.com/ + :target: https://www.dataiku.com/ -.. |inria| image:: images/inria-logo.jpg - :width: 100pt - :target: https://www.inria.fr +.. |nvidia| image:: images/nvidia.png + :target: https://www.nvidia.com +.. |inria| image:: images/inria-logo.jpg + :target: https://www.inria.fr .. raw:: html -
- -.. table:: - :class: sk-sponsor-table align-default - - +---------+----------+ - | |bcg| | - +---------+----------+ - | | - +---------+----------+ - | |axa| | |bnp| | - +---------+----------+ - ||fujitsu|| |msn| | - +---------+----------+ - | | - +---------+----------+ - | |dataiku| | - +---------+----------+ - | | - +---------+----------+ - | |inria| | - +---------+----------+ + -........ +.. div:: sk-text-image-grid-small -.. raw:: html + .. div:: text-box -
-
+ The `Members `_ of + the `Scikit-learn Consortium at Inria Foundation + `_ help at maintaining and + improving the project through their financial support. -`Columbia University `_ funds Andreas MÃŧller since 2016 + .. div:: image-box -.. raw:: html + .. table:: + :class: image-subtable + + +----------+-----------+ + | |chanel| | + +----------+-----------+ + | |axa| | |bnp| | + +----------+-----------+ + | |nvidia| | + +----------+-----------+ + | |dataiku| | + +----------+-----------+ + | |inria| | + +----------+-----------+ -
+.......... -
+.. div:: sk-text-image-grid-small -.. image:: themes/scikit-learn/static/img/columbia.png - :width: 50pt - :align: center - :target: https://www.columbia.edu/ + .. div:: text-box -.. raw:: html + `NVidia `_ funds Tim Head since 2022 + and is part of the scikit-learn consortium at Inria. -
-
+ .. div:: image-box + + .. image:: images/nvidia.png + :target: https://nvidia.com .......... -.. raw:: html +.. div:: sk-text-image-grid-small -
-
+ .. div:: text-box -Andreas MÃŧller received a grant to improve scikit-learn from the -`Alfred P. Sloan Foundation `_ . -This grant supports the position of Nicolas Hug and Thomas J. Fan. + `Microsoft `_ funds Andreas MÃŧller since 2020. -.. raw:: html + .. div:: image-box + + .. image:: images/microsoft.png + :target: https://microsoft.com -
+........... -
+.. div:: sk-text-image-grid-small -.. image:: images/sloan_banner.png - :width: 100pt - :align: center - :target: https://sloan.org/ + .. div:: text-box -.. raw:: html + `Quansight Labs `_ funds Lucy Liu since 2022. -
-
+ .. div:: image-box + + .. image:: images/quansight-labs.png + :target: https://labs.quansight.org ........... -.. raw:: html +.. |czi| image:: images/czi.png + :target: https://chanzuckerberg.com -
-
+.. |wellcome| image:: images/wellcome-trust.png + :target: https://wellcome.org/ -`The University of Sydney `_ funds Joel Nothman since -July 2017. +.. div:: sk-text-image-grid-small -.. raw:: html + .. div:: text-box -
+ `The Chan-Zuckerberg Initiative `_ and + `Wellcome Trust `_ fund scikit-learn through the + `Essential Open Source Software for Science (EOSS) `_ + cycle 6. -
+ It supports Lucy Liu and diversity & inclusion initiatives that will + be announced in the future. -.. image:: themes/scikit-learn/static/img/sydney-primary.jpeg - :width: 100pt - :align: center - :target: https://sydney.edu.au/ + .. div:: image-box -.. raw:: html + .. table:: + :class: image-subtable -
-
+ +----------+----------------+ + | |czi| | |wellcome| | + +----------+----------------+ ........... -.. raw:: html +.. div:: sk-text-image-grid-small -
-
+ .. div:: text-box -`Zalando SE `_ funds Adrin Jalali since -August 2020. + `Tidelift `_ supports the project via their service + agreement. -.. raw:: html + .. div:: image-box -
+ .. image:: images/Tidelift-logo-on-light.svg + :target: https://tidelift.com/ -
+........... -.. image:: images/zalando_logo.png - :width: 100pt - :align: center - :target: https://corporate.zalando.com/en -.. raw:: html +Past Sponsors +------------- -
-
+.. div:: sk-text-image-grid-small -Past Sponsors -............. + .. div:: text-box -.. raw:: html + `Quansight Labs `_ funded Meekail Zain in 2022 and 2023, + and funded Thomas J. Fan from 2021 to 2023. -
-
+ .. div:: image-box -`INRIA `_ actively supports this project. It has -provided funding for Fabian Pedregosa (2010-2012), Jaques Grobler -(2012-2013) and Olivier Grisel (2013-2017) to work on this project -full-time. It also hosts coding sprints and other events. + .. image:: images/quansight-labs.png + :target: https://labs.quansight.org -.. raw:: html +........... -
+.. div:: sk-text-image-grid-small -
+ .. div:: text-box -.. image:: images/inria-logo.jpg - :width: 100pt - :align: center - :target: https://www.inria.fr + `Columbia University `_ funded Andreas MÃŧller + (2016-2020). -.. raw:: html + .. div:: image-box -
-
+ .. image:: images/columbia.png + :target: https://columbia.edu -..................... +........ -.. raw:: html +.. div:: sk-text-image-grid-small -
-
+ .. div:: text-box -`Paris-Saclay Center for Data Science -`_ -funded one year for a developer to work on the project full-time -(2014-2015), 50% of the time of Guillaume Lemaitre (2016-2017) and 50% of the -time of Joris van den Bossche (2017-2018). + `The University of Sydney `_ funded Joel Nothman + (2017-2021). -.. raw:: html + .. div:: image-box -
-
+ .. image:: images/sydney-primary.jpeg + :target: https://sydney.edu.au/ -.. image:: images/cds-logo.png - :width: 100pt - :align: center - :target: https://www.datascience-paris-saclay.fr/ +........... -.. raw:: html +.. div:: sk-text-image-grid-small -
-
+ .. div:: text-box -............ + Andreas MÃŧller received a grant to improve scikit-learn from the + `Alfred P. Sloan Foundation `_ . + This grant supported the position of Nicolas Hug and Thomas J. Fan. -.. raw:: html + .. div:: image-box -
-
+ .. image:: images/sloan_banner.png + :target: https://sloan.org/ -`Anaconda, Inc `_ funded Adrin Jalali in 2019. +............. -.. raw:: html +.. div:: sk-text-image-grid-small -
+ .. div:: text-box -
+ `INRIA `_ actively supports this project. It has + provided funding for Fabian Pedregosa (2010-2012), Jaques Grobler + (2012-2013) and Olivier Grisel (2013-2017) to work on this project + full-time. It also hosts coding sprints and other events. -.. image:: images/anaconda.png - :width: 100pt - :align: center - :target: https://www.anaconda.com/ + .. div:: image-box -.. raw:: html + .. image:: images/inria-logo.jpg + :target: https://www.inria.fr -
-
+..................... -.......................... +.. div:: sk-text-image-grid-small -.. raw:: html + .. div:: text-box -
-
+ `Paris-Saclay Center for Data Science `_ + funded one year for a developer to work on the project full-time (2014-2015), 50% + of the time of Guillaume Lemaitre (2016-2017) and 50% of the time of Joris van den + Bossche (2017-2018). -`NYU Moore-Sloan Data Science Environment `_ -funded Andreas Mueller (2014-2016) to work on this project. The Moore-Sloan -Data Science Environment also funds several students to work on the project -part-time. + .. div:: image-box -.. raw:: html + .. image:: images/cds-logo.png + :target: http://www.datascience-paris-saclay.fr/ -
-
+.......................... -.. image:: images/nyu_short_color.png - :width: 100pt - :align: center - :target: https://cds.nyu.edu/mooresloan/ +.. div:: sk-text-image-grid-small -.. raw:: html + .. div:: text-box + + `NYU Moore-Sloan Data Science Environment `_ + funded Andreas Mueller (2014-2016) to work on this project. The Moore-Sloan + Data Science Environment also funds several students to work on the project + part-time. + + .. div:: image-box -
-
+ .. image:: images/nyu_short_color.png + :target: https://cds.nyu.edu/mooresloan/ ........................ -.. raw:: html +.. div:: sk-text-image-grid-small -
-
+ .. div:: text-box -`TÊlÊcom Paristech `_ funded Manoj Kumar -(2014), Tom DuprÊ la Tour (2015), Raghav RV (2015-2017), Thierry Guillemot -(2016-2017) and Albert Thomas (2017) to work on scikit-learn. + `TÊlÊcom Paristech `_ funded Manoj Kumar + (2014), Tom DuprÊ la Tour (2015), Raghav RV (2015-2017), Thierry Guillemot + (2016-2017) and Albert Thomas (2017) to work on scikit-learn. -.. raw:: html + .. div:: image-box -
-
+ .. image:: images/telecom.png + :target: https://www.telecom-paristech.fr/ -.. image:: themes/scikit-learn/static/img/telecom.png - :width: 50pt - :align: center - :target: https://www.telecom-paristech.fr/ +..................... -.. raw:: html +.. div:: sk-text-image-grid-small -
-
+ .. div:: text-box -..................... + `The Labex DigiCosme `_ funded Nicolas Goix + (2015-2016), Tom DuprÊ la Tour (2015-2016 and 2017-2018), Mathurin Massias + (2018-2019) to work part time on scikit-learn during their PhDs. It also + funded a scikit-learn coding sprint in 2015. -.. raw:: html + .. div:: image-box -
-
+ .. image:: images/digicosme.png + :target: https://digicosme.lri.fr -`The Labex DigiCosme `_ funded Nicolas Goix -(2015-2016), Tom DuprÊ la Tour (2015-2016 and 2017-2018), Mathurin Massias -(2018-2019) to work part time on scikit-learn during their PhDs. It also -funded a scikit-learn coding sprint in 2015. +..................... -.. raw:: html +.. div:: sk-text-image-grid-small -
-
+ .. div:: text-box -.. image:: themes/scikit-learn/static/img/digicosme.png - :width: 100pt - :align: center - :target: https://digicosme.lri.fr + `The Chan-Zuckerberg Initiative `_ funded Nicolas + Hug to work full-time on scikit-learn in 2020. -.. raw:: html + .. div:: image-box -
-
+ .. image:: images/czi.png + :target: https://chanzuckerberg.com ...................... The following students were sponsored by `Google -`_ to work on scikit-learn through +`_ to work on scikit-learn through the `Google Summer of Code `_ program. - 2007 - David Cournapeau - 2011 - `Vlad Niculae`_ -- 2012 - `Vlad Niculae`_, Immanuel Bayer. +- 2012 - `Vlad Niculae`_, Immanuel Bayer - 2013 - Kemal Eren, Nicolas TrÊsegnie -- 2014 - Hamzeh Alsalhi, Issam Laradji, Maheshakya Wijewardena, Manoj Kumar. +- 2014 - Hamzeh Alsalhi, Issam Laradji, Maheshakya Wijewardena, Manoj Kumar - 2015 - `Raghav RV `_, Wei Xue -- 2016 - `Nelson Liu `_, `YenChen Lin `_ +- 2016 - `Nelson Liu `_, `YenChen Lin `_ .. _Vlad Niculae: https://vene.ro/ @@ -485,68 +517,116 @@ The `NeuroDebian `_ project providing `Debian `Dr. James V. Haxby `_ (`Dartmouth College `_). -Sprints -------- +................... -The International 2019 Paris sprint was kindly hosted by `AXA `_. -Also some participants could attend thanks to the support of the `Alfred P. -Sloan Foundation `_, the `Python Software -Foundation `_ (PSF) and the `DATAIA Institute -`_. +The following organizations funded the scikit-learn consortium at Inria in +the past: -..................... +.. |msn| image:: images/microsoft.png + :target: https://www.microsoft.com/ + +.. |bcg| image:: images/bcg.png + :target: https://www.bcg.com/beyond-consulting/bcg-gamma/default.aspx + +.. |fujitsu| image:: images/fujitsu.png + :target: https://www.fujitsu.com/global/ + +.. |aphp| image:: images/logo_APHP_text.png + :target: https://aphp.fr/ + +.. |hf| image:: images/huggingface_logo-noborder.png + :target: https://huggingface.co + +.. raw:: html + + + +.. grid:: 2 2 4 4 + :class-row: image-subgrid + :gutter: 1 -The 2013 International Paris Sprint was made possible thanks to the support of -`TÊlÊcom Paristech `_, `tinyclues -`_, the `French Python Association -`_ and the `Fonds de la Recherche Scientifique -`_. + .. grid-item:: + :class: sd-text-center + :child-align: center -.............. + |msn| -The 2011 International Granada sprint was made possible thanks to the support -of the `PSF `_ and `tinyclues -`_. + .. grid-item:: + :class: sd-text-center + :child-align: center + + |bcg| + + .. grid-item:: + :class: sd-text-center + :child-align: center + + |fujitsu| + + .. grid-item:: + :class: sd-text-center + :child-align: center + + |aphp| + + .. grid-item:: + :class: sd-text-center + :child-align: center + + |hf| + +Coding Sprints +============== + +The scikit-learn project has a long history of `open source coding sprints +`_ with over 50 sprint +events from 2010 to present day. There are scores of sponsors who contributed +to costs which include venue, food, travel, developer time and more. See +`scikit-learn sprints `_ for a full +list of events. Donating to the project -....................... +======================= If you are interested in donating to the project or to one of our code-sprints, -you can use the *Paypal* button below or the `NumFOCUS Donations Page -`_ (if you use the latter, -please indicate that you are donating for the scikit-learn project). - -All donations will be handled by `NumFOCUS -`_, a non-profit-organization which is -managed by a board of `Scipy community members -`_. NumFOCUS's mission is to foster -scientific computing software, in particular in Python. As a fiscal home -of scikit-learn, it ensures that money is available when needed to keep -the project funded and available while in compliance with tax regulations. - -The received donations for the scikit-learn project mostly will go towards -covering travel-expenses for code sprints, as well as towards the organization -budget of the project [#f1]_. - -.. raw :: html - -

- -
+please donate via the `NumFOCUS Donations Page +`_. + +.. raw:: html + +

+ + Help us, donate! + +

+ +All donations will be handled by `NumFOCUS `_, a non-profit +organization which is managed by a board of `Scipy community members +`_. NumFOCUS's mission is to foster scientific +computing software, in particular in Python. As a fiscal home of scikit-learn, it +ensures that money is available when needed to keep the project funded and available +while in compliance with tax regulations. + +The received donations for the scikit-learn project mostly will go towards covering +travel-expenses for code sprints, as well as towards the organization budget of the +project [#f1]_. .. rubric:: Notes .. [#f1] Regarding the organization budget, in particular, we might use some of - the donated funds to pay for other project expenses such as DNS, - hosting or continuous integration services. + the donated funds to pay for other project expenses such as DNS, + hosting or continuous integration services. + Infrastructure support ----------------------- +====================== -- We would also like to thank `Microsoft Azure - `_, `Travis Cl `_, - `CircleCl `_ for free CPU time on their Continuous - Integration servers, and `Anaconda Inc. `_ for the - storage they provide for our staging and nightly builds. +We would also like to thank `Microsoft Azure `_, +`CircleCl `_ for free CPU +time on their Continuous Integration servers, and `Anaconda Inc. `_ +for the storage they provide for our staging and nightly builds. diff --git a/doc/api/deprecated.rst.template b/doc/api/deprecated.rst.template new file mode 100644 index 0000000000000..a48f0180f76ed --- /dev/null +++ b/doc/api/deprecated.rst.template @@ -0,0 +1,24 @@ +:html_theme.sidebar_secondary.remove: + +.. _api_depr_ref: + +Recently Deprecated +=================== + +.. currentmodule:: sklearn + +{% for ver, objs in DEPRECATED_API_REFERENCE %} +.. _api_depr_ref-{{ ver|replace(".", "-") }}: + +.. rubric:: To be removed in {{ ver }} + +.. autosummary:: + :nosignatures: + :toctree: ../modules/generated/ + :template: base.rst + +{% for obj in objs %} + {{ obj }} +{%- endfor %} + +{% endfor %} diff --git a/doc/api/index.rst.template b/doc/api/index.rst.template new file mode 100644 index 0000000000000..b0a3698775a94 --- /dev/null +++ b/doc/api/index.rst.template @@ -0,0 +1,77 @@ +:html_theme.sidebar_secondary.remove: + +.. _api_ref: + +============= +API Reference +============= + +This is the class and function reference of scikit-learn. Please refer to the +:ref:`full user guide ` for further details, as the raw specifications of +classes and functions may not be enough to give full guidelines on their use. For +reference on concepts repeated across the API, see :ref:`glossary`. + +.. toctree:: + :maxdepth: 2 + :hidden: + +{% for module, _ in API_REFERENCE %} + {{ module }} +{%- endfor %} +{%- if DEPRECATED_API_REFERENCE %} + deprecated +{%- endif %} + +.. list-table:: + :header-rows: 1 + :class: apisearch-table + + * - Object + - Description + +{% for module, module_info in API_REFERENCE %} +{% for section in module_info["sections"] %} +{% for obj in section["autosummary"] %} +{% set parts = obj.rsplit(".", 1) %} +{% if parts|length > 1 %} +{% set full_module = module + "." + parts[0] %} +{% else %} +{% set full_module = module %} +{% endif %} + * - :obj:`~{{ module }}.{{ obj }}` + + - .. div:: sk-apisearch-desc + + .. currentmodule:: {{ full_module }} + + .. autoshortsummary:: {{ module }}.{{ obj }} + + .. div:: caption + + :mod:`{{ full_module }}` +{% endfor %} +{% endfor %} +{% endfor %} + +{% for ver, objs in DEPRECATED_API_REFERENCE %} +{% for obj in objs %} +{% set parts = obj.rsplit(".", 1) %} +{% if parts|length > 1 %} +{% set full_module = "sklearn." + parts[0] %} +{% else %} +{% set full_module = "sklearn" %} +{% endif %} + * - :obj:`~sklearn.{{ obj }}` + + - .. div:: sk-apisearch-desc + + .. currentmodule:: {{ full_module }} + + .. autoshortsummary:: sklearn.{{ obj }} + + .. div:: caption + + :mod:`{{ full_module }}` + :bdg-ref-danger-line:`Deprecated in version {{ ver }} ` +{% endfor %} +{% endfor %} diff --git a/doc/api/module.rst.template b/doc/api/module.rst.template new file mode 100644 index 0000000000000..1980f27aad158 --- /dev/null +++ b/doc/api/module.rst.template @@ -0,0 +1,46 @@ +:html_theme.sidebar_secondary.remove: + +{% if module == "sklearn" -%} +{%- set module_hook = "sklearn" -%} +{%- elif module.startswith("sklearn.") -%} +{%- set module_hook = module[8:] -%} +{%- else -%} +{%- set module_hook = None -%} +{%- endif -%} + +{% if module_hook %} +.. _{{ module_hook }}_ref: +{% endif %} + +{{ module }} +{{ "=" * module|length }} + +.. automodule:: {{ module }} + +{% if module_info["description"] %} +{{ module_info["description"] }} +{% endif %} + +{% for section in module_info["sections"] %} +{% if section["title"] and module_hook %} +.. _{{ module_hook }}_ref-{{ section["title"]|lower|replace(" ", "-") }}: +{% endif %} + +{% if section["title"] %} +{{ section["title"] }} +{{ "-" * section["title"]|length }} +{% endif %} + +{% if section["description"] %} +{{ section["description"] }} +{% endif %} + +.. autosummary:: + :nosignatures: + :toctree: ../modules/generated/ + :template: base.rst + +{% for obj in section["autosummary"] %} + {{ obj }} +{%- endfor %} +{% endfor %} diff --git a/doc/api_reference.py b/doc/api_reference.py new file mode 100644 index 0000000000000..c90b115746415 --- /dev/null +++ b/doc/api_reference.py @@ -0,0 +1,1352 @@ +"""Configuration for the API reference documentation.""" + + +def _get_guide(*refs, is_developer=False): + """Get the rst to refer to user/developer guide. + + `refs` is several references that can be used in the :ref:`...` directive. + """ + if len(refs) == 1: + ref_desc = f":ref:`{refs[0]}` section" + elif len(refs) == 2: + ref_desc = f":ref:`{refs[0]}` and :ref:`{refs[1]}` sections" + else: + ref_desc = ", ".join(f":ref:`{ref}`" for ref in refs[:-1]) + ref_desc += f", and :ref:`{refs[-1]}` sections" + + guide_name = "Developer" if is_developer else "User" + return f"**{guide_name} guide.** See the {ref_desc} for further details." + + +def _get_submodule(module_name, submodule_name): + """Get the submodule docstring and automatically add the hook. + + `module_name` is e.g. `sklearn.feature_extraction`, and `submodule_name` is e.g. + `image`, so we get the docstring and hook for `sklearn.feature_extraction.image` + submodule. `module_name` is used to reset the current module because autosummary + automatically changes the current module. + """ + lines = [ + f".. automodule:: {module_name}.{submodule_name}", + f".. currentmodule:: {module_name}", + ] + return "\n\n".join(lines) + + +""" +CONFIGURING API_REFERENCE +========================= + +API_REFERENCE maps each module name to a dictionary that consists of the following +components: + +short_summary (required) + The text to be printed on the index page; it has nothing to do the API reference + page of each module. +description (required, `None` if not needed) + The additional description for the module to be placed under the module + docstring, before the sections start. +sections (required) + A list of sections, each of which consists of: + - title (required, `None` if not needed): the section title, commonly it should + not be `None` except for the first section of a module, + - description (optional): the optional additional description for the section, + - autosummary (required): an autosummary block, assuming current module is the + current module name. + +Essentially, the rendered page would look like the following: + +|---------------------------------------------------------------------------------| +| {{ module_name }} | +| ================= | +| {{ module_docstring }} | +| {{ description }} | +| | +| {{ section_title_1 }} <-------------- Optional if one wants the first | +| --------------------- section to directly follow | +| {{ section_description_1 }} without a second-level heading. | +| {{ section_autosummary_1 }} | +| | +| {{ section_title_2 }} | +| --------------------- | +| {{ section_description_2 }} | +| {{ section_autosummary_2 }} | +| | +| More sections... | +|---------------------------------------------------------------------------------| + +Hooks will be automatically generated for each module and each section. For a module, +e.g., `sklearn.feature_extraction`, the hook would be `feature_extraction_ref`; for a +section, e.g., "From text" under `sklearn.feature_extraction`, the hook would be +`feature_extraction_ref-from-text`. However, note that a better way is to refer using +the :mod: directive, e.g., :mod:`sklearn.feature_extraction` for the module and +:mod:`sklearn.feature_extraction.text` for the section. Only in case that a section +is not a particular submodule does the hook become useful, e.g., the "Loaders" section +under `sklearn.datasets`. +""" + +API_REFERENCE = { + "sklearn": { + "short_summary": "Settings and information tools.", + "description": None, + "sections": [ + { + "title": None, + "autosummary": [ + "config_context", + "get_config", + "set_config", + "show_versions", + ], + }, + ], + }, + "sklearn.base": { + "short_summary": "Base classes and utility functions.", + "description": None, + "sections": [ + { + "title": None, + "autosummary": [ + "BaseEstimator", + "BiclusterMixin", + "ClassNamePrefixFeaturesOutMixin", + "ClassifierMixin", + "ClusterMixin", + "DensityMixin", + "MetaEstimatorMixin", + "OneToOneFeatureMixin", + "OutlierMixin", + "RegressorMixin", + "TransformerMixin", + "clone", + "is_classifier", + "is_clusterer", + "is_regressor", + "is_outlier_detector", + ], + } + ], + }, + "sklearn.calibration": { + "short_summary": "Probability calibration.", + "description": _get_guide("calibration"), + "sections": [ + { + "title": None, + "autosummary": ["CalibratedClassifierCV", "calibration_curve"], + }, + { + "title": "Visualization", + "autosummary": ["CalibrationDisplay"], + }, + ], + }, + "sklearn.cluster": { + "short_summary": "Clustering.", + "description": _get_guide("clustering", "biclustering"), + "sections": [ + { + "title": None, + "autosummary": [ + "AffinityPropagation", + "AgglomerativeClustering", + "Birch", + "BisectingKMeans", + "DBSCAN", + "FeatureAgglomeration", + "HDBSCAN", + "KMeans", + "MeanShift", + "MiniBatchKMeans", + "OPTICS", + "SpectralBiclustering", + "SpectralClustering", + "SpectralCoclustering", + "affinity_propagation", + "cluster_optics_dbscan", + "cluster_optics_xi", + "compute_optics_graph", + "dbscan", + "estimate_bandwidth", + "k_means", + "kmeans_plusplus", + "mean_shift", + "spectral_clustering", + "ward_tree", + ], + }, + ], + }, + "sklearn.compose": { + "short_summary": "Composite estimators.", + "description": _get_guide("combining_estimators"), + "sections": [ + { + "title": None, + "autosummary": [ + "ColumnTransformer", + "TransformedTargetRegressor", + "make_column_selector", + "make_column_transformer", + ], + }, + ], + }, + "sklearn.covariance": { + "short_summary": "Covariance estimation.", + "description": _get_guide("covariance"), + "sections": [ + { + "title": None, + "autosummary": [ + "EllipticEnvelope", + "EmpiricalCovariance", + "GraphicalLasso", + "GraphicalLassoCV", + "LedoitWolf", + "MinCovDet", + "OAS", + "ShrunkCovariance", + "empirical_covariance", + "graphical_lasso", + "ledoit_wolf", + "ledoit_wolf_shrinkage", + "oas", + "shrunk_covariance", + ], + }, + ], + }, + "sklearn.cross_decomposition": { + "short_summary": "Cross decomposition.", + "description": _get_guide("cross_decomposition"), + "sections": [ + { + "title": None, + "autosummary": ["CCA", "PLSCanonical", "PLSRegression", "PLSSVD"], + }, + ], + }, + "sklearn.datasets": { + "short_summary": "Datasets.", + "description": _get_guide("datasets"), + "sections": [ + { + "title": "Loaders", + "autosummary": [ + "clear_data_home", + "dump_svmlight_file", + "fetch_20newsgroups", + "fetch_20newsgroups_vectorized", + "fetch_california_housing", + "fetch_covtype", + "fetch_file", + "fetch_kddcup99", + "fetch_lfw_pairs", + "fetch_lfw_people", + "fetch_olivetti_faces", + "fetch_openml", + "fetch_rcv1", + "fetch_species_distributions", + "get_data_home", + "load_breast_cancer", + "load_diabetes", + "load_digits", + "load_files", + "load_iris", + "load_linnerud", + "load_sample_image", + "load_sample_images", + "load_svmlight_file", + "load_svmlight_files", + "load_wine", + ], + }, + { + "title": "Sample generators", + "autosummary": [ + "make_biclusters", + "make_blobs", + "make_checkerboard", + "make_circles", + "make_classification", + "make_friedman1", + "make_friedman2", + "make_friedman3", + "make_gaussian_quantiles", + "make_hastie_10_2", + "make_low_rank_matrix", + "make_moons", + "make_multilabel_classification", + "make_regression", + "make_s_curve", + "make_sparse_coded_signal", + "make_sparse_spd_matrix", + "make_sparse_uncorrelated", + "make_spd_matrix", + "make_swiss_roll", + ], + }, + ], + }, + "sklearn.decomposition": { + "short_summary": "Matrix decomposition.", + "description": _get_guide("decompositions"), + "sections": [ + { + "title": None, + "autosummary": [ + "DictionaryLearning", + "FactorAnalysis", + "FastICA", + "IncrementalPCA", + "KernelPCA", + "LatentDirichletAllocation", + "MiniBatchDictionaryLearning", + "MiniBatchNMF", + "MiniBatchSparsePCA", + "NMF", + "PCA", + "SparseCoder", + "SparsePCA", + "TruncatedSVD", + "dict_learning", + "dict_learning_online", + "fastica", + "non_negative_factorization", + "sparse_encode", + ], + }, + ], + }, + "sklearn.discriminant_analysis": { + "short_summary": "Discriminant analysis.", + "description": _get_guide("lda_qda"), + "sections": [ + { + "title": None, + "autosummary": [ + "LinearDiscriminantAnalysis", + "QuadraticDiscriminantAnalysis", + ], + }, + ], + }, + "sklearn.dummy": { + "short_summary": "Dummy estimators.", + "description": _get_guide("model_evaluation"), + "sections": [ + { + "title": None, + "autosummary": ["DummyClassifier", "DummyRegressor"], + }, + ], + }, + "sklearn.ensemble": { + "short_summary": "Ensemble methods.", + "description": _get_guide("ensemble"), + "sections": [ + { + "title": None, + "autosummary": [ + "AdaBoostClassifier", + "AdaBoostRegressor", + "BaggingClassifier", + "BaggingRegressor", + "ExtraTreesClassifier", + "ExtraTreesRegressor", + "GradientBoostingClassifier", + "GradientBoostingRegressor", + "HistGradientBoostingClassifier", + "HistGradientBoostingRegressor", + "IsolationForest", + "RandomForestClassifier", + "RandomForestRegressor", + "RandomTreesEmbedding", + "StackingClassifier", + "StackingRegressor", + "VotingClassifier", + "VotingRegressor", + ], + }, + ], + }, + "sklearn.exceptions": { + "short_summary": "Exceptions and warnings.", + "description": None, + "sections": [ + { + "title": None, + "autosummary": [ + "ConvergenceWarning", + "DataConversionWarning", + "DataDimensionalityWarning", + "EfficiencyWarning", + "FitFailedWarning", + "InconsistentVersionWarning", + "NotFittedError", + "UndefinedMetricWarning", + "EstimatorCheckFailedWarning", + ], + }, + ], + }, + "sklearn.experimental": { + "short_summary": "Experimental tools.", + "description": None, + "sections": [ + { + "title": None, + "autosummary": ["enable_halving_search_cv", "enable_iterative_imputer"], + }, + ], + }, + "sklearn.feature_extraction": { + "short_summary": "Feature extraction.", + "description": _get_guide("feature_extraction"), + "sections": [ + { + "title": None, + "autosummary": ["DictVectorizer", "FeatureHasher"], + }, + { + "title": "From images", + "description": _get_submodule("sklearn.feature_extraction", "image"), + "autosummary": [ + "image.PatchExtractor", + "image.extract_patches_2d", + "image.grid_to_graph", + "image.img_to_graph", + "image.reconstruct_from_patches_2d", + ], + }, + { + "title": "From text", + "description": _get_submodule("sklearn.feature_extraction", "text"), + "autosummary": [ + "text.CountVectorizer", + "text.HashingVectorizer", + "text.TfidfTransformer", + "text.TfidfVectorizer", + ], + }, + ], + }, + "sklearn.feature_selection": { + "short_summary": "Feature selection.", + "description": _get_guide("feature_selection"), + "sections": [ + { + "title": None, + "autosummary": [ + "GenericUnivariateSelect", + "RFE", + "RFECV", + "SelectFdr", + "SelectFpr", + "SelectFromModel", + "SelectFwe", + "SelectKBest", + "SelectPercentile", + "SelectorMixin", + "SequentialFeatureSelector", + "VarianceThreshold", + "chi2", + "f_classif", + "f_regression", + "mutual_info_classif", + "mutual_info_regression", + "r_regression", + ], + }, + ], + }, + "sklearn.frozen": { + "short_summary": "Frozen estimators.", + "description": None, + "sections": [ + { + "title": None, + "autosummary": ["FrozenEstimator"], + }, + ], + }, + "sklearn.gaussian_process": { + "short_summary": "Gaussian processes.", + "description": _get_guide("gaussian_process"), + "sections": [ + { + "title": None, + "autosummary": [ + "GaussianProcessClassifier", + "GaussianProcessRegressor", + ], + }, + { + "title": "Kernels", + "description": _get_submodule("sklearn.gaussian_process", "kernels"), + "autosummary": [ + "kernels.CompoundKernel", + "kernels.ConstantKernel", + "kernels.DotProduct", + "kernels.ExpSineSquared", + "kernels.Exponentiation", + "kernels.Hyperparameter", + "kernels.Kernel", + "kernels.Matern", + "kernels.PairwiseKernel", + "kernels.Product", + "kernels.RBF", + "kernels.RationalQuadratic", + "kernels.Sum", + "kernels.WhiteKernel", + ], + }, + ], + }, + "sklearn.impute": { + "short_summary": "Imputation.", + "description": _get_guide("impute"), + "sections": [ + { + "title": None, + "autosummary": [ + "IterativeImputer", + "KNNImputer", + "MissingIndicator", + "SimpleImputer", + ], + }, + ], + }, + "sklearn.inspection": { + "short_summary": "Inspection.", + "description": _get_guide("inspection"), + "sections": [ + { + "title": None, + "autosummary": ["partial_dependence", "permutation_importance"], + }, + { + "title": "Plotting", + "autosummary": ["DecisionBoundaryDisplay", "PartialDependenceDisplay"], + }, + ], + }, + "sklearn.isotonic": { + "short_summary": "Isotonic regression.", + "description": _get_guide("isotonic"), + "sections": [ + { + "title": None, + "autosummary": [ + "IsotonicRegression", + "check_increasing", + "isotonic_regression", + ], + }, + ], + }, + "sklearn.kernel_approximation": { + "short_summary": "Kernel approximation.", + "description": _get_guide("kernel_approximation"), + "sections": [ + { + "title": None, + "autosummary": [ + "AdditiveChi2Sampler", + "Nystroem", + "PolynomialCountSketch", + "RBFSampler", + "SkewedChi2Sampler", + ], + }, + ], + }, + "sklearn.kernel_ridge": { + "short_summary": "Kernel ridge regression.", + "description": _get_guide("kernel_ridge"), + "sections": [ + { + "title": None, + "autosummary": ["KernelRidge"], + }, + ], + }, + "sklearn.linear_model": { + "short_summary": "Generalized linear models.", + "description": ( + _get_guide("linear_model") + + "\n\nThe following subsections are only rough guidelines: the same " + "estimator can fall into multiple categories, depending on its parameters." + ), + "sections": [ + { + "title": "Linear classifiers", + "autosummary": [ + "LogisticRegression", + "LogisticRegressionCV", + "PassiveAggressiveClassifier", + "Perceptron", + "RidgeClassifier", + "RidgeClassifierCV", + "SGDClassifier", + "SGDOneClassSVM", + ], + }, + { + "title": "Classical linear regressors", + "autosummary": ["LinearRegression", "Ridge", "RidgeCV", "SGDRegressor"], + }, + { + "title": "Regressors with variable selection", + "description": ( + "The following estimators have built-in variable selection fitting " + "procedures, but any estimator using a L1 or elastic-net penalty " + "also performs variable selection: typically " + ":class:`~linear_model.SGDRegressor` or " + ":class:`~sklearn.linear_model.SGDClassifier` with an appropriate " + "penalty." + ), + "autosummary": [ + "ElasticNet", + "ElasticNetCV", + "Lars", + "LarsCV", + "Lasso", + "LassoCV", + "LassoLars", + "LassoLarsCV", + "LassoLarsIC", + "OrthogonalMatchingPursuit", + "OrthogonalMatchingPursuitCV", + ], + }, + { + "title": "Bayesian regressors", + "autosummary": ["ARDRegression", "BayesianRidge"], + }, + { + "title": "Multi-task linear regressors with variable selection", + "description": ( + "These estimators fit multiple regression problems (or tasks)" + " jointly, while inducing sparse coefficients. While the inferred" + " coefficients may differ between the tasks, they are constrained" + " to agree on the features that are selected (non-zero" + " coefficients)." + ), + "autosummary": [ + "MultiTaskElasticNet", + "MultiTaskElasticNetCV", + "MultiTaskLasso", + "MultiTaskLassoCV", + ], + }, + { + "title": "Outlier-robust regressors", + "description": ( + "Any estimator using the Huber loss would also be robust to " + "outliers, e.g., :class:`~linear_model.SGDRegressor` with " + "``loss='huber'``." + ), + "autosummary": [ + "HuberRegressor", + "QuantileRegressor", + "RANSACRegressor", + "TheilSenRegressor", + ], + }, + { + "title": "Generalized linear models (GLM) for regression", + "description": ( + "These models allow for response variables to have error " + "distributions other than a normal distribution." + ), + "autosummary": [ + "GammaRegressor", + "PoissonRegressor", + "TweedieRegressor", + ], + }, + { + "title": "Miscellaneous", + "autosummary": [ + "PassiveAggressiveRegressor", + "enet_path", + "lars_path", + "lars_path_gram", + "lasso_path", + "orthogonal_mp", + "orthogonal_mp_gram", + "ridge_regression", + ], + }, + ], + }, + "sklearn.manifold": { + "short_summary": "Manifold learning.", + "description": _get_guide("manifold"), + "sections": [ + { + "title": None, + "autosummary": [ + "Isomap", + "LocallyLinearEmbedding", + "MDS", + "SpectralEmbedding", + "TSNE", + "locally_linear_embedding", + "smacof", + "spectral_embedding", + "trustworthiness", + ], + }, + ], + }, + "sklearn.metrics": { + "short_summary": "Metrics.", + "description": _get_guide("model_evaluation", "metrics"), + "sections": [ + { + "title": "Model selection interface", + "description": _get_guide("scoring_parameter"), + "autosummary": [ + "check_scoring", + "get_scorer", + "get_scorer_names", + "make_scorer", + ], + }, + { + "title": "Classification metrics", + "description": _get_guide("classification_metrics"), + "autosummary": [ + "accuracy_score", + "auc", + "average_precision_score", + "balanced_accuracy_score", + "brier_score_loss", + "class_likelihood_ratios", + "classification_report", + "cohen_kappa_score", + "confusion_matrix", + "d2_log_loss_score", + "dcg_score", + "det_curve", + "f1_score", + "fbeta_score", + "hamming_loss", + "hinge_loss", + "jaccard_score", + "log_loss", + "matthews_corrcoef", + "multilabel_confusion_matrix", + "ndcg_score", + "precision_recall_curve", + "precision_recall_fscore_support", + "precision_score", + "recall_score", + "roc_auc_score", + "roc_curve", + "top_k_accuracy_score", + "zero_one_loss", + ], + }, + { + "title": "Regression metrics", + "description": _get_guide("regression_metrics"), + "autosummary": [ + "d2_absolute_error_score", + "d2_pinball_score", + "d2_tweedie_score", + "explained_variance_score", + "max_error", + "mean_absolute_error", + "mean_absolute_percentage_error", + "mean_gamma_deviance", + "mean_pinball_loss", + "mean_poisson_deviance", + "mean_squared_error", + "mean_squared_log_error", + "mean_tweedie_deviance", + "median_absolute_error", + "r2_score", + "root_mean_squared_error", + "root_mean_squared_log_error", + ], + }, + { + "title": "Multilabel ranking metrics", + "description": _get_guide("multilabel_ranking_metrics"), + "autosummary": [ + "coverage_error", + "label_ranking_average_precision_score", + "label_ranking_loss", + ], + }, + { + "title": "Clustering metrics", + "description": ( + _get_submodule("sklearn.metrics", "cluster") + + "\n\n" + + _get_guide("clustering_evaluation") + ), + "autosummary": [ + "adjusted_mutual_info_score", + "adjusted_rand_score", + "calinski_harabasz_score", + "cluster.contingency_matrix", + "cluster.pair_confusion_matrix", + "completeness_score", + "davies_bouldin_score", + "fowlkes_mallows_score", + "homogeneity_completeness_v_measure", + "homogeneity_score", + "mutual_info_score", + "normalized_mutual_info_score", + "rand_score", + "silhouette_samples", + "silhouette_score", + "v_measure_score", + ], + }, + { + "title": "Biclustering metrics", + "description": _get_guide("biclustering_evaluation"), + "autosummary": ["consensus_score"], + }, + { + "title": "Distance metrics", + "autosummary": ["DistanceMetric"], + }, + { + "title": "Pairwise metrics", + "description": ( + _get_submodule("sklearn.metrics", "pairwise") + + "\n\n" + + _get_guide("metrics") + ), + "autosummary": [ + "pairwise.additive_chi2_kernel", + "pairwise.chi2_kernel", + "pairwise.cosine_distances", + "pairwise.cosine_similarity", + "pairwise.distance_metrics", + "pairwise.euclidean_distances", + "pairwise.haversine_distances", + "pairwise.kernel_metrics", + "pairwise.laplacian_kernel", + "pairwise.linear_kernel", + "pairwise.manhattan_distances", + "pairwise.nan_euclidean_distances", + "pairwise.paired_cosine_distances", + "pairwise.paired_distances", + "pairwise.paired_euclidean_distances", + "pairwise.paired_manhattan_distances", + "pairwise.pairwise_kernels", + "pairwise.polynomial_kernel", + "pairwise.rbf_kernel", + "pairwise.sigmoid_kernel", + "pairwise_distances", + "pairwise_distances_argmin", + "pairwise_distances_argmin_min", + "pairwise_distances_chunked", + ], + }, + { + "title": "Plotting", + "description": _get_guide("visualizations"), + "autosummary": [ + "ConfusionMatrixDisplay", + "DetCurveDisplay", + "PrecisionRecallDisplay", + "PredictionErrorDisplay", + "RocCurveDisplay", + ], + }, + ], + }, + "sklearn.mixture": { + "short_summary": "Gaussian mixture models.", + "description": _get_guide("mixture"), + "sections": [ + { + "title": None, + "autosummary": ["BayesianGaussianMixture", "GaussianMixture"], + }, + ], + }, + "sklearn.model_selection": { + "short_summary": "Model selection.", + "description": _get_guide("cross_validation", "grid_search", "learning_curve"), + "sections": [ + { + "title": "Splitters", + "autosummary": [ + "GroupKFold", + "GroupShuffleSplit", + "KFold", + "LeaveOneGroupOut", + "LeaveOneOut", + "LeavePGroupsOut", + "LeavePOut", + "PredefinedSplit", + "RepeatedKFold", + "RepeatedStratifiedKFold", + "ShuffleSplit", + "StratifiedGroupKFold", + "StratifiedKFold", + "StratifiedShuffleSplit", + "TimeSeriesSplit", + "check_cv", + "train_test_split", + ], + }, + { + "title": "Hyper-parameter optimizers", + "autosummary": [ + "GridSearchCV", + "HalvingGridSearchCV", + "HalvingRandomSearchCV", + "ParameterGrid", + "ParameterSampler", + "RandomizedSearchCV", + ], + }, + { + "title": "Post-fit model tuning", + "autosummary": [ + "FixedThresholdClassifier", + "TunedThresholdClassifierCV", + ], + }, + { + "title": "Model validation", + "autosummary": [ + "cross_val_predict", + "cross_val_score", + "cross_validate", + "learning_curve", + "permutation_test_score", + "validation_curve", + ], + }, + { + "title": "Visualization", + "autosummary": ["LearningCurveDisplay", "ValidationCurveDisplay"], + }, + ], + }, + "sklearn.multiclass": { + "short_summary": "Multiclass classification.", + "description": _get_guide("multiclass_classification"), + "sections": [ + { + "title": None, + "autosummary": [ + "OneVsOneClassifier", + "OneVsRestClassifier", + "OutputCodeClassifier", + ], + }, + ], + }, + "sklearn.multioutput": { + "short_summary": "Multioutput regression and classification.", + "description": _get_guide( + "multilabel_classification", + "multiclass_multioutput_classification", + "multioutput_regression", + ), + "sections": [ + { + "title": None, + "autosummary": [ + "ClassifierChain", + "MultiOutputClassifier", + "MultiOutputRegressor", + "RegressorChain", + ], + }, + ], + }, + "sklearn.naive_bayes": { + "short_summary": "Naive Bayes.", + "description": _get_guide("naive_bayes"), + "sections": [ + { + "title": None, + "autosummary": [ + "BernoulliNB", + "CategoricalNB", + "ComplementNB", + "GaussianNB", + "MultinomialNB", + ], + }, + ], + }, + "sklearn.neighbors": { + "short_summary": "Nearest neighbors.", + "description": _get_guide("neighbors"), + "sections": [ + { + "title": None, + "autosummary": [ + "BallTree", + "KDTree", + "KNeighborsClassifier", + "KNeighborsRegressor", + "KNeighborsTransformer", + "KernelDensity", + "LocalOutlierFactor", + "NearestCentroid", + "NearestNeighbors", + "NeighborhoodComponentsAnalysis", + "RadiusNeighborsClassifier", + "RadiusNeighborsRegressor", + "RadiusNeighborsTransformer", + "kneighbors_graph", + "radius_neighbors_graph", + "sort_graph_by_row_values", + ], + }, + ], + }, + "sklearn.neural_network": { + "short_summary": "Neural network models.", + "description": _get_guide( + "neural_networks_supervised", "neural_networks_unsupervised" + ), + "sections": [ + { + "title": None, + "autosummary": ["BernoulliRBM", "MLPClassifier", "MLPRegressor"], + }, + ], + }, + "sklearn.pipeline": { + "short_summary": "Pipeline.", + "description": _get_guide("combining_estimators"), + "sections": [ + { + "title": None, + "autosummary": [ + "FeatureUnion", + "Pipeline", + "make_pipeline", + "make_union", + ], + }, + ], + }, + "sklearn.preprocessing": { + "short_summary": "Preprocessing and normalization.", + "description": _get_guide("preprocessing"), + "sections": [ + { + "title": None, + "autosummary": [ + "Binarizer", + "FunctionTransformer", + "KBinsDiscretizer", + "KernelCenterer", + "LabelBinarizer", + "LabelEncoder", + "MaxAbsScaler", + "MinMaxScaler", + "MultiLabelBinarizer", + "Normalizer", + "OneHotEncoder", + "OrdinalEncoder", + "PolynomialFeatures", + "PowerTransformer", + "QuantileTransformer", + "RobustScaler", + "SplineTransformer", + "StandardScaler", + "TargetEncoder", + "add_dummy_feature", + "binarize", + "label_binarize", + "maxabs_scale", + "minmax_scale", + "normalize", + "power_transform", + "quantile_transform", + "robust_scale", + "scale", + ], + }, + ], + }, + "sklearn.random_projection": { + "short_summary": "Random projection.", + "description": _get_guide("random_projection"), + "sections": [ + { + "title": None, + "autosummary": [ + "GaussianRandomProjection", + "SparseRandomProjection", + "johnson_lindenstrauss_min_dim", + ], + }, + ], + }, + "sklearn.semi_supervised": { + "short_summary": "Semi-supervised learning.", + "description": _get_guide("semi_supervised"), + "sections": [ + { + "title": None, + "autosummary": [ + "LabelPropagation", + "LabelSpreading", + "SelfTrainingClassifier", + ], + }, + ], + }, + "sklearn.svm": { + "short_summary": "Support vector machines.", + "description": _get_guide("svm"), + "sections": [ + { + "title": None, + "autosummary": [ + "LinearSVC", + "LinearSVR", + "NuSVC", + "NuSVR", + "OneClassSVM", + "SVC", + "SVR", + "l1_min_c", + ], + }, + ], + }, + "sklearn.tree": { + "short_summary": "Decision trees.", + "description": _get_guide("tree"), + "sections": [ + { + "title": None, + "autosummary": [ + "DecisionTreeClassifier", + "DecisionTreeRegressor", + "ExtraTreeClassifier", + "ExtraTreeRegressor", + ], + }, + { + "title": "Exporting", + "autosummary": ["export_graphviz", "export_text"], + }, + { + "title": "Plotting", + "autosummary": ["plot_tree"], + }, + ], + }, + "sklearn.utils": { + "short_summary": "Utilities.", + "description": _get_guide("developers-utils", is_developer=True), + "sections": [ + { + "title": None, + "autosummary": [ + "Bunch", + "_safe_indexing", + "as_float_array", + "assert_all_finite", + "deprecated", + "estimator_html_repr", + "gen_batches", + "gen_even_slices", + "indexable", + "murmurhash3_32", + "resample", + "safe_mask", + "safe_sqr", + "shuffle", + "Tags", + "InputTags", + "TargetTags", + "ClassifierTags", + "RegressorTags", + "TransformerTags", + "get_tags", + ], + }, + { + "title": "Input and parameter validation", + "description": _get_submodule("sklearn.utils", "validation"), + "autosummary": [ + "check_X_y", + "check_array", + "check_consistent_length", + "check_random_state", + "check_scalar", + "validation.check_is_fitted", + "validation.check_memory", + "validation.check_symmetric", + "validation.column_or_1d", + "validation.has_fit_parameter", + "validation.validate_data", + ], + }, + { + "title": "Meta-estimators", + "description": _get_submodule("sklearn.utils", "metaestimators"), + "autosummary": ["metaestimators.available_if"], + }, + { + "title": "Weight handling based on class labels", + "description": _get_submodule("sklearn.utils", "class_weight"), + "autosummary": [ + "class_weight.compute_class_weight", + "class_weight.compute_sample_weight", + ], + }, + { + "title": "Dealing with multiclass target in classifiers", + "description": _get_submodule("sklearn.utils", "multiclass"), + "autosummary": [ + "multiclass.is_multilabel", + "multiclass.type_of_target", + "multiclass.unique_labels", + ], + }, + { + "title": "Optimal mathematical operations", + "description": _get_submodule("sklearn.utils", "extmath"), + "autosummary": [ + "extmath.density", + "extmath.fast_logdet", + "extmath.randomized_range_finder", + "extmath.randomized_svd", + "extmath.safe_sparse_dot", + "extmath.weighted_mode", + ], + }, + { + "title": "Working with sparse matrices and arrays", + "description": _get_submodule("sklearn.utils", "sparsefuncs"), + "autosummary": [ + "sparsefuncs.incr_mean_variance_axis", + "sparsefuncs.inplace_column_scale", + "sparsefuncs.inplace_csr_column_scale", + "sparsefuncs.inplace_row_scale", + "sparsefuncs.inplace_swap_column", + "sparsefuncs.inplace_swap_row", + "sparsefuncs.mean_variance_axis", + ], + }, + { + "title": None, + "description": _get_submodule("sklearn.utils", "sparsefuncs_fast"), + "autosummary": [ + "sparsefuncs_fast.inplace_csr_row_normalize_l1", + "sparsefuncs_fast.inplace_csr_row_normalize_l2", + ], + }, + { + "title": "Working with graphs", + "description": _get_submodule("sklearn.utils", "graph"), + "autosummary": ["graph.single_source_shortest_path_length"], + }, + { + "title": "Random sampling", + "description": _get_submodule("sklearn.utils", "random"), + "autosummary": ["random.sample_without_replacement"], + }, + { + "title": "Auxiliary functions that operate on arrays", + "description": _get_submodule("sklearn.utils", "arrayfuncs"), + "autosummary": ["arrayfuncs.min_pos"], + }, + { + "title": "Metadata routing", + "description": ( + _get_submodule("sklearn.utils", "metadata_routing") + + "\n\n" + + _get_guide("metadata_routing") + ), + "autosummary": [ + "metadata_routing.MetadataRequest", + "metadata_routing.MetadataRouter", + "metadata_routing.MethodMapping", + "metadata_routing.get_routing_for_object", + "metadata_routing.process_routing", + ], + }, + { + "title": "Discovering scikit-learn objects", + "description": _get_submodule("sklearn.utils", "discovery"), + "autosummary": [ + "discovery.all_displays", + "discovery.all_estimators", + "discovery.all_functions", + ], + }, + { + "title": "API compatibility checkers", + "description": _get_submodule("sklearn.utils", "estimator_checks"), + "autosummary": [ + "estimator_checks.check_estimator", + "estimator_checks.parametrize_with_checks", + "estimator_checks.estimator_checks_generator", + ], + }, + { + "title": "Parallel computing", + "description": _get_submodule("sklearn.utils", "parallel"), + "autosummary": [ + "parallel.Parallel", + "parallel.delayed", + ], + }, + ], + }, +} + + +""" +CONFIGURING DEPRECATED_API_REFERENCE +==================================== + +DEPRECATED_API_REFERENCE maps each deprecation target version to a corresponding +autosummary block. It will be placed at the bottom of the API index page under the +"Recently deprecated" section. Essentially, the rendered section would look like the +following: + +|------------------------------------------| +| To be removed in {{ version_1 }} | +| -------------------------------- | +| {{ autosummary_1 }} | +| | +| To be removed in {{ version_2 }} | +| -------------------------------- | +| {{ autosummary_2 }} | +| | +| More versions... | +|------------------------------------------| + +Note that the autosummary here assumes that the current module is `sklearn`, i.e., if +`sklearn.utils.Memory` is deprecated, one should put `utils.Memory` in the "entries" +slot of the autosummary block. + +Example: + +DEPRECATED_API_REFERENCE = { + "0.24": [ + "model_selection.fit_grid_point", + "utils.safe_indexing", + ], +} +""" + +DEPRECATED_API_REFERENCE = {} # type: ignore[var-annotated] diff --git a/doc/authors.rst b/doc/authors.rst deleted file mode 100644 index 861f2c0a884e4..0000000000000 --- a/doc/authors.rst +++ /dev/null @@ -1,92 +0,0 @@ -.. raw :: html - - -
- -
-
-

JÊrÊmie du Boisberranger

-
-
-
-

Joris Van den Bossche

-
-
-
-

Loïc Estève

-
-
-
-

Thomas J. Fan

-
-
-
-

Alexandre Gramfort

-
-
-
-

Olivier Grisel

-
-
-
-

Yaroslav Halchenko

-
-
-
-

Nicolas Hug

-
-
-
-

Adrin Jalali

-
-
-
-

Guillaume Lemaitre

-
-
-
-

Christian Lorentzen

-
-
-
-

Jan Hendrik Metzen

-
-
-
-

Andreas Mueller

-
-
-
-

Vlad Niculae

-
-
-
-

Joel Nothman

-
-
-
-

Hanmin Qin

-
-
-
-

Bertrand Thirion

-
-
-
-

Tom DuprÊ la Tour

-
-
-
-

Gael Varoquaux

-
-
-
-

Nelle Varoquaux

-
-
-
-

Roman Yurchak

-
-
\ No newline at end of file diff --git a/doc/binder/requirements.txt b/doc/binder/requirements.txt index 38619ceae0bc2..92bee596d18ce 100644 --- a/doc/binder/requirements.txt +++ b/doc/binder/requirements.txt @@ -1,5 +1,5 @@ -# A binder requirement file is required by sphinx-gallery. We don't really need -# one since the binder requirement files live in the -# scikit-learn/binder-examples repo and not in the scikit-learn.github.io repo -# that comes from the scikit-learn doc build. This file can be removed if -# 'dependencies' is made an optional key for binder in sphinx-gallery. +# A binder requirement file is required by sphinx-gallery. +# We don't really need one since our binder requirement file lives in the +# .binder directory. +# This file can be removed if 'dependencies' is made an optional key for +# binder in sphinx-gallery. diff --git a/doc/common_pitfalls.rst b/doc/common_pitfalls.rst index 6bc79fbc14c0d..129f9b3990fd5 100644 --- a/doc/common_pitfalls.rst +++ b/doc/common_pitfalls.rst @@ -1,9 +1,3 @@ -.. Places parent toc into the sidebar - -:parenttoc: True - -.. include:: includes/big_toc_css.rst - .. _common_pitfalls: ========================================= @@ -104,6 +98,26 @@ be the average of the train subset, **not** the average of all the data. If the test subset is included in the average calculation, information from the test subset is influencing the model. +How to avoid data leakage +------------------------- + +Below are some tips on avoiding data leakage: + +* Always split the data into train and test subsets first, particularly + before any preprocessing steps. +* Never include test data when using the `fit` and `fit_transform` + methods. Using all the data, e.g., `fit(X)`, can result in overly optimistic + scores. + + Conversely, the `transform` method should be used on both train and test + subsets as the same preprocessing should be applied to all the data. + This can be achieved by using `fit_transform` on the train subset and + `transform` on the test subset. +* The scikit-learn :ref:`pipeline ` is a great way to prevent data + leakage as it ensures that the appropriate method is performed on the + correct data subset. The pipeline is ideal for use in cross-validation + and hyper-parameter tuning functions. + An example of data leakage during preprocessing is detailed below. Data leakage during pre-processing @@ -146,7 +160,7 @@ much higher than expected accuracy score:: >>> from sklearn.model_selection import train_test_split >>> from sklearn.feature_selection import SelectKBest - >>> from sklearn.ensemble import GradientBoostingClassifier + >>> from sklearn.ensemble import HistGradientBoostingClassifier >>> from sklearn.metrics import accuracy_score >>> # Incorrect preprocessing: the entire data is transformed @@ -154,9 +168,9 @@ much higher than expected accuracy score:: >>> X_train, X_test, y_train, y_test = train_test_split( ... X_selected, y, random_state=42) - >>> gbc = GradientBoostingClassifier(random_state=1) + >>> gbc = HistGradientBoostingClassifier(random_state=1) >>> gbc.fit(X_train, y_train) - GradientBoostingClassifier(random_state=1) + HistGradientBoostingClassifier(random_state=1) >>> y_pred = gbc.predict(X_test) >>> accuracy_score(y_test, y_pred) @@ -175,14 +189,14 @@ data, close to chance:: >>> select = SelectKBest(k=25) >>> X_train_selected = select.fit_transform(X_train, y_train) - >>> gbc = GradientBoostingClassifier(random_state=1) + >>> gbc = HistGradientBoostingClassifier(random_state=1) >>> gbc.fit(X_train_selected, y_train) - GradientBoostingClassifier(random_state=1) + HistGradientBoostingClassifier(random_state=1) >>> X_test_selected = select.transform(X_test) >>> y_pred = gbc.predict(X_test_selected) >>> accuracy_score(y_test, y_pred) - 0.46 + 0.5 Here again, we recommend using a :class:`~sklearn.pipeline.Pipeline` to chain together the feature selection and model estimators. The pipeline ensures @@ -193,15 +207,15 @@ is used only for calculating the accuracy score:: >>> X_train, X_test, y_train, y_test = train_test_split( ... X, y, random_state=42) >>> pipeline = make_pipeline(SelectKBest(k=25), - ... GradientBoostingClassifier(random_state=1)) + ... HistGradientBoostingClassifier(random_state=1)) >>> pipeline.fit(X_train, y_train) Pipeline(steps=[('selectkbest', SelectKBest(k=25)), - ('gradientboostingclassifier', - GradientBoostingClassifier(random_state=1))]) + ('histgradientboostingclassifier', + HistGradientBoostingClassifier(random_state=1))]) >>> y_pred = pipeline.predict(X_test) >>> accuracy_score(y_test, y_pred) - 0.46 + 0.5 The pipeline can also be fed into a cross-validation function such as :func:`~sklearn.model_selection.cross_val_score`. @@ -211,27 +225,8 @@ method is used during fitting and predicting:: >>> from sklearn.model_selection import cross_val_score >>> scores = cross_val_score(pipeline, X, y) >>> print(f"Mean accuracy: {scores.mean():.2f}+/-{scores.std():.2f}") - Mean accuracy: 0.45+/-0.07 + Mean accuracy: 0.43+/-0.05 -How to avoid data leakage -------------------------- - -Below are some tips on avoiding data leakage: - -* Always split the data into train and test subsets first, particularly - before any preprocessing steps. -* Never include test data when using the `fit` and `fit_transform` - methods. Using all the data, e.g., `fit(X)`, can result in overly optimistic - scores. - - Conversely, the `transform` method should be used on both train and test - subsets as the same preprocessing should be applied to all the data. - This can be achieved by using `fit_transform` on the train subset and - `transform` on the test subset. -* The scikit-learn :ref:`pipeline ` is a great way to prevent data - leakage as it ensures that the appropriate method is performed on the - correct data subset. The pipeline is ideal for use in cross-validation - and hyper-parameter tuning functions. .. _randomness: @@ -243,7 +238,7 @@ Some scikit-learn objects are inherently random. These are usually estimators splitters (e.g. :class:`~sklearn.model_selection.KFold`). The randomness of these objects is controlled via their `random_state` parameter, as described in the :term:`Glossary `. This section expands on the glossary -entry, and describes good practices and common pitfalls w.r.t. to this +entry, and describes good practices and common pitfalls w.r.t. this subtle parameter. .. note:: Recommendation summary @@ -316,7 +311,7 @@ inter-dependent. For example, two estimators that share the same we discuss cloning. This point is important to keep in mind when debugging. If we had passed an integer to the `random_state` parameter of the -:class:`~sklearn.ensemble.RandomForestClassifier`, we would have obtained the +:class:`~sklearn.linear_model.SGDClassifier`, we would have obtained the same models, and thus the same scores each time. When we pass an integer, the same RNG is used across all calls to `fit`. What internally happens is that even though the RNG is consumed when `fit` is called, it is always reset to @@ -397,7 +392,7 @@ each case**: be the same across all folds. - Since `rf_inst` was passed a `RandomState` instance, each call to `fit` starts from a different RNG. As a result, the random subset of features - will be different for each folds. + will be different for each fold. While having a constant estimator RNG across folds isn't inherently wrong, we usually want CV results that are robust w.r.t. the estimator's randomness. As @@ -413,39 +408,40 @@ it will allow the estimator RNG to vary for each fold. illustration purpose: what matters is what we pass to the :class:`~sklearn.ensemble.RandomForestClassifier` estimator. -**Cloning** +.. dropdown:: Cloning -Another subtle side effect of passing `RandomState` instances is how -:func:`~sklearn.clone` will work:: + Another subtle side effect of passing `RandomState` instances is how + :func:`~sklearn.base.clone` will work:: - >>> from sklearn import clone - >>> from sklearn.ensemble import RandomForestClassifier - >>> import numpy as np + >>> from sklearn import clone + >>> from sklearn.ensemble import RandomForestClassifier + >>> import numpy as np + + >>> rng = np.random.RandomState(0) + >>> a = RandomForestClassifier(random_state=rng) + >>> b = clone(a) + + Since a `RandomState` instance was passed to `a`, `a` and `b` are not clones + in the strict sense, but rather clones in the statistical sense: `a` and `b` + will still be different models, even when calling `fit(X, y)` on the same + data. Moreover, `a` and `b` will influence each other since they share the + same internal RNG: calling `a.fit` will consume `b`'s RNG, and calling + `b.fit` will consume `a`'s RNG, since they are the same. This bit is true for + any estimators that share a `random_state` parameter; it is not specific to + clones. + + If an integer were passed, `a` and `b` would be exact clones and they would not + influence each other. + + .. warning:: + Even though :func:`~sklearn.base.clone` is rarely used in user code, it is + called pervasively throughout scikit-learn codebase: in particular, most + meta-estimators that accept non-fitted estimators call + :func:`~sklearn.base.clone` internally + (:class:`~sklearn.model_selection.GridSearchCV`, + :class:`~sklearn.ensemble.StackingClassifier`, + :class:`~sklearn.calibration.CalibratedClassifierCV`, etc.). - >>> rng = np.random.RandomState(0) - >>> a = RandomForestClassifier(random_state=rng) - >>> b = clone(a) - -Since a `RandomState` instance was passed to `a`, `a` and `b` are not clones -in the strict sense, but rather clones in the statistical sense: `a` and `b` -will still be different models, even when calling `fit(X, y)` on the same -data. Moreover, `a` and `b` will influence each-other since they share the -same internal RNG: calling `a.fit` will consume `b`'s RNG, and calling -`b.fit` will consume `a`'s RNG, since they are the same. This bit is true for -any estimators that share a `random_state` parameter; it is not specific to -clones. - -If an integer were passed, `a` and `b` would be exact clones and they would not -influence each other. - -.. warning:: - Even though :func:`~sklearn.clone` is rarely used in user code, it is - called pervasively throughout scikit-learn codebase: in particular, most - meta-estimators that accept non-fitted estimators call - :func:`~sklearn.clone` internally - (:class:`~sklearn.model_selection.GridSearchCV`, - :class:`~sklearn.ensemble.StackingClassifier`, - :class:`~sklearn.calibration.CalibratedClassifierCV`, etc.). CV splitters ............ @@ -553,18 +549,18 @@ When we evaluate a randomized estimator performance by cross-validation, we want to make sure that the estimator can yield accurate predictions for new data, but we also want to make sure that the estimator is robust w.r.t. its random initialization. For example, we would like the random weights -initialization of a :class:`~sklearn.linear_model.SGDCLassifier` to be +initialization of an :class:`~sklearn.linear_model.SGDClassifier` to be consistently good across all folds: otherwise, when we train that estimator on new data, we might get unlucky and the random initialization may lead to -bad performance. Similarly, we want a random forest to be robust w.r.t the +bad performance. Similarly, we want a random forest to be robust w.r.t. the set of randomly selected features that each tree will be using. For these reasons, it is preferable to evaluate the cross-validation -preformance by letting the estimator use a different RNG on each fold. This +performance by letting the estimator use a different RNG on each fold. This is done by passing a `RandomState` instance (or `None`) to the estimator initialization. -When we pass an integer, the estimator will use the same RNG on each fold: if +When we pass an integer, the estimator will use the same RNG on each fold: if the estimator performs well (or bad), as evaluated by CV, it might just be because we got lucky (or unlucky) with that specific seed. Passing instances leads to more robust CV results, and makes the comparison between various diff --git a/doc/communication_team.rst b/doc/communication_team.rst new file mode 100644 index 0000000000000..fb9666f0b42f7 --- /dev/null +++ b/doc/communication_team.rst @@ -0,0 +1,16 @@ +.. raw :: html + + +
+ +
+
+

Lauren Burke-McCarthy

+
+
+
+

François Goupil

+
+
diff --git a/doc/communication_team_emeritus.rst b/doc/communication_team_emeritus.rst new file mode 100644 index 0000000000000..d5ef7df59238e --- /dev/null +++ b/doc/communication_team_emeritus.rst @@ -0,0 +1 @@ +- Reshama Shaikh diff --git a/doc/computing.rst b/doc/computing.rst index 6732b754918b0..9f166432006b2 100644 --- a/doc/computing.rst +++ b/doc/computing.rst @@ -1,13 +1,7 @@ -.. Places parent toc into the sidebar - -:parenttoc: True - ============================ Computing with scikit-learn ============================ -.. include:: includes/big_toc_css.rst - .. toctree:: :maxdepth: 2 diff --git a/doc/computing/computational_performance.rst b/doc/computing/computational_performance.rst index 32a485e21a2a5..4af79206dae1c 100644 --- a/doc/computing/computational_performance.rst +++ b/doc/computing/computational_performance.rst @@ -1,7 +1,3 @@ -.. Places parent toc into the sidebar - -:parenttoc: True - .. _computational_performance: .. currentmodule:: sklearn @@ -19,9 +15,9 @@ scikit-learn estimators in different contexts and provide some tips and tricks for overcoming performance bottlenecks. Prediction latency is measured as the elapsed time necessary to make a -prediction (e.g. in micro-seconds). Latency is often viewed as a distribution +prediction (e.g. in microseconds). Latency is often viewed as a distribution and operations engineers often focus on the latency at a given percentile of -this distribution (e.g. the 90 percentile). +this distribution (e.g. the 90th percentile). Prediction throughput is defined as the number of predictions the software can deliver in a given amount of time (e.g. in predictions per second). @@ -34,15 +30,16 @@ to take into account the same exact properties of the data as more complex ones. Prediction Latency ------------------ -One of the most straight-forward concerns one may have when using/choosing a +One of the most straightforward concerns one may have when using/choosing a machine learning toolkit is the latency at which predictions can be made in a production environment. The main factors that influence the prediction latency are - 1. Number of features - 2. Input data representation and sparsity - 3. Model complexity - 4. Feature extraction + +1. Number of features +2. Input data representation and sparsity +3. Model complexity +4. Feature extraction A last major parameter is also the possibility to do predictions in bulk or one-at-a-time mode. @@ -128,7 +125,7 @@ by quite a bit as only the non zero valued features impact the dot product and thus the model predictions. Hence if you have 100 non zeros in 1e6 dimensional space, you only need 100 multiply and add operation instead of 1e6. -Calculation over a dense representation, however, may leverage highly optimised +Calculation over a dense representation, however, may leverage highly optimized vector operations and multithreading in BLAS, and tends to result in fewer CPU cache misses. So the sparsity should typically be quite high (10% non-zeros max, to be checked depending on the hardware) for the sparse input @@ -195,7 +192,7 @@ support vectors. .. centered:: |nusvr_model_complexity| For :mod:`sklearn.ensemble` of trees (e.g. RandomForest, GBT, -ExtraTrees etc) the number of trees and their depth play the most +ExtraTrees, etc.) the number of trees and their depth play the most important role. Latency and throughput should scale linearly with the number of trees. In this case we used directly the ``n_estimators`` parameter of :class:`~ensemble.GradientBoostingRegressor`. @@ -224,9 +221,9 @@ files, tokenizing the text and hashing it into a common vector space) is taking 100 to 500 times more time than the actual prediction code, depending on the chosen model. - .. |prediction_time| image:: ../auto_examples/applications/images/sphx_glr_plot_out_of_core_classification_004.png - :target: ../auto_examples/applications/plot_out_of_core_classification.html - :scale: 80 +.. |prediction_time| image:: ../auto_examples/applications/images/sphx_glr_plot_out_of_core_classification_004.png + :target: ../auto_examples/applications/plot_out_of_core_classification.html + :scale: 80 .. centered:: |prediction_time| @@ -278,21 +275,20 @@ BLAS implementation and lead to orders of magnitude speedup over a non-optimized BLAS. You can display the BLAS / LAPACK implementation used by your NumPy / SciPy / -scikit-learn install with the following commands:: +scikit-learn install with the following command:: - from numpy.distutils.system_info import get_info - print(get_info('blas_opt')) - print(get_info('lapack_opt')) + python -c "import sklearn; sklearn.show_versions()" Optimized BLAS / LAPACK implementations include: - - Atlas (need hardware specific tuning by rebuilding on the target machine) - - OpenBLAS - - MKL - - Apple Accelerate and vecLib frameworks (OSX only) -More information can be found on the `Scipy install page `_ +- Atlas (need hardware specific tuning by rebuilding on the target machine) +- OpenBLAS +- MKL +- Apple Accelerate and vecLib frameworks (OSX only) + +More information can be found on the `NumPy install page `_ and in this -`blog post `_ +`blog post `_ from Daniel Nouri which has some nice step by step install instructions for Debian / Ubuntu. @@ -356,7 +352,7 @@ feature selection components in a pipeline once we know which features to keep from a previous run. Finally, it can help reduce processing time and I/O usage upstream in the data access and feature extraction layers by not collecting and building features that are discarded by the model. For instance -if the raw data come from a database, it can make it possible to write simpler +if the raw data come from a database, it is possible to write simpler and faster queries or reduce I/O usage by making the queries return lighter records. At the moment, reshaping needs to be performed manually in scikit-learn. @@ -366,5 +362,5 @@ sufficient to not generate the relevant features, leaving their columns empty. Links ...... - - :ref:`scikit-learn developer performance documentation ` - - `Scipy sparse matrix formats documentation `_ +- :ref:`scikit-learn developer performance documentation ` +- `Scipy sparse matrix formats documentation `_ diff --git a/doc/computing/parallelism.rst b/doc/computing/parallelism.rst index 8605650e8eec5..d2ff106aec3be 100644 --- a/doc/computing/parallelism.rst +++ b/doc/computing/parallelism.rst @@ -1,7 +1,3 @@ -.. Places parent toc into the sidebar - -:parenttoc: True - Parallelism, resource management, and configuration =================================================== @@ -10,22 +6,29 @@ Parallelism, resource management, and configuration Parallelism ----------- -Some scikit-learn estimators and utilities can parallelize costly operations -using multiple CPU cores, thanks to the following components: +Some scikit-learn estimators and utilities parallelize costly operations +using multiple CPU cores. + +Depending on the type of estimator and sometimes the values of the +constructor parameters, this is either done: -- via the `joblib `_ library. In - this case the number of threads or processes can be controlled with the - ``n_jobs`` parameter. -- via OpenMP, used in C or Cython code. +- with higher-level parallelism via `joblib `_. +- with lower-level parallelism via OpenMP, used in C or Cython code. +- with lower-level parallelism via BLAS, used by NumPy and SciPy for generic operations + on arrays. -In addition, some of the numpy routines that are used internally by -scikit-learn may also be parallelized if numpy is installed with specific -numerical libraries such as MKL, OpenBLAS, or BLIS. +The `n_jobs` parameters of estimators always controls the amount of parallelism +managed by joblib (processes or threads depending on the joblib backend). +The thread-level parallelism managed by OpenMP in scikit-learn's own Cython code +or by BLAS & LAPACK libraries used by NumPy and SciPy operations used in scikit-learn +is always controlled by environment variables or `threadpoolctl` as explained below. +Note that some estimators can leverage all three kinds of parallelism at different +points of their training and prediction methods. -We describe these 3 scenarios in the following subsections. +We describe these 3 types of parallelism in the following subsections in more details. -Joblib-based parallelism -........................ +Higher-level parallelism with joblib +.................................... When the underlying implementation uses joblib, the number of workers (threads or processes) that are spawned in parallel can be controlled via the @@ -33,15 +36,16 @@ When the underlying implementation uses joblib, the number of workers .. note:: - Where (and how) parallelization happens in the estimators is currently - poorly documented. Please help us by improving our docs and tackle `issue - 14228 `_! + Where (and how) parallelization happens in the estimators using joblib by + specifying `n_jobs` is currently poorly documented. + Please help us by improving our docs and tackle `issue 14228 + `_! Joblib is able to support both multi-processing and multi-threading. Whether joblib chooses to spawn a thread or a process depends on the **backend** that it's using. -Scikit-learn generally relies on the ``loky`` backend, which is joblib's +scikit-learn generally relies on the ``loky`` backend, which is joblib's default backend. Loky is a multi-processing backend. When doing multi-processing, in order to avoid duplicating the memory in each process (which isn't reasonable with big datasets), joblib will create a `memmap @@ -68,44 +72,63 @@ In practice, whether parallelism is helpful at improving runtime depends on many factors. It is usually a good idea to experiment rather than assuming that increasing the number of workers is always a good thing. In some cases it can be highly detrimental to performance to run multiple copies of some -estimators or functions in parallel (see oversubscription below). +estimators or functions in parallel (see :ref:`oversubscription` below). -OpenMP-based parallelism -........................ +Lower-level parallelism with OpenMP +................................... OpenMP is used to parallelize code written in Cython or C, relying on -multi-threading exclusively. By default (and unless joblib is trying to -avoid oversubscription), the implementation will use as many threads as -possible. +multi-threading exclusively. By default, the implementations using OpenMP +will use as many threads as possible, i.e. as many threads as logical cores. -You can control the exact number of threads that are used via the -``OMP_NUM_THREADS`` environment variable: +You can control the exact number of threads that are used either: -.. prompt:: bash $ +- via the ``OMP_NUM_THREADS`` environment variable, for instance when: + running a python script: + + .. prompt:: bash $ - OMP_NUM_THREADS=4 python my_script.py + OMP_NUM_THREADS=4 python my_script.py -Parallel Numpy routines from numerical libraries -................................................ +- or via `threadpoolctl` as explained by `this piece of documentation + `_. -Scikit-learn relies heavily on NumPy and SciPy, which internally call -multi-threaded linear algebra routines implemented in libraries such as MKL, -OpenBLAS or BLIS. +Parallel NumPy and SciPy routines from numerical libraries +.......................................................... -The number of threads used by the OpenBLAS, MKL or BLIS libraries can be set -via the ``MKL_NUM_THREADS``, ``OPENBLAS_NUM_THREADS``, and -``BLIS_NUM_THREADS`` environment variables. +scikit-learn relies heavily on NumPy and SciPy, which internally call +multi-threaded linear algebra routines (BLAS & LAPACK) implemented in libraries +such as MKL, OpenBLAS or BLIS. + +You can control the exact number of threads used by BLAS for each library +using environment variables, namely: + +- ``MKL_NUM_THREADS`` sets the number of threads MKL uses, +- ``OPENBLAS_NUM_THREADS`` sets the number of threads OpenBLAS uses +- ``BLIS_NUM_THREADS`` sets the number of threads BLIS uses + +Note that BLAS & LAPACK implementations can also be impacted by +`OMP_NUM_THREADS`. To check whether this is the case in your environment, +you can inspect how the number of threads effectively used by those libraries +is affected when running the following command in a bash or zsh terminal +for different values of `OMP_NUM_THREADS`: + +.. prompt:: bash $ -Please note that scikit-learn has no direct control over these -implementations. Scikit-learn solely relies on Numpy and Scipy. + OMP_NUM_THREADS=2 python -m threadpoolctl -i numpy scipy .. note:: - At the time of writing (2019), NumPy and SciPy packages distributed on - pypi.org (used by ``pip``) and on the conda-forge channel are linked - with OpenBLAS, while conda packages shipped on the "defaults" channel - from anaconda.org are linked by default with MKL. + At the time of writing (2022), NumPy and SciPy packages which are + distributed on pypi.org (i.e. the ones installed via ``pip install``) + and on the conda-forge channel (i.e. the ones installed via + ``conda install --channel conda-forge``) are linked with OpenBLAS, while + NumPy and SciPy packages shipped on the ``defaults`` conda + channel from Anaconda.org (i.e. the ones installed via ``conda install``) + are linked by default with MKL. +.. _oversubscription: + Oversubscription: spawning too many threads ........................................... @@ -120,8 +143,8 @@ with ``n_jobs=8`` over a OpenMP). Each instance of :class:`~sklearn.ensemble.HistGradientBoostingClassifier` will spawn 8 threads (since you have 8 CPUs). That's a total of ``8 * 8 = 64`` threads, which -leads to oversubscription of physical CPU resources and to scheduling -overhead. +leads to oversubscription of threads for physical CPU resources and thus +to scheduling overhead. Oversubscription can arise in the exact same fashion with parallelized routines from MKL, OpenBLAS or BLIS that are nested in joblib calls. @@ -146,71 +169,170 @@ Note that: only use ``_NUM_THREADS``. Joblib exposes a context manager for finer control over the number of threads in its workers (see joblib docs linked below). -- Joblib is currently unable to avoid oversubscription in a - multi-threading context. It can only do so with the ``loky`` backend - (which spawns processes). +- When joblib is configured to use the ``threading`` backend, there is no + mechanism to avoid oversubscriptions when calling into parallel native + libraries in the joblib-managed threads. +- All scikit-learn estimators that explicitly rely on OpenMP in their Cython code + always use `threadpoolctl` internally to automatically adapt the numbers of + threads used by OpenMP and potentially nested BLAS calls so as to avoid + oversubscription. You will find additional details about joblib mitigation of oversubscription in `joblib documentation -`_. +`_. +You will find additional details about parallelism in numerical python libraries +in `this document from Thomas J. Fan `_. Configuration switches ----------------------- -Python runtime -.............. +Python API +.......... -:func:`sklearn.set_config` controls the following behaviors: +:func:`sklearn.set_config` and :func:`sklearn.config_context` can be used to change +parameters of the configuration which control aspect of parallelism. -:assume_finite: +.. _environment_variable: - used to skip validation, which enables faster computations but may - lead to segmentation faults if the data contains NaNs. +Environment variables +..................... -:working_memory: +These environment variables should be set before importing scikit-learn. - the optimal size of temporary arrays used by some algorithms. +`SKLEARN_ASSUME_FINITE` +~~~~~~~~~~~~~~~~~~~~~~~ -.. _environment_variable: +Sets the default value for the `assume_finite` argument of +:func:`sklearn.set_config`. -Environment variables -...................... +`SKLEARN_WORKING_MEMORY` +~~~~~~~~~~~~~~~~~~~~~~~~ -These environment variables should be set before importing scikit-learn. +Sets the default value for the `working_memory` argument of +:func:`sklearn.set_config`. + +`SKLEARN_SEED` +~~~~~~~~~~~~~~ + +Sets the seed of the global random generator when running the tests, for +reproducibility. + +Note that scikit-learn tests are expected to run deterministically with +explicit seeding of their own independent RNG instances instead of relying on +the numpy or Python standard library RNG singletons to make sure that test +results are independent of the test execution order. However some tests might +forget to use explicit seeding and this variable is a way to control the initial +state of the aforementioned singletons. + +`SKLEARN_TESTS_GLOBAL_RANDOM_SEED` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Controls the seeding of the random number generator used in tests that rely on +the `global_random_seed` fixture. + +All tests that use this fixture accept the contract that they should +deterministically pass for any seed value from 0 to 99 included. + +In nightly CI builds, the `SKLEARN_TESTS_GLOBAL_RANDOM_SEED` environment +variable is drawn randomly in the above range and all fixtured tests will run +for that specific seed. The goal is to ensure that, over time, our CI will run +all tests with different seeds while keeping the test duration of a single run +of the full test suite limited. This will check that the assertions of tests +written to use this fixture are not dependent on a specific seed value. + +The range of admissible seed values is limited to [0, 99] because it is often +not possible to write a test that can work for any possible seed and we want to +avoid having tests that randomly fail on the CI. + +Valid values for `SKLEARN_TESTS_GLOBAL_RANDOM_SEED`: + +- `SKLEARN_TESTS_GLOBAL_RANDOM_SEED="42"`: run tests with a fixed seed of 42 +- `SKLEARN_TESTS_GLOBAL_RANDOM_SEED="40-42"`: run the tests with all seeds + between 40 and 42 included +- `SKLEARN_TESTS_GLOBAL_RANDOM_SEED="all"`: run the tests with all seeds + between 0 and 99 included. This can take a long time: only use for individual + tests, not the full test suite! + +If the variable is not set, then 42 is used as the global seed in a +deterministic manner. This ensures that, by default, the scikit-learn test +suite is as deterministic as possible to avoid disrupting our friendly +third-party package maintainers. Similarly, this variable should not be set in +the CI config of pull-requests to make sure that our friendly contributors are +not the first people to encounter a seed-sensitivity regression in a test +unrelated to the changes of their own PR. Only the scikit-learn maintainers who +watch the results of the nightly builds are expected to be annoyed by this. + +When writing a new test function that uses this fixture, please use the +following command to make sure that it passes deterministically for all +admissible seeds on your local machine: + +.. prompt:: bash $ + + SKLEARN_TESTS_GLOBAL_RANDOM_SEED="all" pytest -v -k test_your_test_name + +`SKLEARN_SKIP_NETWORK_TESTS` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When this environment variable is set to a non zero value, the tests that need +network access are skipped. When this environment variable is not set then +network tests are skipped. + +`SKLEARN_RUN_FLOAT32_TESTS` +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When this environment variable is set to '1', the tests using the +`global_dtype` fixture are also run on float32 data. +When this environment variable is not set, the tests are only run on +float64 data. + +`SKLEARN_ENABLE_DEBUG_CYTHON_DIRECTIVES` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When this environment variable is set to a non zero value, the `Cython` +derivative, `boundscheck` is set to `True`. This is useful for finding +segfaults. -:SKLEARN_SITE_JOBLIB: +`SKLEARN_BUILD_ENABLE_DEBUG_SYMBOLS` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - When this environment variable is set to a non zero value, - scikit-learn uses the site joblib rather than its vendored version. - Consequently, joblib must be installed for scikit-learn to run. - Note that using the site joblib is at your own risks: the versions of - scikit-learn and joblib need to be compatible. Currently, joblib 0.11+ - is supported. In addition, dumps from joblib.Memory might be incompatible, - and you might loose some caches and have to redownload some datasets. +When this environment variable is set to a non zero value, the debug symbols +will be included in the compiled C extensions. Only debug symbols for POSIX +systems are configured. - .. deprecated:: 0.21 +`SKLEARN_PAIRWISE_DIST_CHUNK_SIZE` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - As of version 0.21 this parameter has no effect, vendored joblib was - removed and site joblib is always used. +This sets the size of chunk to be used by the underlying `PairwiseDistancesReductions` +implementations. The default value is `256` which has been showed to be adequate on +most machines. -:SKLEARN_ASSUME_FINITE: +Users looking for the best performance might want to tune this variable using +powers of 2 so as to get the best parallelism behavior for their hardware, +especially with respect to their caches' sizes. - Sets the default value for the `assume_finite` argument of - :func:`sklearn.set_config`. +`SKLEARN_WARNINGS_AS_ERRORS` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -:SKLEARN_WORKING_MEMORY: +This environment variable is used to turn warnings into errors in tests and +documentation build. - Sets the default value for the `working_memory` argument of - :func:`sklearn.set_config`. +Some CI (Continuous Integration) builds set `SKLEARN_WARNINGS_AS_ERRORS=1`, for +example to make sure that we catch deprecation warnings from our dependencies +and that we adapt our code. -:SKLEARN_SEED: +To locally run with the same "warnings as errors" setting as in these CI builds +you can set `SKLEARN_WARNINGS_AS_ERRORS=1`. - Sets the seed of the global random generator when running the tests, - for reproducibility. +By default, warnings are not turned into errors. This is the case if +`SKLEARN_WARNINGS_AS_ERRORS` is unset, or `SKLEARN_WARNINGS_AS_ERRORS=0`. -:SKLEARN_SKIP_NETWORK_TESTS: +This environment variable uses specific warning filters to ignore some warnings, +since sometimes warnings originate from third-party libraries and there is not +much we can do about it. You can see the warning filters in the +`_get_warnings_filters_info_list` function in `sklearn/utils/_testing.py`. - When this environment variable is set to a non zero value, the tests - that need network access are skipped. When this environment variable is - not set then network tests are skipped. +Note that for documentation build, `SKLEARN_WARNING_AS_ERRORS=1` is checking +that the documentation build, in particular running examples, does not produce +any warnings. This is different from the `-W` `sphinx-build` argument that +catches syntax warnings in the rst files. diff --git a/doc/computing/scaling_strategies.rst b/doc/computing/scaling_strategies.rst index 5eee5728e4b9a..286a1e79d0a8c 100644 --- a/doc/computing/scaling_strategies.rst +++ b/doc/computing/scaling_strategies.rst @@ -1,7 +1,3 @@ -.. Places parent toc into the sidebar - -:parenttoc: True - .. _scaling_strategies: Strategies to scale computationally: bigger data @@ -20,9 +16,9 @@ data that cannot fit in a computer's main memory (RAM). Here is a sketch of a system designed to achieve this goal: - 1. a way to stream instances - 2. a way to extract features from instances - 3. an incremental algorithm +1. a way to stream instances +2. a way to extract features from instances +3. an incremental algorithm Streaming instances .................... @@ -62,28 +58,29 @@ balances relevancy and memory footprint could involve some tuning [1]_. Here is a list of incremental estimators for different tasks: - - Classification - + :class:`sklearn.naive_bayes.MultinomialNB` - + :class:`sklearn.naive_bayes.BernoulliNB` - + :class:`sklearn.linear_model.Perceptron` - + :class:`sklearn.linear_model.SGDClassifier` - + :class:`sklearn.linear_model.PassiveAggressiveClassifier` - + :class:`sklearn.neural_network.MLPClassifier` - - Regression - + :class:`sklearn.linear_model.SGDRegressor` - + :class:`sklearn.linear_model.PassiveAggressiveRegressor` - + :class:`sklearn.neural_network.MLPRegressor` - - Clustering - + :class:`sklearn.cluster.MiniBatchKMeans` - + :class:`sklearn.cluster.Birch` - - Decomposition / feature Extraction - + :class:`sklearn.decomposition.MiniBatchDictionaryLearning` - + :class:`sklearn.decomposition.IncrementalPCA` - + :class:`sklearn.decomposition.LatentDirichletAllocation` - - Preprocessing - + :class:`sklearn.preprocessing.StandardScaler` - + :class:`sklearn.preprocessing.MinMaxScaler` - + :class:`sklearn.preprocessing.MaxAbsScaler` +- Classification + + :class:`sklearn.naive_bayes.MultinomialNB` + + :class:`sklearn.naive_bayes.BernoulliNB` + + :class:`sklearn.linear_model.Perceptron` + + :class:`sklearn.linear_model.SGDClassifier` + + :class:`sklearn.linear_model.PassiveAggressiveClassifier` + + :class:`sklearn.neural_network.MLPClassifier` +- Regression + + :class:`sklearn.linear_model.SGDRegressor` + + :class:`sklearn.linear_model.PassiveAggressiveRegressor` + + :class:`sklearn.neural_network.MLPRegressor` +- Clustering + + :class:`sklearn.cluster.MiniBatchKMeans` + + :class:`sklearn.cluster.Birch` +- Decomposition / feature Extraction + + :class:`sklearn.decomposition.MiniBatchDictionaryLearning` + + :class:`sklearn.decomposition.IncrementalPCA` + + :class:`sklearn.decomposition.LatentDirichletAllocation` + + :class:`sklearn.decomposition.MiniBatchNMF` +- Preprocessing + + :class:`sklearn.preprocessing.StandardScaler` + + :class:`sklearn.preprocessing.MinMaxScaler` + + :class:`sklearn.preprocessing.MaxAbsScaler` For classification, a somewhat important thing to note is that although a stateless feature extraction routine may be able to cope with new/unseen diff --git a/doc/conf.py b/doc/conf.py index 6768aab208a99..71c9ec5bb60c3 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # scikit-learn documentation build configuration file, created by # sphinx-quickstart on Fri Jan 8 09:13:42 2010. # @@ -12,81 +10,141 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys +import json import os -import warnings import re +import sys +import warnings from datetime import datetime -from packaging.version import parse from pathlib import Path -from io import StringIO +from urllib.request import urlopen + +from sklearn.externals._packaging.version import parse +from sklearn.utils._testing import turn_warnings_into_errors # If extensions (or modules to document with autodoc) are in another # directory, add these directories to sys.path here. If the directory # is relative to the documentation root, use os.path.abspath to make it # absolute, like shown here. -sys.path.insert(0, os.path.abspath('sphinxext')) +sys.path.insert(0, os.path.abspath(".")) +sys.path.insert(0, os.path.abspath("sphinxext")) -from github_link import make_linkcode_resolve +import jinja2 import sphinx_gallery +from github_link import make_linkcode_resolve +from sphinx.util.logging import getLogger +from sphinx_gallery.notebook import add_code_cell, add_markdown_cell +from sphinx_gallery.sorting import ExampleTitleSortKey + +logger = getLogger(__name__) + +try: + # Configure plotly to integrate its output into the HTML pages generated by + # sphinx-gallery. + import plotly.io as pio + + pio.renderers.default = "sphinx_gallery" +except ImportError: + # Make it possible to render the doc when not running the examples + # that need plotly. + pass # -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ - 'sphinx.ext.autodoc', 'sphinx.ext.autosummary', - 'numpydoc', - 'sphinx.ext.linkcode', 'sphinx.ext.doctest', - 'sphinx.ext.intersphinx', - 'sphinx.ext.imgconverter', - 'sphinx_gallery.gen_gallery', - 'sphinx_issues', - 'add_toctree_functions', - 'sphinx-prompt', + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "numpydoc", + "sphinx.ext.linkcode", + "sphinx.ext.doctest", + "sphinx.ext.intersphinx", + "sphinx.ext.imgconverter", + "sphinx_gallery.gen_gallery", + "sphinx-prompt", + "sphinx_copybutton", + "sphinxext.opengraph", + "matplotlib.sphinxext.plot_directive", + "sphinxcontrib.sass", + "sphinx_remove_toctrees", + "sphinx_design", + # See sphinxext/ + "allow_nan_estimators", + "autoshortsummary", + "doi_role", + "dropdown_anchors", + "override_pst_pagetoc", + "sphinx_issues", ] -# this is needed for some reason... -# see https://github.com/numpy/numpydoc/issues/69 +# Specify how to identify the prompt when copying code snippets +copybutton_prompt_text = r">>> |\.\.\. " +copybutton_prompt_is_regexp = True +copybutton_exclude = "style" + +try: + import jupyterlite_sphinx # noqa: F401 + + extensions.append("jupyterlite_sphinx") + with_jupyterlite = True +except ImportError: + # In some cases we don't want to require jupyterlite_sphinx to be installed, + # e.g. the doc-min-dependencies build + warnings.warn( + "jupyterlite_sphinx is not installed, you need to install it " + "if you want JupyterLite links to appear in each example" + ) + with_jupyterlite = False + +# Produce `plot::` directives for examples that contain `import matplotlib` or +# `from matplotlib import`. +numpydoc_use_plots = True + +# Options for the `::plot` directive: +# https://matplotlib.org/stable/api/sphinxext_plot_directive_api.html +plot_formats = ["png"] +plot_include_source = True +plot_html_show_formats = False +plot_html_show_source_link = False + +# We do not need the table of class members because `sphinxext/override_pst_pagetoc.py` +# will show them in the secondary sidebar +numpydoc_show_class_members = False +numpydoc_show_inherited_class_members = False + +# We want in-page toc of class members instead of a separate page for each entry numpydoc_class_members_toctree = False # For maths, use mathjax by default and svg if NO_MATHJAX env variable is set # (useful for viewing the doc offline) -if os.environ.get('NO_MATHJAX'): - extensions.append('sphinx.ext.imgmath') - imgmath_image_format = 'svg' - mathjax_path = '' +if os.environ.get("NO_MATHJAX"): + extensions.append("sphinx.ext.imgmath") + imgmath_image_format = "svg" + mathjax_path = "" else: - extensions.append('sphinx.ext.mathjax') - mathjax_path = ('https://cdn.jsdelivr.net/npm/mathjax@3/es5/' - 'tex-chtml.js') - -autodoc_default_options = { - 'members': True, - 'inherited-members': True -} + extensions.append("sphinx.ext.mathjax") + mathjax_path = "https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-chtml.js" # Add any paths that contain templates here, relative to this directory. -templates_path = ['templates'] +templates_path = ["templates"] # generate autosummary even if no references autosummary_generate = True # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8' +source_encoding = "utf-8" # The main toctree document. -main_doc = 'contents' +root_doc = "index" # General information about the project. -project = 'scikit-learn' -copyright = ( - f'2007 - {datetime.now().year}, scikit-learn developers (BSD License)' -) +project = "scikit-learn" +copyright = f"2007 - {datetime.now().year}, scikit-learn developers (BSD License)" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -94,6 +152,7 @@ # # The short X.Y version. import sklearn + parsed_version = parse(sklearn.__version__) version = ".".join(parsed_version.base_version.split(".")[:2]) # The full version, including alpha/beta/rc tags. @@ -105,89 +164,229 @@ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build', 'templates', 'includes', 'themes'] +exclude_patterns = [ + "_build", + "templates", + "includes", + "**/sg_execution_times.rst", + "whats_new/upcoming_changes", +] # The reST default role (used for this markup: `text`) to use for all # documents. -default_role = 'literal' +default_role = "literal" # If true, '()' will be appended to :func: etc. cross-reference text. add_function_parentheses = False # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +# show_authors = False # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. Major themes that come with # Sphinx are currently 'default' and 'sphinxdoc'. -html_theme = 'scikit-learn-modern' +html_theme = "pydata_sphinx_theme" + +# This config option is used to generate the canonical links in the header +# of every page. The canonical link is needed to prevent search engines from +# returning results pointing to old scikit-learn versions. +html_baseurl = "https://scikit-learn.org/stable/" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -html_theme_options = {'google_analytics': True, - 'mathjax_path': mathjax_path} +html_theme_options = { + # -- General configuration ------------------------------------------------ + "sidebar_includehidden": True, + "use_edit_page_button": True, + "external_links": [], + "icon_links_label": "Icon Links", + "icon_links": [ + { + "name": "GitHub", + "url": "https://github.com/scikit-learn/scikit-learn", + "icon": "fa-brands fa-square-github", + "type": "fontawesome", + }, + ], + "analytics": { + "plausible_analytics_domain": "scikit-learn.org", + "plausible_analytics_url": "https://views.scientific-python.org/js/script.js", + }, + # If "prev-next" is included in article_footer_items, then setting show_prev_next + # to True would repeat prev and next links. See + # https://github.com/pydata/pydata-sphinx-theme/blob/b731dc230bc26a3d1d1bb039c56c977a9b3d25d8/src/pydata_sphinx_theme/theme/pydata_sphinx_theme/layout.html#L118-L129 + "show_prev_next": False, + "search_bar_text": "Search the docs ...", + "navigation_with_keys": False, + "collapse_navigation": False, + "navigation_depth": 2, + "show_nav_level": 1, + "show_toc_level": 1, + "navbar_align": "left", + "header_links_before_dropdown": 5, + "header_dropdown_text": "More", + # The switcher requires a JSON file with the list of documentation versions, which + # is generated by the script `build_tools/circle/list_versions.py` and placed under + # the `js/` static directory; it will then be copied to the `_static` directory in + # the built documentation + "switcher": { + "json_url": "https://scikit-learn.org/dev/_static/versions.json", + "version_match": release, + }, + # check_switcher may be set to False if docbuild pipeline fails. See + # https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/version-dropdown.html#configure-switcher-json-url + "check_switcher": True, + "pygments_light_style": "tango", + "pygments_dark_style": "monokai", + "logo": { + "alt_text": "scikit-learn homepage", + "image_relative": "logos/scikit-learn-logo-small.png", + "image_light": "logos/scikit-learn-logo-small.png", + "image_dark": "logos/scikit-learn-logo-small.png", + }, + "surface_warnings": True, + # -- Template placement in theme layouts ---------------------------------- + "navbar_start": ["navbar-logo"], + # Note that the alignment of navbar_center is controlled by navbar_align + "navbar_center": ["navbar-nav"], + "navbar_end": ["theme-switcher", "navbar-icon-links", "version-switcher"], + # navbar_persistent is persistent right (even when on mobiles) + "navbar_persistent": ["search-button"], + "article_header_start": ["breadcrumbs"], + "article_header_end": [], + "article_footer_items": ["prev-next"], + "content_footer_items": [], + # Use html_sidebars that map page patterns to list of sidebar templates + "primary_sidebar_end": [], + "footer_start": ["copyright"], + "footer_center": [], + "footer_end": [], + # When specified as a dictionary, the keys should follow glob-style patterns, as in + # https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-exclude_patterns + # In particular, "**" specifies the default for all pages + # Use :html_theme.sidebar_secondary.remove: for file-wide removal + "secondary_sidebar_items": { + "**": [ + "page-toc", + "sourcelink", + # Sphinx-Gallery-specific sidebar components + # https://sphinx-gallery.github.io/stable/advanced.html#using-sphinx-gallery-sidebar-components + "sg_download_links", + "sg_launcher_links", + ], + }, + "show_version_warning_banner": True, + "announcement": None, +} # Add any paths that contain custom themes here, relative to this directory. -html_theme_path = ['themes'] - +# html_theme_path = ["themes"] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -html_short_title = 'scikit-learn' - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -html_logo = 'logos/scikit-learn-logo-small.png' +html_short_title = "scikit-learn" # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -html_favicon = 'logos/favicon.ico' +html_favicon = "logos/favicon.ico" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['images'] +html_static_path = ["images", "css", "js"] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# Workaround for removing the left sidebar on pages without TOC +# A better solution would be to follow the merge of: +# https://github.com/pydata/pydata-sphinx-theme/pull/1682 +html_sidebars = { + "install": [], + "getting_started": [], + "glossary": [], + "faq": [], + "support": [], + "related_projects": [], + "roadmap": [], + "governance": [], + "about": [], +} # Additional templates that should be rendered to pages, maps page names to # template names. -html_additional_pages = { - 'index': 'index.html', - 'documentation': 'documentation.html'} # redirects to index +html_additional_pages = {"index": "index.html"} + +# Additional files to copy +# html_extra_path = [] + +# Additional JS files +html_js_files = [ + "scripts/dropdown.js", + "scripts/version-switcher.js", + "scripts/sg_plotly_resize.js", +] + +# Compile scss files into css files using sphinxcontrib-sass +sass_src_dir, sass_out_dir = "scss", "css/styles" +sass_targets = { + f"{file.stem}.scss": f"{file.stem}.css" + for file in Path(sass_src_dir).glob("*.scss") +} + +# Additional CSS files, should be subset of the values of `sass_targets` +html_css_files = ["styles/colors.css", "styles/custom.css"] + + +def add_js_css_files(app, pagename, templatename, context, doctree): + """Load additional JS and CSS files only for certain pages. + + Note that `html_js_files` and `html_css_files` are included in all pages and + should be used for the ones that are used by multiple pages. All page-specific + JS and CSS files should be added here instead. + """ + if pagename == "api/index": + # External: jQuery and DataTables + app.add_js_file("https://code.jquery.com/jquery-3.7.0.js") + app.add_js_file("https://cdn.datatables.net/2.0.0/js/dataTables.min.js") + app.add_css_file( + "https://cdn.datatables.net/2.0.0/css/dataTables.dataTables.min.css" + ) + # Internal: API search initialization and styling + app.add_js_file("scripts/api-search.js") + app.add_css_file("styles/api-search.css") + elif pagename == "index": + app.add_css_file("styles/index.css") + elif pagename.startswith("modules/generated/"): + app.add_css_file("styles/api.css") + # If false, no module index is generated. html_domain_indices = False @@ -196,21 +395,21 @@ html_use_index = False # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = '' +# html_file_suffix = '' # Output file base name for HTML help builder. -htmlhelp_basename = 'scikit-learndoc' +htmlhelp_basename = "scikit-learndoc" # If true, the reST sources are included in the HTML build as _sources/name. html_copy_source = True @@ -221,27 +420,111 @@ # index.html release_highlights_dir = Path("..") / "examples" / "release_highlights" # Finds the highlight with the latest version number -latest_highlights = sorted(release_highlights_dir.glob( - "plot_release_highlights_*.py"))[-1] -latest_highlights = latest_highlights.with_suffix('').name -html_context["release_highlights"] = \ +latest_highlights = sorted(release_highlights_dir.glob("plot_release_highlights_*.py"))[ + -1 +] +latest_highlights = latest_highlights.with_suffix("").name +html_context["release_highlights"] = ( f"auto_examples/release_highlights/{latest_highlights}" +) -# get version from higlight name assuming highlights have the form +# get version from highlight name assuming highlights have the form # plot_release_highlights_0_22_0 highlight_version = ".".join(latest_highlights.split("_")[-3:-1]) html_context["release_highlights_version"] = highlight_version + +# redirects dictionary maps from old links to new links +redirects = { + "documentation": "index", + "contents": "index", + "preface": "index", + "modules/classes": "api/index", + "tutorial/machine_learning_map/index": "machine_learning_map", + "auto_examples/feature_selection/plot_permutation_test_for_classification": ( + "auto_examples/model_selection/plot_permutation_tests_for_classification" + ), + "modules/model_persistence": "model_persistence", + "auto_examples/linear_model/plot_bayesian_ridge": ( + "auto_examples/linear_model/plot_ard" + ), + "auto_examples/model_selection/grid_search_text_feature_extraction": ( + "auto_examples/model_selection/plot_grid_search_text_feature_extraction" + ), + "auto_examples/model_selection/plot_validation_curve": ( + "auto_examples/model_selection/plot_train_error_vs_test_error" + ), + "auto_examples/datasets/plot_digits_last_image": ( + "auto_examples/exercises/plot_digits_classification_exercises" + ), + "auto_examples/datasets/plot_random_dataset": ( + "auto_examples/classification/plot_classifier_comparison" + ), + "auto_examples/miscellaneous/plot_changed_only_pprint_parameter": ( + "auto_examples/miscellaneous/plot_estimator_representation" + ), + "auto_examples/decomposition/plot_beta_divergence": ( + "auto_examples/applications/plot_topics_extraction_with_nmf_lda" + ), + "auto_examples/svm/plot_svm_nonlinear": "auto_examples/svm/plot_svm_kernels", + "auto_examples/ensemble/plot_adaboost_hastie_10_2": ( + "auto_examples/ensemble/plot_adaboost_multiclass" + ), + "auto_examples/decomposition/plot_pca_3d": ( + "auto_examples/decomposition/plot_pca_iris" + ), + "auto_examples/exercises/plot_cv_digits": ( + "auto_examples/model_selection/plot_nested_cross_validation_iris" + ), + "auto_examples/linear_model/plot_lasso_lars": ( + "auto_examples/linear_model/plot_lasso_lasso_lars_elasticnet_path" + ), + "auto_examples/linear_model/plot_lasso_coordinate_descent_path": ( + "auto_examples/linear_model/plot_lasso_lasso_lars_elasticnet_path" + ), + "auto_examples/cluster/plot_color_quantization": ( + "auto_examples/cluster/plot_face_compress" + ), + "auto_examples/cluster/plot_cluster_iris": ( + "auto_examples/cluster/plot_kmeans_assumptions" + ), + "auto_examples/ensemble/plot_forest_importances_faces": ( + "auto_examples/ensemble/plot_forest_importances" + ), + "auto_examples/ensemble/plot_voting_probas": ( + "auto_examples/ensemble/plot_voting_decision_regions" + ), + "auto_examples/datasets/plot_iris_dataset": ( + "auto_examples/decomposition/plot_pca_iris" + ), + "auto_examples/linear_model/plot_iris_logistic": ( + "auto_examples/linear_model/plot_logistic_multinomial" + ), + "auto_examples/linear_model/plot_ols_3d": ("auto_examples/linear_model/plot_ols"), + "auto_examples/linear_model/plot_ols": "auto_examples/linear_model/plot_ols_ridge", + "auto_examples/linear_model/plot_ols_ridge_variance": ( + "auto_examples/linear_model/plot_ols_ridge" + ), + "auto_examples/linear_model/plot_sgd_comparison": ( + "auto_examples/linear_model/plot_sgd_loss_functions" + ), +} +html_context["redirects"] = redirects +for old_link in redirects: + html_additional_pages[old_link] = "redirects.html" + +# See https://github.com/scikit-learn/scikit-learn/pull/22550 +html_context["is_devrelease"] = parsed_version.is_devrelease + + # -- Options for LaTeX output ------------------------------------------------ latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. - 'preamble': r""" + "preamble": r""" \usepackage{amsmath}\usepackage{amsfonts}\usepackage{bm} \usepackage{morefloats}\usepackage{enumitem} \setlistdepth{10} \let\oldhref\href @@ -252,8 +535,15 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass # [howto/manual]). -latex_documents = [('contents', 'user_guide.tex', 'scikit-learn user guide', - 'scikit-learn developers', 'manual'), ] +latex_documents = [ + ( + "contents", + "user_guide.tex", + "scikit-learn user guide", + "scikit-learn developers", + "manual", + ), +] # The name of an image file (relative to this directory) to place at the top of # the title page. @@ -269,27 +559,27 @@ # intersphinx configuration intersphinx_mapping = { - 'python': ('https://docs.python.org/{.major}'.format( - sys.version_info), None), - 'numpy': ('https://numpy.org/doc/stable', None), - 'scipy': ('https://docs.scipy.org/doc/scipy/reference', None), - 'matplotlib': ('https://matplotlib.org/', None), - 'pandas': ('https://pandas.pydata.org/pandas-docs/stable/', None), - 'joblib': ('https://joblib.readthedocs.io/en/latest/', None), - 'seaborn': ('https://seaborn.pydata.org/', None), + "python": ("https://docs.python.org/{.major}".format(sys.version_info), None), + "numpy": ("https://numpy.org/doc/stable", None), + "scipy": ("https://docs.scipy.org/doc/scipy/", None), + "matplotlib": ("https://matplotlib.org/", None), + "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None), + "joblib": ("https://joblib.readthedocs.io/en/latest/", None), + "seaborn": ("https://seaborn.pydata.org/", None), + "skops": ("https://skops.readthedocs.io/en/stable/", None), } v = parse(release) if v.release is None: raise ValueError( - 'Ill-formed version: {!r}. Version should follow ' - 'PEP440'.format(version)) + "Ill-formed version: {!r}. Version should follow PEP440".format(version) + ) if v.is_devrelease: - binder_branch = 'main' + binder_branch = "main" else: major, minor = v.release[:2] - binder_branch = '{}.{}.X'.format(major, minor) + binder_branch = "{}.{}.X".format(major, minor) class SubSectionTitleOrder: @@ -298,12 +588,13 @@ class SubSectionTitleOrder: Assumes README.txt exists for all subsections and uses the subsection with dashes, '---', as the adornment. """ + def __init__(self, src_dir): self.src_dir = src_dir self.regex = re.compile(r"^([\w ]+)\n-", re.MULTILINE) def __repr__(self): - return '<%s>' % (self.__class__.__name__,) + return "<%s>" % (self.__class__.__name__,) def __call__(self, directory): src_path = os.path.normpath(os.path.join(self.src_dir, directory)) @@ -315,7 +606,7 @@ def __call__(self, directory): readme = os.path.join(src_path, "README.txt") try: - with open(readme, 'r') as f: + with open(readme, "r") as f: content = f.read() except FileNotFoundError: return directory @@ -326,54 +617,178 @@ def __call__(self, directory): return directory +class SKExampleTitleSortKey(ExampleTitleSortKey): + """Sorts release highlights based on version number.""" + + def __call__(self, filename): + title = super().__call__(filename) + prefix = "plot_release_highlights_" + + # Use title to sort if not a release highlight + if not str(filename).startswith(prefix): + return title + + major_minor = filename[len(prefix) :].split("_")[:2] + version_float = float(".".join(major_minor)) + + # negate to place the newest version highlights first + return -version_float + + +def notebook_modification_function(notebook_content, notebook_filename): + notebook_content_str = str(notebook_content) + warning_template = "\n".join( + [ + "
", + "", + "# JupyterLite warning", + "", + "{message}", + "
", + ] + ) + + message_class = "warning" + message = ( + "Running the scikit-learn examples in JupyterLite is experimental and you may" + " encounter some unexpected behavior.\n\nThe main difference is that imports" + " will take a lot longer than usual, for example the first `import sklearn` can" + " take roughly 10-20s.\n\nIf you notice problems, feel free to open an" + " [issue](https://github.com/scikit-learn/scikit-learn/issues/new/choose)" + " about it." + ) + + markdown = warning_template.format(message_class=message_class, message=message) + + dummy_notebook_content = {"cells": []} + add_markdown_cell(dummy_notebook_content, markdown) + + code_lines = [] + + if "seaborn" in notebook_content_str: + code_lines.append("%pip install seaborn") + if "plotly.express" in notebook_content_str: + code_lines.append("%pip install plotly nbformat") + if "skimage" in notebook_content_str: + code_lines.append("%pip install scikit-image") + if "polars" in notebook_content_str: + code_lines.append("%pip install polars") + if "fetch_" in notebook_content_str: + code_lines.extend( + [ + "%pip install pyodide-http", + "import pyodide_http", + "pyodide_http.patch_all()", + ] + ) + # always import matplotlib and pandas to avoid Pyodide limitation with + # imports inside functions + code_lines.extend(["import matplotlib", "import pandas"]) + + # Work around https://github.com/jupyterlite/pyodide-kernel/issues/166 + # and https://github.com/pyodide/micropip/issues/223 by installing the + # dependencies first, and then scikit-learn from Anaconda.org. + if "dev" in release: + dev_docs_specific_code = [ + "import piplite", + "import joblib", + "import threadpoolctl", + "import scipy", + "await piplite.install(\n" + f" 'scikit-learn=={release}',\n" + " index_urls='https://pypi.anaconda.org/scientific-python-nightly-wheels/simple',\n" + ")", + ] + + code_lines.extend(dev_docs_specific_code) + + if code_lines: + code_lines = ["# JupyterLite-specific code"] + code_lines + code = "\n".join(code_lines) + add_code_cell(dummy_notebook_content, code) + + notebook_content["cells"] = ( + dummy_notebook_content["cells"] + notebook_content["cells"] + ) + + +default_global_config = sklearn.get_config() + + +def reset_sklearn_config(gallery_conf, fname): + """Reset sklearn config to default values.""" + sklearn.set_config(**default_global_config) + + +sg_examples_dir = "../examples" +sg_gallery_dir = "auto_examples" sphinx_gallery_conf = { - 'doc_module': 'sklearn', - 'backreferences_dir': os.path.join('modules', 'generated'), - 'show_memory': False, - 'reference_url': { - 'sklearn': None}, - 'examples_dirs': ['../examples'], - 'gallery_dirs': ['auto_examples'], - 'subsection_order': SubSectionTitleOrder('../examples'), - 'binder': { - 'org': 'scikit-learn', - 'repo': 'scikit-learn', - 'binderhub_url': 'https://mybinder.org', - 'branch': binder_branch, - 'dependencies': './binder/requirements.txt', - 'use_jupyter_lab': True + "doc_module": "sklearn", + "backreferences_dir": os.path.join("modules", "generated"), + "show_memory": False, + "reference_url": {"sklearn": None}, + "examples_dirs": [sg_examples_dir], + "gallery_dirs": [sg_gallery_dir], + "subsection_order": SubSectionTitleOrder(sg_examples_dir), + "within_subsection_order": SKExampleTitleSortKey, + "binder": { + "org": "scikit-learn", + "repo": "scikit-learn", + "binderhub_url": "https://mybinder.org", + "branch": binder_branch, + "dependencies": "./binder/requirements.txt", + "use_jupyter_lab": True, }, # avoid generating too many cross links - 'inspect_global_variables': False, - 'remove_config_comments': True, + "inspect_global_variables": False, + "remove_config_comments": True, + "plot_gallery": "True", + "recommender": {"enable": True, "n_examples": 4, "min_df": 12}, + "reset_modules": ("matplotlib", "seaborn", reset_sklearn_config), } +if with_jupyterlite: + sphinx_gallery_conf["jupyterlite"] = { + "notebook_modification_function": notebook_modification_function + } + +# For the index page of the gallery and each nested section, we hide the secondary +# sidebar by specifying an empty list (no components), because there is no meaningful +# in-page toc for these pages, and they are generated so "sourcelink" is not useful +# either. +html_theme_options["secondary_sidebar_items"][f"{sg_gallery_dir}/index"] = [] +for sub_sg_dir in (Path(".") / sg_examples_dir).iterdir(): + if sub_sg_dir.is_dir(): + html_theme_options["secondary_sidebar_items"][ + f"{sg_gallery_dir}/{sub_sg_dir.name}/index" + ] = [] # The following dictionary contains the information used to create the # thumbnails for the front page of the scikit-learn home page. # key: first image in set # values: (number of plot in set, height of thumbnail) -carousel_thumbs = {'sphx_glr_plot_classifier_comparison_001.png': 600} +carousel_thumbs = {"sphx_glr_plot_classifier_comparison_001.png": 600} # enable experimental module so that experimental estimators can be # discovered properly by sphinx -from sklearn.experimental import enable_hist_gradient_boosting # noqa -from sklearn.experimental import enable_iterative_imputer # noqa -from sklearn.experimental import enable_halving_search_cv # noqa +from sklearn.experimental import ( # noqa: F401 + enable_halving_search_cv, + enable_iterative_imputer, +) def make_carousel_thumbs(app, exception): """produces the final resized carousel images""" if exception is not None: return - print('Preparing carousel images') + print("Preparing carousel images") - image_dir = os.path.join(app.builder.outdir, '_images') + image_dir = os.path.join(app.builder.outdir, "_images") for glr_plot, max_width in carousel_thumbs.items(): image = os.path.join(image_dir, glr_plot) if os.path.exists(image): - c_thumb = os.path.join(image_dir, glr_plot[:-4] + '_carousel.png') + c_thumb = os.path.join(image_dir, glr_plot[:-4] + "_carousel.png") sphinx_gallery.gen_rst.scale_image(image, c_thumb, max_width, 190) @@ -382,129 +797,77 @@ def filter_search_index(app, exception): return # searchindex only exist when generating html - if app.builder.name != 'html': + if app.builder.name != "html": return - print('Removing methods from search index') + print("Removing methods from search index") - searchindex_path = os.path.join(app.builder.outdir, 'searchindex.js') - with open(searchindex_path, 'r') as f: + searchindex_path = os.path.join(app.builder.outdir, "searchindex.js") + with open(searchindex_path, "r") as f: searchindex_text = f.read() - searchindex_text = re.sub(r'{__init__.+?}', '{}', searchindex_text) - searchindex_text = re.sub(r'{__call__.+?}', '{}', searchindex_text) + searchindex_text = re.sub(r"{__init__.+?}", "{}", searchindex_text) + searchindex_text = re.sub(r"{__call__.+?}", "{}", searchindex_text) - with open(searchindex_path, 'w') as f: + with open(searchindex_path, "w") as f: f.write(searchindex_text) -def generate_min_dependency_table(app): - """Generate min dependency table for docs.""" - from sklearn._min_dependencies import dependent_packages - - # get length of header - package_header_len = max(len(package) - for package in dependent_packages) + 4 - version_header_len = len('Minimum Version') + 4 - tags_header_len = max(len(tags) - for _, tags in dependent_packages.values()) + 4 - - output = StringIO() - output.write(' '.join(['=' * package_header_len, - '=' * version_header_len, - '=' * tags_header_len])) - output.write('\n') - dependency_title = "Dependency" - version_title = "Minimum Version" - tags_title = "Purpose" - - output.write(f'{dependency_title:<{package_header_len}} ' - f'{version_title:<{version_header_len}} ' - f'{tags_title}\n') - - output.write(' '.join(['=' * package_header_len, - '=' * version_header_len, - '=' * tags_header_len])) - output.write('\n') - - for package, (version, tags) in dependent_packages.items(): - output.write(f'{package:<{package_header_len}} ' - f'{version:<{version_header_len}} ' - f'{tags}\n') - - output.write(' '.join(['=' * package_header_len, - '=' * version_header_len, - '=' * tags_header_len])) - output.write('\n') - output = output.getvalue() - - with (Path('.') / 'min_dependency_table.rst').open('w') as f: - f.write(output) - - -def generate_min_dependency_substitutions(app): - """Generate min dependency substitutions for docs.""" - from sklearn._min_dependencies import dependent_packages - - output = StringIO() - - for package, (version, _) in dependent_packages.items(): - package = package.capitalize() - output.write(f'.. |{package}MinVersion| replace:: {version}') - output.write('\n') - - output = output.getvalue() - - with (Path('.') / 'min_dependency_substitutions.rst').open('w') as f: - f.write(output) - - # Config for sphinx_issues # we use the issues path for PRs since the issues URL will forward -issues_github_path = 'scikit-learn/scikit-learn' +issues_github_path = "scikit-learn/scikit-learn" -# Hack to get kwargs to appear in docstring #18434 -# TODO: Remove when https://github.com/sphinx-doc/sphinx/pull/8234 gets -# merged -from sphinx.util import inspect # noqa -from sphinx.ext.autodoc import ClassDocumenter # noqa +def disable_plot_gallery_for_linkcheck(app): + if app.builder.name == "linkcheck": + sphinx_gallery_conf["plot_gallery"] = "False" -class PatchedClassDocumenter(ClassDocumenter): - def _get_signature(self): - old_signature = inspect.signature +def skip_properties(app, what, name, obj, skip, options): + """Skip properties that are fitted attributes""" + if isinstance(obj, property): + if name.endswith("_") and not name.startswith("_"): + return True - def patch_signature(subject, bound_method=False, follow_wrapped=True): - # changes the default of follow_wrapped to True - return old_signature(subject, bound_method=bound_method, - follow_wrapped=follow_wrapped) - inspect.signature = patch_signature - result = super()._get_signature() - inspect.signature = old_signature - return result + return skip def setup(app): - app.registry.documenters['class'] = PatchedClassDocumenter - app.connect('builder-inited', generate_min_dependency_table) - app.connect('builder-inited', generate_min_dependency_substitutions) - # to hide/show the prompt in code examples: - app.connect('build-finished', make_carousel_thumbs) - app.connect('build-finished', filter_search_index) + # do not run the examples when using linkcheck by using a small priority + # (default priority is 500 and sphinx-gallery using builder-inited event too) + app.connect("builder-inited", disable_plot_gallery_for_linkcheck, priority=50) + # triggered just before the HTML for an individual page is created + app.connect("html-page-context", add_js_css_files) -# The following is used by sphinx.ext.linkcode to provide links to github -linkcode_resolve = make_linkcode_resolve('sklearn', - 'https://github.com/scikit-learn/' - 'scikit-learn/blob/{revision}/' - '{package}/{path}#L{lineno}') + # to hide/show the prompt in code examples + app.connect("build-finished", make_carousel_thumbs) + app.connect("build-finished", filter_search_index) + + app.connect("autodoc-skip-member", skip_properties) -warnings.filterwarnings("ignore", category=UserWarning, - message='Matplotlib is currently using agg, which is a' - ' non-GUI backend, so cannot show the figure.') +# The following is used by sphinx.ext.linkcode to provide links to github +linkcode_resolve = make_linkcode_resolve( + "sklearn", + ( + "https://github.com/scikit-learn/" + "scikit-learn/blob/{revision}/" + "{package}/{path}#L{lineno}" + ), +) + +warnings.filterwarnings( + "ignore", + category=UserWarning, + message=( + "Matplotlib is currently using agg, which is a" + " non-GUI backend, so cannot show the figure." + ), +) +if os.environ.get("SKLEARN_WARNINGS_AS_ERRORS", "0") != "0": + turn_warnings_into_errors() # maps functions with a class name that is indistinguishable when case is # ignore to another filename @@ -513,3 +876,217 @@ def setup(app): "sklearn.covariance.oas": "oas-function", "sklearn.decomposition.fastica": "fastica-function", } + + +# Config for sphinxext.opengraph + +ogp_site_url = "https://scikit-learn/stable/" +ogp_image = "https://scikit-learn.org/stable/_static/scikit-learn-logo-small.png" +ogp_use_first_image = True +ogp_site_name = "scikit-learn" + +# Config for linkcheck that checks the documentation for broken links + +# ignore all links in 'whats_new' to avoid doing many github requests and +# hitting the github rate threshold that makes linkcheck take a lot of time +linkcheck_exclude_documents = [r"whats_new/.*"] + +# default timeout to make some sites links fail faster +linkcheck_timeout = 10 + +# Allow redirects from doi.org +linkcheck_allowed_redirects = {r"https://doi.org/.+": r".*"} +linkcheck_ignore = [ + # ignore links to local html files e.g. in image directive :target: field + r"^..?/", + # ignore links to specific pdf pages because linkcheck does not handle them + # ('utf-8' codec can't decode byte error) + r"http://www.utstat.toronto.edu/~rsalakhu/sta4273/notes/Lecture2.pdf#page=.*", + ( + "https://www.fordfoundation.org/media/2976/roads-and-bridges" + "-the-unseen-labor-behind-our-digital-infrastructure.pdf#page=.*" + ), + # links falsely flagged as broken + ( + "https://www.researchgate.net/publication/" + "233096619_A_Dendrite_Method_for_Cluster_Analysis" + ), + ( + "https://www.researchgate.net/publication/221114584_Random_Fourier" + "_Approximations_for_Skewed_Multiplicative_Histogram_Kernels" + ), + ( + "https://www.researchgate.net/publication/4974606_" + "Hedonic_housing_prices_and_the_demand_for_clean_air" + ), + ( + "https://www.researchgate.net/profile/Anh-Huy-Phan/publication/220241471_Fast_" + "Local_Algorithms_for_Large_Scale_Nonnegative_Matrix_and_Tensor_Factorizations" + ), + "https://doi.org/10.13140/RG.2.2.35280.02565", + ( + "https://www.microsoft.com/en-us/research/uploads/prod/2006/01/" + "Bishop-Pattern-Recognition-and-Machine-Learning-2006.pdf" + ), + "https://www.microsoft.com/en-us/research/wp-content/uploads/2016/02/tr-99-87.pdf", + "https://microsoft.com/", + "https://www.jstor.org/stable/2984099", + "https://stat.uw.edu/sites/default/files/files/reports/2000/tr371.pdf", + # Broken links from testimonials + "http://www.bestofmedia.com", + "http://www.data-publica.com/", + "https://livelovely.com", + "https://www.mars.com/global", + "https://www.yhat.com", + # Ignore some dynamically created anchors. See + # https://github.com/sphinx-doc/sphinx/issues/9016 for more details about + # the github example + r"https://github.com/conda-forge/miniforge#miniforge", + r"https://github.com/joblib/threadpoolctl/" + "#setting-the-maximum-size-of-thread-pools", + r"https://stackoverflow.com/questions/5836335/" + "consistently-create-same-random-numpy-array/5837352#comment6712034_5837352", +] + +# Use a browser-like user agent to avoid some "403 Client Error: Forbidden for +# url" errors. This is taken from the variable navigator.userAgent inside a +# browser console. +user_agent = ( + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:100.0) Gecko/20100101 Firefox/100.0" +) + +# Use Github token from environment variable to avoid Github rate limits when +# checking Github links +github_token = os.getenv("GITHUB_TOKEN") + +if github_token is None: + linkcheck_request_headers = {} +else: + linkcheck_request_headers = { + "https://github.com/": {"Authorization": f"token {github_token}"}, + } + + +def infer_next_release_versions(): + """Infer the most likely next release versions to make.""" + all_version_full = {"rc": "0.99.0rc1", "final": "0.99.0", "bf": "0.98.1"} + all_version_short = {"rc": "0.99", "final": "0.99", "bf": "0.98"} + all_previous_tag = {"rc": "unused", "final": "0.98.33", "bf": "0.97.22"} + + try: + # Fetch the version switcher JSON; see `html_theme_options` for more details + versions_json = json.loads( + urlopen(html_theme_options["switcher"]["json_url"], timeout=10).read() + ) + + # See `build_tools/circle/list_versions.py`, stable is always the second entry + stable_version = parse(versions_json[1]["version"]) + last_stable_version = parse(versions_json[2]["version"]) + next_major_minor = f"{stable_version.major}.{stable_version.minor + 1}" + + # RC + all_version_full["rc"] = f"{next_major_minor}.0rc1" + all_version_short["rc"] = next_major_minor + + # Major/Minor final + all_version_full["final"] = f"{next_major_minor}.0" + all_version_short["final"] = next_major_minor + all_previous_tag["final"] = stable_version.base_version + + # Bug-fix + all_version_full["bf"] = ( + f"{stable_version.major}.{stable_version.minor}.{stable_version.micro + 1}" + ) + all_version_short["bf"] = f"{stable_version.major}.{stable_version.minor}" + all_previous_tag["bf"] = last_stable_version.base_version + except Exception as e: + logger.warning( + "Failed to infer all possible next release versions because of " + f"{type(e).__name__}: {e}" + ) + + return { + "version_full": all_version_full, + "version_short": all_version_short, + "previous_tag": all_previous_tag, + } + + +# -- Convert .rst.template files to .rst --------------------------------------- + +from api_reference import API_REFERENCE, DEPRECATED_API_REFERENCE + +from sklearn._min_dependencies import dependent_packages + +# If development build, link to local page in the top navbar; otherwise link to the +# development version; see https://github.com/scikit-learn/scikit-learn/pull/22550 +if parsed_version.is_devrelease: + development_link = "developers/index" +else: + development_link = "https://scikit-learn.org/dev/developers/index.html" + +# Define the templates and target files for conversion +# Each entry is in the format (template name, file name, kwargs for rendering) +rst_templates = [ + ("index", "index", {"development_link": development_link}), + ( + "developers/maintainer", + "developers/maintainer", + {"inferred": infer_next_release_versions()}, + ), + ( + "min_dependency_table", + "min_dependency_table", + {"dependent_packages": dependent_packages}, + ), + ( + "min_dependency_substitutions", + "min_dependency_substitutions", + {"dependent_packages": dependent_packages}, + ), + ( + "api/index", + "api/index", + { + "API_REFERENCE": sorted(API_REFERENCE.items(), key=lambda x: x[0]), + "DEPRECATED_API_REFERENCE": sorted( + DEPRECATED_API_REFERENCE.items(), key=lambda x: x[0], reverse=True + ), + }, + ), +] + +# Convert each module API reference page +for module in API_REFERENCE: + rst_templates.append( + ( + "api/module", + f"api/{module}", + {"module": module, "module_info": API_REFERENCE[module]}, + ) + ) + +# Convert the deprecated API reference page (if there exists any) +if DEPRECATED_API_REFERENCE: + rst_templates.append( + ( + "api/deprecated", + "api/deprecated", + { + "DEPRECATED_API_REFERENCE": sorted( + DEPRECATED_API_REFERENCE.items(), key=lambda x: x[0], reverse=True + ) + }, + ) + ) + +for rst_template_name, rst_target_name, kwargs in rst_templates: + # Read the corresponding template file into jinja2 + with (Path(".") / f"{rst_template_name}.rst.template").open( + "r", encoding="utf-8" + ) as f: + t = jinja2.Template(f.read()) + + # Render the template and write to the target + with (Path(".") / f"{rst_target_name}.rst").open("w", encoding="utf-8") as f: + f.write(t.render(**kwargs)) diff --git a/doc/conftest.py b/doc/conftest.py index 5468184bf5509..ad8d6eb8cfb62 100644 --- a/doc/conftest.py +++ b/doc/conftest.py @@ -1,20 +1,20 @@ import os -from os.path import exists -from os.path import join from os import environ -import warnings +from os.path import exists, join + +import pytest +from _pytest.doctest import DoctestItem -from sklearn.utils import IS_PYPY -from sklearn.utils._testing import SkipTest -from sklearn.utils._testing import check_skip_network from sklearn.datasets import get_data_home from sklearn.datasets._base import _pkl_filepath from sklearn.datasets._twenty_newsgroups import CACHE_NAME +from sklearn.utils._testing import SkipTest, check_skip_network +from sklearn.utils.fixes import np_base_version, parse_version, sp_version def setup_labeled_faces(): data_home = get_data_home() - if not exists(join(data_home, 'lfw_home')): + if not exists(join(data_home, "lfw_home")): raise SkipTest("Skipping dataset loading doctests") @@ -27,15 +27,12 @@ def setup_rcv1(): def setup_twenty_newsgroups(): - data_home = get_data_home() cache_path = _pkl_filepath(get_data_home(), CACHE_NAME) if not exists(cache_path): raise SkipTest("Skipping dataset loading doctests") def setup_working_with_text_data(): - if IS_PYPY and os.environ.get('CI', None): - raise SkipTest('Skipping too slow test with PyPy on CI') check_skip_network() cache_path = _pkl_filepath(get_data_home(), CACHE_NAME) if not exists(cache_path): @@ -44,80 +41,139 @@ def setup_working_with_text_data(): def setup_loading_other_datasets(): try: - import pandas # noqa + import pandas # noqa: F401 except ImportError: - raise SkipTest("Skipping loading_other_datasets.rst, " - "pandas not installed") + raise SkipTest("Skipping loading_other_datasets.rst, pandas not installed") # checks SKLEARN_SKIP_NETWORK_TESTS to see if test should run - run_network_tests = environ.get("SKLEARN_SKIP_NETWORK_TESTS", '1') == "0" + run_network_tests = environ.get("SKLEARN_SKIP_NETWORK_TESTS", "1") == "0" if not run_network_tests: - raise SkipTest("Skipping loading_other_datasets.rst, tests can be " - "enabled by settting SKLEARN_SKIP_NETWORK_TESTS=0") + raise SkipTest( + "Skipping loading_other_datasets.rst, tests can be " + "enabled by setting SKLEARN_SKIP_NETWORK_TESTS=0" + ) def setup_compose(): try: - import pandas # noqa + import pandas # noqa: F401 except ImportError: raise SkipTest("Skipping compose.rst, pandas not installed") def setup_impute(): try: - import pandas # noqa + import pandas # noqa: F401 except ImportError: raise SkipTest("Skipping impute.rst, pandas not installed") def setup_grid_search(): try: - import pandas # noqa + import pandas # noqa: F401 except ImportError: raise SkipTest("Skipping grid_search.rst, pandas not installed") def setup_preprocessing(): try: - import pandas # noqa + import pandas # noqa: F401 except ImportError: raise SkipTest("Skipping preprocessing.rst, pandas not installed") -def setup_unsupervised_learning(): +def skip_if_matplotlib_not_installed(fname): try: - import skimage # noqa + import matplotlib # noqa: F401 except ImportError: - raise SkipTest("Skipping unsupervised_learning.rst, scikit-image " - "not installed") - # ignore deprecation warnings from scipy.misc.face - warnings.filterwarnings('ignore', 'The binary mode of fromstring', - DeprecationWarning) + basename = os.path.basename(fname) + raise SkipTest(f"Skipping doctests for {basename}, matplotlib not installed") + + +def skip_if_cupy_not_installed(fname): + try: + import cupy # noqa: F401 + except ImportError: + basename = os.path.basename(fname) + raise SkipTest(f"Skipping doctests for {basename}, cupy not installed") def pytest_runtest_setup(item): fname = item.fspath.strpath - is_index = fname.endswith('datasets/index.rst') - if fname.endswith('datasets/labeled_faces.rst') or is_index: + # normalize filename to use forward slashes on Windows for easier handling + # later + fname = fname.replace(os.sep, "/") + + is_index = fname.endswith("datasets/index.rst") + if fname.endswith("datasets/labeled_faces.rst") or is_index: setup_labeled_faces() - elif fname.endswith('datasets/rcv1.rst') or is_index: + elif fname.endswith("datasets/rcv1.rst") or is_index: setup_rcv1() - elif fname.endswith('datasets/twenty_newsgroups.rst') or is_index: + elif fname.endswith("datasets/twenty_newsgroups.rst") or is_index: setup_twenty_newsgroups() - elif fname.endswith('tutorial/text_analytics/working_with_text_data.rst')\ - or is_index: - setup_working_with_text_data() - elif fname.endswith('modules/compose.rst') or is_index: + elif fname.endswith("modules/compose.rst") or is_index: setup_compose() - elif IS_PYPY and fname.endswith('modules/feature_extraction.rst'): - raise SkipTest('FeatureHasher is not compatible with PyPy') - elif fname.endswith('datasets/loading_other_datasets.rst'): + elif fname.endswith("datasets/loading_other_datasets.rst"): setup_loading_other_datasets() - elif fname.endswith('modules/impute.rst'): + elif fname.endswith("modules/impute.rst"): setup_impute() - elif fname.endswith('modules/grid_search.rst'): + elif fname.endswith("modules/grid_search.rst"): setup_grid_search() - elif fname.endswith('modules/preprocessing.rst'): + elif fname.endswith("modules/preprocessing.rst"): setup_preprocessing() - elif fname.endswith('statistical_inference/unsupervised_learning.rst'): - setup_unsupervised_learning() + + rst_files_requiring_matplotlib = [ + "modules/partial_dependence.rst", + "modules/tree.rst", + ] + for each in rst_files_requiring_matplotlib: + if fname.endswith(each): + skip_if_matplotlib_not_installed(fname) + + if fname.endswith("array_api.rst"): + skip_if_cupy_not_installed(fname) + + +def pytest_configure(config): + # Use matplotlib agg backend during the tests including doctests + try: + import matplotlib + + matplotlib.use("agg") + except ImportError: + pass + + +def pytest_collection_modifyitems(config, items): + """Called after collect is completed. + + Parameters + ---------- + config : pytest config + items : list of collected items + """ + skip_doctests = False + if np_base_version < parse_version("2"): + # TODO: configure numpy to output scalar arrays as regular Python scalars + # once possible to improve readability of the tests docstrings. + # https://numpy.org/neps/nep-0051-scalar-representation.html#implementation + reason = "Due to NEP 51 numpy scalar repr has changed in numpy 2" + skip_doctests = True + + if sp_version < parse_version("1.14"): + reason = "Scipy sparse matrix repr has changed in scipy 1.14" + skip_doctests = True + + # Normally doctest has the entire module's scope. Here we set globs to an empty dict + # to remove the module's scope: + # https://docs.python.org/3/library/doctest.html#what-s-the-execution-context + for item in items: + if isinstance(item, DoctestItem): + item.dtest.globs = {} + + if skip_doctests: + skip_marker = pytest.mark.skip(reason=reason) + + for item in items: + if isinstance(item, DoctestItem): + item.add_marker(skip_marker) diff --git a/doc/contents.rst b/doc/contents.rst deleted file mode 100644 index a28634621d558..0000000000000 --- a/doc/contents.rst +++ /dev/null @@ -1,24 +0,0 @@ -.. include:: includes/big_toc_css.rst -.. include:: tune_toc.rst - -.. Places global toc into the sidebar - -:globalsidebartoc: True - -================= -Table Of Contents -================= - -.. Define an order for the Table of Contents: - -.. toctree:: - :maxdepth: 2 - - preface - tutorial/index - getting_started - user_guide - glossary - auto_examples/index - modules/classes - developers/index diff --git a/doc/contributor_experience_team.rst b/doc/contributor_experience_team.rst new file mode 100644 index 0000000000000..73ccd668b20cd --- /dev/null +++ b/doc/contributor_experience_team.rst @@ -0,0 +1,52 @@ +.. raw :: html + + +
+ +
+
+

Virgil Chan

+
+
+
+

Juan Carlos Alfaro JimÊnez

+
+
+
+

Lucy Liu

+
+
+
+

Maxwell Liu

+
+
+
+

Juan Martin Loyola

+
+
+
+

Sylvain MariÊ

+
+
+
+

Norbert Preining

+
+
+
+

Stefanie Senger

+
+
+
+

Reshama Shaikh

+
+
+
+

Albert Thomas

+
+
+
+

Maren Westermann

+
+
diff --git a/doc/contributor_experience_team_emeritus.rst b/doc/contributor_experience_team_emeritus.rst new file mode 100644 index 0000000000000..a833907dd5e4a --- /dev/null +++ b/doc/contributor_experience_team_emeritus.rst @@ -0,0 +1 @@ +- Chiara Marmo diff --git a/doc/themes/scikit-learn/static/css/examples.css b/doc/css/.gitkeep similarity index 100% rename from doc/themes/scikit-learn/static/css/examples.css rename to doc/css/.gitkeep diff --git a/doc/data_transforms.rst b/doc/data_transforms.rst index 084214cb094f5..536539ec97007 100644 --- a/doc/data_transforms.rst +++ b/doc/data_transforms.rst @@ -1,9 +1,3 @@ -.. Places parent toc into the sidebar - -:parenttoc: True - -.. include:: includes/big_toc_css.rst - .. _data-transforms: Dataset transformations diff --git a/doc/datasets.rst b/doc/datasets.rst index b9484a02ce84c..f12e5095cc6a8 100644 --- a/doc/datasets.rst +++ b/doc/datasets.rst @@ -1,9 +1,3 @@ -.. Places parent toc into the sidebar - -:parenttoc: True - -.. include:: includes/big_toc_css.rst - .. _datasets: ========================= @@ -12,12 +6,9 @@ Dataset loading utilities .. currentmodule:: sklearn.datasets -The ``sklearn.datasets`` package embeds some small toy datasets -as introduced in the :ref:`Getting Started ` section. - -This package also features helpers to fetch larger datasets commonly -used by the machine learning community to benchmark algorithms on data -that comes from the 'real world'. +The ``sklearn.datasets`` package embeds some small toy datasets and provides helpers +to fetch larger datasets commonly used by the machine learning community to benchmark +algorithms on data that comes from the 'real world'. To evaluate the impact of the scale of the dataset (``n_samples`` and ``n_features``) while controlling the statistical properties of the data @@ -42,7 +33,7 @@ length ``n_samples``, containing the target values, with key ``target``. The Bunch object is a dictionary that exposes its keys as attributes. For more information about Bunch object, see :class:`~sklearn.utils.Bunch`. -It's also possible for almost all of these function to constrain the output +It's also possible for almost all of these functions to constrain the output to be a tuple containing only the data and the target, by setting the ``return_X_y`` parameter to ``True``. diff --git a/doc/datasets/loading_other_datasets.rst b/doc/datasets/loading_other_datasets.rst index 131d6ca9757f5..84d042f64c9d3 100644 --- a/doc/datasets/loading_other_datasets.rst +++ b/doc/datasets/loading_other_datasets.rst @@ -1,7 +1,3 @@ -.. Places parent toc into the sidebar - -:parenttoc: True - .. _loading_other_datasets: Loading other datasets @@ -23,24 +19,29 @@ and pipelines on 2D data. load_sample_images load_sample_image -.. image:: ../auto_examples/cluster/images/sphx_glr_plot_color_quantization_001.png - :target: ../auto_examples/cluster/plot_color_quantization.html +.. plot:: + :context: close-figs :scale: 30 :align: right + :include-source: False + import matplotlib.pyplot as plt + from sklearn.datasets import load_sample_image + + china = load_sample_image("china.jpg") + plt.imshow(china) + plt.axis('off') + plt.tight_layout() + plt.show() .. warning:: The default coding of images is based on the ``uint8`` dtype to spare memory. Often machine learning algorithms work best if the input is converted to a floating point representation first. Also, - if you plan to use ``matplotlib.pyplpt.imshow``, don't forget to scale to the range + if you plan to use ``matplotlib.pyplot.imshow``, don't forget to scale to the range 0 - 1 as done in the following example. -.. topic:: Examples: - - * :ref:`sphx_glr_auto_examples_cluster_plot_color_quantization.py` - .. _libsvm_loader: Datasets in svmlight / libsvm format @@ -52,7 +53,7 @@ takes the form ``
- -Note that in order to avoid potential conflicts with other packages it is -strongly recommended to use a `virtual environment (venv) -`_ or a `conda environment -`_. - -Using such an isolated environment makes it possible to install a specific -version of scikit-learn with pip or conda and its dependencies independently of -any previously installed Python packages. In particular under Linux is it -discouraged to install pip packages alongside the packages managed by the + .. tab-item:: pip + :class-label: tab-6 + :sync: package-manager-pip + + Python 3 is usually installed by default on most Linux distributions. To + check if you have it installed, try: + + .. prompt:: bash + + python3 --version + pip3 --version + + If you don't have Python 3 installed, please install `python3` and + `python3-pip` from your distribution's package manager. + + Now create a `virtual environment (venv) + `_ and install scikit-learn. + Note that the virtual environment is optional but strongly recommended, in + order to avoid potential conflicts with other packages. + + .. prompt:: bash + + python3 -m venv sklearn-env + source sklearn-env/bin/activate # activate + pip3 install -U scikit-learn + + In order to check your installation, you can use: + + .. prompt:: bash + + python3 -m pip show scikit-learn # show scikit-learn version and location + python3 -m pip freeze # show all installed packages in the environment + python3 -c "import sklearn; sklearn.show_versions()" + + .. tab-item:: conda + :class-label: tab-6 + :sync: package-manager-conda + + .. include:: ./install_instructions_conda.rst + + +Using an isolated environment such as pip venv or conda makes it possible to +install a specific version of scikit-learn with pip or conda and its dependencies +independently of any previously installed Python packages. In particular under Linux +it is discouraged to install pip packages alongside the packages managed by the package manager of the distribution (apt, dnf, pacman...). Note that you should always remember to activate the environment of your choice @@ -128,11 +191,10 @@ and NumPy and SciPy are not recompiled from source, which can happen when using particular configurations of operating system and hardware (such as Linux on a Raspberry Pi). - -Scikit-learn plotting capabilities (i.e., functions start with "plot\_" -and classes end with "Display") require Matplotlib. The examples require +Scikit-learn plotting capabilities (i.e., functions starting with `plot\_` +and classes ending with `Display`) require Matplotlib. The examples require Matplotlib and some examples require scikit-image, pandas, or seaborn. The -minimum version of Scikit-learn dependencies are listed below along with its +minimum version of scikit-learn dependencies are listed below along with its purpose. .. include:: min_dependency_table.rst @@ -140,37 +202,24 @@ purpose. .. warning:: Scikit-learn 0.20 was the last version to support Python 2.7 and Python 3.4. - Scikit-learn 0.21 supported Python 3.5-3.7. - Scikit-learn 0.22 supported Python 3.5-3.8. - Scikit-learn now requires Python 3.6 or newer. - -.. note:: + Scikit-learn 0.21 supported Python 3.5—3.7. - For installing on PyPy, PyPy3-v5.10+, Numpy 1.14.0+, and scipy 1.1.0+ - are required. + Scikit-learn 0.22 supported Python 3.5—3.8. -.. _install_on_apple_silicon_m1: + Scikit-learn 0.23 required Python 3.6—3.8. -Installing on Apple Silicon M1 hardware -======================================= + Scikit-learn 0.24 required Python 3.6—3.9. -The recently introduced `macos/arm64` platform (sometimes also known as -`macos/aarch64`) requires the open source community to upgrade the build -configuation and automation to properly support it. + Scikit-learn 1.0 supported Python 3.7—3.10. -At the time of writing (January 2021), the only way to get a working -installation of scikit-learn on this hardware is to install scikit-learn and its -dependencies from the conda-forge distribution, for instance using the miniforge -installers: + Scikit-learn 1.1, 1.2 and 1.3 supported Python 3.8—3.12. -https://github.com/conda-forge/miniforge + Scikit-learn 1.4 and 1.5 supported Python 3.9—3.12. -The following issue tracks progress on making it possible to install -scikit-learn from PyPI with pip: - -https://github.com/scikit-learn/scikit-learn/issues/19137 + Scikit-learn 1.6 supported Python 3.9—3.13. + Scikit-learn 1.7 requires Python 3.10 or newer. .. _install_by_distribution: @@ -187,6 +236,19 @@ dependencies (numpy, scipy) that scikit-learn requires. The following is an incomplete list of OS and python distributions that provide their own version of scikit-learn. +Alpine Linux +------------ + +Alpine Linux's package is provided through the `official repositories +`__ as +``py3-scikit-learn`` for Python. +It can be installed by typing the following command: + +.. prompt:: bash + + sudo apk add py3-scikit-learn + + Arch Linux ---------- @@ -195,7 +257,7 @@ Arch Linux's package is provided through the `official repositories ``python-scikit-learn`` for Python. It can be installed by typing the following command: -.. prompt:: bash $ +.. prompt:: bash sudo pacman -S python-scikit-learn @@ -203,26 +265,26 @@ It can be installed by typing the following command: Debian/Ubuntu ------------- -The Debian/Ubuntu package is splitted in three different packages called +The Debian/Ubuntu package is split in three different packages called ``python3-sklearn`` (python modules), ``python3-sklearn-lib`` (low-level -implementations and bindings), ``python3-sklearn-doc`` (documentation). -Only the Python 3 version is available in the Debian Buster (the more recent -Debian distribution). +implementations and bindings), ``python-sklearn-doc`` (documentation). +Note that scikit-learn requires Python 3, hence the need to use the `python3-` +suffixed package names. Packages can be installed using ``apt-get``: -.. prompt:: bash $ +.. prompt:: bash - sudo apt-get install python3-sklearn python3-sklearn-lib python3-sklearn-doc + sudo apt-get install python3-sklearn python3-sklearn-lib python-sklearn-doc Fedora ------ The Fedora package is called ``python3-scikit-learn`` for the python 3 version, -the only one available in Fedora30. +the only one available in Fedora. It can be installed using ``dnf``: -.. prompt:: bash $ +.. prompt:: bash sudo dnf install python3-scikit-learn @@ -230,10 +292,8 @@ It can be installed using ``dnf``: NetBSD ------ -scikit-learn is available via `pkgsrc-wip -`_: - - http://pkgsrc.se/math/py-scikit-learn +scikit-learn is available via `pkgsrc-wip `_: +https://pkgsrc.se/math/py-scikit-learn MacPorts for Mac OSX @@ -244,9 +304,9 @@ where ``XY`` denotes the Python version. It can be installed by typing the following command: -.. prompt:: bash $ +.. prompt:: bash - sudo port install py36-scikit-learn + sudo port install py312-scikit-learn Anaconda and Enthought Deployment Manager for all supported platforms @@ -260,30 +320,39 @@ python library for Windows, Mac OSX and Linux. Anaconda offers scikit-learn as part of its free distribution. -Intel conda channel -------------------- +Intel Extension for Scikit-learn +-------------------------------- -Intel maintains a dedicated conda channel that ships scikit-learn: +Intel maintains an optimized x86_64 package, available in PyPI (via `pip`), +and in the `main`, `conda-forge` and `intel` conda channels: -.. prompt:: bash $ +.. prompt:: bash - conda install -c intel scikit-learn + conda install scikit-learn-intelex -This version of scikit-learn comes with alternative solvers for some common -estimators. Those solvers come from the DAAL C++ library and are optimized for -multi-core Intel CPUs. +This package has an Intel optimized version of many estimators. Whenever +an alternative implementation doesn't exist, scikit-learn implementation +is used as a fallback. Those optimized solvers come from the oneDAL +C++ library and are optimized for the x86_64 architecture, and are +optimized for multi-core Intel CPUs. Note that those solvers are not enabled by default, please refer to the -`daal4py `_ documentation -for more details. +`scikit-learn-intelex `_ +documentation for more details on usage scenarios. Direct export example: + +.. prompt:: python >>> + + from sklearnex.neighbors import NearestNeighbors Compatibility with the standard scikit-learn solvers is checked by running the full scikit-learn test suite via automated continuous integration as reported -on https://github.com/IntelPython/daal4py. +on https://github.com/intel/scikit-learn-intelex. If you observe any issue +with `scikit-learn-intelex`, please report the issue on their +`issue tracker `__. WinPython for Windows ------------------------ +--------------------- The `WinPython `_ project distributes scikit-learn as an additional plugin. @@ -292,6 +361,10 @@ scikit-learn as an additional plugin. Troubleshooting =============== +If you encounter unexpected failures when installing scikit-learn, you may submit +an issue to the `issue tracker `_. +Before that, please also make sure to check the following common issues. + .. _windows_longpath: Error caused by file path length limit on Windows @@ -305,7 +378,7 @@ size limit of Windows if Python is installed in a nested location such as the Collecting scikit-learn ... Installing collected packages: scikit-learn - ERROR: Could not install packages due to an EnvironmentError: [Errno 2] No such file or directory: 'C:\\Users\\username\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.7_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python37\\site-packages\\sklearn\\datasets\\tests\\data\\openml\\292\\api-v1-json-data-list-data_name-australian-limit-2-data_version-1-status-deactivated.json.gz' + ERROR: Could not install packages due to an OSError: [Errno 2] No such file or directory: 'C:\\Users\\username\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.7_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python37\\site-packages\\sklearn\\datasets\\tests\\data\\openml\\292\\api-v1-json-data-list-data_name-australian-limit-2-data_version-1-status-deactivated.json.gz' In this case it is possible to lift that limit in the Windows registry by using the ``regedit`` tool: @@ -321,6 +394,6 @@ using the ``regedit`` tool: #. Reinstall scikit-learn (ignoring the previous broken installation): -.. prompt:: python $ + .. prompt:: powershell - pip install --exists-action=i scikit-learn + pip install --exists-action=i scikit-learn diff --git a/doc/install_instructions_conda.rst b/doc/install_instructions_conda.rst new file mode 100644 index 0000000000000..0b5a57b747021 --- /dev/null +++ b/doc/install_instructions_conda.rst @@ -0,0 +1,16 @@ +Install conda using the +`conda-forge installers `__ (no +administrator permission required). Then run: + +.. prompt:: bash + + conda create -n sklearn-env -c conda-forge scikit-learn + conda activate sklearn-env + +In order to check your installation, you can use: + +.. prompt:: bash + + conda list scikit-learn # show scikit-learn version and location + conda list # show all installed packages in the environment + python -c "import sklearn; sklearn.show_versions()" diff --git a/doc/js/scripts/api-search.js b/doc/js/scripts/api-search.js new file mode 100644 index 0000000000000..2148e0c429aaa --- /dev/null +++ b/doc/js/scripts/api-search.js @@ -0,0 +1,12 @@ +/** + * This script is for initializing the search table on the API index page. See + * DataTables documentation for more information: https://datatables.net/ + */ + +document.addEventListener("DOMContentLoaded", function () { + new DataTable("table.apisearch-table", { + order: [], // Keep original order + lengthMenu: [10, 25, 50, 100, { label: "All", value: -1 }], + pageLength: -1, // Show all entries by default + }); +}); diff --git a/doc/js/scripts/dropdown.js b/doc/js/scripts/dropdown.js new file mode 100644 index 0000000000000..d74d138773eed --- /dev/null +++ b/doc/js/scripts/dropdown.js @@ -0,0 +1,63 @@ +/** + * This script is used to add the functionality of collapsing/expanding all dropdowns + * on the page to the sphinx-design dropdowns. This is because some browsers cannot + * search into collapsed
(such as Firefox). + * + * The reason why the buttons are added to the page with JS (dynamic) instead of with + * sphinx (static) is that the button will not work without JS activated, so we do not + * want them to show up in that case. + */ + +document.addEventListener("DOMContentLoaded", () => { + // Get all sphinx-design dropdowns + const allDropdowns = document.querySelectorAll("details.sd-dropdown"); + + allDropdowns.forEach((dropdown) => { + // Get the summary element of the dropdown, where we will place the buttons + const summaryTitle = dropdown.querySelector("summary.sd-summary-title"); + + // The state marker with the toggle all icon inside + const newStateMarker = document.createElement("span"); + const newIcon = document.createElement("i"); + newIcon.classList.add("fa-solid", "fa-angles-right"); + newStateMarker.appendChild(newIcon); + + // Classes for styling; `sd-summary-state-marker` and `sd-summary-chevron-right` are + // implemented by sphinx-design; `sk-toggle-all` is implemented by us + newStateMarker.classList.add( + "sd-summary-state-marker", + "sd-summary-chevron-right", + "sk-toggle-all" + ); + + // Bootstrap tooltip configurations + newStateMarker.setAttribute("data-bs-toggle", "tooltip"); + newStateMarker.setAttribute("data-bs-placement", "top"); + newStateMarker.setAttribute("data-bs-offset", "0,10"); + newStateMarker.setAttribute("data-bs-title", "Toggle all dropdowns"); + // Enable the tooltip + new bootstrap.Tooltip(newStateMarker); + + // Assign the collapse/expand action to the state marker + newStateMarker.addEventListener("click", () => { + if (dropdown.open) { + console.log("[SK] Collapsing all dropdowns..."); + allDropdowns.forEach((node) => { + if (node !== dropdown) { + node.removeAttribute("open"); + } + }); + } else { + console.log("[SK] Expanding all dropdowns..."); + allDropdowns.forEach((node) => { + if (node !== dropdown) { + node.setAttribute("open", ""); + } + }); + } + }); + + // Append the state marker to the summary element + summaryTitle.insertBefore(newStateMarker, summaryTitle.lastElementChild); + }); +}); diff --git a/doc/js/scripts/sg_plotly_resize.js b/doc/js/scripts/sg_plotly_resize.js new file mode 100644 index 0000000000000..2d2611910db78 --- /dev/null +++ b/doc/js/scripts/sg_plotly_resize.js @@ -0,0 +1,10 @@ +// Related to https://github.com/scikit-learn/scikit-learn/issues/30279 +// There an interaction between plotly and bootstrap/pydata-sphinx-theme +// that causes plotly figures to not detect the right-hand sidebar width + +// Plotly figures are responsive, this triggers a resize event once the DOM has +// finished loading so that they resize themselves. + +document.addEventListener("DOMContentLoaded", () => { + window.dispatchEvent(new Event("resize")); +}); diff --git a/doc/js/scripts/vendor/svg-pan-zoom.min.js b/doc/js/scripts/vendor/svg-pan-zoom.min.js new file mode 100644 index 0000000000000..bde44a689bfe1 --- /dev/null +++ b/doc/js/scripts/vendor/svg-pan-zoom.min.js @@ -0,0 +1,31 @@ +/** + * svg-pan-zoom v3.6.2 + * + * https://github.com/bumbu/svg-pan-zoom + * + * Copyright 2009-2010 Andrea Leofreddi + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, this + * list of conditions and the following disclaimer in the documentation and/or + * other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +!function s(r,a,l){function u(e,t){if(!a[e]){if(!r[e]){var o="function"==typeof require&&require;if(!t&&o)return o(e,!0);if(h)return h(e,!0);var n=new Error("Cannot find module '"+e+"'");throw n.code="MODULE_NOT_FOUND",n}var i=a[e]={exports:{}};r[e][0].call(i.exports,function(t){return u(r[e][1][t]||t)},i,i.exports,s,r,a,l)}return a[e].exports}for(var h="function"==typeof require&&require,t=0;tthis.options.maxZoom*n.zoom&&(t=this.options.maxZoom*n.zoom/this.getZoom());var i=this.viewport.getCTM(),s=e.matrixTransform(i.inverse()),r=this.svg.createSVGMatrix().translate(s.x,s.y).scale(t).translate(-s.x,-s.y),a=i.multiply(r);a.a!==i.a&&this.viewport.setCTM(a)},i.prototype.zoom=function(t,e){this.zoomAtPoint(t,a.getSvgCenterPoint(this.svg,this.width,this.height),e)},i.prototype.publicZoom=function(t,e){e&&(t=this.computeFromRelativeZoom(t)),this.zoom(t,e)},i.prototype.publicZoomAtPoint=function(t,e,o){if(o&&(t=this.computeFromRelativeZoom(t)),"SVGPoint"!==r.getType(e)){if(!("x"in e&&"y"in e))throw new Error("Given point is invalid");e=a.createSVGPoint(this.svg,e.x,e.y)}this.zoomAtPoint(t,e,o)},i.prototype.getZoom=function(){return this.viewport.getZoom()},i.prototype.getRelativeZoom=function(){return this.viewport.getRelativeZoom()},i.prototype.computeFromRelativeZoom=function(t){return t*this.viewport.getOriginalState().zoom},i.prototype.resetZoom=function(){var t=this.viewport.getOriginalState();this.zoom(t.zoom,!0)},i.prototype.resetPan=function(){this.pan(this.viewport.getOriginalState())},i.prototype.reset=function(){this.resetZoom(),this.resetPan()},i.prototype.handleDblClick=function(t){var e;if((this.options.preventMouseEventsDefault&&(t.preventDefault?t.preventDefault():t.returnValue=!1),this.options.controlIconsEnabled)&&-1<(t.target.getAttribute("class")||"").indexOf("svg-pan-zoom-control"))return!1;e=t.shiftKey?1/(2*(1+this.options.zoomScaleSensitivity)):2*(1+this.options.zoomScaleSensitivity);var o=a.getEventPoint(t,this.svg).matrixTransform(this.svg.getScreenCTM().inverse());this.zoomAtPoint(e,o)},i.prototype.handleMouseDown=function(t,e){this.options.preventMouseEventsDefault&&(t.preventDefault?t.preventDefault():t.returnValue=!1),r.mouseAndTouchNormalize(t,this.svg),this.options.dblClickZoomEnabled&&r.isDblClick(t,e)?this.handleDblClick(t):(this.state="pan",this.firstEventCTM=this.viewport.getCTM(),this.stateOrigin=a.getEventPoint(t,this.svg).matrixTransform(this.firstEventCTM.inverse()))},i.prototype.handleMouseMove=function(t){if(this.options.preventMouseEventsDefault&&(t.preventDefault?t.preventDefault():t.returnValue=!1),"pan"===this.state&&this.options.panEnabled){var e=a.getEventPoint(t,this.svg).matrixTransform(this.firstEventCTM.inverse()),o=this.firstEventCTM.translate(e.x-this.stateOrigin.x,e.y-this.stateOrigin.y);this.viewport.setCTM(o)}},i.prototype.handleMouseUp=function(t){this.options.preventMouseEventsDefault&&(t.preventDefault?t.preventDefault():t.returnValue=!1),"pan"===this.state&&(this.state="none")},i.prototype.fit=function(){var t=this.viewport.getViewBox(),e=Math.min(this.width/t.width,this.height/t.height);this.zoom(e,!0)},i.prototype.contain=function(){var t=this.viewport.getViewBox(),e=Math.max(this.width/t.width,this.height/t.height);this.zoom(e,!0)},i.prototype.center=function(){var t=this.viewport.getViewBox(),e=.5*(this.width-(t.width+2*t.x)*this.getZoom()),o=.5*(this.height-(t.height+2*t.y)*this.getZoom());this.getPublicInstance().pan({x:e,y:o})},i.prototype.updateBBox=function(){this.viewport.simpleViewBoxCache()},i.prototype.pan=function(t){var e=this.viewport.getCTM();e.e=t.x,e.f=t.y,this.viewport.setCTM(e)},i.prototype.panBy=function(t){var e=this.viewport.getCTM();e.e+=t.x,e.f+=t.y,this.viewport.setCTM(e)},i.prototype.getPan=function(){var t=this.viewport.getState();return{x:t.x,y:t.y}},i.prototype.resize=function(){var t=a.getBoundingClientRectNormalized(this.svg);this.width=t.width,this.height=t.height;var e=this.viewport;e.options.width=this.width,e.options.height=this.height,e.processCTM(),this.options.controlIconsEnabled&&(this.getPublicInstance().disableControlIcons(),this.getPublicInstance().enableControlIcons())},i.prototype.destroy=function(){var e=this;for(var t in this.beforeZoom=null,this.onZoom=null,this.beforePan=null,this.onPan=null,(this.onUpdatedCTM=null)!=this.options.customEventsHandler&&this.options.customEventsHandler.destroy({svgElement:this.svg,eventsListenerElement:this.options.eventsListenerElement,instance:this.getPublicInstance()}),this.eventListeners)(this.options.eventsListenerElement||this.svg).removeEventListener(t,this.eventListeners[t],!this.options.preventMouseEventsDefault&&h);this.disableMouseWheelZoom(),this.getPublicInstance().disableControlIcons(),this.reset(),c=c.filter(function(t){return t.svg!==e.svg}),delete this.options,delete this.viewport,delete this.publicInstance,delete this.pi,this.getPublicInstance=function(){return null}},i.prototype.getPublicInstance=function(){var o=this;return this.publicInstance||(this.publicInstance=this.pi={enablePan:function(){return o.options.panEnabled=!0,o.pi},disablePan:function(){return o.options.panEnabled=!1,o.pi},isPanEnabled:function(){return!!o.options.panEnabled},pan:function(t){return o.pan(t),o.pi},panBy:function(t){return o.panBy(t),o.pi},getPan:function(){return o.getPan()},setBeforePan:function(t){return o.options.beforePan=null===t?null:r.proxy(t,o.publicInstance),o.pi},setOnPan:function(t){return o.options.onPan=null===t?null:r.proxy(t,o.publicInstance),o.pi},enableZoom:function(){return o.options.zoomEnabled=!0,o.pi},disableZoom:function(){return o.options.zoomEnabled=!1,o.pi},isZoomEnabled:function(){return!!o.options.zoomEnabled},enableControlIcons:function(){return o.options.controlIconsEnabled||(o.options.controlIconsEnabled=!0,s.enable(o)),o.pi},disableControlIcons:function(){return o.options.controlIconsEnabled&&(o.options.controlIconsEnabled=!1,s.disable(o)),o.pi},isControlIconsEnabled:function(){return!!o.options.controlIconsEnabled},enableDblClickZoom:function(){return o.options.dblClickZoomEnabled=!0,o.pi},disableDblClickZoom:function(){return o.options.dblClickZoomEnabled=!1,o.pi},isDblClickZoomEnabled:function(){return!!o.options.dblClickZoomEnabled},enableMouseWheelZoom:function(){return o.enableMouseWheelZoom(),o.pi},disableMouseWheelZoom:function(){return o.disableMouseWheelZoom(),o.pi},isMouseWheelZoomEnabled:function(){return!!o.options.mouseWheelZoomEnabled},setZoomScaleSensitivity:function(t){return o.options.zoomScaleSensitivity=t,o.pi},setMinZoom:function(t){return o.options.minZoom=t,o.pi},setMaxZoom:function(t){return o.options.maxZoom=t,o.pi},setBeforeZoom:function(t){return o.options.beforeZoom=null===t?null:r.proxy(t,o.publicInstance),o.pi},setOnZoom:function(t){return o.options.onZoom=null===t?null:r.proxy(t,o.publicInstance),o.pi},zoom:function(t){return o.publicZoom(t,!0),o.pi},zoomBy:function(t){return o.publicZoom(t,!1),o.pi},zoomAtPoint:function(t,e){return o.publicZoomAtPoint(t,e,!0),o.pi},zoomAtPointBy:function(t,e){return o.publicZoomAtPoint(t,e,!1),o.pi},zoomIn:function(){return this.zoomBy(1+o.options.zoomScaleSensitivity),o.pi},zoomOut:function(){return this.zoomBy(1/(1+o.options.zoomScaleSensitivity)),o.pi},getZoom:function(){return o.getRelativeZoom()},setOnUpdatedCTM:function(t){return o.options.onUpdatedCTM=null===t?null:r.proxy(t,o.publicInstance),o.pi},resetZoom:function(){return o.resetZoom(),o.pi},resetPan:function(){return o.resetPan(),o.pi},reset:function(){return o.reset(),o.pi},fit:function(){return o.fit(),o.pi},contain:function(){return o.contain(),o.pi},center:function(){return o.center(),o.pi},updateBBox:function(){return o.updateBBox(),o.pi},resize:function(){return o.resize(),o.pi},getSizes:function(){return{width:o.width,height:o.height,realZoom:o.getZoom(),viewBox:o.viewport.getViewBox()}},destroy:function(){return o.destroy(),o.pi}}),this.publicInstance};var c=[];e.exports=function(t,e){var o=r.getSvg(t);if(null===o)return null;for(var n=c.length-1;0<=n;n--)if(c[n].svg===o)return c[n].instance.getPublicInstance();return c.push({svg:o,instance:new i(o,e)}),c[c.length-1].instance.getPublicInstance()}},{"./control-icons":1,"./shadow-viewport":2,"./svg-utilities":5,"./uniwheel":6,"./utilities":7}],5:[function(t,e,o){var l=t("./utilities"),s="unknown";document.documentMode&&(s="ie"),e.exports={svgNS:"http://www.w3.org/2000/svg",xmlNS:"http://www.w3.org/XML/1998/namespace",xmlnsNS:"http://www.w3.org/2000/xmlns/",xlinkNS:"http://www.w3.org/1999/xlink",evNS:"http://www.w3.org/2001/xml-events",getBoundingClientRectNormalized:function(t){if(t.clientWidth&&t.clientHeight)return{width:t.clientWidth,height:t.clientHeight};if(t.getBoundingClientRect())return t.getBoundingClientRect();throw new Error("Cannot get BoundingClientRect for SVG.")},getOrCreateViewport:function(t,e){var o=null;if(!(o=l.isElement(e)?e:t.querySelector(e))){var n=Array.prototype.slice.call(t.childNodes||t.children).filter(function(t){return"defs"!==t.nodeName&&"#text"!==t.nodeName});1===n.length&&"g"===n[0].nodeName&&null===n[0].getAttribute("transform")&&(o=n[0])}if(!o){var i="viewport-"+(new Date).toISOString().replace(/\D/g,"");(o=document.createElementNS(this.svgNS,"g")).setAttribute("id",i);var s=t.childNodes||t.children;if(s&&0 + +## Brand Name +The official name of the package is __scikit-learn__. Do not abbreviate or otherwise alter the name. Always spell ‘scikit’ with a lowercase ‘s’. + + +## Color Palette + +![#29ABE2 Cyan](brand_colors/colorswatch_29ABE2_cyan.png) `RGB 41/171/226 | HEX #29ABE2 | scikit-learn Cyan` | More info: [#29ABE2](https://www.color-hex.com/color/29abe2) + +![#F7931E Orange](brand_colors/colorswatch_F7931E_orange.png) `RGB 247/147/30 | HEX #F7931E | scikit-learn Orange` | More info: [#F7931E](https://www.color-hex.com/color/f7931e) + +![#9B4600 Brown](brand_colors/colorswatch_9B4600_brown.png) `RGB 155/70/0| HEX #9B4600 | scikit-learn Brown` | More info: [#9B4600](https://www.color-hex.com/color/9b4600) + + +## Typography +The following typeface is used in the logo: +- "scikit": Helvetica Neue +- "learn": Script MT + + +## Logos +You may highlight or reference your work with scikit-learn by using one of the logos provided below. Any use must abide by the Logo Integrity Standards defined below. + +| | | +| - | - | +| | __Logo 1__
File type: PNG
File size: 49 KB (1280 x 689 px)
File name: [1280px-scikit-learn-logo.png](https://github.com/scikit-learn/scikit-learn/blob/main/doc/logos/1280px-scikit-learn-logo.png) | +| | __Logo 2__
File type: ICO
File size: 2 KB (32 x 32 px)
File name: [favicon.ico](https://github.com/scikit-learn/scikit-learn/blob/main/doc/logos/favicon.ico) | +| | __Logo 3__
File type: SVG
File size: 5 KB
File name: [scikit-learn-logo-without-subtitle.svg](https://github.com/scikit-learn/scikit-learn/blob/main/doc/logos/scikit-learn-logo-without-subtitle.svg) | +| | __Logo 4__
File type: SVG
File size: 4.59 KB
File name: [scikit-learn-logo.svg](https://github.com/scikit-learn/scikit-learn/blob/main/doc/logos/scikit-learn-logo.svg) | + +
+ + +### Logo Integrity Standards + +- __Minimum Size:__ For consistent legibility, please do not display the scikit-learn logo at less than 50px wide. +- __Scale:__ Ensure any logos used are scaled proportionally. Stretched, compressed, or otherwise distorted versions of the logo should not be displayed. + +- __Clear Space:__ To ensure the logo is clearly visible in all uses, surround it with a sufficient amount of clear space that is free of type, graphics, and other elements that might cause visual clutter. Do not overlap or obscure the logo with text, images, or other elements. The image below demonstrates the suggested amount of clear space margins to use around the logo.
+ +- __Colors:__ Only use logos in the approved color palette defined above. Do not recolor the logo. +- __Typeface:__ Do not change the typeface used in the logo. +- __No Modification:__ Do not attempt recreate or otherwise modify the scikit-learn logo. + + + +--- + +## Reference +- [color-hex](https://www.color-hex.com): Glossary of Color Palettes + +## Other +You can find more variations of the logos here: https://github.com/scikit-learn/blog/tree/main/assets/images diff --git a/doc/logos/brand_colors/colorswatch_29ABE2_cyan.png b/doc/logos/brand_colors/colorswatch_29ABE2_cyan.png new file mode 100644 index 0000000000000..b014a859dd4b9 Binary files /dev/null and b/doc/logos/brand_colors/colorswatch_29ABE2_cyan.png differ diff --git a/doc/logos/brand_colors/colorswatch_9B4600_brown.png b/doc/logos/brand_colors/colorswatch_9B4600_brown.png new file mode 100644 index 0000000000000..379400786ef56 Binary files /dev/null and b/doc/logos/brand_colors/colorswatch_9B4600_brown.png differ diff --git a/doc/logos/brand_colors/colorswatch_F7931E_orange.png b/doc/logos/brand_colors/colorswatch_F7931E_orange.png new file mode 100644 index 0000000000000..5b22b575ac411 Binary files /dev/null and b/doc/logos/brand_colors/colorswatch_F7931E_orange.png differ diff --git a/doc/logos/brand_guidelines/scikitlearn_logo_clearspace_updated.png b/doc/logos/brand_guidelines/scikitlearn_logo_clearspace_updated.png new file mode 100644 index 0000000000000..e10bff7a54c63 Binary files /dev/null and b/doc/logos/brand_guidelines/scikitlearn_logo_clearspace_updated.png differ diff --git a/doc/logos/scikit-learn-logo-without-subtitle.svg b/doc/logos/scikit-learn-logo-without-subtitle.svg new file mode 100644 index 0000000000000..932d418672034 --- /dev/null +++ b/doc/logos/scikit-learn-logo-without-subtitle.svg @@ -0,0 +1 @@ +scikit diff --git a/doc/logos/scikit-learn-logo.svg b/doc/logos/scikit-learn-logo.svg index 523a656943772..362542602e0ae 100644 --- a/doc/logos/scikit-learn-logo.svg +++ b/doc/logos/scikit-learn-logo.svg @@ -1,110 +1 @@ - - - -image/svg+xml - - - - - - - - - - - - - - -scikit - - -machine learning in Python - - \ No newline at end of file +scikitmachine learning in Python diff --git a/doc/machine_learning_map.rst b/doc/machine_learning_map.rst new file mode 100644 index 0000000000000..e63ab1b1ddce6 --- /dev/null +++ b/doc/machine_learning_map.rst @@ -0,0 +1,76 @@ +:html_theme.sidebar_secondary.remove: + +.. _ml_map: + +Choosing the right estimator +============================ + +Often the hardest part of solving a machine learning problem can be finding the right +estimator for the job. Different estimators are better suited for different types of +data and different problems. + +The flowchart below is designed to give users a bit of a rough guide on how to approach +problems with regard to which estimators to try on your data. Click on any estimator in +the chart below to see its documentation. The **Try next** orange arrows are to be read as +"if this estimator does not achieve the desired outcome, then follow the arrow and try +the next one". Use scroll wheel to zoom in and out, and click and drag to pan around. +You can also download the chart: :download:`ml_map.svg `. + +.. raw:: html + + + + + + +
+ +.. raw:: html + :file: images/ml_map.svg + +.. raw:: html + +
diff --git a/doc/maintainers.rst b/doc/maintainers.rst new file mode 100644 index 0000000000000..6b4f3a25c0ddc --- /dev/null +++ b/doc/maintainers.rst @@ -0,0 +1,84 @@ +.. raw :: html + + +
+ +
+
+

JÊrÊmie du Boisberranger

+
+
+
+

Loïc Estève

+
+
+
+

Thomas J. Fan

+
+
+
+

Alexandre Gramfort

+
+
+
+

Olivier Grisel

+
+
+
+

Tim Head

+
+
+
+

Nicolas Hug

+
+
+
+

Adrin Jalali

+
+
+
+

Julien Jerphanion

+
+
+
+

Guillaume Lemaitre

+
+
+
+

Adam Li

+
+
+
+

Lucy Liu

+
+
+
+

Christian Lorentzen

+
+
+
+

Andreas Mueller

+
+
+
+

Joel Nothman

+
+
+
+

Omar Salman

+
+
+
+

Gael Varoquaux

+
+
+
+

Yao Xiao

+
+
+
+

Meekail Zain

+
+
diff --git a/doc/authors_emeritus.rst b/doc/maintainers_emeritus.rst similarity index 73% rename from doc/authors_emeritus.rst rename to doc/maintainers_emeritus.rst index bcfd7d7d0514c..f5640ab2caf31 100644 --- a/doc/authors_emeritus.rst +++ b/doc/maintainers_emeritus.rst @@ -1,9 +1,9 @@ - Mathieu Blondel +- Joris Van den Bossche - Matthieu Brucher - Lars Buitinck - David Cournapeau - Noel Dawe -- Shiqiao Du - Vincent Dubourg - Edouard Duchesnay - Alexander Fabisch @@ -12,6 +12,7 @@ - Angel Soler Gollonet - Chris Gorgolewski - Jaques Grobler +- Yaroslav Halchenko - Brian Holt - Arnaud Joly - Thouis (Ray) Jones @@ -21,13 +22,21 @@ - Wei Li - Paolo Losi - Gilles Louppe +- Jan Hendrik Metzen - Vincent Michel - Jarrod Millman +- Vlad Niculae - Alexandre Passos - Fabian Pedregosa - Peter Prettenhofer +- Hanmin Qin - (Venkat) Raghav, Rajagopalan - Jacob Schreiber +- 杜世抋 Du Shiqiao +- Bertrand Thirion +- Tom DuprÊ la Tour - Jake Vanderplas +- Nelle Varoquaux - David Warde-Farley -- Ron Weiss \ No newline at end of file +- Ron Weiss +- Roman Yurchak diff --git a/doc/make.bat b/doc/make.bat index fa8e7171ea7e6..2a32bcb678f62 100644 --- a/doc/make.bat +++ b/doc/make.bat @@ -9,7 +9,7 @@ if NOT "%PAPER%" == "" ( set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% ) -if "%1" == "" goto help +if "%1" == "" goto html-noplot if "%1" == "help" ( :help @@ -29,8 +29,30 @@ if "%1" == "help" ( ) if "%1" == "clean" ( - for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i - del /q /s %BUILDDIR%\* + if exist %BUILDDIR%\ ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s "%%i" + del /q /s %BUILDDIR%\* + echo. Removed %BUILDDIR%\* + ) + if exist auto_examples\ ( + rmdir /q /s auto_examples + echo. Removed auto_examples\ + ) + if exist generated\ ( + for /d %%i in (generated\*) do rmdir /q /s "%%i" + del /q /s generated\* + echo. Removed generated\* + ) + if exist modules\generated\ ( + rmdir /q /s modules\generated + echo. Removed modules\generated\ + ) + if exist css\styles\ ( + rmdir /q /s css\styles + echo. Removed css\styles\ + ) + for %%i in (api\*.rst) do del /q "%%i" + echo. Removed api\*.rst goto end ) @@ -42,9 +64,11 @@ if "%1" == "html" ( ) if "%1" == "html-noplot" ( + :html-noplot %SPHINXBUILD% -D plot_gallery=0 -b html %ALLSPHINXOPTS% %BUILDDIR%/html echo. echo.Build finished. The HTML pages are in %BUILDDIR%/html + goto end ) if "%1" == "dirhtml" ( diff --git a/doc/metadata_routing.rst b/doc/metadata_routing.rst new file mode 100644 index 0000000000000..d302b84c5de68 --- /dev/null +++ b/doc/metadata_routing.rst @@ -0,0 +1,326 @@ +.. currentmodule:: sklearn + +.. _metadata_routing: + +Metadata Routing +================ + +.. note:: + The Metadata Routing API is experimental, and is not yet implemented for all + estimators. Please refer to the :ref:`list of supported and unsupported + models ` for more information. It may change without + the usual deprecation cycle. By default this feature is not enabled. You can + enable it by setting the ``enable_metadata_routing`` flag to + ``True``:: + + >>> import sklearn + >>> sklearn.set_config(enable_metadata_routing=True) + + Note that the methods and requirements introduced in this document are only + relevant if you want to pass :term:`metadata` (e.g. ``sample_weight``) to a method. + If you're only passing ``X`` and ``y`` and no other parameter / metadata to + methods such as :term:`fit`, :term:`transform`, etc., then you don't need to set + anything. + +This guide demonstrates how :term:`metadata` can be routed and passed between objects in +scikit-learn. If you are developing a scikit-learn compatible estimator or +meta-estimator, you can check our related developer guide: +:ref:`sphx_glr_auto_examples_miscellaneous_plot_metadata_routing.py`. + +Metadata is data that an estimator, scorer, or CV splitter takes into account if the +user explicitly passes it as a parameter. For instance, :class:`~cluster.KMeans` accepts +`sample_weight` in its `fit()` method and considers it to calculate its centroids. +`classes` are consumed by some classifiers and `groups` are used in some splitters, but +any data that is passed into an object's methods apart from X and y can be considered as +metadata. Prior to scikit-learn version 1.3, there was no single API for passing +metadata like that if these objects were used in conjunction with other objects, e.g. a +scorer accepting `sample_weight` inside a :class:`~model_selection.GridSearchCV`. + +With the Metadata Routing API, we can transfer metadata to estimators, scorers, and CV +splitters using :term:`meta-estimators` (such as :class:`~pipeline.Pipeline` or +:class:`~model_selection.GridSearchCV`) or functions such as +:func:`~model_selection.cross_validate` which route data to other objects. In order to +pass metadata to a method like ``fit`` or ``score``, the object consuming the metadata, +must *request* it. This is done via `set_{method}_request()` methods, where `{method}` +is substituted by the name of the method that requests the metadata. For instance, +estimators that use the metadata in their `fit()` method would use `set_fit_request()`, +and scorers would use `set_score_request()`. These methods allow us to specify which +metadata to request, for instance `set_fit_request(sample_weight=True)`. + +For grouped splitters such as :class:`~model_selection.GroupKFold`, a +``groups`` parameter is requested by default. This is best demonstrated by the +following examples. + +Usage Examples +************** +Here we present a few examples to show some common use-cases. Our goal is to pass +`sample_weight` and `groups` through :func:`~model_selection.cross_validate`, which +routes the metadata to :class:`~linear_model.LogisticRegressionCV` and to a custom scorer +made with :func:`~metrics.make_scorer`, both of which *can* use the metadata in their +methods. In these examples we want to individually set whether to use the metadata +within the different :term:`consumers `. + +The examples in this section require the following imports and data:: + + >>> import numpy as np + >>> from sklearn.metrics import make_scorer, accuracy_score + >>> from sklearn.linear_model import LogisticRegressionCV, LogisticRegression + >>> from sklearn.model_selection import cross_validate, GridSearchCV, GroupKFold + >>> from sklearn.feature_selection import SelectKBest + >>> from sklearn.pipeline import make_pipeline + >>> n_samples, n_features = 100, 4 + >>> rng = np.random.RandomState(42) + >>> X = rng.rand(n_samples, n_features) + >>> y = rng.randint(0, 2, size=n_samples) + >>> my_groups = rng.randint(0, 10, size=n_samples) + >>> my_weights = rng.rand(n_samples) + >>> my_other_weights = rng.rand(n_samples) + +Weighted scoring and fitting +---------------------------- + +The splitter used internally in :class:`~linear_model.LogisticRegressionCV`, +:class:`~model_selection.GroupKFold`, requests ``groups`` by default. However, we need +to explicitly request `sample_weight` for it and for our custom scorer by specifying +`sample_weight=True` in :class:`~linear_model.LogisticRegressionCV`'s `set_fit_request()` +method and in :func:`~metrics.make_scorer`'s `set_score_request()` method. Both +:term:`consumers ` know how to use ``sample_weight`` in their `fit()` or +`score()` methods. We can then pass the metadata in +:func:`~model_selection.cross_validate` which will route it to any active consumers:: + + >>> weighted_acc = make_scorer(accuracy_score).set_score_request(sample_weight=True) + >>> lr = LogisticRegressionCV( + ... cv=GroupKFold(), + ... scoring=weighted_acc + ... ).set_fit_request(sample_weight=True) + >>> cv_results = cross_validate( + ... lr, + ... X, + ... y, + ... params={"sample_weight": my_weights, "groups": my_groups}, + ... cv=GroupKFold(), + ... scoring=weighted_acc, + ... ) + +Note that in this example, :func:`~model_selection.cross_validate` routes ``my_weights`` +to both the scorer and :class:`~linear_model.LogisticRegressionCV`. + +If we would pass `sample_weight` in the params of +:func:`~model_selection.cross_validate`, but not set any object to request it, +`UnsetMetadataPassedError` would be raised, hinting to us that we need to explicitly set +where to route it. The same applies if ``params={"sample_weights": my_weights, ...}`` +were passed (note the typo, i.e. ``weights`` instead of ``weight``), since +``sample_weights`` was not requested by any of its underlying objects. + +Weighted scoring and unweighted fitting +--------------------------------------- + +When passing metadata such as ``sample_weight`` into a :term:`router` +(:term:`meta-estimators` or routing function), all ``sample_weight`` :term:`consumers +` require weights to be either explicitly requested or explicitly not +requested (i.e. ``True`` or ``False``). Thus, to perform an unweighted fit, we need to +configure :class:`~linear_model.LogisticRegressionCV` to not request sample weights, so +that :func:`~model_selection.cross_validate` does not pass the weights along:: + + >>> weighted_acc = make_scorer(accuracy_score).set_score_request(sample_weight=True) + >>> lr = LogisticRegressionCV( + ... cv=GroupKFold(), scoring=weighted_acc, + ... ).set_fit_request(sample_weight=False) + >>> cv_results = cross_validate( + ... lr, + ... X, + ... y, + ... cv=GroupKFold(), + ... params={"sample_weight": my_weights, "groups": my_groups}, + ... scoring=weighted_acc, + ... ) + +If :meth:`linear_model.LogisticRegressionCV.set_fit_request` had not been called, +:func:`~model_selection.cross_validate` would raise an error because ``sample_weight`` +is passed but :class:`~linear_model.LogisticRegressionCV` would not be explicitly +configured to recognize the weights. + +Unweighted feature selection +---------------------------- + +Routing metadata is only possible if the object's method knows how to use the metadata, +which in most cases means they have it as an explicit parameter. Only then we can set +request values for metadata using `set_fit_request(sample_weight=True)`, for instance. +This makes the object a :term:`consumer `. + +Unlike :class:`~linear_model.LogisticRegressionCV`, +:class:`~feature_selection.SelectKBest` can't consume weights and therefore no request +value for ``sample_weight`` on its instance is set and ``sample_weight`` is not routed +to it:: + + >>> weighted_acc = make_scorer(accuracy_score).set_score_request(sample_weight=True) + >>> lr = LogisticRegressionCV( + ... cv=GroupKFold(), scoring=weighted_acc, + ... ).set_fit_request(sample_weight=True) + >>> sel = SelectKBest(k=2) + >>> pipe = make_pipeline(sel, lr) + >>> cv_results = cross_validate( + ... pipe, + ... X, + ... y, + ... cv=GroupKFold(), + ... params={"sample_weight": my_weights, "groups": my_groups}, + ... scoring=weighted_acc, + ... ) + +Different scoring and fitting weights +------------------------------------- + +Despite :func:`~metrics.make_scorer` and +:class:`~linear_model.LogisticRegressionCV` both expecting the key +``sample_weight``, we can use aliases to pass different weights to different +consumers. In this example, we pass ``scoring_weight`` to the scorer, and +``fitting_weight`` to :class:`~linear_model.LogisticRegressionCV`:: + + >>> weighted_acc = make_scorer(accuracy_score).set_score_request( + ... sample_weight="scoring_weight" + ... ) + >>> lr = LogisticRegressionCV( + ... cv=GroupKFold(), scoring=weighted_acc, + ... ).set_fit_request(sample_weight="fitting_weight") + >>> cv_results = cross_validate( + ... lr, + ... X, + ... y, + ... cv=GroupKFold(), + ... params={ + ... "scoring_weight": my_weights, + ... "fitting_weight": my_other_weights, + ... "groups": my_groups, + ... }, + ... scoring=weighted_acc, + ... ) + +API Interface +************* + +A :term:`consumer` is an object (estimator, meta-estimator, scorer, splitter) which +accepts and uses some :term:`metadata` in at least one of its methods (for instance +``fit``, ``predict``, ``inverse_transform``, ``transform``, ``score``, ``split``). +Meta-estimators which only forward the metadata to other objects (child estimators, +scorers, or splitters) and don't use the metadata themselves are not consumers. +(Meta-)Estimators which route metadata to other objects are :term:`routers `. +A(n) (meta-)estimator can be a :term:`consumer` and a :term:`router` at the same time. +(Meta-)Estimators and splitters expose a `set_{method}_request` method for each method +which accepts at least one metadata. For instance, if an estimator supports +``sample_weight`` in ``fit`` and ``score``, it exposes +``estimator.set_fit_request(sample_weight=value)`` and +``estimator.set_score_request(sample_weight=value)``. Here ``value`` can be: + +- ``True``: method requests a ``sample_weight``. This means if the metadata is provided, + it will be used, otherwise no error is raised. +- ``False``: method does not request a ``sample_weight``. +- ``None``: router will raise an error if ``sample_weight`` is passed. This is in almost + all cases the default value when an object is instantiated and ensures the user sets + the metadata requests explicitly when a metadata is passed. The only exception are + ``Group*Fold`` splitters. +- ``"param_name"``: alias for ``sample_weight`` if we want to pass different weights to + different consumers. If aliasing is used the meta-estimator should not forward + ``"param_name"`` to the consumer, but ``sample_weight`` instead, because the consumer + will expect a param called ``sample_weight``. This means the mapping between the + metadata required by the object, e.g. ``sample_weight`` and the variable name provided + by the user, e.g. ``my_weights`` is done at the router level, and not by the consuming + object itself. + +Metadata are requested in the same way for scorers using ``set_score_request``. + +If a metadata, e.g. ``sample_weight``, is passed by the user, the metadata request for +all objects which potentially can consume ``sample_weight`` should be set by the user, +otherwise an error is raised by the router object. For example, the following code +raises an error, since it hasn't been explicitly specified whether ``sample_weight`` +should be passed to the estimator's scorer or not:: + + >>> param_grid = {"C": [0.1, 1]} + >>> lr = LogisticRegression().set_fit_request(sample_weight=True) + >>> try: + ... GridSearchCV( + ... estimator=lr, param_grid=param_grid + ... ).fit(X, y, sample_weight=my_weights) + ... except ValueError as e: + ... print(e) + [sample_weight] are passed but are not explicitly set as requested or not + requested for LogisticRegression.score, which is used within GridSearchCV.fit. + Call `LogisticRegression.set_score_request({metadata}=True/False)` for each metadata + you want to request/ignore. See the Metadata Routing User guide + for more information. + +The issue can be fixed by explicitly setting the request value:: + + >>> lr = LogisticRegression().set_fit_request( + ... sample_weight=True + ... ).set_score_request(sample_weight=False) + +At the end of the **Usage Examples** section, we disable the configuration flag for +metadata routing:: + + >>> sklearn.set_config(enable_metadata_routing=False) + +.. _metadata_routing_models: + +Metadata Routing Support Status +******************************* +All consumers (i.e. simple estimators which only consume metadata and don't +route them) support metadata routing, meaning they can be used inside +meta-estimators which support metadata routing. However, development of support +for metadata routing for meta-estimators is in progress, and here is a list of +meta-estimators and tools which support and don't yet support metadata routing. + + +Meta-estimators and functions supporting metadata routing: + +- :class:`sklearn.calibration.CalibratedClassifierCV` +- :class:`sklearn.compose.ColumnTransformer` +- :class:`sklearn.compose.TransformedTargetRegressor` +- :class:`sklearn.covariance.GraphicalLassoCV` +- :class:`sklearn.ensemble.StackingClassifier` +- :class:`sklearn.ensemble.StackingRegressor` +- :class:`sklearn.ensemble.VotingClassifier` +- :class:`sklearn.ensemble.VotingRegressor` +- :class:`sklearn.ensemble.BaggingClassifier` +- :class:`sklearn.ensemble.BaggingRegressor` +- :class:`sklearn.feature_selection.RFE` +- :class:`sklearn.feature_selection.RFECV` +- :class:`sklearn.feature_selection.SelectFromModel` +- :class:`sklearn.feature_selection.SequentialFeatureSelector` +- :class:`sklearn.impute.IterativeImputer` +- :class:`sklearn.linear_model.ElasticNetCV` +- :class:`sklearn.linear_model.LarsCV` +- :class:`sklearn.linear_model.LassoCV` +- :class:`sklearn.linear_model.LassoLarsCV` +- :class:`sklearn.linear_model.LogisticRegressionCV` +- :class:`sklearn.linear_model.MultiTaskElasticNetCV` +- :class:`sklearn.linear_model.MultiTaskLassoCV` +- :class:`sklearn.linear_model.OrthogonalMatchingPursuitCV` +- :class:`sklearn.linear_model.RANSACRegressor` +- :class:`sklearn.linear_model.RidgeClassifierCV` +- :class:`sklearn.linear_model.RidgeCV` +- :class:`sklearn.model_selection.GridSearchCV` +- :class:`sklearn.model_selection.HalvingGridSearchCV` +- :class:`sklearn.model_selection.HalvingRandomSearchCV` +- :class:`sklearn.model_selection.RandomizedSearchCV` +- :class:`sklearn.model_selection.permutation_test_score` +- :func:`sklearn.model_selection.cross_validate` +- :func:`sklearn.model_selection.cross_val_score` +- :func:`sklearn.model_selection.cross_val_predict` +- :class:`sklearn.model_selection.learning_curve` +- :class:`sklearn.model_selection.validation_curve` +- :class:`sklearn.multiclass.OneVsOneClassifier` +- :class:`sklearn.multiclass.OneVsRestClassifier` +- :class:`sklearn.multiclass.OutputCodeClassifier` +- :class:`sklearn.multioutput.ClassifierChain` +- :class:`sklearn.multioutput.MultiOutputClassifier` +- :class:`sklearn.multioutput.MultiOutputRegressor` +- :class:`sklearn.multioutput.RegressorChain` +- :class:`sklearn.pipeline.FeatureUnion` +- :class:`sklearn.pipeline.Pipeline` +- :class:`sklearn.semi_supervised.SelfTrainingClassifier` + +Meta-estimators and tools not supporting metadata routing yet: + +- :class:`sklearn.ensemble.AdaBoostClassifier` +- :class:`sklearn.ensemble.AdaBoostRegressor` diff --git a/doc/min_dependency_substitutions.rst.template b/doc/min_dependency_substitutions.rst.template new file mode 100644 index 0000000000000..946de84902b3b --- /dev/null +++ b/doc/min_dependency_substitutions.rst.template @@ -0,0 +1,3 @@ +{% for package, (version, _) in dependent_packages.items() -%} +.. |{{ package|capitalize }}MinVersion| replace:: {{ version }} +{% endfor %} diff --git a/doc/min_dependency_table.rst.template b/doc/min_dependency_table.rst.template new file mode 100644 index 0000000000000..fbe58633e913a --- /dev/null +++ b/doc/min_dependency_table.rst.template @@ -0,0 +1,13 @@ +.. list-table:: + :header-rows: 1 + + * - Dependency + - Minimum Version + - Purpose + + {% for package, (version, tags) in dependent_packages.items() -%} + * - {{ package }} + - {{ version }} + - {{ tags }} + + {% endfor %} diff --git a/doc/model_persistence.rst b/doc/model_persistence.rst new file mode 100644 index 0000000000000..21d6934a48730 --- /dev/null +++ b/doc/model_persistence.rst @@ -0,0 +1,394 @@ +.. _model_persistence: + +================= +Model persistence +================= + +.. list-table:: Summary of model persistence methods + :widths: 25 50 50 + :header-rows: 1 + + * - Persistence method + - Pros + - Risks / Cons + * - :ref:`ONNX ` + - * Serve models without a Python environment + * Serving and training environments independent of one another + * Most secure option + - * Not all scikit-learn models are supported + * Custom estimators require more work to support + * Original Python object is lost and cannot be reconstructed + * - :ref:`skops_persistence` + - * More secure than `pickle` based formats + * Contents can be partly validated without loading + - * Not as fast as `pickle` based formats + * Supports less types than `pickle` based formats + * Requires the same environment as the training environment + * - :mod:`pickle` + - * Native to Python + * Can serialize most Python objects + * Efficient memory usage with `protocol=5` + - * Loading can execute arbitrary code + * Requires the same environment as the training environment + * - :mod:`joblib` + - * Efficient memory usage + * Supports memory mapping + * Easy shortcuts for compression and decompression + - * Pickle based format + * Loading can execute arbitrary code + * Requires the same environment as the training environment + * - `cloudpickle`_ + - * Can serialize non-packaged, custom Python code + * Comparable loading efficiency as :mod:`pickle` with `protocol=5` + - * Pickle based format + * Loading can execute arbitrary code + * No forward compatibility guarantees + * Requires the same environment as the training environment + +After training a scikit-learn model, it is desirable to have a way to persist +the model for future use without having to retrain. Based on your use-case, +there are a few different ways to persist a scikit-learn model, and here we +help you decide which one suits you best. In order to make a decision, you need +to answer the following questions: + +1. Do you need the Python object after persistence, or do you only need to + persist in order to serve the model and get predictions out of it? + +If you only need to serve the model and no further investigation on the Python +object itself is required, then :ref:`ONNX ` might be the +best fit for you. Note that not all models are supported by ONNX. + +In case ONNX is not suitable for your use-case, the next question is: + +2. Do you absolutely trust the source of the model, or are there any security + concerns regarding where the persisted model comes from? + +If you have security concerns, then you should consider using :ref:`skops.io +` which gives you back the Python object, but unlike +`pickle` based persistence solutions, loading the persisted model doesn't +automatically allow arbitrary code execution. Note that this requires manual +investigation of the persisted file, which :mod:`skops.io` allows you to do. + +The other solutions assume you absolutely trust the source of the file to be +loaded, as they are all susceptible to arbitrary code execution upon loading +the persisted file since they all use the pickle protocol under the hood. + +3. Do you care about the performance of loading the model, and sharing it + between processes where a memory mapped object on disk is beneficial? + +If yes, then you can consider using :ref:`joblib `. If this +is not a major concern for you, then you can use the built-in :mod:`pickle` +module. + +4. Did you try :mod:`pickle` or :mod:`joblib` and found that the model cannot + be persisted? It can happen for instance when you have user defined + functions in your model. + +If yes, then you can use `cloudpickle`_ which can serialize certain objects +which cannot be serialized by :mod:`pickle` or :mod:`joblib`. + + +Workflow Overview +----------------- + +In a typical workflow, the first step is to train the model using scikit-learn +and scikit-learn compatible libraries. Note that support for scikit-learn and +third party estimators varies across the different persistence methods. + +Train and Persist the Model +........................... + +Creating an appropriate model depends on your use-case. As an example, here we +train a :class:`sklearn.ensemble.HistGradientBoostingClassifier` on the iris +dataset:: + + >>> from sklearn import ensemble + >>> from sklearn import datasets + >>> clf = ensemble.HistGradientBoostingClassifier() + >>> X, y = datasets.load_iris(return_X_y=True) + >>> clf.fit(X, y) + HistGradientBoostingClassifier() + +Once the model is trained, you can persist it using your desired method, and +then you can load the model in a separate environment and get predictions from +it given input data. Here there are two major paths depending on how you +persist and plan to serve the model: + +- :ref:`ONNX `: You need an `ONNX` runtime and an environment + with appropriate dependencies installed to load the model and use the runtime + to get predictions. This environment can be minimal and does not necessarily + even require Python to be installed to load the model and compute + predictions. Also note that `onnxruntime` typically requires much less RAM + than Python to compute predictions from small models. + +- :mod:`skops.io`, :mod:`pickle`, :mod:`joblib`, `cloudpickle`_: You need a + Python environment with the appropriate dependencies installed to load the + model and get predictions from it. This environment should have the same + **packages** and the same **versions** as the environment where the model was + trained. Note that none of these methods support loading a model trained with + a different version of scikit-learn, and possibly different versions of other + dependencies such as `numpy` and `scipy`. Another concern would be running + the persisted model on a different hardware, and in most cases you should be + able to load your persisted model on a different hardware. + + +.. _onnx_persistence: + +ONNX +---- + +`ONNX`, or `Open Neural Network Exchange `__ format is best +suitable in use-cases where one needs to persist the model and then use the +persisted artifact to get predictions without the need to load the Python +object itself. It is also useful in cases where the serving environment needs +to be lean and minimal, since the `ONNX` runtime does not require `python`. + +`ONNX` is a binary serialization of the model. It has been developed to improve +the usability of the interoperable representation of data models. It aims to +facilitate the conversion of the data models between different machine learning +frameworks, and to improve their portability on different computing +architectures. More details are available from the `ONNX tutorial +`__. To convert scikit-learn model to `ONNX` +`sklearn-onnx `__ has been developed. However, +not all scikit-learn models are supported, and it is limited to the core +scikit-learn and does not support most third party estimators. One can write a +custom converter for third party or custom estimators, but the documentation to +do that is sparse and it might be challenging to do so. + +.. dropdown:: Using ONNX + + To convert the model to `ONNX` format, you need to give the converter some + information about the input as well, about which you can read more `here + `__:: + + from skl2onnx import to_onnx + onx = to_onnx(clf, X[:1].astype(numpy.float32), target_opset=12) + with open("filename.onnx", "wb") as f: + f.write(onx.SerializeToString()) + + You can load the model in Python and use the `ONNX` runtime to get + predictions:: + + from onnxruntime import InferenceSession + with open("filename.onnx", "rb") as f: + onx = f.read() + sess = InferenceSession(onx, providers=["CPUExecutionProvider"]) + pred_ort = sess.run(None, {"X": X_test.astype(numpy.float32)})[0] + +.. _skops_persistence: + +`skops.io` +---------- + +:mod:`skops.io` avoids using :mod:`pickle` and only loads files which have types +and references to functions which are trusted either by default or by the user. +Therefore it provides a more secure format than :mod:`pickle`, :mod:`joblib`, +and `cloudpickle`_. + + +.. dropdown:: Using skops + + The API is very similar to :mod:`pickle`, and you can persist your models as + explained in the `documentation + `__ using + :func:`skops.io.dump` and :func:`skops.io.dumps`:: + + import skops.io as sio + obj = sio.dump(clf, "filename.skops") + + And you can load them back using :func:`skops.io.load` and + :func:`skops.io.loads`. However, you need to specify the types which are + trusted by you. You can get existing unknown types in a dumped object / file + using :func:`skops.io.get_untrusted_types`, and after checking its contents, + pass it to the load function:: + + unknown_types = sio.get_untrusted_types(file="filename.skops") + # investigate the contents of unknown_types, and only load if you trust + # everything you see. + clf = sio.load("filename.skops", trusted=unknown_types) + + Please report issues and feature requests related to this format on the `skops + issue tracker `__. + + +.. _pickle_persistence: + +`pickle`, `joblib`, and `cloudpickle` +------------------------------------- + +These three modules / packages, use the `pickle` protocol under the hood, but +come with slight variations: + +- :mod:`pickle` is a module from the Python Standard Library. It can serialize + and deserialize any Python object, including custom Python classes and + objects. +- :mod:`joblib` is more efficient than `pickle` when working with large machine + learning models or large numpy arrays. +- `cloudpickle`_ can serialize certain objects which cannot be serialized by + :mod:`pickle` or :mod:`joblib`, such as user defined functions and lambda + functions. This can happen for instance, when using a + :class:`~sklearn.preprocessing.FunctionTransformer` and using a custom + function to transform the data. + +.. dropdown:: Using `pickle`, `joblib`, or `cloudpickle` + + Depending on your use-case, you can choose one of these three methods to + persist and load your scikit-learn model, and they all follow the same API:: + + # Here you can replace pickle with joblib or cloudpickle + from pickle import dump + with open("filename.pkl", "wb") as f: + dump(clf, f, protocol=5) + + Using `protocol=5` is recommended to reduce memory usage and make it faster to + store and load any large NumPy array stored as a fitted attribute in the model. + You can alternatively pass `protocol=pickle.HIGHEST_PROTOCOL` which is + equivalent to `protocol=5` in Python 3.8 and later (at the time of writing). + + And later when needed, you can load the same object from the persisted file:: + + # Here you can replace pickle with joblib or cloudpickle + from pickle import load + with open("filename.pkl", "rb") as f: + clf = load(f) + +.. _persistence_limitations: + +Security & Maintainability Limitations +-------------------------------------- + +:mod:`pickle` (and :mod:`joblib` and :mod:`cloudpickle` by extension), has +many documented security vulnerabilities by design and should only be used if +the artifact, i.e. the pickle-file, is coming from a trusted and verified +source. You should never load a pickle file from an untrusted source, similarly +to how you should never execute code from an untrusted source. + +Also note that arbitrary computations can be represented using the `ONNX` +format, and it is therefore recommended to serve models using `ONNX` in a +sandboxed environment to safeguard against computational and memory exploits. + +Also note that there are no supported ways to load a model trained with a +different version of scikit-learn. While using :mod:`skops.io`, :mod:`joblib`, +:mod:`pickle`, or `cloudpickle`_, models saved using one version of +scikit-learn might load in other versions, however, this is entirely +unsupported and inadvisable. It should also be kept in mind that operations +performed on such data could give different and unexpected results, or even +crash your Python process. + +In order to rebuild a similar model with future versions of scikit-learn, +additional metadata should be saved along the pickled model: + +* The training data, e.g. a reference to an immutable snapshot +* The Python source code used to generate the model +* The versions of scikit-learn and its dependencies +* The cross validation score obtained on the training data + +This should make it possible to check that the cross-validation score is in the +same range as before. + +Aside for a few exceptions, persisted models should be portable across +operating systems and hardware architectures assuming the same versions of +dependencies and Python are used. If you encounter an estimator that is not +portable, please open an issue on GitHub. Persisted models are often deployed +in production using containers like Docker, in order to freeze the environment +and dependencies. + +If you want to know more about these issues, please refer to these talks: + +- `Adrin Jalali: Let's exploit pickle, and skops to the rescue! | PyData + Amsterdam 2023 `__. +- `Alex Gaynor: Pickles are for Delis, not Software - PyCon 2014 + `__. + + +.. _serving_environment: + +Replicating the training environment in production +.................................................. + +If the versions of the dependencies used may differ from training to +production, it may result in unexpected behaviour and errors while using the +trained model. To prevent such situations it is recommended to use the same +dependencies and versions in both the training and production environment. +These transitive dependencies can be pinned with the help of package management +tools like `pip`, `mamba`, `conda`, `poetry`, `conda-lock`, `pixi`, etc. + +It is not always possible to load a model trained with older versions of the +scikit-learn library and its dependencies in an updated software environment. +Instead, you might need to retrain the model with the new versions of all +the libraries. So when training a model, it is important to record the training +recipe (e.g. a Python script) and training set information, and metadata about +all the dependencies to be able to automatically reconstruct the same training +environment for the updated software. + +.. dropdown:: InconsistentVersionWarning + + When an estimator is loaded with a scikit-learn version that is inconsistent + with the version the estimator was pickled with, an + :class:`~sklearn.exceptions.InconsistentVersionWarning` is raised. This warning + can be caught to obtain the original version the estimator was pickled with:: + + from sklearn.exceptions import InconsistentVersionWarning + warnings.simplefilter("error", InconsistentVersionWarning) + + try: + with open("model_from_previous_version.pickle", "rb") as f: + est = pickle.load(f) + except InconsistentVersionWarning as w: + print(w.original_sklearn_version) + + +Serving the model artifact +.......................... + +The last step after training a scikit-learn model is serving the model. +Once the trained model is successfully loaded, it can be served to manage +different prediction requests. This can involve deploying the model as a +web service using containerization, or other model deployment strategies, +according to the specifications. + + +Summarizing the key points +-------------------------- + +Based on the different approaches for model persistence, the key points for +each approach can be summarized as follows: + +* `ONNX`: It provides a uniform format for persisting any machine learning or + deep learning model (other than scikit-learn) and is useful for model + inference (predictions). It can however, result in compatibility issues with + different frameworks. +* :mod:`skops.io`: Trained scikit-learn models can be easily shared and put + into production using :mod:`skops.io`. It is more secure compared to + alternate approaches based on :mod:`pickle` because it does not load + arbitrary code unless explicitly asked for by the user. Such code needs to be + packaged and importable in the target Python environment. +* :mod:`joblib`: Efficient memory mapping techniques make it faster when using + the same persisted model in multiple Python processes when using + `mmap_mode="r"`. It also gives easy shortcuts to compress and decompress the + persisted object without the need for extra code. However, it may trigger the + execution of malicious code when loading a model from an untrusted source as + any other pickle-based persistence mechanism. +* :mod:`pickle`: It is native to Python and most Python objects can be + serialized and deserialized using :mod:`pickle`, including custom Python + classes and functions as long as they are defined in a package that can be + imported in the target environment. While :mod:`pickle` can be used to easily + save and load scikit-learn models, it may trigger the execution of malicious + code while loading a model from an untrusted source. :mod:`pickle` can also + be very efficient memorywise if the model was persisted with `protocol=5` but + it does not support memory mapping. +* `cloudpickle`_: It has comparable loading efficiency as :mod:`pickle` and + :mod:`joblib` (without memory mapping), but offers additional flexibility to + serialize custom Python code such as lambda expressions and interactively + defined functions and classes. It might be a last resort to persist pipelines + with custom Python components such as a + :class:`sklearn.preprocessing.FunctionTransformer` that wraps a function + defined in the training script itself or more generally outside of any + importable Python package. Note that `cloudpickle`_ offers no forward + compatibility guarantees and you might need the same version of + `cloudpickle`_ to load the persisted model along with the same version of all + the libraries used to define the model. As the other pickle-based persistence + mechanisms, it may trigger the execution of malicious code while loading + a model from an untrusted source. + +.. _cloudpickle: https://github.com/cloudpipe/cloudpickle diff --git a/doc/model_selection.rst b/doc/model_selection.rst index 25cd2b655ccc5..b78c9ff4c3aa8 100644 --- a/doc/model_selection.rst +++ b/doc/model_selection.rst @@ -1,9 +1,3 @@ -.. Places parent toc into the sidebar - -:parenttoc: True - -.. include:: includes/big_toc_css.rst - .. _model_selection: Model selection and evaluation @@ -14,5 +8,6 @@ Model selection and evaluation modules/cross_validation modules/grid_search + modules/classification_threshold modules/model_evaluation modules/learning_curve diff --git a/doc/modules/array_api.rst b/doc/modules/array_api.rst new file mode 100644 index 0000000000000..ee049937f5ce0 --- /dev/null +++ b/doc/modules/array_api.rst @@ -0,0 +1,266 @@ +.. _array_api: + +================================ +Array API support (experimental) +================================ + +.. currentmodule:: sklearn + +The `Array API `_ specification defines +a standard API for all array manipulation libraries with a NumPy-like API. +Scikit-learn vendors pinned copies of +`array-api-compat `__ +and `array-api-extra `__. + +Scikit-learn's support for the array API standard requires the environment variable +`SCIPY_ARRAY_API` to be set to `1` before importing `scipy` and `scikit-learn`: + +.. prompt:: bash $ + + export SCIPY_ARRAY_API=1 + +Please note that this environment variable is intended for temporary use. +For more details, refer to SciPy's `Array API documentation +`_. + +Some scikit-learn estimators that primarily rely on NumPy (as opposed to using +Cython) to implement the algorithmic logic of their `fit`, `predict` or +`transform` methods can be configured to accept any Array API compatible input +data structures and automatically dispatch operations to the underlying namespace +instead of relying on NumPy. + +At this stage, this support is **considered experimental** and must be enabled +explicitly as explained in the following. + +.. note:: + Currently, only `array-api-strict`, `cupy`, and `PyTorch` are known to work + with scikit-learn's estimators. + +The following video provides an overview of the standard's design principles +and how it facilitates interoperability between array libraries: + +- `Scikit-learn on GPUs with Array API `_ + by :user:`Thomas Fan ` at PyData NYC 2023. + +Example usage +============= + +Here is an example code snippet to demonstrate how to use `CuPy +`_ to run +:class:`~discriminant_analysis.LinearDiscriminantAnalysis` on a GPU:: + + >>> from sklearn.datasets import make_classification + >>> from sklearn import config_context + >>> from sklearn.discriminant_analysis import LinearDiscriminantAnalysis + >>> import cupy + + >>> X_np, y_np = make_classification(random_state=0) + >>> X_cu = cupy.asarray(X_np) + >>> y_cu = cupy.asarray(y_np) + >>> X_cu.device + + + >>> with config_context(array_api_dispatch=True): + ... lda = LinearDiscriminantAnalysis() + ... X_trans = lda.fit_transform(X_cu, y_cu) + >>> X_trans.device + + +After the model is trained, fitted attributes that are arrays will also be +from the same Array API namespace as the training data. For example, if CuPy's +Array API namespace was used for training, then fitted attributes will be on the +GPU. We provide an experimental `_estimator_with_converted_arrays` utility that +transfers an estimator attributes from Array API to a ndarray:: + + >>> from sklearn.utils._array_api import _estimator_with_converted_arrays + >>> cupy_to_ndarray = lambda array : array.get() + >>> lda_np = _estimator_with_converted_arrays(lda, cupy_to_ndarray) + >>> X_trans = lda_np.transform(X_np) + >>> type(X_trans) + + +PyTorch Support +--------------- + +PyTorch Tensors are supported by setting `array_api_dispatch=True` and passing in +the tensors directly:: + + >>> import torch + >>> X_torch = torch.asarray(X_np, device="cuda", dtype=torch.float32) + >>> y_torch = torch.asarray(y_np, device="cuda", dtype=torch.float32) + + >>> with config_context(array_api_dispatch=True): + ... lda = LinearDiscriminantAnalysis() + ... X_trans = lda.fit_transform(X_torch, y_torch) + >>> type(X_trans) + + >>> X_trans.device.type + 'cuda' + +.. _array_api_supported: + +Support for `Array API`-compatible inputs +========================================= + +Estimators and other tools in scikit-learn that support Array API compatible inputs. + +Estimators +---------- + +- :class:`decomposition.PCA` (with `svd_solver="full"`, + `svd_solver="randomized"` and `power_iteration_normalizer="QR"`) +- :class:`linear_model.Ridge` (with `solver="svd"`) +- :class:`discriminant_analysis.LinearDiscriminantAnalysis` (with `solver="svd"`) +- :class:`preprocessing.Binarizer` +- :class:`preprocessing.KernelCenterer` +- :class:`preprocessing.LabelEncoder` +- :class:`preprocessing.MaxAbsScaler` +- :class:`preprocessing.MinMaxScaler` +- :class:`preprocessing.Normalizer` + +Meta-estimators +--------------- + +Meta-estimators that accept Array API inputs conditioned on the fact that the +base estimator also does: + +- :class:`model_selection.GridSearchCV` +- :class:`model_selection.RandomizedSearchCV` +- :class:`model_selection.HalvingGridSearchCV` +- :class:`model_selection.HalvingRandomSearchCV` + +Metrics +------- + +- :func:`sklearn.metrics.cluster.entropy` +- :func:`sklearn.metrics.accuracy_score` +- :func:`sklearn.metrics.d2_tweedie_score` +- :func:`sklearn.metrics.explained_variance_score` +- :func:`sklearn.metrics.f1_score` +- :func:`sklearn.metrics.fbeta_score` +- :func:`sklearn.metrics.hamming_loss` +- :func:`sklearn.metrics.jaccard_score` +- :func:`sklearn.metrics.max_error` +- :func:`sklearn.metrics.mean_absolute_error` +- :func:`sklearn.metrics.mean_absolute_percentage_error` +- :func:`sklearn.metrics.mean_gamma_deviance` +- :func:`sklearn.metrics.mean_pinball_loss` +- :func:`sklearn.metrics.mean_poisson_deviance` (requires `enabling array API support for SciPy `_) +- :func:`sklearn.metrics.mean_squared_error` +- :func:`sklearn.metrics.mean_squared_log_error` +- :func:`sklearn.metrics.mean_tweedie_deviance` +- :func:`sklearn.metrics.median_absolute_error` +- :func:`sklearn.metrics.multilabel_confusion_matrix` +- :func:`sklearn.metrics.pairwise.additive_chi2_kernel` +- :func:`sklearn.metrics.pairwise.chi2_kernel` +- :func:`sklearn.metrics.pairwise.cosine_similarity` +- :func:`sklearn.metrics.pairwise.cosine_distances` +- :func:`sklearn.metrics.pairwise.euclidean_distances` (see :ref:`device_support_for_float64`) +- :func:`sklearn.metrics.pairwise.linear_kernel` +- :func:`sklearn.metrics.pairwise.paired_cosine_distances` +- :func:`sklearn.metrics.pairwise.paired_euclidean_distances` +- :func:`sklearn.metrics.pairwise.polynomial_kernel` +- :func:`sklearn.metrics.pairwise.rbf_kernel` (see :ref:`device_support_for_float64`) +- :func:`sklearn.metrics.pairwise.sigmoid_kernel` +- :func:`sklearn.metrics.precision_score` +- :func:`sklearn.metrics.precision_recall_fscore_support` +- :func:`sklearn.metrics.r2_score` +- :func:`sklearn.metrics.recall_score` +- :func:`sklearn.metrics.root_mean_squared_error` +- :func:`sklearn.metrics.root_mean_squared_log_error` +- :func:`sklearn.metrics.zero_one_loss` + +Tools +----- + +- :func:`model_selection.train_test_split` +- :func:`utils.check_consistent_length` + +Coverage is expected to grow over time. Please follow the dedicated `meta-issue on GitHub +`_ to track progress. + +Type of return values and fitted attributes +------------------------------------------- + +When calling functions or methods with Array API compatible inputs, the +convention is to return array values of the same array container type and +device as the input data. + +Similarly, when an estimator is fitted with Array API compatible inputs, the +fitted attributes will be arrays from the same library as the input and stored +on the same device. The `predict` and `transform` method subsequently expect +inputs from the same array library and device as the data passed to the `fit` +method. + +Note however that scoring functions that return scalar values return Python +scalars (typically a `float` instance) instead of an array scalar value. + +Common estimator checks +======================= + +Add the `array_api_support` tag to an estimator's set of tags to indicate that +it supports the Array API. This will enable dedicated checks as part of the +common tests to verify that the estimators' results are the same when using +vanilla NumPy and Array API inputs. + +To run these checks you need to install +`array-api-strict `_ in your +test environment. This allows you to run checks without having a +GPU. To run the full set of checks you also need to install +`PyTorch `_, `CuPy `_ and have +a GPU. Checks that can not be executed or have missing dependencies will be +automatically skipped. Therefore it's important to run the tests with the +`-v` flag to see which checks are skipped: + +.. prompt:: bash $ + + pip install array-api-strict # and other libraries as needed + pytest -k "array_api" -v + +Running the scikit-learn tests against `array-api-strict` should help reveal +most code problems related to handling multiple device inputs via the use of +simulated non-CPU devices. This allows for fast iterative development and debugging of +array API related code. + +However, to ensure full handling of PyTorch or CuPy inputs allocated on actual GPU +devices, it is necessary to run the tests against those libraries and hardware. +This can either be achieved by using +`Google Colab `_ +or leveraging our CI infrastructure on pull requests (manually triggered by maintainers +for cost reasons). + +.. _mps_support: + +Note on MPS device support +-------------------------- + +On macOS, PyTorch can use the Metal Performance Shaders (MPS) to access +hardware accelerators (e.g. the internal GPU component of the M1 or M2 chips). +However, the MPS device support for PyTorch is incomplete at the time of +writing. See the following github issue for more details: + +- https://github.com/pytorch/pytorch/issues/77764 + +To enable the MPS support in PyTorch, set the environment variable +`PYTORCH_ENABLE_MPS_FALLBACK=1` before running the tests: + +.. prompt:: bash $ + + PYTORCH_ENABLE_MPS_FALLBACK=1 pytest -k "array_api" -v + +At the time of writing all scikit-learn tests should pass, however, the +computational speed is not necessarily better than with the CPU device. + +.. _device_support_for_float64: + +Note on device support for ``float64`` +-------------------------------------- + +Certain operations within scikit-learn will automatically perform operations +on floating-point values with `float64` precision to prevent overflows and ensure +correctness (e.g., :func:`metrics.pairwise.euclidean_distances`). However, +certain combinations of array namespaces and devices, such as `PyTorch on MPS` +(see :ref:`mps_support`) do not support the `float64` data type. In these cases, +scikit-learn will revert to using the `float32` data type instead. This can result in +different behavior (typically numerically unstable results) compared to not using array +API dispatching or using a device with `float64` support. diff --git a/doc/modules/biclustering.rst b/doc/modules/biclustering.rst index 7ec175883d4cd..41c2316c753ad 100644 --- a/doc/modules/biclustering.rst +++ b/doc/modules/biclustering.rst @@ -4,8 +4,7 @@ Biclustering ============ -Biclustering can be performed with the module -:mod:`sklearn.cluster.bicluster`. Biclustering algorithms simultaneously +Biclustering algorithms simultaneously cluster rows and columns of a data matrix. These clusters of rows and columns are known as biclusters. Each determines a submatrix of the original data matrix with some desired properties. @@ -82,7 +81,7 @@ diagonal and checkerboard bicluster structures. these alternate names. -.. currentmodule:: sklearn.cluster.bicluster +.. currentmodule:: sklearn.cluster .. _spectral_coclustering: @@ -148,21 +147,21 @@ Then the rows of :math:`Z` are clustered using :ref:`k-means and the remaining ``n_columns`` labels provide the column partitioning. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_bicluster_plot_spectral_coclustering.py`: A simple example - showing how to generate a data matrix with biclusters and apply - this method to it. +* :ref:`sphx_glr_auto_examples_bicluster_plot_spectral_coclustering.py`: A simple example + showing how to generate a data matrix with biclusters and apply + this method to it. - * :ref:`sphx_glr_auto_examples_bicluster_plot_bicluster_newsgroups.py`: An example of finding - biclusters in the twenty newsgroup dataset. +* :ref:`sphx_glr_auto_examples_bicluster_plot_bicluster_newsgroups.py`: An example of finding + biclusters in the twenty newsgroup dataset. -.. topic:: References: +.. rubric:: References - * Dhillon, Inderjit S, 2001. `Co-clustering documents and words using - bipartite spectral graph partitioning - `__. +* Dhillon, Inderjit S, 2001. :doi:`Co-clustering documents and words using + bipartite spectral graph partitioning + <10.1145/502512.502550>` .. _spectral_biclustering: @@ -221,7 +220,7 @@ Given these singular vectors, they are ranked according to which can be best approximated by a piecewise-constant vector. The approximations for each vector are found using one-dimensional k-means and scored using the Euclidean distance. Some subset of the best left -and right singular vector are selected. Next, the data is projected to +and right singular vectors are selected. Next, the data is projected to this best subset of singular vectors and clustered. For instance, if :math:`p` singular vectors were calculated, the @@ -235,17 +234,17 @@ Similarly, projecting the columns to :math:`A^{\top} * U` and clustering this :math:`n \times q` matrix yields the column labels. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_bicluster_plot_spectral_biclustering.py`: a simple example - showing how to generate a checkerboard matrix and bicluster it. +* :ref:`sphx_glr_auto_examples_bicluster_plot_spectral_biclustering.py`: a simple example + showing how to generate a checkerboard matrix and bicluster it. -.. topic:: References: +.. rubric:: References - * Kluger, Yuval, et. al., 2003. `Spectral biclustering of microarray - data: coclustering genes and conditions - `__. +* Kluger, Yuval, et. al., 2003. :doi:`Spectral biclustering of microarray + data: coclustering genes and conditions + <10.1101/gr.648603>` .. _biclustering_evaluation: @@ -277,7 +276,7 @@ now, only the Jaccard index is implemented: where :math:`A` and :math:`B` are biclusters, :math:`|A \cap B|` is the number of elements in their intersection. The Jaccard index -achieves its minimum of 0 when the biclusters to not overlap at all +achieves its minimum of 0 when the biclusters do not overlap at all and its maximum of 1 when they are identical. Several methods have been developed to compare two sets of biclusters. @@ -289,7 +288,8 @@ available: 2. Assign biclusters from one set to another in a one-to-one fashion to maximize the sum of their similarities. This step is performed - using the Hungarian algorithm. + using :func:`scipy.optimize.linear_sum_assignment`, which uses a + modified Jonker-Volgenant algorithm. 3. The final sum of similarities is divided by the size of the larger set. @@ -299,8 +299,8 @@ are totally dissimilar. The maximum score, 1, occurs when both sets are identical. -.. topic:: References: +.. rubric:: References - * Hochreiter, Bodenhofer, et. al., 2010. `FABIA: factor analysis - for bicluster acquisition - `__. +* Hochreiter, Bodenhofer, et. al., 2010. `FABIA: factor analysis + for bicluster acquisition + `__. diff --git a/doc/modules/calibration.rst b/doc/modules/calibration.rst index d0a9737dac612..e8e6aa8b9953a 100644 --- a/doc/modules/calibration.rst +++ b/doc/modules/calibration.rst @@ -20,21 +20,53 @@ prediction. Well calibrated classifiers are probabilistic classifiers for which the output of the :term:`predict_proba` method can be directly interpreted as a confidence level. -For instance, a well calibrated (binary) classifier should classify the samples -such that among the samples to which it gave a :term:`predict_proba` value -close to 0.8, -approximately 80% actually belong to the positive class. +For instance, a well calibrated (binary) classifier should classify the samples such +that among the samples to which it gave a :term:`predict_proba` value close to, say, +0.8, approximately 80% actually belong to the positive class. + +Before we show how to re-calibrate a classifier, we first need a way to detect how +good a classifier is calibrated. + +.. note:: + Strictly proper scoring rules for probabilistic predictions like + :func:`sklearn.metrics.brier_score_loss` and + :func:`sklearn.metrics.log_loss` assess calibration (reliability) and + discriminative power (resolution) of a model, as well as the randomness of the data + (uncertainty) at the same time. This follows from the well-known Brier score + decomposition of Murphy [1]_. As it is not clear which term dominates, the score is + of limited use for assessing calibration alone (unless one computes each term of + the decomposition). A lower Brier loss, for instance, does not necessarily + mean a better calibrated model, it could also mean a worse calibrated model with much + more discriminatory power, e.g. using many more features. .. _calibration_curve: Calibration curves ------------------ -The following plot compares how well the probabilistic predictions of -different classifiers are calibrated, using :func:`calibration_curve`. -The x axis represents the average predicted probability in each bin. The -y axis is the *fraction of positives*, i.e. the proportion of samples whose -class is the positive class (in each bin). +Calibration curves, also referred to as *reliability diagrams* (Wilks 1995 [2]_), +compare how well the probabilistic predictions of a binary classifier are calibrated. +It plots the frequency of the positive label (to be more precise, an estimation of the +*conditional event probability* :math:`P(Y=1|\text{predict_proba})`) on the y-axis +against the predicted probability :term:`predict_proba` of a model on the x-axis. +The tricky part is to get values for the y-axis. +In scikit-learn, this is accomplished by binning the predictions such that the x-axis +represents the average predicted probability in each bin. +The y-axis is then the *fraction of positives* given the predictions of that bin, i.e. +the proportion of samples whose class is the positive class (in each bin). + +The top calibration curve plot is created with +:func:`CalibrationDisplay.from_estimator`, which uses :func:`calibration_curve` to +calculate the per bin average predicted probabilities and fraction of positives. +:func:`CalibrationDisplay.from_estimator` +takes as input a fitted classifier, which is used to calculate the predicted +probabilities. The classifier thus must have :term:`predict_proba` method. For +the few classifiers that do not have a :term:`predict_proba` method, it is +possible to use :class:`CalibratedClassifierCV` to calibrate the classifier +outputs to probabilities. + +The bottom histogram gives some insight into the behavior of each classifier +by showing the number of samples in each predicted probability bin. .. figure:: ../auto_examples/calibration/images/sphx_glr_plot_compare_calibration_001.png :target: ../auto_examples/calibration/plot_compare_calibration.html @@ -42,13 +74,20 @@ class is the positive class (in each bin). .. currentmodule:: sklearn.linear_model -:class:`LogisticRegression` returns well calibrated predictions by default as it directly -optimizes :ref:`log_loss`. In contrast, the other methods return biased probabilities; -with different biases per method: +:class:`LogisticRegression` is more likely to return well calibrated predictions by itself as it has a +canonical link function for its loss, i.e. the logit-link for the :ref:`log_loss`. +In the unpenalized case, this leads to the so-called **balance property**, see [8]_ and :ref:`Logistic_regression`. +In the plot above, data is generated according to a linear mechanism, which is +consistent with the :class:`LogisticRegression` model (the model is 'well specified'), +and the value of the regularization parameter `C` is tuned to be +appropriate (neither too strong nor too low). As a consequence, this model returns +accurate predictions from its `predict_proba` method. +In contrast to that, the other shown models return biased probabilities; with +different biases per model. .. currentmodule:: sklearn.naive_bayes -:class:`GaussianNB` tends to push probabilities to 0 or 1 (note the counts +:class:`GaussianNB` (Naive Bayes) tends to push probabilities to 0 or 1 (note the counts in the histograms). This is mainly because it makes the assumption that features are conditionally independent given the class, which is not the case in this dataset which contains 2 redundant features. @@ -56,33 +95,31 @@ case in this dataset which contains 2 redundant features. .. currentmodule:: sklearn.ensemble :class:`RandomForestClassifier` shows the opposite behavior: the histograms -show peaks at approximately 0.2 and 0.9 probability, while probabilities +show peaks at probabilities approximately 0.2 and 0.9, while probabilities close to 0 or 1 are very rare. An explanation for this is given by -Niculescu-Mizil and Caruana [1]_: "Methods such as bagging and random +Niculescu-Mizil and Caruana [3]_: "Methods such as bagging and random forests that average predictions from a base set of models can have difficulty making predictions near 0 and 1 because variance in the underlying base models will bias predictions that should be near zero or one away from these values. Because predictions are restricted to the interval [0,1], errors caused by variance tend to be one-sided near zero and one. For -example, if a model should predict p = 0 for a case, the only way bagging +example, if a model should predict :math:`p = 0` for a case, the only way bagging can achieve this is if all bagged trees predict zero. If we add noise to the trees that bagging is averaging over, this noise will cause some trees to predict values larger than 0 for this case, thus moving the average prediction of the bagged ensemble away from 0. We observe this effect most strongly with random forests because the base-level trees trained with random forests have relatively high variance due to feature subsetting." As -a result, the calibration curve also referred to as the reliability diagram -(Wilks 1995 [2]_) shows a characteristic sigmoid shape, indicating that the -classifier could trust its "intuition" more and return probabilities closer +a result, the calibration curve shows a characteristic sigmoid shape, indicating that +the classifier could trust its "intuition" more and return probabilities closer to 0 or 1 typically. .. currentmodule:: sklearn.svm -Linear Support Vector Classification (:class:`LinearSVC`) shows an even more -sigmoid curve than :class:`~sklearn.ensemble.RandomForestClassifier`, which is -typical for maximum-margin methods (compare Niculescu-Mizil and Caruana [1]_), -which focus on difficult to classify samples that are close to the decision -boundary (the support vectors). +:class:`LinearSVC` (SVC) shows an even more sigmoid curve than the random forest, which +is typical for maximum-margin methods (compare Niculescu-Mizil and Caruana [3]_), which +focus on difficult to classify samples that are close to the decision boundary (the +support vectors). Calibrating a classifier ------------------------ @@ -93,10 +130,11 @@ Calibrating a classifier consists of fitting a regressor (called a *calibrator*) that maps the output of the classifier (as given by :term:`decision_function` or :term:`predict_proba`) to a calibrated probability in [0, 1]. Denoting the output of the classifier for a given sample by :math:`f_i`, -the calibrator tries to predict :math:`p(y_i = 1 | f_i)`. +the calibrator tries to predict the conditional event probability +:math:`P(y_i = 1 | f_i)`. -The samples that are used to fit the calibrator should not be the same -samples used to fit the classifier, as this would introduce bias. +Ideally, the calibrator is fit on a dataset independent of the training data used to +fit the classifier in the first place. This is because performance of the classifier on its training data would be better than for novel data. Using the classifier output of training data to fit the calibrator would thus result in a biased calibrator that maps to @@ -108,22 +146,36 @@ Usage The :class:`CalibratedClassifierCV` class is used to calibrate a classifier. :class:`CalibratedClassifierCV` uses a cross-validation approach to ensure -unbiased data is always used to fit the calibrator. The data is split into k +unbiased data is always used to fit the calibrator. The data is split into :math:`k` `(train_set, test_set)` couples (as determined by `cv`). When `ensemble=True` (default), the following procedure is repeated independently for each -cross-validation split: a clone of `base_estimator` is first trained on the -train subset. Then its predictions on the test subset are used to fit a -calibrator (either a sigmoid or isotonic regressor). This results in an -ensemble of k `(classifier, calibrator)` couples where each calibrator maps +cross-validation split: + +1. a clone of `base_estimator` is trained on the train subset +2. the trained `base_estimator` makes predictions on the test subset +3. the predictions are used to fit a calibrator (either a sigmoid or isotonic + regressor) (when the data is multiclass, a calibrator is fit for every class) + +This results in an +ensemble of :math:`k` `(classifier, calibrator)` couples where each calibrator maps the output of its corresponding classifier into [0, 1]. Each couple is exposed in the `calibrated_classifiers_` attribute, where each entry is a calibrated classifier with a :term:`predict_proba` method that outputs calibrated probabilities. The output of :term:`predict_proba` for the main :class:`CalibratedClassifierCV` instance corresponds to the average of the -predicted probabilities of the `k` estimators in the `calibrated_classifiers_` +predicted probabilities of the :math:`k` estimators in the `calibrated_classifiers_` list. The output of :term:`predict` is the class that has the highest probability. +It is important to choose `cv` carefully when using `ensemble=True`. +All classes should be present in both train and test subsets for every split. +When a class is absent in the train subset, the predicted probability for that +class will default to 0 for the `(classifier, calibrator)` couple of that split. +This skews the :term:`predict_proba` as it averages across all couples. +When a class is absent in the test subset, the calibrator for that class +(within the `(classifier, calibrator)` couple of that split) is +fit on data with no positive class. This results in ineffective calibration. + When `ensemble=False`, cross-validation is used to obtain 'unbiased' predictions for all the data, via :func:`~sklearn.model_selection.cross_val_predict`. @@ -141,33 +193,24 @@ The main advantage of using `ensemble=False` is computational: it reduces the overall fit time by training only a single base classifier and calibrator pair, decreases the final model size and increases prediction speed. -Alternatively an already fitted classifier can be calibrated by setting -`cv="prefit"`. In this case, the data is not split and all of it is used to -fit the regressor. It is up to the user to -make sure that the data used for fitting the classifier is disjoint from the -data used for fitting the regressor. - -:func:`sklearn.metrics.brier_score_loss` may be used to assess how -well a classifier is calibrated. However, this metric should be used with care -because a lower Brier score does not always mean a better calibrated model. -This is because the Brier score metric is a combination of calibration loss -and refinement loss. Calibration loss is defined as the mean squared deviation -from empirical probabilities derived from the slope of ROC segments. -Refinement loss can be defined as the expected optimal loss as measured by the -area under the optimal cost curve. As refinement loss can change -independently from calibration loss, a lower Brier score does not necessarily -mean a better calibrated model. - -:class:`CalibratedClassifierCV` supports the use of two 'calibration' -regressors: 'sigmoid' and 'isotonic'. +Alternatively an already fitted classifier can be calibrated by using a +:class:`~sklearn.frozen.FrozenEstimator` as +``CalibratedClassifierCV(estimator=FrozenEstimator(estimator))``. +It is up to the user to make sure that the data used for fitting the classifier +is disjoint from the data used for fitting the regressor. + +:class:`CalibratedClassifierCV` supports the use of two regression techniques +for calibration via the `method` parameter: `"sigmoid"` and `"isotonic"`. + +.. _sigmoid_regressor: Sigmoid ^^^^^^^ -The sigmoid regressor is based on Platt's logistic model [3]_: +The sigmoid regressor, `method="sigmoid"` is based on Platt's logistic model [4]_: .. math:: - p(y_i = 1 | f_i) = \frac{1}{1 + \exp(A f_i + B)} + p(y_i = 1 | f_i) = \frac{1}{1 + \exp(A f_i + B)} \,, where :math:`y_i` is the true label of sample :math:`i` and :math:`f_i` is the output of the un-calibrated classifier for sample :math:`i`. :math:`A` @@ -178,37 +221,46 @@ The sigmoid method assumes the :ref:`calibration curve ` can be corrected by applying a sigmoid function to the raw predictions. This assumption has been empirically justified in the case of :ref:`svm` with common kernel functions on various benchmark datasets in section 2.1 of Platt -1999 [3]_ but does not necessarily hold in general. Additionally, the +1999 [4]_ but does not necessarily hold in general. Additionally, the logistic model works best if the calibration error is symmetrical, meaning the classifier output for each binary class is normally distributed with -the same variance [6]_. This can be a problem for highly imbalanced +the same variance [7]_. This can be a problem for highly imbalanced classification problems, where outputs do not have equal variance. -In general this method is most effective when the un-calibrated model is -under-confident and has similar calibration errors for both high and low -outputs. +In general this method is most effective for small sample sizes or when the +un-calibrated model is under-confident and has similar calibration errors for both +high and low outputs. Isotonic ^^^^^^^^ -The 'isotonic' method fits a non-parametric isotonic regressor, which outputs -a step-wise non-decreasing function (see :mod:`sklearn.isotonic`). It -minimizes: +The `method="isotonic"` fits a non-parametric isotonic regressor, which outputs +a step-wise non-decreasing function, see :mod:`sklearn.isotonic`. It minimizes: .. math:: \sum_{i=1}^{n} (y_i - \hat{f}_i)^2 -subject to :math:`\hat{f}_i >= \hat{f}_j` whenever -:math:`f_i >= f_j`. :math:`y_i` is the true +subject to :math:`\hat{f}_i \geq \hat{f}_j` whenever +:math:`f_i \geq f_j`. :math:`y_i` is the true label of sample :math:`i` and :math:`\hat{f}_i` is the output of the calibrated classifier for sample :math:`i` (i.e., the calibrated probability). -This method is more general when compared to 'sigmoid' as the only restriction +This method is more general when compared to `'sigmoid'` as the only restriction is that the mapping function is monotonically increasing. It is thus more powerful as it can correct any monotonic distortion of the un-calibrated model. -However, it is more prone to overfitting, especially on small datasets [5]_. +However, it is more prone to overfitting, especially on small datasets [6]_. -Overall, 'isotonic' will perform as well as or better than 'sigmoid' when -there is enough data (greater than ~ 1000 samples) to avoid overfitting [1]_. +Overall, `'isotonic'` will perform as well as or better than `'sigmoid'` when +there is enough data (greater than ~ 1000 samples) to avoid overfitting [3]_. + +.. note:: Impact on ranking metrics like AUC + + It is generally expected that calibration does not affect ranking metrics such as + ROC-AUC. However, these metrics might differ after calibration when using + `method="isotonic"` since isotonic regression introduces ties in the predicted + probabilities. This can be seen as within the uncertainty of the model predictions. + In case, you strictly want to keep the ranking and thus AUC scores, use + `method="sigmoid"` which is a strictly monotonic transformation and thus keeps + the ranking. Multiclass support ^^^^^^^^^^^^^^^^^^ @@ -218,46 +270,57 @@ support 1-dimensional data (e.g., binary classification output) but are extended for multiclass classification if the `base_estimator` supports multiclass predictions. For multiclass predictions, :class:`CalibratedClassifierCV` calibrates for -each class separately in a :ref:`ovr_classification` fashion [4]_. When +each class separately in a :ref:`ovr_classification` fashion [5]_. When predicting probabilities, the calibrated probabilities for each class are predicted separately. As those probabilities do not necessarily sum to one, a postprocessing is performed to normalize them. -.. topic:: Examples: +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_calibration_plot_calibration_curve.py` +* :ref:`sphx_glr_auto_examples_calibration_plot_calibration_multiclass.py` +* :ref:`sphx_glr_auto_examples_calibration_plot_calibration.py` +* :ref:`sphx_glr_auto_examples_calibration_plot_compare_calibration.py` + +.. rubric:: References + +.. [1] Allan H. Murphy (1973). + :doi:`"A New Vector Partition of the Probability Score" + <10.1175/1520-0450(1973)012%3C0595:ANVPOT%3E2.0.CO;2>` + Journal of Applied Meteorology and Climatology - * :ref:`sphx_glr_auto_examples_calibration_plot_calibration_curve.py` - * :ref:`sphx_glr_auto_examples_calibration_plot_calibration_multiclass.py` - * :ref:`sphx_glr_auto_examples_calibration_plot_calibration.py` - * :ref:`sphx_glr_auto_examples_calibration_plot_compare_calibration.py` +.. [2] `On the combination of forecast probabilities for + consecutive precipitation periods. + `_ + Wea. Forecasting, 5, 640–650., Wilks, D. S., 1990a -.. topic:: References: +.. [3] `Predicting Good Probabilities with Supervised Learning + `_, + A. Niculescu-Mizil & R. Caruana, ICML 2005 - .. [1] `Predicting Good Probabilities with Supervised Learning - `_, - A. Niculescu-Mizil & R. Caruana, ICML 2005 - .. [2] `On the combination of forecast probabilities for - consecutive precipitation periods. - `_ - Wea. Forecasting, 5, 640–650., Wilks, D. S., 1990a +.. [4] `Probabilistic Outputs for Support Vector Machines and Comparisons + to Regularized Likelihood Methods. + `_ + J. Platt, (1999) - .. [3] `Probabilistic Outputs for Support Vector Machines and Comparisons - to Regularized Likelihood Methods. - `_ - J. Platt, (1999) +.. [5] `Transforming Classifier Scores into Accurate Multiclass + Probability Estimates. + `_ + B. Zadrozny & C. Elkan, (KDD 2002) - .. [4] `Transforming Classifier Scores into Accurate Multiclass - Probability Estimates. - `_ - B. Zadrozny & C. Elkan, (KDD 2002) +.. [6] `Predicting accurate probabilities with a ranking loss. + `_ + Menon AK, Jiang XJ, Vembu S, Elkan C, Ohno-Machado L. + Proc Int Conf Mach Learn. 2012;2012:703-710 - .. [5] `Predicting accurate probabilities with a ranking loss. - `_ - Menon AK, Jiang XJ, Vembu S, Elkan C, Ohno-Machado L. - Proc Int Conf Mach Learn. 2012;2012:703-710 +.. [7] `Beyond sigmoids: How to obtain well-calibrated probabilities from + binary classifiers with beta calibration + `_ + Kull, M., Silva Filho, T. M., & Flach, P. (2017). - .. [6] `Beyond sigmoids: How to obtain well-calibrated probabilities from - binary classifiers with beta calibration - `_ - Kull, M., Silva Filho, T. M., & Flach, P. (2017). +.. [8] Mario V. WÃŧthrich, Michael Merz (2023). + :doi:`"Statistical Foundations of Actuarial Learning and its Applications" + <10.1007/978-3-031-12409-9>` + Springer Actuarial diff --git a/doc/modules/classes.rst b/doc/modules/classes.rst deleted file mode 100644 index c658bc6b12452..0000000000000 --- a/doc/modules/classes.rst +++ /dev/null @@ -1,1643 +0,0 @@ -.. _api_ref: - -============= -API Reference -============= - -This is the class and function reference of scikit-learn. Please refer to -the :ref:`full user guide ` for further details, as the class and -function raw specifications may not be enough to give full guidelines on their -uses. -For reference on concepts repeated across the API, see :ref:`glossary`. - - -:mod:`sklearn.base`: Base classes and utility functions -======================================================= - -.. automodule:: sklearn.base - :no-members: - :no-inherited-members: - -Base classes ------------- -.. currentmodule:: sklearn - -.. autosummary:: - :nosignatures: - :toctree: generated/ - :template: class.rst - - base.BaseEstimator - base.BiclusterMixin - base.ClassifierMixin - base.ClusterMixin - base.DensityMixin - base.RegressorMixin - base.TransformerMixin - feature_selection.SelectorMixin - -Functions ---------- -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - base.clone - base.is_classifier - base.is_regressor - config_context - get_config - set_config - show_versions - -.. _calibration_ref: - -:mod:`sklearn.calibration`: Probability Calibration -=================================================== - -.. automodule:: sklearn.calibration - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`calibration` section for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - calibration.CalibratedClassifierCV - - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - calibration.calibration_curve - -.. _cluster_ref: - -:mod:`sklearn.cluster`: Clustering -================================== - -.. automodule:: sklearn.cluster - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`clustering` and :ref:`biclustering` sections for -further details. - -Classes -------- -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - cluster.AffinityPropagation - cluster.AgglomerativeClustering - cluster.Birch - cluster.DBSCAN - cluster.FeatureAgglomeration - cluster.KMeans - cluster.MiniBatchKMeans - cluster.MeanShift - cluster.OPTICS - cluster.SpectralClustering - cluster.SpectralBiclustering - cluster.SpectralCoclustering - -Functions ---------- -.. autosummary:: - :toctree: generated/ - :template: function.rst - - cluster.affinity_propagation - cluster.cluster_optics_dbscan - cluster.cluster_optics_xi - cluster.compute_optics_graph - cluster.dbscan - cluster.estimate_bandwidth - cluster.k_means - cluster.kmeans_plusplus - cluster.mean_shift - cluster.spectral_clustering - cluster.ward_tree - -.. _compose_ref: - -:mod:`sklearn.compose`: Composite Estimators -============================================ - -.. automodule:: sklearn.compose - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`combining_estimators` section for further -details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated - :template: class.rst - - compose.ColumnTransformer - compose.TransformedTargetRegressor - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - compose.make_column_transformer - compose.make_column_selector - -.. _covariance_ref: - -:mod:`sklearn.covariance`: Covariance Estimators -================================================ - -.. automodule:: sklearn.covariance - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`covariance` section for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - covariance.EmpiricalCovariance - covariance.EllipticEnvelope - covariance.GraphicalLasso - covariance.GraphicalLassoCV - covariance.LedoitWolf - covariance.MinCovDet - covariance.OAS - covariance.ShrunkCovariance - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - covariance.empirical_covariance - covariance.graphical_lasso - covariance.ledoit_wolf - covariance.oas - covariance.shrunk_covariance - -.. _cross_decomposition_ref: - -:mod:`sklearn.cross_decomposition`: Cross decomposition -======================================================= - -.. automodule:: sklearn.cross_decomposition - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`cross_decomposition` section for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - cross_decomposition.CCA - cross_decomposition.PLSCanonical - cross_decomposition.PLSRegression - cross_decomposition.PLSSVD - -.. _datasets_ref: - -:mod:`sklearn.datasets`: Datasets -================================= - -.. automodule:: sklearn.datasets - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`datasets` section for further details. - -Loaders -------- - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - datasets.clear_data_home - datasets.dump_svmlight_file - datasets.fetch_20newsgroups - datasets.fetch_20newsgroups_vectorized - datasets.fetch_california_housing - datasets.fetch_covtype - datasets.fetch_kddcup99 - datasets.fetch_lfw_pairs - datasets.fetch_lfw_people - datasets.fetch_olivetti_faces - datasets.fetch_openml - datasets.fetch_rcv1 - datasets.fetch_species_distributions - datasets.get_data_home - datasets.load_boston - datasets.load_breast_cancer - datasets.load_diabetes - datasets.load_digits - datasets.load_files - datasets.load_iris - datasets.load_linnerud - datasets.load_sample_image - datasets.load_sample_images - datasets.load_svmlight_file - datasets.load_svmlight_files - datasets.load_wine - -Samples generator ------------------ - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - datasets.make_biclusters - datasets.make_blobs - datasets.make_checkerboard - datasets.make_circles - datasets.make_classification - datasets.make_friedman1 - datasets.make_friedman2 - datasets.make_friedman3 - datasets.make_gaussian_quantiles - datasets.make_hastie_10_2 - datasets.make_low_rank_matrix - datasets.make_moons - datasets.make_multilabel_classification - datasets.make_regression - datasets.make_s_curve - datasets.make_sparse_coded_signal - datasets.make_sparse_spd_matrix - datasets.make_sparse_uncorrelated - datasets.make_spd_matrix - datasets.make_swiss_roll - - -.. _decomposition_ref: - -:mod:`sklearn.decomposition`: Matrix Decomposition -================================================== - -.. automodule:: sklearn.decomposition - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`decompositions` section for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - decomposition.DictionaryLearning - decomposition.FactorAnalysis - decomposition.FastICA - decomposition.IncrementalPCA - decomposition.KernelPCA - decomposition.LatentDirichletAllocation - decomposition.MiniBatchDictionaryLearning - decomposition.MiniBatchSparsePCA - decomposition.NMF - decomposition.PCA - decomposition.SparsePCA - decomposition.SparseCoder - decomposition.TruncatedSVD - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - decomposition.dict_learning - decomposition.dict_learning_online - decomposition.fastica - decomposition.non_negative_factorization - decomposition.sparse_encode - -.. _lda_ref: - -:mod:`sklearn.discriminant_analysis`: Discriminant Analysis -=========================================================== - -.. automodule:: sklearn.discriminant_analysis - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`lda_qda` section for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated - :template: class.rst - - discriminant_analysis.LinearDiscriminantAnalysis - discriminant_analysis.QuadraticDiscriminantAnalysis - -.. _dummy_ref: - -:mod:`sklearn.dummy`: Dummy estimators -====================================== - -.. automodule:: sklearn.dummy - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`model_evaluation` section for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - dummy.DummyClassifier - dummy.DummyRegressor - -.. autosummary:: - :toctree: generated/ - :template: function.rst - -.. _ensemble_ref: - -:mod:`sklearn.ensemble`: Ensemble Methods -========================================= - -.. automodule:: sklearn.ensemble - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`ensemble` section for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - ensemble.AdaBoostClassifier - ensemble.AdaBoostRegressor - ensemble.BaggingClassifier - ensemble.BaggingRegressor - ensemble.ExtraTreesClassifier - ensemble.ExtraTreesRegressor - ensemble.GradientBoostingClassifier - ensemble.GradientBoostingRegressor - ensemble.IsolationForest - ensemble.RandomForestClassifier - ensemble.RandomForestRegressor - ensemble.RandomTreesEmbedding - ensemble.StackingClassifier - ensemble.StackingRegressor - ensemble.VotingClassifier - ensemble.VotingRegressor - ensemble.HistGradientBoostingRegressor - ensemble.HistGradientBoostingClassifier - - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - -.. _exceptions_ref: - -:mod:`sklearn.exceptions`: Exceptions and warnings -================================================== - -.. automodule:: sklearn.exceptions - :no-members: - :no-inherited-members: - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - exceptions.ConvergenceWarning - exceptions.DataConversionWarning - exceptions.DataDimensionalityWarning - exceptions.EfficiencyWarning - exceptions.FitFailedWarning - exceptions.NotFittedError - exceptions.UndefinedMetricWarning - - -:mod:`sklearn.experimental`: Experimental -========================================= - -.. automodule:: sklearn.experimental - :no-members: - :no-inherited-members: - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - - experimental.enable_hist_gradient_boosting - experimental.enable_iterative_imputer - experimental.enable_halving_search_cv - - -.. _feature_extraction_ref: - -:mod:`sklearn.feature_extraction`: Feature Extraction -===================================================== - -.. automodule:: sklearn.feature_extraction - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`feature_extraction` section for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - feature_extraction.DictVectorizer - feature_extraction.FeatureHasher - -From images ------------ - -.. automodule:: sklearn.feature_extraction.image - :no-members: - :no-inherited-members: - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - feature_extraction.image.extract_patches_2d - feature_extraction.image.grid_to_graph - feature_extraction.image.img_to_graph - feature_extraction.image.reconstruct_from_patches_2d - - :template: class.rst - - feature_extraction.image.PatchExtractor - -.. _text_feature_extraction_ref: - -From text ---------- - -.. automodule:: sklearn.feature_extraction.text - :no-members: - :no-inherited-members: - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - feature_extraction.text.CountVectorizer - feature_extraction.text.HashingVectorizer - feature_extraction.text.TfidfTransformer - feature_extraction.text.TfidfVectorizer - - -.. _feature_selection_ref: - -:mod:`sklearn.feature_selection`: Feature Selection -=================================================== - -.. automodule:: sklearn.feature_selection - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`feature_selection` section for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - feature_selection.GenericUnivariateSelect - feature_selection.SelectPercentile - feature_selection.SelectKBest - feature_selection.SelectFpr - feature_selection.SelectFdr - feature_selection.SelectFromModel - feature_selection.SelectFwe - feature_selection.SequentialFeatureSelector - feature_selection.RFE - feature_selection.RFECV - feature_selection.VarianceThreshold - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - feature_selection.chi2 - feature_selection.f_classif - feature_selection.f_regression - feature_selection.mutual_info_classif - feature_selection.mutual_info_regression - - -.. _gaussian_process_ref: - -:mod:`sklearn.gaussian_process`: Gaussian Processes -=================================================== - -.. automodule:: sklearn.gaussian_process - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`gaussian_process` section for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - gaussian_process.GaussianProcessClassifier - gaussian_process.GaussianProcessRegressor - -Kernels: - -.. autosummary:: - :toctree: generated/ - :template: class_with_call.rst - - gaussian_process.kernels.CompoundKernel - gaussian_process.kernels.ConstantKernel - gaussian_process.kernels.DotProduct - gaussian_process.kernels.ExpSineSquared - gaussian_process.kernels.Exponentiation - gaussian_process.kernels.Hyperparameter - gaussian_process.kernels.Kernel - gaussian_process.kernels.Matern - gaussian_process.kernels.PairwiseKernel - gaussian_process.kernels.Product - gaussian_process.kernels.RBF - gaussian_process.kernels.RationalQuadratic - gaussian_process.kernels.Sum - gaussian_process.kernels.WhiteKernel - - -.. _impute_ref: - -:mod:`sklearn.impute`: Impute -============================= - -.. automodule:: sklearn.impute - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`Impute` section for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - impute.SimpleImputer - impute.IterativeImputer - impute.MissingIndicator - impute.KNNImputer - - -.. _inspection_ref: - -:mod:`sklearn.inspection`: Inspection -===================================== - -.. automodule:: sklearn.inspection - :no-members: - :no-inherited-members: - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - inspection.partial_dependence - inspection.permutation_importance - -Plotting --------- - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - inspection.PartialDependenceDisplay - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - inspection.plot_partial_dependence - -.. _isotonic_ref: - -:mod:`sklearn.isotonic`: Isotonic regression -============================================ - -.. automodule:: sklearn.isotonic - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`isotonic` section for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - isotonic.IsotonicRegression - -.. autosummary:: - :toctree: generated - :template: function.rst - - isotonic.check_increasing - isotonic.isotonic_regression - - -.. _kernel_approximation_ref: - -:mod:`sklearn.kernel_approximation`: Kernel Approximation -========================================================= - -.. automodule:: sklearn.kernel_approximation - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`kernel_approximation` section for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - kernel_approximation.AdditiveChi2Sampler - kernel_approximation.Nystroem - kernel_approximation.PolynomialCountSketch - kernel_approximation.RBFSampler - kernel_approximation.SkewedChi2Sampler - -.. _kernel_ridge_ref: - -:mod:`sklearn.kernel_ridge`: Kernel Ridge Regression -==================================================== - -.. automodule:: sklearn.kernel_ridge - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`kernel_ridge` section for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - kernel_ridge.KernelRidge - -.. _linear_model_ref: - -:mod:`sklearn.linear_model`: Linear Models -========================================== - -.. automodule:: sklearn.linear_model - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`linear_model` section for further details. - -The following subsections are only rough guidelines: the same estimator can -fall into multiple categories, depending on its parameters. - -.. currentmodule:: sklearn - -Linear classifiers ------------------- -.. autosummary:: - :toctree: generated/ - :template: class.rst - - linear_model.LogisticRegression - linear_model.LogisticRegressionCV - linear_model.PassiveAggressiveClassifier - linear_model.Perceptron - linear_model.RidgeClassifier - linear_model.RidgeClassifierCV - linear_model.SGDClassifier - -Classical linear regressors ---------------------------- - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - linear_model.LinearRegression - linear_model.Ridge - linear_model.RidgeCV - linear_model.SGDRegressor - -Regressors with variable selection ----------------------------------- - -The following estimators have built-in variable selection fitting -procedures, but any estimator using a L1 or elastic-net penalty also -performs variable selection: typically :class:`~linear_model.SGDRegressor` -or :class:`~sklearn.linear_model.SGDClassifier` with an appropriate penalty. - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - linear_model.ElasticNet - linear_model.ElasticNetCV - linear_model.Lars - linear_model.LarsCV - linear_model.Lasso - linear_model.LassoCV - linear_model.LassoLars - linear_model.LassoLarsCV - linear_model.LassoLarsIC - linear_model.OrthogonalMatchingPursuit - linear_model.OrthogonalMatchingPursuitCV - -Bayesian regressors -------------------- - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - linear_model.ARDRegression - linear_model.BayesianRidge - -Multi-task linear regressors with variable selection ----------------------------------------------------- - -These estimators fit multiple regression problems (or tasks) jointly, while -inducing sparse coefficients. While the inferred coefficients may differ -between the tasks, they are constrained to agree on the features that are -selected (non-zero coefficients). - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - linear_model.MultiTaskElasticNet - linear_model.MultiTaskElasticNetCV - linear_model.MultiTaskLasso - linear_model.MultiTaskLassoCV - -Outlier-robust regressors -------------------------- - -Any estimator using the Huber loss would also be robust to outliers, e.g. -:class:`~linear_model.SGDRegressor` with ``loss='huber'``. - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - linear_model.HuberRegressor - linear_model.RANSACRegressor - linear_model.TheilSenRegressor - -Generalized linear models (GLM) for regression ----------------------------------------------- - -These models allow for response variables to have error distributions other -than a normal distribution: - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - linear_model.PoissonRegressor - linear_model.TweedieRegressor - linear_model.GammaRegressor - - -Miscellaneous -------------- - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - linear_model.PassiveAggressiveRegressor - linear_model.enet_path - linear_model.lars_path - linear_model.lars_path_gram - linear_model.lasso_path - linear_model.orthogonal_mp - linear_model.orthogonal_mp_gram - linear_model.ridge_regression - - -.. _manifold_ref: - -:mod:`sklearn.manifold`: Manifold Learning -========================================== - -.. automodule:: sklearn.manifold - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`manifold` section for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated - :template: class.rst - - manifold.Isomap - manifold.LocallyLinearEmbedding - manifold.MDS - manifold.SpectralEmbedding - manifold.TSNE - -.. autosummary:: - :toctree: generated - :template: function.rst - - manifold.locally_linear_embedding - manifold.smacof - manifold.spectral_embedding - manifold.trustworthiness - - -.. _metrics_ref: - -:mod:`sklearn.metrics`: Metrics -=============================== - -See the :ref:`model_evaluation` section and the :ref:`metrics` section of the -user guide for further details. - -.. automodule:: sklearn.metrics - :no-members: - :no-inherited-members: - -.. currentmodule:: sklearn - -Model Selection Interface -------------------------- -See the :ref:`scoring_parameter` section of the user guide for further -details. - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - metrics.check_scoring - metrics.get_scorer - metrics.make_scorer - -Classification metrics ----------------------- - -See the :ref:`classification_metrics` section of the user guide for further -details. - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - metrics.accuracy_score - metrics.auc - metrics.average_precision_score - metrics.balanced_accuracy_score - metrics.brier_score_loss - metrics.classification_report - metrics.cohen_kappa_score - metrics.confusion_matrix - metrics.dcg_score - metrics.det_curve - metrics.f1_score - metrics.fbeta_score - metrics.hamming_loss - metrics.hinge_loss - metrics.jaccard_score - metrics.log_loss - metrics.matthews_corrcoef - metrics.multilabel_confusion_matrix - metrics.ndcg_score - metrics.precision_recall_curve - metrics.precision_recall_fscore_support - metrics.precision_score - metrics.recall_score - metrics.roc_auc_score - metrics.roc_curve - metrics.top_k_accuracy_score - metrics.zero_one_loss - -Regression metrics ------------------- - -See the :ref:`regression_metrics` section of the user guide for further -details. - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - metrics.explained_variance_score - metrics.max_error - metrics.mean_absolute_error - metrics.mean_squared_error - metrics.mean_squared_log_error - metrics.median_absolute_error - metrics.mean_absolute_percentage_error - metrics.r2_score - metrics.mean_poisson_deviance - metrics.mean_gamma_deviance - metrics.mean_tweedie_deviance - metrics.mean_pinball_loss - -Multilabel ranking metrics --------------------------- -See the :ref:`multilabel_ranking_metrics` section of the user guide for further -details. - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - metrics.coverage_error - metrics.label_ranking_average_precision_score - metrics.label_ranking_loss - - -Clustering metrics ------------------- - -See the :ref:`clustering_evaluation` section of the user guide for further -details. - -.. automodule:: sklearn.metrics.cluster - :no-members: - :no-inherited-members: - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - metrics.adjusted_mutual_info_score - metrics.adjusted_rand_score - metrics.calinski_harabasz_score - metrics.davies_bouldin_score - metrics.completeness_score - metrics.cluster.contingency_matrix - metrics.cluster.pair_confusion_matrix - metrics.fowlkes_mallows_score - metrics.homogeneity_completeness_v_measure - metrics.homogeneity_score - metrics.mutual_info_score - metrics.normalized_mutual_info_score - metrics.rand_score - metrics.silhouette_score - metrics.silhouette_samples - metrics.v_measure_score - -Biclustering metrics --------------------- - -See the :ref:`biclustering_evaluation` section of the user guide for -further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - metrics.consensus_score - - -Pairwise metrics ----------------- - -See the :ref:`metrics` section of the user guide for further details. - -.. automodule:: sklearn.metrics.pairwise - :no-members: - :no-inherited-members: - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - metrics.pairwise.additive_chi2_kernel - metrics.pairwise.chi2_kernel - metrics.pairwise.cosine_similarity - metrics.pairwise.cosine_distances - metrics.pairwise.distance_metrics - metrics.pairwise.euclidean_distances - metrics.pairwise.haversine_distances - metrics.pairwise.kernel_metrics - metrics.pairwise.laplacian_kernel - metrics.pairwise.linear_kernel - metrics.pairwise.manhattan_distances - metrics.pairwise.nan_euclidean_distances - metrics.pairwise.pairwise_kernels - metrics.pairwise.polynomial_kernel - metrics.pairwise.rbf_kernel - metrics.pairwise.sigmoid_kernel - metrics.pairwise.paired_euclidean_distances - metrics.pairwise.paired_manhattan_distances - metrics.pairwise.paired_cosine_distances - metrics.pairwise.paired_distances - metrics.pairwise_distances - metrics.pairwise_distances_argmin - metrics.pairwise_distances_argmin_min - metrics.pairwise_distances_chunked - - -Plotting --------- - -See the :ref:`visualizations` section of the user guide for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - metrics.plot_confusion_matrix - metrics.plot_det_curve - metrics.plot_precision_recall_curve - metrics.plot_roc_curve - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - metrics.ConfusionMatrixDisplay - metrics.DetCurveDisplay - metrics.PrecisionRecallDisplay - metrics.RocCurveDisplay - - -.. _mixture_ref: - -:mod:`sklearn.mixture`: Gaussian Mixture Models -=============================================== - -.. automodule:: sklearn.mixture - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`mixture` section for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - mixture.BayesianGaussianMixture - mixture.GaussianMixture - -.. _modelselection_ref: - -:mod:`sklearn.model_selection`: Model Selection -=============================================== - -.. automodule:: sklearn.model_selection - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`cross_validation`, :ref:`grid_search` and -:ref:`learning_curve` sections for further details. - -Splitter Classes ----------------- - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - model_selection.GroupKFold - model_selection.GroupShuffleSplit - model_selection.KFold - model_selection.LeaveOneGroupOut - model_selection.LeavePGroupsOut - model_selection.LeaveOneOut - model_selection.LeavePOut - model_selection.PredefinedSplit - model_selection.RepeatedKFold - model_selection.RepeatedStratifiedKFold - model_selection.ShuffleSplit - model_selection.StratifiedKFold - model_selection.StratifiedShuffleSplit - model_selection.TimeSeriesSplit - -Splitter Functions ------------------- - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - model_selection.check_cv - model_selection.train_test_split - -.. _hyper_parameter_optimizers: - -Hyper-parameter optimizers --------------------------- - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - model_selection.GridSearchCV - model_selection.HalvingGridSearchCV - model_selection.ParameterGrid - model_selection.ParameterSampler - model_selection.RandomizedSearchCV - model_selection.HalvingRandomSearchCV - - -Model validation ----------------- - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - model_selection.cross_validate - model_selection.cross_val_predict - model_selection.cross_val_score - model_selection.learning_curve - model_selection.permutation_test_score - model_selection.validation_curve - -.. _multiclass_ref: - -:mod:`sklearn.multiclass`: Multiclass classification -==================================================== - -.. automodule:: sklearn.multiclass - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`multiclass_classification` section for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated - :template: class.rst - - multiclass.OneVsRestClassifier - multiclass.OneVsOneClassifier - multiclass.OutputCodeClassifier - -.. _multioutput_ref: - -:mod:`sklearn.multioutput`: Multioutput regression and classification -===================================================================== - -.. automodule:: sklearn.multioutput - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`multilabel_classification`, -:ref:`multiclass_multioutput_classification`, and -:ref:`multioutput_regression` sections for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated - :template: class.rst - - multioutput.ClassifierChain - multioutput.MultiOutputRegressor - multioutput.MultiOutputClassifier - multioutput.RegressorChain - -.. _naive_bayes_ref: - -:mod:`sklearn.naive_bayes`: Naive Bayes -======================================= - -.. automodule:: sklearn.naive_bayes - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`naive_bayes` section for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - naive_bayes.BernoulliNB - naive_bayes.CategoricalNB - naive_bayes.ComplementNB - naive_bayes.GaussianNB - naive_bayes.MultinomialNB - - -.. _neighbors_ref: - -:mod:`sklearn.neighbors`: Nearest Neighbors -=========================================== - -.. automodule:: sklearn.neighbors - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`neighbors` section for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - neighbors.BallTree - neighbors.DistanceMetric - neighbors.KDTree - neighbors.KernelDensity - neighbors.KNeighborsClassifier - neighbors.KNeighborsRegressor - neighbors.KNeighborsTransformer - neighbors.LocalOutlierFactor - neighbors.RadiusNeighborsClassifier - neighbors.RadiusNeighborsRegressor - neighbors.RadiusNeighborsTransformer - neighbors.NearestCentroid - neighbors.NearestNeighbors - neighbors.NeighborhoodComponentsAnalysis - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - neighbors.kneighbors_graph - neighbors.radius_neighbors_graph - -.. _neural_network_ref: - -:mod:`sklearn.neural_network`: Neural network models -==================================================== - -.. automodule:: sklearn.neural_network - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`neural_networks_supervised` and :ref:`neural_networks_unsupervised` sections for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - neural_network.BernoulliRBM - neural_network.MLPClassifier - neural_network.MLPRegressor - -.. _pipeline_ref: - -:mod:`sklearn.pipeline`: Pipeline -================================= - -.. automodule:: sklearn.pipeline - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`combining_estimators` section for further -details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - pipeline.FeatureUnion - pipeline.Pipeline - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - pipeline.make_pipeline - pipeline.make_union - -.. _preprocessing_ref: - -:mod:`sklearn.preprocessing`: Preprocessing and Normalization -============================================================= - -.. automodule:: sklearn.preprocessing - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`preprocessing` section for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - preprocessing.Binarizer - preprocessing.FunctionTransformer - preprocessing.KBinsDiscretizer - preprocessing.KernelCenterer - preprocessing.LabelBinarizer - preprocessing.LabelEncoder - preprocessing.MultiLabelBinarizer - preprocessing.MaxAbsScaler - preprocessing.MinMaxScaler - preprocessing.Normalizer - preprocessing.OneHotEncoder - preprocessing.OrdinalEncoder - preprocessing.PolynomialFeatures - preprocessing.PowerTransformer - preprocessing.QuantileTransformer - preprocessing.RobustScaler - preprocessing.SplineTransformer - preprocessing.StandardScaler - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - preprocessing.add_dummy_feature - preprocessing.binarize - preprocessing.label_binarize - preprocessing.maxabs_scale - preprocessing.minmax_scale - preprocessing.normalize - preprocessing.quantile_transform - preprocessing.robust_scale - preprocessing.scale - preprocessing.power_transform - - -.. _random_projection_ref: - -:mod:`sklearn.random_projection`: Random projection -=================================================== - -.. automodule:: sklearn.random_projection - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`random_projection` section for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - random_projection.GaussianRandomProjection - random_projection.SparseRandomProjection - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - random_projection.johnson_lindenstrauss_min_dim - - -.. _semi_supervised_ref: - -:mod:`sklearn.semi_supervised`: Semi-Supervised Learning -======================================================== - -.. automodule:: sklearn.semi_supervised - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`semi_supervised` section for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - semi_supervised.LabelPropagation - semi_supervised.LabelSpreading - semi_supervised.SelfTrainingClassifier - - -.. _svm_ref: - -:mod:`sklearn.svm`: Support Vector Machines -=========================================== - -.. automodule:: sklearn.svm - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`svm` section for further details. - -Estimators ----------- - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - svm.LinearSVC - svm.LinearSVR - svm.NuSVC - svm.NuSVR - svm.OneClassSVM - svm.SVC - svm.SVR - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - svm.l1_min_c - -.. _tree_ref: - -:mod:`sklearn.tree`: Decision Trees -=================================== - -.. automodule:: sklearn.tree - :no-members: - :no-inherited-members: - -**User guide:** See the :ref:`tree` section for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - tree.DecisionTreeClassifier - tree.DecisionTreeRegressor - tree.ExtraTreeClassifier - tree.ExtraTreeRegressor - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - tree.export_graphviz - tree.export_text - -Plotting --------- - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - tree.plot_tree - -.. _utils_ref: - -:mod:`sklearn.utils`: Utilities -=============================== - -.. automodule:: sklearn.utils - :no-members: - :no-inherited-members: - -**Developer guide:** See the :ref:`developers-utils` page for further details. - -.. currentmodule:: sklearn - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - utils.arrayfuncs.min_pos - utils.as_float_array - utils.assert_all_finite - utils.Bunch - utils.check_X_y - utils.check_array - utils.check_scalar - utils.check_consistent_length - utils.check_random_state - utils.class_weight.compute_class_weight - utils.class_weight.compute_sample_weight - utils.deprecated - utils.estimator_checks.check_estimator - utils.estimator_checks.parametrize_with_checks - utils.estimator_html_repr - utils.extmath.safe_sparse_dot - utils.extmath.randomized_range_finder - utils.extmath.randomized_svd - utils.extmath.fast_logdet - utils.extmath.density - utils.extmath.weighted_mode - utils.gen_even_slices - utils.graph.single_source_shortest_path_length - utils.graph_shortest_path.graph_shortest_path - utils.indexable - utils.metaestimators.if_delegate_has_method - utils.multiclass.type_of_target - utils.multiclass.is_multilabel - utils.multiclass.unique_labels - utils.murmurhash3_32 - utils.resample - utils._safe_indexing - utils.safe_mask - utils.safe_sqr - utils.shuffle - utils.sparsefuncs.incr_mean_variance_axis - utils.sparsefuncs.inplace_column_scale - utils.sparsefuncs.inplace_row_scale - utils.sparsefuncs.inplace_swap_row - utils.sparsefuncs.inplace_swap_column - utils.sparsefuncs.mean_variance_axis - utils.sparsefuncs.inplace_csr_column_scale - utils.sparsefuncs_fast.inplace_csr_row_normalize_l1 - utils.sparsefuncs_fast.inplace_csr_row_normalize_l2 - utils.random.sample_without_replacement - utils.validation.check_is_fitted - utils.validation.check_memory - utils.validation.check_symmetric - utils.validation.column_or_1d - utils.validation.has_fit_parameter - utils.all_estimators - -Utilities from joblib: - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - utils.parallel_backend - utils.register_parallel_backend - - -Recently deprecated -=================== - -To be removed in 1.0 (renaming of 0.25) ---------------------------------------- diff --git a/doc/modules/classification_threshold.rst b/doc/modules/classification_threshold.rst new file mode 100644 index 0000000000000..ee7028f469b5f --- /dev/null +++ b/doc/modules/classification_threshold.rst @@ -0,0 +1,159 @@ +.. currentmodule:: sklearn.model_selection + +.. _TunedThresholdClassifierCV: + +================================================== +Tuning the decision threshold for class prediction +================================================== + +Classification is best divided into two parts: + +* the statistical problem of learning a model to predict, ideally, class probabilities; +* the decision problem to take concrete action based on those probability predictions. + +Let's take a straightforward example related to weather forecasting: the first point is +related to answering "what is the chance that it will rain tomorrow?" while the second +point is related to answering "should I take an umbrella tomorrow?". + +When it comes to the scikit-learn API, the first point is addressed by providing scores +using :term:`predict_proba` or :term:`decision_function`. The former returns conditional +probability estimates :math:`P(y|X)` for each class, while the latter returns a decision +score for each class. + +The decision corresponding to the labels is obtained with :term:`predict`. In binary +classification, a decision rule or action is then defined by thresholding the scores, +leading to the prediction of a single class label for each sample. For binary +classification in scikit-learn, class labels predictions are obtained by hard-coded +cut-off rules: a positive class is predicted when the conditional probability +:math:`P(y|X)` is greater than 0.5 (obtained with :term:`predict_proba`) or if the +decision score is greater than 0 (obtained with :term:`decision_function`). + +Here, we show an example that illustrates the relation between conditional +probability estimates :math:`P(y|X)` and class labels:: + + >>> from sklearn.datasets import make_classification + >>> from sklearn.tree import DecisionTreeClassifier + >>> X, y = make_classification(random_state=0) + >>> classifier = DecisionTreeClassifier(max_depth=2, random_state=0).fit(X, y) + >>> classifier.predict_proba(X[:4]) + array([[0.94 , 0.06 ], + [0.94 , 0.06 ], + [0.0416, 0.9583], + [0.0416, 0.9583]]) + >>> classifier.predict(X[:4]) + array([0, 0, 1, 1]) + +While these hard-coded rules might at first seem reasonable as default behavior, they +are most certainly not ideal for most use cases. Let's illustrate with an example. + +Consider a scenario where a predictive model is being deployed to assist +physicians in detecting tumors. In this setting, physicians will most likely be +interested in identifying all patients with cancer and not missing anyone with cancer so +that they can provide them with the right treatment. In other words, physicians +prioritize achieving a high recall rate. This emphasis on recall comes, of course, with +the trade-off of potentially more false-positive predictions, reducing the precision of +the model. That is a risk physicians are willing to take because the cost of a missed +cancer is much higher than the cost of further diagnostic tests. Consequently, when it +comes to deciding whether to classify a patient as having cancer or not, it may be more +beneficial to classify them as positive for cancer when the conditional probability +estimate is much lower than 0.5. + +Post-tuning the decision threshold +================================== + +One solution to address the problem stated in the introduction is to tune the decision +threshold of the classifier once the model has been trained. The +:class:`~sklearn.model_selection.TunedThresholdClassifierCV` tunes this threshold using +an internal cross-validation. The optimum threshold is chosen to maximize a given +metric. + +The following image illustrates the tuning of the decision threshold for a gradient +boosting classifier. While the vanilla and tuned classifiers provide the same +:term:`predict_proba` outputs and thus the same Receiver Operating Characteristic (ROC) +and Precision-Recall curves, the class label predictions differ because of the tuned +decision threshold. The vanilla classifier predicts the class of interest for a +conditional probability greater than 0.5 while the tuned classifier predicts the class +of interest for a very low probability (around 0.02). This decision threshold optimizes +a utility metric defined by the business (in this case an insurance company). + +.. figure:: ../auto_examples/model_selection/images/sphx_glr_plot_cost_sensitive_learning_002.png + :target: ../auto_examples/model_selection/plot_cost_sensitive_learning.html + :align: center + +Options to tune the decision threshold +-------------------------------------- + +The decision threshold can be tuned through different strategies controlled by the +parameter `scoring`. + +One way to tune the threshold is by maximizing a pre-defined scikit-learn metric. These +metrics can be found by calling the function :func:`~sklearn.metrics.get_scorer_names`. +By default, the balanced accuracy is the metric used but be aware that one should choose +a meaningful metric for their use case. + +.. note:: + + It is important to notice that these metrics come with default parameters, notably + the label of the class of interest (i.e. `pos_label`). Thus, if this label is not + the right one for your application, you need to define a scorer and pass the right + `pos_label` (and additional parameters) using the + :func:`~sklearn.metrics.make_scorer`. Refer to :ref:`scoring_callable` to get + information to define your own scoring function. For instance, we show how to pass + the information to the scorer that the label of interest is `0` when maximizing the + :func:`~sklearn.metrics.f1_score`:: + + >>> from sklearn.linear_model import LogisticRegression + >>> from sklearn.model_selection import TunedThresholdClassifierCV + >>> from sklearn.metrics import make_scorer, f1_score + >>> X, y = make_classification( + ... n_samples=1_000, weights=[0.1, 0.9], random_state=0) + >>> pos_label = 0 + >>> scorer = make_scorer(f1_score, pos_label=pos_label) + >>> base_model = LogisticRegression() + >>> model = TunedThresholdClassifierCV(base_model, scoring=scorer) + >>> scorer(model.fit(X, y), X, y) + 0.88 + >>> # compare it with the internal score found by cross-validation + >>> model.best_score_ + np.float64(0.86) + +Important notes regarding the internal cross-validation +------------------------------------------------------- + +By default :class:`~sklearn.model_selection.TunedThresholdClassifierCV` uses a 5-fold +stratified cross-validation to tune the decision threshold. The parameter `cv` allows to +control the cross-validation strategy. It is possible to bypass cross-validation by +setting `cv="prefit"` and providing a fitted classifier. In this case, the decision +threshold is tuned on the data provided to the `fit` method. + +However, you should be extremely careful when using this option. You should never use +the same data for training the classifier and tuning the decision threshold due to the +risk of overfitting. Refer to the following example section for more details (cf. +:ref:`TunedThresholdClassifierCV_no_cv`). If you have limited resources, consider using +a float number for `cv` to limit to an internal single train-test split. + +The option `cv="prefit"` should only be used when the provided classifier was already +trained, and you just want to find the best decision threshold using a new validation +set. + +.. _FixedThresholdClassifier: + +Manually setting the decision threshold +--------------------------------------- + +The previous sections discussed strategies to find an optimal decision threshold. It is +also possible to manually set the decision threshold using the class +:class:`~sklearn.model_selection.FixedThresholdClassifier`. In case that you don't want +to refit the model when calling `fit`, wrap your sub-estimator with a +:class:`~sklearn.frozen.FrozenEstimator` and do +``FixedThresholdClassifier(FrozenEstimator(estimator), ...)``. + +Examples +-------- + +- See the example entitled + :ref:`sphx_glr_auto_examples_model_selection_plot_tuned_decision_threshold.py`, + to get insights on the post-tuning of the decision threshold. +- See the example entitled + :ref:`sphx_glr_auto_examples_model_selection_plot_cost_sensitive_learning.py`, + to learn about cost-sensitive learning and decision threshold tuning. diff --git a/doc/modules/clustering.rst b/doc/modules/clustering.rst index 17ae9eb2651c6..cdf8421a103e3 100644 --- a/doc/modules/clustering.rst +++ b/doc/modules/clustering.rst @@ -89,14 +89,22 @@ Overview of clustering methods * - :ref:`DBSCAN ` - neighborhood size - Very large ``n_samples``, medium ``n_clusters`` - - Non-flat geometry, uneven cluster sizes, transductive + - Non-flat geometry, uneven cluster sizes, outlier removal, + transductive + - Distances between nearest points + + * - :ref:`HDBSCAN ` + - minimum cluster membership, minimum point neighbors + - large ``n_samples``, medium ``n_clusters`` + - Non-flat geometry, uneven cluster sizes, outlier removal, + transductive, hierarchical, variable cluster density - Distances between nearest points * - :ref:`OPTICS ` - minimum cluster membership - Very large ``n_samples``, large ``n_clusters`` - Non-flat geometry, uneven cluster sizes, variable cluster density, - transductive + outlier removal, transductive - Distances between points * - :ref:`Gaussian mixtures ` @@ -111,6 +119,13 @@ Overview of clustering methods - Large dataset, outlier removal, data reduction, inductive - Euclidean distance between points + * - :ref:`Bisecting K-Means ` + - number of clusters + - Very large ``n_samples``, medium ``n_clusters`` + - General-purpose, even cluster size, flat geometry, + no empty clusters, inductive, hierarchical + - Distances between points + Non-flat geometry clustering is useful when the clusters have a specific shape, i.e. a non-flat manifold, and the standard euclidean distance is not the right metric. This case arises in the two top rows of the figure @@ -125,6 +140,11 @@ model with equal covariance per component. :term:`inductive` clustering methods) are not designed to be applied to new, unseen data. +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_cluster_plot_inductive_clustering.py`: An example + of an inductive clustering model for handling new data. + .. _k_means: K-means @@ -133,7 +153,7 @@ K-means The :class:`KMeans` algorithm clusters data by trying to separate samples in n groups of equal variance, minimizing a criterion known as the *inertia* or within-cluster sum-of-squares (see below). This algorithm requires the number -of clusters to be specified. It scales well to large number of samples and has +of clusters to be specified. It scales well to large numbers of samples and has been used across a large range of application areas in many different fields. The k-means algorithm divides a set of :math:`N` samples :math:`X` into @@ -162,11 +182,15 @@ It suffers from various drawbacks: k-means clustering can alleviate this problem and speed up the computations. -.. image:: ../auto_examples/cluster/images/sphx_glr_plot_kmeans_assumptions_001.png +.. image:: ../auto_examples/cluster/images/sphx_glr_plot_kmeans_assumptions_002.png :target: ../auto_examples/cluster/plot_kmeans_assumptions.html :align: center :scale: 50 +For more detailed descriptions of the issues shown above and how to address them, +refer to the examples :ref:`sphx_glr_auto_examples_cluster_plot_kmeans_assumptions.py` +and :ref:`sphx_glr_auto_examples_cluster_plot_kmeans_silhouette_analysis.py`. + K-means is often referred to as Lloyd's algorithm. In basic terms, the algorithm has three steps. The first step chooses the initial centroids, with the most basic method being to choose :math:`k` samples from the dataset @@ -202,10 +226,13 @@ As a result, the computation is often done several times, with different initializations of the centroids. One method to help address this issue is the k-means++ initialization scheme, which has been implemented in scikit-learn (use the ``init='k-means++'`` parameter). This initializes the centroids to be -(generally) distant from each other, leading to provably better results than -random initialization, as shown in the reference. +(generally) distant from each other, leading to probably better results than +random initialization, as shown in the reference. For detailed examples of +comparing different initialization schemes, refer to +:ref:`sphx_glr_auto_examples_cluster_plot_kmeans_digits.py` and +:ref:`sphx_glr_auto_examples_cluster_plot_kmeans_stability_low_dim_dense.py`. -K-means++ can also be called independently to select seeds for other +K-means++ can also be called independently to select seeds for other clustering algorithms, see :func:`sklearn.cluster.kmeans_plusplus` for details and example usage. @@ -215,8 +242,13 @@ computing cluster centers and values of inertia. For example, assigning a weight of 2 to a sample is equivalent to adding a duplicate of that sample to the dataset :math:`X`. -K-means can be used for vector quantization. This is achieved using the -transform method of a trained model of :class:`KMeans`. +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_text_plot_document_clustering.py`: Document clustering + using :class:`KMeans` and :class:`MiniBatchKMeans` based on sparse data + +* :ref:`sphx_glr_auto_examples_cluster_plot_kmeans_plusplus.py`: Using K-means++ + to select seeds for other clustering algorithms. Low-level parallelism --------------------- @@ -226,19 +258,20 @@ chunks of data (256 samples) are processed in parallel, which in addition yields a low memory footprint. For more details on how to control the number of threads, please refer to our :ref:`parallelism` notes. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_cluster_plot_kmeans_assumptions.py`: Demonstrating when - k-means performs intuitively and when it does not - * :ref:`sphx_glr_auto_examples_cluster_plot_kmeans_digits.py`: Clustering handwritten digits +* :ref:`sphx_glr_auto_examples_cluster_plot_kmeans_assumptions.py`: Demonstrating when + k-means performs intuitively and when it does not +* :ref:`sphx_glr_auto_examples_cluster_plot_kmeans_digits.py`: Clustering handwritten digits -.. topic:: References: +.. dropdown:: References + + * `"k-means++: The advantages of careful seeding" + `_ + Arthur, David, and Sergei Vassilvitskii, + *Proceedings of the eighteenth annual ACM-SIAM symposium on Discrete + algorithms*, Society for Industrial and Applied Mathematics (2007) - * `"k-means++: The advantages of careful seeding" - `_ - Arthur, David, and Sergei Vassilvitskii, - *Proceedings of the eighteenth annual ACM-SIAM symposium on Discrete - algorithms*, Society for Industrial and Applied Mathematics (2007) .. _mini_batch_kmeans: @@ -274,23 +307,22 @@ small, as shown in the example and cited reference. :scale: 100 -.. topic:: Examples: - - * :ref:`sphx_glr_auto_examples_cluster_plot_mini_batch_kmeans.py`: Comparison of KMeans and - MiniBatchKMeans +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_text_plot_document_clustering.py`: Document clustering using sparse - MiniBatchKMeans +* :ref:`sphx_glr_auto_examples_cluster_plot_mini_batch_kmeans.py`: Comparison of + :class:`KMeans` and :class:`MiniBatchKMeans` - * :ref:`sphx_glr_auto_examples_cluster_plot_dict_face_patches.py` +* :ref:`sphx_glr_auto_examples_text_plot_document_clustering.py`: Document clustering + using :class:`KMeans` and :class:`MiniBatchKMeans` based on sparse data +* :ref:`sphx_glr_auto_examples_cluster_plot_dict_face_patches.py` -.. topic:: References: +.. dropdown:: References - * `"Web Scale K-Means clustering" - `_ - D. Sculley, *Proceedings of the 19th international conference on World - wide web* (2010) + * `"Web Scale K-Means clustering" + `_ + D. Sculley, *Proceedings of the 19th international conference on World + wide web* (2010) .. _affinity_propagation: @@ -327,52 +359,51 @@ convergence. Further, the memory complexity is of the order sparse similarity matrix is used. This makes Affinity Propagation most appropriate for small to medium sized datasets. -.. topic:: Examples: +.. dropdown:: Algorithm description - * :ref:`sphx_glr_auto_examples_cluster_plot_affinity_propagation.py`: Affinity - Propagation on a synthetic 2D datasets with 3 classes. + The messages sent between points belong to one of two categories. The first is + the responsibility :math:`r(i, k)`, which is the accumulated evidence that + sample :math:`k` should be the exemplar for sample :math:`i`. The second is the + availability :math:`a(i, k)` which is the accumulated evidence that sample + :math:`i` should choose sample :math:`k` to be its exemplar, and considers the + values for all other samples that :math:`k` should be an exemplar. In this way, + exemplars are chosen by samples if they are (1) similar enough to many samples + and (2) chosen by many samples to be representative of themselves. - * :ref:`sphx_glr_auto_examples_applications_plot_stock_market.py` Affinity Propagation on - Financial time series to find groups of companies + More formally, the responsibility of a sample :math:`k` to be the exemplar of + sample :math:`i` is given by: + .. math:: -**Algorithm description:** -The messages sent between points belong to one of two categories. The first is -the responsibility :math:`r(i, k)`, -which is the accumulated evidence that sample :math:`k` -should be the exemplar for sample :math:`i`. -The second is the availability :math:`a(i, k)` -which is the accumulated evidence that sample :math:`i` -should choose sample :math:`k` to be its exemplar, -and considers the values for all other samples that :math:`k` should -be an exemplar. In this way, exemplars are chosen by samples if they are (1) -similar enough to many samples and (2) chosen by many samples to be -representative of themselves. + r(i, k) \leftarrow s(i, k) - max [ a(i, k') + s(i, k') \forall k' \neq k ] -More formally, the responsibility of a sample :math:`k` -to be the exemplar of sample :math:`i` is given by: + Where :math:`s(i, k)` is the similarity between samples :math:`i` and :math:`k`. + The availability of sample :math:`k` to be the exemplar of sample :math:`i` is + given by: -.. math:: + .. math:: - r(i, k) \leftarrow s(i, k) - max [ a(i, k') + s(i, k') \forall k' \neq k ] + a(i, k) \leftarrow min [0, r(k, k) + \sum_{i'~s.t.~i' \notin \{i, k\}}{r(i', + k)}] -Where :math:`s(i, k)` is the similarity between samples :math:`i` and :math:`k`. -The availability of sample :math:`k` -to be the exemplar of sample :math:`i` is given by: + To begin with, all values for :math:`r` and :math:`a` are set to zero, and the + calculation of each iterates until convergence. As discussed above, in order to + avoid numerical oscillations when updating the messages, the damping factor + :math:`\lambda` is introduced to iteration process: -.. math:: + .. math:: r_{t+1}(i, k) = \lambda\cdot r_{t}(i, k) + (1-\lambda)\cdot r_{t+1}(i, k) + .. math:: a_{t+1}(i, k) = \lambda\cdot a_{t}(i, k) + (1-\lambda)\cdot a_{t+1}(i, k) - a(i, k) \leftarrow min [0, r(k, k) + \sum_{i'~s.t.~i' \notin \{i, k\}}{r(i', k)}] + where :math:`t` indicates the iteration times. -To begin with, all values for :math:`r` and :math:`a` are set to zero, -and the calculation of each iterates until convergence. -As discussed above, in order to avoid numerical oscillations when updating the -messages, the damping factor :math:`\lambda` is introduced to iteration process: -.. math:: r_{t+1}(i, k) = \lambda\cdot r_{t}(i, k) + (1-\lambda)\cdot r_{t+1}(i, k) -.. math:: a_{t+1}(i, k) = \lambda\cdot a_{t}(i, k) + (1-\lambda)\cdot a_{t+1}(i, k) +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_cluster_plot_affinity_propagation.py`: Affinity + Propagation on a synthetic 2D datasets with 3 classes +* :ref:`sphx_glr_auto_examples_applications_plot_stock_market.py` Affinity Propagation + on financial time series to find groups of companies -where :math:`t` indicates the iteration times. .. _mean_shift: @@ -384,22 +415,40 @@ for centroids to be the mean of the points within a given region. These candidates are then filtered in a post-processing stage to eliminate near-duplicates to form the final set of centroids. -Given a candidate centroid :math:`x_i` for iteration :math:`t`, the candidate -is updated according to the following equation: +.. dropdown:: Mathematical details -.. math:: + The position of centroid candidates is iteratively adjusted using a technique + called hill climbing, which finds local maxima of the estimated probability + density. Given a candidate centroid :math:`x` for iteration :math:`t`, the + candidate is updated according to the following equation: - x_i^{t+1} = m(x_i^t) + .. math:: -Where :math:`N(x_i)` is the neighborhood of samples within a given distance -around :math:`x_i` and :math:`m` is the *mean shift* vector that is computed for each -centroid that points towards a region of the maximum increase in the density of points. -This is computed using the following equation, effectively updating a centroid -to be the mean of the samples within its neighborhood: + x^{t+1} = x^t + m(x^t) -.. math:: + Where :math:`m` is the *mean shift* vector that is computed for each centroid + that points towards a region of the maximum increase in the density of points. + To compute :math:`m` we define :math:`N(x)` as the neighborhood of samples + within a given distance around :math:`x`. Then :math:`m` is computed using the + following equation, effectively updating a centroid to be the mean of the + samples within its neighborhood: + + .. math:: + + m(x) = \frac{1}{|N(x)|} \sum_{x_j \in N(x)}x_j - x + + In general, the equation for :math:`m` depends on a kernel used for density + estimation. The generic formula is: + + .. math:: + + m(x) = \frac{\sum_{x_j \in N(x)}K(x_j - x)x_j}{\sum_{x_j \in N(x)}K(x_j - + x)} - x + + In our implementation, :math:`K(x)` is equal to 1 if :math:`x` is small enough + and is equal to 0 otherwise. Effectively :math:`K(y - x)` indicates whether + :math:`y` is in the neighborhood of :math:`x`. - m(x_i) = \frac{\sum_{x_j \in N(x_i)}K(x_j - x_i)x_j}{\sum_{x_j \in N(x_i)}K(x_j - x_i)} The algorithm automatically sets the number of clusters, instead of relying on a parameter ``bandwidth``, which dictates the size of the region to search through. @@ -421,16 +470,16 @@ given sample. :scale: 50 -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_cluster_plot_mean_shift.py`: Mean Shift clustering - on a synthetic 2D datasets with 3 classes. +* :ref:`sphx_glr_auto_examples_cluster_plot_mean_shift.py`: Mean Shift clustering + on a synthetic 2D datasets with 3 classes. -.. topic:: References: +.. dropdown:: References - * `"Mean shift: A robust approach toward feature space analysis." - `_ - D. Comaniciu and P. Meer, *IEEE Transactions on Pattern Analysis and Machine Intelligence* (2002) + * :doi:`"Mean shift: A robust approach toward feature space analysis" + <10.1109/34.1000236>` D. Comaniciu and P. Meer, *IEEE Transactions on Pattern + Analysis and Machine Intelligence* (2002) .. _spectral_clustering: @@ -450,7 +499,7 @@ to be specified in advance. It works well for a small number of clusters, but is not advised for many clusters. For two clusters, SpectralClustering solves a convex relaxation of the -`normalised cuts `_ +`normalized cuts `_ problem on the similarity graph: cutting the graph in two so that the weight of the edges cut is small compared to the weights of the edges inside each cluster. This criteria is especially interesting when working on images, where @@ -481,21 +530,26 @@ computed using a function of a gradient of the image. See the examples for such an application. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_cluster_plot_segmentation_toy.py`: Segmenting objects - from a noisy background using spectral clustering. +* :ref:`sphx_glr_auto_examples_cluster_plot_segmentation_toy.py`: Segmenting objects + from a noisy background using spectral clustering. +* :ref:`sphx_glr_auto_examples_cluster_plot_coin_segmentation.py`: Spectral clustering + to split the image of coins in regions. - * :ref:`sphx_glr_auto_examples_cluster_plot_coin_segmentation.py`: Spectral clustering - to split the image of coins in regions. .. |coin_kmeans| image:: ../auto_examples/cluster/images/sphx_glr_plot_coin_segmentation_001.png - :target: ../auto_examples/cluster/plot_coin_segmentation.html - :scale: 65 + :target: ../auto_examples/cluster/plot_coin_segmentation.html + :scale: 35 .. |coin_discretize| image:: ../auto_examples/cluster/images/sphx_glr_plot_coin_segmentation_002.png - :target: ../auto_examples/cluster/plot_coin_segmentation.html - :scale: 65 + :target: ../auto_examples/cluster/plot_coin_segmentation.html + :scale: 35 + +.. |coin_cluster_qr| image:: ../auto_examples/cluster/images/sphx_glr_plot_coin_segmentation_003.png + :target: ../auto_examples/cluster/plot_coin_segmentation.html + :scale: 35 + Different label assignment strategies ------------------------------------- @@ -507,12 +561,27 @@ In particular, unless you control the ``random_state``, it may not be reproducible from run-to-run, as it depends on random initialization. The alternative ``"discretize"`` strategy is 100% reproducible, but tends to create parcels of fairly even and geometrical shape. +The recently added ``"cluster_qr"`` option is a deterministic alternative that +tends to create the visually best partitioning on the example application +below. + +================================ ================================ ================================ + ``assign_labels="kmeans"`` ``assign_labels="discretize"`` ``assign_labels="cluster_qr"`` +================================ ================================ ================================ +|coin_kmeans| |coin_discretize| |coin_cluster_qr| +================================ ================================ ================================ + +.. dropdown:: References + + * `"Multiclass spectral clustering" + `_ + Stella X. Yu, Jianbo Shi, 2003 -===================================== ===================================== - ``assign_labels="kmeans"`` ``assign_labels="discretize"`` -===================================== ===================================== -|coin_kmeans| |coin_discretize| -===================================== ===================================== + * :doi:`"Simple, direct, and efficient multi-way spectral clustering"<10.1093/imaiai/iay008>` + Anil Damle, Victor Minden, Lexing Ying, 2019 + + +.. _spectral_clustering_graph: Spectral Clustering Graphs -------------------------- @@ -526,28 +595,25 @@ graph, and SpectralClustering is initialized with `affinity='precomputed'`:: ... assign_labels='discretize') >>> sc.fit_predict(adjacency_matrix) # doctest: +SKIP -.. topic:: References: +.. dropdown:: References - * `"A Tutorial on Spectral Clustering" - `_ - Ulrike von Luxburg, 2007 + * :doi:`"A Tutorial on Spectral Clustering" <10.1007/s11222-007-9033-z>` Ulrike + von Luxburg, 2007 - * `"Normalized cuts and image segmentation" - `_ - Jianbo Shi, Jitendra Malik, 2000 + * :doi:`"Normalized cuts and image segmentation" <10.1109/34.868688>` Jianbo + Shi, Jitendra Malik, 2000 - * `"A Random Walks View of Spectral Segmentation" - `_ - Marina Meila, Jianbo Shi, 2001 + * `"A Random Walks View of Spectral Segmentation" + `_ + Marina Meila, Jianbo Shi, 2001 - * `"On Spectral Clustering: Analysis and an algorithm" - `_ - Andrew Y. Ng, Michael I. Jordan, Yair Weiss, 2001 + * `"On Spectral Clustering: Analysis and an algorithm" + `_ + Andrew Y. Ng, Michael I. Jordan, Yair Weiss, 2001 + + * :arxiv:`"Preconditioned Spectral Clustering for Stochastic Block Partition + Streaming Graph Challenge" <1708.07481>` David Zhuzhunashvili, Andrew Knyazev - * `"Preconditioned Spectral Clustering for Stochastic - Block Partition Streaming Graph Challenge" - `_ - David Zhuzhunashvili, Andrew Knyazev .. _hierarchical_clustering: @@ -608,10 +674,14 @@ while not robust to noisy data, can be computed very efficiently and can therefore be useful to provide hierarchical clustering of larger datasets. Single linkage can also perform well on non-globular data. -.. topic:: Examples: +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_cluster_plot_digits_linkage.py`: exploration of the + different linkage strategies in a real dataset. + + * :ref:`sphx_glr_auto_examples_cluster_plot_linkage_comparison.py`: exploration of + the different linkage strategies in toy datasets. - * :ref:`sphx_glr_auto_examples_cluster_plot_digits_linkage.py`: exploration of the - different linkage strategies in a real dataset. Visualization of cluster hierarchy ---------------------------------- @@ -624,6 +694,9 @@ of the data, though more so in the case of small sample sizes. :target: ../auto_examples/cluster/plot_agglomerative_dendrogram.html :scale: 42 +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_cluster_plot_agglomerative_dendrogram.py` Adding connectivity constraints @@ -665,21 +738,6 @@ using :func:`sklearn.feature_extraction.image.grid_to_graph` to enable only merging of neighboring pixels on an image, as in the :ref:`coin ` example. -.. topic:: Examples: - - * :ref:`sphx_glr_auto_examples_cluster_plot_coin_ward_segmentation.py`: Ward clustering - to split the image of coins in regions. - - * :ref:`sphx_glr_auto_examples_cluster_plot_ward_structured_vs_unstructured.py`: Example of - Ward algorithm on a swiss-roll, comparison of structured approaches - versus unstructured approaches. - - * :ref:`sphx_glr_auto_examples_cluster_plot_feature_agglomeration_vs_univariate_selection.py`: - Example of dimensionality reduction with feature agglomeration based on - Ward hierarchical clustering. - - * :ref:`sphx_glr_auto_examples_cluster_plot_agglomerative_clustering.py` - .. warning:: **Connectivity constraints with single, average and complete linkage** Connectivity constraints and single, complete or average linkage can enhance @@ -707,6 +765,21 @@ enable only merging of neighboring pixels on an image, as in the :target: ../auto_examples/cluster/plot_agglomerative_clustering.html :scale: 38 +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_cluster_plot_coin_ward_segmentation.py`: Ward + clustering to split the image of coins in regions. + +* :ref:`sphx_glr_auto_examples_cluster_plot_ward_structured_vs_unstructured.py`: Example + of Ward algorithm on a swiss-roll, comparison of structured approaches + versus unstructured approaches. + +* :ref:`sphx_glr_auto_examples_cluster_plot_feature_agglomeration_vs_univariate_selection.py`: Example + of dimensionality reduction with feature agglomeration based on Ward + hierarchical clustering. + +* :ref:`sphx_glr_auto_examples_cluster_plot_agglomerative_clustering.py` + Varying the metric ------------------- @@ -739,9 +812,68 @@ each class. :target: ../auto_examples/cluster/plot_agglomerative_clustering_metrics.html :scale: 32 -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_cluster_plot_agglomerative_clustering_metrics.py` +* :ref:`sphx_glr_auto_examples_cluster_plot_agglomerative_clustering_metrics.py` + + +Bisecting K-Means +----------------- + +.. _bisect_k_means: + +The :class:`BisectingKMeans` is an iterative variant of :class:`KMeans`, using +divisive hierarchical clustering. Instead of creating all centroids at once, centroids +are picked progressively based on a previous clustering: a cluster is split into two +new clusters repeatedly until the target number of clusters is reached. + +:class:`BisectingKMeans` is more efficient than :class:`KMeans` when the number of +clusters is large since it only works on a subset of the data at each bisection +while :class:`KMeans` always works on the entire dataset. + +Although :class:`BisectingKMeans` can't benefit from the advantages of the `"k-means++"` +initialization by design, it will still produce comparable results than +`KMeans(init="k-means++")` in terms of inertia at cheaper computational costs, and will +likely produce better results than `KMeans` with a random initialization. + +This variant is more efficient to agglomerative clustering if the number of clusters is +small compared to the number of data points. + +This variant also does not produce empty clusters. + +There exist two strategies for selecting the cluster to split: + - ``bisecting_strategy="largest_cluster"`` selects the cluster having the most points + - ``bisecting_strategy="biggest_inertia"`` selects the cluster with biggest inertia + (cluster with biggest Sum of Squared Errors within) + +Picking by largest amount of data points in most cases produces result as +accurate as picking by inertia and is faster (especially for larger amount of data +points, where calculating error may be costly). + +Picking by largest amount of data points will also likely produce clusters of similar +sizes while `KMeans` is known to produce clusters of different sizes. + +Difference between Bisecting K-Means and regular K-Means can be seen on example +:ref:`sphx_glr_auto_examples_cluster_plot_bisect_kmeans.py`. +While the regular K-Means algorithm tends to create non-related clusters, +clusters from Bisecting K-Means are well ordered and create quite a visible hierarchy. + +.. dropdown:: References + + * `"A Comparison of Document Clustering Techniques" + `_ Michael + Steinbach, George Karypis and Vipin Kumar, Department of Computer Science and + Egineering, University of Minnesota (June 2000) + * `"Performance Analysis of K-Means and Bisecting K-Means Algorithms in Weblog + Data" + `_ + K.Abirami and Dr.P.Mayilvahanan, International Journal of Emerging + Technologies in Engineering Research (IJETER) Volume 4, Issue 8, (August 2016) + * `"Bisecting K-means Algorithm Based on K-valued Self-determining and + Clustering Center Optimization" + `_ Jian Di, Xinyue Gou School + of Control and Computer Engineering,North China Electric Power University, + Baoding, Hebei, China (August 2017) .. _dbscan: @@ -795,71 +927,179 @@ indicating core samples found by the algorithm. Smaller circles are non-core samples that are still part of a cluster. Moreover, the outliers are indicated by black points below. -.. |dbscan_results| image:: ../auto_examples/cluster/images/sphx_glr_plot_dbscan_001.png - :target: ../auto_examples/cluster/plot_dbscan.html - :scale: 50 +.. |dbscan_results| image:: ../auto_examples/cluster/images/sphx_glr_plot_dbscan_002.png + :target: ../auto_examples/cluster/plot_dbscan.html + :scale: 50 .. centered:: |dbscan_results| -.. topic:: Examples: - - * :ref:`sphx_glr_auto_examples_cluster_plot_dbscan.py` - -.. topic:: Implementation - - The DBSCAN algorithm is deterministic, always generating the same clusters - when given the same data in the same order. However, the results can differ when - data is provided in a different order. First, even though the core samples - will always be assigned to the same clusters, the labels of those clusters - will depend on the order in which those samples are encountered in the data. - Second and more importantly, the clusters to which non-core samples are assigned - can differ depending on the data order. This would happen when a non-core sample - has a distance lower than ``eps`` to two core samples in different clusters. By the - triangular inequality, those two core samples must be more distant than - ``eps`` from each other, or they would be in the same cluster. The non-core - sample is assigned to whichever cluster is generated first in a pass - through the data, and so the results will depend on the data ordering. - - The current implementation uses ball trees and kd-trees - to determine the neighborhood of points, - which avoids calculating the full distance matrix - (as was done in scikit-learn versions before 0.14). - The possibility to use custom metrics is retained; - for details, see :class:`NearestNeighbors`. - -.. topic:: Memory consumption for large sample sizes - - This implementation is by default not memory efficient because it constructs - a full pairwise similarity matrix in the case where kd-trees or ball-trees cannot - be used (e.g., with sparse matrices). This matrix will consume :math:`n^2` floats. - A couple of mechanisms for getting around this are: - - - Use :ref:`OPTICS ` clustering in conjunction with the - `extract_dbscan` method. OPTICS clustering also calculates the full - pairwise matrix, but only keeps one row in memory at a time (memory - complexity n). - - - A sparse radius neighborhood graph (where missing entries are presumed to - be out of eps) can be precomputed in a memory-efficient way and dbscan - can be run over this with ``metric='precomputed'``. See - :meth:`sklearn.neighbors.NearestNeighbors.radius_neighbors_graph`. - - - The dataset can be compressed, either by removing exact duplicates if - these occur in your data, or by using BIRCH. Then you only have a - relatively small number of representatives for a large number of points. - You can then provide a ``sample_weight`` when fitting DBSCAN. - -.. topic:: References: - - * "A Density-Based Algorithm for Discovering Clusters in Large Spatial Databases - with Noise" - Ester, M., H. P. Kriegel, J. Sander, and X. Xu, - In Proceedings of the 2nd International Conference on Knowledge Discovery - and Data Mining, Portland, OR, AAAI Press, pp. 226–231. 1996 - - * "DBSCAN revisited, revisited: why and how you should (still) use DBSCAN. - Schubert, E., Sander, J., Ester, M., Kriegel, H. P., & Xu, X. (2017). - In ACM Transactions on Database Systems (TODS), 42(3), 19. +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_cluster_plot_dbscan.py` + +.. dropdown:: Implementation + + The DBSCAN algorithm is deterministic, always generating the same clusters when + given the same data in the same order. However, the results can differ when + data is provided in a different order. First, even though the core samples will + always be assigned to the same clusters, the labels of those clusters will + depend on the order in which those samples are encountered in the data. Second + and more importantly, the clusters to which non-core samples are assigned can + differ depending on the data order. This would happen when a non-core sample + has a distance lower than ``eps`` to two core samples in different clusters. By + the triangular inequality, those two core samples must be more distant than + ``eps`` from each other, or they would be in the same cluster. The non-core + sample is assigned to whichever cluster is generated first in a pass through the + data, and so the results will depend on the data ordering. + + The current implementation uses ball trees and kd-trees to determine the + neighborhood of points, which avoids calculating the full distance matrix (as + was done in scikit-learn versions before 0.14). The possibility to use custom + metrics is retained; for details, see :class:`NearestNeighbors`. + +.. dropdown:: Memory consumption for large sample sizes + + This implementation is by default not memory efficient because it constructs a + full pairwise similarity matrix in the case where kd-trees or ball-trees cannot + be used (e.g., with sparse matrices). This matrix will consume :math:`n^2` + floats. A couple of mechanisms for getting around this are: + + - Use :ref:`OPTICS ` clustering in conjunction with the `extract_dbscan` + method. OPTICS clustering also calculates the full pairwise matrix, but only + keeps one row in memory at a time (memory complexity n). + + - A sparse radius neighborhood graph (where missing entries are presumed to be + out of eps) can be precomputed in a memory-efficient way and dbscan can be run + over this with ``metric='precomputed'``. See + :meth:`sklearn.neighbors.NearestNeighbors.radius_neighbors_graph`. + + - The dataset can be compressed, either by removing exact duplicates if these + occur in your data, or by using BIRCH. Then you only have a relatively small + number of representatives for a large number of points. You can then provide a + ``sample_weight`` when fitting DBSCAN. + +.. dropdown:: References + +* `A Density-Based Algorithm for Discovering Clusters in Large Spatial + Databases with Noise `_ + Ester, M., H. P. Kriegel, J. Sander, and X. Xu, In Proceedings of the 2nd + International Conference on Knowledge Discovery and Data Mining, Portland, OR, + AAAI Press, pp. 226-231. 1996 + +* :doi:`DBSCAN revisited, revisited: why and how you should (still) use DBSCAN. + <10.1145/3068335>` Schubert, E., Sander, J., Ester, M., Kriegel, H. P., & Xu, + X. (2017). In ACM Transactions on Database Systems (TODS), 42(3), 19. + + +.. _hdbscan: + +HDBSCAN +======= + +The :class:`HDBSCAN` algorithm can be seen as an extension of :class:`DBSCAN` +and :class:`OPTICS`. Specifically, :class:`DBSCAN` assumes that the clustering +criterion (i.e. density requirement) is *globally homogeneous*. +In other words, :class:`DBSCAN` may struggle to successfully capture clusters +with different densities. +:class:`HDBSCAN` alleviates this assumption and explores all possible density +scales by building an alternative representation of the clustering problem. + +.. note:: + + This implementation is adapted from the original implementation of HDBSCAN, + `scikit-learn-contrib/hdbscan `_ based on [LJ2017]_. + +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_cluster_plot_hdbscan.py` + +Mutual Reachability Graph +------------------------- + +HDBSCAN first defines :math:`d_c(x_p)`, the *core distance* of a sample :math:`x_p`, as the +distance to its `min_samples` th-nearest neighbor, counting itself. For example, +if `min_samples=5` and :math:`x_*` is the 5th-nearest neighbor of :math:`x_p` +then the core distance is: + +.. math:: d_c(x_p)=d(x_p, x_*). + +Next it defines :math:`d_m(x_p, x_q)`, the *mutual reachability distance* of two points +:math:`x_p, x_q`, as: + +.. math:: d_m(x_p, x_q) = \max\{d_c(x_p), d_c(x_q), d(x_p, x_q)\} + +These two notions allow us to construct the *mutual reachability graph* +:math:`G_{ms}` defined for a fixed choice of `min_samples` by associating each +sample :math:`x_p` with a vertex of the graph, and thus edges between points +:math:`x_p, x_q` are the mutual reachability distance :math:`d_m(x_p, x_q)` +between them. We may build subsets of this graph, denoted as +:math:`G_{ms,\varepsilon}`, by removing any edges with value greater than :math:`\varepsilon`: +from the original graph. Any points whose core distance is less than :math:`\varepsilon`: +are at this staged marked as noise. The remaining points are then clustered by +finding the connected components of this trimmed graph. + +.. note:: + + Taking the connected components of a trimmed graph :math:`G_{ms,\varepsilon}` is + equivalent to running DBSCAN* with `min_samples` and :math:`\varepsilon`. DBSCAN* is a + slightly modified version of DBSCAN mentioned in [CM2013]_. + +Hierarchical Clustering +----------------------- +HDBSCAN can be seen as an algorithm which performs DBSCAN* clustering across all +values of :math:`\varepsilon`. As mentioned prior, this is equivalent to finding the connected +components of the mutual reachability graphs for all values of :math:`\varepsilon`. To do this +efficiently, HDBSCAN first extracts a minimum spanning tree (MST) from the fully +-connected mutual reachability graph, then greedily cuts the edges with highest +weight. An outline of the HDBSCAN algorithm is as follows: + +1. Extract the MST of :math:`G_{ms}`. +2. Extend the MST by adding a "self edge" for each vertex, with weight equal + to the core distance of the underlying sample. +3. Initialize a single cluster and label for the MST. +4. Remove the edge with the greatest weight from the MST (ties are + removed simultaneously). +5. Assign cluster labels to the connected components which contain the + end points of the now-removed edge. If the component does not have at least + one edge it is instead assigned a "null" label marking it as noise. +6. Repeat 4-5 until there are no more connected components. + +HDBSCAN is therefore able to obtain all possible partitions achievable by +DBSCAN* for a fixed choice of `min_samples` in a hierarchical fashion. +Indeed, this allows HDBSCAN to perform clustering across multiple densities +and as such it no longer needs :math:`\varepsilon` to be given as a hyperparameter. Instead +it relies solely on the choice of `min_samples`, which tends to be a more robust +hyperparameter. + +.. |hdbscan_ground_truth| image:: ../auto_examples/cluster/images/sphx_glr_plot_hdbscan_005.png + :target: ../auto_examples/cluster/plot_hdbscan.html + :scale: 75 +.. |hdbscan_results| image:: ../auto_examples/cluster/images/sphx_glr_plot_hdbscan_007.png + :target: ../auto_examples/cluster/plot_hdbscan.html + :scale: 75 + +.. centered:: |hdbscan_ground_truth| +.. centered:: |hdbscan_results| + +HDBSCAN can be smoothed with an additional hyperparameter `min_cluster_size` +which specifies that during the hierarchical clustering, components with fewer +than `minimum_cluster_size` many samples are considered noise. In practice, one +can set `minimum_cluster_size = min_samples` to couple the parameters and +simplify the hyperparameter space. + +.. rubric:: References + +.. [CM2013] Campello, R.J.G.B., Moulavi, D., Sander, J. (2013). Density-Based + Clustering Based on Hierarchical Density Estimates. In: Pei, J., Tseng, V.S., + Cao, L., Motoda, H., Xu, G. (eds) Advances in Knowledge Discovery and Data + Mining. PAKDD 2013. Lecture Notes in Computer Science(), vol 7819. Springer, + Berlin, Heidelberg. :doi:`Density-Based Clustering Based on Hierarchical + Density Estimates <10.1007/978-3-642-37456-2_14>` + +.. [LJ2017] L. McInnes and J. Healy, (2017). Accelerated Hierarchical Density + Based Clustering. In: IEEE International Conference on Data Mining Workshops + (ICDMW), 2017, pp. 33-42. :doi:`Accelerated Hierarchical Density Based + Clustering <10.1109/ICDMW.2017.12>` .. _optics: @@ -905,47 +1145,48 @@ the linear segment clusters of the reachability plot. Note that the blue and red clusters are adjacent in the reachability plot, and can be hierarchically represented as children of a larger parent cluster. -.. topic:: Examples: +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_cluster_plot_optics.py` - * :ref:`sphx_glr_auto_examples_cluster_plot_optics.py` +.. dropdown:: Comparison with DBSCAN -.. topic:: Comparison with DBSCAN + The results from OPTICS ``cluster_optics_dbscan`` method and DBSCAN are very + similar, but not always identical; specifically, labeling of periphery and noise + points. This is in part because the first samples of each dense area processed + by OPTICS have a large reachability value while being close to other points in + their area, and will thus sometimes be marked as noise rather than periphery. + This affects adjacent points when they are considered as candidates for being + marked as either periphery or noise. - The results from OPTICS ``cluster_optics_dbscan`` method and DBSCAN are - very similar, but not always identical; specifically, labeling of periphery - and noise points. This is in part because the first samples of each dense - area processed by OPTICS have a large reachability value while being close - to other points in their area, and will thus sometimes be marked as noise - rather than periphery. This affects adjacent points when they are - considered as candidates for being marked as either periphery or noise. + Note that for any single value of ``eps``, DBSCAN will tend to have a shorter + run time than OPTICS; however, for repeated runs at varying ``eps`` values, a + single run of OPTICS may require less cumulative runtime than DBSCAN. It is also + important to note that OPTICS' output is close to DBSCAN's only if ``eps`` and + ``max_eps`` are close. - Note that for any single value of ``eps``, DBSCAN will tend to have a - shorter run time than OPTICS; however, for repeated runs at varying ``eps`` - values, a single run of OPTICS may require less cumulative runtime than - DBSCAN. It is also important to note that OPTICS' output is close to - DBSCAN's only if ``eps`` and ``max_eps`` are close. +.. dropdown:: Computational Complexity -.. topic:: Computational Complexity + Spatial indexing trees are used to avoid calculating the full distance matrix, + and allow for efficient memory usage on large sets of samples. Different + distance metrics can be supplied via the ``metric`` keyword. - Spatial indexing trees are used to avoid calculating the full distance - matrix, and allow for efficient memory usage on large sets of samples. - Different distance metrics can be supplied via the ``metric`` keyword. + For large datasets, similar (but not identical) results can be obtained via + :class:`HDBSCAN`. The HDBSCAN implementation is multithreaded, and has better + algorithmic runtime complexity than OPTICS, at the cost of worse memory scaling. + For extremely large datasets that exhaust system memory using HDBSCAN, OPTICS + will maintain :math:`n` (as opposed to :math:`n^2`) memory scaling; however, + tuning of the ``max_eps`` parameter will likely need to be used to give a + solution in a reasonable amount of wall time. - For large datasets, similar (but not identical) results can be obtained via - `HDBSCAN `_. The HDBSCAN implementation is - multithreaded, and has better algorithmic runtime complexity than OPTICS, - at the cost of worse memory scaling. For extremely large datasets that - exhaust system memory using HDBSCAN, OPTICS will maintain :math:`n` (as opposed - to :math:`n^2`) memory scaling; however, tuning of the ``max_eps`` parameter - will likely need to be used to give a solution in a reasonable amount of - wall time. -.. topic:: References: +.. dropdown:: References + + * "OPTICS: ordering points to identify the clustering structure." Ankerst, + Mihael, Markus M. Breunig, Hans-Peter Kriegel, and JÃļrg Sander. In ACM Sigmod + Record, vol. 28, no. 2, pp. 49-60. ACM, 1999. - * "OPTICS: ordering points to identify the clustering structure." - Ankerst, Mihael, Markus M. Breunig, Hans-Peter Kriegel, and JÃļrg Sander. - In ACM Sigmod Record, vol. 28, no. 2, pp. 49-60. ACM, 1999. .. _birch: @@ -981,60 +1222,60 @@ If ``n_clusters`` is set to None, the subclusters from the leaves are directly read off, otherwise a global clustering step labels these subclusters into global clusters (labels) and the samples are mapped to the global label of the nearest subcluster. -**Algorithm description:** - -- A new sample is inserted into the root of the CF Tree which is a CF Node. - It is then merged with the subcluster of the root, that has the smallest - radius after merging, constrained by the threshold and branching factor conditions. - If the subcluster has any child node, then this is done repeatedly till it reaches - a leaf. After finding the nearest subcluster in the leaf, the properties of this - subcluster and the parent subclusters are recursively updated. - -- If the radius of the subcluster obtained by merging the new sample and the - nearest subcluster is greater than the square of the threshold and if the - number of subclusters is greater than the branching factor, then a space is temporarily - allocated to this new sample. The two farthest subclusters are taken and - the subclusters are divided into two groups on the basis of the distance - between these subclusters. - -- If this split node has a parent subcluster and there is room - for a new subcluster, then the parent is split into two. If there is no room, - then this node is again split into two and the process is continued - recursively, till it reaches the root. - -**BIRCH or MiniBatchKMeans?** - - - BIRCH does not scale very well to high dimensional data. As a rule of thumb if - ``n_features`` is greater than twenty, it is generally better to use MiniBatchKMeans. - - If the number of instances of data needs to be reduced, or if one wants a - large number of subclusters either as a preprocessing step or otherwise, - BIRCH is more useful than MiniBatchKMeans. - +.. dropdown:: Algorithm description + + - A new sample is inserted into the root of the CF Tree which is a CF Node. It + is then merged with the subcluster of the root, that has the smallest radius + after merging, constrained by the threshold and branching factor conditions. + If the subcluster has any child node, then this is done repeatedly till it + reaches a leaf. After finding the nearest subcluster in the leaf, the + properties of this subcluster and the parent subclusters are recursively + updated. + + - If the radius of the subcluster obtained by merging the new sample and the + nearest subcluster is greater than the square of the threshold and if the + number of subclusters is greater than the branching factor, then a space is + temporarily allocated to this new sample. The two farthest subclusters are + taken and the subclusters are divided into two groups on the basis of the + distance between these subclusters. + + - If this split node has a parent subcluster and there is room for a new + subcluster, then the parent is split into two. If there is no room, then this + node is again split into two and the process is continued recursively, till it + reaches the root. + +.. dropdown:: BIRCH or MiniBatchKMeans? + + - BIRCH does not scale very well to high dimensional data. As a rule of thumb if + ``n_features`` is greater than twenty, it is generally better to use MiniBatchKMeans. + - If the number of instances of data needs to be reduced, or if one wants a + large number of subclusters either as a preprocessing step or otherwise, + BIRCH is more useful than MiniBatchKMeans. + + .. image:: ../auto_examples/cluster/images/sphx_glr_plot_birch_vs_minibatchkmeans_001.png + :target: ../auto_examples/cluster/plot_birch_vs_minibatchkmeans.html -**How to use partial_fit?** +.. dropdown:: How to use partial_fit? -To avoid the computation of global clustering, for every call of ``partial_fit`` -the user is advised + To avoid the computation of global clustering, for every call of ``partial_fit`` + the user is advised: - 1. To set ``n_clusters=None`` initially - 2. Train all data by multiple calls to partial_fit. - 3. Set ``n_clusters`` to a required value using - ``brc.set_params(n_clusters=n_clusters)``. - 4. Call ``partial_fit`` finally with no arguments, i.e. ``brc.partial_fit()`` - which performs the global clustering. + 1. To set ``n_clusters=None`` initially. + 2. Train all data by multiple calls to partial_fit. + 3. Set ``n_clusters`` to a required value using + ``brc.set_params(n_clusters=n_clusters)``. + 4. Call ``partial_fit`` finally with no arguments, i.e. ``brc.partial_fit()`` + which performs the global clustering. -.. image:: ../auto_examples/cluster/images/sphx_glr_plot_birch_vs_minibatchkmeans_001.png - :target: ../auto_examples/cluster/plot_birch_vs_minibatchkmeans.html +.. dropdown:: References -.. topic:: References: + * Tian Zhang, Raghu Ramakrishnan, Maron Livny BIRCH: An efficient data + clustering method for large databases. + https://www.cs.sfu.ca/CourseCentral/459/han/papers/zhang96.pdf - * Tian Zhang, Raghu Ramakrishnan, Maron Livny - BIRCH: An efficient data clustering method for large databases. - https://www.cs.sfu.ca/CourseCentral/459/han/papers/zhang96.pdf + * Roberto Perdisci JBirch - Java implementation of BIRCH clustering algorithm + https://code.google.com/archive/p/jbirch - * Roberto Perdisci - JBirch - Java implementation of BIRCH clustering algorithm - https://code.google.com/archive/p/jbirch .. _clustering_evaluation: @@ -1069,32 +1310,32 @@ ignoring permutations:: >>> labels_true = [0, 0, 0, 1, 1, 1] >>> labels_pred = [0, 0, 1, 1, 2, 2] >>> metrics.rand_score(labels_true, labels_pred) - 0.66... + 0.66 The Rand index does not ensure to obtain a value close to 0.0 for a random labelling. The adjusted Rand index **corrects for chance** and will give such a baseline. >>> metrics.adjusted_rand_score(labels_true, labels_pred) - 0.24... + 0.24 As with all clustering metrics, one can permute 0 and 1 in the predicted labels, rename 2 to 3, and get the same score:: >>> labels_pred = [1, 1, 0, 0, 3, 3] >>> metrics.rand_score(labels_true, labels_pred) - 0.66... + 0.66 >>> metrics.adjusted_rand_score(labels_true, labels_pred) - 0.24... + 0.24 -Furthermore, both :func:`rand_score` :func:`adjusted_rand_score` are +Furthermore, both :func:`rand_score` and :func:`adjusted_rand_score` are **symmetric**: swapping the argument does not change the scores. They can thus be used as **consensus measures**:: >>> metrics.rand_score(labels_pred, labels_true) - 0.66... + 0.66 >>> metrics.adjusted_rand_score(labels_pred, labels_true) - 0.24... + 0.24 Perfect labeling is scored 1.0:: @@ -1107,114 +1348,104 @@ Perfect labeling is scored 1.0:: Poorly agreeing labels (e.g. independent labelings) have lower scores, and for the adjusted Rand index the score will be negative or close to zero. However, for the unadjusted Rand index the score, while lower, -will not necessarily be close to zero.:: +will not necessarily be close to zero:: >>> labels_true = [0, 0, 0, 0, 0, 0, 1, 1] >>> labels_pred = [0, 1, 2, 3, 4, 5, 5, 6] >>> metrics.rand_score(labels_true, labels_pred) - 0.39... + 0.39 >>> metrics.adjusted_rand_score(labels_true, labels_pred) - -0.07... - + -0.072 -Advantages -~~~~~~~~~~ -- **Interpretability**: The unadjusted Rand index is proportional - to the number of sample pairs whose labels are the same in both - `labels_pred` and `labels_true`, or are different in both. +.. topic:: Advantages: -- **Random (uniform) label assignments have an adjusted Rand index - score close to 0.0** for any value of ``n_clusters`` and - ``n_samples`` (which is not the case for the unadjusted Rand index - or the V-measure for instance). + - **Interpretability**: The unadjusted Rand index is proportional to the + number of sample pairs whose labels are the same in both `labels_pred` and + `labels_true`, or are different in both. -- **Bounded range**: Lower values indicate different labelings, - similar clusterings have a high (adjusted or unadjusted) Rand index, - 1.0 is the perfect match score. The score range is [0, 1] for the - unadjusted Rand index and [-1, 1] for the adjusted Rand index. + - **Random (uniform) label assignments have an adjusted Rand index score close + to 0.0** for any value of ``n_clusters`` and ``n_samples`` (which is not the + case for the unadjusted Rand index or the V-measure for instance). -- **No assumption is made on the cluster structure**: The (adjusted or - unadjusted) Rand index can be used to compare all kinds of - clustering algorithms, and can be used to compare clustering - algorithms such as k-means which assumes isotropic blob shapes with - results of spectral clustering algorithms which can find cluster - with "folded" shapes. + - **Bounded range**: Lower values indicate different labelings, similar + clusterings have a high (adjusted or unadjusted) Rand index, 1.0 is the + perfect match score. The score range is [0, 1] for the unadjusted Rand index + and [-0.5, 1] for the adjusted Rand index. + - **No assumption is made on the cluster structure**: The (adjusted or + unadjusted) Rand index can be used to compare all kinds of clustering + algorithms, and can be used to compare clustering algorithms such as k-means + which assumes isotropic blob shapes with results of spectral clustering + algorithms which can find cluster with "folded" shapes. -Drawbacks -~~~~~~~~~ +.. topic:: Drawbacks: -- Contrary to inertia, the **(adjusted or unadjusted) Rand index - requires knowledge of the ground truth classes** which is almost - never available in practice or requires manual assignment by human - annotators (as in the supervised learning setting). + - Contrary to inertia, the **(adjusted or unadjusted) Rand index requires + knowledge of the ground truth classes** which is almost never available in + practice or requires manual assignment by human annotators (as in the + supervised learning setting). - However (adjusted or unadjusted) Rand index can also be useful in a - purely unsupervised setting as a building block for a Consensus - Index that can be used for clustering model selection (TODO). + However (adjusted or unadjusted) Rand index can also be useful in a purely + unsupervised setting as a building block for a Consensus Index that can be + used for clustering model selection (TODO). -- The **unadjusted Rand index is often close to 1.0** even if the - clusterings themselves differ significantly. This can be understood - when interpreting the Rand index as the accuracy of element pair - labeling resulting from the clusterings: In practice there often is - a majority of element pairs that are assigned the ``different`` pair - label under both the predicted and the ground truth clustering - resulting in a high proportion of pair labels that agree, which - leads subsequently to a high score. + - The **unadjusted Rand index is often close to 1.0** even if the clusterings + themselves differ significantly. This can be understood when interpreting + the Rand index as the accuracy of element pair labeling resulting from the + clusterings: In practice there often is a majority of element pairs that are + assigned the ``different`` pair label under both the predicted and the + ground truth clustering resulting in a high proportion of pair labels that + agree, which leads subsequently to a high score. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_cluster_plot_adjusted_for_chance_measures.py`: - Analysis of the impact of the dataset size on the value of - clustering measures for random assignments. +* :ref:`sphx_glr_auto_examples_cluster_plot_adjusted_for_chance_measures.py`: + Analysis of the impact of the dataset size on the value of + clustering measures for random assignments. +.. dropdown:: Mathematical formulation -Mathematical formulation -~~~~~~~~~~~~~~~~~~~~~~~~ + If C is a ground truth class assignment and K the clustering, let us define + :math:`a` and :math:`b` as: -If C is a ground truth class assignment and K the clustering, let us -define :math:`a` and :math:`b` as: + - :math:`a`, the number of pairs of elements that are in the same set in C and + in the same set in K -- :math:`a`, the number of pairs of elements that are in the same set - in C and in the same set in K + - :math:`b`, the number of pairs of elements that are in different sets in C and + in different sets in K -- :math:`b`, the number of pairs of elements that are in different sets - in C and in different sets in K + The unadjusted Rand index is then given by: -The unadjusted Rand index is then given by: + .. math:: \text{RI} = \frac{a + b}{C_2^{n_{samples}}} -.. math:: \text{RI} = \frac{a + b}{C_2^{n_{samples}}} + where :math:`C_2^{n_{samples}}` is the total number of possible pairs in the + dataset. It does not matter if the calculation is performed on ordered pairs or + unordered pairs as long as the calculation is performed consistently. -where :math:`C_2^{n_{samples}}` is the total number of possible pairs -in the dataset. It does not matter if the calculation is performed on -ordered pairs or unordered pairs as long as the calculation is -performed consistently. + However, the Rand index does not guarantee that random label assignments will + get a value close to zero (esp. if the number of clusters is in the same order + of magnitude as the number of samples). -However, the Rand index does not guarantee that random label assignments -will get a value close to zero (esp. if the number of clusters is in -the same order of magnitude as the number of samples). + To counter this effect we can discount the expected RI :math:`E[\text{RI}]` of + random labelings by defining the adjusted Rand index as follows: -To counter this effect we can discount the expected RI :math:`E[\text{RI}]` of -random labelings by defining the adjusted Rand index as follows: + .. math:: \text{ARI} = \frac{\text{RI} - E[\text{RI}]}{\max(\text{RI}) - E[\text{RI}]} -.. math:: \text{ARI} = \frac{\text{RI} - E[\text{RI}]}{\max(\text{RI}) - E[\text{RI}]} +.. dropdown:: References -.. topic:: References + * `Comparing Partitions + `_ L. Hubert and P. + Arabie, Journal of Classification 1985 - * `Comparing Partitions - `_ - L. Hubert and P. Arabie, Journal of Classification 1985 + * `Properties of the Hubert-Arabie adjusted Rand index + `_ D. Steinley, Psychological + Methods 2004 - * `Properties of the Hubert-Arabie adjusted Rand index - `_ - D. Steinley, Psychological Methods 2004 + * `Wikipedia entry for the Rand index + `_ - * `Wikipedia entry for the Rand index - `_ - - * `Wikipedia entry for the adjusted Rand index - `_ + * :doi:`Minimum adjusted Rand index for two clusterings of a given size, 2022, J. E. ChacÃŗn and A. I. Rastrojo <10.1007/s11634-022-00491-w>` .. _mutual_info_score: @@ -1235,21 +1466,21 @@ proposed more recently and is **normalized against chance**:: >>> labels_pred = [0, 0, 1, 1, 2, 2] >>> metrics.adjusted_mutual_info_score(labels_true, labels_pred) # doctest: +SKIP - 0.22504... + 0.22504 One can permute 0 and 1 in the predicted labels, rename 2 to 3 and get the same score:: >>> labels_pred = [1, 1, 0, 0, 3, 3] >>> metrics.adjusted_mutual_info_score(labels_true, labels_pred) # doctest: +SKIP - 0.22504... + 0.22504 All, :func:`mutual_info_score`, :func:`adjusted_mutual_info_score` and :func:`normalized_mutual_info_score` are symmetric: swapping the argument does not change the score. Thus they can be used as a **consensus measure**:: >>> metrics.adjusted_mutual_info_score(labels_pred, labels_true) # doctest: +SKIP - 0.22504... + 0.22504 Perfect labeling is scored 1.0:: @@ -1263,144 +1494,134 @@ Perfect labeling is scored 1.0:: This is not true for ``mutual_info_score``, which is therefore harder to judge:: >>> metrics.mutual_info_score(labels_true, labels_pred) # doctest: +SKIP - 0.69... + 0.69 Bad (e.g. independent labelings) have non-positive scores:: >>> labels_true = [0, 1, 2, 0, 3, 4, 5, 1] >>> labels_pred = [1, 1, 0, 0, 2, 2, 2, 2] >>> metrics.adjusted_mutual_info_score(labels_true, labels_pred) # doctest: +SKIP - -0.10526... - + -0.10526 -Advantages -~~~~~~~~~~ -- **Random (uniform) label assignments have a AMI score close to 0.0** - for any value of ``n_clusters`` and ``n_samples`` (which is not the - case for raw Mutual Information or the V-measure for instance). +.. topic:: Advantages: -- **Upper bound of 1**: Values close to zero indicate two label - assignments that are largely independent, while values close to one - indicate significant agreement. Further, an AMI of exactly 1 indicates - that the two label assignments are equal (with or without permutation). + - **Random (uniform) label assignments have a AMI score close to 0.0** for any + value of ``n_clusters`` and ``n_samples`` (which is not the case for raw + Mutual Information or the V-measure for instance). + - **Upper bound of 1**: Values close to zero indicate two label assignments + that are largely independent, while values close to one indicate significant + agreement. Further, an AMI of exactly 1 indicates that the two label + assignments are equal (with or without permutation). -Drawbacks -~~~~~~~~~ +.. topic:: Drawbacks: -- Contrary to inertia, **MI-based measures require the knowledge - of the ground truth classes** while almost never available in practice or - requires manual assignment by human annotators (as in the supervised learning - setting). + - Contrary to inertia, **MI-based measures require the knowledge of the ground + truth classes** while almost never available in practice or requires manual + assignment by human annotators (as in the supervised learning setting). - However MI-based measures can also be useful in purely unsupervised setting as a - building block for a Consensus Index that can be used for clustering - model selection. + However MI-based measures can also be useful in purely unsupervised setting + as a building block for a Consensus Index that can be used for clustering + model selection. -- NMI and MI are not adjusted against chance. + - NMI and MI are not adjusted against chance. +.. rubric:: Examples -.. topic:: Examples: +* :ref:`sphx_glr_auto_examples_cluster_plot_adjusted_for_chance_measures.py`: Analysis + of the impact of the dataset size on the value of clustering measures for random + assignments. This example also includes the Adjusted Rand Index. - * :ref:`sphx_glr_auto_examples_cluster_plot_adjusted_for_chance_measures.py`: Analysis of - the impact of the dataset size on the value of clustering measures - for random assignments. This example also includes the Adjusted Rand - Index. +.. dropdown:: Mathematical formulation + Assume two label assignments (of the same N objects), :math:`U` and :math:`V`. + Their entropy is the amount of uncertainty for a partition set, defined by: -Mathematical formulation -~~~~~~~~~~~~~~~~~~~~~~~~ + .. math:: H(U) = - \sum_{i=1}^{|U|}P(i)\log(P(i)) -Assume two label assignments (of the same N objects), :math:`U` and :math:`V`. -Their entropy is the amount of uncertainty for a partition set, defined by: + where :math:`P(i) = |U_i| / N` is the probability that an object picked at + random from :math:`U` falls into class :math:`U_i`. Likewise for :math:`V`: -.. math:: H(U) = - \sum_{i=1}^{|U|}P(i)\log(P(i)) + .. math:: H(V) = - \sum_{j=1}^{|V|}P'(j)\log(P'(j)) -where :math:`P(i) = |U_i| / N` is the probability that an object picked at -random from :math:`U` falls into class :math:`U_i`. Likewise for :math:`V`: + With :math:`P'(j) = |V_j| / N`. The mutual information (MI) between :math:`U` + and :math:`V` is calculated by: -.. math:: H(V) = - \sum_{j=1}^{|V|}P'(j)\log(P'(j)) + .. math:: \text{MI}(U, V) = \sum_{i=1}^{|U|}\sum_{j=1}^{|V|}P(i, j)\log\left(\frac{P(i,j)}{P(i)P'(j)}\right) -With :math:`P'(j) = |V_j| / N`. The mutual information (MI) between :math:`U` -and :math:`V` is calculated by: + where :math:`P(i, j) = |U_i \cap V_j| / N` is the probability that an object + picked at random falls into both classes :math:`U_i` and :math:`V_j`. -.. math:: \text{MI}(U, V) = \sum_{i=1}^{|U|}\sum_{j=1}^{|V|}P(i, j)\log\left(\frac{P(i,j)}{P(i)P'(j)}\right) + It also can be expressed in set cardinality formulation: -where :math:`P(i, j) = |U_i \cap V_j| / N` is the probability that an object -picked at random falls into both classes :math:`U_i` and :math:`V_j`. + .. math:: \text{MI}(U, V) = \sum_{i=1}^{|U|} \sum_{j=1}^{|V|} \frac{|U_i \cap V_j|}{N}\log\left(\frac{N|U_i \cap V_j|}{|U_i||V_j|}\right) -It also can be expressed in set cardinality formulation: + The normalized mutual information is defined as -.. math:: \text{MI}(U, V) = \sum_{i=1}^{|U|} \sum_{j=1}^{|V|} \frac{|U_i \cap V_j|}{N}\log\left(\frac{N|U_i \cap V_j|}{|U_i||V_j|}\right) + .. math:: \text{NMI}(U, V) = \frac{\text{MI}(U, V)}{\text{mean}(H(U), H(V))} -The normalized mutual information is defined as + This value of the mutual information and also the normalized variant is not + adjusted for chance and will tend to increase as the number of different labels + (clusters) increases, regardless of the actual amount of "mutual information" + between the label assignments. -.. math:: \text{NMI}(U, V) = \frac{\text{MI}(U, V)}{\text{mean}(H(U), H(V))} + The expected value for the mutual information can be calculated using the + following equation [VEB2009]_. In this equation, :math:`a_i = |U_i|` (the number + of elements in :math:`U_i`) and :math:`b_j = |V_j|` (the number of elements in + :math:`V_j`). -This value of the mutual information and also the normalized variant is not -adjusted for chance and will tend to increase as the number of different labels -(clusters) increases, regardless of the actual amount of "mutual information" -between the label assignments. + .. math:: E[\text{MI}(U,V)]=\sum_{i=1}^{|U|} \sum_{j=1}^{|V|} \sum_{n_{ij}=(a_i+b_j-N)^+ + }^{\min(a_i, b_j)} \frac{n_{ij}}{N}\log \left( \frac{ N.n_{ij}}{a_i b_j}\right) + \frac{a_i!b_j!(N-a_i)!(N-b_j)!}{N!n_{ij}!(a_i-n_{ij})!(b_j-n_{ij})! + (N-a_i-b_j+n_{ij})!} -The expected value for the mutual information can be calculated using the -following equation [VEB2009]_. In this equation, -:math:`a_i = |U_i|` (the number of elements in :math:`U_i`) and -:math:`b_j = |V_j|` (the number of elements in :math:`V_j`). + Using the expected value, the adjusted mutual information can then be calculated + using a similar form to that of the adjusted Rand index: + .. math:: \text{AMI} = \frac{\text{MI} - E[\text{MI}]}{\text{mean}(H(U), H(V)) - E[\text{MI}]} -.. math:: E[\text{MI}(U,V)]=\sum_{i=1}^{|U|} \sum_{j=1}^{|V|} \sum_{n_{ij}=(a_i+b_j-N)^+ - }^{\min(a_i, b_j)} \frac{n_{ij}}{N}\log \left( \frac{ N.n_{ij}}{a_i b_j}\right) - \frac{a_i!b_j!(N-a_i)!(N-b_j)!}{N!n_{ij}!(a_i-n_{ij})!(b_j-n_{ij})! - (N-a_i-b_j+n_{ij})!} + For normalized mutual information and adjusted mutual information, the + normalizing value is typically some *generalized* mean of the entropies of each + clustering. Various generalized means exist, and no firm rules exist for + preferring one over the others. The decision is largely a field-by-field basis; + for instance, in community detection, the arithmetic mean is most common. Each + normalizing method provides "qualitatively similar behaviours" [YAT2016]_. In + our implementation, this is controlled by the ``average_method`` parameter. -Using the expected value, the adjusted mutual information can then be -calculated using a similar form to that of the adjusted Rand index: + Vinh et al. (2010) named variants of NMI and AMI by their averaging method + [VEB2010]_. Their 'sqrt' and 'sum' averages are the geometric and arithmetic + means; we use these more broadly common names. -.. math:: \text{AMI} = \frac{\text{MI} - E[\text{MI}]}{\text{mean}(H(U), H(V)) - E[\text{MI}]} + .. rubric:: References -For normalized mutual information and adjusted mutual information, the normalizing -value is typically some *generalized* mean of the entropies of each clustering. -Various generalized means exist, and no firm rules exist for preferring one over the -others. The decision is largely a field-by-field basis; for instance, in community -detection, the arithmetic mean is most common. Each -normalizing method provides "qualitatively similar behaviours" [YAT2016]_. In our -implementation, this is controlled by the ``average_method`` parameter. + * Strehl, Alexander, and Joydeep Ghosh (2002). "Cluster ensembles - a + knowledge reuse framework for combining multiple partitions". Journal of + Machine Learning Research 3: 583-617. `doi:10.1162/153244303321897735 + `_. -Vinh et al. (2010) named variants of NMI and AMI by their averaging method [VEB2010]_. Their -'sqrt' and 'sum' averages are the geometric and arithmetic means; we use these -more broadly common names. + * `Wikipedia entry for the (normalized) Mutual Information + `_ -.. topic:: References + * `Wikipedia entry for the Adjusted Mutual Information + `_ - * Strehl, Alexander, and Joydeep Ghosh (2002). "Cluster ensembles – a - knowledge reuse framework for combining multiple partitions". Journal of - Machine Learning Research 3: 583–617. - `doi:10.1162/153244303321897735 `_. + .. [VEB2009] Vinh, Epps, and Bailey, (2009). "Information theoretic measures + for clusterings comparison". Proceedings of the 26th Annual International + Conference on Machine Learning - ICML '09. `doi:10.1145/1553374.1553511 + `_. ISBN + 9781605585161. - * `Wikipedia entry for the (normalized) Mutual Information - `_ + .. [VEB2010] Vinh, Epps, and Bailey, (2010). "Information Theoretic Measures + for Clusterings Comparison: Variants, Properties, Normalization and + Correction for Chance". JMLR + - * `Wikipedia entry for the Adjusted Mutual Information - `_ - - .. [VEB2009] Vinh, Epps, and Bailey, (2009). "Information theoretic measures - for clusterings comparison". Proceedings of the 26th Annual International - Conference on Machine Learning - ICML '09. - `doi:10.1145/1553374.1553511 `_. - ISBN 9781605585161. + .. [YAT2016] Yang, Algesheimer, and Tessone, (2016). "A comparative analysis + of community detection algorithms on artificial networks". Scientific + Reports 6: 30750. `doi:10.1038/srep30750 + `_. - .. [VEB2010] Vinh, Epps, and Bailey, (2010). "Information Theoretic Measures for - Clusterings Comparison: Variants, Properties, Normalization and - Correction for Chance". JMLR - - - .. [YAT2016] Yang, Algesheimer, and Tessone, (2016). "A comparative analysis of - community - detection algorithms on artificial networks". Scientific Reports 6: 30750. - `doi:10.1038/srep30750 `_. - - .. _homogeneity_completeness: @@ -1428,16 +1649,16 @@ We can turn those concept as scores :func:`homogeneity_score` and >>> labels_pred = [0, 0, 1, 1, 2, 2] >>> metrics.homogeneity_score(labels_true, labels_pred) - 0.66... + 0.66 >>> metrics.completeness_score(labels_true, labels_pred) - 0.42... + 0.42 Their harmonic mean called **V-measure** is computed by :func:`v_measure_score`:: >>> metrics.v_measure_score(labels_true, labels_pred) - 0.51... + 0.516 This function's formula is as follows: @@ -1446,12 +1667,12 @@ This function's formula is as follows: `beta` defaults to a value of 1.0, but for using a value less than 1 for beta:: >>> metrics.v_measure_score(labels_true, labels_pred, beta=0.6) - 0.54... + 0.547 more weight will be attributed to homogeneity, and using a value greater than 1:: >>> metrics.v_measure_score(labels_true, labels_pred, beta=1.8) - 0.48... + 0.48 more weight will be attributed to completeness. @@ -1462,14 +1683,14 @@ Homogeneity, completeness and V-measure can be computed at once using :func:`homogeneity_completeness_v_measure` as follows:: >>> metrics.homogeneity_completeness_v_measure(labels_true, labels_pred) - (0.66..., 0.42..., 0.51...) + (0.67, 0.42, 0.52) The following clustering assignment is slightly better, since it is homogeneous but not complete:: >>> labels_pred = [0, 0, 0, 1, 2, 2] >>> metrics.homogeneity_completeness_v_measure(labels_true, labels_pred) - (1.0, 0.68..., 0.81...) + (1.0, 0.68, 0.81) .. note:: @@ -1482,114 +1703,114 @@ homogeneous but not complete:: homogeneity_score(a, b) == completeness_score(b, a) -Advantages -~~~~~~~~~~ +.. topic:: Advantages: -- **Bounded scores**: 0.0 is as bad as it can be, 1.0 is a perfect score. + - **Bounded scores**: 0.0 is as bad as it can be, 1.0 is a perfect score. -- Intuitive interpretation: clustering with bad V-measure can be - **qualitatively analyzed in terms of homogeneity and completeness** - to better feel what 'kind' of mistakes is done by the assignment. + - Intuitive interpretation: clustering with bad V-measure can be + **qualitatively analyzed in terms of homogeneity and completeness** to + better feel what 'kind' of mistakes is done by the assignment. -- **No assumption is made on the cluster structure**: can be used - to compare clustering algorithms such as k-means which assumes isotropic - blob shapes with results of spectral clustering algorithms which can - find cluster with "folded" shapes. + - **No assumption is made on the cluster structure**: can be used to compare + clustering algorithms such as k-means which assumes isotropic blob shapes + with results of spectral clustering algorithms which can find cluster with + "folded" shapes. +.. topic:: Drawbacks: -Drawbacks -~~~~~~~~~ + - The previously introduced metrics are **not normalized with regards to + random labeling**: this means that depending on the number of samples, + clusters and ground truth classes, a completely random labeling will not + always yield the same values for homogeneity, completeness and hence + v-measure. In particular **random labeling won't yield zero scores + especially when the number of clusters is large**. -- The previously introduced metrics are **not normalized with regards to - random labeling**: this means that depending on the number of samples, - clusters and ground truth classes, a completely random labeling will - not always yield the same values for homogeneity, completeness and - hence v-measure. In particular **random labeling won't yield zero - scores especially when the number of clusters is large**. + This problem can safely be ignored when the number of samples is more than a + thousand and the number of clusters is less than 10. **For smaller sample + sizes or larger number of clusters it is safer to use an adjusted index such + as the Adjusted Rand Index (ARI)**. - This problem can safely be ignored when the number of samples is more - than a thousand and the number of clusters is less than 10. **For - smaller sample sizes or larger number of clusters it is safer to use - an adjusted index such as the Adjusted Rand Index (ARI)**. - -.. figure:: ../auto_examples/cluster/images/sphx_glr_plot_adjusted_for_chance_measures_001.png - :target: ../auto_examples/cluster/plot_adjusted_for_chance_measures.html - :align: center - :scale: 100 - -- These metrics **require the knowledge of the ground truth classes** while - almost never available in practice or requires manual assignment by - human annotators (as in the supervised learning setting). + .. figure:: ../auto_examples/cluster/images/sphx_glr_plot_adjusted_for_chance_measures_001.png + :target: ../auto_examples/cluster/plot_adjusted_for_chance_measures.html + :align: center + :scale: 100 + - These metrics **require the knowledge of the ground truth classes** while + almost never available in practice or requires manual assignment by human + annotators (as in the supervised learning setting). -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_cluster_plot_adjusted_for_chance_measures.py`: Analysis of - the impact of the dataset size on the value of clustering measures - for random assignments. +* :ref:`sphx_glr_auto_examples_cluster_plot_adjusted_for_chance_measures.py`: Analysis + of the impact of the dataset size on the value of clustering measures for + random assignments. +.. dropdown:: Mathematical formulation -Mathematical formulation -~~~~~~~~~~~~~~~~~~~~~~~~ + Homogeneity and completeness scores are formally given by: -Homogeneity and completeness scores are formally given by: + .. math:: h = 1 - \frac{H(C|K)}{H(C)} -.. math:: h = 1 - \frac{H(C|K)}{H(C)} + .. math:: c = 1 - \frac{H(K|C)}{H(K)} -.. math:: c = 1 - \frac{H(K|C)}{H(K)} + where :math:`H(C|K)` is the **conditional entropy of the classes given the + cluster assignments** and is given by: -where :math:`H(C|K)` is the **conditional entropy of the classes given -the cluster assignments** and is given by: + .. math:: H(C|K) = - \sum_{c=1}^{|C|} \sum_{k=1}^{|K|} \frac{n_{c,k}}{n} + \cdot \log\left(\frac{n_{c,k}}{n_k}\right) -.. math:: H(C|K) = - \sum_{c=1}^{|C|} \sum_{k=1}^{|K|} \frac{n_{c,k}}{n} - \cdot \log\left(\frac{n_{c,k}}{n_k}\right) + and :math:`H(C)` is the **entropy of the classes** and is given by: -and :math:`H(C)` is the **entropy of the classes** and is given by: + .. math:: H(C) = - \sum_{c=1}^{|C|} \frac{n_c}{n} \cdot \log\left(\frac{n_c}{n}\right) -.. math:: H(C) = - \sum_{c=1}^{|C|} \frac{n_c}{n} \cdot \log\left(\frac{n_c}{n}\right) + with :math:`n` the total number of samples, :math:`n_c` and :math:`n_k` the + number of samples respectively belonging to class :math:`c` and cluster + :math:`k`, and finally :math:`n_{c,k}` the number of samples from class + :math:`c` assigned to cluster :math:`k`. -with :math:`n` the total number of samples, :math:`n_c` and :math:`n_k` -the number of samples respectively belonging to class :math:`c` and -cluster :math:`k`, and finally :math:`n_{c,k}` the number of samples -from class :math:`c` assigned to cluster :math:`k`. + The **conditional entropy of clusters given class** :math:`H(K|C)` and the + **entropy of clusters** :math:`H(K)` are defined in a symmetric manner. -The **conditional entropy of clusters given class** :math:`H(K|C)` and the -**entropy of clusters** :math:`H(K)` are defined in a symmetric manner. + Rosenberg and Hirschberg further define **V-measure** as the **harmonic mean of + homogeneity and completeness**: -Rosenberg and Hirschberg further define **V-measure** as the **harmonic -mean of homogeneity and completeness**: + .. math:: v = 2 \cdot \frac{h \cdot c}{h + c} -.. math:: v = 2 \cdot \frac{h \cdot c}{h + c} +.. rubric:: References -.. topic:: References +* `V-Measure: A conditional entropy-based external cluster evaluation measure + `_ Andrew Rosenberg and Julia + Hirschberg, 2007 - * `V-Measure: A conditional entropy-based external cluster evaluation - measure `_ - Andrew Rosenberg and Julia Hirschberg, 2007 +.. [B2011] `Identification and Characterization of Events in Social Media + `_, Hila + Becker, PhD Thesis. - .. [B2011] `Identication and Characterization of Events in Social Media - `_, Hila - Becker, PhD Thesis. .. _fowlkes_mallows_scores: Fowlkes-Mallows scores ---------------------- -The Fowlkes-Mallows index (:func:`sklearn.metrics.fowlkes_mallows_score`) can be -used when the ground truth class assignments of the samples is known. The -Fowlkes-Mallows score FMI is defined as the geometric mean of the -pairwise precision and recall: +The original Fowlkes-Mallows index (FMI) was intended to measure the similarity +between two clustering results, which is inherently an unsupervised comparison. +The supervised adaptation of the Fowlkes-Mallows index +(as implemented in :func:`sklearn.metrics.fowlkes_mallows_score`) can be used +when the ground truth class assignments of the samples are known. +The FMI is defined as the geometric mean of the pairwise precision and recall: .. math:: \text{FMI} = \frac{\text{TP}}{\sqrt{(\text{TP} + \text{FP}) (\text{TP} + \text{FN})}} -Where ``TP`` is the number of **True Positive** (i.e. the number of pair -of points that belong to the same clusters in both the true labels and the -predicted labels), ``FP`` is the number of **False Positive** (i.e. the number -of pair of points that belong to the same clusters in the true labels and not -in the predicted labels) and ``FN`` is the number of **False Negative** (i.e the -number of pair of points that belongs in the same clusters in the predicted -labels and not in the true labels). +In the above formula: + +* ``TP`` (**True Positive**): The number of pairs of points that are clustered together + both in the true labels and in the predicted labels. + +* ``FP`` (**False Positive**): The number of pairs of points that are clustered together + in the predicted labels but not in the true labels. + +* ``FN`` (**False Negative**): The number of pairs of points that are clustered together + in the true labels but not in the predicted labels. The score ranges from 0 to 1. A high value indicates a good similarity between two clusters. @@ -1599,7 +1820,7 @@ between two clusters. >>> labels_pred = [0, 0, 1, 1, 2, 2] >>> metrics.fowlkes_mallows_score(labels_true, labels_pred) - 0.47140... + 0.47140 One can permute 0 and 1 in the predicted labels, rename 2 to 3 and get the same score:: @@ -1607,7 +1828,7 @@ the same score:: >>> labels_pred = [1, 1, 0, 0, 3, 3] >>> metrics.fowlkes_mallows_score(labels_true, labels_pred) - 0.47140... + 0.47140 Perfect labeling is scored 1.0:: @@ -1622,42 +1843,40 @@ Bad (e.g. independent labelings) have zero scores:: >>> metrics.fowlkes_mallows_score(labels_true, labels_pred) 0.0 -Advantages -~~~~~~~~~~ - -- **Random (uniform) label assignments have a FMI score close to 0.0** - for any value of ``n_clusters`` and ``n_samples`` (which is not the - case for raw Mutual Information or the V-measure for instance). +.. topic:: Advantages: -- **Upper-bounded at 1**: Values close to zero indicate two label - assignments that are largely independent, while values close to one - indicate significant agreement. Further, values of exactly 0 indicate - **purely** independent label assignments and a FMI of exactly 1 indicates - that the two label assignments are equal (with or without permutation). + - **Random (uniform) label assignments have a FMI score close to 0.0** for any + value of ``n_clusters`` and ``n_samples`` (which is not the case for raw + Mutual Information or the V-measure for instance). -- **No assumption is made on the cluster structure**: can be used - to compare clustering algorithms such as k-means which assumes isotropic - blob shapes with results of spectral clustering algorithms which can - find cluster with "folded" shapes. + - **Upper-bounded at 1**: Values close to zero indicate two label assignments + that are largely independent, while values close to one indicate significant + agreement. Further, values of exactly 0 indicate **purely** independent + label assignments and a FMI of exactly 1 indicates that the two label + assignments are equal (with or without permutation). + - **No assumption is made on the cluster structure**: can be used to compare + clustering algorithms such as k-means which assumes isotropic blob shapes + with results of spectral clustering algorithms which can find cluster with + "folded" shapes. -Drawbacks -~~~~~~~~~ +.. topic:: Drawbacks: -- Contrary to inertia, **FMI-based measures require the knowledge - of the ground truth classes** while almost never available in practice or - requires manual assignment by human annotators (as in the supervised learning - setting). + - Contrary to inertia, **FMI-based measures require the knowledge of the + ground truth classes** while almost never available in practice or requires + manual assignment by human annotators (as in the supervised learning + setting). -.. topic:: References +.. dropdown:: References * E. B. Fowkles and C. L. Mallows, 1983. "A method for comparing two hierarchical clusterings". Journal of the American Statistical Association. - http://wildfire.stat.ucla.edu/pdflibrary/fowlkes.pdf + https://www.tandfonline.com/doi/abs/10.1080/01621459.1983.10478008 * `Wikipedia entry for the Fowlkes-Mallows Index `_ + .. _silhouette_coefficient: Silhouette Coefficient @@ -1698,37 +1917,33 @@ cluster analysis. >>> kmeans_model = KMeans(n_clusters=3, random_state=1).fit(X) >>> labels = kmeans_model.labels_ >>> metrics.silhouette_score(X, labels, metric='euclidean') - 0.55... - -.. topic:: References + 0.55 - * Peter J. Rousseeuw (1987). "Silhouettes: a Graphical Aid to the - Interpretation and Validation of Cluster Analysis". Computational - and Applied Mathematics 20: 53–65. - `doi:10.1016/0377-0427(87)90125-7 `_. +.. topic:: Advantages: + - The score is bounded between -1 for incorrect clustering and +1 for highly + dense clustering. Scores around zero indicate overlapping clusters. -Advantages -~~~~~~~~~~ + - The score is higher when clusters are dense and well separated, which + relates to a standard concept of a cluster. -- The score is bounded between -1 for incorrect clustering and +1 for highly - dense clustering. Scores around zero indicate overlapping clusters. +.. topic:: Drawbacks: -- The score is higher when clusters are dense and well separated, which relates - to a standard concept of a cluster. + - The Silhouette Coefficient is generally higher for convex clusters than + other concepts of clusters, such as density based clusters like those + obtained through DBSCAN. +.. rubric:: Examples -Drawbacks -~~~~~~~~~ +* :ref:`sphx_glr_auto_examples_cluster_plot_kmeans_silhouette_analysis.py` : In + this example the silhouette analysis is used to choose an optimal value for + n_clusters. -- The Silhouette Coefficient is generally higher for convex clusters than other - concepts of clusters, such as density based clusters like those obtained - through DBSCAN. +.. dropdown:: References -.. topic:: Examples: - - * :ref:`sphx_glr_auto_examples_cluster_plot_kmeans_silhouette_analysis.py` : In this example - the silhouette analysis is used to choose an optimal value for n_clusters. + * Peter J. Rousseeuw (1987). :doi:`"Silhouettes: a Graphical Aid to the + Interpretation and Validation of Cluster Analysis"<10.1016/0377-0427(87)90125-7>`. + Computational and Applied Mathematics 20: 53-65. .. _calinski_harabasz_index: @@ -1738,8 +1953,8 @@ Calinski-Harabasz Index If the ground truth labels are not known, the Calinski-Harabasz index -(:func:`sklearn.metrics.calinski_harabasz_score`) - also known as the Variance -Ratio Criterion - can be used to evaluate the model, where a higher +(:func:`sklearn.metrics.calinski_harabasz_score`) - also known as the Variance +Ratio Criterion - can be used to evaluate the model, where a higher Calinski-Harabasz score relates to a model with better defined clusters. The index is the ratio of the sum of between-clusters dispersion and of @@ -1759,53 +1974,50 @@ cluster analysis: >>> kmeans_model = KMeans(n_clusters=3, random_state=1).fit(X) >>> labels = kmeans_model.labels_ >>> metrics.calinski_harabasz_score(X, labels) - 561.62... + 561.59 -Advantages -~~~~~~~~~~ -- The score is higher when clusters are dense and well separated, which relates - to a standard concept of a cluster. +.. topic:: Advantages: -- The score is fast to compute. + - The score is higher when clusters are dense and well separated, which + relates to a standard concept of a cluster. + - The score is fast to compute. -Drawbacks -~~~~~~~~~ +.. topic:: Drawbacks: -- The Calinski-Harabasz index is generally higher for convex clusters than other - concepts of clusters, such as density based clusters like those obtained - through DBSCAN. + - The Calinski-Harabasz index is generally higher for convex clusters than + other concepts of clusters, such as density based clusters like those + obtained through DBSCAN. -Mathematical formulation -~~~~~~~~~~~~~~~~~~~~~~~~ +.. dropdown:: Mathematical formulation -For a set of data :math:`E` of size :math:`n_E` which has been clustered into -:math:`k` clusters, the Calinski-Harabasz score :math:`s` is defined as the -ratio of the between-clusters dispersion mean and the within-cluster dispersion: + For a set of data :math:`E` of size :math:`n_E` which has been clustered into + :math:`k` clusters, the Calinski-Harabasz score :math:`s` is defined as the + ratio of the between-clusters dispersion mean and the within-cluster + dispersion: -.. math:: - s = \frac{\mathrm{tr}(B_k)}{\mathrm{tr}(W_k)} \times \frac{n_E - k}{k - 1} + .. math:: + s = \frac{\mathrm{tr}(B_k)}{\mathrm{tr}(W_k)} \times \frac{n_E - k}{k - 1} -where :math:`\mathrm{tr}(B_k)` is trace of the between group dispersion matrix -and :math:`\mathrm{tr}(W_k)` is the trace of the within-cluster dispersion -matrix defined by: + where :math:`\mathrm{tr}(B_k)` is trace of the between group dispersion matrix + and :math:`\mathrm{tr}(W_k)` is the trace of the within-cluster dispersion + matrix defined by: -.. math:: W_k = \sum_{q=1}^k \sum_{x \in C_q} (x - c_q) (x - c_q)^T + .. math:: W_k = \sum_{q=1}^k \sum_{x \in C_q} (x - c_q) (x - c_q)^T -.. math:: B_k = \sum_{q=1}^k n_q (c_q - c_E) (c_q - c_E)^T + .. math:: B_k = \sum_{q=1}^k n_q (c_q - c_E) (c_q - c_E)^T -with :math:`C_q` the set of points in cluster :math:`q`, :math:`c_q` the center -of cluster :math:`q`, :math:`c_E` the center of :math:`E`, and :math:`n_q` the -number of points in cluster :math:`q`. + with :math:`C_q` the set of points in cluster :math:`q`, :math:`c_q` the + center of cluster :math:`q`, :math:`c_E` the center of :math:`E`, and + :math:`n_q` the number of points in cluster :math:`q`. -.. topic:: References +.. dropdown:: References - * Caliński, T., & Harabasz, J. (1974). - `"A Dendrite Method for Cluster Analysis" - `_. - Communications in Statistics-theory and Methods 3: 1-27. - `doi:10.1080/03610927408827101 `_. + * Caliński, T., & Harabasz, J. (1974). `"A Dendrite Method for Cluster Analysis" + `_. + :doi:`Communications in Statistics-theory and Methods 3: 1-27 + <10.1080/03610927408827101>`. .. _davies-bouldin_index: @@ -1836,61 +2048,57 @@ cluster analysis as follows: >>> kmeans = KMeans(n_clusters=3, random_state=1).fit(X) >>> labels = kmeans.labels_ >>> davies_bouldin_score(X, labels) - 0.6619... + 0.666 -Advantages -~~~~~~~~~~ +.. topic:: Advantages: -- The computation of Davies-Bouldin is simpler than that of Silhouette scores. -- The index is computed only quantities and features inherent to the dataset. + - The computation of Davies-Bouldin is simpler than that of Silhouette scores. + - The index is solely based on quantities and features inherent to the dataset + as its computation only uses point-wise distances. -Drawbacks -~~~~~~~~~ +.. topic:: Drawbacks: -- The Davies-Boulding index is generally higher for convex clusters than other - concepts of clusters, such as density based clusters like those obtained from - DBSCAN. -- The usage of centroid distance limits the distance metric to Euclidean space. + - The Davies-Bouldin index is generally higher for convex clusters than other + concepts of clusters, such as density-based clusters like those + obtained from DBSCAN. + - The usage of centroid distance limits the distance metric to Euclidean + space. -Mathematical formulation -~~~~~~~~~~~~~~~~~~~~~~~~ +.. dropdown:: Mathematical formulation -The index is defined as the average similarity between each cluster :math:`C_i` -for :math:`i=1, ..., k` and its most similar one :math:`C_j`. In the context of -this index, similarity is defined as a measure :math:`R_{ij}` that trades off: + The index is defined as the average similarity between each cluster :math:`C_i` + for :math:`i=1, ..., k` and its most similar one :math:`C_j`. In the context of + this index, similarity is defined as a measure :math:`R_{ij}` that trades off: -- :math:`s_i`, the average distance between each point of cluster :math:`i` and - the centroid of that cluster -- also know as cluster diameter. -- :math:`d_{ij}`, the distance between cluster centroids :math:`i` and :math:`j`. + - :math:`s_i`, the average distance between each point of cluster :math:`i` and + the centroid of that cluster -- also known as cluster diameter. + - :math:`d_{ij}`, the distance between cluster centroids :math:`i` and + :math:`j`. -A simple choice to construct :math:`R_{ij}` so that it is nonnegative and -symmetric is: + A simple choice to construct :math:`R_{ij}` so that it is nonnegative and + symmetric is: -.. math:: - R_{ij} = \frac{s_i + s_j}{d_{ij}} - -Then the Davies-Bouldin index is defined as: + .. math:: + R_{ij} = \frac{s_i + s_j}{d_{ij}} -.. math:: - DB = \frac{1}{k} \sum_{i=1}^k \max_{i \neq j} R_{ij} + Then the Davies-Bouldin index is defined as: + .. math:: + DB = \frac{1}{k} \sum_{i=1}^k \max_{i \neq j} R_{ij} -.. topic:: References +.. dropdown:: References - * Davies, David L.; Bouldin, Donald W. (1979). - "A Cluster Separation Measure" - IEEE Transactions on Pattern Analysis and Machine Intelligence. - PAMI-1 (2): 224-227. - `doi:10.1109/TPAMI.1979.4766909 `_. + * Davies, David L.; Bouldin, Donald W. (1979). :doi:`"A Cluster Separation + Measure" <10.1109/TPAMI.1979.4766909>` IEEE Transactions on Pattern Analysis + and Machine Intelligence. PAMI-1 (2): 224-227. - * Halkidi, Maria; Batistakis, Yannis; Vazirgiannis, Michalis (2001). - "On Clustering Validation Techniques" - Journal of Intelligent Information Systems, 17(2-3), 107-145. - `doi:10.1023/A:1012801612483 `_. + * Halkidi, Maria; Batistakis, Yannis; Vazirgiannis, Michalis (2001). :doi:`"On + Clustering Validation Techniques" <10.1023/A:1012801612483>` Journal of + Intelligent Information Systems, 17(2-3), 107-145. - * `Wikipedia entry for Davies-Bouldin index - `_. + * `Wikipedia entry for Davies-Bouldin index + `_. .. _contingency_matrix: @@ -1913,7 +2121,7 @@ Here is an example:: array([[2, 1, 0], [0, 1, 2]]) -The first row of output array indicates that there are three samples whose +The first row of the output array indicates that there are three samples whose true cluster is "a". Of them, two are in predicted cluster 0, one is in 1, and none is in 2. And the second row indicates that there are three samples whose true cluster is "b". Of them, none is in predicted cluster 0, one is in @@ -1924,30 +2132,28 @@ contingency matrix where the order of rows and columns correspond to a list of classes. -Advantages -~~~~~~~~~~ +.. topic:: Advantages: -- Allows to examine the spread of each true cluster across predicted - clusters and vice versa. + - Allows to examine the spread of each true cluster across predicted clusters + and vice versa. -- The contingency table calculated is typically utilized in the calculation - of a similarity statistic (like the others listed in this document) between - the two clusterings. + - The contingency table calculated is typically utilized in the calculation of + a similarity statistic (like the others listed in this document) between the + two clusterings. -Drawbacks -~~~~~~~~~ +.. topic:: Drawbacks: -- Contingency matrix is easy to interpret for a small number of clusters, but - becomes very hard to interpret for a large number of clusters. + - Contingency matrix is easy to interpret for a small number of clusters, but + becomes very hard to interpret for a large number of clusters. -- It doesn't give a single metric to use as an objective for clustering - optimisation. + - It doesn't give a single metric to use as an objective for clustering + optimisation. +.. dropdown:: References -.. topic:: References + * `Wikipedia entry for contingency matrix + `_ - * `Wikipedia entry for contingency matrix - `_ .. _pair_confusion_matrix: @@ -1970,19 +2176,19 @@ under the true and predicted clusterings. It has the following entries: - :math:`C_{00}` : number of pairs with both clusterings having the samples - not clustered together +:math:`C_{00}` : number of pairs with both clusterings having the samples +not clustered together - :math:`C_{10}` : number of pairs with the true label clustering having the - samples clustered together but the other clustering not having the samples - clustered together +:math:`C_{10}` : number of pairs with the true label clustering having the +samples clustered together but the other clustering not having the samples +clustered together - :math:`C_{01}` : number of pairs with the true label clustering not having - the samples clustered together but the other clustering having the samples - clustered together +:math:`C_{01}` : number of pairs with the true label clustering not having +the samples clustered together but the other clustering having the samples +clustered together - :math:`C_{11}` : number of pairs with both clusterings having the samples - clustered together +:math:`C_{11}` : number of pairs with both clusterings having the samples +clustered together Considering a pair of samples that is clustered together a positive pair, then as in binary classification the count of true negatives is @@ -2025,8 +2231,7 @@ diagonal entries:: array([[ 0, 0], [12, 0]]) -.. topic:: References +.. dropdown:: References - * L. Hubert and P. Arabie, Comparing Partitions, Journal of - Classification 1985 - _ + * :doi:`"Comparing Partitions" <10.1007/BF01908075>` L. Hubert and P. Arabie, + Journal of Classification 1985 diff --git a/doc/modules/compose.rst b/doc/modules/compose.rst index 6e827304c38cd..3ef0d94236aa6 100644 --- a/doc/modules/compose.rst +++ b/doc/modules/compose.rst @@ -5,14 +5,24 @@ Pipelines and composite estimators ================================== -Transformers are usually combined with classifiers, regressors or other -estimators to build a composite estimator. The most common tool is a -:ref:`Pipeline `. Pipeline is often used in combination with -:ref:`FeatureUnion ` which concatenates the output of -transformers into a composite feature space. :ref:`TransformedTargetRegressor -` deals with transforming the :term:`target` -(i.e. log-transform :term:`y`). In contrast, Pipelines only transform the -observed data (:term:`X`). +To build a composite estimator, transformers are usually combined with other +transformers or with :term:`predictors` (such as classifiers or regressors). +The most common tool used for composing estimators is a :ref:`Pipeline +`. Pipelines require all steps except the last to be a +:term:`transformer`. The last step can be anything, a transformer, a +:term:`predictor`, or a clustering estimator which might have or not have a +`.predict(...)` method. A pipeline exposes all methods provided by the last +estimator: if the last step provides a `transform` method, then the pipeline +would have a `transform` method and behave like a transformer. If the last step +provides a `predict` method, then the pipeline would expose that method, and +given a data :term:`X`, use all steps except the last to transform the data, +and then give that transformed data to the `predict` method of the last step of +the pipeline. The class :class:`Pipeline` is often used in combination with +:ref:`ColumnTransformer ` or +:ref:`FeatureUnion ` which concatenate the output of transformers +into a composite feature space. +:ref:`TransformedTargetRegressor ` +deals with transforming the :term:`target` (i.e. log-transform :term:`y`). .. _pipeline: @@ -41,12 +51,21 @@ All estimators in a pipeline, except the last one, must be transformers (i.e. must have a :term:`transform` method). The last estimator may be any type (transformer, classifier, etc.). +.. note:: + + Calling ``fit`` on the pipeline is the same as calling ``fit`` on + each estimator in turn, ``transform`` the input and pass it on to the next step. + The pipeline has all the methods that the last estimator in the pipeline has, + i.e. if the last estimator is a classifier, the :class:`Pipeline` can be used + as a classifier. If the last estimator is a transformer, again, so is the + pipeline. + Usage ----- -Construction -............ +Build a pipeline +................ The :class:`Pipeline` is built using a list of ``(key, value)`` pairs, where the ``key`` is a string containing the name you want to give this step and ``value`` @@ -60,38 +79,22 @@ is an estimator object:: >>> pipe Pipeline(steps=[('reduce_dim', PCA()), ('clf', SVC())]) -The utility function :func:`make_pipeline` is a shorthand -for constructing pipelines; -it takes a variable number of estimators and returns a pipeline, -filling in the names automatically:: - - >>> from sklearn.pipeline import make_pipeline - >>> from sklearn.naive_bayes import MultinomialNB - >>> from sklearn.preprocessing import Binarizer - >>> make_pipeline(Binarizer(), MultinomialNB()) - Pipeline(steps=[('binarizer', Binarizer()), ('multinomialnb', MultinomialNB())]) +.. dropdown:: Shorthand version using :func:`make_pipeline` -Accessing steps -............... + The utility function :func:`make_pipeline` is a shorthand + for constructing pipelines; + it takes a variable number of estimators and returns a pipeline, + filling in the names automatically:: -The estimators of a pipeline are stored as a list in the ``steps`` attribute, -but can be accessed by index or name by indexing (with ``[idx]``) the -Pipeline:: + >>> from sklearn.pipeline import make_pipeline + >>> make_pipeline(PCA(), SVC()) + Pipeline(steps=[('pca', PCA()), ('svc', SVC())]) - >>> pipe.steps[0] - ('reduce_dim', PCA()) - >>> pipe[0] - PCA() - >>> pipe['reduce_dim'] - PCA() +Access pipeline steps +..................... -Pipeline's `named_steps` attribute allows accessing steps by name with tab -completion in interactive environments:: - - >>> pipe.named_steps.reduce_dim is pipe['reduce_dim'] - True - -A sub-pipeline can also be extracted using the slicing notation commonly used +The estimators of a pipeline are stored as a list in the ``steps`` attribute. +A sub-pipeline can be extracted using the slicing notation commonly used for Python Sequences such as lists or strings (although only a step of 1 is permitted). This is convenient for performing only some of the transformations (or their inverse): @@ -101,67 +104,97 @@ permitted). This is convenient for performing only some of the transformations >>> pipe[-1:] Pipeline(steps=[('clf', SVC())]) +.. dropdown:: Accessing a step by name or position -.. _pipeline_nested_parameters: - -Nested parameters -................. + A specific step can also be accessed by index or name by indexing (with ``[idx]``) the + pipeline:: -Parameters of the estimators in the pipeline can be accessed using the -``__`` syntax:: + >>> pipe.steps[0] + ('reduce_dim', PCA()) + >>> pipe[0] + PCA() + >>> pipe['reduce_dim'] + PCA() - >>> pipe.set_params(clf__C=10) - Pipeline(steps=[('reduce_dim', PCA()), ('clf', SVC(C=10))]) + `Pipeline`'s `named_steps` attribute allows accessing steps by name with tab + completion in interactive environments:: -This is particularly important for doing grid searches:: + >>> pipe.named_steps.reduce_dim is pipe['reduce_dim'] + True - >>> from sklearn.model_selection import GridSearchCV - >>> param_grid = dict(reduce_dim__n_components=[2, 5, 10], - ... clf__C=[0.1, 10, 100]) - >>> grid_search = GridSearchCV(pipe, param_grid=param_grid) +Tracking feature names in a pipeline +.................................... -Individual steps may also be replaced as parameters, and non-final steps may be -ignored by setting them to ``'passthrough'``:: +To enable model inspection, :class:`~sklearn.pipeline.Pipeline` has a +``get_feature_names_out()`` method, just like all transformers. You can use +pipeline slicing to get the feature names going into each step:: + >>> from sklearn.datasets import load_iris >>> from sklearn.linear_model import LogisticRegression - >>> param_grid = dict(reduce_dim=['passthrough', PCA(5), PCA(10)], - ... clf=[SVC(), LogisticRegression()], - ... clf__C=[0.1, 10, 100]) - >>> grid_search = GridSearchCV(pipe, param_grid=param_grid) + >>> from sklearn.feature_selection import SelectKBest + >>> iris = load_iris() + >>> pipe = Pipeline(steps=[ + ... ('select', SelectKBest(k=2)), + ... ('clf', LogisticRegression())]) + >>> pipe.fit(iris.data, iris.target) + Pipeline(steps=[('select', SelectKBest(...)), ('clf', LogisticRegression(...))]) + >>> pipe[:-1].get_feature_names_out() + array(['x2', 'x3'], ...) + +.. dropdown:: Customize feature names + + You can also provide custom feature names for the input data using + ``get_feature_names_out``:: -The estimators of the pipeline can be retrieved by index: + >>> pipe[:-1].get_feature_names_out(iris.feature_names) + array(['petal length (cm)', 'petal width (cm)'], ...) - >>> pipe[0] - PCA() +.. _pipeline_nested_parameters: -or by name:: +Access to nested parameters +........................... - >>> pipe['reduce_dim'] - PCA() +It is common to adjust the parameters of an estimator within a pipeline. This parameter +is therefore nested because it belongs to a particular sub-step. Parameters of the +estimators in the pipeline are accessible using the ``__`` +syntax:: -.. topic:: Examples: + >>> pipe = Pipeline(steps=[("reduce_dim", PCA()), ("clf", SVC())]) + >>> pipe.set_params(clf__C=10) + Pipeline(steps=[('reduce_dim', PCA()), ('clf', SVC(C=10))]) - * :ref:`sphx_glr_auto_examples_feature_selection_plot_feature_selection_pipeline.py` - * :ref:`sphx_glr_auto_examples_model_selection_grid_search_text_feature_extraction.py` - * :ref:`sphx_glr_auto_examples_compose_plot_digits_pipe.py` - * :ref:`sphx_glr_auto_examples_miscellaneous_plot_kernel_approximation.py` - * :ref:`sphx_glr_auto_examples_svm_plot_svm_anova.py` - * :ref:`sphx_glr_auto_examples_compose_plot_compare_reduction.py` +.. dropdown:: When does it matter? -.. topic:: See Also: + This is particularly important for doing grid searches:: - * :ref:`composite_grid_search` + >>> from sklearn.model_selection import GridSearchCV + >>> param_grid = dict(reduce_dim__n_components=[2, 5, 10], + ... clf__C=[0.1, 10, 100]) + >>> grid_search = GridSearchCV(pipe, param_grid=param_grid) + Individual steps may also be replaced as parameters, and non-final steps may be + ignored by setting them to ``'passthrough'``:: -Notes ------ + >>> param_grid = dict(reduce_dim=['passthrough', PCA(5), PCA(10)], + ... clf=[SVC(), LogisticRegression()], + ... clf__C=[0.1, 10, 100]) + >>> grid_search = GridSearchCV(pipe, param_grid=param_grid) + + .. seealso:: + + * :ref:`composite_grid_search` + + +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_feature_selection_plot_feature_selection_pipeline.py` +* :ref:`sphx_glr_auto_examples_model_selection_plot_grid_search_text_feature_extraction.py` +* :ref:`sphx_glr_auto_examples_compose_plot_digits_pipe.py` +* :ref:`sphx_glr_auto_examples_miscellaneous_plot_kernel_approximation.py` +* :ref:`sphx_glr_auto_examples_svm_plot_svm_anova.py` +* :ref:`sphx_glr_auto_examples_compose_plot_compare_reduction.py` +* :ref:`sphx_glr_auto_examples_miscellaneous_plot_pipeline_display.py` -Calling ``fit`` on the pipeline is the same as calling ``fit`` on -each estimator in turn, ``transform`` the input and pass it on to the next step. -The pipeline has all the methods that the last estimator in the pipeline has, -i.e. if the last estimator is a classifier, the :class:`Pipeline` can be used -as a classifier. If the last estimator is a transformer, again, so is the -pipeline. .. _pipeline_cache: @@ -176,11 +209,11 @@ after calling ``fit``. This feature is used to avoid computing the fit transformers within a pipeline if the parameters and input data are identical. A typical example is the case of a grid search in which the transformers can be fitted only once and reused for -each configuration. +each configuration. The last step will never be cached, even if it is a transformer. The parameter ``memory`` is needed in order to cache the transformers. ``memory`` can be either a string containing the directory where to cache the -transformers or a `joblib.Memory `_ +transformers or a `joblib.Memory `_ object:: >>> from tempfile import mkdtemp @@ -197,47 +230,49 @@ object:: >>> # Clear the cache directory when you don't need it anymore >>> rmtree(cachedir) -.. warning:: **Side effect of caching transformers** - - Using a :class:`Pipeline` without cache enabled, it is possible to - inspect the original instance such as:: - - >>> from sklearn.datasets import load_digits - >>> X_digits, y_digits = load_digits(return_X_y=True) - >>> pca1 = PCA() - >>> svm1 = SVC() - >>> pipe = Pipeline([('reduce_dim', pca1), ('clf', svm1)]) - >>> pipe.fit(X_digits, y_digits) - Pipeline(steps=[('reduce_dim', PCA()), ('clf', SVC())]) - >>> # The pca instance can be inspected directly - >>> print(pca1.components_) - [[-1.77484909e-19 ... 4.07058917e-18]] - - Enabling caching triggers a clone of the transformers before fitting. - Therefore, the transformer instance given to the pipeline cannot be - inspected directly. - In following example, accessing the :class:`PCA` instance ``pca2`` - will raise an ``AttributeError`` since ``pca2`` will be an unfitted - transformer. - Instead, use the attribute ``named_steps`` to inspect estimators within - the pipeline:: - - >>> cachedir = mkdtemp() - >>> pca2 = PCA() - >>> svm2 = SVC() - >>> cached_pipe = Pipeline([('reduce_dim', pca2), ('clf', svm2)], - ... memory=cachedir) - >>> cached_pipe.fit(X_digits, y_digits) - Pipeline(memory=..., - steps=[('reduce_dim', PCA()), ('clf', SVC())]) - >>> print(cached_pipe.named_steps['reduce_dim'].components_) - [[-1.77484909e-19 ... 4.07058917e-18]] - >>> # Remove the cache directory - >>> rmtree(cachedir) - -.. topic:: Examples: - - * :ref:`sphx_glr_auto_examples_compose_plot_compare_reduction.py` +.. dropdown:: Side effect of caching transformers + :color: warning + + Using a :class:`Pipeline` without cache enabled, it is possible to + inspect the original instance such as:: + + >>> from sklearn.datasets import load_digits + >>> X_digits, y_digits = load_digits(return_X_y=True) + >>> pca1 = PCA(n_components=10) + >>> svm1 = SVC() + >>> pipe = Pipeline([('reduce_dim', pca1), ('clf', svm1)]) + >>> pipe.fit(X_digits, y_digits) + Pipeline(steps=[('reduce_dim', PCA(n_components=10)), ('clf', SVC())]) + >>> # The pca instance can be inspected directly + >>> pca1.components_.shape + (10, 64) + + Enabling caching triggers a clone of the transformers before fitting. + Therefore, the transformer instance given to the pipeline cannot be + inspected directly. + In the following example, accessing the :class:`~sklearn.decomposition.PCA` + instance ``pca2`` will raise an ``AttributeError`` since ``pca2`` will be an + unfitted transformer. + Instead, use the attribute ``named_steps`` to inspect estimators within + the pipeline:: + + >>> cachedir = mkdtemp() + >>> pca2 = PCA(n_components=10) + >>> svm2 = SVC() + >>> cached_pipe = Pipeline([('reduce_dim', pca2), ('clf', svm2)], + ... memory=cachedir) + >>> cached_pipe.fit(X_digits, y_digits) + Pipeline(memory=..., + steps=[('reduce_dim', PCA(n_components=10)), ('clf', SVC())]) + >>> cached_pipe.named_steps['reduce_dim'].components_.shape + (10, 64) + >>> # Remove the cache directory + >>> rmtree(cachedir) + + +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_compose_plot_compare_reduction.py` .. _transformed_target_regressor: @@ -310,9 +345,9 @@ each other. However, it is possible to bypass this checking by setting pair of functions ``func`` and ``inverse_func``. However, setting both options will raise an error. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_compose_plot_transformed_target.py` +* :ref:`sphx_glr_auto_examples_compose_plot_transformed_target.py` .. _feature_union: @@ -374,9 +409,9 @@ and ignored by setting to ``'drop'``:: FeatureUnion(transformer_list=[('linear_pca', PCA()), ('kernel_pca', 'drop')]) -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_compose_plot_feature_union.py` +* :ref:`sphx_glr_auto_examples_compose_plot_feature_union.py` .. _column_transformer: @@ -426,21 +461,20 @@ By default, the remaining rating columns are ignored (``remainder='drop'``):: >>> from sklearn.feature_extraction.text import CountVectorizer >>> from sklearn.preprocessing import OneHotEncoder >>> column_trans = ColumnTransformer( - ... [('city_category', OneHotEncoder(dtype='int'),['city']), + ... [('categories', OneHotEncoder(dtype='int'), ['city']), ... ('title_bow', CountVectorizer(), 'title')], - ... remainder='drop') + ... remainder='drop', verbose_feature_names_out=False) >>> column_trans.fit(X) - ColumnTransformer(transformers=[('city_category', OneHotEncoder(dtype='int'), + ColumnTransformer(transformers=[('categories', OneHotEncoder(dtype='int'), ['city']), - ('title_bow', CountVectorizer(), 'title')]) + ('title_bow', CountVectorizer(), 'title')], + verbose_feature_names_out=False) - >>> column_trans.get_feature_names() - ['city_category__x0_London', 'city_category__x0_Paris', 'city_category__x0_Sallisaw', - 'title_bow__bow', 'title_bow__feast', 'title_bow__grapes', 'title_bow__his', - 'title_bow__how', 'title_bow__last', 'title_bow__learned', 'title_bow__moveable', - 'title_bow__of', 'title_bow__the', 'title_bow__trick', 'title_bow__watson', - 'title_bow__wrath'] + >>> column_trans.get_feature_names_out() + array(['city_London', 'city_Paris', 'city_Sallisaw', 'bow', 'feast', + 'grapes', 'his', 'how', 'last', 'learned', 'moveable', 'of', 'the', + 'trick', 'watson', 'wrath'], ...) >>> column_trans.transform(X).toarray() array([[1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0], @@ -470,10 +504,10 @@ on data type or column name:: ... OneHotEncoder(), ... make_column_selector(pattern='city', dtype_include=object))]) >>> ct.fit_transform(X) - array([[ 0.904..., 0. , 1. , 0. , 0. ], - [-1.507..., 1.414..., 1. , 0. , 0. ], - [-0.301..., 0. , 0. , 1. , 0. ], - [ 0.904..., -1.414..., 0. , 0. , 1. ]]) + array([[ 0.904, 0. , 1. , 0. , 0. ], + [-1.507, 1.414, 1. , 0. , 0. ], + [-0.301, 0. , 0. , 1. , 0. ], + [ 0.904, -1.414, 0. , 0. , 1. ]]) Strings can reference columns if the input is a DataFrame, integers are always interpreted as the positional columns. @@ -527,19 +561,37 @@ above example would be:: ('countvectorizer', CountVectorizer(), 'title')]) +If :class:`~sklearn.compose.ColumnTransformer` is fitted with a dataframe +and the dataframe only has string column names, then transforming a dataframe +will use the column names to select the columns:: + + + >>> ct = ColumnTransformer( + ... [("scale", StandardScaler(), ["expert_rating"])]).fit(X) + >>> X_new = pd.DataFrame({"expert_rating": [5, 6, 1], + ... "ignored_new_col": [1.2, 0.3, -0.1]}) + >>> ct.transform(X_new) + array([[ 0.9], + [ 2.1], + [-3.9]]) + .. _visualizing_composite_estimators: Visualizing Composite Estimators ================================ -Estimators can be displayed with a HTML representation when shown in a -jupyter notebook. This can be useful to diagnose or visualize a Pipeline with -many estimators. This visualization is activated by setting the -`display` option in :func:`~sklearn.set_config`:: +Estimators are displayed with an HTML representation when shown in a +jupyter notebook. This is useful to diagnose or visualize a Pipeline with +many estimators. This visualization is activated by default:: + + >>> column_trans # doctest: +SKIP + +It can be deactivated by setting the `display` option in :func:`~sklearn.set_config` +to 'text':: >>> from sklearn import set_config - >>> set_config(display='diagram') # doctest: +SKIP - >>> # diplays HTML representation in a jupyter context + >>> set_config(display='text') # doctest: +SKIP + >>> # displays text representation in a jupyter context >>> column_trans # doctest: +SKIP An example of the HTML output can be seen in the @@ -552,7 +604,7 @@ As an alternative, the HTML can be written to a file using >>> with open('my_estimator.html', 'w') as f: # doctest: +SKIP ... f.write(estimator_html_repr(clf)) -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_compose_plot_column_transformer.py` - * :ref:`sphx_glr_auto_examples_compose_plot_column_transformer_mixed_types.py` +* :ref:`sphx_glr_auto_examples_compose_plot_column_transformer.py` +* :ref:`sphx_glr_auto_examples_compose_plot_column_transformer_mixed_types.py` diff --git a/doc/modules/covariance.rst b/doc/modules/covariance.rst index c97676ea62108..0eadfa2c8c584 100644 --- a/doc/modules/covariance.rst +++ b/doc/modules/covariance.rst @@ -40,11 +40,10 @@ on whether the data are centered, so one may want to use the same mean vector as the training set. If not, both should be centered by the user, and ``assume_centered=True`` should be used. -.. topic:: Examples: +.. rubric:: Examples - * See :ref:`sphx_glr_auto_examples_covariance_plot_covariance_estimation.py` for - an example on how to fit an :class:`EmpiricalCovariance` object - to data. +* See :ref:`sphx_glr_auto_examples_covariance_plot_covariance_estimation.py` for + an example on how to fit an :class:`EmpiricalCovariance` object to data. .. _shrunk_covariance: @@ -77,18 +76,17 @@ smallest and the largest eigenvalues of the empirical covariance matrix. It can be done by simply shifting every eigenvalue according to a given offset, which is equivalent of finding the l2-penalized Maximum Likelihood Estimator of the covariance matrix. In practice, shrinkage -boils down to a simple a convex transformation : :math:`\Sigma_{\rm +boils down to a simple convex transformation : :math:`\Sigma_{\rm shrunk} = (1-\alpha)\hat{\Sigma} + \alpha\frac{{\rm Tr}\hat{\Sigma}}{p}\rm Id`. Choosing the amount of shrinkage, :math:`\alpha` amounts to setting a bias/variance trade-off, and is discussed below. -.. topic:: Examples: +.. rubric:: Examples - * See :ref:`sphx_glr_auto_examples_covariance_plot_covariance_estimation.py` for - an example on how to fit a :class:`ShrunkCovariance` object - to data. +* See :ref:`sphx_glr_auto_examples_covariance_plot_covariance_estimation.py` for + an example on how to fit a :class:`ShrunkCovariance` object to data. Ledoit-Wolf shrinkage @@ -109,30 +107,30 @@ fitting a :class:`LedoitWolf` object to the same sample. It is important to note that when the number of samples is much larger than the number of features, one would expect that no shrinkage would be necessary. The intuition behind this is that if the population covariance - is full rank, when the number of sample grows, the sample covariance will - also become positive definite. As a result, no shrinkage would necessary + is full rank, when the number of samples grows, the sample covariance will + also become positive definite. As a result, no shrinkage would be necessary and the method should automatically do this. This, however, is not the case in the Ledoit-Wolf procedure when the population covariance happens to be a multiple of the identity matrix. In this case, the Ledoit-Wolf shrinkage estimate approaches 1 as the number of samples increases. This indicates that the optimal estimate of the - covariance matrix in the Ledoit-Wolf sense is multiple of the identity. + covariance matrix in the Ledoit-Wolf sense is a multiple of the identity. Since the population covariance is already a multiple of the identity matrix, the Ledoit-Wolf solution is indeed a reasonable estimate. -.. topic:: Examples: +.. rubric:: Examples - * See :ref:`sphx_glr_auto_examples_covariance_plot_covariance_estimation.py` for - an example on how to fit a :class:`LedoitWolf` object to data and - for visualizing the performances of the Ledoit-Wolf estimator in - terms of likelihood. +* See :ref:`sphx_glr_auto_examples_covariance_plot_covariance_estimation.py` for + an example on how to fit a :class:`LedoitWolf` object to data and + for visualizing the performances of the Ledoit-Wolf estimator in + terms of likelihood. -.. topic:: References: +.. rubric:: References - .. [1] O. Ledoit and M. Wolf, "A Well-Conditioned Estimator for Large-Dimensional - Covariance Matrices", Journal of Multivariate Analysis, Volume 88, Issue 2, - February 2004, pages 365-411. +.. [1] O. Ledoit and M. Wolf, "A Well-Conditioned Estimator for Large-Dimensional + Covariance Matrices", Journal of Multivariate Analysis, Volume 88, Issue 2, + February 2004, pages 365-411. .. _oracle_approximating_shrinkage: @@ -158,20 +156,21 @@ object to the same sample. Bias-variance trade-off when setting the shrinkage: comparing the choices of Ledoit-Wolf and OAS estimators -.. topic:: References: +.. rubric:: References - .. [2] Chen et al., "Shrinkage Algorithms for MMSE Covariance Estimation", - IEEE Trans. on Sign. Proc., Volume 58, Issue 10, October 2010. +.. [2] :arxiv:`"Shrinkage algorithms for MMSE covariance estimation.", + Chen, Y., Wiesel, A., Eldar, Y. C., & Hero, A. O. + IEEE Transactions on Signal Processing, 58(10), 5016-5029, 2010. + <0907.4698>` -.. topic:: Examples: +.. rubric:: Examples - * See :ref:`sphx_glr_auto_examples_covariance_plot_covariance_estimation.py` for - an example on how to fit an :class:`OAS` object - to data. +* See :ref:`sphx_glr_auto_examples_covariance_plot_covariance_estimation.py` for + an example on how to fit an :class:`OAS` object to data. - * See :ref:`sphx_glr_auto_examples_covariance_plot_lw_vs_oas.py` to visualize the - Mean Squared Error difference between a :class:`LedoitWolf` and - an :class:`OAS` estimator of the covariance. +* See :ref:`sphx_glr_auto_examples_covariance_plot_lw_vs_oas.py` to visualize the + Mean Squared Error difference between a :class:`LedoitWolf` and + an :class:`OAS` estimator of the covariance. .. figure:: ../auto_examples/covariance/images/sphx_glr_plot_lw_vs_oas_001.png @@ -252,20 +251,20 @@ problem is the GLasso algorithm, from the Friedman 2008 Biostatistics paper. It is the same algorithm as in the R ``glasso`` package. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_covariance_plot_sparse_cov.py`: example on synthetic - data showing some recovery of a structure, and comparing to other - covariance estimators. +* :ref:`sphx_glr_auto_examples_covariance_plot_sparse_cov.py`: example on synthetic + data showing some recovery of a structure, and comparing to other + covariance estimators. - * :ref:`sphx_glr_auto_examples_applications_plot_stock_market.py`: example on real - stock market data, finding which symbols are most linked. +* :ref:`sphx_glr_auto_examples_applications_plot_stock_market.py`: example on real + stock market data, finding which symbols are most linked. -.. topic:: References: +.. rubric:: References - * Friedman et al, `"Sparse inverse covariance estimation with the - graphical lasso" `_, - Biostatistics 9, pp 432, 2008 +* Friedman et al, `"Sparse inverse covariance estimation with the + graphical lasso" `_, + Biostatistics 9, pp 432, 2008 .. _robust_covariance: @@ -311,24 +310,24 @@ the same time. Raw estimates can be accessed as ``raw_location_`` and ``raw_covariance_`` attributes of a :class:`MinCovDet` robust covariance estimator object. -.. topic:: References: +.. rubric:: References - .. [3] P. J. Rousseeuw. Least median of squares regression. - J. Am Stat Ass, 79:871, 1984. - .. [4] A Fast Algorithm for the Minimum Covariance Determinant Estimator, - 1999, American Statistical Association and the American Society - for Quality, TECHNOMETRICS. +.. [3] P. J. Rousseeuw. Least median of squares regression. + J. Am Stat Ass, 79:871, 1984. +.. [4] A Fast Algorithm for the Minimum Covariance Determinant Estimator, + 1999, American Statistical Association and the American Society + for Quality, TECHNOMETRICS. -.. topic:: Examples: +.. rubric:: Examples - * See :ref:`sphx_glr_auto_examples_covariance_plot_robust_vs_empirical_covariance.py` for - an example on how to fit a :class:`MinCovDet` object to data and see how - the estimate remains accurate despite the presence of outliers. +* See :ref:`sphx_glr_auto_examples_covariance_plot_robust_vs_empirical_covariance.py` for + an example on how to fit a :class:`MinCovDet` object to data and see how + the estimate remains accurate despite the presence of outliers. - * See :ref:`sphx_glr_auto_examples_covariance_plot_mahalanobis_distances.py` to - visualize the difference between :class:`EmpiricalCovariance` and - :class:`MinCovDet` covariance estimators in terms of Mahalanobis distance - (so we get a better estimate of the precision matrix too). +* See :ref:`sphx_glr_auto_examples_covariance_plot_mahalanobis_distances.py` to + visualize the difference between :class:`EmpiricalCovariance` and + :class:`MinCovDet` covariance estimators in terms of Mahalanobis distance + (so we get a better estimate of the precision matrix too). .. |robust_vs_emp| image:: ../auto_examples/covariance/images/sphx_glr_plot_robust_vs_empirical_covariance_001.png :target: ../auto_examples/covariance/plot_robust_vs_empirical_covariance.html diff --git a/doc/modules/cross_decomposition.rst b/doc/modules/cross_decomposition.rst index 981f7d98fbbde..01722cbd07ab6 100644 --- a/doc/modules/cross_decomposition.rst +++ b/doc/modules/cross_decomposition.rst @@ -28,9 +28,9 @@ PLS draws similarities with `Principal Component Regression `_ (PCR), where the samples are first projected into a lower-dimensional subspace, and the targets `y` are predicted using `transformed(X)`. One issue with PCR is that -the dimensionality reduction is unsupervized, and may lose some important +the dimensionality reduction is unsupervised, and may lose some important variables: PCR would keep the features with the most variance, but it's -possible that features with a small variances are relevant from predicting +possible that features with small variances are relevant for predicting the target. In a way, PLS allows for the same kind of dimensionality reduction, but by taking into account the targets `y`. An illustration of this fact is given in the following example: @@ -64,7 +64,7 @@ Set :math:`X_1` to :math:`X` and :math:`Y_1` to :math:`Y`. Then, for each :math:`C = X_k^T Y_k`. :math:`u_k` and :math:`v_k` are called the *weights*. By definition, :math:`u_k` and :math:`v_k` are - choosen so that they maximize the covariance between the projected + chosen so that they maximize the covariance between the projected :math:`X_k` and the projected target, that is :math:`\text{Cov}(X_k u_k, Y_k v_k)`. - b) Project :math:`X_k` and :math:`Y_k` on the singular vectors to obtain @@ -88,42 +88,39 @@ Note that the scores matrices :math:`\Xi` and :math:`\Omega` correspond to the projections of the training data :math:`X` and :math:`Y`, respectively. Step *a)* may be performed in two ways: either by computing the whole SVD of -:math:`C` and only retain the singular vectors with the biggest singular +:math:`C` and only retaining the singular vectors with the biggest singular values, or by directly computing the singular vectors using the power method (cf section 11.3 in [1]_), which corresponds to the `'nipals'` option of the `algorithm` parameter. +.. dropdown:: Transforming data -Transforming data -^^^^^^^^^^^^^^^^^ + To transform :math:`X` into :math:`\bar{X}`, we need to find a projection + matrix :math:`P` such that :math:`\bar{X} = XP`. We know that for the + training data, :math:`\Xi = XP`, and :math:`X = \Xi \Gamma^T`. Setting + :math:`P = U(\Gamma^T U)^{-1}` where :math:`U` is the matrix with the + :math:`u_k` in the columns, we have :math:`XP = X U(\Gamma^T U)^{-1} = \Xi + (\Gamma^T U) (\Gamma^T U)^{-1} = \Xi` as desired. The rotation matrix + :math:`P` can be accessed from the `x_rotations_` attribute. -To transform :math:`X` into :math:`\bar{X}`, we need to find a projection -matrix :math:`P` such that :math:`\bar{X} = XP`. We know that for the -training data, :math:`\Xi = XP`, and :math:`X = \Xi \Gamma^T`. Setting -:math:`P = U(\Gamma^T U)^{-1}` where :math:`U` is the matrix with the -:math:`u_k` in the columns, we have :math:`XP = X U(\Gamma^T U)^{-1} = \Xi -(\Gamma^T U) (\Gamma^T U)^{-1} = \Xi` as desired. The rotation matrix -:math:`P` can be accessed from the `x_rotations_` attribute. + Similarly, :math:`Y` can be transformed using the rotation matrix + :math:`V(\Delta^T V)^{-1}`, accessed via the `y_rotations_` attribute. -Similarly, :math:`Y` can be transformed using the rotation matrix -:math:`V(\Delta^T V)^{-1}`, accessed via the `y_rotations_` attribute. +.. dropdown:: Predicting the targets `Y` -Predicting the targets Y -^^^^^^^^^^^^^^^^^^^^^^^^ + To predict the targets of some data :math:`X`, we are looking for a + coefficient matrix :math:`\beta \in R^{d \times t}` such that :math:`Y = + X\beta`. -To predict the targets of some data :math:`X`, we are looking for a -coefficient matrix :math:`\beta \in R^{d \times t}` such that :math:`Y = -X\beta`. + The idea is to try to predict the transformed targets :math:`\Omega` as a + function of the transformed samples :math:`\Xi`, by computing :math:`\alpha + \in \mathbb{R}` such that :math:`\Omega = \alpha \Xi`. -The idea is to try to predict the transformed targets :math:`\Omega` as a -function of the transformed samples :math:`\Xi`, by computing :math:`\alpha -\in \mathbb{R}` such that :math:`\Omega = \alpha \Xi`. + Then, we have :math:`Y = \Omega \Delta^T = \alpha \Xi \Delta^T`, and since + :math:`\Xi` is the transformed training data we have that :math:`Y = X \alpha + P \Delta^T`, and as a result the coefficient matrix :math:`\beta = \alpha P + \Delta^T`. -Then, we have :math:`Y = \Omega \Delta^T = \alpha \Xi \Delta^T`, and since -:math:`\Xi` is the transformed training data we have that :math:`Y = X \alpha -P \Delta^T`, and as a result the coefficient matrix :math:`\beta = \alpha P -\Delta^T`. - -:math:`\beta` can be accessed through the `coef_` attribute. + :math:`\beta` can be accessed through the `coef_` attribute. PLSSVD ------ @@ -180,15 +177,13 @@ Since :class:`CCA` involves the inversion of :math:`X_k^TX_k` and :math:`Y_k^TY_k`, this estimator can be unstable if the number of features or targets is greater than the number of samples. +.. rubric:: References -.. topic:: Reference: - - .. [1] `A survey of Partial Least Squares (PLS) methods, with emphasis on - the two-block case - `_ - JA Wegelin +.. [1] `A survey of Partial Least Squares (PLS) methods, with emphasis on the two-block + case `_, + JA Wegelin -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_cross_decomposition_plot_compare_cross_decomposition.py` - * :ref:`sphx_glr_auto_examples_cross_decomposition_plot_pcr_vs_pls.py` +* :ref:`sphx_glr_auto_examples_cross_decomposition_plot_compare_cross_decomposition.py` +* :ref:`sphx_glr_auto_examples_cross_decomposition_plot_pcr_vs_pls.py` diff --git a/doc/modules/cross_validation.rst b/doc/modules/cross_validation.rst index ae3d38f168f3f..b1c9ccec8f641 100644 --- a/doc/modules/cross_validation.rst +++ b/doc/modules/cross_validation.rst @@ -55,7 +55,7 @@ data for testing (evaluating) our classifier:: >>> clf = svm.SVC(kernel='linear', C=1).fit(X_train, y_train) >>> clf.score(X_test, y_test) - 0.96... + 0.96 When evaluating different settings ("hyperparameters") for estimators, such as the ``C`` setting that must be manually set for an SVM, @@ -86,10 +86,10 @@ the training set is split into *k* smaller sets but generally follow the same principles). The following procedure is followed for each of the *k* "folds": - * A model is trained using :math:`k-1` of the folds as training data; - * the resulting model is validated on the remaining part of the data - (i.e., it is used as a test set to compute a performance measure - such as accuracy). +* A model is trained using :math:`k-1` of the folds as training data; +* the resulting model is validated on the remaining part of the data + (i.e., it is used as a test set to compute a performance measure + such as accuracy). The performance measure reported by *k*-fold cross-validation is then the average of the values computed in the loop. @@ -102,6 +102,7 @@ where the number of samples is very small. .. image:: ../images/grid_search_cross_validation.png :width: 500px :height: 300px + :alt: A depiction of a 5 fold cross validation on a training set, while holding out a test set. :align: center Computing cross-validated metrics @@ -119,7 +120,7 @@ time):: >>> clf = svm.SVC(kernel='linear', C=1, random_state=42) >>> scores = cross_val_score(clf, X, y, cv=5) >>> scores - array([0.96..., 1. , 0.96..., 0.96..., 1. ]) + array([0.96, 1. , 0.96, 0.96, 1. ]) The mean score and the standard deviation are hence given by:: @@ -134,7 +135,7 @@ scoring parameter:: >>> scores = cross_val_score( ... clf, X, y, cv=5, scoring='f1_macro') >>> scores - array([0.96..., 1. ..., 0.96..., 0.96..., 1. ]) + array([0.96, 1., 0.96, 0.96, 1.]) See :ref:`scoring_parameter` for details. In the case of the Iris dataset, the samples are balanced across target @@ -152,7 +153,7 @@ validation iterator instead, for instance:: >>> n_samples = X.shape[0] >>> cv = ShuffleSplit(n_splits=5, test_size=0.3, random_state=0) >>> cross_val_score(clf, X, y, cv=cv) - array([0.977..., 0.977..., 1. ..., 0.955..., 1. ]) + array([0.977, 0.977, 1., 0.955, 1.]) Another option is to use an iterable yielding (train, test) splits as arrays of indices, for example:: @@ -167,34 +168,34 @@ indices, for example:: ... >>> custom_cv = custom_cv_2folds(X) >>> cross_val_score(clf, X, y, cv=custom_cv) - array([1. , 0.973...]) + array([1. , 0.973]) -.. topic:: Data transformation with held out data +.. dropdown:: Data transformation with held-out data - Just as it is important to test a predictor on data held-out from - training, preprocessing (such as standardization, feature selection, etc.) - and similar :ref:`data transformations ` similarly should - be learnt from a training set and applied to held-out data for prediction:: + Just as it is important to test a predictor on data held-out from + training, preprocessing (such as standardization, feature selection, etc.) + and similar :ref:`data transformations ` similarly should + be learnt from a training set and applied to held-out data for prediction:: - >>> from sklearn import preprocessing - >>> X_train, X_test, y_train, y_test = train_test_split( - ... X, y, test_size=0.4, random_state=0) - >>> scaler = preprocessing.StandardScaler().fit(X_train) - >>> X_train_transformed = scaler.transform(X_train) - >>> clf = svm.SVC(C=1).fit(X_train_transformed, y_train) - >>> X_test_transformed = scaler.transform(X_test) - >>> clf.score(X_test_transformed, y_test) - 0.9333... + >>> from sklearn import preprocessing + >>> X_train, X_test, y_train, y_test = train_test_split( + ... X, y, test_size=0.4, random_state=0) + >>> scaler = preprocessing.StandardScaler().fit(X_train) + >>> X_train_transformed = scaler.transform(X_train) + >>> clf = svm.SVC(C=1).fit(X_train_transformed, y_train) + >>> X_test_transformed = scaler.transform(X_test) + >>> clf.score(X_test_transformed, y_test) + 0.9333 - A :class:`Pipeline ` makes it easier to compose - estimators, providing this behavior under cross-validation:: + A :class:`Pipeline ` makes it easier to compose + estimators, providing this behavior under cross-validation:: - >>> from sklearn.pipeline import make_pipeline - >>> clf = make_pipeline(preprocessing.StandardScaler(), svm.SVC(C=1)) - >>> cross_val_score(clf, X, y, cv=cv) - array([0.977..., 0.933..., 0.955..., 0.933..., 0.977...]) + >>> from sklearn.pipeline import make_pipeline + >>> clf = make_pipeline(preprocessing.StandardScaler(), svm.SVC(C=1)) + >>> cross_val_score(clf, X, y, cv=cv) + array([0.977, 0.933, 0.955, 0.933, 0.977]) - See :ref:`combining_estimators`. + See :ref:`combining_estimators`. .. _multimetric_cross_validation: @@ -208,8 +209,8 @@ two ways: - It allows specifying multiple metrics for evaluation. - It returns a dict containing fit-times, score-times - (and optionally training scores as well as fitted estimators) in - addition to the test score. + (and optionally training scores, fitted estimators, train-test split indices) + in addition to the test score. For single metric evaluation, where the scoring parameter is a string, callable or None, the keys will be - ``['test_score', 'fit_time', 'score_time']`` @@ -220,10 +221,10 @@ following keys - ``return_train_score`` is set to ``False`` by default to save computation time. To evaluate the scores on the training set as well you need to set it to -``True``. - -You may also retain the estimator fitted on each training set by setting -``return_estimator=True``. +``True``. You may also retain the estimator fitted on each training set by +setting ``return_estimator=True``. Similarly, you may set +`return_indices=True` to retain the training and testing indices used to split +the dataset into train and test sets for each cv split. The multiple metrics can be specified either as a list, tuple or set of predefined scorer names:: @@ -236,7 +237,7 @@ predefined scorer names:: >>> sorted(scores.keys()) ['fit_time', 'score_time', 'test_precision_macro', 'test_recall_macro'] >>> scores['test_recall_macro'] - array([0.96..., 1. ..., 0.96..., 0.96..., 1. ]) + array([0.96, 1., 0.96, 0.96, 1.]) Or as a dict mapping scorer name to a predefined or custom scoring function:: @@ -249,7 +250,7 @@ Or as a dict mapping scorer name to a predefined or custom scoring function:: ['fit_time', 'score_time', 'test_prec_macro', 'test_rec_macro', 'train_prec_macro', 'train_rec_macro'] >>> scores['train_rec_macro'] - array([0.97..., 0.97..., 0.99..., 0.98..., 0.98...]) + array([0.97, 0.97, 0.99, 0.98, 0.98]) Here is an example of ``cross_validate`` using a single metric:: @@ -278,7 +279,7 @@ can be used (otherwise, an exception is raised). over cross-validation folds, whereas :func:`cross_val_predict` simply returns the labels (or probabilities) from several distinct models undistinguished. Thus, :func:`cross_val_predict` is not an appropriate - measure of generalisation error. + measure of generalization error. The function :func:`cross_val_predict` is appropriate for: @@ -290,14 +291,14 @@ The function :func:`cross_val_predict` is appropriate for: The available cross validation iterators are introduced in the following section. -.. topic:: Examples +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_model_selection_plot_roc_crossval.py`, - * :ref:`sphx_glr_auto_examples_feature_selection_plot_rfe_with_cross_validation.py`, - * :ref:`sphx_glr_auto_examples_model_selection_plot_grid_search_digits.py`, - * :ref:`sphx_glr_auto_examples_model_selection_grid_search_text_feature_extraction.py`, - * :ref:`sphx_glr_auto_examples_model_selection_plot_cv_predict.py`, - * :ref:`sphx_glr_auto_examples_model_selection_plot_nested_cross_validation_iris.py`. +* :ref:`sphx_glr_auto_examples_model_selection_plot_roc_crossval.py`, +* :ref:`sphx_glr_auto_examples_feature_selection_plot_rfe_with_cross_validation.py`, +* :ref:`sphx_glr_auto_examples_model_selection_plot_grid_search_digits.py`, +* :ref:`sphx_glr_auto_examples_model_selection_plot_grid_search_text_feature_extraction.py`, +* :ref:`sphx_glr_auto_examples_model_selection_plot_cv_predict.py`, +* :ref:`sphx_glr_auto_examples_model_selection_plot_nested_cross_validation_iris.py`. Cross validation iterators ========================== @@ -353,7 +354,7 @@ Example of 2-fold cross-validation on a dataset with 4 samples:: Here is a visualization of the cross-validation behavior. Note that :class:`KFold` is not affected by classes or groups. -.. figure:: ../auto_examples/model_selection/images/sphx_glr_plot_cv_indices_004.png +.. figure:: ../auto_examples/model_selection/images/sphx_glr_plot_cv_indices_006.png :target: ../auto_examples/model_selection/plot_cv_indices.html :align: center :scale: 75% @@ -371,8 +372,7 @@ Thus, one can create the training/test sets using numpy indexing:: Repeated K-Fold ^^^^^^^^^^^^^^^ -:class:`RepeatedKFold` repeats K-Fold n times. It can be used when one -requires to run :class:`KFold` n times, producing different splits in +:class:`RepeatedKFold` repeats :class:`KFold` :math:`n` times, producing different splits in each repetition. Example of 2-fold K-Fold repeated 2 times:: @@ -391,7 +391,7 @@ Example of 2-fold K-Fold repeated 2 times:: [1 3] [0 2] -Similarly, :class:`RepeatedStratifiedKFold` repeats Stratified K-Fold n times +Similarly, :class:`RepeatedStratifiedKFold` repeats :class:`StratifiedKFold` :math:`n` times with different randomization in each repetition. .. _leave_one_out: @@ -402,7 +402,7 @@ Leave One Out (LOO) :class:`LeaveOneOut` (or LOO) is a simple cross-validation. Each learning set is created by taking all the samples except one, the test set being the sample left out. Thus, for :math:`n` samples, we have :math:`n` different -training sets and :math:`n` different tests set. This cross-validation +training sets and :math:`n` different test sets. This cross-validation procedure does not waste much data as only one sample is removed from the training set:: @@ -433,25 +433,24 @@ folds are virtually identical to each other and to the model built from the entire training set. However, if the learning curve is steep for the training size in question, -then 5- or 10- fold cross validation can overestimate the generalization error. +then 5 or 10-fold cross validation can overestimate the generalization error. -As a general rule, most authors, and empirical evidence, suggest that 5- or 10- -fold cross validation should be preferred to LOO. +As a general rule, most authors and empirical evidence suggest that 5 or 10-fold +cross validation should be preferred to LOO. +.. dropdown:: References -.. topic:: References: - - * ``_; - * T. Hastie, R. Tibshirani, J. Friedman, `The Elements of Statistical Learning - `_, Springer 2009 - * L. Breiman, P. Spector `Submodel selection and evaluation in regression: The X-random case - `_, International Statistical Review 1992; - * R. Kohavi, `A Study of Cross-Validation and Bootstrap for Accuracy Estimation and Model Selection - `_, Intl. Jnt. Conf. AI - * R. Bharat Rao, G. Fung, R. Rosales, `On the Dangers of Cross-Validation. An Experimental Evaluation - `_, SIAM 2008; - * G. James, D. Witten, T. Hastie, R Tibshirani, `An Introduction to - Statistical Learning `_, Springer 2013. + * ``_; + * T. Hastie, R. Tibshirani, J. Friedman, `The Elements of Statistical Learning + `_, Springer 2009 + * L. Breiman, P. Spector `Submodel selection and evaluation in regression: The X-random case + `_, International Statistical Review 1992; + * R. Kohavi, `A Study of Cross-Validation and Bootstrap for Accuracy Estimation and Model Selection + `_, Intl. Jnt. Conf. AI + * R. Bharat Rao, G. Fung, R. Rosales, `On the Dangers of Cross-Validation. An Experimental Evaluation + `_, SIAM 2008; + * G. James, D. Witten, T. Hastie, R. Tibshirani, `An Introduction to + Statistical Learning `_, Springer 2013. .. _leave_p_out: @@ -509,7 +508,7 @@ Here is a usage example:: Here is a visualization of the cross-validation behavior. Note that :class:`ShuffleSplit` is not affected by classes or groups. -.. figure:: ../auto_examples/model_selection/images/sphx_glr_plot_cv_indices_006.png +.. figure:: ../auto_examples/model_selection/images/sphx_glr_plot_cv_indices_008.png :target: ../auto_examples/model_selection/plot_cv_indices.html :align: center :scale: 75% @@ -520,22 +519,43 @@ the proportion of samples on each side of the train / test split. .. _stratification: -Cross-validation iterators with stratification based on class labels. ---------------------------------------------------------------------- +Cross-validation iterators with stratification based on class labels +-------------------------------------------------------------------- + +Some classification tasks can naturally exhibit rare classes: for instance, +there could be orders of magnitude more negative observations than positive +observations (e.g. medical screening, fraud detection, etc). As a result, +cross-validation splitting can generate train or validation folds without any +occurrence of a particular class. This typically leads to undefined +classification metrics (e.g. ROC AUC), exceptions raised when attempting to +call :term:`fit` or missing columns in the output of the `predict_proba` or +`decision_function` methods of multiclass classifiers trained on different +folds. -Some classification problems can exhibit a large imbalance in the distribution -of the target classes: for instance there could be several times more negative -samples than positive samples. In such cases it is recommended to use -stratified sampling as implemented in :class:`StratifiedKFold` and -:class:`StratifiedShuffleSplit` to ensure that relative class frequencies is -approximately preserved in each train and validation fold. +To mitigate such problems, splitters such as :class:`StratifiedKFold` and +:class:`StratifiedShuffleSplit` implement stratified sampling to ensure that +relative class frequencies are approximately preserved in each fold. + +.. note:: + + Stratified sampling was introduced in scikit-learn to workaround the + aforementioned engineering problems rather than solve a statistical one. + + Stratification makes cross-validation folds more homogeneous, and as a result + hides some of the variability inherent to fitting models with a limited + number of observations. + + As a result, stratification can artificially shrink the spread of the metric + measured across cross-validation iterations: the inter-fold variability does + no longer reflect the uncertainty in the performance of classifiers in the + presence of rare classes. .. _stratified_k_fold: -Stratified k-fold +Stratified K-fold ^^^^^^^^^^^^^^^^^ -:class:`StratifiedKFold` is a variation of *k-fold* which returns *stratified* +:class:`StratifiedKFold` is a variation of *K-fold* which returns *stratified* folds: each set contains approximately the same percentage of samples of each target class as the complete set. @@ -562,11 +582,11 @@ two unbalanced classes. We show the number of samples in each class and compare train - [34] | test - [11 5] We can see that :class:`StratifiedKFold` preserves the class ratios -(approximately 1 / 10) in both train and test dataset. +(approximately 1 / 10) in both train and test datasets. Here is a visualization of the cross-validation behavior. -.. figure:: ../auto_examples/model_selection/images/sphx_glr_plot_cv_indices_007.png +.. figure:: ../auto_examples/model_selection/images/sphx_glr_plot_cv_indices_009.png :target: ../auto_examples/model_selection/plot_cv_indices.html :align: center :scale: 75% @@ -580,22 +600,35 @@ Stratified Shuffle Split ^^^^^^^^^^^^^^^^^^^^^^^^ :class:`StratifiedShuffleSplit` is a variation of *ShuffleSplit*, which returns -stratified splits, *i.e* which creates splits by preserving the same +stratified splits, *i.e.* which creates splits by preserving the same percentage for each target class as in the complete set. Here is a visualization of the cross-validation behavior. -.. figure:: ../auto_examples/model_selection/images/sphx_glr_plot_cv_indices_009.png +.. figure:: ../auto_examples/model_selection/images/sphx_glr_plot_cv_indices_012.png :target: ../auto_examples/model_selection/plot_cv_indices.html :align: center :scale: 75% +.. _predefined_split: + +Predefined fold-splits / Validation-sets +---------------------------------------- + +For some datasets, a pre-defined split of the data into training- and +validation fold or into several cross-validation folds already +exists. Using :class:`PredefinedSplit` it is possible to use these folds +e.g. when searching for hyperparameters. + +For example, when using a validation set, set the ``test_fold`` to 0 for all +samples that are part of the validation set, and to -1 for all other samples. + .. _group_cv: -Cross-validation iterators for grouped data. --------------------------------------------- +Cross-validation iterators for grouped data +------------------------------------------- -The i.i.d. assumption is broken if the underlying generative process yield +The i.i.d. assumption is broken if the underlying generative process yields groups of dependent samples. Such a grouping of data is domain specific. An example would be when there is @@ -614,10 +647,10 @@ parameter. .. _group_k_fold: -Group k-fold +Group K-fold ^^^^^^^^^^^^ -:class:`GroupKFold` is a variation of k-fold which ensures that the same group is +:class:`GroupKFold` is a variation of K-fold which ensures that the same group is not represented in both testing and training sets. For example if the data is obtained from different subjects with several samples per-subject and if the model is flexible enough to learn from highly person specific features it @@ -641,10 +674,71 @@ Imagine you have three subjects, each with an associated number from 1 to 3:: Each subject is in a different testing fold, and the same subject is never in both testing and training. Notice that the folds do not have exactly the same -size due to the imbalance in the data. +size due to the imbalance in the data. If class proportions must be balanced +across folds, :class:`StratifiedGroupKFold` is a better option. Here is a visualization of the cross-validation behavior. +.. figure:: ../auto_examples/model_selection/images/sphx_glr_plot_cv_indices_007.png + :target: ../auto_examples/model_selection/plot_cv_indices.html + :align: center + :scale: 75% + +Similar to :class:`KFold`, the test sets from :class:`GroupKFold` will form a +complete partition of all the data. + +While :class:`GroupKFold` attempts to place the same number of samples in each +fold when ``shuffle=False``, when ``shuffle=True`` it attempts to place an equal +number of distinct groups in each fold (but does not account for group sizes). + +.. _stratified_group_k_fold: + +StratifiedGroupKFold +^^^^^^^^^^^^^^^^^^^^ + +:class:`StratifiedGroupKFold` is a cross-validation scheme that combines both +:class:`StratifiedKFold` and :class:`GroupKFold`. The idea is to try to +preserve the distribution of classes in each split while keeping each group +within a single split. That might be useful when you have an unbalanced +dataset so that using just :class:`GroupKFold` might produce skewed splits. + +Example:: + + >>> from sklearn.model_selection import StratifiedGroupKFold + >>> X = list(range(18)) + >>> y = [1] * 6 + [0] * 12 + >>> groups = [1, 2, 3, 3, 4, 4, 1, 1, 2, 2, 3, 4, 5, 5, 5, 6, 6, 6] + >>> sgkf = StratifiedGroupKFold(n_splits=3) + >>> for train, test in sgkf.split(X, y, groups=groups): + ... print("%s %s" % (train, test)) + [ 0 2 3 4 5 6 7 10 11 15 16 17] [ 1 8 9 12 13 14] + [ 0 1 4 5 6 7 8 9 11 12 13 14] [ 2 3 10 15 16 17] + [ 1 2 3 8 9 10 12 13 14 15 16 17] [ 0 4 5 6 7 11] + +.. dropdown:: Implementation notes + + - With the current implementation full shuffle is not possible in most + scenarios. When shuffle=True, the following happens: + + 1. All groups are shuffled. + 2. Groups are sorted by standard deviation of classes using stable sort. + 3. Sorted groups are iterated over and assigned to folds. + + That means that only groups with the same standard deviation of class + distribution will be shuffled, which might be useful when each group has only + a single class. + - The algorithm greedily assigns each group to one of n_splits test sets, + choosing the test set that minimises the variance in class distribution + across test sets. Group assignment proceeds from groups with highest to + lowest variance in class frequency, i.e. large groups peaked on one or few + classes are assigned first. + - This split is suboptimal in a sense that it might produce imbalanced splits + even if perfect stratification is possible. If you have relatively close + distribution of classes in each group, using :class:`GroupKFold` is better. + + +Here is a visualization of cross-validation behavior for uneven groups: + .. figure:: ../auto_examples/model_selection/images/sphx_glr_plot_cv_indices_005.png :target: ../auto_examples/model_selection/plot_cv_indices.html :align: center @@ -655,13 +749,14 @@ Here is a visualization of the cross-validation behavior. Leave One Group Out ^^^^^^^^^^^^^^^^^^^ -:class:`LeaveOneGroupOut` is a cross-validation scheme which holds out -the samples according to a third-party provided array of integer groups. This -group information can be used to encode arbitrary domain specific pre-defined -cross-validation folds. +:class:`LeaveOneGroupOut` is a cross-validation scheme where each split holds +out samples belonging to one specific group. Group information is +provided via an array that encodes the group of each sample. Each training set is thus constituted by all the samples except the ones -related to a specific group. +related to a specific group. This is the same as :class:`LeavePGroupsOut` with +`n_groups=1` and the same as :class:`GroupKFold` with `n_splits` equal to the +number of unique labels passed to the `groups` parameter. For example, in the cases of multiple experiments, :class:`LeaveOneGroupOut` can be used to create a cross-validation based on the different experiments: @@ -688,8 +783,10 @@ for cross-validation against time-based splits. Leave P Groups Out ^^^^^^^^^^^^^^^^^^ -:class:`LeavePGroupsOut` is similar as :class:`LeaveOneGroupOut`, but removes -samples related to :math:`P` groups for each training/test set. +:class:`LeavePGroupsOut` is similar to :class:`LeaveOneGroupOut`, but removes +samples related to :math:`P` groups for each training/test set. All possible +combinations of :math:`P` groups are left out, meaning test sets will overlap +for :math:`P>1`. Example of Leave-2-Group Out:: @@ -713,7 +810,8 @@ Group Shuffle Split The :class:`GroupShuffleSplit` iterator behaves as a combination of :class:`ShuffleSplit` and :class:`LeavePGroupsOut`, and generates a sequence of randomized partitions in which a subset of groups are held -out for each split. +out for each split. Each train/test split is performed independently meaning +there is no guaranteed relationship between successive test sets. Here is a usage example:: @@ -733,7 +831,7 @@ Here is a usage example:: Here is a visualization of the cross-validation behavior. -.. figure:: ../auto_examples/model_selection/images/sphx_glr_plot_cv_indices_008.png +.. figure:: ../auto_examples/model_selection/images/sphx_glr_plot_cv_indices_011.png :target: ../auto_examples/model_selection/plot_cv_indices.html :align: center :scale: 75% @@ -745,23 +843,10 @@ expensive. In such a scenario, :class:`GroupShuffleSplit` provides a random sample (with replacement) of the train / test splits generated by :class:`LeavePGroupsOut`. -.. _predefined_split: - -Predefined Fold-Splits / Validation-Sets ----------------------------------------- - -For some datasets, a pre-defined split of the data into training- and -validation fold or into several cross-validation folds already -exists. Using :class:`PredefinedSplit` it is possible to use these folds -e.g. when searching for hyperparameters. - -For example, when using a validation set, set the ``test_fold`` to 0 for all -samples that are part of the validation set, and to -1 for all other samples. - Using cross-validation iterators to split train and test -------------------------------------------------------- -The above group cross-validation functions may also be useful for spitting a +The above group cross-validation functions may also be useful for splitting a dataset into training and testing subsets. Note that the convenience function :func:`train_test_split` is a wrapper around :func:`ShuffleSplit` and thus only allows for stratified splitting (using the class labels) @@ -792,13 +877,13 @@ cross-validation splitter. For example:: Cross validation of time series data ------------------------------------ -Time series data is characterised by the correlation between observations +Time series data is characterized by the correlation between observations that are near in time (*autocorrelation*). However, classical cross-validation techniques such as :class:`KFold` and :class:`ShuffleSplit` assume the samples are independent and identically distributed, and would result in unreasonable correlation between training and testing instances (yielding poor estimates of -generalisation error) on time series data. Therefore, it is very important +generalization error) on time series data. Therefore, it is very important to evaluate our model for time series data on the "future" observations least like those that are used to train the model. To achieve this, one solution is provided by :class:`TimeSeriesSplit`. @@ -816,7 +901,8 @@ Also, it adds all surplus data to the first training partition, which is always used to train the model. This class can be used to cross-validate time series data samples -that are observed at fixed time intervals. +that are observed at fixed time intervals. Indeed, the folds must +represent the same duration, in order to have comparable metrics across folds. Example of 3-split time series cross-validation on a dataset with 6 samples:: @@ -835,7 +921,7 @@ Example of 3-split time series cross-validation on a dataset with 6 samples:: Here is a visualization of the cross-validation behavior. -.. figure:: ../auto_examples/model_selection/images/sphx_glr_plot_cv_indices_010.png +.. figure:: ../auto_examples/model_selection/images/sphx_glr_plot_cv_indices_013.png :target: ../auto_examples/model_selection/plot_cv_indices.html :align: center :scale: 75% @@ -844,8 +930,8 @@ A note on shuffling =================== If the data ordering is not arbitrary (e.g. samples with the same class label -are contiguous), shuffling it first may be essential to get a meaningful cross- -validation result. However, the opposite may be true if the samples are not +are contiguous), shuffling it first may be essential to get a meaningful +cross-validation result. However, the opposite may be true if the samples are not independently and identically distributed. For example, if samples correspond to news articles, and are ordered by their time of publication, then shuffling the data will likely lead to a model that is overfit and an inflated validation @@ -856,8 +942,8 @@ Some cross validation iterators, such as :class:`KFold`, have an inbuilt option to shuffle the data indices before splitting them. Note that: * This consumes less memory than shuffling the data directly. -* By default no shuffling occurs, including for the (stratified) K fold cross- - validation performed by specifying ``cv=some_integer`` to +* By default no shuffling occurs, including for the (stratified) K fold + cross-validation performed by specifying ``cv=some_integer`` to :func:`cross_val_score`, grid search, etc. Keep in mind that :func:`train_test_split` still returns a random split. * The ``random_state`` parameter defaults to ``None``, meaning that the @@ -882,56 +968,59 @@ Permutation test score ====================== :func:`~sklearn.model_selection.permutation_test_score` offers another way -to evaluate the performance of classifiers. It provides a permutation-based -p-value, which represents how likely an observed performance of the -classifier would be obtained by chance. The null hypothesis in this test is -that the classifier fails to leverage any statistical dependency between the -features and the labels to make correct predictions on left out data. +to evaluate the performance of a :term:`predictor`. It provides a +permutation-based p-value, which represents how likely an observed performance of the +estimator would be obtained by chance. The null hypothesis in this test is +that the estimator fails to leverage any statistical dependency between the +features and the targets to make correct predictions on left-out data. :func:`~sklearn.model_selection.permutation_test_score` generates a null distribution by calculating `n_permutations` different permutations of the -data. In each permutation the labels are randomly shuffled, thereby removing -any dependency between the features and the labels. The p-value output -is the fraction of permutations for which the average cross-validation score -obtained by the model is better than the cross-validation score obtained by -the model using the original data. For reliable results ``n_permutations`` -should typically be larger than 100 and ``cv`` between 3-10 folds. - -A low p-value provides evidence that the dataset contains real dependency -between features and labels and the classifier was able to utilize this -to obtain good results. A high p-value could be due to a lack of dependency -between features and labels (there is no difference in feature values between -the classes) or because the classifier was not able to use the dependency in -the data. In the latter case, using a more appropriate classifier that -is able to utilize the structure in the data, would result in a low -p-value. - -Cross-validation provides information about how well a classifier generalizes, -specifically the range of expected errors of the classifier. However, a -classifier trained on a high dimensional dataset with no structure may still +data. In each permutation the target values are randomly shuffled, thereby removing +any dependency between the features and the targets. The p-value output is the fraction +of permutations whose cross-validation score is better or equal than the true score +without permuting targets. For reliable results ``n_permutations`` should typically be +larger than 100 and ``cv`` between 3-10 folds. + +A low p-value provides evidence that the dataset contains some real dependency between +features and targets **and** that the estimator was able to utilize this dependency to +obtain good results. A high p-value, in reverse, could be due to either one of these: + +- a lack of dependency between features and targets (i.e., there is no systematic + relationship and any observed patterns are likely due to random chance) +- **or** because the estimator was not able to use the dependency in the data (for + instance because it underfit). + +In the latter case, using a more appropriate estimator that is able to use the +structure in the data, would result in a lower p-value. + +Cross-validation provides information about how well an estimator generalizes +by estimating the range of its expected scores. However, an +estimator trained on a high dimensional dataset with no structure may still perform better than expected on cross-validation, just by chance. This can typically happen with small datasets with less than a few hundred samples. :func:`~sklearn.model_selection.permutation_test_score` provides information -on whether the classifier has found a real class structure and can help in -evaluating the performance of the classifier. +on whether the estimator has found a real dependency between features and targets and +can help in evaluating the performance of the estimator. It is important to note that this test has been shown to produce low p-values even if there is only weak structure in the data because in the corresponding permutated datasets there is absolutely no structure. This -test is therefore only able to show when the model reliably outperforms +test is therefore only able to show whether the model reliably outperforms random guessing. Finally, :func:`~sklearn.model_selection.permutation_test_score` is computed -using brute force and interally fits ``(n_permutations + 1) * n_cv`` models. +using brute force and internally fits ``(n_permutations + 1) * n_cv`` models. It is therefore only tractable with small datasets for which fitting an -individual model is very fast. +individual model is very fast. Using the `n_jobs` parameter parallelizes the +computation and thus speeds it up. -.. topic:: Examples +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_feature_selection_plot_permutation_test_for_classification.py` +* :ref:`sphx_glr_auto_examples_model_selection_plot_permutation_tests_for_classification.py` -.. topic:: References: +.. dropdown:: References - * Ojala and Garriga. `Permutation Tests for Studying Classifier Performance - `_. - J. Mach. Learn. Res. 2010. + * Ojala and Garriga. `Permutation Tests for Studying Classifier Performance + `_. + J. Mach. Learn. Res. 2010. diff --git a/doc/modules/decomposition.rst b/doc/modules/decomposition.rst index 7e8e79d9d8bdd..24fcd43a292c0 100644 --- a/doc/modules/decomposition.rst +++ b/doc/modules/decomposition.rst @@ -51,10 +51,11 @@ data based on the amount of variance it explains. As such it implements a :scale: 75% -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_decomposition_plot_pca_vs_lda.py` - * :ref:`sphx_glr_auto_examples_decomposition_plot_pca_vs_fa_model_selection.py` +* :ref:`sphx_glr_auto_examples_decomposition_plot_pca_iris.py` +* :ref:`sphx_glr_auto_examples_decomposition_plot_pca_vs_lda.py` +* :ref:`sphx_glr_auto_examples_decomposition_plot_pca_vs_fa_model_selection.py` .. _IncrementalPCA: @@ -71,14 +72,14 @@ exactly match the results of :class:`PCA` while processing the data in a minibatch fashion. :class:`IncrementalPCA` makes it possible to implement out-of-core Principal Component Analysis either by: - * Using its ``partial_fit`` method on chunks of data fetched sequentially - from the local hard drive or a network database. +* Using its ``partial_fit`` method on chunks of data fetched sequentially + from the local hard drive or a network database. - * Calling its fit method on a sparse matrix or a memory mapped file using - ``numpy.memmap``. +* Calling its fit method on a memory mapped file using + ``numpy.memmap``. :class:`IncrementalPCA` only stores estimates of component and noise variances, -in order update ``explained_variance_ratio_`` incrementally. This is why +in order to update ``explained_variance_ratio_`` incrementally. This is why memory usage depends on the number of samples per batch, rather than the number of samples to be processed in the dataset. @@ -96,9 +97,9 @@ input data for each feature before applying the SVD. :scale: 75% -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_decomposition_plot_incremental_pca.py` +* :ref:`sphx_glr_auto_examples_decomposition_plot_incremental_pca.py` .. _RandomizedPCA: @@ -119,7 +120,7 @@ pictures of human faces look somewhat alike. The samples lie on a manifold of much lower dimension (say around 200 for instance). The PCA algorithm can be used to linearly transform the data while both reducing the dimensionality -and preserve most of the explained variance at the same time. +and preserving most of the explained variance at the same time. The class :class:`PCA` used with the optional parameter ``svd_solver='randomized'`` is very useful in that case: since we are going @@ -159,39 +160,20 @@ Note: the implementation of ``inverse_transform`` in :class:`PCA` with ``transform`` even when ``whiten=False`` (default). -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_applications_plot_face_recognition.py` - * :ref:`sphx_glr_auto_examples_decomposition_plot_faces_decomposition.py` +* :ref:`sphx_glr_auto_examples_applications_plot_face_recognition.py` +* :ref:`sphx_glr_auto_examples_decomposition_plot_faces_decomposition.py` -.. topic:: References: +.. rubric:: References - * `"Finding structure with randomness: Stochastic algorithms for - constructing approximate matrix decompositions" - `_ - Halko, et al., 2009 - - -.. _kernel_PCA: - -Kernel PCA ----------- - -:class:`KernelPCA` is an extension of PCA which achieves non-linear -dimensionality reduction through the use of kernels (see :ref:`metrics`). It -has many applications including denoising, compression and structured -prediction (kernel dependency estimation). :class:`KernelPCA` supports both -``transform`` and ``inverse_transform``. - -.. figure:: ../auto_examples/decomposition/images/sphx_glr_plot_kernel_pca_001.png - :target: ../auto_examples/decomposition/plot_kernel_pca.html - :align: center - :scale: 75% - -.. topic:: Examples: - - * :ref:`sphx_glr_auto_examples_decomposition_plot_kernel_pca.py` +* Algorithm 4.3 in + :arxiv:`"Finding structure with randomness: Stochastic algorithms for + constructing approximate matrix decompositions" <0909.4061>` + Halko, et al., 2009 +* :arxiv:`"An implementation of a randomized algorithm for principal component + analysis" <1412.3510>` A. Szlam et al. 2014 .. _SparsePCA: @@ -215,7 +197,7 @@ the real underlying components can be more naturally imagined as sparse vectors; for example in face recognition, components might naturally map to parts of faces. -Sparse principal components yields a more parsimonious, interpretable +Sparse principal components yield a more parsimonious, interpretable representation, clearly emphasizing which of the original features contribute to the differences between samples. @@ -246,12 +228,14 @@ problem solved is a PCA problem (dictionary learning) with an .. math:: (U^*, V^*) = \underset{U, V}{\operatorname{arg\,min\,}} & \frac{1}{2} - ||X-UV||_2^2+\alpha||V||_1 \\ - \text{subject to } & ||U_k||_2 = 1 \text{ for all } + ||X-UV||_{\text{Fro}}^2+\alpha||V||_{1,1} \\ + \text{subject to } & ||U_k||_2 \leq 1 \text{ for all } 0 \leq k < n_{components} - -The sparsity-inducing :math:`\ell_1` norm also prevents learning +:math:`||.||_{\text{Fro}}` stands for the Frobenius norm and :math:`||.||_{1,1}` +stands for the entry-wise matrix norm which is the sum of the absolute values +of all the entries in the matrix. +The sparsity-inducing :math:`||.||_{1,1}` matrix norm also prevents learning components from noise when few training samples are available. The degree of penalization (and thus sparsity) can be adjusted through the hyperparameter ``alpha``. Small values lead to a gently regularized @@ -264,98 +248,206 @@ factorization, while larger values shrink many coefficients to zero. the algorithm is online along the features direction, not the samples direction. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_decomposition_plot_faces_decomposition.py` +* :ref:`sphx_glr_auto_examples_decomposition_plot_faces_decomposition.py` -.. topic:: References: +.. rubric:: References - .. [Mrl09] `"Online Dictionary Learning for Sparse Coding" - `_ - J. Mairal, F. Bach, J. Ponce, G. Sapiro, 2009 - .. [Jen09] `"Structured Sparse Principal Component Analysis" - `_ - R. Jenatton, G. Obozinski, F. Bach, 2009 +.. [Mrl09] `"Online Dictionary Learning for Sparse Coding" + `_ + J. Mairal, F. Bach, J. Ponce, G. Sapiro, 2009 +.. [Jen09] `"Structured Sparse Principal Component Analysis" + `_ + R. Jenatton, G. Obozinski, F. Bach, 2009 -.. _LSA: +.. _kernel_PCA: -Truncated singular value decomposition and latent semantic analysis -=================================================================== +Kernel Principal Component Analysis (kPCA) +========================================== -:class:`TruncatedSVD` implements a variant of singular value decomposition -(SVD) that only computes the :math:`k` largest singular values, -where :math:`k` is a user-specified parameter. +Exact Kernel PCA +---------------- -When truncated SVD is applied to term-document matrices -(as returned by :class:`~sklearn.feature_extraction.text.CountVectorizer` or -:class:`~sklearn.feature_extraction.text.TfidfVectorizer`), -this transformation is known as -`latent semantic analysis `_ -(LSA), because it transforms such matrices -to a "semantic" space of low dimensionality. -In particular, LSA is known to combat the effects of synonymy and polysemy -(both of which roughly mean there are multiple meanings per word), -which cause term-document matrices to be overly sparse -and exhibit poor similarity under measures such as cosine similarity. +:class:`KernelPCA` is an extension of PCA which achieves non-linear +dimensionality reduction through the use of kernels (see :ref:`metrics`) [Scholkopf1997]_. It +has many applications including denoising, compression and structured +prediction (kernel dependency estimation). :class:`KernelPCA` supports both +``transform`` and ``inverse_transform``. + +.. figure:: ../auto_examples/decomposition/images/sphx_glr_plot_kernel_pca_002.png + :target: ../auto_examples/decomposition/plot_kernel_pca.html + :align: center + :scale: 75% .. note:: - LSA is also known as latent semantic indexing, LSI, - though strictly that refers to its use in persistent indexes - for information retrieval purposes. + :meth:`KernelPCA.inverse_transform` relies on a kernel ridge to learn the + function mapping samples from the PCA basis into the original feature + space [Bakir2003]_. Thus, the reconstruction obtained with + :meth:`KernelPCA.inverse_transform` is an approximation. See the example + linked below for more details. -Mathematically, truncated SVD applied to training samples :math:`X` -produces a low-rank approximation :math:`X`: +.. rubric:: Examples -.. math:: - X \approx X_k = U_k \Sigma_k V_k^\top +* :ref:`sphx_glr_auto_examples_decomposition_plot_kernel_pca.py` +* :ref:`sphx_glr_auto_examples_applications_plot_digits_denoising.py` -After this operation, :math:`U_k \Sigma_k^\top` -is the transformed training set with :math:`k` features -(called ``n_components`` in the API). +.. rubric:: References -To also transform a test set :math:`X`, we multiply it with :math:`V_k`: +.. [Scholkopf1997] SchÃļlkopf, Bernhard, Alexander Smola, and Klaus-Robert MÃŧller. + `"Kernel principal component analysis." + `_ + International conference on artificial neural networks. + Springer, Berlin, Heidelberg, 1997. -.. math:: - X' = X V_k +.. [Bakir2003] BakÄąr, GÃļkhan H., Jason Weston, and Bernhard SchÃļlkopf. + `"Learning to find pre-images." + `_ + Advances in neural information processing systems 16 (2003): 449-456. -.. note:: - Most treatments of LSA in the natural language processing (NLP) - and information retrieval (IR) literature - swap the axes of the matrix :math:`X` so that it has shape - ``n_features`` × ``n_samples``. - We present LSA in a different way that matches the scikit-learn API better, - but the singular values found are the same. +.. _kPCA_Solvers: + +Choice of solver for Kernel PCA +------------------------------- + +While in :class:`PCA` the number of components is bounded by the number of +features, in :class:`KernelPCA` the number of components is bounded by the +number of samples. Many real-world datasets have large number of samples! In +these cases finding *all* the components with a full kPCA is a waste of +computation time, as data is mostly described by the first few components +(e.g. ``n_components<=100``). In other words, the centered Gram matrix that +is eigendecomposed in the Kernel PCA fitting process has an effective rank that +is much smaller than its size. This is a situation where approximate +eigensolvers can provide speedup with very low precision loss. + + +.. dropdown:: Eigensolvers + + The optional parameter ``eigen_solver='randomized'`` can be used to + *significantly* reduce the computation time when the number of requested + ``n_components`` is small compared with the number of samples. It relies on + randomized decomposition methods to find an approximate solution in a shorter + time. + + The time complexity of the randomized :class:`KernelPCA` is + :math:`O(n_{\mathrm{samples}}^2 \cdot n_{\mathrm{components}})` + instead of :math:`O(n_{\mathrm{samples}}^3)` for the exact method + implemented with ``eigen_solver='dense'``. + + The memory footprint of randomized :class:`KernelPCA` is also proportional to + :math:`2 \cdot n_{\mathrm{samples}} \cdot n_{\mathrm{components}}` instead of + :math:`n_{\mathrm{samples}}^2` for the exact method. + + Note: this technique is the same as in :ref:`RandomizedPCA`. + + In addition to the above two solvers, ``eigen_solver='arpack'`` can be used as + an alternate way to get an approximate decomposition. In practice, this method + only provides reasonable execution times when the number of components to find + is extremely small. It is enabled by default when the desired number of + components is less than 10 (strict) and the number of samples is more than 200 + (strict). See :class:`KernelPCA` for details. + + .. rubric:: References + + * *dense* solver: + `scipy.linalg.eigh documentation + `_ + + * *randomized* solver: + + * Algorithm 4.3 in + :arxiv:`"Finding structure with randomness: Stochastic + algorithms for constructing approximate matrix decompositions" <0909.4061>` + Halko, et al. (2009) + + * :arxiv:`"An implementation of a randomized algorithm + for principal component analysis" <1412.3510>` + A. Szlam et al. (2014) + + * *arpack* solver: + `scipy.sparse.linalg.eigsh documentation + `_ + R. B. Lehoucq, D. C. Sorensen, and C. Yang, (1998) + + +.. _LSA: + +Truncated singular value decomposition and latent semantic analysis +=================================================================== + +:class:`TruncatedSVD` implements a variant of singular value decomposition +(SVD) that only computes the :math:`k` largest singular values, +where :math:`k` is a user-specified parameter. :class:`TruncatedSVD` is very similar to :class:`PCA`, but differs in that the matrix :math:`X` does not need to be centered. When the columnwise (per-feature) means of :math:`X` are subtracted from the feature values, truncated SVD on the resulting matrix is equivalent to PCA. -In practical terms, this means -that the :class:`TruncatedSVD` transformer accepts ``scipy.sparse`` -matrices without the need to densify them, -as densifying may fill up memory even for medium-sized document collections. -While the :class:`TruncatedSVD` transformer -works with any feature matrix, -using it on tf–idf matrices is recommended over raw frequency counts -in an LSA/document processing setting. -In particular, sublinear scaling and inverse document frequency -should be turned on (``sublinear_tf=True, use_idf=True``) -to bring the feature values closer to a Gaussian distribution, -compensating for LSA's erroneous assumptions about textual data. +.. dropdown:: About truncated SVD and latent semantic analysis (LSA) + + When truncated SVD is applied to term-document matrices + (as returned by :class:`~sklearn.feature_extraction.text.CountVectorizer` or + :class:`~sklearn.feature_extraction.text.TfidfVectorizer`), + this transformation is known as + `latent semantic analysis `_ + (LSA), because it transforms such matrices + to a "semantic" space of low dimensionality. + In particular, LSA is known to combat the effects of synonymy and polysemy + (both of which roughly mean there are multiple meanings per word), + which cause term-document matrices to be overly sparse + and exhibit poor similarity under measures such as cosine similarity. + + .. note:: + LSA is also known as latent semantic indexing, LSI, + though strictly that refers to its use in persistent indexes + for information retrieval purposes. + + Mathematically, truncated SVD applied to training samples :math:`X` + produces a low-rank approximation :math:`X`: + + .. math:: + X \approx X_k = U_k \Sigma_k V_k^\top -.. topic:: Examples: + After this operation, :math:`U_k \Sigma_k` + is the transformed training set with :math:`k` features + (called ``n_components`` in the API). - * :ref:`sphx_glr_auto_examples_text_plot_document_clustering.py` + To also transform a test set :math:`X`, we multiply it with :math:`V_k`: -.. topic:: References: + .. math:: + X' = X V_k + + .. note:: + Most treatments of LSA in the natural language processing (NLP) + and information retrieval (IR) literature + swap the axes of the matrix :math:`X` so that it has shape + ``(n_features, n_samples)``. + We present LSA in a different way that matches the scikit-learn API better, + but the singular values found are the same. + + While the :class:`TruncatedSVD` transformer + works with any feature matrix, + using it on tf-idf matrices is recommended over raw frequency counts + in an LSA/document processing setting. + In particular, sublinear scaling and inverse document frequency + should be turned on (``sublinear_tf=True, use_idf=True``) + to bring the feature values closer to a Gaussian distribution, + compensating for LSA's erroneous assumptions about textual data. + +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_text_plot_document_clustering.py` + +.. rubric:: References + +* Christopher D. Manning, Prabhakar Raghavan and Hinrich SchÃŧtze (2008), + *Introduction to Information Retrieval*, Cambridge University Press, + chapter 18: `Matrix decompositions & latent semantic indexing + `_ - * Christopher D. Manning, Prabhakar Raghavan and Hinrich SchÃŧtze (2008), - *Introduction to Information Retrieval*, Cambridge University Press, - chapter 18: `Matrix decompositions & latent semantic indexing - `_ .. _DictionaryLearning: @@ -408,9 +500,9 @@ the split code is filled with the negative part of the code vector, only with a positive sign. Therefore, the split_code is non-negative. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_decomposition_plot_sparse_coding.py` +* :ref:`sphx_glr_auto_examples_decomposition_plot_sparse_coding.py` Generic dictionary learning @@ -432,8 +524,8 @@ dictionary fixed, and then updating the dictionary to best fit the sparse code. .. math:: (U^*, V^*) = \underset{U, V}{\operatorname{arg\,min\,}} & \frac{1}{2} - ||X-UV||_2^2+\alpha||U||_1 \\ - \text{subject to } & ||V_k||_2 = 1 \text{ for all } + ||X-UV||_{\text{Fro}}^2+\alpha||U||_{1,1} \\ + \text{subject to } & ||V_k||_2 \leq 1 \text{ for all } 0 \leq k < n_{\mathrm{atoms}} @@ -441,13 +533,15 @@ dictionary fixed, and then updating the dictionary to best fit the sparse code. :target: ../auto_examples/decomposition/plot_faces_decomposition.html :scale: 60% -.. |dict_img2| image:: ../auto_examples/decomposition/images/sphx_glr_plot_faces_decomposition_006.png +.. |dict_img2| image:: ../auto_examples/decomposition/images/sphx_glr_plot_faces_decomposition_007.png :target: ../auto_examples/decomposition/plot_faces_decomposition.html :scale: 60% .. centered:: |pca_img2| |dict_img2| - +:math:`||.||_{\text{Fro}}` stands for the Frobenius norm and :math:`||.||_{1,1}` +stands for the entry-wise matrix norm which is the sum of the absolute values +of all the entries in the matrix. After using such a procedure to fit the dictionary, the transform is simply a sparse coding step that shares the same implementation with all dictionary learning objects (see :ref:`SparseCoder`). @@ -458,19 +552,19 @@ different positivity constraints applied. Red indicates negative values, blue indicates positive values, and white represents zeros. -.. |dict_img_pos1| image:: ../auto_examples/decomposition/images/sphx_glr_plot_faces_decomposition_011.png +.. |dict_img_pos1| image:: ../auto_examples/decomposition/images/sphx_glr_plot_faces_decomposition_010.png :target: ../auto_examples/decomposition/plot_image_denoising.html :scale: 60% -.. |dict_img_pos2| image:: ../auto_examples/decomposition/images/sphx_glr_plot_faces_decomposition_012.png +.. |dict_img_pos2| image:: ../auto_examples/decomposition/images/sphx_glr_plot_faces_decomposition_011.png :target: ../auto_examples/decomposition/plot_image_denoising.html :scale: 60% -.. |dict_img_pos3| image:: ../auto_examples/decomposition/images/sphx_glr_plot_faces_decomposition_013.png +.. |dict_img_pos3| image:: ../auto_examples/decomposition/images/sphx_glr_plot_faces_decomposition_012.png :target: ../auto_examples/decomposition/plot_image_denoising.html :scale: 60% -.. |dict_img_pos4| image:: ../auto_examples/decomposition/images/sphx_glr_plot_faces_decomposition_014.png +.. |dict_img_pos4| image:: ../auto_examples/decomposition/images/sphx_glr_plot_faces_decomposition_013.png :target: ../auto_examples/decomposition/plot_image_denoising.html :scale: 60% @@ -488,16 +582,16 @@ extracted from part of the image of a raccoon face looks like. :scale: 50% -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_decomposition_plot_image_denoising.py` +* :ref:`sphx_glr_auto_examples_decomposition_plot_image_denoising.py` -.. topic:: References: +.. rubric:: References - * `"Online dictionary learning for sparse coding" - `_ - J. Mairal, F. Bach, J. Ponce, G. Sapiro, 2009 +* `"Online dictionary learning for sparse coding" + `_ + J. Mairal, F. Bach, J. Ponce, G. Sapiro, 2009 .. _MiniBatchDictionaryLearning: @@ -516,7 +610,7 @@ implement a stopping condition. The estimator also implements ``partial_fit``, which updates the dictionary by iterating only once over a mini-batch. This can be used for online learning when the data is not readily available from the start, or for when the data -does not fit into the memory. +does not fit into memory. .. currentmodule:: sklearn.cluster @@ -533,7 +627,7 @@ does not fit into the memory. computationally efficient and implements on-line learning with a ``partial_fit`` method. - Example: :ref:`sphx_glr_auto_examples_cluster_plot_dict_face_patches.py` + Example: :ref:`sphx_glr_auto_examples_cluster_plot_dict_face_patches.py` .. currentmodule:: sklearn.decomposition @@ -599,7 +693,7 @@ about these components (e.g. whether they are orthogonal): :target: ../auto_examples/decomposition/plot_faces_decomposition.html :scale: 60% -.. |fa_img3| image:: ../auto_examples/decomposition/images/sphx_glr_plot_faces_decomposition_009.png +.. |fa_img3| image:: ../auto_examples/decomposition/images/sphx_glr_plot_faces_decomposition_008.png :target: ../auto_examples/decomposition/plot_faces_decomposition.html :scale: 60% @@ -609,7 +703,7 @@ The main advantage for Factor Analysis over :class:`PCA` is that it can model the variance in every direction of the input space independently (heteroscedastic noise): -.. figure:: ../auto_examples/decomposition/images/sphx_glr_plot_faces_decomposition_008.png +.. figure:: ../auto_examples/decomposition/images/sphx_glr_plot_faces_decomposition_009.png :target: ../auto_examples/decomposition/plot_faces_decomposition.html :align: center :scale: 75% @@ -628,10 +722,10 @@ Varimax rotation maximizes the sum of the variances of the squared loadings, i.e., it tends to produce sparser factors, which are influenced by only a few features each (the "simple structure"). See e.g., the first example below. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_decomposition_plot_varimax_fa.py` - * :ref:`sphx_glr_auto_examples_decomposition_plot_pca_vs_fa_model_selection.py` +* :ref:`sphx_glr_auto_examples_decomposition_plot_varimax_fa.py` +* :ref:`sphx_glr_auto_examples_decomposition_plot_pca_vs_fa_model_selection.py` .. _ICA: @@ -645,7 +739,7 @@ implemented in scikit-learn using the :class:`Fast ICA ` algorithm. Typically, ICA is not used for reducing dimensionality but for separating superimposed signals. Since the ICA model does not include a noise term, for the model to be correct, whitening must be applied. -This can be done internally using the whiten argument or manually using one +This can be done internally using the `whiten` argument or manually using one of the PCA variants. It is classically used to separate mixed signals (a problem known as @@ -670,11 +764,11 @@ components with some sparsity: .. centered:: |pca_img4| |ica_img4| -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_decomposition_plot_ica_blind_source_separation.py` - * :ref:`sphx_glr_auto_examples_decomposition_plot_ica_vs_pca.py` - * :ref:`sphx_glr_auto_examples_decomposition_plot_faces_decomposition.py` +* :ref:`sphx_glr_auto_examples_decomposition_plot_ica_blind_source_separation.py` +* :ref:`sphx_glr_auto_examples_decomposition_plot_ica_vs_pca.py` +* :ref:`sphx_glr_auto_examples_decomposition_plot_faces_decomposition.py` .. _NMF: @@ -718,7 +812,7 @@ faces dataset, in comparison with the PCA eigenfaces. .. centered:: |pca_img5| |nmf_img5| -The :attr:`init` attribute determines the initialization method applied, which +The `init` attribute determines the initialization method applied, which has a great impact on the performance of the method. :class:`NMF` implements the method Nonnegative Double Singular Value Decomposition. NNDSVD [4]_ is based on two SVD processes, one approximating the data matrix, the other approximating @@ -735,33 +829,31 @@ basic NNDSVD algorithm which introduces a lot of zeros; in this case, NNDSVDa or NNDSVDar should be preferred. :class:`NMF` can also be initialized with correctly scaled random non-negative -matrices by setting :attr:`init="random"`. An integer seed or a -``RandomState`` can also be passed to :attr:`random_state` to control +matrices by setting `init="random"`. An integer seed or a +``RandomState`` can also be passed to `random_state` to control reproducibility. -In :class:`NMF`, L1 and L2 priors can be added to the loss function in order -to regularize the model. The L2 prior uses the Frobenius norm, while the L1 -prior uses an elementwise L1 norm. As in :class:`ElasticNet`, we control the -combination of L1 and L2 with the :attr:`l1_ratio` (:math:`\rho`) parameter, -and the intensity of the regularization with the :attr:`alpha` -(:math:`\alpha`) parameter. Then the priors terms are: +In :class:`NMF`, L1 and L2 priors can be added to the loss function in order to +regularize the model. The L2 prior uses the Frobenius norm, while the L1 prior +uses an elementwise L1 norm. As in :class:`~sklearn.linear_model.ElasticNet`, +we control the combination of L1 and L2 with the `l1_ratio` (:math:`\rho`) +parameter, and the intensity of the regularization with the `alpha_W` and +`alpha_H` (:math:`\alpha_W` and :math:`\alpha_H`) parameters. The priors are +scaled by the number of samples (:math:`n\_samples`) for `H` and the number of +features (:math:`n\_features`) for `W` to keep their impact balanced with +respect to one another and to the data fit term as independent as possible of +the size of the training set. Then the priors terms are: .. math:: - \alpha \rho ||W||_1 + \alpha \rho ||H||_1 - + \frac{\alpha(1-\rho)}{2} ||W||_{\mathrm{Fro}} ^ 2 - + \frac{\alpha(1-\rho)}{2} ||H||_{\mathrm{Fro}} ^ 2 + (\alpha_W \rho ||W||_1 + \frac{\alpha_W(1-\rho)}{2} ||W||_{\mathrm{Fro}} ^ 2) * n\_features + + (\alpha_H \rho ||H||_1 + \frac{\alpha_H(1-\rho)}{2} ||H||_{\mathrm{Fro}} ^ 2) * n\_samples and the regularized objective function is: .. math:: d_{\mathrm{Fro}}(X, WH) - + \alpha \rho ||W||_1 + \alpha \rho ||H||_1 - + \frac{\alpha(1-\rho)}{2} ||W||_{\mathrm{Fro}} ^ 2 - + \frac{\alpha(1-\rho)}{2} ||H||_{\mathrm{Fro}} ^ 2 - -:class:`NMF` regularizes both W and H by default. The :attr:`regularization` -parameter allows for finer control, with which only W, only H, -or both can be regularized. + + (\alpha_W \rho ||W||_1 + \frac{\alpha_W(1-\rho)}{2} ||W||_{\mathrm{Fro}} ^ 2) * n\_features + + (\alpha_H \rho ||H||_1 + \frac{\alpha_H(1-\rho)}{2} ||H||_{\mathrm{Fro}} ^ 2) * n\_samples NMF with a beta-divergence -------------------------- @@ -785,14 +877,13 @@ Or, the Itakura-Saito (IS) divergence: d_{IS}(X, Y) = \sum_{i,j} (\frac{X_{ij}}{Y_{ij}} - \log(\frac{X_{ij}}{Y_{ij}}) - 1) These three distances are special cases of the beta-divergence family, with -:math:`\beta = 2, 1, 0` respectively [6]_. The beta-divergence are +:math:`\beta = 2, 1, 0` respectively [6]_. The beta-divergence is defined by : .. math:: d_{\beta}(X, Y) = \sum_{i,j} \frac{1}{\beta(\beta - 1)}(X_{ij}^\beta + (\beta-1)Y_{ij}^\beta - \beta X_{ij} Y_{ij}^{\beta - 1}) -.. figure:: ../auto_examples/decomposition/images/sphx_glr_plot_beta_divergence_001.png - :target: ../auto_examples/decomposition/plot_beta_divergence.html +.. image:: ../images/beta_divergence.png :align: center :scale: 75% @@ -800,18 +891,20 @@ Note that this definition is not valid if :math:`\beta \in (0; 1)`, yet it can be continuously extended to the definitions of :math:`d_{KL}` and :math:`d_{IS}` respectively. -:class:`NMF` implements two solvers, using Coordinate Descent ('cd') [5]_, and -Multiplicative Update ('mu') [6]_. The 'mu' solver can optimize every -beta-divergence, including of course the Frobenius norm (:math:`\beta=2`), the -(generalized) Kullback-Leibler divergence (:math:`\beta=1`) and the -Itakura-Saito divergence (:math:`\beta=0`). Note that for -:math:`\beta \in (1; 2)`, the 'mu' solver is significantly faster than for other -values of :math:`\beta`. Note also that with a negative (or 0, i.e. -'itakura-saito') :math:`\beta`, the input matrix cannot contain zero values. +.. dropdown:: NMF implemented solvers + + :class:`NMF` implements two solvers, using Coordinate Descent ('cd') [5]_, and + Multiplicative Update ('mu') [6]_. The 'mu' solver can optimize every + beta-divergence, including of course the Frobenius norm (:math:`\beta=2`), the + (generalized) Kullback-Leibler divergence (:math:`\beta=1`) and the + Itakura-Saito divergence (:math:`\beta=0`). Note that for + :math:`\beta \in (1; 2)`, the 'mu' solver is significantly faster than for other + values of :math:`\beta`. Note also that with a negative (or 0, i.e. + 'itakura-saito') :math:`\beta`, the input matrix cannot contain zero values. -The 'cd' solver can only optimize the Frobenius norm. Due to the -underlying non-convexity of NMF, the different solvers may converge to -different minima, even when optimizing the same distance function. + The 'cd' solver can only optimize the Frobenius norm. Due to the + underlying non-convexity of NMF, the different solvers may converge to + different minima, even when optimizing the same distance function. NMF is best used with the ``fit_transform`` method, which returns the matrix W. The matrix H is stored into the fitted model in the ``components_`` attribute; @@ -827,36 +920,63 @@ stored components:: >>> X_new = np.array([[1, 0], [1, 6.1], [1, 0], [1, 4], [3.2, 1], [0, 4]]) >>> W_new = model.transform(X_new) -.. topic:: Examples: - * :ref:`sphx_glr_auto_examples_decomposition_plot_faces_decomposition.py` - * :ref:`sphx_glr_auto_examples_applications_plot_topics_extraction_with_nmf_lda.py` - * :ref:`sphx_glr_auto_examples_decomposition_plot_beta_divergence.py` -.. topic:: References: +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_decomposition_plot_faces_decomposition.py` +* :ref:`sphx_glr_auto_examples_applications_plot_topics_extraction_with_nmf_lda.py` + +.. _MiniBatchNMF: - .. [1] `"Learning the parts of objects by non-negative matrix factorization" - `_ - D. Lee, S. Seung, 1999 +Mini-batch Non Negative Matrix Factorization +-------------------------------------------- - .. [2] `"Non-negative Matrix Factorization with Sparseness Constraints" - `_ - P. Hoyer, 2004 +:class:`MiniBatchNMF` [7]_ implements a faster, but less accurate version of the +non negative matrix factorization (i.e. :class:`~sklearn.decomposition.NMF`), +better suited for large datasets. - .. [4] `"SVD based initialization: A head start for nonnegative - matrix factorization" - `_ - C. Boutsidis, E. Gallopoulos, 2008 +By default, :class:`MiniBatchNMF` divides the data into mini-batches and +optimizes the NMF model in an online manner by cycling over the mini-batches +for the specified number of iterations. The ``batch_size`` parameter controls +the size of the batches. - .. [5] `"Fast local algorithms for large scale nonnegative matrix and tensor - factorizations." - `_ - A. Cichocki, A. Phan, 2009 +In order to speed up the mini-batch algorithm it is also possible to scale +past batches, giving them less importance than newer batches. This is done +by introducing a so-called forgetting factor controlled by the ``forget_factor`` +parameter. - .. [6] `"Algorithms for nonnegative matrix factorization with the beta-divergence" - `_ - C. Fevotte, J. Idier, 2011 +The estimator also implements ``partial_fit``, which updates ``H`` by iterating +only once over a mini-batch. This can be used for online learning when the data +is not readily available from the start, or when the data does not fit into memory. +.. rubric:: References + +.. [1] `"Learning the parts of objects by non-negative matrix factorization" + `_ + D. Lee, S. Seung, 1999 + +.. [2] `"Non-negative Matrix Factorization with Sparseness Constraints" + `_ + P. Hoyer, 2004 + +.. [4] `"SVD based initialization: A head start for nonnegative + matrix factorization" + `_ + C. Boutsidis, E. Gallopoulos, 2008 + +.. [5] `"Fast local algorithms for large scale nonnegative matrix and tensor + factorizations." + `_ + A. Cichocki, A. Phan, 2009 + +.. [6] :arxiv:`"Algorithms for nonnegative matrix factorization with + the beta-divergence" <1010.1763>` + C. Fevotte, J. Idier, 2011 + +.. [7] :arxiv:`"Online algorithms for nonnegative matrix factorization with the + Itakura-Saito divergence" <1106.4198>` + A. Lefevre, F. Bach, C. Fevotte, 2011 .. _LatentDirichletAllocation: @@ -864,7 +984,7 @@ Latent Dirichlet Allocation (LDA) ================================= Latent Dirichlet Allocation is a generative probabilistic model for collections of -discrete dataset such as text corpora. It is also a topic model that is used for +discrete datasets such as text corpora. It is also a topic model that is used for discovering abstract topics from a collection of documents. The graphical model of LDA is a three-level generative model: @@ -872,66 +992,69 @@ The graphical model of LDA is a three-level generative model: .. image:: ../images/lda_model_graph.png :align: center -Note on notations presented in the graphical model above, which can be found in +Note on notations presented in the graphical model above, which can be found in Hoffman et al. (2013): - * The corpus is a collection of :math:`D` documents. - * A document is a sequence of :math:`N` words. - * There are :math:`K` topics in the corpus. - * The boxes represent repeated sampling. +* The corpus is a collection of :math:`D` documents. +* A document is a sequence of :math:`N` words. +* There are :math:`K` topics in the corpus. +* The boxes represent repeated sampling. -In the graphical model, each node is a random variable and has a role in the -generative process. A shaded node indicates an observed variable and an unshaded -node indicates a hidden (latent) variable. In this case, words in the corpus are -the only data that we observe. The latent variables determine the random mixture -of topics in the corpus and the distribution of words in the documents. -The goal of LDA is to use the observed words to infer the hidden topic -structure. +In the graphical model, each node is a random variable and has a role in the +generative process. A shaded node indicates an observed variable and an unshaded +node indicates a hidden (latent) variable. In this case, words in the corpus are +the only data that we observe. The latent variables determine the random mixture +of topics in the corpus and the distribution of words in the documents. +The goal of LDA is to use the observed words to infer the hidden topic +structure. -When modeling text corpora, the model assumes the following generative process -for a corpus with :math:`D` documents and :math:`K` topics, with :math:`K` -corresponding to :attr:`n_components` in the API: +.. dropdown:: Details on modeling text corpora - 1. For each topic :math:`k \in K`, draw :math:`\beta_k \sim - \mathrm{Dirichlet}(\eta)`. This provides a distribution over the words, - i.e. the probability of a word appearing in topic :math:`k`. - :math:`\eta` corresponds to :attr:`topic_word_prior`. + When modeling text corpora, the model assumes the following generative process + for a corpus with :math:`D` documents and :math:`K` topics, with :math:`K` + corresponding to `n_components` in the API: - 2. For each document :math:`d \in D`, draw the topic proportions - :math:`\theta_d \sim \mathrm{Dirichlet}(\alpha)`. :math:`\alpha` - corresponds to :attr:`doc_topic_prior`. + 1. For each topic :math:`k \in K`, draw :math:`\beta_k \sim + \mathrm{Dirichlet}(\eta)`. This provides a distribution over the words, + i.e. the probability of a word appearing in topic :math:`k`. + :math:`\eta` corresponds to `topic_word_prior`. - 3. For each word :math:`i` in document :math:`d`: + 2. For each document :math:`d \in D`, draw the topic proportions + :math:`\theta_d \sim \mathrm{Dirichlet}(\alpha)`. :math:`\alpha` + corresponds to `doc_topic_prior`. - a. Draw the topic assignment :math:`z_{di} \sim \mathrm{Multinomial} - (\theta_d)` - b. Draw the observed word :math:`w_{ij} \sim \mathrm{Multinomial} - (\beta_{z_{di}})` + 3. For each word :math:`i` in document :math:`d`: -For parameter estimation, the posterior distribution is: + a. Draw the topic assignment :math:`z_{di} \sim \mathrm{Multinomial} + (\theta_d)` + b. Draw the observed word :math:`w_{ij} \sim \mathrm{Multinomial} + (\beta_{z_{di}})` -.. math:: - p(z, \theta, \beta |w, \alpha, \eta) = - \frac{p(z, \theta, \beta|\alpha, \eta)}{p(w|\alpha, \eta)} + For parameter estimation, the posterior distribution is: -Since the posterior is intractable, variational Bayesian method -uses a simpler distribution :math:`q(z,\theta,\beta | \lambda, \phi, \gamma)` -to approximate it, and those variational parameters :math:`\lambda`, -:math:`\phi`, :math:`\gamma` are optimized to maximize the Evidence -Lower Bound (ELBO): + .. math:: + p(z, \theta, \beta |w, \alpha, \eta) = + \frac{p(z, \theta, \beta|\alpha, \eta)}{p(w|\alpha, \eta)} -.. math:: - \log\: P(w | \alpha, \eta) \geq L(w,\phi,\gamma,\lambda) \overset{\triangle}{=} - E_{q}[\log\:p(w,z,\theta,\beta|\alpha,\eta)] - E_{q}[\log\:q(z, \theta, \beta)] + Since the posterior is intractable, variational Bayesian method + uses a simpler distribution :math:`q(z,\theta,\beta | \lambda, \phi, \gamma)` + to approximate it, and those variational parameters :math:`\lambda`, + :math:`\phi`, :math:`\gamma` are optimized to maximize the Evidence + Lower Bound (ELBO): + + .. math:: + \log\: P(w | \alpha, \eta) \geq L(w,\phi,\gamma,\lambda) \overset{\triangle}{=} + E_{q}[\log\:p(w,z,\theta,\beta|\alpha,\eta)] - E_{q}[\log\:q(z, \theta, \beta)] + + Maximizing ELBO is equivalent to minimizing the Kullback-Leibler(KL) divergence + between :math:`q(z,\theta,\beta)` and the true posterior + :math:`p(z, \theta, \beta |w, \alpha, \eta)`. -Maximizing ELBO is equivalent to minimizing the Kullback-Leibler(KL) divergence -between :math:`q(z,\theta,\beta)` and the true posterior -:math:`p(z, \theta, \beta |w, \alpha, \eta)`. -:class:`LatentDirichletAllocation` implements the online variational Bayes +:class:`LatentDirichletAllocation` implements the online variational Bayes algorithm and supports both online and batch update methods. -While the batch method updates variational variables after each full pass through -the data, the online method updates variational variables from mini-batch data +While the batch method updates variational variables after each full pass through +the data, the online method updates variational variables from mini-batch data points. .. note:: @@ -942,33 +1065,33 @@ points. When :class:`LatentDirichletAllocation` is applied on a "document-term" matrix, the matrix will be decomposed into a "topic-term" matrix and a "document-topic" matrix. While -"topic-term" matrix is stored as :attr:`components_` in the model, "document-topic" matrix +"topic-term" matrix is stored as `components_` in the model, "document-topic" matrix can be calculated from ``transform`` method. :class:`LatentDirichletAllocation` also implements ``partial_fit`` method. This is used when data can be fetched sequentially. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_applications_plot_topics_extraction_with_nmf_lda.py` +* :ref:`sphx_glr_auto_examples_applications_plot_topics_extraction_with_nmf_lda.py` -.. topic:: References: +.. rubric:: References - * `"Latent Dirichlet Allocation" - `_ - D. Blei, A. Ng, M. Jordan, 2003 +* `"Latent Dirichlet Allocation" + `_ + D. Blei, A. Ng, M. Jordan, 2003 - * `"Online Learning for Latent Dirichlet Allocation” - `_ - M. Hoffman, D. Blei, F. Bach, 2010 +* `"Online Learning for Latent Dirichlet Allocation” + `_ + M. Hoffman, D. Blei, F. Bach, 2010 - * `"Stochastic Variational Inference" - `_ - M. Hoffman, D. Blei, C. Wang, J. Paisley, 2013 +* `"Stochastic Variational Inference" + `_ + M. Hoffman, D. Blei, C. Wang, J. Paisley, 2013 - * `"The varimax criterion for analytic rotation in factor analysis" - `_ - H. F. Kaiser, 1958 +* `"The varimax criterion for analytic rotation in factor analysis" + `_ + H. F. Kaiser, 1958 See also :ref:`nca_dim_reduction` for dimensionality reduction with Neighborhood Components Analysis. diff --git a/doc/modules/density.rst b/doc/modules/density.rst index 115d318183577..16c73bd5349a2 100644 --- a/doc/modules/density.rst +++ b/doc/modules/density.rst @@ -100,6 +100,10 @@ between bias and variance in the result. A large bandwidth leads to a very smooth (i.e. high-bias) density distribution. A small bandwidth leads to an unsmooth (i.e. high-variance) density distribution. +The parameter `bandwidth` controls this smoothing. One can either set +manually this parameter or use Scott's and Silverman's estimation +methods. + :class:`~sklearn.neighbors.KernelDensity` implements several common kernel forms, which are shown in the following figure: @@ -109,36 +113,39 @@ forms, which are shown in the following figure: .. centered:: |kde_kernels| -The form of these kernels is as follows: +.. dropdown:: Kernels' mathematical expressions + + The form of these kernels is as follows: + + * Gaussian kernel (``kernel = 'gaussian'``) -* Gaussian kernel (``kernel = 'gaussian'``) + :math:`K(x; h) \propto \exp(- \frac{x^2}{2h^2} )` - :math:`K(x; h) \propto \exp(- \frac{x^2}{2h^2} )` + * Tophat kernel (``kernel = 'tophat'``) -* Tophat kernel (``kernel = 'tophat'``) + :math:`K(x; h) \propto 1` if :math:`x < h` - :math:`K(x; h) \propto 1` if :math:`x < h` + * Epanechnikov kernel (``kernel = 'epanechnikov'``) -* Epanechnikov kernel (``kernel = 'epanechnikov'``) + :math:`K(x; h) \propto 1 - \frac{x^2}{h^2}` - :math:`K(x; h) \propto 1 - \frac{x^2}{h^2}` + * Exponential kernel (``kernel = 'exponential'``) -* Exponential kernel (``kernel = 'exponential'``) + :math:`K(x; h) \propto \exp(-x/h)` - :math:`K(x; h) \propto \exp(-x/h)` + * Linear kernel (``kernel = 'linear'``) -* Linear kernel (``kernel = 'linear'``) + :math:`K(x; h) \propto 1 - x/h` if :math:`x < h` - :math:`K(x; h) \propto 1 - x/h` if :math:`x < h` + * Cosine kernel (``kernel = 'cosine'``) -* Cosine kernel (``kernel = 'cosine'``) + :math:`K(x; h) \propto \cos(\frac{\pi x}{2h})` if :math:`x < h` - :math:`K(x; h) \propto \cos(\frac{\pi x}{2h})` if :math:`x < h` The kernel density estimator can be used with any of the valid distance -metrics (see :class:`~sklearn.neighbors.DistanceMetric` for a list of available metrics), though -the results are properly normalized only for the Euclidean metric. One -particularly useful metric is the +metrics (see :class:`~sklearn.metrics.DistanceMetric` for a list of +available metrics), though the results are properly normalized only +for the Euclidean metric. One particularly useful metric is the `Haversine distance `_ which measures the angular distance between points on a sphere. Here is an example of using a kernel density estimate for a visualization @@ -167,14 +174,14 @@ on a PCA projection of the data: The "new" data consists of linear combinations of the input data, with weights probabilistically drawn given the KDE model. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_neighbors_plot_kde_1d.py`: computation of simple kernel - density estimates in one dimension. +* :ref:`sphx_glr_auto_examples_neighbors_plot_kde_1d.py`: computation of simple kernel + density estimates in one dimension. - * :ref:`sphx_glr_auto_examples_neighbors_plot_digits_kde_sampling.py`: an example of using - Kernel Density estimation to learn a generative model of the hand-written - digits data, and drawing new samples from this model. +* :ref:`sphx_glr_auto_examples_neighbors_plot_digits_kde_sampling.py`: an example of using + Kernel Density estimation to learn a generative model of the hand-written + digits data, and drawing new samples from this model. - * :ref:`sphx_glr_auto_examples_neighbors_plot_species_kde.py`: an example of Kernel Density - estimation using the Haversine distance metric to visualize geospatial data +* :ref:`sphx_glr_auto_examples_neighbors_plot_species_kde.py`: an example of Kernel Density + estimation using the Haversine distance metric to visualize geospatial data diff --git a/doc/modules/ensemble.rst b/doc/modules/ensemble.rst index 0e0aaaafaffba..31ca150df372e 100644 --- a/doc/modules/ensemble.rst +++ b/doc/modules/ensemble.rst @@ -1,584 +1,592 @@ .. _ensemble: -================ -Ensemble methods -================ +=========================================================================== +Ensembles: Gradient boosting, random forests, bagging, voting, stacking +=========================================================================== .. currentmodule:: sklearn.ensemble -The goal of **ensemble methods** is to combine the predictions of several +**Ensemble methods** combine the predictions of several base estimators built with a given learning algorithm in order to improve generalizability / robustness over a single estimator. -Two families of ensemble methods are usually distinguished: +Two very famous examples of ensemble methods are :ref:`gradient-boosted trees +` and :ref:`random forests `. -- In **averaging methods**, the driving principle is to build several - estimators independently and then to average their predictions. On average, - the combined estimator is usually better than any of the single base - estimator because its variance is reduced. +More generally, ensemble models can be applied to any base learner beyond +trees, in averaging methods such as :ref:`Bagging methods `, +:ref:`model stacking `, or :ref:`Voting `, or in +boosting, as :ref:`AdaBoost `. - **Examples:** :ref:`Bagging methods `, :ref:`Forests of randomized trees `, ... +.. _gradient_boosting: -- By contrast, in **boosting methods**, base estimators are built sequentially - and one tries to reduce the bias of the combined estimator. The motivation is - to combine several weak models to produce a powerful ensemble. +Gradient-boosted trees +====================== - **Examples:** :ref:`AdaBoost `, :ref:`Gradient Tree Boosting `, ... +`Gradient Tree Boosting `_ +or Gradient Boosted Decision Trees (GBDT) is a generalization +of boosting to arbitrary differentiable loss functions, see the seminal work of +[Friedman2001]_. GBDT is an excellent model for both regression and +classification, in particular for tabular data. + +.. topic:: :class:`GradientBoostingClassifier` vs :class:`HistGradientBoostingClassifier` + + Scikit-learn provides two implementations of gradient-boosted trees: + :class:`HistGradientBoostingClassifier` vs + :class:`GradientBoostingClassifier` for classification, and the + corresponding classes for regression. The former can be **orders of + magnitude faster** than the latter when the number of samples is + larger than tens of thousands of samples. + + Missing values and categorical data are natively supported by the + Hist... version, removing the need for additional preprocessing such as + imputation. + + :class:`GradientBoostingClassifier` and + :class:`GradientBoostingRegressor` might be preferred for small sample + sizes since binning may lead to split points that are too approximate + in this setting. +.. _histogram_based_gradient_boosting: -.. _bagging: +Histogram-Based Gradient Boosting +---------------------------------- -Bagging meta-estimator -====================== +Scikit-learn 0.21 introduced two new implementations of +gradient boosted trees, namely :class:`HistGradientBoostingClassifier` +and :class:`HistGradientBoostingRegressor`, inspired by +`LightGBM `__ (See [LightGBM]_). -In ensemble algorithms, bagging methods form a class of algorithms which build -several instances of a black-box estimator on random subsets of the original -training set and then aggregate their individual predictions to form a final -prediction. These methods are used as a way to reduce the variance of a base -estimator (e.g., a decision tree), by introducing randomization into its -construction procedure and then making an ensemble out of it. In many cases, -bagging methods constitute a very simple way to improve with respect to a -single model, without making it necessary to adapt the underlying base -algorithm. As they provide a way to reduce overfitting, bagging methods work -best with strong and complex models (e.g., fully developed decision trees), in -contrast with boosting methods which usually work best with weak models (e.g., -shallow decision trees). +These histogram-based estimators can be **orders of magnitude faster** +than :class:`GradientBoostingClassifier` and +:class:`GradientBoostingRegressor` when the number of samples is larger +than tens of thousands of samples. -Bagging methods come in many flavours but mostly differ from each other by the -way they draw random subsets of the training set: +They also have built-in support for missing values, which avoids the need +for an imputer. - * When random subsets of the dataset are drawn as random subsets of the - samples, then this algorithm is known as Pasting [B1999]_. +These fast estimators first bin the input samples ``X`` into +integer-valued bins (typically 256 bins) which tremendously reduces the +number of splitting points to consider, and allows the algorithm to +leverage integer-based data structures (histograms) instead of relying on +sorted continuous values when building the trees. The API of these +estimators is slightly different, and some of the features from +:class:`GradientBoostingClassifier` and :class:`GradientBoostingRegressor` +are not yet supported, for instance some loss functions. - * When samples are drawn with replacement, then the method is known as - Bagging [B1996]_. +.. rubric:: Examples - * When random subsets of the dataset are drawn as random subsets of - the features, then the method is known as Random Subspaces [H1998]_. +* :ref:`sphx_glr_auto_examples_inspection_plot_partial_dependence.py` +* :ref:`sphx_glr_auto_examples_ensemble_plot_forest_hist_grad_boosting_comparison.py` - * Finally, when base estimators are built on subsets of both samples and - features, then the method is known as Random Patches [LG2012]_. +Usage +^^^^^ -In scikit-learn, bagging methods are offered as a unified -:class:`BaggingClassifier` meta-estimator (resp. :class:`BaggingRegressor`), -taking as input a user-specified base estimator along with parameters -specifying the strategy to draw random subsets. In particular, ``max_samples`` -and ``max_features`` control the size of the subsets (in terms of samples and -features), while ``bootstrap`` and ``bootstrap_features`` control whether -samples and features are drawn with or without replacement. When using a subset -of the available samples the generalization accuracy can be estimated with the -out-of-bag samples by setting ``oob_score=True``. As an example, the -snippet below illustrates how to instantiate a bagging ensemble of -:class:`KNeighborsClassifier` base estimators, each built on random subsets of -50% of the samples and 50% of the features. +Most of the parameters are unchanged from +:class:`GradientBoostingClassifier` and :class:`GradientBoostingRegressor`. +One exception is the ``max_iter`` parameter that replaces ``n_estimators``, and +controls the number of iterations of the boosting process:: - >>> from sklearn.ensemble import BaggingClassifier - >>> from sklearn.neighbors import KNeighborsClassifier - >>> bagging = BaggingClassifier(KNeighborsClassifier(), - ... max_samples=0.5, max_features=0.5) + >>> from sklearn.ensemble import HistGradientBoostingClassifier + >>> from sklearn.datasets import make_hastie_10_2 -.. topic:: Examples: + >>> X, y = make_hastie_10_2(random_state=0) + >>> X_train, X_test = X[:2000], X[2000:] + >>> y_train, y_test = y[:2000], y[2000:] - * :ref:`sphx_glr_auto_examples_ensemble_plot_bias_variance.py` + >>> clf = HistGradientBoostingClassifier(max_iter=100).fit(X_train, y_train) + >>> clf.score(X_test, y_test) + 0.8965 -.. topic:: References +Available losses for **regression** are: - .. [B1999] L. Breiman, "Pasting small votes for classification in large - databases and on-line", Machine Learning, 36(1), 85-103, 1999. +- 'squared_error', which is the default loss; +- 'absolute_error', which is less sensitive to outliers than the squared error; +- 'gamma', which is well suited to model strictly positive outcomes; +- 'poisson', which is well suited to model counts and frequencies; +- 'quantile', which allows for estimating a conditional quantile that can later + be used to obtain prediction intervals. - .. [B1996] L. Breiman, "Bagging predictors", Machine Learning, 24(2), - 123-140, 1996. +For **classification**, 'log_loss' is the only option. For binary classification +it uses the binary log loss, also known as binomial deviance or binary +cross-entropy. For `n_classes >= 3`, it uses the multi-class log loss function, +with multinomial deviance and categorical cross-entropy as alternative names. +The appropriate loss version is selected based on :term:`y` passed to +:term:`fit`. - .. [H1998] T. Ho, "The random subspace method for constructing decision - forests", Pattern Analysis and Machine Intelligence, 20(8), 832-844, - 1998. +The size of the trees can be controlled through the ``max_leaf_nodes``, +``max_depth``, and ``min_samples_leaf`` parameters. - .. [LG2012] G. Louppe and P. Geurts, "Ensembles on Random Patches", - Machine Learning and Knowledge Discovery in Databases, 346-361, 2012. +The number of bins used to bin the data is controlled with the ``max_bins`` +parameter. Using less bins acts as a form of regularization. It is generally +recommended to use as many bins as possible (255), which is the default. -.. _forest: +The ``l2_regularization`` parameter acts as a regularizer for the loss function, +and corresponds to :math:`\lambda` in the following expression (see equation (2) +in [XGBoost]_): -Forests of randomized trees -=========================== +.. math:: -The :mod:`sklearn.ensemble` module includes two averaging algorithms based -on randomized :ref:`decision trees `: the RandomForest algorithm -and the Extra-Trees method. Both algorithms are perturb-and-combine -techniques [B1998]_ specifically designed for trees. This means a diverse -set of classifiers is created by introducing randomness in the classifier -construction. The prediction of the ensemble is given as the averaged -prediction of the individual classifiers. + \mathcal{L}(\phi) = \sum_i l(\hat{y}_i, y_i) + \frac12 \sum_k \lambda ||w_k||^2 -As other classifiers, forest classifiers have to be fitted with two -arrays: a sparse or dense array X of shape ``(n_samples, n_features)`` -holding the training samples, and an array Y of shape ``(n_samples,)`` -holding the target values (class labels) for the training samples:: +.. dropdown:: Details on l2 regularization - >>> from sklearn.ensemble import RandomForestClassifier - >>> X = [[0, 0], [1, 1]] - >>> Y = [0, 1] - >>> clf = RandomForestClassifier(n_estimators=10) - >>> clf = clf.fit(X, Y) + It is important to notice that the loss term :math:`l(\hat{y}_i, y_i)` describes + only half of the actual loss function except for the pinball loss and absolute + error. -Like :ref:`decision trees `, forests of trees also extend to -:ref:`multi-output problems ` (if Y is an array -of shape ``(n_samples, n_outputs)``). + The index :math:`k` refers to the k-th tree in the ensemble of trees. In the + case of regression and binary classification, gradient boosting models grow one + tree per iteration, then :math:`k` runs up to `max_iter`. In the case of + multiclass classification problems, the maximal value of the index :math:`k` is + `n_classes` :math:`\times` `max_iter`. -Random Forests --------------- + If :math:`T_k` denotes the number of leaves in the k-th tree, then :math:`w_k` + is a vector of length :math:`T_k`, which contains the leaf values of the form `w + = -sum_gradient / (sum_hessian + l2_regularization)` (see equation (5) in + [XGBoost]_). -In random forests (see :class:`RandomForestClassifier` and -:class:`RandomForestRegressor` classes), each tree in the ensemble is built -from a sample drawn with replacement (i.e., a bootstrap sample) from the -training set. + The leaf values :math:`w_k` are derived by dividing the sum of the gradients of + the loss function by the combined sum of hessians. Adding the regularization to + the denominator penalizes the leaves with small hessians (flat regions), + resulting in smaller updates. Those :math:`w_k` values contribute then to the + model's prediction for a given input that ends up in the corresponding leaf. The + final prediction is the sum of the base prediction and the contributions from + each tree. The result of that sum is then transformed by the inverse link + function depending on the choice of the loss function (see + :ref:`gradient_boosting_formulation`). -Furthermore, when splitting each node during the construction of a tree, the -best split is found either from all input features or a random subset of size -``max_features``. (See the :ref:`parameter tuning guidelines -` for more details). + Notice that the original paper [XGBoost]_ introduces a term :math:`\gamma\sum_k + T_k` that penalizes the number of leaves (making it a smooth version of + `max_leaf_nodes`) not presented here as it is not implemented in scikit-learn; + whereas :math:`\lambda` penalizes the magnitude of the individual tree + predictions before being rescaled by the learning rate, see + :ref:`gradient_boosting_shrinkage`. -The purpose of these two sources of randomness is to decrease the variance of -the forest estimator. Indeed, individual decision trees typically exhibit high -variance and tend to overfit. The injected randomness in forests yield decision -trees with somewhat decoupled prediction errors. By taking an average of those -predictions, some errors can cancel out. Random forests achieve a reduced -variance by combining diverse trees, sometimes at the cost of a slight increase -in bias. In practice the variance reduction is often significant hence yielding -an overall better model. -In contrast to the original publication [B2001]_, the scikit-learn -implementation combines classifiers by averaging their probabilistic -prediction, instead of letting each classifier vote for a single class. +Note that **early-stopping is enabled by default if the number of samples is +larger than 10,000**. The early-stopping behaviour is controlled via the +``early_stopping``, ``scoring``, ``validation_fraction``, +``n_iter_no_change``, and ``tol`` parameters. It is possible to early-stop +using an arbitrary :term:`scorer`, or just the training or validation loss. +Note that for technical reasons, using a callable as a scorer is significantly slower +than using the loss. By default, early-stopping is performed if there are at least +10,000 samples in the training set, using the validation loss. -Extremely Randomized Trees --------------------------- +.. _nan_support_hgbt: -In extremely randomized trees (see :class:`ExtraTreesClassifier` -and :class:`ExtraTreesRegressor` classes), randomness goes one step -further in the way splits are computed. As in random forests, a random -subset of candidate features is used, but instead of looking for the -most discriminative thresholds, thresholds are drawn at random for each -candidate feature and the best of these randomly-generated thresholds is -picked as the splitting rule. This usually allows to reduce the variance -of the model a bit more, at the expense of a slightly greater increase -in bias:: +Missing values support +^^^^^^^^^^^^^^^^^^^^^^ - >>> from sklearn.model_selection import cross_val_score - >>> from sklearn.datasets import make_blobs - >>> from sklearn.ensemble import RandomForestClassifier - >>> from sklearn.ensemble import ExtraTreesClassifier - >>> from sklearn.tree import DecisionTreeClassifier +:class:`HistGradientBoostingClassifier` and +:class:`HistGradientBoostingRegressor` have built-in support for missing +values (NaNs). - >>> X, y = make_blobs(n_samples=10000, n_features=10, centers=100, - ... random_state=0) +During training, the tree grower learns at each split point whether samples +with missing values should go to the left or right child, based on the +potential gain. When predicting, samples with missing values are assigned to +the left or right child consequently:: - >>> clf = DecisionTreeClassifier(max_depth=None, min_samples_split=2, - ... random_state=0) - >>> scores = cross_val_score(clf, X, y, cv=5) - >>> scores.mean() - 0.98... + >>> from sklearn.ensemble import HistGradientBoostingClassifier + >>> import numpy as np - >>> clf = RandomForestClassifier(n_estimators=10, max_depth=None, - ... min_samples_split=2, random_state=0) - >>> scores = cross_val_score(clf, X, y, cv=5) - >>> scores.mean() - 0.999... + >>> X = np.array([0, 1, 2, np.nan]).reshape(-1, 1) + >>> y = [0, 0, 1, 1] - >>> clf = ExtraTreesClassifier(n_estimators=10, max_depth=None, - ... min_samples_split=2, random_state=0) - >>> scores = cross_val_score(clf, X, y, cv=5) - >>> scores.mean() > 0.999 - True + >>> gbdt = HistGradientBoostingClassifier(min_samples_leaf=1).fit(X, y) + >>> gbdt.predict(X) + array([0, 0, 1, 1]) -.. figure:: ../auto_examples/ensemble/images/sphx_glr_plot_forest_iris_001.png - :target: ../auto_examples/ensemble/plot_forest_iris.html - :align: center - :scale: 75% +When the missingness pattern is predictive, the splits can be performed on +whether the feature value is missing or not:: -.. _random_forest_parameters: + >>> X = np.array([0, np.nan, 1, 2, np.nan]).reshape(-1, 1) + >>> y = [0, 1, 0, 0, 1] + >>> gbdt = HistGradientBoostingClassifier(min_samples_leaf=1, + ... max_depth=2, + ... learning_rate=1, + ... max_iter=1).fit(X, y) + >>> gbdt.predict(X) + array([0, 1, 0, 0, 1]) -Parameters ----------- +If no missing values were encountered for a given feature during training, +then samples with missing values are mapped to whichever child has the most +samples. -The main parameters to adjust when using these methods is ``n_estimators`` and -``max_features``. The former is the number of trees in the forest. The larger -the better, but also the longer it will take to compute. In addition, note that -results will stop getting significantly better beyond a critical number of -trees. The latter is the size of the random subsets of features to consider -when splitting a node. The lower the greater the reduction of variance, but -also the greater the increase in bias. Empirical good default values are -``max_features=None`` (always considering all features instead of a random -subset) for regression problems, and ``max_features="sqrt"`` (using a random -subset of size ``sqrt(n_features)``) for classification tasks (where -``n_features`` is the number of features in the data). Good results are often -achieved when setting ``max_depth=None`` in combination with -``min_samples_split=2`` (i.e., when fully developing the trees). Bear in mind -though that these values are usually not optimal, and might result in models -that consume a lot of RAM. The best parameter values should always be -cross-validated. In addition, note that in random forests, bootstrap samples -are used by default (``bootstrap=True``) while the default strategy for -extra-trees is to use the whole dataset (``bootstrap=False``). When using -bootstrap sampling the generalization accuracy can be estimated on the left out -or out-of-bag samples. This can be enabled by setting ``oob_score=True``. +.. rubric:: Examples -.. note:: +* :ref:`sphx_glr_auto_examples_ensemble_plot_hgbt_regression.py` - The size of the model with the default parameters is :math:`O( M * N * log (N) )`, - where :math:`M` is the number of trees and :math:`N` is the number of samples. - In order to reduce the size of the model, you can change these parameters: - ``min_samples_split``, ``max_leaf_nodes``, ``max_depth`` and ``min_samples_leaf``. +.. _sw_hgbdt: -Parallelization ---------------- +Sample weight support +^^^^^^^^^^^^^^^^^^^^^ -Finally, this module also features the parallel construction of the trees -and the parallel computation of the predictions through the ``n_jobs`` -parameter. If ``n_jobs=k`` then computations are partitioned into -``k`` jobs, and run on ``k`` cores of the machine. If ``n_jobs=-1`` -then all cores available on the machine are used. Note that because of -inter-process communication overhead, the speedup might not be linear -(i.e., using ``k`` jobs will unfortunately not be ``k`` times as -fast). Significant speedup can still be achieved though when building -a large number of trees, or when building a single tree requires a fair -amount of time (e.g., on large datasets). +:class:`HistGradientBoostingClassifier` and +:class:`HistGradientBoostingRegressor` support sample weights during +:term:`fit`. -.. topic:: Examples: +The following toy example demonstrates that samples with a sample weight of zero are ignored: - * :ref:`sphx_glr_auto_examples_ensemble_plot_forest_iris.py` - * :ref:`sphx_glr_auto_examples_ensemble_plot_forest_importances_faces.py` - * :ref:`sphx_glr_auto_examples_miscellaneous_plot_multioutput_face_completion.py` + >>> X = [[1, 0], + ... [1, 0], + ... [1, 0], + ... [0, 1]] + >>> y = [0, 0, 1, 0] + >>> # ignore the first 2 training samples by setting their weight to 0 + >>> sample_weight = [0, 0, 1, 1] + >>> gb = HistGradientBoostingClassifier(min_samples_leaf=1) + >>> gb.fit(X, y, sample_weight=sample_weight) + HistGradientBoostingClassifier(...) + >>> gb.predict([[1, 0]]) + array([1]) + >>> gb.predict_proba([[1, 0]])[0, 1] + np.float64(0.999) -.. topic:: References +As you can see, the `[1, 0]` is comfortably classified as `1` since the first +two samples are ignored due to their sample weights. - .. [B2001] L. Breiman, "Random Forests", Machine Learning, 45(1), 5-32, 2001. +Implementation detail: taking sample weights into account amounts to +multiplying the gradients (and the hessians) by the sample weights. Note that +the binning stage (specifically the quantiles computation) does not take the +weights into account. - .. [B1998] L. Breiman, "Arcing Classifiers", Annals of Statistics 1998. +.. _categorical_support_gbdt: - * P. Geurts, D. Ernst., and L. Wehenkel, "Extremely randomized - trees", Machine Learning, 63(1), 3-42, 2006. +Categorical Features Support +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. _random_forest_feature_importance: +:class:`HistGradientBoostingClassifier` and +:class:`HistGradientBoostingRegressor` have native support for categorical +features: they can consider splits on non-ordered, categorical data. -Feature importance evaluation ------------------------------ +For datasets with categorical features, using the native categorical support +is often better than relying on one-hot encoding +(:class:`~sklearn.preprocessing.OneHotEncoder`), because one-hot encoding +requires more tree depth to achieve equivalent splits. It is also usually +better to rely on the native categorical support rather than to treat +categorical features as continuous (ordinal), which happens for ordinal-encoded +categorical data, since categories are nominal quantities where order does not +matter. -The relative rank (i.e. depth) of a feature used as a decision node in a -tree can be used to assess the relative importance of that feature with -respect to the predictability of the target variable. Features used at -the top of the tree contribute to the final prediction decision of a -larger fraction of the input samples. The **expected fraction of the -samples** they contribute to can thus be used as an estimate of the -**relative importance of the features**. In scikit-learn, the fraction of -samples a feature contributes to is combined with the decrease in impurity -from splitting them to create a normalized estimate of the predictive power -of that feature. +To enable categorical support, a boolean mask can be passed to the +`categorical_features` parameter, indicating which feature is categorical. In +the following, the first feature will be treated as categorical and the +second feature as numerical:: -By **averaging** the estimates of predictive ability over several randomized -trees one can **reduce the variance** of such an estimate and use it -for feature selection. This is known as the mean decrease in impurity, or MDI. -Refer to [L2014]_ for more information on MDI and feature importance -evaluation with Random Forests. + >>> gbdt = HistGradientBoostingClassifier(categorical_features=[True, False]) -.. warning:: +Equivalently, one can pass a list of integers indicating the indices of the +categorical features:: - The impurity-based feature importances computed on tree-based models suffer - from two flaws that can lead to misleading conclusions. First they are - computed on statistics derived from the training dataset and therefore **do - not necessarily inform us on which features are most important to make good - predictions on held-out dataset**. Secondly, **they favor high cardinality - features**, that is features with many unique values. - :ref:`permutation_importance` is an alternative to impurity-based feature - importance that does not suffer from these flaws. These two methods of - obtaining feature importance are explored in: - :ref:`sphx_glr_auto_examples_inspection_plot_permutation_importance.py`. + >>> gbdt = HistGradientBoostingClassifier(categorical_features=[0]) -The following example shows a color-coded representation of the relative -importances of each individual pixel for a face recognition task using -a :class:`ExtraTreesClassifier` model. +When the input is a DataFrame, it is also possible to pass a list of column +names:: -.. figure:: ../auto_examples/ensemble/images/sphx_glr_plot_forest_importances_faces_001.png - :target: ../auto_examples/ensemble/plot_forest_importances_faces.html - :align: center - :scale: 75 + >>> gbdt = HistGradientBoostingClassifier(categorical_features=["site", "manufacturer"]) -In practice those estimates are stored as an attribute named -``feature_importances_`` on the fitted model. This is an array with shape -``(n_features,)`` whose values are positive and sum to 1.0. The higher -the value, the more important is the contribution of the matching feature -to the prediction function. +Finally, when the input is a DataFrame we can use +`categorical_features="from_dtype"` in which case all columns with a categorical +`dtype` will be treated as categorical features. -.. topic:: Examples: +The cardinality of each categorical feature must be less than the `max_bins` +parameter. For an example using histogram-based gradient boosting on categorical +features, see +:ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_categorical.py`. - * :ref:`sphx_glr_auto_examples_ensemble_plot_forest_importances_faces.py` - * :ref:`sphx_glr_auto_examples_ensemble_plot_forest_importances.py` +If there are missing values during training, the missing values will be +treated as a proper category. If there are no missing values during training, +then at prediction time, missing values are mapped to the child node that has +the most samples (just like for continuous features). When predicting, +categories that were not seen during fit time will be treated as missing +values. -.. topic:: References +.. dropdown:: Split finding with categorical features - .. [L2014] G. Louppe, - "Understanding Random Forests: From Theory to Practice", - PhD Thesis, U. of Liege, 2014. + The canonical way of considering categorical splits in a tree is to consider + all of the :math:`2^{K - 1} - 1` partitions, where :math:`K` is the number of + categories. This can quickly become prohibitive when :math:`K` is large. + Fortunately, since gradient boosting trees are always regression trees (even + for classification problems), there exists a faster strategy that can yield + equivalent splits. First, the categories of a feature are sorted according to + the variance of the target, for each category `k`. Once the categories are + sorted, one can consider *continuous partitions*, i.e. treat the categories + as if they were ordered continuous values (see Fisher [Fisher1958]_ for a + formal proof). As a result, only :math:`K - 1` splits need to be considered + instead of :math:`2^{K - 1} - 1`. The initial sorting is a + :math:`\mathcal{O}(K \log(K))` operation, leading to a total complexity of + :math:`\mathcal{O}(K \log(K) + K)`, instead of :math:`\mathcal{O}(2^K)`. -.. _random_trees_embedding: +.. rubric:: Examples -Totally Random Trees Embedding ------------------------------- +* :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_categorical.py` -:class:`RandomTreesEmbedding` implements an unsupervised transformation of the -data. Using a forest of completely random trees, :class:`RandomTreesEmbedding` -encodes the data by the indices of the leaves a data point ends up in. This -index is then encoded in a one-of-K manner, leading to a high dimensional, -sparse binary coding. -This coding can be computed very efficiently and can then be used as a basis -for other learning tasks. -The size and sparsity of the code can be influenced by choosing the number of -trees and the maximum depth per tree. For each tree in the ensemble, the coding -contains one entry of one. The size of the coding is at most ``n_estimators * 2 -** max_depth``, the maximum number of leaves in the forest. +.. _monotonic_cst_gbdt: -As neighboring data points are more likely to lie within the same leaf of a -tree, the transformation performs an implicit, non-parametric density -estimation. +Monotonic Constraints +^^^^^^^^^^^^^^^^^^^^^ -.. topic:: Examples: +Depending on the problem at hand, you may have prior knowledge indicating +that a given feature should in general have a positive (or negative) effect +on the target value. For example, all else being equal, a higher credit +score should increase the probability of getting approved for a loan. +Monotonic constraints allow you to incorporate such prior knowledge into the +model. - * :ref:`sphx_glr_auto_examples_ensemble_plot_random_forest_embedding.py` +For a predictor :math:`F` with two features: - * :ref:`sphx_glr_auto_examples_manifold_plot_lle_digits.py` compares non-linear - dimensionality reduction techniques on handwritten digits. +- a **monotonic increase constraint** is a constraint of the form: - * :ref:`sphx_glr_auto_examples_ensemble_plot_feature_transformation.py` compares - supervised and unsupervised tree based feature transformations. + .. math:: + x_1 \leq x_1' \implies F(x_1, x_2) \leq F(x_1', x_2) -.. seealso:: +- a **monotonic decrease constraint** is a constraint of the form: - :ref:`manifold` techniques can also be useful to derive non-linear - representations of feature space, also these approaches focus also on - dimensionality reduction. + .. math:: + x_1 \leq x_1' \implies F(x_1, x_2) \geq F(x_1', x_2) +You can specify a monotonic constraint on each feature using the +`monotonic_cst` parameter. For each feature, a value of 0 indicates no +constraint, while 1 and -1 indicate a monotonic increase and +monotonic decrease constraint, respectively:: -.. _adaboost: + >>> from sklearn.ensemble import HistGradientBoostingRegressor -AdaBoost -======== + ... # monotonic increase, monotonic decrease, and no constraint on the 3 features + >>> gbdt = HistGradientBoostingRegressor(monotonic_cst=[1, -1, 0]) -The module :mod:`sklearn.ensemble` includes the popular boosting algorithm -AdaBoost, introduced in 1995 by Freund and Schapire [FS1995]_. +In a binary classification context, imposing a monotonic increase (decrease) constraint means that higher values of the feature are supposed +to have a positive (negative) effect on the probability of samples +to belong to the positive class. -The core principle of AdaBoost is to fit a sequence of weak learners (i.e., -models that are only slightly better than random guessing, such as small -decision trees) on repeatedly modified versions of the data. The predictions -from all of them are then combined through a weighted majority vote (or sum) to -produce the final prediction. The data modifications at each so-called boosting -iteration consist of applying weights :math:`w_1`, :math:`w_2`, ..., :math:`w_N` -to each of the training samples. Initially, those weights are all set to -:math:`w_i = 1/N`, so that the first step simply trains a weak learner on the -original data. For each successive iteration, the sample weights are -individually modified and the learning algorithm is reapplied to the reweighted -data. At a given step, those training examples that were incorrectly predicted -by the boosted model induced at the previous step have their weights increased, -whereas the weights are decreased for those that were predicted correctly. As -iterations proceed, examples that are difficult to predict receive -ever-increasing influence. Each subsequent weak learner is thereby forced to -concentrate on the examples that are missed by the previous ones in the sequence -[HTF]_. +Nevertheless, monotonic constraints only marginally constrain feature effects on the output. +For instance, monotonic increase and decrease constraints cannot be used to enforce the +following modelling constraint: -.. figure:: ../auto_examples/ensemble/images/sphx_glr_plot_adaboost_hastie_10_2_001.png - :target: ../auto_examples/ensemble/plot_adaboost_hastie_10_2.html - :align: center - :scale: 75 +.. math:: + x_1 \leq x_1' \implies F(x_1, x_2) \leq F(x_1', x_2') -AdaBoost can be used both for classification and regression problems: +Also, monotonic constraints are not supported for multiclass classification. - - For multi-class classification, :class:`AdaBoostClassifier` implements - AdaBoost-SAMME and AdaBoost-SAMME.R [ZZRH2009]_. +For a practical implementation of monotonic constraints with the histogram-based +gradient boosting, including how they can improve generalization when domain knowledge +is available, see +:ref:`sphx_glr_auto_examples_ensemble_plot_monotonic_constraints.py`. - - For regression, :class:`AdaBoostRegressor` implements AdaBoost.R2 [D1997]_. +.. note:: + Since categories are unordered quantities, it is not possible to enforce + monotonic constraints on categorical features. -Usage ------ +.. rubric:: Examples -The following example shows how to fit an AdaBoost classifier with 100 weak -learners:: +* :ref:`sphx_glr_auto_examples_ensemble_plot_hgbt_regression.py` - >>> from sklearn.model_selection import cross_val_score - >>> from sklearn.datasets import load_iris - >>> from sklearn.ensemble import AdaBoostClassifier +.. _interaction_cst_hgbt: - >>> X, y = load_iris(return_X_y=True) - >>> clf = AdaBoostClassifier(n_estimators=100) - >>> scores = cross_val_score(clf, X, y, cv=5) - >>> scores.mean() - 0.9... +Interaction constraints +^^^^^^^^^^^^^^^^^^^^^^^ -The number of weak learners is controlled by the parameter ``n_estimators``. The -``learning_rate`` parameter controls the contribution of the weak learners in -the final combination. By default, weak learners are decision stumps. Different -weak learners can be specified through the ``base_estimator`` parameter. -The main parameters to tune to obtain good results are ``n_estimators`` and -the complexity of the base estimators (e.g., its depth ``max_depth`` or -minimum required number of samples to consider a split ``min_samples_split``). +A priori, the histogram gradient boosted trees are allowed to use any feature +to split a node into child nodes. This creates so called interactions between +features, i.e. usage of different features as split along a branch. Sometimes, +one wants to restrict the possible interactions, see [Mayer2022]_. This can be +done by the parameter ``interaction_cst``, where one can specify the indices +of features that are allowed to interact. +For instance, with 3 features in total, ``interaction_cst=[{0}, {1}, {2}]`` +forbids all interactions. +The constraints ``[{0, 1}, {1, 2}]`` specify two groups of possibly +interacting features. Features 0 and 1 may interact with each other, as well +as features 1 and 2. But note that features 0 and 2 are forbidden to interact. +The following depicts a tree and the possible splits of the tree: -.. topic:: Examples: +.. code-block:: none - * :ref:`sphx_glr_auto_examples_ensemble_plot_adaboost_hastie_10_2.py` compares the - classification error of a decision stump, decision tree, and a boosted - decision stump using AdaBoost-SAMME and AdaBoost-SAMME.R. + 1 <- Both constraint groups could be applied from now on + / \ + 1 2 <- Left split still fulfills both constraint groups. + / \ / \ Right split at feature 2 has only group {1, 2} from now on. - * :ref:`sphx_glr_auto_examples_ensemble_plot_adaboost_multiclass.py` shows the performance - of AdaBoost-SAMME and AdaBoost-SAMME.R on a multi-class problem. +LightGBM uses the same logic for overlapping groups. - * :ref:`sphx_glr_auto_examples_ensemble_plot_adaboost_twoclass.py` shows the decision boundary - and decision function values for a non-linearly separable two-class problem - using AdaBoost-SAMME. +Note that features not listed in ``interaction_cst`` are automatically +assigned an interaction group for themselves. With again 3 features, this +means that ``[{0}]`` is equivalent to ``[{0}, {1, 2}]``. - * :ref:`sphx_glr_auto_examples_ensemble_plot_adaboost_regression.py` demonstrates regression - with the AdaBoost.R2 algorithm. +.. rubric:: Examples -.. topic:: References +* :ref:`sphx_glr_auto_examples_inspection_plot_partial_dependence.py` - .. [FS1995] Y. Freund, and R. Schapire, "A Decision-Theoretic Generalization of - On-Line Learning and an Application to Boosting", 1997. +.. rubric:: References - .. [ZZRH2009] J. Zhu, H. Zou, S. Rosset, T. Hastie. "Multi-class AdaBoost", - 2009. +.. [Mayer2022] M. Mayer, S.C. Bourassa, M. Hoesli, and D.F. Scognamiglio. + 2022. :doi:`Machine Learning Applications to Land and Structure Valuation + <10.3390/jrfm15050193>`. + Journal of Risk and Financial Management 15, no. 5: 193 - .. [D1997] H. Drucker. "Improving Regressors using Boosting Techniques", 1997. +Low-level parallelism +^^^^^^^^^^^^^^^^^^^^^ - .. [HTF] T. Hastie, R. Tibshirani and J. Friedman, "Elements of - Statistical Learning Ed. 2", Springer, 2009. +:class:`HistGradientBoostingClassifier` and +:class:`HistGradientBoostingRegressor` use OpenMP +for parallelization through Cython. For more details on how to control the +number of threads, please refer to our :ref:`parallelism` notes. -.. _gradient_boosting: +The following parts are parallelized: -Gradient Tree Boosting -====================== +- mapping samples from real values to integer-valued bins (finding the bin + thresholds is however sequential) +- building histograms is parallelized over features +- finding the best split point at a node is parallelized over features +- during fit, mapping samples into the left and right children is + parallelized over samples +- gradient and hessians computations are parallelized over samples +- predicting is parallelized over samples + +.. _Why_it's_faster: + +Why it's faster +^^^^^^^^^^^^^^^ + +The bottleneck of a gradient boosting procedure is building the decision +trees. Building a traditional decision tree (as in the other GBDTs +:class:`GradientBoostingClassifier` and :class:`GradientBoostingRegressor`) +requires sorting the samples at each node (for +each feature). Sorting is needed so that the potential gain of a split point +can be computed efficiently. Splitting a single node has thus a complexity +of :math:`\mathcal{O}(n_\text{features} \times n \log(n))` where :math:`n` +is the number of samples at the node. + +:class:`HistGradientBoostingClassifier` and +:class:`HistGradientBoostingRegressor`, in contrast, do not require sorting the +feature values and instead use a data-structure called a histogram, where the +samples are implicitly ordered. Building a histogram has a +:math:`\mathcal{O}(n)` complexity, so the node splitting procedure has a +:math:`\mathcal{O}(n_\text{features} \times n)` complexity, much smaller +than the previous one. In addition, instead of considering :math:`n` split +points, we consider only ``max_bins`` split points, which might be much +smaller. -`Gradient Tree Boosting `_ -or Gradient Boosted Decision Trees (GBDT) is a generalization -of boosting to arbitrary -differentiable loss functions. GBDT is an accurate and effective -off-the-shelf procedure that can be used for both regression and -classification problems in a -variety of areas including Web search ranking and ecology. +In order to build histograms, the input data `X` needs to be binned into +integer-valued bins. This binning procedure does require sorting the feature +values, but it only happens once at the very beginning of the boosting process +(not at each node, like in :class:`GradientBoostingClassifier` and +:class:`GradientBoostingRegressor`). -The module :mod:`sklearn.ensemble` provides methods -for both classification and regression via gradient boosted decision -trees. +Finally, many parts of the implementation of +:class:`HistGradientBoostingClassifier` and +:class:`HistGradientBoostingRegressor` are parallelized. -.. note:: +.. rubric:: References - Scikit-learn 0.21 introduces two new experimental implementations of - gradient boosting trees, namely :class:`HistGradientBoostingClassifier` - and :class:`HistGradientBoostingRegressor`, inspired by - `LightGBM `__ (See [LightGBM]_). +.. [XGBoost] Tianqi Chen, Carlos Guestrin, :arxiv:`"XGBoost: A Scalable Tree + Boosting System" <1603.02754>` - These histogram-based estimators can be **orders of magnitude faster** - than :class:`GradientBoostingClassifier` and - :class:`GradientBoostingRegressor` when the number of samples is larger - than tens of thousands of samples. +.. [LightGBM] Ke et. al. `"LightGBM: A Highly Efficient Gradient + BoostingDecision Tree" `_ - They also have built-in support for missing values, which avoids the need - for an imputer. +.. [Fisher1958] Fisher, W.D. (1958). `"On Grouping for Maximum Homogeneity" + `_ + Journal of the American Statistical Association, 53, 789-798. - These estimators are described in more detail below in - :ref:`histogram_based_gradient_boosting`. - The following guide focuses on :class:`GradientBoostingClassifier` and - :class:`GradientBoostingRegressor`, which might be preferred for small - sample sizes since binning may lead to split points that are too approximate - in this setting. +:class:`GradientBoostingClassifier` and :class:`GradientBoostingRegressor` +---------------------------------------------------------------------------- The usage and the parameters of :class:`GradientBoostingClassifier` and :class:`GradientBoostingRegressor` are described below. The 2 most important parameters of these estimators are `n_estimators` and `learning_rate`. -Classification ---------------- - -:class:`GradientBoostingClassifier` supports both binary and multi-class -classification. -The following example shows how to fit a gradient boosting classifier -with 100 decision stumps as weak learners:: - - >>> from sklearn.datasets import make_hastie_10_2 - >>> from sklearn.ensemble import GradientBoostingClassifier - - >>> X, y = make_hastie_10_2(random_state=0) - >>> X_train, X_test = X[:2000], X[2000:] - >>> y_train, y_test = y[:2000], y[2000:] - - >>> clf = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0, - ... max_depth=1, random_state=0).fit(X_train, y_train) - >>> clf.score(X_test, y_test) - 0.913... - -The number of weak learners (i.e. regression trees) is controlled by the -parameter ``n_estimators``; :ref:`The size of each tree -` can be controlled either by setting the tree -depth via ``max_depth`` or by setting the number of leaf nodes via -``max_leaf_nodes``. The ``learning_rate`` is a hyper-parameter in the range -(0.0, 1.0] that controls overfitting via :ref:`shrinkage -` . - -.. note:: - - Classification with more than 2 classes requires the induction - of ``n_classes`` regression trees at each iteration, - thus, the total number of induced trees equals - ``n_classes * n_estimators``. For datasets with a large number - of classes we strongly recommend to use - :class:`HistGradientBoostingClassifier` as an alternative to - :class:`GradientBoostingClassifier` . - -Regression ----------- - -:class:`GradientBoostingRegressor` supports a number of -:ref:`different loss functions ` -for regression which can be specified via the argument -``loss``; the default loss function for regression is least squares (``'ls'``). - -:: - - >>> import numpy as np - >>> from sklearn.metrics import mean_squared_error - >>> from sklearn.datasets import make_friedman1 - >>> from sklearn.ensemble import GradientBoostingRegressor - - >>> X, y = make_friedman1(n_samples=1200, random_state=0, noise=1.0) - >>> X_train, X_test = X[:200], X[200:] - >>> y_train, y_test = y[:200], y[200:] - >>> est = GradientBoostingRegressor(n_estimators=100, learning_rate=0.1, - ... max_depth=1, random_state=0, loss='ls').fit(X_train, y_train) - >>> mean_squared_error(y_test, est.predict(X_test)) - 5.00... - -The figure below shows the results of applying :class:`GradientBoostingRegressor` -with least squares loss and 500 base learners to the diabetes dataset -(:func:`sklearn.datasets.load_diabetes`). -The plot on the left shows the train and test error at each iteration. -The train error at each iteration is stored in the -:attr:`~GradientBoostingRegressor.train_score_` attribute -of the gradient boosting model. The test error at each iterations can be obtained -via the :meth:`~GradientBoostingRegressor.staged_predict` method which returns a -generator that yields the predictions at each stage. Plots like these can be used -to determine the optimal number of trees (i.e. ``n_estimators``) by early stopping. - -.. figure:: ../auto_examples/ensemble/images/sphx_glr_plot_gradient_boosting_regression_001.png - :target: ../auto_examples/ensemble/plot_gradient_boosting_regression.html - :align: center - :scale: 75 +.. dropdown:: Classification + + :class:`GradientBoostingClassifier` supports both binary and multi-class + classification. + The following example shows how to fit a gradient boosting classifier + with 100 decision stumps as weak learners:: + + >>> from sklearn.datasets import make_hastie_10_2 + >>> from sklearn.ensemble import GradientBoostingClassifier + + >>> X, y = make_hastie_10_2(random_state=0) + >>> X_train, X_test = X[:2000], X[2000:] + >>> y_train, y_test = y[:2000], y[2000:] + + >>> clf = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0, + ... max_depth=1, random_state=0).fit(X_train, y_train) + >>> clf.score(X_test, y_test) + 0.913 + + The number of weak learners (i.e. regression trees) is controlled by the + parameter ``n_estimators``; :ref:`The size of each tree + ` can be controlled either by setting the tree + depth via ``max_depth`` or by setting the number of leaf nodes via + ``max_leaf_nodes``. The ``learning_rate`` is a hyper-parameter in the range + (0.0, 1.0] that controls overfitting via :ref:`shrinkage + ` . + + .. note:: + + Classification with more than 2 classes requires the induction + of ``n_classes`` regression trees at each iteration, + thus, the total number of induced trees equals + ``n_classes * n_estimators``. For datasets with a large number + of classes we strongly recommend to use + :class:`HistGradientBoostingClassifier` as an alternative to + :class:`GradientBoostingClassifier` . + +.. dropdown:: Regression + + :class:`GradientBoostingRegressor` supports a number of + :ref:`different loss functions ` + for regression which can be specified via the argument + ``loss``; the default loss function for regression is squared error + (``'squared_error'``). + + :: + + >>> import numpy as np + >>> from sklearn.metrics import mean_squared_error + >>> from sklearn.datasets import make_friedman1 + >>> from sklearn.ensemble import GradientBoostingRegressor + + >>> X, y = make_friedman1(n_samples=1200, random_state=0, noise=1.0) + >>> X_train, X_test = X[:200], X[200:] + >>> y_train, y_test = y[:200], y[200:] + >>> est = GradientBoostingRegressor( + ... n_estimators=100, learning_rate=0.1, max_depth=1, random_state=0, + ... loss='squared_error' + ... ).fit(X_train, y_train) + >>> mean_squared_error(y_test, est.predict(X_test)) + 5.00 + + The figure below shows the results of applying :class:`GradientBoostingRegressor` + with least squares loss and 500 base learners to the diabetes dataset + (:func:`sklearn.datasets.load_diabetes`). + The plot shows the train and test error at each iteration. + The train error at each iteration is stored in the + `train_score_` attribute of the gradient boosting model. + The test error at each iteration can be obtained + via the :meth:`~GradientBoostingRegressor.staged_predict` method which returns a + generator that yields the predictions at each stage. Plots like these can be used + to determine the optimal number of trees (i.e. ``n_estimators``) by early stopping. + + .. figure:: ../auto_examples/ensemble/images/sphx_glr_plot_gradient_boosting_regression_001.png + :target: ../auto_examples/ensemble/plot_gradient_boosting_regression.html + :align: center + :scale: 75 -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_regression.py` - * :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_oob.py` +* :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_regression.py` +* :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_oob.py` .. _gradient_boosting_warm_start: Fitting additional weak-learners --------------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Both :class:`GradientBoostingRegressor` and :class:`GradientBoostingClassifier` support ``warm_start=True`` which allows you to add more estimators to an already @@ -586,15 +594,30 @@ fitted model. :: - >>> _ = est.set_params(n_estimators=200, warm_start=True) # set warm_start and new nr of trees + >>> import numpy as np + >>> from sklearn.metrics import mean_squared_error + >>> from sklearn.datasets import make_friedman1 + >>> from sklearn.ensemble import GradientBoostingRegressor + + >>> X, y = make_friedman1(n_samples=1200, random_state=0, noise=1.0) + >>> X_train, X_test = X[:200], X[200:] + >>> y_train, y_test = y[:200], y[200:] + >>> est = GradientBoostingRegressor( + ... n_estimators=100, learning_rate=0.1, max_depth=1, random_state=0, + ... loss='squared_error' + ... ) + >>> est = est.fit(X_train, y_train) # fit with 100 trees + >>> mean_squared_error(y_test, est.predict(X_test)) + 5.00 + >>> _ = est.set_params(n_estimators=200, warm_start=True) # set warm_start and increase num of trees >>> _ = est.fit(X_train, y_train) # fit additional 100 trees to est >>> mean_squared_error(y_test, est.predict(X_test)) - 3.84... + 3.84 .. _gradient_boosting_tree_size: Controlling the tree size -------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^^^ The size of the regression tree base learners defines the level of variable interactions that can be captured by the gradient boosting model. In general, @@ -617,54 +640,55 @@ We found that ``max_leaf_nodes=k`` gives comparable results to ``max_depth=k-1`` but is significantly faster to train at the expense of a slightly higher training error. The parameter ``max_leaf_nodes`` corresponds to the variable ``J`` in the -chapter on gradient boosting in [F2001]_ and is related to the parameter +chapter on gradient boosting in [Friedman2001]_ and is related to the parameter ``interaction.depth`` in R's gbm package where ``max_leaf_nodes == interaction.depth + 1`` . +.. _gradient_boosting_formulation: + Mathematical formulation -------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^ We first present GBRT for regression, and then detail the classification case. -Regression -^^^^^^^^^^ +.. dropdown:: Regression -GBRT regressors are additive models whose prediction :math:`y_i` for a -given input :math:`x_i` is of the following form: + GBRT regressors are additive models whose prediction :math:`\hat{y}_i` for a + given input :math:`x_i` is of the following form: .. math:: - \hat{y_i} = F_M(x_i) = \sum_{m=1}^{M} h_m(x_i) + \hat{y}_i = F_M(x_i) = \sum_{m=1}^{M} h_m(x_i) -where the :math:`h_m` are estimators called *weak learners* in the context -of boosting. Gradient Tree Boosting uses :ref:`decision tree regressors -` of fixed size as weak learners. The constant M corresponds to the -`n_estimators` parameter. + where the :math:`h_m` are estimators called *weak learners* in the context + of boosting. Gradient Tree Boosting uses :ref:`decision tree regressors + ` of fixed size as weak learners. The constant M corresponds to the + `n_estimators` parameter. -Similar to other boosting algorithms, a GBRT is built in a greedy fashion: + Similar to other boosting algorithms, a GBRT is built in a greedy fashion: .. math:: F_m(x) = F_{m-1}(x) + h_m(x), -where the newly added tree :math:`h_m` is fitted in order to minimize a sum -of losses :math:`L_m`, given the previous ensemble :math:`F_{m-1}`: + where the newly added tree :math:`h_m` is fitted in order to minimize a sum + of losses :math:`L_m`, given the previous ensemble :math:`F_{m-1}`: .. math:: h_m = \arg\min_{h} L_m = \arg\min_{h} \sum_{i=1}^{n} l(y_i, F_{m-1}(x_i) + h(x_i)), -where :math:`l(y_i, F(x_i))` is defined by the `loss` parameter, detailed -in the next section. + where :math:`l(y_i, F(x_i))` is defined by the `loss` parameter, detailed + in the next section. -By default, the initial model :math:`F_{0}` is chosen as the constant that -minimizes the loss: for a least-squares loss, this is the empirical mean of -the target values. The initial model can also be specified via the ``init`` -argument. + By default, the initial model :math:`F_{0}` is chosen as the constant that + minimizes the loss: for a least-squares loss, this is the empirical mean of + the target values. The initial model can also be specified via the ``init`` + argument. -Using a first-order Taylor approximation, the value of :math:`l` can be -approximated as follows: + Using a first-order Taylor approximation, the value of :math:`l` can be + approximated as follows: .. math:: @@ -673,113 +697,112 @@ approximated as follows: + h_m(x_i) \left[ \frac{\partial l(y_i, F(x_i))}{\partial F(x_i)} \right]_{F=F_{m - 1}}. -.. note:: + .. note:: - Briefly, a first-order Taylor approximation says that - :math:`l(z) \approx l(a) + (z - a) \frac{\partial l(a)}{\partial a}`. - Here, :math:`z` corresponds to :math:`F_{m - 1}(x_i) + h_m(x_i)`, and - :math:`a` corresponds to :math:`F_{m-1}(x_i)` + Briefly, a first-order Taylor approximation says that + :math:`l(z) \approx l(a) + (z - a) \frac{\partial l}{\partial z}(a)`. + Here, :math:`z` corresponds to :math:`F_{m - 1}(x_i) + h_m(x_i)`, and + :math:`a` corresponds to :math:`F_{m-1}(x_i)` -The quantity :math:`\left[ \frac{\partial l(y_i, F(x_i))}{\partial F(x_i)} -\right]_{F=F_{m - 1}}` is the derivative of the loss with respect to its -second parameter, evaluated at :math:`F_{m-1}(x)`. It is easy to compute for -any given :math:`F_{m - 1}(x_i)` in a closed form since the loss is -differentiable. We will denote it by :math:`g_i`. + The quantity :math:`\left[ \frac{\partial l(y_i, F(x_i))}{\partial F(x_i)} + \right]_{F=F_{m - 1}}` is the derivative of the loss with respect to its + second parameter, evaluated at :math:`F_{m-1}(x)`. It is easy to compute for + any given :math:`F_{m - 1}(x_i)` in a closed form since the loss is + differentiable. We will denote it by :math:`g_i`. -Removing the constant terms, we have: + Removing the constant terms, we have: .. math:: h_m \approx \arg\min_{h} \sum_{i=1}^{n} h(x_i) g_i -This is minimized if :math:`h(x_i)` is fitted to predict a value that is -proportional to the negative gradient :math:`-g_i`. Therefore, at each -iteration, **the estimator** :math:`h_m` **is fitted to predict the negative -gradients of the samples**. The gradients are updated at each iteration. -This can be considered as some kind of gradient descent in a functional -space. + This is minimized if :math:`h(x_i)` is fitted to predict a value that is + proportional to the negative gradient :math:`-g_i`. Therefore, at each + iteration, **the estimator** :math:`h_m` **is fitted to predict the negative + gradients of the samples**. The gradients are updated at each iteration. + This can be considered as some kind of gradient descent in a functional + space. -.. note:: + .. note:: - For some losses, e.g. the least absolute deviation (LAD) where the gradients - are :math:`\pm 1`, the values predicted by a fitted :math:`h_m` are not - accurate enough: the tree can only output integer values. As a result, the - leaves values of the tree :math:`h_m` are modified once the tree is - fitted, such that the leaves values minimize the loss :math:`L_m`. The - update is loss-dependent: for the LAD loss, the value of a leaf is updated - to the median of the samples in that leaf. + For some losses, e.g. ``'absolute_error'`` where the gradients + are :math:`\pm 1`, the values predicted by a fitted :math:`h_m` are not + accurate enough: the tree can only output integer values. As a result, the + leaves values of the tree :math:`h_m` are modified once the tree is + fitted, such that the leaves values minimize the loss :math:`L_m`. The + update is loss-dependent: for the absolute error loss, the value of + a leaf is updated to the median of the samples in that leaf. -Classification -^^^^^^^^^^^^^^ +.. dropdown:: Classification -Gradient boosting for classification is very similar to the regression case. -However, the sum of the trees :math:`F_M(x_i) = \sum_m h_m(x_i)` is not -homogeneous to a prediction: it cannot be a class, since the trees predict -continuous values. + Gradient boosting for classification is very similar to the regression case. + However, the sum of the trees :math:`F_M(x_i) = \sum_m h_m(x_i)` is not + homogeneous to a prediction: it cannot be a class, since the trees predict + continuous values. -The mapping from the value :math:`F_M(x_i)` to a class or a probability is -loss-dependent. For the deviance (or log-loss), the probability that -:math:`x_i` belongs to the positive class is modeled as :math:`p(y_i = 1 | -x_i) = \sigma(F_M(x_i))` where :math:`\sigma` is the sigmoid function. + The mapping from the value :math:`F_M(x_i)` to a class or a probability is + loss-dependent. For the log-loss, the probability that + :math:`x_i` belongs to the positive class is modeled as :math:`p(y_i = 1 | + x_i) = \sigma(F_M(x_i))` where :math:`\sigma` is the sigmoid or expit function. -For multiclass classification, K trees (for K classes) are built at each of -the :math:`M` iterations. The probability that :math:`x_i` belongs to class -k is modeled as a softmax of the :math:`F_{M,k}(x_i)` values. + For multiclass classification, K trees (for K classes) are built at each of + the :math:`M` iterations. The probability that :math:`x_i` belongs to class + k is modeled as a softmax of the :math:`F_{M,k}(x_i)` values. -Note that even for a classification task, the :math:`h_m` sub-estimator is -still a regressor, not a classifier. This is because the sub-estimators are -trained to predict (negative) *gradients*, which are always continuous -quantities. + Note that even for a classification task, the :math:`h_m` sub-estimator is + still a regressor, not a classifier. This is because the sub-estimators are + trained to predict (negative) *gradients*, which are always continuous + quantities. .. _gradient_boosting_loss: Loss Functions --------------- +^^^^^^^^^^^^^^ The following loss functions are supported and can be specified using the parameter ``loss``: - * Regression - - * Least squares (``'ls'``): The natural choice for regression due - to its superior computational properties. The initial model is - given by the mean of the target values. - * Least absolute deviation (``'lad'``): A robust loss function for - regression. The initial model is given by the median of the - target values. - * Huber (``'huber'``): Another robust loss function that combines - least squares and least absolute deviation; use ``alpha`` to - control the sensitivity with regards to outliers (see [F2001]_ for - more details). - * Quantile (``'quantile'``): A loss function for quantile regression. - Use ``0 < alpha < 1`` to specify the quantile. This loss function - can be used to create prediction intervals - (see :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_quantile.py`). - - * Classification - - * Binomial deviance (``'deviance'``): The negative binomial - log-likelihood loss function for binary classification (provides - probability estimates). The initial model is given by the - log odds-ratio. - * Multinomial deviance (``'deviance'``): The negative multinomial - log-likelihood loss function for multi-class classification with - ``n_classes`` mutually exclusive classes. It provides - probability estimates. The initial model is given by the - prior probability of each class. At each iteration ``n_classes`` - regression trees have to be constructed which makes GBRT rather - inefficient for data sets with a large number of classes. - * Exponential loss (``'exponential'``): The same loss function - as :class:`AdaBoostClassifier`. Less robust to mislabeled - examples than ``'deviance'``; can only be used for binary - classification. +.. dropdown:: Regression + + * Squared error (``'squared_error'``): The natural choice for regression + due to its superior computational properties. The initial model is + given by the mean of the target values. + * Absolute error (``'absolute_error'``): A robust loss function for + regression. The initial model is given by the median of the + target values. + * Huber (``'huber'``): Another robust loss function that combines + least squares and least absolute deviation; use ``alpha`` to + control the sensitivity with regards to outliers (see [Friedman2001]_ for + more details). + * Quantile (``'quantile'``): A loss function for quantile regression. + Use ``0 < alpha < 1`` to specify the quantile. This loss function + can be used to create prediction intervals + (see :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_quantile.py`). + +.. dropdown:: Classification + + * Binary log-loss (``'log-loss'``): The binomial + negative log-likelihood loss function for binary classification. It provides + probability estimates. The initial model is given by the + log odds-ratio. + * Multi-class log-loss (``'log-loss'``): The multinomial + negative log-likelihood loss function for multi-class classification with + ``n_classes`` mutually exclusive classes. It provides + probability estimates. The initial model is given by the + prior probability of each class. At each iteration ``n_classes`` + regression trees have to be constructed which makes GBRT rather + inefficient for data sets with a large number of classes. + * Exponential loss (``'exponential'``): The same loss function + as :class:`AdaBoostClassifier`. Less robust to mislabeled + examples than ``'log-loss'``; can only be used for binary + classification. .. _gradient_boosting_shrinkage: Shrinkage via learning rate ---------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[F2001]_ proposed a simple regularization strategy that scales +[Friedman2001]_ proposed a simple regularization strategy that scales the contribution of each weak learner by a constant factor :math:`\nu`: .. math:: @@ -787,7 +810,7 @@ the contribution of each weak learner by a constant factor :math:`\nu`: F_m(x) = F_{m-1}(x) + \nu h_m(x) The parameter :math:`\nu` is also called the **learning rate** because -it scales the step length the gradient descent procedure; it can +it scales the step length of the gradient descent procedure; it can be set via the ``learning_rate`` parameter. The parameter ``learning_rate`` strongly interacts with the parameter @@ -796,14 +819,16 @@ of ``learning_rate`` require larger numbers of weak learners to maintain a constant training error. Empirical evidence suggests that small values of ``learning_rate`` favor better test error. [HTF]_ recommend to set the learning rate to a small constant -(e.g. ``learning_rate <= 0.1``) and choose ``n_estimators`` by early -stopping. For a more detailed discussion of the interaction between +(e.g. ``learning_rate <= 0.1``) and choose ``n_estimators`` large enough +that early stopping applies, +see :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_early_stopping.py` +for a more detailed discussion of the interaction between ``learning_rate`` and ``n_estimators`` see [R2007]_. Subsampling ------------ +^^^^^^^^^^^^ -[F1999]_ proposed stochastic gradient boosting, which combines gradient +[Friedman2002]_ proposed stochastic gradient boosting, which combines gradient boosting with bootstrap averaging (bagging). At each iteration the base classifier is trained on a fraction ``subsample`` of the available training data. The subsample is drawn without replacement. @@ -821,7 +846,7 @@ does poorly. :scale: 75 Another strategy to reduce the variance is by subsampling the features -analogous to the random splits in :class:`RandomForestClassifier` . +analogous to the random splits in :class:`RandomForestClassifier`. The number of subsampled features can be controlled via the ``max_features`` parameter. @@ -830,23 +855,22 @@ parameter. Stochastic gradient boosting allows to compute out-of-bag estimates of the test deviance by computing the improvement in deviance on the examples that are not included in the bootstrap sample (i.e. the out-of-bag examples). -The improvements are stored in the attribute -:attr:`~GradientBoostingRegressor.oob_improvement_`. ``oob_improvement_[i]`` holds -the improvement in terms of the loss on the OOB samples if you add the i-th stage -to the current predictions. +The improvements are stored in the attribute `oob_improvement_`. +``oob_improvement_[i]`` holds the improvement in terms of the loss on the OOB samples +if you add the i-th stage to the current predictions. Out-of-bag estimates can be used for model selection, for example to determine the optimal number of iterations. OOB estimates are usually very pessimistic thus we recommend to use cross-validation instead and only use OOB if cross-validation is too time consuming. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_regularization.py` - * :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_oob.py` - * :ref:`sphx_glr_auto_examples_ensemble_plot_ensemble_oob.py` +* :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_regularization.py` +* :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_oob.py` +* :ref:`sphx_glr_auto_examples_ensemble_plot_ensemble_oob.py` Interpretation with feature importance --------------------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Individual decision trees can be interpreted easily by simply visualizing the tree structure. Gradient boosting models, however, @@ -859,7 +883,7 @@ Often features do not contribute equally to predict the target response; in many situations the majority of the features are in fact irrelevant. When interpreting a model, the first question usually is: what are -those important features and how do they contributing in predicting +those important features and how do they contribute in predicting the target response? Individual decision trees intrinsically perform feature selection by selecting @@ -880,352 +904,420 @@ accessed via the ``feature_importances_`` property:: >>> clf = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0, ... max_depth=1, random_state=0).fit(X, y) >>> clf.feature_importances_ - array([0.10..., 0.10..., 0.11..., ... + array([0.107, 0.105, 0.113, 0.0987, 0.0947, + 0.107, 0.0916, 0.0972, 0.0958, 0.0906]) Note that this computation of feature importance is based on entropy, and it is distinct from :func:`sklearn.inspection.permutation_importance` which is based on permutation of the features. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_regression.py` +* :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_regression.py` -.. _histogram_based_gradient_boosting: +.. rubric:: References -Histogram-Based Gradient Boosting -================================= +.. [Friedman2001] Friedman, J.H. (2001). :doi:`Greedy function approximation: A gradient + boosting machine <10.1214/aos/1013203451>`. + Annals of Statistics, 29, 1189-1232. -Scikit-learn 0.21 introduced two new experimental implementations of -gradient boosting trees, namely :class:`HistGradientBoostingClassifier` -and :class:`HistGradientBoostingRegressor`, inspired by -`LightGBM `__ (See [LightGBM]_). +.. [Friedman2002] Friedman, J.H. (2002). `Stochastic gradient boosting. + `_. + Computational Statistics & Data Analysis, 38, 367-378. -These histogram-based estimators can be **orders of magnitude faster** -than :class:`GradientBoostingClassifier` and -:class:`GradientBoostingRegressor` when the number of samples is larger -than tens of thousands of samples. +.. [R2007] G. Ridgeway (2006). `Generalized Boosted Models: A guide to the gbm + package `_ -They also have built-in support for missing values, which avoids the need -for an imputer. +.. _forest: -These fast estimators first bin the input samples ``X`` into -integer-valued bins (typically 256 bins) which tremendously reduces the -number of splitting points to consider, and allows the algorithm to -leverage integer-based data structures (histograms) instead of relying on -sorted continuous values when building the trees. The API of these -estimators is slightly different, and some of the features from -:class:`GradientBoostingClassifier` and :class:`GradientBoostingRegressor` -are not yet supported, for instance some loss functions. +Random forests and other randomized tree ensembles +=================================================== -These estimators are still **experimental**: their predictions -and their API might change without any deprecation cycle. To use them, you -need to explicitly import ``enable_hist_gradient_boosting``:: +The :mod:`sklearn.ensemble` module includes two averaging algorithms based +on randomized :ref:`decision trees `: the RandomForest algorithm +and the Extra-Trees method. Both algorithms are perturb-and-combine +techniques [B1998]_ specifically designed for trees. This means a diverse +set of classifiers is created by introducing randomness in the classifier +construction. The prediction of the ensemble is given as the averaged +prediction of the individual classifiers. - >>> # explicitly require this experimental feature - >>> from sklearn.experimental import enable_hist_gradient_boosting # noqa - >>> # now you can import normally from ensemble - >>> from sklearn.ensemble import HistGradientBoostingClassifier +As other classifiers, forest classifiers have to be fitted with two +arrays: a sparse or dense array X of shape ``(n_samples, n_features)`` +holding the training samples, and an array Y of shape ``(n_samples,)`` +holding the target values (class labels) for the training samples:: -.. topic:: Examples: + >>> from sklearn.ensemble import RandomForestClassifier + >>> X = [[0, 0], [1, 1]] + >>> Y = [0, 1] + >>> clf = RandomForestClassifier(n_estimators=10) + >>> clf = clf.fit(X, Y) - * :ref:`sphx_glr_auto_examples_inspection_plot_partial_dependence.py` +Like :ref:`decision trees `, forests of trees also extend to +:ref:`multi-output problems ` (if Y is an array +of shape ``(n_samples, n_outputs)``). -Usage ------ +Random Forests +-------------- -Most of the parameters are unchanged from -:class:`GradientBoostingClassifier` and :class:`GradientBoostingRegressor`. -One exception is the ``max_iter`` parameter that replaces ``n_estimators``, and -controls the number of iterations of the boosting process:: +In random forests (see :class:`RandomForestClassifier` and +:class:`RandomForestRegressor` classes), each tree in the ensemble is built +from a sample drawn with replacement (i.e., a bootstrap sample) from the +training set. - >>> from sklearn.experimental import enable_hist_gradient_boosting - >>> from sklearn.ensemble import HistGradientBoostingClassifier - >>> from sklearn.datasets import make_hastie_10_2 +Furthermore, when splitting each node during the construction of a tree, the +best split is found through an exhaustive search of the features values of +either all input features or a random subset of size ``max_features``. +(See the :ref:`parameter tuning guidelines ` for more details.) - >>> X, y = make_hastie_10_2(random_state=0) - >>> X_train, X_test = X[:2000], X[2000:] - >>> y_train, y_test = y[:2000], y[2000:] +The purpose of these two sources of randomness is to decrease the variance of +the forest estimator. Indeed, individual decision trees typically exhibit high +variance and tend to overfit. The injected randomness in forests yield decision +trees with somewhat decoupled prediction errors. By taking an average of those +predictions, some errors can cancel out. Random forests achieve a reduced +variance by combining diverse trees, sometimes at the cost of a slight increase +in bias. In practice the variance reduction is often significant hence yielding +an overall better model. - >>> clf = HistGradientBoostingClassifier(max_iter=100).fit(X_train, y_train) - >>> clf.score(X_test, y_test) - 0.8965 +In contrast to the original publication [B2001]_, the scikit-learn +implementation combines classifiers by averaging their probabilistic +prediction, instead of letting each classifier vote for a single class. -Available losses for regression are 'least_squares', -'least_absolute_deviation', which is less sensitive to outliers, and -'poisson', which is well suited to model counts and frequencies. For -classification, 'binary_crossentropy' is used for binary classification and -'categorical_crossentropy' is used for multiclass classification. By default -the loss is 'auto' and will select the appropriate loss depending on -:term:`y` passed to :term:`fit`. +A competitive alternative to random forests are +:ref:`histogram_based_gradient_boosting` (HGBT) models: -The size of the trees can be controlled through the ``max_leaf_nodes``, -``max_depth``, and ``min_samples_leaf`` parameters. +- Building trees: Random forests typically rely on deep trees (that overfit + individually) which uses much computational resources, as they require + several splittings and evaluations of candidate splits. Boosting models + build shallow trees (that underfit individually) which are faster to fit + and predict. -The number of bins used to bin the data is controlled with the ``max_bins`` -parameter. Using less bins acts as a form of regularization. It is -generally recommended to use as many bins as possible, which is the default. +- Sequential boosting: In HGBT, the decision trees are built sequentially, + where each tree is trained to correct the errors made by the previous ones. + This allows them to iteratively improve the model's performance using + relatively few trees. In contrast, random forests use a majority vote to + predict the outcome, which can require a larger number of trees to achieve + the same level of accuracy. -The ``l2_regularization`` parameter is a regularizer on the loss function and -corresponds to :math:`\lambda` in equation (2) of [XGBoost]_. +- Efficient binning: HGBT uses an efficient binning algorithm that can handle + large datasets with a high number of features. The binning algorithm can + pre-process the data to speed up the subsequent tree construction (see + :ref:`Why it's faster `). In contrast, the scikit-learn + implementation of random forests does not use binning and relies on exact + splitting, which can be computationally expensive. -Note that **early-stopping is enabled by default if the number of samples is -larger than 10,000**. The early-stopping behaviour is controlled via the -``early-stopping``, ``scoring``, ``validation_fraction``, -``n_iter_no_change``, and ``tol`` parameters. It is possible to early-stop -using an arbitrary :term:`scorer`, or just the training or validation loss. -Note that for technical reasons, using a scorer is significantly slower than -using the loss. By default, early-stopping is performed if there are at least -10,000 samples in the training set, using the validation loss. +Overall, the computational cost of HGBT versus RF depends on the specific +characteristics of the dataset and the modeling task. It's a good idea +to try both models and compare their performance and computational efficiency +on your specific problem to determine which model is the best fit. -Missing values support ----------------------- +.. rubric:: Examples -:class:`HistGradientBoostingClassifier` and -:class:`HistGradientBoostingRegressor` have built-in support for missing -values (NaNs). +* :ref:`sphx_glr_auto_examples_ensemble_plot_forest_hist_grad_boosting_comparison.py` -During training, the tree grower learns at each split point whether samples -with missing values should go to the left or right child, based on the -potential gain. When predicting, samples with missing values are assigned to -the left or right child consequently:: +Extremely Randomized Trees +-------------------------- - >>> from sklearn.experimental import enable_hist_gradient_boosting # noqa - >>> from sklearn.ensemble import HistGradientBoostingClassifier - >>> import numpy as np +In extremely randomized trees (see :class:`ExtraTreesClassifier` +and :class:`ExtraTreesRegressor` classes), randomness goes one step +further in the way splits are computed. As in random forests, a random +subset of candidate features is used, but instead of looking for the +most discriminative thresholds, thresholds are drawn at random for each +candidate feature and the best of these randomly-generated thresholds is +picked as the splitting rule. This usually allows to reduce the variance +of the model a bit more, at the expense of a slightly greater increase +in bias:: - >>> X = np.array([0, 1, 2, np.nan]).reshape(-1, 1) - >>> y = [0, 0, 1, 1] + >>> from sklearn.model_selection import cross_val_score + >>> from sklearn.datasets import make_blobs + >>> from sklearn.ensemble import RandomForestClassifier + >>> from sklearn.ensemble import ExtraTreesClassifier + >>> from sklearn.tree import DecisionTreeClassifier - >>> gbdt = HistGradientBoostingClassifier(min_samples_leaf=1).fit(X, y) - >>> gbdt.predict(X) - array([0, 0, 1, 1]) + >>> X, y = make_blobs(n_samples=10000, n_features=10, centers=100, + ... random_state=0) -When the missingness pattern is predictive, the splits can be done on -whether the feature value is missing or not:: + >>> clf = DecisionTreeClassifier(max_depth=None, min_samples_split=2, + ... random_state=0) + >>> scores = cross_val_score(clf, X, y, cv=5) + >>> scores.mean() + np.float64(0.98) - >>> X = np.array([0, np.nan, 1, 2, np.nan]).reshape(-1, 1) - >>> y = [0, 1, 0, 0, 1] - >>> gbdt = HistGradientBoostingClassifier(min_samples_leaf=1, - ... max_depth=2, - ... learning_rate=1, - ... max_iter=1).fit(X, y) - >>> gbdt.predict(X) - array([0, 1, 0, 0, 1]) + >>> clf = RandomForestClassifier(n_estimators=10, max_depth=None, + ... min_samples_split=2, random_state=0) + >>> scores = cross_val_score(clf, X, y, cv=5) + >>> scores.mean() + np.float64(0.999) -If no missing values were encountered for a given feature during training, -then samples with missing values are mapped to whichever child has the most -samples. + >>> clf = ExtraTreesClassifier(n_estimators=10, max_depth=None, + ... min_samples_split=2, random_state=0) + >>> scores = cross_val_score(clf, X, y, cv=5) + >>> scores.mean() > 0.999 + np.True_ -.. _sw_hgbdt: +.. figure:: ../auto_examples/ensemble/images/sphx_glr_plot_forest_iris_001.png + :target: ../auto_examples/ensemble/plot_forest_iris.html + :align: center + :scale: 75% -Sample weight support ---------------------- +.. _random_forest_parameters: + +Parameters +---------- + +The main parameters to adjust when using these methods is ``n_estimators`` and +``max_features``. The former is the number of trees in the forest. The larger +the better, but also the longer it will take to compute. In addition, note that +results will stop getting significantly better beyond a critical number of +trees. The latter is the size of the random subsets of features to consider +when splitting a node. The lower the greater the reduction of variance, but +also the greater the increase in bias. Empirical good default values are +``max_features=1.0`` or equivalently ``max_features=None`` (always considering +all features instead of a random subset) for regression problems, and +``max_features="sqrt"`` (using a random subset of size ``sqrt(n_features)``) +for classification tasks (where ``n_features`` is the number of features in +the data). The default value of ``max_features=1.0`` is equivalent to bagged +trees and more randomness can be achieved by setting smaller values (e.g. 0.3 +is a typical default in the literature). Good results are often achieved when +setting ``max_depth=None`` in combination with ``min_samples_split=2`` (i.e., +when fully developing the trees). Bear in mind though that these values are +usually not optimal, and might result in models that consume a lot of RAM. +The best parameter values should always be cross-validated. In addition, note +that in random forests, bootstrap samples are used by default +(``bootstrap=True``) while the default strategy for extra-trees is to use the +whole dataset (``bootstrap=False``). When using bootstrap sampling the +generalization error can be estimated on the left out or out-of-bag samples. +This can be enabled by setting ``oob_score=True``. + +.. note:: + + The size of the model with the default parameters is :math:`O( M * N * log (N) )`, + where :math:`M` is the number of trees and :math:`N` is the number of samples. + In order to reduce the size of the model, you can change these parameters: + ``min_samples_split``, ``max_leaf_nodes``, ``max_depth`` and ``min_samples_leaf``. + +Parallelization +--------------- + +Finally, this module also features the parallel construction of the trees +and the parallel computation of the predictions through the ``n_jobs`` +parameter. If ``n_jobs=k`` then computations are partitioned into +``k`` jobs, and run on ``k`` cores of the machine. If ``n_jobs=-1`` +then all cores available on the machine are used. Note that because of +inter-process communication overhead, the speedup might not be linear +(i.e., using ``k`` jobs will unfortunately not be ``k`` times as +fast). Significant speedup can still be achieved though when building +a large number of trees, or when building a single tree requires a fair +amount of time (e.g., on large datasets). + +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_ensemble_plot_forest_iris.py` +* :ref:`sphx_glr_auto_examples_miscellaneous_plot_multioutput_face_completion.py` + +.. rubric:: References + +.. [B2001] L. Breiman, "Random Forests", Machine Learning, 45(1), 5-32, 2001. + +.. [B1998] L. Breiman, "Arcing Classifiers", Annals of Statistics 1998. + +* P. Geurts, D. Ernst., and L. Wehenkel, "Extremely randomized + trees", Machine Learning, 63(1), 3-42, 2006. + +.. _random_forest_feature_importance: + +Feature importance evaluation +----------------------------- + +The relative rank (i.e. depth) of a feature used as a decision node in a +tree can be used to assess the relative importance of that feature with +respect to the predictability of the target variable. Features used at +the top of the tree contribute to the final prediction decision of a +larger fraction of the input samples. The **expected fraction of the +samples** they contribute to can thus be used as an estimate of the +**relative importance of the features**. In scikit-learn, the fraction of +samples a feature contributes to is combined with the decrease in impurity +from splitting them to create a normalized estimate of the predictive power +of that feature. + +By **averaging** the estimates of predictive ability over several randomized +trees one can **reduce the variance** of such an estimate and use it +for feature selection. This is known as the mean decrease in impurity, or MDI. +Refer to [L2014]_ for more information on MDI and feature importance +evaluation with Random Forests. + +.. warning:: + + The impurity-based feature importances computed on tree-based models suffer + from two flaws that can lead to misleading conclusions. First they are + computed on statistics derived from the training dataset and therefore **do + not necessarily inform us on which features are most important to make good + predictions on held-out dataset**. Secondly, **they favor high cardinality + features**, that is features with many unique values. + :ref:`permutation_importance` is an alternative to impurity-based feature + importance that does not suffer from these flaws. These two methods of + obtaining feature importance are explored in: + :ref:`sphx_glr_auto_examples_inspection_plot_permutation_importance.py`. -:class:`HistGradientBoostingClassifier` and -:class:`HistGradientBoostingRegressor` sample support weights during -:term:`fit`. +In practice those estimates are stored as an attribute named +``feature_importances_`` on the fitted model. This is an array with shape +``(n_features,)`` whose values are positive and sum to 1.0. The higher +the value, the more important is the contribution of the matching feature +to the prediction function. -The following toy example demonstrates how the model ignores the samples with -zero sample weights: +.. rubric:: Examples - >>> X = [[1, 0], - ... [1, 0], - ... [1, 0], - ... [0, 1]] - >>> y = [0, 0, 1, 0] - >>> # ignore the first 2 training samples by setting their weight to 0 - >>> sample_weight = [0, 0, 1, 1] - >>> gb = HistGradientBoostingClassifier(min_samples_leaf=1) - >>> gb.fit(X, y, sample_weight=sample_weight) - HistGradientBoostingClassifier(...) - >>> gb.predict([[1, 0]]) - array([1]) - >>> gb.predict_proba([[1, 0]])[0, 1] - 0.99... +* :ref:`sphx_glr_auto_examples_ensemble_plot_forest_importances.py` -As you can see, the `[1, 0]` is comfortably classified as `1` since the first -two samples are ignored due to their sample weights. +.. rubric:: References -Implementation detail: taking sample weights into account amounts to -multiplying the gradients (and the hessians) by the sample weights. Note that -the binning stage (specifically the quantiles computation) does not take the -weights into account. +.. [L2014] G. Louppe, :arxiv:`"Understanding Random Forests: From Theory to + Practice" <1407.7502>`, + PhD Thesis, U. of Liege, 2014. -.. _categorical_support_gbdt: +.. _random_trees_embedding: -Categorical Features Support ----------------------------- +Totally Random Trees Embedding +------------------------------ -:class:`HistGradientBoostingClassifier` and -:class:`HistGradientBoostingRegressor` have native support for categorical -features: they can consider splits on non-ordered, categorical data. +:class:`RandomTreesEmbedding` implements an unsupervised transformation of the +data. Using a forest of completely random trees, :class:`RandomTreesEmbedding` +encodes the data by the indices of the leaves a data point ends up in. This +index is then encoded in a one-of-K manner, leading to a high dimensional, +sparse binary coding. +This coding can be computed very efficiently and can then be used as a basis +for other learning tasks. +The size and sparsity of the code can be influenced by choosing the number of +trees and the maximum depth per tree. For each tree in the ensemble, the coding +contains one entry of one. The size of the coding is at most ``n_estimators * 2 +** max_depth``, the maximum number of leaves in the forest. -For datasets with categorical features, using the native categorical support -is often better than relying on one-hot encoding -(:class:`~sklearn.preprocessing.OneHotEncoder`), because one-hot encoding -requires more tree depth to achieve equivalent splits. It is also usually -better to rely on the native categorical support rather than to treat -categorical features as continuous (ordinal), which happens for ordinal-encoded -categorical data, since categories are nominal quantities where order does not -matter. +As neighboring data points are more likely to lie within the same leaf of a +tree, the transformation performs an implicit, non-parametric density +estimation. -To enable categorical support, a boolean mask can be passed to the -`categorical_features` parameter, indicating which feature is categorical. In -the following, the first feature will be treated as categorical and the -second feature as numerical:: +.. rubric:: Examples - >>> gbdt = HistGradientBoostingClassifier(categorical_features=[True, False]) +* :ref:`sphx_glr_auto_examples_ensemble_plot_random_forest_embedding.py` -Equivalently, one can pass a list of integers indicating the indices of the -categorical features:: +* :ref:`sphx_glr_auto_examples_manifold_plot_lle_digits.py` compares non-linear + dimensionality reduction techniques on handwritten digits. - >>> gbdt = HistGradientBoostingClassifier(categorical_features=[0]) +* :ref:`sphx_glr_auto_examples_ensemble_plot_feature_transformation.py` compares + supervised and unsupervised tree based feature transformations. -The cardinality of each categorical feature should be less than the `max_bins` -parameter, and each categorical feature is expected to be encoded in -`[0, max_bins - 1]`. To that end, it might be useful to pre-process the data -with an :class:`~sklearn.preprocessing.OrdinalEncoder` as done in -:ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_categorical.py`. +.. seealso:: -If there are missing values during training, the missing values will be -treated as a proper category. If there are no missing values during training, -then at prediction time, missing values are mapped to the child node that has -the most samples (just like for continuous features). When predicting, -categories that were not seen during fit time will be treated as missing -values. + :ref:`manifold` techniques can also be useful to derive non-linear + representations of feature space, also these approaches focus also on + dimensionality reduction. -**Split finding with categorical features**: The canonical way of considering -categorical splits in a tree is to consider -all of the :math:`2^{K - 1} - 1` partitions, where :math:`K` is the number of -categories. This can quickly become prohibitive when :math:`K` is large. -Fortunately, since gradient boosting trees are always regression trees (even -for classification problems), there exist a faster strategy that can yield -equivalent splits. First, the categories of a feature are sorted according to -the variance of the target, for each category `k`. Once the categories are -sorted, one can consider *continuous partitions*, i.e. treat the categories -as if they were ordered continuous values (see Fisher [Fisher1958]_ for a -formal proof). As a result, only :math:`K - 1` splits need to be considered -instead of :math:`2^{K - 1} - 1`. The initial sorting is a -:math:`\mathcal{O}(K \log(K))` operation, leading to a total complexity of -:math:`\mathcal{O}(K \log(K) + K)`, instead of :math:`\mathcal{O}(2^K)`. - -.. topic:: Examples: - - * :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_categorical.py` +.. _tree_ensemble_warm_start: -.. _monotonic_cst_gbdt: +Fitting additional trees +------------------------ -Monotonic Constraints ---------------------- +RandomForest, Extra-Trees and :class:`RandomTreesEmbedding` estimators all support +``warm_start=True`` which allows you to add more trees to an already fitted model. -Depending on the problem at hand, you may have prior knowledge indicating -that a given feature should in general have a positive (or negative) effect -on the target value. For example, all else being equal, a higher credit -score should increase the probability of getting approved for a loan. -Monotonic constraints allow you to incorporate such prior knowledge into the -model. +:: -A positive monotonic constraint is a constraint of the form: + >>> from sklearn.datasets import make_classification + >>> from sklearn.ensemble import RandomForestClassifier + + >>> X, y = make_classification(n_samples=100, random_state=1) + >>> clf = RandomForestClassifier(n_estimators=10) + >>> clf = clf.fit(X, y) # fit with 10 trees + >>> len(clf.estimators_) + 10 + >>> # set warm_start and increase num of estimators + >>> _ = clf.set_params(n_estimators=20, warm_start=True) + >>> _ = clf.fit(X, y) # fit additional 10 trees + >>> len(clf.estimators_) + 20 + +When ``random_state`` is also set, the internal random state is also preserved +between ``fit`` calls. This means that training a model once with ``n`` estimators is +the same as building the model iteratively via multiple ``fit`` calls, where the +final number of estimators is equal to ``n``. -:math:`x_1 \leq x_1' \implies F(x_1, x_2) \leq F(x_1', x_2)`, -where :math:`F` is the predictor with two features. +:: -Similarly, a negative monotonic constraint is of the form: + >>> clf = RandomForestClassifier(n_estimators=20) # set `n_estimators` to 10 + 10 + >>> _ = clf.fit(X, y) # fit `estimators_` will be the same as `clf` above -:math:`x_1 \leq x_1' \implies F(x_1, x_2) \geq F(x_1', x_2)`. +Note that this differs from the usual behavior of :term:`random_state` in that it does +*not* result in the same result across different calls. -Note that monotonic constraints only constraint the output "all else being -equal". Indeed, the following relation **is not enforced** by a positive -constraint: :math:`x_1 \leq x_1' \implies F(x_1, x_2) \leq F(x_1', x_2')`. +.. _bagging: -You can specify a monotonic constraint on each feature using the -`monotonic_cst` parameter. For each feature, a value of 0 indicates no -constraint, while -1 and 1 indicate a negative and positive constraint, -respectively:: +Bagging meta-estimator +====================== - >>> from sklearn.experimental import enable_hist_gradient_boosting # noqa - >>> from sklearn.ensemble import HistGradientBoostingRegressor +In ensemble algorithms, bagging methods form a class of algorithms which build +several instances of a black-box estimator on random subsets of the original +training set and then aggregate their individual predictions to form a final +prediction. These methods are used as a way to reduce the variance of a base +estimator (e.g., a decision tree), by introducing randomization into its +construction procedure and then making an ensemble out of it. In many cases, +bagging methods constitute a very simple way to improve with respect to a +single model, without making it necessary to adapt the underlying base +algorithm. As they provide a way to reduce overfitting, bagging methods work +best with strong and complex models (e.g., fully developed decision trees), in +contrast with boosting methods which usually work best with weak models (e.g., +shallow decision trees). - ... # positive, negative, and no constraint on the 3 features - >>> gbdt = HistGradientBoostingRegressor(monotonic_cst=[1, -1, 0]) +Bagging methods come in many flavours but mostly differ from each other by the +way they draw random subsets of the training set: -In a binary classification context, imposing a monotonic constraint means -that the feature is supposed to have a positive / negative effect on the -probability to belong to the positive class. Monotonic constraints are not -supported for multiclass context. +* When random subsets of the dataset are drawn as random subsets of the + samples, then this algorithm is known as Pasting [B1999]_. -.. note:: - Since categories are unordered quantities, it is not possible to enforce - monotonic constraints on categorical features. +* When samples are drawn with replacement, then the method is known as + Bagging [B1996]_. -.. topic:: Examples: +* When random subsets of the dataset are drawn as random subsets of + the features, then the method is known as Random Subspaces [H1998]_. - * :ref:`sphx_glr_auto_examples_ensemble_plot_monotonic_constraints.py` +* Finally, when base estimators are built on subsets of both samples and + features, then the method is known as Random Patches [LG2012]_. -Low-level parallelism ---------------------- +In scikit-learn, bagging methods are offered as a unified +:class:`BaggingClassifier` meta-estimator (resp. :class:`BaggingRegressor`), +taking as input a user-specified estimator along with parameters +specifying the strategy to draw random subsets. In particular, ``max_samples`` +and ``max_features`` control the size of the subsets (in terms of samples and +features), while ``bootstrap`` and ``bootstrap_features`` control whether +samples and features are drawn with or without replacement. When using a subset +of the available samples the generalization accuracy can be estimated with the +out-of-bag samples by setting ``oob_score=True``. As an example, the +snippet below illustrates how to instantiate a bagging ensemble of +:class:`~sklearn.neighbors.KNeighborsClassifier` estimators, each built on random +subsets of 50% of the samples and 50% of the features. -:class:`HistGradientBoostingClassifier` and -:class:`HistGradientBoostingRegressor` have implementations that use OpenMP -for parallelization through Cython. For more details on how to control the -number of threads, please refer to our :ref:`parallelism` notes. + >>> from sklearn.ensemble import BaggingClassifier + >>> from sklearn.neighbors import KNeighborsClassifier + >>> bagging = BaggingClassifier(KNeighborsClassifier(), + ... max_samples=0.5, max_features=0.5) -The following parts are parallelized: +.. rubric:: Examples -- mapping samples from real values to integer-valued bins (finding the bin - thresholds is however sequential) -- building histograms is parallelized over features -- finding the best split point at a node is parallelized over features -- during fit, mapping samples into the left and right children is - parallelized over samples -- gradient and hessians computations are parallelized over samples -- predicting is parallelized over samples +* :ref:`sphx_glr_auto_examples_ensemble_plot_bias_variance.py` -Why it's faster ---------------- +.. rubric:: References -The bottleneck of a gradient boosting procedure is building the decision -trees. Building a traditional decision tree (as in the other GBDTs -:class:`GradientBoostingClassifier` and :class:`GradientBoostingRegressor`) -requires sorting the samples at each node (for -each feature). Sorting is needed so that the potential gain of a split point -can be computed efficiently. Splitting a single node has thus a complexity -of :math:`\mathcal{O}(n_\text{features} \times n \log(n))` where :math:`n` -is the number of samples at the node. +.. [B1999] L. Breiman, "Pasting small votes for classification in large + databases and on-line", Machine Learning, 36(1), 85-103, 1999. -:class:`HistGradientBoostingClassifier` and -:class:`HistGradientBoostingRegressor`, in contrast, do not require sorting the -feature values and instead use a data-structure called a histogram, where the -samples are implicitly ordered. Building a histogram has a -:math:`\mathcal{O}(n)` complexity, so the node splitting procedure has a -:math:`\mathcal{O}(n_\text{features} \times n)` complexity, much smaller -than the previous one. In addition, instead of considering :math:`n` split -points, we here consider only ``max_bins`` split points, which is much -smaller. +.. [B1996] L. Breiman, "Bagging predictors", Machine Learning, 24(2), + 123-140, 1996. -In order to build histograms, the input data `X` needs to be binned into -integer-valued bins. This binning procedure does require sorting the feature -values, but it only happens once at the very beginning of the boosting process -(not at each node, like in :class:`GradientBoostingClassifier` and -:class:`GradientBoostingRegressor`). +.. [H1998] T. Ho, "The random subspace method for constructing decision + forests", Pattern Analysis and Machine Intelligence, 20(8), 832-844, 1998. -Finally, many parts of the implementation of -:class:`HistGradientBoostingClassifier` and -:class:`HistGradientBoostingRegressor` are parallelized. +.. [LG2012] G. Louppe and P. Geurts, "Ensembles on Random Patches", + Machine Learning and Knowledge Discovery in Databases, 346-361, 2012. -.. topic:: References - .. [F1999] Friedmann, Jerome H., 2007, `"Stochastic Gradient Boosting" - `_ - .. [R2007] G. Ridgeway, "Generalized Boosted Models: A guide to the gbm - package", 2007 - .. [XGBoost] Tianqi Chen, Carlos Guestrin, `"XGBoost: A Scalable Tree - Boosting System" `_ - .. [LightGBM] Ke et. al. `"LightGBM: A Highly Efficient Gradient - BoostingDecision Tree" `_ - .. [Fisher1958] Walter D. Fisher. `"On Grouping for Maximum Homogeneity" - `_ .. _voting_classifier: @@ -1235,7 +1327,7 @@ Voting Classifier The idea behind the :class:`VotingClassifier` is to combine conceptually different machine learning classifiers and use a majority vote or the average predicted probabilities (soft vote) to predict the class labels. -Such a classifier can be useful for a set of equally well performing model +Such a classifier can be useful for a set of equally well performing models in order to balance out their individual weaknesses. @@ -1308,7 +1400,7 @@ and averaged. The final class label is then derived from the class label with the highest average probability. To illustrate this with a simple example, let's assume we have 3 -classifiers and a 3-class classification problems where we assign +classifiers and a 3-class classification problem where we assign equal weights to all classifiers: w1=1, w2=1, w3=1. The weighted average probabilities for a sample would then be @@ -1317,69 +1409,27 @@ calculated as follows: ================ ========== ========== ========== classifier class 1 class 2 class 3 ================ ========== ========== ========== -classifier 1 w1 * 0.2 w1 * 0.5 w1 * 0.3 -classifier 2 w2 * 0.6 w2 * 0.3 w2 * 0.1 +classifier 1 w1 * 0.2 w1 * 0.5 w1 * 0.3 +classifier 2 w2 * 0.6 w2 * 0.3 w2 * 0.1 classifier 3 w3 * 0.3 w3 * 0.4 w3 * 0.3 -weighted average 0.37 0.4 0.23 +weighted average 0.37 0.4 0.23 ================ ========== ========== ========== -Here, the predicted class label is 2, since it has the -highest average probability. - -The following example illustrates how the decision regions may change -when a soft :class:`VotingClassifier` is used based on an linear Support -Vector Machine, a Decision Tree, and a K-nearest neighbor classifier:: - - >>> from sklearn import datasets - >>> from sklearn.tree import DecisionTreeClassifier - >>> from sklearn.neighbors import KNeighborsClassifier - >>> from sklearn.svm import SVC - >>> from itertools import product - >>> from sklearn.ensemble import VotingClassifier +Here, the predicted class label is 2, since it has the highest average +predicted probability. See the example on +:ref:`sphx_glr_auto_examples_ensemble_plot_voting_decision_regions.py` for a +demonstration of how the predicted class label can be obtained from the weighted +average of predicted probabilities. - >>> # Loading some example data - >>> iris = datasets.load_iris() - >>> X = iris.data[:, [0, 2]] - >>> y = iris.target +The following figure illustrates how the decision regions may change when +a soft :class:`VotingClassifier` is trained with weights on three linear +models: - >>> # Training classifiers - >>> clf1 = DecisionTreeClassifier(max_depth=4) - >>> clf2 = KNeighborsClassifier(n_neighbors=7) - >>> clf3 = SVC(kernel='rbf', probability=True) - >>> eclf = VotingClassifier(estimators=[('dt', clf1), ('knn', clf2), ('svc', clf3)], - ... voting='soft', weights=[2, 1, 2]) - - >>> clf1 = clf1.fit(X, y) - >>> clf2 = clf2.fit(X, y) - >>> clf3 = clf3.fit(X, y) - >>> eclf = eclf.fit(X, y) - -.. figure:: ../auto_examples/ensemble/images/sphx_glr_plot_voting_decision_regions_001.png +.. figure:: ../auto_examples/ensemble/images/sphx_glr_plot_voting_decision_regions_002.png :target: ../auto_examples/ensemble/plot_voting_decision_regions.html :align: center :scale: 75% -Using the `VotingClassifier` with `GridSearchCV` ------------------------------------------------- - -The :class:`VotingClassifier` can also be used together with -:class:`~sklearn.model_selection.GridSearchCV` in order to tune the -hyperparameters of the individual estimators:: - - >>> from sklearn.model_selection import GridSearchCV - >>> clf1 = LogisticRegression(random_state=1) - >>> clf2 = RandomForestClassifier(random_state=1) - >>> clf3 = GaussianNB() - >>> eclf = VotingClassifier( - ... estimators=[('lr', clf1), ('rf', clf2), ('gnb', clf3)], - ... voting='soft' - ... ) - - >>> params = {'lr__C': [1.0, 100.0], 'rf__n_estimators': [20, 200]} - - >>> grid = GridSearchCV(estimator=eclf, param_grid=params, cv=5) - >>> grid = grid.fit(iris.data, iris.target) - Usage ----- @@ -1399,6 +1449,26 @@ Optionally, weights can be provided for the individual classifiers:: ... voting='soft', weights=[2,5,1] ... ) +.. dropdown:: Using the :class:`VotingClassifier` with :class:`~sklearn.model_selection.GridSearchCV` + + The :class:`VotingClassifier` can also be used together with + :class:`~sklearn.model_selection.GridSearchCV` in order to tune the + hyperparameters of the individual estimators:: + + >>> from sklearn.model_selection import GridSearchCV + >>> clf1 = LogisticRegression(random_state=1) + >>> clf2 = RandomForestClassifier(random_state=1) + >>> clf3 = GaussianNB() + >>> eclf = VotingClassifier( + ... estimators=[('lr', clf1), ('rf', clf2), ('gnb', clf3)], + ... voting='soft' + ... ) + + >>> params = {'lr__C': [1.0, 100.0], 'rf__n_estimators': [20, 200]} + + >>> grid = GridSearchCV(estimator=eclf, param_grid=params, cv=5) + >>> grid = grid.fit(iris.data, iris.target) + .. _voting_regressor: Voting Regressor @@ -1435,9 +1505,9 @@ The following example shows how to fit the VotingRegressor:: :align: center :scale: 75% -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_ensemble_plot_voting_regressor.py` +* :ref:`sphx_glr_auto_examples_ensemble_plot_voting_regressor.py` .. _stacking: @@ -1501,8 +1571,8 @@ availability, tested in the order of preference: `predict_proba`, `decision_function` and `predict`. A :class:`StackingRegressor` and :class:`StackingClassifier` can be used as -any other regressor or classifier, exposing a `predict`, `predict_proba`, and -`decision_function` methods, e.g.:: +any other regressor or classifier, exposing a `predict`, `predict_proba`, or +`decision_function` method, e.g.:: >>> y_pred = reg.predict(X_test) >>> from sklearn.metrics import r2_score @@ -1513,15 +1583,15 @@ Note that it is also possible to get the output of the stacked `estimators` using the `transform` method:: >>> reg.transform(X_test[:5]) - array([[142..., 138..., 146...], - [179..., 182..., 151...], - [139..., 132..., 158...], - [286..., 292..., 225...], - [126..., 124..., 164...]]) + array([[142, 138, 146], + [179, 182, 151], + [139, 132, 158], + [286, 292, 225], + [126, 124, 164]]) In practice, a stacking predictor predicts as good as the best predictor of the base layer and even sometimes outperforms it by combining the different -strengths of the these predictors. However, training a stacking predictor is +strengths of these predictors. However, training a stacking predictor is computationally expensive. .. note:: @@ -1556,7 +1626,99 @@ computationally expensive. ... .format(multi_layer_regressor.score(X_test, y_test))) R2 score: 0.53 -.. topic:: References +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_ensemble_plot_stack_predictors.py` + +.. rubric:: References + +.. [W1992] Wolpert, David H. "Stacked generalization." Neural networks 5.2 + (1992): 241-259. + + + +.. _adaboost: + +AdaBoost +======== + +The module :mod:`sklearn.ensemble` includes the popular boosting algorithm +AdaBoost, introduced in 1995 by Freund and Schapire [FS1995]_. + +The core principle of AdaBoost is to fit a sequence of weak learners (i.e., +models that are only slightly better than random guessing, such as small +decision trees) on repeatedly modified versions of the data. The predictions +from all of them are then combined through a weighted majority vote (or sum) to +produce the final prediction. The data modifications at each so-called boosting +iteration consists of applying weights :math:`w_1`, :math:`w_2`, ..., :math:`w_N` +to each of the training samples. Initially, those weights are all set to +:math:`w_i = 1/N`, so that the first step simply trains a weak learner on the +original data. For each successive iteration, the sample weights are +individually modified and the learning algorithm is reapplied to the reweighted +data. At a given step, those training examples that were incorrectly predicted +by the boosted model induced at the previous step have their weights increased, +whereas the weights are decreased for those that were predicted correctly. As +iterations proceed, examples that are difficult to predict receive +ever-increasing influence. Each subsequent weak learner is thereby forced to +concentrate on the examples that are missed by the previous ones in the sequence +[HTF]_. + +.. figure:: ../auto_examples/ensemble/images/sphx_glr_plot_adaboost_multiclass_001.png + :target: ../auto_examples/ensemble/plot_adaboost_multiclass.html + :align: center + :scale: 75 + +AdaBoost can be used both for classification and regression problems: + +- For multi-class classification, :class:`AdaBoostClassifier` implements + AdaBoost.SAMME [ZZRH2009]_. + +- For regression, :class:`AdaBoostRegressor` implements AdaBoost.R2 [D1997]_. + +Usage +----- + +The following example shows how to fit an AdaBoost classifier with 100 weak +learners:: + + >>> from sklearn.model_selection import cross_val_score + >>> from sklearn.datasets import load_iris + >>> from sklearn.ensemble import AdaBoostClassifier + + >>> X, y = load_iris(return_X_y=True) + >>> clf = AdaBoostClassifier(n_estimators=100) + >>> scores = cross_val_score(clf, X, y, cv=5) + >>> scores.mean() + np.float64(0.95) + +The number of weak learners is controlled by the parameter ``n_estimators``. The +``learning_rate`` parameter controls the contribution of the weak learners in +the final combination. By default, weak learners are decision stumps. Different +weak learners can be specified through the ``estimator`` parameter. +The main parameters to tune to obtain good results are ``n_estimators`` and +the complexity of the base estimators (e.g., its depth ``max_depth`` or +minimum required number of samples to consider a split ``min_samples_split``). + +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_ensemble_plot_adaboost_multiclass.py` shows the performance + of AdaBoost on a multi-class problem. + +* :ref:`sphx_glr_auto_examples_ensemble_plot_adaboost_twoclass.py` shows the decision boundary + and decision function values for a non-linearly separable two-class problem + using AdaBoost-SAMME. + +* :ref:`sphx_glr_auto_examples_ensemble_plot_adaboost_regression.py` demonstrates regression + with the AdaBoost.R2 algorithm. + +.. rubric:: References + +.. [FS1995] Y. Freund, and R. Schapire, "A Decision-Theoretic Generalization of + On-Line Learning and an Application to Boosting", 1997. + +.. [ZZRH2009] J. Zhu, H. Zou, S. Rosset, T. Hastie. "Multi-class AdaBoost", 2009. + +.. [D1997] H. Drucker. "Improving Regressors using Boosting Techniques", 1997. - .. [W1992] Wolpert, David H. "Stacked generalization." Neural networks 5.2 - (1992): 241-259. +.. [HTF] T. Hastie, R. Tibshirani and J. Friedman, "Elements of Statistical Learning + Ed. 2", Springer, 2009. diff --git a/doc/modules/feature_extraction.rst b/doc/modules/feature_extraction.rst index e074d93fad7b8..42bcf18e1d572 100644 --- a/doc/modules/feature_extraction.rst +++ b/doc/modules/feature_extraction.rst @@ -1,4 +1,4 @@ -īģŋ.. _feature_extraction: +.. _feature_extraction: ================== Feature extraction @@ -13,9 +13,9 @@ consisting of formats such as text and image. .. note:: Feature extraction is very different from :ref:`feature_selection`: - the former consists in transforming arbitrary data, such as text or + the former consists of transforming arbitrary data, such as text or images, into numerical features usable for machine learning. The latter - is a machine learning technique applied on these features. + is a machine learning technique applied to these features. .. _dict_feature_extraction: @@ -33,7 +33,7 @@ need not be stored) and storing feature names in addition to values. :class:`DictVectorizer` implements what is called one-of-K or "one-hot" coding for categorical (aka nominal, discrete) features. Categorical features are "attribute-value" pairs where the value is restricted -to a list of discrete of possibilities without ordering (e.g. topic +to a list of discrete possibilities without ordering (e.g. topic identifiers, types of objects, tags, names...). In the following, "city" is a categorical attribute while "temperature" @@ -53,13 +53,13 @@ is a traditional numerical feature:: [ 0., 1., 0., 12.], [ 0., 0., 1., 18.]]) - >>> vec.get_feature_names() - ['city=Dubai', 'city=London', 'city=San Francisco', 'temperature'] + >>> vec.get_feature_names_out() + array(['city=Dubai', 'city=London', 'city=San Francisco', 'temperature'], ...) :class:`DictVectorizer` accepts multiple string values for one feature, like, e.g., multiple categories for a movie. -Assume a database classifies each movie using some categories (not mandatories) +Assume a database classifies each movie using some categories (not mandatory) and its year of release. >>> movie_entry = [{'category': ['thriller', 'drama'], 'year': 2003}, @@ -69,10 +69,9 @@ and its year of release. array([[0.000e+00, 1.000e+00, 0.000e+00, 1.000e+00, 2.003e+03], [1.000e+00, 0.000e+00, 1.000e+00, 0.000e+00, 2.011e+03], [0.000e+00, 0.000e+00, 0.000e+00, 0.000e+00, 1.974e+03]]) - >>> vec.get_feature_names() == ['category=animation', 'category=drama', - ... 'category=family', 'category=thriller', - ... 'year'] - True + >>> vec.get_feature_names_out() + array(['category=animation', 'category=drama', 'category=family', + 'category=thriller', 'year'], ...) >>> vec.transform({'category': ['thriller'], ... 'unseen_feature': '3'}).toarray() array([[0., 0., 0., 1., 0.]]) @@ -107,12 +106,13 @@ suitable for feeding into a classifier (maybe after being piped into a >>> vec = DictVectorizer() >>> pos_vectorized = vec.fit_transform(pos_window) >>> pos_vectorized - <1x6 sparse matrix of type '<... 'numpy.float64'>' - with 6 stored elements in Compressed Sparse ... format> + >>> pos_vectorized.toarray() array([[1., 1., 1., 1., 1., 1.]]) - >>> vec.get_feature_names() - ['pos+1=PP', 'pos-1=NN', 'pos-2=DT', 'word+1=on', 'word-1=cat', 'word-2=the'] + >>> vec.get_feature_names_out() + array(['pos+1=PP', 'pos-1=NN', 'pos-2=DT', 'word+1=on', 'word-1=cat', + 'word-2=the'], ...) As you can imagine, if one extracts such a context around each individual word of a corpus of documents the resulting matrix will be very wide @@ -158,7 +158,7 @@ feature selectors that expect non-negative inputs. (like Python's ``dict`` and its variants in the ``collections`` module), ``(feature, value)`` pairs, or strings, depending on the constructor parameter ``input_type``. -Mapping are treated as lists of ``(feature, value)`` pairs, +Mappings are treated as lists of ``(feature, value)`` pairs, while single strings have an implicit value of 1, so ``['feat1', 'feat2', 'feat3']`` is interpreted as ``[('feat1', 1), ('feat2', 1), ('feat3', 1)]``. @@ -206,32 +206,32 @@ Note the use of a generator comprehension, which introduces laziness into the feature extraction: tokens are only processed on demand from the hasher. -Implementation details ----------------------- +.. dropdown:: Implementation details -:class:`FeatureHasher` uses the signed 32-bit variant of MurmurHash3. -As a result (and because of limitations in ``scipy.sparse``), -the maximum number of features supported is currently :math:`2^{31} - 1`. + :class:`FeatureHasher` uses the signed 32-bit variant of MurmurHash3. + As a result (and because of limitations in ``scipy.sparse``), + the maximum number of features supported is currently :math:`2^{31} - 1`. -The original formulation of the hashing trick by Weinberger et al. -used two separate hash functions :math:`h` and :math:`\xi` -to determine the column index and sign of a feature, respectively. -The present implementation works under the assumption -that the sign bit of MurmurHash3 is independent of its other bits. + The original formulation of the hashing trick by Weinberger et al. + used two separate hash functions :math:`h` and :math:`\xi` + to determine the column index and sign of a feature, respectively. + The present implementation works under the assumption + that the sign bit of MurmurHash3 is independent of its other bits. -Since a simple modulo is used to transform the hash function to a column index, -it is advisable to use a power of two as the ``n_features`` parameter; -otherwise the features will not be mapped evenly to the columns. + Since a simple modulo is used to transform the hash function to a column index, + it is advisable to use a power of two as the ``n_features`` parameter; + otherwise the features will not be mapped evenly to the columns. + .. rubric:: References -.. topic:: References: + * `MurmurHash3 `_. - * Kilian Weinberger, Anirban Dasgupta, John Langford, Alex Smola and - Josh Attenberg (2009). `Feature hashing for large scale multitask learning - `_. Proc. ICML. - * `MurmurHash3 `_. +.. rubric:: References +* Kilian Weinberger, Anirban Dasgupta, John Langford, Alex Smola and + Josh Attenberg (2009). `Feature hashing for large scale multitask learning + `_. Proc. ICML. .. _text_feature_extraction: @@ -245,7 +245,7 @@ The Bag of Words representation ------------------------------- Text Analysis is a major application field for machine learning -algorithms. However the raw data, a sequence of symbols cannot be fed +algorithms. However the raw data, a sequence of symbols, cannot be fed directly to the algorithms themselves as most of them expect numerical feature vectors with a fixed size rather than the raw text documents with variable length. @@ -307,7 +307,7 @@ counting in a single class:: This model has many parameters, however the default values are quite reasonable (please see the :ref:`reference documentation -` for the details):: +` for the details):: >>> vectorizer = CountVectorizer() >>> vectorizer @@ -324,8 +324,8 @@ corpus of text documents:: ... ] >>> X = vectorizer.fit_transform(corpus) >>> X - <4x9 sparse matrix of type '<... 'numpy.int64'>' - with 19 stored elements in Compressed Sparse ... format> + The default configuration tokenizes the string by extracting words of at least 2 letters. The specific function that does this step can be @@ -340,10 +340,9 @@ Each term found by the analyzer during the fit is assigned a unique integer index corresponding to a column in the resulting matrix. This interpretation of the columns can be retrieved as follows:: - >>> vectorizer.get_feature_names() == ( - ... ['and', 'document', 'first', 'is', 'one', - ... 'second', 'the', 'third', 'this']) - True + >>> vectorizer.get_feature_names_out() + array(['and', 'document', 'first', 'is', 'one', 'second', 'the', + 'third', 'this'], ...) >>> X.toarray() array([[0, 1, 1, 1, 0, 0, 1, 0, 1], @@ -397,17 +396,17 @@ last document:: .. _stop_words: Using stop words -................ +---------------- Stop words are words like "and", "the", "him", which are presumed to be uninformative in representing the content of a text, and which may be -removed to avoid them being construed as signal for prediction. Sometimes, +removed to avoid them being construed as informative for prediction. Sometimes, however, similar words are useful for prediction, such as in classifying writing style or personality. There are several known issues in our provided 'english' stop word list. It -does not aim to be a general, 'one-size-fits-all' solution as some tasks -may require a more custom solution. See [NQY18]_ for more details. +does not aim to be a general, 'one-size-fits-all' solution as some tasks +may require a more custom solution. See [NQY18]_ for more details. Please take care in choosing a stop word list. Popular stop word lists may include words that are highly informative to @@ -420,12 +419,13 @@ tokenizer, so if *we've* is in ``stop_words``, but *ve* is not, *ve* will be retained from *we've* in transformed text. Our vectorizers will try to identify and warn about some kinds of inconsistencies. -.. topic:: References +.. rubric:: References + +.. [NQY18] J. Nothman, H. Qin and R. Yurchak (2018). + `"Stop Word Lists in Free Open-source Software Packages" + `__. + In *Proc. Workshop for NLP Open Source Software*. - .. [NQY18] J. Nothman, H. Qin and R. Yurchak (2018). - `"Stop Word Lists in Free Open-source Software Packages" - `__. - In *Proc. Workshop for NLP Open Source Software*. .. _tfidf: @@ -489,126 +489,125 @@ class:: TfidfTransformer(smooth_idf=False) Again please see the :ref:`reference documentation -` for the details on all the parameters. - -Let's take an example with the following counts. The first term is present -100% of the time hence not very interesting. The two other features only -in less than 50% of the time hence probably more representative of the -content of the documents:: - - >>> counts = [[3, 0, 1], - ... [2, 0, 0], - ... [3, 0, 0], - ... [4, 0, 0], - ... [3, 2, 0], - ... [3, 0, 2]] - ... - >>> tfidf = transformer.fit_transform(counts) - >>> tfidf - <6x3 sparse matrix of type '<... 'numpy.float64'>' - with 9 stored elements in Compressed Sparse ... format> +` for the details on all the parameters. - >>> tfidf.toarray() - array([[0.81940995, 0. , 0.57320793], - [1. , 0. , 0. ], - [1. , 0. , 0. ], - [1. , 0. , 0. ], - [0.47330339, 0.88089948, 0. ], - [0.58149261, 0. , 0.81355169]]) +.. dropdown:: Numeric example of a tf-idf matrix -Each row is normalized to have unit Euclidean norm: + Let's take an example with the following counts. The first term is present + 100% of the time hence not very interesting. The two other features only + in less than 50% of the time hence probably more representative of the + content of the documents:: -:math:`v_{norm} = \frac{v}{||v||_2} = \frac{v}{\sqrt{v{_1}^2 + -v{_2}^2 + \dots + v{_n}^2}}` + >>> counts = [[3, 0, 1], + ... [2, 0, 0], + ... [3, 0, 0], + ... [4, 0, 0], + ... [3, 2, 0], + ... [3, 0, 2]] + ... + >>> tfidf = transformer.fit_transform(counts) + >>> tfidf + -For example, we can compute the tf-idf of the first term in the first -document in the `counts` array as follows: + >>> tfidf.toarray() + array([[0.81940995, 0. , 0.57320793], + [1. , 0. , 0. ], + [1. , 0. , 0. ], + [1. , 0. , 0. ], + [0.47330339, 0.88089948, 0. ], + [0.58149261, 0. , 0.81355169]]) -:math:`n = 6` + Each row is normalized to have unit Euclidean norm: -:math:`\text{df}(t)_{\text{term1}} = 6` + :math:`v_{norm} = \frac{v}{||v||_2} = \frac{v}{\sqrt{v{_1}^2 + + v{_2}^2 + \dots + v{_n}^2}}` -:math:`\text{idf}(t)_{\text{term1}} = -\log \frac{n}{\text{df}(t)} + 1 = \log(1)+1 = 1` + For example, we can compute the tf-idf of the first term in the first + document in the `counts` array as follows: -:math:`\text{tf-idf}_{\text{term1}} = \text{tf} \times \text{idf} = 3 \times 1 = 3` + :math:`n = 6` -Now, if we repeat this computation for the remaining 2 terms in the document, -we get + :math:`\text{df}(t)_{\text{term1}} = 6` -:math:`\text{tf-idf}_{\text{term2}} = 0 \times (\log(6/1)+1) = 0` + :math:`\text{idf}(t)_{\text{term1}} = + \log \frac{n}{\text{df}(t)} + 1 = \log(1)+1 = 1` -:math:`\text{tf-idf}_{\text{term3}} = 1 \times (\log(6/2)+1) \approx 2.0986` + :math:`\text{tf-idf}_{\text{term1}} = \text{tf} \times \text{idf} = 3 \times 1 = 3` -and the vector of raw tf-idfs: + Now, if we repeat this computation for the remaining 2 terms in the document, + we get -:math:`\text{tf-idf}_{\text{raw}} = [3, 0, 2.0986].` + :math:`\text{tf-idf}_{\text{term2}} = 0 \times (\log(6/1)+1) = 0` + :math:`\text{tf-idf}_{\text{term3}} = 1 \times (\log(6/2)+1) \approx 2.0986` -Then, applying the Euclidean (L2) norm, we obtain the following tf-idfs -for document 1: + and the vector of raw tf-idfs: -:math:`\frac{[3, 0, 2.0986]}{\sqrt{\big(3^2 + 0^2 + 2.0986^2\big)}} -= [ 0.819, 0, 0.573].` + :math:`\text{tf-idf}_{\text{raw}} = [3, 0, 2.0986].` -Furthermore, the default parameter ``smooth_idf=True`` adds "1" to the numerator -and denominator as if an extra document was seen containing every term in the -collection exactly once, which prevents zero divisions: -:math:`\text{idf}(t) = \log{\frac{1 + n}{1+\text{df}(t)}} + 1` + Then, applying the Euclidean (L2) norm, we obtain the following tf-idfs + for document 1: -Using this modification, the tf-idf of the third term in document 1 changes to -1.8473: + :math:`\frac{[3, 0, 2.0986]}{\sqrt{\big(3^2 + 0^2 + 2.0986^2\big)}} + = [ 0.819, 0, 0.573].` -:math:`\text{tf-idf}_{\text{term3}} = 1 \times \log(7/3)+1 \approx 1.8473` + Furthermore, the default parameter ``smooth_idf=True`` adds "1" to the numerator + and denominator as if an extra document was seen containing every term in the + collection exactly once, which prevents zero divisions: -And the L2-normalized tf-idf changes to + :math:`\text{idf}(t) = \log{\frac{1 + n}{1+\text{df}(t)}} + 1` -:math:`\frac{[3, 0, 1.8473]}{\sqrt{\big(3^2 + 0^2 + 1.8473^2\big)}} -= [0.8515, 0, 0.5243]`:: + Using this modification, the tf-idf of the third term in document 1 changes to + 1.8473: - >>> transformer = TfidfTransformer() - >>> transformer.fit_transform(counts).toarray() - array([[0.85151335, 0. , 0.52433293], - [1. , 0. , 0. ], - [1. , 0. , 0. ], - [1. , 0. , 0. ], - [0.55422893, 0.83236428, 0. ], - [0.63035731, 0. , 0.77630514]]) + :math:`\text{tf-idf}_{\text{term3}} = 1 \times \log(7/3)+1 \approx 1.8473` -The weights of each -feature computed by the ``fit`` method call are stored in a model -attribute:: + And the L2-normalized tf-idf changes to - >>> transformer.idf_ - array([1. ..., 2.25..., 1.84...]) + :math:`\frac{[3, 0, 1.8473]}{\sqrt{\big(3^2 + 0^2 + 1.8473^2\big)}} + = [0.8515, 0, 0.5243]`:: + >>> transformer = TfidfTransformer() + >>> transformer.fit_transform(counts).toarray() + array([[0.85151335, 0. , 0.52433293], + [1. , 0. , 0. ], + [1. , 0. , 0. ], + [1. , 0. , 0. ], + [0.55422893, 0.83236428, 0. ], + [0.63035731, 0. , 0.77630514]]) + The weights of each + feature computed by the ``fit`` method call are stored in a model + attribute:: + >>> transformer.idf_ + array([1., 2.25, 1.84]) -As tf–idf is very often used for text features, there is also another -class called :class:`TfidfVectorizer` that combines all the options of -:class:`CountVectorizer` and :class:`TfidfTransformer` in a single model:: + As tf-idf is very often used for text features, there is also another + class called :class:`TfidfVectorizer` that combines all the options of + :class:`CountVectorizer` and :class:`TfidfTransformer` in a single model:: - >>> from sklearn.feature_extraction.text import TfidfVectorizer - >>> vectorizer = TfidfVectorizer() - >>> vectorizer.fit_transform(corpus) - <4x9 sparse matrix of type '<... 'numpy.float64'>' - with 19 stored elements in Compressed Sparse ... format> + >>> from sklearn.feature_extraction.text import TfidfVectorizer + >>> vectorizer = TfidfVectorizer() + >>> vectorizer.fit_transform(corpus) + -While the tf–idf normalization is often very useful, there might -be cases where the binary occurrence markers might offer better -features. This can be achieved by using the ``binary`` parameter -of :class:`CountVectorizer`. In particular, some estimators such as -:ref:`bernoulli_naive_bayes` explicitly model discrete boolean random -variables. Also, very short texts are likely to have noisy tf–idf values -while the binary occurrence info is more stable. + While the tf-idf normalization is often very useful, there might + be cases where the binary occurrence markers might offer better + features. This can be achieved by using the ``binary`` parameter + of :class:`CountVectorizer`. In particular, some estimators such as + :ref:`bernoulli_naive_bayes` explicitly model discrete boolean random + variables. Also, very short texts are likely to have noisy tf-idf values + while the binary occurrence info is more stable. -As usual the best way to adjust the feature extraction parameters -is to use a cross-validated grid search, for instance by pipelining the -feature extractor with a classifier: + As usual the best way to adjust the feature extraction parameters + is to use a cross-validated grid search, for instance by pipelining the + feature extractor with a classifier: - * :ref:`sphx_glr_auto_examples_model_selection_grid_search_text_feature_extraction.py` + * :ref:`sphx_glr_auto_examples_model_selection_plot_grid_search_text_feature_extraction.py` Decoding text files @@ -638,58 +637,59 @@ or ``"replace"``. See the documentation for the Python function ``bytes.decode`` for more details (type ``help(bytes.decode)`` at the Python prompt). -If you are having trouble decoding text, here are some things to try: - -- Find out what the actual encoding of the text is. The file might come - with a header or README that tells you the encoding, or there might be some - standard encoding you can assume based on where the text comes from. - -- You may be able to find out what kind of encoding it is in general - using the UNIX command ``file``. The Python ``chardet`` module comes with - a script called ``chardetect.py`` that will guess the specific encoding, - though you cannot rely on its guess being correct. - -- You could try UTF-8 and disregard the errors. You can decode byte - strings with ``bytes.decode(errors='replace')`` to replace all - decoding errors with a meaningless character, or set - ``decode_error='replace'`` in the vectorizer. This may damage the - usefulness of your features. - -- Real text may come from a variety of sources that may have used different - encodings, or even be sloppily decoded in a different encoding than the - one it was encoded with. This is common in text retrieved from the Web. - The Python package `ftfy`_ can automatically sort out some classes of - decoding errors, so you could try decoding the unknown text as ``latin-1`` - and then using ``ftfy`` to fix errors. - -- If the text is in a mish-mash of encodings that is simply too hard to sort - out (which is the case for the 20 Newsgroups dataset), you can fall back on - a simple single-byte encoding such as ``latin-1``. Some text may display - incorrectly, but at least the same sequence of bytes will always represent - the same feature. - -For example, the following snippet uses ``chardet`` -(not shipped with scikit-learn, must be installed separately) -to figure out the encoding of three texts. -It then vectorizes the texts and prints the learned vocabulary. -The output is not shown here. - - >>> import chardet # doctest: +SKIP - >>> text1 = b"Sei mir gegr\xc3\xbc\xc3\x9ft mein Sauerkraut" - >>> text2 = b"holdselig sind deine Ger\xfcche" - >>> text3 = b"\xff\xfeA\x00u\x00f\x00 \x00F\x00l\x00\xfc\x00g\x00e\x00l\x00n\x00 \x00d\x00e\x00s\x00 \x00G\x00e\x00s\x00a\x00n\x00g\x00e\x00s\x00,\x00 \x00H\x00e\x00r\x00z\x00l\x00i\x00e\x00b\x00c\x00h\x00e\x00n\x00,\x00 \x00t\x00r\x00a\x00g\x00 \x00i\x00c\x00h\x00 \x00d\x00i\x00c\x00h\x00 \x00f\x00o\x00r\x00t\x00" - >>> decoded = [x.decode(chardet.detect(x)['encoding']) - ... for x in (text1, text2, text3)] # doctest: +SKIP - >>> v = CountVectorizer().fit(decoded).vocabulary_ # doctest: +SKIP - >>> for term in v: print(v) # doctest: +SKIP - -(Depending on the version of ``chardet``, it might get the first one wrong.) - -For an introduction to Unicode and character encodings in general, -see Joel Spolsky's `Absolute Minimum Every Software Developer Must Know -About Unicode `_. - -.. _`ftfy`: https://github.com/LuminosoInsight/python-ftfy +.. dropdown:: Troubleshooting decoding text + + If you are having trouble decoding text, here are some things to try: + + - Find out what the actual encoding of the text is. The file might come + with a header or README that tells you the encoding, or there might be some + standard encoding you can assume based on where the text comes from. + + - You may be able to find out what kind of encoding it is in general + using the UNIX command ``file``. The Python ``chardet`` module comes with + a script called ``chardetect.py`` that will guess the specific encoding, + though you cannot rely on its guess being correct. + + - You could try UTF-8 and disregard the errors. You can decode byte + strings with ``bytes.decode(errors='replace')`` to replace all + decoding errors with a meaningless character, or set + ``decode_error='replace'`` in the vectorizer. This may damage the + usefulness of your features. + + - Real text may come from a variety of sources that may have used different + encodings, or even be sloppily decoded in a different encoding than the + one it was encoded with. This is common in text retrieved from the Web. + The Python package `ftfy `__ + can automatically sort out some classes of + decoding errors, so you could try decoding the unknown text as ``latin-1`` + and then using ``ftfy`` to fix errors. + + - If the text is in a mish-mash of encodings that is simply too hard to sort + out (which is the case for the 20 Newsgroups dataset), you can fall back on + a simple single-byte encoding such as ``latin-1``. Some text may display + incorrectly, but at least the same sequence of bytes will always represent + the same feature. + + For example, the following snippet uses ``chardet`` + (not shipped with scikit-learn, must be installed separately) + to figure out the encoding of three texts. + It then vectorizes the texts and prints the learned vocabulary. + The output is not shown here. + + >>> import chardet # doctest: +SKIP + >>> text1 = b"Sei mir gegr\xc3\xbc\xc3\x9ft mein Sauerkraut" + >>> text2 = b"holdselig sind deine Ger\xfcche" + >>> text3 = b"\xff\xfeA\x00u\x00f\x00 \x00F\x00l\x00\xfc\x00g\x00e\x00l\x00n\x00 \x00d\x00e\x00s\x00 \x00G\x00e\x00s\x00a\x00n\x00g\x00e\x00s\x00,\x00 \x00H\x00e\x00r\x00z\x00l\x00i\x00e\x00b\x00c\x00h\x00e\x00n\x00,\x00 \x00t\x00r\x00a\x00g\x00 \x00i\x00c\x00h\x00 \x00d\x00i\x00c\x00h\x00 \x00f\x00o\x00r\x00t\x00" + >>> decoded = [x.decode(chardet.detect(x)['encoding']) + ... for x in (text1, text2, text3)] # doctest: +SKIP + >>> v = CountVectorizer().fit(decoded).vocabulary_ # doctest: +SKIP + >>> for term in v: print(v) # doctest: +SKIP + + (Depending on the version of ``chardet``, it might get the first one wrong.) + + For an introduction to Unicode and character encodings in general, + see Joel Spolsky's `Absolute Minimum Every Software Developer Must Know + About Unicode `_. Applications and examples @@ -702,18 +702,18 @@ In particular in a **supervised setting** it can be successfully combined with fast and scalable linear models to train **document classifiers**, for instance: - * :ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py` +* :ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py` In an **unsupervised setting** it can be used to group similar documents together by applying clustering algorithms such as :ref:`k_means`: - * :ref:`sphx_glr_auto_examples_text_plot_document_clustering.py` +* :ref:`sphx_glr_auto_examples_text_plot_document_clustering.py` Finally it is possible to discover the main topics of a corpus by relaxing the hard assignment constraint of clustering, for instance by using :ref:`NMF`: - * :ref:`sphx_glr_auto_examples_applications_plot_topics_extraction_with_nmf_lda.py` +* :ref:`sphx_glr_auto_examples_applications_plot_topics_extraction_with_nmf_lda.py` Limitations of the Bag of Words representation @@ -742,9 +742,8 @@ decide better:: >>> ngram_vectorizer = CountVectorizer(analyzer='char_wb', ngram_range=(2, 2)) >>> counts = ngram_vectorizer.fit_transform(['words', 'wprds']) - >>> ngram_vectorizer.get_feature_names() == ( - ... [' w', 'ds', 'or', 'pr', 'rd', 's ', 'wo', 'wp']) - True + >>> ngram_vectorizer.get_feature_names_out() + array([' w', 'ds', 'or', 'pr', 'rd', 's ', 'wo', 'wp'], ...) >>> counts.toarray().astype(int) array([[1, 1, 1, 0, 1, 1, 1, 0], [1, 1, 0, 1, 1, 1, 0, 1]]) @@ -756,19 +755,18 @@ span across words:: >>> ngram_vectorizer = CountVectorizer(analyzer='char_wb', ngram_range=(5, 5)) >>> ngram_vectorizer.fit_transform(['jumpy fox']) - <1x4 sparse matrix of type '<... 'numpy.int64'>' - with 4 stored elements in Compressed Sparse ... format> - >>> ngram_vectorizer.get_feature_names() == ( - ... [' fox ', ' jump', 'jumpy', 'umpy ']) - True + + + >>> ngram_vectorizer.get_feature_names_out() + array([' fox ', ' jump', 'jumpy', 'umpy '], ...) >>> ngram_vectorizer = CountVectorizer(analyzer='char', ngram_range=(5, 5)) >>> ngram_vectorizer.fit_transform(['jumpy fox']) - <1x5 sparse matrix of type '<... 'numpy.int64'>' - with 5 stored elements in Compressed Sparse ... format> - >>> ngram_vectorizer.get_feature_names() == ( - ... ['jumpy', 'mpy f', 'py fo', 'umpy ', 'y fox']) - True + + >>> ngram_vectorizer.get_feature_names_out() + array(['jumpy', 'mpy f', 'py fo', 'umpy ', 'y fox'], ...) The word boundaries-aware variant ``char_wb`` is especially interesting for languages that use white-spaces for word separation as it generates @@ -794,9 +792,9 @@ problems which are currently outside of the scope of scikit-learn. Vectorizing a large text corpus with the hashing trick ------------------------------------------------------ -The above vectorization scheme is simple but the fact that it holds an **in- -memory mapping from the string tokens to the integer feature indices** (the -``vocabulary_`` attribute) causes several **problems when dealing with large +The above vectorization scheme is simple but the fact that it holds an +**in-memory mapping from the string tokens to the integer feature indices** +(the ``vocabulary_`` attribute) causes several **problems when dealing with large datasets**: - the larger the corpus, the larger the vocabulary will grow and hence the @@ -815,7 +813,7 @@ datasets**: - it is not easily possible to split the vectorization work into concurrent sub tasks as the ``vocabulary_`` attribute would have to be a shared state with a fine grained synchronization barrier: the mapping from token string to - feature index is dependent on ordering of the first occurrence of each token + feature index is dependent on the ordering of the first occurrence of each token hence would have to be shared, potentially harming the concurrent workers' performance to the point of making them slower than the sequential variant. @@ -824,7 +822,7 @@ It is possible to overcome those limitations by combining the "hashing trick" :class:`~sklearn.feature_extraction.FeatureHasher` class and the text preprocessing and tokenization features of the :class:`CountVectorizer`. -This combination is implementing in :class:`HashingVectorizer`, +This combination is implemented in :class:`HashingVectorizer`, a transformer class that is mostly API compatible with :class:`CountVectorizer`. :class:`HashingVectorizer` is stateless, meaning that you don't have to call ``fit`` on it:: @@ -832,8 +830,8 @@ meaning that you don't have to call ``fit`` on it:: >>> from sklearn.feature_extraction.text import HashingVectorizer >>> hv = HashingVectorizer(n_features=10) >>> hv.transform(corpus) - <4x10 sparse matrix of type '<... 'numpy.float64'>' - with 16 stored elements in Compressed Sparse ... format> + You can see that 16 non-zero feature tokens were extracted in the vector output: this is less than the 19 non-zeros extracted previously by the @@ -850,14 +848,14 @@ Note that the dimensionality does not affect the CPU training time of algorithms which operate on CSR matrices (``LinearSVC(dual=True)``, ``Perceptron``, ``SGDClassifier``, ``PassiveAggressive``) but it does for algorithms that work with CSC matrices (``LinearSVC(dual=False)``, ``Lasso()``, -etc). +etc.). Let's try again with the default setting:: >>> hv = HashingVectorizer() >>> hv.transform(corpus) - <4x1048576 sparse matrix of type '<... 'numpy.float64'>' - with 19 stored elements in Compressed Sparse ... format> + We no longer get the collisions, but this comes at the expense of a much larger dimensionality of the output space. @@ -874,25 +872,25 @@ The :class:`HashingVectorizer` also comes with the following limitations: model. A :class:`TfidfTransformer` can be appended to it in a pipeline if required. -Performing out-of-core scaling with HashingVectorizer ------------------------------------------------------- +.. dropdown:: Performing out-of-core scaling with HashingVectorizer + + An interesting development of using a :class:`HashingVectorizer` is the ability + to perform `out-of-core`_ scaling. This means that we can learn from data that + does not fit into the computer's main memory. -An interesting development of using a :class:`HashingVectorizer` is the ability -to perform `out-of-core`_ scaling. This means that we can learn from data that -does not fit into the computer's main memory. + .. _out-of-core: https://en.wikipedia.org/wiki/Out-of-core_algorithm -.. _out-of-core: https://en.wikipedia.org/wiki/Out-of-core_algorithm + A strategy to implement out-of-core scaling is to stream data to the estimator + in mini-batches. Each mini-batch is vectorized using :class:`HashingVectorizer` + so as to guarantee that the input space of the estimator has always the same + dimensionality. The amount of memory used at any time is thus bounded by the + size of a mini-batch. Although there is no limit to the amount of data that can + be ingested using such an approach, from a practical point of view the learning + time is often limited by the CPU time one wants to spend on the task. -A strategy to implement out-of-core scaling is to stream data to the estimator -in mini-batches. Each mini-batch is vectorized using :class:`HashingVectorizer` -so as to guarantee that the input space of the estimator has always the same -dimensionality. The amount of memory used at any time is thus bounded by the -size of a mini-batch. Although there is no limit to the amount of data that can -be ingested using such an approach, from a practical point of view the learning -time is often limited by the CPU time one wants to spend on the task. + For a full-fledged example of out-of-core scaling in a text classification + task see :ref:`sphx_glr_auto_examples_applications_plot_out_of_core_classification.py`. -For a full-fledged example of out-of-core scaling in a text classification -task see :ref:`sphx_glr_auto_examples_applications_plot_out_of_core_classification.py`. Customizing the vectorizer classes ---------------------------------- @@ -910,19 +908,19 @@ to the vectorizer constructor:: In particular we name: - * ``preprocessor``: a callable that takes an entire document as input (as a - single string), and returns a possibly transformed version of the document, - still as an entire string. This can be used to remove HTML tags, lowercase - the entire document, etc. +* ``preprocessor``: a callable that takes an entire document as input (as a + single string), and returns a possibly transformed version of the document, + still as an entire string. This can be used to remove HTML tags, lowercase + the entire document, etc. - * ``tokenizer``: a callable that takes the output from the preprocessor - and splits it into tokens, then returns a list of these. +* ``tokenizer``: a callable that takes the output from the preprocessor + and splits it into tokens, then returns a list of these. - * ``analyzer``: a callable that replaces the preprocessor and tokenizer. - The default analyzers all call the preprocessor and tokenizer, but custom - analyzers will skip this. N-gram extraction and stop word filtering take - place at the analyzer level, so a custom analyzer may have to reproduce - these steps. +* ``analyzer``: a callable that replaces the preprocessor and tokenizer. + The default analyzers all call the preprocessor and tokenizer, but custom + analyzers will skip this. N-gram extraction and stop word filtering take + place at the analyzer level, so a custom analyzer may have to reproduce + these steps. (Lucene users might recognize these names, but be aware that scikit-learn concepts may not map one-to-one onto Lucene concepts.) @@ -932,7 +930,8 @@ parameters it is possible to derive from the class and override the ``build_preprocessor``, ``build_tokenizer`` and ``build_analyzer`` factory methods instead of passing custom functions. -Some tips and tricks: +.. dropdown:: Tips and tricks + :color: success * If documents are pre-tokenized by an external package, then store them in files (or strings) with the tokens separated by whitespace and pass @@ -956,7 +955,6 @@ Some tips and tricks: (Note that this will not filter out punctuation.) - The following example will, for instance, transform some British spelling to American spelling:: @@ -980,11 +978,10 @@ Some tips and tricks: for other styles of preprocessing; examples include stemming, lemmatization, or normalizing numerical tokens, with the latter illustrated in: - * :ref:`sphx_glr_auto_examples_bicluster_plot_bicluster_newsgroups.py` - + * :ref:`sphx_glr_auto_examples_bicluster_plot_bicluster_newsgroups.py` -Customizing the vectorizer can also be useful when handling Asian languages -that do not use an explicit word separator such as whitespace. + Customizing the vectorizer can also be useful when handling Asian languages + that do not use an explicit word separator such as whitespace. .. _image_feature_extraction: @@ -999,7 +996,7 @@ Patch extraction The :func:`extract_patches_2d` function extracts patches from an image stored as a two-dimensional array, or three-dimensional with color information along the third axis. For rebuilding an image from all its patches, use -:func:`reconstruct_from_patches_2d`. For example let use generate a 4x4 pixel +:func:`reconstruct_from_patches_2d`. For example let us generate a 4x4 pixel picture with 3 color channels (e.g. in RGB format):: >>> import numpy as np @@ -1037,17 +1034,19 @@ on overlapping areas:: The :class:`PatchExtractor` class works in the same way as :func:`extract_patches_2d`, only it supports multiple images as input. It is -implemented as an estimator, so it can be used in pipelines. See:: +implemented as a scikit-learn transformer, so it can be used in pipelines. See:: >>> five_images = np.arange(5 * 4 * 4 * 3).reshape(5, 4, 4, 3) >>> patches = image.PatchExtractor(patch_size=(2, 2)).transform(five_images) >>> patches.shape (45, 2, 2, 3) +.. _connectivity_graph_image: + Connectivity graph of an image ------------------------------- -Several estimators in the scikit-learn can use connectivity information between +Several estimators in scikit-learn can use connectivity information between features or samples. For instance Ward clustering (:ref:`hierarchical_clustering`) can cluster together only neighboring pixels of an image, thus forming contiguous patches: @@ -1061,8 +1060,8 @@ For this purpose, the estimators use a 'connectivity' matrix, giving which samples are connected. The function :func:`img_to_graph` returns such a matrix from a 2D or 3D -image. Similarly, :func:`grid_to_graph` build a connectivity matrix for -images given the shape of these image. +image. Similarly, :func:`grid_to_graph` builds a connectivity matrix for +images given the shape of these images. These matrices can be used to impose connectivity in estimators that use connectivity information, such as Ward clustering diff --git a/doc/modules/feature_selection.rst b/doc/modules/feature_selection.rst index c27a334c2ed4b..ffee801f34ccc 100644 --- a/doc/modules/feature_selection.rst +++ b/doc/modules/feature_selection.rst @@ -57,29 +57,29 @@ univariate statistical tests. It can be seen as a preprocessing step to an estimator. Scikit-learn exposes feature selection routines as objects that implement the ``transform`` method: - * :class:`SelectKBest` removes all but the :math:`k` highest scoring features +* :class:`SelectKBest` removes all but the :math:`k` highest scoring features - * :class:`SelectPercentile` removes all but a user-specified highest scoring - percentage of features +* :class:`SelectPercentile` removes all but a user-specified highest scoring + percentage of features - * using common univariate statistical tests for each feature: - false positive rate :class:`SelectFpr`, false discovery rate - :class:`SelectFdr`, or family wise error :class:`SelectFwe`. +* using common univariate statistical tests for each feature: + false positive rate :class:`SelectFpr`, false discovery rate + :class:`SelectFdr`, or family wise error :class:`SelectFwe`. - * :class:`GenericUnivariateSelect` allows to perform univariate feature - selection with a configurable strategy. This allows to select the best - univariate selection strategy with hyper-parameter search estimator. +* :class:`GenericUnivariateSelect` allows to perform univariate feature + selection with a configurable strategy. This allows to select the best + univariate selection strategy with hyper-parameter search estimator. -For instance, we can perform a :math:`\chi^2` test to the samples -to retrieve only the two best features as follows: +For instance, we can use a F-test to retrieve the two +best features for a dataset as follows: >>> from sklearn.datasets import load_iris >>> from sklearn.feature_selection import SelectKBest - >>> from sklearn.feature_selection import chi2 + >>> from sklearn.feature_selection import f_classif >>> X, y = load_iris(return_X_y=True) >>> X.shape (150, 4) - >>> X_new = SelectKBest(chi2, k=2).fit_transform(X, y) + >>> X_new = SelectKBest(f_classif, k=2).fit_transform(X, y) >>> X_new.shape (150, 2) @@ -87,14 +87,15 @@ These objects take as input a scoring function that returns univariate scores and p-values (or only scores for :class:`SelectKBest` and :class:`SelectPercentile`): - * For regression: :func:`f_regression`, :func:`mutual_info_regression` +* For regression: :func:`r_regression`, :func:`f_regression`, :func:`mutual_info_regression` - * For classification: :func:`chi2`, :func:`f_classif`, :func:`mutual_info_classif` +* For classification: :func:`chi2`, :func:`f_classif`, :func:`mutual_info_classif` The methods based on F-test estimate the degree of linear dependency between two random variables. On the other hand, mutual information methods can capture any kind of statistical dependency, but being nonparametric, they require more -samples for accurate estimation. +samples for accurate estimation. Note that the :math:`\chi^2`-test should only be +applied to non-negative features, such as frequencies. .. topic:: Feature selection with sparse data @@ -107,11 +108,17 @@ samples for accurate estimation. Beware not to use a regression scoring function with a classification problem, you will get useless results. -.. topic:: Examples: +.. note:: - * :ref:`sphx_glr_auto_examples_feature_selection_plot_feature_selection.py` + The :class:`SelectPercentile` and :class:`SelectKBest` support unsupervised + feature selection as well. One needs to provide a `score_func` where `y=None`. + The `score_func` should use internally `X` to compute the scores. - * :ref:`sphx_glr_auto_examples_feature_selection_plot_f_test_vs_mi.py` +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_feature_selection_plot_feature_selection.py` + +* :ref:`sphx_glr_auto_examples_feature_selection_plot_f_test_vs_mi.py` .. _rfe: @@ -124,21 +131,27 @@ is to select features by recursively considering smaller and smaller sets of features. First, the estimator is trained on the initial set of features and the importance of each feature is obtained either through any specific attribute (such as ``coef_``, ``feature_importances_``) or callable. Then, the least important -features are pruned from current set of features. That procedure is recursively +features are pruned from the current set of features. That procedure is recursively repeated on the pruned set until the desired number of features to select is eventually reached. :class:`RFECV` performs RFE in a cross-validation loop to find the optimal -number of features. +number of features. In more details, the number of features selected is tuned +automatically by fitting an :class:`RFE` selector on the different +cross-validation splits (provided by the `cv` parameter). The performance +of the :class:`RFE` selector is evaluated using `scorer` for different numbers +of selected features and aggregated together. Finally, the scores are averaged +across folds and the number of features selected is set to the number of +features that maximize the cross-validation score. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_feature_selection_plot_rfe_digits.py`: A recursive feature elimination example - showing the relevance of pixels in a digit classification task. +* :ref:`sphx_glr_auto_examples_feature_selection_plot_rfe_digits.py`: A recursive feature elimination example + showing the relevance of pixels in a digit classification task. - * :ref:`sphx_glr_auto_examples_feature_selection_plot_rfe_with_cross_validation.py`: A recursive feature - elimination example with automatic tuning of the number of features - selected with cross-validation. +* :ref:`sphx_glr_auto_examples_feature_selection_plot_rfe_with_cross_validation.py`: A recursive feature + elimination example with automatic tuning of the number of features + selected with cross-validation. .. _select_from_model: @@ -149,7 +162,7 @@ Feature selection using SelectFromModel estimator that assigns importance to each feature through a specific attribute (such as ``coef_``, ``feature_importances_``) or via an `importance_getter` callable after fitting. The features are considered unimportant and removed if the corresponding -importance of the feature values are below the provided +importance of the feature values is below the provided ``threshold`` parameter. Apart from specifying the threshold numerically, there are built-in heuristics for finding a threshold using a string argument. Available heuristics are "mean", "median" and float multiples of these like @@ -158,9 +171,9 @@ Available heuristics are "mean", "median" and float multiples of these like For examples on how it is to be used refer to the sections below. -.. topic:: Examples +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_feature_selection_plot_select_from_model_diabetes.py` +* :ref:`sphx_glr_auto_examples_feature_selection_plot_select_from_model_diabetes.py` .. _l1_feature_selection: @@ -190,41 +203,45 @@ for classification:: >>> X_new.shape (150, 3) -With SVMs and logistic-regression, the parameter C controls the sparsity: +With SVMs and logistic regression, the parameter C controls the sparsity: the smaller C the fewer features selected. With Lasso, the higher the alpha parameter, the fewer features selected. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py`: Comparison - of different algorithms for document classification including L1-based - feature selection. +* :ref:`sphx_glr_auto_examples_linear_model_plot_lasso_dense_vs_sparse_data.py`. .. _compressive_sensing: -.. topic:: **L1-recovery and compressive sensing** - - For a good choice of alpha, the :ref:`lasso` can fully recover the - exact set of non-zero variables using only few observations, provided - certain specific conditions are met. In particular, the number of - samples should be "sufficiently large", or L1 models will perform at - random, where "sufficiently large" depends on the number of non-zero - coefficients, the logarithm of the number of features, the amount of - noise, the smallest absolute value of non-zero coefficients, and the - structure of the design matrix X. In addition, the design matrix must - display certain specific properties, such as not being too correlated. - - There is no general rule to select an alpha parameter for recovery of - non-zero coefficients. It can by set by cross-validation - (:class:`LassoCV` or :class:`LassoLarsCV`), though this may lead to - under-penalized models: including a small number of non-relevant - variables is not detrimental to prediction score. BIC - (:class:`LassoLarsIC`) tends, on the opposite, to set high values of - alpha. - - **Reference** Richard G. Baraniuk "Compressive Sensing", IEEE Signal - Processing Magazine [120] July 2007 - http://users.isr.ist.utl.pt/~aguiar/CS_notes.pdf +.. dropdown:: L1-recovery and compressive sensing + + For a good choice of alpha, the :ref:`lasso` can fully recover the + exact set of non-zero variables using only few observations, provided + certain specific conditions are met. In particular, the number of + samples should be "sufficiently large", or L1 models will perform at + random, where "sufficiently large" depends on the number of non-zero + coefficients, the logarithm of the number of features, the amount of + noise, the smallest absolute value of non-zero coefficients, and the + structure of the design matrix X. In addition, the design matrix must + display certain specific properties, such as not being too correlated. + On the use of Lasso for sparse signal recovery, see this example on + compressive sensing: + :ref:`sphx_glr_auto_examples_applications_plot_tomography_l1_reconstruction.py`. + + There is no general rule to select an alpha parameter for recovery of + non-zero coefficients. It can be set by cross-validation + (:class:`~sklearn.linear_model.LassoCV` or + :class:`~sklearn.linear_model.LassoLarsCV`), though this may lead to + under-penalized models: including a small number of non-relevant variables + is not detrimental to prediction score. BIC + (:class:`~sklearn.linear_model.LassoLarsIC`) tends, on the opposite, to set + high values of alpha. + + .. rubric:: References + + Richard G. Baraniuk "Compressive Sensing", IEEE Signal + Processing Magazine [120] July 2007 + http://users.isr.ist.utl.pt/~aguiar/CS_notes.pdf Tree-based feature selection @@ -245,20 +262,20 @@ meta-transformer):: >>> clf = ExtraTreesClassifier(n_estimators=50) >>> clf = clf.fit(X, y) >>> clf.feature_importances_ # doctest: +SKIP - array([ 0.04..., 0.05..., 0.4..., 0.4...]) + array([ 0.04, 0.05, 0.4, 0.4]) >>> model = SelectFromModel(clf, prefit=True) >>> X_new = model.transform(X) >>> X_new.shape # doctest: +SKIP (150, 2) -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_ensemble_plot_forest_importances.py`: example on - synthetic data showing the recovery of the actually meaningful - features. +* :ref:`sphx_glr_auto_examples_ensemble_plot_forest_importances.py`: example on + synthetic data showing the recovery of the actually meaningful features. - * :ref:`sphx_glr_auto_examples_ensemble_plot_forest_importances_faces.py`: example - on face recognition data. +* :ref:`sphx_glr_auto_examples_inspection_plot_permutation_importance.py`: example + discussing the caveats of using impurity-based feature importances as a proxy for + feature relevance. .. _sequential_feature_selection: @@ -271,43 +288,46 @@ SFS can be either forward or backward: Forward-SFS is a greedy procedure that iteratively finds the best new feature to add to the set of selected features. Concretely, we initially start with -zero feature and find the one feature that maximizes a cross-validated score +zero features and find the one feature that maximizes a cross-validated score when an estimator is trained on this single feature. Once that first feature is selected, we repeat the procedure by adding a new feature to the set of selected features. The procedure stops when the desired number of selected features is reached, as determined by the `n_features_to_select` parameter. Backward-SFS follows the same idea but works in the opposite direction: -instead of starting with no feature and greedily adding features, we start +instead of starting with no features and greedily adding features, we start with *all* the features and greedily *remove* features from the set. The `direction` parameter controls whether forward or backward SFS is used. -In general, forward and backward selection do not yield equivalent results. -Also, one may be much faster than the other depending on the requested number -of selected features: if we have 10 features and ask for 7 selected features, -forward selection would need to perform 7 iterations while backward selection -would only need to perform 3. +.. dropdown:: Details on Sequential Feature Selection -SFS differs from :class:`~sklearn.feature_selection.RFE` and -:class:`~sklearn.feature_selection.SelectFromModel` in that it does not -require the underlying model to expose a `coef_` or `feature_importances_` -attribute. It may however be slower considering that more models need to be -evaluated, compared to the other approaches. For example in backward -selection, the iteration going from `m` features to `m - 1` features using k-fold -cross-validation requires fitting `m * k` models, while -:class:`~sklearn.feature_selection.RFE` would require only a single fit, and -:class:`~sklearn.feature_selection.SelectFromModel` always just does a single -fit and requires no iterations. + In general, forward and backward selection do not yield equivalent results. + Also, one may be much faster than the other depending on the requested number + of selected features: if we have 10 features and ask for 7 selected features, + forward selection would need to perform 7 iterations while backward selection + would only need to perform 3. -.. topic:: Examples + SFS differs from :class:`~sklearn.feature_selection.RFE` and + :class:`~sklearn.feature_selection.SelectFromModel` in that it does not + require the underlying model to expose a `coef_` or `feature_importances_` + attribute. It may however be slower considering that more models need to be + evaluated, compared to the other approaches. For example in backward + selection, the iteration going from `m` features to `m - 1` features using k-fold + cross-validation requires fitting `m * k` models, while + :class:`~sklearn.feature_selection.RFE` would require only a single fit, and + :class:`~sklearn.feature_selection.SelectFromModel` always just does a single + fit and requires no iterations. - * :ref:`sphx_glr_auto_examples_feature_selection_plot_select_from_model_diabetes.py` + .. rubric:: References -.. topic:: References: - - .. [sfs] Ferri et al, `Comparative study of techniques for + .. [sfs] Ferri et al, `Comparative study of techniques for large-scale feature selection - `_. + `_. + + +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_feature_selection_plot_select_from_model_diabetes.py` Feature selection as part of a pipeline ======================================= diff --git a/doc/modules/gaussian_process.rst b/doc/modules/gaussian_process.rst index 6aa9cb417aa5d..46d04ac35d832 100644 --- a/doc/modules/gaussian_process.rst +++ b/doc/modules/gaussian_process.rst @@ -1,5 +1,3 @@ - - .. _gaussian_process: ================== @@ -8,30 +6,30 @@ Gaussian Processes .. currentmodule:: sklearn.gaussian_process -**Gaussian Processes (GP)** are a generic supervised learning method designed +**Gaussian Processes (GP)** are a nonparametric supervised learning method used to solve *regression* and *probabilistic classification* problems. The advantages of Gaussian processes are: - - The prediction interpolates the observations (at least for regular - kernels). +- The prediction interpolates the observations (at least for regular + kernels). - - The prediction is probabilistic (Gaussian) so that one can compute - empirical confidence intervals and decide based on those if one should - refit (online fitting, adaptive fitting) the prediction in some - region of interest. +- The prediction is probabilistic (Gaussian) so that one can compute + empirical confidence intervals and decide based on those if one should + refit (online fitting, adaptive fitting) the prediction in some + region of interest. - - Versatile: different :ref:`kernels - ` can be specified. Common kernels are provided, but - it is also possible to specify custom kernels. +- Versatile: different :ref:`kernels + ` can be specified. Common kernels are provided, but + it is also possible to specify custom kernels. The disadvantages of Gaussian processes include: - - They are not sparse, i.e., they use the whole samples/features information to - perform the prediction. +- Our implementation is not sparse, i.e., they use the whole samples/features + information to perform the prediction. - - They lose efficiency in high dimensional spaces -- namely when the number - of features exceeds a few dozens. +- They lose efficiency in high dimensional spaces -- namely when the number + of features exceeds a few dozens. .. _gpr: @@ -42,31 +40,44 @@ Gaussian Process Regression (GPR) .. currentmodule:: sklearn.gaussian_process The :class:`GaussianProcessRegressor` implements Gaussian processes (GP) for -regression purposes. For this, the prior of the GP needs to be specified. The -prior mean is assumed to be constant and zero (for ``normalize_y=False``) or the -training data's mean (for ``normalize_y=True``). The prior's -covariance is specified by passing a :ref:`kernel ` object. The -hyperparameters of the kernel are optimized during fitting of -GaussianProcessRegressor by maximizing the log-marginal-likelihood (LML) based -on the passed ``optimizer``. As the LML may have multiple local optima, the -optimizer can be started repeatedly by specifying ``n_restarts_optimizer``. The -first run is always conducted starting from the initial hyperparameter values -of the kernel; subsequent runs are conducted from hyperparameter values -that have been chosen randomly from the range of allowed values. -If the initial hyperparameters should be kept fixed, `None` can be passed as -optimizer. +regression purposes. For this, the prior of the GP needs to be specified. GP +will combine this prior and the likelihood function based on training samples. +It allows to give a probabilistic approach to prediction by giving the mean and +standard deviation as output when predicting. -The noise level in the targets can be specified by passing it via the -parameter ``alpha``, either globally as a scalar or per datapoint. -Note that a moderate noise level can also be helpful for dealing with numeric -issues during fitting as it is effectively implemented as Tikhonov -regularization, i.e., by adding it to the diagonal of the kernel matrix. An -alternative to specifying the noise level explicitly is to include a -WhiteKernel component into the kernel, which can estimate the global noise -level from the data (see example below). +.. figure:: ../auto_examples/gaussian_process/images/sphx_glr_plot_gpr_noisy_targets_002.png + :target: ../auto_examples/gaussian_process/plot_gpr_noisy_targets.html + :align: center + +The prior mean is assumed to be constant and zero (for `normalize_y=False`) or +the training data's mean (for `normalize_y=True`). The prior's covariance is +specified by passing a :ref:`kernel ` object. The hyperparameters +of the kernel are optimized when fitting the :class:`GaussianProcessRegressor` +by maximizing the log-marginal-likelihood (LML) based on the passed +`optimizer`. As the LML may have multiple local optima, the optimizer can be +started repeatedly by specifying `n_restarts_optimizer`. The first run is +always conducted starting from the initial hyperparameter values of the kernel; +subsequent runs are conducted from hyperparameter values that have been chosen +randomly from the range of allowed values. If the initial hyperparameters +should be kept fixed, `None` can be passed as optimizer. + +The noise level in the targets can be specified by passing it via the parameter +`alpha`, either globally as a scalar or per datapoint. Note that a moderate +noise level can also be helpful for dealing with numeric instabilities during +fitting as it is effectively implemented as Tikhonov regularization, i.e., by +adding it to the diagonal of the kernel matrix. An alternative to specifying +the noise level explicitly is to include a +:class:`~sklearn.gaussian_process.kernels.WhiteKernel` component into the +kernel, which can estimate the global noise level from the data (see example +below). The figure below shows the effect of noisy target handled by setting +the parameter `alpha`. + +.. figure:: ../auto_examples/gaussian_process/images/sphx_glr_plot_gpr_noisy_targets_003.png + :target: ../auto_examples/gaussian_process/plot_gpr_noisy_targets.html + :align: center The implementation is based on Algorithm 2.1 of [RW2006]_. In addition to -the API of standard scikit-learn estimators, GaussianProcessRegressor: +the API of standard scikit-learn estimators, :class:`GaussianProcessRegressor`: * allows prediction without prior fitting (based on the GP prior) @@ -77,152 +88,12 @@ the API of standard scikit-learn estimators, GaussianProcessRegressor: externally for other ways of selecting hyperparameters, e.g., via Markov chain Monte Carlo. +.. rubric:: Examples -GPR examples -============ - -GPR with noise-level estimation -------------------------------- -This example illustrates that GPR with a sum-kernel including a WhiteKernel can -estimate the noise level of data. An illustration of the -log-marginal-likelihood (LML) landscape shows that there exist two local -maxima of LML. - -.. figure:: ../auto_examples/gaussian_process/images/sphx_glr_plot_gpr_noisy_001.png - :target: ../auto_examples/gaussian_process/plot_gpr_noisy.html - :align: center - -The first corresponds to a model with a high noise level and a -large length scale, which explains all variations in the data by noise. - -.. figure:: ../auto_examples/gaussian_process/images/sphx_glr_plot_gpr_noisy_002.png - :target: ../auto_examples/gaussian_process/plot_gpr_noisy.html - :align: center - -The second one has a smaller noise level and shorter length scale, which explains -most of the variation by the noise-free functional relationship. The second -model has a higher likelihood; however, depending on the initial value for the -hyperparameters, the gradient-based optimization might also converge to the -high-noise solution. It is thus important to repeat the optimization several -times for different initializations. - -.. figure:: ../auto_examples/gaussian_process/images/sphx_glr_plot_gpr_noisy_003.png - :target: ../auto_examples/gaussian_process/plot_gpr_noisy.html - :align: center - - -Comparison of GPR and Kernel Ridge Regression ---------------------------------------------- - -Both kernel ridge regression (KRR) and GPR learn -a target function by employing internally the "kernel trick". KRR learns a -linear function in the space induced by the respective kernel which corresponds -to a non-linear function in the original space. The linear function in the -kernel space is chosen based on the mean-squared error loss with -ridge regularization. GPR uses the kernel to define the covariance of -a prior distribution over the target functions and uses the observed training -data to define a likelihood function. Based on Bayes theorem, a (Gaussian) -posterior distribution over target functions is defined, whose mean is used -for prediction. - -A major difference is that GPR can choose the kernel's hyperparameters based -on gradient-ascent on the marginal likelihood function while KRR needs to -perform a grid search on a cross-validated loss function (mean-squared error -loss). A further difference is that GPR learns a generative, probabilistic -model of the target function and can thus provide meaningful confidence -intervals and posterior samples along with the predictions while KRR only -provides predictions. - -The following figure illustrates both methods on an artificial dataset, which -consists of a sinusoidal target function and strong noise. The figure compares -the learned model of KRR and GPR based on a ExpSineSquared kernel, which is -suited for learning periodic functions. The kernel's hyperparameters control -the smoothness (length_scale) and periodicity of the kernel (periodicity). -Moreover, the noise level -of the data is learned explicitly by GPR by an additional WhiteKernel component -in the kernel and by the regularization parameter alpha of KRR. - -.. figure:: ../auto_examples/gaussian_process/images/sphx_glr_plot_compare_gpr_krr_001.png - :target: ../auto_examples/gaussian_process/plot_compare_gpr_krr.html - :align: center - -The figure shows that both methods learn reasonable models of the target -function. GPR correctly identifies the periodicity of the function to be -roughly :math:`2*\pi` (6.28), while KRR chooses the doubled periodicity -:math:`4*\pi` . Besides -that, GPR provides reasonable confidence bounds on the prediction which are not -available for KRR. A major difference between the two methods is the time -required for fitting and predicting: while fitting KRR is fast in principle, -the grid-search for hyperparameter optimization scales exponentially with the -number of hyperparameters ("curse of dimensionality"). The gradient-based -optimization of the parameters in GPR does not suffer from this exponential -scaling and is thus considerably faster on this example with 3-dimensional -hyperparameter space. The time for predicting is similar; however, generating -the variance of the predictive distribution of GPR takes considerably longer -than just predicting the mean. - -GPR on Mauna Loa CO2 data -------------------------- - -This example is based on Section 5.4.3 of [RW2006]_. -It illustrates an example of complex kernel engineering and -hyperparameter optimization using gradient ascent on the -log-marginal-likelihood. The data consists of the monthly average atmospheric -CO2 concentrations (in parts per million by volume (ppmv)) collected at the -Mauna Loa Observatory in Hawaii, between 1958 and 1997. The objective is to -model the CO2 concentration as a function of the time t. - -The kernel is composed of several terms that are responsible for explaining -different properties of the signal: - -- a long term, smooth rising trend is to be explained by an RBF kernel. The - RBF kernel with a large length-scale enforces this component to be smooth; - it is not enforced that the trend is rising which leaves this choice to the - GP. The specific length-scale and the amplitude are free hyperparameters. - -- a seasonal component, which is to be explained by the periodic - ExpSineSquared kernel with a fixed periodicity of 1 year. The length-scale - of this periodic component, controlling its smoothness, is a free parameter. - In order to allow decaying away from exact periodicity, the product with an - RBF kernel is taken. The length-scale of this RBF component controls the - decay time and is a further free parameter. - -- smaller, medium term irregularities are to be explained by a - RationalQuadratic kernel component, whose length-scale and alpha parameter, - which determines the diffuseness of the length-scales, are to be determined. - According to [RW2006]_, these irregularities can better be explained by - a RationalQuadratic than an RBF kernel component, probably because it can - accommodate several length-scales. - -- a "noise" term, consisting of an RBF kernel contribution, which shall - explain the correlated noise components such as local weather phenomena, - and a WhiteKernel contribution for the white noise. The relative amplitudes - and the RBF's length scale are further free parameters. - -Maximizing the log-marginal-likelihood after subtracting the target's mean -yields the following kernel with an LML of -83.214: - -:: - - 34.4**2 * RBF(length_scale=41.8) - + 3.27**2 * RBF(length_scale=180) * ExpSineSquared(length_scale=1.44, - periodicity=1) - + 0.446**2 * RationalQuadratic(alpha=17.7, length_scale=0.957) - + 0.197**2 * RBF(length_scale=0.138) + WhiteKernel(noise_level=0.0336) - -Thus, most of the target signal (34.4ppm) is explained by a long-term rising -trend (length-scale 41.8 years). The periodic component has an amplitude of -3.27ppm, a decay time of 180 years and a length-scale of 1.44. The long decay -time indicates that we have a locally very close to periodic seasonal -component. The correlated noise has an amplitude of 0.197ppm with a length -scale of 0.138 years and a white-noise contribution of 0.197ppm. Thus, the -overall noise level is very small, indicating that the data can be very well -explained by the model. The figure shows also that the model makes very -confident predictions until around 2015 - -.. figure:: ../auto_examples/gaussian_process/images/sphx_glr_plot_gpr_co2_001.png - :target: ../auto_examples/gaussian_process/plot_gpr_co2.html - :align: center +* :ref:`sphx_glr_auto_examples_gaussian_process_plot_gpr_noisy_targets.py` +* :ref:`sphx_glr_auto_examples_gaussian_process_plot_gpr_noisy.py` +* :ref:`sphx_glr_auto_examples_gaussian_process_plot_compare_gpr_krr.py` +* :ref:`sphx_glr_auto_examples_gaussian_process_plot_gpr_co2.py` .. _gpc: @@ -235,11 +106,11 @@ The :class:`GaussianProcessClassifier` implements Gaussian processes (GP) for classification purposes, more specifically for probabilistic classification, where test predictions take the form of class probabilities. GaussianProcessClassifier places a GP prior on a latent function :math:`f`, -which is then squashed through a link function to obtain the probabilistic +which is then squashed through a link function :math:`\pi` to obtain the probabilistic classification. The latent function :math:`f` is a so-called nuisance function, whose values are not observed and are not relevant by themselves. Its purpose is to allow a convenient formulation of the model, and :math:`f` -is removed (integrated out) during prediction. GaussianProcessClassifier +is removed (integrated out) during prediction. :class:`GaussianProcessClassifier` implements the logistic link function, for which the integral cannot be computed analytically but is easily approximated in the binary case. @@ -263,6 +134,11 @@ that have been chosen randomly from the range of allowed values. If the initial hyperparameters should be kept fixed, `None` can be passed as optimizer. +In some scenarios, information about the latent function :math:`f` is desired +(i.e. the mean :math:`\bar{f_*}` and the variance :math:`\text{Var}[f_*]` described +in Eqs. (3.21) and (3.24) of [RW2006]_). The :class:`GaussianProcessClassifier` +provides access to these quantities via the `latent_mean_and_variance` method. + :class:`GaussianProcessClassifier` supports multi-class classification by performing either one-versus-rest or one-versus-one based training and prediction. In one-versus-rest, one binary Gaussian process classifier is @@ -298,7 +174,7 @@ While the hyperparameters chosen by optimizing LML have a considerably larger LML, they perform slightly worse according to the log-loss on test data. The figure shows that this is because they exhibit a steep change of the class probabilities at the class boundaries (which is good) but have predicted -probabilities close to 0.5 far away from the class boundaries (which is bad) +probabilities close to 0.5 far away from the class boundaries (which is bad). This undesirable effect is caused by the Laplace approximation used internally by GPC. @@ -338,7 +214,7 @@ Gaussian process classification (GPC) on iris dataset ----------------------------------------------------- This example illustrates the predicted probability of GPC for an isotropic -and anisotropic RBF kernel on a two-dimensional version for the iris-dataset. +and anisotropic RBF kernel on a two-dimensional version for the iris dataset. This illustrates the applicability of GPC to non-binary classification. The anisotropic RBF kernel obtains slightly higher log-marginal-likelihood by assigning different length-scales to the two feature dimensions. @@ -365,91 +241,93 @@ translations in the input space, while non-stationary kernels depend also on the specific values of the datapoints. Stationary kernels can further be subdivided into isotropic and anisotropic kernels, where isotropic kernels are also invariant to rotations in the input space. For more details, we refer to -Chapter 4 of [RW2006]_. For guidance on how to best combine different kernels, -we refer to [Duv2014]_. - -Gaussian Process Kernel API ---------------------------- -The main usage of a :class:`Kernel` is to compute the GP's covariance between -datapoints. For this, the method ``__call__`` of the kernel can be called. This -method can either be used to compute the "auto-covariance" of all pairs of -datapoints in a 2d array X, or the "cross-covariance" of all combinations -of datapoints of a 2d array X with datapoints in a 2d array Y. The following -identity holds true for all kernels k (except for the :class:`WhiteKernel`): -``k(X) == K(X, Y=X)`` - -If only the diagonal of the auto-covariance is being used, the method ``diag()`` -of a kernel can be called, which is more computationally efficient than the -equivalent call to ``__call__``: ``np.diag(k(X, X)) == k.diag(X)`` - -Kernels are parameterized by a vector :math:`\theta` of hyperparameters. These -hyperparameters can for instance control length-scales or periodicity of a -kernel (see below). All kernels support computing analytic gradients -of the kernel's auto-covariance with respect to :math:`log(\theta)` via setting -``eval_gradient=True`` in the ``__call__`` method. -That is, a ``(len(X), len(X), len(theta))`` array is returned where the entry -``[i, j, l]`` contains :math:`\frac{\partial k_\theta(x_i, x_j)}{\partial log(\theta_l)}`. -This gradient is used by the Gaussian process (both regressor and classifier) -in computing the gradient of the log-marginal-likelihood, which in turn is used -to determine the value of :math:`\theta`, which maximizes the log-marginal-likelihood, -via gradient ascent. For each hyperparameter, the initial value and the -bounds need to be specified when creating an instance of the kernel. The -current value of :math:`\theta` can be get and set via the property -``theta`` of the kernel object. Moreover, the bounds of the hyperparameters can be -accessed by the property ``bounds`` of the kernel. Note that both properties -(theta and bounds) return log-transformed values of the internally used values -since those are typically more amenable to gradient-based optimization. -The specification of each hyperparameter is stored in the form of an instance of -:class:`Hyperparameter` in the respective kernel. Note that a kernel using a -hyperparameter with name "x" must have the attributes self.x and self.x_bounds. - -The abstract base class for all kernels is :class:`Kernel`. Kernel implements a -similar interface as :class:`Estimator`, providing the methods ``get_params()``, -``set_params()``, and ``clone()``. This allows setting kernel values also via -meta-estimators such as :class:`Pipeline` or :class:`GridSearch`. Note that due to the nested -structure of kernels (by applying kernel operators, see below), the names of -kernel parameters might become relatively complicated. In general, for a -binary kernel operator, parameters of the left operand are prefixed with ``k1__`` -and parameters of the right operand with ``k2__``. An additional convenience -method is ``clone_with_theta(theta)``, which returns a cloned version of the -kernel but with the hyperparameters set to ``theta``. An illustrative example: - - >>> from sklearn.gaussian_process.kernels import ConstantKernel, RBF - >>> kernel = ConstantKernel(constant_value=1.0, constant_value_bounds=(0.0, 10.0)) * RBF(length_scale=0.5, length_scale_bounds=(0.0, 10.0)) + RBF(length_scale=2.0, length_scale_bounds=(0.0, 10.0)) - >>> for hyperparameter in kernel.hyperparameters: print(hyperparameter) - Hyperparameter(name='k1__k1__constant_value', value_type='numeric', bounds=array([[ 0., 10.]]), n_elements=1, fixed=False) - Hyperparameter(name='k1__k2__length_scale', value_type='numeric', bounds=array([[ 0., 10.]]), n_elements=1, fixed=False) - Hyperparameter(name='k2__length_scale', value_type='numeric', bounds=array([[ 0., 10.]]), n_elements=1, fixed=False) - >>> params = kernel.get_params() - >>> for key in sorted(params): print("%s : %s" % (key, params[key])) - k1 : 1**2 * RBF(length_scale=0.5) - k1__k1 : 1**2 - k1__k1__constant_value : 1.0 - k1__k1__constant_value_bounds : (0.0, 10.0) - k1__k2 : RBF(length_scale=0.5) - k1__k2__length_scale : 0.5 - k1__k2__length_scale_bounds : (0.0, 10.0) - k2 : RBF(length_scale=2) - k2__length_scale : 2.0 - k2__length_scale_bounds : (0.0, 10.0) - >>> print(kernel.theta) # Note: log-transformed - [ 0. -0.69314718 0.69314718] - >>> print(kernel.bounds) # Note: log-transformed - [[ -inf 2.30258509] - [ -inf 2.30258509] - [ -inf 2.30258509]] - - -All Gaussian process kernels are interoperable with :mod:`sklearn.metrics.pairwise` -and vice versa: instances of subclasses of :class:`Kernel` can be passed as -``metric`` to ``pairwise_kernels`` from :mod:`sklearn.metrics.pairwise`. Moreover, -kernel functions from pairwise can be used as GP kernels by using the wrapper -class :class:`PairwiseKernel`. The only caveat is that the gradient of -the hyperparameters is not analytic but numeric and all those kernels support -only isotropic distances. The parameter ``gamma`` is considered to be a -hyperparameter and may be optimized. The other kernel parameters are set -directly at initialization and are kept fixed. - +Chapter 4 of [RW2006]_. :ref:`This example +` +shows how to define a custom kernel over discrete data. For guidance on how to best +combine different kernels, we refer to [Duv2014]_. + +.. dropdown:: Gaussian Process Kernel API + + The main usage of a :class:`Kernel` is to compute the GP's covariance between + datapoints. For this, the method ``__call__`` of the kernel can be called. This + method can either be used to compute the "auto-covariance" of all pairs of + datapoints in a 2d array X, or the "cross-covariance" of all combinations + of datapoints of a 2d array X with datapoints in a 2d array Y. The following + identity holds true for all kernels k (except for the :class:`WhiteKernel`): + ``k(X) == K(X, Y=X)`` + + If only the diagonal of the auto-covariance is being used, the method ``diag()`` + of a kernel can be called, which is more computationally efficient than the + equivalent call to ``__call__``: ``np.diag(k(X, X)) == k.diag(X)`` + + Kernels are parameterized by a vector :math:`\theta` of hyperparameters. These + hyperparameters can for instance control length-scales or periodicity of a + kernel (see below). All kernels support computing analytic gradients + of the kernel's auto-covariance with respect to :math:`log(\theta)` via setting + ``eval_gradient=True`` in the ``__call__`` method. + That is, a ``(len(X), len(X), len(theta))`` array is returned where the entry + ``[i, j, l]`` contains :math:`\frac{\partial k_\theta(x_i, x_j)}{\partial log(\theta_l)}`. + This gradient is used by the Gaussian process (both regressor and classifier) + in computing the gradient of the log-marginal-likelihood, which in turn is used + to determine the value of :math:`\theta`, which maximizes the log-marginal-likelihood, + via gradient ascent. For each hyperparameter, the initial value and the + bounds need to be specified when creating an instance of the kernel. The + current value of :math:`\theta` can be get and set via the property + ``theta`` of the kernel object. Moreover, the bounds of the hyperparameters can be + accessed by the property ``bounds`` of the kernel. Note that both properties + (theta and bounds) return log-transformed values of the internally used values + since those are typically more amenable to gradient-based optimization. + The specification of each hyperparameter is stored in the form of an instance of + :class:`Hyperparameter` in the respective kernel. Note that a kernel using a + hyperparameter with name "x" must have the attributes self.x and self.x_bounds. + + The abstract base class for all kernels is :class:`Kernel`. Kernel implements a + similar interface as :class:`~sklearn.base.BaseEstimator`, providing the + methods ``get_params()``, ``set_params()``, and ``clone()``. This allows + setting kernel values also via meta-estimators such as + :class:`~sklearn.pipeline.Pipeline` or + :class:`~sklearn.model_selection.GridSearchCV`. Note that due to the nested + structure of kernels (by applying kernel operators, see below), the names of + kernel parameters might become relatively complicated. In general, for a binary + kernel operator, parameters of the left operand are prefixed with ``k1__`` and + parameters of the right operand with ``k2__``. An additional convenience method + is ``clone_with_theta(theta)``, which returns a cloned version of the kernel + but with the hyperparameters set to ``theta``. An illustrative example: + + >>> from sklearn.gaussian_process.kernels import ConstantKernel, RBF + >>> kernel = ConstantKernel(constant_value=1.0, constant_value_bounds=(0.0, 10.0)) * RBF(length_scale=0.5, length_scale_bounds=(0.0, 10.0)) + RBF(length_scale=2.0, length_scale_bounds=(0.0, 10.0)) + >>> for hyperparameter in kernel.hyperparameters: print(hyperparameter) + Hyperparameter(name='k1__k1__constant_value', value_type='numeric', bounds=array([[ 0., 10.]]), n_elements=1, fixed=False) + Hyperparameter(name='k1__k2__length_scale', value_type='numeric', bounds=array([[ 0., 10.]]), n_elements=1, fixed=False) + Hyperparameter(name='k2__length_scale', value_type='numeric', bounds=array([[ 0., 10.]]), n_elements=1, fixed=False) + >>> params = kernel.get_params() + >>> for key in sorted(params): print("%s : %s" % (key, params[key])) + k1 : 1**2 * RBF(length_scale=0.5) + k1__k1 : 1**2 + k1__k1__constant_value : 1.0 + k1__k1__constant_value_bounds : (0.0, 10.0) + k1__k2 : RBF(length_scale=0.5) + k1__k2__length_scale : 0.5 + k1__k2__length_scale_bounds : (0.0, 10.0) + k2 : RBF(length_scale=2) + k2__length_scale : 2.0 + k2__length_scale_bounds : (0.0, 10.0) + >>> print(kernel.theta) # Note: log-transformed + [ 0. -0.69314718 0.69314718] + >>> print(kernel.bounds) # Note: log-transformed + [[ -inf 2.30258509] + [ -inf 2.30258509] + [ -inf 2.30258509]] + + All Gaussian process kernels are interoperable with :mod:`sklearn.metrics.pairwise` + and vice versa: instances of subclasses of :class:`Kernel` can be passed as + ``metric`` to ``pairwise_kernels`` from :mod:`sklearn.metrics.pairwise`. Moreover, + kernel functions from pairwise can be used as GP kernels by using the wrapper + class :class:`PairwiseKernel`. The only caveat is that the gradient of + the hyperparameters is not analytic but numeric and all those kernels support + only isotropic distances. The parameter ``gamma`` is considered to be a + hyperparameter and may be optimized. The other kernel parameters are set + directly at initialization and are kept fixed. Basic kernels ------------- @@ -484,7 +362,7 @@ Note that magic methods ``__add__``, ``__mul___`` and ``__pow__`` are overridden on the Kernel objects, so one can use e.g. ``RBF() + RBF()`` as a shortcut for ``Sum(RBF(), RBF())``. -Radial-basis function (RBF) kernel +Radial basis function (RBF) kernel ---------------------------------- The :class:`RBF` kernel is a stationary kernel. It is also known as the "squared exponential" kernel. It is parameterized by a length-scale parameter :math:`l>0`, which @@ -510,36 +388,41 @@ MatÊrn kernel ------------- The :class:`Matern` kernel is a stationary kernel and a generalization of the :class:`RBF` kernel. It has an additional parameter :math:`\nu` which controls -the smoothness of the resulting function. It is parameterized by a length-scale parameter :math:`l>0`, which can either be a scalar (isotropic variant of the kernel) or a vector with the same number of dimensions as the inputs :math:`x` (anisotropic variant of the kernel). The kernel is given by: +the smoothness of the resulting function. It is parameterized by a length-scale parameter :math:`l>0`, which can either be a scalar (isotropic variant of the kernel) or a vector with the same number of dimensions as the inputs :math:`x` (anisotropic variant of the kernel). -.. math:: +.. dropdown:: Mathematical implementation of MatÊrn kernel - k(x_i, x_j) = \frac{1}{\Gamma(\nu)2^{\nu-1}}\Bigg(\frac{\sqrt{2\nu}}{l} d(x_i , x_j )\Bigg)^\nu K_\nu\Bigg(\frac{\sqrt{2\nu}}{l} d(x_i , x_j )\Bigg), + The kernel is given by: -where :math:`d(\cdot,\cdot)` is the Euclidean distance, :math:`K_\nu(\cdot)` is a modified Bessel function and :math:`\Gamma(\cdot)` is the gamma function. -As :math:`\nu\rightarrow\infty`, the MatÊrn kernel converges to the RBF kernel. -When :math:`\nu = 1/2`, the MatÊrn kernel becomes identical to the absolute -exponential kernel, i.e., + .. math:: -.. math:: - k(x_i, x_j) = \exp \Bigg(- \frac{1}{l} d(x_i , x_j ) \Bigg) \quad \quad \nu= \tfrac{1}{2} + k(x_i, x_j) = \frac{1}{\Gamma(\nu)2^{\nu-1}}\Bigg(\frac{\sqrt{2\nu}}{l} d(x_i , x_j )\Bigg)^\nu K_\nu\Bigg(\frac{\sqrt{2\nu}}{l} d(x_i , x_j )\Bigg), -In particular, :math:`\nu = 3/2`: + where :math:`d(\cdot,\cdot)` is the Euclidean distance, :math:`K_\nu(\cdot)` is a modified Bessel function and :math:`\Gamma(\cdot)` is the gamma function. + As :math:`\nu\rightarrow\infty`, the MatÊrn kernel converges to the RBF kernel. + When :math:`\nu = 1/2`, the MatÊrn kernel becomes identical to the absolute + exponential kernel, i.e., -.. math:: - k(x_i, x_j) = \Bigg(1 + \frac{\sqrt{3}}{l} d(x_i , x_j )\Bigg) \exp \Bigg(-\frac{\sqrt{3}}{l} d(x_i , x_j ) \Bigg) \quad \quad \nu= \tfrac{3}{2} + .. math:: + k(x_i, x_j) = \exp \Bigg(- \frac{1}{l} d(x_i , x_j ) \Bigg) \quad \quad \nu= \tfrac{1}{2} -and :math:`\nu = 5/2`: + In particular, :math:`\nu = 3/2`: -.. math:: - k(x_i, x_j) = \Bigg(1 + \frac{\sqrt{5}}{l} d(x_i , x_j ) +\frac{5}{3l} d(x_i , x_j )^2 \Bigg) \exp \Bigg(-\frac{\sqrt{5}}{l} d(x_i , x_j ) \Bigg) \quad \quad \nu= \tfrac{5}{2} + .. math:: + k(x_i, x_j) = \Bigg(1 + \frac{\sqrt{3}}{l} d(x_i , x_j )\Bigg) \exp \Bigg(-\frac{\sqrt{3}}{l} d(x_i , x_j ) \Bigg) \quad \quad \nu= \tfrac{3}{2} + + and :math:`\nu = 5/2`: + + .. math:: + k(x_i, x_j) = \Bigg(1 + \frac{\sqrt{5}}{l} d(x_i , x_j ) +\frac{5}{3l} d(x_i , x_j )^2 \Bigg) \exp \Bigg(-\frac{\sqrt{5}}{l} d(x_i , x_j ) \Bigg) \quad \quad \nu= \tfrac{5}{2} + + are popular choices for learning functions that are not infinitely + differentiable (as assumed by the RBF kernel) but at least once (:math:`\nu = + 3/2`) or twice differentiable (:math:`\nu = 5/2`). -are popular choices for learning functions that are not infinitely -differentiable (as assumed by the RBF kernel) but at least once (:math:`\nu = -3/2`) or twice differentiable (:math:`\nu = 5/2`). + The flexibility of controlling the smoothness of the learned function via :math:`\nu` + allows adapting to the properties of the true underlying functional relation. -The flexibility of controlling the smoothness of the learned function via :math:`\nu` -allows adapting to the properties of the true underlying functional relation. The prior and posterior of a GP resulting from a MatÊrn kernel are shown in the following figure: @@ -610,8 +493,11 @@ shown in the following figure: References ---------- -.. [RW2006] Carl Eduard Rasmussen and Christopher K.I. Williams, "Gaussian Processes for Machine Learning", MIT Press 2006, Link to an official complete PDF version of the book `here `_ . +.. [RW2006] `Carl E. Rasmussen and Christopher K.I. Williams, + "Gaussian Processes for Machine Learning", + MIT Press 2006 `_ -.. [Duv2014] David Duvenaud, "The Kernel Cookbook: Advice on Covariance functions", 2014, `Link `_ . +.. [Duv2014] `David Duvenaud, "The Kernel Cookbook: Advice on Covariance functions", 2014 + `_ .. currentmodule:: sklearn.gaussian_process diff --git a/doc/modules/grid_search.rst b/doc/modules/grid_search.rst index 331607c86e657..edb915b193e37 100644 --- a/doc/modules/grid_search.rst +++ b/doc/modules/grid_search.rst @@ -72,35 +72,41 @@ evaluated and the best combination is retained. .. currentmodule:: sklearn.model_selection -.. topic:: Examples: +.. rubric:: Examples - - See :ref:`sphx_glr_auto_examples_model_selection_plot_grid_search_digits.py` for an example of - Grid Search computation on the digits dataset. +- See :ref:`sphx_glr_auto_examples_model_selection_plot_nested_cross_validation_iris.py` + for an example of Grid Search within a cross validation loop on the iris + dataset. This is the best practice for evaluating the performance of a + model with grid search. - - See :ref:`sphx_glr_auto_examples_model_selection_grid_search_text_feature_extraction.py` for an example - of Grid Search coupling parameters from a text documents feature - extractor (n-gram count vectorizer and TF-IDF transformer) with a - classifier (here a linear SVM trained with SGD with either elastic - net or L2 penalty) using a :class:`pipeline.Pipeline` instance. +- See :ref:`sphx_glr_auto_examples_model_selection_plot_grid_search_text_feature_extraction.py` for an example + of Grid Search coupling parameters from a text documents feature + extractor (n-gram count vectorizer and TF-IDF transformer) with a + classifier (here a linear SVM trained with SGD with either elastic + net or L2 penalty) using a :class:`~sklearn.pipeline.Pipeline` instance. - - See :ref:`sphx_glr_auto_examples_model_selection_plot_nested_cross_validation_iris.py` - for an example of Grid Search within a cross validation loop on the iris - dataset. This is the best practice for evaluating the performance of a - model with grid search. - - See :ref:`sphx_glr_auto_examples_model_selection_plot_multi_metric_evaluation.py` - for an example of :class:`GridSearchCV` being used to evaluate multiple - metrics simultaneously. +.. dropdown:: Advanced examples - - See :ref:`sphx_glr_auto_examples_model_selection_plot_grid_search_refit_callable.py` - for an example of using ``refit=callable`` interface in - :class:`GridSearchCV`. The example shows how this interface adds certain - amount of flexibility in identifying the "best" estimator. This interface - can also be used in multiple metrics evaluation. + - See :ref:`sphx_glr_auto_examples_model_selection_plot_nested_cross_validation_iris.py` + for an example of Grid Search within a cross validation loop on the iris + dataset. This is the best practice for evaluating the performance of a + model with grid search. + + - See :ref:`sphx_glr_auto_examples_model_selection_plot_multi_metric_evaluation.py` + for an example of :class:`GridSearchCV` being used to evaluate multiple + metrics simultaneously. + + - See :ref:`sphx_glr_auto_examples_model_selection_plot_grid_search_refit_callable.py` + for an example of using ``refit=callable`` interface in + :class:`GridSearchCV`. The example shows how this interface adds a certain + amount of flexibility in identifying the "best" estimator. This interface + can also be used in multiple metrics evaluation. + + - See :ref:`sphx_glr_auto_examples_model_selection_plot_grid_search_stats.py` + for an example of how to do a statistical comparison on the outputs of + :class:`GridSearchCV`. - - See :ref:`sphx_glr_auto_examples_model_selection_plot_grid_search_stats.py` - for an example of how to do a statistical comparison on the outputs of - :class:`GridSearchCV`. .. _randomized_parameter_search: @@ -108,7 +114,7 @@ Randomized Parameter Optimization ================================= While using a grid of parameter settings is currently the most widely used method for parameter optimization, other search methods have more -favourable properties. +favorable properties. :class:`RandomizedSearchCV` implements a randomized search over parameters, where each setting is sampled from a distribution over possible parameter values. This has two main benefits over an exhaustive search: @@ -128,32 +134,29 @@ discrete choices (which will be sampled uniformly) can be specified:: This example uses the ``scipy.stats`` module, which contains many useful distributions for sampling parameters, such as ``expon``, ``gamma``, -``uniform`` or ``randint``. +``uniform``, ``loguniform`` or ``randint``. In principle, any function can be passed that provides a ``rvs`` (random variate sample) method to sample a value. A call to the ``rvs`` function should provide independent random samples from possible parameter values on consecutive calls. - .. warning:: +.. warning:: - The distributions in ``scipy.stats`` prior to version scipy 0.16 - do not allow specifying a random state. Instead, they use the global - numpy random state, that can be seeded via ``np.random.seed`` or set - using ``np.random.set_state``. However, beginning scikit-learn 0.18, - the :mod:`sklearn.model_selection` module sets the random state provided - by the user if scipy >= 0.16 is also available. + The distributions in ``scipy.stats`` prior to version scipy 0.16 + do not allow specifying a random state. Instead, they use the global + numpy random state, that can be seeded via ``np.random.seed`` or set + using ``np.random.set_state``. However, beginning scikit-learn 0.18, + the :mod:`sklearn.model_selection` module sets the random state provided + by the user if scipy >= 0.16 is also available. For continuous parameters, such as ``C`` above, it is important to specify a continuous distribution to take full advantage of the randomization. This way, increasing ``n_iter`` will always lead to a finer search. -A continuous log-uniform random variable is available through -:class:`~sklearn.utils.fixes.loguniform`. This is a continuous version of -log-spaced parameters. For example to specify ``C`` above, ``loguniform(1, -100)`` can be used instead of ``[1, 10, 100]`` or ``np.logspace(0, 2, -num=1000)``. This is an alias to SciPy's `stats.reciprocal -`_. +A continuous log-uniform random variable is the continuous version of +a log-spaced parameter. For example to specify the equivalent of ``C`` from above, +``loguniform(1, 100)`` can be used instead of ``[1, 10, 100]``. Mirroring the example above in grid search, we can specify a continuous random variable that is log-uniformly distributed between ``1e0`` and ``1e3``:: @@ -164,16 +167,16 @@ variable that is log-uniformly distributed between ``1e0`` and ``1e3``:: 'kernel': ['rbf'], 'class_weight':['balanced', None]} -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_model_selection_plot_randomized_search.py` compares the usage and efficiency - of randomized search and grid search. +* :ref:`sphx_glr_auto_examples_model_selection_plot_randomized_search.py` compares the usage and efficiency + of randomized search and grid search. -.. topic:: References: +.. rubric:: References - * Bergstra, J. and Bengio, Y., - Random search for hyper-parameter optimization, - The Journal of Machine Learning Research (2012) +* Bergstra, J. and Bengio, Y., + Random search for hyper-parameter optimization, + The Journal of Machine Learning Research (2012) .. _successive_halving_user_guide: @@ -191,6 +194,11 @@ iteration, which will be allocated more resources. For parameter tuning, the resource is typically the number of training samples, but it can also be an arbitrary numeric parameter such as `n_estimators` in a random forest. +.. note:: + + The resource increase chosen should be large enough so that a large improvement + in scores is obtained when taking into account statistical significance. + As illustrated in the figure below, only a subset of candidates 'survive' until the last iteration. These are the candidates that have consistently ranked among the top-scoring candidates across all iterations. @@ -202,7 +210,7 @@ here the number of samples. :align: center We here briefly describe the main parameters, but each parameter and their -interactions are described in more details in the sections below. The +interactions are described more in detail in the dropdown section below. The ``factor`` (> 1) parameter controls the rate at which the resources grow, and the rate at which the number of candidates decreases. In each iteration, the number of resources per candidate is multiplied by ``factor`` and the number @@ -219,328 +227,323 @@ These estimators are still **experimental**: their predictions and their API might change without any deprecation cycle. To use them, you need to explicitly import ``enable_halving_search_cv``:: - >>> # explicitly require this experimental feature >>> from sklearn.experimental import enable_halving_search_cv # noqa - >>> # now you can import normally from model_selection >>> from sklearn.model_selection import HalvingGridSearchCV >>> from sklearn.model_selection import HalvingRandomSearchCV -.. topic:: Examples: - - * :ref:`sphx_glr_auto_examples_model_selection_plot_successive_halving_heatmap.py` - * :ref:`sphx_glr_auto_examples_model_selection_plot_successive_halving_iterations.py` - -Choosing ``min_resources`` and the number of candidates -------------------------------------------------------- - -Beside ``factor``, the two main parameters that influence the behaviour of a -successive halving search are the ``min_resources`` parameter, and the -number of candidates (or parameter combinations) that are evaluated. -``min_resources`` is the amount of resources allocated at the first -iteration for each candidate. The number of candidates is specified directly -in :class:`HalvingRandomSearchCV`, and is determined from the ``param_grid`` -parameter of :class:`HalvingGridSearchCV`. - -Consider a case where the resource is the number of samples, and where we -have 1000 samples. In theory, with ``min_resources=10`` and ``factor=2``, we -are able to run **at most** 7 iterations with the following number of -samples: ``[10, 20, 40, 80, 160, 320, 640]``. - -But depending on the number of candidates, we might run less than 7 -iterations: if we start with a **small** number of candidates, the last -iteration might use less than 640 samples, which means not using all the -available resources (samples). For example if we start with 5 candidates, we -only need 2 iterations: 5 candidates for the first iteration, then -`5 // 2 = 2` candidates at the second iteration, after which we know which -candidate performs the best (so we don't need a third one). We would only be -using at most 20 samples which is a waste since we have 1000 samples at our -disposal. On the other hand, if we start with a **high** number of -candidates, we might end up with a lot of candidates at the last iteration, -which may not always be ideal: it means that many candidates will run with -the full resources, basically reducing the procedure to standard search. - -In the case of :class:`HalvingRandomSearchCV`, the number of candidates is set -by default such that the last iteration uses as much of the available -resources as possible. For :class:`HalvingGridSearchCV`, the number of -candidates is determined by the `param_grid` parameter. Changing the value of -``min_resources`` will impact the number of possible iterations, and as a -result will also have an effect on the ideal number of candidates. - -Another consideration when choosing ``min_resources`` is whether or not it -is easy to discriminate between good and bad candidates with a small amount -of resources. For example, if you need a lot of samples to distinguish -between good and bad parameters, a high ``min_resources`` is recommended. On -the other hand if the distinction is clear even with a small amount of -samples, then a small ``min_resources`` may be preferable since it would -speed up the computation. - -Notice in the example above that the last iteration does not use the maximum -amount of resources available: 1000 samples are available, yet only 640 are -used, at most. By default, both :class:`HalvingRandomSearchCV` and -:class:`HalvingGridSearchCV` try to use as many resources as possible in the -last iteration, with the constraint that this amount of resources must be a -multiple of both `min_resources` and `factor` (this constraint will be clear -in the next section). :class:`HalvingRandomSearchCV` achieves this by -sampling the right amount of candidates, while :class:`HalvingGridSearchCV` -achieves this by properly setting `min_resources`. Please see -:ref:`exhausting_the_resources` for details. - -.. _amount_of_resource_and_number_of_candidates: - -Amount of resource and number of candidates at each iteration -------------------------------------------------------------- - -At any iteration `i`, each candidate is allocated a given amount of resources -which we denote `n_resources_i`. This quantity is controlled by the -parameters ``factor`` and ``min_resources`` as follows (`factor` is strictly -greater than 1):: - - n_resources_i = factor**i * min_resources, - -or equivalently:: - - n_resources_{i+1} = n_resources_i * factor - -where ``min_resources == n_resources_0`` is the amount of resources used at -the first iteration. ``factor`` also defines the proportions of candidates -that will be selected for the next iteration:: - - n_candidates_i = n_candidates // (factor ** i) - -or equivalently:: - - n_candidates_0 = n_candidates - n_candidates_{i+1} = n_candidates_i // factor - -So in the first iteration, we use ``min_resources`` resources -``n_candidates`` times. In the second iteration, we use ``min_resources * -factor`` resources ``n_candidates // factor`` times. The third again -multiplies the resources per candidate and divides the number of candidates. -This process stops when the maximum amount of resource per candidate is -reached, or when we have identified the best candidate. The best candidate -is identified at the iteration that is evaluating `factor` or less candidates -(see just below for an explanation). - -Here is an example with ``min_resources=3`` and ``factor=2``, starting with -70 candidates: - -+-----------------------+-----------------------+ -| ``n_resources_i`` | ``n_candidates_i`` | -+=======================+=======================+ -| 3 (=min_resources) | 70 (=n_candidates) | -+-----------------------+-----------------------+ -| 3 * 2 = 6 | 70 // 2 = 35 | -+-----------------------+-----------------------+ -| 6 * 2 = 12 | 35 // 2 = 17 | -+-----------------------+-----------------------+ -| 12 * 2 = 24 | 17 // 2 = 8 | -+-----------------------+-----------------------+ -| 24 * 2 = 48 | 8 // 2 = 4 | -+-----------------------+-----------------------+ -| 48 * 2 = 96 | 4 // 2 = 2 | -+-----------------------+-----------------------+ - -We can note that: - -- the process stops at the first iteration which evaluates `factor=2` - candidates: the best candidate is the best out of these 2 candidates. It - is not necessary to run an additional iteration, since it would only - evaluate one candidate (namely the best one, which we have already - identified). For this reason, in general, we want the last iteration to - run at most ``factor`` candidates. If the last iteration evaluates more - than `factor` candidates, then this last iteration reduces to a regular - search (as in :class:`RandomizedSearchCV` or :class:`GridSearchCV`). -- each ``n_resources_i`` is a multiple of both ``factor`` and - ``min_resources`` (which is confirmed by its definition above). - -The amount of resources that is used at each iteration can be found in the -`n_resources_` attribute. - -Choosing a resource -------------------- - -By default, the resource is defined in terms of number of samples. That is, -each iteration will use an increasing amount of samples to train on. You can -however manually specify a parameter to use as the resource with the -``resource`` parameter. Here is an example where the resource is defined in -terms of the number of estimators of a random forest:: - - >>> from sklearn.datasets import make_classification - >>> from sklearn.ensemble import RandomForestClassifier - >>> from sklearn.experimental import enable_halving_search_cv # noqa - >>> from sklearn.model_selection import HalvingGridSearchCV - >>> import pandas as pd - >>> - >>> param_grid = {'max_depth': [3, 5, 10], - ... 'min_samples_split': [2, 5, 10]} - >>> base_estimator = RandomForestClassifier(random_state=0) - >>> X, y = make_classification(n_samples=1000, random_state=0) - >>> sh = HalvingGridSearchCV(base_estimator, param_grid, cv=5, - ... factor=2, resource='n_estimators', - ... max_resources=30).fit(X, y) - >>> sh.best_estimator_ - RandomForestClassifier(max_depth=5, n_estimators=24, random_state=0) - -Note that it is not possible to budget on a parameter that is part of the -parameter grid. - -.. _exhausting_the_resources: - -Exhausting the available resources ----------------------------------- - -As mentioned above, the number of resources that is used at each iteration -depends on the `min_resources` parameter. -If you have a lot of resources available but start with a low number of -resources, some of them might be wasted (i.e. not used):: - - >>> from sklearn.datasets import make_classification - >>> from sklearn.svm import SVC - >>> from sklearn.experimental import enable_halving_search_cv # noqa - >>> from sklearn.model_selection import HalvingGridSearchCV - >>> import pandas as pd - >>> param_grid= {'kernel': ('linear', 'rbf'), - ... 'C': [1, 10, 100]} - >>> base_estimator = SVC(gamma='scale') - >>> X, y = make_classification(n_samples=1000) - >>> sh = HalvingGridSearchCV(base_estimator, param_grid, cv=5, - ... factor=2, min_resources=20).fit(X, y) - >>> sh.n_resources_ - [20, 40, 80] - -The search process will only use 80 resources at most, while our maximum -amount of available resources is ``n_samples=1000``. Here, we have -``min_resources = r_0 = 20``. - -For :class:`HalvingGridSearchCV`, by default, the `min_resources` parameter -is set to 'exhaust'. This means that `min_resources` is automatically set -such that the last iteration can use as many resources as possible, within -the `max_resources` limit:: - - >>> sh = HalvingGridSearchCV(base_estimator, param_grid, cv=5, - ... factor=2, min_resources='exhaust').fit(X, y) - >>> sh.n_resources_ - [250, 500, 1000] - -`min_resources` was here automatically set to 250, which results in the last -iteration using all the resources. The exact value that is used depends on -the number of candidate parameter, on `max_resources` and on `factor`. - -For :class:`HalvingRandomSearchCV`, exhausting the resources can be done in 2 -ways: - -- by setting `min_resources='exhaust'`, just like for - :class:`HalvingGridSearchCV`; -- by setting `n_candidates='exhaust'`. - -Both options are mutally exclusive: using `min_resources='exhaust'` requires -knowing the number of candidates, and symmetrically `n_candidates='exhaust'` -requires knowing `min_resources`. - -In general, exhausting the total number of resources leads to a better final -candidate parameter, and is slightly more time-intensive. +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_model_selection_plot_successive_halving_heatmap.py` +* :ref:`sphx_glr_auto_examples_model_selection_plot_successive_halving_iterations.py` + +The sections below dive into technical aspects of successive halving. + +.. dropdown:: Choosing ``min_resources`` and the number of candidates + + Beside ``factor``, the two main parameters that influence the behaviour of a + successive halving search are the ``min_resources`` parameter, and the + number of candidates (or parameter combinations) that are evaluated. + ``min_resources`` is the amount of resources allocated at the first + iteration for each candidate. The number of candidates is specified directly + in :class:`HalvingRandomSearchCV`, and is determined from the ``param_grid`` + parameter of :class:`HalvingGridSearchCV`. + + Consider a case where the resource is the number of samples, and where we + have 1000 samples. In theory, with ``min_resources=10`` and ``factor=2``, we + are able to run **at most** 7 iterations with the following number of + samples: ``[10, 20, 40, 80, 160, 320, 640]``. + + But depending on the number of candidates, we might run less than 7 + iterations: if we start with a **small** number of candidates, the last + iteration might use less than 640 samples, which means not using all the + available resources (samples). For example if we start with 5 candidates, we + only need 2 iterations: 5 candidates for the first iteration, then + `5 // 2 = 2` candidates at the second iteration, after which we know which + candidate performs the best (so we don't need a third one). We would only be + using at most 20 samples which is a waste since we have 1000 samples at our + disposal. On the other hand, if we start with a **high** number of + candidates, we might end up with a lot of candidates at the last iteration, + which may not always be ideal: it means that many candidates will run with + the full resources, basically reducing the procedure to standard search. + + In the case of :class:`HalvingRandomSearchCV`, the number of candidates is set + by default such that the last iteration uses as much of the available + resources as possible. For :class:`HalvingGridSearchCV`, the number of + candidates is determined by the `param_grid` parameter. Changing the value of + ``min_resources`` will impact the number of possible iterations, and as a + result will also have an effect on the ideal number of candidates. + + Another consideration when choosing ``min_resources`` is whether or not it + is easy to discriminate between good and bad candidates with a small amount + of resources. For example, if you need a lot of samples to distinguish + between good and bad parameters, a high ``min_resources`` is recommended. On + the other hand if the distinction is clear even with a small amount of + samples, then a small ``min_resources`` may be preferable since it would + speed up the computation. + + Notice in the example above that the last iteration does not use the maximum + amount of resources available: 1000 samples are available, yet only 640 are + used, at most. By default, both :class:`HalvingRandomSearchCV` and + :class:`HalvingGridSearchCV` try to use as many resources as possible in the + last iteration, with the constraint that this amount of resources must be a + multiple of both `min_resources` and `factor` (this constraint will be clear + in the next section). :class:`HalvingRandomSearchCV` achieves this by + sampling the right amount of candidates, while :class:`HalvingGridSearchCV` + achieves this by properly setting `min_resources`. + + +.. dropdown:: Amount of resource and number of candidates at each iteration + + At any iteration `i`, each candidate is allocated a given amount of resources + which we denote `n_resources_i`. This quantity is controlled by the + parameters ``factor`` and ``min_resources`` as follows (`factor` is strictly + greater than 1):: + + n_resources_i = factor**i * min_resources, + + or equivalently:: + + n_resources_{i+1} = n_resources_i * factor + + where ``min_resources == n_resources_0`` is the amount of resources used at + the first iteration. ``factor`` also defines the proportions of candidates + that will be selected for the next iteration:: + + n_candidates_i = n_candidates // (factor ** i) + + or equivalently:: + + n_candidates_0 = n_candidates + n_candidates_{i+1} = n_candidates_i // factor + + So in the first iteration, we use ``min_resources`` resources + ``n_candidates`` times. In the second iteration, we use ``min_resources * + factor`` resources ``n_candidates // factor`` times. The third again + multiplies the resources per candidate and divides the number of candidates. + This process stops when the maximum amount of resource per candidate is + reached, or when we have identified the best candidate. The best candidate + is identified at the iteration that is evaluating `factor` or less candidates + (see just below for an explanation). + + Here is an example with ``min_resources=3`` and ``factor=2``, starting with + 70 candidates: + + +-----------------------+-----------------------+ + | ``n_resources_i`` | ``n_candidates_i`` | + +=======================+=======================+ + | 3 (=min_resources) | 70 (=n_candidates) | + +-----------------------+-----------------------+ + | 3 * 2 = 6 | 70 // 2 = 35 | + +-----------------------+-----------------------+ + | 6 * 2 = 12 | 35 // 2 = 17 | + +-----------------------+-----------------------+ + | 12 * 2 = 24 | 17 // 2 = 8 | + +-----------------------+-----------------------+ + | 24 * 2 = 48 | 8 // 2 = 4 | + +-----------------------+-----------------------+ + | 48 * 2 = 96 | 4 // 2 = 2 | + +-----------------------+-----------------------+ + + We can note that: + + - the process stops at the first iteration which evaluates `factor=2` + candidates: the best candidate is the best out of these 2 candidates. It + is not necessary to run an additional iteration, since it would only + evaluate one candidate (namely the best one, which we have already + identified). For this reason, in general, we want the last iteration to + run at most ``factor`` candidates. If the last iteration evaluates more + than `factor` candidates, then this last iteration reduces to a regular + search (as in :class:`RandomizedSearchCV` or :class:`GridSearchCV`). + - each ``n_resources_i`` is a multiple of both ``factor`` and + ``min_resources`` (which is confirmed by its definition above). + + The amount of resources that is used at each iteration can be found in the + `n_resources_` attribute. + +.. dropdown:: Choosing a resource + + By default, the resource is defined in terms of number of samples. That is, + each iteration will use an increasing amount of samples to train on. You can + however manually specify a parameter to use as the resource with the + ``resource`` parameter. Here is an example where the resource is defined in + terms of the number of estimators of a random forest:: + + >>> from sklearn.datasets import make_classification + >>> from sklearn.ensemble import RandomForestClassifier + >>> from sklearn.experimental import enable_halving_search_cv # noqa + >>> from sklearn.model_selection import HalvingGridSearchCV + >>> import pandas as pd + >>> param_grid = {'max_depth': [3, 5, 10], + ... 'min_samples_split': [2, 5, 10]} + >>> base_estimator = RandomForestClassifier(random_state=0) + >>> X, y = make_classification(n_samples=1000, random_state=0) + >>> sh = HalvingGridSearchCV(base_estimator, param_grid, cv=5, + ... factor=2, resource='n_estimators', + ... max_resources=30).fit(X, y) + >>> sh.best_estimator_ + RandomForestClassifier(max_depth=5, n_estimators=24, random_state=0) + + Note that it is not possible to budget on a parameter that is part of the + parameter grid. + + +.. dropdown:: Exhausting the available resources + + As mentioned above, the number of resources that is used at each iteration + depends on the `min_resources` parameter. + If you have a lot of resources available but start with a low number of + resources, some of them might be wasted (i.e. not used):: + + >>> from sklearn.datasets import make_classification + >>> from sklearn.svm import SVC + >>> from sklearn.experimental import enable_halving_search_cv # noqa + >>> from sklearn.model_selection import HalvingGridSearchCV + >>> import pandas as pd + >>> param_grid= {'kernel': ('linear', 'rbf'), + ... 'C': [1, 10, 100]} + >>> base_estimator = SVC(gamma='scale') + >>> X, y = make_classification(n_samples=1000) + >>> sh = HalvingGridSearchCV(base_estimator, param_grid, cv=5, + ... factor=2, min_resources=20).fit(X, y) + >>> sh.n_resources_ + [20, 40, 80] + + The search process will only use 80 resources at most, while our maximum + amount of available resources is ``n_samples=1000``. Here, we have + ``min_resources = r_0 = 20``. + + For :class:`HalvingGridSearchCV`, by default, the `min_resources` parameter + is set to 'exhaust'. This means that `min_resources` is automatically set + such that the last iteration can use as many resources as possible, within + the `max_resources` limit:: + + >>> sh = HalvingGridSearchCV(base_estimator, param_grid, cv=5, + ... factor=2, min_resources='exhaust').fit(X, y) + >>> sh.n_resources_ + [250, 500, 1000] + + `min_resources` was here automatically set to 250, which results in the last + iteration using all the resources. The exact value that is used depends on + the number of candidate parameters, on `max_resources` and on `factor`. + + For :class:`HalvingRandomSearchCV`, exhausting the resources can be done in 2 + ways: + + - by setting `min_resources='exhaust'`, just like for + :class:`HalvingGridSearchCV`; + - by setting `n_candidates='exhaust'`. + + Both options are mutually exclusive: using `min_resources='exhaust'` requires + knowing the number of candidates, and symmetrically `n_candidates='exhaust'` + requires knowing `min_resources`. + + In general, exhausting the total number of resources leads to a better final + candidate parameter, and is slightly more time-intensive. .. _aggressive_elimination: Aggressive elimination of candidates ------------------------------------ -Ideally, we want the last iteration to evaluate ``factor`` candidates (see -:ref:`amount_of_resource_and_number_of_candidates`). We then just have to -pick the best one. When the number of available resources is small with -respect to the number of candidates, the last iteration may have to evaluate -more than ``factor`` candidates:: - - >>> from sklearn.datasets import make_classification - >>> from sklearn.svm import SVC - >>> from sklearn.experimental import enable_halving_search_cv # noqa - >>> from sklearn.model_selection import HalvingGridSearchCV - >>> import pandas as pd - >>> - >>> - >>> param_grid = {'kernel': ('linear', 'rbf'), - ... 'C': [1, 10, 100]} - >>> base_estimator = SVC(gamma='scale') - >>> X, y = make_classification(n_samples=1000) - >>> sh = HalvingGridSearchCV(base_estimator, param_grid, cv=5, - ... factor=2, max_resources=40, - ... aggressive_elimination=False).fit(X, y) - >>> sh.n_resources_ - [20, 40] - >>> sh.n_candidates_ - [6, 3] - -Since we cannot use more than ``max_resources=40`` resources, the process -has to stop at the second iteration which evaluates more than ``factor=2`` -candidates. - Using the ``aggressive_elimination`` parameter, you can force the search process to end up with less than ``factor`` candidates at the last -iteration. To do this, the process will eliminate as many candidates as -necessary using ``min_resources`` resources:: - - >>> sh = HalvingGridSearchCV(base_estimator, param_grid, cv=5, - ... factor=2, - ... max_resources=40, - ... aggressive_elimination=True, - ... ).fit(X, y) - >>> sh.n_resources_ - [20, 20, 40] - >>> sh.n_candidates_ - [6, 3, 2] - -Notice that we end with 2 candidates at the last iteration since we have -eliminated enough candidates during the first iterations, using ``n_resources = -min_resources = 20``. +iteration. + +.. dropdown:: Code example of aggressive elimination + + Ideally, we want the last iteration to evaluate ``factor`` candidates. We + then just have to pick the best one. When the number of available resources is + small with respect to the number of candidates, the last iteration may have to + evaluate more than ``factor`` candidates:: + + >>> from sklearn.datasets import make_classification + >>> from sklearn.svm import SVC + >>> from sklearn.experimental import enable_halving_search_cv # noqa + >>> from sklearn.model_selection import HalvingGridSearchCV + >>> import pandas as pd + >>> param_grid = {'kernel': ('linear', 'rbf'), + ... 'C': [1, 10, 100]} + >>> base_estimator = SVC(gamma='scale') + >>> X, y = make_classification(n_samples=1000) + >>> sh = HalvingGridSearchCV(base_estimator, param_grid, cv=5, + ... factor=2, max_resources=40, + ... aggressive_elimination=False).fit(X, y) + >>> sh.n_resources_ + [20, 40] + >>> sh.n_candidates_ + [6, 3] + + Since we cannot use more than ``max_resources=40`` resources, the process + has to stop at the second iteration which evaluates more than ``factor=2`` + candidates. + + When using ``aggressive_elimination``, the process will eliminate as many + candidates as necessary using ``min_resources`` resources:: + + >>> sh = HalvingGridSearchCV(base_estimator, param_grid, cv=5, + ... factor=2, + ... max_resources=40, + ... aggressive_elimination=True, + ... ).fit(X, y) + >>> sh.n_resources_ + [20, 20, 40] + >>> sh.n_candidates_ + [6, 3, 2] + + Notice that we end with 2 candidates at the last iteration since we have + eliminated enough candidates during the first iterations, using ``n_resources = + min_resources = 20``. .. _successive_halving_cv_results: -Analysing results with the `cv_results_` attribute +Analyzing results with the `cv_results_` attribute -------------------------------------------------- -The ``cv_results_`` attribute contains useful information for analysing the +The ``cv_results_`` attribute contains useful information for analyzing the results of a search. It can be converted to a pandas dataframe with ``df = pd.DataFrame(est.cv_results_)``. The ``cv_results_`` attribute of :class:`HalvingGridSearchCV` and :class:`HalvingRandomSearchCV` is similar to that of :class:`GridSearchCV` and :class:`RandomizedSearchCV`, with additional information related to the successive halving process. -Here is an example with some of the columns of a (truncated) dataframe: - -==== ====== =============== ================= ======================================================================================= - .. iter n_resources mean_test_score params -==== ====== =============== ================= ======================================================================================= - 0 0 125 0.983667 {'criterion': 'entropy', 'max_depth': None, 'max_features': 9, 'min_samples_split': 5} - 1 0 125 0.983667 {'criterion': 'gini', 'max_depth': None, 'max_features': 8, 'min_samples_split': 7} - 2 0 125 0.983667 {'criterion': 'gini', 'max_depth': None, 'max_features': 10, 'min_samples_split': 10} - 3 0 125 0.983667 {'criterion': 'entropy', 'max_depth': None, 'max_features': 6, 'min_samples_split': 6} - ... ... ... ... ... - 15 2 500 0.951958 {'criterion': 'entropy', 'max_depth': None, 'max_features': 9, 'min_samples_split': 10} - 16 2 500 0.947958 {'criterion': 'gini', 'max_depth': None, 'max_features': 10, 'min_samples_split': 10} - 17 2 500 0.951958 {'criterion': 'gini', 'max_depth': None, 'max_features': 10, 'min_samples_split': 4} - 18 3 1000 0.961009 {'criterion': 'entropy', 'max_depth': None, 'max_features': 9, 'min_samples_split': 10} - 19 3 1000 0.955989 {'criterion': 'gini', 'max_depth': None, 'max_features': 10, 'min_samples_split': 4} -==== ====== =============== ================= ======================================================================================= - -Each row corresponds to a given parameter combination (a candidate) and a given -iteration. The iteration is given by the ``iter`` column. The ``n_resources`` -column tells you how many resources were used. - -In the example above, the best parameter combination is ``{'criterion': -'entropy', 'max_depth': None, 'max_features': 9, 'min_samples_split': 10}`` -since it has reached the last iteration (3) with the highest score: -0.96. - -.. topic:: References: - - .. [1] K. Jamieson, A. Talwalkar, - `Non-stochastic Best Arm Identification and Hyperparameter - Optimization `_, in - proc. of Machine Learning Research, 2016. - .. [2] L. Li, K. Jamieson, G. DeSalvo, A. Rostamizadeh, A. Talwalkar, - `Hyperband: A Novel Bandit-Based Approach to Hyperparameter Optimization - `_, in Machine Learning Research - 18, 2018. +.. dropdown:: Example of a (truncated) output dataframe: + + ==== ====== =============== ================= ======================================================================================== + .. iter n_resources mean_test_score params + ==== ====== =============== ================= ======================================================================================== + 0 0 125 0.983667 {'criterion': 'log_loss', 'max_depth': None, 'max_features': 9, 'min_samples_split': 5} + 1 0 125 0.983667 {'criterion': 'gini', 'max_depth': None, 'max_features': 8, 'min_samples_split': 7} + 2 0 125 0.983667 {'criterion': 'gini', 'max_depth': None, 'max_features': 10, 'min_samples_split': 10} + 3 0 125 0.983667 {'criterion': 'log_loss', 'max_depth': None, 'max_features': 6, 'min_samples_split': 6} + ... ... ... ... ... + 15 2 500 0.951958 {'criterion': 'log_loss', 'max_depth': None, 'max_features': 9, 'min_samples_split': 10} + 16 2 500 0.947958 {'criterion': 'gini', 'max_depth': None, 'max_features': 10, 'min_samples_split': 10} + 17 2 500 0.951958 {'criterion': 'gini', 'max_depth': None, 'max_features': 10, 'min_samples_split': 4} + 18 3 1000 0.961009 {'criterion': 'log_loss', 'max_depth': None, 'max_features': 9, 'min_samples_split': 10} + 19 3 1000 0.955989 {'criterion': 'gini', 'max_depth': None, 'max_features': 10, 'min_samples_split': 4} + ==== ====== =============== ================= ======================================================================================== + + Each row corresponds to a given parameter combination (a candidate) and a given + iteration. The iteration is given by the ``iter`` column. The ``n_resources`` + column tells you how many resources were used. + + In the example above, the best parameter combination is ``{'criterion': + 'log_loss', 'max_depth': None, 'max_features': 9, 'min_samples_split': 10}`` + since it has reached the last iteration (3) with the highest score: + 0.96. + + .. rubric:: References + + .. [1] K. Jamieson, A. Talwalkar, + `Non-stochastic Best Arm Identification and Hyperparameter + Optimization `_, in + proc. of Machine Learning Research, 2016. + + .. [2] L. Li, K. Jamieson, G. DeSalvo, A. Rostamizadeh, A. Talwalkar, + :arxiv:`Hyperband: A Novel Bandit-Based Approach to Hyperparameter Optimization + <1603.06560>`, in Machine Learning Research 18, 2018. + + .. _grid_search_tips: @@ -552,14 +555,15 @@ Tips for parameter search Specifying an objective metric ------------------------------ -By default, parameter search uses the ``score`` function of the estimator -to evaluate a parameter setting. These are the +By default, parameter search uses the ``score`` function of the estimator to +evaluate a parameter setting. These are the :func:`sklearn.metrics.accuracy_score` for classification and -:func:`sklearn.metrics.r2_score` for regression. For some applications, -other scoring functions are better suited (for example in unbalanced -classification, the accuracy score is often uninformative). An alternative -scoring function can be specified via the ``scoring`` parameter of most -parameter search tools. See :ref:`scoring_parameter` for more details. +:func:`sklearn.metrics.r2_score` for regression. For some applications, other +scoring functions are better suited (for example in unbalanced classification, +the accuracy score is often uninformative), see :ref:`which_scoring_function` +for some guidance. An alternative scoring function can be specified via the +``scoring`` parameter of most parameter search tools, see +:ref:`scoring_parameter` for more details. .. _multimetric_grid_search: @@ -603,20 +607,20 @@ parameters of composite or nested estimators such as >>> from sklearn.datasets import make_moons >>> X, y = make_moons() >>> calibrated_forest = CalibratedClassifierCV( - ... base_estimator=RandomForestClassifier(n_estimators=10)) + ... estimator=RandomForestClassifier(n_estimators=10)) >>> param_grid = { - ... 'base_estimator__max_depth': [2, 4, 6, 8]} + ... 'estimator__max_depth': [2, 4, 6, 8]} >>> search = GridSearchCV(calibrated_forest, param_grid, cv=5) >>> search.fit(X, y) GridSearchCV(cv=5, - estimator=CalibratedClassifierCV(...), - param_grid={'base_estimator__max_depth': [2, 4, 6, 8]}) + estimator=CalibratedClassifierCV(estimator=RandomForestClassifier(n_estimators=10)), + param_grid={'estimator__max_depth': [2, 4, 6, 8]}) Here, ```` is the parameter name of the nested estimator, -in this case ``base_estimator``. +in this case ``estimator``. If the meta-estimator is constructed as a collection of estimators as in `pipeline.Pipeline`, then ```` refers to the name of the estimator, -see :ref:`pipeline_nested_parameters`. In practice, there can be several +see :ref:`pipeline_nested_parameters`. In practice, there can be several levels of nesting:: >>> from sklearn.pipeline import Pipeline @@ -626,7 +630,7 @@ levels of nesting:: ... ('model', calibrated_forest)]) >>> param_grid = { ... 'select__k': [1, 2], - ... 'model__base_estimator__max_depth': [2, 4, 6, 8]} + ... 'model__estimator__max_depth': [2, 4, 6, 8]} >>> search = GridSearchCV(pipe, param_grid, cv=5).fit(X, y) Please refer to :ref:`pipeline` for performing parameter searches over @@ -658,12 +662,11 @@ entry for :term:`n_jobs`. Robustness to failure --------------------- -Some parameter settings may result in a failure to ``fit`` one or more folds -of the data. By default, this will cause the entire search to fail, even if -some parameter settings could be fully evaluated. Setting ``error_score=0`` -(or `=np.NaN`) will make the procedure robust to such failure, issuing a -warning and setting the score for that fold to 0 (or `NaN`), but completing -the search. +Some parameter settings may result in a failure to ``fit`` one or more folds of +the data. By default, the score for those settings will be `np.nan`. This can +be controlled by setting `error_score="raise"` to raise an exception if one fit +fails, or for example `error_score=0` to set another value for the score of +failing parameter combinations. .. _alternative_cv: @@ -722,7 +725,7 @@ model selection: Out of Bag Estimates -------------------- -When using ensemble methods base upon bagging, i.e. generating new +When using ensemble methods based upon bagging, i.e. generating new training sets using sampling with replacement, part of the training set remains unused. For each classifier in the ensemble, a different part of the training set is left out. diff --git a/doc/modules/impute.rst b/doc/modules/impute.rst index c3d2165e29d98..59367b647dd58 100644 --- a/doc/modules/impute.rst +++ b/doc/modules/impute.rst @@ -14,7 +14,7 @@ use incomplete datasets is to discard entire rows and/or columns containing missing values. However, this comes at the price of losing data which may be valuable (even though incomplete). A better strategy is to impute the missing values, i.e., to infer them from the known part of the data. See the -:ref:`glossary` entry on imputation. +glossary entry on :term:`imputation`. Univariate vs. Multivariate Imputation @@ -22,9 +22,9 @@ Univariate vs. Multivariate Imputation One type of imputation algorithm is univariate, which imputes values in the i-th feature dimension using only non-missing values in that feature dimension -(e.g. :class:`impute.SimpleImputer`). By contrast, multivariate imputation +(e.g. :class:`SimpleImputer`). By contrast, multivariate imputation algorithms use the entire set of available feature dimensions to estimate the -missing values (e.g. :class:`impute.IterativeImputer`). +missing values (e.g. :class:`IterativeImputer`). .. _single_imputer: @@ -50,7 +50,7 @@ that contain the missing values:: >>> X = [[np.nan, 2], [6, np.nan], [7, 6]] >>> print(imp.transform(X)) [[4. 2. ] - [6. 3.666...] + [6. 3.666] [7. 6. ]] The :class:`SimpleImputer` class also supports sparse matrices:: @@ -87,6 +87,8 @@ string values or pandas categoricals when using the ``'most_frequent'`` or ['a' 'y'] ['b' 'y']] +For another example on usage, see :ref:`sphx_glr_auto_examples_impute_plot_missing_values.py`. + .. _iterative_imputer: @@ -108,9 +110,9 @@ imputation round are returned. This estimator is still **experimental** for now: default parameters or details of behaviour might change without any deprecation cycle. Resolving the following issues would help stabilize :class:`IterativeImputer`: - convergence criteria (:issue:`14338`), default estimators (:issue:`13286`), - and use of random state (:issue:`15611`). To use it, you need to explicitly - import ``enable_iterative_imputer``. + convergence criteria (:issue:`14338`) and default estimators + (:issue:`13286`). To use it, you need to explicitly import + ``enable_iterative_imputer``. :: @@ -173,12 +175,11 @@ Note that a call to the ``transform`` method of :class:`IterativeImputer` is not allowed to change the number of samples. Therefore multiple imputations cannot be achieved by a single call to ``transform``. -References -========== +.. rubric:: References -.. [1] Stef van Buuren, Karin Groothuis-Oudshoorn (2011). "mice: Multivariate +.. [1] `Stef van Buuren, Karin Groothuis-Oudshoorn (2011). "mice: Multivariate Imputation by Chained Equations in R". Journal of Statistical Software 45: - 1-67. + 1-67. `_ .. [2] Roderick J A Little and Donald B Rubin (1986). "Statistical Analysis with Missing Data". John Wiley & Sons, Inc., New York, NY, USA. @@ -190,19 +191,20 @@ Nearest neighbors imputation The :class:`KNNImputer` class provides imputation for filling in missing values using the k-Nearest Neighbors approach. By default, a euclidean distance metric -that supports missing values, :func:`~sklearn.metrics.nan_euclidean_distances`, -is used to find the nearest neighbors. Each missing feature is imputed using -values from ``n_neighbors`` nearest neighbors that have a value for the -feature. The feature of the neighbors are averaged uniformly or weighted by -distance to each neighbor. If a sample has more than one feature missing, then -the neighbors for that sample can be different depending on the particular -feature being imputed. When the number of available neighbors is less than -`n_neighbors` and there are no defined distances to the training set, the -training set average for that feature is used during imputation. If there is at -least one neighbor with a defined distance, the weighted or unweighted average -of the remaining neighbors will be used during imputation. If a feature is -always missing in training, it is removed during `transform`. For more -information on the methodology, see ref. [OL2001]_. +that supports missing values, +:func:`~sklearn.metrics.pairwise.nan_euclidean_distances`, is used to find the +nearest neighbors. Each missing feature is imputed using values from +``n_neighbors`` nearest neighbors that have a value for the feature. The +feature of the neighbors are averaged uniformly or weighted by distance to each +neighbor. If a sample has more than one feature missing, then the neighbors for +that sample can be different depending on the particular feature being imputed. +When the number of available neighbors is less than `n_neighbors` and there are +no defined distances to the training set, the training set average for that +feature is used during imputation. If there is at least one neighbor with a +defined distance, the weighted or unweighted average of the remaining neighbors +will be used during imputation. If a feature is always missing in training, it +is removed during `transform`. For more information on the methodology, see +ref. [OL2001]_. The following snippet demonstrates how to replace missing values, encoded as ``np.nan``, using the mean feature value of the two nearest @@ -219,10 +221,42 @@ neighbors of samples with missing values:: [5.5, 6. , 5. ], [8. , 8. , 7. ]]) -.. [OL2001] Olga Troyanskaya, Michael Cantor, Gavin Sherlock, Pat Brown, +For another example on usage, see :ref:`sphx_glr_auto_examples_impute_plot_missing_values.py`. + +.. rubric:: References + +.. [OL2001] `Olga Troyanskaya, Michael Cantor, Gavin Sherlock, Pat Brown, Trevor Hastie, Robert Tibshirani, David Botstein and Russ B. Altman, Missing value estimation methods for DNA microarrays, BIOINFORMATICS Vol. 17 no. 6, 2001 Pages 520-525. + `_ + +Keeping the number of features constant +======================================= + +By default, the scikit-learn imputers will drop fully empty features, i.e. +columns containing only missing values. For instance:: + + >>> imputer = SimpleImputer() + >>> X = np.array([[np.nan, 1], [np.nan, 2], [np.nan, 3]]) + >>> imputer.fit_transform(X) + array([[1.], + [2.], + [3.]]) + +The first feature in `X` containing only `np.nan` was dropped after the +imputation. While this feature will not help in predictive setting, dropping +the columns will change the shape of `X` which could be problematic when using +imputers in a more complex machine-learning pipeline. The parameter +`keep_empty_features` offers the option to keep the empty features by imputing +with a constant value. In most of the cases, this constant value is zero:: + + >>> imputer.set_params(keep_empty_features=True) + SimpleImputer(keep_empty_features=True) + >>> imputer.fit_transform(X) + array([[0., 1.], + [0., 2.], + [0., 3.]]) .. _missing_indicator: @@ -274,10 +308,12 @@ whether or not they contain missing values:: >>> indicator.features_ array([0, 1, 2, 3]) -When using the :class:`MissingIndicator` in a :class:`Pipeline`, be sure to use -the :class:`FeatureUnion` or :class:`ColumnTransformer` to add the indicator -features to the regular features. First we obtain the `iris` dataset, and add -some missing values to it. +When using the :class:`MissingIndicator` in a +:class:`~sklearn.pipeline.Pipeline`, be sure to use the +:class:`~sklearn.pipeline.FeatureUnion` or +:class:`~sklearn.compose.ColumnTransformer` to add the indicator features to +the regular features. First we obtain the `iris` dataset, and add some missing +values to it. >>> from sklearn.datasets import load_iris >>> from sklearn.impute import SimpleImputer, MissingIndicator @@ -290,9 +326,9 @@ some missing values to it. >>> X_train, X_test, y_train, _ = train_test_split(X, y, test_size=100, ... random_state=0) -Now we create a :class:`FeatureUnion`. All features will be imputed using -:class:`SimpleImputer`, in order to enable classifiers to work with this data. -Additionally, it adds the indicator variables from +Now we create a :class:`~sklearn.pipeline.FeatureUnion`. All features will be +imputed using :class:`SimpleImputer`, in order to enable classifiers to work +with this data. Additionally, it adds the indicator variables from :class:`MissingIndicator`. >>> transformer = FeatureUnion( @@ -305,11 +341,20 @@ Additionally, it adds the indicator variables from (100, 8) Of course, we cannot use the transformer to make any predictions. We should -wrap this in a :class:`Pipeline` with a classifier (e.g., a -:class:`DecisionTreeClassifier`) to be able to make predictions. +wrap this in a :class:`~sklearn.pipeline.Pipeline` with a classifier (e.g., a +:class:`~sklearn.tree.DecisionTreeClassifier`) to be able to make predictions. >>> clf = make_pipeline(transformer, DecisionTreeClassifier()) >>> clf = clf.fit(X_train, y_train) >>> results = clf.predict(X_test) >>> results.shape (100,) + +Estimators that handle NaN values +================================= + +Some estimators are designed to handle NaN values without preprocessing. +Below is the list of these estimators, classified by type +(cluster, regressor, classifier, transform): + +.. allow_nan_estimators:: diff --git a/doc/modules/isotonic.rst b/doc/modules/isotonic.rst index 8967ef18afcb3..50fbdb24e72c7 100644 --- a/doc/modules/isotonic.rst +++ b/doc/modules/isotonic.rst @@ -9,10 +9,10 @@ Isotonic regression The class :class:`IsotonicRegression` fits a non-decreasing real function to 1-dimensional data. It solves the following problem: - minimize :math:`\sum_i w_i (y_i - \hat{y}_i)^2` - - subject to :math:`\hat{y}_i \le \hat{y}_j` whenever :math:`X_i \le X_j`, +.. math:: + \min \sum_i w_i (y_i - \hat{y}_i)^2 +subject to :math:`\hat{y}_i \le \hat{y}_j` whenever :math:`X_i \le X_j`, where the weights :math:`w_i` are strictly positive, and both `X` and `y` are arbitrary real quantities. @@ -31,3 +31,7 @@ thus form a function that is piecewise linear: .. figure:: ../auto_examples/miscellaneous/images/sphx_glr_plot_isotonic_regression_001.png :target: ../auto_examples/miscellaneous/plot_isotonic_regression.html :align: center + +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_miscellaneous_plot_isotonic_regression.py` diff --git a/doc/modules/kernel_approximation.rst b/doc/modules/kernel_approximation.rst index b7de01e308519..0bbd19d05de33 100644 --- a/doc/modules/kernel_approximation.rst +++ b/doc/modules/kernel_approximation.rst @@ -35,13 +35,65 @@ is advisable to compare results against exact kernel methods when possible. Nystroem Method for Kernel Approximation ---------------------------------------- -The Nystroem method, as implemented in :class:`Nystroem` is a general method -for low-rank approximations of kernels. It achieves this by essentially subsampling -the data on which the kernel is evaluated. -By default :class:`Nystroem` uses the ``rbf`` kernel, but it can use any -kernel function or a precomputed kernel matrix. -The number of samples used - which is also the dimensionality of the features computed - -is given by the parameter ``n_components``. +The Nystroem method, as implemented in :class:`Nystroem` is a general method for +reduced rank approximations of kernels. It achieves this by subsampling without +replacement rows/columns of the data on which the kernel is evaluated. While the +computational complexity of the exact method is +:math:`\mathcal{O}(n^3_{\text{samples}})`, the complexity of the approximation +is :math:`\mathcal{O}(n^2_{\text{components}} \cdot n_{\text{samples}})`, where +one can set :math:`n_{\text{components}} \ll n_{\text{samples}}` without a +significant decrease in performance [WS2001]_. + +We can construct the eigendecomposition of the kernel matrix :math:`K`, based +on the features of the data, and then split it into sampled and unsampled data +points. + +.. math:: + + K = U \Lambda U^T + = \begin{bmatrix} U_1 \\ U_2\end{bmatrix} \Lambda \begin{bmatrix} U_1 \\ U_2 \end{bmatrix}^T + = \begin{bmatrix} U_1 \Lambda U_1^T & U_1 \Lambda U_2^T \\ U_2 \Lambda U_1^T & U_2 \Lambda U_2^T \end{bmatrix} + \equiv \begin{bmatrix} K_{11} & K_{12} \\ K_{21} & K_{22} \end{bmatrix} + +where: + +* :math:`U` is orthonormal +* :math:`\Lambda` is diagonal matrix of eigenvalues +* :math:`U_1` is orthonormal matrix of samples that were chosen +* :math:`U_2` is orthonormal matrix of samples that were not chosen + +Given that :math:`U_1 \Lambda U_1^T` can be obtained by orthonormalization of +the matrix :math:`K_{11}`, and :math:`U_2 \Lambda U_1^T` can be evaluated (as +well as its transpose), the only remaining term to elucidate is +:math:`U_2 \Lambda U_2^T`. To do this we can express it in terms of the already +evaluated matrices: + +.. math:: + + \begin{align} U_2 \Lambda U_2^T &= \left(K_{21} U_1 \Lambda^{-1}\right) \Lambda \left(K_{21} U_1 \Lambda^{-1}\right)^T + \\&= K_{21} U_1 (\Lambda^{-1} \Lambda) \Lambda^{-1} U_1^T K_{21}^T + \\&= K_{21} U_1 \Lambda^{-1} U_1^T K_{21}^T + \\&= K_{21} K_{11}^{-1} K_{21}^T + \\&= \left( K_{21} K_{11}^{-\frac12} \right) \left( K_{21} K_{11}^{-\frac12} \right)^T + .\end{align} + +During ``fit``, the class :class:`Nystroem` evaluates the basis :math:`U_1`, and +computes the normalization constant, :math:`K_{11}^{-\frac12}`. Later, during +``transform``, the kernel matrix is determined between the basis (given by the +`components_` attribute) and the new data points, ``X``. This matrix is then +multiplied by the ``normalization_`` matrix for the final result. + +By default :class:`Nystroem` uses the ``rbf`` kernel, but it can use any kernel +function or a precomputed kernel matrix. The number of samples used - which is +also the dimensionality of the features computed - is given by the parameter +``n_components``. + +.. rubric:: Examples + +* See the example entitled + :ref:`sphx_glr_auto_examples_applications_plot_cyclical_feature_engineering.py`, + that shows an efficient machine learning pipeline that uses a + :class:`Nystroem` kernel. .. _rbf_kernel_approx: @@ -91,9 +143,9 @@ use of larger feature spaces more efficient. Comparing an exact RBF kernel (left) with the approximation (right) -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_miscellaneous_plot_kernel_approximation.py` +* :ref:`sphx_glr_auto_examples_miscellaneous_plot_kernel_approximation.py` .. _additive_chi_kernel_approx: @@ -108,7 +160,7 @@ The additive chi squared kernel as used here is given by k(x, y) = \sum_i \frac{2x_iy_i}{x_i+y_i} -This is not exactly the same as :func:`sklearn.metrics.additive_chi2_kernel`. +This is not exactly the same as :func:`sklearn.metrics.pairwise.additive_chi2_kernel`. The authors of [VZ2010]_ prefer the version above as it is always positive definite. Since the kernel is additive, it is possible to treat all components @@ -163,8 +215,8 @@ function given by: where: - * ``x``, ``y`` are the input vectors - * ``d`` is the kernel degree +* ``x``, ``y`` are the input vectors +* ``d`` is the kernel degree Intuitively, the feature space of the polynomial kernel of degree `d` consists of all possible degree-`d` products among input features, which enables @@ -189,9 +241,9 @@ In addition, this method can transform samples in time, where :math:`n_{\text{components}}` is the desired output dimension, determined by ``n_components``. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_kernel_approximation_plot_scalable_poly_kernels.py` +* :ref:`sphx_glr_auto_examples_kernel_approximation_plot_scalable_poly_kernels.py` .. _tensor_sketch_kernel_approx: @@ -231,26 +283,29 @@ The classes in this submodule allow to approximate the embedding or store training examples. -.. topic:: References: - - .. [RR2007] `"Random features for large-scale kernel machines" - `_ - Rahimi, A. and Recht, B. - Advances in neural information processing 2007, - .. [LS2010] `"Random Fourier approximations for skewed multiplicative histogram kernels" - `_ - Random Fourier approximations for skewed multiplicative histogram kernels - - Lecture Notes for Computer Sciencd (DAGM) - .. [VZ2010] `"Efficient additive kernels via explicit feature maps" - `_ - Vedaldi, A. and Zisserman, A. - Computer Vision and Pattern Recognition 2010 - .. [VVZ2010] `"Generalized RBF feature maps for Efficient Detection" - `_ - Vempati, S. and Vedaldi, A. and Zisserman, A. and Jawahar, CV - 2010 - .. [PP2013] `"Fast and scalable polynomial kernels via explicit feature maps" - `_ - Pham, N., & Pagh, R. - 2013 - .. [CCF2002] `"Finding frequent items in data streams" - `_ - Charikar, M., Chen, K., & Farach-Colton - 2002 - .. [WIKICS] `"Wikipedia: Count sketch" - `_ +.. rubric:: References + +.. [WS2001] `"Using the NystrÃļm method to speed up kernel machines" + `_ + Williams, C.K.I.; Seeger, M. - 2001. +.. [RR2007] `"Random features for large-scale kernel machines" + `_ + Rahimi, A. and Recht, B. - Advances in neural information processing 2007, +.. [LS2010] `"Random Fourier approximations for skewed multiplicative histogram kernels" + `_ + Li, F., Ionescu, C., and Sminchisescu, C. + - Pattern Recognition, DAGM 2010, Lecture Notes in Computer Science. +.. [VZ2010] `"Efficient additive kernels via explicit feature maps" + `_ + Vedaldi, A. and Zisserman, A. - Computer Vision and Pattern Recognition 2010 +.. [VVZ2010] `"Generalized RBF feature maps for Efficient Detection" + `_ + Vempati, S. and Vedaldi, A. and Zisserman, A. and Jawahar, CV - 2010 +.. [PP2013] :doi:`"Fast and scalable polynomial kernels via explicit feature maps" + <10.1145/2487575.2487591>` + Pham, N., & Pagh, R. - 2013 +.. [CCF2002] `"Finding frequent items in data streams" + `_ + Charikar, M., Chen, K., & Farach-Colton - 2002 +.. [WIKICS] `"Wikipedia: Count sketch" + `_ diff --git a/doc/modules/kernel_ridge.rst b/doc/modules/kernel_ridge.rst index 286e9d4ac5322..64267f4233a53 100644 --- a/doc/modules/kernel_ridge.rst +++ b/doc/modules/kernel_ridge.rst @@ -7,7 +7,7 @@ Kernel ridge regression .. currentmodule:: sklearn.kernel_ridge Kernel ridge regression (KRR) [M2012]_ combines :ref:`ridge_regression` -(linear least squares with l2-norm regularization) with the `kernel trick +(linear least squares with :math:`L_2`-norm regularization) with the `kernel trick `_. It thus learns a linear function in the space induced by the respective kernel and the data. For non-linear kernels, this corresponds to a non-linear function in the original @@ -16,7 +16,7 @@ space. The form of the model learned by :class:`KernelRidge` is identical to support vector regression (:class:`~sklearn.svm.SVR`). However, different loss functions are used: KRR uses squared error loss while support vector -regression uses :math:`\epsilon`-insensitive loss, both combined with l2 +regression uses :math:`\epsilon`-insensitive loss, both combined with :math:`L_2` regularization. In contrast to :class:`~sklearn.svm.SVR`, fitting :class:`KernelRidge` can be done in closed-form and is typically faster for medium-sized datasets. On the other hand, the learned model is non-sparse and @@ -31,7 +31,7 @@ plotted, where both complexity/regularization and bandwidth of the RBF kernel have been optimized using grid-search. The learned functions are very similar; however, fitting :class:`KernelRidge` is approximately seven times faster than fitting :class:`~sklearn.svm.SVR` (both with grid-search). -However, prediction of 100000 target values is more than three times faster +However, prediction of 100,000 target values is more than three times faster with :class:`~sklearn.svm.SVR` since it has learned a sparse model using only approximately 1/3 of the 100 training datapoints as support vectors. @@ -55,8 +55,11 @@ dense model. :target: ../auto_examples/miscellaneous/plot_kernel_ridge_regression.html :align: center +.. rubric:: Examples -.. topic:: References: +* :ref:`sphx_glr_auto_examples_miscellaneous_plot_kernel_ridge_regression.py` - .. [M2012] "Machine Learning: A Probabilistic Perspective" - Murphy, K. P. - chapter 14.4.3, pp. 492-493, The MIT Press, 2012 +.. rubric:: References + +.. [M2012] "Machine Learning: A Probabilistic Perspective" + Murphy, K. P. - chapter 14.4.3, pp. 492-493, The MIT Press, 2012 diff --git a/doc/modules/lda_qda.rst b/doc/modules/lda_qda.rst index 962d65705f75a..c18835d514a9f 100644 --- a/doc/modules/lda_qda.rst +++ b/doc/modules/lda_qda.rst @@ -29,10 +29,10 @@ Discriminant Analysis can only learn linear boundaries, while Quadratic Discriminant Analysis can learn quadratic boundaries and is therefore more flexible. -.. topic:: Examples: +.. rubric:: Examples - :ref:`sphx_glr_auto_examples_classification_plot_lda_qda.py`: Comparison of LDA and QDA - on synthetic data. +* :ref:`sphx_glr_auto_examples_classification_plot_lda_qda.py`: Comparison of LDA and + QDA on synthetic data. Dimensionality reduction using Linear Discriminant Analysis =========================================================== @@ -49,10 +49,10 @@ This is implemented in the `transform` method. The desired dimensionality can be set using the ``n_components`` parameter. This parameter has no influence on the `fit` and `predict` methods. -.. topic:: Examples: +.. rubric:: Examples - :ref:`sphx_glr_auto_examples_decomposition_plot_pca_vs_lda.py`: Comparison of LDA and PCA - for dimensionality reduction of the Iris dataset +* :ref:`sphx_glr_auto_examples_decomposition_plot_pca_vs_lda.py`: Comparison of LDA and + PCA for dimensionality reduction of the Iris dataset .. _lda_qda_math: @@ -93,10 +93,10 @@ predicted class is the one that maximises this log-posterior. .. note:: **Relation with Gaussian Naive Bayes** - If in the QDA model one assumes that the covariance matrices are diagonal, - then the inputs are assumed to be conditionally independent in each class, - and the resulting classifier is equivalent to the Gaussian Naive Bayes - classifier :class:`naive_bayes.GaussianNB`. + If in the QDA model one assumes that the covariance matrices are diagonal, + then the inputs are assumed to be conditionally independent in each class, + and the resulting classifier is equivalent to the Gaussian Naive Bayes + classifier :class:`naive_bayes.GaussianNB`. LDA --- @@ -137,7 +137,7 @@ Mathematical formulation of LDA dimensionality reduction First note that the K means :math:`\mu_k` are vectors in :math:`\mathcal{R}^d`, and they lie in an affine subspace :math:`H` of dimension at most :math:`K - 1` (2 points lie on a line, 3 points lie on a -plane, etc). +plane, etc.). As mentioned above, we can interpret LDA as assigning :math:`x` to the class whose mean :math:`\mu_k` is the closest in terms of Mahalanobis distance, @@ -173,11 +173,11 @@ In this scenario, the empirical sample covariance is a poor estimator, and shrinkage helps improving the generalization performance of the classifier. Shrinkage LDA can be used by setting the ``shrinkage`` parameter of -the :class:`~discriminant_analysis.LinearDiscriminantAnalysis` class to 'auto'. +the :class:`~discriminant_analysis.LinearDiscriminantAnalysis` class to `'auto'`. This automatically determines the optimal shrinkage parameter in an analytic way following the lemma introduced by Ledoit and Wolf [2]_. Note that -currently shrinkage only works when setting the ``solver`` parameter to 'lsqr' -or 'eigen'. +currently shrinkage only works when setting the ``solver`` parameter to `'lsqr'` +or `'eigen'`. The ``shrinkage`` parameter can also be manually set between 0 and 1. In particular, a value of 0 corresponds to no shrinkage (which means the empirical @@ -187,17 +187,17 @@ an estimate for the covariance matrix). Setting this parameter to a value between these two extrema will estimate a shrunk version of the covariance matrix. -The shrinked Ledoit and Wolf estimator of covariance may not always be the +The shrunk Ledoit and Wolf estimator of covariance may not always be the best choice. For example if the distribution of the data is normally distributed, the -Oracle Shrinkage Approximating estimator :class:`sklearn.covariance.OAS` +Oracle Approximating Shrinkage estimator :class:`sklearn.covariance.OAS` yields a smaller Mean Squared Error than the one given by Ledoit and Wolf's -formula used with shrinkage="auto". In LDA, the data are assumed to be gaussian +formula used with `shrinkage="auto"`. In LDA, the data are assumed to be gaussian conditionally to the class. If these assumptions hold, using LDA with -the OAS estimator of covariance will yield a better classification +the OAS estimator of covariance will yield a better classification accuracy than if Ledoit and Wolf or the empirical covariance estimator is used. -The covariance estimator can be chosen using with the ``covariance_estimator`` +The covariance estimator can be chosen using the ``covariance_estimator`` parameter of the :class:`discriminant_analysis.LinearDiscriminantAnalysis` class. A covariance estimator should have a :term:`fit` method and a ``covariance_`` attribute like all covariance estimators in the @@ -210,10 +210,10 @@ class. A covariance estimator should have a :term:`fit` method and a .. centered:: |shrinkage| -.. topic:: Examples: +.. rubric:: Examples - :ref:`sphx_glr_auto_examples_classification_plot_lda.py`: Comparison of LDA classifiers - with Empirical, Ledoit Wolf and OAS covariance estimator. +* :ref:`sphx_glr_auto_examples_classification_plot_lda.py`: Comparison of LDA classifiers + with Empirical, Ledoit Wolf and OAS covariance estimator. Estimation algorithms ===================== @@ -232,14 +232,14 @@ solver may be preferable in situations where the number of features is large. The 'svd' solver cannot be used with shrinkage. For QDA, the use of the SVD solver relies on the fact that the covariance matrix :math:`\Sigma_k` is, by definition, equal to :math:`\frac{1}{n - 1} -X_k^tX_k = V S^2 V^t` where :math:`V` comes from the SVD of the (centered) +X_k^tX_k = \frac{1}{n - 1} V S^2 V^t` where :math:`V` comes from the SVD of the (centered) matrix: :math:`X_k = U S V^t`. It turns out that we can compute the -log-posterior above without having to explictly compute :math:`\Sigma`: +log-posterior above without having to explicitly compute :math:`\Sigma`: computing :math:`S` and :math:`V` via the SVD of :math:`X` is enough. For LDA, two SVDs are computed: the SVD of the centered input matrix :math:`X` and the SVD of the class-wise mean vectors. -The 'lsqr' solver is an efficient algorithm that only works for +The `'lsqr'` solver is an efficient algorithm that only works for classification. It needs to explicitly compute the covariance matrix :math:`\Sigma`, and supports shrinkage and custom covariance estimators. This solver computes the coefficients @@ -247,19 +247,19 @@ This solver computes the coefficients \mu_k`, thus avoiding the explicit computation of the inverse :math:`\Sigma^{-1}`. -The 'eigen' solver is based on the optimization of the between class scatter to +The `'eigen'` solver is based on the optimization of the between class scatter to within class scatter ratio. It can be used for both classification and -transform, and it supports shrinkage. However, the 'eigen' solver needs to +transform, and it supports shrinkage. However, the `'eigen'` solver needs to compute the covariance matrix, so it might not be suitable for situations with a high number of features. -.. topic:: References: +.. rubric:: References - .. [1] "The Elements of Statistical Learning", Hastie T., Tibshirani R., - Friedman J., Section 4.3, p.106-119, 2008. +.. [1] "The Elements of Statistical Learning", Hastie T., Tibshirani R., + Friedman J., Section 4.3, p.106-119, 2008. - .. [2] Ledoit O, Wolf M. Honey, I Shrunk the Sample Covariance Matrix. - The Journal of Portfolio Management 30(4), 110-119, 2004. +.. [2] Ledoit O, Wolf M. Honey, I Shrunk the Sample Covariance Matrix. + The Journal of Portfolio Management 30(4), 110-119, 2004. - .. [3] R. O. Duda, P. E. Hart, D. G. Stork. Pattern Classification - (Second Edition), section 2.6.2. +.. [3] R. O. Duda, P. E. Hart, D. G. Stork. Pattern Classification + (Second Edition), section 2.6.2. diff --git a/doc/modules/learning_curve.rst b/doc/modules/learning_curve.rst index 4fb90df937e15..6dca0a29af7cb 100644 --- a/doc/modules/learning_curve.rst +++ b/doc/modules/learning_curve.rst @@ -39,11 +39,11 @@ easy to see whether the estimator suffers from bias or variance. However, in high-dimensional spaces, models can become very difficult to visualize. For this reason, it is often helpful to use the tools described below. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_model_selection_plot_underfitting_overfitting.py` - * :ref:`sphx_glr_auto_examples_model_selection_plot_validation_curve.py` - * :ref:`sphx_glr_auto_examples_model_selection_plot_learning_curve.py` +* :ref:`sphx_glr_auto_examples_model_selection_plot_underfitting_overfitting.py` +* :ref:`sphx_glr_auto_examples_model_selection_plot_train_error_vs_test_error.py` +* :ref:`sphx_glr_auto_examples_model_selection_plot_learning_curve.py` .. _validation_curve: @@ -53,9 +53,9 @@ Validation curve To validate a model we need a scoring function (see :ref:`model_evaluation`), for example accuracy for classifiers. The proper way of choosing multiple -hyperparameters of an estimator are of course grid search or similar methods +hyperparameters of an estimator is of course grid search or similar methods (see :ref:`grid_search`) that select the hyperparameter with the maximum score -on a validation set or multiple validation sets. Note that if we optimized +on a validation set or multiple validation sets. Note that if we optimize the hyperparameters based on a validation score the validation score is biased and not a good estimate of the generalization any longer. To get a proper estimate of the generalization we have to compute the score on another test @@ -71,7 +71,7 @@ The function :func:`validation_curve` can help in this case:: >>> import numpy as np >>> from sklearn.model_selection import validation_curve >>> from sklearn.datasets import load_iris - >>> from sklearn.linear_model import Ridge + >>> from sklearn.svm import SVC >>> np.random.seed(0) >>> X, y = load_iris(return_X_y=True) @@ -79,30 +79,43 @@ The function :func:`validation_curve` can help in this case:: >>> np.random.shuffle(indices) >>> X, y = X[indices], y[indices] - >>> train_scores, valid_scores = validation_curve(Ridge(), X, y, "alpha", - ... np.logspace(-7, 3, 3), - ... cv=5) + >>> train_scores, valid_scores = validation_curve( + ... SVC(kernel="linear"), X, y, param_name="C", param_range=np.logspace(-7, 3, 3), + ... ) >>> train_scores - array([[0.93..., 0.94..., 0.92..., 0.91..., 0.92...], - [0.93..., 0.94..., 0.92..., 0.91..., 0.92...], - [0.51..., 0.52..., 0.49..., 0.47..., 0.49...]]) + array([[0.90, 0.94, 0.91, 0.89, 0.92], + [0.9 , 0.92, 0.93, 0.92, 0.93], + [0.97, 1 , 0.98, 0.97, 0.99]]) >>> valid_scores - array([[0.90..., 0.84..., 0.94..., 0.96..., 0.93...], - [0.90..., 0.84..., 0.94..., 0.96..., 0.93...], - [0.46..., 0.25..., 0.50..., 0.49..., 0.52...]]) + array([[0.9, 0.9 , 0.9 , 0.96, 0.9 ], + [0.9, 0.83, 0.96, 0.96, 0.93], + [1. , 0.93, 1 , 1 , 0.9 ]]) + +If you intend to plot the validation curves only, the class +:class:`~sklearn.model_selection.ValidationCurveDisplay` is more direct than +using matplotlib manually on the results of a call to :func:`validation_curve`. +You can use the method +:meth:`~sklearn.model_selection.ValidationCurveDisplay.from_estimator` similarly +to :func:`validation_curve` to generate and plot the validation curve: + +.. plot:: + :context: close-figs + :align: center + + from sklearn.datasets import load_iris + from sklearn.model_selection import ValidationCurveDisplay + from sklearn.svm import SVC + from sklearn.utils import shuffle + X, y = load_iris(return_X_y=True) + X, y = shuffle(X, y, random_state=0) + ValidationCurveDisplay.from_estimator( + SVC(kernel="linear"), X, y, param_name="C", param_range=np.logspace(-7, 3, 10) + ) If the training score and the validation score are both low, the estimator will be underfitting. If the training score is high and the validation score is low, the estimator is overfitting and otherwise it is working very well. A low -training score and a high validation score is usually not possible. Underfitting, -overfitting, and a working model are shown in the in the plot below where we vary -the parameter :math:`\gamma` of an SVM on the digits dataset. - -.. figure:: ../auto_examples/model_selection/images/sphx_glr_plot_validation_curve_001.png - :target: ../auto_examples/model_selection/plot_validation_curve.html - :align: center - :scale: 50% - +training score and a high validation score is usually not possible. .. _learning_curve: @@ -141,11 +154,34 @@ average scores on the validation sets):: >>> train_sizes array([ 50, 80, 110]) >>> train_scores - array([[0.98..., 0.98 , 0.98..., 0.98..., 0.98...], - [0.98..., 1. , 0.98..., 0.98..., 0.98...], - [0.98..., 1. , 0.98..., 0.98..., 0.99...]]) + array([[0.98, 0.98 , 0.98, 0.98, 0.98], + [0.98, 1. , 0.98, 0.98, 0.98], + [0.98, 1. , 0.98, 0.98, 0.99]]) >>> valid_scores - array([[1. , 0.93..., 1. , 1. , 0.96...], - [1. , 0.96..., 1. , 1. , 0.96...], - [1. , 0.96..., 1. , 1. , 0.96...]]) + array([[1. , 0.93, 1. , 1. , 0.96], + [1. , 0.96, 1. , 1. , 0.96], + [1. , 0.96, 1. , 1. , 0.96]]) + +If you intend to plot the learning curves only, the class +:class:`~sklearn.model_selection.LearningCurveDisplay` will be easier to use. +You can use the method +:meth:`~sklearn.model_selection.LearningCurveDisplay.from_estimator` similarly +to :func:`learning_curve` to generate and plot the learning curve: + +.. plot:: + :context: close-figs + :align: center + + from sklearn.datasets import load_iris + from sklearn.model_selection import LearningCurveDisplay + from sklearn.svm import SVC + from sklearn.utils import shuffle + X, y = load_iris(return_X_y=True) + X, y = shuffle(X, y, random_state=0) + LearningCurveDisplay.from_estimator( + SVC(kernel="linear"), X, y, train_sizes=[50, 80, 110], cv=5) + +.. rubric:: Examples +* See :ref:`sphx_glr_auto_examples_model_selection_plot_learning_curve.py` for an + example of using learning curves to check the scalability of a predictive model. diff --git a/doc/modules/linear_model.rst b/doc/modules/linear_model.rst index f1f376dc641c9..9edd90321bd02 100644 --- a/doc/modules/linear_model.rst +++ b/doc/modules/linear_model.rst @@ -32,14 +32,14 @@ solves a problem of the form: .. math:: \min_{w} || X w - y||_2^2 -.. figure:: ../auto_examples/linear_model/images/sphx_glr_plot_ols_001.png - :target: ../auto_examples/linear_model/plot_ols.html +.. figure:: ../auto_examples/linear_model/images/sphx_glr_plot_ols_ridge_001.png + :target: ../auto_examples/linear_model/plot_ols_ridge.html :align: center :scale: 50% -:class:`LinearRegression` will take in its ``fit`` method arrays X, y -and will store the coefficients :math:`w` of the linear model in its -``coef_`` member:: +:class:`LinearRegression` takes in its ``fit`` method arguments ``X``, ``y``, +``sample_weight`` and stores the coefficients :math:`w` of the linear model in its +``coef_`` and ``intercept_`` attributes:: >>> from sklearn import linear_model >>> reg = linear_model.LinearRegression() @@ -47,19 +47,21 @@ and will store the coefficients :math:`w` of the linear model in its LinearRegression() >>> reg.coef_ array([0.5, 0.5]) + >>> reg.intercept_ + 0.0 The coefficient estimates for Ordinary Least Squares rely on the -independence of the features. When features are correlated and the -columns of the design matrix :math:`X` have an approximate linear +independence of the features. When features are correlated and some +columns of the design matrix :math:`X` have an approximately linear dependence, the design matrix becomes close to singular and as a result, the least-squares estimate becomes highly sensitive to random errors in the observed target, producing a large variance. This situation of *multicollinearity* can arise, for example, when data are collected without an experimental design. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_linear_model_plot_ols.py` +* :ref:`sphx_glr_auto_examples_linear_model_plot_ols_ridge.py` Non-Negative Least Squares -------------------------- @@ -68,18 +70,18 @@ It is possible to constrain all the coefficients to be non-negative, which may be useful when they represent some physical or naturally non-negative quantities (e.g., frequency counts or prices of goods). :class:`LinearRegression` accepts a boolean ``positive`` -parameter: when set to `True` `Non Negative Least Squares +parameter: when set to `True` `Non-Negative Least Squares `_ are then applied. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_linear_model_plot_nnls.py` +* :ref:`sphx_glr_auto_examples_linear_model_plot_nnls.py` Ordinary Least Squares Complexity --------------------------------- The least squares solution is computed using the singular value -decomposition of X. If X is a matrix of shape `(n_samples, n_features)` +decomposition of :math:`X`. If :math:`X` is a matrix of shape `(n_samples, n_features)` this method has a cost of :math:`O(n_{\text{samples}} n_{\text{features}}^2)`, assuming that :math:`n_{\text{samples}} \geq n_{\text{features}}`. @@ -114,7 +116,7 @@ of shrinkage and thus the coefficients become more robust to collinearity. As with other linear models, :class:`Ridge` will take in its ``fit`` method -arrays X, y and will store the coefficients :math:`w` of the linear model in +arrays ``X``, ``y`` and will store the coefficients :math:`w` of the linear model in its ``coef_`` member:: >>> from sklearn import linear_model @@ -124,8 +126,30 @@ its ``coef_`` member:: >>> reg.coef_ array([0.34545455, 0.34545455]) >>> reg.intercept_ - 0.13636... - + np.float64(0.13636) + +Note that the class :class:`Ridge` allows for the user to specify that the +solver be automatically chosen by setting `solver="auto"`. When this option +is specified, :class:`Ridge` will choose between the `"lbfgs"`, `"cholesky"`, +and `"sparse_cg"` solvers. :class:`Ridge` will begin checking the conditions +shown in the following table from top to bottom. If the condition is true, +the corresponding solver is chosen. + ++-------------+----------------------------------------------------+ +| **Solver** | **Condition** | ++-------------+----------------------------------------------------+ +| 'lbfgs' | The ``positive=True`` option is specified. | ++-------------+----------------------------------------------------+ +| 'cholesky' | The input array X is not sparse. | ++-------------+----------------------------------------------------+ +| 'sparse_cg' | None of the above conditions are fulfilled. | ++-------------+----------------------------------------------------+ + +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_linear_model_plot_ols_ridge.py` +* :ref:`sphx_glr_auto_examples_linear_model_plot_ridge_path.py` +* :ref:`sphx_glr_auto_examples_inspection_plot_linear_model_coefficient_interpretation.py` Classification -------------- @@ -140,26 +164,25 @@ the output with the highest value. It might seem questionable to use a (penalized) Least Squares loss to fit a classification model instead of the more traditional logistic or hinge -losses. However in practice all those models can lead to similar +losses. However, in practice, all those models can lead to similar cross-validation scores in terms of accuracy or precision/recall, while the penalized least squares loss used by the :class:`RidgeClassifier` allows for a very different choice of the numerical solvers with distinct computational performance profiles. The :class:`RidgeClassifier` can be significantly faster than e.g. -:class:`LogisticRegression` with a high number of classes, because it is -able to compute the projection matrix :math:`(X^T X)^{-1} X^T` only once. +:class:`LogisticRegression` with a high number of classes because it can +compute the projection matrix :math:`(X^T X)^{-1} X^T` only once. This classifier is sometimes referred to as a `Least Squares Support Vector -Machines +Machine `_ with a linear kernel. -.. topic:: Examples: +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py` - * :ref:`sphx_glr_auto_examples_linear_model_plot_ridge_path.py` - * :ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py` - * :ref:`sphx_glr_auto_examples_inspection_plot_linear_model_coefficient_interpretation.py` Ridge Complexity ---------------- @@ -176,9 +199,14 @@ This method has the same order of complexity as Setting the regularization parameter: leave-one-out Cross-Validation -------------------------------------------------------------------- -:class:`RidgeCV` implements ridge regression with built-in -cross-validation of the alpha parameter. The object works in the same way -as GridSearchCV except that it defaults to Leave-One-Out Cross-Validation:: +:class:`RidgeCV` and :class:`RidgeClassifierCV` implement ridge +regression/classification with built-in cross-validation of the alpha parameter. +They work in the same way as :class:`~sklearn.model_selection.GridSearchCV` except +that it defaults to efficient Leave-One-Out :term:`cross-validation`. +When using the default :term:`cross-validation`, alpha cannot be 0 due to the +formulation used to calculate Leave-One-Out error. See [RL2007]_ for details. + +Usage example:: >>> import numpy as np >>> from sklearn import linear_model @@ -187,20 +215,18 @@ as GridSearchCV except that it defaults to Leave-One-Out Cross-Validation:: RidgeCV(alphas=array([1.e-06, 1.e-05, 1.e-04, 1.e-03, 1.e-02, 1.e-01, 1.e+00, 1.e+01, 1.e+02, 1.e+03, 1.e+04, 1.e+05, 1.e+06])) >>> reg.alpha_ - 0.01 + np.float64(0.01) Specifying the value of the :term:`cv` attribute will trigger the use of cross-validation with :class:`~sklearn.model_selection.GridSearchCV`, for example `cv=10` for 10-fold cross-validation, rather than Leave-One-Out Cross-Validation. -.. topic:: References - - * "Notes on Regularized Least Squares", Rifkin & Lippert (`technical report - `_, - `course slides - `_). +.. dropdown:: References + .. [RL2007] "Notes on Regularized Least Squares", Rifkin & Lippert (`technical report + `_, + `course slides `_). .. _lasso: @@ -210,7 +236,7 @@ Lasso The :class:`Lasso` is a linear model that estimates sparse coefficients. It is useful in some contexts due to its tendency to prefer solutions with fewer non-zero coefficients, effectively reducing the number of -features upon which the given solution is dependent. For this reason +features upon which the given solution is dependent. For this reason, Lasso and its variants are fundamental to the field of compressed sensing. Under certain conditions, it can recover the exact set of non-zero coefficients (see @@ -240,11 +266,11 @@ for another implementation:: The function :func:`lasso_path` is useful for lower-level tasks, as it computes the coefficients along the full path of possible values. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_linear_model_plot_lasso_and_elasticnet.py` - * :ref:`sphx_glr_auto_examples_applications_plot_tomography_l1_reconstruction.py` - * :ref:`sphx_glr_auto_examples_inspection_plot_linear_model_coefficient_interpretation.py` +* :ref:`sphx_glr_auto_examples_linear_model_plot_lasso_and_elasticnet.py` +* :ref:`sphx_glr_auto_examples_applications_plot_tomography_l1_reconstruction.py` +* :ref:`sphx_glr_auto_examples_inspection_plot_linear_model_coefficient_interpretation.py` .. note:: **Feature selection with Lasso** @@ -253,20 +279,19 @@ computes the coefficients along the full path of possible values. thus be used to perform feature selection, as detailed in :ref:`l1_feature_selection`. -The following two references explain the iterations -used in the coordinate descent solver of scikit-learn, as well as -the duality gap computation used for convergence control. +.. dropdown:: References -.. topic:: References - - * "Regularization Path For Generalized linear Models by Coordinate Descent", - Friedman, Hastie & Tibshirani, J Stat Softw, 2010 (`Paper - `__). - * "An Interior-Point Method for Large-Scale L1-Regularized Least Squares," - S. J. Kim, K. Koh, M. Lustig, S. Boyd and D. Gorinevsky, - in IEEE Journal of Selected Topics in Signal Processing, 2007 - (`Paper `__) + The following two references explain the iterations + used in the coordinate descent solver of scikit-learn, as well as + the duality gap computation used for convergence control. + * "Regularization Path For Generalized linear Models by Coordinate Descent", + Friedman, Hastie & Tibshirani, J Stat Softw, 2010 (`Paper + `__). + * "An Interior-Point Method for Large-Scale L1-Regularized Least Squares," + S. J. Kim, K. Koh, M. Lustig, S. Boyd and D. Gorinevsky, + in IEEE Journal of Selected Topics in Signal Processing, 2007 + (`Paper `__) Setting regularization parameter -------------------------------- @@ -298,6 +323,7 @@ features, it is often faster than :class:`LassoCV`. .. centered:: |lasso_cv_1| |lasso_cv_2| +.. _lasso_lars_ic: Information-criteria based model selection ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -306,22 +332,97 @@ Alternatively, the estimator :class:`LassoLarsIC` proposes to use the Akaike information criterion (AIC) and the Bayes Information criterion (BIC). It is a computationally cheaper alternative to find the optimal value of alpha as the regularization path is computed only once instead of k+1 times -when using k-fold cross-validation. However, such criteria needs a -proper estimation of the degrees of freedom of the solution, are -derived for large samples (asymptotic results) and assume the model -is correct, i.e. that the data are actually generated by this model. -They also tend to break when the problem is badly conditioned -(more features than samples). - -.. figure:: ../auto_examples/linear_model/images/sphx_glr_plot_lasso_model_selection_001.png - :target: ../auto_examples/linear_model/plot_lasso_model_selection.html +when using k-fold cross-validation. + +Indeed, these criteria are computed on the in-sample training set. In short, +they penalize the over-optimistic scores of the different Lasso models by +their flexibility (cf. to "Mathematical details" section below). + +However, such criteria need a proper estimation of the degrees of freedom of +the solution, are derived for large samples (asymptotic results) and assume the +correct model is candidates under investigation. They also tend to break when +the problem is badly conditioned (e.g. more features than samples). + +.. figure:: ../auto_examples/linear_model/images/sphx_glr_plot_lasso_lars_ic_001.png + :target: ../auto_examples/linear_model/plot_lasso_lars_ic.html :align: center :scale: 50% +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_linear_model_plot_lasso_model_selection.py` +* :ref:`sphx_glr_auto_examples_linear_model_plot_lasso_lars_ic.py` + +.. _aic_bic: + +AIC and BIC criteria +^^^^^^^^^^^^^^^^^^^^ + +The definition of AIC (and thus BIC) might differ in the literature. In this +section, we give more information regarding the criterion computed in +scikit-learn. + +.. dropdown:: Mathematical details + + The AIC criterion is defined as: + + .. math:: + AIC = -2 \log(\hat{L}) + 2 d + + where :math:`\hat{L}` is the maximum likelihood of the model and + :math:`d` is the number of parameters (as well referred to as degrees of + freedom in the previous section). + + The definition of BIC replaces the constant :math:`2` by :math:`\log(N)`: + + .. math:: + BIC = -2 \log(\hat{L}) + \log(N) d + + where :math:`N` is the number of samples. + + For a linear Gaussian model, the maximum log-likelihood is defined as: + + .. math:: + \log(\hat{L}) = - \frac{n}{2} \log(2 \pi) - \frac{n}{2} \ln(\sigma^2) - \frac{\sum_{i=1}^{n} (y_i - \hat{y}_i)^2}{2\sigma^2} + + where :math:`\sigma^2` is an estimate of the noise variance, + :math:`y_i` and :math:`\hat{y}_i` are respectively the true and predicted + targets, and :math:`n` is the number of samples. + + Plugging the maximum log-likelihood in the AIC formula yields: + + .. math:: + AIC = n \log(2 \pi \sigma^2) + \frac{\sum_{i=1}^{n} (y_i - \hat{y}_i)^2}{\sigma^2} + 2 d + + The first term of the above expression is sometimes discarded since it is a + constant when :math:`\sigma^2` is provided. In addition, + it is sometimes stated that the AIC is equivalent to the :math:`C_p` statistic + [12]_. In a strict sense, however, it is equivalent only up to some constant + and a multiplicative factor. + + At last, we mentioned above that :math:`\sigma^2` is an estimate of the + noise variance. In :class:`LassoLarsIC` when the parameter `noise_variance` is + not provided (default), the noise variance is estimated via the unbiased + estimator [13]_ defined as: + + .. math:: + \sigma^2 = \frac{\sum_{i=1}^{n} (y_i - \hat{y}_i)^2}{n - p} + + where :math:`p` is the number of features and :math:`\hat{y}_i` is the + predicted target using an ordinary least squares regression. Note, that this + formula is valid only when `n_samples > n_features`. -.. topic:: Examples: + .. rubric:: References - * :ref:`sphx_glr_auto_examples_linear_model_plot_lasso_model_selection.py` + .. [12] :arxiv:`Zou, Hui, Trevor Hastie, and Robert Tibshirani. + "On the degrees of freedom of the lasso." + The Annals of Statistics 35.5 (2007): 2173-2192. + <0712.0881.pdf>` + + .. [13] :doi:`Cherkassky, Vladimir, and Yunqian Ma. + "Comparison of model selection for regression." + Neural computation 15.7 (2003): 1691-1714. + <10.1162/089976603321891864>` Comparison with the regularization parameter of SVM ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -358,28 +459,29 @@ the MultiTaskLasso are full columns. .. centered:: Fitting a time-series model, imposing that any active feature be active at all times. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_linear_model_plot_multi_task_lasso_support.py` +* :ref:`sphx_glr_auto_examples_linear_model_plot_multi_task_lasso_support.py` -Mathematically, it consists of a linear model trained with a mixed -:math:`\ell_1` :math:`\ell_2`-norm for regularization. -The objective function to minimize is: +.. dropdown:: Mathematical details -.. math:: \min_{W} { \frac{1}{2n_{\text{samples}}} ||X W - Y||_{\text{Fro}} ^ 2 + \alpha ||W||_{21}} + Mathematically, it consists of a linear model trained with a mixed + :math:`\ell_1` :math:`\ell_2`-norm for regularization. + The objective function to minimize is: -where :math:`\text{Fro}` indicates the Frobenius norm + .. math:: \min_{W} { \frac{1}{2n_{\text{samples}}} ||X W - Y||_{\text{Fro}} ^ 2 + \alpha ||W||_{21}} -.. math:: ||A||_{\text{Fro}} = \sqrt{\sum_{ij} a_{ij}^2} + where :math:`\text{Fro}` indicates the Frobenius norm -and :math:`\ell_1` :math:`\ell_2` reads + .. math:: ||A||_{\text{Fro}} = \sqrt{\sum_{ij} a_{ij}^2} -.. math:: ||A||_{2 1} = \sum_i \sqrt{\sum_j a_{ij}^2}. + and :math:`\ell_1` :math:`\ell_2` reads -The implementation in the class :class:`MultiTaskLasso` uses -coordinate descent as the algorithm to fit the coefficients. + .. math:: ||A||_{2 1} = \sum_i \sqrt{\sum_j a_{ij}^2}. + The implementation in the class :class:`MultiTaskLasso` uses + coordinate descent as the algorithm to fit the coefficients. .. _elastic_net: @@ -393,7 +495,7 @@ the regularization properties of :class:`Ridge`. We control the convex combination of :math:`\ell_1` and :math:`\ell_2` using the ``l1_ratio`` parameter. -Elastic-net is useful when there are multiple features which are +Elastic-net is useful when there are multiple features that are correlated with one another. Lasso is likely to pick one of these at random, while elastic-net is likely to pick both. @@ -408,32 +510,33 @@ The objective function to minimize is in this case \frac{\alpha(1-\rho)}{2} ||w||_2 ^ 2} -.. figure:: ../auto_examples/linear_model/images/sphx_glr_plot_lasso_coordinate_descent_path_001.png - :target: ../auto_examples/linear_model/plot_lasso_coordinate_descent_path.html +.. figure:: ../auto_examples/linear_model/images/sphx_glr_plot_lasso_lasso_lars_elasticnet_path_002.png + :target: ../auto_examples/linear_model/plot_lasso_lasso_lars_elasticnet_path.html :align: center :scale: 50% The class :class:`ElasticNetCV` can be used to set the parameters ``alpha`` (:math:`\alpha`) and ``l1_ratio`` (:math:`\rho`) by cross-validation. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_linear_model_plot_lasso_and_elasticnet.py` - * :ref:`sphx_glr_auto_examples_linear_model_plot_lasso_coordinate_descent_path.py` +* :ref:`sphx_glr_auto_examples_linear_model_plot_lasso_and_elasticnet.py` +* :ref:`sphx_glr_auto_examples_linear_model_plot_lasso_lasso_lars_elasticnet_path.py` +* :ref:`sphx_glr_auto_examples_linear_model_plot_elastic_net_precomputed_gram_matrix_with_weighted_samples.py` -The following two references explain the iterations -used in the coordinate descent solver of scikit-learn, as well as -the duality gap computation used for convergence control. +.. dropdown:: References -.. topic:: References + The following two references explain the iterations + used in the coordinate descent solver of scikit-learn, as well as + the duality gap computation used for convergence control. - * "Regularization Path For Generalized linear Models by Coordinate Descent", - Friedman, Hastie & Tibshirani, J Stat Softw, 2010 (`Paper - `__). - * "An Interior-Point Method for Large-Scale L1-Regularized Least Squares," - S. J. Kim, K. Koh, M. Lustig, S. Boyd and D. Gorinevsky, - in IEEE Journal of Selected Topics in Signal Processing, 2007 - (`Paper `__) + * "Regularization Path For Generalized linear Models by Coordinate Descent", + Friedman, Hastie & Tibshirani, J Stat Softw, 2010 (`Paper + `__). + * "An Interior-Point Method for Large-Scale L1-Regularized Least Squares," + S. J. Kim, K. Koh, M. Lustig, S. Boyd and D. Gorinevsky, + in IEEE Journal of Selected Topics in Signal Processing, 2007 + (`Paper `__) .. _multi_task_elastic_net: @@ -475,32 +578,32 @@ between the features. The advantages of LARS are: - - It is numerically efficient in contexts where the number of features - is significantly greater than the number of samples. +- It is numerically efficient in contexts where the number of features + is significantly greater than the number of samples. - - It is computationally just as fast as forward selection and has - the same order of complexity as ordinary least squares. +- It is computationally just as fast as forward selection and has + the same order of complexity as ordinary least squares. - - It produces a full piecewise linear solution path, which is - useful in cross-validation or similar attempts to tune the model. +- It produces a full piecewise linear solution path, which is + useful in cross-validation or similar attempts to tune the model. - - If two features are almost equally correlated with the target, - then their coefficients should increase at approximately the same - rate. The algorithm thus behaves as intuition would expect, and - also is more stable. +- If two features are almost equally correlated with the target, + then their coefficients should increase at approximately the same + rate. The algorithm thus behaves as intuition would expect, and + also is more stable. - - It is easily modified to produce solutions for other estimators, - like the Lasso. +- It is easily modified to produce solutions for other estimators, + like the Lasso. The disadvantages of the LARS method include: - - Because LARS is based upon an iterative refitting of the - residuals, it would appear to be especially sensitive to the - effects of noise. This problem is discussed in detail by Weisberg - in the discussion section of the Efron et al. (2004) Annals of - Statistics article. +- Because LARS is based upon an iterative refitting of the + residuals, it would appear to be especially sensitive to the + effects of noise. This problem is discussed in detail by Weisberg + in the discussion section of the Efron et al. (2004) Annals of + Statistics article. -The LARS model can be used using estimator :class:`Lars`, or its +The LARS model can be used via the estimator :class:`Lars`, or its low-level implementation :func:`lars_path` or :func:`lars_path_gram`. @@ -512,8 +615,8 @@ algorithm, and unlike the implementation based on coordinate descent, this yields the exact solution, which is piecewise linear as a function of the norm of its coefficients. -.. figure:: ../auto_examples/linear_model/images/sphx_glr_plot_lasso_lars_001.png - :target: ../auto_examples/linear_model/plot_lasso_lars.html +.. figure:: ../auto_examples/linear_model/images/sphx_glr_plot_lasso_lasso_lars_elasticnet_path_001.png + :target: ../auto_examples/linear_model/plot_lasso_lasso_lars_elasticnet_path.html :align: center :scale: 50% @@ -524,52 +627,50 @@ function of the norm of its coefficients. >>> reg.fit([[0, 0], [1, 1]], [0, 1]) LassoLars(alpha=0.1) >>> reg.coef_ - array([0.717157..., 0. ]) + array([0.6, 0. ]) -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_linear_model_plot_lasso_lars.py` +* :ref:`sphx_glr_auto_examples_linear_model_plot_lasso_lasso_lars_elasticnet_path.py` -The Lars algorithm provides the full path of the coefficients along +The LARS algorithm provides the full path of the coefficients along the regularization parameter almost for free, thus a common operation is to retrieve the path with one of the functions :func:`lars_path` or :func:`lars_path_gram`. -Mathematical formulation ------------------------- - -The algorithm is similar to forward stepwise regression, but instead -of including features at each step, the estimated coefficients are -increased in a direction equiangular to each one's correlations with -the residual. +.. dropdown:: Mathematical formulation -Instead of giving a vector result, the LARS solution consists of a -curve denoting the solution for each value of the :math:`\ell_1` norm of the -parameter vector. The full coefficients path is stored in the array -``coef_path_``, which has size (n_features, max_features+1). The first -column is always zero. + The algorithm is similar to forward stepwise regression, but instead + of including features at each step, the estimated coefficients are + increased in a direction equiangular to each one's correlations with + the residual. -.. topic:: References: + Instead of giving a vector result, the LARS solution consists of a + curve denoting the solution for each value of the :math:`\ell_1` norm of the + parameter vector. The full coefficients path is stored in the array + ``coef_path_`` of shape `(n_features, max_features + 1)`. The first + column is always zero. - * Original Algorithm is detailed in the paper `Least Angle Regression - `_ - by Hastie et al. + .. rubric:: References + * Original Algorithm is detailed in the paper `Least Angle Regression + `_ + by Hastie et al. .. _omp: Orthogonal Matching Pursuit (OMP) ================================= -:class:`OrthogonalMatchingPursuit` and :func:`orthogonal_mp` implements the OMP +:class:`OrthogonalMatchingPursuit` and :func:`orthogonal_mp` implement the OMP algorithm for approximating the fit of a linear model with constraints imposed -on the number of non-zero coefficients (ie. the :math:`\ell_0` pseudo-norm). +on the number of non-zero coefficients (i.e. the :math:`\ell_0` pseudo-norm). Being a forward feature selection method like :ref:`least_angle_regression`, orthogonal matching pursuit can approximate the optimum solution vector with a fixed number of non-zero elements: .. math:: - \underset{w}{\operatorname{arg\,min\,}} ||y - Xw||_2^2 \text{ subject to } ||w||_0 \leq n_{\text{nonzero\_coefs}} + \underset{w}{\operatorname{arg\,min\,}} ||y - Xw||_2^2 \text{ subject to } ||w||_0 \leq n_{\text{nonzero_coefs}} Alternatively, orthogonal matching pursuit can target a specific error instead of a specific number of non-zero coefficients. This can be expressed as: @@ -585,18 +686,17 @@ residual is recomputed using an orthogonal projection on the space of the previously chosen dictionary elements. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_linear_model_plot_omp.py` +* :ref:`sphx_glr_auto_examples_linear_model_plot_omp.py` -.. topic:: References: +.. dropdown:: References - * https://www.cs.technion.ac.il/~ronrubin/Publications/KSVD-OMP-v2.pdf - - * `Matching pursuits with time-frequency dictionaries - `_, - S. G. Mallat, Z. Zhang, + * https://www.cs.technion.ac.il/~ronrubin/Publications/KSVD-OMP-v2.pdf + * `Matching pursuits with time-frequency dictionaries + `_, + S. G. Mallat, Z. Zhang, .. _bayesian_regression: @@ -619,29 +719,29 @@ variable to be estimated from the data. To obtain a fully probabilistic model, the output :math:`y` is assumed to be Gaussian distributed around :math:`X w`: -.. math:: p(y|X,w,\alpha) = \mathcal{N}(y|X w,\alpha) +.. math:: p(y|X,w,\alpha) = \mathcal{N}(y|X w,\alpha^{-1}) where :math:`\alpha` is again treated as a random variable that is to be estimated from the data. The advantages of Bayesian Regression are: - - It adapts to the data at hand. +- It adapts to the data at hand. - - It can be used to include regularization parameters in the - estimation procedure. +- It can be used to include regularization parameters in the + estimation procedure. The disadvantages of Bayesian regression include: - - Inference of the model can be time consuming. +- Inference of the model can be time consuming. -.. topic:: References +.. dropdown:: References - * A good introduction to Bayesian methods is given in C. Bishop: Pattern - Recognition and Machine learning + * A good introduction to Bayesian methods is given in C. Bishop: Pattern + Recognition and Machine learning - * Original Algorithm is detailed in the book `Bayesian learning for neural - networks` by Radford M. Neal + * Original Algorithm is detailed in the book `Bayesian learning for neural + networks` by Radford M. Neal .. _bayesian_ridge_regression: @@ -675,13 +775,6 @@ There are four more hyperparameters, :math:`\alpha_1`, :math:`\alpha_2`, :math:`\alpha` and :math:`\lambda`. These are usually chosen to be *non-informative*. By default :math:`\alpha_1 = \alpha_2 = \lambda_1 = \lambda_2 = 10^{-6}`. - -.. figure:: ../auto_examples/linear_model/images/sphx_glr_plot_bayesian_ridge_001.png - :target: ../auto_examples/linear_model/plot_bayesian_ridge.html - :align: center - :scale: 50% - - Bayesian Ridge Regression is used for regression:: >>> from sklearn import linear_model @@ -701,212 +794,344 @@ The coefficients :math:`w` of the model can be accessed:: >>> reg.coef_ array([0.49999993, 0.49999993]) -Due to the Bayesian framework, the weights found are slightly different to the +Due to the Bayesian framework, the weights found are slightly different from the ones found by :ref:`ordinary_least_squares`. However, Bayesian Ridge Regression is more robust to ill-posed problems. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_linear_model_plot_bayesian_ridge.py` - * :ref:`sphx_glr_auto_examples_linear_model_plot_bayesian_ridge_curvefit.py` +* :ref:`sphx_glr_auto_examples_linear_model_plot_bayesian_ridge_curvefit.py` -.. topic:: References: +.. dropdown:: References - * Section 3.3 in Christopher M. Bishop: Pattern Recognition and Machine Learning, 2006 + * Section 3.3 in Christopher M. Bishop: Pattern Recognition and Machine Learning, 2006 - * David J. C. MacKay, `Bayesian Interpolation `_, 1992. + * David J. C. MacKay, `Bayesian Interpolation `_, 1992. - * Michael E. Tipping, `Sparse Bayesian Learning and the Relevance Vector Machine `_, 2001. + * Michael E. Tipping, `Sparse Bayesian Learning and the Relevance Vector Machine `_, 2001. +.. _automatic_relevance_determination: Automatic Relevance Determination - ARD --------------------------------------- -:class:`ARDRegression` is very similar to `Bayesian Ridge Regression`_, -but can lead to sparser coefficients :math:`w` [1]_ [2]_. -:class:`ARDRegression` poses a different prior over :math:`w`, by dropping the -assumption of the Gaussian being spherical. +The Automatic Relevance Determination (as being implemented in +:class:`ARDRegression`) is a kind of linear model which is very similar to the +`Bayesian Ridge Regression`_, but that leads to sparser coefficients :math:`w` +[1]_ [2]_. -Instead, the distribution over :math:`w` is assumed to be an axis-parallel, -elliptical Gaussian distribution. - -This means each coefficient :math:`w_{i}` is drawn from a Gaussian distribution, -centered on zero and with a precision :math:`\lambda_{i}`: +:class:`ARDRegression` poses a different prior over :math:`w`: it drops +the spherical Gaussian distribution for a centered elliptic Gaussian +distribution. This means each coefficient :math:`w_{i}` can itself be drawn from +a Gaussian distribution, centered on zero and with a precision +:math:`\lambda_{i}`: .. math:: p(w|\lambda) = \mathcal{N}(w|0,A^{-1}) -with :math:`\text{diag}(A) = \lambda = \{\lambda_{1},...,\lambda_{p}\}`. +with :math:`A` being a positive definite diagonal matrix and +:math:`\text{diag}(A) = \lambda = \{\lambda_{1},...,\lambda_{p}\}`. -In contrast to `Bayesian Ridge Regression`_, each coordinate of :math:`w_{i}` -has its own standard deviation :math:`\lambda_i`. The prior over all -:math:`\lambda_i` is chosen to be the same gamma distribution given by -hyperparameters :math:`\lambda_1` and :math:`\lambda_2`. +In contrast to the `Bayesian Ridge Regression`_, each coordinate of +:math:`w_{i}` has its own standard deviation :math:`\frac{1}{\lambda_i}`. The +prior over all :math:`\lambda_i` is chosen to be the same gamma distribution +given by the hyperparameters :math:`\lambda_1` and :math:`\lambda_2`. -.. figure:: ../auto_examples/linear_model/images/sphx_glr_plot_ard_001.png - :target: ../auto_examples/linear_model/plot_ard.html - :align: center - :scale: 50% - -ARD is also known in the literature as *Sparse Bayesian Learning* and -*Relevance Vector Machine* [3]_ [4]_. +ARD is also known in the literature as *Sparse Bayesian Learning* and *Relevance +Vector Machine* [3]_ [4]_. For a worked-out comparison between ARD and `Bayesian +Ridge Regression`_, see the example below. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_linear_model_plot_ard.py` +* :ref:`sphx_glr_auto_examples_linear_model_plot_ard.py` -.. topic:: References: - .. [1] Christopher M. Bishop: Pattern Recognition and Machine Learning, Chapter 7.2.1 +.. rubric:: References - .. [2] David Wipf and Srikantan Nagarajan: `A new view of automatic relevance determination `_ +.. [1] Christopher M. Bishop: Pattern Recognition and Machine Learning, Chapter 7.2.1 - .. [3] Michael E. Tipping: `Sparse Bayesian Learning and the Relevance Vector Machine `_ +.. [2] David Wipf and Srikantan Nagarajan: `A New View of Automatic Relevance Determination `_ - .. [4] Tristan Fletcher: `Relevance Vector Machines explained `_ +.. [3] Michael E. Tipping: `Sparse Bayesian Learning and the Relevance Vector Machine `_ +.. [4] Tristan Fletcher: `Relevance Vector Machines Explained `_ .. _Logistic_regression: Logistic regression =================== -Logistic regression, despite its name, is a linear model for classification -rather than regression. Logistic regression is also known in the literature as -logit regression, maximum-entropy classification (MaxEnt) or the log-linear -classifier. In this model, the probabilities describing the possible outcomes -of a single trial are modeled using a -`logistic function `_. +The logistic regression is implemented in :class:`LogisticRegression`. Despite +its name, it is implemented as a linear model for classification rather than +regression in terms of the scikit-learn/ML nomenclature. The logistic +regression is also known in the literature as logit regression, +maximum-entropy classification (MaxEnt) or the log-linear classifier. In this +model, the probabilities describing the possible outcomes of a single trial +are modeled using a `logistic function +`_. -Logistic regression is implemented in :class:`LogisticRegression`. This implementation can fit binary, One-vs-Rest, or multinomial logistic regression with optional :math:`\ell_1`, :math:`\ell_2` or Elastic-Net regularization. -.. note:: +.. note:: **Regularization** Regularization is applied by default, which is common in machine learning but not in statistics. Another advantage of regularization is that it improves numerical stability. No regularization amounts to setting C to a very high value. -As an optimization problem, binary class :math:`\ell_2` penalized logistic -regression minimizes the following cost function: +.. note:: **Logistic Regression as a special case of the Generalized Linear Models (GLM)** -.. math:: \min_{w, c} \frac{1}{2}w^T w + C \sum_{i=1}^n \log(\exp(- y_i (X_i^T w + c)) + 1) . + Logistic regression is a special case of + :ref:`generalized_linear_models` with a Binomial / Bernoulli conditional + distribution and a Logit link. The numerical output of the logistic + regression, which is the predicted probability, can be used as a classifier + by applying a threshold (by default 0.5) to it. This is how it is + implemented in scikit-learn, so it expects a categorical target, making + the Logistic Regression a classifier. -Similarly, :math:`\ell_1` regularized logistic regression solves the following -optimization problem: +.. rubric:: Examples -.. math:: \min_{w, c} \|w\|_1 + C \sum_{i=1}^n \log(\exp(- y_i (X_i^T w + c)) + 1). +* :ref:`sphx_glr_auto_examples_linear_model_plot_logistic_l1_l2_sparsity.py` +* :ref:`sphx_glr_auto_examples_linear_model_plot_logistic_path.py` +* :ref:`sphx_glr_auto_examples_linear_model_plot_logistic_multinomial.py` +* :ref:`sphx_glr_auto_examples_linear_model_plot_sparse_logistic_regression_20newsgroups.py` +* :ref:`sphx_glr_auto_examples_linear_model_plot_sparse_logistic_regression_mnist.py` +* :ref:`sphx_glr_auto_examples_classification_plot_classification_probability.py` -Elastic-Net regularization is a combination of :math:`\ell_1` and -:math:`\ell_2`, and minimizes the following cost function: +Binary Case +----------- -.. math:: \min_{w, c} \frac{1 - \rho}{2}w^T w + \rho \|w\|_1 + C \sum_{i=1}^n \log(\exp(- y_i (X_i^T w + c)) + 1), +For notational ease, we assume that the target :math:`y_i` takes values in the +set :math:`\{0, 1\}` for data point :math:`i`. +Once fitted, the :meth:`~sklearn.linear_model.LogisticRegression.predict_proba` +method of :class:`~sklearn.linear_model.LogisticRegression` predicts +the probability of the positive class :math:`P(y_i=1|X_i)` as -where :math:`\rho` controls the strength of :math:`\ell_1` regularization vs. -:math:`\ell_2` regularization (it corresponds to the `l1_ratio` parameter). +.. math:: \hat{p}(X_i) = \operatorname{expit}(X_i w + w_0) = \frac{1}{1 + \exp(-X_i w - w_0)}. -Note that, in this notation, it's assumed that the target :math:`y_i` takes -values in the set :math:`{-1, 1}` at trial :math:`i`. We can also see that -Elastic-Net is equivalent to :math:`\ell_1` when :math:`\rho = 1` and equivalent -to :math:`\ell_2` when :math:`\rho=0`. -The solvers implemented in the class :class:`LogisticRegression` -are "liblinear", "newton-cg", "lbfgs", "sag" and "saga": - -The solver "liblinear" uses a coordinate descent (CD) algorithm, and relies -on the excellent C++ `LIBLINEAR library -`_, which is shipped with -scikit-learn. However, the CD algorithm implemented in liblinear cannot learn -a true multinomial (multiclass) model; instead, the optimization problem is -decomposed in a "one-vs-rest" fashion so separate binary classifiers are -trained for all classes. This happens under the hood, so -:class:`LogisticRegression` instances using this solver behave as multiclass -classifiers. For :math:`\ell_1` regularization :func:`sklearn.svm.l1_min_c` allows to -calculate the lower bound for C in order to get a non "null" (all feature -weights to zero) model. - -The "lbfgs", "sag" and "newton-cg" solvers only support :math:`\ell_2` -regularization or no regularization, and are found to converge faster for some -high-dimensional data. Setting `multi_class` to "multinomial" with these solvers -learns a true multinomial logistic regression model [5]_, which means that its -probability estimates should be better calibrated than the default "one-vs-rest" -setting. - -The "sag" solver uses Stochastic Average Gradient descent [6]_. It is faster -than other solvers for large datasets, when both the number of samples and the -number of features are large. - -The "saga" solver [7]_ is a variant of "sag" that also supports the -non-smooth `penalty="l1"`. This is therefore the solver of choice for sparse -multinomial logistic regression. It is also the only solver that supports -`penalty="elasticnet"`. - -The "lbfgs" is an optimization algorithm that approximates the -Broyden–Fletcher–Goldfarb–Shanno algorithm [8]_, which belongs to -quasi-Newton methods. The "lbfgs" solver is recommended for use for -small data-sets but for larger datasets its performance suffers. [9]_ - -The following table summarizes the penalties supported by each solver: - -+------------------------------+-----------------+-------------+-----------------+-----------+------------+ -| | **Solvers** | -+------------------------------+-----------------+-------------+-----------------+-----------+------------+ -| **Penalties** | **'liblinear'** | **'lbfgs'** | **'newton-cg'** | **'sag'** | **'saga'** | -+------------------------------+-----------------+-------------+-----------------+-----------+------------+ -| Multinomial + L2 penalty | no | yes | yes | yes | yes | -+------------------------------+-----------------+-------------+-----------------+-----------+------------+ -| OVR + L2 penalty | yes | yes | yes | yes | yes | -+------------------------------+-----------------+-------------+-----------------+-----------+------------+ -| Multinomial + L1 penalty | no | no | no | no | yes | -+------------------------------+-----------------+-------------+-----------------+-----------+------------+ -| OVR + L1 penalty | yes | no | no | no | yes | -+------------------------------+-----------------+-------------+-----------------+-----------+------------+ -| Elastic-Net | no | no | no | no | yes | -+------------------------------+-----------------+-------------+-----------------+-----------+------------+ -| No penalty ('none') | no | yes | yes | yes | yes | -+------------------------------+-----------------+-------------+-----------------+-----------+------------+ -| **Behaviors** | | -+------------------------------+-----------------+-------------+-----------------+-----------+------------+ -| Penalize the intercept (bad) | yes | no | no | no | no | -+------------------------------+-----------------+-------------+-----------------+-----------+------------+ -| Faster for large datasets | no | no | no | yes | yes | -+------------------------------+-----------------+-------------+-----------------+-----------+------------+ -| Robust to unscaled datasets | yes | yes | yes | no | no | -+------------------------------+-----------------+-------------+-----------------+-----------+------------+ - -The "lbfgs" solver is used by default for its robustness. For large datasets -the "saga" solver is usually faster. -For large dataset, you may also consider using :class:`SGDClassifier` -with 'log' loss, which might be even faster but requires more tuning. - -.. topic:: Examples: - - * :ref:`sphx_glr_auto_examples_linear_model_plot_logistic_l1_l2_sparsity.py` +As an optimization problem, binary +class logistic regression with regularization term :math:`r(w)` minimizes the +following cost function: - * :ref:`sphx_glr_auto_examples_linear_model_plot_logistic_path.py` +.. math:: + :name: regularized-logistic-loss + + \min_{w} \frac{1}{S}\sum_{i=1}^n s_i + \left(-y_i \log(\hat{p}(X_i)) - (1 - y_i) \log(1 - \hat{p}(X_i))\right) + + \frac{r(w)}{S C}\,, + +where :math:`{s_i}` corresponds to the weights assigned by the user to a +specific training sample (the vector :math:`s` is formed by element-wise +multiplication of the class weights and sample weights), +and the sum :math:`S = \sum_{i=1}^n s_i`. + +We currently provide four choices for the regularization term :math:`r(w)` via +the `penalty` argument: + ++----------------+-------------------------------------------------+ +| penalty | :math:`r(w)` | ++================+=================================================+ +| `None` | :math:`0` | ++----------------+-------------------------------------------------+ +| :math:`\ell_1` | :math:`\|w\|_1` | ++----------------+-------------------------------------------------+ +| :math:`\ell_2` | :math:`\frac{1}{2}\|w\|_2^2 = \frac{1}{2}w^T w` | ++----------------+-------------------------------------------------+ +| `ElasticNet` | :math:`\frac{1 - \rho}{2}w^T w + \rho \|w\|_1` | ++----------------+-------------------------------------------------+ + +For ElasticNet, :math:`\rho` (which corresponds to the `l1_ratio` parameter) +controls the strength of :math:`\ell_1` regularization vs. :math:`\ell_2` +regularization. Elastic-Net is equivalent to :math:`\ell_1` when +:math:`\rho = 1` and equivalent to :math:`\ell_2` when :math:`\rho=0`. + +Note that the scale of the class weights and the sample weights will influence +the optimization problem. For instance, multiplying the sample weights by a +constant :math:`b>0` is equivalent to multiplying the (inverse) regularization +strength `C` by :math:`b`. + +Multinomial Case +---------------- - * :ref:`sphx_glr_auto_examples_linear_model_plot_logistic_multinomial.py` +The binary case can be extended to :math:`K` classes leading to the multinomial +logistic regression, see also `log-linear model +`_. - * :ref:`sphx_glr_auto_examples_linear_model_plot_sparse_logistic_regression_20newsgroups.py` +.. note:: + It is possible to parameterize a :math:`K`-class classification model + using only :math:`K-1` weight vectors, leaving one class probability fully + determined by the other class probabilities by leveraging the fact that all + class probabilities must sum to one. We deliberately choose to overparameterize the model + using :math:`K` weight vectors for ease of implementation and to preserve the + symmetrical inductive bias regarding ordering of classes, see [16]_. This effect becomes + especially important when using regularization. The choice of overparameterization can be + detrimental for unpenalized models since then the solution may not be unique, as shown in [16]_. + +.. dropdown:: Mathematical details + + Let :math:`y_i \in \{1, \ldots, K\}` be the label (ordinal) encoded target variable for observation :math:`i`. + Instead of a single coefficient vector, we now have + a matrix of coefficients :math:`W` where each row vector :math:`W_k` corresponds to class + :math:`k`. We aim at predicting the class probabilities :math:`P(y_i=k|X_i)` via + :meth:`~sklearn.linear_model.LogisticRegression.predict_proba` as: + + .. math:: \hat{p}_k(X_i) = \frac{\exp(X_i W_k + W_{0, k})}{\sum_{l=0}^{K-1} \exp(X_i W_l + W_{0, l})}. + + The objective for the optimization becomes + + .. math:: + \min_W -\frac{1}{S}\sum_{i=1}^n \sum_{k=0}^{K-1} s_{ik} [y_i = k] \log(\hat{p}_k(X_i)) + + \frac{r(W)}{S C}\,, + + where :math:`[P]` represents the Iverson bracket which evaluates to :math:`0` + if :math:`P` is false, otherwise it evaluates to :math:`1`. + + Again, :math:`s_{ik}` are the weights assigned by the user (multiplication of sample + weights and class weights) with their sum :math:`S = \sum_{i=1}^n \sum_{k=0}^{K-1} s_{ik}`. + + We currently provide four choices + for the regularization term :math:`r(W)` via the `penalty` argument, where :math:`m` + is the number of features: + + +----------------+----------------------------------------------------------------------------------+ + | penalty | :math:`r(W)` | + +================+==================================================================================+ + | `None` | :math:`0` | + +----------------+----------------------------------------------------------------------------------+ + | :math:`\ell_1` | :math:`\|W\|_{1,1} = \sum_{i=1}^m\sum_{j=1}^{K}|W_{i,j}|` | + +----------------+----------------------------------------------------------------------------------+ + | :math:`\ell_2` | :math:`\frac{1}{2}\|W\|_F^2 = \frac{1}{2}\sum_{i=1}^m\sum_{j=1}^{K} W_{i,j}^2` | + +----------------+----------------------------------------------------------------------------------+ + | `ElasticNet` | :math:`\frac{1 - \rho}{2}\|W\|_F^2 + \rho \|W\|_{1,1}` | + +----------------+----------------------------------------------------------------------------------+ + +.. _logistic_regression_solvers: + +Solvers +------- - * :ref:`sphx_glr_auto_examples_linear_model_plot_sparse_logistic_regression_mnist.py` +The solvers implemented in the class :class:`LogisticRegression` +are "lbfgs", "liblinear", "newton-cg", "newton-cholesky", "sag" and "saga": + +The following table summarizes the penalties and multinomial multiclass supported by each solver: + ++------------------------------+-----------------+-------------+-----------------+-----------------------+-----------+------------+ +| | **Solvers** | ++------------------------------+-------------+-----------------+-----------------+-----------------------+-----------+------------+ +| **Penalties** | **'lbfgs'** | **'liblinear'** | **'newton-cg'** | **'newton-cholesky'** | **'sag'** | **'saga'** | ++------------------------------+-------------+-----------------+-----------------+-----------------------+-----------+------------+ +| L2 penalty | yes | yes | yes | yes | yes | yes | ++------------------------------+-------------+-----------------+-----------------+-----------------------+-----------+------------+ +| L1 penalty | no | yes | no | no | no | yes | ++------------------------------+-------------+-----------------+-----------------+-----------------------+-----------+------------+ +| Elastic-Net (L1 + L2) | no | no | no | no | no | yes | ++------------------------------+-------------+-----------------+-----------------+-----------------------+-----------+------------+ +| No penalty ('none') | yes | no | yes | yes | yes | yes | ++------------------------------+-------------+-----------------+-----------------+-----------------------+-----------+------------+ +| **Multiclass support** | | ++------------------------------+-------------+-----------------+-----------------+-----------------------+-----------+------------+ +| multinomial multiclass | yes | no | yes | yes | yes | yes | ++------------------------------+-------------+-----------------+-----------------+-----------------------+-----------+------------+ +| **Behaviors** | | ++------------------------------+-------------+-----------------+-----------------+-----------------------+-----------+------------+ +| Penalize the intercept (bad) | no | yes | no | no | no | no | ++------------------------------+-------------+-----------------+-----------------+-----------------------+-----------+------------+ +| Faster for large datasets | no | no | no | no | yes | yes | ++------------------------------+-------------+-----------------+-----------------+-----------------------+-----------+------------+ +| Robust to unscaled datasets | yes | yes | yes | yes | no | no | ++------------------------------+-------------+-----------------+-----------------+-----------------------+-----------+------------+ + +The "lbfgs" solver is used by default for its robustness. For +`n_samples >> n_features`, "newton-cholesky" is a good choice and can reach high +precision (tiny `tol` values). For large datasets +the "saga" solver is usually faster (than "lbfgs"), in particular for low precision +(high `tol`). +For large dataset, you may also consider using :class:`SGDClassifier` +with `loss="log_loss"`, which might be even faster but requires more tuning. .. _liblinear_differences: -.. topic:: Differences from liblinear: +Differences between solvers +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +There might be a difference in the scores obtained between +:class:`LogisticRegression` with ``solver=liblinear`` or +:class:`~sklearn.svm.LinearSVC` and the external liblinear library directly, +when ``fit_intercept=False`` and the fit ``coef_`` (or) the data to be predicted +are zeroes. This is because for the sample(s) with ``decision_function`` zero, +:class:`LogisticRegression` and :class:`~sklearn.svm.LinearSVC` predict the +negative class, while liblinear predicts the positive class. Note that a model +with ``fit_intercept=False`` and having many samples with ``decision_function`` +zero, is likely to be an underfit, bad model and you are advised to set +``fit_intercept=True`` and increase the ``intercept_scaling``. + +.. dropdown:: Solvers' details + + * The solver "liblinear" uses a coordinate descent (CD) algorithm, and relies + on the excellent C++ `LIBLINEAR library + `_, which is shipped with + scikit-learn. However, the CD algorithm implemented in liblinear cannot learn + a true multinomial (multiclass) model; instead, the optimization problem is + decomposed in a "one-vs-rest" fashion so separate binary classifiers are + trained for all classes. This happens under the hood, so + :class:`LogisticRegression` instances using this solver behave as multiclass + classifiers. For :math:`\ell_1` regularization :func:`sklearn.svm.l1_min_c` allows to + calculate the lower bound for C in order to get a non "null" (all feature + weights to zero) model. + + * The "lbfgs", "newton-cg" and "sag" solvers only support :math:`\ell_2` + regularization or no regularization, and are found to converge faster for some + high-dimensional data. Setting `multi_class` to "multinomial" with these solvers + learns a true multinomial logistic regression model [5]_, which means that its + probability estimates should be better calibrated than the default "one-vs-rest" + setting. + + * The "sag" solver uses Stochastic Average Gradient descent [6]_. It is faster + than other solvers for large datasets, when both the number of samples and the + number of features are large. + + * The "saga" solver [7]_ is a variant of "sag" that also supports the + non-smooth `penalty="l1"`. This is therefore the solver of choice for sparse + multinomial logistic regression. It is also the only solver that supports + `penalty="elasticnet"`. + + * The "lbfgs" is an optimization algorithm that approximates the + Broyden–Fletcher–Goldfarb–Shanno algorithm [8]_, which belongs to + quasi-Newton methods. As such, it can deal with a wide range of different training + data and is therefore the default solver. Its performance, however, suffers on poorly + scaled datasets and on datasets with one-hot encoded categorical features with rare + categories. + + * The "newton-cholesky" solver is an exact Newton solver that calculates the Hessian + matrix and solves the resulting linear system. It is a very good choice for + `n_samples` >> `n_features` and can reach high precision (tiny values of `tol`), + but has a few shortcomings: Only :math:`\ell_2` regularization is supported. + Furthermore, because the Hessian matrix is explicitly computed, the memory usage + has a quadratic dependency on `n_features` as well as on `n_classes`. + + For a comparison of some of these solvers, see [9]_. + + .. rubric:: References + + .. [5] Christopher M. Bishop: Pattern Recognition and Machine Learning, Chapter 4.3.4 + + .. [6] Mark Schmidt, Nicolas Le Roux, and Francis Bach: `Minimizing Finite Sums with the Stochastic Average Gradient. `_ + + .. [7] Aaron Defazio, Francis Bach, Simon Lacoste-Julien: + :arxiv:`SAGA: A Fast Incremental Gradient Method With Support for + Non-Strongly Convex Composite Objectives. <1407.0202>` + + .. [8] https://en.wikipedia.org/wiki/Broyden%E2%80%93Fletcher%E2%80%93Goldfarb%E2%80%93Shanno_algorithm + + .. [9] Thomas P. Minka `"A comparison of numerical optimizers for logistic regression" + `_ + + .. [16] :arxiv:`Simon, Noah, J. Friedman and T. Hastie. + "A Blockwise Descent Algorithm for Group-penalized Multiresponse and + Multinomial Regression." <1311.6529>` - There might be a difference in the scores obtained between - :class:`LogisticRegression` with ``solver=liblinear`` - or :class:`LinearSVC` and the external liblinear library directly, - when ``fit_intercept=False`` and the fit ``coef_`` (or) the data to - be predicted are zeroes. This is because for the sample(s) with - ``decision_function`` zero, :class:`LogisticRegression` and :class:`LinearSVC` - predict the negative class, while liblinear predicts the positive class. - Note that a model with ``fit_intercept=False`` and having many samples with - ``decision_function`` zero, is likely to be a underfit, bad model and you are - advised to set ``fit_intercept=True`` and increase the intercept_scaling. .. note:: **Feature selection with sparse logistic regression** @@ -918,7 +1143,7 @@ with 'log' loss, which might be even faster but requires more tuning. It is possible to obtain the p-values and confidence intervals for coefficients in cases of regression without penalization. The `statsmodels - package ` natively supports this. + package `_ natively supports this. Within sklearn, one could use bootstrapping instead as well. @@ -928,23 +1153,12 @@ according to the ``scoring`` attribute. The "newton-cg", "sag", "saga" and "lbfgs" solvers are found to be faster for high-dimensional dense data, due to warm-starting (see :term:`Glossary `). -.. topic:: References: - - .. [5] Christopher M. Bishop: Pattern Recognition and Machine Learning, Chapter 4.3.4 - - .. [6] Mark Schmidt, Nicolas Le Roux, and Francis Bach: `Minimizing Finite Sums with the Stochastic Average Gradient. `_ - - .. [7] Aaron Defazio, Francis Bach, Simon Lacoste-Julien: `SAGA: A Fast Incremental Gradient Method With Support for Non-Strongly Convex Composite Objectives. `_ - - .. [8] https://en.wikipedia.org/wiki/Broyden%E2%80%93Fletcher%E2%80%93Goldfarb%E2%80%93Shanno_algorithm - - .. [9] `"Performance Evaluation of Lbfgs vs other solvers" - `_ - .. _Generalized_linear_regression: -Generalized Linear Regression -============================= +.. _Generalized_linear_models: + +Generalized Linear Models +========================= Generalized Linear Models (GLM) extend linear models in two ways [10]_. First, the predicted values :math:`\hat{y}` are linked to a linear @@ -959,22 +1173,23 @@ reproductive exponential dispersion model (EDM) [11]_). The minimization problem becomes: -.. math:: \min_{w} \frac{1}{2 n_{\text{samples}}} \sum_i d(y_i, \hat{y}_i) + \frac{\alpha}{2} ||w||_2, +.. math:: \min_{w} \frac{1}{2 n_{\text{samples}}} \sum_i d(y_i, \hat{y}_i) + \frac{\alpha}{2} ||w||_2^2, where :math:`\alpha` is the L2 regularization penalty. When sample weights are provided, the average becomes a weighted average. -The following table lists some specific EDMs and their unit deviance (all of -these are instances of the Tweedie family): +The following table lists some specific EDMs and their unit deviance : -================= =============================== ============================================ +================= ================================ ============================================ Distribution Target Domain Unit Deviance :math:`d(y, \hat{y})` -================= =============================== ============================================ -Normal :math:`y \in (-\infty, \infty)` :math:`(y-\hat{y})^2` -Poisson :math:`y \in [0, \infty)` :math:`2(y\log\frac{y}{\hat{y}}-y+\hat{y})` -Gamma :math:`y \in (0, \infty)` :math:`2(\log\frac{\hat{y}}{y}+\frac{y}{\hat{y}}-1)` -Inverse Gaussian :math:`y \in (0, \infty)` :math:`\frac{(y-\hat{y})^2}{y\hat{y}^2}` -================= =============================== ============================================ +================= ================================ ============================================ +Normal :math:`y \in (-\infty, \infty)` :math:`(y-\hat{y})^2` +Bernoulli :math:`y \in \{0, 1\}` :math:`2({y}\log\frac{y}{\hat{y}}+({1}-{y})\log\frac{{1}-{y}}{{1}-\hat{y}})` +Categorical :math:`y \in \{0, 1, ..., k\}` :math:`2\sum_{i \in \{0, 1, ..., k\}} I(y = i) y_\text{i}\log\frac{I(y = i)}{\hat{I(y = i)}}` +Poisson :math:`y \in [0, \infty)` :math:`2(y\log\frac{y}{\hat{y}}-y+\hat{y})` +Gamma :math:`y \in (0, \infty)` :math:`2(\log\frac{\hat{y}}{y}+\frac{y}{\hat{y}}-1)` +Inverse Gaussian :math:`y \in (0, \infty)` :math:`\frac{(y-\hat{y})^2}{y\hat{y}^2}` +================= ================================ ============================================ The Probability Density Functions (PDF) of these distributions are illustrated in the following figure, @@ -989,40 +1204,58 @@ in the following figure, distribution, but not for the Gamma distribution which has a strictly positive target domain. +The Bernoulli distribution is a discrete probability distribution modelling a +Bernoulli trial - an event that has only two mutually exclusive outcomes. +The Categorical distribution is a generalization of the Bernoulli distribution +for a categorical random variable. While a random variable in a Bernoulli +distribution has two possible outcomes, a Categorical random variable can take +on one of K possible categories, with the probability of each category +specified separately. + The choice of the distribution depends on the problem at hand: * If the target values :math:`y` are counts (non-negative integer valued) or - relative frequencies (non-negative), you might use a Poisson deviance - with log-link. -* If the target values are positive valued and skewed, you might try a - Gamma deviance with log-link. -* If the target values seem to be heavier tailed than a Gamma distribution, - you might try an Inverse Gaussian deviance (or even higher variance powers - of the Tweedie family). - - -Examples of use cases include: - -* Agriculture / weather modeling: number of rain events per year (Poisson), - amount of rainfall per event (Gamma), total rainfall per year (Tweedie / - Compound Poisson Gamma). -* Risk modeling / insurance policy pricing: number of claim events / - policyholder per year (Poisson), cost per event (Gamma), total cost per - policyholder per year (Tweedie / Compound Poisson Gamma). -* Predictive maintenance: number of production interruption events per year - (Poisson), duration of interruption (Gamma), total interruption time per year - (Tweedie / Compound Poisson Gamma). - - -.. topic:: References: - - .. [10] McCullagh, Peter; Nelder, John (1989). Generalized Linear Models, - Second Edition. Boca Raton: Chapman and Hall/CRC. ISBN 0-412-31760-5. - - .. [11] Jørgensen, B. (1992). The theory of exponential dispersion models - and analysis of deviance. Monografias de matemÃĄtica, no. 51. See also - `Exponential dispersion model. - `_ + relative frequencies (non-negative), you might use a Poisson distribution + with a log-link. +* If the target values are positive valued and skewed, you might try a Gamma + distribution with a log-link. +* If the target values seem to be heavier tailed than a Gamma distribution, you + might try an Inverse Gaussian distribution (or even higher variance powers of + the Tweedie family). +* If the target values :math:`y` are probabilities, you can use the Bernoulli + distribution. The Bernoulli distribution with a logit link can be used for + binary classification. The Categorical distribution with a softmax link can be + used for multiclass classification. + + +.. dropdown:: Examples of use cases + + * Agriculture / weather modeling: number of rain events per year (Poisson), + amount of rainfall per event (Gamma), total rainfall per year (Tweedie / + Compound Poisson Gamma). + * Risk modeling / insurance policy pricing: number of claim events / + policyholder per year (Poisson), cost per event (Gamma), total cost per + policyholder per year (Tweedie / Compound Poisson Gamma). + * Credit Default: probability that a loan can't be paid back (Bernoulli). + * Fraud Detection: probability that a financial transaction like a cash transfer + is a fraudulent transaction (Bernoulli). + * Predictive maintenance: number of production interruption events per year + (Poisson), duration of interruption (Gamma), total interruption time per year + (Tweedie / Compound Poisson Gamma). + * Medical Drug Testing: probability of curing a patient in a set of trials or + probability that a patient will experience side effects (Bernoulli). + * News Classification: classification of news articles into three categories + namely Business News, Politics and Entertainment news (Categorical). + +.. rubric:: References + +.. [10] McCullagh, Peter; Nelder, John (1989). Generalized Linear Models, + Second Edition. Boca Raton: Chapman and Hall/CRC. ISBN 0-412-31760-5. + +.. [11] Jørgensen, B. (1992). The theory of exponential dispersion models + and analysis of deviance. Monografias de matemÃĄtica, no. 51. See also + `Exponential dispersion model. + `_ Usage ----- @@ -1051,39 +1284,38 @@ Usage example:: >>> reg.fit([[0, 0], [0, 1], [2, 2]], [0, 1, 2]) TweedieRegressor(alpha=0.5, link='log', power=1) >>> reg.coef_ - array([0.2463..., 0.4337...]) + array([0.2463, 0.4337]) >>> reg.intercept_ - -0.7638... + np.float64(-0.7638) -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_linear_model_plot_poisson_regression_non_normal_loss.py` - * :ref:`sphx_glr_auto_examples_linear_model_plot_tweedie_regression_insurance_claims.py` +* :ref:`sphx_glr_auto_examples_linear_model_plot_poisson_regression_non_normal_loss.py` +* :ref:`sphx_glr_auto_examples_linear_model_plot_tweedie_regression_insurance_claims.py` -Practical considerations ------------------------- +.. dropdown:: Practical considerations -The feature matrix `X` should be standardized before fitting. This ensures -that the penalty treats features equally. + The feature matrix `X` should be standardized before fitting. This ensures + that the penalty treats features equally. -Since the linear predictor :math:`Xw` can be negative and Poisson, -Gamma and Inverse Gaussian distributions don't support negative values, it -is necessary to apply an inverse link function that guarantees the -non-negativeness. For example with `link='log'`, the inverse link function -becomes :math:`h(Xw)=\exp(Xw)`. + Since the linear predictor :math:`Xw` can be negative and Poisson, + Gamma and Inverse Gaussian distributions don't support negative values, it + is necessary to apply an inverse link function that guarantees the + non-negativeness. For example with `link='log'`, the inverse link function + becomes :math:`h(Xw)=\exp(Xw)`. -If you want to model a relative frequency, i.e. counts per exposure (time, -volume, ...) you can do so by using a Poisson distribution and passing -:math:`y=\frac{\mathrm{counts}}{\mathrm{exposure}}` as target values -together with :math:`\mathrm{exposure}` as sample weights. For a concrete -example see e.g. -:ref:`sphx_glr_auto_examples_linear_model_plot_tweedie_regression_insurance_claims.py`. + If you want to model a relative frequency, i.e. counts per exposure (time, + volume, ...) you can do so by using a Poisson distribution and passing + :math:`y=\frac{\mathrm{counts}}{\mathrm{exposure}}` as target values + together with :math:`\mathrm{exposure}` as sample weights. For a concrete + example see e.g. + :ref:`sphx_glr_auto_examples_linear_model_plot_tweedie_regression_insurance_claims.py`. -When performing cross-validation for the `power` parameter of -`TweedieRegressor`, it is advisable to specify an explicit `scoring` function, -because the default scorer :meth:`TweedieRegressor.score` is a function of -`power` itself. + When performing cross-validation for the `power` parameter of + `TweedieRegressor`, it is advisable to specify an explicit `scoring` function, + because the default scorer :meth:`TweedieRegressor.score` is a function of + `power` itself. Stochastic Gradient Descent - SGD ================================= @@ -1100,9 +1332,7 @@ E.g., with ``loss="log"``, :class:`SGDClassifier` fits a logistic regression model, while with ``loss="hinge"`` it fits a linear support vector machine (SVM). -.. topic:: References - - * :ref:`sgd` +You can refer to the dedicated :ref:`sgd` documentation section for more details. .. _perceptron: @@ -1112,16 +1342,21 @@ Perceptron The :class:`Perceptron` is another simple classification algorithm suitable for large scale learning. By default: - - It does not require a learning rate. +- It does not require a learning rate. - - It is not regularized (penalized). +- It is not regularized (penalized). - - It updates its model only on mistakes. +- It updates its model only on mistakes. The last characteristic implies that the Perceptron is slightly faster to train than SGD with the hinge loss and that the resulting models are sparser. +In fact, the :class:`Perceptron` is a wrapper around the :class:`SGDClassifier` +class using a perceptron loss and a constant learning rate. Refer to +:ref:`mathematical section ` of the SGD procedure +for more details. + .. _passive_aggressive: Passive Aggressive Algorithms @@ -1138,13 +1373,11 @@ For classification, :class:`PassiveAggressiveClassifier` can be used with ``loss='epsilon_insensitive'`` (PA-I) or ``loss='squared_epsilon_insensitive'`` (PA-II). -.. topic:: References: - - - * `"Online Passive-Aggressive Algorithms" - `_ - K. Crammer, O. Dekel, J. Keshat, S. Shalev-Shwartz, Y. Singer - JMLR 7 (2006) +.. dropdown:: References + * `"Online Passive-Aggressive Algorithms" + `_ + K. Crammer, O. Dekel, J. Keshat, S. Shalev-Shwartz, Y. Singer - JMLR 7 (2006) Robustness regression: outliers and modeling errors ===================================================== @@ -1203,7 +1436,7 @@ Note that in general, robust fitting in high-dimensional setting (large in these settings. -.. topic:: **Trade-offs: which estimator?** +.. topic:: Trade-offs: which estimator ? Scikit-learn provides 3 robust regression estimators: :ref:`RANSAC `, @@ -1212,7 +1445,7 @@ in these settings. * :ref:`HuberRegressor ` should be faster than :ref:`RANSAC ` and :ref:`Theil Sen ` - unless the number of samples are very large, i.e ``n_samples`` >> ``n_features``. + unless the number of samples is very large, i.e. ``n_samples`` >> ``n_features``. This is because :ref:`RANSAC ` and :ref:`Theil Sen ` fit on smaller subsets of the data. However, both :ref:`Theil Sen ` and :ref:`RANSAC ` are unlikely to be as robust as @@ -1228,7 +1461,7 @@ in these settings. medium-size outliers in the X direction, but this property will disappear in high-dimensional settings. - When in doubt, use :ref:`RANSAC `. + When in doubt, use :ref:`RANSAC `. .. _ransac_regression: @@ -1254,50 +1487,48 @@ estimated only from the determined inliers. :align: center :scale: 50% -Details of the algorithm -^^^^^^^^^^^^^^^^^^^^^^^^ - -Each iteration performs the following steps: - -1. Select ``min_samples`` random samples from the original data and check - whether the set of data is valid (see ``is_data_valid``). -2. Fit a model to the random subset (``base_estimator.fit``) and check - whether the estimated model is valid (see ``is_model_valid``). -3. Classify all data as inliers or outliers by calculating the residuals - to the estimated model (``base_estimator.predict(X) - y``) - all data - samples with absolute residuals smaller than the ``residual_threshold`` - are considered as inliers. -4. Save fitted model as best model if number of inlier samples is - maximal. In case the current estimated model has the same number of - inliers, it is only considered as the best model if it has better score. - -These steps are performed either a maximum number of times (``max_trials``) or -until one of the special stop criteria are met (see ``stop_n_inliers`` and -``stop_score``). The final model is estimated using all inlier samples (consensus -set) of the previously determined best model. - -The ``is_data_valid`` and ``is_model_valid`` functions allow to identify and reject -degenerate combinations of random sub-samples. If the estimated model is not -needed for identifying degenerate cases, ``is_data_valid`` should be used as it -is called prior to fitting the model and thus leading to better computational -performance. - - -.. topic:: Examples: - - * :ref:`sphx_glr_auto_examples_linear_model_plot_ransac.py` - * :ref:`sphx_glr_auto_examples_linear_model_plot_robust_fit.py` - -.. topic:: References: - - * https://en.wikipedia.org/wiki/RANSAC - * `"Random Sample Consensus: A Paradigm for Model Fitting with Applications to - Image Analysis and Automated Cartography" - `_ - Martin A. Fischler and Robert C. Bolles - SRI International (1981) - * `"Performance Evaluation of RANSAC Family" - `_ - Sunglok Choi, Taemin Kim and Wonpil Yu - BMVC (2009) +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_linear_model_plot_ransac.py` +* :ref:`sphx_glr_auto_examples_linear_model_plot_robust_fit.py` + +.. dropdown:: Details of the algorithm + + Each iteration performs the following steps: + + 1. Select ``min_samples`` random samples from the original data and check + whether the set of data is valid (see ``is_data_valid``). + 2. Fit a model to the random subset (``estimator.fit``) and check + whether the estimated model is valid (see ``is_model_valid``). + 3. Classify all data as inliers or outliers by calculating the residuals + to the estimated model (``estimator.predict(X) - y``) - all data + samples with absolute residuals smaller than or equal to the + ``residual_threshold`` are considered as inliers. + 4. Save fitted model as best model if number of inlier samples is + maximal. In case the current estimated model has the same number of + inliers, it is only considered as the best model if it has better score. + + These steps are performed either a maximum number of times (``max_trials``) or + until one of the special stop criteria are met (see ``stop_n_inliers`` and + ``stop_score``). The final model is estimated using all inlier samples (consensus + set) of the previously determined best model. + + The ``is_data_valid`` and ``is_model_valid`` functions allow to identify and reject + degenerate combinations of random sub-samples. If the estimated model is not + needed for identifying degenerate cases, ``is_data_valid`` should be used as it + is called prior to fitting the model and thus leading to better computational + performance. + +.. dropdown:: References + + * https://en.wikipedia.org/wiki/RANSAC + * `"Random Sample Consensus: A Paradigm for Model Fitting with Applications to + Image Analysis and Automated Cartography" + `_ + Martin A. Fischler and Robert C. Bolles - SRI International (1981) + * `"Performance Evaluation of RANSAC Family" + `_ + Sunglok Choi, Taemin Kim and Wonpil Yu - BMVC (2009) .. _theil_sen_regression: @@ -1310,67 +1541,62 @@ that the robustness of the estimator decreases quickly with the dimensionality of the problem. It loses its robustness properties and becomes no better than an ordinary least squares in high dimension. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_linear_model_plot_theilsen.py` - * :ref:`sphx_glr_auto_examples_linear_model_plot_robust_fit.py` +* :ref:`sphx_glr_auto_examples_linear_model_plot_theilsen.py` +* :ref:`sphx_glr_auto_examples_linear_model_plot_robust_fit.py` -.. topic:: References: - * https://en.wikipedia.org/wiki/Theil%E2%80%93Sen_estimator +.. dropdown:: Theoretical considerations -Theoretical considerations -^^^^^^^^^^^^^^^^^^^^^^^^^^ + :class:`TheilSenRegressor` is comparable to the :ref:`Ordinary Least Squares + (OLS) ` in terms of asymptotic efficiency and as an + unbiased estimator. In contrast to OLS, Theil-Sen is a non-parametric + method which means it makes no assumption about the underlying + distribution of the data. Since Theil-Sen is a median-based estimator, it + is more robust against corrupted data aka outliers. In univariate + setting, Theil-Sen has a breakdown point of about 29.3% in case of a + simple linear regression which means that it can tolerate arbitrary + corrupted data of up to 29.3%. -:class:`TheilSenRegressor` is comparable to the :ref:`Ordinary Least Squares -(OLS) ` in terms of asymptotic efficiency and as an -unbiased estimator. In contrast to OLS, Theil-Sen is a non-parametric -method which means it makes no assumption about the underlying -distribution of the data. Since Theil-Sen is a median-based estimator, it -is more robust against corrupted data aka outliers. In univariate -setting, Theil-Sen has a breakdown point of about 29.3% in case of a -simple linear regression which means that it can tolerate arbitrary -corrupted data of up to 29.3%. - -.. figure:: ../auto_examples/linear_model/images/sphx_glr_plot_theilsen_001.png - :target: ../auto_examples/linear_model/plot_theilsen.html - :align: center - :scale: 50% + .. figure:: ../auto_examples/linear_model/images/sphx_glr_plot_theilsen_001.png + :target: ../auto_examples/linear_model/plot_theilsen.html + :align: center + :scale: 50% -The implementation of :class:`TheilSenRegressor` in scikit-learn follows a -generalization to a multivariate linear regression model [#f1]_ using the -spatial median which is a generalization of the median to multiple -dimensions [#f2]_. + The implementation of :class:`TheilSenRegressor` in scikit-learn follows a + generalization to a multivariate linear regression model [#f1]_ using the + spatial median which is a generalization of the median to multiple + dimensions [#f2]_. -In terms of time and space complexity, Theil-Sen scales according to + In terms of time and space complexity, Theil-Sen scales according to -.. math:: - \binom{n_{\text{samples}}}{n_{\text{subsamples}}} + .. math:: + \binom{n_{\text{samples}}}{n_{\text{subsamples}}} -which makes it infeasible to be applied exhaustively to problems with a -large number of samples and features. Therefore, the magnitude of a -subpopulation can be chosen to limit the time and space complexity by -considering only a random subset of all possible combinations. + which makes it infeasible to be applied exhaustively to problems with a + large number of samples and features. Therefore, the magnitude of a + subpopulation can be chosen to limit the time and space complexity by + considering only a random subset of all possible combinations. -.. topic:: Examples: + .. rubric:: References - * :ref:`sphx_glr_auto_examples_linear_model_plot_theilsen.py` + .. [#f1] Xin Dang, Hanxiang Peng, Xueqin Wang and Heping Zhang: `Theil-Sen Estimators in a Multiple Linear Regression Model. `_ -.. topic:: References: + .. [#f2] T. Kärkkäinen and S. ÄyrämÃļ: `On Computation of Spatial Median for Robust Data Mining. `_ - .. [#f1] Xin Dang, Hanxiang Peng, Xueqin Wang and Heping Zhang: `Theil-Sen Estimators in a Multiple Linear Regression Model. `_ + Also see the `Wikipedia page `_ - .. [#f2] T. Kärkkäinen and S. ÄyrämÃļ: `On Computation of Spatial Median for Robust Data Mining. `_ .. _huber_regression: Huber Regression ---------------- -The :class:`HuberRegressor` is different to :class:`Ridge` because it applies a -linear loss to samples that are classified as outliers. +The :class:`HuberRegressor` is different from :class:`Ridge` because it applies a +linear loss to samples that are defined as outliers by the `epsilon` parameter. A sample is classified as an inlier if the absolute error of that sample is -lesser than a certain threshold. It differs from :class:`TheilSenRegressor` +less than the threshold `epsilon`. It differs from :class:`TheilSenRegressor` and :class:`RANSACRegressor` because it does not ignore the effect of the outliers but gives a lesser weight to them. @@ -1379,25 +1605,35 @@ but gives a lesser weight to them. :align: center :scale: 50% -The loss function that :class:`HuberRegressor` minimizes is given by +.. rubric:: Examples -.. math:: +* :ref:`sphx_glr_auto_examples_linear_model_plot_huber_vs_ridge.py` - \min_{w, \sigma} {\sum_{i=1}^n\left(\sigma + H_{\epsilon}\left(\frac{X_{i}w - y_{i}}{\sigma}\right)\sigma\right) + \alpha {||w||_2}^2} +.. dropdown:: Mathematical details -where + :class:`HuberRegressor` minimizes -.. math:: + .. math:: - H_{\epsilon}(z) = \begin{cases} - z^2, & \text {if } |z| < \epsilon, \\ - 2\epsilon|z| - \epsilon^2, & \text{otherwise} - \end{cases} + \min_{w, \sigma} {\sum_{i=1}^n\left(\sigma + H_{\epsilon}\left(\frac{X_{i}w - y_{i}}{\sigma}\right)\sigma\right) + \alpha {||w||_2}^2} -It is advised to set the parameter ``epsilon`` to 1.35 to achieve 95% statistical efficiency. + where the loss function is given by + + .. math:: + + H_{\epsilon}(z) = \begin{cases} + z^2, & \text {if } |z| < \epsilon, \\ + 2\epsilon|z| - \epsilon^2, & \text{otherwise} + \end{cases} + + It is advised to set the parameter ``epsilon`` to 1.35 to achieve 95% + statistical efficiency. + + .. rubric:: References + + * Peter J. Huber, Elvezio M. Ronchetti: Robust Statistics, Concomitant scale + estimates, p. 172. -Notes ------ The :class:`HuberRegressor` differs from using :class:`SGDRegressor` with loss set to `huber` in the following ways. @@ -1410,18 +1646,89 @@ in the following ways. samples while :class:`SGDRegressor` needs a number of passes on the training data to produce the same robustness. -.. topic:: Examples: +Note that this estimator is different from the `R implementation of Robust +Regression `_ because the R +implementation does a weighted least squares implementation with weights given to each +sample on the basis of how much the residual is greater than a certain threshold. + +.. _quantile_regression: + +Quantile Regression +=================== + +Quantile regression estimates the median or other quantiles of :math:`y` +conditional on :math:`X`, while ordinary least squares (OLS) estimates the +conditional mean. + +Quantile regression may be useful if one is interested in predicting an +interval instead of point prediction. Sometimes, prediction intervals are +calculated based on the assumption that prediction error is distributed +normally with zero mean and constant variance. Quantile regression provides +sensible prediction intervals even for errors with non-constant (but +predictable) variance or non-normal distribution. + +.. figure:: /auto_examples/linear_model/images/sphx_glr_plot_quantile_regression_002.png + :target: ../auto_examples/linear_model/plot_quantile_regression.html + :align: center + :scale: 50% + +Based on minimizing the pinball loss, conditional quantiles can also be +estimated by models other than linear models. For example, +:class:`~sklearn.ensemble.GradientBoostingRegressor` can predict conditional +quantiles if its parameter ``loss`` is set to ``"quantile"`` and parameter +``alpha`` is set to the quantile that should be predicted. See the example in +:ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_quantile.py`. + +Most implementations of quantile regression are based on linear programming +problem. The current implementation is based on +:func:`scipy.optimize.linprog`. + +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_linear_model_plot_quantile_regression.py` + +.. dropdown:: Mathematical details + + As a linear model, the :class:`QuantileRegressor` gives linear predictions + :math:`\hat{y}(w, X) = Xw` for the :math:`q`-th quantile, :math:`q \in (0, 1)`. + The weights or coefficients :math:`w` are then found by the following + minimization problem: - * :ref:`sphx_glr_auto_examples_linear_model_plot_huber_vs_ridge.py` + .. math:: + \min_{w} {\frac{1}{n_{\text{samples}}} + \sum_i PB_q(y_i - X_i w) + \alpha ||w||_1}. -.. topic:: References: + This consists of the pinball loss (also known as linear loss), + see also :class:`~sklearn.metrics.mean_pinball_loss`, - * Peter J. Huber, Elvezio M. Ronchetti: Robust Statistics, Concomitant scale estimates, pg 172 + .. math:: + PB_q(t) = q \max(t, 0) + (1 - q) \max(-t, 0) = + \begin{cases} + q t, & t > 0, \\ + 0, & t = 0, \\ + (q-1) t, & t < 0 + \end{cases} + + and the L1 penalty controlled by parameter ``alpha``, similar to + :class:`Lasso`. + + As the pinball loss is only linear in the residuals, quantile regression is + much more robust to outliers than squared error based estimation of the mean. + Somewhat in between is the :class:`HuberRegressor`. + +.. dropdown:: References + + * Koenker, R., & Bassett Jr, G. (1978). `Regression quantiles. + `_ + Econometrica: journal of the Econometric Society, 33-50. + + * Portnoy, S., & Koenker, R. (1997). :doi:`The Gaussian hare and the Laplacian + tortoise: computability of squared-error versus absolute-error estimators. + Statistical Science, 12, 279-300 <10.1214/ss/1030037960>`. + + * Koenker, R. (2005). :doi:`Quantile Regression <10.1017/CBO9780511754098>`. + Cambridge University Press. -Note that this estimator is different from the R implementation of Robust Regression -(http://www.ats.ucla.edu/stat/r/dae/rreg.htm) because the R implementation does a weighted least -squares implementation with weights given to each sample on the basis of how much the residual is -greater than a certain threshold. .. _polynomial_regression: @@ -1435,32 +1742,34 @@ on nonlinear functions of the data. This approach maintains the generally fast performance of linear methods, while allowing them to fit a much wider range of data. -For example, a simple linear regression can be extended by constructing -**polynomial features** from the coefficients. In the standard linear -regression case, you might have a model that looks like this for -two-dimensional data: +.. dropdown:: Mathematical details + + For example, a simple linear regression can be extended by constructing + **polynomial features** from the coefficients. In the standard linear + regression case, you might have a model that looks like this for + two-dimensional data: -.. math:: \hat{y}(w, x) = w_0 + w_1 x_1 + w_2 x_2 + .. math:: \hat{y}(w, x) = w_0 + w_1 x_1 + w_2 x_2 -If we want to fit a paraboloid to the data instead of a plane, we can combine -the features in second-order polynomials, so that the model looks like this: + If we want to fit a paraboloid to the data instead of a plane, we can combine + the features in second-order polynomials, so that the model looks like this: -.. math:: \hat{y}(w, x) = w_0 + w_1 x_1 + w_2 x_2 + w_3 x_1 x_2 + w_4 x_1^2 + w_5 x_2^2 + .. math:: \hat{y}(w, x) = w_0 + w_1 x_1 + w_2 x_2 + w_3 x_1 x_2 + w_4 x_1^2 + w_5 x_2^2 -The (sometimes surprising) observation is that this is *still a linear model*: -to see this, imagine creating a new set of features + The (sometimes surprising) observation is that this is *still a linear model*: + to see this, imagine creating a new set of features -.. math:: z = [x_1, x_2, x_1 x_2, x_1^2, x_2^2] + .. math:: z = [x_1, x_2, x_1 x_2, x_1^2, x_2^2] -With this re-labeling of the data, our problem can be written + With this re-labeling of the data, our problem can be written -.. math:: \hat{y}(w, z) = w_0 + w_1 z_1 + w_2 z_2 + w_3 z_3 + w_4 z_4 + w_5 z_5 + .. math:: \hat{y}(w, z) = w_0 + w_1 z_1 + w_2 z_2 + w_3 z_3 + w_4 z_4 + w_5 z_5 -We see that the resulting *polynomial regression* is in the same class of -linear models we considered above (i.e. the model is linear in :math:`w`) -and can be solved by the same techniques. By considering linear fits within -a higher-dimensional space built with these basis functions, the model has the -flexibility to fit a much broader range of data. + We see that the resulting *polynomial regression* is in the same class of + linear models we considered above (i.e. the model is linear in :math:`w`) + and can be solved by the same techniques. By considering linear fits within + a higher-dimensional space built with these basis functions, the model has the + flexibility to fit a much broader range of data. Here is an example of applying this idea to one-dimensional data, using polynomial features of varying degrees: diff --git a/doc/modules/manifold.rst b/doc/modules/manifold.rst index 8de2a73477c87..aec992a8f9dc1 100644 --- a/doc/modules/manifold.rst +++ b/doc/modules/manifold.rst @@ -7,23 +7,40 @@ Manifold learning ================= -.. rst-class:: quote - - | Look for the bare necessities - | The simple bare necessities - | Forget about your worries and your strife - | I mean the bare necessities - | Old Mother Nature's recipes - | That bring the bare necessities of life - | - | -- Baloo's song [The Jungle Book] +| Look for the bare necessities +| The simple bare necessities +| Forget about your worries and your strife +| I mean the bare necessities +| Old Mother Nature's recipes +| That bring the bare necessities of life +| +| -- Baloo's song [The Jungle Book] .. figure:: ../auto_examples/manifold/images/sphx_glr_plot_compare_methods_001.png :target: ../auto_examples/manifold/plot_compare_methods.html :align: center - :scale: 60 + :scale: 70% + +.. |manifold_img3| image:: ../auto_examples/manifold/images/sphx_glr_plot_compare_methods_003.png + :target: ../auto_examples/manifold/plot_compare_methods.html + :scale: 60% + +.. |manifold_img4| image:: ../auto_examples/manifold/images/sphx_glr_plot_compare_methods_004.png + :target: ../auto_examples/manifold/plot_compare_methods.html + :scale: 60% + +.. |manifold_img5| image:: ../auto_examples/manifold/images/sphx_glr_plot_compare_methods_005.png + :target: ../auto_examples/manifold/plot_compare_methods.html + :scale: 60% + +.. |manifold_img6| image:: ../auto_examples/manifold/images/sphx_glr_plot_compare_methods_006.png + :target: ../auto_examples/manifold/plot_compare_methods.html + :scale: 60% + +.. centered:: |manifold_img3| |manifold_img4| |manifold_img5| |manifold_img6| + Manifold learning is an approach to non-linear dimensionality reduction. Algorithms for this task are based on the idea that the dimensionality of @@ -83,13 +100,23 @@ unsupervised: it learns the high-dimensional structure of the data from the data itself, without the use of predetermined classifications. -.. topic:: Examples: +.. rubric:: Examples + +* See :ref:`sphx_glr_auto_examples_manifold_plot_lle_digits.py` for an example of + dimensionality reduction on handwritten digits. + +* See :ref:`sphx_glr_auto_examples_manifold_plot_compare_methods.py` for an example of + dimensionality reduction on a toy "S-curve" dataset. - * See :ref:`sphx_glr_auto_examples_manifold_plot_lle_digits.py` for an example of - dimensionality reduction on handwritten digits. +* See :ref:`sphx_glr_auto_examples_applications_plot_stock_market.py` for an example of + using manifold learning to map the stock market structure based on historical stock + prices. - * See :ref:`sphx_glr_auto_examples_manifold_plot_compare_methods.py` for an example of - dimensionality reduction on a toy "S-curve" dataset. +* See :ref:`sphx_glr_auto_examples_manifold_plot_manifold_sphere.py` for an example of + manifold learning techniques applied to a spherical data-set. + +* See :ref:`sphx_glr_auto_examples_manifold_plot_swissroll.py` for an example of using + manifold learning techniques on a Swiss Roll dataset. The manifold learning implementations available in scikit-learn are summarized below @@ -111,43 +138,43 @@ distances between all points. Isomap can be performed with the object :align: center :scale: 50 -Complexity ----------- -The Isomap algorithm comprises three stages: +.. dropdown:: Complexity + + The Isomap algorithm comprises three stages: -1. **Nearest neighbor search.** Isomap uses - :class:`~sklearn.neighbors.BallTree` for efficient neighbor search. - The cost is approximately :math:`O[D \log(k) N \log(N)]`, for :math:`k` - nearest neighbors of :math:`N` points in :math:`D` dimensions. + 1. **Nearest neighbor search.** Isomap uses + :class:`~sklearn.neighbors.BallTree` for efficient neighbor search. + The cost is approximately :math:`O[D \log(k) N \log(N)]`, for :math:`k` + nearest neighbors of :math:`N` points in :math:`D` dimensions. -2. **Shortest-path graph search.** The most efficient known algorithms - for this are *Dijkstra's Algorithm*, which is approximately - :math:`O[N^2(k + \log(N))]`, or the *Floyd-Warshall algorithm*, which - is :math:`O[N^3]`. The algorithm can be selected by the user with - the ``path_method`` keyword of ``Isomap``. If unspecified, the code - attempts to choose the best algorithm for the input data. + 2. **Shortest-path graph search.** The most efficient known algorithms + for this are *Dijkstra's Algorithm*, which is approximately + :math:`O[N^2(k + \log(N))]`, or the *Floyd-Warshall algorithm*, which + is :math:`O[N^3]`. The algorithm can be selected by the user with + the ``path_method`` keyword of ``Isomap``. If unspecified, the code + attempts to choose the best algorithm for the input data. -3. **Partial eigenvalue decomposition.** The embedding is encoded in the - eigenvectors corresponding to the :math:`d` largest eigenvalues of the - :math:`N \times N` isomap kernel. For a dense solver, the cost is - approximately :math:`O[d N^2]`. This cost can often be improved using - the ``ARPACK`` solver. The eigensolver can be specified by the user - with the ``eigen_solver`` keyword of ``Isomap``. If unspecified, the - code attempts to choose the best algorithm for the input data. + 3. **Partial eigenvalue decomposition.** The embedding is encoded in the + eigenvectors corresponding to the :math:`d` largest eigenvalues of the + :math:`N \times N` isomap kernel. For a dense solver, the cost is + approximately :math:`O[d N^2]`. This cost can often be improved using + the ``ARPACK`` solver. The eigensolver can be specified by the user + with the ``eigen_solver`` keyword of ``Isomap``. If unspecified, the + code attempts to choose the best algorithm for the input data. -The overall complexity of Isomap is -:math:`O[D \log(k) N \log(N)] + O[N^2(k + \log(N))] + O[d N^2]`. + The overall complexity of Isomap is + :math:`O[D \log(k) N \log(N)] + O[N^2(k + \log(N))] + O[d N^2]`. -* :math:`N` : number of training data points -* :math:`D` : input dimension -* :math:`k` : number of nearest neighbors -* :math:`d` : output dimension + * :math:`N` : number of training data points + * :math:`D` : input dimension + * :math:`k` : number of nearest neighbors + * :math:`d` : output dimension -.. topic:: References: +.. rubric:: References - * `"A global geometric framework for nonlinear dimensionality reduction" - `_ - Tenenbaum, J.B.; De Silva, V.; & Langford, J.C. Science 290 (5500) +* `"A global geometric framework for nonlinear dimensionality reduction" + `_ + Tenenbaum, J.B.; De Silva, V.; & Langford, J.C. Science 290 (5500) .. _locally_linear_embedding: @@ -168,33 +195,32 @@ Locally linear embedding can be performed with function :align: center :scale: 50 -Complexity ----------- +.. dropdown:: Complexity -The standard LLE algorithm comprises three stages: + The standard LLE algorithm comprises three stages: -1. **Nearest Neighbors Search**. See discussion under Isomap above. + 1. **Nearest Neighbors Search**. See discussion under Isomap above. -2. **Weight Matrix Construction**. :math:`O[D N k^3]`. - The construction of the LLE weight matrix involves the solution of a - :math:`k \times k` linear equation for each of the :math:`N` local - neighborhoods + 2. **Weight Matrix Construction**. :math:`O[D N k^3]`. + The construction of the LLE weight matrix involves the solution of a + :math:`k \times k` linear equation for each of the :math:`N` local + neighborhoods. -3. **Partial Eigenvalue Decomposition**. See discussion under Isomap above. + 3. **Partial Eigenvalue Decomposition**. See discussion under Isomap above. -The overall complexity of standard LLE is -:math:`O[D \log(k) N \log(N)] + O[D N k^3] + O[d N^2]`. + The overall complexity of standard LLE is + :math:`O[D \log(k) N \log(N)] + O[D N k^3] + O[d N^2]`. -* :math:`N` : number of training data points -* :math:`D` : input dimension -* :math:`k` : number of nearest neighbors -* :math:`d` : output dimension + * :math:`N` : number of training data points + * :math:`D` : input dimension + * :math:`k` : number of nearest neighbors + * :math:`d` : output dimension -.. topic:: References: +.. rubric:: References - * `"Nonlinear dimensionality reduction by locally linear embedding" - `_ - Roweis, S. & Saul, L. Science 290:2323 (2000) +* `"Nonlinear dimensionality reduction by locally linear embedding" + `_ + Roweis, S. & Saul, L. Science 290:2323 (2000) Modified Locally Linear Embedding @@ -222,35 +248,34 @@ It requires ``n_neighbors > n_components``. :align: center :scale: 50 -Complexity ----------- +.. dropdown:: Complexity -The MLLE algorithm comprises three stages: + The MLLE algorithm comprises three stages: -1. **Nearest Neighbors Search**. Same as standard LLE + 1. **Nearest Neighbors Search**. Same as standard LLE -2. **Weight Matrix Construction**. Approximately - :math:`O[D N k^3] + O[N (k-D) k^2]`. The first term is exactly equivalent - to that of standard LLE. The second term has to do with constructing the - weight matrix from multiple weights. In practice, the added cost of - constructing the MLLE weight matrix is relatively small compared to the - cost of stages 1 and 3. + 2. **Weight Matrix Construction**. Approximately + :math:`O[D N k^3] + O[N (k-D) k^2]`. The first term is exactly equivalent + to that of standard LLE. The second term has to do with constructing the + weight matrix from multiple weights. In practice, the added cost of + constructing the MLLE weight matrix is relatively small compared to the + cost of stages 1 and 3. -3. **Partial Eigenvalue Decomposition**. Same as standard LLE + 3. **Partial Eigenvalue Decomposition**. Same as standard LLE -The overall complexity of MLLE is -:math:`O[D \log(k) N \log(N)] + O[D N k^3] + O[N (k-D) k^2] + O[d N^2]`. + The overall complexity of MLLE is + :math:`O[D \log(k) N \log(N)] + O[D N k^3] + O[N (k-D) k^2] + O[d N^2]`. -* :math:`N` : number of training data points -* :math:`D` : input dimension -* :math:`k` : number of nearest neighbors -* :math:`d` : output dimension + * :math:`N` : number of training data points + * :math:`D` : input dimension + * :math:`k` : number of nearest neighbors + * :math:`d` : output dimension -.. topic:: References: +.. rubric:: References - * `"MLLE: Modified Locally Linear Embedding Using Multiple Weights" - `_ - Zhang, Z. & Wang, J. +* `"MLLE: Modified Locally Linear Embedding Using Multiple Weights" + `_ + Zhang, Z. & Wang, J. Hessian Eigenmapping @@ -272,33 +297,32 @@ It requires ``n_neighbors > n_components * (n_components + 3) / 2``. :align: center :scale: 50 -Complexity ----------- +.. dropdown:: Complexity -The HLLE algorithm comprises three stages: + The HLLE algorithm comprises three stages: -1. **Nearest Neighbors Search**. Same as standard LLE + 1. **Nearest Neighbors Search**. Same as standard LLE -2. **Weight Matrix Construction**. Approximately - :math:`O[D N k^3] + O[N d^6]`. The first term reflects a similar - cost to that of standard LLE. The second term comes from a QR - decomposition of the local hessian estimator. + 2. **Weight Matrix Construction**. Approximately + :math:`O[D N k^3] + O[N d^6]`. The first term reflects a similar + cost to that of standard LLE. The second term comes from a QR + decomposition of the local hessian estimator. -3. **Partial Eigenvalue Decomposition**. Same as standard LLE + 3. **Partial Eigenvalue Decomposition**. Same as standard LLE. -The overall complexity of standard HLLE is -:math:`O[D \log(k) N \log(N)] + O[D N k^3] + O[N d^6] + O[d N^2]`. + The overall complexity of standard HLLE is + :math:`O[D \log(k) N \log(N)] + O[D N k^3] + O[N d^6] + O[d N^2]`. -* :math:`N` : number of training data points -* :math:`D` : input dimension -* :math:`k` : number of nearest neighbors -* :math:`d` : output dimension + * :math:`N` : number of training data points + * :math:`D` : input dimension + * :math:`k` : number of nearest neighbors + * :math:`d` : output dimension -.. topic:: References: +.. rubric:: References - * `"Hessian Eigenmaps: Locally linear embedding techniques for - high-dimensional data" `_ - Donoho, D. & Grimes, C. Proc Natl Acad Sci USA. 100:5591 (2003) +* `"Hessian Eigenmaps: Locally linear embedding techniques for + high-dimensional data" `_ + Donoho, D. & Grimes, C. Proc Natl Acad Sci USA. 100:5591 (2003) .. _spectral_embedding: @@ -316,35 +340,34 @@ preserving local distances. Spectral embedding can be performed with the function :func:`spectral_embedding` or its object-oriented counterpart :class:`SpectralEmbedding`. -Complexity ----------- +.. dropdown:: Complexity -The Spectral Embedding (Laplacian Eigenmaps) algorithm comprises three stages: + The Spectral Embedding (Laplacian Eigenmaps) algorithm comprises three stages: -1. **Weighted Graph Construction**. Transform the raw input data into - graph representation using affinity (adjacency) matrix representation. + 1. **Weighted Graph Construction**. Transform the raw input data into + graph representation using affinity (adjacency) matrix representation. -2. **Graph Laplacian Construction**. unnormalized Graph Laplacian - is constructed as :math:`L = D - A` for and normalized one as - :math:`L = D^{-\frac{1}{2}} (D - A) D^{-\frac{1}{2}}`. + 2. **Graph Laplacian Construction**. unnormalized Graph Laplacian + is constructed as :math:`L = D - A` for and normalized one as + :math:`L = D^{-\frac{1}{2}} (D - A) D^{-\frac{1}{2}}`. -3. **Partial Eigenvalue Decomposition**. Eigenvalue decomposition is - done on graph Laplacian + 3. **Partial Eigenvalue Decomposition**. Eigenvalue decomposition is + done on graph Laplacian. -The overall complexity of spectral embedding is -:math:`O[D \log(k) N \log(N)] + O[D N k^3] + O[d N^2]`. + The overall complexity of spectral embedding is + :math:`O[D \log(k) N \log(N)] + O[D N k^3] + O[d N^2]`. -* :math:`N` : number of training data points -* :math:`D` : input dimension -* :math:`k` : number of nearest neighbors -* :math:`d` : output dimension + * :math:`N` : number of training data points + * :math:`D` : input dimension + * :math:`k` : number of nearest neighbors + * :math:`d` : output dimension -.. topic:: References: +.. rubric:: References - * `"Laplacian Eigenmaps for Dimensionality Reduction - and Data Representation" - `_ - M. Belkin, P. Niyogi, Neural Computation, June 2003; 15 (6):1373-1396 +* `"Laplacian Eigenmaps for Dimensionality Reduction + and Data Representation" + `_ + M. Belkin, P. Niyogi, Neural Computation, June 2003; 15 (6):1373-1396 Local Tangent Space Alignment @@ -364,33 +387,32 @@ tangent spaces to learn the embedding. LTSA can be performed with function :align: center :scale: 50 -Complexity ----------- +.. dropdown:: Complexity -The LTSA algorithm comprises three stages: + The LTSA algorithm comprises three stages: -1. **Nearest Neighbors Search**. Same as standard LLE + 1. **Nearest Neighbors Search**. Same as standard LLE -2. **Weight Matrix Construction**. Approximately - :math:`O[D N k^3] + O[k^2 d]`. The first term reflects a similar - cost to that of standard LLE. + 2. **Weight Matrix Construction**. Approximately + :math:`O[D N k^3] + O[k^2 d]`. The first term reflects a similar + cost to that of standard LLE. -3. **Partial Eigenvalue Decomposition**. Same as standard LLE + 3. **Partial Eigenvalue Decomposition**. Same as standard LLE -The overall complexity of standard LTSA is -:math:`O[D \log(k) N \log(N)] + O[D N k^3] + O[k^2 d] + O[d N^2]`. + The overall complexity of standard LTSA is + :math:`O[D \log(k) N \log(N)] + O[D N k^3] + O[k^2 d] + O[d N^2]`. -* :math:`N` : number of training data points -* :math:`D` : input dimension -* :math:`k` : number of nearest neighbors -* :math:`d` : output dimension + * :math:`N` : number of training data points + * :math:`D` : input dimension + * :math:`k` : number of nearest neighbors + * :math:`d` : output dimension -.. topic:: References: +.. rubric:: References - * `"Principal manifolds and nonlinear dimensionality reduction via - tangent space alignment" - `_ - Zhang, Z. & Zha, H. Journal of Shanghai Univ. 8:406 (2004) +* :arxiv:`"Principal manifolds and nonlinear dimensionality reduction via + tangent space alignment" + ` + Zhang, Z. & Zha, H. Journal of Shanghai Univ. 8:406 (2004) .. _multidimensional_scaling: @@ -402,20 +424,19 @@ Multi-dimensional Scaling (MDS) representation of the data in which the distances respect well the distances in the original high-dimensional space. -In general, :class:`MDS` is a technique used for analyzing similarity or -dissimilarity data. It attempts to model similarity or dissimilarity data as -distances in a geometric spaces. The data can be ratings of similarity between +In general, :class:`MDS` is a technique used for analyzing +dissimilarity data. It attempts to model dissimilarities as +distances in a Euclidean space. The data can be ratings of dissimilarity between objects, interaction frequencies of molecules, or trade indices between countries. -There exists two types of MDS algorithm: metric and non metric. In the -scikit-learn, the class :class:`MDS` implements both. In Metric MDS, the input -similarity matrix arises from a metric (and thus respects the triangular -inequality), the distances between output two points are then set to be as -close as possible to the similarity or dissimilarity data. In the non-metric -version, the algorithms will try to preserve the order of the distances, and +There exist two types of MDS algorithm: metric and non-metric. In +scikit-learn, the class :class:`MDS` implements both. In metric MDS, +the distances in the embedding space are set as +close as possible to the dissimilarity data. In the non-metric +version, the algorithm will try to preserve the order of the distances, and hence seek for a monotonic relationship between the distances in the embedded -space and the similarities/dissimilarities. +space and the input dissimilarities. .. figure:: ../auto_examples/manifold/images/sphx_glr_plot_lle_digits_010.png :target: ../auto_examples/manifold/plot_lle_digits.html @@ -423,54 +444,68 @@ space and the similarities/dissimilarities. :scale: 50 -Let :math:`S` be the similarity matrix, and :math:`X` the coordinates of the -:math:`n` input points. Disparities :math:`\hat{d}_{ij}` are transformation of -the similarities chosen in some optimal ways. The objective, called the -stress, is then defined by :math:`\sum_{i < j} d_{ij}(X) - \hat{d}_{ij}(X)` +Let :math:`\delta_{ij}` be the dissimilarity matrix between the +:math:`n` input points (possibly arising as some pairwise distances +:math:`d_{ij}(X)` between the coordinates :math:`X` of the input points). +Disparities :math:`\hat{d}_{ij} = f(\delta_{ij})` are some transformation of +the dissimilarities. The MDS objective, called the raw stress, is then +defined by :math:`\sum_{i < j} (\hat{d}_{ij} - d_{ij}(Z))^2`, +where :math:`d_{ij}(Z)` are the pairwise distances between the +coordinates :math:`Z` of the embedded points. -Metric MDS ----------- +.. dropdown:: Metric MDS -The simplest metric :class:`MDS` model, called *absolute MDS*, disparities are defined by -:math:`\hat{d}_{ij} = S_{ij}`. With absolute MDS, the value :math:`S_{ij}` -should then correspond exactly to the distance between point :math:`i` and -:math:`j` in the embedding point. + In the metric :class:`MDS` model (sometimes also called *absolute MDS*), + disparities are simply equal to the input dissimilarities + :math:`\hat{d}_{ij} = \delta_{ij}`. -Most commonly, disparities are set to :math:`\hat{d}_{ij} = b S_{ij}`. +.. dropdown:: Nonmetric MDS -Nonmetric MDS -------------- + Non metric :class:`MDS` focuses on the ordination of the data. If + :math:`\delta_{ij} > \delta_{kl}`, then the embedding + seeks to enforce :math:`d_{ij}(Z) > d_{kl}(Z)`. A simple algorithm + to enforce proper ordination is to use an + isotonic regression of :math:`d_{ij}(Z)` on :math:`\delta_{ij}`, yielding + disparities :math:`\hat{d}_{ij}` that are a monotonic transformation + of dissimilarities :math:`\delta_{ij}` and hence having the same ordering. + This is done repeatedly after every step of the optimization algorithm. + In order to avoid the trivial solution where all embedding points are + overlapping, the disparities :math:`\hat{d}_{ij}` are normalized. -Non metric :class:`MDS` focuses on the ordination of the data. If -:math:`S_{ij} < S_{jk}`, then the embedding should enforce :math:`d_{ij} < -d_{jk}`. A simple algorithm to enforce that is to use a monotonic regression -of :math:`d_{ij}` on :math:`S_{ij}`, yielding disparities :math:`\hat{d}_{ij}` -in the same order as :math:`S_{ij}`. + Note that since we only care about relative ordering, our objective should be + invariant to simple translation and scaling, however the stress used in metric + MDS is sensitive to scaling. To address this, non-metric MDS returns + normalized stress, also known as Stress-1, defined as -A trivial solution to this problem is to set all the points on the origin. In -order to avoid that, the disparities :math:`\hat{d}_{ij}` are normalized. + .. math:: + \sqrt{\frac{\sum_{i < j} (\hat{d}_{ij} - d_{ij}(Z))^2}{\sum_{i < j} + d_{ij}(Z)^2}}. + Normalized Stress-1 is returned if `normalized_stress=True`. -.. figure:: ../auto_examples/manifold/images/sphx_glr_plot_mds_001.png - :target: ../auto_examples/manifold/plot_mds.html - :align: center - :scale: 60 + .. figure:: ../auto_examples/manifold/images/sphx_glr_plot_mds_001.png + :target: ../auto_examples/manifold/plot_mds.html + :align: center + :scale: 60 +.. rubric:: References -.. topic:: References: +* `"More on Multidimensional Scaling and Unfolding in R: smacof Version 2" + `_ + Mair P, Groenen P., de Leeuw J. Journal of Statistical Software (2022) - * `"Modern Multidimensional Scaling - Theory and Applications" - `_ - Borg, I.; Groenen P. Springer Series in Statistics (1997) +* `"Modern Multidimensional Scaling - Theory and Applications" + `_ + Borg, I.; Groenen P. Springer Series in Statistics (1997) - * `"Nonmetric multidimensional scaling: a numerical method" - `_ - Kruskal, J. Psychometrika, 29 (1964) +* `"Nonmetric multidimensional scaling: a numerical method" + `_ + Kruskal, J. Psychometrika, 29 (1964) - * `"Multidimensional scaling by optimizing goodness of fit to a nonmetric hypothesis" - `_ - Kruskal, J. Psychometrika, 29, (1964) +* `"Multidimensional scaling by optimizing goodness of fit to a nonmetric hypothesis" + `_ + Kruskal, J. Psychometrika, 29, (1964) .. _t_sne: @@ -518,105 +553,110 @@ The disadvantages to using t-SNE are roughly: :align: center :scale: 50 -Optimizing t-SNE ----------------- -The main purpose of t-SNE is visualization of high-dimensional data. Hence, -it works best when the data will be embedded on two or three dimensions. - -Optimizing the KL divergence can be a little bit tricky sometimes. There are -five parameters that control the optimization of t-SNE and therefore possibly -the quality of the resulting embedding: - -* perplexity -* early exaggeration factor -* learning rate -* maximum number of iterations -* angle (not used in the exact method) - -The perplexity is defined as :math:`k=2^{(S)}` where :math:`S` is the Shannon -entropy of the conditional probability distribution. The perplexity of a -:math:`k`-sided die is :math:`k`, so that :math:`k` is effectively the number of -nearest neighbors t-SNE considers when generating the conditional probabilities. -Larger perplexities lead to more nearest neighbors and less sensitive to small -structure. Conversely a lower perplexity considers a smaller number of -neighbors, and thus ignores more global information in favour of the -local neighborhood. As dataset sizes get larger more points will be -required to get a reasonable sample of the local neighborhood, and hence -larger perplexities may be required. Similarly noisier datasets will require -larger perplexity values to encompass enough local neighbors to see beyond -the background noise. - -The maximum number of iterations is usually high enough and does not need -any tuning. The optimization consists of two phases: the early exaggeration -phase and the final optimization. During early exaggeration the joint -probabilities in the original space will be artificially increased by -multiplication with a given factor. Larger factors result in larger gaps -between natural clusters in the data. If the factor is too high, the KL -divergence could increase during this phase. Usually it does not have to be -tuned. A critical parameter is the learning rate. If it is too low gradient -descent will get stuck in a bad local minimum. If it is too high the KL -divergence will increase during optimization. More tips can be found in -Laurens van der Maaten's FAQ (see references). The last parameter, angle, -is a tradeoff between performance and accuracy. Larger angles imply that we -can approximate larger regions by a single point, leading to better speed -but less accurate results. - -`"How to Use t-SNE Effectively" `_ -provides a good discussion of the effects of the various parameters, as well -as interactive plots to explore the effects of different parameters. - -Barnes-Hut t-SNE ----------------- - -The Barnes-Hut t-SNE that has been implemented here is usually much slower than -other manifold learning algorithms. The optimization is quite difficult -and the computation of the gradient is :math:`O[d N log(N)]`, where :math:`d` -is the number of output dimensions and :math:`N` is the number of samples. The -Barnes-Hut method improves on the exact method where t-SNE complexity is -:math:`O[d N^2]`, but has several other notable differences: - -* The Barnes-Hut implementation only works when the target dimensionality is 3 - or less. The 2D case is typical when building visualizations. -* Barnes-Hut only works with dense input data. Sparse data matrices can only be - embedded with the exact method or can be approximated by a dense low rank - projection for instance using :class:`~sklearn.decomposition.TruncatedSVD` -* Barnes-Hut is an approximation of the exact method. The approximation is - parameterized with the angle parameter, therefore the angle parameter is - unused when method="exact" -* Barnes-Hut is significantly more scalable. Barnes-Hut can be used to embed - hundred of thousands of data points while the exact method can handle - thousands of samples before becoming computationally intractable - -For visualization purpose (which is the main use case of t-SNE), using the -Barnes-Hut method is strongly recommended. The exact t-SNE method is useful -for checking the theoretically properties of the embedding possibly in higher -dimensional space but limit to small datasets due to computational constraints. - -Also note that the digits labels roughly match the natural grouping found by -t-SNE while the linear 2D projection of the PCA model yields a representation -where label regions largely overlap. This is a strong clue that this data can -be well separated by non linear methods that focus on the local structure (e.g. -an SVM with a Gaussian RBF kernel). However, failing to visualize well -separated homogeneously labeled groups with t-SNE in 2D does not necessarily -imply that the data cannot be correctly classified by a supervised model. It -might be the case that 2 dimensions are not low enough to accurately represents -the internal structure of the data. - - -.. topic:: References: - - * `"Visualizing High-Dimensional Data Using t-SNE" - `_ - van der Maaten, L.J.P.; Hinton, G. Journal of Machine Learning Research - (2008) - - * `"t-Distributed Stochastic Neighbor Embedding" - `_ - van der Maaten, L.J.P. - - * `"Accelerating t-SNE using Tree-Based Algorithms." - `_ - L.J.P. van der Maaten. Journal of Machine Learning Research 15(Oct):3221-3245, 2014. +.. dropdown:: Optimizing t-SNE + + The main purpose of t-SNE is visualization of high-dimensional data. Hence, + it works best when the data will be embedded on two or three dimensions. + + Optimizing the KL divergence can be a little bit tricky sometimes. There are + five parameters that control the optimization of t-SNE and therefore possibly + the quality of the resulting embedding: + + * perplexity + * early exaggeration factor + * learning rate + * maximum number of iterations + * angle (not used in the exact method) + + The perplexity is defined as :math:`k=2^{(S)}` where :math:`S` is the Shannon + entropy of the conditional probability distribution. The perplexity of a + :math:`k`-sided die is :math:`k`, so that :math:`k` is effectively the number of + nearest neighbors t-SNE considers when generating the conditional probabilities. + Larger perplexities lead to more nearest neighbors and less sensitive to small + structure. Conversely a lower perplexity considers a smaller number of + neighbors, and thus ignores more global information in favour of the + local neighborhood. As dataset sizes get larger more points will be + required to get a reasonable sample of the local neighborhood, and hence + larger perplexities may be required. Similarly noisier datasets will require + larger perplexity values to encompass enough local neighbors to see beyond + the background noise. + + The maximum number of iterations is usually high enough and does not need + any tuning. The optimization consists of two phases: the early exaggeration + phase and the final optimization. During early exaggeration the joint + probabilities in the original space will be artificially increased by + multiplication with a given factor. Larger factors result in larger gaps + between natural clusters in the data. If the factor is too high, the KL + divergence could increase during this phase. Usually it does not have to be + tuned. A critical parameter is the learning rate. If it is too low gradient + descent will get stuck in a bad local minimum. If it is too high the KL + divergence will increase during optimization. A heuristic suggested in + Belkina et al. (2019) is to set the learning rate to the sample size + divided by the early exaggeration factor. We implement this heuristic + as `learning_rate='auto'` argument. More tips can be found in + Laurens van der Maaten's FAQ (see references). The last parameter, angle, + is a tradeoff between performance and accuracy. Larger angles imply that we + can approximate larger regions by a single point, leading to better speed + but less accurate results. + + `"How to Use t-SNE Effectively" `_ + provides a good discussion of the effects of the various parameters, as well + as interactive plots to explore the effects of different parameters. + +.. dropdown:: Barnes-Hut t-SNE + + The Barnes-Hut t-SNE that has been implemented here is usually much slower than + other manifold learning algorithms. The optimization is quite difficult + and the computation of the gradient is :math:`O[d N log(N)]`, where :math:`d` + is the number of output dimensions and :math:`N` is the number of samples. The + Barnes-Hut method improves on the exact method where t-SNE complexity is + :math:`O[d N^2]`, but has several other notable differences: + + * The Barnes-Hut implementation only works when the target dimensionality is 3 + or less. The 2D case is typical when building visualizations. + * Barnes-Hut only works with dense input data. Sparse data matrices can only be + embedded with the exact method or can be approximated by a dense low rank + projection for instance using :class:`~sklearn.decomposition.PCA` + * Barnes-Hut is an approximation of the exact method. The approximation is + parameterized with the angle parameter, therefore the angle parameter is + unused when method="exact" + * Barnes-Hut is significantly more scalable. Barnes-Hut can be used to embed + hundreds of thousands of data points while the exact method can handle + thousands of samples before becoming computationally intractable + + For visualization purpose (which is the main use case of t-SNE), using the + Barnes-Hut method is strongly recommended. The exact t-SNE method is useful + for checking the theoretical properties of the embedding possibly in higher + dimensional space but limited to small datasets due to computational constraints. + + Also note that the digits labels roughly match the natural grouping found by + t-SNE while the linear 2D projection of the PCA model yields a representation + where label regions largely overlap. This is a strong clue that this data can + be well separated by non linear methods that focus on the local structure (e.g. + an SVM with a Gaussian RBF kernel). However, failing to visualize well + separated homogeneously labeled groups with t-SNE in 2D does not necessarily + imply that the data cannot be correctly classified by a supervised model. It + might be the case that 2 dimensions are not high enough to accurately represent + the internal structure of the data. + +.. rubric:: References + +* `"Visualizing High-Dimensional Data Using t-SNE" + `_ + van der Maaten, L.J.P.; Hinton, G. Journal of Machine Learning Research (2008) + +* `"t-Distributed Stochastic Neighbor Embedding" + `_ van der Maaten, L.J.P. + +* `"Accelerating t-SNE using Tree-Based Algorithms" + `_ + van der Maaten, L.J.P.; Journal of Machine Learning Research 15(Oct):3221-3245, 2014. + +* `"Automated optimized parameters for T-distributed stochastic neighbor + embedding improve visualization and analysis of large datasets" + `_ + Belkina, A.C., Ciccolella, C.O., Anno, R., Halpert, R., Spidlen, J., + Snyder-Cappione, J.E., Nature Communications 10, 5415 (2019). Tips on practical use ===================== @@ -649,5 +689,5 @@ Tips on practical use .. seealso:: :ref:`random_trees_embedding` can also be useful to derive non-linear - representations of feature space, also it does not perform + representations of feature space, but it does not perform dimensionality reduction. diff --git a/doc/modules/metrics.rst b/doc/modules/metrics.rst index 0926980aaaf8a..f65d86a758b03 100644 --- a/doc/modules/metrics.rst +++ b/doc/modules/metrics.rst @@ -28,9 +28,9 @@ There are a number of ways to convert between a distance metric and a similarity measure, such as a kernel. Let ``D`` be the distance, and ``S`` be the kernel: - 1. ``S = np.exp(-D * gamma)``, where one heuristic for choosing - ``gamma`` is ``1 / num_features`` - 2. ``S = 1. / (D / np.max(D))`` +1. ``S = np.exp(-D * gamma)``, where one heuristic for choosing + ``gamma`` is ``1 / num_features`` +2. ``S = 1. / (D / np.max(D))`` .. currentmodule:: sklearn.metrics @@ -87,11 +87,11 @@ represented as tf-idf vectors. can produce normalized vectors, in which case :func:`cosine_similarity` is equivalent to :func:`linear_kernel`, only slower.) -.. topic:: References: +.. rubric:: References - * C.D. Manning, P. Raghavan and H. SchÃŧtze (2008). Introduction to - Information Retrieval. Cambridge University Press. - https://nlp.stanford.edu/IR-book/html/htmledition/the-vector-space-model-for-scoring-1.html +* C.D. Manning, P. Raghavan and H. SchÃŧtze (2008). Introduction to + Information Retrieval. Cambridge University Press. + https://nlp.stanford.edu/IR-book/html/htmledition/the-vector-space-model-for-scoring-1.html .. _linear_kernel: @@ -111,7 +111,7 @@ Polynomial kernel ----------------- The function :func:`polynomial_kernel` computes the degree-d polynomial kernel between two vectors. The polynomial kernel represents the similarity between two -vectors. Conceptually, the polynomial kernels considers not only the similarity +vectors. Conceptually, the polynomial kernel considers not only the similarity between vectors under the same dimension, but also across dimensions. When used in machine learning algorithms, this allows to account for feature interaction. @@ -123,8 +123,8 @@ The polynomial kernel is defined as: where: - * ``x``, ``y`` are the input vectors - * ``d`` is the kernel degree +* ``x``, ``y`` are the input vectors +* ``d`` is the kernel degree If :math:`c_0 = 0` the kernel is said to be homogeneous. @@ -143,9 +143,9 @@ activation function). It is defined as: where: - * ``x``, ``y`` are the input vectors - * :math:`\gamma` is known as slope - * :math:`c_0` is known as intercept +* ``x``, ``y`` are the input vectors +* :math:`\gamma` is known as slope +* :math:`c_0` is known as intercept .. _rbf_kernel: @@ -165,14 +165,14 @@ the kernel is known as the Gaussian kernel of variance :math:`\sigma^2`. Laplacian kernel ---------------- -The function :func:`laplacian_kernel` is a variant on the radial basis +The function :func:`laplacian_kernel` is a variant on the radial basis function kernel defined as: .. math:: k(x, y) = \exp( -\gamma \| x-y \|_1) -where ``x`` and ``y`` are the input vectors and :math:`\|x-y\|_1` is the +where ``x`` and ``y`` are the input vectors and :math:`\|x-y\|_1` is the Manhattan distance between the input vectors. It has proven useful in ML applied to noiseless data. @@ -222,11 +222,10 @@ which is a distance between discrete probability distributions. The chi squared kernel is most commonly used on histograms (bags) of visual words. -.. topic:: References: - - * Zhang, J. and Marszalek, M. and Lazebnik, S. and Schmid, C. - Local features and kernels for classification of texture and object - categories: A comprehensive study - International Journal of Computer Vision 2007 - https://research.microsoft.com/en-us/um/people/manik/projects/trade-off/papers/ZhangIJCV06.pdf +.. rubric:: References +* Zhang, J. and Marszalek, M. and Lazebnik, S. and Schmid, C. + Local features and kernels for classification of texture and object + categories: A comprehensive study + International Journal of Computer Vision 2007 + https://hal.archives-ouvertes.fr/hal-00171412/document diff --git a/doc/modules/mixture.rst b/doc/modules/mixture.rst index fb8e897270f0b..694bde784d61e 100644 --- a/doc/modules/mixture.rst +++ b/doc/modules/mixture.rst @@ -14,13 +14,13 @@ matrices supported), sample them, and estimate them from data. Facilities to help determine the appropriate number of components are also provided. - .. figure:: ../auto_examples/mixture/images/sphx_glr_plot_gmm_pdf_001.png - :target: ../auto_examples/mixture/plot_gmm_pdf.html - :align: center - :scale: 50% +.. figure:: ../auto_examples/mixture/images/sphx_glr_plot_gmm_pdf_001.png + :target: ../auto_examples/mixture/plot_gmm_pdf.html + :align: center + :scale: 50% - **Two-component Gaussian mixture model:** *data points, and equi-probability - surfaces of the model.* + **Two-component Gaussian mixture model:** *data points, and equi-probability + surfaces of the model.* A Gaussian mixture model is a probabilistic model that assumes all the data points are generated from a mixture of a finite number of @@ -42,8 +42,8 @@ algorithm for fitting mixture-of-Gaussian models. It can also draw confidence ellipsoids for multivariate models, and compute the Bayesian Information Criterion to assess the number of clusters in the data. A :meth:`GaussianMixture.fit` method is provided that learns a Gaussian -Mixture Model from train data. Given test data, it can assign to each -sample the Gaussian it mostly probably belong to using +Mixture Model from training data. Given test data, it can assign to each +sample the Gaussian it most probably belongs to using the :meth:`GaussianMixture.predict` method. .. @@ -60,80 +60,111 @@ full covariance. :align: center :scale: 75% -.. topic:: Examples: +.. rubric:: Examples - * See :ref:`sphx_glr_auto_examples_mixture_plot_gmm_covariances.py` for an example of - using the Gaussian mixture as clustering on the iris dataset. +* See :ref:`sphx_glr_auto_examples_mixture_plot_gmm_covariances.py` for an example of + using the Gaussian mixture as clustering on the iris dataset. - * See :ref:`sphx_glr_auto_examples_mixture_plot_gmm_pdf.py` for an example on plotting the - density estimation. +* See :ref:`sphx_glr_auto_examples_mixture_plot_gmm_pdf.py` for an example on plotting the + density estimation. -Pros and cons of class :class:`GaussianMixture` ------------------------------------------------ +.. dropdown:: Pros and cons of class GaussianMixture -Pros -.... + .. rubric:: Pros -:Speed: It is the fastest algorithm for learning mixture models + :Speed: It is the fastest algorithm for learning mixture models -:Agnostic: As this algorithm maximizes only the likelihood, it - will not bias the means towards zero, or bias the cluster sizes to - have specific structures that might or might not apply. + :Agnostic: As this algorithm maximizes only the likelihood, it + will not bias the means towards zero, or bias the cluster sizes to + have specific structures that might or might not apply. -Cons -.... + .. rubric:: Cons -:Singularities: When one has insufficiently many points per - mixture, estimating the covariance matrices becomes difficult, - and the algorithm is known to diverge and find solutions with - infinite likelihood unless one regularizes the covariances artificially. + :Singularities: When one has insufficiently many points per + mixture, estimating the covariance matrices becomes difficult, + and the algorithm is known to diverge and find solutions with + infinite likelihood unless one regularizes the covariances artificially. -:Number of components: This algorithm will always use all the - components it has access to, needing held-out data - or information theoretical criteria to decide how many components to use - in the absence of external cues. + :Number of components: This algorithm will always use all the + components it has access to, needing held-out data + or information theoretical criteria to decide how many components to use + in the absence of external cues. -Selecting the number of components in a classical Gaussian Mixture Model ------------------------------------------------------------------------- +.. dropdown:: Selecting the number of components in a classical Gaussian Mixture model -The BIC criterion can be used to select the number of components in a Gaussian -Mixture in an efficient way. In theory, it recovers the true number of -components only in the asymptotic regime (i.e. if much data is available and -assuming that the data was actually generated i.i.d. from a mixture of Gaussian -distribution). Note that using a :ref:`Variational Bayesian Gaussian mixture ` -avoids the specification of the number of components for a Gaussian mixture -model. + The BIC criterion can be used to select the number of components in a Gaussian + Mixture in an efficient way. In theory, it recovers the true number of + components only in the asymptotic regime (i.e. if much data is available and + assuming that the data was actually generated i.i.d. from a mixture of Gaussian + distributions). Note that using a :ref:`Variational Bayesian Gaussian mixture ` + avoids the specification of the number of components for a Gaussian mixture + model. -.. figure:: ../auto_examples/mixture/images/sphx_glr_plot_gmm_selection_001.png - :target: ../auto_examples/mixture/plot_gmm_selection.html - :align: center - :scale: 50% + .. figure:: ../auto_examples/mixture/images/sphx_glr_plot_gmm_selection_002.png + :target: ../auto_examples/mixture/plot_gmm_selection.html + :align: center + :scale: 50% -.. topic:: Examples: + .. rubric:: Examples - * See :ref:`sphx_glr_auto_examples_mixture_plot_gmm_selection.py` for an example - of model selection performed with classical Gaussian mixture. + * See :ref:`sphx_glr_auto_examples_mixture_plot_gmm_selection.py` for an example + of model selection performed with classical Gaussian mixture. .. _expectation_maximization: -Estimation algorithm Expectation-maximization ------------------------------------------------ - -The main difficulty in learning Gaussian mixture models from unlabeled -data is that it is one usually doesn't know which points came from -which latent component (if one has access to this information it gets -very easy to fit a separate Gaussian distribution to each set of -points). `Expectation-maximization -`_ -is a well-founded statistical -algorithm to get around this problem by an iterative process. First -one assumes random components (randomly centered on data points, -learned from k-means, or even just normally distributed around the -origin) and computes for each point a probability of being generated by -each component of the model. Then, one tweaks the -parameters to maximize the likelihood of the data given those -assignments. Repeating this process is guaranteed to always converge -to a local optimum. +.. dropdown:: Estimation algorithm expectation-maximization + + The main difficulty in learning Gaussian mixture models from unlabeled + data is that one usually doesn't know which points came from + which latent component (if one has access to this information it gets + very easy to fit a separate Gaussian distribution to each set of + points). `Expectation-maximization + `_ + is a well-founded statistical + algorithm to get around this problem by an iterative process. First + one assumes random components (randomly centered on data points, + learned from k-means, or even just normally distributed around the + origin) and computes for each point a probability of being generated by + each component of the model. Then, one tweaks the + parameters to maximize the likelihood of the data given those + assignments. Repeating this process is guaranteed to always converge + to a local optimum. + +.. dropdown:: Choice of the Initialization method + + There is a choice of four initialization methods (as well as inputting user defined + initial means) to generate the initial centers for the model components: + + k-means (default) + This applies a traditional k-means clustering algorithm. + This can be computationally expensive compared to other initialization methods. + + k-means++ + This uses the initialization method of k-means clustering: k-means++. + This will pick the first center at random from the data. Subsequent centers will be + chosen from a weighted distribution of the data favouring points further away from + existing centers. k-means++ is the default initialization for k-means so will be + quicker than running a full k-means but can still take a significant amount of + time for large data sets with many components. + + random_from_data + This will pick random data points from the input data as the initial + centers. This is a very fast method of initialization but can produce non-convergent + results if the chosen points are too close to each other. + + random + Centers are chosen as a small perturbation away from the mean of all data. + This method is simple but can lead to the model taking longer to converge. + + .. figure:: ../auto_examples/mixture/images/sphx_glr_plot_gmm_init_001.png + :target: ../auto_examples/mixture/plot_gmm_init.html + :align: center + :scale: 50% + + .. rubric:: Examples + + * See :ref:`sphx_glr_auto_examples_mixture_plot_gmm_init.py` for an example of + using different initializations in Gaussian Mixture. .. _bgmm: @@ -142,12 +173,11 @@ Variational Bayesian Gaussian Mixture The :class:`BayesianGaussianMixture` object implements a variant of the Gaussian mixture model with variational inference algorithms. The API is -similar as the one defined by :class:`GaussianMixture`. +similar to the one defined by :class:`GaussianMixture`. .. _variational_inference: -Estimation algorithm: variational inference ---------------------------------------------- +**Estimation algorithm: variational inference** Variational inference is an extension of expectation-maximization that maximizes a lower bound on model evidence (including @@ -162,13 +192,13 @@ expectation-maximization solutions but introduces some subtle biases to the model. Inference is often notably slower, but not usually as much so as to render usage unpractical. -Due to its Bayesian nature, the variational algorithm needs more hyper- -parameters than expectation-maximization, the most important of these being the +Due to its Bayesian nature, the variational algorithm needs more hyperparameters +than expectation-maximization, the most important of these being the concentration parameter ``weight_concentration_prior``. Specifying a low value -for the concentration prior will make the model put most of the weight on few -components set the remaining components weights very close to zero. High values -of the concentration prior will allow a larger number of components to be active -in the mixture. +for the concentration prior will make the model put most of the weight on a few +components and set the remaining components' weights very close to zero. High +values of the concentration prior will allow a larger number of components to +be active in the mixture. The parameters implementation of the :class:`BayesianGaussianMixture` class proposes two types of prior for the weights distribution: a finite mixture model @@ -178,7 +208,7 @@ uses a truncated distribution with a fixed maximum number of components (called the Stick-breaking representation). The number of components actually used almost always depends on the data. -The next figure compares the results obtained for the different type of the +The next figure compares the results obtained for the different types of the weight concentration prior (parameter ``weight_concentration_prior_type``) for different values of ``weight_concentration_prior``. Here, we can see the value of the ``weight_concentration_prior`` parameter @@ -229,64 +259,58 @@ from the two resulting mixtures. -.. topic:: Examples: - - * See :ref:`sphx_glr_auto_examples_mixture_plot_gmm.py` for an example on - plotting the confidence ellipsoids for both :class:`GaussianMixture` - and :class:`BayesianGaussianMixture`. - - * :ref:`sphx_glr_auto_examples_mixture_plot_gmm_sin.py` shows using - :class:`GaussianMixture` and :class:`BayesianGaussianMixture` to fit a - sine wave. - - * See :ref:`sphx_glr_auto_examples_mixture_plot_concentration_prior.py` - for an example plotting the confidence ellipsoids for the - :class:`BayesianGaussianMixture` with different - ``weight_concentration_prior_type`` for different values of the parameter - ``weight_concentration_prior``. +.. rubric:: Examples +* See :ref:`sphx_glr_auto_examples_mixture_plot_gmm.py` for an example on + plotting the confidence ellipsoids for both :class:`GaussianMixture` + and :class:`BayesianGaussianMixture`. -Pros and cons of variational inference with :class:`BayesianGaussianMixture` ----------------------------------------------------------------------------- +* :ref:`sphx_glr_auto_examples_mixture_plot_gmm_sin.py` shows using + :class:`GaussianMixture` and :class:`BayesianGaussianMixture` to fit a + sine wave. -Pros -..... +* See :ref:`sphx_glr_auto_examples_mixture_plot_concentration_prior.py` + for an example plotting the confidence ellipsoids for the + :class:`BayesianGaussianMixture` with different + ``weight_concentration_prior_type`` for different values of the parameter + ``weight_concentration_prior``. -:Automatic selection: when ``weight_concentration_prior`` is small enough and - ``n_components`` is larger than what is found necessary by the model, the - Variational Bayesian mixture model has a natural tendency to set some mixture - weights values close to zero. This makes it possible to let the model choose - a suitable number of effective components automatically. Only an upper bound - of this number needs to be provided. Note however that the "ideal" number of - active components is very application specific and is typically ill-defined - in a data exploration setting. +.. dropdown:: Pros and cons of variational inference with BayesianGaussianMixture -:Less sensitivity to the number of parameters: unlike finite models, which will - almost always use all components as much as they can, and hence will produce - wildly different solutions for different numbers of components, the - variational inference with a Dirichlet process prior - (``weight_concentration_prior_type='dirichlet_process'``) won't change much - with changes to the parameters, leading to more stability and less tuning. + .. rubric:: Pros -:Regularization: due to the incorporation of prior information, - variational solutions have less pathological special cases than - expectation-maximization solutions. + :Automatic selection: When ``weight_concentration_prior`` is small enough and + ``n_components`` is larger than what is found necessary by the model, the + Variational Bayesian mixture model has a natural tendency to set some mixture + weights values close to zero. This makes it possible to let the model choose + a suitable number of effective components automatically. Only an upper bound + of this number needs to be provided. Note however that the "ideal" number of + active components is very application specific and is typically ill-defined + in a data exploration setting. + :Less sensitivity to the number of parameters: Unlike finite models, which will + almost always use all components as much as they can, and hence will produce + wildly different solutions for different numbers of components, the + variational inference with a Dirichlet process prior + (``weight_concentration_prior_type='dirichlet_process'``) won't change much + with changes to the parameters, leading to more stability and less tuning. -Cons -..... + :Regularization: Due to the incorporation of prior information, + variational solutions have less pathological special cases than + expectation-maximization solutions. -:Speed: the extra parametrization necessary for variational inference make - inference slower, although not by much. + .. rubric:: Cons -:Hyperparameters: this algorithm needs an extra hyperparameter - that might need experimental tuning via cross-validation. + :Speed: The extra parametrization necessary for variational inference makes + inference slower, although not by much. -:Bias: there are many implicit biases in the inference algorithms (and also in - the Dirichlet process if used), and whenever there is a mismatch between - these biases and the data it might be possible to fit better models using a - finite mixture. + :Hyperparameters: This algorithm needs an extra hyperparameter + that might need experimental tuning via cross-validation. + :Bias: There are many implicit biases in the inference algorithms (and also in + the Dirichlet process if used), and whenever there is a mismatch between + these biases and the data it might be possible to fit better models using a + finite mixture. .. _dirichlet_process: @@ -312,7 +336,7 @@ group of the mixture. At the end, to represent the infinite mixture, we associate the last remaining piece of the stick to the proportion of points that don't fall into all the other groups. The length of each piece is a random variable with probability proportional to the concentration parameter. Smaller -value of the concentration will divide the unit-length into larger pieces of +values of the concentration will divide the unit-length into larger pieces of the stick (defining more concentrated distribution). Larger concentration values will create smaller pieces of the stick (increasing the number of components with non zero weights). diff --git a/doc/modules/model_evaluation.rst b/doc/modules/model_evaluation.rst index c807af982e277..c304966fccdb2 100644 --- a/doc/modules/model_evaluation.rst +++ b/doc/modules/model_evaluation.rst @@ -6,18 +6,158 @@ Metrics and scoring: quantifying the quality of predictions =========================================================== +.. _which_scoring_function: + +Which scoring function should I use? +==================================== + +Before we take a closer look into the details of the many scores and +:term:`evaluation metrics`, we want to give some guidance, inspired by statistical +decision theory, on the choice of **scoring functions** for **supervised learning**, +see [Gneiting2009]_: + +- *Which scoring function should I use?* +- *Which scoring function is a good one for my task?* + +In a nutshell, if the scoring function is given, e.g. in a kaggle competition +or in a business context, use that one. +If you are free to choose, it starts by considering the ultimate goal and application +of the prediction. It is useful to distinguish two steps: + +* Predicting +* Decision making + +**Predicting:** +Usually, the response variable :math:`Y` is a random variable, in the sense that there +is *no deterministic* function :math:`Y = g(X)` of the features :math:`X`. +Instead, there is a probability distribution :math:`F` of :math:`Y`. +One can aim to predict the whole distribution, known as *probabilistic prediction*, +or---more the focus of scikit-learn---issue a *point prediction* (or point forecast) +by choosing a property or functional of that distribution :math:`F`. +Typical examples are the mean (expected value), the median or a quantile of the +response variable :math:`Y` (conditionally on :math:`X`). + +Once that is settled, use a **strictly consistent** scoring function for that +(target) functional, see [Gneiting2009]_. +This means using a scoring function that is aligned with *measuring the distance +between predictions* `y_pred` *and the true target functional using observations of* +:math:`Y`, i.e. `y_true`. +For classification **strictly proper scoring rules**, see +`Wikipedia entry for Scoring rule `_ +and [Gneiting2007]_, coincide with strictly consistent scoring functions. +The table further below provides examples. +One could say that consistent scoring functions act as *truth serum* in that +they guarantee *"that truth telling [. . .] is an optimal strategy in +expectation"* [Gneiting2014]_. + +Once a strictly consistent scoring function is chosen, it is best used for both: as +loss function for model training and as metric/score in model evaluation and model +comparison. + +Note that for regressors, the prediction is done with :term:`predict` while for +classifiers it is usually :term:`predict_proba`. + +**Decision Making:** +The most common decisions are done on binary classification tasks, where the result of +:term:`predict_proba` is turned into a single outcome, e.g., from the predicted +probability of rain a decision is made on how to act (whether to take mitigating +measures like an umbrella or not). +For classifiers, this is what :term:`predict` returns. +See also :ref:`TunedThresholdClassifierCV`. +There are many scoring functions which measure different aspects of such a +decision, most of them are covered with or derived from the +:func:`metrics.confusion_matrix`. + +**List of strictly consistent scoring functions:** +Here, we list some of the most relevant statistical functionals and corresponding +strictly consistent scoring functions for tasks in practice. Note that the list is not +complete and that there are more of them. +For further criteria on how to select a specific one, see [Fissler2022]_. + +================== =================================================== ==================== ================================= +functional scoring or loss function response `y` prediction +================== =================================================== ==================== ================================= +**Classification** +mean :ref:`Brier score ` :sup:`1` multi-class ``predict_proba`` +mean :ref:`log loss ` multi-class ``predict_proba`` +mode :ref:`zero-one loss ` :sup:`2` multi-class ``predict``, categorical +**Regression** +mean :ref:`squared error ` :sup:`3` all reals ``predict``, all reals +mean :ref:`Poisson deviance ` non-negative ``predict``, strictly positive +mean :ref:`Gamma deviance ` strictly positive ``predict``, strictly positive +mean :ref:`Tweedie deviance ` depends on ``power`` ``predict``, depends on ``power`` +median :ref:`absolute error ` all reals ``predict``, all reals +quantile :ref:`pinball loss ` all reals ``predict``, all reals +mode no consistent one exists reals +================== =================================================== ==================== ================================= + +:sup:`1` The Brier score is just a different name for the squared error in case of +classification. + +:sup:`2` The zero-one loss is only consistent but not strictly consistent for the mode. +The zero-one loss is equivalent to one minus the accuracy score, meaning it gives +different score values but the same ranking. + +:sup:`3` R² gives the same ranking as squared error. + +**Fictitious Example:** +Let's make the above arguments more tangible. Consider a setting in network reliability +engineering, such as maintaining stable internet or Wi-Fi connections. +As provider of the network, you have access to the dataset of log entries of network +connections containing network load over time and many interesting features. +Your goal is to improve the reliability of the connections. +In fact, you promise your customers that on at least 99% of all days there are no +connection discontinuities larger than 1 minute. +Therefore, you are interested in a prediction of the 99% quantile (of longest +connection interruption duration per day) in order to know in advance when to add +more bandwidth and thereby satisfy your customers. So the *target functional* is the +99% quantile. From the table above, you choose the pinball loss as scoring function +(fair enough, not much choice given), for model training (e.g. +`HistGradientBoostingRegressor(loss="quantile", quantile=0.99)`) as well as model +evaluation (`mean_pinball_loss(..., alpha=0.99)` - we apologize for the different +argument names, `quantile` and `alpha`) be it in grid search for finding +hyperparameters or in comparing to other models like +`QuantileRegressor(quantile=0.99)`. + +.. rubric:: References + +.. [Gneiting2007] T. Gneiting and A. E. Raftery. :doi:`Strictly Proper + Scoring Rules, Prediction, and Estimation <10.1198/016214506000001437>` + In: Journal of the American Statistical Association 102 (2007), + pp. 359– 378. + `link to pdf `_ + +.. [Gneiting2009] T. Gneiting. :arxiv:`Making and Evaluating Point Forecasts + <0912.0902>` + Journal of the American Statistical Association 106 (2009): 746 - 762. + +.. [Gneiting2014] T. Gneiting and M. Katzfuss. :doi:`Probabilistic Forecasting + <10.1146/annurev-statistics-062713-085831>`. In: Annual Review of Statistics and Its Application 1.1 (2014), pp. 125–151. + +.. [Fissler2022] T. Fissler, C. Lorentzen and M. Mayer. :arxiv:`Model + Comparison and Calibration Assessment: User Guide for Consistent Scoring + Functions in Machine Learning and Actuarial Practice. <2202.12780>` + +.. _scoring_api_overview: + +Scoring API overview +==================== + There are 3 different APIs for evaluating the quality of a model's predictions: * **Estimator score method**: Estimators have a ``score`` method providing a default evaluation criterion for the problem they are designed to solve. - This is not discussed on this page, but in each estimator's documentation. + Most commonly this is :ref:`accuracy ` for classifiers and the + :ref:`coefficient of determination ` (:math:`R^2`) for regressors. + Details for each estimator can be found in its documentation. -* **Scoring parameter**: Model-evaluation tools using +* **Scoring parameter**: Model-evaluation tools that use :ref:`cross-validation ` (such as - :func:`model_selection.cross_val_score` and - :class:`model_selection.GridSearchCV`) rely on an internal *scoring* strategy. - This is discussed in the section :ref:`scoring_parameter`. + :class:`model_selection.GridSearchCV`, :func:`model_selection.validation_curve` and + :class:`linear_model.LogisticRegressionCV`) rely on an internal *scoring* strategy. + This can be specified using the `scoring` parameter of that tool and is discussed + in the section :ref:`scoring_parameter`. * **Metric functions**: The :mod:`sklearn.metrics` module implements functions assessing prediction error for specific purposes. These metrics are detailed @@ -38,24 +178,39 @@ value of those metrics for random predictions. The ``scoring`` parameter: defining model evaluation rules ========================================================== -Model selection and evaluation using tools, such as -:class:`model_selection.GridSearchCV` and -:func:`model_selection.cross_val_score`, take a ``scoring`` parameter that +Model selection and evaluation tools that internally use +:ref:`cross-validation ` (such as +:class:`model_selection.GridSearchCV`, :func:`model_selection.validation_curve` and +:class:`linear_model.LogisticRegressionCV`) take a ``scoring`` parameter that controls what metric they apply to the estimators evaluated. -Common cases: predefined values -------------------------------- +They can be specified in several ways: + +* `None`: the estimator's default evaluation criterion (i.e., the metric used in the + estimator's `score` method) is used. +* :ref:`String name `: common metrics can be passed via a string + name. +* :ref:`Callable `: more complex metrics can be passed via a custom + metric callable (e.g., function). + +Some tools do also accept multiple metric evaluation. See :ref:`multimetric_scoring` +for details. + +.. _scoring_string_names: + +String name scorers +------------------- For the most common use cases, you can designate a scorer object with the -``scoring`` parameter; the table below shows all possible values. +``scoring`` parameter via a string name; the table below shows all possible values. All scorer objects follow the convention that **higher return values are better -than lower return values**. Thus metrics which measure the distance between +than lower return values**. Thus metrics which measure the distance between the model and the data, like :func:`metrics.mean_squared_error`, are -available as neg_mean_squared_error which return the negated value +available as 'neg_mean_squared_error' which return the negated value of the metric. ==================================== ============================================== ================================== -Scoring Function Comment +Scoring string name Function Comment ==================================== ============================================== ================================== **Classification** 'accuracy' :func:`metrics.accuracy_score` @@ -77,6 +232,7 @@ Scoring Function 'roc_auc_ovo' :func:`metrics.roc_auc_score` 'roc_auc_ovr_weighted' :func:`metrics.roc_auc_score` 'roc_auc_ovo_weighted' :func:`metrics.roc_auc_score` +'d2_log_loss_score' :func:`metrics.d2_log_loss_score` **Clustering** 'adjusted_mutual_info_score' :func:`metrics.adjusted_mutual_info_score` @@ -91,19 +247,20 @@ Scoring Function **Regression** 'explained_variance' :func:`metrics.explained_variance_score` -'max_error' :func:`metrics.max_error` +'neg_max_error' :func:`metrics.max_error` 'neg_mean_absolute_error' :func:`metrics.mean_absolute_error` 'neg_mean_squared_error' :func:`metrics.mean_squared_error` -'neg_root_mean_squared_error' :func:`metrics.mean_squared_error` +'neg_root_mean_squared_error' :func:`metrics.root_mean_squared_error` 'neg_mean_squared_log_error' :func:`metrics.mean_squared_log_error` +'neg_root_mean_squared_log_error' :func:`metrics.root_mean_squared_log_error` 'neg_median_absolute_error' :func:`metrics.median_absolute_error` 'r2' :func:`metrics.r2_score` 'neg_mean_poisson_deviance' :func:`metrics.mean_poisson_deviance` 'neg_mean_gamma_deviance' :func:`metrics.mean_gamma_deviance` 'neg_mean_absolute_percentage_error' :func:`metrics.mean_absolute_percentage_error` +'d2_absolute_error_score' :func:`metrics.d2_absolute_error_score` ==================================== ============================================== ================================== - Usage examples: >>> from sklearn import svm, datasets @@ -111,47 +268,51 @@ Usage examples: >>> X, y = datasets.load_iris(return_X_y=True) >>> clf = svm.SVC(random_state=0) >>> cross_val_score(clf, X, y, cv=5, scoring='recall_macro') - array([0.96..., 0.96..., 0.96..., 0.93..., 1. ]) - >>> model = svm.SVC() - >>> cross_val_score(model, X, y, cv=5, scoring='wrong_choice') - Traceback (most recent call last): - ValueError: 'wrong_choice' is not a valid scoring value. Use sorted(sklearn.metrics.SCORERS.keys()) to get valid options. + array([0.96, 0.96, 0.96, 0.93, 1. ]) .. note:: - The values listed by the ``ValueError`` exception correspond to the functions measuring - prediction accuracy described in the following sections. - The scorer objects for those functions are stored in the dictionary - ``sklearn.metrics.SCORERS``. + If a wrong scoring name is passed, an ``InvalidParameterError`` is raised. + You can retrieve the names of all available scorers by calling + :func:`~sklearn.metrics.get_scorer_names`. .. currentmodule:: sklearn.metrics -.. _scoring: +.. _scoring_callable: -Defining your scoring strategy from metric functions ------------------------------------------------------ +Callable scorers +---------------- -The module :mod:`sklearn.metrics` also exposes a set of simple functions -measuring a prediction error given ground truth and prediction: +For more complex use cases and more flexibility, you can pass a callable to +the `scoring` parameter. This can be done by: -- functions ending with ``_score`` return a value to - maximize, the higher the better. +* :ref:`scoring_adapt_metric` +* :ref:`scoring_custom` (most flexible) -- functions ending with ``_error`` or ``_loss`` return a - value to minimize, the lower the better. When converting - into a scorer object using :func:`make_scorer`, set - the ``greater_is_better`` parameter to ``False`` (``True`` by default; see the - parameter description below). +.. _scoring_adapt_metric: -Metrics available for various machine learning tasks are detailed in sections -below. +Adapting predefined metrics via `make_scorer` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Many metrics are not given names to be used as ``scoring`` values, +The following metric functions are not implemented as named scorers, sometimes because they require additional parameters, such as -:func:`fbeta_score`. In such cases, you need to generate an appropriate -scoring object. The simplest way to generate a callable object for scoring -is by using :func:`make_scorer`. That function converts metrics -into callables that can be used for model evaluation. +:func:`fbeta_score`. They cannot be passed to the ``scoring`` +parameters; instead their callable needs to be passed to +:func:`make_scorer` together with the value of the user-settable +parameters. + +===================================== ========= ============================================== +Function Parameter Example usage +===================================== ========= ============================================== +**Classification** +:func:`metrics.fbeta_score` ``beta`` ``make_scorer(fbeta_score, beta=2)`` + +**Regression** +:func:`metrics.mean_tweedie_deviance` ``power`` ``make_scorer(mean_tweedie_deviance, power=1.5)`` +:func:`metrics.mean_pinball_loss` ``alpha`` ``make_scorer(mean_pinball_loss, alpha=0.95)`` +:func:`metrics.d2_tweedie_score` ``power`` ``make_scorer(d2_tweedie_score, power=1.5)`` +:func:`metrics.d2_pinball_score` ``alpha`` ``make_scorer(d2_pinball_score, alpha=0.95)`` +===================================== ========= ============================================== One typical use case is to wrap an existing metric function from the library with non-default values for its parameters, such as the ``beta`` parameter for @@ -164,67 +325,101 @@ the :func:`fbeta_score` function:: >>> grid = GridSearchCV(LinearSVC(), param_grid={'C': [1, 10]}, ... scoring=ftwo_scorer, cv=5) -The second use case is to build a completely custom scorer object -from a simple python function using :func:`make_scorer`, which can -take several parameters: - -* the python function you want to use (``my_custom_loss_func`` - in the example below) - -* whether the python function returns a score (``greater_is_better=True``, - the default) or a loss (``greater_is_better=False``). If a loss, the output - of the python function is negated by the scorer object, conforming to - the cross validation convention that scorers return higher values for better models. - -* for classification metrics only: whether the python function you provided requires continuous decision - certainties (``needs_threshold=True``). The default value is - False. +The module :mod:`sklearn.metrics` also exposes a set of simple functions +measuring a prediction error given ground truth and prediction: -* any additional parameters, such as ``beta`` or ``labels`` in :func:`f1_score`. +- functions ending with ``_score`` return a value to + maximize, the higher the better. -Here is an example of building custom scorers, and of using the -``greater_is_better`` parameter:: +- functions ending with ``_error``, ``_loss``, or ``_deviance`` return a + value to minimize, the lower the better. When converting + into a scorer object using :func:`make_scorer`, set + the ``greater_is_better`` parameter to ``False`` (``True`` by default; see the + parameter description below). - >>> import numpy as np - >>> def my_custom_loss_func(y_true, y_pred): - ... diff = np.abs(y_true - y_pred).max() - ... return np.log1p(diff) - ... - >>> # score will negate the return value of my_custom_loss_func, - >>> # which will be np.log(2), 0.693, given the values for X - >>> # and y defined below. - >>> score = make_scorer(my_custom_loss_func, greater_is_better=False) - >>> X = [[1], [1]] - >>> y = [0, 1] - >>> from sklearn.dummy import DummyClassifier - >>> clf = DummyClassifier(strategy='most_frequent', random_state=0) - >>> clf = clf.fit(X, y) - >>> my_custom_loss_func(y, clf.predict(X)) - 0.69... - >>> score(clf, X, y) - -0.69... - - -.. _diy_scoring: - -Implementing your own scoring object ------------------------------------- -You can generate even more flexible model scorers by constructing your own -scoring object from scratch, without using the :func:`make_scorer` factory. -For a callable to be a scorer, it needs to meet the protocol specified by -the following two rules: - -- It can be called with parameters ``(estimator, X, y)``, where ``estimator`` - is the model that should be evaluated, ``X`` is validation data, and ``y`` is - the ground truth target for ``X`` (in the supervised case) or ``None`` (in the - unsupervised case). - -- It returns a floating point number that quantifies the - ``estimator`` prediction quality on ``X``, with reference to ``y``. - Again, by convention higher numbers are better, so if your scorer - returns loss, that value should be negated. - -.. note:: **Using custom scorers in functions where n_jobs > 1** +.. _scoring_custom: + +Creating a custom scorer object +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You can create your own custom scorer object using +:func:`make_scorer` or for the most flexibility, from scratch. See below for details. + +.. dropdown:: Custom scorer objects using `make_scorer` + + You can build a completely custom scorer object + from a simple python function using :func:`make_scorer`, which can + take several parameters: + + * the python function you want to use (``my_custom_loss_func`` + in the example below) + + * whether the python function returns a score (``greater_is_better=True``, + the default) or a loss (``greater_is_better=False``). If a loss, the output + of the python function is negated by the scorer object, conforming to + the cross validation convention that scorers return higher values for better models. + + * for classification metrics only: whether the python function you provided requires + continuous decision certainties. If the scoring function only accepts probability + estimates (e.g. :func:`metrics.log_loss`), then one needs to set the parameter + `response_method="predict_proba"`. Some scoring + functions do not necessarily require probability estimates but rather non-thresholded + decision values (e.g. :func:`metrics.roc_auc_score`). In this case, one can provide a + list (e.g., `response_method=["decision_function", "predict_proba"]`), + and scorer will use the first available method, in the order given in the list, + to compute the scores. + + * any additional parameters of the scoring function, such as ``beta`` or ``labels``. + + Here is an example of building custom scorers, and of using the + ``greater_is_better`` parameter:: + + >>> import numpy as np + >>> def my_custom_loss_func(y_true, y_pred): + ... diff = np.abs(y_true - y_pred).max() + ... return float(np.log1p(diff)) + ... + >>> # score will negate the return value of my_custom_loss_func, + >>> # which will be np.log(2), 0.693, given the values for X + >>> # and y defined below. + >>> score = make_scorer(my_custom_loss_func, greater_is_better=False) + >>> X = [[1], [1]] + >>> y = [0, 1] + >>> from sklearn.dummy import DummyClassifier + >>> clf = DummyClassifier(strategy='most_frequent', random_state=0) + >>> clf = clf.fit(X, y) + >>> my_custom_loss_func(y, clf.predict(X)) + 0.69 + >>> score(clf, X, y) + -0.69 + +.. dropdown:: Custom scorer objects from scratch + + You can generate even more flexible model scorers by constructing your own + scoring object from scratch, without using the :func:`make_scorer` factory. + + For a callable to be a scorer, it needs to meet the protocol specified by + the following two rules: + + - It can be called with parameters ``(estimator, X, y)``, where ``estimator`` + is the model that should be evaluated, ``X`` is validation data, and ``y`` is + the ground truth target for ``X`` (in the supervised case) or ``None`` (in the + unsupervised case). + + - It returns a floating point number that quantifies the + ``estimator`` prediction quality on ``X``, with reference to ``y``. + Again, by convention higher numbers are better, so if your scorer + returns loss, that value should be negated. + + - Advanced: If it requires extra metadata to be passed to it, it should expose + a ``get_metadata_routing`` method returning the requested metadata. The user + should be able to set the requested metadata via a ``set_score_request`` + method. Please see :ref:`User Guide ` and :ref:`Developer + Guide ` for + more details. + + +.. dropdown:: Using custom scorers in functions where n_jobs > 1 While defining the custom scoring function alongside the calling function should work out of the box with the default joblib backend (loky), @@ -255,13 +450,15 @@ There are three ways to specify multiple scoring metrics for the ``scoring`` parameter: - As an iterable of string metrics:: - >>> scoring = ['accuracy', 'precision'] + + >>> scoring = ['accuracy', 'precision'] - As a ``dict`` mapping the scorer name to the scoring function:: - >>> from sklearn.metrics import accuracy_score - >>> from sklearn.metrics import make_scorer - >>> scoring = {'accuracy': make_scorer(accuracy_score), - ... 'prec': 'precision'} + + >>> from sklearn.metrics import accuracy_score + >>> from sklearn.metrics import make_scorer + >>> scoring = {'accuracy': make_scorer(accuracy_score), + ... 'prec': 'precision'} Note that the dict values can either be scorer functions or one of the predefined metric strings. @@ -307,6 +504,7 @@ Some of these are restricted to the binary classification case: precision_recall_curve roc_curve + class_likelihood_ratios det_curve @@ -340,6 +538,7 @@ Some also work in the multilabel case: recall_score roc_auc_score zero_one_loss + d2_log_loss_score And some work with binary and multilabel (but not multiclass) problems: @@ -428,18 +627,18 @@ where :math:`1(x)` is the `indicator function >>> accuracy_score(y_true, y_pred) 0.5 >>> accuracy_score(y_true, y_pred, normalize=False) - 2 + 2.0 In the multilabel case with binary label indicators:: >>> accuracy_score(np.array([[0, 1], [1, 1]]), np.ones((2, 2))) 0.5 -.. topic:: Example: +.. rubric:: Examples - * See :ref:`sphx_glr_auto_examples_feature_selection_plot_permutation_test_for_classification.py` - for an example of accuracy score usage using permutations of - the dataset. +* See :ref:`sphx_glr_auto_examples_model_selection_plot_permutation_tests_for_classification.py` + for an example of accuracy score usage using permutations of + the dataset. .. _top_k_accuracy_score: @@ -477,7 +676,7 @@ where :math:`k` is the number of guesses allowed and :math:`1(x)` is the 0.75 >>> # Not normalizing gives the number of "correctly" classified samples >>> top_k_accuracy_score(y_true, y_score, k=2, normalize=False) - 3 + 3.0 .. _balanced_accuracy_score: @@ -510,7 +709,7 @@ In contrast, if the conventional accuracy is above chance only because the classifier takes advantage of an imbalanced test set, then the balanced accuracy, as appropriate, will drop to :math:`\frac{1}{n\_classes}`. -The score ranges from 0 to 1, or when ``adjusted=True`` is used, it rescaled to +The score ranges from 0 to 1, or when ``adjusted=True`` is used, it is rescaled to the range :math:`\frac{1}{1 - n\_classes}` to 1, inclusive, with performance at random scoring 0. @@ -550,21 +749,20 @@ or *informedness*. * Balanced Accuracy as described in [Urbanowicz2015]_: the average of sensitivity and specificity is computed for each class and then averaged over total number of classes. -.. topic:: References: - - .. [Guyon2015] I. Guyon, K. Bennett, G. Cawley, H.J. Escalante, S. Escalera, T.K. Ho, N. Macià, - B. Ray, M. Saeed, A.R. Statnikov, E. Viegas, `Design of the 2015 ChaLearn AutoML Challenge - `_, - IJCNN 2015. - .. [Mosley2013] L. Mosley, `A balanced approach to the multi-class imbalance problem - `_, - IJCV 2010. - .. [Kelleher2015] John. D. Kelleher, Brian Mac Namee, Aoife D'Arcy, `Fundamentals of - Machine Learning for Predictive Data Analytics: Algorithms, Worked Examples, - and Case Studies `_, - 2015. - .. [Urbanowicz2015] Urbanowicz R.J., Moore, J.H. `ExSTraCS 2.0: description and evaluation of a scalable learning - classifier system `_, Evol. Intel. (2015) 8: 89. +.. rubric:: References + +.. [Guyon2015] I. Guyon, K. Bennett, G. Cawley, H.J. Escalante, S. Escalera, T.K. Ho, N. Macià, + B. Ray, M. Saeed, A.R. Statnikov, E. Viegas, `Design of the 2015 ChaLearn AutoML Challenge + `_, IJCNN 2015. +.. [Mosley2013] L. Mosley, `A balanced approach to the multi-class imbalance problem + `_, IJCV 2010. +.. [Kelleher2015] John. D. Kelleher, Brian Mac Namee, Aoife D'Arcy, `Fundamentals of + Machine Learning for Predictive Data Analytics: Algorithms, Worked Examples, + and Case Studies `_, + 2015. +.. [Urbanowicz2015] Urbanowicz R.J., Moore, J.H. :doi:`ExSTraCS 2.0: description + and evaluation of a scalable learning classifier + system <10.1007/s12065-015-0128-8>`, Evol. Intel. (2015) 8: 89. .. _cohen_kappa: @@ -576,7 +774,7 @@ The function :func:`cohen_kappa_score` computes `Cohen's kappa This measure is intended to compare labelings by different human annotators, not a classifier versus a ground truth. -The kappa score (see docstring) is a number between -1 and 1. +The kappa score is a number between -1 and 1. Scores above .8 are generally considered good agreement; zero or lower means no agreement (practically random labels). @@ -585,9 +783,9 @@ but not for multilabel problems (except by manually computing a per-label score) and not for more than two annotators. >>> from sklearn.metrics import cohen_kappa_score - >>> y_true = [2, 0, 2, 2, 0, 1] - >>> y_pred = [0, 0, 2, 2, 0, 2] - >>> cohen_kappa_score(y_true, y_pred) + >>> labeling1 = [2, 0, 2, 2, 0, 1] + >>> labeling2 = [0, 0, 2, 2, 0, 2] + >>> cohen_kappa_score(labeling1, labeling2) 0.4285714285714286 .. _confusion_matrix: @@ -639,23 +837,23 @@ false negatives and true positives as follows:: >>> y_true = [0, 0, 0, 1, 1, 1, 1, 1] >>> y_pred = [0, 1, 0, 1, 0, 1, 0, 1] - >>> tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel() + >>> tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel().tolist() >>> tn, fp, fn, tp (2, 1, 2, 3) -.. topic:: Example: +.. rubric:: Examples - * See :ref:`sphx_glr_auto_examples_model_selection_plot_confusion_matrix.py` - for an example of using a confusion matrix to evaluate classifier output - quality. +* See :ref:`sphx_glr_auto_examples_model_selection_plot_confusion_matrix.py` + for an example of using a confusion matrix to evaluate classifier output + quality. - * See :ref:`sphx_glr_auto_examples_classification_plot_digits_classification.py` - for an example of using a confusion matrix to classify - hand-written digits. +* See :ref:`sphx_glr_auto_examples_classification_plot_digits_classification.py` + for an example of using a confusion matrix to classify + hand-written digits. - * See :ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py` - for an example of using a confusion matrix to classify text - documents. +* See :ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py` + for an example of using a confusion matrix to classify text + documents. .. _classification_report: @@ -682,19 +880,15 @@ and inferred labels:: weighted avg 0.67 0.60 0.59 5 -.. topic:: Example: - - * See :ref:`sphx_glr_auto_examples_classification_plot_digits_classification.py` - for an example of classification report usage for - hand-written digits. +.. rubric:: Examples - * See :ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py` - for an example of classification report usage for text - documents. +* See :ref:`sphx_glr_auto_examples_classification_plot_digits_classification.py` + for an example of classification report usage for + hand-written digits. - * See :ref:`sphx_glr_auto_examples_model_selection_plot_grid_search_digits.py` - for an example of classification report usage for - grid search with nested cross-validation. +* See :ref:`sphx_glr_auto_examples_model_selection_plot_grid_search_digits.py` + for an example of classification report usage for + grid search with nested cross-validation. .. _hamming_loss: @@ -705,17 +899,21 @@ The :func:`hamming_loss` computes the average Hamming loss or `Hamming distance `_ between two sets of samples. -If :math:`\hat{y}_j` is the predicted value for the :math:`j`-th label of -a given sample, :math:`y_j` is the corresponding true value, and -:math:`n_\text{labels}` is the number of classes or labels, then the -Hamming loss :math:`L_{Hamming}` between two samples is defined as: +If :math:`\hat{y}_{i,j}` is the predicted value for the :math:`j`-th label of a +given sample :math:`i`, :math:`y_{i,j}` is the corresponding true value, +:math:`n_\text{samples}` is the number of samples and :math:`n_\text{labels}` +is the number of labels, then the Hamming loss :math:`L_{Hamming}` is defined +as: .. math:: - L_{Hamming}(y, \hat{y}) = \frac{1}{n_\text{labels}} \sum_{j=0}^{n_\text{labels} - 1} 1(\hat{y}_j \not= y_j) + L_{Hamming}(y, \hat{y}) = \frac{1}{n_\text{samples} * n_\text{labels}} \sum_{i=0}^{n_\text{samples}-1} \sum_{j=0}^{n_\text{labels} - 1} 1(\hat{y}_{i,j} \not= y_{i,j}) where :math:`1(x)` is the `indicator function -`_. :: +`_. + +The equation above does not hold true in the case of multiclass classification. +Please refer to the note below for more information. :: >>> from sklearn.metrics import hamming_loss >>> y_pred = [1, 2, 3, 4] @@ -795,47 +993,44 @@ score: recall_score Note that the :func:`precision_recall_curve` function is restricted to the -binary case. The :func:`average_precision_score` function works only in -binary classification and multilabel indicator format. The -:func:`plot_precision_recall_curve` function plots the precision recall as -follows. +binary case. The :func:`average_precision_score` function supports multiclass +and multilabel formats by computing each class score in a One-vs-the-rest (OvR) +fashion and averaging them or not depending of its ``average`` argument value. + +The :func:`PrecisionRecallDisplay.from_estimator` and +:func:`PrecisionRecallDisplay.from_predictions` functions will plot the +precision-recall curve as follows. .. image:: ../auto_examples/model_selection/images/sphx_glr_plot_precision_recall_001.png :target: ../auto_examples/model_selection/plot_precision_recall.html#plot-the-precision-recall-curve :scale: 75 :align: center -.. topic:: Examples: +.. rubric:: Examples - * See :ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py` - for an example of :func:`f1_score` usage to classify text - documents. +* See :ref:`sphx_glr_auto_examples_model_selection_plot_grid_search_digits.py` + for an example of :func:`precision_score` and :func:`recall_score` usage + to estimate parameters using grid search with nested cross-validation. - * See :ref:`sphx_glr_auto_examples_model_selection_plot_grid_search_digits.py` - for an example of :func:`precision_score` and :func:`recall_score` usage - to estimate parameters using grid search with nested cross-validation. +* See :ref:`sphx_glr_auto_examples_model_selection_plot_precision_recall.py` + for an example of :func:`precision_recall_curve` usage to evaluate + classifier output quality. - * See :ref:`sphx_glr_auto_examples_model_selection_plot_precision_recall.py` - for an example of :func:`precision_recall_curve` usage to evaluate - classifier output quality. - - -.. topic:: References: - - .. [Manning2008] C.D. Manning, P. Raghavan, H. SchÃŧtze, `Introduction to Information Retrieval - `_, - 2008. - .. [Everingham2010] M. Everingham, L. Van Gool, C.K.I. Williams, J. Winn, A. Zisserman, - `The Pascal Visual Object Classes (VOC) Challenge - `_, - IJCV 2010. - .. [Davis2006] J. Davis, M. Goadrich, `The Relationship Between Precision-Recall and ROC Curves - `_, - ICML 2006. - .. [Flach2015] P.A. Flach, M. Kull, `Precision-Recall-Gain Curves: PR Analysis Done Right - `_, - NIPS 2015. +.. rubric:: References +.. [Manning2008] C.D. Manning, P. Raghavan, H. SchÃŧtze, `Introduction to Information Retrieval + `_, + 2008. +.. [Everingham2010] M. Everingham, L. Van Gool, C.K.I. Williams, J. Winn, A. Zisserman, + `The Pascal Visual Object Classes (VOC) Challenge + `_, + IJCV 2010. +.. [Davis2006] J. Davis, M. Goadrich, `The Relationship Between Precision-Recall and ROC Curves + `_, + ICML 2006. +.. [Flach2015] P.A. Flach, M. Kull, `Precision-Recall-Gain Curves: PR Analysis Done Right + `_, + NIPS 2015. Binary classification ^^^^^^^^^^^^^^^^^^^^^ @@ -856,20 +1051,36 @@ following table: | | Missing result | Correct absence of result| +-------------------+---------------------+--------------------------+ -In this context, we can define the notions of precision, recall and F-measure: +In this context, we can define the notions of precision and recall: .. math:: - \text{precision} = \frac{tp}{tp + fp}, + \text{precision} = \frac{\text{tp}}{\text{tp} + \text{fp}}, .. math:: - \text{recall} = \frac{tp}{tp + fn}, + \text{recall} = \frac{\text{tp}}{\text{tp} + \text{fn}}, + +(Sometimes recall is also called ''sensitivity'') + +F-measure is the weighted harmonic mean of precision and recall, with precision's +contribution to the mean weighted by some parameter :math:`\beta`: .. math:: - F_\beta = (1 + \beta^2) \frac{\text{precision} \times \text{recall}}{\beta^2 \text{precision} + \text{recall}}. + F_\beta = (1 + \beta^2) \frac{\text{precision} \times \text{recall}}{\beta^2 \text{precision} + \text{recall}} +To avoid division by zero when precision and recall are zero, Scikit-Learn calculates F-measure with this +otherwise-equivalent formula: + +.. math:: + + F_\beta = \frac{(1 + \beta^2) \text{tp}}{(1 + \beta^2) \text{tp} + \text{fp} + \beta^2 \text{fn}} + +Note that this formula is still undefined when there are no true positives, false +positives, or false negatives. By default, F-1 for a set of exclusively true negatives +is calculated as 0, however this behavior can be changed using the `zero_division` +parameter. Here are some small examples in binary classification:: >>> from sklearn import metrics @@ -880,15 +1091,15 @@ Here are some small examples in binary classification:: >>> metrics.recall_score(y_true, y_pred) 0.5 >>> metrics.f1_score(y_true, y_pred) - 0.66... + 0.66 >>> metrics.fbeta_score(y_true, y_pred, beta=0.5) - 0.83... + 0.83 >>> metrics.fbeta_score(y_true, y_pred, beta=1) - 0.66... + 0.66 >>> metrics.fbeta_score(y_true, y_pred, beta=2) - 0.55... + 0.55 >>> metrics.precision_recall_fscore_support(y_true, y_pred, beta=0.5) - (array([0.66..., 1. ]), array([1. , 0.5]), array([0.71..., 0.83...]), array([2, 2])) + (array([0.66, 1. ]), array([1. , 0.5]), array([0.71, 0.83]), array([2, 2])) >>> import numpy as np @@ -898,34 +1109,41 @@ Here are some small examples in binary classification:: >>> y_scores = np.array([0.1, 0.4, 0.35, 0.8]) >>> precision, recall, threshold = precision_recall_curve(y_true, y_scores) >>> precision - array([0.66..., 0.5 , 1. , 1. ]) + array([0.5 , 0.66, 0.5 , 1. , 1. ]) >>> recall - array([1. , 0.5, 0.5, 0. ]) + array([1. , 1. , 0.5, 0.5, 0. ]) >>> threshold - array([0.35, 0.4 , 0.8 ]) + array([0.1 , 0.35, 0.4 , 0.8 ]) >>> average_precision_score(y_true, y_scores) - 0.83... + 0.83 Multiclass and multilabel classification ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -In multiclass and multilabel classification task, the notions of precision, +In a multiclass and multilabel classification task, the notions of precision, recall, and F-measures can be applied to each label independently. There are a few ways to combine results across labels, specified by the ``average`` argument to the -:func:`average_precision_score` (multilabel only), :func:`f1_score`, +:func:`average_precision_score`, :func:`f1_score`, :func:`fbeta_score`, :func:`precision_recall_fscore_support`, :func:`precision_score` and :func:`recall_score` functions, as described -:ref:`above `. Note that if all labels are included, "micro"-averaging -in a multiclass setting will produce precision, recall and :math:`F` -that are all identical to accuracy. Also note that "weighted" averaging may -produce an F-score that is not between precision and recall. +:ref:`above `. + +Note the following behaviors when averaging: + +* If all labels are included, "micro"-averaging in a multiclass setting will produce + precision, recall and :math:`F` that are all identical to accuracy. +* "weighted" averaging may produce a F-score that is not between precision and recall. +* "macro" averaging for F-measures is calculated as the arithmetic mean over + per-label/class F-measures, not the harmonic mean over the arithmetic precision and + recall means. Both calculations can be seen in the literature but are not equivalent, + see [OB2019]_ for details. To make this more explicit, consider the following notation: -* :math:`y` the set of *predicted* :math:`(sample, label)` pairs -* :math:`\hat{y}` the set of *true* :math:`(sample, label)` pairs +* :math:`y` the set of *true* :math:`(sample, label)` pairs +* :math:`\hat{y}` the set of *predicted* :math:`(sample, label)` pairs * :math:`L` the set of labels * :math:`S` the set of samples * :math:`y_s` the subset of :math:`y` with sample :math:`s`, @@ -933,10 +1151,10 @@ To make this more explicit, consider the following notation: * :math:`y_l` the subset of :math:`y` with label :math:`l` * similarly, :math:`\hat{y}_s` and :math:`\hat{y}_l` are subsets of :math:`\hat{y}` -* :math:`P(A, B) := \frac{\left| A \cap B \right|}{\left|A\right|}` for some +* :math:`P(A, B) := \frac{\left| A \cap B \right|}{\left|B\right|}` for some sets :math:`A` and :math:`B` -* :math:`R(A, B) := \frac{\left| A \cap B \right|}{\left|B\right|}` - (Conventions vary on handling :math:`B = \emptyset`; this implementation uses +* :math:`R(A, B) := \frac{\left| A \cap B \right|}{\left|A\right|}` + (Conventions vary on handling :math:`A = \emptyset`; this implementation uses :math:`R(A, B):=0`, and similar for :math:`P`.) * :math:`F_\beta(A, B) := \left(1 + \beta^2\right) \frac{P(A, B) \times R(A, B)}{\beta^2 P(A, B) + R(A, B)}` @@ -951,7 +1169,7 @@ Then the metrics are defined as: +---------------+------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------+ |``"macro"`` | :math:`\frac{1}{\left|L\right|} \sum_{l \in L} P(y_l, \hat{y}_l)` | :math:`\frac{1}{\left|L\right|} \sum_{l \in L} R(y_l, \hat{y}_l)` | :math:`\frac{1}{\left|L\right|} \sum_{l \in L} F_\beta(y_l, \hat{y}_l)` | +---------------+------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------+ -|``"weighted"`` | :math:`\frac{1}{\sum_{l \in L} \left|\hat{y}_l\right|} \sum_{l \in L} \left|\hat{y}_l\right| P(y_l, \hat{y}_l)` | :math:`\frac{1}{\sum_{l \in L} \left|\hat{y}_l\right|} \sum_{l \in L} \left|\hat{y}_l\right| R(y_l, \hat{y}_l)` | :math:`\frac{1}{\sum_{l \in L} \left|\hat{y}_l\right|} \sum_{l \in L} \left|\hat{y}_l\right| F_\beta(y_l, \hat{y}_l)`| +|``"weighted"`` | :math:`\frac{1}{\sum_{l \in L} \left|y_l\right|} \sum_{l \in L} \left|y_l\right| P(y_l, \hat{y}_l)` | :math:`\frac{1}{\sum_{l \in L} \left|y_l\right|} \sum_{l \in L} \left|y_l\right| R(y_l, \hat{y}_l)` | :math:`\frac{1}{\sum_{l \in L} \left|y_l\right|} \sum_{l \in L} \left|y_l\right| F_\beta(y_l, \hat{y}_l)` | +---------------+------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------+ |``None`` | :math:`\langle P(y_l, \hat{y}_l) | l \in L \rangle` | :math:`\langle R(y_l, \hat{y}_l) | l \in L \rangle` | :math:`\langle F_\beta(y_l, \hat{y}_l) | l \in L \rangle` | +---------------+------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------+ @@ -960,15 +1178,15 @@ Then the metrics are defined as: >>> y_true = [0, 1, 2, 0, 1, 2] >>> y_pred = [0, 2, 1, 0, 0, 1] >>> metrics.precision_score(y_true, y_pred, average='macro') - 0.22... + 0.22 >>> metrics.recall_score(y_true, y_pred, average='micro') - 0.33... + 0.33 >>> metrics.f1_score(y_true, y_pred, average='weighted') - 0.26... + 0.267 >>> metrics.fbeta_score(y_true, y_pred, average='macro', beta=0.5) - 0.23... + 0.238 >>> metrics.precision_recall_fscore_support(y_true, y_pred, beta=0.5, average=None) - (array([0.66..., 0. , 0. ]), array([1., 0., 0.]), array([0.71..., 0. , 0. ]), array([2, 2, 2]...)) + (array([0.667, 0., 0.]), array([1., 0., 0.]), array([0.714, 0., 0.]), array([2, 2, 2])) For multiclass classification with a "negative class", it is possible to exclude some labels: @@ -979,7 +1197,12 @@ For multiclass classification with a "negative class", it is possible to exclude Similarly, labels not present in the data sample may be accounted for in macro-averaging. >>> metrics.precision_score(y_true, y_pred, labels=[0, 1, 2, 3], average='macro') - 0.166... + 0.166 + +.. rubric:: References + +.. [OB2019] :arxiv:`Opitz, J., & Burst, S. (2019). "Macro f1 and macro f1." + <1911.03347>` .. _jaccard_similarity_score: @@ -990,17 +1213,16 @@ The :func:`jaccard_score` function computes the average of `Jaccard similarity coefficients `_, also called the Jaccard index, between pairs of label sets. -The Jaccard similarity coefficient of the :math:`i`-th samples, -with a ground truth label set :math:`y_i` and predicted label set -:math:`\hat{y}_i`, is defined as +The Jaccard similarity coefficient with a ground truth label set :math:`y` and +predicted label set :math:`\hat{y}`, is defined as .. math:: - J(y_i, \hat{y}_i) = \frac{|y_i \cap \hat{y}_i|}{|y_i \cup \hat{y}_i|}. + J(y, \hat{y}) = \frac{|y \cap \hat{y}|}{|y \cup \hat{y}|}. -:func:`jaccard_score` works like :func:`precision_recall_fscore_support` as a -naively set-wise measure applying natively to binary targets, and extended to -apply to multilabel and multiclass through the use of `average` (see +The :func:`jaccard_score` (like :func:`precision_recall_fscore_support`) applies +natively to binary targets. By computing it set-wise it can be extended to apply +to multilabel and multiclass through the use of `average` (see :ref:`above `). In the binary case:: @@ -1012,14 +1234,19 @@ In the binary case:: >>> y_pred = np.array([[1, 1, 1], ... [1, 0, 0]]) >>> jaccard_score(y_true[0], y_pred[0]) - 0.6666... + 0.6666 + +In the 2D comparison case (e.g. image similarity): + + >>> jaccard_score(y_true, y_pred, average="micro") + 0.6 In the multilabel case with binary label indicators:: >>> jaccard_score(y_true, y_pred, average='samples') - 0.5833... + 0.5833 >>> jaccard_score(y_true, y_pred, average='macro') - 0.6666... + 0.6666 >>> jaccard_score(y_true, y_pred, average=None) array([0.5, 0.5, 1. ]) @@ -1029,11 +1256,11 @@ multilabel problem:: >>> y_pred = [0, 2, 1, 2] >>> y_true = [0, 1, 2, 2] >>> jaccard_score(y_true, y_pred, average=None) - array([1. , 0. , 0.33...]) + array([1. , 0. , 0.33]) >>> jaccard_score(y_true, y_pred, average='macro') - 0.44... + 0.44 >>> jaccard_score(y_true, y_pred, average='micro') - 0.33... + 0.33 .. _hinge_loss: @@ -1046,29 +1273,35 @@ the model and the data using that considers only prediction errors. (Hinge loss is used in maximal margin classifiers such as support vector machines.) -If the labels are encoded with +1 and -1, :math:`y`: is the true -value, and :math:`w` is the predicted decisions as output by -``decision_function``, then the hinge loss is defined as: +If the true label :math:`y_i` of a binary classification task is encoded as +:math:`y_i=\left\{-1, +1\right\}` for every sample :math:`i`; and :math:`w_i` +is the corresponding predicted decision (an array of shape (`n_samples`,) as +output by the `decision_function` method), then the hinge loss is defined as: .. math:: - L_\text{Hinge}(y, w) = \max\left\{1 - wy, 0\right\} = \left|1 - wy\right|_+ + L_\text{Hinge}(y, w) = \frac{1}{n_\text{samples}} \sum_{i=0}^{n_\text{samples}-1} \max\left\{1 - w_i y_i, 0\right\} If there are more than two labels, :func:`hinge_loss` uses a multiclass variant due to Crammer & Singer. -`Here `_ is +`Here `_ is the paper describing it. -If :math:`y_w` is the predicted decision for true label and :math:`y_t` is the -maximum of the predicted decisions for all other labels, where predicted -decisions are output by decision function, then multiclass hinge loss is defined -by: +In this case the predicted decision is an array of shape (`n_samples`, +`n_labels`). If :math:`w_{i, y_i}` is the predicted decision for the true label +:math:`y_i` of the :math:`i`-th sample; and +:math:`\hat{w}_{i, y_i} = \max\left\{w_{i, y_j}~|~y_j \ne y_i \right\}` +is the maximum of the +predicted decisions for all the other labels, then the multi-class hinge loss +is defined by: .. math:: - L_\text{Hinge}(y_w, y_t) = \max\left\{1 + y_t - y_w, 0\right\} + L_\text{Hinge}(y, w) = \frac{1}{n_\text{samples}} + \sum_{i=0}^{n_\text{samples}-1} \max\left\{1 + \hat{w}_{i, y_i} + - w_{i, y_i}, 0\right\} -Here a small example demonstrating the use of the :func:`hinge_loss` function +Here is a small example demonstrating the use of the :func:`hinge_loss` function with a svm classifier in a binary class problem:: >>> from sklearn import svm @@ -1080,9 +1313,9 @@ with a svm classifier in a binary class problem:: LinearSVC(random_state=0) >>> pred_decision = est.decision_function([[-2], [3], [0.5]]) >>> pred_decision - array([-2.18..., 2.36..., 0.09...]) + array([-2.18, 2.36, 0.09]) >>> hinge_loss([-1, 1, 1], pred_decision) - 0.3... + 0.3 Here is an example demonstrating the use of the :func:`hinge_loss` function with a svm classifier in a multiclass problem:: @@ -1095,8 +1328,8 @@ with a svm classifier in a multiclass problem:: LinearSVC() >>> pred_decision = est.decision_function([[-1], [2], [3]]) >>> y_true = [0, 2, 3] - >>> hinge_loss(y_true, pred_decision, labels) - 0.56... + >>> hinge_loss(y_true, pred_decision, labels=labels) + 0.56 .. _log_loss: @@ -1111,30 +1344,30 @@ probability outputs (``predict_proba``) of a classifier instead of its discrete predictions. For binary classification with a true label :math:`y \in \{0,1\}` -and a probability estimate :math:`p = \operatorname{Pr}(y = 1)`, +and a probability estimate :math:`\hat{p} \approx \operatorname{Pr}(y = 1)`, the log loss per sample is the negative log-likelihood of the classifier given the true label: .. math:: - L_{\log}(y, p) = -\log \operatorname{Pr}(y|p) = -(y \log (p) + (1 - y) \log (1 - p)) + L_{\log}(y, \hat{p}) = -\log \operatorname{Pr}(y|\hat{p}) = -(y \log (\hat{p}) + (1 - y) \log (1 - \hat{p})) This extends to the multiclass case as follows. Let the true labels for a set of samples be encoded as a 1-of-K binary indicator matrix :math:`Y`, i.e., :math:`y_{i,k} = 1` if sample :math:`i` has label :math:`k` taken from a set of :math:`K` labels. -Let :math:`P` be a matrix of probability estimates, -with :math:`p_{i,k} = \operatorname{Pr}(y_{i,k} = 1)`. +Let :math:`\hat{P}` be a matrix of probability estimates, +with elements :math:`\hat{p}_{i,k} \approx \operatorname{Pr}(y_{i,k} = 1)`. Then the log loss of the whole set is .. math:: - L_{\log}(Y, P) = -\log \operatorname{Pr}(Y|P) = - \frac{1}{N} \sum_{i=0}^{N-1} \sum_{k=0}^{K-1} y_{i,k} \log p_{i,k} + L_{\log}(Y, \hat{P}) = -\log \operatorname{Pr}(Y|\hat{P}) = - \frac{1}{N} \sum_{i=0}^{N-1} \sum_{k=0}^{K-1} y_{i,k} \log \hat{p}_{i,k} To see how this generalizes the binary log loss given above, note that in the binary case, -:math:`p_{i,0} = 1 - p_{i,1}` and :math:`y_{i,0} = 1 - y_{i,1}`, +:math:`\hat{p}_{i,0} = 1 - \hat{p}_{i,1}` and :math:`y_{i,0} = 1 - y_{i,1}`, so expanding the inner sum over :math:`y_{i,k} \in \{0,1\}` gives the binary log loss. @@ -1146,7 +1379,7 @@ method. >>> y_true = [0, 0, 1, 1] >>> y_pred = [[.9, .1], [.8, .2], [.3, .7], [.01, .99]] >>> log_loss(y_true, y_pred) - 0.1738... + 0.1738 The first ``[.9, .1]`` in ``y_pred`` denotes 90% probability that the first sample has label 0. The log loss is non-negative. @@ -1201,8 +1434,9 @@ Then the multiclass MCC is defined as: When there are more than two labels, the value of the MCC will no longer range between -1 and +1. Instead the minimum value will be somewhere between -1 and 0 -depending on the number and distribution of ground true labels. The maximum +depending on the number and distribution of ground truth labels. The maximum value is always +1. +For additional information, see [WikipediaMCC2021]_. Here is a small example illustrating the usage of the :func:`matthews_corrcoef` function: @@ -1211,7 +1445,14 @@ function: >>> y_true = [+1, +1, +1, -1] >>> y_pred = [+1, -1, +1, +1] >>> matthews_corrcoef(y_true, y_pred) - -0.33... + -0.33 + +.. rubric:: References + +.. [WikipediaMCC2021] Wikipedia contributors. Phi coefficient. + Wikipedia, The Free Encyclopedia. April 21, 2021, 12:21 CEST. + Available at: https://en.wikipedia.org/wiki/Phi_coefficient + Accessed April 21, 2021. .. _multilabel_confusion_matrix: @@ -1336,10 +1577,10 @@ Quoting Wikipedia : positive rate), at various threshold settings. TPR is also known as sensitivity, and FPR is one minus the specificity or true negative rate." -This function requires the true binary -value and the target scores, which can either be probability estimates of the -positive class, confidence values, or binary decisions. -Here is a small example of how to use the :func:`roc_curve` function:: +This function requires the true binary value and the target scores, which can +either be probability estimates of the positive class, confidence values, or +binary decisions. Here is a small example of how to use the :func:`roc_curve` +function:: >>> import numpy as np >>> from sklearn.metrics import roc_curve @@ -1351,25 +1592,29 @@ Here is a small example of how to use the :func:`roc_curve` function:: >>> tpr array([0. , 0.5, 0.5, 1. , 1. ]) >>> thresholds - array([1.8 , 0.8 , 0.4 , 0.35, 0.1 ]) + array([ inf, 0.8 , 0.4 , 0.35, 0.1 ]) + +Compared to metrics such as the subset accuracy, the Hamming loss, or the +F1 score, ROC doesn't require optimizing a threshold for each label. + +The :func:`roc_auc_score` function, denoted by ROC-AUC or AUROC, computes the +area under the ROC curve. By doing so, the curve information is summarized in +one number. -This figure shows an example of such an ROC curve: +The following figure shows the ROC curve and ROC-AUC score for a classifier +aimed to distinguish the virginica flower from the rest of the species in the +:ref:`iris_dataset`: .. image:: ../auto_examples/model_selection/images/sphx_glr_plot_roc_001.png :target: ../auto_examples/model_selection/plot_roc.html :scale: 75 :align: center -The :func:`roc_auc_score` function computes the area under the receiver -operating characteristic (ROC) curve, which is also denoted by -AUC or AUROC. By computing the -area under the roc curve, the curve information is summarized in one number. + + For more information see the `Wikipedia article on AUC `_. -Compared to metrics such as the subset accuracy, the Hamming loss, or the -F1 score, ROC doesn't require optimizing a threshold for each label. - .. _roc_auc_binary: Binary case @@ -1387,7 +1632,7 @@ Therefore, the `y_score` parameter is of size (n_samples,). >>> from sklearn.linear_model import LogisticRegression >>> from sklearn.metrics import roc_auc_score >>> X, y = load_breast_cancer(return_X_y=True) - >>> clf = LogisticRegression(solver="liblinear").fit(X, y) + >>> clf = LogisticRegression().fit(X, y) >>> clf.classes_ array([0, 1]) @@ -1395,12 +1640,12 @@ We can use the probability estimates corresponding to `clf.classes_[1]`. >>> y_score = clf.predict_proba(X)[:, 1] >>> roc_auc_score(y, y_score) - 0.99... + 0.99 Otherwise, we can use the non-thresholded decision values >>> roc_auc_score(y, clf.decision_function(X)) - 0.99... + 0.99 .. _roc_auc_multiclass: @@ -1417,50 +1662,57 @@ correspond to the probability estimates that a sample belongs to a particular class. The OvO and OvR algorithms support weighting uniformly (``average='macro'``) and by prevalence (``average='weighted'``). -**One-vs-one Algorithm**: Computes the average AUC of all possible pairwise -combinations of classes. [HT2001]_ defines a multiclass AUC metric weighted -uniformly: +.. dropdown:: One-vs-one Algorithm -.. math:: + Computes the average AUC of all possible pairwise + combinations of classes. [HT2001]_ defines a multiclass AUC metric weighted + uniformly: - \frac{1}{c(c-1)}\sum_{j=1}^{c}\sum_{k > j}^c (\text{AUC}(j | k) + - \text{AUC}(k | j)) + .. math:: -where :math:`c` is the number of classes and :math:`\text{AUC}(j | k)` is the -AUC with class :math:`j` as the positive class and class :math:`k` as the -negative class. In general, -:math:`\text{AUC}(j | k) \neq \text{AUC}(k | j))` in the multiclass -case. This algorithm is used by setting the keyword argument ``multiclass`` -to ``'ovo'`` and ``average`` to ``'macro'``. + \frac{1}{c(c-1)}\sum_{j=1}^{c}\sum_{k > j}^c (\text{AUC}(j | k) + + \text{AUC}(k | j)) -The [HT2001]_ multiclass AUC metric can be extended to be weighted by the -prevalence: + where :math:`c` is the number of classes and :math:`\text{AUC}(j | k)` is the + AUC with class :math:`j` as the positive class and class :math:`k` as the + negative class. In general, + :math:`\text{AUC}(j | k) \neq \text{AUC}(k | j))` in the multiclass + case. This algorithm is used by setting the keyword argument ``multiclass`` + to ``'ovo'`` and ``average`` to ``'macro'``. -.. math:: + The [HT2001]_ multiclass AUC metric can be extended to be weighted by the + prevalence: - \frac{1}{c(c-1)}\sum_{j=1}^{c}\sum_{k > j}^c p(j \cup k)( - \text{AUC}(j | k) + \text{AUC}(k | j)) + .. math:: -where :math:`c` is the number of classes. This algorithm is used by setting -the keyword argument ``multiclass`` to ``'ovo'`` and ``average`` to -``'weighted'``. The ``'weighted'`` option returns a prevalence-weighted average -as described in [FC2009]_. + \frac{1}{c(c-1)}\sum_{j=1}^{c}\sum_{k > j}^c p(j \cup k)( + \text{AUC}(j | k) + \text{AUC}(k | j)) -**One-vs-rest Algorithm**: Computes the AUC of each class against the rest -[PD2000]_. The algorithm is functionally the same as the multilabel case. To -enable this algorithm set the keyword argument ``multiclass`` to ``'ovr'``. -Like OvO, OvR supports two types of averaging: ``'macro'`` [F2006]_ and -``'weighted'`` [F2001]_. + where :math:`c` is the number of classes. This algorithm is used by setting + the keyword argument ``multiclass`` to ``'ovo'`` and ``average`` to + ``'weighted'``. The ``'weighted'`` option returns a prevalence-weighted average + as described in [FC2009]_. -In applications where a high false positive rate is not tolerable the parameter -``max_fpr`` of :func:`roc_auc_score` can be used to summarize the ROC curve up -to the given limit. +.. dropdown:: One-vs-rest Algorithm + Computes the AUC of each class against the rest + [PD2000]_. The algorithm is functionally the same as the multilabel case. To + enable this algorithm set the keyword argument ``multiclass`` to ``'ovr'``. + Additionally to ``'macro'`` [F2006]_ and ``'weighted'`` [F2001]_ averaging, OvR + supports ``'micro'`` averaging. -.. image:: ../auto_examples/model_selection/images/sphx_glr_plot_roc_002.png - :target: ../auto_examples/model_selection/plot_roc.html - :scale: 75 - :align: center + In applications where a high false positive rate is not tolerable the parameter + ``max_fpr`` of :func:`roc_auc_score` can be used to summarize the ROC curve up + to the given limit. + + The following figure shows the micro-averaged ROC curve and its corresponding + ROC-AUC score for a classifier aimed to distinguish the different species in + the :ref:`iris_dataset`: + + .. image:: ../auto_examples/model_selection/images/sphx_glr_plot_roc_002.png + :target: ../auto_examples/model_selection/plot_roc.html + :scale: 75 + :align: center .. _roc_auc_multilabel: @@ -1476,11 +1728,11 @@ class with the greater label for each output. >>> from sklearn.datasets import make_multilabel_classification >>> from sklearn.multioutput import MultiOutputClassifier >>> X, y = make_multilabel_classification(random_state=0) - >>> inner_clf = LogisticRegression(solver="liblinear", random_state=0) + >>> inner_clf = LogisticRegression(random_state=0) >>> clf = MultiOutputClassifier(inner_clf).fit(X, y) >>> y_score = np.transpose([y_pred[:, 1] for y_pred in clf.predict_proba(X)]) >>> roc_auc_score(y, y_score, average=None) - array([0.82..., 0.86..., 0.94..., 0.85... , 0.94...]) + array([0.828, 0.851, 0.94, 0.87, 0.95]) And the decision values do not require such processing. @@ -1488,46 +1740,45 @@ And the decision values do not require such processing. >>> clf = RidgeClassifierCV().fit(X, y) >>> y_score = clf.decision_function(X) >>> roc_auc_score(y, y_score, average=None) - array([0.81..., 0.84... , 0.93..., 0.87..., 0.94...]) + array([0.82, 0.85, 0.93, 0.87, 0.94]) -.. topic:: Examples: +.. rubric:: Examples - * See :ref:`sphx_glr_auto_examples_model_selection_plot_roc.py` - for an example of using ROC to - evaluate the quality of the output of a classifier. +* See :ref:`sphx_glr_auto_examples_model_selection_plot_roc.py` for an example of + using ROC to evaluate the quality of the output of a classifier. - * See :ref:`sphx_glr_auto_examples_model_selection_plot_roc_crossval.py` - for an example of using ROC to - evaluate classifier output quality, using cross-validation. +* See :ref:`sphx_glr_auto_examples_model_selection_plot_roc_crossval.py` for an + example of using ROC to evaluate classifier output quality, using cross-validation. - * See :ref:`sphx_glr_auto_examples_applications_plot_species_distribution_modeling.py` - for an example of using ROC to - model species distribution. +* See :ref:`sphx_glr_auto_examples_applications_plot_species_distribution_modeling.py` + for an example of using ROC to model species distribution. -.. topic:: References: +.. rubric:: References - .. [HT2001] Hand, D.J. and Till, R.J., (2001). `A simple generalisation - of the area under the ROC curve for multiple class classification problems. - `_ - Machine learning, 45(2), pp.171-186. +.. [HT2001] Hand, D.J. and Till, R.J., (2001). `A simple generalisation + of the area under the ROC curve for multiple class classification problems. + `_ + Machine learning, 45(2), pp. 171-186. - .. [FC2009] Ferri, Cèsar & Hernandez-Orallo, Jose & Modroiu, R. (2009). - `An Experimental Comparison of Performance Measures for Classification. - `_ - Pattern Recognition Letters. 30. 27-38. +.. [FC2009] Ferri, Cèsar & Hernandez-Orallo, Jose & Modroiu, R. (2009). + `An Experimental Comparison of Performance Measures for Classification. + `_ + Pattern Recognition Letters. 30. 27-38. - .. [PD2000] Provost, F., Domingos, P. (2000). Well-trained PETs: Improving - probability estimation trees (Section 6.2), CeDER Working Paper #IS-00-04, - Stern School of Business, New York University. +.. [PD2000] Provost, F., Domingos, P. (2000). `Well-trained PETs: Improving + probability estimation trees + `_ + (Section 6.2), CeDER Working Paper #IS-00-04, Stern School of Business, + New York University. - .. [F2006] Fawcett, T., 2006. `An introduction to ROC analysis. - `_ - Pattern Recognition Letters, 27(8), pp. 861-874. +.. [F2006] Fawcett, T., 2006. `An introduction to ROC analysis. + `_ + Pattern Recognition Letters, 27(8), pp. 861-874. - .. [F2001] Fawcett, T., 2001. `Using rule sets to maximize - ROC performance `_ - In Data Mining, 2001. - Proceedings IEEE International Conference, pp. 131-138. +.. [F2001] Fawcett, T., 2001. `Using rule sets to maximize + ROC performance `_ + In Data Mining, 2001. + Proceedings IEEE International Conference, pp. 131-138. .. _det_curve: @@ -1563,59 +1814,57 @@ same classification task: :scale: 75 :align: center -**Properties:** +.. dropdown:: Properties -* DET curves form a linear curve in normal deviate scale if the detection - scores are normally (or close-to normally) distributed. - It was shown by [Navratil2007]_ that the reverse it not necessarily true and - even more general distributions are able produce linear DET curves. + * DET curves form a linear curve in normal deviate scale if the detection + scores are normally (or close-to normally) distributed. + It was shown by [Navratil2007]_ that the reverse is not necessarily true and + even more general distributions are able to produce linear DET curves. -* The normal deviate scale transformation spreads out the points such that a - comparatively larger space of plot is occupied. - Therefore curves with similar classification performance might be easier to - distinguish on a DET plot. + * The normal deviate scale transformation spreads out the points such that a + comparatively larger space of plot is occupied. + Therefore curves with similar classification performance might be easier to + distinguish on a DET plot. -* With False Negative Rate being "inverse" to True Positive Rate the point - of perfection for DET curves is the origin (in contrast to the top left - corner for ROC curves). + * With False Negative Rate being "inverse" to True Positive Rate the point + of perfection for DET curves is the origin (in contrast to the top left + corner for ROC curves). -**Applications and limitations:** +.. dropdown:: Applications and limitations -DET curves are intuitive to read and hence allow quick visual assessment of a -classifier's performance. -Additionally DET curves can be consulted for threshold analysis and operating -point selection. -This is particularly helpful if a comparison of error types is required. + DET curves are intuitive to read and hence allow quick visual assessment of a + classifier's performance. + Additionally DET curves can be consulted for threshold analysis and operating + point selection. + This is particularly helpful if a comparison of error types is required. -One the other hand DET curves do not provide their metric as a single number. -Therefore for either automated evaluation or comparison to other -classification tasks metrics like the derived area under ROC curve might be -better suited. + On the other hand DET curves do not provide their metric as a single number. + Therefore for either automated evaluation or comparison to other + classification tasks metrics like the derived area under ROC curve might be + better suited. -.. topic:: Examples: +.. rubric:: Examples - * See :ref:`sphx_glr_auto_examples_model_selection_plot_det.py` - for an example comparison between receiver operating characteristic (ROC) - curves and Detection error tradeoff (DET) curves. +* See :ref:`sphx_glr_auto_examples_model_selection_plot_det.py` + for an example comparison between receiver operating characteristic (ROC) + curves and Detection error tradeoff (DET) curves. -.. topic:: References: +.. rubric:: References - .. [WikipediaDET2017] Wikipedia contributors. Detection error tradeoff. - Wikipedia, The Free Encyclopedia. September 4, 2017, 23:33 UTC. - Available at: https://en.wikipedia.org/w/index.php?title=Detection_error_tradeoff&oldid=798982054. - Accessed February 19, 2018. +.. [WikipediaDET2017] Wikipedia contributors. Detection error tradeoff. + Wikipedia, The Free Encyclopedia. September 4, 2017, 23:33 UTC. + Available at: https://en.wikipedia.org/w/index.php?title=Detection_error_tradeoff&oldid=798982054. + Accessed February 19, 2018. - .. [Martin1997] A. Martin, G. Doddington, T. Kamm, M. Ordowski, and M. Przybocki, - `The DET Curve in Assessment of Detection Task Performance - `_, - NIST 1997. +.. [Martin1997] A. Martin, G. Doddington, T. Kamm, M. Ordowski, and M. Przybocki, + `The DET Curve in Assessment of Detection Task Performance + `_, NIST 1997. - .. [Navratil2007] J. Navractil and D. Klusacek, - "`On Linear DETs, - `_" - 2007 IEEE International Conference on Acoustics, - Speech and Signal Processing - ICASSP '07, Honolulu, - HI, 2007, pp. IV-229-IV-232. +.. [Navratil2007] J. Navratil and D. Klusacek, + `"On Linear DETs" `_, + 2007 IEEE International Conference on Acoustics, + Speech and Signal Processing - ICASSP '07, Honolulu, + HI, 2007, pp. IV-229-IV-232. .. _zero_one_loss: @@ -1631,7 +1880,7 @@ In multilabel classification, the :func:`zero_one_loss` scores a subset as one if its labels strictly match the predictions, and as a zero if there are any errors. By default, the function returns the percentage of imperfectly predicted subsets. To get the count of such subsets instead, set -``normalize`` to ``False`` +``normalize`` to ``False``. If :math:`\hat{y}_i` is the predicted value of the :math:`i`-th sample and :math:`y_i` is the corresponding true value, @@ -1639,10 +1888,11 @@ then the 0-1 loss :math:`L_{0-1}` is defined as: .. math:: - L_{0-1}(y_i, \hat{y}_i) = 1(\hat{y}_i \not= y_i) + L_{0-1}(y, \hat{y}) = \frac{1}{n_\text{samples}} \sum_{i=0}^{n_\text{samples}-1} 1(\hat{y}_i \not= y_i) where :math:`1(x)` is the `indicator function -`_. +`_. The zero-one +loss can also be computed as :math:`\text{zero-one loss} = 1 - \text{accuracy}`. >>> from sklearn.metrics import zero_one_loss @@ -1651,7 +1901,7 @@ where :math:`1(x)` is the `indicator function >>> zero_one_loss(y_true, y_pred) 0.25 >>> zero_one_loss(y_true, y_pred, normalize=False) - 1 + 1.0 In the multilabel case with binary label indicators, where the first label set [0,1] has an error:: @@ -1660,54 +1910,77 @@ set [0,1] has an error:: 0.5 >>> zero_one_loss(np.array([[0, 1], [1, 1]]), np.ones((2, 2)), normalize=False) - 1 + 1.0 -.. topic:: Example: +.. rubric:: Examples - * See :ref:`sphx_glr_auto_examples_feature_selection_plot_rfe_with_cross_validation.py` - for an example of zero one loss usage to perform recursive feature - elimination with cross-validation. +* See :ref:`sphx_glr_auto_examples_feature_selection_plot_rfe_with_cross_validation.py` + for an example of zero one loss usage to perform recursive feature + elimination with cross-validation. .. _brier_score_loss: Brier score loss ---------------- -The :func:`brier_score_loss` function computes the -`Brier score `_ -for binary classes [Brier1950]_. Quoting Wikipedia: +The :func:`brier_score_loss` function computes the `Brier score +`_ for binary and multiclass +probabilistic predictions and is equivalent to the mean squared error. +Quoting Wikipedia: + + "The Brier score is a strictly proper scoring rule that measures the accuracy of + probabilistic predictions. [...] [It] is applicable to tasks in which predictions + must assign probabilities to a set of mutually exclusive discrete outcomes or + classes." + +Let the true labels for a set of :math:`N` data points be encoded as a 1-of-K binary +indicator matrix :math:`Y`, i.e., :math:`y_{i,k} = 1` if sample :math:`i` has +label :math:`k` taken from a set of :math:`K` labels. Let :math:`\hat{P}` be a matrix +of probability estimates with elements :math:`\hat{p}_{i,k} \approx \operatorname{Pr}(y_{i,k} = 1)`. +Following the original definition by [Brier1950]_, the Brier score is given by: + +.. math:: + + BS(Y, \hat{P}) = \frac{1}{N}\sum_{i=0}^{N-1}\sum_{k=0}^{K-1}(y_{i,k} - \hat{p}_{i,k})^{2} - "The Brier score is a proper score function that measures the accuracy of - probabilistic predictions. It is applicable to tasks in which predictions - must assign probabilities to a set of mutually exclusive discrete outcomes." +The Brier score lies in the interval :math:`[0, 2]` and the lower the value the +better the probability estimates are (the mean squared difference is smaller). +Actually, the Brier score is a strictly proper scoring rule, meaning that it +achieves the best score only when the estimated probabilities equal the +true ones. -This function returns the mean squared error of the actual outcome -:math:`y \in \{0,1\}` and the predicted probability estimate -:math:`p = \operatorname{Pr}(y = 1)` (:term:`predict_proba`) as outputted by: +Note that in the binary case, the Brier score is usually divided by two and +ranges between :math:`[0,1]`. For binary targets :math:`y_i \in {0, 1}` and +probability estimates :math:`\hat{p}_i \approx \operatorname{Pr}(y_i = 1)` +for the positive class, the Brier score is then equal to: .. math:: - BS = \frac{1}{n_{\text{samples}}} \sum_{i=0}^{n_{\text{samples}} - 1}(y_i - p_i)^2 + BS(y, \hat{p}) = \frac{1}{N} \sum_{i=0}^{N - 1}(y_i - \hat{p}_i)^2 -The Brier score loss is also between 0 to 1 and the lower the value (the mean -square difference is smaller), the more accurate the prediction is. +The :func:`brier_score_loss` function computes the Brier score given the +ground-truth labels and predicted probabilities, as returned by an estimator's +``predict_proba`` method. The `scale_by_half` parameter controls which of the +two above definitions to follow. -Here is a small example of usage of this function:: >>> import numpy as np >>> from sklearn.metrics import brier_score_loss >>> y_true = np.array([0, 1, 1, 0]) >>> y_true_categorical = np.array(["spam", "ham", "ham", "spam"]) >>> y_prob = np.array([0.1, 0.9, 0.8, 0.4]) - >>> y_pred = np.array([0, 1, 1, 0]) >>> brier_score_loss(y_true, y_prob) 0.055 >>> brier_score_loss(y_true, 1 - y_prob, pos_label=0) 0.055 >>> brier_score_loss(y_true_categorical, y_prob, pos_label="ham") 0.055 - >>> brier_score_loss(y_true, y_prob > 0.5) - 0.0 + >>> brier_score_loss( + ... ["eggs", "ham", "spam"], + ... [[0.8, 0.1, 0.1], [0.2, 0.7, 0.1], [0.2, 0.2, 0.6]], + ... labels=["eggs", "ham", "spam"], + ... ) + 0.146 The Brier score can be used to assess how well a classifier is calibrated. However, a lower Brier score loss does not always mean a better calibration. @@ -1722,28 +1995,220 @@ necessarily mean a better calibrated model. "Only when refinement loss remains the same does a lower Brier score loss always mean better calibration" [Bella2012]_, [Flach2008]_. -.. topic:: Example: +.. rubric:: Examples + +* See :ref:`sphx_glr_auto_examples_calibration_plot_calibration.py` + for an example of Brier score loss usage to perform probability + calibration of classifiers. + +.. rubric:: References - * See :ref:`sphx_glr_auto_examples_calibration_plot_calibration.py` - for an example of Brier score loss usage to perform probability - calibration of classifiers. +.. [Brier1950] G. Brier, `Verification of forecasts expressed in terms of probability + `_, + Monthly weather review 78.1 (1950) -.. topic:: References: +.. [Bella2012] Bella, Ferri, HernÃĄndez-Orallo, and Ramírez-Quintana + `"Calibration of Machine Learning Models" + `_ + in Khosrow-Pour, M. "Machine learning: concepts, methodologies, tools + and applications." Hershey, PA: Information Science Reference (2012). - .. [Brier1950] G. Brier, `Verification of forecasts expressed in terms of - probability - `_, - Monthly weather review 78.1 (1950) +.. [Flach2008] Flach, Peter, and Edson Matsubara. `"On classification, ranking, + and probability estimation." `_ + Dagstuhl Seminar Proceedings. Schloss Dagstuhl-Leibniz-Zentrum fÃŧr Informatik (2008). - .. [Bella2012] Bella, Ferri, HernÃĄndez-Orallo, and Ramírez-Quintana - `"Calibration of Machine Learning Models" - `_ - in Khosrow-Pour, M. "Machine learning: concepts, methodologies, tools - and applications." Hershey, PA: Information Science Reference (2012). +.. _class_likelihood_ratios: + +Class likelihood ratios +----------------------- + +The :func:`class_likelihood_ratios` function computes the `positive and negative +likelihood ratios +`_ +:math:`LR_\pm` for binary classes, which can be interpreted as the ratio of +post-test to pre-test odds as explained below. As a consequence, this metric is +invariant w.r.t. the class prevalence (the number of samples in the positive +class divided by the total number of samples) and **can be extrapolated between +populations regardless of any possible class imbalance.** + +The :math:`LR_\pm` metrics are therefore very useful in settings where the data +available to learn and evaluate a classifier is a study population with nearly +balanced classes, such as a case-control study, while the target application, +i.e. the general population, has very low prevalence. + +The positive likelihood ratio :math:`LR_+` is the probability of a classifier to +correctly predict that a sample belongs to the positive class divided by the +probability of predicting the positive class for a sample belonging to the +negative class: + +.. math:: + + LR_+ = \frac{\text{PR}(P+|T+)}{\text{PR}(P+|T-)}. + +The notation here refers to predicted (:math:`P`) or true (:math:`T`) label and +the sign :math:`+` and :math:`-` refer to the positive and negative class, +respectively, e.g. :math:`P+` stands for "predicted positive". + +Analogously, the negative likelihood ratio :math:`LR_-` is the probability of a +sample of the positive class being classified as belonging to the negative class +divided by the probability of a sample of the negative class being correctly +classified: + +.. math:: + + LR_- = \frac{\text{PR}(P-|T+)}{\text{PR}(P-|T-)}. + +For classifiers above chance :math:`LR_+` above 1 **higher is better**, while +:math:`LR_-` ranges from 0 to 1 and **lower is better**. +Values of :math:`LR_\pm\approx 1` correspond to chance level. + +Notice that probabilities differ from counts, for instance +:math:`\operatorname{PR}(P+|T+)` is not equal to the number of true positive +counts ``tp`` (see `the wikipedia page +`_ for +the actual formulas). + +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_model_selection_plot_likelihood_ratios.py` + +.. dropdown:: Interpretation across varying prevalence + + Both class likelihood ratios are interpretable in terms of an odds ratio + (pre-test and post-tests): + + .. math:: + + \text{post-test odds} = \text{Likelihood ratio} \times \text{pre-test odds}. + + Odds are in general related to probabilities via + + .. math:: + + \text{odds} = \frac{\text{probability}}{1 - \text{probability}}, + + or equivalently + + .. math:: + + \text{probability} = \frac{\text{odds}}{1 + \text{odds}}. + + On a given population, the pre-test probability is given by the prevalence. By + converting odds to probabilities, the likelihood ratios can be translated into a + probability of truly belonging to either class before and after a classifier + prediction: + + .. math:: + + \text{post-test odds} = \text{Likelihood ratio} \times + \frac{\text{pre-test probability}}{1 - \text{pre-test probability}}, + + .. math:: + + \text{post-test probability} = \frac{\text{post-test odds}}{1 + \text{post-test odds}}. + +.. dropdown:: Mathematical divergences + + The positive likelihood ratio (`LR+`) is undefined when :math:`fp=0`, meaning the + classifier does not misclassify any negative labels as positives. This condition can + either indicate a perfect identification of all the negative cases or, if there are + also no true positive predictions (:math:`tp=0`), that the classifier does not predict + the positive class at all. In the first case, `LR+` can be interpreted as `np.inf`, in + the second case (for instance, with highly imbalanced data) it can be interpreted as + `np.nan`. + + The negative likelihood ratio (`LR-`) is undefined when :math:`tn=0`. Such + divergence is invalid, as :math:`LR_- > 1.0` would indicate an increase in the odds of + a sample belonging to the positive class after being classified as negative, as if the + act of classifying caused the positive condition. This includes the case of a + :class:`~sklearn.dummy.DummyClassifier` that always predicts the positive class + (i.e. when :math:`tn=fn=0`). + + Both class likelihood ratios (`LR+ and LR-`) are undefined when :math:`tp=fn=0`, which + means that no samples of the positive class were present in the test set. This can + happen when cross-validating on highly imbalanced data and also leads to a division by + zero. + + If a division by zero occurs and `raise_warning` is set to `True` (default), + :func:`class_likelihood_ratios` raises an `UndefinedMetricWarning` and returns + `np.nan` by default to avoid pollution when averaging over cross-validation folds. + Users can set return values in case of a division by zero with the + `replace_undefined_by` param. + + For a worked-out demonstration of the :func:`class_likelihood_ratios` function, + see the example below. + +.. dropdown:: References + + * `Wikipedia entry for Likelihood ratios in diagnostic testing + `_ + + * Brenner, H., & Gefeller, O. (1997). + Variation of sensitivity, specificity, likelihood ratios and predictive + values with disease prevalence. Statistics in medicine, 16(9), 981-991. + + +.. _d2_score_classification: + +D² score for classification +--------------------------- + +The D² score computes the fraction of deviance explained. +It is a generalization of R², where the squared error is generalized and replaced +by a classification deviance of choice :math:`\text{dev}(y, \hat{y})` +(e.g., Log loss). D² is a form of a *skill score*. +It is calculated as + +.. math:: + + D^2(y, \hat{y}) = 1 - \frac{\text{dev}(y, \hat{y})}{\text{dev}(y, y_{\text{null}})} \,. + +Where :math:`y_{\text{null}}` is the optimal prediction of an intercept-only model +(e.g., the per-class proportion of `y_true` in the case of the Log loss). + +Like R², the best possible score is 1.0 and it can be negative (because the +model can be arbitrarily worse). A constant model that always predicts +:math:`y_{\text{null}}`, disregarding the input features, would get a D² score +of 0.0. + +.. dropdown:: D2 log loss score + + The :func:`d2_log_loss_score` function implements the special case + of D² with the log loss, see :ref:`log_loss`, i.e.: + + .. math:: + + \text{dev}(y, \hat{y}) = \text{log_loss}(y, \hat{y}). + + Here are some usage examples of the :func:`d2_log_loss_score` function:: + + >>> from sklearn.metrics import d2_log_loss_score + >>> y_true = [1, 1, 2, 3] + >>> y_pred = [ + ... [0.5, 0.25, 0.25], + ... [0.5, 0.25, 0.25], + ... [0.5, 0.25, 0.25], + ... [0.5, 0.25, 0.25], + ... ] + >>> d2_log_loss_score(y_true, y_pred) + 0.0 + >>> y_true = [1, 2, 3] + >>> y_pred = [ + ... [0.98, 0.01, 0.01], + ... [0.01, 0.98, 0.01], + ... [0.01, 0.01, 0.98], + ... ] + >>> d2_log_loss_score(y_true, y_pred) + 0.981 + >>> y_true = [1, 2, 3] + >>> y_pred = [ + ... [0.1, 0.6, 0.3], + ... [0.1, 0.6, 0.3], + ... [0.4, 0.5, 0.1], + ... ] + >>> d2_log_loss_score(y_true, y_pred) + -0.552 - .. [Flach2008] Flach, Peter, and Edson Matsubara. `"On classification, ranking, - and probability estimation." `_ - Dagstuhl Seminar Proceedings. Schloss Dagstuhl-Leibniz-Zentrum fr Informatik (2008). .. _multilabel_ranking_metrics: @@ -1765,7 +2230,7 @@ The :func:`coverage_error` function computes the average number of labels that have to be included in the final prediction such that all true labels are predicted. This is useful if you want to know how many top-scored-labels you have to predict in average without missing any true one. The best value -of this metrics is thus the average number of true labels. +of this metric is thus the average number of true labels. .. note:: @@ -1841,7 +2306,7 @@ Here is a small example of usage of this function:: >>> y_true = np.array([[1, 0, 0], [0, 0, 1]]) >>> y_score = np.array([[0.75, 0.5, 1], [1, 0.2, 0.1]]) >>> label_ranking_average_precision_score(y_true, y_score) - 0.416... + 0.416 .. _label_ranking_loss: @@ -1876,18 +2341,19 @@ Here is a small example of usage of this function:: >>> y_true = np.array([[1, 0, 0], [0, 0, 1]]) >>> y_score = np.array([[0.75, 0.5, 1], [1, 0.2, 0.1]]) >>> label_ranking_loss(y_true, y_score) - 0.75... + 0.75 >>> # With the following prediction, we have perfect and minimal loss >>> y_score = np.array([[1.0, 0.1, 0.2], [0.1, 0.2, 0.9]]) >>> label_ranking_loss(y_true, y_score) 0.0 -.. topic:: References: +.. dropdown:: References * Tsoumakas, G., Katakis, I., & Vlahavas, I. (2010). Mining multi-label data. In Data mining and knowledge discovery handbook (pp. 667-685). Springer US. + .. _ndcg: Normalized Discounted Cumulative Gain @@ -1906,7 +2372,7 @@ engine algorithms or related applications. Using a graded relevance scale of documents in a search-engine result set, DCG measures the usefulness, or gain, of a document based on its position in the result list. The gain is accumulated from the top of the result list to the bottom, with the gain of each result -discounted at lower ranks" +discounted at lower ranks." DCG orders the true targets (e.g. relevance of query answers) in the predicted order, then multiplies them by a logarithmic decay and sums the result. The sum @@ -1932,7 +2398,7 @@ DCG score is and the NDCG score is the DCG score divided by the DCG score obtained for :math:`y`. -.. topic:: References: +.. dropdown:: References * `Wikipedia entry for Discounted Cumulative Gain `_ @@ -1950,6 +2416,7 @@ and the NDCG score is the DCG score divided by the DCG score obtained for European conference on information retrieval (pp. 414-421). Springer, Berlin, Heidelberg. + .. _regression_metrics: Regression metrics @@ -1960,17 +2427,18 @@ Regression metrics The :mod:`sklearn.metrics` module implements several loss, score, and utility functions to measure regression performance. Some of those have been enhanced to handle the multioutput case: :func:`mean_squared_error`, -:func:`mean_absolute_error`, :func:`explained_variance_score`, -:func:`r2_score` and :func:`mean_pinball_loss`. +:func:`mean_absolute_error`, :func:`r2_score`, +:func:`explained_variance_score`, :func:`mean_pinball_loss`, :func:`d2_pinball_score` +and :func:`d2_absolute_error_score`. -These functions have an ``multioutput`` keyword argument which specifies the +These functions have a ``multioutput`` keyword argument which specifies the way the scores or losses for each individual target should be averaged. The default is ``'uniform_average'``, which specifies a uniformly weighted mean over outputs. If an ``ndarray`` of shape ``(n_outputs,)`` is passed, then its entries are interpreted as weights and an according weighted average is -returned. If ``multioutput`` is ``'raw_values'`` is specified, then all -unaltered individual scores or losses will be returned in an array of shape +returned. If ``multioutput`` is ``'raw_values'``, then all unaltered +individual scores or losses will be returned in an array of shape ``(n_outputs,)``. @@ -1979,76 +2447,89 @@ value ``'variance_weighted'`` for the ``multioutput`` parameter. This option leads to a weighting of each individual score by the variance of the corresponding target variable. This setting quantifies the globally captured unscaled variance. If the target variables are of different scale, then this -score puts more importance on well explaining the higher variance variables. -``multioutput='variance_weighted'`` is the default value for :func:`r2_score` -for backward compatibility. This will be changed to ``uniform_average`` in the -future. +score puts more importance on explaining the higher variance variables. -.. _explained_variance_score: - -Explained variance score -------------------------- - -The :func:`explained_variance_score` computes the `explained variance -regression score `_. +.. _r2_score: -If :math:`\hat{y}` is the estimated target output, :math:`y` the corresponding -(correct) target output, and :math:`Var` is `Variance -`_, the square of the standard deviation, -then the explained variance is estimated as follow: +R² score, the coefficient of determination +------------------------------------------- -.. math:: +The :func:`r2_score` function computes the `coefficient of +determination `_, +usually denoted as :math:`R^2`. - explained\_{}variance(y, \hat{y}) = 1 - \frac{Var\{ y - \hat{y}\}}{Var\{y\}} +It represents the proportion of variance (of y) that has been explained by the +independent variables in the model. It provides an indication of goodness of +fit and therefore a measure of how well unseen samples are likely to be +predicted by the model, through the proportion of explained variance. -The best possible score is 1.0, lower values are worse. +As such variance is dataset dependent, :math:`R^2` may not be meaningfully comparable +across different datasets. Best possible score is 1.0 and it can be negative +(because the model can be arbitrarily worse). A constant model that always +predicts the expected (average) value of y, disregarding the input features, +would get an :math:`R^2` score of 0.0. -Here is a small example of usage of the :func:`explained_variance_score` -function:: +Note: when the prediction residuals have zero mean, the :math:`R^2` score and +the :ref:`explained_variance_score` are identical. - >>> from sklearn.metrics import explained_variance_score - >>> y_true = [3, -0.5, 2, 7] - >>> y_pred = [2.5, 0.0, 2, 8] - >>> explained_variance_score(y_true, y_pred) - 0.957... - >>> y_true = [[0.5, 1], [-1, 1], [7, -6]] - >>> y_pred = [[0, 2], [-1, 2], [8, -5]] - >>> explained_variance_score(y_true, y_pred, multioutput='raw_values') - array([0.967..., 1. ]) - >>> explained_variance_score(y_true, y_pred, multioutput=[0.3, 0.7]) - 0.990... - -.. _max_error: +If :math:`\hat{y}_i` is the predicted value of the :math:`i`-th sample +and :math:`y_i` is the corresponding true value for total :math:`n` samples, +the estimated :math:`R^2` is defined as: -Max error -------------------- +.. math:: -The :func:`max_error` function computes the maximum `residual error -`_ , a metric -that captures the worst case error between the predicted value and -the true value. In a perfectly fitted single output regression -model, ``max_error`` would be ``0`` on the training set and though this -would be highly unlikely in the real world, this metric shows the -extent of error that the model had when it was fitted. + R^2(y, \hat{y}) = 1 - \frac{\sum_{i=1}^{n} (y_i - \hat{y}_i)^2}{\sum_{i=1}^{n} (y_i - \bar{y})^2} +where :math:`\bar{y} = \frac{1}{n} \sum_{i=1}^{n} y_i` and :math:`\sum_{i=1}^{n} (y_i - \hat{y}_i)^2 = \sum_{i=1}^{n} \epsilon_i^2`. -If :math:`\hat{y}_i` is the predicted value of the :math:`i`-th sample, -and :math:`y_i` is the corresponding true value, then the max error is -defined as +Note that :func:`r2_score` calculates unadjusted :math:`R^2` without correcting for +bias in sample variance of y. -.. math:: +In the particular case where the true target is constant, the :math:`R^2` score is +not finite: it is either ``NaN`` (perfect predictions) or ``-Inf`` (imperfect +predictions). Such non-finite scores may prevent correct model optimization +such as grid-search cross-validation to be performed correctly. For this reason +the default behaviour of :func:`r2_score` is to replace them with 1.0 (perfect +predictions) or 0.0 (imperfect predictions). If ``force_finite`` +is set to ``False``, this score falls back on the original :math:`R^2` definition. - \text{Max Error}(y, \hat{y}) = max(| y_i - \hat{y}_i |) +Here is a small example of usage of the :func:`r2_score` function:: -Here is a small example of usage of the :func:`max_error` function:: + >>> from sklearn.metrics import r2_score + >>> y_true = [3, -0.5, 2, 7] + >>> y_pred = [2.5, 0.0, 2, 8] + >>> r2_score(y_true, y_pred) + 0.948 + >>> y_true = [[0.5, 1], [-1, 1], [7, -6]] + >>> y_pred = [[0, 2], [-1, 2], [8, -5]] + >>> r2_score(y_true, y_pred, multioutput='variance_weighted') + 0.938 + >>> y_true = [[0.5, 1], [-1, 1], [7, -6]] + >>> y_pred = [[0, 2], [-1, 2], [8, -5]] + >>> r2_score(y_true, y_pred, multioutput='uniform_average') + 0.936 + >>> r2_score(y_true, y_pred, multioutput='raw_values') + array([0.965, 0.908]) + >>> r2_score(y_true, y_pred, multioutput=[0.3, 0.7]) + 0.925 + >>> y_true = [-2, -2, -2] + >>> y_pred = [-2, -2, -2] + >>> r2_score(y_true, y_pred) + 1.0 + >>> r2_score(y_true, y_pred, force_finite=False) + nan + >>> y_true = [-2, -2, -2] + >>> y_pred = [-2, -2, -2 + 1e-8] + >>> r2_score(y_true, y_pred) + 0.0 + >>> r2_score(y_true, y_pred, force_finite=False) + -inf - >>> from sklearn.metrics import max_error - >>> y_true = [3, 2, 7, 1] - >>> y_pred = [9, 2, 7, 1] - >>> max_error(y_true, y_pred) - 6 +.. rubric:: Examples -The :func:`max_error` does not support multioutput. +* See :ref:`sphx_glr_auto_examples_linear_model_plot_lasso_and_elasticnet.py` + for an example of R² score usage to + evaluate Lasso and Elastic Net on sparse signals. .. _mean_absolute_error: @@ -2082,14 +2563,14 @@ Here is a small example of usage of the :func:`mean_absolute_error` function:: >>> mean_absolute_error(y_true, y_pred, multioutput='raw_values') array([0.5, 1. ]) >>> mean_absolute_error(y_true, y_pred, multioutput=[0.3, 0.7]) - 0.85... + 0.85 .. _mean_squared_error: Mean squared error ------------------- -The :func:`mean_squared_error` function computes `mean square +The :func:`mean_squared_error` function computes `mean squared error `_, a risk metric corresponding to the expected value of the squared (quadratic) error or loss. @@ -2113,13 +2594,16 @@ function:: >>> y_true = [[0.5, 1], [-1, 1], [7, -6]] >>> y_pred = [[0, 2], [-1, 2], [8, -5]] >>> mean_squared_error(y_true, y_pred) - 0.7083... + 0.7083 + +.. rubric:: Examples -.. topic:: Examples: +* See :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_regression.py` + for an example of mean squared error usage to evaluate gradient boosting regression. - * See :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_regression.py` - for an example of mean squared error usage to - evaluate gradient boosting regression. +Taking the square root of the MSE, called the root mean squared error (RMSE), is another +common metric that provides a measure in the same units as the target variable. RMSE is +available through the :func:`root_mean_squared_error` function. .. _mean_squared_log_error: @@ -2152,11 +2636,14 @@ function:: >>> y_true = [3, 5, 2.5, 7] >>> y_pred = [2.5, 5, 4, 8] >>> mean_squared_log_error(y_true, y_pred) - 0.039... + 0.0397 >>> y_true = [[0.5, 1], [1, 2], [7, 6]] >>> y_pred = [[0.5, 2], [1, 2.5], [8, 8]] >>> mean_squared_log_error(y_true, y_pred) - 0.044... + 0.044 + +The root mean squared logarithmic error (RMSLE) is available through the +:func:`root_mean_squared_log_error` function. .. _mean_absolute_percentage_error: @@ -2173,7 +2660,7 @@ error (MAPE) estimated over :math:`n_{\text{samples}}` is defined as .. math:: - \text{MAPE}(y, \hat{y}) = \frac{1}{n_{\text{samples}}} \sum_{i=0}^{n_{\text{samples}}-1} \frac{{}\left| y_i - \hat{y}_i \right|}{max(\epsilon, \left| y_i \right|)} + \text{MAPE}(y, \hat{y}) = \frac{1}{n_{\text{samples}}} \sum_{i=0}^{n_{\text{samples}}-1} \frac{{}\left| y_i - \hat{y}_i \right|}{\max(\epsilon, \left| y_i \right|)} where :math:`\epsilon` is an arbitrary small yet strictly positive number to avoid undefined results when y is zero. @@ -2187,13 +2674,29 @@ function:: >>> y_true = [1, 10, 1e6] >>> y_pred = [0.9, 15, 1.2e6] >>> mean_absolute_percentage_error(y_true, y_pred) - 0.2666... + 0.2666 In above example, if we had used `mean_absolute_error`, it would have ignored the small magnitude values and only reflected the error in prediction of highest magnitude value. But that problem is resolved in case of MAPE because it calculates relative percentage error with respect to actual output. +.. note:: + + The MAPE formula here does not represent the common "percentage" definition: the + percentage in the range [0, 100] is converted to a relative value in the range [0, + 1] by dividing by 100. Thus, an error of 200% corresponds to a relative error of 2. + The motivation here is to have a range of values that is more consistent with other + error metrics in scikit-learn, such as `accuracy_score`. + + To obtain the mean absolute percentage error as per the Wikipedia formula, + multiply the `mean_absolute_percentage_error` computed here by 100. + +.. dropdown:: References + + * `Wikipedia entry for Mean Absolute Percentage Error + `_ + .. _median_absolute_error: Median absolute error @@ -2222,65 +2725,102 @@ function:: >>> median_absolute_error(y_true, y_pred) 0.5 -.. _r2_score: -R² score, the coefficient of determination -------------------------------------------- -The :func:`r2_score` function computes the `coefficient of -determination `_, -usually denoted as R². +.. _max_error: -It represents the proportion of variance (of y) that has been explained by the -independent variables in the model. It provides an indication of goodness of -fit and therefore a measure of how well unseen samples are likely to be -predicted by the model, through the proportion of explained variance. +Max error +------------------- -As such variance is dataset dependent, R² may not be meaningfully comparable -across different datasets. Best possible score is 1.0 and it can be negative -(because the model can be arbitrarily worse). A constant model that always -predicts the expected value of y, disregarding the input features, would get a -R² score of 0.0. +The :func:`max_error` function computes the maximum `residual error +`_ , a metric +that captures the worst case error between the predicted value and +the true value. In a perfectly fitted single output regression +model, ``max_error`` would be ``0`` on the training set and though this +would be highly unlikely in the real world, this metric shows the +extent of error that the model had when it was fitted. -If :math:`\hat{y}_i` is the predicted value of the :math:`i`-th sample -and :math:`y_i` is the corresponding true value for total :math:`n` samples, -the estimated R² is defined as: + +If :math:`\hat{y}_i` is the predicted value of the :math:`i`-th sample, +and :math:`y_i` is the corresponding true value, then the max error is +defined as .. math:: - R^2(y, \hat{y}) = 1 - \frac{\sum_{i=1}^{n} (y_i - \hat{y}_i)^2}{\sum_{i=1}^{n} (y_i - \bar{y})^2} + \text{Max Error}(y, \hat{y}) = \max(| y_i - \hat{y}_i |) -where :math:`\bar{y} = \frac{1}{n} \sum_{i=1}^{n} y_i` and :math:`\sum_{i=1}^{n} (y_i - \hat{y}_i)^2 = \sum_{i=1}^{n} \epsilon_i^2`. +Here is a small example of usage of the :func:`max_error` function:: -Note that :func:`r2_score` calculates unadjusted R² without correcting for -bias in sample variance of y. + >>> from sklearn.metrics import max_error + >>> y_true = [3, 2, 7, 1] + >>> y_pred = [9, 2, 7, 1] + >>> max_error(y_true, y_pred) + 6.0 -Here is a small example of usage of the :func:`r2_score` function:: +The :func:`max_error` does not support multioutput. - >>> from sklearn.metrics import r2_score - >>> y_true = [3, -0.5, 2, 7] - >>> y_pred = [2.5, 0.0, 2, 8] - >>> r2_score(y_true, y_pred) - 0.948... - >>> y_true = [[0.5, 1], [-1, 1], [7, -6]] - >>> y_pred = [[0, 2], [-1, 2], [8, -5]] - >>> r2_score(y_true, y_pred, multioutput='variance_weighted') - 0.938... - >>> y_true = [[0.5, 1], [-1, 1], [7, -6]] - >>> y_pred = [[0, 2], [-1, 2], [8, -5]] - >>> r2_score(y_true, y_pred, multioutput='uniform_average') - 0.936... - >>> r2_score(y_true, y_pred, multioutput='raw_values') - array([0.965..., 0.908...]) - >>> r2_score(y_true, y_pred, multioutput=[0.3, 0.7]) - 0.925... +.. _explained_variance_score: + +Explained variance score +------------------------- + +The :func:`explained_variance_score` computes the `explained variance +regression score `_. +If :math:`\hat{y}` is the estimated target output, :math:`y` the corresponding +(correct) target output, and :math:`Var` is `Variance +`_, the square of the standard deviation, +then the explained variance is estimated as follow: + +.. math:: + + explained\_{}variance(y, \hat{y}) = 1 - \frac{Var\{ y - \hat{y}\}}{Var\{y\}} + +The best possible score is 1.0, lower values are worse. + +.. topic:: Link to :ref:`r2_score` -.. topic:: Example: + The difference between the explained variance score and the :ref:`r2_score` + is that the explained variance score does not account for + systematic offset in the prediction. For this reason, the + :ref:`r2_score` should be preferred in general. - * See :ref:`sphx_glr_auto_examples_linear_model_plot_lasso_and_elasticnet.py` - for an example of R² score usage to - evaluate Lasso and Elastic Net on sparse signals. +In the particular case where the true target is constant, the Explained +Variance score is not finite: it is either ``NaN`` (perfect predictions) or +``-Inf`` (imperfect predictions). Such non-finite scores may prevent correct +model optimization such as grid-search cross-validation to be performed +correctly. For this reason the default behaviour of +:func:`explained_variance_score` is to replace them with 1.0 (perfect +predictions) or 0.0 (imperfect predictions). You can set the ``force_finite`` +parameter to ``False`` to prevent this fix from happening and fallback on the +original Explained Variance score. + +Here is a small example of usage of the :func:`explained_variance_score` +function:: + + >>> from sklearn.metrics import explained_variance_score + >>> y_true = [3, -0.5, 2, 7] + >>> y_pred = [2.5, 0.0, 2, 8] + >>> explained_variance_score(y_true, y_pred) + 0.957 + >>> y_true = [[0.5, 1], [-1, 1], [7, -6]] + >>> y_pred = [[0, 2], [-1, 2], [8, -5]] + >>> explained_variance_score(y_true, y_pred, multioutput='raw_values') + array([0.967, 1. ]) + >>> explained_variance_score(y_true, y_pred, multioutput=[0.3, 0.7]) + 0.990 + >>> y_true = [-2, -2, -2] + >>> y_pred = [-2, -2, -2] + >>> explained_variance_score(y_true, y_pred) + 1.0 + >>> explained_variance_score(y_true, y_pred, force_finite=False) + nan + >>> y_true = [-2, -2, -2] + >>> y_pred = [-2, -2, -2 + 1e-8] + >>> explained_variance_score(y_true, y_pred) + 0.0 + >>> explained_variance_score(y_true, y_pred, force_finite=False) + -inf .. _mean_tweedie_deviance: @@ -2310,10 +2850,10 @@ is defined as \sum_{i=0}^{n_\text{samples} - 1} \begin{cases} (y_i-\hat{y}_i)^2, & \text{for }p=0\text{ (Normal)}\\ - 2(y_i \log(y/\hat{y}_i) + \hat{y}_i - y_i), & \text{for}p=1\text{ (Poisson)}\\ - 2(\log(\hat{y}_i/y_i) + y_i/\hat{y}_i - 1), & \text{for}p=2\text{ (Gamma)}\\ + 2(y_i \log(y_i/\hat{y}_i) + \hat{y}_i - y_i), & \text{for }p=1\text{ (Poisson)}\\ + 2(\log(\hat{y}_i/y_i) + y_i/\hat{y}_i - 1), & \text{for }p=2\text{ (Gamma)}\\ 2\left(\frac{\max(y_i,0)^{2-p}}{(1-p)(2-p)}- - \frac{y\,\hat{y}^{1-p}_i}{1-p}+\frac{\hat{y}^{2-p}_i}{2-p}\right), + \frac{y_i\,\hat{y}_i^{1-p}}{1-p}+\frac{\hat{y}_i^{2-p}}{2-p}\right), & \text{otherwise} \end{cases} @@ -2325,8 +2865,8 @@ distribution (``power=0``), quadratically. In general, the higher ``power`` the less weight is given to extreme deviations between true and predicted targets. -For instance, let's compare the two predictions 1.0 and 100 that are both -50% of their corresponding true value. +For instance, let's compare the two predictions 1.5 and 150 that are both +50% larger than their corresponding true value. The mean squared error (``power=0``) is very sensitive to the prediction difference of the second point,:: @@ -2340,16 +2880,16 @@ prediction difference of the second point,:: If we increase ``power`` to 1,:: >>> mean_tweedie_deviance([1.0], [1.5], power=1) - 0.18... + 0.189 >>> mean_tweedie_deviance([100.], [150.], power=1) - 18.9... + 18.9 the difference in errors decreases. Finally, by setting, ``power=2``:: >>> mean_tweedie_deviance([1.0], [1.5], power=2) - 0.14... + 0.144 >>> mean_tweedie_deviance([100.], [150.], power=2) - 0.14... + 0.144 we would get identical errors. The deviance when ``power=2`` is thus only sensitive to relative errors. @@ -2360,33 +2900,35 @@ Pinball loss ------------ The :func:`mean_pinball_loss` function is used to evaluate the predictive -performance of quantile regression models. The `pinball loss -`_ is equivalent -to :func:`mean_absolute_error` when the quantile parameter ``alpha`` is set to -0.5. +performance of `quantile regression +`_ models. .. math:: \text{pinball}(y, \hat{y}) = \frac{1}{n_{\text{samples}}} \sum_{i=0}^{n_{\text{samples}}-1} \alpha \max(y_i - \hat{y}_i, 0) + (1 - \alpha) \max(\hat{y}_i - y_i, 0) +The value of pinball loss is equivalent to half of :func:`mean_absolute_error` when the quantile +parameter ``alpha`` is set to 0.5. + + Here is a small example of usage of the :func:`mean_pinball_loss` function:: >>> from sklearn.metrics import mean_pinball_loss >>> y_true = [1, 2, 3] >>> mean_pinball_loss(y_true, [0, 2, 3], alpha=0.1) - 0.03... + 0.033 >>> mean_pinball_loss(y_true, [1, 2, 4], alpha=0.1) - 0.3... + 0.3 >>> mean_pinball_loss(y_true, [0, 2, 3], alpha=0.9) - 0.3... + 0.3 >>> mean_pinball_loss(y_true, [1, 2, 4], alpha=0.9) - 0.03... + 0.033 >>> mean_pinball_loss(y_true, y_true, alpha=0.1) 0.0 >>> mean_pinball_loss(y_true, y_true, alpha=0.9) 0.0 -It is possible to build a scorer object with a specific choice of alpha:: +It is possible to build a scorer object with a specific choice of ``alpha``:: >>> from sklearn.metrics import make_scorer >>> mean_pinball_loss_95p = make_scorer(mean_pinball_loss, alpha=0.95) @@ -2405,32 +2947,190 @@ quantile regressor via cross-validation: ... random_state=0, ... ) >>> cross_val_score(estimator, X, y, cv=5, scoring=mean_pinball_loss_95p) - array([11.1..., 10.4... , 24.4..., 9.2..., 12.9...]) + array([13.6, 9.7, 23.3, 9.5, 10.4]) It is also possible to build scorer objects for hyper-parameter tuning. The sign of the loss must be switched to ensure that greater means better as explained in the example linked below. -.. topic:: Example: +.. rubric:: Examples + +* See :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_quantile.py` + for an example of using the pinball loss to evaluate and tune the + hyper-parameters of quantile regression models on data with non-symmetric + noise and outliers. + +.. _d2_score: + +D² score +-------- + +The D² score computes the fraction of deviance explained. +It is a generalization of R², where the squared error is generalized and replaced +by a deviance of choice :math:`\text{dev}(y, \hat{y})` +(e.g., Tweedie, pinball or mean absolute error). D² is a form of a *skill score*. +It is calculated as + +.. math:: + + D^2(y, \hat{y}) = 1 - \frac{\text{dev}(y, \hat{y})}{\text{dev}(y, y_{\text{null}})} \,. + +Where :math:`y_{\text{null}}` is the optimal prediction of an intercept-only model +(e.g., the mean of `y_true` for the Tweedie case, the median for absolute +error and the alpha-quantile for pinball loss). + +Like R², the best possible score is 1.0 and it can be negative (because the +model can be arbitrarily worse). A constant model that always predicts +:math:`y_{\text{null}}`, disregarding the input features, would get a D² score +of 0.0. + +.. dropdown:: D² Tweedie score + + The :func:`d2_tweedie_score` function implements the special case of D² + where :math:`\text{dev}(y, \hat{y})` is the Tweedie deviance, see :ref:`mean_tweedie_deviance`. + It is also known as D² Tweedie and is related to McFadden's likelihood ratio index. + + The argument ``power`` defines the Tweedie power as for + :func:`mean_tweedie_deviance`. Note that for `power=0`, + :func:`d2_tweedie_score` equals :func:`r2_score` (for single targets). + + A scorer object with a specific choice of ``power`` can be built by:: + + >>> from sklearn.metrics import d2_tweedie_score, make_scorer + >>> d2_tweedie_score_15 = make_scorer(d2_tweedie_score, power=1.5) + +.. dropdown:: D² pinball score - * See :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_quantile.py` - for an example of using a the pinball loss to evaluate and tune the - hyper-parameters of quantile regression models on data with non-symmetric - noise and outliers. + The :func:`d2_pinball_score` function implements the special case + of D² with the pinball loss, see :ref:`pinball_loss`, i.e.: + .. math:: + + \text{dev}(y, \hat{y}) = \text{pinball}(y, \hat{y}). + + The argument ``alpha`` defines the slope of the pinball loss as for + :func:`mean_pinball_loss` (:ref:`pinball_loss`). It determines the + quantile level ``alpha`` for which the pinball loss and also D² + are optimal. Note that for `alpha=0.5` (the default) :func:`d2_pinball_score` + equals :func:`d2_absolute_error_score`. + + A scorer object with a specific choice of ``alpha`` can be built by:: + + >>> from sklearn.metrics import d2_pinball_score, make_scorer + >>> d2_pinball_score_08 = make_scorer(d2_pinball_score, alpha=0.8) + +.. dropdown:: D² absolute error score + + The :func:`d2_absolute_error_score` function implements the special case of + the :ref:`mean_absolute_error`: + + .. math:: + + \text{dev}(y, \hat{y}) = \text{MAE}(y, \hat{y}). + + Here are some usage examples of the :func:`d2_absolute_error_score` function:: + + >>> from sklearn.metrics import d2_absolute_error_score + >>> y_true = [3, -0.5, 2, 7] + >>> y_pred = [2.5, 0.0, 2, 8] + >>> d2_absolute_error_score(y_true, y_pred) + 0.764 + >>> y_true = [1, 2, 3] + >>> y_pred = [1, 2, 3] + >>> d2_absolute_error_score(y_true, y_pred) + 1.0 + >>> y_true = [1, 2, 3] + >>> y_pred = [2, 2, 2] + >>> d2_absolute_error_score(y_true, y_pred) + 0.0 + + +.. _visualization_regression_evaluation: + +Visual evaluation of regression models +-------------------------------------- + +Among methods to assess the quality of regression models, scikit-learn provides +the :class:`~sklearn.metrics.PredictionErrorDisplay` class. It allows to +visually inspect the prediction errors of a model in two different manners. + +.. image:: ../auto_examples/model_selection/images/sphx_glr_plot_cv_predict_001.png + :target: ../auto_examples/model_selection/plot_cv_predict.html + :scale: 75 + :align: center + +The plot on the left shows the actual values vs predicted values. For a +noise-free regression task aiming to predict the (conditional) expectation of +`y`, a perfect regression model would display data points on the diagonal +defined by predicted equal to actual values. The further away from this optimal +line, the larger the error of the model. In a more realistic setting with +irreducible noise, that is, when not all the variations of `y` can be explained +by features in `X`, then the best model would lead to a cloud of points densely +arranged around the diagonal. + +Note that the above only holds when the predicted values is the expected value +of `y` given `X`. This is typically the case for regression models that +minimize the mean squared error objective function or more generally the +:ref:`mean Tweedie deviance ` for any value of its +"power" parameter. + +When plotting the predictions of an estimator that predicts a quantile +of `y` given `X`, e.g. :class:`~sklearn.linear_model.QuantileRegressor` +or any other model minimizing the :ref:`pinball loss `, a +fraction of the points are either expected to lie above or below the diagonal +depending on the estimated quantile level. + +All in all, while intuitive to read, this plot does not really inform us on +what to do to obtain a better model. + +The right-hand side plot shows the residuals (i.e. the difference between the +actual and the predicted values) vs. the predicted values. + +This plot makes it easier to visualize if the residuals follow and +`homoscedastic or heteroschedastic +`_ +distribution. + +In particular, if the true distribution of `y|X` is Poisson or Gamma +distributed, it is expected that the variance of the residuals of the optimal +model would grow with the predicted value of `E[y|X]` (either linearly for +Poisson or quadratically for Gamma). + +When fitting a linear least squares regression model (see +:class:`~sklearn.linear_model.LinearRegression` and +:class:`~sklearn.linear_model.Ridge`), we can use this plot to check +if some of the `model assumptions +`_ +are met, in particular that the residuals should be uncorrelated, their +expected value should be null and that their variance should be constant +(homoschedasticity). + +If this is not the case, and in particular if the residuals plot show some +banana-shaped structure, this is a hint that the model is likely mis-specified +and that non-linear feature engineering or switching to a non-linear regression +model might be useful. + +Refer to the example below to see a model evaluation that makes use of this +display. + +.. rubric:: Examples + +* See :ref:`sphx_glr_auto_examples_compose_plot_transformed_target.py` for + an example on how to use :class:`~sklearn.metrics.PredictionErrorDisplay` + to visualize the prediction quality improvement of a regression model + obtained by transforming the target before learning. .. _clustering_metrics: Clustering metrics -====================== +================== .. currentmodule:: sklearn.metrics The :mod:`sklearn.metrics` module implements several loss, score, and utility -functions. For more information see the :ref:`clustering_evaluation` -section for instance clustering, and :ref:`biclustering_evaluation` for -biclustering. - +functions to measure clustering performance. For more information see the +:ref:`clustering_evaluation` section for instance clustering, and +:ref:`biclustering_evaluation` for biclustering. .. _dummy_estimators: @@ -2472,19 +3172,19 @@ Next, let's compare the accuracy of ``SVC`` and ``most_frequent``:: >>> from sklearn.svm import SVC >>> clf = SVC(kernel='linear', C=1).fit(X_train, y_train) >>> clf.score(X_test, y_test) - 0.63... + 0.63 >>> clf = DummyClassifier(strategy='most_frequent', random_state=0) >>> clf.fit(X_train, y_train) DummyClassifier(random_state=0, strategy='most_frequent') >>> clf.score(X_test, y_test) - 0.57... + 0.579 We see that ``SVC`` doesn't do much better than a dummy classifier. Now, let's change the kernel:: >>> clf = SVC(kernel='rbf', C=1).fit(X_train, y_train) >>> clf.score(X_test, y_test) - 0.94... + 0.94 We see that the accuracy was boosted to almost 100%. A cross validation strategy is recommended for a better estimate of the accuracy, if it diff --git a/doc/modules/model_persistence.rst b/doc/modules/model_persistence.rst deleted file mode 100644 index e00212d80fd10..0000000000000 --- a/doc/modules/model_persistence.rst +++ /dev/null @@ -1,121 +0,0 @@ -.. _model_persistence: - -================= -Model persistence -================= - -After training a scikit-learn model, it is desirable to have a way to persist -the model for future use without having to retrain. The following sections give -you some hints on how to persist a scikit-learn model. - -Python specific serialization ------------------------------ - -It is possible to save a model in scikit-learn by using Python's built-in -persistence model, namely `pickle -`_:: - - >>> from sklearn import svm - >>> from sklearn import datasets - >>> clf = svm.SVC() - >>> X, y= datasets.load_iris(return_X_y=True) - >>> clf.fit(X, y) - SVC() - - >>> import pickle - >>> s = pickle.dumps(clf) - >>> clf2 = pickle.loads(s) - >>> clf2.predict(X[0:1]) - array([0]) - >>> y[0] - 0 - -In the specific case of scikit-learn, it may be better to use joblib's -replacement of pickle (``dump`` & ``load``), which is more efficient on -objects that carry large numpy arrays internally as is often the case for -fitted scikit-learn estimators, but can only pickle to the disk and not to a -string:: - - >>> from joblib import dump, load - >>> dump(clf, 'filename.joblib') # doctest: +SKIP - -Later you can load back the pickled model (possibly in another Python process) -with:: - - >>> clf = load('filename.joblib') # doctest:+SKIP - -.. note:: - - ``dump`` and ``load`` functions also accept file-like object - instead of filenames. More information on data persistence with Joblib is - available `here - `_. - -.. _persistence_limitations: - -Security & maintainability limitations -...................................... - -pickle (and joblib by extension), has some issues regarding maintainability -and security. Because of this, - -* Never unpickle untrusted data as it could lead to malicious code being - executed upon loading. -* While models saved using one version of scikit-learn might load in - other versions, this is entirely unsupported and inadvisable. It should - also be kept in mind that operations performed on such data could give - different and unexpected results. - -In order to rebuild a similar model with future versions of scikit-learn, -additional metadata should be saved along the pickled model: - -* The training data, e.g. a reference to an immutable snapshot -* The python source code used to generate the model -* The versions of scikit-learn and its dependencies -* The cross validation score obtained on the training data - -This should make it possible to check that the cross-validation score is in the -same range as before. - -Aside for a few exceptions, pickled models should be portable across -architectures assuming the same versions of dependencies and Python are used. -If you encounter an estimator that is not portable please open an issue on -GitHub. Pickled models are often deployed in production using containers, like -Docker, in order to freeze the environment and dependencies. - -If you want to know more about these issues and explore other possible -serialization methods, please refer to this -`talk by Alex Gaynor -`_. - -Interoperable formats ---------------------- - -For reproducibility and quality control needs, when different architectures -and environments should be taken into account, exporting the model in -`Open Neural Network -Exchange `_ format or `Predictive Model Markup Language -(PMML) `_ format -might be a better approach than using `pickle` alone. -These are helpful where you may want to use your model for prediction in a -different environment from where the model was trained. - -ONNX is a binary serialization of the model. It has been developed to improve -the usability of the interoperable representation of data models. -It aims to facilitate the conversion of the data -models between different machine learning frameworks, and to improve their -portability on different computing architectures. More details are available -from the `ONNX tutorial `_. -To convert scikit-learn model to ONNX a specific tool `sklearn-onnx -`_ has been developed. - -PMML is an implementation of the `XML -`_ document standard -defined to represent data models together with the data used to generate them. -Being human and machine readable, -PMML is a good option for model validation on different platforms and -long term archiving. On the other hand, as XML in general, its verbosity does -not help in production when performance is critical. -To convert scikit-learn model to PMML you can use for example `sklearn2pmml -`_ distributed under the Affero GPLv3 -license. diff --git a/doc/modules/multiclass.rst b/doc/modules/multiclass.rst index b3ea8d838574e..ef7d6ab3000e1 100644 --- a/doc/modules/multiclass.rst +++ b/doc/modules/multiclass.rst @@ -63,8 +63,8 @@ can provide additional strategies beyond what is built-in: - :class:`semi_supervised.LabelSpreading` - :class:`discriminant_analysis.LinearDiscriminantAnalysis` - :class:`svm.LinearSVC` (setting multi_class="crammer_singer") - - :class:`linear_model.LogisticRegression` (setting multi_class="multinomial") - - :class:`linear_model.LogisticRegressionCV` (setting multi_class="multinomial") + - :class:`linear_model.LogisticRegression` (with most solvers) + - :class:`linear_model.LogisticRegressionCV` (with most solvers) - :class:`neural_network.MLPClassifier` - :class:`neighbors.NearestCentroid` - :class:`discriminant_analysis.QuadraticDiscriminantAnalysis` @@ -86,8 +86,8 @@ can provide additional strategies beyond what is built-in: - :class:`ensemble.GradientBoostingClassifier` - :class:`gaussian_process.GaussianProcessClassifier` (setting multi_class = "one_vs_rest") - :class:`svm.LinearSVC` (setting multi_class="ovr") - - :class:`linear_model.LogisticRegression` (setting multi_class="ovr") - - :class:`linear_model.LogisticRegressionCV` (setting multi_class="ovr") + - :class:`linear_model.LogisticRegression` (most solvers) + - :class:`linear_model.LogisticRegressionCV` (most solvers) - :class:`linear_model.SGDClassifier` - :class:`linear_model.Perceptron` - :class:`linear_model.PassiveAggressiveClassifier` @@ -102,6 +102,7 @@ can provide additional strategies beyond what is built-in: - :class:`neural_network.MLPClassifier` - :class:`neighbors.RadiusNeighborsClassifier` - :class:`ensemble.RandomForestClassifier` + - :class:`linear_model.RidgeClassifier` - :class:`linear_model.RidgeClassifierCV` @@ -146,35 +147,38 @@ Target format Valid :term:`multiclass` representations for :func:`~sklearn.utils.multiclass.type_of_target` (`y`) are: - - 1d or column vector containing more than two discrete values. An - example of a vector ``y`` for 4 samples: - - >>> import numpy as np - >>> y = np.array(['apple', 'pear', 'apple', 'orange']) - >>> print(y) - ['apple' 'pear' 'apple' 'orange'] - - - Dense or sparse :term:`binary` matrix of shape ``(n_samples, n_classes)`` - with a single sample per row, where each column represents one class. An - example of both a dense and sparse :term:`binary` matrix ``y`` for 4 - samples, where the columns, in order, are apple, orange, and pear: - - >>> import numpy as np - >>> from sklearn.preprocessing import LabelBinarizer - >>> y = np.array(['apple', 'pear', 'apple', 'orange']) - >>> y_dense = LabelBinarizer().fit_transform(y) - >>> print(y_dense) - [[1 0 0] - [0 0 1] - [1 0 0] - [0 1 0]] - >>> from scipy import sparse - >>> y_sparse = sparse.csr_matrix(y_dense) - >>> print(y_sparse) - (0, 0) 1 - (1, 2) 1 - (2, 0) 1 - (3, 1) 1 +- 1d or column vector containing more than two discrete values. An + example of a vector ``y`` for 4 samples: + + >>> import numpy as np + >>> y = np.array(['apple', 'pear', 'apple', 'orange']) + >>> print(y) + ['apple' 'pear' 'apple' 'orange'] + +- Dense or sparse :term:`binary` matrix of shape ``(n_samples, n_classes)`` + with a single sample per row, where each column represents one class. An + example of both a dense and sparse :term:`binary` matrix ``y`` for 4 + samples, where the columns, in order, are apple, orange, and pear: + + >>> import numpy as np + >>> from sklearn.preprocessing import LabelBinarizer + >>> y = np.array(['apple', 'pear', 'apple', 'orange']) + >>> y_dense = LabelBinarizer().fit_transform(y) + >>> print(y_dense) + [[1 0 0] + [0 0 1] + [1 0 0] + [0 1 0]] + >>> from scipy import sparse + >>> y_sparse = sparse.csr_matrix(y_dense) + >>> print(y_sparse) + + Coords Values + (0, 0) 1 + (1, 2) 1 + (2, 0) 1 + (3, 1) 1 For more information about :class:`~sklearn.preprocessing.LabelBinarizer`, refer to :ref:`preprocessing_targets`. @@ -221,9 +225,11 @@ in which cell [i, j] indicates the presence of label j in sample i. :scale: 75% -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_miscellaneous_plot_multilabel.py` +* :ref:`sphx_glr_auto_examples_miscellaneous_plot_multilabel.py` +* :ref:`sphx_glr_auto_examples_classification_plot_classification_probability.py` +* :ref:`sphx_glr_auto_examples_linear_model_plot_logistic_multinomial.py` .. _ovo_classification: @@ -262,10 +268,10 @@ Below is an example of multiclass learning using OvO:: 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]) -.. topic:: References: +.. rubric:: References - * "Pattern Recognition and Machine Learning. Springer", - Christopher M. Bishop, page 183, (First Edition) +* "Pattern Recognition and Machine Learning. Springer", + Christopher M. Bishop, page 183, (First Edition) .. _ecoc: @@ -295,7 +301,7 @@ used. It is a percentage of the total number of classes. A number between 0 and 1 will require fewer classifiers than one-vs-the-rest. In theory, ``log2(n_classes) / n_classes`` is sufficient to represent each class unambiguously. However, in practice, it may not lead to -good accuracy since ``log2(n_classes)`` is much smaller than n_classes. +good accuracy since ``log2(n_classes)`` is much smaller than `n_classes`. A number greater than 1 will require more classifiers than one-vs-the-rest. In this case, some classifiers will in theory correct for @@ -310,8 +316,7 @@ Below is an example of multiclass learning using Output-Codes:: >>> from sklearn.multiclass import OutputCodeClassifier >>> from sklearn.svm import LinearSVC >>> X, y = datasets.load_iris(return_X_y=True) - >>> clf = OutputCodeClassifier(LinearSVC(random_state=0), - ... code_size=2, random_state=0) + >>> clf = OutputCodeClassifier(LinearSVC(random_state=0), code_size=2, random_state=0) >>> clf.fit(X, y).predict(X) array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -321,21 +326,16 @@ Below is an example of multiclass learning using Output-Codes:: 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]) -.. topic:: References: +.. rubric:: References - * "Solving multiclass learning problems via error-correcting output codes", - Dietterich T., Bakiri G., - Journal of Artificial Intelligence Research 2, - 1995. +* "Solving multiclass learning problems via error-correcting output codes", + Dietterich T., Bakiri G., Journal of Artificial Intelligence Research 2, 1995. - .. [3] "The error coding method and PICTs", - James G., Hastie T., - Journal of Computational and Graphical statistics 7, - 1998. +.. [3] "The error coding method and PICTs", James G., Hastie T., + Journal of Computational and Graphical statistics 7, 1998. - * "The Elements of Statistical Learning", - Hastie T., Tibshirani R., Friedman J., page 606 (second-edition) - 2008. +* "The Elements of Statistical Learning", + Hastie T., Tibshirani R., Friedman J., page 606 (second-edition), 2008. .. _multilabel_classification: @@ -382,10 +382,13 @@ An example of the same ``y`` in sparse matrix form: >>> y_sparse = sparse.csr_matrix(y) >>> print(y_sparse) - (0, 0) 1 - (0, 3) 1 - (1, 2) 1 - (1, 3) 1 + + Coords Values + (0, 0) 1 + (0, 3) 1 + (1, 2) 1 + (1, 3) 1 .. _multioutputclassfier: @@ -400,33 +403,11 @@ to be able to estimate a series of target functions (f1,f2,f3...,fn) that are trained on a single X predictor matrix to predict a series of responses (y1,y2,y3...,yn). -Below is an example of multilabel classification: - - >>> from sklearn.datasets import make_classification - >>> from sklearn.multioutput import MultiOutputClassifier - >>> from sklearn.ensemble import RandomForestClassifier - >>> from sklearn.utils import shuffle - >>> import numpy as np - >>> X, y1 = make_classification(n_samples=10, n_features=100, n_informative=30, n_classes=3, random_state=1) - >>> y2 = shuffle(y1, random_state=1) - >>> y3 = shuffle(y1, random_state=2) - >>> Y = np.vstack((y1, y2, y3)).T - >>> n_samples, n_features = X.shape # 10,100 - >>> n_outputs = Y.shape[1] # 3 - >>> n_classes = 3 - >>> forest = RandomForestClassifier(random_state=1) - >>> multi_target_forest = MultiOutputClassifier(forest, n_jobs=-1) - >>> multi_target_forest.fit(X, Y).predict(X) - array([[2, 2, 0], - [1, 2, 1], - [2, 1, 0], - [0, 0, 2], - [0, 2, 1], - [0, 0, 2], - [1, 1, 0], - [1, 1, 1], - [0, 0, 2], - [2, 0, 0]]) +You can find a usage example for +:class:`~sklearn.multioutput.MultiOutputClassifier` +as part of the section on :ref:`multiclass_multioutput_classification` +since it is a generalization of multilabel classification to +multiclass outputs instead of binary outputs. .. _classifierchain: @@ -454,10 +435,10 @@ one does not know the optimal ordering of the models in the chain so typically many randomly ordered chains are fit and their predictions are averaged together. -.. topic:: References: +.. rubric:: References - Jesse Read, Bernhard Pfahringer, Geoff Holmes, Eibe Frank, - "Classifier Chains for Multi-label Classification", 2009. +* Jesse Read, Bernhard Pfahringer, Geoff Holmes, Eibe Frank, + "Classifier Chains for Multi-label Classification", 2009. .. _multiclass_multioutput_classification: @@ -487,6 +468,36 @@ as a special case. Multitask classification is similar to the multioutput classification task with different model formulations. For more information, see the relevant estimator documentation. +Below is an example of multiclass-multioutput classification: + + >>> from sklearn.datasets import make_classification + >>> from sklearn.multioutput import MultiOutputClassifier + >>> from sklearn.ensemble import RandomForestClassifier + >>> from sklearn.utils import shuffle + >>> import numpy as np + >>> X, y1 = make_classification(n_samples=10, n_features=100, + ... n_informative=30, n_classes=3, + ... random_state=1) + >>> y2 = shuffle(y1, random_state=1) + >>> y3 = shuffle(y1, random_state=2) + >>> Y = np.vstack((y1, y2, y3)).T + >>> n_samples, n_features = X.shape # 10,100 + >>> n_outputs = Y.shape[1] # 3 + >>> n_classes = 3 + >>> forest = RandomForestClassifier(random_state=1) + >>> multi_target_forest = MultiOutputClassifier(forest, n_jobs=2) + >>> multi_target_forest.fit(X, Y).predict(X) + array([[2, 2, 0], + [1, 2, 1], + [2, 1, 0], + [0, 0, 2], + [0, 2, 1], + [0, 0, 2], + [1, 1, 0], + [1, 1, 1], + [0, 0, 2], + [2, 0, 0]]) + .. warning:: At present, no metric in :mod:`sklearn.metrics` supports the multiclass-multioutput classification task. @@ -520,11 +531,42 @@ using data obtained at a certain location. Each sample would be data obtained at one location and both wind speed and direction would be output for each sample. +The following regressors natively support multioutput regression: + +- :class:`cross_decomposition.CCA` +- :class:`tree.DecisionTreeRegressor` +- :class:`dummy.DummyRegressor` +- :class:`linear_model.ElasticNet` +- :class:`tree.ExtraTreeRegressor` +- :class:`ensemble.ExtraTreesRegressor` +- :class:`gaussian_process.GaussianProcessRegressor` +- :class:`neighbors.KNeighborsRegressor` +- :class:`kernel_ridge.KernelRidge` +- :class:`linear_model.Lars` +- :class:`linear_model.Lasso` +- :class:`linear_model.LassoLars` +- :class:`linear_model.LinearRegression` +- :class:`multioutput.MultiOutputRegressor` +- :class:`linear_model.MultiTaskElasticNet` +- :class:`linear_model.MultiTaskElasticNetCV` +- :class:`linear_model.MultiTaskLasso` +- :class:`linear_model.MultiTaskLassoCV` +- :class:`linear_model.OrthogonalMatchingPursuit` +- :class:`cross_decomposition.PLSCanonical` +- :class:`cross_decomposition.PLSRegression` +- :class:`linear_model.RANSACRegressor` +- :class:`neighbors.RadiusNeighborsRegressor` +- :class:`ensemble.RandomForestRegressor` +- :class:`multioutput.RegressorChain` +- :class:`linear_model.Ridge` +- :class:`linear_model.RidgeCV` +- :class:`compose.TransformedTargetRegressor` + Target format ------------- A valid representation of :term:`multioutput` `y` is a dense matrix of shape -``(n_samples, n_classes)`` of floats. A column wise concatenation of +``(n_samples, n_output)`` of floats. A column wise concatenation of :term:`continuous` variables. An example of ``y`` for 3 samples: >>> y = np.array([[31.4, 94], [40.5, 109], [25.0, 30]]) diff --git a/doc/modules/naive_bayes.rst b/doc/modules/naive_bayes.rst index b2dd4cf5a7cd3..b25334a902050 100644 --- a/doc/modules/naive_bayes.rst +++ b/doc/modules/naive_bayes.rst @@ -69,11 +69,11 @@ On the flip side, although naive Bayes is known as a decent classifier, it is known to be a bad estimator, so the probability outputs from ``predict_proba`` are not to be taken too seriously. -.. topic:: References: +.. dropdown:: References - * H. Zhang (2004). `The optimality of Naive Bayes. - `_ - Proc. FLAIRS. + * H. Zhang (2004). `The optimality of Naive Bayes. + `_ + Proc. FLAIRS. .. _gaussian_naive_bayes: @@ -117,7 +117,7 @@ for each class :math:`y`, where :math:`n` is the number of features and :math:`\theta_{yi}` is the probability :math:`P(x_i \mid y)` of feature :math:`i` appearing in a sample belonging to class :math:`y`. -The parameters :math:`\theta_y` is estimated by a smoothed +The parameters :math:`\theta_y` are estimated by a smoothed version of maximum likelihood, i.e. relative frequency counting: .. math:: @@ -125,13 +125,13 @@ version of maximum likelihood, i.e. relative frequency counting: \hat{\theta}_{yi} = \frac{ N_{yi} + \alpha}{N_y + \alpha n} where :math:`N_{yi} = \sum_{x \in T} x_i` is -the number of times feature :math:`i` appears in a sample of class :math:`y` +the number of times feature :math:`i` appears in all samples of class :math:`y` in the training set :math:`T`, and :math:`N_{y} = \sum_{i=1}^{n} N_{yi}` is the total count of all features for class :math:`y`. -The smoothing priors :math:`\alpha \ge 0` accounts for -features not present in the learning samples and prevents zero probabilities +The smoothing priors :math:`\alpha \ge 0` account for +features not present in the learning samples and prevent zero probabilities in further computations. Setting :math:`\alpha = 1` is called Laplace smoothing, while :math:`\alpha < 1` is called Lidstone smoothing. @@ -147,38 +147,42 @@ that is particularly suited for imbalanced data sets. Specifically, CNB uses statistics from the *complement* of each class to compute the model's weights. The inventors of CNB show empirically that the parameter estimates for CNB are more stable than those for MNB. Further, CNB regularly outperforms MNB (often -by a considerable margin) on text classification tasks. The procedure for -calculating the weights is as follows: +by a considerable margin) on text classification tasks. -.. math:: +.. dropdown:: Weights calculation - \hat{\theta}_{ci} = \frac{\alpha_i + \sum_{j:y_j \neq c} d_{ij}} - {\alpha + \sum_{j:y_j \neq c} \sum_{k} d_{kj}} + The procedure for calculating the weights is as follows: - w_{ci} = \log \hat{\theta}_{ci} + .. math:: - w_{ci} = \frac{w_{ci}}{\sum_{j} |w_{cj}|} + \hat{\theta}_{ci} = \frac{\alpha_i + \sum_{j:y_j \neq c} d_{ij}} + {\alpha + \sum_{j:y_j \neq c} \sum_{k} d_{kj}} -where the summations are over all documents :math:`j` not in class :math:`c`, -:math:`d_{ij}` is either the count or tf-idf value of term :math:`i` in document -:math:`j`, :math:`\alpha_i` is a smoothing hyperparameter like that found in -MNB, and :math:`\alpha = \sum_{i} \alpha_i`. The second normalization addresses -the tendency for longer documents to dominate parameter estimates in MNB. The -classification rule is: + w_{ci} = \log \hat{\theta}_{ci} -.. math:: + w_{ci} = \frac{w_{ci}}{\sum_{j} |w_{cj}|} + + where the summations are over all documents :math:`j` not in class :math:`c`, + :math:`d_{ij}` is either the count or tf-idf value of term :math:`i` in document + :math:`j`, :math:`\alpha_i` is a smoothing hyperparameter like that found in + MNB, and :math:`\alpha = \sum_{i} \alpha_i`. The second normalization addresses + the tendency for longer documents to dominate parameter estimates in MNB. The + classification rule is: + + .. math:: - \hat{c} = \arg\min_c \sum_{i} t_i w_{ci} + \hat{c} = \arg\min_c \sum_{i} t_i w_{ci} -i.e., a document is assigned to the class that is the *poorest* complement -match. + i.e., a document is assigned to the class that is the *poorest* complement + match. -.. topic:: References: +.. dropdown:: References + + * Rennie, J. D., Shih, L., Teevan, J., & Karger, D. R. (2003). + `Tackling the poor assumptions of naive bayes text classifiers. + `_ + In ICML (Vol. 3, pp. 616-623). - * Rennie, J. D., Shih, L., Teevan, J., & Karger, D. R. (2003). - `Tackling the poor assumptions of naive bayes text classifiers. - `_ - In ICML (Vol. 3, pp. 616-623). .. _bernoulli_naive_bayes: @@ -190,14 +194,14 @@ algorithms for data that is distributed according to multivariate Bernoulli distributions; i.e., there may be multiple features but each one is assumed to be a binary-valued (Bernoulli, boolean) variable. Therefore, this class requires samples to be represented as binary-valued -feature vectors; if handed any other kind of data, a ``BernoulliNB`` instance +feature vectors; if handed any other kind of data, a :class:`BernoulliNB` instance may binarize its input (depending on the ``binarize`` parameter). The decision rule for Bernoulli naive Bayes is based on .. math:: - P(x_i \mid y) = P(i \mid y) x_i + (1 - P(i \mid y)) (1 - x_i) + P(x_i \mid y) = P(x_i = 1 \mid y) x_i + (1 - P(x_i = 1 \mid y)) (1 - x_i) which differs from multinomial NB's rule in that it explicitly penalizes the non-occurrence of a feature :math:`i` @@ -205,57 +209,60 @@ that is an indicator for class :math:`y`, where the multinomial variant would simply ignore a non-occurring feature. In the case of text classification, word occurrence vectors (rather than word -count vectors) may be used to train and use this classifier. ``BernoulliNB`` +count vectors) may be used to train and use this classifier. :class:`BernoulliNB` might perform better on some datasets, especially those with shorter documents. It is advisable to evaluate both models, if time permits. -.. topic:: References: +.. dropdown:: References + + * C.D. Manning, P. Raghavan and H. SchÃŧtze (2008). Introduction to + Information Retrieval. Cambridge University Press, pp. 234-265. - * C.D. Manning, P. Raghavan and H. SchÃŧtze (2008). Introduction to - Information Retrieval. Cambridge University Press, pp. 234-265. + * A. McCallum and K. Nigam (1998). + `A comparison of event models for Naive Bayes text classification. + `_ + Proc. AAAI/ICML-98 Workshop on Learning for Text Categorization, pp. 41-48. - * A. McCallum and K. Nigam (1998). - `A comparison of event models for Naive Bayes text classification. - `_ - Proc. AAAI/ICML-98 Workshop on Learning for Text Categorization, pp. 41-48. + * V. Metsis, I. Androutsopoulos and G. Paliouras (2006). + `Spam filtering with Naive Bayes -- Which Naive Bayes? + `_ + 3rd Conf. on Email and Anti-Spam (CEAS). - * V. Metsis, I. Androutsopoulos and G. Paliouras (2006). - `Spam filtering with Naive Bayes -- Which Naive Bayes? - `_ - 3rd Conf. on Email and Anti-Spam (CEAS). .. _categorical_naive_bayes: Categorical Naive Bayes ----------------------- -:class:`CategoricalNB` implements the categorical naive Bayes -algorithm for categorically distributed data. It assumes that each feature, -which is described by the index :math:`i`, has its own categorical -distribution. +:class:`CategoricalNB` implements the categorical naive Bayes +algorithm for categorically distributed data. It assumes that each feature, +which is described by the index :math:`i`, has its own categorical +distribution. For each feature :math:`i` in the training set :math:`X`, :class:`CategoricalNB` estimates a categorical distribution for each feature i of X conditioned on the class y. The index set of the samples is defined as :math:`J = \{ 1, \dots, m \}`, with :math:`m` as the number of samples. -The probability of category :math:`t` in feature :math:`i` given class -:math:`c` is estimated as: +.. dropdown:: Probability calculation -.. math:: + The probability of category :math:`t` in feature :math:`i` given class + :math:`c` is estimated as: + + .. math:: - P(x_i = t \mid y = c \: ;\, \alpha) = \frac{ N_{tic} + \alpha}{N_{c} + - \alpha n_i}, + P(x_i = t \mid y = c \: ;\, \alpha) = \frac{ N_{tic} + \alpha}{N_{c} + + \alpha n_i}, -where :math:`N_{tic} = |\{j \in J \mid x_{ij} = t, y_j = c\}|` is the number -of times category :math:`t` appears in the samples :math:`x_{i}`, which belong -to class :math:`c`, :math:`N_{c} = |\{ j \in J\mid y_j = c\}|` is the number -of samples with class c, :math:`\alpha` is a smoothing parameter and -:math:`n_i` is the number of available categories of feature :math:`i`. + where :math:`N_{tic} = |\{j \in J \mid x_{ij} = t, y_j = c\}|` is the number + of times category :math:`t` appears in the samples :math:`x_{i}`, which belong + to class :math:`c`, :math:`N_{c} = |\{ j \in J\mid y_j = c\}|` is the number + of samples with class c, :math:`\alpha` is a smoothing parameter and + :math:`n_i` is the number of available categories of feature :math:`i`. -:class:`CategoricalNB` assumes that the sample matrix :math:`X` is encoded -(for instance with the help of :class:`OrdinalEncoder`) such that all -categories for each feature :math:`i` are represented with numbers +:class:`CategoricalNB` assumes that the sample matrix :math:`X` is encoded (for +instance with the help of :class:`~sklearn.preprocessing.OrdinalEncoder`) such +that all categories for each feature :math:`i` are represented with numbers :math:`0, ..., n_i - 1` where :math:`n_i` is the number of available categories of feature :math:`i`. diff --git a/doc/modules/neighbors.rst b/doc/modules/neighbors.rst index bb84b79e8570a..82caa397b60d2 100644 --- a/doc/modules/neighbors.rst +++ b/doc/modules/neighbors.rst @@ -59,12 +59,12 @@ The choice of neighbors search algorithm is controlled through the keyword from the training data. For a discussion of the strengths and weaknesses of each option, see `Nearest Neighbor Algorithms`_. - .. warning:: +.. warning:: - Regarding the Nearest Neighbors algorithms, if two - neighbors :math:`k+1` and :math:`k` have identical distances - but different labels, the result will depend on the ordering of the - training data. + Regarding the Nearest Neighbors algorithms, if two + neighbors :math:`k+1` and :math:`k` have identical distances + but different labels, the result will depend on the ordering of the + training data. Finding the Nearest Neighbors ----------------------------- @@ -136,8 +136,13 @@ have the same interface; we'll show an example of using the KD Tree here: Refer to the :class:`KDTree` and :class:`BallTree` class documentation for more information on the options available for nearest neighbors searches, including specification of query strategies, distance metrics, etc. For a list -of available metrics, see the documentation of the :class:`DistanceMetric` -class. +of valid metrics use `KDTree.valid_metrics` and `BallTree.valid_metrics`: + + >>> from sklearn.neighbors import KDTree, BallTree + >>> KDTree.valid_metrics + ['euclidean', 'l2', 'minkowski', 'p', 'manhattan', 'cityblock', 'l1', 'chebyshev', 'infinity'] + >>> BallTree.valid_metrics + ['euclidean', 'l2', 'minkowski', 'p', 'manhattan', 'cityblock', 'l1', 'chebyshev', 'infinity', 'seuclidean', 'mahalanobis', 'hamming', 'canberra', 'braycurtis', 'jaccard', 'dice', 'rogerstanimoto', 'russellrao', 'sokalmichener', 'sokalsneath', 'haversine', 'pyfunc'] .. _classification: @@ -183,18 +188,14 @@ distance can be supplied to compute the weights. .. |classification_1| image:: ../auto_examples/neighbors/images/sphx_glr_plot_classification_001.png :target: ../auto_examples/neighbors/plot_classification.html - :scale: 50 - -.. |classification_2| image:: ../auto_examples/neighbors/images/sphx_glr_plot_classification_002.png - :target: ../auto_examples/neighbors/plot_classification.html - :scale: 50 + :scale: 75 -.. centered:: |classification_1| |classification_2| +.. centered:: |classification_1| -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_neighbors_plot_classification.py`: an example of - classification using nearest neighbors. +* :ref:`sphx_glr_auto_examples_neighbors_plot_classification.py`: an example of + classification using nearest neighbors. .. _regression: @@ -240,13 +241,13 @@ the lower half of those faces. :align: center -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_neighbors_plot_regression.py`: an example of regression - using nearest neighbors. +* :ref:`sphx_glr_auto_examples_neighbors_plot_regression.py`: an example of regression + using nearest neighbors. - * :ref:`sphx_glr_auto_examples_miscellaneous_plot_multioutput_face_completion.py`: an example of - multi-output regression using nearest neighbors. +* :ref:`sphx_glr_auto_examples_miscellaneous_plot_multioutput_face_completion.py`: + an example of multi-output regression using nearest neighbors. Nearest Neighbor Algorithms @@ -304,11 +305,11 @@ keyword ``algorithm = 'kd_tree'``, and are computed using the class :class:`KDTree`. -.. topic:: References: +.. dropdown:: References - * `"Multidimensional binary search trees used for associative searching" - `_, - Bentley, J.L., Communications of the ACM (1975) + * `"Multidimensional binary search trees used for associative searching" + `_, + Bentley, J.L., Communications of the ACM (1975) .. _ball_tree: @@ -342,127 +343,142 @@ neighbors searches are specified using the keyword ``algorithm = 'ball_tree'``, and are computed using the class :class:`BallTree`. Alternatively, the user can work with the :class:`BallTree` class directly. -.. topic:: References: - - * `"Five balltree construction algorithms" - `_, - Omohundro, S.M., International Computer Science Institute - Technical Report (1989) - -Choice of Nearest Neighbors Algorithm -------------------------------------- -The optimal algorithm for a given dataset is a complicated choice, and -depends on a number of factors: - -* number of samples :math:`N` (i.e. ``n_samples``) and dimensionality - :math:`D` (i.e. ``n_features``). - - * *Brute force* query time grows as :math:`O[D N]` - * *Ball tree* query time grows as approximately :math:`O[D \log(N)]` - * *KD tree* query time changes with :math:`D` in a way that is difficult - to precisely characterise. For small :math:`D` (less than 20 or so) - the cost is approximately :math:`O[D\log(N)]`, and the KD tree - query can be very efficient. - For larger :math:`D`, the cost increases to nearly :math:`O[DN]`, and - the overhead due to the tree - structure can lead to queries which are slower than brute force. - - For small data sets (:math:`N` less than 30 or so), :math:`\log(N)` is - comparable to :math:`N`, and brute force algorithms can be more efficient - than a tree-based approach. Both :class:`KDTree` and :class:`BallTree` - address this through providing a *leaf size* parameter: this controls the - number of samples at which a query switches to brute-force. This allows both - algorithms to approach the efficiency of a brute-force computation for small - :math:`N`. - -* data structure: *intrinsic dimensionality* of the data and/or *sparsity* - of the data. Intrinsic dimensionality refers to the dimension - :math:`d \le D` of a manifold on which the data lies, which can be linearly - or non-linearly embedded in the parameter space. Sparsity refers to the - degree to which the data fills the parameter space (this is to be - distinguished from the concept as used in "sparse" matrices. The data - matrix may have no zero entries, but the **structure** can still be - "sparse" in this sense). - - * *Brute force* query time is unchanged by data structure. - * *Ball tree* and *KD tree* query times can be greatly influenced - by data structure. In general, sparser data with a smaller intrinsic - dimensionality leads to faster query times. Because the KD tree - internal representation is aligned with the parameter axes, it will not - generally show as much improvement as ball tree for arbitrarily - structured data. - - Datasets used in machine learning tend to be very structured, and are - very well-suited for tree-based queries. - -* number of neighbors :math:`k` requested for a query point. - - * *Brute force* query time is largely unaffected by the value of :math:`k` - * *Ball tree* and *KD tree* query time will become slower as :math:`k` - increases. This is due to two effects: first, a larger :math:`k` leads - to the necessity to search a larger portion of the parameter space. - Second, using :math:`k > 1` requires internal queueing of results - as the tree is traversed. - - As :math:`k` becomes large compared to :math:`N`, the ability to prune - branches in a tree-based query is reduced. In this situation, Brute force - queries can be more efficient. - -* number of query points. Both the ball tree and the KD Tree - require a construction phase. The cost of this construction becomes - negligible when amortized over many queries. If only a small number of - queries will be performed, however, the construction can make up - a significant fraction of the total cost. If very few query points - will be required, brute force is better than a tree-based method. - -Currently, ``algorithm = 'auto'`` selects ``'brute'`` if any of the following -conditions are verified: - -* input data is sparse -* ``metric = 'precomputed'`` -* :math:`D > 15` -* :math:`k >= N/2` -* ``effective_metric_`` isn't in the ``VALID_METRICS`` list for either - ``'kd_tree'`` or ``'ball_tree'`` - -Otherwise, it selects the first out of ``'kd_tree'`` and ``'ball_tree'`` that -has ``effective_metric_`` in its ``VALID_METRICS`` list. This heuristic is -based on the following assumptions: - -* the number of query points is at least the same order as the number of - training points -* ``leaf_size`` is close to its default value of ``30`` -* when :math:`D > 15`, the intrinsic dimensionality of the data is generally - too high for tree-based methods - -Effect of ``leaf_size`` ------------------------ -As noted above, for small sample sizes a brute force search can be more -efficient than a tree-based query. This fact is accounted for in the ball -tree and KD tree by internally switching to brute force searches within -leaf nodes. The level of this switch can be specified with the parameter -``leaf_size``. This parameter choice has many effects: - -**construction time** - A larger ``leaf_size`` leads to a faster tree construction time, because - fewer nodes need to be created - -**query time** - Both a large or small ``leaf_size`` can lead to suboptimal query cost. - For ``leaf_size`` approaching 1, the overhead involved in traversing - nodes can significantly slow query times. For ``leaf_size`` approaching - the size of the training set, queries become essentially brute force. - A good compromise between these is ``leaf_size = 30``, the default value - of the parameter. - -**memory** - As ``leaf_size`` increases, the memory required to store a tree structure - decreases. This is especially important in the case of ball tree, which - stores a :math:`D`-dimensional centroid for each node. The required - storage space for :class:`BallTree` is approximately ``1 / leaf_size`` times - the size of the training set. - -``leaf_size`` is not referenced for brute force queries. + +.. dropdown:: References + + * `"Five Balltree Construction Algorithms" + `_, + Omohundro, S.M., International Computer Science Institute + Technical Report (1989) + +.. dropdown:: Choice of Nearest Neighbors Algorithm + + The optimal algorithm for a given dataset is a complicated choice, and + depends on a number of factors: + + * number of samples :math:`N` (i.e. ``n_samples``) and dimensionality + :math:`D` (i.e. ``n_features``). + + * *Brute force* query time grows as :math:`O[D N]` + * *Ball tree* query time grows as approximately :math:`O[D \log(N)]` + * *KD tree* query time changes with :math:`D` in a way that is difficult + to precisely characterise. For small :math:`D` (less than 20 or so) + the cost is approximately :math:`O[D\log(N)]`, and the KD tree + query can be very efficient. + For larger :math:`D`, the cost increases to nearly :math:`O[DN]`, and + the overhead due to the tree + structure can lead to queries which are slower than brute force. + + For small data sets (:math:`N` less than 30 or so), :math:`\log(N)` is + comparable to :math:`N`, and brute force algorithms can be more efficient + than a tree-based approach. Both :class:`KDTree` and :class:`BallTree` + address this through providing a *leaf size* parameter: this controls the + number of samples at which a query switches to brute-force. This allows both + algorithms to approach the efficiency of a brute-force computation for small + :math:`N`. + + * data structure: *intrinsic dimensionality* of the data and/or *sparsity* + of the data. Intrinsic dimensionality refers to the dimension + :math:`d \le D` of a manifold on which the data lies, which can be linearly + or non-linearly embedded in the parameter space. Sparsity refers to the + degree to which the data fills the parameter space (this is to be + distinguished from the concept as used in "sparse" matrices. The data + matrix may have no zero entries, but the **structure** can still be + "sparse" in this sense). + + * *Brute force* query time is unchanged by data structure. + * *Ball tree* and *KD tree* query times can be greatly influenced + by data structure. In general, sparser data with a smaller intrinsic + dimensionality leads to faster query times. Because the KD tree + internal representation is aligned with the parameter axes, it will not + generally show as much improvement as ball tree for arbitrarily + structured data. + + Datasets used in machine learning tend to be very structured, and are + very well-suited for tree-based queries. + + * number of neighbors :math:`k` requested for a query point. + + * *Brute force* query time is largely unaffected by the value of :math:`k` + * *Ball tree* and *KD tree* query time will become slower as :math:`k` + increases. This is due to two effects: first, a larger :math:`k` leads + to the necessity to search a larger portion of the parameter space. + Second, using :math:`k > 1` requires internal queueing of results + as the tree is traversed. + + As :math:`k` becomes large compared to :math:`N`, the ability to prune + branches in a tree-based query is reduced. In this situation, Brute force + queries can be more efficient. + + * number of query points. Both the ball tree and the KD Tree + require a construction phase. The cost of this construction becomes + negligible when amortized over many queries. If only a small number of + queries will be performed, however, the construction can make up + a significant fraction of the total cost. If very few query points + will be required, brute force is better than a tree-based method. + + Currently, ``algorithm = 'auto'`` selects ``'brute'`` if any of the following + conditions are verified: + + * input data is sparse + * ``metric = 'precomputed'`` + * :math:`D > 15` + * :math:`k >= N/2` + * ``effective_metric_`` isn't in the ``VALID_METRICS`` list for either + ``'kd_tree'`` or ``'ball_tree'`` + + Otherwise, it selects the first out of ``'kd_tree'`` and ``'ball_tree'`` that + has ``effective_metric_`` in its ``VALID_METRICS`` list. This heuristic is + based on the following assumptions: + + * the number of query points is at least the same order as the number of + training points + * ``leaf_size`` is close to its default value of ``30`` + * when :math:`D > 15`, the intrinsic dimensionality of the data is generally + too high for tree-based methods + +.. dropdown:: Effect of ``leaf_size`` + + As noted above, for small sample sizes a brute force search can be more + efficient than a tree-based query. This fact is accounted for in the ball + tree and KD tree by internally switching to brute force searches within + leaf nodes. The level of this switch can be specified with the parameter + ``leaf_size``. This parameter choice has many effects: + + **construction time** + A larger ``leaf_size`` leads to a faster tree construction time, because + fewer nodes need to be created + + **query time** + Both a large or small ``leaf_size`` can lead to suboptimal query cost. + For ``leaf_size`` approaching 1, the overhead involved in traversing + nodes can significantly slow query times. For ``leaf_size`` approaching + the size of the training set, queries become essentially brute force. + A good compromise between these is ``leaf_size = 30``, the default value + of the parameter. + + **memory** + As ``leaf_size`` increases, the memory required to store a tree structure + decreases. This is especially important in the case of ball tree, which + stores a :math:`D`-dimensional centroid for each node. The required + storage space for :class:`BallTree` is approximately ``1 / leaf_size`` times + the size of the training set. + + ``leaf_size`` is not referenced for brute force queries. + +.. dropdown:: Valid Metrics for Nearest Neighbor Algorithms + + For a list of available metrics, see the documentation of the + :class:`~sklearn.metrics.DistanceMetric` class and the metrics listed in + `sklearn.metrics.pairwise.PAIRWISE_DISTANCE_FUNCTIONS`. Note that the "cosine" + metric uses :func:`~sklearn.metrics.pairwise.cosine_distances`. + + A list of valid metrics for any of the above algorithms can be obtained by using their + ``valid_metric`` attribute. For example, valid metrics for ``KDTree`` can be generated by: + + >>> from sklearn.neighbors import KDTree + >>> print(sorted(KDTree.valid_metrics)) + ['chebyshev', 'cityblock', 'euclidean', 'infinity', 'l1', 'l2', 'manhattan', 'minkowski', 'p'] .. _nearest_centroid_classifier: @@ -515,10 +531,10 @@ the model from 0.81 to 0.82. .. centered:: |nearest_centroid_1| |nearest_centroid_2| -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_neighbors_plot_nearest_centroid.py`: an example of - classification using nearest centroid with different shrink thresholds. +* :ref:`sphx_glr_auto_examples_neighbors_plot_nearest_centroid.py`: an example of + classification using nearest centroid with different shrink thresholds. .. _neighbors_transformer: @@ -544,17 +560,24 @@ a scikit-learn pipeline, one can also use the corresponding classes :class:`KNeighborsTransformer` and :class:`RadiusNeighborsTransformer`. The benefits of this sparse graph API are multiple. -First, the precomputed graph can be re-used multiple times, for instance while +First, the precomputed graph can be reused multiple times, for instance while varying a parameter of the estimator. This can be done manually by the user, or using the caching properties of the scikit-learn pipeline: + >>> import tempfile >>> from sklearn.manifold import Isomap >>> from sklearn.neighbors import KNeighborsTransformer >>> from sklearn.pipeline import make_pipeline + >>> from sklearn.datasets import make_regression + >>> cache_path = tempfile.gettempdir() # we use a temporary folder here + >>> X, _ = make_regression(n_samples=50, n_features=25, random_state=0) >>> estimator = make_pipeline( - ... KNeighborsTransformer(n_neighbors=5, mode='distance'), - ... Isomap(neighbors_algorithm='precomputed'), - ... memory='/path/to/cache') + ... KNeighborsTransformer(mode='distance'), + ... Isomap(n_components=3, metric='precomputed'), + ... memory=cache_path) + >>> X_embedded = estimator.fit_transform(X) + >>> X_embedded.shape + (50, 3) Second, precomputing the graph can give finer control on the nearest neighbors estimation, for instance enabling multiprocessing though the parameter @@ -596,17 +619,17 @@ implementation with special data types. The precomputed neighbors include one extra neighbor in a custom nearest neighbors estimator, since unnecessary neighbors will be filtered by following estimators. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_neighbors_approximate_nearest_neighbors.py`: - an example of pipelining :class:`KNeighborsTransformer` and - :class:`~sklearn.manifold.TSNE`. Also proposes two custom nearest neighbors - estimators based on external packages. +* :ref:`sphx_glr_auto_examples_neighbors_approximate_nearest_neighbors.py`: + an example of pipelining :class:`KNeighborsTransformer` and + :class:`~sklearn.manifold.TSNE`. Also proposes two custom nearest neighbors + estimators based on external packages. - * :ref:`sphx_glr_auto_examples_neighbors_plot_caching_nearest_neighbors.py`: - an example of pipelining :class:`KNeighborsTransformer` and - :class:`KNeighborsClassifier` to enable caching of the neighbors graph - during a hyper-parameter grid-search. +* :ref:`sphx_glr_auto_examples_neighbors_plot_caching_nearest_neighbors.py`: + an example of pipelining :class:`KNeighborsTransformer` and + :class:`KNeighborsClassifier` to enable caching of the neighbors graph + during a hyper-parameter grid-search. .. _nca: @@ -730,11 +753,11 @@ by each method. Each data sample belongs to one of 10 classes. .. centered:: |nca_dim_reduction_1| |nca_dim_reduction_2| |nca_dim_reduction_3| -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_neighbors_plot_nca_classification.py` - * :ref:`sphx_glr_auto_examples_neighbors_plot_nca_dim_reduction.py` - * :ref:`sphx_glr_auto_examples_manifold_plot_lle_digits.py` +* :ref:`sphx_glr_auto_examples_neighbors_plot_nca_classification.py` +* :ref:`sphx_glr_auto_examples_neighbors_plot_nca_dim_reduction.py` +* :ref:`sphx_glr_auto_examples_manifold_plot_lle_digits.py` .. _nca_mathematical_formulation: @@ -767,18 +790,16 @@ space: p_{i j} = \frac{\exp(-||L x_i - L x_j||^2)}{\sum\limits_{k \ne i} {\exp{-(||L x_i - L x_k||^2)}}} , \quad p_{i i} = 0 +.. dropdown:: Mahalanobis distance -Mahalanobis distance -^^^^^^^^^^^^^^^^^^^^ + NCA can be seen as learning a (squared) Mahalanobis distance metric: -NCA can be seen as learning a (squared) Mahalanobis distance metric: - -.. math:: + .. math:: - || L(x_i - x_j)||^2 = (x_i - x_j)^TM(x_i - x_j), + || L(x_i - x_j)||^2 = (x_i - x_j)^TM(x_i - x_j), -where :math:`M = L^T L` is a symmetric positive semi-definite matrix of size -``(n_features, n_features)``. + where :math:`M = L^T L` is a symmetric positive semi-definite matrix of size + ``(n_features, n_features)``. Implementation @@ -811,12 +832,12 @@ complexity equals ``n_components * n_features * n_samples_test``. There is no added space complexity in the operation. -.. topic:: References: +.. rubric:: References - .. [1] `"Neighbourhood Components Analysis" - `_, - J. Goldberger, S. Roweis, G. Hinton, R. Salakhutdinov, Advances in - Neural Information Processing Systems, Vol. 17, May 2005, pp. 513-520. +.. [1] `"Neighbourhood Components Analysis" + `_, + J. Goldberger, S. Roweis, G. Hinton, R. Salakhutdinov, Advances in + Neural Information Processing Systems, Vol. 17, May 2005, pp. 513-520. - `Wikipedia entry on Neighborhood Components Analysis - `_ +* `Wikipedia entry on Neighborhood Components Analysis + `_ diff --git a/doc/modules/neural_networks_supervised.rst b/doc/modules/neural_networks_supervised.rst index b9f4e17c6a575..13611b7f52775 100644 --- a/doc/modules/neural_networks_supervised.rst +++ b/doc/modules/neural_networks_supervised.rst @@ -20,7 +20,7 @@ Multi-layer Perceptron ====================== **Multi-layer Perceptron (MLP)** is a supervised learning algorithm that learns -a function :math:`f(\cdot): R^m \rightarrow R^o` by training on a dataset, +a function :math:`f: R^m \rightarrow R^o` by training on a dataset, where :math:`m` is the number of dimensions for input and :math:`o` is the number of dimensions for output. Given a set of features :math:`X = {x_1, x_2, ..., x_m}` and a target :math:`y`, it can learn a non-linear function approximator for either @@ -49,27 +49,29 @@ The module contains the public attributes ``coefs_`` and ``intercepts_``. :math:`i+1`. ``intercepts_`` is a list of bias vectors, where the vector at index :math:`i` represents the bias values added to layer :math:`i+1`. -The advantages of Multi-layer Perceptron are: +.. dropdown:: Advantages and disadvantages of Multi-layer Perceptron - + Capability to learn non-linear models. + The advantages of Multi-layer Perceptron are: - + Capability to learn models in real-time (on-line learning) - using ``partial_fit``. + + Capability to learn non-linear models. + + Capability to learn models in real-time (on-line learning) + using ``partial_fit``. -The disadvantages of Multi-layer Perceptron (MLP) include: - + MLP with hidden layers have a non-convex loss function where there exists - more than one local minimum. Therefore different random weight - initializations can lead to different validation accuracy. + The disadvantages of Multi-layer Perceptron (MLP) include: - + MLP requires tuning a number of hyperparameters such as the number of - hidden neurons, layers, and iterations. + + MLP with hidden layers has a non-convex loss function where there exists + more than one local minimum. Therefore, different random weight + initializations can lead to different validation accuracy. - + MLP is sensitive to feature scaling. + + MLP requires tuning a number of hyperparameters such as the number of + hidden neurons, layers, and iterations. -Please see :ref:`Tips on Practical Use ` section that addresses -some of these disadvantages. + + MLP is sensitive to feature scaling. + + Please see :ref:`Tips on Practical Use ` section that addresses + some of these disadvantages. Classification @@ -114,8 +116,8 @@ classification, it minimizes the Cross-Entropy loss function, giving a vector of probability estimates :math:`P(y|x)` per sample :math:`x`:: >>> clf.predict_proba([[2., 2.], [1., 2.]]) - array([[1.967...e-04, 9.998...-01], - [1.967...e-04, 9.998...-01]]) + array([[1.967e-04, 9.998e-01], + [1.967e-04, 9.998e-01]]) :class:`MLPClassifier` supports multi-class classification by applying `Softmax `_ @@ -125,7 +127,7 @@ Further, the model supports :ref:`multi-label classification ` in which a sample can belong to more than one class. For each class, the raw output passes through the logistic function. Values larger or equal to `0.5` are rounded to `1`, otherwise to `0`. For a predicted output of a sample, the -indices where the value is `1` represents the assigned classes of that sample:: +indices where the value is `1` represent the assigned classes of that sample:: >>> X = [[0., 0.], [1., 1.]] >>> y = [[0, 1], [1, 1]] @@ -143,10 +145,11 @@ indices where the value is `1` represents the assigned classes of that sample:: See the examples below and the docstring of :meth:`MLPClassifier.fit` for further information. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_neural_networks_plot_mlp_training_curves.py` - * :ref:`sphx_glr_auto_examples_neural_networks_plot_mnist_filters.py` +* :ref:`sphx_glr_auto_examples_neural_networks_plot_mlp_training_curves.py` +* See :ref:`sphx_glr_auto_examples_neural_networks_plot_mnist_filters.py` for + visualized representation of trained weights. Regression ========== @@ -175,16 +178,16 @@ decision function with value of alpha. See the examples below for further information. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_neural_networks_plot_mlp_alpha.py` +* :ref:`sphx_glr_auto_examples_neural_networks_plot_mlp_alpha.py` Algorithms ========== MLP trains using `Stochastic Gradient Descent `_, -`Adam `_, or +:arxiv:`Adam <1412.6980>`, or `L-BFGS `__. Stochastic Gradient Descent (SGD) updates parameters using the gradient of the loss function with respect to a parameter that needs adaptation, i.e. @@ -199,7 +202,7 @@ the parameter space search. :math:`Loss` is the loss function used for the network. More details can be found in the documentation of -`SGD `_ +`SGD `_ Adam is similar to SGD in a sense that it is a stochastic optimizer, but it can automatically adjust the amount to update parameters based on adaptive estimates @@ -223,86 +226,82 @@ Complexity Suppose there are :math:`n` training samples, :math:`m` features, :math:`k` hidden layers, each containing :math:`h` neurons - for simplicity, and :math:`o` output neurons. The time complexity of backpropagation is -:math:`O(n\cdot m \cdot h^k \cdot o \cdot i)`, where :math:`i` is the number +:math:`O(i \cdot n \cdot (m \cdot h + (k - 1) \cdot h \cdot h + h \cdot o))`, where :math:`i` is the number of iterations. Since backpropagation has a high time complexity, it is advisable to start with smaller number of hidden neurons and few hidden layers for training. +.. dropdown:: Mathematical formulation -Mathematical formulation -======================== - -Given a set of training examples :math:`(x_1, y_1), (x_2, y_2), \ldots, (x_n, y_n)` -where :math:`x_i \in \mathbf{R}^n` and :math:`y_i \in \{0, 1\}`, a one hidden -layer one hidden neuron MLP learns the function :math:`f(x) = W_2 g(W_1^T x + b_1) + b_2` -where :math:`W_1 \in \mathbf{R}^m` and :math:`W_2, b_1, b_2 \in \mathbf{R}` are -model parameters. :math:`W_1, W_2` represent the weights of the input layer and -hidden layer, respectively; and :math:`b_1, b_2` represent the bias added to -the hidden layer and the output layer, respectively. -:math:`g(\cdot) : R \rightarrow R` is the activation function, set by default as -the hyperbolic tan. It is given as, - -.. math:: - g(z)= \frac{e^z-e^{-z}}{e^z+e^{-z}} - -For binary classification, :math:`f(x)` passes through the logistic function -:math:`g(z)=1/(1+e^{-z})` to obtain output values between zero and one. A -threshold, set to 0.5, would assign samples of outputs larger or equal 0.5 -to the positive class, and the rest to the negative class. - -If there are more than two classes, :math:`f(x)` itself would be a vector of -size (n_classes,). Instead of passing through logistic function, it passes -through the softmax function, which is written as, + Given a set of training examples :math:`(x_1, y_1), (x_2, y_2), \ldots, (x_n, y_n)` + where :math:`x_i \in \mathbf{R}^n` and :math:`y_i \in \{0, 1\}`, a one hidden + layer one hidden neuron MLP learns the function :math:`f(x) = W_2 g(W_1^T x + b_1) + b_2` + where :math:`W_1 \in \mathbf{R}^m` and :math:`W_2, b_1, b_2 \in \mathbf{R}` are + model parameters. :math:`W_1, W_2` represent the weights of the input layer and + hidden layer, respectively; and :math:`b_1, b_2` represent the bias added to + the hidden layer and the output layer, respectively. + :math:`g(\cdot) : R \rightarrow R` is the activation function, set by default as + the hyperbolic tan. It is given as, -.. math:: - \text{softmax}(z)_i = \frac{\exp(z_i)}{\sum_{l=1}^k\exp(z_l)} + .. math:: + g(z)= \frac{e^z-e^{-z}}{e^z+e^{-z}} -where :math:`z_i` represents the :math:`i` th element of the input to softmax, -which corresponds to class :math:`i`, and :math:`K` is the number of classes. -The result is a vector containing the probabilities that sample :math:`x` -belong to each class. The output is the class with the highest probability. + For binary classification, :math:`f(x)` passes through the logistic function + :math:`g(z)=1/(1+e^{-z})` to obtain output values between zero and one. A + threshold, set to 0.5, would assign samples of outputs larger or equal 0.5 + to the positive class, and the rest to the negative class. -In regression, the output remains as :math:`f(x)`; therefore, output activation -function is just the identity function. + If there are more than two classes, :math:`f(x)` itself would be a vector of + size (n_classes,). Instead of passing through logistic function, it passes + through the softmax function, which is written as, -MLP uses different loss functions depending on the problem type. The loss -function for classification is Cross-Entropy, which in binary case is given as, + .. math:: + \text{softmax}(z)_i = \frac{\exp(z_i)}{\sum_{l=1}^k\exp(z_l)} -.. math:: + where :math:`z_i` represents the :math:`i` th element of the input to softmax, + which corresponds to class :math:`i`, and :math:`K` is the number of classes. + The result is a vector containing the probabilities that sample :math:`x` + belongs to each class. The output is the class with the highest probability. - Loss(\hat{y},y,W) = -y \ln {\hat{y}} - (1-y) \ln{(1-\hat{y})} + \alpha ||W||_2^2 + In regression, the output remains as :math:`f(x)`; therefore, output activation + function is just the identity function. -where :math:`\alpha ||W||_2^2` is an L2-regularization term (aka penalty) -that penalizes complex models; and :math:`\alpha > 0` is a non-negative -hyperparameter that controls the magnitude of the penalty. + MLP uses different loss functions depending on the problem type. The loss + function for classification is Average Cross-Entropy, which in binary case is + given as, -For regression, MLP uses the Square Error loss function; written as, + .. math:: -.. math:: + Loss(\hat{y},y,W) = -\dfrac{1}{n}\sum_{i=0}^n(y_i \ln {\hat{y_i}} + (1-y_i) \ln{(1-\hat{y_i})}) + \dfrac{\alpha}{2n} ||W||_2^2 - Loss(\hat{y},y,W) = \frac{1}{2}||\hat{y} - y ||_2^2 + \frac{\alpha}{2} ||W||_2^2 + where :math:`\alpha ||W||_2^2` is an L2-regularization term (aka penalty) + that penalizes complex models; and :math:`\alpha > 0` is a non-negative + hyperparameter that controls the magnitude of the penalty. + For regression, MLP uses the Mean Square Error loss function; written as, -Starting from initial random weights, multi-layer perceptron (MLP) minimizes -the loss function by repeatedly updating these weights. After computing the -loss, a backward pass propagates it from the output layer to the previous -layers, providing each weight parameter with an update value meant to decrease -the loss. + .. math:: -In gradient descent, the gradient :math:`\nabla Loss_{W}` of the loss with respect -to the weights is computed and deducted from :math:`W`. -More formally, this is expressed as, + Loss(\hat{y},y,W) = \frac{1}{2n}\sum_{i=0}^n||\hat{y}_i - y_i ||_2^2 + \frac{\alpha}{2n} ||W||_2^2 -.. math:: - W^{i+1} = W^i - \epsilon \nabla {Loss}_{W}^{i} + Starting from initial random weights, multi-layer perceptron (MLP) minimizes + the loss function by repeatedly updating these weights. After computing the + loss, a backward pass propagates it from the output layer to the previous + layers, providing each weight parameter with an update value meant to decrease + the loss. + In gradient descent, the gradient :math:`\nabla Loss_{W}` of the loss with respect + to the weights is computed and deducted from :math:`W`. + More formally, this is expressed as, -where :math:`i` is the iteration step, and :math:`\epsilon` is the learning rate -with a value larger than 0. + .. math:: + W^{i+1} = W^i - \epsilon \nabla {Loss}_{W}^{i} -The algorithm stops when it reaches a preset maximum number of iterations; or -when the improvement in loss is below a certain, small number. + where :math:`i` is the iteration step, and :math:`\epsilon` is the learning rate + with a value larger than 0. + The algorithm stops when it reaches a preset maximum number of iterations; or + when the improvement in loss is below a certain, small number. .. _mlp_tips: @@ -310,34 +309,35 @@ when the improvement in loss is below a certain, small number. Tips on Practical Use ===================== - * Multi-layer Perceptron is sensitive to feature scaling, so it - is highly recommended to scale your data. For example, scale each - attribute on the input vector X to [0, 1] or [-1, +1], or standardize - it to have mean 0 and variance 1. Note that you must apply the *same* - scaling to the test set for meaningful results. - You can use :class:`StandardScaler` for standardization. - - >>> from sklearn.preprocessing import StandardScaler # doctest: +SKIP - >>> scaler = StandardScaler() # doctest: +SKIP - >>> # Don't cheat - fit only on training data - >>> scaler.fit(X_train) # doctest: +SKIP - >>> X_train = scaler.transform(X_train) # doctest: +SKIP - >>> # apply same transformation to test data - >>> X_test = scaler.transform(X_test) # doctest: +SKIP - - An alternative and recommended approach is to use :class:`StandardScaler` - in a :class:`Pipeline` - - * Finding a reasonable regularization parameter :math:`\alpha` is - best done using :class:`GridSearchCV`, usually in the - range ``10.0 ** -np.arange(1, 7)``. - - * Empirically, we observed that `L-BFGS` converges faster and - with better solutions on small datasets. For relatively large - datasets, however, `Adam` is very robust. It usually converges - quickly and gives pretty good performance. `SGD` with momentum or - nesterov's momentum, on the other hand, can perform better than - those two algorithms if learning rate is correctly tuned. +* Multi-layer Perceptron is sensitive to feature scaling, so it + is highly recommended to scale your data. For example, scale each + attribute on the input vector X to [0, 1] or [-1, +1], or standardize + it to have mean 0 and variance 1. Note that you must apply the *same* + scaling to the test set for meaningful results. + You can use :class:`~sklearn.preprocessing.StandardScaler` for standardization. + + >>> from sklearn.preprocessing import StandardScaler # doctest: +SKIP + >>> scaler = StandardScaler() # doctest: +SKIP + >>> # Don't cheat - fit only on training data + >>> scaler.fit(X_train) # doctest: +SKIP + >>> X_train = scaler.transform(X_train) # doctest: +SKIP + >>> # apply same transformation to test data + >>> X_test = scaler.transform(X_test) # doctest: +SKIP + + An alternative and recommended approach is to use + :class:`~sklearn.preprocessing.StandardScaler` in a + :class:`~sklearn.pipeline.Pipeline` + +* Finding a reasonable regularization parameter :math:`\alpha` is best done + using :class:`~sklearn.model_selection.GridSearchCV`, usually in the range + ``10.0 ** -np.arange(1, 7)``. + +* Empirically, we observed that `L-BFGS` converges faster and + with better solutions on small datasets. For relatively large + datasets, however, `Adam` is very robust. It usually converges + quickly and gives pretty good performance. `SGD` with momentum or + nesterov's momentum, on the other hand, can perform better than + those two algorithms if learning rate is correctly tuned. More control with warm_start ============================ @@ -353,21 +353,19 @@ or want to do additional monitoring, using ``warm_start=True`` and ... # additional monitoring / inspection MLPClassifier(... -.. topic:: References: +.. dropdown:: References - * `"Learning representations by back-propagating errors." - `_ - Rumelhart, David E., Geoffrey E. Hinton, and Ronald J. Williams. + * `"Learning representations by back-propagating errors." + `_ + Rumelhart, David E., Geoffrey E. Hinton, and Ronald J. Williams. - * `"Stochastic Gradient Descent" `_ L. Bottou - Website, 2010. + * `"Stochastic Gradient Descent" `_ L. Bottou - Website, 2010. - * `"Backpropagation" `_ - Andrew Ng, Jiquan Ngiam, Chuan Yu Foo, Yifan Mai, Caroline Suen - Website, 2011. + * `"Backpropagation" `_ + Andrew Ng, Jiquan Ngiam, Chuan Yu Foo, Yifan Mai, Caroline Suen - Website, 2011. - * `"Efficient BackProp" `_ - Y. LeCun, L. Bottou, G. Orr, K. MÃŧller - In Neural Networks: Tricks - of the Trade 1998. + * `"Efficient BackProp" `_ + Y. LeCun, L. Bottou, G. Orr, K. MÃŧller - In Neural Networks: Tricks of the Trade 1998. - * `"Adam: A method for stochastic optimization." - `_ - Kingma, Diederik, and Jimmy Ba. arXiv preprint arXiv:1412.6980 (2014). + * :arxiv:`"Adam: A method for stochastic optimization." <1412.6980>` + Kingma, Diederik, and Jimmy Ba (2014) diff --git a/doc/modules/neural_networks_unsupervised.rst b/doc/modules/neural_networks_unsupervised.rst index aca56ae8aaf2e..7f6c0016d183b 100644 --- a/doc/modules/neural_networks_unsupervised.rst +++ b/doc/modules/neural_networks_unsupervised.rst @@ -37,9 +37,9 @@ weights of independent RBMs. This method is known as unsupervised pre-training. :align: center :scale: 100% -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_neural_networks_plot_rbm_logistic_classification.py` +* :ref:`sphx_glr_auto_examples_neural_networks_plot_rbm_logistic_classification.py` Graphical model and parametrization @@ -57,7 +57,7 @@ visible and hidden unit, omitted from the image for simplicity. The energy function measures the quality of a joint assignment: -.. math:: +.. math:: E(\mathbf{v}, \mathbf{h}) = -\sum_i \sum_j w_{ij}v_ih_j - \sum_i b_iv_i - \sum_j c_jh_j @@ -149,13 +149,13 @@ step, in PCD we keep a number of chains (fantasy particles) that are updated :math:`k` Gibbs steps after each weight update. This allows the particles to explore the space more thoroughly. -.. topic:: References: +.. rubric:: References - * `"A fast learning algorithm for deep belief nets" - `_ - G. Hinton, S. Osindero, Y.-W. Teh, 2006 +* `"A fast learning algorithm for deep belief nets" + `_, + G. Hinton, S. Osindero, Y.-W. Teh, 2006 - * `"Training Restricted Boltzmann Machines using Approximations to - the Likelihood Gradient" - `_ - T. Tieleman, 2008 +* `"Training Restricted Boltzmann Machines using Approximations to + the Likelihood Gradient" + `_, + T. Tieleman, 2008 diff --git a/doc/modules/outlier_detection.rst b/doc/modules/outlier_detection.rst index 5d2008f3c3f58..7de2da4f1818e 100644 --- a/doc/modules/outlier_detection.rst +++ b/doc/modules/outlier_detection.rst @@ -74,20 +74,22 @@ not available. When ``novelty`` is set to ``True`` be aware that you must only use ``predict``, ``decision_function`` and ``score_samples`` on new unseen data and not on the training samples as this would lead to wrong results. + I.e., the result of ``predict`` will not be the same as ``fit_predict``. The scores of abnormality of the training samples are always accessible through the ``negative_outlier_factor_`` attribute. The behavior of :class:`neighbors.LocalOutlierFactor` is summarized in the following table. -===================== ================================ ===================== -Method Outlier detection Novelty detection -===================== ================================ ===================== -``fit_predict`` OK Not available -``predict`` Not available Use only on new data -``decision_function`` Not available Use only on new data -``score_samples`` Use ``negative_outlier_factor_`` Use only on new data -===================== ================================ ===================== +============================ ================================ ===================== +Method Outlier detection Novelty detection +============================ ================================ ===================== +``fit_predict`` OK Not available +``predict`` Not available Use only on new data +``decision_function`` Not available Use only on new data +``score_samples`` Use ``negative_outlier_factor_`` Use only on new data +``negative_outlier_factor_`` OK OK +============================ ================================ ===================== Overview of outlier detection methods @@ -110,19 +112,30 @@ does not perform very well for outlier detection. That being said, outlier detection in high-dimension, or without any assumptions on the distribution of the inlying data is very challenging. :class:`svm.OneClassSVM` may still be used with outlier detection but requires fine-tuning of its hyperparameter -`nu` to handle outliers and prevent overfitting. Finally, -:class:`covariance.EllipticEnvelope` assumes the data is Gaussian and learns -an ellipse. For more details on the different estimators refer to the example +`nu` to handle outliers and prevent overfitting. +:class:`linear_model.SGDOneClassSVM` provides an implementation of a +linear One-Class SVM with a linear complexity in the number of samples. This +implementation is here used with a kernel approximation technique to obtain +results similar to :class:`svm.OneClassSVM` which uses a Gaussian kernel +by default. Finally, :class:`covariance.EllipticEnvelope` assumes the data is +Gaussian and learns an ellipse. For more details on the different estimators +refer to the example :ref:`sphx_glr_auto_examples_miscellaneous_plot_anomaly_comparison.py` and the sections hereunder. -.. topic:: Examples: +.. rubric:: Examples - * See :ref:`sphx_glr_auto_examples_miscellaneous_plot_anomaly_comparison.py` - for a comparison of the :class:`svm.OneClassSVM`, the - :class:`ensemble.IsolationForest`, the - :class:`neighbors.LocalOutlierFactor` and - :class:`covariance.EllipticEnvelope`. +* See :ref:`sphx_glr_auto_examples_miscellaneous_plot_anomaly_comparison.py` + for a comparison of the :class:`svm.OneClassSVM`, the + :class:`ensemble.IsolationForest`, the + :class:`neighbors.LocalOutlierFactor` and + :class:`covariance.EllipticEnvelope`. + +* See :ref:`sphx_glr_auto_examples_miscellaneous_plot_outlier_detection_bench.py` + for an example showing how to evaluate outlier detection estimators, + the :class:`neighbors.LocalOutlierFactor` and the + :class:`ensemble.IsolationForest`, using ROC curves from + :class:`metrics.RocCurveDisplay`. Novelty Detection ================= @@ -140,7 +153,7 @@ In general, it is about to learn a rough, close frontier delimiting the contour of the initial observations distribution, plotted in embedding :math:`p`-dimensional space. Then, if further observations lay within the frontier-delimited subspace, they are considered as -coming from the same population than the initial +coming from the same population as the initial observations. Otherwise, if they lay outside the frontier, we can say that they are abnormal with a given confidence in our assessment. @@ -154,18 +167,18 @@ implementation. The `nu` parameter, also known as the margin of the One-Class SVM, corresponds to the probability of finding a new, but regular, observation outside the frontier. -.. topic:: References: +.. rubric:: References + +* `Estimating the support of a high-dimensional distribution + `_ + SchÃļlkopf, Bernhard, et al. Neural computation 13.7 (2001): 1443-1471. - * `Estimating the support of a high-dimensional distribution - `_ - SchÃļlkopf, Bernhard, et al. Neural computation 13.7 (2001): 1443-1471. +.. rubric:: Examples -.. topic:: Examples: +* See :ref:`sphx_glr_auto_examples_svm_plot_oneclass.py` for visualizing the + frontier learned around some data by a :class:`svm.OneClassSVM` object. - * See :ref:`sphx_glr_auto_examples_svm_plot_oneclass.py` for visualizing the - frontier learned around some data by a - :class:`svm.OneClassSVM` object. - * :ref:`sphx_glr_auto_examples_applications_plot_species_distribution_modeling.py` +* :ref:`sphx_glr_auto_examples_applications_plot_species_distribution_modeling.py` .. figure:: ../auto_examples/svm/images/sphx_glr_plot_oneclass_001.png :target: ../auto_examples/svm/plot_oneclass.html @@ -173,6 +186,23 @@ but regular, observation outside the frontier. :scale: 75% +Scaling up the One-Class SVM +---------------------------- + +An online linear version of the One-Class SVM is implemented in +:class:`linear_model.SGDOneClassSVM`. This implementation scales linearly with +the number of samples and can be used with a kernel approximation to +approximate the solution of a kernelized :class:`svm.OneClassSVM` whose +complexity is at best quadratic in the number of samples. See section +:ref:`sgd_online_one_class_svm` for more details. + +.. rubric:: Examples + +* See :ref:`sphx_glr_auto_examples_linear_model_plot_sgdocsvm_vs_ocsvm.py` + for an illustration of the approximation of a kernelized One-Class SVM + with the `linear_model.SGDOneClassSVM` combined with kernel approximation. + + Outlier Detection ================= @@ -200,7 +230,7 @@ points, ignoring points outside the central mode. For instance, assuming that the inlier data are Gaussian distributed, it will estimate the inlier location and covariance in a robust way (i.e. without being influenced by outliers). The Mahalanobis distances -obtained from this estimate is used to derive a measure of outlyingness. +obtained from this estimate are used to derive a measure of outlyingness. This strategy is illustrated below. .. figure:: ../auto_examples/covariance/images/sphx_glr_plot_mahalanobis_distances_001.png @@ -208,18 +238,22 @@ This strategy is illustrated below. :align: center :scale: 75% -.. topic:: Examples: +.. rubric:: Examples + +* See :ref:`sphx_glr_auto_examples_covariance_plot_mahalanobis_distances.py` for + an illustration of the difference between using a standard + (:class:`covariance.EmpiricalCovariance`) or a robust estimate + (:class:`covariance.MinCovDet`) of location and covariance to + assess the degree of outlyingness of an observation. - * See :ref:`sphx_glr_auto_examples_covariance_plot_mahalanobis_distances.py` for - an illustration of the difference between using a standard - (:class:`covariance.EmpiricalCovariance`) or a robust estimate - (:class:`covariance.MinCovDet`) of location and covariance to - assess the degree of outlyingness of an observation. +* See :ref:`sphx_glr_auto_examples_applications_plot_outlier_detection_wine.py` + for an example of robust covariance estimation on a real data set. -.. topic:: References: - * Rousseeuw, P.J., Van Driessen, K. "A fast algorithm for the minimum - covariance determinant estimator" Technometrics 41(3), 212 (1999) +.. rubric:: References + +* Rousseeuw, P.J., Van Driessen, K. "A fast algorithm for the minimum + covariance determinant estimator" Technometrics 41(3), 212 (1999) .. _isolation_forest: @@ -251,7 +285,7 @@ the maximum depth of each tree is set to :math:`\lceil \log_2(n) \rceil` where This algorithm is illustrated below. -.. figure:: ../auto_examples/ensemble/images/sphx_glr_plot_isolation_forest_001.png +.. figure:: ../auto_examples/ensemble/images/sphx_glr_plot_isolation_forest_003.png :target: ../auto_examples/ensemble/plot_isolation_forest.html :align: center :scale: 75% @@ -269,23 +303,24 @@ allows you to add more trees to an already fitted model:: >>> clf.set_params(n_estimators=20) # add 10 more trees # doctest: +SKIP >>> clf.fit(X) # fit the added trees # doctest: +SKIP -.. topic:: Examples: +.. rubric:: Examples - * See :ref:`sphx_glr_auto_examples_ensemble_plot_isolation_forest.py` for - an illustration of the use of IsolationForest. +* See :ref:`sphx_glr_auto_examples_ensemble_plot_isolation_forest.py` for + an illustration of the use of IsolationForest. - * See :ref:`sphx_glr_auto_examples_miscellaneous_plot_anomaly_comparison.py` - for a comparison of :class:`ensemble.IsolationForest` with - :class:`neighbors.LocalOutlierFactor`, - :class:`svm.OneClassSVM` (tuned to perform like an outlier detection - method) and a covariance-based outlier detection with - :class:`covariance.EllipticEnvelope`. +* See :ref:`sphx_glr_auto_examples_miscellaneous_plot_anomaly_comparison.py` + for a comparison of :class:`ensemble.IsolationForest` with + :class:`neighbors.LocalOutlierFactor`, + :class:`svm.OneClassSVM` (tuned to perform like an outlier detection + method), :class:`linear_model.SGDOneClassSVM`, and a covariance-based + outlier detection with :class:`covariance.EllipticEnvelope`. -.. topic:: References: +.. rubric:: References - * Liu, Fei Tony, Ting, Kai Ming and Zhou, Zhi-Hua. "Isolation forest." - Data Mining, 2008. ICDM'08. Eighth IEEE International Conference on. +* Liu, Fei Tony, Ting, Kai Ming and Zhou, Zhi-Hua. "Isolation forest." + Data Mining, 2008. ICDM'08. Eighth IEEE International Conference on. +.. _local_outlier_factor: Local Outlier Factor -------------------- @@ -301,20 +336,18 @@ lower density than their neighbors. In practice the local density is obtained from the k-nearest neighbors. The LOF score of an observation is equal to the ratio of the -average local density of his k-nearest neighbors, and its own local density: +average local density of its k-nearest neighbors, and its own local density: a normal instance is expected to have a local density similar to that of its neighbors, while abnormal data are expected to have much smaller local density. -The number k of neighbors considered, (alias parameter n_neighbors) is typically -chosen 1) greater than the minimum number of objects a cluster has to contain, -so that other objects can be local outliers relative to this cluster, and 2) -smaller than the maximum number of close by objects that can potentially be -local outliers. -In practice, such informations are generally not available, and taking -n_neighbors=20 appears to work well in general. -When the proportion of outliers is high (i.e. greater than 10 \%, as in the -example below), n_neighbors should be greater (n_neighbors=35 in the example -below). +The number k of neighbors considered, (alias parameter `n_neighbors`) is +typically chosen 1) greater than the minimum number of objects a cluster has to +contain, so that other objects can be local outliers relative to this cluster, +and 2) smaller than the maximum number of close by objects that can potentially +be local outliers. In practice, such information is generally not available, and +taking `n_neighbors=20` appears to work well in general. When the proportion of +outliers is high (i.e. greater than 10 \%, as in the example below), +`n_neighbors` should be greater (`n_neighbors=35` in the example below). The strength of the LOF algorithm is that it takes both local and global properties of datasets into consideration: it can perform well even in datasets @@ -328,30 +361,31 @@ method. The scores of abnormality of the training samples are accessible through the ``negative_outlier_factor_`` attribute. Note that ``predict``, ``decision_function`` and ``score_samples`` can be used on new unseen data when LOF is applied for novelty detection, i.e. when the -``novelty`` parameter is set to ``True``. See :ref:`novelty_with_lof`. +``novelty`` parameter is set to ``True``, but the result of ``predict`` may +differ from that of ``fit_predict``. See :ref:`novelty_with_lof`. This strategy is illustrated below. .. figure:: ../auto_examples/neighbors/images/sphx_glr_plot_lof_outlier_detection_001.png - :target: ../auto_examples/neighbors/sphx_glr_plot_lof_outlier_detection.html + :target: ../auto_examples/neighbors/plot_lof_outlier_detection.html :align: center :scale: 75% -.. topic:: Examples: +.. rubric:: Examples - * See :ref:`sphx_glr_auto_examples_neighbors_plot_lof_outlier_detection.py` - for an illustration of the use of :class:`neighbors.LocalOutlierFactor`. +* See :ref:`sphx_glr_auto_examples_neighbors_plot_lof_outlier_detection.py` + for an illustration of the use of :class:`neighbors.LocalOutlierFactor`. - * See :ref:`sphx_glr_auto_examples_miscellaneous_plot_anomaly_comparison.py` - for a comparison with other anomaly detection methods. +* See :ref:`sphx_glr_auto_examples_miscellaneous_plot_anomaly_comparison.py` + for a comparison with other anomaly detection methods. -.. topic:: References: +.. rubric:: References - * Breunig, Kriegel, Ng, and Sander (2000) - `LOF: identifying density-based local outliers. - `_ - Proc. ACM SIGMOD +* Breunig, Kriegel, Ng, and Sander (2000) + `LOF: identifying density-based local outliers. + `_ + Proc. ACM SIGMOD .. _novelty_with_lof: @@ -366,19 +400,20 @@ set to ``True`` before fitting the estimator:: lof = LocalOutlierFactor(novelty=True) lof.fit(X_train) -Note that ``fit_predict`` is not available in this case. +Note that ``fit_predict`` is not available in this case to avoid inconsistencies. -.. warning:: **Novelty detection with Local Outlier Factor`** +.. warning:: **Novelty detection with Local Outlier Factor** When ``novelty`` is set to ``True`` be aware that you must only use ``predict``, ``decision_function`` and ``score_samples`` on new unseen data and not on the training samples as this would lead to wrong results. + I.e., the result of ``predict`` will not be the same as ``fit_predict``. The scores of abnormality of the training samples are always accessible through the ``negative_outlier_factor_`` attribute. Novelty detection with Local Outlier Factor is illustrated below. - .. figure:: ../auto_examples/neighbors/images/sphx_glr_plot_lof_novelty_detection_001.png - :target: ../auto_examples/neighbors/sphx_glr_plot_lof_novelty_detection.html - :align: center - :scale: 75% +.. figure:: ../auto_examples/neighbors/images/sphx_glr_plot_lof_novelty_detection_001.png + :target: ../auto_examples/neighbors/plot_lof_novelty_detection.html + :align: center + :scale: 75% diff --git a/doc/modules/partial_dependence.rst b/doc/modules/partial_dependence.rst index 89fd0f17e38bb..7f30a3a7e6731 100644 --- a/doc/modules/partial_dependence.rst +++ b/doc/modules/partial_dependence.rst @@ -11,10 +11,10 @@ Partial dependence plots (PDP) and individual conditional expectation (ICE) plots can be used to visualize and analyze interaction between the target response [1]_ and a set of input features of interest. -Both PDPs and ICEs assume that the input features of interest are independent -from the complement features, and this assumption is often violated in practice. -Thus, in the case of correlated features, we will create absurd data points to -compute the PDP/ICE. +Both PDPs [H2009]_ and ICEs [G2015]_ assume that the input features of interest +are independent from the complement features, and this assumption is often +violated in practice. Thus, in the case of correlated features, we will +create absurd data points to compute the PDP/ICE [M2019]_. Partial dependence plots ======================== @@ -25,66 +25,75 @@ of all other input features (the 'complement' features). Intuitively, we can interpret the partial dependence as the expected target response as a function of the input features of interest. -Due to the limits of human perception the size of the set of input feature of +Due to the limits of human perception, the size of the set of input features of interest must be small (usually, one or two) thus the input features of interest are usually chosen among the most important features. The figure below shows two one-way and one two-way partial dependence plots for -the California housing dataset, with a :class:`HistGradientBoostingRegressor -`: +the bike sharing dataset, with a +:class:`~sklearn.ensemble.HistGradientBoostingRegressor`: -.. figure:: ../auto_examples/inspection/images/sphx_glr_plot_partial_dependence_003.png +.. figure:: ../auto_examples/inspection/images/sphx_glr_plot_partial_dependence_006.png :target: ../auto_examples/inspection/plot_partial_dependence.html :align: center :scale: 70 -One-way PDPs tell us about the interaction between the target response and an -input feature of interest feature (e.g. linear, non-linear). The left plot -in the above figure shows the effect of the average occupancy on the median -house price; we can clearly see a linear relationship among them when the -average occupancy is inferior to 3 persons. Similarly, we could analyze the -effect of the house age on the median house price (middle plot). Thus, these -interpretations are marginal, considering a feature at a time. - -PDPs with two input features of interest show the interactions among the two -features. For example, the two-variable PDP in the above figure shows the -dependence of median house price on joint values of house age and average -occupants per household. We can clearly see an interaction between the two -features: for an average occupancy greater than two, the house price is nearly -independent of the house age, whereas for values less than 2 there is a strong -dependence on age. +One-way PDPs tell us about the interaction between the target response and an input +feature of interest (e.g. linear, non-linear). The left plot in the above figure +shows the effect of the temperature on the number of bike rentals; we can clearly see +that a higher temperature is related with a higher number of bike rentals. Similarly, we +could analyze the effect of the humidity on the number of bike rentals (middle plot). +Thus, these interpretations are marginal, considering a feature at a time. + +PDPs with two input features of interest show the interactions among the two features. +For example, the two-variable PDP in the above figure shows the dependence of the number +of bike rentals on joint values of temperature and humidity. We can clearly see an +interaction between the two features: with a temperature higher than 20 degrees Celsius, +mainly the humidity has a strong impact on the number of bike rentals. For lower +temperatures, both the temperature and the humidity have an impact on the number of bike +rentals. The :mod:`sklearn.inspection` module provides a convenience function -:func:`plot_partial_dependence` to create one-way and two-way partial +:func:`~PartialDependenceDisplay.from_estimator` to create one-way and two-way partial dependence plots. In the below example we show how to create a grid of partial dependence plots: two one-way PDPs for the features ``0`` and ``1`` and a two-way PDP between the two features:: >>> from sklearn.datasets import make_hastie_10_2 >>> from sklearn.ensemble import GradientBoostingClassifier - >>> from sklearn.inspection import plot_partial_dependence + >>> from sklearn.inspection import PartialDependenceDisplay >>> X, y = make_hastie_10_2(random_state=0) >>> clf = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0, ... max_depth=1, random_state=0).fit(X, y) >>> features = [0, 1, (0, 1)] - >>> plot_partial_dependence(clf, X, features) #doctest: +SKIP + >>> PartialDependenceDisplay.from_estimator(clf, X, features) + <...> You can access the newly created figure and Axes objects using ``plt.gcf()`` and ``plt.gca()``. -For multi-class classification, you need to set the class label for which -the PDPs should be created via the ``target`` argument:: +To make a partial dependence plot with categorical features, you need to specify +which features are categorical using the parameter `categorical_features`. This +parameter takes a list of indices, names of the categorical features or a boolean +mask. The graphical representation of partial dependence for categorical features is +a bar plot or a 2D heatmap. - >>> from sklearn.datasets import load_iris - >>> iris = load_iris() - >>> mc_clf = GradientBoostingClassifier(n_estimators=10, - ... max_depth=1).fit(iris.data, iris.target) - >>> features = [3, 2, (3, 2)] - >>> plot_partial_dependence(mc_clf, X, features, target=0) #doctest: +SKIP +.. dropdown:: PDPs for multi-class classification -The same parameter ``target`` is used to specify the target in multi-output -regression settings. + For multi-class classification, you need to set the class label for which + the PDPs should be created via the ``target`` argument:: + + >>> from sklearn.datasets import load_iris + >>> iris = load_iris() + >>> mc_clf = GradientBoostingClassifier(n_estimators=10, + ... max_depth=1).fit(iris.data, iris.target) + >>> features = [3, 2, (3, 2)] + >>> PartialDependenceDisplay.from_estimator(mc_clf, X, features, target=0) + <...> + + The same parameter ``target`` is used to specify the target in multi-output + regression settings. If you need the raw values of the partial dependence function rather than the plots, you can use the @@ -92,10 +101,10 @@ the plots, you can use the >>> from sklearn.inspection import partial_dependence - >>> pdp, axes = partial_dependence(clf, X, [0]) - >>> pdp + >>> results = partial_dependence(clf, X, [0]) + >>> results["average"] array([[ 2.466..., 2.466..., ... - >>> axes + >>> results["grid_values"] [array([-1.624..., -1.592..., ... The values at which the partial dependence should be evaluated are directly @@ -118,12 +127,11 @@ feature for each sample separately with one line per sample. Due to the limits of human perception, only one input feature of interest is supported for ICE plots. -The figures below show four ICE plots for the California housing dataset, -with a :class:`HistGradientBoostingRegressor -`. The second figure plots +The figures below show two ICE plots for the bike sharing dataset, +with a :class:`~sklearn.ensemble.HistGradientBoostingRegressor`. The figures plot the corresponding PD line overlaid on ICE lines. -.. figure:: ../auto_examples/inspection/images/sphx_glr_plot_partial_dependence_002.png +.. figure:: ../auto_examples/inspection/images/sphx_glr_plot_partial_dependence_004.png :target: ../auto_examples/inspection/plot_partial_dependence.html :align: center :scale: 70 @@ -131,34 +139,49 @@ the corresponding PD line overlaid on ICE lines. While the PDPs are good at showing the average effect of the target features, they can obscure a heterogeneous relationship created by interactions. When interactions are present the ICE plot will provide many more insights. -For example, we could observe a linear relationship between the median income -and the house price in the PD line. However, the ICE lines show that there -are some exceptions, where the house price remains constant in some ranges of -the median income. +For example, we see that the ICE for the temperature feature gives us some +additional information: some of the ICE lines are flat while some others +show a decrease of the dependence for temperature above 35 degrees Celsius. +We observe a similar pattern for the humidity feature: some of the ICE +lines show a sharp decrease when the humidity is above 80%. -The :mod:`sklearn.inspection` module's :func:`plot_partial_dependence` +The :mod:`sklearn.inspection` module's :meth:`PartialDependenceDisplay.from_estimator` convenience function can be used to create ICE plots by setting ``kind='individual'``. In the example below, we show how to create a grid of ICE plots: >>> from sklearn.datasets import make_hastie_10_2 >>> from sklearn.ensemble import GradientBoostingClassifier - >>> from sklearn.inspection import plot_partial_dependence + >>> from sklearn.inspection import PartialDependenceDisplay >>> X, y = make_hastie_10_2(random_state=0) >>> clf = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0, ... max_depth=1, random_state=0).fit(X, y) >>> features = [0, 1] - >>> plot_partial_dependence(clf, X, features, - ... kind='individual') # doctest: +SKIP + >>> PartialDependenceDisplay.from_estimator(clf, X, features, + ... kind='individual') + <...> In ICE plots it might not be easy to see the average effect of the input feature of interest. Hence, it is recommended to use ICE plots alongside PDPs. They can be plotted together with ``kind='both'``. - >>> plot_partial_dependence(clf, X, features, - ... kind='both') # doctest: +SKIP + >>> PartialDependenceDisplay.from_estimator(clf, X, features, + ... kind='both') + <...> + +If there are too many lines in an ICE plot, it can be difficult to see +differences between individual samples and interpret the model. Centering the +ICE at the first value on the x-axis, produces centered Individual Conditional +Expectation (cICE) plots [G2015]_. This puts emphasis on the divergence of +individual conditional expectations from the mean line, thus making it easier +to explore heterogeneous relationships. cICE plots can be plotted by setting +`centered=True`: + + >>> PartialDependenceDisplay.from_estimator(clf, X, features, + ... kind='both', centered=True) + <...> Mathematical Definition ======================= @@ -188,11 +211,11 @@ Computation methods =================== There are two main methods to approximate the integral above, namely the -'brute' and 'recursion' methods. The `method` parameter controls which method +`'brute'` and `'recursion'` methods. The `method` parameter controls which method to use. -The 'brute' method is a generic method that works with any estimator. Note that -computing ICE plots is only supported with the 'brute' method. It +The `'brute'` method is a generic method that works with any estimator. Note that +computing ICE plots is only supported with the `'brute'` method. It approximates the above integral by computing an average over the data `X`: .. math:: @@ -206,23 +229,23 @@ over the dataset `X` which is computationally intensive. Each of the :math:`f(x_{S}, x_{C}^{(i)})` corresponds to one ICE line evaluated at :math:`x_{S}`. Computing this for multiple values of :math:`x_{S}`, one obtains a full ICE line. As one can see, the average of the ICE lines -correspond to the partial dependence line. +corresponds to the partial dependence line. -The 'recursion' method is faster than the 'brute' method, but it is only +The `'recursion'` method is faster than the `'brute'` method, but it is only supported for PDP plots by some tree-based estimators. It is computed as follows. For a given point :math:`x_S`, a weighted tree traversal is performed: if a split node involves an input feature of interest, the corresponding left or right branch is followed; otherwise both branches are followed, each branch being weighted by the fraction of training samples that entered that branch. Finally, the partial dependence is given by a weighted average of all the -visited leaves values. +visited leaves' values. -With the 'brute' method, the parameter `X` is used both for generating the +With the `'brute'` method, the parameter `X` is used both for generating the grid of values :math:`x_S` and the complement feature values :math:`x_C`. However with the 'recursion' method, `X` is only used for the grid values: implicitly, the :math:`x_C` values are those of the training data. -By default, the 'recursion' method is used for plotting PDPs on tree-based +By default, the `'recursion'` method is used for plotting PDPs on tree-based estimators that support it, and 'brute' is used for the rest. .. _pdp_method_differences: @@ -230,18 +253,18 @@ estimators that support it, and 'brute' is used for the rest. .. note:: While both methods should be close in general, they might differ in some - specific settings. The 'brute' method assumes the existence of the + specific settings. The `'brute'` method assumes the existence of the data points :math:`(x_S, x_C^{(i)})`. When the features are correlated, - such artificial samples may have a very low probability mass. The 'brute' - and 'recursion' methods will likely disagree regarding the value of the + such artificial samples may have a very low probability mass. The `'brute'` + and `'recursion'` methods will likely disagree regarding the value of the partial dependence, because they will treat these unlikely samples differently. Remember, however, that the primary assumption for interpreting PDPs is that the features should be independent. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_inspection_plot_partial_dependence.py` +* :ref:`sphx_glr_auto_examples_inspection_plot_partial_dependence.py` .. rubric:: Footnotes @@ -249,17 +272,20 @@ estimators that support it, and 'brute' is used for the rest. class (the positive class for binary classification), or the decision function. -.. topic:: References +.. rubric:: References - T. Hastie, R. Tibshirani and J. Friedman, `The Elements of - Statistical Learning `_, +.. [H2009] T. Hastie, R. Tibshirani and J. Friedman, + `The Elements of Statistical Learning + `_, Second Edition, Section 10.13.2, Springer, 2009. - C. Molnar, `Interpretable Machine Learning - `_, Section 5.1, 2019. +.. [M2019] C. Molnar, + `Interpretable Machine Learning + `_, + Section 5.1, 2019. - A. Goldstein, A. Kapelner, J. Bleich, and E. Pitkin, `Peeking Inside the - Black Box: Visualizing Statistical Learning With Plots of Individual - Conditional Expectation `_, - Journal of Computational and Graphical Statistics, 24(1): 44-65, Springer, - 2015. +.. [G2015] :arxiv:`A. Goldstein, A. Kapelner, J. Bleich, and E. Pitkin, + "Peeking Inside the Black Box: Visualizing Statistical + Learning With Plots of Individual Conditional Expectation" + Journal of Computational and Graphical Statistics, + 24(1): 44-65, Springer, 2015. <1309.6392>` diff --git a/doc/modules/permutation_importance.rst b/doc/modules/permutation_importance.rst index 833c9fb9a696e..80bb5ef0eb650 100644 --- a/doc/modules/permutation_importance.rst +++ b/doc/modules/permutation_importance.rst @@ -6,15 +6,46 @@ Permutation feature importance .. currentmodule:: sklearn.inspection -Permutation feature importance is a model inspection technique that can be used -for any :term:`fitted` :term:`estimator` when the data is tabular. This is -especially useful for non-linear or opaque :term:`estimators`. The permutation -feature importance is defined to be the decrease in a model score when a single -feature value is randomly shuffled [1]_. This procedure breaks the relationship -between the feature and the target, thus the drop in the model score is -indicative of how much the model depends on the feature. This technique -benefits from being model agnostic and can be calculated many times with -different permutations of the feature. +Permutation feature importance is a model inspection technique that measures the +contribution of each feature to a :term:`fitted` model's statistical performance +on a given tabular dataset. This technique is particularly useful for non-linear +or opaque :term:`estimators`, and involves randomly shuffling the values of a +single feature and observing the resulting degradation of the model's score +[1]_. By breaking the relationship between the feature and the target, we +determine how much the model relies on such particular feature. + +In the following figures, we observe the effect of permuting features on the correlation +between the feature and the target and consequently on the model's statistical +performance. + +.. image:: ../images/permuted_predictive_feature.png + :align: center + +.. image:: ../images/permuted_non_predictive_feature.png + :align: center + +On the top figure, we observe that permuting a predictive feature breaks the +correlation between the feature and the target, and consequently the model's +statistical performance decreases. On the bottom figure, we observe that permuting +a non-predictive feature does not significantly degrade the model's statistical +performance. + +One key advantage of permutation feature importance is that it is +model-agnostic, i.e. it can be applied to any fitted estimator. Moreover, it can +be calculated multiple times with different permutations of the feature, further +providing a measure of the variance in the estimated feature importances for the +specific trained model. + +The figure below shows the permutation feature importance of a +:class:`~sklearn.ensemble.RandomForestClassifier` trained on an augmented +version of the titanic dataset that contains a `random_cat` and a `random_num` +features, i.e. a categorical and a numerical feature that are not correlated in +any way with the target variable: + +.. figure:: ../auto_examples/inspection/images/sphx_glr_plot_permutation_importance_002.png + :target: ../auto_examples/inspection/plot_permutation_importance.html + :align: center + :scale: 70 .. warning:: @@ -22,7 +53,7 @@ different permutations of the feature. cross-validation score) could be **very important for a good model**. Therefore it is always important to evaluate the predictive power of a model using a held-out set (or better with cross-validation) prior to computing - importances. Permutation importance does not reflect to the intrinsic + importances. Permutation importance does not reflect the intrinsic predictive value of a feature by itself but **how important this feature is for a particular model**. @@ -74,49 +105,50 @@ highlight which features contribute the most to the generalization power of the inspected model. Features that are important on the training set but not on the held-out set might cause the model to overfit. -The permutation feature importance is the decrease in a model score when a single -feature value is randomly shuffled. The score function to be used for the -computation of importances can be specified with the `scoring` argument, -which also accepts multiple scorers. Using multiple scorers is more computationally -efficient than sequentially calling :func:`permutation_importance` several times -with a different scorer, as it reuses model predictions. - -An example of using multiple scorers is shown below, employing a list of metrics, -but more input formats are possible, as documented in :ref:`multimetric_scoring`. - - >>> scoring = ['r2', 'neg_mean_absolute_percentage_error', 'neg_mean_squared_error'] - >>> r_multi = permutation_importance( - ... model, X_val, y_val, n_repeats=30, random_state=0, scoring=scoring) - ... - >>> for metric in r_multi: - ... print(f"{metric}") - ... r = r_multi[metric] - ... for i in r.importances_mean.argsort()[::-1]: - ... if r.importances_mean[i] - 2 * r.importances_std[i] > 0: - ... print(f" {diabetes.feature_names[i]:<8}" - ... f"{r.importances_mean[i]:.3f}" - ... f" +/- {r.importances_std[i]:.3f}") - ... - r2 - s5 0.204 +/- 0.050 - bmi 0.176 +/- 0.048 - bp 0.088 +/- 0.033 - sex 0.056 +/- 0.023 - neg_mean_absolute_percentage_error - s5 0.081 +/- 0.020 - bmi 0.064 +/- 0.015 - bp 0.029 +/- 0.010 - neg_mean_squared_error - s5 1013.903 +/- 246.460 - bmi 872.694 +/- 240.296 - bp 438.681 +/- 163.025 - sex 277.382 +/- 115.126 - -The ranking of the features is approximately the same for different metrics even -if the scales of the importance values are very different. However, this is not -guaranteed and different metrics might lead to significantly different feature -importances, in particular for models trained for imbalanced classification problems, -for which the choice of the classification metric can be critical. +The permutation feature importance depends on the score function that is +specified with the `scoring` argument. This argument accepts multiple scorers, +which is more computationally efficient than sequentially calling +:func:`permutation_importance` several times with a different scorer, as it +reuses model predictions. + +.. dropdown:: Example of permutation feature importance using multiple scorers + + In the example below we use a list of metrics, but more input formats are + possible, as documented in :ref:`multimetric_scoring`. + + >>> scoring = ['r2', 'neg_mean_absolute_percentage_error', 'neg_mean_squared_error'] + >>> r_multi = permutation_importance( + ... model, X_val, y_val, n_repeats=30, random_state=0, scoring=scoring) + ... + >>> for metric in r_multi: + ... print(f"{metric}") + ... r = r_multi[metric] + ... for i in r.importances_mean.argsort()[::-1]: + ... if r.importances_mean[i] - 2 * r.importances_std[i] > 0: + ... print(f" {diabetes.feature_names[i]:<8}" + ... f"{r.importances_mean[i]:.3f}" + ... f" +/- {r.importances_std[i]:.3f}") + ... + r2 + s5 0.204 +/- 0.050 + bmi 0.176 +/- 0.048 + bp 0.088 +/- 0.033 + sex 0.056 +/- 0.023 + neg_mean_absolute_percentage_error + s5 0.081 +/- 0.020 + bmi 0.064 +/- 0.015 + bp 0.029 +/- 0.010 + neg_mean_squared_error + s5 1013.866 +/- 246.445 + bmi 872.726 +/- 240.298 + bp 438.663 +/- 163.022 + sex 277.376 +/- 115.123 + + The ranking of the features is approximately the same for different metrics even + if the scales of the importance values are very different. However, this is not + guaranteed and different metrics might lead to significantly different feature + importances, in particular for models trained for imbalanced classification problems, + for which **the choice of the classification metric can be critical**. Outline of the permutation importance algorithm ----------------------------------------------- @@ -145,20 +177,20 @@ Relation to impurity-based importance in trees Tree-based models provide an alternative measure of :ref:`feature importances based on the mean decrease in impurity ` (MDI). Impurity is quantified by the splitting criterion of the decision trees -(Gini, Entropy or Mean Squared Error). However, this method can give high +(Gini, Log Loss or Mean Squared Error). However, this method can give high importance to features that may not be predictive on unseen data when the model is overfitting. Permutation-based feature importance, on the other hand, avoids this issue, since it can be computed on unseen data. -Furthermore, impurity-based feature importance for trees are **strongly +Furthermore, impurity-based feature importance for trees is **strongly biased** and **favor high cardinality features** (typically numerical features) over low cardinality features such as binary features or categorical variables with a small number of possible categories. Permutation-based feature importances do not exhibit such a bias. Additionally, -the permutation feature importance may be computed performance metric on the -model predictions predictions and can be used to analyze any model class (not -just tree-based models). +the permutation feature importance may be computed with any performance metric +on the model predictions and can be used to analyze any model class (not just +tree-based models). The following example highlights the limitations of impurity-based feature importance in contrast to permutation-based feature importance: @@ -168,21 +200,37 @@ Misleading values on strongly correlated features ------------------------------------------------- When two features are correlated and one of the features is permuted, the model -will still have access to the feature through its correlated feature. This will -result in a lower importance value for both features, where they might -*actually* be important. +still has access to the latter through its correlated feature. This results in a +lower reported importance value for both features, though they might *actually* +be important. + +The figure below shows the permutation feature importance of a +:class:`~sklearn.ensemble.RandomForestClassifier` trained using the +:ref:`breast_cancer_dataset`, which contains strongly correlated features. A +naive interpretation would suggest that all features are unimportant: + +.. figure:: ../auto_examples/inspection/images/sphx_glr_plot_permutation_importance_multicollinear_002.png + :target: ../auto_examples/inspection/plot_permutation_importance_multicollinear.html + :align: center + :scale: 70 + +One way to handle the issue is to cluster features that are correlated and only +keep one feature from each cluster. + +.. figure:: ../auto_examples/inspection/images/sphx_glr_plot_permutation_importance_multicollinear_004.png + :target: ../auto_examples/inspection/plot_permutation_importance_multicollinear.html + :align: center + :scale: 70 -One way to handle this is to cluster features that are correlated and only -keep one feature from each cluster. This strategy is explored in the following -example: +For more details on such strategy, see the example :ref:`sphx_glr_auto_examples_inspection_plot_permutation_importance_multicollinear.py`. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_inspection_plot_permutation_importance.py` - * :ref:`sphx_glr_auto_examples_inspection_plot_permutation_importance_multicollinear.py` +* :ref:`sphx_glr_auto_examples_inspection_plot_permutation_importance.py` +* :ref:`sphx_glr_auto_examples_inspection_plot_permutation_importance_multicollinear.py` -.. topic:: References: +.. rubric:: References - .. [1] L. Breiman, "Random Forests", Machine Learning, 45(1), 5-32, - 2001. https://doi.org/10.1023/A:1010933404324 +.. [1] L. Breiman, :doi:`"Random Forests" <10.1023/A:1010933404324>`, + Machine Learning, 45(1), 5-32, 2001. diff --git a/doc/modules/preprocessing.rst b/doc/modules/preprocessing.rst index b87971ec4ae5a..69dff95518c41 100644 --- a/doc/modules/preprocessing.rst +++ b/doc/modules/preprocessing.rst @@ -10,10 +10,11 @@ The ``sklearn.preprocessing`` package provides several common utility functions and transformer classes to change raw feature vectors into a representation that is more suitable for the downstream estimators. -In general, learning algorithms benefit from standardization of the data set. If -some outliers are present in the set, robust scalers or transformers are more -appropriate. The behaviors of the different scalers, transformers, and -normalizers on a dataset containing marginal outliers is highlighted in +In general, many learning algorithms such as linear models benefit from standardization of the data set +(see :ref:`sphx_glr_auto_examples_preprocessing_plot_scaling_importance.py`). +If some outliers are present in the set, robust scalers or other transformers can +be more appropriate. The behaviors of the different scalers, transformers, and +normalizers on a dataset containing marginal outliers are highlighted in :ref:`sphx_glr_auto_examples_preprocessing_plot_all_scaling.py`. @@ -34,8 +35,8 @@ standard deviation. For instance, many elements used in the objective function of a learning algorithm (such as the RBF kernel of Support Vector -Machines or the l1 and l2 regularizers of linear models) assume that -all features are centered around zero and have variance in the same +Machines or the l1 and l2 regularizers of linear models) may assume that +all features are centered around zero or have variance in the same order. If a feature has a variance that is orders of magnitude larger than others, it might dominate the objective function and make the estimator unable to learn from other features correctly as expected. @@ -56,16 +57,16 @@ dataset:: StandardScaler() >>> scaler.mean_ - array([1. ..., 0. ..., 0.33...]) + array([1., 0., 0.33]) >>> scaler.scale_ - array([0.81..., 0.81..., 1.24...]) + array([0.81, 0.81, 1.24]) >>> X_scaled = scaler.transform(X_train) >>> X_scaled - array([[ 0. ..., -1.22..., 1.33...], - [ 1.22..., 0. ..., -0.26...], - [-1.22..., 1.22..., -1.06...]]) + array([[ 0. , -1.22, 1.33 ], + [ 1.22, 0. , -0.267], + [-1.22, 1.22, -1.06 ]]) .. >>> import numpy as np @@ -117,7 +118,7 @@ or so that the maximum absolute value of each feature is scaled to unit size. This can be achieved using :class:`MinMaxScaler` or :class:`MaxAbsScaler`, respectively. -The motivation to use this scaling include robustness to very small +The motivation to use this scaling includes robustness to very small standard deviations of features and preserving zero entries in sparse data. Here is an example to scale a toy data matrix to the ``[0, 1]`` range:: @@ -146,10 +147,10 @@ It is possible to introspect the scaler attributes to find about the exact nature of the transformation learned on the training data:: >>> min_max_scaler.scale_ - array([0.5 , 0.5 , 0.33...]) + array([0.5 , 0.5 , 0.33]) >>> min_max_scaler.min_ - array([0. , 0.5 , 0.33...]) + array([0. , 0.5 , 0.33]) If :class:`MinMaxScaler` is given an explicit ``feature_range=(min, max)`` the full formula is:: @@ -219,13 +220,13 @@ of the data is likely to not work very well. In these cases, you can use more robust estimates for the center and range of your data. -.. topic:: References: +.. dropdown:: References Further discussion on the importance of centering and scaling data is available on this FAQ: `Should I normalize/standardize/rescale the data? `_ -.. topic:: Scaling vs Whitening +.. dropdown:: Scaling vs Whitening It is sometimes not enough to center and scale the features independently, since a downstream model can further make some assumption @@ -234,16 +235,72 @@ more robust estimates for the center and range of your data. To address this issue you can use :class:`~sklearn.decomposition.PCA` with ``whiten=True`` to further remove the linear correlation across features. + .. _kernel_centering: Centering kernel matrices ------------------------- If you have a kernel matrix of a kernel :math:`K` that computes a dot product -in a feature space defined by function :math:`\phi`, -a :class:`KernelCenterer` can transform the kernel matrix -so that it contains inner products in the feature space -defined by :math:`\phi` followed by removal of the mean in that space. +in a feature space (possibly implicitly) defined by a function +:math:`\phi(\cdot)`, a :class:`KernelCenterer` can transform the kernel matrix +so that it contains inner products in the feature space defined by :math:`\phi` +followed by the removal of the mean in that space. In other words, +:class:`KernelCenterer` computes the centered Gram matrix associated to a +positive semidefinite kernel :math:`K`. + +.. dropdown:: Mathematical formulation + + We can have a look at the mathematical formulation now that we have the + intuition. Let :math:`K` be a kernel matrix of shape `(n_samples, n_samples)` + computed from :math:`X`, a data matrix of shape `(n_samples, n_features)`, + during the `fit` step. :math:`K` is defined by + + .. math:: + K(X, X) = \phi(X) . \phi(X)^{T} + + :math:`\phi(X)` is a function mapping of :math:`X` to a Hilbert space. A + centered kernel :math:`\tilde{K}` is defined as: + + .. math:: + \tilde{K}(X, X) = \tilde{\phi}(X) . \tilde{\phi}(X)^{T} + + where :math:`\tilde{\phi}(X)` results from centering :math:`\phi(X)` in the + Hilbert space. + + Thus, one could compute :math:`\tilde{K}` by mapping :math:`X` using the + function :math:`\phi(\cdot)` and center the data in this new space. However, + kernels are often used because they allow some algebra calculations that + avoid computing explicitly this mapping using :math:`\phi(\cdot)`. Indeed, one + can implicitly center as shown in Appendix B in [Scholkopf1998]_: + + .. math:: + \tilde{K} = K - 1_{\text{n}_{samples}} K - K 1_{\text{n}_{samples}} + 1_{\text{n}_{samples}} K 1_{\text{n}_{samples}} + + :math:`1_{\text{n}_{samples}}` is a matrix of `(n_samples, n_samples)` where + all entries are equal to :math:`\frac{1}{\text{n}_{samples}}`. In the + `transform` step, the kernel becomes :math:`K_{test}(X, Y)` defined as: + + .. math:: + K_{test}(X, Y) = \phi(Y) . \phi(X)^{T} + + :math:`Y` is the test dataset of shape `(n_samples_test, n_features)` and thus + :math:`K_{test}` is of shape `(n_samples_test, n_samples)`. In this case, + centering :math:`K_{test}` is done as: + + .. math:: + \tilde{K}_{test}(X, Y) = K_{test} - 1'_{\text{n}_{samples}} K - K_{test} 1_{\text{n}_{samples}} + 1'_{\text{n}_{samples}} K 1_{\text{n}_{samples}} + + :math:`1'_{\text{n}_{samples}}` is a matrix of shape + `(n_samples_test, n_samples)` where all entries are equal to + :math:`\frac{1}{\text{n}_{samples}}`. + + .. rubric:: References + + .. [Scholkopf1998] B. SchÃļlkopf, A. Smola, and K.R. MÃŧller, + `"Nonlinear component analysis as a kernel eigenvalue problem." + `_ + Neural computation 10.5 (1998): 1299-1319. .. _preprocessing_transformer: @@ -289,21 +346,21 @@ with values between 0 and 1:: array([ 4.3, 5.1, 5.8, 6.5, 7.9]) This feature corresponds to the sepal length in cm. Once the quantile -transformation applied, those landmarks approach closely the percentiles +transformation is applied, those landmarks approach closely the percentiles previously defined:: >>> np.percentile(X_train_trans[:, 0], [0, 25, 50, 75, 100]) ... # doctest: +SKIP - array([ 0.00... , 0.24..., 0.49..., 0.73..., 0.99... ]) + array([ 0.00 , 0.24, 0.49, 0.73, 0.99 ]) -This can be confirmed on a independent testing set with similar remarks:: +This can be confirmed on an independent testing set with similar remarks:: >>> np.percentile(X_test[:, 0], [0, 25, 50, 75, 100]) ... # doctest: +SKIP array([ 4.4 , 5.125, 5.75 , 6.175, 7.3 ]) >>> np.percentile(X_test_trans[:, 0], [0, 25, 50, 75, 100]) ... # doctest: +SKIP - array([ 0.01..., 0.25..., 0.46..., 0.60... , 0.94...]) + array([ 0.01, 0.25, 0.46, 0.60 , 0.94]) Mapping to a Gaussian distribution ---------------------------------- @@ -316,46 +373,46 @@ possible in order to stabilize variance and minimize skewness. :class:`PowerTransformer` currently provides two such power transformations, the Yeo-Johnson transform and the Box-Cox transform. -The Yeo-Johnson transform is given by: - -.. math:: - x_i^{(\lambda)} = - \begin{cases} - [(x_i + 1)^\lambda - 1] / \lambda & \text{if } \lambda \neq 0, x_i \geq 0, \\[8pt] - \ln{(x_i + 1)} & \text{if } \lambda = 0, x_i \geq 0 \\[8pt] - -[(-x_i + 1)^{2 - \lambda} - 1] / (2 - \lambda) & \text{if } \lambda \neq 2, x_i < 0, \\[8pt] - - \ln (- x_i + 1) & \text{if } \lambda = 2, x_i < 0 - \end{cases} - -while the Box-Cox transform is given by: - -.. math:: - x_i^{(\lambda)} = - \begin{cases} - \dfrac{x_i^\lambda - 1}{\lambda} & \text{if } \lambda \neq 0, \\[8pt] - \ln{(x_i)} & \text{if } \lambda = 0, - \end{cases} - - -Box-Cox can only be applied to strictly positive data. In both methods, the -transformation is parameterized by :math:`\lambda`, which is determined through -maximum likelihood estimation. Here is an example of using Box-Cox to map -samples drawn from a lognormal distribution to a normal distribution:: - - >>> pt = preprocessing.PowerTransformer(method='box-cox', standardize=False) - >>> X_lognormal = np.random.RandomState(616).lognormal(size=(3, 3)) - >>> X_lognormal - array([[1.28..., 1.18..., 0.84...], - [0.94..., 1.60..., 0.38...], - [1.35..., 0.21..., 1.09...]]) - >>> pt.fit_transform(X_lognormal) - array([[ 0.49..., 0.17..., -0.15...], - [-0.05..., 0.58..., -0.57...], - [ 0.69..., -0.84..., 0.10...]]) - -While the above example sets the `standardize` option to `False`, -:class:`PowerTransformer` will apply zero-mean, unit-variance normalization -to the transformed output by default. +.. dropdown:: Yeo-Johnson transform + + .. math:: + x_i^{(\lambda)} = + \begin{cases} + [(x_i + 1)^\lambda - 1] / \lambda & \text{if } \lambda \neq 0, x_i \geq 0, \\[8pt] + \ln{(x_i + 1)} & \text{if } \lambda = 0, x_i \geq 0 \\[8pt] + -[(-x_i + 1)^{2 - \lambda} - 1] / (2 - \lambda) & \text{if } \lambda \neq 2, x_i < 0, \\[8pt] + - \ln (- x_i + 1) & \text{if } \lambda = 2, x_i < 0 + \end{cases} + +.. dropdown:: Box-Cox transform + + .. math:: + x_i^{(\lambda)} = + \begin{cases} + \dfrac{x_i^\lambda - 1}{\lambda} & \text{if } \lambda \neq 0, \\[8pt] + \ln{(x_i)} & \text{if } \lambda = 0, + \end{cases} + + Box-Cox can only be applied to strictly positive data. In both methods, the + transformation is parameterized by :math:`\lambda`, which is determined through + maximum likelihood estimation. Here is an example of using Box-Cox to map + samples drawn from a lognormal distribution to a normal distribution:: + + >>> pt = preprocessing.PowerTransformer(method='box-cox', standardize=False) + >>> X_lognormal = np.random.RandomState(616).lognormal(size=(3, 3)) + >>> X_lognormal + array([[1.28, 1.18 , 0.84 ], + [0.94, 1.60 , 0.388], + [1.35, 0.217, 1.09 ]]) + >>> pt.fit_transform(X_lognormal) + array([[ 0.49 , 0.179, -0.156], + [-0.051, 0.589, -0.576], + [ 0.69 , -0.849, 0.101]]) + + While the above example sets the `standardize` option to `False`, + :class:`PowerTransformer` will apply zero-mean, unit-variance normalization + to the transformed output by default. + Below are examples of Box-Cox and Yeo-Johnson applied to various probability distributions. Note that when applied to certain distributions, the power @@ -413,9 +470,9 @@ operation on a single array-like dataset, either using the ``l1``, ``l2``, or >>> X_normalized = preprocessing.normalize(X, norm='l2') >>> X_normalized - array([[ 0.40..., -0.40..., 0.81...], - [ 1. ..., 0. ..., 0. ...], - [ 0. ..., 0.70..., -0.70...]]) + array([[ 0.408, -0.408, 0.812], + [ 1. , 0. , 0. ], + [ 0. , 0.707, -0.707]]) The ``preprocessing`` module further provides a utility class :class:`Normalizer` that implements the same operation using the @@ -433,17 +490,17 @@ This class is hence suitable for use in the early steps of a The normalizer instance can then be used on sample vectors as any transformer:: >>> normalizer.transform(X) - array([[ 0.40..., -0.40..., 0.81...], - [ 1. ..., 0. ..., 0. ...], - [ 0. ..., 0.70..., -0.70...]]) + array([[ 0.408, -0.408, 0.812], + [ 1. , 0. , 0. ], + [ 0. , 0.707, -0.707]]) >>> normalizer.transform([[-1., 1., 0.]]) - array([[-0.70..., 0.70..., 0. ...]]) + array([[-0.707, 0.707, 0.]]) Note: L2 normalization is also known as spatial sign preprocessing. -.. topic:: Sparse input +.. dropdown:: Sparse input :func:`normalize` and :class:`Normalizer` accept **both dense array-like and sparse matrices from scipy.sparse as input**. @@ -457,6 +514,7 @@ Note: L2 normalization is also known as spatial sign preprocessing. Encoding categorical features ============================= + Often features are not given as continuous values but categorical. For example a person could have features ``["male", "female"]``, ``["from Europe", "from US", "from Asia"]``, @@ -482,8 +540,8 @@ scikit-learn estimators, as these expect continuous input, and would interpret the categories as being ordered, which is often not desired (i.e. the set of browsers was ordered arbitrarily). -:class:`OrdinalEncoder` will also passthrough missing values that are -indicated by `np.nan`. +By default, :class:`OrdinalEncoder` will also passthrough missing values that +are indicated by `np.nan`. >>> enc = preprocessing.OrdinalEncoder() >>> X = [['male'], ['female'], [np.nan], ['female']] @@ -493,6 +551,32 @@ indicated by `np.nan`. [nan], [ 0.]]) +:class:`OrdinalEncoder` provides a parameter `encoded_missing_value` to encode +the missing values without the need to create a pipeline and using +:class:`~sklearn.impute.SimpleImputer`. + + >>> enc = preprocessing.OrdinalEncoder(encoded_missing_value=-1) + >>> X = [['male'], ['female'], [np.nan], ['female']] + >>> enc.fit_transform(X) + array([[ 1.], + [ 0.], + [-1.], + [ 0.]]) + +The above processing is equivalent to the following pipeline:: + + >>> from sklearn.pipeline import Pipeline + >>> from sklearn.impute import SimpleImputer + >>> enc = Pipeline(steps=[ + ... ("encoder", preprocessing.OrdinalEncoder()), + ... ("imputer", SimpleImputer(strategy="constant", fill_value=-1)), + ... ]) + >>> enc.fit_transform(X) + array([[ 1.], + [ 0.], + [-1.], + [ 0.]]) + Another possibility to convert categorical features to features that can be used with scikit-learn estimators is to use a one-of-K, also known as one-hot or dummy encoding. @@ -539,17 +623,19 @@ dataset:: array([[1., 0., 0., 1., 0., 0., 1., 0., 0., 0.]]) If there is a possibility that the training data might have missing categorical -features, it can often be better to specify ``handle_unknown='ignore'`` instead -of setting the ``categories`` manually as above. When -``handle_unknown='ignore'`` is specified and unknown categories are encountered -during transform, no error will be raised but the resulting one-hot encoded -columns for this feature will be all zeros -(``handle_unknown='ignore'`` is only supported for one-hot encoding):: - - >>> enc = preprocessing.OneHotEncoder(handle_unknown='ignore') +features, it can often be better to specify +`handle_unknown='infrequent_if_exist'` instead of setting the `categories` +manually as above. When `handle_unknown='infrequent_if_exist'` is specified +and unknown categories are encountered during transform, no error will be +raised but the resulting one-hot encoded columns for this feature will be all +zeros or considered as an infrequent category if enabled. +(`handle_unknown='infrequent_if_exist'` is only supported for one-hot +encoding):: + + >>> enc = preprocessing.OneHotEncoder(handle_unknown='infrequent_if_exist') >>> X = [['male', 'from US', 'uses Safari'], ['female', 'from Europe', 'uses Firefox']] >>> enc.fit(X) - OneHotEncoder(handle_unknown='ignore') + OneHotEncoder(handle_unknown='infrequent_if_exist') >>> enc.transform([['female', 'from Asia', 'uses Chrome']]).toarray() array([[1., 0., 0., 0., 0., 0.]]) @@ -560,15 +646,14 @@ parameter allows the user to specify a category for each feature to be dropped. This is useful to avoid co-linearity in the input matrix in some classifiers. Such functionality is useful, for example, when using non-regularized regression (:class:`LinearRegression `), -since co-linearity would cause the covariance matrix to be non-invertible. -When this parameter is not None, ``handle_unknown`` must be set to -``error``:: +since co-linearity would cause the covariance matrix to be non-invertible:: >>> X = [['male', 'from US', 'uses Safari'], ... ['female', 'from Europe', 'uses Firefox']] >>> drop_enc = preprocessing.OneHotEncoder(drop='first').fit(X) >>> drop_enc.categories_ - [array(['female', 'male'], dtype=object), array(['from Europe', 'from US'], dtype=object), array(['uses Firefox', 'uses Safari'], dtype=object)] + [array(['female', 'male'], dtype=object), array(['from Europe', 'from US'], dtype=object), + array(['uses Firefox', 'uses Safari'], dtype=object)] >>> drop_enc.transform(X).toarray() array([[1., 1., 1.], [0., 0., 0.]]) @@ -581,46 +666,326 @@ categories. In this case, you can set the parameter `drop='if_binary'`. ... ['female', 'Asia', 'Chrome']] >>> drop_enc = preprocessing.OneHotEncoder(drop='if_binary').fit(X) >>> drop_enc.categories_ - [array(['female', 'male'], dtype=object), array(['Asia', 'Europe', 'US'], dtype=object), array(['Chrome', 'Firefox', 'Safari'], dtype=object)] + [array(['female', 'male'], dtype=object), array(['Asia', 'Europe', 'US'], dtype=object), + array(['Chrome', 'Firefox', 'Safari'], dtype=object)] >>> drop_enc.transform(X).toarray() array([[1., 0., 0., 1., 0., 0., 1.], [0., 0., 1., 0., 0., 1., 0.], [0., 1., 0., 0., 1., 0., 0.]]) In the transformed `X`, the first column is the encoding of the feature with -categories "male"/"female", while the remaining 6 columns is the encoding of +categories "male"/"female", while the remaining 6 columns are the encoding of the 2 features with respectively 3 categories each. -:class:`OneHotEncoder` supports categorical features with missing values by -considering the missing values as an additional category:: +When `handle_unknown='ignore'` and `drop` is not None, unknown categories will +be encoded as all zeros:: + + >>> drop_enc = preprocessing.OneHotEncoder(drop='first', + ... handle_unknown='ignore').fit(X) + >>> X_test = [['unknown', 'America', 'IE']] + >>> drop_enc.transform(X_test).toarray() + array([[0., 0., 0., 0., 0.]]) + +All the categories in `X_test` are unknown during transform and will be mapped +to all zeros. This means that unknown categories will have the same mapping as +the dropped category. :meth:`OneHotEncoder.inverse_transform` will map all zeros +to the dropped category if a category is dropped and `None` if a category is +not dropped:: + + >>> drop_enc = preprocessing.OneHotEncoder(drop='if_binary', sparse_output=False, + ... handle_unknown='ignore').fit(X) + >>> X_test = [['unknown', 'America', 'IE']] + >>> X_trans = drop_enc.transform(X_test) + >>> X_trans + array([[0., 0., 0., 0., 0., 0., 0.]]) + >>> drop_enc.inverse_transform(X_trans) + array([['female', None, None]], dtype=object) + +.. dropdown:: Support of categorical features with missing values + + :class:`OneHotEncoder` supports categorical features with missing values by + considering the missing values as an additional category:: + + >>> X = [['male', 'Safari'], + ... ['female', None], + ... [np.nan, 'Firefox']] + >>> enc = preprocessing.OneHotEncoder(handle_unknown='error').fit(X) + >>> enc.categories_ + [array(['female', 'male', nan], dtype=object), + array(['Firefox', 'Safari', None], dtype=object)] + >>> enc.transform(X).toarray() + array([[0., 1., 0., 0., 1., 0.], + [1., 0., 0., 0., 0., 1.], + [0., 0., 1., 1., 0., 0.]]) + + If a feature contains both `np.nan` and `None`, they will be considered + separate categories:: + + >>> X = [['Safari'], [None], [np.nan], ['Firefox']] + >>> enc = preprocessing.OneHotEncoder(handle_unknown='error').fit(X) + >>> enc.categories_ + [array(['Firefox', 'Safari', None, nan], dtype=object)] + >>> enc.transform(X).toarray() + array([[0., 1., 0., 0.], + [0., 0., 1., 0.], + [0., 0., 0., 1.], + [1., 0., 0., 0.]]) + + See :ref:`dict_feature_extraction` for categorical features that are + represented as a dict, not as scalars. + + +.. _encoder_infrequent_categories: + +Infrequent categories +--------------------- - >>> X = [['male', 'Safari'], - ... ['female', None], - ... [np.nan, 'Firefox']] - >>> enc = preprocessing.OneHotEncoder(handle_unknown='error').fit(X) - >>> enc.categories_ - [array(['female', 'male', nan], dtype=object), - array(['Firefox', 'Safari', None], dtype=object)] - >>> enc.transform(X).toarray() - array([[0., 1., 0., 0., 1., 0.], - [1., 0., 0., 0., 0., 1.], - [0., 0., 1., 1., 0., 0.]]) - -If a feature contains both `np.nan` and `None`, they will be considered -separate categories:: - - >>> X = [['Safari'], [None], [np.nan], ['Firefox']] - >>> enc = preprocessing.OneHotEncoder(handle_unknown='error').fit(X) - >>> enc.categories_ - [array(['Firefox', 'Safari', None, nan], dtype=object)] - >>> enc.transform(X).toarray() - array([[0., 1., 0., 0.], - [0., 0., 1., 0.], - [0., 0., 0., 1.], - [1., 0., 0., 0.]]) +:class:`OneHotEncoder` and :class:`OrdinalEncoder` support aggregating +infrequent categories into a single output for each feature. The parameters to +enable the gathering of infrequent categories are `min_frequency` and +`max_categories`. + +1. `min_frequency` is either an integer greater or equal to 1, or a float in + the interval `(0.0, 1.0)`. If `min_frequency` is an integer, categories with + a cardinality smaller than `min_frequency` will be considered infrequent. + If `min_frequency` is a float, categories with a cardinality smaller than + this fraction of the total number of samples will be considered infrequent. + The default value is 1, which means every category is encoded separately. + +2. `max_categories` is either `None` or any integer greater than 1. This + parameter sets an upper limit to the number of output features for each + input feature. `max_categories` includes the feature that combines + infrequent categories. + +In the following example with :class:`OrdinalEncoder`, the categories `'dog'` +and `'snake'` are considered infrequent:: + + >>> X = np.array([['dog'] * 5 + ['cat'] * 20 + ['rabbit'] * 10 + + ... ['snake'] * 3], dtype=object).T + >>> enc = preprocessing.OrdinalEncoder(min_frequency=6).fit(X) + >>> enc.infrequent_categories_ + [array(['dog', 'snake'], dtype=object)] + >>> enc.transform(np.array([['dog'], ['cat'], ['rabbit'], ['snake']])) + array([[2.], + [0.], + [1.], + [2.]]) + +:class:`OrdinalEncoder`'s `max_categories` do **not** take into account missing +or unknown categories. Setting `unknown_value` or `encoded_missing_value` to an +integer will increase the number of unique integer codes by one each. This can +result in up to `max_categories + 2` integer codes. In the following example, +"a" and "d" are considered infrequent and grouped together into a single +category, "b" and "c" are their own categories, unknown values are encoded as 3 +and missing values are encoded as 4. + + >>> X_train = np.array( + ... [["a"] * 5 + ["b"] * 20 + ["c"] * 10 + ["d"] * 3 + [np.nan]], + ... dtype=object).T + >>> enc = preprocessing.OrdinalEncoder( + ... handle_unknown="use_encoded_value", unknown_value=3, + ... max_categories=3, encoded_missing_value=4) + >>> _ = enc.fit(X_train) + >>> X_test = np.array([["a"], ["b"], ["c"], ["d"], ["e"], [np.nan]], dtype=object) + >>> enc.transform(X_test) + array([[2.], + [0.], + [1.], + [2.], + [3.], + [4.]]) + +Similarly, :class:`OneHotEncoder` can be configured to group together infrequent +categories:: + + >>> enc = preprocessing.OneHotEncoder(min_frequency=6, sparse_output=False).fit(X) + >>> enc.infrequent_categories_ + [array(['dog', 'snake'], dtype=object)] + >>> enc.transform(np.array([['dog'], ['cat'], ['rabbit'], ['snake']])) + array([[0., 0., 1.], + [1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]) + +By setting handle_unknown to `'infrequent_if_exist'`, unknown categories will +be considered infrequent:: + + >>> enc = preprocessing.OneHotEncoder( + ... handle_unknown='infrequent_if_exist', sparse_output=False, min_frequency=6) + >>> enc = enc.fit(X) + >>> enc.transform(np.array([['dragon']])) + array([[0., 0., 1.]]) + +:meth:`OneHotEncoder.get_feature_names_out` uses 'infrequent' as the infrequent +feature name:: + + >>> enc.get_feature_names_out() + array(['x0_cat', 'x0_rabbit', 'x0_infrequent_sklearn'], dtype=object) + +When `'handle_unknown'` is set to `'infrequent_if_exist'` and an unknown +category is encountered in transform: + +1. If infrequent category support was not configured or there was no + infrequent category during training, the resulting one-hot encoded columns + for this feature will be all zeros. In the inverse transform, an unknown + category will be denoted as `None`. + +2. If there is an infrequent category during training, the unknown category + will be considered infrequent. In the inverse transform, 'infrequent_sklearn' + will be used to represent the infrequent category. + +Infrequent categories can also be configured using `max_categories`. In the +following example, we set `max_categories=2` to limit the number of features in +the output. This will result in all but the `'cat'` category to be considered +infrequent, leading to two features, one for `'cat'` and one for infrequent +categories - which are all the others:: + + >>> enc = preprocessing.OneHotEncoder(max_categories=2, sparse_output=False) + >>> enc = enc.fit(X) + >>> enc.transform([['dog'], ['cat'], ['rabbit'], ['snake']]) + array([[0., 1.], + [1., 0.], + [0., 1.], + [0., 1.]]) + +If both `max_categories` and `min_frequency` are non-default values, then +categories are selected based on `min_frequency` first and `max_categories` +categories are kept. In the following example, `min_frequency=4` considers +only `snake` to be infrequent, but `max_categories=3`, forces `dog` to also be +infrequent:: + + >>> enc = preprocessing.OneHotEncoder(min_frequency=4, max_categories=3, sparse_output=False) + >>> enc = enc.fit(X) + >>> enc.transform([['dog'], ['cat'], ['rabbit'], ['snake']]) + array([[0., 0., 1.], + [1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]) + +If there are infrequent categories with the same cardinality at the cutoff of +`max_categories`, then the first `max_categories` are taken based on lexicon +ordering. In the following example, "b", "c", and "d", have the same cardinality +and with `max_categories=2`, "b" and "c" are infrequent because they have a higher +lexicon order. + + >>> X = np.asarray([["a"] * 20 + ["b"] * 10 + ["c"] * 10 + ["d"] * 10], dtype=object).T + >>> enc = preprocessing.OneHotEncoder(max_categories=3).fit(X) + >>> enc.infrequent_categories_ + [array(['b', 'c'], dtype=object)] + +.. _target_encoder: + +Target Encoder +-------------- + +.. currentmodule:: sklearn.preprocessing + +The :class:`TargetEncoder` uses the target mean conditioned on the categorical +feature for encoding unordered categories, i.e. nominal categories [PAR]_ +[MIC]_. This encoding scheme is useful with categorical features with high +cardinality, where one-hot encoding would inflate the feature space making it +more expensive for a downstream model to process. A classical example of high +cardinality categories are location based such as zip code or region. + +.. dropdown:: Binary classification targets + + For the binary classification target, the target encoding is given by: + + .. math:: + S_i = \lambda_i\frac{n_{iY}}{n_i} + (1 - \lambda_i)\frac{n_Y}{n} + + where :math:`S_i` is the encoding for category :math:`i`, :math:`n_{iY}` is the + number of observations with :math:`Y=1` and category :math:`i`, :math:`n_i` is + the number of observations with category :math:`i`, :math:`n_Y` is the number of + observations with :math:`Y=1`, :math:`n` is the number of observations, and + :math:`\lambda_i` is a shrinkage factor for category :math:`i`. The shrinkage + factor is given by: + + .. math:: + \lambda_i = \frac{n_i}{m + n_i} + + where :math:`m` is a smoothing factor, which is controlled with the `smooth` + parameter in :class:`TargetEncoder`. Large smoothing factors will put more + weight on the global mean. When `smooth="auto"`, the smoothing factor is + computed as an empirical Bayes estimate: :math:`m=\sigma_i^2/\tau^2`, where + :math:`\sigma_i^2` is the variance of `y` with category :math:`i` and + :math:`\tau^2` is the global variance of `y`. + +.. dropdown:: Multiclass classification targets + + For multiclass classification targets, the formulation is similar to binary + classification: + + .. math:: + S_{ij} = \lambda_i\frac{n_{iY_j}}{n_i} + (1 - \lambda_i)\frac{n_{Y_j}}{n} + + where :math:`S_{ij}` is the encoding for category :math:`i` and class :math:`j`, + :math:`n_{iY_j}` is the number of observations with :math:`Y=j` and category + :math:`i`, :math:`n_i` is the number of observations with category :math:`i`, + :math:`n_{Y_j}` is the number of observations with :math:`Y=j`, :math:`n` is the + number of observations, and :math:`\lambda_i` is a shrinkage factor for category + :math:`i`. + +.. dropdown:: Continuous targets + + For continuous targets, the formulation is similar to binary classification: -See :ref:`dict_feature_extraction` for categorical features that are -represented as a dict, not as scalars. + .. math:: + S_i = \lambda_i\frac{\sum_{k\in L_i}Y_k}{n_i} + (1 - \lambda_i)\frac{\sum_{k=1}^{n}Y_k}{n} + + where :math:`L_i` is the set of observations with category :math:`i` and + :math:`n_i` is the number of observations with category :math:`i`. + + +:meth:`~TargetEncoder.fit_transform` internally relies on a :term:`cross fitting` +scheme to prevent target information from leaking into the train-time +representation, especially for non-informative high-cardinality categorical +variables, and help prevent the downstream model from overfitting spurious +correlations. Note that as a result, `fit(X, y).transform(X)` does not equal +`fit_transform(X, y)`. In :meth:`~TargetEncoder.fit_transform`, the training +data is split into *k* folds (determined by the `cv` parameter) and each fold is +encoded using the encodings learnt using the other *k-1* folds. The following +diagram shows the :term:`cross fitting` scheme in +:meth:`~TargetEncoder.fit_transform` with the default `cv=5`: + +.. image:: ../images/target_encoder_cross_validation.svg + :width: 600 + :align: center + +:meth:`~TargetEncoder.fit_transform` also learns a 'full data' encoding using +the whole training set. This is never used in +:meth:`~TargetEncoder.fit_transform` but is saved to the attribute `encodings_`, +for use when :meth:`~TargetEncoder.transform` is called. Note that the encodings +learned for each fold during the :term:`cross fitting` scheme are not saved to +an attribute. + +The :meth:`~TargetEncoder.fit` method does **not** use any :term:`cross fitting` +schemes and learns one encoding on the entire training set, which is used to +encode categories in :meth:`~TargetEncoder.transform`. +This encoding is the same as the 'full data' +encoding learned in :meth:`~TargetEncoder.fit_transform`. + +.. note:: + :class:`TargetEncoder` considers missing values, such as `np.nan` or `None`, + as another category and encodes them like any other category. Categories + that are not seen during `fit` are encoded with the target mean, i.e. + `target_mean_`. + +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_preprocessing_plot_target_encoder.py` +* :ref:`sphx_glr_auto_examples_preprocessing_plot_target_encoder_cross_val.py` + +.. rubric:: References + +.. [MIC] :doi:`Micci-Barreca, Daniele. "A preprocessing scheme for high-cardinality + categorical attributes in classification and prediction problems" + SIGKDD Explor. Newsl. 3, 1 (July 2001), 27-32. <10.1145/507533.507538>` + +.. [PAR] :doi:`Pargent, F., Pfisterer, F., Thomas, J. et al. "Regularized target + encoding outperforms traditional methods in supervised machine learning with + high cardinality features" Comput Stat 37, 2671-2692 (2022) + <10.1007/s00180-022-01207-6>` .. _preprocessing_discretization: @@ -656,9 +1021,9 @@ For each feature, the bin edges are computed during ``fit`` and together with the number of bins, they will define the intervals. Therefore, for the current example, these intervals are defined as: - - feature 1: :math:`{[-\infty, -1), [-1, 2), [2, \infty)}` - - feature 2: :math:`{[-\infty, 5), [5, \infty)}` - - feature 3: :math:`{[-\infty, 14), [14, \infty)}` +- feature 1: :math:`{[-\infty, -1), [-1, 2), [2, \infty)}` +- feature 2: :math:`{[-\infty, 5), [5, \infty)}` +- feature 3: :math:`{[-\infty, 14), [14, \infty)}` Based on these bin intervals, ``X`` is transformed as follows:: @@ -686,6 +1051,8 @@ For instance, we can use the Pandas function :func:`pandas.cut`:: >>> import pandas as pd >>> import numpy as np + >>> from sklearn import preprocessing + >>> >>> bins = [0, 1, 13, 20, 60, np.inf] >>> labels = ['infant', 'kid', 'teen', 'adult', 'senior citizen'] >>> transformer = preprocessing.FunctionTransformer( @@ -696,11 +1063,11 @@ For instance, we can use the Pandas function :func:`pandas.cut`:: ['infant', 'kid', 'teen', 'adult', 'senior citizen'] Categories (5, object): ['infant' < 'kid' < 'teen' < 'adult' < 'senior citizen'] -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_preprocessing_plot_discretization.py` - * :ref:`sphx_glr_auto_examples_preprocessing_plot_discretization_classification.py` - * :ref:`sphx_glr_auto_examples_preprocessing_plot_discretization_strategies.py` +* :ref:`sphx_glr_auto_examples_preprocessing_plot_discretization.py` +* :ref:`sphx_glr_auto_examples_preprocessing_plot_discretization_classification.py` +* :ref:`sphx_glr_auto_examples_preprocessing_plot_discretization_strategies.py` .. _preprocessing_binarization: @@ -847,23 +1214,23 @@ below. Some of the advantages of splines over polynomials are: - - B-splines are very flexible and robust if you keep a fixed low degree, - usually 3, and parsimoniously adapt the number of knots. Polynomials - would need a higher degree, which leads to the next point. - - B-splines do not have oscillatory behaviour at the boundaries as have - polynomials (the higher the degree, the worse). This is known as `Runge's - phenomenon `_. - - B-splines provide good options for extrapolation beyond the boundaries, - i.e. beyond the range of fitted values. Have a look at the option - ``extrapolation``. - - B-splines generate a feature matrix with a banded structure. For a single - feature, every row contains only ``degree + 1`` non-zero elements, which - occur consecutively and are even positive. This results in a matrix with - good numerical properties, e.g. a low condition number, in sharp contrast - to a matrix of polynomials, which goes under the name - `Vandermonde matrix `_. - A low condition number is important for stable algorithms of linear - models. +- B-splines are very flexible and robust if you keep a fixed low degree, + usually 3, and parsimoniously adapt the number of knots. Polynomials + would need a higher degree, which leads to the next point. +- B-splines do not have oscillatory behaviour at the boundaries as have + polynomials (the higher the degree, the worse). This is known as `Runge's + phenomenon `_. +- B-splines provide good options for extrapolation beyond the boundaries, + i.e. beyond the range of fitted values. Have a look at the option + ``extrapolation``. +- B-splines generate a feature matrix with a banded structure. For a single + feature, every row contains only ``degree + 1`` non-zero elements, which + occur consecutively and are even positive. This results in a matrix with + good numerical properties, e.g. a low condition number, in sharp contrast + to a matrix of polynomials, which goes under the name + `Vandermonde matrix `_. + A low condition number is important for stable algorithms of linear + models. The following code snippet shows splines in action:: @@ -893,20 +1260,20 @@ Interestingly, a :class:`SplineTransformer` of ``degree=0`` is the same as ``encode='onehot-dense'`` and ``n_bins = n_knots - 1`` if ``knots = strategy``. -.. topic:: Examples: +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_linear_model_plot_polynomial_interpolation.py` +* :ref:`sphx_glr_auto_examples_applications_plot_cyclical_feature_engineering.py` - * :ref:`sphx_glr_auto_examples_linear_model_plot_polynomial_interpolation.py` +.. dropdown:: References -.. topic:: References: + * Eilers, P., & Marx, B. (1996). :doi:`Flexible Smoothing with B-splines and + Penalties <10.1214/ss/1038425655>`. Statist. Sci. 11 (1996), no. 2, 89--121. - * Eilers, P., & Marx, B. (1996). Flexible Smoothing with B-splines and - Penalties. Statist. Sci. 11 (1996), no. 2, 89--121. - `doi:10.1214/ss/1038425655 `_ + * Perperoglou, A., Sauerbrei, W., Abrahamowicz, M. et al. :doi:`A review of + spline function procedures in R <10.1186/s12874-019-0666-3>`. + BMC Med Res Methodol 19, 46 (2019). - * Perperoglou, A., Sauerbrei, W., Abrahamowicz, M. et al. A review of - spline function procedures in R. BMC Med Res Methodol 19, 46 (2019). - `doi:10.1186/s12874-019-0666-3 - `_ .. _function_transformer: @@ -922,6 +1289,7 @@ a transformer that applies a log transformation in a pipeline, do:: >>> from sklearn.preprocessing import FunctionTransformer >>> transformer = FunctionTransformer(np.log1p, validate=True) >>> X = np.array([[0, 1], [2, 3]]) + >>> # Since FunctionTransformer is no-op during fit, we can call transform directly >>> transformer.transform(X) array([[0. , 0.69314718], [1.09861229, 1.38629436]]) @@ -937,4 +1305,5 @@ error with a ``filterwarnings``:: For a full code example that demonstrates using a :class:`FunctionTransformer` to extract features from text data see -:ref:`sphx_glr_auto_examples_compose_plot_column_transformer.py` +:ref:`sphx_glr_auto_examples_compose_plot_column_transformer.py` and +:ref:`sphx_glr_auto_examples_applications_plot_cyclical_feature_engineering.py`. diff --git a/doc/modules/preprocessing_targets.rst b/doc/modules/preprocessing_targets.rst index b7e8802785257..f8035bc059af4 100644 --- a/doc/modules/preprocessing_targets.rst +++ b/doc/modules/preprocessing_targets.rst @@ -95,8 +95,8 @@ hashable and comparable) to numerical labels:: >>> le.fit(["paris", "paris", "tokyo", "amsterdam"]) LabelEncoder() >>> list(le.classes_) - ['amsterdam', 'paris', 'tokyo'] + [np.str_('amsterdam'), np.str_('paris'), np.str_('tokyo')] >>> le.transform(["tokyo", "tokyo", "paris"]) array([2, 2, 1]) >>> list(le.inverse_transform([2, 2, 1])) - ['tokyo', 'tokyo', 'paris'] + [np.str_('tokyo'), np.str_('tokyo'), np.str_('paris')] diff --git a/doc/modules/random_projection.rst b/doc/modules/random_projection.rst index b8a87fe602c9a..ec437c60c7d4c 100644 --- a/doc/modules/random_projection.rst +++ b/doc/modules/random_projection.rst @@ -19,19 +19,19 @@ samples of the dataset. Thus random projection is a suitable approximation technique for distance based method. -.. topic:: References: +.. rubric:: References - * Sanjoy Dasgupta. 2000. - `Experiments with random projection. `_ - In Proceedings of the Sixteenth conference on Uncertainty in artificial - intelligence (UAI'00), Craig Boutilier and MoisÊs Goldszmidt (Eds.). Morgan - Kaufmann Publishers Inc., San Francisco, CA, USA, 143-151. +* Sanjoy Dasgupta. 2000. + `Experiments with random projection. `_ + In Proceedings of the Sixteenth conference on Uncertainty in artificial + intelligence (UAI'00), Craig Boutilier and MoisÊs Goldszmidt (Eds.). Morgan + Kaufmann Publishers Inc., San Francisco, CA, USA, 143-151. - * Ella Bingham and Heikki Mannila. 2001. - `Random projection in dimensionality reduction: applications to image and text data. `_ - In Proceedings of the seventh ACM SIGKDD international conference on - Knowledge discovery and data mining (KDD '01). ACM, New York, NY, USA, - 245-250. +* Ella Bingham and Heikki Mannila. 2001. + `Random projection in dimensionality reduction: applications to image and text data. `_ + In Proceedings of the seventh ACM SIGKDD international conference on + Knowledge discovery and data mining (KDD '01). ACM, New York, NY, USA, + 245-250. .. _johnson_lindenstrauss: @@ -58,7 +58,7 @@ bounded distortion introduced by the random projection:: >>> from sklearn.random_projection import johnson_lindenstrauss_min_dim >>> johnson_lindenstrauss_min_dim(n_samples=1e6, eps=0.5) - 663 + np.int64(663) >>> johnson_lindenstrauss_min_dim(n_samples=1e6, eps=[0.5, 0.1, 0.01]) array([ 663, 11841, 1112658]) >>> johnson_lindenstrauss_min_dim(n_samples=[1e4, 1e5, 1e6], eps=0.1) @@ -74,17 +74,17 @@ bounded distortion introduced by the random projection:: :scale: 75 :align: center -.. topic:: Example: +.. rubric:: Examples - * See :ref:`sphx_glr_auto_examples_miscellaneous_plot_johnson_lindenstrauss_bound.py` - for a theoretical explication on the Johnson-Lindenstrauss lemma and an - empirical validation using sparse random matrices. +* See :ref:`sphx_glr_auto_examples_miscellaneous_plot_johnson_lindenstrauss_bound.py` + for a theoretical explication on the Johnson-Lindenstrauss lemma and an + empirical validation using sparse random matrices. -.. topic:: References: +.. rubric:: References - * Sanjoy Dasgupta and Anupam Gupta, 1999. - `An elementary proof of the Johnson-Lindenstrauss Lemma. - `_ +* Sanjoy Dasgupta and Anupam Gupta, 1999. + `An elementary proof of the Johnson-Lindenstrauss Lemma. + `_ .. _gaussian_random_matrix: @@ -95,7 +95,7 @@ dimensionality by projecting the original input space on a randomly generated matrix where components are drawn from the following distribution :math:`N(0, \frac{1}{n_{components}})`. -Here a small excerpt which illustrates how to use the Gaussian random +Here is a small excerpt which illustrates how to use the Gaussian random projection transformer:: >>> import numpy as np @@ -136,7 +136,7 @@ where :math:`n_{\text{components}}` is the size of the projected subspace. By default the density of non zero elements is set to the minimum density as recommended by Ping Li et al.: :math:`1 / \sqrt{n_{\text{features}}}`. -Here a small excerpt which illustrates how to use the sparse random +Here is a small excerpt which illustrates how to use the sparse random projection transformer:: >>> import numpy as np @@ -148,15 +148,53 @@ projection transformer:: (100, 3947) -.. topic:: References: +.. rubric:: References - * D. Achlioptas. 2003. - `Database-friendly random projections: Johnson-Lindenstrauss with binary - coins `_. - Journal of Computer and System Sciences 66 (2003) 671–687 +* D. Achlioptas. 2003. + `Database-friendly random projections: Johnson-Lindenstrauss with binary + coins `_. + Journal of Computer and System Sciences 66 (2003) 671-687. - * Ping Li, Trevor J. Hastie, and Kenneth W. Church. 2006. - `Very sparse random projections. `_ - In Proceedings of the 12th ACM SIGKDD international conference on - Knowledge discovery and data mining (KDD '06). ACM, New York, NY, USA, - 287-296. +* Ping Li, Trevor J. Hastie, and Kenneth W. Church. 2006. + `Very sparse random projections. `_ + In Proceedings of the 12th ACM SIGKDD international conference on + Knowledge discovery and data mining (KDD '06). ACM, New York, NY, USA, 287-296. + + +.. _random_projection_inverse_transform: + +Inverse Transform +================= +The random projection transformers have ``compute_inverse_components`` parameter. When +set to True, after creating the random ``components_`` matrix during fitting, +the transformer computes the pseudo-inverse of this matrix and stores it as +``inverse_components_``. The ``inverse_components_`` matrix has shape +:math:`n_{features} \times n_{components}`, and it is always a dense matrix, +regardless of whether the components matrix is sparse or dense. So depending on +the number of features and components, it may use a lot of memory. + +When the ``inverse_transform`` method is called, it computes the product of the +input ``X`` and the transpose of the inverse components. If the inverse components have +been computed during fit, they are reused at each call to ``inverse_transform``. +Otherwise they are recomputed each time, which can be costly. The result is always +dense, even if ``X`` is sparse. + +Here is a small code example which illustrates how to use the inverse transform +feature:: + + >>> import numpy as np + >>> from sklearn.random_projection import SparseRandomProjection + >>> X = np.random.rand(100, 10000) + >>> transformer = SparseRandomProjection( + ... compute_inverse_components=True + ... ) + ... + >>> X_new = transformer.fit_transform(X) + >>> X_new.shape + (100, 3947) + >>> X_new_inversed = transformer.inverse_transform(X_new) + >>> X_new_inversed.shape + (100, 10000) + >>> X_new_again = transformer.transform(X_new_inversed) + >>> np.allclose(X_new, X_new_again) + True diff --git a/doc/modules/semi_supervised.rst b/doc/modules/semi_supervised.rst index 7c1ea8f296a49..6c050b698f42c 100644 --- a/doc/modules/semi_supervised.rst +++ b/doc/modules/semi_supervised.rst @@ -40,8 +40,8 @@ this algorithm, a given supervised classifier can function as a semi-supervised classifier, allowing it to learn from unlabeled data. :class:`SelfTrainingClassifier` can be called with any classifier that -implements `predict_proba`, passed as the parameter `base_classifier`. In -each iteration, the `base_classifier` predicts labels for the unlabeled +implements `predict_proba`, passed as the parameter `estimator`. In +each iteration, the `estimator` predicts labels for the unlabeled samples and adds a subset of these labels to the labeled dataset. The choice of this subset is determined by the selection criterion. This @@ -60,18 +60,18 @@ until all samples have labels or no new samples are selected in that iteration. When using the self-training classifier, the :ref:`calibration ` of the classifier is important. -.. topic:: Examples +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_semi_supervised_plot_self_training_varying_threshold.py` - * :ref:`sphx_glr_auto_examples_semi_supervised_plot_semi_supervised_versus_svm_iris.py` +* :ref:`sphx_glr_auto_examples_semi_supervised_plot_self_training_varying_threshold.py` +* :ref:`sphx_glr_auto_examples_semi_supervised_plot_semi_supervised_versus_svm_iris.py` -.. topic:: References +.. rubric:: References - .. [1] David Yarowsky. 1995. Unsupervised word sense disambiguation rivaling - supervised methods. In Proceedings of the 33rd annual meeting on - Association for Computational Linguistics (ACL '95). Association for - Computational Linguistics, Stroudsburg, PA, USA, 189-196. DOI: - https://doi.org/10.3115/981658.981684 +.. [1] :doi:`"Unsupervised word sense disambiguation rivaling supervised methods" + <10.3115/981658.981684>` + David Yarowsky, Proceedings of the 33rd annual meeting on Association for + Computational Linguistics (ACL '95). Association for Computational Linguistics, + Stroudsburg, PA, USA, 189-196. .. _label_propagation: @@ -79,7 +79,7 @@ Label Propagation ================= Label propagation denotes a few variations of semi-supervised graph -inference algorithms. +inference algorithms. A few features available in this model: * Used for classification tasks @@ -87,7 +87,7 @@ A few features available in this model: `scikit-learn` provides two label propagation models: :class:`LabelPropagation` and :class:`LabelSpreading`. Both work by -constructing a similarity graph over all items in the input dataset. +constructing a similarity graph over all items in the input dataset. .. figure:: ../auto_examples/semi_supervised/images/sphx_glr_plot_label_propagation_structure_001.png :target: ../auto_examples/semi_supervised/plot_label_propagation_structure.html @@ -118,14 +118,14 @@ computing the normalized graph Laplacian matrix. This procedure is also used in :ref:`spectral_clustering`. Label propagation models have two built-in kernel methods. Choice of kernel -effects both scalability and performance of the algorithms. The following are +affects both scalability and performance of the algorithms. The following are available: - * rbf (:math:`\exp(-\gamma |x-y|^2), \gamma > 0`). :math:`\gamma` is - specified by keyword gamma. +* rbf (:math:`\exp(-\gamma |x-y|^2), \gamma > 0`). :math:`\gamma` is + specified by keyword gamma. - * knn (:math:`1[x' \in kNN(x)]`). :math:`k` is specified by keyword - n_neighbors. +* knn (:math:`1[x' \in kNN(x)]`). :math:`k` is specified by keyword + n_neighbors. The RBF kernel will produce a fully connected graph which is represented in memory by a dense matrix. This matrix may be very large and combined with the cost of @@ -134,18 +134,18 @@ algorithm can lead to prohibitively long running times. On the other hand, the KNN kernel will produce a much more memory-friendly sparse matrix which can drastically reduce running times. -.. topic:: Examples +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_semi_supervised_plot_semi_supervised_versus_svm_iris.py` - * :ref:`sphx_glr_auto_examples_semi_supervised_plot_label_propagation_structure.py` - * :ref:`sphx_glr_auto_examples_semi_supervised_plot_label_propagation_digits.py` - * :ref:`sphx_glr_auto_examples_semi_supervised_plot_label_propagation_digits_active_learning.py` +* :ref:`sphx_glr_auto_examples_semi_supervised_plot_semi_supervised_versus_svm_iris.py` +* :ref:`sphx_glr_auto_examples_semi_supervised_plot_label_propagation_structure.py` +* :ref:`sphx_glr_auto_examples_semi_supervised_plot_label_propagation_digits.py` +* :ref:`sphx_glr_auto_examples_semi_supervised_plot_label_propagation_digits_active_learning.py` -.. topic:: References +.. rubric:: References - [2] Yoshua Bengio, Olivier Delalleau, Nicolas Le Roux. In Semi-Supervised - Learning (2006), pp. 193-216 +[2] Yoshua Bengio, Olivier Delalleau, Nicolas Le Roux. In Semi-Supervised +Learning (2006), pp. 193-216 - [3] Olivier Delalleau, Yoshua Bengio, Nicolas Le Roux. Efficient - Non-Parametric Function Induction in Semi-Supervised Learning. AISTAT 2005 - https://research.microsoft.com/en-us/people/nicolasl/efficient_ssl.pdf +[3] Olivier Delalleau, Yoshua Bengio, Nicolas Le Roux. Efficient +Non-Parametric Function Induction in Semi-Supervised Learning. AISTAT 2005 +https://www.gatsby.ucl.ac.uk/aistats/fullpapers/204.pdf diff --git a/doc/modules/sgd.rst b/doc/modules/sgd.rst index 95a5111747509..84812a0cccf12 100644 --- a/doc/modules/sgd.rst +++ b/doc/modules/sgd.rst @@ -18,34 +18,34 @@ recently in the context of large-scale learning. SGD has been successfully applied to large-scale and sparse machine learning problems often encountered in text classification and natural language processing. Given that the data is sparse, the classifiers -in this module easily scale to problems with more than 10^5 training -examples and more than 10^5 features. +in this module easily scale to problems with more than :math:`10^5` training +examples and more than :math:`10^5` features. Strictly speaking, SGD is merely an optimization technique and does not correspond to a specific family of machine learning models. It is only a *way* to train a model. Often, an instance of :class:`SGDClassifier` or :class:`SGDRegressor` will have an equivalent estimator in the scikit-learn API, potentially using a different optimization technique. -For example, using `SGDClassifier(loss='log')` results in logistic regression, +For example, using `SGDClassifier(loss='log_loss')` results in logistic regression, i.e. a model equivalent to :class:`~sklearn.linear_model.LogisticRegression` which is fitted via SGD instead of being fitted by one of the other solvers in :class:`~sklearn.linear_model.LogisticRegression`. Similarly, -`SGDRegressor(loss='squared_loss', penalty='l2')` and +`SGDRegressor(loss='squared_error', penalty='l2')` and :class:`~sklearn.linear_model.Ridge` solve the same optimization problem, via different means. The advantages of Stochastic Gradient Descent are: - + Efficiency. ++ Efficiency. - + Ease of implementation (lots of opportunities for code tuning). ++ Ease of implementation (lots of opportunities for code tuning). The disadvantages of Stochastic Gradient Descent include: - + SGD requires a number of hyperparameters such as the regularization - parameter and the number of iterations. ++ SGD requires a number of hyperparameters such as the regularization + parameter and the number of iterations. - + SGD is sensitive to feature scaling. ++ SGD is sensitive to feature scaling. .. warning:: @@ -71,7 +71,7 @@ penalties for classification. Below is the decision boundary of a As other classifiers, SGD has to be fitted with two arrays: an array `X` of shape (n_samples, n_features) holding the training samples, and an -array y of shape (n_samples,) holding the target values (class labels) +array `y` of shape (n_samples,) holding the target values (class labels) for the training samples:: >>> from sklearn.linear_model import SGDClassifier @@ -91,12 +91,12 @@ SGD fits a linear model to the training data. The ``coef_`` attribute holds the model parameters:: >>> clf.coef_ - array([[9.9..., 9.9...]]) + array([[9.9, 9.9]]) The ``intercept_`` attribute holds the intercept (aka offset or bias):: >>> clf.intercept_ - array([-9.9...]) + array([-9.9]) Whether or not the model should use an intercept, i.e. a biased hyperplane, is controlled by the parameter ``fit_intercept``. @@ -106,46 +106,46 @@ the coefficients and the input sample, plus the intercept) is given by :meth:`SGDClassifier.decision_function`:: >>> clf.decision_function([[2., 2.]]) - array([29.6...]) + array([29.6]) The concrete loss function can be set via the ``loss`` parameter. :class:`SGDClassifier` supports the following loss functions: - * ``loss="hinge"``: (soft-margin) linear Support Vector Machine, - * ``loss="modified_huber"``: smoothed hinge loss, - * ``loss="log"``: logistic regression, - * and all regression losses below. In this case the target is encoded as -1 - or 1, and the problem is treated as a regression problem. The predicted - class then correspond to the sign of the predicted target. +* ``loss="hinge"``: (soft-margin) linear Support Vector Machine, +* ``loss="modified_huber"``: smoothed hinge loss, +* ``loss="log_loss"``: logistic regression, +* and all regression losses below. In this case the target is encoded as :math:`-1` + or :math:`1`, and the problem is treated as a regression problem. The predicted + class then corresponds to the sign of the predicted target. Please refer to the :ref:`mathematical section below ` for formulas. The first two loss functions are lazy, they only update the model parameters if an example violates the margin constraint, which makes training very efficient and may result in sparser models (i.e. with more zero -coefficents), even when L2 penalty is used. +coefficients), even when :math:`L_2` penalty is used. -Using ``loss="log"`` or ``loss="modified_huber"`` enables the +Using ``loss="log_loss"`` or ``loss="modified_huber"`` enables the ``predict_proba`` method, which gives a vector of probability estimates :math:`P(y|x)` per sample :math:`x`:: - >>> clf = SGDClassifier(loss="log", max_iter=5).fit(X, y) - >>> clf.predict_proba([[1., 1.]]) - array([[0.00..., 0.99...]]) + >>> clf = SGDClassifier(loss="log_loss", max_iter=5).fit(X, y) + >>> clf.predict_proba([[1., 1.]]) # doctest: +SKIP + array([[0.00, 0.99]]) The concrete penalty can be set via the ``penalty`` parameter. SGD supports the following penalties: - * ``penalty="l2"``: L2 norm penalty on ``coef_``. - * ``penalty="l1"``: L1 norm penalty on ``coef_``. - * ``penalty="elasticnet"``: Convex combination of L2 and L1; - ``(1 - l1_ratio) * L2 + l1_ratio * L1``. +* ``penalty="l2"``: :math:`L_2` norm penalty on ``coef_``. +* ``penalty="l1"``: :math:`L_1` norm penalty on ``coef_``. +* ``penalty="elasticnet"``: Convex combination of :math:`L_2` and :math:`L_1`; + ``(1 - l1_ratio) * L2 + l1_ratio * L1``. -The default setting is ``penalty="l2"``. The L1 penalty leads to sparse +The default setting is ``penalty="l2"``. The :math:`L_1` penalty leads to sparse solutions, driving most coefficients to zero. The Elastic Net [#5]_ solves -some deficiencies of the L1 penalty in the presence of highly correlated +some deficiencies of the :math:`L_1` penalty in the presence of highly correlated attributes. The parameter ``l1_ratio`` controls the convex combination -of L1 and L2 penalty. +of :math:`L_1` and :math:`L_2` penalty. :class:`SGDClassifier` supports multi-class classification by combining multiple binary classifiers in a "one versus all" (OVA) scheme. For each @@ -164,11 +164,11 @@ the decision surface induced by the three classifiers. In the case of multi-class classification ``coef_`` is a two-dimensional array of shape (n_classes, n_features) and ``intercept_`` is a -one-dimensional array of shape (n_classes,). The i-th row of ``coef_`` holds -the weight vector of the OVA classifier for the i-th class; classes are +one-dimensional array of shape (n_classes,). The :math:`i`-th row of ``coef_`` holds +the weight vector of the OVA classifier for the :math:`i`-th class; classes are indexed in ascending order (see attribute ``classes_``). Note that, in principle, since they allow to create a probability model, -``loss="log"`` and ``loss="modified_huber"`` are more suitable for +``loss="log_loss"`` and ``loss="modified_huber"`` are more suitable for one-vs-all classification. :class:`SGDClassifier` supports both weighted classes and weighted @@ -189,14 +189,13 @@ For classification with a logistic loss, another variant of SGD with an averaging strategy is available with Stochastic Average Gradient (SAG) algorithm, available as a solver in :class:`LogisticRegression`. -.. topic:: Examples: +.. rubric:: Examples - - :ref:`sphx_glr_auto_examples_linear_model_plot_sgd_separating_hyperplane.py`, - - :ref:`sphx_glr_auto_examples_linear_model_plot_sgd_iris.py` - - :ref:`sphx_glr_auto_examples_linear_model_plot_sgd_weighted_samples.py` - - :ref:`sphx_glr_auto_examples_linear_model_plot_sgd_comparison.py` - - :ref:`sphx_glr_auto_examples_svm_plot_separating_hyperplane_unbalanced.py` - (See the Note in the example) +- :ref:`sphx_glr_auto_examples_linear_model_plot_sgd_separating_hyperplane.py` +- :ref:`sphx_glr_auto_examples_linear_model_plot_sgd_iris.py` +- :ref:`sphx_glr_auto_examples_linear_model_plot_sgd_weighted_samples.py` +- :ref:`sphx_glr_auto_examples_svm_plot_separating_hyperplane_unbalanced.py` + (See the Note in the example) Regression ========== @@ -211,9 +210,9 @@ samples (> 10.000), for other problems we recommend :class:`Ridge`, The concrete loss function can be set via the ``loss`` parameter. :class:`SGDRegressor` supports the following loss functions: - * ``loss="squared_loss"``: Ordinary least squares, - * ``loss="huber"``: Huber loss for robust regression, - * ``loss="epsilon_insensitive"``: linear Support Vector Regression. +* ``loss="squared_error"``: Ordinary least squares, +* ``loss="huber"``: Huber loss for robust regression, +* ``loss="epsilon_insensitive"``: linear Support Vector Regression. Please refer to the :ref:`mathematical section below ` for formulas. @@ -228,10 +227,72 @@ description above in the classification section). :class:`SGDRegressor` also supports averaged SGD [#4]_ (here again, see description above in the classification section). -For regression with a squared loss and a l2 penalty, another variant of +For regression with a squared loss and a :math:`L_2` penalty, another variant of SGD with an averaging strategy is available with Stochastic Average Gradient (SAG) algorithm, available as a solver in :class:`Ridge`. +.. rubric:: Examples + +- :ref:`sphx_glr_auto_examples_applications_plot_prediction_latency.py` + +.. _sgd_online_one_class_svm: + +Online One-Class SVM +==================== + +The class :class:`sklearn.linear_model.SGDOneClassSVM` implements an online +linear version of the One-Class SVM using a stochastic gradient descent. +Combined with kernel approximation techniques, +:class:`sklearn.linear_model.SGDOneClassSVM` can be used to approximate the +solution of a kernelized One-Class SVM, implemented in +:class:`sklearn.svm.OneClassSVM`, with a linear complexity in the number of +samples. Note that the complexity of a kernelized One-Class SVM is at best +quadratic in the number of samples. +:class:`sklearn.linear_model.SGDOneClassSVM` is thus well suited for datasets +with a large number of training samples (over 10,000) for which the SGD +variant can be several orders of magnitude faster. + +.. dropdown:: Mathematical details + + Its implementation is based on the implementation of the stochastic + gradient descent. Indeed, the original optimization problem of the One-Class + SVM is given by + + .. math:: + + \begin{aligned} + \min_{w, \rho, \xi} & \quad \frac{1}{2}\Vert w \Vert^2 - \rho + \frac{1}{\nu n} \sum_{i=1}^n \xi_i \\ + \text{s.t.} & \quad \langle w, x_i \rangle \geq \rho - \xi_i \quad 1 \leq i \leq n \\ + & \quad \xi_i \geq 0 \quad 1 \leq i \leq n + \end{aligned} + + where :math:`\nu \in (0, 1]` is the user-specified parameter controlling the + proportion of outliers and the proportion of support vectors. Getting rid of + the slack variables :math:`\xi_i` this problem is equivalent to + + .. math:: + + \min_{w, \rho} \frac{1}{2}\Vert w \Vert^2 - \rho + \frac{1}{\nu n} \sum_{i=1}^n \max(0, \rho - \langle w, x_i \rangle) \, . + + Multiplying by the constant :math:`\nu` and introducing the intercept + :math:`b = 1 - \rho` we obtain the following equivalent optimization problem + + .. math:: + + \min_{w, b} \frac{\nu}{2}\Vert w \Vert^2 + b\nu + \frac{1}{n} \sum_{i=1}^n \max(0, 1 - (\langle w, x_i \rangle + b)) \, . + + This is similar to the optimization problems studied in section + :ref:`sgd_mathematical_formulation` with :math:`y_i = 1, 1 \leq i \leq n` and + :math:`\alpha = \nu/2`, :math:`L` being the hinge loss function and :math:`R` + being the :math:`L_2` norm. We just need to add the term :math:`b\nu` in the + optimization loop. + +As :class:`SGDClassifier` and :class:`SGDRegressor`, :class:`SGDOneClassSVM` +supports averaged SGD. Averaging can be enabled by setting ``average=True``. + +.. rubric:: Examples + +- :ref:`sphx_glr_auto_examples_linear_model_plot_sgdocsvm_vs_ocsvm.py` Stochastic Gradient Descent for sparse data =========================================== @@ -247,16 +308,17 @@ efficiency, however, use the CSR matrix format as defined in `scipy.sparse.csr_matrix `_. -.. topic:: Examples: +.. rubric:: Examples - - :ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py` +- :ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py` Complexity ========== The major advantage of SGD is its efficiency, which is basically -linear in the number of training examples. If X is a matrix of size (n, p) -training has a cost of :math:`O(k n \bar p)`, where k is the number +linear in the number of training examples. If :math:`X` is a matrix of size +:math:`n \times p` (with :math:`n` samples and :math:`p` features), +training has a cost of :math:`O(k n \bar p)`, where :math:`k` is the number of iterations (epochs) and :math:`\bar p` is the average number of non-zero attributes per sample. @@ -269,69 +331,71 @@ Stopping criterion The classes :class:`SGDClassifier` and :class:`SGDRegressor` provide two criteria to stop the algorithm when a given level of convergence is reached: - * With ``early_stopping=True``, the input data is split into a training set - and a validation set. The model is then fitted on the training set, and the - stopping criterion is based on the prediction score (using the `score` - method) computed on the validation set. The size of the validation set - can be changed with the parameter ``validation_fraction``. - * With ``early_stopping=False``, the model is fitted on the entire input data - and the stopping criterion is based on the objective function computed on - the training data. +* With ``early_stopping=True``, the input data is split into a training set + and a validation set. The model is then fitted on the training set, and the + stopping criterion is based on the prediction score (using the `score` + method) computed on the validation set. The size of the validation set + can be changed with the parameter ``validation_fraction``. +* With ``early_stopping=False``, the model is fitted on the entire input data + and the stopping criterion is based on the objective function computed on + the training data. In both cases, the criterion is evaluated once by epoch, and the algorithm stops when the criterion does not improve ``n_iter_no_change`` times in a row. The improvement is evaluated with absolute tolerance ``tol``, and the algorithm -stops in any case after a maximum number of iteration ``max_iter``. +stops in any case after a maximum number of iterations ``max_iter``. +See :ref:`sphx_glr_auto_examples_linear_model_plot_sgd_early_stopping.py` for an +example of the effects of early stopping. Tips on Practical Use ===================== - * Stochastic Gradient Descent is sensitive to feature scaling, so it - is highly recommended to scale your data. For example, scale each - attribute on the input vector X to [0,1] or [-1,+1], or standardize - it to have mean 0 and variance 1. Note that the *same* scaling - must be applied to the test vector to obtain meaningful - results. This can be easily done using :class:`StandardScaler`:: +* Stochastic Gradient Descent is sensitive to feature scaling, so it + is highly recommended to scale your data. For example, scale each + attribute on the input vector :math:`X` to :math:`[0,1]` or :math:`[-1,1]`, or standardize + it to have mean :math:`0` and variance :math:`1`. Note that the *same* scaling must be + applied to the test vector to obtain meaningful results. This can be easily + done using :class:`~sklearn.preprocessing.StandardScaler`:: - from sklearn.preprocessing import StandardScaler - scaler = StandardScaler() - scaler.fit(X_train) # Don't cheat - fit only on training data - X_train = scaler.transform(X_train) - X_test = scaler.transform(X_test) # apply same transformation to test data + from sklearn.preprocessing import StandardScaler + scaler = StandardScaler() + scaler.fit(X_train) # Don't cheat - fit only on training data + X_train = scaler.transform(X_train) + X_test = scaler.transform(X_test) # apply same transformation to test data - # Or better yet: use a pipeline! - from sklearn.pipeline import make_pipeline - est = make_pipeline(StandardScaler(), SGDClassifier()) - est.fit(X_train) - est.predict(X_test) + # Or better yet: use a pipeline! + from sklearn.pipeline import make_pipeline + est = make_pipeline(StandardScaler(), SGDClassifier()) + est.fit(X_train) + est.predict(X_test) - If your attributes have an intrinsic scale (e.g. word frequencies or - indicator features) scaling is not needed. + If your attributes have an intrinsic scale (e.g. word frequencies or + indicator features) scaling is not needed. - * Finding a reasonable regularization term :math:`\alpha` is - best done using automatic hyper-parameter search, e.g. - :class:`~sklearn.model_selection.GridSearchCV` or - :class:`~sklearn.model_selection.RandomizedSearchCV`, usually in the - range ``10.0**-np.arange(1,7)``. +* Finding a reasonable regularization term :math:`\alpha` is + best done using automatic hyper-parameter search, e.g. + :class:`~sklearn.model_selection.GridSearchCV` or + :class:`~sklearn.model_selection.RandomizedSearchCV`, usually in the + range ``10.0**-np.arange(1,7)``. - * Empirically, we found that SGD converges after observing - approximately 10^6 training samples. Thus, a reasonable first guess - for the number of iterations is ``max_iter = np.ceil(10**6 / n)``, - where ``n`` is the size of the training set. +* Empirically, we found that SGD converges after observing + approximately :math:`10^6` training samples. Thus, a reasonable first guess + for the number of iterations is ``max_iter = np.ceil(10**6 / n)``, + where ``n`` is the size of the training set. - * If you apply SGD to features extracted using PCA we found that - it is often wise to scale the feature values by some constant `c` - such that the average L2 norm of the training data equals one. +* If you apply SGD to features extracted using PCA we found that + it is often wise to scale the feature values by some constant `c` + such that the average :math:`L_2` norm of the training data equals one. - * We found that Averaged SGD works best with a larger number of features - and a higher eta0 +* We found that Averaged SGD works best with a larger number of features + and a higher `eta0`. -.. topic:: References: +.. rubric:: References - * `"Efficient BackProp" `_ - Y. LeCun, L. Bottou, G. Orr, K. MÃŧller - In Neural Networks: Tricks - of the Trade 1998. +* `"Efficient BackProp" `_ + Y. LeCun, L. Bottou, G. Orr, K. MÃŧller - In Neural Networks: Tricks + of the Trade 1998. .. _sgd_mathematical_formulation: @@ -342,8 +406,9 @@ We describe here the mathematical details of the SGD procedure. A good overview with convergence rates can be found in [#6]_. Given a set of training examples :math:`(x_1, y_1), \ldots, (x_n, y_n)` where -:math:`x_i \in \mathbf{R}^m` and :math:`y_i \in \mathcal{R}` (:math:`y_i \in -{-1, 1}` for classification), our goal is to learn a linear scoring function +:math:`x_i \in \mathbf{R}^m` and :math:`y_i \in \mathbf{R}` +(:math:`y_i \in \{-1, 1\}` for classification), +our goal is to learn a linear scoring function :math:`f(x) = w^T x + b` with model parameters :math:`w \in \mathbf{R}^m` and intercept :math:`b \in \mathbf{R}`. In order to make predictions for binary classification, we simply look at the sign of :math:`f(x)`. To find the model @@ -356,28 +421,30 @@ parameters, we minimize the regularized training error given by where :math:`L` is a loss function that measures model (mis)fit and :math:`R` is a regularization term (aka penalty) that penalizes model complexity; :math:`\alpha > 0` is a non-negative hyperparameter that controls -the regularization stength. - -Different choices for :math:`L` entail different classifiers or regressors: - -- Hinge (soft-margin): equivalent to Support Vector Classification. - :math:`L(y_i, f(x_i)) = \max(0, 1 - y_i f(x_i))`. -- Perceptron: - :math:`L(y_i, f(x_i)) = \max(0, - y_i f(x_i))`. -- Modified Huber: - :math:`L(y_i, f(x_i)) = \max(0, 1 - y_i f(x_i))^2` if :math:`y_i f(x_i) > - 1`, and :math:`L(y_i, f(x_i)) = -4 y_i f(x_i)` otherwise. -- Log: equivalent to Logistic Regression. - :math:`L(y_i, f(x_i)) = \log(1 + \exp (-y_i f(x_i)))`. -- Least-Squares: Linear regression (Ridge or Lasso depending on - :math:`R`). - :math:`L(y_i, f(x_i)) = \frac{1}{2}(y_i - f(x_i))^2`. -- Huber: less sensitive to outliers than least-squares. It is equivalent to - least squares when :math:`|y_i - f(x_i)| \leq \varepsilon`, and - :math:`L(y_i, f(x_i)) = \varepsilon |y_i - f(x_i)| - \frac{1}{2} - \varepsilon^2` otherwise. -- Epsilon-Insensitive: (soft-margin) equivalent to Support Vector Regression. - :math:`L(y_i, f(x_i)) = \max(0, |y_i - f(x_i)| - \varepsilon)`. +the regularization strength. + +.. dropdown:: Loss functions details + + Different choices for :math:`L` entail different classifiers or regressors: + + - Hinge (soft-margin): equivalent to Support Vector Classification. + :math:`L(y_i, f(x_i)) = \max(0, 1 - y_i f(x_i))`. + - Perceptron: + :math:`L(y_i, f(x_i)) = \max(0, - y_i f(x_i))`. + - Modified Huber: + :math:`L(y_i, f(x_i)) = \max(0, 1 - y_i f(x_i))^2` if :math:`y_i f(x_i) > + -1`, and :math:`L(y_i, f(x_i)) = -4 y_i f(x_i)` otherwise. + - Log Loss: equivalent to Logistic Regression. + :math:`L(y_i, f(x_i)) = \log(1 + \exp (-y_i f(x_i)))`. + - Squared Error: Linear regression (Ridge or Lasso depending on + :math:`R`). + :math:`L(y_i, f(x_i)) = \frac{1}{2}(y_i - f(x_i))^2`. + - Huber: less sensitive to outliers than least-squares. It is equivalent to + least squares when :math:`|y_i - f(x_i)| \leq \varepsilon`, and + :math:`L(y_i, f(x_i)) = \varepsilon |y_i - f(x_i)| - \frac{1}{2} + \varepsilon^2` otherwise. + - Epsilon-Insensitive: (soft-margin) equivalent to Support Vector Regression. + :math:`L(y_i, f(x_i)) = \max(0, |y_i - f(x_i)| - \varepsilon)`. All of the above loss functions can be regarded as an upper bound on the misclassification error (Zero-one loss) as shown in the Figure below. @@ -390,12 +457,12 @@ misclassification error (Zero-one loss) as shown in the Figure below. Popular choices for the regularization term :math:`R` (the `penalty` parameter) include: - - L2 norm: :math:`R(w) := \frac{1}{2} \sum_{j=1}^{m} w_j^2 = ||w||_2^2`, - - L1 norm: :math:`R(w) := \sum_{j=1}^{m} |w_j|`, which leads to sparse - solutions. - - Elastic Net: :math:`R(w) := \frac{\rho}{2} \sum_{j=1}^{n} w_j^2 + - (1-\rho) \sum_{j=1}^{m} |w_j|`, a convex combination of L2 and L1, where - :math:`\rho` is given by ``1 - l1_ratio``. +- :math:`L_2` norm: :math:`R(w) := \frac{1}{2} \sum_{j=1}^{m} w_j^2 = ||w||_2^2`, +- :math:`L_1` norm: :math:`R(w) := \sum_{j=1}^{m} |w_j|`, which leads to sparse + solutions. +- Elastic Net: :math:`R(w) := \frac{\rho}{2} \sum_{j=1}^{n} w_j^2 + + (1-\rho) \sum_{j=1}^{m} |w_j|`, a convex combination of :math:`L_2` and :math:`L_1`, where + :math:`\rho` is given by ``1 - l1_ratio``. The Figure below shows the contours of the different regularization terms in a 2-dimensional parameter space (:math:`m=2`) when :math:`R(w) = 1`. @@ -438,8 +505,8 @@ is given by where :math:`t` is the time step (there are a total of `n_samples * n_iter` time steps), :math:`t_0` is determined based on a heuristic proposed by LÊon Bottou such that the expected initial updates are comparable with the expected -size of the weights (this assuming that the norm of the training samples is -approx. 1). The exact definition can be found in ``_init_t`` in :class:`BaseSGD`. +size of the weights (this assumes that the norm of the training samples is +approximately 1). The exact definition can be found in ``_init_t`` in `BaseSGD`. For regression the default learning rate schedule is inverse scaling @@ -450,7 +517,7 @@ For regression the default learning rate schedule is inverse scaling \eta^{(t)} = \frac{eta_0}{t^{power\_t}} where :math:`eta_0` and :math:`power\_t` are hyperparameters chosen by the -user via ``eta0`` and ``power_t``, resp. +user via ``eta0`` and ``power_t``, respectively. For a constant learning rate use ``learning_rate='constant'`` and use ``eta0`` to specify the learning rate. @@ -458,7 +525,7 @@ to specify the learning rate. For an adaptively decreasing learning rate, use ``learning_rate='adaptive'`` and use ``eta0`` to specify the starting learning rate. When the stopping criterion is reached, the learning rate is divided by 5, and the algorithm -does not stop. The algorithm stops when the learning rate goes below 1e-6. +does not stop. The algorithm stops when the learning rate goes below `1e-6`. The model parameters can be accessed through the ``coef_`` and ``intercept_`` attributes: ``coef_`` holds the weights :math:`w` and @@ -478,7 +545,7 @@ The implementation of SGD is influenced by the `Stochastic Gradient SVM` of [#1]_. Similar to SvmSGD, the weight vector is represented as the product of a scalar and a vector -which allows an efficient weight update in the case of L2 regularization. +which allows an efficient weight update in the case of :math:`L_2` regularization. In the case of sparse input `X`, the intercept is updated with a smaller learning rate (multiplied by 0.01) to account for the fact that it is updated more frequently. Training examples are picked up sequentially @@ -486,35 +553,32 @@ and the learning rate is lowered after each observed example. We adopted the learning rate schedule from [#2]_. For multi-class classification, a "one versus all" approach is used. We use the truncated gradient algorithm proposed in [#3]_ -for L1 regularization (and the Elastic Net). +for :math:`L_1` regularization (and the Elastic Net). The code is written in Cython. -.. topic:: References: +.. rubric:: References - .. [#1] `"Stochastic Gradient Descent" - `_ L. Bottou - Website, 2010. +.. [#1] `"Stochastic Gradient Descent" + `_ L. Bottou - Website, 2010. - .. [#2] `"Pegasos: Primal estimated sub-gradient solver for svm" - `_ - S. Shalev-Shwartz, Y. Singer, N. Srebro - In Proceedings of ICML '07. +.. [#2] :doi:`"Pegasos: Primal estimated sub-gradient solver for svm" + <10.1145/1273496.1273598>` + S. Shalev-Shwartz, Y. Singer, N. Srebro - In Proceedings of ICML '07. - .. [#3] `"Stochastic gradient descent training for l1-regularized - log-linear models with cumulative penalty" - `_ - Y. Tsuruoka, J. Tsujii, S. Ananiadou - In Proceedings of the AFNLP/ACL - '09. +.. [#3] `"Stochastic gradient descent training for l1-regularized + log-linear models with cumulative penalty" + `_ + Y. Tsuruoka, J. Tsujii, S. Ananiadou - In Proceedings of the AFNLP/ACL'09. - .. [#4] `"Towards Optimal One Pass Large Scale Learning with - Averaged Stochastic Gradient Descent" - `_ - Xu, Wei +.. [#4] :arxiv:`"Towards Optimal One Pass Large Scale Learning with + Averaged Stochastic Gradient Descent" + <1107.2490v2>`. Xu, Wei (2011) - .. [#5] `"Regularization and variable selection via the elastic net" - `_ - H. Zou, T. Hastie - Journal of the Royal Statistical Society Series B, - 67 (2), 301-320. +.. [#5] :doi:`"Regularization and variable selection via the elastic net" + <10.1111/j.1467-9868.2005.00503.x>` + H. Zou, T. Hastie - Journal of the Royal Statistical Society Series B, + 67 (2), 301-320. - .. [#6] `"Solving large scale linear prediction problems using stochastic - gradient descent algorithms" - `_ - T. Zhang - In Proceedings of ICML '04. +.. [#6] :doi:`"Solving large scale linear prediction problems using stochastic + gradient descent algorithms" <10.1145/1015330.1015332>` + T. Zhang - In Proceedings of ICML '04. diff --git a/doc/modules/svm.rst b/doc/modules/svm.rst index 57d2cfb3cb7a7..ac9fbdb12e58d 100644 --- a/doc/modules/svm.rst +++ b/doc/modules/svm.rst @@ -16,27 +16,27 @@ methods used for :ref:`classification `, The advantages of support vector machines are: - - Effective in high dimensional spaces. +- Effective in high dimensional spaces. - - Still effective in cases where number of dimensions is greater - than the number of samples. +- Still effective in cases where number of dimensions is greater + than the number of samples. - - Uses a subset of training points in the decision function (called - support vectors), so it is also memory efficient. +- Uses a subset of training points in the decision function (called + support vectors), so it is also memory efficient. - - Versatile: different :ref:`svm_kernels` can be - specified for the decision function. Common kernels are - provided, but it is also possible to specify custom kernels. +- Versatile: different :ref:`svm_kernels` can be + specified for the decision function. Common kernels are + provided, but it is also possible to specify custom kernels. The disadvantages of support vector machines include: - - If the number of features is much greater than the number of - samples, avoid over-fitting in choosing :ref:`svm_kernels` and regularization - term is crucial. +- If the number of features is much greater than the number of + samples, avoid over-fitting in choosing :ref:`svm_kernels` and regularization + term is crucial. - - SVMs do not directly provide probability estimates, these are - calculated using an expensive five-fold cross-validation - (see :ref:`Scores and probabilities `, below). +- SVMs do not directly provide probability estimates, these are + calculated using an expensive five-fold cross-validation + (see :ref:`Scores and probabilities `, below). The support vector machines in scikit-learn support both dense (``numpy.ndarray`` and convertible to that by ``numpy.asarray``) and @@ -60,14 +60,19 @@ capable of performing binary and multi-class classification on a dataset. :align: center -:class:`SVC` and :class:`NuSVC` are similar methods, but accept -slightly different sets of parameters and have different mathematical -formulations (see section :ref:`svm_mathematical_formulation`). On the -other hand, :class:`LinearSVC` is another (faster) implementation of Support -Vector Classification for the case of a linear kernel. Note that -:class:`LinearSVC` does not accept parameter ``kernel``, as this is -assumed to be linear. It also lacks some of the attributes of -:class:`SVC` and :class:`NuSVC`, like ``support_``. +:class:`SVC` and :class:`NuSVC` are similar methods, but accept slightly +different sets of parameters and have different mathematical formulations (see +section :ref:`svm_mathematical_formulation`). On the other hand, +:class:`LinearSVC` is another (faster) implementation of Support Vector +Classification for the case of a linear kernel. It also +lacks some of the attributes of :class:`SVC` and :class:`NuSVC`, like +`support_`. :class:`LinearSVC` uses `squared_hinge` loss and due to its +implementation in `liblinear` it also regularizes the intercept, if considered. +This effect can however be reduced by carefully fine tuning its +`intercept_scaling` parameter, which allows the intercept term to have a +different regularization behavior compared to the other features. The +classification results and score can therefore differ from the other two +classifiers. As other classifiers, :class:`SVC`, :class:`NuSVC` and :class:`LinearSVC` take as input two arrays: an array `X` of shape @@ -103,11 +108,11 @@ properties of these support vectors can be found in attributes >>> clf.n_support_ array([1, 1]...) -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_svm_plot_separating_hyperplane.py`, - * :ref:`sphx_glr_auto_examples_svm_plot_svm_nonlinear.py` - * :ref:`sphx_glr_auto_examples_svm_plot_svm_anova.py`, +* :ref:`sphx_glr_auto_examples_svm_plot_separating_hyperplane.py` +* :ref:`sphx_glr_auto_examples_svm_plot_svm_anova.py` +* :ref:`sphx_glr_auto_examples_classification_plot_classification_probability.py` .. _svm_multi_class: @@ -121,7 +126,8 @@ classifiers are constructed and each one trains data from two classes. To provide a consistent interface with other classifiers, the ``decision_function_shape`` option allows to monotonically transform the results of the "one-versus-one" classifiers to a "one-vs-rest" decision -function of shape ``(n_samples, n_classes)``. +function of shape ``(n_samples, n_classes)``, which is the default setting +of the parameter (default='ovr'). >>> X = [[0], [1], [2], [3]] >>> Y = [0, 1, 2, 3] @@ -129,7 +135,7 @@ function of shape ``(n_samples, n_classes)``. >>> clf.fit(X, Y) SVC(decision_function_shape='ovo') >>> dec = clf.decision_function([[1]]) - >>> dec.shape[1] # 4 classes: 4*3/2 = 6 + >>> dec.shape[1] # 6 classes: 4*3/2 = 6 6 >>> clf.decision_function_shape = "ovr" >>> dec = clf.decision_function([[1]]) @@ -149,64 +155,61 @@ multi-class strategy, thus training `n_classes` models. See :ref:`svm_mathematical_formulation` for a complete description of the decision function. -Note that the :class:`LinearSVC` also implements an alternative multi-class -strategy, the so-called multi-class SVM formulated by Crammer and Singer -[#8]_, by using the option ``multi_class='crammer_singer'``. In practice, -one-vs-rest classification is usually preferred, since the results are mostly -similar, but the runtime is significantly less. - -For "one-vs-rest" :class:`LinearSVC` the attributes ``coef_`` and ``intercept_`` -have the shape ``(n_classes, n_features)`` and ``(n_classes,)`` respectively. -Each row of the coefficients corresponds to one of the ``n_classes`` -"one-vs-rest" classifiers and similar for the intercepts, in the -order of the "one" class. - -In the case of "one-vs-one" :class:`SVC` and :class:`NuSVC`, the layout of -the attributes is a little more involved. In the case of a linear -kernel, the attributes ``coef_`` and ``intercept_`` have the shape -``(n_classes * (n_classes - 1) / 2, n_features)`` and ``(n_classes * -(n_classes - 1) / 2)`` respectively. This is similar to the layout for -:class:`LinearSVC` described above, with each row now corresponding -to a binary classifier. The order for classes -0 to n is "0 vs 1", "0 vs 2" , ... "0 vs n", "1 vs 2", "1 vs 3", "1 vs n", . . -. "n-1 vs n". - -The shape of ``dual_coef_`` is ``(n_classes-1, n_SV)`` with -a somewhat hard to grasp layout. -The columns correspond to the support vectors involved in any -of the ``n_classes * (n_classes - 1) / 2`` "one-vs-one" classifiers. -Each of the support vectors is used in ``n_classes - 1`` classifiers. -The ``n_classes - 1`` entries in each row correspond to the dual coefficients -for these classifiers. - -This might be clearer with an example: consider a three class problem with -class 0 having three support vectors -:math:`v^{0}_0, v^{1}_0, v^{2}_0` and class 1 and 2 having two support vectors -:math:`v^{0}_1, v^{1}_1` and :math:`v^{0}_2, v^{1}_2` respectively. For each -support vector :math:`v^{j}_i`, there are two dual coefficients. Let's call -the coefficient of support vector :math:`v^{j}_i` in the classifier between -classes :math:`i` and :math:`k` :math:`\alpha^{j}_{i,k}`. -Then ``dual_coef_`` looks like this: - -+------------------------+------------------------+------------------+ -|:math:`\alpha^{0}_{0,1}`|:math:`\alpha^{0}_{0,2}`|Coefficients | -+------------------------+------------------------+for SVs of class 0| -|:math:`\alpha^{1}_{0,1}`|:math:`\alpha^{1}_{0,2}`| | -+------------------------+------------------------+ | -|:math:`\alpha^{2}_{0,1}`|:math:`\alpha^{2}_{0,2}`| | -+------------------------+------------------------+------------------+ -|:math:`\alpha^{0}_{1,0}`|:math:`\alpha^{0}_{1,2}`|Coefficients | -+------------------------+------------------------+for SVs of class 1| -|:math:`\alpha^{1}_{1,0}`|:math:`\alpha^{1}_{1,2}`| | -+------------------------+------------------------+------------------+ -|:math:`\alpha^{0}_{2,0}`|:math:`\alpha^{0}_{2,1}`|Coefficients | -+------------------------+------------------------+for SVs of class 2| -|:math:`\alpha^{1}_{2,0}`|:math:`\alpha^{1}_{2,1}`| | -+------------------------+------------------------+------------------+ - -.. topic:: Examples: - - * :ref:`sphx_glr_auto_examples_svm_plot_iris_svc.py`, +.. dropdown:: Details on multi-class strategies + + Note that the :class:`LinearSVC` also implements an alternative multi-class + strategy, the so-called multi-class SVM formulated by Crammer and Singer + [#8]_, by using the option ``multi_class='crammer_singer'``. In practice, + one-vs-rest classification is usually preferred, since the results are mostly + similar, but the runtime is significantly less. + + For "one-vs-rest" :class:`LinearSVC` the attributes ``coef_`` and ``intercept_`` + have the shape ``(n_classes, n_features)`` and ``(n_classes,)`` respectively. + Each row of the coefficients corresponds to one of the ``n_classes`` + "one-vs-rest" classifiers and similar for the intercepts, in the + order of the "one" class. + + In the case of "one-vs-one" :class:`SVC` and :class:`NuSVC`, the layout of + the attributes is a little more involved. In the case of a linear + kernel, the attributes ``coef_`` and ``intercept_`` have the shape + ``(n_classes * (n_classes - 1) / 2, n_features)`` and ``(n_classes * + (n_classes - 1) / 2)`` respectively. This is similar to the layout for + :class:`LinearSVC` described above, with each row now corresponding + to a binary classifier. The order for classes + 0 to n is "0 vs 1", "0 vs 2" , ... "0 vs n", "1 vs 2", "1 vs 3", "1 vs n", . . + . "n-1 vs n". + + The shape of ``dual_coef_`` is ``(n_classes-1, n_SV)`` with + a somewhat hard to grasp layout. + The columns correspond to the support vectors involved in any + of the ``n_classes * (n_classes - 1) / 2`` "one-vs-one" classifiers. + Each support vector ``v`` has a dual coefficient in each of the + ``n_classes - 1`` classifiers comparing the class of ``v`` against another class. + Note that some, but not all, of these dual coefficients, may be zero. + The ``n_classes - 1`` entries in each column are these dual coefficients, + ordered by the opposing class. + + This might be clearer with an example: consider a three class problem with + class 0 having three support vectors + :math:`v^{0}_0, v^{1}_0, v^{2}_0` and class 1 and 2 having two support vectors + :math:`v^{0}_1, v^{1}_1` and :math:`v^{0}_2, v^{1}_2` respectively. For each + support vector :math:`v^{j}_i`, there are two dual coefficients. Let's call + the coefficient of support vector :math:`v^{j}_i` in the classifier between + classes :math:`i` and :math:`k` :math:`\alpha^{j}_{i,k}`. + Then ``dual_coef_`` looks like this: + + +------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+ + |:math:`\alpha^{0}_{0,1}`|:math:`\alpha^{1}_{0,1}`|:math:`\alpha^{2}_{0,1}`|:math:`\alpha^{0}_{1,0}`|:math:`\alpha^{1}_{1,0}`|:math:`\alpha^{0}_{2,0}`|:math:`\alpha^{1}_{2,0}`| + +------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+ + |:math:`\alpha^{0}_{0,2}`|:math:`\alpha^{1}_{0,2}`|:math:`\alpha^{2}_{0,2}`|:math:`\alpha^{0}_{1,2}`|:math:`\alpha^{1}_{1,2}`|:math:`\alpha^{0}_{2,1}`|:math:`\alpha^{1}_{2,1}`| + +------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+ + |Coefficients |Coefficients |Coefficients | + |for SVs of class 0 |for SVs of class 1 |for SVs of class 2 | + +--------------------------------------------------------------------------+-------------------------------------------------+-------------------------------------------------+ + +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_svm_plot_iris_svc.py` .. _scores_probabilities: @@ -227,7 +230,7 @@ In the multiclass case, this is extended as per [#2]_. The same probability calibration procedure is available for all estimators via the :class:`~sklearn.calibration.CalibratedClassifierCV` (see :ref:`calibration`). In the case of :class:`SVC` and :class:`NuSVC`, this - procedure is builtin in `libsvm`_ which is used under the hood, so it does + procedure is builtin to `libsvm`_ which is used under the hood, so it does not rely on scikit-learn's :class:`~sklearn.calibration.CalibratedClassifierCV`. @@ -289,10 +292,10 @@ to the sample weights: :align: center :scale: 75 -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_svm_plot_separating_hyperplane_unbalanced.py` - * :ref:`sphx_glr_auto_examples_svm_plot_weighted_samples.py`, +* :ref:`sphx_glr_auto_examples_svm_plot_separating_hyperplane_unbalanced.py` +* :ref:`sphx_glr_auto_examples_svm_plot_weighted_samples.py` .. _svm_regression: @@ -313,10 +316,15 @@ target. There are three different implementations of Support Vector Regression: :class:`SVR`, :class:`NuSVR` and :class:`LinearSVR`. :class:`LinearSVR` -provides a faster implementation than :class:`SVR` but only considers -the linear kernel, while :class:`NuSVR` implements a slightly different -formulation than :class:`SVR` and :class:`LinearSVR`. See -:ref:`svm_implementation_details` for further details. +provides a faster implementation than :class:`SVR` but only considers the +linear kernel, while :class:`NuSVR` implements a slightly different formulation +than :class:`SVR` and :class:`LinearSVR`. Due to its implementation in +`liblinear` :class:`LinearSVR` also regularizes the intercept, if considered. +This effect can however be reduced by carefully fine tuning its +`intercept_scaling` parameter, which allows the intercept term to have a +different regularization behavior compared to the other features. The +classification results and score can therefore differ from the other two +classifiers. See :ref:`svm_implementation_details` for further details. As with classification classes, the fit method will take as argument vectors X, y, only that in this case y is expected to have @@ -332,9 +340,9 @@ floating point values instead of integer values:: array([1.5]) -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_svm_plot_svm_regression.py` +* :ref:`sphx_glr_auto_examples_svm_plot_svm_regression.py` .. _svm_outlier_detection: @@ -370,95 +378,95 @@ Tips on Practical Use ===================== - * **Avoiding data copy**: For :class:`SVC`, :class:`SVR`, :class:`NuSVC` and - :class:`NuSVR`, if the data passed to certain methods is not C-ordered - contiguous and double precision, it will be copied before calling the - underlying C implementation. You can check whether a given numpy array is - C-contiguous by inspecting its ``flags`` attribute. - - For :class:`LinearSVC` (and :class:`LogisticRegression - `) any input passed as a numpy - array will be copied and converted to the `liblinear`_ internal sparse data - representation (double precision floats and int32 indices of non-zero - components). If you want to fit a large-scale linear classifier without - copying a dense numpy C-contiguous double precision array as input, we - suggest to use the :class:`SGDClassifier - ` class instead. The objective - function can be configured to be almost the same as the :class:`LinearSVC` - model. - - * **Kernel cache size**: For :class:`SVC`, :class:`SVR`, :class:`NuSVC` and - :class:`NuSVR`, the size of the kernel cache has a strong impact on run - times for larger problems. If you have enough RAM available, it is - recommended to set ``cache_size`` to a higher value than the default of - 200(MB), such as 500(MB) or 1000(MB). - - - * **Setting C**: ``C`` is ``1`` by default and it's a reasonable default - choice. If you have a lot of noisy observations you should decrease it: - decreasing C corresponds to more regularization. - - :class:`LinearSVC` and :class:`LinearSVR` are less sensitive to ``C`` when - it becomes large, and prediction results stop improving after a certain - threshold. Meanwhile, larger ``C`` values will take more time to train, - sometimes up to 10 times longer, as shown in [#3]_. - - * Support Vector Machine algorithms are not scale invariant, so **it - is highly recommended to scale your data**. For example, scale each - attribute on the input vector X to [0,1] or [-1,+1], or standardize it - to have mean 0 and variance 1. Note that the *same* scaling must be - applied to the test vector to obtain meaningful results. This can be done - easily by using a :class:`~sklearn.pipeline.Pipeline`:: - - >>> from sklearn.pipeline import make_pipeline - >>> from sklearn.preprocessing import StandardScaler - >>> from sklearn.svm import SVC - - >>> clf = make_pipeline(StandardScaler(), SVC()) - - See section :ref:`preprocessing` for more details on scaling and - normalization. - - .. _shrinking_svm: - - * Regarding the `shrinking` parameter, quoting [#4]_: *We found that if the - number of iterations is large, then shrinking can shorten the training - time. However, if we loosely solve the optimization problem (e.g., by - using a large stopping tolerance), the code without using shrinking may - be much faster* - - * Parameter ``nu`` in :class:`NuSVC`/:class:`OneClassSVM`/:class:`NuSVR` - approximates the fraction of training errors and support vectors. - - * In :class:`SVC`, if the data is unbalanced (e.g. many - positive and few negative), set ``class_weight='balanced'`` and/or try - different penalty parameters ``C``. - - * **Randomness of the underlying implementations**: The underlying - implementations of :class:`SVC` and :class:`NuSVC` use a random number - generator only to shuffle the data for probability estimation (when - ``probability`` is set to ``True``). This randomness can be controlled - with the ``random_state`` parameter. If ``probability`` is set to ``False`` - these estimators are not random and ``random_state`` has no effect on the - results. The underlying :class:`OneClassSVM` implementation is similar to - the ones of :class:`SVC` and :class:`NuSVC`. As no probability estimation - is provided for :class:`OneClassSVM`, it is not random. - - The underlying :class:`LinearSVC` implementation uses a random number - generator to select features when fitting the model with a dual coordinate - descent (i.e when ``dual`` is set to ``True``). It is thus not uncommon - to have slightly different results for the same input data. If that - happens, try with a smaller `tol` parameter. This randomness can also be - controlled with the ``random_state`` parameter. When ``dual`` is - set to ``False`` the underlying implementation of :class:`LinearSVC` is - not random and ``random_state`` has no effect on the results. - - * Using L1 penalization as provided by ``LinearSVC(penalty='l1', - dual=False)`` yields a sparse solution, i.e. only a subset of feature - weights is different from zero and contribute to the decision function. - Increasing ``C`` yields a more complex model (more features are selected). - The ``C`` value that yields a "null" model (all weights equal to zero) can - be calculated using :func:`l1_min_c`. +* **Avoiding data copy**: For :class:`SVC`, :class:`SVR`, :class:`NuSVC` and + :class:`NuSVR`, if the data passed to certain methods is not C-ordered + contiguous and double precision, it will be copied before calling the + underlying C implementation. You can check whether a given numpy array is + C-contiguous by inspecting its ``flags`` attribute. + + For :class:`LinearSVC` (and :class:`LogisticRegression + `) any input passed as a numpy + array will be copied and converted to the `liblinear`_ internal sparse data + representation (double precision floats and int32 indices of non-zero + components). If you want to fit a large-scale linear classifier without + copying a dense numpy C-contiguous double precision array as input, we + suggest to use the :class:`SGDClassifier + ` class instead. The objective + function can be configured to be almost the same as the :class:`LinearSVC` + model. + +* **Kernel cache size**: For :class:`SVC`, :class:`SVR`, :class:`NuSVC` and + :class:`NuSVR`, the size of the kernel cache has a strong impact on run + times for larger problems. If you have enough RAM available, it is + recommended to set ``cache_size`` to a higher value than the default of + 200(MB), such as 500(MB) or 1000(MB). + + +* **Setting C**: ``C`` is ``1`` by default and it's a reasonable default + choice. If you have a lot of noisy observations you should decrease it: + decreasing C corresponds to more regularization. + + :class:`LinearSVC` and :class:`LinearSVR` are less sensitive to ``C`` when + it becomes large, and prediction results stop improving after a certain + threshold. Meanwhile, larger ``C`` values will take more time to train, + sometimes up to 10 times longer, as shown in [#3]_. + +* Support Vector Machine algorithms are not scale invariant, so **it + is highly recommended to scale your data**. For example, scale each + attribute on the input vector X to [0,1] or [-1,+1], or standardize it + to have mean 0 and variance 1. Note that the *same* scaling must be + applied to the test vector to obtain meaningful results. This can be done + easily by using a :class:`~sklearn.pipeline.Pipeline`:: + + >>> from sklearn.pipeline import make_pipeline + >>> from sklearn.preprocessing import StandardScaler + >>> from sklearn.svm import SVC + + >>> clf = make_pipeline(StandardScaler(), SVC()) + + See section :ref:`preprocessing` for more details on scaling and + normalization. + +.. _shrinking_svm: + +* Regarding the `shrinking` parameter, quoting [#4]_: *We found that if the + number of iterations is large, then shrinking can shorten the training + time. However, if we loosely solve the optimization problem (e.g., by + using a large stopping tolerance), the code without using shrinking may + be much faster* + +* Parameter ``nu`` in :class:`NuSVC`/:class:`OneClassSVM`/:class:`NuSVR` + approximates the fraction of training errors and support vectors. + +* In :class:`SVC`, if the data is unbalanced (e.g. many + positive and few negative), set ``class_weight='balanced'`` and/or try + different penalty parameters ``C``. + +* **Randomness of the underlying implementations**: The underlying + implementations of :class:`SVC` and :class:`NuSVC` use a random number + generator only to shuffle the data for probability estimation (when + ``probability`` is set to ``True``). This randomness can be controlled + with the ``random_state`` parameter. If ``probability`` is set to ``False`` + these estimators are not random and ``random_state`` has no effect on the + results. The underlying :class:`OneClassSVM` implementation is similar to + the ones of :class:`SVC` and :class:`NuSVC`. As no probability estimation + is provided for :class:`OneClassSVM`, it is not random. + + The underlying :class:`LinearSVC` implementation uses a random number + generator to select features when fitting the model with a dual coordinate + descent (i.e. when ``dual`` is set to ``True``). It is thus not uncommon + to have slightly different results for the same input data. If that + happens, try with a smaller `tol` parameter. This randomness can also be + controlled with the ``random_state`` parameter. When ``dual`` is + set to ``False`` the underlying implementation of :class:`LinearSVC` is + not random and ``random_state`` has no effect on the results. + +* Using L1 penalization as provided by ``LinearSVC(penalty='l1', + dual=False)`` yields a sparse solution, i.e. only a subset of feature + weights is different from zero and contribute to the decision function. + Increasing ``C`` yields a more complex model (more features are selected). + The ``C`` value that yields a "null" model (all weights equal to zero) can + be calculated using :func:`l1_min_c`. .. _svm_kernels: @@ -468,16 +476,16 @@ Kernel functions The *kernel function* can be any of the following: - * linear: :math:`\langle x, x'\rangle`. +* linear: :math:`\langle x, x'\rangle`. - * polynomial: :math:`(\gamma \langle x, x'\rangle + r)^d`, where - :math:`d` is specified by parameter ``degree``, :math:`r` by ``coef0``. +* polynomial: :math:`(\gamma \langle x, x'\rangle + r)^d`, where + :math:`d` is specified by parameter ``degree``, :math:`r` by ``coef0``. - * rbf: :math:`\exp(-\gamma \|x-x'\|^2)`, where :math:`\gamma` is - specified by parameter ``gamma``, must be greater than 0. +* rbf: :math:`\exp(-\gamma \|x-x'\|^2)`, where :math:`\gamma` is + specified by parameter ``gamma``, must be greater than 0. - * sigmoid :math:`\tanh(\gamma \langle x,x'\rangle + r)`, - where :math:`r` is specified by ``coef0``. +* sigmoid :math:`\tanh(\gamma \langle x,x'\rangle + r)`, + where :math:`r` is specified by ``coef0``. Different kernels are specified by the `kernel` parameter:: @@ -488,6 +496,8 @@ Different kernels are specified by the `kernel` parameter:: >>> rbf_svc.kernel 'rbf' +See also :ref:`kernel_approximation` for a solution to use RBF kernels that is much faster and more scalable. + Parameters of the RBF Kernel ---------------------------- @@ -503,11 +513,10 @@ Proper choice of ``C`` and ``gamma`` is critical to the SVM's performance. One is advised to use :class:`~sklearn.model_selection.GridSearchCV` with ``C`` and ``gamma`` spaced exponentially far apart to choose good values. -.. topic:: Examples: - - * :ref:`sphx_glr_auto_examples_svm_plot_rbf_parameters.py` - * :ref:`sphx_glr_auto_examples_svm_plot_svm_nonlinear.py` +.. rubric:: Examples +* :ref:`sphx_glr_auto_examples_svm_plot_rbf_parameters.py` +* :ref:`sphx_glr_auto_examples_svm_plot_svm_scale_c.py` Custom Kernels -------------- @@ -518,62 +527,60 @@ python function or by precomputing the Gram matrix. Classifiers with custom kernels behave the same way as any other classifiers, except that: - * Field ``support_vectors_`` is now empty, only indices of support - vectors are stored in ``support_`` +* Field ``support_vectors_`` is now empty, only indices of support + vectors are stored in ``support_`` - * A reference (and not a copy) of the first argument in the ``fit()`` - method is stored for future reference. If that array changes between the - use of ``fit()`` and ``predict()`` you will have unexpected results. +* A reference (and not a copy) of the first argument in the ``fit()`` + method is stored for future reference. If that array changes between the + use of ``fit()`` and ``predict()`` you will have unexpected results. -Using Python functions as kernels -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. dropdown:: Using Python functions as kernels -You can use your own defined kernels by passing a function to the -``kernel`` parameter. + You can use your own defined kernels by passing a function to the + ``kernel`` parameter. -Your kernel must take as arguments two matrices of shape -``(n_samples_1, n_features)``, ``(n_samples_2, n_features)`` -and return a kernel matrix of shape ``(n_samples_1, n_samples_2)``. + Your kernel must take as arguments two matrices of shape + ``(n_samples_1, n_features)``, ``(n_samples_2, n_features)`` + and return a kernel matrix of shape ``(n_samples_1, n_samples_2)``. -The following code defines a linear kernel and creates a classifier -instance that will use that kernel:: + The following code defines a linear kernel and creates a classifier + instance that will use that kernel:: - >>> import numpy as np - >>> from sklearn import svm - >>> def my_kernel(X, Y): - ... return np.dot(X, Y.T) - ... - >>> clf = svm.SVC(kernel=my_kernel) + >>> import numpy as np + >>> from sklearn import svm + >>> def my_kernel(X, Y): + ... return np.dot(X, Y.T) + ... + >>> clf = svm.SVC(kernel=my_kernel) -.. topic:: Examples: - * :ref:`sphx_glr_auto_examples_svm_plot_custom_kernel.py`. +.. dropdown:: Using the Gram matrix -Using the Gram matrix -~~~~~~~~~~~~~~~~~~~~~ + You can pass pre-computed kernels by using the ``kernel='precomputed'`` + option. You should then pass Gram matrix instead of X to the `fit` and + `predict` methods. The kernel values between *all* training vectors and the + test vectors must be provided: -You can pass pre-computed kernels by using the ``kernel='precomputed'`` -option. You should then pass Gram matrix instead of X to the `fit` and -`predict` methods. The kernel values between *all* training vectors and the -test vectors must be provided: + >>> import numpy as np + >>> from sklearn.datasets import make_classification + >>> from sklearn.model_selection import train_test_split + >>> from sklearn import svm + >>> X, y = make_classification(n_samples=10, random_state=0) + >>> X_train , X_test , y_train, y_test = train_test_split(X, y, random_state=0) + >>> clf = svm.SVC(kernel='precomputed') + >>> # linear kernel computation + >>> gram_train = np.dot(X_train, X_train.T) + >>> clf.fit(gram_train, y_train) + SVC(kernel='precomputed') + >>> # predict on training examples + >>> gram_test = np.dot(X_test, X_train.T) + >>> clf.predict(gram_test) + array([0, 1, 0]) - >>> import numpy as np - >>> from sklearn.datasets import make_classification - >>> from sklearn.model_selection import train_test_split - >>> from sklearn import svm - >>> X, y = make_classification(n_samples=10, random_state=0) - >>> X_train , X_test , y_train, y_test = train_test_split(X, y, random_state=0) - >>> clf = svm.SVC(kernel='precomputed') - >>> # linear kernel computation - >>> gram_train = np.dot(X_train, X_train.T) - >>> clf.fit(gram_train, y_train) - SVC(kernel='precomputed') - >>> # predict on training examples - >>> gram_test = np.dot(X_test, X_train.T) - >>> clf.predict(gram_test) - array([0, 1, 0]) +.. rubric:: Examples +* :ref:`sphx_glr_auto_examples_svm_plot_custom_kernel.py` .. _svm_mathematical_formulation: @@ -623,7 +630,7 @@ misclassified or within the margin boundary. Ideally, the value :math:`y_i (w^T \phi (x_i) + b)` would be :math:`\geq 1` for all samples, which indicates a perfect prediction. But problems are usually not always perfectly separable with a hyperplane, so we allow some samples to be at a distance :math:`\zeta_i` from -their correct margin boundary. The penalty term `C` controls the strengh of +their correct margin boundary. The penalty term `C` controls the strength of this penalty, and as a result, acts as an inverse regularization parameter (see note below). @@ -652,14 +659,14 @@ Once the optimization problem is solved, the output of .. math:: \sum_{i\in SV} y_i \alpha_i K(x_i, x) + b, -and the predicted class correspond to its sign. We only need to sum over the +and the predicted class corresponds to its sign. We only need to sum over the support vectors (i.e. the samples that lie within the margin) because the dual coefficients :math:`\alpha_i` are zero for the other samples. These parameters can be accessed through the attributes ``dual_coef_`` which holds the product :math:`y_i \alpha_i`, ``support_vectors_`` which holds the support vectors, and ``intercept_`` which holds the independent -term :math:`b` +term :math:`b`. .. note:: @@ -668,40 +675,37 @@ term :math:`b` equivalence between the amount of regularization of two models depends on the exact objective function optimized by the model. For example, when the estimator used is :class:`~sklearn.linear_model.Ridge` regression, - the relation between them is given as :math:`C = \frac{1}{alpha}`. + the relation between them is given as :math:`C = \frac{1}{\alpha}`. -LinearSVC ---------- +.. dropdown:: LinearSVC -The primal problem can be equivalently formulated as + The primal problem can be equivalently formulated as -.. math:: + .. math:: - \min_ {w, b} \frac{1}{2} w^T w + C \sum_{i=1}\max(0, y_i (w^T \phi(x_i) + b)), + \min_ {w, b} \frac{1}{2} w^T w + C \sum_{i=1}^{n}\max(0, 1 - y_i (w^T \phi(x_i) + b)), -where we make use of the `hinge loss -`_. This is the form that is -directly optimized by :class:`LinearSVC`, but unlike the dual form, this one -does not involve inner products between samples, so the famous kernel trick -cannot be applied. This is why only the linear kernel is supported by -:class:`LinearSVC` (:math:`\phi` is the identity function). + where we make use of the `hinge loss + `_. This is the form that is + directly optimized by :class:`LinearSVC`, but unlike the dual form, this one + does not involve inner products between samples, so the famous kernel trick + cannot be applied. This is why only the linear kernel is supported by + :class:`LinearSVC` (:math:`\phi` is the identity function). .. _nu_svc: -NuSVC ------ +.. dropdown:: NuSVC -The :math:`\nu`-SVC formulation [#7]_ is a reparameterization of the -:math:`C`-SVC and therefore mathematically equivalent. - -We introduce a new parameter :math:`\nu` (instead of :math:`C`) which -controls the number of support vectors and *margin errors*: -:math:`\nu \in (0, 1]` is an upper bound on the fraction of margin errors and -a lower bound of the fraction of support vectors. A margin error corresponds -to a sample that lies on the wrong side of its margin boundary: it is either -misclassified, or it is correctly classified but does not lie beyond the -margin. + The :math:`\nu`-SVC formulation [#7]_ is a reparameterization of the + :math:`C`-SVC and therefore mathematically equivalent. + We introduce a new parameter :math:`\nu` (instead of :math:`C`) which + controls the number of support vectors and *margin errors*: + :math:`\nu \in (0, 1]` is an upper bound on the fraction of margin errors and + a lower bound of the fraction of support vectors. A margin error corresponds + to a sample that lies on the wrong side of its margin boundary: it is either + misclassified, or it is correctly classified but does not lie beyond the + margin. SVR --- @@ -750,18 +754,17 @@ which holds the difference :math:`\alpha_i - \alpha_i^*`, ``support_vectors_`` w holds the support vectors, and ``intercept_`` which holds the independent term :math:`b` -LinearSVR ---------- +.. dropdown:: LinearSVR -The primal problem can be equivalently formulated as + The primal problem can be equivalently formulated as -.. math:: + .. math:: - \min_ {w, b} \frac{1}{2} w^T w + C \sum_{i=1}\max(0, |y_i - (w^T \phi(x_i) + b)| - \varepsilon), + \min_ {w, b} \frac{1}{2} w^T w + C \sum_{i=1}^{n}\max(0, |y_i - (w^T \phi(x_i) + b)| - \varepsilon), -where we make use of the epsilon-insensitive loss, i.e. errors of less than -:math:`\varepsilon` are ignored. This is the form that is directly optimized -by :class:`LinearSVR`. + where we make use of the epsilon-insensitive loss, i.e. errors of less than + :math:`\varepsilon` are ignored. This is the form that is directly optimized + by :class:`LinearSVR`. .. _svm_implementation_details: @@ -777,38 +780,38 @@ used, please refer to their respective papers. .. _`libsvm`: https://www.csie.ntu.edu.tw/~cjlin/libsvm/ .. _`liblinear`: https://www.csie.ntu.edu.tw/~cjlin/liblinear/ -.. topic:: References: - - .. [#1] Platt `"Probabilistic outputs for SVMs and comparisons to - regularized likelihood methods" - `_. - - .. [#2] Wu, Lin and Weng, `"Probability estimates for multi-class - classification by pairwise coupling" - `_, JMLR - 5:975-1005, 2004. - - .. [#3] Fan, Rong-En, et al., - `"LIBLINEAR: A library for large linear classification." - `_, - Journal of machine learning research 9.Aug (2008): 1871-1874. - - .. [#4] Chang and Lin, `LIBSVM: A Library for Support Vector Machines - `_. - - .. [#5] Bishop, `Pattern recognition and machine learning - `_, - chapter 7 Sparse Kernel Machines - - .. [#6] `"A Tutorial on Support Vector Regression" - `_, - Alex J. Smola, Bernhard SchÃļlkopf - Statistics and Computing archive - Volume 14 Issue 3, August 2004, p. 199-222. - - .. [#7] SchÃļlkopf et. al `New Support Vector Algorithms - `_ - - .. [#8] Crammer and Singer `On the Algorithmic Implementation ofMulticlass - Kernel-based Vector Machines - `_, - JMLR 2001. +.. rubric:: References + +.. [#1] Platt `"Probabilistic outputs for SVMs and comparisons to + regularized likelihood methods" + `_. + +.. [#2] Wu, Lin and Weng, `"Probability estimates for multi-class + classification by pairwise coupling" + `_, + JMLR 5:975-1005, 2004. + +.. [#3] Fan, Rong-En, et al., + `"LIBLINEAR: A library for large linear classification." + `_, + Journal of machine learning research 9.Aug (2008): 1871-1874. + +.. [#4] Chang and Lin, `LIBSVM: A Library for Support Vector Machines + `_. + +.. [#5] Bishop, `Pattern recognition and machine learning + `_, + chapter 7 Sparse Kernel Machines. + +.. [#6] :doi:`"A Tutorial on Support Vector Regression" + <10.1023/B:STCO.0000035301.49549.88>` + Alex J. Smola, Bernhard SchÃļlkopf - Statistics and Computing archive + Volume 14 Issue 3, August 2004, p. 199-222. + +.. [#7] SchÃļlkopf et. al `New Support Vector Algorithms + `_, + Neural Computation 12, 1207-1245 (2000). + +.. [#8] Crammer and Singer `On the Algorithmic Implementation of Multiclass + Kernel-based Vector Machines + `_, JMLR 2001. diff --git a/doc/modules/tree.rst b/doc/modules/tree.rst index 6d30fdcc6bf2f..ee36d9f6af1b2 100644 --- a/doc/modules/tree.rst +++ b/doc/modules/tree.rst @@ -23,68 +23,68 @@ the tree, the more complex the decision rules and the fitter the model. Some advantages of decision trees are: - - Simple to understand and to interpret. Trees can be visualised. +- Simple to understand and to interpret. Trees can be visualized. - - Requires little data preparation. Other techniques often require data - normalisation, dummy variables need to be created and blank values to - be removed. Note however that this module does not support missing - values. +- Requires little data preparation. Other techniques often require data + normalization, dummy variables need to be created and blank values to + be removed. Some tree and algorithm combinations support + :ref:`missing values `. - - The cost of using the tree (i.e., predicting data) is logarithmic in the - number of data points used to train the tree. +- The cost of using the tree (i.e., predicting data) is logarithmic in the + number of data points used to train the tree. - - Able to handle both numerical and categorical data. However scikit-learn - implementation does not support categorical variables for now. Other - techniques are usually specialised in analysing datasets that have only one type - of variable. See :ref:`algorithms ` for more - information. +- Able to handle both numerical and categorical data. However, the scikit-learn + implementation does not support categorical variables for now. Other + techniques are usually specialized in analyzing datasets that have only one type + of variable. See :ref:`algorithms ` for more + information. - - Able to handle multi-output problems. +- Able to handle multi-output problems. - - Uses a white box model. If a given situation is observable in a model, - the explanation for the condition is easily explained by boolean logic. - By contrast, in a black box model (e.g., in an artificial neural - network), results may be more difficult to interpret. +- Uses a white box model. If a given situation is observable in a model, + the explanation for the condition is easily explained by boolean logic. + By contrast, in a black box model (e.g., in an artificial neural + network), results may be more difficult to interpret. - - Possible to validate a model using statistical tests. That makes it - possible to account for the reliability of the model. +- Possible to validate a model using statistical tests. That makes it + possible to account for the reliability of the model. - - Performs well even if its assumptions are somewhat violated by - the true model from which the data were generated. +- Performs well even if its assumptions are somewhat violated by + the true model from which the data were generated. The disadvantages of decision trees include: - - Decision-tree learners can create over-complex trees that do not - generalise the data well. This is called overfitting. Mechanisms - such as pruning, setting the minimum number of samples required - at a leaf node or setting the maximum depth of the tree are - necessary to avoid this problem. +- Decision-tree learners can create over-complex trees that do not + generalize the data well. This is called overfitting. Mechanisms + such as pruning, setting the minimum number of samples required + at a leaf node or setting the maximum depth of the tree are + necessary to avoid this problem. - - Decision trees can be unstable because small variations in the - data might result in a completely different tree being generated. - This problem is mitigated by using decision trees within an - ensemble. +- Decision trees can be unstable because small variations in the + data might result in a completely different tree being generated. + This problem is mitigated by using decision trees within an + ensemble. - - Predictions of decision trees are neither smooth nor continuous, but - piecewise constant approximations as seen in the above figure. Therefore, - they are not good at extrapolation. +- Predictions of decision trees are neither smooth nor continuous, but + piecewise constant approximations as seen in the above figure. Therefore, + they are not good at extrapolation. - - The problem of learning an optimal decision tree is known to be - NP-complete under several aspects of optimality and even for simple - concepts. Consequently, practical decision-tree learning algorithms - are based on heuristic algorithms such as the greedy algorithm where - locally optimal decisions are made at each node. Such algorithms - cannot guarantee to return the globally optimal decision tree. This - can be mitigated by training multiple trees in an ensemble learner, - where the features and samples are randomly sampled with replacement. +- The problem of learning an optimal decision tree is known to be + NP-complete under several aspects of optimality and even for simple + concepts. Consequently, practical decision-tree learning algorithms + are based on heuristic algorithms such as the greedy algorithm where + locally optimal decisions are made at each node. Such algorithms + cannot guarantee to return the globally optimal decision tree. This + can be mitigated by training multiple trees in an ensemble learner, + where the features and samples are randomly sampled with replacement. - - There are concepts that are hard to learn because decision trees - do not express them easily, such as XOR, parity or multiplexer problems. +- There are concepts that are hard to learn because decision trees + do not express them easily, such as XOR, parity or multiplexer problems. - - Decision tree learners create biased trees if some classes dominate. - It is therefore recommended to balance the dataset prior to fitting - with the decision tree. +- Decision tree learners create biased trees if some classes dominate. + It is therefore recommended to balance the dataset prior to fitting + with the decision tree. .. _tree_classification: @@ -130,90 +130,94 @@ Using the Iris dataset, we can construct a tree as follows:: >>> from sklearn.datasets import load_iris >>> from sklearn import tree - >>> X, y = load_iris(return_X_y=True) + >>> iris = load_iris() + >>> X, y = iris.data, iris.target >>> clf = tree.DecisionTreeClassifier() >>> clf = clf.fit(X, y) Once trained, you can plot the tree with the :func:`plot_tree` function:: - >>> tree.plot_tree(clf) # doctest: +SKIP + >>> tree.plot_tree(clf) + [...] .. figure:: ../auto_examples/tree/images/sphx_glr_plot_iris_dtc_002.png :target: ../auto_examples/tree/plot_iris_dtc.html :scale: 75 :align: center -We can also export the tree in `Graphviz -`_ format using the :func:`export_graphviz` -exporter. If you use the `conda `_ package manager, the graphviz binaries -and the python package can be installed with `conda install python-graphviz`. +.. dropdown:: Alternative ways to export trees -Alternatively binaries for graphviz can be downloaded from the graphviz project homepage, -and the Python wrapper installed from pypi with `pip install graphviz`. + We can also export the tree in `Graphviz + `_ format using the :func:`export_graphviz` + exporter. If you use the `conda `_ package manager, the graphviz binaries + and the python package can be installed with `conda install python-graphviz`. -Below is an example graphviz export of the above tree trained on the entire -iris dataset; the results are saved in an output file `iris.pdf`:: + Alternatively binaries for graphviz can be downloaded from the graphviz project homepage, + and the Python wrapper installed from pypi with `pip install graphviz`. + Below is an example graphviz export of the above tree trained on the entire + iris dataset; the results are saved in an output file `iris.pdf`:: - >>> import graphviz # doctest: +SKIP - >>> dot_data = tree.export_graphviz(clf, out_file=None) # doctest: +SKIP - >>> graph = graphviz.Source(dot_data) # doctest: +SKIP - >>> graph.render("iris") # doctest: +SKIP -The :func:`export_graphviz` exporter also supports a variety of aesthetic -options, including coloring nodes by their class (or value for regression) and -using explicit variable and class names if desired. Jupyter notebooks also -render these plots inline automatically:: + >>> import graphviz # doctest: +SKIP + >>> dot_data = tree.export_graphviz(clf, out_file=None) # doctest: +SKIP + >>> graph = graphviz.Source(dot_data) # doctest: +SKIP + >>> graph.render("iris") # doctest: +SKIP - >>> dot_data = tree.export_graphviz(clf, out_file=None, # doctest: +SKIP - ... feature_names=iris.feature_names, # doctest: +SKIP - ... class_names=iris.target_names, # doctest: +SKIP - ... filled=True, rounded=True, # doctest: +SKIP - ... special_characters=True) # doctest: +SKIP - >>> graph = graphviz.Source(dot_data) # doctest: +SKIP - >>> graph # doctest: +SKIP + The :func:`export_graphviz` exporter also supports a variety of aesthetic + options, including coloring nodes by their class (or value for regression) and + using explicit variable and class names if desired. Jupyter notebooks also + render these plots inline automatically:: -.. only:: html + >>> dot_data = tree.export_graphviz(clf, out_file=None, # doctest: +SKIP + ... feature_names=iris.feature_names, # doctest: +SKIP + ... class_names=iris.target_names, # doctest: +SKIP + ... filled=True, rounded=True, # doctest: +SKIP + ... special_characters=True) # doctest: +SKIP + >>> graph = graphviz.Source(dot_data) # doctest: +SKIP + >>> graph # doctest: +SKIP - .. figure:: ../images/iris.svg - :align: center + .. only:: html -.. only:: latex + .. figure:: ../images/iris.svg + :align: center - .. figure:: ../images/iris.pdf - :align: center + .. only:: latex -.. figure:: ../auto_examples/tree/images/sphx_glr_plot_iris_dtc_001.png - :target: ../auto_examples/tree/plot_iris_dtc.html - :align: center - :scale: 75 + .. figure:: ../images/iris.pdf + :align: center -Alternatively, the tree can also be exported in textual format with the -function :func:`export_text`. This method doesn't require the installation -of external libraries and is more compact: + .. figure:: ../auto_examples/tree/images/sphx_glr_plot_iris_dtc_001.png + :target: ../auto_examples/tree/plot_iris_dtc.html + :align: center + :scale: 75 - >>> from sklearn.datasets import load_iris - >>> from sklearn.tree import DecisionTreeClassifier - >>> from sklearn.tree import export_text - >>> iris = load_iris() - >>> decision_tree = DecisionTreeClassifier(random_state=0, max_depth=2) - >>> decision_tree = decision_tree.fit(iris.data, iris.target) - >>> r = export_text(decision_tree, feature_names=iris['feature_names']) - >>> print(r) - |--- petal width (cm) <= 0.80 - | |--- class: 0 - |--- petal width (cm) > 0.80 - | |--- petal width (cm) <= 1.75 - | | |--- class: 1 - | |--- petal width (cm) > 1.75 - | | |--- class: 2 - - -.. topic:: Examples: - - * :ref:`sphx_glr_auto_examples_tree_plot_iris_dtc.py` - * :ref:`sphx_glr_auto_examples_tree_plot_unveil_tree_structure.py` + Alternatively, the tree can also be exported in textual format with the + function :func:`export_text`. This method doesn't require the installation + of external libraries and is more compact: + + >>> from sklearn.datasets import load_iris + >>> from sklearn.tree import DecisionTreeClassifier + >>> from sklearn.tree import export_text + >>> iris = load_iris() + >>> decision_tree = DecisionTreeClassifier(random_state=0, max_depth=2) + >>> decision_tree = decision_tree.fit(iris.data, iris.target) + >>> r = export_text(decision_tree, feature_names=iris['feature_names']) + >>> print(r) + |--- petal width (cm) <= 0.80 + | |--- class: 0 + |--- petal width (cm) > 0.80 + | |--- petal width (cm) <= 1.75 + | | |--- class: 1 + | |--- petal width (cm) > 1.75 + | | |--- class: 2 + + +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_tree_plot_iris_dtc.py` +* :ref:`sphx_glr_auto_examples_tree_plot_unveil_tree_structure.py` .. _tree_regression: @@ -240,9 +244,9 @@ instead of integer values:: >>> clf.predict([[1, 1]]) array([0.5]) -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_tree_plot_tree_regression.py` +* :ref:`sphx_glr_auto_examples_tree_plot_tree_regression.py` .. _tree_multioutput: @@ -265,27 +269,26 @@ generalization accuracy of the resulting estimator may often be increased. With regard to decision trees, this strategy can readily be used to support multi-output problems. This requires the following changes: - - Store n output values in leaves, instead of 1; - - Use splitting criteria that compute the average reduction across all - n outputs. +- Store n output values in leaves, instead of 1; +- Use splitting criteria that compute the average reduction across all + n outputs. This module offers support for multi-output problems by implementing this strategy in both :class:`DecisionTreeClassifier` and :class:`DecisionTreeRegressor`. If a decision tree is fit on an output array Y of shape ``(n_samples, n_outputs)`` then the resulting estimator will: - * Output n_output values upon ``predict``; - - * Output a list of n_output arrays of class probabilities upon - ``predict_proba``. +* Output n_output values upon ``predict``; +* Output a list of n_output arrays of class probabilities upon + ``predict_proba``. The use of multi-output trees for regression is demonstrated in -:ref:`sphx_glr_auto_examples_tree_plot_tree_regression_multioutput.py`. In this example, the input +:ref:`sphx_glr_auto_examples_tree_plot_tree_regression.py`. In this example, the input X is a single real value and the outputs Y are the sine and cosine of X. -.. figure:: ../auto_examples/tree/images/sphx_glr_plot_tree_regression_multioutput_001.png - :target: ../auto_examples/tree/plot_tree_regression_multioutput.html +.. figure:: ../auto_examples/tree/images/sphx_glr_plot_tree_regression_002.png + :target: ../auto_examples/tree/plot_tree_regression.html :scale: 75 :align: center @@ -299,17 +302,16 @@ the lower half of those faces. :scale: 75 :align: center -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_tree_plot_tree_regression_multioutput.py` - * :ref:`sphx_glr_auto_examples_miscellaneous_plot_multioutput_face_completion.py` +* :ref:`sphx_glr_auto_examples_miscellaneous_plot_multioutput_face_completion.py` -.. topic:: References: +.. rubric:: References - * M. Dumont et al, `Fast multi-class image annotation with random subwindows - and multiple output randomized trees - `_, International Conference on - Computer Vision Theory and Applications 2009 +* M. Dumont et al, `Fast multi-class image annotation with random subwindows + and multiple output randomized trees + `_, + International Conference on Computer Vision Theory and Applications 2009 .. _tree_complexity: @@ -322,7 +324,8 @@ In general, the run time cost to construct a balanced binary tree is to generate balanced trees, they will not always be balanced. Assuming that the subtrees remain approximately balanced, the cost at each node consists of searching through :math:`O(n_{features})` to find the feature that offers the -largest reduction in entropy. This has a cost of +largest reduction in the impurity criterion, e.g. log loss (which is equivalent to an +information gain). This has a cost of :math:`O(n_{features}n_{samples}\log(n_{samples}))` at each node, leading to a total cost over the entire trees (by summing the cost at each node) of :math:`O(n_{features}n_{samples}^{2}\log(n_{samples}))`. @@ -331,65 +334,65 @@ total cost over the entire trees (by summing the cost at each node) of Tips on practical use ===================== - * Decision trees tend to overfit on data with a large number of features. - Getting the right ratio of samples to number of features is important, since - a tree with few samples in high dimensional space is very likely to overfit. - - * Consider performing dimensionality reduction (:ref:`PCA `, - :ref:`ICA `, or :ref:`feature_selection`) beforehand to - give your tree a better chance of finding features that are discriminative. - - * :ref:`sphx_glr_auto_examples_tree_plot_unveil_tree_structure.py` will help - in gaining more insights about how the decision tree makes predictions, which is - important for understanding the important features in the data. - - * Visualise your tree as you are training by using the ``export`` - function. Use ``max_depth=3`` as an initial tree depth to get a feel for - how the tree is fitting to your data, and then increase the depth. - - * Remember that the number of samples required to populate the tree doubles - for each additional level the tree grows to. Use ``max_depth`` to control - the size of the tree to prevent overfitting. - - * Use ``min_samples_split`` or ``min_samples_leaf`` to ensure that multiple - samples inform every decision in the tree, by controlling which splits will - be considered. A very small number will usually mean the tree will overfit, - whereas a large number will prevent the tree from learning the data. Try - ``min_samples_leaf=5`` as an initial value. If the sample size varies - greatly, a float number can be used as percentage in these two parameters. - While ``min_samples_split`` can create arbitrarily small leaves, - ``min_samples_leaf`` guarantees that each leaf has a minimum size, avoiding - low-variance, over-fit leaf nodes in regression problems. For - classification with few classes, ``min_samples_leaf=1`` is often the best - choice. - - Note that ``min_samples_split`` considers samples directly and independent of - ``sample_weight``, if provided (e.g. a node with m weighted samples is still - treated as having exactly m samples). Consider ``min_weight_fraction_leaf`` or - ``min_impurity_decrease`` if accounting for sample weights is required at splits. - - * Balance your dataset before training to prevent the tree from being biased - toward the classes that are dominant. Class balancing can be done by - sampling an equal number of samples from each class, or preferably by - normalizing the sum of the sample weights (``sample_weight``) for each - class to the same value. Also note that weight-based pre-pruning criteria, - such as ``min_weight_fraction_leaf``, will then be less biased toward - dominant classes than criteria that are not aware of the sample weights, - like ``min_samples_leaf``. - - * If the samples are weighted, it will be easier to optimize the tree - structure using weight-based pre-pruning criterion such as - ``min_weight_fraction_leaf``, which ensure that leaf nodes contain at least - a fraction of the overall sum of the sample weights. - - * All decision trees use ``np.float32`` arrays internally. - If training data is not in this format, a copy of the dataset will be made. - - * If the input matrix X is very sparse, it is recommended to convert to sparse - ``csc_matrix`` before calling fit and sparse ``csr_matrix`` before calling - predict. Training time can be orders of magnitude faster for a sparse - matrix input compared to a dense matrix when features have zero values in - most of the samples. +* Decision trees tend to overfit on data with a large number of features. + Getting the right ratio of samples to number of features is important, since + a tree with few samples in high dimensional space is very likely to overfit. + +* Consider performing dimensionality reduction (:ref:`PCA `, + :ref:`ICA `, or :ref:`feature_selection`) beforehand to + give your tree a better chance of finding features that are discriminative. + +* :ref:`sphx_glr_auto_examples_tree_plot_unveil_tree_structure.py` will help + in gaining more insights about how the decision tree makes predictions, which is + important for understanding the important features in the data. + +* Visualize your tree as you are training by using the ``export`` + function. Use ``max_depth=3`` as an initial tree depth to get a feel for + how the tree is fitting to your data, and then increase the depth. + +* Remember that the number of samples required to populate the tree doubles + for each additional level the tree grows to. Use ``max_depth`` to control + the size of the tree to prevent overfitting. + +* Use ``min_samples_split`` or ``min_samples_leaf`` to ensure that multiple + samples inform every decision in the tree, by controlling which splits will + be considered. A very small number will usually mean the tree will overfit, + whereas a large number will prevent the tree from learning the data. Try + ``min_samples_leaf=5`` as an initial value. If the sample size varies + greatly, a float number can be used as percentage in these two parameters. + While ``min_samples_split`` can create arbitrarily small leaves, + ``min_samples_leaf`` guarantees that each leaf has a minimum size, avoiding + low-variance, over-fit leaf nodes in regression problems. For + classification with few classes, ``min_samples_leaf=1`` is often the best + choice. + + Note that ``min_samples_split`` considers samples directly and independent of + ``sample_weight``, if provided (e.g. a node with m weighted samples is still + treated as having exactly m samples). Consider ``min_weight_fraction_leaf`` or + ``min_impurity_decrease`` if accounting for sample weights is required at splits. + +* Balance your dataset before training to prevent the tree from being biased + toward the classes that are dominant. Class balancing can be done by + sampling an equal number of samples from each class, or preferably by + normalizing the sum of the sample weights (``sample_weight``) for each + class to the same value. Also note that weight-based pre-pruning criteria, + such as ``min_weight_fraction_leaf``, will then be less biased toward + dominant classes than criteria that are not aware of the sample weights, + like ``min_samples_leaf``. + +* If the samples are weighted, it will be easier to optimize the tree + structure using weight-based pre-pruning criterion such as + ``min_weight_fraction_leaf``, which ensures that leaf nodes contain at least + a fraction of the overall sum of the sample weights. + +* All decision trees use ``np.float32`` arrays internally. + If training data is not in this format, a copy of the dataset will be made. + +* If the input matrix X is very sparse, it is recommended to convert to sparse + ``csc_matrix`` before calling fit and sparse ``csr_matrix`` before calling + predict. Training time can be orders of magnitude faster for a sparse + matrix input compared to a dense matrix when features have zero values in + most of the samples. .. _tree_algorithms: @@ -400,36 +403,37 @@ Tree algorithms: ID3, C4.5, C5.0 and CART What are all the various decision tree algorithms and how do they differ from each other? Which one is implemented in scikit-learn? -ID3_ (Iterative Dichotomiser 3) was developed in 1986 by Ross Quinlan. -The algorithm creates a multiway tree, finding for each node (i.e. in -a greedy manner) the categorical feature that will yield the largest -information gain for categorical targets. Trees are grown to their -maximum size and then a pruning step is usually applied to improve the -ability of the tree to generalise to unseen data. - -C4.5 is the successor to ID3 and removed the restriction that features -must be categorical by dynamically defining a discrete attribute (based -on numerical variables) that partitions the continuous attribute value -into a discrete set of intervals. C4.5 converts the trained trees -(i.e. the output of the ID3 algorithm) into sets of if-then rules. -These accuracy of each rule is then evaluated to determine the order -in which they should be applied. Pruning is done by removing a rule's -precondition if the accuracy of the rule improves without it. - -C5.0 is Quinlan's latest version release under a proprietary license. -It uses less memory and builds smaller rulesets than C4.5 while being -more accurate. - -CART_ (Classification and Regression Trees) is very similar to C4.5, but -it differs in that it supports numerical target variables (regression) and -does not compute rule sets. CART constructs binary trees using the feature -and threshold that yield the largest information gain at each node. - -scikit-learn uses an optimised version of the CART algorithm; however, scikit-learn -implementation does not support categorical variables for now. +.. dropdown:: Various decision tree algorithms + + ID3_ (Iterative Dichotomiser 3) was developed in 1986 by Ross Quinlan. + The algorithm creates a multiway tree, finding for each node (i.e. in + a greedy manner) the categorical feature that will yield the largest + information gain for categorical targets. Trees are grown to their + maximum size and then a pruning step is usually applied to improve the + ability of the tree to generalize to unseen data. + + C4.5 is the successor to ID3 and removed the restriction that features + must be categorical by dynamically defining a discrete attribute (based + on numerical variables) that partitions the continuous attribute value + into a discrete set of intervals. C4.5 converts the trained trees + (i.e. the output of the ID3 algorithm) into sets of if-then rules. + The accuracy of each rule is then evaluated to determine the order + in which they should be applied. Pruning is done by removing a rule's + precondition if the accuracy of the rule improves without it. + + C5.0 is Quinlan's latest version release under a proprietary license. + It uses less memory and builds smaller rulesets than C4.5 while being + more accurate. + + CART (Classification and Regression Trees) is very similar to C4.5, but + it differs in that it supports numerical target variables (regression) and + does not compute rule sets. CART constructs binary trees using the feature + and threshold that yield the largest information gain at each node. + +scikit-learn uses an optimized version of the CART algorithm; however, the +scikit-learn implementation does not support categorical variables for now. .. _ID3: https://en.wikipedia.org/wiki/ID3_algorithm -.. _CART: https://en.wikipedia.org/wiki/Predictive_analytics#Classification_and_regression_trees_.28CART.29 .. _tree_mathematical_formulation: @@ -442,14 +446,14 @@ Given training vectors :math:`x_i \in R^n`, i=1,..., l and a label vector such that the samples with the same labels or similar target values are grouped together. -Let the data at node :math:`m` be represented by :math:`Q_m` with :math:`N_m` +Let the data at node :math:`m` be represented by :math:`Q_m` with :math:`n_m` samples. For each candidate split :math:`\theta = (j, t_m)` consisting of a feature :math:`j` and threshold :math:`t_m`, partition the data into :math:`Q_m^{left}(\theta)` and :math:`Q_m^{right}(\theta)` subsets .. math:: - Q_m^{left}(\theta) = \{(x, y) | x_j <= t_m\} + Q_m^{left}(\theta) = \{(x, y) | x_j \leq t_m\} Q_m^{right}(\theta) = Q_m \setminus Q_m^{left}(\theta) @@ -459,8 +463,8 @@ the task being solved (classification or regression) .. math:: - G(Q_m, \theta) = \frac{N_m^{left}}{N_m} H(Q_m^{left}(\theta)) - + \frac{N_m^{right}}{N_m} H(Q_m^{right}(\theta)) + G(Q_m, \theta) = \frac{n_m^{left}}{n_m} H(Q_m^{left}(\theta)) + + \frac{n_m^{right}}{n_m} H(Q_m^{right}(\theta)) Select the parameters that minimises the impurity @@ -470,7 +474,7 @@ Select the parameters that minimises the impurity Recurse for subsets :math:`Q_m^{left}(\theta^*)` and :math:`Q_m^{right}(\theta^*)` until the maximum allowable depth is reached, -:math:`N_m < \min_{samples}` or :math:`N_m = 1`. +:math:`n_m < \min_{samples}` or :math:`n_m = 1`. Classification criteria ----------------------- @@ -480,7 +484,7 @@ for node :math:`m`, let .. math:: - p_{mk} = 1/ N_m \sum_{y \in Q_m} I(y = k) + p_{mk} = \frac{1}{n_m} \sum_{y \in Q_m} I(y = k) be the proportion of class k observations in node :math:`m`. If :math:`m` is a terminal node, `predict_proba` for this region is set to :math:`p_{mk}`. @@ -492,17 +496,41 @@ Gini: H(Q_m) = \sum_k p_{mk} (1 - p_{mk}) -Entropy: +Log Loss or Entropy: .. math:: H(Q_m) = - \sum_k p_{mk} \log(p_{mk}) -Misclassification: +.. dropdown:: Shannon entropy -.. math:: + The entropy criterion computes the Shannon entropy of the possible classes. It + takes the class frequencies of the training data points that reached a given + leaf :math:`m` as their probability. Using the **Shannon entropy as tree node + splitting criterion is equivalent to minimizing the log loss** (also known as + cross-entropy and multinomial deviance) between the true labels :math:`y_i` + and the probabilistic predictions :math:`T_k(x_i)` of the tree model :math:`T` for class :math:`k`. + + To see this, first recall that the log loss of a tree model :math:`T` + computed on a dataset :math:`D` is defined as follows: + + .. math:: + + \mathrm{LL}(D, T) = -\frac{1}{n} \sum_{(x_i, y_i) \in D} \sum_k I(y_i = k) \log(T_k(x_i)) - H(Q_m) = 1 - \max(p_{mk}) + where :math:`D` is a training dataset of :math:`n` pairs :math:`(x_i, y_i)`. + + In a classification tree, the predicted class probabilities within leaf nodes + are constant, that is: for all :math:`(x_i, y_i) \in Q_m`, one has: + :math:`T_k(x_i) = p_{mk}` for each class :math:`k`. + + This property makes it possible to rewrite :math:`\mathrm{LL}(D, T)` as the + sum of the Shannon entropies computed for each leaf of :math:`T` weighted by + the number of training data points that reached each leaf: + + .. math:: + + \mathrm{LL}(D, T) = \sum_{m \in T} \frac{n_m}{n} H(Q_m) Regression criteria ------------------- @@ -519,21 +547,22 @@ Mean Squared Error: .. math:: - \bar{y}_m = \frac{1}{N_m} \sum_{y \in Q_m} y + \bar{y}_m = \frac{1}{n_m} \sum_{y \in Q_m} y - H(Q_m) = \frac{1}{N_m} \sum_{y \in Q_m} (y - \bar{y}_m)^2 + H(Q_m) = \frac{1}{n_m} \sum_{y \in Q_m} (y - \bar{y}_m)^2 -Half Poisson deviance: +Mean Poisson deviance: .. math:: - H(Q_m) = \frac{1}{N_m} \sum_{y \in Q_m} (y \log\frac{y}{\bar{y}_m} + H(Q_m) = \frac{2}{n_m} \sum_{y \in Q_m} (y \log\frac{y}{\bar{y}_m} - y + \bar{y}_m) Setting `criterion="poisson"` might be a good choice if your target is a count or a frequency (count per some unit). In any case, :math:`y >= 0` is a necessary condition to use this criterion. Note that it fits much slower than -the MSE criterion. +the MSE criterion. For performance reasons the actual implementation minimizes +the half mean poisson deviance, i.e. the mean poisson deviance divided by 2. Mean Absolute Error: @@ -541,10 +570,95 @@ Mean Absolute Error: median(y)_m = \underset{y \in Q_m}{\mathrm{median}}(y) - H(Q_m) = \frac{1}{N_m} \sum_{y \in Q_m} |y - median(y)_m| + H(Q_m) = \frac{1}{n_m} \sum_{y \in Q_m} |y - median(y)_m| Note that it fits much slower than the MSE criterion. +.. _tree_missing_value_support: + +Missing Values Support +====================== + +:class:`DecisionTreeClassifier`, :class:`DecisionTreeRegressor` +have built-in support for missing values using `splitter='best'`, where +the splits are determined in a greedy fashion. +:class:`ExtraTreeClassifier`, and :class:`ExtraTreeRegressor` have built-in +support for missing values for `splitter='random'`, where the splits +are determined randomly. For more details on how the splitter differs on +non-missing values, see the :ref:`Forest section `. + +The criterion supported when there are missing values are +`'gini'`, `'entropy'`, or `'log_loss'`, for classification or +`'squared_error'`, `'friedman_mse'`, or `'poisson'` for regression. + +First we will describe how :class:`DecisionTreeClassifier`, :class:`DecisionTreeRegressor` +handle missing-values in the data. + +For each potential threshold on the non-missing data, the splitter will evaluate +the split with all the missing values going to the left node or the right node. + +Decisions are made as follows: + +- By default when predicting, the samples with missing values are classified + with the class used in the split found during training:: + + >>> from sklearn.tree import DecisionTreeClassifier + >>> import numpy as np + + >>> X = np.array([0, 1, 6, np.nan]).reshape(-1, 1) + >>> y = [0, 0, 1, 1] + + >>> tree = DecisionTreeClassifier(random_state=0).fit(X, y) + >>> tree.predict(X) + array([0, 0, 1, 1]) + +- If the criterion evaluation is the same for both nodes, + then the tie for missing value at predict time is broken by going to the + right node. The splitter also checks the split where all the missing + values go to one child and non-missing values go to the other:: + + >>> from sklearn.tree import DecisionTreeClassifier + >>> import numpy as np + + >>> X = np.array([np.nan, -1, np.nan, 1]).reshape(-1, 1) + >>> y = [0, 0, 1, 1] + + >>> tree = DecisionTreeClassifier(random_state=0).fit(X, y) + + >>> X_test = np.array([np.nan]).reshape(-1, 1) + >>> tree.predict(X_test) + array([1]) + +- If no missing values are seen during training for a given feature, then during + prediction missing values are mapped to the child with the most samples:: + + >>> from sklearn.tree import DecisionTreeClassifier + >>> import numpy as np + + >>> X = np.array([0, 1, 2, 3]).reshape(-1, 1) + >>> y = [0, 1, 1, 1] + + >>> tree = DecisionTreeClassifier(random_state=0).fit(X, y) + + >>> X_test = np.array([np.nan]).reshape(-1, 1) + >>> tree.predict(X_test) + array([1]) + +:class:`ExtraTreeClassifier`, and :class:`ExtraTreeRegressor` handle missing values +in a slightly different way. When splitting a node, a random threshold will be chosen +to split the non-missing values on. Then the non-missing values will be sent to the +left and right child based on the randomly selected threshold, while the missing +values will also be randomly sent to the left or right child. This is repeated for +every feature considered at each split. The best split among these is chosen. + +During prediction, the treatment of missing-values is the same as that of the +decision tree: + +- By default when predicting, the samples with missing values are classified + with the class used in the split found during training. + +- If no missing values are seen during training for a given feature, then during + prediction missing values are mapped to the child with the most samples. .. _minimal_cost_complexity_pruning: @@ -581,21 +695,21 @@ with the smallest value of :math:`\alpha_{eff}` is the weakest link and will be pruned. This process stops when the pruned tree's minimal :math:`\alpha_{eff}` is greater than the ``ccp_alpha`` parameter. -.. topic:: Examples: +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_tree_plot_cost_complexity_pruning.py` +* :ref:`sphx_glr_auto_examples_tree_plot_cost_complexity_pruning.py` -.. topic:: References: +.. rubric:: References - .. [BRE] L. Breiman, J. Friedman, R. Olshen, and C. Stone. Classification - and Regression Trees. Wadsworth, Belmont, CA, 1984. +.. [BRE] L. Breiman, J. Friedman, R. Olshen, and C. Stone. Classification + and Regression Trees. Wadsworth, Belmont, CA, 1984. - * https://en.wikipedia.org/wiki/Decision_tree_learning +* https://en.wikipedia.org/wiki/Decision_tree_learning - * https://en.wikipedia.org/wiki/Predictive_analytics +* https://en.wikipedia.org/wiki/Predictive_analytics - * J.R. Quinlan. C4. 5: programs for machine learning. Morgan - Kaufmann, 1993. +* J.R. Quinlan. C4. 5: programs for machine learning. Morgan + Kaufmann, 1993. - * T. Hastie, R. Tibshirani and J. Friedman. Elements of Statistical - Learning, Springer, 2009. +* T. Hastie, R. Tibshirani and J. Friedman. Elements of Statistical + Learning, Springer, 2009. diff --git a/doc/modules/unsupervised_reduction.rst b/doc/modules/unsupervised_reduction.rst index 6e16886064cfc..12f3647454861 100644 --- a/doc/modules/unsupervised_reduction.rst +++ b/doc/modules/unsupervised_reduction.rst @@ -9,7 +9,7 @@ If your number of features is high, it may be useful to reduce it with an unsupervised step prior to supervised steps. Many of the :ref:`unsupervised-learning` methods implement a ``transform`` method that can be used to reduce the dimensionality. Below we discuss two specific -example of this pattern that are heavily used. +examples of this pattern that are heavily used. .. topic:: **Pipelining** @@ -24,20 +24,20 @@ PCA: principal component analysis :class:`decomposition.PCA` looks for a combination of features that capture well the variance of the original features. See :ref:`decompositions`. -.. topic:: **Examples** +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_applications_plot_face_recognition.py` +* :ref:`sphx_glr_auto_examples_applications_plot_face_recognition.py` Random projections ------------------- -The module: :mod:`random_projection` provides several tools for data +The module: :mod:`~sklearn.random_projection` provides several tools for data reduction by random projections. See the relevant section of the documentation: :ref:`random_projection`. -.. topic:: **Examples** +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_miscellaneous_plot_johnson_lindenstrauss_bound.py` +* :ref:`sphx_glr_auto_examples_miscellaneous_plot_johnson_lindenstrauss_bound.py` Feature agglomeration ------------------------ @@ -46,15 +46,14 @@ Feature agglomeration :ref:`hierarchical_clustering` to group together features that behave similarly. -.. topic:: **Examples** +.. rubric:: Examples - * :ref:`sphx_glr_auto_examples_cluster_plot_feature_agglomeration_vs_univariate_selection.py` - * :ref:`sphx_glr_auto_examples_cluster_plot_digits_agglomeration.py` +* :ref:`sphx_glr_auto_examples_cluster_plot_feature_agglomeration_vs_univariate_selection.py` +* :ref:`sphx_glr_auto_examples_cluster_plot_digits_agglomeration.py` .. topic:: **Feature scaling** Note that if features have very different scaling or statistical properties, :class:`cluster.FeatureAgglomeration` may not be able to - capture the links between related features. Using a + capture the links between related features. Using a :class:`preprocessing.StandardScaler` can be useful in these settings. - diff --git a/doc/preface.rst b/doc/preface.rst deleted file mode 100644 index 447083a3a8136..0000000000000 --- a/doc/preface.rst +++ /dev/null @@ -1,32 +0,0 @@ -.. This helps define the TOC ordering for "about us" sections. Particularly - useful for PDF output as this section is not linked from elsewhere. - -.. Places global toc into the sidebar - -:globalsidebartoc: True - -.. _preface_menu: - -.. include:: includes/big_toc_css.rst -.. include:: tune_toc.rst - -======================= -Welcome to scikit-learn -======================= - -| - -.. toctree:: - :maxdepth: 2 - - install - faq - support - related_projects - about - testimonials/testimonials - whats_new - roadmap - governance - -| diff --git a/doc/presentations.rst b/doc/presentations.rst index 15b02469d3a6c..25a947d180e00 100644 --- a/doc/presentations.rst +++ b/doc/presentations.rst @@ -1,15 +1,52 @@ +.. _external_resources: + =========================================== External Resources, Videos and Talks =========================================== -For written tutorials, see the :ref:`Tutorial section ` of -the documentation. +The scikit-learn MOOC +===================== + +If you are new to scikit-learn, or looking to strengthen your understanding, +we highly recommend the **scikit-learn MOOC (Massive Open Online Course)**. + +The MOOC, created and maintained by some of the scikit-learn core-contributors, +is **free of charge** and is designed to help learners of all levels master +machine learning using scikit-learn. It covers topics +from the fundamental machine learning concepts to more advanced areas like +predictive modeling pipelines and model evaluation. + +The course materials are available on the +`scikit-learn MOOC website `_. + +This course is also hosted on the `FUN platform +`_, +which additionally makes the content interactive without the need to install +anything, and gives access to a discussion forum. + +The videos are available on the +`Inria Learning Lab channel `_ +in a +`playlist `__. + +.. _videos: + +Videos +====== + +- The `scikit-learn YouTube channel `_ + features a + `playlist `__ + of videos + showcasing talks by maintainers + and community members. New to Scientific Python? ========================== + For those that are still new to the scientific Python ecosystem, we highly recommend the `Python Scientific Lecture Notes -`_. This will help you find your footing a +`_. This will help you find your footing a bit and will definitely improve your scikit-learn experience. A basic understanding of NumPy arrays is recommended to make the most of scikit-learn. @@ -21,58 +58,3 @@ specific subject areas: - `Machine Learning for NeuroImaging in Python `_ - `Machine Learning for Astronomical Data Analysis `_ - -.. _videos: - -Videos -====== - -- An introduction to scikit-learn `Part - I `_ and - `Part II `_ at Scipy 2013 - by `Gael Varoquaux`_, `Jake Vanderplas`_ and `Olivier Grisel`_. Notebooks on - `github `_. - -- `Introduction to scikit-learn - `_ by `Gael Varoquaux`_ at - ICML 2010 - - A three minute video from a very early stage of scikit-learn, explaining the - basic idea and approach we are following. - -- `Introduction to statistical learning with scikit-learn `_ - by `Gael Varoquaux`_ at SciPy 2011 - - An extensive tutorial, consisting of four sessions of one hour. - The tutorial covers the basics of machine learning, - many algorithms and how to apply them using scikit-learn. The - material corresponding is now in the scikit-learn documentation - section :ref:`stat_learn_tut_index`. - -- `Statistical Learning for Text Classification with scikit-learn and NLTK - `_ - (and `slides `_) - by `Olivier Grisel`_ at PyCon 2011 - - Thirty minute introduction to text classification. Explains how to - use NLTK and scikit-learn to solve real-world text classification - tasks and compares against cloud-based solutions. - -- `Introduction to Interactive Predictive Analytics in Python with scikit-learn `_ - by `Olivier Grisel`_ at PyCon 2012 - - 3-hours long introduction to prediction tasks using scikit-learn. - -- `scikit-learn - Machine Learning in Python `_ - by `Jake Vanderplas`_ at the 2012 PyData workshop at Google - - Interactive demonstration of some scikit-learn features. 75 minutes. - -- `scikit-learn tutorial `_ by `Jake Vanderplas`_ at PyData NYC 2012 - - Presentation using the online tutorial, 45 minutes. - - -.. _Gael Varoquaux: http://gael-varoquaux.info -.. _Jake Vanderplas: https://staff.washington.edu/jakevdp -.. _Olivier Grisel: https://twitter.com/ogrisel diff --git a/doc/related_projects.rst b/doc/related_projects.rst index acc2689388896..a7a10aef7929e 100644 --- a/doc/related_projects.rst +++ b/doc/related_projects.rst @@ -7,7 +7,7 @@ Related Projects Projects implementing the scikit-learn estimator API are encouraged to use the `scikit-learn-contrib template `_ which facilitates best practices for testing and documenting estimators. -The `scikit-learn-contrib GitHub organisation `_ +The `scikit-learn-contrib GitHub organization `_ also accepts high-quality contributions of repositories conforming to this template. @@ -19,17 +19,6 @@ Interoperability and framework enhancements These tools adapt scikit-learn for use with other technologies or otherwise enhance the functionality of scikit-learn's estimators. -**Data formats** - -- `Fast svmlight / libsvm file loader `_ - Fast and memory-efficient svmlight / libsvm file loader for Python. - -- `sklearn_pandas `_ bridge for - scikit-learn pipelines and pandas data frame with dedicated transformers. - -- `sklearn_xarray `_ provides - compatibility of scikit-learn estimators with xarray data structures. - **Auto-ML** - `auto-sklearn `_ @@ -45,74 +34,99 @@ enhance the functionality of scikit-learn's estimators. operators to design a machine learning pipeline, including data and feature preprocessors as well as the estimators. Works as a drop-in replacement for a scikit-learn estimator. - -- `Featuretools `_ - A framework to perform automated feature engineering. It can be used for - transforming temporal and relational datasets into feature matrices for + +- `Featuretools `_ + A framework to perform automated feature engineering. It can be used for + transforming temporal and relational datasets into feature matrices for machine learning. -- `Neuraxle `_ - A library for building neat pipelines, providing the right abstractions to - both ease research, development, and deployment of machine learning - applications. Compatible with deep learning frameworks and scikit-learn API, - it can stream minibatches, use data checkpoints, build funky pipelines, and - serialize models with custom per-step savers. +- `EvalML `_ + An AutoML library which builds, optimizes, and evaluates + machine learning pipelines using domain-specific objective functions. + It incorporates multiple modeling libraries under one API, and + the objects that EvalML creates use an sklearn-compatible API. + +- `MLJAR AutoML `_ + A Python package for AutoML on Tabular Data with Feature Engineering, + Hyper-Parameters Tuning, Explanations and Automatic Documentation. -**Experimentation frameworks** +**Experimentation and model registry frameworks** -- `REP `_ Environment for conducting data-driven - research in a consistent and reproducible way +- `MLFlow `_ An open source platform to manage the ML + lifecycle, including experimentation, reproducibility, deployment, and a central + model registry. + +- `Neptune `_ A metadata store for MLOps, + built for teams that run a lot of experiments. It gives you a single + place to log, store, display, organize, compare, and query all your + model building metadata. + +- `Sacred `_ A tool to help you configure, + organize, log and reproduce experiments - `Scikit-Learn Laboratory `_ A command-line wrapper around scikit-learn that makes it easy to run machine learning experiments with multiple learners and large feature sets. -**Model inspection and visualisation** +**Model inspection and visualization** -- `dtreeviz `_ A python library for +- `dtreeviz `_ A Python library for decision tree visualization and model interpretation. -- `eli5 `_ A library for - debugging/inspecting machine learning models and explaining their - predictions. +- `model-diagnostics `_ Tools for + diagnostics and assessment of (machine learning) models (in Python). -- `mlxtend `_ Includes model visualization - utilities. +- `sklearn-evaluation `_ + Machine learning model evaluation made easy: plots, tables, HTML reports, + experiment tracking and Jupyter notebook analysis. Visual analysis, model + selection, evaluation and diagnostics. - `yellowbrick `_ A suite of custom matplotlib visualizers for scikit-learn estimators to support visual feature analysis, model selection, evaluation, and diagnostics. -**Model selection** - -- `scikit-optimize `_ - A library to minimize (very) expensive and noisy black-box functions. It - implements several methods for sequential model-based optimization, and - includes a replacement for ``GridSearchCV`` or ``RandomizedSearchCV`` to do - cross-validated parameter search using any of these strategies. - -- `sklearn-deap `_ Use evolutionary - algorithms instead of gridsearch in scikit-learn. - **Model export for production** - `sklearn-onnx `_ Serialization of many Scikit-learn pipelines to `ONNX `_ for interchange and prediction. +- `skops.io `__ A + persistence model more secure than pickle, which can be used instead of + pickle in most common cases. + - `sklearn2pmml `_ Serialization of a wide variety of scikit-learn estimators and transformers into PMML with the help of `JPMML-SkLearn `_ library. -- `sklearn-porter `_ - Transpile trained scikit-learn models to C, Java, Javascript and others. - - `treelite `_ Compiles tree-based ensemble models into C code for minimizing prediction latency. +- `emlearn `_ + Implements scikit-learn estimators in C99 for embedded devices and microcontrollers. + Supports several classifier, regression and outlier detection models. + +**Model throughput** + +- `Intel(R) Extension for scikit-learn `_ + Mostly on high end Intel(R) hardware, accelerates some scikit-learn models + for both training and inference under certain circumstances. This project is + maintained by Intel(R) and scikit-learn's maintainers are not involved in the + development of this project. Also note that in some cases using the tools and + estimators under ``scikit-learn-intelex`` would give different results than + ``scikit-learn`` itself. If you encounter issues while using this project, + make sure you report potential issues in their respective repositories. + +**Interface to R with genomic applications** + +- `BiocSklearn `_ + Exposes a small number of dimension reduction facilities as an illustration + of the basilisk protocol for interfacing Python with R. Intended as a + springboard for more complete interop. + Other estimators and tasks -------------------------- @@ -122,37 +136,52 @@ project. The following are projects providing interfaces similar to scikit-learn for additional learning algorithms, infrastructures and tasks. -**Structured learning** +**Time series and forecasting** -- `tslearn `_ A machine learning library for time series - that offers tools for pre-processing and feature extraction as well as dedicated models for clustering, classification and regression. +- `aeon `_ A + scikit-learn compatible toolbox for machine learning with time series + (fork of `sktime`_). -- `sktime `_ A scikit-learn compatible toolbox for machine learning with time series including time series classification/regression and (supervised/panel) forecasting. +- `Darts `_ A Python library for + user-friendly forecasting and anomaly detection on time series. It contains a variety + of models, from classics such as ARIMA to deep neural networks. The forecasting + models can all be used in the same way, using fit() and predict() functions, similar + to scikit-learn. -- `HMMLearn `_ Implementation of hidden - markov models that was previously part of scikit-learn. +- `sktime `_ A scikit-learn compatible + toolbox for machine learning with time series including time series + classification/regression and (supervised/panel) forecasting. -- `PyStruct `_ General conditional random fields - and structured prediction. +- `skforecast `_ A Python library + that eases using scikit-learn regressors as multi-step forecasters. It also works + with any regressor compatible with the scikit-learn API. -- `pomegranate `_ Probabilistic modelling - for Python, with an emphasis on hidden Markov models. +- `tslearn `_ A machine learning library for + time series that offers tools for pre-processing and feature extraction as well as + dedicated models for clustering, classification and regression. -- `sklearn-crfsuite `_ - Linear-chain conditional random fields - (`CRFsuite `_ wrapper with - sklearn-like API). +**Gradient (tree) boosting** -**Deep neural networks etc.** +Note scikit-learn own modern gradient boosting estimators +:class:`~sklearn.ensemble.HistGradientBoostingClassifier` and +:class:`~sklearn.ensemble.HistGradientBoostingRegressor`. -- `nolearn `_ A number of wrappers and - abstractions around existing neural network libraries +- `XGBoost `_ XGBoost is an optimized distributed + gradient boosting library designed to be highly efficient, flexible and portable. -- `Keras `_ High-level API for - TensorFlow with a scikit-learn inspired API. +- `LightGBM `_ LightGBM is a gradient boosting + framework that uses tree based learning algorithms. It is designed to be distributed + and efficient. -- `lasagne `_ A lightweight library to - build and train neural networks in Theano. +**Structured learning** + +- `HMMLearn `_ Implementation of hidden + markov models that was previously part of scikit-learn. + +- `pomegranate `_ Probabilistic modelling + for Python, with an emphasis on hidden Markov models. + +**Deep neural networks etc.** - `skorch `_ A scikit-learn compatible neural network library that wraps PyTorch. @@ -161,47 +190,35 @@ and tasks. Keras to interface it with scikit-learn. SciKeras is the successor of `tf.keras.wrappers.scikit_learn`. +**Federated Learning** + +- `Flower `_ A friendly federated learning framework with a + unified approach that can federate any workload, any ML framework, and any programming language. + +**Privacy Preserving Machine Learning** + +- `Concrete ML `_ A privacy preserving + ML framework built on top of `Concrete + `_, with bindings to traditional ML + frameworks, thanks to fully homomorphic encryption. APIs of so-called + Concrete ML built-in models are very close to scikit-learn APIs. + **Broad scope** - `mlxtend `_ Includes a number of additional estimators as well as model visualization utilities. -- `scikit-lego `_ A number of scikit-learn compatible +- `scikit-lego `_ A number of scikit-learn compatible custom transformers, models and metrics, focusing on solving practical industry tasks. **Other regression and classification** -- `xgboost `_ Optimised gradient boosted decision - tree library. - -- `ML-Ensemble `_ Generalized - ensemble learning (stacking, blending, subsemble, deep ensembles, - etc.). - -- `lightning `_ Fast - state-of-the-art linear model solvers (SDCA, AdaGrad, SVRG, SAG, etc...). - -- `py-earth `_ Multivariate - adaptive regression splines - -- `Kernel Regression `_ - Implementation of Nadaraya-Watson kernel regression with automatic bandwidth - selection - - `gplearn `_ Genetic Programming for symbolic regression tasks. - `scikit-multilearn `_ Multi-label classification with focus on label space manipulation. -- `seglearn `_ Time series and sequence - learning using sliding window segmentation. - -- `libOPF `_ Optimal path forest classifier - -- `fastFM `_ Fast factorization machine - implementation compatible with scikit-learn - **Decomposition and clustering** - `lda `_: Fast implementation of latent @@ -218,16 +235,19 @@ and tasks. - `hdbscan `_ HDBSCAN and Robust Single Linkage clustering algorithms for robust variable density clustering. - -- `spherecluster `_ Spherical - K-means and mixture of von Mises Fisher clustering routines for data on the - unit hypersphere. + As of scikit-learn version 1.3.0, there is :class:`~sklearn.cluster.HDBSCAN`. **Pre-processing** - `categorical-encoding `_ A library of sklearn compatible categorical variable encoders. + As of scikit-learn version 1.3.0, there is + :class:`~sklearn.preprocessing.TargetEncoder`. + +- `skrub `_ : facilitate learning on dataframes, + with sklearn compatible encoders (of categories, dates, strings) and + more. - `imbalanced-learn `_ Various @@ -261,15 +281,16 @@ Other packages useful for data analysis and machine learning. statistical models. More focused on statistical tests and less on prediction than scikit-learn. -- `PyMC `_ Bayesian statistical models and +- `PyMC `_ Bayesian statistical models and fitting algorithms. -- `Sacred `_ Tool to help you configure, - organize, log and reproduce experiments - -- `Seaborn `_ Visualization library based on +- `Seaborn `_ A visualization library based on matplotlib. It provides a high-level interface for drawing attractive statistical graphics. +- `scikit-survival `_ A library implementing + models to learn from censored time-to-event data (also called survival analysis). + Models are fully compatible with scikit-learn. + Recommendation Engine packages ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -279,20 +300,16 @@ Recommendation Engine packages - `lightfm `_ A Python/Cython implementation of a hybrid recommender system. -- `OpenRec `_ TensorFlow-based - neural-network inspired recommendation algorithms. - -- `Spotlight `_ Pytorch-based - implementation of deep recommender models. - -- `Surprise Lib `_ Library for explicit feedback +- `Surprise Lib `_ Library for explicit feedback datasets. Domain specific packages ~~~~~~~~~~~~~~~~~~~~~~~~ +- `scikit-network `_ Machine learning on graphs. + - `scikit-image `_ Image processing and computer - vision in python. + vision in Python. - `Natural language toolkit (nltk) `_ Natural language processing and some machine learning. @@ -304,13 +321,10 @@ Domain specific packages - `AstroML `_ Machine learning for astronomy. -- `MSMBuilder `_ Machine learning for protein - conformational dynamics time series. - Translations of scikit-learn documentation ------------------------------------------ -Translation’s purpose is to ease reading and understanding in languages +Translation's purpose is to ease reading and understanding in languages other than English. Its aim is to help people who do not understand English or have doubts about its interpretation. Additionally, some people prefer to read documentation in their native language, but please bear in mind that @@ -327,9 +341,13 @@ and promote community efforts. (`source `__) - `Persian translation `_ (`source `__) +- `Spanish translation `_ + (`source `__) +- `Korean translation `_ + (`source `__) + .. rubric:: Footnotes .. [#f1] following `linux documentation Disclaimer `__ - diff --git a/doc/roadmap.rst b/doc/roadmap.rst index 30c9f58339502..a9e3e73d01deb 100644 --- a/doc/roadmap.rst +++ b/doc/roadmap.rst @@ -1,5 +1,3 @@ -īģŋ.. _roadmap: - .. |ss| raw:: html @@ -8,12 +6,14 @@ +.. _roadmap: + Roadmap ======= Purpose of this document ------------------------ -This document list general directions that core contributors are interested +This document lists general directions that core contributors are interested to see developed in scikit-learn. The fact that an item is listed here is in no way a promise that it will happen, as resources are limited. Rather, it is an indication that help is welcomed on this topic. @@ -51,7 +51,7 @@ external to the core library. (i.e. rectangular data largely invariant to column and row order; predicting targets with simple structure) * improve the ease for users to develop and publish external components -* improve inter-operability with modern data science tools (e.g. Pandas, Dask) +* improve interoperability with modern data science tools (e.g. Pandas, Dask) and infrastructures (e.g. distributed processing) Many of the more fine-grained goals can be found under the `API tag @@ -69,29 +69,17 @@ the document up to date as we work on these issues. #. Improved handling of Pandas DataFrames * document current handling - * column reordering issue :issue:`7242` - * avoiding unnecessary conversion to ndarray :issue:`12147` - * returning DataFrames from transformers :issue:`5523` - * getting DataFrames from dataset loaders :issue:`10733`, - |ss| :issue:`13902` |se| - * Sparse currently not considered :issue:`12800` #. Improved handling of categorical features * Tree-based models should be able to handle both continuous and categorical - features :issue:`12866` and :issue:`15550`. - * |ss| In dataset loaders :issue:`13902` |se| - * As generic transformers to be used with ColumnTransforms (e.g. ordinal - encoding supervised by correlation with target variable) :issue:`5853`, - :issue:`11805` + features :issue:`29437`. * Handling mixtures of categorical and continuous variables #. Improved handling of missing data - * Making sure meta-estimators are lenient towards missing data, - :issue:`15319` - * Non-trivial imputers |ss| :issue:`11977`, :issue:`12852` |se| - * Learners directly handling missing data |ss| :issue:`13911` |se| + * Making sure meta-estimators are lenient towards missing data by implementing + a common test. * An amputation sample generator to make parts of a dataset go missing :issue:`6284` @@ -101,16 +89,8 @@ the document up to date as we work on these issues. documentation is crowded which makes it hard for beginners to get the big picture. Some work could be done in prioritizing the information. -#. Passing around information that is not (X, y): Sample properties - - * We need to be able to pass sample weights to scorers in cross validation. - * We should have standard/generalised ways of passing sample-wise properties - around in meta-estimators. :issue:`4497` :issue:`7646` - #. Passing around information that is not (X, y): Feature properties - * Feature names or descriptions should ideally be available to fit for, e.g. - . :issue:`6425` :issue:`6424` * Per-feature handling (e.g. "is this a nominal / ordinal / English language text?") should also not need to be provided to estimator constructors, ideally, but should be available as metadata alongside X. :issue:`8480` @@ -124,27 +104,21 @@ the document up to date as we work on these issues. #. Make it easier for external users to write Scikit-learn-compatible components - * More flexible estimator checks that do not select by estimator name - :issue:`6599` :issue:`6715` - * Example of how to develop an estimator or a meta-estimator, :issue:`14582` * More self-sufficient running of scikit-learn-contrib or a similar resource #. Support resampling and sample reduction * Allow subsampling of majority classes (in a pipeline?) :issue:`3855` - * Implement random forests with resampling :issue:`8732` #. Better interfaces for interactive development - * |ss| __repr__ |se| and HTML visualisations of estimators - |ss| :issue:`6323` |se| and :pr:`14180`. - * Include plotting tools, not just as examples. :issue:`9173` + * Improve the HTML visualisations of estimators via the `estimator_html_repr`. + * Include more plotting tools, not just as examples. #. Improved tools for model diagnostics and basic inference - * |ss| alternative feature importances implementations, :issue:`13146` |se| + * work on a unified interface for "feature importance" * better ways to handle validation sets when fitting - * better ways to find thresholds / create decision rules :issue:`8614` #. Better tools for selecting hyperparameters with transductive estimators @@ -175,11 +149,6 @@ the document up to date as we work on these issues. learning is on smaller data than ETL, hence we can maybe adapt to very large scale while supporting only a fraction of the patterns. -#. Support for working with pre-trained models - - * Estimator "freezing". In particular, right now it's impossible to clone a - `CalibratedClassifierCV` with prefit. :issue:`8370`. :issue:`6451` - #. Backwards-compatible de/serialization of some estimators * Currently serialization (with pickle) breaks across versions. While we may @@ -201,15 +170,15 @@ the document up to date as we work on these issues. versions: * Try to load the old pickle, if it works, use the validation set - prediction snapshot to detect that the serialized model still behave + prediction snapshot to detect that the serialized model still behaves the same; - * If joblib.load / pickle.load not work, use the versioned control + * If joblib.load / pickle.load does not work, use the versioned control training script + historical training set to retrain the model and use the validation set prediction snapshot to assert that it is possible to recover the previous predictive performance: if this is not the case there is probably a bug in scikit-learn that needs to be reported. -#. Everything in Scikit-learn should probably conform to our API contract. +#. Everything in scikit-learn should probably conform to our API contract. We are still in the process of making decisions on some of these related issues. @@ -229,43 +198,3 @@ the document up to date as we work on these issues. * Document good practices to detect temporal distribution drift for deployed model and good practices for re-training on fresh data without causing catastrophic predictive performance regressions. - - -Subpackage-specific goals -------------------------- - -:mod:`sklearn.ensemble` - -* |ss| a stacking implementation, :issue:`11047` |se| - -:mod:`sklearn.cluster` - -* kmeans variants for non-Euclidean distances, if we can show these have - benefits beyond hierarchical clustering. - -:mod:`sklearn.model_selection` - -* |ss| multi-metric scoring is slow :issue:`9326` |se| -* perhaps we want to be able to get back more than multiple metrics -* the handling of random states in CV splitters is a poor design and - contradicts the validation of similar parameters in estimators, - :issue:`15177` -* exploit warm-starting and path algorithms so the benefits of `EstimatorCV` - objects can be accessed via `GridSearchCV` and used in Pipelines. - :issue:`1626` -* Cross-validation should be able to be replaced by OOB estimates whenever a - cross-validation iterator is used. -* Redundant computations in pipelines should be avoided (related to point - above) cf `daskml - `_ - -:mod:`sklearn.neighbors` - -* |ss| Ability to substitute a custom/approximate/precomputed nearest neighbors - implementation for ours in all/most contexts that nearest neighbors are used - for learning. :issue:`10463` |se| - -:mod:`sklearn.pipeline` - -* Performance issues with `Pipeline.memory` -* see "Everything in Scikit-learn should conform to our API contract" above diff --git a/doc/scss/api-search.scss b/doc/scss/api-search.scss new file mode 100644 index 0000000000000..51cf15f92c1cb --- /dev/null +++ b/doc/scss/api-search.scss @@ -0,0 +1,111 @@ +/** + * This is the styling for the API index page (`api/index`), in particular for the API + * search table. It involves overriding the style sheet of DataTables which does not + * fit well into the theme, especially in dark theme; see https://datatables.net/ + */ + +.dt-container { + margin-bottom: 2rem; + + // Fix the selection box for entries per page + select.dt-input { + padding: 0 !important; + margin-right: 0.4rem !important; + + > option { + color: var(--pst-color-text-base); + background-color: var(--pst-color-background); + } + } + + // Fix the search box + input.dt-input { + width: 50%; + line-height: normal; + padding: 0.1rem 0.3rem !important; + margin-left: 0.4rem !important; + } + + table.dataTable { + th { + // Avoid table header being too tall + p { + margin-bottom: 0; + } + + // Fix the ascending/descending order buttons in the header + span.dt-column-order { + &::before, + &::after { + color: var(--pst-color-text-base); + line-height: 0.7rem !important; + } + } + } + + td { + // Fix color of text warning no records found + &.dt-empty { + color: var(--pst-color-text-base) !important; + } + } + + // Unset bottom border of the last row + tr:last-child > * { + border-bottom: unset !important; + } + } + + div.dt-paging button.dt-paging-button { + padding: 0 0.5rem; + + &.disabled { + color: var(--pst-color-border) !important; + + // Overwrite the !important color assigned by DataTables because we must keep + // the color of disabled buttons consistent with and without hovering + &:hover { + color: var(--pst-color-border) !important; + } + } + + // Fix colors of paging buttons + &.current, + &:not(.disabled):not(.current):hover { + color: var(--pst-color-on-surface) !important; + border-color: var(--pst-color-surface) !important; + background: var(--pst-color-surface) !important; + } + + // Highlight the border of the current selected paging button + &.current { + border-color: var(--pst-color-text-base) !important; + } + } +} + +// Styling the object description cells in the table +div.sk-apisearch-desc { + p { + margin-bottom: 0; + } + + div.caption > p { + a, + code { + color: var(--pst-color-text-muted); + } + + code { + padding: 0; + font-size: 0.7rem; + font-weight: var(--pst-font-weight-caption); + background-color: transparent; + } + + .sd-badge { + font-size: 0.7rem; + margin-left: 0.3rem; + } + } +} diff --git a/doc/scss/api.scss b/doc/scss/api.scss new file mode 100644 index 0000000000000..d7110def4ac09 --- /dev/null +++ b/doc/scss/api.scss @@ -0,0 +1,52 @@ +/** + * This is the styling for API reference pages, currently under `modules/generated`. + * Note that it should be applied *ONLY* to API reference pages, as the selectors are + * designed based on how `autodoc` and `autosummary` generate the stuff. + */ + +// Make the admonitions more compact +div.versionadded, +div.versionchanged, +div.deprecated { + margin: 1rem auto; + + > p { + margin: 0.3rem auto; + } +} + +// Make docstrings more compact +dd { + p:not(table *) { + margin-bottom: 0.5rem !important; + } + + ul { + margin-bottom: 0.5rem !important; + padding-left: 2rem !important; + } +} + +// The first method is too close the the docstring above +dl.py.method:first-of-type { + margin-top: 2rem; +} + +// https://github.com/pydata/pydata-sphinx-theme/blob/8cf45f835bfdafc5f3821014a18f3b7e0fc2d44b/src/pydata_sphinx_theme/assets/styles/content/_api.scss +dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.glossary):not(.simple) { + margin-bottom: 1.5rem; + + dd { + margin-left: 1.2rem; + } + + // "Parameters", "Returns", etc. in the docstring + dt.field-odd, + dt.field-even { + margin: 0.5rem 0; + + + dd > dl { + margin-bottom: 0.5rem; + } + } +} diff --git a/doc/scss/colors.scss b/doc/scss/colors.scss new file mode 100644 index 0000000000000..bbc6aa6c2a3d6 --- /dev/null +++ b/doc/scss/colors.scss @@ -0,0 +1,51 @@ +/** + * This is the style sheet for customized colors of scikit-learn. + * Tints and shades are generated by https://colorkit.co/color-shades-generator/ + * + * This file is compiled into styles/colors.css by sphinxcontrib.sass, see: + * https://sass-lang.com/guide/ + */ + +:root { + /* scikit-learn cyan */ + --sk-cyan-tint-9: #edf7fd; + --sk-cyan-tint-8: #daeffa; + --sk-cyan-tint-7: #c8e6f8; + --sk-cyan-tint-6: #b5def5; + --sk-cyan-tint-5: #a2d6f2; + --sk-cyan-tint-4: #8fcdef; + --sk-cyan-tint-3: #7ac5ec; + --sk-cyan-tint-2: #64bce9; + --sk-cyan-tint-1: #4bb4e5; + --sk-cyan: #29abe2; + --sk-cyan-shades-1: #2294c4; + --sk-cyan-shades-2: #1c7ea8; + --sk-cyan-shades-3: #15688c; + --sk-cyan-shades-4: #0f5471; + --sk-cyan-shades-5: #094057; + --sk-cyan-shades-6: #052d3e; + --sk-cyan-shades-7: #021b27; + --sk-cyan-shades-8: #010b12; + --sk-cyan-shades-9: #000103; + + /* scikit-learn orange */ + --sk-orange-tint-9: #fff5ec; + --sk-orange-tint-8: #ffead9; + --sk-orange-tint-7: #ffe0c5; + --sk-orange-tint-6: #ffd5b2; + --sk-orange-tint-5: #fecb9e; + --sk-orange-tint-4: #fdc08a; + --sk-orange-tint-3: #fcb575; + --sk-orange-tint-2: #fbaa5e; + --sk-orange-tint-1: #f99f44; + --sk-orange: #f7931e; + --sk-orange-shades-1: #d77f19; + --sk-orange-shades-2: #b76c13; + --sk-orange-shades-3: #99590e; + --sk-orange-shades-4: #7c4709; + --sk-orange-shades-5: #603605; + --sk-orange-shades-6: #452503; + --sk-orange-shades-7: #2c1601; + --sk-orange-shades-8: #150800; + --sk-orange-shades-9: #030100; +} diff --git a/doc/scss/custom.scss b/doc/scss/custom.scss new file mode 100644 index 0000000000000..cac81b03e7ce2 --- /dev/null +++ b/doc/scss/custom.scss @@ -0,0 +1,253 @@ +/** + * This is a general styling sheet. + * It should be used for customizations that affect multiple pages. + * + * This file is compiled into styles/custom.css by sphinxcontrib.sass, see: + * https://sass-lang.com/guide/ + */ + +/* Global */ + +code.literal { + border: 0; +} + +/* Version switcher */ + +.version-switcher__menu.dropdown-menu { + // The version switcher is aligned right so we need to avoid the dropdown menu + // to be cut off by the right boundary + left: unset; + right: 0; + + a.list-group-item.sk-avail-docs-link { + display: flex; + align-items: center; + + &:after { + content: var(--pst-icon-external-link); + font: var(--fa-font-solid); + font-size: 0.75rem; + margin-left: 0.5rem; + } + } +} + +/* Primary sidebar */ + +.bd-sidebar-primary { + width: 22.5%; + min-width: 16rem; + + // The version switcher button in the sidebar is ill-styled + button.version-switcher__button { + margin-bottom: unset; + margin-left: 0.3rem; + font-size: 1rem; + } + + // The section navigation part is to close to the right boundary (originally an even + // larger negative right margin was used) + nav.bd-links { + margin-right: -0.5rem; + } +} + +/* Article content */ + +.bd-article { + h1 { + font-weight: 500; + margin-bottom: 2rem; + } + + h2 { + font-weight: 500; + margin-bottom: 1.5rem; + } + + // Avoid changing the aspect ratio of images; add some padding so that at least + // there is some space between image and background in dark mode + img { + height: unset !important; + padding: 1%; + } + + // Resize table of contents to make the top few levels of headings more visible + li.toctree-l1 { + padding-bottom: 0.5em; + + > a { + font-size: 150%; + font-weight: bold; + } + } + + li.toctree-l2, + li.toctree-l3, + li.toctree-l4 { + margin-left: 15px; + } +} + +/* Dropdowns (sphinx-design) */ + +details.sd-dropdown { + &:hover > summary.sd-summary-title { + > .sd-summary-text > a.headerlink { + visibility: visible; + } + + > .sk-toggle-all { + opacity: 1; + } + } + + > summary.sd-summary-title { + > .sd-summary-text > a.headerlink { + font-size: 1rem; + } + + // See `js/scripts/dropdown.js`: this is styling the "expand/collapse all" button + > .sk-toggle-all { + color: var(--pst-sd-dropdown-color); + margin-right: 0.5rem; + pointer-events: auto !important; + opacity: 0; + } + } +} + +/* Tabs (sphinx-design) */ + +.sd-tab-set { + --tab-caption-width: 0%; // No tab caption by default + margin-top: 1.5rem; + + &::before { + // Set `content` for tab caption + width: var(--tab-caption-width); + display: flex; + align-items: center; + font-weight: bold; + } + + .sd-tab-content { + padding: 0.5rem 0 0 0 !important; + background-color: transparent !important; + border: none !important; + + > p:first-child { + margin-top: 1rem !important; + } + } + + > label.sd-tab-label { + margin: 0 3px; + display: flex; + align-items: center; + justify-content: center; + border-radius: 5px !important; + + &.tab-6 { + width: calc((100% - var(--tab-caption-width)) / 2 - 6px) !important; + } + + &.tab-4 { + width: calc((100% - var(--tab-caption-width)) / 3 - 6px) !important; + } + } + + > input:checked + label.sd-tab-label { + transform: unset; + border: 2px solid var(--pst-color-primary); + } +} + +/* Download/launcher links and top hint (sphinx-gallery) */ + +// https://sphinx-gallery.github.io/stable/advanced.html#using-sphinx-gallery-sidebar-components +.sphx-glr-download-link-note, +.binder-badge, +.lite-badge, +.sphx-glr-download-jupyter, +.sphx-glr-download-python, +.sphx-glr-download-zip { + display: none; +} + +/* scikit-learn buttons */ + +a.btn { + &.sk-btn-orange { + background-color: var(--sk-orange-tint-1); + color: black !important; + + &:hover { + background-color: var(--sk-orange-tint-3); + } + } + + &.sk-btn-cyan { + background-color: var(--sk-cyan-shades-2); + color: white !important; + + &:hover { + background-color: var(--sk-cyan-shades-1); + } + } +} + +/* scikit-learn avatar grid, see build_tools/generate_authors_table.py */ + +div.sk-authors-container { + display: flex; + flex-wrap: wrap; + justify-content: center; + + > div { + width: 6rem; + margin: 0.5rem; + font-size: 0.9rem; + } +} + +/* scikit-learn text-image grid, used in testimonials and sponsors pages */ + +@mixin sk-text-image-grid($img-max-height) { + display: flex; + align-items: center; + flex-wrap: wrap; + + div.text-box, + div.image-box { + width: 50%; + + @media screen and (max-width: 500px) { + width: 100%; + } + } + + div.text-box .annotation { + font-size: 0.9rem; + font-style: italic; + color: var(--pst-color-text-muted); + } + + div.image-box { + text-align: center; + + img { + max-height: $img-max-height; + max-width: 50%; + } + } +} + +div.sk-text-image-grid-small { + @include sk-text-image-grid(60px); +} + +div.sk-text-image-grid-large { + @include sk-text-image-grid(100px); +} diff --git a/doc/scss/index.scss b/doc/scss/index.scss new file mode 100644 index 0000000000000..c3bb8e86b41c6 --- /dev/null +++ b/doc/scss/index.scss @@ -0,0 +1,176 @@ +/** + * Styling sheet for the scikit-learn landing page. This should be loaded only for the + * landing page. + * + * This file is compiled into styles/index.css by sphinxcontrib.sass, see: + * https://sass-lang.com/guide/ + */ + +/* Theme-aware colors for the landing page */ + +html { + &[data-theme="light"] { + --sk-landing-bg-1: var(--sk-cyan-shades-3); + --sk-landing-bg-2: var(--sk-cyan); + --sk-landing-bg-3: var(--sk-orange-tint-8); + --sk-landing-bg-4: var(--sk-orange-tint-3); + } + + &[data-theme="dark"] { + --sk-landing-bg-1: var(--sk-cyan-shades-5); + --sk-landing-bg-2: var(--sk-cyan-shades-2); + --sk-landing-bg-3: var(--sk-orange-tint-4); + --sk-landing-bg-4: var(--sk-orange-tint-1); + } +} + +/* General */ + +div.sk-landing-container { + max-width: 1400px; +} + +/* Top bar */ + +div.sk-landing-top-bar { + background-image: linear-gradient( + 160deg, + var(--sk-landing-bg-1) 0%, + var(--sk-landing-bg-2) 17%, + var(--sk-landing-bg-3) 59%, + var(--sk-landing-bg-4) 100% + ); + + .sk-landing-header, + .sk-landing-subheader { + color: white; + text-shadow: 0px 0px 8px var(--sk-landing-bg-1); + } + + .sk-landing-header { + font-size: 3.2rem; + margin-bottom: 0.5rem; + } + + .sk-landing-subheader { + letter-spacing: 0.17rem; + margin-top: 0; + font-weight: 500; + } + + a.sk-btn-orange { + font-size: 1.1rem; + font-weight: 500; + } + + ul.sk-landing-header-body { + margin-top: auto; + margin-bottom: auto; + font-size: 1.2rem; + font-weight: 500; + color: black; + } +} + +/* Body */ + +div.sk-landing-body { + div.card { + background-color: var(--pst-color-background); + border-color: var(--pst-color-border); + } + + .sk-px-xl-4 { + @media screen and (min-width: 1200px) { + padding-left: 1.3rem !important; + padding-right: 1.3rem !important; + } + } + + .card-body { + p { + margin-bottom: 0.8rem; + color: var(--pst-color-text-base); + } + + .sk-card-title { + font-weight: 700; + margin: 0 0 1rem 0; + } + } + + .sk-card-img-container { + display: flex; + justify-content: center; + align-items: end; + margin-bottom: 1rem; + + img { + max-width: unset; + height: 15rem; + } + } +} + +/* More info */ + +div.sk-landing-more-info { + font-size: 0.96rem; + background-color: var(--pst-color-surface); + + .sk-landing-call-header { + font-weight: 700; + margin-top: 0; + + html[data-theme="light"] & { + color: var(--sk-orange-shades-1); + } + + html[data-theme="dark"] & { + color: var(--sk-orange); + } + } + + ul.sk-landing-call-list > li { + margin-bottom: 0.25rem; + } + + .sk-who-uses-carousel { + min-height: 200px; + + .carousel-item img { + max-height: 100px; + max-width: 50%; + margin: 0.5rem; + } + } + + .sk-more-testimonials { + text-align: right !important; + } +} + +/* Footer */ + +div.sk-landing-footer { + a.sk-footer-funding-link { + text-decoration: none; + + p.sk-footer-funding-text { + color: var(--pst-color-link); + + &:hover { + color: var(--pst-color-secondary); + } + } + + div.sk-footer-funding-logos > img { + max-height: 40px; + max-width: 85px; + margin: 0 8px 8px 8px; + padding: 5px; + border-radius: 3px; + background-color: white; + } + } +} diff --git a/doc/sphinxext/add_toctree_functions.py b/doc/sphinxext/add_toctree_functions.py deleted file mode 100644 index 7cd0e7a29bb28..0000000000000 --- a/doc/sphinxext/add_toctree_functions.py +++ /dev/null @@ -1,154 +0,0 @@ -"""Inspired by https://github.com/pandas-dev/pydata-sphinx-theme - -BSD 3-Clause License - -Copyright (c) 2018, pandas -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -""" - -import docutils - - -def add_toctree_functions(app, pagename, templatename, context, doctree): - """Add functions so Jinja templates can add toctree objects. - - This converts the docutils nodes into a nested dictionary that Jinja can - use in our templating. - """ - from sphinx.environment.adapters.toctree import TocTree - - def get_nav_object(maxdepth=None, collapse=True, numbered=False, **kwargs): - """Return a list of nav links that can be accessed from Jinja. - - Parameters - ---------- - maxdepth: int - How many layers of TocTree will be returned - collapse: bool - Whether to only include sub-pages of the currently-active page, - instead of sub-pages of all top-level pages of the site. - numbered: bool - Whether to add section number to title - kwargs: key/val pairs - Passed to the `TocTree.get_toctree_for` Sphinx method - """ - # The TocTree will contain the full site TocTree including sub-pages. - # "collapse=True" collapses sub-pages of non-active TOC pages. - # maxdepth controls how many TOC levels are returned - toctree = TocTree(app.env).get_toctree_for( - pagename, app.builder, collapse=collapse, maxdepth=maxdepth, - **kwargs) - # If no toctree is defined (AKA a single-page site), skip this - if toctree is None: - return [] - - # toctree has this structure - # - # - # - # - # `list_item`s are the actual TOC links and are the only thing we want - toc_items = [item for child in toctree.children for item in child - if isinstance(item, docutils.nodes.list_item)] - - # Now convert our docutils nodes into dicts that Jinja can use - nav = [docutils_node_to_jinja(child, only_pages=True, - numbered=numbered) - for child in toc_items] - - return nav - - context["get_nav_object"] = get_nav_object - - -def docutils_node_to_jinja(list_item, only_pages=False, numbered=False): - """Convert a docutils node to a structure that can be read by Jinja. - - Parameters - ---------- - list_item : docutils list_item node - A parent item, potentially with children, corresponding to the level - of a TocTree. - only_pages : bool - Only include items for full pages in the output dictionary. Exclude - anchor links (TOC items with a URL that starts with #) - numbered: bool - Whether to add section number to title - - Returns - ------- - nav : dict - The TocTree, converted into a dictionary with key/values that work - within Jinja. - """ - if not list_item.children: - return None - - # We assume this structure of a list item: - # - # - # <-- the thing we want - reference = list_item.children[0].children[0] - title = reference.astext() - url = reference.attributes["refuri"] - active = "current" in list_item.attributes["classes"] - - secnumber = reference.attributes.get("secnumber", None) - if numbered and secnumber is not None: - secnumber = ".".join(str(n) for n in secnumber) - title = f"{secnumber}. {title}" - - # If we've got an anchor link, skip it if we wish - if only_pages and '#' in url: - return None - - # Converting the docutils attributes into jinja-friendly objects - nav = {} - nav["title"] = title - nav["url"] = url - nav["active"] = active - - # Recursively convert children as well - # If there are sub-pages for this list_item, there should be two children: - # a paragraph, and a bullet_list. - nav["children"] = [] - if len(list_item.children) > 1: - # The `.children` of the bullet_list has the nodes of the sub-pages. - subpage_list = list_item.children[1].children - for sub_page in subpage_list: - child_nav = docutils_node_to_jinja(sub_page, only_pages=only_pages, - numbered=numbered) - if child_nav is not None: - nav["children"].append(child_nav) - return nav - - -def setup(app): - app.connect("html-page-context", add_toctree_functions) - - return {'parallel_read_safe': True, 'parallel_write_safe': True} diff --git a/doc/sphinxext/allow_nan_estimators.py b/doc/sphinxext/allow_nan_estimators.py new file mode 100644 index 0000000000000..3b85ce6c87508 --- /dev/null +++ b/doc/sphinxext/allow_nan_estimators.py @@ -0,0 +1,58 @@ +from contextlib import suppress + +from docutils import nodes +from docutils.parsers.rst import Directive + +from sklearn.utils import all_estimators +from sklearn.utils._test_common.instance_generator import _construct_instances +from sklearn.utils._testing import SkipTest + + +class AllowNanEstimators(Directive): + @staticmethod + def make_paragraph_for_estimator_type(estimator_type): + intro = nodes.list_item() + intro += nodes.strong(text="Estimators that allow NaN values for type ") + intro += nodes.literal(text=f"{estimator_type}") + intro += nodes.strong(text=":\n") + exists = False + lst = nodes.bullet_list() + for name, est_class in all_estimators(type_filter=estimator_type): + with suppress(SkipTest): + # Here we generate the text only for one instance. This directive + # should not be used for meta-estimators where tags depend on the + # sub-estimator. + est = next(_construct_instances(est_class)) + + if est.__sklearn_tags__().input_tags.allow_nan: + module_name = ".".join(est_class.__module__.split(".")[:2]) + class_title = f"{est_class.__name__}" + class_url = f"./generated/{module_name}.{class_title}.html" + item = nodes.list_item() + para = nodes.paragraph() + para += nodes.reference( + class_title, text=class_title, internal=False, refuri=class_url + ) + exists = True + item += para + lst += item + intro += lst + return [intro] if exists else None + + def run(self): + lst = nodes.bullet_list() + for i in ["cluster", "regressor", "classifier", "transformer"]: + item = self.make_paragraph_for_estimator_type(i) + if item is not None: + lst += item + return [lst] + + +def setup(app): + app.add_directive("allow_nan_estimators", AllowNanEstimators) + + return { + "version": "0.1", + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/doc/sphinxext/autoshortsummary.py b/doc/sphinxext/autoshortsummary.py new file mode 100644 index 0000000000000..8451f3133d05b --- /dev/null +++ b/doc/sphinxext/autoshortsummary.py @@ -0,0 +1,53 @@ +from sphinx.ext.autodoc import ModuleLevelDocumenter + + +class ShortSummaryDocumenter(ModuleLevelDocumenter): + """An autodocumenter that only renders the short summary of the object.""" + + # Defines the usage: .. autoshortsummary:: {{ object }} + objtype = "shortsummary" + + # Disable content indentation + content_indent = "" + + # Avoid being selected as the default documenter for some objects, because we are + # returning `can_document_member` as True for all objects + priority = -99 + + @classmethod + def can_document_member(cls, member, membername, isattr, parent): + """Allow documenting any object.""" + return True + + def get_object_members(self, want_all): + """Document no members.""" + return (False, []) + + def add_directive_header(self, sig): + """Override default behavior to add no directive header or options.""" + pass + + def add_content(self, more_content): + """Override default behavior to add only the first line of the docstring. + + Modified based on the part of processing docstrings in the original + implementation of this method. + + https://github.com/sphinx-doc/sphinx/blob/faa33a53a389f6f8bc1f6ae97d6015fa92393c4a/sphinx/ext/autodoc/__init__.py#L609-L622 + """ + sourcename = self.get_sourcename() + docstrings = self.get_doc() + + if docstrings is not None: + if not docstrings: + docstrings.append([]) + # Get the first non-empty line of the processed docstring; this could lead + # to unexpected results if the object does not have a short summary line. + short_summary = next( + (s for s in self.process_doc(docstrings) if s), "" + ) + self.add_line(short_summary, sourcename, 0) + + +def setup(app): + app.add_autodocumenter(ShortSummaryDocumenter) diff --git a/doc/sphinxext/custom_references_resolver.py b/doc/sphinxext/custom_references_resolver.py deleted file mode 100644 index 2fd32b7da785e..0000000000000 --- a/doc/sphinxext/custom_references_resolver.py +++ /dev/null @@ -1,122 +0,0 @@ -"""Adapted from -sphinx.transforms.post_transforms.ReferencesResolver.resolve_anyref - -If 'py' is one of the domains and `py:class` is defined, -the Python domain will be processed before the 'std' domain. - -License for Sphinx -================== - -Copyright (c) 2007-2019 by the Sphinx team (see AUTHORS file). -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -* Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -""" -from contextlib import suppress - -from docutils import nodes -from sphinx.transforms.post_transforms import ReferencesResolver - - -class CustomReferencesResolver(ReferencesResolver): - def resolve_anyref(self, refdoc, node, contnode): - """Resolve reference generated by the "any" role.""" - stddomain = self.env.get_domain('std') - target = node['reftarget'] - - # process 'py' domain first for python classes - if "py:class" in node: - with suppress(KeyError): - py_domain = self.env.domains['py'] - py_ref = py_domain.resolve_any_xref( - self.env, refdoc, self.app.builder, target, node, contnode) - if py_ref: - return self.create_node(py_ref[0]) - - # resolve :term: - term_ref = stddomain.resolve_xref(self.env, refdoc, self.app.builder, - 'term', target, node, contnode) - if term_ref: - # replace literal nodes with inline nodes - if not isinstance(term_ref[0], nodes.inline): - inline_node = nodes.inline(rawsource=term_ref[0].rawsource, - classes=term_ref[0].get('classes')) - if term_ref[0]: - inline_node.append(term_ref[0][0]) - term_ref[0] = inline_node - return self.create_node(("std:term", term_ref)) - - # next, do the standard domain - std_ref = stddomain.resolve_any_xref( - self.env, refdoc, self.app.builder, target, node, contnode) - if std_ref: - return self.create_node(std_ref[0]) - - for domain in self.env.domains.values(): - try: - ref = domain.resolve_any_xref( - self.env, refdoc, self.app.builder, target, node, contnode) - if ref: - return self.create_node(ref[0]) - except NotImplementedError: - # the domain doesn't yet support the new interface - # we have to manually collect possible references (SLOW) - for role in domain.roles: - res = domain.resolve_xref(self.env, refdoc, - self.app.builder, role, target, - node, contnode) - if res and isinstance(res[0], nodes.Element): - result = ('%s:%s' % (domain.name, role), res) - return self.create_node(result) - - # no results considered to be - contnode['classes'] = [] - return contnode - - def create_node(self, result): - res_role, newnode = result - # Override "any" class with the actual role type to get the styling - # approximately correct. - res_domain = res_role.split(':')[0] - if (len(newnode) > 0 and isinstance(newnode[0], nodes.Element) - and newnode[0].get('classes')): - newnode[0]['classes'].append(res_domain) - newnode[0]['classes'].append(res_role.replace(':', '-')) - return newnode - - -def setup(app): - if (hasattr(app.registry, "get_post_transforms") - and callable(app.registry.get_post_transforms)): - post_transforms = app.registry.get_post_transforms() - else: - # Support sphinx 1.6.* - post_transforms = app.post_transforms - - for i, transform_class in enumerate(post_transforms): - if transform_class == ReferencesResolver: - post_transforms[i] = CustomReferencesResolver - break - else: - raise RuntimeError("ReferencesResolver not found") diff --git a/doc/sphinxext/doi_role.py b/doc/sphinxext/doi_role.py new file mode 100644 index 0000000000000..9f117b07fa6a3 --- /dev/null +++ b/doc/sphinxext/doi_role.py @@ -0,0 +1,47 @@ +""" +doilinks +~~~~~~~~ +Extension to add links to DOIs. With this extension you can use e.g. +:doi:`10.1016/S0022-2836(05)80360-2` in your documents. This will +create a link to a DOI resolver +(``https://doi.org/10.1016/S0022-2836(05)80360-2``). +The link caption will be the raw DOI. +You can also give an explicit caption, e.g. +:doi:`Basic local alignment search tool <10.1016/S0022-2836(05)80360-2>`. + +:copyright: Copyright 2015 Jon Lund Steffensen. Based on extlinks by + the Sphinx team. +:license: BSD. +""" + +from docutils import nodes, utils +from sphinx.util.nodes import split_explicit_title + + +def reference_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): + text = utils.unescape(text) + has_explicit_title, title, part = split_explicit_title(text) + if typ in ["arXiv", "arxiv"]: + full_url = "https://arxiv.org/abs/" + part + if not has_explicit_title: + title = "arXiv:" + part + pnode = nodes.reference(title, title, internal=False, refuri=full_url) + return [pnode], [] + if typ in ["doi", "DOI"]: + full_url = "https://doi.org/" + part + if not has_explicit_title: + title = "DOI:" + part + pnode = nodes.reference(title, title, internal=False, refuri=full_url) + return [pnode], [] + + +def setup_link_role(app): + app.add_role("arxiv", reference_role, override=True) + app.add_role("arXiv", reference_role, override=True) + app.add_role("doi", reference_role, override=True) + app.add_role("DOI", reference_role, override=True) + + +def setup(app): + app.connect("builder-inited", setup_link_role) + return {"version": "0.1", "parallel_read_safe": True} diff --git a/doc/sphinxext/dropdown_anchors.py b/doc/sphinxext/dropdown_anchors.py new file mode 100644 index 0000000000000..a001dfa11d403 --- /dev/null +++ b/doc/sphinxext/dropdown_anchors.py @@ -0,0 +1,58 @@ +import re + +from docutils import nodes +from sphinx.transforms.post_transforms import SphinxPostTransform +from sphinx_design.dropdown import dropdown_main + + +class DropdownAnchorAdder(SphinxPostTransform): + """Insert anchor links to the sphinx-design dropdowns. + + Some of the dropdowns were originally headers that had automatic anchors, so we + need to make sure that the old anchors still work. See the original implementation + (in JS): https://github.com/scikit-learn/scikit-learn/pull/27409 + + The anchor links are inserted at the end of the node with class "sd-summary-text" + which includes only the title text part of the dropdown (no icon, markers, etc). + """ + + default_priority = 9999 # Apply later than everything else + formats = ["html"] + + def run(self): + """Run the post transformation.""" + # Counter to store the duplicated summary text to add it as a suffix in the + # anchor ID + anchor_id_counters = {} + + for sd_dropdown in self.document.findall(dropdown_main): + # Grab the summary text node + sd_summary_text = sd_dropdown.next_node( + lambda node: "sd-summary-text" in node.get("classes", []) + ) + + # Concatenate the text of relevant nodes as the title text + title_text = "".join(node.astext() for node in sd_summary_text.children) + + # The ID uses the first line, lowercased, with spaces replaced by dashes; + # suffix the anchor ID with a counter if it already exists + anchor_id = re.sub(r"\s+", "-", title_text.strip().split("\n")[0]).lower() + if anchor_id in anchor_id_counters: + anchor_id_counters[anchor_id] += 1 + anchor_id = f"{anchor_id}-{anchor_id_counters[anchor_id]}" + else: + anchor_id_counters[anchor_id] = 1 + sd_dropdown["ids"].append(anchor_id) + + # Create the anchor element and insert after the title text; we do this + # directly with raw HTML + anchor_html = ( + f'#' + ) + anchor_node = nodes.raw("", anchor_html, format="html") + sd_summary_text.append(anchor_node) + + +def setup(app): + app.add_post_transform(DropdownAnchorAdder) diff --git a/doc/sphinxext/github_link.py b/doc/sphinxext/github_link.py index 1592b266a548a..2cd1fbd83af47 100644 --- a/doc/sphinxext/github_link.py +++ b/doc/sphinxext/github_link.py @@ -1,20 +1,20 @@ -from operator import attrgetter import inspect -import subprocess import os +import subprocess import sys from functools import partial +from operator import attrgetter -REVISION_CMD = 'git rev-parse --short HEAD' +REVISION_CMD = "git rev-parse --short HEAD" def _get_git_revision(): try: revision = subprocess.check_output(REVISION_CMD.split()).strip() except (subprocess.CalledProcessError, OSError): - print('Failed to execute git to get revision') + print("Failed to execute git to get revision") return None - return revision.decode('utf-8') + return revision.decode("utf-8") def _linkcode_resolve(domain, info, package, url_fmt, revision): @@ -26,22 +26,22 @@ def _linkcode_resolve(domain, info, package, url_fmt, revision): >>> _linkcode_resolve('py', {'module': 'tty', ... 'fullname': 'setraw'}, ... package='tty', - ... url_fmt='http://hg.python.org/cpython/file/' + ... url_fmt='https://hg.python.org/cpython/file/' ... '{revision}/Lib/{package}/{path}#L{lineno}', ... revision='xxxx') - 'http://hg.python.org/cpython/file/xxxx/Lib/tty/tty.py#L18' + 'https://hg.python.org/cpython/file/xxxx/Lib/tty/tty.py#L18' """ if revision is None: return - if domain not in ('py', 'pyx'): + if domain not in ("py", "pyx"): return - if not info.get('module') or not info.get('fullname'): + if not info.get("module") or not info.get("fullname"): return - class_name = info['fullname'].split('.')[0] - module = __import__(info['module'], fromlist=[class_name]) - obj = attrgetter(info['fullname'])(module) + class_name = info["fullname"].split(".")[0] + module = __import__(info["module"], fromlist=[class_name]) + obj = attrgetter(info["fullname"])(module) # Unwrap the object to get the correct source # file in case that is wrapped by a decorator @@ -59,14 +59,12 @@ def _linkcode_resolve(domain, info, package, url_fmt, revision): if not fn: return - fn = os.path.relpath(fn, - start=os.path.dirname(__import__(package).__file__)) + fn = os.path.relpath(fn, start=os.path.dirname(__import__(package).__file__)) try: lineno = inspect.getsourcelines(obj)[1] except Exception: - lineno = '' - return url_fmt.format(revision=revision, package=package, - path=fn, lineno=lineno) + lineno = "" + return url_fmt.format(revision=revision, package=package, path=fn, lineno=lineno) def make_linkcode_resolve(package, url_fmt): @@ -81,5 +79,6 @@ def make_linkcode_resolve(package, url_fmt): '{path}#L{lineno}') """ revision = _get_git_revision() - return partial(_linkcode_resolve, revision=revision, package=package, - url_fmt=url_fmt) + return partial( + _linkcode_resolve, revision=revision, package=package, url_fmt=url_fmt + ) diff --git a/doc/sphinxext/override_pst_pagetoc.py b/doc/sphinxext/override_pst_pagetoc.py new file mode 100644 index 0000000000000..f5697de8ef155 --- /dev/null +++ b/doc/sphinxext/override_pst_pagetoc.py @@ -0,0 +1,84 @@ +from functools import cache + +from sphinx.util.logging import getLogger + +logger = getLogger(__name__) + + +def override_pst_pagetoc(app, pagename, templatename, context, doctree): + """Overrides the `generate_toc_html` function of pydata-sphinx-theme for API.""" + + @cache + def generate_api_toc_html(kind="html"): + """Generate the in-page toc for an API page. + + This relies on the `generate_toc_html` function added by pydata-sphinx-theme + into the context. We save the original function into `pst_generate_toc_html` + and override `generate_toc_html` with this function for generated API pages. + + The pagetoc of an API page would look like the following: + +
    <-- Unwrap +
  • <-- Unwrap + {{obj}} <-- Decompose + +
      +
    • + ...object +
        <-- Set visible if exists +
      • ...method 1
        • <-- Shorten +
      • ...method 2
        • <-- Shorten + ...more methods <-- Shorten +
      +
      • +
    • ...gallery examples
      • +
    + +
    • <-- Unwrapped +
<-- Unwrapped + """ + soup = context["pst_generate_toc_html"](kind="soup") + + try: + # Unwrap the outermost level + soup.ul.unwrap() + soup.li.unwrap() + soup.a.decompose() + + # Get all toc-h2 level entries, where the first one should be the function + # or class, and the second one, if exists, should be the examples; there + # should be no more than two entries at this level for generated API pages + lis = soup.ul.select("li.toc-h2") + main_li = lis[0] + meth_list = main_li.ul + + if meth_list is not None: + # This is a class API page, we remove the class name from the method + # names to make them better fit into the secondary sidebar; also we + # make the toc-h3 level entries always visible to more easily navigate + # through the methods + meth_list["class"].append("visible") + for meth in meth_list.find_all("li", {"class": "toc-h3"}): + target = meth.a.code.span + target.string = target.string.split(".", 1)[1] + + # This corresponds to the behavior of `generate_toc_html` + return str(soup) if kind == "html" else soup + + except Exception as e: + # Upon any failure we return the original pagetoc + logger.warning( + f"Failed to generate API pagetoc for {pagename}: {e}; falling back" + ) + return context["pst_generate_toc_html"](kind=kind) + + # Override the pydata-sphinx-theme implementation for generate API pages + if pagename.startswith("modules/generated/"): + context["pst_generate_toc_html"] = context["generate_toc_html"] + context["generate_toc_html"] = generate_api_toc_html + + +def setup(app): + # Need to be triggered after `pydata_sphinx_theme.toctree.add_toctree_functions`, + # and since default priority is 500 we set 900 for safety + app.connect("html-page-context", override_pst_pagetoc, priority=900) diff --git a/doc/sphinxext/sphinx_issues.py b/doc/sphinxext/sphinx_issues.py index ba14de62d7a2e..206359a1bd703 100644 --- a/doc/sphinxext/sphinx_issues.py +++ b/doc/sphinxext/sphinx_issues.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """A Sphinx extension for linking to your project's issue tracker. Copyright 2014 Steven Loria @@ -19,6 +18,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ + import re from docutils import nodes, utils @@ -76,7 +76,6 @@ def cve_role(name, rawtext, text, lineno, inliner, options=None, content=None): class IssueRole(object): - EXTERNAL_REPO_REGEX = re.compile(r"^(\w+)/(.+)([#@])([\w]+)$") def __init__( @@ -120,8 +119,9 @@ def make_node(self, name, issue_no, config, options=None): ) else: raise ValueError( - "Neither {} nor issues_github_path " - "is set".format(self.uri_config_option) + "Neither {} nor issues_github_path is set".format( + self.uri_config_option + ) ) issue_text = self.format_text(issue_no) link = nodes.reference(text=issue_text, refuri=ref, **options) diff --git a/doc/supervised_learning.rst b/doc/supervised_learning.rst index e8578fa1e0307..ba24e8ee23c6f 100644 --- a/doc/supervised_learning.rst +++ b/doc/supervised_learning.rst @@ -1,13 +1,7 @@ -.. Places parent toc into the sidebar - -:parenttoc: True - -.. include:: includes/big_toc_css.rst - .. _supervised-learning: Supervised learning ------------------------ +------------------- .. toctree:: :maxdepth: 2 diff --git a/doc/support.rst b/doc/support.rst index f96609bbab937..eb90ff6dd3d94 100644 --- a/doc/support.rst +++ b/doc/support.rst @@ -2,96 +2,131 @@ Support ======= -There are several ways to get in touch with the developers. +There are several channels to connect with scikit-learn developers for assistance, feedback, or contributions. +**Note**: Communications on all channels should respect our `Code of Conduct `_. -.. _mailing_lists: -Mailing List -============ +.. _announcements_and_notification: -- The main mailing list is `scikit-learn - `_. +Mailing Lists +============= -- There is also a commit list `scikit-learn-commits - `_, - where updates to the main repository and test failures get notified. +- **Main Mailing List**: Join the primary discussion + platform for scikit-learn at `scikit-learn Mailing List + `_. +- **Commit Updates**: Stay informed about repository + updates and test failures on the `scikit-learn-commits list + `_. .. _user_questions: -User questions +User Questions ============== -- Some scikit-learn developers support users on StackOverflow using - the `[scikit-learn] `_ +If you have questions, this is our general workflow. + +- **Stack Overflow**: Some scikit-learn developers support users using the + `[scikit-learn] `_ tag. -- For general theoretical or methodological Machine Learning questions - `stack exchange `_ is probably a more - suitable venue. +- **General Machine Learning Queries**: For broader machine learning + discussions, visit `Stack Exchange `_. + +When posting questions: + +- Please use a descriptive question in the title field (e.g. no "Please + help with scikit-learn!" as this is not a question) + +- Provide detailed context, expected results, and actual observations. + +- Include code and data snippets (preferably minimalistic scripts, + up to ~20 lines). + +- Describe your data and preprocessing steps, including sample size, + feature types (categorical or numerical), and the target for supervised + learning tasks (classification type or regression). + +**Note**: Avoid asking user questions on the bug tracker to keep +the focus on development. -In both cases please use a descriptive question in the title field (e.g. -no "Please help with scikit-learn!" as this is not a question) and put -details on what you tried to achieve, what were the expected results and -what you observed instead in the details field. +- `GitHub Discussions `_ + Usage questions such as methodological -Code and data snippets are welcome. Minimalistic (up to ~20 lines long) -reproduction script very helpful. +- `Stack Overflow `_ + Programming/user questions with `[scikit-learn]` tag -Please describe the nature of your data and how you preprocessed it: -what is the number of samples, what is the number and type of features -(i.d. categorical or numerical) and for supervised learning tasks, -what target are your trying to predict: binary, multiclass (1 out of -``n_classes``) or multilabel (``k`` out of ``n_classes``) classification -or continuous variable regression. +- `GitHub Bug Tracker `_ + Bug reports - Please do not ask usage questions on the issue tracker. -User questions should **not be asked on the bug tracker**, as it crowds -the list of issues and makes the development of the project harder. +- `Discord Server `_ + Current pull requests - Post any specific PR-related questions on your PR, + and you can share a link to your PR on this server. .. _bug_tracker: -Bug tracker +Bug Tracker =========== -If you think you've encountered a bug, please report it to the issue tracker: +Encountered a bug? Report it on our `issue tracker +`_ -https://github.com/scikit-learn/scikit-learn/issues +Include in your report: -Don't forget to include: +- Steps or scripts to reproduce the bug. - - steps (or better script) to reproduce, +- Expected and observed outcomes. - - expected outcome, +- Python or gdb tracebacks, if applicable. - - observed outcome or Python (or gdb) tracebacks +- The ideal bug report contains a :ref:`short reproducible code snippet + `, this way anyone can try to reproduce the bug easily. -To help developers fix your bug faster, please link to a https://gist.github.com -holding a standalone minimalistic python script that reproduces your bug and -optionally a minimalistic subsample of your dataset (for instance, exported -as CSV files using ``numpy.savetxt``). +- If your snippet is longer than around 50 lines, please link to a + `gist `_ or a github repo. -Note: Gists are Git cloneable repositories and thus you can use Git to -push datafiles to them. +**Tip**: Gists are Git repositories; you can push data files to them using Git. +Paid support +============ + +The following companies (listed in alphabetical order) offer support services +related to scikit-learn and have a proven track record of employing long-term +maintainers of scikit-learn and related open source projects: + +- `:probabl. `__ +- `Quansight `__ + +.. _social_media: + +Social Media +============ + +scikit-learn has presence on various social media platforms to share +updates with the community. The platforms are not monitored for user +questions. .. _gitter: Gitter -=== - -Some developers like to hang out on scikit-learn Gitter room: -https://gitter.im/scikit-learn/scikit-learn. +====== +**Note**: The scikit-learn Gitter room is no longer an active community. +For live discussions and support, please refer to the other channels +mentioned in this document. .. _documentation_resources: -Documentation resources +Documentation Resources ======================= -This documentation is relative to |release|. Documentation for -other versions can be found `here -`__. +This documentation is for |release|. Documentation for other versions can be found `here +`__, including zip archives which can be +downloaded for offline access. -Printable pdf documentation for old versions can be found `here -`_. +We no longer provide a PDF version of the documentation, but you can still generate it +locally by following the :ref:`building documentation instructions `. +The most recent version with a PDF documentation is quite old, 0.23.2 (released +in August 2020), but the PDF is available `here +`__. diff --git a/doc/templates/base.rst b/doc/templates/base.rst new file mode 100644 index 0000000000000..ee86bd8a18dbe --- /dev/null +++ b/doc/templates/base.rst @@ -0,0 +1,36 @@ +{{ objname | escape | underline(line="=") }} + +{% if objtype == "module" -%} + +.. automodule:: {{ fullname }} + +{%- elif objtype == "function" -%} + +.. currentmodule:: {{ module }} + +.. autofunction:: {{ objname }} + +.. minigallery:: {{ module }}.{{ objname }} + :add-heading: Gallery examples + :heading-level: - + +{%- elif objtype == "class" -%} + +.. currentmodule:: {{ module }} + +.. autoclass:: {{ objname }} + :members: + :inherited-members: + :special-members: __call__ + +.. minigallery:: {{ module }}.{{ objname }} {% for meth in methods %}{{ module }}.{{ objname }}.{{ meth }} {% endfor %} + :add-heading: Gallery examples + :heading-level: - + +{%- else -%} + +.. currentmodule:: {{ module }} + +.. auto{{ objtype }}:: {{ objname }} + +{%- endif -%} diff --git a/doc/templates/class.rst b/doc/templates/class.rst deleted file mode 100644 index 79ff2cf807794..0000000000000 --- a/doc/templates/class.rst +++ /dev/null @@ -1,12 +0,0 @@ -:mod:`{{module}}`.{{objname}} -{{ underline }}============== - -.. currentmodule:: {{ module }} - -.. autoclass:: {{ objname }} - -.. include:: {{module}}.{{objname}}.examples - -.. raw:: html - -
diff --git a/doc/templates/class_with_call.rst b/doc/templates/class_with_call.rst deleted file mode 100644 index f98b7dbbf6578..0000000000000 --- a/doc/templates/class_with_call.rst +++ /dev/null @@ -1,16 +0,0 @@ -:mod:`{{module}}`.{{objname}} -{{ underline }}=============== - -.. currentmodule:: {{ module }} - -.. autoclass:: {{ objname }} - - {% block methods %} - .. automethod:: __call__ - {% endblock %} - -.. include:: {{module}}.{{objname}}.examples - -.. raw:: html - -
diff --git a/doc/templates/deprecated_class.rst b/doc/templates/deprecated_class.rst deleted file mode 100644 index 857e2c28ce1da..0000000000000 --- a/doc/templates/deprecated_class.rst +++ /dev/null @@ -1,23 +0,0 @@ -:mod:`{{module}}`.{{objname}} -{{ underline }}============== - -.. meta:: - :robots: noindex - -.. warning:: - **DEPRECATED** - - -.. currentmodule:: {{ module }} - -.. autoclass:: {{ objname }} - - {% block methods %} - .. automethod:: __init__ - {% endblock %} - -.. include:: {{module}}.{{objname}}.examples - -.. raw:: html - -
diff --git a/doc/templates/deprecated_class_with_call.rst b/doc/templates/deprecated_class_with_call.rst deleted file mode 100644 index a04efcb80be07..0000000000000 --- a/doc/templates/deprecated_class_with_call.rst +++ /dev/null @@ -1,24 +0,0 @@ -:mod:`{{module}}`.{{objname}} -{{ underline }}=============== - -.. meta:: - :robots: noindex - -.. warning:: - **DEPRECATED** - - -.. currentmodule:: {{ module }} - -.. autoclass:: {{ objname }} - - {% block methods %} - .. automethod:: __init__ - .. automethod:: __call__ - {% endblock %} - -.. include:: {{module}}.{{objname}}.examples - -.. raw:: html - -
diff --git a/doc/templates/deprecated_class_without_init.rst b/doc/templates/deprecated_class_without_init.rst deleted file mode 100644 index c019992493610..0000000000000 --- a/doc/templates/deprecated_class_without_init.rst +++ /dev/null @@ -1,19 +0,0 @@ -:mod:`{{module}}`.{{objname}} -{{ underline }}============== - -.. meta:: - :robots: noindex - -.. warning:: - **DEPRECATED** - - -.. currentmodule:: {{ module }} - -.. autoclass:: {{ objname }} - -.. include:: {{module}}.{{objname}}.examples - -.. raw:: html - -
diff --git a/doc/templates/deprecated_function.rst b/doc/templates/deprecated_function.rst deleted file mode 100644 index 6d13ac6aca2de..0000000000000 --- a/doc/templates/deprecated_function.rst +++ /dev/null @@ -1,19 +0,0 @@ -:mod:`{{module}}`.{{objname}} -{{ underline }}==================== - -.. meta:: - :robots: noindex - -.. warning:: - **DEPRECATED** - - -.. currentmodule:: {{ module }} - -.. autofunction:: {{ objname }} - -.. include:: {{module}}.{{objname}}.examples - -.. raw:: html - -
diff --git a/doc/templates/function.rst b/doc/templates/function.rst deleted file mode 100644 index f4b11eda770e4..0000000000000 --- a/doc/templates/function.rst +++ /dev/null @@ -1,12 +0,0 @@ -:mod:`{{module}}`.{{objname}} -{{ underline }}==================== - -.. currentmodule:: {{ module }} - -.. autofunction:: {{ objname }} - -.. include:: {{module}}.{{objname}}.examples - -.. raw:: html - -
diff --git a/doc/templates/generate_deprecated.sh b/doc/templates/generate_deprecated.sh deleted file mode 100755 index a7301fb5dc419..0000000000000 --- a/doc/templates/generate_deprecated.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -for f in [^d]*; do (head -n2 < $f; echo ' -.. meta:: - :robots: noindex - -.. warning:: - **DEPRECATED** -'; tail -n+3 $f) > deprecated_$f; done diff --git a/doc/templates/index.html b/doc/templates/index.html index d333530ef8376..ff71b52ebd59c 100644 --- a/doc/templates/index.html +++ b/doc/templates/index.html @@ -1,15 +1,27 @@ {% extends "layout.html" %} {% set title = 'scikit-learn: machine learning in Python' %} -{% block content %} -
+ +{% if is_devrelease|tobool %} + {%- set contributing_link = pathto("developers/contributing") %} + {%- set contributing_attrs = "" %} +{%- else %} + {%- set contributing_link = "https://scikit-learn.org/dev/developers/contributing.html" %} + {%- set contributing_attrs = 'target="_blank" rel="noopener noreferrer"' %} +{%- endif %} + +{%- import "static/webpack-macros.html" as _webpack with context %} + +{% block docs_navbar %} +{{ super() }} + +
-

scikit-learn

-

Machine Learning in Python

- Getting Started - Release Highlights for {{ release_highlights_version }} - GitHub +

scikit-learn

+

Machine Learning in Python

+ Getting Started + Release Highlights for {{ release_highlights_version }}
    @@ -23,231 +35,279 @@

    Machine Learning in

-
+{% endblock docs_navbar %} + +{% block docs_main %} + +
+
-
+
-

Classification

 -

Identifying which category an object belongs to.

-

Applications: Spam detection, image recognition.
- Algorithms: - SVM, - nearest neighbors, - random forest, - and more...

+

+ Classification +

+

Identifying which category an object belongs to.

+

+ Applications: Spam detection, image recognition.
+ Algorithms: + Gradient boosting, + nearest neighbors, + random forest, + logistic regression, + and more... +

-
- - Classifier comparison +
+ + Classifier comparison
- Examples + Examples
+
-
+
-

Regression

 -

Predicting a continuous-valued attribute associated with an object.

-

Applications: Drug response, Stock prices.
- Algorithms: - SVR, - nearest neighbors, - random forest, - and more...

+

+ Regression +

+

Predicting a continuous-valued attribute associated with an object.

+

+ Applications: Drug response, stock prices.
+ Algorithms: + Gradient boosting, + nearest neighbors, + random forest, + ridge, + and more... +

-
- - Decision Tree Regression with AdaBoost +
+ + Decision Tree Regression with HGBT
- Examples + Examples
+
-
+
-

Clustering

 -

Automatic grouping of similar objects into sets.

-

Applications: Customer segmentation, Grouping experiment outcomes
- Algorithms: - k-Means, - spectral clustering, - mean-shift, - and more...

+

+ Clustering +

+

Automatic grouping of similar objects into sets.

+

+ Applications: Customer segmentation, grouping experiment outcomes.
+ Algorithms: + k-Means, + HDBSCAN, + hierarchical clustering, + and more... +

-
- - A demo of K-Means clustering on the handwritten digits data +
+ + A demo of K-Means clustering on the handwritten digits data
- Examples + Examples
+
-
+
-

Dimensionality reduction

 -

Reducing the number of random variables to consider.

-

Applications: Visualization, Increased efficiency
- Algorithms: - k-Means, - feature selection, - non-negative matrix factorization, - and more...

+

+ Dimensionality reduction +

+

Reducing the number of random variables to consider.

+

+ Applications: Visualization, increased efficiency.
+ Algorithms: + PCA, + feature selection, + non-negative matrix factorization, + and more... +

-
- - PCA example with Iris Data-set +
+ + PCA example with Iris Data-set
- Examples + Examples
+
-
+
-

Model selection

 -

Comparing, validating and choosing parameters and models.

-

Applications: Improved accuracy via parameter tuning
- Algorithms: - grid search, - cross validation, - metrics, - and more...

+

+ Model selection +

+

Comparing, validating and choosing parameters and models.

+

+ Applications: Improved accuracy via parameter tuning.
+ Algorithms: + Grid search, + cross validation, + metrics, + and more... +

-
- - Demonstration of multi-metric evaluation on cross_val_score and GridSearchCV +
+ + Demonstration of multi-metric evaluation on cross_val_score and GridSearchCV
- Examples + Examples
+
-
+
-

Preprocessing

 -

Feature extraction and normalization.

-

Applications: Transforming input data such as text for use with machine learning algorithms.
- Algorithms: - preprocessing, - feature extraction, - and more...

+

+ Preprocessing +

+

Feature extraction and normalization.

+

+ Applications: Transforming input data such as text for use with machine learning algorithms.
+ Algorithms: + Preprocessing, + feature extraction, + and more... +

-
- - Demonstrating the different strategies of KBinsDiscretizer +
+ + Demonstrating the different strategies of KBinsDiscretizer
- Examples + Examples
-
-
+{% endblock docs_main %} + +{% block footer %} + +
+
+

News

    -
  • On-going development: - What's new (Changelog) -
  • December 2020. scikit-learn 0.24.0 is available for download (Changelog). -
    • -
  • August 2020. scikit-learn 0.23.2 is available for download (Changelog). -
    • -
  • May 2020. scikit-learn 0.23.1 is available for download (Changelog). -
    • -
  • May 2020. scikit-learn 0.23.0 is available for download (Changelog). -
    • -
  • Scikit-learn from 0.23 requires Python 3.6 or greater. -
    • -
  • March 2020. scikit-learn 0.22.2 is available for download (Changelog). -
  • January 2020. scikit-learn 0.22.1 is available for download (Changelog). -
  • December 2019. scikit-learn 0.22 is available for download (Changelog and Release Highlights). -
    • -
  • Scikit-learn from 0.21 requires Python 3.5 or greater. -
    • -
  • July 2019. scikit-learn 0.21.3 (Changelog) and 0.20.4 (Changelog) are available for download. -
    • -
  • May 2019. scikit-learn 0.21.0 to 0.21.2 are available for download (Changelog). -
    • +
  • On-going development: scikit-learn 1.8 (Changelog).
    • +
  • June 2025. scikit-learn 1.7.0 is available for download (Changelog).
    • +
  • January 2025. scikit-learn 1.6.1 is available for download (Changelog).
    • +
  • December 2024. scikit-learn 1.6.0 is available for download (Changelog).
    • +
  • September 2024. scikit-learn 1.5.2 is available for download (Changelog).
    • +
  • July 2024. scikit-learn 1.5.1 is available for download (Changelog).
    • +
  • May 2024. scikit-learn 1.5.0 is available for download (Changelog).
    • +
  • All releases: What's new (Changelog).
+

Community

- - Help us, donate! - Cite us! +

+ Help us, donate! + Cite us! +

+

Who uses scikit-learn?

-
-
+ +
- -
-

- scikit-learn development and maintenance are financially supported by -

- - - - - - - - - - - + +
+

+ scikit-learn development and maintenance are financially supported by +

+
+ + + + + + + + + + +
- +
+
-{% endblock %} + +{% endblock footer %} + +{%- block scripts_end %} +{{ _webpack.body_post() }} +{%- endblock scripts_end %} diff --git a/doc/templates/documentation.html b/doc/templates/redirects.html similarity index 52% rename from doc/templates/documentation.html rename to doc/templates/redirects.html index e255877fc89fb..6cca53544d5e7 100644 --- a/doc/templates/documentation.html +++ b/doc/templates/redirects.html @@ -1,14 +1,15 @@ +{% set redirect = pathto(redirects[pagename]) %} - + - + scikit-learn: machine learning in Python -

You will be automatically redirected to the main page.

+

You will be automatically redirected to the new location of this page.

diff --git a/doc/testimonials/README.txt b/doc/testimonials/README.txt index 1ba1f31bd367f..d12a3f3d2a1b9 100644 --- a/doc/testimonials/README.txt +++ b/doc/testimonials/README.txt @@ -5,4 +5,3 @@ https://docs.google.com/spreadsheet/ccc?key=0AhGnAxuBDhjmdDYwNzlZVE5SMkFsMjNBbGl To obtain access to this file, send an email to: nelle dot varoquaux at gmail dot com - diff --git a/doc/testimonials/testimonials.rst b/doc/testimonials/testimonials.rst index 88997285e347e..3c8c15b2e25ee 100644 --- a/doc/testimonials/testimonials.rst +++ b/doc/testimonials/testimonials.rst @@ -1,1151 +1,752 @@ -.. _testimonials: - -================================================================================ -Who is using scikit-learn? -================================================================================ +:orphan: -.. raw:: html - -
+.. title:: Testimonials +.. _testimonials: -.. to add a testimonials, just XXX +========================== +Who is using scikit-learn? +========================== `J.P.Morgan `_ ------------------------------------------- - -.. raw:: html - -
-
- -Scikit-learn is an indispensable part of the Python machine learning -toolkit at JPMorgan. It is very widely used across all parts of the bank -for classification, predictive analytics, and very many other machine -learning tasks. Its straightforward API, its breadth of algorithms, and -the quality of its documentation combine to make scikit-learn -simultaneously very approachable and very powerful. +---------------------------------------- -.. raw:: html +.. div:: sk-text-image-grid-large - + .. div:: text-box -Stephen Simmons, VP, Athena Research, JPMorgan + Scikit-learn is an indispensable part of the Python machine learning + toolkit at JPMorgan. It is very widely used across all parts of the bank + for classification, predictive analytics, and very many other machine + learning tasks. Its straightforward API, its breadth of algorithms, and + the quality of its documentation combine to make scikit-learn + simultaneously very approachable and very powerful. -.. raw:: html + .. rst-class:: annotation - -
-
+ Stephen Simmons, VP, Athena Research, JPMorgan -.. image:: images/jpmorgan.png - :width: 120pt - :align: center - :target: https://www.jpmorgan.com + .. div:: image-box -.. raw:: html + .. image:: images/jpmorgan.png + :target: https://www.jpmorgan.com -
-
`Spotify `_ ------------------------------------ -.. raw:: html - -
-
- -Scikit-learn provides a toolbox with solid implementations of a bunch of -state-of-the-art models and makes it easy to plug them into existing -applications. We've been using it quite a lot for music recommendations at -Spotify and I think it's the most well-designed ML package I've seen so -far. - -.. raw:: html +.. div:: sk-text-image-grid-large - + .. div:: text-box -Erik Bernhardsson, Engineering Manager Music Discovery & Machine Learning, Spotify + Scikit-learn provides a toolbox with solid implementations of a bunch of + state-of-the-art models and makes it easy to plug them into existing + applications. We've been using it quite a lot for music recommendations at + Spotify and I think it's the most well-designed ML package I've seen so far. -.. raw:: html + .. rst-class:: annotation - -
-
+ Erik Bernhardsson, Engineering Manager Music Discovery & Machine Learning, Spotify -.. image:: images/spotify.png - :width: 120pt - :align: center - :target: https://www.spotify.com + .. div:: image-box -.. raw:: html + .. image:: images/spotify.png + :target: https://www.spotify.com -
-
`Inria `_ -------------------------------- -.. raw:: html - -
-
- -.. title Scikit-learn for efficient and easier machine learning research -.. Author: GaÃĢl Varoquaux - - -At INRIA, we use scikit-learn to support leading-edge basic research in many -teams: `Parietal `_ for neuroimaging, `Lear -`_ for computer vision, `Visages -`_ for medical image analysis, `Privatics -`_ for security. The project is a fantastic -tool to address difficult applications of machine learning in an academic -environment as it is performant and versatile, but all easy-to-use and well -documented, which makes it well suited to grad students. +.. div:: sk-text-image-grid-large + .. div:: text-box -.. raw:: html + At INRIA, we use scikit-learn to support leading-edge basic research in many + teams: `Parietal `_ for neuroimaging, `Lear + `_ for computer vision, `Visages + `_ for medical image analysis, `Privatics + `_ for security. The project is a fantastic + tool to address difficult applications of machine learning in an academic + environment as it is performant and versatile, but all easy-to-use and well + documented, which makes it well suited to grad students. - + .. rst-class:: annotation -GaÃĢl Varoquaux, research at Parietal + GaÃĢl Varoquaux, research at Parietal -.. raw:: html + .. div:: image-box - -
-
- -.. image:: images/inria.png - :width: 120pt - :align: center - :target: https://www.inria.fr/ - -.. raw:: html - -
-
+ .. image:: images/inria.png + :target: https://www.inria.fr/ `betaworks `_ ------------------------------------ -.. raw:: html - -
-
- -Betaworks is a NYC-based startup studio that builds new products, grows -companies, and invests in others. Over the past 8 years we’ve launched a -handful of social data analytics-driven services, such as Bitly, Chartbeat, -digg and Scale Model. Consistently the betaworks data science team uses -Scikit-learn for a variety of tasks. From exploratory analysis, to product -development, it is an essential part of our toolkit. Recent uses are included -in `digg’s new video recommender system -`_, -and Poncho’s `dynamic heuristic subspace clustering -`_. +.. div:: sk-text-image-grid-large -.. raw:: html + .. div:: text-box - + Betaworks is a NYC-based startup studio that builds new products, grows + companies, and invests in others. Over the past 8 years we've launched a + handful of social data analytics-driven services, such as Bitly, Chartbeat, + digg and Scale Model. Consistently the betaworks data science team uses + Scikit-learn for a variety of tasks. From exploratory analysis, to product + development, it is an essential part of our toolkit. Recent uses are included + in `digg's new video recommender system + `_, + and Poncho's `dynamic heuristic subspace clustering + `_. -Gilad Lotan, Chief Data Scientist + .. rst-class:: annotation -.. raw:: html + Gilad Lotan, Chief Data Scientist - -
-
+ .. div:: image-box -.. image:: images/betaworks.png - :width: 120pt - :align: center - :target: https://betaworks.com - -.. raw:: html - -
-
+ .. image:: images/betaworks.png + :target: https://betaworks.com `Hugging Face `_ ---------------------------------------- -.. raw:: html - -
-
- -At Hugging Face we're using NLP and probabilistic models to generate -conversational Artificial intelligences that are fun to chat with. Despite using -deep neural nets for `a few `_ -of our `NLP tasks `_, scikit-learn is still the bread-and-butter of -our daily machine learning routine. The ease of use and predictability of the -interface, as well as the straightforward mathematical explanations that are -here when you need them, is the killer feature. We use a variety of scikit-learn -models in production and they are also operationally very pleasant to work with. +.. div:: sk-text-image-grid-large -.. raw:: html + .. div:: text-box - + At Hugging Face we're using NLP and probabilistic models to generate + conversational Artificial intelligences that are fun to chat with. Despite using + deep neural nets for `a few `_ + of our `NLP tasks `_, scikit-learn is still the + bread-and-butter of our daily machine learning routine. The ease of use and + predictability of the interface, as well as the straightforward mathematical + explanations that are here when you need them, is the killer feature. We use a + variety of scikit-learn models in production and they are also operationally very + pleasant to work with. -Julien Chaumond, Chief Technology Officer + .. rst-class:: annotation -.. raw:: html + Julien Chaumond, Chief Technology Officer - -
-
+ .. div:: image-box -.. image:: images/huggingface.png - :width: 120pt - :align: center - :target: https://huggingface.co - -.. raw:: html - -
-
+ .. image:: images/huggingface.png + :target: https://huggingface.co `Evernote `_ ---------------------------------- -.. raw:: html - -
-
- -Building a classifier is typically an iterative process of exploring -the data, selecting the features (the attributes of the data believed -to be predictive in some way), training the models, and finally -evaluating them. For many of these tasks, we relied on the excellent -scikit-learn package for Python. +.. div:: sk-text-image-grid-large -`Read more `_ + .. div:: text-box -.. raw:: html + Building a classifier is typically an iterative process of exploring + the data, selecting the features (the attributes of the data believed + to be predictive in some way), training the models, and finally + evaluating them. For many of these tasks, we relied on the excellent + scikit-learn package for Python. - + `Read more `_ -Mark Ayzenshtat, VP, Augmented Intelligence + .. rst-class:: annotation -.. raw:: html + Mark Ayzenshtat, VP, Augmented Intelligence - -
-
+ .. div:: image-box -.. image:: images/evernote.png - :width: 120pt - :align: center - :target: https://evernote.com + .. image:: images/evernote.png + :target: https://evernote.com -.. raw:: html - -
-
`TÊlÊcom ParisTech `_ -------------------------------------------------------- -.. raw:: html - -
-
- -At Telecom ParisTech, scikit-learn is used for hands-on sessions and home -assignments in introductory and advanced machine learning courses. The classes -are for undergrads and masters students. The great benefit of scikit-learn is -its fast learning curve that allows students to quickly start working on -interesting and motivating problems. - -.. raw:: html +.. div:: sk-text-image-grid-large - + .. div:: text-box -Alexandre Gramfort, Assistant Professor + At Telecom ParisTech, scikit-learn is used for hands-on sessions and home + assignments in introductory and advanced machine learning courses. The classes + are for undergrads and masters students. The great benefit of scikit-learn is + its fast learning curve that allows students to quickly start working on + interesting and motivating problems. -.. raw:: html + .. rst-class:: annotation - -
-
+ Alexandre Gramfort, Assistant Professor -.. image:: images/telecomparistech.jpg - :width: 120pt - :align: center - :target: https://www.telecom-paristech.fr/ + .. div:: image-box -.. raw:: html - -
-
+ .. image:: images/telecomparistech.jpg + :target: https://www.telecom-paristech.fr/ `Booking.com `_ ------------------------------------------ -.. raw:: html - -
-
- -At Booking.com, we use machine learning algorithms for many different -applications, such as recommending hotels and destinations to our customers, -detecting fraudulent reservations, or scheduling our customer service agents. -Scikit-learn is one of the tools we use when implementing standard algorithms -for prediction tasks. Its API and documentations are excellent and make it easy -to use. The scikit-learn developers do a great job of incorporating state of -the art implementations and new algorithms into the package. Thus, scikit-learn -provides convenient access to a wide spectrum of algorithms, and allows us to -readily find the right tool for the right job. - +---------------------------------------- -.. raw:: html +.. div:: sk-text-image-grid-large - + .. div:: text-box -Melanie Mueller, Data Scientist + At Booking.com, we use machine learning algorithms for many different + applications, such as recommending hotels and destinations to our customers, + detecting fraudulent reservations, or scheduling our customer service agents. + Scikit-learn is one of the tools we use when implementing standard algorithms + for prediction tasks. Its API and documentations are excellent and make it easy + to use. The scikit-learn developers do a great job of incorporating state of + the art implementations and new algorithms into the package. Thus, scikit-learn + provides convenient access to a wide spectrum of algorithms, and allows us to + readily find the right tool for the right job. -.. raw:: html + .. rst-class:: annotation - -
-
+ Melanie Mueller, Data Scientist -.. image:: images/booking.png - :width: 120pt - :align: center - :target: https://www.booking.com + .. div:: image-box -.. raw:: html + .. image:: images/booking.png + :target: https://www.booking.com -
-
`AWeber `_ ------------------------------------------- - -.. raw:: html - -
-
- -The scikit-learn toolkit is indispensable for the Data Analysis and Management -team at AWeber. It allows us to do AWesome stuff we would not otherwise have -the time or resources to accomplish. The documentation is excellent, allowing -new engineers to quickly evaluate and apply many different algorithms to our -data. The text feature extraction utilities are useful when working with the -large volume of email content we have at AWeber. The RandomizedPCA -implementation, along with Pipelining and FeatureUnions, allows us to develop -complex machine learning algorithms efficiently and reliably. +----------------------------------- -Anyone interested in learning more about how AWeber deploys scikit-learn in a -production environment should check out talks from PyData Boston by AWeber's -Michael Becker available at https://github.com/mdbecker/pydata_2013 +.. div:: sk-text-image-grid-large -.. raw:: html + .. div:: text-box - + The scikit-learn toolkit is indispensable for the Data Analysis and Management + team at AWeber. It allows us to do AWesome stuff we would not otherwise have + the time or resources to accomplish. The documentation is excellent, allowing + new engineers to quickly evaluate and apply many different algorithms to our + data. The text feature extraction utilities are useful when working with the + large volume of email content we have at AWeber. The RandomizedPCA + implementation, along with Pipelining and FeatureUnions, allows us to develop + complex machine learning algorithms efficiently and reliably. -Michael Becker, Software Engineer, Data Analysis and Management Ninjas + Anyone interested in learning more about how AWeber deploys scikit-learn in a + production environment should check out talks from PyData Boston by AWeber's + Michael Becker available at https://github.com/mdbecker/pydata_2013. -.. raw:: html + .. rst-class:: annotation - -
-
+ Michael Becker, Software Engineer, Data Analysis and Management Ninjas -.. image:: images/aweber.png - :width: 120pt - :align: center - :target: https://www.aweber.com/ + .. div:: image-box -.. raw:: html + .. image:: images/aweber.png + :target: https://www.aweber.com -
-
`Yhat `_ ------------------------------------------- - -.. raw:: html +------------------------------ -
-
+.. div:: sk-text-image-grid-large -The combination of consistent APIs, thorough documentation, and top notch -implementation make scikit-learn our favorite machine learning package in -Python. scikit-learn makes doing advanced analysis in Python accessible to -anyone. At Yhat, we make it easy to integrate these models into your production -applications. Thus eliminating the unnecessary dev time encountered -productionizing analytical work. + .. div:: text-box + The combination of consistent APIs, thorough documentation, and top notch + implementation make scikit-learn our favorite machine learning package in + Python. scikit-learn makes doing advanced analysis in Python accessible to + anyone. At Yhat, we make it easy to integrate these models into your production + applications. Thus eliminating the unnecessary dev time encountered + productionizing analytical work. -.. raw:: html + .. rst-class:: annotation - + Greg Lamp, Co-founder -Greg Lamp, Co-founder Yhat + .. div:: image-box -.. raw:: html + .. image:: images/yhat.png + :target: https://www.yhat.com - -
-
- -.. image:: images/yhat.png - :width: 120pt - :align: center - :target: https://www.yhat.com - -.. raw:: html - -
-
`Rangespan `_ ----------------------------------------- +--------------------------------------- -.. raw:: html +.. div:: sk-text-image-grid-large -
-
+ .. div:: text-box -The Python scikit-learn toolkit is a core tool in the data science -group at Rangespan. Its large collection of well documented models and -algorithms allow our team of data scientists to prototype fast and -quickly iterate to find the right solution to our learning problems. -We find that scikit-learn is not only the right tool for prototyping, -but its careful and well tested implementation give us the confidence -to run scikit-learn models in production. + The Python scikit-learn toolkit is a core tool in the data science + group at Rangespan. Its large collection of well documented models and + algorithms allow our team of data scientists to prototype fast and + quickly iterate to find the right solution to our learning problems. + We find that scikit-learn is not only the right tool for prototyping, + but its careful and well tested implementation give us the confidence + to run scikit-learn models in production. -.. raw:: html + .. rst-class:: annotation - + Jurgen Van Gael, Data Science Director -Jurgen Van Gael, Data Science Director at Rangespan Ltd + .. div:: image-box -.. raw:: html + .. image:: images/rangespan.png + :target: http://www.rangespan.com - -
-
- -.. image:: images/rangespan.png - :width: 120pt - :align: center - :target: http://www.rangespan.com - -.. raw:: html - -
-
`Birchbox `_ ------------------------------------------- - -.. raw:: html - -
-
- -At Birchbox, we face a range of machine learning problems typical to -E-commerce: product recommendation, user clustering, inventory prediction, -trends detection, etc. Scikit-learn lets us experiment with many models, -especially in the exploration phase of a new project: the data can be passed -around in a consistent way; models are easy to save and reuse; updates keep us -informed of new developments from the pattern discovery research community. -Scikit-learn is an important tool for our team, built the right way in the -right language. - -.. raw:: html +-------------------------------------- - +.. div:: sk-text-image-grid-large -Thierry Bertin-Mahieux, Birchbox, Data Scientist + .. div:: text-box -.. raw:: html + At Birchbox, we face a range of machine learning problems typical to + E-commerce: product recommendation, user clustering, inventory prediction, + trends detection, etc. Scikit-learn lets us experiment with many models, + especially in the exploration phase of a new project: the data can be passed + around in a consistent way; models are easy to save and reuse; updates keep us + informed of new developments from the pattern discovery research community. + Scikit-learn is an important tool for our team, built the right way in the + right language. - -
-
+ .. rst-class:: annotation -.. image:: images/birchbox.jpg - :width: 120pt - :align: center - :target: https://www.birchbox.com + Thierry Bertin-Mahieux, Data Scientist -.. raw:: html + .. div:: image-box -
-
+ .. image:: images/birchbox.jpg + :target: https://www.birchbox.com `Bestofmedia Group `_ --------------------------------------------------- +------------------------------------------------- -.. raw:: html +.. div:: sk-text-image-grid-large -
-
+ .. div:: text-box -Scikit-learn is our #1 toolkit for all things machine learning -at Bestofmedia. We use it for a variety of tasks (e.g. spam fighting, -ad click prediction, various ranking models) thanks to the varied, -state-of-the-art algorithm implementations packaged into it. -In the lab it accelerates prototyping of complex pipelines. In -production I can say it has proven to be robust and efficient enough -to be deployed for business critical components. + Scikit-learn is our #1 toolkit for all things machine learning + at Bestofmedia. We use it for a variety of tasks (e.g. spam fighting, + ad click prediction, various ranking models) thanks to the varied, + state-of-the-art algorithm implementations packaged into it. + In the lab it accelerates prototyping of complex pipelines. In + production I can say it has proven to be robust and efficient enough + to be deployed for business critical components. -.. raw:: html + .. rst-class:: annotation - + Eustache Diemert, Lead Scientist -Eustache Diemert, Lead Scientist Bestofmedia Group + .. div:: image-box -.. raw:: html + .. image:: images/bestofmedia-logo.png + :target: http://www.bestofmedia.com - -
-
- -.. image:: images/bestofmedia-logo.png - :width: 120pt - :align: center - :target: http://www.bestofmedia.com - -.. raw:: html - -
-
`Change.org `_ -------------------------------------- -.. raw:: html - -
-
+.. div:: sk-text-image-grid-large -At change.org we automate the use of scikit-learn's RandomForestClassifier -in our production systems to drive email targeting that reaches millions -of users across the world each week. In the lab, scikit-learn's ease-of-use, -performance, and overall variety of algorithms implemented has proved invaluable -in giving us a single reliable source to turn to for our machine-learning needs. + .. div:: text-box -.. raw:: html + At change.org we automate the use of scikit-learn's RandomForestClassifier + in our production systems to drive email targeting that reaches millions + of users across the world each week. In the lab, scikit-learn's ease-of-use, + performance, and overall variety of algorithms implemented has proved invaluable + in giving us a single reliable source to turn to for our machine-learning needs. - + .. rst-class:: annotation -Vijay Ramesh, Software Engineer in Data/science at Change.org + Vijay Ramesh, Software Engineer in Data/science at Change.org -.. raw:: html + .. div:: image-box - -
-
+ .. image:: images/change-logo.png + :target: https://www.change.org -.. image:: images/change-logo.png - :width: 120pt - :align: center - :target: https://www.change.org - -.. raw:: html - -
-
`PHIMECA Engineering `_ ----------------------------------------------------------- - -.. raw:: html - -
-
- -At PHIMECA Engineering, we use scikit-learn estimators as surrogates for -expensive-to-evaluate numerical models (mostly but not exclusively -finite-element mechanical models) for speeding up the intensive post-processing -operations involved in our simulation-based decision making framework. -Scikit-learn's fit/predict API together with its efficient cross-validation -tools considerably eases the task of selecting the best-fit estimator. We are -also using scikit-learn for illustrating concepts in our training sessions. -Trainees are always impressed by the ease-of-use of scikit-learn despite the -apparent theoretical complexity of machine learning. +--------------------------------------------------------- -.. raw:: html +.. div:: sk-text-image-grid-large - + .. div:: text-box -Vincent Dubourg, PHIMECA Engineering, PhD Engineer + At PHIMECA Engineering, we use scikit-learn estimators as surrogates for + expensive-to-evaluate numerical models (mostly but not exclusively + finite-element mechanical models) for speeding up the intensive post-processing + operations involved in our simulation-based decision making framework. + Scikit-learn's fit/predict API together with its efficient cross-validation + tools considerably eases the task of selecting the best-fit estimator. We are + also using scikit-learn for illustrating concepts in our training sessions. + Trainees are always impressed by the ease-of-use of scikit-learn despite the + apparent theoretical complexity of machine learning. -.. raw:: html + .. rst-class:: annotation - -
-
+ Vincent Dubourg, PHIMECA Engineering, PhD Engineer -.. image:: images/phimeca.png - :width: 120pt - :align: center - :target: https://www.phimeca.com/?lang=en + .. div:: image-box -.. raw:: html + .. image:: images/phimeca.png + :target: https://www.phimeca.com/?lang=en -
-
`HowAboutWe `_ ----------------------------------------------------------- - -.. raw:: html - -
-
- -At HowAboutWe, scikit-learn lets us implement a wide array of machine learning -techniques in analysis and in production, despite having a small team. We use -scikit-learn’s classification algorithms to predict user behavior, enabling us -to (for example) estimate the value of leads from a given traffic source early -in the lead’s tenure on our site. Also, our users' profiles consist of -primarily unstructured data (answers to open-ended questions), so we use -scikit-learn’s feature extraction and dimensionality reduction tools to -translate these unstructured data into inputs for our matchmaking system. - -.. raw:: html +------------------------------------------ - +.. div:: sk-text-image-grid-large -Daniel Weitzenfeld, Senior Data Scientist at HowAboutWe + .. div:: text-box -.. raw:: html + At HowAboutWe, scikit-learn lets us implement a wide array of machine learning + techniques in analysis and in production, despite having a small team. We use + scikit-learn's classification algorithms to predict user behavior, enabling us + to (for example) estimate the value of leads from a given traffic source early + in the lead's tenure on our site. Also, our users' profiles consist of + primarily unstructured data (answers to open-ended questions), so we use + scikit-learn's feature extraction and dimensionality reduction tools to + translate these unstructured data into inputs for our matchmaking system. - -
-
+ .. rst-class:: annotation -.. image:: images/howaboutwe.png - :width: 120pt - :align: center - :target: http://www.howaboutwe.com/ + Daniel Weitzenfeld, Senior Data Scientist at HowAboutWe -.. raw:: html + .. div:: image-box -
-
+ .. image:: images/howaboutwe.png + :target: http://www.howaboutwe.com/ `PeerIndex `_ ------------------------------------------------------------------ -.. raw:: html - -
-
- -At PeerIndex we use scientific methodology to build the Influence Graph - a -unique dataset that allows us to identify who’s really influential and in which -context. To do this, we have to tackle a range of machine learning and -predictive modeling problems. Scikit-learn has emerged as our primary tool for -developing prototypes and making quick progress. From predicting missing data -and classifying tweets to clustering communities of social media users, scikit- -learn proved useful in a variety of applications. Its very intuitive interface -and excellent compatibility with other python tools makes it and indispensable -tool in our daily research efforts. +.. div:: sk-text-image-grid-large -.. raw:: html + .. div:: text-box - + At PeerIndex we use scientific methodology to build the Influence Graph - a + unique dataset that allows us to identify who's really influential and in which + context. To do this, we have to tackle a range of machine learning and + predictive modeling problems. Scikit-learn has emerged as our primary tool for + developing prototypes and making quick progress. From predicting missing data + and classifying tweets to clustering communities of social media users, scikit- + learn proved useful in a variety of applications. Its very intuitive interface + and excellent compatibility with other python tools makes it and indispensable + tool in our daily research efforts. -Ferenc Huszar - Senior Data Scientist at Peerindex + .. rst-class:: annotation -.. raw:: html + Ferenc Huszar, Senior Data Scientist at Peerindex - -
-
+ .. div:: image-box -.. image:: images/peerindex.png - :width: 120pt - :align: center - :target: https://www.brandwatch.com/peerindex-and-brandwatch - -.. raw:: html - -
-
+ .. image:: images/peerindex.png + :target: https://www.brandwatch.com/peerindex-and-brandwatch `DataRobot `_ ---------------------------------------- -.. raw:: html - -
-
- -DataRobot is building next generation predictive analytics software to make data scientists more productive, and scikit-learn is an integral part of our system. The variety of machine learning techniques in combination with the solid implementations that scikit-learn offers makes it a one-stop-shopping library for machine learning in Python. Moreover, its consistent API, well-tested code and permissive licensing allow us to use it in a production environment. Scikit-learn has literally saved us years of work we would have had to do ourselves to bring our product to market. - -.. raw:: html +.. div:: sk-text-image-grid-large - + .. div:: text-box -Jeremy Achin, CEO & Co-founder DataRobot Inc. + DataRobot is building next generation predictive analytics software to make data + scientists more productive, and scikit-learn is an integral part of our system. The + variety of machine learning techniques in combination with the solid implementations + that scikit-learn offers makes it a one-stop-shopping library for machine learning + in Python. Moreover, its consistent API, well-tested code and permissive licensing + allow us to use it in a production environment. Scikit-learn has literally saved us + years of work we would have had to do ourselves to bring our product to market. -.. raw:: html + .. rst-class:: annotation - -
-
+ Jeremy Achin, CEO & Co-founder DataRobot Inc. -.. image:: images/datarobot.png - :width: 120pt - :align: center - :target: https://www.datarobot.com + .. div:: image-box -.. raw:: html - -
-
+ .. image:: images/datarobot.png + :target: https://www.datarobot.com `OkCupid `_ --------------------------------------- - -.. raw:: html - -
-
+------------------------------------- -We're using scikit-learn at OkCupid to evaluate and improve our matchmaking -system. The range of features it has, especially preprocessing utilities, means -we can use it for a wide variety of projects, and it's performant enough to -handle the volume of data that we need to sort through. The documentation is -really thorough, as well, which makes the library quite easy to use. +.. div:: sk-text-image-grid-large -.. raw:: html + .. div:: text-box - + We're using scikit-learn at OkCupid to evaluate and improve our matchmaking + system. The range of features it has, especially preprocessing utilities, means + we can use it for a wide variety of projects, and it's performant enough to + handle the volume of data that we need to sort through. The documentation is + really thorough, as well, which makes the library quite easy to use. -David Koh - Senior Data Scientist at OkCupid + .. rst-class:: annotation -.. raw:: html + David Koh - Senior Data Scientist at OkCupid - -
-
+ .. div:: image-box -.. image:: images/okcupid.png - :width: 120pt - :align: center - :target: https://www.okcupid.com - -.. raw:: html - -
-
+ .. image:: images/okcupid.png + :target: https://www.okcupid.com `Lovely `_ ----------------------------------- -.. raw:: html - -
-
- -At Lovely, we strive to deliver the best apartment marketplace, with respect to -our users and our listings. From understanding user behavior, improving data -quality, and detecting fraud, scikit-learn is a regular tool for gathering -insights, predictive modeling and improving our product. The easy-to-read -documentation and intuitive architecture of the API makes machine learning both -explorable and accessible to a wide range of python developers. I'm constantly -recommending that more developers and scientists try scikit-learn. - -.. raw:: html +.. div:: sk-text-image-grid-large - + .. div:: text-box -Simon Frid - Data Scientist, Lead at Lovely + At Lovely, we strive to deliver the best apartment marketplace, with respect to + our users and our listings. From understanding user behavior, improving data + quality, and detecting fraud, scikit-learn is a regular tool for gathering + insights, predictive modeling and improving our product. The easy-to-read + documentation and intuitive architecture of the API makes machine learning both + explorable and accessible to a wide range of python developers. I'm constantly + recommending that more developers and scientists try scikit-learn. -.. raw:: html + .. rst-class:: annotation - -
-
+ Simon Frid - Data Scientist, Lead at Lovely -.. image:: images/lovely.png - :width: 120pt - :align: center - :target: https://livelovely.com - -.. raw:: html - -
-
+ .. div:: image-box + .. image:: images/lovely.png + :target: https://livelovely.com `Data Publica `_ ---------------------------------------------- -.. raw:: html - -
-
- -Data Publica builds a new predictive sales tool for commercial and marketing teams called C-Radar. -We extensively use scikit-learn to build segmentations of customers through clustering, and to predict future customers based on past partnerships success or failure. -We also categorize companies using their website communication thanks to scikit-learn and its machine learning algorithm implementations. -Eventually, machine learning makes it possible to detect weak signals that traditional tools cannot see. -All these complex tasks are performed in an easy and straightforward way thanks to the great quality of the scikit-learn framework. +.. div:: sk-text-image-grid-large -.. raw:: html + .. div:: text-box - + Data Publica builds a new predictive sales tool for commercial and marketing teams + called C-Radar. We extensively use scikit-learn to build segmentations of customers + through clustering, and to predict future customers based on past partnerships + success or failure. We also categorize companies using their website communication + thanks to scikit-learn and its machine learning algorithm implementations. + Eventually, machine learning makes it possible to detect weak signals that + traditional tools cannot see. All these complex tasks are performed in an easy and + straightforward way thanks to the great quality of the scikit-learn framework. -Guillaume Lebourgeois & Samuel Charron - Data Scientists at Data Publica + .. rst-class:: annotation -.. raw:: html + Guillaume Lebourgeois & Samuel Charron - Data Scientists at Data Publica - -
-
- -.. image:: images/datapublica.png - :width: 120pt - :align: center - :target: http://www.data-publica.com/ - -.. raw:: html - -
-
+ .. div:: image-box + .. image:: images/datapublica.png + :target: http://www.data-publica.com/ `Machinalis `_ ------------------------------------------- -.. raw:: html - -
-
+.. div:: sk-text-image-grid-large -Scikit-learn is the cornerstone of all the machine learning projects carried at -Machinalis. It has a consistent API, a wide selection of algorithms and lots -of auxiliary tools to deal with the boilerplate. -We have used it in production environments on a variety of projects -including click-through rate prediction, `information extraction `_, -and even counting sheep! + .. div:: text-box -In fact, we use it so much that we've started to freeze our common use cases -into Python packages, some of them open-sourced, like -`FeatureForge `_ . -Scikit-learn in one word: Awesome. + Scikit-learn is the cornerstone of all the machine learning projects carried at + Machinalis. It has a consistent API, a wide selection of algorithms and lots of + auxiliary tools to deal with the boilerplate. We have used it in production + environments on a variety of projects including click-through rate prediction, + `information extraction `_, and even counting + sheep! -.. raw:: html + In fact, we use it so much that we've started to freeze our common use cases + into Python packages, some of them open-sourced, like `FeatureForge + `_. Scikit-learn in one word: Awesome. - + .. rst-class:: annotation -Rafael Carrascosa, Lead developer + Rafael Carrascosa, Lead developer -.. raw:: html + .. div:: image-box - -
-
- -.. image:: images/machinalis.png - :width: 120pt - :align: center - :target: https://www.machinalis.com/ - -.. raw:: html - -
-
+ .. image:: images/machinalis.png + :target: https://www.machinalis.com/ `solido `_ ----------------------------------------- -.. raw:: html - -
-
+.. div:: sk-text-image-grid-large -Scikit-learn is helping to drive Moore’s Law, via Solido. Solido creates -computer-aided design tools used by the majority of top-20 semiconductor -companies and fabs, to design the bleeding-edge chips inside smartphones, -automobiles, and more. Scikit-learn helps to power Solido’s algorithms for -rare-event estimation, worst-case verification, optimization, and more. At -Solido, we are particularly fond of scikit-learn’s libraries for Gaussian -Process models, large-scale regularized linear regression, and classification. -Scikit-learn has increased our productivity, because for many ML problems we no -longer need to “roll our own” code. `This PyData 2014 talk `_ has details. + .. div:: text-box + Scikit-learn is helping to drive Moore's Law, via Solido. Solido creates + computer-aided design tools used by the majority of top-20 semiconductor + companies and fabs, to design the bleeding-edge chips inside smartphones, + automobiles, and more. Scikit-learn helps to power Solido's algorithms for + rare-event estimation, worst-case verification, optimization, and more. At + Solido, we are particularly fond of scikit-learn's libraries for Gaussian + Process models, large-scale regularized linear regression, and classification. + Scikit-learn has increased our productivity, because for many ML problems we no + longer need to “roll our own” code. `This PyData 2014 talk + `_ has details. -.. raw:: html + .. rst-class:: annotation - + Trent McConaghy, founder, Solido Design Automation Inc. -Trent McConaghy, founder, Solido Design Automation Inc. - -.. raw:: html - - -
-
- -.. image:: images/solido_logo.png - :width: 120pt - :align: center - :target: https://www.solidodesign.com/ - -.. raw:: html - -
-
+ .. div:: image-box + .. image:: images/solido_logo.png + :target: https://www.solidodesign.com/ `INFONEA `_ ------------------------------------------ - -.. raw:: html - -
-
- -We employ scikit-learn for rapid prototyping and custom-made Data Science -solutions within our in-memory based Business Intelligence Software -INFONEAÂŽ. As a well-documented and comprehensive collection of -state-of-the-art algorithms and pipelining methods, scikit-learn enables -us to provide flexible and scalable scientific analysis solutions. Thus, -scikit-learn is immensely valuable in realizing a powerful integration of -Data Science technology within self-service business analytics. +--------------------------------------- -.. raw:: html +.. div:: sk-text-image-grid-large - + .. div:: text-box -Thorsten Kranz, Data Scientist, Coma Soft AG. + We employ scikit-learn for rapid prototyping and custom-made Data Science + solutions within our in-memory based Business Intelligence Software + INFONEAÂŽ. As a well-documented and comprehensive collection of + state-of-the-art algorithms and pipelining methods, scikit-learn enables + us to provide flexible and scalable scientific analysis solutions. Thus, + scikit-learn is immensely valuable in realizing a powerful integration of + Data Science technology within self-service business analytics. -.. raw:: html + .. rst-class:: annotation - -
-
+ Thorsten Kranz, Data Scientist, Coma Soft AG. -.. image:: images/infonea.jpg - :width: 120pt - :align: center - :target: http://www.infonea.com/en/ + .. div:: image-box -.. raw:: html - -
-
+ .. image:: images/infonea.jpg + :target: http://www.infonea.com/en/ `Dataiku `_ ------------------------------------------ - -.. raw:: html - -
-
+------------------------------------- -Our software, Data Science Studio (DSS), enables users to create data services -that combine `ETL `_ with -Machine Learning. Our Machine Learning module integrates -many scikit-learn algorithms. The scikit-learn library is a perfect integration -with DSS because it offers algorithms for virtually all business cases. Our goal -is to offer a transparent and flexible tool that makes it easier to optimize -time consuming aspects of building a data service, preparing data, and training -machine learning algorithms on all types of data. +.. div:: sk-text-image-grid-large + .. div:: text-box -.. raw:: html + Our software, Data Science Studio (DSS), enables users to create data services + that combine `ETL `_ with + Machine Learning. Our Machine Learning module integrates + many scikit-learn algorithms. The scikit-learn library is a perfect integration + with DSS because it offers algorithms for virtually all business cases. Our goal + is to offer a transparent and flexible tool that makes it easier to optimize + time consuming aspects of building a data service, preparing data, and training + machine learning algorithms on all types of data. - + .. rst-class:: annotation -Florian Douetteau, CEO, Dataiku + Florian Douetteau, CEO, Dataiku -.. raw:: html + .. div:: image-box - -
-
+ .. image:: images/dataiku_logo.png + :target: https://www.dataiku.com/ -.. image:: images/dataiku_logo.png - :width: 120pt - :align: center - :target: https://www.dataiku.com/ - -.. raw:: html - -
-
`Otto Group `_ ------------------------------------------ - -.. raw:: html - -
-
- -Here at Otto Group, one of global Big Five B2C online retailers, we are using -scikit-learn in all aspects of our daily work from data exploration to development -of machine learning application to the productive deployment of those services. -It helps us to tackle machine learning problems ranging from e-commerce to logistics. -It consistent APIs enabled us to build the `Palladium REST-API framework -`_ around it and continuously deliver -scikit-learn based services. - +-------------------------------------- -.. raw:: html +.. div:: sk-text-image-grid-large - + .. div:: text-box -Christian Rammig, Head of Data Science, Otto Group + Here at Otto Group, one of global Big Five B2C online retailers, we are using + scikit-learn in all aspects of our daily work from data exploration to development + of machine learning application to the productive deployment of those services. + It helps us to tackle machine learning problems ranging from e-commerce to logistics. + It consistent APIs enabled us to build the `Palladium REST-API framework + `_ around it and continuously deliver + scikit-learn based services. -.. raw:: html + .. rst-class:: annotation - -
-
+ Christian Rammig, Head of Data Science, Otto Group -.. image:: images/ottogroup_logo.png - :width: 120pt - :align: center - :target: https://ottogroup.com + .. div:: image-box -.. raw:: html + .. image:: images/ottogroup_logo.png + :target: https://ottogroup.com -
-
`Zopa `_ ------------------------------------------ +--------------------------- -.. raw:: html +.. div:: sk-text-image-grid-large -
-
+ .. div:: text-box -At Zopa, the first ever Peer-to-Peer lending platform, we extensively use scikit-learn -to run the business and optimize our users' experience. It powers our -Machine Learning models involved in credit risk, fraud risk, marketing, and pricing, -and has been used for originating at least 1 billion GBP worth of Zopa loans. -It is very well documented, powerful, and simple to use. We are grateful for the -capabilities it has provided, and for allowing us to deliver on our mission of making -money simple and fair. + At Zopa, the first ever Peer-to-Peer lending platform, we extensively use + scikit-learn to run the business and optimize our users' experience. It powers our + Machine Learning models involved in credit risk, fraud risk, marketing, and pricing, + and has been used for originating at least 1 billion GBP worth of Zopa loans. It is + very well documented, powerful, and simple to use. We are grateful for the + capabilities it has provided, and for allowing us to deliver on our mission of + making money simple and fair. -.. raw:: html + .. rst-class:: annotation - + Vlasios Vasileiou, Head of Data Science, Zopa -Vlasios Vasileiou, Head of Data Science, Zopa + .. div:: image-box -.. raw:: html + .. image:: images/zopa.png + :target: https://zopa.com - -
-
- -.. image:: images/zopa.png - :width: 120pt - :align: center - :target: https://zopa.com - -.. raw:: html - -
-
`MARS `_ --------------------------------------- +------------------------------------- -.. raw:: html +.. div:: sk-text-image-grid-large -
-
+ .. div:: text-box -Scikit-Learn is integral to the Machine Learning Ecosystem at Mars. Whether -we're designing better recipes for petfood or closely analysing our cocoa -supply chain, Scikit-Learn is used as a tool for rapidly prototyping ideas -and taking them to production. This allows us to better understand and meet -the needs of our consumers worldwide. Scikit-Learn's feature-rich toolset is -easy to use and equips our associates with the capabilities they need to -solve the business challenges they face every day. + Scikit-Learn is integral to the Machine Learning Ecosystem at Mars. Whether + we're designing better recipes for petfood or closely analysing our cocoa + supply chain, Scikit-Learn is used as a tool for rapidly prototyping ideas + and taking them to production. This allows us to better understand and meet + the needs of our consumers worldwide. Scikit-Learn's feature-rich toolset is + easy to use and equips our associates with the capabilities they need to + solve the business challenges they face every day. -.. raw:: html + .. rst-class:: annotation - + Michael Fitzke, Next Generation Technologies Sr Leader, Mars Inc. -Michael Fitzke Next Generation Technologies Sr Leader, Mars Inc. + .. div:: image-box -.. raw:: html - - -
-
- -.. image:: images/mars.png - :width: 120pt - :align: center - :target: https://www.mars.com/global - -.. raw:: html - -
-
+ .. image:: images/mars.png + :target: https://www.mars.com/global `BNP Paribas Cardif `_ --------------------------------------------------------- -.. raw:: html - -
-
- -BNP Paribas Cardif uses scikit-learn for several of its machine learning models -in production. Our internal community of developers and data scientists has -been using scikit-learn since 2015, for several reasons: the quality of the -developments, documentation and contribution governance, and the sheer size of -the contributing community. We even explicitly mention the use of -scikit-learn's pipelines in our internal model risk governance as one of our -good practices to decrease operational risks and overfitting risk. As a way to -support open source software development and in particular scikit-learn -project, we decided to participate to scikit-learn's consortium at La Fondation -Inria since its creation in 2018. - -.. raw:: html - - +.. div:: sk-text-image-grid-large -SÊbastien Conort, Chief Data Scientist, BNP Paribas Cardif + .. div:: text-box -.. raw:: html + BNP Paribas Cardif uses scikit-learn for several of its machine learning models + in production. Our internal community of developers and data scientists has + been using scikit-learn since 2015, for several reasons: the quality of the + developments, documentation and contribution governance, and the sheer size of + the contributing community. We even explicitly mention the use of + scikit-learn's pipelines in our internal model risk governance as one of our + good practices to decrease operational risks and overfitting risk. As a way to + support open source software development and in particular scikit-learn + project, we decided to participate to scikit-learn's consortium at La Fondation + Inria since its creation in 2018. - -
-
+ .. rst-class:: annotation -.. image:: images/bnp_paribas_cardif.png - :width: 120pt - :align: center - :target: https://www.bnpparibascardif.com/ + SÊbastien Conort, Chief Data Scientist, BNP Paribas Cardif -.. raw:: html + .. div:: image-box -
-
+ .. image:: images/bnp_paribas_cardif.png + :target: https://www.bnpparibascardif.com/ diff --git a/doc/themes/scikit-learn-modern/javascript.html b/doc/themes/scikit-learn-modern/javascript.html deleted file mode 100644 index fc0dca1040e03..0000000000000 --- a/doc/themes/scikit-learn-modern/javascript.html +++ /dev/null @@ -1,149 +0,0 @@ -{% if theme_google_analytics|tobool %} - - -{% endif %} - - -{%- if pagename != 'index' and pagename != 'documentation' %} - {% if theme_mathjax_path %} - - {% endif %} -{%- endif %} diff --git a/doc/themes/scikit-learn-modern/layout.html b/doc/themes/scikit-learn-modern/layout.html deleted file mode 100644 index a4b9733b68709..0000000000000 --- a/doc/themes/scikit-learn-modern/layout.html +++ /dev/null @@ -1,158 +0,0 @@ -{# TEMPLATE VAR SETTINGS #} -{%- set url_root = pathto('', 1) %} -{%- if url_root == '#' %}{% set url_root = '' %}{% endif %} -{%- if not embedded and docstitle %} - {%- set titlesuffix = " — "|safe + docstitle|e %} -{%- else %} - {%- set titlesuffix = "" %} -{%- endif %} -{%- set lang_attr = 'en' %} - - - - - - - {{ metatags }} - - - {% block htmltitle %} - {{ title|striptags|e }}{{ titlesuffix }} - {% endblock %} - - - {% if favicon %} - - {% endif %} - - - {%- for css in css_files %} - {%- if css|attr("rel") %} - - {%- else %} - - {%- endif %} - {%- endfor %} - - - -{%- block extrahead %} {% endblock %} - - -{% include "nav.html" %} -{%- block content %} -
- - -
-
- -
- {%- if prev %} - Prev - {%- else %} - Prev - {%- endif %} - {%- if parents -%} - Up - {%- else %} - Up - {%- endif %} - {%- if next %} - Next - {%- else %} - Next - {%- endif %} -
- {%- if pagename != "install" %} - - {%- endif %} - - {%- if meta and meta['parenttoc']|tobool %} -
- {% set nav = get_nav_object(maxdepth=3, collapse=True, numbered=True) %} -
    - {% for main_nav_item in nav %} - {% if main_nav_item.active %} -
  • - {{ main_nav_item.title }} -
    • -
      - {% for nav_item in main_nav_item.children %} -
    • - {{ nav_item.title }} - {% if nav_item.children %} - - {% endif %} -
      • - {% endfor %} -
    - {% endif %} - {% endfor %} -
-
- {%- elif meta and meta['globalsidebartoc']|tobool %} -
- {{ toctree(maxdepth=2, titles_only=True) }} -
- {%- else %} -
- {{ toc }} -
- {%- endif %} -
-
-
-
- {% block body %}{% endblock %} -
-
-
- {%- if pagename != 'index' %} - {%- if show_copyright %} - {%- if hasdoc('copyright') %} - {% trans path=pathto('copyright'), copyright=copyright|e %}© {{ copyright }}.{% endtrans %} - {%- else %} - {% trans copyright=copyright|e %}© {{ copyright }}.{% endtrans %} - {%- endif %} - {%- endif %} - {%- if last_updated %} - {% trans last_updated=last_updated|e %}Last updated on {{ last_updated }}.{% endtrans %} - {%- endif %} - {%- if show_source and has_source and sourcename %} - {{ _('Show this page source') }} - {%- endif %} - {%- endif %} -
-
-
-
-{%- endblock %} - -{% include "javascript.html" %} - - diff --git a/doc/themes/scikit-learn-modern/nav.html b/doc/themes/scikit-learn-modern/nav.html deleted file mode 100644 index 03a6b3c6f33b9..0000000000000 --- a/doc/themes/scikit-learn-modern/nav.html +++ /dev/null @@ -1,87 +0,0 @@ -{%- if pagename != 'index' and pagename != 'documentation' %} - {%- set nav_bar_class = "sk-docs-navbar" %} - {%- set top_container_cls = "sk-docs-container" %} -{%- else %} - {%- set nav_bar_class = "sk-landing-navbar" %} - {%- set top_container_cls = "sk-landing-container" %} -{%- endif %} - -{%- set drop_down_navigation = [ - ('Getting Started', pathto('getting_started')), - ('Tutorial', pathto('tutorial/index')), - ("What's new", pathto('whats_new/v' + version)), - ('Glossary', pathto('glossary')), - ('Development', pathto('developers/index')), - ('FAQ', pathto('faq')), - ('Support', pathto('support')), - ('Related packages', pathto('related_projects')), - ('Roadmap', pathto('roadmap')), - ('About us', pathto('about')), - ('GitHub', 'https://github.com/scikit-learn/scikit-learn'), - ('Other Versions and Download', 'https://scikit-learn.org/dev/versions.html')] --%} - - diff --git a/doc/themes/scikit-learn-modern/search.html b/doc/themes/scikit-learn-modern/search.html deleted file mode 100644 index 2d06835afabb6..0000000000000 --- a/doc/themes/scikit-learn-modern/search.html +++ /dev/null @@ -1,8 +0,0 @@ -{%- extends "basic/search.html" %} -{% block extrahead %} - - - - - -{% endblock %} diff --git a/doc/themes/scikit-learn-modern/static/css/theme.css b/doc/themes/scikit-learn-modern/static/css/theme.css deleted file mode 100644 index 5fa26391886e0..0000000000000 --- a/doc/themes/scikit-learn-modern/static/css/theme.css +++ /dev/null @@ -1,1304 +0,0 @@ -/* Elements */ -a { - color: #2878A2; - word-wrap: break-word; -} - -a:focus { - outline: none; -} - -/* Anchor links */ - -a.headerlink { - color: #c60f0f; - font-size: 0.8em; - padding: 0 4px 0 4px; - text-decoration: none; - visibility: hidden; -} - -a.headerlink:hover { - background-color: #c60f0f; - color: white; -} - -p { - word-break: break-word; - hyphens: auto; -} - -input:focus { - outline: none; -} - -code { - color: #222; - background-color: #ecf0f3; - border-radius: 0.2rem; - padding: 0.15rem; - word-break: normal; -} - -nav { - z-index: 3; -} - -h1 code, h2 code, h3 code, h4 code, h5 code, h6 code { - background-color: transparent; -} - -h1:hover a.headerlink, -h2:hover a.headerlink, -h3:hover a.headerlink, -h4:hover a.headerlink, -h5:hover a.headerlink, -h6:hover a.headerlink, -dt:hover a.headerlink { - visibility: visible; -} - -strong { - font-weight: bold; -} - -a code { - color: inherit; -} - -a code { - background-color: transparent; - font-weight: bold; - color: #2878A2; - border-radius: 0; - padding: 0; - white-space: nowrap; -} - -img { - max-width: 100%; -} - -span.highlighted { - background-color: #fbe54e; -} - -div.highlight { - border: 1px solid #ddd; - margin-bottom: 1rem; -} - -div.highlight pre { - padding: 0.2rem 0.5rem; - margin-bottom: 0; - line-height: 1.2rem; -} - -div.highlight a { - text-decoration: underline; -} - -.versionmodified { - font-style: italic; -} - -a.sk-landing-btn { - background-color: #ff9c34; - color: black; - cursor: pointer; - font-size: 1.1rem; - font-weight: 500; -} - -a.sk-landing-btn:hover { - background-color: #ffb05f; -} - -.sk-donate-btn { - cursor: pointer; -} - -.sk-page-content div.logo { - float: left; - width: 200px; -} - -@media screen and (min-width: 992px) { - .sk-page-content { - padding-left: 2rem!important; - padding-right: 2rem!important; - } -} - -@media screen and (min-width: 1200px) { - .sk-px-xl-4 { - padding-left: 1.3rem!important; - padding-right: 1.3rem!important; - } -} - -/* clearfix */ - -div.clearer { - clear: both; -} - -/* Button */ - -.sk-btn-primary { - background-color: #30799C; - border-color: #30799C; - color: white; -} - -.sk-btn-primary:hover, -.sk-btn-primary:active { - background-color: #3499cd; - border-color: #3499cd; -} - -/* Quote */ - -.quote { - text-align: right; - line-height: 1.5em; - font-style: italic; - margin: 2em 3em 1em 3em; -} - -.line-block { - display: block; - margin-top: 1em; - margin-bottom: 1em; -} - -/* Search */ - -#search-results { - margin-top: 1rem; -} - -#searchbox { - padding-top: 0.1rem; -} - -.sk-search-text-input { - width: 12rem; -} - -.sk-search-text-btn { - padding-left: 0.2rem; - padding-right: 0.2rem; -} - -ul.search li div.context { - color: #888; - margin: 0.1rem 0 0 0; - text-align: left; -} - -@media screen and (min-width: 768px) { - ul.search li div.context { - margin-left: 1rem; - } -} - -ul.search li a { - font-weight: bold; -} -/* navbar */ - -img.sk-brand-img { - height: 48px; -} - -.navbar-light .navbar-nav a.nav-link, a.sk-dropdown-item { - color: rgba(77, 77, 77, 1); - font-weight: 500; -} - -.navbar-light .navbar-nav a.nav-link:hover, a.sk-dropdown-item:hover { - color: rgba(246, 126, 0, 1); -} - -a.sk-nav-dropdown-item:active { - color: white; - background-color: rgba(246, 126, 0, 1); -} - -.nav-more-item-mobile-items { - display: inherit; -} - -.nav-more-item-dropdown { - display: none; -} - -@media screen and (min-width: 768px) { - .nav-more-item-dropdown { - display: inherit; - } - - .nav-more-item-mobile-items { - display: none; - } -} -/* LANDING PAGE STYLE */ - -div.sk-landing-container { - max-width: 1400px; -} - -div.sk-landing-container .text-white { - text-shadow: 0px 0px 8px rgb(42, 98, 128); -} - -ul.sk-landing-header-body { - margin-top: auto; - margin-bottom: auto; - font-size: 1.2rem; - font-weight: 500; -} - -div.sk-landing-bg-more-info dd { - padding-left: 0; -} - -div.sk-landing-bg { - background-image: linear-gradient(160deg, rgba(42,98,128,1) 0%, rgba(52,153,205,1) 17%, rgba(255,243,211,1) 59%, rgba(255,178,96,1) 100%); -} - -div.sk-landing-bg-more-info { - background-color: #f8f8f8; - font-size: 0.96rem; -} - -.sk-card-title { - font-weight: 700; -} - -.sk-landing-header { - font-size: 3.2rem; -} - -.sk-landing-subheader { - letter-spacing: 0.17rem; -} - -.sk-landing-call-header { - color: #E07200; - font-weight: 700; -} - -img.sk-index-img { - max-height: 240px; - margin: auto; - margin-bottom: 1em; - width: auto; -} - -@media screen and (min-width: 768px) { - img.sk-index-img { - width: 100% - } -} - -img.sk-who-uses-carousel-img { - max-height: 100px; - max-width: 50%; -} - -div#carouselExampleSlidesOnly { - min-height: 200px; -} - -ul.sk-landing-call-list li { - margin-bottom: 0.25rem; -} - -img.sk-footer-funding-logo { - max-height: 36px; - max-width: 80px; - margin: 0 8px; - margin-bottom: 8px; -} - -a.sk-footer-funding-link:hover { - text-decoration: none; -} -/* DOCS STYLE */ - -.navbar > .sk-docs-container { - max-width: 1400px; - margin: 0 auto; -} - -#sk-sidebar-wrapper { - height: 100%; - overflow-y: hidden; - overflow-x: hidden; - position: fixed; - margin-left: -240px; - width: 240px; - -webkit-transition: margin 0.25s ease-out, opacity 0.25s ease-out; - -moz-transition: margin 0.25s ease-out, opacity 0.25s ease-out; - -o-transition: margin 0.25s ease-out, opacity 0.25s ease-out; - transition: margin 0.25s ease-out, opacity 0.25s ease-out; - background-color: white; - opacity: 0; - top: 0; - padding: 0 0.5rem 0.5rem 0.5rem; - z-index: 2; -} - -#sk-toggle-checkbox { - display: none; -} - -#sk-toggle-checkbox:checked ~ #sk-sidebar-wrapper { - margin-left: 0; - opacity: 1; -} - -#sk-doc-wrapper { - max-width: 1400px; - margin: 0 auto; -} - -#sk-page-content-wrapper { - width: 100%; -} - -div.sk-page-content { - background-color: white; - position: relative; - margin-top: 0.5rem; -} - -div.sk-page-content { - table-layout: fixed; - max-width: 100%; -} - -div.section h2, -div.section h3, -div.section h4, -div.section h5, -div.section h6 { - margin-top: 1rem; -} - -.sk-btn-toggle-toc { - position: fixed; - bottom: 0; - margin: 0; - border-radius: 0; - border-top-right-radius: 0.5rem; - z-index: 3; - cursor: pointer; -} - -div.sk-page-content { - margin-top: 52px; -} - -@media screen and (min-width: 1400px) { - .sk-btn-toggle-toc { - border-top-left-radius: 0.5rem; - } -} - -.sk-btn-toggle-toc:hover { - color: white; - background-color: #297ca7; -} - -footer.sk-content-footer { - padding: 1rem 0; - color: #999; - text-align: right; -} - -nav.sk-docs-navbar { - width: 100%; - z-index: 3; - -webkit-transition: top .2s ease-in-out; - -moz-transition: top .2s ease-in-out .05s; - -o-transition: top .2s ease-in-out .05s; - transition: top .2s ease-in-out .05s; - position: fixed; - max-height: 100vh; - overflow-y: auto; - align-items: initial; -} - -div.sk-navbar-collapse { - padding-bottom: 4rem; -} - -@media screen and (min-width: 768px) { - - nav.sk-docs-navbar { - overflow-y: visible; - max-height: none; - } - - div.sk-navbar-collapse { - padding-bottom: 0; - } - - #sk-page-content-wrapper { - padding-left: 240px; - max-width: 1240px; - margin-left: auto; - margin-right: auto; - } - - #sk-sidebar-wrapper { - margin-left: 0; - opacity: 1; - } - - #sk-toggle-checkbox:checked ~ #sk-sidebar-wrapper { - margin-left: -240px; - opacity: 0; - } - - #sk-toggle-checkbox:checked ~ #sk-page-content-wrapper { - padding-left: 0; - margin-left: auto; - margin-right: auto; - } -} - -.centered { - text-align: center; -} - -dl.citation > dd > ol > li { - display: inline; -} - -dl.citation > dd > ol { - margin-bottom: 0; -} - -/* docs index */ - -div.sk-documentation-index-card { - border-left: 0.15rem solid #ff9c34; -} -div.sk-documentation-index-card:hover { - box-shadow: 0 0.5rem 1rem rgba(0, 0, 0, 0.15); -} - -a.sk-documentation-index-anchor:hover { - text-decoration: none; - color: #2878A2; -} - -.sk-documentation-index-header { - background-color: #cde8ef; - padding: 0.5rem; - border-radius: 0 1rem; - text-align: center; - font-size: 2rem; - font-weight: 500; -} - -/* toc */ - -div.sk-sidebar-toc-logo { - height: 52px; -} - -.sk-toc-active { - font-weight: bold; -} - -div.sk-sidebar-toc-wrapper { - font-size: 0.9rem; - width: 252px; - overflow-x: hidden; - overflow-y: scroll; - height: 100vh; - padding-right: 1.75rem; - - /* Hide scrollbar for IE and Edge */ - -ms-overflow-style: none; - - /* Hide scrollbar for Firefox */ - scrollbar-width: none; -} - -div.sk-sidebar-toc-wrapper::-webkit-scrollbar { - display: none; -} - -div.sk-sidebar-toc-wrapper::after { - display: block; - content: ""; - height: 3rem; - visibility: hidden; -} - -div.sk-sidebar-toc > ul > li > a{ - font-weight: bold; -} - -div.sk-sidebar-toc > ul, -div.sk-sidebar-toc ul ul { - list-style: none; - margin-left: 0; - padding-left: 0; -} - -div.sk-sidebar-toc ul ul ul { - margin-left: 1rem; -} - - -div.sk-sidebar-toc ul li ul li ul{ - display: none; -} - -div.sk-sidebar-toc span { - white-space: pre; -} - -div.sk-sidebar-global-toc ul ul { - padding-left: 0.75rem; -} -/* content styling element style */ - -div.sk-page-content h1 { - background-color: #cde8ef; - padding: 0.5rem; - border-radius: 0 1rem; - text-align: center; - font-size: 2rem; - word-wrap: break-word; -} - -div.sk-page-content h2 { - padding: 0.5rem; - background-color: #BED4EB; - border-radius: 0.3rem; - font-size: 1.5rem; - margin-bottom: 1rem; - word-wrap: break-word; -} - -div.sk-page-content h3 { - padding: 0.3rem; - background-color: #eee; - border-radius: 0.3rem; - font-size: 1.2rem; - word-wrap: break-word; -} - -div.sk-page-content h4 { - padding: 0.2rem; - background-color: #F4F4F4; - border-radius: 0.3rem; - font-size: 1.2rem; - word-wrap: break-word; -} - -div.sk-page-content h1 code, -div.sk-page-content h2 code, -div.sk-page-content h3 code, -div.sk-page-content h4 code { - white-space: normal; -} - -/* longtables */ - -table.longtable p { - -moz-hyphens: none; - -ms-hyphens: none; - -webkit-hyphens: none; - hyphens: none; - line-height: 1.1em; - margin-bottom: 0; -} - -table.longtable td, table.longtable th { - border-top: 1px solid #ddd; - border-bottom: 1px solid #ddd; - padding-right: 0.5rem; - white-space:nowrap; -} - -table.longtable tr.row-odd { - background-color: #F0F7FA; -} - -/* api docs */ - -.class > dt, .function > dt, .method > dt { - padding: 0.5rem; - background-color: #f8f8f8; - font-weight: normal; - border: 1px solid rgba(0, 0, 0, 0.125); - border-left: 2px solid #ff9c34; - overflow: auto; - margin-bottom: 1rem; -} - -.class > dt::after, .function > dt::after, .method > dt::after { - overflow: auto; -} - -code.descname { - font-weight: bold; - background-color: transparent; - padding: 0; -} - -code.descclassname { - background-color: transparent; -} - -.viewcode-link { - float: right; -} - -dl.field-list { - display: flex; - flex-wrap: wrap; - overflow-x: scroll; -} - -dl.field-list > dt { - flex-basis: 100%; - font-weight: bold; - word-break: break-word; -} - -dl.field-list > dd { - flex-basis: 100%; - margin-bottom: 0; -} - -@media screen and (min-width: 768px) { - dl.field-list > dt { - flex-basis: 110px; - } - dl.field-list > dd { - flex: 1 0 calc(100% - 110px); - max-width: calc(100% - 110px); - } - -} - -dt.field-odd, dt.field-even { - background-color: #F0F7FA; - padding-left: 0.25rem; -} - -.field-odd, .field-even { - margin-top: 0; - border-bottom: 1px solid #ddd; - border-top: 1px solid #ddd; - box-sizing: border-box; -} - -dl.field-list > dt:after { - content: ":"; -} - -.classifier { - font-style: italic; -} - -.classifier::before { - font-style: normal; - margin: 0.3em; - content: ":"; -} - -dd { - padding-left: 1rem; -} - -dl.class > dd { - padding-left: 0; -} - -@media screen and (min-width: 768px) { - dl.class > dd { - padding-left: 1rem; - } -} - -.rubric { - font-weight: bold; - margin-top: 1rem; -} - -ul.simple li p, ol.simple li p { - margin-bottom: 0; -} - -ul.simple, ol.simple { - padding-left: 1.5rem; -} - -/* info boxes */ - -div.topic { - padding: 0.5rem; - background-color: #eee; - margin-bottom: 1rem; - border-radius: 0.25rem; - border: 1px solid #CCC; -} - -div.topic p { - margin-bottom: 0.25rem; -} - -div.topic dd { - margin-bottom: 0.25rem; -} - -p.topic-title { - font-weight: bold; - margin-bottom: 0.5rem; -} - -div.topic > ul.simple { - margin-bottom: 0.25rem; -} - -p.admonition-title { - margin-right: 0.5rem; - font-weight: bold; - display: inline; -} - -p.admonition-title:after { - content: ":"; -} - -div.admonition p.admonition-title + p, div.deprecated p { - display: inline; -} - -div.admonition, div.deprecated, -div.versionchanged, div.versionadded{ - margin-top: 0.5rem; - padding: 0.5rem; - border-radius: 0.5rem; - margin-bottom: 0.5rem; - border: 1px solid #ddd; -} - -div.admonition { - background-color: #eee; -} - -div.admonition p, div.admonition dl, div.admonition dd, -div.deprecated p, div.versionchanged p, div.versionadded p{ - margin-bottom: 0 -} - -div.deprecated { - color: #b94a48; - background-color: #F3E5E5; - border-color: #eed3d7; -} - -div.seealso { - background-color: #FFFBE8; - border-color: #fbeed5; - color: #AF8A4B; -} - -div.versionchanged { - background-color: #FFFBE8; - border-color: #fbeed5; -} - -dt.label { - float: left; - padding-right: 0.5rem; -} - -/* copy buttonn */ -div.highlight:hover span.copybutton { - background-color: #3F556B; - color: white; -} - -div.highlight:hover span.copybutton:hover { - background-color: #20252B; -} - -div.body img { - max-width: 100%; - height: unset!important; /* Needed because sphinx sets the height */ -} - -div.body dd > p { - hyphens: none; -} - -img.align-center, .figure.align-center, object.align-center { - display: block; - margin-left: auto; - margin-right: auto; - margin-bottom: 1rem; - text-align: center; -} - -img.align-right, .figure.align-right, object.align-right { - clear: right; - float: right; - margin-left: 1em; -} - -a.brackets::after, span.brackets > a::after { - content: "]"; -} - -a.brackets::before, span.brackets > a::before { - content: "["; -} - -/* copybutton */ - -.copybutton { - cursor: pointer; - position: absolute; - top: 0px; - right: 0px; - border: 1px solid rgb(221, 221, 221); - color: rgb(221, 221, 221); - font-family: monospace; - padding-left: 0.2rem; - padding-right: 0.2rem; -} - -div.highlight:hover span.copybutton::after { - background: #3F556B; - border-radius: 0.25rem; - color: white; - content: attr(title); - padding: 0.25rem; - position: absolute; - z-index: 98; - width: 100px; - font-size: 0.7rem; - top: 0; - right: 0; -} - -/* world */ - -img.avatar { - width: 100%; -} - -/* table */ -table.align-default { - margin-left: auto; - margin-right: auto; -} - -table.docutils tr:nth-child(odd) { - background-color: #F0F7FA; -} - -table.docutils tr { - border-style: solid none solid none; - border-width: 1px 0; - border-color: #ddd; -} - -table.docutils td, table.docutils th { - padding: 0.125rem 0.5rem 0.125rem 0.25rem; -} - -table.docutils { - margin-bottom: 1rem; - line-height: 1rem; - max-width: 100%; - display: block; - overflow-x: scroll; -} - -table.docutils p { - margin-bottom: 0; -} - -table.docutils p { - white-space: pre-wrap; - word-wrap: break-word; - word-break: initial; -} - -/* gallery */ - -div.sphx-glr-thumbcontainer { - min-height: 250px; - font-size: 0.9rem; -} - -.sphx-glr-example-title > :target::before { - display: block; - content: ""; - margin-top: -150px; - height: 150px; - visibility: hidden; -} - -.sphx-glr-script-out .highlight pre { - padding: 1ex; -} - -.sphx-glr-script-out div.highlight { - padding: 0; -} - - -@media screen and (min-width: 1540px) { - .sphx-glr-download-link-note { - position: absolute; - position: absolute; - left: 98%; - width: 20ex; - } -} - -/* Pandas dataframe css */ -/* Taken from: https://github.com/spatialaudio/nbsphinx/blob/fb3ba670fc1ba5f54d4c487573dbc1b4ecf7e9ff/src/nbsphinx.py#L587-L619 */ -/* FIXME: to be removed when sphinx-gallery >= 5.0 will be released */ - -table.dataframe { - border: none !important; - border-collapse: collapse; - border-spacing: 0; - border-color: transparent; - color: black; - font-size: 12px; - table-layout: fixed; -} -table.dataframe thead { - border-bottom: 1px solid black; - vertical-align: bottom; -} -table.dataframe tr, -table.dataframe th, -table.dataframe td { - text-align: right; - vertical-align: middle; - padding: 0.5em 0.5em; - line-height: normal; - white-space: normal; - max-width: none; - border: none; -} -table.dataframe th { - font-weight: bold; -} -table.dataframe tbody tr:nth-child(odd) { - background: #f5f5f5; -} -table.dataframe tbody tr:hover { - background: rgba(66, 165, 245, 0.2); -} - -/* rellinks */ - -.sk-btn-rellink { - background-color: #ff9c34; - border-color: #ff9c34; - color: white; - cursor: pointer; - font-size: 0.8rem; - font-weight: bold; -} - -.sk-btn-rellink:hover { - color: black; - border: 1px solid black; -} - -[sk-rellink-tooltip] { - position: relative; - cursor: pointer; -} - -[sk-rellink-tooltip]::before { - visibility: hidden; - position: absolute; - padding: 0.5rem; - overflow: hidden; - background-color: #ff9c34; - border: 1px solid #ff9c34; - white-space: pre; - content: attr(sk-rellink-tooltip); - text-align: left; - width: 222px; - top: 100%; - left: -78px; - border: 1px solid black; -} - -[sk-rellink-tooltip]:first-child::before { - left: 0; -} - -[sk-rellink-tooltip]:last-child::before { - left: -144px; -} - -[sk-rellink-tooltip]:hover::before { - visibility: visible; - white-space: pre-wrap; - word-wrap: break-word; -} - -/* authors */ -.sk-authors-container { - display: flex; - flex-wrap: wrap; - justify-content: center; -} - -.sk-authors-container > div { - width: 100px; - margin: 5px; - font-size: 0.9rem; -} - - -/* testimonial */ - -div.testimonial h2 { - background-color: transparent; - color: #008EB2; - padding: 0; - height: 26px; - line-height: 1.1em; - font-size: 22px; - font-weight: bold; - text-align: left; -} - -div.testimonial p { - color: #1c1c1c; -} - -div.testimonial span.testimonial-author p { - font-size: 0.8em; - font-style: italic; - color: #808080; -} - -div.testimonial p { - color: #1c1c1c; -} - -/* Installation quickstart */ -/* This quickstart installation is a hack of the awesome - https://spacy.io/usage/#quickstart page. - See the original javascript implementation - https://github.com/ines/quickstart */ - -/* style input radio and checkbox */ - -div.install > input { - -moz-appearance: none; - -webkit-appearance: none; - appearance: none; - opacity: 0; -} - -/* Style the button */ -div.install > label { - display: inline-block; - margin-top: 12px; - padding: 5px 11px; - background-color: #fff3cd; - border: none; - border-radius: 3px; - color: black; -} - -div.install > label:hover { - cursor: pointer; -} - -/* Style the button when the checkbox is checked */ -div.install > input:checked + label { - background-color: #ff9c34; - color: white; -} - -/* Hide expandable content by default */ -.sk-expandable { - display: none; -} - -div.highlight span.sk-expandable:before { - content: "$ "; -} - -/* Show hidden content when the checkbox is checked */ -/* for conda */ -#quickstart-conda:checked ~* [data-packager="conda"] { - display: block; -} - -#quickstart-conda:checked ~ label[for="quickstart-venv"]:before { - content: "Use conda environment"; -} - -/* for pip */ -#quickstart-pip:checked ~* [data-packager="pip"] { - display: block; -} - -#quickstart-pip:checked ~ label[for="quickstart-venv"]:before { - content: "Use pip virtualenv"; -} - -#quickstart-win:not(:checked) ~* [data-os="windows"] { - display: none; -} -#quickstart-lin:not(:checked) ~* [data-os="linux"] { - display: none; -} -#quickstart-mac:not(:checked) ~* [data-os="mac"] { - display: none; -} - -#quickstart-venv:not(:checked) ~* [data-venv=""] { - display: none; -} - -#quickstart-venv:checked ~* [data-venv="no"] { - display: none; -} - -/* Algorithm cheet-sheet */ - -div.sk-page-content img.map { - position: absolute; - max-width: none; - transform-origin: left top; - -webkit-transform: scale(0.5); - -ms-transform: scale(0.5); - transform: scale(0.5); -} - -/* sponsors and testimonials */ - -div.sk-sponsor-div, div.sk-testimonial-div { - display: flex; - flex-wrap: wrap; - -webkit-flex-align: center; - -ms-flex-align: center; - -webkit-align-items: center; - align-items: center; -} - -div.sk-sponsor-div-box, div.sk-testimonial-div-box { - width: 100%; -} - -@media screen and (min-width: 500px) { - div.sk-sponsor-div-box, div.sk-testimonial-div-box { - width: 50%; - } -} - -table.sk-sponsor-table tr, table.sk-sponsor-table tr:nth-child(odd) { - border-style: none; - background-color: white; - vertical-align: middle; - text-align: center; -} - -table.sk-sponsor-table td { - padding: 0.30rem; -} - -.caption { - text-align: center -} - -/* pygments - highlightning */ - -.highlight .hll { background-color: #ffffcc } -.highlight { background: #f8f8f8; } -.highlight .c { color: #408090; font-style: italic } /* Comment */ -.highlight .err { border: 1px solid #FF0000 } /* Error */ -.highlight .k { color: #007020; font-weight: bold } /* Keyword */ -.highlight .o { color: #666666 } /* Operator */ -.highlight .ch { color: #408090; font-style: italic } /* Comment.Hashbang */ -.highlight .cm { color: #408090; font-style: italic } /* Comment.Multiline */ -.highlight .cp { color: #007020 } /* Comment.Preproc */ -.highlight .cpf { color: #408090; font-style: italic } /* Comment.PreprocFile */ -.highlight .c1 { color: #408090; font-style: italic } /* Comment.Single */ -.highlight .cs { color: #408090; background-color: #fff0f0 } /* Comment.Special */ -.highlight .gd { color: #A00000 } /* Generic.Deleted */ -.highlight .ge { font-style: italic } /* Generic.Emph */ -.highlight .gr { color: #FF0000 } /* Generic.Error */ -.highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */ -.highlight .gi { color: #00A000 } /* Generic.Inserted */ -.highlight .go { color: #333333 } /* Generic.Output */ -.highlight .gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */ -.highlight .gs { font-weight: bold } /* Generic.Strong */ -.highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ -.highlight .gt { color: #0044DD } /* Generic.Traceback */ -.highlight .kc { color: #007020; font-weight: bold } /* Keyword.Constant */ -.highlight .kd { color: #007020; font-weight: bold } /* Keyword.Declaration */ -.highlight .kn { color: #007020; font-weight: bold } /* Keyword.Namespace */ -.highlight .kp { color: #007020 } /* Keyword.Pseudo */ -.highlight .kr { color: #007020; font-weight: bold } /* Keyword.Reserved */ -.highlight .kt { color: #902000 } /* Keyword.Type */ -.highlight .m { color: #208050 } /* Literal.Number */ -.highlight .s { color: #4070a0 } /* Literal.String */ -.highlight .na { color: #4070a0 } /* Name.Attribute */ -.highlight .nb { color: #007020 } /* Name.Builtin */ -.highlight .nc { color: #0e84b5; font-weight: bold } /* Name.Class */ -.highlight .no { color: #60add5 } /* Name.Constant */ -.highlight .nd { color: #555555; font-weight: bold } /* Name.Decorator */ -.highlight .ni { color: #d55537; font-weight: bold } /* Name.Entity */ -.highlight .ne { color: #007020 } /* Name.Exception */ -.highlight .nf { color: #06287e } /* Name.Function */ -.highlight .nl { color: #002070; font-weight: bold } /* Name.Label */ -.highlight .nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */ -.highlight .nt { color: #062873; font-weight: bold } /* Name.Tag */ -.highlight .nv { color: #bb60d5 } /* Name.Variable */ -.highlight .ow { color: #007020; font-weight: bold } /* Operator.Word */ -.highlight .w { color: #bbbbbb } /* Text.Whitespace */ -.highlight .mb { color: #208050 } /* Literal.Number.Bin */ -.highlight .mf { color: #208050 } /* Literal.Number.Float */ -.highlight .mh { color: #208050 } /* Literal.Number.Hex */ -.highlight .mi { color: #208050 } /* Literal.Number.Integer */ -.highlight .mo { color: #208050 } /* Literal.Number.Oct */ -.highlight .sa { color: #4070a0 } /* Literal.String.Affix */ -.highlight .sb { color: #4070a0 } /* Literal.String.Backtick */ -.highlight .sc { color: #4070a0 } /* Literal.String.Char */ -.highlight .dl { color: #4070a0 } /* Literal.String.Delimiter */ -.highlight .sd { color: #4070a0; font-style: italic } /* Literal.String.Doc */ -.highlight .s2 { color: #4070a0 } /* Literal.String.Double */ -.highlight .se { color: #4070a0; font-weight: bold } /* Literal.String.Escape */ -.highlight .sh { color: #4070a0 } /* Literal.String.Heredoc */ -.highlight .si { color: #70a0d0; font-style: italic } /* Literal.String.Interpol */ -.highlight .sx { color: #c65d09 } /* Literal.String.Other */ -.highlight .sr { color: #235388 } /* Literal.String.Regex */ -.highlight .s1 { color: #4070a0 } /* Literal.String.Single */ -.highlight .ss { color: #517918 } /* Literal.String.Symbol */ -.highlight .bp { color: #007020 } /* Name.Builtin.Pseudo */ -.highlight .fm { color: #06287e } /* Name.Function.Magic */ -.highlight .vc { color: #bb60d5 } /* Name.Variable.Class */ -.highlight .vg { color: #bb60d5 } /* Name.Variable.Global */ -.highlight .vi { color: #bb60d5 } /* Name.Variable.Instance */ -.highlight .vm { color: #bb60d5 } /* Name.Variable.Magic */ -.highlight .il { color: #208050 } /* Literal.Number.Integer.Long */ diff --git a/doc/themes/scikit-learn-modern/static/css/vendor/bootstrap.min.css b/doc/themes/scikit-learn-modern/static/css/vendor/bootstrap.min.css deleted file mode 100644 index 326cf7fb8aef2..0000000000000 --- a/doc/themes/scikit-learn-modern/static/css/vendor/bootstrap.min.css +++ /dev/null @@ -1,6 +0,0 @@ -/*! - * Bootstrap v4.3.1 (https://getbootstrap.com/) - * Copyright 2011-2019 The Bootstrap Authors - * Copyright 2011-2019 Twitter, Inc. - * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) - */:root{--blue:#007bff;--indigo:#6610f2;--purple:#6f42c1;--pink:#e83e8c;--red:#dc3545;--orange:#fd7e14;--yellow:#ffc107;--green:#28a745;--teal:#20c997;--cyan:#17a2b8;--white:#fff;--gray:#6c757d;--gray-dark:#343a40;--primary:#007bff;--secondary:#6c757d;--success:#28a745;--info:#17a2b8;--warning:#ffc107;--danger:#dc3545;--light:#f8f9fa;--dark:#343a40;--breakpoint-xs:0;--breakpoint-sm:576px;--breakpoint-md:768px;--breakpoint-lg:992px;--breakpoint-xl:1200px;--font-family-sans-serif:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";--font-family-monospace:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace}*,::after,::before{box-sizing:border-box}html{font-family:sans-serif;line-height:1.15;-webkit-text-size-adjust:100%;-webkit-tap-highlight-color:transparent}article,aside,figcaption,figure,footer,header,hgroup,main,nav,section{display:block}body{margin:0;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-size:1rem;font-weight:400;line-height:1.5;color:#212529;text-align:left;background-color:#fff}[tabindex="-1"]:focus{outline:0!important}hr{box-sizing:content-box;height:0;overflow:visible}h1,h2,h3,h4,h5,h6{margin-top:0;margin-bottom:.5rem}p{margin-top:0;margin-bottom:1rem}abbr[data-original-title],abbr[title]{text-decoration:underline;-webkit-text-decoration:underline dotted;text-decoration:underline dotted;cursor:help;border-bottom:0;-webkit-text-decoration-skip-ink:none;text-decoration-skip-ink:none}address{margin-bottom:1rem;font-style:normal;line-height:inherit}dl,ol,ul{margin-top:0;margin-bottom:1rem}ol ol,ol ul,ul ol,ul ul{margin-bottom:0}dt{font-weight:700}dd{margin-bottom:.5rem;margin-left:0}blockquote{margin:0 0 1rem}b,strong{font-weight:bolder}small{font-size:80%}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}a{color:#007bff;text-decoration:none;background-color:transparent}a:hover{color:#0056b3;text-decoration:underline}a:not([href]):not([tabindex]){color:inherit;text-decoration:none}a:not([href]):not([tabindex]):focus,a:not([href]):not([tabindex]):hover{color:inherit;text-decoration:none}a:not([href]):not([tabindex]):focus{outline:0}code,kbd,pre,samp{font-family:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;font-size:1em}pre{margin-top:0;margin-bottom:1rem;overflow:auto}figure{margin:0 0 1rem}img{vertical-align:middle;border-style:none}svg{overflow:hidden;vertical-align:middle}table{border-collapse:collapse}caption{padding-top:.75rem;padding-bottom:.75rem;color:#6c757d;text-align:left;caption-side:bottom}th{text-align:inherit}label{display:inline-block;margin-bottom:.5rem}button{border-radius:0}button:focus{outline:1px dotted;outline:5px auto -webkit-focus-ring-color}button,input,optgroup,select,textarea{margin:0;font-family:inherit;font-size:inherit;line-height:inherit}button,input{overflow:visible}button,select{text-transform:none}select{word-wrap:normal}[type=button],[type=reset],[type=submit],button{-webkit-appearance:button}[type=button]:not(:disabled),[type=reset]:not(:disabled),[type=submit]:not(:disabled),button:not(:disabled){cursor:pointer}[type=button]::-moz-focus-inner,[type=reset]::-moz-focus-inner,[type=submit]::-moz-focus-inner,button::-moz-focus-inner{padding:0;border-style:none}input[type=checkbox],input[type=radio]{box-sizing:border-box;padding:0}input[type=date],input[type=datetime-local],input[type=month],input[type=time]{-webkit-appearance:listbox}textarea{overflow:auto;resize:vertical}fieldset{min-width:0;padding:0;margin:0;border:0}legend{display:block;width:100%;max-width:100%;padding:0;margin-bottom:.5rem;font-size:1.5rem;line-height:inherit;color:inherit;white-space:normal}progress{vertical-align:baseline}[type=number]::-webkit-inner-spin-button,[type=number]::-webkit-outer-spin-button{height:auto}[type=search]{outline-offset:-2px;-webkit-appearance:none}[type=search]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{font:inherit;-webkit-appearance:button}output{display:inline-block}summary{display:list-item;cursor:pointer}template{display:none}[hidden]{display:none!important}.h1,.h2,.h3,.h4,.h5,.h6,h1,h2,h3,h4,h5,h6{margin-bottom:.5rem;font-weight:500;line-height:1.2}.h1,h1{font-size:2.5rem}.h2,h2{font-size:2rem}.h3,h3{font-size:1.75rem}.h4,h4{font-size:1.5rem}.h5,h5{font-size:1.25rem}.h6,h6{font-size:1rem}.lead{font-size:1.25rem;font-weight:300}.display-1{font-size:6rem;font-weight:300;line-height:1.2}.display-2{font-size:5.5rem;font-weight:300;line-height:1.2}.display-3{font-size:4.5rem;font-weight:300;line-height:1.2}.display-4{font-size:3.5rem;font-weight:300;line-height:1.2}hr{margin-top:1rem;margin-bottom:1rem;border:0;border-top:1px solid rgba(0,0,0,.1)}.small,small{font-size:80%;font-weight:400}.mark,mark{padding:.2em;background-color:#fcf8e3}.list-unstyled{padding-left:0;list-style:none}.list-inline{padding-left:0;list-style:none}.list-inline-item{display:inline-block}.list-inline-item:not(:last-child){margin-right:.5rem}.initialism{font-size:90%;text-transform:uppercase}.blockquote{margin-bottom:1rem;font-size:1.25rem}.blockquote-footer{display:block;font-size:80%;color:#6c757d}.blockquote-footer::before{content:"\2014\00A0"}.img-fluid{max-width:100%;height:auto}.img-thumbnail{padding:.25rem;background-color:#fff;border:1px solid #dee2e6;border-radius:.25rem;max-width:100%;height:auto}.figure{display:inline-block}.figure-img{margin-bottom:.5rem;line-height:1}.figure-caption{font-size:90%;color:#6c757d}code{font-size:87.5%;color:#e83e8c;word-break:break-word}a>code{color:inherit}kbd{padding:.2rem .4rem;font-size:87.5%;color:#fff;background-color:#212529;border-radius:.2rem}kbd kbd{padding:0;font-size:100%;font-weight:700}pre{display:block;font-size:87.5%;color:#212529}pre code{font-size:inherit;color:inherit;word-break:normal}.pre-scrollable{max-height:340px;overflow-y:scroll}.container{width:100%;padding-right:15px;padding-left:15px;margin-right:auto;margin-left:auto}@media (min-width:576px){.container{max-width:540px}}@media (min-width:768px){.container{max-width:720px}}@media (min-width:992px){.container{max-width:960px}}@media (min-width:1200px){.container{max-width:1140px}}.container-fluid{width:100%;padding-right:15px;padding-left:15px;margin-right:auto;margin-left:auto}.row{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;margin-right:-15px;margin-left:-15px}.no-gutters{margin-right:0;margin-left:0}.no-gutters>.col,.no-gutters>[class*=col-]{padding-right:0;padding-left:0}.col,.col-1,.col-10,.col-11,.col-12,.col-2,.col-3,.col-4,.col-5,.col-6,.col-7,.col-8,.col-9,.col-auto,.col-lg,.col-lg-1,.col-lg-10,.col-lg-11,.col-lg-12,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9,.col-lg-auto,.col-md,.col-md-1,.col-md-10,.col-md-11,.col-md-12,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9,.col-md-auto,.col-sm,.col-sm-1,.col-sm-10,.col-sm-11,.col-sm-12,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9,.col-sm-auto,.col-xl,.col-xl-1,.col-xl-10,.col-xl-11,.col-xl-12,.col-xl-2,.col-xl-3,.col-xl-4,.col-xl-5,.col-xl-6,.col-xl-7,.col-xl-8,.col-xl-9,.col-xl-auto{position:relative;width:100%;padding-right:15px;padding-left:15px}.col{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;max-width:100%}.col-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-first{-ms-flex-order:-1;order:-1}.order-last{-ms-flex-order:13;order:13}.order-0{-ms-flex-order:0;order:0}.order-1{-ms-flex-order:1;order:1}.order-2{-ms-flex-order:2;order:2}.order-3{-ms-flex-order:3;order:3}.order-4{-ms-flex-order:4;order:4}.order-5{-ms-flex-order:5;order:5}.order-6{-ms-flex-order:6;order:6}.order-7{-ms-flex-order:7;order:7}.order-8{-ms-flex-order:8;order:8}.order-9{-ms-flex-order:9;order:9}.order-10{-ms-flex-order:10;order:10}.order-11{-ms-flex-order:11;order:11}.order-12{-ms-flex-order:12;order:12}.offset-1{margin-left:8.333333%}.offset-2{margin-left:16.666667%}.offset-3{margin-left:25%}.offset-4{margin-left:33.333333%}.offset-5{margin-left:41.666667%}.offset-6{margin-left:50%}.offset-7{margin-left:58.333333%}.offset-8{margin-left:66.666667%}.offset-9{margin-left:75%}.offset-10{margin-left:83.333333%}.offset-11{margin-left:91.666667%}@media (min-width:576px){.col-sm{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;max-width:100%}.col-sm-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-sm-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-sm-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-sm-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-sm-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-sm-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-sm-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-sm-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-sm-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-sm-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-sm-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-sm-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-sm-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-sm-first{-ms-flex-order:-1;order:-1}.order-sm-last{-ms-flex-order:13;order:13}.order-sm-0{-ms-flex-order:0;order:0}.order-sm-1{-ms-flex-order:1;order:1}.order-sm-2{-ms-flex-order:2;order:2}.order-sm-3{-ms-flex-order:3;order:3}.order-sm-4{-ms-flex-order:4;order:4}.order-sm-5{-ms-flex-order:5;order:5}.order-sm-6{-ms-flex-order:6;order:6}.order-sm-7{-ms-flex-order:7;order:7}.order-sm-8{-ms-flex-order:8;order:8}.order-sm-9{-ms-flex-order:9;order:9}.order-sm-10{-ms-flex-order:10;order:10}.order-sm-11{-ms-flex-order:11;order:11}.order-sm-12{-ms-flex-order:12;order:12}.offset-sm-0{margin-left:0}.offset-sm-1{margin-left:8.333333%}.offset-sm-2{margin-left:16.666667%}.offset-sm-3{margin-left:25%}.offset-sm-4{margin-left:33.333333%}.offset-sm-5{margin-left:41.666667%}.offset-sm-6{margin-left:50%}.offset-sm-7{margin-left:58.333333%}.offset-sm-8{margin-left:66.666667%}.offset-sm-9{margin-left:75%}.offset-sm-10{margin-left:83.333333%}.offset-sm-11{margin-left:91.666667%}}@media (min-width:768px){.col-md{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;max-width:100%}.col-md-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-md-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-md-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-md-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-md-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-md-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-md-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-md-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-md-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-md-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-md-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-md-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-md-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-md-first{-ms-flex-order:-1;order:-1}.order-md-last{-ms-flex-order:13;order:13}.order-md-0{-ms-flex-order:0;order:0}.order-md-1{-ms-flex-order:1;order:1}.order-md-2{-ms-flex-order:2;order:2}.order-md-3{-ms-flex-order:3;order:3}.order-md-4{-ms-flex-order:4;order:4}.order-md-5{-ms-flex-order:5;order:5}.order-md-6{-ms-flex-order:6;order:6}.order-md-7{-ms-flex-order:7;order:7}.order-md-8{-ms-flex-order:8;order:8}.order-md-9{-ms-flex-order:9;order:9}.order-md-10{-ms-flex-order:10;order:10}.order-md-11{-ms-flex-order:11;order:11}.order-md-12{-ms-flex-order:12;order:12}.offset-md-0{margin-left:0}.offset-md-1{margin-left:8.333333%}.offset-md-2{margin-left:16.666667%}.offset-md-3{margin-left:25%}.offset-md-4{margin-left:33.333333%}.offset-md-5{margin-left:41.666667%}.offset-md-6{margin-left:50%}.offset-md-7{margin-left:58.333333%}.offset-md-8{margin-left:66.666667%}.offset-md-9{margin-left:75%}.offset-md-10{margin-left:83.333333%}.offset-md-11{margin-left:91.666667%}}@media (min-width:992px){.col-lg{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;max-width:100%}.col-lg-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-lg-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-lg-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-lg-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-lg-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-lg-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-lg-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-lg-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-lg-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-lg-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-lg-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-lg-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-lg-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-lg-first{-ms-flex-order:-1;order:-1}.order-lg-last{-ms-flex-order:13;order:13}.order-lg-0{-ms-flex-order:0;order:0}.order-lg-1{-ms-flex-order:1;order:1}.order-lg-2{-ms-flex-order:2;order:2}.order-lg-3{-ms-flex-order:3;order:3}.order-lg-4{-ms-flex-order:4;order:4}.order-lg-5{-ms-flex-order:5;order:5}.order-lg-6{-ms-flex-order:6;order:6}.order-lg-7{-ms-flex-order:7;order:7}.order-lg-8{-ms-flex-order:8;order:8}.order-lg-9{-ms-flex-order:9;order:9}.order-lg-10{-ms-flex-order:10;order:10}.order-lg-11{-ms-flex-order:11;order:11}.order-lg-12{-ms-flex-order:12;order:12}.offset-lg-0{margin-left:0}.offset-lg-1{margin-left:8.333333%}.offset-lg-2{margin-left:16.666667%}.offset-lg-3{margin-left:25%}.offset-lg-4{margin-left:33.333333%}.offset-lg-5{margin-left:41.666667%}.offset-lg-6{margin-left:50%}.offset-lg-7{margin-left:58.333333%}.offset-lg-8{margin-left:66.666667%}.offset-lg-9{margin-left:75%}.offset-lg-10{margin-left:83.333333%}.offset-lg-11{margin-left:91.666667%}}@media (min-width:1200px){.col-xl{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;max-width:100%}.col-xl-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-xl-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-xl-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-xl-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-xl-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-xl-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-xl-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-xl-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-xl-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-xl-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-xl-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-xl-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-xl-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-xl-first{-ms-flex-order:-1;order:-1}.order-xl-last{-ms-flex-order:13;order:13}.order-xl-0{-ms-flex-order:0;order:0}.order-xl-1{-ms-flex-order:1;order:1}.order-xl-2{-ms-flex-order:2;order:2}.order-xl-3{-ms-flex-order:3;order:3}.order-xl-4{-ms-flex-order:4;order:4}.order-xl-5{-ms-flex-order:5;order:5}.order-xl-6{-ms-flex-order:6;order:6}.order-xl-7{-ms-flex-order:7;order:7}.order-xl-8{-ms-flex-order:8;order:8}.order-xl-9{-ms-flex-order:9;order:9}.order-xl-10{-ms-flex-order:10;order:10}.order-xl-11{-ms-flex-order:11;order:11}.order-xl-12{-ms-flex-order:12;order:12}.offset-xl-0{margin-left:0}.offset-xl-1{margin-left:8.333333%}.offset-xl-2{margin-left:16.666667%}.offset-xl-3{margin-left:25%}.offset-xl-4{margin-left:33.333333%}.offset-xl-5{margin-left:41.666667%}.offset-xl-6{margin-left:50%}.offset-xl-7{margin-left:58.333333%}.offset-xl-8{margin-left:66.666667%}.offset-xl-9{margin-left:75%}.offset-xl-10{margin-left:83.333333%}.offset-xl-11{margin-left:91.666667%}}.table{width:100%;margin-bottom:1rem;color:#212529}.table td,.table th{padding:.75rem;vertical-align:top;border-top:1px solid #dee2e6}.table thead th{vertical-align:bottom;border-bottom:2px solid #dee2e6}.table tbody+tbody{border-top:2px solid #dee2e6}.table-sm td,.table-sm th{padding:.3rem}.table-bordered{border:1px solid #dee2e6}.table-bordered td,.table-bordered th{border:1px solid #dee2e6}.table-bordered thead td,.table-bordered thead th{border-bottom-width:2px}.table-borderless tbody+tbody,.table-borderless td,.table-borderless th,.table-borderless thead th{border:0}.table-striped tbody tr:nth-of-type(odd){background-color:rgba(0,0,0,.05)}.table-hover tbody tr:hover{color:#212529;background-color:rgba(0,0,0,.075)}.table-primary,.table-primary>td,.table-primary>th{background-color:#b8daff}.table-primary tbody+tbody,.table-primary td,.table-primary th,.table-primary thead th{border-color:#7abaff}.table-hover .table-primary:hover{background-color:#9fcdff}.table-hover .table-primary:hover>td,.table-hover .table-primary:hover>th{background-color:#9fcdff}.table-secondary,.table-secondary>td,.table-secondary>th{background-color:#d6d8db}.table-secondary tbody+tbody,.table-secondary td,.table-secondary th,.table-secondary thead th{border-color:#b3b7bb}.table-hover .table-secondary:hover{background-color:#c8cbcf}.table-hover .table-secondary:hover>td,.table-hover .table-secondary:hover>th{background-color:#c8cbcf}.table-success,.table-success>td,.table-success>th{background-color:#c3e6cb}.table-success tbody+tbody,.table-success td,.table-success th,.table-success thead th{border-color:#8fd19e}.table-hover .table-success:hover{background-color:#b1dfbb}.table-hover .table-success:hover>td,.table-hover .table-success:hover>th{background-color:#b1dfbb}.table-info,.table-info>td,.table-info>th{background-color:#bee5eb}.table-info tbody+tbody,.table-info td,.table-info th,.table-info thead th{border-color:#86cfda}.table-hover .table-info:hover{background-color:#abdde5}.table-hover .table-info:hover>td,.table-hover .table-info:hover>th{background-color:#abdde5}.table-warning,.table-warning>td,.table-warning>th{background-color:#ffeeba}.table-warning tbody+tbody,.table-warning td,.table-warning th,.table-warning thead th{border-color:#ffdf7e}.table-hover .table-warning:hover{background-color:#ffe8a1}.table-hover .table-warning:hover>td,.table-hover .table-warning:hover>th{background-color:#ffe8a1}.table-danger,.table-danger>td,.table-danger>th{background-color:#f5c6cb}.table-danger tbody+tbody,.table-danger td,.table-danger th,.table-danger thead th{border-color:#ed969e}.table-hover .table-danger:hover{background-color:#f1b0b7}.table-hover .table-danger:hover>td,.table-hover .table-danger:hover>th{background-color:#f1b0b7}.table-light,.table-light>td,.table-light>th{background-color:#fdfdfe}.table-light tbody+tbody,.table-light td,.table-light th,.table-light thead th{border-color:#fbfcfc}.table-hover .table-light:hover{background-color:#ececf6}.table-hover .table-light:hover>td,.table-hover .table-light:hover>th{background-color:#ececf6}.table-dark,.table-dark>td,.table-dark>th{background-color:#c6c8ca}.table-dark tbody+tbody,.table-dark td,.table-dark th,.table-dark thead th{border-color:#95999c}.table-hover .table-dark:hover{background-color:#b9bbbe}.table-hover .table-dark:hover>td,.table-hover .table-dark:hover>th{background-color:#b9bbbe}.table-active,.table-active>td,.table-active>th{background-color:rgba(0,0,0,.075)}.table-hover .table-active:hover{background-color:rgba(0,0,0,.075)}.table-hover .table-active:hover>td,.table-hover .table-active:hover>th{background-color:rgba(0,0,0,.075)}.table .thead-dark th{color:#fff;background-color:#343a40;border-color:#454d55}.table .thead-light th{color:#495057;background-color:#e9ecef;border-color:#dee2e6}.table-dark{color:#fff;background-color:#343a40}.table-dark td,.table-dark th,.table-dark thead th{border-color:#454d55}.table-dark.table-bordered{border:0}.table-dark.table-striped tbody tr:nth-of-type(odd){background-color:rgba(255,255,255,.05)}.table-dark.table-hover tbody tr:hover{color:#fff;background-color:rgba(255,255,255,.075)}@media (max-width:575.98px){.table-responsive-sm{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-sm>.table-bordered{border:0}}@media (max-width:767.98px){.table-responsive-md{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-md>.table-bordered{border:0}}@media (max-width:991.98px){.table-responsive-lg{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-lg>.table-bordered{border:0}}@media (max-width:1199.98px){.table-responsive-xl{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-xl>.table-bordered{border:0}}.table-responsive{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive>.table-bordered{border:0}.form-control{display:block;width:100%;height:calc(1.5em + .75rem + 2px);padding:.375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:#495057;background-color:#fff;background-clip:padding-box;border:1px solid #ced4da;border-radius:.25rem;transition:border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.form-control{transition:none}}.form-control::-ms-expand{background-color:transparent;border:0}.form-control:focus{color:#495057;background-color:#fff;border-color:#80bdff;outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.form-control::-webkit-input-placeholder{color:#6c757d;opacity:1}.form-control::-moz-placeholder{color:#6c757d;opacity:1}.form-control:-ms-input-placeholder{color:#6c757d;opacity:1}.form-control::-ms-input-placeholder{color:#6c757d;opacity:1}.form-control::placeholder{color:#6c757d;opacity:1}.form-control:disabled,.form-control[readonly]{background-color:#e9ecef;opacity:1}select.form-control:focus::-ms-value{color:#495057;background-color:#fff}.form-control-file,.form-control-range{display:block;width:100%}.col-form-label{padding-top:calc(.375rem + 1px);padding-bottom:calc(.375rem + 1px);margin-bottom:0;font-size:inherit;line-height:1.5}.col-form-label-lg{padding-top:calc(.5rem + 1px);padding-bottom:calc(.5rem + 1px);font-size:1.25rem;line-height:1.5}.col-form-label-sm{padding-top:calc(.25rem + 1px);padding-bottom:calc(.25rem + 1px);font-size:.875rem;line-height:1.5}.form-control-plaintext{display:block;width:100%;padding-top:.375rem;padding-bottom:.375rem;margin-bottom:0;line-height:1.5;color:#212529;background-color:transparent;border:solid transparent;border-width:1px 0}.form-control-plaintext.form-control-lg,.form-control-plaintext.form-control-sm{padding-right:0;padding-left:0}.form-control-sm{height:calc(1.5em + .5rem + 2px);padding:.25rem .5rem;font-size:.875rem;line-height:1.5;border-radius:.2rem}.form-control-lg{height:calc(1.5em + 1rem + 2px);padding:.5rem 1rem;font-size:1.25rem;line-height:1.5;border-radius:.3rem}select.form-control[multiple],select.form-control[size]{height:auto}textarea.form-control{height:auto}.form-group{margin-bottom:1rem}.form-text{display:block;margin-top:.25rem}.form-row{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;margin-right:-5px;margin-left:-5px}.form-row>.col,.form-row>[class*=col-]{padding-right:5px;padding-left:5px}.form-check{position:relative;display:block;padding-left:1.25rem}.form-check-input{position:absolute;margin-top:.3rem;margin-left:-1.25rem}.form-check-input:disabled~.form-check-label{color:#6c757d}.form-check-label{margin-bottom:0}.form-check-inline{display:-ms-inline-flexbox;display:inline-flex;-ms-flex-align:center;align-items:center;padding-left:0;margin-right:.75rem}.form-check-inline .form-check-input{position:static;margin-top:0;margin-right:.3125rem;margin-left:0}.valid-feedback{display:none;width:100%;margin-top:.25rem;font-size:80%;color:#28a745}.valid-tooltip{position:absolute;top:100%;z-index:5;display:none;max-width:100%;padding:.25rem .5rem;margin-top:.1rem;font-size:.875rem;line-height:1.5;color:#fff;background-color:rgba(40,167,69,.9);border-radius:.25rem}.form-control.is-valid,.was-validated .form-control:valid{border-color:#28a745;padding-right:calc(1.5em + .75rem);background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 8 8'%3e%3cpath fill='%2328a745' d='M2.3 6.73L.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e");background-repeat:no-repeat;background-position:center right calc(.375em + .1875rem);background-size:calc(.75em + .375rem) calc(.75em + .375rem)}.form-control.is-valid:focus,.was-validated .form-control:valid:focus{border-color:#28a745;box-shadow:0 0 0 .2rem rgba(40,167,69,.25)}.form-control.is-valid~.valid-feedback,.form-control.is-valid~.valid-tooltip,.was-validated .form-control:valid~.valid-feedback,.was-validated .form-control:valid~.valid-tooltip{display:block}.was-validated textarea.form-control:valid,textarea.form-control.is-valid{padding-right:calc(1.5em + .75rem);background-position:top calc(.375em + .1875rem) right calc(.375em + .1875rem)}.custom-select.is-valid,.was-validated .custom-select:valid{border-color:#28a745;padding-right:calc((1em + .75rem) * 3 / 4 + 1.75rem);background:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e") no-repeat right .75rem center/8px 10px,url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 8 8'%3e%3cpath fill='%2328a745' d='M2.3 6.73L.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e") #fff no-repeat center right 1.75rem/calc(.75em + .375rem) calc(.75em + .375rem)}.custom-select.is-valid:focus,.was-validated .custom-select:valid:focus{border-color:#28a745;box-shadow:0 0 0 .2rem rgba(40,167,69,.25)}.custom-select.is-valid~.valid-feedback,.custom-select.is-valid~.valid-tooltip,.was-validated .custom-select:valid~.valid-feedback,.was-validated .custom-select:valid~.valid-tooltip{display:block}.form-control-file.is-valid~.valid-feedback,.form-control-file.is-valid~.valid-tooltip,.was-validated .form-control-file:valid~.valid-feedback,.was-validated .form-control-file:valid~.valid-tooltip{display:block}.form-check-input.is-valid~.form-check-label,.was-validated .form-check-input:valid~.form-check-label{color:#28a745}.form-check-input.is-valid~.valid-feedback,.form-check-input.is-valid~.valid-tooltip,.was-validated .form-check-input:valid~.valid-feedback,.was-validated .form-check-input:valid~.valid-tooltip{display:block}.custom-control-input.is-valid~.custom-control-label,.was-validated .custom-control-input:valid~.custom-control-label{color:#28a745}.custom-control-input.is-valid~.custom-control-label::before,.was-validated .custom-control-input:valid~.custom-control-label::before{border-color:#28a745}.custom-control-input.is-valid~.valid-feedback,.custom-control-input.is-valid~.valid-tooltip,.was-validated .custom-control-input:valid~.valid-feedback,.was-validated .custom-control-input:valid~.valid-tooltip{display:block}.custom-control-input.is-valid:checked~.custom-control-label::before,.was-validated .custom-control-input:valid:checked~.custom-control-label::before{border-color:#34ce57;background-color:#34ce57}.custom-control-input.is-valid:focus~.custom-control-label::before,.was-validated .custom-control-input:valid:focus~.custom-control-label::before{box-shadow:0 0 0 .2rem rgba(40,167,69,.25)}.custom-control-input.is-valid:focus:not(:checked)~.custom-control-label::before,.was-validated .custom-control-input:valid:focus:not(:checked)~.custom-control-label::before{border-color:#28a745}.custom-file-input.is-valid~.custom-file-label,.was-validated .custom-file-input:valid~.custom-file-label{border-color:#28a745}.custom-file-input.is-valid~.valid-feedback,.custom-file-input.is-valid~.valid-tooltip,.was-validated .custom-file-input:valid~.valid-feedback,.was-validated .custom-file-input:valid~.valid-tooltip{display:block}.custom-file-input.is-valid:focus~.custom-file-label,.was-validated .custom-file-input:valid:focus~.custom-file-label{border-color:#28a745;box-shadow:0 0 0 .2rem rgba(40,167,69,.25)}.invalid-feedback{display:none;width:100%;margin-top:.25rem;font-size:80%;color:#dc3545}.invalid-tooltip{position:absolute;top:100%;z-index:5;display:none;max-width:100%;padding:.25rem .5rem;margin-top:.1rem;font-size:.875rem;line-height:1.5;color:#fff;background-color:rgba(220,53,69,.9);border-radius:.25rem}.form-control.is-invalid,.was-validated .form-control:invalid{border-color:#dc3545;padding-right:calc(1.5em + .75rem);background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='%23dc3545' viewBox='-2 -2 7 7'%3e%3cpath stroke='%23dc3545' d='M0 0l3 3m0-3L0 3'/%3e%3ccircle r='.5'/%3e%3ccircle cx='3' r='.5'/%3e%3ccircle cy='3' r='.5'/%3e%3ccircle cx='3' cy='3' r='.5'/%3e%3c/svg%3E");background-repeat:no-repeat;background-position:center right calc(.375em + .1875rem);background-size:calc(.75em + .375rem) calc(.75em + .375rem)}.form-control.is-invalid:focus,.was-validated .form-control:invalid:focus{border-color:#dc3545;box-shadow:0 0 0 .2rem rgba(220,53,69,.25)}.form-control.is-invalid~.invalid-feedback,.form-control.is-invalid~.invalid-tooltip,.was-validated .form-control:invalid~.invalid-feedback,.was-validated .form-control:invalid~.invalid-tooltip{display:block}.was-validated textarea.form-control:invalid,textarea.form-control.is-invalid{padding-right:calc(1.5em + .75rem);background-position:top calc(.375em + .1875rem) right calc(.375em + .1875rem)}.custom-select.is-invalid,.was-validated .custom-select:invalid{border-color:#dc3545;padding-right:calc((1em + .75rem) * 3 / 4 + 1.75rem);background:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e") no-repeat right .75rem center/8px 10px,url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='%23dc3545' viewBox='-2 -2 7 7'%3e%3cpath stroke='%23dc3545' d='M0 0l3 3m0-3L0 3'/%3e%3ccircle r='.5'/%3e%3ccircle cx='3' r='.5'/%3e%3ccircle cy='3' r='.5'/%3e%3ccircle cx='3' cy='3' r='.5'/%3e%3c/svg%3E") #fff no-repeat center right 1.75rem/calc(.75em + .375rem) calc(.75em + .375rem)}.custom-select.is-invalid:focus,.was-validated .custom-select:invalid:focus{border-color:#dc3545;box-shadow:0 0 0 .2rem rgba(220,53,69,.25)}.custom-select.is-invalid~.invalid-feedback,.custom-select.is-invalid~.invalid-tooltip,.was-validated .custom-select:invalid~.invalid-feedback,.was-validated .custom-select:invalid~.invalid-tooltip{display:block}.form-control-file.is-invalid~.invalid-feedback,.form-control-file.is-invalid~.invalid-tooltip,.was-validated .form-control-file:invalid~.invalid-feedback,.was-validated .form-control-file:invalid~.invalid-tooltip{display:block}.form-check-input.is-invalid~.form-check-label,.was-validated .form-check-input:invalid~.form-check-label{color:#dc3545}.form-check-input.is-invalid~.invalid-feedback,.form-check-input.is-invalid~.invalid-tooltip,.was-validated .form-check-input:invalid~.invalid-feedback,.was-validated .form-check-input:invalid~.invalid-tooltip{display:block}.custom-control-input.is-invalid~.custom-control-label,.was-validated .custom-control-input:invalid~.custom-control-label{color:#dc3545}.custom-control-input.is-invalid~.custom-control-label::before,.was-validated .custom-control-input:invalid~.custom-control-label::before{border-color:#dc3545}.custom-control-input.is-invalid~.invalid-feedback,.custom-control-input.is-invalid~.invalid-tooltip,.was-validated .custom-control-input:invalid~.invalid-feedback,.was-validated .custom-control-input:invalid~.invalid-tooltip{display:block}.custom-control-input.is-invalid:checked~.custom-control-label::before,.was-validated .custom-control-input:invalid:checked~.custom-control-label::before{border-color:#e4606d;background-color:#e4606d}.custom-control-input.is-invalid:focus~.custom-control-label::before,.was-validated .custom-control-input:invalid:focus~.custom-control-label::before{box-shadow:0 0 0 .2rem rgba(220,53,69,.25)}.custom-control-input.is-invalid:focus:not(:checked)~.custom-control-label::before,.was-validated .custom-control-input:invalid:focus:not(:checked)~.custom-control-label::before{border-color:#dc3545}.custom-file-input.is-invalid~.custom-file-label,.was-validated .custom-file-input:invalid~.custom-file-label{border-color:#dc3545}.custom-file-input.is-invalid~.invalid-feedback,.custom-file-input.is-invalid~.invalid-tooltip,.was-validated .custom-file-input:invalid~.invalid-feedback,.was-validated .custom-file-input:invalid~.invalid-tooltip{display:block}.custom-file-input.is-invalid:focus~.custom-file-label,.was-validated .custom-file-input:invalid:focus~.custom-file-label{border-color:#dc3545;box-shadow:0 0 0 .2rem rgba(220,53,69,.25)}.form-inline{display:-ms-flexbox;display:flex;-ms-flex-flow:row wrap;flex-flow:row wrap;-ms-flex-align:center;align-items:center}.form-inline .form-check{width:100%}@media (min-width:576px){.form-inline label{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center;margin-bottom:0}.form-inline .form-group{display:-ms-flexbox;display:flex;-ms-flex:0 0 auto;flex:0 0 auto;-ms-flex-flow:row wrap;flex-flow:row wrap;-ms-flex-align:center;align-items:center;margin-bottom:0}.form-inline .form-control{display:inline-block;width:auto;vertical-align:middle}.form-inline .form-control-plaintext{display:inline-block}.form-inline .custom-select,.form-inline .input-group{width:auto}.form-inline .form-check{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center;width:auto;padding-left:0}.form-inline .form-check-input{position:relative;-ms-flex-negative:0;flex-shrink:0;margin-top:0;margin-right:.25rem;margin-left:0}.form-inline .custom-control{-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center}.form-inline .custom-control-label{margin-bottom:0}}.btn{display:inline-block;font-weight:400;color:#212529;text-align:center;vertical-align:middle;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;background-color:transparent;border:1px solid transparent;padding:.375rem .75rem;font-size:1rem;line-height:1.5;border-radius:.25rem;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.btn{transition:none}}.btn:hover{color:#212529;text-decoration:none}.btn.focus,.btn:focus{outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.btn.disabled,.btn:disabled{opacity:.65}a.btn.disabled,fieldset:disabled a.btn{pointer-events:none}.btn-primary{color:#fff;background-color:#007bff;border-color:#007bff}.btn-primary:hover{color:#fff;background-color:#0069d9;border-color:#0062cc}.btn-primary.focus,.btn-primary:focus{box-shadow:0 0 0 .2rem rgba(38,143,255,.5)}.btn-primary.disabled,.btn-primary:disabled{color:#fff;background-color:#007bff;border-color:#007bff}.btn-primary:not(:disabled):not(.disabled).active,.btn-primary:not(:disabled):not(.disabled):active,.show>.btn-primary.dropdown-toggle{color:#fff;background-color:#0062cc;border-color:#005cbf}.btn-primary:not(:disabled):not(.disabled).active:focus,.btn-primary:not(:disabled):not(.disabled):active:focus,.show>.btn-primary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(38,143,255,.5)}.btn-secondary{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-secondary:hover{color:#fff;background-color:#5a6268;border-color:#545b62}.btn-secondary.focus,.btn-secondary:focus{box-shadow:0 0 0 .2rem rgba(130,138,145,.5)}.btn-secondary.disabled,.btn-secondary:disabled{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-secondary:not(:disabled):not(.disabled).active,.btn-secondary:not(:disabled):not(.disabled):active,.show>.btn-secondary.dropdown-toggle{color:#fff;background-color:#545b62;border-color:#4e555b}.btn-secondary:not(:disabled):not(.disabled).active:focus,.btn-secondary:not(:disabled):not(.disabled):active:focus,.show>.btn-secondary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(130,138,145,.5)}.btn-success{color:#fff;background-color:#28a745;border-color:#28a745}.btn-success:hover{color:#fff;background-color:#218838;border-color:#1e7e34}.btn-success.focus,.btn-success:focus{box-shadow:0 0 0 .2rem rgba(72,180,97,.5)}.btn-success.disabled,.btn-success:disabled{color:#fff;background-color:#28a745;border-color:#28a745}.btn-success:not(:disabled):not(.disabled).active,.btn-success:not(:disabled):not(.disabled):active,.show>.btn-success.dropdown-toggle{color:#fff;background-color:#1e7e34;border-color:#1c7430}.btn-success:not(:disabled):not(.disabled).active:focus,.btn-success:not(:disabled):not(.disabled):active:focus,.show>.btn-success.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(72,180,97,.5)}.btn-info{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-info:hover{color:#fff;background-color:#138496;border-color:#117a8b}.btn-info.focus,.btn-info:focus{box-shadow:0 0 0 .2rem rgba(58,176,195,.5)}.btn-info.disabled,.btn-info:disabled{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-info:not(:disabled):not(.disabled).active,.btn-info:not(:disabled):not(.disabled):active,.show>.btn-info.dropdown-toggle{color:#fff;background-color:#117a8b;border-color:#10707f}.btn-info:not(:disabled):not(.disabled).active:focus,.btn-info:not(:disabled):not(.disabled):active:focus,.show>.btn-info.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(58,176,195,.5)}.btn-warning{color:#212529;background-color:#ffc107;border-color:#ffc107}.btn-warning:hover{color:#212529;background-color:#e0a800;border-color:#d39e00}.btn-warning.focus,.btn-warning:focus{box-shadow:0 0 0 .2rem rgba(222,170,12,.5)}.btn-warning.disabled,.btn-warning:disabled{color:#212529;background-color:#ffc107;border-color:#ffc107}.btn-warning:not(:disabled):not(.disabled).active,.btn-warning:not(:disabled):not(.disabled):active,.show>.btn-warning.dropdown-toggle{color:#212529;background-color:#d39e00;border-color:#c69500}.btn-warning:not(:disabled):not(.disabled).active:focus,.btn-warning:not(:disabled):not(.disabled):active:focus,.show>.btn-warning.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(222,170,12,.5)}.btn-danger{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-danger:hover{color:#fff;background-color:#c82333;border-color:#bd2130}.btn-danger.focus,.btn-danger:focus{box-shadow:0 0 0 .2rem rgba(225,83,97,.5)}.btn-danger.disabled,.btn-danger:disabled{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-danger:not(:disabled):not(.disabled).active,.btn-danger:not(:disabled):not(.disabled):active,.show>.btn-danger.dropdown-toggle{color:#fff;background-color:#bd2130;border-color:#b21f2d}.btn-danger:not(:disabled):not(.disabled).active:focus,.btn-danger:not(:disabled):not(.disabled):active:focus,.show>.btn-danger.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(225,83,97,.5)}.btn-light{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-light:hover{color:#212529;background-color:#e2e6ea;border-color:#dae0e5}.btn-light.focus,.btn-light:focus{box-shadow:0 0 0 .2rem rgba(216,217,219,.5)}.btn-light.disabled,.btn-light:disabled{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-light:not(:disabled):not(.disabled).active,.btn-light:not(:disabled):not(.disabled):active,.show>.btn-light.dropdown-toggle{color:#212529;background-color:#dae0e5;border-color:#d3d9df}.btn-light:not(:disabled):not(.disabled).active:focus,.btn-light:not(:disabled):not(.disabled):active:focus,.show>.btn-light.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(216,217,219,.5)}.btn-dark{color:#fff;background-color:#343a40;border-color:#343a40}.btn-dark:hover{color:#fff;background-color:#23272b;border-color:#1d2124}.btn-dark.focus,.btn-dark:focus{box-shadow:0 0 0 .2rem rgba(82,88,93,.5)}.btn-dark.disabled,.btn-dark:disabled{color:#fff;background-color:#343a40;border-color:#343a40}.btn-dark:not(:disabled):not(.disabled).active,.btn-dark:not(:disabled):not(.disabled):active,.show>.btn-dark.dropdown-toggle{color:#fff;background-color:#1d2124;border-color:#171a1d}.btn-dark:not(:disabled):not(.disabled).active:focus,.btn-dark:not(:disabled):not(.disabled):active:focus,.show>.btn-dark.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(82,88,93,.5)}.btn-outline-primary{color:#007bff;border-color:#007bff}.btn-outline-primary:hover{color:#fff;background-color:#007bff;border-color:#007bff}.btn-outline-primary.focus,.btn-outline-primary:focus{box-shadow:0 0 0 .2rem rgba(0,123,255,.5)}.btn-outline-primary.disabled,.btn-outline-primary:disabled{color:#007bff;background-color:transparent}.btn-outline-primary:not(:disabled):not(.disabled).active,.btn-outline-primary:not(:disabled):not(.disabled):active,.show>.btn-outline-primary.dropdown-toggle{color:#fff;background-color:#007bff;border-color:#007bff}.btn-outline-primary:not(:disabled):not(.disabled).active:focus,.btn-outline-primary:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-primary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(0,123,255,.5)}.btn-outline-secondary{color:#6c757d;border-color:#6c757d}.btn-outline-secondary:hover{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-outline-secondary.focus,.btn-outline-secondary:focus{box-shadow:0 0 0 .2rem rgba(108,117,125,.5)}.btn-outline-secondary.disabled,.btn-outline-secondary:disabled{color:#6c757d;background-color:transparent}.btn-outline-secondary:not(:disabled):not(.disabled).active,.btn-outline-secondary:not(:disabled):not(.disabled):active,.show>.btn-outline-secondary.dropdown-toggle{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-outline-secondary:not(:disabled):not(.disabled).active:focus,.btn-outline-secondary:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-secondary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(108,117,125,.5)}.btn-outline-success{color:#28a745;border-color:#28a745}.btn-outline-success:hover{color:#fff;background-color:#28a745;border-color:#28a745}.btn-outline-success.focus,.btn-outline-success:focus{box-shadow:0 0 0 .2rem rgba(40,167,69,.5)}.btn-outline-success.disabled,.btn-outline-success:disabled{color:#28a745;background-color:transparent}.btn-outline-success:not(:disabled):not(.disabled).active,.btn-outline-success:not(:disabled):not(.disabled):active,.show>.btn-outline-success.dropdown-toggle{color:#fff;background-color:#28a745;border-color:#28a745}.btn-outline-success:not(:disabled):not(.disabled).active:focus,.btn-outline-success:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-success.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(40,167,69,.5)}.btn-outline-info{color:#17a2b8;border-color:#17a2b8}.btn-outline-info:hover{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-outline-info.focus,.btn-outline-info:focus{box-shadow:0 0 0 .2rem rgba(23,162,184,.5)}.btn-outline-info.disabled,.btn-outline-info:disabled{color:#17a2b8;background-color:transparent}.btn-outline-info:not(:disabled):not(.disabled).active,.btn-outline-info:not(:disabled):not(.disabled):active,.show>.btn-outline-info.dropdown-toggle{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-outline-info:not(:disabled):not(.disabled).active:focus,.btn-outline-info:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-info.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(23,162,184,.5)}.btn-outline-warning{color:#ffc107;border-color:#ffc107}.btn-outline-warning:hover{color:#212529;background-color:#ffc107;border-color:#ffc107}.btn-outline-warning.focus,.btn-outline-warning:focus{box-shadow:0 0 0 .2rem rgba(255,193,7,.5)}.btn-outline-warning.disabled,.btn-outline-warning:disabled{color:#ffc107;background-color:transparent}.btn-outline-warning:not(:disabled):not(.disabled).active,.btn-outline-warning:not(:disabled):not(.disabled):active,.show>.btn-outline-warning.dropdown-toggle{color:#212529;background-color:#ffc107;border-color:#ffc107}.btn-outline-warning:not(:disabled):not(.disabled).active:focus,.btn-outline-warning:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-warning.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(255,193,7,.5)}.btn-outline-danger{color:#dc3545;border-color:#dc3545}.btn-outline-danger:hover{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-outline-danger.focus,.btn-outline-danger:focus{box-shadow:0 0 0 .2rem rgba(220,53,69,.5)}.btn-outline-danger.disabled,.btn-outline-danger:disabled{color:#dc3545;background-color:transparent}.btn-outline-danger:not(:disabled):not(.disabled).active,.btn-outline-danger:not(:disabled):not(.disabled):active,.show>.btn-outline-danger.dropdown-toggle{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-outline-danger:not(:disabled):not(.disabled).active:focus,.btn-outline-danger:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-danger.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(220,53,69,.5)}.btn-outline-light{color:#f8f9fa;border-color:#f8f9fa}.btn-outline-light:hover{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-outline-light.focus,.btn-outline-light:focus{box-shadow:0 0 0 .2rem rgba(248,249,250,.5)}.btn-outline-light.disabled,.btn-outline-light:disabled{color:#f8f9fa;background-color:transparent}.btn-outline-light:not(:disabled):not(.disabled).active,.btn-outline-light:not(:disabled):not(.disabled):active,.show>.btn-outline-light.dropdown-toggle{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-outline-light:not(:disabled):not(.disabled).active:focus,.btn-outline-light:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-light.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(248,249,250,.5)}.btn-outline-dark{color:#343a40;border-color:#343a40}.btn-outline-dark:hover{color:#fff;background-color:#343a40;border-color:#343a40}.btn-outline-dark.focus,.btn-outline-dark:focus{box-shadow:0 0 0 .2rem rgba(52,58,64,.5)}.btn-outline-dark.disabled,.btn-outline-dark:disabled{color:#343a40;background-color:transparent}.btn-outline-dark:not(:disabled):not(.disabled).active,.btn-outline-dark:not(:disabled):not(.disabled):active,.show>.btn-outline-dark.dropdown-toggle{color:#fff;background-color:#343a40;border-color:#343a40}.btn-outline-dark:not(:disabled):not(.disabled).active:focus,.btn-outline-dark:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-dark.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(52,58,64,.5)}.btn-link{font-weight:400;color:#007bff;text-decoration:none}.btn-link:hover{color:#0056b3;text-decoration:underline}.btn-link.focus,.btn-link:focus{text-decoration:underline;box-shadow:none}.btn-link.disabled,.btn-link:disabled{color:#6c757d;pointer-events:none}.btn-group-lg>.btn,.btn-lg{padding:.5rem 1rem;font-size:1.25rem;line-height:1.5;border-radius:.3rem}.btn-group-sm>.btn,.btn-sm{padding:.25rem .5rem;font-size:.875rem;line-height:1.5;border-radius:.2rem}.btn-block{display:block;width:100%}.btn-block+.btn-block{margin-top:.5rem}input[type=button].btn-block,input[type=reset].btn-block,input[type=submit].btn-block{width:100%}.fade{transition:opacity .15s linear}@media (prefers-reduced-motion:reduce){.fade{transition:none}}.fade:not(.show){opacity:0}.collapse:not(.show){display:none}.collapsing{position:relative;height:0;overflow:hidden;transition:height .35s ease}@media (prefers-reduced-motion:reduce){.collapsing{transition:none}}.dropdown,.dropleft,.dropright,.dropup{position:relative}.dropdown-toggle{white-space:nowrap}.dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:.3em solid;border-right:.3em solid transparent;border-bottom:0;border-left:.3em solid transparent}.dropdown-toggle:empty::after{margin-left:0}.dropdown-menu{position:absolute;top:100%;left:0;z-index:1000;display:none;float:left;min-width:10rem;padding:.5rem 0;margin:.125rem 0 0;font-size:1rem;color:#212529;text-align:left;list-style:none;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.15);border-radius:.25rem}.dropdown-menu-left{right:auto;left:0}.dropdown-menu-right{right:0;left:auto}@media (min-width:576px){.dropdown-menu-sm-left{right:auto;left:0}.dropdown-menu-sm-right{right:0;left:auto}}@media (min-width:768px){.dropdown-menu-md-left{right:auto;left:0}.dropdown-menu-md-right{right:0;left:auto}}@media (min-width:992px){.dropdown-menu-lg-left{right:auto;left:0}.dropdown-menu-lg-right{right:0;left:auto}}@media (min-width:1200px){.dropdown-menu-xl-left{right:auto;left:0}.dropdown-menu-xl-right{right:0;left:auto}}.dropup .dropdown-menu{top:auto;bottom:100%;margin-top:0;margin-bottom:.125rem}.dropup .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:0;border-right:.3em solid transparent;border-bottom:.3em solid;border-left:.3em solid transparent}.dropup .dropdown-toggle:empty::after{margin-left:0}.dropright .dropdown-menu{top:0;right:auto;left:100%;margin-top:0;margin-left:.125rem}.dropright .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:.3em solid transparent;border-right:0;border-bottom:.3em solid transparent;border-left:.3em solid}.dropright .dropdown-toggle:empty::after{margin-left:0}.dropright .dropdown-toggle::after{vertical-align:0}.dropleft .dropdown-menu{top:0;right:100%;left:auto;margin-top:0;margin-right:.125rem}.dropleft .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:""}.dropleft .dropdown-toggle::after{display:none}.dropleft .dropdown-toggle::before{display:inline-block;margin-right:.255em;vertical-align:.255em;content:"";border-top:.3em solid transparent;border-right:.3em solid;border-bottom:.3em solid transparent}.dropleft .dropdown-toggle:empty::after{margin-left:0}.dropleft .dropdown-toggle::before{vertical-align:0}.dropdown-menu[x-placement^=bottom],.dropdown-menu[x-placement^=left],.dropdown-menu[x-placement^=right],.dropdown-menu[x-placement^=top]{right:auto;bottom:auto}.dropdown-divider{height:0;margin:.5rem 0;overflow:hidden;border-top:1px solid #e9ecef}.dropdown-item{display:block;width:100%;padding:.25rem 1.5rem;clear:both;font-weight:400;color:#212529;text-align:inherit;white-space:nowrap;background-color:transparent;border:0}.dropdown-item:focus,.dropdown-item:hover{color:#16181b;text-decoration:none;background-color:#f8f9fa}.dropdown-item.active,.dropdown-item:active{color:#fff;text-decoration:none;background-color:#007bff}.dropdown-item.disabled,.dropdown-item:disabled{color:#6c757d;pointer-events:none;background-color:transparent}.dropdown-menu.show{display:block}.dropdown-header{display:block;padding:.5rem 1.5rem;margin-bottom:0;font-size:.875rem;color:#6c757d;white-space:nowrap}.dropdown-item-text{display:block;padding:.25rem 1.5rem;color:#212529}.btn-group,.btn-group-vertical{position:relative;display:-ms-inline-flexbox;display:inline-flex;vertical-align:middle}.btn-group-vertical>.btn,.btn-group>.btn{position:relative;-ms-flex:1 1 auto;flex:1 1 auto}.btn-group-vertical>.btn:hover,.btn-group>.btn:hover{z-index:1}.btn-group-vertical>.btn.active,.btn-group-vertical>.btn:active,.btn-group-vertical>.btn:focus,.btn-group>.btn.active,.btn-group>.btn:active,.btn-group>.btn:focus{z-index:1}.btn-toolbar{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-pack:start;justify-content:flex-start}.btn-toolbar .input-group{width:auto}.btn-group>.btn-group:not(:first-child),.btn-group>.btn:not(:first-child){margin-left:-1px}.btn-group>.btn-group:not(:last-child)>.btn,.btn-group>.btn:not(:last-child):not(.dropdown-toggle){border-top-right-radius:0;border-bottom-right-radius:0}.btn-group>.btn-group:not(:first-child)>.btn,.btn-group>.btn:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.dropdown-toggle-split{padding-right:.5625rem;padding-left:.5625rem}.dropdown-toggle-split::after,.dropright .dropdown-toggle-split::after,.dropup .dropdown-toggle-split::after{margin-left:0}.dropleft .dropdown-toggle-split::before{margin-right:0}.btn-group-sm>.btn+.dropdown-toggle-split,.btn-sm+.dropdown-toggle-split{padding-right:.375rem;padding-left:.375rem}.btn-group-lg>.btn+.dropdown-toggle-split,.btn-lg+.dropdown-toggle-split{padding-right:.75rem;padding-left:.75rem}.btn-group-vertical{-ms-flex-direction:column;flex-direction:column;-ms-flex-align:start;align-items:flex-start;-ms-flex-pack:center;justify-content:center}.btn-group-vertical>.btn,.btn-group-vertical>.btn-group{width:100%}.btn-group-vertical>.btn-group:not(:first-child),.btn-group-vertical>.btn:not(:first-child){margin-top:-1px}.btn-group-vertical>.btn-group:not(:last-child)>.btn,.btn-group-vertical>.btn:not(:last-child):not(.dropdown-toggle){border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn-group:not(:first-child)>.btn,.btn-group-vertical>.btn:not(:first-child){border-top-left-radius:0;border-top-right-radius:0}.btn-group-toggle>.btn,.btn-group-toggle>.btn-group>.btn{margin-bottom:0}.btn-group-toggle>.btn input[type=checkbox],.btn-group-toggle>.btn input[type=radio],.btn-group-toggle>.btn-group>.btn input[type=checkbox],.btn-group-toggle>.btn-group>.btn input[type=radio]{position:absolute;clip:rect(0,0,0,0);pointer-events:none}.input-group{position:relative;display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-align:stretch;align-items:stretch;width:100%}.input-group>.custom-file,.input-group>.custom-select,.input-group>.form-control,.input-group>.form-control-plaintext{position:relative;-ms-flex:1 1 auto;flex:1 1 auto;width:1%;margin-bottom:0}.input-group>.custom-file+.custom-file,.input-group>.custom-file+.custom-select,.input-group>.custom-file+.form-control,.input-group>.custom-select+.custom-file,.input-group>.custom-select+.custom-select,.input-group>.custom-select+.form-control,.input-group>.form-control+.custom-file,.input-group>.form-control+.custom-select,.input-group>.form-control+.form-control,.input-group>.form-control-plaintext+.custom-file,.input-group>.form-control-plaintext+.custom-select,.input-group>.form-control-plaintext+.form-control{margin-left:-1px}.input-group>.custom-file .custom-file-input:focus~.custom-file-label,.input-group>.custom-select:focus,.input-group>.form-control:focus{z-index:3}.input-group>.custom-file .custom-file-input:focus{z-index:4}.input-group>.custom-select:not(:last-child),.input-group>.form-control:not(:last-child){border-top-right-radius:0;border-bottom-right-radius:0}.input-group>.custom-select:not(:first-child),.input-group>.form-control:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.input-group>.custom-file{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center}.input-group>.custom-file:not(:last-child) .custom-file-label,.input-group>.custom-file:not(:last-child) .custom-file-label::after{border-top-right-radius:0;border-bottom-right-radius:0}.input-group>.custom-file:not(:first-child) .custom-file-label{border-top-left-radius:0;border-bottom-left-radius:0}.input-group-append,.input-group-prepend{display:-ms-flexbox;display:flex}.input-group-append .btn,.input-group-prepend .btn{position:relative;z-index:2}.input-group-append .btn:focus,.input-group-prepend .btn:focus{z-index:3}.input-group-append .btn+.btn,.input-group-append .btn+.input-group-text,.input-group-append .input-group-text+.btn,.input-group-append .input-group-text+.input-group-text,.input-group-prepend .btn+.btn,.input-group-prepend .btn+.input-group-text,.input-group-prepend .input-group-text+.btn,.input-group-prepend .input-group-text+.input-group-text{margin-left:-1px}.input-group-prepend{margin-right:-1px}.input-group-append{margin-left:-1px}.input-group-text{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;padding:.375rem .75rem;margin-bottom:0;font-size:1rem;font-weight:400;line-height:1.5;color:#495057;text-align:center;white-space:nowrap;background-color:#e9ecef;border:1px solid #ced4da;border-radius:.25rem}.input-group-text input[type=checkbox],.input-group-text input[type=radio]{margin-top:0}.input-group-lg>.custom-select,.input-group-lg>.form-control:not(textarea){height:calc(1.5em + 1rem + 2px)}.input-group-lg>.custom-select,.input-group-lg>.form-control,.input-group-lg>.input-group-append>.btn,.input-group-lg>.input-group-append>.input-group-text,.input-group-lg>.input-group-prepend>.btn,.input-group-lg>.input-group-prepend>.input-group-text{padding:.5rem 1rem;font-size:1.25rem;line-height:1.5;border-radius:.3rem}.input-group-sm>.custom-select,.input-group-sm>.form-control:not(textarea){height:calc(1.5em + .5rem + 2px)}.input-group-sm>.custom-select,.input-group-sm>.form-control,.input-group-sm>.input-group-append>.btn,.input-group-sm>.input-group-append>.input-group-text,.input-group-sm>.input-group-prepend>.btn,.input-group-sm>.input-group-prepend>.input-group-text{padding:.25rem .5rem;font-size:.875rem;line-height:1.5;border-radius:.2rem}.input-group-lg>.custom-select,.input-group-sm>.custom-select{padding-right:1.75rem}.input-group>.input-group-append:last-child>.btn:not(:last-child):not(.dropdown-toggle),.input-group>.input-group-append:last-child>.input-group-text:not(:last-child),.input-group>.input-group-append:not(:last-child)>.btn,.input-group>.input-group-append:not(:last-child)>.input-group-text,.input-group>.input-group-prepend>.btn,.input-group>.input-group-prepend>.input-group-text{border-top-right-radius:0;border-bottom-right-radius:0}.input-group>.input-group-append>.btn,.input-group>.input-group-append>.input-group-text,.input-group>.input-group-prepend:first-child>.btn:not(:first-child),.input-group>.input-group-prepend:first-child>.input-group-text:not(:first-child),.input-group>.input-group-prepend:not(:first-child)>.btn,.input-group>.input-group-prepend:not(:first-child)>.input-group-text{border-top-left-radius:0;border-bottom-left-radius:0}.custom-control{position:relative;display:block;min-height:1.5rem;padding-left:1.5rem}.custom-control-inline{display:-ms-inline-flexbox;display:inline-flex;margin-right:1rem}.custom-control-input{position:absolute;z-index:-1;opacity:0}.custom-control-input:checked~.custom-control-label::before{color:#fff;border-color:#007bff;background-color:#007bff}.custom-control-input:focus~.custom-control-label::before{box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.custom-control-input:focus:not(:checked)~.custom-control-label::before{border-color:#80bdff}.custom-control-input:not(:disabled):active~.custom-control-label::before{color:#fff;background-color:#b3d7ff;border-color:#b3d7ff}.custom-control-input:disabled~.custom-control-label{color:#6c757d}.custom-control-input:disabled~.custom-control-label::before{background-color:#e9ecef}.custom-control-label{position:relative;margin-bottom:0;vertical-align:top}.custom-control-label::before{position:absolute;top:.25rem;left:-1.5rem;display:block;width:1rem;height:1rem;pointer-events:none;content:"";background-color:#fff;border:#adb5bd solid 1px}.custom-control-label::after{position:absolute;top:.25rem;left:-1.5rem;display:block;width:1rem;height:1rem;content:"";background:no-repeat 50%/50% 50%}.custom-checkbox .custom-control-label::before{border-radius:.25rem}.custom-checkbox .custom-control-input:checked~.custom-control-label::after{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 8 8'%3e%3cpath fill='%23fff' d='M6.564.75l-3.59 3.612-1.538-1.55L0 4.26 2.974 7.25 8 2.193z'/%3e%3c/svg%3e")}.custom-checkbox .custom-control-input:indeterminate~.custom-control-label::before{border-color:#007bff;background-color:#007bff}.custom-checkbox .custom-control-input:indeterminate~.custom-control-label::after{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 4 4'%3e%3cpath stroke='%23fff' d='M0 2h4'/%3e%3c/svg%3e")}.custom-checkbox .custom-control-input:disabled:checked~.custom-control-label::before{background-color:rgba(0,123,255,.5)}.custom-checkbox .custom-control-input:disabled:indeterminate~.custom-control-label::before{background-color:rgba(0,123,255,.5)}.custom-radio .custom-control-label::before{border-radius:50%}.custom-radio .custom-control-input:checked~.custom-control-label::after{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='%23fff'/%3e%3c/svg%3e")}.custom-radio .custom-control-input:disabled:checked~.custom-control-label::before{background-color:rgba(0,123,255,.5)}.custom-switch{padding-left:2.25rem}.custom-switch .custom-control-label::before{left:-2.25rem;width:1.75rem;pointer-events:all;border-radius:.5rem}.custom-switch .custom-control-label::after{top:calc(.25rem + 2px);left:calc(-2.25rem + 2px);width:calc(1rem - 4px);height:calc(1rem - 4px);background-color:#adb5bd;border-radius:.5rem;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out,-webkit-transform .15s ease-in-out;transition:transform .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:transform .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out,-webkit-transform .15s ease-in-out}@media (prefers-reduced-motion:reduce){.custom-switch .custom-control-label::after{transition:none}}.custom-switch .custom-control-input:checked~.custom-control-label::after{background-color:#fff;-webkit-transform:translateX(.75rem);transform:translateX(.75rem)}.custom-switch .custom-control-input:disabled:checked~.custom-control-label::before{background-color:rgba(0,123,255,.5)}.custom-select{display:inline-block;width:100%;height:calc(1.5em + .75rem + 2px);padding:.375rem 1.75rem .375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:#495057;vertical-align:middle;background:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e") no-repeat right .75rem center/8px 10px;background-color:#fff;border:1px solid #ced4da;border-radius:.25rem;-webkit-appearance:none;-moz-appearance:none;appearance:none}.custom-select:focus{border-color:#80bdff;outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.custom-select:focus::-ms-value{color:#495057;background-color:#fff}.custom-select[multiple],.custom-select[size]:not([size="1"]){height:auto;padding-right:.75rem;background-image:none}.custom-select:disabled{color:#6c757d;background-color:#e9ecef}.custom-select::-ms-expand{display:none}.custom-select-sm{height:calc(1.5em + .5rem + 2px);padding-top:.25rem;padding-bottom:.25rem;padding-left:.5rem;font-size:.875rem}.custom-select-lg{height:calc(1.5em + 1rem + 2px);padding-top:.5rem;padding-bottom:.5rem;padding-left:1rem;font-size:1.25rem}.custom-file{position:relative;display:inline-block;width:100%;height:calc(1.5em + .75rem + 2px);margin-bottom:0}.custom-file-input{position:relative;z-index:2;width:100%;height:calc(1.5em + .75rem + 2px);margin:0;opacity:0}.custom-file-input:focus~.custom-file-label{border-color:#80bdff;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.custom-file-input:disabled~.custom-file-label{background-color:#e9ecef}.custom-file-input:lang(en)~.custom-file-label::after{content:"Browse"}.custom-file-input~.custom-file-label[data-browse]::after{content:attr(data-browse)}.custom-file-label{position:absolute;top:0;right:0;left:0;z-index:1;height:calc(1.5em + .75rem + 2px);padding:.375rem .75rem;font-weight:400;line-height:1.5;color:#495057;background-color:#fff;border:1px solid #ced4da;border-radius:.25rem}.custom-file-label::after{position:absolute;top:0;right:0;bottom:0;z-index:3;display:block;height:calc(1.5em + .75rem);padding:.375rem .75rem;line-height:1.5;color:#495057;content:"Browse";background-color:#e9ecef;border-left:inherit;border-radius:0 .25rem .25rem 0}.custom-range{width:100%;height:calc(1rem + .4rem);padding:0;background-color:transparent;-webkit-appearance:none;-moz-appearance:none;appearance:none}.custom-range:focus{outline:0}.custom-range:focus::-webkit-slider-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .2rem rgba(0,123,255,.25)}.custom-range:focus::-moz-range-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .2rem rgba(0,123,255,.25)}.custom-range:focus::-ms-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .2rem rgba(0,123,255,.25)}.custom-range::-moz-focus-outer{border:0}.custom-range::-webkit-slider-thumb{width:1rem;height:1rem;margin-top:-.25rem;background-color:#007bff;border:0;border-radius:1rem;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;-webkit-appearance:none;appearance:none}@media (prefers-reduced-motion:reduce){.custom-range::-webkit-slider-thumb{transition:none}}.custom-range::-webkit-slider-thumb:active{background-color:#b3d7ff}.custom-range::-webkit-slider-runnable-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:#dee2e6;border-color:transparent;border-radius:1rem}.custom-range::-moz-range-thumb{width:1rem;height:1rem;background-color:#007bff;border:0;border-radius:1rem;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;-moz-appearance:none;appearance:none}@media (prefers-reduced-motion:reduce){.custom-range::-moz-range-thumb{transition:none}}.custom-range::-moz-range-thumb:active{background-color:#b3d7ff}.custom-range::-moz-range-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:#dee2e6;border-color:transparent;border-radius:1rem}.custom-range::-ms-thumb{width:1rem;height:1rem;margin-top:0;margin-right:.2rem;margin-left:.2rem;background-color:#007bff;border:0;border-radius:1rem;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;appearance:none}@media (prefers-reduced-motion:reduce){.custom-range::-ms-thumb{transition:none}}.custom-range::-ms-thumb:active{background-color:#b3d7ff}.custom-range::-ms-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:transparent;border-color:transparent;border-width:.5rem}.custom-range::-ms-fill-lower{background-color:#dee2e6;border-radius:1rem}.custom-range::-ms-fill-upper{margin-right:15px;background-color:#dee2e6;border-radius:1rem}.custom-range:disabled::-webkit-slider-thumb{background-color:#adb5bd}.custom-range:disabled::-webkit-slider-runnable-track{cursor:default}.custom-range:disabled::-moz-range-thumb{background-color:#adb5bd}.custom-range:disabled::-moz-range-track{cursor:default}.custom-range:disabled::-ms-thumb{background-color:#adb5bd}.custom-control-label::before,.custom-file-label,.custom-select{transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.custom-control-label::before,.custom-file-label,.custom-select{transition:none}}.nav{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;padding-left:0;margin-bottom:0;list-style:none}.nav-link{display:block;padding:.5rem 1rem}.nav-link:focus,.nav-link:hover{text-decoration:none}.nav-link.disabled{color:#6c757d;pointer-events:none;cursor:default}.nav-tabs{border-bottom:1px solid #dee2e6}.nav-tabs .nav-item{margin-bottom:-1px}.nav-tabs .nav-link{border:1px solid transparent;border-top-left-radius:.25rem;border-top-right-radius:.25rem}.nav-tabs .nav-link:focus,.nav-tabs .nav-link:hover{border-color:#e9ecef #e9ecef #dee2e6}.nav-tabs .nav-link.disabled{color:#6c757d;background-color:transparent;border-color:transparent}.nav-tabs .nav-item.show .nav-link,.nav-tabs .nav-link.active{color:#495057;background-color:#fff;border-color:#dee2e6 #dee2e6 #fff}.nav-tabs .dropdown-menu{margin-top:-1px;border-top-left-radius:0;border-top-right-radius:0}.nav-pills .nav-link{border-radius:.25rem}.nav-pills .nav-link.active,.nav-pills .show>.nav-link{color:#fff;background-color:#007bff}.nav-fill .nav-item{-ms-flex:1 1 auto;flex:1 1 auto;text-align:center}.nav-justified .nav-item{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;text-align:center}.tab-content>.tab-pane{display:none}.tab-content>.active{display:block}.navbar{position:relative;display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-align:center;align-items:center;-ms-flex-pack:justify;justify-content:space-between;padding:.5rem 1rem}.navbar>.container,.navbar>.container-fluid{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-align:center;align-items:center;-ms-flex-pack:justify;justify-content:space-between}.navbar-brand{display:inline-block;padding-top:.3125rem;padding-bottom:.3125rem;margin-right:1rem;font-size:1.25rem;line-height:inherit;white-space:nowrap}.navbar-brand:focus,.navbar-brand:hover{text-decoration:none}.navbar-nav{display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;padding-left:0;margin-bottom:0;list-style:none}.navbar-nav .nav-link{padding-right:0;padding-left:0}.navbar-nav .dropdown-menu{position:static;float:none}.navbar-text{display:inline-block;padding-top:.5rem;padding-bottom:.5rem}.navbar-collapse{-ms-flex-preferred-size:100%;flex-basis:100%;-ms-flex-positive:1;flex-grow:1;-ms-flex-align:center;align-items:center}.navbar-toggler{padding:.25rem .75rem;font-size:1.25rem;line-height:1;background-color:transparent;border:1px solid transparent;border-radius:.25rem}.navbar-toggler:focus,.navbar-toggler:hover{text-decoration:none}.navbar-toggler-icon{display:inline-block;width:1.5em;height:1.5em;vertical-align:middle;content:"";background:no-repeat center center;background-size:100% 100%}@media (max-width:575.98px){.navbar-expand-sm>.container,.navbar-expand-sm>.container-fluid{padding-right:0;padding-left:0}}@media (min-width:576px){.navbar-expand-sm{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-sm .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-sm .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-sm .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-sm>.container,.navbar-expand-sm>.container-fluid{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-sm .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-sm .navbar-toggler{display:none}}@media (max-width:767.98px){.navbar-expand-md>.container,.navbar-expand-md>.container-fluid{padding-right:0;padding-left:0}}@media (min-width:768px){.navbar-expand-md{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-md .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-md .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-md .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-md>.container,.navbar-expand-md>.container-fluid{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-md .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-md .navbar-toggler{display:none}}@media (max-width:991.98px){.navbar-expand-lg>.container,.navbar-expand-lg>.container-fluid{padding-right:0;padding-left:0}}@media (min-width:992px){.navbar-expand-lg{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-lg .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-lg .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-lg .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-lg>.container,.navbar-expand-lg>.container-fluid{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-lg .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-lg .navbar-toggler{display:none}}@media (max-width:1199.98px){.navbar-expand-xl>.container,.navbar-expand-xl>.container-fluid{padding-right:0;padding-left:0}}@media (min-width:1200px){.navbar-expand-xl{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-xl .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-xl .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-xl .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-xl>.container,.navbar-expand-xl>.container-fluid{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-xl .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-xl .navbar-toggler{display:none}}.navbar-expand{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand>.container,.navbar-expand>.container-fluid{padding-right:0;padding-left:0}.navbar-expand .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand .navbar-nav .dropdown-menu{position:absolute}.navbar-expand .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand>.container,.navbar-expand>.container-fluid{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand .navbar-toggler{display:none}.navbar-light .navbar-brand{color:rgba(0,0,0,.9)}.navbar-light .navbar-brand:focus,.navbar-light .navbar-brand:hover{color:rgba(0,0,0,.9)}.navbar-light .navbar-nav .nav-link{color:rgba(0,0,0,.5)}.navbar-light .navbar-nav .nav-link:focus,.navbar-light .navbar-nav .nav-link:hover{color:rgba(0,0,0,.7)}.navbar-light .navbar-nav .nav-link.disabled{color:rgba(0,0,0,.3)}.navbar-light .navbar-nav .active>.nav-link,.navbar-light .navbar-nav .nav-link.active,.navbar-light .navbar-nav .nav-link.show,.navbar-light .navbar-nav .show>.nav-link{color:rgba(0,0,0,.9)}.navbar-light .navbar-toggler{color:rgba(0,0,0,.5);border-color:rgba(0,0,0,.1)}.navbar-light .navbar-toggler-icon{background-image:url("data:image/svg+xml,%3csvg viewBox='0 0 30 30' xmlns='http://www.w3.org/2000/svg'%3e%3cpath stroke='rgba(0, 0, 0, 0.5)' stroke-width='2' stroke-linecap='round' stroke-miterlimit='10' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e")}.navbar-light .navbar-text{color:rgba(0,0,0,.5)}.navbar-light .navbar-text a{color:rgba(0,0,0,.9)}.navbar-light .navbar-text a:focus,.navbar-light .navbar-text a:hover{color:rgba(0,0,0,.9)}.navbar-dark .navbar-brand{color:#fff}.navbar-dark .navbar-brand:focus,.navbar-dark .navbar-brand:hover{color:#fff}.navbar-dark .navbar-nav .nav-link{color:rgba(255,255,255,.5)}.navbar-dark .navbar-nav .nav-link:focus,.navbar-dark .navbar-nav .nav-link:hover{color:rgba(255,255,255,.75)}.navbar-dark .navbar-nav .nav-link.disabled{color:rgba(255,255,255,.25)}.navbar-dark .navbar-nav .active>.nav-link,.navbar-dark .navbar-nav .nav-link.active,.navbar-dark .navbar-nav .nav-link.show,.navbar-dark .navbar-nav .show>.nav-link{color:#fff}.navbar-dark .navbar-toggler{color:rgba(255,255,255,.5);border-color:rgba(255,255,255,.1)}.navbar-dark .navbar-toggler-icon{background-image:url("data:image/svg+xml,%3csvg viewBox='0 0 30 30' xmlns='http://www.w3.org/2000/svg'%3e%3cpath stroke='rgba(255, 255, 255, 0.5)' stroke-width='2' stroke-linecap='round' stroke-miterlimit='10' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e")}.navbar-dark .navbar-text{color:rgba(255,255,255,.5)}.navbar-dark .navbar-text a{color:#fff}.navbar-dark .navbar-text a:focus,.navbar-dark .navbar-text a:hover{color:#fff}.card{position:relative;display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;min-width:0;word-wrap:break-word;background-color:#fff;background-clip:border-box;border:1px solid rgba(0,0,0,.125);border-radius:.25rem}.card>hr{margin-right:0;margin-left:0}.card>.list-group:first-child .list-group-item:first-child{border-top-left-radius:.25rem;border-top-right-radius:.25rem}.card>.list-group:last-child .list-group-item:last-child{border-bottom-right-radius:.25rem;border-bottom-left-radius:.25rem}.card-body{-ms-flex:1 1 auto;flex:1 1 auto;padding:1.25rem}.card-title{margin-bottom:.75rem}.card-subtitle{margin-top:-.375rem;margin-bottom:0}.card-text:last-child{margin-bottom:0}.card-link:hover{text-decoration:none}.card-link+.card-link{margin-left:1.25rem}.card-header{padding:.75rem 1.25rem;margin-bottom:0;background-color:rgba(0,0,0,.03);border-bottom:1px solid rgba(0,0,0,.125)}.card-header:first-child{border-radius:calc(.25rem - 1px) calc(.25rem - 1px) 0 0}.card-header+.list-group .list-group-item:first-child{border-top:0}.card-footer{padding:.75rem 1.25rem;background-color:rgba(0,0,0,.03);border-top:1px solid rgba(0,0,0,.125)}.card-footer:last-child{border-radius:0 0 calc(.25rem - 1px) calc(.25rem - 1px)}.card-header-tabs{margin-right:-.625rem;margin-bottom:-.75rem;margin-left:-.625rem;border-bottom:0}.card-header-pills{margin-right:-.625rem;margin-left:-.625rem}.card-img-overlay{position:absolute;top:0;right:0;bottom:0;left:0;padding:1.25rem}.card-img{width:100%;border-radius:calc(.25rem - 1px)}.card-img-top{width:100%;border-top-left-radius:calc(.25rem - 1px);border-top-right-radius:calc(.25rem - 1px)}.card-img-bottom{width:100%;border-bottom-right-radius:calc(.25rem - 1px);border-bottom-left-radius:calc(.25rem - 1px)}.card-deck{display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column}.card-deck .card{margin-bottom:15px}@media (min-width:576px){.card-deck{-ms-flex-flow:row wrap;flex-flow:row wrap;margin-right:-15px;margin-left:-15px}.card-deck .card{display:-ms-flexbox;display:flex;-ms-flex:1 0 0%;flex:1 0 0%;-ms-flex-direction:column;flex-direction:column;margin-right:15px;margin-bottom:0;margin-left:15px}}.card-group{display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column}.card-group>.card{margin-bottom:15px}@media (min-width:576px){.card-group{-ms-flex-flow:row wrap;flex-flow:row wrap}.card-group>.card{-ms-flex:1 0 0%;flex:1 0 0%;margin-bottom:0}.card-group>.card+.card{margin-left:0;border-left:0}.card-group>.card:not(:last-child){border-top-right-radius:0;border-bottom-right-radius:0}.card-group>.card:not(:last-child) .card-header,.card-group>.card:not(:last-child) .card-img-top{border-top-right-radius:0}.card-group>.card:not(:last-child) .card-footer,.card-group>.card:not(:last-child) .card-img-bottom{border-bottom-right-radius:0}.card-group>.card:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.card-group>.card:not(:first-child) .card-header,.card-group>.card:not(:first-child) .card-img-top{border-top-left-radius:0}.card-group>.card:not(:first-child) .card-footer,.card-group>.card:not(:first-child) .card-img-bottom{border-bottom-left-radius:0}}.card-columns .card{margin-bottom:.75rem}@media (min-width:576px){.card-columns{-webkit-column-count:3;-moz-column-count:3;column-count:3;-webkit-column-gap:1.25rem;-moz-column-gap:1.25rem;column-gap:1.25rem;orphans:1;widows:1}.card-columns .card{display:inline-block;width:100%}}.accordion>.card{overflow:hidden}.accordion>.card:not(:first-of-type) .card-header:first-child{border-radius:0}.accordion>.card:not(:first-of-type):not(:last-of-type){border-bottom:0;border-radius:0}.accordion>.card:first-of-type{border-bottom:0;border-bottom-right-radius:0;border-bottom-left-radius:0}.accordion>.card:last-of-type{border-top-left-radius:0;border-top-right-radius:0}.accordion>.card .card-header{margin-bottom:-1px}.breadcrumb{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;padding:.75rem 1rem;margin-bottom:1rem;list-style:none;background-color:#e9ecef;border-radius:.25rem}.breadcrumb-item+.breadcrumb-item{padding-left:.5rem}.breadcrumb-item+.breadcrumb-item::before{display:inline-block;padding-right:.5rem;color:#6c757d;content:"/"}.breadcrumb-item+.breadcrumb-item:hover::before{text-decoration:underline}.breadcrumb-item+.breadcrumb-item:hover::before{text-decoration:none}.breadcrumb-item.active{color:#6c757d}.pagination{display:-ms-flexbox;display:flex;padding-left:0;list-style:none;border-radius:.25rem}.page-link{position:relative;display:block;padding:.5rem .75rem;margin-left:-1px;line-height:1.25;color:#007bff;background-color:#fff;border:1px solid #dee2e6}.page-link:hover{z-index:2;color:#0056b3;text-decoration:none;background-color:#e9ecef;border-color:#dee2e6}.page-link:focus{z-index:2;outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.page-item:first-child .page-link{margin-left:0;border-top-left-radius:.25rem;border-bottom-left-radius:.25rem}.page-item:last-child .page-link{border-top-right-radius:.25rem;border-bottom-right-radius:.25rem}.page-item.active .page-link{z-index:1;color:#fff;background-color:#007bff;border-color:#007bff}.page-item.disabled .page-link{color:#6c757d;pointer-events:none;cursor:auto;background-color:#fff;border-color:#dee2e6}.pagination-lg .page-link{padding:.75rem 1.5rem;font-size:1.25rem;line-height:1.5}.pagination-lg .page-item:first-child .page-link{border-top-left-radius:.3rem;border-bottom-left-radius:.3rem}.pagination-lg .page-item:last-child .page-link{border-top-right-radius:.3rem;border-bottom-right-radius:.3rem}.pagination-sm .page-link{padding:.25rem .5rem;font-size:.875rem;line-height:1.5}.pagination-sm .page-item:first-child .page-link{border-top-left-radius:.2rem;border-bottom-left-radius:.2rem}.pagination-sm .page-item:last-child .page-link{border-top-right-radius:.2rem;border-bottom-right-radius:.2rem}.badge{display:inline-block;padding:.25em .4em;font-size:75%;font-weight:700;line-height:1;text-align:center;white-space:nowrap;vertical-align:baseline;border-radius:.25rem;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.badge{transition:none}}a.badge:focus,a.badge:hover{text-decoration:none}.badge:empty{display:none}.btn .badge{position:relative;top:-1px}.badge-pill{padding-right:.6em;padding-left:.6em;border-radius:10rem}.badge-primary{color:#fff;background-color:#007bff}a.badge-primary:focus,a.badge-primary:hover{color:#fff;background-color:#0062cc}a.badge-primary.focus,a.badge-primary:focus{outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.5)}.badge-secondary{color:#fff;background-color:#6c757d}a.badge-secondary:focus,a.badge-secondary:hover{color:#fff;background-color:#545b62}a.badge-secondary.focus,a.badge-secondary:focus{outline:0;box-shadow:0 0 0 .2rem rgba(108,117,125,.5)}.badge-success{color:#fff;background-color:#28a745}a.badge-success:focus,a.badge-success:hover{color:#fff;background-color:#1e7e34}a.badge-success.focus,a.badge-success:focus{outline:0;box-shadow:0 0 0 .2rem rgba(40,167,69,.5)}.badge-info{color:#fff;background-color:#17a2b8}a.badge-info:focus,a.badge-info:hover{color:#fff;background-color:#117a8b}a.badge-info.focus,a.badge-info:focus{outline:0;box-shadow:0 0 0 .2rem rgba(23,162,184,.5)}.badge-warning{color:#212529;background-color:#ffc107}a.badge-warning:focus,a.badge-warning:hover{color:#212529;background-color:#d39e00}a.badge-warning.focus,a.badge-warning:focus{outline:0;box-shadow:0 0 0 .2rem rgba(255,193,7,.5)}.badge-danger{color:#fff;background-color:#dc3545}a.badge-danger:focus,a.badge-danger:hover{color:#fff;background-color:#bd2130}a.badge-danger.focus,a.badge-danger:focus{outline:0;box-shadow:0 0 0 .2rem rgba(220,53,69,.5)}.badge-light{color:#212529;background-color:#f8f9fa}a.badge-light:focus,a.badge-light:hover{color:#212529;background-color:#dae0e5}a.badge-light.focus,a.badge-light:focus{outline:0;box-shadow:0 0 0 .2rem rgba(248,249,250,.5)}.badge-dark{color:#fff;background-color:#343a40}a.badge-dark:focus,a.badge-dark:hover{color:#fff;background-color:#1d2124}a.badge-dark.focus,a.badge-dark:focus{outline:0;box-shadow:0 0 0 .2rem rgba(52,58,64,.5)}.jumbotron{padding:2rem 1rem;margin-bottom:2rem;background-color:#e9ecef;border-radius:.3rem}@media (min-width:576px){.jumbotron{padding:4rem 2rem}}.jumbotron-fluid{padding-right:0;padding-left:0;border-radius:0}.alert{position:relative;padding:.75rem 1.25rem;margin-bottom:1rem;border:1px solid transparent;border-radius:.25rem}.alert-heading{color:inherit}.alert-link{font-weight:700}.alert-dismissible{padding-right:4rem}.alert-dismissible .close{position:absolute;top:0;right:0;padding:.75rem 1.25rem;color:inherit}.alert-primary{color:#004085;background-color:#cce5ff;border-color:#b8daff}.alert-primary hr{border-top-color:#9fcdff}.alert-primary .alert-link{color:#002752}.alert-secondary{color:#383d41;background-color:#e2e3e5;border-color:#d6d8db}.alert-secondary hr{border-top-color:#c8cbcf}.alert-secondary .alert-link{color:#202326}.alert-success{color:#155724;background-color:#d4edda;border-color:#c3e6cb}.alert-success hr{border-top-color:#b1dfbb}.alert-success .alert-link{color:#0b2e13}.alert-info{color:#0c5460;background-color:#d1ecf1;border-color:#bee5eb}.alert-info hr{border-top-color:#abdde5}.alert-info .alert-link{color:#062c33}.alert-warning{color:#856404;background-color:#fff3cd;border-color:#ffeeba}.alert-warning hr{border-top-color:#ffe8a1}.alert-warning .alert-link{color:#533f03}.alert-danger{color:#721c24;background-color:#f8d7da;border-color:#f5c6cb}.alert-danger hr{border-top-color:#f1b0b7}.alert-danger .alert-link{color:#491217}.alert-light{color:#818182;background-color:#fefefe;border-color:#fdfdfe}.alert-light hr{border-top-color:#ececf6}.alert-light .alert-link{color:#686868}.alert-dark{color:#1b1e21;background-color:#d6d8d9;border-color:#c6c8ca}.alert-dark hr{border-top-color:#b9bbbe}.alert-dark .alert-link{color:#040505}@-webkit-keyframes progress-bar-stripes{from{background-position:1rem 0}to{background-position:0 0}}@keyframes progress-bar-stripes{from{background-position:1rem 0}to{background-position:0 0}}.progress{display:-ms-flexbox;display:flex;height:1rem;overflow:hidden;font-size:.75rem;background-color:#e9ecef;border-radius:.25rem}.progress-bar{display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;-ms-flex-pack:center;justify-content:center;color:#fff;text-align:center;white-space:nowrap;background-color:#007bff;transition:width .6s ease}@media (prefers-reduced-motion:reduce){.progress-bar{transition:none}}.progress-bar-striped{background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-size:1rem 1rem}.progress-bar-animated{-webkit-animation:progress-bar-stripes 1s linear infinite;animation:progress-bar-stripes 1s linear infinite}@media (prefers-reduced-motion:reduce){.progress-bar-animated{-webkit-animation:none;animation:none}}.media{display:-ms-flexbox;display:flex;-ms-flex-align:start;align-items:flex-start}.media-body{-ms-flex:1;flex:1}.list-group{display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;padding-left:0;margin-bottom:0}.list-group-item-action{width:100%;color:#495057;text-align:inherit}.list-group-item-action:focus,.list-group-item-action:hover{z-index:1;color:#495057;text-decoration:none;background-color:#f8f9fa}.list-group-item-action:active{color:#212529;background-color:#e9ecef}.list-group-item{position:relative;display:block;padding:.75rem 1.25rem;margin-bottom:-1px;background-color:#fff;border:1px solid rgba(0,0,0,.125)}.list-group-item:first-child{border-top-left-radius:.25rem;border-top-right-radius:.25rem}.list-group-item:last-child{margin-bottom:0;border-bottom-right-radius:.25rem;border-bottom-left-radius:.25rem}.list-group-item.disabled,.list-group-item:disabled{color:#6c757d;pointer-events:none;background-color:#fff}.list-group-item.active{z-index:2;color:#fff;background-color:#007bff;border-color:#007bff}.list-group-horizontal{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal .list-group-item{margin-right:-1px;margin-bottom:0}.list-group-horizontal .list-group-item:first-child{border-top-left-radius:.25rem;border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal .list-group-item:last-child{margin-right:0;border-top-right-radius:.25rem;border-bottom-right-radius:.25rem;border-bottom-left-radius:0}@media (min-width:576px){.list-group-horizontal-sm{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal-sm .list-group-item{margin-right:-1px;margin-bottom:0}.list-group-horizontal-sm .list-group-item:first-child{border-top-left-radius:.25rem;border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-sm .list-group-item:last-child{margin-right:0;border-top-right-radius:.25rem;border-bottom-right-radius:.25rem;border-bottom-left-radius:0}}@media (min-width:768px){.list-group-horizontal-md{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal-md .list-group-item{margin-right:-1px;margin-bottom:0}.list-group-horizontal-md .list-group-item:first-child{border-top-left-radius:.25rem;border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-md .list-group-item:last-child{margin-right:0;border-top-right-radius:.25rem;border-bottom-right-radius:.25rem;border-bottom-left-radius:0}}@media (min-width:992px){.list-group-horizontal-lg{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal-lg .list-group-item{margin-right:-1px;margin-bottom:0}.list-group-horizontal-lg .list-group-item:first-child{border-top-left-radius:.25rem;border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-lg .list-group-item:last-child{margin-right:0;border-top-right-radius:.25rem;border-bottom-right-radius:.25rem;border-bottom-left-radius:0}}@media (min-width:1200px){.list-group-horizontal-xl{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal-xl .list-group-item{margin-right:-1px;margin-bottom:0}.list-group-horizontal-xl .list-group-item:first-child{border-top-left-radius:.25rem;border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-xl .list-group-item:last-child{margin-right:0;border-top-right-radius:.25rem;border-bottom-right-radius:.25rem;border-bottom-left-radius:0}}.list-group-flush .list-group-item{border-right:0;border-left:0;border-radius:0}.list-group-flush .list-group-item:last-child{margin-bottom:-1px}.list-group-flush:first-child .list-group-item:first-child{border-top:0}.list-group-flush:last-child .list-group-item:last-child{margin-bottom:0;border-bottom:0}.list-group-item-primary{color:#004085;background-color:#b8daff}.list-group-item-primary.list-group-item-action:focus,.list-group-item-primary.list-group-item-action:hover{color:#004085;background-color:#9fcdff}.list-group-item-primary.list-group-item-action.active{color:#fff;background-color:#004085;border-color:#004085}.list-group-item-secondary{color:#383d41;background-color:#d6d8db}.list-group-item-secondary.list-group-item-action:focus,.list-group-item-secondary.list-group-item-action:hover{color:#383d41;background-color:#c8cbcf}.list-group-item-secondary.list-group-item-action.active{color:#fff;background-color:#383d41;border-color:#383d41}.list-group-item-success{color:#155724;background-color:#c3e6cb}.list-group-item-success.list-group-item-action:focus,.list-group-item-success.list-group-item-action:hover{color:#155724;background-color:#b1dfbb}.list-group-item-success.list-group-item-action.active{color:#fff;background-color:#155724;border-color:#155724}.list-group-item-info{color:#0c5460;background-color:#bee5eb}.list-group-item-info.list-group-item-action:focus,.list-group-item-info.list-group-item-action:hover{color:#0c5460;background-color:#abdde5}.list-group-item-info.list-group-item-action.active{color:#fff;background-color:#0c5460;border-color:#0c5460}.list-group-item-warning{color:#856404;background-color:#ffeeba}.list-group-item-warning.list-group-item-action:focus,.list-group-item-warning.list-group-item-action:hover{color:#856404;background-color:#ffe8a1}.list-group-item-warning.list-group-item-action.active{color:#fff;background-color:#856404;border-color:#856404}.list-group-item-danger{color:#721c24;background-color:#f5c6cb}.list-group-item-danger.list-group-item-action:focus,.list-group-item-danger.list-group-item-action:hover{color:#721c24;background-color:#f1b0b7}.list-group-item-danger.list-group-item-action.active{color:#fff;background-color:#721c24;border-color:#721c24}.list-group-item-light{color:#818182;background-color:#fdfdfe}.list-group-item-light.list-group-item-action:focus,.list-group-item-light.list-group-item-action:hover{color:#818182;background-color:#ececf6}.list-group-item-light.list-group-item-action.active{color:#fff;background-color:#818182;border-color:#818182}.list-group-item-dark{color:#1b1e21;background-color:#c6c8ca}.list-group-item-dark.list-group-item-action:focus,.list-group-item-dark.list-group-item-action:hover{color:#1b1e21;background-color:#b9bbbe}.list-group-item-dark.list-group-item-action.active{color:#fff;background-color:#1b1e21;border-color:#1b1e21}.close{float:right;font-size:1.5rem;font-weight:700;line-height:1;color:#000;text-shadow:0 1px 0 #fff;opacity:.5}.close:hover{color:#000;text-decoration:none}.close:not(:disabled):not(.disabled):focus,.close:not(:disabled):not(.disabled):hover{opacity:.75}button.close{padding:0;background-color:transparent;border:0;-webkit-appearance:none;-moz-appearance:none;appearance:none}a.close.disabled{pointer-events:none}.toast{max-width:350px;overflow:hidden;font-size:.875rem;background-color:rgba(255,255,255,.85);background-clip:padding-box;border:1px solid rgba(0,0,0,.1);box-shadow:0 .25rem .75rem rgba(0,0,0,.1);-webkit-backdrop-filter:blur(10px);backdrop-filter:blur(10px);opacity:0;border-radius:.25rem}.toast:not(:last-child){margin-bottom:.75rem}.toast.showing{opacity:1}.toast.show{display:block;opacity:1}.toast.hide{display:none}.toast-header{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;padding:.25rem .75rem;color:#6c757d;background-color:rgba(255,255,255,.85);background-clip:padding-box;border-bottom:1px solid rgba(0,0,0,.05)}.toast-body{padding:.75rem}.modal-open{overflow:hidden}.modal-open .modal{overflow-x:hidden;overflow-y:auto}.modal{position:fixed;top:0;left:0;z-index:1050;display:none;width:100%;height:100%;overflow:hidden;outline:0}.modal-dialog{position:relative;width:auto;margin:.5rem;pointer-events:none}.modal.fade .modal-dialog{transition:-webkit-transform .3s ease-out;transition:transform .3s ease-out;transition:transform .3s ease-out,-webkit-transform .3s ease-out;-webkit-transform:translate(0,-50px);transform:translate(0,-50px)}@media (prefers-reduced-motion:reduce){.modal.fade .modal-dialog{transition:none}}.modal.show .modal-dialog{-webkit-transform:none;transform:none}.modal-dialog-scrollable{display:-ms-flexbox;display:flex;max-height:calc(100% - 1rem)}.modal-dialog-scrollable .modal-content{max-height:calc(100vh - 1rem);overflow:hidden}.modal-dialog-scrollable .modal-footer,.modal-dialog-scrollable .modal-header{-ms-flex-negative:0;flex-shrink:0}.modal-dialog-scrollable .modal-body{overflow-y:auto}.modal-dialog-centered{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;min-height:calc(100% - 1rem)}.modal-dialog-centered::before{display:block;height:calc(100vh - 1rem);content:""}.modal-dialog-centered.modal-dialog-scrollable{-ms-flex-direction:column;flex-direction:column;-ms-flex-pack:center;justify-content:center;height:100%}.modal-dialog-centered.modal-dialog-scrollable .modal-content{max-height:none}.modal-dialog-centered.modal-dialog-scrollable::before{content:none}.modal-content{position:relative;display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;width:100%;pointer-events:auto;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.2);border-radius:.3rem;outline:0}.modal-backdrop{position:fixed;top:0;left:0;z-index:1040;width:100vw;height:100vh;background-color:#000}.modal-backdrop.fade{opacity:0}.modal-backdrop.show{opacity:.5}.modal-header{display:-ms-flexbox;display:flex;-ms-flex-align:start;align-items:flex-start;-ms-flex-pack:justify;justify-content:space-between;padding:1rem 1rem;border-bottom:1px solid #dee2e6;border-top-left-radius:.3rem;border-top-right-radius:.3rem}.modal-header .close{padding:1rem 1rem;margin:-1rem -1rem -1rem auto}.modal-title{margin-bottom:0;line-height:1.5}.modal-body{position:relative;-ms-flex:1 1 auto;flex:1 1 auto;padding:1rem}.modal-footer{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:end;justify-content:flex-end;padding:1rem;border-top:1px solid #dee2e6;border-bottom-right-radius:.3rem;border-bottom-left-radius:.3rem}.modal-footer>:not(:first-child){margin-left:.25rem}.modal-footer>:not(:last-child){margin-right:.25rem}.modal-scrollbar-measure{position:absolute;top:-9999px;width:50px;height:50px;overflow:scroll}@media (min-width:576px){.modal-dialog{max-width:500px;margin:1.75rem auto}.modal-dialog-scrollable{max-height:calc(100% - 3.5rem)}.modal-dialog-scrollable .modal-content{max-height:calc(100vh - 3.5rem)}.modal-dialog-centered{min-height:calc(100% - 3.5rem)}.modal-dialog-centered::before{height:calc(100vh - 3.5rem)}.modal-sm{max-width:300px}}@media (min-width:992px){.modal-lg,.modal-xl{max-width:800px}}@media (min-width:1200px){.modal-xl{max-width:1140px}}.tooltip{position:absolute;z-index:1070;display:block;margin:0;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;white-space:normal;line-break:auto;font-size:.875rem;word-wrap:break-word;opacity:0}.tooltip.show{opacity:.9}.tooltip .arrow{position:absolute;display:block;width:.8rem;height:.4rem}.tooltip .arrow::before{position:absolute;content:"";border-color:transparent;border-style:solid}.bs-tooltip-auto[x-placement^=top],.bs-tooltip-top{padding:.4rem 0}.bs-tooltip-auto[x-placement^=top] .arrow,.bs-tooltip-top .arrow{bottom:0}.bs-tooltip-auto[x-placement^=top] .arrow::before,.bs-tooltip-top .arrow::before{top:0;border-width:.4rem .4rem 0;border-top-color:#000}.bs-tooltip-auto[x-placement^=right],.bs-tooltip-right{padding:0 .4rem}.bs-tooltip-auto[x-placement^=right] .arrow,.bs-tooltip-right .arrow{left:0;width:.4rem;height:.8rem}.bs-tooltip-auto[x-placement^=right] .arrow::before,.bs-tooltip-right .arrow::before{right:0;border-width:.4rem .4rem .4rem 0;border-right-color:#000}.bs-tooltip-auto[x-placement^=bottom],.bs-tooltip-bottom{padding:.4rem 0}.bs-tooltip-auto[x-placement^=bottom] .arrow,.bs-tooltip-bottom .arrow{top:0}.bs-tooltip-auto[x-placement^=bottom] .arrow::before,.bs-tooltip-bottom .arrow::before{bottom:0;border-width:0 .4rem .4rem;border-bottom-color:#000}.bs-tooltip-auto[x-placement^=left],.bs-tooltip-left{padding:0 .4rem}.bs-tooltip-auto[x-placement^=left] .arrow,.bs-tooltip-left .arrow{right:0;width:.4rem;height:.8rem}.bs-tooltip-auto[x-placement^=left] .arrow::before,.bs-tooltip-left .arrow::before{left:0;border-width:.4rem 0 .4rem .4rem;border-left-color:#000}.tooltip-inner{max-width:200px;padding:.25rem .5rem;color:#fff;text-align:center;background-color:#000;border-radius:.25rem}.popover{position:absolute;top:0;left:0;z-index:1060;display:block;max-width:276px;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;white-space:normal;line-break:auto;font-size:.875rem;word-wrap:break-word;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.2);border-radius:.3rem}.popover .arrow{position:absolute;display:block;width:1rem;height:.5rem;margin:0 .3rem}.popover .arrow::after,.popover .arrow::before{position:absolute;display:block;content:"";border-color:transparent;border-style:solid}.bs-popover-auto[x-placement^=top],.bs-popover-top{margin-bottom:.5rem}.bs-popover-auto[x-placement^=top]>.arrow,.bs-popover-top>.arrow{bottom:calc((.5rem + 1px) * -1)}.bs-popover-auto[x-placement^=top]>.arrow::before,.bs-popover-top>.arrow::before{bottom:0;border-width:.5rem .5rem 0;border-top-color:rgba(0,0,0,.25)}.bs-popover-auto[x-placement^=top]>.arrow::after,.bs-popover-top>.arrow::after{bottom:1px;border-width:.5rem .5rem 0;border-top-color:#fff}.bs-popover-auto[x-placement^=right],.bs-popover-right{margin-left:.5rem}.bs-popover-auto[x-placement^=right]>.arrow,.bs-popover-right>.arrow{left:calc((.5rem + 1px) * -1);width:.5rem;height:1rem;margin:.3rem 0}.bs-popover-auto[x-placement^=right]>.arrow::before,.bs-popover-right>.arrow::before{left:0;border-width:.5rem .5rem .5rem 0;border-right-color:rgba(0,0,0,.25)}.bs-popover-auto[x-placement^=right]>.arrow::after,.bs-popover-right>.arrow::after{left:1px;border-width:.5rem .5rem .5rem 0;border-right-color:#fff}.bs-popover-auto[x-placement^=bottom],.bs-popover-bottom{margin-top:.5rem}.bs-popover-auto[x-placement^=bottom]>.arrow,.bs-popover-bottom>.arrow{top:calc((.5rem + 1px) * -1)}.bs-popover-auto[x-placement^=bottom]>.arrow::before,.bs-popover-bottom>.arrow::before{top:0;border-width:0 .5rem .5rem .5rem;border-bottom-color:rgba(0,0,0,.25)}.bs-popover-auto[x-placement^=bottom]>.arrow::after,.bs-popover-bottom>.arrow::after{top:1px;border-width:0 .5rem .5rem .5rem;border-bottom-color:#fff}.bs-popover-auto[x-placement^=bottom] .popover-header::before,.bs-popover-bottom .popover-header::before{position:absolute;top:0;left:50%;display:block;width:1rem;margin-left:-.5rem;content:"";border-bottom:1px solid #f7f7f7}.bs-popover-auto[x-placement^=left],.bs-popover-left{margin-right:.5rem}.bs-popover-auto[x-placement^=left]>.arrow,.bs-popover-left>.arrow{right:calc((.5rem + 1px) * -1);width:.5rem;height:1rem;margin:.3rem 0}.bs-popover-auto[x-placement^=left]>.arrow::before,.bs-popover-left>.arrow::before{right:0;border-width:.5rem 0 .5rem .5rem;border-left-color:rgba(0,0,0,.25)}.bs-popover-auto[x-placement^=left]>.arrow::after,.bs-popover-left>.arrow::after{right:1px;border-width:.5rem 0 .5rem .5rem;border-left-color:#fff}.popover-header{padding:.5rem .75rem;margin-bottom:0;font-size:1rem;background-color:#f7f7f7;border-bottom:1px solid #ebebeb;border-top-left-radius:calc(.3rem - 1px);border-top-right-radius:calc(.3rem - 1px)}.popover-header:empty{display:none}.popover-body{padding:.5rem .75rem;color:#212529}.carousel{position:relative}.carousel.pointer-event{-ms-touch-action:pan-y;touch-action:pan-y}.carousel-inner{position:relative;width:100%;overflow:hidden}.carousel-inner::after{display:block;clear:both;content:""}.carousel-item{position:relative;display:none;float:left;width:100%;margin-right:-100%;-webkit-backface-visibility:hidden;backface-visibility:hidden;transition:-webkit-transform .6s ease-in-out;transition:transform .6s ease-in-out;transition:transform .6s ease-in-out,-webkit-transform .6s ease-in-out}@media (prefers-reduced-motion:reduce){.carousel-item{transition:none}}.carousel-item-next,.carousel-item-prev,.carousel-item.active{display:block}.active.carousel-item-right,.carousel-item-next:not(.carousel-item-left){-webkit-transform:translateX(100%);transform:translateX(100%)}.active.carousel-item-left,.carousel-item-prev:not(.carousel-item-right){-webkit-transform:translateX(-100%);transform:translateX(-100%)}.carousel-fade .carousel-item{opacity:0;transition-property:opacity;-webkit-transform:none;transform:none}.carousel-fade .carousel-item-next.carousel-item-left,.carousel-fade .carousel-item-prev.carousel-item-right,.carousel-fade .carousel-item.active{z-index:1;opacity:1}.carousel-fade .active.carousel-item-left,.carousel-fade .active.carousel-item-right{z-index:0;opacity:0;transition:0s .6s opacity}@media (prefers-reduced-motion:reduce){.carousel-fade .active.carousel-item-left,.carousel-fade .active.carousel-item-right{transition:none}}.carousel-control-next,.carousel-control-prev{position:absolute;top:0;bottom:0;z-index:1;display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center;width:15%;color:#fff;text-align:center;opacity:.5;transition:opacity .15s ease}@media (prefers-reduced-motion:reduce){.carousel-control-next,.carousel-control-prev{transition:none}}.carousel-control-next:focus,.carousel-control-next:hover,.carousel-control-prev:focus,.carousel-control-prev:hover{color:#fff;text-decoration:none;outline:0;opacity:.9}.carousel-control-prev{left:0}.carousel-control-next{right:0}.carousel-control-next-icon,.carousel-control-prev-icon{display:inline-block;width:20px;height:20px;background:no-repeat 50%/100% 100%}.carousel-control-prev-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='%23fff' viewBox='0 0 8 8'%3e%3cpath d='M5.25 0l-4 4 4 4 1.5-1.5-2.5-2.5 2.5-2.5-1.5-1.5z'/%3e%3c/svg%3e")}.carousel-control-next-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='%23fff' viewBox='0 0 8 8'%3e%3cpath d='M2.75 0l-1.5 1.5 2.5 2.5-2.5 2.5 1.5 1.5 4-4-4-4z'/%3e%3c/svg%3e")}.carousel-indicators{position:absolute;right:0;bottom:0;left:0;z-index:15;display:-ms-flexbox;display:flex;-ms-flex-pack:center;justify-content:center;padding-left:0;margin-right:15%;margin-left:15%;list-style:none}.carousel-indicators li{box-sizing:content-box;-ms-flex:0 1 auto;flex:0 1 auto;width:30px;height:3px;margin-right:3px;margin-left:3px;text-indent:-999px;cursor:pointer;background-color:#fff;background-clip:padding-box;border-top:10px solid transparent;border-bottom:10px solid transparent;opacity:.5;transition:opacity .6s ease}@media (prefers-reduced-motion:reduce){.carousel-indicators li{transition:none}}.carousel-indicators .active{opacity:1}.carousel-caption{position:absolute;right:15%;bottom:20px;left:15%;z-index:10;padding-top:20px;padding-bottom:20px;color:#fff;text-align:center}@-webkit-keyframes spinner-border{to{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}@keyframes spinner-border{to{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}.spinner-border{display:inline-block;width:2rem;height:2rem;vertical-align:text-bottom;border:.25em solid currentColor;border-right-color:transparent;border-radius:50%;-webkit-animation:spinner-border .75s linear infinite;animation:spinner-border .75s linear infinite}.spinner-border-sm{width:1rem;height:1rem;border-width:.2em}@-webkit-keyframes spinner-grow{0%{-webkit-transform:scale(0);transform:scale(0)}50%{opacity:1}}@keyframes spinner-grow{0%{-webkit-transform:scale(0);transform:scale(0)}50%{opacity:1}}.spinner-grow{display:inline-block;width:2rem;height:2rem;vertical-align:text-bottom;background-color:currentColor;border-radius:50%;opacity:0;-webkit-animation:spinner-grow .75s linear infinite;animation:spinner-grow .75s linear infinite}.spinner-grow-sm{width:1rem;height:1rem}.align-baseline{vertical-align:baseline!important}.align-top{vertical-align:top!important}.align-middle{vertical-align:middle!important}.align-bottom{vertical-align:bottom!important}.align-text-bottom{vertical-align:text-bottom!important}.align-text-top{vertical-align:text-top!important}.bg-primary{background-color:#007bff!important}a.bg-primary:focus,a.bg-primary:hover,button.bg-primary:focus,button.bg-primary:hover{background-color:#0062cc!important}.bg-secondary{background-color:#6c757d!important}a.bg-secondary:focus,a.bg-secondary:hover,button.bg-secondary:focus,button.bg-secondary:hover{background-color:#545b62!important}.bg-success{background-color:#28a745!important}a.bg-success:focus,a.bg-success:hover,button.bg-success:focus,button.bg-success:hover{background-color:#1e7e34!important}.bg-info{background-color:#17a2b8!important}a.bg-info:focus,a.bg-info:hover,button.bg-info:focus,button.bg-info:hover{background-color:#117a8b!important}.bg-warning{background-color:#ffc107!important}a.bg-warning:focus,a.bg-warning:hover,button.bg-warning:focus,button.bg-warning:hover{background-color:#d39e00!important}.bg-danger{background-color:#dc3545!important}a.bg-danger:focus,a.bg-danger:hover,button.bg-danger:focus,button.bg-danger:hover{background-color:#bd2130!important}.bg-light{background-color:#f8f9fa!important}a.bg-light:focus,a.bg-light:hover,button.bg-light:focus,button.bg-light:hover{background-color:#dae0e5!important}.bg-dark{background-color:#343a40!important}a.bg-dark:focus,a.bg-dark:hover,button.bg-dark:focus,button.bg-dark:hover{background-color:#1d2124!important}.bg-white{background-color:#fff!important}.bg-transparent{background-color:transparent!important}.border{border:1px solid #dee2e6!important}.border-top{border-top:1px solid #dee2e6!important}.border-right{border-right:1px solid #dee2e6!important}.border-bottom{border-bottom:1px solid #dee2e6!important}.border-left{border-left:1px solid #dee2e6!important}.border-0{border:0!important}.border-top-0{border-top:0!important}.border-right-0{border-right:0!important}.border-bottom-0{border-bottom:0!important}.border-left-0{border-left:0!important}.border-primary{border-color:#007bff!important}.border-secondary{border-color:#6c757d!important}.border-success{border-color:#28a745!important}.border-info{border-color:#17a2b8!important}.border-warning{border-color:#ffc107!important}.border-danger{border-color:#dc3545!important}.border-light{border-color:#f8f9fa!important}.border-dark{border-color:#343a40!important}.border-white{border-color:#fff!important}.rounded-sm{border-radius:.2rem!important}.rounded{border-radius:.25rem!important}.rounded-top{border-top-left-radius:.25rem!important;border-top-right-radius:.25rem!important}.rounded-right{border-top-right-radius:.25rem!important;border-bottom-right-radius:.25rem!important}.rounded-bottom{border-bottom-right-radius:.25rem!important;border-bottom-left-radius:.25rem!important}.rounded-left{border-top-left-radius:.25rem!important;border-bottom-left-radius:.25rem!important}.rounded-lg{border-radius:.3rem!important}.rounded-circle{border-radius:50%!important}.rounded-pill{border-radius:50rem!important}.rounded-0{border-radius:0!important}.clearfix::after{display:block;clear:both;content:""}.d-none{display:none!important}.d-inline{display:inline!important}.d-inline-block{display:inline-block!important}.d-block{display:block!important}.d-table{display:table!important}.d-table-row{display:table-row!important}.d-table-cell{display:table-cell!important}.d-flex{display:-ms-flexbox!important;display:flex!important}.d-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}@media (min-width:576px){.d-sm-none{display:none!important}.d-sm-inline{display:inline!important}.d-sm-inline-block{display:inline-block!important}.d-sm-block{display:block!important}.d-sm-table{display:table!important}.d-sm-table-row{display:table-row!important}.d-sm-table-cell{display:table-cell!important}.d-sm-flex{display:-ms-flexbox!important;display:flex!important}.d-sm-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:768px){.d-md-none{display:none!important}.d-md-inline{display:inline!important}.d-md-inline-block{display:inline-block!important}.d-md-block{display:block!important}.d-md-table{display:table!important}.d-md-table-row{display:table-row!important}.d-md-table-cell{display:table-cell!important}.d-md-flex{display:-ms-flexbox!important;display:flex!important}.d-md-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:992px){.d-lg-none{display:none!important}.d-lg-inline{display:inline!important}.d-lg-inline-block{display:inline-block!important}.d-lg-block{display:block!important}.d-lg-table{display:table!important}.d-lg-table-row{display:table-row!important}.d-lg-table-cell{display:table-cell!important}.d-lg-flex{display:-ms-flexbox!important;display:flex!important}.d-lg-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:1200px){.d-xl-none{display:none!important}.d-xl-inline{display:inline!important}.d-xl-inline-block{display:inline-block!important}.d-xl-block{display:block!important}.d-xl-table{display:table!important}.d-xl-table-row{display:table-row!important}.d-xl-table-cell{display:table-cell!important}.d-xl-flex{display:-ms-flexbox!important;display:flex!important}.d-xl-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media print{.d-print-none{display:none!important}.d-print-inline{display:inline!important}.d-print-inline-block{display:inline-block!important}.d-print-block{display:block!important}.d-print-table{display:table!important}.d-print-table-row{display:table-row!important}.d-print-table-cell{display:table-cell!important}.d-print-flex{display:-ms-flexbox!important;display:flex!important}.d-print-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}.embed-responsive{position:relative;display:block;width:100%;padding:0;overflow:hidden}.embed-responsive::before{display:block;content:""}.embed-responsive .embed-responsive-item,.embed-responsive embed,.embed-responsive iframe,.embed-responsive object,.embed-responsive video{position:absolute;top:0;bottom:0;left:0;width:100%;height:100%;border:0}.embed-responsive-21by9::before{padding-top:42.857143%}.embed-responsive-16by9::before{padding-top:56.25%}.embed-responsive-4by3::before{padding-top:75%}.embed-responsive-1by1::before{padding-top:100%}.flex-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-center{-ms-flex-align:center!important;align-items:center!important}.align-items-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}@media (min-width:576px){.flex-sm-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-sm-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-sm-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-sm-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-sm-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-sm-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-sm-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-sm-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-sm-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-sm-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-sm-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-sm-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-sm-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-sm-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-sm-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-sm-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-sm-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-sm-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-sm-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-sm-center{-ms-flex-align:center!important;align-items:center!important}.align-items-sm-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-sm-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-sm-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-sm-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-sm-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-sm-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-sm-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-sm-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-sm-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-sm-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-sm-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-sm-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-sm-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-sm-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}@media (min-width:768px){.flex-md-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-md-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-md-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-md-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-md-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-md-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-md-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-md-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-md-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-md-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-md-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-md-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-md-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-md-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-md-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-md-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-md-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-md-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-md-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-md-center{-ms-flex-align:center!important;align-items:center!important}.align-items-md-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-md-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-md-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-md-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-md-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-md-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-md-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-md-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-md-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-md-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-md-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-md-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-md-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-md-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}@media (min-width:992px){.flex-lg-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-lg-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-lg-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-lg-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-lg-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-lg-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-lg-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-lg-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-lg-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-lg-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-lg-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-lg-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-lg-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-lg-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-lg-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-lg-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-lg-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-lg-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-lg-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-lg-center{-ms-flex-align:center!important;align-items:center!important}.align-items-lg-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-lg-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-lg-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-lg-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-lg-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-lg-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-lg-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-lg-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-lg-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-lg-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-lg-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-lg-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-lg-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-lg-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}@media (min-width:1200px){.flex-xl-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-xl-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-xl-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-xl-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-xl-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-xl-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-xl-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-xl-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-xl-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-xl-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-xl-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-xl-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-xl-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-xl-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-xl-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-xl-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-xl-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-xl-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-xl-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-xl-center{-ms-flex-align:center!important;align-items:center!important}.align-items-xl-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-xl-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-xl-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-xl-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-xl-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-xl-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-xl-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-xl-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-xl-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-xl-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-xl-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-xl-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-xl-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-xl-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}.float-left{float:left!important}.float-right{float:right!important}.float-none{float:none!important}@media (min-width:576px){.float-sm-left{float:left!important}.float-sm-right{float:right!important}.float-sm-none{float:none!important}}@media (min-width:768px){.float-md-left{float:left!important}.float-md-right{float:right!important}.float-md-none{float:none!important}}@media (min-width:992px){.float-lg-left{float:left!important}.float-lg-right{float:right!important}.float-lg-none{float:none!important}}@media (min-width:1200px){.float-xl-left{float:left!important}.float-xl-right{float:right!important}.float-xl-none{float:none!important}}.overflow-auto{overflow:auto!important}.overflow-hidden{overflow:hidden!important}.position-static{position:static!important}.position-relative{position:relative!important}.position-absolute{position:absolute!important}.position-fixed{position:fixed!important}.position-sticky{position:-webkit-sticky!important;position:sticky!important}.fixed-top{position:fixed;top:0;right:0;left:0;z-index:1030}.fixed-bottom{position:fixed;right:0;bottom:0;left:0;z-index:1030}@supports ((position:-webkit-sticky) or (position:sticky)){.sticky-top{position:-webkit-sticky;position:sticky;top:0;z-index:1020}}.sr-only{position:absolute;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap;border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;overflow:visible;clip:auto;white-space:normal}.shadow-sm{box-shadow:0 .125rem .25rem rgba(0,0,0,.075)!important}.shadow{box-shadow:0 .5rem 1rem rgba(0,0,0,.15)!important}.shadow-lg{box-shadow:0 1rem 3rem rgba(0,0,0,.175)!important}.shadow-none{box-shadow:none!important}.w-25{width:25%!important}.w-50{width:50%!important}.w-75{width:75%!important}.w-100{width:100%!important}.w-auto{width:auto!important}.h-25{height:25%!important}.h-50{height:50%!important}.h-75{height:75%!important}.h-100{height:100%!important}.h-auto{height:auto!important}.mw-100{max-width:100%!important}.mh-100{max-height:100%!important}.min-vw-100{min-width:100vw!important}.min-vh-100{min-height:100vh!important}.vw-100{width:100vw!important}.vh-100{height:100vh!important}.stretched-link::after{position:absolute;top:0;right:0;bottom:0;left:0;z-index:1;pointer-events:auto;content:"";background-color:rgba(0,0,0,0)}.m-0{margin:0!important}.mt-0,.my-0{margin-top:0!important}.mr-0,.mx-0{margin-right:0!important}.mb-0,.my-0{margin-bottom:0!important}.ml-0,.mx-0{margin-left:0!important}.m-1{margin:.25rem!important}.mt-1,.my-1{margin-top:.25rem!important}.mr-1,.mx-1{margin-right:.25rem!important}.mb-1,.my-1{margin-bottom:.25rem!important}.ml-1,.mx-1{margin-left:.25rem!important}.m-2{margin:.5rem!important}.mt-2,.my-2{margin-top:.5rem!important}.mr-2,.mx-2{margin-right:.5rem!important}.mb-2,.my-2{margin-bottom:.5rem!important}.ml-2,.mx-2{margin-left:.5rem!important}.m-3{margin:1rem!important}.mt-3,.my-3{margin-top:1rem!important}.mr-3,.mx-3{margin-right:1rem!important}.mb-3,.my-3{margin-bottom:1rem!important}.ml-3,.mx-3{margin-left:1rem!important}.m-4{margin:1.5rem!important}.mt-4,.my-4{margin-top:1.5rem!important}.mr-4,.mx-4{margin-right:1.5rem!important}.mb-4,.my-4{margin-bottom:1.5rem!important}.ml-4,.mx-4{margin-left:1.5rem!important}.m-5{margin:3rem!important}.mt-5,.my-5{margin-top:3rem!important}.mr-5,.mx-5{margin-right:3rem!important}.mb-5,.my-5{margin-bottom:3rem!important}.ml-5,.mx-5{margin-left:3rem!important}.p-0{padding:0!important}.pt-0,.py-0{padding-top:0!important}.pr-0,.px-0{padding-right:0!important}.pb-0,.py-0{padding-bottom:0!important}.pl-0,.px-0{padding-left:0!important}.p-1{padding:.25rem!important}.pt-1,.py-1{padding-top:.25rem!important}.pr-1,.px-1{padding-right:.25rem!important}.pb-1,.py-1{padding-bottom:.25rem!important}.pl-1,.px-1{padding-left:.25rem!important}.p-2{padding:.5rem!important}.pt-2,.py-2{padding-top:.5rem!important}.pr-2,.px-2{padding-right:.5rem!important}.pb-2,.py-2{padding-bottom:.5rem!important}.pl-2,.px-2{padding-left:.5rem!important}.p-3{padding:1rem!important}.pt-3,.py-3{padding-top:1rem!important}.pr-3,.px-3{padding-right:1rem!important}.pb-3,.py-3{padding-bottom:1rem!important}.pl-3,.px-3{padding-left:1rem!important}.p-4{padding:1.5rem!important}.pt-4,.py-4{padding-top:1.5rem!important}.pr-4,.px-4{padding-right:1.5rem!important}.pb-4,.py-4{padding-bottom:1.5rem!important}.pl-4,.px-4{padding-left:1.5rem!important}.p-5{padding:3rem!important}.pt-5,.py-5{padding-top:3rem!important}.pr-5,.px-5{padding-right:3rem!important}.pb-5,.py-5{padding-bottom:3rem!important}.pl-5,.px-5{padding-left:3rem!important}.m-n1{margin:-.25rem!important}.mt-n1,.my-n1{margin-top:-.25rem!important}.mr-n1,.mx-n1{margin-right:-.25rem!important}.mb-n1,.my-n1{margin-bottom:-.25rem!important}.ml-n1,.mx-n1{margin-left:-.25rem!important}.m-n2{margin:-.5rem!important}.mt-n2,.my-n2{margin-top:-.5rem!important}.mr-n2,.mx-n2{margin-right:-.5rem!important}.mb-n2,.my-n2{margin-bottom:-.5rem!important}.ml-n2,.mx-n2{margin-left:-.5rem!important}.m-n3{margin:-1rem!important}.mt-n3,.my-n3{margin-top:-1rem!important}.mr-n3,.mx-n3{margin-right:-1rem!important}.mb-n3,.my-n3{margin-bottom:-1rem!important}.ml-n3,.mx-n3{margin-left:-1rem!important}.m-n4{margin:-1.5rem!important}.mt-n4,.my-n4{margin-top:-1.5rem!important}.mr-n4,.mx-n4{margin-right:-1.5rem!important}.mb-n4,.my-n4{margin-bottom:-1.5rem!important}.ml-n4,.mx-n4{margin-left:-1.5rem!important}.m-n5{margin:-3rem!important}.mt-n5,.my-n5{margin-top:-3rem!important}.mr-n5,.mx-n5{margin-right:-3rem!important}.mb-n5,.my-n5{margin-bottom:-3rem!important}.ml-n5,.mx-n5{margin-left:-3rem!important}.m-auto{margin:auto!important}.mt-auto,.my-auto{margin-top:auto!important}.mr-auto,.mx-auto{margin-right:auto!important}.mb-auto,.my-auto{margin-bottom:auto!important}.ml-auto,.mx-auto{margin-left:auto!important}@media (min-width:576px){.m-sm-0{margin:0!important}.mt-sm-0,.my-sm-0{margin-top:0!important}.mr-sm-0,.mx-sm-0{margin-right:0!important}.mb-sm-0,.my-sm-0{margin-bottom:0!important}.ml-sm-0,.mx-sm-0{margin-left:0!important}.m-sm-1{margin:.25rem!important}.mt-sm-1,.my-sm-1{margin-top:.25rem!important}.mr-sm-1,.mx-sm-1{margin-right:.25rem!important}.mb-sm-1,.my-sm-1{margin-bottom:.25rem!important}.ml-sm-1,.mx-sm-1{margin-left:.25rem!important}.m-sm-2{margin:.5rem!important}.mt-sm-2,.my-sm-2{margin-top:.5rem!important}.mr-sm-2,.mx-sm-2{margin-right:.5rem!important}.mb-sm-2,.my-sm-2{margin-bottom:.5rem!important}.ml-sm-2,.mx-sm-2{margin-left:.5rem!important}.m-sm-3{margin:1rem!important}.mt-sm-3,.my-sm-3{margin-top:1rem!important}.mr-sm-3,.mx-sm-3{margin-right:1rem!important}.mb-sm-3,.my-sm-3{margin-bottom:1rem!important}.ml-sm-3,.mx-sm-3{margin-left:1rem!important}.m-sm-4{margin:1.5rem!important}.mt-sm-4,.my-sm-4{margin-top:1.5rem!important}.mr-sm-4,.mx-sm-4{margin-right:1.5rem!important}.mb-sm-4,.my-sm-4{margin-bottom:1.5rem!important}.ml-sm-4,.mx-sm-4{margin-left:1.5rem!important}.m-sm-5{margin:3rem!important}.mt-sm-5,.my-sm-5{margin-top:3rem!important}.mr-sm-5,.mx-sm-5{margin-right:3rem!important}.mb-sm-5,.my-sm-5{margin-bottom:3rem!important}.ml-sm-5,.mx-sm-5{margin-left:3rem!important}.p-sm-0{padding:0!important}.pt-sm-0,.py-sm-0{padding-top:0!important}.pr-sm-0,.px-sm-0{padding-right:0!important}.pb-sm-0,.py-sm-0{padding-bottom:0!important}.pl-sm-0,.px-sm-0{padding-left:0!important}.p-sm-1{padding:.25rem!important}.pt-sm-1,.py-sm-1{padding-top:.25rem!important}.pr-sm-1,.px-sm-1{padding-right:.25rem!important}.pb-sm-1,.py-sm-1{padding-bottom:.25rem!important}.pl-sm-1,.px-sm-1{padding-left:.25rem!important}.p-sm-2{padding:.5rem!important}.pt-sm-2,.py-sm-2{padding-top:.5rem!important}.pr-sm-2,.px-sm-2{padding-right:.5rem!important}.pb-sm-2,.py-sm-2{padding-bottom:.5rem!important}.pl-sm-2,.px-sm-2{padding-left:.5rem!important}.p-sm-3{padding:1rem!important}.pt-sm-3,.py-sm-3{padding-top:1rem!important}.pr-sm-3,.px-sm-3{padding-right:1rem!important}.pb-sm-3,.py-sm-3{padding-bottom:1rem!important}.pl-sm-3,.px-sm-3{padding-left:1rem!important}.p-sm-4{padding:1.5rem!important}.pt-sm-4,.py-sm-4{padding-top:1.5rem!important}.pr-sm-4,.px-sm-4{padding-right:1.5rem!important}.pb-sm-4,.py-sm-4{padding-bottom:1.5rem!important}.pl-sm-4,.px-sm-4{padding-left:1.5rem!important}.p-sm-5{padding:3rem!important}.pt-sm-5,.py-sm-5{padding-top:3rem!important}.pr-sm-5,.px-sm-5{padding-right:3rem!important}.pb-sm-5,.py-sm-5{padding-bottom:3rem!important}.pl-sm-5,.px-sm-5{padding-left:3rem!important}.m-sm-n1{margin:-.25rem!important}.mt-sm-n1,.my-sm-n1{margin-top:-.25rem!important}.mr-sm-n1,.mx-sm-n1{margin-right:-.25rem!important}.mb-sm-n1,.my-sm-n1{margin-bottom:-.25rem!important}.ml-sm-n1,.mx-sm-n1{margin-left:-.25rem!important}.m-sm-n2{margin:-.5rem!important}.mt-sm-n2,.my-sm-n2{margin-top:-.5rem!important}.mr-sm-n2,.mx-sm-n2{margin-right:-.5rem!important}.mb-sm-n2,.my-sm-n2{margin-bottom:-.5rem!important}.ml-sm-n2,.mx-sm-n2{margin-left:-.5rem!important}.m-sm-n3{margin:-1rem!important}.mt-sm-n3,.my-sm-n3{margin-top:-1rem!important}.mr-sm-n3,.mx-sm-n3{margin-right:-1rem!important}.mb-sm-n3,.my-sm-n3{margin-bottom:-1rem!important}.ml-sm-n3,.mx-sm-n3{margin-left:-1rem!important}.m-sm-n4{margin:-1.5rem!important}.mt-sm-n4,.my-sm-n4{margin-top:-1.5rem!important}.mr-sm-n4,.mx-sm-n4{margin-right:-1.5rem!important}.mb-sm-n4,.my-sm-n4{margin-bottom:-1.5rem!important}.ml-sm-n4,.mx-sm-n4{margin-left:-1.5rem!important}.m-sm-n5{margin:-3rem!important}.mt-sm-n5,.my-sm-n5{margin-top:-3rem!important}.mr-sm-n5,.mx-sm-n5{margin-right:-3rem!important}.mb-sm-n5,.my-sm-n5{margin-bottom:-3rem!important}.ml-sm-n5,.mx-sm-n5{margin-left:-3rem!important}.m-sm-auto{margin:auto!important}.mt-sm-auto,.my-sm-auto{margin-top:auto!important}.mr-sm-auto,.mx-sm-auto{margin-right:auto!important}.mb-sm-auto,.my-sm-auto{margin-bottom:auto!important}.ml-sm-auto,.mx-sm-auto{margin-left:auto!important}}@media (min-width:768px){.m-md-0{margin:0!important}.mt-md-0,.my-md-0{margin-top:0!important}.mr-md-0,.mx-md-0{margin-right:0!important}.mb-md-0,.my-md-0{margin-bottom:0!important}.ml-md-0,.mx-md-0{margin-left:0!important}.m-md-1{margin:.25rem!important}.mt-md-1,.my-md-1{margin-top:.25rem!important}.mr-md-1,.mx-md-1{margin-right:.25rem!important}.mb-md-1,.my-md-1{margin-bottom:.25rem!important}.ml-md-1,.mx-md-1{margin-left:.25rem!important}.m-md-2{margin:.5rem!important}.mt-md-2,.my-md-2{margin-top:.5rem!important}.mr-md-2,.mx-md-2{margin-right:.5rem!important}.mb-md-2,.my-md-2{margin-bottom:.5rem!important}.ml-md-2,.mx-md-2{margin-left:.5rem!important}.m-md-3{margin:1rem!important}.mt-md-3,.my-md-3{margin-top:1rem!important}.mr-md-3,.mx-md-3{margin-right:1rem!important}.mb-md-3,.my-md-3{margin-bottom:1rem!important}.ml-md-3,.mx-md-3{margin-left:1rem!important}.m-md-4{margin:1.5rem!important}.mt-md-4,.my-md-4{margin-top:1.5rem!important}.mr-md-4,.mx-md-4{margin-right:1.5rem!important}.mb-md-4,.my-md-4{margin-bottom:1.5rem!important}.ml-md-4,.mx-md-4{margin-left:1.5rem!important}.m-md-5{margin:3rem!important}.mt-md-5,.my-md-5{margin-top:3rem!important}.mr-md-5,.mx-md-5{margin-right:3rem!important}.mb-md-5,.my-md-5{margin-bottom:3rem!important}.ml-md-5,.mx-md-5{margin-left:3rem!important}.p-md-0{padding:0!important}.pt-md-0,.py-md-0{padding-top:0!important}.pr-md-0,.px-md-0{padding-right:0!important}.pb-md-0,.py-md-0{padding-bottom:0!important}.pl-md-0,.px-md-0{padding-left:0!important}.p-md-1{padding:.25rem!important}.pt-md-1,.py-md-1{padding-top:.25rem!important}.pr-md-1,.px-md-1{padding-right:.25rem!important}.pb-md-1,.py-md-1{padding-bottom:.25rem!important}.pl-md-1,.px-md-1{padding-left:.25rem!important}.p-md-2{padding:.5rem!important}.pt-md-2,.py-md-2{padding-top:.5rem!important}.pr-md-2,.px-md-2{padding-right:.5rem!important}.pb-md-2,.py-md-2{padding-bottom:.5rem!important}.pl-md-2,.px-md-2{padding-left:.5rem!important}.p-md-3{padding:1rem!important}.pt-md-3,.py-md-3{padding-top:1rem!important}.pr-md-3,.px-md-3{padding-right:1rem!important}.pb-md-3,.py-md-3{padding-bottom:1rem!important}.pl-md-3,.px-md-3{padding-left:1rem!important}.p-md-4{padding:1.5rem!important}.pt-md-4,.py-md-4{padding-top:1.5rem!important}.pr-md-4,.px-md-4{padding-right:1.5rem!important}.pb-md-4,.py-md-4{padding-bottom:1.5rem!important}.pl-md-4,.px-md-4{padding-left:1.5rem!important}.p-md-5{padding:3rem!important}.pt-md-5,.py-md-5{padding-top:3rem!important}.pr-md-5,.px-md-5{padding-right:3rem!important}.pb-md-5,.py-md-5{padding-bottom:3rem!important}.pl-md-5,.px-md-5{padding-left:3rem!important}.m-md-n1{margin:-.25rem!important}.mt-md-n1,.my-md-n1{margin-top:-.25rem!important}.mr-md-n1,.mx-md-n1{margin-right:-.25rem!important}.mb-md-n1,.my-md-n1{margin-bottom:-.25rem!important}.ml-md-n1,.mx-md-n1{margin-left:-.25rem!important}.m-md-n2{margin:-.5rem!important}.mt-md-n2,.my-md-n2{margin-top:-.5rem!important}.mr-md-n2,.mx-md-n2{margin-right:-.5rem!important}.mb-md-n2,.my-md-n2{margin-bottom:-.5rem!important}.ml-md-n2,.mx-md-n2{margin-left:-.5rem!important}.m-md-n3{margin:-1rem!important}.mt-md-n3,.my-md-n3{margin-top:-1rem!important}.mr-md-n3,.mx-md-n3{margin-right:-1rem!important}.mb-md-n3,.my-md-n3{margin-bottom:-1rem!important}.ml-md-n3,.mx-md-n3{margin-left:-1rem!important}.m-md-n4{margin:-1.5rem!important}.mt-md-n4,.my-md-n4{margin-top:-1.5rem!important}.mr-md-n4,.mx-md-n4{margin-right:-1.5rem!important}.mb-md-n4,.my-md-n4{margin-bottom:-1.5rem!important}.ml-md-n4,.mx-md-n4{margin-left:-1.5rem!important}.m-md-n5{margin:-3rem!important}.mt-md-n5,.my-md-n5{margin-top:-3rem!important}.mr-md-n5,.mx-md-n5{margin-right:-3rem!important}.mb-md-n5,.my-md-n5{margin-bottom:-3rem!important}.ml-md-n5,.mx-md-n5{margin-left:-3rem!important}.m-md-auto{margin:auto!important}.mt-md-auto,.my-md-auto{margin-top:auto!important}.mr-md-auto,.mx-md-auto{margin-right:auto!important}.mb-md-auto,.my-md-auto{margin-bottom:auto!important}.ml-md-auto,.mx-md-auto{margin-left:auto!important}}@media (min-width:992px){.m-lg-0{margin:0!important}.mt-lg-0,.my-lg-0{margin-top:0!important}.mr-lg-0,.mx-lg-0{margin-right:0!important}.mb-lg-0,.my-lg-0{margin-bottom:0!important}.ml-lg-0,.mx-lg-0{margin-left:0!important}.m-lg-1{margin:.25rem!important}.mt-lg-1,.my-lg-1{margin-top:.25rem!important}.mr-lg-1,.mx-lg-1{margin-right:.25rem!important}.mb-lg-1,.my-lg-1{margin-bottom:.25rem!important}.ml-lg-1,.mx-lg-1{margin-left:.25rem!important}.m-lg-2{margin:.5rem!important}.mt-lg-2,.my-lg-2{margin-top:.5rem!important}.mr-lg-2,.mx-lg-2{margin-right:.5rem!important}.mb-lg-2,.my-lg-2{margin-bottom:.5rem!important}.ml-lg-2,.mx-lg-2{margin-left:.5rem!important}.m-lg-3{margin:1rem!important}.mt-lg-3,.my-lg-3{margin-top:1rem!important}.mr-lg-3,.mx-lg-3{margin-right:1rem!important}.mb-lg-3,.my-lg-3{margin-bottom:1rem!important}.ml-lg-3,.mx-lg-3{margin-left:1rem!important}.m-lg-4{margin:1.5rem!important}.mt-lg-4,.my-lg-4{margin-top:1.5rem!important}.mr-lg-4,.mx-lg-4{margin-right:1.5rem!important}.mb-lg-4,.my-lg-4{margin-bottom:1.5rem!important}.ml-lg-4,.mx-lg-4{margin-left:1.5rem!important}.m-lg-5{margin:3rem!important}.mt-lg-5,.my-lg-5{margin-top:3rem!important}.mr-lg-5,.mx-lg-5{margin-right:3rem!important}.mb-lg-5,.my-lg-5{margin-bottom:3rem!important}.ml-lg-5,.mx-lg-5{margin-left:3rem!important}.p-lg-0{padding:0!important}.pt-lg-0,.py-lg-0{padding-top:0!important}.pr-lg-0,.px-lg-0{padding-right:0!important}.pb-lg-0,.py-lg-0{padding-bottom:0!important}.pl-lg-0,.px-lg-0{padding-left:0!important}.p-lg-1{padding:.25rem!important}.pt-lg-1,.py-lg-1{padding-top:.25rem!important}.pr-lg-1,.px-lg-1{padding-right:.25rem!important}.pb-lg-1,.py-lg-1{padding-bottom:.25rem!important}.pl-lg-1,.px-lg-1{padding-left:.25rem!important}.p-lg-2{padding:.5rem!important}.pt-lg-2,.py-lg-2{padding-top:.5rem!important}.pr-lg-2,.px-lg-2{padding-right:.5rem!important}.pb-lg-2,.py-lg-2{padding-bottom:.5rem!important}.pl-lg-2,.px-lg-2{padding-left:.5rem!important}.p-lg-3{padding:1rem!important}.pt-lg-3,.py-lg-3{padding-top:1rem!important}.pr-lg-3,.px-lg-3{padding-right:1rem!important}.pb-lg-3,.py-lg-3{padding-bottom:1rem!important}.pl-lg-3,.px-lg-3{padding-left:1rem!important}.p-lg-4{padding:1.5rem!important}.pt-lg-4,.py-lg-4{padding-top:1.5rem!important}.pr-lg-4,.px-lg-4{padding-right:1.5rem!important}.pb-lg-4,.py-lg-4{padding-bottom:1.5rem!important}.pl-lg-4,.px-lg-4{padding-left:1.5rem!important}.p-lg-5{padding:3rem!important}.pt-lg-5,.py-lg-5{padding-top:3rem!important}.pr-lg-5,.px-lg-5{padding-right:3rem!important}.pb-lg-5,.py-lg-5{padding-bottom:3rem!important}.pl-lg-5,.px-lg-5{padding-left:3rem!important}.m-lg-n1{margin:-.25rem!important}.mt-lg-n1,.my-lg-n1{margin-top:-.25rem!important}.mr-lg-n1,.mx-lg-n1{margin-right:-.25rem!important}.mb-lg-n1,.my-lg-n1{margin-bottom:-.25rem!important}.ml-lg-n1,.mx-lg-n1{margin-left:-.25rem!important}.m-lg-n2{margin:-.5rem!important}.mt-lg-n2,.my-lg-n2{margin-top:-.5rem!important}.mr-lg-n2,.mx-lg-n2{margin-right:-.5rem!important}.mb-lg-n2,.my-lg-n2{margin-bottom:-.5rem!important}.ml-lg-n2,.mx-lg-n2{margin-left:-.5rem!important}.m-lg-n3{margin:-1rem!important}.mt-lg-n3,.my-lg-n3{margin-top:-1rem!important}.mr-lg-n3,.mx-lg-n3{margin-right:-1rem!important}.mb-lg-n3,.my-lg-n3{margin-bottom:-1rem!important}.ml-lg-n3,.mx-lg-n3{margin-left:-1rem!important}.m-lg-n4{margin:-1.5rem!important}.mt-lg-n4,.my-lg-n4{margin-top:-1.5rem!important}.mr-lg-n4,.mx-lg-n4{margin-right:-1.5rem!important}.mb-lg-n4,.my-lg-n4{margin-bottom:-1.5rem!important}.ml-lg-n4,.mx-lg-n4{margin-left:-1.5rem!important}.m-lg-n5{margin:-3rem!important}.mt-lg-n5,.my-lg-n5{margin-top:-3rem!important}.mr-lg-n5,.mx-lg-n5{margin-right:-3rem!important}.mb-lg-n5,.my-lg-n5{margin-bottom:-3rem!important}.ml-lg-n5,.mx-lg-n5{margin-left:-3rem!important}.m-lg-auto{margin:auto!important}.mt-lg-auto,.my-lg-auto{margin-top:auto!important}.mr-lg-auto,.mx-lg-auto{margin-right:auto!important}.mb-lg-auto,.my-lg-auto{margin-bottom:auto!important}.ml-lg-auto,.mx-lg-auto{margin-left:auto!important}}@media (min-width:1200px){.m-xl-0{margin:0!important}.mt-xl-0,.my-xl-0{margin-top:0!important}.mr-xl-0,.mx-xl-0{margin-right:0!important}.mb-xl-0,.my-xl-0{margin-bottom:0!important}.ml-xl-0,.mx-xl-0{margin-left:0!important}.m-xl-1{margin:.25rem!important}.mt-xl-1,.my-xl-1{margin-top:.25rem!important}.mr-xl-1,.mx-xl-1{margin-right:.25rem!important}.mb-xl-1,.my-xl-1{margin-bottom:.25rem!important}.ml-xl-1,.mx-xl-1{margin-left:.25rem!important}.m-xl-2{margin:.5rem!important}.mt-xl-2,.my-xl-2{margin-top:.5rem!important}.mr-xl-2,.mx-xl-2{margin-right:.5rem!important}.mb-xl-2,.my-xl-2{margin-bottom:.5rem!important}.ml-xl-2,.mx-xl-2{margin-left:.5rem!important}.m-xl-3{margin:1rem!important}.mt-xl-3,.my-xl-3{margin-top:1rem!important}.mr-xl-3,.mx-xl-3{margin-right:1rem!important}.mb-xl-3,.my-xl-3{margin-bottom:1rem!important}.ml-xl-3,.mx-xl-3{margin-left:1rem!important}.m-xl-4{margin:1.5rem!important}.mt-xl-4,.my-xl-4{margin-top:1.5rem!important}.mr-xl-4,.mx-xl-4{margin-right:1.5rem!important}.mb-xl-4,.my-xl-4{margin-bottom:1.5rem!important}.ml-xl-4,.mx-xl-4{margin-left:1.5rem!important}.m-xl-5{margin:3rem!important}.mt-xl-5,.my-xl-5{margin-top:3rem!important}.mr-xl-5,.mx-xl-5{margin-right:3rem!important}.mb-xl-5,.my-xl-5{margin-bottom:3rem!important}.ml-xl-5,.mx-xl-5{margin-left:3rem!important}.p-xl-0{padding:0!important}.pt-xl-0,.py-xl-0{padding-top:0!important}.pr-xl-0,.px-xl-0{padding-right:0!important}.pb-xl-0,.py-xl-0{padding-bottom:0!important}.pl-xl-0,.px-xl-0{padding-left:0!important}.p-xl-1{padding:.25rem!important}.pt-xl-1,.py-xl-1{padding-top:.25rem!important}.pr-xl-1,.px-xl-1{padding-right:.25rem!important}.pb-xl-1,.py-xl-1{padding-bottom:.25rem!important}.pl-xl-1,.px-xl-1{padding-left:.25rem!important}.p-xl-2{padding:.5rem!important}.pt-xl-2,.py-xl-2{padding-top:.5rem!important}.pr-xl-2,.px-xl-2{padding-right:.5rem!important}.pb-xl-2,.py-xl-2{padding-bottom:.5rem!important}.pl-xl-2,.px-xl-2{padding-left:.5rem!important}.p-xl-3{padding:1rem!important}.pt-xl-3,.py-xl-3{padding-top:1rem!important}.pr-xl-3,.px-xl-3{padding-right:1rem!important}.pb-xl-3,.py-xl-3{padding-bottom:1rem!important}.pl-xl-3,.px-xl-3{padding-left:1rem!important}.p-xl-4{padding:1.5rem!important}.pt-xl-4,.py-xl-4{padding-top:1.5rem!important}.pr-xl-4,.px-xl-4{padding-right:1.5rem!important}.pb-xl-4,.py-xl-4{padding-bottom:1.5rem!important}.pl-xl-4,.px-xl-4{padding-left:1.5rem!important}.p-xl-5{padding:3rem!important}.pt-xl-5,.py-xl-5{padding-top:3rem!important}.pr-xl-5,.px-xl-5{padding-right:3rem!important}.pb-xl-5,.py-xl-5{padding-bottom:3rem!important}.pl-xl-5,.px-xl-5{padding-left:3rem!important}.m-xl-n1{margin:-.25rem!important}.mt-xl-n1,.my-xl-n1{margin-top:-.25rem!important}.mr-xl-n1,.mx-xl-n1{margin-right:-.25rem!important}.mb-xl-n1,.my-xl-n1{margin-bottom:-.25rem!important}.ml-xl-n1,.mx-xl-n1{margin-left:-.25rem!important}.m-xl-n2{margin:-.5rem!important}.mt-xl-n2,.my-xl-n2{margin-top:-.5rem!important}.mr-xl-n2,.mx-xl-n2{margin-right:-.5rem!important}.mb-xl-n2,.my-xl-n2{margin-bottom:-.5rem!important}.ml-xl-n2,.mx-xl-n2{margin-left:-.5rem!important}.m-xl-n3{margin:-1rem!important}.mt-xl-n3,.my-xl-n3{margin-top:-1rem!important}.mr-xl-n3,.mx-xl-n3{margin-right:-1rem!important}.mb-xl-n3,.my-xl-n3{margin-bottom:-1rem!important}.ml-xl-n3,.mx-xl-n3{margin-left:-1rem!important}.m-xl-n4{margin:-1.5rem!important}.mt-xl-n4,.my-xl-n4{margin-top:-1.5rem!important}.mr-xl-n4,.mx-xl-n4{margin-right:-1.5rem!important}.mb-xl-n4,.my-xl-n4{margin-bottom:-1.5rem!important}.ml-xl-n4,.mx-xl-n4{margin-left:-1.5rem!important}.m-xl-n5{margin:-3rem!important}.mt-xl-n5,.my-xl-n5{margin-top:-3rem!important}.mr-xl-n5,.mx-xl-n5{margin-right:-3rem!important}.mb-xl-n5,.my-xl-n5{margin-bottom:-3rem!important}.ml-xl-n5,.mx-xl-n5{margin-left:-3rem!important}.m-xl-auto{margin:auto!important}.mt-xl-auto,.my-xl-auto{margin-top:auto!important}.mr-xl-auto,.mx-xl-auto{margin-right:auto!important}.mb-xl-auto,.my-xl-auto{margin-bottom:auto!important}.ml-xl-auto,.mx-xl-auto{margin-left:auto!important}}.text-monospace{font-family:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace!important}.text-justify{text-align:justify!important}.text-wrap{white-space:normal!important}.text-nowrap{white-space:nowrap!important}.text-truncate{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.text-left{text-align:left!important}.text-right{text-align:right!important}.text-center{text-align:center!important}@media (min-width:576px){.text-sm-left{text-align:left!important}.text-sm-right{text-align:right!important}.text-sm-center{text-align:center!important}}@media (min-width:768px){.text-md-left{text-align:left!important}.text-md-right{text-align:right!important}.text-md-center{text-align:center!important}}@media (min-width:992px){.text-lg-left{text-align:left!important}.text-lg-right{text-align:right!important}.text-lg-center{text-align:center!important}}@media (min-width:1200px){.text-xl-left{text-align:left!important}.text-xl-right{text-align:right!important}.text-xl-center{text-align:center!important}}.text-lowercase{text-transform:lowercase!important}.text-uppercase{text-transform:uppercase!important}.text-capitalize{text-transform:capitalize!important}.font-weight-light{font-weight:300!important}.font-weight-lighter{font-weight:lighter!important}.font-weight-normal{font-weight:400!important}.font-weight-bold{font-weight:700!important}.font-weight-bolder{font-weight:bolder!important}.font-italic{font-style:italic!important}.text-white{color:#fff!important}.text-primary{color:#007bff!important}a.text-primary:focus,a.text-primary:hover{color:#0056b3!important}.text-secondary{color:#6c757d!important}a.text-secondary:focus,a.text-secondary:hover{color:#494f54!important}.text-success{color:#28a745!important}a.text-success:focus,a.text-success:hover{color:#19692c!important}.text-info{color:#17a2b8!important}a.text-info:focus,a.text-info:hover{color:#0f6674!important}.text-warning{color:#ffc107!important}a.text-warning:focus,a.text-warning:hover{color:#ba8b00!important}.text-danger{color:#dc3545!important}a.text-danger:focus,a.text-danger:hover{color:#a71d2a!important}.text-light{color:#f8f9fa!important}a.text-light:focus,a.text-light:hover{color:#cbd3da!important}.text-dark{color:#343a40!important}a.text-dark:focus,a.text-dark:hover{color:#121416!important}.text-body{color:#212529!important}.text-muted{color:#6c757d!important}.text-black-50{color:rgba(0,0,0,.5)!important}.text-white-50{color:rgba(255,255,255,.5)!important}.text-hide{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.text-decoration-none{text-decoration:none!important}.text-break{word-break:break-word!important;overflow-wrap:break-word!important}.text-reset{color:inherit!important}.visible{visibility:visible!important}.invisible{visibility:hidden!important}@media print{*,::after,::before{text-shadow:none!important;box-shadow:none!important}a:not(.btn){text-decoration:underline}abbr[title]::after{content:" (" attr(title) ")"}pre{white-space:pre-wrap!important}blockquote,pre{border:1px solid #adb5bd;page-break-inside:avoid}thead{display:table-header-group}img,tr{page-break-inside:avoid}h2,h3,p{orphans:3;widows:3}h2,h3{page-break-after:avoid}@page{size:a3}body{min-width:992px!important}.container{min-width:992px!important}.navbar{display:none}.badge{border:1px solid #000}.table{border-collapse:collapse!important}.table td,.table th{background-color:#fff!important}.table-bordered td,.table-bordered th{border:1px solid #dee2e6!important}.table-dark{color:inherit}.table-dark tbody+tbody,.table-dark td,.table-dark th,.table-dark thead th{border-color:#dee2e6}.table .thead-dark th{color:inherit;border-color:#dee2e6}} \ No newline at end of file diff --git a/doc/themes/scikit-learn-modern/static/js/searchtools.js b/doc/themes/scikit-learn-modern/static/js/searchtools.js deleted file mode 100644 index 0d4ca2328b079..0000000000000 --- a/doc/themes/scikit-learn-modern/static/js/searchtools.js +++ /dev/null @@ -1,595 +0,0 @@ -/* - * searchtools.js - * ~~~~~~~~~~~~~~~~ - * - * Sphinx JavaScript utilities for the full-text search. - * - * :copyright: Copyright 2007-2019 by the Sphinx team, see AUTHORS. - * :license: BSD, see LICENSE for details. - * - * CHANGELOG: - * - Removes ajax call to get context for each result - * - Adjusts Search.query to remove duplicates in search results. - * - Adjusts Scorer to rank objects higher. - * - Adds Search._total_non_object_results to limit the number of search non - * object results. Object results do not perform another GET resquest, so they - * are cheap to display. - */ - -if (!Scorer) { - /** - * Simple result scoring code. - */ - var Scorer = { - // Implement the following function to further tweak the score for each result - // The function takes a result array [filename, title, anchor, descr, score] - // and returns the new score. - /* - score: function(result) { - return result[4]; - }, - */ - - // query matches the full name of an object - objNameMatch: 15, - // or matches in the last dotted part of the object name - objPartialMatch: 15, - // Additive scores depending on the priority of the object - objPrio: { - 0: 15, // used to be importantResults - 1: 5, // used to be objectResults - 2: -5 - }, // used to be unimportantResults - // Used when the priority is not in the mapping. - objPrioDefault: 0, - - // query found in title - title: 15, - partialTitle: 7, - // query found in terms - term: 10, - partialTerm: 2 - }; -} - -if (!splitQuery) { - function splitQuery(query) { - return query.split(/\s+/); - } -} - -/** - * Search Module - */ -var Search = { - _index: null, - _queued_query: null, - _pulse_status: -1, - _total_non_object_results: 10, - - htmlToText: function (htmlString) { - var htmlString = htmlString.replace(//g, ""); - var htmlElement = document.createElement("span"); - htmlElement.innerHTML = htmlString; - $(htmlElement) - .find(".headerlink") - .remove(); - docContent = $(htmlElement).find("[role=main]")[0]; - return docContent.textContent || docContent.innerText; - }, - - init: function () { - var params = $.getQueryParameters(); - if (params.q) { - var query = params.q[0]; - $('input[name="q"]')[0].value = query; - this.performSearch(query); - } - }, - - loadIndex: function (url) { - $.ajax({ - type: "GET", - url: url, - data: null, - dataType: "script", - cache: true, - complete: function (jqxhr, textstatus) { - if (textstatus != "success") { - document.getElementById("searchindexloader").src = url; - } - } - }); - }, - - setIndex: function (index) { - var q; - this._index = index; - if ((q = this._queued_query) !== null) { - this._queued_query = null; - Search.query(q); - } - }, - - hasIndex: function () { - return this._index !== null; - }, - - deferQuery: function (query) { - this._queued_query = query; - }, - - stopPulse: function () { - this._pulse_status = 0; - }, - - startPulse: function () { - if (this._pulse_status >= 0) return; - function pulse() { - var i; - Search._pulse_status = (Search._pulse_status + 1) % 4; - var dotString = ""; - for (i = 0; i < Search._pulse_status; i++) dotString += "."; - Search.dots.text(dotString); - if (Search._pulse_status > -1) window.setTimeout(pulse, 500); - } - pulse(); - }, - - /** - * perform a search for something (or wait until index is loaded) - */ - performSearch: function (query) { - // create the required interface elements - this.out = $("#search-results"); - this.title = $("

" + _("Searching") + "

").appendTo(this.out); - this.dots = $("").appendTo(this.title); - this.status = $('

 

').appendTo(this.out); - this.output = $('
',trigger:"hover focus",title:"",delay:0,html:!1,selector:!1,placement:"top",offset:0,container:!1,fallbackPlacement:"flip",boundary:"scrollParent",sanitize:!0,sanitizeFn:null,whiteList:Ee},je="show",He="out",Re={HIDE:"hide"+De,HIDDEN:"hidden"+De,SHOW:"show"+De,SHOWN:"shown"+De,INSERTED:"inserted"+De,CLICK:"click"+De,FOCUSIN:"focusin"+De,FOCUSOUT:"focusout"+De,MOUSEENTER:"mouseenter"+De,MOUSELEAVE:"mouseleave"+De},xe="fade",Fe="show",Ue=".tooltip-inner",We=".arrow",qe="hover",Me="focus",Ke="click",Qe="manual",Be=function(){function i(t,e){if("undefined"==typeof u)throw new TypeError("Bootstrap's tooltips require Popper.js (https://popper.js.org/)");this._isEnabled=!0,this._timeout=0,this._hoverState="",this._activeTrigger={},this._popper=null,this.element=t,this.config=this._getConfig(e),this.tip=null,this._setListeners()}var t=i.prototype;return t.enable=function(){this._isEnabled=!0},t.disable=function(){this._isEnabled=!1},t.toggleEnabled=function(){this._isEnabled=!this._isEnabled},t.toggle=function(t){if(this._isEnabled)if(t){var e=this.constructor.DATA_KEY,n=g(t.currentTarget).data(e);n||(n=new this.constructor(t.currentTarget,this._getDelegateConfig()),g(t.currentTarget).data(e,n)),n._activeTrigger.click=!n._activeTrigger.click,n._isWithActiveTrigger()?n._enter(null,n):n._leave(null,n)}else{if(g(this.getTipElement()).hasClass(Fe))return void this._leave(null,this);this._enter(null,this)}},t.dispose=function(){clearTimeout(this._timeout),g.removeData(this.element,this.constructor.DATA_KEY),g(this.element).off(this.constructor.EVENT_KEY),g(this.element).closest(".modal").off("hide.bs.modal"),this.tip&&g(this.tip).remove(),this._isEnabled=null,this._timeout=null,this._hoverState=null,(this._activeTrigger=null)!==this._popper&&this._popper.destroy(),this._popper=null,this.element=null,this.config=null,this.tip=null},t.show=function(){var e=this;if("none"===g(this.element).css("display"))throw new Error("Please use show on visible elements");var t=g.Event(this.constructor.Event.SHOW);if(this.isWithContent()&&this._isEnabled){g(this.element).trigger(t);var n=_.findShadowRoot(this.element),i=g.contains(null!==n?n:this.element.ownerDocument.documentElement,this.element);if(t.isDefaultPrevented()||!i)return;var o=this.getTipElement(),r=_.getUID(this.constructor.NAME);o.setAttribute("id",r),this.element.setAttribute("aria-describedby",r),this.setContent(),this.config.animation&&g(o).addClass(xe);var s="function"==typeof this.config.placement?this.config.placement.call(this,o,this.element):this.config.placement,a=this._getAttachment(s);this.addAttachmentClass(a);var l=this._getContainer();g(o).data(this.constructor.DATA_KEY,this),g.contains(this.element.ownerDocument.documentElement,this.tip)||g(o).appendTo(l),g(this.element).trigger(this.constructor.Event.INSERTED),this._popper=new u(this.element,o,{placement:a,modifiers:{offset:this._getOffset(),flip:{behavior:this.config.fallbackPlacement},arrow:{element:We},preventOverflow:{boundariesElement:this.config.boundary}},onCreate:function(t){t.originalPlacement!==t.placement&&e._handlePopperPlacementChange(t)},onUpdate:function(t){return e._handlePopperPlacementChange(t)}}),g(o).addClass(Fe),"ontouchstart"in document.documentElement&&g(document.body).children().on("mouseover",null,g.noop);var c=function(){e.config.animation&&e._fixTransition();var t=e._hoverState;e._hoverState=null,g(e.element).trigger(e.constructor.Event.SHOWN),t===He&&e._leave(null,e)};if(g(this.tip).hasClass(xe)){var h=_.getTransitionDurationFromElement(this.tip);g(this.tip).one(_.TRANSITION_END,c).emulateTransitionEnd(h)}else c()}},t.hide=function(t){var e=this,n=this.getTipElement(),i=g.Event(this.constructor.Event.HIDE),o=function(){e._hoverState!==je&&n.parentNode&&n.parentNode.removeChild(n),e._cleanTipClass(),e.element.removeAttribute("aria-describedby"),g(e.element).trigger(e.constructor.Event.HIDDEN),null!==e._popper&&e._popper.destroy(),t&&t()};if(g(this.element).trigger(i),!i.isDefaultPrevented()){if(g(n).removeClass(Fe),"ontouchstart"in document.documentElement&&g(document.body).children().off("mouseover",null,g.noop),this._activeTrigger[Ke]=!1,this._activeTrigger[Me]=!1,this._activeTrigger[qe]=!1,g(this.tip).hasClass(xe)){var r=_.getTransitionDurationFromElement(n);g(n).one(_.TRANSITION_END,o).emulateTransitionEnd(r)}else o();this._hoverState=""}},t.update=function(){null!==this._popper&&this._popper.scheduleUpdate()},t.isWithContent=function(){return Boolean(this.getTitle())},t.addAttachmentClass=function(t){g(this.getTipElement()).addClass(Ae+"-"+t)},t.getTipElement=function(){return this.tip=this.tip||g(this.config.template)[0],this.tip},t.setContent=function(){var t=this.getTipElement();this.setElementContent(g(t.querySelectorAll(Ue)),this.getTitle()),g(t).removeClass(xe+" "+Fe)},t.setElementContent=function(t,e){"object"!=typeof e||!e.nodeType&&!e.jquery?this.config.html?(this.config.sanitize&&(e=Se(e,this.config.whiteList,this.config.sanitizeFn)),t.html(e)):t.text(e):this.config.html?g(e).parent().is(t)||t.empty().append(e):t.text(g(e).text())},t.getTitle=function(){var t=this.element.getAttribute("data-original-title");return t||(t="function"==typeof this.config.title?this.config.title.call(this.element):this.config.title),t},t._getOffset=function(){var e=this,t={};return"function"==typeof this.config.offset?t.fn=function(t){return t.offsets=l({},t.offsets,e.config.offset(t.offsets,e.element)||{}),t}:t.offset=this.config.offset,t},t._getContainer=function(){return!1===this.config.container?document.body:_.isElement(this.config.container)?g(this.config.container):g(document).find(this.config.container)},t._getAttachment=function(t){return Pe[t.toUpperCase()]},t._setListeners=function(){var i=this;this.config.trigger.split(" ").forEach(function(t){if("click"===t)g(i.element).on(i.constructor.Event.CLICK,i.config.selector,function(t){return i.toggle(t)});else if(t!==Qe){var e=t===qe?i.constructor.Event.MOUSEENTER:i.constructor.Event.FOCUSIN,n=t===qe?i.constructor.Event.MOUSELEAVE:i.constructor.Event.FOCUSOUT;g(i.element).on(e,i.config.selector,function(t){return i._enter(t)}).on(n,i.config.selector,function(t){return i._leave(t)})}}),g(this.element).closest(".modal").on("hide.bs.modal",function(){i.element&&i.hide()}),this.config.selector?this.config=l({},this.config,{trigger:"manual",selector:""}):this._fixTitle()},t._fixTitle=function(){var t=typeof this.element.getAttribute("data-original-title");(this.element.getAttribute("title")||"string"!==t)&&(this.element.setAttribute("data-original-title",this.element.getAttribute("title")||""),this.element.setAttribute("title",""))},t._enter=function(t,e){var n=this.constructor.DATA_KEY;(e=e||g(t.currentTarget).data(n))||(e=new this.constructor(t.currentTarget,this._getDelegateConfig()),g(t.currentTarget).data(n,e)),t&&(e._activeTrigger["focusin"===t.type?Me:qe]=!0),g(e.getTipElement()).hasClass(Fe)||e._hoverState===je?e._hoverState=je:(clearTimeout(e._timeout),e._hoverState=je,e.config.delay&&e.config.delay.show?e._timeout=setTimeout(function(){e._hoverState===je&&e.show()},e.config.delay.show):e.show())},t._leave=function(t,e){var n=this.constructor.DATA_KEY;(e=e||g(t.currentTarget).data(n))||(e=new this.constructor(t.currentTarget,this._getDelegateConfig()),g(t.currentTarget).data(n,e)),t&&(e._activeTrigger["focusout"===t.type?Me:qe]=!1),e._isWithActiveTrigger()||(clearTimeout(e._timeout),e._hoverState=He,e.config.delay&&e.config.delay.hide?e._timeout=setTimeout(function(){e._hoverState===He&&e.hide()},e.config.delay.hide):e.hide())},t._isWithActiveTrigger=function(){for(var t in this._activeTrigger)if(this._activeTrigger[t])return!0;return!1},t._getConfig=function(t){var e=g(this.element).data();return Object.keys(e).forEach(function(t){-1!==Oe.indexOf(t)&&delete e[t]}),"number"==typeof(t=l({},this.constructor.Default,e,"object"==typeof t&&t?t:{})).delay&&(t.delay={show:t.delay,hide:t.delay}),"number"==typeof t.title&&(t.title=t.title.toString()),"number"==typeof t.content&&(t.content=t.content.toString()),_.typeCheckConfig(be,t,this.constructor.DefaultType),t.sanitize&&(t.template=Se(t.template,t.whiteList,t.sanitizeFn)),t},t._getDelegateConfig=function(){var t={};if(this.config)for(var e in this.config)this.constructor.Default[e]!==this.config[e]&&(t[e]=this.config[e]);return t},t._cleanTipClass=function(){var t=g(this.getTipElement()),e=t.attr("class").match(Ne);null!==e&&e.length&&t.removeClass(e.join(""))},t._handlePopperPlacementChange=function(t){var e=t.instance;this.tip=e.popper,this._cleanTipClass(),this.addAttachmentClass(this._getAttachment(t.placement))},t._fixTransition=function(){var t=this.getTipElement(),e=this.config.animation;null===t.getAttribute("x-placement")&&(g(t).removeClass(xe),this.config.animation=!1,this.hide(),this.show(),this.config.animation=e)},i._jQueryInterface=function(n){return this.each(function(){var t=g(this).data(Ie),e="object"==typeof n&&n;if((t||!/dispose|hide/.test(n))&&(t||(t=new i(this,e),g(this).data(Ie,t)),"string"==typeof n)){if("undefined"==typeof t[n])throw new TypeError('No method named "'+n+'"');t[n]()}})},s(i,null,[{key:"VERSION",get:function(){return"4.3.1"}},{key:"Default",get:function(){return Le}},{key:"NAME",get:function(){return be}},{key:"DATA_KEY",get:function(){return Ie}},{key:"Event",get:function(){return Re}},{key:"EVENT_KEY",get:function(){return De}},{key:"DefaultType",get:function(){return ke}}]),i}();g.fn[be]=Be._jQueryInterface,g.fn[be].Constructor=Be,g.fn[be].noConflict=function(){return g.fn[be]=we,Be._jQueryInterface};var Ve="popover",Ye="bs.popover",ze="."+Ye,Xe=g.fn[Ve],$e="bs-popover",Ge=new RegExp("(^|\\s)"+$e+"\\S+","g"),Je=l({},Be.Default,{placement:"right",trigger:"click",content:"",template:''}),Ze=l({},Be.DefaultType,{content:"(string|element|function)"}),tn="fade",en="show",nn=".popover-header",on=".popover-body",rn={HIDE:"hide"+ze,HIDDEN:"hidden"+ze,SHOW:"show"+ze,SHOWN:"shown"+ze,INSERTED:"inserted"+ze,CLICK:"click"+ze,FOCUSIN:"focusin"+ze,FOCUSOUT:"focusout"+ze,MOUSEENTER:"mouseenter"+ze,MOUSELEAVE:"mouseleave"+ze},sn=function(t){var e,n;function i(){return t.apply(this,arguments)||this}n=t,(e=i).prototype=Object.create(n.prototype),(e.prototype.constructor=e).__proto__=n;var o=i.prototype;return o.isWithContent=function(){return this.getTitle()||this._getContent()},o.addAttachmentClass=function(t){g(this.getTipElement()).addClass($e+"-"+t)},o.getTipElement=function(){return this.tip=this.tip||g(this.config.template)[0],this.tip},o.setContent=function(){var t=g(this.getTipElement());this.setElementContent(t.find(nn),this.getTitle());var e=this._getContent();"function"==typeof e&&(e=e.call(this.element)),this.setElementContent(t.find(on),e),t.removeClass(tn+" "+en)},o._getContent=function(){return this.element.getAttribute("data-content")||this.config.content},o._cleanTipClass=function(){var t=g(this.getTipElement()),e=t.attr("class").match(Ge);null!==e&&0=this._offsets[o]&&("undefined"==typeof this._offsets[o+1]||t - - - -{% endblock %} - -{% block extrahead %} - {% if pagename != 'index' %} - {% endif %} - - - - - - -{% endblock %} - -{%- if pagename == 'index' %} - {% set title = 'scikit-learn: machine learning in Python' %} -{%- endif %} - -{%- if pagename == 'documentation' %} - {% set title = 'Documentation scikit-learn: machine learning in Python' %} -{%- endif %} - -{% block header %} - -
-
- {%- if logo %} - - {%- endif %} - - {%- block navbar -%} - - - {%- endblock -%} - -
-
- - -{%- if pagename == 'index' %} - - -{%- endif %} - -{% endblock %} - -{% block content %} - - - Fork me on GitHub - - -
- -{%- if pagename != 'index' and pagename != 'documentation' %} -
-
- - {%- if rellinks %} - - {%- if parents %} -
- {% else %} -
- {% endif %} - - {%- for rellink in rellinks|reverse %} -
- {{ rellink[3]|capitalize }} -
- - {{ rellink[1]|striptags|truncate(16, killwords=True) }} - - {%- if rellink[1]|striptags %} - - {{ rellink[1]|striptags}} - - {%- endif %} - -
- {%- if not loop.last %} -
-   -
- {%- endif %} - {%- endfor %} - - - - {%- if parents %} -
-   -
-
- - Up -
- - {{ parents[-1].title|striptags|truncate(16, killwords=True) }} - - {%- if parents[-1].title|striptags != '' %} - - {{ parents[-1].title|striptags }} - - {% endif %} - -
- {%- endif %} -
- {% endif %} - - {%- if pagename != "install" %} -

{{project}} v{{ release|e }}
- Other versions

- {%- endif %} -

Please cite us if you use the software.

- {{ toc }} -
-
-{% if theme_collapsiblesidebar|tobool %} - - -{% endif %} -{% endif %} - - -
- {%- block document %} - {{ super() }} - {%- endblock %} -
-
-
-{% endblock %} - -{% block relbar1 %}{% endblock %} -{% block relbar2 %}{% endblock %} - - -{%- block footer %} -
- {%- if pagename != 'index' %} - {%- if show_copyright %} - {%- if hasdoc('copyright') %} - {% trans path=pathto('copyright'), copyright=copyright|e %}© {{ copyright }}.{% endtrans %} - {%- else %} - {% trans copyright=copyright|e %}© {{ copyright }}.{% endtrans %} - {%- endif %} - {%- endif %} - {%- if last_updated %} - {% trans last_updated=last_updated|e %}Last updated on {{ last_updated }}.{% endtrans %} - {%- endif %} - {%- if show_source and has_source and sourcename %} - {{ _('Show this page source') }} - {%- endif %} - {%- endif %} -
- - {%- if pagename != 'index' %} - {%- if parents %} -
- {% else %} -
- {% endif %} - {%- for rellink in rellinks|reverse %} -
- {{ loop.cycle('Previous', 'Next') }} - -
- {%- endfor %} - {% endif %} -
- - {% if theme_google_analytics|tobool %} - - - {% endif %} - -{%- endblock %} diff --git a/doc/themes/scikit-learn/static/ML_MAPS_README.rst b/doc/themes/scikit-learn/static/ML_MAPS_README.rst deleted file mode 100644 index 47fe633767995..0000000000000 --- a/doc/themes/scikit-learn/static/ML_MAPS_README.rst +++ /dev/null @@ -1,93 +0,0 @@ -Machine Learning Cheat Sheet (for scikit-learn) -=============================================== - -This document is intended to explain how to edit -the machine learning cheat sheet, originally created -by Andreas Mueller: - -(https://peekaboo-vision.blogspot.de/2013/01/machine-learning-cheat-sheet-for-scikit.html) - -The image is made interactive using an imagemap, and uses the jQuery Map Hilight plugin module -by David Lynch (https://davidlynch.org/projects/maphilight/docs/) to highlight -the different items on the image upon mouseover. - -Modifying the map on the docs is currently a little bit tedious, -so I'll try to make it as simple as possible. - -1. Editing the layout of the map and its paths. ------------------------------------------------- - -Use a Graphics editor like Inkscape Vector Graphics Editor -to open the ml_map.svg file, in this folder. From there -you can move objects around, etc. as you need. - -Save when done, and make sure to export a .PNG file -to replace the old-outdated ml_map.png, as that file -is used as a background image. - -2. Accessing the paths of the SVG file and exporting them. ----------------------------------------------------------- - -Use an image manipulation package like GIMP Image Editor to open -the ml_map.svg file, in this folder. With GIMP, make sure -to select 'Import paths'. - -Once the image has been opened, you can see all imported paths on the paths tab. -You can edit these or create new paths. In GIMP, right-clicking one of the -paths and choosing: Path Tool will allow you to see the paths on -the image. The paths will be exported later and will be used to -make the click able regions on our image map. - -3. Export paths as SVG files ----------------------------- - -After you've edited a path or created a new one, right click it on -the paths menu and choose 'Export Path..'. This way we extract just -that path on its own as 'new_area.svg' for example. - -4. Edit the SVG file ---------------------- -Using a script made by David Lynch, we will convert the svg files into -html maps. To do this, open the svg file in question in any text editor. -Make sure that the 'width' and 'height' are not in 'in' or 'px', i.e -"100" is OK, but "100px" or "1.25in" are not. - -Then wrap the tags in and tags. -Then the file is ready for the script. - -5. From SVG to HTML map ------------------------ - -Use the provided svg2imagemap.py script on your edited svg file: - -$ python svg2imagemap.py new_area.svg - -where new_area.svg is our file. - -6. Add the new map to the main html file ------------------------------------------- - -Copy the code from the newly created 'new_area.html' -file. Open the ml_map.html file. - -Add the that you copied -after the last tag in the ml_map.html file. - -Add the link address to 'href' and a tooltip to -'title' within your tag. - -If you wish to add the green and blue hover effect -to the area, add -data-maphilight='{"strokeColor":"0000ff","strokeWidth":5,"fillColor":"66FF66","fillOpacity":0.4}' - -to your area tag, as done in the other tags above. - -Save the file, and you're done. - ------------------------------------------------------ - -I'll take some time to make some scripts to automate this process -a bit more at some point, as it is not difficult to do, -but tedious. - --Jaques Grobler diff --git a/doc/themes/scikit-learn/static/css/bootstrap-responsive.css b/doc/themes/scikit-learn/static/css/bootstrap-responsive.css deleted file mode 100644 index 09e88ce3fecc2..0000000000000 --- a/doc/themes/scikit-learn/static/css/bootstrap-responsive.css +++ /dev/null @@ -1,1109 +0,0 @@ -/*! - * Bootstrap Responsive v2.3.2 - * - * Copyright 2012 Twitter, Inc - * Licensed under the Apache License v2.0 - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Designed and built with all the love in the world @twitter by @mdo and @fat. - */ - -.clearfix { - *zoom: 1; -} - -.clearfix:before, -.clearfix:after { - display: table; - line-height: 0; - content: ""; -} - -.clearfix:after { - clear: both; -} - -.hide-text { - font: 0/0 a; - color: transparent; - text-shadow: none; - background-color: transparent; - border: 0; -} - -.input-block-level { - display: block; - width: 100%; - min-height: 30px; - -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - box-sizing: border-box; -} - -@-ms-viewport { - width: device-width; -} - -.hidden { - display: none; - visibility: hidden; -} - -.visible-phone { - display: none !important; -} - -.visible-tablet { - display: none !important; -} - -.hidden-desktop { - display: none !important; -} - -.visible-desktop { - display: inherit !important; -} - -@media (min-width: 768px) and (max-width: 979px) { - .hidden-desktop { - display: inherit !important; - } - .visible-desktop { - display: none !important ; - } - .visible-tablet { - display: inherit !important; - } - .hidden-tablet { - display: none !important; - } -} - -@media (max-width: 767px) { - .hidden-desktop { - display: inherit !important; - } - .visible-desktop { - display: none !important; - } - .visible-phone { - display: inherit !important; - } - .hidden-phone { - display: none !important; - } -} - -.visible-print { - display: none !important; -} - -@media print { - .visible-print { - display: inherit !important; - } - .hidden-print { - display: none !important; - } -} - -@media (min-width: 1200px) { - .row { - margin-left: -30px; - *zoom: 1; - } - .row:before, - .row:after { - display: table; - line-height: 0; - content: ""; - } - .row:after { - clear: both; - } - [class*="span"] { - float: left; - min-height: 1px; - margin-left: 30px; - } - .container, - .navbar-static-top .container, - .navbar-fixed-top .container, - .navbar-fixed-bottom .container { - width: 1170px; - } - .span12 { - width: 1170px; - } - .span11 { - width: 1070px; - } - .span10 { - width: 970px; - } - .span9 { - width: 870px; - } - .span8 { - width: 770px; - } - .span7 { - width: 670px; - } - .span6 { - width: 570px; - } - .span5 { - width: 470px; - } - .span4 { - width: 370px; - } - .span3 { - width: 270px; - } - .span2 { - width: 170px; - } - .span1 { - width: 70px; - } - .offset12 { - margin-left: 1230px; - } - .offset11 { - margin-left: 1130px; - } - .offset10 { - margin-left: 1030px; - } - .offset9 { - margin-left: 930px; - } - .offset8 { - margin-left: 830px; - } - .offset7 { - margin-left: 730px; - } - .offset6 { - margin-left: 630px; - } - .offset5 { - margin-left: 530px; - } - .offset4 { - margin-left: 430px; - } - .offset3 { - margin-left: 330px; - } - .offset2 { - margin-left: 230px; - } - .offset1 { - margin-left: 130px; - } - .row-fluid { - width: 100%; - *zoom: 1; - } - .row-fluid:before, - .row-fluid:after { - display: table; - line-height: 0; - content: ""; - } - .row-fluid:after { - clear: both; - } - .row-fluid [class*="span"] { - display: block; - float: left; - width: 100%; - min-height: 30px; - margin-left: 2.564102564102564%; - *margin-left: 2.5109110747408616%; - -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - box-sizing: border-box; - } - .row-fluid [class*="span"]:first-child { - margin-left: 0; - } - .row-fluid .controls-row [class*="span"] + [class*="span"] { - margin-left: 2.564102564102564%; - } - .row-fluid .span12 { - width: 100%; - *width: 99.94680851063829%; - } - .row-fluid .span11 { - width: 91.45299145299145%; - *width: 91.39979996362975%; - } - .row-fluid .span10 { - width: 82.90598290598291%; - *width: 82.8527914166212%; - } - .row-fluid .span9 { - width: 74.35897435897436%; - *width: 74.30578286961266%; - } - .row-fluid .span8 { - width: 65.81196581196582%; - *width: 65.75877432260411%; - } - .row-fluid .span7 { - width: 57.26495726495726%; - *width: 57.21176577559556%; - } - .row-fluid .span6 { - width: 48.717948717948715%; - *width: 48.664757228587014%; - } - .row-fluid .span5 { - width: 40.17094017094017%; - *width: 40.11774868157847%; - } - .row-fluid .span4 { - width: 31.623931623931625%; - *width: 31.570740134569924%; - } - .row-fluid .span3 { - width: 23.076923076923077%; - *width: 23.023731587561375%; - } - .row-fluid .span2 { - width: 14.52991452991453%; - *width: 14.476723040552828%; - } - .row-fluid .span1 { - width: 5.982905982905983%; - *width: 5.929714493544281%; - } - .row-fluid .offset12 { - margin-left: 105.12820512820512%; - *margin-left: 105.02182214948171%; - } - .row-fluid .offset12:first-child { - margin-left: 102.56410256410257%; - *margin-left: 102.45771958537915%; - } - .row-fluid .offset11 { - margin-left: 96.58119658119658%; - *margin-left: 96.47481360247316%; - } - .row-fluid .offset11:first-child { - margin-left: 94.01709401709402%; - *margin-left: 93.91071103837061%; - } - .row-fluid .offset10 { - margin-left: 88.03418803418803%; - *margin-left: 87.92780505546462%; - } - .row-fluid .offset10:first-child { - margin-left: 85.47008547008548%; - *margin-left: 85.36370249136206%; - } - .row-fluid .offset9 { - margin-left: 79.48717948717949%; - *margin-left: 79.38079650845607%; - } - .row-fluid .offset9:first-child { - margin-left: 76.92307692307693%; - *margin-left: 76.81669394435352%; - } - .row-fluid .offset8 { - margin-left: 70.94017094017094%; - *margin-left: 70.83378796144753%; - } - .row-fluid .offset8:first-child { - margin-left: 68.37606837606839%; - *margin-left: 68.26968539734497%; - } - .row-fluid .offset7 { - margin-left: 62.393162393162385%; - *margin-left: 62.28677941443899%; - } - .row-fluid .offset7:first-child { - margin-left: 59.82905982905982%; - *margin-left: 59.72267685033642%; - } - .row-fluid .offset6 { - margin-left: 53.84615384615384%; - *margin-left: 53.739770867430444%; - } - .row-fluid .offset6:first-child { - margin-left: 51.28205128205128%; - *margin-left: 51.175668303327875%; - } - .row-fluid .offset5 { - margin-left: 45.299145299145295%; - *margin-left: 45.1927623204219%; - } - .row-fluid .offset5:first-child { - margin-left: 42.73504273504273%; - *margin-left: 42.62865975631933%; - } - .row-fluid .offset4 { - margin-left: 36.75213675213675%; - *margin-left: 36.645753773413354%; - } - .row-fluid .offset4:first-child { - margin-left: 34.18803418803419%; - *margin-left: 34.081651209310785%; - } - .row-fluid .offset3 { - margin-left: 28.205128205128204%; - *margin-left: 28.0987452264048%; - } - .row-fluid .offset3:first-child { - margin-left: 25.641025641025642%; - *margin-left: 25.53464266230224%; - } - .row-fluid .offset2 { - margin-left: 19.65811965811966%; - *margin-left: 19.551736679396257%; - } - .row-fluid .offset2:first-child { - margin-left: 17.094017094017094%; - *margin-left: 16.98763411529369%; - } - .row-fluid .offset1 { - margin-left: 11.11111111111111%; - *margin-left: 11.004728132387708%; - } - .row-fluid .offset1:first-child { - margin-left: 8.547008547008547%; - *margin-left: 8.440625568285142%; - } - input, - textarea, - .uneditable-input { - margin-left: 0; - } - .controls-row [class*="span"] + [class*="span"] { - margin-left: 30px; - } - input.span12, - textarea.span12, - .uneditable-input.span12 { - width: 1156px; - } - input.span11, - textarea.span11, - .uneditable-input.span11 { - width: 1056px; - } - input.span10, - textarea.span10, - .uneditable-input.span10 { - width: 956px; - } - input.span9, - textarea.span9, - .uneditable-input.span9 { - width: 856px; - } - input.span8, - textarea.span8, - .uneditable-input.span8 { - width: 756px; - } - input.span7, - textarea.span7, - .uneditable-input.span7 { - width: 656px; - } - input.span6, - textarea.span6, - .uneditable-input.span6 { - width: 556px; - } - input.span5, - textarea.span5, - .uneditable-input.span5 { - width: 456px; - } - input.span4, - textarea.span4, - .uneditable-input.span4 { - width: 356px; - } - input.span3, - textarea.span3, - .uneditable-input.span3 { - width: 256px; - } - input.span2, - textarea.span2, - .uneditable-input.span2 { - width: 156px; - } - input.span1, - textarea.span1, - .uneditable-input.span1 { - width: 56px; - } - .thumbnails { - margin-left: -30px; - } - .thumbnails > li { - margin-left: 30px; - } - .row-fluid .thumbnails { - margin-left: 0; - } -} - -@media (min-width: 768px) and (max-width: 979px) { - .row { - margin-left: -20px; - *zoom: 1; - } - .row:before, - .row:after { - display: table; - line-height: 0; - content: ""; - } - .row:after { - clear: both; - } - [class*="span"] { - float: left; - min-height: 1px; - margin-left: 20px; - } - .container, - .navbar-static-top .container, - .navbar-fixed-top .container, - .navbar-fixed-bottom .container { - width: 724px; - } - .span12 { - width: 724px; - } - .span11 { - width: 662px; - } - .span10 { - width: 600px; - } - .span9 { - width: 538px; - } - .span8 { - width: 476px; - } - .span7 { - width: 414px; - } - .span6 { - width: 352px; - } - .span5 { - width: 290px; - } - .span4 { - width: 228px; - } - .span3 { - width: 166px; - } - .span2 { - width: 104px; - } - .span1 { - width: 42px; - } - .offset12 { - margin-left: 764px; - } - .offset11 { - margin-left: 702px; - } - .offset10 { - margin-left: 640px; - } - .offset9 { - margin-left: 578px; - } - .offset8 { - margin-left: 516px; - } - .offset7 { - margin-left: 454px; - } - .offset6 { - margin-left: 392px; - } - .offset5 { - margin-left: 330px; - } - .offset4 { - margin-left: 268px; - } - .offset3 { - margin-left: 206px; - } - .offset2 { - margin-left: 144px; - } - .offset1 { - margin-left: 82px; - } - .row-fluid { - width: 100%; - *zoom: 1; - } - .row-fluid:before, - .row-fluid:after { - display: table; - line-height: 0; - content: ""; - } - .row-fluid:after { - clear: both; - } - .row-fluid [class*="span"] { - display: block; - float: left; - width: 100%; - min-height: 30px; - margin-left: 2.7624309392265194%; - *margin-left: 2.709239449864817%; - -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - box-sizing: border-box; - } - .row-fluid [class*="span"]:first-child { - margin-left: 0; - } - .row-fluid .controls-row [class*="span"] + [class*="span"] { - margin-left: 2.7624309392265194%; - } - .row-fluid .span12 { - width: 100%; - *width: 99.94680851063829%; - } - .row-fluid .span11 { - width: 91.43646408839778%; - *width: 91.38327259903608%; - } - .row-fluid .span10 { - width: 82.87292817679558%; - *width: 82.81973668743387%; - } - .row-fluid .span9 { - width: 74.30939226519337%; - *width: 74.25620077583166%; - } - .row-fluid .span8 { - width: 65.74585635359117%; - *width: 65.69266486422946%; - } - .row-fluid .span7 { - width: 57.18232044198895%; - *width: 57.12912895262725%; - } - .row-fluid .span6 { - width: 48.61878453038674%; - *width: 48.56559304102504%; - } - .row-fluid .span5 { - width: 40.05524861878453%; - *width: 40.00205712942283%; - } - .row-fluid .span4 { - width: 31.491712707182323%; - *width: 31.43852121782062%; - } - .row-fluid .span3 { - width: 22.92817679558011%; - *width: 22.87498530621841%; - } - .row-fluid .span2 { - width: 14.3646408839779%; - *width: 14.311449394616199%; - } - .row-fluid .span1 { - width: 5.801104972375691%; - *width: 5.747913483013988%; - } - .row-fluid .offset12 { - margin-left: 105.52486187845304%; - *margin-left: 105.41847889972962%; - } - .row-fluid .offset12:first-child { - margin-left: 102.76243093922652%; - *margin-left: 102.6560479605031%; - } - .row-fluid .offset11 { - margin-left: 96.96132596685082%; - *margin-left: 96.8549429881274%; - } - .row-fluid .offset11:first-child { - margin-left: 94.1988950276243%; - *margin-left: 94.09251204890089%; - } - .row-fluid .offset10 { - margin-left: 88.39779005524862%; - *margin-left: 88.2914070765252%; - } - .row-fluid .offset10:first-child { - margin-left: 85.6353591160221%; - *margin-left: 85.52897613729868%; - } - .row-fluid .offset9 { - margin-left: 79.8342541436464%; - *margin-left: 79.72787116492299%; - } - .row-fluid .offset9:first-child { - margin-left: 77.07182320441989%; - *margin-left: 76.96544022569647%; - } - .row-fluid .offset8 { - margin-left: 71.2707182320442%; - *margin-left: 71.16433525332079%; - } - .row-fluid .offset8:first-child { - margin-left: 68.50828729281768%; - *margin-left: 68.40190431409427%; - } - .row-fluid .offset7 { - margin-left: 62.70718232044199%; - *margin-left: 62.600799341718584%; - } - .row-fluid .offset7:first-child { - margin-left: 59.94475138121547%; - *margin-left: 59.838368402492065%; - } - .row-fluid .offset6 { - margin-left: 54.14364640883978%; - *margin-left: 54.037263430116376%; - } - .row-fluid .offset6:first-child { - margin-left: 51.38121546961326%; - *margin-left: 51.27483249088986%; - } - .row-fluid .offset5 { - margin-left: 45.58011049723757%; - *margin-left: 45.47372751851417%; - } - .row-fluid .offset5:first-child { - margin-left: 42.81767955801105%; - *margin-left: 42.71129657928765%; - } - .row-fluid .offset4 { - margin-left: 37.01657458563536%; - *margin-left: 36.91019160691196%; - } - .row-fluid .offset4:first-child { - margin-left: 34.25414364640884%; - *margin-left: 34.14776066768544%; - } - .row-fluid .offset3 { - margin-left: 28.45303867403315%; - *margin-left: 28.346655695309746%; - } - .row-fluid .offset3:first-child { - margin-left: 25.69060773480663%; - *margin-left: 25.584224756083227%; - } - .row-fluid .offset2 { - margin-left: 19.88950276243094%; - *margin-left: 19.783119783707537%; - } - .row-fluid .offset2:first-child { - margin-left: 17.12707182320442%; - *margin-left: 17.02068884448102%; - } - .row-fluid .offset1 { - margin-left: 11.32596685082873%; - *margin-left: 11.219583872105325%; - } - .row-fluid .offset1:first-child { - margin-left: 8.56353591160221%; - *margin-left: 8.457152932878806%; - } - input, - textarea, - .uneditable-input { - margin-left: 0; - } - .controls-row [class*="span"] + [class*="span"] { - margin-left: 20px; - } - input.span12, - textarea.span12, - .uneditable-input.span12 { - width: 710px; - } - input.span11, - textarea.span11, - .uneditable-input.span11 { - width: 648px; - } - input.span10, - textarea.span10, - .uneditable-input.span10 { - width: 586px; - } - input.span9, - textarea.span9, - .uneditable-input.span9 { - width: 524px; - } - input.span8, - textarea.span8, - .uneditable-input.span8 { - width: 462px; - } - input.span7, - textarea.span7, - .uneditable-input.span7 { - width: 400px; - } - input.span6, - textarea.span6, - .uneditable-input.span6 { - width: 338px; - } - input.span5, - textarea.span5, - .uneditable-input.span5 { - width: 276px; - } - input.span4, - textarea.span4, - .uneditable-input.span4 { - width: 214px; - } - input.span3, - textarea.span3, - .uneditable-input.span3 { - width: 152px; - } - input.span2, - textarea.span2, - .uneditable-input.span2 { - width: 90px; - } - input.span1, - textarea.span1, - .uneditable-input.span1 { - width: 28px; - } -} - -@media (max-width: 767px) { - body { - padding-right: 20px; - padding-left: 20px; - } - .navbar-fixed-top, - .navbar-fixed-bottom, - .navbar-static-top { - margin-right: -20px; - margin-left: -20px; - } - .container-fluid { - padding: 0; - } - .dl-horizontal dt { - float: none; - width: auto; - clear: none; - text-align: left; - } - .dl-horizontal dd { - margin-left: 0; - } - .container { - width: auto; - } - .row-fluid { - width: 100%; - } - .row, - .thumbnails { - margin-left: 0; - } - .thumbnails > li { - float: none; - margin-left: 0; - } - [class*="span"], - .uneditable-input[class*="span"], - .row-fluid [class*="span"] { - display: block; - float: none; - width: 100%; - margin-left: 0; - -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - box-sizing: border-box; - } - .span12, - .row-fluid .span12 { - width: 100%; - -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - box-sizing: border-box; - } - .row-fluid [class*="offset"]:first-child { - margin-left: 0; - } - .input-large, - .input-xlarge, - .input-xxlarge, - input[class*="span"], - select[class*="span"], - textarea[class*="span"], - .uneditable-input { - display: block; - width: 100%; - min-height: 30px; - -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - box-sizing: border-box; - } - .input-prepend input, - .input-append input, - .input-prepend input[class*="span"], - .input-append input[class*="span"] { - display: inline-block; - width: auto; - } - .controls-row [class*="span"] + [class*="span"] { - margin-left: 0; - } - .modal { - position: fixed; - top: 20px; - right: 20px; - left: 20px; - width: auto; - margin: 0; - } - .modal.fade { - top: -100px; - } - .modal.fade.in { - top: 20px; - } -} - -@media (max-width: 480px) { - .nav-collapse { - -webkit-transform: translate3d(0, 0, 0); - } - .page-header h1 small { - display: block; - line-height: 20px; - } - input[type="checkbox"], - input[type="radio"] { - border: 1px solid #ccc; - } - .form-horizontal .control-label { - float: none; - width: auto; - padding-top: 0; - text-align: left; - } - .form-horizontal .controls { - margin-left: 0; - } - .form-horizontal .control-list { - padding-top: 0; - } - .form-horizontal .form-actions { - padding-right: 10px; - padding-left: 10px; - } - .media .pull-left, - .media .pull-right { - display: block; - float: none; - margin-bottom: 10px; - } - .media-object { - margin-right: 0; - margin-left: 0; - } - .modal { - top: 10px; - right: 10px; - left: 10px; - } - .modal-header .close { - padding: 10px; - margin: -10px; - } - .carousel-caption { - position: static; - } -} - -@media (max-width: 979px) { - body { - padding-top: 0; - } - .navbar-fixed-top, - .navbar-fixed-bottom { - position: static; - } - .navbar-fixed-top { - margin-bottom: 20px; - } - .navbar-fixed-bottom { - margin-top: 20px; - } - .navbar-fixed-top .navbar-inner, - .navbar-fixed-bottom .navbar-inner { - padding: 5px; - } - .navbar .container { - width: auto; - padding: 0; - } - .navbar .brand { - padding-right: 10px; - padding-left: 10px; - margin: 0 0 0 -5px; - } - .nav-collapse { - clear: both; - } - .nav-collapse .nav { - float: none; - margin: 0 0 10px; - } - .nav-collapse .nav > li { - float: none; - } - .nav-collapse .nav > li > a { - margin-bottom: 2px; - } - .nav-collapse .nav > .divider-vertical { - display: none; - } - .nav-collapse .nav .nav-header { - color: #777777; - text-shadow: none; - } - .nav-collapse .nav > li > a, - .nav-collapse .dropdown-menu a { - padding: 9px 15px; - font-weight: bold; - color: #777777; - -webkit-border-radius: 3px; - -moz-border-radius: 3px; - border-radius: 3px; - } - .nav-collapse .btn { - padding: 4px 10px 4px; - font-weight: normal; - -webkit-border-radius: 4px; - -moz-border-radius: 4px; - border-radius: 4px; - } - .nav-collapse .dropdown-menu li + li a { - margin-bottom: 2px; - } - .nav-collapse .nav > li > a:hover, - .nav-collapse .nav > li > a:focus, - .nav-collapse .dropdown-menu a:hover, - .nav-collapse .dropdown-menu a:focus { - background-color: #f2f2f2; - } - .navbar-inverse .nav-collapse .nav > li > a, - .navbar-inverse .nav-collapse .dropdown-menu a { - color: #999999; - } - .navbar-inverse .nav-collapse .nav > li > a:hover, - .navbar-inverse .nav-collapse .nav > li > a:focus, - .navbar-inverse .nav-collapse .dropdown-menu a:hover, - .navbar-inverse .nav-collapse .dropdown-menu a:focus { - background-color: #111111; - } - .nav-collapse.in .btn-group { - padding: 0; - margin-top: 5px; - } - .nav-collapse .dropdown-menu { - position: static; - top: auto; - left: auto; - display: none; - float: none; - max-width: none; - padding: 0; - margin: 0 15px; - background-color: transparent; - border: none; - -webkit-border-radius: 0; - -moz-border-radius: 0; - border-radius: 0; - -webkit-box-shadow: none; - -moz-box-shadow: none; - box-shadow: none; - } - .nav-collapse .open > .dropdown-menu { - display: block; - } - .nav-collapse .dropdown-menu:before, - .nav-collapse .dropdown-menu:after { - display: none; - } - .nav-collapse .dropdown-menu .divider { - display: none; - } - .nav-collapse .nav > li > .dropdown-menu:before, - .nav-collapse .nav > li > .dropdown-menu:after { - display: none; - } - .nav-collapse .navbar-form, - .nav-collapse .navbar-search { - float: none; - padding: 10px 15px; - margin: 10px 0; - border-top: 1px solid #f2f2f2; - border-bottom: 1px solid #f2f2f2; - -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1); - -moz-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1); - box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1); - } - .navbar-inverse .nav-collapse .navbar-form, - .navbar-inverse .nav-collapse .navbar-search { - border-top-color: #111111; - border-bottom-color: #111111; - } - .navbar .nav-collapse .nav.pull-right { - float: none; - margin-left: 0; - } - .nav-collapse, - .nav-collapse.collapse { - height: 0; - overflow: hidden; - } - .navbar .btn-navbar { - display: block; - } - .navbar-static .navbar-inner { - padding-right: 10px; - padding-left: 10px; - } -} - -@media (min-width: 980px) { - .nav-collapse.collapse { - height: auto !important; - overflow: visible !important; - } -} diff --git a/doc/themes/scikit-learn/static/css/bootstrap-responsive.min.css b/doc/themes/scikit-learn/static/css/bootstrap-responsive.min.css deleted file mode 100644 index f4ede63f32e2e..0000000000000 --- a/doc/themes/scikit-learn/static/css/bootstrap-responsive.min.css +++ /dev/null @@ -1,9 +0,0 @@ -/*! - * Bootstrap Responsive v2.3.2 - * - * Copyright 2012 Twitter, Inc - * Licensed under the Apache License v2.0 - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Designed and built with all the love in the world @twitter by @mdo and @fat. - */.clearfix{*zoom:1}.clearfix:before,.clearfix:after{display:table;line-height:0;content:""}.clearfix:after{clear:both}.hide-text{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.input-block-level{display:block;width:100%;min-height:30px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}@-ms-viewport{width:device-width}.hidden{display:none;visibility:hidden}.visible-phone{display:none!important}.visible-tablet{display:none!important}.hidden-desktop{display:none!important}.visible-desktop{display:inherit!important}@media(min-width:768px) and (max-width:979px){.hidden-desktop{display:inherit!important}.visible-desktop{display:none!important}.visible-tablet{display:inherit!important}.hidden-tablet{display:none!important}}@media(max-width:767px){.hidden-desktop{display:inherit!important}.visible-desktop{display:none!important}.visible-phone{display:inherit!important}.hidden-phone{display:none!important}}.visible-print{display:none!important}@media print{.visible-print{display:inherit!important}.hidden-print{display:none!important}}@media(min-width:1200px){.row{margin-left:-30px;*zoom:1}.row:before,.row:after{display:table;line-height:0;content:""}.row:after{clear:both}[class*="span"]{float:left;min-height:1px;margin-left:30px}.container,.navbar-static-top .container,.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:1170px}.span12{width:1170px}.span11{width:1070px}.span10{width:970px}.span9{width:870px}.span8{width:770px}.span7{width:670px}.span6{width:570px}.span5{width:470px}.span4{width:370px}.span3{width:270px}.span2{width:170px}.span1{width:70px}.offset12{margin-left:1230px}.offset11{margin-left:1130px}.offset10{margin-left:1030px}.offset9{margin-left:930px}.offset8{margin-left:830px}.offset7{margin-left:730px}.offset6{margin-left:630px}.offset5{margin-left:530px}.offset4{margin-left:430px}.offset3{margin-left:330px}.offset2{margin-left:230px}.offset1{margin-left:130px}.row-fluid{width:100%;*zoom:1}.row-fluid:before,.row-fluid:after{display:table;line-height:0;content:""}.row-fluid:after{clear:both}.row-fluid [class*="span"]{display:block;float:left;width:100%;min-height:30px;margin-left:2.564102564102564%;*margin-left:2.5109110747408616%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.row-fluid [class*="span"]:first-child{margin-left:0}.row-fluid .controls-row [class*="span"]+[class*="span"]{margin-left:2.564102564102564%}.row-fluid .span12{width:100%;*width:99.94680851063829%}.row-fluid .span11{width:91.45299145299145%;*width:91.39979996362975%}.row-fluid .span10{width:82.90598290598291%;*width:82.8527914166212%}.row-fluid .span9{width:74.35897435897436%;*width:74.30578286961266%}.row-fluid .span8{width:65.81196581196582%;*width:65.75877432260411%}.row-fluid .span7{width:57.26495726495726%;*width:57.21176577559556%}.row-fluid .span6{width:48.717948717948715%;*width:48.664757228587014%}.row-fluid .span5{width:40.17094017094017%;*width:40.11774868157847%}.row-fluid .span4{width:31.623931623931625%;*width:31.570740134569924%}.row-fluid .span3{width:23.076923076923077%;*width:23.023731587561375%}.row-fluid .span2{width:14.52991452991453%;*width:14.476723040552828%}.row-fluid .span1{width:5.982905982905983%;*width:5.929714493544281%}.row-fluid .offset12{margin-left:105.12820512820512%;*margin-left:105.02182214948171%}.row-fluid .offset12:first-child{margin-left:102.56410256410257%;*margin-left:102.45771958537915%}.row-fluid .offset11{margin-left:96.58119658119658%;*margin-left:96.47481360247316%}.row-fluid .offset11:first-child{margin-left:94.01709401709402%;*margin-left:93.91071103837061%}.row-fluid .offset10{margin-left:88.03418803418803%;*margin-left:87.92780505546462%}.row-fluid .offset10:first-child{margin-left:85.47008547008548%;*margin-left:85.36370249136206%}.row-fluid .offset9{margin-left:79.48717948717949%;*margin-left:79.38079650845607%}.row-fluid .offset9:first-child{margin-left:76.92307692307693%;*margin-left:76.81669394435352%}.row-fluid .offset8{margin-left:70.94017094017094%;*margin-left:70.83378796144753%}.row-fluid .offset8:first-child{margin-left:68.37606837606839%;*margin-left:68.26968539734497%}.row-fluid .offset7{margin-left:62.393162393162385%;*margin-left:62.28677941443899%}.row-fluid .offset7:first-child{margin-left:59.82905982905982%;*margin-left:59.72267685033642%}.row-fluid .offset6{margin-left:53.84615384615384%;*margin-left:53.739770867430444%}.row-fluid .offset6:first-child{margin-left:51.28205128205128%;*margin-left:51.175668303327875%}.row-fluid .offset5{margin-left:45.299145299145295%;*margin-left:45.1927623204219%}.row-fluid .offset5:first-child{margin-left:42.73504273504273%;*margin-left:42.62865975631933%}.row-fluid .offset4{margin-left:36.75213675213675%;*margin-left:36.645753773413354%}.row-fluid .offset4:first-child{margin-left:34.18803418803419%;*margin-left:34.081651209310785%}.row-fluid .offset3{margin-left:28.205128205128204%;*margin-left:28.0987452264048%}.row-fluid .offset3:first-child{margin-left:25.641025641025642%;*margin-left:25.53464266230224%}.row-fluid .offset2{margin-left:19.65811965811966%;*margin-left:19.551736679396257%}.row-fluid .offset2:first-child{margin-left:17.094017094017094%;*margin-left:16.98763411529369%}.row-fluid .offset1{margin-left:11.11111111111111%;*margin-left:11.004728132387708%}.row-fluid .offset1:first-child{margin-left:8.547008547008547%;*margin-left:8.440625568285142%}input,textarea,.uneditable-input{margin-left:0}.controls-row [class*="span"]+[class*="span"]{margin-left:30px}input.span12,textarea.span12,.uneditable-input.span12{width:1156px}input.span11,textarea.span11,.uneditable-input.span11{width:1056px}input.span10,textarea.span10,.uneditable-input.span10{width:956px}input.span9,textarea.span9,.uneditable-input.span9{width:856px}input.span8,textarea.span8,.uneditable-input.span8{width:756px}input.span7,textarea.span7,.uneditable-input.span7{width:656px}input.span6,textarea.span6,.uneditable-input.span6{width:556px}input.span5,textarea.span5,.uneditable-input.span5{width:456px}input.span4,textarea.span4,.uneditable-input.span4{width:356px}input.span3,textarea.span3,.uneditable-input.span3{width:256px}input.span2,textarea.span2,.uneditable-input.span2{width:156px}input.span1,textarea.span1,.uneditable-input.span1{width:56px}.thumbnails{margin-left:-30px}.thumbnails>li{margin-left:30px}.row-fluid .thumbnails{margin-left:0}}@media(min-width:768px) and (max-width:979px){.row{margin-left:-20px;*zoom:1}.row:before,.row:after{display:table;line-height:0;content:""}.row:after{clear:both}[class*="span"]{float:left;min-height:1px;margin-left:20px}.container,.navbar-static-top .container,.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:724px}.span12{width:724px}.span11{width:662px}.span10{width:600px}.span9{width:538px}.span8{width:476px}.span7{width:414px}.span6{width:352px}.span5{width:290px}.span4{width:228px}.span3{width:166px}.span2{width:104px}.span1{width:42px}.offset12{margin-left:764px}.offset11{margin-left:702px}.offset10{margin-left:640px}.offset9{margin-left:578px}.offset8{margin-left:516px}.offset7{margin-left:454px}.offset6{margin-left:392px}.offset5{margin-left:330px}.offset4{margin-left:268px}.offset3{margin-left:206px}.offset2{margin-left:144px}.offset1{margin-left:82px}.row-fluid{width:100%;*zoom:1}.row-fluid:before,.row-fluid:after{display:table;line-height:0;content:""}.row-fluid:after{clear:both}.row-fluid [class*="span"]{display:block;float:left;width:100%;min-height:30px;margin-left:2.7624309392265194%;*margin-left:2.709239449864817%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.row-fluid [class*="span"]:first-child{margin-left:0}.row-fluid .controls-row [class*="span"]+[class*="span"]{margin-left:2.7624309392265194%}.row-fluid .span12{width:100%;*width:99.94680851063829%}.row-fluid .span11{width:91.43646408839778%;*width:91.38327259903608%}.row-fluid .span10{width:82.87292817679558%;*width:82.81973668743387%}.row-fluid .span9{width:74.30939226519337%;*width:74.25620077583166%}.row-fluid .span8{width:65.74585635359117%;*width:65.69266486422946%}.row-fluid .span7{width:57.18232044198895%;*width:57.12912895262725%}.row-fluid .span6{width:48.61878453038674%;*width:48.56559304102504%}.row-fluid .span5{width:40.05524861878453%;*width:40.00205712942283%}.row-fluid .span4{width:31.491712707182323%;*width:31.43852121782062%}.row-fluid .span3{width:22.92817679558011%;*width:22.87498530621841%}.row-fluid .span2{width:14.3646408839779%;*width:14.311449394616199%}.row-fluid .span1{width:5.801104972375691%;*width:5.747913483013988%}.row-fluid .offset12{margin-left:105.52486187845304%;*margin-left:105.41847889972962%}.row-fluid .offset12:first-child{margin-left:102.76243093922652%;*margin-left:102.6560479605031%}.row-fluid .offset11{margin-left:96.96132596685082%;*margin-left:96.8549429881274%}.row-fluid .offset11:first-child{margin-left:94.1988950276243%;*margin-left:94.09251204890089%}.row-fluid .offset10{margin-left:88.39779005524862%;*margin-left:88.2914070765252%}.row-fluid .offset10:first-child{margin-left:85.6353591160221%;*margin-left:85.52897613729868%}.row-fluid .offset9{margin-left:79.8342541436464%;*margin-left:79.72787116492299%}.row-fluid .offset9:first-child{margin-left:77.07182320441989%;*margin-left:76.96544022569647%}.row-fluid .offset8{margin-left:71.2707182320442%;*margin-left:71.16433525332079%}.row-fluid .offset8:first-child{margin-left:68.50828729281768%;*margin-left:68.40190431409427%}.row-fluid .offset7{margin-left:62.70718232044199%;*margin-left:62.600799341718584%}.row-fluid .offset7:first-child{margin-left:59.94475138121547%;*margin-left:59.838368402492065%}.row-fluid .offset6{margin-left:54.14364640883978%;*margin-left:54.037263430116376%}.row-fluid .offset6:first-child{margin-left:51.38121546961326%;*margin-left:51.27483249088986%}.row-fluid .offset5{margin-left:45.58011049723757%;*margin-left:45.47372751851417%}.row-fluid .offset5:first-child{margin-left:42.81767955801105%;*margin-left:42.71129657928765%}.row-fluid .offset4{margin-left:37.01657458563536%;*margin-left:36.91019160691196%}.row-fluid .offset4:first-child{margin-left:34.25414364640884%;*margin-left:34.14776066768544%}.row-fluid .offset3{margin-left:28.45303867403315%;*margin-left:28.346655695309746%}.row-fluid .offset3:first-child{margin-left:25.69060773480663%;*margin-left:25.584224756083227%}.row-fluid .offset2{margin-left:19.88950276243094%;*margin-left:19.783119783707537%}.row-fluid .offset2:first-child{margin-left:17.12707182320442%;*margin-left:17.02068884448102%}.row-fluid .offset1{margin-left:11.32596685082873%;*margin-left:11.219583872105325%}.row-fluid .offset1:first-child{margin-left:8.56353591160221%;*margin-left:8.457152932878806%}input,textarea,.uneditable-input{margin-left:0}.controls-row [class*="span"]+[class*="span"]{margin-left:20px}input.span12,textarea.span12,.uneditable-input.span12{width:710px}input.span11,textarea.span11,.uneditable-input.span11{width:648px}input.span10,textarea.span10,.uneditable-input.span10{width:586px}input.span9,textarea.span9,.uneditable-input.span9{width:524px}input.span8,textarea.span8,.uneditable-input.span8{width:462px}input.span7,textarea.span7,.uneditable-input.span7{width:400px}input.span6,textarea.span6,.uneditable-input.span6{width:338px}input.span5,textarea.span5,.uneditable-input.span5{width:276px}input.span4,textarea.span4,.uneditable-input.span4{width:214px}input.span3,textarea.span3,.uneditable-input.span3{width:152px}input.span2,textarea.span2,.uneditable-input.span2{width:90px}input.span1,textarea.span1,.uneditable-input.span1{width:28px}}@media(max-width:767px){body{padding-right:20px;padding-left:20px}.navbar-fixed-top,.navbar-fixed-bottom,.navbar-static-top{margin-right:-20px;margin-left:-20px}.container-fluid{padding:0}.dl-horizontal dt{float:none;width:auto;clear:none;text-align:left}.dl-horizontal dd{margin-left:0}.container{width:auto}.row-fluid{width:100%}.row,.thumbnails{margin-left:0}.thumbnails>li{float:none;margin-left:0}[class*="span"],.uneditable-input[class*="span"],.row-fluid [class*="span"]{display:block;float:none;width:100%;margin-left:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.span12,.row-fluid .span12{width:100%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.row-fluid [class*="offset"]:first-child{margin-left:0}.input-large,.input-xlarge,.input-xxlarge,input[class*="span"],select[class*="span"],textarea[class*="span"],.uneditable-input{display:block;width:100%;min-height:30px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.input-prepend input,.input-append input,.input-prepend input[class*="span"],.input-append input[class*="span"]{display:inline-block;width:auto}.controls-row [class*="span"]+[class*="span"]{margin-left:0}.modal{position:fixed;top:20px;right:20px;left:20px;width:auto;margin:0}.modal.fade{top:-100px}.modal.fade.in{top:20px}}@media(max-width:480px){.nav-collapse{-webkit-transform:translate3d(0,0,0)}.page-header h1 small{display:block;line-height:20px}input[type="checkbox"],input[type="radio"]{border:1px solid #ccc}.form-horizontal .control-label{float:none;width:auto;padding-top:0;text-align:left}.form-horizontal .controls{margin-left:0}.form-horizontal .control-list{padding-top:0}.form-horizontal .form-actions{padding-right:10px;padding-left:10px}.media .pull-left,.media .pull-right{display:block;float:none;margin-bottom:10px}.media-object{margin-right:0;margin-left:0}.modal{top:10px;right:10px;left:10px}.modal-header .close{padding:10px;margin:-10px}.carousel-caption{position:static}}@media(max-width:979px){body{padding-top:0}.navbar-fixed-top,.navbar-fixed-bottom{position:static}.navbar-fixed-top{margin-bottom:20px}.navbar-fixed-bottom{margin-top:20px}.navbar-fixed-top .navbar-inner,.navbar-fixed-bottom .navbar-inner{padding:5px}.navbar .container{width:auto;padding:0}.navbar .brand{padding-right:10px;padding-left:10px;margin:0 0 0 -5px}.nav-collapse{clear:both}.nav-collapse .nav{float:none;margin:0 0 10px}.nav-collapse .nav>li{float:none}.nav-collapse .nav>li>a{margin-bottom:2px}.nav-collapse .nav>.divider-vertical{display:none}.nav-collapse .nav .nav-header{color:#777;text-shadow:none}.nav-collapse .nav>li>a,.nav-collapse .dropdown-menu a{padding:9px 15px;font-weight:bold;color:#777;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}.nav-collapse .btn{padding:4px 10px 4px;font-weight:normal;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.nav-collapse .dropdown-menu li+li a{margin-bottom:2px}.nav-collapse .nav>li>a:hover,.nav-collapse .nav>li>a:focus,.nav-collapse .dropdown-menu a:hover,.nav-collapse .dropdown-menu a:focus{background-color:#f2f2f2}.navbar-inverse .nav-collapse .nav>li>a,.navbar-inverse .nav-collapse .dropdown-menu a{color:#999}.navbar-inverse .nav-collapse .nav>li>a:hover,.navbar-inverse .nav-collapse .nav>li>a:focus,.navbar-inverse .nav-collapse .dropdown-menu a:hover,.navbar-inverse .nav-collapse .dropdown-menu a:focus{background-color:#111}.nav-collapse.in .btn-group{padding:0;margin-top:5px}.nav-collapse .dropdown-menu{position:static;top:auto;left:auto;display:none;float:none;max-width:none;padding:0;margin:0 15px;background-color:transparent;border:0;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none}.nav-collapse .open>.dropdown-menu{display:block}.nav-collapse .dropdown-menu:before,.nav-collapse .dropdown-menu:after{display:none}.nav-collapse .dropdown-menu .divider{display:none}.nav-collapse .nav>li>.dropdown-menu:before,.nav-collapse .nav>li>.dropdown-menu:after{display:none}.nav-collapse .navbar-form,.nav-collapse .navbar-search{float:none;padding:10px 15px;margin:10px 0;border-top:1px solid #f2f2f2;border-bottom:1px solid #f2f2f2;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1);-moz-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1);box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1)}.navbar-inverse .nav-collapse .navbar-form,.navbar-inverse .nav-collapse .navbar-search{border-top-color:#111;border-bottom-color:#111}.navbar .nav-collapse .nav.pull-right{float:none;margin-left:0}.nav-collapse,.nav-collapse.collapse{height:0;overflow:hidden}.navbar .btn-navbar{display:block}.navbar-static .navbar-inner{padding-right:10px;padding-left:10px}}@media(min-width:980px){.nav-collapse.collapse{height:auto!important;overflow:visible!important}} diff --git a/doc/themes/scikit-learn/static/css/bootstrap.css b/doc/themes/scikit-learn/static/css/bootstrap.css deleted file mode 100644 index 2cdd4ac5f938b..0000000000000 --- a/doc/themes/scikit-learn/static/css/bootstrap.css +++ /dev/null @@ -1,6320 +0,0 @@ -/*! - * Bootstrap v2.3.2 - * - * Copyright 2012 Twitter, Inc - * Licensed under the Apache License v2.0 - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Designed and built with all the love in the world @twitter by @mdo and @fat. - */ -.clearfix { - *zoom: 1; -} -.clearfix:before, -.clearfix:after { - display: table; - content: ""; - line-height: 0; -} -.clearfix:after { - clear: both; -} -.hide-text { - font: 0/0 a; - color: transparent; - text-shadow: none; - background-color: transparent; - border: 0; -} -.input-block-level { - display: block; - width: 100%; - min-height: 30px; - -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - box-sizing: border-box; -} -article, -aside, -details, -figcaption, -figure, -footer, -header, -hgroup, -nav, -section { - display: block; -} -audio, -canvas, -video { - display: inline-block; - *display: inline; - *zoom: 1; -} -audio:not([controls]) { - display: none; -} -html { - font-size: 100%; - -webkit-text-size-adjust: 100%; - -ms-text-size-adjust: 100%; -} -a:focus { - outline: thin dotted #333; - outline: 5px auto -webkit-focus-ring-color; - outline-offset: -2px; -} -a:hover, -a:active { - outline: 0; -} -sub, -sup { - position: relative; - font-size: 75%; - line-height: 0; - vertical-align: baseline; -} -sup { - top: -0.5em; -} -sub { - bottom: -0.25em; -} -img { - /* Responsive images (ensure images don't scale beyond their parents) */ - - max-width: 100%; - /* Part 1: Set a maximum relative to the parent */ - - width: auto\9; - /* IE7-8 need help adjusting responsive images */ - - height: auto; - /* Part 2: Scale the height according to the width, otherwise you get stretching */ - - vertical-align: middle; - border: 0; - -ms-interpolation-mode: bicubic; -} -#map_canvas img, -.google-maps img { - max-width: none; -} -button, -input, -select, -textarea { - margin: 0; - font-size: 100%; - vertical-align: middle; -} -button, -input { - *overflow: visible; - line-height: normal; -} -button::-moz-focus-inner, -input::-moz-focus-inner { - padding: 0; - border: 0; -} -button, -html input[type="button"], -input[type="reset"], -input[type="submit"] { - -webkit-appearance: button; - cursor: pointer; -} -label, -select, -button, -input[type="button"], -input[type="reset"], -input[type="submit"], -input[type="radio"], -input[type="checkbox"] { - cursor: pointer; -} -input[type="search"] { - -webkit-box-sizing: content-box; - -moz-box-sizing: content-box; - box-sizing: content-box; - -webkit-appearance: textfield; -} -input[type="search"]::-webkit-search-decoration, -input[type="search"]::-webkit-search-cancel-button { - -webkit-appearance: none; -} -textarea { - overflow: auto; - vertical-align: top; -} -@media print { - * { - text-shadow: none !important; - color: #000 !important; - background: transparent !important; - box-shadow: none !important; - } - a, - a:visited { - text-decoration: underline; - } - a[href]:after { - content: " (" attr(href) ")"; - } - abbr[title]:after { - content: " (" attr(title) ")"; - } - .ir a:after, - a[href^="javascript:"]:after, - a[href^="#"]:after { - content: ""; - } - pre, - blockquote { - border: 1px solid #999; - page-break-inside: avoid; - } - thead { - display: table-header-group; - } - tr, - img { - page-break-inside: avoid; - } - img { - max-width: 100% !important; - } - @page { - margin: 0.5cm; - } - p, - h2, - h3 { - orphans: 3; - widows: 3; - } - h2, - h3 { - page-break-after: avoid; - } -} -body { - margin: 0; - font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; - font-size: 14px; - line-height: 20px; - color: #333333; - background-color: #ffffff; -} -a { - color: #0088cc; - text-decoration: none; -} -a:hover, -a:focus { - color: #005580; - text-decoration: underline; -} -.img-rounded { - -webkit-border-radius: 6px; - -moz-border-radius: 6px; - border-radius: 6px; -} -.img-polaroid { - padding: 4px; - background-color: #fff; - border: 1px solid #ccc; - border: 1px solid rgba(0, 0, 0, 0.2); - -webkit-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); - -moz-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); - box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); -} -.img-circle { - -webkit-border-radius: 500px; - -moz-border-radius: 500px; - border-radius: 500px; -} -.row { - margin-left: -20px; - *zoom: 1; -} -.row:before, -.row:after { - display: table; - content: ""; - line-height: 0; -} -.row:after { - clear: both; -} -[class*="span"] { - float: left; - min-height: 1px; - margin-left: 20px; -} -.container, -.navbar-static-top .container, -.navbar-fixed-top .container, -.navbar-fixed-bottom .container { - width: 940px; -} -.span12 { - width: 940px; -} -.span11 { - width: 860px; -} -.span10 { - width: 780px; -} -.span9 { - width: 700px; -} -.span8 { - width: 620px; -} -.span7 { - width: 540px; -} -.span6 { - width: 460px; -} -.span5 { - width: 380px; -} -.span4 { - width: 300px; -} -.span3 { - width: 220px; -} -.span2 { - width: 140px; -} -.span1 { - width: 60px; -} -.offset12 { - margin-left: 980px; -} -.offset11 { - margin-left: 900px; -} -.offset10 { - margin-left: 820px; -} -.offset9 { - margin-left: 740px; -} -.offset8 { - margin-left: 660px; -} -.offset7 { - margin-left: 580px; -} -.offset6 { - margin-left: 500px; -} -.offset5 { - margin-left: 420px; -} -.offset4 { - margin-left: 340px; -} -.offset3 { - margin-left: 260px; -} -.offset2 { - margin-left: 180px; -} -.offset1 { - margin-left: 100px; -} -.row-fluid { - width: 100%; - *zoom: 1; -} -.row-fluid:before, -.row-fluid:after { - display: table; - content: ""; - line-height: 0; -} -.row-fluid:after { - clear: both; -} -.row-fluid [class*="span"] { - display: block; - width: 100%; - min-height: 30px; - -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - box-sizing: border-box; - float: left; - margin-left: 2.127659574468085%; - *margin-left: 2.074468085106383%; -} -.row-fluid [class*="span"]:first-child { - margin-left: 0; -} -.row-fluid .controls-row [class*="span"] + [class*="span"] { - margin-left: 2.127659574468085%; -} -.row-fluid .span12 { - width: 100%; - *width: 99.94680851063829%; -} -.row-fluid .span11 { - width: 91.48936170212765%; - *width: 91.43617021276594%; -} -.row-fluid .span10 { - width: 82.97872340425532%; - *width: 82.92553191489361%; -} -.row-fluid .span9 { - width: 74.46808510638297%; - *width: 74.41489361702126%; -} -.row-fluid .span8 { - width: 65.95744680851064%; - *width: 65.90425531914893%; -} -.row-fluid .span7 { - width: 57.44680851063829%; - *width: 57.39361702127659%; -} -.row-fluid .span6 { - width: 48.93617021276595%; - *width: 48.88297872340425%; -} -.row-fluid .span5 { - width: 40.42553191489362%; - *width: 40.37234042553192%; -} -.row-fluid .span4 { - width: 31.914893617021278%; - *width: 31.861702127659576%; -} -.row-fluid .span3 { - width: 23.404255319148934%; - *width: 23.351063829787233%; -} -.row-fluid .span2 { - width: 14.893617021276595%; - *width: 14.840425531914894%; -} -.row-fluid .span1 { - width: 6.382978723404255%; - *width: 6.329787234042553%; -} -.row-fluid .offset12 { - margin-left: 104.25531914893617%; - *margin-left: 104.14893617021275%; -} -.row-fluid .offset12:first-child { - margin-left: 102.12765957446808%; - *margin-left: 102.02127659574467%; -} -.row-fluid .offset11 { - margin-left: 95.74468085106382%; - *margin-left: 95.6382978723404%; -} -.row-fluid .offset11:first-child { - margin-left: 93.61702127659574%; - *margin-left: 93.51063829787232%; -} -.row-fluid .offset10 { - margin-left: 87.23404255319149%; - *margin-left: 87.12765957446807%; -} -.row-fluid .offset10:first-child { - margin-left: 85.1063829787234%; - *margin-left: 84.99999999999999%; -} -.row-fluid .offset9 { - margin-left: 78.72340425531914%; - *margin-left: 78.61702127659572%; -} -.row-fluid .offset9:first-child { - margin-left: 76.59574468085106%; - *margin-left: 76.48936170212764%; -} -.row-fluid .offset8 { - margin-left: 70.2127659574468%; - *margin-left: 70.10638297872339%; -} -.row-fluid .offset8:first-child { - margin-left: 68.08510638297872%; - *margin-left: 67.9787234042553%; -} -.row-fluid .offset7 { - margin-left: 61.70212765957446%; - *margin-left: 61.59574468085106%; -} -.row-fluid .offset7:first-child { - margin-left: 59.574468085106375%; - *margin-left: 59.46808510638297%; -} -.row-fluid .offset6 { - margin-left: 53.191489361702125%; - *margin-left: 53.085106382978715%; -} -.row-fluid .offset6:first-child { - margin-left: 51.063829787234035%; - *margin-left: 50.95744680851063%; -} -.row-fluid .offset5 { - margin-left: 44.68085106382979%; - *margin-left: 44.57446808510638%; -} -.row-fluid .offset5:first-child { - margin-left: 42.5531914893617%; - *margin-left: 42.4468085106383%; -} -.row-fluid .offset4 { - margin-left: 36.170212765957444%; - *margin-left: 36.06382978723405%; -} -.row-fluid .offset4:first-child { - margin-left: 34.04255319148936%; - *margin-left: 33.93617021276596%; -} -.row-fluid .offset3 { - margin-left: 27.659574468085104%; - *margin-left: 27.5531914893617%; -} -.row-fluid .offset3:first-child { - margin-left: 25.53191489361702%; - *margin-left: 25.425531914893618%; -} -.row-fluid .offset2 { - margin-left: 19.148936170212764%; - *margin-left: 19.04255319148936%; -} -.row-fluid .offset2:first-child { - margin-left: 17.02127659574468%; - *margin-left: 16.914893617021278%; -} -.row-fluid .offset1 { - margin-left: 10.638297872340425%; - *margin-left: 10.53191489361702%; -} -.row-fluid .offset1:first-child { - margin-left: 8.51063829787234%; - *margin-left: 8.404255319148938%; -} -[class*="span"].hide, -.row-fluid [class*="span"].hide { - display: none; -} -[class*="span"].pull-right, -.row-fluid [class*="span"].pull-right { - float: right; -} -.container { - margin-right: auto; - margin-left: auto; - *zoom: 1; -} -.container:before, -.container:after { - display: table; - content: ""; - line-height: 0; -} -.container:after { - clear: both; -} -.container-fluid { - padding-right: 20px; - padding-left: 20px; - *zoom: 1; -} -.container-fluid:before, -.container-fluid:after { - display: table; - content: ""; - line-height: 0; -} -.container-fluid:after { - clear: both; -} -p { - margin: 0 0 10px; -} -.lead { - margin-bottom: 20px; - font-size: 21px; - font-weight: 200; - line-height: 30px; -} -small { - font-size: 85%; -} -strong { - font-weight: bold; -} -em { - font-style: italic; -} -cite { - font-style: normal; -} -.muted { - color: #999999; -} -a.muted:hover, -a.muted:focus { - color: #808080; -} -.text-warning { - color: #c09853; -} -a.text-warning:hover, -a.text-warning:focus { - color: #a47e3c; -} -.text-error { - color: #b94a48; -} -a.text-error:hover, -a.text-error:focus { - color: #953b39; -} -.text-info { - color: #3a87ad; -} -a.text-info:hover, -a.text-info:focus { - color: #2d6987; -} -.text-success { - color: #468847; -} -a.text-success:hover, -a.text-success:focus { - color: #356635; -} -.text-left { - text-align: left; -} -.text-right { - text-align: right; -} -.text-center { - text-align: center; -} -h1, -h2, -h3, -h4, -h5, -h6 { - margin: 10px 0; - font-family: inherit; - font-weight: bold; - line-height: 20px; - color: inherit; - text-rendering: optimizelegibility; -} -h1 small, -h2 small, -h3 small, -h4 small, -h5 small, -h6 small { - font-weight: normal; - line-height: 1; - color: #999999; -} -h1, -h2, -h3 { - line-height: 40px; -} -h1 { - font-size: 38.5px; -} -h2 { - font-size: 31.5px; -} -h3 { - font-size: 24.5px; -} -h4 { - font-size: 17.5px; -} -h5 { - font-size: 14px; -} -h6 { - font-size: 11.9px; -} -h1 small { - font-size: 24.5px; -} -h2 small { - font-size: 17.5px; -} -h3 small { - font-size: 14px; -} -h4 small { - font-size: 14px; -} -.page-header { - padding-bottom: 9px; - margin: 20px 0 30px; - border-bottom: 1px solid #eeeeee; -} -ul, -ol { - padding: 0; - margin: 0 0 10px 25px; -} -ul ul, -ul ol, -ol ol, -ol ul { - margin-bottom: 0; -} -li { - line-height: 20px; -} -ul.unstyled, -ol.unstyled { - margin-left: 0; - list-style: none; -} -ul.inline, -ol.inline { - margin-left: 0; - list-style: none; -} -ul.inline > li, -ol.inline > li { - display: inline-block; - *display: inline; - /* IE7 inline-block hack */ - - *zoom: 1; - padding-left: 5px; - padding-right: 5px; -} -dl { - margin-bottom: 20px; -} -dt, -dd { - line-height: 20px; -} -dt { - font-weight: bold; -} -dd { - margin-left: 10px; -} -.dl-horizontal { - *zoom: 1; -} -.dl-horizontal:before, -.dl-horizontal:after { - display: table; - content: ""; - line-height: 0; -} -.dl-horizontal:after { - clear: both; -} -.dl-horizontal dt { - float: left; - width: 160px; - clear: left; - text-align: right; - overflow: hidden; - text-overflow: ellipsis; - white-space: nowrap; -} -.dl-horizontal dd { - margin-left: 180px; -} -hr { - margin: 20px 0; - border: 0; - border-top: 1px solid #eeeeee; - border-bottom: 1px solid #ffffff; -} -abbr[title], -abbr[data-original-title] { - cursor: help; - border-bottom: 1px dotted #999999; -} -abbr.initialism { - font-size: 90%; - text-transform: uppercase; -} -blockquote { - padding: 0 0 0 15px; - margin: 0 0 20px; - border-left: 5px solid #eeeeee; -} -blockquote p { - margin-bottom: 0; - font-size: 17.5px; - font-weight: 300; - line-height: 1.25; -} -blockquote small { - display: block; - line-height: 20px; - color: #999999; -} -blockquote small:before { - content: '\2014 \00A0'; -} -blockquote.pull-right { - float: right; - padding-right: 15px; - padding-left: 0; - border-right: 5px solid #eeeeee; - border-left: 0; -} -blockquote.pull-right p, -blockquote.pull-right small { - text-align: right; -} -blockquote.pull-right small:before { - content: ''; -} -blockquote.pull-right small:after { - content: '\00A0 \2014'; -} -q:before, -q:after, -blockquote:before, -blockquote:after { - content: ""; -} -address { - display: block; - margin-bottom: 20px; - font-style: normal; - line-height: 20px; -} -code, -pre { - padding: 0 3px 2px; - font-family: Monaco, Menlo, Consolas, "Courier New", monospace; - font-size: 12px; - color: #333333; - -webkit-border-radius: 3px; - -moz-border-radius: 3px; - border-radius: 3px; -} -code { - padding: 2px 4px; - color: #d14; - background-color: #f7f7f9; - border: 1px solid #e1e1e8; - white-space: nowrap; -} -pre { - display: block; - padding: 9.5px; - margin: 0 0 10px; - font-size: 13px; - line-height: 20px; - word-break: break-all; - word-wrap: break-word; - white-space: pre; - white-space: pre-wrap; - background-color: #f5f5f5; - border: 1px solid #ccc; - border: 1px solid rgba(0, 0, 0, 0.15); - -webkit-border-radius: 4px; - -moz-border-radius: 4px; - border-radius: 4px; -} -pre.prettyprint { - margin-bottom: 20px; -} -pre code { - padding: 0; - color: inherit; - white-space: pre; - white-space: pre-wrap; - background-color: transparent; - border: 0; -} -.pre-scrollable { - max-height: 340px; - overflow-y: scroll; -} -.label, -.badge { - display: inline-block; - padding: 2px 4px; - font-size: 11.844px; - font-weight: bold; - line-height: 14px; - color: #ffffff; - vertical-align: baseline; - white-space: nowrap; - text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25); - background-color: #999999; -} -.label { - -webkit-border-radius: 3px; - -moz-border-radius: 3px; - border-radius: 3px; -} -.badge { - padding-left: 9px; - padding-right: 9px; - -webkit-border-radius: 9px; - -moz-border-radius: 9px; - border-radius: 9px; -} -.label:empty, -.badge:empty { - display: none; -} -a.label:hover, -a.label:focus, -a.badge:hover, -a.badge:focus { - color: #ffffff; - text-decoration: none; - cursor: pointer; -} -.label-important, -.badge-important { - background-color: #b94a48; -} -.label-important[href], -.badge-important[href] { - background-color: #953b39; -} -.label-warning, -.badge-warning { - background-color: #f89406; -} -.label-warning[href], -.badge-warning[href] { - background-color: #c67605; -} -.label-danger, -.badge-danger { - /* XXX: backported from later bootstrap */ - background-color: #d9534f; -} -.label-success, -.badge-success { - background-color: #468847; -} -.label-success[href], -.badge-success[href] { - background-color: #356635; -} -.label-info, -.badge-info { - background-color: #3a87ad; -} -.label-info[href], -.badge-info[href] { - background-color: #2d6987; -} -.label-inverse, -.badge-inverse { - background-color: #333333; -} -.label-inverse[href], -.badge-inverse[href] { - background-color: #1a1a1a; -} -.btn .label, -.btn .badge { - position: relative; - top: -1px; -} -.btn-mini .label, -.btn-mini .badge { - top: 0; -} -table { - max-width: 100%; - background-color: transparent; - border-collapse: collapse; - border-spacing: 0; -} -.table { - width: 100%; - margin-bottom: 20px; -} -.table th, -.table td { - padding: 8px; - line-height: 20px; - text-align: left; - vertical-align: top; - border-top: 1px solid #dddddd; -} -.table th { - font-weight: bold; -} -.table thead th { - vertical-align: bottom; -} -.table caption + thead tr:first-child th, -.table caption + thead tr:first-child td, -.table colgroup + thead tr:first-child th, -.table colgroup + thead tr:first-child td, -.table thead:first-child tr:first-child th, -.table thead:first-child tr:first-child td { - border-top: 0; -} -.table tbody + tbody { - border-top: 2px solid #dddddd; -} -.table .table { - background-color: #ffffff; -} -.table-condensed th, -.table-condensed td { - padding: 4px 5px; -} -.table-bordered { - border: 1px solid #dddddd; - border-collapse: separate; - *border-collapse: collapse; - border-left: 0; - -webkit-border-radius: 4px; - -moz-border-radius: 4px; - border-radius: 4px; -} -.table-bordered th, -.table-bordered td { - border-left: 1px solid #dddddd; -} -.table-bordered caption + thead tr:first-child th, -.table-bordered caption + tbody tr:first-child th, -.table-bordered caption + tbody tr:first-child td, -.table-bordered colgroup + thead tr:first-child th, -.table-bordered colgroup + tbody tr:first-child th, -.table-bordered colgroup + tbody tr:first-child td, -.table-bordered thead:first-child tr:first-child th, -.table-bordered tbody:first-child tr:first-child th, -.table-bordered tbody:first-child tr:first-child td { - border-top: 0; -} -.table-bordered thead:first-child tr:first-child > th:first-child, -.table-bordered tbody:first-child tr:first-child > td:first-child, -.table-bordered tbody:first-child tr:first-child > th:first-child { - -webkit-border-top-left-radius: 4px; - -moz-border-radius-topleft: 4px; - border-top-left-radius: 4px; -} -.table-bordered thead:first-child tr:first-child > th:last-child, -.table-bordered tbody:first-child tr:first-child > td:last-child, -.table-bordered tbody:first-child tr:first-child > th:last-child { - -webkit-border-top-right-radius: 4px; - -moz-border-radius-topright: 4px; - border-top-right-radius: 4px; -} -.table-bordered thead:last-child tr:last-child > th:first-child, -.table-bordered tbody:last-child tr:last-child > td:first-child, -.table-bordered tbody:last-child tr:last-child > th:first-child, -.table-bordered tfoot:last-child tr:last-child > td:first-child, -.table-bordered tfoot:last-child tr:last-child > th:first-child { - -webkit-border-bottom-left-radius: 4px; - -moz-border-radius-bottomleft: 4px; - border-bottom-left-radius: 4px; -} -.table-bordered thead:last-child tr:last-child > th:last-child, -.table-bordered tbody:last-child tr:last-child > td:last-child, -.table-bordered tbody:last-child tr:last-child > th:last-child, -.table-bordered tfoot:last-child tr:last-child > td:last-child, -.table-bordered tfoot:last-child tr:last-child > th:last-child { - -webkit-border-bottom-right-radius: 4px; - -moz-border-radius-bottomright: 4px; - border-bottom-right-radius: 4px; -} -.table-bordered tfoot + tbody:last-child tr:last-child td:first-child { - -webkit-border-bottom-left-radius: 0; - -moz-border-radius-bottomleft: 0; - border-bottom-left-radius: 0; -} -.table-bordered tfoot + tbody:last-child tr:last-child td:last-child { - -webkit-border-bottom-right-radius: 0; - -moz-border-radius-bottomright: 0; - border-bottom-right-radius: 0; -} -.table-bordered caption + thead tr:first-child th:first-child, -.table-bordered caption + tbody tr:first-child td:first-child, -.table-bordered colgroup + thead tr:first-child th:first-child, -.table-bordered colgroup + tbody tr:first-child td:first-child { - -webkit-border-top-left-radius: 4px; - -moz-border-radius-topleft: 4px; - border-top-left-radius: 4px; -} -.table-bordered caption + thead tr:first-child th:last-child, -.table-bordered caption + tbody tr:first-child td:last-child, -.table-bordered colgroup + thead tr:first-child th:last-child, -.table-bordered colgroup + tbody tr:first-child td:last-child { - -webkit-border-top-right-radius: 4px; - -moz-border-radius-topright: 4px; - border-top-right-radius: 4px; -} -.table-striped tbody > tr:nth-child(odd) > td, -.table-striped tbody > tr:nth-child(odd) > th { - background-color: #f9f9f9; -} -.table-hover tbody tr:hover > td, -.table-hover tbody tr:hover > th { - background-color: #f5f5f5; -} -table td[class*="span"], -table th[class*="span"], -.row-fluid table td[class*="span"], -.row-fluid table th[class*="span"] { - display: table-cell; - float: none; - margin-left: 0; -} -.table td.span1, -.table th.span1 { - float: none; - width: 44px; - margin-left: 0; -} -.table td.span2, -.table th.span2 { - float: none; - width: 124px; - margin-left: 0; -} -.table td.span3, -.table th.span3 { - float: none; - width: 204px; - margin-left: 0; -} -.table td.span4, -.table th.span4 { - float: none; - width: 284px; - margin-left: 0; -} -.table td.span5, -.table th.span5 { - float: none; - width: 364px; - margin-left: 0; -} -.table td.span6, -.table th.span6 { - float: none; - width: 444px; - margin-left: 0; -} -.table td.span7, -.table th.span7 { - float: none; - width: 524px; - margin-left: 0; -} -.table td.span8, -.table th.span8 { - float: none; - width: 604px; - margin-left: 0; -} -.table td.span9, -.table th.span9 { - float: none; - width: 684px; - margin-left: 0; -} -.table td.span10, -.table th.span10 { - float: none; - width: 764px; - margin-left: 0; -} -.table td.span11, -.table th.span11 { - float: none; - width: 844px; - margin-left: 0; -} -.table td.span12, -.table th.span12 { - float: none; - width: 924px; - margin-left: 0; -} -.table tbody tr.success > td { - background-color: #dff0d8; -} -.table tbody tr.error > td { - background-color: #f2dede; -} -.table tbody tr.warning > td { - background-color: #fcf8e3; -} -.table tbody tr.info > td { - background-color: #d9edf7; -} -.table-hover tbody tr.success:hover > td { - background-color: #d0e9c6; -} -.table-hover tbody tr.error:hover > td { - background-color: #ebcccc; -} -.table-hover tbody tr.warning:hover > td { - background-color: #faf2cc; -} -.table-hover tbody tr.info:hover > td { - background-color: #c4e3f3; -} -form { - margin: 0 0 20px; -} -fieldset { - padding: 0; - margin: 0; - border: 0; -} -legend { - display: block; - width: 100%; - padding: 0; - margin-bottom: 20px; - font-size: 21px; - line-height: 40px; - color: #333333; - border: 0; - border-bottom: 1px solid #e5e5e5; -} -legend small { - font-size: 15px; - color: #999999; -} -label, -input, -button, -select, -textarea { - font-size: 14px; - font-weight: normal; - line-height: 20px; -} -input, -button, -select, -textarea { - font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; -} -label { - display: block; - margin-bottom: 5px; -} -select, -textarea, -input[type="text"], -input[type="password"], -input[type="datetime"], -input[type="datetime-local"], -input[type="date"], -input[type="month"], -input[type="time"], -input[type="week"], -input[type="number"], -input[type="email"], -input[type="url"], -input[type="search"], -input[type="tel"], -input[type="color"], -.uneditable-input { - display: inline-block; - height: 20px; - padding: 4px 6px; - margin-bottom: 10px; - font-size: 14px; - line-height: 20px; - color: #555555; - -webkit-border-radius: 4px; - -moz-border-radius: 4px; - border-radius: 4px; - vertical-align: middle; -} -input, -textarea, -.uneditable-input { - width: 206px; -} -textarea { - height: auto; -} -textarea, -input[type="text"], -input[type="password"], -input[type="datetime"], -input[type="datetime-local"], -input[type="date"], -input[type="month"], -input[type="time"], -input[type="week"], -input[type="number"], -input[type="email"], -input[type="url"], -input[type="search"], -input[type="tel"], -input[type="color"], -.uneditable-input { - background-color: #ffffff; - border: 1px solid #cccccc; - -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); - -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); - box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); - -webkit-transition: border linear .2s, box-shadow linear .2s; - -moz-transition: border linear .2s, box-shadow linear .2s; - -o-transition: border linear .2s, box-shadow linear .2s; - transition: border linear .2s, box-shadow linear .2s; -} -textarea:focus, -input[type="text"]:focus, -input[type="password"]:focus, -input[type="datetime"]:focus, -input[type="datetime-local"]:focus, -input[type="date"]:focus, -input[type="month"]:focus, -input[type="time"]:focus, -input[type="week"]:focus, -input[type="number"]:focus, -input[type="email"]:focus, -input[type="url"]:focus, -input[type="search"]:focus, -input[type="tel"]:focus, -input[type="color"]:focus, -.uneditable-input:focus { - border-color: rgba(82, 168, 236, 0.8); - outline: 0; - outline: thin dotted \9; - /* IE6-9 */ - - -webkit-box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(82,168,236,.6); - -moz-box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(82,168,236,.6); - box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(82,168,236,.6); -} -input[type="radio"], -input[type="checkbox"] { - margin: 4px 0 0; - *margin-top: 0; - /* IE7 */ - - margin-top: 1px \9; - /* IE8-9 */ - - line-height: normal; -} -input[type="file"], -input[type="image"], -input[type="submit"], -input[type="reset"], -input[type="button"], -input[type="radio"], -input[type="checkbox"] { - width: auto; -} -select, -input[type="file"] { - height: 30px; - /* In IE7, the height of the select element cannot be changed by height, only font-size */ - - *margin-top: 4px; - /* For IE7, add top margin to align select with labels */ - - line-height: 30px; -} -select { - width: 220px; - border: 1px solid #cccccc; - background-color: #ffffff; -} -select[multiple], -select[size] { - height: auto; -} -select:focus, -input[type="file"]:focus, -input[type="radio"]:focus, -input[type="checkbox"]:focus { - outline: thin dotted #333; - outline: 5px auto -webkit-focus-ring-color; - outline-offset: -2px; -} -.uneditable-input, -.uneditable-textarea { - color: #999999; - background-color: #fcfcfc; - border-color: #cccccc; - -webkit-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.025); - -moz-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.025); - box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.025); - cursor: not-allowed; -} -.uneditable-input { - overflow: hidden; - white-space: nowrap; -} -.uneditable-textarea { - width: auto; - height: auto; -} -input:-moz-placeholder, -textarea:-moz-placeholder { - color: #999999; -} -input:-ms-input-placeholder, -textarea:-ms-input-placeholder { - color: #999999; -} -input::-webkit-input-placeholder, -textarea::-webkit-input-placeholder { - color: #999999; -} -.radio, -.checkbox { - min-height: 20px; - padding-left: 20px; -} -.radio input[type="radio"], -.checkbox input[type="checkbox"] { - float: left; - margin-left: -20px; -} -.controls > .radio:first-child, -.controls > .checkbox:first-child { - padding-top: 5px; -} -.radio.inline, -.checkbox.inline { - display: inline-block; - padding-top: 5px; - margin-bottom: 0; - vertical-align: middle; -} -.radio.inline + .radio.inline, -.checkbox.inline + .checkbox.inline { - margin-left: 10px; -} -.input-mini { - width: 60px; -} -.input-small { - width: 90px; -} -.input-medium { - width: 150px; -} -.input-large { - width: 210px; -} -.input-xlarge { - width: 270px; -} -.input-xxlarge { - width: 530px; -} -input[class*="span"], -select[class*="span"], -textarea[class*="span"], -.uneditable-input[class*="span"], -.row-fluid input[class*="span"], -.row-fluid select[class*="span"], -.row-fluid textarea[class*="span"], -.row-fluid .uneditable-input[class*="span"] { - float: none; - margin-left: 0; -} -.input-append input[class*="span"], -.input-append .uneditable-input[class*="span"], -.input-prepend input[class*="span"], -.input-prepend .uneditable-input[class*="span"], -.row-fluid input[class*="span"], -.row-fluid select[class*="span"], -.row-fluid textarea[class*="span"], -.row-fluid .uneditable-input[class*="span"], -.row-fluid .input-prepend [class*="span"], -.row-fluid .input-append [class*="span"] { - display: inline-block; -} -input, -textarea, -.uneditable-input { - margin-left: 0; -} -.controls-row [class*="span"] + [class*="span"] { - margin-left: 20px; -} -input.span12, -textarea.span12, -.uneditable-input.span12 { - width: 926px; -} -input.span11, -textarea.span11, -.uneditable-input.span11 { - width: 846px; -} -input.span10, -textarea.span10, -.uneditable-input.span10 { - width: 766px; -} -input.span9, -textarea.span9, -.uneditable-input.span9 { - width: 686px; -} -input.span8, -textarea.span8, -.uneditable-input.span8 { - width: 606px; -} -input.span7, -textarea.span7, -.uneditable-input.span7 { - width: 526px; -} -input.span6, -textarea.span6, -.uneditable-input.span6 { - width: 446px; -} -input.span5, -textarea.span5, -.uneditable-input.span5 { - width: 366px; -} -input.span4, -textarea.span4, -.uneditable-input.span4 { - width: 286px; -} -input.span3, -textarea.span3, -.uneditable-input.span3 { - width: 206px; -} -input.span2, -textarea.span2, -.uneditable-input.span2 { - width: 126px; -} -input.span1, -textarea.span1, -.uneditable-input.span1 { - width: 46px; -} -.controls-row { - *zoom: 1; -} -.controls-row:before, -.controls-row:after { - display: table; - content: ""; - line-height: 0; -} -.controls-row:after { - clear: both; -} -.controls-row [class*="span"], -.row-fluid .controls-row [class*="span"] { - float: left; -} -.controls-row .checkbox[class*="span"], -.controls-row .radio[class*="span"] { - padding-top: 5px; -} -input[disabled], -select[disabled], -textarea[disabled], -input[readonly], -select[readonly], -textarea[readonly] { - cursor: not-allowed; - background-color: #eeeeee; -} -input[type="radio"][disabled], -input[type="checkbox"][disabled], -input[type="radio"][readonly], -input[type="checkbox"][readonly] { - background-color: transparent; -} -.control-group.warning .control-label, -.control-group.warning .help-block, -.control-group.warning .help-inline { - color: #c09853; -} -.control-group.warning .checkbox, -.control-group.warning .radio, -.control-group.warning input, -.control-group.warning select, -.control-group.warning textarea { - color: #c09853; -} -.control-group.warning input, -.control-group.warning select, -.control-group.warning textarea { - border-color: #c09853; - -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); - -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); - box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); -} -.control-group.warning input:focus, -.control-group.warning select:focus, -.control-group.warning textarea:focus { - border-color: #a47e3c; - -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #dbc59e; - -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #dbc59e; - box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #dbc59e; -} -.control-group.warning .input-prepend .add-on, -.control-group.warning .input-append .add-on { - color: #c09853; - background-color: #fcf8e3; - border-color: #c09853; -} -.control-group.error .control-label, -.control-group.error .help-block, -.control-group.error .help-inline { - color: #b94a48; -} -.control-group.error .checkbox, -.control-group.error .radio, -.control-group.error input, -.control-group.error select, -.control-group.error textarea { - color: #b94a48; -} -.control-group.error input, -.control-group.error select, -.control-group.error textarea { - border-color: #b94a48; - -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); - -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); - box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); -} -.control-group.error input:focus, -.control-group.error select:focus, -.control-group.error textarea:focus { - border-color: #953b39; - -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #d59392; - -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #d59392; - box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #d59392; -} -.control-group.error .input-prepend .add-on, -.control-group.error .input-append .add-on { - color: #b94a48; - background-color: #f2dede; - border-color: #b94a48; -} -.control-group.success .control-label, -.control-group.success .help-block, -.control-group.success .help-inline { - color: #468847; -} -.control-group.success .checkbox, -.control-group.success .radio, -.control-group.success input, -.control-group.success select, -.control-group.success textarea { - color: #468847; -} -.control-group.success input, -.control-group.success select, -.control-group.success textarea { - border-color: #468847; - -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); - -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); - box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); -} -.control-group.success input:focus, -.control-group.success select:focus, -.control-group.success textarea:focus { - border-color: #356635; - -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7aba7b; - -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7aba7b; - box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7aba7b; -} -.control-group.success .input-prepend .add-on, -.control-group.success .input-append .add-on { - color: #468847; - background-color: #dff0d8; - border-color: #468847; -} -.control-group.info .control-label, -.control-group.info .help-block, -.control-group.info .help-inline { - color: #3a87ad; -} -.control-group.info .checkbox, -.control-group.info .radio, -.control-group.info input, -.control-group.info select, -.control-group.info textarea { - color: #3a87ad; -} -.control-group.info input, -.control-group.info select, -.control-group.info textarea { - border-color: #3a87ad; - -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); - -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); - box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); -} -.control-group.info input:focus, -.control-group.info select:focus, -.control-group.info textarea:focus { - border-color: #2d6987; - -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7ab5d3; - -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7ab5d3; - box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7ab5d3; -} -.control-group.info .input-prepend .add-on, -.control-group.info .input-append .add-on { - color: #3a87ad; - background-color: #d9edf7; - border-color: #3a87ad; -} -input:focus:invalid, -textarea:focus:invalid, -select:focus:invalid { - color: #b94a48; - border-color: #ee5f5b; -} -input:focus:invalid:focus, -textarea:focus:invalid:focus, -select:focus:invalid:focus { - border-color: #e9322d; - -webkit-box-shadow: 0 0 6px #f8b9b7; - -moz-box-shadow: 0 0 6px #f8b9b7; - box-shadow: 0 0 6px #f8b9b7; -} -.form-actions { - padding: 19px 20px 20px; - margin-top: 20px; - margin-bottom: 20px; - background-color: #f5f5f5; - border-top: 1px solid #e5e5e5; - *zoom: 1; -} -.form-actions:before, -.form-actions:after { - display: table; - content: ""; - line-height: 0; -} -.form-actions:after { - clear: both; -} -.help-block, -.help-inline { - color: #595959; -} -.help-block { - display: block; - margin-bottom: 10px; -} -.help-inline { - display: inline-block; - *display: inline; - /* IE7 inline-block hack */ - - *zoom: 1; - vertical-align: middle; - padding-left: 5px; -} -.input-append, -.input-prepend { - display: inline-block; - margin-bottom: 10px; - vertical-align: middle; - font-size: 0; - white-space: nowrap; -} -.input-append input, -.input-prepend input, -.input-append select, -.input-prepend select, -.input-append .uneditable-input, -.input-prepend .uneditable-input, -.input-append .dropdown-menu, -.input-prepend .dropdown-menu, -.input-append .popover, -.input-prepend .popover { - font-size: 14px; -} -.input-append input, -.input-prepend input, -.input-append select, -.input-prepend select, -.input-append .uneditable-input, -.input-prepend .uneditable-input { - position: relative; - margin-bottom: 0; - *margin-left: 0; - vertical-align: top; - -webkit-border-radius: 0 4px 4px 0; - -moz-border-radius: 0 4px 4px 0; - border-radius: 0 4px 4px 0; -} -.input-append input:focus, -.input-prepend input:focus, -.input-append select:focus, -.input-prepend select:focus, -.input-append .uneditable-input:focus, -.input-prepend .uneditable-input:focus { - z-index: 2; -} -.input-append .add-on, -.input-prepend .add-on { - display: inline-block; - width: auto; - height: 20px; - min-width: 16px; - padding: 4px 5px; - font-size: 14px; - font-weight: normal; - line-height: 20px; - text-align: center; - text-shadow: 0 1px 0 #ffffff; - background-color: #eeeeee; - border: 1px solid #ccc; -} -.input-append .add-on, -.input-prepend .add-on, -.input-append .btn, -.input-prepend .btn, -.input-append .btn-group > .dropdown-toggle, -.input-prepend .btn-group > .dropdown-toggle { - vertical-align: top; - -webkit-border-radius: 0; - -moz-border-radius: 0; - border-radius: 0; -} -.input-append .active, -.input-prepend .active { - background-color: #a9dba9; - border-color: #46a546; -} -.input-prepend .add-on, -.input-prepend .btn { - margin-right: -1px; -} -.input-prepend .add-on:first-child, -.input-prepend .btn:first-child { - -webkit-border-radius: 4px 0 0 4px; - -moz-border-radius: 4px 0 0 4px; - border-radius: 4px 0 0 4px; -} -.input-append input, -.input-append select, -.input-append .uneditable-input { - -webkit-border-radius: 4px 0 0 4px; - -moz-border-radius: 4px 0 0 4px; - border-radius: 4px 0 0 4px; -} -.input-append input + .btn-group .btn:last-child, -.input-append select + .btn-group .btn:last-child, -.input-append .uneditable-input + .btn-group .btn:last-child { - -webkit-border-radius: 0 4px 4px 0; - -moz-border-radius: 0 4px 4px 0; - border-radius: 0 4px 4px 0; -} -.input-append .add-on, -.input-append .btn, -.input-append .btn-group { - margin-left: -1px; -} -.input-append .add-on:last-child, -.input-append .btn:last-child, -.input-append .btn-group:last-child > .dropdown-toggle { - -webkit-border-radius: 0 4px 4px 0; - -moz-border-radius: 0 4px 4px 0; - border-radius: 0 4px 4px 0; -} -.input-prepend.input-append input, -.input-prepend.input-append select, -.input-prepend.input-append .uneditable-input { - -webkit-border-radius: 0; - -moz-border-radius: 0; - border-radius: 0; -} -.input-prepend.input-append input + .btn-group .btn, -.input-prepend.input-append select + .btn-group .btn, -.input-prepend.input-append .uneditable-input + .btn-group .btn { - -webkit-border-radius: 0 4px 4px 0; - -moz-border-radius: 0 4px 4px 0; - border-radius: 0 4px 4px 0; -} -.input-prepend.input-append .add-on:first-child, -.input-prepend.input-append .btn:first-child { - margin-right: -1px; - -webkit-border-radius: 4px 0 0 4px; - -moz-border-radius: 4px 0 0 4px; - border-radius: 4px 0 0 4px; -} -.input-prepend.input-append .add-on:last-child, -.input-prepend.input-append .btn:last-child { - margin-left: -1px; - -webkit-border-radius: 0 4px 4px 0; - -moz-border-radius: 0 4px 4px 0; - border-radius: 0 4px 4px 0; -} -.input-prepend.input-append .btn-group:first-child { - margin-left: 0; -} -input.search-query { - padding-right: 14px; - padding-right: 4px \9; - padding-left: 14px; - padding-left: 4px \9; - /* IE7-8 doesn't have border-radius, so don't indent the padding */ - - margin-bottom: 0; - -webkit-border-radius: 15px; - -moz-border-radius: 15px; - border-radius: 15px; -} -/* Allow for input prepend/append in search forms */ -.form-search .input-append .search-query, -.form-search .input-prepend .search-query { - -webkit-border-radius: 0; - -moz-border-radius: 0; - border-radius: 0; -} -.form-search .input-append .search-query { - -webkit-border-radius: 14px 0 0 14px; - -moz-border-radius: 14px 0 0 14px; - border-radius: 14px 0 0 14px; -} -.form-search .input-append .btn { - -webkit-border-radius: 0 14px 14px 0; - -moz-border-radius: 0 14px 14px 0; - border-radius: 0 14px 14px 0; -} -.form-search .input-prepend .search-query { - -webkit-border-radius: 0 14px 14px 0; - -moz-border-radius: 0 14px 14px 0; - border-radius: 0 14px 14px 0; -} -.form-search .input-prepend .btn { - -webkit-border-radius: 14px 0 0 14px; - -moz-border-radius: 14px 0 0 14px; - border-radius: 14px 0 0 14px; -} -.form-search input, -.form-inline input, -.form-horizontal input, -.form-search textarea, -.form-inline textarea, -.form-horizontal textarea, -.form-search select, -.form-inline select, -.form-horizontal select, -.form-search .help-inline, -.form-inline .help-inline, -.form-horizontal .help-inline, -.form-search .uneditable-input, -.form-inline .uneditable-input, -.form-horizontal .uneditable-input, -.form-search .input-prepend, -.form-inline .input-prepend, -.form-horizontal .input-prepend, -.form-search .input-append, -.form-inline .input-append, -.form-horizontal .input-append { - display: inline-block; - *display: inline; - /* IE7 inline-block hack */ - - *zoom: 1; - margin-bottom: 0; - vertical-align: middle; -} -.form-search .hide, -.form-inline .hide, -.form-horizontal .hide { - display: none; -} -.form-search label, -.form-inline label, -.form-search .btn-group, -.form-inline .btn-group { - display: inline-block; -} -.form-search .input-append, -.form-inline .input-append, -.form-search .input-prepend, -.form-inline .input-prepend { - margin-bottom: 0; -} -.form-search .radio, -.form-search .checkbox, -.form-inline .radio, -.form-inline .checkbox { - padding-left: 0; - margin-bottom: 0; - vertical-align: middle; -} -.form-search .radio input[type="radio"], -.form-search .checkbox input[type="checkbox"], -.form-inline .radio input[type="radio"], -.form-inline .checkbox input[type="checkbox"] { - float: left; - margin-right: 3px; - margin-left: 0; -} -.control-group { - margin-bottom: 10px; -} -legend + .control-group { - margin-top: 20px; - -webkit-margin-top-collapse: separate; -} -.form-horizontal .control-group { - margin-bottom: 20px; - *zoom: 1; -} -.form-horizontal .control-group:before, -.form-horizontal .control-group:after { - display: table; - content: ""; - line-height: 0; -} -.form-horizontal .control-group:after { - clear: both; -} -.form-horizontal .control-label { - float: left; - width: 160px; - padding-top: 5px; - text-align: right; -} -.form-horizontal .controls { - *display: inline-block; - *padding-left: 20px; - margin-left: 180px; - *margin-left: 0; -} -.form-horizontal .controls:first-child { - *padding-left: 180px; -} -.form-horizontal .help-block { - margin-bottom: 0; -} -.form-horizontal input + .help-block, -.form-horizontal select + .help-block, -.form-horizontal textarea + .help-block, -.form-horizontal .uneditable-input + .help-block, -.form-horizontal .input-prepend + .help-block, -.form-horizontal .input-append + .help-block { - margin-top: 10px; -} -.form-horizontal .form-actions { - padding-left: 180px; -} -.btn { - display: inline-block; - *display: inline; - /* IE7 inline-block hack */ - - *zoom: 1; - padding: 4px 12px; - margin-bottom: 0; - font-size: 14px; - line-height: 20px; - text-align: center; - vertical-align: middle; - cursor: pointer; - color: #333333; - text-shadow: 0 1px 1px rgba(255, 255, 255, 0.75); - background-color: #f5f5f5; - background-image: -moz-linear-gradient(top, #ffffff, #e6e6e6); - background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ffffff), to(#e6e6e6)); - background-image: -webkit-linear-gradient(top, #ffffff, #e6e6e6); - background-image: -o-linear-gradient(top, #ffffff, #e6e6e6); - background-image: linear-gradient(to bottom, #ffffff, #e6e6e6); - background-repeat: repeat-x; - filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffffff', endColorstr='#ffe6e6e6', GradientType=0); - border-color: #e6e6e6 #e6e6e6 #bfbfbf; - border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); - *background-color: #e6e6e6; - /* Darken IE7 buttons by default so they stand out more given they won't have borders */ - - filter: progid:DXImageTransform.Microsoft.gradient(enabled = false); - border: 1px solid #cccccc; - *border: 0; - border-bottom-color: #b3b3b3; - -webkit-border-radius: 4px; - -moz-border-radius: 4px; - border-radius: 4px; - *margin-left: .3em; - -webkit-box-shadow: inset 0 1px 0 rgba(255,255,255,.2), 0 1px 2px rgba(0,0,0,.05); - -moz-box-shadow: inset 0 1px 0 rgba(255,255,255,.2), 0 1px 2px rgba(0,0,0,.05); - box-shadow: inset 0 1px 0 rgba(255,255,255,.2), 0 1px 2px rgba(0,0,0,.05); -} -.btn:hover, -.btn:focus, -.btn:active, -.btn.active, -.btn.disabled, -.btn[disabled] { - color: #333333; - background-color: #e6e6e6; - *background-color: #d9d9d9; -} -.btn:active, -.btn.active { - background-color: #cccccc \9; -} -.btn:first-child { - *margin-left: 0; -} -.btn:hover, -.btn:focus { - color: #333333; - text-decoration: none; - background-position: 0 -15px; - -webkit-transition: background-position 0.1s linear; - -moz-transition: background-position 0.1s linear; - -o-transition: background-position 0.1s linear; - transition: background-position 0.1s linear; -} -.btn:focus { - outline: thin dotted #333; - outline: 5px auto -webkit-focus-ring-color; - outline-offset: -2px; -} -.btn.active, -.btn:active { - background-image: none; - outline: 0; - -webkit-box-shadow: inset 0 2px 4px rgba(0,0,0,.15), 0 1px 2px rgba(0,0,0,.05); - -moz-box-shadow: inset 0 2px 4px rgba(0,0,0,.15), 0 1px 2px rgba(0,0,0,.05); - box-shadow: inset 0 2px 4px rgba(0,0,0,.15), 0 1px 2px rgba(0,0,0,.05); -} -.btn.disabled, -.btn[disabled] { - cursor: default; - background-image: none; - opacity: 0.65; - filter: alpha(opacity=65); - -webkit-box-shadow: none; - -moz-box-shadow: none; - box-shadow: none; -} -.btn-large { - padding: 11px 19px; - font-size: 17.5px; - -webkit-border-radius: 6px; - -moz-border-radius: 6px; - border-radius: 6px; -} -.btn-large [class^="icon-"], -.btn-large [class*=" icon-"] { - margin-top: 4px; -} -.btn-small { - padding: 2px 10px; - font-size: 11.9px; - -webkit-border-radius: 3px; - -moz-border-radius: 3px; - border-radius: 3px; -} -.btn-small [class^="icon-"], -.btn-small [class*=" icon-"] { - margin-top: 0; -} -.btn-mini [class^="icon-"], -.btn-mini [class*=" icon-"] { - margin-top: -1px; -} -.btn-mini { - padding: 0 6px; - font-size: 10.5px; - -webkit-border-radius: 3px; - -moz-border-radius: 3px; - border-radius: 3px; -} -.btn-block { - display: block; - width: 100%; - padding-left: 0; - padding-right: 0; - -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - box-sizing: border-box; -} -.btn-block + .btn-block { - margin-top: 5px; -} -input[type="submit"].btn-block, -input[type="reset"].btn-block, -input[type="button"].btn-block { - width: 100%; -} -.btn-primary.active, -.btn-warning.active, -.btn-danger.active, -.btn-success.active, -.btn-info.active, -.btn-inverse.active { - color: rgba(255, 255, 255, 0.75); -} -.btn-primary { - color: #ffffff; - text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25); - background-color: #006dcc; - background-image: -moz-linear-gradient(top, #0088cc, #0044cc); - background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#0088cc), to(#0044cc)); - background-image: -webkit-linear-gradient(top, #0088cc, #0044cc); - background-image: -o-linear-gradient(top, #0088cc, #0044cc); - background-image: linear-gradient(to bottom, #0088cc, #0044cc); - background-repeat: repeat-x; - filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff0088cc', endColorstr='#ff0044cc', GradientType=0); - border-color: #0044cc #0044cc #002a80; - border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); - *background-color: #0044cc; - /* Darken IE7 buttons by default so they stand out more given they won't have borders */ - - filter: progid:DXImageTransform.Microsoft.gradient(enabled = false); -} -.btn-primary:hover, -.btn-primary:focus, -.btn-primary:active, -.btn-primary.active, -.btn-primary.disabled, -.btn-primary[disabled] { - color: #ffffff; - background-color: #0044cc; - *background-color: #003bb3; -} -.btn-primary:active, -.btn-primary.active { - background-color: #003399 \9; -} -.btn-warning { - color: #ffffff; - text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25); - background-color: #faa732; - background-image: -moz-linear-gradient(top, #fbb450, #f89406); - background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#fbb450), to(#f89406)); - background-image: -webkit-linear-gradient(top, #fbb450, #f89406); - background-image: -o-linear-gradient(top, #fbb450, #f89406); - background-image: linear-gradient(to bottom, #fbb450, #f89406); - background-repeat: repeat-x; - filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#fffbb450', endColorstr='#fff89406', GradientType=0); - border-color: #f89406 #f89406 #ad6704; - border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); - *background-color: #f89406; - /* Darken IE7 buttons by default so they stand out more given they won't have borders */ - - filter: progid:DXImageTransform.Microsoft.gradient(enabled = false); -} -.btn-warning:hover, -.btn-warning:focus, -.btn-warning:active, -.btn-warning.active, -.btn-warning.disabled, -.btn-warning[disabled] { - color: #ffffff; - background-color: #f89406; - *background-color: #df8505; -} -.btn-warning:active, -.btn-warning.active { - background-color: #c67605 \9; -} -.btn-danger { - color: #ffffff; - text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25); - background-color: #da4f49; - background-image: -moz-linear-gradient(top, #ee5f5b, #bd362f); - background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ee5f5b), to(#bd362f)); - background-image: -webkit-linear-gradient(top, #ee5f5b, #bd362f); - background-image: -o-linear-gradient(top, #ee5f5b, #bd362f); - background-image: linear-gradient(to bottom, #ee5f5b, #bd362f); - background-repeat: repeat-x; - filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffee5f5b', endColorstr='#ffbd362f', GradientType=0); - border-color: #bd362f #bd362f #802420; - border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); - *background-color: #bd362f; - /* Darken IE7 buttons by default so they stand out more given they won't have borders */ - - filter: progid:DXImageTransform.Microsoft.gradient(enabled = false); -} -.btn-danger:hover, -.btn-danger:focus, -.btn-danger:active, -.btn-danger.active, -.btn-danger.disabled, -.btn-danger[disabled] { - color: #ffffff; - background-color: #bd362f; - *background-color: #a9302a; -} -.btn-danger:active, -.btn-danger.active { - background-color: #942a25 \9; -} -.btn-success { - color: #ffffff; - text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25); - background-color: #5bb75b; - background-image: -moz-linear-gradient(top, #62c462, #51a351); - background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#62c462), to(#51a351)); - background-image: -webkit-linear-gradient(top, #62c462, #51a351); - background-image: -o-linear-gradient(top, #62c462, #51a351); - background-image: linear-gradient(to bottom, #62c462, #51a351); - background-repeat: repeat-x; - filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff62c462', endColorstr='#ff51a351', GradientType=0); - border-color: #51a351 #51a351 #387038; - border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); - *background-color: #51a351; - /* Darken IE7 buttons by default so they stand out more given they won't have borders */ - - filter: progid:DXImageTransform.Microsoft.gradient(enabled = false); -} -.btn-success:hover, -.btn-success:focus, -.btn-success:active, -.btn-success.active, -.btn-success.disabled, -.btn-success[disabled] { - color: #ffffff; - background-color: #51a351; - *background-color: #499249; -} -.btn-success:active, -.btn-success.active { - background-color: #408140 \9; -} -.btn-info { - color: #ffffff; - text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25); - background-color: #49afcd; - background-image: -moz-linear-gradient(top, #5bc0de, #2f96b4); - background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#5bc0de), to(#2f96b4)); - background-image: -webkit-linear-gradient(top, #5bc0de, #2f96b4); - background-image: -o-linear-gradient(top, #5bc0de, #2f96b4); - background-image: linear-gradient(to bottom, #5bc0de, #2f96b4); - background-repeat: repeat-x; - filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff5bc0de', endColorstr='#ff2f96b4', GradientType=0); - border-color: #2f96b4 #2f96b4 #1f6377; - border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); - *background-color: #2f96b4; - /* Darken IE7 buttons by default so they stand out more given they won't have borders */ - - filter: progid:DXImageTransform.Microsoft.gradient(enabled = false); -} -.btn-info:hover, -.btn-info:focus, -.btn-info:active, -.btn-info.active, -.btn-info.disabled, -.btn-info[disabled] { - color: #ffffff; - background-color: #2f96b4; - *background-color: #2a85a0; -} -.btn-info:active, -.btn-info.active { - background-color: #24748c \9; -} -.btn-inverse { - color: #ffffff; - text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25); - background-color: #363636; - background-image: -moz-linear-gradient(top, #444444, #222222); - background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#444444), to(#222222)); - background-image: -webkit-linear-gradient(top, #444444, #222222); - background-image: -o-linear-gradient(top, #444444, #222222); - background-image: linear-gradient(to bottom, #444444, #222222); - background-repeat: repeat-x; - filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff444444', endColorstr='#ff222222', GradientType=0); - border-color: #222222 #222222 #000000; - border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); - *background-color: #222222; - /* Darken IE7 buttons by default so they stand out more given they won't have borders */ - - filter: progid:DXImageTransform.Microsoft.gradient(enabled = false); -} -.btn-inverse:hover, -.btn-inverse:focus, -.btn-inverse:active, -.btn-inverse.active, -.btn-inverse.disabled, -.btn-inverse[disabled] { - color: #ffffff; - background-color: #222222; - *background-color: #151515; -} -.btn-inverse:active, -.btn-inverse.active { - background-color: #080808 \9; -} -button.btn, -input[type="submit"].btn { - *padding-top: 3px; - *padding-bottom: 3px; -} -button.btn::-moz-focus-inner, -input[type="submit"].btn::-moz-focus-inner { - padding: 0; - border: 0; -} -button.btn.btn-large, -input[type="submit"].btn.btn-large { - *padding-top: 7px; - *padding-bottom: 7px; -} -button.btn.btn-small, -input[type="submit"].btn.btn-small { - *padding-top: 3px; - *padding-bottom: 3px; -} -button.btn.btn-mini, -input[type="submit"].btn.btn-mini { - *padding-top: 1px; - *padding-bottom: 1px; -} -.btn-link, -.btn-link:active, -.btn-link[disabled] { - background-color: transparent; - background-image: none; - -webkit-box-shadow: none; - -moz-box-shadow: none; - box-shadow: none; -} -.btn-link { - border-color: transparent; - cursor: pointer; - color: #0088cc; - -webkit-border-radius: 0; - -moz-border-radius: 0; - border-radius: 0; -} -.btn-link:hover, -.btn-link:focus { - color: #005580; - text-decoration: underline; - background-color: transparent; -} -.btn-link[disabled]:hover, -.btn-link[disabled]:focus { - color: #333333; - text-decoration: none; -} -[class^="icon-"], -[class*=" icon-"] { - display: inline-block; - width: 14px; - height: 14px; - *margin-right: .3em; - line-height: 14px; - vertical-align: text-top; - background-image: url("../img/glyphicons-halflings.png"); - background-position: 14px 14px; - background-repeat: no-repeat; - margin-top: 1px; -} -/* White icons with optional class, or on hover/focus/active states of certain elements */ -.icon-white, -.nav-pills > .active > a > [class^="icon-"], -.nav-pills > .active > a > [class*=" icon-"], -.nav-list > .active > a > [class^="icon-"], -.nav-list > .active > a > [class*=" icon-"], -.navbar-inverse .nav > .active > a > [class^="icon-"], -.navbar-inverse .nav > .active > a > [class*=" icon-"], -.dropdown-menu > li > a:hover > [class^="icon-"], -.dropdown-menu > li > a:focus > [class^="icon-"], -.dropdown-menu > li > a:hover > [class*=" icon-"], -.dropdown-menu > li > a:focus > [class*=" icon-"], -.dropdown-menu > .active > a > [class^="icon-"], -.dropdown-menu > .active > a > [class*=" icon-"], -.dropdown-submenu:hover > a > [class^="icon-"], -.dropdown-submenu:focus > a > [class^="icon-"], -.dropdown-submenu:hover > a > [class*=" icon-"], -.dropdown-submenu:focus > a > [class*=" icon-"] { - background-image: url("../img/glyphicons-halflings-white.png"); -} -.icon-glass { - background-position: 0 0; -} -.icon-music { - background-position: -24px 0; -} -.icon-search { - background-position: -48px 0; -} -.icon-envelope { - background-position: -72px 0; -} -.icon-heart { - background-position: -96px 0; -} -.icon-star { - background-position: -120px 0; -} -.icon-star-empty { - background-position: -144px 0; -} -.icon-user { - background-position: -168px 0; -} -.icon-film { - background-position: -192px 0; -} -.icon-th-large { - background-position: -216px 0; -} -.icon-th { - background-position: -240px 0; -} -.icon-th-list { - background-position: -264px 0; -} -.icon-ok { - background-position: -288px 0; -} -.icon-remove { - background-position: -312px 0; -} -.icon-zoom-in { - background-position: -336px 0; -} -.icon-zoom-out { - background-position: -360px 0; -} -.icon-off { - background-position: -384px 0; -} -.icon-signal { - background-position: -408px 0; -} -.icon-cog { - background-position: -432px 0; -} -.icon-trash { - background-position: -456px 0; -} -.icon-home { - background-position: 0 -24px; -} -.icon-file { - background-position: -24px -24px; -} -.icon-time { - background-position: -48px -24px; -} -.icon-road { - background-position: -72px -24px; -} -.icon-download-alt { - background-position: -96px -24px; -} -.icon-download { - background-position: -120px -24px; -} -.icon-upload { - background-position: -144px -24px; -} -.icon-inbox { - background-position: -168px -24px; -} -.icon-play-circle { - background-position: -192px -24px; -} -.icon-repeat { - background-position: -216px -24px; -} -.icon-refresh { - background-position: -240px -24px; -} -.icon-list-alt { - background-position: -264px -24px; -} -.icon-lock { - background-position: -287px -24px; -} -.icon-flag { - background-position: -312px -24px; -} -.icon-headphones { - background-position: -336px -24px; -} -.icon-volume-off { - background-position: -360px -24px; -} -.icon-volume-down { - background-position: -384px -24px; -} -.icon-volume-up { - background-position: -408px -24px; -} -.icon-qrcode { - background-position: -432px -24px; -} -.icon-barcode { - background-position: -456px -24px; -} -.icon-tag { - background-position: 0 -48px; -} -.icon-tags { - background-position: -25px -48px; -} -.icon-book { - background-position: -48px -48px; -} -.icon-bookmark { - background-position: -72px -48px; -} -.icon-print { - background-position: -96px -48px; -} -.icon-camera { - background-position: -120px -48px; -} -.icon-font { - background-position: -144px -48px; -} -.icon-bold { - background-position: -167px -48px; -} -.icon-italic { - background-position: -192px -48px; -} -.icon-text-height { - background-position: -216px -48px; -} -.icon-text-width { - background-position: -240px -48px; -} -.icon-align-left { - background-position: -264px -48px; -} -.icon-align-center { - background-position: -288px -48px; -} -.icon-align-right { - background-position: -312px -48px; -} -.icon-align-justify { - background-position: -336px -48px; -} -.icon-list { - background-position: -360px -48px; -} -.icon-indent-left { - background-position: -384px -48px; -} -.icon-indent-right { - background-position: -408px -48px; -} -.icon-facetime-video { - background-position: -432px -48px; -} -.icon-picture { - background-position: -456px -48px; -} -.icon-pencil { - background-position: 0 -72px; -} -.icon-map-marker { - background-position: -24px -72px; -} -.icon-adjust { - background-position: -48px -72px; -} -.icon-tint { - background-position: -72px -72px; -} -.icon-edit { - background-position: -96px -72px; -} -.icon-share { - background-position: -120px -72px; -} -.icon-check { - background-position: -144px -72px; -} -.icon-move { - background-position: -168px -72px; -} -.icon-step-backward { - background-position: -192px -72px; -} -.icon-fast-backward { - background-position: -216px -72px; -} -.icon-backward { - background-position: -240px -72px; -} -.icon-play { - background-position: -264px -72px; -} -.icon-pause { - background-position: -288px -72px; -} -.icon-stop { - background-position: -312px -72px; -} -.icon-forward { - background-position: -336px -72px; -} -.icon-fast-forward { - background-position: -360px -72px; -} -.icon-step-forward { - background-position: -384px -72px; -} -.icon-eject { - background-position: -408px -72px; -} -.icon-chevron-left { - background-position: -432px -72px; -} -.icon-chevron-right { - background-position: -456px -72px; -} -.icon-plus-sign { - background-position: 0 -96px; -} -.icon-minus-sign { - background-position: -24px -96px; -} -.icon-remove-sign { - background-position: -48px -96px; -} -.icon-ok-sign { - background-position: -72px -96px; -} -.icon-question-sign { - background-position: -96px -96px; -} -.icon-info-sign { - background-position: -120px -96px; -} -.icon-screenshot { - background-position: -144px -96px; -} -.icon-remove-circle { - background-position: -168px -96px; -} -.icon-ok-circle { - background-position: -192px -96px; -} -.icon-ban-circle { - background-position: -216px -96px; -} -.icon-arrow-left { - background-position: -240px -96px; -} -.icon-arrow-right { - background-position: -264px -96px; -} -.icon-arrow-up { - background-position: -289px -96px; -} -.icon-arrow-down { - background-position: -312px -96px; -} -.icon-share-alt { - background-position: -336px -96px; -} -.icon-resize-full { - background-position: -360px -96px; -} -.icon-resize-small { - background-position: -384px -96px; -} -.icon-plus { - background-position: -408px -96px; -} -.icon-minus { - background-position: -433px -96px; -} -.icon-asterisk { - background-position: -456px -96px; -} -.icon-exclamation-sign { - background-position: 0 -120px; -} -.icon-gift { - background-position: -24px -120px; -} -.icon-leaf { - background-position: -48px -120px; -} -.icon-fire { - background-position: -72px -120px; -} -.icon-eye-open { - background-position: -96px -120px; -} -.icon-eye-close { - background-position: -120px -120px; -} -.icon-warning-sign { - background-position: -144px -120px; -} -.icon-plane { - background-position: -168px -120px; -} -.icon-calendar { - background-position: -192px -120px; -} -.icon-random { - background-position: -216px -120px; - width: 16px; -} -.icon-comment { - background-position: -240px -120px; -} -.icon-magnet { - background-position: -264px -120px; -} -.icon-chevron-up { - background-position: -288px -120px; -} -.icon-chevron-down { - background-position: -313px -119px; -} -.icon-retweet { - background-position: -336px -120px; -} -.icon-shopping-cart { - background-position: -360px -120px; -} -.icon-folder-close { - background-position: -384px -120px; - width: 16px; -} -.icon-folder-open { - background-position: -408px -120px; - width: 16px; -} -.icon-resize-vertical { - background-position: -432px -119px; -} -.icon-resize-horizontal { - background-position: -456px -118px; -} -.icon-hdd { - background-position: 0 -144px; -} -.icon-bullhorn { - background-position: -24px -144px; -} -.icon-bell { - background-position: -48px -144px; -} -.icon-certificate { - background-position: -72px -144px; -} -.icon-thumbs-up { - background-position: -96px -144px; -} -.icon-thumbs-down { - background-position: -120px -144px; -} -.icon-hand-right { - background-position: -144px -144px; -} -.icon-hand-left { - background-position: -168px -144px; -} -.icon-hand-up { - background-position: -192px -144px; -} -.icon-hand-down { - background-position: -216px -144px; -} -.icon-circle-arrow-right { - background-position: -240px -144px; -} -.icon-circle-arrow-left { - background-position: -264px -144px; -} -.icon-circle-arrow-up { - background-position: -288px -144px; -} -.icon-circle-arrow-down { - background-position: -312px -144px; -} -.icon-globe { - background-position: -336px -144px; -} -.icon-wrench { - background-position: -360px -144px; -} -.icon-tasks { - background-position: -384px -144px; -} -.icon-filter { - background-position: -408px -144px; -} -.icon-briefcase { - background-position: -432px -144px; -} -.icon-fullscreen { - background-position: -456px -144px; -} -.btn-group { - position: relative; - display: inline-block; - *display: inline; - /* IE7 inline-block hack */ - - *zoom: 1; - font-size: 0; - vertical-align: middle; - white-space: nowrap; - *margin-left: .3em; -} -.btn-group:first-child { - *margin-left: 0; -} -.btn-group + .btn-group { - margin-left: 5px; -} -.btn-toolbar { - font-size: 0; - margin-top: 10px; - margin-bottom: 10px; -} -.btn-toolbar > .btn + .btn, -.btn-toolbar > .btn-group + .btn, -.btn-toolbar > .btn + .btn-group { - margin-left: 5px; -} -.btn-group > .btn { - position: relative; - -webkit-border-radius: 0; - -moz-border-radius: 0; - border-radius: 0; -} -.btn-group > .btn + .btn { - margin-left: -1px; -} -.btn-group > .btn, -.btn-group > .dropdown-menu, -.btn-group > .popover { - font-size: 14px; -} -.btn-group > .btn-mini { - font-size: 10.5px; -} -.btn-group > .btn-small { - font-size: 11.9px; -} -.btn-group > .btn-large { - font-size: 17.5px; -} -.btn-group > .btn:first-child { - margin-left: 0; - -webkit-border-top-left-radius: 4px; - -moz-border-radius-topleft: 4px; - border-top-left-radius: 4px; - -webkit-border-bottom-left-radius: 4px; - -moz-border-radius-bottomleft: 4px; - border-bottom-left-radius: 4px; -} -.btn-group > .btn:last-child, -.btn-group > .dropdown-toggle { - -webkit-border-top-right-radius: 4px; - -moz-border-radius-topright: 4px; - border-top-right-radius: 4px; - -webkit-border-bottom-right-radius: 4px; - -moz-border-radius-bottomright: 4px; - border-bottom-right-radius: 4px; -} -.btn-group > .btn.large:first-child { - margin-left: 0; - -webkit-border-top-left-radius: 6px; - -moz-border-radius-topleft: 6px; - border-top-left-radius: 6px; - -webkit-border-bottom-left-radius: 6px; - -moz-border-radius-bottomleft: 6px; - border-bottom-left-radius: 6px; -} -.btn-group > .btn.large:last-child, -.btn-group > .large.dropdown-toggle { - -webkit-border-top-right-radius: 6px; - -moz-border-radius-topright: 6px; - border-top-right-radius: 6px; - -webkit-border-bottom-right-radius: 6px; - -moz-border-radius-bottomright: 6px; - border-bottom-right-radius: 6px; -} -.btn-group > .btn:hover, -.btn-group > .btn:focus, -.btn-group > .btn:active, -.btn-group > .btn.active { - z-index: 2; -} -.btn-group .dropdown-toggle:active, -.btn-group.open .dropdown-toggle { - outline: 0; -} -.btn-group > .btn + .dropdown-toggle { - padding-left: 8px; - padding-right: 8px; - -webkit-box-shadow: inset 1px 0 0 rgba(255,255,255,.125), inset 0 1px 0 rgba(255,255,255,.2), 0 1px 2px rgba(0,0,0,.05); - -moz-box-shadow: inset 1px 0 0 rgba(255,255,255,.125), inset 0 1px 0 rgba(255,255,255,.2), 0 1px 2px rgba(0,0,0,.05); - box-shadow: inset 1px 0 0 rgba(255,255,255,.125), inset 0 1px 0 rgba(255,255,255,.2), 0 1px 2px rgba(0,0,0,.05); - *padding-top: 5px; - *padding-bottom: 5px; -} -.btn-group > .btn-mini + .dropdown-toggle { - padding-left: 5px; - padding-right: 5px; - *padding-top: 2px; - *padding-bottom: 2px; -} -.btn-group > .btn-small + .dropdown-toggle { - *padding-top: 5px; - *padding-bottom: 4px; -} -.btn-group > .btn-large + .dropdown-toggle { - padding-left: 12px; - padding-right: 12px; - *padding-top: 7px; - *padding-bottom: 7px; -} -.btn-group.open .dropdown-toggle { - background-image: none; - -webkit-box-shadow: inset 0 2px 4px rgba(0,0,0,.15), 0 1px 2px rgba(0,0,0,.05); - -moz-box-shadow: inset 0 2px 4px rgba(0,0,0,.15), 0 1px 2px rgba(0,0,0,.05); - box-shadow: inset 0 2px 4px rgba(0,0,0,.15), 0 1px 2px rgba(0,0,0,.05); -} -.btn-group.open .btn.dropdown-toggle { - background-color: #e6e6e6; -} -.btn-group.open .btn-primary.dropdown-toggle { - background-color: #0044cc; -} -.btn-group.open .btn-warning.dropdown-toggle { - background-color: #f89406; -} -.btn-group.open .btn-danger.dropdown-toggle { - background-color: #bd362f; -} -.btn-group.open .btn-success.dropdown-toggle { - background-color: #51a351; -} -.btn-group.open .btn-info.dropdown-toggle { - background-color: #2f96b4; -} -.btn-group.open .btn-inverse.dropdown-toggle { - background-color: #222222; -} -.btn .caret { - margin-top: 8px; - margin-left: 0; -} -.btn-large .caret { - margin-top: 6px; -} -.btn-large .caret { - border-left-width: 5px; - border-right-width: 5px; - border-top-width: 5px; -} -.btn-mini .caret, -.btn-small .caret { - margin-top: 8px; -} -.dropup .btn-large .caret { - border-bottom-width: 5px; -} -.btn-primary .caret, -.btn-warning .caret, -.btn-danger .caret, -.btn-info .caret, -.btn-success .caret, -.btn-inverse .caret { - border-top-color: #ffffff; - border-bottom-color: #ffffff; -} -.btn-group-vertical { - display: inline-block; - *display: inline; - /* IE7 inline-block hack */ - - *zoom: 1; -} -.btn-group-vertical > .btn { - display: block; - float: none; - max-width: 100%; - -webkit-border-radius: 0; - -moz-border-radius: 0; - border-radius: 0; -} -.btn-group-vertical > .btn + .btn { - margin-left: 0; - margin-top: -1px; -} -.btn-group-vertical > .btn:first-child { - -webkit-border-radius: 4px 4px 0 0; - -moz-border-radius: 4px 4px 0 0; - border-radius: 4px 4px 0 0; -} -.btn-group-vertical > .btn:last-child { - -webkit-border-radius: 0 0 4px 4px; - -moz-border-radius: 0 0 4px 4px; - border-radius: 0 0 4px 4px; -} -.btn-group-vertical > .btn-large:first-child { - -webkit-border-radius: 6px 6px 0 0; - -moz-border-radius: 6px 6px 0 0; - border-radius: 6px 6px 0 0; -} -.btn-group-vertical > .btn-large:last-child { - -webkit-border-radius: 0 0 6px 6px; - -moz-border-radius: 0 0 6px 6px; - border-radius: 0 0 6px 6px; -} -.nav { - margin-left: 0; - margin-bottom: 20px; - list-style: none; -} -.nav > li > a { - display: block; -} -.nav > li > a:hover, -.nav > li > a:focus { - text-decoration: none; - background-color: #eeeeee; -} -.nav > li > a > img { - max-width: none; -} -.nav > .pull-right { - float: right; -} -.nav-header { - display: block; - padding: 3px 15px; - font-size: 11px; - font-weight: bold; - line-height: 20px; - color: #999999; - text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5); - text-transform: uppercase; -} -.nav li + .nav-header { - margin-top: 9px; -} -.nav-list { - padding-left: 15px; - padding-right: 15px; - margin-bottom: 0; -} -.nav-list > li > a, -.nav-list .nav-header { - margin-left: -15px; - margin-right: -15px; - text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5); -} -.nav-list > li > a { - padding: 3px 15px; -} -.nav-list > .active > a, -.nav-list > .active > a:hover, -.nav-list > .active > a:focus { - color: #ffffff; - text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.2); - background-color: #0088cc; -} -.nav-list [class^="icon-"], -.nav-list [class*=" icon-"] { - margin-right: 2px; -} -.nav-list .divider { - *width: 100%; - height: 1px; - margin: 9px 1px; - *margin: -5px 0 5px; - overflow: hidden; - background-color: #e5e5e5; - border-bottom: 1px solid #ffffff; -} -.nav-tabs, -.nav-pills { - *zoom: 1; -} -.nav-tabs:before, -.nav-pills:before, -.nav-tabs:after, -.nav-pills:after { - display: table; - content: ""; - line-height: 0; -} -.nav-tabs:after, -.nav-pills:after { - clear: both; -} -.nav-tabs > li, -.nav-pills > li { - float: left; -} -.nav-tabs > li > a, -.nav-pills > li > a { - padding-right: 12px; - padding-left: 12px; - margin-right: 2px; - line-height: 14px; -} -.nav-tabs { - border-bottom: 1px solid #ddd; -} -.nav-tabs > li { - margin-bottom: -1px; -} -.nav-tabs > li > a { - padding-top: 8px; - padding-bottom: 8px; - line-height: 20px; - border: 1px solid transparent; - -webkit-border-radius: 4px 4px 0 0; - -moz-border-radius: 4px 4px 0 0; - border-radius: 4px 4px 0 0; -} -.nav-tabs > li > a:hover, -.nav-tabs > li > a:focus { - border-color: #eeeeee #eeeeee #dddddd; -} -.nav-tabs > .active > a, -.nav-tabs > .active > a:hover, -.nav-tabs > .active > a:focus { - color: #555555; - background-color: #ffffff; - border: 1px solid #ddd; - border-bottom-color: transparent; - cursor: default; -} -.nav-pills > li > a { - padding-top: 8px; - padding-bottom: 8px; - margin-top: 2px; - margin-bottom: 2px; - -webkit-border-radius: 5px; - -moz-border-radius: 5px; - border-radius: 5px; -} -.nav-pills > .active > a, -.nav-pills > .active > a:hover, -.nav-pills > .active > a:focus { - color: #ffffff; - background-color: #0088cc; -} -.nav-stacked > li { - float: none; -} -.nav-stacked > li > a { - margin-right: 0; -} -.nav-tabs.nav-stacked { - border-bottom: 0; -} -.nav-tabs.nav-stacked > li > a { - border: 1px solid #ddd; - -webkit-border-radius: 0; - -moz-border-radius: 0; - border-radius: 0; -} -.nav-tabs.nav-stacked > li:first-child > a { - -webkit-border-top-right-radius: 4px; - -moz-border-radius-topright: 4px; - border-top-right-radius: 4px; - -webkit-border-top-left-radius: 4px; - -moz-border-radius-topleft: 4px; - border-top-left-radius: 4px; -} -.nav-tabs.nav-stacked > li:last-child > a { - -webkit-border-bottom-right-radius: 4px; - -moz-border-radius-bottomright: 4px; - border-bottom-right-radius: 4px; - -webkit-border-bottom-left-radius: 4px; - -moz-border-radius-bottomleft: 4px; - border-bottom-left-radius: 4px; -} -.nav-tabs.nav-stacked > li > a:hover, -.nav-tabs.nav-stacked > li > a:focus { - border-color: #ddd; - z-index: 2; -} -.nav-pills.nav-stacked > li > a { - margin-bottom: 3px; -} -.nav-pills.nav-stacked > li:last-child > a { - margin-bottom: 1px; -} -.nav-tabs .dropdown-menu { - -webkit-border-radius: 0 0 6px 6px; - -moz-border-radius: 0 0 6px 6px; - border-radius: 0 0 6px 6px; -} -.nav-pills .dropdown-menu { - -webkit-border-radius: 6px; - -moz-border-radius: 6px; - border-radius: 6px; -} -.nav .dropdown-toggle .caret { - border-top-color: #0088cc; - border-bottom-color: #0088cc; - margin-top: 6px; -} -.nav .dropdown-toggle:hover .caret, -.nav .dropdown-toggle:focus .caret { - border-top-color: #005580; - border-bottom-color: #005580; -} -/* move down carets for tabs */ -.nav-tabs .dropdown-toggle .caret { - margin-top: 8px; -} -.nav .active .dropdown-toggle .caret { - border-top-color: #fff; - border-bottom-color: #fff; -} -.nav-tabs .active .dropdown-toggle .caret { - border-top-color: #555555; - border-bottom-color: #555555; -} -.nav > .dropdown.active > a:hover, -.nav > .dropdown.active > a:focus { - cursor: pointer; -} -.nav-tabs .open .dropdown-toggle, -.nav-pills .open .dropdown-toggle, -.nav > li.dropdown.open.active > a:hover, -.nav > li.dropdown.open.active > a:focus { - color: #ffffff; - background-color: #999999; - border-color: #999999; -} -.nav li.dropdown.open .caret, -.nav li.dropdown.open.active .caret, -.nav li.dropdown.open a:hover .caret, -.nav li.dropdown.open a:focus .caret { - border-top-color: #ffffff; - border-bottom-color: #ffffff; - opacity: 1; - filter: alpha(opacity=100); -} -.tabs-stacked .open > a:hover, -.tabs-stacked .open > a:focus { - border-color: #999999; -} -.tabbable { - *zoom: 1; -} -.tabbable:before, -.tabbable:after { - display: table; - content: ""; - line-height: 0; -} -.tabbable:after { - clear: both; -} -.tab-content { - overflow: auto; -} -.tabs-below > .nav-tabs, -.tabs-right > .nav-tabs, -.tabs-left > .nav-tabs { - border-bottom: 0; -} -.tab-content > .tab-pane, -.pill-content > .pill-pane { - display: none; -} -.tab-content > .active, -.pill-content > .active { - display: block; -} -.tabs-below > .nav-tabs { - border-top: 1px solid #ddd; -} -.tabs-below > .nav-tabs > li { - margin-top: -1px; - margin-bottom: 0; -} -.tabs-below > .nav-tabs > li > a { - -webkit-border-radius: 0 0 4px 4px; - -moz-border-radius: 0 0 4px 4px; - border-radius: 0 0 4px 4px; -} -.tabs-below > .nav-tabs > li > a:hover, -.tabs-below > .nav-tabs > li > a:focus { - border-bottom-color: transparent; - border-top-color: #ddd; -} -.tabs-below > .nav-tabs > .active > a, -.tabs-below > .nav-tabs > .active > a:hover, -.tabs-below > .nav-tabs > .active > a:focus { - border-color: transparent #ddd #ddd #ddd; -} -.tabs-left > .nav-tabs > li, -.tabs-right > .nav-tabs > li { - float: none; -} -.tabs-left > .nav-tabs > li > a, -.tabs-right > .nav-tabs > li > a { - min-width: 74px; - margin-right: 0; - margin-bottom: 3px; -} -.tabs-left > .nav-tabs { - float: left; - margin-right: 19px; - border-right: 1px solid #ddd; -} -.tabs-left > .nav-tabs > li > a { - margin-right: -1px; - -webkit-border-radius: 4px 0 0 4px; - -moz-border-radius: 4px 0 0 4px; - border-radius: 4px 0 0 4px; -} -.tabs-left > .nav-tabs > li > a:hover, -.tabs-left > .nav-tabs > li > a:focus { - border-color: #eeeeee #dddddd #eeeeee #eeeeee; -} -.tabs-left > .nav-tabs .active > a, -.tabs-left > .nav-tabs .active > a:hover, -.tabs-left > .nav-tabs .active > a:focus { - border-color: #ddd transparent #ddd #ddd; - *border-right-color: #ffffff; -} -.tabs-right > .nav-tabs { - float: right; - margin-left: 19px; - border-left: 1px solid #ddd; -} -.tabs-right > .nav-tabs > li > a { - margin-left: -1px; - -webkit-border-radius: 0 4px 4px 0; - -moz-border-radius: 0 4px 4px 0; - border-radius: 0 4px 4px 0; -} -.tabs-right > .nav-tabs > li > a:hover, -.tabs-right > .nav-tabs > li > a:focus { - border-color: #eeeeee #eeeeee #eeeeee #dddddd; -} -.tabs-right > .nav-tabs .active > a, -.tabs-right > .nav-tabs .active > a:hover, -.tabs-right > .nav-tabs .active > a:focus { - border-color: #ddd #ddd #ddd transparent; - *border-left-color: #ffffff; -} -.nav > .disabled > a { - color: #999999; -} -.nav > .disabled > a:hover, -.nav > .disabled > a:focus { - text-decoration: none; - background-color: transparent; - cursor: default; -} -.navbar { - overflow: visible; - margin-bottom: 20px; - *position: relative; - *z-index: 2; -} -.navbar-inner { - min-height: 40px; - padding-left: 20px; - padding-right: 20px; - background-color: #fafafa; - background-image: -moz-linear-gradient(top, #ffffff, #f2f2f2); - background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ffffff), to(#f2f2f2)); - background-image: -webkit-linear-gradient(top, #ffffff, #f2f2f2); - background-image: -o-linear-gradient(top, #ffffff, #f2f2f2); - background-image: linear-gradient(to bottom, #ffffff, #f2f2f2); - background-repeat: repeat-x; - filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffffff', endColorstr='#fff2f2f2', GradientType=0); - border: 1px solid #d4d4d4; - -webkit-border-radius: 4px; - -moz-border-radius: 4px; - border-radius: 4px; - -webkit-box-shadow: 0 1px 4px rgba(0, 0, 0, 0.065); - -moz-box-shadow: 0 1px 4px rgba(0, 0, 0, 0.065); - box-shadow: 0 1px 4px rgba(0, 0, 0, 0.065); - *zoom: 1; -} -.navbar-inner:before, -.navbar-inner:after { - display: table; - content: ""; - line-height: 0; -} -.navbar-inner:after { - clear: both; -} -.navbar .container { - width: auto; -} -.nav-collapse.collapse { - height: auto; - overflow: visible; -} -.navbar .brand { - float: left; - display: block; - padding: 10px 20px 10px; - margin-left: -20px; - font-size: 20px; - font-weight: 200; - color: #777777; - text-shadow: 0 1px 0 #ffffff; -} -.navbar .brand:hover, -.navbar .brand:focus { - text-decoration: none; -} -.navbar-text { - margin-bottom: 0; - line-height: 40px; - color: #777777; -} -.navbar-link { - color: #777777; -} -.navbar-link:hover, -.navbar-link:focus { - color: #333333; -} -.navbar .divider-vertical { - height: 40px; - margin: 0 9px; - border-left: 1px solid #f2f2f2; - border-right: 1px solid #ffffff; -} -.navbar .btn, -.navbar .btn-group { - margin-top: 5px; -} -.navbar .btn-group .btn, -.navbar .input-prepend .btn, -.navbar .input-append .btn, -.navbar .input-prepend .btn-group, -.navbar .input-append .btn-group { - margin-top: 0; -} -.navbar-form { - margin-bottom: 0; - *zoom: 1; -} -.navbar-form:before, -.navbar-form:after { - display: table; - content: ""; - line-height: 0; -} -.navbar-form:after { - clear: both; -} -.navbar-form input, -.navbar-form select, -.navbar-form .radio, -.navbar-form .checkbox { - margin-top: 5px; -} -.navbar-form input, -.navbar-form select, -.navbar-form .btn { - display: inline-block; - margin-bottom: 0; -} -.navbar-form input[type="image"], -.navbar-form input[type="checkbox"], -.navbar-form input[type="radio"] { - margin-top: 3px; -} -.navbar-form .input-append, -.navbar-form .input-prepend { - margin-top: 5px; - white-space: nowrap; -} -.navbar-form .input-append input, -.navbar-form .input-prepend input { - margin-top: 0; -} -.navbar-search { - position: relative; - float: left; - margin-top: 5px; - margin-bottom: 0; -} -.navbar-search .search-query { - margin-bottom: 0; - padding: 4px 14px; - font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; - font-size: 13px; - font-weight: normal; - line-height: 1; - -webkit-border-radius: 15px; - -moz-border-radius: 15px; - border-radius: 15px; -} -.navbar-static-top { - position: static; - margin-bottom: 0; -} -.navbar-static-top .navbar-inner { - -webkit-border-radius: 0; - -moz-border-radius: 0; - border-radius: 0; -} -.navbar-fixed-top, -.navbar-fixed-bottom { - position: fixed; - right: 0; - left: 0; - z-index: 1030; - margin-bottom: 0; -} -.navbar-fixed-top .navbar-inner, -.navbar-static-top .navbar-inner { - border-width: 0 0 1px; -} -.navbar-fixed-bottom .navbar-inner { - border-width: 1px 0 0; -} -.navbar-fixed-top .navbar-inner, -.navbar-fixed-bottom .navbar-inner { - padding-left: 0; - padding-right: 0; - -webkit-border-radius: 0; - -moz-border-radius: 0; - border-radius: 0; -} -.navbar-static-top .container, -.navbar-fixed-top .container, -.navbar-fixed-bottom .container { - width: 940px; -} -.navbar-fixed-top { - top: 0; -} -.navbar-fixed-top .navbar-inner, -.navbar-static-top .navbar-inner { - -webkit-box-shadow: 0 1px 10px rgba(0,0,0,.1); - -moz-box-shadow: 0 1px 10px rgba(0,0,0,.1); - box-shadow: 0 1px 10px rgba(0,0,0,.1); -} -.navbar-fixed-bottom { - bottom: 0; -} -.navbar-fixed-bottom .navbar-inner { - -webkit-box-shadow: 0 -1px 10px rgba(0,0,0,.1); - -moz-box-shadow: 0 -1px 10px rgba(0,0,0,.1); - box-shadow: 0 -1px 10px rgba(0,0,0,.1); -} -.navbar .nav { - position: relative; - left: 0; - display: block; - float: left; - margin: 0 10px 0 0; -} -.navbar .nav.pull-right { - float: right; - margin-right: 0; -} -.navbar .nav > li { - float: left; -} -.navbar .nav > li > a { - float: none; - padding: 10px 15px 10px; - color: #777777; - text-decoration: none; - text-shadow: 0 1px 0 #ffffff; -} -.navbar .nav .dropdown-toggle .caret { - margin-top: 8px; -} -.navbar .nav > li > a:focus, -.navbar .nav > li > a:hover { - background-color: transparent; - color: #333333; - text-decoration: none; -} -.navbar .nav > .active > a, -.navbar .nav > .active > a:hover, -.navbar .nav > .active > a:focus { - color: #555555; - text-decoration: none; - background-color: #e5e5e5; - -webkit-box-shadow: inset 0 3px 8px rgba(0, 0, 0, 0.125); - -moz-box-shadow: inset 0 3px 8px rgba(0, 0, 0, 0.125); - box-shadow: inset 0 3px 8px rgba(0, 0, 0, 0.125); -} -.navbar .btn-navbar { - display: none; - float: right; - padding: 7px 10px; - margin-left: 5px; - margin-right: 5px; - color: #ffffff; - text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25); - background-color: #ededed; - background-image: -moz-linear-gradient(top, #f2f2f2, #e5e5e5); - background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#f2f2f2), to(#e5e5e5)); - background-image: -webkit-linear-gradient(top, #f2f2f2, #e5e5e5); - background-image: -o-linear-gradient(top, #f2f2f2, #e5e5e5); - background-image: linear-gradient(to bottom, #f2f2f2, #e5e5e5); - background-repeat: repeat-x; - filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff2f2f2', endColorstr='#ffe5e5e5', GradientType=0); - border-color: #e5e5e5 #e5e5e5 #bfbfbf; - border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); - *background-color: #e5e5e5; - /* Darken IE7 buttons by default so they stand out more given they won't have borders */ - - filter: progid:DXImageTransform.Microsoft.gradient(enabled = false); - -webkit-box-shadow: inset 0 1px 0 rgba(255,255,255,.1), 0 1px 0 rgba(255,255,255,.075); - -moz-box-shadow: inset 0 1px 0 rgba(255,255,255,.1), 0 1px 0 rgba(255,255,255,.075); - box-shadow: inset 0 1px 0 rgba(255,255,255,.1), 0 1px 0 rgba(255,255,255,.075); -} -.navbar .btn-navbar:hover, -.navbar .btn-navbar:focus, -.navbar .btn-navbar:active, -.navbar .btn-navbar.active, -.navbar .btn-navbar.disabled, -.navbar .btn-navbar[disabled] { - color: #ffffff; - background-color: #e5e5e5; - *background-color: #d9d9d9; -} -.navbar .btn-navbar:active, -.navbar .btn-navbar.active { - background-color: #cccccc \9; -} -.navbar .btn-navbar .icon-bar { - display: block; - width: 18px; - height: 2px; - background-color: #f5f5f5; - -webkit-border-radius: 1px; - -moz-border-radius: 1px; - border-radius: 1px; - -webkit-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25); - -moz-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25); - box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25); -} -.btn-navbar .icon-bar + .icon-bar { - margin-top: 3px; -} -.navbar .nav > li > .dropdown-menu:before { - content: ''; - display: inline-block; - border-left: 7px solid transparent; - border-right: 7px solid transparent; - border-bottom: 7px solid #ccc; - border-bottom-color: rgba(0, 0, 0, 0.2); - position: absolute; - top: -7px; - left: 9px; -} -.navbar .nav > li > .dropdown-menu:after { - content: ''; - display: inline-block; - border-left: 6px solid transparent; - border-right: 6px solid transparent; - border-bottom: 6px solid #ffffff; - position: absolute; - top: -6px; - left: 10px; -} -.navbar-fixed-bottom .nav > li > .dropdown-menu:before { - border-top: 7px solid #ccc; - border-top-color: rgba(0, 0, 0, 0.2); - border-bottom: 0; - bottom: -7px; - top: auto; -} -.navbar-fixed-bottom .nav > li > .dropdown-menu:after { - border-top: 6px solid #ffffff; - border-bottom: 0; - bottom: -6px; - top: auto; -} -.navbar .nav li.dropdown > a:hover .caret, -.navbar .nav li.dropdown > a:focus .caret { - border-top-color: #333333; - border-bottom-color: #333333; -} -.navbar .nav li.dropdown.open > .dropdown-toggle, -.navbar .nav li.dropdown.active > .dropdown-toggle, -.navbar .nav li.dropdown.open.active > .dropdown-toggle { - background-color: #e5e5e5; - color: #555555; -} -.navbar .nav li.dropdown > .dropdown-toggle .caret { - border-top-color: #777777; - border-bottom-color: #777777; -} -.navbar .nav li.dropdown.open > .dropdown-toggle .caret, -.navbar .nav li.dropdown.active > .dropdown-toggle .caret, -.navbar .nav li.dropdown.open.active > .dropdown-toggle .caret { - border-top-color: #555555; - border-bottom-color: #555555; -} -.navbar .pull-right > li > .dropdown-menu, -.navbar .nav > li > .dropdown-menu.pull-right { - left: auto; - right: 0; -} -.navbar .pull-right > li > .dropdown-menu:before, -.navbar .nav > li > .dropdown-menu.pull-right:before { - left: auto; - right: 12px; -} -.navbar .pull-right > li > .dropdown-menu:after, -.navbar .nav > li > .dropdown-menu.pull-right:after { - left: auto; - right: 13px; -} -.navbar .pull-right > li > .dropdown-menu .dropdown-menu, -.navbar .nav > li > .dropdown-menu.pull-right .dropdown-menu { - left: auto; - right: 100%; - margin-left: 0; - margin-right: -1px; - -webkit-border-radius: 6px 0 6px 6px; - -moz-border-radius: 6px 0 6px 6px; - border-radius: 6px 0 6px 6px; -} -.navbar-inverse .navbar-inner { - background-color: #1b1b1b; - background-image: -moz-linear-gradient(top, #222222, #111111); - background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#222222), to(#111111)); - background-image: -webkit-linear-gradient(top, #222222, #111111); - background-image: -o-linear-gradient(top, #222222, #111111); - background-image: linear-gradient(to bottom, #222222, #111111); - background-repeat: repeat-x; - filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff222222', endColorstr='#ff111111', GradientType=0); - border-color: #252525; -} -.navbar-inverse .brand, -.navbar-inverse .nav > li > a { - color: #999999; - text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25); -} -.navbar-inverse .brand:hover, -.navbar-inverse .nav > li > a:hover, -.navbar-inverse .brand:focus, -.navbar-inverse .nav > li > a:focus { - color: #ffffff; -} -.navbar-inverse .brand { - color: #999999; -} -.navbar-inverse .navbar-text { - color: #999999; -} -.navbar-inverse .nav > li > a:focus, -.navbar-inverse .nav > li > a:hover { - background-color: transparent; - color: #ffffff; -} -.navbar-inverse .nav .active > a, -.navbar-inverse .nav .active > a:hover, -.navbar-inverse .nav .active > a:focus { - color: #ffffff; - background-color: #111111; -} -.navbar-inverse .navbar-link { - color: #999999; -} -.navbar-inverse .navbar-link:hover, -.navbar-inverse .navbar-link:focus { - color: #ffffff; -} -.navbar-inverse .divider-vertical { - border-left-color: #111111; - border-right-color: #222222; -} -.navbar-inverse .nav li.dropdown.open > .dropdown-toggle, -.navbar-inverse .nav li.dropdown.active > .dropdown-toggle, -.navbar-inverse .nav li.dropdown.open.active > .dropdown-toggle { - background-color: #111111; - color: #ffffff; -} -.navbar-inverse .nav li.dropdown > a:hover .caret, -.navbar-inverse .nav li.dropdown > a:focus .caret { - border-top-color: #ffffff; - border-bottom-color: #ffffff; -} -.navbar-inverse .nav li.dropdown > .dropdown-toggle .caret { - border-top-color: #999999; - border-bottom-color: #999999; -} -.navbar-inverse .nav li.dropdown.open > .dropdown-toggle .caret, -.navbar-inverse .nav li.dropdown.active > .dropdown-toggle .caret, -.navbar-inverse .nav li.dropdown.open.active > .dropdown-toggle .caret { - border-top-color: #ffffff; - border-bottom-color: #ffffff; -} -.navbar-inverse .navbar-search .search-query { - color: #ffffff; - background-color: #515151; - border-color: #111111; - -webkit-box-shadow: inset 0 1px 2px rgba(0,0,0,.1), 0 1px 0 rgba(255,255,255,.15); - -moz-box-shadow: inset 0 1px 2px rgba(0,0,0,.1), 0 1px 0 rgba(255,255,255,.15); - box-shadow: inset 0 1px 2px rgba(0,0,0,.1), 0 1px 0 rgba(255,255,255,.15); - -webkit-transition: none; - -moz-transition: none; - -o-transition: none; - transition: none; -} -.navbar-inverse .navbar-search .search-query:-moz-placeholder { - color: #cccccc; -} -.navbar-inverse .navbar-search .search-query:-ms-input-placeholder { - color: #cccccc; -} -.navbar-inverse .navbar-search .search-query::-webkit-input-placeholder { - color: #cccccc; -} -.navbar-inverse .navbar-search .search-query:focus, -.navbar-inverse .navbar-search .search-query.focused { - padding: 5px 15px; - color: #333333; - text-shadow: 0 1px 0 #ffffff; - background-color: #ffffff; - border: 0; - -webkit-box-shadow: 0 0 3px rgba(0, 0, 0, 0.15); - -moz-box-shadow: 0 0 3px rgba(0, 0, 0, 0.15); - box-shadow: 0 0 3px rgba(0, 0, 0, 0.15); - outline: 0; -} -.navbar-inverse .btn-navbar { - color: #ffffff; - text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25); - background-color: #0e0e0e; - background-image: -moz-linear-gradient(top, #151515, #040404); - background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#151515), to(#040404)); - background-image: -webkit-linear-gradient(top, #151515, #040404); - background-image: -o-linear-gradient(top, #151515, #040404); - background-image: linear-gradient(to bottom, #151515, #040404); - background-repeat: repeat-x; - filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff151515', endColorstr='#ff040404', GradientType=0); - border-color: #040404 #040404 #000000; - border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); - *background-color: #040404; - /* Darken IE7 buttons by default so they stand out more given they won't have borders */ - - filter: progid:DXImageTransform.Microsoft.gradient(enabled = false); -} -.navbar-inverse .btn-navbar:hover, -.navbar-inverse .btn-navbar:focus, -.navbar-inverse .btn-navbar:active, -.navbar-inverse .btn-navbar.active, -.navbar-inverse .btn-navbar.disabled, -.navbar-inverse .btn-navbar[disabled] { - color: #ffffff; - background-color: #040404; - *background-color: #000000; -} -.navbar-inverse .btn-navbar:active, -.navbar-inverse .btn-navbar.active { - background-color: #000000 \9; -} -.breadcrumb { - padding: 8px 15px; - margin: 0 0 20px; - list-style: none; - background-color: #f5f5f5; - -webkit-border-radius: 4px; - -moz-border-radius: 4px; - border-radius: 4px; -} -.breadcrumb > li { - display: inline-block; - *display: inline; - /* IE7 inline-block hack */ - - *zoom: 1; - text-shadow: 0 1px 0 #ffffff; -} -.breadcrumb > li > .divider { - padding: 0 5px; - color: #ccc; -} -.breadcrumb > .active { - color: #999999; -} -.pagination { - margin: 20px 0; -} -.pagination ul { - display: inline-block; - *display: inline; - /* IE7 inline-block hack */ - - *zoom: 1; - margin-left: 0; - margin-bottom: 0; - -webkit-border-radius: 4px; - -moz-border-radius: 4px; - border-radius: 4px; - -webkit-box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05); - -moz-box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05); - box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05); -} -.pagination ul > li { - display: inline; -} -.pagination ul > li > a, -.pagination ul > li > span { - float: left; - padding: 4px 12px; - line-height: 20px; - text-decoration: none; - background-color: #ffffff; - border: 1px solid #dddddd; - border-left-width: 0; -} -.pagination ul > li > a:hover, -.pagination ul > li > a:focus, -.pagination ul > .active > a, -.pagination ul > .active > span { - background-color: #f5f5f5; -} -.pagination ul > .active > a, -.pagination ul > .active > span { - color: #999999; - cursor: default; -} -.pagination ul > .disabled > span, -.pagination ul > .disabled > a, -.pagination ul > .disabled > a:hover, -.pagination ul > .disabled > a:focus { - color: #999999; - background-color: transparent; - cursor: default; -} -.pagination ul > li:first-child > a, -.pagination ul > li:first-child > span { - border-left-width: 1px; - -webkit-border-top-left-radius: 4px; - -moz-border-radius-topleft: 4px; - border-top-left-radius: 4px; - -webkit-border-bottom-left-radius: 4px; - -moz-border-radius-bottomleft: 4px; - border-bottom-left-radius: 4px; -} -.pagination ul > li:last-child > a, -.pagination ul > li:last-child > span { - -webkit-border-top-right-radius: 4px; - -moz-border-radius-topright: 4px; - border-top-right-radius: 4px; - -webkit-border-bottom-right-radius: 4px; - -moz-border-radius-bottomright: 4px; - border-bottom-right-radius: 4px; -} -.pagination-centered { - text-align: center; -} -.pagination-right { - text-align: right; -} -.pagination-large ul > li > a, -.pagination-large ul > li > span { - padding: 11px 19px; - font-size: 17.5px; -} -.pagination-large ul > li:first-child > a, -.pagination-large ul > li:first-child > span { - -webkit-border-top-left-radius: 6px; - -moz-border-radius-topleft: 6px; - border-top-left-radius: 6px; - -webkit-border-bottom-left-radius: 6px; - -moz-border-radius-bottomleft: 6px; - border-bottom-left-radius: 6px; -} -.pagination-large ul > li:last-child > a, -.pagination-large ul > li:last-child > span { - -webkit-border-top-right-radius: 6px; - -moz-border-radius-topright: 6px; - border-top-right-radius: 6px; - -webkit-border-bottom-right-radius: 6px; - -moz-border-radius-bottomright: 6px; - border-bottom-right-radius: 6px; -} -.pagination-mini ul > li:first-child > a, -.pagination-small ul > li:first-child > a, -.pagination-mini ul > li:first-child > span, -.pagination-small ul > li:first-child > span { - -webkit-border-top-left-radius: 3px; - -moz-border-radius-topleft: 3px; - border-top-left-radius: 3px; - -webkit-border-bottom-left-radius: 3px; - -moz-border-radius-bottomleft: 3px; - border-bottom-left-radius: 3px; -} -.pagination-mini ul > li:last-child > a, -.pagination-small ul > li:last-child > a, -.pagination-mini ul > li:last-child > span, -.pagination-small ul > li:last-child > span { - -webkit-border-top-right-radius: 3px; - -moz-border-radius-topright: 3px; - border-top-right-radius: 3px; - -webkit-border-bottom-right-radius: 3px; - -moz-border-radius-bottomright: 3px; - border-bottom-right-radius: 3px; -} -.pagination-small ul > li > a, -.pagination-small ul > li > span { - padding: 2px 10px; - font-size: 11.9px; -} -.pagination-mini ul > li > a, -.pagination-mini ul > li > span { - padding: 0 6px; - font-size: 10.5px; -} -.pager { - margin: 20px 0; - list-style: none; - text-align: center; - *zoom: 1; -} -.pager:before, -.pager:after { - display: table; - content: ""; - line-height: 0; -} -.pager:after { - clear: both; -} -.pager li { - display: inline; -} -.pager li > a, -.pager li > span { - display: inline-block; - padding: 5px 14px; - background-color: #fff; - border: 1px solid #ddd; - -webkit-border-radius: 15px; - -moz-border-radius: 15px; - border-radius: 15px; -} -.pager li > a:hover, -.pager li > a:focus { - text-decoration: none; - background-color: #f5f5f5; -} -.pager .next > a, -.pager .next > span { - float: right; -} -.pager .previous > a, -.pager .previous > span { - float: left; -} -.pager .disabled > a, -.pager .disabled > a:hover, -.pager .disabled > a:focus, -.pager .disabled > span { - color: #999999; - background-color: #fff; - cursor: default; -} -.thumbnails { - margin-left: -20px; - list-style: none; - *zoom: 1; -} -.thumbnails:before, -.thumbnails:after { - display: table; - content: ""; - line-height: 0; -} -.thumbnails:after { - clear: both; -} -.row-fluid .thumbnails { - margin-left: 0; -} -.thumbnails > li { - float: left; - margin-bottom: 20px; - margin-left: 20px; -} -.thumbnail { - display: block; - padding: 4px; - line-height: 20px; - border: 1px solid #ddd; - -webkit-border-radius: 4px; - -moz-border-radius: 4px; - border-radius: 4px; - -webkit-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.055); - -moz-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.055); - box-shadow: 0 1px 3px rgba(0, 0, 0, 0.055); - -webkit-transition: all 0.2s ease-in-out; - -moz-transition: all 0.2s ease-in-out; - -o-transition: all 0.2s ease-in-out; - transition: all 0.2s ease-in-out; -} -a.thumbnail:hover, -a.thumbnail:focus { - border-color: #0088cc; - -webkit-box-shadow: 0 1px 4px rgba(0, 105, 214, 0.25); - -moz-box-shadow: 0 1px 4px rgba(0, 105, 214, 0.25); - box-shadow: 0 1px 4px rgba(0, 105, 214, 0.25); -} -.thumbnail > img { - display: block; - max-width: 100%; - margin-left: auto; - margin-right: auto; -} -.thumbnail .caption { - padding: 9px; - color: #555555; -} -.alert { - padding: 8px 35px 8px 14px; - margin-bottom: 20px; - text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5); - background-color: #fcf8e3; - border: 1px solid #fbeed5; - -webkit-border-radius: 4px; - -moz-border-radius: 4px; - border-radius: 4px; -} -.alert, -.alert h4 { - color: #c09853; -} -.alert h4 { - margin: 0; -} -.alert .close { - position: relative; - top: -2px; - right: -21px; - line-height: 20px; -} -.alert-success { - background-color: #dff0d8; - border-color: #d6e9c6; - color: #468847; -} -.alert-success h4 { - color: #468847; -} -.alert-danger, -.alert-error { - background-color: #f2dede; - border-color: #eed3d7; - color: #b94a48; -} -.alert-danger h4, -.alert-error h4 { - color: #b94a48; -} -.alert-info { - background-color: #d9edf7; - border-color: #bce8f1; - color: #3a87ad; -} -.alert-info h4 { - color: #3a87ad; -} -.alert-block { - padding-top: 14px; - padding-bottom: 14px; -} -.alert-block > p, -.alert-block > ul { - margin-bottom: 0; -} -.alert-block p + p { - margin-top: 5px; -} -@-webkit-keyframes progress-bar-stripes { - from { - background-position: 40px 0; - } - to { - background-position: 0 0; - } -} -@-moz-keyframes progress-bar-stripes { - from { - background-position: 40px 0; - } - to { - background-position: 0 0; - } -} -@-ms-keyframes progress-bar-stripes { - from { - background-position: 40px 0; - } - to { - background-position: 0 0; - } -} -@-o-keyframes progress-bar-stripes { - from { - background-position: 0 0; - } - to { - background-position: 40px 0; - } -} -@keyframes progress-bar-stripes { - from { - background-position: 40px 0; - } - to { - background-position: 0 0; - } -} -.progress { - overflow: hidden; - height: 20px; - margin-bottom: 20px; - background-color: #f7f7f7; - background-image: -moz-linear-gradient(top, #f5f5f5, #f9f9f9); - background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#f5f5f5), to(#f9f9f9)); - background-image: -webkit-linear-gradient(top, #f5f5f5, #f9f9f9); - background-image: -o-linear-gradient(top, #f5f5f5, #f9f9f9); - background-image: linear-gradient(to bottom, #f5f5f5, #f9f9f9); - background-repeat: repeat-x; - filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff5f5f5', endColorstr='#fff9f9f9', GradientType=0); - -webkit-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1); - -moz-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1); - box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1); - -webkit-border-radius: 4px; - -moz-border-radius: 4px; - border-radius: 4px; -} -.progress .bar { - width: 0%; - height: 100%; - color: #ffffff; - float: left; - font-size: 12px; - text-align: center; - text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25); - background-color: #0e90d2; - background-image: -moz-linear-gradient(top, #149bdf, #0480be); - background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#149bdf), to(#0480be)); - background-image: -webkit-linear-gradient(top, #149bdf, #0480be); - background-image: -o-linear-gradient(top, #149bdf, #0480be); - background-image: linear-gradient(to bottom, #149bdf, #0480be); - background-repeat: repeat-x; - filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff149bdf', endColorstr='#ff0480be', GradientType=0); - -webkit-box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.15); - -moz-box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.15); - box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.15); - -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - box-sizing: border-box; - -webkit-transition: width 0.6s ease; - -moz-transition: width 0.6s ease; - -o-transition: width 0.6s ease; - transition: width 0.6s ease; -} -.progress .bar + .bar { - -webkit-box-shadow: inset 1px 0 0 rgba(0,0,0,.15), inset 0 -1px 0 rgba(0,0,0,.15); - -moz-box-shadow: inset 1px 0 0 rgba(0,0,0,.15), inset 0 -1px 0 rgba(0,0,0,.15); - box-shadow: inset 1px 0 0 rgba(0,0,0,.15), inset 0 -1px 0 rgba(0,0,0,.15); -} -.progress-striped .bar { - background-color: #149bdf; - background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent)); - background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); - background-image: -moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); - background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); - background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); - -webkit-background-size: 40px 40px; - -moz-background-size: 40px 40px; - -o-background-size: 40px 40px; - background-size: 40px 40px; -} -.progress.active .bar { - -webkit-animation: progress-bar-stripes 2s linear infinite; - -moz-animation: progress-bar-stripes 2s linear infinite; - -ms-animation: progress-bar-stripes 2s linear infinite; - -o-animation: progress-bar-stripes 2s linear infinite; - animation: progress-bar-stripes 2s linear infinite; -} -.progress-danger .bar, -.progress .bar-danger { - background-color: #dd514c; - background-image: -moz-linear-gradient(top, #ee5f5b, #c43c35); - background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ee5f5b), to(#c43c35)); - background-image: -webkit-linear-gradient(top, #ee5f5b, #c43c35); - background-image: -o-linear-gradient(top, #ee5f5b, #c43c35); - background-image: linear-gradient(to bottom, #ee5f5b, #c43c35); - background-repeat: repeat-x; - filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffee5f5b', endColorstr='#ffc43c35', GradientType=0); -} -.progress-danger.progress-striped .bar, -.progress-striped .bar-danger { - background-color: #ee5f5b; - background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent)); - background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); - background-image: -moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); - background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); - background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); -} -.progress-success .bar, -.progress .bar-success { - background-color: #5eb95e; - background-image: -moz-linear-gradient(top, #62c462, #57a957); - background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#62c462), to(#57a957)); - background-image: -webkit-linear-gradient(top, #62c462, #57a957); - background-image: -o-linear-gradient(top, #62c462, #57a957); - background-image: linear-gradient(to bottom, #62c462, #57a957); - background-repeat: repeat-x; - filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff62c462', endColorstr='#ff57a957', GradientType=0); -} -.progress-success.progress-striped .bar, -.progress-striped .bar-success { - background-color: #62c462; - background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent)); - background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); - background-image: -moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); - background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); - background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); -} -.progress-info .bar, -.progress .bar-info { - background-color: #4bb1cf; - background-image: -moz-linear-gradient(top, #5bc0de, #339bb9); - background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#5bc0de), to(#339bb9)); - background-image: -webkit-linear-gradient(top, #5bc0de, #339bb9); - background-image: -o-linear-gradient(top, #5bc0de, #339bb9); - background-image: linear-gradient(to bottom, #5bc0de, #339bb9); - background-repeat: repeat-x; - filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff5bc0de', endColorstr='#ff339bb9', GradientType=0); -} -.progress-info.progress-striped .bar, -.progress-striped .bar-info { - background-color: #5bc0de; - background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent)); - background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); - background-image: -moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); - background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); - background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); -} -.progress-warning .bar, -.progress .bar-warning { - background-color: #faa732; - background-image: -moz-linear-gradient(top, #fbb450, #f89406); - background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#fbb450), to(#f89406)); - background-image: -webkit-linear-gradient(top, #fbb450, #f89406); - background-image: -o-linear-gradient(top, #fbb450, #f89406); - background-image: linear-gradient(to bottom, #fbb450, #f89406); - background-repeat: repeat-x; - filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#fffbb450', endColorstr='#fff89406', GradientType=0); -} -.progress-warning.progress-striped .bar, -.progress-striped .bar-warning { - background-color: #fbb450; - background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent)); - background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); - background-image: -moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); - background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); - background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); -} -.hero-unit { - padding: 60px; - margin-bottom: 30px; - font-size: 18px; - font-weight: 200; - line-height: 30px; - color: inherit; - background-color: #4eacc5; - -webkit-border-radius: 6px; - -moz-border-radius: 6px; - border-radius: 6px; -} -.hero-unit h1 { - margin-bottom: 0; - font-size: 60px; - line-height: 1; - color: inherit; - letter-spacing: -1px; -} -.hero-unit li { - line-height: 30px; -} -.media, -.media-body { - overflow: hidden; - *overflow: visible; - zoom: 1; -} -.media, -.media .media { - margin-top: 15px; -} -.media:first-child { - margin-top: 0; -} -.media-object { - display: block; -} -.media-heading { - margin: 0 0 5px; -} -.media > .pull-left { - margin-right: 10px; -} -.media > .pull-right { - margin-left: 10px; -} -.media-list { - margin-left: 0; - list-style: none; -} -.tooltip { - position: absolute; - z-index: 1030; - display: block; - visibility: visible; - font-size: 11px; - line-height: 1.4; - opacity: 0; - filter: alpha(opacity=0); -} -.tooltip.in { - opacity: 0.8; - filter: alpha(opacity=80); -} -.tooltip.top { - margin-top: -3px; - padding: 5px 0; -} -.tooltip.right { - margin-left: 3px; - padding: 0 5px; -} -.tooltip.bottom { - margin-top: 3px; - padding: 5px 0; -} -.tooltip.left { - margin-left: -3px; - padding: 0 5px; -} -.tooltip-inner { - max-width: 200px; - padding: 8px; - color: #ffffff; - text-align: center; - text-decoration: none; - background-color: #000000; - -webkit-border-radius: 4px; - -moz-border-radius: 4px; - border-radius: 4px; -} -.tooltip-arrow { - position: absolute; - width: 0; - height: 0; - border-color: transparent; - border-style: solid; -} -.tooltip.top .tooltip-arrow { - bottom: 0; - left: 50%; - margin-left: -5px; - border-width: 5px 5px 0; - border-top-color: #000000; -} -.tooltip.right .tooltip-arrow { - top: 50%; - left: 0; - margin-top: -5px; - border-width: 5px 5px 5px 0; - border-right-color: #000000; -} -.tooltip.left .tooltip-arrow { - top: 50%; - right: 0; - margin-top: -5px; - border-width: 5px 0 5px 5px; - border-left-color: #000000; -} -.tooltip.bottom .tooltip-arrow { - top: 0; - left: 50%; - margin-left: -5px; - border-width: 0 5px 5px; - border-bottom-color: #000000; -} -.popover { - position: absolute; - top: 0; - left: 0; - z-index: 1010; - display: none; - max-width: 276px; - padding: 1px; - text-align: left; - background-color: #ffffff; - -webkit-background-clip: padding-box; - -moz-background-clip: padding; - background-clip: padding-box; - border: 1px solid #ccc; - border: 1px solid rgba(0, 0, 0, 0.2); - -webkit-border-radius: 6px; - -moz-border-radius: 6px; - border-radius: 6px; - -webkit-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2); - -moz-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2); - box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2); - white-space: normal; -} -.popover.top { - margin-top: -10px; -} -.popover.right { - margin-left: 10px; -} -.popover.bottom { - margin-top: 10px; -} -.popover.left { - margin-left: -10px; -} -.popover-title { - margin: 0; - padding: 8px 14px; - font-size: 14px; - font-weight: normal; - line-height: 18px; - background-color: #f7f7f7; - border-bottom: 1px solid #ebebeb; - -webkit-border-radius: 5px 5px 0 0; - -moz-border-radius: 5px 5px 0 0; - border-radius: 5px 5px 0 0; -} -.popover-title:empty { - display: none; -} -.popover-content { - padding: 9px 14px; -} -.popover .arrow, -.popover .arrow:after { - position: absolute; - display: block; - width: 0; - height: 0; - border-color: transparent; - border-style: solid; -} -.popover .arrow { - border-width: 11px; -} -.popover .arrow:after { - border-width: 10px; - content: ""; -} -.popover.top .arrow { - left: 50%; - margin-left: -11px; - border-bottom-width: 0; - border-top-color: #999; - border-top-color: rgba(0, 0, 0, 0.25); - bottom: -11px; -} -.popover.top .arrow:after { - bottom: 1px; - margin-left: -10px; - border-bottom-width: 0; - border-top-color: #ffffff; -} -.popover.right .arrow { - top: 50%; - left: -11px; - margin-top: -11px; - border-left-width: 0; - border-right-color: #999; - border-right-color: rgba(0, 0, 0, 0.25); -} -.popover.right .arrow:after { - left: 1px; - bottom: -10px; - border-left-width: 0; - border-right-color: #ffffff; -} -.popover.bottom .arrow { - left: 50%; - margin-left: -11px; - border-top-width: 0; - border-bottom-color: #999; - border-bottom-color: rgba(0, 0, 0, 0.25); - top: -11px; -} -.popover.bottom .arrow:after { - top: 1px; - margin-left: -10px; - border-top-width: 0; - border-bottom-color: #ffffff; -} -.popover.left .arrow { - top: 50%; - right: -11px; - margin-top: -11px; - border-right-width: 0; - border-left-color: #999; - border-left-color: rgba(0, 0, 0, 0.25); -} -.popover.left .arrow:after { - right: 1px; - border-right-width: 0; - border-left-color: #ffffff; - bottom: -10px; -} -.modal-backdrop { - position: fixed; - top: 0; - right: 0; - bottom: 0; - left: 0; - z-index: 1040; - background-color: #000000; -} -.modal-backdrop.fade { - opacity: 0; -} -.modal-backdrop, -.modal-backdrop.fade.in { - opacity: 0.8; - filter: alpha(opacity=80); -} -.modal { - position: fixed; - top: 10%; - left: 50%; - z-index: 1050; - width: 560px; - margin-left: -280px; - background-color: #ffffff; - border: 1px solid #999; - border: 1px solid rgba(0, 0, 0, 0.3); - *border: 1px solid #999; - /* IE6-7 */ - - -webkit-border-radius: 6px; - -moz-border-radius: 6px; - border-radius: 6px; - -webkit-box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3); - -moz-box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3); - box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3); - -webkit-background-clip: padding-box; - -moz-background-clip: padding-box; - background-clip: padding-box; - outline: none; -} -.modal.fade { - -webkit-transition: opacity .3s linear, top .3s ease-out; - -moz-transition: opacity .3s linear, top .3s ease-out; - -o-transition: opacity .3s linear, top .3s ease-out; - transition: opacity .3s linear, top .3s ease-out; - top: -25%; -} -.modal.fade.in { - top: 10%; -} -.modal-header { - padding: 9px 15px; - border-bottom: 1px solid #eee; -} -.modal-header .close { - margin-top: 2px; -} -.modal-header h3 { - margin: 0; - line-height: 30px; -} -.modal-body { - position: relative; - overflow-y: auto; - max-height: 400px; - padding: 15px; -} -.modal-form { - margin-bottom: 0; -} -.modal-footer { - padding: 14px 15px 15px; - margin-bottom: 0; - text-align: right; - background-color: #f5f5f5; - border-top: 1px solid #ddd; - -webkit-border-radius: 0 0 6px 6px; - -moz-border-radius: 0 0 6px 6px; - border-radius: 0 0 6px 6px; - -webkit-box-shadow: inset 0 1px 0 #ffffff; - -moz-box-shadow: inset 0 1px 0 #ffffff; - box-shadow: inset 0 1px 0 #ffffff; - *zoom: 1; -} -.modal-footer:before, -.modal-footer:after { - display: table; - content: ""; - line-height: 0; -} -.modal-footer:after { - clear: both; -} -.modal-footer .btn + .btn { - margin-left: 5px; - margin-bottom: 0; -} -.modal-footer .btn-group .btn + .btn { - margin-left: -1px; -} -.modal-footer .btn-block + .btn-block { - margin-left: 0; -} -.dropup, -.dropdown { - position: relative; -} -.dropdown-toggle { - *margin-bottom: -3px; -} -.dropdown-toggle:active, -.open .dropdown-toggle { - outline: 0; -} -.caret { - display: inline-block; - width: 0; - height: 0; - vertical-align: top; - border-top: 4px solid #000000; - border-right: 4px solid transparent; - border-left: 4px solid transparent; - content: ""; -} -.dropdown .caret { - margin-top: 8px; - margin-left: 2px; -} -.dropdown-menu { - position: absolute; - top: 100%; - left: 0; - z-index: 1000; - display: none; - float: left; - min-width: 160px; - padding: 5px 0; - margin: 2px 0 0; - list-style: none; - background-color: #ffffff; - border: 1px solid #ccc; - border: 1px solid rgba(0, 0, 0, 0.2); - *border-right-width: 2px; - *border-bottom-width: 2px; - -webkit-border-radius: 6px; - -moz-border-radius: 6px; - border-radius: 6px; - -webkit-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2); - -moz-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2); - box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2); - -webkit-background-clip: padding-box; - -moz-background-clip: padding; - background-clip: padding-box; -} -.dropdown-menu.pull-right { - right: 0; - left: auto; -} -.dropdown-menu .divider { - *width: 100%; - height: 1px; - margin: 9px 1px; - *margin: -5px 0 5px; - overflow: hidden; - background-color: #e5e5e5; - border-bottom: 1px solid #ffffff; -} -.dropdown-menu > li > a { - display: block; - padding: 3px 20px; - clear: both; - font-weight: normal; - line-height: 20px; - color: #333333; - white-space: nowrap; -} -.dropdown-menu > li > a:hover, -.dropdown-menu > li > a:focus, -.dropdown-submenu:hover > a, -.dropdown-submenu:focus > a { - text-decoration: none; - color: #ffffff; - background-color: #0081c2; - background-image: -moz-linear-gradient(top, #0088cc, #0077b3); - background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#0088cc), to(#0077b3)); - background-image: -webkit-linear-gradient(top, #0088cc, #0077b3); - background-image: -o-linear-gradient(top, #0088cc, #0077b3); - background-image: linear-gradient(to bottom, #0088cc, #0077b3); - background-repeat: repeat-x; - filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff0088cc', endColorstr='#ff0077b3', GradientType=0); -} -.dropdown-menu > .active > a, -.dropdown-menu > .active > a:hover, -.dropdown-menu > .active > a:focus { - color: #ffffff; - text-decoration: none; - outline: 0; - background-color: #0081c2; - background-image: -moz-linear-gradient(top, #0088cc, #0077b3); - background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#0088cc), to(#0077b3)); - background-image: -webkit-linear-gradient(top, #0088cc, #0077b3); - background-image: -o-linear-gradient(top, #0088cc, #0077b3); - background-image: linear-gradient(to bottom, #0088cc, #0077b3); - background-repeat: repeat-x; - filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff0088cc', endColorstr='#ff0077b3', GradientType=0); -} -.dropdown-menu > .disabled > a, -.dropdown-menu > .disabled > a:hover, -.dropdown-menu > .disabled > a:focus { - color: #999999; -} -.dropdown-menu > .disabled > a:hover, -.dropdown-menu > .disabled > a:focus { - text-decoration: none; - background-color: transparent; - background-image: none; - filter: progid:DXImageTransform.Microsoft.gradient(enabled = false); - cursor: default; -} -.open { - *z-index: 1000; -} -.open > .dropdown-menu { - display: block; -} -.dropdown-backdrop { - position: fixed; - left: 0; - right: 0; - bottom: 0; - top: 0; - z-index: 990; -} -.pull-right > .dropdown-menu { - right: 0; - left: auto; -} -.dropup .caret, -.navbar-fixed-bottom .dropdown .caret { - border-top: 0; - border-bottom: 4px solid #000000; - content: ""; -} -.dropup .dropdown-menu, -.navbar-fixed-bottom .dropdown .dropdown-menu { - top: auto; - bottom: 100%; - margin-bottom: 1px; -} -.dropdown-submenu { - position: relative; -} -.dropdown-submenu > .dropdown-menu { - top: 0; - left: 100%; - margin-top: -6px; - margin-left: -1px; - -webkit-border-radius: 0 6px 6px 6px; - -moz-border-radius: 0 6px 6px 6px; - border-radius: 0 6px 6px 6px; -} -.dropdown-submenu:hover > .dropdown-menu { - display: block; -} -.dropup .dropdown-submenu > .dropdown-menu { - top: auto; - bottom: 0; - margin-top: 0; - margin-bottom: -2px; - -webkit-border-radius: 5px 5px 5px 0; - -moz-border-radius: 5px 5px 5px 0; - border-radius: 5px 5px 5px 0; -} -.dropdown-submenu > a:after { - display: block; - content: " "; - float: right; - width: 0; - height: 0; - border-color: transparent; - border-style: solid; - border-width: 5px 0 5px 5px; - border-left-color: #cccccc; - margin-top: 5px; - margin-right: -10px; -} -.dropdown-submenu:hover > a:after { - border-left-color: #ffffff; -} -.dropdown-submenu.pull-left { - float: none; -} -.dropdown-submenu.pull-left > .dropdown-menu { - left: -100%; - margin-left: 10px; - -webkit-border-radius: 6px 0 6px 6px; - -moz-border-radius: 6px 0 6px 6px; - border-radius: 6px 0 6px 6px; -} -.dropdown .dropdown-menu .nav-header { - padding-left: 20px; - padding-right: 20px; -} -.typeahead { - z-index: 1051; - margin-top: 2px; - -webkit-border-radius: 4px; - -moz-border-radius: 4px; - border-radius: 4px; -} -.accordion { - margin-bottom: 20px; -} -.accordion-group { - margin-bottom: 2px; - border: 1px solid #e5e5e5; - -webkit-border-radius: 4px; - -moz-border-radius: 4px; - border-radius: 4px; -} -.accordion-heading { - border-bottom: 0; -} -.accordion-heading .accordion-toggle { - display: block; - padding: 8px 15px; -} -.accordion-toggle { - cursor: pointer; -} -.accordion-inner { - padding: 9px 15px; - border-top: 1px solid #e5e5e5; -} -.carousel { - position: relative; - margin-bottom: 20px; - line-height: 1; -} -.carousel-inner { - overflow: hidden; - width: 100%; - position: relative; -} -.carousel-inner > .item { - display: none; - position: relative; - -webkit-transition: 0.6s ease-in-out left; - -moz-transition: 0.6s ease-in-out left; - -o-transition: 0.6s ease-in-out left; - transition: 0.6s ease-in-out left; -} -.carousel-inner > .item > img, -.carousel-inner > .item > a > img { - display: block; - line-height: 1; -} -.carousel-inner > .active, -.carousel-inner > .next, -.carousel-inner > .prev { - display: block; -} -.carousel-inner > .active { - left: 0; -} -.carousel-inner > .next, -.carousel-inner > .prev { - position: absolute; - top: 0; - width: 100%; -} -.carousel-inner > .next { - left: 100%; -} -.carousel-inner > .prev { - left: -100%; -} -.carousel-inner > .next.left, -.carousel-inner > .prev.right { - left: 0; -} -.carousel-inner > .active.left { - left: -100%; -} -.carousel-inner > .active.right { - left: 100%; -} -.carousel-control { - position: absolute; - top: 40%; - left: 15px; - width: 40px; - height: 40px; - margin-top: -20px; - font-size: 60px; - font-weight: 100; - line-height: 30px; - color: #ffffff; - text-align: center; - background: #222222; - border: 3px solid #ffffff; - -webkit-border-radius: 23px; - -moz-border-radius: 23px; - border-radius: 23px; - opacity: 0.5; - filter: alpha(opacity=50); -} -.carousel-control.right { - left: auto; - right: 15px; -} -.carousel-control:hover, -.carousel-control:focus { - color: #ffffff; - text-decoration: none; - opacity: 0.9; - filter: alpha(opacity=90); -} -.carousel-indicators { - position: absolute; - top: 15px; - right: 15px; - z-index: 5; - margin: 0; - list-style: none; -} -.carousel-indicators li { - display: block; - float: left; - width: 10px; - height: 10px; - margin-left: 5px; - text-indent: -999px; - background-color: #ccc; - background-color: rgba(255, 255, 255, 0.25); - border-radius: 5px; -} -.carousel-indicators .active { - background-color: #fff; -} -.carousel-caption { - position: absolute; - left: 0; - right: 0; - bottom: 0; - padding: 15px; - background: #333333; - background: rgba(0, 0, 0, 0.75); -} -.carousel-caption h4, -.carousel-caption p { - color: #ffffff; - line-height: 20px; -} -.carousel-caption h4 { - margin: 0 0 5px; -} -.carousel-caption p { - margin-bottom: 0; -} -.well { - min-height: 20px; - padding: 19px; - margin-bottom: 20px; - background-color: #f5f5f5; - border: 1px solid #e3e3e3; - -webkit-border-radius: 4px; - -moz-border-radius: 4px; - border-radius: 4px; - -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05); - -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05); - box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05); -} -.well blockquote { - border-color: #ddd; - border-color: rgba(0, 0, 0, 0.15); -} -.well-large { - padding: 24px; - -webkit-border-radius: 6px; - -moz-border-radius: 6px; - border-radius: 6px; -} -.well-small { - padding: 9px; - -webkit-border-radius: 3px; - -moz-border-radius: 3px; - border-radius: 3px; -} -.close { - float: right; - font-size: 20px; - font-weight: bold; - line-height: 20px; - color: #000000; - text-shadow: 0 1px 0 #ffffff; - opacity: 0.2; - filter: alpha(opacity=20); -} -.close:hover, -.close:focus { - color: #000000; - text-decoration: none; - cursor: pointer; - opacity: 0.4; - filter: alpha(opacity=40); -} -button.close { - padding: 0; - cursor: pointer; - background: transparent; - border: 0; - -webkit-appearance: none; -} -.pull-right { - float: right; -} -.pull-left { - float: left; -} -.hide { - display: none; -} -.show { - display: block; -} -.invisible { - visibility: hidden; -} -.affix { - position: fixed; -} -.fade { - opacity: 0; - -webkit-transition: opacity 0.15s linear; - -moz-transition: opacity 0.15s linear; - -o-transition: opacity 0.15s linear; - transition: opacity 0.15s linear; -} -.fade.in { - opacity: 1; -} -.collapse { - position: relative; - height: 0; - overflow: hidden; - -webkit-transition: height 0.35s ease; - -moz-transition: height 0.35s ease; - -o-transition: height 0.35s ease; - transition: height 0.35s ease; -} -.collapse.in { - height: auto; -} -@-ms-viewport { - width: device-width; -} -.hidden { - display: none; - visibility: hidden; -} -.visible-phone { - display: none !important; -} -.visible-tablet { - display: none !important; -} -.hidden-desktop { - display: none !important; -} -.visible-desktop { - display: inherit !important; -} -@media (min-width: 768px) and (max-width: 979px) { - .hidden-desktop { - display: inherit !important; - } - .visible-desktop { - display: none !important ; - } - .visible-tablet { - display: inherit !important; - } - .hidden-tablet { - display: none !important; - } -} -@media (max-width: 767px) { - .hidden-desktop { - display: inherit !important; - } - .visible-desktop { - display: none !important; - } - .visible-phone { - display: inherit !important; - } - .hidden-phone { - display: none !important; - } -} -.visible-print { - display: none !important; -} -@media print { - .visible-print { - display: inherit !important; - } - .hidden-print { - display: none !important; - } -} -@media (max-width: 767px) { - body { - padding-left: 20px; - padding-right: 20px; - } - .navbar-fixed-top, - .navbar-fixed-bottom, - .navbar-static-top { - margin-left: -20px; - margin-right: -20px; - } - .container-fluid { - padding: 0; - } - .dl-horizontal dt { - float: none; - clear: none; - width: auto; - text-align: left; - } - .dl-horizontal dd { - margin-left: 0; - } - .container { - width: auto; - } - .row-fluid { - width: 100%; - } - .row, - .thumbnails { - margin-left: 0; - } - .thumbnails > li { - float: none; - margin-left: 0; - } - [class*="span"], - .uneditable-input[class*="span"], - .row-fluid [class*="span"] { - float: none; - display: block; - width: 100%; - margin-left: 0; - -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - box-sizing: border-box; - } - .span12, - .row-fluid .span12 { - width: 100%; - -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - box-sizing: border-box; - } - .row-fluid [class*="offset"]:first-child { - margin-left: 0; - } - .input-large, - .input-xlarge, - .input-xxlarge, - input[class*="span"], - select[class*="span"], - textarea[class*="span"], - .uneditable-input { - display: block; - width: 100%; - min-height: 30px; - -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - box-sizing: border-box; - } - .input-prepend input, - .input-append input, - .input-prepend input[class*="span"], - .input-append input[class*="span"] { - display: inline-block; - width: auto; - } - .controls-row [class*="span"] + [class*="span"] { - margin-left: 0; - } - .modal { - position: fixed; - top: 20px; - left: 20px; - right: 20px; - width: auto; - margin: 0; - } - .modal.fade { - top: -100px; - } - .modal.fade.in { - top: 20px; - } -} -@media (max-width: 480px) { - .nav-collapse { - -webkit-transform: translate3d(0, 0, 0); - } - .page-header h1 small { - display: block; - line-height: 20px; - } - input[type="checkbox"], - input[type="radio"] { - border: 1px solid #ccc; - } - .form-horizontal .control-label { - float: none; - width: auto; - padding-top: 0; - text-align: left; - } - .form-horizontal .controls { - margin-left: 0; - } - .form-horizontal .control-list { - padding-top: 0; - } - .form-horizontal .form-actions { - padding-left: 10px; - padding-right: 10px; - } - .media .pull-left, - .media .pull-right { - float: none; - display: block; - margin-bottom: 10px; - } - .media-object { - margin-right: 0; - margin-left: 0; - } - .modal { - top: 10px; - left: 10px; - right: 10px; - } - .modal-header .close { - padding: 10px; - margin: -10px; - } - .carousel-caption { - position: static; - } -} -@media (min-width: 768px) and (max-width: 979px) { - .row { - margin-left: -20px; - *zoom: 1; - } - .row:before, - .row:after { - display: table; - content: ""; - line-height: 0; - } - .row:after { - clear: both; - } - [class*="span"] { - float: left; - min-height: 1px; - margin-left: 20px; - } - .container, - .navbar-static-top .container, - .navbar-fixed-top .container, - .navbar-fixed-bottom .container { - width: 724px; - } - .span12 { - width: 724px; - } - .span11 { - width: 662px; - } - .span10 { - width: 600px; - } - .span9 { - width: 538px; - } - .span8 { - width: 476px; - } - .span7 { - width: 414px; - } - .span6 { - width: 352px; - } - .span5 { - width: 290px; - } - .span4 { - width: 228px; - } - .span3 { - width: 166px; - } - .span2 { - width: 104px; - } - .span1 { - width: 42px; - } - .offset12 { - margin-left: 764px; - } - .offset11 { - margin-left: 702px; - } - .offset10 { - margin-left: 640px; - } - .offset9 { - margin-left: 578px; - } - .offset8 { - margin-left: 516px; - } - .offset7 { - margin-left: 454px; - } - .offset6 { - margin-left: 392px; - } - .offset5 { - margin-left: 330px; - } - .offset4 { - margin-left: 268px; - } - .offset3 { - margin-left: 206px; - } - .offset2 { - margin-left: 144px; - } - .offset1 { - margin-left: 82px; - } - .row-fluid { - width: 100%; - *zoom: 1; - } - .row-fluid:before, - .row-fluid:after { - display: table; - content: ""; - line-height: 0; - } - .row-fluid:after { - clear: both; - } - .row-fluid [class*="span"] { - display: block; - width: 100%; - min-height: 30px; - -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - box-sizing: border-box; - float: left; - margin-left: 2.7624309392265194%; - *margin-left: 2.709239449864817%; - } - .row-fluid [class*="span"]:first-child { - margin-left: 0; - } - .row-fluid .controls-row [class*="span"] + [class*="span"] { - margin-left: 2.7624309392265194%; - } - .row-fluid .span12 { - width: 100%; - *width: 99.94680851063829%; - } - .row-fluid .span11 { - width: 91.43646408839778%; - *width: 91.38327259903608%; - } - .row-fluid .span10 { - width: 82.87292817679558%; - *width: 82.81973668743387%; - } - .row-fluid .span9 { - width: 74.30939226519337%; - *width: 74.25620077583166%; - } - .row-fluid .span8 { - width: 65.74585635359117%; - *width: 65.69266486422946%; - } - .row-fluid .span7 { - width: 57.18232044198895%; - *width: 57.12912895262725%; - } - .row-fluid .span6 { - width: 48.61878453038674%; - *width: 48.56559304102504%; - } - .row-fluid .span5 { - width: 40.05524861878453%; - *width: 40.00205712942283%; - } - .row-fluid .span4 { - width: 31.491712707182323%; - *width: 31.43852121782062%; - } - .row-fluid .span3 { - width: 22.92817679558011%; - *width: 22.87498530621841%; - } - .row-fluid .span2 { - width: 14.3646408839779%; - *width: 14.311449394616199%; - } - .row-fluid .span1 { - width: 5.801104972375691%; - *width: 5.747913483013988%; - } - .row-fluid .offset12 { - margin-left: 105.52486187845304%; - *margin-left: 105.41847889972962%; - } - .row-fluid .offset12:first-child { - margin-left: 102.76243093922652%; - *margin-left: 102.6560479605031%; - } - .row-fluid .offset11 { - margin-left: 96.96132596685082%; - *margin-left: 96.8549429881274%; - } - .row-fluid .offset11:first-child { - margin-left: 94.1988950276243%; - *margin-left: 94.09251204890089%; - } - .row-fluid .offset10 { - margin-left: 88.39779005524862%; - *margin-left: 88.2914070765252%; - } - .row-fluid .offset10:first-child { - margin-left: 85.6353591160221%; - *margin-left: 85.52897613729868%; - } - .row-fluid .offset9 { - margin-left: 79.8342541436464%; - *margin-left: 79.72787116492299%; - } - .row-fluid .offset9:first-child { - margin-left: 77.07182320441989%; - *margin-left: 76.96544022569647%; - } - .row-fluid .offset8 { - margin-left: 71.2707182320442%; - *margin-left: 71.16433525332079%; - } - .row-fluid .offset8:first-child { - margin-left: 68.50828729281768%; - *margin-left: 68.40190431409427%; - } - .row-fluid .offset7 { - margin-left: 62.70718232044199%; - *margin-left: 62.600799341718584%; - } - .row-fluid .offset7:first-child { - margin-left: 59.94475138121547%; - *margin-left: 59.838368402492065%; - } - .row-fluid .offset6 { - margin-left: 54.14364640883978%; - *margin-left: 54.037263430116376%; - } - .row-fluid .offset6:first-child { - margin-left: 51.38121546961326%; - *margin-left: 51.27483249088986%; - } - .row-fluid .offset5 { - margin-left: 45.58011049723757%; - *margin-left: 45.47372751851417%; - } - .row-fluid .offset5:first-child { - margin-left: 42.81767955801105%; - *margin-left: 42.71129657928765%; - } - .row-fluid .offset4 { - margin-left: 37.01657458563536%; - *margin-left: 36.91019160691196%; - } - .row-fluid .offset4:first-child { - margin-left: 34.25414364640884%; - *margin-left: 34.14776066768544%; - } - .row-fluid .offset3 { - margin-left: 28.45303867403315%; - *margin-left: 28.346655695309746%; - } - .row-fluid .offset3:first-child { - margin-left: 25.69060773480663%; - *margin-left: 25.584224756083227%; - } - .row-fluid .offset2 { - margin-left: 19.88950276243094%; - *margin-left: 19.783119783707537%; - } - .row-fluid .offset2:first-child { - margin-left: 17.12707182320442%; - *margin-left: 17.02068884448102%; - } - .row-fluid .offset1 { - margin-left: 11.32596685082873%; - *margin-left: 11.219583872105325%; - } - .row-fluid .offset1:first-child { - margin-left: 8.56353591160221%; - *margin-left: 8.457152932878806%; - } - input, - textarea, - .uneditable-input { - margin-left: 0; - } - .controls-row [class*="span"] + [class*="span"] { - margin-left: 20px; - } - input.span12, - textarea.span12, - .uneditable-input.span12 { - width: 710px; - } - input.span11, - textarea.span11, - .uneditable-input.span11 { - width: 648px; - } - input.span10, - textarea.span10, - .uneditable-input.span10 { - width: 586px; - } - input.span9, - textarea.span9, - .uneditable-input.span9 { - width: 524px; - } - input.span8, - textarea.span8, - .uneditable-input.span8 { - width: 462px; - } - input.span7, - textarea.span7, - .uneditable-input.span7 { - width: 400px; - } - input.span6, - textarea.span6, - .uneditable-input.span6 { - width: 338px; - } - input.span5, - textarea.span5, - .uneditable-input.span5 { - width: 276px; - } - input.span4, - textarea.span4, - .uneditable-input.span4 { - width: 214px; - } - input.span3, - textarea.span3, - .uneditable-input.span3 { - width: 152px; - } - input.span2, - textarea.span2, - .uneditable-input.span2 { - width: 90px; - } - input.span1, - textarea.span1, - .uneditable-input.span1 { - width: 28px; - } -} -@media (min-width: 1200px) { - .row { - margin-left: -30px; - *zoom: 1; - } - .row:before, - .row:after { - display: table; - content: ""; - line-height: 0; - } - .row:after { - clear: both; - } - [class*="span"] { - float: left; - min-height: 1px; - margin-left: 30px; - } - .container, - .navbar-static-top .container, - .navbar-fixed-top .container, - .navbar-fixed-bottom .container { - width: 1170px; - } - .span12 { - width: 1170px; - } - .span11 { - width: 1070px; - } - .span10 { - width: 970px; - } - .span9 { - width: 870px; - } - .span8 { - width: 770px; - } - .span7 { - width: 670px; - } - .span6 { - width: 570px; - } - .span5 { - width: 470px; - } - .span4 { - width: 370px; - } - .span3 { - width: 270px; - } - .span2 { - width: 170px; - } - .span1 { - width: 70px; - } - .offset12 { - margin-left: 1230px; - } - .offset11 { - margin-left: 1130px; - } - .offset10 { - margin-left: 1030px; - } - .offset9 { - margin-left: 930px; - } - .offset8 { - margin-left: 830px; - } - .offset7 { - margin-left: 730px; - } - .offset6 { - margin-left: 630px; - } - .offset5 { - margin-left: 530px; - } - .offset4 { - margin-left: 430px; - } - .offset3 { - margin-left: 330px; - } - .offset2 { - margin-left: 230px; - } - .offset1 { - margin-left: 130px; - } - .row-fluid { - width: 100%; - *zoom: 1; - } - .row-fluid:before, - .row-fluid:after { - display: table; - content: ""; - line-height: 0; - } - .row-fluid:after { - clear: both; - } - .row-fluid [class*="span"] { - display: block; - width: 100%; - min-height: 30px; - -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - box-sizing: border-box; - float: left; - margin-left: 2.564102564102564%; - *margin-left: 2.5109110747408616%; - } - .row-fluid [class*="span"]:first-child { - margin-left: 0; - } - .row-fluid .controls-row [class*="span"] + [class*="span"] { - margin-left: 2.564102564102564%; - } - .row-fluid .span12 { - width: 100%; - *width: 99.94680851063829%; - } - .row-fluid .span11 { - width: 91.45299145299145%; - *width: 91.39979996362975%; - } - .row-fluid .span10 { - width: 82.90598290598291%; - *width: 82.8527914166212%; - } - .row-fluid .span9 { - width: 74.35897435897436%; - *width: 74.30578286961266%; - } - .row-fluid .span8 { - width: 65.81196581196582%; - *width: 65.75877432260411%; - } - .row-fluid .span7 { - width: 57.26495726495726%; - *width: 57.21176577559556%; - } - .row-fluid .span6 { - width: 48.717948717948715%; - *width: 48.664757228587014%; - } - .row-fluid .span5 { - width: 40.17094017094017%; - *width: 40.11774868157847%; - } - .row-fluid .span4 { - width: 31.623931623931625%; - *width: 31.570740134569924%; - } - .row-fluid .span3 { - width: 23.076923076923077%; - *width: 23.023731587561375%; - } - .row-fluid .span2 { - width: 14.52991452991453%; - *width: 14.476723040552828%; - } - .row-fluid .span1 { - width: 5.982905982905983%; - *width: 5.929714493544281%; - } - .row-fluid .offset12 { - margin-left: 105.12820512820512%; - *margin-left: 105.02182214948171%; - } - .row-fluid .offset12:first-child { - margin-left: 102.56410256410257%; - *margin-left: 102.45771958537915%; - } - .row-fluid .offset11 { - margin-left: 96.58119658119658%; - *margin-left: 96.47481360247316%; - } - .row-fluid .offset11:first-child { - margin-left: 94.01709401709402%; - *margin-left: 93.91071103837061%; - } - .row-fluid .offset10 { - margin-left: 88.03418803418803%; - *margin-left: 87.92780505546462%; - } - .row-fluid .offset10:first-child { - margin-left: 85.47008547008548%; - *margin-left: 85.36370249136206%; - } - .row-fluid .offset9 { - margin-left: 79.48717948717949%; - *margin-left: 79.38079650845607%; - } - .row-fluid .offset9:first-child { - margin-left: 76.92307692307693%; - *margin-left: 76.81669394435352%; - } - .row-fluid .offset8 { - margin-left: 70.94017094017094%; - *margin-left: 70.83378796144753%; - } - .row-fluid .offset8:first-child { - margin-left: 68.37606837606839%; - *margin-left: 68.26968539734497%; - } - .row-fluid .offset7 { - margin-left: 62.393162393162385%; - *margin-left: 62.28677941443899%; - } - .row-fluid .offset7:first-child { - margin-left: 59.82905982905982%; - *margin-left: 59.72267685033642%; - } - .row-fluid .offset6 { - margin-left: 53.84615384615384%; - *margin-left: 53.739770867430444%; - } - .row-fluid .offset6:first-child { - margin-left: 51.28205128205128%; - *margin-left: 51.175668303327875%; - } - .row-fluid .offset5 { - margin-left: 45.299145299145295%; - *margin-left: 45.1927623204219%; - } - .row-fluid .offset5:first-child { - margin-left: 42.73504273504273%; - *margin-left: 42.62865975631933%; - } - .row-fluid .offset4 { - margin-left: 36.75213675213675%; - *margin-left: 36.645753773413354%; - } - .row-fluid .offset4:first-child { - margin-left: 34.18803418803419%; - *margin-left: 34.081651209310785%; - } - .row-fluid .offset3 { - margin-left: 28.205128205128204%; - *margin-left: 28.0987452264048%; - } - .row-fluid .offset3:first-child { - margin-left: 25.641025641025642%; - *margin-left: 25.53464266230224%; - } - .row-fluid .offset2 { - margin-left: 19.65811965811966%; - *margin-left: 19.551736679396257%; - } - .row-fluid .offset2:first-child { - margin-left: 17.094017094017094%; - *margin-left: 16.98763411529369%; - } - .row-fluid .offset1 { - margin-left: 11.11111111111111%; - *margin-left: 11.004728132387708%; - } - .row-fluid .offset1:first-child { - margin-left: 8.547008547008547%; - *margin-left: 8.440625568285142%; - } - input, - textarea, - .uneditable-input { - margin-left: 0; - } - .controls-row [class*="span"] + [class*="span"] { - margin-left: 30px; - } - input.span12, - textarea.span12, - .uneditable-input.span12 { - width: 1156px; - } - input.span11, - textarea.span11, - .uneditable-input.span11 { - width: 1056px; - } - input.span10, - textarea.span10, - .uneditable-input.span10 { - width: 956px; - } - input.span9, - textarea.span9, - .uneditable-input.span9 { - width: 856px; - } - input.span8, - textarea.span8, - .uneditable-input.span8 { - width: 756px; - } - input.span7, - textarea.span7, - .uneditable-input.span7 { - width: 656px; - } - input.span6, - textarea.span6, - .uneditable-input.span6 { - width: 556px; - } - input.span5, - textarea.span5, - .uneditable-input.span5 { - width: 456px; - } - input.span4, - textarea.span4, - .uneditable-input.span4 { - width: 356px; - } - input.span3, - textarea.span3, - .uneditable-input.span3 { - width: 256px; - } - input.span2, - textarea.span2, - .uneditable-input.span2 { - width: 156px; - } - input.span1, - textarea.span1, - .uneditable-input.span1 { - width: 56px; - } - .thumbnails { - margin-left: -30px; - } - .thumbnails > li { - margin-left: 30px; - } - .row-fluid .thumbnails { - margin-left: 0; - } -} -@media (max-width: 979px) { - body { - padding-top: 0; - } - .navbar-fixed-top, - .navbar-fixed-bottom { - position: static; - } - .navbar-fixed-top { - margin-bottom: 20px; - } - .navbar-fixed-bottom { - margin-top: 20px; - } - .navbar-fixed-top .navbar-inner, - .navbar-fixed-bottom .navbar-inner { - padding: 5px; - } - .navbar .container { - width: auto; - padding: 0; - } - .navbar .brand { - padding-left: 10px; - padding-right: 10px; - margin: 0 0 0 -5px; - } - .nav-collapse { - clear: both; - } - .nav-collapse .nav { - float: none; - margin: 0 0 10px; - } - .nav-collapse .nav > li { - float: none; - } - .nav-collapse .nav > li > a { - margin-bottom: 2px; - } - .nav-collapse .nav > .divider-vertical { - display: none; - } - .nav-collapse .nav .nav-header { - color: #777777; - text-shadow: none; - } - .nav-collapse .nav > li > a, - .nav-collapse .dropdown-menu a { - padding: 9px 15px; - font-weight: bold; - color: #777777; - -webkit-border-radius: 3px; - -moz-border-radius: 3px; - border-radius: 3px; - } - .nav-collapse .btn { - padding: 4px 10px 4px; - font-weight: normal; - -webkit-border-radius: 4px; - -moz-border-radius: 4px; - border-radius: 4px; - } - .nav-collapse .dropdown-menu li + li a { - margin-bottom: 2px; - } - .nav-collapse .nav > li > a:hover, - .nav-collapse .nav > li > a:focus, - .nav-collapse .dropdown-menu a:hover, - .nav-collapse .dropdown-menu a:focus { - background-color: #f2f2f2; - } - .navbar-inverse .nav-collapse .nav > li > a, - .navbar-inverse .nav-collapse .dropdown-menu a { - color: #999999; - } - .navbar-inverse .nav-collapse .nav > li > a:hover, - .navbar-inverse .nav-collapse .nav > li > a:focus, - .navbar-inverse .nav-collapse .dropdown-menu a:hover, - .navbar-inverse .nav-collapse .dropdown-menu a:focus { - background-color: #111111; - } - .nav-collapse.in .btn-group { - margin-top: 5px; - padding: 0; - } - .nav-collapse .dropdown-menu { - position: static; - top: auto; - left: auto; - float: none; - display: none; - max-width: none; - margin: 0 15px; - padding: 0; - background-color: transparent; - border: none; - -webkit-border-radius: 0; - -moz-border-radius: 0; - border-radius: 0; - -webkit-box-shadow: none; - -moz-box-shadow: none; - box-shadow: none; - } - .nav-collapse .open > .dropdown-menu { - display: block; - } - .nav-collapse .dropdown-menu:before, - .nav-collapse .dropdown-menu:after { - display: none; - } - .nav-collapse .dropdown-menu .divider { - display: none; - } - .nav-collapse .nav > li > .dropdown-menu:before, - .nav-collapse .nav > li > .dropdown-menu:after { - display: none; - } - .nav-collapse .navbar-form, - .nav-collapse .navbar-search { - float: none; - padding: 10px 15px; - margin: 10px 0; - border-top: 1px solid #f2f2f2; - border-bottom: 1px solid #f2f2f2; - -webkit-box-shadow: inset 0 1px 0 rgba(255,255,255,.1), 0 1px 0 rgba(255,255,255,.1); - -moz-box-shadow: inset 0 1px 0 rgba(255,255,255,.1), 0 1px 0 rgba(255,255,255,.1); - box-shadow: inset 0 1px 0 rgba(255,255,255,.1), 0 1px 0 rgba(255,255,255,.1); - } - .navbar-inverse .nav-collapse .navbar-form, - .navbar-inverse .nav-collapse .navbar-search { - border-top-color: #111111; - border-bottom-color: #111111; - } - .navbar .nav-collapse .nav.pull-right { - float: none; - margin-left: 0; - } - .nav-collapse, - .nav-collapse.collapse { - overflow: hidden; - height: 0; - } - .navbar .btn-navbar { - display: block; - } - .navbar-static .navbar-inner { - padding-left: 10px; - padding-right: 10px; - } -} -@media (min-width: 980px) { - .nav-collapse.collapse { - height: auto !important; - overflow: visible !important; - } -} diff --git a/doc/themes/scikit-learn/static/css/bootstrap.min.css b/doc/themes/scikit-learn/static/css/bootstrap.min.css deleted file mode 100644 index 0243215b66555..0000000000000 --- a/doc/themes/scikit-learn/static/css/bootstrap.min.css +++ /dev/null @@ -1,858 +0,0 @@ -/*! - * Bootstrap v2.3.2 - * - * Copyright 2012 Twitter, Inc - * Licensed under the Apache License v2.0 - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Designed and built with all the love in the world @twitter by @mdo and @fat. - */ -.clearfix{*zoom:1;}.clearfix:before,.clearfix:after{display:table;content:"";line-height:0;} -.clearfix:after{clear:both;} -.hide-text{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0;} -.input-block-level{display:block;width:100%;min-height:30px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;} -article,aside,details,figcaption,figure,footer,header,hgroup,nav,section{display:block;} -audio,canvas,video{display:inline-block;*display:inline;*zoom:1;} -audio:not([controls]){display:none;} -html{font-size:100%;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%;} -a:focus{outline:thin dotted #333;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px;} -a:hover,a:active{outline:0;} -sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline;} -sup{top:-0.5em;} -sub{bottom:-0.25em;} -img{max-width:100%;width:auto\9;height:auto;vertical-align:middle;border:0;-ms-interpolation-mode:bicubic;} -#map_canvas img,.google-maps img{max-width:none;} -button,input,select,textarea{margin:0;font-size:100%;vertical-align:middle;} -button,input{*overflow:visible;line-height:normal;} -button::-moz-focus-inner,input::-moz-focus-inner{padding:0;border:0;} -button,html input[type="button"],input[type="reset"],input[type="submit"]{-webkit-appearance:button;cursor:pointer;} -label,select,button,input[type="button"],input[type="reset"],input[type="submit"],input[type="radio"],input[type="checkbox"]{cursor:pointer;} -input[type="search"]{-webkit-box-sizing:content-box;-moz-box-sizing:content-box;box-sizing:content-box;-webkit-appearance:textfield;} -input[type="search"]::-webkit-search-decoration,input[type="search"]::-webkit-search-cancel-button{-webkit-appearance:none;} -textarea{overflow:auto;vertical-align:top;} -.img-rounded{-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px;} -.img-polaroid{padding:4px;background-color:#fff;border:1px solid #ccc;border:1px solid rgba(0, 0, 0, 0.2);-webkit-box-shadow:0 1px 3px rgba(0, 0, 0, 0.1);-moz-box-shadow:0 1px 3px rgba(0, 0, 0, 0.1);box-shadow:0 1px 3px rgba(0, 0, 0, 0.1);} -.img-circle{-webkit-border-radius:500px;-moz-border-radius:500px;border-radius:500px;} -.row{margin-left:-20px;*zoom:1;}.row:before,.row:after{display:table;content:"";line-height:0;} -.row:after{clear:both;} -[class*="span"]{float:left;min-height:1px;margin-left:20px;} -.container,.navbar-static-top .container,.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:940px;} -.span12{width:940px;} -.span11{width:860px;} -.span10{width:780px;} -.span9{width:700px;} -.span8{width:620px;} -.span7{width:540px;} -.span6{width:460px;} -.span5{width:380px;} -.span4{width:300px;} -.span3{width:220px;} -.span2{width:140px;} -.span1{width:60px;} -.offset12{margin-left:980px;} -.offset11{margin-left:900px;} -.offset10{margin-left:820px;} -.offset9{margin-left:740px;} -.offset8{margin-left:660px;} -.offset7{margin-left:580px;} -.offset6{margin-left:500px;} -.offset5{margin-left:420px;} -.offset4{margin-left:340px;} -.offset3{margin-left:260px;} -.offset2{margin-left:180px;} -.offset1{margin-left:100px;} -.row-fluid{width:100%;*zoom:1;}.row-fluid:before,.row-fluid:after{display:table;content:"";line-height:0;} -.row-fluid:after{clear:both;} -.row-fluid [class*="span"]{display:block;width:100%;min-height:30px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;float:left;margin-left:2.127659574468085%;*margin-left:2.074468085106383%;} -.row-fluid [class*="span"]:first-child{margin-left:0;} -.row-fluid .controls-row [class*="span"]+[class*="span"]{margin-left:2.127659574468085%;} -.row-fluid .span12{width:100%;*width:99.94680851063829%;} -.row-fluid .span11{width:91.48936170212765%;*width:91.43617021276594%;} -.row-fluid .span10{width:82.97872340425532%;*width:82.92553191489361%;} -.row-fluid .span9{width:74.46808510638297%;*width:74.41489361702126%;} -.row-fluid .span8{width:65.95744680851064%;*width:65.90425531914893%;} -.row-fluid .span7{width:57.44680851063829%;*width:57.39361702127659%;} -.row-fluid .span6{width:48.93617021276595%;*width:48.88297872340425%;} -.row-fluid .span5{width:40.42553191489362%;*width:40.37234042553192%;} -.row-fluid .span4{width:31.914893617021278%;*width:31.861702127659576%;} -.row-fluid .span3{width:23.404255319148934%;*width:23.351063829787233%;} -.row-fluid .span2{width:14.893617021276595%;*width:14.840425531914894%;} -.row-fluid .span1{width:6.382978723404255%;*width:6.329787234042553%;} -.row-fluid .offset12{margin-left:104.25531914893617%;*margin-left:104.14893617021275%;} -.row-fluid .offset12:first-child{margin-left:102.12765957446808%;*margin-left:102.02127659574467%;} -.row-fluid .offset11{margin-left:95.74468085106382%;*margin-left:95.6382978723404%;} -.row-fluid .offset11:first-child{margin-left:93.61702127659574%;*margin-left:93.51063829787232%;} -.row-fluid .offset10{margin-left:87.23404255319149%;*margin-left:87.12765957446807%;} -.row-fluid .offset10:first-child{margin-left:85.1063829787234%;*margin-left:84.99999999999999%;} -.row-fluid .offset9{margin-left:78.72340425531914%;*margin-left:78.61702127659572%;} -.row-fluid .offset9:first-child{margin-left:76.59574468085106%;*margin-left:76.48936170212764%;} -.row-fluid .offset8{margin-left:70.2127659574468%;*margin-left:70.10638297872339%;} -.row-fluid .offset8:first-child{margin-left:68.08510638297872%;*margin-left:67.9787234042553%;} -.row-fluid .offset7{margin-left:61.70212765957446%;*margin-left:61.59574468085106%;} -.row-fluid .offset7:first-child{margin-left:59.574468085106375%;*margin-left:59.46808510638297%;} -.row-fluid .offset6{margin-left:53.191489361702125%;*margin-left:53.085106382978715%;} -.row-fluid .offset6:first-child{margin-left:51.063829787234035%;*margin-left:50.95744680851063%;} -.row-fluid .offset5{margin-left:44.68085106382979%;*margin-left:44.57446808510638%;} -.row-fluid .offset5:first-child{margin-left:42.5531914893617%;*margin-left:42.4468085106383%;} -.row-fluid .offset4{margin-left:36.170212765957444%;*margin-left:36.06382978723405%;} -.row-fluid .offset4:first-child{margin-left:34.04255319148936%;*margin-left:33.93617021276596%;} -.row-fluid .offset3{margin-left:27.659574468085104%;*margin-left:27.5531914893617%;} -.row-fluid .offset3:first-child{margin-left:25.53191489361702%;*margin-left:25.425531914893618%;} -.row-fluid .offset2{margin-left:19.148936170212764%;*margin-left:19.04255319148936%;} -.row-fluid .offset2:first-child{margin-left:17.02127659574468%;*margin-left:16.914893617021278%;} -.row-fluid .offset1{margin-left:10.638297872340425%;*margin-left:10.53191489361702%;} -.row-fluid .offset1:first-child{margin-left:8.51063829787234%;*margin-left:8.404255319148938%;} -[class*="span"].hide,.row-fluid [class*="span"].hide{display:none;} -[class*="span"].pull-right,.row-fluid [class*="span"].pull-right{float:right;} -.container{margin-right:auto;margin-left:auto;*zoom:1;}.container:before,.container:after{display:table;content:"";line-height:0;} -.container:after{clear:both;} -.container-fluid{padding-right:20px;padding-left:20px;*zoom:1;}.container-fluid:before,.container-fluid:after{display:table;content:"";line-height:0;} -.container-fluid:after{clear:both;} -p{margin:0 0 10px;} -.lead{margin-bottom:20px;font-size:21px;font-weight:200;line-height:30px;} -small{font-size:85%;} -strong{font-weight:bold;} -em{font-style:italic;} -cite{font-style:normal;} -.muted{color:#999999;} -a.muted:hover,a.muted:focus{color:#808080;} -.text-warning{color:#c09853;} -a.text-warning:hover,a.text-warning:focus{color:#a47e3c;} -.text-error{color:#b94a48;} -a.text-error:hover,a.text-error:focus{color:#953b39;} -.text-info{color:#3a87ad;} -a.text-info:hover,a.text-info:focus{color:#2d6987;} -.text-success{color:#468847;} -a.text-success:hover,a.text-success:focus{color:#356635;} -.text-left{text-align:left;} -.text-right{text-align:right;} -.text-center{text-align:center;} -.page-header{padding-bottom:9px;margin:20px 0 30px;border-bottom:1px solid #eeeeee;} -ul,ol{padding:0;margin:0 0 10px 25px;} -ul ul,ul ol,ol ol,ol ul{margin-bottom:0;} -li{line-height:20px;} -ul.unstyled,ol.unstyled{margin-left:0;list-style:none;} -ul.inline,ol.inline{margin-left:0;list-style:none;}ul.inline>li,ol.inline>li{display:inline-block;*display:inline;*zoom:1;padding-left:5px;padding-right:5px;} -dl{margin-bottom:20px;} -dt,dd{line-height:20px;} -dt{font-weight:bold;} -dd{margin-left:10px;} -.dl-horizontal{*zoom:1;}.dl-horizontal:before,.dl-horizontal:after{display:table;content:"";line-height:0;} -.dl-horizontal:after{clear:both;} -.dl-horizontal dt{float:left;width:160px;clear:left;text-align:right;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;} -.dl-horizontal dd{margin-left:180px;} -hr{margin:20px 0;border:0;border-top:1px solid #eeeeee;border-bottom:1px solid #ffffff;} -abbr[title],abbr[data-original-title]{cursor:help;border-bottom:1px dotted #999999;} -abbr.initialism{font-size:90%;text-transform:uppercase;} -blockquote small{display:block;line-height:20px;color:#999999;}blockquote small:before{content:'\2014 \00A0';} -blockquote.pull-right{float:right;padding-right:15px;padding-left:0;border-right:5px solid #eeeeee;border-left:0;}blockquote.pull-right p,blockquote.pull-right small{text-align:right;} -blockquote.pull-right small:before{content:'';} -blockquote.pull-right small:after{content:'\00A0 \2014';} -q:before,q:after,blockquote:before,blockquote:after{content:"";} -address{display:block;margin-bottom:20px;font-style:normal;line-height:20px;} -code,pre{padding:0 3px 2px;font-family:Monaco,Menlo,Consolas,"Courier New",monospace;font-size:12px;color:#333333;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px;} -code{padding:2px 4px;color:#d14;background-color:#f7f7f9;border:1px solid #e1e1e8;white-space:nowrap;} -pre{display:block;padding:9.5px;margin:0 0 10px;font-size:13px;line-height:20px;word-break:break-all;word-wrap:break-word;white-space:pre;white-space:pre-wrap;background-color:#f5f5f5;border:1px solid #ccc;border:1px solid rgba(0, 0, 0, 0.15);-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;}pre.prettyprint{margin-bottom:20px;} -pre code{padding:0;color:inherit;white-space:pre;white-space:pre-wrap;background-color:transparent;border:0;} -.pre-scrollable{max-height:340px;overflow-y:scroll;} -.label,.badge{display:inline-block;padding:2px 4px;font-size:11.844px;font-weight:bold;line-height:14px;color:#ffffff;vertical-align:baseline;white-space:nowrap;text-shadow:0 -1px 0 rgba(0, 0, 0, 0.25);background-color:#999999;} -.label{-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px;} -.badge{padding-left:9px;padding-right:9px;-webkit-border-radius:9px;-moz-border-radius:9px;border-radius:9px;} -.label:empty,.badge:empty{display:none;} -a.label:hover,a.label:focus,a.badge:hover,a.badge:focus{color:#ffffff;text-decoration:none;cursor:pointer;} -.label-important,.badge-important{background-color:#b94a48;} -.label-important[href],.badge-important[href]{background-color:#953b39;} -.label-warning,.badge-warning{background-color:#f89406;} -.label-warning[href],.badge-warning[href]{background-color:#c67605;} -.label-danger,.badge-danger {/* XXX: backported from later bootstrap */background-color: #d9534f;} -.label-success,.badge-success{background-color:#468847;} -.label-success[href],.badge-success[href]{background-color:#356635;} -.label-info,.badge-info{background-color:#3a87ad;} -.label-info[href],.badge-info[href]{background-color:#2d6987;} -.label-inverse,.badge-inverse{background-color:#333333;} -.label-inverse[href],.badge-inverse[href]{background-color:#1a1a1a;} -.btn .label,.btn .badge{position:relative;top:-1px;} -.btn-mini .label,.btn-mini .badge{top:0;} -table{max-width:100%;background-color:transparent;border-collapse:collapse;border-spacing:0;} -.table{width:100%;margin-bottom:20px;}.table th,.table td{padding:8px;line-height:20px;text-align:left;vertical-align:top;border-top:1px solid #dddddd;} -.table th{font-weight:bold;} -.table thead th{vertical-align:bottom;} -.table caption+thead tr:first-child th,.table caption+thead tr:first-child td,.table colgroup+thead tr:first-child th,.table colgroup+thead tr:first-child td,.table thead:first-child tr:first-child th,.table thead:first-child tr:first-child td{border-top:0;} -.table tbody+tbody{border-top:2px solid #dddddd;} -.table .table{background-color:#ffffff;} -.table-condensed th,.table-condensed td{padding:4px 5px;} -.table-bordered{border:1px solid #dddddd;border-collapse:separate;*border-collapse:collapse;border-left:0;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;}.table-bordered th,.table-bordered td{border-left:1px solid #dddddd;} -.table-bordered caption+thead tr:first-child th,.table-bordered caption+tbody tr:first-child th,.table-bordered caption+tbody tr:first-child td,.table-bordered colgroup+thead tr:first-child th,.table-bordered colgroup+tbody tr:first-child th,.table-bordered colgroup+tbody tr:first-child td,.table-bordered thead:first-child tr:first-child th,.table-bordered tbody:first-child tr:first-child th,.table-bordered tbody:first-child tr:first-child td{border-top:0;} -.table-bordered thead:first-child tr:first-child>th:first-child,.table-bordered tbody:first-child tr:first-child>td:first-child,.table-bordered tbody:first-child tr:first-child>th:first-child{-webkit-border-top-left-radius:4px;-moz-border-radius-topleft:4px;border-top-left-radius:4px;} -.table-bordered thead:first-child tr:first-child>th:last-child,.table-bordered tbody:first-child tr:first-child>td:last-child,.table-bordered tbody:first-child tr:first-child>th:last-child{-webkit-border-top-right-radius:4px;-moz-border-radius-topright:4px;border-top-right-radius:4px;} -.table-bordered thead:last-child tr:last-child>th:first-child,.table-bordered tbody:last-child tr:last-child>td:first-child,.table-bordered tbody:last-child tr:last-child>th:first-child,.table-bordered tfoot:last-child tr:last-child>td:first-child,.table-bordered tfoot:last-child tr:last-child>th:first-child{-webkit-border-bottom-left-radius:4px;-moz-border-radius-bottomleft:4px;border-bottom-left-radius:4px;} -.table-bordered thead:last-child tr:last-child>th:last-child,.table-bordered tbody:last-child tr:last-child>td:last-child,.table-bordered tbody:last-child tr:last-child>th:last-child,.table-bordered tfoot:last-child tr:last-child>td:last-child,.table-bordered tfoot:last-child tr:last-child>th:last-child{-webkit-border-bottom-right-radius:4px;-moz-border-radius-bottomright:4px;border-bottom-right-radius:4px;} -.table-bordered tfoot+tbody:last-child tr:last-child td:first-child{-webkit-border-bottom-left-radius:0;-moz-border-radius-bottomleft:0;border-bottom-left-radius:0;} -.table-bordered tfoot+tbody:last-child tr:last-child td:last-child{-webkit-border-bottom-right-radius:0;-moz-border-radius-bottomright:0;border-bottom-right-radius:0;} -.table-bordered caption+thead tr:first-child th:first-child,.table-bordered caption+tbody tr:first-child td:first-child,.table-bordered colgroup+thead tr:first-child th:first-child,.table-bordered colgroup+tbody tr:first-child td:first-child{-webkit-border-top-left-radius:4px;-moz-border-radius-topleft:4px;border-top-left-radius:4px;} -.table-bordered caption+thead tr:first-child th:last-child,.table-bordered caption+tbody tr:first-child td:last-child,.table-bordered colgroup+thead tr:first-child th:last-child,.table-bordered colgroup+tbody tr:first-child td:last-child{-webkit-border-top-right-radius:4px;-moz-border-radius-topright:4px;border-top-right-radius:4px;} -.table-striped tbody>tr:nth-child(odd)>td,.table-striped tbody>tr:nth-child(odd)>th{background-color:#f9f9f9;} -.table-hover tbody tr:hover>td,.table-hover tbody tr:hover>th{background-color:#f5f5f5;} -table td[class*="span"],table th[class*="span"],.row-fluid table td[class*="span"],.row-fluid table th[class*="span"]{display:table-cell;float:none;margin-left:0;} -.table td.span1,.table th.span1{float:none;width:44px;margin-left:0;} -.table td.span2,.table th.span2{float:none;width:124px;margin-left:0;} -.table td.span3,.table th.span3{float:none;width:204px;margin-left:0;} -.table td.span4,.table th.span4{float:none;width:284px;margin-left:0;} -.table td.span5,.table th.span5{float:none;width:364px;margin-left:0;} -.table td.span6,.table th.span6{float:none;width:444px;margin-left:0;} -.table td.span7,.table th.span7{float:none;width:524px;margin-left:0;} -.table td.span8,.table th.span8{float:none;width:604px;margin-left:0;} -.table td.span9,.table th.span9{float:none;width:684px;margin-left:0;} -.table td.span10,.table th.span10{float:none;width:764px;margin-left:0;} -.table td.span11,.table th.span11{float:none;width:844px;margin-left:0;} -.table td.span12,.table th.span12{float:none;width:924px;margin-left:0;} -.table tbody tr.success>td{background-color:#dff0d8;} -.table tbody tr.error>td{background-color:#f2dede;} -.table tbody tr.warning>td{background-color:#fcf8e3;} -.table tbody tr.info>td{background-color:#d9edf7;} -.table-hover tbody tr.success:hover>td{background-color:#d0e9c6;} -.table-hover tbody tr.error:hover>td{background-color:#ebcccc;} -.table-hover tbody tr.warning:hover>td{background-color:#faf2cc;} -.table-hover tbody tr.info:hover>td{background-color:#c4e3f3;} -form{margin:0 0 20px;} -fieldset{padding:0;margin:0;border:0;} -legend{display:block;width:100%;padding:0;margin-bottom:20px;font-size:21px;line-height:40px;color:#333333;border:0;border-bottom:1px solid #e5e5e5;}legend small{font-size:15px;color:#999999;} -label,input,button,select,textarea{font-size:14px;font-weight:normal;line-height:20px;} -input,button,select,textarea{font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;} -label{display:block;margin-bottom:5px;} -select,textarea,input[type="text"],input[type="password"],input[type="datetime"],input[type="datetime-local"],input[type="date"],input[type="month"],input[type="time"],input[type="week"],input[type="number"],input[type="email"],input[type="url"],input[type="search"],input[type="tel"],input[type="color"],.uneditable-input{display:inline-block;height:20px;padding:4px 6px;margin-bottom:10px;font-size:14px;line-height:20px;color:#555555;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;vertical-align:middle;} -input,textarea,.uneditable-input{width:206px;} -textarea{height:auto;} -textarea,input[type="text"],input[type="password"],input[type="datetime"],input[type="datetime-local"],input[type="date"],input[type="month"],input[type="time"],input[type="week"],input[type="number"],input[type="email"],input[type="url"],input[type="search"],input[type="tel"],input[type="color"],.uneditable-input{background-color:#ffffff;border:1px solid #cccccc;-webkit-box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075);-moz-box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075);box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075);-webkit-transition:border linear .2s, box-shadow linear .2s;-moz-transition:border linear .2s, box-shadow linear .2s;-o-transition:border linear .2s, box-shadow linear .2s;transition:border linear .2s, box-shadow linear .2s;}textarea:focus,input[type="text"]:focus,input[type="password"]:focus,input[type="datetime"]:focus,input[type="datetime-local"]:focus,input[type="date"]:focus,input[type="month"]:focus,input[type="time"]:focus,input[type="week"]:focus,input[type="number"]:focus,input[type="email"]:focus,input[type="url"]:focus,input[type="search"]:focus,input[type="tel"]:focus,input[type="color"]:focus,.uneditable-input:focus{border-color:rgba(82, 168, 236, 0.8);outline:0;outline:thin dotted \9;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(82,168,236,.6);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(82,168,236,.6);box-shadow:inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(82,168,236,.6);} -input[type="radio"],input[type="checkbox"]{margin:4px 0 0;*margin-top:0;margin-top:1px \9;line-height:normal;} -input[type="file"],input[type="image"],input[type="submit"],input[type="reset"],input[type="button"],input[type="radio"],input[type="checkbox"]{width:auto;} -select,input[type="file"]{height:30px;*margin-top:4px;line-height:30px;} -select{width:220px;border:1px solid #cccccc;background-color:#ffffff;} -select[multiple],select[size]{height:auto;} -select:focus,input[type="file"]:focus,input[type="radio"]:focus,input[type="checkbox"]:focus{outline:thin dotted #333;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px;} -.uneditable-input,.uneditable-textarea{color:#999999;background-color:#fcfcfc;border-color:#cccccc;-webkit-box-shadow:inset 0 1px 2px rgba(0, 0, 0, 0.025);-moz-box-shadow:inset 0 1px 2px rgba(0, 0, 0, 0.025);box-shadow:inset 0 1px 2px rgba(0, 0, 0, 0.025);cursor:not-allowed;} -.uneditable-input{overflow:hidden;white-space:nowrap;} -.uneditable-textarea{width:auto;height:auto;} -input:-moz-placeholder,textarea:-moz-placeholder{color:#999999;} -input:-ms-input-placeholder,textarea:-ms-input-placeholder{color:#999999;} -input::-webkit-input-placeholder,textarea::-webkit-input-placeholder{color:#999999;} -.radio,.checkbox{min-height:20px;padding-left:20px;} -.radio input[type="radio"],.checkbox input[type="checkbox"]{float:left;margin-left:-20px;} -.controls>.radio:first-child,.controls>.checkbox:first-child{padding-top:5px;} -.radio.inline,.checkbox.inline{display:inline-block;padding-top:5px;margin-bottom:0;vertical-align:middle;} -.radio.inline+.radio.inline,.checkbox.inline+.checkbox.inline{margin-left:10px;} -.input-mini{width:60px;} -.input-small{width:90px;} -.input-medium{width:150px;} -.input-large{width:210px;} -.input-xlarge{width:270px;} -.input-xxlarge{width:530px;} -input[class*="span"],select[class*="span"],textarea[class*="span"],.uneditable-input[class*="span"],.row-fluid input[class*="span"],.row-fluid select[class*="span"],.row-fluid textarea[class*="span"],.row-fluid .uneditable-input[class*="span"]{float:none;margin-left:0;} -.input-append input[class*="span"],.input-append .uneditable-input[class*="span"],.input-prepend input[class*="span"],.input-prepend .uneditable-input[class*="span"],.row-fluid input[class*="span"],.row-fluid select[class*="span"],.row-fluid textarea[class*="span"],.row-fluid .uneditable-input[class*="span"],.row-fluid .input-prepend [class*="span"],.row-fluid .input-append [class*="span"]{display:inline-block;} -input,textarea,.uneditable-input{margin-left:0;} -.controls-row [class*="span"]+[class*="span"]{margin-left:20px;} -input.span12,textarea.span12,.uneditable-input.span12{width:926px;} -input.span11,textarea.span11,.uneditable-input.span11{width:846px;} -input.span10,textarea.span10,.uneditable-input.span10{width:766px;} -input.span9,textarea.span9,.uneditable-input.span9{width:686px;} -input.span8,textarea.span8,.uneditable-input.span8{width:606px;} -input.span7,textarea.span7,.uneditable-input.span7{width:526px;} -input.span6,textarea.span6,.uneditable-input.span6{width:446px;} -input.span5,textarea.span5,.uneditable-input.span5{width:366px;} -input.span4,textarea.span4,.uneditable-input.span4{width:286px;} -input.span3,textarea.span3,.uneditable-input.span3{width:206px;} -input.span2,textarea.span2,.uneditable-input.span2{width:126px;} -input.span1,textarea.span1,.uneditable-input.span1{width:46px;} -.controls-row{*zoom:1;}.controls-row:before,.controls-row:after{display:table;content:"";line-height:0;} -.controls-row:after{clear:both;} -.controls-row [class*="span"],.row-fluid .controls-row [class*="span"]{float:left;} -.controls-row .checkbox[class*="span"],.controls-row .radio[class*="span"]{padding-top:5px;} -input[disabled],select[disabled],textarea[disabled],input[readonly],select[readonly],textarea[readonly]{cursor:not-allowed;background-color:#eeeeee;} -input[type="radio"][disabled],input[type="checkbox"][disabled],input[type="radio"][readonly],input[type="checkbox"][readonly]{background-color:transparent;} -.control-group.warning .control-label,.control-group.warning .help-block,.control-group.warning .help-inline{color:#c09853;} -.control-group.warning .checkbox,.control-group.warning .radio,.control-group.warning input,.control-group.warning select,.control-group.warning textarea{color:#c09853;} -.control-group.warning input,.control-group.warning select,.control-group.warning textarea{border-color:#c09853;-webkit-box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075);-moz-box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075);box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075);}.control-group.warning input:focus,.control-group.warning select:focus,.control-group.warning textarea:focus{border-color:#a47e3c;-webkit-box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075),0 0 6px #dbc59e;-moz-box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075),0 0 6px #dbc59e;box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075),0 0 6px #dbc59e;} -.control-group.warning .input-prepend .add-on,.control-group.warning .input-append .add-on{color:#c09853;background-color:#fcf8e3;border-color:#c09853;} -.control-group.error .control-label,.control-group.error .help-block,.control-group.error .help-inline{color:#b94a48;} -.control-group.error .checkbox,.control-group.error .radio,.control-group.error input,.control-group.error select,.control-group.error textarea{color:#b94a48;} -.control-group.error input,.control-group.error select,.control-group.error textarea{border-color:#b94a48;-webkit-box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075);-moz-box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075);box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075);}.control-group.error input:focus,.control-group.error select:focus,.control-group.error textarea:focus{border-color:#953b39;-webkit-box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075),0 0 6px #d59392;-moz-box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075),0 0 6px #d59392;box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075),0 0 6px #d59392;} -.control-group.error .input-prepend .add-on,.control-group.error .input-append .add-on{color:#b94a48;background-color:#f2dede;border-color:#b94a48;} -.control-group.success .control-label,.control-group.success .help-block,.control-group.success .help-inline{color:#468847;} -.control-group.success .checkbox,.control-group.success .radio,.control-group.success input,.control-group.success select,.control-group.success textarea{color:#468847;} -.control-group.success input,.control-group.success select,.control-group.success textarea{border-color:#468847;-webkit-box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075);-moz-box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075);box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075);}.control-group.success input:focus,.control-group.success select:focus,.control-group.success textarea:focus{border-color:#356635;-webkit-box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075),0 0 6px #7aba7b;-moz-box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075),0 0 6px #7aba7b;box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075),0 0 6px #7aba7b;} -.control-group.success .input-prepend .add-on,.control-group.success .input-append .add-on{color:#468847;background-color:#dff0d8;border-color:#468847;} -.control-group.info .control-label,.control-group.info .help-block,.control-group.info .help-inline{color:#3a87ad;} -.control-group.info .checkbox,.control-group.info .radio,.control-group.info input,.control-group.info select,.control-group.info textarea{color:#3a87ad;} -.control-group.info input,.control-group.info select,.control-group.info textarea{border-color:#3a87ad;-webkit-box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075);-moz-box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075);box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075);}.control-group.info input:focus,.control-group.info select:focus,.control-group.info textarea:focus{border-color:#2d6987;-webkit-box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075),0 0 6px #7ab5d3;-moz-box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075),0 0 6px #7ab5d3;box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.075),0 0 6px #7ab5d3;} -.control-group.info .input-prepend .add-on,.control-group.info .input-append .add-on{color:#3a87ad;background-color:#d9edf7;border-color:#3a87ad;} -input:focus:invalid,textarea:focus:invalid,select:focus:invalid{color:#b94a48;border-color:#ee5f5b;}input:focus:invalid:focus,textarea:focus:invalid:focus,select:focus:invalid:focus{border-color:#e9322d;-webkit-box-shadow:0 0 6px #f8b9b7;-moz-box-shadow:0 0 6px #f8b9b7;box-shadow:0 0 6px #f8b9b7;} -.form-actions{padding:19px 20px 20px;margin-top:20px;margin-bottom:20px;background-color:#f5f5f5;border-top:1px solid #e5e5e5;*zoom:1;}.form-actions:before,.form-actions:after{display:table;content:"";line-height:0;} -.form-actions:after{clear:both;} -.help-block,.help-inline{color:#595959;} -.help-block{display:block;margin-bottom:10px;} -.help-inline{display:inline-block;*display:inline;*zoom:1;vertical-align:middle;padding-left:5px;} -.input-append,.input-prepend{display:inline-block;margin-bottom:10px;vertical-align:middle;font-size:0;white-space:nowrap;}.input-append input,.input-prepend input,.input-append select,.input-prepend select,.input-append .uneditable-input,.input-prepend .uneditable-input,.input-append .dropdown-menu,.input-prepend .dropdown-menu,.input-append .popover,.input-prepend .popover{font-size:14px;} -.input-append input,.input-prepend input,.input-append select,.input-prepend select,.input-append .uneditable-input,.input-prepend .uneditable-input{position:relative;margin-bottom:0;*margin-left:0;vertical-align:top;-webkit-border-radius:0 4px 4px 0;-moz-border-radius:0 4px 4px 0;border-radius:0 4px 4px 0;}.input-append input:focus,.input-prepend input:focus,.input-append select:focus,.input-prepend select:focus,.input-append .uneditable-input:focus,.input-prepend .uneditable-input:focus{z-index:2;} -.input-append .add-on,.input-prepend .add-on{display:inline-block;width:auto;height:20px;min-width:16px;padding:4px 5px;font-size:14px;font-weight:normal;line-height:20px;text-align:center;text-shadow:0 1px 0 #ffffff;background-color:#eeeeee;border:1px solid #ccc;} -.input-append .add-on,.input-prepend .add-on,.input-append .btn,.input-prepend .btn,.input-append .btn-group>.dropdown-toggle,.input-prepend .btn-group>.dropdown-toggle{vertical-align:top;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;} -.input-append .active,.input-prepend .active{background-color:#a9dba9;border-color:#46a546;} -.input-prepend .add-on,.input-prepend .btn{margin-right:-1px;} -.input-prepend .add-on:first-child,.input-prepend .btn:first-child{-webkit-border-radius:4px 0 0 4px;-moz-border-radius:4px 0 0 4px;border-radius:4px 0 0 4px;} -.input-append input,.input-append select,.input-append .uneditable-input{-webkit-border-radius:4px 0 0 4px;-moz-border-radius:4px 0 0 4px;border-radius:4px 0 0 4px;}.input-append input+.btn-group .btn:last-child,.input-append select+.btn-group .btn:last-child,.input-append .uneditable-input+.btn-group .btn:last-child{-webkit-border-radius:0 4px 4px 0;-moz-border-radius:0 4px 4px 0;border-radius:0 4px 4px 0;} -.input-append .add-on,.input-append .btn,.input-append .btn-group{margin-left:-1px;} -.input-append .add-on:last-child,.input-append .btn:last-child,.input-append .btn-group:last-child>.dropdown-toggle{-webkit-border-radius:0 4px 4px 0;-moz-border-radius:0 4px 4px 0;border-radius:0 4px 4px 0;} -.input-prepend.input-append input,.input-prepend.input-append select,.input-prepend.input-append .uneditable-input{-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;}.input-prepend.input-append input+.btn-group .btn,.input-prepend.input-append select+.btn-group .btn,.input-prepend.input-append .uneditable-input+.btn-group .btn{-webkit-border-radius:0 4px 4px 0;-moz-border-radius:0 4px 4px 0;border-radius:0 4px 4px 0;} -.input-prepend.input-append .add-on:first-child,.input-prepend.input-append .btn:first-child{margin-right:-1px;-webkit-border-radius:4px 0 0 4px;-moz-border-radius:4px 0 0 4px;border-radius:4px 0 0 4px;} -.input-prepend.input-append .add-on:last-child,.input-prepend.input-append .btn:last-child{margin-left:-1px;-webkit-border-radius:0 4px 4px 0;-moz-border-radius:0 4px 4px 0;border-radius:0 4px 4px 0;} -.input-prepend.input-append .btn-group:first-child{margin-left:0;} -input.search-query{padding-right:14px;padding-right:4px \9;padding-left:14px;padding-left:4px \9;margin-bottom:0;-webkit-border-radius:15px;-moz-border-radius:15px;border-radius:15px;} -.form-search .input-append .search-query,.form-search .input-prepend .search-query{-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;} -.form-search .input-append .search-query{-webkit-border-radius:14px 0 0 14px;-moz-border-radius:14px 0 0 14px;border-radius:14px 0 0 14px;} -.form-search .input-append .btn{-webkit-border-radius:0 14px 14px 0;-moz-border-radius:0 14px 14px 0;border-radius:0 14px 14px 0;} -.form-search .input-prepend .search-query{-webkit-border-radius:0 14px 14px 0;-moz-border-radius:0 14px 14px 0;border-radius:0 14px 14px 0;} -.form-search .input-prepend .btn{-webkit-border-radius:14px 0 0 14px;-moz-border-radius:14px 0 0 14px;border-radius:14px 0 0 14px;} -.form-search input,.form-inline input,.form-horizontal input,.form-search textarea,.form-inline textarea,.form-horizontal textarea,.form-search select,.form-inline select,.form-horizontal select,.form-search .help-inline,.form-inline .help-inline,.form-horizontal .help-inline,.form-search .uneditable-input,.form-inline .uneditable-input,.form-horizontal .uneditable-input,.form-search .input-prepend,.form-inline .input-prepend,.form-horizontal .input-prepend,.form-search .input-append,.form-inline .input-append,.form-horizontal .input-append{display:inline-block;*display:inline;*zoom:1;margin-bottom:0;vertical-align:middle;} -.form-search .hide,.form-inline .hide,.form-horizontal .hide{display:none;} -.form-search label,.form-inline label,.form-search .btn-group,.form-inline .btn-group{display:inline-block;} -.form-search .input-append,.form-inline .input-append,.form-search .input-prepend,.form-inline .input-prepend{margin-bottom:0;} -.form-search .radio,.form-search .checkbox,.form-inline .radio,.form-inline .checkbox{padding-left:0;margin-bottom:0;vertical-align:middle;} -.form-search .radio input[type="radio"],.form-search .checkbox input[type="checkbox"],.form-inline .radio input[type="radio"],.form-inline .checkbox input[type="checkbox"]{float:left;margin-right:3px;margin-left:0;} -.control-group{margin-bottom:10px;} -legend+.control-group{margin-top:20px;-webkit-margin-top-collapse:separate;} -.form-horizontal .control-group{margin-bottom:20px;*zoom:1;}.form-horizontal .control-group:before,.form-horizontal .control-group:after{display:table;content:"";line-height:0;} -.form-horizontal .control-group:after{clear:both;} -.form-horizontal .control-label{float:left;width:160px;padding-top:5px;text-align:right;} -.form-horizontal .controls{*display:inline-block;*padding-left:20px;margin-left:180px;*margin-left:0;}.form-horizontal .controls:first-child{*padding-left:180px;} -.form-horizontal .help-block{margin-bottom:0;} -.form-horizontal input+.help-block,.form-horizontal select+.help-block,.form-horizontal textarea+.help-block,.form-horizontal .uneditable-input+.help-block,.form-horizontal .input-prepend+.help-block,.form-horizontal .input-append+.help-block{margin-top:10px;} -.form-horizontal .form-actions{padding-left:180px;} -.btn{display:inline-block;*display:inline;*zoom:1;padding:4px 12px;margin-bottom:0;font-size:14px;line-height:20px;text-align:center;vertical-align:middle;cursor:pointer;color:#333333;text-shadow:0 1px 1px rgba(255, 255, 255, 0.75);background-color:#f5f5f5;background-image:-moz-linear-gradient(top, #ffffff, #e6e6e6);background-image:-webkit-gradient(linear, 0 0, 0 100%, from(#ffffff), to(#e6e6e6));background-image:-webkit-linear-gradient(top, #ffffff, #e6e6e6);background-image:-o-linear-gradient(top, #ffffff, #e6e6e6);background-image:linear-gradient(to bottom, #ffffff, #e6e6e6);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffffff', endColorstr='#ffe6e6e6', GradientType=0);border-color:#e6e6e6 #e6e6e6 #bfbfbf;border-color:rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);*background-color:#e6e6e6;filter:progid:DXImageTransform.Microsoft.gradient(enabled = false);border:1px solid #cccccc;*border:0;border-bottom-color:#b3b3b3;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;*margin-left:.3em;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,.2), 0 1px 2px rgba(0,0,0,.05);-moz-box-shadow:inset 0 1px 0 rgba(255,255,255,.2), 0 1px 2px rgba(0,0,0,.05);box-shadow:inset 0 1px 0 rgba(255,255,255,.2), 0 1px 2px rgba(0,0,0,.05);}.btn:hover,.btn:focus,.btn:active,.btn.active,.btn.disabled,.btn[disabled]{color:#333333;background-color:#e6e6e6;*background-color:#d9d9d9;} -.btn:active,.btn.active{background-color:#cccccc \9;} -.btn:first-child{*margin-left:0;} -.btn:hover,.btn:focus{color:#333333;text-decoration:none;background-position:0 -15px;-webkit-transition:background-position 0.1s linear;-moz-transition:background-position 0.1s linear;-o-transition:background-position 0.1s linear;transition:background-position 0.1s linear;} -.btn:focus{outline:thin dotted #333;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px;} -.btn.active,.btn:active{background-image:none;outline:0;-webkit-box-shadow:inset 0 2px 4px rgba(0,0,0,.15), 0 1px 2px rgba(0,0,0,.05);-moz-box-shadow:inset 0 2px 4px rgba(0,0,0,.15), 0 1px 2px rgba(0,0,0,.05);box-shadow:inset 0 2px 4px rgba(0,0,0,.15), 0 1px 2px rgba(0,0,0,.05);} -.btn.disabled,.btn[disabled]{cursor:default;background-image:none;opacity:0.65;filter:alpha(opacity=65);-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none;} -.btn-large{padding:11px 19px;font-size:17.5px;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px;} -.btn-large [class^="icon-"],.btn-large [class*=" icon-"]{margin-top:4px;} -.btn-small{padding:2px 10px;font-size:11.9px;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px;} -.btn-small [class^="icon-"],.btn-small [class*=" icon-"]{margin-top:0;} -.btn-mini [class^="icon-"],.btn-mini [class*=" icon-"]{margin-top:-1px;} -.btn-mini{padding:0 6px;font-size:10.5px;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px;} -.btn-block{display:block;width:100%;padding-left:0;padding-right:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;} -.btn-block+.btn-block{margin-top:5px;} -input[type="submit"].btn-block,input[type="reset"].btn-block,input[type="button"].btn-block{width:100%;} -.btn-primary.active,.btn-warning.active,.btn-danger.active,.btn-success.active,.btn-info.active,.btn-inverse.active{color:rgba(255, 255, 255, 0.75);} -.btn-primary{color:#ffffff;text-shadow:0 -1px 0 rgba(0, 0, 0, 0.25);background-color:#006dcc;background-image:-moz-linear-gradient(top, #0088cc, #0044cc);background-image:-webkit-gradient(linear, 0 0, 0 100%, from(#0088cc), to(#0044cc));background-image:-webkit-linear-gradient(top, #0088cc, #0044cc);background-image:-o-linear-gradient(top, #0088cc, #0044cc);background-image:linear-gradient(to bottom, #0088cc, #0044cc);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff0088cc', endColorstr='#ff0044cc', GradientType=0);border-color:#0044cc #0044cc #002a80;border-color:rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);*background-color:#0044cc;filter:progid:DXImageTransform.Microsoft.gradient(enabled = false);}.btn-primary:hover,.btn-primary:focus,.btn-primary:active,.btn-primary.active,.btn-primary.disabled,.btn-primary[disabled]{color:#ffffff;background-color:#0044cc;*background-color:#003bb3;} -.btn-primary:active,.btn-primary.active{background-color:#003399 \9;} -.btn-warning{color:#ffffff;text-shadow:0 -1px 0 rgba(0, 0, 0, 0.25);background-color:#faa732;background-image:-moz-linear-gradient(top, #fbb450, #f89406);background-image:-webkit-gradient(linear, 0 0, 0 100%, from(#fbb450), to(#f89406));background-image:-webkit-linear-gradient(top, #fbb450, #f89406);background-image:-o-linear-gradient(top, #fbb450, #f89406);background-image:linear-gradient(to bottom, #fbb450, #f89406);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fffbb450', endColorstr='#fff89406', GradientType=0);border-color:#f89406 #f89406 #ad6704;border-color:rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);*background-color:#f89406;filter:progid:DXImageTransform.Microsoft.gradient(enabled = false);}.btn-warning:hover,.btn-warning:focus,.btn-warning:active,.btn-warning.active,.btn-warning.disabled,.btn-warning[disabled]{color:#ffffff;background-color:#f89406;*background-color:#df8505;} -.btn-warning:active,.btn-warning.active{background-color:#c67605 \9;} -.btn-danger{color:#ffffff;text-shadow:0 -1px 0 rgba(0, 0, 0, 0.25);background-color:#da4f49;background-image:-moz-linear-gradient(top, #ee5f5b, #bd362f);background-image:-webkit-gradient(linear, 0 0, 0 100%, from(#ee5f5b), to(#bd362f));background-image:-webkit-linear-gradient(top, #ee5f5b, #bd362f);background-image:-o-linear-gradient(top, #ee5f5b, #bd362f);background-image:linear-gradient(to bottom, #ee5f5b, #bd362f);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffee5f5b', endColorstr='#ffbd362f', GradientType=0);border-color:#bd362f #bd362f #802420;border-color:rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);*background-color:#bd362f;filter:progid:DXImageTransform.Microsoft.gradient(enabled = false);}.btn-danger:hover,.btn-danger:focus,.btn-danger:active,.btn-danger.active,.btn-danger.disabled,.btn-danger[disabled]{color:#ffffff;background-color:#bd362f;*background-color:#a9302a;} -.btn-danger:active,.btn-danger.active{background-color:#942a25 \9;} -.btn-success{color:#ffffff;text-shadow:0 -1px 0 rgba(0, 0, 0, 0.25);background-color:#5bb75b;background-image:-moz-linear-gradient(top, #62c462, #51a351);background-image:-webkit-gradient(linear, 0 0, 0 100%, from(#62c462), to(#51a351));background-image:-webkit-linear-gradient(top, #62c462, #51a351);background-image:-o-linear-gradient(top, #62c462, #51a351);background-image:linear-gradient(to bottom, #62c462, #51a351);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff62c462', endColorstr='#ff51a351', GradientType=0);border-color:#51a351 #51a351 #387038;border-color:rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);*background-color:#51a351;filter:progid:DXImageTransform.Microsoft.gradient(enabled = false);}.btn-success:hover,.btn-success:focus,.btn-success:active,.btn-success.active,.btn-success.disabled,.btn-success[disabled]{color:#ffffff;background-color:#51a351;*background-color:#499249;} -.btn-success:active,.btn-success.active{background-color:#408140 \9;} -.btn-info{color:#ffffff;text-shadow:0 -1px 0 rgba(0, 0, 0, 0.25);background-color:#49afcd;background-image:-moz-linear-gradient(top, #5bc0de, #2f96b4);background-image:-webkit-gradient(linear, 0 0, 0 100%, from(#5bc0de), to(#2f96b4));background-image:-webkit-linear-gradient(top, #5bc0de, #2f96b4);background-image:-o-linear-gradient(top, #5bc0de, #2f96b4);background-image:linear-gradient(to bottom, #5bc0de, #2f96b4);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff5bc0de', endColorstr='#ff2f96b4', GradientType=0);border-color:#2f96b4 #2f96b4 #1f6377;border-color:rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);*background-color:#2f96b4;filter:progid:DXImageTransform.Microsoft.gradient(enabled = false);}.btn-info:hover,.btn-info:focus,.btn-info:active,.btn-info.active,.btn-info.disabled,.btn-info[disabled]{color:#ffffff;background-color:#2f96b4;*background-color:#2a85a0;} -.btn-info:active,.btn-info.active{background-color:#24748c \9;} -.btn-inverse{color:#ffffff;text-shadow:0 -1px 0 rgba(0, 0, 0, 0.25);background-color:#363636;background-image:-moz-linear-gradient(top, #444444, #222222);background-image:-webkit-gradient(linear, 0 0, 0 100%, from(#444444), to(#222222));background-image:-webkit-linear-gradient(top, #444444, #222222);background-image:-o-linear-gradient(top, #444444, #222222);background-image:linear-gradient(to bottom, #444444, #222222);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff444444', endColorstr='#ff222222', GradientType=0);border-color:#222222 #222222 #000000;border-color:rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);*background-color:#222222;filter:progid:DXImageTransform.Microsoft.gradient(enabled = false);}.btn-inverse:hover,.btn-inverse:focus,.btn-inverse:active,.btn-inverse.active,.btn-inverse.disabled,.btn-inverse[disabled]{color:#ffffff;background-color:#222222;*background-color:#151515;} -.btn-inverse:active,.btn-inverse.active{background-color:#080808 \9;} -button.btn,input[type="submit"].btn{*padding-top:3px;*padding-bottom:3px;}button.btn::-moz-focus-inner,input[type="submit"].btn::-moz-focus-inner{padding:0;border:0;} -button.btn.btn-large,input[type="submit"].btn.btn-large{*padding-top:7px;*padding-bottom:7px;} -button.btn.btn-small,input[type="submit"].btn.btn-small{*padding-top:3px;*padding-bottom:3px;} -button.btn.btn-mini,input[type="submit"].btn.btn-mini{*padding-top:1px;*padding-bottom:1px;} -.btn-link,.btn-link:active,.btn-link[disabled]{background-color:transparent;background-image:none;-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none;} -.btn-link{border-color:transparent;cursor:pointer;color:#0088cc;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;} -.btn-link:hover,.btn-link:focus{color:#005580;text-decoration:underline;background-color:transparent;} -.btn-link[disabled]:hover,.btn-link[disabled]:focus{color:#333333;text-decoration:none;} -[class^="icon-"],[class*=" icon-"]{display:inline-block;width:14px;height:14px;*margin-right:.3em;line-height:14px;vertical-align:text-top;background-image:url("../img/glyphicons-halflings.png");background-position:14px 14px;background-repeat:no-repeat;margin-top:1px;} -.icon-white,.nav-pills>.active>a>[class^="icon-"],.nav-pills>.active>a>[class*=" icon-"],.nav-list>.active>a>[class^="icon-"],.nav-list>.active>a>[class*=" icon-"],.navbar-inverse .nav>.active>a>[class^="icon-"],.navbar-inverse .nav>.active>a>[class*=" icon-"],.dropdown-menu>li>a:hover>[class^="icon-"],.dropdown-menu>li>a:focus>[class^="icon-"],.dropdown-menu>li>a:hover>[class*=" icon-"],.dropdown-menu>li>a:focus>[class*=" icon-"],.dropdown-menu>.active>a>[class^="icon-"],.dropdown-menu>.active>a>[class*=" icon-"],.dropdown-submenu:hover>a>[class^="icon-"],.dropdown-submenu:focus>a>[class^="icon-"],.dropdown-submenu:hover>a>[class*=" icon-"],.dropdown-submenu:focus>a>[class*=" icon-"]{background-image:url("../img/glyphicons-halflings-white.png");} -.icon-glass{background-position:0 0;} -.icon-music{background-position:-24px 0;} -.icon-search{background-position:-48px 0;} -.icon-envelope{background-position:-72px 0;} -.icon-heart{background-position:-96px 0;} -.icon-star{background-position:-120px 0;} -.icon-star-empty{background-position:-144px 0;} -.icon-user{background-position:-168px 0;} -.icon-film{background-position:-192px 0;} -.icon-th-large{background-position:-216px 0;} -.icon-th{background-position:-240px 0;} -.icon-th-list{background-position:-264px 0;} -.icon-ok{background-position:-288px 0;} -.icon-remove{background-position:-312px 0;} -.icon-zoom-in{background-position:-336px 0;} -.icon-zoom-out{background-position:-360px 0;} -.icon-off{background-position:-384px 0;} -.icon-signal{background-position:-408px 0;} -.icon-cog{background-position:-432px 0;} -.icon-trash{background-position:-456px 0;} -.icon-home{background-position:0 -24px;} -.icon-file{background-position:-24px -24px;} -.icon-time{background-position:-48px -24px;} -.icon-road{background-position:-72px -24px;} -.icon-download-alt{background-position:-96px -24px;} -.icon-download{background-position:-120px -24px;} -.icon-upload{background-position:-144px -24px;} -.icon-inbox{background-position:-168px -24px;} -.icon-play-circle{background-position:-192px -24px;} -.icon-repeat{background-position:-216px -24px;} -.icon-refresh{background-position:-240px -24px;} -.icon-list-alt{background-position:-264px -24px;} -.icon-lock{background-position:-287px -24px;} -.icon-flag{background-position:-312px -24px;} -.icon-headphones{background-position:-336px -24px;} -.icon-volume-off{background-position:-360px -24px;} -.icon-volume-down{background-position:-384px -24px;} -.icon-volume-up{background-position:-408px -24px;} -.icon-qrcode{background-position:-432px -24px;} -.icon-barcode{background-position:-456px -24px;} -.icon-tag{background-position:0 -48px;} -.icon-tags{background-position:-25px -48px;} -.icon-book{background-position:-48px -48px;} -.icon-bookmark{background-position:-72px -48px;} -.icon-print{background-position:-96px -48px;} -.icon-camera{background-position:-120px -48px;} -.icon-font{background-position:-144px -48px;} -.icon-bold{background-position:-167px -48px;} -.icon-italic{background-position:-192px -48px;} -.icon-text-height{background-position:-216px -48px;} -.icon-text-width{background-position:-240px -48px;} -.icon-align-left{background-position:-264px -48px;} -.icon-align-center{background-position:-288px -48px;} -.icon-align-right{background-position:-312px -48px;} -.icon-align-justify{background-position:-336px -48px;} -.icon-list{background-position:-360px -48px;} -.icon-indent-left{background-position:-384px -48px;} -.icon-indent-right{background-position:-408px -48px;} -.icon-facetime-video{background-position:-432px -48px;} -.icon-picture{background-position:-456px -48px;} -.icon-pencil{background-position:0 -72px;} -.icon-map-marker{background-position:-24px -72px;} -.icon-adjust{background-position:-48px -72px;} -.icon-tint{background-position:-72px -72px;} -.icon-edit{background-position:-96px -72px;} -.icon-share{background-position:-120px -72px;} -.icon-check{background-position:-144px -72px;} -.icon-move{background-position:-168px -72px;} -.icon-step-backward{background-position:-192px -72px;} -.icon-fast-backward{background-position:-216px -72px;} -.icon-backward{background-position:-240px -72px;} -.icon-play{background-position:-264px -72px;} -.icon-pause{background-position:-288px -72px;} -.icon-stop{background-position:-312px -72px;} -.icon-forward{background-position:-336px -72px;} -.icon-fast-forward{background-position:-360px -72px;} -.icon-step-forward{background-position:-384px -72px;} -.icon-eject{background-position:-408px -72px;} -.icon-chevron-left{background-position:-432px -72px;} -.icon-chevron-right{background-position:-456px -72px;} -.icon-plus-sign{background-position:0 -96px;} -.icon-minus-sign{background-position:-24px -96px;} -.icon-remove-sign{background-position:-48px -96px;} -.icon-ok-sign{background-position:-72px -96px;} -.icon-question-sign{background-position:-96px -96px;} -.icon-info-sign{background-position:-120px -96px;} -.icon-screenshot{background-position:-144px -96px;} -.icon-remove-circle{background-position:-168px -96px;} -.icon-ok-circle{background-position:-192px -96px;} -.icon-ban-circle{background-position:-216px -96px;} -.icon-arrow-left{background-position:-240px -96px;} -.icon-arrow-right{background-position:-264px -96px;} -.icon-arrow-up{background-position:-289px -96px;} -.icon-arrow-down{background-position:-312px -96px;} -.icon-share-alt{background-position:-336px -96px;} -.icon-resize-full{background-position:-360px -96px;} -.icon-resize-small{background-position:-384px -96px;} -.icon-plus{background-position:-408px -96px;} -.icon-minus{background-position:-433px -96px;} -.icon-asterisk{background-position:-456px -96px;} -.icon-exclamation-sign{background-position:0 -120px;} -.icon-gift{background-position:-24px -120px;} -.icon-leaf{background-position:-48px -120px;} -.icon-fire{background-position:-72px -120px;} -.icon-eye-open{background-position:-96px -120px;} -.icon-eye-close{background-position:-120px -120px;} -.icon-warning-sign{background-position:-144px -120px;} -.icon-plane{background-position:-168px -120px;} -.icon-calendar{background-position:-192px -120px;} -.icon-random{background-position:-216px -120px;width:16px;} -.icon-comment{background-position:-240px -120px;} -.icon-magnet{background-position:-264px -120px;} -.icon-chevron-up{background-position:-288px -120px;} -.icon-chevron-down{background-position:-313px -119px;} -.icon-retweet{background-position:-336px -120px;} -.icon-shopping-cart{background-position:-360px -120px;} -.icon-folder-close{background-position:-384px -120px;width:16px;} -.icon-folder-open{background-position:-408px -120px;width:16px;} -.icon-resize-vertical{background-position:-432px -119px;} -.icon-resize-horizontal{background-position:-456px -118px;} -.icon-hdd{background-position:0 -144px;} -.icon-bullhorn{background-position:-24px -144px;} -.icon-bell{background-position:-48px -144px;} -.icon-certificate{background-position:-72px -144px;} -.icon-thumbs-up{background-position:-96px -144px;} -.icon-thumbs-down{background-position:-120px -144px;} -.icon-hand-right{background-position:-144px -144px;} -.icon-hand-left{background-position:-168px -144px;} -.icon-hand-up{background-position:-192px -144px;} -.icon-hand-down{background-position:-216px -144px;} -.icon-circle-arrow-right{background-position:-240px -144px;} -.icon-circle-arrow-left{background-position:-264px -144px;} -.icon-circle-arrow-up{background-position:-288px -144px;} -.icon-circle-arrow-down{background-position:-312px -144px;} -.icon-globe{background-position:-336px -144px;} -.icon-wrench{background-position:-360px -144px;} -.icon-tasks{background-position:-384px -144px;} -.icon-filter{background-position:-408px -144px;} -.icon-briefcase{background-position:-432px -144px;} -.icon-fullscreen{background-position:-456px -144px;} -.btn-group{position:relative;display:inline-block;*display:inline;*zoom:1;font-size:0;vertical-align:middle;white-space:nowrap;*margin-left:.3em;}.btn-group:first-child{*margin-left:0;} -.btn-group+.btn-group{margin-left:5px;} -.btn-toolbar{font-size:0;margin-top:10px;margin-bottom:10px;}.btn-toolbar>.btn+.btn,.btn-toolbar>.btn-group+.btn,.btn-toolbar>.btn+.btn-group{margin-left:5px;} -.btn-group>.btn{position:relative;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;} -.btn-group>.btn+.btn{margin-left:-1px;} -.btn-group>.btn,.btn-group>.dropdown-menu,.btn-group>.popover{font-size:14px;} -.btn-group>.btn-mini{font-size:10.5px;} -.btn-group>.btn-small{font-size:11.9px;} -.btn-group>.btn-large{font-size:17.5px;} -.btn-group>.btn:first-child{margin-left:0;-webkit-border-top-left-radius:4px;-moz-border-radius-topleft:4px;border-top-left-radius:4px;-webkit-border-bottom-left-radius:4px;-moz-border-radius-bottomleft:4px;border-bottom-left-radius:4px;} -.btn-group>.btn:last-child,.btn-group>.dropdown-toggle{-webkit-border-top-right-radius:4px;-moz-border-radius-topright:4px;border-top-right-radius:4px;-webkit-border-bottom-right-radius:4px;-moz-border-radius-bottomright:4px;border-bottom-right-radius:4px;} -.btn-group>.btn.large:first-child{margin-left:0;-webkit-border-top-left-radius:6px;-moz-border-radius-topleft:6px;border-top-left-radius:6px;-webkit-border-bottom-left-radius:6px;-moz-border-radius-bottomleft:6px;border-bottom-left-radius:6px;} -.btn-group>.btn.large:last-child,.btn-group>.large.dropdown-toggle{-webkit-border-top-right-radius:6px;-moz-border-radius-topright:6px;border-top-right-radius:6px;-webkit-border-bottom-right-radius:6px;-moz-border-radius-bottomright:6px;border-bottom-right-radius:6px;} -.btn-group>.btn:hover,.btn-group>.btn:focus,.btn-group>.btn:active,.btn-group>.btn.active{z-index:2;} -.btn-group .dropdown-toggle:active,.btn-group.open .dropdown-toggle{outline:0;} -.btn-group>.btn+.dropdown-toggle{padding-left:8px;padding-right:8px;-webkit-box-shadow:inset 1px 0 0 rgba(255,255,255,.125), inset 0 1px 0 rgba(255,255,255,.2), 0 1px 2px rgba(0,0,0,.05);-moz-box-shadow:inset 1px 0 0 rgba(255,255,255,.125), inset 0 1px 0 rgba(255,255,255,.2), 0 1px 2px rgba(0,0,0,.05);box-shadow:inset 1px 0 0 rgba(255,255,255,.125), inset 0 1px 0 rgba(255,255,255,.2), 0 1px 2px rgba(0,0,0,.05);*padding-top:5px;*padding-bottom:5px;} -.btn-group>.btn-mini+.dropdown-toggle{padding-left:5px;padding-right:5px;*padding-top:2px;*padding-bottom:2px;} -.btn-group>.btn-small+.dropdown-toggle{*padding-top:5px;*padding-bottom:4px;} -.btn-group>.btn-large+.dropdown-toggle{padding-left:12px;padding-right:12px;*padding-top:7px;*padding-bottom:7px;} -.btn-group.open .dropdown-toggle{background-image:none;-webkit-box-shadow:inset 0 2px 4px rgba(0,0,0,.15), 0 1px 2px rgba(0,0,0,.05);-moz-box-shadow:inset 0 2px 4px rgba(0,0,0,.15), 0 1px 2px rgba(0,0,0,.05);box-shadow:inset 0 2px 4px rgba(0,0,0,.15), 0 1px 2px rgba(0,0,0,.05);} -.btn-group.open .btn.dropdown-toggle{background-color:#e6e6e6;} -.btn-group.open .btn-primary.dropdown-toggle{background-color:#0044cc;} -.btn-group.open .btn-warning.dropdown-toggle{background-color:#f89406;} -.btn-group.open .btn-danger.dropdown-toggle{background-color:#bd362f;} -.btn-group.open .btn-success.dropdown-toggle{background-color:#51a351;} -.btn-group.open .btn-info.dropdown-toggle{background-color:#2f96b4;} -.btn-group.open .btn-inverse.dropdown-toggle{background-color:#222222;} -.btn .caret{margin-top:8px;margin-left:0;} -.btn-large .caret{margin-top:6px;} -.btn-large .caret{border-left-width:5px;border-right-width:5px;border-top-width:5px;} -.btn-mini .caret,.btn-small .caret{margin-top:8px;} -.dropup .btn-large .caret{border-bottom-width:5px;} -.btn-primary .caret,.btn-warning .caret,.btn-danger .caret,.btn-info .caret,.btn-success .caret,.btn-inverse .caret{border-top-color:#ffffff;border-bottom-color:#ffffff;} -.btn-group-vertical{display:inline-block;*display:inline;*zoom:1;} -.btn-group-vertical>.btn{display:block;float:none;max-width:100%;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;} -.btn-group-vertical>.btn+.btn{margin-left:0;margin-top:-1px;} -.btn-group-vertical>.btn:first-child{-webkit-border-radius:4px 4px 0 0;-moz-border-radius:4px 4px 0 0;border-radius:4px 4px 0 0;} -.btn-group-vertical>.btn:last-child{-webkit-border-radius:0 0 4px 4px;-moz-border-radius:0 0 4px 4px;border-radius:0 0 4px 4px;} -.btn-group-vertical>.btn-large:first-child{-webkit-border-radius:6px 6px 0 0;-moz-border-radius:6px 6px 0 0;border-radius:6px 6px 0 0;} -.btn-group-vertical>.btn-large:last-child{-webkit-border-radius:0 0 6px 6px;-moz-border-radius:0 0 6px 6px;border-radius:0 0 6px 6px;} -.nav{margin-left:0;margin-bottom:20px;list-style:none;} -.nav>li>a{display:block;} -.nav>li>a:hover,.nav>li>a:focus{text-decoration:none;background-color:#eeeeee;} -.nav>li>a>img{max-width:none;} -.nav>.pull-right{float:right;} -.nav-header{display:block;padding:3px 15px;font-size:11px;font-weight:bold;line-height:20px;color:#999999;text-shadow:0 1px 0 rgba(255, 255, 255, 0.5);text-transform:uppercase;} -.nav li+.nav-header{margin-top:9px;} -.nav-list{padding-left:15px;padding-right:15px;margin-bottom:0;} -.nav-list>li>a,.nav-list .nav-header{margin-left:-15px;margin-right:-15px;text-shadow:0 1px 0 rgba(255, 255, 255, 0.5);} -.nav-list>li>a{padding:3px 15px;} -.nav-list>.active>a,.nav-list>.active>a:hover,.nav-list>.active>a:focus{color:#ffffff;text-shadow:0 -1px 0 rgba(0, 0, 0, 0.2);background-color:#0088cc;} -.nav-list [class^="icon-"],.nav-list [class*=" icon-"]{margin-right:2px;} -.nav-list .divider{*width:100%;height:1px;margin:9px 1px;*margin:-5px 0 5px;overflow:hidden;background-color:#e5e5e5;border-bottom:1px solid #ffffff;} -.nav-tabs,.nav-pills{*zoom:1;}.nav-tabs:before,.nav-pills:before,.nav-tabs:after,.nav-pills:after{display:table;content:"";line-height:0;} -.nav-tabs:after,.nav-pills:after{clear:both;} -.nav-tabs>li,.nav-pills>li{float:left;} -.nav-tabs>li>a,.nav-pills>li>a{padding-right:12px;padding-left:12px;margin-right:2px;line-height:14px;} -.nav-tabs{border-bottom:1px solid #ddd;} -.nav-tabs>li{margin-bottom:-1px;} -.nav-tabs>li>a{padding-top:8px;padding-bottom:8px;line-height:20px;border:1px solid transparent;-webkit-border-radius:4px 4px 0 0;-moz-border-radius:4px 4px 0 0;border-radius:4px 4px 0 0;}.nav-tabs>li>a:hover,.nav-tabs>li>a:focus{border-color:#eeeeee #eeeeee #dddddd;} -.nav-tabs>.active>a,.nav-tabs>.active>a:hover,.nav-tabs>.active>a:focus{color:#555555;background-color:#ffffff;border:1px solid #ddd;border-bottom-color:transparent;cursor:default;} -.nav-pills>li>a{padding-top:8px;padding-bottom:8px;margin-top:2px;margin-bottom:2px;-webkit-border-radius:5px;-moz-border-radius:5px;border-radius:5px;} -.nav-pills>.active>a,.nav-pills>.active>a:hover,.nav-pills>.active>a:focus{color:#ffffff;background-color:#0088cc;} -.nav-stacked>li{float:none;} -.nav-stacked>li>a{margin-right:0;} -.nav-tabs.nav-stacked{border-bottom:0;} -.nav-tabs.nav-stacked>li>a{border:1px solid #ddd;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;} -.nav-tabs.nav-stacked>li:first-child>a{-webkit-border-top-right-radius:4px;-moz-border-radius-topright:4px;border-top-right-radius:4px;-webkit-border-top-left-radius:4px;-moz-border-radius-topleft:4px;border-top-left-radius:4px;} -.nav-tabs.nav-stacked>li:last-child>a{-webkit-border-bottom-right-radius:4px;-moz-border-radius-bottomright:4px;border-bottom-right-radius:4px;-webkit-border-bottom-left-radius:4px;-moz-border-radius-bottomleft:4px;border-bottom-left-radius:4px;} -.nav-tabs.nav-stacked>li>a:hover,.nav-tabs.nav-stacked>li>a:focus{border-color:#ddd;z-index:2;} -.nav-pills.nav-stacked>li>a{margin-bottom:3px;} -.nav-pills.nav-stacked>li:last-child>a{margin-bottom:1px;} -.nav-tabs .dropdown-menu{-webkit-border-radius:0 0 6px 6px;-moz-border-radius:0 0 6px 6px;border-radius:0 0 6px 6px;} -.nav-pills .dropdown-menu{-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px;} -.nav .dropdown-toggle .caret{border-top-color:#0088cc;border-bottom-color:#0088cc;margin-top:6px;} -.nav .dropdown-toggle:hover .caret,.nav .dropdown-toggle:focus .caret{border-top-color:#005580;border-bottom-color:#005580;} -.nav-tabs .dropdown-toggle .caret{margin-top:8px;} -.nav .active .dropdown-toggle .caret{border-top-color:#fff;border-bottom-color:#fff;} -.nav-tabs .active .dropdown-toggle .caret{border-top-color:#555555;border-bottom-color:#555555;} -.nav>.dropdown.active>a:hover,.nav>.dropdown.active>a:focus{cursor:pointer;} -.nav-tabs .open .dropdown-toggle,.nav-pills .open .dropdown-toggle,.nav>li.dropdown.open.active>a:hover,.nav>li.dropdown.open.active>a:focus{color:#ffffff;background-color:#999999;border-color:#999999;} -.nav li.dropdown.open .caret,.nav li.dropdown.open.active .caret,.nav li.dropdown.open a:hover .caret,.nav li.dropdown.open a:focus .caret{border-top-color:#ffffff;border-bottom-color:#ffffff;opacity:1;filter:alpha(opacity=100);} -.tabs-stacked .open>a:hover,.tabs-stacked .open>a:focus{border-color:#999999;} -.tabbable{*zoom:1;}.tabbable:before,.tabbable:after{display:table;content:"";line-height:0;} -.tabbable:after{clear:both;} -.tab-content{overflow:auto;} -.tabs-below>.nav-tabs,.tabs-right>.nav-tabs,.tabs-left>.nav-tabs{border-bottom:0;} -.tab-content>.tab-pane,.pill-content>.pill-pane{display:none;} -.tab-content>.active,.pill-content>.active{display:block;} -.tabs-below>.nav-tabs{border-top:1px solid #ddd;} -.tabs-below>.nav-tabs>li{margin-top:-1px;margin-bottom:0;} -.tabs-below>.nav-tabs>li>a{-webkit-border-radius:0 0 4px 4px;-moz-border-radius:0 0 4px 4px;border-radius:0 0 4px 4px;}.tabs-below>.nav-tabs>li>a:hover,.tabs-below>.nav-tabs>li>a:focus{border-bottom-color:transparent;border-top-color:#ddd;} -.tabs-below>.nav-tabs>.active>a,.tabs-below>.nav-tabs>.active>a:hover,.tabs-below>.nav-tabs>.active>a:focus{border-color:transparent #ddd #ddd #ddd;} -.tabs-left>.nav-tabs>li,.tabs-right>.nav-tabs>li{float:none;} -.tabs-left>.nav-tabs>li>a,.tabs-right>.nav-tabs>li>a{min-width:74px;margin-right:0;margin-bottom:3px;} -.tabs-left>.nav-tabs{float:left;margin-right:19px;border-right:1px solid #ddd;} -.tabs-left>.nav-tabs>li>a{margin-right:-1px;-webkit-border-radius:4px 0 0 4px;-moz-border-radius:4px 0 0 4px;border-radius:4px 0 0 4px;} -.tabs-left>.nav-tabs>li>a:hover,.tabs-left>.nav-tabs>li>a:focus{border-color:#eeeeee #dddddd #eeeeee #eeeeee;} -.tabs-left>.nav-tabs .active>a,.tabs-left>.nav-tabs .active>a:hover,.tabs-left>.nav-tabs .active>a:focus{border-color:#ddd transparent #ddd #ddd;*border-right-color:#ffffff;} -.tabs-right>.nav-tabs{float:right;margin-left:19px;border-left:1px solid #ddd;} -.tabs-right>.nav-tabs>li>a{margin-left:-1px;-webkit-border-radius:0 4px 4px 0;-moz-border-radius:0 4px 4px 0;border-radius:0 4px 4px 0;} -.tabs-right>.nav-tabs>li>a:hover,.tabs-right>.nav-tabs>li>a:focus{border-color:#eeeeee #eeeeee #eeeeee #dddddd;} -.tabs-right>.nav-tabs .active>a,.tabs-right>.nav-tabs .active>a:hover,.tabs-right>.nav-tabs .active>a:focus{border-color:#ddd #ddd #ddd transparent;*border-left-color:#ffffff;} -.nav>.disabled>a{color:#999999;} -.nav>.disabled>a:hover,.nav>.disabled>a:focus{text-decoration:none;background-color:transparent;cursor:default;} -.navbar-inner{min-height:40px;padding-left:20px;padding-right:20px;background-color:#fafafa;background-image:-moz-linear-gradient(top, #ffffff, #f2f2f2);background-image:-webkit-gradient(linear, 0 0, 0 100%, from(#ffffff), to(#f2f2f2));background-image:-webkit-linear-gradient(top, #ffffff, #f2f2f2);background-image:-o-linear-gradient(top, #ffffff, #f2f2f2);background-image:linear-gradient(to bottom, #ffffff, #f2f2f2);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffffff', endColorstr='#fff2f2f2', GradientType=0);border:1px solid #d4d4d4;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;-webkit-box-shadow:0 1px 4px rgba(0, 0, 0, 0.065);-moz-box-shadow:0 1px 4px rgba(0, 0, 0, 0.065);box-shadow:0 1px 4px rgba(0, 0, 0, 0.065);*zoom:1;}.navbar-inner:before,.navbar-inner:after{display:table;content:"";line-height:0;} -.navbar-inner:after{clear:both;} -.navbar .container{width:auto;} -.nav-collapse.collapse{height:auto;overflow:visible;} -.navbar .brand{float:left;display:block;padding:10px 20px 10px;margin-left:-20px;font-size:20px;font-weight:200;color:#777777;text-shadow:0 1px 0 #ffffff;}.navbar .brand:hover,.navbar .brand:focus{text-decoration:none;} -.navbar-text{margin-bottom:0;line-height:40px;color:#777777;} -.navbar-link{color:#777777;}.navbar-link:hover,.navbar-link:focus{color:#333333;} -.navbar .divider-vertical{height:40px;margin:0 9px;border-left:1px solid #f2f2f2;border-right:1px solid #ffffff;} -.navbar .btn,.navbar .btn-group{margin-top:5px;} -.navbar .btn-group .btn,.navbar .input-prepend .btn,.navbar .input-append .btn,.navbar .input-prepend .btn-group,.navbar .input-append .btn-group{margin-top:0;} -.navbar-form{margin-bottom:0;*zoom:1;}.navbar-form:before,.navbar-form:after{display:table;content:"";line-height:0;} -.navbar-form:after{clear:both;} -.navbar-form input,.navbar-form select,.navbar-form .radio,.navbar-form .checkbox{margin-top:5px;} -.navbar-form input,.navbar-form select,.navbar-form .btn{display:inline-block;margin-bottom:0;} -.navbar-form input[type="image"],.navbar-form input[type="checkbox"],.navbar-form input[type="radio"]{margin-top:3px;} -.navbar-form .input-append,.navbar-form .input-prepend{margin-top:5px;white-space:nowrap;}.navbar-form .input-append input,.navbar-form .input-prepend input{margin-top:0;} -.navbar-search{position:relative;float:left;margin-top:5px;margin-bottom:0;}.navbar-search .search-query{margin-bottom:0;padding:4px 14px;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:13px;font-weight:normal;line-height:1;-webkit-border-radius:15px;-moz-border-radius:15px;border-radius:15px;} -.navbar-static-top{position:static;margin-bottom:0;}.navbar-static-top .navbar-inner{-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;} -.navbar-fixed-top,.navbar-fixed-bottom{position:fixed;right:0;left:0;z-index:1030;margin-bottom:0;} -.navbar-fixed-top .navbar-inner,.navbar-static-top .navbar-inner{border-width:0 0 1px;} -.navbar-fixed-bottom .navbar-inner{border-width:1px 0 0;} -.navbar-fixed-top .navbar-inner,.navbar-fixed-bottom .navbar-inner{padding-left:0;padding-right:0;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;} -.navbar-static-top .container,.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:940px;} -.navbar-fixed-top{top:0;} -.navbar-fixed-top .navbar-inner,.navbar-static-top .navbar-inner{-webkit-box-shadow:0 1px 10px rgba(0,0,0,.1);-moz-box-shadow:0 1px 10px rgba(0,0,0,.1);box-shadow:0 1px 10px rgba(0,0,0,.1);} -.navbar-fixed-bottom{bottom:0;}.navbar-fixed-bottom .navbar-inner{-webkit-box-shadow:0 -1px 10px rgba(0,0,0,.1);-moz-box-shadow:0 -1px 10px rgba(0,0,0,.1);box-shadow:0 -1px 10px rgba(0,0,0,.1);} -.navbar .nav{position:relative;left:0;display:block;float:left;margin:0 10px 0 0;} -.navbar .nav.pull-right{float:right;margin-right:0;} -.navbar .nav>li{float:left;} -.navbar .nav>li>a{float:none;padding:10px 15px 10px;color:#777777;text-decoration:none;text-shadow:0 1px 0 #ffffff;} -.navbar .nav .dropdown-toggle .caret{margin-top:8px;} -.navbar .nav>li>a:focus,.navbar .nav>li>a:hover{background-color:transparent;color:#333333;text-decoration:none;} -.navbar .nav>.active>a,.navbar .nav>.active>a:hover,.navbar .nav>.active>a:focus{color:#555555;text-decoration:none;background-color:#e5e5e5;-webkit-box-shadow:inset 0 3px 8px rgba(0, 0, 0, 0.125);-moz-box-shadow:inset 0 3px 8px rgba(0, 0, 0, 0.125);box-shadow:inset 0 3px 8px rgba(0, 0, 0, 0.125);} -.navbar .btn-navbar{display:none;float:right;padding:7px 10px;margin-left:5px;margin-right:5px;color:#ffffff;text-shadow:0 -1px 0 rgba(0, 0, 0, 0.25);background-color:#ededed;background-image:-moz-linear-gradient(top, #f2f2f2, #e5e5e5);background-image:-webkit-gradient(linear, 0 0, 0 100%, from(#f2f2f2), to(#e5e5e5));background-image:-webkit-linear-gradient(top, #f2f2f2, #e5e5e5);background-image:-o-linear-gradient(top, #f2f2f2, #e5e5e5);background-image:linear-gradient(to bottom, #f2f2f2, #e5e5e5);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff2f2f2', endColorstr='#ffe5e5e5', GradientType=0);border-color:#e5e5e5 #e5e5e5 #bfbfbf;border-color:rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);*background-color:#e5e5e5;filter:progid:DXImageTransform.Microsoft.gradient(enabled = false);-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,.1), 0 1px 0 rgba(255,255,255,.075);-moz-box-shadow:inset 0 1px 0 rgba(255,255,255,.1), 0 1px 0 rgba(255,255,255,.075);box-shadow:inset 0 1px 0 rgba(255,255,255,.1), 0 1px 0 rgba(255,255,255,.075);}.navbar .btn-navbar:hover,.navbar .btn-navbar:focus,.navbar .btn-navbar:active,.navbar .btn-navbar.active,.navbar .btn-navbar.disabled,.navbar .btn-navbar[disabled]{color:#ffffff;background-color:#e5e5e5;*background-color:#d9d9d9;} -.navbar .btn-navbar:active,.navbar .btn-navbar.active{background-color:#cccccc \9;} -.navbar .btn-navbar .icon-bar{display:block;width:18px;height:2px;background-color:#f5f5f5;-webkit-border-radius:1px;-moz-border-radius:1px;border-radius:1px;-webkit-box-shadow:0 1px 0 rgba(0, 0, 0, 0.25);-moz-box-shadow:0 1px 0 rgba(0, 0, 0, 0.25);box-shadow:0 1px 0 rgba(0, 0, 0, 0.25);} -.btn-navbar .icon-bar+.icon-bar{margin-top:3px;} -.navbar .nav>li>.dropdown-menu:before{content:'';display:inline-block;border-left:7px solid transparent;border-right:7px solid transparent;border-bottom:7px solid #ccc;border-bottom-color:rgba(0, 0, 0, 0.2);position:absolute;top:-7px;left:9px;} -.navbar .nav>li>.dropdown-menu:after{content:'';display:inline-block;border-left:6px solid transparent;border-right:6px solid transparent;border-bottom:6px solid #ffffff;position:absolute;top:-6px;left:10px;} -.navbar-fixed-bottom .nav>li>.dropdown-menu:before{border-top:7px solid #ccc;border-top-color:rgba(0, 0, 0, 0.2);border-bottom:0;bottom:-7px;top:auto;} -.navbar-fixed-bottom .nav>li>.dropdown-menu:after{border-top:6px solid #ffffff;border-bottom:0;bottom:-6px;top:auto;} -.navbar .nav li.dropdown>a:hover .caret,.navbar .nav li.dropdown>a:focus .caret{border-top-color:#333333;border-bottom-color:#333333;} -.navbar .nav li.dropdown.open>.dropdown-toggle,.navbar .nav li.dropdown.active>.dropdown-toggle,.navbar .nav li.dropdown.open.active>.dropdown-toggle{background-color:#e5e5e5;color:#555555;} -.navbar .nav li.dropdown>.dropdown-toggle .caret{border-top-color:#777777;border-bottom-color:#777777;} -.navbar .nav li.dropdown.open>.dropdown-toggle .caret,.navbar .nav li.dropdown.active>.dropdown-toggle .caret,.navbar .nav li.dropdown.open.active>.dropdown-toggle .caret{border-top-color:#555555;border-bottom-color:#555555;} -.navbar .pull-right>li>.dropdown-menu,.navbar .nav>li>.dropdown-menu.pull-right{left:auto;right:0;}.navbar .pull-right>li>.dropdown-menu:before,.navbar .nav>li>.dropdown-menu.pull-right:before{left:auto;right:12px;} -.navbar .pull-right>li>.dropdown-menu:after,.navbar .nav>li>.dropdown-menu.pull-right:after{left:auto;right:13px;} -.navbar .pull-right>li>.dropdown-menu .dropdown-menu,.navbar .nav>li>.dropdown-menu.pull-right .dropdown-menu{left:auto;right:100%;margin-left:0;margin-right:-1px;-webkit-border-radius:6px 0 6px 6px;-moz-border-radius:6px 0 6px 6px;border-radius:6px 0 6px 6px;} -.navbar-inverse .navbar-inner{background-color:#1b1b1b;background-image:-moz-linear-gradient(top, #222222, #111111);background-image:-webkit-gradient(linear, 0 0, 0 100%, from(#222222), to(#111111));background-image:-webkit-linear-gradient(top, #222222, #111111);background-image:-o-linear-gradient(top, #222222, #111111);background-image:linear-gradient(to bottom, #222222, #111111);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff222222', endColorstr='#ff111111', GradientType=0);border-color:#252525;} -.navbar-inverse .brand,.navbar-inverse .nav>li>a{color:#999999;text-shadow:0 -1px 0 rgba(0, 0, 0, 0.25);}.navbar-inverse .brand:hover,.navbar-inverse .nav>li>a:hover,.navbar-inverse .brand:focus,.navbar-inverse .nav>li>a:focus{color:#ffffff;} -.navbar-inverse .brand{color:#999999;} -.navbar-inverse .navbar-text{color:#999999;} -.navbar-inverse .nav>li>a:focus,.navbar-inverse .nav>li>a:hover{background-color:transparent;color:#ffffff;} -.navbar-inverse .nav .active>a,.navbar-inverse .nav .active>a:hover,.navbar-inverse .nav .active>a:focus{color:#ffffff;background-color:#111111;} -.navbar-inverse .navbar-link{color:#999999;}.navbar-inverse .navbar-link:hover,.navbar-inverse .navbar-link:focus{color:#ffffff;} -.navbar-inverse .divider-vertical{border-left-color:#111111;border-right-color:#222222;} -.navbar-inverse .nav li.dropdown.open>.dropdown-toggle,.navbar-inverse .nav li.dropdown.active>.dropdown-toggle,.navbar-inverse .nav li.dropdown.open.active>.dropdown-toggle{background-color:#111111;color:#ffffff;} -.navbar-inverse .nav li.dropdown>a:hover .caret,.navbar-inverse .nav li.dropdown>a:focus .caret{border-top-color:#ffffff;border-bottom-color:#ffffff;} -.navbar-inverse .nav li.dropdown>.dropdown-toggle .caret{border-top-color:#999999;border-bottom-color:#999999;} -.navbar-inverse .nav li.dropdown.open>.dropdown-toggle .caret,.navbar-inverse .nav li.dropdown.active>.dropdown-toggle .caret,.navbar-inverse .nav li.dropdown.open.active>.dropdown-toggle .caret{border-top-color:#ffffff;border-bottom-color:#ffffff;} -.navbar-inverse .navbar-search .search-query{color:#ffffff;background-color:#515151;border-color:#111111;-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,.1), 0 1px 0 rgba(255,255,255,.15);-moz-box-shadow:inset 0 1px 2px rgba(0,0,0,.1), 0 1px 0 rgba(255,255,255,.15);box-shadow:inset 0 1px 2px rgba(0,0,0,.1), 0 1px 0 rgba(255,255,255,.15);-webkit-transition:none;-moz-transition:none;-o-transition:none;transition:none;}.navbar-inverse .navbar-search .search-query:-moz-placeholder{color:#cccccc;} -.navbar-inverse .navbar-search .search-query:-ms-input-placeholder{color:#cccccc;} -.navbar-inverse .navbar-search .search-query::-webkit-input-placeholder{color:#cccccc;} -.navbar-inverse .navbar-search .search-query:focus,.navbar-inverse .navbar-search .search-query.focused{padding:5px 15px;color:#333333;text-shadow:0 1px 0 #ffffff;background-color:#ffffff;border:0;-webkit-box-shadow:0 0 3px rgba(0, 0, 0, 0.15);-moz-box-shadow:0 0 3px rgba(0, 0, 0, 0.15);box-shadow:0 0 3px rgba(0, 0, 0, 0.15);outline:0;} -.navbar-inverse .btn-navbar{color:#ffffff;text-shadow:0 -1px 0 rgba(0, 0, 0, 0.25);background-color:#0e0e0e;background-image:-moz-linear-gradient(top, #151515, #040404);background-image:-webkit-gradient(linear, 0 0, 0 100%, from(#151515), to(#040404));background-image:-webkit-linear-gradient(top, #151515, #040404);background-image:-o-linear-gradient(top, #151515, #040404);background-image:linear-gradient(to bottom, #151515, #040404);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff151515', endColorstr='#ff040404', GradientType=0);border-color:#040404 #040404 #000000;border-color:rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);*background-color:#040404;filter:progid:DXImageTransform.Microsoft.gradient(enabled = false);}.navbar-inverse .btn-navbar:hover,.navbar-inverse .btn-navbar:focus,.navbar-inverse .btn-navbar:active,.navbar-inverse .btn-navbar.active,.navbar-inverse .btn-navbar.disabled,.navbar-inverse .btn-navbar[disabled]{color:#ffffff;background-color:#040404;*background-color:#000000;} -.navbar-inverse .btn-navbar:active,.navbar-inverse .btn-navbar.active{background-color:#000000 \9;} -.breadcrumb{padding:8px 15px;margin:0 0 20px;list-style:none;background-color:#f5f5f5;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;}.breadcrumb>li{display:inline-block;*display:inline;*zoom:1;text-shadow:0 1px 0 #ffffff;}.breadcrumb>li>.divider{padding:0 5px;color:#ccc;} -.breadcrumb>.active{color:#999999;} -.pagination{margin:20px 0;} -.pagination ul{display:inline-block;*display:inline;*zoom:1;margin-left:0;margin-bottom:0;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;-webkit-box-shadow:0 1px 2px rgba(0, 0, 0, 0.05);-moz-box-shadow:0 1px 2px rgba(0, 0, 0, 0.05);box-shadow:0 1px 2px rgba(0, 0, 0, 0.05);} -.pagination ul>li{display:inline;} -.pagination ul>li>a,.pagination ul>li>span{float:left;padding:4px 12px;line-height:20px;text-decoration:none;background-color:#ffffff;border:1px solid #dddddd;border-left-width:0;} -.pagination ul>li>a:hover,.pagination ul>li>a:focus,.pagination ul>.active>a,.pagination ul>.active>span{background-color:#f5f5f5;} -.pagination ul>.active>a,.pagination ul>.active>span{color:#999999;cursor:default;} -.pagination ul>.disabled>span,.pagination ul>.disabled>a,.pagination ul>.disabled>a:hover,.pagination ul>.disabled>a:focus{color:#999999;background-color:transparent;cursor:default;} -.pagination ul>li:first-child>a,.pagination ul>li:first-child>span{border-left-width:1px;-webkit-border-top-left-radius:4px;-moz-border-radius-topleft:4px;border-top-left-radius:4px;-webkit-border-bottom-left-radius:4px;-moz-border-radius-bottomleft:4px;border-bottom-left-radius:4px;} -.pagination ul>li:last-child>a,.pagination ul>li:last-child>span{-webkit-border-top-right-radius:4px;-moz-border-radius-topright:4px;border-top-right-radius:4px;-webkit-border-bottom-right-radius:4px;-moz-border-radius-bottomright:4px;border-bottom-right-radius:4px;} -.pagination-centered{text-align:center;} -.pagination-right{text-align:right;} -.pagination-large ul>li>a,.pagination-large ul>li>span{padding:11px 19px;font-size:17.5px;} -.pagination-large ul>li:first-child>a,.pagination-large ul>li:first-child>span{-webkit-border-top-left-radius:6px;-moz-border-radius-topleft:6px;border-top-left-radius:6px;-webkit-border-bottom-left-radius:6px;-moz-border-radius-bottomleft:6px;border-bottom-left-radius:6px;} -.pagination-large ul>li:last-child>a,.pagination-large ul>li:last-child>span{-webkit-border-top-right-radius:6px;-moz-border-radius-topright:6px;border-top-right-radius:6px;-webkit-border-bottom-right-radius:6px;-moz-border-radius-bottomright:6px;border-bottom-right-radius:6px;} -.pagination-mini ul>li:first-child>a,.pagination-small ul>li:first-child>a,.pagination-mini ul>li:first-child>span,.pagination-small ul>li:first-child>span{-webkit-border-top-left-radius:3px;-moz-border-radius-topleft:3px;border-top-left-radius:3px;-webkit-border-bottom-left-radius:3px;-moz-border-radius-bottomleft:3px;border-bottom-left-radius:3px;} -.pagination-mini ul>li:last-child>a,.pagination-small ul>li:last-child>a,.pagination-mini ul>li:last-child>span,.pagination-small ul>li:last-child>span{-webkit-border-top-right-radius:3px;-moz-border-radius-topright:3px;border-top-right-radius:3px;-webkit-border-bottom-right-radius:3px;-moz-border-radius-bottomright:3px;border-bottom-right-radius:3px;} -.pagination-small ul>li>a,.pagination-small ul>li>span{padding:2px 10px;font-size:11.9px;} -.pagination-mini ul>li>a,.pagination-mini ul>li>span{padding:0 6px;font-size:10.5px;} -.pager{margin:20px 0;list-style:none;text-align:center;*zoom:1;}.pager:before,.pager:after{display:table;content:"";line-height:0;} -.pager:after{clear:both;} -.pager li{display:inline;} -.pager li>a,.pager li>span{display:inline-block;padding:5px 14px;background-color:#fff;border:1px solid #ddd;-webkit-border-radius:15px;-moz-border-radius:15px;border-radius:15px;} -.pager li>a:hover,.pager li>a:focus{text-decoration:none;background-color:#f5f5f5;} -.pager .next>a,.pager .next>span{float:right;} -.pager .previous>a,.pager .previous>span{float:left;} -.pager .disabled>a,.pager .disabled>a:hover,.pager .disabled>a:focus,.pager .disabled>span{color:#999999;background-color:#fff;cursor:default;} -.thumbnails{margin-left:-20px;list-style:none;*zoom:1;}.thumbnails:before,.thumbnails:after{display:table;content:"";line-height:0;} -.thumbnails:after{clear:both;} -.row-fluid .thumbnails{margin-left:0;} -.thumbnails>li{float:left;margin-bottom:20px;margin-left:20px;} -.thumbnail{display:block;padding:4px;line-height:20px;border:1px solid #ddd;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;-webkit-box-shadow:0 1px 3px rgba(0, 0, 0, 0.055);-moz-box-shadow:0 1px 3px rgba(0, 0, 0, 0.055);box-shadow:0 1px 3px rgba(0, 0, 0, 0.055);-webkit-transition:all 0.2s ease-in-out;-moz-transition:all 0.2s ease-in-out;-o-transition:all 0.2s ease-in-out;transition:all 0.2s ease-in-out;} -a.thumbnail:hover,a.thumbnail:focus{border-color:#0088cc;-webkit-box-shadow:0 1px 4px rgba(0, 105, 214, 0.25);-moz-box-shadow:0 1px 4px rgba(0, 105, 214, 0.25);box-shadow:0 1px 4px rgba(0, 105, 214, 0.25);} -.thumbnail>img{display:block;max-width:100%;margin-left:auto;margin-right:auto;} -.thumbnail .caption{padding:9px;color:#555555;} -.alert{padding:8px 35px 8px 14px;margin-bottom:20px;text-shadow:0 1px 0 rgba(255, 255, 255, 0.5);background-color:#fcf8e3;border:1px solid #fbeed5;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;} -.alert,.alert h4{color:#c09853;} -.alert h4{margin:0;} -.alert .close{position:relative;top:-2px;right:-21px;line-height:20px;} -.alert-success{background-color:#dff0d8;border-color:#d6e9c6;color:#468847;} -.alert-success h4{color:#468847;} -.alert-danger,.alert-error{background-color:#f2dede;border-color:#eed3d7;color:#b94a48;} -.alert-danger h4,.alert-error h4{color:#b94a48;} -.alert-info{background-color:#d9edf7;border-color:#bce8f1;color:#3a87ad;} -.alert-info h4{color:#3a87ad;} -.alert-block{padding-top:14px;padding-bottom:14px;} -.alert-block>p,.alert-block>ul{margin-bottom:0;} -.alert-block p+p{margin-top:5px;} -@-webkit-keyframes progress-bar-stripes{from{background-position:40px 0;} to{background-position:0 0;}}@-moz-keyframes progress-bar-stripes{from{background-position:40px 0;} to{background-position:0 0;}}@-ms-keyframes progress-bar-stripes{from{background-position:40px 0;} to{background-position:0 0;}}@-o-keyframes progress-bar-stripes{from{background-position:0 0;} to{background-position:40px 0;}}@keyframes progress-bar-stripes{from{background-position:40px 0;} to{background-position:0 0;}}.progress{overflow:hidden;height:20px;margin-bottom:20px;background-color:#f7f7f7;background-image:-moz-linear-gradient(top, #f5f5f5, #f9f9f9);background-image:-webkit-gradient(linear, 0 0, 0 100%, from(#f5f5f5), to(#f9f9f9));background-image:-webkit-linear-gradient(top, #f5f5f5, #f9f9f9);background-image:-o-linear-gradient(top, #f5f5f5, #f9f9f9);background-image:linear-gradient(to bottom, #f5f5f5, #f9f9f9);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff5f5f5', endColorstr='#fff9f9f9', GradientType=0);-webkit-box-shadow:inset 0 1px 2px rgba(0, 0, 0, 0.1);-moz-box-shadow:inset 0 1px 2px rgba(0, 0, 0, 0.1);box-shadow:inset 0 1px 2px rgba(0, 0, 0, 0.1);-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;} -.progress .bar{width:0%;height:100%;color:#ffffff;float:left;font-size:12px;text-align:center;text-shadow:0 -1px 0 rgba(0, 0, 0, 0.25);background-color:#0e90d2;background-image:-moz-linear-gradient(top, #149bdf, #0480be);background-image:-webkit-gradient(linear, 0 0, 0 100%, from(#149bdf), to(#0480be));background-image:-webkit-linear-gradient(top, #149bdf, #0480be);background-image:-o-linear-gradient(top, #149bdf, #0480be);background-image:linear-gradient(to bottom, #149bdf, #0480be);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff149bdf', endColorstr='#ff0480be', GradientType=0);-webkit-box-shadow:inset 0 -1px 0 rgba(0, 0, 0, 0.15);-moz-box-shadow:inset 0 -1px 0 rgba(0, 0, 0, 0.15);box-shadow:inset 0 -1px 0 rgba(0, 0, 0, 0.15);-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;-webkit-transition:width 0.6s ease;-moz-transition:width 0.6s ease;-o-transition:width 0.6s ease;transition:width 0.6s ease;} -.progress .bar+.bar{-webkit-box-shadow:inset 1px 0 0 rgba(0,0,0,.15), inset 0 -1px 0 rgba(0,0,0,.15);-moz-box-shadow:inset 1px 0 0 rgba(0,0,0,.15), inset 0 -1px 0 rgba(0,0,0,.15);box-shadow:inset 1px 0 0 rgba(0,0,0,.15), inset 0 -1px 0 rgba(0,0,0,.15);} -.progress-striped .bar{background-color:#149bdf;background-image:-webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));background-image:-webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);background-image:-moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);background-image:-o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);background-image:linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);-webkit-background-size:40px 40px;-moz-background-size:40px 40px;-o-background-size:40px 40px;background-size:40px 40px;} -.progress.active .bar{-webkit-animation:progress-bar-stripes 2s linear infinite;-moz-animation:progress-bar-stripes 2s linear infinite;-ms-animation:progress-bar-stripes 2s linear infinite;-o-animation:progress-bar-stripes 2s linear infinite;animation:progress-bar-stripes 2s linear infinite;} -.progress-danger .bar,.progress .bar-danger{background-color:#dd514c;background-image:-moz-linear-gradient(top, #ee5f5b, #c43c35);background-image:-webkit-gradient(linear, 0 0, 0 100%, from(#ee5f5b), to(#c43c35));background-image:-webkit-linear-gradient(top, #ee5f5b, #c43c35);background-image:-o-linear-gradient(top, #ee5f5b, #c43c35);background-image:linear-gradient(to bottom, #ee5f5b, #c43c35);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffee5f5b', endColorstr='#ffc43c35', GradientType=0);} -.progress-danger.progress-striped .bar,.progress-striped .bar-danger{background-color:#ee5f5b;background-image:-webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));background-image:-webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);background-image:-moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);background-image:-o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);background-image:linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);} -.progress-success .bar,.progress .bar-success{background-color:#5eb95e;background-image:-moz-linear-gradient(top, #62c462, #57a957);background-image:-webkit-gradient(linear, 0 0, 0 100%, from(#62c462), to(#57a957));background-image:-webkit-linear-gradient(top, #62c462, #57a957);background-image:-o-linear-gradient(top, #62c462, #57a957);background-image:linear-gradient(to bottom, #62c462, #57a957);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff62c462', endColorstr='#ff57a957', GradientType=0);} -.progress-success.progress-striped .bar,.progress-striped .bar-success{background-color:#62c462;background-image:-webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));background-image:-webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);background-image:-moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);background-image:-o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);background-image:linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);} -.progress-info .bar,.progress .bar-info{background-color:#4bb1cf;background-image:-moz-linear-gradient(top, #5bc0de, #339bb9);background-image:-webkit-gradient(linear, 0 0, 0 100%, from(#5bc0de), to(#339bb9));background-image:-webkit-linear-gradient(top, #5bc0de, #339bb9);background-image:-o-linear-gradient(top, #5bc0de, #339bb9);background-image:linear-gradient(to bottom, #5bc0de, #339bb9);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff5bc0de', endColorstr='#ff339bb9', GradientType=0);} -.progress-info.progress-striped .bar,.progress-striped .bar-info{background-color:#5bc0de;background-image:-webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));background-image:-webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);background-image:-moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);background-image:-o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);background-image:linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);} -.progress-warning .bar,.progress .bar-warning{background-color:#faa732;background-image:-moz-linear-gradient(top, #fbb450, #f89406);background-image:-webkit-gradient(linear, 0 0, 0 100%, from(#fbb450), to(#f89406));background-image:-webkit-linear-gradient(top, #fbb450, #f89406);background-image:-o-linear-gradient(top, #fbb450, #f89406);background-image:linear-gradient(to bottom, #fbb450, #f89406);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fffbb450', endColorstr='#fff89406', GradientType=0);} -.progress-warning.progress-striped .bar,.progress-striped .bar-warning{background-color:#fbb450;background-image:-webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));background-image:-webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);background-image:-moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);background-image:-o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);background-image:linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);} -.hero-unit{padding:60px;margin-bottom:30px;font-size:18px;font-weight:200;line-height:30px;color:inherit;background-color:#4eacc5;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px;}.hero-unit h1{margin-bottom:0;font-size:60px;line-height:1;color:inherit;letter-spacing:-1px;} -.hero-unit li{line-height:30px;} -.media,.media-body{overflow:hidden;*overflow:visible;zoom:1;} -.media,.media .media{margin-top:15px;} -.media:first-child{margin-top:0;} -.media-object{display:block;} -.media-heading{margin:0 0 5px;} -.media>.pull-left{margin-right:10px;} -.media>.pull-right{margin-left:10px;} -.media-list{margin-left:0;list-style:none;} -.tooltip{position:absolute;z-index:1030;display:block;visibility:visible;font-size:11px;line-height:1.4;opacity:0;filter:alpha(opacity=0);}.tooltip.in{opacity:0.8;filter:alpha(opacity=80);} -.tooltip.top{margin-top:-3px;padding:5px 0;} -.tooltip.right{margin-left:3px;padding:0 5px;} -.tooltip.bottom{margin-top:3px;padding:5px 0;} -.tooltip.left{margin-left:-3px;padding:0 5px;} -.tooltip-inner{max-width:200px;padding:8px;color:#ffffff;text-align:center;text-decoration:none;background-color:#000000;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;} -.tooltip-arrow{position:absolute;width:0;height:0;border-color:transparent;border-style:solid;} -.tooltip.top .tooltip-arrow{bottom:0;left:50%;margin-left:-5px;border-width:5px 5px 0;border-top-color:#000000;} -.tooltip.right .tooltip-arrow{top:50%;left:0;margin-top:-5px;border-width:5px 5px 5px 0;border-right-color:#000000;} -.tooltip.left .tooltip-arrow{top:50%;right:0;margin-top:-5px;border-width:5px 0 5px 5px;border-left-color:#000000;} -.tooltip.bottom .tooltip-arrow{top:0;left:50%;margin-left:-5px;border-width:0 5px 5px;border-bottom-color:#000000;} -.popover{position:absolute;top:0;left:0;z-index:1010;display:none;max-width:276px;padding:1px;text-align:left;background-color:#ffffff;-webkit-background-clip:padding-box;-moz-background-clip:padding;background-clip:padding-box;border:1px solid #ccc;border:1px solid rgba(0, 0, 0, 0.2);-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px;-webkit-box-shadow:0 5px 10px rgba(0, 0, 0, 0.2);-moz-box-shadow:0 5px 10px rgba(0, 0, 0, 0.2);box-shadow:0 5px 10px rgba(0, 0, 0, 0.2);white-space:normal;}.popover.top{margin-top:-10px;} -.popover.right{margin-left:10px;} -.popover.bottom{margin-top:10px;} -.popover.left{margin-left:-10px;} -.popover-title{margin:0;padding:8px 14px;font-size:14px;font-weight:normal;line-height:18px;background-color:#f7f7f7;border-bottom:1px solid #ebebeb;-webkit-border-radius:5px 5px 0 0;-moz-border-radius:5px 5px 0 0;border-radius:5px 5px 0 0;}.popover-title:empty{display:none;} -.popover-content{padding:9px 14px;} -.popover .arrow,.popover .arrow:after{position:absolute;display:block;width:0;height:0;border-color:transparent;border-style:solid;} -.popover .arrow{border-width:11px;} -.popover .arrow:after{border-width:10px;content:"";} -.popover.top .arrow{left:50%;margin-left:-11px;border-bottom-width:0;border-top-color:#999;border-top-color:rgba(0, 0, 0, 0.25);bottom:-11px;}.popover.top .arrow:after{bottom:1px;margin-left:-10px;border-bottom-width:0;border-top-color:#ffffff;} -.popover.right .arrow{top:50%;left:-11px;margin-top:-11px;border-left-width:0;border-right-color:#999;border-right-color:rgba(0, 0, 0, 0.25);}.popover.right .arrow:after{left:1px;bottom:-10px;border-left-width:0;border-right-color:#ffffff;} -.popover.bottom .arrow{left:50%;margin-left:-11px;border-top-width:0;border-bottom-color:#999;border-bottom-color:rgba(0, 0, 0, 0.25);top:-11px;}.popover.bottom .arrow:after{top:1px;margin-left:-10px;border-top-width:0;border-bottom-color:#ffffff;} -.popover.left .arrow{top:50%;right:-11px;margin-top:-11px;border-right-width:0;border-left-color:#999;border-left-color:rgba(0, 0, 0, 0.25);}.popover.left .arrow:after{right:1px;border-right-width:0;border-left-color:#ffffff;bottom:-10px;} -.modal-backdrop{position:fixed;top:0;right:0;bottom:0;left:0;z-index:1040;background-color:#000000;}.modal-backdrop.fade{opacity:0;} -.modal-backdrop,.modal-backdrop.fade.in{opacity:0.8;filter:alpha(opacity=80);} -.modal{position:fixed;top:10%;left:50%;z-index:1050;width:560px;margin-left:-280px;background-color:#ffffff;border:1px solid #999;border:1px solid rgba(0, 0, 0, 0.3);*border:1px solid #999;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px;-webkit-box-shadow:0 3px 7px rgba(0, 0, 0, 0.3);-moz-box-shadow:0 3px 7px rgba(0, 0, 0, 0.3);box-shadow:0 3px 7px rgba(0, 0, 0, 0.3);-webkit-background-clip:padding-box;-moz-background-clip:padding-box;background-clip:padding-box;outline:none;}.modal.fade{-webkit-transition:opacity .3s linear, top .3s ease-out;-moz-transition:opacity .3s linear, top .3s ease-out;-o-transition:opacity .3s linear, top .3s ease-out;transition:opacity .3s linear, top .3s ease-out;top:-25%;} -.modal.fade.in{top:10%;} -.modal-header{padding:9px 15px;border-bottom:1px solid #eee;}.modal-header .close{margin-top:2px;} -.modal-header h3{margin:0;line-height:30px;} -.modal-body{position:relative;overflow-y:auto;max-height:400px;padding:15px;} -.modal-form{margin-bottom:0;} -.modal-footer{padding:14px 15px 15px;margin-bottom:0;text-align:right;background-color:#f5f5f5;border-top:1px solid #ddd;-webkit-border-radius:0 0 6px 6px;-moz-border-radius:0 0 6px 6px;border-radius:0 0 6px 6px;-webkit-box-shadow:inset 0 1px 0 #ffffff;-moz-box-shadow:inset 0 1px 0 #ffffff;box-shadow:inset 0 1px 0 #ffffff;*zoom:1;}.modal-footer:before,.modal-footer:after{display:table;content:"";line-height:0;} -.modal-footer:after{clear:both;} -.modal-footer .btn+.btn{margin-left:5px;margin-bottom:0;} -.modal-footer .btn-group .btn+.btn{margin-left:-1px;} -.modal-footer .btn-block+.btn-block{margin-left:0;} -.dropup,.dropdown{position:relative;} -.dropdown-toggle{*margin-bottom:-3px;} -.dropdown-toggle:active,.open .dropdown-toggle{outline:0;} -.caret{display:inline-block;width:0;height:0;vertical-align:top;border-top:4px solid #000000;border-right:4px solid transparent;border-left:4px solid transparent;content:"";} -.dropdown .caret{margin-top:8px;margin-left:2px;} -.dropdown-menu{position:absolute;top:100%;left:0;z-index:1000;display:none;float:left;min-width:160px;padding:5px 0;margin:2px 0 0;list-style:none;background-color:#ffffff;border:1px solid #ccc;border:1px solid rgba(0, 0, 0, 0.2);*border-right-width:2px;*border-bottom-width:2px;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px;-webkit-box-shadow:0 5px 10px rgba(0, 0, 0, 0.2);-moz-box-shadow:0 5px 10px rgba(0, 0, 0, 0.2);box-shadow:0 5px 10px rgba(0, 0, 0, 0.2);-webkit-background-clip:padding-box;-moz-background-clip:padding;background-clip:padding-box;}.dropdown-menu.pull-right{right:0;left:auto;} -.dropdown-menu .divider{*width:100%;height:1px;margin:9px 1px;*margin:-5px 0 5px;overflow:hidden;background-color:#e5e5e5;border-bottom:1px solid #ffffff;} -.dropdown-menu>li>a{display:block;padding:3px 20px;clear:both;font-weight:normal;line-height:20px;color:#333333;white-space:nowrap;} -.dropdown-menu>li>a:hover,.dropdown-menu>li>a:focus,.dropdown-submenu:hover>a,.dropdown-submenu:focus>a{text-decoration:none;color:#ffffff;background-color:#0081c2;background-image:-moz-linear-gradient(top, #0088cc, #0077b3);background-image:-webkit-gradient(linear, 0 0, 0 100%, from(#0088cc), to(#0077b3));background-image:-webkit-linear-gradient(top, #0088cc, #0077b3);background-image:-o-linear-gradient(top, #0088cc, #0077b3);background-image:linear-gradient(to bottom, #0088cc, #0077b3);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff0088cc', endColorstr='#ff0077b3', GradientType=0);} -.dropdown-menu>.active>a,.dropdown-menu>.active>a:hover,.dropdown-menu>.active>a:focus{color:#ffffff;text-decoration:none;outline:0;background-color:#0081c2;background-image:-moz-linear-gradient(top, #0088cc, #0077b3);background-image:-webkit-gradient(linear, 0 0, 0 100%, from(#0088cc), to(#0077b3));background-image:-webkit-linear-gradient(top, #0088cc, #0077b3);background-image:-o-linear-gradient(top, #0088cc, #0077b3);background-image:linear-gradient(to bottom, #0088cc, #0077b3);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff0088cc', endColorstr='#ff0077b3', GradientType=0);} -.dropdown-menu>.disabled>a,.dropdown-menu>.disabled>a:hover,.dropdown-menu>.disabled>a:focus{color:#999999;} -.dropdown-menu>.disabled>a:hover,.dropdown-menu>.disabled>a:focus{text-decoration:none;background-color:transparent;background-image:none;filter:progid:DXImageTransform.Microsoft.gradient(enabled = false);cursor:default;} -.open{*z-index:1000;}.open>.dropdown-menu{display:block;} -.dropdown-backdrop{position:fixed;left:0;right:0;bottom:0;top:0;z-index:990;} -.pull-right>.dropdown-menu{right:0;left:auto;} -.dropup .caret,.navbar-fixed-bottom .dropdown .caret{border-top:0;border-bottom:4px solid #000000;content:"";} -.dropup .dropdown-menu,.navbar-fixed-bottom .dropdown .dropdown-menu{top:auto;bottom:100%;margin-bottom:1px;} -.dropdown-submenu{position:relative;} -.dropdown-submenu>.dropdown-menu{top:0;left:100%;margin-top:-6px;margin-left:-1px;-webkit-border-radius:0 6px 6px 6px;-moz-border-radius:0 6px 6px 6px;border-radius:0 6px 6px 6px;} -.dropdown-submenu:hover>.dropdown-menu{display:block;} -.dropup .dropdown-submenu>.dropdown-menu{top:auto;bottom:0;margin-top:0;margin-bottom:-2px;-webkit-border-radius:5px 5px 5px 0;-moz-border-radius:5px 5px 5px 0;border-radius:5px 5px 5px 0;} -.dropdown-submenu>a:after{display:block;content:" ";float:right;width:0;height:0;border-color:transparent;border-style:solid;border-width:5px 0 5px 5px;border-left-color:#cccccc;margin-top:5px;margin-right:-10px;} -.dropdown-submenu:hover>a:after{border-left-color:#ffffff;} -.dropdown-submenu.pull-left{float:none;}.dropdown-submenu.pull-left>.dropdown-menu{left:-100%;margin-left:10px;-webkit-border-radius:6px 0 6px 6px;-moz-border-radius:6px 0 6px 6px;border-radius:6px 0 6px 6px;} -.dropdown .dropdown-menu .nav-header{padding-left:20px;padding-right:20px;} -.typeahead{z-index:1051;margin-top:2px;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;} -.accordion{margin-bottom:20px;} -.accordion-group{margin-bottom:2px;border:1px solid #e5e5e5;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;} -.accordion-heading{border-bottom:0;} -.accordion-heading .accordion-toggle{display:block;padding:8px 15px;} -.accordion-toggle{cursor:pointer;} -.accordion-inner{padding:9px 15px;border-top:1px solid #e5e5e5;} -.carousel{position:relative;margin-bottom:20px;line-height:1;} -.carousel-inner{overflow:hidden;width:100%;position:relative;} -.carousel-inner>.item{display:none;position:relative;-webkit-transition:0.6s ease-in-out left;-moz-transition:0.6s ease-in-out left;-o-transition:0.6s ease-in-out left;transition:0.6s ease-in-out left;}.carousel-inner>.item>img,.carousel-inner>.item>a>img{display:block;line-height:1;} -.carousel-inner>.active,.carousel-inner>.next,.carousel-inner>.prev{display:block;} -.carousel-inner>.active{left:0;} -.carousel-inner>.next,.carousel-inner>.prev{position:absolute;top:0;width:100%;} -.carousel-inner>.next{left:100%;} -.carousel-inner>.prev{left:-100%;} -.carousel-inner>.next.left,.carousel-inner>.prev.right{left:0;} -.carousel-inner>.active.left{left:-100%;} -.carousel-inner>.active.right{left:100%;} -.carousel-control{position:absolute;top:40%;left:15px;width:40px;height:40px;margin-top:-20px;font-size:60px;font-weight:100;line-height:30px;color:#ffffff;text-align:center;background:#222222;border:3px solid #ffffff;-webkit-border-radius:23px;-moz-border-radius:23px;border-radius:23px;opacity:0.5;filter:alpha(opacity=50);}.carousel-control.right{left:auto;right:15px;} -.carousel-control:hover,.carousel-control:focus{color:#ffffff;text-decoration:none;opacity:0.9;filter:alpha(opacity=90);} -.carousel-indicators{position:absolute;top:15px;right:15px;z-index:5;margin:0;list-style:none;}.carousel-indicators li{display:block;float:left;width:10px;height:10px;margin-left:5px;text-indent:-999px;background-color:#ccc;background-color:rgba(255, 255, 255, 0.25);border-radius:5px;} -.carousel-indicators .active{background-color:#fff;} -.carousel-caption{position:absolute;left:0;right:0;bottom:0;padding:15px;background:#333333;background:rgba(0, 0, 0, 0.75);} -.carousel-caption h4,.carousel-caption p{color:#ffffff;line-height:20px;} -.carousel-caption h4{margin:0 0 5px;} -.carousel-caption p{margin-bottom:0;} -.well{min-height:20px;padding:19px;margin-bottom:20px;background-color:#f5f5f5;border:1px solid #e3e3e3;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;-webkit-box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.05);-moz-box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.05);box-shadow:inset 0 1px 1px rgba(0, 0, 0, 0.05);}.well blockquote{border-color:#ddd;border-color:rgba(0, 0, 0, 0.15);} -.well-large{padding:24px;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px;} -.well-small{padding:9px;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px;} -.close{float:right;font-size:20px;font-weight:bold;line-height:20px;color:#000000;text-shadow:0 1px 0 #ffffff;opacity:0.2;filter:alpha(opacity=20);}.close:hover,.close:focus{color:#000000;text-decoration:none;cursor:pointer;opacity:0.4;filter:alpha(opacity=40);} -button.close{padding:0;cursor:pointer;background:transparent;border:0;-webkit-appearance:none;} -.pull-right{float:right;} -.pull-left{float:left;} -.hide{display:none;} -.show{display:block;} -.invisible{visibility:hidden;} -.affix{position:fixed;} -.fade{opacity:0;-webkit-transition:opacity 0.15s linear;-moz-transition:opacity 0.15s linear;-o-transition:opacity 0.15s linear;transition:opacity 0.15s linear;}.fade.in{opacity:1;} -.collapse{position:relative;height:0;overflow:hidden;-webkit-transition:height 0.35s ease;-moz-transition:height 0.35s ease;-o-transition:height 0.35s ease;transition:height 0.35s ease;}.collapse.in{height:auto;} -@-ms-viewport{width:device-width;}.hidden{display:none;visibility:hidden;} -.visible-phone{display:none !important;} -.visible-tablet{display:none !important;} -.hidden-desktop{display:none !important;} -.visible-desktop{display:inherit !important;} -@media (min-width:768px) and (max-width:979px){.hidden-desktop{display:inherit !important;} .visible-desktop{display:none !important ;} .visible-tablet{display:inherit !important;} .hidden-tablet{display:none !important;}}@media (max-width:767px){.hidden-desktop{display:inherit !important;} .visible-desktop{display:none !important;} .visible-phone{display:inherit !important;} .hidden-phone{display:none !important;}}.visible-print{display:none !important;} -@media print{.visible-print{display:inherit !important;} .hidden-print{display:none !important;}}@media (max-width:767px){body{padding-left:20px;padding-right:20px;} .navbar-fixed-top,.navbar-fixed-bottom,.navbar-static-top{margin-left:-20px;margin-right:-20px;} .container-fluid{padding:0;} .dl-horizontal dt{float:none;clear:none;width:auto;text-align:left;} .dl-horizontal dd{margin-left:0;} .container{width:auto;} .row-fluid{width:100%;} .row,.thumbnails{margin-left:0;} .thumbnails>li{float:none;margin-left:0;} [class*="span"],.uneditable-input[class*="span"],.row-fluid [class*="span"]{float:none;display:block;width:100%;margin-left:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;} .span12,.row-fluid .span12{width:100%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;} .row-fluid [class*="offset"]:first-child{margin-left:0;} .input-large,.input-xlarge,.input-xxlarge,input[class*="span"],select[class*="span"],textarea[class*="span"],.uneditable-input{display:block;width:100%;min-height:30px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;} .input-prepend input,.input-append input,.input-prepend input[class*="span"],.input-append input[class*="span"]{display:inline-block;width:auto;} .controls-row [class*="span"]+[class*="span"]{margin-left:0;} .modal{position:fixed;top:20px;left:20px;right:20px;width:auto;margin:0;}.modal.fade{top:-100px;} .modal.fade.in{top:20px;}}@media (max-width:480px){.nav-collapse{-webkit-transform:translate3d(0, 0, 0);} .page-header h1 small{display:block;line-height:20px;} input[type="checkbox"],input[type="radio"]{border:1px solid #ccc;} .form-horizontal .control-label{float:none;width:auto;padding-top:0;text-align:left;} .form-horizontal .controls{margin-left:0;} .form-horizontal .control-list{padding-top:0;} .form-horizontal .form-actions{padding-left:10px;padding-right:10px;} .media .pull-left,.media .pull-right{float:none;display:block;margin-bottom:10px;} .media-object{margin-right:0;margin-left:0;} .modal{top:10px;left:10px;right:10px;} .modal-header .close{padding:10px;margin:-10px;} .carousel-caption{position:static;}}@media (min-width:768px) and (max-width:979px){.row{margin-left:-20px;*zoom:1;}.row:before,.row:after{display:table;content:"";line-height:0;} .row:after{clear:both;} [class*="span"]{float:left;min-height:1px;margin-left:20px;} .container,.navbar-static-top .container,.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:724px;} .span12{width:724px;} .span11{width:662px;} .span10{width:600px;} .span9{width:538px;} .span8{width:476px;} .span7{width:414px;} .span6{width:352px;} .span5{width:290px;} .span4{width:228px;} .span3{width:166px;} .span2{width:104px;} .span1{width:42px;} .offset12{margin-left:764px;} .offset11{margin-left:702px;} .offset10{margin-left:640px;} .offset9{margin-left:578px;} .offset8{margin-left:516px;} .offset7{margin-left:454px;} .offset6{margin-left:392px;} .offset5{margin-left:330px;} .offset4{margin-left:268px;} .offset3{margin-left:206px;} .offset2{margin-left:144px;} .offset1{margin-left:82px;} .row-fluid{width:100%;*zoom:1;}.row-fluid:before,.row-fluid:after{display:table;content:"";line-height:0;} .row-fluid:after{clear:both;} .row-fluid [class*="span"]{display:block;width:100%;min-height:30px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;float:left;margin-left:2.7624309392265194%;*margin-left:2.709239449864817%;} .row-fluid [class*="span"]:first-child{margin-left:0;} .row-fluid .controls-row [class*="span"]+[class*="span"]{margin-left:2.7624309392265194%;} .row-fluid .span12{width:100%;*width:99.94680851063829%;} .row-fluid .span11{width:91.43646408839778%;*width:91.38327259903608%;} .row-fluid .span10{width:82.87292817679558%;*width:82.81973668743387%;} .row-fluid .span9{width:74.30939226519337%;*width:74.25620077583166%;} .row-fluid .span8{width:65.74585635359117%;*width:65.69266486422946%;} .row-fluid .span7{width:57.18232044198895%;*width:57.12912895262725%;} .row-fluid .span6{width:48.61878453038674%;*width:48.56559304102504%;} .row-fluid .span5{width:40.05524861878453%;*width:40.00205712942283%;} .row-fluid .span4{width:31.491712707182323%;*width:31.43852121782062%;} .row-fluid .span3{width:22.92817679558011%;*width:22.87498530621841%;} .row-fluid .span2{width:14.3646408839779%;*width:14.311449394616199%;} .row-fluid .span1{width:5.801104972375691%;*width:5.747913483013988%;} .row-fluid .offset12{margin-left:105.52486187845304%;*margin-left:105.41847889972962%;} .row-fluid .offset12:first-child{margin-left:102.76243093922652%;*margin-left:102.6560479605031%;} .row-fluid .offset11{margin-left:96.96132596685082%;*margin-left:96.8549429881274%;} .row-fluid .offset11:first-child{margin-left:94.1988950276243%;*margin-left:94.09251204890089%;} .row-fluid .offset10{margin-left:88.39779005524862%;*margin-left:88.2914070765252%;} .row-fluid .offset10:first-child{margin-left:85.6353591160221%;*margin-left:85.52897613729868%;} .row-fluid .offset9{margin-left:79.8342541436464%;*margin-left:79.72787116492299%;} .row-fluid .offset9:first-child{margin-left:77.07182320441989%;*margin-left:76.96544022569647%;} .row-fluid .offset8{margin-left:71.2707182320442%;*margin-left:71.16433525332079%;} .row-fluid .offset8:first-child{margin-left:68.50828729281768%;*margin-left:68.40190431409427%;} .row-fluid .offset7{margin-left:62.70718232044199%;*margin-left:62.600799341718584%;} .row-fluid .offset7:first-child{margin-left:59.94475138121547%;*margin-left:59.838368402492065%;} .row-fluid .offset6{margin-left:54.14364640883978%;*margin-left:54.037263430116376%;} .row-fluid .offset6:first-child{margin-left:51.38121546961326%;*margin-left:51.27483249088986%;} .row-fluid .offset5{margin-left:45.58011049723757%;*margin-left:45.47372751851417%;} .row-fluid .offset5:first-child{margin-left:42.81767955801105%;*margin-left:42.71129657928765%;} .row-fluid .offset4{margin-left:37.01657458563536%;*margin-left:36.91019160691196%;} .row-fluid .offset4:first-child{margin-left:34.25414364640884%;*margin-left:34.14776066768544%;} .row-fluid .offset3{margin-left:28.45303867403315%;*margin-left:28.346655695309746%;} .row-fluid .offset3:first-child{margin-left:25.69060773480663%;*margin-left:25.584224756083227%;} .row-fluid .offset2{margin-left:19.88950276243094%;*margin-left:19.783119783707537%;} .row-fluid .offset2:first-child{margin-left:17.12707182320442%;*margin-left:17.02068884448102%;} .row-fluid .offset1{margin-left:11.32596685082873%;*margin-left:11.219583872105325%;} .row-fluid .offset1:first-child{margin-left:8.56353591160221%;*margin-left:8.457152932878806%;} input,textarea,.uneditable-input{margin-left:0;} .controls-row [class*="span"]+[class*="span"]{margin-left:20px;} input.span12,textarea.span12,.uneditable-input.span12{width:710px;} input.span11,textarea.span11,.uneditable-input.span11{width:648px;} input.span10,textarea.span10,.uneditable-input.span10{width:586px;} input.span9,textarea.span9,.uneditable-input.span9{width:524px;} input.span8,textarea.span8,.uneditable-input.span8{width:462px;} input.span7,textarea.span7,.uneditable-input.span7{width:400px;} input.span6,textarea.span6,.uneditable-input.span6{width:338px;} input.span5,textarea.span5,.uneditable-input.span5{width:276px;} input.span4,textarea.span4,.uneditable-input.span4{width:214px;} input.span3,textarea.span3,.uneditable-input.span3{width:152px;} input.span2,textarea.span2,.uneditable-input.span2{width:90px;} input.span1,textarea.span1,.uneditable-input.span1{width:28px;}}@media (min-width:1200px){.row{margin-left:-30px;*zoom:1;}.row:before,.row:after{display:table;content:"";line-height:0;} .row:after{clear:both;} [class*="span"]{float:left;min-height:1px;margin-left:30px;} .container,.navbar-static-top .container,.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:1170px;} .span12{width:1170px;} .span11{width:1070px;} .span10{width:970px;} .span9{width:870px;} .span8{width:770px;} .span7{width:670px;} .span6{width:570px;} .span5{width:470px;} .span4{width:370px;} .span3{width:270px;} .span2{width:170px;} .span1{width:70px;} .offset12{margin-left:1230px;} .offset11{margin-left:1130px;} .offset10{margin-left:1030px;} .offset9{margin-left:930px;} .offset8{margin-left:830px;} .offset7{margin-left:730px;} .offset6{margin-left:630px;} .offset5{margin-left:530px;} .offset4{margin-left:430px;} .offset3{margin-left:330px;} .offset2{margin-left:230px;} .offset1{margin-left:130px;} .row-fluid{width:100%;*zoom:1;}.row-fluid:before,.row-fluid:after{display:table;content:"";line-height:0;} .row-fluid:after{clear:both;} .row-fluid [class*="span"]{display:block;width:100%;min-height:30px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;float:left;margin-left:2.564102564102564%;*margin-left:2.5109110747408616%;} .row-fluid [class*="span"]:first-child{margin-left:0;} .row-fluid .controls-row [class*="span"]+[class*="span"]{margin-left:2.564102564102564%;} .row-fluid .span12{width:100%;*width:99.94680851063829%;} .row-fluid .span11{width:91.45299145299145%;*width:91.39979996362975%;} .row-fluid .span10{width:82.90598290598291%;*width:82.8527914166212%;} .row-fluid .span9{width:74.35897435897436%;*width:74.30578286961266%;} .row-fluid .span8{width:65.81196581196582%;*width:65.75877432260411%;} .row-fluid .span7{width:57.26495726495726%;*width:57.21176577559556%;} .row-fluid .span6{width:48.717948717948715%;*width:48.664757228587014%;} .row-fluid .span5{width:40.17094017094017%;*width:40.11774868157847%;} .row-fluid .span4{width:31.623931623931625%;*width:31.570740134569924%;} .row-fluid .span3{width:23.076923076923077%;*width:23.023731587561375%;} .row-fluid .span2{width:14.52991452991453%;*width:14.476723040552828%;} .row-fluid .span1{width:5.982905982905983%;*width:5.929714493544281%;} .row-fluid .offset12{margin-left:105.12820512820512%;*margin-left:105.02182214948171%;} .row-fluid .offset12:first-child{margin-left:102.56410256410257%;*margin-left:102.45771958537915%;} .row-fluid .offset11{margin-left:96.58119658119658%;*margin-left:96.47481360247316%;} .row-fluid .offset11:first-child{margin-left:94.01709401709402%;*margin-left:93.91071103837061%;} .row-fluid .offset10{margin-left:88.03418803418803%;*margin-left:87.92780505546462%;} .row-fluid .offset10:first-child{margin-left:85.47008547008548%;*margin-left:85.36370249136206%;} .row-fluid .offset9{margin-left:79.48717948717949%;*margin-left:79.38079650845607%;} .row-fluid .offset9:first-child{margin-left:76.92307692307693%;*margin-left:76.81669394435352%;} .row-fluid .offset8{margin-left:70.94017094017094%;*margin-left:70.83378796144753%;} .row-fluid .offset8:first-child{margin-left:68.37606837606839%;*margin-left:68.26968539734497%;} .row-fluid .offset7{margin-left:62.393162393162385%;*margin-left:62.28677941443899%;} .row-fluid .offset7:first-child{margin-left:59.82905982905982%;*margin-left:59.72267685033642%;} .row-fluid .offset6{margin-left:53.84615384615384%;*margin-left:53.739770867430444%;} .row-fluid .offset6:first-child{margin-left:51.28205128205128%;*margin-left:51.175668303327875%;} .row-fluid .offset5{margin-left:45.299145299145295%;*margin-left:45.1927623204219%;} .row-fluid .offset5:first-child{margin-left:42.73504273504273%;*margin-left:42.62865975631933%;} .row-fluid .offset4{margin-left:36.75213675213675%;*margin-left:36.645753773413354%;} .row-fluid .offset4:first-child{margin-left:34.18803418803419%;*margin-left:34.081651209310785%;} .row-fluid .offset3{margin-left:28.205128205128204%;*margin-left:28.0987452264048%;} .row-fluid .offset3:first-child{margin-left:25.641025641025642%;*margin-left:25.53464266230224%;} .row-fluid .offset2{margin-left:19.65811965811966%;*margin-left:19.551736679396257%;} .row-fluid .offset2:first-child{margin-left:17.094017094017094%;*margin-left:16.98763411529369%;} .row-fluid .offset1{margin-left:11.11111111111111%;*margin-left:11.004728132387708%;} .row-fluid .offset1:first-child{margin-left:8.547008547008547%;*margin-left:8.440625568285142%;} input,textarea,.uneditable-input{margin-left:0;} .controls-row [class*="span"]+[class*="span"]{margin-left:30px;} input.span12,textarea.span12,.uneditable-input.span12{width:1156px;} input.span11,textarea.span11,.uneditable-input.span11{width:1056px;} input.span10,textarea.span10,.uneditable-input.span10{width:956px;} input.span9,textarea.span9,.uneditable-input.span9{width:856px;} input.span8,textarea.span8,.uneditable-input.span8{width:756px;} input.span7,textarea.span7,.uneditable-input.span7{width:656px;} input.span6,textarea.span6,.uneditable-input.span6{width:556px;} input.span5,textarea.span5,.uneditable-input.span5{width:456px;} input.span4,textarea.span4,.uneditable-input.span4{width:356px;} input.span3,textarea.span3,.uneditable-input.span3{width:256px;} input.span2,textarea.span2,.uneditable-input.span2{width:156px;} input.span1,textarea.span1,.uneditable-input.span1{width:56px;} .thumbnails{margin-left:-30px;} .thumbnails>li{margin-left:30px;} .row-fluid .thumbnails{margin-left:0;}}@media (max-width:979px){body{padding-top:0;} .navbar-fixed-top,.navbar-fixed-bottom{position:static;} .navbar-fixed-top{margin-bottom:20px;} .navbar-fixed-bottom{margin-top:20px;} .navbar-fixed-top .navbar-inner,.navbar-fixed-bottom .navbar-inner{padding:5px;} .navbar .container{width:auto;padding:0;} .navbar .brand{padding-left:10px;padding-right:10px;margin:0 0 0 -5px;} .nav-collapse{clear:both;} .nav-collapse .nav{float:none;margin:0 0 10px;} .nav-collapse .nav>li{float:none;} .nav-collapse .nav>li>a{margin-bottom:2px;} .nav-collapse .nav>.divider-vertical{display:none;} .nav-collapse .nav .nav-header{color:#777777;text-shadow:none;} .nav-collapse .nav>li>a,.nav-collapse .dropdown-menu a{padding:9px 15px;font-weight:bold;color:#777777;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px;} .nav-collapse .btn{padding:4px 10px 4px;font-weight:normal;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;} .nav-collapse .dropdown-menu li+li a{margin-bottom:2px;} .nav-collapse .nav>li>a:hover,.nav-collapse .nav>li>a:focus,.nav-collapse .dropdown-menu a:hover,.nav-collapse .dropdown-menu a:focus{background-color:#f2f2f2;} .navbar-inverse .nav-collapse .nav>li>a,.navbar-inverse .nav-collapse .dropdown-menu a{color:#999999;} .navbar-inverse .nav-collapse .nav>li>a:hover,.navbar-inverse .nav-collapse .nav>li>a:focus,.navbar-inverse .nav-collapse .dropdown-menu a:hover,.navbar-inverse .nav-collapse .dropdown-menu a:focus{background-color:#111111;} .nav-collapse.in .btn-group{margin-top:5px;padding:0;} .nav-collapse .dropdown-menu{position:static;top:auto;left:auto;float:none;display:none;max-width:none;margin:0 15px;padding:0;background-color:transparent;border:none;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none;} .nav-collapse .open>.dropdown-menu{display:block;} .nav-collapse .dropdown-menu:before,.nav-collapse .dropdown-menu:after{display:none;} .nav-collapse .dropdown-menu .divider{display:none;} .nav-collapse .nav>li>.dropdown-menu:before,.nav-collapse .nav>li>.dropdown-menu:after{display:none;} .nav-collapse .navbar-form,.nav-collapse .navbar-search{float:none;padding:10px 15px;margin:10px 0;border-top:1px solid #f2f2f2;border-bottom:1px solid #f2f2f2;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,.1), 0 1px 0 rgba(255,255,255,.1);-moz-box-shadow:inset 0 1px 0 rgba(255,255,255,.1), 0 1px 0 rgba(255,255,255,.1);box-shadow:inset 0 1px 0 rgba(255,255,255,.1), 0 1px 0 rgba(255,255,255,.1);} .navbar-inverse .nav-collapse .navbar-form,.navbar-inverse .nav-collapse .navbar-search{border-top-color:#111111;border-bottom-color:#111111;} .navbar .nav-collapse .nav.pull-right{float:none;margin-left:0;} .nav-collapse,.nav-collapse.collapse{overflow:hidden;height:0;} .navbar .btn-navbar{display:block;} .navbar-static .navbar-inner{padding-left:10px;padding-right:10px;}}@media (min-width:980px){.nav-collapse.collapse{height:auto !important;overflow:visible !important;}} diff --git a/doc/themes/scikit-learn/static/img/FNRS-logo.png b/doc/themes/scikit-learn/static/img/FNRS-logo.png deleted file mode 100644 index 925559035ca9c..0000000000000 Binary files a/doc/themes/scikit-learn/static/img/FNRS-logo.png and /dev/null differ diff --git a/doc/themes/scikit-learn/static/img/forkme.png b/doc/themes/scikit-learn/static/img/forkme.png deleted file mode 100644 index 5cf7a6c2162ff..0000000000000 Binary files a/doc/themes/scikit-learn/static/img/forkme.png and /dev/null differ diff --git a/doc/themes/scikit-learn/static/img/glyphicons-halflings-white.png b/doc/themes/scikit-learn/static/img/glyphicons-halflings-white.png deleted file mode 100644 index 3bf6484a29d8d..0000000000000 Binary files a/doc/themes/scikit-learn/static/img/glyphicons-halflings-white.png and /dev/null differ diff --git a/doc/themes/scikit-learn/static/img/glyphicons-halflings.png b/doc/themes/scikit-learn/static/img/glyphicons-halflings.png deleted file mode 100644 index 8bc9e642b762a..0000000000000 Binary files a/doc/themes/scikit-learn/static/img/glyphicons-halflings.png and /dev/null differ diff --git a/doc/themes/scikit-learn/static/img/google.png b/doc/themes/scikit-learn/static/img/google.png deleted file mode 100644 index a4f0830735994..0000000000000 Binary files a/doc/themes/scikit-learn/static/img/google.png and /dev/null differ diff --git a/doc/themes/scikit-learn/static/img/inria-small.jpg b/doc/themes/scikit-learn/static/img/inria-small.jpg deleted file mode 100644 index 7e2e30825569b..0000000000000 Binary files a/doc/themes/scikit-learn/static/img/inria-small.jpg and /dev/null differ diff --git a/doc/themes/scikit-learn/static/img/inria-small.png b/doc/themes/scikit-learn/static/img/inria-small.png deleted file mode 100644 index a77ace28d4807..0000000000000 Binary files a/doc/themes/scikit-learn/static/img/inria-small.png and /dev/null differ diff --git a/doc/themes/scikit-learn/static/img/nyu_short_color.png b/doc/themes/scikit-learn/static/img/nyu_short_color.png deleted file mode 100644 index e9eb5216057c8..0000000000000 Binary files a/doc/themes/scikit-learn/static/img/nyu_short_color.png and /dev/null differ diff --git a/doc/themes/scikit-learn/static/img/plot_classifier_comparison_1.png b/doc/themes/scikit-learn/static/img/plot_classifier_comparison_1.png deleted file mode 100644 index 387adf0b03f83..0000000000000 Binary files a/doc/themes/scikit-learn/static/img/plot_classifier_comparison_1.png and /dev/null differ diff --git a/doc/themes/scikit-learn/static/img/plot_manifold_sphere_1.png b/doc/themes/scikit-learn/static/img/plot_manifold_sphere_1.png deleted file mode 100644 index 17630fb0f9913..0000000000000 Binary files a/doc/themes/scikit-learn/static/img/plot_manifold_sphere_1.png and /dev/null differ diff --git a/doc/themes/scikit-learn/static/img/scikit-learn-logo-notext.png b/doc/themes/scikit-learn/static/img/scikit-learn-logo-notext.png deleted file mode 100644 index ce1e1056c66a6..0000000000000 Binary files a/doc/themes/scikit-learn/static/img/scikit-learn-logo-notext.png and /dev/null differ diff --git a/doc/themes/scikit-learn/static/img/scikit-learn-logo-small.png b/doc/themes/scikit-learn/static/img/scikit-learn-logo-small.png deleted file mode 100644 index 5882603c25a00..0000000000000 Binary files a/doc/themes/scikit-learn/static/img/scikit-learn-logo-small.png and /dev/null differ diff --git a/doc/themes/scikit-learn/static/img/scikit-learn-logo.png b/doc/themes/scikit-learn/static/img/scikit-learn-logo.png deleted file mode 100644 index 2c75cc2a1386b..0000000000000 Binary files a/doc/themes/scikit-learn/static/img/scikit-learn-logo.png and /dev/null differ diff --git a/doc/themes/scikit-learn/static/img/scikit-learn-logo.svg b/doc/themes/scikit-learn/static/img/scikit-learn-logo.svg deleted file mode 100644 index a4beb202d0967..0000000000000 --- a/doc/themes/scikit-learn/static/img/scikit-learn-logo.svg +++ /dev/null @@ -1,1049 +0,0 @@ - - - - - - - - - scikit-learn/doc/logos/scikit-learn-logo.svg at main ¡ scikit-learn/scikit-learn - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - - - - -
-
- - - - - -
- - - - -
- - -
-
- - - - - -
- - This repository - - -
-
- -
- - -
This repository
-
- -
- - -
All repositories
-
- -
-
-
- - - - - - - - -
- -
- - - - - - - - - - - - -
-
- - - - - - - - -
- -
-
- - -
    - -
  • -
    - -
    - - - - - Unwatch - - - -
    -
    -
    - Notification status - -
    - -
    - -
    - -
    - -

    Not watching

    - You only receive notifications for discussions in which you participate or are @mentioned. - - - Watch - -
    -
    - -
    - -
    - -

    Watching

    - You receive notifications for all discussions in this repository. - - - Unwatch - -
    -
    - -
    - -
    - -

    Ignoring

    - You do not receive any notifications for discussions in this repository. - - - Stop ignoring - -
    -
    - -
    - -
    -
    -
    - -
    -
    • - -
  • - -
    - - Unstar - - - Star - - -
    - -
    • - - -
  • - - Fork - - -
    • - - -
- -

- public - - - /scikit-learn - - - Octocat-spinner-32 - - -

-
-
- -
- -
- -
- - -
-
- -
- - -
- -
-
- -
- - - - -
-

HTTPS clone URL

- - - - -
- - - -
-

SSH clone URL

- - - - -
- - - -
-

Subversion checkout URL

- - - - -
- - - -

You can clone with - HTTPS, - SSH, - Subversion, - and other methods. -

- - - - - - Download ZIP - -
-
- -
- - - - - - - - -Show File Finder - -
- - - -
- - - branch: - main - - -
- -
-
- Switch branches/tags - -
- -
-
- -
-
- -
-
- -
- -
- - -
- - 0.6.X -
-
- - 0.7.X -
-
- - 0.8.X -
-
- - 0.9.X -
-
- - 0.10.X -
-
- - 0.11.X -
-
- - 0.12.X -
-
- - 0.13.X -
-
- - debian -
-
- - invalid-n-folds -
-
- - main -
-
- -
- -
-

Create branch:

- from ‘main’ -
- - - -
- -
- -
-
- - -
- - sprint01 -
-
- - debian/0.12.0-1 -
-
- - debian/0.11.0-2 -
-
- - debian/0.11.0-1 -
-
- - debian/0.10.0-1 -
-
- - debian/0.9.0.dfsg-1 -
-
- - debian/0.8.1.dfsg-1 -
-
- - debian/0.8.0.dfsg-1 -
-
- - debian/0.7.1.dfsg-3 -
-
- - debian/0.7.1.dfsg-1 -
-
- - debian/0.6.0.dfsg-1 -
-
- - debian/0.5-1 -
-
- - debian/0.4-3 -
-
- - debian/0.4-2 -
-
- - debian/0.4-1 -
-
- - debian/0.3-4 -
-
- - debian/0.3-3 -
-
- - debian/0.3-2 -
-
- - debian/0.3-1 -
-
- - debian/0.2+svn625-1 -
-
- - 0.13-branching -
-
- - 0.13.1 -
-
- - 0.13 -
-
- - 0.12-branching -
-
- - 0.12.1 -
-
- - 0.12 -
-
- - 0.11-branching -
-
- - 0.11-beta -
-
- - 0.11 -
-
- - 0.10-branching -
-
- - 0.10 -
-
- - 0.9-branching -
-
- - 0.9 -
-
- - 0.8-branching -
-
- - 0.8.1 -
-
- - 0.8 -
-
- - 0.7-branching -
-
- - 0.7.1 -
-
- - 0.7 -
-
- - 0.6-rc -
-
- - 0.6.0 -
-
- - 0.5.rc3 -
-
- - 0.5.rc2 -
-
- - 0.5.rc -
-
- - 0.5 -
-
- - 0.4 -
-
- - 0.3 -
-
- - 0.2-beta -
-
- - 0.2 -
-
- - 0.1-beta -
-
- - 0.1 -
-
- -
Nothing to show
-
- -
-
-
- -
- scikit-learn / doc / logos / scikit-learn-logo.svg -
-
- - -
- Fetching contributorsâ€Ļ - -
-

Octocat-spinner-32-eaf2f5

-

Cannot retrieve contributors at this time

-
-
- -
-
-
-
- - file - 110 lines (107 sloc) - 6.944 kb -
-
-
- Edit - Raw - Blame - History -
- - Delete - -
- -
-
- - - - - -
- 1 -2 -3 -4 -5 -6 -7 -8 -9 -10 -11 -12 -13 -14 -15 -16 -17 -18 -19 -20 -21 -22 -23 -24 -25 -26 -27 -28 -29 -30 -31 -32 -33 -34 -35 -36 -37 -38 -39 -40 -41 -42 -43 -44 -45 -46 -47 -48 -49 -50 -51 -52 -53 -54 -55 -56 -57 -58 -59 -60 -61 -62 -63 -64 -65 -66 -67 -68 -69 -70 -71 -72 -73 -74 -75 -76 -77 -78 -79 -80 -81 -82 -83 -84 -85 -86 -87 -88 -89 -90 -91 -92 -93 -94 -95 -96 -97 -98 -99 -100 -101 -102 -103 -104 -105 -106 -107 -108 -109 -110 - - - 
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Generator: Adobe Illustrator 14.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 43363)  -->

<svg
   xmlns:dc="http://purl.org/dc/elements/1.1/"
   xmlns:cc="http://creativecommons.org/ns#"
   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
   xmlns:svg="http://www.w3.org/2000/svg"
   xmlns="http://www.w3.org/2000/svg"
   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
   version="1.1"
   id="Layer_1"
   x="0px"
   y="0px"
   width="792px"
   height="612px"
   viewBox="0 0 792 612"
   enable-background="new 0 0 792 612"
   xml:space="preserve"
   inkscape:version="0.48.2 r9819"
   sodipodi:docname="scikit-learn-logo.svg"><metadata
   id="metadata35"><rdf:RDF><cc:Work
       rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
         rdf:resource="http://purl.org/dc/dcmitype/StillImage" /></cc:Work></rdf:RDF></metadata><defs
   id="defs33" /><sodipodi:namedview
   pagecolor="#ffffff"
   bordercolor="#666666"
   borderopacity="1"
   objecttolerance="10"
   gridtolerance="10"
   guidetolerance="10"
   inkscape:pageopacity="0"
   inkscape:pageshadow="2"
   inkscape:window-width="1019"
   inkscape:window-height="958"
   id="namedview31"
   showgrid="false"
   inkscape:zoom="2.1814013"
   inkscape:cx="276.47817"
   inkscape:cy="330.56501"
   inkscape:window-x="246"
   inkscape:window-y="247"
   inkscape:window-maximized="0"
   inkscape:current-layer="Layer_1" />
<g
   id="g3">
	<path
   fill="#F89939"
   d="M333.32,347.348c33.869-33.867,39.498-83.146,12.572-110.07c-26.922-26.921-76.199-21.293-110.066,12.572   c-33.867,33.866-24.07,98.568-12.57,110.07C232.549,369.213,299.455,381.213,333.32,347.348z"
   id="path5" />
	<path
   fill="#3499CD"
   d="M194.35,298.411c-19.648-19.648-48.242-22.919-63.867-7.295c-15.621,15.622-12.355,44.22,7.297,63.865   c19.652,19.654,57.195,13.969,63.863,7.295C207.039,356.889,214.004,318.061,194.35,298.411z"
   id="path7" />
</g>
<g
   id="g9">
	<g
   id="g11">
		<path
   fill="#010101"
   d="M262.143,339.047c-3.471,3.195-6.516,5.553-9.133,7.068c-2.617,1.52-5.113,2.279-7.488,2.279    c-2.732,0-4.936-1.059-6.607-3.178c-1.674-2.121-2.508-4.965-2.508-8.543c0-5.361,1.162-11.797,3.486-19.301    c2.32-7.51,5.145-14.43,8.463-20.761l9.729-3.602c0.305-0.102,0.537-0.154,0.691-0.154c0.738,0,1.348,0.544,1.816,1.627    c0.473,1.088,0.711,2.55,0.711,4.388c0,5.209-1.199,10.252-3.602,15.129c-2.402,4.879-6.154,10.086-11.26,15.627    c-0.205,2.656-0.307,4.48-0.307,5.477c0,2.223,0.408,3.982,1.225,5.285c0.818,1.305,1.902,1.953,3.256,1.953    c1.381,0,2.848-0.494,4.406-1.49c1.555-0.998,3.93-3.064,7.121-6.207V339.047z M247.475,324.074    c3.242-3.605,5.875-7.648,7.891-12.121c2.016-4.475,3.023-8.324,3.023-11.549c0-0.94-0.139-1.704-0.418-2.278    c-0.281-0.575-0.641-0.864-1.074-0.864c-0.941,0-2.316,2.352-4.117,7.057C250.979,309.023,249.211,315.609,247.475,324.074z"
   id="path13" />
		<path
   fill="#010101"
   d="M290.795,339.047c-3.242,3.195-6.152,5.553-8.732,7.068c-2.58,1.52-5.424,2.279-8.541,2.279    c-3.473,0-6.275-1.111-8.41-3.33c-2.131-2.225-3.195-5.146-3.195-8.773c0-5.412,1.875-10.309,5.633-14.688    c3.75-4.381,7.914-6.57,12.484-6.57c2.375,0,4.275,0.615,5.707,1.84c1.43,1.227,2.145,2.834,2.145,4.826    c0,5.287-5.617,9.574-16.852,12.869c1.02,4.977,3.688,7.469,8.004,7.469c1.686,0,3.293-0.453,4.824-1.357    c1.535-0.908,3.844-2.922,6.934-6.035V339.047z M270.725,331.963c6.535-1.84,9.805-5.234,9.805-10.188    c0-2.451-0.895-3.676-2.68-3.676c-1.686,0-3.293,1.281-4.824,3.85C271.49,324.514,270.725,327.85,270.725,331.963z"
   id="path15" />
		<path
   fill="#010101"
   d="M331.701,339.047c-4.086,3.881-7.01,6.412-8.77,7.588c-1.762,1.174-3.447,1.76-5.057,1.76    c-4.035,0-5.936-3.561-5.707-10.686c-2.553,3.65-4.91,6.344-7.068,8.084c-2.156,1.736-4.383,2.602-6.684,2.602    c-2.244,0-4.152-1.051-5.725-3.158s-2.354-4.691-2.354-7.758c0-3.828,1.051-7.48,3.156-10.955    c2.109-3.473,4.809-6.279,8.102-8.424s6.207-3.219,8.732-3.219c3.193,0,5.428,1.469,6.705,4.404l7.828-4.326h2.148l-3.381,11.221    c-1.736,5.645-2.607,9.514-2.607,11.607c0,2.195,0.777,3.293,2.336,3.293c0.992,0,2.09-0.529,3.291-1.59s2.883-2.676,5.053-4.846    V339.047z M303.664,341.156c2.553,0,4.959-2.176,7.223-6.529c2.26-4.355,3.389-8.373,3.389-12.049c0-1.428-0.322-2.547-0.957-3.35    c-0.641-0.807-1.496-1.207-2.566-1.207c-2.555,0-4.977,2.17-7.258,6.512c-2.285,4.342-3.43,8.338-3.43,11.986    c0,1.381,0.34,2.498,1.016,3.354S302.615,341.156,303.664,341.156z"
   id="path17" />
		<path
   fill="#010101"
   d="M360.314,339.047c-6.41,6.281-11.352,9.424-14.824,9.424c-1.559,0-2.875-0.658-3.945-1.969    c-1.07-1.316-1.609-2.945-1.609-4.887c0-3.6,1.93-8.424,5.785-14.477c-1.891,0.971-3.957,1.645-6.205,2.029    c-1.66,3.064-4.266,6.359-7.814,9.879h-0.879v-3.443c1.99-2.068,3.791-4.291,5.4-6.666c-2.199-0.971-3.295-2.414-3.295-4.326    c0-1.969,0.668-4.068,2.012-6.305c1.34-2.232,3.184-3.348,5.535-3.348c1.992,0,2.986,1.018,2.986,3.062    c0,1.609-0.574,3.906-1.725,6.895c4.238-0.461,7.941-3.701,11.109-9.729l3.484-0.154l-3.562,9.805    c-1.48,4.137-2.438,6.955-2.871,8.447s-0.652,2.816-0.652,3.963c0,1.074,0.25,1.932,0.746,2.566    c0.498,0.643,1.17,0.959,2.012,0.959c0.918,0,1.801-0.314,2.643-0.936c0.842-0.631,2.732-2.359,5.67-5.193V339.047z"
   id="path19" />
		<path
   fill="#010101"
   d="M397.928,339.047c-5.898,6.234-10.957,9.348-15.168,9.348c-1.711,0-3.09-0.6-4.137-1.801    c-1.049-1.199-1.572-2.807-1.572-4.824c0-2.732,1.125-6.908,3.373-12.523c1.199-3.014,1.801-4.932,1.801-5.746    c0-0.818-0.322-1.227-0.957-1.227c-0.357,0-0.832,0.18-1.418,0.535c-0.539,0.357-1.164,0.859-1.879,1.496    c-0.637,0.586-1.354,1.301-2.145,2.141c-0.691,0.721-1.432,1.537-2.219,2.453l-2.148,2.492c-0.943,1.148-1.531,2.359-1.76,3.637    c-0.385,2.17-0.639,4.164-0.768,5.979c-0.078,1.35-0.115,3.174-0.115,5.477l-8.465,1.988c-0.279-3.447-0.422-6.014-0.422-7.697    c0-4.111,0.479-8.006,1.438-11.682c0.957-3.68,2.494-7.814,4.615-12.412l9.344-1.799c-1.965,5.287-3.254,9.447-3.867,12.484    c4.188-4.672,7.508-7.906,9.969-9.709c2.457-1.801,4.645-2.697,6.557-2.697c1.299,0,2.385,0.49,3.25,1.471    c0.869,0.982,1.301,2.215,1.301,3.689c0,2.449-1.098,6.484-3.291,12.104c-1.508,3.854-2.262,6.355-2.262,7.51    c0,1.537,0.627,2.305,1.881,2.305c1.867,0,4.891-2.465,9.064-7.393V339.047z"
   id="path21" />
	</g>
</g>
<rect
   x="273.943"
   y="285.613"
   fill="none"
   width="225.242"
   height="83.948"
   id="rect23" />
<text
   transform="translate(273.9414,302.2061)"
   font-size="23.0795"
   id="text25"
   style="font-size:23.0795002px;fill:#ffffff;font-family:Helvetica">scikit</text>

<rect
   x="345.568"
   y="357.979"
   fill="none"
   width="327.432"
   height="42.963"
   id="rect27" />
<text
   transform="matrix(1 0 0 1 345.5684 375.5176)"
   fill="#010101"
   font-family="'Verdana'"
   font-size="23.0795"
   id="text29">machine learning in Python</text>

</svg>
-
-
- -
-
- - - - -
- -
-
-
-
- - -
- -
-
- - - - - - - -
-
- - -
-
-
- -
-
-
-
-
-
-
- - - - - - -
-
- - - -
- - - Something went wrong with that request. Please try again. -
- - - - diff --git a/doc/themes/scikit-learn/static/img/sloan_logo.jpg b/doc/themes/scikit-learn/static/img/sloan_logo.jpg deleted file mode 100644 index ea714312753a2..0000000000000 Binary files a/doc/themes/scikit-learn/static/img/sloan_logo.jpg and /dev/null differ diff --git a/doc/themes/scikit-learn/static/img/sydney-stacked.jpeg b/doc/themes/scikit-learn/static/img/sydney-stacked.jpeg deleted file mode 100644 index d35e8c724f435..0000000000000 Binary files a/doc/themes/scikit-learn/static/img/sydney-stacked.jpeg and /dev/null differ diff --git a/doc/themes/scikit-learn/static/jquery.js b/doc/themes/scikit-learn/static/jquery.js deleted file mode 100644 index 4c5be4c0fbe23..0000000000000 --- a/doc/themes/scikit-learn/static/jquery.js +++ /dev/null @@ -1,4 +0,0 @@ -/*! jQuery v3.1.1 | (c) jQuery Foundation | jquery.org/license */ -!function(a,b){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=a.document?b(a,!0):function(a){if(!a.document)throw new Error("jQuery requires a window with a document");return b(a)}:b(a)}("undefined"!=typeof window?window:this,function(a,b){"use strict";var c=[],d=a.document,e=Object.getPrototypeOf,f=c.slice,g=c.concat,h=c.push,i=c.indexOf,j={},k=j.toString,l=j.hasOwnProperty,m=l.toString,n=m.call(Object),o={};function p(a,b){b=b||d;var c=b.createElement("script");c.text=a,b.head.appendChild(c).parentNode.removeChild(c)}var q="3.1.1",r=function(a,b){return new r.fn.init(a,b)},s=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,t=/^-ms-/,u=/-([a-z])/g,v=function(a,b){return b.toUpperCase()};r.fn=r.prototype={jquery:q,constructor:r,length:0,toArray:function(){return f.call(this)},get:function(a){return null==a?f.call(this):a<0?this[a+this.length]:this[a]},pushStack:function(a){var b=r.merge(this.constructor(),a);return b.prevObject=this,b},each:function(a){return r.each(this,a)},map:function(a){return this.pushStack(r.map(this,function(b,c){return a.call(b,c,b)}))},slice:function(){return this.pushStack(f.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(a){var b=this.length,c=+a+(a<0?b:0);return this.pushStack(c>=0&&c0&&b-1 in a)}var x=function(a){var b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u="sizzle"+1*new Date,v=a.document,w=0,x=0,y=ha(),z=ha(),A=ha(),B=function(a,b){return a===b&&(l=!0),0},C={}.hasOwnProperty,D=[],E=D.pop,F=D.push,G=D.push,H=D.slice,I=function(a,b){for(var c=0,d=a.length;c+~]|"+K+")"+K+"*"),S=new RegExp("="+K+"*([^\\]'\"]*?)"+K+"*\\]","g"),T=new RegExp(N),U=new RegExp("^"+L+"$"),V={ID:new RegExp("^#("+L+")"),CLASS:new RegExp("^\\.("+L+")"),TAG:new RegExp("^("+L+"|[*])"),ATTR:new RegExp("^"+M),PSEUDO:new RegExp("^"+N),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+K+"*(even|odd|(([+-]|)(\\d*)n|)"+K+"*(?:([+-]|)"+K+"*(\\d+)|))"+K+"*\\)|)","i"),bool:new RegExp("^(?:"+J+")$","i"),needsContext:new RegExp("^"+K+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+K+"*((?:-\\d)?\\d*)"+K+"*\\)|)(?=[^-]|$)","i")},W=/^(?:input|select|textarea|button)$/i,X=/^h\d$/i,Y=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,$=/[+~]/,_=new RegExp("\\\\([\\da-f]{1,6}"+K+"?|("+K+")|.)","ig"),aa=function(a,b,c){var d="0x"+b-65536;return d!==d||c?b:d<0?String.fromCharCode(d+65536):String.fromCharCode(d>>10|55296,1023&d|56320)},ba=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ca=function(a,b){return b?"\0"===a?"\ufffd":a.slice(0,-1)+"\\"+a.charCodeAt(a.length-1).toString(16)+" ":"\\"+a},da=function(){m()},ea=ta(function(a){return a.disabled===!0&&("form"in a||"label"in a)},{dir:"parentNode",next:"legend"});try{G.apply(D=H.call(v.childNodes),v.childNodes),D[v.childNodes.length].nodeType}catch(fa){G={apply:D.length?function(a,b){F.apply(a,H.call(b))}:function(a,b){var c=a.length,d=0;while(a[c++]=b[d++]);a.length=c-1}}}function ga(a,b,d,e){var f,h,j,k,l,o,r,s=b&&b.ownerDocument,w=b?b.nodeType:9;if(d=d||[],"string"!=typeof a||!a||1!==w&&9!==w&&11!==w)return d;if(!e&&((b?b.ownerDocument||b:v)!==n&&m(b),b=b||n,p)){if(11!==w&&(l=Z.exec(a)))if(f=l[1]){if(9===w){if(!(j=b.getElementById(f)))return d;if(j.id===f)return d.push(j),d}else if(s&&(j=s.getElementById(f))&&t(b,j)&&j.id===f)return d.push(j),d}else{if(l[2])return G.apply(d,b.getElementsByTagName(a)),d;if((f=l[3])&&c.getElementsByClassName&&b.getElementsByClassName)return G.apply(d,b.getElementsByClassName(f)),d}if(c.qsa&&!A[a+" "]&&(!q||!q.test(a))){if(1!==w)s=b,r=a;else if("object"!==b.nodeName.toLowerCase()){(k=b.getAttribute("id"))?k=k.replace(ba,ca):b.setAttribute("id",k=u),o=g(a),h=o.length;while(h--)o[h]="#"+k+" "+sa(o[h]);r=o.join(","),s=$.test(a)&&qa(b.parentNode)||b}if(r)try{return G.apply(d,s.querySelectorAll(r)),d}catch(x){}finally{k===u&&b.removeAttribute("id")}}}return i(a.replace(P,"$1"),b,d,e)}function ha(){var a=[];function b(c,e){return a.push(c+" ")>d.cacheLength&&delete b[a.shift()],b[c+" "]=e}return b}function ia(a){return a[u]=!0,a}function ja(a){var b=n.createElement("fieldset");try{return!!a(b)}catch(c){return!1}finally{b.parentNode&&b.parentNode.removeChild(b),b=null}}function ka(a,b){var c=a.split("|"),e=c.length;while(e--)d.attrHandle[c[e]]=b}function la(a,b){var c=b&&a,d=c&&1===a.nodeType&&1===b.nodeType&&a.sourceIndex-b.sourceIndex;if(d)return d;if(c)while(c=c.nextSibling)if(c===b)return-1;return a?1:-1}function ma(a){return function(b){var c=b.nodeName.toLowerCase();return"input"===c&&b.type===a}}function na(a){return function(b){var c=b.nodeName.toLowerCase();return("input"===c||"button"===c)&&b.type===a}}function oa(a){return function(b){return"form"in b?b.parentNode&&b.disabled===!1?"label"in b?"label"in b.parentNode?b.parentNode.disabled===a:b.disabled===a:b.isDisabled===a||b.isDisabled!==!a&&ea(b)===a:b.disabled===a:"label"in b&&b.disabled===a}}function pa(a){return ia(function(b){return b=+b,ia(function(c,d){var e,f=a([],c.length,b),g=f.length;while(g--)c[e=f[g]]&&(c[e]=!(d[e]=c[e]))})})}function qa(a){return a&&"undefined"!=typeof a.getElementsByTagName&&a}c=ga.support={},f=ga.isXML=function(a){var b=a&&(a.ownerDocument||a).documentElement;return!!b&&"HTML"!==b.nodeName},m=ga.setDocument=function(a){var b,e,g=a?a.ownerDocument||a:v;return g!==n&&9===g.nodeType&&g.documentElement?(n=g,o=n.documentElement,p=!f(n),v!==n&&(e=n.defaultView)&&e.top!==e&&(e.addEventListener?e.addEventListener("unload",da,!1):e.attachEvent&&e.attachEvent("onunload",da)),c.attributes=ja(function(a){return a.className="i",!a.getAttribute("className")}),c.getElementsByTagName=ja(function(a){return a.appendChild(n.createComment("")),!a.getElementsByTagName("*").length}),c.getElementsByClassName=Y.test(n.getElementsByClassName),c.getById=ja(function(a){return o.appendChild(a).id=u,!n.getElementsByName||!n.getElementsByName(u).length}),c.getById?(d.filter.ID=function(a){var b=a.replace(_,aa);return function(a){return a.getAttribute("id")===b}},d.find.ID=function(a,b){if("undefined"!=typeof b.getElementById&&p){var c=b.getElementById(a);return c?[c]:[]}}):(d.filter.ID=function(a){var b=a.replace(_,aa);return function(a){var c="undefined"!=typeof a.getAttributeNode&&a.getAttributeNode("id");return c&&c.value===b}},d.find.ID=function(a,b){if("undefined"!=typeof b.getElementById&&p){var c,d,e,f=b.getElementById(a);if(f){if(c=f.getAttributeNode("id"),c&&c.value===a)return[f];e=b.getElementsByName(a),d=0;while(f=e[d++])if(c=f.getAttributeNode("id"),c&&c.value===a)return[f]}return[]}}),d.find.TAG=c.getElementsByTagName?function(a,b){return"undefined"!=typeof b.getElementsByTagName?b.getElementsByTagName(a):c.qsa?b.querySelectorAll(a):void 0}:function(a,b){var c,d=[],e=0,f=b.getElementsByTagName(a);if("*"===a){while(c=f[e++])1===c.nodeType&&d.push(c);return d}return f},d.find.CLASS=c.getElementsByClassName&&function(a,b){if("undefined"!=typeof b.getElementsByClassName&&p)return b.getElementsByClassName(a)},r=[],q=[],(c.qsa=Y.test(n.querySelectorAll))&&(ja(function(a){o.appendChild(a).innerHTML="",a.querySelectorAll("[msallowcapture^='']").length&&q.push("[*^$]="+K+"*(?:''|\"\")"),a.querySelectorAll("[selected]").length||q.push("\\["+K+"*(?:value|"+J+")"),a.querySelectorAll("[id~="+u+"-]").length||q.push("~="),a.querySelectorAll(":checked").length||q.push(":checked"),a.querySelectorAll("a#"+u+"+*").length||q.push(".#.+[+~]")}),ja(function(a){a.innerHTML="";var b=n.createElement("input");b.setAttribute("type","hidden"),a.appendChild(b).setAttribute("name","D"),a.querySelectorAll("[name=d]").length&&q.push("name"+K+"*[*^$|!~]?="),2!==a.querySelectorAll(":enabled").length&&q.push(":enabled",":disabled"),o.appendChild(a).disabled=!0,2!==a.querySelectorAll(":disabled").length&&q.push(":enabled",":disabled"),a.querySelectorAll("*,:x"),q.push(",.*:")})),(c.matchesSelector=Y.test(s=o.matches||o.webkitMatchesSelector||o.mozMatchesSelector||o.oMatchesSelector||o.msMatchesSelector))&&ja(function(a){c.disconnectedMatch=s.call(a,"*"),s.call(a,"[s!='']:x"),r.push("!=",N)}),q=q.length&&new RegExp(q.join("|")),r=r.length&&new RegExp(r.join("|")),b=Y.test(o.compareDocumentPosition),t=b||Y.test(o.contains)?function(a,b){var c=9===a.nodeType?a.documentElement:a,d=b&&b.parentNode;return a===d||!(!d||1!==d.nodeType||!(c.contains?c.contains(d):a.compareDocumentPosition&&16&a.compareDocumentPosition(d)))}:function(a,b){if(b)while(b=b.parentNode)if(b===a)return!0;return!1},B=b?function(a,b){if(a===b)return l=!0,0;var d=!a.compareDocumentPosition-!b.compareDocumentPosition;return d?d:(d=(a.ownerDocument||a)===(b.ownerDocument||b)?a.compareDocumentPosition(b):1,1&d||!c.sortDetached&&b.compareDocumentPosition(a)===d?a===n||a.ownerDocument===v&&t(v,a)?-1:b===n||b.ownerDocument===v&&t(v,b)?1:k?I(k,a)-I(k,b):0:4&d?-1:1)}:function(a,b){if(a===b)return l=!0,0;var c,d=0,e=a.parentNode,f=b.parentNode,g=[a],h=[b];if(!e||!f)return a===n?-1:b===n?1:e?-1:f?1:k?I(k,a)-I(k,b):0;if(e===f)return la(a,b);c=a;while(c=c.parentNode)g.unshift(c);c=b;while(c=c.parentNode)h.unshift(c);while(g[d]===h[d])d++;return d?la(g[d],h[d]):g[d]===v?-1:h[d]===v?1:0},n):n},ga.matches=function(a,b){return ga(a,null,null,b)},ga.matchesSelector=function(a,b){if((a.ownerDocument||a)!==n&&m(a),b=b.replace(S,"='$1']"),c.matchesSelector&&p&&!A[b+" "]&&(!r||!r.test(b))&&(!q||!q.test(b)))try{var d=s.call(a,b);if(d||c.disconnectedMatch||a.document&&11!==a.document.nodeType)return d}catch(e){}return ga(b,n,null,[a]).length>0},ga.contains=function(a,b){return(a.ownerDocument||a)!==n&&m(a),t(a,b)},ga.attr=function(a,b){(a.ownerDocument||a)!==n&&m(a);var e=d.attrHandle[b.toLowerCase()],f=e&&C.call(d.attrHandle,b.toLowerCase())?e(a,b,!p):void 0;return void 0!==f?f:c.attributes||!p?a.getAttribute(b):(f=a.getAttributeNode(b))&&f.specified?f.value:null},ga.escape=function(a){return(a+"").replace(ba,ca)},ga.error=function(a){throw new Error("Syntax error, unrecognized expression: "+a)},ga.uniqueSort=function(a){var b,d=[],e=0,f=0;if(l=!c.detectDuplicates,k=!c.sortStable&&a.slice(0),a.sort(B),l){while(b=a[f++])b===a[f]&&(e=d.push(f));while(e--)a.splice(d[e],1)}return k=null,a},e=ga.getText=function(a){var b,c="",d=0,f=a.nodeType;if(f){if(1===f||9===f||11===f){if("string"==typeof a.textContent)return a.textContent;for(a=a.firstChild;a;a=a.nextSibling)c+=e(a)}else if(3===f||4===f)return a.nodeValue}else while(b=a[d++])c+=e(b);return c},d=ga.selectors={cacheLength:50,createPseudo:ia,match:V,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(a){return a[1]=a[1].replace(_,aa),a[3]=(a[3]||a[4]||a[5]||"").replace(_,aa),"~="===a[2]&&(a[3]=" "+a[3]+" "),a.slice(0,4)},CHILD:function(a){return a[1]=a[1].toLowerCase(),"nth"===a[1].slice(0,3)?(a[3]||ga.error(a[0]),a[4]=+(a[4]?a[5]+(a[6]||1):2*("even"===a[3]||"odd"===a[3])),a[5]=+(a[7]+a[8]||"odd"===a[3])):a[3]&&ga.error(a[0]),a},PSEUDO:function(a){var b,c=!a[6]&&a[2];return V.CHILD.test(a[0])?null:(a[3]?a[2]=a[4]||a[5]||"":c&&T.test(c)&&(b=g(c,!0))&&(b=c.indexOf(")",c.length-b)-c.length)&&(a[0]=a[0].slice(0,b),a[2]=c.slice(0,b)),a.slice(0,3))}},filter:{TAG:function(a){var b=a.replace(_,aa).toLowerCase();return"*"===a?function(){return!0}:function(a){return a.nodeName&&a.nodeName.toLowerCase()===b}},CLASS:function(a){var b=y[a+" "];return b||(b=new RegExp("(^|"+K+")"+a+"("+K+"|$)"))&&y(a,function(a){return b.test("string"==typeof a.className&&a.className||"undefined"!=typeof a.getAttribute&&a.getAttribute("class")||"")})},ATTR:function(a,b,c){return function(d){var e=ga.attr(d,a);return null==e?"!="===b:!b||(e+="","="===b?e===c:"!="===b?e!==c:"^="===b?c&&0===e.indexOf(c):"*="===b?c&&e.indexOf(c)>-1:"$="===b?c&&e.slice(-c.length)===c:"~="===b?(" "+e.replace(O," ")+" ").indexOf(c)>-1:"|="===b&&(e===c||e.slice(0,c.length+1)===c+"-"))}},CHILD:function(a,b,c,d,e){var f="nth"!==a.slice(0,3),g="last"!==a.slice(-4),h="of-type"===b;return 1===d&&0===e?function(a){return!!a.parentNode}:function(b,c,i){var j,k,l,m,n,o,p=f!==g?"nextSibling":"previousSibling",q=b.parentNode,r=h&&b.nodeName.toLowerCase(),s=!i&&!h,t=!1;if(q){if(f){while(p){m=b;while(m=m[p])if(h?m.nodeName.toLowerCase()===r:1===m.nodeType)return!1;o=p="only"===a&&!o&&"nextSibling"}return!0}if(o=[g?q.firstChild:q.lastChild],g&&s){m=q,l=m[u]||(m[u]={}),k=l[m.uniqueID]||(l[m.uniqueID]={}),j=k[a]||[],n=j[0]===w&&j[1],t=n&&j[2],m=n&&q.childNodes[n];while(m=++n&&m&&m[p]||(t=n=0)||o.pop())if(1===m.nodeType&&++t&&m===b){k[a]=[w,n,t];break}}else if(s&&(m=b,l=m[u]||(m[u]={}),k=l[m.uniqueID]||(l[m.uniqueID]={}),j=k[a]||[],n=j[0]===w&&j[1],t=n),t===!1)while(m=++n&&m&&m[p]||(t=n=0)||o.pop())if((h?m.nodeName.toLowerCase()===r:1===m.nodeType)&&++t&&(s&&(l=m[u]||(m[u]={}),k=l[m.uniqueID]||(l[m.uniqueID]={}),k[a]=[w,t]),m===b))break;return t-=e,t===d||t%d===0&&t/d>=0}}},PSEUDO:function(a,b){var c,e=d.pseudos[a]||d.setFilters[a.toLowerCase()]||ga.error("unsupported pseudo: "+a);return e[u]?e(b):e.length>1?(c=[a,a,"",b],d.setFilters.hasOwnProperty(a.toLowerCase())?ia(function(a,c){var d,f=e(a,b),g=f.length;while(g--)d=I(a,f[g]),a[d]=!(c[d]=f[g])}):function(a){return e(a,0,c)}):e}},pseudos:{not:ia(function(a){var b=[],c=[],d=h(a.replace(P,"$1"));return d[u]?ia(function(a,b,c,e){var f,g=d(a,null,e,[]),h=a.length;while(h--)(f=g[h])&&(a[h]=!(b[h]=f))}):function(a,e,f){return b[0]=a,d(b,null,f,c),b[0]=null,!c.pop()}}),has:ia(function(a){return function(b){return ga(a,b).length>0}}),contains:ia(function(a){return a=a.replace(_,aa),function(b){return(b.textContent||b.innerText||e(b)).indexOf(a)>-1}}),lang:ia(function(a){return U.test(a||"")||ga.error("unsupported lang: "+a),a=a.replace(_,aa).toLowerCase(),function(b){var c;do if(c=p?b.lang:b.getAttribute("xml:lang")||b.getAttribute("lang"))return c=c.toLowerCase(),c===a||0===c.indexOf(a+"-");while((b=b.parentNode)&&1===b.nodeType);return!1}}),target:function(b){var c=a.location&&a.location.hash;return c&&c.slice(1)===b.id},root:function(a){return a===o},focus:function(a){return a===n.activeElement&&(!n.hasFocus||n.hasFocus())&&!!(a.type||a.href||~a.tabIndex)},enabled:oa(!1),disabled:oa(!0),checked:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&!!a.checked||"option"===b&&!!a.selected},selected:function(a){return a.parentNode&&a.parentNode.selectedIndex,a.selected===!0},empty:function(a){for(a=a.firstChild;a;a=a.nextSibling)if(a.nodeType<6)return!1;return!0},parent:function(a){return!d.pseudos.empty(a)},header:function(a){return X.test(a.nodeName)},input:function(a){return W.test(a.nodeName)},button:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&"button"===a.type||"button"===b},text:function(a){var b;return"input"===a.nodeName.toLowerCase()&&"text"===a.type&&(null==(b=a.getAttribute("type"))||"text"===b.toLowerCase())},first:pa(function(){return[0]}),last:pa(function(a,b){return[b-1]}),eq:pa(function(a,b,c){return[c<0?c+b:c]}),even:pa(function(a,b){for(var c=0;c=0;)a.push(d);return a}),gt:pa(function(a,b,c){for(var d=c<0?c+b:c;++d1?function(b,c,d){var e=a.length;while(e--)if(!a[e](b,c,d))return!1;return!0}:a[0]}function va(a,b,c){for(var d=0,e=b.length;d-1&&(f[j]=!(g[j]=l))}}else r=wa(r===g?r.splice(o,r.length):r),e?e(null,g,r,i):G.apply(g,r)})}function ya(a){for(var b,c,e,f=a.length,g=d.relative[a[0].type],h=g||d.relative[" "],i=g?1:0,k=ta(function(a){return a===b},h,!0),l=ta(function(a){return I(b,a)>-1},h,!0),m=[function(a,c,d){var e=!g&&(d||c!==j)||((b=c).nodeType?k(a,c,d):l(a,c,d));return b=null,e}];i1&&ua(m),i>1&&sa(a.slice(0,i-1).concat({value:" "===a[i-2].type?"*":""})).replace(P,"$1"),c,i0,e=a.length>0,f=function(f,g,h,i,k){var l,o,q,r=0,s="0",t=f&&[],u=[],v=j,x=f||e&&d.find.TAG("*",k),y=w+=null==v?1:Math.random()||.1,z=x.length;for(k&&(j=g===n||g||k);s!==z&&null!=(l=x[s]);s++){if(e&&l){o=0,g||l.ownerDocument===n||(m(l),h=!p);while(q=a[o++])if(q(l,g||n,h)){i.push(l);break}k&&(w=y)}c&&((l=!q&&l)&&r--,f&&t.push(l))}if(r+=s,c&&s!==r){o=0;while(q=b[o++])q(t,u,g,h);if(f){if(r>0)while(s--)t[s]||u[s]||(u[s]=E.call(i));u=wa(u)}G.apply(i,u),k&&!f&&u.length>0&&r+b.length>1&&ga.uniqueSort(i)}return k&&(w=y,j=v),t};return c?ia(f):f}return h=ga.compile=function(a,b){var c,d=[],e=[],f=A[a+" "];if(!f){b||(b=g(a)),c=b.length;while(c--)f=ya(b[c]),f[u]?d.push(f):e.push(f);f=A(a,za(e,d)),f.selector=a}return f},i=ga.select=function(a,b,c,e){var f,i,j,k,l,m="function"==typeof a&&a,n=!e&&g(a=m.selector||a);if(c=c||[],1===n.length){if(i=n[0]=n[0].slice(0),i.length>2&&"ID"===(j=i[0]).type&&9===b.nodeType&&p&&d.relative[i[1].type]){if(b=(d.find.ID(j.matches[0].replace(_,aa),b)||[])[0],!b)return c;m&&(b=b.parentNode),a=a.slice(i.shift().value.length)}f=V.needsContext.test(a)?0:i.length;while(f--){if(j=i[f],d.relative[k=j.type])break;if((l=d.find[k])&&(e=l(j.matches[0].replace(_,aa),$.test(i[0].type)&&qa(b.parentNode)||b))){if(i.splice(f,1),a=e.length&&sa(i),!a)return G.apply(c,e),c;break}}}return(m||h(a,n))(e,b,!p,c,!b||$.test(a)&&qa(b.parentNode)||b),c},c.sortStable=u.split("").sort(B).join("")===u,c.detectDuplicates=!!l,m(),c.sortDetached=ja(function(a){return 1&a.compareDocumentPosition(n.createElement("fieldset"))}),ja(function(a){return a.innerHTML="","#"===a.firstChild.getAttribute("href")})||ka("type|href|height|width",function(a,b,c){if(!c)return a.getAttribute(b,"type"===b.toLowerCase()?1:2)}),c.attributes&&ja(function(a){return a.innerHTML="",a.firstChild.setAttribute("value",""),""===a.firstChild.getAttribute("value")})||ka("value",function(a,b,c){if(!c&&"input"===a.nodeName.toLowerCase())return a.defaultValue}),ja(function(a){return null==a.getAttribute("disabled")})||ka(J,function(a,b,c){var d;if(!c)return a[b]===!0?b.toLowerCase():(d=a.getAttributeNode(b))&&d.specified?d.value:null}),ga}(a);r.find=x,r.expr=x.selectors,r.expr[":"]=r.expr.pseudos,r.uniqueSort=r.unique=x.uniqueSort,r.text=x.getText,r.isXMLDoc=x.isXML,r.contains=x.contains,r.escapeSelector=x.escape;var y=function(a,b,c){var d=[],e=void 0!==c;while((a=a[b])&&9!==a.nodeType)if(1===a.nodeType){if(e&&r(a).is(c))break;d.push(a)}return d},z=function(a,b){for(var c=[];a;a=a.nextSibling)1===a.nodeType&&a!==b&&c.push(a);return c},A=r.expr.match.needsContext,B=/^<([a-z][^\/\0>:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i,C=/^.[^:#\[\.,]*$/;function D(a,b,c){return r.isFunction(b)?r.grep(a,function(a,d){return!!b.call(a,d,a)!==c}):b.nodeType?r.grep(a,function(a){return a===b!==c}):"string"!=typeof b?r.grep(a,function(a){return i.call(b,a)>-1!==c}):C.test(b)?r.filter(b,a,c):(b=r.filter(b,a),r.grep(a,function(a){return i.call(b,a)>-1!==c&&1===a.nodeType}))}r.filter=function(a,b,c){var d=b[0];return c&&(a=":not("+a+")"),1===b.length&&1===d.nodeType?r.find.matchesSelector(d,a)?[d]:[]:r.find.matches(a,r.grep(b,function(a){return 1===a.nodeType}))},r.fn.extend({find:function(a){var b,c,d=this.length,e=this;if("string"!=typeof a)return this.pushStack(r(a).filter(function(){for(b=0;b1?r.uniqueSort(c):c},filter:function(a){return this.pushStack(D(this,a||[],!1))},not:function(a){return this.pushStack(D(this,a||[],!0))},is:function(a){return!!D(this,"string"==typeof a&&A.test(a)?r(a):a||[],!1).length}});var E,F=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]+))$/,G=r.fn.init=function(a,b,c){var e,f;if(!a)return this;if(c=c||E,"string"==typeof a){if(e="<"===a[0]&&">"===a[a.length-1]&&a.length>=3?[null,a,null]:F.exec(a),!e||!e[1]&&b)return!b||b.jquery?(b||c).find(a):this.constructor(b).find(a);if(e[1]){if(b=b instanceof r?b[0]:b,r.merge(this,r.parseHTML(e[1],b&&b.nodeType?b.ownerDocument||b:d,!0)),B.test(e[1])&&r.isPlainObject(b))for(e in b)r.isFunction(this[e])?this[e](b[e]):this.attr(e,b[e]);return this}return f=d.getElementById(e[2]),f&&(this[0]=f,this.length=1),this}return a.nodeType?(this[0]=a,this.length=1,this):r.isFunction(a)?void 0!==c.ready?c.ready(a):a(r):r.makeArray(a,this)};G.prototype=r.fn,E=r(d);var H=/^(?:parents|prev(?:Until|All))/,I={children:!0,contents:!0,next:!0,prev:!0};r.fn.extend({has:function(a){var b=r(a,this),c=b.length;return this.filter(function(){for(var a=0;a-1:1===c.nodeType&&r.find.matchesSelector(c,a))){f.push(c);break}return this.pushStack(f.length>1?r.uniqueSort(f):f)},index:function(a){return a?"string"==typeof a?i.call(r(a),this[0]):i.call(this,a.jquery?a[0]:a):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(a,b){return this.pushStack(r.uniqueSort(r.merge(this.get(),r(a,b))))},addBack:function(a){return this.add(null==a?this.prevObject:this.prevObject.filter(a))}});function J(a,b){while((a=a[b])&&1!==a.nodeType);return a}r.each({parent:function(a){var b=a.parentNode;return b&&11!==b.nodeType?b:null},parents:function(a){return y(a,"parentNode")},parentsUntil:function(a,b,c){return y(a,"parentNode",c)},next:function(a){return J(a,"nextSibling")},prev:function(a){return J(a,"previousSibling")},nextAll:function(a){return y(a,"nextSibling")},prevAll:function(a){return y(a,"previousSibling")},nextUntil:function(a,b,c){return y(a,"nextSibling",c)},prevUntil:function(a,b,c){return y(a,"previousSibling",c)},siblings:function(a){return z((a.parentNode||{}).firstChild,a)},children:function(a){return z(a.firstChild)},contents:function(a){return a.contentDocument||r.merge([],a.childNodes)}},function(a,b){r.fn[a]=function(c,d){var e=r.map(this,b,c);return"Until"!==a.slice(-5)&&(d=c),d&&"string"==typeof d&&(e=r.filter(d,e)),this.length>1&&(I[a]||r.uniqueSort(e),H.test(a)&&e.reverse()),this.pushStack(e)}});var K=/[^\x20\t\r\n\f]+/g;function L(a){var b={};return r.each(a.match(K)||[],function(a,c){b[c]=!0}),b}r.Callbacks=function(a){a="string"==typeof a?L(a):r.extend({},a);var b,c,d,e,f=[],g=[],h=-1,i=function(){for(e=a.once,d=b=!0;g.length;h=-1){c=g.shift();while(++h-1)f.splice(c,1),c<=h&&h--}),this},has:function(a){return a?r.inArray(a,f)>-1:f.length>0},empty:function(){return f&&(f=[]),this},disable:function(){return e=g=[],f=c="",this},disabled:function(){return!f},lock:function(){return e=g=[],c||b||(f=c=""),this},locked:function(){return!!e},fireWith:function(a,c){return e||(c=c||[],c=[a,c.slice?c.slice():c],g.push(c),b||i()),this},fire:function(){return j.fireWith(this,arguments),this},fired:function(){return!!d}};return j};function M(a){return a}function N(a){throw a}function O(a,b,c){var d;try{a&&r.isFunction(d=a.promise)?d.call(a).done(b).fail(c):a&&r.isFunction(d=a.then)?d.call(a,b,c):b.call(void 0,a)}catch(a){c.call(void 0,a)}}r.extend({Deferred:function(b){var c=[["notify","progress",r.Callbacks("memory"),r.Callbacks("memory"),2],["resolve","done",r.Callbacks("once memory"),r.Callbacks("once memory"),0,"resolved"],["reject","fail",r.Callbacks("once memory"),r.Callbacks("once memory"),1,"rejected"]],d="pending",e={state:function(){return d},always:function(){return f.done(arguments).fail(arguments),this},"catch":function(a){return e.then(null,a)},pipe:function(){var a=arguments;return r.Deferred(function(b){r.each(c,function(c,d){var e=r.isFunction(a[d[4]])&&a[d[4]];f[d[1]](function(){var a=e&&e.apply(this,arguments);a&&r.isFunction(a.promise)?a.promise().progress(b.notify).done(b.resolve).fail(b.reject):b[d[0]+"With"](this,e?[a]:arguments)})}),a=null}).promise()},then:function(b,d,e){var f=0;function g(b,c,d,e){return function(){var h=this,i=arguments,j=function(){var a,j;if(!(b=f&&(d!==N&&(h=void 0,i=[a]),c.rejectWith(h,i))}};b?k():(r.Deferred.getStackHook&&(k.stackTrace=r.Deferred.getStackHook()),a.setTimeout(k))}}return r.Deferred(function(a){c[0][3].add(g(0,a,r.isFunction(e)?e:M,a.notifyWith)),c[1][3].add(g(0,a,r.isFunction(b)?b:M)),c[2][3].add(g(0,a,r.isFunction(d)?d:N))}).promise()},promise:function(a){return null!=a?r.extend(a,e):e}},f={};return r.each(c,function(a,b){var g=b[2],h=b[5];e[b[1]]=g.add,h&&g.add(function(){d=h},c[3-a][2].disable,c[0][2].lock),g.add(b[3].fire),f[b[0]]=function(){return f[b[0]+"With"](this===f?void 0:this,arguments),this},f[b[0]+"With"]=g.fireWith}),e.promise(f),b&&b.call(f,f),f},when:function(a){var b=arguments.length,c=b,d=Array(c),e=f.call(arguments),g=r.Deferred(),h=function(a){return function(c){d[a]=this,e[a]=arguments.length>1?f.call(arguments):c,--b||g.resolveWith(d,e)}};if(b<=1&&(O(a,g.done(h(c)).resolve,g.reject),"pending"===g.state()||r.isFunction(e[c]&&e[c].then)))return g.then();while(c--)O(e[c],h(c),g.reject);return g.promise()}});var P=/^(Eval|Internal|Range|Reference|Syntax|Type|URI)Error$/;r.Deferred.exceptionHook=function(b,c){a.console&&a.console.warn&&b&&P.test(b.name)&&a.console.warn("jQuery.Deferred exception: "+b.message,b.stack,c)},r.readyException=function(b){a.setTimeout(function(){throw b})};var Q=r.Deferred();r.fn.ready=function(a){return Q.then(a)["catch"](function(a){r.readyException(a)}),this},r.extend({isReady:!1,readyWait:1,holdReady:function(a){a?r.readyWait++:r.ready(!0)},ready:function(a){(a===!0?--r.readyWait:r.isReady)||(r.isReady=!0,a!==!0&&--r.readyWait>0||Q.resolveWith(d,[r]))}}),r.ready.then=Q.then;function R(){d.removeEventListener("DOMContentLoaded",R), -a.removeEventListener("load",R),r.ready()}"complete"===d.readyState||"loading"!==d.readyState&&!d.documentElement.doScroll?a.setTimeout(r.ready):(d.addEventListener("DOMContentLoaded",R),a.addEventListener("load",R));var S=function(a,b,c,d,e,f,g){var h=0,i=a.length,j=null==c;if("object"===r.type(c)){e=!0;for(h in c)S(a,b,h,c[h],!0,f,g)}else if(void 0!==d&&(e=!0,r.isFunction(d)||(g=!0),j&&(g?(b.call(a,d),b=null):(j=b,b=function(a,b,c){return j.call(r(a),c)})),b))for(;h1,null,!0)},removeData:function(a){return this.each(function(){W.remove(this,a)})}}),r.extend({queue:function(a,b,c){var d;if(a)return b=(b||"fx")+"queue",d=V.get(a,b),c&&(!d||r.isArray(c)?d=V.access(a,b,r.makeArray(c)):d.push(c)),d||[]},dequeue:function(a,b){b=b||"fx";var c=r.queue(a,b),d=c.length,e=c.shift(),f=r._queueHooks(a,b),g=function(){r.dequeue(a,b)};"inprogress"===e&&(e=c.shift(),d--),e&&("fx"===b&&c.unshift("inprogress"),delete f.stop,e.call(a,g,f)),!d&&f&&f.empty.fire()},_queueHooks:function(a,b){var c=b+"queueHooks";return V.get(a,c)||V.access(a,c,{empty:r.Callbacks("once memory").add(function(){V.remove(a,[b+"queue",c])})})}}),r.fn.extend({queue:function(a,b){var c=2;return"string"!=typeof a&&(b=a,a="fx",c--),arguments.length\x20\t\r\n\f]+)/i,ka=/^$|\/(?:java|ecma)script/i,la={option:[1,""],thead:[1,"","
"],col:[2,"","
"],tr:[2,"","
"],td:[3,"","
"],_default:[0,"",""]};la.optgroup=la.option,la.tbody=la.tfoot=la.colgroup=la.caption=la.thead,la.th=la.td;function ma(a,b){var c;return c="undefined"!=typeof a.getElementsByTagName?a.getElementsByTagName(b||"*"):"undefined"!=typeof a.querySelectorAll?a.querySelectorAll(b||"*"):[],void 0===b||b&&r.nodeName(a,b)?r.merge([a],c):c}function na(a,b){for(var c=0,d=a.length;c-1)e&&e.push(f);else if(j=r.contains(f.ownerDocument,f),g=ma(l.appendChild(f),"script"),j&&na(g),c){k=0;while(f=g[k++])ka.test(f.type||"")&&c.push(f)}return l}!function(){var a=d.createDocumentFragment(),b=a.appendChild(d.createElement("div")),c=d.createElement("input");c.setAttribute("type","radio"),c.setAttribute("checked","checked"),c.setAttribute("name","t"),b.appendChild(c),o.checkClone=b.cloneNode(!0).cloneNode(!0).lastChild.checked,b.innerHTML="",o.noCloneChecked=!!b.cloneNode(!0).lastChild.defaultValue}();var qa=d.documentElement,ra=/^key/,sa=/^(?:mouse|pointer|contextmenu|drag|drop)|click/,ta=/^([^.]*)(?:\.(.+)|)/;function ua(){return!0}function va(){return!1}function wa(){try{return d.activeElement}catch(a){}}function xa(a,b,c,d,e,f){var g,h;if("object"==typeof b){"string"!=typeof c&&(d=d||c,c=void 0);for(h in b)xa(a,h,c,d,b[h],f);return a}if(null==d&&null==e?(e=c,d=c=void 0):null==e&&("string"==typeof c?(e=d,d=void 0):(e=d,d=c,c=void 0)),e===!1)e=va;else if(!e)return a;return 1===f&&(g=e,e=function(a){return r().off(a),g.apply(this,arguments)},e.guid=g.guid||(g.guid=r.guid++)),a.each(function(){r.event.add(this,b,e,d,c)})}r.event={global:{},add:function(a,b,c,d,e){var f,g,h,i,j,k,l,m,n,o,p,q=V.get(a);if(q){c.handler&&(f=c,c=f.handler,e=f.selector),e&&r.find.matchesSelector(qa,e),c.guid||(c.guid=r.guid++),(i=q.events)||(i=q.events={}),(g=q.handle)||(g=q.handle=function(b){return"undefined"!=typeof r&&r.event.triggered!==b.type?r.event.dispatch.apply(a,arguments):void 0}),b=(b||"").match(K)||[""],j=b.length;while(j--)h=ta.exec(b[j])||[],n=p=h[1],o=(h[2]||"").split(".").sort(),n&&(l=r.event.special[n]||{},n=(e?l.delegateType:l.bindType)||n,l=r.event.special[n]||{},k=r.extend({type:n,origType:p,data:d,handler:c,guid:c.guid,selector:e,needsContext:e&&r.expr.match.needsContext.test(e),namespace:o.join(".")},f),(m=i[n])||(m=i[n]=[],m.delegateCount=0,l.setup&&l.setup.call(a,d,o,g)!==!1||a.addEventListener&&a.addEventListener(n,g)),l.add&&(l.add.call(a,k),k.handler.guid||(k.handler.guid=c.guid)),e?m.splice(m.delegateCount++,0,k):m.push(k),r.event.global[n]=!0)}},remove:function(a,b,c,d,e){var f,g,h,i,j,k,l,m,n,o,p,q=V.hasData(a)&&V.get(a);if(q&&(i=q.events)){b=(b||"").match(K)||[""],j=b.length;while(j--)if(h=ta.exec(b[j])||[],n=p=h[1],o=(h[2]||"").split(".").sort(),n){l=r.event.special[n]||{},n=(d?l.delegateType:l.bindType)||n,m=i[n]||[],h=h[2]&&new RegExp("(^|\\.)"+o.join("\\.(?:.*\\.|)")+"(\\.|$)"),g=f=m.length;while(f--)k=m[f],!e&&p!==k.origType||c&&c.guid!==k.guid||h&&!h.test(k.namespace)||d&&d!==k.selector&&("**"!==d||!k.selector)||(m.splice(f,1),k.selector&&m.delegateCount--,l.remove&&l.remove.call(a,k));g&&!m.length&&(l.teardown&&l.teardown.call(a,o,q.handle)!==!1||r.removeEvent(a,n,q.handle),delete i[n])}else for(n in i)r.event.remove(a,n+b[j],c,d,!0);r.isEmptyObject(i)&&V.remove(a,"handle events")}},dispatch:function(a){var b=r.event.fix(a),c,d,e,f,g,h,i=new Array(arguments.length),j=(V.get(this,"events")||{})[b.type]||[],k=r.event.special[b.type]||{};for(i[0]=b,c=1;c=1))for(;j!==this;j=j.parentNode||this)if(1===j.nodeType&&("click"!==a.type||j.disabled!==!0)){for(f=[],g={},c=0;c-1:r.find(e,this,null,[j]).length),g[e]&&f.push(d);f.length&&h.push({elem:j,handlers:f})}return j=this,i\x20\t\r\n\f]*)[^>]*)\/>/gi,za=/\s*$/g;function Da(a,b){return r.nodeName(a,"table")&&r.nodeName(11!==b.nodeType?b:b.firstChild,"tr")?a.getElementsByTagName("tbody")[0]||a:a}function Ea(a){return a.type=(null!==a.getAttribute("type"))+"/"+a.type,a}function Fa(a){var b=Ba.exec(a.type);return b?a.type=b[1]:a.removeAttribute("type"),a}function Ga(a,b){var c,d,e,f,g,h,i,j;if(1===b.nodeType){if(V.hasData(a)&&(f=V.access(a),g=V.set(b,f),j=f.events)){delete g.handle,g.events={};for(e in j)for(c=0,d=j[e].length;c1&&"string"==typeof q&&!o.checkClone&&Aa.test(q))return a.each(function(e){var f=a.eq(e);s&&(b[0]=q.call(this,e,f.html())),Ia(f,b,c,d)});if(m&&(e=pa(b,a[0].ownerDocument,!1,a,d),f=e.firstChild,1===e.childNodes.length&&(e=f),f||d)){for(h=r.map(ma(e,"script"),Ea),i=h.length;l")},clone:function(a,b,c){var d,e,f,g,h=a.cloneNode(!0),i=r.contains(a.ownerDocument,a);if(!(o.noCloneChecked||1!==a.nodeType&&11!==a.nodeType||r.isXMLDoc(a)))for(g=ma(h),f=ma(a),d=0,e=f.length;d0&&na(g,!i&&ma(a,"script")),h},cleanData:function(a){for(var b,c,d,e=r.event.special,f=0;void 0!==(c=a[f]);f++)if(T(c)){if(b=c[V.expando]){if(b.events)for(d in b.events)e[d]?r.event.remove(c,d):r.removeEvent(c,d,b.handle);c[V.expando]=void 0}c[W.expando]&&(c[W.expando]=void 0)}}}),r.fn.extend({detach:function(a){return Ja(this,a,!0)},remove:function(a){return Ja(this,a)},text:function(a){return S(this,function(a){return void 0===a?r.text(this):this.empty().each(function(){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||(this.textContent=a)})},null,a,arguments.length)},append:function(){return Ia(this,arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=Da(this,a);b.appendChild(a)}})},prepend:function(){return Ia(this,arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=Da(this,a);b.insertBefore(a,b.firstChild)}})},before:function(){return Ia(this,arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this)})},after:function(){return Ia(this,arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this.nextSibling)})},empty:function(){for(var a,b=0;null!=(a=this[b]);b++)1===a.nodeType&&(r.cleanData(ma(a,!1)),a.textContent="");return this},clone:function(a,b){return a=null!=a&&a,b=null==b?a:b,this.map(function(){return r.clone(this,a,b)})},html:function(a){return S(this,function(a){var b=this[0]||{},c=0,d=this.length;if(void 0===a&&1===b.nodeType)return b.innerHTML;if("string"==typeof a&&!za.test(a)&&!la[(ja.exec(a)||["",""])[1].toLowerCase()]){a=r.htmlPrefilter(a);try{for(;c1)}});function Ya(a,b,c,d,e){return new Ya.prototype.init(a,b,c,d,e)}r.Tween=Ya,Ya.prototype={constructor:Ya,init:function(a,b,c,d,e,f){this.elem=a,this.prop=c,this.easing=e||r.easing._default,this.options=b,this.start=this.now=this.cur(),this.end=d,this.unit=f||(r.cssNumber[c]?"":"px")},cur:function(){var a=Ya.propHooks[this.prop];return a&&a.get?a.get(this):Ya.propHooks._default.get(this)},run:function(a){var b,c=Ya.propHooks[this.prop];return this.options.duration?this.pos=b=r.easing[this.easing](a,this.options.duration*a,0,1,this.options.duration):this.pos=b=a,this.now=(this.end-this.start)*b+this.start,this.options.step&&this.options.step.call(this.elem,this.now,this),c&&c.set?c.set(this):Ya.propHooks._default.set(this),this}},Ya.prototype.init.prototype=Ya.prototype,Ya.propHooks={_default:{get:function(a){var b;return 1!==a.elem.nodeType||null!=a.elem[a.prop]&&null==a.elem.style[a.prop]?a.elem[a.prop]:(b=r.css(a.elem,a.prop,""),b&&"auto"!==b?b:0)},set:function(a){r.fx.step[a.prop]?r.fx.step[a.prop](a):1!==a.elem.nodeType||null==a.elem.style[r.cssProps[a.prop]]&&!r.cssHooks[a.prop]?a.elem[a.prop]=a.now:r.style(a.elem,a.prop,a.now+a.unit)}}},Ya.propHooks.scrollTop=Ya.propHooks.scrollLeft={set:function(a){a.elem.nodeType&&a.elem.parentNode&&(a.elem[a.prop]=a.now)}},r.easing={linear:function(a){return a},swing:function(a){return.5-Math.cos(a*Math.PI)/2},_default:"swing"},r.fx=Ya.prototype.init,r.fx.step={};var Za,$a,_a=/^(?:toggle|show|hide)$/,ab=/queueHooks$/;function bb(){$a&&(a.requestAnimationFrame(bb),r.fx.tick())}function cb(){return a.setTimeout(function(){Za=void 0}),Za=r.now()}function db(a,b){var c,d=0,e={height:a};for(b=b?1:0;d<4;d+=2-b)c=ba[d],e["margin"+c]=e["padding"+c]=a;return b&&(e.opacity=e.width=a),e}function eb(a,b,c){for(var d,e=(hb.tweeners[b]||[]).concat(hb.tweeners["*"]),f=0,g=e.length;f1)},removeAttr:function(a){return this.each(function(){r.removeAttr(this,a)})}}),r.extend({attr:function(a,b,c){var d,e,f=a.nodeType;if(3!==f&&8!==f&&2!==f)return"undefined"==typeof a.getAttribute?r.prop(a,b,c):(1===f&&r.isXMLDoc(a)||(e=r.attrHooks[b.toLowerCase()]||(r.expr.match.bool.test(b)?ib:void 0)), -void 0!==c?null===c?void r.removeAttr(a,b):e&&"set"in e&&void 0!==(d=e.set(a,c,b))?d:(a.setAttribute(b,c+""),c):e&&"get"in e&&null!==(d=e.get(a,b))?d:(d=r.find.attr(a,b),null==d?void 0:d))},attrHooks:{type:{set:function(a,b){if(!o.radioValue&&"radio"===b&&r.nodeName(a,"input")){var c=a.value;return a.setAttribute("type",b),c&&(a.value=c),b}}}},removeAttr:function(a,b){var c,d=0,e=b&&b.match(K);if(e&&1===a.nodeType)while(c=e[d++])a.removeAttribute(c)}}),ib={set:function(a,b,c){return b===!1?r.removeAttr(a,c):a.setAttribute(c,c),c}},r.each(r.expr.match.bool.source.match(/\w+/g),function(a,b){var c=jb[b]||r.find.attr;jb[b]=function(a,b,d){var e,f,g=b.toLowerCase();return d||(f=jb[g],jb[g]=e,e=null!=c(a,b,d)?g:null,jb[g]=f),e}});var kb=/^(?:input|select|textarea|button)$/i,lb=/^(?:a|area)$/i;r.fn.extend({prop:function(a,b){return S(this,r.prop,a,b,arguments.length>1)},removeProp:function(a){return this.each(function(){delete this[r.propFix[a]||a]})}}),r.extend({prop:function(a,b,c){var d,e,f=a.nodeType;if(3!==f&&8!==f&&2!==f)return 1===f&&r.isXMLDoc(a)||(b=r.propFix[b]||b,e=r.propHooks[b]),void 0!==c?e&&"set"in e&&void 0!==(d=e.set(a,c,b))?d:a[b]=c:e&&"get"in e&&null!==(d=e.get(a,b))?d:a[b]},propHooks:{tabIndex:{get:function(a){var b=r.find.attr(a,"tabindex");return b?parseInt(b,10):kb.test(a.nodeName)||lb.test(a.nodeName)&&a.href?0:-1}}},propFix:{"for":"htmlFor","class":"className"}}),o.optSelected||(r.propHooks.selected={get:function(a){var b=a.parentNode;return b&&b.parentNode&&b.parentNode.selectedIndex,null},set:function(a){var b=a.parentNode;b&&(b.selectedIndex,b.parentNode&&b.parentNode.selectedIndex)}}),r.each(["tabIndex","readOnly","maxLength","cellSpacing","cellPadding","rowSpan","colSpan","useMap","frameBorder","contentEditable"],function(){r.propFix[this.toLowerCase()]=this});function mb(a){var b=a.match(K)||[];return b.join(" ")}function nb(a){return a.getAttribute&&a.getAttribute("class")||""}r.fn.extend({addClass:function(a){var b,c,d,e,f,g,h,i=0;if(r.isFunction(a))return this.each(function(b){r(this).addClass(a.call(this,b,nb(this)))});if("string"==typeof a&&a){b=a.match(K)||[];while(c=this[i++])if(e=nb(c),d=1===c.nodeType&&" "+mb(e)+" "){g=0;while(f=b[g++])d.indexOf(" "+f+" ")<0&&(d+=f+" ");h=mb(d),e!==h&&c.setAttribute("class",h)}}return this},removeClass:function(a){var b,c,d,e,f,g,h,i=0;if(r.isFunction(a))return this.each(function(b){r(this).removeClass(a.call(this,b,nb(this)))});if(!arguments.length)return this.attr("class","");if("string"==typeof a&&a){b=a.match(K)||[];while(c=this[i++])if(e=nb(c),d=1===c.nodeType&&" "+mb(e)+" "){g=0;while(f=b[g++])while(d.indexOf(" "+f+" ")>-1)d=d.replace(" "+f+" "," ");h=mb(d),e!==h&&c.setAttribute("class",h)}}return this},toggleClass:function(a,b){var c=typeof a;return"boolean"==typeof b&&"string"===c?b?this.addClass(a):this.removeClass(a):r.isFunction(a)?this.each(function(c){r(this).toggleClass(a.call(this,c,nb(this),b),b)}):this.each(function(){var b,d,e,f;if("string"===c){d=0,e=r(this),f=a.match(K)||[];while(b=f[d++])e.hasClass(b)?e.removeClass(b):e.addClass(b)}else void 0!==a&&"boolean"!==c||(b=nb(this),b&&V.set(this,"__className__",b),this.setAttribute&&this.setAttribute("class",b||a===!1?"":V.get(this,"__className__")||""))})},hasClass:function(a){var b,c,d=0;b=" "+a+" ";while(c=this[d++])if(1===c.nodeType&&(" "+mb(nb(c))+" ").indexOf(b)>-1)return!0;return!1}});var ob=/\r/g;r.fn.extend({val:function(a){var b,c,d,e=this[0];{if(arguments.length)return d=r.isFunction(a),this.each(function(c){var e;1===this.nodeType&&(e=d?a.call(this,c,r(this).val()):a,null==e?e="":"number"==typeof e?e+="":r.isArray(e)&&(e=r.map(e,function(a){return null==a?"":a+""})),b=r.valHooks[this.type]||r.valHooks[this.nodeName.toLowerCase()],b&&"set"in b&&void 0!==b.set(this,e,"value")||(this.value=e))});if(e)return b=r.valHooks[e.type]||r.valHooks[e.nodeName.toLowerCase()],b&&"get"in b&&void 0!==(c=b.get(e,"value"))?c:(c=e.value,"string"==typeof c?c.replace(ob,""):null==c?"":c)}}}),r.extend({valHooks:{option:{get:function(a){var b=r.find.attr(a,"value");return null!=b?b:mb(r.text(a))}},select:{get:function(a){var b,c,d,e=a.options,f=a.selectedIndex,g="select-one"===a.type,h=g?null:[],i=g?f+1:e.length;for(d=f<0?i:g?f:0;d-1)&&(c=!0);return c||(a.selectedIndex=-1),f}}}}),r.each(["radio","checkbox"],function(){r.valHooks[this]={set:function(a,b){if(r.isArray(b))return a.checked=r.inArray(r(a).val(),b)>-1}},o.checkOn||(r.valHooks[this].get=function(a){return null===a.getAttribute("value")?"on":a.value})});var pb=/^(?:focusinfocus|focusoutblur)$/;r.extend(r.event,{trigger:function(b,c,e,f){var g,h,i,j,k,m,n,o=[e||d],p=l.call(b,"type")?b.type:b,q=l.call(b,"namespace")?b.namespace.split("."):[];if(h=i=e=e||d,3!==e.nodeType&&8!==e.nodeType&&!pb.test(p+r.event.triggered)&&(p.indexOf(".")>-1&&(q=p.split("."),p=q.shift(),q.sort()),k=p.indexOf(":")<0&&"on"+p,b=b[r.expando]?b:new r.Event(p,"object"==typeof b&&b),b.isTrigger=f?2:3,b.namespace=q.join("."),b.rnamespace=b.namespace?new RegExp("(^|\\.)"+q.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,b.result=void 0,b.target||(b.target=e),c=null==c?[b]:r.makeArray(c,[b]),n=r.event.special[p]||{},f||!n.trigger||n.trigger.apply(e,c)!==!1)){if(!f&&!n.noBubble&&!r.isWindow(e)){for(j=n.delegateType||p,pb.test(j+p)||(h=h.parentNode);h;h=h.parentNode)o.push(h),i=h;i===(e.ownerDocument||d)&&o.push(i.defaultView||i.parentWindow||a)}g=0;while((h=o[g++])&&!b.isPropagationStopped())b.type=g>1?j:n.bindType||p,m=(V.get(h,"events")||{})[b.type]&&V.get(h,"handle"),m&&m.apply(h,c),m=k&&h[k],m&&m.apply&&T(h)&&(b.result=m.apply(h,c),b.result===!1&&b.preventDefault());return b.type=p,f||b.isDefaultPrevented()||n._default&&n._default.apply(o.pop(),c)!==!1||!T(e)||k&&r.isFunction(e[p])&&!r.isWindow(e)&&(i=e[k],i&&(e[k]=null),r.event.triggered=p,e[p](),r.event.triggered=void 0,i&&(e[k]=i)),b.result}},simulate:function(a,b,c){var d=r.extend(new r.Event,c,{type:a,isSimulated:!0});r.event.trigger(d,null,b)}}),r.fn.extend({trigger:function(a,b){return this.each(function(){r.event.trigger(a,b,this)})},triggerHandler:function(a,b){var c=this[0];if(c)return r.event.trigger(a,b,c,!0)}}),r.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(a,b){r.fn[b]=function(a,c){return arguments.length>0?this.on(b,null,a,c):this.trigger(b)}}),r.fn.extend({hover:function(a,b){return this.mouseenter(a).mouseleave(b||a)}}),o.focusin="onfocusin"in a,o.focusin||r.each({focus:"focusin",blur:"focusout"},function(a,b){var c=function(a){r.event.simulate(b,a.target,r.event.fix(a))};r.event.special[b]={setup:function(){var d=this.ownerDocument||this,e=V.access(d,b);e||d.addEventListener(a,c,!0),V.access(d,b,(e||0)+1)},teardown:function(){var d=this.ownerDocument||this,e=V.access(d,b)-1;e?V.access(d,b,e):(d.removeEventListener(a,c,!0),V.remove(d,b))}}});var qb=a.location,rb=r.now(),sb=/\?/;r.parseXML=function(b){var c;if(!b||"string"!=typeof b)return null;try{c=(new a.DOMParser).parseFromString(b,"text/xml")}catch(d){c=void 0}return c&&!c.getElementsByTagName("parsererror").length||r.error("Invalid XML: "+b),c};var tb=/\[\]$/,ub=/\r?\n/g,vb=/^(?:submit|button|image|reset|file)$/i,wb=/^(?:input|select|textarea|keygen)/i;function xb(a,b,c,d){var e;if(r.isArray(b))r.each(b,function(b,e){c||tb.test(a)?d(a,e):xb(a+"["+("object"==typeof e&&null!=e?b:"")+"]",e,c,d)});else if(c||"object"!==r.type(b))d(a,b);else for(e in b)xb(a+"["+e+"]",b[e],c,d)}r.param=function(a,b){var c,d=[],e=function(a,b){var c=r.isFunction(b)?b():b;d[d.length]=encodeURIComponent(a)+"="+encodeURIComponent(null==c?"":c)};if(r.isArray(a)||a.jquery&&!r.isPlainObject(a))r.each(a,function(){e(this.name,this.value)});else for(c in a)xb(c,a[c],b,e);return d.join("&")},r.fn.extend({serialize:function(){return r.param(this.serializeArray())},serializeArray:function(){return this.map(function(){var a=r.prop(this,"elements");return a?r.makeArray(a):this}).filter(function(){var a=this.type;return this.name&&!r(this).is(":disabled")&&wb.test(this.nodeName)&&!vb.test(a)&&(this.checked||!ia.test(a))}).map(function(a,b){var c=r(this).val();return null==c?null:r.isArray(c)?r.map(c,function(a){return{name:b.name,value:a.replace(ub,"\r\n")}}):{name:b.name,value:c.replace(ub,"\r\n")}}).get()}});var yb=/%20/g,zb=/#.*$/,Ab=/([?&])_=[^&]*/,Bb=/^(.*?):[ \t]*([^\r\n]*)$/gm,Cb=/^(?:about|app|app-storage|.+-extension|file|res|widget):$/,Db=/^(?:GET|HEAD)$/,Eb=/^\/\//,Fb={},Gb={},Hb="*/".concat("*"),Ib=d.createElement("a");Ib.href=qb.href;function Jb(a){return function(b,c){"string"!=typeof b&&(c=b,b="*");var d,e=0,f=b.toLowerCase().match(K)||[];if(r.isFunction(c))while(d=f[e++])"+"===d[0]?(d=d.slice(1)||"*",(a[d]=a[d]||[]).unshift(c)):(a[d]=a[d]||[]).push(c)}}function Kb(a,b,c,d){var e={},f=a===Gb;function g(h){var i;return e[h]=!0,r.each(a[h]||[],function(a,h){var j=h(b,c,d);return"string"!=typeof j||f||e[j]?f?!(i=j):void 0:(b.dataTypes.unshift(j),g(j),!1)}),i}return g(b.dataTypes[0])||!e["*"]&&g("*")}function Lb(a,b){var c,d,e=r.ajaxSettings.flatOptions||{};for(c in b)void 0!==b[c]&&((e[c]?a:d||(d={}))[c]=b[c]);return d&&r.extend(!0,a,d),a}function Mb(a,b,c){var d,e,f,g,h=a.contents,i=a.dataTypes;while("*"===i[0])i.shift(),void 0===d&&(d=a.mimeType||b.getResponseHeader("Content-Type"));if(d)for(e in h)if(h[e]&&h[e].test(d)){i.unshift(e);break}if(i[0]in c)f=i[0];else{for(e in c){if(!i[0]||a.converters[e+" "+i[0]]){f=e;break}g||(g=e)}f=f||g}if(f)return f!==i[0]&&i.unshift(f),c[f]}function Nb(a,b,c,d){var e,f,g,h,i,j={},k=a.dataTypes.slice();if(k[1])for(g in a.converters)j[g.toLowerCase()]=a.converters[g];f=k.shift();while(f)if(a.responseFields[f]&&(c[a.responseFields[f]]=b),!i&&d&&a.dataFilter&&(b=a.dataFilter(b,a.dataType)),i=f,f=k.shift())if("*"===f)f=i;else if("*"!==i&&i!==f){if(g=j[i+" "+f]||j["* "+f],!g)for(e in j)if(h=e.split(" "),h[1]===f&&(g=j[i+" "+h[0]]||j["* "+h[0]])){g===!0?g=j[e]:j[e]!==!0&&(f=h[0],k.unshift(h[1]));break}if(g!==!0)if(g&&a["throws"])b=g(b);else try{b=g(b)}catch(l){return{state:"parsererror",error:g?l:"No conversion from "+i+" to "+f}}}return{state:"success",data:b}}r.extend({active:0,lastModified:{},etag:{},ajaxSettings:{url:qb.href,type:"GET",isLocal:Cb.test(qb.protocol),global:!0,processData:!0,async:!0,contentType:"application/x-www-form-urlencoded; charset=UTF-8",accepts:{"*":Hb,text:"text/plain",html:"text/html",xml:"application/xml, text/xml",json:"application/json, text/javascript"},contents:{xml:/\bxml\b/,html:/\bhtml/,json:/\bjson\b/},responseFields:{xml:"responseXML",text:"responseText",json:"responseJSON"},converters:{"* text":String,"text html":!0,"text json":JSON.parse,"text xml":r.parseXML},flatOptions:{url:!0,context:!0}},ajaxSetup:function(a,b){return b?Lb(Lb(a,r.ajaxSettings),b):Lb(r.ajaxSettings,a)},ajaxPrefilter:Jb(Fb),ajaxTransport:Jb(Gb),ajax:function(b,c){"object"==typeof b&&(c=b,b=void 0),c=c||{};var e,f,g,h,i,j,k,l,m,n,o=r.ajaxSetup({},c),p=o.context||o,q=o.context&&(p.nodeType||p.jquery)?r(p):r.event,s=r.Deferred(),t=r.Callbacks("once memory"),u=o.statusCode||{},v={},w={},x="canceled",y={readyState:0,getResponseHeader:function(a){var b;if(k){if(!h){h={};while(b=Bb.exec(g))h[b[1].toLowerCase()]=b[2]}b=h[a.toLowerCase()]}return null==b?null:b},getAllResponseHeaders:function(){return k?g:null},setRequestHeader:function(a,b){return null==k&&(a=w[a.toLowerCase()]=w[a.toLowerCase()]||a,v[a]=b),this},overrideMimeType:function(a){return null==k&&(o.mimeType=a),this},statusCode:function(a){var b;if(a)if(k)y.always(a[y.status]);else for(b in a)u[b]=[u[b],a[b]];return this},abort:function(a){var b=a||x;return e&&e.abort(b),A(0,b),this}};if(s.promise(y),o.url=((b||o.url||qb.href)+"").replace(Eb,qb.protocol+"//"),o.type=c.method||c.type||o.method||o.type,o.dataTypes=(o.dataType||"*").toLowerCase().match(K)||[""],null==o.crossDomain){j=d.createElement("a");try{j.href=o.url,j.href=j.href,o.crossDomain=Ib.protocol+"//"+Ib.host!=j.protocol+"//"+j.host}catch(z){o.crossDomain=!0}}if(o.data&&o.processData&&"string"!=typeof o.data&&(o.data=r.param(o.data,o.traditional)),Kb(Fb,o,c,y),k)return y;l=r.event&&o.global,l&&0===r.active++&&r.event.trigger("ajaxStart"),o.type=o.type.toUpperCase(),o.hasContent=!Db.test(o.type),f=o.url.replace(zb,""),o.hasContent?o.data&&o.processData&&0===(o.contentType||"").indexOf("application/x-www-form-urlencoded")&&(o.data=o.data.replace(yb,"+")):(n=o.url.slice(f.length),o.data&&(f+=(sb.test(f)?"&":"?")+o.data,delete o.data),o.cache===!1&&(f=f.replace(Ab,"$1"),n=(sb.test(f)?"&":"?")+"_="+rb++ +n),o.url=f+n),o.ifModified&&(r.lastModified[f]&&y.setRequestHeader("If-Modified-Since",r.lastModified[f]),r.etag[f]&&y.setRequestHeader("If-None-Match",r.etag[f])),(o.data&&o.hasContent&&o.contentType!==!1||c.contentType)&&y.setRequestHeader("Content-Type",o.contentType),y.setRequestHeader("Accept",o.dataTypes[0]&&o.accepts[o.dataTypes[0]]?o.accepts[o.dataTypes[0]]+("*"!==o.dataTypes[0]?", "+Hb+"; q=0.01":""):o.accepts["*"]);for(m in o.headers)y.setRequestHeader(m,o.headers[m]);if(o.beforeSend&&(o.beforeSend.call(p,y,o)===!1||k))return y.abort();if(x="abort",t.add(o.complete),y.done(o.success),y.fail(o.error),e=Kb(Gb,o,c,y)){if(y.readyState=1,l&&q.trigger("ajaxSend",[y,o]),k)return y;o.async&&o.timeout>0&&(i=a.setTimeout(function(){y.abort("timeout")},o.timeout));try{k=!1,e.send(v,A)}catch(z){if(k)throw z;A(-1,z)}}else A(-1,"No Transport");function A(b,c,d,h){var j,m,n,v,w,x=c;k||(k=!0,i&&a.clearTimeout(i),e=void 0,g=h||"",y.readyState=b>0?4:0,j=b>=200&&b<300||304===b,d&&(v=Mb(o,y,d)),v=Nb(o,v,y,j),j?(o.ifModified&&(w=y.getResponseHeader("Last-Modified"),w&&(r.lastModified[f]=w),w=y.getResponseHeader("etag"),w&&(r.etag[f]=w)),204===b||"HEAD"===o.type?x="nocontent":304===b?x="notmodified":(x=v.state,m=v.data,n=v.error,j=!n)):(n=x,!b&&x||(x="error",b<0&&(b=0))),y.status=b,y.statusText=(c||x)+"",j?s.resolveWith(p,[m,x,y]):s.rejectWith(p,[y,x,n]),y.statusCode(u),u=void 0,l&&q.trigger(j?"ajaxSuccess":"ajaxError",[y,o,j?m:n]),t.fireWith(p,[y,x]),l&&(q.trigger("ajaxComplete",[y,o]),--r.active||r.event.trigger("ajaxStop")))}return y},getJSON:function(a,b,c){return r.get(a,b,c,"json")},getScript:function(a,b){return r.get(a,void 0,b,"script")}}),r.each(["get","post"],function(a,b){r[b]=function(a,c,d,e){return r.isFunction(c)&&(e=e||d,d=c,c=void 0),r.ajax(r.extend({url:a,type:b,dataType:e,data:c,success:d},r.isPlainObject(a)&&a))}}),r._evalUrl=function(a){return r.ajax({url:a,type:"GET",dataType:"script",cache:!0,async:!1,global:!1,"throws":!0})},r.fn.extend({wrapAll:function(a){var b;return this[0]&&(r.isFunction(a)&&(a=a.call(this[0])),b=r(a,this[0].ownerDocument).eq(0).clone(!0),this[0].parentNode&&b.insertBefore(this[0]),b.map(function(){var a=this;while(a.firstElementChild)a=a.firstElementChild;return a}).append(this)),this},wrapInner:function(a){return r.isFunction(a)?this.each(function(b){r(this).wrapInner(a.call(this,b))}):this.each(function(){var b=r(this),c=b.contents();c.length?c.wrapAll(a):b.append(a)})},wrap:function(a){var b=r.isFunction(a);return this.each(function(c){r(this).wrapAll(b?a.call(this,c):a)})},unwrap:function(a){return this.parent(a).not("body").each(function(){r(this).replaceWith(this.childNodes)}),this}}),r.expr.pseudos.hidden=function(a){return!r.expr.pseudos.visible(a)},r.expr.pseudos.visible=function(a){return!!(a.offsetWidth||a.offsetHeight||a.getClientRects().length)},r.ajaxSettings.xhr=function(){try{return new a.XMLHttpRequest}catch(b){}};var Ob={0:200,1223:204},Pb=r.ajaxSettings.xhr();o.cors=!!Pb&&"withCredentials"in Pb,o.ajax=Pb=!!Pb,r.ajaxTransport(function(b){var c,d;if(o.cors||Pb&&!b.crossDomain)return{send:function(e,f){var g,h=b.xhr();if(h.open(b.type,b.url,b.async,b.username,b.password),b.xhrFields)for(g in b.xhrFields)h[g]=b.xhrFields[g];b.mimeType&&h.overrideMimeType&&h.overrideMimeType(b.mimeType),b.crossDomain||e["X-Requested-With"]||(e["X-Requested-With"]="XMLHttpRequest");for(g in e)h.setRequestHeader(g,e[g]);c=function(a){return function(){c&&(c=d=h.onload=h.onerror=h.onabort=h.onreadystatechange=null,"abort"===a?h.abort():"error"===a?"number"!=typeof h.status?f(0,"error"):f(h.status,h.statusText):f(Ob[h.status]||h.status,h.statusText,"text"!==(h.responseType||"text")||"string"!=typeof h.responseText?{binary:h.response}:{text:h.responseText},h.getAllResponseHeaders()))}},h.onload=c(),d=h.onerror=c("error"),void 0!==h.onabort?h.onabort=d:h.onreadystatechange=function(){4===h.readyState&&a.setTimeout(function(){c&&d()})},c=c("abort");try{h.send(b.hasContent&&b.data||null)}catch(i){if(c)throw i}},abort:function(){c&&c()}}}),r.ajaxPrefilter(function(a){a.crossDomain&&(a.contents.script=!1)}),r.ajaxSetup({accepts:{script:"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"},contents:{script:/\b(?:java|ecma)script\b/},converters:{"text script":function(a){return r.globalEval(a),a}}}),r.ajaxPrefilter("script",function(a){void 0===a.cache&&(a.cache=!1),a.crossDomain&&(a.type="GET")}),r.ajaxTransport("script",function(a){if(a.crossDomain){var b,c;return{send:function(e,f){b=r(" - - - - diff --git a/doc/tutorial/basic/tutorial.rst b/doc/tutorial/basic/tutorial.rst deleted file mode 100644 index 5199ad226243f..0000000000000 --- a/doc/tutorial/basic/tutorial.rst +++ /dev/null @@ -1,338 +0,0 @@ -.. _introduction: - -An introduction to machine learning with scikit-learn -===================================================== - -.. topic:: Section contents - - In this section, we introduce the `machine learning - `_ - vocabulary that we use throughout scikit-learn and give a - simple learning example. - - -Machine learning: the problem setting -------------------------------------- - -In general, a learning problem considers a set of n -`samples `_ of -data and then tries to predict properties of unknown data. If each sample is -more than a single number and, for instance, a multi-dimensional entry -(aka `multivariate `_ -data), it is said to have several attributes or **features**. - -Learning problems fall into a few categories: - - * `supervised learning `_, - in which the data comes with additional attributes that we want to predict - (:ref:`Click here ` - to go to the scikit-learn supervised learning page).This problem - can be either: - - * `classification - `_: - samples belong to two or more classes and we - want to learn from already labeled data how to predict the class - of unlabeled data. An example of a classification problem would - be handwritten digit recognition, in which the aim is - to assign each input vector to one of a finite number of discrete - categories. Another way to think of classification is as a discrete - (as opposed to continuous) form of supervised learning where one has a - limited number of categories and for each of the n samples provided, - one is to try to label them with the correct category or class. - - * `regression `_: - if the desired output consists of one or more - continuous variables, then the task is called *regression*. An - example of a regression problem would be the prediction of the - length of a salmon as a function of its age and weight. - - * `unsupervised learning `_, - in which the training data consists of a set of input vectors x - without any corresponding target values. The goal in such problems - may be to discover groups of similar examples within the data, where - it is called `clustering `_, - or to determine the distribution of data within the input space, known as - `density estimation `_, or - to project the data from a high-dimensional space down to two or three - dimensions for the purpose of *visualization* - (:ref:`Click here ` - to go to the Scikit-Learn unsupervised learning page). - -.. topic:: Training set and testing set - - Machine learning is about learning some properties of a data set - and then testing those properties against another data set. A common - practice in machine learning is to evaluate an algorithm by splitting a data - set into two. We call one of those sets the **training set**, on which we - learn some properties; we call the other set the **testing set**, on which - we test the learned properties. - - -.. _loading_example_dataset: - -Loading an example dataset --------------------------- - -`scikit-learn` comes with a few standard datasets, for instance the -`iris `_ and `digits -`_ -datasets for classification and the `diabetes dataset -`_ for regression. - -In the following, we start a Python interpreter from our shell and then -load the ``iris`` and ``digits`` datasets. Our notational convention is that -``$`` denotes the shell prompt while ``>>>`` denotes the Python -interpreter prompt:: - - $ python - >>> from sklearn import datasets - >>> iris = datasets.load_iris() - >>> digits = datasets.load_digits() - -A dataset is a dictionary-like object that holds all the data and some -metadata about the data. This data is stored in the ``.data`` member, -which is a ``n_samples, n_features`` array. In the case of supervised -problem, one or more response variables are stored in the ``.target`` member. More -details on the different datasets can be found in the :ref:`dedicated -section `. - -For instance, in the case of the digits dataset, ``digits.data`` gives -access to the features that can be used to classify the digits samples:: - - >>> print(digits.data) - [[ 0. 0. 5. ... 0. 0. 0.] - [ 0. 0. 0. ... 10. 0. 0.] - [ 0. 0. 0. ... 16. 9. 0.] - ... - [ 0. 0. 1. ... 6. 0. 0.] - [ 0. 0. 2. ... 12. 0. 0.] - [ 0. 0. 10. ... 12. 1. 0.]] - -and ``digits.target`` gives the ground truth for the digit dataset, that -is the number corresponding to each digit image that we are trying to -learn:: - - >>> digits.target - array([0, 1, 2, ..., 8, 9, 8]) - -.. topic:: Shape of the data arrays - - The data is always a 2D array, shape ``(n_samples, n_features)``, although - the original data may have had a different shape. In the case of the - digits, each original sample is an image of shape ``(8, 8)`` and can be - accessed using:: - - >>> digits.images[0] - array([[ 0., 0., 5., 13., 9., 1., 0., 0.], - [ 0., 0., 13., 15., 10., 15., 5., 0.], - [ 0., 3., 15., 2., 0., 11., 8., 0.], - [ 0., 4., 12., 0., 0., 8., 8., 0.], - [ 0., 5., 8., 0., 0., 9., 8., 0.], - [ 0., 4., 11., 0., 1., 12., 7., 0.], - [ 0., 2., 14., 5., 10., 12., 0., 0.], - [ 0., 0., 6., 13., 10., 0., 0., 0.]]) - - The :ref:`simple example on this dataset - ` illustrates how starting - from the original problem one can shape the data for consumption in - scikit-learn. - -.. topic:: Loading from external datasets - - To load from an external dataset, please refer to :ref:`loading external datasets `. - -Learning and predicting ------------------------- - -In the case of the digits dataset, the task is to predict, given an image, -which digit it represents. We are given samples of each of the 10 -possible classes (the digits zero through nine) on which we *fit* an -`estimator `_ to be able to *predict* -the classes to which unseen samples belong. - -In scikit-learn, an estimator for classification is a Python object that -implements the methods ``fit(X, y)`` and ``predict(T)``. - -An example of an estimator is the class ``sklearn.svm.SVC``, which -implements `support vector classification -`_. The -estimator's constructor takes as arguments the model's parameters. - -For now, we will consider the estimator as a black box:: - - >>> from sklearn import svm - >>> clf = svm.SVC(gamma=0.001, C=100.) - -.. topic:: Choosing the parameters of the model - - In this example, we set the value of ``gamma`` manually. - To find good values for these parameters, we can use tools - such as :ref:`grid search ` and :ref:`cross validation - `. - -The ``clf`` (for classifier) estimator instance is first -fitted to the model; that is, it must *learn* from the model. This is -done by passing our training set to the ``fit`` method. For the training -set, we'll use all the images from our dataset, except for the last -image, which we'll reserve for our predicting. We select the training set with -the ``[:-1]`` Python syntax, which produces a new array that contains all but -the last item from ``digits.data``:: - - >>> clf.fit(digits.data[:-1], digits.target[:-1]) - SVC(C=100.0, gamma=0.001) - -Now you can *predict* new values. In this case, you'll predict using the last -image from ``digits.data``. By predicting, you'll determine the image from the -training set that best matches the last image. - - - >>> clf.predict(digits.data[-1:]) - array([8]) - -The corresponding image is: - -.. image:: /auto_examples/datasets/images/sphx_glr_plot_digits_last_image_001.png - :target: ../../auto_examples/datasets/plot_digits_last_image.html - :align: center - :scale: 50 - -As you can see, it is a challenging task: after all, the images are of poor -resolution. Do you agree with the classifier? - -A complete example of this classification problem is available as an -example that you can run and study: -:ref:`sphx_glr_auto_examples_classification_plot_digits_classification.py`. - -Conventions ------------ - -scikit-learn estimators follow certain rules to make their behavior more -predictive. These are described in more detail in the :ref:`glossary`. - -Type casting -~~~~~~~~~~~~ - -Unless otherwise specified, input will be cast to ``float64``:: - - >>> import numpy as np - >>> from sklearn import random_projection - - >>> rng = np.random.RandomState(0) - >>> X = rng.rand(10, 2000) - >>> X = np.array(X, dtype='float32') - >>> X.dtype - dtype('float32') - - >>> transformer = random_projection.GaussianRandomProjection() - >>> X_new = transformer.fit_transform(X) - >>> X_new.dtype - dtype('float64') - -In this example, ``X`` is ``float32``, which is cast to ``float64`` by -``fit_transform(X)``. - -Regression targets are cast to ``float64`` and classification targets are -maintained:: - - >>> from sklearn import datasets - >>> from sklearn.svm import SVC - >>> iris = datasets.load_iris() - >>> clf = SVC() - >>> clf.fit(iris.data, iris.target) - SVC() - - >>> list(clf.predict(iris.data[:3])) - [0, 0, 0] - - >>> clf.fit(iris.data, iris.target_names[iris.target]) - SVC() - - >>> list(clf.predict(iris.data[:3])) - ['setosa', 'setosa', 'setosa'] - -Here, the first ``predict()`` returns an integer array, since ``iris.target`` -(an integer array) was used in ``fit``. The second ``predict()`` returns a string -array, since ``iris.target_names`` was for fitting. - -Refitting and updating parameters -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Hyper-parameters of an estimator can be updated after it has been constructed -via the :term:`set_params()` method. Calling ``fit()`` more than -once will overwrite what was learned by any previous ``fit()``:: - - >>> import numpy as np - >>> from sklearn.datasets import load_iris - >>> from sklearn.svm import SVC - >>> X, y = load_iris(return_X_y=True) - - >>> clf = SVC() - >>> clf.set_params(kernel='linear').fit(X, y) - SVC(kernel='linear') - >>> clf.predict(X[:5]) - array([0, 0, 0, 0, 0]) - - >>> clf.set_params(kernel='rbf').fit(X, y) - SVC() - >>> clf.predict(X[:5]) - array([0, 0, 0, 0, 0]) - -Here, the default kernel ``rbf`` is first changed to ``linear`` via -:func:`SVC.set_params()` after the estimator has -been constructed, and changed back to ``rbf`` to refit the estimator and to -make a second prediction. - -Multiclass vs. multilabel fitting -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -When using :class:`multiclass classifiers `, -the learning and prediction task that is performed is dependent on the format of -the target data fit upon:: - - >>> from sklearn.svm import SVC - >>> from sklearn.multiclass import OneVsRestClassifier - >>> from sklearn.preprocessing import LabelBinarizer - - >>> X = [[1, 2], [2, 4], [4, 5], [3, 2], [3, 1]] - >>> y = [0, 0, 1, 1, 2] - - >>> classif = OneVsRestClassifier(estimator=SVC(random_state=0)) - >>> classif.fit(X, y).predict(X) - array([0, 0, 1, 1, 2]) - -In the above case, the classifier is fit on a 1d array of multiclass labels and -the ``predict()`` method therefore provides corresponding multiclass predictions. -It is also possible to fit upon a 2d array of binary label indicators:: - - >>> y = LabelBinarizer().fit_transform(y) - >>> classif.fit(X, y).predict(X) - array([[1, 0, 0], - [1, 0, 0], - [0, 1, 0], - [0, 0, 0], - [0, 0, 0]]) - -Here, the classifier is ``fit()`` on a 2d binary label representation of ``y``, -using the :class:`LabelBinarizer `. -In this case ``predict()`` returns a 2d array representing the corresponding -multilabel predictions. - -Note that the fourth and fifth instances returned all zeroes, indicating that -they matched none of the three labels ``fit`` upon. With multilabel outputs, it -is similarly possible for an instance to be assigned multiple labels:: - - >>> from sklearn.preprocessing import MultiLabelBinarizer - >>> y = [[0, 1], [0, 2], [1, 3], [0, 2, 3], [2, 4]] - >>> y = MultiLabelBinarizer().fit_transform(y) - >>> classif.fit(X, y).predict(X) - array([[1, 1, 0, 0, 0], - [1, 0, 1, 0, 0], - [0, 1, 0, 1, 0], - [1, 0, 1, 0, 0], - [1, 0, 1, 0, 0]]) - -In this case, the classifier is fit upon instances each assigned multiple labels. -The :class:`MultiLabelBinarizer ` is -used to binarize the 2d array of multilabels to ``fit`` upon. As a result, -``predict()`` returns a 2d array with multiple predicted labels for each instance. diff --git a/doc/tutorial/common_includes/info.txt b/doc/tutorial/common_includes/info.txt deleted file mode 100644 index f8e44fec90f2f..0000000000000 --- a/doc/tutorial/common_includes/info.txt +++ /dev/null @@ -1,3 +0,0 @@ -Meant to share common RST file snippets that we want to reuse by inclusion -in the real tutorial in order to lower the maintenance burden -of redundant sections. diff --git a/doc/tutorial/index.rst b/doc/tutorial/index.rst deleted file mode 100644 index cfd63719321f2..0000000000000 --- a/doc/tutorial/index.rst +++ /dev/null @@ -1,38 +0,0 @@ -.. Places global toc into the sidebar - -:globalsidebartoc: True - -.. _tutorial_menu: - - -.. include:: ../includes/big_toc_css.rst -.. include:: ../tune_toc.rst - -====================== -scikit-learn Tutorials -====================== - -| - -.. toctree:: - :maxdepth: 2 - - basic/tutorial.rst - statistical_inference/index.rst - text_analytics/working_with_text_data.rst - machine_learning_map/index - ../presentations - -| - -.. note:: **Doctest Mode** - - The code-examples in the above tutorials are written in a - *python-console* format. If you wish to easily execute these examples - in **IPython**, use:: - - %doctest_mode - - in the IPython-console. You can then simply copy and paste the examples - directly into IPython without having to worry about removing the **>>>** - manually. diff --git a/doc/tutorial/machine_learning_map/ML_MAPS_README.txt b/doc/tutorial/machine_learning_map/ML_MAPS_README.txt deleted file mode 100644 index 47fe633767995..0000000000000 --- a/doc/tutorial/machine_learning_map/ML_MAPS_README.txt +++ /dev/null @@ -1,93 +0,0 @@ -Machine Learning Cheat Sheet (for scikit-learn) -=============================================== - -This document is intended to explain how to edit -the machine learning cheat sheet, originally created -by Andreas Mueller: - -(https://peekaboo-vision.blogspot.de/2013/01/machine-learning-cheat-sheet-for-scikit.html) - -The image is made interactive using an imagemap, and uses the jQuery Map Hilight plugin module -by David Lynch (https://davidlynch.org/projects/maphilight/docs/) to highlight -the different items on the image upon mouseover. - -Modifying the map on the docs is currently a little bit tedious, -so I'll try to make it as simple as possible. - -1. Editing the layout of the map and its paths. ------------------------------------------------- - -Use a Graphics editor like Inkscape Vector Graphics Editor -to open the ml_map.svg file, in this folder. From there -you can move objects around, etc. as you need. - -Save when done, and make sure to export a .PNG file -to replace the old-outdated ml_map.png, as that file -is used as a background image. - -2. Accessing the paths of the SVG file and exporting them. ----------------------------------------------------------- - -Use an image manipulation package like GIMP Image Editor to open -the ml_map.svg file, in this folder. With GIMP, make sure -to select 'Import paths'. - -Once the image has been opened, you can see all imported paths on the paths tab. -You can edit these or create new paths. In GIMP, right-clicking one of the -paths and choosing: Path Tool will allow you to see the paths on -the image. The paths will be exported later and will be used to -make the click able regions on our image map. - -3. Export paths as SVG files ----------------------------- - -After you've edited a path or created a new one, right click it on -the paths menu and choose 'Export Path..'. This way we extract just -that path on its own as 'new_area.svg' for example. - -4. Edit the SVG file ---------------------- -Using a script made by David Lynch, we will convert the svg files into -html maps. To do this, open the svg file in question in any text editor. -Make sure that the 'width' and 'height' are not in 'in' or 'px', i.e -"100" is OK, but "100px" or "1.25in" are not. - -Then wrap the tags in and tags. -Then the file is ready for the script. - -5. From SVG to HTML map ------------------------ - -Use the provided svg2imagemap.py script on your edited svg file: - -$ python svg2imagemap.py new_area.svg - -where new_area.svg is our file. - -6. Add the new map to the main html file ------------------------------------------- - -Copy the code from the newly created 'new_area.html' -file. Open the ml_map.html file. - -Add the that you copied -after the last tag in the ml_map.html file. - -Add the link address to 'href' and a tooltip to -'title' within your tag. - -If you wish to add the green and blue hover effect -to the area, add -data-maphilight='{"strokeColor":"0000ff","strokeWidth":5,"fillColor":"66FF66","fillOpacity":0.4}' - -to your area tag, as done in the other tags above. - -Save the file, and you're done. - ------------------------------------------------------ - -I'll take some time to make some scripts to automate this process -a bit more at some point, as it is not difficult to do, -but tedious. - --Jaques Grobler diff --git a/doc/tutorial/machine_learning_map/index.rst b/doc/tutorial/machine_learning_map/index.rst deleted file mode 100644 index 257bad51b42b4..0000000000000 --- a/doc/tutorial/machine_learning_map/index.rst +++ /dev/null @@ -1,55 +0,0 @@ -.. _ml_map: - - -.. include:: ../../includes/big_toc_css.rst - -Choosing the right estimator -======================================================= - - -Often the hardest part of solving a machine learning problem can -be finding the right estimator for the job. - -Different estimators are better suited for different types of data -and different problems. - -The flowchart below is designed to give users a bit of -a rough guide on how to approach problems with regard to -which estimators to try on your data. - -Click on any estimator in the chart below to see its documentation. - - - -.. raw:: html - - Move mouse over image - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/doc/tutorial/machine_learning_map/parse_path.py b/doc/tutorial/machine_learning_map/parse_path.py deleted file mode 100644 index 770fd1481f53b..0000000000000 --- a/doc/tutorial/machine_learning_map/parse_path.py +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/local/bin/python - -""" -Based on: http://wxpsvg.googlecode.com/svn/trunk/svg/pathdata.py -According to that project, this file is licensed under the LGPL -""" - -try: - from pyparsing import (ParserElement, Literal, Word, CaselessLiteral, - Optional, Combine, Forward, ZeroOrMore, nums, oneOf, Group, ParseException, OneOrMore) -except ImportError: - import sys - sys.exit("pyparsing is required") - - -#ParserElement.enablePackrat() - -def Command(char): - """ Case insensitive but case preserving""" - return CaselessPreservingLiteral(char) - -def Arguments(token): - return Group(token) - - -class CaselessPreservingLiteral(CaselessLiteral): - """ Like CaselessLiteral, but returns the match as found - instead of as defined. - """ - def __init__( self, matchString ): - super().__init__(matchString.upper()) - self.name = "'%s'" % matchString - self.errmsg = "Expected " + self.name - self.myException.msg = self.errmsg - - def parseImpl( self, instring, loc, doActions=True ): - test = instring[ loc:loc+self.matchLen ] - if test.upper() == self.match: - return loc+self.matchLen, test - #~ raise ParseException( instring, loc, self.errmsg ) - exc = self.myException - exc.loc = loc - exc.pstr = instring - raise exc - -def Sequence(token): - """ A sequence of the token""" - return OneOrMore(token+maybeComma) - -digit_sequence = Word(nums) - -sign = oneOf("+ -") - -def convertToFloat(s, loc, toks): - try: - return float(toks[0]) - except BaseException as e: - raise ParseException(loc, "invalid float format %s" % toks[0]) from e - -exponent = CaselessLiteral("e")+Optional(sign)+Word(nums) - -#note that almost all these fields are optional, -#and this can match almost anything. We rely on Pythons built-in -#float() function to clear out invalid values - loosely matching like this -#speeds up parsing quite a lot -floatingPointConstant = Combine( - Optional(sign) + - Optional(Word(nums)) + - Optional(Literal(".") + Optional(Word(nums)))+ - Optional(exponent) -) - -floatingPointConstant.setParseAction(convertToFloat) - -number = floatingPointConstant - -#same as FP constant but don't allow a - sign -nonnegativeNumber = Combine( - Optional(Word(nums)) + - Optional(Literal(".") + Optional(Word(nums)))+ - Optional(exponent) -) -nonnegativeNumber.setParseAction(convertToFloat) - -coordinate = number - -#comma or whitespace can separate values all over the place in SVG -maybeComma = Optional(Literal(',')).suppress() - -coordinateSequence = Sequence(coordinate) - -coordinatePair = (coordinate + maybeComma + coordinate).setParseAction(lambda t: tuple(t)) -coordinatePairSequence = Sequence(coordinatePair) - -coordinatePairPair = coordinatePair + maybeComma + coordinatePair -coordinatePairPairSequence = Sequence(Group(coordinatePairPair)) - -coordinatePairTriple = coordinatePair + maybeComma + coordinatePair + maybeComma + coordinatePair -coordinatePairTripleSequence = Sequence(Group(coordinatePairTriple)) - -#commands -lineTo = Group(Command("L") + Arguments(coordinatePairSequence)) -curve = Group(Command("C") + Arguments(coordinatePairSequence)) - -moveTo = Group(Command("M") + Arguments(coordinatePairSequence)) - -closePath = Group(Command("Z")).setParseAction(lambda t: ('Z', (None,))) - -flag = oneOf("1 0").setParseAction(lambda t: bool(int((t[0])))) - -arcRadius = ( - nonnegativeNumber + maybeComma + #rx - nonnegativeNumber #ry -).setParseAction(lambda t: tuple(t)) - -arcFlags = (flag + maybeComma + flag).setParseAction(lambda t: tuple(t)) - -ellipticalArcArgument = Group( - arcRadius + maybeComma + #rx, ry - number + maybeComma +#rotation - arcFlags + #large-arc-flag, sweep-flag - coordinatePair #(x,y) -) - -ellipticalArc = Group(Command("A") + Arguments(Sequence(ellipticalArcArgument))) - -smoothQuadraticBezierCurveto = Group(Command("T") + Arguments(coordinatePairSequence)) - -quadraticBezierCurveto = Group(Command("Q") + Arguments(coordinatePairPairSequence)) - -smoothCurve = Group(Command("S") + Arguments(coordinatePairPairSequence)) - -#curve = Group(Command("C") + Arguments(coordinatePairTripleSequence)) - -horizontalLine = Group(Command("H") + Arguments(coordinateSequence)) -verticalLine = Group(Command("V") + Arguments(coordinateSequence)) - -drawToCommand = ( - lineTo | moveTo | closePath | ellipticalArc | smoothQuadraticBezierCurveto | - quadraticBezierCurveto | smoothCurve | curve | horizontalLine | verticalLine - ) - -#~ number.debug = True -moveToDrawToCommands = moveTo + ZeroOrMore(drawToCommand) - -path = ZeroOrMore(moveToDrawToCommands) -path.keepTabs = True - -def get_points(d): - commands = path.parseString(d) - points = [] - currentset = None - for command in commands: - if command[0] == 'M' or command[0] == 'm': - currentset = [] - points.append(currentset) - currentset.append(command[1][-1]) - elif command[0] == 'L' or command[0] == 'l': - currentset.extend(command[1]) - elif command[0] == 'C' or command[0] == 'c': - currentset.extend(command[1]) - return points - -if __name__ == "__main__": - s = ("M 242.96145,653.59282 L 244.83646,650.1553 L 247.02397,649.8428 " - "L 247.33647,650.62405 L 245.30521,653.59282 L 242.96145,653.59282 z " - "M 252.80525,649.99905 L 258.74278,652.49906 L 260.77404,652.18656 " - "L 262.33654,648.43654 L 261.71154,645.15528 L 257.64902,644.68653 " - "L 253.74275,646.40528 L 252.80525,649.99905 z M 282.49289,659.6866 " - "L 286.08665,664.99912 L 288.43041,664.68662 L 289.52417,664.21787 " - "L 290.93042,665.46787 L 294.52419,665.31162 L 295.4617,663.90537 " - "L 292.64918,662.18661 L 290.77417,658.59284 L 288.74291,655.15533 " - "L 283.11789,657.96784 L 282.49289,659.6866 z M 302.02423,668.28039 " - "L 303.27423,666.40538 L 307.8055,667.34288 L 308.43051,666.87413 " - "L 314.36803,667.49913 L 314.05553,668.74914 L 311.55552,670.15539 " - "L 307.33675,669.84289 L 302.02423,668.28039 z M 307.1805,673.28041 " - "L 309.05551,677.03043 L 312.02427,675.93667 L 312.33677,674.37416 " - "L 310.77427,672.3429 L 307.1805,672.0304 L 307.1805,673.28041 z " - "M 313.89928,672.18665 L 316.08679,669.37414 L 320.61806,671.7179 " - "L 324.83683,672.81166 L 329.0556,675.46792 L 329.0556,677.34293 " - "L 325.61809,679.06169 L 320.93056,679.99919 L 318.5868,678.59293 " - "L 313.89928,672.18665 z M 329.99311,687.18672 L 331.55561,685.93672 " - "L 334.83688,687.49923 L 342.18066,690.93674 L 345.46193,692.968 " - "L 347.02443,695.31176 L 348.89944,699.53053 L 352.80571,702.03054 " - "L 352.49321,703.28055 L 348.74319,706.40556 L 344.68067,707.81182 " - "L 343.27442,707.18682 L 340.30565,708.90557 L 337.96189,712.03059 " - "L 335.77438,714.8431 L 334.05562,714.68685 L 330.61811,712.18684 " - "L 330.30561,707.81182 L 330.93061,705.46806 L 329.3681,699.99928 " - "L 327.33684,698.28052 L 327.18059,695.78051 L 329.3681,694.84301 " - "L 331.39936,691.87425 L 331.86811,690.93674 L 330.30561,689.21798 " - "L 329.99311,687.18672 z ") - print(path.parseString(s)) diff --git a/doc/tutorial/machine_learning_map/pyparsing.py b/doc/tutorial/machine_learning_map/pyparsing.py deleted file mode 100644 index 0c5fca5cf891d..0000000000000 --- a/doc/tutorial/machine_learning_map/pyparsing.py +++ /dev/null @@ -1,5715 +0,0 @@ -# module pyparsing.py -# -# Copyright (c) 2003-2016 Paul T. McGuire -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# -# flake8: noqa - -__doc__ = \ -""" -pyparsing module - Classes and methods to define and execute parsing grammars - -The pyparsing module is an alternative approach to creating and executing simple grammars, -vs. the traditional lex/yacc approach, or the use of regular expressions. With pyparsing, you -don't need to learn a new syntax for defining grammars or matching expressions - the parsing module -provides a library of classes that you use to construct the grammar directly in Python. - -Here is a program to parse "Hello, World!" (or any greeting of the form -C{", !"}), built up using L{Word}, L{Literal}, and L{And} elements -(L{'+'} operator gives L{And} expressions, strings are auto-converted to -L{Literal} expressions):: - - from pyparsing import Word, alphas - - # define grammar of a greeting - greet = Word(alphas) + "," + Word(alphas) + "!" - - hello = "Hello, World!" - print (hello, "->", greet.parseString(hello)) - -The program outputs the following:: - - Hello, World! -> ['Hello', ',', 'World', '!'] - -The Python representation of the grammar is quite readable, owing to the self-explanatory -class names, and the use of '+', '|' and '^' operators. - -The L{ParseResults} object returned from L{ParserElement.parseString} can be accessed as a nested list, a dictionary, or an -object with named attributes. - -The pyparsing module handles some of the problems that are typically vexing when writing text parsers: - - extra or missing whitespace (the above program will also handle "Hello,World!", "Hello , World !", etc.) - - quoted strings - - embedded comments -""" - -__version__ = "2.2.0" -__versionTime__ = "06 Mar 2017 02:06 UTC" -__author__ = "Paul McGuire " - -import string -from weakref import ref as wkref -import copy -import sys -import warnings -import re -import sre_constants -import collections -import pprint -import traceback -import types -from datetime import datetime - -try: - from _thread import RLock -except ImportError: - from threading import RLock - -try: - from collections import OrderedDict as _OrderedDict -except ImportError: - try: - from ordereddict import OrderedDict as _OrderedDict - except ImportError: - _OrderedDict = None - -#~ sys.stderr.write( "testing pyparsing module, version %s, %s\n" % (__version__,__versionTime__ ) ) - -__all__ = [ -'And', 'CaselessKeyword', 'CaselessLiteral', 'CharsNotIn', 'Combine', 'Dict', 'Each', 'Empty', -'FollowedBy', 'Forward', 'GoToColumn', 'Group', 'Keyword', 'LineEnd', 'LineStart', 'Literal', -'MatchFirst', 'NoMatch', 'NotAny', 'OneOrMore', 'OnlyOnce', 'Optional', 'Or', -'ParseBaseException', 'ParseElementEnhance', 'ParseException', 'ParseExpression', 'ParseFatalException', -'ParseResults', 'ParseSyntaxException', 'ParserElement', 'QuotedString', 'RecursiveGrammarException', -'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter', -'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore', -'alphanums', 'alphas', 'alphas8bit', 'anyCloseTag', 'anyOpenTag', 'cStyleComment', 'col', -'commaSeparatedList', 'commonHTMLEntity', 'countedArray', 'cppStyleComment', 'dblQuotedString', -'dblSlashComment', 'delimitedList', 'dictOf', 'downcaseTokens', 'empty', 'hexnums', -'htmlComment', 'javaStyleComment', 'line', 'lineEnd', 'lineStart', 'lineno', -'makeHTMLTags', 'makeXMLTags', 'matchOnlyAtCol', 'matchPreviousExpr', 'matchPreviousLiteral', -'nestedExpr', 'nullDebugAction', 'nums', 'oneOf', 'opAssoc', 'operatorPrecedence', 'printables', -'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity', -'replaceWith', 'restOfLine', 'sglQuotedString', 'srange', 'stringEnd', -'stringStart', 'traceParseAction', 'unicodeString', 'upcaseTokens', 'withAttribute', -'indentedBlock', 'originalTextFor', 'ungroup', 'infixNotation','locatedExpr', 'withClass', -'CloseMatch', 'tokenMap', 'pyparsing_common', -] - -system_version = tuple(sys.version_info)[:3] -PY_3 = system_version[0] == 3 -if PY_3: - _MAX_INT = sys.maxsize - basestring = str - unichr = chr - _ustr = str - - # build list of single arg builtins, that can be used as parse actions - singleArgBuiltins = [sum, len, sorted, reversed, list, tuple, set, any, all, min, max] - -else: - _MAX_INT = sys.maxint - range = xrange - - def _ustr(obj): - """Drop-in replacement for str(obj) that tries to be Unicode friendly. It first tries - str(obj). If that fails with a UnicodeEncodeError, then it tries unicode(obj). It - then < returns the unicode object | encodes it with the default encoding | ... >. - """ - if isinstance(obj,unicode): - return obj - - try: - # If this works, then _ustr(obj) has the same behaviour as str(obj), so - # it won't break any existing code. - return str(obj) - - except UnicodeEncodeError: - # Else encode it - ret = unicode(obj).encode(sys.getdefaultencoding(), 'xmlcharrefreplace') - xmlcharref = Regex(r'&#\d+;') - xmlcharref.setParseAction(lambda t: '\\u' + hex(int(t[0][2:-1]))[2:]) - return xmlcharref.transformString(ret) - - # build list of single arg builtins, tolerant of Python version, that can be used as parse actions - singleArgBuiltins = [] - import __builtin__ - for fname in "sum len sorted reversed list tuple set any all min max".split(): - try: - singleArgBuiltins.append(getattr(__builtin__,fname)) - except AttributeError: - continue - -_generatorType = type((y for y in range(1))) - -def _xml_escape(data): - """Escape &, <, >, ", ', etc. in a string of data.""" - - # ampersand must be replaced first - from_symbols = '&><"\'' - to_symbols = ('&'+s+';' for s in "amp gt lt quot apos".split()) - for from_,to_ in zip(from_symbols, to_symbols): - data = data.replace(from_, to_) - return data - -class _Constants(object): - pass - -alphas = string.ascii_uppercase + string.ascii_lowercase -nums = "0123456789" -hexnums = nums + "ABCDEFabcdef" -alphanums = alphas + nums -_bslash = chr(92) -printables = "".join(c for c in string.printable if c not in string.whitespace) - -class ParseBaseException(Exception): - """base exception class for all parsing runtime exceptions""" - # Performance tuning: we construct a *lot* of these, so keep this - # constructor as small and fast as possible - def __init__( self, pstr, loc=0, msg=None, elem=None ): - self.loc = loc - if msg is None: - self.msg = pstr - self.pstr = "" - else: - self.msg = msg - self.pstr = pstr - self.parserElement = elem - self.args = (pstr, loc, msg) - - @classmethod - def _from_exception(cls, pe): - """ - internal factory method to simplify creating one type of ParseException - from another - avoids having __init__ signature conflicts among subclasses - """ - return cls(pe.pstr, pe.loc, pe.msg, pe.parserElement) - - def __getattr__( self, aname ): - """supported attributes by name are: - - lineno - returns the line number of the exception text - - col - returns the column number of the exception text - - line - returns the line containing the exception text - """ - if( aname == "lineno" ): - return lineno( self.loc, self.pstr ) - elif( aname in ("col", "column") ): - return col( self.loc, self.pstr ) - elif( aname == "line" ): - return line( self.loc, self.pstr ) - else: - raise AttributeError(aname) - - def __str__( self ): - return "%s (at char %d), (line:%d, col:%d)" % \ - ( self.msg, self.loc, self.lineno, self.column ) - def __repr__( self ): - return _ustr(self) - def markInputline( self, markerString = ">!<" ): - """Extracts the exception line from the input string, and marks - the location of the exception with a special symbol. - """ - line_str = self.line - line_column = self.column - 1 - if markerString: - line_str = "".join((line_str[:line_column], - markerString, line_str[line_column:])) - return line_str.strip() - def __dir__(self): - return "lineno col line".split() + dir(type(self)) - -class ParseException(ParseBaseException): - """ - Exception thrown when parse expressions don't match class; - supported attributes by name are: - - lineno - returns the line number of the exception text - - col - returns the column number of the exception text - - line - returns the line containing the exception text - - Example:: - try: - Word(nums).setName("integer").parseString("ABC") - except ParseException as pe: - print(pe) - print("column: {}".format(pe.col)) - - prints:: - Expected integer (at char 0), (line:1, col:1) - column: 1 - """ - pass - -class ParseFatalException(ParseBaseException): - """user-throwable exception thrown when inconsistent parse content - is found; stops all parsing immediately""" - pass - -class ParseSyntaxException(ParseFatalException): - """just like L{ParseFatalException}, but thrown internally when an - L{ErrorStop} ('-' operator) indicates that parsing is to stop - immediately because an unbacktrackable syntax error has been found""" - pass - -#~ class ReparseException(ParseBaseException): - #~ """Experimental class - parse actions can raise this exception to cause - #~ pyparsing to reparse the input string: - #~ - with a modified input string, and/or - #~ - with a modified start location - #~ Set the values of the ReparseException in the constructor, and raise the - #~ exception in a parse action to cause pyparsing to use the new string/location. - #~ Setting the values as None causes no change to be made. - #~ """ - #~ def __init_( self, newstring, restartLoc ): - #~ self.newParseText = newstring - #~ self.reparseLoc = restartLoc - -class RecursiveGrammarException(Exception): - """exception thrown by L{ParserElement.validate} if the grammar could be improperly recursive""" - def __init__( self, parseElementList ): - self.parseElementTrace = parseElementList - - def __str__( self ): - return "RecursiveGrammarException: %s" % self.parseElementTrace - -class _ParseResultsWithOffset(object): - def __init__(self,p1,p2): - self.tup = (p1,p2) - def __getitem__(self,i): - return self.tup[i] - def __repr__(self): - return repr(self.tup[0]) - def setOffset(self,i): - self.tup = (self.tup[0],i) - -class ParseResults(object): - """ - Structured parse results, to provide multiple means of access to the parsed data: - - as a list (C{len(results)}) - - by list index (C{results[0], results[1]}, etc.) - - by attribute (C{results.} - see L{ParserElement.setResultsName}) - - Example:: - integer = Word(nums) - date_str = (integer.setResultsName("year") + '/' - + integer.setResultsName("month") + '/' - + integer.setResultsName("day")) - # equivalent form: - # date_str = integer("year") + '/' + integer("month") + '/' + integer("day") - - # parseString returns a ParseResults object - result = date_str.parseString("1999/12/31") - - def test(s, fn=repr): - print("%s -> %s" % (s, fn(eval(s)))) - test("list(result)") - test("result[0]") - test("result['month']") - test("result.day") - test("'month' in result") - test("'minutes' in result") - test("result.dump()", str) - prints:: - list(result) -> ['1999', '/', '12', '/', '31'] - result[0] -> '1999' - result['month'] -> '12' - result.day -> '31' - 'month' in result -> True - 'minutes' in result -> False - result.dump() -> ['1999', '/', '12', '/', '31'] - - day: 31 - - month: 12 - - year: 1999 - """ - def __new__(cls, toklist=None, name=None, asList=True, modal=True ): - if isinstance(toklist, cls): - return toklist - retobj = object.__new__(cls) - retobj.__doinit = True - return retobj - - # Performance tuning: we construct a *lot* of these, so keep this - # constructor as small and fast as possible - def __init__( self, toklist=None, name=None, asList=True, modal=True, isinstance=isinstance ): - if self.__doinit: - self.__doinit = False - self.__name = None - self.__parent = None - self.__accumNames = {} - self.__asList = asList - self.__modal = modal - if toklist is None: - toklist = [] - if isinstance(toklist, list): - self.__toklist = toklist[:] - elif isinstance(toklist, _generatorType): - self.__toklist = list(toklist) - else: - self.__toklist = [toklist] - self.__tokdict = dict() - - if name is not None and name: - if not modal: - self.__accumNames[name] = 0 - if isinstance(name,int): - name = _ustr(name) # will always return a str, but use _ustr for consistency - self.__name = name - if not (isinstance(toklist, (type(None), basestring, list)) and toklist in (None,'',[])): - if isinstance(toklist,basestring): - toklist = [ toklist ] - if asList: - if isinstance(toklist,ParseResults): - self[name] = _ParseResultsWithOffset(toklist.copy(),0) - else: - self[name] = _ParseResultsWithOffset(ParseResults(toklist[0]),0) - self[name].__name = name - else: - try: - self[name] = toklist[0] - except (KeyError,TypeError,IndexError): - self[name] = toklist - - def __getitem__( self, i ): - if isinstance( i, (int,slice) ): - return self.__toklist[i] - else: - if i not in self.__accumNames: - return self.__tokdict[i][-1][0] - else: - return ParseResults([ v[0] for v in self.__tokdict[i] ]) - - def __setitem__( self, k, v, isinstance=isinstance ): - if isinstance(v,_ParseResultsWithOffset): - self.__tokdict[k] = self.__tokdict.get(k,list()) + [v] - sub = v[0] - elif isinstance(k,(int,slice)): - self.__toklist[k] = v - sub = v - else: - self.__tokdict[k] = self.__tokdict.get(k,list()) + [_ParseResultsWithOffset(v,0)] - sub = v - if isinstance(sub,ParseResults): - sub.__parent = wkref(self) - - def __delitem__( self, i ): - if isinstance(i,(int,slice)): - mylen = len( self.__toklist ) - del self.__toklist[i] - - # convert int to slice - if isinstance(i, int): - if i < 0: - i += mylen - i = slice(i, i+1) - # get removed indices - removed = list(range(*i.indices(mylen))) - removed.reverse() - # fixup indices in token dictionary - for name,occurrences in self.__tokdict.items(): - for j in removed: - for k, (value, position) in enumerate(occurrences): - occurrences[k] = _ParseResultsWithOffset(value, position - (position > j)) - else: - del self.__tokdict[i] - - def __contains__( self, k ): - return k in self.__tokdict - - def __len__( self ): return len( self.__toklist ) - def __bool__(self): return ( not not self.__toklist ) - __nonzero__ = __bool__ - def __iter__( self ): return iter( self.__toklist ) - def __reversed__( self ): return iter( self.__toklist[::-1] ) - def _iterkeys( self ): - if hasattr(self.__tokdict, "iterkeys"): - return self.__tokdict.iterkeys() - else: - return iter(self.__tokdict) - - def _itervalues( self ): - return (self[k] for k in self._iterkeys()) - - def _iteritems( self ): - return ((k, self[k]) for k in self._iterkeys()) - - if PY_3: - keys = _iterkeys - """Returns an iterator of all named result keys (Python 3.x only).""" - - values = _itervalues - """Returns an iterator of all named result values (Python 3.x only).""" - - items = _iteritems - """Returns an iterator of all named result key-value tuples (Python 3.x only).""" - - else: - iterkeys = _iterkeys - """Returns an iterator of all named result keys (Python 2.x only).""" - - itervalues = _itervalues - """Returns an iterator of all named result values (Python 2.x only).""" - - iteritems = _iteritems - """Returns an iterator of all named result key-value tuples (Python 2.x only).""" - - def keys( self ): - """Returns all named result keys (as a list in Python 2.x, as an iterator in Python 3.x).""" - return list(self.iterkeys()) - - def values( self ): - """Returns all named result values (as a list in Python 2.x, as an iterator in Python 3.x).""" - return list(self.itervalues()) - - def items( self ): - """Returns all named result key-values (as a list of tuples in Python 2.x, as an iterator in Python 3.x).""" - return list(self.iteritems()) - - def haskeys( self ): - """Since keys() returns an iterator, this method is helpful in bypassing - code that looks for the existence of any defined results names.""" - return bool(self.__tokdict) - - def pop( self, *args, **kwargs): - """ - Removes and returns item at specified index (default=C{last}). - Supports both C{list} and C{dict} semantics for C{pop()}. If passed no - argument or an integer argument, it will use C{list} semantics - and pop tokens from the list of parsed tokens. If passed a - non-integer argument (most likely a string), it will use C{dict} - semantics and pop the corresponding value from any defined - results names. A second default return value argument is - supported, just as in C{dict.pop()}. - - Example:: - def remove_first(tokens): - tokens.pop(0) - print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321'] - print(OneOrMore(Word(nums)).addParseAction(remove_first).parseString("0 123 321")) # -> ['123', '321'] - - label = Word(alphas) - patt = label("LABEL") + OneOrMore(Word(nums)) - print(patt.parseString("AAB 123 321").dump()) - - # Use pop() in a parse action to remove named result (note that corresponding value is not - # removed from list form of results) - def remove_LABEL(tokens): - tokens.pop("LABEL") - return tokens - patt.addParseAction(remove_LABEL) - print(patt.parseString("AAB 123 321").dump()) - prints:: - ['AAB', '123', '321'] - - LABEL: AAB - - ['AAB', '123', '321'] - """ - if not args: - args = [-1] - for k,v in kwargs.items(): - if k == 'default': - args = (args[0], v) - else: - raise TypeError("pop() got an unexpected keyword argument '%s'" % k) - if (isinstance(args[0], int) or - len(args) == 1 or - args[0] in self): - index = args[0] - ret = self[index] - del self[index] - return ret - else: - defaultvalue = args[1] - return defaultvalue - - def get(self, key, defaultValue=None): - """ - Returns named result matching the given key, or if there is no - such name, then returns the given C{defaultValue} or C{None} if no - C{defaultValue} is specified. - - Similar to C{dict.get()}. - - Example:: - integer = Word(nums) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") - - result = date_str.parseString("1999/12/31") - print(result.get("year")) # -> '1999' - print(result.get("hour", "not specified")) # -> 'not specified' - print(result.get("hour")) # -> None - """ - if key in self: - return self[key] - else: - return defaultValue - - def insert( self, index, insStr ): - """ - Inserts new element at location index in the list of parsed tokens. - - Similar to C{list.insert()}. - - Example:: - print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321'] - - # use a parse action to insert the parse location in the front of the parsed results - def insert_locn(locn, tokens): - tokens.insert(0, locn) - print(OneOrMore(Word(nums)).addParseAction(insert_locn).parseString("0 123 321")) # -> [0, '0', '123', '321'] - """ - self.__toklist.insert(index, insStr) - # fixup indices in token dictionary - for name,occurrences in self.__tokdict.items(): - for k, (value, position) in enumerate(occurrences): - occurrences[k] = _ParseResultsWithOffset(value, position + (position > index)) - - def append( self, item ): - """ - Add single element to end of ParseResults list of elements. - - Example:: - print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321'] - - # use a parse action to compute the sum of the parsed integers, and add it to the end - def append_sum(tokens): - tokens.append(sum(map(int, tokens))) - print(OneOrMore(Word(nums)).addParseAction(append_sum).parseString("0 123 321")) # -> ['0', '123', '321', 444] - """ - self.__toklist.append(item) - - def extend( self, itemseq ): - """ - Add sequence of elements to end of ParseResults list of elements. - - Example:: - patt = OneOrMore(Word(alphas)) - - # use a parse action to append the reverse of the matched strings, to make a palindrome - def make_palindrome(tokens): - tokens.extend(reversed([t[::-1] for t in tokens])) - return ''.join(tokens) - print(patt.addParseAction(make_palindrome).parseString("lskdj sdlkjf lksd")) # -> 'lskdjsdlkjflksddsklfjkldsjdksl' - """ - if isinstance(itemseq, ParseResults): - self += itemseq - else: - self.__toklist.extend(itemseq) - - def clear( self ): - """ - Clear all elements and results names. - """ - del self.__toklist[:] - self.__tokdict.clear() - - def __getattr__( self, name ): - try: - return self[name] - except KeyError: - return "" - - if name in self.__tokdict: - if name not in self.__accumNames: - return self.__tokdict[name][-1][0] - else: - return ParseResults([ v[0] for v in self.__tokdict[name] ]) - else: - return "" - - def __add__( self, other ): - ret = self.copy() - ret += other - return ret - - def __iadd__( self, other ): - if other.__tokdict: - offset = len(self.__toklist) - addoffset = lambda a: offset if a<0 else a+offset - otheritems = other.__tokdict.items() - otherdictitems = [(k, _ParseResultsWithOffset(v[0],addoffset(v[1])) ) - for (k,vlist) in otheritems for v in vlist] - for k,v in otherdictitems: - self[k] = v - if isinstance(v[0],ParseResults): - v[0].__parent = wkref(self) - - self.__toklist += other.__toklist - self.__accumNames.update( other.__accumNames ) - return self - - def __radd__(self, other): - if isinstance(other,int) and other == 0: - # useful for merging many ParseResults using sum() builtin - return self.copy() - else: - # this may raise a TypeError - so be it - return other + self - - def __repr__( self ): - return "(%s, %s)" % ( repr( self.__toklist ), repr( self.__tokdict ) ) - - def __str__( self ): - return '[' + ', '.join(_ustr(i) if isinstance(i, ParseResults) else repr(i) for i in self.__toklist) + ']' - - def _asStringList( self, sep='' ): - out = [] - for item in self.__toklist: - if out and sep: - out.append(sep) - if isinstance( item, ParseResults ): - out += item._asStringList() - else: - out.append( _ustr(item) ) - return out - - def asList( self ): - """ - Returns the parse results as a nested list of matching tokens, all converted to strings. - - Example:: - patt = OneOrMore(Word(alphas)) - result = patt.parseString("sldkj lsdkj sldkj") - # even though the result prints in string-like form, it is actually a pyparsing ParseResults - print(type(result), result) # -> ['sldkj', 'lsdkj', 'sldkj'] - - # Use asList() to create an actual list - result_list = result.asList() - print(type(result_list), result_list) # -> ['sldkj', 'lsdkj', 'sldkj'] - """ - return [res.asList() if isinstance(res,ParseResults) else res for res in self.__toklist] - - def asDict( self ): - """ - Returns the named parse results as a nested dictionary. - - Example:: - integer = Word(nums) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") - - result = date_str.parseString('12/31/1999') - print(type(result), repr(result)) # -> (['12', '/', '31', '/', '1999'], {'day': [('1999', 4)], 'year': [('12', 0)], 'month': [('31', 2)]}) - - result_dict = result.asDict() - print(type(result_dict), repr(result_dict)) # -> {'day': '1999', 'year': '12', 'month': '31'} - - # even though a ParseResults supports dict-like access, sometime you just need to have a dict - import json - print(json.dumps(result)) # -> Exception: TypeError: ... is not JSON serializable - print(json.dumps(result.asDict())) # -> {"month": "31", "day": "1999", "year": "12"} - """ - if PY_3: - item_fn = self.items - else: - item_fn = self.iteritems - - def toItem(obj): - if isinstance(obj, ParseResults): - if obj.haskeys(): - return obj.asDict() - else: - return [toItem(v) for v in obj] - else: - return obj - - return dict((k,toItem(v)) for k,v in item_fn()) - - def copy( self ): - """ - Returns a new copy of a C{ParseResults} object. - """ - ret = ParseResults( self.__toklist ) - ret.__tokdict = self.__tokdict.copy() - ret.__parent = self.__parent - ret.__accumNames.update( self.__accumNames ) - ret.__name = self.__name - return ret - - def asXML( self, doctag=None, namedItemsOnly=False, indent="", formatted=True ): - """ - (Deprecated) Returns the parse results as XML. Tags are created for tokens and lists that have defined results names. - """ - nl = "\n" - out = [] - namedItems = dict((v[1],k) for (k,vlist) in self.__tokdict.items() - for v in vlist) - nextLevelIndent = indent + " " - - # collapse out indents if formatting is not desired - if not formatted: - indent = "" - nextLevelIndent = "" - nl = "" - - selfTag = None - if doctag is not None: - selfTag = doctag - else: - if self.__name: - selfTag = self.__name - - if not selfTag: - if namedItemsOnly: - return "" - else: - selfTag = "ITEM" - - out += [ nl, indent, "<", selfTag, ">" ] - - for i,res in enumerate(self.__toklist): - if isinstance(res,ParseResults): - if i in namedItems: - out += [ res.asXML(namedItems[i], - namedItemsOnly and doctag is None, - nextLevelIndent, - formatted)] - else: - out += [ res.asXML(None, - namedItemsOnly and doctag is None, - nextLevelIndent, - formatted)] - else: - # individual token, see if there is a name for it - resTag = None - if i in namedItems: - resTag = namedItems[i] - if not resTag: - if namedItemsOnly: - continue - else: - resTag = "ITEM" - xmlBodyText = _xml_escape(_ustr(res)) - out += [ nl, nextLevelIndent, "<", resTag, ">", - xmlBodyText, - "" ] - - out += [ nl, indent, "" ] - return "".join(out) - - def __lookup(self,sub): - for k,vlist in self.__tokdict.items(): - for v,loc in vlist: - if sub is v: - return k - return None - - def getName(self): - r""" - Returns the results name for this token expression. Useful when several - different expressions might match at a particular location. - - Example:: - integer = Word(nums) - ssn_expr = Regex(r"\d\d\d-\d\d-\d\d\d\d") - house_number_expr = Suppress('#') + Word(nums, alphanums) - user_data = (Group(house_number_expr)("house_number") - | Group(ssn_expr)("ssn") - | Group(integer)("age")) - user_info = OneOrMore(user_data) - - result = user_info.parseString("22 111-22-3333 #221B") - for item in result: - print(item.getName(), ':', item[0]) - prints:: - age : 22 - ssn : 111-22-3333 - house_number : 221B - """ - if self.__name: - return self.__name - elif self.__parent: - par = self.__parent() - if par: - return par.__lookup(self) - else: - return None - elif (len(self) == 1 and - len(self.__tokdict) == 1 and - next(iter(self.__tokdict.values()))[0][1] in (0,-1)): - return next(iter(self.__tokdict.keys())) - else: - return None - - def dump(self, indent='', depth=0, full=True): - """ - Diagnostic method for listing out the contents of a C{ParseResults}. - Accepts an optional C{indent} argument so that this string can be embedded - in a nested display of other data. - - Example:: - integer = Word(nums) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") - - result = date_str.parseString('12/31/1999') - print(result.dump()) - prints:: - ['12', '/', '31', '/', '1999'] - - day: 1999 - - month: 31 - - year: 12 - """ - out = [] - NL = '\n' - out.append( indent+_ustr(self.asList()) ) - if full: - if self.haskeys(): - items = sorted((str(k), v) for k,v in self.items()) - for k,v in items: - if out: - out.append(NL) - out.append( "%s%s- %s: " % (indent,(' '*depth), k) ) - if isinstance(v,ParseResults): - if v: - out.append( v.dump(indent,depth+1) ) - else: - out.append(_ustr(v)) - else: - out.append(repr(v)) - elif any(isinstance(vv,ParseResults) for vv in self): - v = self - for i,vv in enumerate(v): - if isinstance(vv,ParseResults): - out.append("\n%s%s[%d]:\n%s%s%s" % (indent,(' '*(depth)),i,indent,(' '*(depth+1)),vv.dump(indent,depth+1) )) - else: - out.append("\n%s%s[%d]:\n%s%s%s" % (indent,(' '*(depth)),i,indent,(' '*(depth+1)),_ustr(vv))) - - return "".join(out) - - def pprint(self, *args, **kwargs): - """ - Pretty-printer for parsed results as a list, using the C{pprint} module. - Accepts additional positional or keyword args as defined for the - C{pprint.pprint} method. (U{https://docs.python.org/3/library/pprint.html#pprint.pprint}) - - Example:: - ident = Word(alphas, alphanums) - num = Word(nums) - func = Forward() - term = ident | num | Group('(' + func + ')') - func <<= ident + Group(Optional(delimitedList(term))) - result = func.parseString("fna a,b,(fnb c,d,200),100") - result.pprint(width=40) - prints:: - ['fna', - ['a', - 'b', - ['(', 'fnb', ['c', 'd', '200'], ')'], - '100']] - """ - pprint.pprint(self.asList(), *args, **kwargs) - - # add support for pickle protocol - def __getstate__(self): - return ( self.__toklist, - ( self.__tokdict.copy(), - self.__parent is not None and self.__parent() or None, - self.__accumNames, - self.__name ) ) - - def __setstate__(self,state): - self.__toklist = state[0] - (self.__tokdict, - par, - inAccumNames, - self.__name) = state[1] - self.__accumNames = {} - self.__accumNames.update(inAccumNames) - if par is not None: - self.__parent = wkref(par) - else: - self.__parent = None - - def __getnewargs__(self): - return self.__toklist, self.__name, self.__asList, self.__modal - - def __dir__(self): - return (dir(type(self)) + list(self.keys())) - -collections.MutableMapping.register(ParseResults) - -def col (loc,strg): - """Returns current column within a string, counting newlines as line separators. - The first column is number 1. - - Note: the default parsing behavior is to expand tabs in the input string - before starting the parsing process. See L{I{ParserElement.parseString}} for more information - on parsing strings containing C{}s, and suggested methods to maintain a - consistent view of the parsed string, the parse location, and line and column - positions within the parsed string. - """ - s = strg - return 1 if 0} for more information - on parsing strings containing C{}s, and suggested methods to maintain a - consistent view of the parsed string, the parse location, and line and column - positions within the parsed string. - """ - return strg.count("\n",0,loc) + 1 - -def line( loc, strg ): - """Returns the line of text containing loc within a string, counting newlines as line separators. - """ - lastCR = strg.rfind("\n", 0, loc) - nextCR = strg.find("\n", loc) - if nextCR >= 0: - return strg[lastCR+1:nextCR] - else: - return strg[lastCR+1:] - -def _defaultStartDebugAction( instring, loc, expr ): - print (("Match " + _ustr(expr) + " at loc " + _ustr(loc) + "(%d,%d)" % ( lineno(loc,instring), col(loc,instring) ))) - -def _defaultSuccessDebugAction( instring, startloc, endloc, expr, toks ): - print ("Matched " + _ustr(expr) + " -> " + str(toks.asList())) - -def _defaultExceptionDebugAction( instring, loc, expr, exc ): - print ("Exception raised:" + _ustr(exc)) - -def nullDebugAction(*args): - """'Do-nothing' debug action, to suppress debugging output during parsing.""" - pass - -# Only works on Python 3.x - nonlocal is toxic to Python 2 installs -#~ 'decorator to trim function calls to match the arity of the target' -#~ def _trim_arity(func, maxargs=3): - #~ if func in singleArgBuiltins: - #~ return lambda s,l,t: func(t) - #~ limit = 0 - #~ foundArity = False - #~ def wrapper(*args): - #~ nonlocal limit,foundArity - #~ while 1: - #~ try: - #~ ret = func(*args[limit:]) - #~ foundArity = True - #~ return ret - #~ except TypeError: - #~ if limit == maxargs or foundArity: - #~ raise - #~ limit += 1 - #~ continue - #~ return wrapper - -# this version is Python 2.x-3.x cross-compatible -'decorator to trim function calls to match the arity of the target' -def _trim_arity(func, maxargs=2): - if func in singleArgBuiltins: - return lambda s,l,t: func(t) - limit = [0] - foundArity = [False] - - def extract_stack(limit=0): - offset = -2 - frame_summary = traceback.extract_stack(limit=-offset+limit-1)[offset] - return [(frame_summary.filename, frame_summary.lineno)] - def extract_tb(tb, limit=0): - frames = traceback.extract_tb(tb, limit=limit) - frame_summary = frames[-1] - return [(frame_summary.filename, frame_summary.lineno)] - - # synthesize what would be returned by traceback.extract_stack at the call to - # user's parse action 'func', so that we don't incur call penalty at parse time - - LINE_DIFF = 6 - # IF ANY CODE CHANGES, EVEN JUST COMMENTS OR BLANK LINES, BETWEEN THE NEXT LINE AND - # THE CALL TO FUNC INSIDE WRAPPER, LINE_DIFF MUST BE MODIFIED!!!! - this_line = extract_stack(limit=2)[-1] - pa_call_line_synth = (this_line[0], this_line[1]+LINE_DIFF) - - def wrapper(*args): - while 1: - try: - ret = func(*args[limit[0]:]) - foundArity[0] = True - return ret - except TypeError: - # re-raise TypeErrors if they did not come from our arity testing - if foundArity[0]: - raise - else: - try: - tb = sys.exc_info()[-1] - if not extract_tb(tb, limit=2)[-1][:2] == pa_call_line_synth: - raise - finally: - del tb - - if limit[0] <= maxargs: - limit[0] += 1 - continue - raise - - # copy func name to wrapper for sensible debug output - func_name = "" - try: - func_name = getattr(func, '__name__', - getattr(func, '__class__').__name__) - except Exception: - func_name = str(func) - wrapper.__name__ = func_name - - return wrapper - -class ParserElement(object): - """Abstract base level parser element class.""" - DEFAULT_WHITE_CHARS = " \n\t\r" - verbose_stacktrace = False - - @staticmethod - def setDefaultWhitespaceChars( chars ): - r""" - Overrides the default whitespace chars - - Example:: - # default whitespace chars are space, and newline - OneOrMore(Word(alphas)).parseString("abc def\nghi jkl") # -> ['abc', 'def', 'ghi', 'jkl'] - - # change to just treat newline as significant - ParserElement.setDefaultWhitespaceChars(" \t") - OneOrMore(Word(alphas)).parseString("abc def\nghi jkl") # -> ['abc', 'def'] - """ - ParserElement.DEFAULT_WHITE_CHARS = chars - - @staticmethod - def inlineLiteralsUsing(cls): - """ - Set class to be used for inclusion of string literals into a parser. - - Example:: - # default literal class used is Literal - integer = Word(nums) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") - - date_str.parseString("1999/12/31") # -> ['1999', '/', '12', '/', '31'] - - - # change to Suppress - ParserElement.inlineLiteralsUsing(Suppress) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") - - date_str.parseString("1999/12/31") # -> ['1999', '12', '31'] - """ - ParserElement._literalStringClass = cls - - def __init__( self, savelist=False ): - self.parseAction = list() - self.failAction = None - #~ self.name = "" # don't define self.name, let subclasses try/except upcall - self.strRepr = None - self.resultsName = None - self.saveAsList = savelist - self.skipWhitespace = True - self.whiteChars = ParserElement.DEFAULT_WHITE_CHARS - self.copyDefaultWhiteChars = True - self.mayReturnEmpty = False # used when checking for left-recursion - self.keepTabs = False - self.ignoreExprs = list() - self.debug = False - self.streamlined = False - self.mayIndexError = True # used to optimize exception handling for subclasses that don't advance parse index - self.errmsg = "" - self.modalResults = True # used to mark results names as modal (report only last) or cumulative (list all) - self.debugActions = ( None, None, None ) #custom debug actions - self.re = None - self.callPreparse = True # used to avoid redundant calls to preParse - self.callDuringTry = False - - def copy( self ): - """ - Make a copy of this C{ParserElement}. Useful for defining different parse actions - for the same parsing pattern, using copies of the original parse element. - - Example:: - integer = Word(nums).setParseAction(lambda toks: int(toks[0])) - integerK = integer.copy().addParseAction(lambda toks: toks[0]*1024) + Suppress("K") - integerM = integer.copy().addParseAction(lambda toks: toks[0]*1024*1024) + Suppress("M") - - print(OneOrMore(integerK | integerM | integer).parseString("5K 100 640K 256M")) - prints:: - [5120, 100, 655360, 268435456] - Equivalent form of C{expr.copy()} is just C{expr()}:: - integerM = integer().addParseAction(lambda toks: toks[0]*1024*1024) + Suppress("M") - """ - cpy = copy.copy( self ) - cpy.parseAction = self.parseAction[:] - cpy.ignoreExprs = self.ignoreExprs[:] - if self.copyDefaultWhiteChars: - cpy.whiteChars = ParserElement.DEFAULT_WHITE_CHARS - return cpy - - def setName( self, name ): - """ - Define name for this expression, makes debugging and exception messages clearer. - - Example:: - Word(nums).parseString("ABC") # -> Exception: Expected W:(0123...) (at char 0), (line:1, col:1) - Word(nums).setName("integer").parseString("ABC") # -> Exception: Expected integer (at char 0), (line:1, col:1) - """ - self.name = name - self.errmsg = "Expected " + self.name - if hasattr(self,"exception"): - self.exception.msg = self.errmsg - return self - - def setResultsName( self, name, listAllMatches=False ): - """ - Define name for referencing matching tokens as a nested attribute - of the returned parse results. - NOTE: this returns a *copy* of the original C{ParserElement} object; - this is so that the client can define a basic element, such as an - integer, and reference it in multiple places with different names. - - You can also set results names using the abbreviated syntax, - C{expr("name")} in place of C{expr.setResultsName("name")} - - see L{I{__call__}<__call__>}. - - Example:: - date_str = (integer.setResultsName("year") + '/' - + integer.setResultsName("month") + '/' - + integer.setResultsName("day")) - - # equivalent form: - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") - """ - newself = self.copy() - if name.endswith("*"): - name = name[:-1] - listAllMatches=True - newself.resultsName = name - newself.modalResults = not listAllMatches - return newself - - def setBreak(self,breakFlag = True): - """Method to invoke the Python pdb debugger when this element is - about to be parsed. Set C{breakFlag} to True to enable, False to - disable. - """ - if breakFlag: - _parseMethod = self._parse - def breaker(instring, loc, doActions=True, callPreParse=True): - import pdb - pdb.set_trace() - return _parseMethod( instring, loc, doActions, callPreParse ) - breaker._originalParseMethod = _parseMethod - self._parse = breaker - else: - if hasattr(self._parse,"_originalParseMethod"): - self._parse = self._parse._originalParseMethod - return self - - def setParseAction( self, *fns, **kwargs ): - """ - Define one or more actions to perform when successfully matching parse element definition. - Parse action fn is a callable method with 0-3 arguments, called as C{fn(s,loc,toks)}, - C{fn(loc,toks)}, C{fn(toks)}, or just C{fn()}, where: - - s = the original string being parsed (see note below) - - loc = the location of the matching substring - - toks = a list of the matched tokens, packaged as a C{L{ParseResults}} object - If the functions in fns modify the tokens, they can return them as the return - value from fn, and the modified list of tokens will replace the original. - Otherwise, fn does not need to return any value. - - Optional keyword arguments: - - callDuringTry = (default=C{False}) indicate if parse action should be run during lookaheads and alternate testing - - Note: the default parsing behavior is to expand tabs in the input string - before starting the parsing process. See L{I{parseString}} for more information - on parsing strings containing C{}s, and suggested methods to maintain a - consistent view of the parsed string, the parse location, and line and column - positions within the parsed string. - - Example:: - integer = Word(nums) - date_str = integer + '/' + integer + '/' + integer - - date_str.parseString("1999/12/31") # -> ['1999', '/', '12', '/', '31'] - - # use parse action to convert to ints at parse time - integer = Word(nums).setParseAction(lambda toks: int(toks[0])) - date_str = integer + '/' + integer + '/' + integer - - # note that integer fields are now ints, not strings - date_str.parseString("1999/12/31") # -> [1999, '/', 12, '/', 31] - """ - self.parseAction = list(map(_trim_arity, list(fns))) - self.callDuringTry = kwargs.get("callDuringTry", False) - return self - - def addParseAction( self, *fns, **kwargs ): - """ - Add one or more parse actions to expression's list of parse actions. See L{I{setParseAction}}. - - See examples in L{I{copy}}. - """ - self.parseAction += list(map(_trim_arity, list(fns))) - self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False) - return self - - def addCondition(self, *fns, **kwargs): - """Add a boolean predicate function to expression's list of parse actions. See - L{I{setParseAction}} for function call signatures. Unlike C{setParseAction}, - functions passed to C{addCondition} need to return boolean success/fail of the condition. - - Optional keyword arguments: - - message = define a custom message to be used in the raised exception - - fatal = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise ParseException - - Example:: - integer = Word(nums).setParseAction(lambda toks: int(toks[0])) - year_int = integer.copy() - year_int.addCondition(lambda toks: toks[0] >= 2000, message="Only support years 2000 and later") - date_str = year_int + '/' + integer + '/' + integer - - result = date_str.parseString("1999/12/31") # -> Exception: Only support years 2000 and later (at char 0), (line:1, col:1) - """ - msg = kwargs.get("message", "failed user-defined condition") - exc_type = ParseFatalException if kwargs.get("fatal", False) else ParseException - for fn in fns: - def pa(s,l,t): - if not bool(_trim_arity(fn)(s,l,t)): - raise exc_type(s,l,msg) - self.parseAction.append(pa) - self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False) - return self - - def setFailAction( self, fn ): - """Define action to perform if parsing fails at this expression. - Fail action fn is a callable function that takes the arguments - C{fn(s,loc,expr,err)} where: - - s = string being parsed - - loc = location where expression match was attempted and failed - - expr = the parse expression that failed - - err = the exception thrown - The function returns no value. It may throw C{L{ParseFatalException}} - if it is desired to stop parsing immediately.""" - self.failAction = fn - return self - - def _skipIgnorables( self, instring, loc ): - exprsFound = True - while exprsFound: - exprsFound = False - for e in self.ignoreExprs: - try: - while 1: - loc,dummy = e._parse( instring, loc ) - exprsFound = True - except ParseException: - pass - return loc - - def preParse( self, instring, loc ): - if self.ignoreExprs: - loc = self._skipIgnorables( instring, loc ) - - if self.skipWhitespace: - wt = self.whiteChars - instrlen = len(instring) - while loc < instrlen and instring[loc] in wt: - loc += 1 - - return loc - - def parseImpl( self, instring, loc, doActions=True ): - return loc, [] - - def postParse( self, instring, loc, tokenlist ): - return tokenlist - - #~ @profile - def _parseNoCache( self, instring, loc, doActions=True, callPreParse=True ): - debugging = ( self.debug ) #and doActions ) - - if debugging or self.failAction: - #~ print ("Match",self,"at loc",loc,"(%d,%d)" % ( lineno(loc,instring), col(loc,instring) )) - if (self.debugActions[0] ): - self.debugActions[0]( instring, loc, self ) - if callPreParse and self.callPreparse: - preloc = self.preParse( instring, loc ) - else: - preloc = loc - tokensStart = preloc - try: - try: - loc,tokens = self.parseImpl( instring, preloc, doActions ) - except IndexError: - raise ParseException( instring, len(instring), self.errmsg, self ) - except ParseBaseException as err: - #~ print ("Exception raised:", err) - if self.debugActions[2]: - self.debugActions[2]( instring, tokensStart, self, err ) - if self.failAction: - self.failAction( instring, tokensStart, self, err ) - raise - else: - if callPreParse and self.callPreparse: - preloc = self.preParse( instring, loc ) - else: - preloc = loc - tokensStart = preloc - if self.mayIndexError or loc >= len(instring): - try: - loc,tokens = self.parseImpl( instring, preloc, doActions ) - except IndexError: - raise ParseException( instring, len(instring), self.errmsg, self ) - else: - loc,tokens = self.parseImpl( instring, preloc, doActions ) - - tokens = self.postParse( instring, loc, tokens ) - - retTokens = ParseResults( tokens, self.resultsName, asList=self.saveAsList, modal=self.modalResults ) - if self.parseAction and (doActions or self.callDuringTry): - if debugging: - try: - for fn in self.parseAction: - tokens = fn( instring, tokensStart, retTokens ) - if tokens is not None: - retTokens = ParseResults( tokens, - self.resultsName, - asList=self.saveAsList and isinstance(tokens,(ParseResults,list)), - modal=self.modalResults ) - except ParseBaseException as err: - #~ print "Exception raised in user parse action:", err - if (self.debugActions[2] ): - self.debugActions[2]( instring, tokensStart, self, err ) - raise - else: - for fn in self.parseAction: - tokens = fn( instring, tokensStart, retTokens ) - if tokens is not None: - retTokens = ParseResults( tokens, - self.resultsName, - asList=self.saveAsList and isinstance(tokens,(ParseResults,list)), - modal=self.modalResults ) - - if debugging: - #~ print ("Matched",self,"->",retTokens.asList()) - if (self.debugActions[1] ): - self.debugActions[1]( instring, tokensStart, loc, self, retTokens ) - - return loc, retTokens - - def tryParse( self, instring, loc ): - try: - return self._parse( instring, loc, doActions=False )[0] - except ParseFatalException: - raise ParseException( instring, loc, self.errmsg, self) - - def canParseNext(self, instring, loc): - try: - self.tryParse(instring, loc) - except (ParseException, IndexError): - return False - else: - return True - - class _UnboundedCache(object): - def __init__(self): - cache = {} - self.not_in_cache = not_in_cache = object() - - def get(self, key): - return cache.get(key, not_in_cache) - - def set(self, key, value): - cache[key] = value - - def clear(self): - cache.clear() - - def cache_len(self): - return len(cache) - - self.get = types.MethodType(get, self) - self.set = types.MethodType(set, self) - self.clear = types.MethodType(clear, self) - self.__len__ = types.MethodType(cache_len, self) - - if _OrderedDict is not None: - class _FifoCache(object): - def __init__(self, size): - self.not_in_cache = not_in_cache = object() - - cache = _OrderedDict() - - def get(self, key): - return cache.get(key, not_in_cache) - - def set(self, key, value): - cache[key] = value - while len(cache) > size: - try: - cache.popitem(False) - except KeyError: - pass - - def clear(self): - cache.clear() - - def cache_len(self): - return len(cache) - - self.get = types.MethodType(get, self) - self.set = types.MethodType(set, self) - self.clear = types.MethodType(clear, self) - self.__len__ = types.MethodType(cache_len, self) - - else: - class _FifoCache(object): - def __init__(self, size): - self.not_in_cache = not_in_cache = object() - - cache = {} - key_fifo = collections.deque([], size) - - def get(self, key): - return cache.get(key, not_in_cache) - - def set(self, key, value): - cache[key] = value - while len(key_fifo) > size: - cache.pop(key_fifo.popleft(), None) - key_fifo.append(key) - - def clear(self): - cache.clear() - key_fifo.clear() - - def cache_len(self): - return len(cache) - - self.get = types.MethodType(get, self) - self.set = types.MethodType(set, self) - self.clear = types.MethodType(clear, self) - self.__len__ = types.MethodType(cache_len, self) - - # argument cache for optimizing repeated calls when backtracking through recursive expressions - packrat_cache = {} # this is set later by enabledPackrat(); this is here so that resetCache() doesn't fail - packrat_cache_lock = RLock() - packrat_cache_stats = [0, 0] - - # this method gets repeatedly called during backtracking with the same arguments - - # we can cache these arguments and save ourselves the trouble of re-parsing the contained expression - def _parseCache( self, instring, loc, doActions=True, callPreParse=True ): - HIT, MISS = 0, 1 - lookup = (self, instring, loc, callPreParse, doActions) - with ParserElement.packrat_cache_lock: - cache = ParserElement.packrat_cache - value = cache.get(lookup) - if value is cache.not_in_cache: - ParserElement.packrat_cache_stats[MISS] += 1 - try: - value = self._parseNoCache(instring, loc, doActions, callPreParse) - except ParseBaseException as pe: - # cache a copy of the exception, without the traceback - cache.set(lookup, pe.__class__(*pe.args)) - raise - else: - cache.set(lookup, (value[0], value[1].copy())) - return value - else: - ParserElement.packrat_cache_stats[HIT] += 1 - if isinstance(value, Exception): - raise value - return (value[0], value[1].copy()) - - _parse = _parseNoCache - - @staticmethod - def resetCache(): - ParserElement.packrat_cache.clear() - ParserElement.packrat_cache_stats[:] = [0] * len(ParserElement.packrat_cache_stats) - - _packratEnabled = False - @staticmethod - def enablePackrat(cache_size_limit=128): - """Enables "packrat" parsing, which adds memoizing to the parsing logic. - Repeated parse attempts at the same string location (which happens - often in many complex grammars) can immediately return a cached value, - instead of re-executing parsing/validating code. Memoizing is done of - both valid results and parsing exceptions. - - Parameters: - - cache_size_limit - (default=C{128}) - if an integer value is provided - will limit the size of the packrat cache; if None is passed, then - the cache size will be unbounded; if 0 is passed, the cache will - be effectively disabled. - - This speedup may break existing programs that use parse actions that - have side-effects. For this reason, packrat parsing is disabled when - you first import pyparsing. To activate the packrat feature, your - program must call the class method C{ParserElement.enablePackrat()}. If - your program uses C{psyco} to "compile as you go", you must call - C{enablePackrat} before calling C{psyco.full()}. If you do not do this, - Python will crash. For best results, call C{enablePackrat()} immediately - after importing pyparsing. - - Example:: - import pyparsing - pyparsing.ParserElement.enablePackrat() - """ - if not ParserElement._packratEnabled: - ParserElement._packratEnabled = True - if cache_size_limit is None: - ParserElement.packrat_cache = ParserElement._UnboundedCache() - else: - ParserElement.packrat_cache = ParserElement._FifoCache(cache_size_limit) - ParserElement._parse = ParserElement._parseCache - - def parseString( self, instring, parseAll=False ): - """ - Execute the parse expression with the given string. - This is the main interface to the client code, once the complete - expression has been built. - - If you want the grammar to require that the entire input string be - successfully parsed, then set C{parseAll} to True (equivalent to ending - the grammar with C{L{StringEnd()}}). - - Note: C{parseString} implicitly calls C{expandtabs()} on the input string, - in order to report proper column numbers in parse actions. - If the input string contains tabs and - the grammar uses parse actions that use the C{loc} argument to index into the - string being parsed, you can ensure you have a consistent view of the input - string by: - - calling C{parseWithTabs} on your grammar before calling C{parseString} - (see L{I{parseWithTabs}}) - - define your parse action using the full C{(s,loc,toks)} signature, and - reference the input string using the parse action's C{s} argument - - explicitly expand the tabs in your input string before calling - C{parseString} - - Example:: - Word('a').parseString('aaaaabaaa') # -> ['aaaaa'] - Word('a').parseString('aaaaabaaa', parseAll=True) # -> Exception: Expected end of text - """ - ParserElement.resetCache() - if not self.streamlined: - self.streamline() - #~ self.saveAsList = True - for e in self.ignoreExprs: - e.streamline() - if not self.keepTabs: - instring = instring.expandtabs() - try: - loc, tokens = self._parse( instring, 0 ) - if parseAll: - loc = self.preParse( instring, loc ) - se = Empty() + StringEnd() - se._parse( instring, loc ) - except ParseBaseException as exc: - if ParserElement.verbose_stacktrace: - raise - else: - # catch and re-raise exception from here, clears out pyparsing internal stack trace - raise exc - else: - return tokens - - def scanString( self, instring, maxMatches=_MAX_INT, overlap=False ): - """ - Scan the input string for expression matches. Each match will return the - matching tokens, start location, and end location. May be called with optional - C{maxMatches} argument, to clip scanning after 'n' matches are found. If - C{overlap} is specified, then overlapping matches will be reported. - - Note that the start and end locations are reported relative to the string - being parsed. See L{I{parseString}} for more information on parsing - strings with embedded tabs. - - Example:: - source = "sldjf123lsdjjkf345sldkjf879lkjsfd987" - print(source) - for tokens,start,end in Word(alphas).scanString(source): - print(' '*start + '^'*(end-start)) - print(' '*start + tokens[0]) - - prints:: - - sldjf123lsdjjkf345sldkjf879lkjsfd987 - ^^^^^ - sldjf - ^^^^^^^ - lsdjjkf - ^^^^^^ - sldkjf - ^^^^^^ - lkjsfd - """ - if not self.streamlined: - self.streamline() - for e in self.ignoreExprs: - e.streamline() - - if not self.keepTabs: - instring = _ustr(instring).expandtabs() - instrlen = len(instring) - loc = 0 - preparseFn = self.preParse - parseFn = self._parse - ParserElement.resetCache() - matches = 0 - try: - while loc <= instrlen and matches < maxMatches: - try: - preloc = preparseFn( instring, loc ) - nextLoc,tokens = parseFn( instring, preloc, callPreParse=False ) - except ParseException: - loc = preloc+1 - else: - if nextLoc > loc: - matches += 1 - yield tokens, preloc, nextLoc - if overlap: - nextloc = preparseFn( instring, loc ) - if nextloc > loc: - loc = nextLoc - else: - loc += 1 - else: - loc = nextLoc - else: - loc = preloc+1 - except ParseBaseException as exc: - if ParserElement.verbose_stacktrace: - raise - else: - # catch and re-raise exception from here, clears out pyparsing internal stack trace - raise exc - - def transformString( self, instring ): - """ - Extension to C{L{scanString}}, to modify matching text with modified tokens that may - be returned from a parse action. To use C{transformString}, define a grammar and - attach a parse action to it that modifies the returned token list. - Invoking C{transformString()} on a target string will then scan for matches, - and replace the matched text patterns according to the logic in the parse - action. C{transformString()} returns the resulting transformed string. - - Example:: - wd = Word(alphas) - wd.setParseAction(lambda toks: toks[0].title()) - - print(wd.transformString("now is the winter of our discontent made glorious summer by this sun of york.")) - Prints:: - Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York. - """ - out = [] - lastE = 0 - # force preservation of s, to minimize unwanted transformation of string, and to - # keep string locs straight between transformString and scanString - self.keepTabs = True - try: - for t,s,e in self.scanString( instring ): - out.append( instring[lastE:s] ) - if t: - if isinstance(t,ParseResults): - out += t.asList() - elif isinstance(t,list): - out += t - else: - out.append(t) - lastE = e - out.append(instring[lastE:]) - out = [o for o in out if o] - return "".join(map(_ustr,_flatten(out))) - except ParseBaseException as exc: - if ParserElement.verbose_stacktrace: - raise - else: - # catch and re-raise exception from here, clears out pyparsing internal stack trace - raise exc - - def searchString( self, instring, maxMatches=_MAX_INT ): - """ - Another extension to C{L{scanString}}, simplifying the access to the tokens found - to match the given parse expression. May be called with optional - C{maxMatches} argument, to clip searching after 'n' matches are found. - - Example:: - # a capitalized word starts with an uppercase letter, followed by zero or more lowercase letters - cap_word = Word(alphas.upper(), alphas.lower()) - - print(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity")) - - # the sum() builtin can be used to merge results into a single ParseResults object - print(sum(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity"))) - prints:: - [['More'], ['Iron'], ['Lead'], ['Gold'], ['I'], ['Electricity']] - ['More', 'Iron', 'Lead', 'Gold', 'I', 'Electricity'] - """ - try: - return ParseResults([ t for t,s,e in self.scanString( instring, maxMatches ) ]) - except ParseBaseException as exc: - if ParserElement.verbose_stacktrace: - raise - else: - # catch and re-raise exception from here, clears out pyparsing internal stack trace - raise exc - - def split(self, instring, maxsplit=_MAX_INT, includeSeparators=False): - """ - Generator method to split a string using the given expression as a separator. - May be called with optional C{maxsplit} argument, to limit the number of splits; - and the optional C{includeSeparators} argument (default=C{False}), if the separating - matching text should be included in the split results. - - Example:: - punc = oneOf(list(".,;:/-!?")) - print(list(punc.split("This, this?, this sentence, is badly punctuated!"))) - prints:: - ['This', ' this', '', ' this sentence', ' is badly punctuated', ''] - """ - splits = 0 - last = 0 - for t,s,e in self.scanString(instring, maxMatches=maxsplit): - yield instring[last:s] - if includeSeparators: - yield t[0] - last = e - yield instring[last:] - - def __add__(self, other ): - """ - Implementation of + operator - returns C{L{And}}. Adding strings to a ParserElement - converts them to L{Literal}s by default. - - Example:: - greet = Word(alphas) + "," + Word(alphas) + "!" - hello = "Hello, World!" - print (hello, "->", greet.parseString(hello)) - Prints:: - Hello, World! -> ['Hello', ',', 'World', '!'] - """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) - return None - return And( [ self, other ] ) - - def __radd__(self, other ): - """ - Implementation of + operator when left operand is not a C{L{ParserElement}} - """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) - return None - return other + self - - def __sub__(self, other): - """ - Implementation of - operator, returns C{L{And}} with error stop - """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) - return None - return self + And._ErrorStop() + other - - def __rsub__(self, other ): - """ - Implementation of - operator when left operand is not a C{L{ParserElement}} - """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) - return None - return other - self - - def __mul__(self,other): - """ - Implementation of * operator, allows use of C{expr * 3} in place of - C{expr + expr + expr}. Expressions may also me multiplied by a 2-integer - tuple, similar to C{{min,max}} multipliers in regular expressions. Tuples - may also include C{None} as in: - - C{expr*(n,None)} or C{expr*(n,)} is equivalent - to C{expr*n + L{ZeroOrMore}(expr)} - (read as "at least n instances of C{expr}") - - C{expr*(None,n)} is equivalent to C{expr*(0,n)} - (read as "0 to n instances of C{expr}") - - C{expr*(None,None)} is equivalent to C{L{ZeroOrMore}(expr)} - - C{expr*(1,None)} is equivalent to C{L{OneOrMore}(expr)} - - Note that C{expr*(None,n)} does not raise an exception if - more than n exprs exist in the input stream; that is, - C{expr*(None,n)} does not enforce a maximum number of expr - occurrences. If this behavior is desired, then write - C{expr*(None,n) + ~expr} - """ - if isinstance(other,int): - minElements, optElements = other,0 - elif isinstance(other,tuple): - other = (other + (None, None))[:2] - if other[0] is None: - other = (0, other[1]) - if isinstance(other[0],int) and other[1] is None: - if other[0] == 0: - return ZeroOrMore(self) - if other[0] == 1: - return OneOrMore(self) - else: - return self*other[0] + ZeroOrMore(self) - elif isinstance(other[0],int) and isinstance(other[1],int): - minElements, optElements = other - optElements -= minElements - else: - raise TypeError("cannot multiply 'ParserElement' and ('%s','%s') objects", type(other[0]),type(other[1])) - else: - raise TypeError("cannot multiply 'ParserElement' and '%s' objects", type(other)) - - if minElements < 0: - raise ValueError("cannot multiply ParserElement by negative value") - if optElements < 0: - raise ValueError("second tuple value must be greater or equal to first tuple value") - if minElements == optElements == 0: - raise ValueError("cannot multiply ParserElement by 0 or (0,0)") - - if (optElements): - def makeOptionalList(n): - if n>1: - return Optional(self + makeOptionalList(n-1)) - else: - return Optional(self) - if minElements: - if minElements == 1: - ret = self + makeOptionalList(optElements) - else: - ret = And([self]*minElements) + makeOptionalList(optElements) - else: - ret = makeOptionalList(optElements) - else: - if minElements == 1: - ret = self - else: - ret = And([self]*minElements) - return ret - - def __rmul__(self, other): - return self.__mul__(other) - - def __or__(self, other ): - """ - Implementation of | operator - returns C{L{MatchFirst}} - """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) - return None - return MatchFirst( [ self, other ] ) - - def __ror__(self, other ): - """ - Implementation of | operator when left operand is not a C{L{ParserElement}} - """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) - return None - return other | self - - def __xor__(self, other ): - """ - Implementation of ^ operator - returns C{L{Or}} - """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) - return None - return Or( [ self, other ] ) - - def __rxor__(self, other ): - """ - Implementation of ^ operator when left operand is not a C{L{ParserElement}} - """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) - return None - return other ^ self - - def __and__(self, other ): - """ - Implementation of & operator - returns C{L{Each}} - """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) - return None - return Each( [ self, other ] ) - - def __rand__(self, other ): - """ - Implementation of & operator when left operand is not a C{L{ParserElement}} - """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) - return None - return other & self - - def __invert__( self ): - """ - Implementation of ~ operator - returns C{L{NotAny}} - """ - return NotAny( self ) - - def __call__(self, name=None): - """ - Shortcut for C{L{setResultsName}}, with C{listAllMatches=False}. - - If C{name} is given with a trailing C{'*'} character, then C{listAllMatches} will be - passed as C{True}. - - If C{name} is omitted, same as calling C{L{copy}}. - - Example:: - # these are equivalent - userdata = Word(alphas).setResultsName("name") + Word(nums+"-").setResultsName("socsecno") - userdata = Word(alphas)("name") + Word(nums+"-")("socsecno") - """ - if name is not None: - return self.setResultsName(name) - else: - return self.copy() - - def suppress( self ): - """ - Suppresses the output of this C{ParserElement}; useful to keep punctuation from - cluttering up returned output. - """ - return Suppress( self ) - - def leaveWhitespace( self ): - """ - Disables the skipping of whitespace before matching the characters in the - C{ParserElement}'s defined pattern. This is normally only used internally by - the pyparsing module, but may be needed in some whitespace-sensitive grammars. - """ - self.skipWhitespace = False - return self - - def setWhitespaceChars( self, chars ): - """ - Overrides the default whitespace chars - """ - self.skipWhitespace = True - self.whiteChars = chars - self.copyDefaultWhiteChars = False - return self - - def parseWithTabs( self ): - """ - Overrides default behavior to expand C{}s to spaces before parsing the input string. - Must be called before C{parseString} when the input grammar contains elements that - match C{} characters. - """ - self.keepTabs = True - return self - - def ignore( self, other ): - """ - Define expression to be ignored (e.g., comments) while doing pattern - matching; may be called repeatedly, to define multiple comment or other - ignorable patterns. - - Example:: - patt = OneOrMore(Word(alphas)) - patt.parseString('ablaj /* comment */ lskjd') # -> ['ablaj'] - - patt.ignore(cStyleComment) - patt.parseString('ablaj /* comment */ lskjd') # -> ['ablaj', 'lskjd'] - """ - if isinstance(other, basestring): - other = Suppress(other) - - if isinstance( other, Suppress ): - if other not in self.ignoreExprs: - self.ignoreExprs.append(other) - else: - self.ignoreExprs.append( Suppress( other.copy() ) ) - return self - - def setDebugActions( self, startAction, successAction, exceptionAction ): - """ - Enable display of debugging messages while doing pattern matching. - """ - self.debugActions = (startAction or _defaultStartDebugAction, - successAction or _defaultSuccessDebugAction, - exceptionAction or _defaultExceptionDebugAction) - self.debug = True - return self - - def setDebug( self, flag=True ): - """ - Enable display of debugging messages while doing pattern matching. - Set C{flag} to True to enable, False to disable. - - Example:: - wd = Word(alphas).setName("alphaword") - integer = Word(nums).setName("numword") - term = wd | integer - - # turn on debugging for wd - wd.setDebug() - - OneOrMore(term).parseString("abc 123 xyz 890") - - prints:: - Match alphaword at loc 0(1,1) - Matched alphaword -> ['abc'] - Match alphaword at loc 3(1,4) - Exception raised:Expected alphaword (at char 4), (line:1, col:5) - Match alphaword at loc 7(1,8) - Matched alphaword -> ['xyz'] - Match alphaword at loc 11(1,12) - Exception raised:Expected alphaword (at char 12), (line:1, col:13) - Match alphaword at loc 15(1,16) - Exception raised:Expected alphaword (at char 15), (line:1, col:16) - - The output shown is that produced by the default debug actions - custom debug actions can be - specified using L{setDebugActions}. Prior to attempting - to match the C{wd} expression, the debugging message C{"Match at loc (,)"} - is shown. Then if the parse succeeds, a C{"Matched"} message is shown, or an C{"Exception raised"} - message is shown. Also note the use of L{setName} to assign a human-readable name to the expression, - which makes debugging and exception messages easier to understand - for instance, the default - name created for the C{Word} expression without calling C{setName} is C{"W:(ABCD...)"}. - """ - if flag: - self.setDebugActions( _defaultStartDebugAction, _defaultSuccessDebugAction, _defaultExceptionDebugAction ) - else: - self.debug = False - return self - - def __str__( self ): - return self.name - - def __repr__( self ): - return _ustr(self) - - def streamline( self ): - self.streamlined = True - self.strRepr = None - return self - - def checkRecursion( self, parseElementList ): - pass - - def validate( self, validateTrace=[] ): - """ - Check defined expressions for valid structure, check for infinite recursive definitions. - """ - self.checkRecursion( [] ) - - def parseFile( self, file_or_filename, parseAll=False ): - """ - Execute the parse expression on the given file or filename. - If a filename is specified (instead of a file object), - the entire file is opened, read, and closed before parsing. - """ - try: - file_contents = file_or_filename.read() - except AttributeError: - with open(file_or_filename, "r") as f: - file_contents = f.read() - try: - return self.parseString(file_contents, parseAll) - except ParseBaseException as exc: - if ParserElement.verbose_stacktrace: - raise - else: - # catch and re-raise exception from here, clears out pyparsing internal stack trace - raise exc - - def __eq__(self,other): - if isinstance(other, ParserElement): - return self is other or vars(self) == vars(other) - elif isinstance(other, basestring): - return self.matches(other) - else: - return super(ParserElement,self)==other - - def __ne__(self,other): - return not (self == other) - - def __hash__(self): - return hash(id(self)) - - def __req__(self,other): - return self == other - - def __rne__(self,other): - return not (self == other) - - def matches(self, testString, parseAll=True): - """ - Method for quick testing of a parser against a test string. Good for simple - inline microtests of sub expressions while building up larger parser. - - Parameters: - - testString - to test against this expression for a match - - parseAll - (default=C{True}) - flag to pass to C{L{parseString}} when running tests - - Example:: - expr = Word(nums) - assert expr.matches("100") - """ - try: - self.parseString(_ustr(testString), parseAll=parseAll) - return True - except ParseBaseException: - return False - - def runTests(self, tests, parseAll=True, comment='#', fullDump=True, printResults=True, failureTests=False): - """ - Execute the parse expression on a series of test strings, showing each - test, the parsed results or where the parse failed. Quick and easy way to - run a parse expression against a list of sample strings. - - Parameters: - - tests - a list of separate test strings, or a multiline string of test strings - - parseAll - (default=C{True}) - flag to pass to C{L{parseString}} when running tests - - comment - (default=C{'#'}) - expression for indicating embedded comments in the test - string; pass None to disable comment filtering - - fullDump - (default=C{True}) - dump results as list followed by results names in nested outline; - if False, only dump nested list - - printResults - (default=C{True}) prints test output to stdout - - failureTests - (default=C{False}) indicates if these tests are expected to fail parsing - - Returns: a (success, results) tuple, where success indicates that all tests succeeded - (or failed if C{failureTests} is True), and the results contain a list of lines of each - test's output - - Example:: - number_expr = pyparsing_common.number.copy() - - result = number_expr.runTests(''' - # unsigned integer - 100 - # negative integer - -100 - # float with scientific notation - 6.02e23 - # integer with scientific notation - 1e-12 - ''') - print("Success" if result[0] else "Failed!") - - result = number_expr.runTests(''' - # stray character - 100Z - # missing leading digit before '.' - -.100 - # too many '.' - 3.14.159 - ''', failureTests=True) - print("Success" if result[0] else "Failed!") - prints:: - # unsigned integer - 100 - [100] - - # negative integer - -100 - [-100] - - # float with scientific notation - 6.02e23 - [6.02e+23] - - # integer with scientific notation - 1e-12 - [1e-12] - - Success - - # stray character - 100Z - ^ - FAIL: Expected end of text (at char 3), (line:1, col:4) - - # missing leading digit before '.' - -.100 - ^ - FAIL: Expected {real number with scientific notation | real number | signed integer} (at char 0), (line:1, col:1) - - # too many '.' - 3.14.159 - ^ - FAIL: Expected end of text (at char 4), (line:1, col:5) - - Success - - Each test string must be on a single line. If you want to test a string that spans multiple - lines, create a test like this:: - - expr.runTest(r"this is a test\\n of strings that spans \\n 3 lines") - - (Note that this is a raw string literal, you must include the leading 'r'.) - """ - if isinstance(tests, basestring): - tests = list(map(str.strip, tests.rstrip().splitlines())) - if isinstance(comment, basestring): - comment = Literal(comment) - allResults = [] - comments = [] - success = True - for t in tests: - if comment is not None and comment.matches(t, False) or comments and not t: - comments.append(t) - continue - if not t: - continue - out = ['\n'.join(comments), t] - comments = [] - try: - t = t.replace(r'\n','\n') - result = self.parseString(t, parseAll=parseAll) - out.append(result.dump(full=fullDump)) - success = success and not failureTests - except ParseBaseException as pe: - fatal = "(FATAL)" if isinstance(pe, ParseFatalException) else "" - if '\n' in t: - out.append(line(pe.loc, t)) - out.append(' '*(col(pe.loc,t)-1) + '^' + fatal) - else: - out.append(' '*pe.loc + '^' + fatal) - out.append("FAIL: " + str(pe)) - success = success and failureTests - result = pe - except Exception as exc: - out.append("FAIL-EXCEPTION: " + str(exc)) - success = success and failureTests - result = exc - - if printResults: - if fullDump: - out.append('') - print('\n'.join(out)) - - allResults.append((t, result)) - - return success, allResults - - -class Token(ParserElement): - """ - Abstract C{ParserElement} subclass, for defining atomic matching patterns. - """ - def __init__( self ): - super(Token,self).__init__( savelist=False ) - - -class Empty(Token): - """ - An empty token, will always match. - """ - def __init__( self ): - super(Empty,self).__init__() - self.name = "Empty" - self.mayReturnEmpty = True - self.mayIndexError = False - - -class NoMatch(Token): - """ - A token that will never match. - """ - def __init__( self ): - super(NoMatch,self).__init__() - self.name = "NoMatch" - self.mayReturnEmpty = True - self.mayIndexError = False - self.errmsg = "Unmatchable token" - - def parseImpl( self, instring, loc, doActions=True ): - raise ParseException(instring, loc, self.errmsg, self) - - -class Literal(Token): - """ - Token to exactly match a specified string. - - Example:: - Literal('blah').parseString('blah') # -> ['blah'] - Literal('blah').parseString('blahfooblah') # -> ['blah'] - Literal('blah').parseString('bla') # -> Exception: Expected "blah" - - For case-insensitive matching, use L{CaselessLiteral}. - - For keyword matching (force word break before and after the matched string), - use L{Keyword} or L{CaselessKeyword}. - """ - def __init__( self, matchString ): - super(Literal,self).__init__() - self.match = matchString - self.matchLen = len(matchString) - try: - self.firstMatchChar = matchString[0] - except IndexError: - warnings.warn("null string passed to Literal; use Empty() instead", - SyntaxWarning, stacklevel=2) - self.__class__ = Empty - self.name = '"%s"' % _ustr(self.match) - self.errmsg = "Expected " + self.name - self.mayReturnEmpty = False - self.mayIndexError = False - - # Performance tuning: this routine gets called a *lot* - # if this is a single character match string and the first character matches, - # short-circuit as quickly as possible, and avoid calling startswith - #~ @profile - def parseImpl( self, instring, loc, doActions=True ): - if (instring[loc] == self.firstMatchChar and - (self.matchLen==1 or instring.startswith(self.match,loc)) ): - return loc+self.matchLen, self.match - raise ParseException(instring, loc, self.errmsg, self) -_L = Literal -ParserElement._literalStringClass = Literal - -class Keyword(Token): - """ - Token to exactly match a specified string as a keyword, that is, it must be - immediately followed by a non-keyword character. Compare with C{L{Literal}}: - - C{Literal("if")} will match the leading C{'if'} in C{'ifAndOnlyIf'}. - - C{Keyword("if")} will not; it will only match the leading C{'if'} in C{'if x=1'}, or C{'if(y==2)'} - Accepts two optional constructor arguments in addition to the keyword string: - - C{identChars} is a string of characters that would be valid identifier characters, - defaulting to all alphanumerics + "_" and "$" - - C{caseless} allows case-insensitive matching, default is C{False}. - - Example:: - Keyword("start").parseString("start") # -> ['start'] - Keyword("start").parseString("starting") # -> Exception - - For case-insensitive matching, use L{CaselessKeyword}. - """ - DEFAULT_KEYWORD_CHARS = alphanums+"_$" - - def __init__( self, matchString, identChars=None, caseless=False ): - super(Keyword,self).__init__() - if identChars is None: - identChars = Keyword.DEFAULT_KEYWORD_CHARS - self.match = matchString - self.matchLen = len(matchString) - try: - self.firstMatchChar = matchString[0] - except IndexError: - warnings.warn("null string passed to Keyword; use Empty() instead", - SyntaxWarning, stacklevel=2) - self.name = '"%s"' % self.match - self.errmsg = "Expected " + self.name - self.mayReturnEmpty = False - self.mayIndexError = False - self.caseless = caseless - if caseless: - self.caselessmatch = matchString.upper() - identChars = identChars.upper() - self.identChars = set(identChars) - - def parseImpl( self, instring, loc, doActions=True ): - if self.caseless: - if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and - (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) and - (loc == 0 or instring[loc-1].upper() not in self.identChars) ): - return loc+self.matchLen, self.match - else: - if (instring[loc] == self.firstMatchChar and - (self.matchLen==1 or instring.startswith(self.match,loc)) and - (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen] not in self.identChars) and - (loc == 0 or instring[loc-1] not in self.identChars) ): - return loc+self.matchLen, self.match - raise ParseException(instring, loc, self.errmsg, self) - - def copy(self): - c = super(Keyword,self).copy() - c.identChars = Keyword.DEFAULT_KEYWORD_CHARS - return c - - @staticmethod - def setDefaultKeywordChars( chars ): - """Overrides the default Keyword chars - """ - Keyword.DEFAULT_KEYWORD_CHARS = chars - -class CaselessLiteral(Literal): - """ - Token to match a specified string, ignoring case of letters. - Note: the matched results will always be in the case of the given - match string, NOT the case of the input text. - - Example:: - OneOrMore(CaselessLiteral("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD', 'CMD'] - - (Contrast with example for L{CaselessKeyword}.) - """ - def __init__( self, matchString ): - super(CaselessLiteral,self).__init__( matchString.upper() ) - # Preserve the defining literal. - self.returnString = matchString - self.name = "'%s'" % self.returnString - self.errmsg = "Expected " + self.name - - def parseImpl( self, instring, loc, doActions=True ): - if instring[ loc:loc+self.matchLen ].upper() == self.match: - return loc+self.matchLen, self.returnString - raise ParseException(instring, loc, self.errmsg, self) - -class CaselessKeyword(Keyword): - """ - Caseless version of L{Keyword}. - - Example:: - OneOrMore(CaselessKeyword("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD'] - - (Contrast with example for L{CaselessLiteral}.) - """ - def __init__( self, matchString, identChars=None ): - super(CaselessKeyword,self).__init__( matchString, identChars, caseless=True ) - - def parseImpl( self, instring, loc, doActions=True ): - if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and - (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) ): - return loc+self.matchLen, self.match - raise ParseException(instring, loc, self.errmsg, self) - -class CloseMatch(Token): - """ - A variation on L{Literal} which matches "close" matches, that is, - strings with at most 'n' mismatching characters. C{CloseMatch} takes parameters: - - C{match_string} - string to be matched - - C{maxMismatches} - (C{default=1}) maximum number of mismatches allowed to count as a match - - The results from a successful parse will contain the matched text from the input string and the following named results: - - C{mismatches} - a list of the positions within the match_string where mismatches were found - - C{original} - the original match_string used to compare against the input string - - If C{mismatches} is an empty list, then the match was an exact match. - - Example:: - patt = CloseMatch("ATCATCGAATGGA") - patt.parseString("ATCATCGAAXGGA") # -> (['ATCATCGAAXGGA'], {'mismatches': [[9]], 'original': ['ATCATCGAATGGA']}) - patt.parseString("ATCAXCGAAXGGA") # -> Exception: Expected 'ATCATCGAATGGA' (with up to 1 mismatches) (at char 0), (line:1, col:1) - - # exact match - patt.parseString("ATCATCGAATGGA") # -> (['ATCATCGAATGGA'], {'mismatches': [[]], 'original': ['ATCATCGAATGGA']}) - - # close match allowing up to 2 mismatches - patt = CloseMatch("ATCATCGAATGGA", maxMismatches=2) - patt.parseString("ATCAXCGAAXGGA") # -> (['ATCAXCGAAXGGA'], {'mismatches': [[4, 9]], 'original': ['ATCATCGAATGGA']}) - """ - def __init__(self, match_string, maxMismatches=1): - super(CloseMatch,self).__init__() - self.name = match_string - self.match_string = match_string - self.maxMismatches = maxMismatches - self.errmsg = "Expected %r (with up to %d mismatches)" % (self.match_string, self.maxMismatches) - self.mayIndexError = False - self.mayReturnEmpty = False - - def parseImpl( self, instring, loc, doActions=True ): - start = loc - instrlen = len(instring) - maxloc = start + len(self.match_string) - - if maxloc <= instrlen: - match_string = self.match_string - match_stringloc = 0 - mismatches = [] - maxMismatches = self.maxMismatches - - for match_stringloc,s_m in enumerate(zip(instring[loc:maxloc], self.match_string)): - src,mat = s_m - if src != mat: - mismatches.append(match_stringloc) - if len(mismatches) > maxMismatches: - break - else: - loc = match_stringloc + 1 - results = ParseResults([instring[start:loc]]) - results['original'] = self.match_string - results['mismatches'] = mismatches - return loc, results - - raise ParseException(instring, loc, self.errmsg, self) - - -class Word(Token): - """ - Token for matching words composed of allowed character sets. - Defined with string containing all allowed initial characters, - an optional string containing allowed body characters (if omitted, - defaults to the initial character set), and an optional minimum, - maximum, and/or exact length. The default value for C{min} is 1 (a - minimum value < 1 is not valid); the default values for C{max} and C{exact} - are 0, meaning no maximum or exact length restriction. An optional - C{excludeChars} parameter can list characters that might be found in - the input C{bodyChars} string; useful to define a word of all printables - except for one or two characters, for instance. - - L{srange} is useful for defining custom character set strings for defining - C{Word} expressions, using range notation from regular expression character sets. - - A common mistake is to use C{Word} to match a specific literal string, as in - C{Word("Address")}. Remember that C{Word} uses the string argument to define - I{sets} of matchable characters. This expression would match "Add", "AAA", - "dAred", or any other word made up of the characters 'A', 'd', 'r', 'e', and 's'. - To match an exact literal string, use L{Literal} or L{Keyword}. - - pyparsing includes helper strings for building Words: - - L{alphas} - - L{nums} - - L{alphanums} - - L{hexnums} - - L{alphas8bit} (alphabetic characters in ASCII range 128-255 - accented, tilded, umlauted, etc.) - - L{punc8bit} (non-alphabetic characters in ASCII range 128-255 - currency, symbols, superscripts, diacriticals, etc.) - - L{printables} (any non-whitespace character) - - Example:: - # a word composed of digits - integer = Word(nums) # equivalent to Word("0123456789") or Word(srange("0-9")) - - # a word with a leading capital, and zero or more lowercase - capital_word = Word(alphas.upper(), alphas.lower()) - - # hostnames are alphanumeric, with leading alpha, and '-' - hostname = Word(alphas, alphanums+'-') - - # roman numeral (not a strict parser, accepts invalid mix of characters) - roman = Word("IVXLCDM") - - # any string of non-whitespace characters, except for ',' - csv_value = Word(printables, excludeChars=",") - """ - def __init__( self, initChars, bodyChars=None, min=1, max=0, exact=0, asKeyword=False, excludeChars=None ): - super(Word,self).__init__() - if excludeChars: - initChars = ''.join(c for c in initChars if c not in excludeChars) - if bodyChars: - bodyChars = ''.join(c for c in bodyChars if c not in excludeChars) - self.initCharsOrig = initChars - self.initChars = set(initChars) - if bodyChars : - self.bodyCharsOrig = bodyChars - self.bodyChars = set(bodyChars) - else: - self.bodyCharsOrig = initChars - self.bodyChars = set(initChars) - - self.maxSpecified = max > 0 - - if min < 1: - raise ValueError("cannot specify a minimum length < 1; use Optional(Word()) if zero-length word is permitted") - - self.minLen = min - - if max > 0: - self.maxLen = max - else: - self.maxLen = _MAX_INT - - if exact > 0: - self.maxLen = exact - self.minLen = exact - - self.name = _ustr(self) - self.errmsg = "Expected " + self.name - self.mayIndexError = False - self.asKeyword = asKeyword - - if ' ' not in self.initCharsOrig+self.bodyCharsOrig and (min==1 and max==0 and exact==0): - if self.bodyCharsOrig == self.initCharsOrig: - self.reString = "[%s]+" % _escapeRegexRangeChars(self.initCharsOrig) - elif len(self.initCharsOrig) == 1: - self.reString = "%s[%s]*" % \ - (re.escape(self.initCharsOrig), - _escapeRegexRangeChars(self.bodyCharsOrig),) - else: - self.reString = "[%s][%s]*" % \ - (_escapeRegexRangeChars(self.initCharsOrig), - _escapeRegexRangeChars(self.bodyCharsOrig),) - if self.asKeyword: - self.reString = r"\b"+self.reString+r"\b" - try: - self.re = re.compile( self.reString ) - except Exception: - self.re = None - - def parseImpl( self, instring, loc, doActions=True ): - if self.re: - result = self.re.match(instring,loc) - if not result: - raise ParseException(instring, loc, self.errmsg, self) - - loc = result.end() - return loc, result.group() - - if not(instring[ loc ] in self.initChars): - raise ParseException(instring, loc, self.errmsg, self) - - start = loc - loc += 1 - instrlen = len(instring) - bodychars = self.bodyChars - maxloc = start + self.maxLen - maxloc = min( maxloc, instrlen ) - while loc < maxloc and instring[loc] in bodychars: - loc += 1 - - throwException = False - if loc - start < self.minLen: - throwException = True - if self.maxSpecified and loc < instrlen and instring[loc] in bodychars: - throwException = True - if self.asKeyword: - if (start>0 and instring[start-1] in bodychars) or (loc4: - return s[:4]+"..." - else: - return s - - if ( self.initCharsOrig != self.bodyCharsOrig ): - self.strRepr = "W:(%s,%s)" % ( charsAsStr(self.initCharsOrig), charsAsStr(self.bodyCharsOrig) ) - else: - self.strRepr = "W:(%s)" % charsAsStr(self.initCharsOrig) - - return self.strRepr - - -class Regex(Token): - r""" - Token for matching strings that match a given regular expression. - Defined with string specifying the regular expression in a form recognized by the inbuilt Python re module. - If the given regex contains named groups (defined using C{(?P...)}), these will be preserved as - named parse results. - - Example:: - realnum = Regex(r"[+-]?\d+\.\d*") - date = Regex(r'(?P\d{4})-(?P\d\d?)-(?P\d\d?)') - # ref: https://stackoverflow.com/questions/267399/how-do-you-match-only-valid-roman-numerals-with-a-regular-expression - roman = Regex(r"M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})") - """ - compiledREtype = type(re.compile("[A-Z]")) - def __init__( self, pattern, flags=0): - """The parameters C{pattern} and C{flags} are passed to the C{re.compile()} function as-is. See the Python C{re} module for an explanation of the acceptable patterns and flags.""" - super(Regex,self).__init__() - - if isinstance(pattern, basestring): - if not pattern: - warnings.warn("null string passed to Regex; use Empty() instead", - SyntaxWarning, stacklevel=2) - - self.pattern = pattern - self.flags = flags - - try: - self.re = re.compile(self.pattern, self.flags) - self.reString = self.pattern - except sre_constants.error: - warnings.warn("invalid pattern (%s) passed to Regex" % pattern, - SyntaxWarning, stacklevel=2) - raise - - elif isinstance(pattern, Regex.compiledREtype): - self.re = pattern - self.pattern = \ - self.reString = str(pattern) - self.flags = flags - - else: - raise ValueError("Regex may only be constructed with a string or a compiled RE object") - - self.name = _ustr(self) - self.errmsg = "Expected " + self.name - self.mayIndexError = False - self.mayReturnEmpty = True - - def parseImpl( self, instring, loc, doActions=True ): - result = self.re.match(instring,loc) - if not result: - raise ParseException(instring, loc, self.errmsg, self) - - loc = result.end() - d = result.groupdict() - ret = ParseResults(result.group()) - if d: - for k in d: - ret[k] = d[k] - return loc,ret - - def __str__( self ): - try: - return super(Regex,self).__str__() - except Exception: - pass - - if self.strRepr is None: - self.strRepr = "Re:(%s)" % repr(self.pattern) - - return self.strRepr - - -class QuotedString(Token): - r""" - Token for matching strings that are delimited by quoting characters. - - Defined with the following parameters: - - quoteChar - string of one or more characters defining the quote delimiting string - - escChar - character to escape quotes, typically backslash (default=C{None}) - - escQuote - special quote sequence to escape an embedded quote string (such as SQL's "" to escape an embedded ") (default=C{None}) - - multiline - boolean indicating whether quotes can span multiple lines (default=C{False}) - - unquoteResults - boolean indicating whether the matched text should be unquoted (default=C{True}) - - endQuoteChar - string of one or more characters defining the end of the quote delimited string (default=C{None} => same as quoteChar) - - convertWhitespaceEscapes - convert escaped whitespace (C{'\t'}, C{'\n'}, etc.) to actual whitespace (default=C{True}) - - Example:: - qs = QuotedString('"') - print(qs.searchString('lsjdf "This is the quote" sldjf')) - complex_qs = QuotedString('{{', endQuoteChar='}}') - print(complex_qs.searchString('lsjdf {{This is the "quote"}} sldjf')) - sql_qs = QuotedString('"', escQuote='""') - print(sql_qs.searchString('lsjdf "This is the quote with ""embedded"" quotes" sldjf')) - prints:: - [['This is the quote']] - [['This is the "quote"']] - [['This is the quote with "embedded" quotes']] - """ - def __init__( self, quoteChar, escChar=None, escQuote=None, multiline=False, unquoteResults=True, endQuoteChar=None, convertWhitespaceEscapes=True): - super(QuotedString,self).__init__() - - # remove white space from quote chars - wont work anyway - quoteChar = quoteChar.strip() - if not quoteChar: - warnings.warn("quoteChar cannot be the empty string",SyntaxWarning,stacklevel=2) - raise SyntaxError() - - if endQuoteChar is None: - endQuoteChar = quoteChar - else: - endQuoteChar = endQuoteChar.strip() - if not endQuoteChar: - warnings.warn("endQuoteChar cannot be the empty string",SyntaxWarning,stacklevel=2) - raise SyntaxError() - - self.quoteChar = quoteChar - self.quoteCharLen = len(quoteChar) - self.firstQuoteChar = quoteChar[0] - self.endQuoteChar = endQuoteChar - self.endQuoteCharLen = len(endQuoteChar) - self.escChar = escChar - self.escQuote = escQuote - self.unquoteResults = unquoteResults - self.convertWhitespaceEscapes = convertWhitespaceEscapes - - if multiline: - self.flags = re.MULTILINE | re.DOTALL - self.pattern = r'%s(?:[^%s%s]' % \ - ( re.escape(self.quoteChar), - _escapeRegexRangeChars(self.endQuoteChar[0]), - (escChar is not None and _escapeRegexRangeChars(escChar) or '') ) - else: - self.flags = 0 - self.pattern = r'%s(?:[^%s\n\r%s]' % \ - ( re.escape(self.quoteChar), - _escapeRegexRangeChars(self.endQuoteChar[0]), - (escChar is not None and _escapeRegexRangeChars(escChar) or '') ) - if len(self.endQuoteChar) > 1: - self.pattern += ( - '|(?:' + ')|(?:'.join("%s[^%s]" % (re.escape(self.endQuoteChar[:i]), - _escapeRegexRangeChars(self.endQuoteChar[i])) - for i in range(len(self.endQuoteChar)-1,0,-1)) + ')' - ) - if escQuote: - self.pattern += (r'|(?:%s)' % re.escape(escQuote)) - if escChar: - self.pattern += (r'|(?:%s.)' % re.escape(escChar)) - self.escCharReplacePattern = re.escape(self.escChar)+"(.)" - self.pattern += (r')*%s' % re.escape(self.endQuoteChar)) - - try: - self.re = re.compile(self.pattern, self.flags) - self.reString = self.pattern - except sre_constants.error: - warnings.warn("invalid pattern (%s) passed to Regex" % self.pattern, - SyntaxWarning, stacklevel=2) - raise - - self.name = _ustr(self) - self.errmsg = "Expected " + self.name - self.mayIndexError = False - self.mayReturnEmpty = True - - def parseImpl( self, instring, loc, doActions=True ): - result = instring[loc] == self.firstQuoteChar and self.re.match(instring,loc) or None - if not result: - raise ParseException(instring, loc, self.errmsg, self) - - loc = result.end() - ret = result.group() - - if self.unquoteResults: - - # strip off quotes - ret = ret[self.quoteCharLen:-self.endQuoteCharLen] - - if isinstance(ret,basestring): - # replace escaped whitespace - if '\\' in ret and self.convertWhitespaceEscapes: - ws_map = { - r'\t' : '\t', - r'\n' : '\n', - r'\f' : '\f', - r'\r' : '\r', - } - for wslit,wschar in ws_map.items(): - ret = ret.replace(wslit, wschar) - - # replace escaped characters - if self.escChar: - ret = re.sub(self.escCharReplacePattern, r"\g<1>", ret) - - # replace escaped quotes - if self.escQuote: - ret = ret.replace(self.escQuote, self.endQuoteChar) - - return loc, ret - - def __str__( self ): - try: - return super(QuotedString,self).__str__() - except Exception: - pass - - if self.strRepr is None: - self.strRepr = "quoted string, starting with %s ending with %s" % (self.quoteChar, self.endQuoteChar) - - return self.strRepr - - -class CharsNotIn(Token): - """ - Token for matching words composed of characters I{not} in a given set (will - include whitespace in matched characters if not listed in the provided exclusion set - see example). - Defined with string containing all disallowed characters, and an optional - minimum, maximum, and/or exact length. The default value for C{min} is 1 (a - minimum value < 1 is not valid); the default values for C{max} and C{exact} - are 0, meaning no maximum or exact length restriction. - - Example:: - # define a comma-separated-value as anything that is not a ',' - csv_value = CharsNotIn(',') - print(delimitedList(csv_value).parseString("dkls,lsdkjf,s12 34,@!#,213")) - prints:: - ['dkls', 'lsdkjf', 's12 34', '@!#', '213'] - """ - def __init__( self, notChars, min=1, max=0, exact=0 ): - super(CharsNotIn,self).__init__() - self.skipWhitespace = False - self.notChars = notChars - - if min < 1: - raise ValueError("cannot specify a minimum length < 1; use Optional(CharsNotIn()) if zero-length char group is permitted") - - self.minLen = min - - if max > 0: - self.maxLen = max - else: - self.maxLen = _MAX_INT - - if exact > 0: - self.maxLen = exact - self.minLen = exact - - self.name = _ustr(self) - self.errmsg = "Expected " + self.name - self.mayReturnEmpty = ( self.minLen == 0 ) - self.mayIndexError = False - - def parseImpl( self, instring, loc, doActions=True ): - if instring[loc] in self.notChars: - raise ParseException(instring, loc, self.errmsg, self) - - start = loc - loc += 1 - notchars = self.notChars - maxlen = min( start+self.maxLen, len(instring) ) - while loc < maxlen and \ - (instring[loc] not in notchars): - loc += 1 - - if loc - start < self.minLen: - raise ParseException(instring, loc, self.errmsg, self) - - return loc, instring[start:loc] - - def __str__( self ): - try: - return super(CharsNotIn, self).__str__() - except Exception: - pass - - if self.strRepr is None: - if len(self.notChars) > 4: - self.strRepr = "!W:(%s...)" % self.notChars[:4] - else: - self.strRepr = "!W:(%s)" % self.notChars - - return self.strRepr - -class White(Token): - """ - Special matching class for matching whitespace. Normally, whitespace is ignored - by pyparsing grammars. This class is included when some whitespace structures - are significant. Define with a string containing the whitespace characters to be - matched; default is C{" \\t\\r\\n"}. Also takes optional C{min}, C{max}, and C{exact} arguments, - as defined for the C{L{Word}} class. - """ - whiteStrs = { - " " : "", - "\t": "", - "\n": "", - "\r": "", - "\f": "", - } - def __init__(self, ws=" \t\r\n", min=1, max=0, exact=0): - super(White,self).__init__() - self.matchWhite = ws - self.setWhitespaceChars( "".join(c for c in self.whiteChars if c not in self.matchWhite) ) - #~ self.leaveWhitespace() - self.name = ("".join(White.whiteStrs[c] for c in self.matchWhite)) - self.mayReturnEmpty = True - self.errmsg = "Expected " + self.name - - self.minLen = min - - if max > 0: - self.maxLen = max - else: - self.maxLen = _MAX_INT - - if exact > 0: - self.maxLen = exact - self.minLen = exact - - def parseImpl( self, instring, loc, doActions=True ): - if not(instring[ loc ] in self.matchWhite): - raise ParseException(instring, loc, self.errmsg, self) - start = loc - loc += 1 - maxloc = start + self.maxLen - maxloc = min( maxloc, len(instring) ) - while loc < maxloc and instring[loc] in self.matchWhite: - loc += 1 - - if loc - start < self.minLen: - raise ParseException(instring, loc, self.errmsg, self) - - return loc, instring[start:loc] - - -class _PositionToken(Token): - def __init__( self ): - super(_PositionToken,self).__init__() - self.name=self.__class__.__name__ - self.mayReturnEmpty = True - self.mayIndexError = False - -class GoToColumn(_PositionToken): - """ - Token to advance to a specific column of input text; useful for tabular report scraping. - """ - def __init__( self, colno ): - super(GoToColumn,self).__init__() - self.col = colno - - def preParse( self, instring, loc ): - if col(loc,instring) != self.col: - instrlen = len(instring) - if self.ignoreExprs: - loc = self._skipIgnorables( instring, loc ) - while loc < instrlen and instring[loc].isspace() and col( loc, instring ) != self.col : - loc += 1 - return loc - - def parseImpl( self, instring, loc, doActions=True ): - thiscol = col( loc, instring ) - if thiscol > self.col: - raise ParseException( instring, loc, "Text not in expected column", self ) - newloc = loc + self.col - thiscol - ret = instring[ loc: newloc ] - return newloc, ret - - -class LineStart(_PositionToken): - """ - Matches if current position is at the beginning of a line within the parse string - - Example:: - - test = '''\ - AAA this line - AAA and this line - AAA but not this one - B AAA and definitely not this one - ''' - - for t in (LineStart() + 'AAA' + restOfLine).searchString(test): - print(t) - - Prints:: - ['AAA', ' this line'] - ['AAA', ' and this line'] - - """ - def __init__( self ): - super(LineStart,self).__init__() - self.errmsg = "Expected start of line" - - def parseImpl( self, instring, loc, doActions=True ): - if col(loc, instring) == 1: - return loc, [] - raise ParseException(instring, loc, self.errmsg, self) - -class LineEnd(_PositionToken): - """ - Matches if current position is at the end of a line within the parse string - """ - def __init__( self ): - super(LineEnd,self).__init__() - self.setWhitespaceChars( ParserElement.DEFAULT_WHITE_CHARS.replace("\n","") ) - self.errmsg = "Expected end of line" - - def parseImpl( self, instring, loc, doActions=True ): - if loc len(instring): - return loc, [] - else: - raise ParseException(instring, loc, self.errmsg, self) - -class WordStart(_PositionToken): - """ - Matches if the current position is at the beginning of a Word, and - is not preceded by any character in a given set of C{wordChars} - (default=C{printables}). To emulate the C{\b} behavior of regular expressions, - use C{WordStart(alphanums)}. C{WordStart} will also match at the beginning of - the string being parsed, or at the beginning of a line. - """ - def __init__(self, wordChars = printables): - super(WordStart,self).__init__() - self.wordChars = set(wordChars) - self.errmsg = "Not at the start of a word" - - def parseImpl(self, instring, loc, doActions=True ): - if loc != 0: - if (instring[loc-1] in self.wordChars or - instring[loc] not in self.wordChars): - raise ParseException(instring, loc, self.errmsg, self) - return loc, [] - -class WordEnd(_PositionToken): - """ - Matches if the current position is at the end of a Word, and - is not followed by any character in a given set of C{wordChars} - (default=C{printables}). To emulate the C{\b} behavior of regular expressions, - use C{WordEnd(alphanums)}. C{WordEnd} will also match at the end of - the string being parsed, or at the end of a line. - """ - def __init__(self, wordChars = printables): - super(WordEnd,self).__init__() - self.wordChars = set(wordChars) - self.skipWhitespace = False - self.errmsg = "Not at the end of a word" - - def parseImpl(self, instring, loc, doActions=True ): - instrlen = len(instring) - if instrlen>0 and loc maxExcLoc: - maxException = err - maxExcLoc = err.loc - except IndexError: - if len(instring) > maxExcLoc: - maxException = ParseException(instring,len(instring),e.errmsg,self) - maxExcLoc = len(instring) - else: - # save match among all matches, to retry longest to shortest - matches.append((loc2, e)) - - if matches: - matches.sort(key=lambda x: -x[0]) - for _,e in matches: - try: - return e._parse( instring, loc, doActions ) - except ParseException as err: - err.__traceback__ = None - if err.loc > maxExcLoc: - maxException = err - maxExcLoc = err.loc - - if maxException is not None: - maxException.msg = self.errmsg - raise maxException - else: - raise ParseException(instring, loc, "no defined alternatives to match", self) - - - def __ixor__(self, other ): - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - return self.append( other ) #Or( [ self, other ] ) - - def __str__( self ): - if hasattr(self,"name"): - return self.name - - if self.strRepr is None: - self.strRepr = "{" + " ^ ".join(_ustr(e) for e in self.exprs) + "}" - - return self.strRepr - - def checkRecursion( self, parseElementList ): - subRecCheckList = parseElementList[:] + [ self ] - for e in self.exprs: - e.checkRecursion( subRecCheckList ) - - -class MatchFirst(ParseExpression): - """ - Requires that at least one C{ParseExpression} is found. - If two expressions match, the first one listed is the one that will match. - May be constructed using the C{'|'} operator. - - Example:: - # construct MatchFirst using '|' operator - - # watch the order of expressions to match - number = Word(nums) | Combine(Word(nums) + '.' + Word(nums)) - print(number.searchString("123 3.1416 789")) # Fail! -> [['123'], ['3'], ['1416'], ['789']] - - # put more selective expression first - number = Combine(Word(nums) + '.' + Word(nums)) | Word(nums) - print(number.searchString("123 3.1416 789")) # Better -> [['123'], ['3.1416'], ['789']] - """ - def __init__( self, exprs, savelist = False ): - super(MatchFirst,self).__init__(exprs, savelist) - if self.exprs: - self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs) - else: - self.mayReturnEmpty = True - - def parseImpl( self, instring, loc, doActions=True ): - maxExcLoc = -1 - maxException = None - for e in self.exprs: - try: - ret = e._parse( instring, loc, doActions ) - return ret - except ParseException as err: - if err.loc > maxExcLoc: - maxException = err - maxExcLoc = err.loc - except IndexError: - if len(instring) > maxExcLoc: - maxException = ParseException(instring,len(instring),e.errmsg,self) - maxExcLoc = len(instring) - - # only got here if no expression matched, raise exception for match that made it the furthest - else: - if maxException is not None: - maxException.msg = self.errmsg - raise maxException - else: - raise ParseException(instring, loc, "no defined alternatives to match", self) - - def __ior__(self, other ): - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - return self.append( other ) #MatchFirst( [ self, other ] ) - - def __str__( self ): - if hasattr(self,"name"): - return self.name - - if self.strRepr is None: - self.strRepr = "{" + " | ".join(_ustr(e) for e in self.exprs) + "}" - - return self.strRepr - - def checkRecursion( self, parseElementList ): - subRecCheckList = parseElementList[:] + [ self ] - for e in self.exprs: - e.checkRecursion( subRecCheckList ) - - -class Each(ParseExpression): - """ - Requires all given C{ParseExpression}s to be found, but in any order. - Expressions may be separated by whitespace. - May be constructed using the C{'&'} operator. - - Example:: - color = oneOf("RED ORANGE YELLOW GREEN BLUE PURPLE BLACK WHITE BROWN") - shape_type = oneOf("SQUARE CIRCLE TRIANGLE STAR HEXAGON OCTAGON") - integer = Word(nums) - shape_attr = "shape:" + shape_type("shape") - posn_attr = "posn:" + Group(integer("x") + ',' + integer("y"))("posn") - color_attr = "color:" + color("color") - size_attr = "size:" + integer("size") - - # use Each (using operator '&') to accept attributes in any order - # (shape and posn are required, color and size are optional) - shape_spec = shape_attr & posn_attr & Optional(color_attr) & Optional(size_attr) - - shape_spec.runTests(''' - shape: SQUARE color: BLACK posn: 100, 120 - shape: CIRCLE size: 50 color: BLUE posn: 50,80 - color:GREEN size:20 shape:TRIANGLE posn:20,40 - ''' - ) - prints:: - shape: SQUARE color: BLACK posn: 100, 120 - ['shape:', 'SQUARE', 'color:', 'BLACK', 'posn:', ['100', ',', '120']] - - color: BLACK - - posn: ['100', ',', '120'] - - x: 100 - - y: 120 - - shape: SQUARE - - - shape: CIRCLE size: 50 color: BLUE posn: 50,80 - ['shape:', 'CIRCLE', 'size:', '50', 'color:', 'BLUE', 'posn:', ['50', ',', '80']] - - color: BLUE - - posn: ['50', ',', '80'] - - x: 50 - - y: 80 - - shape: CIRCLE - - size: 50 - - - color: GREEN size: 20 shape: TRIANGLE posn: 20,40 - ['color:', 'GREEN', 'size:', '20', 'shape:', 'TRIANGLE', 'posn:', ['20', ',', '40']] - - color: GREEN - - posn: ['20', ',', '40'] - - x: 20 - - y: 40 - - shape: TRIANGLE - - size: 20 - """ - def __init__( self, exprs, savelist = True ): - super(Each,self).__init__(exprs, savelist) - self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) - self.skipWhitespace = True - self.initExprGroups = True - - def parseImpl( self, instring, loc, doActions=True ): - if self.initExprGroups: - self.opt1map = dict((id(e.expr),e) for e in self.exprs if isinstance(e,Optional)) - opt1 = [ e.expr for e in self.exprs if isinstance(e,Optional) ] - opt2 = [ e for e in self.exprs if e.mayReturnEmpty and not isinstance(e,Optional)] - self.optionals = opt1 + opt2 - self.multioptionals = [ e.expr for e in self.exprs if isinstance(e,ZeroOrMore) ] - self.multirequired = [ e.expr for e in self.exprs if isinstance(e,OneOrMore) ] - self.required = [ e for e in self.exprs if not isinstance(e,(Optional,ZeroOrMore,OneOrMore)) ] - self.required += self.multirequired - self.initExprGroups = False - tmpLoc = loc - tmpReqd = self.required[:] - tmpOpt = self.optionals[:] - matchOrder = [] - - keepMatching = True - while keepMatching: - tmpExprs = tmpReqd + tmpOpt + self.multioptionals + self.multirequired - failed = [] - for e in tmpExprs: - try: - tmpLoc = e.tryParse( instring, tmpLoc ) - except ParseException: - failed.append(e) - else: - matchOrder.append(self.opt1map.get(id(e),e)) - if e in tmpReqd: - tmpReqd.remove(e) - elif e in tmpOpt: - tmpOpt.remove(e) - if len(failed) == len(tmpExprs): - keepMatching = False - - if tmpReqd: - missing = ", ".join(_ustr(e) for e in tmpReqd) - raise ParseException(instring,loc,"Missing one or more required elements (%s)" % missing ) - - # add any unmatched Optionals, in case they have default values defined - matchOrder += [e for e in self.exprs if isinstance(e,Optional) and e.expr in tmpOpt] - - resultlist = [] - for e in matchOrder: - loc,results = e._parse(instring,loc,doActions) - resultlist.append(results) - - finalResults = sum(resultlist, ParseResults([])) - return loc, finalResults - - def __str__( self ): - if hasattr(self,"name"): - return self.name - - if self.strRepr is None: - self.strRepr = "{" + " & ".join(_ustr(e) for e in self.exprs) + "}" - - return self.strRepr - - def checkRecursion( self, parseElementList ): - subRecCheckList = parseElementList[:] + [ self ] - for e in self.exprs: - e.checkRecursion( subRecCheckList ) - - -class ParseElementEnhance(ParserElement): - """ - Abstract subclass of C{ParserElement}, for combining and post-processing parsed tokens. - """ - def __init__( self, expr, savelist=False ): - super(ParseElementEnhance,self).__init__(savelist) - if isinstance( expr, basestring ): - if issubclass(ParserElement._literalStringClass, Token): - expr = ParserElement._literalStringClass(expr) - else: - expr = ParserElement._literalStringClass(Literal(expr)) - self.expr = expr - self.strRepr = None - if expr is not None: - self.mayIndexError = expr.mayIndexError - self.mayReturnEmpty = expr.mayReturnEmpty - self.setWhitespaceChars( expr.whiteChars ) - self.skipWhitespace = expr.skipWhitespace - self.saveAsList = expr.saveAsList - self.callPreparse = expr.callPreparse - self.ignoreExprs.extend(expr.ignoreExprs) - - def parseImpl( self, instring, loc, doActions=True ): - if self.expr is not None: - return self.expr._parse( instring, loc, doActions, callPreParse=False ) - else: - raise ParseException("",loc,self.errmsg,self) - - def leaveWhitespace( self ): - self.skipWhitespace = False - self.expr = self.expr.copy() - if self.expr is not None: - self.expr.leaveWhitespace() - return self - - def ignore( self, other ): - if isinstance( other, Suppress ): - if other not in self.ignoreExprs: - super( ParseElementEnhance, self).ignore( other ) - if self.expr is not None: - self.expr.ignore( self.ignoreExprs[-1] ) - else: - super( ParseElementEnhance, self).ignore( other ) - if self.expr is not None: - self.expr.ignore( self.ignoreExprs[-1] ) - return self - - def streamline( self ): - super(ParseElementEnhance,self).streamline() - if self.expr is not None: - self.expr.streamline() - return self - - def checkRecursion( self, parseElementList ): - if self in parseElementList: - raise RecursiveGrammarException( parseElementList+[self] ) - subRecCheckList = parseElementList[:] + [ self ] - if self.expr is not None: - self.expr.checkRecursion( subRecCheckList ) - - def validate( self, validateTrace=[] ): - tmp = validateTrace[:]+[self] - if self.expr is not None: - self.expr.validate(tmp) - self.checkRecursion( [] ) - - def __str__( self ): - try: - return super(ParseElementEnhance,self).__str__() - except Exception: - pass - - if self.strRepr is None and self.expr is not None: - self.strRepr = "%s:(%s)" % ( self.__class__.__name__, _ustr(self.expr) ) - return self.strRepr - - -class FollowedBy(ParseElementEnhance): - """ - Lookahead matching of the given parse expression. C{FollowedBy} - does I{not} advance the parsing position within the input string, it only - verifies that the specified parse expression matches at the current - position. C{FollowedBy} always returns a null token list. - - Example:: - # use FollowedBy to match a label only if it is followed by a ':' - data_word = Word(alphas) - label = data_word + FollowedBy(':') - attr_expr = Group(label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) - - OneOrMore(attr_expr).parseString("shape: SQUARE color: BLACK posn: upper left").pprint() - prints:: - [['shape', 'SQUARE'], ['color', 'BLACK'], ['posn', 'upper left']] - """ - def __init__( self, expr ): - super(FollowedBy,self).__init__(expr) - self.mayReturnEmpty = True - - def parseImpl( self, instring, loc, doActions=True ): - self.expr.tryParse( instring, loc ) - return loc, [] - - -class NotAny(ParseElementEnhance): - """ - Lookahead to disallow matching with the given parse expression. C{NotAny} - does I{not} advance the parsing position within the input string, it only - verifies that the specified parse expression does I{not} match at the current - position. Also, C{NotAny} does I{not} skip over leading whitespace. C{NotAny} - always returns a null token list. May be constructed using the '~' operator. - - Example:: - - """ - def __init__( self, expr ): - super(NotAny,self).__init__(expr) - #~ self.leaveWhitespace() - self.skipWhitespace = False # do NOT use self.leaveWhitespace(), don't want to propagate to exprs - self.mayReturnEmpty = True - self.errmsg = "Found unwanted token, "+_ustr(self.expr) - - def parseImpl( self, instring, loc, doActions=True ): - if self.expr.canParseNext(instring, loc): - raise ParseException(instring, loc, self.errmsg, self) - return loc, [] - - def __str__( self ): - if hasattr(self,"name"): - return self.name - - if self.strRepr is None: - self.strRepr = "~{" + _ustr(self.expr) + "}" - - return self.strRepr - -class _MultipleMatch(ParseElementEnhance): - def __init__( self, expr, stopOn=None): - super(_MultipleMatch, self).__init__(expr) - self.saveAsList = True - ender = stopOn - if isinstance(ender, basestring): - ender = ParserElement._literalStringClass(ender) - self.not_ender = ~ender if ender is not None else None - - def parseImpl( self, instring, loc, doActions=True ): - self_expr_parse = self.expr._parse - self_skip_ignorables = self._skipIgnorables - check_ender = self.not_ender is not None - if check_ender: - try_not_ender = self.not_ender.tryParse - - # must be at least one (but first see if we are the stopOn sentinel; - # if so, fail) - if check_ender: - try_not_ender(instring, loc) - loc, tokens = self_expr_parse( instring, loc, doActions, callPreParse=False ) - try: - hasIgnoreExprs = (not not self.ignoreExprs) - while 1: - if check_ender: - try_not_ender(instring, loc) - if hasIgnoreExprs: - preloc = self_skip_ignorables( instring, loc ) - else: - preloc = loc - loc, tmptokens = self_expr_parse( instring, preloc, doActions ) - if tmptokens or tmptokens.haskeys(): - tokens += tmptokens - except (ParseException,IndexError): - pass - - return loc, tokens - -class OneOrMore(_MultipleMatch): - """ - Repetition of one or more of the given expression. - - Parameters: - - expr - expression that must match one or more times - - stopOn - (default=C{None}) - expression for a terminating sentinel - (only required if the sentinel would ordinarily match the repetition - expression) - - Example:: - data_word = Word(alphas) - label = data_word + FollowedBy(':') - attr_expr = Group(label + Suppress(':') + OneOrMore(data_word).setParseAction(' '.join)) - - text = "shape: SQUARE posn: upper left color: BLACK" - OneOrMore(attr_expr).parseString(text).pprint() # Fail! read 'color' as data instead of next label -> [['shape', 'SQUARE color']] - - # use stopOn attribute for OneOrMore to avoid reading label string as part of the data - attr_expr = Group(label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) - OneOrMore(attr_expr).parseString(text).pprint() # Better -> [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'BLACK']] - - # could also be written as - (attr_expr * (1,)).parseString(text).pprint() - """ - - def __str__( self ): - if hasattr(self,"name"): - return self.name - - if self.strRepr is None: - self.strRepr = "{" + _ustr(self.expr) + "}..." - - return self.strRepr - -class ZeroOrMore(_MultipleMatch): - """ - Optional repetition of zero or more of the given expression. - - Parameters: - - expr - expression that must match zero or more times - - stopOn - (default=C{None}) - expression for a terminating sentinel - (only required if the sentinel would ordinarily match the repetition - expression) - - Example: similar to L{OneOrMore} - """ - def __init__( self, expr, stopOn=None): - super(ZeroOrMore,self).__init__(expr, stopOn=stopOn) - self.mayReturnEmpty = True - - def parseImpl( self, instring, loc, doActions=True ): - try: - return super(ZeroOrMore, self).parseImpl(instring, loc, doActions) - except (ParseException,IndexError): - return loc, [] - - def __str__( self ): - if hasattr(self,"name"): - return self.name - - if self.strRepr is None: - self.strRepr = "[" + _ustr(self.expr) + "]..." - - return self.strRepr - -class _NullToken(object): - def __bool__(self): - return False - __nonzero__ = __bool__ - def __str__(self): - return "" - -_optionalNotMatched = _NullToken() -class Optional(ParseElementEnhance): - """ - Optional matching of the given expression. - - Parameters: - - expr - expression that must match zero or more times - - default (optional) - value to be returned if the optional expression is not found. - - Example:: - # US postal code can be a 5-digit zip, plus optional 4-digit qualifier - zip = Combine(Word(nums, exact=5) + Optional('-' + Word(nums, exact=4))) - zip.runTests(''' - # traditional ZIP code - 12345 - - # ZIP+4 form - 12101-0001 - - # invalid ZIP - 98765- - ''') - prints:: - # traditional ZIP code - 12345 - ['12345'] - - # ZIP+4 form - 12101-0001 - ['12101-0001'] - - # invalid ZIP - 98765- - ^ - FAIL: Expected end of text (at char 5), (line:1, col:6) - """ - def __init__( self, expr, default=_optionalNotMatched ): - super(Optional,self).__init__( expr, savelist=False ) - self.saveAsList = self.expr.saveAsList - self.defaultValue = default - self.mayReturnEmpty = True - - def parseImpl( self, instring, loc, doActions=True ): - try: - loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False ) - except (ParseException,IndexError): - if self.defaultValue is not _optionalNotMatched: - if self.expr.resultsName: - tokens = ParseResults([ self.defaultValue ]) - tokens[self.expr.resultsName] = self.defaultValue - else: - tokens = [ self.defaultValue ] - else: - tokens = [] - return loc, tokens - - def __str__( self ): - if hasattr(self,"name"): - return self.name - - if self.strRepr is None: - self.strRepr = "[" + _ustr(self.expr) + "]" - - return self.strRepr - -class SkipTo(ParseElementEnhance): - """ - Token for skipping over all undefined text until the matched expression is found. - - Parameters: - - expr - target expression marking the end of the data to be skipped - - include - (default=C{False}) if True, the target expression is also parsed - (the skipped text and target expression are returned as a 2-element list). - - ignore - (default=C{None}) used to define grammars (typically quoted strings and - comments) that might contain false matches to the target expression - - failOn - (default=C{None}) define expressions that are not allowed to be - included in the skipped test; if found before the target expression is found, - the SkipTo is not a match - - Example:: - report = ''' - Outstanding Issues Report - 1 Jan 2000 - - # | Severity | Description | Days Open - -----+----------+-------------------------------------------+----------- - 101 | Critical | Intermittent system crash | 6 - 94 | Cosmetic | Spelling error on Login ('log|n') | 14 - 79 | Minor | System slow when running too many reports | 47 - ''' - integer = Word(nums) - SEP = Suppress('|') - # use SkipTo to simply match everything up until the next SEP - # - ignore quoted strings, so that a '|' character inside a quoted string does not match - # - parse action will call token.strip() for each matched token, i.e., the description body - string_data = SkipTo(SEP, ignore=quotedString) - string_data.setParseAction(tokenMap(str.strip)) - ticket_expr = (integer("issue_num") + SEP - + string_data("sev") + SEP - + string_data("desc") + SEP - + integer("days_open")) - - for tkt in ticket_expr.searchString(report): - print tkt.dump() - prints:: - ['101', 'Critical', 'Intermittent system crash', '6'] - - days_open: 6 - - desc: Intermittent system crash - - issue_num: 101 - - sev: Critical - ['94', 'Cosmetic', "Spelling error on Login ('log|n')", '14'] - - days_open: 14 - - desc: Spelling error on Login ('log|n') - - issue_num: 94 - - sev: Cosmetic - ['79', 'Minor', 'System slow when running too many reports', '47'] - - days_open: 47 - - desc: System slow when running too many reports - - issue_num: 79 - - sev: Minor - """ - def __init__( self, other, include=False, ignore=None, failOn=None ): - super( SkipTo, self ).__init__( other ) - self.ignoreExpr = ignore - self.mayReturnEmpty = True - self.mayIndexError = False - self.includeMatch = include - self.asList = False - if isinstance(failOn, basestring): - self.failOn = ParserElement._literalStringClass(failOn) - else: - self.failOn = failOn - self.errmsg = "No match found for "+_ustr(self.expr) - - def parseImpl( self, instring, loc, doActions=True ): - startloc = loc - instrlen = len(instring) - expr = self.expr - expr_parse = self.expr._parse - self_failOn_canParseNext = self.failOn.canParseNext if self.failOn is not None else None - self_ignoreExpr_tryParse = self.ignoreExpr.tryParse if self.ignoreExpr is not None else None - - tmploc = loc - while tmploc <= instrlen: - if self_failOn_canParseNext is not None: - # break if failOn expression matches - if self_failOn_canParseNext(instring, tmploc): - break - - if self_ignoreExpr_tryParse is not None: - # advance past ignore expressions - while 1: - try: - tmploc = self_ignoreExpr_tryParse(instring, tmploc) - except ParseBaseException: - break - - try: - expr_parse(instring, tmploc, doActions=False, callPreParse=False) - except (ParseException, IndexError): - # no match, advance loc in string - tmploc += 1 - else: - # matched skipto expr, done - break - - else: - # ran off the end of the input string without matching skipto expr, fail - raise ParseException(instring, loc, self.errmsg, self) - - # build up return values - loc = tmploc - skiptext = instring[startloc:loc] - skipresult = ParseResults(skiptext) - - if self.includeMatch: - loc, mat = expr_parse(instring,loc,doActions,callPreParse=False) - skipresult += mat - - return loc, skipresult - -class Forward(ParseElementEnhance): - """ - Forward declaration of an expression to be defined later - - used for recursive grammars, such as algebraic infix notation. - When the expression is known, it is assigned to the C{Forward} variable using the '<<' operator. - - Note: take care when assigning to C{Forward} not to overlook precedence of operators. - Specifically, '|' has a lower precedence than '<<', so that:: - fwdExpr << a | b | c - will actually be evaluated as:: - (fwdExpr << a) | b | c - thereby leaving b and c out as parseable alternatives. It is recommended that you - explicitly group the values inserted into the C{Forward}:: - fwdExpr << (a | b | c) - Converting to use the '<<=' operator instead will avoid this problem. - - See L{ParseResults.pprint} for an example of a recursive parser created using - C{Forward}. - """ - def __init__( self, other=None ): - super(Forward,self).__init__( other, savelist=False ) - - def __lshift__( self, other ): - if isinstance( other, basestring ): - other = ParserElement._literalStringClass(other) - self.expr = other - self.strRepr = None - self.mayIndexError = self.expr.mayIndexError - self.mayReturnEmpty = self.expr.mayReturnEmpty - self.setWhitespaceChars( self.expr.whiteChars ) - self.skipWhitespace = self.expr.skipWhitespace - self.saveAsList = self.expr.saveAsList - self.ignoreExprs.extend(self.expr.ignoreExprs) - return self - - def __ilshift__(self, other): - return self << other - - def leaveWhitespace( self ): - self.skipWhitespace = False - return self - - def streamline( self ): - if not self.streamlined: - self.streamlined = True - if self.expr is not None: - self.expr.streamline() - return self - - def validate( self, validateTrace=[] ): - if self not in validateTrace: - tmp = validateTrace[:]+[self] - if self.expr is not None: - self.expr.validate(tmp) - self.checkRecursion([]) - - def __str__( self ): - if hasattr(self,"name"): - return self.name - return self.__class__.__name__ + ": ..." - - # stubbed out for now - creates awful memory and perf issues - self._revertClass = self.__class__ - self.__class__ = _ForwardNoRecurse - try: - if self.expr is not None: - retString = _ustr(self.expr) - else: - retString = "None" - finally: - self.__class__ = self._revertClass - return self.__class__.__name__ + ": " + retString - - def copy(self): - if self.expr is not None: - return super(Forward,self).copy() - else: - ret = Forward() - ret <<= self - return ret - -class _ForwardNoRecurse(Forward): - def __str__( self ): - return "..." - -class TokenConverter(ParseElementEnhance): - """ - Abstract subclass of C{ParseExpression}, for converting parsed results. - """ - def __init__( self, expr, savelist=False ): - super(TokenConverter,self).__init__( expr )#, savelist ) - self.saveAsList = False - -class Combine(TokenConverter): - """ - Converter to concatenate all matching tokens to a single string. - By default, the matching patterns must also be contiguous in the input string; - this can be disabled by specifying C{'adjacent=False'} in the constructor. - - Example:: - real = Word(nums) + '.' + Word(nums) - print(real.parseString('3.1416')) # -> ['3', '.', '1416'] - # will also erroneously match the following - print(real.parseString('3. 1416')) # -> ['3', '.', '1416'] - - real = Combine(Word(nums) + '.' + Word(nums)) - print(real.parseString('3.1416')) # -> ['3.1416'] - # no match when there are internal spaces - print(real.parseString('3. 1416')) # -> Exception: Expected W:(0123...) - """ - def __init__( self, expr, joinString="", adjacent=True ): - super(Combine,self).__init__( expr ) - # suppress whitespace-stripping in contained parse expressions, but re-enable it on the Combine itself - if adjacent: - self.leaveWhitespace() - self.adjacent = adjacent - self.skipWhitespace = True - self.joinString = joinString - self.callPreparse = True - - def ignore( self, other ): - if self.adjacent: - ParserElement.ignore(self, other) - else: - super( Combine, self).ignore( other ) - return self - - def postParse( self, instring, loc, tokenlist ): - retToks = tokenlist.copy() - del retToks[:] - retToks += ParseResults([ "".join(tokenlist._asStringList(self.joinString)) ], modal=self.modalResults) - - if self.resultsName and retToks.haskeys(): - return [ retToks ] - else: - return retToks - -class Group(TokenConverter): - """ - Converter to return the matched tokens as a list - useful for returning tokens of C{L{ZeroOrMore}} and C{L{OneOrMore}} expressions. - - Example:: - ident = Word(alphas) - num = Word(nums) - term = ident | num - func = ident + Optional(delimitedList(term)) - print(func.parseString("fn a,b,100")) # -> ['fn', 'a', 'b', '100'] - - func = ident + Group(Optional(delimitedList(term))) - print(func.parseString("fn a,b,100")) # -> ['fn', ['a', 'b', '100']] - """ - def __init__( self, expr ): - super(Group,self).__init__( expr ) - self.saveAsList = True - - def postParse( self, instring, loc, tokenlist ): - return [ tokenlist ] - -class Dict(TokenConverter): - """ - Converter to return a repetitive expression as a list, but also as a dictionary. - Each element can also be referenced using the first token in the expression as its key. - Useful for tabular report scraping when the first column can be used as a item key. - - Example:: - data_word = Word(alphas) - label = data_word + FollowedBy(':') - attr_expr = Group(label + Suppress(':') + OneOrMore(data_word).setParseAction(' '.join)) - - text = "shape: SQUARE posn: upper left color: light blue texture: burlap" - attr_expr = (label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) - - # print attributes as plain groups - print(OneOrMore(attr_expr).parseString(text).dump()) - - # instead of OneOrMore(expr), parse using Dict(OneOrMore(Group(expr))) - Dict will auto-assign names - result = Dict(OneOrMore(Group(attr_expr))).parseString(text) - print(result.dump()) - - # access named fields as dict entries, or output as dict - print(result['shape']) - print(result.asDict()) - prints:: - ['shape', 'SQUARE', 'posn', 'upper left', 'color', 'light blue', 'texture', 'burlap'] - - [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']] - - color: light blue - - posn: upper left - - shape: SQUARE - - texture: burlap - SQUARE - {'color': 'light blue', 'posn': 'upper left', 'texture': 'burlap', 'shape': 'SQUARE'} - See more examples at L{ParseResults} of accessing fields by results name. - """ - def __init__( self, expr ): - super(Dict,self).__init__( expr ) - self.saveAsList = True - - def postParse( self, instring, loc, tokenlist ): - for i,tok in enumerate(tokenlist): - if len(tok) == 0: - continue - ikey = tok[0] - if isinstance(ikey,int): - ikey = _ustr(tok[0]).strip() - if len(tok)==1: - tokenlist[ikey] = _ParseResultsWithOffset("",i) - elif len(tok)==2 and not isinstance(tok[1],ParseResults): - tokenlist[ikey] = _ParseResultsWithOffset(tok[1],i) - else: - dictvalue = tok.copy() #ParseResults(i) - del dictvalue[0] - if len(dictvalue)!= 1 or (isinstance(dictvalue,ParseResults) and dictvalue.haskeys()): - tokenlist[ikey] = _ParseResultsWithOffset(dictvalue,i) - else: - tokenlist[ikey] = _ParseResultsWithOffset(dictvalue[0],i) - - if self.resultsName: - return [ tokenlist ] - else: - return tokenlist - - -class Suppress(TokenConverter): - """ - Converter for ignoring the results of a parsed expression. - - Example:: - source = "a, b, c,d" - wd = Word(alphas) - wd_list1 = wd + ZeroOrMore(',' + wd) - print(wd_list1.parseString(source)) - - # often, delimiters that are useful during parsing are just in the - # way afterward - use Suppress to keep them out of the parsed output - wd_list2 = wd + ZeroOrMore(Suppress(',') + wd) - print(wd_list2.parseString(source)) - prints:: - ['a', ',', 'b', ',', 'c', ',', 'd'] - ['a', 'b', 'c', 'd'] - (See also L{delimitedList}.) - """ - def postParse( self, instring, loc, tokenlist ): - return [] - - def suppress( self ): - return self - - -class OnlyOnce(object): - """ - Wrapper for parse actions, to ensure they are only called once. - """ - def __init__(self, methodCall): - self.callable = _trim_arity(methodCall) - self.called = False - def __call__(self,s,l,t): - if not self.called: - results = self.callable(s,l,t) - self.called = True - return results - raise ParseException(s,l,"") - def reset(self): - self.called = False - -def traceParseAction(f): - """ - Decorator for debugging parse actions. - - When the parse action is called, this decorator will print C{">> entering I{method-name}(line:I{current_source_line}, I{parse_location}, I{matched_tokens})".} - When the parse action completes, the decorator will print C{"<<"} followed by the returned value, or any exception that the parse action raised. - - Example:: - wd = Word(alphas) - - @traceParseAction - def remove_duplicate_chars(tokens): - return ''.join(sorted(set(''.join(tokens))) - - wds = OneOrMore(wd).setParseAction(remove_duplicate_chars) - print(wds.parseString("slkdjs sld sldd sdlf sdljf")) - prints:: - >>entering remove_duplicate_chars(line: 'slkdjs sld sldd sdlf sdljf', 0, (['slkdjs', 'sld', 'sldd', 'sdlf', 'sdljf'], {})) - <3: - thisFunc = paArgs[0].__class__.__name__ + '.' + thisFunc - sys.stderr.write( ">>entering %s(line: '%s', %d, %r)\n" % (thisFunc,line(l,s),l,t) ) - try: - ret = f(*paArgs) - except Exception as exc: - sys.stderr.write( "< ['aa', 'bb', 'cc'] - delimitedList(Word(hexnums), delim=':', combine=True).parseString("AA:BB:CC:DD:EE") # -> ['AA:BB:CC:DD:EE'] - """ - dlName = _ustr(expr)+" ["+_ustr(delim)+" "+_ustr(expr)+"]..." - if combine: - return Combine( expr + ZeroOrMore( delim + expr ) ).setName(dlName) - else: - return ( expr + ZeroOrMore( Suppress( delim ) + expr ) ).setName(dlName) - -def countedArray( expr, intExpr=None ): - """ - Helper to define a counted list of expressions. - This helper defines a pattern of the form:: - integer expr expr expr... - where the leading integer tells how many expr expressions follow. - The matched tokens returns the array of expr tokens as a list - the leading count token is suppressed. - - If C{intExpr} is specified, it should be a pyparsing expression that produces an integer value. - - Example:: - countedArray(Word(alphas)).parseString('2 ab cd ef') # -> ['ab', 'cd'] - - # in this parser, the leading integer value is given in binary, - # '10' indicating that 2 values are in the array - binaryConstant = Word('01').setParseAction(lambda t: int(t[0], 2)) - countedArray(Word(alphas), intExpr=binaryConstant).parseString('10 ab cd ef') # -> ['ab', 'cd'] - """ - arrayExpr = Forward() - def countFieldParseAction(s,l,t): - n = t[0] - arrayExpr << (n and Group(And([expr]*n)) or Group(empty)) - return [] - if intExpr is None: - intExpr = Word(nums).setParseAction(lambda t:int(t[0])) - else: - intExpr = intExpr.copy() - intExpr.setName("arrayLen") - intExpr.addParseAction(countFieldParseAction, callDuringTry=True) - return ( intExpr + arrayExpr ).setName('(len) ' + _ustr(expr) + '...') - -def _flatten(L): - ret = [] - for i in L: - if isinstance(i,list): - ret.extend(_flatten(i)) - else: - ret.append(i) - return ret - -def matchPreviousLiteral(expr): - """ - Helper to define an expression that is indirectly defined from - the tokens matched in a previous expression, that is, it looks - for a 'repeat' of a previous expression. For example:: - first = Word(nums) - second = matchPreviousLiteral(first) - matchExpr = first + ":" + second - will match C{"1:1"}, but not C{"1:2"}. Because this matches a - previous literal, will also match the leading C{"1:1"} in C{"1:10"}. - If this is not desired, use C{matchPreviousExpr}. - Do I{not} use with packrat parsing enabled. - """ - rep = Forward() - def copyTokenToRepeater(s,l,t): - if t: - if len(t) == 1: - rep << t[0] - else: - # flatten t tokens - tflat = _flatten(t.asList()) - rep << And(Literal(tt) for tt in tflat) - else: - rep << Empty() - expr.addParseAction(copyTokenToRepeater, callDuringTry=True) - rep.setName('(prev) ' + _ustr(expr)) - return rep - -def matchPreviousExpr(expr): - """ - Helper to define an expression that is indirectly defined from - the tokens matched in a previous expression, that is, it looks - for a 'repeat' of a previous expression. For example:: - first = Word(nums) - second = matchPreviousExpr(first) - matchExpr = first + ":" + second - will match C{"1:1"}, but not C{"1:2"}. Because this matches by - expressions, will I{not} match the leading C{"1:1"} in C{"1:10"}; - the expressions are evaluated first, and then compared, so - C{"1"} is compared with C{"10"}. - Do I{not} use with packrat parsing enabled. - """ - rep = Forward() - e2 = expr.copy() - rep <<= e2 - def copyTokenToRepeater(s,l,t): - matchTokens = _flatten(t.asList()) - def mustMatchTheseTokens(s,l,t): - theseTokens = _flatten(t.asList()) - if theseTokens != matchTokens: - raise ParseException("",0,"") - rep.setParseAction( mustMatchTheseTokens, callDuringTry=True ) - expr.addParseAction(copyTokenToRepeater, callDuringTry=True) - rep.setName('(prev) ' + _ustr(expr)) - return rep - -def _escapeRegexRangeChars(s): - #~ escape these chars: ^-] - for c in r"\^-]": - s = s.replace(c,_bslash+c) - s = s.replace("\n",r"\n") - s = s.replace("\t",r"\t") - return _ustr(s) - -def oneOf( strs, caseless=False, useRegex=True ): - """ - Helper to quickly define a set of alternative Literals, and makes sure to do - longest-first testing when there is a conflict, regardless of the input order, - but returns a C{L{MatchFirst}} for best performance. - - Parameters: - - strs - a string of space-delimited literals, or a collection of string literals - - caseless - (default=C{False}) - treat all literals as caseless - - useRegex - (default=C{True}) - as an optimization, will generate a Regex - object; otherwise, will generate a C{MatchFirst} object (if C{caseless=True}, or - if creating a C{Regex} raises an exception) - - Example:: - comp_oper = oneOf("< = > <= >= !=") - var = Word(alphas) - number = Word(nums) - term = var | number - comparison_expr = term + comp_oper + term - print(comparison_expr.searchString("B = 12 AA=23 B<=AA AA>12")) - prints:: - [['B', '=', '12'], ['AA', '=', '23'], ['B', '<=', 'AA'], ['AA', '>', '12']] - """ - if caseless: - isequal = ( lambda a,b: a.upper() == b.upper() ) - masks = ( lambda a,b: b.upper().startswith(a.upper()) ) - parseElementClass = CaselessLiteral - else: - isequal = ( lambda a,b: a == b ) - masks = ( lambda a,b: b.startswith(a) ) - parseElementClass = Literal - - symbols = [] - if isinstance(strs,basestring): - symbols = strs.split() - elif isinstance(strs, collections.Iterable): - symbols = list(strs) - else: - warnings.warn("Invalid argument to oneOf, expected string or iterable", - SyntaxWarning, stacklevel=2) - if not symbols: - return NoMatch() - - i = 0 - while i < len(symbols)-1: - cur = symbols[i] - for j,other in enumerate(symbols[i+1:]): - if ( isequal(other, cur) ): - del symbols[i+j+1] - break - elif ( masks(cur, other) ): - del symbols[i+j+1] - symbols.insert(i,other) - cur = other - break - else: - i += 1 - - if not caseless and useRegex: - #~ print (strs,"->", "|".join( [ _escapeRegexChars(sym) for sym in symbols] )) - try: - if len(symbols)==len("".join(symbols)): - return Regex( "[%s]" % "".join(_escapeRegexRangeChars(sym) for sym in symbols) ).setName(' | '.join(symbols)) - else: - return Regex( "|".join(re.escape(sym) for sym in symbols) ).setName(' | '.join(symbols)) - except Exception: - warnings.warn("Exception creating Regex for oneOf, building MatchFirst", - SyntaxWarning, stacklevel=2) - - - # last resort, just use MatchFirst - return MatchFirst(parseElementClass(sym) for sym in symbols).setName(' | '.join(symbols)) - -def dictOf( key, value ): - """ - Helper to easily and clearly define a dictionary by specifying the respective patterns - for the key and value. Takes care of defining the C{L{Dict}}, C{L{ZeroOrMore}}, and C{L{Group}} tokens - in the proper order. The key pattern can include delimiting markers or punctuation, - as long as they are suppressed, thereby leaving the significant key text. The value - pattern can include named results, so that the C{Dict} results can include named token - fields. - - Example:: - text = "shape: SQUARE posn: upper left color: light blue texture: burlap" - attr_expr = (label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) - print(OneOrMore(attr_expr).parseString(text).dump()) - - attr_label = label - attr_value = Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join) - - # similar to Dict, but simpler call format - result = dictOf(attr_label, attr_value).parseString(text) - print(result.dump()) - print(result['shape']) - print(result.shape) # object attribute access works too - print(result.asDict()) - prints:: - [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']] - - color: light blue - - posn: upper left - - shape: SQUARE - - texture: burlap - SQUARE - SQUARE - {'color': 'light blue', 'shape': 'SQUARE', 'posn': 'upper left', 'texture': 'burlap'} - """ - return Dict( ZeroOrMore( Group ( key + value ) ) ) - -def originalTextFor(expr, asString=True): - """ - Helper to return the original, untokenized text for a given expression. Useful to - restore the parsed fields of an HTML start tag into the raw tag text itself, or to - revert separate tokens with intervening whitespace back to the original matching - input text. By default, returns astring containing the original parsed text. - - If the optional C{asString} argument is passed as C{False}, then the return value is a - C{L{ParseResults}} containing any results names that were originally matched, and a - single token containing the original matched text from the input string. So if - the expression passed to C{L{originalTextFor}} contains expressions with defined - results names, you must set C{asString} to C{False} if you want to preserve those - results name values. - - Example:: - src = "this is test bold text normal text " - for tag in ("b","i"): - opener,closer = makeHTMLTags(tag) - patt = originalTextFor(opener + SkipTo(closer) + closer) - print(patt.searchString(src)[0]) - prints:: - [' bold text '] - ['text'] - """ - locMarker = Empty().setParseAction(lambda s,loc,t: loc) - endlocMarker = locMarker.copy() - endlocMarker.callPreparse = False - matchExpr = locMarker("_original_start") + expr + endlocMarker("_original_end") - if asString: - extractText = lambda s,l,t: s[t._original_start:t._original_end] - else: - def extractText(s,l,t): - t[:] = [s[t.pop('_original_start'):t.pop('_original_end')]] - matchExpr.setParseAction(extractText) - matchExpr.ignoreExprs = expr.ignoreExprs - return matchExpr - -def ungroup(expr): - """ - Helper to undo pyparsing's default grouping of And expressions, even - if all but one are non-empty. - """ - return TokenConverter(expr).setParseAction(lambda t:t[0]) - -def locatedExpr(expr): - """ - Helper to decorate a returned token with its starting and ending locations in the input string. - This helper adds the following results names: - - locn_start = location where matched expression begins - - locn_end = location where matched expression ends - - value = the actual parsed results - - Be careful if the input text contains C{} characters, you may want to call - C{L{ParserElement.parseWithTabs}} - - Example:: - wd = Word(alphas) - for match in locatedExpr(wd).searchString("ljsdf123lksdjjf123lkkjj1222"): - print(match) - prints:: - [[0, 'ljsdf', 5]] - [[8, 'lksdjjf', 15]] - [[18, 'lkkjj', 23]] - """ - locator = Empty().setParseAction(lambda s,l,t: l) - return Group(locator("locn_start") + expr("value") + locator.copy().leaveWhitespace()("locn_end")) - - -# convenience constants for positional expressions -empty = Empty().setName("empty") -lineStart = LineStart().setName("lineStart") -lineEnd = LineEnd().setName("lineEnd") -stringStart = StringStart().setName("stringStart") -stringEnd = StringEnd().setName("stringEnd") - -_escapedPunc = Word( _bslash, r"\[]-*.$+^?()~ ", exact=2 ).setParseAction(lambda s,l,t:t[0][1]) -_escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").setParseAction(lambda s,l,t:unichr(int(t[0].lstrip(r'\0x'),16))) -_escapedOctChar = Regex(r"\\0[0-7]+").setParseAction(lambda s,l,t:unichr(int(t[0][1:],8))) -_singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | Word(printables, excludeChars=r'\]', exact=1) | Regex(r"\w", re.UNICODE) -_charRange = Group(_singleChar + Suppress("-") + _singleChar) -_reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group( OneOrMore( _charRange | _singleChar ) ).setResultsName("body") + "]" - -def srange(s): - r""" - Helper to easily define string ranges for use in Word construction. Borrows - syntax from regexp '[]' string range definitions:: - srange("[0-9]") -> "0123456789" - srange("[a-z]") -> "abcdefghijklmnopqrstuvwxyz" - srange("[a-z$_]") -> "abcdefghijklmnopqrstuvwxyz$_" - The input string must be enclosed in []'s, and the returned string is the expanded - character set joined into a single string. - The values enclosed in the []'s may be: - - a single character - - an escaped character with a leading backslash (such as C{\-} or C{\]}) - - an escaped hex character with a leading C{'\x'} (C{\x21}, which is a C{'!'} character) - (C{\0x##} is also supported for backwards compatibility) - - an escaped octal character with a leading C{'\0'} (C{\041}, which is a C{'!'} character) - - a range of any of the above, separated by a dash (C{'a-z'}, etc.) - - any combination of the above (C{'aeiouy'}, C{'a-zA-Z0-9_$'}, etc.) - """ - _expanded = lambda p: p if not isinstance(p,ParseResults) else ''.join(unichr(c) for c in range(ord(p[0]),ord(p[1])+1)) - try: - return "".join(_expanded(part) for part in _reBracketExpr.parseString(s).body) - except Exception: - return "" - -def matchOnlyAtCol(n): - """ - Helper method for defining parse actions that require matching at a specific - column in the input text. - """ - def verifyCol(strg,locn,toks): - if col(locn,strg) != n: - raise ParseException(strg,locn,"matched token not at column %d" % n) - return verifyCol - -def replaceWith(replStr): - """ - Helper method for common parse actions that simply return a literal value. Especially - useful when used with C{L{transformString}()}. - - Example:: - num = Word(nums).setParseAction(lambda toks: int(toks[0])) - na = oneOf("N/A NA").setParseAction(replaceWith(math.nan)) - term = na | num - - OneOrMore(term).parseString("324 234 N/A 234") # -> [324, 234, nan, 234] - """ - return lambda s,l,t: [replStr] - -def removeQuotes(s,l,t): - """ - Helper parse action for removing quotation marks from parsed quoted strings. - - Example:: - # by default, quotation marks are included in parsed results - quotedString.parseString("'Now is the Winter of our Discontent'") # -> ["'Now is the Winter of our Discontent'"] - - # use removeQuotes to strip quotation marks from parsed results - quotedString.setParseAction(removeQuotes) - quotedString.parseString("'Now is the Winter of our Discontent'") # -> ["Now is the Winter of our Discontent"] - """ - return t[0][1:-1] - -def tokenMap(func, *args): - """ - Helper to define a parse action by mapping a function to all elements of a ParseResults list.If any additional - args are passed, they are forwarded to the given function as additional arguments after - the token, as in C{hex_integer = Word(hexnums).setParseAction(tokenMap(int, 16))}, which will convert the - parsed data to an integer using base 16. - - Example (compare the last to example in L{ParserElement.transformString}:: - hex_ints = OneOrMore(Word(hexnums)).setParseAction(tokenMap(int, 16)) - hex_ints.runTests(''' - 00 11 22 aa FF 0a 0d 1a - ''') - - upperword = Word(alphas).setParseAction(tokenMap(str.upper)) - OneOrMore(upperword).runTests(''' - my kingdom for a horse - ''') - - wd = Word(alphas).setParseAction(tokenMap(str.title)) - OneOrMore(wd).setParseAction(' '.join).runTests(''' - now is the winter of our discontent made glorious summer by this sun of york - ''') - prints:: - 00 11 22 aa FF 0a 0d 1a - [0, 17, 34, 170, 255, 10, 13, 26] - - my kingdom for a horse - ['MY', 'KINGDOM', 'FOR', 'A', 'HORSE'] - - now is the winter of our discontent made glorious summer by this sun of york - ['Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York'] - """ - def pa(s,l,t): - return [func(tokn, *args) for tokn in t] - - try: - func_name = getattr(func, '__name__', - getattr(func, '__class__').__name__) - except Exception: - func_name = str(func) - pa.__name__ = func_name - - return pa - -upcaseTokens = tokenMap(lambda t: _ustr(t).upper()) -"""(Deprecated) Helper parse action to convert tokens to upper case. Deprecated in favor of L{pyparsing_common.upcaseTokens}""" - -downcaseTokens = tokenMap(lambda t: _ustr(t).lower()) -"""(Deprecated) Helper parse action to convert tokens to lower case. Deprecated in favor of L{pyparsing_common.downcaseTokens}""" - -def _makeTags(tagStr, xml): - """Internal helper to construct opening and closing tag expressions, given a tag name""" - if isinstance(tagStr,basestring): - resname = tagStr - tagStr = Keyword(tagStr, caseless=not xml) - else: - resname = tagStr.name - - tagAttrName = Word(alphas,alphanums+"_-:") - if (xml): - tagAttrValue = dblQuotedString.copy().setParseAction( removeQuotes ) - openTag = Suppress("<") + tagStr("tag") + \ - Dict(ZeroOrMore(Group( tagAttrName + Suppress("=") + tagAttrValue ))) + \ - Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">") - else: - printablesLessRAbrack = "".join(c for c in printables if c not in ">") - tagAttrValue = quotedString.copy().setParseAction( removeQuotes ) | Word(printablesLessRAbrack) - openTag = Suppress("<") + tagStr("tag") + \ - Dict(ZeroOrMore(Group( tagAttrName.setParseAction(downcaseTokens) + \ - Optional( Suppress("=") + tagAttrValue ) ))) + \ - Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">") - closeTag = Combine(_L("") - - openTag = openTag.setResultsName("start"+"".join(resname.replace(":"," ").title().split())).setName("<%s>" % resname) - closeTag = closeTag.setResultsName("end"+"".join(resname.replace(":"," ").title().split())).setName("" % resname) - openTag.tag = resname - closeTag.tag = resname - return openTag, closeTag - -def makeHTMLTags(tagStr): - """ - Helper to construct opening and closing tag expressions for HTML, given a tag name. Matches - tags in either upper or lower case, attributes with namespaces and with quoted or unquoted values. - - Example:: - text = 'More info at the pyparsing wiki page' - # makeHTMLTags returns pyparsing expressions for the opening and closing tags as a 2-tuple - a,a_end = makeHTMLTags("A") - link_expr = a + SkipTo(a_end)("link_text") + a_end - - for link in link_expr.searchString(text): - # attributes in the tag (like "href" shown here) are also accessible as named results - print(link.link_text, '->', link.href) - prints:: - pyparsing -> http://pyparsing.wikispaces.com - """ - return _makeTags( tagStr, False ) - -def makeXMLTags(tagStr): - """ - Helper to construct opening and closing tag expressions for XML, given a tag name. Matches - tags only in the given upper/lower case. - - Example: similar to L{makeHTMLTags} - """ - return _makeTags( tagStr, True ) - -def withAttribute(*args,**attrDict): - """ - Helper to create a validating parse action to be used with start tags created - with C{L{makeXMLTags}} or C{L{makeHTMLTags}}. Use C{withAttribute} to qualify a starting tag - with a required attribute value, to avoid false matches on common tags such as - C{} or C{
}. - - Call C{withAttribute} with a series of attribute names and values. Specify the list - of filter attributes names and values as: - - keyword arguments, as in C{(align="right")}, or - - as an explicit dict with C{**} operator, when an attribute name is also a Python - reserved word, as in C{**{"class":"Customer", "align":"right"}} - - a list of name-value tuples, as in ( ("ns1:class", "Customer"), ("ns2:align","right") ) - For attribute names with a namespace prefix, you must use the second form. Attribute - names are matched insensitive to upper/lower case. - - If just testing for C{class} (with or without a namespace), use C{L{withClass}}. - - To verify that the attribute exists, but without specifying a value, pass - C{withAttribute.ANY_VALUE} as the value. - - Example:: - html = ''' -
- Some text -
1 4 0 1 0
-
1,3 2,3 1,1
-
this has no type
-
- - ''' - div,div_end = makeHTMLTags("div") - - # only match div tag having a type attribute with value "grid" - div_grid = div().setParseAction(withAttribute(type="grid")) - grid_expr = div_grid + SkipTo(div | div_end)("body") - for grid_header in grid_expr.searchString(html): - print(grid_header.body) - - # construct a match with any div tag having a type attribute, regardless of the value - div_any_type = div().setParseAction(withAttribute(type=withAttribute.ANY_VALUE)) - div_expr = div_any_type + SkipTo(div | div_end)("body") - for div_header in div_expr.searchString(html): - print(div_header.body) - prints:: - 1 4 0 1 0 - - 1 4 0 1 0 - 1,3 2,3 1,1 - """ - if args: - attrs = args[:] - else: - attrs = attrDict.items() - attrs = [(k,v) for k,v in attrs] - def pa(s,l,tokens): - for attrName,attrValue in attrs: - if attrName not in tokens: - raise ParseException(s,l,"no matching attribute " + attrName) - if attrValue != withAttribute.ANY_VALUE and tokens[attrName] != attrValue: - raise ParseException(s,l,"attribute '%s' has value '%s', must be '%s'" % - (attrName, tokens[attrName], attrValue)) - return pa -withAttribute.ANY_VALUE = object() - -def withClass(classname, namespace=''): - """ - Simplified version of C{L{withAttribute}} when matching on a div class - made - difficult because C{class} is a reserved word in Python. - - Example:: - html = ''' -
- Some text -
1 4 0 1 0
-
1,3 2,3 1,1
-
this <div> has no class
-
- - ''' - div,div_end = makeHTMLTags("div") - div_grid = div().setParseAction(withClass("grid")) - - grid_expr = div_grid + SkipTo(div | div_end)("body") - for grid_header in grid_expr.searchString(html): - print(grid_header.body) - - div_any_type = div().setParseAction(withClass(withAttribute.ANY_VALUE)) - div_expr = div_any_type + SkipTo(div | div_end)("body") - for div_header in div_expr.searchString(html): - print(div_header.body) - prints:: - 1 4 0 1 0 - - 1 4 0 1 0 - 1,3 2,3 1,1 - """ - classattr = "%s:class" % namespace if namespace else "class" - return withAttribute(**{classattr : classname}) - -opAssoc = _Constants() -opAssoc.LEFT = object() -opAssoc.RIGHT = object() - -def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): - """ - Helper method for constructing grammars of expressions made up of - operators working in a precedence hierarchy. Operators may be unary or - binary, left- or right-associative. Parse actions can also be attached - to operator expressions. The generated parser will also recognize the use - of parentheses to override operator precedences (see example below). - - Note: if you define a deep operator list, you may see performance issues - when using infixNotation. See L{ParserElement.enablePackrat} for a - mechanism to potentially improve your parser performance. - - Parameters: - - baseExpr - expression representing the most basic element for the nested - - opList - list of tuples, one for each operator precedence level in the - expression grammar; each tuple is of the form - (opExpr, numTerms, rightLeftAssoc, parseAction), where: - - opExpr is the pyparsing expression for the operator; - may also be a string, which will be converted to a Literal; - if numTerms is 3, opExpr is a tuple of two expressions, for the - two operators separating the 3 terms - - numTerms is the number of terms for this operator (must - be 1, 2, or 3) - - rightLeftAssoc is the indicator whether the operator is - right or left associative, using the pyparsing-defined - constants C{opAssoc.RIGHT} and C{opAssoc.LEFT}. - - parseAction is the parse action to be associated with - expressions matching this operator expression (the - parse action tuple member may be omitted); if the parse action - is passed a tuple or list of functions, this is equivalent to - calling C{setParseAction(*fn)} (L{ParserElement.setParseAction}) - - lpar - expression for matching left-parentheses (default=C{Suppress('(')}) - - rpar - expression for matching right-parentheses (default=C{Suppress(')')}) - - Example:: - # simple example of four-function arithmetic with ints and variable names - integer = pyparsing_common.signed_integer - varname = pyparsing_common.identifier - - arith_expr = infixNotation(integer | varname, - [ - ('-', 1, opAssoc.RIGHT), - (oneOf('* /'), 2, opAssoc.LEFT), - (oneOf('+ -'), 2, opAssoc.LEFT), - ]) - - arith_expr.runTests(''' - 5+3*6 - (5+3)*6 - -2--11 - ''', fullDump=False) - prints:: - 5+3*6 - [[5, '+', [3, '*', 6]]] - - (5+3)*6 - [[[5, '+', 3], '*', 6]] - - -2--11 - [[['-', 2], '-', ['-', 11]]] - """ - ret = Forward() - lastExpr = baseExpr | ( lpar + ret + rpar ) - for i,operDef in enumerate(opList): - opExpr,arity,rightLeftAssoc,pa = (operDef + (None,))[:4] - termName = "%s term" % opExpr if arity < 3 else "%s%s term" % opExpr - if arity == 3: - if opExpr is None or len(opExpr) != 2: - raise ValueError("if numterms=3, opExpr must be a tuple or list of two expressions") - opExpr1, opExpr2 = opExpr - thisExpr = Forward().setName(termName) - if rightLeftAssoc == opAssoc.LEFT: - if arity == 1: - matchExpr = FollowedBy(lastExpr + opExpr) + Group( lastExpr + OneOrMore( opExpr ) ) - elif arity == 2: - if opExpr is not None: - matchExpr = FollowedBy(lastExpr + opExpr + lastExpr) + Group( lastExpr + OneOrMore( opExpr + lastExpr ) ) - else: - matchExpr = FollowedBy(lastExpr+lastExpr) + Group( lastExpr + OneOrMore(lastExpr) ) - elif arity == 3: - matchExpr = FollowedBy(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr) + \ - Group( lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr ) - else: - raise ValueError("operator must be unary (1), binary (2), or ternary (3)") - elif rightLeftAssoc == opAssoc.RIGHT: - if arity == 1: - # try to avoid LR with this extra test - if not isinstance(opExpr, Optional): - opExpr = Optional(opExpr) - matchExpr = FollowedBy(opExpr.expr + thisExpr) + Group( opExpr + thisExpr ) - elif arity == 2: - if opExpr is not None: - matchExpr = FollowedBy(lastExpr + opExpr + thisExpr) + Group( lastExpr + OneOrMore( opExpr + thisExpr ) ) - else: - matchExpr = FollowedBy(lastExpr + thisExpr) + Group( lastExpr + OneOrMore( thisExpr ) ) - elif arity == 3: - matchExpr = FollowedBy(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) + \ - Group( lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr ) - else: - raise ValueError("operator must be unary (1), binary (2), or ternary (3)") - else: - raise ValueError("operator must indicate right or left associativity") - if pa: - if isinstance(pa, (tuple, list)): - matchExpr.setParseAction(*pa) - else: - matchExpr.setParseAction(pa) - thisExpr <<= ( matchExpr.setName(termName) | lastExpr ) - lastExpr = thisExpr - ret <<= lastExpr - return ret - -operatorPrecedence = infixNotation -"""(Deprecated) Former name of C{L{infixNotation}}, will be dropped in a future release.""" - -dblQuotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*')+'"').setName("string enclosed in double quotes") -sglQuotedString = Combine(Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*")+"'").setName("string enclosed in single quotes") -quotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*')+'"'| - Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*")+"'").setName("quotedString using single or double quotes") -unicodeString = Combine(_L('u') + quotedString.copy()).setName("unicode string literal") - -def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.copy()): - """ - Helper method for defining nested lists enclosed in opening and closing - delimiters ("(" and ")" are the default). - - Parameters: - - opener - opening character for a nested list (default=C{"("}); can also be a pyparsing expression - - closer - closing character for a nested list (default=C{")"}); can also be a pyparsing expression - - content - expression for items within the nested lists (default=C{None}) - - ignoreExpr - expression for ignoring opening and closing delimiters (default=C{quotedString}) - - If an expression is not provided for the content argument, the nested - expression will capture all whitespace-delimited content between delimiters - as a list of separate values. - - Use the C{ignoreExpr} argument to define expressions that may contain - opening or closing characters that should not be treated as opening - or closing characters for nesting, such as quotedString or a comment - expression. Specify multiple expressions using an C{L{Or}} or C{L{MatchFirst}}. - The default is L{quotedString}, but if no expressions are to be ignored, - then pass C{None} for this argument. - - Example:: - data_type = oneOf("void int short long char float double") - decl_data_type = Combine(data_type + Optional(Word('*'))) - ident = Word(alphas+'_', alphanums+'_') - number = pyparsing_common.number - arg = Group(decl_data_type + ident) - LPAR,RPAR = map(Suppress, "()") - - code_body = nestedExpr('{', '}', ignoreExpr=(quotedString | cStyleComment)) - - c_function = (decl_data_type("type") - + ident("name") - + LPAR + Optional(delimitedList(arg), [])("args") + RPAR - + code_body("body")) - c_function.ignore(cStyleComment) - - source_code = ''' - int is_odd(int x) { - return (x%2); - } - - int dec_to_hex(char hchar) { - if (hchar >= '0' && hchar <= '9') { - return (ord(hchar)-ord('0')); - } else { - return (10+ord(hchar)-ord('A')); - } - } - ''' - for func in c_function.searchString(source_code): - print("%(name)s (%(type)s) args: %(args)s" % func) - - prints:: - is_odd (int) args: [['int', 'x']] - dec_to_hex (int) args: [['char', 'hchar']] - """ - if opener == closer: - raise ValueError("opening and closing strings cannot be the same") - if content is None: - if isinstance(opener,basestring) and isinstance(closer,basestring): - if len(opener) == 1 and len(closer)==1: - if ignoreExpr is not None: - content = (Combine(OneOrMore(~ignoreExpr + - CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS,exact=1)) - ).setParseAction(lambda t:t[0].strip())) - else: - content = (empty.copy()+CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS - ).setParseAction(lambda t:t[0].strip())) - else: - if ignoreExpr is not None: - content = (Combine(OneOrMore(~ignoreExpr + - ~Literal(opener) + ~Literal(closer) + - CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1)) - ).setParseAction(lambda t:t[0].strip())) - else: - content = (Combine(OneOrMore(~Literal(opener) + ~Literal(closer) + - CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1)) - ).setParseAction(lambda t:t[0].strip())) - else: - raise ValueError("opening and closing arguments must be strings if no content expression is given") - ret = Forward() - if ignoreExpr is not None: - ret <<= Group( Suppress(opener) + ZeroOrMore( ignoreExpr | ret | content ) + Suppress(closer) ) - else: - ret <<= Group( Suppress(opener) + ZeroOrMore( ret | content ) + Suppress(closer) ) - ret.setName('nested %s%s expression' % (opener,closer)) - return ret - -def indentedBlock(blockStatementExpr, indentStack, indent=True): - """ - Helper method for defining space-delimited indentation blocks, such as - those used to define block statements in Python source code. - - Parameters: - - blockStatementExpr - expression defining syntax of statement that - is repeated within the indented block - - indentStack - list created by caller to manage indentation stack - (multiple statementWithIndentedBlock expressions within a single grammar - should share a common indentStack) - - indent - boolean indicating whether block must be indented beyond the - the current level; set to False for block of left-most statements - (default=C{True}) - - A valid block must contain at least one C{blockStatement}. - - Example:: - data = ''' - def A(z): - A1 - B = 100 - G = A2 - A2 - A3 - B - def BB(a,b,c): - BB1 - def BBA(): - bba1 - bba2 - bba3 - C - D - def spam(x,y): - def eggs(z): - pass - ''' - - - indentStack = [1] - stmt = Forward() - - identifier = Word(alphas, alphanums) - funcDecl = ("def" + identifier + Group( "(" + Optional( delimitedList(identifier) ) + ")" ) + ":") - func_body = indentedBlock(stmt, indentStack) - funcDef = Group( funcDecl + func_body ) - - rvalue = Forward() - funcCall = Group(identifier + "(" + Optional(delimitedList(rvalue)) + ")") - rvalue << (funcCall | identifier | Word(nums)) - assignment = Group(identifier + "=" + rvalue) - stmt << ( funcDef | assignment | identifier ) - - module_body = OneOrMore(stmt) - - parseTree = module_body.parseString(data) - parseTree.pprint() - prints:: - [['def', - 'A', - ['(', 'z', ')'], - ':', - [['A1'], [['B', '=', '100']], [['G', '=', 'A2']], ['A2'], ['A3']]], - 'B', - ['def', - 'BB', - ['(', 'a', 'b', 'c', ')'], - ':', - [['BB1'], [['def', 'BBA', ['(', ')'], ':', [['bba1'], ['bba2'], ['bba3']]]]]], - 'C', - 'D', - ['def', - 'spam', - ['(', 'x', 'y', ')'], - ':', - [[['def', 'eggs', ['(', 'z', ')'], ':', [['pass']]]]]]] - """ - def checkPeerIndent(s,l,t): - if l >= len(s): return - curCol = col(l,s) - if curCol != indentStack[-1]: - if curCol > indentStack[-1]: - raise ParseFatalException(s,l,"illegal nesting") - raise ParseException(s,l,"not a peer entry") - - def checkSubIndent(s,l,t): - curCol = col(l,s) - if curCol > indentStack[-1]: - indentStack.append( curCol ) - else: - raise ParseException(s,l,"not a subentry") - - def checkUnindent(s,l,t): - if l >= len(s): return - curCol = col(l,s) - if not(indentStack and curCol < indentStack[-1] and curCol <= indentStack[-2]): - raise ParseException(s,l,"not an unindent") - indentStack.pop() - - NL = OneOrMore(LineEnd().setWhitespaceChars("\t ").suppress()) - INDENT = (Empty() + Empty().setParseAction(checkSubIndent)).setName('INDENT') - PEER = Empty().setParseAction(checkPeerIndent).setName('') - UNDENT = Empty().setParseAction(checkUnindent).setName('UNINDENT') - if indent: - smExpr = Group( Optional(NL) + - #~ FollowedBy(blockStatementExpr) + - INDENT + (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) + UNDENT) - else: - smExpr = Group( Optional(NL) + - (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) ) - blockStatementExpr.ignore(_bslash + LineEnd()) - return smExpr.setName('indented block') - -alphas8bit = srange(r"[\0xc0-\0xd6\0xd8-\0xf6\0xf8-\0xff]") -punc8bit = srange(r"[\0xa1-\0xbf\0xd7\0xf7]") - -anyOpenTag,anyCloseTag = makeHTMLTags(Word(alphas,alphanums+"_:").setName('any tag')) -_htmlEntityMap = dict(zip("gt lt amp nbsp quot apos".split(),'><& "\'')) -commonHTMLEntity = Regex('&(?P' + '|'.join(_htmlEntityMap.keys()) +");").setName("common HTML entity") -def replaceHTMLEntity(t): - """Helper parser action to replace common HTML entities with their special characters""" - return _htmlEntityMap.get(t.entity) - -# it's easy to get these comment structures wrong - they're very common, so may as well make them available -cStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/').setName("C style comment") -"Comment of the form C{/* ... */}" - -htmlComment = Regex(r"").setName("HTML comment") -"Comment of the form C{}" - -restOfLine = Regex(r".*").leaveWhitespace().setName("rest of line") -dblSlashComment = Regex(r"//(?:\\\n|[^\n])*").setName("// comment") -"Comment of the form C{// ... (to end of line)}" - -cppStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/'| dblSlashComment).setName("C++ style comment") -"Comment of either form C{L{cStyleComment}} or C{L{dblSlashComment}}" - -javaStyleComment = cppStyleComment -"Same as C{L{cppStyleComment}}" - -pythonStyleComment = Regex(r"#.*").setName("Python style comment") -"Comment of the form C{# ... (to end of line)}" - -_commasepitem = Combine(OneOrMore(Word(printables, excludeChars=',') + - Optional( Word(" \t") + - ~Literal(",") + ~LineEnd() ) ) ).streamline().setName("commaItem") -commaSeparatedList = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("commaSeparatedList") -"""(Deprecated) Predefined expression of 1 or more printable words or quoted strings, separated by commas. - This expression is deprecated in favor of L{pyparsing_common.comma_separated_list}.""" - -# some other useful expressions - using lower-case class name since we are really using this as a namespace -class pyparsing_common: - """ - Here are some common low-level expressions that may be useful in jump-starting parser development: - - numeric forms (L{integers}, L{reals}, L{scientific notation}) - - common L{programming identifiers} - - network addresses (L{MAC}, L{IPv4}, L{IPv6}) - - ISO8601 L{dates} and L{datetime} - - L{UUID} - - L{comma-separated list} - Parse actions: - - C{L{convertToInteger}} - - C{L{convertToFloat}} - - C{L{convertToDate}} - - C{L{convertToDatetime}} - - C{L{stripHTMLTags}} - - C{L{upcaseTokens}} - - C{L{downcaseTokens}} - - Example:: - pyparsing_common.number.runTests(''' - # any int or real number, returned as the appropriate type - 100 - -100 - +100 - 3.14159 - 6.02e23 - 1e-12 - ''') - - pyparsing_common.fnumber.runTests(''' - # any int or real number, returned as float - 100 - -100 - +100 - 3.14159 - 6.02e23 - 1e-12 - ''') - - pyparsing_common.hex_integer.runTests(''' - # hex numbers - 100 - FF - ''') - - pyparsing_common.fraction.runTests(''' - # fractions - 1/2 - -3/4 - ''') - - pyparsing_common.mixed_integer.runTests(''' - # mixed fractions - 1 - 1/2 - -3/4 - 1-3/4 - ''') - - import uuid - pyparsing_common.uuid.setParseAction(tokenMap(uuid.UUID)) - pyparsing_common.uuid.runTests(''' - # uuid - 12345678-1234-5678-1234-567812345678 - ''') - prints:: - # any int or real number, returned as the appropriate type - 100 - [100] - - -100 - [-100] - - +100 - [100] - - 3.14159 - [3.14159] - - 6.02e23 - [6.02e+23] - - 1e-12 - [1e-12] - - # any int or real number, returned as float - 100 - [100.0] - - -100 - [-100.0] - - +100 - [100.0] - - 3.14159 - [3.14159] - - 6.02e23 - [6.02e+23] - - 1e-12 - [1e-12] - - # hex numbers - 100 - [256] - - FF - [255] - - # fractions - 1/2 - [0.5] - - -3/4 - [-0.75] - - # mixed fractions - 1 - [1] - - 1/2 - [0.5] - - -3/4 - [-0.75] - - 1-3/4 - [1.75] - - # uuid - 12345678-1234-5678-1234-567812345678 - [UUID('12345678-1234-5678-1234-567812345678')] - """ - - convertToInteger = tokenMap(int) - """ - Parse action for converting parsed integers to Python int - """ - - convertToFloat = tokenMap(float) - """ - Parse action for converting parsed numbers to Python float - """ - - integer = Word(nums).setName("integer").setParseAction(convertToInteger) - """expression that parses an unsigned integer, returns an int""" - - hex_integer = Word(hexnums).setName("hex integer").setParseAction(tokenMap(int,16)) - """expression that parses a hexadecimal integer, returns an int""" - - signed_integer = Regex(r'[+-]?\d+').setName("signed integer").setParseAction(convertToInteger) - """expression that parses an integer with optional leading sign, returns an int""" - - fraction = (signed_integer().setParseAction(convertToFloat) + '/' + signed_integer().setParseAction(convertToFloat)).setName("fraction") - """fractional expression of an integer divided by an integer, returns a float""" - fraction.addParseAction(lambda t: t[0]/t[-1]) - - mixed_integer = (fraction | signed_integer + Optional(Optional('-').suppress() + fraction)).setName("fraction or mixed integer-fraction") - """mixed integer of the form 'integer - fraction', with optional leading integer, returns float""" - mixed_integer.addParseAction(sum) - - real = Regex(r'[+-]?\d+\.\d*').setName("real number").setParseAction(convertToFloat) - """expression that parses a floating point number and returns a float""" - - sci_real = Regex(r'[+-]?\d+([eE][+-]?\d+|\.\d*([eE][+-]?\d+)?)').setName("real number with scientific notation").setParseAction(convertToFloat) - """expression that parses a floating point number with optional scientific notation and returns a float""" - - # streamlining this expression makes the docs nicer-looking - number = (sci_real | real | signed_integer).streamline() - """any numeric expression, returns the corresponding Python type""" - - fnumber = Regex(r'[+-]?\d+\.?\d*([eE][+-]?\d+)?').setName("fnumber").setParseAction(convertToFloat) - """any int or real number, returned as float""" - - identifier = Word(alphas+'_', alphanums+'_').setName("identifier") - """typical code identifier (leading alpha or '_', followed by 0 or more alphas, nums, or '_')""" - - ipv4_address = Regex(r'(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})(\.(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})){3}').setName("IPv4 address") - "IPv4 address (C{0.0.0.0 - 255.255.255.255})" - - _ipv6_part = Regex(r'[0-9a-fA-F]{1,4}').setName("hex_integer") - _full_ipv6_address = (_ipv6_part + (':' + _ipv6_part)*7).setName("full IPv6 address") - _short_ipv6_address = (Optional(_ipv6_part + (':' + _ipv6_part)*(0,6)) + "::" + Optional(_ipv6_part + (':' + _ipv6_part)*(0,6))).setName("short IPv6 address") - _short_ipv6_address.addCondition(lambda t: sum(1 for tt in t if pyparsing_common._ipv6_part.matches(tt)) < 8) - _mixed_ipv6_address = ("::ffff:" + ipv4_address).setName("mixed IPv6 address") - ipv6_address = Combine((_full_ipv6_address | _mixed_ipv6_address | _short_ipv6_address).setName("IPv6 address")).setName("IPv6 address") - "IPv6 address (long, short, or mixed form)" - - mac_address = Regex(r'[0-9a-fA-F]{2}([:.-])[0-9a-fA-F]{2}(?:\1[0-9a-fA-F]{2}){4}').setName("MAC address") - "MAC address xx:xx:xx:xx:xx (may also have '-' or '.' delimiters)" - - @staticmethod - def convertToDate(fmt="%Y-%m-%d"): - """ - Helper to create a parse action for converting parsed date string to Python datetime.date - - Params - - - fmt - format to be passed to datetime.strptime (default=C{"%Y-%m-%d"}) - - Example:: - date_expr = pyparsing_common.iso8601_date.copy() - date_expr.setParseAction(pyparsing_common.convertToDate()) - print(date_expr.parseString("1999-12-31")) - prints:: - [datetime.date(1999, 12, 31)] - """ - def cvt_fn(s,l,t): - try: - return datetime.strptime(t[0], fmt).date() - except ValueError as ve: - raise ParseException(s, l, str(ve)) - return cvt_fn - - @staticmethod - def convertToDatetime(fmt="%Y-%m-%dT%H:%M:%S.%f"): - """ - Helper to create a parse action for converting parsed datetime string to Python datetime.datetime - - Params - - - fmt - format to be passed to datetime.strptime (default=C{"%Y-%m-%dT%H:%M:%S.%f"}) - - Example:: - dt_expr = pyparsing_common.iso8601_datetime.copy() - dt_expr.setParseAction(pyparsing_common.convertToDatetime()) - print(dt_expr.parseString("1999-12-31T23:59:59.999")) - prints:: - [datetime.datetime(1999, 12, 31, 23, 59, 59, 999000)] - """ - def cvt_fn(s,l,t): - try: - return datetime.strptime(t[0], fmt) - except ValueError as ve: - raise ParseException(s, l, str(ve)) - return cvt_fn - - iso8601_date = Regex(r'(?P\d{4})(?:-(?P\d\d)(?:-(?P\d\d))?)?').setName("ISO8601 date") - "ISO8601 date (C{yyyy-mm-dd})" - - iso8601_datetime = Regex(r'(?P\d{4})-(?P\d\d)-(?P\d\d)[T ](?P\d\d):(?P\d\d)(:(?P\d\d(\.\d*)?)?)?(?PZ|[+-]\d\d:?\d\d)?').setName("ISO8601 datetime") - "ISO8601 datetime (C{yyyy-mm-ddThh:mm:ss.s(Z|+-00:00)}) - trailing seconds, milliseconds, and timezone optional; accepts separating C{'T'} or C{' '}" - - uuid = Regex(r'[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}').setName("UUID") - "UUID (C{xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx})" - - _html_stripper = anyOpenTag.suppress() | anyCloseTag.suppress() - @staticmethod - def stripHTMLTags(s, l, tokens): - """ - Parse action to remove HTML tags from web page HTML source - - Example:: - # strip HTML links from normal text - text = 'More info at the pyparsing wiki page' - td,td_end = makeHTMLTags("TD") - table_text = td + SkipTo(td_end).setParseAction(pyparsing_common.stripHTMLTags)("body") + td_end - - print(table_text.parseString(text).body) # -> 'More info at the pyparsing wiki page' - """ - return pyparsing_common._html_stripper.transformString(tokens[0]) - - _commasepitem = Combine(OneOrMore(~Literal(",") + ~LineEnd() + Word(printables, excludeChars=',') - + Optional( White(" \t") ) ) ).streamline().setName("commaItem") - comma_separated_list = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("comma separated list") - """Predefined expression of 1 or more printable words or quoted strings, separated by commas.""" - - upcaseTokens = staticmethod(tokenMap(lambda t: _ustr(t).upper())) - """Parse action to convert tokens to upper case.""" - - downcaseTokens = staticmethod(tokenMap(lambda t: _ustr(t).lower())) - """Parse action to convert tokens to lower case.""" - - -if __name__ == "__main__": - - selectToken = CaselessLiteral("select") - fromToken = CaselessLiteral("from") - - ident = Word(alphas, alphanums + "_$") - - columnName = delimitedList(ident, ".", combine=True).setParseAction(upcaseTokens) - columnNameList = Group(delimitedList(columnName)).setName("columns") - columnSpec = ('*' | columnNameList) - - tableName = delimitedList(ident, ".", combine=True).setParseAction(upcaseTokens) - tableNameList = Group(delimitedList(tableName)).setName("tables") - - simpleSQL = selectToken("command") + columnSpec("columns") + fromToken + tableNameList("tables") - - # demo runTests method, including embedded comments in test string - simpleSQL.runTests(""" - # '*' as column list and dotted table name - select * from SYS.XYZZY - - # caseless match on "SELECT", and casts back to "select" - SELECT * from XYZZY, ABC - - # list of column names, and mixed case SELECT keyword - Select AA,BB,CC from Sys.dual - - # multiple tables - Select A, B, C from Sys.dual, Table2 - - # invalid SELECT keyword - should fail - Xelect A, B, C from Sys.dual - - # incomplete command - should fail - Select - - # invalid column name - should fail - Select ^^^ frox Sys.dual - - """) - - pyparsing_common.number.runTests(""" - 100 - -100 - +100 - 3.14159 - 6.02e23 - 1e-12 - """) - - # any int or real number, returned as float - pyparsing_common.fnumber.runTests(""" - 100 - -100 - +100 - 3.14159 - 6.02e23 - 1e-12 - """) - - pyparsing_common.hex_integer.runTests(""" - 100 - FF - """) - - import uuid - pyparsing_common.uuid.setParseAction(tokenMap(uuid.UUID)) - pyparsing_common.uuid.runTests(""" - 12345678-1234-5678-1234-567812345678 - """) diff --git a/doc/tutorial/machine_learning_map/svg2imagemap.py b/doc/tutorial/machine_learning_map/svg2imagemap.py deleted file mode 100644 index 53cd37d0b7cc0..0000000000000 --- a/doc/tutorial/machine_learning_map/svg2imagemap.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/local/bin/python - -""" -This script converts a subset of SVG into an HTML imagemap - -Note *subset*. It only handles elements, for which it only pays -attention to the M and L commands. Further, it only notices the "translate" -transform. - -It was written to generate the examples in the documentation for maphilight, -and thus is very squarely aimed at handling several SVG maps from wikipedia. -It *assumes* that all the s it will need are inside a . Any -outside of a will be ignored. - -It takes several possible arguments, in the form: -$ svn2imagemap.py FILENAME [x y [group1 group2 ... groupN]] - -FILENAME must be the name of an SVG file. All other arguments are optional. - -x and y, if present, are the dimensions of the image you'll be creating from -the SVG. If not present, it assumes the values of the width and height -attributes in the SVG file. - -group1 through groupN are group ids. If only want particular groups used, -enter their ids here and all others will be ignored. -""" -import os -import re -import sys -import xml.dom.minidom - -import parse_path - -if len(sys.argv) == 1: - sys.exit("svn2imagemap.py FILENAME [x y [group1 group2 ... groupN]]") -if not os.path.exists(sys.argv[1]): - sys.exit("Input file does not exist") -x, y, groups = None, None, None -if len(sys.argv) >= 3: - x = float(sys.argv[2]) - y = float(sys.argv[3]) - if len(sys.argv) > 3: - groups = sys.argv[4:] - -svg_file = xml.dom.minidom.parse(sys.argv[1]) -svg = svg_file.getElementsByTagName('svg')[0] - -raw_width = float(svg.getAttribute('width')) -raw_height = float(svg.getAttribute('height')) -width_ratio = x and (x / raw_width) or 1 -height_ratio = y and (y / raw_height) or 1 - -if groups: - elements = [g for g in svg.getElementsByTagName('g') if (g.hasAttribute('id') and g.getAttribute('id') in groups)] - elements.extend([p for p in svg.getElementsByTagName('path') if (p.hasAttribute('id') and p.getAttribute('id') in groups)]) -else: - elements = svg.getElementsByTagName('g') - -parsed_groups = {} -for e in elements: - paths = [] - if e.nodeName == 'g': - for path in e.getElementsByTagName('path'): - points = parse_path.get_points(path.getAttribute('d')) - for pointset in points: - paths.append([path.getAttribute('id'), pointset]) - else: - points = parse_path.get_points(e.getAttribute('d')) - for pointset in points: - paths.append([e.getAttribute('id'), pointset]) - if e.hasAttribute('transform'): - print(e.getAttribute('id'), e.getAttribute('transform')) - for transform in re.findall(r'(\w+)\((-?\d+.?\d*),(-?\d+.?\d*)\)', e.getAttribute('transform')): - if transform[0] == 'translate': - x_shift = float(transform[1]) - y_shift = float(transform[2]) - for path in paths: - path[1] = [(p[0] + x_shift, p[1] + y_shift) for p in path[1]] - - parsed_groups[e.getAttribute('id')] = paths - -out = [] -for g in parsed_groups: - for path in parsed_groups[g]: - out.append('' % - (path[0], ', '.join([("%d,%d" % (p[0]*width_ratio, p[1]*height_ratio)) for p in path[1]]))) - -outfile = open(sys.argv[1].replace('.svg', '.html'), 'w') -outfile.write('\n'.join(out)) diff --git a/doc/tutorial/statistical_inference/index.rst b/doc/tutorial/statistical_inference/index.rst deleted file mode 100644 index 1ea527054fc38..0000000000000 --- a/doc/tutorial/statistical_inference/index.rst +++ /dev/null @@ -1,36 +0,0 @@ -.. _stat_learn_tut_index: - -========================================================================== -A tutorial on statistical-learning for scientific data processing -========================================================================== - -.. topic:: Statistical learning - - `Machine learning `_ is - a technique with a growing importance, as the - size of the datasets experimental sciences are facing is rapidly - growing. Problems it tackles range from building a prediction function - linking different observations, to classifying observations, or - learning the structure in an unlabeled dataset. - - This tutorial will explore *statistical learning*, the use of - machine learning techniques with the goal of `statistical inference - `_: - drawing conclusions on the data at hand. - - Scikit-learn is a Python module integrating classic machine - learning algorithms in the tightly-knit world of scientific Python - packages (`NumPy `_, `SciPy - `_, `matplotlib - `_). - -.. include:: ../../includes/big_toc_css.rst - -.. toctree:: - :maxdepth: 2 - - settings - supervised_learning - model_selection - unsupervised_learning - putting_together diff --git a/doc/tutorial/statistical_inference/model_selection.rst b/doc/tutorial/statistical_inference/model_selection.rst deleted file mode 100644 index 070e86c18e8b1..0000000000000 --- a/doc/tutorial/statistical_inference/model_selection.rst +++ /dev/null @@ -1,281 +0,0 @@ -.. _model_selection_tut: - -============================================================ -Model selection: choosing estimators and their parameters -============================================================ - -Score, and cross-validated scores -================================== - -As we have seen, every estimator exposes a ``score`` method that can judge -the quality of the fit (or the prediction) on new data. **Bigger is -better**. - -:: - - >>> from sklearn import datasets, svm - >>> X_digits, y_digits = datasets.load_digits(return_X_y=True) - >>> svc = svm.SVC(C=1, kernel='linear') - >>> svc.fit(X_digits[:-100], y_digits[:-100]).score(X_digits[-100:], y_digits[-100:]) - 0.98 - -To get a better measure of prediction accuracy (which we can use as a -proxy for goodness of fit of the model), we can successively split the -data in *folds* that we use for training and testing:: - - >>> import numpy as np - >>> X_folds = np.array_split(X_digits, 3) - >>> y_folds = np.array_split(y_digits, 3) - >>> scores = list() - >>> for k in range(3): - ... # We use 'list' to copy, in order to 'pop' later on - ... X_train = list(X_folds) - ... X_test = X_train.pop(k) - ... X_train = np.concatenate(X_train) - ... y_train = list(y_folds) - ... y_test = y_train.pop(k) - ... y_train = np.concatenate(y_train) - ... scores.append(svc.fit(X_train, y_train).score(X_test, y_test)) - >>> print(scores) - [0.934..., 0.956..., 0.939...] - -.. currentmodule:: sklearn.model_selection - -This is called a :class:`KFold` cross-validation. - -.. _cv_generators_tut: - -Cross-validation generators -============================= - -Scikit-learn has a collection of classes which can be used to generate lists of -train/test indices for popular cross-validation strategies. - -They expose a ``split`` method which accepts the input -dataset to be split and yields the train/test set indices for each iteration -of the chosen cross-validation strategy. - -This example shows an example usage of the ``split`` method. - - >>> from sklearn.model_selection import KFold, cross_val_score - >>> X = ["a", "a", "a", "b", "b", "c", "c", "c", "c", "c"] - >>> k_fold = KFold(n_splits=5) - >>> for train_indices, test_indices in k_fold.split(X): - ... print('Train: %s | test: %s' % (train_indices, test_indices)) - Train: [2 3 4 5 6 7 8 9] | test: [0 1] - Train: [0 1 4 5 6 7 8 9] | test: [2 3] - Train: [0 1 2 3 6 7 8 9] | test: [4 5] - Train: [0 1 2 3 4 5 8 9] | test: [6 7] - Train: [0 1 2 3 4 5 6 7] | test: [8 9] - -The cross-validation can then be performed easily:: - - >>> [svc.fit(X_digits[train], y_digits[train]).score(X_digits[test], y_digits[test]) - ... for train, test in k_fold.split(X_digits)] - [0.963..., 0.922..., 0.963..., 0.963..., 0.930...] - -The cross-validation score can be directly calculated using the -:func:`cross_val_score` helper. Given an estimator, the cross-validation object -and the input dataset, the :func:`cross_val_score` splits the data repeatedly into -a training and a testing set, trains the estimator using the training set and -computes the scores based on the testing set for each iteration of cross-validation. - -By default the estimator's ``score`` method is used to compute the individual scores. - -Refer the :ref:`metrics module ` to learn more on the available scoring -methods. - - >>> cross_val_score(svc, X_digits, y_digits, cv=k_fold, n_jobs=-1) - array([0.96388889, 0.92222222, 0.9637883 , 0.9637883 , 0.93036212]) - -`n_jobs=-1` means that the computation will be dispatched on all the CPUs -of the computer. - -Alternatively, the ``scoring`` argument can be provided to specify an alternative -scoring method. - - >>> cross_val_score(svc, X_digits, y_digits, cv=k_fold, - ... scoring='precision_macro') - array([0.96578289, 0.92708922, 0.96681476, 0.96362897, 0.93192644]) - - **Cross-validation generators** - - -.. list-table:: - - * - - - :class:`KFold` **(n_splits, shuffle, random_state)** - - - :class:`StratifiedKFold` **(n_splits, shuffle, random_state)** - - - :class:`GroupKFold` **(n_splits)** - - - * - - - Splits it into K folds, trains on K-1 and then tests on the left-out. - - - Same as K-Fold but preserves the class distribution within each fold. - - - Ensures that the same group is not in both testing and training sets. - - -.. list-table:: - - * - - - :class:`ShuffleSplit` **(n_splits, test_size, train_size, random_state)** - - - :class:`StratifiedShuffleSplit` - - - :class:`GroupShuffleSplit` - - * - - - Generates train/test indices based on random permutation. - - - Same as shuffle split but preserves the class distribution within each iteration. - - - Ensures that the same group is not in both testing and training sets. - - -.. list-table:: - - * - - - :class:`LeaveOneGroupOut` **()** - - - :class:`LeavePGroupsOut` **(n_groups)** - - - :class:`LeaveOneOut` **()** - - - - * - - - Takes a group array to group observations. - - - Leave P groups out. - - - Leave one observation out. - - - -.. list-table:: - - * - - - :class:`LeavePOut` **(p)** - - - :class:`PredefinedSplit` - - * - - - Leave P observations out. - - - Generates train/test indices based on predefined splits. - - -.. currentmodule:: sklearn.svm - -.. topic:: **Exercise** - - On the digits dataset, plot the cross-validation score of a :class:`SVC` - estimator with an linear kernel as a function of parameter ``C`` (use a - logarithmic grid of points, from 1 to 10). - - .. literalinclude:: ../../auto_examples/exercises/plot_cv_digits.py - :lines: 13-23 - - .. image:: /auto_examples/exercises/images/sphx_glr_plot_cv_digits_001.png - :target: ../../auto_examples/exercises/plot_cv_digits.html - :align: center - :scale: 90 - - **Solution:** :ref:`sphx_glr_auto_examples_exercises_plot_cv_digits.py` - -Grid-search and cross-validated estimators -============================================ - -Grid-search -------------- - -.. currentmodule:: sklearn.model_selection - -scikit-learn provides an object that, given data, computes the score -during the fit of an estimator on a parameter grid and chooses the -parameters to maximize the cross-validation score. This object takes an -estimator during the construction and exposes an estimator API:: - - >>> from sklearn.model_selection import GridSearchCV, cross_val_score - >>> Cs = np.logspace(-6, -1, 10) - >>> clf = GridSearchCV(estimator=svc, param_grid=dict(C=Cs), - ... n_jobs=-1) - >>> clf.fit(X_digits[:1000], y_digits[:1000]) # doctest: +SKIP - GridSearchCV(cv=None,... - >>> clf.best_score_ # doctest: +SKIP - 0.925... - >>> clf.best_estimator_.C # doctest: +SKIP - 0.0077... - - >>> # Prediction performance on test set is not as good as on train set - >>> clf.score(X_digits[1000:], y_digits[1000:]) # doctest: +SKIP - 0.943... - - -By default, the :class:`GridSearchCV` uses a 5-fold cross-validation. However, -if it detects that a classifier is passed, rather than a regressor, it uses -a stratified 5-fold. - -.. topic:: Nested cross-validation - - :: - - >>> cross_val_score(clf, X_digits, y_digits) # doctest: +SKIP - array([0.938..., 0.963..., 0.944...]) - - Two cross-validation loops are performed in parallel: one by the - :class:`GridSearchCV` estimator to set ``gamma`` and the other one by - ``cross_val_score`` to measure the prediction performance of the - estimator. The resulting scores are unbiased estimates of the - prediction score on new data. - -.. warning:: - - You cannot nest objects with parallel computing (``n_jobs`` different - than 1). - -.. _cv_estimators_tut: - -Cross-validated estimators ----------------------------- - -Cross-validation to set a parameter can be done more efficiently on an -algorithm-by-algorithm basis. This is why, for certain estimators, -scikit-learn exposes :ref:`cross_validation` estimators that set their -parameter automatically by cross-validation:: - - >>> from sklearn import linear_model, datasets - >>> lasso = linear_model.LassoCV() - >>> X_diabetes, y_diabetes = datasets.load_diabetes(return_X_y=True) - >>> lasso.fit(X_diabetes, y_diabetes) - LassoCV() - >>> # The estimator chose automatically its lambda: - >>> lasso.alpha_ - 0.00375... - -These estimators are called similarly to their counterparts, with 'CV' -appended to their name. - -.. topic:: **Exercise** - - On the diabetes dataset, find the optimal regularization parameter - alpha. - - **Bonus**: How much can you trust the selection of alpha? - - .. literalinclude:: ../../auto_examples/exercises/plot_cv_diabetes.py - :lines: 17-24 - - **Solution:** :ref:`sphx_glr_auto_examples_exercises_plot_cv_diabetes.py` diff --git a/doc/tutorial/statistical_inference/putting_together.rst b/doc/tutorial/statistical_inference/putting_together.rst deleted file mode 100644 index 033bed2e33884..0000000000000 --- a/doc/tutorial/statistical_inference/putting_together.rst +++ /dev/null @@ -1,62 +0,0 @@ -========================= -Putting it all together -========================= - -.. Imports - >>> import numpy as np - -Pipelining -============ - -We have seen that some estimators can transform data and that some estimators -can predict variables. We can also create combined estimators: - -.. literalinclude:: ../../auto_examples/compose/plot_digits_pipe.py - :lines: 23-63 - -.. image:: ../../auto_examples/compose/images/sphx_glr_plot_digits_pipe_001.png - :target: ../../auto_examples/compose/plot_digits_pipe.html - :scale: 65 - :align: center - -Face recognition with eigenfaces -================================= - -The dataset used in this example is a preprocessed excerpt of the -"Labeled Faces in the Wild", also known as LFW_: - - http://vis-www.cs.umass.edu/lfw/lfw-funneled.tgz (233MB) - -.. _LFW: http://vis-www.cs.umass.edu/lfw/ - -.. literalinclude:: ../../auto_examples/applications/plot_face_recognition.py - -.. figure:: ../../images/plot_face_recognition_1.png - :scale: 50 - - **Prediction** - -.. figure:: ../../images/plot_face_recognition_2.png - :scale: 50 - - **Eigenfaces** - -Expected results for the top 5 most represented people in the dataset:: - - precision recall f1-score support - - Gerhard_Schroeder 0.91 0.75 0.82 28 - Donald_Rumsfeld 0.84 0.82 0.83 33 - Tony_Blair 0.65 0.82 0.73 34 - Colin_Powell 0.78 0.88 0.83 58 - George_W_Bush 0.93 0.86 0.90 129 - - avg / total 0.86 0.84 0.85 282 - - -Open problem: Stock Market Structure -===================================== - -Can we predict the variation in stock prices for Google over a given time frame? - -:ref:`stock_market` diff --git a/doc/tutorial/statistical_inference/settings.rst b/doc/tutorial/statistical_inference/settings.rst deleted file mode 100644 index 0ab9e39d63345..0000000000000 --- a/doc/tutorial/statistical_inference/settings.rst +++ /dev/null @@ -1,92 +0,0 @@ - -========================================================================== -Statistical learning: the setting and the estimator object in scikit-learn -========================================================================== - -Datasets -========= - -Scikit-learn deals with learning information from one or more -datasets that are represented as 2D arrays. They can be understood as a -list of multi-dimensional observations. We say that the first axis of -these arrays is the **samples** axis, while the second is the -**features** axis. - -.. topic:: A simple example shipped with scikit-learn: iris dataset - - :: - - >>> from sklearn import datasets - >>> iris = datasets.load_iris() - >>> data = iris.data - >>> data.shape - (150, 4) - - It is made of 150 observations of irises, each described by 4 - features: their sepal and petal length and width, as detailed in - ``iris.DESCR``. - -When the data is not initially in the ``(n_samples, n_features)`` shape, it -needs to be preprocessed in order to be used by scikit-learn. - -.. topic:: An example of reshaping data would be the digits dataset - - The digits dataset is made of 1797 8x8 images of hand-written - digits :: - - >>> digits = datasets.load_digits() - >>> digits.images.shape - (1797, 8, 8) - >>> import matplotlib.pyplot as plt #doctest: +SKIP - >>> plt.imshow(digits.images[-1], - ... cmap=plt.cm.gray_r) #doctest: +SKIP - - - .. image:: /auto_examples/datasets/images/sphx_glr_plot_digits_last_image_001.png - :target: ../../auto_examples/datasets/plot_digits_last_image.html - :align: center - - To use this dataset with scikit-learn, we transform each 8x8 image into a - feature vector of length 64 :: - - >>> data = digits.images.reshape( - ... (digits.images.shape[0], -1) - ... ) - -Estimators objects -=================== - -.. Some code to make the doctests run - - >>> from sklearn.base import BaseEstimator - >>> class Estimator(BaseEstimator): - ... def __init__(self, param1=0, param2=0): - ... self.param1 = param1 - ... self.param2 = param2 - ... def fit(self, data): - ... pass - >>> estimator = Estimator() - -**Fitting data**: the main API implemented by scikit-learn is that of the -`estimator`. An estimator is any object that learns from data; -it may be a classification, regression or clustering algorithm or -a *transformer* that extracts/filters useful features from raw data. - -All estimator objects expose a ``fit`` method that takes a dataset -(usually a 2-d array): - - >>> estimator.fit(data) - -**Estimator parameters**: All the parameters of an estimator can be set -when it is instantiated or by modifying the corresponding attribute:: - - >>> estimator = Estimator(param1=1, param2=2) - >>> estimator.param1 - 1 - -**Estimated parameters**: When data is fitted with an estimator, -parameters are estimated from the data at hand. All the estimated -parameters are attributes of the estimator object ending by an -underscore:: - - >>> estimator.estimated_param_ #doctest: +SKIP diff --git a/doc/tutorial/statistical_inference/supervised_learning.rst b/doc/tutorial/statistical_inference/supervised_learning.rst deleted file mode 100644 index 3d87830fa0b26..0000000000000 --- a/doc/tutorial/statistical_inference/supervised_learning.rst +++ /dev/null @@ -1,524 +0,0 @@ -.. _supervised_learning_tut: - -======================================================================================= -Supervised learning: predicting an output variable from high-dimensional observations -======================================================================================= - - -.. topic:: The problem solved in supervised learning - - :ref:`Supervised learning ` - consists in learning the link between two - datasets: the observed data ``X`` and an external variable ``y`` that we - are trying to predict, usually called "target" or "labels". Most often, - ``y`` is a 1D array of length ``n_samples``. - - All supervised `estimators `_ - in scikit-learn implement a ``fit(X, y)`` method to fit the model - and a ``predict(X)`` method that, given unlabeled observations ``X``, - returns the predicted labels ``y``. - -.. topic:: Vocabulary: classification and regression - - If the prediction task is to classify the observations in a set of - finite labels, in other words to "name" the objects observed, the task - is said to be a **classification** task. On the other hand, if the goal - is to predict a continuous target variable, it is said to be a - **regression** task. - - When doing classification in scikit-learn, ``y`` is a vector of integers - or strings. - - Note: See the :ref:`Introduction to machine learning with scikit-learn - Tutorial ` for a quick run-through on the basic machine - learning vocabulary used within scikit-learn. - -Nearest neighbor and the curse of dimensionality -================================================= - -.. topic:: Classifying irises: - - The iris dataset is a classification task consisting in identifying 3 - different types of irises (Setosa, Versicolour, and Virginica) from - their petal and sepal length and width:: - - >>> import numpy as np - >>> from sklearn import datasets - >>> iris_X, iris_y = datasets.load_iris(return_X_y=True) - >>> np.unique(iris_y) - array([0, 1, 2]) - - .. image:: /auto_examples/datasets/images/sphx_glr_plot_iris_dataset_001.png - :target: ../../auto_examples/datasets/plot_iris_dataset.html - :align: center - :scale: 50 - -k-Nearest neighbors classifier -------------------------------- - -The simplest possible classifier is the -`nearest neighbor `_: -given a new observation ``X_test``, find in the training set (i.e. the data -used to train the estimator) the observation with the closest feature vector. -(Please see the :ref:`Nearest Neighbors section` of the online -Scikit-learn documentation for more information about this type of classifier.) - -.. topic:: Training set and testing set - - While experimenting with any learning algorithm, it is important not to - test the prediction of an estimator on the data used to fit the - estimator as this would not be evaluating the performance of the - estimator on **new data**. This is why datasets are often split into - *train* and *test* data. - -**KNN (k nearest neighbors) classification example**: - -.. image:: /auto_examples/neighbors/images/sphx_glr_plot_classification_001.png - :target: ../../auto_examples/neighbors/plot_classification.html - :align: center - :scale: 70 - -:: - - >>> # Split iris data in train and test data - >>> # A random permutation, to split the data randomly - >>> np.random.seed(0) - >>> indices = np.random.permutation(len(iris_X)) - >>> iris_X_train = iris_X[indices[:-10]] - >>> iris_y_train = iris_y[indices[:-10]] - >>> iris_X_test = iris_X[indices[-10:]] - >>> iris_y_test = iris_y[indices[-10:]] - >>> # Create and fit a nearest-neighbor classifier - >>> from sklearn.neighbors import KNeighborsClassifier - >>> knn = KNeighborsClassifier() - >>> knn.fit(iris_X_train, iris_y_train) - KNeighborsClassifier() - >>> knn.predict(iris_X_test) - array([1, 2, 1, 0, 0, 0, 2, 1, 2, 0]) - >>> iris_y_test - array([1, 1, 1, 0, 0, 0, 2, 1, 2, 0]) - -.. _curse_of_dimensionality: - -The curse of dimensionality -------------------------------- - -For an estimator to be effective, you need the distance between neighboring -points to be less than some value :math:`d`, which depends on the problem. -In one dimension, this requires on average :math:`n \sim 1/d` points. -In the context of the above :math:`k`-NN example, if the data is described by -just one feature with values ranging from 0 to 1 and with :math:`n` training -observations, then new data will be no further away than :math:`1/n`. -Therefore, the nearest neighbor decision rule will be efficient as soon as -:math:`1/n` is small compared to the scale of between-class feature variations. - -If the number of features is :math:`p`, you now require :math:`n \sim 1/d^p` -points. Let's say that we require 10 points in one dimension: now :math:`10^p` -points are required in :math:`p` dimensions to pave the :math:`[0, 1]` space. -As :math:`p` becomes large, the number of training points required for a good -estimator grows exponentially. - -For example, if each point is just a single number (8 bytes), then an -effective :math:`k`-NN estimator in a paltry :math:`p \sim 20` dimensions would -require more training data than the current estimated size of the entire -internet (Âą1000 Exabytes or so). - -This is called the -`curse of dimensionality `_ -and is a core problem that machine learning addresses. - -Linear model: from regression to sparsity -========================================== - -.. topic:: Diabetes dataset - - The diabetes dataset consists of 10 physiological variables (age, - sex, weight, blood pressure) measure on 442 patients, and an - indication of disease progression after one year:: - - >>> diabetes_X, diabetes_y = datasets.load_diabetes(return_X_y=True) - >>> diabetes_X_train = diabetes_X[:-20] - >>> diabetes_X_test = diabetes_X[-20:] - >>> diabetes_y_train = diabetes_y[:-20] - >>> diabetes_y_test = diabetes_y[-20:] - - The task at hand is to predict disease progression from physiological - variables. - -Linear regression ------------------- - -.. currentmodule:: sklearn.linear_model - -:class:`LinearRegression`, -in its simplest form, fits a linear model to the data set by adjusting -a set of parameters in order to make the sum of the squared residuals -of the model as small as possible. - -Linear models: :math:`y = X\beta + \epsilon` - - * :math:`X`: data - * :math:`y`: target variable - * :math:`\beta`: Coefficients - * :math:`\epsilon`: Observation noise - -.. image:: /auto_examples/linear_model/images/sphx_glr_plot_ols_001.png - :target: ../../auto_examples/linear_model/plot_ols.html - :scale: 50 - :align: center - -:: - - >>> from sklearn import linear_model - >>> regr = linear_model.LinearRegression() - >>> regr.fit(diabetes_X_train, diabetes_y_train) - LinearRegression() - >>> print(regr.coef_) - [ 0.30349955 -237.63931533 510.53060544 327.73698041 -814.13170937 - 492.81458798 102.84845219 184.60648906 743.51961675 76.09517222] - - - >>> # The mean square error - >>> np.mean((regr.predict(diabetes_X_test) - diabetes_y_test)**2) - 2004.56760268... - - >>> # Explained variance score: 1 is perfect prediction - >>> # and 0 means that there is no linear relationship - >>> # between X and y. - >>> regr.score(diabetes_X_test, diabetes_y_test) - 0.5850753022690... - - -.. _shrinkage: - -Shrinkage ----------- - -If there are few data points per dimension, noise in the observations -induces high variance: - -:: - - >>> X = np.c_[ .5, 1].T - >>> y = [.5, 1] - >>> test = np.c_[ 0, 2].T - >>> regr = linear_model.LinearRegression() - - >>> import matplotlib.pyplot as plt # doctest: +SKIP - >>> plt.figure() # doctest: +SKIP - - >>> np.random.seed(0) - >>> for _ in range(6): # doctest: +SKIP - ... this_X = .1 * np.random.normal(size=(2, 1)) + X - ... regr.fit(this_X, y) - ... plt.plot(test, regr.predict(test)) # doctest: +SKIP - ... plt.scatter(this_X, y, s=3) # doctest: +SKIP - -.. image:: /auto_examples/linear_model/images/sphx_glr_plot_ols_ridge_variance_001.png - :target: ../../auto_examples/linear_model/plot_ols_ridge_variance.html - :align: center - -A solution in high-dimensional statistical learning is to *shrink* the -regression coefficients to zero: any two randomly chosen set of -observations are likely to be uncorrelated. This is called :class:`Ridge` -regression: - -:: - - >>> regr = linear_model.Ridge(alpha=.1) - - >>> plt.figure() # doctest: +SKIP - - >>> np.random.seed(0) - >>> for _ in range(6): # doctest: +SKIP - ... this_X = .1 * np.random.normal(size=(2, 1)) + X - ... regr.fit(this_X, y) - ... plt.plot(test, regr.predict(test)) # doctest: +SKIP - ... plt.scatter(this_X, y, s=3) # doctest: +SKIP - -.. image:: /auto_examples/linear_model/images/sphx_glr_plot_ols_ridge_variance_002.png - :target: ../../auto_examples/linear_model/plot_ols_ridge_variance.html - :align: center - -This is an example of **bias/variance tradeoff**: the larger the ridge -``alpha`` parameter, the higher the bias and the lower the variance. - -We can choose ``alpha`` to minimize left out error, this time using the -diabetes dataset rather than our synthetic data:: - - >>> alphas = np.logspace(-4, -1, 6) - >>> print([regr.set_params(alpha=alpha) - ... .fit(diabetes_X_train, diabetes_y_train) - ... .score(diabetes_X_test, diabetes_y_test) - ... for alpha in alphas]) - [0.5851110683883..., 0.5852073015444..., 0.5854677540698..., - 0.5855512036503..., 0.5830717085554..., 0.57058999437...] - - -.. note:: - - Capturing in the fitted parameters noise that prevents the model to - generalize to new data is called - `overfitting `_. The bias introduced - by the ridge regression is called a - `regularization `_. - -.. _sparsity: - -Sparsity ----------- - - -.. |diabetes_ols_1| image:: /auto_examples/linear_model/images/sphx_glr_plot_ols_3d_001.png - :target: ../../auto_examples/linear_model/plot_ols_3d.html - :scale: 65 - -.. |diabetes_ols_3| image:: /auto_examples/linear_model/images/sphx_glr_plot_ols_3d_003.png - :target: ../../auto_examples/linear_model/plot_ols_3d.html - :scale: 65 - -.. |diabetes_ols_2| image:: /auto_examples/linear_model/images/sphx_glr_plot_ols_3d_002.png - :target: ../../auto_examples/linear_model/plot_ols_3d.html - :scale: 65 - - - - -.. rst-class:: centered - - **Fitting only features 1 and 2** - -.. centered:: |diabetes_ols_1| |diabetes_ols_3| |diabetes_ols_2| - -.. note:: - - A representation of the full diabetes dataset would involve 11 - dimensions (10 feature dimensions and one of the target variable). It - is hard to develop an intuition on such representation, but it may be - useful to keep in mind that it would be a fairly *empty* space. - - - -We can see that, although feature 2 has a strong coefficient on the full -model, it conveys little information on ``y`` when considered with feature 1. - -To improve the conditioning of the problem (i.e. mitigating the -:ref:`curse_of_dimensionality`), it would be interesting to select only the -informative features and set non-informative ones, like feature 2 to 0. Ridge -regression will decrease their contribution, but not set them to zero. Another -penalization approach, called :ref:`lasso` (least absolute shrinkage and -selection operator), can set some coefficients to zero. Such methods are -called **sparse method** and sparsity can be seen as an -application of Occam's razor: *prefer simpler models*. - -:: - - >>> regr = linear_model.Lasso() - >>> scores = [regr.set_params(alpha=alpha) - ... .fit(diabetes_X_train, diabetes_y_train) - ... .score(diabetes_X_test, diabetes_y_test) - ... for alpha in alphas] - >>> best_alpha = alphas[scores.index(max(scores))] - >>> regr.alpha = best_alpha - >>> regr.fit(diabetes_X_train, diabetes_y_train) - Lasso(alpha=0.025118864315095794) - >>> print(regr.coef_) - [ 0. -212.437... 517.194... 313.779... -160.830... - -0. -187.195... 69.382... 508.660... 71.842...] - -.. topic:: **Different algorithms for the same problem** - - Different algorithms can be used to solve the same mathematical - problem. For instance the ``Lasso`` object in scikit-learn - solves the lasso regression problem using a - `coordinate descent `_ method, - that is efficient on large datasets. However, scikit-learn also - provides the :class:`LassoLars` object using the *LARS* algorithm, - which is very efficient for problems in which the weight vector estimated - is very sparse (i.e. problems with very few observations). - -.. _clf_tut: - -Classification ---------------- - -For classification, as in the labeling -`iris `_ task, linear -regression is not the right approach as it will give too much weight to -data far from the decision frontier. A linear approach is to fit a sigmoid -function or **logistic** function: - -.. image:: /auto_examples/linear_model/images/sphx_glr_plot_logistic_001.png - :target: ../../auto_examples/linear_model/plot_logistic.html - :scale: 70 - :align: center - -.. math:: - - y = \textrm{sigmoid}(X\beta - \textrm{offset}) + \epsilon = - \frac{1}{1 + \textrm{exp}(- X\beta + \textrm{offset})} + \epsilon - -:: - - >>> log = linear_model.LogisticRegression(C=1e5) - >>> log.fit(iris_X_train, iris_y_train) - LogisticRegression(C=100000.0) - -This is known as :class:`LogisticRegression`. - -.. image:: /auto_examples/linear_model/images/sphx_glr_plot_iris_logistic_001.png - :target: ../../auto_examples/linear_model/plot_iris_logistic.html - :scale: 83 - :align: center - -.. topic:: Multiclass classification - - If you have several classes to predict, an option often used is to fit - one-versus-all classifiers and then use a voting heuristic for the final - decision. - -.. topic:: Shrinkage and sparsity with logistic regression - - The ``C`` parameter controls the amount of regularization in the - :class:`LogisticRegression` object: a large value for ``C`` results in - less regularization. - ``penalty="l2"`` gives :ref:`shrinkage` (i.e. non-sparse coefficients), while - ``penalty="l1"`` gives :ref:`sparsity`. - -.. topic:: **Exercise** - :class: green - - Try classifying the digits dataset with nearest neighbors and a linear - model. Leave out the last 10% and test prediction performance on these - observations. - - .. literalinclude:: ../../auto_examples/exercises/plot_digits_classification_exercise.py - :lines: 15-19 - - A solution can be downloaded :download:`here <../../auto_examples/exercises/plot_digits_classification_exercise.py>`. - - -Support vector machines (SVMs) -================================ - -Linear SVMs -------------- - - -:ref:`svm` belong to the discriminant model family: they try to find a combination of -samples to build a plane maximizing the margin between the two classes. -Regularization is set by the ``C`` parameter: a small value for ``C`` means the margin -is calculated using many or all of the observations around the separating line -(more regularization); -a large value for ``C`` means the margin is calculated on observations close to -the separating line (less regularization). - -.. currentmodule :: sklearn.svm - -.. figure:: /auto_examples/svm/images/sphx_glr_plot_svm_margin_001.png - :target: ../../auto_examples/svm/plot_svm_margin.html - - **Unregularized SVM** - -.. figure:: /auto_examples/svm/images/sphx_glr_plot_svm_margin_002.png - :target: ../../auto_examples/svm/plot_svm_margin.html - - **Regularized SVM (default)** - -.. topic:: Example: - - - :ref:`sphx_glr_auto_examples_svm_plot_iris_svc.py` - - -SVMs can be used in regression --:class:`SVR` (Support Vector Regression)--, or in -classification --:class:`SVC` (Support Vector Classification). - -:: - - >>> from sklearn import svm - >>> svc = svm.SVC(kernel='linear') - >>> svc.fit(iris_X_train, iris_y_train) - SVC(kernel='linear') - - -.. warning:: **Normalizing data** - - For many estimators, including the SVMs, having datasets with unit - standard deviation for each feature is important to get good - prediction. - -.. _using_kernels_tut: - -Using kernels -------------- - -Classes are not always linearly separable in feature space. The solution is to -build a decision function that is not linear but may be polynomial instead. -This is done using the *kernel trick* that can be seen as -creating a decision energy by positioning *kernels* on observations: - -Linear kernel -^^^^^^^^^^^^^ - -:: - - >>> svc = svm.SVC(kernel='linear') - -.. image:: /auto_examples/svm/images/sphx_glr_plot_svm_kernels_001.png - :target: ../../auto_examples/svm/plot_svm_kernels.html - -Polynomial kernel -^^^^^^^^^^^^^^^^^ - -:: - - >>> svc = svm.SVC(kernel='poly', - ... degree=3) - >>> # degree: polynomial degree - -.. image:: /auto_examples/svm/images/sphx_glr_plot_svm_kernels_002.png - :target: ../../auto_examples/svm/plot_svm_kernels.html - -RBF kernel (Radial Basis Function) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:: - - >>> svc = svm.SVC(kernel='rbf') - >>> # gamma: inverse of size of - >>> # radial kernel - -.. image:: /auto_examples/svm/images/sphx_glr_plot_svm_kernels_003.png - :target: ../../auto_examples/svm/plot_svm_kernels.html - - - -.. topic:: **Interactive example** - - See the :ref:`SVM GUI ` to download - ``svm_gui.py``; add data points of both classes with right and left button, - fit the model and change parameters and data. - -.. topic:: **Exercise** - :class: green - - Try classifying classes 1 and 2 from the iris dataset with SVMs, with - the 2 first features. Leave out 10% of each class and test prediction - performance on these observations. - - **Warning**: the classes are ordered, do not leave out the last 10%, - you would be testing on only one class. - - **Hint**: You can use the ``decision_function`` method on a grid to get - intuitions. - - .. literalinclude:: ../../auto_examples/exercises/plot_iris_exercise.py - :lines: 18-23 - - .. image:: /auto_examples/datasets/images/sphx_glr_plot_iris_dataset_001.png - :target: ../../auto_examples/datasets/plot_iris_dataset.html - :align: center - :scale: 70 - - - A solution can be downloaded :download:`here <../../auto_examples/exercises/plot_iris_exercise.py>` diff --git a/doc/tutorial/statistical_inference/unsupervised_learning.rst b/doc/tutorial/statistical_inference/unsupervised_learning.rst deleted file mode 100644 index 033872fac895e..0000000000000 --- a/doc/tutorial/statistical_inference/unsupervised_learning.rst +++ /dev/null @@ -1,312 +0,0 @@ -============================================================ -Unsupervised learning: seeking representations of the data -============================================================ - -Clustering: grouping observations together -============================================ - -.. topic:: The problem solved in clustering - - Given the iris dataset, if we knew that there were 3 types of iris, but - did not have access to a taxonomist to label them: we could try a - **clustering task**: split the observations into well-separated group - called *clusters*. - -.. - >>> # Set the PRNG - >>> import numpy as np - >>> np.random.seed(1) - -K-means clustering -------------------- - -Note that there exist a lot of different clustering criteria and associated -algorithms. The simplest clustering algorithm is :ref:`k_means`. - -.. image:: /auto_examples/cluster/images/sphx_glr_plot_cluster_iris_002.png - :target: ../../auto_examples/cluster/plot_cluster_iris.html - :scale: 70 - :align: center - -:: - - >>> from sklearn import cluster, datasets - >>> X_iris, y_iris = datasets.load_iris(return_X_y=True) - - >>> k_means = cluster.KMeans(n_clusters=3) - >>> k_means.fit(X_iris) - KMeans(n_clusters=3) - >>> print(k_means.labels_[::10]) - [1 1 1 1 1 0 0 0 0 0 2 2 2 2 2] - >>> print(y_iris[::10]) - [0 0 0 0 0 1 1 1 1 1 2 2 2 2 2] - -.. warning:: - - There is absolutely no guarantee of recovering a ground truth. First, - choosing the right number of clusters is hard. Second, the algorithm - is sensitive to initialization, and can fall into local minima, - although scikit-learn employs several tricks to mitigate this issue. - - | - - .. figure:: /auto_examples/cluster/images/sphx_glr_plot_cluster_iris_003.png - :target: ../../auto_examples/cluster/plot_cluster_iris.html - :scale: 63 - - **Bad initialization** - - .. figure:: /auto_examples/cluster/images/sphx_glr_plot_cluster_iris_001.png - :target: ../../auto_examples/cluster/plot_cluster_iris.html - :scale: 63 - - **8 clusters** - - .. figure:: /auto_examples/cluster/images/sphx_glr_plot_cluster_iris_004.png - :target: ../../auto_examples/cluster/plot_cluster_iris.html - :scale: 63 - - **Ground truth** - - **Don't over-interpret clustering results** - -.. topic:: **Application example: vector quantization** - - Clustering in general and KMeans, in particular, can be seen as a way - of choosing a small number of exemplars to compress the information. - The problem is sometimes known as - `vector quantization `_. - For instance, this can be used to posterize an image:: - - >>> import scipy as sp - >>> try: - ... face = sp.face(gray=True) - ... except AttributeError: - ... from scipy import misc - ... face = misc.face(gray=True) - >>> X = face.reshape((-1, 1)) # We need an (n_sample, n_feature) array - >>> k_means = cluster.KMeans(n_clusters=5, n_init=1) - >>> k_means.fit(X) - KMeans(n_clusters=5, n_init=1) - >>> values = k_means.cluster_centers_.squeeze() - >>> labels = k_means.labels_ - >>> face_compressed = np.choose(labels, values) - >>> face_compressed.shape = face.shape - - - .. figure:: /auto_examples/cluster/images/sphx_glr_plot_face_compress_001.png - :target: ../../auto_examples/cluster/plot_face_compress.html - - **Raw image** - - .. figure:: /auto_examples/cluster/images/sphx_glr_plot_face_compress_003.png - :target: ../../auto_examples/cluster/plot_face_compress.html - - **K-means quantization** - - .. figure:: /auto_examples/cluster/images/sphx_glr_plot_face_compress_002.png - :target: ../../auto_examples/cluster/plot_face_compress.html - - **Equal bins** - - - .. figure:: /auto_examples/cluster/images/sphx_glr_plot_face_compress_004.png - :target: ../../auto_examples/cluster/plot_face_compress.html - - **Image histogram** - -Hierarchical agglomerative clustering: Ward ---------------------------------------------- - -A :ref:`hierarchical_clustering` method is a type of cluster analysis -that aims to build a hierarchy of clusters. In general, the various approaches -of this technique are either: - - * **Agglomerative** - bottom-up approaches: each observation starts in its - own cluster, and clusters are iteratively merged in such a way to - minimize a *linkage* criterion. This approach is particularly interesting - when the clusters of interest are made of only a few observations. When - the number of clusters is large, it is much more computationally efficient - than k-means. - - * **Divisive** - top-down approaches: all observations start in one - cluster, which is iteratively split as one moves down the hierarchy. - For estimating large numbers of clusters, this approach is both slow (due - to all observations starting as one cluster, which it splits recursively) - and statistically ill-posed. - -Connectivity-constrained clustering -..................................... - -With agglomerative clustering, it is possible to specify which samples can be -clustered together by giving a connectivity graph. Graphs in scikit-learn -are represented by their adjacency matrix. Often, a sparse matrix is used. -This can be useful, for instance, to retrieve connected regions (sometimes -also referred to as connected components) when clustering an image. - -.. image:: /auto_examples/cluster/images/sphx_glr_plot_coin_ward_segmentation_001.png - :target: ../../auto_examples/cluster/plot_coin_ward_segmentation.html - :scale: 40 - :align: center - -:: - - >>> from skimage.data import coins - >>> from scipy.ndimage.filters import gaussian_filter - >>> from skimage.transform import rescale - >>> rescaled_coins = rescale( - ... gaussian_filter(coins(), sigma=2), - ... 0.2, mode='reflect', anti_aliasing=False, multichannel=False - ... ) - >>> X = np.reshape(rescaled_coins, (-1, 1)) - -We need a vectorized version of the image. `'rescaled_coins'` is a down-scaled -version of the coins image to speed up the process:: - - >>> from sklearn.feature_extraction import grid_to_graph - >>> connectivity = grid_to_graph(*rescaled_coins.shape) - -Define the graph structure of the data. Pixels connected to their neighbors:: - - >>> n_clusters = 27 # number of regions - - >>> from sklearn.cluster import AgglomerativeClustering - >>> ward = AgglomerativeClustering(n_clusters=n_clusters, linkage='ward', - ... connectivity=connectivity) - >>> ward.fit(X) - AgglomerativeClustering(connectivity=..., n_clusters=27) - >>> label = np.reshape(ward.labels_, rescaled_coins.shape) - -Feature agglomeration -...................... - -We have seen that sparsity could be used to mitigate the curse of -dimensionality, *i.e* an insufficient amount of observations compared to the -number of features. Another approach is to merge together similar -features: **feature agglomeration**. This approach can be implemented by -clustering in the feature direction, in other words clustering the -transposed data. - -.. image:: /auto_examples/cluster/images/sphx_glr_plot_digits_agglomeration_001.png - :target: ../../auto_examples/cluster/plot_digits_agglomeration.html - :align: center - :scale: 57 - -:: - - >>> digits = datasets.load_digits() - >>> images = digits.images - >>> X = np.reshape(images, (len(images), -1)) - >>> connectivity = grid_to_graph(*images[0].shape) - - >>> agglo = cluster.FeatureAgglomeration(connectivity=connectivity, - ... n_clusters=32) - >>> agglo.fit(X) - FeatureAgglomeration(connectivity=..., n_clusters=32) - >>> X_reduced = agglo.transform(X) - - >>> X_approx = agglo.inverse_transform(X_reduced) - >>> images_approx = np.reshape(X_approx, images.shape) - -.. topic:: ``transform`` and ``inverse_transform`` methods - - Some estimators expose a ``transform`` method, for instance to reduce - the dimensionality of the dataset. - -Decompositions: from a signal to components and loadings -=========================================================== - -.. topic:: **Components and loadings** - - If X is our multivariate data, then the problem that we are trying to solve - is to rewrite it on a different observational basis: we want to learn - loadings L and a set of components C such that *X = L C*. - Different criteria exist to choose the components - -Principal component analysis: PCA ------------------------------------ - -:ref:`PCA` selects the successive components that -explain the maximum variance in the signal. - -.. |pca_3d_axis| image:: /auto_examples/decomposition/images/sphx_glr_plot_pca_3d_001.png - :target: ../../auto_examples/decomposition/plot_pca_3d.html - :scale: 70 - -.. |pca_3d_aligned| image:: /auto_examples/decomposition/images/sphx_glr_plot_pca_3d_002.png - :target: ../../auto_examples/decomposition/plot_pca_3d.html - :scale: 70 - -.. rst-class:: centered - - |pca_3d_axis| |pca_3d_aligned| - -The point cloud spanned by the observations above is very flat in one -direction: one of the three univariate features can almost be exactly -computed using the other two. PCA finds the directions in which the data is -not *flat* - -When used to *transform* data, PCA can reduce the dimensionality of the -data by projecting on a principal subspace. - -.. np.random.seed(0) - -:: - - >>> # Create a signal with only 2 useful dimensions - >>> x1 = np.random.normal(size=100) - >>> x2 = np.random.normal(size=100) - >>> x3 = x1 + x2 - >>> X = np.c_[x1, x2, x3] - - >>> from sklearn import decomposition - >>> pca = decomposition.PCA() - >>> pca.fit(X) - PCA() - >>> print(pca.explained_variance_) # doctest: +SKIP - [ 2.18565811e+00 1.19346747e+00 8.43026679e-32] - - >>> # As we can see, only the 2 first components are useful - >>> pca.n_components = 2 - >>> X_reduced = pca.fit_transform(X) - >>> X_reduced.shape - (100, 2) - -.. Eigenfaces here? - -Independent Component Analysis: ICA -------------------------------------- - -:ref:`ICA` selects components so that the distribution of their loadings carries -a maximum amount of independent information. It is able to recover -**non-Gaussian** independent signals: - -.. image:: /auto_examples/decomposition/images/sphx_glr_plot_ica_blind_source_separation_001.png - :target: ../../auto_examples/decomposition/plot_ica_blind_source_separation.html - :scale: 70 - :align: center - -.. np.random.seed(0) - -:: - - >>> # Generate sample data - >>> import numpy as np - >>> from scipy import signal - >>> time = np.linspace(0, 10, 2000) - >>> s1 = np.sin(2 * time) # Signal 1 : sinusoidal signal - >>> s2 = np.sign(np.sin(3 * time)) # Signal 2 : square signal - >>> s3 = signal.sawtooth(2 * np.pi * time) # Signal 3: saw tooth signal - >>> S = np.c_[s1, s2, s3] - >>> S += 0.2 * np.random.normal(size=S.shape) # Add noise - >>> S /= S.std(axis=0) # Standardize data - >>> # Mix data - >>> A = np.array([[1, 1, 1], [0.5, 2, 1], [1.5, 1, 2]]) # Mixing matrix - >>> X = np.dot(S, A.T) # Generate observations - - >>> # Compute ICA - >>> ica = decomposition.FastICA() - >>> S_ = ica.fit_transform(X) # Get the estimated sources - >>> A_ = ica.mixing_.T - >>> np.allclose(X, np.dot(S_, A_) + ica.mean_) - True diff --git a/doc/tutorial/text_analytics/.gitignore b/doc/tutorial/text_analytics/.gitignore deleted file mode 100644 index 54c78634d9dd1..0000000000000 --- a/doc/tutorial/text_analytics/.gitignore +++ /dev/null @@ -1,25 +0,0 @@ -# cruft -.*.swp -*.pyc -.DS_Store -*.pdf - -# folder to be used for working on the exercises -workspace - -# output of the sphinx build of the documentation -tutorial/_build - -# datasets to be fetched from the web and cached locally -data/twenty_newsgroups/20news-bydate.tar.gz -data/twenty_newsgroups/20news-bydate-train -data/twenty_newsgroups/20news-bydate-test - -data/movie_reviews/txt_sentoken -data/movie_reviews/poldata.README.2.0 - -data/languages/paragraphs -data/languages/short_paragraphs -data/languages/html - -data/labeled_faces_wild/lfw_preprocessed/ diff --git a/doc/tutorial/text_analytics/data/languages/fetch_data.py b/doc/tutorial/text_analytics/data/languages/fetch_data.py deleted file mode 100644 index a4f577bd7a76e..0000000000000 --- a/doc/tutorial/text_analytics/data/languages/fetch_data.py +++ /dev/null @@ -1,101 +0,0 @@ - -# simple python script to collect text paragraphs from various languages on the -# same topic namely the Wikipedia encyclopedia itself - -import os -from urllib.request import Request, build_opener - -import lxml.html -from lxml.etree import ElementTree -import numpy as np - -import codecs - -pages = { - 'ar': 'http://ar.wikipedia.org/wiki/%D9%88%D9%8A%D9%83%D9%8A%D8%A8%D9%8A%D8%AF%D9%8A%D8%A7', # noqa: E501 - 'de': 'http://de.wikipedia.org/wiki/Wikipedia', - 'en': 'https://en.wikipedia.org/wiki/Wikipedia', - 'es': 'http://es.wikipedia.org/wiki/Wikipedia', - 'fr': 'http://fr.wikipedia.org/wiki/Wikip%C3%A9dia', - 'it': 'http://it.wikipedia.org/wiki/Wikipedia', - 'ja': 'http://ja.wikipedia.org/wiki/Wikipedia', - 'nl': 'http://nl.wikipedia.org/wiki/Wikipedia', - 'pl': 'http://pl.wikipedia.org/wiki/Wikipedia', - 'pt': 'http://pt.wikipedia.org/wiki/Wikip%C3%A9dia', - 'ru': 'http://ru.wikipedia.org/wiki/%D0%92%D0%B8%D0%BA%D0%B8%D0%BF%D0%B5%D0%B4%D0%B8%D1%8F', # noqa: E501 -# u'zh': u'http://zh.wikipedia.org/wiki/Wikipedia', -} - -html_folder = 'html' -text_folder = 'paragraphs' -short_text_folder = 'short_paragraphs' -n_words_per_short_text = 5 - - -if not os.path.exists(html_folder): - os.makedirs(html_folder) - -for lang, page in pages.items(): - - text_lang_folder = os.path.join(text_folder, lang) - if not os.path.exists(text_lang_folder): - os.makedirs(text_lang_folder) - - short_text_lang_folder = os.path.join(short_text_folder, lang) - if not os.path.exists(short_text_lang_folder): - os.makedirs(short_text_lang_folder) - - opener = build_opener() - html_filename = os.path.join(html_folder, lang + '.html') - if not os.path.exists(html_filename): - print("Downloading %s" % page) - request = Request(page) - # change the User Agent to avoid being blocked by Wikipedia - # downloading a couple of articles should not be considered abusive - request.add_header('User-Agent', 'OpenAnything/1.0') - html_content = opener.open(request).read() - open(html_filename, 'wb').write(html_content) - - # decode the payload explicitly as UTF-8 since lxml is confused for some - # reason - with codecs.open(html_filename,'r','utf-8') as html_file: - html_content = html_file.read() - tree = ElementTree(lxml.html.document_fromstring(html_content)) - i = 0 - j = 0 - for p in tree.findall('//p'): - content = p.text_content() - if len(content) < 100: - # skip paragraphs that are too short - probably too noisy and not - # representative of the actual language - continue - - text_filename = os.path.join(text_lang_folder, - '%s_%04d.txt' % (lang, i)) - print("Writing %s" % text_filename) - open(text_filename, 'wb').write(content.encode('utf-8', 'ignore')) - i += 1 - - # split the paragraph into fake smaller paragraphs to make the - # problem harder e.g. more similar to tweets - if lang in ('zh', 'ja'): - # FIXME: whitespace tokenizing does not work on chinese and japanese - continue - words = content.split() - n_groups = len(words) / n_words_per_short_text - if n_groups < 1: - continue - groups = np.array_split(words, n_groups) - - for group in groups: - small_content = " ".join(group) - - short_text_filename = os.path.join(short_text_lang_folder, - '%s_%04d.txt' % (lang, j)) - print("Writing %s" % short_text_filename) - open(short_text_filename, 'wb').write( - small_content.encode('utf-8', 'ignore')) - j += 1 - if j >= 1000: - break - diff --git a/doc/tutorial/text_analytics/data/movie_reviews/fetch_data.py b/doc/tutorial/text_analytics/data/movie_reviews/fetch_data.py deleted file mode 100644 index e591aca0f241b..0000000000000 --- a/doc/tutorial/text_analytics/data/movie_reviews/fetch_data.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Script to download the movie review dataset""" - -import os -import tarfile -from contextlib import closing -from urllib.request import urlopen - - -URL = ("http://www.cs.cornell.edu/people/pabo/" - "movie-review-data/review_polarity.tar.gz") - -ARCHIVE_NAME = URL.rsplit('/', 1)[1] -DATA_FOLDER = "txt_sentoken" - - -if not os.path.exists(DATA_FOLDER): - - if not os.path.exists(ARCHIVE_NAME): - print("Downloading dataset from %s (3 MB)" % URL) - opener = urlopen(URL) - with open(ARCHIVE_NAME, 'wb') as archive: - archive.write(opener.read()) - - print("Decompressing %s" % ARCHIVE_NAME) - with closing(tarfile.open(ARCHIVE_NAME, "r:gz")) as archive: - archive.extractall(path='.') - os.remove(ARCHIVE_NAME) diff --git a/doc/tutorial/text_analytics/data/twenty_newsgroups/fetch_data.py b/doc/tutorial/text_analytics/data/twenty_newsgroups/fetch_data.py deleted file mode 100644 index 983989d8c0e25..0000000000000 --- a/doc/tutorial/text_analytics/data/twenty_newsgroups/fetch_data.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Script to download the 20 newsgroups text classification set""" - -import os -import tarfile -from contextlib import closing -from urllib.request import urlopen - -URL = ("http://people.csail.mit.edu/jrennie/" - "20Newsgroups/20news-bydate.tar.gz") - -ARCHIVE_NAME = URL.rsplit('/', 1)[1] -TRAIN_FOLDER = "20news-bydate-train" -TEST_FOLDER = "20news-bydate-test" - - -if not os.path.exists(TRAIN_FOLDER) or not os.path.exists(TEST_FOLDER): - - if not os.path.exists(ARCHIVE_NAME): - print("Downloading dataset from %s (14 MB)" % URL) - opener = urlopen(URL) - with open(ARCHIVE_NAME, 'wb') as archive: - archive.write(opener.read()) - - print("Decompressing %s" % ARCHIVE_NAME) - with closing(tarfile.open(ARCHIVE_NAME, "r:gz")) as archive: - archive.extractall(path='.') - os.remove(ARCHIVE_NAME) diff --git a/doc/tutorial/text_analytics/skeletons/exercise_01_language_train_model.py b/doc/tutorial/text_analytics/skeletons/exercise_01_language_train_model.py deleted file mode 100644 index 438481120d126..0000000000000 --- a/doc/tutorial/text_analytics/skeletons/exercise_01_language_train_model.py +++ /dev/null @@ -1,62 +0,0 @@ -"""Build a language detector model - -The goal of this exercise is to train a linear classifier on text features -that represent sequences of up to 3 consecutive characters so as to be -recognize natural languages by using the frequencies of short character -sequences as 'fingerprints'. - -""" -# Author: Olivier Grisel -# License: Simplified BSD - -import sys - -from sklearn.feature_extraction.text import TfidfVectorizer -from sklearn.linear_model import Perceptron -from sklearn.pipeline import Pipeline -from sklearn.datasets import load_files -from sklearn.model_selection import train_test_split -from sklearn import metrics - - -# The training data folder must be passed as first argument -languages_data_folder = sys.argv[1] -dataset = load_files(languages_data_folder) - -# Split the dataset in training and test set: -docs_train, docs_test, y_train, y_test = train_test_split( - dataset.data, dataset.target, test_size=0.5) - - -# TASK: Build a vectorizer that splits strings into sequence of 1 to 3 -# characters instead of word tokens - -# TASK: Build a vectorizer / classifier pipeline using the previous analyzer -# the pipeline instance should stored in a variable named clf - -# TASK: Fit the pipeline on the training set - -# TASK: Predict the outcome on the testing set in a variable named y_predicted - -# Print the classification report -print(metrics.classification_report(y_test, y_predicted, - target_names=dataset.target_names)) - -# Plot the confusion matrix -cm = metrics.confusion_matrix(y_test, y_predicted) -print(cm) - -#import matplotlib.pyplot as plt -#plt.matshow(cm, cmap=plt.cm.jet) -#plt.show() - -# Predict the result on some short new sentences: -sentences = [ - 'This is a language detection test.', - 'Ceci est un test de d\xe9tection de la langue.', - 'Dies ist ein Test, um die Sprache zu erkennen.', -] -predicted = clf.predict(sentences) - -for s, p in zip(sentences, predicted): - print('The language of "%s" is "%s"' % (s, dataset.target_names[p])) diff --git a/doc/tutorial/text_analytics/skeletons/exercise_02_sentiment.py b/doc/tutorial/text_analytics/skeletons/exercise_02_sentiment.py deleted file mode 100644 index 23299f5f01b3d..0000000000000 --- a/doc/tutorial/text_analytics/skeletons/exercise_02_sentiment.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Build a sentiment analysis / polarity model - -Sentiment analysis can be casted as a binary text classification problem, -that is fitting a linear classifier on features extracted from the text -of the user messages so as to guess whether the opinion of the author is -positive or negative. - -In this examples we will use a movie review dataset. - -""" -# Author: Olivier Grisel -# License: Simplified BSD - -import sys -from sklearn.feature_extraction.text import TfidfVectorizer -from sklearn.svm import LinearSVC -from sklearn.pipeline import Pipeline -from sklearn.model_selection import GridSearchCV -from sklearn.datasets import load_files -from sklearn.model_selection import train_test_split -from sklearn import metrics - - -if __name__ == "__main__": - # NOTE: we put the following in a 'if __name__ == "__main__"' protected - # block to be able to use a multi-core grid search that also works under - # Windows, see: http://docs.python.org/library/multiprocessing.html#windows - # The multiprocessing module is used as the backend of joblib.Parallel - # that is used when n_jobs != 1 in GridSearchCV - - # the training data folder must be passed as first argument - movie_reviews_data_folder = sys.argv[1] - dataset = load_files(movie_reviews_data_folder, shuffle=False) - print("n_samples: %d" % len(dataset.data)) - - # split the dataset in training and test set: - docs_train, docs_test, y_train, y_test = train_test_split( - dataset.data, dataset.target, test_size=0.25, random_state=None) - - # TASK: Build a vectorizer / classifier pipeline that filters out tokens - # that are too rare or too frequent - - # TASK: Build a grid search to find out whether unigrams or bigrams are - # more useful. - # Fit the pipeline on the training set using grid search for the parameters - - # TASK: print the cross-validated scores for the each parameters set - # explored by the grid search - - # TASK: Predict the outcome on the testing set and store it in a variable - # named y_predicted - - # Print the classification report - print(metrics.classification_report(y_test, y_predicted, - target_names=dataset.target_names)) - - # Print and plot the confusion matrix - cm = metrics.confusion_matrix(y_test, y_predicted) - print(cm) - - # import matplotlib.pyplot as plt - # plt.matshow(cm) - # plt.show() diff --git a/doc/tutorial/text_analytics/solutions/exercise_01_language_train_model.py b/doc/tutorial/text_analytics/solutions/exercise_01_language_train_model.py deleted file mode 100644 index 21cee0c80e00e..0000000000000 --- a/doc/tutorial/text_analytics/solutions/exercise_01_language_train_model.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Build a language detector model - -The goal of this exercise is to train a linear classifier on text features -that represent sequences of up to 3 consecutive characters so as to be -recognize natural languages by using the frequencies of short character -sequences as 'fingerprints'. - -""" -# Author: Olivier Grisel -# License: Simplified BSD - -import sys - -from sklearn.feature_extraction.text import TfidfVectorizer -from sklearn.linear_model import Perceptron -from sklearn.pipeline import Pipeline -from sklearn.datasets import load_files -from sklearn.model_selection import train_test_split -from sklearn import metrics - - -# The training data folder must be passed as first argument -languages_data_folder = sys.argv[1] -dataset = load_files(languages_data_folder) - -# Split the dataset in training and test set: -docs_train, docs_test, y_train, y_test = train_test_split( - dataset.data, dataset.target, test_size=0.5) - - -# TASK: Build a vectorizer that splits strings into sequence of 1 to 3 -# characters instead of word tokens -vectorizer = TfidfVectorizer(ngram_range=(1, 3), analyzer='char', - use_idf=False) - -# TASK: Build a vectorizer / classifier pipeline using the previous analyzer -# the pipeline instance should stored in a variable named clf -clf = Pipeline([ - ('vec', vectorizer), - ('clf', Perceptron()), -]) - -# TASK: Fit the pipeline on the training set -clf.fit(docs_train, y_train) - -# TASK: Predict the outcome on the testing set in a variable named y_predicted -y_predicted = clf.predict(docs_test) - -# Print the classification report -print(metrics.classification_report(y_test, y_predicted, - target_names=dataset.target_names)) - -# Plot the confusion matrix -cm = metrics.confusion_matrix(y_test, y_predicted) -print(cm) - -#import matlotlib.pyplot as plt -#plt.matshow(cm, cmap=plt.cm.jet) -#plt.show() - -# Predict the result on some short new sentences: -sentences = [ - 'This is a language detection test.', - 'Ceci est un test de d\xe9tection de la langue.', - 'Dies ist ein Test, um die Sprache zu erkennen.', -] -predicted = clf.predict(sentences) - -for s, p in zip(sentences, predicted): - print('The language of "%s" is "%s"' % (s, dataset.target_names[p])) diff --git a/doc/tutorial/text_analytics/solutions/exercise_02_sentiment.py b/doc/tutorial/text_analytics/solutions/exercise_02_sentiment.py deleted file mode 100644 index 434bece341975..0000000000000 --- a/doc/tutorial/text_analytics/solutions/exercise_02_sentiment.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Build a sentiment analysis / polarity model - -Sentiment analysis can be casted as a binary text classification problem, -that is fitting a linear classifier on features extracted from the text -of the user messages so as to guess whether the opinion of the author is -positive or negative. - -In this examples we will use a movie review dataset. - -""" -# Author: Olivier Grisel -# License: Simplified BSD - -import sys -from sklearn.feature_extraction.text import TfidfVectorizer -from sklearn.svm import LinearSVC -from sklearn.pipeline import Pipeline -from sklearn.model_selection import GridSearchCV -from sklearn.datasets import load_files -from sklearn.model_selection import train_test_split -from sklearn import metrics - - -if __name__ == "__main__": - # NOTE: we put the following in a 'if __name__ == "__main__"' protected - # block to be able to use a multi-core grid search that also works under - # Windows, see: http://docs.python.org/library/multiprocessing.html#windows - # The multiprocessing module is used as the backend of joblib.Parallel - # that is used when n_jobs != 1 in GridSearchCV - - # the training data folder must be passed as first argument - movie_reviews_data_folder = sys.argv[1] - dataset = load_files(movie_reviews_data_folder, shuffle=False) - print("n_samples: %d" % len(dataset.data)) - - # split the dataset in training and test set: - docs_train, docs_test, y_train, y_test = train_test_split( - dataset.data, dataset.target, test_size=0.25, random_state=None) - - # TASK: Build a vectorizer / classifier pipeline that filters out tokens - # that are too rare or too frequent - pipeline = Pipeline([ - ('vect', TfidfVectorizer(min_df=3, max_df=0.95)), - ('clf', LinearSVC(C=1000)), - ]) - - # TASK: Build a grid search to find out whether unigrams or bigrams are - # more useful. - # Fit the pipeline on the training set using grid search for the parameters - parameters = { - 'vect__ngram_range': [(1, 1), (1, 2)], - } - grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1) - grid_search.fit(docs_train, y_train) - - # TASK: print the mean and std for each candidate along with the parameter - # settings for all the candidates explored by grid search. - n_candidates = len(grid_search.cv_results_['params']) - for i in range(n_candidates): - print(i, 'params - %s; mean - %0.2f; std - %0.2f' - % (grid_search.cv_results_['params'][i], - grid_search.cv_results_['mean_test_score'][i], - grid_search.cv_results_['std_test_score'][i])) - - # TASK: Predict the outcome on the testing set and store it in a variable - # named y_predicted - y_predicted = grid_search.predict(docs_test) - - # Print the classification report - print(metrics.classification_report(y_test, y_predicted, - target_names=dataset.target_names)) - - # Print and plot the confusion matrix - cm = metrics.confusion_matrix(y_test, y_predicted) - print(cm) - - # import matplotlib.pyplot as plt - # plt.matshow(cm) - # plt.show() diff --git a/doc/tutorial/text_analytics/solutions/generate_skeletons.py b/doc/tutorial/text_analytics/solutions/generate_skeletons.py deleted file mode 100644 index 4729b976530c7..0000000000000 --- a/doc/tutorial/text_analytics/solutions/generate_skeletons.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Generate skeletons from the example code""" -import os - -exercise_dir = os.path.dirname(__file__) -if exercise_dir == '': - exercise_dir = '.' - -skeleton_dir = os.path.abspath(os.path.join(exercise_dir, '..', 'skeletons')) -if not os.path.exists(skeleton_dir): - os.makedirs(skeleton_dir) - -solutions = os.listdir(exercise_dir) - -for f in solutions: - if not f.endswith('.py'): - continue - - if f == os.path.basename(__file__): - continue - - print("Generating skeleton for %s" % f) - - input_file = open(os.path.join(exercise_dir, f)) - output_file = open(os.path.join(skeleton_dir, f), 'w') - - in_exercise_region = False - - for line in input_file: - linestrip = line.strip() - if len(linestrip) == 0: - in_exercise_region = False - elif linestrip.startswith('# TASK:'): - in_exercise_region = True - - if not in_exercise_region or linestrip.startswith('#'): - output_file.write(line) - - output_file.close() diff --git a/doc/tutorial/text_analytics/working_with_text_data.rst b/doc/tutorial/text_analytics/working_with_text_data.rst deleted file mode 100644 index 26ff51b0cf3df..0000000000000 --- a/doc/tutorial/text_analytics/working_with_text_data.rst +++ /dev/null @@ -1,586 +0,0 @@ -.. _text_data_tutorial: - -====================== -Working With Text Data -====================== - -The goal of this guide is to explore some of the main ``scikit-learn`` -tools on a single practical task: analyzing a collection of text -documents (newsgroups posts) on twenty different topics. - -In this section we will see how to: - - - load the file contents and the categories - - - extract feature vectors suitable for machine learning - - - train a linear model to perform categorization - - - use a grid search strategy to find a good configuration of both - the feature extraction components and the classifier - - -Tutorial setup --------------- - -To get started with this tutorial, you must first install -*scikit-learn* and all of its required dependencies. - -Please refer to the :ref:`installation instructions ` -page for more information and for system-specific instructions. - -The source of this tutorial can be found within your scikit-learn folder:: - - scikit-learn/doc/tutorial/text_analytics/ - -The source can also be found `on Github -`_. - -The tutorial folder should contain the following sub-folders: - - * ``*.rst files`` - the source of the tutorial document written with sphinx - - * ``data`` - folder to put the datasets used during the tutorial - - * ``skeletons`` - sample incomplete scripts for the exercises - - * ``solutions`` - solutions of the exercises - - -You can already copy the skeletons into a new folder somewhere -on your hard-drive named ``sklearn_tut_workspace`` where you -will edit your own files for the exercises while keeping -the original skeletons intact: - -.. prompt:: bash $ - - cp -r skeletons work_directory/sklearn_tut_workspace - - -Machine learning algorithms need data. Go to each ``$TUTORIAL_HOME/data`` -sub-folder and run the ``fetch_data.py`` script from there (after -having read them first). - -For instance: - -.. prompt:: bash $ - - cd $TUTORIAL_HOME/data/languages - less fetch_data.py - python fetch_data.py - - -Loading the 20 newsgroups dataset ---------------------------------- - -The dataset is called "Twenty Newsgroups". Here is the official -description, quoted from the `website -`_: - - The 20 Newsgroups data set is a collection of approximately 20,000 - newsgroup documents, partitioned (nearly) evenly across 20 different - newsgroups. To the best of our knowledge, it was originally collected - by Ken Lang, probably for his paper "Newsweeder: Learning to filter - netnews," though he does not explicitly mention this collection. - The 20 newsgroups collection has become a popular data set for - experiments in text applications of machine learning techniques, - such as text classification and text clustering. - -In the following we will use the built-in dataset loader for 20 newsgroups -from scikit-learn. Alternatively, it is possible to download the dataset -manually from the website and use the :func:`sklearn.datasets.load_files` -function by pointing it to the ``20news-bydate-train`` sub-folder of the -uncompressed archive folder. - -In order to get faster execution times for this first example we will -work on a partial dataset with only 4 categories out of the 20 available -in the dataset:: - - >>> categories = ['alt.atheism', 'soc.religion.christian', - ... 'comp.graphics', 'sci.med'] - -We can now load the list of files matching those categories as follows:: - - >>> from sklearn.datasets import fetch_20newsgroups - >>> twenty_train = fetch_20newsgroups(subset='train', - ... categories=categories, shuffle=True, random_state=42) - -The returned dataset is a ``scikit-learn`` "bunch": a simple holder -object with fields that can be both accessed as python ``dict`` -keys or ``object`` attributes for convenience, for instance the -``target_names`` holds the list of the requested category names:: - - >>> twenty_train.target_names - ['alt.atheism', 'comp.graphics', 'sci.med', 'soc.religion.christian'] - -The files themselves are loaded in memory in the ``data`` attribute. For -reference the filenames are also available:: - - >>> len(twenty_train.data) - 2257 - >>> len(twenty_train.filenames) - 2257 - -Let's print the first lines of the first loaded file:: - - >>> print("\n".join(twenty_train.data[0].split("\n")[:3])) - From: sd345@city.ac.uk (Michael Collier) - Subject: Converting images to HP LaserJet III? - Nntp-Posting-Host: hampton - - >>> print(twenty_train.target_names[twenty_train.target[0]]) - comp.graphics - -Supervised learning algorithms will require a category label for each -document in the training set. In this case the category is the name of the -newsgroup which also happens to be the name of the folder holding the -individual documents. - -For speed and space efficiency reasons ``scikit-learn`` loads the -target attribute as an array of integers that corresponds to the -index of the category name in the ``target_names`` list. The category -integer id of each sample is stored in the ``target`` attribute:: - - >>> twenty_train.target[:10] - array([1, 1, 3, 3, 3, 3, 3, 2, 2, 2]) - -It is possible to get back the category names as follows:: - - >>> for t in twenty_train.target[:10]: - ... print(twenty_train.target_names[t]) - ... - comp.graphics - comp.graphics - soc.religion.christian - soc.religion.christian - soc.religion.christian - soc.religion.christian - soc.religion.christian - sci.med - sci.med - sci.med - -You might have noticed that the samples were shuffled randomly when we called -``fetch_20newsgroups(..., shuffle=True, random_state=42)``: this is useful if -you wish to select only a subset of samples to quickly train a model and get a -first idea of the results before re-training on the complete dataset later. - - -Extracting features from text files ------------------------------------ - -In order to perform machine learning on text documents, we first need to -turn the text content into numerical feature vectors. - -.. currentmodule:: sklearn.feature_extraction.text - - -Bags of words -~~~~~~~~~~~~~ - -The most intuitive way to do so is to use a bags of words representation: - - 1. Assign a fixed integer id to each word occurring in any document - of the training set (for instance by building a dictionary - from words to integer indices). - - 2. For each document ``#i``, count the number of occurrences of each - word ``w`` and store it in ``X[i, j]`` as the value of feature - ``#j`` where ``j`` is the index of word ``w`` in the dictionary. - -The bags of words representation implies that ``n_features`` is -the number of distinct words in the corpus: this number is typically -larger than 100,000. - -If ``n_samples == 10000``, storing ``X`` as a NumPy array of type -float32 would require 10000 x 100000 x 4 bytes = **4GB in RAM** which -is barely manageable on today's computers. - -Fortunately, **most values in X will be zeros** since for a given -document less than a few thousand distinct words will be -used. For this reason we say that bags of words are typically -**high-dimensional sparse datasets**. We can save a lot of memory by -only storing the non-zero parts of the feature vectors in memory. - -``scipy.sparse`` matrices are data structures that do exactly this, -and ``scikit-learn`` has built-in support for these structures. - - -Tokenizing text with ``scikit-learn`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Text preprocessing, tokenizing and filtering of stopwords are all included -in :class:`CountVectorizer`, which builds a dictionary of features and -transforms documents to feature vectors:: - - >>> from sklearn.feature_extraction.text import CountVectorizer - >>> count_vect = CountVectorizer() - >>> X_train_counts = count_vect.fit_transform(twenty_train.data) - >>> X_train_counts.shape - (2257, 35788) - -:class:`CountVectorizer` supports counts of N-grams of words or consecutive -characters. Once fitted, the vectorizer has built a dictionary of feature -indices:: - - >>> count_vect.vocabulary_.get(u'algorithm') - 4690 - -The index value of a word in the vocabulary is linked to its frequency -in the whole training corpus. - -.. note: - - The method ``count_vect.fit_transform`` performs two actions: - it learns the vocabulary and transforms the documents into count vectors. - It's possible to separate these steps by calling - ``count_vect.fit(twenty_train.data)`` followed by - ``X_train_counts = count_vect.transform(twenty_train.data)``, - but doing so would tokenize and vectorize each text file twice. - - -From occurrences to frequencies -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Occurrence count is a good start but there is an issue: longer -documents will have higher average count values than shorter documents, -even though they might talk about the same topics. - -To avoid these potential discrepancies it suffices to divide the -number of occurrences of each word in a document by the total number -of words in the document: these new features are called ``tf`` for Term -Frequencies. - -Another refinement on top of tf is to downscale weights for words -that occur in many documents in the corpus and are therefore less -informative than those that occur only in a smaller portion of the -corpus. - -This downscaling is called `tf–idf`_ for "Term Frequency times -Inverse Document Frequency". - -.. _`tf–idf`: https://en.wikipedia.org/wiki/Tf-idf - - -Both **tf** and **tf–idf** can be computed as follows using -:class:`TfidfTransformer`:: - - >>> from sklearn.feature_extraction.text import TfidfTransformer - >>> tf_transformer = TfidfTransformer(use_idf=False).fit(X_train_counts) - >>> X_train_tf = tf_transformer.transform(X_train_counts) - >>> X_train_tf.shape - (2257, 35788) - -In the above example-code, we firstly use the ``fit(..)`` method to fit our -estimator to the data and secondly the ``transform(..)`` method to transform -our count-matrix to a tf-idf representation. -These two steps can be combined to achieve the same end result faster -by skipping redundant processing. This is done through using the -``fit_transform(..)`` method as shown below, and as mentioned in the note -in the previous section:: - - >>> tfidf_transformer = TfidfTransformer() - >>> X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts) - >>> X_train_tfidf.shape - (2257, 35788) - - -Training a classifier ---------------------- - -Now that we have our features, we can train a classifier to try to predict -the category of a post. Let's start with a :ref:`naïve Bayes ` -classifier, which -provides a nice baseline for this task. ``scikit-learn`` includes several -variants of this classifier; the one most suitable for word counts is the -multinomial variant:: - - >>> from sklearn.naive_bayes import MultinomialNB - >>> clf = MultinomialNB().fit(X_train_tfidf, twenty_train.target) - -To try to predict the outcome on a new document we need to extract -the features using almost the same feature extracting chain as before. -The difference is that we call ``transform`` instead of ``fit_transform`` -on the transformers, since they have already been fit to the training set:: - - >>> docs_new = ['God is love', 'OpenGL on the GPU is fast'] - >>> X_new_counts = count_vect.transform(docs_new) - >>> X_new_tfidf = tfidf_transformer.transform(X_new_counts) - - >>> predicted = clf.predict(X_new_tfidf) - - >>> for doc, category in zip(docs_new, predicted): - ... print('%r => %s' % (doc, twenty_train.target_names[category])) - ... - 'God is love' => soc.religion.christian - 'OpenGL on the GPU is fast' => comp.graphics - - -Building a pipeline -------------------- - -In order to make the vectorizer => transformer => classifier easier -to work with, ``scikit-learn`` provides a :class:`~sklearn.pipeline.Pipeline` class that behaves -like a compound classifier:: - - >>> from sklearn.pipeline import Pipeline - >>> text_clf = Pipeline([ - ... ('vect', CountVectorizer()), - ... ('tfidf', TfidfTransformer()), - ... ('clf', MultinomialNB()), - ... ]) - - -The names ``vect``, ``tfidf`` and ``clf`` (classifier) are arbitrary. -We will use them to perform grid search for suitable hyperparameters below. -We can now train the model with a single command:: - - >>> text_clf.fit(twenty_train.data, twenty_train.target) - Pipeline(...) - - -Evaluation of the performance on the test set ---------------------------------------------- - -Evaluating the predictive accuracy of the model is equally easy:: - - >>> import numpy as np - >>> twenty_test = fetch_20newsgroups(subset='test', - ... categories=categories, shuffle=True, random_state=42) - >>> docs_test = twenty_test.data - >>> predicted = text_clf.predict(docs_test) - >>> np.mean(predicted == twenty_test.target) - 0.8348... - -We achieved 83.5% accuracy. Let's see if we can do better with a -linear :ref:`support vector machine (SVM) `, -which is widely regarded as one of -the best text classification algorithms (although it's also a bit slower -than naïve Bayes). We can change the learner by simply plugging a different -classifier object into our pipeline:: - - >>> from sklearn.linear_model import SGDClassifier - >>> text_clf = Pipeline([ - ... ('vect', CountVectorizer()), - ... ('tfidf', TfidfTransformer()), - ... ('clf', SGDClassifier(loss='hinge', penalty='l2', - ... alpha=1e-3, random_state=42, - ... max_iter=5, tol=None)), - ... ]) - - >>> text_clf.fit(twenty_train.data, twenty_train.target) - Pipeline(...) - >>> predicted = text_clf.predict(docs_test) - >>> np.mean(predicted == twenty_test.target) - 0.9101... - -We achieved 91.3% accuracy using the SVM. ``scikit-learn`` provides further -utilities for more detailed performance analysis of the results:: - - >>> from sklearn import metrics - >>> print(metrics.classification_report(twenty_test.target, predicted, - ... target_names=twenty_test.target_names)) - precision recall f1-score support - - alt.atheism 0.95 0.80 0.87 319 - comp.graphics 0.87 0.98 0.92 389 - sci.med 0.94 0.89 0.91 396 - soc.religion.christian 0.90 0.95 0.93 398 - - accuracy 0.91 1502 - macro avg 0.91 0.91 0.91 1502 - weighted avg 0.91 0.91 0.91 1502 - - - >>> metrics.confusion_matrix(twenty_test.target, predicted) - array([[256, 11, 16, 36], - [ 4, 380, 3, 2], - [ 5, 35, 353, 3], - [ 5, 11, 4, 378]]) - -As expected the confusion matrix shows that posts from the newsgroups -on atheism and Christianity are more often confused for one another than -with computer graphics. - -.. note: - - SGD stands for Stochastic Gradient Descent. This is a simple - optimization algorithms that is known to be scalable when the dataset - has many samples. - - By setting ``loss="hinge"`` and ``penalty="l2"`` we are configuring - the classifier model to tune its parameters for the linear Support - Vector Machine cost function. - - Alternatively we could have used ``sklearn.svm.LinearSVC`` (Linear - Support Vector Machine Classifier) that provides an alternative - optimizer for the same cost function based on the liblinear_ C++ - library. - -.. _liblinear: https://www.csie.ntu.edu.tw/~cjlin/liblinear/ - - -Parameter tuning using grid search ----------------------------------- - -We've already encountered some parameters such as ``use_idf`` in the -``TfidfTransformer``. Classifiers tend to have many parameters as well; -e.g., ``MultinomialNB`` includes a smoothing parameter ``alpha`` and -``SGDClassifier`` has a penalty parameter ``alpha`` and configurable loss -and penalty terms in the objective function (see the module documentation, -or use the Python ``help`` function to get a description of these). - -Instead of tweaking the parameters of the various components of the -chain, it is possible to run an exhaustive search of the best -parameters on a grid of possible values. We try out all classifiers -on either words or bigrams, with or without idf, and with a penalty -parameter of either 0.01 or 0.001 for the linear SVM:: - - >>> from sklearn.model_selection import GridSearchCV - >>> parameters = { - ... 'vect__ngram_range': [(1, 1), (1, 2)], - ... 'tfidf__use_idf': (True, False), - ... 'clf__alpha': (1e-2, 1e-3), - ... } - - -Obviously, such an exhaustive search can be expensive. If we have multiple -CPU cores at our disposal, we can tell the grid searcher to try these eight -parameter combinations in parallel with the ``n_jobs`` parameter. If we give -this parameter a value of ``-1``, grid search will detect how many cores -are installed and use them all:: - - >>> gs_clf = GridSearchCV(text_clf, parameters, cv=5, n_jobs=-1) - -The grid search instance behaves like a normal ``scikit-learn`` -model. Let's perform the search on a smaller subset of the training data -to speed up the computation:: - - >>> gs_clf = gs_clf.fit(twenty_train.data[:400], twenty_train.target[:400]) - -The result of calling ``fit`` on a ``GridSearchCV`` object is a classifier -that we can use to ``predict``:: - - >>> twenty_train.target_names[gs_clf.predict(['God is love'])[0]] - 'soc.religion.christian' - -The object's ``best_score_`` and ``best_params_`` attributes store the best -mean score and the parameters setting corresponding to that score:: - - >>> gs_clf.best_score_ - 0.9... - >>> for param_name in sorted(parameters.keys()): - ... print("%s: %r" % (param_name, gs_clf.best_params_[param_name])) - ... - clf__alpha: 0.001 - tfidf__use_idf: True - vect__ngram_range: (1, 1) - -A more detailed summary of the search is available at ``gs_clf.cv_results_``. - -The ``cv_results_`` parameter can be easily imported into pandas as a -``DataFrame`` for further inspection. - -.. note: - - A ``GridSearchCV`` object also stores the best classifier that it trained - as its ``best_estimator_`` attribute. In this case, that isn't much use as - we trained on a small, 400-document subset of our full training set. - - -Exercises -~~~~~~~~~ - -To do the exercises, copy the content of the 'skeletons' folder as -a new folder named 'workspace': - -.. prompt:: bash $ - - cp -r skeletons workspace - - -You can then edit the content of the workspace without fear of losing -the original exercise instructions. - -Then fire an ipython shell and run the work-in-progress script with:: - - [1] %run workspace/exercise_XX_script.py arg1 arg2 arg3 - -If an exception is triggered, use ``%debug`` to fire-up a post -mortem ipdb session. - -Refine the implementation and iterate until the exercise is solved. - -**For each exercise, the skeleton file provides all the necessary import -statements, boilerplate code to load the data and sample code to evaluate -the predictive accuracy of the model.** - - -Exercise 1: Language identification ------------------------------------ - -- Write a text classification pipeline using a custom preprocessor and - ``CharNGramAnalyzer`` using data from Wikipedia articles as training set. - -- Evaluate the performance on some held out test set. - -ipython command line:: - - %run workspace/exercise_01_language_train_model.py data/languages/paragraphs/ - - -Exercise 2: Sentiment Analysis on movie reviews ------------------------------------------------ - -- Write a text classification pipeline to classify movie reviews as either - positive or negative. - -- Find a good set of parameters using grid search. - -- Evaluate the performance on a held out test set. - -ipython command line:: - - %run workspace/exercise_02_sentiment.py data/movie_reviews/txt_sentoken/ - - -Exercise 3: CLI text classification utility -------------------------------------------- - -Using the results of the previous exercises and the ``cPickle`` -module of the standard library, write a command line utility that -detects the language of some text provided on ``stdin`` and estimate -the polarity (positive or negative) if the text is written in -English. - -Bonus point if the utility is able to give a confidence level for its -predictions. - - -Where to from here ------------------- - -Here are a few suggestions to help further your scikit-learn intuition -upon the completion of this tutorial: - - -* Try playing around with the ``analyzer`` and ``token normalisation`` under - :class:`CountVectorizer`. - -* If you don't have labels, try using - :ref:`Clustering ` - on your problem. - -* If you have multiple labels per document, e.g categories, have a look - at the :ref:`Multiclass and multilabel section `. - -* Try using :ref:`Truncated SVD ` for - `latent semantic analysis `_. - -* Have a look at using - :ref:`Out-of-core Classification - ` to - learn from data that would not fit into the computer main memory. - -* Have a look at the :ref:`Hashing Vectorizer ` - as a memory efficient alternative to :class:`CountVectorizer`. diff --git a/doc/unsupervised_learning.rst b/doc/unsupervised_learning.rst index 9c1de0c134623..57e8b429fd67e 100644 --- a/doc/unsupervised_learning.rst +++ b/doc/unsupervised_learning.rst @@ -1,9 +1,3 @@ -.. Places parent toc into the sidebar - -:parenttoc: True - -.. include:: includes/big_toc_css.rst - .. _unsupervised-learning: Unsupervised learning diff --git a/doc/user_guide.rst b/doc/user_guide.rst index 7d48934d32727..0c1a6ee66ebf9 100644 --- a/doc/user_guide.rst +++ b/doc/user_guide.rst @@ -1,21 +1,9 @@ -.. Places parent toc into the sidebar - -:parenttoc: True - -.. title:: User guide: contents - .. _user_guide: ========== User Guide ========== -.. include:: includes/big_toc_css.rst - -.. nice layout in the toc - -.. include:: tune_toc.rst - .. toctree:: :numbered: :maxdepth: 3 @@ -23,10 +11,14 @@ User Guide supervised_learning.rst unsupervised_learning.rst model_selection.rst + metadata_routing.rst inspection.rst visualizations.rst data_transforms.rst datasets.rst computing.rst - modules/model_persistence.rst + model_persistence.rst common_pitfalls.rst + dispatching.rst + machine_learning_map.rst + presentations.rst diff --git a/doc/visualizations.rst b/doc/visualizations.rst index a2d40408b403f..e9d38f25e1e0d 100644 --- a/doc/visualizations.rst +++ b/doc/visualizations.rst @@ -1,9 +1,3 @@ -.. Places parent toc into the sidebar - -:parenttoc: True - -.. include:: includes/big_toc_css.rst - .. _visualizations: ============== @@ -12,77 +6,119 @@ Visualizations Scikit-learn defines a simple API for creating visualizations for machine learning. The key feature of this API is to allow for quick plotting and -visual adjustments without recalculation. In the following example, we plot a -ROC curve for a fitted support vector machine: +visual adjustments without recalculation. We provide `Display` classes that +expose two methods for creating plots: `from_estimator` and +`from_predictions`. + +The `from_estimator` method generates a `Display` object from a fitted estimator, +input data (`X`, `y`), and a plot. +The `from_predictions` method creates a `Display` object from true and predicted +values (`y_test`, `y_pred`), and a plot. + +Using `from_predictions` avoids having to recompute predictions, +but the user needs to take care that the prediction values passed correspond +to the `pos_label`. For :term:`predict_proba`, select the column corresponding +to the `pos_label` class while for :term:`decision_function`, revert the score +(i.e. multiply by -1) if `pos_label` is not the last class in the +`classes_` attribute of your estimator. + +The `Display` object stores the computed values (e.g., metric values or +feature importance) required for plotting with Matplotlib. These values are the +results derived from the raw predictions passed to `from_predictions`, or +an estimator and `X` passed to `from_estimator`. + +Display objects have a plot method that creates a matplotlib plot once the display +object has been initialized (note that we recommend that display objects are created +via `from_estimator` or `from_predictions` instead of initialized directly). +The plot method allows adding to an existing plot by passing the existing plots +:class:`matplotlib.axes.Axes` to the `ax` parameter. + +In the following example, we plot a ROC curve for a fitted Logistic Regression +model `from_estimator`: + +.. plot:: + :context: close-figs + :align: center + + from sklearn.model_selection import train_test_split + from sklearn.linear_model import LogisticRegression + from sklearn.metrics import RocCurveDisplay + from sklearn.datasets import load_iris -.. code-block:: python + X, y = load_iris(return_X_y=True) + y = y == 2 # make binary + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=.8, random_state=42 + ) + clf = LogisticRegression(random_state=42, C=.01) + clf.fit(X_train, y_train) + + clf_disp = RocCurveDisplay.from_estimator(clf, X_test, y_test) + +If you already have the prediction values, you could instead use +`from_predictions` to do the same thing (and save on compute): + + +.. plot:: + :context: close-figs + :align: center from sklearn.model_selection import train_test_split - from sklearn.svm import SVC - from sklearn.metrics import plot_roc_curve - from sklearn.datasets import load_wine + from sklearn.linear_model import LogisticRegression + from sklearn.metrics import RocCurveDisplay + from sklearn.datasets import load_iris + + X, y = load_iris(return_X_y=True) + y = y == 2 # make binary + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=.8, random_state=42 + ) + clf = LogisticRegression(random_state=42, C=.01) + clf.fit(X_train, y_train) - X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42) - svc = SVC(random_state=42) - svc.fit(X_train, y_train) + # select the probability of the class that we considered to be the positive label + y_pred = clf.predict_proba(X_test)[:, 1] - svc_disp = plot_roc_curve(svc, X_test, y_test) + clf_disp = RocCurveDisplay.from_predictions(y_test, y_pred) -.. figure:: auto_examples/miscellaneous/images/sphx_glr_plot_roc_curve_visualization_api_001.png - :target: auto_examples/miscellaneous/plot_roc_curve_visualization_api.html - :align: center - :scale: 75% -The returned `svc_disp` object allows us to continue using the already computed -ROC curve for SVC in future plots. In this case, the `svc_disp` is a -:class:`~sklearn.metrics.RocCurveDisplay` that stores the computed values as -attributes called `roc_auc`, `fpr`, and `tpr`. Next, we train a random forest -classifier and plot the previously computed roc curve again by using the `plot` -method of the `Display` object. +The returned `clf_disp` object allows us to add another curve to the already computed +ROC curve. In this case, the `clf_disp` is a :class:`~sklearn.metrics.RocCurveDisplay` +that stores the computed values as attributes called `roc_auc`, `fpr`, and `tpr`. -.. code-block:: python +Next, we train a random forest classifier and plot the previously computed ROC curve +again by using the `plot` method of the `Display` object. + +.. plot:: + :context: close-figs + :align: center import matplotlib.pyplot as plt from sklearn.ensemble import RandomForestClassifier - rfc = RandomForestClassifier(random_state=42) + rfc = RandomForestClassifier(n_estimators=10, random_state=42) rfc.fit(X_train, y_train) ax = plt.gca() - rfc_disp = plot_roc_curve(rfc, X_test, y_test, ax=ax, alpha=0.8) - svc_disp.plot(ax=ax, alpha=0.8) - -.. figure:: auto_examples/miscellaneous/images/sphx_glr_plot_roc_curve_visualization_api_002.png - :target: auto_examples/miscellaneous/plot_roc_curve_visualization_api.html - :align: center - :scale: 75% + rfc_disp = RocCurveDisplay.from_estimator( + rfc, X_test, y_test, ax=ax, curve_kwargs={"alpha": 0.8} + ) + clf_disp.plot(ax=ax, curve_kwargs={"alpha": 0.8}) Notice that we pass `alpha=0.8` to the plot functions to adjust the alpha values of the curves. -.. topic:: Examples: - * :ref:`sphx_glr_auto_examples_miscellaneous_plot_roc_curve_visualization_api.py` - * :ref:`sphx_glr_auto_examples_miscellaneous_plot_partial_dependence_visualization_api.py` - * :ref:`sphx_glr_auto_examples_miscellaneous_plot_display_object_visualization.py` +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_miscellaneous_plot_roc_curve_visualization_api.py` +* :ref:`sphx_glr_auto_examples_miscellaneous_plot_partial_dependence_visualization_api.py` +* :ref:`sphx_glr_auto_examples_miscellaneous_plot_display_object_visualization.py` +* :ref:`sphx_glr_auto_examples_calibration_plot_compare_calibration.py` Available Plotting Utilities ============================ -Functions ---------- - -.. currentmodule:: sklearn - -.. autosummary:: - - inspection.plot_partial_dependence - metrics.plot_confusion_matrix - metrics.plot_det_curve - metrics.plot_precision_recall_curve - metrics.plot_roc_curve - - Display Objects --------------- @@ -90,8 +126,13 @@ Display Objects .. autosummary:: + calibration.CalibrationDisplay inspection.PartialDependenceDisplay + inspection.DecisionBoundaryDisplay metrics.ConfusionMatrixDisplay metrics.DetCurveDisplay metrics.PrecisionRecallDisplay + metrics.PredictionErrorDisplay metrics.RocCurveDisplay + model_selection.LearningCurveDisplay + model_selection.ValidationCurveDisplay diff --git a/doc/whats_new.rst b/doc/whats_new.rst index 2231173d62224..1e9d0316691e1 100644 --- a/doc/whats_new.rst +++ b/doc/whats_new.rst @@ -1,28 +1,39 @@ .. currentmodule:: sklearn + .. include:: whats_new/_contributors.rst Release History =============== -Release notes for all scikit-learn releases are linked in this page. +Changelogs and release notes for all scikit-learn releases are linked in this page. + +.. tip:: -**Tip:** `Subscribe to scikit-learn releases `__ -on libraries.io to be notified when new versions are released. + `Subscribe to scikit-learn releases `__ + on libraries.io to be notified when new versions are released. .. toctree:: - :maxdepth: 1 + :maxdepth: 2 - Version 1.0 - Version 0.24 - Version 0.23 - Version 0.22 - Version 0.21 - Version 0.20 - Version 0.19 - Version 0.18 - Version 0.17 - Version 0.16 - Version 0.15 - Version 0.14 - Version 0.13 - Older Versions + whats_new/v1.8.rst + whats_new/v1.7.rst + whats_new/v1.6.rst + whats_new/v1.5.rst + whats_new/v1.4.rst + whats_new/v1.3.rst + whats_new/v1.2.rst + whats_new/v1.1.rst + whats_new/v1.0.rst + whats_new/v0.24.rst + whats_new/v0.23.rst + whats_new/v0.22.rst + whats_new/v0.21.rst + whats_new/v0.20.rst + whats_new/v0.19.rst + whats_new/v0.18.rst + whats_new/v0.17.rst + whats_new/v0.16.rst + whats_new/v0.15.rst + whats_new/v0.14.rst + whats_new/v0.13.rst + whats_new/older_versions.rst diff --git a/doc/whats_new/_contributors.rst b/doc/whats_new/_contributors.rst index ca0f8ede93afa..c74a2964e57bc 100644 --- a/doc/whats_new/_contributors.rst +++ b/doc/whats_new/_contributors.rst @@ -12,15 +12,15 @@ .. role:: raw-latex(raw) :format: latex -.. |MajorFeature| replace:: :raw-html:`Major Feature` :raw-latex:`{\small\sc [Major Feature]}` -.. |Feature| replace:: :raw-html:`Feature` :raw-latex:`{\small\sc [Feature]}` -.. |Efficiency| replace:: :raw-html:`Efficiency` :raw-latex:`{\small\sc [Efficiency]}` -.. |Enhancement| replace:: :raw-html:`Enhancement` :raw-latex:`{\small\sc [Enhancement]}` -.. |Fix| replace:: :raw-html:`Fix` :raw-latex:`{\small\sc [Fix]}` -.. |API| replace:: :raw-html:`API Change` :raw-latex:`{\small\sc [API Change]}` +.. |MajorFeature| replace:: :raw-html:`Major Feature` :raw-latex:`{\small\sc [Major Feature]}` +.. |Feature| replace:: :raw-html:`Feature` :raw-latex:`{\small\sc [Feature]}` +.. |Efficiency| replace:: :raw-html:`Efficiency` :raw-latex:`{\small\sc [Efficiency]}` +.. |Enhancement| replace:: :raw-html:`Enhancement` :raw-latex:`{\small\sc [Enhancement]}` +.. |Fix| replace:: :raw-html:`Fix` :raw-latex:`{\small\sc [Fix]}` +.. |API| replace:: :raw-html:`API Change` :raw-latex:`{\small\sc [API Change]}` -.. _Olivier Grisel: https://twitter.com/ogrisel +.. _Olivier Grisel: https://bsky.app/profile/ogrisel.bsky.social .. _Gael Varoquaux: http://gael-varoquaux.info @@ -38,7 +38,7 @@ .. _Vlad Niculae: https://vene.ro/ -.. _Edouard Duchesnay: https://sites.google.com/site/duchesnay/home +.. _Edouard Duchesnay: https://duchesnay.github.io/ .. _Peter Prettenhofer: https://sites.google.com/site/peterprettenhofer/ @@ -176,4 +176,6 @@ .. _Nicolas Hug: https://github.com/NicolasHug -.. _Guillaume Lemaitre: https://github.com/glemaitre \ No newline at end of file +.. _Guillaume Lemaitre: https://github.com/glemaitre + +.. _Tim Head: https://betatim.github.io/ diff --git a/doc/whats_new/changelog_legend.inc b/doc/whats_new/changelog_legend.inc index e1b053bc6ee4c..6611571301ff1 100644 --- a/doc/whats_new/changelog_legend.inc +++ b/doc/whats_new/changelog_legend.inc @@ -1,12 +1,11 @@ -Legend for changelogs ---------------------- +.. rubric:: Legend for changelogs -- |MajorFeature|: something big that you couldn't do before. -- |Feature|: something that you couldn't do before. -- |Efficiency|: an existing feature now may not require as much computation or +- |MajorFeature| something big that you couldn't do before. +- |Feature| something that you couldn't do before. +- |Efficiency| an existing feature now may not require as much computation or memory. -- |Enhancement|: a miscellaneous minor improvement. -- |Fix|: something that previously didn't work as documentated -- or according +- |Enhancement| a miscellaneous minor improvement. +- |Fix| something that previously didn't work as documented -- or according to reasonable expectations -- should now work. -- |API|: you will need to change your code to have the same effect in the +- |API| you will need to change your code to have the same effect in the future; or a feature will be removed in the future. diff --git a/doc/whats_new/older_versions.rst b/doc/whats_new/older_versions.rst index 63b5c6e9ea4cb..4f808850e750a 100644 --- a/doc/whats_new/older_versions.rst +++ b/doc/whats_new/older_versions.rst @@ -2,6 +2,10 @@ .. currentmodule:: sklearn +============== +Older Versions +============== + .. _changes_0_12.1: Version 0.12.1 @@ -40,14 +44,14 @@ Changelog People ------ - * 14 `Peter Prettenhofer`_ - * 12 `Gael Varoquaux`_ - * 10 `Andreas MÃŧller`_ - * 5 `Lars Buitinck`_ - * 3 :user:`Virgile Fritsch ` - * 1 `Alexandre Gramfort`_ - * 1 `Gilles Louppe`_ - * 1 `Mathieu Blondel`_ +* 14 `Peter Prettenhofer`_ +* 12 `Gael Varoquaux`_ +* 10 `Andreas MÃŧller`_ +* 5 `Lars Buitinck`_ +* 3 :user:`Virgile Fritsch ` +* 1 `Alexandre Gramfort`_ +* 1 `Gilles Louppe`_ +* 1 `Mathieu Blondel`_ .. _changes_0_12: @@ -101,7 +105,7 @@ Changelog - Add MultiTaskLasso and MultiTaskElasticNet for joint feature selection, by `Alexandre Gramfort`_. -- Added :func:`metrics.auc_score` and +- Added `metrics.auc_score` and :func:`metrics.average_precision_score` convenience functions by `Andreas MÃŧller`_. @@ -121,7 +125,7 @@ Changelog and OrthogonalMatchingPursuit) by `Vlad Niculae`_ and `Alexandre Gramfort`_. -- Fixes in :class:`~decomposition.ProbabilisticPCA` score function by Wei Li. +- Fixes in `decomposition.ProbabilisticPCA` score function by Wei Li. - Fixed feature importance computation in :ref:`gradient_boosting`. @@ -133,11 +137,11 @@ API changes summary from ``sklearn`` instead, which was introduced in 0.9. - In :func:`metrics.roc_curve`, the ``thresholds`` array is now returned - with it's order reversed, in order to keep it consistent with the order + with its order reversed, in order to keep it consistent with the order of the returned ``fpr`` and ``tpr``. -- In :class:`hmm` objects, like :class:`~hmm.GaussianHMM`, - :class:`~hmm.MultinomialHMM`, etc., all parameters must be passed to the +- In `hmm` objects, like `hmm.GaussianHMM`, + `hmm.MultinomialHMM`, etc., all parameters must be passed to the object when initialising it and not through ``fit``. Now ``fit`` will only accept the data as an input parameter. @@ -180,9 +184,9 @@ API changes summary :meth:`~ensemble.GradientBoostingClassifier.staged_predict_proba`, and :meth:`~ensemble.GradientBoostingClassifier.staged_predict`. -- :class:`~svm.sparse.SVC` and other sparse SVM classes are now deprecated. - The all classes in the :ref:`svm` module now automatically select the - sparse or dense representation base on the input. +- `svm.sparse.SVC` and other sparse SVM classes are now deprecated. + All classes in the :ref:`svm` module now automatically select the + sparse or dense representation based on the input. - All clustering algorithms now interpret the array ``X`` given to ``fit`` as input data, in particular :class:`~cluster.SpectralClustering` and @@ -194,53 +198,53 @@ API changes summary People ------ - * 267 `Andreas MÃŧller`_ - * 94 `Gilles Louppe`_ - * 89 `Gael Varoquaux`_ - * 79 `Peter Prettenhofer`_ - * 60 `Mathieu Blondel`_ - * 57 `Alexandre Gramfort`_ - * 52 `Vlad Niculae`_ - * 45 `Lars Buitinck`_ - * 44 Nelle Varoquaux - * 37 `Jaques Grobler`_ - * 30 Alexis Mignon - * 30 Immanuel Bayer - * 27 `Olivier Grisel`_ - * 16 Subhodeep Moitra - * 13 Yannick Schwartz - * 12 :user:`@kernc ` - * 11 :user:`Virgile Fritsch ` - * 9 Daniel Duckworth - * 9 `Fabian Pedregosa`_ - * 9 `Robert Layton`_ - * 8 John Benediktsson - * 7 Marko Burjek - * 5 `Nicolas Pinto`_ - * 4 Alexandre Abraham - * 4 `Jake Vanderplas`_ - * 3 `Brian Holt`_ - * 3 `Edouard Duchesnay`_ - * 3 Florian Hoenig - * 3 flyingimmidev - * 2 Francois Savard - * 2 Hannes Schulz - * 2 Peter Welinder - * 2 `Yaroslav Halchenko`_ - * 2 Wei Li - * 1 Alex Companioni - * 1 Brandyn A. White - * 1 Bussonnier Matthias - * 1 Charles-Pierre Astolfi - * 1 Dan O'Huiginn - * 1 David Cournapeau - * 1 Keith Goodman - * 1 Ludwig Schwardt - * 1 Olivier Hervieu - * 1 Sergio Medina - * 1 Shiqiao Du - * 1 Tim Sheerman-Chase - * 1 buguen +* 267 `Andreas MÃŧller`_ +* 94 `Gilles Louppe`_ +* 89 `Gael Varoquaux`_ +* 79 `Peter Prettenhofer`_ +* 60 `Mathieu Blondel`_ +* 57 `Alexandre Gramfort`_ +* 52 `Vlad Niculae`_ +* 45 `Lars Buitinck`_ +* 44 Nelle Varoquaux +* 37 `Jaques Grobler`_ +* 30 Alexis Mignon +* 30 Immanuel Bayer +* 27 `Olivier Grisel`_ +* 16 Subhodeep Moitra +* 13 Yannick Schwartz +* 12 :user:`@kernc ` +* 11 :user:`Virgile Fritsch ` +* 9 Daniel Duckworth +* 9 `Fabian Pedregosa`_ +* 9 `Robert Layton`_ +* 8 John Benediktsson +* 7 Marko Burjek +* 5 `Nicolas Pinto`_ +* 4 Alexandre Abraham +* 4 `Jake Vanderplas`_ +* 3 `Brian Holt`_ +* 3 `Edouard Duchesnay`_ +* 3 Florian Hoenig +* 3 flyingimmidev +* 2 Francois Savard +* 2 Hannes Schulz +* 2 Peter Welinder +* 2 `Yaroslav Halchenko`_ +* 2 Wei Li +* 1 Alex Companioni +* 1 Brandyn A. White +* 1 Bussonnier Matthias +* 1 Charles-Pierre Astolfi +* 1 Dan O'Huiginn +* 1 David Cournapeau +* 1 Keith Goodman +* 1 Ludwig Schwardt +* 1 Olivier Hervieu +* 1 Sergio Medina +* 1 Shiqiao Du +* 1 Tim Sheerman-Chase +* 1 buguen @@ -282,8 +286,8 @@ Highlights - Added BIC/AIC model selection to classical :ref:`gmm` and unified the API with the remainder of scikit-learn, by `Bertrand Thirion`_ -- Added :class:`~sklearn.cross_validation.StratifiedShuffleSplit`, which is - a :class:`~sklearn.cross_validation.ShuffleSplit` with balanced splits, +- Added `sklearn.cross_validation.StratifiedShuffleSplit`, which is + a `sklearn.cross_validation.ShuffleSplit` with balanced splits, by Yannick Schwartz. - :class:`~sklearn.neighbors.NearestCentroid` classifier added, along with a @@ -307,15 +311,15 @@ Other changes - Regressors can now be used as base estimator in the :ref:`multiclass` module by `Mathieu Blondel`_. -- Added n_jobs option to :func:`metrics.pairwise.pairwise_distances` +- Added n_jobs option to :func:`metrics.pairwise_distances` and :func:`metrics.pairwise.pairwise_kernels` for parallel computation, by `Mathieu Blondel`_. - :ref:`k_means` can now be run in parallel, using the ``n_jobs`` argument - to either :ref:`k_means` or :class:`KMeans`, by `Robert Layton`_. + to either :ref:`k_means` or :class:`cluster.KMeans`, by `Robert Layton`_. - Improved :ref:`cross_validation` and :ref:`grid_search` documentation - and introduced the new :func:`cross_validation.train_test_split` + and introduced the new `cross_validation.train_test_split` helper function by `Olivier Grisel`_ - :class:`~svm.SVC` members ``coef_`` and ``intercept_`` changed sign for @@ -330,7 +334,7 @@ Other changes API and fixed a bug that caused possible negative IDF, by `Olivier Grisel`_. -- Beam pruning option in :class:`_BaseHMM` module has been removed since it +- Beam pruning option in `_BaseHMM` module has been removed since it is difficult to Cythonize. If you are interested in contributing a Cython version, you can use the python version in the git history as a reference. @@ -340,31 +344,31 @@ Other changes API changes summary ------------------- -- :class:`~covariance.EllipticEnvelop` is now deprecated - Please use :class:`~covariance.EllipticEnvelope` - instead. +- `covariance.EllipticEnvelop` is now deprecated. + Please use :class:`~covariance.EllipticEnvelope` instead. - ``NeighborsClassifier`` and ``NeighborsRegressor`` are gone in the module - :ref:`neighbors`. Use the classes :class:`KNeighborsClassifier`, - :class:`RadiusNeighborsClassifier`, :class:`KNeighborsRegressor` - and/or :class:`RadiusNeighborsRegressor` instead. + :ref:`neighbors`. Use the classes :class:`~neighbors.KNeighborsClassifier`, + :class:`~neighbors.RadiusNeighborsClassifier`, :class:`~neighbors.KNeighborsRegressor` + and/or :class:`~neighbors.RadiusNeighborsRegressor` instead. - Sparse classes in the :ref:`sgd` module are now deprecated. -- In :class:`~mixture.GMM`, :class:`~mixture.DPGMM` and :class:`~mixture.VBGMM`, +- In `mixture.GMM`, `mixture.DPGMM` and `mixture.VBGMM`, parameters must be passed to an object when initialising it and not through ``fit``. Now ``fit`` will only accept the data as an input parameter. -- methods ``rvs`` and ``decode`` in :class:`GMM` module are now deprecated. +- methods ``rvs`` and ``decode`` in `GMM` module are now deprecated. ``sample`` and ``score`` or ``predict`` should be used instead. - attribute ``_scores`` and ``_pvalues`` in univariate feature selection objects are now deprecated. ``scores_`` or ``pvalues_`` should be used instead. -- In :class:`LogisticRegression`, :class:`LinearSVC`, :class:`SVC` and - :class:`NuSVC`, the ``class_weight`` parameter is now an initialization - parameter, not a parameter to fit. This makes grid searches - over this parameter possible. +- In :class:`~linear_model.LogisticRegression`, :class:`~svm.LinearSVC`, + :class:`~svm.SVC` and :class:`~svm.NuSVC`, the ``class_weight`` parameter is + now an initialization parameter, not a parameter to fit. This makes grid + searches over this parameter possible. - LFW ``data`` is now always shape ``(n_samples, n_features)`` to be consistent with the Olivetti faces dataset. Use ``images`` and @@ -375,14 +379,14 @@ API changes summary ``'ovr'`` being the default. This does not change the default behavior but hopefully is less confusing. -- Class :class:`~feature_selection.text.Vectorizer` is deprecated and - replaced by :class:`~feature_selection.text.TfidfVectorizer`. +- Class `feature_selection.text.Vectorizer` is deprecated and + replaced by `feature_selection.text.TfidfVectorizer`. - The preprocessor / analyzer nested structure for text feature extraction has been removed. All those features are now directly passed as flat constructor arguments - to :class:`~feature_selection.text.TfidfVectorizer` and - :class:`~feature_selection.text.CountVectorizer`, in particular the + to `feature_selection.text.TfidfVectorizer` and + `feature_selection.text.CountVectorizer`, in particular the following parameters are now used: - ``analyzer`` can be ``'word'`` or ``'char'`` to switch the default @@ -401,27 +405,27 @@ API changes summary ``vocabulary_`` attribute to be consistent with the project conventions. -- Class :class:`~feature_selection.text.TfidfVectorizer` now derives directly - from :class:`~feature_selection.text.CountVectorizer` to make grid +- Class `feature_selection.text.TfidfVectorizer` now derives directly + from `feature_selection.text.CountVectorizer` to make grid search trivial. -- methods ``rvs`` in :class:`_BaseHMM` module are now deprecated. +- methods ``rvs`` in `_BaseHMM` module are now deprecated. ``sample`` should be used instead. -- Beam pruning option in :class:`_BaseHMM` module is removed since it is +- Beam pruning option in `_BaseHMM` module is removed since it is difficult to be Cythonized. If you are interested, you can look in the history codes by git. - The SVMlight format loader now supports files with both zero-based and one-based column indices, since both occur "in the wild". -- Arguments in class :class:`ShuffleSplit` are now consistent with - :class:`StratifiedShuffleSplit`. Arguments ``test_fraction`` and +- Arguments in class :class:`~model_selection.ShuffleSplit` are now consistent with + :class:`~model_selection.StratifiedShuffleSplit`. Arguments ``test_fraction`` and ``train_fraction`` are deprecated and renamed to ``test_size`` and ``train_size`` and can accept both ``float`` and ``int``. -- Arguments in class :class:`Bootstrap` are now consistent with - :class:`StratifiedShuffleSplit`. Arguments ``n_test`` and +- Arguments in class `Bootstrap` are now consistent with + :class:`~model_selection.StratifiedShuffleSplit`. Arguments ``n_test`` and ``n_train`` are deprecated and renamed to ``test_size`` and ``train_size`` and can accept both ``float`` and ``int``. @@ -431,54 +435,55 @@ API changes summary People ------ - * 282 `Andreas MÃŧller`_ - * 239 `Peter Prettenhofer`_ - * 198 `Gael Varoquaux`_ - * 129 `Olivier Grisel`_ - * 114 `Mathieu Blondel`_ - * 103 Clay Woolam - * 96 `Lars Buitinck`_ - * 88 `Jaques Grobler`_ - * 82 `Alexandre Gramfort`_ - * 50 `Bertrand Thirion`_ - * 42 `Robert Layton`_ - * 28 flyingimmidev - * 26 `Jake Vanderplas`_ - * 26 Shiqiao Du - * 21 `Satrajit Ghosh`_ - * 17 `David Marek`_ - * 17 `Gilles Louppe`_ - * 14 `Vlad Niculae`_ - * 11 Yannick Schwartz - * 10 `Fabian Pedregosa`_ - * 9 fcostin - * 7 Nick Wilson - * 5 Adrien Gaidon - * 5 `Nicolas Pinto`_ - * 4 `David Warde-Farley`_ - * 5 Nelle Varoquaux - * 5 Emmanuelle Gouillart - * 3 Joonas Sillanpää - * 3 Paolo Losi - * 2 Charles McCarthy - * 2 Roy Hyunjin Han - * 2 Scott White - * 2 ibayer - * 1 Brandyn White - * 1 Carlos Scheidegger - * 1 Claire Revillet - * 1 Conrad Lee - * 1 `Edouard Duchesnay`_ - * 1 Jan Hendrik Metzen - * 1 Meng Xinfan - * 1 `Rob Zinkov`_ - * 1 Shiqiao - * 1 Udi Weinsberg - * 1 Virgile Fritsch - * 1 Xinfan Meng - * 1 Yaroslav Halchenko - * 1 jansoe - * 1 Leon Palafox + +* 282 `Andreas MÃŧller`_ +* 239 `Peter Prettenhofer`_ +* 198 `Gael Varoquaux`_ +* 129 `Olivier Grisel`_ +* 114 `Mathieu Blondel`_ +* 103 Clay Woolam +* 96 `Lars Buitinck`_ +* 88 `Jaques Grobler`_ +* 82 `Alexandre Gramfort`_ +* 50 `Bertrand Thirion`_ +* 42 `Robert Layton`_ +* 28 flyingimmidev +* 26 `Jake Vanderplas`_ +* 26 Shiqiao Du +* 21 `Satrajit Ghosh`_ +* 17 `David Marek`_ +* 17 `Gilles Louppe`_ +* 14 `Vlad Niculae`_ +* 11 Yannick Schwartz +* 10 `Fabian Pedregosa`_ +* 9 fcostin +* 7 Nick Wilson +* 5 Adrien Gaidon +* 5 `Nicolas Pinto`_ +* 4 `David Warde-Farley`_ +* 5 Nelle Varoquaux +* 5 Emmanuelle Gouillart +* 3 Joonas Sillanpää +* 3 Paolo Losi +* 2 Charles McCarthy +* 2 Roy Hyunjin Han +* 2 Scott White +* 2 ibayer +* 1 Brandyn White +* 1 Carlos Scheidegger +* 1 Claire Revillet +* 1 Conrad Lee +* 1 `Edouard Duchesnay`_ +* 1 Jan Hendrik Metzen +* 1 Meng Xinfan +* 1 `Rob Zinkov`_ +* 1 Shiqiao +* 1 Udi Weinsberg +* 1 Virgile Fritsch +* 1 Xinfan Meng +* 1 Yaroslav Halchenko +* 1 jansoe +* 1 Leon Palafox .. _changes_0_10: @@ -557,7 +562,7 @@ Changelog by `Mathieu Blondel`_. - Make :func:`~sklearn.preprocessing.scale` and - :class:`~sklearn.preprocessing.Scaler` work on sparse matrices by + `sklearn.preprocessing.Scaler` work on sparse matrices by `Olivier Grisel`_ - Feature importances using decision trees and/or forest of trees, @@ -566,7 +571,7 @@ Changelog - Parallel implementation of forests of randomized trees by `Gilles Louppe`_. -- :class:`~sklearn.cross_validation.ShuffleSplit` can subsample the train +- `sklearn.cross_validation.ShuffleSplit` can subsample the train sets as well as the test sets by `Olivier Grisel`_. - Errors in the build of the documentation fixed by `Andreas MÃŧller`_. @@ -582,7 +587,7 @@ version 0.9: had ``overwrite_`` parameters; these have been replaced with ``copy_`` parameters with exactly the opposite meaning. - This particularly affects some of the estimators in :mod:`linear_model`. + This particularly affects some of the estimators in :mod:`~sklearn.linear_model`. The default behavior is still to copy everything passed in. - The SVMlight dataset loader :func:`~sklearn.datasets.load_svmlight_file` no @@ -596,17 +601,17 @@ version 0.9: - The :ref:`covariance` module now has a robust estimator of covariance, the Minimum Covariance Determinant estimator. -- Cluster evaluation metrics in :mod:`metrics.cluster` have been refactored +- Cluster evaluation metrics in :mod:`~sklearn.metrics.cluster` have been refactored but the changes are backwards compatible. They have been moved to the - :mod:`metrics.cluster.supervised`, along with - :mod:`metrics.cluster.unsupervised` which contains the Silhouette + `metrics.cluster.supervised`, along with + `metrics.cluster.unsupervised` which contains the Silhouette Coefficient. - The ``permutation_test_score`` function now behaves the same way as ``cross_val_score`` (i.e. uses the mean score across the folds.) - Cross Validation generators now use integer indices (``indices=True``) - by default instead of boolean masks. This make it more intuitive to + by default instead of boolean masks. This makes it more intuitive to use with sparse matrix data. - The functions used for sparse coding, ``sparse_encode`` and @@ -622,7 +627,7 @@ version 0.9: - ``BaseDictionaryLearning`` class replaced by ``SparseCodingMixin``. -- :func:`~sklearn.utils.extmath.fast_svd` has been renamed +- `sklearn.utils.extmath.fast_svd` has been renamed :func:`~sklearn.utils.extmath.randomized_svd` and the default oversampling is now fixed to 10 additional random vectors instead of doubling the number of components to extract. The new behavior @@ -634,37 +639,37 @@ People The following people contributed to scikit-learn since last release: - * 246 `Andreas MÃŧller`_ - * 242 `Olivier Grisel`_ - * 220 `Gilles Louppe`_ - * 183 `Brian Holt`_ - * 166 `Gael Varoquaux`_ - * 144 `Lars Buitinck`_ - * 73 `Vlad Niculae`_ - * 65 `Peter Prettenhofer`_ - * 64 `Fabian Pedregosa`_ - * 60 Robert Layton - * 55 `Mathieu Blondel`_ - * 52 `Jake Vanderplas`_ - * 44 Noel Dawe - * 38 `Alexandre Gramfort`_ - * 24 :user:`Virgile Fritsch ` - * 23 `Satrajit Ghosh`_ - * 3 Jan Hendrik Metzen - * 3 Kenneth C. Arnold - * 3 Shiqiao Du - * 3 Tim Sheerman-Chase - * 3 `Yaroslav Halchenko`_ - * 2 Bala Subrahmanyam Varanasi - * 2 DraXus - * 2 Michael Eickenberg - * 1 Bogdan Trach - * 1 FÊlix-Antoine Fortin - * 1 Juan Manuel Caicedo Carvajal - * 1 Nelle Varoquaux - * 1 `Nicolas Pinto`_ - * 1 Tiziano Zito - * 1 Xinfan Meng +* 246 `Andreas MÃŧller`_ +* 242 `Olivier Grisel`_ +* 220 `Gilles Louppe`_ +* 183 `Brian Holt`_ +* 166 `Gael Varoquaux`_ +* 144 `Lars Buitinck`_ +* 73 `Vlad Niculae`_ +* 65 `Peter Prettenhofer`_ +* 64 `Fabian Pedregosa`_ +* 60 Robert Layton +* 55 `Mathieu Blondel`_ +* 52 `Jake Vanderplas`_ +* 44 Noel Dawe +* 38 `Alexandre Gramfort`_ +* 24 :user:`Virgile Fritsch ` +* 23 `Satrajit Ghosh`_ +* 3 Jan Hendrik Metzen +* 3 Kenneth C. Arnold +* 3 Shiqiao Du +* 3 Tim Sheerman-Chase +* 3 `Yaroslav Halchenko`_ +* 2 Bala Subrahmanyam Varanasi +* 2 DraXus +* 2 Michael Eickenberg +* 1 Bogdan Trach +* 1 FÊlix-Antoine Fortin +* 1 Juan Manuel Caicedo Carvajal +* 1 Nelle Varoquaux +* 1 `Nicolas Pinto`_ +* 1 Tiziano Zito +* 1 Xinfan Meng @@ -744,7 +749,7 @@ Changelog - Text feature extraction optimizations by Lars Buitinck - Chi-Square feature selection - (:func:`feature_selection.univariate_selection.chi2`) by `Lars Buitinck`_. + (:func:`feature_selection.chi2`) by `Lars Buitinck`_. - :ref:`sample_generators` module refactoring by `Gilles Louppe`_ @@ -778,7 +783,7 @@ Changelog - Scalability improvements to :func:`metrics.roc_curve` by Olivier Hervieu -- Distance helper functions :func:`metrics.pairwise.pairwise_distances` +- Distance helper functions :func:`metrics.pairwise_distances` and :func:`metrics.pairwise.pairwise_kernels` by Robert Layton - :class:`Mini-Batch K-Means ` by Nelle Varoquaux and Peter Prettenhofer. @@ -925,7 +930,7 @@ enhancements and bug fixes. Changelog --------- -Several new modules where introduced during this release: +Several new modules were introduced during this release: - New :ref:`hierarchical_clustering` module by Vincent Michel, `Bertrand Thirion`_, `Alexandre Gramfort`_ and `Gael Varoquaux`_. @@ -993,20 +998,20 @@ People that made this release possible preceded by number of commits: - 25 `Peter Prettenhofer`_ - 22 `Nicolas Pinto`_ - 11 :user:`Virgile Fritsch ` - - 7 Lars Buitinck - - 6 Vincent Michel - - 5 `Bertrand Thirion`_ - - 4 Thouis (Ray) Jones - - 4 Vincent Schut - - 3 Jan SchlÃŧter - - 2 Julien Miotte - - 2 `Matthieu Perrot`_ - - 2 Yann Malet - - 2 `Yaroslav Halchenko`_ - - 1 Amit Aides - - 1 `Andreas MÃŧller`_ - - 1 Feth Arezki - - 1 Meng Xinfan +- 7 Lars Buitinck +- 6 Vincent Michel +- 5 `Bertrand Thirion`_ +- 4 Thouis (Ray) Jones +- 4 Vincent Schut +- 3 Jan SchlÃŧter +- 2 Julien Miotte +- 2 `Matthieu Perrot`_ +- 2 Yann Malet +- 2 `Yaroslav Halchenko`_ +- 1 Amit Aides +- 1 `Andreas MÃŧller`_ +- 1 Feth Arezki +- 1 Meng Xinfan .. _changes_0_7: @@ -1021,7 +1026,7 @@ after the 0.6 release. This release is marked by the speed improvements in existing algorithms like k-Nearest Neighbors and K-Means algorithm and by the inclusion of an efficient algorithm for computing the Ridge Generalized Cross Validation solution. Unlike the -preceding release, no new modules where added to this release. +preceding release, no new modules were added to this release. Changelog --------- @@ -1047,7 +1052,7 @@ Changelog - Sanity checks for SVM-based classes [`Mathieu Blondel`_]. -- Refactoring of :class:`~neighbors.NeighborsClassifier` and +- Refactoring of `neighbors.NeighborsClassifier` and :func:`neighbors.kneighbors_graph`: added different algorithms for the k-Nearest Neighbor Search and implemented a more stable algorithm for finding barycenter weights. Also added some @@ -1055,7 +1060,7 @@ Changelog `notes_neighbors `_ for more information [`Fabian Pedregosa`_]. -- Documentation improvements: Added :class:`~pca.RandomizedPCA` and +- Documentation improvements: Added `pca.RandomizedPCA` and :class:`~linear_model.LogisticRegression` to the class reference. Also added references of matrices used for clustering and other fixes [`Gael Varoquaux`_, `Fabian Pedregosa`_, `Mathieu @@ -1067,12 +1072,12 @@ Changelog :class:`~linear_model.LogisticRegression` [`Fabian Pedregosa`_]. - Performance and API improvements to - :func:`metrics.euclidean_distances` and to - :class:`~pca.RandomizedPCA` [`James Bergstra`_]. + :func:`metrics.pairwise.euclidean_distances` and to + `pca.RandomizedPCA` [`James Bergstra`_]. - Fix compilation issues under NetBSD [Kamel Ibn Hassen Derouiche] -- Allow input sequences of different lengths in :class:`~hmm.GaussianHMM` +- Allow input sequences of different lengths in `hmm.GaussianHMM` [`Ron Weiss`_]. - Fix bug in affinity propagation caused by incorrect indexing [Xinfan Meng] @@ -1119,7 +1124,7 @@ Changelog --------- - New `stochastic gradient - `_ descent + `_ descent module by Peter Prettenhofer. The module comes with complete documentation and examples. @@ -1134,21 +1139,20 @@ Changelog example_gaussian_process_plot_gp_probabilistic_classification_after_regression.py for a taste of what can be done. -- It is now possible to use liblinear’s Multi-class SVC (option +- It is now possible to use liblinear's Multi-class SVC (option multi_class in :class:`~svm.LinearSVC`) - New features and performance improvements of text feature extraction. - Improved sparse matrix support, both in main classes - (:class:`~grid_search.GridSearchCV`) as in modules + (:class:`~model_selection.GridSearchCV`) as in modules sklearn.svm.sparse and sklearn.linear_model.sparse. - Lots of cool new examples and a new section that uses real-world datasets was created. These include: :ref:`sphx_glr_auto_examples_applications_plot_face_recognition.py`, :ref:`sphx_glr_auto_examples_applications_plot_species_distribution_modeling.py`, - :ref:`sphx_glr_auto_examples_applications_svm_gui.py`, :ref:`sphx_glr_auto_examples_applications_wikipedia_principal_eigenvector.py` and others. @@ -1175,31 +1179,31 @@ People People that made this release possible preceded by number of commits: - * 207 `Olivier Grisel`_ +* 207 `Olivier Grisel`_ - * 167 `Fabian Pedregosa`_ +* 167 `Fabian Pedregosa`_ - * 97 `Peter Prettenhofer`_ +* 97 `Peter Prettenhofer`_ - * 68 `Alexandre Gramfort`_ +* 68 `Alexandre Gramfort`_ - * 59 `Mathieu Blondel`_ +* 59 `Mathieu Blondel`_ - * 55 `Gael Varoquaux`_ +* 55 `Gael Varoquaux`_ - * 33 Vincent Dubourg +* 33 Vincent Dubourg - * 21 `Ron Weiss`_ +* 21 `Ron Weiss`_ - * 9 Bertrand Thirion +* 9 Bertrand Thirion - * 3 `Alexandre Passos`_ +* 3 `Alexandre Passos`_ - * 3 Anne-Laure Fouque +* 3 Anne-Laure Fouque - * 2 Ronan Amicel +* 2 Ronan Amicel - * 1 `Christian Osendorfer`_ +* 1 `Christian Osendorfer`_ @@ -1218,9 +1222,9 @@ New classes ----------- - Support for sparse matrices in some classifiers of modules - ``svm`` and ``linear_model`` (see :class:`~svm.sparse.SVC`, - :class:`~svm.sparse.SVR`, :class:`~svm.sparse.LinearSVC`, - :class:`~linear_model.sparse.Lasso`, :class:`~linear_model.sparse.ElasticNet`) + ``svm`` and ``linear_model`` (see `svm.sparse.SVC`, + `svm.sparse.SVR`, `svm.sparse.LinearSVC`, + `linear_model.sparse.Lasso`, `linear_model.sparse.ElasticNet`) - New :class:`~pipeline.Pipeline` object to compose different estimators. @@ -1237,8 +1241,7 @@ New classes :class:`~linear_model.LassoLars`. - New Hidden Markov Models module (see classes - :class:`~hmm.GaussianHMM`, :class:`~hmm.MultinomialHMM`, - :class:`~hmm.GMMHMM`) + `hmm.GaussianHMM`, `hmm.MultinomialHMM`, `hmm.GMMHMM`) - New module feature_extraction (see :ref:`class reference `) @@ -1252,9 +1255,9 @@ Documentation - Improved documentation for many modules, now separating narrative documentation from the class reference. As an example, see `documentation for the SVM module - `_ and the + `_ and the complete `class reference - `_. + `_. Fixes ----- @@ -1276,7 +1279,7 @@ Examples :ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py` - Many more examples. `See here - `_ + `_ the full list of examples. @@ -1305,20 +1308,20 @@ Authors The following is a list of authors for this release, preceded by number of commits: - * 262 Fabian Pedregosa - * 240 Gael Varoquaux - * 149 Alexandre Gramfort - * 116 Olivier Grisel - * 40 Vincent Michel - * 38 Ron Weiss - * 23 Matthieu Perrot - * 10 Bertrand Thirion - * 7 Yaroslav Halchenko - * 9 VirgileFritsch - * 6 Edouard Duchesnay - * 4 Mathieu Blondel - * 1 Ariel Rokem - * 1 Matthieu Brucher +* 262 Fabian Pedregosa +* 240 Gael Varoquaux +* 149 Alexandre Gramfort +* 116 Olivier Grisel +* 40 Vincent Michel +* 38 Ron Weiss +* 23 Matthieu Perrot +* 10 Bertrand Thirion +* 7 Yaroslav Halchenko +* 9 VirgileFritsch +* 6 Edouard Duchesnay +* 4 Mathieu Blondel +* 1 Ariel Rokem +* 1 Matthieu Brucher Version 0.4 =========== @@ -1369,13 +1372,13 @@ Authors The committer list for this release is the following (preceded by number of commits): - * 143 Fabian Pedregosa - * 35 Alexandre Gramfort - * 34 Olivier Grisel - * 11 Gael Varoquaux - * 5 Yaroslav Halchenko - * 2 Vincent Michel - * 1 Chris Filo Gorgolewski +* 143 Fabian Pedregosa +* 35 Alexandre Gramfort +* 34 Olivier Grisel +* 11 Gael Varoquaux +* 5 Yaroslav Halchenko +* 2 Vincent Michel +* 1 Chris Filo Gorgolewski Earlier versions @@ -1383,4 +1386,3 @@ Earlier versions Earlier versions included contributions by Fred Mailhot, David Cooke, David Huard, Dave Morrill, Ed Schofield, Travis Oliphant, Pearu Peterson. - diff --git a/doc/whats_new/upcoming_changes/.gitkeep b/doc/whats_new/upcoming_changes/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/README.md b/doc/whats_new/upcoming_changes/README.md new file mode 100644 index 0000000000000..3524eebb0e339 --- /dev/null +++ b/doc/whats_new/upcoming_changes/README.md @@ -0,0 +1,52 @@ +# Changelog instructions + +This directory (`doc/whats_new/upcoming_changes`) contains "news fragments", +which are short files that contain a small **ReST**-formatted text that will be +added to the next release changelog. + +Each file should be named like `..rst`, where +`` is a pull request number, and `` is one of: + +* `major-feature` +* `feature` +* `efficiency` +* `enhancement` +* `fix` +* `api` +* `other` (see [](#custom-top-level-folder)) + +See [this](https://github.com/scikit-learn/scikit-learn/blob/main/doc/whats_new/changelog_legend.inc) +for more details about the meaning of each type. + +This file needs to be added to the right folder like `sklearn.linear_model` or +`sklearn.tree` depending on which part of scikit-learn your PR changes. There +are also a few folders for some topics like `array-api`, `metadata-routing` or `security`. + +In almost all cases, your fragment should be formatted as a bullet point. + +For example, `28268.feature.rst` would be added to the `sklearn.ensemble` +folder with the following content:: + +```rst +- :class:`ensemble.ExtraTreesClassifier` and :class:`ensemble.ExtraTreesRegressor` + now supports missing values in the data matrix `X`. Missing-values are + handled by randomly moving all of the samples to the left, or right child + node as the tree is traversed. + By :user:`Adam Li ` +``` + +If you are unsure how to name the news fragment or which folder to use, don't +hesitate to ask in your pull request! + +You can install [`towncrier`](https://github.com/twisted/towncrier) and run +`towncrier create` to help you create a news fragment. You can also run +`towncrier build --draft --version ` if +you want to get a preview of how your change will look in the final release +notes. + + +## `custom-top-level` folder + +The `custom-top-level` folder is for changes for which there is no good +folder and are somewhat one-off topics. Type `other` is mostly meant to be used +in the `custom-top-level` section. diff --git a/doc/whats_new/upcoming_changes/array-api/.gitkeep b/doc/whats_new/upcoming_changes/array-api/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/changed-models/.gitkeep b/doc/whats_new/upcoming_changes/changed-models/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/custom-top-level/.gitkeep b/doc/whats_new/upcoming_changes/custom-top-level/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/many-modules/.gitkeep b/doc/whats_new/upcoming_changes/many-modules/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/metadata-routing/.gitkeep b/doc/whats_new/upcoming_changes/metadata-routing/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/security/.gitkeep b/doc/whats_new/upcoming_changes/security/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.base/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.base/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.calibration/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.calibration/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.cluster/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.cluster/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.compose/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.compose/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.covariance/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.covariance/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.cross_decomposition/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.cross_decomposition/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.datasets/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.datasets/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.decomposition/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.decomposition/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.discriminant_analysis/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.discriminant_analysis/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.dummy/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.dummy/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.ensemble/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.ensemble/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.exceptions/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.exceptions/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.feature_extraction/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.feature_extraction/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.feature_selection/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.feature_selection/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.frozen/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.frozen/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.gaussian_process/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.gaussian_process/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.impute/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.impute/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.inspection/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.inspection/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.isotonic/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.isotonic/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.kernel_approximation/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.kernel_approximation/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.kernel_ridge/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.kernel_ridge/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.linear_model/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.linear_model/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.manifold/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.manifold/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.metrics/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.metrics/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.metrics/31406.enhancement.rst b/doc/whats_new/upcoming_changes/sklearn.metrics/31406.enhancement.rst new file mode 100644 index 0000000000000..4736c67c80132 --- /dev/null +++ b/doc/whats_new/upcoming_changes/sklearn.metrics/31406.enhancement.rst @@ -0,0 +1,2 @@ +- :func:`metrics.median_absolute_error` now supports Array API compatible inputs. + By :user:`Lucy Liu `. diff --git a/doc/whats_new/upcoming_changes/sklearn.mixture/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.mixture/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.model_selection/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.model_selection/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.multiclass/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.multiclass/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.multioutput/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.multioutput/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.naive_bayes/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.naive_bayes/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.neighbors/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.neighbors/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.neural_network/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.neural_network/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.pipeline/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.pipeline/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.preprocessing/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.preprocessing/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.random_projection/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.random_projection/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.semi_supervised/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.semi_supervised/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.svm/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.svm/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.tree/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.tree/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/sklearn.utils/.gitkeep b/doc/whats_new/upcoming_changes/sklearn.utils/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/doc/whats_new/upcoming_changes/towncrier_template.rst.jinja2 b/doc/whats_new/upcoming_changes/towncrier_template.rst.jinja2 new file mode 100644 index 0000000000000..98c84a1d85b91 --- /dev/null +++ b/doc/whats_new/upcoming_changes/towncrier_template.rst.jinja2 @@ -0,0 +1,49 @@ +{% set version_underscore = versiondata.version.replace('.', '_') %} +.. _changes_{{ version_underscore }}: + +{% set title = "Version " + versiondata.version %} +{{ title }} +{{ top_underline * title|length }} + +{% set month_names = { + '01': 'January', '02': 'February', '03': 'March', '04': 'April', + '05': 'May', '06': 'June', '07': 'July', '08': 'August', + '09': 'September', '10': 'October', '11': 'November', '12': 'December' +} %} +{% set year, month, _ = versiondata.date.split('-') %} +{% set release_date = month_names[month] + ' ' + year %} +**{{ release_date }}** + +{% set underline = underlines[0] %} +{% for section, content_per_category in sections.items() if content_per_category %} +{% if section != 'custom-top-level' %} +{{ section }} +{{ underline * section|length }} + +{% endif %} +{# section-specific description #} +{% if section == 'Support for Array API' %} +Additional estimators and functions have been updated to include support for all +`Array API `_ compliant inputs. + +See :ref:`array_api` for more details. + +{% endif %} +{% if section == 'Metadata routing' %} +Refer to the :ref:`Metadata Routing User Guide ` for +more details. + +{% endif %} +{# We loop over definitions because, contrary to content_per_category, it follow the category order as defined in pyproject.toml #} +{% for category in definitions if category in content_per_category %} +{% set content = content_per_category[category] %} +{% for text, issue_links in content.items() %} +{% set tag = definitions[category]['name'] %} +{# If category != 'other' add tag like |Fix| or |Feature|. This assumes the text is formatted as a bullet point #} +{% set text_with_tag = text if category == 'other' else '{0} {1}{2}'.format(text[0], tag, text[1:]) %} +{# issue_links is a list so need to join. For our purposes, issue_links is always of length 1 #} +{{ text_with_tag }} {{ issue_links|join(', ') }} + +{% endfor %} +{% endfor %} +{% endfor %} diff --git a/doc/whats_new/v0.13.rst b/doc/whats_new/v0.13.rst index 10b4d3b5b783f..2c48f48be9b01 100644 --- a/doc/whats_new/v0.13.rst +++ b/doc/whats_new/v0.13.rst @@ -2,6 +2,10 @@ .. currentmodule:: sklearn +============ +Version 0.13 +============ + .. _changes_0_13_1: Version 0.13.1 @@ -14,7 +18,7 @@ The 0.13.1 release only fixes some bugs and does not add any new functionality. Changelog --------- -- Fixed a testing error caused by the function :func:`cross_validation.train_test_split` being +- Fixed a testing error caused by the function `cross_validation.train_test_split` being interpreted as a test by `Yaroslav Halchenko`_. - Fixed a bug in the reassignment of small clusters in the :class:`cluster.MiniBatchKMeans` @@ -33,21 +37,22 @@ Changelog People ------ List of contributors for release 0.13.1 by number of commits. - * 16 `Lars Buitinck`_ - * 12 `Andreas MÃŧller`_ - * 8 `Gael Varoquaux`_ - * 5 Robert Marchman - * 3 `Peter Prettenhofer`_ - * 2 Hrishikesh Huilgolkar - * 1 Bastiaan van den Berg - * 1 Diego Molla - * 1 `Gilles Louppe`_ - * 1 `Mathieu Blondel`_ - * 1 `Nelle Varoquaux`_ - * 1 Rafael Cunha de Almeida - * 1 Rolando Espinoza La fuente - * 1 `Vlad Niculae`_ - * 1 `Yaroslav Halchenko`_ + +* 16 `Lars Buitinck`_ +* 12 `Andreas MÃŧller`_ +* 8 `Gael Varoquaux`_ +* 5 Robert Marchman +* 3 `Peter Prettenhofer`_ +* 2 Hrishikesh Huilgolkar +* 1 Bastiaan van den Berg +* 1 Diego Molla +* 1 `Gilles Louppe`_ +* 1 `Mathieu Blondel`_ +* 1 `Nelle Varoquaux`_ +* 1 Rafael Cunha de Almeida +* 1 Rolando Espinoza La fuente +* 1 `Vlad Niculae`_ +* 1 `Yaroslav Halchenko`_ .. _changes_0_13: @@ -117,7 +122,7 @@ Changelog --------- - :func:`metrics.zero_one_loss` (formerly ``metrics.zero_one``) now has - option for normalized output that reports the fraction of + an option for normalized output that reports the fraction of misclassifications, rather than the raw number of misclassifications. By Kyle Beauchamp. @@ -128,7 +133,7 @@ Changelog trees, by `Peter Prettenhofer`_ and `Gilles Louppe`_. - Partial dependence plots for :ref:`gradient_boosting` in - :func:`ensemble.partial_dependence.partial_dependence` by `Peter + `ensemble.partial_dependence.partial_dependence` by `Peter Prettenhofer`_. See :ref:`sphx_glr_auto_examples_inspection_plot_partial_dependence.py` for an example. @@ -161,7 +166,7 @@ Changelog - Faster and more robust :func:`metrics.confusion_matrix` and :ref:`clustering_evaluation` by Wei Li. -- :func:`cross_validation.cross_val_score` now works with precomputed kernels +- `cross_validation.cross_val_score` now works with precomputed kernels and affinity matrices, by `Andreas MÃŧller`_. - LARS algorithm made more numerically stable with heuristics to drop @@ -171,7 +176,7 @@ Changelog - Faster implementation of :func:`metrics.precision_recall_curve` by Conrad Lee. -- New kernel :class:`metrics.chi2_kernel` by `Andreas MÃŧller`_, often used +- New kernel `metrics.chi2_kernel` by `Andreas MÃŧller`_, often used in computer vision applications. - Fix of longstanding bug in :class:`naive_bayes.BernoulliNB` fixed by @@ -184,7 +189,7 @@ Changelog :class:`ensemble.GradientBoostingRegressor` and :class:`ensemble.GradientBoostingClassifier` use the estimator :class:`tree.DecisionTreeRegressor` instead of the - :class:`tree._tree.Tree` data structure by `Arnaud Joly`_. + `tree._tree.Tree` data structure by `Arnaud Joly`_. - Fixed a floating point exception in the :ref:`decision trees ` module, by Seberg. @@ -209,7 +214,7 @@ Changelog - Fixed a bug in :class:`sklearn.svm.SVC` when using csr-matrices with unsorted indices by Xinfan Meng and `Andreas MÃŧller`_. -- :class:`MiniBatchKMeans`: Add random reassignment of cluster centers +- :class:`cluster.MiniBatchKMeans`: Add random reassignment of cluster centers with little observations attached to them, by `Gael Varoquaux`_. @@ -221,18 +226,18 @@ API changes summary :func:`decomposition.dict_learning`, :func:`decomposition.dict_learning_online`. - Renamed all occurrences of ``max_iters`` to ``max_iter`` for consistency. - This applies to :class:`semi_supervised.LabelPropagation` and - :class:`semi_supervised.label_propagation.LabelSpreading`. + This applies to `semi_supervised.LabelPropagation` and + `semi_supervised.label_propagation.LabelSpreading`. - Renamed all occurrences of ``learn_rate`` to ``learning_rate`` for - consistency in :class:`ensemble.BaseGradientBoosting` and + consistency in `ensemble.BaseGradientBoosting` and :class:`ensemble.GradientBoostingRegressor`. - The module ``sklearn.linear_model.sparse`` is gone. Sparse matrix support was already integrated into the "regular" linear models. -- :func:`sklearn.metrics.mean_square_error`, which incorrectly returned the - accumulated error, was removed. Use ``mean_squared_error`` instead. +- `sklearn.metrics.mean_square_error`, which incorrectly returned the + accumulated error, was removed. Use :func:`metrics.mean_squared_error` instead. - Passing ``class_weight`` parameters to ``fit`` methods is no longer supported. Pass them to estimator constructors instead. @@ -244,17 +249,18 @@ API changes summary deprecated and will be removed in v0.14. Use the constructor option instead. -- :class:`feature_extraction.text.DictVectorizer` now returns sparse +- `feature_extraction.text.DictVectorizer` now returns sparse matrices in the CSR format, instead of COO. -- Renamed ``k`` in :class:`cross_validation.KFold` and - :class:`cross_validation.StratifiedKFold` to ``n_folds``, renamed +- Renamed ``k`` in `cross_validation.KFold` and + `cross_validation.StratifiedKFold` to ``n_folds``, renamed ``n_bootstraps`` to ``n_iter`` in ``cross_validation.Bootstrap``. - Renamed all occurrences of ``n_iterations`` to ``n_iter`` for consistency. - This applies to :class:`cross_validation.ShuffleSplit`, - :class:`cross_validation.StratifiedShuffleSplit`, - :func:`utils.randomized_range_finder` and :func:`utils.randomized_svd`. + This applies to `cross_validation.ShuffleSplit`, + `cross_validation.StratifiedShuffleSplit`, + :func:`utils.extmath.randomized_range_finder` and + :func:`utils.extmath.randomized_svd`. - Replaced ``rho`` in :class:`linear_model.ElasticNet` and :class:`linear_model.SGDClassifier` by ``l1_ratio``. The ``rho`` parameter @@ -267,10 +273,10 @@ API changes summary store a list of paths in the case of multiple targets, rather than an array of paths. -- The attribute ``gmm`` of :class:`hmm.GMMHMM` was renamed to ``gmm_`` +- The attribute ``gmm`` of `hmm.GMMHMM` was renamed to ``gmm_`` to adhere more strictly with the API. -- :func:`cluster.spectral_embedding` was moved to +- `cluster.spectral_embedding` was moved to :func:`manifold.spectral_embedding`. - Renamed ``eig_tol`` in :func:`manifold.spectral_embedding`, @@ -286,9 +292,9 @@ API changes summary multi-output problems. - The ``estimators_`` attribute of - :class:`ensemble.gradient_boosting.GradientBoostingRegressor` and - :class:`ensemble.gradient_boosting.GradientBoostingClassifier` is now an - array of :class:'tree.DecisionTreeRegressor'. + :class:`ensemble.GradientBoostingRegressor` and + :class:`ensemble.GradientBoostingClassifier` is now an + array of :class:`tree.DecisionTreeRegressor`. - Renamed ``chunk_size`` to ``batch_size`` in :class:`decomposition.MiniBatchDictionaryLearning` and @@ -299,18 +305,18 @@ API changes summary Also, the dtype returned by ``predict`` now reflects the dtype of ``y`` during ``fit`` (used to be ``np.float``). -- Changed default test_size in :func:`cross_validation.train_test_split` +- Changed default test_size in `cross_validation.train_test_split` to None, added possibility to infer ``test_size`` from ``train_size`` in - :class:`cross_validation.ShuffleSplit` and - :class:`cross_validation.StratifiedShuffleSplit`. + `cross_validation.ShuffleSplit` and + `cross_validation.StratifiedShuffleSplit`. -- Renamed function :func:`sklearn.metrics.zero_one` to - :func:`sklearn.metrics.zero_one_loss`. Be aware that the default behavior - in :func:`sklearn.metrics.zero_one_loss` is different from - :func:`sklearn.metrics.zero_one`: ``normalize=False`` is changed to +- Renamed function `sklearn.metrics.zero_one` to + `sklearn.metrics.zero_one_loss`. Be aware that the default behavior + in `sklearn.metrics.zero_one_loss` is different from + `sklearn.metrics.zero_one`: ``normalize=False`` is changed to ``normalize=True``. -- Renamed function :func:`metrics.zero_one_score` to +- Renamed function `metrics.zero_one_score` to :func:`metrics.accuracy_score`. - :func:`datasets.make_circles` now has the same number of inner and outer points. @@ -322,70 +328,69 @@ People ------ List of contributors for release 0.13 by number of commits. - * 364 `Andreas MÃŧller`_ - * 143 `Arnaud Joly`_ - * 137 `Peter Prettenhofer`_ - * 131 `Gael Varoquaux`_ - * 117 `Mathieu Blondel`_ - * 108 `Lars Buitinck`_ - * 106 Wei Li - * 101 `Olivier Grisel`_ - * 65 `Vlad Niculae`_ - * 54 `Gilles Louppe`_ - * 40 `Jaques Grobler`_ - * 38 `Alexandre Gramfort`_ - * 30 `Rob Zinkov`_ - * 19 Aymeric Masurelle - * 18 Andrew Winterman - * 17 `Fabian Pedregosa`_ - * 17 Nelle Varoquaux - * 16 `Christian Osendorfer`_ - * 14 `Daniel Nouri`_ - * 13 :user:`Virgile Fritsch ` - * 13 syhw - * 12 `Satrajit Ghosh`_ - * 10 Corey Lynch - * 10 Kyle Beauchamp - * 9 Brian Cheung - * 9 Immanuel Bayer - * 9 mr.Shu - * 8 Conrad Lee - * 8 `James Bergstra`_ - * 7 Tadej JaneÅž - * 6 Brian Cajes - * 6 `Jake Vanderplas`_ - * 6 Michael - * 6 Noel Dawe - * 6 Tiago Nunes - * 6 cow - * 5 Anze - * 5 Shiqiao Du - * 4 Christian Jauvin - * 4 Jacques Kvam - * 4 Richard T. Guy - * 4 `Robert Layton`_ - * 3 Alexandre Abraham - * 3 Doug Coleman - * 3 Scott Dickerson - * 2 ApproximateIdentity - * 2 John Benediktsson - * 2 Mark Veronda - * 2 Matti Lyra - * 2 Mikhail Korobov - * 2 Xinfan Meng - * 1 Alejandro Weinstein - * 1 `Alexandre Passos`_ - * 1 Christoph Deil - * 1 Eugene Nizhibitsky - * 1 Kenneth C. Arnold - * 1 Luis Pedro Coelho - * 1 Miroslav Batchkarov - * 1 Pavel - * 1 Sebastian Berg - * 1 Shaun Jackman - * 1 Subhodeep Moitra - * 1 bob - * 1 dengemann - * 1 emanuele - * 1 x006 - +* 364 `Andreas MÃŧller`_ +* 143 `Arnaud Joly`_ +* 137 `Peter Prettenhofer`_ +* 131 `Gael Varoquaux`_ +* 117 `Mathieu Blondel`_ +* 108 `Lars Buitinck`_ +* 106 Wei Li +* 101 `Olivier Grisel`_ +* 65 `Vlad Niculae`_ +* 54 `Gilles Louppe`_ +* 40 `Jaques Grobler`_ +* 38 `Alexandre Gramfort`_ +* 30 `Rob Zinkov`_ +* 19 Aymeric Masurelle +* 18 Andrew Winterman +* 17 `Fabian Pedregosa`_ +* 17 Nelle Varoquaux +* 16 `Christian Osendorfer`_ +* 14 `Daniel Nouri`_ +* 13 :user:`Virgile Fritsch ` +* 13 syhw +* 12 `Satrajit Ghosh`_ +* 10 Corey Lynch +* 10 Kyle Beauchamp +* 9 Brian Cheung +* 9 Immanuel Bayer +* 9 mr.Shu +* 8 Conrad Lee +* 8 `James Bergstra`_ +* 7 Tadej JaneÅž +* 6 Brian Cajes +* 6 `Jake Vanderplas`_ +* 6 Michael +* 6 Noel Dawe +* 6 Tiago Nunes +* 6 cow +* 5 Anze +* 5 Shiqiao Du +* 4 Christian Jauvin +* 4 Jacques Kvam +* 4 Richard T. Guy +* 4 `Robert Layton`_ +* 3 Alexandre Abraham +* 3 Doug Coleman +* 3 Scott Dickerson +* 2 ApproximateIdentity +* 2 John Benediktsson +* 2 Mark Veronda +* 2 Matti Lyra +* 2 Mikhail Korobov +* 2 Xinfan Meng +* 1 Alejandro Weinstein +* 1 `Alexandre Passos`_ +* 1 Christoph Deil +* 1 Eugene Nizhibitsky +* 1 Kenneth C. Arnold +* 1 Luis Pedro Coelho +* 1 Miroslav Batchkarov +* 1 Pavel +* 1 Sebastian Berg +* 1 Shaun Jackman +* 1 Subhodeep Moitra +* 1 bob +* 1 dengemann +* 1 emanuele +* 1 x006 diff --git a/doc/whats_new/v0.14.rst b/doc/whats_new/v0.14.rst index 5abe7d12d2051..15b3320c26531 100644 --- a/doc/whats_new/v0.14.rst +++ b/doc/whats_new/v0.14.rst @@ -2,6 +2,10 @@ .. currentmodule:: sklearn +============ +Version 0.14 +============ + .. _changes_0_14: Version 0.14 @@ -13,9 +17,9 @@ Changelog --------- - Missing values with sparse and dense matrices can be imputed with the - transformer :class:`preprocessing.Imputer` by `Nicolas TrÊsegnie`_. + transformer `preprocessing.Imputer` by `Nicolas TrÊsegnie`_. -- The core implementation of decisions trees has been rewritten from +- The core implementation of decision trees has been rewritten from scratch, allowing for faster tree induction and lower memory consumption in all tree-based estimators. By `Gilles Louppe`_. @@ -24,13 +28,13 @@ Changelog `Gilles Louppe`_. See the :ref:`AdaBoost ` section of the user guide for details and examples. -- Added :class:`grid_search.RandomizedSearchCV` and - :class:`grid_search.ParameterSampler` for randomized hyperparameter +- Added `grid_search.RandomizedSearchCV` and + `grid_search.ParameterSampler` for randomized hyperparameter optimization. By `Andreas MÃŧller`_. - Added :ref:`biclustering ` algorithms - (:class:`sklearn.cluster.bicluster.SpectralCoclustering` and - :class:`sklearn.cluster.bicluster.SpectralBiclustering`), data + (`sklearn.cluster.bicluster.SpectralCoclustering` and + `sklearn.cluster.bicluster.SpectralBiclustering`), data generation methods (:func:`sklearn.datasets.make_biclusters` and :func:`sklearn.datasets.make_checkerboard`), and scoring metrics (:func:`sklearn.metrics.consensus_score`). By `Kemal Eren`_. @@ -45,7 +49,7 @@ Changelog - Ability to pass one penalty (alpha value) per target in :class:`linear_model.Ridge`, by @eickenberg and `Mathieu Blondel`_. -- Fixed :mod:`sklearn.linear_model.stochastic_gradient.py` L2 regularization +- Fixed `sklearn.linear_model.stochastic_gradient.py` L2 regularization issue (minor practical significance). By :user:`Norbert Crombach ` and `Mathieu Blondel`_ . @@ -55,9 +59,9 @@ Changelog to the documentation. See :ref:`Choosing the right estimator `. By `Jaques Grobler`_. -- :class:`grid_search.GridSearchCV` and - :func:`cross_validation.cross_val_score` now support the use of advanced - scoring function such as area under the ROC curve and f-beta scores. +- `grid_search.GridSearchCV` and + `cross_validation.cross_val_score` now support the use of advanced + scoring functions such as area under the ROC curve and f-beta scores. See :ref:`scoring_parameter` for details. By `Andreas MÃŧller`_ and `Lars Buitinck`_. Passing a function from :mod:`sklearn.metrics` as ``score_func`` is @@ -71,7 +75,7 @@ Changelog by `Arnaud Joly`_. - Two new metrics :func:`metrics.hamming_loss` and - :func:`metrics.jaccard_similarity_score` + `metrics.jaccard_similarity_score` are added with multi-label support by `Arnaud Joly`_. - Speed and memory usage improvements in @@ -103,13 +107,13 @@ Changelog - ``max_features`` in :class:`tree.DecisionTreeClassifier`, :class:`tree.DecisionTreeRegressor` and all derived ensemble estimators - now supports percentage values. By `Gilles Louppe`_. + now support percentage values. By `Gilles Louppe`_. - Performance improvements in :class:`isotonic.IsotonicRegression` by `Nelle Varoquaux`_. - :func:`metrics.accuracy_score` has an option normalize to return - the fraction or the number of correctly classified sample + the fraction or the number of correctly classified samples by `Arnaud Joly`_. - Added :func:`metrics.log_loss` that computes log loss, aka cross-entropy @@ -121,8 +125,8 @@ Changelog - Feature selectors now share a mixin providing consistent ``transform``, ``inverse_transform`` and ``get_support`` methods. By `Joel Nothman`_. -- A fitted :class:`grid_search.GridSearchCV` or - :class:`grid_search.RandomizedSearchCV` can now generally be pickled. +- A fitted `grid_search.GridSearchCV` or + `grid_search.RandomizedSearchCV` can now generally be pickled. By `Joel Nothman`_. - Refactored and vectorized implementation of :func:`metrics.roc_curve` @@ -138,7 +142,7 @@ Changelog By :user:`Eustache Diemert `. - The default number of components for - :class:`sklearn.decomposition.RandomizedPCA` is now correctly documented + `sklearn.decomposition.RandomizedPCA` is now correctly documented to be ``n_features``. This was the default behavior, so programs using it will continue to work as they did. @@ -149,12 +153,12 @@ Changelog - Reduce memory footprint of FastICA by `Denis Engemann`_ and `Alexandre Gramfort`_. -- Verbose output in :mod:`sklearn.ensemble.gradient_boosting` now uses +- Verbose output in `sklearn.ensemble.gradient_boosting` now uses a column format and prints progress in decreasing frequency. It also shows the remaining time. By `Peter Prettenhofer`_. -- :mod:`sklearn.ensemble.gradient_boosting` provides out-of-bag improvement - :attr:`~sklearn.ensemble.GradientBoostingRegressor.oob_improvement_` +- `sklearn.ensemble.gradient_boosting` provides out-of-bag improvement + `oob_improvement_` rather than the OOB score for model selection. An example that shows how to use OOB estimates to select the number of trees was added. By `Peter Prettenhofer`_. @@ -165,17 +169,17 @@ Changelog - New OrthogonalMatchingPursuitCV class by `Alexandre Gramfort`_ and `Vlad Niculae`_. -- Fixed a bug in :class:`sklearn.covariance.GraphLassoCV`: the +- Fixed a bug in `sklearn.covariance.GraphLassoCV`: the 'alphas' parameter now works as expected when given a list of values. By Philippe Gervais. -- Fixed an important bug in :class:`sklearn.covariance.GraphLassoCV` +- Fixed an important bug in `sklearn.covariance.GraphLassoCV` that prevented all folds provided by a CV object to be used (only the first 3 were used). When providing a CV object, execution time may thus increase significantly compared to the previous version (bug results are correct now). By Philippe Gervais. -- :class:`cross_validation.cross_val_score` and the :mod:`grid_search` +- `cross_validation.cross_val_score` and the `grid_search` module is now tested with multi-output data by `Arnaud Joly`_. - :func:`datasets.make_multilabel_classification` can now return @@ -187,8 +191,8 @@ Changelog :class:`neighbors.RadiusNeighborsClassifier` support multioutput data by `Arnaud Joly`_. -- Random state in LibSVM-based estimators (:class:`svm.SVC`, :class:`NuSVC`, - :class:`OneClassSVM`, :class:`svm.SVR`, :class:`svm.NuSVR`) can now be +- Random state in LibSVM-based estimators (:class:`svm.SVC`, :class:`svm.NuSVC`, + :class:`svm.OneClassSVM`, :class:`svm.SVR`, :class:`svm.NuSVR`) can now be controlled. This is useful to ensure consistency in the probability estimates for the classifiers trained with ``probability=True``. By `Vlad Niculae`_. @@ -204,10 +208,10 @@ Changelog - Improved documentation on :ref:`multi-class, multi-label and multi-output classification ` by `Yannick Schwartz`_ and `Arnaud Joly`_. -- Better input and error handling in the :mod:`metrics` module by +- Better input and error handling in the :mod:`sklearn.metrics` module by `Arnaud Joly`_ and `Joel Nothman`_. -- Speed optimization of the :mod:`hmm` module by :user:`Mikhail Korobov ` +- Speed optimization of the `hmm` module by :user:`Mikhail Korobov ` - Significant speed improvements for :class:`sklearn.cluster.DBSCAN` by `cleverless `_ @@ -216,7 +220,7 @@ Changelog API changes summary ------------------- -- The :func:`auc_score` was renamed :func:`roc_auc_score`. +- The `auc_score` was renamed :func:`metrics.roc_auc_score`. - Testing scikit-learn with ``sklearn.test()`` is deprecated. Use ``nosetests sklearn`` from the command line. @@ -233,10 +237,9 @@ API changes summary setting the ``return_models`` parameter to ``False``. By `Jaques Grobler`_ and `Alexandre Gramfort`_ -- :class:`grid_search.IterGrid` was renamed to - :class:`grid_search.ParameterGrid`. +- `grid_search.IterGrid` was renamed to `grid_search.ParameterGrid`. -- Fixed bug in :class:`KFold` causing imperfect class balance in some +- Fixed bug in `KFold` causing imperfect class balance in some cases. By `Alexandre Gramfort`_ and Tadej JaneÅž. - :class:`sklearn.neighbors.BallTree` has been refactored, and a @@ -249,8 +252,8 @@ API changes summary By `Jake Vanderplas`_ - Support for scipy.spatial.cKDTree within neighbors queries has been - removed, and the functionality replaced with the new :class:`KDTree` - class. + removed, and the functionality replaced with the new + :class:`sklearn.neighbors.KDTree` class. - :class:`sklearn.neighbors.KernelDensity` has been added, which performs efficient kernel density estimation with a variety of kernels. @@ -264,11 +267,11 @@ API changes summary - ``gcv_mode="auto"`` no longer tries to perform SVD on a densified sparse matrix in :class:`sklearn.linear_model.RidgeCV`. -- Sparse matrix support in :class:`sklearn.decomposition.RandomizedPCA` +- Sparse matrix support in `sklearn.decomposition.RandomizedPCA` is now deprecated in favor of the new ``TruncatedSVD``. -- :class:`cross_validation.KFold` and - :class:`cross_validation.StratifiedKFold` now enforce `n_folds >= 2` +- `cross_validation.KFold` and + `cross_validation.StratifiedKFold` now enforce `n_folds >= 2` otherwise a ``ValueError`` is raised. By `Olivier Grisel`_. - :func:`datasets.load_files`'s ``charset`` and ``charset_errors`` @@ -298,92 +301,91 @@ People ------ List of contributors for release 0.14 by number of commits. - * 277 Gilles Louppe - * 245 Lars Buitinck - * 187 Andreas Mueller - * 124 Arnaud Joly - * 112 Jaques Grobler - * 109 Gael Varoquaux - * 107 Olivier Grisel - * 102 Noel Dawe - * 99 Kemal Eren - * 79 Joel Nothman - * 75 Jake VanderPlas - * 73 Nelle Varoquaux - * 71 Vlad Niculae - * 65 Peter Prettenhofer - * 64 Alexandre Gramfort - * 54 Mathieu Blondel - * 38 Nicolas TrÊsegnie - * 35 eustache - * 27 Denis Engemann - * 25 Yann N. Dauphin - * 19 Justin Vincent - * 17 Robert Layton - * 15 Doug Coleman - * 14 Michael Eickenberg - * 13 Robert Marchman - * 11 Fabian Pedregosa - * 11 Philippe Gervais - * 10 Jim HolmstrÃļm - * 10 Tadej JaneÅž - * 10 syhw - * 9 Mikhail Korobov - * 9 Steven De Gryze - * 8 sergeyf - * 7 Ben Root - * 7 Hrishikesh Huilgolkar - * 6 Kyle Kastner - * 6 Martin Luessi - * 6 Rob Speer - * 5 Federico Vaggi - * 5 Raul Garreta - * 5 Rob Zinkov - * 4 Ken Geis - * 3 A. Flaxman - * 3 Denton Cockburn - * 3 Dougal Sutherland - * 3 Ian Ozsvald - * 3 Johannes SchÃļnberger - * 3 Robert McGibbon - * 3 Roman Sinayev - * 3 Szabo Roland - * 2 Diego Molla - * 2 Imran Haque - * 2 Jochen WersdÃļrfer - * 2 Sergey Karayev - * 2 Yannick Schwartz - * 2 jamestwebber - * 1 Abhijeet Kolhe - * 1 Alexander Fabisch - * 1 Bastiaan van den Berg - * 1 Benjamin Peterson - * 1 Daniel Velkov - * 1 Fazlul Shahriar - * 1 Felix Brockherde - * 1 FÊlix-Antoine Fortin - * 1 Harikrishnan S - * 1 Jack Hale - * 1 JakeMick - * 1 James McDermott - * 1 John Benediktsson - * 1 John Zwinck - * 1 Joshua Vredevoogd - * 1 Justin Pati - * 1 Kevin Hughes - * 1 Kyle Kelley - * 1 Matthias Ekman - * 1 Miroslav Shubernetskiy - * 1 Naoki Orii - * 1 Norbert Crombach - * 1 Rafael Cunha de Almeida - * 1 Rolando Espinoza La fuente - * 1 Seamus Abshere - * 1 Sergey Feldman - * 1 Sergio Medina - * 1 Stefano Lattarini - * 1 Steve Koch - * 1 Sturla Molden - * 1 Thomas Jarosch - * 1 Yaroslav Halchenko - +* 277 Gilles Louppe +* 245 Lars Buitinck +* 187 Andreas Mueller +* 124 Arnaud Joly +* 112 Jaques Grobler +* 109 Gael Varoquaux +* 107 Olivier Grisel +* 102 Noel Dawe +* 99 Kemal Eren +* 79 Joel Nothman +* 75 Jake VanderPlas +* 73 Nelle Varoquaux +* 71 Vlad Niculae +* 65 Peter Prettenhofer +* 64 Alexandre Gramfort +* 54 Mathieu Blondel +* 38 Nicolas TrÊsegnie +* 35 eustache +* 27 Denis Engemann +* 25 Yann N. Dauphin +* 19 Justin Vincent +* 17 Robert Layton +* 15 Doug Coleman +* 14 Michael Eickenberg +* 13 Robert Marchman +* 11 Fabian Pedregosa +* 11 Philippe Gervais +* 10 Jim HolmstrÃļm +* 10 Tadej JaneÅž +* 10 syhw +* 9 Mikhail Korobov +* 9 Steven De Gryze +* 8 sergeyf +* 7 Ben Root +* 7 Hrishikesh Huilgolkar +* 6 Kyle Kastner +* 6 Martin Luessi +* 6 Rob Speer +* 5 Federico Vaggi +* 5 Raul Garreta +* 5 Rob Zinkov +* 4 Ken Geis +* 3 A. Flaxman +* 3 Denton Cockburn +* 3 Dougal Sutherland +* 3 Ian Ozsvald +* 3 Johannes SchÃļnberger +* 3 Robert McGibbon +* 3 Roman Sinayev +* 3 Szabo Roland +* 2 Diego Molla +* 2 Imran Haque +* 2 Jochen WersdÃļrfer +* 2 Sergey Karayev +* 2 Yannick Schwartz +* 2 jamestwebber +* 1 Abhijeet Kolhe +* 1 Alexander Fabisch +* 1 Bastiaan van den Berg +* 1 Benjamin Peterson +* 1 Daniel Velkov +* 1 Fazlul Shahriar +* 1 Felix Brockherde +* 1 FÊlix-Antoine Fortin +* 1 Harikrishnan S +* 1 Jack Hale +* 1 JakeMick +* 1 James McDermott +* 1 John Benediktsson +* 1 John Zwinck +* 1 Joshua Vredevoogd +* 1 Justin Pati +* 1 Kevin Hughes +* 1 Kyle Kelley +* 1 Matthias Ekman +* 1 Miroslav Shubernetskiy +* 1 Naoki Orii +* 1 Norbert Crombach +* 1 Rafael Cunha de Almeida +* 1 Rolando Espinoza La fuente +* 1 Seamus Abshere +* 1 Sergey Feldman +* 1 Sergio Medina +* 1 Stefano Lattarini +* 1 Steve Koch +* 1 Sturla Molden +* 1 Thomas Jarosch +* 1 Yaroslav Halchenko diff --git a/doc/whats_new/v0.15.rst b/doc/whats_new/v0.15.rst index a2eafc63b0617..c98cd07adfffe 100644 --- a/doc/whats_new/v0.15.rst +++ b/doc/whats_new/v0.15.rst @@ -2,6 +2,10 @@ .. currentmodule:: sklearn +============ +Version 0.15 +============ + .. _changes_0_15_2: Version 0.15.2 @@ -42,7 +46,7 @@ Bug fixes - Performance optimization in :class:`isotonic.IsotonicRegression`. By Robert Bradshaw. -- ``nose`` is non-longer a runtime dependency to import ``sklearn``, only for +- ``nose`` is no longer a runtime dependency to import ``sklearn``, only for running the tests. By `Joel Nothman`_. - Many documentation and website fixes by `Joel Nothman`_, `Lars Buitinck`_ @@ -58,9 +62,9 @@ Version 0.15.1 Bug fixes --------- -- Made :func:`cross_validation.cross_val_score` use - :class:`cross_validation.KFold` instead of - :class:`cross_validation.StratifiedKFold` on multi-output classification +- Made `cross_validation.cross_val_score` use + `cross_validation.KFold` instead of + `cross_validation.StratifiedKFold` on multi-output classification problems. By :user:`Nikolay Mayorov `. - Support unseen labels :class:`preprocessing.LabelBinarizer` to restore @@ -74,8 +78,8 @@ Bug fixes in case of ties at the per-class vote level by computing the correct per-class sum of prediction scores. By `Andreas MÃŧller`_. -- Made :func:`cross_validation.cross_val_score` and - :class:`grid_search.GridSearchCV` accept Python lists as input data. +- Made `cross_validation.cross_val_score` and + `grid_search.GridSearchCV` accept Python lists as input data. This is especially useful for cross-validation and model selection of text processing pipelines. By `Andreas MÃŧller`_. @@ -141,7 +145,7 @@ New features - Shorthand constructors :func:`pipeline.make_pipeline` and :func:`pipeline.make_union` were added by `Lars Buitinck`_. -- Shuffle option for :class:`cross_validation.StratifiedKFold`. +- Shuffle option for `cross_validation.StratifiedKFold`. By :user:`Jeffrey Blackburne `. - Incremental learning (``partial_fit``) for Gaussian Naive Bayes by @@ -151,7 +155,7 @@ New features ` By :user:`Danny Sullivan `. -- Added :func:`learning_curve ` utility to +- Added `learning_curve` utility to chart performance with respect to training size. See :ref:`sphx_glr_auto_examples_model_selection_plot_learning_curve.py`. By Alexander Fabisch. @@ -180,8 +184,8 @@ Enhancements - Decision trees can now be fitted on fortran- and c-style arrays, and non-continuous arrays without the need to make a copy. - If the input array has a different dtype than ``np.float32``, a fortran- - style copy will be made since fortran-style memory layout has speed + If the input array has a different dtype than ``np.float32``, a + fortran-style copy will be made since fortran-style memory layout has speed advantages. By `Peter Prettenhofer`_ and `Gilles Louppe`_. - Speed improvement of regression trees by optimizing the @@ -203,16 +207,16 @@ Enhancements threading backend of joblib 0.8 and releasing the GIL in the tree fitting Cython code. By `Olivier Grisel`_ and `Gilles Louppe`_. -- Speed improvement of the :mod:`sklearn.ensemble.gradient_boosting` module. +- Speed improvement of the `sklearn.ensemble.gradient_boosting` module. By `Gilles Louppe`_ and `Peter Prettenhofer`_. -- Various enhancements to the :mod:`sklearn.ensemble.gradient_boosting` +- Various enhancements to the `sklearn.ensemble.gradient_boosting` module: a ``warm_start`` argument to fit additional trees, a ``max_leaf_nodes`` argument to fit GBM style trees, a ``monitor`` fit argument to inspect the estimator during training, and refactoring of the verbose code. By `Peter Prettenhofer`_. -- Faster :class:`sklearn.ensemble.ExtraTrees` by caching feature values. +- Faster `sklearn.ensemble.ExtraTrees` by caching feature values. By `Arnaud Joly`_. - Faster depth-based tree building algorithm such as decision tree, @@ -246,13 +250,13 @@ Enhancements significantly speedup computation by `Denis Engemann`_, and `Alexandre Gramfort`_. -- Changed :class:`cross_validation.StratifiedKFold` to try and +- Changed `cross_validation.StratifiedKFold` to try and preserve as much of the original ordering of samples as possible so as not to hide overfitting on datasets with a non-negligible level of samples dependency. By `Daniel Nouri`_ and `Olivier Grisel`_. -- Add multi-output support to :class:`gaussian_process.GaussianProcess` +- Add multi-output support to :class:`gaussian_process.GaussianProcessRegressor` by John Novak. - Support for precomputed distance matrices in nearest neighbor estimators @@ -282,9 +286,8 @@ Enhancements By `Lars Buitinck`_. - Grid search and cross validation allow NaNs in the input arrays so that - preprocessors such as :class:`preprocessing.Imputer - ` can be trained within the cross validation loop, - avoiding potentially skewed results. + preprocessors such as `preprocessing.Imputer` can be trained within the cross + validation loop, avoiding potentially skewed results. - Ridge regression can now deal with sample weights in feature space (only sample space until then). By :user:`Michael Eickenberg `. @@ -312,7 +315,7 @@ Enhancements Documentation improvements ........................... -- The :ref:`Working With Text Data ` tutorial +- The Working With Text Data tutorial has now been worked in to the main documentation's tutorial section. Includes exercises and skeletons for tutorial presentation. Original tutorial created by several authors including @@ -333,11 +336,11 @@ Bug fixes - Fixed bug in :class:`decomposition.MiniBatchDictionaryLearning` : ``partial_fit`` was not working properly. -- Fixed bug in :class:`linear_model.stochastic_gradient` : +- Fixed bug in `linear_model.stochastic_gradient` : ``l1_ratio`` was used as ``(1.0 - l1_ratio)`` . - Fixed bug in :class:`multiclass.OneVsOneClassifier` with string - labels + labels. - Fixed a bug in :class:`LassoCV ` and :class:`ElasticNetCV `: they would not @@ -353,10 +356,10 @@ Bug fixes By `Olivier Grisel`_. - Raise error in :class:`cluster.FeatureAgglomeration` and - :class:`cluster.WardAgglomeration` when no samples are given, + `cluster.WardAgglomeration` when no samples are given, rather than returning meaningless clustering. -- Fixed bug in :class:`gradient_boosting.GradientBoostingRegressor` with +- Fixed bug in `gradient_boosting.GradientBoostingRegressor` with ``loss='huber'``: ``gamma`` might have not been initialized. - Fixed feature importances as computed with a forest of randomized trees @@ -366,36 +369,36 @@ Bug fixes API changes summary ------------------- -- :mod:`sklearn.hmm` is deprecated. Its removal is planned +- `sklearn.hmm` is deprecated. Its removal is planned for the 0.17 release. -- Use of :class:`covariance.EllipticEnvelop` has now been removed after +- Use of `covariance.EllipticEnvelop` has now been removed after deprecation. Please use :class:`covariance.EllipticEnvelope` instead. -- :class:`cluster.Ward` is deprecated. Use +- `cluster.Ward` is deprecated. Use :class:`cluster.AgglomerativeClustering` instead. -- :class:`cluster.WardClustering` is deprecated. Use +- `cluster.WardClustering` is deprecated. Use - :class:`cluster.AgglomerativeClustering` instead. -- :class:`cross_validation.Bootstrap` is deprecated. - :class:`cross_validation.KFold` or - :class:`cross_validation.ShuffleSplit` are recommended instead. +- `cross_validation.Bootstrap` is deprecated. + `cross_validation.KFold` or + `cross_validation.ShuffleSplit` are recommended instead. - Direct support for the sequence of sequences (or list of lists) multilabel format is deprecated. To convert to and from the supported binary indicator matrix format, use - :class:`MultiLabelBinarizer `. + :class:`preprocessing.MultiLabelBinarizer`. By `Joel Nothman`_. -- Add score method to :class:`PCA ` following the model of +- Add score method to :class:`decomposition.PCA` following the model of probabilistic PCA and deprecate - :class:`ProbabilisticPCA ` model whose + `ProbabilisticPCA` model whose score implementation is not correct. The computation now also exploits the matrix inversion lemma for faster computation. By `Alexandre Gramfort`_. -- The score method of :class:`FactorAnalysis ` +- The score method of :class:`decomposition.FactorAnalysis` now returns the average log-likelihood of the samples. Use score_samples to get log-likelihood of each sample. By `Alexandre Gramfort`_. @@ -410,7 +413,7 @@ API changes summary from version 0.13 in some classifiers. By `Joel Nothman`_. - Fix wrong ``explained_variance_ratio_`` attribute in - :class:`RandomizedPCA `. + `RandomizedPCA`. By `Alexandre Gramfort`_. - Fit alphas for each ``l1_ratio`` instead of ``mean_l1_ratio`` in @@ -445,11 +448,11 @@ API changes summary performance, you should modify the value of ``max_features``. By `Arnaud Joly`_. -- Fix :func:`utils.compute_class_weight` when ``class_weight=="auto"``. +- Fix :func:`utils.class_weight.compute_class_weight` when ``class_weight=="auto"``. Previously it was broken for input of non-integer ``dtype`` and the weighted array that was returned was wrong. By `Manoj Kumar`_. -- Fix :class:`cross_validation.Bootstrap` to return ``ValueError`` +- Fix `cross_validation.Bootstrap` to return ``ValueError`` when ``n_train + n_test > n``. By :user:`Ronald Phlypo `. @@ -458,166 +461,165 @@ People List of contributors for release 0.15 by number of commits. -* 312 Olivier Grisel -* 275 Lars Buitinck -* 221 Gael Varoquaux -* 148 Arnaud Joly -* 134 Johannes SchÃļnberger -* 119 Gilles Louppe -* 113 Joel Nothman -* 111 Alexandre Gramfort -* 95 Jaques Grobler -* 89 Denis Engemann -* 83 Peter Prettenhofer -* 83 Alexander Fabisch -* 62 Mathieu Blondel -* 60 Eustache Diemert -* 60 Nelle Varoquaux -* 49 Michael Bommarito -* 45 Manoj-Kumar-S -* 28 Kyle Kastner -* 26 Andreas Mueller -* 22 Noel Dawe -* 21 Maheshakya Wijewardena -* 21 Brooke Osborn -* 21 Hamzeh Alsalhi -* 21 Jake VanderPlas -* 21 Philippe Gervais -* 19 Bala Subrahmanyam Varanasi -* 12 Ronald Phlypo -* 10 Mikhail Korobov -* 8 Thomas Unterthiner -* 8 Jeffrey Blackburne -* 8 eltermann -* 8 bwignall -* 7 Ankit Agrawal -* 7 CJ Carey -* 6 Daniel Nouri -* 6 Chen Liu -* 6 Michael Eickenberg -* 6 ugurthemaster -* 5 Aaron Schumacher -* 5 Baptiste Lagarde -* 5 Rajat Khanduja -* 5 Robert McGibbon -* 5 Sergio Pascual -* 4 Alexis Metaireau -* 4 Ignacio Rossi -* 4 Virgile Fritsch -* 4 Sebastian Säger -* 4 Ilambharathi Kanniah -* 4 sdenton4 -* 4 Robert Layton -* 4 Alyssa -* 4 Amos Waterland -* 3 Andrew Tulloch -* 3 murad -* 3 Steven Maude -* 3 Karol Pysniak -* 3 Jacques Kvam -* 3 cgohlke -* 3 cjlin -* 3 Michael Becker -* 3 hamzeh -* 3 Eric Jacobsen -* 3 john collins -* 3 kaushik94 -* 3 Erwin Marsi -* 2 csytracy -* 2 LK -* 2 Vlad Niculae -* 2 Laurent Direr -* 2 Erik Shilts -* 2 Raul Garreta -* 2 Yoshiki VÃĄzquez Baeza -* 2 Yung Siang Liau -* 2 abhishek thakur -* 2 James Yu -* 2 Rohit Sivaprasad -* 2 Roland Szabo -* 2 amormachine -* 2 Alexis Mignon -* 2 Oscar Carlsson -* 2 Nantas Nardelli -* 2 jess010 -* 2 kowalski87 -* 2 Andrew Clegg -* 2 Federico Vaggi -* 2 Simon Frid -* 2 FÊlix-Antoine Fortin -* 1 Ralf Gommers -* 1 t-aft -* 1 Ronan Amicel -* 1 Rupesh Kumar Srivastava -* 1 Ryan Wang -* 1 Samuel Charron -* 1 Samuel St-Jean -* 1 Fabian Pedregosa -* 1 Skipper Seabold -* 1 Stefan Walk -* 1 Stefan van der Walt -* 1 Stephan Hoyer -* 1 Allen Riddell -* 1 Valentin Haenel -* 1 Vijay Ramesh -* 1 Will Myers -* 1 Yaroslav Halchenko -* 1 Yoni Ben-Meshulam -* 1 Yury V. Zaytsev -* 1 adrinjalali -* 1 ai8rahim -* 1 alemagnani -* 1 alex -* 1 benjamin wilson -* 1 chalmerlowe -* 1 dzikie droÅŧdÅŧe -* 1 jamestwebber -* 1 matrixorz -* 1 popo -* 1 samuela -* 1 François Boulogne -* 1 Alexander Measure -* 1 Ethan White -* 1 Guilherme Trein -* 1 Hendrik Heuer -* 1 IvicaJovic -* 1 Jan Hendrik Metzen -* 1 Jean Michel Rouly -* 1 Eduardo AriÃąo de la Rubia -* 1 Jelle Zijlstra -* 1 Eddy L O Jansson -* 1 Denis -* 1 John -* 1 John Schmidt -* 1 Jorge CaÃąardo Alastuey -* 1 Joseph Perla -* 1 Joshua Vredevoogd -* 1 JosÊ Ricardo -* 1 Julien Miotte -* 1 Kemal Eren -* 1 Kenta Sato -* 1 David Cournapeau -* 1 Kyle Kelley -* 1 Daniele Medri -* 1 Laurent Luce -* 1 Laurent Pierron -* 1 Luis Pedro Coelho -* 1 DanielWeitzenfeld -* 1 Craig Thompson -* 1 Chyi-Kwei Yau -* 1 Matthew Brett -* 1 Matthias Feurer -* 1 Max Linke -* 1 Chris Filo Gorgolewski -* 1 Charles Earl -* 1 Michael Hanke -* 1 Michele OrrÚ -* 1 Bryan Lunt -* 1 Brian Kearns -* 1 Paul Butler -* 1 Paweł Mandera -* 1 Peter -* 1 Andrew Ash -* 1 Pietro Zambelli -* 1 staubda - +* 312 Olivier Grisel +* 275 Lars Buitinck +* 221 Gael Varoquaux +* 148 Arnaud Joly +* 134 Johannes SchÃļnberger +* 119 Gilles Louppe +* 113 Joel Nothman +* 111 Alexandre Gramfort +* 95 Jaques Grobler +* 89 Denis Engemann +* 83 Peter Prettenhofer +* 83 Alexander Fabisch +* 62 Mathieu Blondel +* 60 Eustache Diemert +* 60 Nelle Varoquaux +* 49 Michael Bommarito +* 45 Manoj-Kumar-S +* 28 Kyle Kastner +* 26 Andreas Mueller +* 22 Noel Dawe +* 21 Maheshakya Wijewardena +* 21 Brooke Osborn +* 21 Hamzeh Alsalhi +* 21 Jake VanderPlas +* 21 Philippe Gervais +* 19 Bala Subrahmanyam Varanasi +* 12 Ronald Phlypo +* 10 Mikhail Korobov +* 8 Thomas Unterthiner +* 8 Jeffrey Blackburne +* 8 eltermann +* 8 bwignall +* 7 Ankit Agrawal +* 7 CJ Carey +* 6 Daniel Nouri +* 6 Chen Liu +* 6 Michael Eickenberg +* 6 ugurthemaster +* 5 Aaron Schumacher +* 5 Baptiste Lagarde +* 5 Rajat Khanduja +* 5 Robert McGibbon +* 5 Sergio Pascual +* 4 Alexis Metaireau +* 4 Ignacio Rossi +* 4 Virgile Fritsch +* 4 Sebastian Säger +* 4 Ilambharathi Kanniah +* 4 sdenton4 +* 4 Robert Layton +* 4 Alyssa +* 4 Amos Waterland +* 3 Andrew Tulloch +* 3 murad +* 3 Steven Maude +* 3 Karol Pysniak +* 3 Jacques Kvam +* 3 cgohlke +* 3 cjlin +* 3 Michael Becker +* 3 hamzeh +* 3 Eric Jacobsen +* 3 john collins +* 3 kaushik94 +* 3 Erwin Marsi +* 2 csytracy +* 2 LK +* 2 Vlad Niculae +* 2 Laurent Direr +* 2 Erik Shilts +* 2 Raul Garreta +* 2 Yoshiki VÃĄzquez Baeza +* 2 Yung Siang Liau +* 2 abhishek thakur +* 2 James Yu +* 2 Rohit Sivaprasad +* 2 Roland Szabo +* 2 amormachine +* 2 Alexis Mignon +* 2 Oscar Carlsson +* 2 Nantas Nardelli +* 2 jess010 +* 2 kowalski87 +* 2 Andrew Clegg +* 2 Federico Vaggi +* 2 Simon Frid +* 2 FÊlix-Antoine Fortin +* 1 Ralf Gommers +* 1 t-aft +* 1 Ronan Amicel +* 1 Rupesh Kumar Srivastava +* 1 Ryan Wang +* 1 Samuel Charron +* 1 Samuel St-Jean +* 1 Fabian Pedregosa +* 1 Skipper Seabold +* 1 Stefan Walk +* 1 Stefan van der Walt +* 1 Stephan Hoyer +* 1 Allen Riddell +* 1 Valentin Haenel +* 1 Vijay Ramesh +* 1 Will Myers +* 1 Yaroslav Halchenko +* 1 Yoni Ben-Meshulam +* 1 Yury V. Zaytsev +* 1 adrinjalali +* 1 ai8rahim +* 1 alemagnani +* 1 alex +* 1 benjamin wilson +* 1 chalmerlowe +* 1 dzikie droÅŧdÅŧe +* 1 jamestwebber +* 1 matrixorz +* 1 popo +* 1 samuela +* 1 François Boulogne +* 1 Alexander Measure +* 1 Ethan White +* 1 Guilherme Trein +* 1 Hendrik Heuer +* 1 IvicaJovic +* 1 Jan Hendrik Metzen +* 1 Jean Michel Rouly +* 1 Eduardo AriÃąo de la Rubia +* 1 Jelle Zijlstra +* 1 Eddy L O Jansson +* 1 Denis +* 1 John +* 1 John Schmidt +* 1 Jorge CaÃąardo Alastuey +* 1 Joseph Perla +* 1 Joshua Vredevoogd +* 1 JosÊ Ricardo +* 1 Julien Miotte +* 1 Kemal Eren +* 1 Kenta Sato +* 1 David Cournapeau +* 1 Kyle Kelley +* 1 Daniele Medri +* 1 Laurent Luce +* 1 Laurent Pierron +* 1 Luis Pedro Coelho +* 1 DanielWeitzenfeld +* 1 Craig Thompson +* 1 Chyi-Kwei Yau +* 1 Matthew Brett +* 1 Matthias Feurer +* 1 Max Linke +* 1 Chris Filo Gorgolewski +* 1 Charles Earl +* 1 Michael Hanke +* 1 Michele OrrÚ +* 1 Bryan Lunt +* 1 Brian Kearns +* 1 Paul Butler +* 1 Paweł Mandera +* 1 Peter +* 1 Andrew Ash +* 1 Pietro Zambelli +* 1 staubda diff --git a/doc/whats_new/v0.16.rst b/doc/whats_new/v0.16.rst index 931c7e0fbb923..b5656d3bff64c 100644 --- a/doc/whats_new/v0.16.rst +++ b/doc/whats_new/v0.16.rst @@ -2,6 +2,10 @@ .. currentmodule:: sklearn +============ +Version 0.16 +============ + .. _changes_0_16_1: Version 0.16.1 @@ -22,7 +26,7 @@ Bug fixes caused unstable result in :class:`calibration.CalibratedClassifierCV` by `Jan Hendrik Metzen`_. -- Fix sorting of labels in func:`preprocessing.label_binarize` by Michael Heilman. +- Fix sorting of labels in :func:`preprocessing.label_binarize` by Michael Heilman. - Fix several stability and convergence issues in :class:`cross_decomposition.CCA` and @@ -54,13 +58,13 @@ Highlights - Out-of core learning of PCA via :class:`decomposition.IncrementalPCA`. -- Probability callibration of classifiers using +- Probability calibration of classifiers using :class:`calibration.CalibratedClassifierCV`. - :class:`cluster.Birch` clustering method for large-scale datasets. - Scalable approximate nearest neighbors search with Locality-sensitive - hashing forests in :class:`neighbors.LSHForest`. + hashing forests in `neighbors.LSHForest`. - Improved error messages and better validation when using malformed input data. @@ -72,7 +76,7 @@ Changelog New features ............ -- The new :class:`neighbors.LSHForest` implements locality-sensitive hashing +- The new `neighbors.LSHForest` implements locality-sensitive hashing for approximate nearest neighbors search. By :user:`Maheshakya Wijewardena`. - Added :class:`svm.LinearSVR`. This class uses the liblinear implementation @@ -109,7 +113,7 @@ New features and :class:`SGDRegressor ` By :user:`Danny Sullivan `. -- Added :func:`cross_val_predict ` +- Added `cross_val_predict` function which computes cross-validated estimates. By `Luis Pedro Coelho`_ - Added :class:`linear_model.TheilSenRegressor`, a robust @@ -131,7 +135,7 @@ New features - All solvers in :class:`linear_model.Ridge` now support `sample_weight`. By `Mathieu Blondel`_. -- Added :class:`cross_validation.PredefinedSplit` cross-validation +- Added `cross_validation.PredefinedSplit` cross-validation for fixed user-provided cross-validation folds. By :user:`Thomas Unterthiner `. @@ -144,10 +148,10 @@ New features Enhancements ............ -- Add option ``return_distance`` in :func:`hierarchical.ward_tree` +- Add option ``return_distance`` in `hierarchical.ward_tree` to return distances between nodes for both structured and unstructured versions of the algorithm. By `Matteo Visconti di Oleggio Castello`_. - The same option was added in :func:`hierarchical.linkage_tree`. + The same option was added in `hierarchical.linkage_tree`. By `Manoj Kumar`_ - Add support for sample weights in scorer objects. Metrics with sample @@ -162,7 +166,7 @@ Enhancements and related. By `Manoj Kumar`_. - Add ``sample_weight`` parameter to - :func:`metrics.jaccard_similarity_score` and :func:`metrics.log_loss`. + `metrics.jaccard_similarity_score` and :func:`metrics.log_loss`. By :user:`Jatin Shah `. - Support sparse multilabel indicator representation in @@ -191,11 +195,11 @@ Enhancements single pass, when giving the option ``sort=False``. By :user:`Dan Blanchard `. -- :class:`GridSearchCV` and :class:`RandomizedSearchCV` can now be - configured to work with estimators that may fail and raise errors on - individual folds. This option is controlled by the `error_score` - parameter. This does not affect errors raised on re-fit. By - :user:`Michal Romaniuk `. +- :class:`model_selection.GridSearchCV` and + :class:`model_selection.RandomizedSearchCV` can now be configured to work + with estimators that may fail and raise errors on individual folds. This + option is controlled by the `error_score` parameter. This does not affect + errors raised on re-fit. By :user:`Michal Romaniuk `. - Add ``digits`` parameter to `metrics.classification_report` to allow report to show different precision of floating point numbers. By @@ -214,7 +218,7 @@ Enhancements - Optimized :class:`cluster.AffinityPropagation` by reducing the number of memory allocations of large temporary data-structures. By `Antony Lee`_. -- Parellization of the computation of feature importances in random forest. +- Parallelization of the computation of feature importances in random forest. By `Olivier Grisel`_ and `Arnaud Joly`_. - Add ``n_iter_`` attribute to estimators that accept a ``max_iter`` attribute @@ -223,14 +227,14 @@ Enhancements - Added decision function for :class:`multiclass.OneVsOneClassifier` By `Raghav RV`_ and :user:`Kyle Beauchamp `. -- :func:`neighbors.kneighbors_graph` and :func:`radius_neighbors_graph` +- `neighbors.kneighbors_graph` and `radius_neighbors_graph` support non-Euclidean metrics. By `Manoj Kumar`_ - Parameter ``connectivity`` in :class:`cluster.AgglomerativeClustering` and family now accept callables that return a connectivity matrix. By `Manoj Kumar`_. -- Sparse support for :func:`paired_distances`. By `Joel Nothman`_. +- Sparse support for :func:`metrics.pairwise.paired_distances`. By `Joel Nothman`_. - :class:`cluster.DBSCAN` now supports sparse input and sample weights and has been optimized: the inner loop has been rewritten in Cython and @@ -242,10 +246,10 @@ Enhancements :class:`tree.DecisionTreeClassifier`, :class:`ensemble.ExtraTreesClassifier` and :class:`tree.ExtraTreeClassifier`. By `Trevor Stephens`_. -- :class:`grid_search.RandomizedSearchCV` now does sampling without +- `grid_search.RandomizedSearchCV` now does sampling without replacement if all parameters are given as lists. By `Andreas MÃŧller`_. -- Parallelized calculation of :func:`pairwise_distances` is now supported +- Parallelized calculation of :func:`metrics.pairwise_distances` is now supported for scipy metrics and custom callables. By `Joel Nothman`_. - Allow the fitting and scoring of all clustering algorithms in @@ -254,8 +258,8 @@ Enhancements - More robust seeding and improved error messages in :class:`cluster.MeanShift` by `Andreas MÃŧller`_. -- Make the stopping criterion for :class:`mixture.GMM`, - :class:`mixture.DPGMM` and :class:`mixture.VBGMM` less dependent on the +- Make the stopping criterion for `mixture.GMM`, + `mixture.DPGMM` and `mixture.VBGMM` less dependent on the number of samples by thresholding the average log-likelihood change instead of its sum over all samples. By `HervÊ Bredin`_. @@ -271,14 +275,14 @@ Enhancements - :class:`svm.SVC` fitted on sparse input now implements ``decision_function``. By `Rob Zinkov`_ and `Andreas MÃŧller`_. -- :func:`cross_validation.train_test_split` now preserves the input type, +- `cross_validation.train_test_split` now preserves the input type, instead of converting to numpy arrays. Documentation improvements .......................... -- Added example of using :class:`FeatureUnion` for heterogeneous input. +- Added example of using :class:`pipeline.FeatureUnion` for heterogeneous input. By :user:`Matt Terry ` - Documentation on scorers was improved, to highlight the handling of loss @@ -306,16 +310,16 @@ Bug fixes ......... - Metaestimators now support ducktyping for the presence of ``decision_function``, ``predict_proba`` and other methods. This fixes behavior of - :class:`grid_search.GridSearchCV`, - :class:`grid_search.RandomizedSearchCV`, :class:`pipeline.Pipeline`, + `grid_search.GridSearchCV`, + `grid_search.RandomizedSearchCV`, :class:`pipeline.Pipeline`, :class:`feature_selection.RFE`, :class:`feature_selection.RFECV` when nested. By `Joel Nothman`_ - The ``scoring`` attribute of grid-search and cross-validation methods is no longer - ignored when a :class:`grid_search.GridSearchCV` is given as a base estimator or + ignored when a `grid_search.GridSearchCV` is given as a base estimator or the base estimator doesn't have predict. -- The function :func:`hierarchical.ward_tree` now returns the children in +- The function `hierarchical.ward_tree` now returns the children in the same order for both the structured and unstructured versions. By `Matteo Visconti di Oleggio Castello`_. @@ -327,13 +331,13 @@ Bug fixes length. By :user:`Michael Eickenberg `. - Fix incomplete download of the dataset when - :func:`datasets.download_20newsgroups` is called. By `Manoj Kumar`_. + `datasets.download_20newsgroups` is called. By `Manoj Kumar`_. - Various fixes to the Gaussian processes subpackage by Vincent Dubourg and Jan Hendrik Metzen. - Calling ``partial_fit`` with ``class_weight=='auto'`` throws an - appropriate error message and suggests a work around. + appropriate error message and suggests a workaround. By :user:`Danny Sullivan `. - :class:`RBFSampler ` with ``gamma=g`` @@ -343,7 +347,7 @@ Bug fixes (If you cross-validated over ``gamma``, it probably doesn't matter too much.) By :user:`Dougal Sutherland `. -- Pipeline object delegate the ``classes_`` attribute to the underlying +- Pipeline object delegates the ``classes_`` attribute to the underlying estimator. It allows, for instance, to make bagging of a pipeline object. By `Arnaud Joly`_ @@ -359,7 +363,7 @@ Bug fixes - When `compute_full_tree` is set to "auto", the full tree is built when n_clusters is high and is early stopped when n_clusters is - low, while the behavior should be vice-versa in + low, while the behavior should be vice versa in :class:`cluster.AgglomerativeClustering` (and friends). This has been fixed By `Manoj Kumar`_ @@ -384,7 +388,7 @@ Bug fixes :class:`sklearn.neighbors.NearestNeighbors` and family, when the query data is not the same as fit data. By `Manoj Kumar`_. -- Fix log-density calculation in the :class:`mixture.GMM` with +- Fix log-density calculation in the `mixture.GMM` with tied covariance. By `Will Dawson`_ - Fixed a scaling error in :class:`feature_selection.SelectFdr` @@ -397,7 +401,7 @@ Bug fixes - Fixed round off errors with non positive-definite covariance matrices in GMM. By :user:`Alexis Mignon `. -- Fixed a error in the computation of conditional probabilities in +- Fixed an error in the computation of conditional probabilities in :class:`naive_bayes.BernoulliNB`. By `Hanna Wallach`_. - Make the method ``radius_neighbors`` of @@ -415,15 +419,15 @@ Bug fixes API changes summary ------------------- -- :class:`GridSearchCV ` and - :func:`cross_val_score ` and other +- `GridSearchCV` and + `cross_val_score` and other meta-estimators don't convert pandas DataFrames into arrays any more, allowing DataFrame specific operations in custom estimators. -- :func:`multiclass.fit_ovr`, :func:`multiclass.predict_ovr`, - :func:`predict_proba_ovr`, - :func:`multiclass.fit_ovo`, :func:`multiclass.predict_ovo`, - :func:`multiclass.fit_ecoc` and :func:`multiclass.predict_ecoc` +- `multiclass.fit_ovr`, `multiclass.predict_ovr`, + `predict_proba_ovr`, + `multiclass.fit_ovo`, `multiclass.predict_ovo`, + `multiclass.fit_ecoc` and `multiclass.predict_ecoc` are deprecated. Use the underlying estimators instead. - Nearest neighbors estimators used to take arbitrary keyword arguments @@ -439,11 +443,11 @@ API changes summary but previous versions accidentally returned only the positive probability. Fixed by Will Lamond and `Lars Buitinck`_. -- Change default value of precompute in :class:`ElasticNet` and :class:`Lasso` - to False. Setting precompute to "auto" was found to be slower when - n_samples > n_features since the computation of the Gram matrix is - computationally expensive and outweighs the benefit of fitting the Gram - for just one alpha. +- Change default value of precompute in :class:`linear_model.ElasticNet` and + :class:`linear_model.Lasso` to False. Setting precompute to "auto" was found + to be slower when n_samples > n_features since the computation of the Gram + matrix is computationally expensive and outweighs the benefit of fitting the + Gram for just one alpha. ``precompute="auto"`` is now deprecated and will be removed in 0.18 By `Manoj Kumar`_. @@ -467,8 +471,8 @@ API changes summary been removed. They were deprecated since 0.14 - From now onwards, all estimators will uniformly raise ``NotFittedError`` - (:class:`utils.validation.NotFittedError`), when any of the ``predict`` - like methods are called before the model is fit. By `Raghav RV`_. + when any of the ``predict`` like methods are called before the model is fit. + By `Raghav RV`_. - Input data validation was refactored for more consistent input validation. The ``check_arrays`` function was replaced by ``check_array`` @@ -486,7 +490,7 @@ API changes summary as the first nearest neighbor. - `thresh` parameter is deprecated in favor of new `tol` parameter in - :class:`GMM`, :class:`DPGMM` and :class:`VBGMM`. See `Enhancements` + `GMM`, `DPGMM` and `VBGMM`. See `Enhancements` section for details. By `HervÊ Bredin`_. - Estimators will treat input with dtype object as numeric when possible. @@ -538,4 +542,3 @@ terrycojones, Thomas Delteil, Thomas Unterthiner, Tomas Kazmar, trevorstephens, tttthomasssss, Tzu-Ming Kuo, ugurcaliskan, ugurthemaster, Vinayak Mehta, Vincent Dubourg, Vjacheslav Murashkin, Vlad Niculae, wadawson, Wei Xue, Will Lamond, Wu Jiang, x0l, Xinfan Meng, Yan Yi, Yu-Chin - diff --git a/doc/whats_new/v0.17.rst b/doc/whats_new/v0.17.rst index 7657d07712ab5..33e5ab9baf123 100644 --- a/doc/whats_new/v0.17.rst +++ b/doc/whats_new/v0.17.rst @@ -2,6 +2,10 @@ .. currentmodule:: sklearn +============ +Version 0.17 +============ + .. _changes_0_17_1: Version 0.17.1 @@ -75,10 +79,10 @@ New features function into a ``Pipeline``-compatible transformer object. By Joe Jevnik. -- The new classes :class:`cross_validation.LabelKFold` and - :class:`cross_validation.LabelShuffleSplit` generate train-test folds, - respectively similar to :class:`cross_validation.KFold` and - :class:`cross_validation.ShuffleSplit`, except that the folds are +- The new classes `cross_validation.LabelKFold` and + `cross_validation.LabelShuffleSplit` generate train-test folds, + respectively similar to `cross_validation.KFold` and + `cross_validation.ShuffleSplit`, except that the folds are conditioned on a label array. By `Brian McFee`_, :user:`Jean Kossaifi ` and `Gilles Louppe`_. @@ -97,7 +101,7 @@ New features :class:`decomposition.NMF`. Previous solver based on Projected Gradient is still available setting new parameter ``solver`` to ``pg``, but is deprecated and will be removed in 0.19, along with - :class:`decomposition.ProjectedGradientNMF` and parameters ``sparseness``, + `decomposition.ProjectedGradientNMF` and parameters ``sparseness``, ``eta``, ``beta`` and ``nls_max_iter``. New parameters ``alpha`` and ``l1_ratio`` control L1 and L2 regularization, and ``shuffle`` adds a shuffling step in the ``cd`` solver. @@ -109,7 +113,7 @@ Enhancements Barnes-Hut method, leading to much faster fitting. By Christopher Erick Moody. (:issue:`4025`) -- :class:`cluster.mean_shift_.MeanShift` now supports parallel execution, +- :class:`cluster.MeanShift` now supports parallel execution, as implemented in the ``mean_shift`` function. By :user:`Martino Sorbaro `. @@ -119,7 +123,7 @@ Enhancements - :class:`dummy.DummyClassifier` now supports a prior fitting strategy. By `Arnaud Joly`_. -- Added a ``fit_predict`` method for :class:`mixture.GMM` and subclasses. +- Added a ``fit_predict`` method for `mixture.GMM` and subclasses. By :user:`Cory Lorenz `. - Added the :func:`metrics.label_ranking_loss` metric. @@ -133,7 +137,7 @@ Enhancements - Added option to use multi-output regression metrics without averaging. By Konstantin Shmelkov and :user:`Michael Eickenberg`. -- Added ``stratify`` option to :func:`cross_validation.train_test_split` +- Added ``stratify`` option to `cross_validation.train_test_split` for stratified splitting. By Miroslav Batchkarov. - The :func:`tree.export_graphviz` function now supports aesthetic @@ -172,8 +176,8 @@ Enhancements :func:`sklearn.metrics.pairwise.cosine_similarity`. By :user:`Jaidev Deshpande `. -- Add :func:`minmax_scale` to provide a function interface for - :class:`MinMaxScaler`. By :user:`Thomas Unterthiner `. +- Add :func:`preprocessing.minmax_scale` to provide a function interface for + :class:`preprocessing.MinMaxScaler`. By :user:`Thomas Unterthiner `. - ``dump_svmlight_file`` now handles multi-label datasets. By Chih-Wei Chang. @@ -183,12 +187,12 @@ Enhancements - The "Wisconsin Breast Cancer" classical two-class classification dataset is now included in scikit-learn, available with - :func:`sklearn.dataset.load_breast_cancer`. + :func:`datasets.load_breast_cancer`. - Upgraded to joblib 0.9.3 to benefit from the new automatic batching of short tasks. This makes it possible for scikit-learn to benefit from parallelism when many very short tasks are executed in parallel, for - instance by the :class:`grid_search.GridSearchCV` meta-estimator + instance by the `grid_search.GridSearchCV` meta-estimator with ``n_jobs > 1`` used with a large grid of parameters on a small dataset. By `Vlad Niculae`_, `Olivier Grisel`_ and `Loic Esteve`_. @@ -196,7 +200,7 @@ Enhancements https://github.com/joblib/joblib/blob/master/CHANGES.rst#release-093 - Improved speed (3 times per iteration) of - :class:`decomposition.DictLearning` with coordinate descent method + `decomposition.DictLearning` with coordinate descent method from :class:`linear_model.Lasso`. By :user:`Arthur Mensch `. - Parallel processing (threaded) for queries of nearest neighbors @@ -264,7 +268,7 @@ Enhancements - Added :func:`metrics.pairwise.laplacian_kernel`. By `Clyde Fare `_. -- :class:`covariance.GraphLasso` allows separate control of the convergence criterion +- `covariance.GraphLasso` allows separate control of the convergence criterion for the Elastic-Net subproblem via the ``enet_tol`` parameter. - Improved verbosity in :class:`decomposition.DictionaryLearning`. @@ -283,7 +287,7 @@ Enhancements - Added the ``fit_predict`` method to :class:`pipeline.Pipeline`. -- Added the :func:`preprocessing.min_max_scale` function. +- Added the :func:`preprocessing.minmax_scale` function. Bug fixes ......... @@ -294,16 +298,16 @@ Bug fixes - Fixed the output shape of :class:`linear_model.RANSACRegressor` to ``(n_samples, )``. By `Andreas MÃŧller`_. -- Fixed bug in :class:`decomposition.DictLearning` when ``n_jobs < 0``. By +- Fixed bug in `decomposition.DictLearning` when ``n_jobs < 0``. By `Andreas MÃŧller`_. -- Fixed bug where :class:`grid_search.RandomizedSearchCV` could consume a +- Fixed bug where `grid_search.RandomizedSearchCV` could consume a lot of memory for large discrete grids. By `Joel Nothman`_. - Fixed bug in :class:`linear_model.LogisticRegressionCV` where `penalty` was ignored in the final fit. By `Manoj Kumar`_. -- Fixed bug in :class:`ensemble.forest.ForestClassifier` while computing +- Fixed bug in `ensemble.forest.ForestClassifier` while computing oob_score and X is a sparse.csc_matrix. By :user:`Ankur Ankan `. - All regressors now consistently handle and warn when given ``y`` that is of @@ -313,17 +317,18 @@ Bug fixes - Fix in :class:`cluster.KMeans` cluster reassignment for sparse input by `Lars Buitinck`_. -- Fixed a bug in :class:`lda.LDA` that could cause asymmetric covariance - matrices when using shrinkage. By `Martin Billinger`_. +- Fixed a bug in :class:`discriminant_analysis.LinearDiscriminantAnalysis` that + could cause asymmetric covariance matrices when using shrinkage. By `Martin + Billinger`_. -- Fixed :func:`cross_validation.cross_val_predict` for estimators with +- Fixed `cross_validation.cross_val_predict` for estimators with sparse predictions. By Buddha Prakash. - Fixed the ``predict_proba`` method of :class:`linear_model.LogisticRegression` to use soft-max instead of one-vs-rest normalization. By `Manoj Kumar`_. (:issue:`5182`) -- Fixed the :func:`partial_fit` method of :class:`linear_model.SGDClassifier` +- Fixed the `partial_fit` method of :class:`linear_model.SGDClassifier` when called with ``average=True``. By :user:`Andrew Lamb `. (:issue:`5282`) @@ -339,17 +344,17 @@ Bug fixes automatically changes the solver to 'sag' in this case. :issue:`5360` by `Tom Dupre la Tour`_. -- Fixed a performance bug in :class:`decomposition.RandomizedPCA` on data +- Fixed a performance bug in `decomposition.RandomizedPCA` on data with a large number of features and fewer samples. (:issue:`4478`) By `Andreas MÃŧller`_, `Loic Esteve`_ and :user:`Giorgio Patrini `. -- Fixed bug in :class:`cross_decomposition.PLS` that yielded unstable and +- Fixed bug in `cross_decomposition.PLS` that yielded unstable and platform dependent output, and failed on `fit_transform`. By :user:`Arthur Mensch `. - Fixes to the ``Bunch`` class used to store datasets. -- Fixed :func:`ensemble.plot_partial_dependence` ignoring the +- Fixed `ensemble.plot_partial_dependence` ignoring the ``percentiles`` parameter. - Providing a ``set`` as vocabulary in ``CountVectorizer`` no longer @@ -361,8 +366,8 @@ Bug fixes :class:`linear_model.Lasso` and :class:`linear_model.ElasticNet`. - Fixed inconsistent memory layout in the coordinate descent solver - that affected :class:`linear_model.DictionaryLearning` and - :class:`covariance.GraphLasso`. (:issue:`5337`) + that affected `linear_model.DictionaryLearning` and + `covariance.GraphLasso`. (:issue:`5337`) By `Olivier Grisel`_. - :class:`manifold.LocallyLinearEmbedding` no longer ignores the ``reg`` @@ -396,7 +401,7 @@ API changes summary in :class:`preprocessing.StandardScaler` is deprecated and superseded by `scale_`; it won't be available in 0.19. By :user:`Giorgio Patrini `. -- :class:`svm.SVC`` and :class:`svm.NuSVC` now have an ``decision_function_shape`` +- :class:`svm.SVC` and :class:`svm.NuSVC` now have an ``decision_function_shape`` parameter to make their decision function of shape ``(n_samples, n_classes)`` by setting ``decision_function_shape='ovr'``. This will be the default behavior starting in 0.19. By `Andreas MÃŧller`_. @@ -407,7 +412,7 @@ API changes summary to be explicitly shaped ``(n_samples, n_features)``. By :user:`Vighnesh Birodkar `. -- :class:`lda.LDA` and :class:`qda.QDA` have been moved to +- `lda.LDA` and `qda.QDA` have been moved to :class:`discriminant_analysis.LinearDiscriminantAnalysis` and :class:`discriminant_analysis.QuadraticDiscriminantAnalysis`. @@ -438,7 +443,7 @@ API changes summary - The ``decision_function`` on all regressors was deprecated and will be removed in 0.19. Use ``predict`` instead. -- :func:`datasets.load_lfw_pairs` is deprecated and will be removed in 0.19. +- `datasets.load_lfw_pairs` is deprecated and will be removed in 0.19. Use :func:`datasets.fetch_lfw_pairs` instead. - The deprecated ``hmm`` module was removed. @@ -446,9 +451,9 @@ API changes summary - The deprecated ``Bootstrap`` cross-validation iterator was removed. - The deprecated ``Ward`` and ``WardAgglomerative`` classes have been removed. - Use :class:`clustering.AgglomerativeClustering` instead. + Use :class:`cluster.AgglomerativeClustering` instead. -- :func:`cross_validation.check_cv` is now a public function. +- `cross_validation.check_cv` is now a public function. - The property ``residues_`` of :class:`linear_model.LinearRegression` is deprecated and will be removed in 0.19. diff --git a/doc/whats_new/v0.18.rst b/doc/whats_new/v0.18.rst index ea3548c0b9a0c..23de532c917f2 100644 --- a/doc/whats_new/v0.18.rst +++ b/doc/whats_new/v0.18.rst @@ -2,6 +2,16 @@ .. currentmodule:: sklearn +============ +Version 0.18 +============ + +.. warning:: + + Scikit-learn 0.18 is the last major release of scikit-learn to support Python 2.6. + Later versions of scikit-learn will require Python 2.7 or above. + + .. _changes_0_18_2: Version 0.18.2 @@ -9,12 +19,6 @@ Version 0.18.2 **June 20, 2017** -.. topic:: Last release with Python 2.6 support - - Scikit-learn 0.18 is the last major release of scikit-learn to support Python 2.6. - Later versions of scikit-learn will require Python 2.7 or above. - - Changelog --------- @@ -176,11 +180,6 @@ Version 0.18 **September 28, 2016** -.. topic:: Last release with Python 2.6 support - - Scikit-learn 0.18 will be the last version of scikit-learn to support Python 2.6. - Later versions of scikit-learn will require Python 2.7 or above. - .. _model_selection_changes: Model Selection Enhancements and API Changes @@ -189,8 +188,8 @@ Model Selection Enhancements and API Changes - **The model_selection module** The new module :mod:`sklearn.model_selection`, which groups together the - functionalities of formerly :mod:`sklearn.cross_validation`, - :mod:`sklearn.grid_search` and :mod:`sklearn.learning_curve`, introduces new + functionalities of formerly `sklearn.cross_validation`, + `sklearn.grid_search` and `sklearn.learning_curve`, introduces new possibilities such as nested cross-validation and better manipulation of parameter searches with Pandas. @@ -202,7 +201,7 @@ Model Selection Enhancements and API Changes The new cross-validation splitters, defined in the :mod:`sklearn.model_selection`, are no longer initialized with any data-dependent parameters such as ``y``. Instead they expose a - :func:`split` method that takes in the data and yields a generator for the + `split` method that takes in the data and yields a generator for the different splits. This change makes it possible to use the cross-validation splitters to @@ -258,7 +257,7 @@ Model Selection Enhancements and API Changes - **Fit parameter labels renamed to groups** - The ``labels`` parameter in the :func:`split` method of the newly renamed + The ``labels`` parameter in the `split` method of the newly renamed splitters :class:`model_selection.GroupKFold`, :class:`model_selection.LeaveOneGroupOut`, :class:`model_selection.LeavePGroupsOut`, @@ -314,7 +313,7 @@ Other estimators for sounder results. :issue:`7295` by :user:`Wei Xue ` and :user:`Thierry Guillemot `. -- Class :class:`decomposition.RandomizedPCA` is now factored into :class:`decomposition.PCA` +- Class `decomposition.RandomizedPCA` is now factored into :class:`decomposition.PCA` and it is available calling with parameter ``svd_solver='randomized'``. The default number of ``n_iter`` for ``'randomized'`` has changed to 4. The old behavior of PCA is recovered by ``svd_solver='full'``. An additional solver @@ -337,11 +336,11 @@ Other estimators Model selection and evaluation -- Added :func:`metrics.cluster.fowlkes_mallows_score`, the Fowlkes Mallows +- Added :func:`metrics.fowlkes_mallows_score`, the Fowlkes Mallows Index which measures the similarity of two clusterings of a set of points By :user:`Arnaud Fouchet ` and :user:`Thierry Guillemot `. -- Added :func:`metrics.calinski_harabaz_score`, which computes the Calinski +- Added `metrics.calinski_harabaz_score`, which computes the Calinski and Harabaz score to evaluate the resulting clustering of a set of points. By :user:`Arnaud Fouchet ` and :user:`Thierry Guillemot `. @@ -384,7 +383,7 @@ Trees and ensembles :issue:`6667` by :user:`Nelson Liu `. - The memory footprint is reduced (sometimes greatly) for - :class:`ensemble.bagging.BaseBagging` and classes that inherit from it, + `ensemble.bagging.BaseBagging` and classes that inherit from it, i.e, :class:`ensemble.BaggingClassifier`, :class:`ensemble.BaggingRegressor`, and :class:`ensemble.IsolationForest`, by dynamically generating attribute ``estimators_samples_`` only when it is @@ -462,7 +461,7 @@ Model evaluation and meta-estimators - Added support for substituting or disabling :class:`pipeline.Pipeline` and :class:`pipeline.FeatureUnion` components using the ``set_params`` - interface that powers :mod:`sklearn.grid_search`. + interface that powers `sklearn.grid_search`. See :ref:`sphx_glr_auto_examples_compose_plot_compare_reduction.py` By `Joel Nothman`_ and :user:`Robert McGibbon `. @@ -489,7 +488,7 @@ Metrics :user:`Mads Jensen ` and :user:`Nelson Liu `. - Support sparse contingency matrices in cluster evaluation - (:mod:`metrics.cluster.supervised`) to scale to a large number of + (`metrics.cluster.supervised`) to scale to a large number of clusters. :issue:`7419` by :user:`Gregory Stupp ` and `Joel Nothman`_. @@ -512,22 +511,22 @@ Miscellaneous C/C++ files. By :user:`Arthur Mensch `. - Reduce the memory usage for 32-bit float input arrays of - :func:`utils.sparse_func.mean_variance_axis` and - :func:`utils.sparse_func.incr_mean_variance_axis` by supporting cython + `utils.sparse_func.mean_variance_axis` and + `utils.sparse_func.incr_mean_variance_axis` by supporting cython fused types. By :user:`YenChen Lin `. -- The :func:`ignore_warnings` now accept a category argument to ignore only +- The `ignore_warnings` now accept a category argument to ignore only the warnings of a specified type. By :user:`Thierry Guillemot `. - Added parameter ``return_X_y`` and return type ``(data, target) : tuple`` option to - :func:`load_iris` dataset + :func:`datasets.load_iris` dataset :issue:`7049`, - :func:`load_breast_cancer` dataset + :func:`datasets.load_breast_cancer` dataset :issue:`7152`, - :func:`load_digits` dataset, - :func:`load_diabetes` dataset, - :func:`load_linnerud` dataset, - :func:`load_boston` dataset + :func:`datasets.load_digits` dataset, + :func:`datasets.load_diabetes` dataset, + :func:`datasets.load_linnerud` dataset, + `datasets.load_boston` dataset :issue:`7154` by :user:`Manvendra Singh`. @@ -584,26 +583,26 @@ Linear, kernelized and related models Decomposition, manifold learning and clustering -- :class:`decomposition.RandomizedPCA` default number of `iterated_power` is 4 instead of 3. +- `decomposition.RandomizedPCA` default number of `iterated_power` is 4 instead of 3. :issue:`5141` by :user:`Giorgio Patrini `. -- :func:`utils.extmath.randomized_svd` performs 4 power iterations by default, instead or 0. - In practice this is enough for obtaining a good approximation of the +- :func:`utils.extmath.randomized_svd` performs 4 power iterations by default, instead + of 0. In practice this is enough for obtaining a good approximation of the true eigenvalues/vectors in the presence of noise. When `n_components` is small (``< .1 * min(X.shape)``) `n_iter` is set to 7, unless the user specifies a higher number. This improves precision with few components. :issue:`5299` by :user:`Giorgio Patrini`. - Whiten/non-whiten inconsistency between components of :class:`decomposition.PCA` - and :class:`decomposition.RandomizedPCA` (now factored into PCA, see the + and `decomposition.RandomizedPCA` (now factored into PCA, see the New features) is fixed. `components_` are stored with no whitening. :issue:`5299` by :user:`Giorgio Patrini `. - Fixed bug in :func:`manifold.spectral_embedding` where diagonal of unnormalized Laplacian matrix was incorrectly set to 1. :issue:`4995` by :user:`Peter Fischer `. -- Fixed incorrect initialization of :func:`utils.arpack.eigsh` on all - occurrences. Affects :class:`cluster.bicluster.SpectralBiclustering`, +- Fixed incorrect initialization of `utils.arpack.eigsh` on all + occurrences. Affects `cluster.bicluster.SpectralBiclustering`, :class:`decomposition.KernelPCA`, :class:`manifold.LocallyLinearEmbedding`, and :class:`manifold.SpectralEmbedding` (:issue:`5012`). By :user:`Peter Fischer `. @@ -614,7 +613,7 @@ Decomposition, manifold learning and clustering Preprocessing and feature selection -- :func:`preprocessing.data._transform_selected` now always passes a copy +- `preprocessing.data._transform_selected` now always passes a copy of ``X`` to transform function when ``copy=True`` (:issue:`7194`). By `Caio Oliveira `_. @@ -633,8 +632,8 @@ Model evaluation and meta-estimators return splits of size ``train_size`` and ``test_size`` in all cases (:issue:`6472`). By `Andreas MÃŧller`_. -- Cross-validation of :class:`OneVsOneClassifier` and - :class:`OneVsRestClassifier` now works with precomputed kernels. +- Cross-validation of :class:`multiclass.OneVsOneClassifier` and + :class:`multiclass.OneVsRestClassifier` now works with precomputed kernels. :issue:`7350` by :user:`Russell Smith `. - Fix incomplete ``predict_proba`` method delegation from @@ -654,7 +653,7 @@ Metrics - Fix bug where expected and adjusted mutual information were incorrect if cluster contingency cells exceeded ``2**16``. By `Joel Nothman`_. -- :func:`metrics.pairwise.pairwise_distances` now converts arrays to +- :func:`metrics.pairwise_distances` now converts arrays to boolean arrays when required in ``scipy.spatial.distance``. :issue:`5460` by `Tom Dupre la Tour`_. @@ -667,7 +666,7 @@ Metrics Miscellaneous -- :func:`model_selection.tests._search._check_param_grid` now works correctly with all types +- `model_selection.tests._search._check_param_grid` now works correctly with all types that extends/implements `Sequence` (except string), including range (Python 3.x) and xrange (Python 2.x). :issue:`7323` by Viacheslav Kovalevskyi. @@ -698,7 +697,7 @@ Linear, kernelized and related models Decomposition, manifold learning and clustering -- The old :class:`mixture.DPGMM` is deprecated in favor of the new +- The old `mixture.DPGMM` is deprecated in favor of the new :class:`mixture.BayesianGaussianMixture` (with the parameter ``weight_concentration_prior_type='dirichlet_process'``). The new class solves the computational @@ -706,7 +705,7 @@ Decomposition, manifold learning and clustering Dirichlet process prior faster than before. :issue:`7295` by :user:`Wei Xue ` and :user:`Thierry Guillemot `. -- The old :class:`mixture.VBGMM` is deprecated in favor of the new +- The old `mixture.VBGMM` is deprecated in favor of the new :class:`mixture.BayesianGaussianMixture` (with the parameter ``weight_concentration_prior_type='dirichlet_distribution'``). The new class solves the computational @@ -714,15 +713,15 @@ Decomposition, manifold learning and clustering mixture faster than before. :issue:`6651` by :user:`Wei Xue ` and :user:`Thierry Guillemot `. -- The old :class:`mixture.GMM` is deprecated in favor of the new +- The old `mixture.GMM` is deprecated in favor of the new :class:`mixture.GaussianMixture`. The new class computes the Gaussian mixture faster than before and some of computational problems have been solved. :issue:`6666` by :user:`Wei Xue ` and :user:`Thierry Guillemot `. Model evaluation and meta-estimators -- The :mod:`sklearn.cross_validation`, :mod:`sklearn.grid_search` and - :mod:`sklearn.learning_curve` have been deprecated and the classes and +- The `sklearn.cross_validation`, `sklearn.grid_search` and + `sklearn.learning_curve` have been deprecated and the classes and functions have been reorganized into the :mod:`sklearn.model_selection` module. Ref :ref:`model_selection_changes` for more information. :issue:`4294` by `Raghav RV`_. @@ -747,7 +746,7 @@ Model evaluation and meta-estimators :class:`model_selection.GroupShuffleSplit`, :class:`model_selection.LeaveOneGroupOut` and :class:`model_selection.LeavePGroupsOut` respectively. - Also the parameter ``labels`` in the :func:`split` method of the newly + Also the parameter ``labels`` in the `split` method of the newly renamed splitters :class:`model_selection.LeaveOneGroupOut` and :class:`model_selection.LeavePGroupsOut` is renamed to ``groups``. Additionally in :class:`model_selection.LeavePGroupsOut`, @@ -813,4 +812,3 @@ Hauck, trevorstephens, Tue Vo, Varun, Varun Jewalikar, Viacheslav, Vighnesh Birodkar, Vikram, Villu Ruusmann, Vinayak Mehta, walter, waterponey, Wenhua Yang, Wenjian Huang, Will Welch, wyseguy7, xyguo, yanlend, Yaroslav Halchenko, yelite, Yen, YenChenLin, Yichuan Liu, Yoav Ram, Yoshiki, Zheng RuiFeng, zivori, Óscar NÃĄjera - diff --git a/doc/whats_new/v0.19.rst b/doc/whats_new/v0.19.rst index 1b847b574399b..2d47afb0af1cf 100644 --- a/doc/whats_new/v0.19.rst +++ b/doc/whats_new/v0.19.rst @@ -2,6 +2,10 @@ .. currentmodule:: sklearn +============ +Version 0.19 +============ + .. _changes_0_19: Version 0.19.2 @@ -94,9 +98,9 @@ Regressions in 0.19.0 fixed in 0.19.1: longer accepted ``X`` as a list. :issue:`9600` by :user:`Rasul Kerimov `. -- Fixed handling of :func:`cross_val_predict` for binary classification with - ``method='decision_function'``. :issue:`9593` by :user:`Reiichiro Nakano - ` and core devs. +- Fixed handling of :func:`model_selection.cross_val_predict` for binary + classification with ``method='decision_function'``. :issue:`9593` by + :user:`Reiichiro Nakano ` and core devs. - Fix regression in :class:`pipeline.Pipeline` where it no longer accepted ``steps`` as a tuple. :issue:`9604` by :user:`Joris Van den Bossche @@ -119,13 +123,13 @@ Regressions in 0.19.0 fixed in 0.19.1: Enhancements ............ -- Our test suite and :func:`utils.estimator_checks.check_estimators` can now be +- Our test suite and :func:`utils.estimator_checks.check_estimator` can now be run without Nose installed. :issue:`9697` by :user:`Joan Massich `. - To improve usability of version 0.19's :class:`pipeline.Pipeline` caching, ``memory`` now allows ``joblib.Memory`` instances. This make use of the new :func:`utils.validation.check_memory` helper. - issue:`9584` by :user:`Kumar Ashutosh ` + :issue:`9584` by :user:`Kumar Ashutosh ` - Some fixes to examples: :issue:`9750`, :issue:`9788`, :issue:`9815` @@ -291,7 +295,7 @@ Miscellaneous and may be particularly useful for prediction time. :issue:`7548` by `Joel Nothman`_. -- Added a test to ensure parameter listing in docstrings match the +- Added a test to ensure parameter listing in docstrings matches the function/class signature. :issue:`9206` by `Alexandre Gramfort`_ and `Raghav RV`_. @@ -362,11 +366,11 @@ Linear, kernelized and related models Other predictors -- Custom metrics for the :mod:`neighbors` binary trees now have +- Custom metrics for the :mod:`sklearn.neighbors` binary trees now have fewer constraints: they must take two 1d-arrays and return a float. :issue:`6288` by `Jake Vanderplas`_. -- ``algorithm='auto`` in :mod:`neighbors` estimators now chooses the most +- ``algorithm='auto`` in :mod:`sklearn.neighbors` estimators now chooses the most appropriate algorithm for all input types and metrics. :issue:`9145` by :user:`Herilalaina Rakotoarison ` and :user:`Reddy Chinthala `. @@ -396,7 +400,7 @@ Decomposition, manifold learning and clustering - Memory usage enhancements: Prevent cast from float32 to float64 in :class:`decomposition.PCA` and - :func:`decomposition.randomized_svd_low_rank`. + `decomposition.randomized_svd_low_rank`. :issue:`9067` by `Raghav RV`_. Preprocessing and feature selection @@ -409,7 +413,7 @@ Preprocessing and feature selection with ``center=True``. :issue:`8065` by :user:`Daniel LeJeune `. - Small performance improvement to n-gram creation in - :mod:`feature_extraction.text` by binding methods for loops and + :mod:`sklearn.feature_extraction.text` by binding methods for loops and special-casing unigrams. :issue:`7567` by :user:`Jaye Doepke ` - Relax assumption on the data for the @@ -478,7 +482,7 @@ Model evaluation and meta-estimators Metrics -- :func:`metrics.matthews_corrcoef` now support multiclass classification. +- :func:`metrics.matthews_corrcoef` now supports multiclass classification. :issue:`8094` by :user:`Jon Crall `. - Add ``sample_weight`` parameter to :func:`metrics.cohen_kappa_score`. @@ -486,12 +490,12 @@ Metrics Miscellaneous -- :func:`utils.check_estimator` now attempts to ensure that methods +- :func:`utils.estimator_checks.check_estimator` now attempts to ensure that methods transform, predict, etc. do not set attributes on the estimator. :issue:`7533` by :user:`Ekaterina Krivich `. - Added type checking to the ``accept_sparse`` parameter in - :mod:`utils.validation` methods. This parameter now accepts only boolean, + :mod:`sklearn.utils.validation` methods. This parameter now accepts only boolean, string, or list/tuple of strings. ``accept_sparse=None`` is deprecated and should be replaced by ``accept_sparse=False``. :issue:`7880` by :user:`Josh Karnofsky `. @@ -512,7 +516,7 @@ Trees and ensembles :issue:`8002` by `Raghav RV`_. - Fixed a bug where :class:`ensemble.IsolationForest` uses an - an incorrect formula for the average path length + incorrect formula for the average path length :issue:`8549` by `Peter Wang `_. - Fixed a bug where :class:`ensemble.AdaBoostClassifier` throws @@ -570,7 +574,7 @@ Linear, kernelized and related models the same result as the LassoLars implementation available in R (lars library). :issue:`7849` by :user:`Jair Montoya Martinez `. -- Fixed a bug in :class:`linear_model.RandomizedLasso`, +- Fixed a bug in `linear_model.RandomizedLasso`, :class:`linear_model.Lars`, :class:`linear_model.LassoLars`, :class:`linear_model.LarsCV` and :class:`linear_model.LassoLarsCV`, where the parameter ``precompute`` was not used consistently across @@ -605,13 +609,13 @@ Linear, kernelized and related models - Fixed a bug in :class:`gaussian_process.GaussianProcessRegressor` when the standard deviation and covariance predicted without fit - would fail with a unmeaningful error by default. + would fail with a meaningless error by default. :issue:`6573` by :user:`Quazi Marufur Rahman ` and `Manoj Kumar`_. Other predictors -- Fix :class:`semi_supervised.BaseLabelPropagation` to correctly implement +- Fix `semi_supervised.BaseLabelPropagation` to correctly implement ``LabelPropagation`` and ``LabelSpreading`` as done in the referenced papers. :issue:`9239` by :user:`Andre Ambrosio Boechat `, :user:`Utkarsh Upadhyay @@ -620,7 +624,7 @@ Other predictors Decomposition, manifold learning and clustering - Fixed the implementation of :class:`manifold.TSNE`: -- ``early_exageration`` parameter had no effect and is now used for the +- ``early_exaggeration`` parameter had no effect and is now used for the first 250 optimization iterations. - Fixed the ``AssertionError: Tree consistency failed`` exception reported in :issue:`8992`. @@ -642,7 +646,7 @@ Decomposition, manifold learning and clustering - Fixed the implementation of ``explained_variance_`` in :class:`decomposition.PCA`, - :class:`decomposition.RandomizedPCA` and + `decomposition.RandomizedPCA` and :class:`decomposition.IncrementalPCA`. :issue:`9105` by `Hanmin Qin `_. @@ -674,13 +678,13 @@ Decomposition, manifold learning and clustering - Fixed improper scaling in :class:`cross_decomposition.PLSRegression` with ``scale=True``. :issue:`7819` by :user:`jayzed82 `. -- :class:`cluster.bicluster.SpectralCoclustering` and - :class:`cluster.bicluster.SpectralBiclustering` ``fit`` method conforms +- :class:`cluster.SpectralCoclustering` and + :class:`cluster.SpectralBiclustering` ``fit`` method conforms with API by accepting ``y`` and returning the object. :issue:`6126`, :issue:`7814` by :user:`Laurent Direr ` and :user:`Maniteja Nandana `. -- Fix bug where :mod:`mixture` ``sample`` methods did not return as many +- Fix bug where :mod:`sklearn.mixture` ``sample`` methods did not return as many samples as requested. :issue:`7702` by :user:`Levi John Wolf `. - Fixed the shrinkage implementation in :class:`neighbors.NearestCentroid`. @@ -698,8 +702,8 @@ Preprocessing and feature selection selected fewer features than it should. :issue:`7490` by :user:`Peng Meng `. -- Fixed a bug where :class:`linear_model.RandomizedLasso` and - :class:`linear_model.RandomizedLogisticRegression` breaks for +- Fixed a bug where `linear_model.RandomizedLasso` and + `linear_model.RandomizedLogisticRegression` break for sparse input. :issue:`8259` by :user:`Aman Dalmia `. - Fix a bug where :class:`feature_extraction.FeatureHasher` @@ -715,14 +719,14 @@ Preprocessing and feature selection Model evaluation and meta-estimators -- Fixed a bug where :func:`model_selection.BaseSearchCV.inverse_transform` +- Fixed a bug where `model_selection.BaseSearchCV.inverse_transform` returns ``self.best_estimator_.transform()`` instead of ``self.best_estimator_.inverse_transform()``. :issue:`8344` by :user:`Akshay Gupta ` and :user:`Rasmus Eriksson `. - Added ``classes_`` attribute to :class:`model_selection.GridSearchCV`, - :class:`model_selection.RandomizedSearchCV`, :class:`grid_search.GridSearchCV`, - and :class:`grid_search.RandomizedSearchCV` that matches the ``classes_`` + :class:`model_selection.RandomizedSearchCV`, `grid_search.GridSearchCV`, + and `grid_search.RandomizedSearchCV` that matches the ``classes_`` attribute of ``best_estimator_``. :issue:`7661` and :issue:`8295` by :user:`Alyssa Batula `, :user:`Dylan Werner-Meier `, and :user:`Stephen Hoover `. @@ -748,7 +752,7 @@ Model evaluation and meta-estimators raised on trying to stack matrices with different dimensions. :issue:`8093` by :user:`Peter Bull `. -- Cross validation now works with Pandas datatypes that that have a +- Cross validation now works with Pandas datatypes that have a read-only index. :issue:`9507` by `Loic Esteve`_. Metrics @@ -760,7 +764,7 @@ Metrics (`#7356 `_). By :user:`Nick Dingwall ` and `Gael Varoquaux`_. -- Fix a bug in :func:`metrics.classification._check_targets` +- Fix a bug in `metrics.classification._check_targets` which would return ``'binary'`` if ``y_true`` and ``y_pred`` were both ``'binary'`` but the union of ``y_true`` and ``y_pred`` was ``'multiclass'``. :issue:`8377` by `Loic Esteve`_. @@ -784,7 +788,7 @@ Miscellaneous incorrect result when ``n_samples`` is odd. :issue:`8198` by :user:`Josh Levy `. -- Some ``fetch_`` functions in :mod:`datasets` were ignoring the +- Some ``fetch_`` functions in :mod:`sklearn.datasets` were ignoring the ``download_if_missing`` keyword. :issue:`7944` by :user:`Ralf Gommers `. - Fix estimators to accept a ``sample_weight`` parameter of type @@ -795,7 +799,7 @@ Miscellaneous raising an exception if instability is identified. :issue:`7376` and :issue:`7331` by `Joel Nothman`_ and :user:`yangarbiter`. -- Fix a bug where :meth:`base.BaseEstimator.__getstate__` +- Fix a bug where `base.BaseEstimator.__getstate__` obstructed pickling customizations of child-classes, when used in a multiple inheritance context. :issue:`8316` by :user:`Holger Peters `. @@ -821,7 +825,7 @@ Trees and ensembles - Gradient boosting base models are no longer estimators. By `Andreas MÃŧller`_. -- All tree based estimators now accept a ``min_impurity_decrease`` +- All tree-based estimators now accept a ``min_impurity_decrease`` parameter in lieu of the ``min_impurity_split``, which is now deprecated. The ``min_impurity_decrease`` helps stop splitting the nodes in which the weighted impurity decrease from splitting is no longer at least @@ -837,7 +841,7 @@ Linear, kernelized and related models Other predictors -- :class:`neighbors.LSHForest` has been deprecated and will be +- `neighbors.LSHForest` has been deprecated and will be removed in 0.21 due to poor performance. :issue:`9078` by :user:`Laurent Direr `. @@ -876,7 +880,7 @@ Preprocessing and feature selection - :class:`feature_selection.SelectFromModel` now validates the ``threshold`` parameter and sets the ``threshold_`` attribute during the call to - ``fit``, and no longer during the call to ``transform```. By `Andreas + ``fit``, and no longer during the call to ``transform``. By `Andreas MÃŧller`_. - The ``non_negative`` parameter in :class:`feature_extraction.FeatureHasher` @@ -884,8 +888,8 @@ Preprocessing and feature selection ``alternate_sign``. :issue:`7565` by :user:`Roman Yurchak `. -- :class:`linear_model.RandomizedLogisticRegression`, - and :class:`linear_model.RandomizedLasso` have been deprecated and will +- `linear_model.RandomizedLogisticRegression`, + and `linear_model.RandomizedLasso` have been deprecated and will be removed in version 0.21. :issue:`8995` by :user:`Ramana.S `. @@ -938,18 +942,18 @@ Model evaluation and meta-estimators Miscellaneous - Deprecate the ``y`` parameter in ``transform`` and ``inverse_transform``. - The method should not accept ``y`` parameter, as it's used at the prediction time. + The method should not accept ``y`` parameter, as it's used at the prediction time. :issue:`8174` by :user:`Tahar Zanouda `, `Alexandre Gramfort`_ and `Raghav RV`_. - SciPy >= 0.13.3 and NumPy >= 1.8.2 are now the minimum supported versions for scikit-learn. The following backported functions in - :mod:`utils` have been removed or deprecated accordingly. + :mod:`sklearn.utils` have been removed or deprecated accordingly. :issue:`8854` and :issue:`8874` by :user:`Naoya Kanai ` - The ``store_covariances`` and ``covariances_`` parameters of :class:`discriminant_analysis.QuadraticDiscriminantAnalysis` - has been renamed to ``store_covariance`` and ``covariance_`` to be + have been renamed to ``store_covariance`` and ``covariance_`` to be consistent with the corresponding parameter names of the :class:`discriminant_analysis.LinearDiscriminantAnalysis`. They will be removed in version 0.21. :issue:`7998` by :user:`Jiacheng ` @@ -994,7 +998,7 @@ Miscellaneous - Ensure that estimators' attributes ending with ``_`` are not set in the constructor but only in the ``fit`` method. Most notably, - ensemble estimators (deriving from :class:`ensemble.BaseEnsemble`) + ensemble estimators (deriving from `ensemble.BaseEnsemble`) now only have ``self.estimators_`` available after ``fit``. :issue:`7464` by `Lars Buitinck`_ and `Loic Esteve`_. @@ -1066,4 +1070,3 @@ Dai, Greg Stupp, Grzegorz Szpak, Bertrand Thirion, Hadrien Bertrand, Harizo Rajaona, zxcvbnius, Henry Lin, Holger Peters, Icyblade Dai, Igor Andriushchenko, Ilya, Isaac Laughlin, IvÃĄn VallÊs, AurÊlien Bellet, JPFrancoia, Jacob Schreiber, Asish Mahapatra - diff --git a/doc/whats_new/v0.20.rst b/doc/whats_new/v0.20.rst index 2eaf3199fbc3c..a7d43d2d45d85 100644 --- a/doc/whats_new/v0.20.rst +++ b/doc/whats_new/v0.20.rst @@ -2,6 +2,17 @@ .. currentmodule:: sklearn +============ +Version 0.20 +============ + +.. warning:: + + Version 0.20 is the last version of scikit-learn to support Python 2.7 and Python 3.4. + Scikit-learn 0.21 will require Python 3.5 or higher. + +.. include:: changelog_legend.inc + .. _changes_0_20_4: Version 0.20.4 @@ -26,15 +37,15 @@ The bundled version of joblib was upgraded from 0.13.0 to 0.13.2. ....................... - |Fix| Fixed an issue in :class:`compose.ColumnTransformer` where using - DataFrames whose column order differs between :func:``fit`` and - :func:``transform`` could lead to silently passing incorrect columns to the + DataFrames whose column order differs between :func:`fit` and + :func:`transform` could lead to silently passing incorrect columns to the ``remainder`` transformer. :pr:`14237` by `Andreas Schuderer `. :mod:`sklearn.decomposition` ............................ -- |Fix| Fixed a bug in :class:`cross_decomposition.CCA` improving numerical +- |Fix| Fixed a bug in :class:`cross_decomposition.CCA` improving numerical stability when `Y` is close to zero. :pr:`13903` by `Thomas Fan`_. @@ -53,7 +64,7 @@ The bundled version of joblib was upgraded from 0.13.0 to 0.13.2. restored from a pickle if ``sample_weight`` had been used. :issue:`13772` by :user:`Aditya Vyas `. - .. _changes_0_20_3: +.. _changes_0_20_3: Version 0.20.3 ============== @@ -104,7 +115,7 @@ Changelog :mod:`sklearn.feature_extraction` ................................. -- |Fix| Fixed a bug in :class:`feature_extraction.text.CountVectorizer` which +- |Fix| Fixed a bug in :class:`feature_extraction.text.CountVectorizer` which would result in the sparse feature matrix having conflicting `indptr` and `indices` precisions under very large vocabularies. :issue:`11295` by :user:`Gabriel Vacaliuc `. @@ -209,7 +220,7 @@ Changelog :mod:`sklearn.neighbors` ........................ -- |Fix| Fixed :class:`sklearn.neighbors.DistanceMetric` jaccard distance +- |Fix| Fixed `sklearn.neighbors.DistanceMetric` jaccard distance function to return 0 when two all-zero vectors are compared. :issue:`12685` by :user:`Thomas Fan `. @@ -342,7 +353,7 @@ Changelog those estimators as part of parallel parameter search or cross-validation. :issue:`12122` by :user:`Olivier Grisel `. -- |Fix| Fixed a bug affecting :class:`SGDClassifier` in the multiclass +- |Fix| Fixed a bug affecting :class:`linear_model.SGDClassifier` in the multiclass case. Each one-versus-all step is run in a :class:`joblib.Parallel` call and mutating a common parameter, causing a segmentation fault if called within a backend using processes and not threads. We now use ``require=sharedmem`` @@ -352,16 +363,16 @@ Changelog :mod:`sklearn.metrics` ...................... -- |Fix| Fixed a bug in :func:`metrics.pairwise.pairwise_distances_argmin_min` +- |Fix| Fixed a bug in `metrics.pairwise.pairwise_distances_argmin_min` which returned the square root of the distance when the metric parameter was set to "euclidean". :issue:`12481` by :user:`JÊrÊmie du Boisberranger `. -- |Fix| Fixed a bug in :func:`metrics.pairwise.pairwise_distances_chunked` +- |Fix| Fixed a bug in `metrics.pairwise.pairwise_distances_chunked` which didn't ensure the diagonal is zero for euclidean distances. :issue:`12612` by :user:`Andreas MÃŧller `. -- |API| The :func:`metrics.calinski_harabaz_score` has been renamed to +- |API| The `metrics.calinski_harabaz_score` has been renamed to :func:`metrics.calinski_harabasz_score` and will be removed in version 0.23. :issue:`12211` by :user:`Lisa Thomas `, :user:`Mark Hannel ` and :user:`Melissa Ferrari `. @@ -399,7 +410,7 @@ Changelog :issue:`12522` by :user:`Nicolas Hug`. - |Fix| Fixed a bug in :class:`preprocessing.OneHotEncoder` where transform - failed when set to ignore unknown numpy strings of different lengths + failed when set to ignore unknown numpy strings of different lengths :issue:`12471` by :user:`Gabriel Marzinotto`. - |API| The default value of the :code:`method` argument in @@ -419,7 +430,7 @@ Changelog - |Fix| Calling :func:`utils.check_array` on `pandas.Series`, which raised an error in 0.20.0, now returns the expected output again. :issue:`12625` by `Andreas MÃŧller`_ - + Miscellaneous ............. @@ -434,7 +445,7 @@ Miscellaneous - |API| Removed all mentions of ``sklearn.externals.joblib``, and deprecated joblib methods exposed in ``sklearn.utils``, except for - :func:`utils.parallel_backend` and :func:`utils.register_parallel_backend`, + `utils.parallel_backend` and `utils.register_parallel_backend`, which allow users to configure parallel computation in scikit-learn. Other functionalities are part of `joblib `_. package and should be used directly, by installing it. @@ -480,11 +491,6 @@ Thanks to our contributors! This release is dedicated to the memory of Raghav Rajagopalan. -.. warning:: - - Version 0.20 is the last version of scikit-learn to support Python 2.7 and Python 3.4. - Scikit-learn 0.21 will require Python 3.5 or higher. - Highlights ---------- @@ -493,7 +499,7 @@ including missing values, categorical variables, heterogeneous data, and features/targets with unusual distributions. Missing values in features, represented by NaNs, are now accepted in column-wise preprocessing such as scalers. Each feature is fitted disregarding -NaNs, and data containing NaNs can be transformed. The new :mod:`impute` +NaNs, and data containing NaNs can be transformed. The new :mod:`sklearn.impute` module provides estimators for learning despite missing data. :class:`~compose.ColumnTransformer` handles the case where different features @@ -545,7 +551,7 @@ random sampling procedures. - :class:`linear_model.SGDRegressor` (bug fix) - :class:`metrics.roc_auc_score` (bug fix) - :class:`metrics.roc_curve` (bug fix) -- :class:`neural_network.BaseMultilayerPerceptron` (bug fix) +- `neural_network.BaseMultilayerPerceptron` (bug fix) - :class:`neural_network.MLPClassifier` (bug fix) - :class:`neural_network.MLPRegressor` (bug fix) - The v0.19.0 release notes failed to mention a backwards incompatibility with @@ -590,7 +596,7 @@ Support for Python 3.3 has been officially dropped. :issue:`10933` by :user:`Johannes Hansen `. - |Efficiency| :class:`cluster.KMeans`, :class:`cluster.MiniBatchKMeans` and - :func:`cluster.k_means` passed with ``algorithm='full'`` now enforces + :func:`cluster.k_means` passed with ``algorithm='full'`` now enforce row-major ordering, improving runtime. :issue:`10471` by :user:`Gaurav Dhingra `. @@ -616,7 +622,7 @@ Support for Python 3.3 has been officially dropped. by :user:`Jan Margeta `, :user:`Guillaume Lemaitre `, and :user:`Devansh D. `. -- |Fix| Fixed a bug in :func:`cluster.k_means_elkan` where the returned +- |Fix| Fixed a bug in `cluster.k_means_elkan` where the returned ``iteration`` was 1 less than the correct value. Also added the missing ``n_iter_`` attribute in the docstring of :class:`cluster.KMeans`. :issue:`11353` by :user:`Jeremie du Boisberranger `. @@ -654,8 +660,8 @@ Support for Python 3.3 has been officially dropped. - |Efficiency| Runtime improvements to :class:`covariance.GraphicalLasso`. :issue:`9858` by :user:`Steven Brown `. -- |API| The :func:`covariance.graph_lasso`, - :class:`covariance.GraphLasso` and :class:`covariance.GraphLassoCV` have been +- |API| The `covariance.graph_lasso`, + `covariance.GraphLasso` and `covariance.GraphLassoCV` have been renamed to :func:`covariance.graphical_lasso`, :class:`covariance.GraphicalLasso` and :class:`covariance.GraphicalLassoCV` respectively and will be removed in version 0.22. @@ -675,14 +681,14 @@ Support for Python 3.3 has been officially dropped. cluster. :issue:`8617` by :user:`Maskani Filali Mohamed ` and :user:`Konstantinos Katrioplas `. -- |Feature| Add ``filename`` attribute to :mod:`datasets` that have a CSV file. +- |Feature| Add ``filename`` attribute to :mod:`sklearn.datasets` that have a CSV file. :issue:`9101` by :user:`alex-33 ` and :user:`Maskani Filali Mohamed `. - |Feature| ``return_X_y`` parameter has been added to several dataset loaders. :issue:`10774` by :user:`Chris Catalfo `. -- |Fix| Fixed a bug in :func:`datasets.load_boston` which had a wrong data +- |Fix| Fixed a bug in `datasets.load_boston` which had a wrong data point. :issue:`10795` by :user:`Takeshi Yoshizawa `. - |Fix| Fixed a bug in :func:`datasets.load_iris` which had two wrong data points. @@ -696,7 +702,7 @@ Support for Python 3.3 has been officially dropped. data points could be generated. :issue:`10045` by :user:`Christian Braune `. -- |API| Deprecated :func:`sklearn.datasets.fetch_mldata` to be removed in +- |API| Deprecated `sklearn.datasets.fetch_mldata` to be removed in version 0.22. mldata.org is no longer operational. Until removal it will remain possible to load cached datasets. :issue:`11466` by `Joel Nothman`_. @@ -751,8 +757,8 @@ Support for Python 3.3 has been officially dropped. :mod:`sklearn.discriminant_analysis` .................................... -- |Efficiency| Memory usage improvement for :func:`_class_means` and - :func:`_class_cov` in :mod:`discriminant_analysis`. :issue:`10898` by +- |Efficiency| Memory usage improvement for `_class_means` and + `_class_cov` in :mod:`sklearn.discriminant_analysis`. :issue:`10898` by :user:`Nanxin Chen `. @@ -809,14 +815,14 @@ Support for Python 3.3 has been officially dropped. to 100 in 0.22. A FutureWarning is raised when the default value is used. :issue:`11542` by :user:`Anna Ayzenshtat `. -- |API| Classes derived from :class:`ensemble.BaseBagging`. The attribute +- |API| Classes derived from `ensemble.BaseBagging`. The attribute ``estimators_samples_`` will return a list of arrays containing the indices selected for each bootstrap instead of a list of arrays containing the mask of the samples selected for each bootstrap. Indices allows to repeat samples while mask does not allow this functionality. :issue:`9524` by :user:`Guillaume Lemaitre `. -- |Fix| :class:`ensemble.BaseBagging` where one could not deterministically +- |Fix| `ensemble.BaseBagging` where one could not deterministically reproduce ``fit`` result using the object attributes when ``random_state`` is set. :issue:`9723` by :user:`Guillaume Lemaitre `. @@ -824,7 +830,7 @@ Support for Python 3.3 has been officially dropped. :mod:`sklearn.feature_extraction` ................................. -- |Feature| Enable the call to :term:`get_feature_names` in unfitted +- |Feature| Enable the call to `get_feature_names` in unfitted :class:`feature_extraction.text.CountVectorizer` initialized with a vocabulary. :issue:`10908` by :user:`Mohamed Maskani `. @@ -925,7 +931,7 @@ Support for Python 3.3 has been officially dropped. :class:`linear_model.BayesianRidge` for weighted linear regression. :issue:`10112` by :user:`Peter St. John `. -- |Fix| Fixed a bug in :func:`logistic.logistic_regression_path` to ensure +- |Fix| Fixed a bug in `logistic.logistic_regression_path` to ensure that the returned coefficients are correct when ``multiclass='multinomial'``. Previously, some of the coefficients would override each other, leading to incorrect results in :class:`linear_model.LogisticRegressionCV`. @@ -1027,7 +1033,7 @@ Support for Python 3.3 has been officially dropped. - |Feature| Support sparse input in :meth:`manifold.Isomap.fit`. :issue:`8554` by :user:`Leland McInnes `. -- |Feature| :func:`manifold.t_sne.trustworthiness` accepts metrics other than +- |Feature| `manifold.t_sne.trustworthiness` accepts metrics other than Euclidean. :issue:`9775` by :user:`William de Vazelhes `. - |Fix| Fixed a bug in :func:`manifold.spectral_embedding` where the @@ -1037,14 +1043,14 @@ Support for Python 3.3 has been officially dropped. `. - |API| |Feature| Deprecate ``precomputed`` parameter in function - :func:`manifold.t_sne.trustworthiness`. Instead, the new parameter ``metric`` + `manifold.t_sne.trustworthiness`. Instead, the new parameter ``metric`` should be used with any compatible metric including 'precomputed', in which case the input matrix ``X`` should be a matrix of pairwise distances or squared distances. :issue:`9775` by :user:`William de Vazelhes `. - |API| Deprecate ``precomputed`` parameter in function - :func:`manifold.t_sne.trustworthiness`. Instead, the new parameter + `manifold.t_sne.trustworthiness`. Instead, the new parameter ``metric`` should be used with any compatible metric including 'precomputed', in which case the input matrix ``X`` should be a matrix of pairwise distances or squared distances. :issue:`9775` by @@ -1161,12 +1167,12 @@ Support for Python 3.3 has been officially dropped. calling :term:`fit` and :term:`predict`. :issue:`10336` by :user:`Shu Haoran ` and :user:`Andrew Peng `. -- |Fix| Fixed a bug in :class:`mixture.BaseMixture` where the reported `n_iter_` was +- |Fix| Fixed a bug in `mixture.BaseMixture` where the reported `n_iter_` was missing an iteration. It affected :class:`mixture.GaussianMixture` and :class:`mixture.BayesianGaussianMixture`. :issue:`10740` by :user:`Erich Schubert ` and :user:`Guillaume Lemaitre `. -- |Fix| Fixed a bug in :class:`mixture.BaseMixture` and its subclasses +- |Fix| Fixed a bug in `mixture.BaseMixture` and its subclasses :class:`mixture.GaussianMixture` and :class:`mixture.BayesianGaussianMixture` where the ``lower_bound_`` was not the max lower bound across all initializations (when ``n_init > 1``), but just the lower bound of the last @@ -1192,7 +1198,7 @@ Support for Python 3.3 has been officially dropped. :func:`model_selection.cross_val_score`, :func:`model_selection.learning_curve` and :func:`model_selection.validation_curve` to control the behavior triggered - when an error occurs in :func:`model_selection._fit_and_score`. + when an error occurs in `model_selection._fit_and_score`. :issue:`11576` by :user:`Samuel O. Ronsin `. - |Feature| `BaseSearchCV` now has an experimental, private interface to @@ -1271,7 +1277,7 @@ Support for Python 3.3 has been officially dropped. parallelized according to ``n_jobs`` regardless of ``algorithm``. :issue:`10887` by :user:`JoÃĢl Billaud `. -- |Efficiency| :mod:`Nearest neighbors ` query methods are now more +- |Efficiency| :mod:`sklearn.neighbors` query methods are now more memory efficient when ``algorithm='brute'``. :issue:`11136` by `Joel Nothman`_ and :user:`Aman Dalmia `. @@ -1286,7 +1292,7 @@ Support for Python 3.3 has been officially dropped. be used for novelty detection, i.e. predict on new unseen data. Available prediction methods are ``predict``, ``decision_function`` and ``score_samples``. By default, ``novelty`` is set to ``False``, and only - the ``fit_predict`` method is avaiable. + the ``fit_predict`` method is available. By :user:`Albert Thomas `. - |Fix| Fixed a bug in :class:`neighbors.NearestNeighbors` where fitting a @@ -1305,7 +1311,7 @@ Support for Python 3.3 has been officially dropped. :issue:`11556` by :user:`Jake VanderPlas ` - |Fix| Fixed a bug in :class:`neighbors.KDTree` and :class:`neighbors.BallTree` where - pickled tree objects would change their type to the super class :class:`BinaryTree`. + pickled tree objects would change their type to the super class `BinaryTree`. :issue:`11774` by :user:`Nicolas Hug `. @@ -1313,13 +1319,13 @@ Support for Python 3.3 has been officially dropped. ............................. - |Feature| Add `n_iter_no_change` parameter in - :class:`neural_network.BaseMultilayerPerceptron`, + `neural_network.BaseMultilayerPerceptron`, :class:`neural_network.MLPRegressor`, and :class:`neural_network.MLPClassifier` to give control over maximum number of epochs to not meet ``tol`` improvement. :issue:`9456` by :user:`Nicholas Nadeau `. -- |Fix| Fixed a bug in :class:`neural_network.BaseMultilayerPerceptron`, +- |Fix| Fixed a bug in `neural_network.BaseMultilayerPerceptron`, :class:`neural_network.MLPRegressor`, and :class:`neural_network.MLPClassifier` with new ``n_iter_no_change`` parameter now at 10 from previously hardcoded 2. @@ -1398,7 +1404,7 @@ Support for Python 3.3 has been officially dropped. :issue:`11308` by :user:`Guillaume Lemaitre `. - |Feature| :class:`preprocessing.OneHotEncoder` now supports the - :term:`get_feature_names` method to obtain the transformed feature names. + `get_feature_names` method to obtain the transformed feature names. :issue:`10181` by :user:`Nirvan Anjirbag ` and `Joris Van den Bossche`_. @@ -1441,13 +1447,13 @@ Support for Python 3.3 has been officially dropped. :class:`compose.ColumnTransformer`. :issue:`10521` by `Joris Van den Bossche`_. -- |API| Deprecate :class:`preprocessing.Imputer` and move +- |API| Deprecate `preprocessing.Imputer` and move the corresponding module to :class:`impute.SimpleImputer`. :issue:`9726` by :user:`Kumar Ashutosh `. - |API| The ``axis`` parameter that was in - :class:`preprocessing.Imputer` is no longer present in + `preprocessing.Imputer` is no longer present in :class:`impute.SimpleImputer`. The behavior is equivalent to ``axis=0`` (impute along columns). Row-wise imputation can be performed with FunctionTransformer @@ -1457,8 +1463,8 @@ Support for Python 3.3 has been officially dropped. :user:`Gilberto Olimpio `. - |API| The NaN marker for the missing values has been changed - between the :class:`preprocessing.Imputer` and the - :class:`impute.SimpleImputer`. + between the `preprocessing.Imputer` and the + `impute.SimpleImputer`. ``missing_values='NaN'`` should now be ``missing_values=np.nan``. :issue:`11211` by :user:`Jeremie du Boisberranger `. @@ -1491,15 +1497,15 @@ Support for Python 3.3 has been officially dropped. ................... - |Enhancement| Although private (and hence not assured API stability), - :class:`tree._criterion.ClassificationCriterion` and - :class:`tree._criterion.RegressionCriterion` may now be cimported and + `tree._criterion.ClassificationCriterion` and + `tree._criterion.RegressionCriterion` may now be cimported and extended. :issue:`10325` by :user:`Camil Staps `. -- |Fix| Fixed a bug in :class:`tree.BaseDecisionTree` with `splitter="best"` +- |Fix| Fixed a bug in `tree.BaseDecisionTree` with `splitter="best"` where split threshold could become infinite when values in X were near infinite. :issue:`10536` by :user:`Jonathan Ohayon `. -- |Fix| Fixed a bug in :class:`tree.MAE` to ensure sample weights are being +- |Fix| Fixed a bug in `tree.MAE` to ensure sample weights are being used during the calculation of tree MAE impurity. Previous behaviour could cause suboptimal splits to be chosen since the impurity calculation considered all samples to be of equal weight importance. @@ -1559,7 +1565,7 @@ Multiple modules - |API| Changed warning type from :class:`UserWarning` to :class:`exceptions.ConvergenceWarning` for failing convergence in - :func:`linear_model.logistic_regression_path`, + `linear_model.logistic_regression_path`, :class:`linear_model.RANSACRegressor`, :func:`linear_model.ridge_regression`, :class:`gaussian_process.GaussianProcessRegressor`, :class:`gaussian_process.GaussianProcessClassifier`, @@ -1596,7 +1602,7 @@ Miscellaneous PyPy3-v5.10+, Numpy 1.14.0+, and scipy 1.1.0+ are required. :issue:`11010` by :user:`Ronan Lamy ` and `Roman Yurchak`_. -- |Feature| A utility method :func:`sklearn.show_versions()` was added to +- |Feature| A utility method :func:`sklearn.show_versions` was added to print out information relevant for debugging. It includes the user system, the Python executable, the version of the main libraries and BLAS binding information. :issue:`11596` by :user:`Alexandre Boucaud ` diff --git a/doc/whats_new/v0.21.rst b/doc/whats_new/v0.21.rst index cf9886a6636af..f7e708fc713fd 100644 --- a/doc/whats_new/v0.21.rst +++ b/doc/whats_new/v0.21.rst @@ -2,13 +2,17 @@ .. currentmodule:: sklearn +============ +Version 0.21 +============ + +.. include:: changelog_legend.inc + .. _changes_0_21_3: Version 0.21.3 ============== -.. include:: changelog_legend.inc - **July 30, 2019** Changed models @@ -47,8 +51,8 @@ Changelog ...................... - |Fix| Fixed an issue in :class:`compose.ColumnTransformer` where using - DataFrames whose column order differs between :func:``fit`` and - :func:``transform`` could lead to silently passing incorrect columns to the + DataFrames whose column order differs between :func:`fit` and + :func:`transform` could lead to silently passing incorrect columns to the ``remainder`` transformer. :pr:`14237` by `Andreas Schuderer `. @@ -67,8 +71,8 @@ Changelog :mod:`sklearn.ensemble` ....................... -- |Fix| Fix zero division error in :func:`HistGradientBoostingClassifier` and - :func:`HistGradientBoostingRegressor`. +- |Fix| Fix zero division error in :class:`ensemble.HistGradientBoostingClassifier` and + :class:`ensemble.HistGradientBoostingRegressor`. :pr:`14024` by `Nicolas Hug `. :mod:`sklearn.impute` @@ -81,7 +85,7 @@ Changelog :mod:`sklearn.inspection` ......................... -- |Fix| Fixed a bug in :func:`inspection.plot_partial_dependence` where +- |Fix| Fixed a bug in `inspection.plot_partial_dependence` where ``target`` parameter was not being taken into account for multiclass problems. :pr:`14393` by :user:`Guillem G. Subies `. @@ -109,10 +113,10 @@ Changelog :mod:`sklearn.tree` ................... -- |Fix| Fixed bug in :func:`tree.export_text` when the tree has one feature and +- |Fix| Fixed bug in :func:`tree.export_text` when the tree has one feature and a single feature name is passed in. :pr:`14053` by `Thomas Fan`. -- |Fix| Fixed an issue with :func:`plot_tree` where it displayed +- |Fix| Fixed an issue with :func:`tree.plot_tree` where it displayed entropy calculations even for `gini` criterion in DecisionTreeClassifiers. :pr:`13947` by :user:`Frank Hoang `. @@ -129,14 +133,14 @@ Changelog :mod:`sklearn.decomposition` ............................ -- |Fix| Fixed a bug in :class:`cross_decomposition.CCA` improving numerical +- |Fix| Fixed a bug in :class:`cross_decomposition.CCA` improving numerical stability when `Y` is close to zero. :pr:`13903` by `Thomas Fan`_. :mod:`sklearn.metrics` ...................... - |Fix| Fixed a bug in :func:`metrics.pairwise.euclidean_distances` where a - part of the distance matrix was left un-instanciated for suffiently large + part of the distance matrix was left un-instanciated for sufficiently large float32 datasets (regression introduced in 0.21). :pr:`13910` by :user:`JÊrÊmie du Boisberranger `. @@ -148,11 +152,11 @@ Changelog by :user:`James Myatt `. -:mod:`sklearn.utils.sparsefuncs` -................................ +`sklearn.utils.sparsefuncs` +........................... -- |Fix| Fixed a bug where :func:`min_max_axis` would fail on 32-bit systems - for certain large inputs. This affects :class:`preprocessing.MaxAbsScaler`, +- |Fix| Fixed a bug where `min_max_axis` would fail on 32-bit systems + for certain large inputs. This affects :class:`preprocessing.MaxAbsScaler`, :func:`preprocessing.normalize` and :class:`preprocessing.LabelBinarizer`. :pr:`13741` by :user:`Roddy MacSween `. @@ -230,7 +234,7 @@ random sampling procedures. - :func:`svm.SVC.decision_function` and :func:`multiclass.OneVsOneClassifier.decision_function`. |Fix| - :class:`linear_model.SGDClassifier` and any derived classifiers. |Fix| -- Any model using the :func:`linear_model._sag.sag_solver` function with a `0` +- Any model using the `linear_model._sag.sag_solver` function with a `0` seed, including :class:`linear_model.LogisticRegression`, :class:`linear_model.LogisticRegressionCV`, :class:`linear_model.Ridge`, and :class:`linear_model.RidgeCV` with 'sag' solver. |Fix| @@ -421,6 +425,11 @@ Support for Python 3.4 and below has been officially dropped. >>> # now you can import normally from sklearn.ensemble >>> from sklearn.ensemble import HistGradientBoostingClassifier + .. note:: + Update: since version 1.0, these estimators are not experimental + anymore and you don't need to use `from sklearn.experimental import + enable_hist_gradient_boosting`. + :pr:`12807` by :user:`Nicolas Hug`. - |Feature| Add :class:`ensemble.VotingRegressor` @@ -431,7 +440,7 @@ Support for Python 3.4 and below has been officially dropped. - |Efficiency| Make :class:`ensemble.IsolationForest` prefer threads over processes when running with ``n_jobs > 1`` as the underlying decision tree - fit calls do release the GIL. This changes reduces memory usage and + fit calls do release the GIL. This change reduces memory usage and communication overhead. :pr:`12543` by :user:`Isaac Storch ` and `Olivier Grisel`_. @@ -503,24 +512,24 @@ Support for Python 3.4 and below has been officially dropped. if any targets were strings. :pr:`12834` by :user:`Elizabeth Sander `. -- |Fix| Fixed a bug in :class:`ensemble.gradient_boosting.LossFunction` and - :class:`ensemble.gradient_boosting.LeastSquaresError` where the default +- |Fix| Fixed a bug in `ensemble.gradient_boosting.LossFunction` and + `ensemble.gradient_boosting.LeastSquaresError` where the default value of ``learning_rate`` in ``update_terminal_regions`` is not consistent with the document and the caller functions. Note however that directly using these loss functions is deprecated. :pr:`6463` by :user:`movelikeriver `. -- |Fix| :func:`ensemble.partial_dependence` (and consequently the new +- |Fix| `ensemble.partial_dependence` (and consequently the new version :func:`sklearn.inspection.partial_dependence`) now takes sample weights into account for the partial dependence computation when the gradient boosting model has been trained with sample weights. :pr:`13193` by :user:`Samuel O. Ronsin `. -- |API| :func:`ensemble.partial_dependence` and - :func:`ensemble.plot_partial_dependence` are now deprecated in favor of +- |API| `ensemble.partial_dependence` and + `ensemble.plot_partial_dependence` are now deprecated in favor of :func:`inspection.partial_dependence` and - :func:`inspection.plot_partial_dependence`. + `inspection.plot_partial_dependence`. :pr:`12599` by :user:`Trevor Stephens` and :user:`Nicolas Hug`. @@ -535,10 +544,10 @@ Support for Python 3.4 and below has been officially dropped. :class:`pipeline.FeatureUnion` and :class:`compose.ColumnTransformer`). :pr:`13780` by :user:`Guillaume Lemaitre `. -:mod:`sklearn.externals` -........................ +`sklearn.externals` +................... -- |API| Deprecated :mod:`externals.six` since we have dropped support for +- |API| Deprecated `externals.six` since we have dropped support for Python 2.7. :pr:`12916` by :user:`Hanmin Qin `. :mod:`sklearn.feature_extraction` @@ -578,7 +587,7 @@ Support for Python 3.4 and below has been officially dropped. `. - |Fix| In :class:`impute.MissingIndicator` avoid implicit densification by - raising an exception if input is sparse add `missing_values` property + raising an exception if input is sparse and `missing_values` property is set to 0. :pr:`13240` by :user:`Bartosz Telenczuk `. - |Fix| Fixed two bugs in :class:`impute.MissingIndicator`. First, when @@ -594,7 +603,7 @@ Support for Python 3.4 and below has been officially dropped. (new subpackage) - |Feature| Partial dependence plots - (:func:`inspection.plot_partial_dependence`) are now supported for + (`inspection.plot_partial_dependence`) are now supported for any regressor or classifier (provided that they have a `predict_proba` method). :pr:`12599` by :user:`Trevor Stephens ` and :user:`Nicolas Hug `. @@ -622,7 +631,7 @@ Support for Python 3.4 and below has been officially dropped. users to compute :class:`linear_model.lars_path` without providing ``X`` and ``y``. :pr:`11699` by :user:`Kuai Yu `. -- |Efficiency| :func:`linear_model.make_dataset` now preserves +- |Efficiency| `linear_model.make_dataset` now preserves ``float32`` and ``float64`` dtypes, reducing memory consumption in stochastic gradient, SAG and SAGA solvers. :pr:`8769` and :pr:`11000` by @@ -641,7 +650,7 @@ Support for Python 3.4 and below has been officially dropped. - |Enhancement| The coordinate descent solver used in `Lasso`, `ElasticNet`, etc. now issues a `ConvergenceWarning` when it completes without meeting the - desired toleranbce. + desired tolerance. :pr:`11754` and :pr:`13397` by :user:`Brent Fagan ` and :user:`Adrin Jalali `. @@ -665,7 +674,7 @@ Support for Python 3.4 and below has been officially dropped. :pr:`12972` by :user:`Lucio Fernandez-Arjona ` - |Fix| Fixed a bug in :class:`linear_model.LinearRegression` that - was not returning the same coeffecients and intercepts with + was not returning the same coefficients and intercepts with ``fit_intercept=True`` in sparse and dense case. :pr:`13279` by `Alexandre Gramfort`_ @@ -678,7 +687,7 @@ Support for Python 3.4 and below has been officially dropped. case. :pr:`13389` by :user:`Pierre Glaser `. - |Fix| Fixed a bug in - :class:`linear_model.stochastic_gradient.BaseSGDClassifier` that was not + `linear_model.stochastic_gradient.BaseSGDClassifier` that was not deterministic when trained in a multi-class setting on several threads. :pr:`13422` by :user:`ClÊment Doumouro `. @@ -703,7 +712,7 @@ Support for Python 3.4 and below has been officially dropped. in version 0.23. Use :class:`linear_model.lars_path_gram` instead. :pr:`11699` by :user:`Kuai Yu `. -- |API| :func:`linear_model.logistic_regression_path` is deprecated +- |API| `linear_model.logistic_regression_path` is deprecated in version 0.21 and will be removed in version 0.23. :pr:`12821` by :user:`Nicolas Hug `. @@ -714,7 +723,7 @@ Support for Python 3.4 and below has been officially dropped. :mod:`sklearn.manifold` ....................... -- |Efficiency| Make :func:`manifold.tsne.trustworthiness` use an inverted index +- |Efficiency| Make :func:`manifold.trustworthiness` use an inverted index instead of an `np.where` lookup to find the rank of neighbors in the input space. This improves efficiency in particular when computed with lots of neighbors and/or small datasets. @@ -784,13 +793,13 @@ Support for Python 3.4 and below has been officially dropped. in version 0.21 and will be removed in version 0.23. :pr:`10580` by :user:`Reshama Shaikh ` and :user:`Sandra Mitrovic `. -- |Fix| The function :func:`metrics.pairwise.euclidean_distances`, and - therefore several estimators with ``metric='euclidean'``, suffered from - numerical precision issues with ``float32`` features. Precision has been - increased at the cost of a small drop of performance. :pr:`13554` by +- |Fix| The function :func:`metrics.pairwise.euclidean_distances`, and + therefore several estimators with ``metric='euclidean'``, suffered from + numerical precision issues with ``float32`` features. Precision has been + increased at the cost of a small drop of performance. :pr:`13554` by :user:`Celelibi` and :user:`JÊrÊmie du Boisberranger `. -- |API| :func:`metrics.jaccard_similarity_score` is deprecated in favour of +- |API| `metrics.jaccard_similarity_score` is deprecated in favour of the more consistent :func:`metrics.jaccard_score`. The former behavior for binary and multiclass targets is broken. :pr:`13151` by `Joel Nothman`_. @@ -798,7 +807,7 @@ Support for Python 3.4 and below has been officially dropped. :mod:`sklearn.mixture` ...................... -- |Fix| Fixed a bug in :class:`mixture.BaseMixture` and therefore on estimators +- |Fix| Fixed a bug in `mixture.BaseMixture` and therefore on estimators based on it, i.e. :class:`mixture.GaussianMixture` and :class:`mixture.BayesianGaussianMixture`, where ``fit_predict`` and ``fit.predict`` were not equivalent. :pr:`13142` by @@ -860,7 +869,7 @@ Support for Python 3.4 and below has been officially dropped. `predict_proba` method incorrectly checked for `predict_proba` attribute in the estimator object. :pr:`12222` by :user:`Rebekah Kim ` - + :mod:`sklearn.neighbors` ........................ @@ -953,7 +962,7 @@ Support for Python 3.4 and below has been officially dropped. - |API| The default value of `copy` in :func:`preprocessing.quantile_transform` will change from False to True in 0.23 in order to make it more consistent with the default `copy` values of other functions in - :mod:`preprocessing` and prevent unexpected side effects by modifying + :mod:`sklearn.preprocessing` and prevent unexpected side effects by modifying the value of `X` inplace. :pr:`13459` by :user:`Hunter McGushion `. @@ -971,7 +980,7 @@ Support for Python 3.4 and below has been officially dropped. ................... - |Feature| Decision Trees can now be plotted with matplotlib using - :func:`tree.plot_tree` without relying on the ``dot`` library, + `tree.plot_tree` without relying on the ``dot`` library, removing a hard-to-install dependency. :pr:`8508` by `Andreas MÃŧller`_. - |Feature| Decision Trees can now be exported in a human readable @@ -979,7 +988,7 @@ Support for Python 3.4 and below has been officially dropped. :pr:`6261` by `Giuseppe Vettigli `. - |Feature| ``get_n_leaves()`` and ``get_depth()`` have been added to - :class:`tree.BaseDecisionTree` and consequently all estimators based + `tree.BaseDecisionTree` and consequently all estimators based on it, including :class:`tree.DecisionTreeClassifier`, :class:`tree.DecisionTreeRegressor`, :class:`tree.ExtraTreeClassifier`, and :class:`tree.ExtraTreeRegressor`. @@ -989,7 +998,7 @@ Support for Python 3.4 and below has been officially dropped. classification targets with string labels, despite accepting them in `fit`. :pr:`11458` by :user:`Mitar Milutinovic `. -- |Fix| Fixed an issue with :class:`tree.BaseDecisionTree` +- |Fix| Fixed an issue with `tree.BaseDecisionTree` and consequently all estimators based on it, including :class:`tree.DecisionTreeClassifier`, :class:`tree.DecisionTreeRegressor`, :class:`tree.ExtraTreeClassifier`, @@ -1008,7 +1017,7 @@ Support for Python 3.4 and below has been officially dropped. - |API| Deprecated ``warn_on_dtype`` parameter from :func:`utils.check_array` and :func:`utils.check_X_y`. Added explicit warning for dtype conversion - in :func:`check_pairwise_arrays` if the ``metric`` being passed is a + in `check_pairwise_arrays` if the ``metric`` being passed is a pairwise boolean metric. :pr:`13382` by :user:`Prathmesh Savale `. @@ -1033,7 +1042,7 @@ Multiple modules dtype in multiple estimators. :pr:`11973` by :user:`Roman Yurchak `. -- |Fix| Fixed a bug in the implementation of the :func:`our_rand_r` +- |Fix| Fixed a bug in the implementation of the `our_rand_r` helper function that was not behaving consistently across platforms. :pr:`13422` by :user:`Madhura Parikh ` and :user:`ClÊment Doumouro `. @@ -1055,15 +1064,14 @@ These changes mostly affect library developers. - Add ``check_fit_idempotent`` to :func:`~utils.estimator_checks.check_estimator`, which checks that - when `fit` is called twice with the same data, the ouput of + when `fit` is called twice with the same data, the output of `predict`, `predict_proba`, `transform`, and `decision_function` does not change. :pr:`12328` by :user:`Nicolas Hug ` - Many checks can now be disabled or configured with :ref:`estimator_tags`. :pr:`8022` by :user:`Andreas MÃŧller `. -Code and Documentation Contributors ------------------------------------ +.. rubric:: Code and documentation contributors Thanks to everyone who has contributed to the maintenance and improvement of the project since version 0.20, including: @@ -1078,7 +1086,7 @@ Baibak, daten-kieker, Denis Kataev, Didi Bar-Zev, Dillon Gardner, Dmitry Mottl, Dmitry Vukolov, Dougal J. Sutherland, Dowon, drewmjohnston, Dror Atariah, Edward J Brown, Ekaterina Krivich, Elizabeth Sander, Emmanuel Arias, Eric Chang, Eric Larson, Erich Schubert, esvhd, Falak, Feda Curic, Federico Caselli, -Frank Hoang, Fibinse Xavier`, Finn O'Shea, Gabriel Marzinotto, Gabriel Vacaliuc, +Frank Hoang, Fibinse Xavier`, Finn O'Shea, Gabriel Marzinotto, Gabriel Vacaliuc, Gabriele Calvo, Gael Varoquaux, GauravAhlawat, Giuseppe Vettigli, Greg Gandenberger, Guillaume Fournier, Guillaume Lemaitre, Gustavo De Mari Pereira, Hanmin Qin, haroldfox, hhu-luqi, Hunter McGushion, Ian Sanders, JackLangerman, Jacopo diff --git a/doc/whats_new/v0.22.rst b/doc/whats_new/v0.22.rst index 6acabdb5c0037..e700ad569b168 100644 --- a/doc/whats_new/v0.22.rst +++ b/doc/whats_new/v0.22.rst @@ -2,6 +2,17 @@ .. currentmodule:: sklearn +.. _release_notes_0_22: + +============ +Version 0.22 +============ + +For a short description of the main highlights of the release, please refer to +:ref:`sphx_glr_auto_examples_release_highlights_plot_release_highlights_0_22_0.py`. + +.. include:: changelog_legend.inc + .. _changes_0_22_2: Version 0.22.2.post1 @@ -27,13 +38,13 @@ Changelog :mod:`sklearn.metrics` ...................... -- |Fix| Fixed a bug in :func:`metrics.plot_roc_curve` where +- |Fix| Fixed a bug in `metrics.plot_roc_curve` where the name of the estimator was passed in the :class:`metrics.RocCurveDisplay` instead of the parameter `name`. It results in a different plot when calling :meth:`metrics.RocCurveDisplay.plot` for the subsequent times. :pr:`16500` by :user:`Guillaume Lemaitre `. -- |Fix| Fixed a bug in :func:`metrics.plot_precision_recall_curve` where the +- |Fix| Fixed a bug in `metrics.plot_precision_recall_curve` where the name of the estimator was passed in the :class:`metrics.PrecisionRecallDisplay` instead of the parameter `name`. It results in a different plot when calling @@ -41,12 +52,12 @@ Changelog :pr:`16505` by :user:`Guillaume Lemaitre `. :mod:`sklearn.neighbors` -.............................. +........................ -- |Fix| Fix a bug which converted a list of arrays into a 2-D object +- |Fix| Fix a bug which converted a list of arrays into a 2-D object array instead of a 1-D array containing NumPy arrays. This bug was affecting :meth:`neighbors.NearestNeighbors.radius_neighbors`. - :pr:`16076` by :user:`Guillaume Lemaitre ` and + :pr:`16076` by :user:`Guillaume Lemaitre ` and :user:`Alex Shacked `. .. _changes_0_22_1: @@ -82,18 +93,18 @@ Changelog Follow-up of :pr:`15898` by :user:`Shivam Gargsya `. :pr:`15933` by :user:`Guillaume Lemaitre ` and `Olivier Grisel`_. -- |Fix| :func:`inspection.plot_partial_dependence` and +- |Fix| `inspection.plot_partial_dependence` and :meth:`inspection.PartialDependenceDisplay.plot` now consistently checks the number of axes passed in. :pr:`15760` by `Thomas Fan`_. :mod:`sklearn.metrics` ...................... -- |Fix| :func:`metrics.plot_confusion_matrix` now raises error when `normalize` +- |Fix| `metrics.plot_confusion_matrix` now raises error when `normalize` is invalid. Previously, it runs fine with no normalization. :pr:`15888` by `Hanmin Qin`_. -- |Fix| :func:`metrics.plot_confusion_matrix` now colors the label color +- |Fix| `metrics.plot_confusion_matrix` now colors the label color correctly to maximize contrast with its background. :pr:`15936` by `Thomas Fan`_ and :user:`DizietAsahi`. @@ -101,8 +112,8 @@ Changelog value of the ``zero_division`` keyword argument. :pr:`15879` by :user:`Bibhash Chandra Mitra `. -- |Fix| Fixed a bug in :func:`metrics.plot_confusion_matrix` to correctly - pass the `values_format` parameter to the :class:`ConfusionMatrixDisplay` +- |Fix| Fixed a bug in `metrics.plot_confusion_matrix` to correctly + pass the `values_format` parameter to the :class:`metrics.ConfusionMatrixDisplay` plot() call. :pr:`15937` by :user:`Stephen Blystone `. :mod:`sklearn.model_selection` @@ -118,7 +129,7 @@ Changelog .......................... - |Fix| Removed `abstractmethod` decorator for the method `_check_X` in - :class:`naive_bayes.BaseNB` that could break downstream projects inheriting + `naive_bayes.BaseNB` that could break downstream projects inheriting from this deprecated public base class. :pr:`15996` by :user:`Brigitta Sipőcz `. @@ -143,10 +154,10 @@ Changelog - |Fix| :func:`utils.check_array` now correctly converts pandas DataFrame with boolean columns to floats. :pr:`15797` by `Thomas Fan`_. -- |Fix| :func:`utils.check_is_fitted` accepts back an explicit ``attributes`` +- |Fix| :func:`utils.validation.check_is_fitted` accepts back an explicit ``attributes`` argument to check for specific attributes as explicit markers of a fitted estimator. When no explicit ``attributes`` are provided, only the attributes - that end with a underscore and do not start with double underscore are used + that end with an underscore and do not start with double underscore are used as "fitted" markers. The ``all_or_any`` argument is also no longer deprecated. This change is made to restore some backward compatibility with the behavior of this utility in version 0.21. :pr:`15947` by `Thomas Fan`_. @@ -158,12 +169,6 @@ Version 0.22.0 **December 3 2019** -For a short description of the main highlights of the release, please -refer to -:ref:`sphx_glr_auto_examples_release_highlights_plot_release_highlights_0_22_0.py`. - -.. include:: changelog_legend.inc - Website update -------------- @@ -390,12 +395,12 @@ Changelog :mod:`sklearn.decomposition` ............................ -- |Efficiency| :class:`decomposition.NMF(solver='mu')` fitted on sparse input +- |Efficiency| :class:`decomposition.NMF` with `solver="mu"` fitted on sparse input matrices now uses batching to avoid briefly allocating an array with size - (#non-zero elements, n_components). :pr:`15257` by `Mart Willocx `_. + (#non-zero elements, n_components). :pr:`15257` by :user:`Mart Willocx `. -- |Enhancement| :func:`decomposition.dict_learning()` and - :func:`decomposition.dict_learning_online()` now accept `method_max_iter` and +- |Enhancement| :func:`decomposition.dict_learning` and + :func:`decomposition.dict_learning_online` now accept `method_max_iter` and pass it to :meth:`decomposition.sparse_encode`. :issue:`12650` by `Adrin Jalali`_. @@ -403,15 +408,15 @@ Changelog :class:`decomposition.DictionaryLearning`, and :class:`decomposition.MiniBatchDictionaryLearning` now take a `transform_max_iter` parameter and pass it to either - :func:`decomposition.dict_learning()` or - :func:`decomposition.sparse_encode()`. :issue:`12650` by `Adrin Jalali`_. + :func:`decomposition.dict_learning` or + :func:`decomposition.sparse_encode`. :issue:`12650` by `Adrin Jalali`_. - |Enhancement| :class:`decomposition.IncrementalPCA` now accepts sparse matrices as input, converting them to dense in batches thereby avoiding the need to store the entire dense matrix at once. :pr:`13960` by :user:`Scott Gigante `. -- |Fix| :func:`decomposition.sparse_encode()` now passes the `max_iter` to the +- |Fix| :func:`decomposition.sparse_encode` now passes the `max_iter` to the underlying :class:`linear_model.LassoLars` when `algorithm='lasso_lars'`. :issue:`12650` by `Adrin Jalali`_. @@ -419,7 +424,7 @@ Changelog .................... - |Fix| :class:`dummy.DummyClassifier` now handles checking the existence - of the provided constant in multiouput cases. + of the provided constant in multioutput cases. :pr:`14908` by :user:`Martina G. Vilas `. - |API| The default value of the `strategy` parameter in @@ -451,7 +456,7 @@ Changelog - |Feature| Estimators now have an additional `warm_start` parameter that enables warm starting. :pr:`14012` by :user:`Johann Faouzi `. - |Feature| :func:`inspection.partial_dependence` and - :func:`inspection.plot_partial_dependence` now support the fast 'recursion' + `inspection.plot_partial_dependence` now support the fast 'recursion' method for both estimators. :pr:`13769` by `Nicolas Hug`_. - |Enhancement| for :class:`ensemble.HistGradientBoostingClassifier` the training loss or score is now monitored on a class-wise stratified @@ -503,7 +508,7 @@ Changelog - |Fix| Stacking and Voting estimators now ensure that their underlying estimators are either all classifiers or all regressors. :class:`ensemble.StackingClassifier`, :class:`ensemble.StackingRegressor`, - and :class:`ensemble.VotingClassifier` and :class:`VotingRegressor` + and :class:`ensemble.VotingClassifier` and :class:`ensemble.VotingRegressor` now raise consistent error messages. :pr:`15084` by `Guillaume Lemaitre`_. @@ -529,10 +534,10 @@ Changelog :pr:`14602` by :user:`Gaurav Chawla `. - |Fix| Functions created by ``build_preprocessor`` and ``build_analyzer`` of - :class:`feature_extraction.text.VectorizerMixin` can now be pickled. + `feature_extraction.text.VectorizerMixin` can now be pickled. :pr:`14430` by :user:`Dillon Niederhut `. -- |Fix| :func:`feature_extraction.text.strip_accents_unicode` now correctly +- |Fix| `feature_extraction.text.strip_accents_unicode` now correctly removes accents from strings that are in NFKD normalized form. :pr:`15100` by :user:`Daniel Grady `. @@ -548,8 +553,8 @@ Changelog :mod:`sklearn.feature_selection` ................................ -- |Enhancement| Updated the following :mod:`feature_selection` estimators to allow - NaN/Inf values in ``transform`` and ``fit``: +- |Enhancement| Updated the following :mod:`sklearn.feature_selection` + estimators to allow NaN/Inf values in ``transform`` and ``fit``: :class:`feature_selection.RFE`, :class:`feature_selection.RFECV`, :class:`feature_selection.SelectFromModel`, and :class:`feature_selection.VarianceThreshold`. Note that if the underlying @@ -570,7 +575,7 @@ Changelog of generic objects (e.g. strings, trees, graphs, etc.) as the ``X`` argument to their training/prediction methods. A user-defined kernel should be provided for computing the kernel matrix among - the generic objects, and should inherit from :class:`gaussian_process.kernels.GenericKernelMixin` + the generic objects, and should inherit from `gaussian_process.kernels.GenericKernelMixin` to notify the GPR/GPC model that it handles non-vectorial samples. :pr:`15557` by :user:`Yu-Hang Tang `. @@ -603,7 +608,7 @@ Changelog - |Fix| :class:`impute.IterativeImputer` now works when there is only one feature. By :user:`Sergey Feldman `. -- |Fix| Fixed a bug in :class:`impute.IterativeImputer` where features where +- |Fix| Fixed a bug in :class:`impute.IterativeImputer` where features were imputed in the reverse desired order with ``imputation_order`` either ``"ascending"`` or ``"descending"``. :pr:`15393` by :user:`Venkatachalam N `. @@ -616,18 +621,18 @@ Changelog respect to a given scoring function. :issue:`13146` by `Thomas Fan`_. - |Feature| :func:`inspection.partial_dependence` and - :func:`inspection.plot_partial_dependence` now support the fast 'recursion' + `inspection.plot_partial_dependence` now support the fast 'recursion' method for :class:`ensemble.HistGradientBoostingClassifier` and :class:`ensemble.HistGradientBoostingRegressor`. :pr:`13769` by `Nicolas Hug`_. -- |Enhancement| :func:`inspection.plot_partial_dependence` has been extended to +- |Enhancement| `inspection.plot_partial_dependence` has been extended to now support the new visualization API described in the :ref:`User Guide `. :pr:`14646` by `Thomas Fan`_. - |Enhancement| :func:`inspection.partial_dependence` accepts pandas DataFrame and :class:`pipeline.Pipeline` containing :class:`compose.ColumnTransformer`. - In addition :func:`inspection.plot_partial_dependence` will use the column + In addition `inspection.plot_partial_dependence` will use the column names by default when a dataframe is passed. :pr:`14028` and :pr:`15429` by `Guillaume Lemaitre`_. @@ -712,14 +717,15 @@ Changelog :mod:`sklearn.metrics` ...................... -- |MajorFeature| :func:`metrics.plot_roc_curve` has been added to plot roc +- |MajorFeature| `metrics.plot_roc_curve` has been added to plot roc curves. This function introduces the visualization API described in the :ref:`User Guide `. :pr:`14357` by `Thomas Fan`_. - |Feature| Added a new parameter ``zero_division`` to multiple classification - metrics: :func:`precision_score`, :func:`recall_score`, :func:`f1_score`, - :func:`fbeta_score`, :func:`precision_recall_fscore_support`, - :func:`classification_report`. This allows to set returned value for + metrics: :func:`metrics.precision_score`, :func:`metrics.recall_score`, + :func:`metrics.f1_score`, :func:`metrics.fbeta_score`, + :func:`metrics.precision_recall_fscore_support`, + :func:`metrics.classification_report`. This allows to set returned value for ill-defined metrics. :pr:`14900` by :user:`Marc Torrellas Socastro `. @@ -732,16 +738,16 @@ Changelog Gain and Normalized Discounted Cumulative Gain. :pr:`9951` by :user:`JÊrôme Dockès `. -- |Feature| :func:`metrics.plot_precision_recall_curve` has been added to plot +- |Feature| `metrics.plot_precision_recall_curve` has been added to plot precision recall curves. :pr:`14936` by `Thomas Fan`_. -- |Feature| :func:`metrics.plot_confusion_matrix` has been added to plot +- |Feature| `metrics.plot_confusion_matrix` has been added to plot confusion matrices. :pr:`15083` by `Thomas Fan`_. - |Feature| Added multiclass support to :func:`metrics.roc_auc_score` with corresponding scorers `'roc_auc_ovr'`, `'roc_auc_ovo'`, `'roc_auc_ovr_weighted'`, and `'roc_auc_ovo_weighted'`. - :pr:`12789` and :pr:`15274` by + :pr:`12789` and :pr:`15274` by :user:`Kathy Chen `, :user:`Mohamed Maskani `, and `Thomas Fan`_. @@ -877,7 +883,7 @@ Changelog ............................. - |Feature| Add `max_fun` parameter in - :class:`neural_network.BaseMultilayerPerceptron`, + `neural_network.BaseMultilayerPerceptron`, :class:`neural_network.MLPRegressor`, and :class:`neural_network.MLPClassifier` to give control over maximum number of function evaluation to not meet ``tol`` improvement. @@ -918,7 +924,7 @@ Changelog - |Fix| :class:`model_selection.GridSearchCV` and `model_selection.RandomizedSearchCV` now supports the - :term:`_pairwise` property, which prevents an error during cross-validation + `_pairwise` property, which prevents an error during cross-validation for estimators with pairwise inputs (such as :class:`neighbors.KNeighborsClassifier` when :term:`metric` is set to 'precomputed'). @@ -949,7 +955,7 @@ Changelog :class:`svm.OneClassSVM` was previously non-initialized, and had size 2. It has now size 1 with the correct value. :pr:`15099` by `Nicolas Hug`_. -- |Fix| fixed a bug in :class:`BaseLibSVM._sparse_fit` where n_SV=0 raised a +- |Fix| fixed a bug in `BaseLibSVM._sparse_fit` where n_SV=0 raised a ZeroDivisionError. :pr:`14894` by :user:`Danna Naser `. - |Fix| The liblinear solver now supports ``sample_weight``. @@ -993,14 +999,14 @@ Changelog :func:`~utils.estimator_checks.parametrize_with_checks`, to parametrize estimator checks for a list of estimators. :pr:`14381` by `Thomas Fan`_. -- |Feature| A new random variable, :class:`utils.fixes.loguniform` implements a +- |Feature| A new random variable, `utils.fixes.loguniform` implements a log-uniform random variable (e.g., for use in RandomizedSearchCV). For example, the outcomes ``1``, ``10`` and ``100`` are all equally likely for ``loguniform(1, 100)``. See :issue:`11232` by :user:`Scott Sievert ` and :user:`Nathaniel Saul `, - and `SciPy PR 10815 `. + and `SciPy PR 10815 `_. -- |Enhancement| :func:`utils.safe_indexing` (now deprecated) accepts an +- |Enhancement| `utils.safe_indexing` (now deprecated) accepts an ``axis`` parameter to index array-like across rows and columns. The column indexing can be done on NumPy array, SciPy sparse matrix, and Pandas DataFrame. An additional refactoring was done. :pr:`14035` and :pr:`14475` @@ -1092,8 +1098,8 @@ These changes mostly affect library developers. :pr:`14336` by :user:`Gregory Dexter `. - Added two common multioutput estimator tests - :func:`~utils.estimator_checks.check_classifier_multioutput` and - :func:`~utils.estimator_checks.check_regressor_multioutput`. + `utils.estimator_checks.check_classifier_multioutput` and + `utils.estimator_checks.check_regressor_multioutput`. :pr:`13392` by :user:`Rok Mihevc `. - |Fix| Added ``check_transformer_data_not_an_array`` to checks where missing @@ -1102,8 +1108,7 @@ These changes mostly affect library developers. to be overridable only once. :pr:`14884` by `Andreas MÃŧller`_. -Code and Documentation Contributors ------------------------------------ +.. rubric:: Code and documentation contributors Thanks to everyone who has contributed to the maintenance and improvement of the project since version 0.21, including: diff --git a/doc/whats_new/v0.23.rst b/doc/whats_new/v0.23.rst index 598d9adc5cef4..379fa7adfe7aa 100644 --- a/doc/whats_new/v0.23.rst +++ b/doc/whats_new/v0.23.rst @@ -2,6 +2,17 @@ .. currentmodule:: sklearn +.. _release_notes_0_23: + +============ +Version 0.23 +============ + +For a short description of the main highlights of the release, please refer to +:ref:`sphx_glr_auto_examples_release_highlights_plot_release_highlights_0_23_0.py`. + +.. include:: changelog_legend.inc + .. _changes_0_23_2: Version 0.23.2 @@ -65,7 +76,7 @@ Changelog :mod:`sklearn.ensemble` ....................... -- |Fix| Fixed bug in :class:`ensemble.MultinomialDeviance` where the +- |Fix| Fixed bug in `ensemble.MultinomialDeviance` where the average of logloss was incorrectly calculated as sum of logloss. :pr:`17694` by :user:`Markus Rempfler ` and :user:`Tsutomu Kusanagi `. @@ -152,12 +163,6 @@ Version 0.23.0 **May 12 2020** -For a short description of the main highlights of the release, please -refer to -:ref:`sphx_glr_auto_examples_release_highlights_plot_release_highlights_0_23_0.py`. - - -.. include:: changelog_legend.inc Enforcing keyword-only arguments -------------------------------- @@ -186,7 +191,7 @@ random sampling procedures. - |Fix| :class:`cluster.KMeans` with ``algorithm="elkan"`` and ``algorithm="full"``. - |Fix| :class:`cluster.Birch` -- |Fix| :func:`compose.ColumnTransformer.get_feature_names` +- |Fix| `compose.ColumnTransformer.get_feature_names` - |Fix| :func:`compose.ColumnTransformer.fit` - |Fix| :func:`datasets.make_multilabel_classification` - |Fix| :class:`decomposition.PCA` with `n_components='mle'` @@ -210,7 +215,7 @@ random sampling procedures. - |Fix| :class:`preprocessing.StandardScaler` with `partial_fit` and sparse input. - |Fix| :class:`preprocessing.Normalizer` with norm='max' -- |Fix| Any model using the :func:`svm.libsvm` or the :func:`svm.liblinear` solver, +- |Fix| Any model using the `svm.libsvm` or the `svm.liblinear` solver, including :class:`svm.LinearSVC`, :class:`svm.LinearSVR`, :class:`svm.NuSVC`, :class:`svm.NuSVR`, :class:`svm.OneClassSVM`, :class:`svm.SVC`, :class:`svm.SVR`, :class:`linear_model.LogisticRegression`. @@ -269,7 +274,7 @@ Changelog could not have a `np.int64` type. :pr:`16484` by :user:`Jeremie du Boisberranger `. -- |Fix| :class:`cluster.AgglomerativeCluClustering` add specific error when +- |Fix| :class:`cluster.AgglomerativeClustering` add specific error when distance matrix is not square and `affinity=precomputed`. :pr:`16257` by :user:`Simona Maggio `. @@ -320,10 +325,11 @@ Changelog by :user:`Stephanie Andrews ` and :user:`Reshama Shaikh `. -- |Feature| embedded dataset loaders :func:`load_breast_cancer`, - :func:`load_diabetes`, :func:`load_digits`, :func:`load_iris`, - :func:`load_linnerud` and :func:`load_wine` now support loading as a pandas - ``DataFrame`` by setting `as_frame=True`. :pr:`15980` by :user:`wconnell` and +- |Feature| embedded dataset loaders :func:`datasets.load_breast_cancer`, + :func:`datasets.load_diabetes`, :func:`datasets.load_digits`, + :func:`datasets.load_iris`, :func:`datasets.load_linnerud` and + :func:`datasets.load_wine` now support loading as a pandas ``DataFrame`` by + setting `as_frame=True`. :pr:`15980` by :user:`wconnell` and :user:`Reshama Shaikh `. - |Enhancement| Added ``return_centers`` parameter in @@ -341,7 +347,7 @@ Changelog :pr:`16006` by :user:`Rushabh Vasani `. - |API| The `StreamHandler` was removed from `sklearn.logger` to avoid - double logging of messages in common cases where a hander is attached + double logging of messages in common cases where a handler is attached to the root logger, and to follow the Python logging documentation recommendation for libraries to leave the log message handling to users and application code. :pr:`16451` by :user:`Christoph Deil `. @@ -353,8 +359,8 @@ Changelog :func:`decomposition.non_negative_factorization` now preserves float32 dtype. :pr:`16280` by :user:`Jeremie du Boisberranger `. -- |Enhancement| :func:`TruncatedSVD.transform` is now faster on given sparse - ``csc`` matrices. :pr:`16837` by :user:`wornbb`. +- |Enhancement| :func:`decomposition.TruncatedSVD.transform` is now faster on + given sparse ``csc`` matrices. :pr:`16837` by :user:`wornbb`. - |Fix| :class:`decomposition.PCA` with a float `n_components` parameter, will exclusively choose the components that explain the variance greater than @@ -484,7 +490,7 @@ Changelog ......................... - |Feature| :func:`inspection.partial_dependence` and - :func:`inspection.plot_partial_dependence` now support the fast 'recursion' + `inspection.plot_partial_dependence` now support the fast 'recursion' method for :class:`ensemble.RandomForestRegressor` and :class:`tree.DecisionTreeRegressor`. :pr:`15864` by `Nicolas Hug`_. @@ -505,7 +511,7 @@ Changelog `. - |Efficiency| :class:`linear_model.RidgeCV` and - :class:`linear_model.RidgeClassifierCV` now does not allocate a + :class:`linear_model.RidgeClassifierCV` now do not allocate a potentially large array to store dual coefficients for all hyperparameters during its `fit`, nor an array to store all error or LOO predictions unless `store_cv_values` is `True`. @@ -565,7 +571,7 @@ Changelog :mod:`sklearn.metrics` ...................... -- |Enhancement| :func:`metrics.pairwise.pairwise_distances_chunked` now allows +- |Enhancement| :func:`metrics.pairwise_distances_chunked` now allows its ``reduce_func`` to not have a return value, enabling in-place operations. :pr:`16397` by `Joel Nothman`_. @@ -584,11 +590,11 @@ Changelog - |API| Changed the formatting of values in :meth:`metrics.ConfusionMatrixDisplay.plot` and - :func:`metrics.plot_confusion_matrix` to pick the shorter format (either '2g' + `metrics.plot_confusion_matrix` to pick the shorter format (either '2g' or 'd'). :pr:`16159` by :user:`Rick Mackenbach ` and `Thomas Fan`_. -- |API| From version 0.25, :func:`metrics.pairwise.pairwise_distances` will no +- |API| From version 0.25, :func:`metrics.pairwise_distances` will no longer automatically compute the ``VI`` parameter for Mahalanobis distance and the ``V`` parameter for seuclidean distance if ``Y`` is passed. The user will be expected to compute this parameter on the training data of their @@ -607,7 +613,7 @@ Changelog `method="predict_proba"` when `y=None`. :pr:`15918` by :user:`Luca Kubin `. -- |Fix| :func:`model_selection.fit_grid_point` is deprecated in 0.23 and will +- |Fix| `model_selection.fit_grid_point` is deprecated in 0.23 and will be removed in 0.25. :pr:`16401` by :user:`Arie Pratama Sutiono ` @@ -703,7 +709,7 @@ Changelog crude "modulo" postprocessor used to get a random number in a bounded interval was replaced by the tweaked Lemire method as suggested by `this blog post `_. - Any model using the :func:`svm.libsvm` or the :func:`svm.liblinear` solver, + Any model using the `svm.libsvm` or the `svm.liblinear` solver, including :class:`svm.LinearSVC`, :class:`svm.LinearSVR`, :class:`svm.NuSVC`, :class:`svm.NuSVR`, :class:`svm.OneClassSVM`, :class:`svm.SVC`, :class:`svm.SVR`, :class:`linear_model.LogisticRegression`, @@ -712,7 +718,7 @@ Changelog :pr:`13511` by :user:`Sylvain MariÊ `. - |Fix| Fix use of custom kernel not taking float entries such as string - kernels in :class:`svm.SVC` and :class:`svm.SVR`. Note that custom kennels + kernels in :class:`svm.SVC` and :class:`svm.SVR`. Note that custom kernels are now expected to validate their input where they previously received valid numeric arrays. :pr:`11296` by `Alexandre Gramfort`_ and :user:`Georgi Peev `. @@ -756,7 +762,7 @@ Changelog matrix from a pandas DataFrame that contains only `SparseArray` columns. :pr:`16728` by `Thomas Fan`_. -- |Enhancement| :func:`utils.validation.check_array` supports pandas' +- |Enhancement| :func:`utils.check_array` supports pandas' nullable integer dtype with missing values when `force_all_finite` is set to `False` or `'allow-nan'` in which case the data is converted to floating point values where `pd.NA` values are replaced by `np.nan`. As a consequence, @@ -776,14 +782,14 @@ Changelog in the MRO for `_get_tags()` to work properly. :pr:`16950` by `Nicolas Hug`_. -- |FIX| :func:`utils.all_estimators` now only returns public estimators. +- |FIX| `utils.all_estimators` now only returns public estimators. :pr:`15380` by `Thomas Fan`_. Miscellaneous ............. - |MajorFeature| Adds a HTML representation of estimators to be shown in - a jupyter notebook or lab. This visualization is acitivated by setting the + a jupyter notebook or lab. This visualization is activated by setting the `display` option in :func:`sklearn.set_config`. :pr:`14180` by `Thomas Fan`_. @@ -810,8 +816,7 @@ Miscellaneous always possible to quickly inspect the parameters of any estimator using `est.get_params(deep=False)`. :pr:`17061` by `Nicolas Hug`_. -Code and Documentation Contributors ------------------------------------ +.. rubric:: Code and documentation contributors Thanks to everyone who has contributed to the maintenance and improvement of the project since version 0.22, including: diff --git a/doc/whats_new/v0.24.rst b/doc/whats_new/v0.24.rst index 68ea8ba0f7a72..70bfa0907c146 100644 --- a/doc/whats_new/v0.24.rst +++ b/doc/whats_new/v0.24.rst @@ -2,12 +2,23 @@ .. currentmodule:: sklearn +.. _release_notes_0_24: + +============ +Version 0.24 +============ + +For a short description of the main highlights of the release, please refer to +:ref:`sphx_glr_auto_examples_release_highlights_plot_release_highlights_0_24_0.py`. + +.. include:: changelog_legend.inc + .. _changes_0_24_2: Version 0.24.2 ============== -**TBD 2021** +**April 2021** Changelog --------- @@ -15,10 +26,26 @@ Changelog :mod:`sklearn.compose` ...................... -- |Fix| :meth:`compose.ColumnTransformer.get_feature_names` does not call - :term:`get_feature_names` on transformers with an empty column selection. +- |Fix| `compose.ColumnTransformer.get_feature_names` does not call + `get_feature_names` on transformers with an empty column selection. :pr:`19579` by `Thomas Fan`_. +:mod:`sklearn.cross_decomposition` +.................................. + +- |Fix| Fixed a regression in :class:`cross_decomposition.CCA`. :pr:`19646` + by `Thomas Fan`_. + +- |Fix| :class:`cross_decomposition.PLSRegression` raises warning for + constant y residuals instead of a `StopIteration` error. :pr:`19922` + by `Thomas Fan`_. + +:mod:`sklearn.decomposition` +............................ + +- |Fix| Fixed a bug in :class:`decomposition.KernelPCA`'s + ``inverse_transform``. :pr:`19732` by :user:`Kei Ishikawa `. + :mod:`sklearn.ensemble` ....................... @@ -26,6 +53,31 @@ Changelog with `sample_weight` parameter and `least_absolute_deviation` loss function. :pr:`19407` by :user:`Vadim Ushtanit `. +:mod:`sklearn.feature_extraction` +................................. + +- |Fix| Fixed a bug to support multiple strings for a category when + `sparse=False` in :class:`feature_extraction.DictVectorizer`. + :pr:`19982` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.gaussian_process` +............................... + +- |Fix| Avoid explicitly forming inverse covariance matrix in + :class:`gaussian_process.GaussianProcessRegressor` when set to output + standard deviation. With certain covariance matrices this inverse is unstable + to compute explicitly. Calling Cholesky solver mitigates this issue in + computation. + :pr:`19939` by :user:`Ian Halvic `. + +- |Fix| Avoid division by zero when scaling constant target in + :class:`gaussian_process.GaussianProcessRegressor`. It was due to a std. dev. + equal to 0. Now, such case is detected and the std. dev. is affected to 1 + avoiding a division by zero and thus the presence of NaN values in the + normalized target. + :pr:`19703` by :user:`sobkevich`, :user:`Boris VillazÃŗn-Terrazas ` + and :user:`Alexandr Fonari `. + :mod:`sklearn.linear_model` ........................... @@ -33,13 +85,38 @@ Changelog sample_weight object is not modified anymore. :pr:`19182` by :user:`Yosuke KOBAYASHI `. +:mod:`sklearn.metrics` +...................... + +- |Fix| :func:`metrics.top_k_accuracy_score` now supports multiclass + problems where only two classes appear in `y_true` and all the classes + are specified in `labels`. + :pr:`19721` by :user:`Joris Clement `. + :mod:`sklearn.model_selection` .............................. - |Fix| :class:`model_selection.RandomizedSearchCV` and - :class:`model_selection.GridSearchCV` now correctly shows the score for + :class:`model_selection.GridSearchCV` now correctly show the score for single metrics and verbose > 2. :pr:`19659` by `Thomas Fan`_. +- |Fix| Some values in the `cv_results_` attribute of + :class:`model_selection.HalvingRandomSearchCV` and + :class:`model_selection.HalvingGridSearchCV` were not properly converted to + numpy arrays. :pr:`19211` by `Nicolas Hug`_. + +- |Fix| The `fit` method of the successive halving parameter search + (:class:`model_selection.HalvingGridSearchCV`, and + :class:`model_selection.HalvingRandomSearchCV`) now correctly handles the + `groups` parameter. :pr:`19847` by :user:`Xiaoyu Chai `. + +:mod:`sklearn.multioutput` +.......................... + +- |Fix| :class:`multioutput.MultiOutputRegressor` now works with estimators + that dynamically define `predict` during fitting, such as + :class:`ensemble.StackingRegressor`. :pr:`19308` by `Thomas Fan`_. + :mod:`sklearn.preprocessing` ............................ @@ -48,6 +125,17 @@ Changelog `'use_encoded_value'` strategies. :pr:`19234` by `Guillaume Lemaitre `. +- |Fix| Fix encoder categories having dtype='S' + :class:`preprocessing.OneHotEncoder` and + :class:`preprocessing.OrdinalEncoder`. + :pr:`19727` by :user:`Andrew Delong `. + +- |Fix| :meth:`preprocessing.OrdinalEncoder.transform` correctly handles + unknown values for string dtypes. :pr:`19888` by `Thomas Fan`_. + +- |Fix| :meth:`preprocessing.OneHotEncoder.fit` no longer alters the `drop` + parameter. :pr:`19924` by `Thomas Fan`_. + :mod:`sklearn.semi_supervised` .............................. @@ -56,9 +144,9 @@ Changelog :pr:`19271` by :user:`Zhaowei Wang `. :mod:`sklearn.tree` -....................... +................... -- |Fix| Fix a bug in `fit` of :class:`tree.BaseDecisionTree` that caused +- |Fix| Fix a bug in `fit` of `tree.BaseDecisionTree` that caused segmentation faults under certain conditions. `fit` now deep copies the `Criterion` object to prevent shared concurrent accesses. :pr:`19580` by :user:`Samuel Brice ` and @@ -114,14 +202,6 @@ Version 0.24.0 **December 2020** -For a short description of the main highlights of the release, please -refer to -:ref:`sphx_glr_auto_examples_release_highlights_plot_release_highlights_0_24_0.py`. - -.. include:: changelog_legend.inc - -Put the changes in their relevant module. - Changed models -------------- @@ -139,6 +219,9 @@ random sampling procedures. - |Fix| :class:`linear_model.Perceptron` when `penalty='elasticnet'`. +- |Fix| Change in the random sampling procedures for the center initialization + of :class:`cluster.KMeans`. + Details are listed in the changelog below. (While we are trying to better inform users by providing this information, we @@ -240,12 +323,6 @@ Changelog - |Fix| Increases the stability of :class:`cross_decomposition.CCA` :pr:`18746` by `Thomas Fan`_. -- |API| For :class:`cross_decomposition.NMF`, - the `init` value, when 'init=None' and - n_components <= min(n_samples, n_features) will be changed from - `'nndsvd'` to `'nndsvda'` in 1.1 (renaming of 0.26). - :pr:`18525` by :user:`Chiara Marmo `. - - |API| The bounds of the `n_components` parameter is now restricted: - into `[1, min(n_samples, n_features, n_targets)]`, for @@ -290,13 +367,13 @@ Changelog unless data is sparse. :pr:`17396` by :user:`Jiaxiang `. -- |Enhancement| :func:`datasets.fetch_covtype` now now supports the optional +- |Enhancement| :func:`datasets.fetch_covtype` now supports the optional argument `as_frame`; when it is set to True, the returned Bunch object's `data` and `frame` members are pandas DataFrames, and the `target` member is a pandas Series. :pr:`17491` by :user:`Alex Liang `. -- |Enhancement| :func:`datasets.fetch_kddcup99` now now supports the optional +- |Enhancement| :func:`datasets.fetch_kddcup99` now supports the optional argument `as_frame`; when it is set to True, the returned Bunch object's `data` and `frame` members are pandas DataFrames, and the `target` member is a pandas Series. @@ -315,6 +392,12 @@ Changelog :mod:`sklearn.decomposition` ............................ +- |API| For :class:`decomposition.NMF`, + the `init` value, when 'init=None' and + n_components <= min(n_samples, n_features) will be changed from + `'nndsvd'` to `'nndsvda'` in 1.1 (renaming of 0.26). + :pr:`18525` by :user:`Chiara Marmo `. + - |Enhancement| :func:`decomposition.FactorAnalysis` now supports the optional argument `rotation`, which can take the value `None`, `'varimax'` or `'quartimax'`. :pr:`11064` by :user:`Jona Sassenhagen `. @@ -322,8 +405,8 @@ Changelog - |Enhancement| :class:`decomposition.NMF` now supports the optional parameter `regularization`, which can take the values `None`, 'components', 'transformation' or 'both', in accordance with - :func:`decomposition.NMF.non_negative_factorization`. - :pr:`17414` by :user:`Bharat Raghunathan `. + `decomposition.NMF.non_negative_factorization`. + :pr:`17414` by :user:`Bharat Raghunathan `. - |Fix| :class:`decomposition.KernelPCA` behaviour is now more consistent between 32-bits and 64-bits data input when the kernel has small positive @@ -332,14 +415,15 @@ Changelog :pr:`18149` by :user:`Sylvain MariÊ `. - |Fix| Fix :class:`decomposition.SparseCoder` such that it follows - scikit-learn API and support cloning. The attribute `components_` is + scikit-learn API and supports cloning. The attribute `components_` is deprecated in 0.24 and will be removed in 1.1 (renaming of 0.26). This attribute was redundant with the `dictionary` attribute and constructor parameter. :pr:`17679` by :user:`Xavier DuprÊ `. -- |Fix| :meth:`TruncatedSVD.fit_transform` consistently returns the same - as :meth:`TruncatedSVD.fit` followed by :meth:`TruncatedSVD.transform`. +- |Fix| :meth:`decomposition.TruncatedSVD.fit_transform` consistently returns + the same as :meth:`decomposition.TruncatedSVD.fit` followed by + :meth:`decomposition.TruncatedSVD.transform`. :pr:`18528` by :user:`Albert Villanova del Moral ` and :user:`Ruifeng Zheng `. @@ -394,8 +478,8 @@ Changelog :mod:`sklearn.exceptions` ......................... -- |API| :class:`exceptions.ChangedBehaviorWarning` and - :class:`exceptions.NonBLASDotWarning` are deprecated and will be removed in +- |API| `exceptions.ChangedBehaviorWarning` and + `exceptions.NonBLASDotWarning` are deprecated and will be removed in 1.1 (renaming of 0.26). :pr:`17804` by `Adrin Jalali`_. @@ -406,8 +490,8 @@ Changelog values for one categorical feature. :pr:`17367` by :user:`Peng Yu ` and :user:`Chiara Marmo `. -- |Fix| :class:`feature_extraction.CountVectorizer` raises an issue if a - custom token pattern which capture more than one group is provided. +- |Fix| :class:`feature_extraction.text.CountVectorizer` raises an issue if a + custom token pattern which captures more than one group is provided. :pr:`15427` by :user:`Gangesh Gudmalwar ` and :user:`Erin R Hoffman `. @@ -440,7 +524,7 @@ Changelog ............................... - |Enhancement| A new method - :meth:`gaussian_process.Kernel._check_bounds_params` is called after + `gaussian_process.kernel._check_bounds_params` is called after fitting a Gaussian Process and raises a ``ConvergenceWarning`` if the bounds of the hyperparameters are too tight. :issue:`12638` by :user:`Sylvain Lannuzel `. @@ -475,7 +559,7 @@ Changelog ......................... - |Feature| :func:`inspection.partial_dependence` and - :func:`inspection.plot_partial_dependence` now support calculating and + `inspection.plot_partial_dependence` now support calculating and plotting Individual Conditional Expectation (ICE) curves controlled by the ``kind`` parameter. :pr:`16619` by :user:`Madhura Jayratne `. @@ -572,7 +656,7 @@ Changelog generalization of :func:`metrics.top_k_accuracy_score`, the difference is that a prediction is considered correct as long as the true label is associated with one of the `k` highest predicted scores. - :func:`accuracy_score` is the special case of `k = 1`. + :func:`metrics.accuracy_score` is the special case of `k = 1`. :pr:`16625` by :user:`Geoffrey Bolmier `. - |Feature| Added :func:`metrics.det_curve` to compute Detection Error Tradeoff @@ -580,7 +664,7 @@ Changelog :pr:`10591` by :user:`Jeremy Karnowski ` and :user:`Daniel Mohns `. -- |Feature| Added :func:`metrics.plot_det_curve` and +- |Feature| Added `metrics.plot_det_curve` and :class:`metrics.DetCurveDisplay` to ease the plot of DET curves. :pr:`18176` by :user:`Guillaume Lemaitre `. @@ -594,25 +678,21 @@ Changelog Rand index. :pr:`17412` by :user:`Uwe F Mayer `. -- |Feature| :func:`metrics.plot_confusion_matrix` now supports making colorbar +- |Feature| `metrics.plot_confusion_matrix` now supports making colorbar optional in the matplotlib plot by setting `colorbar=False`. :pr:`17192` by :user:`Avi Gupta ` -- |Feature| :func:`metrics.plot_confusion_matrix` now supports making colorbar - optional in the matplotlib plot by setting colorbar=False. :pr:`17192` by - :user:`Avi Gupta `. - - |Enhancement| Add `sample_weight` parameter to :func:`metrics.median_absolute_error`. :pr:`17225` by :user:`Lucy Liu `. - |Enhancement| Add `pos_label` parameter in - :func:`metrics.plot_precision_recall_curve` in order to specify the positive + `metrics.plot_precision_recall_curve` in order to specify the positive class to be used when computing the precision and recall statistics. :pr:`17569` by :user:`Guillaume Lemaitre `. - |Enhancement| Add `pos_label` parameter in - :func:`metrics.plot_roc_curve` in order to specify the positive + `metrics.plot_roc_curve` in order to specify the positive class to be used when computing the roc auc statistics. :pr:`17651` by :user:`Clara Matos `. @@ -633,7 +713,7 @@ Changelog :user:`Joseph Willard ` - |Fix| bug in :func:`metrics.hinge_loss` where error occurs when - ``y_true`` is missing some labels that are provided explictly in the + ``y_true`` is missing some labels that are provided explicitly in the ``labels`` parameter. :pr:`17935` by :user:`Cary Goltermann `. @@ -644,7 +724,7 @@ Changelog classifiers directly with string labeled target classes. :pr:`18114` by :user:`Guillaume Lemaitre `. -- |Fix| Fixed bug in :func:`metrics.plot_confusion_matrix` where error occurs +- |Fix| Fixed bug in `metrics.plot_confusion_matrix` where error occurs when `y_true` contains labels that were not previously seen by the classifier while the `labels` and `display_labels` parameters are set to `None`. :pr:`18405` by :user:`Thomas J. Fan ` and @@ -703,7 +783,7 @@ Changelog - |Enhancement| :class:`model_selection.GridSearchCV`, :class:`model_selection.RandomizedSearchCV` and :func:`model_selection.cross_validate` support `scoring` being a callable - returning a dictionary of of multiple metric names/values association. + returning a dictionary of multiple metric names/values association. :pr:`15126` by `Thomas Fan`_. :mod:`sklearn.multiclass` @@ -754,7 +834,7 @@ Changelog ........................ - |Efficiency| Speed up ``seuclidean``, ``wminkowski``, ``mahalanobis`` and - ``haversine`` metrics in :class:`neighbors.DistanceMetric` by avoiding + ``haversine`` metrics in `neighbors.DistanceMetric` by avoiding unexpected GIL acquiring in Cython when setting ``n_jobs>1`` in :class:`neighbors.KNeighborsClassifier`, :class:`neighbors.KNeighborsRegressor`, @@ -764,13 +844,13 @@ Changelog and by validating data out of loops. :pr:`17038` by :user:`Wenbo Zhao `. -- |Efficiency| :class:`neighbors.NeighborsBase` benefits of an improved +- |Efficiency| `neighbors.NeighborsBase` benefits of an improved `algorithm = 'auto'` heuristic. In addition to the previous set of rules, now, when the number of features exceeds 15, `brute` is selected, assuming the data intrinsic dimensionality is too high for tree-based methods. :pr:`17148` by :user:`Geoffrey Bolmier `. -- |Fix| :class:`neighbors.BinaryTree` +- |Fix| `neighbors.BinaryTree` will raise a `ValueError` when fitting on data array having points with different dimensions. :pr:`18691` by :user:`Chiara Marmo `. @@ -803,7 +883,7 @@ Changelog :class:`neural_network.MLPRegressor`. :pr:`17759` by :user:`Srimukh Sripada `. -- |Fix| Fix method :func:`fit` of :class:`neural_network.MLPClassifier` +- |Fix| Fix method :meth:`neural_network.MLPClassifier.fit` not iterating to ``max_iter`` if warm started. :pr:`18269` by :user:`Norbert Preining ` and :user:`Guillaume Lemaitre `. @@ -881,7 +961,7 @@ Changelog - |Enhancement| invoke SciPy BLAS API for SVM kernel function in ``fit``, ``predict`` and related methods of :class:`svm.SVC`, :class:`svm.NuSVC`, - :class:`svm.SVR`, :class:`svm.NuSVR`, :class:`OneClassSVM`. + :class:`svm.SVR`, :class:`svm.NuSVR`, :class:`svm.OneClassSVM`. :pr:`16530` by :user:`Shuhua Fan `. :mod:`sklearn.tree` @@ -896,7 +976,7 @@ Changelog - |API| The parameter ``X_idx_sorted`` is now deprecated in :meth:`tree.DecisionTreeClassifier.fit` and - :meth:`tree.DecisionTreeRegressor.fit`, and has not effect. + :meth:`tree.DecisionTreeRegressor.fit`, and has no effect. :pr:`17614` by :user:`Juan Carlos Alfaro JimÊnez `. :mod:`sklearn.utils` @@ -908,10 +988,10 @@ Changelog with different sample order :pr:`17598` by :user:`Jason Ngo `. - |Enhancement| Add support for weights in - :func:`utils.sparse_func.incr_mean_variance_axis`. + `utils.sparse_func.incr_mean_variance_axis`. By :user:`Maria Telenczuk ` and :user:`Alex Gramfort `. -- |Fix| Raise ValueError with clear error message in :func:`check_array` +- |Fix| Raise ValueError with clear error message in :func:`utils.check_array` for sparse DataFrames with mixed types. :pr:`17992` by :user:`Thomas J. Fan ` and :user:`Alex Shacked `. @@ -921,7 +1001,7 @@ Changelog :pr:`17644` by :user:`Qi Zhang `. - |Fix| Check that we raise proper error when axis=1 and the - dimensions do not match in :func:`utils.sparse_func.incr_mean_variance_axis`. + dimensions do not match in `utils.sparse_func.incr_mean_variance_axis`. By :user:`Alex Gramfort `. Miscellaneous @@ -931,8 +1011,7 @@ Miscellaneous when `print_changed_only=True`, especially with meta-estimators. :pr:`18508` by :user:`Nathan C. `. -Code and Documentation Contributors ------------------------------------ +.. rubric:: Code and documentation contributors Thanks to everyone who has contributed to the maintenance and improvement of the project since version 0.23, including: diff --git a/doc/whats_new/v1.0.rst b/doc/whats_new/v1.0.rst index a1f21723bac28..d5e4a2c302d6a 100644 --- a/doc/whats_new/v1.0.rst +++ b/doc/whats_new/v1.0.rst @@ -2,17 +2,310 @@ .. currentmodule:: sklearn +.. _release_notes_1_0: + +=========== +Version 1.0 +=========== + +For a short description of the main highlights of the release, please refer to +:ref:`sphx_glr_auto_examples_release_highlights_plot_release_highlights_1_0_0.py`. + +.. include:: changelog_legend.inc + +.. _changes_1_0_2: + +Version 1.0.2 +============= + +**December 2021** + +- |Fix| :class:`cluster.Birch`, + :class:`feature_selection.RFECV`, :class:`ensemble.RandomForestRegressor`, + :class:`ensemble.RandomForestClassifier`, + :class:`ensemble.GradientBoostingRegressor`, and + :class:`ensemble.GradientBoostingClassifier` do not raise warning when fitted + on a pandas DataFrame anymore. :pr:`21578` by `Thomas Fan`_. + +Changelog +--------- + +:mod:`sklearn.cluster` +...................... + +- |Fix| Fixed an infinite loop in :func:`cluster.SpectralClustering` by + moving an iteration counter from try to except. + :pr:`21271` by :user:`Tyler Martin `. + +:mod:`sklearn.datasets` +....................... + +- |Fix| :func:`datasets.fetch_openml` is now thread safe. Data is first + downloaded to a temporary subfolder and then renamed. + :pr:`21833` by :user:`Siavash Rezazadeh `. + +:mod:`sklearn.decomposition` +............................ + +- |Fix| Fixed the constraint on the objective function of + :class:`decomposition.DictionaryLearning`, + :class:`decomposition.MiniBatchDictionaryLearning`, :class:`decomposition.SparsePCA` + and :class:`decomposition.MiniBatchSparsePCA` to be convex and match the referenced + article. :pr:`19210` by :user:`JÊrÊmie du Boisberranger `. + +:mod:`sklearn.ensemble` +....................... + +- |Fix| :class:`ensemble.RandomForestClassifier`, + :class:`ensemble.RandomForestRegressor`, + :class:`ensemble.ExtraTreesClassifier`, :class:`ensemble.ExtraTreesRegressor`, + and :class:`ensemble.RandomTreesEmbedding` now raise a ``ValueError`` when + ``bootstrap=False`` and ``max_samples`` is not ``None``. + :pr:`21295` :user:`Haoyin Xu `. + +- |Fix| Solve a bug in :class:`ensemble.GradientBoostingClassifier` where the + exponential loss was computing the positive gradient instead of the + negative one. + :pr:`22050` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.feature_selection` +................................ + +- |Fix| Fixed :class:`feature_selection.SelectFromModel` by improving support + for base estimators that do not set `feature_names_in_`. :pr:`21991` by + `Thomas Fan`_. + +:mod:`sklearn.impute` +..................... + +- |Fix| Fix a bug in :class:`linear_model.RidgeClassifierCV` where the method + `predict` was performing an `argmax` on the scores obtained from + `decision_function` instead of returning the multilabel indicator matrix. + :pr:`19869` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.linear_model` +........................... + +- |Fix| :class:`linear_model.LassoLarsIC` now correctly computes AIC + and BIC. An error is now raised when `n_features > n_samples` and + when the noise variance is not provided. + :pr:`21481` by :user:`Guillaume Lemaitre ` and + :user:`AndrÊs Babino `. + +:mod:`sklearn.manifold` +....................... + +- |Fix| Fixed an unnecessary error when fitting :class:`manifold.Isomap` with a + precomputed dense distance matrix where the neighbors graph has multiple + disconnected components. :pr:`21915` by `Tom Dupre la Tour`_. + +:mod:`sklearn.metrics` +...................... + +- |Fix| All :class:`sklearn.metrics.DistanceMetric` subclasses now correctly support + read-only buffer attributes. + This fixes a regression introduced in 1.0.0 with respect to 0.24.2. + :pr:`21694` by :user:`Julien Jerphanion `. + +- |Fix| All `sklearn.metrics.MinkowskiDistance` now accepts a weight + parameter that makes it possible to write code that behaves consistently both + with scipy 1.8 and earlier versions. In turn this means that all + neighbors-based estimators (except those that use `algorithm="kd_tree"`) now + accept a weight parameter with `metric="minkowski"` to yield results that + are always consistent with `scipy.spatial.distance.cdist`. + :pr:`21741` by :user:`Olivier Grisel `. + +:mod:`sklearn.multiclass` +......................... + +- |Fix| :meth:`multiclass.OneVsRestClassifier.predict_proba` does not error when + fitted on constant integer targets. :pr:`21871` by `Thomas Fan`_. + +:mod:`sklearn.neighbors` +........................ + +- |Fix| :class:`neighbors.KDTree` and :class:`neighbors.BallTree` correctly support + read-only buffer attributes. :pr:`21845` by `Thomas Fan`_. + +:mod:`sklearn.preprocessing` +............................ + +- |Fix| Fixes compatibility bug with NumPy 1.22 in :class:`preprocessing.OneHotEncoder`. + :pr:`21517` by `Thomas Fan`_. + +:mod:`sklearn.tree` +................... + +- |Fix| Prevents :func:`tree.plot_tree` from drawing out of the boundary of + the figure. :pr:`21917` by `Thomas Fan`_. + +- |Fix| Support loading pickles of decision tree models when the pickle has + been generated on a platform with a different bitness. A typical example is + to train and pickle the model on 64 bit machine and load the model on a 32 + bit machine for prediction. :pr:`21552` by :user:`Loïc Estève `. + +:mod:`sklearn.utils` +.................... + +- |Fix| :func:`utils.estimator_html_repr` now escapes all the estimator + descriptions in the generated HTML. :pr:`21493` by + :user:`AurÊlien Geron `. + +.. _changes_1_0_1: + +Version 1.0.1 +============= + +**October 2021** + +Fixed models +------------ + +- |Fix| Non-fit methods in the following classes do not raise a UserWarning + when fitted on DataFrames with valid feature names: + :class:`covariance.EllipticEnvelope`, :class:`ensemble.IsolationForest`, + :class:`ensemble.AdaBoostClassifier`, :class:`neighbors.KNeighborsClassifier`, + :class:`neighbors.KNeighborsRegressor`, + :class:`neighbors.RadiusNeighborsClassifier`, + :class:`neighbors.RadiusNeighborsRegressor`. :pr:`21199` by `Thomas Fan`_. + +:mod:`sklearn.calibration` +.......................... + +- |Fix| Fixed :class:`calibration.CalibratedClassifierCV` to take into account + `sample_weight` when computing the base estimator prediction when + `ensemble=False`. + :pr:`20638` by :user:`Julien BohnÊ `. + +- |Fix| Fixed a bug in :class:`calibration.CalibratedClassifierCV` with + `method="sigmoid"` that was ignoring the `sample_weight` when computing the + Bayesian priors. + :pr:`21179` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.cluster` +...................... + +- |Fix| Fixed a bug in :class:`cluster.KMeans`, ensuring reproducibility and equivalence + between sparse and dense input. :pr:`21195` + by :user:`JÊrÊmie du Boisberranger `. + +:mod:`sklearn.ensemble` +....................... + +- |Fix| Fixed a bug that could produce a segfault in rare cases for + :class:`ensemble.HistGradientBoostingClassifier` and + :class:`ensemble.HistGradientBoostingRegressor`. + :pr:`21130` :user:`Christian Lorentzen `. + +:mod:`sklearn.gaussian_process` +............................... + +- |Fix| Compute `y_std` properly with multi-target in + :class:`sklearn.gaussian_process.GaussianProcessRegressor` allowing + proper normalization in multi-target scene. + :pr:`20761` by :user:`Patrick de C. T. R. Ferreira `. + +:mod:`sklearn.feature_extraction` +................................. + +- |Efficiency| Fixed an efficiency regression introduced in version 1.0.0 in the + `transform` method of :class:`feature_extraction.text.CountVectorizer` which no + longer checks for uppercase characters in the provided vocabulary. :pr:`21251` + by :user:`JÊrÊmie du Boisberranger `. + +- |Fix| Fixed a bug in :class:`feature_extraction.text.CountVectorizer` and + :class:`feature_extraction.text.TfidfVectorizer` by raising an + error when 'min_idf' or 'max_idf' are floating-point numbers greater than 1. + :pr:`20752` by :user:`Alek Lefebvre `. + +:mod:`sklearn.linear_model` +........................... + +- |Fix| Improves stability of :class:`linear_model.LassoLars` for different + versions of openblas. :pr:`21340` by `Thomas Fan`_. + +- |Fix| :class:`linear_model.LogisticRegression` now raises a better error + message when the solver does not support sparse matrices with int64 indices. + :pr:`21093` by `Tom Dupre la Tour`_. + +:mod:`sklearn.neighbors` +........................ + +- |Fix| :class:`neighbors.KNeighborsClassifier`, + :class:`neighbors.KNeighborsRegressor`, + :class:`neighbors.RadiusNeighborsClassifier`, + :class:`neighbors.RadiusNeighborsRegressor` with `metric="precomputed"` raises + an error for `bsr` and `dok` sparse matrices in methods: `fit`, `kneighbors` + and `radius_neighbors`, due to handling of explicit zeros in `bsr` and `dok` + :term:`sparse graph` formats. :pr:`21199` by `Thomas Fan`_. + +:mod:`sklearn.pipeline` +....................... + +- |Fix| :meth:`pipeline.Pipeline.get_feature_names_out` correctly passes feature + names out from one step of a pipeline to the next. :pr:`21351` by + `Thomas Fan`_. + +:mod:`sklearn.svm` +.................. + +- |Fix| :class:`svm.SVC` and :class:`svm.SVR` check for an inconsistency + in its internal representation and raise an error instead of segfaulting. + This fix also resolves + `CVE-2020-28975 `__. + :pr:`21336` by `Thomas Fan`_. + +:mod:`sklearn.utils` +.................... + +- |Enhancement| `utils.validation._check_sample_weight` can perform a + non-negativity check on the sample weights. It can be turned on + using the only_non_negative bool parameter. + Estimators that check for non-negative weights are updated: + :func:`linear_model.LinearRegression` (here the previous + error message was misleading), + :func:`ensemble.AdaBoostClassifier`, + :func:`ensemble.AdaBoostRegressor`, + :func:`neighbors.KernelDensity`. + :pr:`20880` by :user:`Guillaume Lemaitre ` + and :user:`AndrÃĄs Simon `. + +- |Fix| Solve a bug in ``sklearn.utils.metaestimators.if_delegate_has_method`` + where the underlying check for an attribute did not work with NumPy arrays. + :pr:`21145` by :user:`Zahlii `. + +Miscellaneous +............. + +- |Fix| Fitting an estimator on a dataset that has no feature names, that was previously + fitted on a dataset with feature names no longer keeps the old feature names stored in + the `feature_names_in_` attribute. :pr:`21389` by + :user:`JÊrÊmie du Boisberranger `. + .. _changes_1_0: Version 1.0.0 ============= -**In Development** +**September 2021** +Minimal dependencies +-------------------- -.. include:: changelog_legend.inc +Version 1.0.0 of scikit-learn requires python 3.7+, numpy 1.14.6+ and +scipy 1.1.0+. Optional minimal dependency is matplotlib 2.2.2+. -Put the changes in their relevant module. +Enforcing keyword-only arguments +-------------------------------- + +In an effort to promote clear and non-ambiguous use of the library, most +constructor and function parameters must now be passed as keyword arguments +(i.e. using the `param=value` syntax) instead of positional. If a keyword-only +parameter is used as positional, a `TypeError` is now raised. +:issue:`15005` :pr:`20002` by `Joel Nothman`_, `Adrin Jalali`_, `Thomas Fan`_, +`Nicolas Hug`_, and `Tom Dupre la Tour`_. See `SLEP009 +`_ +for more details. Changed models -------------- @@ -25,6 +318,18 @@ random sampling procedures. - |Fix| :class:`manifold.TSNE` now avoids numerical underflow issues during affinity matrix computation. +- |Fix| :class:`manifold.Isomap` now connects disconnected components of the + neighbors graph along some minimum distance pairs, instead of changing + every infinite distances to zero. + +- |Fix| The splitting criterion of :class:`tree.DecisionTreeClassifier` and + :class:`tree.DecisionTreeRegressor` can be impacted by a fix in the handling + of rounding errors. Previously some extra spurious splits could occur. + +- |Fix| :func:`model_selection.train_test_split` with a `stratify` parameter + and :class:`model_selection.StratifiedShuffleSplit` may lead to slightly + different results. + Details are listed in the changelog below. (While we are trying to better inform users by providing this information, we @@ -45,21 +350,204 @@ Changelog :pr:`123456` by :user:`Joe Bloggs `. where 123456 is the *pull request* number, not the issue number. +- |API| The option for using the squared error via ``loss`` and + ``criterion`` parameters was made more consistent. The preferred way is by + setting the value to `"squared_error"`. Old option names are still valid, + produce the same models, but are deprecated and will be removed in version + 1.2. + :pr:`19310` by :user:`Christian Lorentzen `. + + - For :class:`ensemble.ExtraTreesRegressor`, `criterion="mse"` is deprecated, + use `"squared_error"` instead which is now the default. + + - For :class:`ensemble.GradientBoostingRegressor`, `loss="ls"` is deprecated, + use `"squared_error"` instead which is now the default. + + - For :class:`ensemble.RandomForestRegressor`, `criterion="mse"` is deprecated, + use `"squared_error"` instead which is now the default. + + - For :class:`ensemble.HistGradientBoostingRegressor`, `loss="least_squares"` + is deprecated, use `"squared_error"` instead which is now the default. + + - For :class:`linear_model.RANSACRegressor`, `loss="squared_loss"` is + deprecated, use `"squared_error"` instead. + + - For :class:`linear_model.SGDRegressor`, `loss="squared_loss"` is + deprecated, use `"squared_error"` instead which is now the default. + + - For :class:`tree.DecisionTreeRegressor`, `criterion="mse"` is deprecated, + use `"squared_error"` instead which is now the default. + + - For :class:`tree.ExtraTreeRegressor`, `criterion="mse"` is deprecated, + use `"squared_error"` instead which is now the default. + +- |API| The option for using the absolute error via ``loss`` and + ``criterion`` parameters was made more consistent. The preferred way is by + setting the value to `"absolute_error"`. Old option names are still valid, + produce the same models, but are deprecated and will be removed in version + 1.2. + :pr:`19733` by :user:`Christian Lorentzen `. + + - For :class:`ensemble.ExtraTreesRegressor`, `criterion="mae"` is deprecated, + use `"absolute_error"` instead. + + - For :class:`ensemble.GradientBoostingRegressor`, `loss="lad"` is deprecated, + use `"absolute_error"` instead. + + - For :class:`ensemble.RandomForestRegressor`, `criterion="mae"` is deprecated, + use `"absolute_error"` instead. + + - For :class:`ensemble.HistGradientBoostingRegressor`, + `loss="least_absolute_deviation"` is deprecated, use `"absolute_error"` + instead. + + - For :class:`linear_model.RANSACRegressor`, `loss="absolute_loss"` is + deprecated, use `"absolute_error"` instead which is now the default. + + - For :class:`tree.DecisionTreeRegressor`, `criterion="mae"` is deprecated, + use `"absolute_error"` instead. + + - For :class:`tree.ExtraTreeRegressor`, `criterion="mae"` is deprecated, + use `"absolute_error"` instead. + +- |API| `np.matrix` usage is deprecated in 1.0 and will raise a `TypeError` in + 1.2. :pr:`20165` by `Thomas Fan`_. + +- |API| :term:`get_feature_names_out` has been added to the transformer API + to get the names of the output features. `get_feature_names` has in + turn been deprecated. :pr:`18444` by `Thomas Fan`_. + +- |API| All estimators store `feature_names_in_` when fitted on pandas Dataframes. + These feature names are compared to names seen in non-`fit` methods, e.g. + `transform` and will raise a `FutureWarning` if they are not consistent, see also + :ref:`sphx_glr_auto_examples_release_highlights_plot_release_highlights_1_0_0.py`. + These ``FutureWarning`` s will become ``ValueError`` s in 1.2. :pr:`18010` by + `Thomas Fan`_. + +:mod:`sklearn.base` +................... + +- |Fix| :func:`config_context` is now threadsafe. :pr:`18736` by `Thomas Fan`_. + +:mod:`sklearn.calibration` +.......................... + +- |Feature| :func:`calibration.CalibrationDisplay` added to plot + calibration curves. :pr:`17443` by :user:`Lucy Liu `. + +- |Fix| The ``predict`` and ``predict_proba`` methods of + :class:`calibration.CalibratedClassifierCV` can now properly be used on + prefitted pipelines. :pr:`19641` by :user:`Alek Lefebvre `. + +- |Fix| Fixed an error when using a :class:`ensemble.VotingClassifier` + as `base_estimator` in :class:`calibration.CalibratedClassifierCV`. + :pr:`20087` by :user:`ClÊment Fauchereau `. + + :mod:`sklearn.cluster` ...................... -- |Efficiency| The "k-means++" initialization of :class:`cluster.KMeans` and - :class:`cluster.MiniBatchKMeans` is now faster, especially in multicore - settings. :pr:`19002` by :user:`Jon Crall ` and - :user:`JÊrÊmie du Boisberranger `. +- |Efficiency| The ``"k-means++"`` initialization of :class:`cluster.KMeans` + and :class:`cluster.MiniBatchKMeans` is now faster, especially in multicore + settings. :pr:`19002` by :user:`Jon Crall ` and :user:`JÊrÊmie du + Boisberranger `. - |Efficiency| :class:`cluster.KMeans` with `algorithm='elkan'` is now faster in multicore settings. :pr:`19052` by :user:`Yusuke Nagasaka `. +- |Efficiency| :class:`cluster.MiniBatchKMeans` is now faster in multicore + settings. :pr:`17622` by :user:`JÊrÊmie du Boisberranger `. + +- |Efficiency| :class:`cluster.OPTICS` can now cache the output of the + computation of the tree, using the `memory` parameter. :pr:`19024` by + :user:`Frankie Robertson `. + +- |Enhancement| The `predict` and `fit_predict` methods of + :class:`cluster.AffinityPropagation` now accept sparse data type for input + data. + :pr:`20117` by :user:`Venkatachalam Natchiappan ` + +- |Fix| Fixed a bug in :class:`cluster.MiniBatchKMeans` where the sample + weights were partially ignored when the input is sparse. :pr:`17622` by + :user:`JÊrÊmie du Boisberranger `. + +- |Fix| Improved convergence detection based on center change in + :class:`cluster.MiniBatchKMeans` which was almost never achievable. + :pr:`17622` by :user:`JÊrÊmie du Boisberranger `. + +- |FIX| :class:`cluster.AgglomerativeClustering` now supports readonly + memory-mapped datasets. + :pr:`19883` by :user:`Julien Jerphanion `. + +- |Fix| :class:`cluster.AgglomerativeClustering` correctly connects components + when connectivity and affinity are both precomputed and the number + of connected components is greater than 1. :pr:`20597` by + `Thomas Fan`_. + +- |Fix| :class:`cluster.FeatureAgglomeration` does not accept a ``**params`` kwarg in + the ``fit`` function anymore, resulting in a more concise error message. :pr:`20899` + by :user:`Adam Li `. + +- |Fix| Fixed a bug in :class:`cluster.KMeans`, ensuring reproducibility and equivalence + between sparse and dense input. :pr:`20200` + by :user:`JÊrÊmie du Boisberranger `. + - |API| :class:`cluster.Birch` attributes, `fit_` and `partial_fit_`, are deprecated and will be removed in 1.2. :pr:`19297` by `Thomas Fan`_. +- |API| the default value for the `batch_size` parameter of + :class:`cluster.MiniBatchKMeans` was changed from 100 to 1024 due to + efficiency reasons. The `n_iter_` attribute of + :class:`cluster.MiniBatchKMeans` now reports the number of started epochs and + the `n_steps_` attribute reports the number of mini batches processed. + :pr:`17622` by :user:`JÊrÊmie du Boisberranger `. + +- |API| :func:`cluster.spectral_clustering` raises an improved error when passed + a `np.matrix`. :pr:`20560` by `Thomas Fan`_. + +:mod:`sklearn.compose` +...................... + +- |Enhancement| :class:`compose.ColumnTransformer` now records the output + of each transformer in `output_indices_`. :pr:`18393` by + :user:`Luca Bittarello `. + +- |Enhancement| :class:`compose.ColumnTransformer` now allows DataFrame input to + have its columns appear in a changed order in `transform`. Further, columns that + are dropped will not be required in transform, and additional columns will be + ignored if `remainder='drop'`. :pr:`19263` by `Thomas Fan`_. + +- |Enhancement| Adds `**predict_params` keyword argument to + :meth:`compose.TransformedTargetRegressor.predict` that passes keyword + argument to the regressor. + :pr:`19244` by :user:`Ricardo `. + +- |FIX| `compose.ColumnTransformer.get_feature_names` supports + non-string feature names returned by any of its transformers. However, note + that ``get_feature_names`` is deprecated, use ``get_feature_names_out`` + instead. :pr:`18459` by :user:`Albert Villanova del Moral ` + and :user:`Alonso Silva Allende `. + +- |Fix| :class:`compose.TransformedTargetRegressor` now takes nD targets with + an adequate transformer. + :pr:`18898` by :user:`Oras Phongpanagnam `. + +- |API| Adds `verbose_feature_names_out` to :class:`compose.ColumnTransformer`. + This flag controls the prefixing of feature names out in + :term:`get_feature_names_out`. :pr:`18444` and :pr:`21080` by `Thomas Fan`_. + +:mod:`sklearn.covariance` +......................... + +- |Fix| Adds arrays check to :func:`covariance.ledoit_wolf` and + :func:`covariance.ledoit_wolf_shrinkage`. :pr:`20416` by :user:`Hugo Defois + `. + +- |API| Deprecates the following keys in `cv_results_`: `'mean_score'`, + `'std_score'`, and `'split(k)_score'` in favor of `'mean_test_score'` + `'std_test_score'`, and `'split(k)_test_score'`. :pr:`20583` by `Thomas Fan`_. + :mod:`sklearn.datasets` ....................... @@ -68,27 +556,107 @@ Changelog `Thomas Fan`_ and :user:`Amanda Dsouza ` and :user:`EL-ATEIF Sara `. +- |Enhancement| :func:`datasets.fetch_kddcup99` raises a better message + when the cached file is invalid. :pr:`19669` `Thomas Fan`_. + +- |Enhancement| Replace usages of ``__file__`` related to resource file I/O + with ``importlib.resources`` to avoid the assumption that these resource + files (e.g. ``iris.csv``) already exist on a filesystem, and by extension + to enable compatibility with tools such as ``PyOxidizer``. + :pr:`20297` by :user:`Jack Liu `. + +- |Fix| Shorten data file names in the openml tests to better support + installing on Windows and its default 260 character limit on file names. + :pr:`20209` by `Thomas Fan`_. + +- |Fix| :func:`datasets.fetch_kddcup99` returns dataframes when + `return_X_y=True` and `as_frame=True`. :pr:`19011` by `Thomas Fan`_. + +- |API| Deprecates `datasets.load_boston` in 1.0 and it will be removed + in 1.2. Alternative code snippets to load similar datasets are provided. + Please report to the docstring of the function for details. + :pr:`20729` by `Guillaume Lemaitre`_. + + :mod:`sklearn.decomposition` ............................ -- |API| In :class:`decomposition.DictionaryLearning`, - :class:`decomposition.MiniBatchDictionaryLearning`, - :func:`dict_learning` and :func:`dict_learning_online`, - `transform_alpha` will be equal to `alpha` instead of 1.0 by default - starting from version 1.2 - :pr:`19159` by :user:`BenoÃŽt MalÊzieux `. +- |Enhancement| added a new approximate solver (randomized SVD, available with + `eigen_solver='randomized'`) to :class:`decomposition.KernelPCA`. This + significantly accelerates computation when the number of samples is much + larger than the desired number of components. + :pr:`12069` by :user:`Sylvain MariÊ `. - |Fix| Fixes incorrect multiple data-conversion warnings when clustering boolean data. :pr:`19046` by :user:`Surya Prakash `. -- |Fix| Fixed :func:`dict_learning`, used by :class:`DictionaryLearning`, to - ensure determinism of the output. Achieved by flipping signs of the SVD - output which is used to initialize the code. - :pr:`18433` by :user:`Bruno Charron `. +- |Fix| Fixed :func:`decomposition.dict_learning`, used by + :class:`decomposition.DictionaryLearning`, to ensure determinism of the + output. Achieved by flipping signs of the SVD output which is used to + initialize the code. :pr:`18433` by :user:`Bruno Charron `. + +- |Fix| Fixed a bug in :class:`decomposition.MiniBatchDictionaryLearning`, + :class:`decomposition.MiniBatchSparsePCA` and + :func:`decomposition.dict_learning_online` where the update of the dictionary + was incorrect. :pr:`19198` by :user:`JÊrÊmie du Boisberranger `. + +- |Fix| Fixed a bug in :class:`decomposition.DictionaryLearning`, + :class:`decomposition.SparsePCA`, + :class:`decomposition.MiniBatchDictionaryLearning`, + :class:`decomposition.MiniBatchSparsePCA`, + :func:`decomposition.dict_learning` and + :func:`decomposition.dict_learning_online` where the restart of unused atoms + during the dictionary update was not working as expected. :pr:`19198` by + :user:`JÊrÊmie du Boisberranger `. + +- |API| In :class:`decomposition.DictionaryLearning`, + :class:`decomposition.MiniBatchDictionaryLearning`, + :func:`decomposition.dict_learning` and + :func:`decomposition.dict_learning_online`, `transform_alpha` will be equal + to `alpha` instead of 1.0 by default starting from version 1.2 :pr:`19159` by + :user:`BenoÃŽt MalÊzieux `. + +- |API| Rename variable names in :class:`decomposition.KernelPCA` to improve + readability. `lambdas_` and `alphas_` are renamed to `eigenvalues_` + and `eigenvectors_`, respectively. `lambdas_` and `alphas_` are + deprecated and will be removed in 1.2. + :pr:`19908` by :user:`Kei Ishikawa `. + +- |API| The `alpha` and `regularization` parameters of :class:`decomposition.NMF` and + :func:`decomposition.non_negative_factorization` are deprecated and will be removed + in 1.2. Use the new parameters `alpha_W` and `alpha_H` instead. :pr:`20512` by + :user:`JÊrÊmie du Boisberranger `. + +:mod:`sklearn.dummy` +.................... + +- |API| Attribute `n_features_in_` in :class:`dummy.DummyRegressor` and + :class:`dummy.DummyRegressor` is deprecated and will be removed in 1.2. + :pr:`20960` by `Thomas Fan`_. :mod:`sklearn.ensemble` ....................... +- |Enhancement| :class:`~sklearn.ensemble.HistGradientBoostingClassifier` and + :class:`~sklearn.ensemble.HistGradientBoostingRegressor` take cgroups quotas + into account when deciding the number of threads used by OpenMP. This + avoids performance problems caused by over-subscription when using those + classes in a docker container for instance. :pr:`20477` + by `Thomas Fan`_. + +- |Enhancement| :class:`~sklearn.ensemble.HistGradientBoostingClassifier` and + :class:`~sklearn.ensemble.HistGradientBoostingRegressor` are no longer + experimental. They are now considered stable and are subject to the same + deprecation cycles as all other estimators. :pr:`19799` by `Nicolas Hug`_. + +- |Enhancement| Improve the HTML rendering of the + :class:`ensemble.StackingClassifier` and :class:`ensemble.StackingRegressor`. + :pr:`19564` by `Thomas Fan`_. + +- |Enhancement| Added Poisson criterion to + :class:`ensemble.RandomForestRegressor`. :pr:`19836` by :user:`Brian Sun + `. + - |Fix| Do not allow to compute out-of-bag (OOB) score in :class:`ensemble.RandomForestClassifier` and :class:`ensemble.ExtraTreesClassifier` with multiclass-multioutput target @@ -96,87 +664,340 @@ Changelog target. Additional private refactoring was performed. :pr:`19162` by :user:`Guillaume Lemaitre `. +- |Fix| Improve numerical precision for weights boosting in + :class:`ensemble.AdaBoostClassifier` and :class:`ensemble.AdaBoostRegressor` + to avoid underflows. + :pr:`10096` by :user:`Fenil Suchak `. + +- |Fix| Fixed the range of the argument ``max_samples`` to be ``(0.0, 1.0]`` + in :class:`ensemble.RandomForestClassifier`, + :class:`ensemble.RandomForestRegressor`, where `max_samples=1.0` is + interpreted as using all `n_samples` for bootstrapping. :pr:`20159` by + :user:`murata-yu`. + +- |Fix| Fixed a bug in :class:`ensemble.AdaBoostClassifier` and + :class:`ensemble.AdaBoostRegressor` where the `sample_weight` parameter + got overwritten during `fit`. + :pr:`20534` by :user:`Guillaume Lemaitre `. + +- |API| Removes `tol=None` option in + :class:`ensemble.HistGradientBoostingClassifier` and + :class:`ensemble.HistGradientBoostingRegressor`. Please use `tol=0` for + the same behavior. :pr:`19296` by `Thomas Fan`_. + :mod:`sklearn.feature_extraction` ................................. -- |Fix| Fixed a bug in class:`feature_extraction.HashingVectorizer` where some - input strings would result in negative indices in the transformed data. - :pr:`19035` by :user:`Liu Yu `. +- |Fix| Fixed a bug in :class:`feature_extraction.text.HashingVectorizer` + where some input strings would result in negative indices in the transformed + data. :pr:`19035` by :user:`Liu Yu `. + +- |Fix| Fixed a bug in :class:`feature_extraction.DictVectorizer` by raising an + error with unsupported value type. + :pr:`19520` by :user:`Jeff Zhao `. + +- |Fix| Fixed a bug in :func:`feature_extraction.image.img_to_graph` + and :func:`feature_extraction.image.grid_to_graph` where singleton connected + components were not handled properly, resulting in a wrong vertex indexing. + :pr:`18964` by `Bertrand Thirion`_. + +- |Fix| Raise a warning in :class:`feature_extraction.text.CountVectorizer` + with `lowercase=True` when there are vocabulary entries with uppercase + characters to avoid silent misses in the resulting feature vectors. + :pr:`19401` by :user:`Zito Relova ` + +:mod:`sklearn.feature_selection` +................................ + +- |Feature| :func:`feature_selection.r_regression` computes Pearson's R + correlation coefficients between the features and the target. + :pr:`17169` by :user:`Dmytro Lituiev ` + and :user:`Julien Jerphanion `. + +- |Enhancement| :func:`feature_selection.RFE.fit` accepts additional estimator + parameters that are passed directly to the estimator's `fit` method. + :pr:`20380` by :user:`IvÃĄn Pulido `, :user:`Felipe Bidu `, + :user:`Gil Rutter `, and :user:`Adrin Jalali `. + +- |FIX| Fix a bug in :func:`isotonic.isotonic_regression` where the + `sample_weight` passed by a user were overwritten during ``fit``. + :pr:`20515` by :user:`Carsten Allefeld `. + +- |Fix| Change :func:`feature_selection.SequentialFeatureSelector` to + allow for unsupervised modelling so that the `fit` signature need not + do any `y` validation and allow for `y=None`. + :pr:`19568` by :user:`Shyam Desai `. + +- |API| Raises an error in :class:`feature_selection.VarianceThreshold` + when the variance threshold is negative. + :pr:`20207` by :user:`Tomohiro Endo ` + +- |API| Deprecates `grid_scores_` in favor of split scores in `cv_results_` in + :class:`feature_selection.RFECV`. `grid_scores_` will be removed in + version 1.2. + :pr:`20161` by :user:`Shuhei Kayawari ` and :user:`arka204`. :mod:`sklearn.inspection` ......................... +- |Enhancement| Add `max_samples` parameter in + :func:`inspection.permutation_importance`. It enables to draw a subset of the + samples to compute the permutation importance. This is useful to keep the + method tractable when evaluating feature importance on large datasets. + :pr:`20431` by :user:`Oliver Pfaffel `. + +- |Enhancement| Add kwargs to format ICE and PD lines separately in partial + dependence plots `inspection.plot_partial_dependence` and + :meth:`inspection.PartialDependenceDisplay.plot`. :pr:`19428` by :user:`Mehdi + Hamoumi `. + - |Fix| Allow multiple scorers input to - :func:`~sklearn.inspection.permutation_importance`. - :pr:`19411` by :user:`Simona Maggio `. + :func:`inspection.permutation_importance`. :pr:`19411` by :user:`Simona + Maggio `. + +- |API| :class:`inspection.PartialDependenceDisplay` exposes a class method: + :func:`~inspection.PartialDependenceDisplay.from_estimator`. + `inspection.plot_partial_dependence` is deprecated in favor of the + class method and will be removed in 1.2. :pr:`20959` by `Thomas Fan`_. + +:mod:`sklearn.kernel_approximation` +................................... + +- |Fix| Fix a bug in :class:`kernel_approximation.Nystroem` + where the attribute `component_indices_` did not correspond to the subset of + sample indices used to generate the approximated kernel. :pr:`20554` by + :user:`Xiangyin Kong `. :mod:`sklearn.linear_model` ........................... -- |Enhancement| Validate user-supplied gram matrix passed to linear models - via the `precompute` argument. :pr:`19004` by :user:`Adam Midvidy `. +- |MajorFeature| Added :class:`linear_model.QuantileRegressor` which implements + linear quantile regression with L1 penalty. + :pr:`9978` by :user:`David Dale ` and + :user:`Christian Lorentzen `. -- |Fix| :meth:`ElasticNet.fit` no longer modifies `sample_weight` in place. - :pr:`19055` by `Thomas Fan`_. +- |Feature| The new :class:`linear_model.SGDOneClassSVM` provides an SGD + implementation of the linear One-Class SVM. Combined with kernel + approximation techniques, this implementation approximates the solution of + a kernelized One Class SVM while benefiting from a linear + complexity in the number of samples. + :pr:`10027` by :user:`Albert Thomas `. -- |Fix| :class:`Lasso`, :class:`ElasticNet` no longer have a `dual_gap_` - not corresponding to their objective. :pr:`19172` by - :user:`Mathurin Massias ` +- |Feature| Added `sample_weight` parameter to + :class:`linear_model.LassoCV` and :class:`linear_model.ElasticNetCV`. + :pr:`16449` by :user:`Christian Lorentzen `. -:mod:`sklearn.preprocessing` -............................ +- |Feature| Added new solver `lbfgs` (available with `solver="lbfgs"`) + and `positive` argument to :class:`linear_model.Ridge`. When `positive` is + set to `True`, forces the coefficients to be positive (only supported by + `lbfgs`). :pr:`20231` by :user:`Toshihiro Nakae `. -- |Feature| :class:`preprocessing.OrdinalEncoder` supports passing through - missing values by default. :pr:`19069` by `Thomas Fan`_. +- |Efficiency| The implementation of :class:`linear_model.LogisticRegression` + has been optimised for dense matrices when using `solver='newton-cg'` and + `multi_class!='multinomial'`. + :pr:`19571` by :user:`Julien Jerphanion `. -- |API|: The parameter ``normalize`` of :class:`linear_model.LinearRegression` - is deprecated and will be removed in 1.2. - Motivation for this deprecation: ``normalize`` parameter did not take any - effect if ``fit_intercept`` was set to False and therefore was deemed - confusing. - The behavior of the deprecated LinearRegression(normalize=True) can be - reproduced with :class:`~sklearn.pipeline.Pipeline` with - :class:`~sklearn.preprocessing.StandardScaler`as follows: - make_pipeline(StandardScaler(with_mean=False), LinearRegression()). - :pr:`17743` by :user:`Maria Telenczuk ` and - :user:`Alexandre Gramfort `. - -- |Fix|: `sample_weight` are now fully taken into account in linear models +- |Enhancement| `fit` method preserves dtype for numpy.float32 in + :class:`linear_model.Lars`, :class:`linear_model.LassoLars`, + :class:`linear_model.LassoLars`, :class:`linear_model.LarsCV` and + :class:`linear_model.LassoLarsCV`. :pr:`20155` by :user:`Takeshi Oura + `. + +- |Enhancement| Validate user-supplied gram matrix passed to linear models + via the `precompute` argument. :pr:`19004` by :user:`Adam Midvidy `. + +- |Fix| :meth:`linear_model.ElasticNet.fit` no longer modifies `sample_weight` + in place. :pr:`19055` by `Thomas Fan`_. + +- |Fix| :class:`linear_model.Lasso` and :class:`linear_model.ElasticNet` no + longer have a `dual_gap_` not corresponding to their objective. :pr:`19172` + by :user:`Mathurin Massias ` + +- |Fix| `sample_weight` are now fully taken into account in linear models when `normalize=True` for both feature centering and feature scaling. :pr:`19426` by :user:`Alexandre Gramfort ` and :user:`Maria Telenczuk `. +- |Fix| Points with residuals equal to ``residual_threshold`` are now considered + as inliers for :class:`linear_model.RANSACRegressor`. This allows fitting + a model perfectly on some datasets when `residual_threshold=0`. + :pr:`19499` by :user:`Gregory Strubel `. + +- |Fix| Sample weight invariance for :class:`linear_model.Ridge` was fixed in + :pr:`19616` by :user:`Oliver Grisel ` and :user:`Christian Lorentzen + `. + +- |Fix| The dictionary `params` in :func:`linear_model.enet_path` and + :func:`linear_model.lasso_path` should only contain parameter of the + coordinate descent solver. Otherwise, an error will be raised. + :pr:`19391` by :user:`Shao Yang Hong `. + +- |API| Raise a warning in :class:`linear_model.RANSACRegressor` that from + version 1.2, `min_samples` need to be set explicitly for models other than + :class:`linear_model.LinearRegression`. :pr:`19390` by :user:`Shao Yang Hong + `. + +- |API|: The parameter ``normalize`` of :class:`linear_model.LinearRegression` + is deprecated and will be removed in 1.2. Motivation for this deprecation: + ``normalize`` parameter did not take any effect if ``fit_intercept`` was set + to False and therefore was deemed confusing. The behavior of the deprecated + ``LinearModel(normalize=True)`` can be reproduced with a + :class:`~sklearn.pipeline.Pipeline` with ``LinearModel`` (where + ``LinearModel`` is :class:`~linear_model.LinearRegression`, + :class:`~linear_model.Ridge`, :class:`~linear_model.RidgeClassifier`, + :class:`~linear_model.RidgeCV` or :class:`~linear_model.RidgeClassifierCV`) + as follows: ``make_pipeline(StandardScaler(with_mean=False), + LinearModel())``. The ``normalize`` parameter in + :class:`~linear_model.LinearRegression` was deprecated in :pr:`17743` by + :user:`Maria Telenczuk ` and :user:`Alexandre Gramfort `. + Same for :class:`~linear_model.Ridge`, + :class:`~linear_model.RidgeClassifier`, :class:`~linear_model.RidgeCV`, and + :class:`~linear_model.RidgeClassifierCV`, in: :pr:`17772` by :user:`Maria + Telenczuk ` and :user:`Alexandre Gramfort `. Same for + :class:`~linear_model.BayesianRidge`, :class:`~linear_model.ARDRegression` + in: :pr:`17746` by :user:`Maria Telenczuk `. Same for + :class:`~linear_model.Lasso`, :class:`~linear_model.LassoCV`, + :class:`~linear_model.ElasticNet`, :class:`~linear_model.ElasticNetCV`, + :class:`~linear_model.MultiTaskLasso`, + :class:`~linear_model.MultiTaskLassoCV`, + :class:`~linear_model.MultiTaskElasticNet`, + :class:`~linear_model.MultiTaskElasticNetCV`, in: :pr:`17785` by :user:`Maria + Telenczuk ` and :user:`Alexandre Gramfort `. + +- |API| The ``normalize`` parameter of + :class:`~linear_model.OrthogonalMatchingPursuit` and + :class:`~linear_model.OrthogonalMatchingPursuitCV` will default to False in + 1.2 and will be removed in 1.4. :pr:`17750` by :user:`Maria Telenczuk + ` and :user:`Alexandre Gramfort `. Same for + :class:`~linear_model.Lars` :class:`~linear_model.LarsCV` + :class:`~linear_model.LassoLars` :class:`~linear_model.LassoLarsCV` + :class:`~linear_model.LassoLarsIC`, in :pr:`17769` by :user:`Maria Telenczuk + ` and :user:`Alexandre Gramfort `. + +- |API| Keyword validation has moved from `__init__` and `set_params` to `fit` + for the following estimators conforming to scikit-learn's conventions: + :class:`~linear_model.SGDClassifier`, + :class:`~linear_model.SGDRegressor`, + :class:`~linear_model.SGDOneClassSVM`, + :class:`~linear_model.PassiveAggressiveClassifier`, and + :class:`~linear_model.PassiveAggressiveRegressor`. + :pr:`20683` by `Guillaume Lemaitre`_. + :mod:`sklearn.manifold` ....................... +- |Enhancement| Implement `'auto'` heuristic for the `learning_rate` in + :class:`manifold.TSNE`. It will become default in 1.2. The default + initialization will change to `pca` in 1.2. PCA initialization will + be scaled to have standard deviation 1e-4 in 1.2. + :pr:`19491` by :user:`Dmitry Kobak `. + - |Fix| Change numerical precision to prevent underflow issues during affinity matrix computation for :class:`manifold.TSNE`. :pr:`19472` by :user:`Dmitry Kobak `. +- |Fix| :class:`manifold.Isomap` now uses `scipy.sparse.csgraph.shortest_path` + to compute the graph shortest path. It also connects disconnected components + of the neighbors graph along some minimum distance pairs, instead of changing + every infinite distances to zero. :pr:`20531` by `Roman Yurchak`_ and `Tom + Dupre la Tour`_. + +- |Fix| Decrease the numerical default tolerance in the lobpcg call + in :func:`manifold.spectral_embedding` to prevent numerical instability. + :pr:`21194` by :user:`Andrew Knyazev `. + :mod:`sklearn.metrics` ...................... +- |Feature| :func:`metrics.mean_pinball_loss` exposes the pinball loss for + quantile regression. :pr:`19415` by :user:`Xavier DuprÊ ` + and :user:`Oliver Grisel `. + +- |Feature| :func:`metrics.d2_tweedie_score` calculates the D^2 regression + score for Tweedie deviances with power parameter ``power``. This is a + generalization of the `r2_score` and can be interpreted as percentage of + Tweedie deviance explained. + :pr:`17036` by :user:`Christian Lorentzen `. + +- |Feature| :func:`metrics.mean_squared_log_error` now supports + `squared=False`. + :pr:`20326` by :user:`Uttam kumar `. + +- |Efficiency| Improved speed of :func:`metrics.confusion_matrix` when labels + are integral. + :pr:`9843` by :user:`Jon Crall `. + +- |Enhancement| A fix to raise an error in :func:`metrics.hinge_loss` when + ``pred_decision`` is 1d whereas it is a multiclass classification or when + ``pred_decision`` parameter is not consistent with the ``labels`` parameter. + :pr:`19643` by :user:`Pierre Attard `. + +- |Fix| :meth:`metrics.ConfusionMatrixDisplay.plot` uses the correct max + for colormap. :pr:`19784` by `Thomas Fan`_. + +- |Fix| Samples with zero `sample_weight` values do not affect the results + from :func:`metrics.det_curve`, :func:`metrics.precision_recall_curve` + and :func:`metrics.roc_curve`. + :pr:`18328` by :user:`Albert Villanova del Moral ` and + :user:`Alonso Silva Allende `. + +- |Fix| avoid overflow in :func:`metrics.adjusted_rand_score` with + large amount of data. :pr:`20312` by :user:`Divyanshu Deoli + `. + - |API| :class:`metrics.ConfusionMatrixDisplay` exposes two class methods :func:`~metrics.ConfusionMatrixDisplay.from_estimator` and :func:`~metrics.ConfusionMatrixDisplay.from_predictions` allowing to create a confusion matrix plot using an estimator or the predictions. - :func:`metrics.plot_confusion_matrix` is deprecated in favor of these two + `metrics.plot_confusion_matrix` is deprecated in favor of these two class methods and will be removed in 1.2. :pr:`18543` by `Guillaume Lemaitre`_. -- |Enhancement| A fix to raise an error in :func:`metrics.hinge_loss` when - ``pred_decision`` is 1d whereas it is a multiclass classification or when - ``pred_decision`` parameter is not consistent with the ``labels`` parameter. - :pr:`19643` by :user:`Pierre Attard `. +- |API| :class:`metrics.PrecisionRecallDisplay` exposes two class methods + :func:`~metrics.PrecisionRecallDisplay.from_estimator` and + :func:`~metrics.PrecisionRecallDisplay.from_predictions` allowing to create + a precision-recall curve using an estimator or the predictions. + `metrics.plot_precision_recall_curve` is deprecated in favor of these + two class methods and will be removed in 1.2. + :pr:`20552` by `Guillaume Lemaitre`_. -- |Feature| :func:`metrics.mean_pinball_loss` exposes the pinball loss for - quantile regression. :pr:`19415` by :user:`Xavier DuprÊ ` - and :user:`Oliver Grisel `. +- |API| :class:`metrics.DetCurveDisplay` exposes two class methods + :func:`~metrics.DetCurveDisplay.from_estimator` and + :func:`~metrics.DetCurveDisplay.from_predictions` allowing to create + a confusion matrix plot using an estimator or the predictions. + `metrics.plot_det_curve` is deprecated in favor of these two + class methods and will be removed in 1.2. + :pr:`19278` by `Guillaume Lemaitre`_. -- |Efficiency| Improved speed of :func:`metrics.confusion_matrix` when labels - are integral. - :pr:`9843` by :user:`Jon Crall `. +:mod:`sklearn.mixture` +...................... + +- |Fix| Ensure that the best parameters are set appropriately + in the case of divergency for :class:`mixture.GaussianMixture` and + :class:`mixture.BayesianGaussianMixture`. + :pr:`20030` by :user:`Tingshan Liu ` and + :user:`Benjamin Pedigo `. + +:mod:`sklearn.model_selection` +.............................. + +- |Feature| added :class:`model_selection.StratifiedGroupKFold`, that combines + :class:`model_selection.StratifiedKFold` and + :class:`model_selection.GroupKFold`, providing an ability to split data + preserving the distribution of classes in each split while keeping each + group within a single split. + :pr:`18649` by :user:`Leandro Hermida ` and + :user:`Rodion Martynov `. + +- |Enhancement| warn only once in the main process for per-split fit failures + in cross-validation. :pr:`20619` by :user:`Loïc Estève ` + +- |Enhancement| The `model_selection.BaseShuffleSplit` base class is + now public. :pr:`20056` by :user:`pabloduque0`. + +- |Fix| Avoid premature overflow in :func:`model_selection.train_test_split`. + :pr:`20904` by :user:`Tomasz Jakubek `. :mod:`sklearn.naive_bayes` .......................... @@ -193,6 +1014,42 @@ Changelog Use ``var_`` instead. :pr:`18842` by :user:`Hong Shao Yang `. +:mod:`sklearn.neighbors` +........................ + +- |Enhancement| The creation of :class:`neighbors.KDTree` and + :class:`neighbors.BallTree` has been improved for their worst-cases time + complexity from :math:`\mathcal{O}(n^2)` to :math:`\mathcal{O}(n)`. + :pr:`19473` by :user:`jiefangxuanyan ` and + :user:`Julien Jerphanion `. + +- |FIX| `neighbors.DistanceMetric` subclasses now support readonly + memory-mapped datasets. :pr:`19883` by :user:`Julien Jerphanion `. + +- |FIX| :class:`neighbors.NearestNeighbors`, :class:`neighbors.KNeighborsClassifier`, + :class:`neighbors.RadiusNeighborsClassifier`, :class:`neighbors.KNeighborsRegressor` + and :class:`neighbors.RadiusNeighborsRegressor` do not validate `weights` in + `__init__` and validate `weights` in `fit` instead. :pr:`20072` by + :user:`Juan Carlos Alfaro JimÊnez `. + +- |API| The parameter `kwargs` of :class:`neighbors.RadiusNeighborsClassifier` is + deprecated and will be removed in 1.2. + :pr:`20842` by :user:`Juan Martín Loyola `. + +:mod:`sklearn.neural_network` +............................. + +- |Fix| :class:`neural_network.MLPClassifier` and + :class:`neural_network.MLPRegressor` now correctly support continued training + when loading from a pickled file. :pr:`19631` by `Thomas Fan`_. + +:mod:`sklearn.pipeline` +....................... + +- |API| The `predict_proba` and `predict_log_proba` methods of the + :class:`pipeline.Pipeline` now support passing prediction kwargs to the final + estimator. :pr:`19790` by :user:`Christopher Flynn `. + :mod:`sklearn.preprocessing` ............................ @@ -201,13 +1058,79 @@ Changelog polynomial ``degree`` of the splines, number of knots ``n_knots`` and knot positioning strategy ``knots``. :pr:`18368` by :user:`Christian Lorentzen `. + :class:`preprocessing.SplineTransformer` also supports periodic + splines via the ``extrapolation`` argument. + :pr:`19483` by :user:`Malte Londschien `. + :class:`preprocessing.SplineTransformer` supports sample weights for + knot position strategy ``"quantile"``. + :pr:`20526` by :user:`Malte Londschien `. + +- |Feature| :class:`preprocessing.OrdinalEncoder` supports passing through + missing values by default. :pr:`19069` by `Thomas Fan`_. + +- |Feature| :class:`preprocessing.OneHotEncoder` now supports + `handle_unknown='ignore'` and dropping categories. :pr:`19041` by + `Thomas Fan`_. + +- |Feature| :class:`preprocessing.PolynomialFeatures` now supports passing + a tuple to `degree`, i.e. `degree=(min_degree, max_degree)`. + :pr:`20250` by :user:`Christian Lorentzen `. + +- |Efficiency| :class:`preprocessing.StandardScaler` is faster and more memory + efficient. :pr:`20652` by `Thomas Fan`_. + +- |Efficiency| Changed ``algorithm`` argument for :class:`cluster.KMeans` in + :class:`preprocessing.KBinsDiscretizer` from ``auto`` to ``full``. + :pr:`19934` by :user:`Hleb Levitski `. + +- |Efficiency| The implementation of `fit` for + :class:`preprocessing.PolynomialFeatures` transformer is now faster. This is + especially noticeable on large sparse input. :pr:`19734` by :user:`Fred + Robinson `. + +- |Fix| The :func:`preprocessing.StandardScaler.inverse_transform` method + now raises error when the input data is 1D. :pr:`19752` by :user:`Zhehao Liu + `. - |Fix| :func:`preprocessing.scale`, :class:`preprocessing.StandardScaler` and similar scalers detect near-constant features to avoid scaling them to very large values. This problem happens in particular when using a scaler on sparse data with a constant column with sample weights, in which case centering is typically disabled. :pr:`19527` by :user:`Oliver Grisel - ` and :user:`Maria Telenczuk `. + ` and :user:`Maria Telenczuk ` and :pr:`19788` by + :user:`JÊrÊmie du Boisberranger `. + +- |Fix| :meth:`preprocessing.StandardScaler.inverse_transform` now + correctly handles integer dtypes. :pr:`19356` by :user:`makoeppel`. + +- |Fix| :meth:`preprocessing.OrdinalEncoder.inverse_transform` is not + supporting sparse matrix and raises the appropriate error message. + :pr:`19879` by :user:`Guillaume Lemaitre `. + +- |Fix| The `fit` method of :class:`preprocessing.OrdinalEncoder` will not + raise error when `handle_unknown='ignore'` and unknown categories are given + to `fit`. + :pr:`19906` by :user:`Zhehao Liu `. + +- |Fix| Fix a regression in :class:`preprocessing.OrdinalEncoder` where large + Python numeric would raise an error due to overflow when casted to C type + (`np.float64` or `np.int64`). + :pr:`20727` by `Guillaume Lemaitre`_. + +- |Fix| :class:`preprocessing.FunctionTransformer` does not set `n_features_in_` + based on the input to `inverse_transform`. :pr:`20961` by `Thomas Fan`_. + +- |API| The `n_input_features_` attribute of + :class:`preprocessing.PolynomialFeatures` is deprecated in favor of + `n_features_in_` and will be removed in 1.2. :pr:`20240` by + :user:`JÊrÊmie du Boisberranger `. + +:mod:`sklearn.svm` +................... + +- |API| The parameter `**params` of :func:`svm.OneClassSVM.fit` is + deprecated and will be removed in 1.2. + :pr:`20843` by :user:`Juan Martín Loyola `. :mod:`sklearn.tree` ................... @@ -216,17 +1139,140 @@ Changelog for non-English characters. :pr:`18959` by :user:`Zero ` and :user:`wstates `. -:mod:`sklearn.calibration` -............................ +- |Fix| Improves compatibility of :func:`tree.plot_tree` with high DPI screens. + :pr:`20023` by `Thomas Fan`_. + +- |Fix| Fixed a bug in :class:`tree.DecisionTreeClassifier`, + :class:`tree.DecisionTreeRegressor` where a node could be split whereas it + should not have been due to incorrect handling of rounding errors. + :pr:`19336` by :user:`JÊrÊmie du Boisberranger `. + +- |API| The `n_features_` attribute of :class:`tree.DecisionTreeClassifier`, + :class:`tree.DecisionTreeRegressor`, :class:`tree.ExtraTreeClassifier` and + :class:`tree.ExtraTreeRegressor` is deprecated in favor of `n_features_in_` + and will be removed in 1.2. :pr:`20272` by + :user:`JÊrÊmie du Boisberranger `. + +:mod:`sklearn.utils` +.................... + +- |Enhancement| Deprecated the default value of the `random_state=0` in + :func:`~sklearn.utils.extmath.randomized_svd`. Starting in 1.2, + the default value of `random_state` will be set to `None`. + :pr:`19459` by :user:`Cindy Bezuidenhout ` and + :user:`Clifford Akai-Nettey`. + +- |Enhancement| Added helper decorator :func:`utils.metaestimators.available_if` + to provide flexibility in metaestimators making methods available or + unavailable on the basis of state, in a more readable way. + :pr:`19948` by `Joel Nothman`_. + +- |Enhancement| :func:`utils.validation.check_is_fitted` now uses + ``__sklearn_is_fitted__`` if available, instead of checking for attributes + ending with an underscore. This also makes :class:`pipeline.Pipeline` and + :class:`preprocessing.FunctionTransformer` pass + ``check_is_fitted(estimator)``. :pr:`20657` by `Adrin Jalali`_. + +- |Fix| Fixed a bug in :func:`utils.sparsefuncs.mean_variance_axis` where the + precision of the computed variance was very poor when the real variance is + exactly zero. :pr:`19766` by :user:`JÊrÊmie du Boisberranger `. + +- |Fix| The docstrings of properties that are decorated with + :func:`utils.deprecated` are now properly wrapped. :pr:`20385` by `Thomas + Fan`_. + +- |Fix| `utils.stats._weighted_percentile` now correctly ignores + zero-weighted observations smaller than the smallest observation with + positive weight for ``percentile=0``. Affected classes are + :class:`dummy.DummyRegressor` for ``quantile=0`` and + `ensemble.HuberLossFunction` and `ensemble.HuberLossFunction` + for ``alpha=0``. :pr:`20528` by :user:`Malte Londschien `. + +- |Fix| :func:`utils._safe_indexing` explicitly takes a dataframe copy when + integer indices are provided avoiding to raise a warning from Pandas. This + warning was previously raised in resampling utilities and functions using + those utilities (e.g. :func:`model_selection.train_test_split`, + :func:`model_selection.cross_validate`, + :func:`model_selection.cross_val_score`, + :func:`model_selection.cross_val_predict`). + :pr:`20673` by :user:`Joris Van den Bossche `. + +- |Fix| Fix a regression in `utils.is_scalar_nan` where large Python + numbers would raise an error due to overflow in C types (`np.float64` or + `np.int64`). + :pr:`20727` by `Guillaume Lemaitre`_. + +- |Fix| Support for `np.matrix` is deprecated in + :func:`~sklearn.utils.check_array` in 1.0 and will raise a `TypeError` in + 1.2. :pr:`20165` by `Thomas Fan`_. + +- |API| `utils._testing.assert_warns` and `utils._testing.assert_warns_message` + are deprecated in 1.0 and will be removed in 1.2. Used `pytest.warns` context + manager instead. Note that these functions were not documented and part from + the public API. :pr:`20521` by :user:`Olivier Grisel `. -- |Fix| The predict and predict_proba methods of - :class:`calibration.CalibratedClassifierCV can now properly be used on - prefitted pipelines. :pr:`19641` by :user:`Alek Lefebvre ` +- |API| Fixed several bugs in `utils.graph.graph_shortest_path`, which is + now deprecated. Use `scipy.sparse.csgraph.shortest_path` instead. :pr:`20531` + by `Tom Dupre la Tour`_. -Code and Documentation Contributors ------------------------------------ +.. rubric:: Code and documentation contributors Thanks to everyone who has contributed to the maintenance and improvement of the project since version 0.24, including: -TODO: update at the time of the release. +Abdulelah S. Al Mesfer, Abhinav Gupta, Adam J. Stewart, Adam Li, Adam Midvidy, +Adrian Garcia Badaracco, Adrian Sadłocha, Adrin Jalali, Agamemnon Krasoulis, +Alberto Rubiales, Albert Thomas, Albert Villanova del Moral, Alek Lefebvre, +Alessia Marcolini, Alexandr Fonari, Alihan Zihna, Aline Ribeiro de Almeida, +Amanda, Amanda Dsouza, Amol Deshmukh, Ana Pessoa, Anavelyz, Andreas Mueller, +Andrew Delong, Ashish, Ashvith Shetty, Atsushi Nukariya, AurÊlien Geron, Avi +Gupta, Ayush Singh, baam, BaptBillard, Benjamin Pedigo, Bertrand Thirion, +Bharat Raghunathan, bmalezieux, Brian Rice, Brian Sun, Bruno Charron, Bryan +Chen, bumblebee, caherrera-meli, Carsten Allefeld, CeeThinwa, Chiara Marmo, +chrissobel, Christian Lorentzen, Christopher Yeh, Chuliang Xiao, ClÊment +Fauchereau, cliffordEmmanuel, Conner Shen, Connor Tann, David Dale, David Katz, +David Poznik, Dimitri Papadopoulos Orfanos, Divyanshu Deoli, dmallia17, +Dmitry Kobak, DS_anas, Eduardo Jardim, EdwinWenink, EL-ATEIF Sara, Eleni +Markou, EricEllwanger, Eric Fiegel, Erich Schubert, Ezri-Mudde, Fatos Morina, +Felipe Rodrigues, Felix Hafner, Fenil Suchak, flyingdutchman23, Flynn, Fortune +Uwha, Francois Berenger, Frankie Robertson, Frans Larsson, Frederick Robinson, +frellwan, Gabriel S Vicente, Gael Varoquaux, genvalen, Geoffrey Thomas, +geroldcsendes, Hleb Levitski, Glen, GlÃ˛ria Macià MuÃąoz, gregorystrubel, +groceryheist, Guillaume Lemaitre, guiweber, Haidar Almubarak, Hans Moritz +GÃŧnther, Haoyin Xu, Harris Mirza, Harry Wei, Harutaka Kawamura, Hassan +Alsawadi, Helder Geovane Gomes de Lima, Hugo DEFOIS, Igor Ilic, Ikko Ashimine, +Isaack Mungui, Ishaan Bhat, Ishan Mishra, IvÃĄn Pulido, iwhalvic, J Alexander, +Jack Liu, James Alan Preiss, James Budarz, James Lamb, Jannik, Jeff Zhao, +Jennifer Maldonado, JÊrÊmie du Boisberranger, Jesse Lima, Jianzhu Guo, jnboehm, +Joel Nothman, JohanWork, John Paton, Jonathan Schneider, Jon Crall, Jon Haitz +Legarreta GorroÃąo, Joris Van den Bossche, JosÊ Manuel NÃĄpoles Duarte, Juan +Carlos Alfaro JimÊnez, Juan Martin Loyola, Julien Jerphanion, Julio Batista +Silva, julyrashchenko, JVM, Kadatatlu Kishore, Karen Palacio, Kei Ishikawa, +kmatt10, kobaski, Kot271828, Kunj, KurumeYuta, kxytim, lacrosse91, LalliAcqua, +Laveen Bagai, Leonardo Rocco, Leonardo Uieda, Leopoldo Corona, Loic Esteve, +LSturtew, Luca Bittarello, Luccas Quadros, Lucy JimÊnez, Lucy Liu, ly648499246, +Mabu Manaileng, Manimaran, makoeppel, Marco Gorelli, Maren Westermann, +Mariangela, Maria Telenczuk, marielaraj, Martin Hirzel, Mateo NoreÃąa, Mathieu +Blondel, Mathis Batoul, mathurinm, Matthew Calcote, Maxime Prieur, Maxwell, +Mehdi Hamoumi, Mehmet Ali Özer, Miao Cai, Michal Karbownik, michalkrawczyk, +Mitzi, mlondschien, Mohamed Haseeb, Mohamed Khoualed, Muhammad Jarir Kanji, +murata-yu, Nadim Kawwa, Nanshan Li, naozin555, Nate Parsons, Neal Fultz, Nic +Annau, Nicolas Hug, Nicolas Miller, Nico Stefani, Nigel Bosch, Nikita Titov, +Nodar Okroshiashvili, Norbert Preining, novaya, Ogbonna Chibuike Stephen, +OGordon100, Oliver Pfaffel, Olivier Grisel, Oras Phongpanangam, Pablo Duque, +Pablo Ibieta-Jimenez, Patric Lacouth, Paulo S. Costa, Paweł Olszewski, Peter +Dye, PierreAttard, Pierre-Yves Le Borgne, PranayAnchuri, Prince Canuma, +putschblos, qdeffense, RamyaNP, ranjanikrishnan, Ray Bell, Rene Jean Corneille, +Reshama Shaikh, ricardojnf, RichardScottOZ, Rodion Martynov, Rohan Paul, Roman +Lutz, Roman Yurchak, Samuel Brice, Sandy Khosasi, Sean Benhur J, Sebastian +Flores, Sebastian PÃļlsterl, Shao Yang Hong, shinehide, shinnar, shivamgargsya, +Shooter23, Shuhei Kayawari, Shyam Desai, simonamaggio, Sina Tootoonian, +solosilence, Steven Kolawole, Steve Stagg, Surya Prakash, swpease, Sylvain +MariÊ, Takeshi Oura, Terence Honles, TFiFiE, Thomas A Caswell, Thomas J. Fan, +Tim Gates, TimotheeMathieu, Timothy Wolodzko, Tim Vink, t-jakubek, t-kusanagi, +tliu68, Tobias Uhmann, tom1092, TomÃĄs Moreyra, TomÃĄs Ronald Hughes, Tom +DuprÊ la Tour, Tommaso Di Noto, Tomohiro Endo, TONY GEORGE, Toshihiro NAKAE, +tsuga, Uttam kumar, vadim-ushtanit, Vangelis Gkiastas, Venkatachalam N, VilÊm +Zouhar, Vinicius Rios Fuck, Vlasovets, waijean, Whidou, xavier duprÊ, +xiaoyuchai, Yasmeen Alsaedy, yoch, Yosuke KOBAYASHI, Yu Feng, YusukeNagasaka, +yzhenman, Zero, ZeyuSun, ZhaoweiWang, Zito, Zito Relova diff --git a/doc/whats_new/v1.1.rst b/doc/whats_new/v1.1.rst new file mode 100644 index 0000000000000..f1bdcfd544166 --- /dev/null +++ b/doc/whats_new/v1.1.rst @@ -0,0 +1,1409 @@ +.. include:: _contributors.rst + +.. currentmodule:: sklearn + +.. _release_notes_1_1: + +=========== +Version 1.1 +=========== + +For a short description of the main highlights of the release, please refer to +:ref:`sphx_glr_auto_examples_release_highlights_plot_release_highlights_1_1_0.py`. + +.. include:: changelog_legend.inc + +.. _changes_1_1_3: + +Version 1.1.3 +============= + +**October 2022** + +This bugfix release only includes fixes for compatibility with the latest +SciPy release >= 1.9.2. Notable changes include: + +- |Fix| Include `msvcp140.dll` in the scikit-learn wheels since it has been + removed in the latest SciPy wheels. + :pr:`24631` by :user:`Chiara Marmo `. + +- |Enhancement| Create wheels for Python 3.11. + :pr:`24446` by :user:`Chiara Marmo `. + +Other bug fixes will be available in the next 1.2 release, which will be +released in the coming weeks. + +Note that support for 32-bit Python on Windows has been dropped in this release. This +is due to the fact that SciPy 1.9.2 also dropped the support for that platform. +Windows users are advised to install the 64-bit version of Python instead. + +.. _changes_1_1_2: + +Version 1.1.2 +============= + +**August 2022** + +Changed models +-------------- + +The following estimators and functions, when fit with the same data and +parameters, may produce different models from the previous version. This often +occurs due to changes in the modelling logic (bug fixes or enhancements), or in +random sampling procedures. + +- |Fix| :class:`manifold.TSNE` now throws a `ValueError` when fit with + `perplexity>=n_samples` to ensure mathematical correctness of the algorithm. + :pr:`10805` by :user:`Mathias Andersen ` and + :pr:`23471` by :user:`Meekail Zain `. + +Changelog +--------- + +- |Fix| A default HTML representation is shown for meta-estimators with invalid + parameters. :pr:`24015` by `Thomas Fan`_. + +- |Fix| Add support for F-contiguous arrays for estimators and functions whose back-end + have been changed in 1.1. + :pr:`23990` by :user:`Julien Jerphanion `. + +- |Fix| Wheels are now available for MacOS 10.9 and greater. :pr:`23833` by + `Thomas Fan`_. + +:mod:`sklearn.base` +................... + +- |Fix| The `get_params` method of the :class:`base.BaseEstimator` class now supports + estimators with `type`-type params that have the `get_params` method. + :pr:`24017` by :user:`Henry Sorsky `. + +:mod:`sklearn.cluster` +...................... + +- |Fix| Fixed a bug in :class:`cluster.Birch` that could trigger an error when splitting + a node if there are duplicates in the dataset. + :pr:`23395` by :user:`JÊrÊmie du Boisberranger `. + +:mod:`sklearn.feature_selection` +................................ + +- |Fix| :class:`feature_selection.SelectFromModel` defaults to selection + threshold 1e-5 when the estimator is either :class:`linear_model.ElasticNet` + or :class:`linear_model.ElasticNetCV` with `l1_ratio` equals 1 or + :class:`linear_model.LassoCV`. + :pr:`23636` by :user:`Hao Chun Chang `. + +:mod:`sklearn.impute` +..................... + +- |Fix| :class:`impute.SimpleImputer` uses the dtype seen in `fit` for + `transform` when the dtype is object. :pr:`22063` by `Thomas Fan`_. + +:mod:`sklearn.linear_model` +........................... + +- |Fix| Use dtype-aware tolerances for the validation of gram matrices (passed by users + or precomputed). :pr:`22059` by :user:`Malte S. Kurz `. + +- |Fix| Fixed an error in :class:`linear_model.LogisticRegression` with + `solver="newton-cg"`, `fit_intercept=True`, and a single feature. :pr:`23608` + by `Tom Dupre la Tour`_. + +:mod:`sklearn.manifold` +....................... + +- |Fix| :class:`manifold.TSNE` now throws a `ValueError` when fit with + `perplexity>=n_samples` to ensure mathematical correctness of the algorithm. + :pr:`10805` by :user:`Mathias Andersen ` and + :pr:`23471` by :user:`Meekail Zain `. + +:mod:`sklearn.metrics` +...................... + +- |Fix| Fixed error message of :class:`metrics.coverage_error` for 1D array input. + :pr:`23548` by :user:`Hao Chun Chang `. + +:mod:`sklearn.preprocessing` +............................ + +- |Fix| :meth:`preprocessing.OrdinalEncoder.inverse_transform` correctly handles + use cases where `unknown_value` or `encoded_missing_value` is `nan`. :pr:`24087` + by `Thomas Fan`_. + +:mod:`sklearn.tree` +................... + +- |Fix| Fixed invalid memory access bug during fit in + :class:`tree.DecisionTreeRegressor` and :class:`tree.DecisionTreeClassifier`. + :pr:`23273` by `Thomas Fan`_. + +.. _changes_1_1_1: + +Version 1.1.1 +============= + +**May 2022** + +Changelog +--------- + +- |Enhancement| The error message is improved when importing + :class:`model_selection.HalvingGridSearchCV`, + :class:`model_selection.HalvingRandomSearchCV`, or + :class:`impute.IterativeImputer` without importing the experimental flag. + :pr:`23194` by `Thomas Fan`_. + +- |Enhancement| Added an extension in doc/conf.py to automatically generate + the list of estimators that handle NaN values. + :pr:`23198` by :user:`Lise Kleiber `, :user:`Zhehao Liu ` + and :user:`Chiara Marmo `. + +:mod:`sklearn.datasets` +....................... + +- |Fix| Avoid timeouts in :func:`datasets.fetch_openml` by not passing a + `timeout` argument, :pr:`23358` by :user:`Loïc Estève `. + +:mod:`sklearn.decomposition` +............................ + +- |Fix| Avoid spurious warning in :class:`decomposition.IncrementalPCA` when + `n_samples == n_components`. :pr:`23264` by :user:`Lucy Liu `. + +:mod:`sklearn.feature_selection` +................................ + +- |Fix| The `partial_fit` method of :class:`feature_selection.SelectFromModel` + now conducts validation for `max_features` and `feature_names_in` parameters. + :pr:`23299` by :user:`Long Bao `. + +:mod:`sklearn.metrics` +...................... + +- |Fix| Fixes :func:`metrics.precision_recall_curve` to compute precision-recall at 100% + recall. The Precision-Recall curve now displays the last point corresponding to a + classifier that always predicts the positive class: recall=100% and + precision=class balance. + :pr:`23214` by :user:`StÊphane Collot ` and :user:`Max Baak `. + +:mod:`sklearn.preprocessing` +............................ + +- |Fix| :class:`preprocessing.PolynomialFeatures` with ``degree`` equal to 0 + will raise error when ``include_bias`` is set to False, and outputs a single + constant array when ``include_bias`` is set to True. + :pr:`23370` by :user:`Zhehao Liu `. + +:mod:`sklearn.tree` +................... + +- |Fix| Fixes performance regression with low cardinality features for + :class:`tree.DecisionTreeClassifier`, + :class:`tree.DecisionTreeRegressor`, + :class:`ensemble.RandomForestClassifier`, + :class:`ensemble.RandomForestRegressor`, + :class:`ensemble.GradientBoostingClassifier`, and + :class:`ensemble.GradientBoostingRegressor`. + :pr:`23410` by :user:`Loïc Estève `. + +:mod:`sklearn.utils` +.................... + +- |Fix| :func:`utils.class_weight.compute_sample_weight` now works with sparse `y`. + :pr:`23115` by :user:`kernc `. + +.. _changes_1_1: + +Version 1.1.0 +============= + +**May 2022** + +Minimal dependencies +-------------------- + +Version 1.1.0 of scikit-learn requires python 3.8+, numpy 1.17.3+ and +scipy 1.3.2+. Optional minimal dependency is matplotlib 3.1.2+. + +Changed models +-------------- + +The following estimators and functions, when fit with the same data and +parameters, may produce different models from the previous version. This often +occurs due to changes in the modelling logic (bug fixes or enhancements), or in +random sampling procedures. + +- |Efficiency| :class:`cluster.KMeans` now defaults to ``algorithm="lloyd"`` + instead of ``algorithm="auto"``, which was equivalent to + ``algorithm="elkan"``. Lloyd's algorithm and Elkan's algorithm converge to the + same solution, up to numerical rounding errors, but in general Lloyd's + algorithm uses much less memory, and it is often faster. + +- |Efficiency| Fitting :class:`tree.DecisionTreeClassifier`, + :class:`tree.DecisionTreeRegressor`, + :class:`ensemble.RandomForestClassifier`, + :class:`ensemble.RandomForestRegressor`, + :class:`ensemble.GradientBoostingClassifier`, and + :class:`ensemble.GradientBoostingRegressor` is on average 15% faster than in + previous versions thanks to a new sort algorithm to find the best split. + Models might be different because of a different handling of splits + with tied criterion values: both the old and the new sorting algorithm + are unstable sorting algorithms. :pr:`22868` by `Thomas Fan`_. + +- |Fix| The eigenvectors initialization for :class:`cluster.SpectralClustering` + and :class:`manifold.SpectralEmbedding` now samples from a Gaussian when + using the `'amg'` or `'lobpcg'` solver. This change improves numerical + stability of the solver, but may result in a different model. + +- |Fix| :func:`feature_selection.f_regression` and + :func:`feature_selection.r_regression` will now return finite score by + default instead of `np.nan` and `np.inf` for some corner case. You can use + `force_finite=False` if you really want to get non-finite values and keep + the old behavior. + +- |Fix| Panda's DataFrames with all non-string columns such as a MultiIndex no + longer warns when passed into an Estimator. Estimators will continue to + ignore the column names in DataFrames with non-string columns. For + `feature_names_in_` to be defined, columns must be all strings. :pr:`22410` by + `Thomas Fan`_. + +- |Fix| :class:`preprocessing.KBinsDiscretizer` changed handling of bin edges + slightly, which might result in a different encoding with the same data. + +- |Fix| :func:`calibration.calibration_curve` changed handling of bin + edges slightly, which might result in a different output curve given the same + data. + +- |Fix| :class:`discriminant_analysis.LinearDiscriminantAnalysis` now uses + the correct variance-scaling coefficient which may result in different model + behavior. + +- |Fix| :meth:`feature_selection.SelectFromModel.fit` and + :meth:`feature_selection.SelectFromModel.partial_fit` can now be called with + `prefit=True`. `estimators_` will be a deep copy of `estimator` when + `prefit=True`. :pr:`23271` by :user:`Guillaume Lemaitre `. + +Changelog +--------- + +.. + Entries should be grouped by module (in alphabetic order) and prefixed with + one of the labels: |MajorFeature|, |Feature|, |Efficiency|, |Enhancement|, + |Fix| or |API| (see whats_new.rst for descriptions). + Entries should be ordered by those labels (e.g. |Fix| after |Efficiency|). + Changes not specific to a module should be listed under *Multiple Modules* + or *Miscellaneous*. + Entries should end with: + :pr:`123456` by :user:`Joe Bloggs `. + where 123456 is the *pull request* number, not the issue number. + + +- |Efficiency| Low-level routines for reductions on pairwise distances + for dense float64 datasets have been refactored. The following functions + and estimators now benefit from improved performances in terms of hardware + scalability and speed-ups: + + - :func:`sklearn.metrics.pairwise_distances_argmin` + - :func:`sklearn.metrics.pairwise_distances_argmin_min` + - :class:`sklearn.cluster.AffinityPropagation` + - :class:`sklearn.cluster.Birch` + - :class:`sklearn.cluster.MeanShift` + - :class:`sklearn.cluster.OPTICS` + - :class:`sklearn.cluster.SpectralClustering` + - :func:`sklearn.feature_selection.mutual_info_regression` + - :class:`sklearn.neighbors.KNeighborsClassifier` + - :class:`sklearn.neighbors.KNeighborsRegressor` + - :class:`sklearn.neighbors.RadiusNeighborsClassifier` + - :class:`sklearn.neighbors.RadiusNeighborsRegressor` + - :class:`sklearn.neighbors.LocalOutlierFactor` + - :class:`sklearn.neighbors.NearestNeighbors` + - :class:`sklearn.manifold.Isomap` + - :class:`sklearn.manifold.LocallyLinearEmbedding` + - :class:`sklearn.manifold.TSNE` + - :func:`sklearn.manifold.trustworthiness` + - :class:`sklearn.semi_supervised.LabelPropagation` + - :class:`sklearn.semi_supervised.LabelSpreading` + + For instance :class:`sklearn.neighbors.NearestNeighbors.kneighbors` and + :class:`sklearn.neighbors.NearestNeighbors.radius_neighbors` + can respectively be up to ×20 and ×5 faster than previously on a laptop. + + Moreover, implementations of those two algorithms are now suitable + for machine with many cores, making them usable for datasets consisting + of millions of samples. + + :pr:`21987`, :pr:`22064`, :pr:`22065`, :pr:`22288` and :pr:`22320` + by :user:`Julien Jerphanion `. + +- |Enhancement| All scikit-learn models now generate a more informative + error message when some input contains unexpected `NaN` or infinite values. + In particular the message contains the input name ("X", "y" or + "sample_weight") and if an unexpected `NaN` value is found in `X`, the error + message suggests potential solutions. + :pr:`21219` by :user:`Olivier Grisel `. + +- |Enhancement| All scikit-learn models now generate a more informative + error message when setting invalid hyper-parameters with `set_params`. + :pr:`21542` by :user:`Olivier Grisel `. + +- |Enhancement| Removes random unique identifiers in the HTML representation. + With this change, jupyter notebooks are reproducible as long as the cells are + run in the same order. :pr:`23098` by `Thomas Fan`_. + +- |Fix| Estimators with `non_deterministic` tag set to `True` will skip both + `check_methods_sample_order_invariance` and `check_methods_subset_invariance` tests. + :pr:`22318` by :user:`Zhehao Liu `. + +- |API| The option for using the log loss, aka binomial or multinomial deviance, via + the `loss` parameters was made more consistent. The preferred way is by + setting the value to `"log_loss"`. Old option names are still valid and + produce the same models, but are deprecated and will be removed in version + 1.3. + + - For :class:`ensemble.GradientBoostingClassifier`, the `loss` parameter name + "deviance" is deprecated in favor of the new name "log_loss", which is now the + default. + :pr:`23036` by :user:`Christian Lorentzen `. + + - For :class:`ensemble.HistGradientBoostingClassifier`, the `loss` parameter names + "auto", "binary_crossentropy" and "categorical_crossentropy" are deprecated in + favor of the new name "log_loss", which is now the default. + :pr:`23040` by :user:`Christian Lorentzen `. + + - For :class:`linear_model.SGDClassifier`, the `loss` parameter name + "log" is deprecated in favor of the new name "log_loss". + :pr:`23046` by :user:`Christian Lorentzen `. + +- |API| Rich html representation of estimators is now enabled by default in Jupyter + notebooks. It can be deactivated by setting `display='text'` in + :func:`sklearn.set_config`. + :pr:`22856` by :user:`JÊrÊmie du Boisberranger `. + +:mod:`sklearn.calibration` +.......................... + +- |Enhancement| :func:`calibration.calibration_curve` accepts a parameter + `pos_label` to specify the positive class label. + :pr:`21032` by :user:`Guillaume Lemaitre `. + +- |Enhancement| :meth:`calibration.CalibratedClassifierCV.fit` now supports passing + `fit_params`, which are routed to the `base_estimator`. + :pr:`18170` by :user:`Benjamin Bossan `. + +- |Enhancement| :class:`calibration.CalibrationDisplay` accepts a parameter `pos_label` + to add this information to the plot. + :pr:`21038` by :user:`Guillaume Lemaitre `. + +- |Fix| :func:`calibration.calibration_curve` handles bin edges more consistently now. + :pr:`14975` by `Andreas MÃŧller`_ and :pr:`22526` by :user:`Meekail Zain `. + +- |API| :func:`calibration.calibration_curve`'s `normalize` parameter is + now deprecated and will be removed in version 1.3. It is recommended that + a proper probability (i.e. a classifier's :term:`predict_proba` positive + class) is used for `y_prob`. + :pr:`23095` by :user:`Jordan Silke `. + +:mod:`sklearn.cluster` +...................... + +- |MajorFeature| :class:`cluster.BisectingKMeans` introducing Bisecting K-Means algorithm + :pr:`20031` by :user:`Michal Krawczyk `, + :user:`Tom Dupre la Tour ` + and :user:`JÊrÊmie du Boisberranger `. + +- |Enhancement| :class:`cluster.SpectralClustering` and + :func:`cluster.spectral_clustering` now include the new `'cluster_qr'` method that + clusters samples in the embedding space as an alternative to the existing `'kmeans'` + and `'discrete'` methods. See :func:`cluster.spectral_clustering` for more details. + :pr:`21148` by :user:`Andrew Knyazev `. + +- |Enhancement| Adds :term:`get_feature_names_out` to :class:`cluster.Birch`, + :class:`cluster.FeatureAgglomeration`, :class:`cluster.KMeans`, + :class:`cluster.MiniBatchKMeans`. :pr:`22255` by `Thomas Fan`_. + +- |Enhancement| :class:`cluster.SpectralClustering` now raises consistent + error messages when passed invalid values for `n_clusters`, `n_init`, + `gamma`, `n_neighbors`, `eigen_tol` or `degree`. + :pr:`21881` by :user:`Hugo Vassard `. + +- |Enhancement| :class:`cluster.AffinityPropagation` now returns cluster + centers and labels if they exist, even if the model has not fully converged. + When returning these potentially-degenerate cluster centers and labels, a new + warning message is shown. If no cluster centers were constructed, + then the cluster centers remain an empty list with labels set to + `-1` and the original warning message is shown. + :pr:`22217` by :user:`Meekail Zain `. + +- |Efficiency| In :class:`cluster.KMeans`, the default ``algorithm`` is now + ``"lloyd"`` which is the full classical EM-style algorithm. Both ``"auto"`` + and ``"full"`` are deprecated and will be removed in version 1.3. They are + now aliases for ``"lloyd"``. The previous default was ``"auto"``, which relied + on Elkan's algorithm. Lloyd's algorithm uses less memory than Elkan's, it + is faster on many datasets, and its results are identical, hence the change. + :pr:`21735` by :user:`AurÊlien Geron `. + +- |Fix| :class:`cluster.KMeans`'s `init` parameter now properly supports + array-like input and NumPy string scalars. :pr:`22154` by `Thomas Fan`_. + +:mod:`sklearn.compose` +...................... + +- |Fix| :class:`compose.ColumnTransformer` now removes validation errors from + `__init__` and `set_params` methods. + :pr:`22537` by :user:`iofall ` and :user:`Arisa Y. `. + +- |Fix| :term:`get_feature_names_out` functionality in + :class:`compose.ColumnTransformer` was broken when columns were specified + using `slice`. This is fixed in :pr:`22775` and :pr:`22913` by + :user:`randomgeek78 `. + +:mod:`sklearn.covariance` +......................... + +- |Fix| :class:`covariance.GraphicalLassoCV` now accepts NumPy array for the + parameter `alphas`. + :pr:`22493` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.cross_decomposition` +.................................. + +- |Enhancement| the `inverse_transform` method of + :class:`cross_decomposition.PLSRegression`, :class:`cross_decomposition.PLSCanonical` + and :class:`cross_decomposition.CCA` now allows reconstruction of a `X` target when + a `Y` parameter is given. :pr:`19680` by + :user:`Robin Thibaut `. + +- |Enhancement| Adds :term:`get_feature_names_out` to all transformers in the + :mod:`~sklearn.cross_decomposition` module: + :class:`cross_decomposition.CCA`, + :class:`cross_decomposition.PLSSVD`, + :class:`cross_decomposition.PLSRegression`, + and :class:`cross_decomposition.PLSCanonical`. :pr:`22119` by `Thomas Fan`_. + +- |Fix| The shape of the :term:`coef_` attribute of :class:`cross_decomposition.CCA`, + :class:`cross_decomposition.PLSCanonical` and + :class:`cross_decomposition.PLSRegression` will change in version 1.3, from + `(n_features, n_targets)` to `(n_targets, n_features)`, to be consistent + with other linear models and to make it work with interface expecting a + specific shape for `coef_` (e.g. :class:`feature_selection.RFE`). + :pr:`22016` by :user:`Guillaume Lemaitre `. + +- |API| add the fitted attribute `intercept_` to + :class:`cross_decomposition.PLSCanonical`, + :class:`cross_decomposition.PLSRegression`, and + :class:`cross_decomposition.CCA`. The method `predict` is indeed equivalent to + `Y = X @ coef_ + intercept_`. + :pr:`22015` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.datasets` +....................... + +- |Feature| :func:`datasets.load_files` now accepts an ignore list and + an allow list based on file extensions. + :pr:`19747` by :user:`Tony Attalla ` and :pr:`22498` by + :user:`Meekail Zain `. + +- |Enhancement| :func:`datasets.make_swiss_roll` now supports the optional argument + hole; when set to True, it returns the swiss-hole dataset. :pr:`21482` by + :user:`Sebastian Pujalte `. + +- |Enhancement| :func:`datasets.make_blobs` no longer copies data during the generation + process, therefore uses less memory. + :pr:`22412` by :user:`Zhehao Liu `. + +- |Enhancement| :func:`datasets.load_diabetes` now accepts the parameter + ``scaled``, to allow loading unscaled data. The scaled version of this + dataset is now computed from the unscaled data, and can produce slightly + different results than in previous version (within a 1e-4 absolute + tolerance). + :pr:`16605` by :user:`Mandy Gu `. + +- |Enhancement| :func:`datasets.fetch_openml` now has two optional arguments + `n_retries` and `delay`. By default, :func:`datasets.fetch_openml` will retry + 3 times in case of a network failure with a delay between each try. + :pr:`21901` by :user:`Rileran `. + +- |Fix| :func:`datasets.fetch_covtype` is now concurrent-safe: data is downloaded + to a temporary directory before being moved to the data directory. + :pr:`23113` by :user:`Ilion Beyst `. + +- |API| :func:`datasets.make_sparse_coded_signal` now accepts a parameter + `data_transposed` to explicitly specify the shape of matrix `X`. The default + behavior `True` is to return a transposed matrix `X` corresponding to a + `(n_features, n_samples)` shape. The default value will change to `False` in + version 1.3. :pr:`21425` by :user:`Gabriel Stefanini Vicente `. + +:mod:`sklearn.decomposition` +............................ + +- |MajorFeature| Added a new estimator :class:`decomposition.MiniBatchNMF`. It is a + faster but less accurate version of non-negative matrix factorization, better suited + for large datasets. :pr:`16948` by :user:`Chiara Marmo `, + :user:`Patricio Cerda ` and :user:`JÊrÊmie du Boisberranger `. + +- |Enhancement| :func:`decomposition.dict_learning`, + :func:`decomposition.dict_learning_online` + and :func:`decomposition.sparse_encode` preserve dtype for `numpy.float32`. + :class:`decomposition.DictionaryLearning`, + :class:`decomposition.MiniBatchDictionaryLearning` + and :class:`decomposition.SparseCoder` preserve dtype for `numpy.float32`. + :pr:`22002` by :user:`Takeshi Oura `. + +- |Enhancement| :class:`decomposition.PCA` exposes a parameter `n_oversamples` to tune + :func:`utils.extmath.randomized_svd` and get accurate results when the number of + features is large. + :pr:`21109` by :user:`Smile `. + +- |Enhancement| The :class:`decomposition.MiniBatchDictionaryLearning` and + :func:`decomposition.dict_learning_online` have been refactored and now have a + stopping criterion based on a small change of the dictionary or objective function, + controlled by the new `max_iter`, `tol` and `max_no_improvement` parameters. In + addition, some of their parameters and attributes are deprecated. + + - the `n_iter` parameter of both is deprecated. Use `max_iter` instead. + - the `iter_offset`, `return_inner_stats`, `inner_stats` and `return_n_iter` + parameters of :func:`decomposition.dict_learning_online` serve internal purpose + and are deprecated. + - the `inner_stats_`, `iter_offset_` and `random_state_` attributes of + :class:`decomposition.MiniBatchDictionaryLearning` serve internal purpose and are + deprecated. + - the default value of the `batch_size` parameter of both will change from 3 to 256 + in version 1.3. + + :pr:`18975` by :user:`JÊrÊmie du Boisberranger `. + +- |Enhancement| :class:`decomposition.SparsePCA` and :class:`decomposition.MiniBatchSparsePCA` + preserve dtype for `numpy.float32`. + :pr:`22111` by :user:`Takeshi Oura `. + +- |Enhancement| :class:`decomposition.TruncatedSVD` now allows + `n_components == n_features`, if `algorithm='randomized'`. + :pr:`22181` by :user:`Zach Deane-Mayer `. + +- |Enhancement| Adds :term:`get_feature_names_out` to all transformers in the + :mod:`~sklearn.decomposition` module: + :class:`decomposition.DictionaryLearning`, + :class:`decomposition.FactorAnalysis`, + :class:`decomposition.FastICA`, + :class:`decomposition.IncrementalPCA`, + :class:`decomposition.KernelPCA`, + :class:`decomposition.LatentDirichletAllocation`, + :class:`decomposition.MiniBatchDictionaryLearning`, + :class:`decomposition.MiniBatchSparsePCA`, + :class:`decomposition.NMF`, + :class:`decomposition.PCA`, + :class:`decomposition.SparsePCA`, + and :class:`decomposition.TruncatedSVD`. :pr:`21334` by + `Thomas Fan`_. + +- |Enhancement| :class:`decomposition.TruncatedSVD` exposes the parameter + `n_oversamples` and `power_iteration_normalizer` to tune + :func:`utils.extmath.randomized_svd` and get accurate results when the number + of features is large, the rank of the matrix is high, or other features of + the matrix make low rank approximation difficult. + :pr:`21705` by :user:`Jay S. Stanley III `. + +- |Enhancement| :class:`decomposition.PCA` exposes the parameter + `power_iteration_normalizer` to tune :func:`utils.extmath.randomized_svd` and + get more accurate results when low rank approximation is difficult. + :pr:`21705` by :user:`Jay S. Stanley III `. + +- |Fix| :class:`decomposition.FastICA` now validates input parameters in `fit` + instead of `__init__`. + :pr:`21432` by :user:`Hannah Bohle ` and + :user:`Maren Westermann `. + +- |Fix| :class:`decomposition.FastICA` now accepts `np.float32` data without + silent upcasting. The dtype is preserved by `fit` and `fit_transform` and the + main fitted attributes use a dtype of the same precision as the training + data. :pr:`22806` by :user:`Jihane Bennis ` and + :user:`Olivier Grisel `. + +- |Fix| :class:`decomposition.FactorAnalysis` now validates input parameters + in `fit` instead of `__init__`. + :pr:`21713` by :user:`Haya ` and :user:`Krum Arnaudov `. + +- |Fix| :class:`decomposition.KernelPCA` now validates input parameters in + `fit` instead of `__init__`. + :pr:`21567` by :user:`Maggie Chege `. + +- |Fix| :class:`decomposition.PCA` and :class:`decomposition.IncrementalPCA` + more safely calculate precision using the inverse of the covariance matrix + if `self.noise_variance_` is zero. + :pr:`22300` by :user:`Meekail Zain ` and :pr:`15948` by :user:`sysuresh`. + +- |Fix| Greatly reduced peak memory usage in :class:`decomposition.PCA` when + calling `fit` or `fit_transform`. + :pr:`22553` by :user:`Meekail Zain `. + +- |API| :func:`decomposition.FastICA` now supports unit variance for whitening. + The default value of its `whiten` argument will change from `True` + (which behaves like `'arbitrary-variance'`) to `'unit-variance'` in version 1.3. + :pr:`19490` by :user:`Facundo Ferrin ` and + :user:`Julien Jerphanion `. + +:mod:`sklearn.discriminant_analysis` +.................................... + +- |Enhancement| Adds :term:`get_feature_names_out` to + :class:`discriminant_analysis.LinearDiscriminantAnalysis`. :pr:`22120` by + `Thomas Fan`_. + +- |Fix| :class:`discriminant_analysis.LinearDiscriminantAnalysis` now uses + the correct variance-scaling coefficient which may result in different model + behavior. :pr:`15984` by :user:`Okon Samuel ` and :pr:`22696` by + :user:`Meekail Zain `. + +:mod:`sklearn.dummy` +.................... + +- |Fix| :class:`dummy.DummyRegressor` no longer overrides the `constant` + parameter during `fit`. :pr:`22486` by `Thomas Fan`_. + +:mod:`sklearn.ensemble` +....................... + +- |MajorFeature| Added additional option `loss="quantile"` to + :class:`ensemble.HistGradientBoostingRegressor` for modelling quantiles. + The quantile level can be specified with the new parameter `quantile`. + :pr:`21800` and :pr:`20567` by :user:`Christian Lorentzen `. + +- |Efficiency| `fit` of :class:`ensemble.GradientBoostingClassifier` + and :class:`ensemble.GradientBoostingRegressor` now calls :func:`utils.check_array` + with parameter `force_all_finite=False` for non initial warm-start runs as it has + already been checked before. + :pr:`22159` by :user:`Geoffrey Paris `. + +- |Enhancement| :class:`ensemble.HistGradientBoostingClassifier` is faster, + for binary and in particular for multiclass problems thanks to the new private loss + function module. + :pr:`20811`, :pr:`20567` and :pr:`21814` by + :user:`Christian Lorentzen `. + +- |Enhancement| Adds support to use pre-fit models with `cv="prefit"` + in :class:`ensemble.StackingClassifier` and :class:`ensemble.StackingRegressor`. + :pr:`16748` by :user:`Siqi He ` and :pr:`22215` by + :user:`Meekail Zain `. + +- |Enhancement| :class:`ensemble.RandomForestClassifier` and + :class:`ensemble.ExtraTreesClassifier` have the new `criterion="log_loss"`, which is + equivalent to `criterion="entropy"`. + :pr:`23047` by :user:`Christian Lorentzen `. + +- |Enhancement| Adds :term:`get_feature_names_out` to + :class:`ensemble.VotingClassifier`, :class:`ensemble.VotingRegressor`, + :class:`ensemble.StackingClassifier`, and + :class:`ensemble.StackingRegressor`. :pr:`22695` and :pr:`22697` by `Thomas Fan`_. + +- |Enhancement| :class:`ensemble.RandomTreesEmbedding` now has an informative + :term:`get_feature_names_out` function that includes both tree index and leaf index in + the output feature names. + :pr:`21762` by :user:`Zhehao Liu ` and `Thomas Fan`_. + +- |Efficiency| Fitting a :class:`ensemble.RandomForestClassifier`, + :class:`ensemble.RandomForestRegressor`, :class:`ensemble.ExtraTreesClassifier`, + :class:`ensemble.ExtraTreesRegressor`, and :class:`ensemble.RandomTreesEmbedding` + is now faster in a multiprocessing setting, especially for subsequent fits with + `warm_start` enabled. + :pr:`22106` by :user:`Pieter Gijsbers `. + +- |Fix| Change the parameter `validation_fraction` in + :class:`ensemble.GradientBoostingClassifier` and + :class:`ensemble.GradientBoostingRegressor` so that an error is raised if anything + other than a float is passed in as an argument. + :pr:`21632` by :user:`Genesis Valencia `. + +- |Fix| Removed a potential source of CPU oversubscription in + :class:`ensemble.HistGradientBoostingClassifier` and + :class:`ensemble.HistGradientBoostingRegressor` when CPU resource usage is limited, + for instance using cgroups quota in a docker container. :pr:`22566` by + :user:`JÊrÊmie du Boisberranger `. + +- |Fix| :class:`ensemble.HistGradientBoostingClassifier` and + :class:`ensemble.HistGradientBoostingRegressor` no longer warn when + fitting on a pandas DataFrame with a non-default `scoring` parameter and + early_stopping enabled. :pr:`22908` by `Thomas Fan`_. + +- |Fix| Fixes HTML repr for :class:`ensemble.StackingClassifier` and + :class:`ensemble.StackingRegressor`. :pr:`23097` by `Thomas Fan`_. + +- |API| The attribute `loss_` of :class:`ensemble.GradientBoostingClassifier` and + :class:`ensemble.GradientBoostingRegressor` has been deprecated and will be removed + in version 1.3. + :pr:`23079` by :user:`Christian Lorentzen `. + +- |API| Changed the default of `max_features` to 1.0 for + :class:`ensemble.RandomForestRegressor` and to `"sqrt"` for + :class:`ensemble.RandomForestClassifier`. Note that these give the same fit + results as before, but are much easier to understand. The old default value + `"auto"` has been deprecated and will be removed in version 1.3. The same + changes are also applied for :class:`ensemble.ExtraTreesRegressor` and + :class:`ensemble.ExtraTreesClassifier`. + :pr:`20803` by :user:`Brian Sun `. + +- |Efficiency| Improve runtime performance of :class:`ensemble.IsolationForest` + by skipping repetitive input checks. :pr:`23149` by :user:`Zhehao Liu `. + +:mod:`sklearn.feature_extraction` +................................. + +- |Feature| :class:`feature_extraction.FeatureHasher` now supports PyPy. + :pr:`23023` by `Thomas Fan`_. + +- |Fix| :class:`feature_extraction.FeatureHasher` now validates input parameters + in `transform` instead of `__init__`. :pr:`21573` by + :user:`Hannah Bohle ` and :user:`Maren Westermann `. + +- |Fix| :class:`feature_extraction.text.TfidfVectorizer` now does not create + a :class:`feature_extraction.text.TfidfTransformer` at `__init__` as required + by our API. + :pr:`21832` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.feature_selection` +................................ + +- |Feature| Added auto mode to :class:`feature_selection.SequentialFeatureSelector`. + If the argument `n_features_to_select` is `'auto'`, select features until the score + improvement does not exceed the argument `tol`. The default value of + `n_features_to_select` changed from `None` to `'warn'` in 1.1 and will become + `'auto'` in 1.3. `None` and `'warn'` will be removed in 1.3. :pr:`20145` by + :user:`murata-yu `. + +- |Feature| Added the ability to pass callables to the `max_features` parameter + of :class:`feature_selection.SelectFromModel`. Also introduced new attribute + `max_features_` which is inferred from `max_features` and the data during + `fit`. If `max_features` is an integer, then `max_features_ = max_features`. + If `max_features` is a callable, then `max_features_ = max_features(X)`. + :pr:`22356` by :user:`Meekail Zain `. + +- |Enhancement| :class:`feature_selection.GenericUnivariateSelect` preserves + float32 dtype. :pr:`18482` by :user:`Thierry Gameiro ` + and :user:`Daniel Kharsa ` and :pr:`22370` by + :user:`Meekail Zain `. + +- |Enhancement| Add a parameter `force_finite` to + :func:`feature_selection.f_regression` and + :func:`feature_selection.r_regression`. This parameter allows to force the + output to be finite in the case where a feature or the target is constant + or that the feature and target are perfectly correlated (only for the + F-statistic). + :pr:`17819` by :user:`Juan Carlos Alfaro JimÊnez `. + +- |Efficiency| Improve runtime performance of :func:`feature_selection.chi2` + with boolean arrays. :pr:`22235` by `Thomas Fan`_. + +- |Efficiency| Reduced memory usage of :func:`feature_selection.chi2`. + :pr:`21837` by :user:`Louis Wagner `. + +:mod:`sklearn.gaussian_process` +............................... + +- |Fix| `predict` and `sample_y` methods of + :class:`gaussian_process.GaussianProcessRegressor` now return + arrays of the correct shape in single-target and multi-target cases, and for + both `normalize_y=False` and `normalize_y=True`. + :pr:`22199` by :user:`Guillaume Lemaitre `, + :user:`Aidar Shakerimoff ` and + :user:`Tenavi Nakamura-Zimmerer `. + +- |Fix| :class:`gaussian_process.GaussianProcessClassifier` raises + a more informative error if `CompoundKernel` is passed via `kernel`. + :pr:`22223` by :user:`MarcoM `. + +:mod:`sklearn.impute` +..................... + +- |Enhancement| :class:`impute.SimpleImputer` now warns with feature names when features + which are skipped due to the lack of any observed values in the training set. + :pr:`21617` by :user:`Christian Ritter `. + +- |Enhancement| Added support for `pd.NA` in :class:`impute.SimpleImputer`. + :pr:`21114` by :user:`Ying Xiong `. + +- |Enhancement| Adds :term:`get_feature_names_out` to + :class:`impute.SimpleImputer`, :class:`impute.KNNImputer`, + :class:`impute.IterativeImputer`, and :class:`impute.MissingIndicator`. + :pr:`21078` by `Thomas Fan`_. + +- |API| The `verbose` parameter was deprecated for :class:`impute.SimpleImputer`. + A warning will always be raised upon the removal of empty columns. + :pr:`21448` by :user:`Oleh Kozynets ` and + :user:`Christian Ritter `. + +:mod:`sklearn.inspection` +......................... + +- |Feature| Add a display to plot the boundary decision of a classifier by + using the method :func:`inspection.DecisionBoundaryDisplay.from_estimator`. + :pr:`16061` by `Thomas Fan`_. + +- |Enhancement| In + :meth:`inspection.PartialDependenceDisplay.from_estimator`, allow + `kind` to accept a list of strings to specify which type of + plot to draw for each feature interaction. + :pr:`19438` by :user:`Guillaume Lemaitre `. + +- |Enhancement| :meth:`inspection.PartialDependenceDisplay.from_estimator`, + :meth:`inspection.PartialDependenceDisplay.plot`, and + `inspection.plot_partial_dependence` now support plotting centered + Individual Conditional Expectation (cICE) and centered PDP curves controlled + by setting the parameter `centered`. + :pr:`18310` by :user:`Johannes Elfner ` and + :user:`Guillaume Lemaitre `. + +:mod:`sklearn.isotonic` +....................... + +- |Enhancement| Adds :term:`get_feature_names_out` to + :class:`isotonic.IsotonicRegression`. + :pr:`22249` by `Thomas Fan`_. + +:mod:`sklearn.kernel_approximation` +................................... + +- |Enhancement| Adds :term:`get_feature_names_out` to + :class:`kernel_approximation.AdditiveChi2Sampler`. + :class:`kernel_approximation.Nystroem`, + :class:`kernel_approximation.PolynomialCountSketch`, + :class:`kernel_approximation.RBFSampler`, and + :class:`kernel_approximation.SkewedChi2Sampler`. + :pr:`22137` and :pr:`22694` by `Thomas Fan`_. + +:mod:`sklearn.linear_model` +........................... + +- |Feature| :class:`linear_model.ElasticNet`, :class:`linear_model.ElasticNetCV`, + :class:`linear_model.Lasso` and :class:`linear_model.LassoCV` support `sample_weight` + for sparse input `X`. + :pr:`22808` by :user:`Christian Lorentzen `. + +- |Feature| :class:`linear_model.Ridge` with `solver="lsqr"` now supports to fit sparse + input with `fit_intercept=True`. + :pr:`22950` by :user:`Christian Lorentzen `. + +- |Enhancement| :class:`linear_model.QuantileRegressor` support sparse input + for the highs based solvers. + :pr:`21086` by :user:`Venkatachalam Natchiappan `. + In addition, those solvers now use the CSC matrix right from the + beginning which speeds up fitting. + :pr:`22206` by :user:`Christian Lorentzen `. + +- |Enhancement| :class:`linear_model.LogisticRegression` is faster for + ``solvers="lbfgs"`` and ``solver="newton-cg"``, for binary and in particular for + multiclass problems thanks to the new private loss function module. In the multiclass + case, the memory consumption has also been reduced for these solvers as the target is + now label encoded (mapped to integers) instead of label binarized (one-hot encoded). + The more classes, the larger the benefit. + :pr:`21808`, :pr:`20567` and :pr:`21814` by + :user:`Christian Lorentzen `. + +- |Enhancement| :class:`linear_model.GammaRegressor`, + :class:`linear_model.PoissonRegressor` and :class:`linear_model.TweedieRegressor` + are faster for ``solvers="lbfgs"``. + :pr:`22548`, :pr:`21808` and :pr:`20567` by + :user:`Christian Lorentzen `. + +- |Enhancement| Rename parameter `base_estimator` to `estimator` in + :class:`linear_model.RANSACRegressor` to improve readability and consistency. + `base_estimator` is deprecated and will be removed in 1.3. + :pr:`22062` by :user:`Adrian Trujillo `. + +- |Enhancement| :func:`linear_model.ElasticNet` and + other linear model classes using coordinate descent show error + messages when non-finite parameter weights are produced. :pr:`22148` + by :user:`Christian Ritter ` and :user:`Norbert Preining `. + +- |Enhancement| :class:`linear_model.ElasticNet` and :class:`linear_model.Lasso` + now raise consistent error messages when passed invalid values for `l1_ratio`, + `alpha`, `max_iter` and `tol`. + :pr:`22240` by :user:`Arturo Amor `. + +- |Enhancement| :class:`linear_model.BayesianRidge` and + :class:`linear_model.ARDRegression` now preserve float32 dtype. :pr:`9087` by + :user:`Arthur Imbert ` and :pr:`22525` by :user:`Meekail Zain `. + +- |Enhancement| :class:`linear_model.RidgeClassifier` is now supporting + multilabel classification. + :pr:`19689` by :user:`Guillaume Lemaitre `. + +- |Enhancement| :class:`linear_model.RidgeCV` and + :class:`linear_model.RidgeClassifierCV` now raise consistent error message + when passed invalid values for `alphas`. + :pr:`21606` by :user:`Arturo Amor `. + +- |Enhancement| :class:`linear_model.Ridge` and :class:`linear_model.RidgeClassifier` + now raise consistent error message when passed invalid values for `alpha`, + `max_iter` and `tol`. + :pr:`21341` by :user:`Arturo Amor `. + +- |Enhancement| :func:`linear_model.orthogonal_mp_gram` preserves dtype for + `numpy.float32`. + :pr:`22002` by :user:`Takeshi Oura `. + +- |Fix| :class:`linear_model.LassoLarsIC` now correctly computes AIC + and BIC. An error is now raised when `n_features > n_samples` and + when the noise variance is not provided. + :pr:`21481` by :user:`Guillaume Lemaitre ` and + :user:`AndrÊs Babino `. + +- |Fix| :class:`linear_model.TheilSenRegressor` now validates input parameter + ``max_subpopulation`` in `fit` instead of `__init__`. + :pr:`21767` by :user:`Maren Westermann `. + +- |Fix| :class:`linear_model.ElasticNetCV` now produces correct + warning when `l1_ratio=0`. + :pr:`21724` by :user:`Yar Khine Phyo `. + +- |Fix| :class:`linear_model.LogisticRegression` and + :class:`linear_model.LogisticRegressionCV` now set the `n_iter_` attribute + with a shape that respects the docstring and that is consistent with the shape + obtained when using the other solvers in the one-vs-rest setting. Previously, + it would record only the maximum of the number of iterations for each binary + sub-problem while now all of them are recorded. :pr:`21998` by + :user:`Olivier Grisel `. + +- |Fix| The property `family` of :class:`linear_model.TweedieRegressor` is not + validated in `__init__` anymore. Instead, this (private) property is deprecated in + :class:`linear_model.GammaRegressor`, :class:`linear_model.PoissonRegressor` and + :class:`linear_model.TweedieRegressor`, and will be removed in 1.3. + :pr:`22548` by :user:`Christian Lorentzen `. + +- |Fix| The `coef_` and `intercept_` attributes of + :class:`linear_model.LinearRegression` are now correctly computed in the presence of + sample weights when the input is sparse. + :pr:`22891` by :user:`JÊrÊmie du Boisberranger `. + +- |Fix| The `coef_` and `intercept_` attributes of :class:`linear_model.Ridge` with + `solver="sparse_cg"` and `solver="lbfgs"` are now correctly computed in the presence + of sample weights when the input is sparse. + :pr:`22899` by :user:`JÊrÊmie du Boisberranger `. + +- |Fix| :class:`linear_model.SGDRegressor` and :class:`linear_model.SGDClassifier` now + compute the validation error correctly when early stopping is enabled. + :pr:`23256` by :user:`Zhehao Liu `. + +- |API| :class:`linear_model.LassoLarsIC` now exposes `noise_variance` as + a parameter in order to provide an estimate of the noise variance. + This is particularly relevant when `n_features > n_samples` and the + estimator of the noise variance cannot be computed. + :pr:`21481` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.manifold` +....................... + +- |Feature| :class:`manifold.Isomap` now supports radius-based + neighbors via the `radius` argument. + :pr:`19794` by :user:`Zhehao Liu `. + +- |Enhancement| :func:`manifold.spectral_embedding` and + :class:`manifold.SpectralEmbedding` support `np.float32` dtype and will + preserve this dtype. + :pr:`21534` by :user:`Andrew Knyazev `. + +- |Enhancement| Adds :term:`get_feature_names_out` to :class:`manifold.Isomap` + and :class:`manifold.LocallyLinearEmbedding`. :pr:`22254` by `Thomas Fan`_. + +- |Enhancement| added `metric_params` to :class:`manifold.TSNE` constructor for + additional parameters of distance metric to use in optimization. + :pr:`21805` by :user:`Jeanne Dionisi ` and :pr:`22685` by + :user:`Meekail Zain `. + +- |Enhancement| :func:`manifold.trustworthiness` raises an error if + `n_neighbours >= n_samples / 2` to ensure a correct support for the function. + :pr:`18832` by :user:`Hong Shao Yang ` and :pr:`23033` by + :user:`Meekail Zain `. + +- |Fix| :func:`manifold.spectral_embedding` now uses Gaussian instead of + the previous uniform on [0, 1] random initial approximations to eigenvectors + in eigen_solvers `lobpcg` and `amg` to improve their numerical stability. + :pr:`21565` by :user:`Andrew Knyazev `. + +:mod:`sklearn.metrics` +...................... + +- |Feature| :func:`metrics.r2_score` and :func:`metrics.explained_variance_score` have a + new `force_finite` parameter. Setting this parameter to `False` will return the + actual non-finite score in case of perfect predictions or constant `y_true`, + instead of the finite approximation (`1.0` and `0.0` respectively) currently + returned by default. :pr:`17266` by :user:`Sylvain MariÊ `. + +- |Feature| :func:`metrics.d2_pinball_score` and :func:`metrics.d2_absolute_error_score` + calculate the :math:`D^2` regression score for the pinball loss and the + absolute error respectively. :func:`metrics.d2_absolute_error_score` is a special case + of :func:`metrics.d2_pinball_score` with a fixed quantile parameter `alpha=0.5` + for ease of use and discovery. The :math:`D^2` scores are generalizations + of the `r2_score` and can be interpreted as the fraction of deviance explained. + :pr:`22118` by :user:`Ohad Michel `. + +- |Enhancement| :func:`metrics.top_k_accuracy_score` raises an improved error + message when `y_true` is binary and `y_score` is 2d. :pr:`22284` by `Thomas Fan`_. + +- |Enhancement| :func:`metrics.roc_auc_score` now supports ``average=None`` + in the multiclass case when ``multiclass='ovr'`` which will return the score + per class. :pr:`19158` by :user:`Nicki Skafte `. + +- |Enhancement| Adds `im_kw` parameter to + :meth:`metrics.ConfusionMatrixDisplay.from_estimator` + :meth:`metrics.ConfusionMatrixDisplay.from_predictions`, and + :meth:`metrics.ConfusionMatrixDisplay.plot`. The `im_kw` parameter is passed + to the `matplotlib.pyplot.imshow` call when plotting the confusion matrix. + :pr:`20753` by `Thomas Fan`_. + +- |Fix| :func:`metrics.silhouette_score` now supports integer input for precomputed + distances. :pr:`22108` by `Thomas Fan`_. + +- |Fix| Fixed a bug in :func:`metrics.normalized_mutual_info_score` which could return + unbounded values. :pr:`22635` by :user:`JÊrÊmie du Boisberranger `. + +- |Fix| Fixes :func:`metrics.precision_recall_curve` and + :func:`metrics.average_precision_score` when true labels are all negative. + :pr:`19085` by :user:`Varun Agrawal `. + +- |API| `metrics.SCORERS` is now deprecated and will be removed in 1.3. Please + use :func:`metrics.get_scorer_names` to retrieve the names of all available + scorers. :pr:`22866` by `Adrin Jalali`_. + +- |API| Parameters ``sample_weight`` and ``multioutput`` of + :func:`metrics.mean_absolute_percentage_error` are now keyword-only, in accordance + with `SLEP009 `_. + A deprecation cycle was introduced. + :pr:`21576` by :user:`Paul-Emile Dugnat `. + +- |API| The `"wminkowski"` metric of :class:`metrics.DistanceMetric` is deprecated + and will be removed in version 1.3. Instead the existing `"minkowski"` metric now takes + in an optional `w` parameter for weights. This deprecation aims at remaining consistent + with SciPy 1.8 convention. :pr:`21873` by :user:`Yar Khine Phyo `. + +- |API| :class:`metrics.DistanceMetric` has been moved from + :mod:`sklearn.neighbors` to :mod:`sklearn.metrics`. + Using `neighbors.DistanceMetric` for imports is still valid for + backward compatibility, but this alias will be removed in 1.3. + :pr:`21177` by :user:`Julien Jerphanion `. + +:mod:`sklearn.mixture` +...................... + +- |Enhancement| :class:`mixture.GaussianMixture` and + :class:`mixture.BayesianGaussianMixture` can now be initialized using + k-means++ and random data points. :pr:`20408` by + :user:`Gordon Walsh `, :user:`Alberto Ceballos` + and :user:`Andres Rios`. + +- |Fix| Fix a bug that correctly initializes `precisions_cholesky_` in + :class:`mixture.GaussianMixture` when providing `precisions_init` by taking + its square root. + :pr:`22058` by :user:`Guillaume Lemaitre `. + +- |Fix| :class:`mixture.GaussianMixture` now normalizes `weights_` more safely, + preventing rounding errors when calling :meth:`mixture.GaussianMixture.sample` with + `n_components=1`. + :pr:`23034` by :user:`Meekail Zain `. + +:mod:`sklearn.model_selection` +.............................. + +- |Enhancement| it is now possible to pass `scoring="matthews_corrcoef"` to all + model selection tools with a `scoring` argument to use the Matthews + correlation coefficient (MCC). + :pr:`22203` by :user:`Olivier Grisel `. + +- |Enhancement| raise an error during cross-validation when the fits for all the + splits failed. Similarly raise an error during grid-search when the fits for + all the models and all the splits failed. + :pr:`21026` by :user:`Loïc Estève `. + +- |Fix| :class:`model_selection.GridSearchCV`, + :class:`model_selection.HalvingGridSearchCV` + now validates input parameters in `fit` instead of `__init__`. + :pr:`21880` by :user:`Mrinal Tyagi `. + +- |Fix| :func:`model_selection.learning_curve` now supports `partial_fit` + with regressors. :pr:`22982` by `Thomas Fan`_. + +:mod:`sklearn.multiclass` +......................... + +- |Enhancement| :class:`multiclass.OneVsRestClassifier` now supports a `verbose` + parameter so progress on fitting can be seen. + :pr:`22508` by :user:`Chris Combs `. + +- |Fix| :meth:`multiclass.OneVsOneClassifier.predict` returns correct predictions when + the inner classifier only has a :term:`predict_proba`. :pr:`22604` by `Thomas Fan`_. + +:mod:`sklearn.neighbors` +........................ + +- |Enhancement| Adds :term:`get_feature_names_out` to + :class:`neighbors.RadiusNeighborsTransformer`, + :class:`neighbors.KNeighborsTransformer` + and :class:`neighbors.NeighborhoodComponentsAnalysis`. + :pr:`22212` by :user:`Meekail Zain `. + +- |Fix| :class:`neighbors.KernelDensity` now validates input parameters in `fit` + instead of `__init__`. :pr:`21430` by :user:`Desislava Vasileva ` and + :user:`Lucy Jimenez `. + +- |Fix| :func:`neighbors.KNeighborsRegressor.predict` now works properly when + given an array-like input if `KNeighborsRegressor` is first constructed with a + callable passed to the `weights` parameter. :pr:`22687` by + :user:`Meekail Zain `. + +:mod:`sklearn.neural_network` +............................. + +- |Enhancement| :func:`neural_network.MLPClassifier` and + :func:`neural_network.MLPRegressor` show error + messages when optimizers produce non-finite parameter weights. :pr:`22150` + by :user:`Christian Ritter ` and :user:`Norbert Preining `. + +- |Enhancement| Adds :term:`get_feature_names_out` to + :class:`neural_network.BernoulliRBM`. :pr:`22248` by `Thomas Fan`_. + +:mod:`sklearn.pipeline` +....................... + +- |Enhancement| Added support for "passthrough" in :class:`pipeline.FeatureUnion`. + Setting a transformer to "passthrough" will pass the features unchanged. + :pr:`20860` by :user:`Shubhraneel Pal `. + +- |Fix| :class:`pipeline.Pipeline` now does not validate hyper-parameters in + `__init__` but in `.fit()`. + :pr:`21888` by :user:`iofall ` and :user:`Arisa Y. `. + +- |Fix| :class:`pipeline.FeatureUnion` does not validate hyper-parameters in + `__init__`. Validation is now handled in `.fit()` and `.fit_transform()`. + :pr:`21954` by :user:`iofall ` and :user:`Arisa Y. `. + +- |Fix| Defines `__sklearn_is_fitted__` in :class:`pipeline.FeatureUnion` to + return correct result with :func:`utils.validation.check_is_fitted`. + :pr:`22953` by :user:`randomgeek78 `. + +:mod:`sklearn.preprocessing` +............................ + +- |Feature| :class:`preprocessing.OneHotEncoder` now supports grouping + infrequent categories into a single feature. Grouping infrequent categories + is enabled by specifying how to select infrequent categories with + `min_frequency` or `max_categories`. :pr:`16018` by `Thomas Fan`_. + +- |Enhancement| Adds a `subsample` parameter to :class:`preprocessing.KBinsDiscretizer`. + This allows specifying a maximum number of samples to be used while fitting + the model. The option is only available when `strategy` is set to `quantile`. + :pr:`21445` by :user:`Felipe Bidu ` and :user:`Amanda Dsouza `. + +- |Enhancement| Adds `encoded_missing_value` to :class:`preprocessing.OrdinalEncoder` + to configure the encoded value for missing data. :pr:`21988` by `Thomas Fan`_. + +- |Enhancement| Added the `get_feature_names_out` method and a new parameter + `feature_names_out` to :class:`preprocessing.FunctionTransformer`. You can set + `feature_names_out` to 'one-to-one' to use the input features names as the + output feature names, or you can set it to a callable that returns the output + feature names. This is especially useful when the transformer changes the + number of features. If `feature_names_out` is None (which is the default), + then `get_output_feature_names` is not defined. + :pr:`21569` by :user:`AurÊlien Geron `. + +- |Enhancement| Adds :term:`get_feature_names_out` to + :class:`preprocessing.Normalizer`, + :class:`preprocessing.KernelCenterer`, + :class:`preprocessing.OrdinalEncoder`, and + :class:`preprocessing.Binarizer`. :pr:`21079` by `Thomas Fan`_. + +- |Fix| :class:`preprocessing.PowerTransformer` with `method='yeo-johnson'` + better supports significantly non-Gaussian data when searching for an optimal + lambda. :pr:`20653` by `Thomas Fan`_. + +- |Fix| :class:`preprocessing.LabelBinarizer` now validates input parameters in + `fit` instead of `__init__`. + :pr:`21434` by :user:`Krum Arnaudov `. + +- |Fix| :class:`preprocessing.FunctionTransformer` with `check_inverse=True` + now provides informative error message when input has mixed dtypes. :pr:`19916` by + :user:`Zhehao Liu `. + +- |Fix| :class:`preprocessing.KBinsDiscretizer` handles bin edges more consistently now. + :pr:`14975` by `Andreas MÃŧller`_ and :pr:`22526` by :user:`Meekail Zain `. + +- |Fix| Adds :meth:`preprocessing.KBinsDiscretizer.get_feature_names_out` support when + `encode="ordinal"`. :pr:`22735` by `Thomas Fan`_. + +:mod:`sklearn.random_projection` +................................ + +- |Enhancement| Adds an `inverse_transform` method and a `compute_inverse_transform` + parameter to :class:`random_projection.GaussianRandomProjection` and + :class:`random_projection.SparseRandomProjection`. When the parameter is set + to True, the pseudo-inverse of the components is computed during `fit` and stored as + `inverse_components_`. :pr:`21701` by :user:`AurÊlien Geron `. + +- |Enhancement| :class:`random_projection.SparseRandomProjection` and + :class:`random_projection.GaussianRandomProjection` preserve dtype for + `numpy.float32`. :pr:`22114` by :user:`Takeshi Oura `. + +- |Enhancement| Adds :term:`get_feature_names_out` to all transformers in the + :mod:`sklearn.random_projection` module: + :class:`random_projection.GaussianRandomProjection` and + :class:`random_projection.SparseRandomProjection`. :pr:`21330` by + :user:`Loïc Estève `. + +:mod:`sklearn.svm` +.................. + +- |Enhancement| :class:`svm.OneClassSVM`, :class:`svm.NuSVC`, + :class:`svm.NuSVR`, :class:`svm.SVC` and :class:`svm.SVR` now expose + `n_iter_`, the number of iterations of the libsvm optimization routine. + :pr:`21408` by :user:`Juan Martín Loyola `. + +- |Enhancement| :func:`svm.SVR`, :func:`svm.SVC`, :func:`svm.NuSVR`, + :func:`svm.OneClassSVM`, :func:`svm.NuSVC` now raise an error + when the dual-gap estimation produces non-finite parameter weights. + :pr:`22149` by :user:`Christian Ritter ` and + :user:`Norbert Preining `. + +- |Fix| :class:`svm.NuSVC`, :class:`svm.NuSVR`, :class:`svm.SVC`, + :class:`svm.SVR`, :class:`svm.OneClassSVM` now validate input + parameters in `fit` instead of `__init__`. + :pr:`21436` by :user:`Haidar Almubarak `. + +:mod:`sklearn.tree` +................... + +- |Enhancement| :class:`tree.DecisionTreeClassifier` and + :class:`tree.ExtraTreeClassifier` have the new `criterion="log_loss"`, which is + equivalent to `criterion="entropy"`. + :pr:`23047` by :user:`Christian Lorentzen `. + +- |Fix| Fix a bug in the Poisson splitting criterion for + :class:`tree.DecisionTreeRegressor`. + :pr:`22191` by :user:`Christian Lorentzen `. + +- |API| Changed the default value of `max_features` to 1.0 for + :class:`tree.ExtraTreeRegressor` and to `"sqrt"` for + :class:`tree.ExtraTreeClassifier`, which will not change the fit result. The original + default value `"auto"` has been deprecated and will be removed in version 1.3. + Setting `max_features` to `"auto"` is also deprecated + for :class:`tree.DecisionTreeClassifier` and :class:`tree.DecisionTreeRegressor`. + :pr:`22476` by :user:`Zhehao Liu `. + +:mod:`sklearn.utils` +.................... + +- |Enhancement| :func:`utils.check_array` and + :func:`utils.multiclass.type_of_target` now accept an `input_name` parameter to make + the error message more informative when passed invalid input data (e.g. with NaN or + infinite values). + :pr:`21219` by :user:`Olivier Grisel `. + +- |Enhancement| :func:`utils.check_array` returns a float + ndarray with `np.nan` when passed a `Float32` or `Float64` pandas extension + array with `pd.NA`. :pr:`21278` by `Thomas Fan`_. + +- |Enhancement| :func:`utils.estimator_html_repr` shows a more helpful error + message when running in a jupyter notebook that is not trusted. :pr:`21316` + by `Thomas Fan`_. + +- |Enhancement| :func:`utils.estimator_html_repr` displays an arrow on the top + left corner of the HTML representation to show how the elements are + clickable. :pr:`21298` by `Thomas Fan`_. + +- |Enhancement| :func:`utils.check_array` with `dtype=None` returns numeric + arrays when passed in a pandas DataFrame with mixed dtypes. `dtype="numeric"` + will also make better infer the dtype when the DataFrame has mixed dtypes. + :pr:`22237` by `Thomas Fan`_. + +- |Enhancement| :func:`utils.check_scalar` now has better messages + when displaying the type. :pr:`22218` by `Thomas Fan`_. + +- |Fix| Changes the error message of the `ValidationError` raised by + :func:`utils.check_X_y` when y is None so that it is compatible + with the `check_requires_y_none` estimator check. :pr:`22578` by + :user:`Claudio Salvatore Arcidiacono `. + +- |Fix| :func:`utils.class_weight.compute_class_weight` now only requires that + all classes in `y` have a weight in `class_weight`. An error is still raised + when a class is present in `y` but not in `class_weight`. :pr:`22595` by + `Thomas Fan`_. + +- |Fix| :func:`utils.estimator_html_repr` has an improved visualization for nested + meta-estimators. :pr:`21310` by `Thomas Fan`_. + +- |Fix| :func:`utils.check_scalar` raises an error when + `include_boundaries={"left", "right"}` and the boundaries are not set. + :pr:`22027` by :user:`Marie Lanternier `. + +- |Fix| :func:`utils.metaestimators.available_if` correctly returns a bounded + method that can be pickled. :pr:`23077` by `Thomas Fan`_. + +- |API| :func:`utils.estimator_checks.check_estimator`'s argument is now called + `estimator` (previous name was `Estimator`). :pr:`22188` by + :user:`Mathurin Massias `. + +- |API| ``utils.metaestimators.if_delegate_has_method`` is deprecated and will be + removed in version 1.3. Use :func:`utils.metaestimators.available_if` instead. + :pr:`22830` by :user:`JÊrÊmie du Boisberranger `. + +.. rubric:: Code and documentation contributors + +Thanks to everyone who has contributed to the maintenance and improvement of +the project since version 1.0, including: + +2357juan, Abhishek Gupta, adamgonzo, Adam Li, adijohar, Aditya Kumawat, Aditya +Raghuwanshi, Aditya Singh, Adrian Trujillo Duron, Adrin Jalali, ahmadjubair33, +AJ Druck, aj-white, Alan Peixinho, Alberto Mario Ceballos-Arroyo, Alek +Lefebvre, Alex, Alexandr, Alexandre Gramfort, alexanmv, almeidayoel, Amanda +Dsouza, Aman Sharma, Amar pratap singh, Amit, amrcode, AndrÃĄs Simon, Andreas +Grivas, Andreas Mueller, Andrew Knyazev, Andriy, Angus L'Herrou, Ankit Sharma, +Anne Ducout, Arisa, Arth, arthurmello, Arturo Amor, ArturoAmor, Atharva Patil, +aufarkari, AurÊlien Geron, avm19, Ayan Bag, baam, Bardiya Ak, Behrouz B, +Ben3940, Benjamin Bossan, Bharat Raghunathan, Bijil Subhash, bmreiniger, +Brandon Truth, Brenden Kadota, Brian Sun, cdrig, Chalmer Lowe, Chiara Marmo, +Chitteti Srinath Reddy, Chloe-Agathe Azencott, Christian Lorentzen, Christian +Ritter, christopherlim98, Christoph T. Weidemann, Christos Aridas, Claudio +Salvatore Arcidiacono, combscCode, Daniela Fernandes, darioka, Darren Nguyen, +Dave Eargle, David Gilbertson, David Poznik, Dea María LÊon, Dennis Osei, +DessyVV, Dev514, Dimitri Papadopoulos Orfanos, Diwakar Gupta, Dr. Felix M. +Riese, drskd, Emiko Sano, Emmanouil Gionanidis, EricEllwanger, Erich Schubert, +Eric Larson, Eric Ndirangu, ErmolaevPA, Estefania Barreto-Ojeda, eyast, Fatima +GASMI, Federico Luna, Felix Glushchenkov, fkaren27, Fortune Uwha, FPGAwesome, +francoisgoupil, Frans Larsson, ftorres16, Gabor Berei, Gabor Kertesz, Gabriel +Stefanini Vicente, Gabriel S Vicente, Gael Varoquaux, GAURAV CHOUDHARY, +Gauthier I, genvalen, Geoffrey-Paris, Giancarlo Pablo, glennfrutiz, gpapadok, +Guillaume Lemaitre, Guillermo TomÃĄs FernÃĄndez Martín, Gustavo Oliveira, Haidar +Almubarak, Hannah Bohle, Hansin Ahuja, Haoyin Xu, Haya, Helder Geovane Gomes de +Lima, henrymooresc, Hideaki Imamura, Himanshu Kumar, Hind-M, hmasdev, hvassard, +i-aki-y, iasoon, Inclusive Coding Bot, Ingela, iofall, Ishan Kumar, Jack Liu, +Jake Cowton, jalexand3r, J Alexander, Jauhar, Jaya Surya Kommireddy, Jay +Stanley, Jeff Hale, je-kr, JElfner, Jenny Vo, JÊrÊmie du Boisberranger, Jihane, +Jirka Borovec, Joel Nothman, Jon Haitz Legarreta GorroÃąo, Jordan Silke, Jorge +CipriÃĄn, Jorge Loayza, Joseph Chazalon, Joseph Schwartz-Messing, Jovan +Stojanovic, JSchuerz, Juan Carlos Alfaro JimÊnez, Juan Martin Loyola, Julien +Jerphanion, katotten, Kaushik Roy Chowdhury, Ken4git, Kenneth Prabakaran, +kernc, Kevin Doucet, KimAYoung, Koushik Joshi, Kranthi Sedamaki, krishna kumar, +krumetoft, lesnee, Lisa Casino, Logan Thomas, Loic Esteve, Louis Wagner, +LucieClair, Lucy Liu, Luiz Eduardo Amaral, Magali, MaggieChege, Mai, +mandjevant, Mandy Gu, Manimaran, MarcoM, Marco Wurps, Maren Westermann, Maria +Boerner, MarieS-WiMLDS, Martel Corentin, martin-kokos, mathurinm, Matías, +matjansen, Matteo Francia, Maxwell, Meekail Zain, Megabyte, Mehrdad +Moradizadeh, melemo2, Michael I Chen, michalkrawczyk, Micky774, milana2, +millawell, Ming-Yang Ho, Mitzi, miwojc, Mizuki, mlant, Mohamed Haseeb, Mohit +Sharma, Moonkyung94, mpoemsl, MrinalTyagi, Mr. Leu, msabatier, murata-yu, N, +Nadirhan Şahin, Naipawat Poolsawat, NartayXD, nastegiano, nathansquan, +nat-salt, Nicki Skafte Detlefsen, Nicolas Hug, Niket Jain, Nikhil Suresh, +Nikita Titov, Nikolay Kondratyev, Ohad Michel, Oleksandr Husak, Olivier Grisel, +partev, Patrick Ferreira, Paul, pelennor, PierreAttard, Piet BrÃļmmel, Pieter +Gijsbers, Pinky, poloso, Pramod Anantharam, puhuk, Purna Chandra Mansingh, +QuadV, Rahil Parikh, Randall Boyes, randomgeek78, Raz Hoshia, Reshama Shaikh, +Ricardo Ferreira, Richard Taylor, Rileran, Rishabh, Robin Thibaut, Rocco Meli, +Roman Feldbauer, Roman Yurchak, Ross Barnowski, rsnegrin, Sachin Yadav, +sakinaOuisrani, Sam Adam Day, Sanjay Marreddi, Sebastian Pujalte, SEELE, SELEE, +Seyedsaman (Sam) Emami, ShanDeng123, Shao Yang Hong, sharmadharmpal, +shaymerNaturalint, Shuangchi He, Shubhraneel Pal, siavrez, slishak, Smile, +spikebh, sply88, Srinath Kailasa, StÊphane Collot, Sultan Orazbayev, Sumit +Saha, Sven Eschlbeck, Sven Stehle, Swapnil Jha, Sylvain MariÊ, Takeshi Oura, +Tamires Santana, Tenavi, teunpe, Theis FerrÊ HjortkjÃĻr, Thiruvenkadam, Thomas +J. Fan, t-jakubek, toastedyeast, Tom DuprÊ la Tour, Tom McTiernan, TONY GEORGE, +Tyler Martin, Tyler Reddy, Udit Gupta, Ugo Marchand, Varun Agrawal, +Venkatachalam N, Vera Komeyer, victoirelouis, Vikas Vishwakarma, Vikrant +khedkar, Vladimir Chernyy, Vladimir Kim, WeijiaDu, Xiao Yuan, Yar Khine Phyo, +Ying Xiong, yiyangq, Yosshi999, Yuki Koyama, Zach Deane-Mayer, Zeel B Patel, +zempleni, zhenfisher, čĩĩ丰 (Zhao Feng) diff --git a/doc/whats_new/v1.2.rst b/doc/whats_new/v1.2.rst new file mode 100644 index 0000000000000..d2d5521508715 --- /dev/null +++ b/doc/whats_new/v1.2.rst @@ -0,0 +1,1071 @@ +.. include:: _contributors.rst + +.. currentmodule:: sklearn + +.. _release_notes_1_2: + +=========== +Version 1.2 +=========== + +For a short description of the main highlights of the release, please refer to +:ref:`sphx_glr_auto_examples_release_highlights_plot_release_highlights_1_2_0.py`. + +.. include:: changelog_legend.inc + +.. _changes_1_2_2: + +Version 1.2.2 +============= + +**March 2023** + +Changelog +--------- + +:mod:`sklearn.base` +................... + +- |Fix| When `set_output(transform="pandas")`, :class:`base.TransformerMixin` maintains + the index if the :term:`transform` output is already a DataFrame. :pr:`25747` by + `Thomas Fan`_. + +:mod:`sklearn.calibration` +.......................... + +- |Fix| A deprecation warning is raised when using the `base_estimator__` prefix to + set parameters of the estimator used in :class:`calibration.CalibratedClassifierCV`. + :pr:`25477` by :user:`Tim Head `. + +:mod:`sklearn.cluster` +...................... + +- |Fix| Fixed a bug in :class:`cluster.BisectingKMeans`, preventing `fit` from randomly + failing due to a permutation of the labels when running multiple inits. + :pr:`25563` by :user:`JÊrÊmie du Boisberranger `. + +:mod:`sklearn.compose` +...................... + +- |Fix| Fixes a bug in :class:`compose.ColumnTransformer` which now supports + empty selection of columns when `set_output(transform="pandas")`. + :pr:`25570` by `Thomas Fan`_. + +:mod:`sklearn.ensemble` +....................... + +- |Fix| A deprecation warning is raised when using the `base_estimator__` prefix + to set parameters of the estimator used in :class:`ensemble.AdaBoostClassifier`, + :class:`ensemble.AdaBoostRegressor`, :class:`ensemble.BaggingClassifier`, + and :class:`ensemble.BaggingRegressor`. + :pr:`25477` by :user:`Tim Head `. + +:mod:`sklearn.feature_selection` +................................ + +- |Fix| Fixed a regression where a negative `tol` would not be accepted any more by + :class:`feature_selection.SequentialFeatureSelector`. + :pr:`25664` by :user:`JÊrÊmie du Boisberranger `. + +:mod:`sklearn.inspection` +......................... + +- |Fix| Raise a more informative error message in :func:`inspection.partial_dependence` + when dealing with mixed data type categories that cannot be sorted by + :func:`numpy.unique`. This problem usually happens when categories are `str` and + missing values are present using `np.nan`. + :pr:`25774` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.isotonic` +....................... + +- |Fix| Fixes a bug in :class:`isotonic.IsotonicRegression` where + :meth:`isotonic.IsotonicRegression.predict` would return a pandas DataFrame + when the global configuration sets `transform_output="pandas"`. + :pr:`25500` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.preprocessing` +............................ + +- |Fix| `preprocessing.OneHotEncoder.drop_idx_` now properly + references the dropped category in the `categories_` attribute + when there are infrequent categories. :pr:`25589` by `Thomas Fan`_. + +- |Fix| :class:`preprocessing.OrdinalEncoder` now correctly supports + `encoded_missing_value` or `unknown_value` set to a categories' cardinality + when there is missing values in the training data. :pr:`25704` by `Thomas Fan`_. + +:mod:`sklearn.tree` +................... + +- |Fix| Fixed a regression in :class:`tree.DecisionTreeClassifier`, + :class:`tree.DecisionTreeRegressor`, :class:`tree.ExtraTreeClassifier` and + :class:`tree.ExtraTreeRegressor` where an error was no longer raised in version + 1.2 when `min_sample_split=1`. + :pr:`25744` by :user:`JÊrÊmie du Boisberranger `. + +:mod:`sklearn.utils` +.................... + +- |Fix| Fixes a bug in :func:`utils.check_array` which now correctly performs + non-finite validation with the Array API specification. :pr:`25619` by + `Thomas Fan`_. + +- |Fix| :func:`utils.multiclass.type_of_target` can identify pandas + nullable data types as classification targets. :pr:`25638` by `Thomas Fan`_. + +.. _changes_1_2_1: + +Version 1.2.1 +============= + +**January 2023** + +Changed models +-------------- + +The following estimators and functions, when fit with the same data and +parameters, may produce different models from the previous version. This often +occurs due to changes in the modelling logic (bug fixes or enhancements), or in +random sampling procedures. + +- |Fix| The fitted components in + :class:`decomposition.MiniBatchDictionaryLearning` might differ. The online + updates of the sufficient statistics now properly take the sizes of the + batches into account. + :pr:`25354` by :user:`JÊrÊmie du Boisberranger `. + +- |Fix| The `categories_` attribute of :class:`preprocessing.OneHotEncoder` now + always contains an array of `object`s when using predefined categories that + are strings. Predefined categories encoded as bytes will no longer work + with `X` encoded as strings. :pr:`25174` by :user:`Tim Head `. + +Changes impacting all modules +----------------------------- + +- |Fix| Support `pandas.Int64` dtyped `y` for classifiers and regressors. + :pr:`25089` by :user:`Tim Head `. + +- |Fix| Remove spurious warnings for estimators internally using neighbors search methods. + :pr:`25129` by :user:`Julien Jerphanion `. + +- |Fix| Fix a bug where the current configuration was ignored in estimators using + `n_jobs > 1`. This bug was triggered for tasks dispatched by the auxiliary + thread of `joblib` as :func:`sklearn.get_config` used to access an empty thread + local configuration instead of the configuration visible from the thread where + `joblib.Parallel` was first called. + :pr:`25363` by :user:`Guillaume Lemaitre `. + +Changelog +--------- + +:mod:`sklearn.base` +................... + +- |Fix| Fix a regression in `BaseEstimator.__getstate__` that would prevent + certain estimators from being pickled when using Python 3.11. :pr:`25188` by + :user:`Benjamin Bossan `. + +- |Fix| Inheriting from :class:`base.TransformerMixin` will only wrap the `transform` + method if the class defines `transform` itself. :pr:`25295` by `Thomas Fan`_. + +:mod:`sklearn.datasets` +....................... + +- |Fix| Fixes an inconsistency in :func:`datasets.fetch_openml` between liac-arff + and pandas parser when a leading space is introduced after the delimiter. + The ARFF specs require ignoring the leading space. + :pr:`25312` by :user:`Guillaume Lemaitre `. + +- |Fix| Fixes a bug in :func:`datasets.fetch_openml` when using `parser="pandas"` + where single quote and backslash escape characters were not properly handled. + :pr:`25511` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.decomposition` +............................ + +- |Fix| Fixed a bug in :class:`decomposition.MiniBatchDictionaryLearning` where the + online updates of the sufficient statistics were not correct when calling + `partial_fit` on batches of different sizes. + :pr:`25354` by :user:`JÊrÊmie du Boisberranger `. + +- |Fix| :class:`decomposition.DictionaryLearning` better supports readonly NumPy + arrays. In particular, it better supports large datasets which are memory-mapped + when it is used with coordinate descent algorithms (i.e. when `fit_algorithm='cd'`). + :pr:`25172` by :user:`Julien Jerphanion `. + +:mod:`sklearn.ensemble` +....................... + +- |Fix| :class:`ensemble.RandomForestClassifier`, + :class:`ensemble.RandomForestRegressor`, :class:`ensemble.ExtraTreesClassifier` + and :class:`ensemble.ExtraTreesRegressor` now support sparse readonly datasets. + :pr:`25341` by :user:`Julien Jerphanion ` + +:mod:`sklearn.feature_extraction` +................................. + +- |Fix| :class:`feature_extraction.FeatureHasher` raises an informative error + when the input is a list of strings. :pr:`25094` by `Thomas Fan`_. + +:mod:`sklearn.linear_model` +........................... + +- |Fix| Fix a regression in :class:`linear_model.SGDClassifier` and + :class:`linear_model.SGDRegressor` that makes them unusable with the + `verbose` parameter set to a value greater than 0. + :pr:`25250` by :user:`JÊrÊmie Du Boisberranger `. + +:mod:`sklearn.manifold` +....................... + +- |Fix| :class:`manifold.TSNE` now works correctly when output type is + set to pandas :pr:`25370` by :user:`Tim Head `. + +:mod:`sklearn.model_selection` +.............................. + +- |Fix| :func:`model_selection.cross_validate` with multimetric scoring in + case of some failing scorers the non-failing scorers now return proper + scores instead of `error_score` values. + :pr:`23101` by :user:`AndrÃĄs Simon ` and `Thomas Fan`_. + +:mod:`sklearn.neural_network` +............................. + +- |Fix| :class:`neural_network.MLPClassifier` and :class:`neural_network.MLPRegressor` + no longer raise warnings when fitting data with feature names. + :pr:`24873` by :user:`Tim Head `. + +- |Fix| Improves error message in :class:`neural_network.MLPClassifier` and + :class:`neural_network.MLPRegressor`, when `early_stopping=True` and + `partial_fit` is called. :pr:`25694` by `Thomas Fan`_. + +:mod:`sklearn.preprocessing` +............................ + +- |Fix| :meth:`preprocessing.FunctionTransformer.inverse_transform` correctly + supports DataFrames that are all numerical when `check_inverse=True`. + :pr:`25274` by `Thomas Fan`_. + +- |Fix| :meth:`preprocessing.SplineTransformer.get_feature_names_out` correctly + returns feature names when `extrapolations="periodic"`. :pr:`25296` by + `Thomas Fan`_. + +:mod:`sklearn.tree` +................... + +- |Fix| :class:`tree.DecisionTreeClassifier`, :class:`tree.DecisionTreeRegressor` + :class:`tree.ExtraTreeClassifier` and :class:`tree.ExtraTreeRegressor` + now support sparse readonly datasets. + :pr:`25341` by :user:`Julien Jerphanion ` + +:mod:`sklearn.utils` +.................... + +- |Fix| Restore :func:`utils.check_array`'s behaviour for pandas Series of type + boolean. The type is maintained, instead of converting to `float64.` + :pr:`25147` by :user:`Tim Head `. + +- |API| `utils.fixes.delayed` is deprecated in 1.2.1 and will be removed + in 1.5. Instead, import :func:`utils.parallel.delayed` and use it in + conjunction with the newly introduced :func:`utils.parallel.Parallel` + to ensure proper propagation of the scikit-learn configuration to + the workers. + :pr:`25363` by :user:`Guillaume Lemaitre `. + +.. _changes_1_2: + +Version 1.2.0 +============= + +**December 2022** + +Changed models +-------------- + +The following estimators and functions, when fit with the same data and +parameters, may produce different models from the previous version. This often +occurs due to changes in the modelling logic (bug fixes or enhancements), or in +random sampling procedures. + +- |Enhancement| The default `eigen_tol` for :class:`cluster.SpectralClustering`, + :class:`manifold.SpectralEmbedding`, :func:`cluster.spectral_clustering`, + and :func:`manifold.spectral_embedding` is now `None` when using the `'amg'` + or `'lobpcg'` solvers. This change improves numerical stability of the + solver, but may result in a different model. + +- |Enhancement| :class:`linear_model.GammaRegressor`, + :class:`linear_model.PoissonRegressor` and :class:`linear_model.TweedieRegressor` + can reach higher precision with the lbfgs solver, in particular when `tol` is set + to a tiny value. Moreover, `verbose` is now properly propagated to L-BFGS-B. + :pr:`23619` by :user:`Christian Lorentzen `. + +- |Enhancement| The default value for `eps` :func:`metrics.log_loss` has changed + from `1e-15` to `"auto"`. `"auto"` sets `eps` to `np.finfo(y_pred.dtype).eps`. + :pr:`24354` by :user:`Safiuddin Khaja ` and :user:`gsiisg `. + +- |Fix| Make sign of `components_` deterministic in :class:`decomposition.SparsePCA`. + :pr:`23935` by :user:`Guillaume Lemaitre `. + +- |Fix| The `components_` signs in :class:`decomposition.FastICA` might differ. + It is now consistent and deterministic with all SVD solvers. + :pr:`22527` by :user:`Meekail Zain ` and `Thomas Fan`_. + +- |Fix| The condition for early stopping has now been changed in + `linear_model._sgd_fast._plain_sgd` which is used by + :class:`linear_model.SGDRegressor` and :class:`linear_model.SGDClassifier`. The old + condition did not disambiguate between + training and validation set and had an effect of overscaling the error tolerance. + This has been fixed in :pr:`23798` by :user:`Harsh Agrawal `. + +- |Fix| For :class:`model_selection.GridSearchCV` and + :class:`model_selection.RandomizedSearchCV` ranks corresponding to nan + scores will all be set to the maximum possible rank. + :pr:`24543` by :user:`Guillaume Lemaitre `. + +- |API| The default value of `tol` was changed from `1e-3` to `1e-4` for + :func:`linear_model.ridge_regression`, :class:`linear_model.Ridge` and + :class:`linear_model.RidgeClassifier`. + :pr:`24465` by :user:`Christian Lorentzen `. + +Changes impacting all modules +----------------------------- + +- |MajorFeature| The `set_output` API has been adopted by all transformers. + Meta-estimators that contain transformers such as :class:`pipeline.Pipeline` + or :class:`compose.ColumnTransformer` also define a `set_output`. + For details, see + `SLEP018 `__. + :pr:`23734` and :pr:`24699` by `Thomas Fan`_. + +- |Efficiency| Low-level routines for reductions on pairwise distances + for dense float32 datasets have been refactored. The following functions + and estimators now benefit from improved performances in terms of hardware + scalability and speed-ups: + + - :func:`sklearn.metrics.pairwise_distances_argmin` + - :func:`sklearn.metrics.pairwise_distances_argmin_min` + - :class:`sklearn.cluster.AffinityPropagation` + - :class:`sklearn.cluster.Birch` + - :class:`sklearn.cluster.MeanShift` + - :class:`sklearn.cluster.OPTICS` + - :class:`sklearn.cluster.SpectralClustering` + - :func:`sklearn.feature_selection.mutual_info_regression` + - :class:`sklearn.neighbors.KNeighborsClassifier` + - :class:`sklearn.neighbors.KNeighborsRegressor` + - :class:`sklearn.neighbors.RadiusNeighborsClassifier` + - :class:`sklearn.neighbors.RadiusNeighborsRegressor` + - :class:`sklearn.neighbors.LocalOutlierFactor` + - :class:`sklearn.neighbors.NearestNeighbors` + - :class:`sklearn.manifold.Isomap` + - :class:`sklearn.manifold.LocallyLinearEmbedding` + - :class:`sklearn.manifold.TSNE` + - :func:`sklearn.manifold.trustworthiness` + - :class:`sklearn.semi_supervised.LabelPropagation` + - :class:`sklearn.semi_supervised.LabelSpreading` + + For instance :meth:`sklearn.neighbors.NearestNeighbors.kneighbors` and + :meth:`sklearn.neighbors.NearestNeighbors.radius_neighbors` + can respectively be up to ×20 and ×5 faster than previously on a laptop. + + Moreover, implementations of those two algorithms are now suitable + for machine with many cores, making them usable for datasets consisting + of millions of samples. + + :pr:`23865` by :user:`Julien Jerphanion `. + +- |Enhancement| Finiteness checks (detection of NaN and infinite values) in all + estimators are now significantly more efficient for float32 data by leveraging + NumPy's SIMD optimized primitives. + :pr:`23446` by :user:`Meekail Zain ` + +- |Enhancement| Finiteness checks (detection of NaN and infinite values) in all + estimators are now faster by utilizing a more efficient stop-on-first + second-pass algorithm. + :pr:`23197` by :user:`Meekail Zain ` + +- |Enhancement| Support for combinations of dense and sparse datasets pairs + for all distance metrics and for float32 and float64 datasets has been added + or has seen its performance improved for the following estimators: + + - :func:`sklearn.metrics.pairwise_distances_argmin` + - :func:`sklearn.metrics.pairwise_distances_argmin_min` + - :class:`sklearn.cluster.AffinityPropagation` + - :class:`sklearn.cluster.Birch` + - :class:`sklearn.cluster.SpectralClustering` + - :class:`sklearn.neighbors.KNeighborsClassifier` + - :class:`sklearn.neighbors.KNeighborsRegressor` + - :class:`sklearn.neighbors.RadiusNeighborsClassifier` + - :class:`sklearn.neighbors.RadiusNeighborsRegressor` + - :class:`sklearn.neighbors.LocalOutlierFactor` + - :class:`sklearn.neighbors.NearestNeighbors` + - :class:`sklearn.manifold.Isomap` + - :class:`sklearn.manifold.TSNE` + - :func:`sklearn.manifold.trustworthiness` + + :pr:`23604` and :pr:`23585` by :user:`Julien Jerphanion `, + :user:`Olivier Grisel `, and `Thomas Fan`_, + :pr:`24556` by :user:`Vincent Maladière `. + +- |Fix| Systematically check the sha256 digest of dataset tarballs used in code + examples in the documentation. + :pr:`24617` by :user:`Olivier Grisel ` and `Thomas Fan`_. Thanks to + `Sim4n6 `_ for the report. + +Changelog +--------- + +.. + Entries should be grouped by module (in alphabetic order) and prefixed with + one of the labels: |MajorFeature|, |Feature|, |Efficiency|, |Enhancement|, + |Fix| or |API| (see whats_new.rst for descriptions). + Entries should be ordered by those labels (e.g. |Fix| after |Efficiency|). + Changes not specific to a module should be listed under *Multiple Modules* + or *Miscellaneous*. + Entries should end with: + :pr:`123456` by :user:`Joe Bloggs `. + where 123456 is the *pull request* number, not the issue number. + +:mod:`sklearn.base` +................... + +- |Enhancement| Introduces :class:`base.ClassNamePrefixFeaturesOutMixin` and + :class:`base.ClassNamePrefixFeaturesOutMixin` mixins that define + :term:`get_feature_names_out` for common transformer use cases. + :pr:`24688` by `Thomas Fan`_. + +:mod:`sklearn.calibration` +.......................... + +- |API| Rename `base_estimator` to `estimator` in + :class:`calibration.CalibratedClassifierCV` to improve readability and consistency. + The parameter `base_estimator` is deprecated and will be removed in 1.4. + :pr:`22054` by :user:`Kevin Roice `. + +:mod:`sklearn.cluster` +...................... + +- |Efficiency| :class:`cluster.KMeans` with `algorithm="lloyd"` is now faster + and uses less memory. :pr:`24264` by + :user:`Vincent Maladiere `. + +- |Enhancement| The `predict` and `fit_predict` methods of :class:`cluster.OPTICS` now + accept sparse data type for input data. :pr:`14736` by :user:`Hunt Zhan `, + :pr:`20802` by :user:`Brandon Pokorny `, + and :pr:`22965` by :user:`Meekail Zain `. + +- |Enhancement| :class:`cluster.Birch` now preserves dtype for `numpy.float32` + inputs. :pr:`22968` by `Meekail Zain `. + +- |Enhancement| :class:`cluster.KMeans` and :class:`cluster.MiniBatchKMeans` + now accept a new `'auto'` option for `n_init` which changes the number of + random initializations to one when using `init='k-means++'` for efficiency. + This begins deprecation for the default values of `n_init` in the two classes + and both will have their defaults changed to `n_init='auto'` in 1.4. + :pr:`23038` by :user:`Meekail Zain `. + +- |Enhancement| :class:`cluster.SpectralClustering` and + :func:`cluster.spectral_clustering` now propagate the `eigen_tol` parameter + to all choices of `eigen_solver`. Includes a new option `eigen_tol="auto"` + and begins deprecation to change the default from `eigen_tol=0` to + `eigen_tol="auto"` in version 1.3. + :pr:`23210` by :user:`Meekail Zain `. + +- |Fix| :class:`cluster.KMeans` now supports readonly attributes when predicting. + :pr:`24258` by `Thomas Fan`_ + +- |API| The `affinity` attribute is now deprecated for + :class:`cluster.AgglomerativeClustering` and will be renamed to `metric` in v1.4. + :pr:`23470` by :user:`Meekail Zain `. + +:mod:`sklearn.datasets` +....................... + +- |Enhancement| Introduce the new parameter `parser` in + :func:`datasets.fetch_openml`. `parser="pandas"` allows to use the very CPU + and memory efficient `pandas.read_csv` parser to load dense ARFF + formatted dataset files. It is possible to pass `parser="liac-arff"` + to use the old LIAC parser. + When `parser="auto"`, dense datasets are loaded with "pandas" and sparse + datasets are loaded with "liac-arff". + Currently, `parser="liac-arff"` by default and will change to `parser="auto"` + in version 1.4 + :pr:`21938` by :user:`Guillaume Lemaitre `. + +- |Enhancement| :func:`datasets.dump_svmlight_file` is now accelerated with a + Cython implementation, providing 2-4x speedups. + :pr:`23127` by :user:`Meekail Zain ` + +- |Enhancement| Path-like objects, such as those created with pathlib are now + allowed as paths in :func:`datasets.load_svmlight_file` and + :func:`datasets.load_svmlight_files`. + :pr:`19075` by :user:`Carlos Ramos CarreÃąo `. + +- |Fix| Make sure that :func:`datasets.fetch_lfw_people` and + :func:`datasets.fetch_lfw_pairs` internally crop images based on the + `slice_` parameter. + :pr:`24951` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.decomposition` +............................ + +- |Efficiency| :func:`decomposition.FastICA.fit` has been optimised w.r.t + its memory footprint and runtime. + :pr:`22268` by :user:`MohamedBsh `. + +- |Enhancement| :class:`decomposition.SparsePCA` and + :class:`decomposition.MiniBatchSparsePCA` now implement an `inverse_transform` + function. + :pr:`23905` by :user:`Guillaume Lemaitre `. + +- |Enhancement| :class:`decomposition.FastICA` now allows the user to select + how whitening is performed through the new `whiten_solver` parameter, which + supports `svd` and `eigh`. `whiten_solver` defaults to `svd` although `eigh` + may be faster and more memory efficient in cases where + `num_features > num_samples`. + :pr:`11860` by :user:`Pierre Ablin `, + :pr:`22527` by :user:`Meekail Zain ` and `Thomas Fan`_. + +- |Enhancement| :class:`decomposition.LatentDirichletAllocation` now preserves dtype + for `numpy.float32` input. :pr:`24528` by :user:`Takeshi Oura ` and + :user:`JÊrÊmie du Boisberranger `. + +- |Fix| Make sign of `components_` deterministic in :class:`decomposition.SparsePCA`. + :pr:`23935` by :user:`Guillaume Lemaitre `. + +- |API| The `n_iter` parameter of :class:`decomposition.MiniBatchSparsePCA` is + deprecated and replaced by the parameters `max_iter`, `tol`, and + `max_no_improvement` to be consistent with + :class:`decomposition.MiniBatchDictionaryLearning`. `n_iter` will be removed + in version 1.3. :pr:`23726` by :user:`Guillaume Lemaitre `. + +- |API| The `n_features_` attribute of + :class:`decomposition.PCA` is deprecated in favor of + `n_features_in_` and will be removed in 1.4. :pr:`24421` by + :user:`Kshitij Mathur `. + +:mod:`sklearn.discriminant_analysis` +.................................... + +- |MajorFeature| :class:`discriminant_analysis.LinearDiscriminantAnalysis` now + supports the `Array API `_ for + `solver="svd"`. Array API support is considered experimental and might evolve + without being subjected to our usual rolling deprecation cycle policy. See + :ref:`array_api` for more details. :pr:`22554` by `Thomas Fan`_. + +- |Fix| Validate parameters only in `fit` and not in `__init__` + for :class:`discriminant_analysis.QuadraticDiscriminantAnalysis`. + :pr:`24218` by :user:`Stefanie Molin `. + +:mod:`sklearn.ensemble` +....................... + +- |MajorFeature| :class:`ensemble.HistGradientBoostingClassifier` and + :class:`ensemble.HistGradientBoostingRegressor` now support + interaction constraints via the argument `interaction_cst` of their + constructors. + :pr:`21020` by :user:`Christian Lorentzen `. + Using interaction constraints also makes fitting faster. + :pr:`24856` by :user:`Christian Lorentzen `. + +- |Feature| Adds `class_weight` to :class:`ensemble.HistGradientBoostingClassifier`. + :pr:`22014` by `Thomas Fan`_. + +- |Efficiency| Improve runtime performance of :class:`ensemble.IsolationForest` + by avoiding data copies. :pr:`23252` by :user:`Zhehao Liu `. + +- |Enhancement| :class:`ensemble.StackingClassifier` now accepts any kind of + base estimator. + :pr:`24538` by :user:`Guillem G Subies `. + +- |Enhancement| Make it possible to pass the `categorical_features` parameter + of :class:`ensemble.HistGradientBoostingClassifier` and + :class:`ensemble.HistGradientBoostingRegressor` as feature names. + :pr:`24889` by :user:`Olivier Grisel `. + +- |Enhancement| :class:`ensemble.StackingClassifier` now supports + multilabel-indicator target + :pr:`24146` by :user:`Nicolas Peretti `, + :user:`Nestor Navarro `, :user:`Nati Tomattis `, + and :user:`Vincent Maladiere `. + +- |Enhancement| :class:`ensemble.HistGradientBoostingClassifier` and + :class:`ensemble.HistGradientBoostingRegressor` now accept their + `monotonic_cst` parameter to be passed as a dictionary in addition + to the previously supported array-like format. + Such dictionary have feature names as keys and one of `-1`, `0`, `1` + as value to specify monotonicity constraints for each feature. + :pr:`24855` by :user:`Olivier Grisel `. + +- |Enhancement| Interaction constraints for + :class:`ensemble.HistGradientBoostingClassifier` + and :class:`ensemble.HistGradientBoostingRegressor` can now be specified + as strings for two common cases: "no_interactions" and "pairwise" interactions. + :pr:`24849` by :user:`Tim Head `. + +- |Fix| Fixed the issue where :class:`ensemble.AdaBoostClassifier` outputs + NaN in feature importance when fitted with very small sample weight. + :pr:`20415` by :user:`Zhehao Liu `. + +- |Fix| :class:`ensemble.HistGradientBoostingClassifier` and + :class:`ensemble.HistGradientBoostingRegressor` no longer error when predicting + on categories encoded as negative values and instead consider them a member + of the "missing category". :pr:`24283` by `Thomas Fan`_. + +- |Fix| :class:`ensemble.HistGradientBoostingClassifier` and + :class:`ensemble.HistGradientBoostingRegressor`, with `verbose>=1`, print detailed + timing information on computing histograms and finding best splits. The time spent in + the root node was previously missing and is now included in the printed information. + :pr:`24894` by :user:`Christian Lorentzen `. + +- |API| Rename the constructor parameter `base_estimator` to `estimator` in + the following classes: + :class:`ensemble.BaggingClassifier`, + :class:`ensemble.BaggingRegressor`, + :class:`ensemble.AdaBoostClassifier`, + :class:`ensemble.AdaBoostRegressor`. + `base_estimator` is deprecated in 1.2 and will be removed in 1.4. + :pr:`23819` by :user:`Adrian Trujillo ` and + :user:`Edoardo Abati `. + +- |API| Rename the fitted attribute `base_estimator_` to `estimator_` in + the following classes: + :class:`ensemble.BaggingClassifier`, + :class:`ensemble.BaggingRegressor`, + :class:`ensemble.AdaBoostClassifier`, + :class:`ensemble.AdaBoostRegressor`, + :class:`ensemble.RandomForestClassifier`, + :class:`ensemble.RandomForestRegressor`, + :class:`ensemble.ExtraTreesClassifier`, + :class:`ensemble.ExtraTreesRegressor`, + :class:`ensemble.RandomTreesEmbedding`, + :class:`ensemble.IsolationForest`. + `base_estimator_` is deprecated in 1.2 and will be removed in 1.4. + :pr:`23819` by :user:`Adrian Trujillo ` and + :user:`Edoardo Abati `. + +:mod:`sklearn.feature_selection` +................................ + +- |Fix| Fix a bug in :func:`feature_selection.mutual_info_regression` and + :func:`feature_selection.mutual_info_classif`, where the continuous features + in `X` should be scaled to a unit variance independently if the target `y` is + continuous or discrete. + :pr:`24747` by :user:`Guillaume Lemaitre ` + +:mod:`sklearn.gaussian_process` +............................... + +- |Fix| Fix :class:`gaussian_process.kernels.Matern` gradient computation with + `nu=0.5` for PyPy (and possibly other non CPython interpreters). :pr:`24245` + by :user:`Loïc Estève `. + +- |Fix| The `fit` method of :class:`gaussian_process.GaussianProcessRegressor` + will not modify the input X in case a custom kernel is used, with a `diag` + method that returns part of the input X. :pr:`24405` + by :user:`Omar Salman `. + +:mod:`sklearn.impute` +..................... + +- |Enhancement| Added `keep_empty_features` parameter to + :class:`impute.SimpleImputer`, :class:`impute.KNNImputer` and + :class:`impute.IterativeImputer`, preventing removal of features + containing only missing values when transforming. + :pr:`16695` by :user:`Vitor Santa Rosa `. + +:mod:`sklearn.inspection` +......................... + +- |MajorFeature| Extended :func:`inspection.partial_dependence` and + :class:`inspection.PartialDependenceDisplay` to handle categorical features. + :pr:`18298` by :user:`Madhura Jayaratne ` and + :user:`Guillaume Lemaitre `. + +- |Fix| :class:`inspection.DecisionBoundaryDisplay` now raises error if input + data is not 2-dimensional. + :pr:`25077` by :user:`Arturo Amor `. + +:mod:`sklearn.kernel_approximation` +................................... + +- |Enhancement| :class:`kernel_approximation.RBFSampler` now preserves + dtype for `numpy.float32` inputs. :pr:`24317` by `Tim Head `. + +- |Enhancement| :class:`kernel_approximation.SkewedChi2Sampler` now preserves + dtype for `numpy.float32` inputs. :pr:`24350` by :user:`Rahil Parikh `. + +- |Enhancement| :class:`kernel_approximation.RBFSampler` now accepts + `'scale'` option for parameter `gamma`. + :pr:`24755` by :user:`Hleb Levitski `. + +:mod:`sklearn.linear_model` +........................... + +- |Enhancement| :class:`linear_model.LogisticRegression`, + :class:`linear_model.LogisticRegressionCV`, :class:`linear_model.GammaRegressor`, + :class:`linear_model.PoissonRegressor` and :class:`linear_model.TweedieRegressor` got + a new solver `solver="newton-cholesky"`. This is a 2nd order (Newton) optimisation + routine that uses a Cholesky decomposition of the hessian matrix. + When `n_samples >> n_features`, the `"newton-cholesky"` solver has been observed to + converge both faster and to a higher precision solution than the `"lbfgs"` solver on + problems with one-hot encoded categorical variables with some rare categorical + levels. + :pr:`24637` and :pr:`24767` by :user:`Christian Lorentzen `. + +- |Enhancement| :class:`linear_model.GammaRegressor`, + :class:`linear_model.PoissonRegressor` and :class:`linear_model.TweedieRegressor` + can reach higher precision with the lbfgs solver, in particular when `tol` is set + to a tiny value. Moreover, `verbose` is now properly propagated to L-BFGS-B. + :pr:`23619` by :user:`Christian Lorentzen `. + +- |Fix| :class:`linear_model.SGDClassifier` and :class:`linear_model.SGDRegressor` will + raise an error when all the validation samples have zero sample weight. + :pr:`23275` by `Zhehao Liu `. + +- |Fix| :class:`linear_model.SGDOneClassSVM` no longer performs parameter + validation in the constructor. All validation is now handled in `fit()` and + `partial_fit()`. + :pr:`24433` by :user:`Yogendrasingh `, :user:`Arisa Y. ` + and :user:`Tim Head `. + +- |Fix| Fix average loss calculation when early stopping is enabled in + :class:`linear_model.SGDRegressor` and :class:`linear_model.SGDClassifier`. + Also updated the condition for early stopping accordingly. + :pr:`23798` by :user:`Harsh Agrawal `. + +- |API| The default value for the `solver` parameter in + :class:`linear_model.QuantileRegressor` will change from `"interior-point"` + to `"highs"` in version 1.4. + :pr:`23637` by :user:`Guillaume Lemaitre `. + +- |API| String option `"none"` is deprecated for `penalty` argument + in :class:`linear_model.LogisticRegression`, and will be removed in version 1.4. + Use `None` instead. :pr:`23877` by :user:`Zhehao Liu `. + +- |API| The default value of `tol` was changed from `1e-3` to `1e-4` for + :func:`linear_model.ridge_regression`, :class:`linear_model.Ridge` and + :class:`linear_model.RidgeClassifier`. + :pr:`24465` by :user:`Christian Lorentzen `. + +:mod:`sklearn.manifold` +....................... + +- |Feature| Adds option to use the normalized stress in :class:`manifold.MDS`. This is + enabled by setting the new `normalize` parameter to `True`. + :pr:`10168` by :user:`Łukasz Borchmann `, + :pr:`12285` by :user:`Matthias Miltenberger `, + :pr:`13042` by :user:`Matthieu Parizy `, + :pr:`18094` by :user:`Roth E Conrad ` and + :pr:`22562` by :user:`Meekail Zain `. + +- |Enhancement| Adds `eigen_tol` parameter to + :class:`manifold.SpectralEmbedding`. Both :func:`manifold.spectral_embedding` + and :class:`manifold.SpectralEmbedding` now propagate `eigen_tol` to all + choices of `eigen_solver`. Includes a new option `eigen_tol="auto"` + and begins deprecation to change the default from `eigen_tol=0` to + `eigen_tol="auto"` in version 1.3. + :pr:`23210` by :user:`Meekail Zain `. + +- |Enhancement| :class:`manifold.Isomap` now preserves + dtype for `np.float32` inputs. :pr:`24714` by :user:`Rahil Parikh `. + +- |API| Added an `"auto"` option to the `normalized_stress` argument in + :class:`manifold.MDS` and :func:`manifold.smacof`. Note that + `normalized_stress` is only valid for non-metric MDS, therefore the `"auto"` + option enables `normalized_stress` when `metric=False` and disables it when + `metric=True`. `"auto"` will become the default value for `normalized_stress` + in version 1.4. + :pr:`23834` by :user:`Meekail Zain ` + +:mod:`sklearn.metrics` +...................... + +- |Feature| :func:`metrics.ConfusionMatrixDisplay.from_estimator`, + :func:`metrics.ConfusionMatrixDisplay.from_predictions`, and + :meth:`metrics.ConfusionMatrixDisplay.plot` accepts a `text_kw` parameter which is + passed to matplotlib's `text` function. :pr:`24051` by `Thomas Fan`_. + +- |Feature| :func:`metrics.class_likelihood_ratios` is added to compute the positive and + negative likelihood ratios derived from the confusion matrix + of a binary classification problem. :pr:`22518` by + :user:`Arturo Amor `. + +- |Feature| Add :class:`metrics.PredictionErrorDisplay` to plot residuals vs + predicted and actual vs predicted to qualitatively assess the behavior of a + regressor. The display can be created with the class methods + :func:`metrics.PredictionErrorDisplay.from_estimator` and + :func:`metrics.PredictionErrorDisplay.from_predictions`. :pr:`18020` by + :user:`Guillaume Lemaitre `. + +- |Feature| :func:`metrics.roc_auc_score` now supports micro-averaging + (`average="micro"`) for the One-vs-Rest multiclass case (`multi_class="ovr"`). + :pr:`24338` by :user:`Arturo Amor `. + +- |Enhancement| Adds an `"auto"` option to `eps` in :func:`metrics.log_loss`. + This option will automatically set the `eps` value depending on the data + type of `y_pred`. In addition, the default value of `eps` is changed from + `1e-15` to the new `"auto"` option. + :pr:`24354` by :user:`Safiuddin Khaja ` and :user:`gsiisg `. + +- |Fix| Allows `csr_matrix` as input for parameter: `y_true` of + the :func:`metrics.label_ranking_average_precision_score` metric. + :pr:`23442` by :user:`Sean Atukorala ` + +- |Fix| :func:`metrics.ndcg_score` will now trigger a warning when the `y_true` + value contains a negative value. Users may still use negative values, but the + result may not be between 0 and 1. Starting in v1.4, passing in negative + values for `y_true` will raise an error. + :pr:`22710` by :user:`Conroy Trinh ` and + :pr:`23461` by :user:`Meekail Zain `. + +- |Fix| :func:`metrics.log_loss` with `eps=0` now returns a correct value of 0 or + `np.inf` instead of `nan` for predictions at the boundaries (0 or 1). It also accepts + integer input. + :pr:`24365` by :user:`Christian Lorentzen `. + +- |API| The parameter `sum_over_features` of + :func:`metrics.pairwise.manhattan_distances` is deprecated and will be removed in 1.4. + :pr:`24630` by :user:`Rushil Desai `. + +:mod:`sklearn.model_selection` +.............................. + +- |Feature| Added the class :class:`model_selection.LearningCurveDisplay` + that allows to make easy plotting of learning curves obtained by the function + :func:`model_selection.learning_curve`. + :pr:`24084` by :user:`Guillaume Lemaitre `. + +- |Fix| For all `SearchCV` classes and scipy >= 1.10, rank corresponding to a + nan score is correctly set to the maximum possible rank, rather than + `np.iinfo(np.int32).min`. :pr:`24141` by :user:`Loïc Estève `. + +- |Fix| In both :class:`model_selection.HalvingGridSearchCV` and + :class:`model_selection.HalvingRandomSearchCV` parameter + combinations with a NaN score now share the lowest rank. + :pr:`24539` by :user:`Tim Head `. + +- |Fix| For :class:`model_selection.GridSearchCV` and + :class:`model_selection.RandomizedSearchCV` ranks corresponding to nan + scores will all be set to the maximum possible rank. + :pr:`24543` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.multioutput` +.......................... + +- |Feature| Added boolean `verbose` flag to classes: + :class:`multioutput.ClassifierChain` and :class:`multioutput.RegressorChain`. + :pr:`23977` by :user:`Eric Fiegel `, + :user:`Chiara Marmo `, + :user:`Lucy Liu `, and + :user:`Guillaume Lemaitre `. + +:mod:`sklearn.naive_bayes` +.......................... + +- |Feature| Add methods `predict_joint_log_proba` to all naive Bayes classifiers. + :pr:`23683` by :user:`Andrey Melnik `. + +- |Enhancement| A new parameter `force_alpha` was added to + :class:`naive_bayes.BernoulliNB`, :class:`naive_bayes.ComplementNB`, + :class:`naive_bayes.CategoricalNB`, and :class:`naive_bayes.MultinomialNB`, + allowing user to set parameter alpha to a very small number, greater or equal + 0, which was earlier automatically changed to `1e-10` instead. + :pr:`16747` by :user:`arka204`, + :pr:`18805` by :user:`hongshaoyang`, + :pr:`22269` by :user:`Meekail Zain `. + +:mod:`sklearn.neighbors` +........................ + +- |Feature| Adds new function :func:`neighbors.sort_graph_by_row_values` to + sort a CSR sparse graph such that each row is stored with increasing values. + This is useful to improve efficiency when using precomputed sparse distance + matrices in a variety of estimators and avoid an `EfficiencyWarning`. + :pr:`23139` by `Tom Dupre la Tour`_. + +- |Efficiency| :class:`neighbors.NearestCentroid` is faster and requires + less memory as it better leverages CPUs' caches to compute predictions. + :pr:`24645` by :user:`Olivier Grisel `. + +- |Enhancement| :class:`neighbors.KernelDensity` bandwidth parameter now accepts + definition using Scott's and Silverman's estimation methods. + :pr:`10468` by :user:`Ruben ` and :pr:`22993` by + :user:`Jovan Stojanovic `. + +- |Enhancement| `neighbors.NeighborsBase` now accepts + Minkowski semi-metric (i.e. when :math:`0 < p < 1` for + `metric="minkowski"`) for `algorithm="auto"` or `algorithm="brute"`. + :pr:`24750` by :user:`Rudresh Veerkhare ` + +- |Fix| :class:`neighbors.NearestCentroid` now raises an informative error message at fit-time + instead of failing with a low-level error message at predict-time. + :pr:`23874` by :user:`Juan Gomez <2357juan>`. + +- |Fix| Set `n_jobs=None` by default (instead of `1`) for + :class:`neighbors.KNeighborsTransformer` and + :class:`neighbors.RadiusNeighborsTransformer`. + :pr:`24075` by :user:`Valentin Laurent `. + +- |Enhancement| :class:`neighbors.LocalOutlierFactor` now preserves + dtype for `numpy.float32` inputs. + :pr:`22665` by :user:`Julien Jerphanion `. + +:mod:`sklearn.neural_network` +............................. + +- |Fix| :class:`neural_network.MLPClassifier` and + :class:`neural_network.MLPRegressor` always expose the parameters `best_loss_`, + `validation_scores_`, and `best_validation_score_`. `best_loss_` is set to + `None` when `early_stopping=True`, while `validation_scores_` and + `best_validation_score_` are set to `None` when `early_stopping=False`. + :pr:`24683` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.pipeline` +....................... + +- |Enhancement| :meth:`pipeline.FeatureUnion.get_feature_names_out` can now + be used when one of the transformers in the :class:`pipeline.FeatureUnion` is + `"passthrough"`. :pr:`24058` by :user:`Diederik Perdok ` + +- |Enhancement| The :class:`pipeline.FeatureUnion` class now has a `named_transformers` + attribute for accessing transformers by name. + :pr:`20331` by :user:`Christopher Flynn `. + +:mod:`sklearn.preprocessing` +............................ + +- |Enhancement| :class:`preprocessing.FunctionTransformer` will always try to set + `n_features_in_` and `feature_names_in_` regardless of the `validate` parameter. + :pr:`23993` by `Thomas Fan`_. + +- |Fix| :class:`preprocessing.LabelEncoder` correctly encodes NaNs in `transform`. + :pr:`22629` by `Thomas Fan`_. + +- |API| The `sparse` parameter of :class:`preprocessing.OneHotEncoder` + is now deprecated and will be removed in version 1.4. Use `sparse_output` instead. + :pr:`24412` by :user:`Rushil Desai `. + +:mod:`sklearn.svm` +.................. + +- |API| The `class_weight_` attribute is now deprecated for + :class:`svm.NuSVR`, :class:`svm.SVR`, :class:`svm.OneClassSVM`. + :pr:`22898` by :user:`Meekail Zain `. + +:mod:`sklearn.tree` +................... + +- |Enhancement| :func:`tree.plot_tree`, :func:`tree.export_graphviz` now uses + a lower case `x[i]` to represent feature `i`. :pr:`23480` by `Thomas Fan`_. + +:mod:`sklearn.utils` +.................... + +- |Feature| A new module exposes development tools to discover estimators (i.e. + :func:`utils.discovery.all_estimators`), displays (i.e. + :func:`utils.discovery.all_displays`) and functions (i.e. + :func:`utils.discovery.all_functions`) in scikit-learn. + :pr:`21469` by :user:`Guillaume Lemaitre `. + +- |Enhancement| :func:`utils.extmath.randomized_svd` now accepts an argument, + `lapack_svd_driver`, to specify the lapack driver used in the internal + deterministic SVD used by the randomized SVD algorithm. + :pr:`20617` by :user:`Srinath Kailasa ` + +- |Enhancement| :func:`utils.validation.column_or_1d` now accepts a `dtype` + parameter to specific `y`'s dtype. :pr:`22629` by `Thomas Fan`_. + +- |Enhancement| `utils.extmath.cartesian` now accepts arrays with different + `dtype` and will cast the output to the most permissive `dtype`. + :pr:`25067` by :user:`Guillaume Lemaitre `. + +- |Fix| :func:`utils.multiclass.type_of_target` now properly handles sparse matrices. + :pr:`14862` by :user:`LÊonard Binet `. + +- |Fix| HTML representation no longer errors when an estimator class is a value in + `get_params`. :pr:`24512` by `Thomas Fan`_. + +- |Fix| :func:`utils.estimator_checks.check_estimator` now takes into account + the `requires_positive_X` tag correctly. :pr:`24667` by `Thomas Fan`_. + +- |Fix| :func:`utils.check_array` now supports Pandas Series with `pd.NA` + by raising a better error message or returning a compatible `ndarray`. + :pr:`25080` by `Thomas Fan`_. + +- |API| The extra keyword parameters of :func:`utils.extmath.density` are deprecated + and will be removed in 1.4. + :pr:`24523` by :user:`Mia Bajic `. + +.. rubric:: Code and documentation contributors + +Thanks to everyone who has contributed to the maintenance and improvement of +the project since version 1.1, including: + +2357juan, 3lLobo, Adam J. Stewart, Adam Kania, Adam Li, Aditya Anulekh, Admir +Demiraj, adoublet, Adrin Jalali, Ahmedbgh, Aiko, Akshita Prasanth, Ala-Na, +Alessandro Miola, Alex, Alexandr, Alexandre Perez-Lebel, Alex Buzenet, Ali H. +El-Kassas, aman kumar, Amit Bera, AndrÃĄs Simon, Andreas Grivas, Andreas +Mueller, Andrew Wang, angela-maennel, Aniket Shirsat, Anthony22-dev, Antony +Lee, anupam, Apostolos Tsetoglou, Aravindh R, Artur Hermano, Arturo Amor, +as-90, ashah002, Ashwin Mathur, avm19, Azaria Gebremichael, b0rxington, Badr +MOUFAD, Bardiya Ak, Bartłomiej Gońda, BdeGraaff, Benjamin Bossan, Benjamin +Carter, berkecanrizai, Bernd Fritzke, Bhoomika, Biswaroop Mitra, Brandon TH +Chen, Brett Cannon, Bsh, cache-missing, carlo, Carlos Ramos CarreÃąo, ceh, +chalulu, Changyao Chen, Charles Zablit, Chiara Marmo, Christian Lorentzen, +Christian Ritter, Christian Veenhuis, christianwaldmann, Christine P. Chai, +Claudio Salvatore Arcidiacono, ClÊment Verrier, crispinlogan, Da-Lan, +DanGonite57, Daniela Fernandes, DanielGaerber, darioka, Darren Nguyen, +davidblnc, david-cortes, David Gilbertson, David Poznik, Dayne, Dea María +LÊon, Denis, Dev Khant, Dhanshree Arora, Diadochokinetic, diederikwp, Dimitri +Papadopoulos Orfanos, Dimitris Litsidis, drewhogg, Duarte OC, Dwight Lindquist, +Eden Brekke, Edern, Edoardo Abati, Eleanore Denies, EliaSchiavon, Emir, +ErmolaevPA, Fabrizio Damicelli, fcharras, Felipe Siola, Flynn, +francesco-tuveri, Franck Charras, ftorres16, Gael Varoquaux, Geevarghese +George, genvalen, GeorgiaMayDay, Gianr Lazz, Hleb Levitski, GlÃ˛ria Macià +MuÃąoz, Guillaume Lemaitre, Guillem García Subies, Guitared, gunesbayir, +Haesun Park, Hansin Ahuja, Hao Chun Chang, Harsh Agrawal, harshit5674, +hasan-yaman, henrymooresc, Henry Sorsky, Hristo Vrigazov, htsedebenham, humahn, +i-aki-y, Ian Thompson, Ido M, Iglesys, Iliya Zhechev, Irene, ivanllt, Ivan +Sedykh, Jack McIvor, jakirkham, JanFidor, Jason G, JÊrÊmie du Boisberranger, +Jiten Sidhpura, jkarolczak, JoÃŖo David, JohnathanPi, John Koumentis, John P, +John Pangas, johnthagen, Jordan Fleming, Joshua Choo Yun Keat, Jovan +Stojanovic, Juan Carlos Alfaro JimÊnez, juanfe88, Juan Felipe Arias, +JuliaSchoepp, Julien Jerphanion, jygerardy, ka00ri, Kanishk Sachdev, Kanissh, +Kaushik Amar Das, Kendall, Kenneth Prabakaran, Kento Nozawa, kernc, Kevin +Roice, Kian Eliasi, Kilian Kluge, Kilian Lieret, Kirandevraj, Kraig, krishna +kumar, krishna vamsi, Kshitij Kapadni, Kshitij Mathur, Lauren Burke, LÊonard +Binet, lingyi1110, Lisa Casino, Logan Thomas, Loic Esteve, Luciano Mantovani, +Lucy Liu, Maascha, Madhura Jayaratne, madinak, Maksym, Malte S. Kurz, Mansi +Agrawal, Marco Edward Gorelli, Marco Wurps, Maren Westermann, Maria Telenczuk, +Mario Kostelac, martin-kokos, Marvin Krawutschke, Masanori Kanazu, mathurinm, +Matt Haberland, mauroantonioserrano, Max Halford, Maxi Marufo, maximeSaur, +Maxim Smolskiy, Maxwell, m. bou, Meekail Zain, Mehgarg, mehmetcanakbay, Mia +Bajić, Michael Flaks, Michael Hornstein, Michel de Ruiter, Michelle Paradis, +Mikhail Iljin, Misa Ogura, Moritz Wilksch, mrastgoo, Naipawat Poolsawat, Naoise +Holohan, Nass, Nathan Jacobi, Nawazish Alam, Nguyáģ…n Văn Diáģ…n, Nicola +Fanelli, Nihal Thukarama Rao, Nikita Jare, nima10khodaveisi, Nima Sarajpoor, +nitinramvelraj, NNLNR, npache, Nwanna-Joseph, Nymark Kho, o-holman, Olivier +Grisel, Olle Lukowski, Omar Hassoun, Omar Salman, osman tamer, ouss1508, +Oyindamola Olatunji, PAB, Pandata, partev, Paulo Sergio Soares, Petar +Mlinarić, Peter Jansson, Peter Steinbach, Philipp Jung, Piet BrÃļmmel, Pooja +M, Pooja Subramaniam, priyam kakati, puhuk, Rachel Freeland, Rachit Keerti Das, +Rafal Wojdyla, Raghuveer Bhat, Rahil Parikh, Ralf Gommers, ram vikram singh, +Ravi Makhija, Rehan Guha, Reshama Shaikh, Richard Klima, Rob Crockett, Robert +Hommes, Robert Juergens, Robin Lenz, Rocco Meli, Roman4oo, Ross Barnowski, +Rowan Mankoo, Rudresh Veerkhare, Rushil Desai, Sabri Monaf Sabri, Safikh, +Safiuddin Khaja, Salahuddin, Sam Adam Day, Sandra Yojana Meneses, Sandro +Ephrem, Sangam, SangamSwadik, SANJAI_3, SarahRemus, Sashka Warner, SavkoMax, +Scott Gigante, Scott Gustafson, Sean Atukorala, sec65, SELEE, seljaks, Shady el +Gewily, Shane, shellyfung, Shinsuke Mori, Shiva chauhan, Shoaib Khan, Shogo +Hida, Shrankhla Srivastava, Shuangchi He, Simon, sonnivs, Sortofamudkip, +Srinath Kailasa, Stanislav (Stanley) Modrak, Stefanie Molin, stellalin7, +StÊphane Collot, Steven Van Vaerenbergh, Steve Schmerler, Sven Stehle, Tabea +Kossen, TheDevPanda, the-syd-sre, Thijs van Weezel, Thomas Bonald, Thomas +Germer, Thomas J. Fan, Ti-Ion, Tim Head, Timofei Kornev, toastedyeast, Tobias +Pitters, Tom DuprÊ la Tour, tomiock, Tom Mathews, Tom McTiernan, tspeng, Tyler +Egashira, Valentin Laurent, Varun Jain, Vera Komeyer, Vicente Reyes-Puerta, +Vinayak Mehta, Vincent M, Vishal, Vyom Pathak, wattai, wchathura, WEN Hao, +William M, x110, Xiao Yuan, Xunius, yanhong-zhao-ef, Yusuf Raji, Z Adil Khwaja, +zeeshan lone diff --git a/doc/whats_new/v1.3.rst b/doc/whats_new/v1.3.rst new file mode 100644 index 0000000000000..f523c02e14447 --- /dev/null +++ b/doc/whats_new/v1.3.rst @@ -0,0 +1,1003 @@ +.. include:: _contributors.rst + +.. currentmodule:: sklearn + +.. _release_notes_1_3: + +=========== +Version 1.3 +=========== + +For a short description of the main highlights of the release, please refer to +:ref:`sphx_glr_auto_examples_release_highlights_plot_release_highlights_1_3_0.py`. + +.. include:: changelog_legend.inc + +.. _changes_1_3_2: + +Version 1.3.2 +============= + +**October 2023** + +Changelog +--------- + +:mod:`sklearn.datasets` +....................... + +- |Fix| All dataset fetchers now accept `data_home` as any object that implements + the :class:`os.PathLike` interface, for instance, :class:`pathlib.Path`. + :pr:`27468` by :user:`Yao Xiao `. + +:mod:`sklearn.decomposition` +............................ + +- |Fix| Fixes a bug in :class:`decomposition.KernelPCA` by forcing the output of + the internal :class:`preprocessing.KernelCenterer` to be a default array. When the + arpack solver is used, it expects an array with a `dtype` attribute. + :pr:`27583` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.metrics` +...................... + +- |Fix| Fixes a bug for metrics using `zero_division=np.nan` + (e.g. :func:`~metrics.precision_score`) within a parallel loop + (e.g. :func:`~model_selection.cross_val_score`) where the singleton for `np.nan` + will be different in the sub-processes. + :pr:`27573` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.tree` +................... + +- |Fix| Do not leak data via non-initialized memory in decision tree pickle files and make + the generation of those files deterministic. :pr:`27580` by :user:`Loïc Estève `. + + +.. _changes_1_3_1: + +Version 1.3.1 +============= + +**September 2023** + +Changed models +-------------- + +The following estimators and functions, when fit with the same data and +parameters, may produce different models from the previous version. This often +occurs due to changes in the modelling logic (bug fixes or enhancements), or in +random sampling procedures. + +- |Fix| Ridge models with `solver='sparse_cg'` may have slightly different + results with scipy>=1.12, because of an underlying change in the scipy solver + (see `scipy#18488 `_ for more + details) + :pr:`26814` by :user:`Loïc Estève ` + +Changes impacting all modules +----------------------------- + +- |Fix| The `set_output` API correctly works with list input. :pr:`27044` by + `Thomas Fan`_. + +Changelog +--------- + +:mod:`sklearn.calibration` +.......................... + +- |Fix| :class:`calibration.CalibratedClassifierCV` can now handle models that + produce large prediction scores. Before it was numerically unstable. + :pr:`26913` by :user:`Omar Salman `. + +:mod:`sklearn.cluster` +...................... + +- |Fix| :class:`cluster.BisectingKMeans` could crash when predicting on data + with a different scale than the data used to fit the model. + :pr:`27167` by `Olivier Grisel`_. + +- |Fix| :class:`cluster.BisectingKMeans` now works with data that has a single feature. + :pr:`27243` by :user:`JÊrÊmie du Boisberranger `. + +:mod:`sklearn.cross_decomposition` +.................................. + +- |Fix| :class:`cross_decomposition.PLSRegression` now automatically ravels the output + of `predict` if fitted with one dimensional `y`. + :pr:`26602` by :user:`Yao Xiao `. + +:mod:`sklearn.ensemble` +....................... + +- |Fix| Fix a bug in :class:`ensemble.AdaBoostClassifier` with `algorithm="SAMME"` + where the decision function of each weak learner should be symmetric (i.e. + the sum of the scores should sum to zero for a sample). + :pr:`26521` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.feature_selection` +................................ + +- |Fix| :func:`feature_selection.mutual_info_regression` now correctly computes the + result when `X` is of integer dtype. :pr:`26748` by :user:`Yao Xiao `. + +:mod:`sklearn.impute` +..................... + +- |Fix| :class:`impute.KNNImputer` now correctly adds a missing indicator column in + ``transform`` when ``add_indicator`` is set to ``True`` and missing values are observed + during ``fit``. :pr:`26600` by :user:`Shreesha Kumar Bhat `. + +:mod:`sklearn.metrics` +...................... + +- |Fix| Scorers used with :func:`metrics.get_scorer` handle properly + multilabel-indicator matrix. + :pr:`27002` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.mixture` +...................... + +- |Fix| The initialization of :class:`mixture.GaussianMixture` from user-provided + `precisions_init` for `covariance_type` of `full` or `tied` was not correct, + and has been fixed. + :pr:`26416` by :user:`Yang Tao `. + +:mod:`sklearn.neighbors` +........................ + +- |Fix| :meth:`neighbors.KNeighborsClassifier.predict` no longer raises an + exception for `pandas.DataFrames` input. + :pr:`26772` by :user:`JÊrÊmie du Boisberranger `. + +- |Fix| Reintroduce `sklearn.neighbors.BallTree.valid_metrics` and + `sklearn.neighbors.KDTree.valid_metrics` as public class attributes. + :pr:`26754` by :user:`Julien Jerphanion `. + +- |Fix| :class:`sklearn.model_selection.HalvingRandomSearchCV` no longer raises + when the input to the `param_distributions` parameter is a list of dicts. + :pr:`26893` by :user:`Stefanie Senger `. + +- |Fix| Neighbors based estimators now correctly work when `metric="minkowski"` and the + metric parameter `p` is in the range `0 < p < 1`, regardless of the `dtype` of `X`. + :pr:`26760` by :user:`Shreesha Kumar Bhat `. + +:mod:`sklearn.preprocessing` +............................ + +- |Fix| :class:`preprocessing.LabelEncoder` correctly accepts `y` as a keyword + argument. :pr:`26940` by `Thomas Fan`_. + +- |Fix| :class:`preprocessing.OneHotEncoder` shows a more informative error message + when `sparse_output=True` and the output is configured to be pandas. + :pr:`26931` by `Thomas Fan`_. + +:mod:`sklearn.tree` +................... + +- |Fix| :func:`tree.plot_tree` now accepts `class_names=True` as documented. + :pr:`26903` by :user:`Thomas Roehr <2maz>` + +- |Fix| The `feature_names` parameter of :func:`tree.plot_tree` now accepts any kind of + array-like instead of just a list. :pr:`27292` by :user:`Rahil Parikh `. + +.. _changes_1_3: + +Version 1.3.0 +============= + +**June 2023** + +Changed models +-------------- + +The following estimators and functions, when fit with the same data and +parameters, may produce different models from the previous version. This often +occurs due to changes in the modelling logic (bug fixes or enhancements), or in +random sampling procedures. + +- |Enhancement| :meth:`multiclass.OutputCodeClassifier.predict` now uses a more + efficient pairwise distance reduction. As a consequence, the tie-breaking + strategy is different and thus the predicted labels may be different. + :pr:`25196` by :user:`Guillaume Lemaitre `. + +- |Enhancement| The `fit_transform` method of :class:`decomposition.DictionaryLearning` + is more efficient but may produce different results as in previous versions when + `transform_algorithm` is not the same as `fit_algorithm` and the number of iterations + is small. :pr:`24871` by :user:`Omar Salman `. + +- |Enhancement| The `sample_weight` parameter now will be used in centroids + initialization for :class:`cluster.KMeans`, :class:`cluster.BisectingKMeans` + and :class:`cluster.MiniBatchKMeans`. + This change will break backward compatibility, since numbers generated + from same random seeds will be different. + :pr:`25752` by :user:`Hleb Levitski `, + :user:`JÊrÊmie du Boisberranger `, + :user:`Guillaume Lemaitre `. + +- |Fix| Treat more consistently small values in the `W` and `H` matrices during the + `fit` and `transform` steps of :class:`decomposition.NMF` and + :class:`decomposition.MiniBatchNMF` which can produce different results than previous + versions. :pr:`25438` by :user:`Yotam Avidar-Constantini `. + +- |Fix| :class:`decomposition.KernelPCA` may produce different results through + `inverse_transform` if `gamma` is `None`. Now it will be chosen correctly as + `1/n_features` of the data that it is fitted on, while previously it might be + incorrectly chosen as `1/n_features` of the data passed to `inverse_transform`. + A new attribute `gamma_` is provided for revealing the actual value of `gamma` + used each time the kernel is called. + :pr:`26337` by :user:`Yao Xiao `. + +Changed displays +---------------- + +- |Enhancement| :class:`model_selection.LearningCurveDisplay` displays both the + train and test curves by default. You can set `score_type="test"` to keep the + past behaviour. + :pr:`25120` by :user:`Guillaume Lemaitre `. + +- |Fix| :class:`model_selection.ValidationCurveDisplay` now accepts passing a + list to the `param_range` parameter. + :pr:`27311` by :user:`Arturo Amor `. + +Changes impacting all modules +----------------------------- + +- |Enhancement| The `get_feature_names_out` method of the following classes now + raises a `NotFittedError` if the instance is not fitted. This ensures the error is + consistent in all estimators with the `get_feature_names_out` method. + + - :class:`impute.MissingIndicator` + - :class:`feature_extraction.DictVectorizer` + - :class:`feature_extraction.text.TfidfTransformer` + - :class:`feature_selection.GenericUnivariateSelect` + - :class:`feature_selection.RFE` + - :class:`feature_selection.RFECV` + - :class:`feature_selection.SelectFdr` + - :class:`feature_selection.SelectFpr` + - :class:`feature_selection.SelectFromModel` + - :class:`feature_selection.SelectFwe` + - :class:`feature_selection.SelectKBest` + - :class:`feature_selection.SelectPercentile` + - :class:`feature_selection.SequentialFeatureSelector` + - :class:`feature_selection.VarianceThreshold` + - :class:`kernel_approximation.AdditiveChi2Sampler` + - :class:`impute.IterativeImputer` + - :class:`impute.KNNImputer` + - :class:`impute.SimpleImputer` + - :class:`isotonic.IsotonicRegression` + - :class:`preprocessing.Binarizer` + - :class:`preprocessing.KBinsDiscretizer` + - :class:`preprocessing.MaxAbsScaler` + - :class:`preprocessing.MinMaxScaler` + - :class:`preprocessing.Normalizer` + - :class:`preprocessing.OrdinalEncoder` + - :class:`preprocessing.PowerTransformer` + - :class:`preprocessing.QuantileTransformer` + - :class:`preprocessing.RobustScaler` + - :class:`preprocessing.SplineTransformer` + - :class:`preprocessing.StandardScaler` + - :class:`random_projection.GaussianRandomProjection` + - :class:`random_projection.SparseRandomProjection` + + The `NotFittedError` displays an informative message asking to fit the instance + with the appropriate arguments. + + :pr:`25294`, :pr:`25308`, :pr:`25291`, :pr:`25367`, :pr:`25402`, + by :user:`John Pangas `, :user:`Rahil Parikh ` , + and :user:`Alex Buzenet `. + +- |Enhancement| Added a multi-threaded Cython routine to the compute squared + Euclidean distances (sometimes followed by a fused reduction operation) for a + pair of datasets consisting of a sparse CSR matrix and a dense NumPy. + + This can improve the performance of following functions and estimators: + + - :func:`sklearn.metrics.pairwise_distances_argmin` + - :func:`sklearn.metrics.pairwise_distances_argmin_min` + - :class:`sklearn.cluster.AffinityPropagation` + - :class:`sklearn.cluster.Birch` + - :class:`sklearn.cluster.MeanShift` + - :class:`sklearn.cluster.OPTICS` + - :class:`sklearn.cluster.SpectralClustering` + - :func:`sklearn.feature_selection.mutual_info_regression` + - :class:`sklearn.neighbors.KNeighborsClassifier` + - :class:`sklearn.neighbors.KNeighborsRegressor` + - :class:`sklearn.neighbors.RadiusNeighborsClassifier` + - :class:`sklearn.neighbors.RadiusNeighborsRegressor` + - :class:`sklearn.neighbors.LocalOutlierFactor` + - :class:`sklearn.neighbors.NearestNeighbors` + - :class:`sklearn.manifold.Isomap` + - :class:`sklearn.manifold.LocallyLinearEmbedding` + - :class:`sklearn.manifold.TSNE` + - :func:`sklearn.manifold.trustworthiness` + - :class:`sklearn.semi_supervised.LabelPropagation` + - :class:`sklearn.semi_supervised.LabelSpreading` + + A typical example of this performance improvement happens when passing a sparse + CSR matrix to the `predict` or `transform` method of estimators that rely on + a dense NumPy representation to store their fitted parameters (or the reverse). + + For instance, :meth:`sklearn.neighbors.NearestNeighbors.kneighbors` is now up + to 2 times faster for this case on commonly available laptops. + + :pr:`25044` by :user:`Julien Jerphanion `. + +- |Enhancement| All estimators that internally rely on OpenMP multi-threading + (via Cython) now use a number of threads equal to the number of physical + (instead of logical) cores by default. In the past, we observed that using as + many threads as logical cores on SMT hosts could sometimes cause severe + performance problems depending on the algorithms and the shape of the data. + Note that it is still possible to manually adjust the number of threads used + by OpenMP as documented in :ref:`parallelism`. + + :pr:`26082` by :user:`JÊrÊmie du Boisberranger ` and + :user:`Olivier Grisel `. + +Experimental / Under Development +-------------------------------- + +- |MajorFeature| :ref:`Metadata routing `'s related base + methods are included in this release. This feature is only available via the + `enable_metadata_routing` feature flag which can be enabled using + :func:`sklearn.set_config` and :func:`sklearn.config_context`. For now this + feature is mostly useful for third party developers to prepare their code + base for metadata routing, and we strongly recommend that they also hide it + behind the same feature flag, rather than having it enabled by default. + :pr:`24027` by `Adrin Jalali`_, :user:`Benjamin Bossan `, and + :user:`Omar Salman `. + +Changelog +--------- + +.. + Entries should be grouped by module (in alphabetic order) and prefixed with + one of the labels: |MajorFeature|, |Feature|, |Efficiency|, |Enhancement|, + |Fix| or |API| (see whats_new.rst for descriptions). + Entries should be ordered by those labels (e.g. |Fix| after |Efficiency|). + Changes not specific to a module should be listed under *Multiple Modules* + or *Miscellaneous*. + Entries should end with: + :pr:`123456` by :user:`Joe Bloggs `. + where 123456 is the *pull request* number, not the issue number. + +`sklearn` +......... + +- |Feature| Added a new option `skip_parameter_validation`, to the function + :func:`sklearn.set_config` and context manager :func:`sklearn.config_context`, that + allows to skip the validation of the parameters passed to the estimators and public + functions. This can be useful to speed up the code but should be used with care + because it can lead to unexpected behaviors or raise obscure error messages when + setting invalid parameters. + :pr:`25815` by :user:`JÊrÊmie du Boisberranger `. + +:mod:`sklearn.base` +................... + +- |Feature| A `__sklearn_clone__` protocol is now available to override the + default behavior of :func:`base.clone`. :pr:`24568` by `Thomas Fan`_. + +- |Fix| :class:`base.TransformerMixin` now currently keeps a namedtuple's class + if `transform` returns a namedtuple. :pr:`26121` by `Thomas Fan`_. + +:mod:`sklearn.calibration` +.......................... + +- |Fix| :class:`calibration.CalibratedClassifierCV` now does not enforce sample + alignment on `fit_params`. :pr:`25805` by `Adrin Jalali`_. + +:mod:`sklearn.cluster` +...................... + +- |MajorFeature| Added :class:`cluster.HDBSCAN`, a modern hierarchical density-based + clustering algorithm. Similarly to :class:`cluster.OPTICS`, it can be seen as a + generalization of :class:`cluster.DBSCAN` by allowing for hierarchical instead of flat + clustering, however it varies in its approach from :class:`cluster.OPTICS`. This + algorithm is very robust with respect to its hyperparameters' values and can + be used on a wide variety of data without much, if any, tuning. + + This implementation is an adaptation from the original implementation of HDBSCAN in + `scikit-learn-contrib/hdbscan `_, + by :user:`Leland McInnes ` et al. + + :pr:`26385` by :user:`Meekail Zain ` + +- |Enhancement| The `sample_weight` parameter now will be used in centroids + initialization for :class:`cluster.KMeans`, :class:`cluster.BisectingKMeans` + and :class:`cluster.MiniBatchKMeans`. + This change will break backward compatibility, since numbers generated + from same random seeds will be different. + :pr:`25752` by :user:`Hleb Levitski `, + :user:`JÊrÊmie du Boisberranger `, + :user:`Guillaume Lemaitre `. + +- |Fix| :class:`cluster.KMeans`, :class:`cluster.MiniBatchKMeans` and + :func:`cluster.k_means` now correctly handle the combination of `n_init="auto"` + and `init` being an array-like, running one initialization in that case. + :pr:`26657` by :user:`Binesh Bannerjee `. + +- |API| The `sample_weight` parameter in `predict` for + :meth:`cluster.KMeans.predict` and :meth:`cluster.MiniBatchKMeans.predict` + is now deprecated and will be removed in v1.5. + :pr:`25251` by :user:`Hleb Levitski `. + +- |API| The `Xred` argument in :func:`cluster.FeatureAgglomeration.inverse_transform` + is renamed to `Xt` and will be removed in v1.5. :pr:`26503` by `Adrin Jalali`_. + +:mod:`sklearn.compose` +...................... + +- |Fix| :class:`compose.ColumnTransformer` raises an informative error when the individual + transformers of `ColumnTransformer` output pandas dataframes with indexes that are + not consistent with each other and the output is configured to be pandas. + :pr:`26286` by `Thomas Fan`_. + +- |Fix| :class:`compose.ColumnTransformer` correctly sets the output of the + remainder when `set_output` is called. :pr:`26323` by `Thomas Fan`_. + +:mod:`sklearn.covariance` +......................... + +- |Fix| Allows `alpha=0` in :class:`covariance.GraphicalLasso` to be + consistent with :func:`covariance.graphical_lasso`. + :pr:`26033` by :user:`Genesis Valencia `. + +- |Fix| :func:`covariance.empirical_covariance` now gives an informative + error message when input is not appropriate. + :pr:`26108` by :user:`Quentin BarthÊlemy `. + +- |API| Deprecates `cov_init` in :func:`covariance.graphical_lasso` in 1.3 since + the parameter has no effect. It will be removed in 1.5. + :pr:`26033` by :user:`Genesis Valencia `. + +- |API| Adds `costs_` fitted attribute in :class:`covariance.GraphicalLasso` and + :class:`covariance.GraphicalLassoCV`. + :pr:`26033` by :user:`Genesis Valencia `. + +- |API| Adds `covariance` parameter in :class:`covariance.GraphicalLasso`. + :pr:`26033` by :user:`Genesis Valencia `. + +- |API| Adds `eps` parameter in :class:`covariance.GraphicalLasso`, + :func:`covariance.graphical_lasso`, and :class:`covariance.GraphicalLassoCV`. + :pr:`26033` by :user:`Genesis Valencia `. + +:mod:`sklearn.datasets` +....................... + +- |Enhancement| Allows to overwrite the parameters used to open the ARFF file using + the parameter `read_csv_kwargs` in :func:`datasets.fetch_openml` when using the + pandas parser. + :pr:`26433` by :user:`Guillaume Lemaitre `. + +- |Fix| :func:`datasets.fetch_openml` returns improved data types when + `as_frame=True` and `parser="liac-arff"`. :pr:`26386` by `Thomas Fan`_. + +- |Fix| Following the ARFF specs, only the marker `"?"` is now considered as a missing + values when opening ARFF files fetched using :func:`datasets.fetch_openml` when using + the pandas parser. The parameter `read_csv_kwargs` allows to overwrite this behaviour. + :pr:`26551` by :user:`Guillaume Lemaitre `. + +- |Fix| :func:`datasets.fetch_openml` will consistently use `np.nan` as missing marker + with both parsers `"pandas"` and `"liac-arff"`. + :pr:`26579` by :user:`Guillaume Lemaitre `. + +- |API| The `data_transposed` argument of :func:`datasets.make_sparse_coded_signal` + is deprecated and will be removed in v1.5. + :pr:`25784` by :user:`JÊrÊmie du Boisberranger`. + +:mod:`sklearn.decomposition` +............................ + +- |Efficiency| :class:`decomposition.MiniBatchDictionaryLearning` and + :class:`decomposition.MiniBatchSparsePCA` are now faster for small batch sizes by + avoiding duplicate validations. + :pr:`25490` by :user:`JÊrÊmie du Boisberranger `. + +- |Enhancement| :class:`decomposition.DictionaryLearning` now accepts the parameter + `callback` for consistency with the function :func:`decomposition.dict_learning`. + :pr:`24871` by :user:`Omar Salman `. + +- |Fix| Treat more consistently small values in the `W` and `H` matrices during the + `fit` and `transform` steps of :class:`decomposition.NMF` and + :class:`decomposition.MiniBatchNMF` which can produce different results than previous + versions. :pr:`25438` by :user:`Yotam Avidar-Constantini `. + +- |API| The `W` argument in :func:`decomposition.NMF.inverse_transform` and + :class:`decomposition.MiniBatchNMF.inverse_transform` is renamed to `Xt` and + will be removed in v1.5. :pr:`26503` by `Adrin Jalali`_. + +:mod:`sklearn.discriminant_analysis` +.................................... + +- |Enhancement| :class:`discriminant_analysis.LinearDiscriminantAnalysis` now + supports the `PyTorch `__. See + :ref:`array_api` for more details. :pr:`25956` by `Thomas Fan`_. + +:mod:`sklearn.ensemble` +....................... + +- |Feature| :class:`ensemble.HistGradientBoostingRegressor` now supports + the Gamma deviance loss via `loss="gamma"`. + Using the Gamma deviance as loss function comes in handy for modelling skewed + distributed, strictly positive valued targets. + :pr:`22409` by :user:`Christian Lorentzen `. + +- |Feature| Compute a custom out-of-bag score by passing a callable to + :class:`ensemble.RandomForestClassifier`, :class:`ensemble.RandomForestRegressor`, + :class:`ensemble.ExtraTreesClassifier` and :class:`ensemble.ExtraTreesRegressor`. + :pr:`25177` by `Tim Head`_. + +- |Feature| :class:`ensemble.GradientBoostingClassifier` now exposes + out-of-bag scores via the `oob_scores_` or `oob_score_` attributes. + :pr:`24882` by :user:`Ashwin Mathur `. + +- |Efficiency| :class:`ensemble.IsolationForest` predict time is now faster + (typically by a factor of 8 or more). Internally, the estimator now precomputes + decision path lengths per tree at `fit` time. It is therefore not possible + to load an estimator trained with scikit-learn 1.2 to make it predict with + scikit-learn 1.3: retraining with scikit-learn 1.3 is required. + :pr:`25186` by :user:`Felipe Breve Siola `. + +- |Efficiency| :class:`ensemble.RandomForestClassifier` and + :class:`ensemble.RandomForestRegressor` with `warm_start=True` now only + recomputes out-of-bag scores when there are actually more `n_estimators` + in subsequent `fit` calls. + :pr:`26318` by :user:`Joshua Choo Yun Keat `. + +- |Enhancement| :class:`ensemble.BaggingClassifier` and + :class:`ensemble.BaggingRegressor` expose the `allow_nan` tag from the + underlying estimator. :pr:`25506` by `Thomas Fan`_. + +- |Fix| :meth:`ensemble.RandomForestClassifier.fit` sets `max_samples = 1` + when `max_samples` is a float and `round(n_samples * max_samples) < 1`. + :pr:`25601` by :user:`Jan Fidor `. + +- |Fix| :meth:`ensemble.IsolationForest.fit` no longer warns about missing + feature names when called with `contamination` not `"auto"` on a pandas + dataframe. + :pr:`25931` by :user:`Yao Xiao `. + +- |Fix| :class:`ensemble.HistGradientBoostingRegressor` and + :class:`ensemble.HistGradientBoostingClassifier` treats negative values for + categorical features consistently as missing values, following LightGBM's and + pandas' conventions. + :pr:`25629` by `Thomas Fan`_. + +- |Fix| Fix deprecation of `base_estimator` in :class:`ensemble.AdaBoostClassifier` + and :class:`ensemble.AdaBoostRegressor` that was introduced in :pr:`23819`. + :pr:`26242` by :user:`Marko Toplak `. + +:mod:`sklearn.exceptions` +......................... + +- |Feature| Added :class:`exceptions.InconsistentVersionWarning` which is raised + when a scikit-learn estimator is unpickled with a scikit-learn version that is + inconsistent with the scikit-learn version the estimator was pickled with. + :pr:`25297` by `Thomas Fan`_. + +:mod:`sklearn.feature_extraction` +................................. + +- |API| :class:`feature_extraction.image.PatchExtractor` now follows the + transformer API of scikit-learn. This class is defined as a stateless transformer + meaning that it is not required to call `fit` before calling `transform`. + Parameter validation only happens at `fit` time. + :pr:`24230` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.feature_selection` +................................ + +- |Enhancement| All selectors in :mod:`sklearn.feature_selection` will preserve + a DataFrame's dtype when transformed. :pr:`25102` by `Thomas Fan`_. + +- |Fix| :class:`feature_selection.SequentialFeatureSelector`'s `cv` parameter + now supports generators. :pr:`25973` by `Yao Xiao `. + +:mod:`sklearn.impute` +..................... + +- |Enhancement| Added the parameter `fill_value` to :class:`impute.IterativeImputer`. + :pr:`25232` by :user:`Thijs van Weezel `. + +- |Fix| :class:`impute.IterativeImputer` now correctly preserves the Pandas + Index when the `set_config(transform_output="pandas")`. :pr:`26454` by `Thomas Fan`_. + +:mod:`sklearn.inspection` +......................... + +- |Enhancement| Added support for `sample_weight` in + :func:`inspection.partial_dependence` and + :meth:`inspection.PartialDependenceDisplay.from_estimator`. This allows for + weighted averaging when aggregating for each value of the grid we are making the + inspection on. The option is only available when `method` is set to `brute`. + :pr:`25209` and :pr:`26644` by :user:`Carlo Lemos `. + +- |API| :func:`inspection.partial_dependence` returns a :class:`utils.Bunch` with + new key: `grid_values`. The `values` key is deprecated in favor of `grid_values` + and the `values` key will be removed in 1.5. + :pr:`21809` and :pr:`25732` by `Thomas Fan`_. + +:mod:`sklearn.kernel_approximation` +................................... + +- |Fix| :class:`kernel_approximation.AdditiveChi2Sampler` is now stateless. + The `sample_interval_` attribute is deprecated and will be removed in 1.5. + :pr:`25190` by :user:`Vincent Maladière `. + +:mod:`sklearn.linear_model` +........................... + +- |Efficiency| Avoid data scaling when `sample_weight=None` and other + unnecessary data copies and unexpected dense to sparse data conversion in + :class:`linear_model.LinearRegression`. + :pr:`26207` by :user:`Olivier Grisel `. + +- |Enhancement| :class:`linear_model.SGDClassifier`, + :class:`linear_model.SGDRegressor` and :class:`linear_model.SGDOneClassSVM` + now preserve dtype for `numpy.float32`. + :pr:`25587` by :user:`Omar Salman `. + +- |Enhancement| The `n_iter_` attribute has been included in + :class:`linear_model.ARDRegression` to expose the actual number of iterations + required to reach the stopping criterion. + :pr:`25697` by :user:`John Pangas `. + +- |Fix| Use a more robust criterion to detect convergence of + :class:`linear_model.LogisticRegression` with `penalty="l1"` and `solver="liblinear"` + on linearly separable problems. + :pr:`25214` by `Tom Dupre la Tour`_. + +- |Fix| Fix a crash when calling `fit` on + :class:`linear_model.LogisticRegression` with `solver="newton-cholesky"` and + `max_iter=0` which failed to inspect the state of the model prior to the + first parameter update. + :pr:`26653` by :user:`Olivier Grisel `. + +- |API| Deprecates `n_iter` in favor of `max_iter` in + :class:`linear_model.BayesianRidge` and :class:`linear_model.ARDRegression`. + `n_iter` will be removed in scikit-learn 1.5. This change makes those + estimators consistent with the rest of estimators. + :pr:`25697` by :user:`John Pangas `. + +:mod:`sklearn.manifold` +....................... + +- |Fix| :class:`manifold.Isomap` now correctly preserves the Pandas + Index when the `set_config(transform_output="pandas")`. :pr:`26454` by `Thomas Fan`_. + +:mod:`sklearn.metrics` +...................... + +- |Feature| Adds `zero_division=np.nan` to multiple classification metrics: + :func:`metrics.precision_score`, :func:`metrics.recall_score`, + :func:`metrics.f1_score`, :func:`metrics.fbeta_score`, + :func:`metrics.precision_recall_fscore_support`, + :func:`metrics.classification_report`. When `zero_division=np.nan` and there is a + zero division, the metric is undefined and is excluded from averaging. When not used + for averages, the value returned is `np.nan`. + :pr:`25531` by :user:`Marc Torrellas Socastro `. + +- |Feature| :func:`metrics.average_precision_score` now supports the + multiclass case. + :pr:`17388` by :user:`Geoffrey Bolmier ` and + :pr:`24769` by :user:`Ashwin Mathur `. + +- |Efficiency| The computation of the expected mutual information in + :func:`metrics.adjusted_mutual_info_score` is now faster when the number of + unique labels is large and its memory usage is reduced in general. + :pr:`25713` by :user:`Kshitij Mathur `, + :user:`Guillaume Lemaitre `, :user:`Omar Salman ` and + :user:`JÊrÊmie du Boisberranger `. + +- |Enhancement| :class:`metrics.silhouette_samples` now accepts a sparse + matrix of pairwise distances between samples, or a feature array. + :pr:`18723` by :user:`Sahil Gupta ` and + :pr:`24677` by :user:`Ashwin Mathur `. + +- |Enhancement| A new parameter `drop_intermediate` was added to + :func:`metrics.precision_recall_curve`, + :func:`metrics.PrecisionRecallDisplay.from_estimator`, + :func:`metrics.PrecisionRecallDisplay.from_predictions`, + which drops some suboptimal thresholds to create lighter precision-recall + curves. + :pr:`24668` by :user:`dberenbaum`. + +- |Enhancement| :meth:`metrics.RocCurveDisplay.from_estimator` and + :meth:`metrics.RocCurveDisplay.from_predictions` now accept two new keywords, + `plot_chance_level` and `chance_level_kw` to plot the baseline chance + level. This line is exposed in the `chance_level_` attribute. + :pr:`25987` by :user:`Yao Xiao `. + +- |Enhancement| :meth:`metrics.PrecisionRecallDisplay.from_estimator` and + :meth:`metrics.PrecisionRecallDisplay.from_predictions` now accept two new + keywords, `plot_chance_level` and `chance_level_kw` to plot the baseline + chance level. This line is exposed in the `chance_level_` attribute. + :pr:`26019` by :user:`Yao Xiao `. + +- |Fix| :func:`metrics.pairwise.manhattan_distances` now supports readonly sparse datasets. + :pr:`25432` by :user:`Julien Jerphanion `. + +- |Fix| Fixed :func:`metrics.classification_report` so that empty input will return + `np.nan`. Previously, "macro avg" and `weighted avg` would return + e.g. `f1-score=np.nan` and `f1-score=0.0`, being inconsistent. Now, they + both return `np.nan`. + :pr:`25531` by :user:`Marc Torrellas Socastro `. + +- |Fix| :func:`metrics.ndcg_score` now gives a meaningful error message for input of + length 1. + :pr:`25672` by :user:`Lene Preuss ` and :user:`Wei-Chun Chu `. + +- |Fix| :func:`metrics.log_loss` raises a warning if the values of the parameter + `y_pred` are not normalized, instead of actually normalizing them in the metric. + Starting from 1.5 this will raise an error. + :pr:`25299` by :user:`Omar Salman `. + +- |Fix| The `'matching'` metric has been removed when using SciPy>=1.9 + to be consistent with `scipy.spatial.distance` which does not support + `'matching'` anymore. + :pr:`26264` by :user:`Barata T. Onggo ` + +- |API| The `eps` parameter of the :func:`metrics.log_loss` has been deprecated and + will be removed in 1.5. :pr:`25299` by :user:`Omar Salman `. + +:mod:`sklearn.gaussian_process` +............................... + +- |Fix| :class:`gaussian_process.GaussianProcessRegressor` has a new argument + `n_targets`, which is used to decide the number of outputs when sampling + from the prior distributions. :pr:`23099` by :user:`Zhehao Liu `. + +:mod:`sklearn.mixture` +...................... + +- |Efficiency| :class:`mixture.GaussianMixture` is more efficient now and will bypass + unnecessary initialization if the weights, means, and precisions are + given by users. + :pr:`26021` by :user:`Jiawei Zhang `. + +:mod:`sklearn.model_selection` +.............................. + +- |MajorFeature| Added the class :class:`model_selection.ValidationCurveDisplay` + that allows easy plotting of validation curves obtained by the function + :func:`model_selection.validation_curve`. + :pr:`25120` by :user:`Guillaume Lemaitre `. + +- |API| The parameter `log_scale` in the class + :class:`model_selection.LearningCurveDisplay` has been deprecated in 1.3 and + will be removed in 1.5. The default scale can be overridden by setting it + directly on the `ax` object and will be set automatically from the spacing + of the data points otherwise. + :pr:`25120` by :user:`Guillaume Lemaitre `. + +- |Enhancement| :func:`model_selection.cross_validate` accepts a new parameter + `return_indices` to return the train-test indices of each cv split. + :pr:`25659` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.multioutput` +.......................... + +- |Fix| :func:`getattr` on :meth:`multioutput.MultiOutputRegressor.partial_fit` + and :meth:`multioutput.MultiOutputClassifier.partial_fit` now correctly raise + an `AttributeError` if done before calling `fit`. :pr:`26333` by `Adrin + Jalali`_. + +:mod:`sklearn.naive_bayes` +.......................... + +- |Fix| :class:`naive_bayes.GaussianNB` does not raise anymore a `ZeroDivisionError` + when the provided `sample_weight` reduces the problem to a single class in `fit`. + :pr:`24140` by :user:`Jonathan Ohayon ` and :user:`Chiara Marmo `. + +:mod:`sklearn.neighbors` +........................ + +- |Enhancement| The performance of :meth:`neighbors.KNeighborsClassifier.predict` + and of :meth:`neighbors.KNeighborsClassifier.predict_proba` has been improved + when `n_neighbors` is large and `algorithm="brute"` with non Euclidean metrics. + :pr:`24076` by :user:`Meekail Zain `, :user:`Julien Jerphanion `. + +- |Fix| Remove support for `KulsinskiDistance` in :class:`neighbors.BallTree`. This + dissimilarity is not a metric and cannot be supported by the BallTree. + :pr:`25417` by :user:`Guillaume Lemaitre `. + +- |API| The support for metrics other than `euclidean` and `manhattan` and for + callables in :class:`neighbors.NearestNeighbors` is deprecated and will be removed in + version 1.5. :pr:`24083` by :user:`Valentin Laurent `. + +:mod:`sklearn.neural_network` +............................. + +- |Fix| :class:`neural_network.MLPRegressor` and :class:`neural_network.MLPClassifier` + reports the right `n_iter_` when `warm_start=True`. It corresponds to the number + of iterations performed on the current call to `fit` instead of the total number + of iterations performed since the initialization of the estimator. + :pr:`25443` by :user:`Marvin Krawutschke `. + +:mod:`sklearn.pipeline` +....................... + +- |Feature| :class:`pipeline.FeatureUnion` can now use indexing notation (e.g. + `feature_union["scalar"]`) to access transformers by name. :pr:`25093` by + `Thomas Fan`_. + +- |Feature| :class:`pipeline.FeatureUnion` can now access the + `feature_names_in_` attribute if the `X` value seen during `.fit` has a + `columns` attribute and all columns are strings. e.g. when `X` is a + `pandas.DataFrame` + :pr:`25220` by :user:`Ian Thompson `. + +- |Fix| :meth:`pipeline.Pipeline.fit_transform` now raises an `AttributeError` + if the last step of the pipeline does not support `fit_transform`. + :pr:`26325` by `Adrin Jalali`_. + +:mod:`sklearn.preprocessing` +............................ + +- |MajorFeature| Introduces :class:`preprocessing.TargetEncoder` which is a + categorical encoding based on target mean conditioned on the value of the + category. :pr:`25334` by `Thomas Fan`_. + +- |Feature| :class:`preprocessing.OrdinalEncoder` now supports grouping + infrequent categories into a single feature. Grouping infrequent categories + is enabled by specifying how to select infrequent categories with + `min_frequency` or `max_categories`. :pr:`25677` by `Thomas Fan`_. + +- |Enhancement| :class:`preprocessing.PolynomialFeatures` now calculates the + number of expanded terms a-priori when dealing with sparse `csr` matrices + in order to optimize the choice of `dtype` for `indices` and `indptr`. It + can now output `csr` matrices with `np.int32` `indices/indptr` components + when there are few enough elements, and will automatically use `np.int64` + for sufficiently large matrices. + :pr:`20524` by :user:`niuk-a ` and + :pr:`23731` by :user:`Meekail Zain ` + +- |Enhancement| A new parameter `sparse_output` was added to + :class:`preprocessing.SplineTransformer`, available as of SciPy 1.8. If + `sparse_output=True`, :class:`preprocessing.SplineTransformer` returns a sparse + CSR matrix. :pr:`24145` by :user:`Christian Lorentzen `. + +- |Enhancement| Adds a `feature_name_combiner` parameter to + :class:`preprocessing.OneHotEncoder`. This specifies a custom callable to + create feature names to be returned by + :meth:`preprocessing.OneHotEncoder.get_feature_names_out`. The callable + combines input arguments `(input_feature, category)` to a string. + :pr:`22506` by :user:`Mario Kostelac `. + +- |Enhancement| Added support for `sample_weight` in + :class:`preprocessing.KBinsDiscretizer`. This allows specifying the parameter + `sample_weight` for each sample to be used while fitting. The option is only + available when `strategy` is set to `quantile` and `kmeans`. + :pr:`24935` by :user:`Seladus `, :user:`Guillaume Lemaitre `, and + :user:`Dea María LÊon `, :pr:`25257` by :user:`Hleb Levitski `. + +- |Enhancement| Subsampling through the `subsample` parameter can now be used in + :class:`preprocessing.KBinsDiscretizer` regardless of the strategy used. + :pr:`26424` by :user:`JÊrÊmie du Boisberranger `. + +- |Fix| :class:`preprocessing.PowerTransformer` now correctly preserves the Pandas + Index when the `set_config(transform_output="pandas")`. :pr:`26454` by `Thomas Fan`_. + +- |Fix| :class:`preprocessing.PowerTransformer` now correctly raises error when + using `method="box-cox"` on data with a constant `np.nan` column. + :pr:`26400` by :user:`Yao Xiao `. + +- |Fix| :class:`preprocessing.PowerTransformer` with `method="yeo-johnson"` now leaves + constant features unchanged instead of transforming with an arbitrary value for + the `lambdas_` fitted parameter. + :pr:`26566` by :user:`JÊrÊmie du Boisberranger `. + +- |API| The default value of the `subsample` parameter of + :class:`preprocessing.KBinsDiscretizer` will change from `None` to `200_000` in + version 1.5 when `strategy="kmeans"` or `strategy="uniform"`. + :pr:`26424` by :user:`JÊrÊmie du Boisberranger `. + +:mod:`sklearn.svm` +.................. + +- |API| `dual` parameter now accepts `auto` option for + :class:`svm.LinearSVC` and :class:`svm.LinearSVR`. + :pr:`26093` by :user:`Hleb Levitski `. + +:mod:`sklearn.tree` +................... + +- |MajorFeature| :class:`tree.DecisionTreeRegressor` and + :class:`tree.DecisionTreeClassifier` support missing values when + `splitter='best'` and criterion is `gini`, `entropy`, or `log_loss`, + for classification or `squared_error`, `friedman_mse`, or `poisson` + for regression. :pr:`23595`, :pr:`26376` by `Thomas Fan`_. + +- |Enhancement| Adds a `class_names` parameter to + :func:`tree.export_text`. This allows specifying the parameter `class_names` + for each target class in ascending numerical order. + :pr:`25387` by :user:`William M ` and :user:`crispinlogan `. + +- |Fix| :func:`tree.export_graphviz` and :func:`tree.export_text` now accepts + `feature_names` and `class_names` as array-like rather than lists. + :pr:`26289` by :user:`Yao Xiao ` + +:mod:`sklearn.utils` +.................... + +- |FIX| Fixes :func:`utils.check_array` to properly convert pandas + extension arrays. :pr:`25813` and :pr:`26106` by `Thomas Fan`_. + +- |Fix| :func:`utils.check_array` now supports pandas DataFrames with + extension arrays and object dtypes by returning an ndarray with object dtype. + :pr:`25814` by `Thomas Fan`_. + +- |API| `utils.estimator_checks.check_transformers_unfitted_stateless` has been + introduced to ensure stateless transformers don't raise `NotFittedError` + during `transform` with no prior call to `fit` or `fit_transform`. + :pr:`25190` by :user:`Vincent Maladière `. + +- |API| A `FutureWarning` is now raised when instantiating a class which inherits from + a deprecated base class (i.e. decorated by :class:`utils.deprecated`) and which + overrides the `__init__` method. + :pr:`25733` by :user:`Brigitta Sipőcz ` and + :user:`JÊrÊmie du Boisberranger `. + +:mod:`sklearn.semi_supervised` +.............................. + +- |Enhancement| :meth:`semi_supervised.LabelSpreading.fit` and + :meth:`semi_supervised.LabelPropagation.fit` now accepts sparse metrics. + :pr:`19664` by :user:`Kaushik Amar Das `. + +Miscellaneous +............. + +- |Enhancement| Replace obsolete exceptions `EnvironmentError`, `IOError` and + `WindowsError`. + :pr:`26466` by :user:`Dimitri Papadopoulos ORfanos `. + +.. rubric:: Code and documentation contributors + +Thanks to everyone who has contributed to the maintenance and improvement of +the project since version 1.2, including: + +2357juan, Abhishek Singh Kushwah, Adam Handke, Adam Kania, Adam Li, adienes, +Admir Demiraj, adoublet, Adrin Jalali, A.H.Mansouri, Ahmedbgh, Ala-Na, Alex +Buzenet, AlexL, Ali H. El-Kassas, amay, AndrÃĄs Simon, AndrÊ Pedersen, Andrew +Wang, Ankur Singh, annegnx, Ansam Zedan, Anthony22-dev, Artur Hermano, Arturo +Amor, as-90, ashah002, Ashish Dutt, Ashwin Mathur, AymericBasset, Azaria +Gebremichael, Barata Tripramudya Onggo, Benedek Harsanyi, Benjamin Bossan, +Bharat Raghunathan, Binesh Bannerjee, Boris Feld, Brendan Lu, Brevin Kunde, +cache-missing, Camille Troillard, Carla J, carlo, Carlo Lemos, c-git, Changyao +Chen, Chiara Marmo, Christian Lorentzen, Christian Veenhuis, Christine P. Chai, +crispinlogan, Da-Lan, DanGonite57, Dave Berenbaum, davidblnc, david-cortes, +Dayne, Dea María LÊon, Denis, Dimitri Papadopoulos Orfanos, Dimitris +Litsidis, Dmitry Nesterov, Dominic Fox, Dominik Prodinger, Edern, Ekaterina +Butyugina, Elabonga Atuo, Emir, farhan khan, Felipe Siola, futurewarning, Gael +Varoquaux, genvalen, Hleb Levitski, Guillaume Lemaitre, gunesbayir, Haesun +Park, hujiahong726, i-aki-y, Ian Thompson, Ido M, Ily, Irene, Jack McIvor, +jakirkham, James Dean, JanFidor, Jarrod Millman, JB Mountford, JÊrÊmie du +Boisberranger, Jessicakk0711, Jiawei Zhang, Joey Ortiz, JohnathanPi, John +Pangas, Joshua Choo Yun Keat, Joshua Hedlund, JuliaSchoepp, Julien Jerphanion, +jygerardy, ka00ri, Kaushik Amar Das, Kento Nozawa, Kian Eliasi, Kilian Kluge, +Lene Preuss, Linus, Logan Thomas, Loic Esteve, Louis Fouquet, Lucy Liu, Madhura +Jayaratne, Marc Torrellas Socastro, Maren Westermann, Mario Kostelac, Mark +Harfouche, Marko Toplak, Marvin Krawutschke, Masanori Kanazu, mathurinm, Matt +Haberland, Max Halford, maximeSaur, Maxwell Liu, m. bou, mdarii, Meekail Zain, +Mikhail Iljin, murezzda, Nawazish Alam, Nicola Fanelli, Nightwalkx, Nikolay +Petrov, Nishu Choudhary, NNLNR, npache, Olivier Grisel, Omar Salman, ouss1508, +PAB, Pandata, partev, Peter Piontek, Phil, pnucci, Pooja M, Pooja Subramaniam, +precondition, Quentin BarthÊlemy, Rafal Wojdyla, Raghuveer Bhat, Rahil Parikh, +Ralf Gommers, ram vikram singh, Rushil Desai, Sadra Barikbin, SANJAI_3, Sashka +Warner, Scott Gigante, Scott Gustafson, searchforpassion, Seoeun +Hong, Shady el Gewily, Shiva chauhan, Shogo Hida, Shreesha Kumar Bhat, sonnivs, +Sortofamudkip, Stanislav (Stanley) Modrak, Stefanie Senger, Steven Van +Vaerenbergh, Tabea Kossen, ThÊophile Baranger, Thijs van Weezel, Thomas A +Caswell, Thomas Germer, Thomas J. Fan, Tim Head, Tim P, Tom DuprÊ la Tour, +tomiock, tspeng, Valentin Laurent, Veghit, VIGNESH D, Vijeth Moudgalya, Vinayak +Mehta, Vincent M, Vincent-violet, Vyom Pathak, William M, windiana42, Xiao +Yuan, Yao Xiao, Yaroslav Halchenko, Yotam Avidar-Constantini, Yuchen Zhou, +Yusuf Raji, zeeshan lone diff --git a/doc/whats_new/v1.4.rst b/doc/whats_new/v1.4.rst new file mode 100644 index 0000000000000..3dfcde90c9e81 --- /dev/null +++ b/doc/whats_new/v1.4.rst @@ -0,0 +1,1034 @@ +.. include:: _contributors.rst + +.. currentmodule:: sklearn + +.. _release_notes_1_4: + +=========== +Version 1.4 +=========== + +For a short description of the main highlights of the release, please refer to +:ref:`sphx_glr_auto_examples_release_highlights_plot_release_highlights_1_4_0.py`. + +.. include:: changelog_legend.inc + +.. _changes_1_4_2: + +Version 1.4.2 +============= + +**April 2024** + +This release only includes support for numpy 2. + +.. _changes_1_4_1: + +Version 1.4.1 +============= + +**February 2024** + +Changed models +-------------- + +- |API| The `tree_.value` attribute in :class:`tree.DecisionTreeClassifier`, + :class:`tree.DecisionTreeRegressor`, :class:`tree.ExtraTreeClassifier` and + :class:`tree.ExtraTreeRegressor` changed from a weighted absolute count + of number of samples to a weighted fraction of the total number of samples. + :pr:`27639` by :user:`Samuel Ronsin `. + +Metadata Routing +---------------- + +- |FIX| Fix routing issue with :class:`~compose.ColumnTransformer` when used + inside another meta-estimator. + :pr:`28188` by `Adrin Jalali`_. + +- |Fix| No error is raised when no metadata is passed to a metaestimator that + includes a sub-estimator which doesn't support metadata routing. + :pr:`28256` by `Adrin Jalali`_. + +- |Fix| Fix :class:`multioutput.MultiOutputRegressor` and + :class:`multioutput.MultiOutputClassifier` to work with estimators that don't + consume any metadata when metadata routing is enabled. + :pr:`28240` by `Adrin Jalali`_. + +DataFrame Support +----------------- + +- |Enhancement| |Fix| Pandas and Polars dataframe are validated directly without + ducktyping checks. + :pr:`28195` by `Thomas Fan`_. + +Changes impacting many modules +------------------------------ + +- |Efficiency| |Fix| Partial revert of :pr:`28191` to avoid a performance regression for + estimators relying on euclidean pairwise computation with + sparse matrices. The impacted estimators are: + + - :func:`sklearn.metrics.pairwise_distances_argmin` + - :func:`sklearn.metrics.pairwise_distances_argmin_min` + - :class:`sklearn.cluster.AffinityPropagation` + - :class:`sklearn.cluster.Birch` + - :class:`sklearn.cluster.SpectralClustering` + - :class:`sklearn.neighbors.KNeighborsClassifier` + - :class:`sklearn.neighbors.KNeighborsRegressor` + - :class:`sklearn.neighbors.RadiusNeighborsClassifier` + - :class:`sklearn.neighbors.RadiusNeighborsRegressor` + - :class:`sklearn.neighbors.LocalOutlierFactor` + - :class:`sklearn.neighbors.NearestNeighbors` + - :class:`sklearn.manifold.Isomap` + - :class:`sklearn.manifold.TSNE` + - :func:`sklearn.manifold.trustworthiness` + + :pr:`28235` by :user:`Julien Jerphanion `. + +- |Fix| Fixes a bug for all scikit-learn transformers when using `set_output` with + `transform` set to `pandas` or `polars`. The bug could lead to wrong naming of the + columns of the returned dataframe. + :pr:`28262` by :user:`Guillaume Lemaitre `. + +- |Fix| When users try to use a method in :class:`~ensemble.StackingClassifier`, + :class:`~ensemble.StackingClassifier`, :class:`~ensemble.StackingClassifier`, + :class:`~feature_selection.SelectFromModel`, :class:`~feature_selection.RFE`, + :class:`~semi_supervised.SelfTrainingClassifier`, + :class:`~multiclass.OneVsOneClassifier`, :class:`~multiclass.OutputCodeClassifier` or + :class:`~multiclass.OneVsRestClassifier` that their sub-estimators don't implement, + the `AttributeError` now reraises in the traceback. + :pr:`28167` by :user:`Stefanie Senger `. + +Changelog +--------- + +:mod:`sklearn.calibration` +.......................... + +- |Fix| `calibration.CalibratedClassifierCV` supports :term:`predict_proba` with + float32 output from the inner estimator. :pr:`28247` by `Thomas Fan`_. + +:mod:`sklearn.cluster` +...................... + +- |Fix| :class:`cluster.AffinityPropagation` now avoids assigning multiple different + clusters for equal points. + :pr:`28121` by :user:`Pietro Peterlongo ` and + :user:`Yao Xiao `. + +- |Fix| Avoid infinite loop in :class:`cluster.KMeans` when the number of clusters is + larger than the number of non-duplicate samples. + :pr:`28165` by :user:`JÊrÊmie du Boisberranger `. + +:mod:`sklearn.compose` +...................... + +- |Fix| :class:`compose.ColumnTransformer` now transforms into a polars dataframe when + `verbose_feature_names_out=True` and the transformers internally used several times + the same columns. Previously, it would raise a due to duplicated column names. + :pr:`28262` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.ensemble` +....................... + +- |Fix| :class:`HistGradientBoostingClassifier` and + :class:`HistGradientBoostingRegressor` when fitted on `pandas` `DataFrame` + with extension dtypes, for example `pd.Int64Dtype` + :pr:`28385` by :user:`Loïc Estève `. + +- |Fix| Fixes error message raised by :class:`ensemble.VotingClassifier` when the + target is multilabel or multiclass-multioutput in a DataFrame format. + :pr:`27702` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.impute` +..................... + +- |Fix|: :class:`impute.SimpleImputer` now raises an error in `.fit` and + `.transform` if `fill_value` can not be cast to input value dtype with + `casting='same_kind'`. + :pr:`28365` by :user:`Leo Grinsztajn `. + +:mod:`sklearn.inspection` +......................... + +- |Fix| :func:`inspection.permutation_importance` now handles properly `sample_weight` + together with subsampling (i.e. `max_features` < 1.0). + :pr:`28184` by :user:`Michael Mayer `. + +:mod:`sklearn.linear_model` +........................... + +- |Fix| :class:`linear_model.ARDRegression` now handles pandas input types + for `predict(X, return_std=True)`. + :pr:`28377` by :user:`Eddie Bergman `. + +:mod:`sklearn.preprocessing` +............................ + +- |Fix| make :class:`preprocessing.FunctionTransformer` more lenient and overwrite + output column names with the `get_feature_names_out` in the following cases: + (i) the input and output column names remain the same (happen when using NumPy + `ufunc`); (ii) the input column names are numbers; (iii) the output will be set to + Pandas or Polars dataframe. + :pr:`28241` by :user:`Guillaume Lemaitre `. + +- |Fix| :class:`preprocessing.FunctionTransformer` now also warns when `set_output` + is called with `transform="polars"` and `func` does not return a Polars dataframe or + `feature_names_out` is not specified. + :pr:`28263` by :user:`Guillaume Lemaitre `. + +- |Fix| :class:`preprocessing.TargetEncoder` no longer fails when + `target_type="continuous"` and the input is read-only. In particular, it now + works with pandas copy-on-write mode enabled. + :pr:`28233` by :user:`John Hopfensperger `. + +:mod:`sklearn.tree` +................... + +- |Fix| :class:`tree.DecisionTreeClassifier` and + :class:`tree.DecisionTreeRegressor` are handling missing values properly. The internal + criterion was not initialized when no missing values were present in the data, leading + to potentially wrong criterion values. + :pr:`28295` by :user:`Guillaume Lemaitre ` and + :pr:`28327` by :user:`Adam Li `. + +:mod:`sklearn.utils` +.................... + +- |Enhancement| |Fix| :func:`utils.metaestimators.available_if` now reraises the error + from the `check` function as the cause of the `AttributeError`. + :pr:`28198` by `Thomas Fan`_. + +- |Fix| :func:`utils._safe_indexing` now raises a `ValueError` when `X` is a Python list + and `axis=1`, as documented in the docstring. + :pr:`28222` by :user:`Guillaume Lemaitre `. + +.. _changes_1_4: + +Version 1.4.0 +============= + +**January 2024** + +Changed models +-------------- + +The following estimators and functions, when fit with the same data and +parameters, may produce different models from the previous version. This often +occurs due to changes in the modelling logic (bug fixes or enhancements), or in +random sampling procedures. + +- |Efficiency| :class:`linear_model.LogisticRegression` and + :class:`linear_model.LogisticRegressionCV` now have much better convergence for + solvers `"lbfgs"` and `"newton-cg"`. Both solvers can now reach much higher precision + for the coefficients depending on the specified `tol`. Additionally, lbfgs can + make better use of `tol`, i.e., stop sooner or reach higher precision. + Note: The lbfgs is the default solver, so this change might affect many models. + This change also means that with this new version of scikit-learn, the resulting + coefficients `coef_` and `intercept_` of your models will change for these two + solvers (when fit on the same data again). The amount of change depends on the + specified `tol`, for small values you will get more precise results. + :pr:`26721` by :user:`Christian Lorentzen `. + +- |Fix| fixes a memory leak seen in PyPy for estimators using the Cython loss functions. + :pr:`27670` by :user:`Guillaume Lemaitre `. + +Changes impacting all modules +----------------------------- + +- |MajorFeature| Transformers now support polars output with + `set_output(transform="polars")`. + :pr:`27315` by `Thomas Fan`_. + +- |Enhancement| All estimators now recognize the column names from any dataframe + that adopts the + `DataFrame Interchange Protocol `__. + Dataframes that return a correct representation through `np.asarray(df)` is expected + to work with our estimators and functions. + :pr:`26464` by `Thomas Fan`_. + +- |Enhancement| The HTML representation of estimators now includes a link to the + documentation and is color-coded to denote whether the estimator is fitted or + not (unfitted estimators are orange, fitted estimators are blue). + :pr:`26616` by :user:`Riccardo Cappuzzo `, + :user:`Ines Ibnukhsein `, :user:`Gael Varoquaux `, + `Joel Nothman`_ and :user:`Lilian Boulard `. + +- |Fix| Fixed a bug in most estimators and functions where setting a parameter to + a large integer would cause a `TypeError`. + :pr:`26648` by :user:`Naoise Holohan `. + +Metadata Routing +---------------- + +The following models now support metadata routing in one or more of their +methods. Refer to the :ref:`Metadata Routing User Guide ` for +more details. + +- |Feature| :class:`LarsCV` and :class:`LassoLarsCV` now support metadata + routing in their `fit` method and route metadata to the CV splitter. + :pr:`27538` by :user:`Omar Salman `. + +- |Feature| :class:`multiclass.OneVsRestClassifier`, + :class:`multiclass.OneVsOneClassifier` and + :class:`multiclass.OutputCodeClassifier` now support metadata routing in + their ``fit`` and ``partial_fit``, and route metadata to the underlying + estimator's ``fit`` and ``partial_fit``. + :pr:`27308` by :user:`Stefanie Senger `. + +- |Feature| :class:`pipeline.Pipeline` now supports metadata routing according + to :ref:`metadata routing user guide `. + :pr:`26789` by `Adrin Jalali`_. + +- |Feature| :func:`~model_selection.cross_validate`, + :func:`~model_selection.cross_val_score`, and + :func:`~model_selection.cross_val_predict` now support metadata routing. The + metadata are routed to the estimator's `fit`, the scorer, and the CV + splitter's `split`. The metadata is accepted via the new `params` parameter. + `fit_params` is deprecated and will be removed in version 1.6. `groups` + parameter is also not accepted as a separate argument when metadata routing + is enabled and should be passed via the `params` parameter. + :pr:`26896` by `Adrin Jalali`_. + +- |Feature| :class:`~model_selection.GridSearchCV`, + :class:`~model_selection.RandomizedSearchCV`, + :class:`~model_selection.HalvingGridSearchCV`, and + :class:`~model_selection.HalvingRandomSearchCV` now support metadata routing + in their ``fit`` and ``score``, and route metadata to the underlying + estimator's ``fit``, the CV splitter, and the scorer. + :pr:`27058` by `Adrin Jalali`_. + +- |Feature| :class:`~compose.ColumnTransformer` now supports metadata routing + according to :ref:`metadata routing user guide `. + :pr:`27005` by `Adrin Jalali`_. + +- |Feature| :class:`linear_model.LogisticRegressionCV` now supports + metadata routing. :meth:`linear_model.LogisticRegressionCV.fit` now + accepts ``**params`` which are passed to the underlying splitter and + scorer. :meth:`linear_model.LogisticRegressionCV.score` now accepts + ``**score_params`` which are passed to the underlying scorer. + :pr:`26525` by :user:`Omar Salman `. + +- |Feature| :class:`feature_selection.SelectFromModel` now supports metadata + routing in `fit` and `partial_fit`. + :pr:`27490` by :user:`Stefanie Senger `. + +- |Feature| :class:`linear_model.OrthogonalMatchingPursuitCV` now supports + metadata routing. Its `fit` now accepts ``**fit_params``, which are passed to + the underlying splitter. + :pr:`27500` by :user:`Stefanie Senger `. + +- |Feature| :class:`ElasticNetCV`, :class:`LassoCV`, + :class:`MultiTaskElasticNetCV` and :class:`MultiTaskLassoCV` + now support metadata routing and route metadata to the CV splitter. + :pr:`27478` by :user:`Omar Salman `. + +- |Fix| All meta-estimators for which metadata routing is not yet implemented + now raise a `NotImplementedError` on `get_metadata_routing` and on `fit` if + metadata routing is enabled and any metadata is passed to them. + :pr:`27389` by `Adrin Jalali`_. + + +Support for SciPy sparse arrays +------------------------------- + +Several estimators are now supporting SciPy sparse arrays. The following functions +and classes are impacted: + +**Functions:** + +- :func:`cluster.compute_optics_graph` in :pr:`27104` by + :user:`Maren Westermann ` and in :pr:`27250` by + :user:`Yao Xiao `; +- :func:`cluster.kmeans_plusplus` in :pr:`27179` by :user:`Nurseit Kamchyev `; +- :func:`decomposition.non_negative_factorization` in :pr:`27100` by + :user:`Isaac Virshup `; +- :func:`feature_selection.f_regression` in :pr:`27239` by + :user:`Yaroslav Korobko `; +- :func:`feature_selection.r_regression` in :pr:`27239` by + :user:`Yaroslav Korobko `; +- :func:`manifold.trustworthiness` in :pr:`27250` by :user:`Yao Xiao `; +- :func:`manifold.spectral_embedding` in :pr:`27240` by :user:`Yao Xiao `; +- :func:`metrics.pairwise_distances` in :pr:`27250` by :user:`Yao Xiao `; +- :func:`metrics.pairwise_distances_chunked` in :pr:`27250` by + :user:`Yao Xiao `; +- :func:`metrics.pairwise.pairwise_kernels` in :pr:`27250` by + :user:`Yao Xiao `; +- :func:`utils.multiclass.type_of_target` in :pr:`27274` by + :user:`Yao Xiao `. + +**Classes:** + +- :class:`cluster.HDBSCAN` in :pr:`27250` by :user:`Yao Xiao `; +- :class:`cluster.KMeans` in :pr:`27179` by :user:`Nurseit Kamchyev `; +- :class:`cluster.MiniBatchKMeans` in :pr:`27179` by :user:`Nurseit Kamchyev `; +- :class:`cluster.OPTICS` in :pr:`27104` by + :user:`Maren Westermann ` and in :pr:`27250` by + :user:`Yao Xiao `; +- :class:`cluster.SpectralClustering` in :pr:`27161` by + :user:`Bharat Raghunathan `; +- :class:`decomposition.MiniBatchNMF` in :pr:`27100` by + :user:`Isaac Virshup `; +- :class:`decomposition.NMF` in :pr:`27100` by :user:`Isaac Virshup `; +- :class:`feature_extraction.text.TfidfTransformer` in :pr:`27219` by + :user:`Yao Xiao `; +- :class:`manifold.Isomap` in :pr:`27250` by :user:`Yao Xiao `; +- :class:`manifold.SpectralEmbedding` in :pr:`27240` by :user:`Yao Xiao `; +- :class:`manifold.TSNE` in :pr:`27250` by :user:`Yao Xiao `; +- :class:`impute.SimpleImputer` in :pr:`27277` by :user:`Yao Xiao `; +- :class:`impute.IterativeImputer` in :pr:`27277` by :user:`Yao Xiao `; +- :class:`impute.KNNImputer` in :pr:`27277` by :user:`Yao Xiao `; +- :class:`kernel_approximation.PolynomialCountSketch` in :pr:`27301` by + :user:`Lohit SundaramahaLingam `; +- :class:`neural_network.BernoulliRBM` in :pr:`27252` by + :user:`Yao Xiao `; +- :class:`preprocessing.PolynomialFeatures` in :pr:`27166` by + :user:`Mohit Joshi `; +- :class:`random_projection.GaussianRandomProjection` in :pr:`27314` by + :user:`Stefanie Senger `; +- :class:`random_projection.SparseRandomProjection` in :pr:`27314` by + :user:`Stefanie Senger `. + +Support for Array API +--------------------- + +Several estimators and functions support the +`Array API `_. Such changes allow for using +the estimators and functions with other libraries such as JAX, CuPy, and PyTorch. +This therefore enables some GPU-accelerated computations. + +See :ref:`array_api` for more details. + +**Functions:** + +- :func:`sklearn.metrics.accuracy_score` and :func:`sklearn.metrics.zero_one_loss` in + :pr:`27137` by :user:`Edoardo Abati `; +- :func:`sklearn.model_selection.train_test_split` in :pr:`26855` by `Tim Head`_; +- :func:`~utils.multiclass.is_multilabel` in :pr:`27601` by + :user:`Yaroslav Korobko `. + +**Classes:** + +- :class:`decomposition.PCA` for the `full` and `randomized` solvers (with QR power + iterations) in :pr:`26315`, :pr:`27098` and :pr:`27431` by + :user:`Mateusz SokÃŗł `, :user:`Olivier Grisel ` and + :user:`Edoardo Abati `; +- :class:`preprocessing.KernelCenterer` in :pr:`27556` by + :user:`Edoardo Abati `; +- :class:`preprocessing.MaxAbsScaler` in :pr:`27110` by :user:`Edoardo Abati `; +- :class:`preprocessing.MinMaxScaler` in :pr:`26243` by `Tim Head`_; +- :class:`preprocessing.Normalizer` in :pr:`27558` by :user:`Edoardo Abati `. + +Private Loss Function Module +---------------------------- + +- |FIX| The gradient computation of the binomial log loss is now numerically + more stable for very large, in absolute value, input (raw predictions). Before, it + could result in `np.nan`. Among the models that profit from this change are + :class:`ensemble.GradientBoostingClassifier`, + :class:`ensemble.HistGradientBoostingClassifier` and + :class:`linear_model.LogisticRegression`. + :pr:`28048` by :user:`Christian Lorentzen `. + +Changelog +--------- + +.. + Entries should be grouped by module (in alphabetic order) and prefixed with + one of the labels: |MajorFeature|, |Feature|, |Efficiency|, |Enhancement|, + |Fix| or |API| (see whats_new.rst for descriptions). + Entries should be ordered by those labels (e.g. |Fix| after |Efficiency|). + Changes not specific to a module should be listed under *Multiple Modules* + or *Miscellaneous*. + Entries should end with: + :pr:`123456` by :user:`Joe Bloggs `. + where 123455 is the *pull request* number, not the issue number. + + +:mod:`sklearn.base` +................... + +- |Enhancement| :meth:`base.ClusterMixin.fit_predict` and + :meth:`base.OutlierMixin.fit_predict` now accept ``**kwargs`` which are + passed to the ``fit`` method of the estimator. + :pr:`26506` by `Adrin Jalali`_. + +- |Enhancement| :meth:`base.TransformerMixin.fit_transform` and + :meth:`base.OutlierMixin.fit_predict` now raise a warning if ``transform`` / + ``predict`` consume metadata, but no custom ``fit_transform`` / ``fit_predict`` + is defined in the class inheriting from them correspondingly. + :pr:`26831` by `Adrin Jalali`_. + +- |Enhancement| :func:`base.clone` now supports `dict` as input and creates a + copy. + :pr:`26786` by `Adrin Jalali`_. + +- |API|:func:`~utils.metadata_routing.process_routing` now has a different + signature. The first two (the object and the method) are positional only, + and all metadata are passed as keyword arguments. + :pr:`26909` by `Adrin Jalali`_. + +:mod:`sklearn.calibration` +.......................... + +- |Enhancement| The internal objective and gradient of the `sigmoid` method + of :class:`calibration.CalibratedClassifierCV` have been replaced by the + private loss module. + :pr:`27185` by :user:`Omar Salman `. + +:mod:`sklearn.cluster` +...................... + +- |Fix| The `degree` parameter in the :class:`cluster.SpectralClustering` + constructor now accepts real values instead of only integral values in + accordance with the `degree` parameter of the + :class:`sklearn.metrics.pairwise.polynomial_kernel`. + :pr:`27668` by :user:`Nolan McMahon `. + +- |Fix| Fixes a bug in :class:`cluster.OPTICS` where the cluster correction based + on predecessor was not using the right indexing. It would lead to inconsistent results + dependent on the order of the data. + :pr:`26459` by :user:`Haoying Zhang ` and + :user:`Guillaume Lemaitre `. + +- |Fix| Improve error message when checking the number of connected components + in the `fit` method of :class:`cluster.HDBSCAN`. + :pr:`27678` by :user:`Ganesh Tata `. + +- |Fix| Create copy of precomputed sparse matrix within the + `fit` method of :class:`cluster.DBSCAN` to avoid in-place modification of + the sparse matrix. + :pr:`27651` by :user:`Ganesh Tata `. + +- |Fix| Raises a proper `ValueError` when `metric="precomputed"` and requested storing + centers via the parameter `store_centers`. + :pr:`27898` by :user:`Guillaume Lemaitre `. + +- |API| `kdtree` and `balltree` values are now deprecated and are renamed as + `kd_tree` and `ball_tree` respectively for the `algorithm` parameter of + :class:`cluster.HDBSCAN` ensuring consistency in naming convention. + `kdtree` and `balltree` values will be removed in 1.6. + :pr:`26744` by :user:`Shreesha Kumar Bhat `. + +- |API| The option `metric=None` in + :class:`cluster.AgglomerativeClustering` and :class:`cluster.FeatureAgglomeration` + is deprecated in version 1.4 and will be removed in version 1.6. Use the default + value instead. + :pr:`27828` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.compose` +...................... + +- |MajorFeature| Adds `polars `__ input support to + :class:`compose.ColumnTransformer` through the `DataFrame Interchange Protocol + `__. + The minimum supported version for polars is `0.19.12`. + :pr:`26683` by `Thomas Fan`_. + +- |Fix| :func:`cluster.spectral_clustering` and :class:`cluster.SpectralClustering` + now raise an explicit error message indicating that sparse matrices and arrays + with `np.int64` indices are not supported. + :pr:`27240` by :user:`Yao Xiao `. + +- |API| outputs that use pandas extension dtypes and contain `pd.NA` in + :class:`~compose.ColumnTransformer` now result in a `FutureWarning` and will + cause a `ValueError` in version 1.6, unless the output container has been + configured as "pandas" with `set_output(transform="pandas")`. Before, such + outputs resulted in numpy arrays of dtype `object` containing `pd.NA` which + could not be converted to numpy floats and caused errors when passed to other + scikit-learn estimators. + :pr:`27734` by :user:`JÊrôme Dockès `. + +:mod:`sklearn.covariance` +......................... + +- |Enhancement| Allow :func:`covariance.shrunk_covariance` to process + multiple covariance matrices at once by handling nd-arrays. + :pr:`25275` by :user:`Quentin BarthÊlemy `. + +- |API| |FIX| :class:`~compose.ColumnTransformer` now replaces `"passthrough"` + with a corresponding :class:`~preprocessing.FunctionTransformer` in the + fitted ``transformers_`` attribute. + :pr:`27204` by `Adrin Jalali`_. + +:mod:`sklearn.datasets` +....................... + +- |Enhancement| :func:`datasets.make_sparse_spd_matrix` now uses a more memory-efficient + sparse layout. It also accepts a new keyword `sparse_format` that allows + specifying the output format of the sparse matrix. By default `sparse_format=None`, + which returns a dense numpy ndarray as before. + :pr:`27438` by :user:`Yao Xiao `. + +- |Fix| :func:`datasets.dump_svmlight_file` now does not raise `ValueError` when `X` + is read-only, e.g., a `numpy.memmap` instance. + :pr:`28111` by :user:`Yao Xiao `. + +- |API| :func:`datasets.make_sparse_spd_matrix` deprecated the keyword argument ``dim`` + in favor of ``n_dim``. ``dim`` will be removed in version 1.6. + :pr:`27718` by :user:`Adam Li `. + +:mod:`sklearn.decomposition` +............................ + +- |Feature| :class:`decomposition.PCA` now supports :class:`scipy.sparse.sparray` + and :class:`scipy.sparse.spmatrix` inputs when using the `arpack` solver. + When used on sparse data like :func:`datasets.fetch_20newsgroups_vectorized` this + can lead to speed-ups of 100x (single threaded) and 70x lower memory usage. + Based on :user:`Alexander Tarashansky `'s implementation in + `scanpy `_. + :pr:`18689` by :user:`Isaac Virshup ` and + :user:`Andrey Portnoy `. + +- |Enhancement| An "auto" option was added to the `n_components` parameter of + :func:`decomposition.non_negative_factorization`, :class:`decomposition.NMF` and + :class:`decomposition.MiniBatchNMF` to automatically infer the number of components + from W or H shapes when using a custom initialization. The default value of this + parameter will change from `None` to `auto` in version 1.6. + :pr:`26634` by :user:`Alexandre Landeau ` and :user:`Alexandre Vigny `. + +- |Fix| :func:`decomposition.dict_learning_online` does not ignore anymore the parameter + `max_iter`. + :pr:`27834` by :user:`Guillaume Lemaitre `. + +- |Fix| The `degree` parameter in the :class:`decomposition.KernelPCA` + constructor now accepts real values instead of only integral values in + accordance with the `degree` parameter of the + :class:`sklearn.metrics.pairwise.polynomial_kernel`. + :pr:`27668` by :user:`Nolan McMahon `. + +- |API| The option `max_iter=None` in + :class:`decomposition.MiniBatchDictionaryLearning`, + :class:`decomposition.MiniBatchSparsePCA`, and + :func:`decomposition.dict_learning_online` is deprecated and will be removed in + version 1.6. Use the default value instead. + :pr:`27834` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.ensemble` +....................... + +- |MajorFeature| :class:`ensemble.RandomForestClassifier` and + :class:`ensemble.RandomForestRegressor` support missing values when + the criterion is `gini`, `entropy`, or `log_loss`, + for classification or `squared_error`, `friedman_mse`, or `poisson` + for regression. + :pr:`26391` by `Thomas Fan`_. + +- |MajorFeature| :class:`ensemble.HistGradientBoostingClassifier` and + :class:`ensemble.HistGradientBoostingRegressor` support + `categorical_features="from_dtype"`, which treats columns with Pandas or + Polars Categorical dtype as categories in the algorithm. + `categorical_features="from_dtype"` will become the default in v1.6. + Categorical features no longer need to be encoded with numbers. When + categorical features are numbers, the maximum value no longer needs to be + smaller than `max_bins`; only the number of (unique) categories must be + smaller than `max_bins`. + :pr:`26411` by `Thomas Fan`_ and :pr:`27835` by :user:`JÊrôme Dockès `. + +- |MajorFeature| :class:`ensemble.HistGradientBoostingClassifier` and + :class:`ensemble.HistGradientBoostingRegressor` got the new parameter + `max_features` to specify the proportion of randomly chosen features considered + in each split. + :pr:`27139` by :user:`Christian Lorentzen `. + +- |Feature| :class:`ensemble.RandomForestClassifier`, + :class:`ensemble.RandomForestRegressor`, :class:`ensemble.ExtraTreesClassifier` + and :class:`ensemble.ExtraTreesRegressor` now support monotonic constraints, + useful when features are supposed to have a positive/negative effect on the target. + Missing values in the train data and multi-output targets are not supported. + :pr:`13649` by :user:`Samuel Ronsin `, + initiated by :user:`Patrick O'Reilly `. + +- |Efficiency| :class:`ensemble.HistGradientBoostingClassifier` and + :class:`ensemble.HistGradientBoostingRegressor` are now a bit faster by reusing + the parent node's histogram as children node's histogram in the subtraction trick. + In effect, less memory has to be allocated and deallocated. + :pr:`27865` by :user:`Christian Lorentzen `. + +- |Efficiency| :class:`ensemble.GradientBoostingClassifier` is faster, + for binary and in particular for multiclass problems thanks to the private loss + function module. + :pr:`26278` and :pr:`28095` by :user:`Christian Lorentzen `. + +- |Efficiency| Improves runtime and memory usage for + :class:`ensemble.GradientBoostingClassifier` and + :class:`ensemble.GradientBoostingRegressor` when trained on sparse data. + :pr:`26957` by `Thomas Fan`_. + +- |Efficiency| :class:`ensemble.HistGradientBoostingClassifier` and + :class:`ensemble.HistGradientBoostingRegressor` is now faster when `scoring` + is a predefined metric listed in :func:`metrics.get_scorer_names` and + early stopping is enabled. + :pr:`26163` by `Thomas Fan`_. + +- |Enhancement| A fitted property, ``estimators_samples_``, was added to all Forest + methods, including + :class:`ensemble.RandomForestClassifier`, :class:`ensemble.RandomForestRegressor`, + :class:`ensemble.ExtraTreesClassifier` and :class:`ensemble.ExtraTreesRegressor`, + which allows to retrieve the training sample indices used for each tree estimator. + :pr:`26736` by :user:`Adam Li `. + +- |Fix| Fixes :class:`ensemble.IsolationForest` when the input is a sparse matrix and + `contamination` is set to a float value. + :pr:`27645` by :user:`Guillaume Lemaitre `. + +- |Fix| Raises a `ValueError` in :class:`ensemble.RandomForestRegressor` and + :class:`ensemble.ExtraTreesRegressor` when requesting OOB score with multioutput model + for the targets being all rounded to integer. It was recognized as a multiclass + problem. + :pr:`27817` by :user:`Daniele Ongari ` + +- |Fix| Changes estimator tags to acknowledge that + :class:`ensemble.VotingClassifier`, :class:`ensemble.VotingRegressor`, + :class:`ensemble.StackingClassifier`, :class:`ensemble.StackingRegressor`, + support missing values if all `estimators` support missing values. + :pr:`27710` by :user:`Guillaume Lemaitre `. + +- |Fix| Support loading pickles of :class:`ensemble.HistGradientBoostingClassifier` and + :class:`ensemble.HistGradientBoostingRegressor` when the pickle has + been generated on a platform with a different bitness. A typical example is + to train and pickle the model on 64 bit machine and load the model on a 32 + bit machine for prediction. + :pr:`28074` by :user:`Christian Lorentzen ` and + :user:`Loïc Estève `. + +- |API| In :class:`ensemble.AdaBoostClassifier`, the `algorithm` argument `SAMME.R` was + deprecated and will be removed in 1.6. + :pr:`26830` by :user:`Stefanie Senger `. + +:mod:`sklearn.feature_extraction` +................................. + +- |API| Changed error type from :class:`AttributeError` to + :class:`exceptions.NotFittedError` in unfitted instances of + :class:`feature_extraction.DictVectorizer` for the following methods: + :func:`feature_extraction.DictVectorizer.inverse_transform`, + :func:`feature_extraction.DictVectorizer.restrict`, + :func:`feature_extraction.DictVectorizer.transform`. + :pr:`24838` by :user:`Lorenz Hertel `. + +:mod:`sklearn.feature_selection` +................................ + +- |Enhancement| :class:`feature_selection.SelectKBest`, + :class:`feature_selection.SelectPercentile`, and + :class:`feature_selection.GenericUnivariateSelect` now support unsupervised + feature selection by providing a `score_func` taking `X` and `y=None`. + :pr:`27721` by :user:`Guillaume Lemaitre `. + +- |Enhancement| :class:`feature_selection.SelectKBest` and + :class:`feature_selection.GenericUnivariateSelect` with `mode='k_best'` + now shows a warning when `k` is greater than the number of features. + :pr:`27841` by `Thomas Fan`_. + +- |Fix| :class:`feature_selection.RFE` and :class:`feature_selection.RFECV` do + not check for nans during input validation. + :pr:`21807` by `Thomas Fan`_. + +:mod:`sklearn.inspection` +......................... + +- |Enhancement| :class:`inspection.DecisionBoundaryDisplay` now accepts a parameter + `class_of_interest` to select the class of interest when plotting the response + provided by `response_method="predict_proba"` or + `response_method="decision_function"`. It allows to plot the decision boundary for + both binary and multiclass classifiers. + :pr:`27291` by :user:`Guillaume Lemaitre `. + +- |Fix| :meth:`inspection.DecisionBoundaryDisplay.from_estimator` and + :class:`inspection.PartialDependenceDisplay.from_estimator` now return the correct + type for subclasses. + :pr:`27675` by :user:`John Cant `. + +- |API| :class:`inspection.DecisionBoundaryDisplay` raises an `AttributeError` instead + of a `ValueError` when an estimator does not implement the requested response method. + :pr:`27291` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.kernel_ridge` +........................... + +- |Fix| The `degree` parameter in the :class:`kernel_ridge.KernelRidge` + constructor now accepts real values instead of only integral values in + accordance with the `degree` parameter of the + :class:`sklearn.metrics.pairwise.polynomial_kernel`. + :pr:`27668` by :user:`Nolan McMahon `. + +:mod:`sklearn.linear_model` +........................... + +- |Efficiency| :class:`linear_model.LogisticRegression` and + :class:`linear_model.LogisticRegressionCV` now have much better convergence for + solvers `"lbfgs"` and `"newton-cg"`. Both solvers can now reach much higher precision + for the coefficients depending on the specified `tol`. Additionally, lbfgs can + make better use of `tol`, i.e., stop sooner or reach higher precision. This is + accomplished by better scaling of the objective function, i.e., using average per + sample losses instead of sum of per sample losses. + :pr:`26721` by :user:`Christian Lorentzen `. + +- |Efficiency| :class:`linear_model.LogisticRegression` and + :class:`linear_model.LogisticRegressionCV` with solver `"newton-cg"` can now be + considerably faster for some data and parameter settings. This is accomplished by a + better line search convergence check for negligible loss improvements that takes into + account gradient information. + :pr:`26721` by :user:`Christian Lorentzen `. + +- |Efficiency| Solver `"newton-cg"` in :class:`linear_model.LogisticRegression` and + :class:`linear_model.LogisticRegressionCV` uses a little less memory. The effect is + proportional to the number of coefficients (`n_features * n_classes`). + :pr:`27417` by :user:`Christian Lorentzen `. + +- |Fix| Ensure that the `sigma_` attribute of + :class:`linear_model.ARDRegression` and :class:`linear_model.BayesianRidge` + always has a `float32` dtype when fitted on `float32` data, even with the + type promotion rules of NumPy 2. + :pr:`27899` by :user:`Olivier Grisel `. + +- |API| The attribute `loss_function_` of :class:`linear_model.SGDClassifier` and + :class:`linear_model.SGDOneClassSVM` has been deprecated and will be removed in + version 1.6. + :pr:`27979` by :user:`Christian Lorentzen `. + +:mod:`sklearn.metrics` +...................... + +- |Efficiency| Computing pairwise distances via :class:`metrics.DistanceMetric` + for CSR x CSR, Dense x CSR, and CSR x Dense datasets is now 1.5x faster. + :pr:`26765` by :user:`Meekail Zain `. + +- |Efficiency| Computing distances via :class:`metrics.DistanceMetric` + for CSR x CSR, Dense x CSR, and CSR x Dense now uses ~50% less memory, + and outputs distances in the same dtype as the provided data. + :pr:`27006` by :user:`Meekail Zain `. + +- |Enhancement| Improve the rendering of the plot obtained with the + :class:`metrics.PrecisionRecallDisplay` and :class:`metrics.RocCurveDisplay` + classes. The x- and y-axis limits are set to [0, 1] and the aspect ratio between + both axes is set to be 1 to get a square plot. + :pr:`26366` by :user:`Mojdeh Rastgoo `. + +- |Enhancement| Added `neg_root_mean_squared_log_error_scorer` as scorer + :pr:`26734` by :user:`Alejandro Martin Gil <101AlexMartin>`. + +- |Enhancement| :func:`metrics.confusion_matrix` now warns when only one label was + found in `y_true` and `y_pred`. + :pr:`27650` by :user:`Lucy Liu `. + +- |Fix| computing pairwise distances with :func:`metrics.pairwise.euclidean_distances` + no longer raises an exception when `X` is provided as a `float64` array and + `X_norm_squared` as a `float32` array. + :pr:`27624` by :user:`JÊrôme Dockès `. + +- |Fix| :func:`f1_score` now provides correct values when handling various + cases in which division by zero occurs by using a formulation that does not + depend on the precision and recall values. + :pr:`27577` by :user:`Omar Salman ` and + :user:`Guillaume Lemaitre `. + +- |Fix| :func:`metrics.make_scorer` now raises an error when using a regressor on a + scorer requesting a non-thresholded decision function (from `decision_function` or + `predict_proba`). Such scorers are specific to classification. + :pr:`26840` by :user:`Guillaume Lemaitre `. + +- |Fix| :meth:`metrics.DetCurveDisplay.from_predictions`, + :class:`metrics.PrecisionRecallDisplay.from_predictions`, + :class:`metrics.PredictionErrorDisplay.from_predictions`, and + :class:`metrics.RocCurveDisplay.from_predictions` now return the correct type + for subclasses. + :pr:`27675` by :user:`John Cant `. + +- |API| Deprecated `needs_threshold` and `needs_proba` from :func:`metrics.make_scorer`. + These parameters will be removed in version 1.6. Instead, use `response_method` that + accepts `"predict"`, `"predict_proba"` or `"decision_function"` or a list of such + values. `needs_proba=True` is equivalent to `response_method="predict_proba"` and + `needs_threshold=True` is equivalent to + `response_method=("decision_function", "predict_proba")`. + :pr:`26840` by :user:`Guillaume Lemaitre `. + +- |API| The `squared` parameter of :func:`metrics.mean_squared_error` and + :func:`metrics.mean_squared_log_error` is deprecated and will be removed in 1.6. + Use the new functions :func:`metrics.root_mean_squared_error` and + :func:`metrics.root_mean_squared_log_error` instead. + :pr:`26734` by :user:`Alejandro Martin Gil <101AlexMartin>`. + +:mod:`sklearn.model_selection` +.............................. + +- |Enhancement| :func:`model_selection.learning_curve` raises a warning when + every cross validation fold fails. + :pr:`26299` by :user:`Rahil Parikh `. + +- |Fix| :class:`model_selection.GridSearchCV`, + :class:`model_selection.RandomizedSearchCV`, and + :class:`model_selection.HalvingGridSearchCV` now don't change the given + object in the parameter grid if it's an estimator. + :pr:`26786` by `Adrin Jalali`_. + +:mod:`sklearn.multioutput` +.......................... + +- |Enhancement| Add method `predict_log_proba` to :class:`multioutput.ClassifierChain`. + :pr:`27720` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.neighbors` +........................ + +- |Efficiency| :meth:`sklearn.neighbors.KNeighborsRegressor.predict` and + :meth:`sklearn.neighbors.KNeighborsClassifier.predict_proba` now efficiently support + pairs of dense and sparse datasets. + :pr:`27018` by :user:`Julien Jerphanion `. + +- |Efficiency| The performance of :meth:`neighbors.RadiusNeighborsClassifier.predict` + and of :meth:`neighbors.RadiusNeighborsClassifier.predict_proba` has been improved + when `radius` is large and `algorithm="brute"` with non-Euclidean metrics. + :pr:`26828` by :user:`Omar Salman `. + +- |Fix| Improve error message for :class:`neighbors.LocalOutlierFactor` + when it is invoked with `n_samples=n_neighbors`. + :pr:`23317` by :user:`Bharat Raghunathan `. + +- |Fix| :meth:`neighbors.KNeighborsClassifier.predict` and + :meth:`neighbors.KNeighborsClassifier.predict_proba` now raise an error when the + weights of all neighbors of some sample are zero. This can happen when `weights` + is a user-defined function. + :pr:`26410` by :user:`Yao Xiao `. + +- |API| :class:`neighbors.KNeighborsRegressor` now accepts + :class:`metrics.DistanceMetric` objects directly via the `metric` keyword + argument allowing for the use of accelerated third-party + :class:`metrics.DistanceMetric` objects. + :pr:`26267` by :user:`Meekail Zain `. + +:mod:`sklearn.preprocessing` +............................ + +- |Efficiency| :class:`preprocessing.OrdinalEncoder` avoids calculating + missing indices twice to improve efficiency. + :pr:`27017` by :user:`Xuefeng Xu `. + +- |Efficiency| Improves efficiency in :class:`preprocessing.OneHotEncoder` and + :class:`preprocessing.OrdinalEncoder` in checking `nan`. + :pr:`27760` by :user:`Xuefeng Xu `. + +- |Enhancement| Improves warnings in :class:`preprocessing.FunctionTransformer` when + `func` returns a pandas dataframe and the output is configured to be pandas. + :pr:`26944` by `Thomas Fan`_. + +- |Enhancement| :class:`preprocessing.TargetEncoder` now supports `target_type` + 'multiclass'. + :pr:`26674` by :user:`Lucy Liu `. + +- |Fix| :class:`preprocessing.OneHotEncoder` and :class:`preprocessing.OrdinalEncoder` + raise an exception when `nan` is a category and is not the last in the user's + provided categories. + :pr:`27309` by :user:`Xuefeng Xu `. + +- |Fix| :class:`preprocessing.OneHotEncoder` and :class:`preprocessing.OrdinalEncoder` + raise an exception if the user provided categories contain duplicates. + :pr:`27328` by :user:`Xuefeng Xu `. + +- |Fix| :class:`preprocessing.FunctionTransformer` raises an error at `transform` if + the output of `get_feature_names_out` is not consistent with the column names of the + output container if those are defined. + :pr:`27801` by :user:`Guillaume Lemaitre `. + +- |Fix| Raise a `NotFittedError` in :class:`preprocessing.OrdinalEncoder` when calling + `transform` without calling `fit` since `categories` always requires to be checked. + :pr:`27821` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.tree` +................... + +- |Feature| :class:`tree.DecisionTreeClassifier`, :class:`tree.DecisionTreeRegressor`, + :class:`tree.ExtraTreeClassifier` and :class:`tree.ExtraTreeRegressor` now support + monotonic constraints, useful when features are supposed to have a positive/negative + effect on the target. Missing values in the train data and multi-output targets are + not supported. + :pr:`13649` by :user:`Samuel Ronsin `, initiated by + :user:`Patrick O'Reilly `. + +:mod:`sklearn.utils` +.................... + +- |Enhancement| :func:`sklearn.utils.estimator_html_repr` dynamically adapts + diagram colors based on the browser's `prefers-color-scheme`, providing + improved adaptability to dark mode environments. + :pr:`26862` by :user:`Andrew Goh Yisheng <9y5>`, `Thomas Fan`_, `Adrin + Jalali`_. + +- |Enhancement| :class:`~utils.metadata_routing.MetadataRequest` and + :class:`~utils.metadata_routing.MetadataRouter` now have a ``consumes`` method + which can be used to check whether a given set of parameters would be consumed. + :pr:`26831` by `Adrin Jalali`_. + +- |Enhancement| Make :func:`sklearn.utils.check_array` attempt to output + `int32`-indexed CSR and COO arrays when converting from DIA arrays if the number of + non-zero entries is small enough. This ensures that estimators implemented in Cython + and that do not accept `int64`-indexed sparse datastucture, now consistently + accept the same sparse input formats for SciPy sparse matrices and arrays. + :pr:`27372` by :user:`Guillaume Lemaitre `. + +- |Fix| :func:`sklearn.utils.check_array` should accept both matrix and array from + the sparse SciPy module. The previous implementation would fail if `copy=True` by + calling specific NumPy `np.may_share_memory` that does not work with SciPy sparse + array and does not return the correct result for SciPy sparse matrix. + :pr:`27336` by :user:`Guillaume Lemaitre `. + +- |Fix| :func:`~utils.estimator_checks.check_estimators_pickle` with + `readonly_memmap=True` now relies on joblib's own capability to allocate + aligned memory mapped arrays when loading a serialized estimator instead of + calling a dedicated private function that would crash when OpenBLAS + misdetects the CPU architecture. + :pr:`27614` by :user:`Olivier Grisel `. + +- |Fix| Error message in :func:`~utils.check_array` when a sparse matrix was + passed but `accept_sparse` is `False` now suggests to use `.toarray()` and not + `X.toarray()`. + :pr:`27757` by :user:`Lucy Liu `. + +- |Fix| Fix the function :func:`~utils.check_array` to output the right error message + when the input is a Series instead of a DataFrame. + :pr:`28090` by :user:`Stan Furrer ` and :user:`Yao Xiao `. + +- |API| :func:`sklearn.extmath.log_logistic` is deprecated and will be removed in 1.6. + Use `-np.logaddexp(0, -x)` instead. + :pr:`27544` by :user:`Christian Lorentzen `. + +.. rubric:: Code and documentation contributors + +Thanks to everyone who has contributed to the maintenance and improvement of +the project since version 1.3, including: + +101AlexMartin, Abhishek Singh Kushwah, Adam Li, Adarsh Wase, Adrin Jalali, +Advik Sinha, Alex, Alexander Al-Feghali, Alexis IMBERT, AlexL, Alex Molas, Anam +Fatima, Andrew Goh, andyscanzio, Aniket Patil, Artem Kislovskiy, Arturo Amor, +ashah002, avm19, Ben Holmes, Ben Mares, Benoit Chevallier-Mames, Bharat +Raghunathan, Binesh Bannerjee, Brendan Lu, Brevin Kunde, Camille Troillard, +Carlo Lemos, Chad Parmet, Christian Clauss, Christian Lorentzen, Christian +Veenhuis, Christos Aridas, Cindy Liang, Claudio Salvatore Arcidiacono, Connor +Boyle, cynthias13w, DaminK, Daniele Ongari, Daniel Schmitz, Daniel Tinoco, +David Brochart, Deborah L. Haar, DevanshKyada27, Dimitri Papadopoulos Orfanos, +Dmitry Nesterov, DUONG, Edoardo Abati, Eitan Hemed, Elabonga Atuo, Elisabeth +GÃŧnther, Emma Carballal, Emmanuel Ferdman, epimorphic, Erwan Le Floch, Fabian +Egli, Filip Karlo DoÅĄilović, Florian Idelberger, Franck Charras, Gael +Varoquaux, Ganesh Tata, Hleb Levitski, Guillaume Lemaitre, Haoying Zhang, +Harmanan Kohli, Ily, ioangatop, IsaacTrost, Isaac Virshup, Iwona Zdzieblo, +Jakub Kaczmarzyk, James McDermott, Jarrod Millman, JB Mountford, JÊrÊmie du +Boisberranger, JÊrôme Dockès, Jiawei Zhang, Joel Nothman, John Cant, John +Hopfensperger, Jona Sassenhagen, Jon Nordby, Julien Jerphanion, Kennedy Waweru, +kevin moore, Kian Eliasi, Kishan Ved, Konstantinos Pitas, Koustav Ghosh, Kushan +Sharma, ldwy4, Linus, Lohit SundaramahaLingam, Loic Esteve, Lorenz, Louis +Fouquet, Lucy Liu, Luis Silvestrin, LukÃĄÅĄ FolwarcznÃŊ, Lukas Geiger, Malte +Londschien, Marcus Fraaß, Marek HanuÅĄ, Maren Westermann, Mark Elliot, Martin +Larralde, Mateusz SokÃŗł, mathurinm, mecopur, Meekail Zain, Michael Higgins, +Miki Watanabe, Milton Gomez, MN193, Mohammed Hamdy, Mohit Joshi, mrastgoo, +Naman Dhingra, Naoise Holohan, Narendra Singh dangi, Noa Malem-Shinitski, +Nolan, Nurseit Kamchyev, Oleksii Kachaiev, Olivier Grisel, Omar Salman, partev, +Peter Hull, Peter Steinbach, Pierre de FrÊminville, Pooja Subramaniam, Puneeth +K, qmarcou, Quentin BarthÊlemy, Rahil Parikh, Rahul Mahajan, Raj Pulapakura, +Raphael, Ricardo Peres, Riccardo Cappuzzo, Roman Lutz, Salim Dohri, Samuel O. +Ronsin, Sandip Dutta, Sayed Qaiser Ali, scaja, scikit-learn-bot, Sebastian +Berg, Shreesha Kumar Bhat, Shubhal Gupta, Søren Fuglede Jørgensen, Stefanie +Senger, Tamara, Tanjina Afroj, THARAK HEGDE, thebabush, Thomas J. Fan, Thomas +Roehr, Tialo, Tim Head, tongyu, Venkatachalam N, Vijeth Moudgalya, Vincent M, +Vivek Reddy P, Vladimir Fokow, Xiao Yuan, Xuefeng Xu, Yang Tao, Yao Xiao, +Yuchen Zhou, Yuusuke Hiramatsu diff --git a/doc/whats_new/v1.5.rst b/doc/whats_new/v1.5.rst new file mode 100644 index 0000000000000..411a1b6b5dd95 --- /dev/null +++ b/doc/whats_new/v1.5.rst @@ -0,0 +1,713 @@ +.. include:: _contributors.rst + +.. currentmodule:: sklearn + +.. _release_notes_1_5: + +=========== +Version 1.5 +=========== + +For a short description of the main highlights of the release, please refer to +:ref:`sphx_glr_auto_examples_release_highlights_plot_release_highlights_1_5_0.py`. + +.. include:: changelog_legend.inc + +.. _changes_1_5_2: + +Version 1.5.2 +============= + +**September 2024** + +Changes impacting many modules +------------------------------ + +- |Fix| Fixed performance regression in a few Cython modules in + `sklearn._loss`, `sklearn.manifold`, `sklearn.metrics` and `sklearn.utils`, + which were built without OpenMP support. + :pr:`29694` by :user:`Loïc Estèvce `. + +Changelog +--------- + +:mod:`sklearn.calibration` +.......................... + +- |Fix| Raise error when :class:`~sklearn.model_selection.LeaveOneOut` used in + `cv`, matching what would happen if `KFold(n_splits=n_samples)` was used. + :pr:`29545` by :user:`Lucy Liu ` + +:mod:`sklearn.compose` +...................... + +- |Fix| Fixed :class:`compose.TransformedTargetRegressor` not to raise `UserWarning` if + transform output is set to `pandas` or `polars`, since it isn't a transformer. + :pr:`29401` by :user:`Stefanie Senger `. + +:mod:`sklearn.decomposition` +............................ + +- |Fix| Increase rank deficiency threshold in the whitening step of + :class:`decomposition.FastICA` with `whiten_solver="eigh"` to improve the + platform-agnosticity of the estimator. + :pr:`29612` by :user:`Olivier Grisel `. + +:mod:`sklearn.metrics` +...................... + +- |Fix| Fix a regression in :func:`metrics.accuracy_score` and in + :func:`metrics.zero_one_loss` causing an error for Array API dispatch with multilabel + inputs. + :pr:`29336` by :user:`Edoardo Abati `. + +:mod:`sklearn.svm` +.................. + +- |Fix| Fixed a regression in :class:`svm.SVC` and :class:`svm.SVR` such that we accept + `C=float("inf")`. + :pr:`29780` by :user:`Guillaume Lemaitre `. + +.. _changes_1_5_1: + +Version 1.5.1 +============= + +**July 2024** + +Changes impacting many modules +------------------------------ + +- |Fix| Fixed a regression in the validation of the input data of all estimators where + an unexpected error was raised when passing a DataFrame backed by a read-only buffer. + :pr:`29018` by :user:`JÊrÊmie du Boisberranger `. + +- |Fix| Fixed a regression causing a dead-lock at import time in some settings. + :pr:`29235` by :user:`JÊrÊmie du Boisberranger `. + +Changelog +--------- + +:mod:`sklearn.compose` +...................... + +- |Efficiency| Fix a performance regression in :class:`compose.ColumnTransformer` + where the full input data was copied for each transformer when `n_jobs > 1`. + :pr:`29330` by :user:`JÊrÊmie du Boisberranger `. + +:mod:`sklearn.metrics` +...................... + +- |Fix| Fix a regression in :func:`metrics.r2_score`. Passing torch CPU tensors + with array API dispatched disabled would complain about non-CPU devices + instead of implicitly converting those inputs as regular NumPy arrays. + :pr:`29119` by :user:`Olivier Grisel`. + +- |Fix| Fix a regression in + :func:`metrics.zero_one_loss` causing an error for Array API dispatch with multilabel + inputs. + :pr:`29269` by :user:`Yaroslav Korobko `. + +:mod:`sklearn.model_selection` +.............................. + +- |Fix| Fix a regression in :class:`model_selection.GridSearchCV` for parameter + grids that have heterogeneous parameter values. + :pr:`29078` by :user:`Loïc Estève `. + +- |Fix| Fix a regression in :class:`model_selection.GridSearchCV` for parameter + grids that have estimators as parameter values. + :pr:`29179` by :user:`Marco Gorelli`. + +- |Fix| Fix a regression in :class:`model_selection.GridSearchCV` for parameter + grids that have arrays of different sizes as parameter values. + :pr:`29314` by :user:`Marco Gorelli`. + +:mod:`sklearn.tree` +................... + +- |Fix| Fix an issue in :func:`tree.export_graphviz` and :func:`tree.plot_tree` + that could potentially result in exception or wrong results on 32bit OSes. + :pr:`29327` by :user:`Loïc Estève`. + +:mod:`sklearn.utils` +.................... + +- |API| :func:`utils.validation.check_array` has a new parameter, `force_writeable`, to + control the writeability of the output array. If set to `True`, the output array will + be guaranteed to be writeable and a copy will be made if the input array is read-only. + If set to `False`, no guarantee is made about the writeability of the output array. + :pr:`29018` by :user:`JÊrÊmie du Boisberranger `. + +.. _changes_1_5: + +Version 1.5.0 +============= + +**May 2024** + +Security +-------- + +- |Fix| :class:`feature_extraction.text.CountVectorizer` and + :class:`feature_extraction.text.TfidfVectorizer` no longer store discarded + tokens from the training set in their `stop_words_` attribute. This attribute + would hold too frequent (above `max_df`) but also too rare tokens (below + `min_df`). This fixes a potential security issue (data leak) if the discarded + rare tokens hold sensitive information from the training set without the + model developer's knowledge. + + Note: users of those classes are encouraged to either retrain their pipelines + with the new scikit-learn version or to manually clear the `stop_words_` + attribute from previously trained instances of those transformers. This + attribute was designed only for model inspection purposes and has no impact + on the behavior of the transformers. + :pr:`28823` by :user:`Olivier Grisel `. + +Changed models +-------------- + +- |Efficiency| The subsampling in :class:`preprocessing.QuantileTransformer` is now + more efficient for dense arrays but the fitted quantiles and the results of + `transform` may be slightly different than before (keeping the same statistical + properties). + :pr:`27344` by :user:`Xuefeng Xu `. + +- |Enhancement| :class:`decomposition.PCA`, :class:`decomposition.SparsePCA` + and :class:`decomposition.TruncatedSVD` now set the sign of the `components_` + attribute based on the component values instead of using the transformed data + as reference. This change is needed to be able to offer consistent component + signs across all `PCA` solvers, including the new + `svd_solver="covariance_eigh"` option introduced in this release. + +Changes impacting many modules +------------------------------ + +- |Fix| Raise `ValueError` with an informative error message when passing 1D + sparse arrays to methods that expect 2D sparse inputs. + :pr:`28988` by :user:`Olivier Grisel `. + +- |API| The name of the input of the `inverse_transform` method of estimators has been + standardized to `X`. As a consequence, `Xt` is deprecated and will be removed in + version 1.7 in the following estimators: :class:`cluster.FeatureAgglomeration`, + :class:`decomposition.MiniBatchNMF`, :class:`decomposition.NMF`, + :class:`model_selection.GridSearchCV`, :class:`model_selection.RandomizedSearchCV`, + :class:`pipeline.Pipeline` and :class:`preprocessing.KBinsDiscretizer`. + :pr:`28756` by :user:`Will Dean `. + +Support for Array API +--------------------- + +Additional estimators and functions have been updated to include support for all +`Array API `_ compliant inputs. + +See :ref:`array_api` for more details. + +**Functions:** + +- :func:`sklearn.metrics.r2_score` now supports Array API compliant inputs. + :pr:`27904` by :user:`Eric Lindgren `, :user:`Franck Charras `, + :user:`Olivier Grisel ` and :user:`Tim Head `. + +**Classes:** + +- :class:`linear_model.Ridge` now supports the Array API for the `svd` solver. + See :ref:`array_api` for more details. + :pr:`27800` by :user:`Franck Charras `, :user:`Olivier Grisel ` + and :user:`Tim Head `. + +Support for building with Meson +------------------------------- + +From scikit-learn 1.5 onwards, Meson is the main supported way to build +scikit-learn, see :ref:`Building from source ` for more +details. + +Unless we discover a major blocker, setuptools support will be dropped in +scikit-learn 1.6. The 1.5.x releases will support building scikit-learn with +setuptools. + +Meson support for building scikit-learn was added in :pr:`28040` by +:user:`Loïc Estève ` + +Metadata Routing +---------------- + +The following models now support metadata routing in one or more of their +methods. Refer to the :ref:`Metadata Routing User Guide ` for +more details. + +- |Feature| :class:`impute.IterativeImputer` now supports metadata routing in + its `fit` method. :pr:`28187` by :user:`Stefanie Senger `. + +- |Feature| :class:`ensemble.BaggingClassifier` and :class:`ensemble.BaggingRegressor` + now support metadata routing. The fit methods now + accept ``**fit_params`` which are passed to the underlying estimators + via their `fit` methods. + :pr:`28432` by :user:`Adam Li ` and + :user:`Benjamin Bossan `. + +- |Feature| :class:`linear_model.RidgeCV` and + :class:`linear_model.RidgeClassifierCV` now support metadata routing in + their `fit` method and route metadata to the underlying + :class:`model_selection.GridSearchCV` object or the underlying scorer. + :pr:`27560` by :user:`Omar Salman `. + +- |Feature| :class:`GraphicalLassoCV` now supports metadata routing in its + `fit` method and routes metadata to the CV splitter. + :pr:`27566` by :user:`Omar Salman `. + +- |Feature| :class:`linear_model.RANSACRegressor` now supports metadata routing + in its ``fit``, ``score`` and ``predict`` methods and route metadata to its + underlying estimator's ``fit``, ``score`` and ``predict`` methods. + :pr:`28261` by :user:`Stefanie Senger `. + +- |Feature| :class:`ensemble.VotingClassifier` and + :class:`ensemble.VotingRegressor` now support metadata routing and pass + ``**fit_params`` to the underlying estimators via their `fit` methods. + :pr:`27584` by :user:`Stefanie Senger `. + +- |Feature| :class:`pipeline.FeatureUnion` now supports metadata routing in its + ``fit`` and ``fit_transform`` methods and route metadata to the underlying + transformers' ``fit`` and ``fit_transform``. + :pr:`28205` by :user:`Stefanie Senger `. + +- |Fix| Fix an issue when resolving default routing requests set via class + attributes. + :pr:`28435` by `Adrin Jalali`_. + +- |Fix| Fix an issue when `set_{method}_request` methods are used as unbound + methods, which can happen if one tries to decorate them. + :pr:`28651` by `Adrin Jalali`_. + +- |FIX| Prevent a `RecursionError` when estimators with the default `scoring` + param (`None`) route metadata. + :pr:`28712` by :user:`Stefanie Senger `. + +Changelog +--------- + +.. + Entries should be grouped by module (in alphabetic order) and prefixed with + one of the labels: |MajorFeature|, |Feature|, |Efficiency|, |Enhancement|, + |Fix| or |API| (see whats_new.rst for descriptions). + Entries should be ordered by those labels (e.g. |Fix| after |Efficiency|). + Changes not specific to a module should be listed under *Multiple Modules* + or *Miscellaneous*. + Entries should end with: + :pr:`123456` by :user:`Joe Bloggs `. + where 123455 is the *pull request* number, not the issue number. + +:mod:`sklearn.calibration` +.......................... + +- |Fix| Fixed a regression in :class:`calibration.CalibratedClassifierCV` where + an error was wrongly raised with string targets. + :pr:`28843` by :user:`JÊrÊmie du Boisberranger `. + +:mod:`sklearn.cluster` +...................... + +- |Fix| The :class:`cluster.MeanShift` class now properly converges for constant data. + :pr:`28951` by :user:`Akihiro Kuno `. + +- |FIX| Create copy of precomputed sparse matrix within the `fit` method of + :class:`~cluster.OPTICS` to avoid in-place modification of the sparse matrix. + :pr:`28491` by :user:`Thanh Lam Dang `. + +- |Fix| :class:`cluster.HDBSCAN` now supports all metrics supported by + :func:`sklearn.metrics.pairwise_distances` when `algorithm="brute"` or `"auto"`. + :pr:`28664` by :user:`Manideep Yenugula `. + +:mod:`sklearn.compose` +...................... + +- |Feature| A fitted :class:`compose.ColumnTransformer` now implements `__getitem__` + which returns the fitted transformers by name. :pr:`27990` by `Thomas Fan`_. + +- |Enhancement| :class:`compose.TransformedTargetRegressor` now raises an error in `fit` + if only `inverse_func` is provided without `func` (that would default to identity) + being explicitly set as well. + :pr:`28483` by :user:`Stefanie Senger `. + +- |Enhancement| :class:`compose.ColumnTransformer` can now expose the "remainder" + columns in the fitted `transformers_` attribute as column names or boolean + masks, rather than column indices. + :pr:`27657` by :user:`JÊrôme Dockès `. + +- |Fix| Fixed a bug in :class:`compose.ColumnTransformer` with `n_jobs > 1`, where the + intermediate selected columns were passed to the transformers as read-only arrays. + :pr:`28822` by :user:`JÊrÊmie du Boisberranger `. + +:mod:`sklearn.cross_decomposition` +.................................. + +- |Fix| The `coef_` fitted attribute of :class:`cross_decomposition.PLSRegression` + now takes into account both the scale of `X` and `Y` when `scale=True`. Note that + the previous predicted values were not affected by this bug. + :pr:`28612` by :user:`Guillaume Lemaitre `. + +- |API| Deprecates `Y` in favor of `y` in the methods fit, transform and + inverse_transform of: + :class:`cross_decomposition.PLSRegression`. + :class:`cross_decomposition.PLSCanonical`, + :class:`cross_decomposition.CCA`, + and :class:`cross_decomposition.PLSSVD`. + `Y` will be removed in version 1.7. + :pr:`28604` by :user:`David Leon `. + +:mod:`sklearn.datasets` +....................... + +- |Enhancement| Adds optional arguments `n_retries` and `delay` to functions + :func:`datasets.fetch_20newsgroups`, + :func:`datasets.fetch_20newsgroups_vectorized`, + :func:`datasets.fetch_california_housing`, + :func:`datasets.fetch_covtype`, + :func:`datasets.fetch_kddcup99`, + :func:`datasets.fetch_lfw_pairs`, + :func:`datasets.fetch_lfw_people`, + :func:`datasets.fetch_olivetti_faces`, + :func:`datasets.fetch_rcv1`, + and :func:`datasets.fetch_species_distributions`. + By default, the functions will retry up to 3 times in case of network failures. + :pr:`28160` by :user:`Zhehao Liu ` and + :user:`Filip Karlo DoÅĄilović `. + +:mod:`sklearn.decomposition` +............................ + +- |Efficiency| :class:`decomposition.PCA` with `svd_solver="full"` now assigns + a contiguous `components_` attribute instead of a non-contiguous slice of + the singular vectors. When `n_components << n_features`, this can save some + memory and, more importantly, help speed-up subsequent calls to the `transform` + method by more than an order of magnitude by leveraging cache locality of + BLAS GEMM on contiguous arrays. + :pr:`27491` by :user:`Olivier Grisel `. + +- |Enhancement| :class:`~decomposition.PCA` now automatically selects the ARPACK solver + for sparse inputs when `svd_solver="auto"` instead of raising an error. + :pr:`28498` by :user:`Thanh Lam Dang `. + +- |Enhancement| :class:`decomposition.PCA` now supports a new solver option + named `svd_solver="covariance_eigh"` which offers an order of magnitude + speed-up and reduced memory usage for datasets with a large number of data + points and a small number of features (say, `n_samples >> 1000 > + n_features`). The `svd_solver="auto"` option has been updated to use the new + solver automatically for such datasets. This solver also accepts sparse input + data. + :pr:`27491` by :user:`Olivier Grisel `. + +- |Fix| :class:`decomposition.PCA` fit with `svd_solver="arpack"`, + `whiten=True` and a value for `n_components` that is larger than the rank of + the training set, no longer returns infinite values when transforming + hold-out data. + :pr:`27491` by :user:`Olivier Grisel `. + +:mod:`sklearn.dummy` +.................... + +- |Enhancement| :class:`dummy.DummyClassifier` and :class:`dummy.DummyRegressor` now + have the `n_features_in_` and `feature_names_in_` attributes after `fit`. + :pr:`27937` by :user:`Marco vd Boom `. + +:mod:`sklearn.ensemble` +....................... + +- |Efficiency| Improves runtime of `predict` of + :class:`ensemble.HistGradientBoostingClassifier` by avoiding to call `predict_proba`. + :pr:`27844` by :user:`Christian Lorentzen `. + +- |Efficiency| :class:`ensemble.HistGradientBoostingClassifier` and + :class:`ensemble.HistGradientBoostingRegressor` are now a tiny bit faster by + pre-sorting the data before finding the thresholds for binning. + :pr:`28102` by :user:`Christian Lorentzen `. + +- |Fix| Fixes a bug in :class:`ensemble.HistGradientBoostingClassifier` and + :class:`ensemble.HistGradientBoostingRegressor` when `monotonic_cst` is specified + for non-categorical features. + :pr:`28925` by :user:`Xiao Yuan `. + +:mod:`sklearn.feature_extraction` +................................. + +- |Efficiency| :class:`feature_extraction.text.TfidfTransformer` is now faster + and more memory-efficient by using a NumPy vector instead of a sparse matrix + for storing the inverse document frequency. + :pr:`18843` by :user:`Paolo Montesel `. + +- |Enhancement| :class:`feature_extraction.text.TfidfTransformer` now preserves + the data type of the input matrix if it is `np.float64` or `np.float32`. + :pr:`28136` by :user:`Guillaume Lemaitre `. + +:mod:`sklearn.feature_selection` +................................ + +- |Enhancement| :func:`feature_selection.mutual_info_regression` and + :func:`feature_selection.mutual_info_classif` now support `n_jobs` parameter. + :pr:`28085` by :user:`Neto Menoci ` and + :user:`Florin Andrei `. + +- |Enhancement| The `cv_results_` attribute of :class:`feature_selection.RFECV` has + a new key, `n_features`, containing an array with the number of features selected + at each step. + :pr:`28670` by :user:`Miguel Silva `. + +:mod:`sklearn.impute` +..................... + +- |Enhancement| :class:`impute.SimpleImputer` now supports custom strategies + by passing a function in place of a strategy name. + :pr:`28053` by :user:`Mark Elliot `. + +:mod:`sklearn.inspection` +......................... + +- |Fix| :meth:`inspection.DecisionBoundaryDisplay.from_estimator` no longer + warns about missing feature names when provided a `polars.DataFrame`. + :pr:`28718` by :user:`Patrick Wang `. + +:mod:`sklearn.linear_model` +........................... + +- |Enhancement| Solver `"newton-cg"` in :class:`linear_model.LogisticRegression` and + :class:`linear_model.LogisticRegressionCV` now emits information when `verbose` is + set to positive values. + :pr:`27526` by :user:`Christian Lorentzen `. + +- |Fix| :class:`linear_model.ElasticNet`, :class:`linear_model.ElasticNetCV`, + :class:`linear_model.Lasso` and :class:`linear_model.LassoCV` now explicitly don't + accept large sparse data formats. + :pr:`27576` by :user:`Stefanie Senger `. + +- |Fix| :class:`linear_model.RidgeCV` and :class:`RidgeClassifierCV` correctly pass + `sample_weight` to the underlying scorer when `cv` is None. + :pr:`27560` by :user:`Omar Salman `. + +- |Fix| `n_nonzero_coefs_` attribute in :class:`linear_model.OrthogonalMatchingPursuit` + will now always be `None` when `tol` is set, as `n_nonzero_coefs` is ignored in + this case. :pr:`28557` by :user:`Lucy Liu `. + +- |API| :class:`linear_model.RidgeCV` and :class:`linear_model.RidgeClassifierCV` + will now allow `alpha=0` when `cv != None`, which is consistent with + :class:`linear_model.Ridge` and :class:`linear_model.RidgeClassifier`. + :pr:`28425` by :user:`Lucy Liu `. + +- |API| Passing `average=0` to disable averaging is deprecated in + :class:`linear_model.PassiveAggressiveClassifier`, + :class:`linear_model.PassiveAggressiveRegressor`, + :class:`linear_model.SGDClassifier`, :class:`linear_model.SGDRegressor` and + :class:`linear_model.SGDOneClassSVM`. Pass `average=False` instead. + :pr:`28582` by :user:`JÊrÊmie du Boisberranger `. + +- |API| Parameter `multi_class` was deprecated in + :class:`linear_model.LogisticRegression` and + :class:`linear_model.LogisticRegressionCV`. `multi_class` will be removed in 1.7, + and internally, for 3 and more classes, it will always use multinomial. + If you still want to use the one-vs-rest scheme, you can use + `OneVsRestClassifier(LogisticRegression(..))`. + :pr:`28703` by :user:`Christian Lorentzen `. + +- |API| `store_cv_values` and `cv_values_` are deprecated in favor of + `store_cv_results` and `cv_results_` in `~linear_model.RidgeCV` and + `~linear_model.RidgeClassifierCV`. + :pr:`28915` by :user:`Lucy Liu `. + +:mod:`sklearn.manifold` +....................... + +- |API| Deprecates `n_iter` in favor of `max_iter` in :class:`manifold.TSNE`. + `n_iter` will be removed in version 1.7. This makes :class:`manifold.TSNE` + consistent with the rest of the estimators. :pr:`28471` by + :user:`Lucy Liu ` + +:mod:`sklearn.metrics` +...................... + +- |Feature| :func:`metrics.pairwise_distances` accepts calculating pairwise distances + for non-numeric arrays as well. This is supported through custom metrics only. + :pr:`27456` by :user:`Venkatachalam N `, :user:`Kshitij Mathur ` + and :user:`Julian Libiseller-Egger `. + +- |Feature| :func:`sklearn.metrics.check_scoring` now returns a multi-metric scorer + when `scoring` as a `dict`, `set`, `tuple`, or `list`. :pr:`28360` by `Thomas Fan`_. + +- |Feature| :func:`metrics.d2_log_loss_score` has been added which + calculates the D^2 score for the log loss. + :pr:`28351` by :user:`Omar Salman `. + +- |Efficiency| Improve efficiency of functions :func:`~metrics.brier_score_loss`, + :func:`~calibration.calibration_curve`, :func:`~metrics.det_curve`, + :func:`~metrics.precision_recall_curve`, + :func:`~metrics.roc_curve` when `pos_label` argument is specified. + Also improve efficiency of methods `from_estimator` + and `from_predictions` in :class:`~metrics.RocCurveDisplay`, + :class:`~metrics.PrecisionRecallDisplay`, :class:`~metrics.DetCurveDisplay`, + :class:`~calibration.CalibrationDisplay`. + :pr:`28051` by :user:`Pierre de FrÊminville `. + +- |Fix|:class:`metrics.classification_report` now shows only accuracy and not + micro-average when input is a subset of labels. + :pr:`28399` by :user:`Vineet Joshi `. + +- |Fix| Fix OpenBLAS 0.3.26 dead-lock on Windows in pairwise distances + computation. This is likely to affect neighbor-based algorithms. + :pr:`28692` by :user:`Loïc Estève `. + +- |API| :func:`metrics.precision_recall_curve` deprecated the keyword argument + `probas_pred` in favor of `y_score`. `probas_pred` will be removed in version 1.7. + :pr:`28092` by :user:`Adam Li `. + +- |API| :func:`metrics.brier_score_loss` deprecated the keyword argument `y_prob` + in favor of `y_proba`. `y_prob` will be removed in version 1.7. + :pr:`28092` by :user:`Adam Li `. + +- |API| For classifiers and classification metrics, labels encoded as bytes + is deprecated and will raise an error in v1.7. + :pr:`18555` by :user:`Kaushik Amar Das `. + +:mod:`sklearn.mixture` +...................... + +- |Fix| The `converged_` attribute of :class:`mixture.GaussianMixture` and + :class:`mixture.BayesianGaussianMixture` now reflects the convergence status of + the best fit whereas it was previously `True` if any of the fits converged. + :pr:`26837` by :user:`Krsto Proroković `. + +:mod:`sklearn.model_selection` +.............................. + +- |MajorFeature| :class:`model_selection.TunedThresholdClassifierCV` finds + the decision threshold of a binary classifier that maximizes a + classification metric through cross-validation. + :class:`model_selection.FixedThresholdClassifier` is an alternative when one wants + to use a fixed decision threshold without any tuning scheme. + :pr:`26120` by :user:`Guillaume Lemaitre `. + +- |Enhancement| :term:`CV splitters ` that ignores the group parameter now + raises a warning when groups are passed in to :term:`split`. :pr:`28210` by + `Thomas Fan`_. + +- |Enhancement| The HTML diagram representation of + :class:`~model_selection.GridSearchCV`, + :class:`~model_selection.RandomizedSearchCV`, + :class:`~model_selection.HalvingGridSearchCV`, and + :class:`~model_selection.HalvingRandomSearchCV` will show the best estimator when + `refit=True`. :pr:`28722` by :user:`Yao Xiao ` and `Thomas Fan`_. + +- |Fix| the ``cv_results_`` attribute (of :class:`model_selection.GridSearchCV`) now + returns masked arrays of the appropriate NumPy dtype, as opposed to always returning + dtype ``object``. :pr:`28352` by :user:`Marco Gorelli`. + +- |Fix| :func:`model_selection.train_test_split` works with Array API inputs. + Previously indexing was not handled correctly leading to exceptions when using strict + implementations of the Array API like CuPY. + :pr:`28407` by :user:`Tim Head `. + +:mod:`sklearn.multioutput` +.......................... + +- |Enhancement| `chain_method` parameter added to :class:`multioutput.ClassifierChain`. + :pr:`27700` by :user:`Lucy Liu `. + +:mod:`sklearn.neighbors` +........................ + +- |Fix| Fixes :class:`neighbors.NeighborhoodComponentsAnalysis` such that + `get_feature_names_out` returns the correct number of feature names. + :pr:`28306` by :user:`Brendan Lu `. + +:mod:`sklearn.pipeline` +....................... + +- |Feature| :class:`pipeline.FeatureUnion` can now use the + `verbose_feature_names_out` attribute. If `True`, `get_feature_names_out` + will prefix all feature names with the name of the transformer + that generated that feature. If `False`, `get_feature_names_out` will not + prefix any feature names and will error if feature names are not unique. + :pr:`25991` by :user:`Jiawei Zhang `. + +:mod:`sklearn.preprocessing` +............................ + +- |Enhancement| :class:`preprocessing.QuantileTransformer` and + :func:`preprocessing.quantile_transform` now supports disabling + subsampling explicitly. + :pr:`27636` by :user:`Ralph Urlus `. + +:mod:`sklearn.tree` +................... + +- |Enhancement| Plotting trees in matplotlib via :func:`tree.plot_tree` now + show a "True/False" label to indicate the directionality the samples traverse + given the split condition. + :pr:`28552` by :user:`Adam Li `. + +:mod:`sklearn.utils` +.................... + +- |Fix| :func:`~utils._safe_indexing` now works correctly for polars DataFrame when + `axis=0` and supports indexing polars Series. + :pr:`28521` by :user:`Yao Xiao `. + +- |API| :data:`utils.IS_PYPY` is deprecated and will be removed in version 1.7. + :pr:`28768` by :user:`JÊrÊmie du Boisberranger `. + +- |API| :func:`utils.tosequence` is deprecated and will be removed in version 1.7. + :pr:`28763` by :user:`JÊrÊmie du Boisberranger `. + +- |API| `utils.parallel_backend` and `utils.register_parallel_backend` are + deprecated and will be removed in version 1.7. Use `joblib.parallel_backend` and + `joblib.register_parallel_backend` instead. + :pr:`28847` by :user:`JÊrÊmie du Boisberranger `. + +- |API| Raise informative warning message in :func:`~utils.multiclass.type_of_target` + when represented as bytes. For classifiers and classification metrics, labels encoded + as bytes is deprecated and will raise an error in v1.7. + :pr:`18555` by :user:`Kaushik Amar Das `. + +- |API| :func:`utils.estimator_checks.check_estimator_sparse_data` was split into two + functions: :func:`utils.estimator_checks.check_estimator_sparse_matrix` and + :func:`utils.estimator_checks.check_estimator_sparse_array`. + :pr:`27576` by :user:`Stefanie Senger `. + +.. rubric:: Code and documentation contributors + +Thanks to everyone who has contributed to the maintenance and improvement of +the project since version 1.4, including: + +101AlexMartin, Abdulaziz Aloqeely, Adam J. Stewart, Adam Li, Adarsh Wase, +Adeyemi Biola, Aditi Juneja, Adrin Jalali, Advik Sinha, Aisha, Akash +Srivastava, Akihiro Kuno, Alan Guedes, Alberto Torres, Alexis IMBERT, alexqiao, +Ana Paula Gomes, Anderson Nelson, Andrei Dzis, Arif Qodari, Arnaud Capitaine, +Arturo Amor, Aswathavicky, Audrey Flanders, awwwyan, baggiponte, Bharat +Raghunathan, bme-git, brdav, Brendan Lu, Brigitta Sipőcz, Bruno, Cailean +Carter, Cemlyn, Christian Lorentzen, Christian Veenhuis, Cindy Liang, Claudio +Salvatore Arcidiacono, Connor Boyle, Conrad Stevens, crispinlogan, David +Matthew Cherney, Davide Chicco, davidleon123, dependabot[bot], DerWeh, dinga92, +Dipan Banik, Drew Craeton, Duarte SÃŖo JosÊ, DUONG, Eddie Bergman, Edoardo +Abati, Egehan Gunduz, Emad Izadifar, EmilyXinyi, Erich Schubert, Evelyn, Filip +Karlo DoÅĄilović, Franck Charras, Gael Varoquaux, GÃļnÃŧl AycÄą, Guillaume +Lemaitre, Gyeongjae Choi, Harmanan Kohli, Hong Xiang Yue, Ian Faust, Ilya +Komarov, itsaphel, Ivan Wiryadi, Jack Bowyer, Javier Marin Tur, JÊrÊmie du +Boisberranger, JÊrôme Dockès, Jiawei Zhang, JoÃŖo Morais, Joe Cainey, Joel +Nothman, Johanna Bayer, John Cant, John Enblom, John Hopfensperger, jpcars, +jpienaar-tuks, Julian Chan, Julian Libiseller-Egger, Julien Jerphanion, +KanchiMoe, Kaushik Amar Das, keyber, Koustav Ghosh, kraktus, Krsto Proroković, +Lars, ldwy4, LeoGrin, lihaitao, Linus Sommer, Loic Esteve, Lucy Liu, Lukas +Geiger, m-maggi, manasimj, Manuel LabbÊ, Manuel Morales, Marco Edward Gorelli, +Marco Wolsza, Maren Westermann, Marija Vlajic, Mark Elliot, Martin Helm, +Mateusz SokÃŗł, mathurinm, Mavs, Michael Dawson, Michael Higgins, Michael Mayer, +miguelcsilva, Miki Watanabe, Mohammed Hamdy, myenugula, Nathan Goldbaum, Naziya +Mahimkar, nbrown-ScottLogic, Neto, Nithish Bolleddula, notPlancha, Olivier +Grisel, Omar Salman, ParsifalXu, Patrick Wang, Pierre de FrÊminville, Piotr, +Priyank Shroff, Priyansh Gupta, Priyash Shah, Puneeth K, Rahil Parikh, raisadz, +Raj Pulapakura, Ralf Gommers, Ralph Urlus, Randolf Scholz, renaissance0ne, +Reshama Shaikh, Richard Barnes, Robert Pollak, Roberto Rosati, Rodrigo Romero, +rwelsch427, Saad Mahmood, Salim Dohri, Sandip Dutta, SarahRemus, +scikit-learn-bot, Shaharyar Choudhry, Shubham, sperret6, Stefanie Senger, +Steffen Schneider, Suha Siddiqui, Thanh Lam DANG, thebabush, Thomas, Thomas J. +Fan, Thomas Lazarus, Tialo, Tim Head, Tuhin Sharma, Tushar Parimi, +VarunChaduvula, Vineet Joshi, virchan, WaÃĢl Boukhobza, Weyb, Will Dean, Xavier +Beltran, Xiao Yuan, Xuefeng Xu, Yao Xiao, yareyaredesuyo, Ziad Amerr, Å těpÃĄn +SrÅĄeň diff --git a/doc/whats_new/v1.6.rst b/doc/whats_new/v1.6.rst new file mode 100644 index 0000000000000..e219f81be6268 --- /dev/null +++ b/doc/whats_new/v1.6.rst @@ -0,0 +1,785 @@ +.. include:: _contributors.rst + +.. currentmodule:: sklearn + +.. _release_notes_1_6: + +=========== +Version 1.6 +=========== + +For a short description of the main highlights of the release, please refer to +:ref:`sphx_glr_auto_examples_release_highlights_plot_release_highlights_1_6_0.py`. + +.. include:: changelog_legend.inc + +.. towncrier release notes start + +.. _changes_1_6_1: + +Version 1.6.1 +============= + +**January 2025** + +Changed models +-------------- + +- |Fix| The `tags.input_tags.sparse` flag was corrected for a majority of estimators. + By :user:`Antoine Baker ` :pr:`30187` + +Changes impacting many modules +------------------------------ + +- |Fix| `_more_tags`, `_get_tags`, and `_safe_tags` are now raising a + :class:`DeprecationWarning` instead of a :class:`FutureWarning` to only notify + developers instead of end-users. + By :user:`Guillaume Lemaitre ` in :pr:`30573` + +:mod:`sklearn.metrics` +---------------------- + +- |Fix| Fix regression when scikit-learn metric called on PyTorch CPU tensors would + raise an error (with array API dispatch disabled which is the default). + By :user:`Loïc Estève ` :pr:`30454` + +:mod:`sklearn.model_selection` +------------------------------ + +- |Fix| :func:`~model_selection.cross_validate`, :func:`~model_selection.cross_val_predict`, + and :func:`~model_selection.cross_val_score` now accept `params=None` when metadata + routing is enabled. By `Adrin Jalali`_ :pr:`30451` + +:mod:`sklearn.tree` +------------------- + +- |Fix| Use `log2` instead of `ln` for building trees to maintain behavior of previous + versions. By `Thomas Fan`_ :pr:`30557` + +:mod:`sklearn.utils` +-------------------- + +- |Enhancement| :func:`utils.estimator_checks.check_estimator_sparse_tag` ensures that + the estimator tag `input_tags.sparse` is consistent with its `fit` + method (accepting sparse input `X` or raising the appropriate error). + By :user:`Antoine Baker ` :pr:`30187` + +- |Fix| Raise a `DeprecationWarning` when there is no concrete implementation of `__sklearn_tags__` + in the MRO of the estimator. We request to inherit from `BaseEstimator` that + implements `__sklearn_tags__`. + By :user:`Guillaume Lemaitre ` :pr:`30516` + +.. _changes_1_6_0: + +Version 1.6.0 +============= + +**December 2024** + +Changes impacting many modules +------------------------------ + +- |Enhancement| `__sklearn_tags__` was introduced for setting tags in estimators. + More details in :ref:`estimator_tags`. + By :user:`Thomas Fan ` and :user:`Adrin Jalali ` :pr:`29677` + +- |Enhancement| Scikit-learn classes and functions can be used while only having a + `import sklearn` import line. For example, `import sklearn; sklearn.svm.SVC()` now works. + By :user:`Thomas Fan ` :pr:`29793` + +- |Fix| Classes :class:`metrics.ConfusionMatrixDisplay`, + :class:`metrics.RocCurveDisplay`, :class:`calibration.CalibrationDisplay`, + :class:`metrics.PrecisionRecallDisplay`, :class:`metrics.PredictionErrorDisplay` and + :class:`inspection.PartialDependenceDisplay` now properly handle Matplotlib aliases + for style parameters (e.g., `c` and `color`, `ls` and `linestyle`, etc). + By :user:`Joseph Barbier ` :pr:`30023` + +- |API| :func:`utils.validation.validate_data` is introduced and replaces previously + private `base.BaseEstimator._validate_data` method. This is intended for third party + estimator developers, who should use this function in most cases instead of + :func:`utils.check_array` and :func:`utils.check_X_y`. + By :user:`Adrin Jalali ` :pr:`29696` + +Support for Array API +--------------------- + +Additional estimators and functions have been updated to include support for all +`Array API `_ compliant inputs. + +See :ref:`array_api` for more details. + +- |Feature| :class:`model_selection.GridSearchCV`, + :class:`model_selection.RandomizedSearchCV`, + :class:`model_selection.HalvingGridSearchCV` and + :class:`model_selection.HalvingRandomSearchCV` now support Array API + compatible inputs when their base estimators do. + By :user:`Tim Head ` and :user:`Olivier Grisel ` :pr:`27096` + +- |Feature| :func:`sklearn.metrics.f1_score` now supports Array API compatible + inputs. + By :user:`Omar Salman ` :pr:`27369` + +- |Feature| :class:`preprocessing.LabelEncoder` now supports Array API compatible inputs. + By :user:`Omar Salman ` :pr:`27381` + +- |Feature| :func:`sklearn.metrics.mean_absolute_error` now supports Array API compatible + inputs. + By :user:`Edoardo Abati ` :pr:`27736` + +- |Feature| :func:`sklearn.metrics.mean_tweedie_deviance` now supports Array API + compatible inputs. + By :user:`Thomas Li ` :pr:`28106` + +- |Feature| :func:`sklearn.metrics.pairwise.cosine_similarity` now supports Array API + compatible inputs. + By :user:`Edoardo Abati ` :pr:`29014` + +- |Feature| :func:`sklearn.metrics.pairwise.paired_cosine_distances` now supports Array + API compatible inputs. + By :user:`Edoardo Abati ` :pr:`29112` + +- |Feature| :func:`sklearn.metrics.cluster.entropy` now supports Array API compatible + inputs. + By :user:`Yaroslav Korobko ` :pr:`29141` + +- |Feature| :func:`sklearn.metrics.mean_squared_error` now supports Array API compatible + inputs. + By :user:`Yaroslav Korobko ` :pr:`29142` + +- |Feature| :func:`sklearn.metrics.pairwise.additive_chi2_kernel` now supports Array API + compatible inputs. + By :user:`Yaroslav Korobko ` :pr:`29144` + +- |Feature| :func:`sklearn.metrics.d2_tweedie_score` now supports Array API compatible + inputs. + By :user:`Emily Chen ` :pr:`29207` + +- |Feature| :func:`sklearn.metrics.max_error` now supports Array API compatible inputs. + By :user:`Edoardo Abati ` :pr:`29212` + +- |Feature| :func:`sklearn.metrics.mean_poisson_deviance` now supports Array API + compatible inputs. + By :user:`Emily Chen ` :pr:`29227` + +- |Feature| :func:`sklearn.metrics.mean_gamma_deviance` now supports Array API compatible + inputs. + By :user:`Emily Chen ` :pr:`29239` + +- |Feature| :func:`sklearn.metrics.pairwise.cosine_distances` now supports Array API + compatible inputs. + By :user:`Emily Chen ` :pr:`29265` + +- |Feature| :func:`sklearn.metrics.pairwise.chi2_kernel` now supports Array API + compatible inputs. + By :user:`Yaroslav Korobko ` :pr:`29267` + +- |Feature| :func:`sklearn.metrics.mean_absolute_percentage_error` now supports Array API + compatible inputs. + By :user:`Emily Chen ` :pr:`29300` + +- |Feature| :func:`sklearn.metrics.pairwise.paired_euclidean_distances` now supports + Array API compatible inputs. + By :user:`Emily Chen ` :pr:`29389` + +- |Feature| :func:`sklearn.metrics.pairwise.euclidean_distances` and + :func:`sklearn.metrics.pairwise.rbf_kernel` now support Array API compatible + inputs. + By :user:`Omar Salman ` :pr:`29433` + +- |Feature| :func:`sklearn.metrics.pairwise.linear_kernel`, + :func:`sklearn.metrics.pairwise.sigmoid_kernel`, and + :func:`sklearn.metrics.pairwise.polynomial_kernel` now support Array API + compatible inputs. + By :user:`Omar Salman ` :pr:`29475` + +- |Feature| :func:`sklearn.metrics.mean_squared_log_error` and + :func:`sklearn.metrics.root_mean_squared_log_error` + now support Array API compatible inputs. + By :user:`Virgil Chan ` :pr:`29709` + +- |Feature| :class:`preprocessing.MinMaxScaler` with `clip=True` now supports Array API + compatible inputs. + By :user:`Shreekant Nandiyawar ` :pr:`29751` + +- Support for the soon to be deprecated `cupy.array_api` module has been + removed in favor of directly supporting the top level `cupy` module, possibly + via the `array_api_compat.cupy` compatibility wrapper. + By :user:`Olivier Grisel ` :pr:`29639` + +Metadata routing +---------------- + +Refer to the :ref:`Metadata Routing User Guide ` for +more details. + +- |Feature| :class:`semi_supervised.SelfTrainingClassifier` + now supports metadata routing. The fit method now accepts ``**fit_params`` + which are passed to the underlying estimators via their `fit` methods. + In addition, the + :meth:`~semi_supervised.SelfTrainingClassifier.predict`, + :meth:`~semi_supervised.SelfTrainingClassifier.predict_proba`, + :meth:`~semi_supervised.SelfTrainingClassifier.predict_log_proba`, + :meth:`~semi_supervised.SelfTrainingClassifier.score` + and :meth:`~semi_supervised.SelfTrainingClassifier.decision_function` + methods also accept ``**params`` which are + passed to the underlying estimators via their respective methods. + By :user:`Adam Li ` :pr:`28494` + +- |Feature| :class:`ensemble.StackingClassifier` and + :class:`ensemble.StackingRegressor` now support metadata routing and pass + ``**fit_params`` to the underlying estimators via their `fit` methods. + By :user:`Stefanie Senger ` :pr:`28701` + +- |Feature| :func:`model_selection.learning_curve` now supports metadata routing for the + `fit` method of its estimator and for its underlying CV splitter and scorer. + By :user:`Stefanie Senger ` :pr:`28975` + +- |Feature| :class:`compose.TransformedTargetRegressor` now supports metadata + routing in its :meth:`~compose.TransformedTargetRegressor.fit` and + :meth:`~compose.TransformedTargetRegressor.predict` methods and routes the + corresponding params to the underlying regressor. + By :user:`Omar Salman ` :pr:`29136` + +- |Feature| :class:`feature_selection.SequentialFeatureSelector` now supports + metadata routing in its `fit` method and passes the corresponding params to + the :func:`model_selection.cross_val_score` function. + By :user:`Omar Salman ` :pr:`29260` + +- |Feature| :func:`model_selection.permutation_test_score` now supports metadata routing + for the `fit` method of its estimator and for its underlying CV splitter and scorer. + By :user:`Adam Li ` :pr:`29266` + +- |Feature| :class:`feature_selection.RFE` and :class:`feature_selection.RFECV` + now support metadata routing. + By :user:`Omar Salman ` :pr:`29312` + +- |Feature| :func:`model_selection.validation_curve` now supports metadata routing for + the `fit` method of its estimator and for its underlying CV splitter and scorer. + By :user:`Stefanie Senger ` :pr:`29329` + +- |Fix| Metadata is routed correctly to grouped CV splitters via + :class:`linear_model.RidgeCV` and :class:`linear_model.RidgeClassifierCV` and + `UnsetMetadataPassedError` is fixed for :class:`linear_model.RidgeClassifierCV` with + default scoring. + By :user:`Stefanie Senger ` :pr:`29634` + +- |Fix| Many method arguments which shouldn't be included in the routing mechanism are + now excluded and the `set_{method}_request` methods are not generated for them. + By `Adrin Jalali`_ :pr:`29920` + +Dropping official support for PyPy +---------------------------------- + +Due to limited maintainer resources and small number of users, official PyPy +support has been dropped. Some parts of scikit-learn may still work but PyPy is +not tested anymore in the scikit-learn Continuous Integration. +By :user:`Loïc Estève ` :pr:`29128` + +Dropping support for building with setuptools +--------------------------------------------- + +From scikit-learn 1.6 onwards, support for building with setuptools has been +removed. Meson is the only supported way to build scikit-learn, see +:ref:`Building from source ` for more details. +By :user:`Loïc Estève ` :pr:`29400` + +Free-threaded CPython 3.13 support +---------------------------------- + +scikit-learn has preliminary support for free-threaded CPython, in particular +free-threaded wheels are available for all of our supported platforms. + +Free-threaded (also known as nogil) CPython 3.13 is an experimental version of +CPython 3.13 which aims at enabling efficient multi-threaded use cases by +removing the Global Interpreter Lock (GIL). + +For more details about free-threaded CPython see `py-free-threading doc `_, +in particular `how to install a free-threaded CPython `_ +and `Ecosystem compatibility tracking `_. + +Feel free to try free-threaded on your use case and report any issues! + +By :user:`Loïc Estève ` and many other people in the wider Scientific +Python and CPython ecosystem, for example :user:`Nathan Goldbaum `, +:user:`Ralf Gommers `, :user:`Edgar AndrÊs Margffoy Tuay `. :pr:`30360` + +:mod:`sklearn.base` +------------------- + +- |Enhancement| Added a function :func:`base.is_clusterer` which determines whether a given + estimator is of category clusterer. + By :user:`Christian Veenhuis ` :pr:`28936` + +- |API| Passing a class object to :func:`~sklearn.base.is_classifier`, + :func:`~sklearn.base.is_regressor`, and + :func:`~sklearn.base.is_outlier_detector` is now deprecated. Pass an instance + instead. + By `Adrin Jalali`_ :pr:`30122` + +:mod:`sklearn.calibration` +-------------------------- + +- |API| `cv="prefit"` is deprecated for :class:`~sklearn.calibration.CalibratedClassifierCV`. + Use :class:`~sklearn.frozen.FrozenEstimator` instead, as + `CalibratedClassifierCV(FrozenEstimator(estimator))`. + By `Adrin Jalali`_ :pr:`30171` + +:mod:`sklearn.cluster` +---------------------- + +- |API| The `copy` parameter of :class:`cluster.Birch` was deprecated in 1.6 and will be + removed in 1.8. It has no effect as the estimator does not perform in-place operations + on the input data. + By :user:`Yao Xiao ` :pr:`29124` + +:mod:`sklearn.compose` +---------------------- + +- |Enhancement| :func:`sklearn.compose.ColumnTransformer` `verbose_feature_names_out` + now accepts string format or callable to generate feature names. + By :user:`Marc Bresson ` :pr:`28934` + +:mod:`sklearn.covariance` +------------------------- + +- |Efficiency| :class:`covariance.MinCovDet` fitting is now slightly faster. + By :user:`Antony Lee ` :pr:`29835` + +:mod:`sklearn.cross_decomposition` +---------------------------------- + +- |Fix| :class:`cross_decomposition.PLSRegression` properly raises an error when + `n_components` is larger than `n_samples`. + By :user:`Thomas Fan ` :pr:`29710` + +:mod:`sklearn.datasets` +----------------------- + +- |Feature| :func:`datasets.fetch_file` allows downloading arbitrary data-file + from the web. It handles local caching, integrity checks with SHA256 digests + and automatic retries in case of HTTP errors. + By :user:`Olivier Grisel ` :pr:`29354` + +:mod:`sklearn.decomposition` +---------------------------- + +- |Enhancement| :class:`~sklearn.decomposition.LatentDirichletAllocation` now has a + ``normalize`` parameter in + :meth:`~sklearn.decomposition.LatentDirichletAllocation.transform` and + :meth:`~sklearn.decomposition.LatentDirichletAllocation.fit_transform` + methods to control whether the document topic distribution is normalized. + By `Adrin Jalali`_ :pr:`30097` + +- |Fix| :class:`~sklearn.decomposition.IncrementalPCA` + will now only raise a ``ValueError`` when the number of samples in the + input data to ``partial_fit`` is less than the number of components + on the first call to ``partial_fit``. Subsequent calls to ``partial_fit`` + no longer face this restriction. + By :user:`Thomas Gessey-Jones ` :pr:`30224` + +:mod:`sklearn.discriminant_analysis` +------------------------------------ + +- |Fix| :class:`discriminant_analysis.QuadraticDiscriminantAnalysis` + will now cause `LinAlgWarning` in case of collinear variables. These errors + can be silenced using the `reg_param` attribute. + By :user:`Alihan Zihna ` :pr:`19731` + +:mod:`sklearn.ensemble` +----------------------- + +- |Feature| :class:`ensemble.ExtraTreesClassifier` and + :class:`ensemble.ExtraTreesRegressor` now support missing-values in the data matrix + `X`. Missing-values are handled by randomly moving all of the samples to the left, or + right child node as the tree is traversed. + By :user:`Adam Li ` :pr:`28268` + +- |Efficiency| Small runtime improvement of fitting + :class:`ensemble.HistGradientBoostingClassifier` and + :class:`ensemble.HistGradientBoostingRegressor` by parallelizing the initial search + for bin thresholds. + By :user:`Christian Lorentzen ` :pr:`28064` + +- |Efficiency| :class:`ensemble.IsolationForest` now runs parallel jobs + during :term:`predict` offering a speedup of up to 2-4x on sample sizes + larger than 2000 using `joblib`. + By :user:`Adam Li ` and :user:`SÊrgio Pereira ` :pr:`28622` + +- |Enhancement| The verbosity of :class:`ensemble.HistGradientBoostingClassifier` + and :class:`ensemble.HistGradientBoostingRegressor` got a more granular control. Now, + `verbose = 1` prints only summary messages, `verbose >= 2` prints the full + information as before. + By :user:`Christian Lorentzen ` :pr:`28179` + +- |API| The parameter `algorithm` of :class:`ensemble.AdaBoostClassifier` is deprecated + and will be removed in 1.8. + By :user:`JÊrÊmie du Boisberranger ` :pr:`29997` + +:mod:`sklearn.feature_extraction` +--------------------------------- + +- |Fix| :class:`feature_extraction.text.TfidfVectorizer` now correctly preserves the + `dtype` of `idf_` based on the input data. + By :user:`Guillaume Lemaitre ` :pr:`30022` + +:mod:`sklearn.frozen` +--------------------- + +- |MajorFeature| :class:`~sklearn.frozen.FrozenEstimator` is now introduced which allows + freezing an estimator. This means calling `.fit` on it has no effect, and doing a + `clone(frozenestimator)` returns the same estimator instead of an unfitted clone. + :pr:`29705` By `Adrin Jalali`_ :pr:`29705` + +:mod:`sklearn.impute` +--------------------- + +- |Fix| :class:`impute.KNNImputer` excludes samples with nan distances when + computing the mean value for uniform weights. + By :user:`Xuefeng Xu ` :pr:`29135` + +- |Fix| When `min_value` and `max_value` are array-like and some features are dropped due to + `keep_empty_features=False`, :class:`impute.IterativeImputer` no longer raises an + error and now indexes correctly. + By :user:`Guntitat Sawadwuthikul ` :pr:`29451` + +- |Fix| Fixed :class:`impute.IterativeImputer` to make sure that it does not skip + the iterative process when `keep_empty_features` is set to `True`. + By :user:`Arif Qodari ` :pr:`29779` + +- |API| Add a warning in :class:`impute.SimpleImputer` when `keep_empty_feature=False` and + `strategy="constant"`. In this case empty features are not dropped and this behaviour + will change in 1.8. + By :user:`Arthur Courselle ` and :user:`Simon Riou ` :pr:`29950` + +:mod:`sklearn.linear_model` +--------------------------- + +- |Enhancement| The `solver="newton-cholesky"` in + :class:`linear_model.LogisticRegression` and + :class:`linear_model.LogisticRegressionCV` is extended to support the full + multinomial loss in a multiclass setting. + By :user:`Christian Lorentzen ` :pr:`28840` + +- |Fix| In :class:`linear_model.Ridge` and :class:`linear_model.RidgeCV`, after `fit`, + the `coef_` attribute is now of shape `(n_samples,)` like other linear models. + By :user:`Maxwell Liu`, `Guillaume Lemaitre`_, and `Adrin Jalali`_ :pr:`19746` + +- |Fix| :class:`linear_model.LogisticRegressionCV` corrects sample weight handling + for the calculation of test scores. + By :user:`Shruti Nath ` :pr:`29419` + +- |Fix| :class:`linear_model.LassoCV` and :class:`linear_model.ElasticNetCV` now + take sample weights into accounts to define the search grid for the internally tuned + `alpha` hyper-parameter. + By :user:`John Hopfensperger ` and :user:`Shruti Nath ` :pr:`29442` + +- |Fix| :class:`linear_model.LogisticRegression`, :class:`linear_model.PoissonRegressor`, + :class:`linear_model.GammaRegressor`, :class:`linear_model.TweedieRegressor` + now take sample weights into account to decide when to fall back to `solver='lbfgs'` + whenever `solver='newton-cholesky'` becomes numerically unstable. + By :user:`Antoine Baker ` :pr:`29818` + +- |Fix| :class:`linear_model.RidgeCV` now properly uses predictions on the same scale as + the target seen during `fit`. These predictions are stored in `cv_results_` when + `scoring != None`. Previously, the predictions were rescaled by the square root of the + sample weights and offset by the mean of the target, leading to an incorrect estimate + of the score. + By :user:`Guillaume Lemaitre `, + :user:`JÊrôme Dockes ` and + :user:`Hanmin Qin ` :pr:`29842` + +- |Fix| :class:`linear_model.RidgeCV` now properly supports custom multioutput scorers + by letting the scorer manage the multioutput averaging. Previously, the predictions + and true targets were both squeezed to a 1D array before computing the error. + By :user:`Guillaume Lemaitre ` :pr:`29884` + +- |Fix| :class:`linear_model.LinearRegression` now sets the `cond` parameter when + calling the `scipy.linalg.lstsq` solver on dense input data. This ensures + more numerically robust results on rank-deficient data. In particular, it + empirically fixes the expected equivalence property between fitting with + reweighted or with repeated data points. + By :user:`Antoine Baker ` :pr:`30040` + +- |Fix| :class:`linear_model.LogisticRegression` and other linear models that + accept `solver="newton-cholesky"` now report the correct number of iterations + when they fall back to the `"lbfgs"` solver because of a rank deficient + Hessian matrix. + By :user:`Olivier Grisel ` :pr:`30100` + +- |Fix| :class:`~sklearn.linear_model.SGDOneClassSVM` now correctly inherits from + :class:`~sklearn.base.OutlierMixin` and the tags are correctly set. + By :user:`Guillaume Lemaitre ` :pr:`30227` + +- |API| Deprecates `copy_X` in :class:`linear_model.TheilSenRegressor` as the parameter + has no effect. `copy_X` will be removed in 1.8. + By :user:`Adam Li ` :pr:`29105` + +:mod:`sklearn.manifold` +----------------------- + +- |Efficiency| :func:`manifold.locally_linear_embedding` and + :class:`manifold.LocallyLinearEmbedding` now allocate more efficiently the memory of + sparse matrices in the Hessian, Modified and LTSA methods. + By :user:`Giorgio Angelotti ` :pr:`28096` + +:mod:`sklearn.metrics` +---------------------- + +- |Efficiency| :func:`sklearn.metrics.classification_report` is now faster by caching + classification labels. + By :user:`Adrin Jalali ` :pr:`29738` + +- |Enhancement| :meth:`metrics.RocCurveDisplay.from_estimator`, + :meth:`metrics.RocCurveDisplay.from_predictions`, + :meth:`metrics.PrecisionRecallDisplay.from_estimator`, and + :meth:`metrics.PrecisionRecallDisplay.from_predictions` now accept a new keyword + `despine` to remove the top and right spines of the plot in order to make it clearer. + By :user:`Yao Xiao ` :pr:`26367` + +- |Enhancement| :func:`sklearn.metrics.check_scoring` now accepts `raise_exc` to specify + whether to raise an exception if a subset of the scorers in multimetric scoring fails + or to return an error code. + By :user:`Stefanie Senger ` :pr:`28992` + +- |Fix| :func:`metrics.roc_auc_score` will now correctly return np.nan and + warn user if only one class is present in the labels. + By :user:`Hleb Levitski ` and :user:`Janez DemÅĄar ` :pr:`27412`, :pr:`30013` + +- |Fix| The functions :func:`metrics.mean_squared_log_error` and + :func:`metrics.root_mean_squared_log_error` now check whether the inputs are within + the correct domain for the function :math:`y=\log(1+x)`, rather than + :math:`y=\log(x)`. The functions :func:`metrics.mean_absolute_error`, + :func:`metrics.mean_absolute_percentage_error`, :func:`metrics.mean_squared_error` + and :func:`metrics.root_mean_squared_error` now explicitly check whether a scalar + will be returned when `multioutput=uniform_average`. + By :user:`Virgil Chan ` :pr:`29709` + +- |API| The `assert_all_finite` parameter of functions + :func:`metrics.pairwise.check_pairwise_arrays` and :func:`metrics.pairwise_distances` + is renamed into `ensure_all_finite`. `force_all_finite` will be removed in 1.8. + By :user:`JÊrÊmie du Boisberranger ` :pr:`29404` + +- |API| `scoring="neg_max_error"` should be used instead of `scoring="max_error"` + which is now deprecated. + By :user:`Farid "Freddie" Taba ` :pr:`29462` + +- |API| The default value of the `response_method` parameter of + :func:`metrics.make_scorer` will change from `None` to `"predict"` and `None` will be + removed in 1.8. In the meantime, `None` is equivalent to `"predict"`. + By :user:`JÊrÊmie du Boisberranger ` :pr:`30001` + +:mod:`sklearn.model_selection` +------------------------------ + +- |Enhancement| :class:`~model_selection.GroupKFold` now has the ability to shuffle groups into + different folds when `shuffle=True`. + By :user:`Zachary Vealey ` :pr:`28519` + +- |Enhancement| There is no need to call `fit` on a + :class:`~sklearn.model_selection.FixedThresholdClassifier` if the underlying + estimator is already fitted. + By :user:`Adrin Jalali ` :pr:`30172` + +- |Fix| Improve error message when :func:`model_selection.RepeatedStratifiedKFold.split` + is called without a `y` argument + By :user:`Anurag Varma ` :pr:`29402` + +:mod:`sklearn.neighbors` +------------------------ + +- |Enhancement| :class:`neighbors.NearestNeighbors`, + :class:`neighbors.KNeighborsClassifier`, + :class:`neighbors.KNeighborsRegressor`, + :class:`neighbors.RadiusNeighborsClassifier`, + :class:`neighbors.RadiusNeighborsRegressor`, + :class:`neighbors.KNeighborsTransformer`, + :class:`neighbors.RadiusNeighborsTransformer`, and + :class:`neighbors.LocalOutlierFactor` + now work with `metric="nan_euclidean"`, supporting `nan` inputs. + By :user:`Carlo Lemos `, `Guillaume Lemaitre`_, and `Adrin Jalali`_ :pr:`25330` + +- |Enhancement| Add :meth:`neighbors.NearestCentroid.decision_function`, + :meth:`neighbors.NearestCentroid.predict_proba` and + :meth:`neighbors.NearestCentroid.predict_log_proba` + to the :class:`neighbors.NearestCentroid` estimator class. + Support the case when `X` is sparse and `shrinking_threshold` + is not `None` in :class:`neighbors.NearestCentroid`. + By :user:`Matthew Ning ` :pr:`26689` + +- |Enhancement| Make `predict`, `predict_proba`, and `score` of + :class:`neighbors.KNeighborsClassifier` and + :class:`neighbors.RadiusNeighborsClassifier` accept `X=None` as input. In this case + predictions for all training set points are returned, and points are not included + into their own neighbors. + By :user:`Dmitry Kobak ` :pr:`30047` + +- |Fix| :class:`neighbors.LocalOutlierFactor` raises a warning in the `fit` method + when duplicate values in the training data lead to inaccurate outlier detection. + By :user:`Henrique Caroço ` :pr:`28773` + +:mod:`sklearn.neural_network` +----------------------------- + +- |Fix| :class:`neural_network.MLPRegressor` does no longer crash when the model + diverges and that `early_stopping` is enabled. + By :user:`Marc Bresson ` :pr:`29773` + +:mod:`sklearn.pipeline` +----------------------- + +- |MajorFeature| :class:`pipeline.Pipeline` can now transform metadata up to the step requiring the + metadata, which can be set using the `transform_input` parameter. + By `Adrin Jalali`_ :pr:`28901` + +- |Enhancement| :class:`pipeline.Pipeline` now warns about not being fitted before calling methods + that require the pipeline to be fitted. This warning will become an error in 1.8. + By `Adrin Jalali`_ :pr:`29868` + +- |Fix| Fixed an issue with tags and estimator type of :class:`~sklearn.pipeline.Pipeline` + when pipeline is empty. This allows the HTML representation of an empty + pipeline to be rendered correctly. + By :user:`Gennaro Daniele Acciaro ` :pr:`30203` + +:mod:`sklearn.preprocessing` +---------------------------- + +- |Enhancement| Added `warn` option to `handle_unknown` parameter in + :class:`preprocessing.OneHotEncoder`. + By :user:`Hleb Levitski ` :pr:`28637` + +- |Enhancement| The HTML representation of :class:`preprocessing.FunctionTransformer` + will show the function name in the label. + By :user:`Yao Xiao ` :pr:`29158` + +- |Fix| :class:`preprocessing.PowerTransformer` now uses `scipy.special.inv_boxcox` + to output `nan` if the input of BoxCox's inverse is invalid. + By :user:`Xuefeng Xu ` :pr:`27875` + +:mod:`sklearn.semi_supervised` +------------------------------ + +- |API| :class:`semi_supervised.SelfTrainingClassifier` + deprecated the `base_estimator` parameter in favor of `estimator`. + By :user:`Adam Li ` :pr:`28494` + +:mod:`sklearn.tree` +------------------- + +- |Feature| :class:`tree.ExtraTreeClassifier` and :class:`tree.ExtraTreeRegressor` now + support missing-values in the data matrix ``X``. Missing-values are handled by + randomly moving all of the samples to the left, or right child node as the tree is + traversed. + By :user:`Adam Li ` and :user:`Loïc Estève ` :pr:`27966`, :pr:`30318` + +- |Fix| Escape double quotes for labels and feature names when exporting trees to Graphviz + format. + By :user:`Santiago M. Mola `. :pr:`17575` + +:mod:`sklearn.utils` +-------------------- + +- |Enhancement| :func:`utils.check_array` now accepts `ensure_non_negative` + to check for negative values in the passed array, until now only available through + calling :func:`utils.check_non_negative`. + By :user:`Tamara Atanasoska ` :pr:`29540` + +- |Enhancement| :func:`~sklearn.utils.estimator_checks.check_estimator` and + :func:`~sklearn.utils.estimator_checks.parametrize_with_checks` now check and fail if + the classifier has the `tags.classifier_tags.multi_class = False` tag but does not + fail on multi-class data. + By `Adrin Jalali`_ :pr:`29874` + +- |Enhancement| :func:`utils.validation.check_is_fitted` now passes on stateless + estimators. An estimator can indicate it's stateless by setting the `requires_fit` + tag. See :ref:`estimator_tags` for more information. + By :user:`Adrin Jalali ` :pr:`29880` + +- |Enhancement| Changes to :func:`~utils.estimator_checks.check_estimator` and + :func:`~utils.estimator_checks.parametrize_with_checks`. + + - :func:`~utils.estimator_checks.check_estimator` introduces new arguments: + ``on_skip``, ``on_fail``, and ``callback`` to control the behavior of the check + runner. Refer to the API documentation for more details. + + - ``generate_only=True`` is deprecated in + :func:`~utils.estimator_checks.check_estimator`. Use + :func:`~utils.estimator_checks.estimator_checks_generator` instead. + + - The ``_xfail_checks`` estimator tag is now removed, and now in order to indicate + which tests are expected to fail, you can pass a dictionary to the + :func:`~utils.estimator_checks.check_estimator` as the ``expected_failed_checks`` + parameter. Similarly, the ``expected_failed_checks`` parameter in + :func:`~utils.estimator_checks.parametrize_with_checks` can be used, which is a + callable returning a dictionary of the form:: + + { + "check_name": "reason to mark this check as xfail", + } + + By `Adrin Jalali`_ :pr:`30149` + +- |Fix| :func:`utils.estimator_checks.parametrize_with_checks` and + :func:`utils.estimator_checks.check_estimator` now support estimators that + have `set_output` called on them. + By :user:`Adrin Jalali ` :pr:`29869` + +- |API| The `assert_all_finite` parameter of functions :func:`utils.check_array`, + :func:`utils.check_X_y`, :func:`utils.as_float_array` is renamed into + `ensure_all_finite`. `force_all_finite` will be removed in 1.8. + By :user:`JÊrÊmie du Boisberranger ` :pr:`29404` + +- |API| `utils.estimator_checks.check_sample_weights_invariance` + replaced by + `utils.estimator_checks.check_sample_weight_equivalence_on_dense_data` + which uses integer (including zero) weights and + `utils.estimator_checks.check_sample_weight_equivalence_on_sparse_data` + which does the same on sparse data. + By :user:`Antoine Baker ` :pr:`29818`, :pr:`30137` + +- |API| Using `_estimator_type` to set the estimator type is deprecated. Inherit from + :class:`~sklearn.base.ClassifierMixin`, :class:`~sklearn.base.RegressorMixin`, + :class:`~sklearn.base.TransformerMixin`, or :class:`~sklearn.base.OutlierMixin` + instead. Alternatively, you can set `estimator_type` in :class:`~sklearn.utils.Tags` + in the `__sklearn_tags__` method. + By `Adrin Jalali`_ :pr:`30122` + +.. rubric:: Code and documentation contributors + +Thanks to everyone who has contributed to the maintenance and improvement of +the project since version 1.5, including: + +Aaron Schumacher, Abdulaziz Aloqeely, abhi-jha, Acciaro Gennaro Daniele, Adam +J. Stewart, Adam Li, Adeel Hassan, Adeyemi Biola, Aditi Juneja, Adrin Jalali, +Aisha, Akanksha Mhadolkar, Akihiro Kuno, Alberto Torres, alexqiao, Alihan +Zihna, Aniruddha Saha, antoinebaker, Antony Lee, Anurag Varma, Arif Qodari, +Arthur Courselle, ArthurDbrn, Arturo Amor, Aswathavicky, Audrey Flanders, +aurelienmorgan, Austin, awwwyan, AyGeeEm, a.zy.lee, baggiponte, BlazeStorm001, +bme-git, Boney Patel, brdav, Brigitta Sipőcz, Cailean Carter, Camille +Troillard, Carlo Lemos, Christian Lorentzen, Christian Veenhuis, Christine P. +Chai, claudio, Conrad Stevens, datarollhexasphericon, Davide Chicco, David +Matthew Cherney, Dea María LÊon, Deepak Saldanha, Deepyaman Datta, +dependabot[bot], dinga92, Dmitry Kobak, Domenico, Drew Craeton, dymil, Edoardo +Abati, EmilyXinyi, Eric Larson, Evelyn, fabianhenning, Farid "Freddie" Taba, +Gael Varoquaux, Giorgio Angelotti, Hleb Levitski, Guillaume Lemaitre, Guntitat +Sawadwuthikul, Haesun Park, Hanjun Kim, Henrique Caroço, hhchen1105, Hugo +Boulenger, Ilya Komarov, Inessa Pawson, Ivan Pan, Ivan Wiryadi, Jaimin Chauhan, +Jakob Bull, James Lamb, Janez DemÅĄar, JÊrÊmie du Boisberranger, JÊrôme +Dockès, Jirair Aroyan, JoÃŖo Morais, Joe Cainey, Joel Nothman, John Enblom, +JorgeCardenas, Joseph Barbier, jpienaar-tuks, Julian Chan, K.Bharat Reddy, +Kevin Doshi, Lars, Loic Esteve, Lucas Colley, Lucy Liu, lunovian, Marc Bresson, +Marco Edward Gorelli, Marco Maggi, Marco Wolsza, Maren Westermann, +MarieS-WiMLDS, Martin Helm, Mathew Shen, mathurinm, Matthew Feickert, Maxwell +Liu, Meekail Zain, Michael Dawson, Miguel CÃĄrdenas, m-maggi, mrastgoo, Natalia +Mokeeva, Nathan Goldbaum, Nathan Orgera, nbrown-ScottLogic, Nikita Chistyakov, +Nithish Bolleddula, Noam Keidar, NoPenguinsLand, Norbert Preining, notPlancha, +Olivier Grisel, Omar Salman, ParsifalXu, Piotr, Priyank Shroff, Priyansh Gupta, +Quentin BarthÊlemy, Rachit23110261, Rahil Parikh, raisadz, Rajath, +renaissance0ne, Reshama Shaikh, Roberto Rosati, Robert Pollak, rwelsch427, +Santiago Castro, Santiago M. Mola, scikit-learn-bot, sean moiselle, SHREEKANT +VITTHAL NANDIYAWAR, Shruti Nath, Søren Bredlund Caspersen, Stefanie Senger, +Stefano Gaspari, Steffen Schneider, Å těpÃĄn SrÅĄeň, Sylvain Combettes, +Tamara, Thomas, Thomas Gessey-Jones, Thomas J. Fan, Thomas Li, ThorbenMaa, +Tialo, Tim Head, Tuhin Sharma, Tushar Parimi, Umberto Fasci, UV, vedpawar2254, +Velislav Babatchev, Victoria Shevchenko, viktor765, Vince Carey, Virgil Chan, +Wang Jiayi, Xiao Yuan, Xuefeng Xu, Yao Xiao, yareyaredesuyo, Zachary Vealey, +Ziad Amerr diff --git a/doc/whats_new/v1.7.rst b/doc/whats_new/v1.7.rst new file mode 100644 index 0000000000000..ab022414982ff --- /dev/null +++ b/doc/whats_new/v1.7.rst @@ -0,0 +1,514 @@ +.. include:: _contributors.rst + +.. currentmodule:: sklearn + +.. _release_notes_1_7: + +=========== +Version 1.7 +=========== + +For a short description of the main highlights of the release, please refer to +:ref:`sphx_glr_auto_examples_release_highlights_plot_release_highlights_1_7_0.py`. + +.. include:: changelog_legend.inc + +.. towncrier release notes start + +.. _changes_1_7_0: + +Version 1.7.0 +============= + +**June 2025** + +Changed models +-------------- + +- |Fix| Change the `ConvergenceWarning` message of estimators that rely on the + `"lbfgs"` optimizer internally to be more informative and to avoid + suggesting to increase the maximum number of iterations when it is not + user-settable or when the convergence problem happens before reaching it. + By :user:`Olivier Grisel `. :pr:`31316` + +Changes impacting many modules +------------------------------ + +- Sparse update: As part of the SciPy change from spmatrix to sparray, all + internal use of sparse now supports both sparray and spmatrix. + All manipulations of sparse objects should work for either spmatrix or sparray. + This is pass 1 of a migration toward sparray (see + `SciPy migration to sparray `_ + By :user:`Dan Schult ` :pr:`30858` + +Support for Array API +--------------------- + +Additional estimators and functions have been updated to include support for all +`Array API `_ compliant inputs. + +See :ref:`array_api` for more details. + +- |Feature| :func:`sklearn.utils.check_consistent_length` now supports Array API compatible + inputs. + By :user:`Stefanie Senger ` :pr:`29519` + +- |Feature| :func:`sklearn.metrics.explained_variance_score` and + :func:`sklearn.metrics.mean_pinball_loss` now support Array API compatible inputs. + By :user:`Virgil Chan ` :pr:`29978` + +- |Feature| :func:`sklearn.metrics.fbeta_score`, + :func:`sklearn.metrics.precision_score` and + :func:`sklearn.metrics.recall_score` now support Array API compatible inputs. + By :user:`Omar Salman ` :pr:`30395` + +- |Feature| :func:`sklearn.utils.extmath.randomized_svd` now support Array API compatible inputs. + By :user:`Connor Lane ` and :user:`JÊrÊmie du Boisberranger `. :pr:`30819` + +- |Feature| :func:`sklearn.metrics.hamming_loss` now support Array API compatible inputs. + By :user:`Thomas Li ` :pr:`30838` + +- |Feature| :class:`preprocessing.Binarizer` now supports Array API compatible inputs. + By :user:`Yaroslav Korobko `, :user:`Olivier Grisel `, and :user:`Thomas Li `. :pr:`31190` + +- |Feature| :func:`sklearn.metrics.jaccard_score` now supports Array API compatible inputs. + By :user:`Omar Salman ` :pr:`31204` + +- array-api-compat and array-api-extra are now vendored within the + scikit-learn source. Users of the experimental array API standard + support no longer need to install array-api-compat in their environment. + by :user:`Lucas Colley ` :pr:`30340` + +Metadata routing +---------------- + +Refer to the :ref:`Metadata Routing User Guide ` for +more details. + +- |Feature| :class:`ensemble.BaggingClassifier` and :class:`ensemble.BaggingRegressor` now support + metadata routing through their `predict`, `predict_proba`, `predict_log_proba` and + `decision_function` methods and pass `**params` to the underlying estimators. + By :user:`Stefanie Senger `. :pr:`30833` + +:mod:`sklearn.base` +------------------- + +- |Enhancement| :class:`base.BaseEstimator` now has a parameter table added to the + estimators HTML representation that can be visualized with jupyter. + By :user:`Guillaume Lemaitre ` and + :user:`Dea María LÊon ` :pr:`30763` + +:mod:`sklearn.calibration` +-------------------------- + +- |Fix| :class:`~calibration.CalibratedClassifierCV` now raises `FutureWarning` + instead of `UserWarning` when passing `cv="prefit`". By + :user:`Olivier Grisel ` +- :class:`~calibration.CalibratedClassifierCV` with `method="sigmoid"` no + longer crashes when passing `float64`-dtyped `sample_weight` along with a + base estimator that outputs `float32`-dtyped predictions. By :user:`Olivier + Grisel ` :pr:`30873` + +:mod:`sklearn.compose` +---------------------- + +- |API| The `force_int_remainder_cols` parameter of :class:`compose.ColumnTransformer` and + :func:`compose.make_column_transformer` is deprecated and will be removed in 1.9. + It has no effect. + By :user:`JÊrÊmie du Boisberranger ` :pr:`31167` + +:mod:`sklearn.covariance` +------------------------- + +- |Fix| Support for ``n_samples == n_features`` in `sklearn.covariance.MinCovDet` has + been restored. By :user:`Antony Lee `. :pr:`30483` + +:mod:`sklearn.datasets` +----------------------- + +- |Enhancement| New parameter ``return_X_y`` added to :func:`datasets.make_classification`. The + default value of the parameter does not change how the function behaves. + By :user:`Success Moses ` and :user:`Adam Cooper ` :pr:`30196` + +:mod:`sklearn.decomposition` +---------------------------- + +- |Feature| :class:`~sklearn.decomposition.DictionaryLearning`, + :class:`~sklearn.decomposition.SparseCoder` and + :class:`~sklearn.decomposition.MiniBatchDictionaryLearning` now have a + ``inverse_transform`` method. By :user:`RÊmi Flamary ` :pr:`30443` + +:mod:`sklearn.ensemble` +----------------------- + +- |Feature| :class:`ensemble.HistGradientBoostingClassifier` and + :class:`ensemble.HistGradientBoostingRegressor` allow for more control over the + validation set used for early stopping. You can now pass data to be used for + validation directly to `fit` via the arguments `X_val`, `y_val` and + `sample_weight_val`. + By :user:`Christian Lorentzen `. :pr:`27124` + +- |Fix| :class:`ensemble.VotingClassifier` and :class:`ensemble.VotingRegressor` + validate `estimators` to make sure it is a list of tuples. By `Thomas Fan`_. :pr:`30649` + +:mod:`sklearn.feature_selection` +-------------------------------- + +- |Enhancement| :class:`feature_selection.RFECV` now gives access to the ranking and support in each + iteration and cv step of feature selection. + By :user:`Marie S. ` :pr:`30179` + +- |Fix| :class:`feature_selection.SelectFromModel` now correctly works when the estimator + is an instance of :class:`linear_model.ElasticNetCV` with its `l1_ratio` parameter + being an array-like. + By :user:`Vasco Pereira `. :pr:`31107` + +:mod:`sklearn.gaussian_process` +------------------------------- + +- |Enhancement| :class:`gaussian_process.GaussianProcessClassifier` now includes a `latent_mean_and_variance` method that exposes the mean and the variance of the latent function, :math:`f`, used in the Laplace approximation. By :user:`Miguel GonzÃĄlez Duque ` :pr:`22227` + +:mod:`sklearn.inspection` +------------------------- + +- |Enhancement| Add `custom_values` parameter in :func:`inspection.partial_dependence`. It enables + users to pass their own grid of values at which the partial dependence should be + calculated. + By :user:`Freddy A. Boulton ` and :user:`Stephen Pardy + ` :pr:`26202` + +- |Enhancement| :class:`inspection.DecisionBoundaryDisplay` now supports + plotting all classes for multi-class problems when `response_method` is + 'decision_function', 'predict_proba' or 'auto'. + By :user:`Lucy Liu ` :pr:`29797` + +- |Fix| :func:`inspection.partial_dependence` now raises an informative error when passing + an empty list as the `categorical_features` parameter. `None` should be used instead + to indicate that no categorical features are present. + By :user:`Pedro Lopes `. :pr:`31146` + +- |API| :func:`inspection.partial_dependence` does no longer accept integer dtype for + numerical feature columns. Explicit conversion to floating point values is + now required before calling this tool (and preferably even before fitting the + model to inspect). + By :user:`Olivier Grisel ` :pr:`30409` + +:mod:`sklearn.linear_model` +--------------------------- + +- |Enhancement| :class:`linear_model.SGDClassifier` and :class:`linear_model.SGDRegressor` now accept + `l1_ratio=None` when `penalty` is not `"elasticnet"`. + By :user:`Marc Bresson `. :pr:`30730` + +- |Enhancement| Fitting :class:`linear_model.Lasso` and :class:`linear_model.ElasticNet` with + `fit_intercept=True` is faster for sparse input `X` because an unnecessary + re-computation of the sum of residuals is avoided. + By :user:`Christian Lorentzen ` :pr:`31387` + +- |Fix| :class:`linear_model.LogisticRegression` and + :class:`linear_model.LogisticRegressionCV` now properly pass sample weights to + :func:`utils.class_weight.compute_class_weight` when fit with + `class_weight="balanced"`. + By :user:`Shruti Nath ` and :user:`Olivier Grisel ` :pr:`30057` + +- |Fix| Added a new parameter `tol` to + :class:`linear_model.LinearRegression` that determines the precision of the + solution `coef_` when fitting on sparse data. + By :user:`Success Moses ` :pr:`30521` + +- |Fix| The update and initialization of the hyperparameters now properly handle + sample weights in :class:`linear_model.BayesianRidge`. + By :user:`Antoine Baker `. :pr:`30644` + +- |Fix| :class:`linear_model.BayesianRidge` now uses the full SVD to correctly estimate + the posterior covariance matrix `sigma_` when `n_samples < n_features`. + By :user:`Antoine Baker ` :pr:`31094` + +- |API| The parameter `n_alphas` has been deprecated in the following classes: + :class:`linear_model.ElasticNetCV` and :class:`linear_model.LassoCV` + and :class:`linear_model.MultiTaskElasticNetCV` + and :class:`linear_model.MultiTaskLassoCV`, and will be removed in 1.9. The parameter + `alphas` now supports both integers and array-likes, removing the need for `n_alphas`. + From now on, only `alphas` should be set to either indicate the number of alphas to + automatically generate (int) or to provide a list of alphas (array-like) to test along + the regularization path. + By :user:`Siddharth Bansal `. :pr:`30616` + +- |API| Using the `"liblinear"` solver for multiclass classification with a one-versus-rest + scheme in :class:`linear_model.LogisticRegression` and + :class:`linear_model.LogisticRegressionCV` is deprecated and will raise an error in + version 1.8. Either use a solver which supports the multinomial loss or wrap the + estimator in a :class:`sklearn.multiclass.OneVsRestClassifier` to keep applying a + one-versus-rest scheme. + By :user:`JÊrÊmie du Boisberranger `. :pr:`31241` + +:mod:`sklearn.manifold` +----------------------- + +- |Enhancement| :class:`manifold.MDS` will switch to use `n_init=1` by default, + starting from version 1.9. + By :user:`Dmitry Kobak ` :pr:`31117` + +- |Fix| :class:`manifold.MDS` now correctly handles non-metric MDS. Furthermore, + the returned stress value now corresponds to the returned embedding and + normalized stress is now allowed for metric MDS. + By :user:`Dmitry Kobak ` :pr:`30514` + +- |Fix| :class:`manifold.MDS` now uses `eps=1e-6` by default and the convergence + criterion was adjusted to make sense for both metric and non-metric MDS + and to follow the reference R implementation. The formula for normalized + stress was adjusted to follow the original definition by Kruskal. + By :user:`Dmitry Kobak ` :pr:`31117` + +:mod:`sklearn.metrics` +---------------------- + +- |Feature| :func:`metrics.brier_score_loss` implements the Brier score for multiclass + classification problems and adds a `scale_by_half` argument. This metric is + notably useful to assess both sharpness and calibration of probabilistic + classifiers. See the docstrings for more details. By + :user:`Varun Aggarwal `, :user:`Olivier Grisel ` and + :user:`Antoine Baker `. :pr:`22046` + +- |Feature| Add class method `from_cv_results` to :class:`metrics.RocCurveDisplay`, which allows + easy plotting of multiple ROC curves from :func:`model_selection.cross_validate` + results. + By :user:`Lucy Liu ` :pr:`30399` + +- |Enhancement| :func:`metrics.det_curve`, :class:`metrics.DetCurveDisplay.from_estimator`, + and :class:`metrics.DetCurveDisplay.from_estimator` now accept a + `drop_intermediate` option to drop thresholds where true positives (tp) do not + change from the previous or subsequent thresholds. All points with the same tp + value have the same `fnr` and thus same y coordinate in a DET curve. + By :user:`Arturo Amor ` :pr:`29151` + +- |Enhancement| :func:`~metrics.class_likelihood_ratios` now has a `replace_undefined_by` param. + When there is a division by zero, the metric is undefined and the set values are + returned for `LR+` and `LR-`. + By :user:`Stefanie Senger ` :pr:`29288` + +- |Fix| :func:`metrics.log_loss` now raises a `ValueError` if values of `y_true` + are missing in `labels`. By :user:`Varun Aggarwal `, + :user:`Olivier Grisel ` and :user:`Antoine Baker `. :pr:`22046` + +- |Fix| :func:`metrics.det_curve` and :class:`metrics.DetCurveDisplay` now return an + extra threshold at infinity where the classifier always predicts the negative + class i.e. tps = fps = 0. + By :user:`Arturo Amor ` :pr:`29151` + +- |Fix| :func:`~metrics.class_likelihood_ratios` now raises `UndefinedMetricWarning` instead + of `UserWarning` when a division by zero occurs. + By :user:`Stefanie Senger ` :pr:`29288` + +- |Fix| :class:`metrics.RocCurveDisplay` will no longer set a legend when + `label` is `None` in both the `line_kwargs` and the `chance_level_kw`. + By :user:`Arturo Amor ` :pr:`29727` + +- |Fix| Additional `sample_weight` checking has been added to + :func:`metrics.mean_absolute_error`, + :func:`metrics.mean_pinball_loss`, + :func:`metrics.mean_absolute_percentage_error`, + :func:`metrics.mean_squared_error`, + :func:`metrics.root_mean_squared_error`, + :func:`metrics.mean_squared_log_error`, + :func:`metrics.root_mean_squared_log_error`, + :func:`metrics.explained_variance_score`, + :func:`metrics.r2_score`, + :func:`metrics.mean_tweedie_deviance`, + :func:`metrics.mean_poisson_deviance`, + :func:`metrics.mean_gamma_deviance` and + :func:`metrics.d2_tweedie_score`. + `sample_weight` can only be 1D, consistent to `y_true` and `y_pred` in length + or a scalar. + By :user:`Lucy Liu `. :pr:`30886` + +- |Fix| :func:`~metrics.d2_log_loss_score` now properly handles the case when `labels` is + passed and not all of the labels are present in `y_true`. + By :user:`Vassilis Margonis ` :pr:`30903` + +- |Fix| Fix :func:`metrics.adjusted_mutual_info_score` numerical issue when number of + classes and samples is low. + By :user:`Hleb Levitski ` :pr:`31065` + +- |API| The `sparse` parameter of :func:`metrics.fowlkes_mallows_score` is deprecated and + will be removed in 1.9. It has no effect. + By :user:`Luc Rocher `. :pr:`28981` + +- |API| The `raise_warning` parameter of :func:`metrics.class_likelihood_ratios` is deprecated + and will be removed in 1.9. An `UndefinedMetricWarning` will always be raised in case + of a division by zero. + By :user:`Stefanie Senger `. :pr:`29288` + +- |API| In :meth:`sklearn.metrics.RocCurveDisplay.from_predictions`, + the argument `y_pred` has been renamed to `y_score` to better reflect its purpose. + `y_pred` will be removed in 1.9. + By :user:`Bagus Tris Atmaja ` in :pr:`29865` + +:mod:`sklearn.mixture` +---------------------- + +- |Feature| Added an attribute `lower_bounds_` in the :class:`mixture.BaseMixture` + class to save the list of lower bounds for each iteration thereby providing + insights into the convergence behavior of mixture models like + :class:`mixture.GaussianMixture`. + By :user:`Manideep Yenugula ` :pr:`28559` + +- |Efficiency| Simplified redundant computation when estimating covariances in + :class:`~mixture.GaussianMixture` with a `covariance_type="spherical"` or + `covariance_type="diag"`. + By :user:`Leonce Mekinda ` and :user:`Olivier Grisel ` :pr:`30414` + +- |Efficiency| :class:`~mixture.GaussianMixture` now consistently operates at `float32` + precision when fitted with `float32` data to improve training speed and + memory efficiency. Previously, part of the computation would be implicitly + cast to `float64`. By :user:`Olivier Grisel ` and :user:`Omar Salman + `. :pr:`30415` + +:mod:`sklearn.model_selection` +------------------------------ + +- |Fix| Hyper-parameter optimizers such as :class:`model_selection.GridSearchCV` + now forward `sample_weight` to the scorer even when metadata routing is not enabled. + By :user:`Antoine Baker ` :pr:`30743` + +:mod:`sklearn.multiclass` +------------------------- + +- |Fix| The `predict_proba` method of :class:`sklearn.multiclass.OneVsRestClassifier` now + returns zero for all classes when all inner estimators never predict their positive + class. + By :user:`Luis M. B. Varona `, :user:`Marc Bresson `, and + :user:`JÊrÊmie du Boisberranger `. :pr:`31228` + +:mod:`sklearn.multioutput` +-------------------------- + +- |Enhancement| The parameter `base_estimator` has been deprecated in favour of `estimator` for + :class:`multioutput.RegressorChain` and :class:`multioutput.ClassifierChain`. + By :user:`Success Moses ` and :user:`dikraMasrour ` :pr:`30152` + +:mod:`sklearn.neural_network` +----------------------------- + +- |Feature| Added support for `sample_weight` in :class:`neural_network.MLPClassifier` and + :class:`neural_network.MLPRegressor`. + By :user:`Zach Shu ` and :user:`Christian Lorentzen ` :pr:`30155` + +- |Feature| Added parameter for `loss` in :class:`neural_network.MLPRegressor` with options + `"squared_error"` (default) and `"poisson"` (new). + By :user:`Christian Lorentzen ` :pr:`30712` + +- |Fix| :class:`neural_network.MLPRegressor` now raises an informative error when + `early_stopping` is set and the computed validation set is too small. + By :user:`David Shumway `. :pr:`24788` + +:mod:`sklearn.pipeline` +----------------------- + +- |Enhancement| Expose the ``verbose_feature_names_out`` argument in the + :func:`pipeline.make_union` function, allowing users to control + feature name uniqueness in the :class:`pipeline.FeatureUnion`. + By :user:`Abhijeetsingh Meena ` :pr:`30406` + +:mod:`sklearn.preprocessing` +---------------------------- + +- |Enhancement| :class:`preprocessing.KBinsDiscretizer` with `strategy="uniform"` now + accepts `sample_weight`. Additionally with `strategy="quantile"` the + `quantile_method` can now be specified (in the future + `quantile_method="averaged_inverted_cdf"` will become the default). + By :user:`Shruti Nath ` and :user:`Olivier Grisel + ` :pr:`29907` + +- |Fix| :class:`preprocessing.KBinsDiscretizer` now uses weighted resampling when + sample weights are given and subsampling is used. This may change results + even when not using sample weights, although in absolute and not in terms + of statistical properties. + By :user:`Shruti Nath ` and :user:`JÊrÊmie du Boisberranger + ` :pr:`29907` + +- |Fix| Now using ``scipy.stats.yeojohnson`` instead of our own implementation of the Yeo-Johnson transform. + Fixed numerical stability (mostly overflows) of the Yeo-Johnson transform with + `PowerTransformer(method="yeo-johnson")` when scipy version is `>= 1.12`. + Initial PR by :user:`Xuefeng Xu ` completed by :user:`Mohamed Yaich `, + :user:`Oussama Er-rabie `, :user:`Mohammed Yaslam Dlimi `, + :user:`Hamza Zaroual `, :user:`Amine Hannoun ` and :user:`Sylvain MariÊ `. :pr:`31227` + +:mod:`sklearn.svm` +------------------ + +- |Fix| :class:`svm.LinearSVC` now properly passes sample weights to + :func:`utils.class_weight.compute_class_weight` when fit with + `class_weight="balanced"`. + By :user:`Shruti Nath ` :pr:`30057` + +:mod:`sklearn.utils` +-------------------- + +- |Enhancement| :func:`utils.multiclass.type_of_target` raises a warning when the number + of unique classes is greater than 50% of the number of samples. This warning is raised + only if `y` has more than 20 samples. + By :user:`Rahil Parikh `. :pr:`26335` + +- |Enhancement| :func: `resample` now handles sample weights which allows + weighted resampling. + By :user:`Shruti Nath ` and :user:`Olivier Grisel + ` :pr:`29907` + +- |Enhancement| :func:`utils.class_weight.compute_class_weight` now properly accounts for + sample weights when using strategy "balanced" to calculate class weights. + By :user:`Shruti Nath ` :pr:`30057` + +- |Enhancement| Warning filters from the main process are propagated to joblib workers. + By `Thomas Fan`_ :pr:`30380` + +- |Enhancement| The private helper function :func:`utils._safe_indexing` now officially supports + pyarrow data. For instance, passing a pyarrow `Table` as `X` in a + :class:`compose.ColumnTransformer` is now possible. + By :user:`Christian Lorentzen ` :pr:`31040` + +- |Fix| In :mod:`utils.estimator_checks` we now enforce for binary classifiers a + binary `y` by taking the minimum as the negative class instead of the first + element, which makes it robust to `y` shuffling. It prevents two checks from + wrongly failing on binary classifiers. + By :user:`Antoine Baker `. :pr:`30775` + +- |Fix| :func:`utils.extmath.randomized_svd` and :func:`utils.extmath.randomized_range_finder` + now validate their input array to fail early with an informative error message on + invalid input. + By :user:`Connor Lane `. :pr:`30819` + +.. rubric:: Code and documentation contributors + +Thanks to everyone who has contributed to the maintenance and improvement of +the project since version 1.6, including: + +4hm3d, Aaron Schumacher, Abhijeetsingh Meena, Acciaro Gennaro Daniele, +Achraf Tasfaout, Adrien Linares, Adrin Jalali, Agriya Khetarpal, Aiden Frank, +Aitsaid Azzedine Idir, ajay-sentry, Akanksha Mhadolkar, Alfredo Saucedo, +Anderson Chaves, Andres Guzman-Ballen, Aniruddha Saha, antoinebaker, Antony +Lee, Arjun S, ArthurDbrn, Arturo, Arturo Amor, ash, Ashton Powell, +ayoub.agouzoul, Bagus Tris Atmaja, Benjamin Danek, Boney Patel, Camille +Troillard, Chems Ben, Christian Lorentzen, Christian Veenhuis, Christine P. +Chai, claudio, Code_Blooded, Colas, Colin Coe, Connor Lane, Corey Farwell, +Daniel Agyapong, Dan Schult, Dea María LÊon, Deepak Saldanha, +dependabot[bot], Dimitri Papadopoulos Orfanos, Dmitry Kobak, Domenico, Elham +Babaei, emelia-hdz, EmilyXinyi, Emma Carballal, Eric Larson, fabianhenning, +Gael Varoquaux, Gil Ramot, Gordon Grey, Goutam, G Sreeja, Guillaume Lemaitre, +Haesun Park, Hanjun Kim, Helder Geovane Gomes de Lima, Henri Bonamy, Hleb +Levitski, Hugo Boulenger, IlyaSolomatin, Irene, JÊrÊmie du Boisberranger, +JÊrôme Dockès, JoaoRodriguesIST, Joel Nothman, Josh, Kevin Klein, Loic +Esteve, Lucas Colley, Luc Rocher, Lucy Liu, Luis M. B. Varona, lunovian, Mamduh +Zabidi, Marc Bresson, Marco Edward Gorelli, Marco Maggi, Maren Westermann, +Marie Sacksick, Martin Jurča, Miguel GonzÃĄlez Duque, Mihir Waknis, Mohamed +Ali SRIR, Mohamed DHIFALLAH, mohammed benyamna, Mohit Singh Thakur, Mounir +Lbath, myenugula, Natalia Mokeeva, Olivier Grisel, omahs, Omar Salman, Pedro +Lopes, Pedro Olivares, Preyas Shah, Radovenchyk, Rahil Parikh, RÊmi Flamary, +Reshama Shaikh, Rishab Saini, rolandrmgservices, SanchitD, Santiago Castro, +Santiago Víquez, scikit-learn-bot, Scott Huberty, Shruti Nath, Siddharth +Bansal, Simarjot Sidhu, Sortofamudkip, sotagg, Sourabh Kumar, Stefan, Stefanie +Senger, Stefano Gaspari, Stephen Pardy, Success Moses, Sylvain Combettes, Tahar +Allouche, Thomas J. Fan, Thomas Li, ThorbenMaa, Tim Head, Umberto Fasci, UV, +Vasco Pereira, Vassilis Margonis, Velislav Babatchev, Victoria Shevchenko, +viktor765, Vipsa Kamani, Virgil Chan, vpz, Xiao Yuan, Yaich Mohamed, Yair +Shimony, Yao Xiao, Yaroslav Halchenko, Yulia Vilensky, Yuvi Panda diff --git a/doc/whats_new/v1.8.rst b/doc/whats_new/v1.8.rst new file mode 100644 index 0000000000000..603373824d395 --- /dev/null +++ b/doc/whats_new/v1.8.rst @@ -0,0 +1,34 @@ +.. include:: _contributors.rst + +.. currentmodule:: sklearn + +.. _release_notes_1_8: + +=========== +Version 1.8 +=========== + +.. + -- UNCOMMENT WHEN 1.8.0 IS RELEASED -- + For a short description of the main highlights of the release, please refer to + :ref:`sphx_glr_auto_examples_release_highlights_plot_release_highlights_1_7_0.py`. + + +.. + DELETE WHEN 1.8.0 IS RELEASED + Since October 2024, DO NOT add your changelog entry in this file. +.. + Instead, create a file named `..rst` in the relevant sub-folder in + `doc/whats_new/upcoming_changes/`. For full details, see: + https://github.com/scikit-learn/scikit-learn/blob/main/doc/whats_new/upcoming_changes/README.md + +.. include:: changelog_legend.inc + +.. towncrier release notes start + +.. rubric:: Code and documentation contributors + +Thanks to everyone who has contributed to the maintenance and improvement of +the project since version 1.7, including: + +TODO: update at the time of the release. diff --git a/examples/README.txt b/examples/README.txt index 958de667a5c69..57a4e7e60eb32 100644 --- a/examples/README.txt +++ b/examples/README.txt @@ -2,3 +2,8 @@ Examples ======== + +This is the gallery of examples that showcase how scikit-learn can be used. Some +examples demonstrate the use of the :ref:`API ` in general and some +demonstrate specific applications in tutorial form. Also check out our +:ref:`user guide ` for more detailed illustrations. diff --git a/examples/applications/plot_cyclical_feature_engineering.py b/examples/applications/plot_cyclical_feature_engineering.py new file mode 100644 index 0000000000000..253316d7dd4fd --- /dev/null +++ b/examples/applications/plot_cyclical_feature_engineering.py @@ -0,0 +1,836 @@ +""" +================================ +Time-related feature engineering +================================ + +This notebook introduces different strategies to leverage time-related features +for a bike sharing demand regression task that is highly dependent on business +cycles (days, weeks, months) and yearly season cycles. + +In the process, we introduce how to perform periodic feature engineering using +the :class:`sklearn.preprocessing.SplineTransformer` class and its +`extrapolation="periodic"` option. + +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Data exploration on the Bike Sharing Demand dataset +# --------------------------------------------------- +# +# We start by loading the data from the OpenML repository. +from sklearn.datasets import fetch_openml + +bike_sharing = fetch_openml("Bike_Sharing_Demand", version=2, as_frame=True) +df = bike_sharing.frame + +# %% +# To get a quick understanding of the periodic patterns of the data, let us +# have a look at the average demand per hour during a week. +# +# Note that the week starts on a Sunday, during the weekend. We can clearly +# distinguish the commute patterns in the morning and evenings of the work days +# and the leisure use of the bikes on the weekends with a more spread peak +# demand around the middle of the days: +import matplotlib.pyplot as plt + +fig, ax = plt.subplots(figsize=(12, 4)) +average_week_demand = df.groupby(["weekday", "hour"])["count"].mean() +average_week_demand.plot(ax=ax) +_ = ax.set( + title="Average hourly bike demand during the week", + xticks=[i * 24 for i in range(7)], + xticklabels=["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"], + xlabel="Time of the week", + ylabel="Number of bike rentals", +) + +# %% +# +# The target of the prediction problem is the absolute count of bike rentals on +# a hourly basis: +df["count"].max() + +# %% +# +# Let us rescale the target variable (number of hourly bike rentals) to predict +# a relative demand so that the mean absolute error is more easily interpreted +# as a fraction of the maximum demand. +# +# .. note:: +# +# The fit method of the models used in this notebook all minimize the +# mean squared error to estimate the conditional mean. +# The absolute error, however, would estimate the conditional median. +# +# Nevertheless, when reporting performance measures on the test set in +# the discussion, we choose to focus on the mean absolute error instead +# of the (root) mean squared error because it is more intuitive to +# interpret. Note, however, that in this study the best models for one +# metric are also the best ones in terms of the other metric. +y = df["count"] / df["count"].max() + +# %% +fig, ax = plt.subplots(figsize=(12, 4)) +y.hist(bins=30, ax=ax) +_ = ax.set( + xlabel="Fraction of rented fleet demand", + ylabel="Number of hours", +) + +# %% +# The input feature data frame is a time annotated hourly log of variables +# describing the weather conditions. It includes both numerical and categorical +# variables. Note that the time information has already been expanded into +# several complementary columns. +# +X = df.drop("count", axis="columns") +X + +# %% +# .. note:: +# +# If the time information was only present as a date or datetime column, we +# could have expanded it into hour-in-the-day, day-in-the-week, +# day-in-the-month, month-in-the-year using pandas: +# https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#time-date-components +# +# We now introspect the distribution of the categorical variables, starting +# with `"weather"`: +# +X["weather"].value_counts() + +# %% +# Since there are only 3 `"heavy_rain"` events, we cannot use this category to +# train machine learning models with cross validation. Instead, we simplify the +# representation by collapsing those into the `"rain"` category. +# +X["weather"] = ( + X["weather"] + .astype(object) + .replace(to_replace="heavy_rain", value="rain") + .astype("category") +) + +# %% +X["weather"].value_counts() + +# %% +# As expected, the `"season"` variable is well balanced: +# +X["season"].value_counts() + +# %% +# Time-based cross-validation +# --------------------------- +# +# Since the dataset is a time-ordered event log (hourly demand), we will use a +# time-sensitive cross-validation splitter to evaluate our demand forecasting +# model as realistically as possible. We use a gap of 2 days between the train +# and test side of the splits. We also limit the training set size to make the +# performance of the CV folds more stable. +# +# 1000 test datapoints should be enough to quantify the performance of the +# model. This represents a bit less than a month and a half of contiguous test +# data: + +from sklearn.model_selection import TimeSeriesSplit + +ts_cv = TimeSeriesSplit( + n_splits=5, + gap=48, + max_train_size=10000, + test_size=1000, +) + +# %% +# Let us manually inspect the various splits to check that the +# `TimeSeriesSplit` works as we expect, starting with the first split: +all_splits = list(ts_cv.split(X, y)) +train_0, test_0 = all_splits[0] + +# %% +X.iloc[test_0] + +# %% +X.iloc[train_0] + +# %% +# We now inspect the last split: +train_4, test_4 = all_splits[4] + +# %% +X.iloc[test_4] + +# %% +X.iloc[train_4] + +# %% +# All is well. We are now ready to do some predictive modeling! +# +# Gradient Boosting +# ----------------- +# +# Gradient Boosting Regression with decision trees is often flexible enough to +# efficiently handle heterogeneous tabular data with a mix of categorical and +# numerical features as long as the number of samples is large enough. +# +# Here, we use the modern +# :class:`~sklearn.ensemble.HistGradientBoostingRegressor` with native support +# for categorical features. Therefore, we only need to set +# `categorical_features="from_dtype"` such that features with categorical dtype +# are considered categorical features. For reference, we extract the categorical +# features from the dataframe based on the dtype. The internal trees use a dedicated +# tree splitting rule for these features. +# +# The numerical variables need no preprocessing and, for the sake of simplicity, +# we only try the default hyper-parameters for this model: +from sklearn.compose import ColumnTransformer +from sklearn.ensemble import HistGradientBoostingRegressor +from sklearn.model_selection import cross_validate +from sklearn.pipeline import make_pipeline + +gbrt = HistGradientBoostingRegressor(categorical_features="from_dtype", random_state=42) +categorical_columns = X.columns[X.dtypes == "category"] +print("Categorical features:", categorical_columns.tolist()) + +# %% +# +# Let's evaluate our gradient boosting model with the mean absolute error of the +# relative demand averaged across our 5 time-based cross-validation splits: +import numpy as np + + +def evaluate(model, X, y, cv, model_prop=None, model_step=None): + cv_results = cross_validate( + model, + X, + y, + cv=cv, + scoring=["neg_mean_absolute_error", "neg_root_mean_squared_error"], + return_estimator=model_prop is not None, + ) + if model_prop is not None: + if model_step is not None: + values = [ + getattr(m[model_step], model_prop) for m in cv_results["estimator"] + ] + else: + values = [getattr(m, model_prop) for m in cv_results["estimator"]] + print(f"Mean model.{model_prop} = {np.mean(values)}") + mae = -cv_results["test_neg_mean_absolute_error"] + rmse = -cv_results["test_neg_root_mean_squared_error"] + print( + f"Mean Absolute Error: {mae.mean():.3f} +/- {mae.std():.3f}\n" + f"Root Mean Squared Error: {rmse.mean():.3f} +/- {rmse.std():.3f}" + ) + + +evaluate(gbrt, X, y, cv=ts_cv, model_prop="n_iter_") + +# %% +# We see that we set `max_iter` large enough such that early stopping took place. +# +# This model has an average error around 4 to 5% of the maximum demand. This is +# quite good for a first trial without any hyper-parameter tuning! We just had +# to make the categorical variables explicit. Note that the time related +# features are passed as is, i.e. without processing them. But this is not much +# of a problem for tree-based models as they can learn a non-monotonic +# relationship between ordinal input features and the target. +# +# This is not the case for linear regression models as we will see in the +# following. +# +# Naive linear regression +# ----------------------- +# +# As usual for linear models, categorical variables need to be one-hot encoded. +# For consistency, we scale the numerical features to the same 0-1 range using +# :class:`~sklearn.preprocessing.MinMaxScaler`, although in this case it does not +# impact the results much because they are already on comparable scales: +from sklearn.linear_model import RidgeCV +from sklearn.preprocessing import MinMaxScaler, OneHotEncoder + +one_hot_encoder = OneHotEncoder(handle_unknown="ignore", sparse_output=False) +alphas = np.logspace(-6, 6, 25) +naive_linear_pipeline = make_pipeline( + ColumnTransformer( + transformers=[ + ("categorical", one_hot_encoder, categorical_columns), + ], + remainder=MinMaxScaler(), + ), + RidgeCV(alphas=alphas), +) + + +evaluate( + naive_linear_pipeline, X, y, cv=ts_cv, model_prop="alpha_", model_step="ridgecv" +) + + +# %% +# It is affirmative to see that the selected `alpha_` is in our specified +# range. +# +# The performance is not good: the average error is around 14% of the maximum +# demand. This is more than three times higher than the average error of the +# gradient boosting model. We can suspect that the naive original encoding +# (merely min-max scaled) of the periodic time-related features might prevent +# the linear regression model to properly leverage the time information: linear +# regression does not automatically model non-monotonic relationships between +# the input features and the target. Non-linear terms have to be engineered in +# the input. +# +# For example, the raw numerical encoding of the `"hour"` feature prevents the +# linear model from recognizing that an increase of hour in the morning from 6 +# to 8 should have a strong positive impact on the number of bike rentals while +# an increase of similar magnitude in the evening from 18 to 20 should have a +# strong negative impact on the predicted number of bike rentals. +# +# Time-steps as categories +# ------------------------ +# +# Since the time features are encoded in a discrete manner using integers (24 +# unique values in the "hours" feature), we could decide to treat those as +# categorical variables using a one-hot encoding and thereby ignore any +# assumption implied by the ordering of the hour values. +# +# Using one-hot encoding for the time features gives the linear model a lot +# more flexibility as we introduce one additional feature per discrete time +# level. +one_hot_linear_pipeline = make_pipeline( + ColumnTransformer( + transformers=[ + ("categorical", one_hot_encoder, categorical_columns), + ("one_hot_time", one_hot_encoder, ["hour", "weekday", "month"]), + ], + remainder=MinMaxScaler(), + ), + RidgeCV(alphas=alphas), +) + +evaluate(one_hot_linear_pipeline, X, y, cv=ts_cv) + +# %% +# The average error rate of this model is 10% which is much better than using +# the original (ordinal) encoding of the time feature, confirming our intuition +# that the linear regression model benefits from the added flexibility to not +# treat time progression in a monotonic manner. +# +# However, this introduces a very large number of new features. If the time of +# the day was represented in minutes since the start of the day instead of +# hours, one-hot encoding would have introduced 1440 features instead of 24. +# This could cause some significant overfitting. To avoid this we could use +# :func:`sklearn.preprocessing.KBinsDiscretizer` instead to re-bin the number +# of levels of fine-grained ordinal or numerical variables while still +# benefitting from the non-monotonic expressivity advantages of one-hot +# encoding. +# +# Finally, we also observe that one-hot encoding completely ignores the +# ordering of the hour levels while this could be an interesting inductive bias +# to preserve to some level. In the following we try to explore smooth, +# non-monotonic encoding that locally preserves the relative ordering of time +# features. +# +# Trigonometric features +# ---------------------- +# +# As a first attempt, we can try to encode each of those periodic features +# using a sine and cosine transformation with the matching period. +# +# Each ordinal time feature is transformed into 2 features that together encode +# equivalent information in a non-monotonic way, and more importantly without +# any jump between the first and the last value of the periodic range. +from sklearn.preprocessing import FunctionTransformer + + +def sin_transformer(period): + return FunctionTransformer(lambda x: np.sin(x / period * 2 * np.pi)) + + +def cos_transformer(period): + return FunctionTransformer(lambda x: np.cos(x / period * 2 * np.pi)) + + +# %% +# +# Let us visualize the effect of this feature expansion on some synthetic hour +# data with a bit of extrapolation beyond hour=23: +import pandas as pd + +hour_df = pd.DataFrame( + np.arange(26).reshape(-1, 1), + columns=["hour"], +) +hour_df["hour_sin"] = sin_transformer(24).fit_transform(hour_df)["hour"] +hour_df["hour_cos"] = cos_transformer(24).fit_transform(hour_df)["hour"] +hour_df.plot(x="hour") +_ = plt.title("Trigonometric encoding for the 'hour' feature") + +# %% +# +# Let's use a 2D scatter plot with the hours encoded as colors to better see +# how this representation maps the 24 hours of the day to a 2D space, akin to +# some sort of a 24 hour version of an analog clock. Note that the "25th" hour +# is mapped back to the 1st hour because of the periodic nature of the +# sine/cosine representation. +fig, ax = plt.subplots(figsize=(7, 5)) +sp = ax.scatter(hour_df["hour_sin"], hour_df["hour_cos"], c=hour_df["hour"]) +ax.set( + xlabel="sin(hour)", + ylabel="cos(hour)", +) +_ = fig.colorbar(sp) + +# %% +# +# We can now build a feature extraction pipeline using this strategy: +cyclic_cossin_transformer = ColumnTransformer( + transformers=[ + ("categorical", one_hot_encoder, categorical_columns), + ("month_sin", sin_transformer(12), ["month"]), + ("month_cos", cos_transformer(12), ["month"]), + ("weekday_sin", sin_transformer(7), ["weekday"]), + ("weekday_cos", cos_transformer(7), ["weekday"]), + ("hour_sin", sin_transformer(24), ["hour"]), + ("hour_cos", cos_transformer(24), ["hour"]), + ], + remainder=MinMaxScaler(), +) +cyclic_cossin_linear_pipeline = make_pipeline( + cyclic_cossin_transformer, + RidgeCV(alphas=alphas), +) +evaluate(cyclic_cossin_linear_pipeline, X, y, cv=ts_cv) + + +# %% +# +# The performance of our linear regression model with this simple feature +# engineering is a bit better than using the original ordinal time features but +# worse than using the one-hot encoded time features. We will further analyze +# possible reasons for this disappointing outcome at the end of this notebook. +# +# Periodic spline features +# ------------------------ +# +# We can try an alternative encoding of the periodic time-related features +# using spline transformations with a large enough number of splines, and as a +# result a larger number of expanded features compared to the sine/cosine +# transformation: +from sklearn.preprocessing import SplineTransformer + + +def periodic_spline_transformer(period, n_splines=None, degree=3): + if n_splines is None: + n_splines = period + n_knots = n_splines + 1 # periodic and include_bias is True + return SplineTransformer( + degree=degree, + n_knots=n_knots, + knots=np.linspace(0, period, n_knots).reshape(n_knots, 1), + extrapolation="periodic", + include_bias=True, + ) + + +# %% +# +# Again, let us visualize the effect of this feature expansion on some +# synthetic hour data with a bit of extrapolation beyond hour=23: +hour_df = pd.DataFrame( + np.linspace(0, 26, 1000).reshape(-1, 1), + columns=["hour"], +) +splines = periodic_spline_transformer(24, n_splines=12).fit_transform(hour_df) +splines_df = pd.DataFrame( + splines, + columns=[f"spline_{i}" for i in range(splines.shape[1])], +) +pd.concat([hour_df, splines_df], axis="columns").plot(x="hour", cmap=plt.cm.tab20b) +_ = plt.title("Periodic spline-based encoding for the 'hour' feature") + + +# %% +# Thanks to the use of the `extrapolation="periodic"` parameter, we observe +# that the feature encoding stays smooth when extrapolating beyond midnight. +# +# We can now build a predictive pipeline using this alternative periodic +# feature engineering strategy. +# +# It is possible to use fewer splines than discrete levels for those ordinal +# values. This makes spline-based encoding more efficient than one-hot encoding +# while preserving most of the expressivity: +cyclic_spline_transformer = ColumnTransformer( + transformers=[ + ("categorical", one_hot_encoder, categorical_columns), + ("cyclic_month", periodic_spline_transformer(12, n_splines=6), ["month"]), + ("cyclic_weekday", periodic_spline_transformer(7, n_splines=3), ["weekday"]), + ("cyclic_hour", periodic_spline_transformer(24, n_splines=12), ["hour"]), + ], + remainder=MinMaxScaler(), +) +cyclic_spline_linear_pipeline = make_pipeline( + cyclic_spline_transformer, + RidgeCV(alphas=alphas), +) +evaluate(cyclic_spline_linear_pipeline, X, y, cv=ts_cv) + +# %% +# Spline features make it possible for the linear model to successfully +# leverage the periodic time-related features and reduce the error from ~14% to +# ~10% of the maximum demand, which is similar to what we observed with the +# one-hot encoded features. +# +# Qualitative analysis of the impact of features on linear model predictions +# -------------------------------------------------------------------------- +# +# Here, we want to visualize the impact of the feature engineering choices on +# the time related shape of the predictions. +# +# To do so we consider an arbitrary time-based split to compare the predictions +# on a range of held out data points. +naive_linear_pipeline.fit(X.iloc[train_0], y.iloc[train_0]) +naive_linear_predictions = naive_linear_pipeline.predict(X.iloc[test_0]) + +one_hot_linear_pipeline.fit(X.iloc[train_0], y.iloc[train_0]) +one_hot_linear_predictions = one_hot_linear_pipeline.predict(X.iloc[test_0]) + +cyclic_cossin_linear_pipeline.fit(X.iloc[train_0], y.iloc[train_0]) +cyclic_cossin_linear_predictions = cyclic_cossin_linear_pipeline.predict(X.iloc[test_0]) + +cyclic_spline_linear_pipeline.fit(X.iloc[train_0], y.iloc[train_0]) +cyclic_spline_linear_predictions = cyclic_spline_linear_pipeline.predict(X.iloc[test_0]) + +# %% +# We visualize those predictions by zooming on the last 96 hours (4 days) of +# the test set to get some qualitative insights: +last_hours = slice(-96, None) +fig, ax = plt.subplots(figsize=(12, 4)) +fig.suptitle("Predictions by linear models") +ax.plot( + y.iloc[test_0].values[last_hours], + "x-", + alpha=0.2, + label="Actual demand", + color="black", +) +ax.plot(naive_linear_predictions[last_hours], "x-", label="Ordinal time features") +ax.plot( + cyclic_cossin_linear_predictions[last_hours], + "x-", + label="Trigonometric time features", +) +ax.plot( + cyclic_spline_linear_predictions[last_hours], + "x-", + label="Spline-based time features", +) +ax.plot( + one_hot_linear_predictions[last_hours], + "x-", + label="One-hot time features", +) +_ = ax.legend() + +# %% +# We can draw the following conclusions from the above plot: +# +# - The **raw ordinal time-related features** are problematic because they do +# not capture the natural periodicity: we observe a big jump in the +# predictions at the end of each day when the hour features goes from 23 back +# to 0. We can expect similar artifacts at the end of each week or each year. +# +# - As expected, the **trigonometric features** (sine and cosine) do not have +# these discontinuities at midnight, but the linear regression model fails to +# leverage those features to properly model intra-day variations. +# Using trigonometric features for higher harmonics or additional +# trigonometric features for the natural period with different phases could +# potentially fix this problem. +# +# - the **periodic spline-based features** fix those two problems at once: they +# give more expressivity to the linear model by making it possible to focus +# on specific hours thanks to the use of 12 splines. Furthermore the +# `extrapolation="periodic"` option enforces a smooth representation between +# `hour=23` and `hour=0`. +# +# - The **one-hot encoded features** behave similarly to the periodic +# spline-based features but are more spiky: for instance they can better +# model the morning peak during the week days since this peak lasts shorter +# than an hour. However, we will see in the following that what can be an +# advantage for linear models is not necessarily one for more expressive +# models. + +# %% +# We can also compare the number of features extracted by each feature +# engineering pipeline: +naive_linear_pipeline[:-1].transform(X).shape + +# %% +one_hot_linear_pipeline[:-1].transform(X).shape + +# %% +cyclic_cossin_linear_pipeline[:-1].transform(X).shape + +# %% +cyclic_spline_linear_pipeline[:-1].transform(X).shape + +# %% +# This confirms that the one-hot encoding and the spline encoding strategies +# create a lot more features for the time representation than the alternatives, +# which in turn gives the downstream linear model more flexibility (degrees of +# freedom) to avoid underfitting. +# +# Finally, we observe that none of the linear models can approximate the true +# bike rentals demand, especially for the peaks that can be very sharp at rush +# hours during the working days but much flatter during the week-ends: the most +# accurate linear models based on splines or one-hot encoding tend to forecast +# peaks of commuting-related bike rentals even on the week-ends and +# under-estimate the commuting-related events during the working days. +# +# These systematic prediction errors reveal a form of under-fitting and can be +# explained by the lack of interactions terms between features, e.g. +# "workingday" and features derived from "hours". This issue will be addressed +# in the following section. + +# %% +# Modeling pairwise interactions with splines and polynomial features +# ------------------------------------------------------------------- +# +# Linear models do not automatically capture interaction effects between input +# features. It does not help that some features are marginally non-linear as is +# the case with features constructed by `SplineTransformer` (or one-hot +# encoding or binning). +# +# However, it is possible to use the `PolynomialFeatures` class on coarse +# grained spline encoded hours to model the "workingday"/"hours" interaction +# explicitly without introducing too many new variables: +from sklearn.pipeline import FeatureUnion +from sklearn.preprocessing import PolynomialFeatures + +hour_workday_interaction = make_pipeline( + ColumnTransformer( + [ + ("cyclic_hour", periodic_spline_transformer(24, n_splines=8), ["hour"]), + ("workingday", FunctionTransformer(lambda x: x == "True"), ["workingday"]), + ] + ), + PolynomialFeatures(degree=2, interaction_only=True, include_bias=False), +) + +# %% +# Those features are then combined with the ones already computed in the +# previous spline-base pipeline. We can observe a nice performance improvement +# by modeling this pairwise interaction explicitly: + +cyclic_spline_interactions_pipeline = make_pipeline( + FeatureUnion( + [ + ("marginal", cyclic_spline_transformer), + ("interactions", hour_workday_interaction), + ] + ), + RidgeCV(alphas=alphas), +) +evaluate(cyclic_spline_interactions_pipeline, X, y, cv=ts_cv) + +# %% +# Modeling non-linear feature interactions with kernels +# ----------------------------------------------------- +# +# The previous analysis highlighted the need to model the interactions between +# `"workingday"` and `"hours"`. Another example of a such a non-linear +# interaction that we would like to model could be the impact of the rain that +# might not be the same during the working days and the week-ends and holidays +# for instance. +# +# To model all such interactions, we could either use a polynomial expansion on +# all marginal features at once, after their spline-based expansion. However, +# this would create a quadratic number of features which can cause overfitting +# and computational tractability issues. +# +# Alternatively, we can use the NystrÃļm method to compute an approximate +# polynomial kernel expansion. Let us try the latter: +from sklearn.kernel_approximation import Nystroem + +cyclic_spline_poly_pipeline = make_pipeline( + cyclic_spline_transformer, + Nystroem(kernel="poly", degree=2, n_components=300, random_state=0), + RidgeCV(alphas=alphas), +) +evaluate(cyclic_spline_poly_pipeline, X, y, cv=ts_cv) + +# %% +# +# We observe that this model can almost rival the performance of the gradient +# boosted trees with an average error around 5% of the maximum demand. +# +# Note that while the final step of this pipeline is a linear regression model, +# the intermediate steps such as the spline feature extraction and the NystrÃļm +# kernel approximation are highly non-linear. As a result the compound pipeline +# is much more expressive than a simple linear regression model with raw features. +# +# For the sake of completeness, we also evaluate the combination of one-hot +# encoding and kernel approximation: + +one_hot_poly_pipeline = make_pipeline( + ColumnTransformer( + transformers=[ + ("categorical", one_hot_encoder, categorical_columns), + ("one_hot_time", one_hot_encoder, ["hour", "weekday", "month"]), + ], + remainder="passthrough", + ), + Nystroem(kernel="poly", degree=2, n_components=300, random_state=0), + RidgeCV(alphas=alphas), +) +evaluate(one_hot_poly_pipeline, X, y, cv=ts_cv) + + +# %% +# While one-hot encoded features were competitive with spline-based features +# when using linear models, this is no longer the case when using a low-rank +# approximation of a non-linear kernel: this can be explained by the fact that +# spline features are smoother and allow the kernel approximation to find a +# more expressive decision function. +# +# Let us now have a qualitative look at the predictions of the kernel models +# and of the gradient boosted trees that should be able to better model +# non-linear interactions between features: +gbrt.fit(X.iloc[train_0], y.iloc[train_0]) +gbrt_predictions = gbrt.predict(X.iloc[test_0]) + +one_hot_poly_pipeline.fit(X.iloc[train_0], y.iloc[train_0]) +one_hot_poly_predictions = one_hot_poly_pipeline.predict(X.iloc[test_0]) + +cyclic_spline_poly_pipeline.fit(X.iloc[train_0], y.iloc[train_0]) +cyclic_spline_poly_predictions = cyclic_spline_poly_pipeline.predict(X.iloc[test_0]) + +# %% +# Again we zoom on the last 4 days of the test set: + +last_hours = slice(-96, None) +fig, ax = plt.subplots(figsize=(12, 4)) +fig.suptitle("Predictions by non-linear regression models") +ax.plot( + y.iloc[test_0].values[last_hours], + "x-", + alpha=0.2, + label="Actual demand", + color="black", +) +ax.plot( + gbrt_predictions[last_hours], + "x-", + label="Gradient Boosted Trees", +) +ax.plot( + one_hot_poly_predictions[last_hours], + "x-", + label="One-hot + polynomial kernel", +) +ax.plot( + cyclic_spline_poly_predictions[last_hours], + "x-", + label="Splines + polynomial kernel", +) +_ = ax.legend() + + +# %% +# First, note that trees can naturally model non-linear feature interactions +# since, by default, decision trees are allowed to grow beyond a depth of 2 +# levels. +# +# Here, we can observe that the combinations of spline features and non-linear +# kernels works quite well and can almost rival the accuracy of the gradient +# boosting regression trees. +# +# On the contrary, one-hot encoded time features do not perform that well with +# the low rank kernel model. In particular, they significantly over-estimate +# the low demand hours more than the competing models. +# +# We also observe that none of the models can successfully predict some of the +# peak rentals at the rush hours during the working days. It is possible that +# access to additional features would be required to further improve the +# accuracy of the predictions. For instance, it could be useful to have access +# to the geographical repartition of the fleet at any point in time or the +# fraction of bikes that are immobilized because they need servicing. +# +# Let us finally get a more quantitative look at the prediction errors of those +# three models using the true vs predicted demand scatter plots: +from sklearn.metrics import PredictionErrorDisplay + +fig, axes = plt.subplots(nrows=2, ncols=3, figsize=(13, 7), sharex=True, sharey="row") +fig.suptitle("Non-linear regression models", y=1.0) +predictions = [ + one_hot_poly_predictions, + cyclic_spline_poly_predictions, + gbrt_predictions, +] +labels = [ + "One hot +\npolynomial kernel", + "Splines +\npolynomial kernel", + "Gradient Boosted\nTrees", +] +plot_kinds = ["actual_vs_predicted", "residual_vs_predicted"] +for axis_idx, kind in enumerate(plot_kinds): + for ax, pred, label in zip(axes[axis_idx], predictions, labels): + disp = PredictionErrorDisplay.from_predictions( + y_true=y.iloc[test_0], + y_pred=pred, + kind=kind, + scatter_kwargs={"alpha": 0.3}, + ax=ax, + ) + ax.set_xticks(np.linspace(0, 1, num=5)) + if axis_idx == 0: + ax.set_yticks(np.linspace(0, 1, num=5)) + ax.legend( + ["Best model", label], + loc="upper center", + bbox_to_anchor=(0.5, 1.3), + ncol=2, + ) + ax.set_aspect("equal", adjustable="box") +plt.show() +# %% +# This visualization confirms the conclusions we draw on the previous plot. +# +# All models under-estimate the high demand events (working day rush hours), +# but gradient boosting a bit less so. The low demand events are well predicted +# on average by gradient boosting while the one-hot polynomial regression +# pipeline seems to systematically over-estimate demand in that regime. Overall +# the predictions of the gradient boosted trees are closer to the diagonal than +# for the kernel models. +# +# Concluding remarks +# ------------------ +# +# We note that we could have obtained slightly better results for kernel models +# by using more components (higher rank kernel approximation) at the cost of +# longer fit and prediction durations. For large values of `n_components`, the +# performance of the one-hot encoded features would even match the spline +# features. +# +# The `Nystroem` + `RidgeCV` regressor could also have been replaced by +# :class:`~sklearn.neural_network.MLPRegressor` with one or two hidden layers +# and we would have obtained quite similar results. +# +# The dataset we used in this case study is sampled on a hourly basis. However +# cyclic spline-based features could model time-within-day or time-within-week +# very efficiently with finer-grained time resolutions (for instance with +# measurements taken every minute instead of every hours) without introducing +# more features. One-hot encoding time representations would not offer this +# flexibility. +# +# Finally, in this notebook we used `RidgeCV` because it is very efficient from +# a computational point of view. However, it models the target variable as a +# Gaussian random variable with constant variance. For positive regression +# problems, it is likely that using a Poisson or Gamma distribution would make +# more sense. This could be achieved by using +# `GridSearchCV(TweedieRegressor(power=2), param_grid({"alpha": alphas}))` +# instead of `RidgeCV`. diff --git a/examples/applications/plot_digits_denoising.py b/examples/applications/plot_digits_denoising.py new file mode 100644 index 0000000000000..8ca31da6a74d2 --- /dev/null +++ b/examples/applications/plot_digits_denoising.py @@ -0,0 +1,151 @@ +""" +================================ +Image denoising using kernel PCA +================================ + +This example shows how to use :class:`~sklearn.decomposition.KernelPCA` to +denoise images. In short, we take advantage of the approximation function +learned during `fit` to reconstruct the original image. + +We will compare the results with an exact reconstruction using +:class:`~sklearn.decomposition.PCA`. + +We will use USPS digits dataset to reproduce presented in Sect. 4 of [1]_. + +.. rubric:: References + +.. [1] `BakÄąr, GÃļkhan H., Jason Weston, and Bernhard SchÃļlkopf. + "Learning to find pre-images." + Advances in neural information processing systems 16 (2004): 449-456. + `_ + +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Load the dataset via OpenML +# --------------------------- +# +# The USPS digits datasets is available in OpenML. We use +# :func:`~sklearn.datasets.fetch_openml` to get this dataset. In addition, we +# normalize the dataset such that all pixel values are in the range (0, 1). +import numpy as np + +from sklearn.datasets import fetch_openml +from sklearn.model_selection import train_test_split +from sklearn.preprocessing import MinMaxScaler + +X, y = fetch_openml(data_id=41082, as_frame=False, return_X_y=True) +X = MinMaxScaler().fit_transform(X) + +# %% +# The idea will be to learn a PCA basis (with and without a kernel) on +# noisy images and then use these models to reconstruct and denoise these +# images. +# +# Thus, we split our dataset into a training and testing set composed of 1,000 +# samples for the training and 100 samples for testing. These images are +# noise-free and we will use them to evaluate the efficiency of the denoising +# approaches. In addition, we create a copy of the original dataset and add a +# Gaussian noise. +# +# The idea of this application, is to show that we can denoise corrupted images +# by learning a PCA basis on some uncorrupted images. We will use both a PCA +# and a kernel-based PCA to solve this problem. +X_train, X_test, y_train, y_test = train_test_split( + X, y, stratify=y, random_state=0, train_size=1_000, test_size=100 +) + +rng = np.random.RandomState(0) +noise = rng.normal(scale=0.25, size=X_test.shape) +X_test_noisy = X_test + noise + +noise = rng.normal(scale=0.25, size=X_train.shape) +X_train_noisy = X_train + noise + +# %% +# In addition, we will create a helper function to qualitatively assess the +# image reconstruction by plotting the test images. +import matplotlib.pyplot as plt + + +def plot_digits(X, title): + """Small helper function to plot 100 digits.""" + fig, axs = plt.subplots(nrows=10, ncols=10, figsize=(8, 8)) + for img, ax in zip(X, axs.ravel()): + ax.imshow(img.reshape((16, 16)), cmap="Greys") + ax.axis("off") + fig.suptitle(title, fontsize=24) + + +# %% +# In addition, we will use the mean squared error (MSE) to quantitatively +# assess the image reconstruction. +# +# Let's first have a look to see the difference between noise-free and noisy +# images. We will check the test set in this regard. +plot_digits(X_test, "Uncorrupted test images") +plot_digits( + X_test_noisy, f"Noisy test images\nMSE: {np.mean((X_test - X_test_noisy) ** 2):.2f}" +) + +# %% +# Learn the `PCA` basis +# --------------------- +# +# We can now learn our PCA basis using both a linear PCA and a kernel PCA that +# uses a radial basis function (RBF) kernel. +from sklearn.decomposition import PCA, KernelPCA + +pca = PCA(n_components=32, random_state=42) +kernel_pca = KernelPCA( + n_components=400, + kernel="rbf", + gamma=1e-3, + fit_inverse_transform=True, + alpha=5e-3, + random_state=42, +) + +pca.fit(X_train_noisy) +_ = kernel_pca.fit(X_train_noisy) + +# %% +# Reconstruct and denoise test images +# ----------------------------------- +# +# Now, we can transform and reconstruct the noisy test set. Since we used less +# components than the number of original features, we will get an approximation +# of the original set. Indeed, by dropping the components explaining variance +# in PCA the least, we hope to remove noise. Similar thinking happens in kernel +# PCA; however, we expect a better reconstruction because we use a non-linear +# kernel to learn the PCA basis and a kernel ridge to learn the mapping +# function. +X_reconstructed_kernel_pca = kernel_pca.inverse_transform( + kernel_pca.transform(X_test_noisy) +) +X_reconstructed_pca = pca.inverse_transform(pca.transform(X_test_noisy)) + +# %% +plot_digits(X_test, "Uncorrupted test images") +plot_digits( + X_reconstructed_pca, + f"PCA reconstruction\nMSE: {np.mean((X_test - X_reconstructed_pca) ** 2):.2f}", +) +plot_digits( + X_reconstructed_kernel_pca, + ( + "Kernel PCA reconstruction\n" + f"MSE: {np.mean((X_test - X_reconstructed_kernel_pca) ** 2):.2f}" + ), +) + +# %% +# PCA has a lower MSE than kernel PCA. However, the qualitative analysis might +# not favor PCA instead of kernel PCA. We observe that kernel PCA is able to +# remove background noise and provide a smoother image. +# +# However, it should be noted that the results of the denoising with kernel PCA +# will depend of the parameters `n_components`, `gamma`, and `alpha`. diff --git a/examples/applications/plot_face_recognition.py b/examples/applications/plot_face_recognition.py index 64a18601cd710..add219aed1610 100644 --- a/examples/applications/plot_face_recognition.py +++ b/examples/applications/plot_face_recognition.py @@ -4,49 +4,28 @@ =================================================== The dataset used in this example is a preprocessed excerpt of the -"Labeled Faces in the Wild", aka LFW_: +"Labeled Faces in the Wild", aka LFW: +https://www.kaggle.com/datasets/jessicali9530/lfw-dataset - http://vis-www.cs.umass.edu/lfw/lfw-funneled.tgz (233MB) - -.. _LFW: http://vis-www.cs.umass.edu/lfw/ - -Expected results for the top 5 most represented people in the dataset: - -================== ============ ======= ========== ======= - precision recall f1-score support -================== ============ ======= ========== ======= - Ariel Sharon 0.67 0.92 0.77 13 - Colin Powell 0.75 0.78 0.76 60 - Donald Rumsfeld 0.78 0.67 0.72 27 - George W Bush 0.86 0.86 0.86 146 -Gerhard Schroeder 0.76 0.76 0.76 25 - Hugo Chavez 0.67 0.67 0.67 15 - Tony Blair 0.81 0.69 0.75 36 +""" - avg / total 0.80 0.80 0.80 322 -================== ============ ======= ========== ======= +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -""" +# %% from time import time -import logging + import matplotlib.pyplot as plt +from scipy.stats import loguniform -from sklearn.model_selection import train_test_split -from sklearn.model_selection import GridSearchCV from sklearn.datasets import fetch_lfw_people -from sklearn.metrics import classification_report -from sklearn.metrics import confusion_matrix from sklearn.decomposition import PCA +from sklearn.metrics import ConfusionMatrixDisplay, classification_report +from sklearn.model_selection import RandomizedSearchCV, train_test_split +from sklearn.preprocessing import StandardScaler from sklearn.svm import SVC - -print(__doc__) - -# Display progress logs on stdout -logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') - - -# ############################################################################# +# %% # Download the data, if not already on disk and load it as numpy arrays lfw_people = fetch_lfw_people(min_faces_per_person=70, resize=0.4) @@ -70,24 +49,28 @@ print("n_classes: %d" % n_classes) -# ############################################################################# -# Split into a training set and a test set using a stratified k fold +# %% +# Split into a training set and a test and keep 25% of the data for testing. -# split into a training and testing set X_train, X_test, y_train, y_test = train_test_split( - X, y, test_size=0.25, random_state=42) + X, y, test_size=0.25, random_state=42 +) +scaler = StandardScaler() +X_train = scaler.fit_transform(X_train) +X_test = scaler.transform(X_test) -# ############################################################################# +# %% # Compute a PCA (eigenfaces) on the face dataset (treated as unlabeled # dataset): unsupervised feature extraction / dimensionality reduction + n_components = 150 -print("Extracting the top %d eigenfaces from %d faces" - % (n_components, X_train.shape[0])) +print( + "Extracting the top %d eigenfaces from %d faces" % (n_components, X_train.shape[0]) +) t0 = time() -pca = PCA(n_components=n_components, svd_solver='randomized', - whiten=True).fit(X_train) +pca = PCA(n_components=n_components, svd_solver="randomized", whiten=True).fit(X_train) print("done in %0.3fs" % (time() - t0)) eigenfaces = pca.components_.reshape((n_components, h, w)) @@ -99,15 +82,17 @@ print("done in %0.3fs" % (time() - t0)) -# ############################################################################# +# %% # Train a SVM classification model print("Fitting the classifier to the training set") t0 = time() -param_grid = {'C': [1e3, 5e3, 1e4, 5e4, 1e5], - 'gamma': [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.1], } -clf = GridSearchCV( - SVC(kernel='rbf', class_weight='balanced'), param_grid +param_grid = { + "C": loguniform(1e3, 1e5), + "gamma": loguniform(1e-4, 1e-1), +} +clf = RandomizedSearchCV( + SVC(kernel="rbf", class_weight="balanced"), param_grid, n_iter=10 ) clf = clf.fit(X_train_pca, y_train) print("done in %0.3fs" % (time() - t0)) @@ -115,7 +100,7 @@ print(clf.best_estimator_) -# ############################################################################# +# %% # Quantitative evaluation of the model quality on the test set print("Predicting people's names on the test set") @@ -124,16 +109,21 @@ print("done in %0.3fs" % (time() - t0)) print(classification_report(y_test, y_pred, target_names=target_names)) -print(confusion_matrix(y_test, y_pred, labels=range(n_classes))) +ConfusionMatrixDisplay.from_estimator( + clf, X_test_pca, y_test, display_labels=target_names, xticks_rotation="vertical" +) +plt.tight_layout() +plt.show() -# ############################################################################# +# %% # Qualitative evaluation of the predictions using matplotlib + def plot_gallery(images, titles, h, w, n_row=3, n_col=4): """Helper function to plot a gallery of portraits""" plt.figure(figsize=(1.8 * n_col, 2.4 * n_row)) - plt.subplots_adjust(bottom=0, left=.01, right=.99, top=.90, hspace=.35) + plt.subplots_adjust(bottom=0, left=0.01, right=0.99, top=0.90, hspace=0.35) for i in range(n_row * n_col): plt.subplot(n_row, n_col, i + 1) plt.imshow(images[i].reshape((h, w)), cmap=plt.cm.gray) @@ -142,21 +132,31 @@ def plot_gallery(images, titles, h, w, n_row=3, n_col=4): plt.yticks(()) +# %% # plot the result of the prediction on a portion of the test set + def title(y_pred, y_test, target_names, i): - pred_name = target_names[y_pred[i]].rsplit(' ', 1)[-1] - true_name = target_names[y_test[i]].rsplit(' ', 1)[-1] - return 'predicted: %s\ntrue: %s' % (pred_name, true_name) + pred_name = target_names[y_pred[i]].rsplit(" ", 1)[-1] + true_name = target_names[y_test[i]].rsplit(" ", 1)[-1] + return "predicted: %s\ntrue: %s" % (pred_name, true_name) -prediction_titles = [title(y_pred, y_test, target_names, i) - for i in range(y_pred.shape[0])] -plot_gallery(X_test, prediction_titles, h, w) +prediction_titles = [ + title(y_pred, y_test, target_names, i) for i in range(y_pred.shape[0]) +] +plot_gallery(X_test, prediction_titles, h, w) +# %% # plot the gallery of the most significative eigenfaces eigenface_titles = ["eigenface %d" % i for i in range(eigenfaces.shape[0])] plot_gallery(eigenfaces, eigenface_titles, h, w) plt.show() + +# %% +# Face recognition problem would be much more effectively solved by training +# convolutional neural networks but this family of models is outside of the scope of +# the scikit-learn library. Interested readers should instead try to use pytorch or +# tensorflow to implement such models. diff --git a/examples/applications/plot_model_complexity_influence.py b/examples/applications/plot_model_complexity_influence.py index 927fcd8e85e15..342dd4d899dab 100644 --- a/examples/applications/plot_model_complexity_influence.py +++ b/examples/applications/plot_model_complexity_influence.py @@ -21,8 +21,12 @@ - :class:`~sklearn.svm.NuSVR` (for regression data) which implements Nu support vector regression; - - :class:`~sklearn.ensemble.GradientBoostingRegressor` (for regression - data) which builds an additive model in a forward stage-wise fashion. + - :class:`~sklearn.ensemble.GradientBoostingRegressor` builds an additive + model in a forward stage-wise fashion. Notice that + :class:`~sklearn.ensemble.HistGradientBoostingRegressor` is much faster + than :class:`~sklearn.ensemble.GradientBoostingRegressor` starting with + intermediate datasets (`n_samples >= 10_000`), which is not the case for + this example. We make the model complexity vary through the choice of relevant model @@ -32,25 +36,20 @@ """ -print(__doc__) - -# Authors: Eustache Diemert -# Maria Telenczuk -# Guillaume Lemaitre -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import time -import numpy as np + import matplotlib.pyplot as plt +import numpy as np from sklearn import datasets -from sklearn.utils import shuffle -from sklearn.metrics import mean_squared_error -from sklearn.svm import NuSVR from sklearn.ensemble import GradientBoostingRegressor from sklearn.linear_model import SGDClassifier -from sklearn.metrics import hamming_loss - +from sklearn.metrics import hamming_loss, mean_squared_error +from sklearn.model_selection import train_test_split +from sklearn.svm import NuSVR # Initialize random generator np.random.seed(0) @@ -72,23 +71,23 @@ def generate_data(case): """Generate regression/classification data.""" - if case == 'regression': + if case == "regression": X, y = datasets.load_diabetes(return_X_y=True) - elif case == 'classification': - X, y = datasets.fetch_20newsgroups_vectorized(subset='all', - return_X_y=True) - X, y = shuffle(X, y) - offset = int(X.shape[0] * 0.8) - X_train, y_train = X[:offset], y[:offset] - X_test, y_test = X[offset:], y[offset:] - - data = {'X_train': X_train, 'X_test': X_test, 'y_train': y_train, - 'y_test': y_test} + train_size = 0.8 + elif case == "classification": + X, y = datasets.fetch_20newsgroups_vectorized(subset="all", return_X_y=True) + train_size = 0.4 # to make the example run faster + + X_train, X_test, y_train, y_test = train_test_split( + X, y, train_size=train_size, random_state=0 + ) + + data = {"X_train": X_train, "X_test": X_test, "y_train": y_train, "y_test": y_test} return data -regression_data = generate_data('regression') -classification_data = generate_data('classification') +regression_data = generate_data("regression") +classification_data = generate_data("classification") ############################################################################## @@ -110,26 +109,33 @@ def benchmark_influence(conf): prediction_times = [] prediction_powers = [] complexities = [] - for param_value in conf['changing_param_values']: - conf['tuned_params'][conf['changing_param']] = param_value - estimator = conf['estimator'](**conf['tuned_params']) + for param_value in conf["changing_param_values"]: + conf["tuned_params"][conf["changing_param"]] = param_value + estimator = conf["estimator"](**conf["tuned_params"]) print("Benchmarking %s" % estimator) - estimator.fit(conf['data']['X_train'], conf['data']['y_train']) - conf['postfit_hook'](estimator) - complexity = conf['complexity_computer'](estimator) + estimator.fit(conf["data"]["X_train"], conf["data"]["y_train"]) + conf["postfit_hook"](estimator) + complexity = conf["complexity_computer"](estimator) complexities.append(complexity) start_time = time.time() - for _ in range(conf['n_samples']): - y_pred = estimator.predict(conf['data']['X_test']) - elapsed_time = (time.time() - start_time) / float(conf['n_samples']) + for _ in range(conf["n_samples"]): + y_pred = estimator.predict(conf["data"]["X_test"]) + elapsed_time = (time.time() - start_time) / float(conf["n_samples"]) prediction_times.append(elapsed_time) - pred_score = conf['prediction_performance_computer']( - conf['data']['y_test'], y_pred) + pred_score = conf["prediction_performance_computer"]( + conf["data"]["y_test"], y_pred + ) prediction_powers.append(pred_score) - print("Complexity: %d | %s: %.4f | Pred. Time: %fs\n" % ( - complexity, conf['prediction_performance_label'], pred_score, - elapsed_time)) + print( + "Complexity: %d | %s: %.4f | Pred. Time: %fs\n" + % ( + complexity, + conf["prediction_performance_label"], + pred_score, + elapsed_time, + ) + ) return prediction_powers, prediction_times, complexities @@ -147,46 +153,63 @@ def benchmark_influence(conf): # different data. # + def _count_nonzero_coefficients(estimator): a = estimator.coef_.toarray() return np.count_nonzero(a) configurations = [ - {'estimator': SGDClassifier, - 'tuned_params': {'penalty': 'elasticnet', 'alpha': 0.001, 'loss': - 'modified_huber', 'fit_intercept': True, 'tol': 1e-3}, - 'changing_param': 'l1_ratio', - 'changing_param_values': [0.25, 0.5, 0.75, 0.9], - 'complexity_label': 'non_zero coefficients', - 'complexity_computer': _count_nonzero_coefficients, - 'prediction_performance_computer': hamming_loss, - 'prediction_performance_label': 'Hamming Loss (Misclassification Ratio)', - 'postfit_hook': lambda x: x.sparsify(), - 'data': classification_data, - 'n_samples': 30}, - {'estimator': NuSVR, - 'tuned_params': {'C': 1e3, 'gamma': 2 ** -15}, - 'changing_param': 'nu', - 'changing_param_values': [0.1, 0.25, 0.5, 0.75, 0.9], - 'complexity_label': 'n_support_vectors', - 'complexity_computer': lambda x: len(x.support_vectors_), - 'data': regression_data, - 'postfit_hook': lambda x: x, - 'prediction_performance_computer': mean_squared_error, - 'prediction_performance_label': 'MSE', - 'n_samples': 30}, - {'estimator': GradientBoostingRegressor, - 'tuned_params': {'loss': 'ls'}, - 'changing_param': 'n_estimators', - 'changing_param_values': [10, 50, 100, 200, 500], - 'complexity_label': 'n_trees', - 'complexity_computer': lambda x: x.n_estimators, - 'data': regression_data, - 'postfit_hook': lambda x: x, - 'prediction_performance_computer': mean_squared_error, - 'prediction_performance_label': 'MSE', - 'n_samples': 30}, + { + "estimator": SGDClassifier, + "tuned_params": { + "penalty": "elasticnet", + "alpha": 0.001, + "loss": "modified_huber", + "fit_intercept": True, + "tol": 1e-1, + "n_iter_no_change": 2, + }, + "changing_param": "l1_ratio", + "changing_param_values": [0.25, 0.5, 0.75, 0.9], + "complexity_label": "non_zero coefficients", + "complexity_computer": _count_nonzero_coefficients, + "prediction_performance_computer": hamming_loss, + "prediction_performance_label": "Hamming Loss (Misclassification Ratio)", + "postfit_hook": lambda x: x.sparsify(), + "data": classification_data, + "n_samples": 5, + }, + { + "estimator": NuSVR, + "tuned_params": {"C": 1e3, "gamma": 2**-15}, + "changing_param": "nu", + "changing_param_values": [0.05, 0.1, 0.2, 0.35, 0.5], + "complexity_label": "n_support_vectors", + "complexity_computer": lambda x: len(x.support_vectors_), + "data": regression_data, + "postfit_hook": lambda x: x, + "prediction_performance_computer": mean_squared_error, + "prediction_performance_label": "MSE", + "n_samples": 15, + }, + { + "estimator": GradientBoostingRegressor, + "tuned_params": { + "loss": "squared_error", + "learning_rate": 0.05, + "max_depth": 2, + }, + "changing_param": "n_estimators", + "changing_param_values": [10, 25, 50, 75, 100], + "complexity_label": "n_trees", + "complexity_computer": lambda x: x.n_estimators, + "data": regression_data, + "postfit_hook": lambda x: x, + "prediction_performance_computer": mean_squared_error, + "prediction_performance_label": "MSE", + "n_samples": 15, + }, ] @@ -209,6 +232,7 @@ def _count_nonzero_coefficients(estimator): # ensemble is not as detrimental. # + def plot_influence(conf, mse_values, prediction_times, complexities): """ Plot influence of model complexity on both accuracy and latency. @@ -219,41 +243,41 @@ def plot_influence(conf, mse_values, prediction_times, complexities): # first axes (prediction error) ax1 = fig.add_subplot(111) - line1 = ax1.plot(complexities, mse_values, c='tab:blue', ls='-')[0] - ax1.set_xlabel('Model Complexity (%s)' % conf['complexity_label']) - y1_label = conf['prediction_performance_label'] + line1 = ax1.plot(complexities, mse_values, c="tab:blue", ls="-")[0] + ax1.set_xlabel("Model Complexity (%s)" % conf["complexity_label"]) + y1_label = conf["prediction_performance_label"] ax1.set_ylabel(y1_label) - ax1.spines['left'].set_color(line1.get_color()) + ax1.spines["left"].set_color(line1.get_color()) ax1.yaxis.label.set_color(line1.get_color()) - ax1.tick_params(axis='y', colors=line1.get_color()) + ax1.tick_params(axis="y", colors=line1.get_color()) # second axes (latency) ax2 = fig.add_subplot(111, sharex=ax1, frameon=False) - line2 = ax2.plot(complexities, prediction_times, c='tab:orange', ls='-')[0] + line2 = ax2.plot(complexities, prediction_times, c="tab:orange", ls="-")[0] ax2.yaxis.tick_right() ax2.yaxis.set_label_position("right") y2_label = "Time (s)" ax2.set_ylabel(y2_label) - ax1.spines['right'].set_color(line2.get_color()) + ax1.spines["right"].set_color(line2.get_color()) ax2.yaxis.label.set_color(line2.get_color()) - ax2.tick_params(axis='y', colors=line2.get_color()) + ax2.tick_params(axis="y", colors=line2.get_color()) - plt.legend((line1, line2), ("prediction error", "latency"), - loc='upper right') + plt.legend( + (line1, line2), ("prediction error", "prediction latency"), loc="upper center" + ) - plt.title("Influence of varying '%s' on %s" % (conf['changing_param'], - conf['estimator'].__name__)) + plt.title( + "Influence of varying '%s' on %s" + % (conf["changing_param"], conf["estimator"].__name__) + ) for conf in configurations: - prediction_performances, prediction_times, complexities = \ - benchmark_influence(conf) - plot_influence(conf, prediction_performances, prediction_times, - complexities) + prediction_performances, prediction_times, complexities = benchmark_influence(conf) + plot_influence(conf, prediction_performances, prediction_times, complexities) plt.show() - ############################################################################## # Conclusion # ---------- diff --git a/examples/applications/plot_out_of_core_classification.py b/examples/applications/plot_out_of_core_classification.py index 14fb2d260a61f..ad0ff9638e41c 100644 --- a/examples/applications/plot_out_of_core_classification.py +++ b/examples/applications/plot_out_of_core_classification.py @@ -11,37 +11,36 @@ HashingVectorizer that will project each example into the same feature space. This is especially useful in the case of text classification where new features (words) may appear in each batch. + """ -# Authors: Eustache Diemert -# @FedericoV -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -from glob import glob import itertools -import os.path import re +import sys import tarfile import time -import sys +from hashlib import sha256 +from html.parser import HTMLParser +from pathlib import Path +from urllib.request import urlretrieve -import numpy as np import matplotlib.pyplot as plt +import numpy as np from matplotlib import rcParams -from html.parser import HTMLParser -from urllib.request import urlretrieve from sklearn.datasets import get_data_home from sklearn.feature_extraction.text import HashingVectorizer -from sklearn.linear_model import SGDClassifier -from sklearn.linear_model import PassiveAggressiveClassifier -from sklearn.linear_model import Perceptron +from sklearn.linear_model import PassiveAggressiveClassifier, Perceptron, SGDClassifier from sklearn.naive_bayes import MultinomialNB def _not_in_sphinx(): # Hack to detect whether we are running by the sphinx builder - return '__file__' in globals() + return "__file__" in globals() + # %% # Reuters Dataset related routines @@ -52,21 +51,20 @@ def _not_in_sphinx(): # run. - class ReutersParser(HTMLParser): """Utility class to parse a SGML file and yield documents one at a time.""" - def __init__(self, encoding='latin-1'): + def __init__(self, encoding="latin-1"): HTMLParser.__init__(self) self._reset() self.encoding = encoding def handle_starttag(self, tag, attrs): - method = 'start_' + tag + method = "start_" + tag getattr(self, method, lambda x: None)(attrs) def handle_endtag(self, tag): - method = 'end_' + tag + method = "end_" + tag getattr(self, method, lambda: None)() def _reset(self): @@ -100,10 +98,10 @@ def start_reuters(self, attributes): pass def end_reuters(self): - self.body = re.sub(r'\s+', r' ', self.body) - self.docs.append({'title': self.title, - 'body': self.body, - 'topics': self.topics}) + self.body = re.sub(r"\s+", r" ", self.body) + self.docs.append( + {"title": self.title, "body": self.body, "topics": self.topics} + ) self._reset() def start_title(self, attributes): @@ -144,37 +142,42 @@ def stream_reuters_documents(data_path=None): """ - DOWNLOAD_URL = ('http://archive.ics.uci.edu/ml/machine-learning-databases/' - 'reuters21578-mld/reuters21578.tar.gz') - ARCHIVE_FILENAME = 'reuters21578.tar.gz' + DOWNLOAD_URL = "https://kdd.ics.uci.edu/databases/reuters21578/reuters21578.tar.gz" + ARCHIVE_SHA256 = "3bae43c9b14e387f76a61b6d82bf98a4fb5d3ef99ef7e7075ff2ccbcf59f9d30" + ARCHIVE_FILENAME = "reuters21578.tar.gz" if data_path is None: - data_path = os.path.join(get_data_home(), "reuters") - if not os.path.exists(data_path): + data_path = Path(get_data_home()) / "reuters" + else: + data_path = Path(data_path) + if not data_path.exists(): """Download the dataset.""" - print("downloading dataset (once and for all) into %s" % - data_path) - os.mkdir(data_path) + print("downloading dataset (once and for all) into %s" % data_path) + data_path.mkdir(parents=True, exist_ok=True) def progress(blocknum, bs, size): - total_sz_mb = '%.2f MB' % (size / 1e6) - current_sz_mb = '%.2f MB' % ((blocknum * bs) / 1e6) + total_sz_mb = "%.2f MB" % (size / 1e6) + current_sz_mb = "%.2f MB" % ((blocknum * bs) / 1e6) if _not_in_sphinx(): - sys.stdout.write( - '\rdownloaded %s / %s' % (current_sz_mb, total_sz_mb)) + sys.stdout.write("\rdownloaded %s / %s" % (current_sz_mb, total_sz_mb)) + + archive_path = data_path / ARCHIVE_FILENAME - archive_path = os.path.join(data_path, ARCHIVE_FILENAME) - urlretrieve(DOWNLOAD_URL, filename=archive_path, - reporthook=progress) + urlretrieve(DOWNLOAD_URL, filename=archive_path, reporthook=progress) if _not_in_sphinx(): - sys.stdout.write('\r') + sys.stdout.write("\r") + + # Check that the archive was not tampered: + assert sha256(archive_path.read_bytes()).hexdigest() == ARCHIVE_SHA256 + print("untarring Reuters dataset...") - tarfile.open(archive_path, 'r:gz').extractall(data_path) + with tarfile.open(archive_path, "r:gz") as fp: + fp.extractall(data_path, filter="data") print("done.") parser = ReutersParser() - for filename in glob(os.path.join(data_path, "*.sgm")): - for doc in parser.parse(open(filename, 'rb')): + for filename in data_path.glob("*.sgm"): + for doc in parser.parse(open(filename, "rb")): yield doc @@ -185,8 +188,9 @@ def progress(blocknum, bs, size): # Create the vectorizer and limit the number of features to a reasonable # maximum -vectorizer = HashingVectorizer(decode_error='ignore', n_features=2 ** 18, - alternate_sign=False) +vectorizer = HashingVectorizer( + decode_error="ignore", n_features=2**18, alternate_sign=False +) # Iterator over parsed Reuters SGML files. @@ -197,14 +201,14 @@ def progress(blocknum, bs, size): # files. For other datasets, one should take care of creating a test set with # a realistic portion of positive instances. all_classes = np.array([0, 1]) -positive_class = 'acq' +positive_class = "acq" # Here are some classifiers that support the `partial_fit` method partial_fit_classifiers = { - 'SGD': SGDClassifier(max_iter=5), - 'Perceptron': Perceptron(), - 'NB Multinomial': MultinomialNB(alpha=0.01), - 'Passive-Aggressive': PassiveAggressiveClassifier(), + "SGD": SGDClassifier(max_iter=5), + "Perceptron": Perceptron(), + "NB Multinomial": MultinomialNB(alpha=0.01), + "Passive-Aggressive": PassiveAggressiveClassifier(), } @@ -214,9 +218,11 @@ def get_minibatch(doc_iter, size, pos_class=positive_class): Note: size is before excluding invalid docs with no topics assigned. """ - data = [('{title}\n\n{body}'.format(**doc), pos_class in doc['topics']) - for doc in itertools.islice(doc_iter, size) - if doc['topics']] + data = [ + ("{title}\n\n{body}".format(**doc), pos_class in doc["topics"]) + for doc in itertools.islice(doc_iter, size) + if doc["topics"] + ] if not len(data): return np.asarray([], dtype=int), np.asarray([], dtype=int) X_text, y = zip(*data) @@ -232,7 +238,7 @@ def iter_minibatches(doc_iter, minibatch_size): # test data statistics -test_stats = {'n_test': 0, 'n_test_pos': 0} +test_stats = {"n_test": 0, "n_test_pos": 0} # First we hold out a number of examples to estimate accuracy n_test_documents = 1000 @@ -242,28 +248,34 @@ def iter_minibatches(doc_iter, minibatch_size): tick = time.time() X_test = vectorizer.transform(X_test_text) vectorizing_time = time.time() - tick -test_stats['n_test'] += len(y_test) -test_stats['n_test_pos'] += sum(y_test) +test_stats["n_test"] += len(y_test) +test_stats["n_test_pos"] += sum(y_test) print("Test set is %d documents (%d positive)" % (len(y_test), sum(y_test))) def progress(cls_name, stats): """Report progress information, return a string.""" - duration = time.time() - stats['t0'] + duration = time.time() - stats["t0"] s = "%20s classifier : \t" % cls_name s += "%(n_train)6d train docs (%(n_train_pos)6d positive) " % stats s += "%(n_test)6d test docs (%(n_test_pos)6d positive) " % test_stats s += "accuracy: %(accuracy).3f " % stats - s += "in %.2fs (%5d docs/s)" % (duration, stats['n_train'] / duration) + s += "in %.2fs (%5d docs/s)" % (duration, stats["n_train"] / duration) return s cls_stats = {} for cls_name in partial_fit_classifiers: - stats = {'n_train': 0, 'n_train_pos': 0, - 'accuracy': 0.0, 'accuracy_history': [(0, 0)], 't0': time.time(), - 'runtime_history': [(0, 0)], 'total_fit_time': 0.0} + stats = { + "n_train": 0, + "n_train_pos": 0, + "accuracy": 0.0, + "accuracy_history": [(0, 0)], + "t0": time.time(), + "runtime_history": [(0, 0)], + "total_fit_time": 0.0, + } cls_stats[cls_name] = stats get_minibatch(data_stream, n_test_documents) @@ -281,7 +293,6 @@ def progress(cls_name, stats): # Main loop : iterate on mini-batches of examples for i, (X_train_text, y_train) in enumerate(minibatch_iterators): - tick = time.time() X_train = vectorizer.transform(X_train_text) total_vect_time += time.time() - tick @@ -292,23 +303,24 @@ def progress(cls_name, stats): cls.partial_fit(X_train, y_train, classes=all_classes) # accumulate test accuracy stats - cls_stats[cls_name]['total_fit_time'] += time.time() - tick - cls_stats[cls_name]['n_train'] += X_train.shape[0] - cls_stats[cls_name]['n_train_pos'] += sum(y_train) + cls_stats[cls_name]["total_fit_time"] += time.time() - tick + cls_stats[cls_name]["n_train"] += X_train.shape[0] + cls_stats[cls_name]["n_train_pos"] += sum(y_train) tick = time.time() - cls_stats[cls_name]['accuracy'] = cls.score(X_test, y_test) - cls_stats[cls_name]['prediction_time'] = time.time() - tick - acc_history = (cls_stats[cls_name]['accuracy'], - cls_stats[cls_name]['n_train']) - cls_stats[cls_name]['accuracy_history'].append(acc_history) - run_history = (cls_stats[cls_name]['accuracy'], - total_vect_time + cls_stats[cls_name]['total_fit_time']) - cls_stats[cls_name]['runtime_history'].append(run_history) + cls_stats[cls_name]["accuracy"] = cls.score(X_test, y_test) + cls_stats[cls_name]["prediction_time"] = time.time() - tick + acc_history = (cls_stats[cls_name]["accuracy"], cls_stats[cls_name]["n_train"]) + cls_stats[cls_name]["accuracy_history"].append(acc_history) + run_history = ( + cls_stats[cls_name]["accuracy"], + total_vect_time + cls_stats[cls_name]["total_fit_time"], + ) + cls_stats[cls_name]["runtime_history"].append(run_history) if i % 3 == 0: print(progress(cls_name, cls_stats[cls_name])) if i % 3 == 0: - print('\n') + print("\n") # %% @@ -327,64 +339,66 @@ def plot_accuracy(x, y, x_legend): """Plot accuracy as a function of x.""" x = np.array(x) y = np.array(y) - plt.title('Classification accuracy as a function of %s' % x_legend) - plt.xlabel('%s' % x_legend) - plt.ylabel('Accuracy') + plt.title("Classification accuracy as a function of %s" % x_legend) + plt.xlabel("%s" % x_legend) + plt.ylabel("Accuracy") plt.grid(True) plt.plot(x, y) -rcParams['legend.fontsize'] = 10 +rcParams["legend.fontsize"] = 10 cls_names = list(sorted(cls_stats.keys())) # Plot accuracy evolution plt.figure() for _, stats in sorted(cls_stats.items()): # Plot accuracy evolution with #examples - accuracy, n_examples = zip(*stats['accuracy_history']) + accuracy, n_examples = zip(*stats["accuracy_history"]) plot_accuracy(n_examples, accuracy, "training examples (#)") ax = plt.gca() ax.set_ylim((0.8, 1)) -plt.legend(cls_names, loc='best') +plt.legend(cls_names, loc="best") plt.figure() for _, stats in sorted(cls_stats.items()): # Plot accuracy evolution with runtime - accuracy, runtime = zip(*stats['runtime_history']) - plot_accuracy(runtime, accuracy, 'runtime (s)') + accuracy, runtime = zip(*stats["runtime_history"]) + plot_accuracy(runtime, accuracy, "runtime (s)") ax = plt.gca() ax.set_ylim((0.8, 1)) -plt.legend(cls_names, loc='best') +plt.legend(cls_names, loc="best") # Plot fitting times plt.figure() fig = plt.gcf() -cls_runtime = [stats['total_fit_time'] - for cls_name, stats in sorted(cls_stats.items())] +cls_runtime = [stats["total_fit_time"] for cls_name, stats in sorted(cls_stats.items())] cls_runtime.append(total_vect_time) -cls_names.append('Vectorization') -bar_colors = ['b', 'g', 'r', 'c', 'm', 'y'] +cls_names.append("Vectorization") +bar_colors = ["b", "g", "r", "c", "m", "y"] ax = plt.subplot(111) -rectangles = plt.bar(range(len(cls_names)), cls_runtime, width=0.5, - color=bar_colors) +rectangles = plt.bar(range(len(cls_names)), cls_runtime, width=0.5, color=bar_colors) ax.set_xticks(np.linspace(0, len(cls_names) - 1, len(cls_names))) ax.set_xticklabels(cls_names, fontsize=10) ymax = max(cls_runtime) * 1.2 ax.set_ylim((0, ymax)) -ax.set_ylabel('runtime (s)') -ax.set_title('Training Times') +ax.set_ylabel("runtime (s)") +ax.set_title("Training Times") def autolabel(rectangles): """attach some text vi autolabel on rectangles.""" for rect in rectangles: height = rect.get_height() - ax.text(rect.get_x() + rect.get_width() / 2., - 1.05 * height, '%.4f' % height, - ha='center', va='bottom') + ax.text( + rect.get_x() + rect.get_width() / 2.0, + 1.05 * height, + "%.4f" % height, + ha="center", + va="bottom", + ) plt.setp(plt.xticks()[1], rotation=30) @@ -397,23 +411,22 @@ def autolabel(rectangles): cls_runtime = [] cls_names = list(sorted(cls_stats.keys())) for cls_name, stats in sorted(cls_stats.items()): - cls_runtime.append(stats['prediction_time']) + cls_runtime.append(stats["prediction_time"]) cls_runtime.append(parsing_time) -cls_names.append('Read/Parse\n+Feat.Extr.') +cls_names.append("Read/Parse\n+Feat.Extr.") cls_runtime.append(vectorizing_time) -cls_names.append('Hashing\n+Vect.') +cls_names.append("Hashing\n+Vect.") ax = plt.subplot(111) -rectangles = plt.bar(range(len(cls_names)), cls_runtime, width=0.5, - color=bar_colors) +rectangles = plt.bar(range(len(cls_names)), cls_runtime, width=0.5, color=bar_colors) ax.set_xticks(np.linspace(0, len(cls_names) - 1, len(cls_names))) ax.set_xticklabels(cls_names, fontsize=8) plt.setp(plt.xticks()[1], rotation=30) ymax = max(cls_runtime) * 1.2 ax.set_ylim((0, ymax)) -ax.set_ylabel('runtime (s)') -ax.set_title('Prediction Times (%d instances)' % n_test_documents) +ax.set_ylabel("runtime (s)") +ax.set_title("Prediction Times (%d instances)" % n_test_documents) autolabel(rectangles) plt.tight_layout() plt.show() diff --git a/examples/applications/plot_outlier_detection_wine.py b/examples/applications/plot_outlier_detection_wine.py index d649cd18a3a84..6940a562119ae 100644 --- a/examples/applications/plot_outlier_detection_wine.py +++ b/examples/applications/plot_outlier_detection_wine.py @@ -7,7 +7,7 @@ on a real data set. It is useful both for outlier detection and for a better understanding of the data structure. -We selected two sets of two variables from the Boston housing data set +We selected two sets of two variables from the Wine data set as an illustration of what kind of analysis can be done with several outlier detection tools. For the purpose of visualization, we are working with two-dimensional examples, but one should be aware that things are @@ -21,82 +21,83 @@ estimation of the data structure, but yet accurate to some extent. The One-Class SVM does not assume any parametric form of the data distribution and can therefore model the complex shape of the data much better. - -First example -------------- -The first example illustrates how the Minimum Covariance Determinant -robust estimator can help concentrate on a relevant cluster when outlying -points exist. Here the empirical covariance estimation is skewed by points -outside of the main cluster. Of course, some screening tools would have pointed -out the presence of two clusters (Support Vector Machines, Gaussian Mixture -Models, univariate outlier detection, ...). But had it been a high-dimensional -example, none of these could be applied that easily. - """ -print(__doc__) -# Author: Virgile Fritsch -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np +# %% +# First example +# ------------- +# +# The first example illustrates how the Minimum Covariance Determinant +# robust estimator can help concentrate on a relevant cluster when outlying +# points exist. Here the empirical covariance estimation is skewed by points +# outside of the main cluster. Of course, some screening tools would have pointed +# out the presence of two clusters (Support Vector Machines, Gaussian Mixture +# Models, univariate outlier detection, ...). But had it been a high-dimensional +# example, none of these could be applied that easily. from sklearn.covariance import EllipticEnvelope +from sklearn.inspection import DecisionBoundaryDisplay from sklearn.svm import OneClassSVM + +estimators = { + "Empirical Covariance": EllipticEnvelope(support_fraction=1.0, contamination=0.25), + "Robust Covariance (Minimum Covariance Determinant)": EllipticEnvelope( + contamination=0.25 + ), + "OCSVM": OneClassSVM(nu=0.25, gamma=0.35), +} + +# %% +import matplotlib.lines as mlines import matplotlib.pyplot as plt -import matplotlib.font_manager -from sklearn.datasets import load_wine -# Define "classifiers" to be used -classifiers = { - "Empirical Covariance": EllipticEnvelope(support_fraction=1., - contamination=0.25), - "Robust Covariance (Minimum Covariance Determinant)": - EllipticEnvelope(contamination=0.25), - "OCSVM": OneClassSVM(nu=0.25, gamma=0.35)} -colors = ['m', 'g', 'b'] -legend1 = {} -legend2 = {} +from sklearn.datasets import load_wine -# Get data -X1 = load_wine()['data'][:, [1, 2]] # two clusters +X = load_wine()["data"][:, [1, 2]] # two clusters +fig, ax = plt.subplots() +colors = ["tab:blue", "tab:orange", "tab:red"] # Learn a frontier for outlier detection with several classifiers -xx1, yy1 = np.meshgrid(np.linspace(0, 6, 500), np.linspace(1, 4.5, 500)) -for i, (clf_name, clf) in enumerate(classifiers.items()): - plt.figure(1) - clf.fit(X1) - Z1 = clf.decision_function(np.c_[xx1.ravel(), yy1.ravel()]) - Z1 = Z1.reshape(xx1.shape) - legend1[clf_name] = plt.contour( - xx1, yy1, Z1, levels=[0], linewidths=2, colors=colors[i]) - -legend1_values_list = list(legend1.values()) -legend1_keys_list = list(legend1.keys()) - -# Plot the results (= shape of the data points cloud) -plt.figure(1) # two clusters -plt.title("Outlier detection on a real data set (wine recognition)") -plt.scatter(X1[:, 0], X1[:, 1], color='black') +legend_lines = [] +for color, (name, estimator) in zip(colors, estimators.items()): + estimator.fit(X) + DecisionBoundaryDisplay.from_estimator( + estimator, + X, + response_method="decision_function", + plot_method="contour", + levels=[0], + colors=color, + ax=ax, + ) + legend_lines.append(mlines.Line2D([], [], color=color, label=name)) + + +ax.scatter(X[:, 0], X[:, 1], color="black") bbox_args = dict(boxstyle="round", fc="0.8") arrow_args = dict(arrowstyle="->") -plt.annotate("outlying points", xy=(4, 2), - xycoords="data", textcoords="data", - xytext=(3, 1.25), bbox=bbox_args, arrowprops=arrow_args) -plt.xlim((xx1.min(), xx1.max())) -plt.ylim((yy1.min(), yy1.max())) -plt.legend((legend1_values_list[0].collections[0], - legend1_values_list[1].collections[0], - legend1_values_list[2].collections[0]), - (legend1_keys_list[0], legend1_keys_list[1], legend1_keys_list[2]), - loc="upper center", - prop=matplotlib.font_manager.FontProperties(size=11)) -plt.ylabel("ash") -plt.xlabel("malic_acid") - -plt.show() +ax.annotate( + "outlying points", + xy=(4, 2), + xycoords="data", + textcoords="data", + xytext=(3, 1.25), + bbox=bbox_args, + arrowprops=arrow_args, +) +ax.legend(handles=legend_lines, loc="upper center") +_ = ax.set( + xlabel="ash", + ylabel="malic_acid", + title="Outlier detection on a real data set (wine recognition)", +) # %% # Second example # -------------- +# # The second example shows the ability of the Minimum Covariance Determinant # robust estimator of covariance to concentrate on the main mode of the data # distribution: the location seems to be well estimated, although the @@ -105,36 +106,32 @@ # capture the real data structure, but the difficulty is to adjust its kernel # bandwidth parameter so as to obtain a good compromise between the shape of # the data scatter matrix and the risk of over-fitting the data. +X = load_wine()["data"][:, [6, 9]] # "banana"-shaped -# Get data -X2 = load_wine()['data'][:, [6, 9]] # "banana"-shaped - +fig, ax = plt.subplots() +colors = ["tab:blue", "tab:orange", "tab:red"] # Learn a frontier for outlier detection with several classifiers -xx2, yy2 = np.meshgrid(np.linspace(-1, 5.5, 500), np.linspace(-2.5, 19, 500)) -for i, (clf_name, clf) in enumerate(classifiers.items()): - plt.figure(2) - clf.fit(X2) - Z2 = clf.decision_function(np.c_[xx2.ravel(), yy2.ravel()]) - Z2 = Z2.reshape(xx2.shape) - legend2[clf_name] = plt.contour( - xx2, yy2, Z2, levels=[0], linewidths=2, colors=colors[i]) - -legend2_values_list = list(legend2.values()) -legend2_keys_list = list(legend2.keys()) - -# Plot the results (= shape of the data points cloud) -plt.figure(2) # "banana" shape -plt.title("Outlier detection on a real data set (wine recognition)") -plt.scatter(X2[:, 0], X2[:, 1], color='black') -plt.xlim((xx2.min(), xx2.max())) -plt.ylim((yy2.min(), yy2.max())) -plt.legend((legend2_values_list[0].collections[0], - legend2_values_list[1].collections[0], - legend2_values_list[2].collections[0]), - (legend2_keys_list[0], legend2_keys_list[1], legend2_keys_list[2]), - loc="upper center", - prop=matplotlib.font_manager.FontProperties(size=11)) -plt.ylabel("color_intensity") -plt.xlabel("flavanoids") +legend_lines = [] +for color, (name, estimator) in zip(colors, estimators.items()): + estimator.fit(X) + DecisionBoundaryDisplay.from_estimator( + estimator, + X, + response_method="decision_function", + plot_method="contour", + levels=[0], + colors=color, + ax=ax, + ) + legend_lines.append(mlines.Line2D([], [], color=color, label=name)) + + +ax.scatter(X[:, 0], X[:, 1], color="black") +ax.legend(handles=legend_lines, loc="upper center") +ax.set( + xlabel="flavanoids", + ylabel="color_intensity", + title="Outlier detection on a real data set (wine recognition)", +) plt.show() diff --git a/examples/applications/plot_prediction_latency.py b/examples/applications/plot_prediction_latency.py index e59402e47fe17..c67fd651cb083 100644 --- a/examples/applications/plot_prediction_latency.py +++ b/examples/applications/plot_prediction_latency.py @@ -13,29 +13,33 @@ """ -# Authors: Eustache Diemert -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +import gc +import time from collections import defaultdict -import time -import gc -import numpy as np import matplotlib.pyplot as plt +import numpy as np -from sklearn.preprocessing import StandardScaler -from sklearn.model_selection import train_test_split from sklearn.datasets import make_regression from sklearn.ensemble import RandomForestRegressor -from sklearn.linear_model import Ridge -from sklearn.linear_model import SGDRegressor +from sklearn.linear_model import Ridge, SGDRegressor +from sklearn.model_selection import train_test_split +from sklearn.preprocessing import StandardScaler from sklearn.svm import SVR from sklearn.utils import shuffle def _not_in_sphinx(): # Hack to detect whether we are running by the sphinx builder - return '__file__' in globals() + return "__file__" in globals() + + +# %% +# Benchmark and plot helper functions +# ----------------------------------- def atomic_benchmark_estimator(estimator, X_test, verbose=False): @@ -48,8 +52,12 @@ def atomic_benchmark_estimator(estimator, X_test, verbose=False): estimator.predict(instance) runtimes[i] = time.time() - start if verbose: - print("atomic_benchmark runtimes:", min(runtimes), np.percentile( - runtimes, 50), max(runtimes)) + print( + "atomic_benchmark runtimes:", + min(runtimes), + np.percentile(runtimes, 50), + max(runtimes), + ) return runtimes @@ -63,8 +71,12 @@ def bulk_benchmark_estimator(estimator, X_test, n_bulk_repeats, verbose): runtimes[i] = time.time() - start runtimes = np.array(list(map(lambda x: x / float(n_instances), runtimes))) if verbose: - print("bulk_benchmark runtimes:", min(runtimes), np.percentile( - runtimes, 50), max(runtimes)) + print( + "bulk_benchmark runtimes:", + min(runtimes), + np.percentile(runtimes, 50), + max(runtimes), + ) return runtimes @@ -85,8 +97,7 @@ def benchmark_estimator(estimator, X_test, n_bulk_repeats=30, verbose=False): """ atomic_runtimes = atomic_benchmark_estimator(estimator, X_test, verbose) - bulk_runtimes = bulk_benchmark_estimator(estimator, X_test, n_bulk_repeats, - verbose) + bulk_runtimes = bulk_benchmark_estimator(estimator, X_test, n_bulk_repeats, verbose) return atomic_runtimes, bulk_runtimes @@ -95,12 +106,14 @@ def generate_dataset(n_train, n_test, n_features, noise=0.1, verbose=False): if verbose: print("generating dataset...") - X, y, coef = make_regression(n_samples=n_train + n_test, - n_features=n_features, noise=noise, coef=True) + X, y, coef = make_regression( + n_samples=n_train + n_test, n_features=n_features, noise=noise, coef=True + ) random_seed = 13 X_train, X_test, y_train, y_test = train_test_split( - X, y, train_size=n_train, test_size=n_test, random_state=random_seed) + X, y, train_size=n_train, test_size=n_test, random_state=random_seed + ) X_train, y_train = shuffle(X_train, y_train, random_state=random_seed) X_scaler = StandardScaler() @@ -130,26 +143,32 @@ def boxplot_runtimes(runtimes, pred_type, configuration): """ fig, ax1 = plt.subplots(figsize=(10, 6)) - bp = plt.boxplot(runtimes, ) - - cls_infos = ['%s\n(%d %s)' % (estimator_conf['name'], - estimator_conf['complexity_computer']( - estimator_conf['instance']), - estimator_conf['complexity_label']) for - estimator_conf in configuration['estimators']] + bp = plt.boxplot( + runtimes, + ) + + cls_infos = [ + "%s\n(%d %s)" + % ( + estimator_conf["name"], + estimator_conf["complexity_computer"](estimator_conf["instance"]), + estimator_conf["complexity_label"], + ) + for estimator_conf in configuration["estimators"] + ] plt.setp(ax1, xticklabels=cls_infos) - plt.setp(bp['boxes'], color='black') - plt.setp(bp['whiskers'], color='black') - plt.setp(bp['fliers'], color='red', marker='+') + plt.setp(bp["boxes"], color="black") + plt.setp(bp["whiskers"], color="black") + plt.setp(bp["fliers"], color="red", marker="+") - ax1.yaxis.grid(True, linestyle='-', which='major', color='lightgrey', - alpha=0.5) + ax1.yaxis.grid(True, linestyle="-", which="major", color="lightgrey", alpha=0.5) ax1.set_axisbelow(True) - ax1.set_title('Prediction Time per Instance - %s, %d feats.' % ( - pred_type.capitalize(), - configuration['n_features'])) - ax1.set_ylabel('Prediction Time (us)') + ax1.set_title( + "Prediction Time per Instance - %s, %d feats." + % (pred_type.capitalize(), configuration["n_features"]) + ) + ax1.set_ylabel("Prediction Time (us)") plt.show() @@ -157,24 +176,24 @@ def boxplot_runtimes(runtimes, pred_type, configuration): def benchmark(configuration): """Run the whole benchmark.""" X_train, y_train, X_test, y_test = generate_dataset( - configuration['n_train'], configuration['n_test'], - configuration['n_features']) + configuration["n_train"], configuration["n_test"], configuration["n_features"] + ) stats = {} - for estimator_conf in configuration['estimators']: - print("Benchmarking", estimator_conf['instance']) - estimator_conf['instance'].fit(X_train, y_train) + for estimator_conf in configuration["estimators"]: + print("Benchmarking", estimator_conf["instance"]) + estimator_conf["instance"].fit(X_train, y_train) gc.collect() - a, b = benchmark_estimator(estimator_conf['instance'], X_test) - stats[estimator_conf['name']] = {'atomic': a, 'bulk': b} + a, b = benchmark_estimator(estimator_conf["instance"], X_test) + stats[estimator_conf["name"]] = {"atomic": a, "bulk": b} - cls_names = [estimator_conf['name'] for estimator_conf in configuration[ - 'estimators']] - runtimes = [1e6 * stats[clf_name]['atomic'] for clf_name in cls_names] - boxplot_runtimes(runtimes, 'atomic', configuration) - runtimes = [1e6 * stats[clf_name]['bulk'] for clf_name in cls_names] - boxplot_runtimes(runtimes, 'bulk (%d)' % configuration['n_test'], - configuration) + cls_names = [ + estimator_conf["name"] for estimator_conf in configuration["estimators"] + ] + runtimes = [1e6 * stats[clf_name]["atomic"] for clf_name in cls_names] + boxplot_runtimes(runtimes, "atomic", configuration) + runtimes = [1e6 * stats[clf_name]["bulk"] for clf_name in cls_names] + boxplot_runtimes(runtimes, "bulk (%d)" % configuration["n_test"], configuration) def n_feature_influence(estimators, n_train, n_test, n_features, percentile): @@ -205,105 +224,125 @@ def n_feature_influence(estimators, n_train, n_test, n_features, percentile): estimator.fit(X_train, y_train) gc.collect() runtimes = bulk_benchmark_estimator(estimator, X_test, 30, False) - percentiles[cls_name][n] = 1e6 * np.percentile(runtimes, - percentile) + percentiles[cls_name][n] = 1e6 * np.percentile(runtimes, percentile) return percentiles def plot_n_features_influence(percentiles, percentile): fig, ax1 = plt.subplots(figsize=(10, 6)) - colors = ['r', 'g', 'b'] + colors = ["r", "g", "b"] for i, cls_name in enumerate(percentiles.keys()): - x = np.array(sorted([n for n in percentiles[cls_name].keys()])) + x = np.array(sorted(percentiles[cls_name].keys())) y = np.array([percentiles[cls_name][n] for n in x]) - plt.plot(x, y, color=colors[i], ) - ax1.yaxis.grid(True, linestyle='-', which='major', color='lightgrey', - alpha=0.5) + plt.plot( + x, + y, + color=colors[i], + ) + ax1.yaxis.grid(True, linestyle="-", which="major", color="lightgrey", alpha=0.5) ax1.set_axisbelow(True) - ax1.set_title('Evolution of Prediction Time with #Features') - ax1.set_xlabel('#Features') - ax1.set_ylabel('Prediction Time at %d%%-ile (us)' % percentile) + ax1.set_title("Evolution of Prediction Time with #Features") + ax1.set_xlabel("#Features") + ax1.set_ylabel("Prediction Time at %d%%-ile (us)" % percentile) plt.show() def benchmark_throughputs(configuration, duration_secs=0.1): """benchmark throughput for different estimators.""" X_train, y_train, X_test, y_test = generate_dataset( - configuration['n_train'], configuration['n_test'], - configuration['n_features']) + configuration["n_train"], configuration["n_test"], configuration["n_features"] + ) throughputs = dict() - for estimator_config in configuration['estimators']: - estimator_config['instance'].fit(X_train, y_train) + for estimator_config in configuration["estimators"]: + estimator_config["instance"].fit(X_train, y_train) start_time = time.time() n_predictions = 0 while (time.time() - start_time) < duration_secs: - estimator_config['instance'].predict(X_test[[0]]) + estimator_config["instance"].predict(X_test[[0]]) n_predictions += 1 - throughputs[estimator_config['name']] = n_predictions / duration_secs + throughputs[estimator_config["name"]] = n_predictions / duration_secs return throughputs def plot_benchmark_throughput(throughputs, configuration): fig, ax = plt.subplots(figsize=(10, 6)) - colors = ['r', 'g', 'b'] - cls_infos = ['%s\n(%d %s)' % (estimator_conf['name'], - estimator_conf['complexity_computer']( - estimator_conf['instance']), - estimator_conf['complexity_label']) for - estimator_conf in configuration['estimators']] - cls_values = [throughputs[estimator_conf['name']] for estimator_conf in - configuration['estimators']] + colors = ["r", "g", "b"] + cls_infos = [ + "%s\n(%d %s)" + % ( + estimator_conf["name"], + estimator_conf["complexity_computer"](estimator_conf["instance"]), + estimator_conf["complexity_label"], + ) + for estimator_conf in configuration["estimators"] + ] + cls_values = [ + throughputs[estimator_conf["name"]] + for estimator_conf in configuration["estimators"] + ] plt.bar(range(len(throughputs)), cls_values, width=0.5, color=colors) ax.set_xticks(np.linspace(0.25, len(throughputs) - 0.75, len(throughputs))) ax.set_xticklabels(cls_infos, fontsize=10) ymax = max(cls_values) * 1.2 ax.set_ylim((0, ymax)) - ax.set_ylabel('Throughput (predictions/sec)') - ax.set_title('Prediction Throughput for different estimators (%d ' - 'features)' % configuration['n_features']) + ax.set_ylabel("Throughput (predictions/sec)") + ax.set_title( + "Prediction Throughput for different estimators (%d features)" + % configuration["n_features"] + ) plt.show() -# ############################################################################# -# Main code - -start_time = time.time() - -# ############################################################################# +# %% # Benchmark bulk/atomic prediction speed for various regressors +# ------------------------------------------------------------- + configuration = { - 'n_train': int(1e3), - 'n_test': int(1e2), - 'n_features': int(1e2), - 'estimators': [ - {'name': 'Linear Model', - 'instance': SGDRegressor(penalty='elasticnet', alpha=0.01, - l1_ratio=0.25, tol=1e-4), - 'complexity_label': 'non-zero coefficients', - 'complexity_computer': lambda clf: np.count_nonzero(clf.coef_)}, - {'name': 'RandomForest', - 'instance': RandomForestRegressor(), - 'complexity_label': 'estimators', - 'complexity_computer': lambda clf: clf.n_estimators}, - {'name': 'SVR', - 'instance': SVR(kernel='rbf'), - 'complexity_label': 'support vectors', - 'complexity_computer': lambda clf: len(clf.support_vectors_)}, - ] + "n_train": int(1e3), + "n_test": int(1e2), + "n_features": int(1e2), + "estimators": [ + { + "name": "Linear Model", + "instance": SGDRegressor( + penalty="elasticnet", alpha=0.01, l1_ratio=0.25, tol=1e-4 + ), + "complexity_label": "non-zero coefficients", + "complexity_computer": lambda clf: np.count_nonzero(clf.coef_), + }, + { + "name": "RandomForest", + "instance": RandomForestRegressor(), + "complexity_label": "estimators", + "complexity_computer": lambda clf: clf.n_estimators, + }, + { + "name": "SVR", + "instance": SVR(kernel="rbf"), + "complexity_label": "support vectors", + "complexity_computer": lambda clf: len(clf.support_vectors_), + }, + ], } benchmark(configuration) -# benchmark n_features influence on prediction speed +# %% +# Benchmark n_features influence on prediction speed +# -------------------------------------------------- + percentile = 90 -percentiles = n_feature_influence({'ridge': Ridge()}, - configuration['n_train'], - configuration['n_test'], - [100, 250, 500], percentile) +percentiles = n_feature_influence( + {"ridge": Ridge()}, + configuration["n_train"], + configuration["n_test"], + [100, 250, 500], + percentile, +) plot_n_features_influence(percentiles, percentile) -# benchmark throughput +# %% +# Benchmark throughput +# -------------------- + throughputs = benchmark_throughputs(configuration) plot_benchmark_throughput(throughputs, configuration) - -stop_time = time.time() -print("example run in %.2fs" % (stop_time - start_time)) diff --git a/examples/applications/plot_species_distribution_modeling.py b/examples/applications/plot_species_distribution_modeling.py index 4216f4fda0c2f..e2edda813c25d 100644 --- a/examples/applications/plot_species_distribution_modeling.py +++ b/examples/applications/plot_species_distribution_modeling.py @@ -4,8 +4,8 @@ ============================= Modeling species' geographic distributions is an important -problem in conservation biology. In this example we -model the geographic distribution of two south american +problem in conservation biology. In this example, we +model the geographic distribution of two South American mammals given past observations and 14 environmental variables. Since we have only positive examples (there are no unsuccessful observations), we cast this problem as a @@ -17,48 +17,46 @@ The two species are: - - `"Bradypus variegatus" - `_ , - the Brown-throated Sloth. +- `Bradypus variegatus + `_, + the brown-throated sloth. - - `"Microryzomys minutus" - `_ , - also known as the Forest Small Rice Rat, a rodent that lives in Peru, - Colombia, Ecuador, Peru, and Venezuela. +- `Microryzomys minutus + `_, + also known as the forest small rice rat, a rodent that lives in Peru, + Colombia, Ecuador, Peru, and Venezuela. References ---------- - * `"Maximum entropy modeling of species geographic distributions" - `_ - S. J. Phillips, R. P. Anderson, R. E. Schapire - Ecological Modelling, - 190:231-259, 2006. +- `"Maximum entropy modeling of species geographic distributions" + `_ + S. J. Phillips, R. P. Anderson, R. E. Schapire - Ecological Modelling, + 190:231-259, 2006. + """ -# Authors: Peter Prettenhofer -# Jake Vanderplas -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause from time import time -import numpy as np import matplotlib.pyplot as plt +import numpy as np -from sklearn.utils import Bunch +from sklearn import metrics, svm from sklearn.datasets import fetch_species_distributions -from sklearn import svm, metrics +from sklearn.utils import Bunch # if basemap is available, we'll use it. # otherwise, we'll improvise later... try: from mpl_toolkits.basemap import Basemap + basemap = True except ImportError: basemap = False -print(__doc__) - def construct_grids(batch): """Construct the map grid from the batch object @@ -93,31 +91,34 @@ def create_species_bunch(species_name, train, test, coverages, xgrid, ygrid): This will use the test/train record arrays to extract the data specific to the given species name. """ - bunch = Bunch(name=' '.join(species_name.split("_")[:2])) - species_name = species_name.encode('ascii') + bunch = Bunch(name=" ".join(species_name.split("_")[:2])) + species_name = species_name.encode("ascii") points = dict(test=test, train=train) for label, pts in points.items(): # choose points associated with the desired species - pts = pts[pts['species'] == species_name] - bunch['pts_%s' % label] = pts + pts = pts[pts["species"] == species_name] + bunch["pts_%s" % label] = pts # determine coverage values for each of the training & testing points - ix = np.searchsorted(xgrid, pts['dd long']) - iy = np.searchsorted(ygrid, pts['dd lat']) - bunch['cov_%s' % label] = coverages[:, -iy, ix].T + ix = np.searchsorted(xgrid, pts["dd long"]) + iy = np.searchsorted(ygrid, pts["dd lat"]) + bunch["cov_%s" % label] = coverages[:, -iy, ix].T return bunch -def plot_species_distribution(species=("bradypus_variegatus_0", - "microryzomys_minutus_0")): +def plot_species_distribution( + species=("bradypus_variegatus_0", "microryzomys_minutus_0"), +): """ Plot the species distribution. """ if len(species) > 2: - print("Note: when more than two species are provided," - " only the first two will be used") + print( + "Note: when more than two species are provided," + " only the first two will be used" + ) t0 = time() @@ -131,19 +132,19 @@ def plot_species_distribution(species=("bradypus_variegatus_0", X, Y = np.meshgrid(xgrid, ygrid[::-1]) # create a bunch for each species - BV_bunch = create_species_bunch(species[0], - data.train, data.test, - data.coverages, xgrid, ygrid) - MM_bunch = create_species_bunch(species[1], - data.train, data.test, - data.coverages, xgrid, ygrid) + BV_bunch = create_species_bunch( + species[0], data.train, data.test, data.coverages, xgrid, ygrid + ) + MM_bunch = create_species_bunch( + species[1], data.train, data.test, data.coverages, xgrid, ygrid + ) # background points (grid coordinates) for evaluation np.random.seed(13) - background_points = np.c_[np.random.randint(low=0, high=data.Ny, - size=10000), - np.random.randint(low=0, high=data.Nx, - size=10000)].T + background_points = np.c_[ + np.random.randint(low=0, high=data.Ny, size=10000), + np.random.randint(low=0, high=data.Nx, size=10000), + ].T # We'll make use of the fact that coverages[6] has measurements at all # land points. This will help us decide between land and water. @@ -160,7 +161,7 @@ def plot_species_distribution(species=("bradypus_variegatus_0", train_cover_std = (species.cov_train - mean) / std # Fit OneClassSVM - print(" - fit OneClassSVM ... ", end='') + print(" - fit OneClassSVM ... ", end="") clf = svm.OneClassSVM(nu=0.1, kernel="rbf", gamma=0.5) clf.fit(train_cover_std) print("done.") @@ -169,16 +170,21 @@ def plot_species_distribution(species=("bradypus_variegatus_0", plt.subplot(1, 2, i + 1) if basemap: print(" - plot coastlines using basemap") - m = Basemap(projection='cyl', llcrnrlat=Y.min(), - urcrnrlat=Y.max(), llcrnrlon=X.min(), - urcrnrlon=X.max(), resolution='c') + m = Basemap( + projection="cyl", + llcrnrlat=Y.min(), + urcrnrlat=Y.max(), + llcrnrlon=X.min(), + urcrnrlon=X.max(), + resolution="c", + ) m.drawcoastlines() m.drawcountries() else: print(" - plot coastlines from coverage") - plt.contour(X, Y, land_reference, - levels=[-9998], colors="k", - linestyles="solid") + plt.contour( + X, Y, land_reference, levels=[-9998], colors="k", linestyles="solid" + ) plt.xticks([]) plt.yticks([]) @@ -188,7 +194,7 @@ def plot_species_distribution(species=("bradypus_variegatus_0", Z = np.ones((data.Ny, data.Nx), dtype=np.float64) # We'll predict only for the land points. - idx = np.where(land_reference > -9999) + idx = (land_reference > -9999).nonzero() coverages_land = data.coverages[:, idx[0], idx[1]].T pred = clf.decision_function((coverages_land - mean) / std) @@ -200,18 +206,28 @@ def plot_species_distribution(species=("bradypus_variegatus_0", # plot contours of the prediction plt.contourf(X, Y, Z, levels=levels, cmap=plt.cm.Reds) - plt.colorbar(format='%.2f') + plt.colorbar(format="%.2f") # scatter training/testing points - plt.scatter(species.pts_train['dd long'], species.pts_train['dd lat'], - s=2 ** 2, c='black', - marker='^', label='train') - plt.scatter(species.pts_test['dd long'], species.pts_test['dd lat'], - s=2 ** 2, c='black', - marker='x', label='test') + plt.scatter( + species.pts_train["dd long"], + species.pts_train["dd lat"], + s=2**2, + c="black", + marker="^", + label="train", + ) + plt.scatter( + species.pts_test["dd long"], + species.pts_test["dd lat"], + s=2**2, + c="black", + marker="x", + label="test", + ) plt.legend() plt.title(species.name) - plt.axis('equal') + plt.axis("equal") # Compute AUC with regards to background points pred_background = Z[background_points[0], background_points[1]] diff --git a/examples/applications/plot_stock_market.py b/examples/applications/plot_stock_market.py index fb6f9b2ec27d8..40f778c785723 100644 --- a/examples/applications/plot_stock_market.py +++ b/examples/applications/plot_stock_market.py @@ -7,140 +7,85 @@ the stock market structure from variations in historical quotes. The quantity that we use is the daily variation in quote price: quotes -that are linked tend to cofluctuate during a day. - -.. _stock_market: - -Learning a graph structure --------------------------- - -We use sparse inverse covariance estimation to find which quotes are -correlated conditionally on the others. Specifically, sparse inverse -covariance gives us a graph, that is a list of connection. For each -symbol, the symbols that it is connected too are those useful to explain -its fluctuations. - -Clustering ----------- - -We use clustering to group together quotes that behave similarly. Here, -amongst the :ref:`various clustering techniques ` available -in the scikit-learn, we use :ref:`affinity_propagation` as it does -not enforce equal-size clusters, and it can choose automatically the -number of clusters from the data. - -Note that this gives us a different indication than the graph, as the -graph reflects conditional relations between variables, while the -clustering reflects marginal properties: variables clustered together can -be considered as having a similar impact at the level of the full stock -market. - -Embedding in 2D space ---------------------- - -For visualization purposes, we need to lay out the different symbols on a -2D canvas. For this we use :ref:`manifold` techniques to retrieve 2D -embedding. - - -Visualization -------------- - -The output of the 3 models are combined in a 2D graph where nodes -represents the stocks and edges the: - -- cluster labels are used to define the color of the nodes -- the sparse covariance model is used to display the strength of the edges -- the 2D embedding is used to position the nodes in the plan - -This example has a fair amount of visualization-related code, as -visualization is crucial here to display the graph. One of the challenge -is to position the labels minimizing overlap. For this we use an -heuristic based on the direction of the nearest neighbor along each -axis. +that are linked tend to fluctuate in relation to each other during a day. """ -# Author: Gael Varoquaux gael.varoquaux@normalesup.org -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Retrieve the data from Internet +# ------------------------------- +# +# The data is from 2003 - 2008. This is reasonably calm: not too long ago so +# that we get high-tech firms, and before the 2008 crash. This kind of +# historical data can be obtained from APIs like the +# `data.nasdaq.com `_ and +# `alphavantage.co `_. import sys import numpy as np -import matplotlib.pyplot as plt -from matplotlib.collections import LineCollection - import pandas as pd -from sklearn import cluster, covariance, manifold - -print(__doc__) - - -# ############################################################################# -# Retrieve the data from Internet - -# The data is from 2003 - 2008. This is reasonably calm: (not too long ago so -# that we get high-tech firms, and before the 2008 crash). This kind of -# historical data can be obtained for from APIs like the quandl.com and -# alphavantage.co ones. - symbol_dict = { - 'TOT': 'Total', - 'XOM': 'Exxon', - 'CVX': 'Chevron', - 'COP': 'ConocoPhillips', - 'VLO': 'Valero Energy', - 'MSFT': 'Microsoft', - 'IBM': 'IBM', - 'TWX': 'Time Warner', - 'CMCSA': 'Comcast', - 'CVC': 'Cablevision', - 'YHOO': 'Yahoo', - 'DELL': 'Dell', - 'HPQ': 'HP', - 'AMZN': 'Amazon', - 'TM': 'Toyota', - 'CAJ': 'Canon', - 'SNE': 'Sony', - 'F': 'Ford', - 'HMC': 'Honda', - 'NAV': 'Navistar', - 'NOC': 'Northrop Grumman', - 'BA': 'Boeing', - 'KO': 'Coca Cola', - 'MMM': '3M', - 'MCD': 'McDonald\'s', - 'PEP': 'Pepsi', - 'K': 'Kellogg', - 'UN': 'Unilever', - 'MAR': 'Marriott', - 'PG': 'Procter Gamble', - 'CL': 'Colgate-Palmolive', - 'GE': 'General Electrics', - 'WFC': 'Wells Fargo', - 'JPM': 'JPMorgan Chase', - 'AIG': 'AIG', - 'AXP': 'American express', - 'BAC': 'Bank of America', - 'GS': 'Goldman Sachs', - 'AAPL': 'Apple', - 'SAP': 'SAP', - 'CSCO': 'Cisco', - 'TXN': 'Texas Instruments', - 'XRX': 'Xerox', - 'WMT': 'Wal-Mart', - 'HD': 'Home Depot', - 'GSK': 'GlaxoSmithKline', - 'PFE': 'Pfizer', - 'SNY': 'Sanofi-Aventis', - 'NVS': 'Novartis', - 'KMB': 'Kimberly-Clark', - 'R': 'Ryder', - 'GD': 'General Dynamics', - 'RTN': 'Raytheon', - 'CVS': 'CVS', - 'CAT': 'Caterpillar', - 'DD': 'DuPont de Nemours'} + "TOT": "Total", + "XOM": "Exxon", + "CVX": "Chevron", + "COP": "ConocoPhillips", + "VLO": "Valero Energy", + "MSFT": "Microsoft", + "IBM": "IBM", + "TWX": "Time Warner", + "CMCSA": "Comcast", + "CVC": "Cablevision", + "YHOO": "Yahoo", + "DELL": "Dell", + "HPQ": "HP", + "AMZN": "Amazon", + "TM": "Toyota", + "CAJ": "Canon", + "SNE": "Sony", + "F": "Ford", + "HMC": "Honda", + "NAV": "Navistar", + "NOC": "Northrop Grumman", + "BA": "Boeing", + "KO": "Coca Cola", + "MMM": "3M", + "MCD": "McDonald's", + "PEP": "Pepsi", + "K": "Kellogg", + "UN": "Unilever", + "MAR": "Marriott", + "PG": "Procter Gamble", + "CL": "Colgate-Palmolive", + "GE": "General Electrics", + "WFC": "Wells Fargo", + "JPM": "JPMorgan Chase", + "AIG": "AIG", + "AXP": "American express", + "BAC": "Bank of America", + "GS": "Goldman Sachs", + "AAPL": "Apple", + "SAP": "SAP", + "CSCO": "Cisco", + "TXN": "Texas Instruments", + "XRX": "Xerox", + "WMT": "Wal-Mart", + "HD": "Home Depot", + "GSK": "GlaxoSmithKline", + "PFE": "Pfizer", + "SNY": "Sanofi-Aventis", + "NVS": "Novartis", + "KMB": "Kimberly-Clark", + "R": "Ryder", + "GD": "General Dynamics", + "RTN": "Raytheon", + "CVS": "CVS", + "CAT": "Caterpillar", + "DD": "DuPont de Nemours", +} symbols, names = np.array(sorted(symbol_dict.items())).T @@ -148,87 +93,143 @@ quotes = [] for symbol in symbols: - print('Fetching quote history for %r' % symbol, file=sys.stderr) - url = ('https://raw.githubusercontent.com/scikit-learn/examples-data/' - 'master/financial-data/{}.csv') + print("Fetching quote history for %r" % symbol, file=sys.stderr) + url = ( + "https://raw.githubusercontent.com/scikit-learn/examples-data/" + "master/financial-data/{}.csv" + ) quotes.append(pd.read_csv(url.format(symbol))) -close_prices = np.vstack([q['close'] for q in quotes]) -open_prices = np.vstack([q['open'] for q in quotes]) +close_prices = np.vstack([q["close"] for q in quotes]) +open_prices = np.vstack([q["open"] for q in quotes]) -# The daily variations of the quotes are what carry most information +# The daily variations of the quotes are what carry the most information variation = close_prices - open_prices +# %% +# .. _stock_market: +# +# Learning a graph structure +# -------------------------- +# +# We use sparse inverse covariance estimation to find which quotes are +# correlated conditionally on the others. Specifically, sparse inverse +# covariance gives us a graph, that is a list of connections. For each +# symbol, the symbols that it is connected to are those useful to explain +# its fluctuations. + +from sklearn import covariance -# ############################################################################# -# Learn a graphical structure from the correlations -edge_model = covariance.GraphicalLassoCV() +alphas = np.logspace(-1.5, 1, num=10) +edge_model = covariance.GraphicalLassoCV(alphas=alphas) # standardize the time series: using correlations rather than covariance -# is more efficient for structure recovery +# former is more efficient for structure recovery X = variation.copy().T X /= X.std(axis=0) edge_model.fit(X) -# ############################################################################# -# Cluster using affinity propagation - -_, labels = cluster.affinity_propagation(edge_model.covariance_, - random_state=0) +# %% +# Clustering using affinity propagation +# ------------------------------------- +# +# We use clustering to group together quotes that behave similarly. Here, +# amongst the :ref:`various clustering techniques ` available +# in the scikit-learn, we use :ref:`affinity_propagation` as it does +# not enforce equal-size clusters, and it can choose automatically the +# number of clusters from the data. +# +# Note that this gives us a different indication than the graph, as the +# graph reflects conditional relations between variables, while the +# clustering reflects marginal properties: variables clustered together can +# be considered as having a similar impact at the level of the full stock +# market. + +from sklearn import cluster + +_, labels = cluster.affinity_propagation(edge_model.covariance_, random_state=0) n_labels = labels.max() for i in range(n_labels + 1): - print('Cluster %i: %s' % ((i + 1), ', '.join(names[labels == i]))) - -# ############################################################################# -# Find a low-dimension embedding for visualization: find the best position of + print(f"Cluster {i + 1}: {', '.join(names[labels == i])}") + +# %% +# Embedding in 2D space +# --------------------- +# +# For visualization purposes, we need to lay out the different symbols on a +# 2D canvas. For this, we use :ref:`manifold` techniques to retrieve 2D +# embedding. +# We use a dense ``eigen_solver`` to achieve reproducibility (arpack is initiated +# with the random vectors that we do not control). In addition, we use a large +# number of neighbors to capture the large-scale structure. + +# Finding a low-dimension embedding for visualization: find the best position of # the nodes (the stocks) on a 2D plane -# We use a dense eigen_solver to achieve reproducibility (arpack is -# initiated with random vectors that we don't control). In addition, we -# use a large number of neighbors to capture the large-scale structure. +from sklearn import manifold + node_position_model = manifold.LocallyLinearEmbedding( - n_components=2, eigen_solver='dense', n_neighbors=6) + n_components=2, eigen_solver="dense", n_neighbors=6 +) embedding = node_position_model.fit_transform(X.T).T -# ############################################################################# +# %% # Visualization -plt.figure(1, facecolor='w', figsize=(10, 8)) +# ------------- +# +# The output of the 3 models are combined in a 2D graph where nodes +# represent the stocks and edges the connections (partial correlations): +# +# - cluster labels are used to define the color of the nodes +# - the sparse covariance model is used to display the strength of the edges +# - the 2D embedding is used to position the nodes in the plan +# +# This example has a fair amount of visualization-related code, as +# visualization is crucial here to display the graph. One of the challenges +# is to position the labels minimizing overlap. For this, we use an +# heuristic based on the direction of the nearest neighbor along each +# axis. + +import matplotlib.pyplot as plt +from matplotlib.collections import LineCollection + +plt.figure(1, facecolor="w", figsize=(10, 8)) plt.clf() -ax = plt.axes([0., 0., 1., 1.]) -plt.axis('off') +ax = plt.axes([0.0, 0.0, 1.0, 1.0]) +plt.axis("off") -# Display a graph of the partial correlations +# Plot the graph of partial correlations partial_correlations = edge_model.precision_.copy() d = 1 / np.sqrt(np.diag(partial_correlations)) partial_correlations *= d partial_correlations *= d[:, np.newaxis] -non_zero = (np.abs(np.triu(partial_correlations, k=1)) > 0.02) +non_zero = np.abs(np.triu(partial_correlations, k=1)) > 0.02 # Plot the nodes using the coordinates of our embedding -plt.scatter(embedding[0], embedding[1], s=100 * d ** 2, c=labels, - cmap=plt.cm.nipy_spectral) +plt.scatter( + embedding[0], embedding[1], s=100 * d**2, c=labels, cmap=plt.cm.nipy_spectral +) # Plot the edges -start_idx, end_idx = np.where(non_zero) +start_idx, end_idx = non_zero.nonzero() # a sequence of (*line0*, *line1*, *line2*), where:: # linen = (x0, y0), (x1, y1), ... (xm, ym) -segments = [[embedding[:, start], embedding[:, stop]] - for start, stop in zip(start_idx, end_idx)] +segments = [ + [embedding[:, start], embedding[:, stop]] for start, stop in zip(start_idx, end_idx) +] values = np.abs(partial_correlations[non_zero]) -lc = LineCollection(segments, - zorder=0, cmap=plt.cm.hot_r, - norm=plt.Normalize(0, .7 * values.max())) +lc = LineCollection( + segments, zorder=0, cmap=plt.cm.hot_r, norm=plt.Normalize(0, 0.7 * values.max()) +) lc.set_array(values) lc.set_linewidths(15 * values) ax.add_collection(lc) # Add a label to each node. The challenge here is that we want to # position the labels to avoid overlap with other labels -for index, (name, label, (x, y)) in enumerate( - zip(names, labels, embedding.T)): - +for index, (name, label, (x, y)) in enumerate(zip(names, labels, embedding.T)): dx = x - embedding[0] dx[index] = 1 dy = y - embedding[1] @@ -236,27 +237,38 @@ this_dx = dx[np.argmin(np.abs(dy))] this_dy = dy[np.argmin(np.abs(dx))] if this_dx > 0: - horizontalalignment = 'left' - x = x + .002 + horizontalalignment = "left" + x = x + 0.002 else: - horizontalalignment = 'right' - x = x - .002 + horizontalalignment = "right" + x = x - 0.002 if this_dy > 0: - verticalalignment = 'bottom' - y = y + .002 + verticalalignment = "bottom" + y = y + 0.002 else: - verticalalignment = 'top' - y = y - .002 - plt.text(x, y, name, size=10, - horizontalalignment=horizontalalignment, - verticalalignment=verticalalignment, - bbox=dict(facecolor='w', - edgecolor=plt.cm.nipy_spectral(label / float(n_labels)), - alpha=.6)) - -plt.xlim(embedding[0].min() - .15 * embedding[0].ptp(), - embedding[0].max() + .10 * embedding[0].ptp(),) -plt.ylim(embedding[1].min() - .03 * embedding[1].ptp(), - embedding[1].max() + .03 * embedding[1].ptp()) + verticalalignment = "top" + y = y - 0.002 + plt.text( + x, + y, + name, + size=10, + horizontalalignment=horizontalalignment, + verticalalignment=verticalalignment, + bbox=dict( + facecolor="w", + edgecolor=plt.cm.nipy_spectral(label / float(n_labels)), + alpha=0.6, + ), + ) + +plt.xlim( + embedding[0].min() - 0.15 * np.ptp(embedding[0]), + embedding[0].max() + 0.10 * np.ptp(embedding[0]), +) +plt.ylim( + embedding[1].min() - 0.03 * np.ptp(embedding[1]), + embedding[1].max() + 0.03 * np.ptp(embedding[1]), +) plt.show() diff --git a/examples/applications/plot_time_series_lagged_features.py b/examples/applications/plot_time_series_lagged_features.py new file mode 100644 index 0000000000000..7c5b75e12ccfd --- /dev/null +++ b/examples/applications/plot_time_series_lagged_features.py @@ -0,0 +1,438 @@ +""" +=========================================== +Lagged features for time series forecasting +=========================================== + +This example demonstrates how Polars-engineered lagged features can be used +for time series forecasting with +:class:`~sklearn.ensemble.HistGradientBoostingRegressor` on the Bike Sharing +Demand dataset. + +See the example on +:ref:`sphx_glr_auto_examples_applications_plot_cyclical_feature_engineering.py` +for some data exploration on this dataset and a demo on periodic feature +engineering. + +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Analyzing the Bike Sharing Demand dataset +# ----------------------------------------- +# +# We start by loading the data from the OpenML repository as a raw parquet file +# to illustrate how to work with an arbitrary parquet file instead of hiding this +# step in a convenience tool such as `sklearn.datasets.fetch_openml`. +# +# The URL of the parquet file can be found in the JSON description of the +# Bike Sharing Demand dataset with id 44063 on openml.org +# (https://openml.org/search?type=data&status=active&id=44063). +# +# The `sha256` hash of the file is also provided to ensure the integrity of the +# downloaded file. +import numpy as np +import polars as pl + +from sklearn.datasets import fetch_file + +pl.Config.set_fmt_str_lengths(20) + +bike_sharing_data_file = fetch_file( + "https://data.openml.org/datasets/0004/44063/dataset_44063.pq", + sha256="d120af76829af0d256338dc6dd4be5df4fd1f35bf3a283cab66a51c1c6abd06a", +) +bike_sharing_data_file + +# %% +# We load the parquet file with Polars for feature engineering. Polars +# automatically caches common subexpressions which are reused in multiple +# expressions (like `pl.col("count").shift(1)` below). See +# https://docs.pola.rs/user-guide/lazy/optimizations/ for more information. + +df = pl.read_parquet(bike_sharing_data_file) + +# %% +# Next, we take a look at the statistical summary of the dataset +# so that we can better understand the data that we are working with. +import polars.selectors as cs + +summary = df.select(cs.numeric()).describe() +summary + +# %% +# Let us look at the count of the seasons `"fall"`, `"spring"`, `"summer"` +# and `"winter"` present in the dataset to confirm they are balanced. + +import matplotlib.pyplot as plt + +df["season"].value_counts() + + +# %% +# Generating Polars-engineered lagged features +# -------------------------------------------- +# Let's consider the problem of predicting the demand at the +# next hour given past demands. Since the demand is a continuous +# variable, one could intuitively use any regression model. However, we do +# not have the usual `(X_train, y_train)` dataset. Instead, we just have +# the `y_train` demand data sequentially organized by time. +lagged_df = df.select( + "count", + *[pl.col("count").shift(i).alias(f"lagged_count_{i}h") for i in [1, 2, 3]], + lagged_count_1d=pl.col("count").shift(24), + lagged_count_1d_1h=pl.col("count").shift(24 + 1), + lagged_count_7d=pl.col("count").shift(7 * 24), + lagged_count_7d_1h=pl.col("count").shift(7 * 24 + 1), + lagged_mean_24h=pl.col("count").shift(1).rolling_mean(24), + lagged_max_24h=pl.col("count").shift(1).rolling_max(24), + lagged_min_24h=pl.col("count").shift(1).rolling_min(24), + lagged_mean_7d=pl.col("count").shift(1).rolling_mean(7 * 24), + lagged_max_7d=pl.col("count").shift(1).rolling_max(7 * 24), + lagged_min_7d=pl.col("count").shift(1).rolling_min(7 * 24), +) +lagged_df.tail(10) + +# %% +# Watch out however, the first lines have undefined values because their own +# past is unknown. This depends on how much lag we used: +lagged_df.head(10) + +# %% +# We can now separate the lagged features in a matrix `X` and the target variable +# (the counts to predict) in an array of the same first dimension `y`. +lagged_df = lagged_df.drop_nulls() +X = lagged_df.drop("count") +y = lagged_df["count"] +print("X shape: {}\ny shape: {}".format(X.shape, y.shape)) + +# %% +# Naive evaluation of the next hour bike demand regression +# -------------------------------------------------------- +# Let's randomly split our tabularized dataset to train a gradient +# boosting regression tree (GBRT) model and evaluate it using Mean +# Absolute Percentage Error (MAPE). If our model is aimed at forecasting +# (i.e., predicting future data from past data), we should not use training +# data that are ulterior to the testing data. In time series machine learning +# the "i.i.d" (independent and identically distributed) assumption does not +# hold true as the data points are not independent and have a temporal +# relationship. +from sklearn.ensemble import HistGradientBoostingRegressor +from sklearn.model_selection import train_test_split + +X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.2, random_state=42 +) + +model = HistGradientBoostingRegressor().fit(X_train, y_train) + +# %% +# Taking a look at the performance of the model. +from sklearn.metrics import mean_absolute_percentage_error + +y_pred = model.predict(X_test) +mean_absolute_percentage_error(y_test, y_pred) + +# %% +# Proper next hour forecasting evaluation +# --------------------------------------- +# Let's use a proper evaluation splitting strategies that takes into account +# the temporal structure of the dataset to evaluate our model's ability to +# predict data points in the future (to avoid cheating by reading values from +# the lagged features in the training set). +from sklearn.model_selection import TimeSeriesSplit + +ts_cv = TimeSeriesSplit( + n_splits=3, # to keep the notebook fast enough on common laptops + gap=48, # 2 days data gap between train and test + max_train_size=10000, # keep train sets of comparable sizes + test_size=3000, # for 2 or 3 digits of precision in scores +) +all_splits = list(ts_cv.split(X, y)) + +# %% +# Training the model and evaluating its performance based on MAPE. +train_idx, test_idx = all_splits[0] +X_train, X_test = X[train_idx, :], X[test_idx, :] +y_train, y_test = y[train_idx], y[test_idx] + +model = HistGradientBoostingRegressor().fit(X_train, y_train) +y_pred = model.predict(X_test) +mean_absolute_percentage_error(y_test, y_pred) + +# %% +# The generalization error measured via a shuffled trained test split +# is too optimistic. The generalization via a time-based split is likely to +# be more representative of the true performance of the regression model. +# Let's assess this variability of our error evaluation with proper +# cross-validation: +from sklearn.model_selection import cross_val_score + +cv_mape_scores = -cross_val_score( + model, X, y, cv=ts_cv, scoring="neg_mean_absolute_percentage_error" +) +cv_mape_scores + +# %% +# The variability across splits is quite large! In a real life setting +# it would be advised to use more splits to better assess the variability. +# Let's report the mean CV scores and their standard deviation from now on. +print(f"CV MAPE: {cv_mape_scores.mean():.3f} Âą {cv_mape_scores.std():.3f}") + +# %% +# We can compute several combinations of evaluation metrics and loss functions, +# which are reported a bit below. +from collections import defaultdict + +from sklearn.metrics import ( + make_scorer, + mean_absolute_error, + mean_pinball_loss, + root_mean_squared_error, +) +from sklearn.model_selection import cross_validate + + +def consolidate_scores(cv_results, scores, metric): + if metric == "MAPE": + scores[metric].append(f"{value.mean():.2f} Âą {value.std():.2f}") + else: + scores[metric].append(f"{value.mean():.1f} Âą {value.std():.1f}") + + return scores + + +scoring = { + "MAPE": make_scorer(mean_absolute_percentage_error), + "RMSE": make_scorer(root_mean_squared_error), + "MAE": make_scorer(mean_absolute_error), + "pinball_loss_05": make_scorer(mean_pinball_loss, alpha=0.05), + "pinball_loss_50": make_scorer(mean_pinball_loss, alpha=0.50), + "pinball_loss_95": make_scorer(mean_pinball_loss, alpha=0.95), +} +loss_functions = ["squared_error", "poisson", "absolute_error"] +scores = defaultdict(list) +for loss_func in loss_functions: + model = HistGradientBoostingRegressor(loss=loss_func) + cv_results = cross_validate( + model, + X, + y, + cv=ts_cv, + scoring=scoring, + n_jobs=2, + ) + time = cv_results["fit_time"] + scores["loss"].append(loss_func) + scores["fit_time"].append(f"{time.mean():.2f} Âą {time.std():.2f} s") + + for key, value in cv_results.items(): + if key.startswith("test_"): + metric = key.split("test_")[1] + scores = consolidate_scores(cv_results, scores, metric) + + +# %% +# Modeling predictive uncertainty via quantile regression +# ------------------------------------------------------- +# Instead of modeling the expected value of the distribution of +# :math:`Y|X` like the least squares and Poisson losses do, one could try to +# estimate quantiles of the conditional distribution. +# +# :math:`Y|X=x_i` is expected to be a random variable for a given data point +# :math:`x_i` because we expect that the number of rentals cannot be 100% +# accurately predicted from the features. It can be influenced by other +# variables not properly captured by the existing lagged features. For +# instance whether or not it will rain in the next hour cannot be fully +# anticipated from the past hours bike rental data. This is what we +# call aleatoric uncertainty. +# +# Quantile regression makes it possible to give a finer description of that +# distribution without making strong assumptions on its shape. +quantile_list = [0.05, 0.5, 0.95] + +for quantile in quantile_list: + model = HistGradientBoostingRegressor(loss="quantile", quantile=quantile) + cv_results = cross_validate( + model, + X, + y, + cv=ts_cv, + scoring=scoring, + n_jobs=2, + ) + time = cv_results["fit_time"] + scores["fit_time"].append(f"{time.mean():.2f} Âą {time.std():.2f} s") + + scores["loss"].append(f"quantile {int(quantile * 100)}") + for key, value in cv_results.items(): + if key.startswith("test_"): + metric = key.split("test_")[1] + scores = consolidate_scores(cv_results, scores, metric) + +scores_df = pl.DataFrame(scores) +scores_df + + +# %% +# Let us take a look at the losses that minimise each metric. +def min_arg(col): + col_split = pl.col(col).str.split(" ") + return pl.arg_sort_by( + col_split.list.get(0).cast(pl.Float64), + col_split.list.get(2).cast(pl.Float64), + ).first() + + +scores_df.select( + pl.col("loss").get(min_arg(col_name)).alias(col_name) + for col_name in scores_df.columns + if col_name != "loss" +) + +# %% +# Even if the score distributions overlap due to the variance in the dataset, +# it is true that the average RMSE is lower when `loss="squared_error"`, whereas +# the average MAPE is lower when `loss="absolute_error"` as expected. That is +# also the case for the Mean Pinball Loss with the quantiles 5 and 95. The score +# corresponding to the 50 quantile loss is overlapping with the score obtained +# by minimizing other loss functions, which is also the case for the MAE. +# +# A qualitative look at the predictions +# ------------------------------------- +# We can now visualize the performance of the model with regards +# to the 5th percentile, median and the 95th percentile: +all_splits = list(ts_cv.split(X, y)) +train_idx, test_idx = all_splits[0] + +X_train, X_test = X[train_idx, :], X[test_idx, :] +y_train, y_test = y[train_idx], y[test_idx] + +max_iter = 50 +gbrt_mean_poisson = HistGradientBoostingRegressor(loss="poisson", max_iter=max_iter) +gbrt_mean_poisson.fit(X_train, y_train) +mean_predictions = gbrt_mean_poisson.predict(X_test) + +gbrt_median = HistGradientBoostingRegressor( + loss="quantile", quantile=0.5, max_iter=max_iter +) +gbrt_median.fit(X_train, y_train) +median_predictions = gbrt_median.predict(X_test) + +gbrt_percentile_5 = HistGradientBoostingRegressor( + loss="quantile", quantile=0.05, max_iter=max_iter +) +gbrt_percentile_5.fit(X_train, y_train) +percentile_5_predictions = gbrt_percentile_5.predict(X_test) + +gbrt_percentile_95 = HistGradientBoostingRegressor( + loss="quantile", quantile=0.95, max_iter=max_iter +) +gbrt_percentile_95.fit(X_train, y_train) +percentile_95_predictions = gbrt_percentile_95.predict(X_test) + +# %% +# We can now take a look at the predictions made by the regression models: +last_hours = slice(-96, None) +fig, ax = plt.subplots(figsize=(15, 7)) +plt.title("Predictions by regression models") +ax.plot( + y_test[last_hours], + "x-", + alpha=0.2, + label="Actual demand", + color="black", +) +ax.plot( + median_predictions[last_hours], + "^-", + label="GBRT median", +) +ax.plot( + mean_predictions[last_hours], + "x-", + label="GBRT mean (Poisson)", +) +ax.fill_between( + np.arange(96), + percentile_5_predictions[last_hours], + percentile_95_predictions[last_hours], + alpha=0.3, + label="GBRT 90% interval", +) +_ = ax.legend() + +# %% +# Here it's interesting to notice that the blue area between the 5% and 95% +# percentile estimators has a width that varies with the time of the day: +# +# - At night, the blue band is much narrower: the pair of models is quite +# certain that there will be a small number of bike rentals. And furthermore +# these seem correct in the sense that the actual demand stays in that blue +# band. +# - During the day, the blue band is much wider: the uncertainty grows, probably +# because of the variability of the weather that can have a very large impact, +# especially on week-ends. +# - We can also see that during week-days, the commute pattern is still visible in +# the 5% and 95% estimations. +# - Finally, it is expected that 10% of the time, the actual demand does not lie +# between the 5% and 95% percentile estimates. On this test span, the actual +# demand seems to be higher, especially during the rush hours. It might reveal that +# our 95% percentile estimator underestimates the demand peaks. This could be be +# quantitatively confirmed by computing empirical coverage numbers as done in +# the :ref:`calibration of confidence intervals `. +# +# Looking at the performance of non-linear regression models vs +# the best models: +from sklearn.metrics import PredictionErrorDisplay + +fig, axes = plt.subplots(ncols=3, figsize=(15, 6), sharey=True) +fig.suptitle("Non-linear regression models") +predictions = [ + median_predictions, + percentile_5_predictions, + percentile_95_predictions, +] +labels = [ + "Median", + "5th percentile", + "95th percentile", +] +for ax, pred, label in zip(axes, predictions, labels): + PredictionErrorDisplay.from_predictions( + y_true=y_test, + y_pred=pred, + kind="residual_vs_predicted", + scatter_kwargs={"alpha": 0.3}, + ax=ax, + ) + ax.set(xlabel="Predicted demand", ylabel="True demand") + ax.legend(["Best model", label]) + +plt.show() + +# %% +# Conclusion +# ---------- +# Through this example we explored time series forecasting using lagged +# features. We compared a naive regression (using the standardized +# :class:`~sklearn.model_selection.train_test_split`) with a proper time +# series evaluation strategy using +# :class:`~sklearn.model_selection.TimeSeriesSplit`. We observed that the +# model trained using :class:`~sklearn.model_selection.train_test_split`, +# having a default value of `shuffle` set to `True` produced an overly +# optimistic Mean Average Percentage Error (MAPE). The results +# produced from the time-based split better represent the performance +# of our time-series regression model. We also analyzed the predictive uncertainty +# of our model via Quantile Regression. Predictions based on the 5th and +# 95th percentile using `loss="quantile"` provide us with a quantitative estimate +# of the uncertainty of the forecasts made by our time series regression model. +# Uncertainty estimation can also be performed +# using `MAPIE `_, +# that provides an implementation based on recent work on conformal prediction +# methods and estimates both aleatoric and epistemic uncertainty at the same time. +# Furthermore, functionalities provided +# by `sktime `_ +# can be used to extend scikit-learn estimators by making use of recursive time +# series forecasting, that enables dynamic predictions of future values. diff --git a/examples/applications/plot_tomography_l1_reconstruction.py b/examples/applications/plot_tomography_l1_reconstruction.py index 0c218c1b0fa4a..02d4594b90518 100644 --- a/examples/applications/plot_tomography_l1_reconstruction.py +++ b/examples/applications/plot_tomography_l1_reconstruction.py @@ -33,19 +33,17 @@ class :class:`~sklearn.linear_model.Lasso`, that uses the coordinate descent reconstructed image, contrary to the L1 penalization. Note in particular the circular artifact separating the pixels in the corners, that have contributed to fewer projections than the central disk. -""" -print(__doc__) +""" -# Author: Emmanuelle Gouillart -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np -from scipy import sparse -from scipy import ndimage -from sklearn.linear_model import Lasso -from sklearn.linear_model import Ridge import matplotlib.pyplot as plt +import numpy as np +from scipy import ndimage, sparse + +from sklearn.linear_model import Lasso, Ridge def _weights(x, dx=1, orig=0): @@ -57,14 +55,14 @@ def _weights(x, dx=1, orig=0): def _generate_center_coordinates(l_x): X, Y = np.mgrid[:l_x, :l_x].astype(np.float64) - center = l_x / 2. + center = l_x / 2.0 X += 0.5 - center Y += 0.5 - center return X, Y def build_projection_operator(l_x, n_dir): - """ Compute the tomography design matrix. + """Compute the tomography design matrix. Parameters ---------- @@ -82,9 +80,8 @@ def build_projection_operator(l_x, n_dir): X, Y = _generate_center_coordinates(l_x) angles = np.linspace(0, np.pi, n_dir, endpoint=False) data_inds, weights, camera_inds = [], [], [] - data_unravel_indices = np.arange(l_x ** 2) - data_unravel_indices = np.hstack((data_unravel_indices, - data_unravel_indices)) + data_unravel_indices = np.arange(l_x**2) + data_unravel_indices = np.hstack((data_unravel_indices, data_unravel_indices)) for i, angle in enumerate(angles): Xrot = np.cos(angle) * X - np.sin(angle) * Y inds, w = _weights(Xrot, dx=1, orig=X.min()) @@ -97,11 +94,11 @@ def build_projection_operator(l_x, n_dir): def generate_synthetic_data(): - """ Synthetic binary data """ + """Synthetic binary data""" rs = np.random.RandomState(0) n_pts = 36 x, y = np.ogrid[0:l, 0:l] - mask_outer = (x - l / 2.) ** 2 + (y - l / 2.) ** 2 < (l / 2.) ** 2 + mask_outer = (x - l / 2.0) ** 2 + (y - l / 2.0) ** 2 < (l / 2.0) ** 2 mask = np.zeros((l, l)) points = l * rs.rand(2, n_pts) mask[(points[0]).astype(int), (points[1]).astype(int)] = 1 @@ -131,19 +128,18 @@ def generate_synthetic_data(): plt.figure(figsize=(8, 3.3)) plt.subplot(131) -plt.imshow(data, cmap=plt.cm.gray, interpolation='nearest') -plt.axis('off') -plt.title('original image') +plt.imshow(data, cmap=plt.cm.gray, interpolation="nearest") +plt.axis("off") +plt.title("original image") plt.subplot(132) -plt.imshow(rec_l2, cmap=plt.cm.gray, interpolation='nearest') -plt.title('L2 penalization') -plt.axis('off') +plt.imshow(rec_l2, cmap=plt.cm.gray, interpolation="nearest") +plt.title("L2 penalization") +plt.axis("off") plt.subplot(133) -plt.imshow(rec_l1, cmap=plt.cm.gray, interpolation='nearest') -plt.title('L1 penalization') -plt.axis('off') +plt.imshow(rec_l1, cmap=plt.cm.gray, interpolation="nearest") +plt.title("L1 penalization") +plt.axis("off") -plt.subplots_adjust(hspace=0.01, wspace=0.01, top=1, bottom=0, left=0, - right=1) +plt.subplots_adjust(hspace=0.01, wspace=0.01, top=1, bottom=0, left=0, right=1) plt.show() diff --git a/examples/applications/plot_topics_extraction_with_nmf_lda.py b/examples/applications/plot_topics_extraction_with_nmf_lda.py index 95e4ebadc512b..a6f774d01e2de 100644 --- a/examples/applications/plot_topics_extraction_with_nmf_lda.py +++ b/examples/applications/plot_topics_extraction_with_nmf_lda.py @@ -21,39 +21,38 @@ """ -# Author: Olivier Grisel -# Lars Buitinck -# Chyi-Kwei Yau -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause from time import time + import matplotlib.pyplot as plt -from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer -from sklearn.decomposition import NMF, LatentDirichletAllocation from sklearn.datasets import fetch_20newsgroups +from sklearn.decomposition import NMF, LatentDirichletAllocation, MiniBatchNMF +from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer n_samples = 2000 n_features = 1000 n_components = 10 n_top_words = 20 +batch_size = 128 +init = "nndsvda" def plot_top_words(model, feature_names, n_top_words, title): fig, axes = plt.subplots(2, 5, figsize=(30, 15), sharex=True) axes = axes.flatten() for topic_idx, topic in enumerate(model.components_): - top_features_ind = topic.argsort()[:-n_top_words - 1:-1] - top_features = [feature_names[i] for i in top_features_ind] + top_features_ind = topic.argsort()[-n_top_words:] + top_features = feature_names[top_features_ind] weights = topic[top_features_ind] ax = axes[topic_idx] ax.barh(top_features, weights, height=0.7) - ax.set_title(f'Topic {topic_idx +1}', - fontdict={'fontsize': 30}) - ax.invert_yaxis() - ax.tick_params(axis='both', which='major', labelsize=20) - for i in 'top right left'.split(): + ax.set_title(f"Topic {topic_idx + 1}", fontdict={"fontsize": 30}) + ax.tick_params(axis="both", which="major", labelsize=20) + for i in "top right left".split(): ax.spines[i].set_visible(False) fig.suptitle(title, fontsize=40) @@ -68,69 +67,158 @@ def plot_top_words(model, feature_names, n_top_words, title): print("Loading dataset...") t0 = time() -data, _ = fetch_20newsgroups(shuffle=True, random_state=1, - remove=('headers', 'footers', 'quotes'), - return_X_y=True) +data, _ = fetch_20newsgroups( + shuffle=True, + random_state=1, + remove=("headers", "footers", "quotes"), + return_X_y=True, +) data_samples = data[:n_samples] print("done in %0.3fs." % (time() - t0)) # Use tf-idf features for NMF. print("Extracting tf-idf features for NMF...") -tfidf_vectorizer = TfidfVectorizer(max_df=0.95, min_df=2, - max_features=n_features, - stop_words='english') +tfidf_vectorizer = TfidfVectorizer( + max_df=0.95, min_df=2, max_features=n_features, stop_words="english" +) t0 = time() tfidf = tfidf_vectorizer.fit_transform(data_samples) print("done in %0.3fs." % (time() - t0)) # Use tf (raw term count) features for LDA. print("Extracting tf features for LDA...") -tf_vectorizer = CountVectorizer(max_df=0.95, min_df=2, - max_features=n_features, - stop_words='english') +tf_vectorizer = CountVectorizer( + max_df=0.95, min_df=2, max_features=n_features, stop_words="english" +) t0 = time() tf = tf_vectorizer.fit_transform(data_samples) print("done in %0.3fs." % (time() - t0)) print() # Fit the NMF model -print("Fitting the NMF model (Frobenius norm) with tf-idf features, " - "n_samples=%d and n_features=%d..." - % (n_samples, n_features)) +print( + "Fitting the NMF model (Frobenius norm) with tf-idf features, " + "n_samples=%d and n_features=%d..." % (n_samples, n_features) +) t0 = time() -nmf = NMF(n_components=n_components, random_state=1, - alpha=.1, l1_ratio=.5).fit(tfidf) +nmf = NMF( + n_components=n_components, + random_state=1, + init=init, + beta_loss="frobenius", + alpha_W=0.00005, + alpha_H=0.00005, + l1_ratio=1, +).fit(tfidf) print("done in %0.3fs." % (time() - t0)) -tfidf_feature_names = tfidf_vectorizer.get_feature_names() -plot_top_words(nmf, tfidf_feature_names, n_top_words, - 'Topics in NMF model (Frobenius norm)') +tfidf_feature_names = tfidf_vectorizer.get_feature_names_out() +plot_top_words( + nmf, tfidf_feature_names, n_top_words, "Topics in NMF model (Frobenius norm)" +) # Fit the NMF model -print('\n' * 2, "Fitting the NMF model (generalized Kullback-Leibler " - "divergence) with tf-idf features, n_samples=%d and n_features=%d..." - % (n_samples, n_features)) +print( + "\n" * 2, + "Fitting the NMF model (generalized Kullback-Leibler " + "divergence) with tf-idf features, n_samples=%d and n_features=%d..." + % (n_samples, n_features), +) +t0 = time() +nmf = NMF( + n_components=n_components, + random_state=1, + init=init, + beta_loss="kullback-leibler", + solver="mu", + max_iter=1000, + alpha_W=0.00005, + alpha_H=0.00005, + l1_ratio=0.5, +).fit(tfidf) +print("done in %0.3fs." % (time() - t0)) + +tfidf_feature_names = tfidf_vectorizer.get_feature_names_out() +plot_top_words( + nmf, + tfidf_feature_names, + n_top_words, + "Topics in NMF model (generalized Kullback-Leibler divergence)", +) + +# Fit the MiniBatchNMF model +print( + "\n" * 2, + "Fitting the MiniBatchNMF model (Frobenius norm) with tf-idf " + "features, n_samples=%d and n_features=%d, batch_size=%d..." + % (n_samples, n_features, batch_size), +) +t0 = time() +mbnmf = MiniBatchNMF( + n_components=n_components, + random_state=1, + batch_size=batch_size, + init=init, + beta_loss="frobenius", + alpha_W=0.00005, + alpha_H=0.00005, + l1_ratio=0.5, +).fit(tfidf) +print("done in %0.3fs." % (time() - t0)) + + +tfidf_feature_names = tfidf_vectorizer.get_feature_names_out() +plot_top_words( + mbnmf, + tfidf_feature_names, + n_top_words, + "Topics in MiniBatchNMF model (Frobenius norm)", +) + +# Fit the MiniBatchNMF model +print( + "\n" * 2, + "Fitting the MiniBatchNMF model (generalized Kullback-Leibler " + "divergence) with tf-idf features, n_samples=%d and n_features=%d, " + "batch_size=%d..." % (n_samples, n_features, batch_size), +) t0 = time() -nmf = NMF(n_components=n_components, random_state=1, - beta_loss='kullback-leibler', solver='mu', max_iter=1000, alpha=.1, - l1_ratio=.5).fit(tfidf) +mbnmf = MiniBatchNMF( + n_components=n_components, + random_state=1, + batch_size=batch_size, + init=init, + beta_loss="kullback-leibler", + alpha_W=0.00005, + alpha_H=0.00005, + l1_ratio=0.5, +).fit(tfidf) print("done in %0.3fs." % (time() - t0)) -tfidf_feature_names = tfidf_vectorizer.get_feature_names() -plot_top_words(nmf, tfidf_feature_names, n_top_words, - 'Topics in NMF model (generalized Kullback-Leibler divergence)') - -print('\n' * 2, "Fitting LDA models with tf features, " - "n_samples=%d and n_features=%d..." - % (n_samples, n_features)) -lda = LatentDirichletAllocation(n_components=n_components, max_iter=5, - learning_method='online', - learning_offset=50., - random_state=0) +tfidf_feature_names = tfidf_vectorizer.get_feature_names_out() +plot_top_words( + mbnmf, + tfidf_feature_names, + n_top_words, + "Topics in MiniBatchNMF model (generalized Kullback-Leibler divergence)", +) + +print( + "\n" * 2, + "Fitting LDA models with tf features, n_samples=%d and n_features=%d..." + % (n_samples, n_features), +) +lda = LatentDirichletAllocation( + n_components=n_components, + max_iter=5, + learning_method="online", + learning_offset=50.0, + random_state=0, +) t0 = time() lda.fit(tf) print("done in %0.3fs." % (time() - t0)) -tf_feature_names = tf_vectorizer.get_feature_names() -plot_top_words(lda, tf_feature_names, n_top_words, 'Topics in LDA model') +tf_feature_names = tf_vectorizer.get_feature_names_out() +plot_top_words(lda, tf_feature_names, n_top_words, "Topics in LDA model") diff --git a/examples/applications/svm_gui.py b/examples/applications/svm_gui.py deleted file mode 100644 index d085851422e18..0000000000000 --- a/examples/applications/svm_gui.py +++ /dev/null @@ -1,337 +0,0 @@ -""" -========== -Libsvm GUI -========== - -A simple graphical frontend for Libsvm mainly intended for didactic -purposes. You can create data points by point and click and visualize -the decision region induced by different kernels and parameter settings. - -To create positive examples click the left mouse button; to create -negative examples click the right button. - -If all examples are from the same class, it uses a one-class SVM. - -""" - -print(__doc__) - -# Author: Peter Prettenhoer -# -# License: BSD 3 clause - -import matplotlib -matplotlib.use('TkAgg') -from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg -try: - from matplotlib.backends.backend_tkagg import NavigationToolbar2Tk -except ImportError: - # NavigationToolbar2TkAgg was deprecated in matplotlib 2.2 - from matplotlib.backends.backend_tkagg import ( - NavigationToolbar2TkAgg as NavigationToolbar2Tk - ) -from matplotlib.figure import Figure -from matplotlib.contour import ContourSet - -import sys -import numpy as np -import tkinter as Tk - -from sklearn import svm -from sklearn.datasets import dump_svmlight_file - -y_min, y_max = -50, 50 -x_min, x_max = -50, 50 - - -class Model: - """The Model which hold the data. It implements the - observable in the observer pattern and notifies the - registered observers on change event. - """ - - def __init__(self): - self.observers = [] - self.surface = None - self.data = [] - self.cls = None - self.surface_type = 0 - - def changed(self, event): - """Notify the observers. """ - for observer in self.observers: - observer.update(event, self) - - def add_observer(self, observer): - """Register an observer. """ - self.observers.append(observer) - - def set_surface(self, surface): - self.surface = surface - - def dump_svmlight_file(self, file): - data = np.array(self.data) - X = data[:, 0:2] - y = data[:, 2] - dump_svmlight_file(X, y, file) - - -class Controller: - def __init__(self, model): - self.model = model - self.kernel = Tk.IntVar() - self.surface_type = Tk.IntVar() - # Whether or not a model has been fitted - self.fitted = False - - def fit(self): - print("fit the model") - train = np.array(self.model.data) - X = train[:, 0:2] - y = train[:, 2] - - C = float(self.complexity.get()) - gamma = float(self.gamma.get()) - coef0 = float(self.coef0.get()) - degree = int(self.degree.get()) - kernel_map = {0: "linear", 1: "rbf", 2: "poly"} - if len(np.unique(y)) == 1: - clf = svm.OneClassSVM(kernel=kernel_map[self.kernel.get()], - gamma=gamma, coef0=coef0, degree=degree) - clf.fit(X) - else: - clf = svm.SVC(kernel=kernel_map[self.kernel.get()], C=C, - gamma=gamma, coef0=coef0, degree=degree) - clf.fit(X, y) - if hasattr(clf, 'score'): - print("Accuracy:", clf.score(X, y) * 100) - X1, X2, Z = self.decision_surface(clf) - self.model.clf = clf - self.model.set_surface((X1, X2, Z)) - self.model.surface_type = self.surface_type.get() - self.fitted = True - self.model.changed("surface") - - def decision_surface(self, cls): - delta = 1 - x = np.arange(x_min, x_max + delta, delta) - y = np.arange(y_min, y_max + delta, delta) - X1, X2 = np.meshgrid(x, y) - Z = cls.decision_function(np.c_[X1.ravel(), X2.ravel()]) - Z = Z.reshape(X1.shape) - return X1, X2, Z - - def clear_data(self): - self.model.data = [] - self.fitted = False - self.model.changed("clear") - - def add_example(self, x, y, label): - self.model.data.append((x, y, label)) - self.model.changed("example_added") - - # update decision surface if already fitted. - self.refit() - - def refit(self): - """Refit the model if already fitted. """ - if self.fitted: - self.fit() - - -class View: - """Test docstring. """ - def __init__(self, root, controller): - f = Figure() - ax = f.add_subplot(111) - ax.set_xticks([]) - ax.set_yticks([]) - ax.set_xlim((x_min, x_max)) - ax.set_ylim((y_min, y_max)) - canvas = FigureCanvasTkAgg(f, master=root) - try: - canvas.draw() - except AttributeError: - # support for matplotlib (1.*) - canvas.show() - canvas.get_tk_widget().pack(side=Tk.TOP, fill=Tk.BOTH, expand=1) - canvas._tkcanvas.pack(side=Tk.TOP, fill=Tk.BOTH, expand=1) - canvas.mpl_connect('button_press_event', self.onclick) - toolbar = NavigationToolbar2Tk(canvas, root) - toolbar.update() - self.controllbar = ControllBar(root, controller) - self.f = f - self.ax = ax - self.canvas = canvas - self.controller = controller - self.contours = [] - self.c_labels = None - self.plot_kernels() - - def plot_kernels(self): - self.ax.text(-50, -60, "Linear: $u^T v$") - self.ax.text(-20, -60, r"RBF: $\exp (-\gamma \| u-v \|^2)$") - self.ax.text(10, -60, r"Poly: $(\gamma \, u^T v + r)^d$") - - def onclick(self, event): - if event.xdata and event.ydata: - if event.button == 1: - self.controller.add_example(event.xdata, event.ydata, 1) - elif event.button == 3: - self.controller.add_example(event.xdata, event.ydata, -1) - - def update_example(self, model, idx): - x, y, l = model.data[idx] - if l == 1: - color = 'w' - elif l == -1: - color = 'k' - self.ax.plot([x], [y], "%so" % color, scalex=0.0, scaley=0.0) - - def update(self, event, model): - if event == "examples_loaded": - for i in range(len(model.data)): - self.update_example(model, i) - - if event == "example_added": - self.update_example(model, -1) - - if event == "clear": - self.ax.clear() - self.ax.set_xticks([]) - self.ax.set_yticks([]) - self.contours = [] - self.c_labels = None - self.plot_kernels() - - if event == "surface": - self.remove_surface() - self.plot_support_vectors(model.clf.support_vectors_) - self.plot_decision_surface(model.surface, model.surface_type) - - self.canvas.draw() - - def remove_surface(self): - """Remove old decision surface.""" - if len(self.contours) > 0: - for contour in self.contours: - if isinstance(contour, ContourSet): - for lineset in contour.collections: - lineset.remove() - else: - contour.remove() - self.contours = [] - - def plot_support_vectors(self, support_vectors): - """Plot the support vectors by placing circles over the - corresponding data points and adds the circle collection - to the contours list.""" - cs = self.ax.scatter(support_vectors[:, 0], support_vectors[:, 1], - s=80, edgecolors="k", facecolors="none") - self.contours.append(cs) - - def plot_decision_surface(self, surface, type): - X1, X2, Z = surface - if type == 0: - levels = [-1.0, 0.0, 1.0] - linestyles = ['dashed', 'solid', 'dashed'] - colors = 'k' - self.contours.append(self.ax.contour(X1, X2, Z, levels, - colors=colors, - linestyles=linestyles)) - elif type == 1: - self.contours.append(self.ax.contourf(X1, X2, Z, 10, - cmap=matplotlib.cm.bone, - origin='lower', alpha=0.85)) - self.contours.append(self.ax.contour(X1, X2, Z, [0.0], colors='k', - linestyles=['solid'])) - else: - raise ValueError("surface type unknown") - - -class ControllBar: - def __init__(self, root, controller): - fm = Tk.Frame(root) - kernel_group = Tk.Frame(fm) - Tk.Radiobutton(kernel_group, text="Linear", variable=controller.kernel, - value=0, command=controller.refit).pack(anchor=Tk.W) - Tk.Radiobutton(kernel_group, text="RBF", variable=controller.kernel, - value=1, command=controller.refit).pack(anchor=Tk.W) - Tk.Radiobutton(kernel_group, text="Poly", variable=controller.kernel, - value=2, command=controller.refit).pack(anchor=Tk.W) - kernel_group.pack(side=Tk.LEFT) - - valbox = Tk.Frame(fm) - controller.complexity = Tk.StringVar() - controller.complexity.set("1.0") - c = Tk.Frame(valbox) - Tk.Label(c, text="C:", anchor="e", width=7).pack(side=Tk.LEFT) - Tk.Entry(c, width=6, textvariable=controller.complexity).pack( - side=Tk.LEFT) - c.pack() - - controller.gamma = Tk.StringVar() - controller.gamma.set("0.01") - g = Tk.Frame(valbox) - Tk.Label(g, text="gamma:", anchor="e", width=7).pack(side=Tk.LEFT) - Tk.Entry(g, width=6, textvariable=controller.gamma).pack(side=Tk.LEFT) - g.pack() - - controller.degree = Tk.StringVar() - controller.degree.set("3") - d = Tk.Frame(valbox) - Tk.Label(d, text="degree:", anchor="e", width=7).pack(side=Tk.LEFT) - Tk.Entry(d, width=6, textvariable=controller.degree).pack(side=Tk.LEFT) - d.pack() - - controller.coef0 = Tk.StringVar() - controller.coef0.set("0") - r = Tk.Frame(valbox) - Tk.Label(r, text="coef0:", anchor="e", width=7).pack(side=Tk.LEFT) - Tk.Entry(r, width=6, textvariable=controller.coef0).pack(side=Tk.LEFT) - r.pack() - valbox.pack(side=Tk.LEFT) - - cmap_group = Tk.Frame(fm) - Tk.Radiobutton(cmap_group, text="Hyperplanes", - variable=controller.surface_type, value=0, - command=controller.refit).pack(anchor=Tk.W) - Tk.Radiobutton(cmap_group, text="Surface", - variable=controller.surface_type, value=1, - command=controller.refit).pack(anchor=Tk.W) - - cmap_group.pack(side=Tk.LEFT) - - train_button = Tk.Button(fm, text='Fit', width=5, - command=controller.fit) - train_button.pack() - fm.pack(side=Tk.LEFT) - Tk.Button(fm, text='Clear', width=5, - command=controller.clear_data).pack(side=Tk.LEFT) - - -def get_parser(): - from optparse import OptionParser - op = OptionParser() - op.add_option("--output", - action="store", type="str", dest="output", - help="Path where to dump data.") - return op - - -def main(argv): - op = get_parser() - opts, args = op.parse_args(argv[1:]) - root = Tk.Tk() - model = Model() - controller = Controller(model) - root.wm_title("Scikit-learn Libsvm GUI") - view = View(root, controller) - model.add_observer(view) - Tk.mainloop() - - if opts.output: - model.dump_svmlight_file(opts.output) - -if __name__ == "__main__": - main(sys.argv) diff --git a/examples/applications/wikipedia_principal_eigenvector.py b/examples/applications/wikipedia_principal_eigenvector.py index 097bab6c7d4d5..2ccd028b9a00d 100644 --- a/examples/applications/wikipedia_principal_eigenvector.py +++ b/examples/applications/wikipedia_principal_eigenvector.py @@ -6,10 +6,7 @@ A classical way to assert the relative importance of vertices in a graph is to compute the principal eigenvector of the adjacency matrix so as to assign to each vertex the values of the components of the first -eigenvector as a centrality score: - - https://en.wikipedia.org/wiki/Eigenvector_centrality - +eigenvector as a centrality score: https://en.wikipedia.org/wiki/Eigenvector_centrality. On the graph of webpages and links those values are called the PageRank scores by Google. @@ -18,38 +15,33 @@ this eigenvector centrality. The traditional way to compute the principal eigenvector is to use the -power iteration method: - - https://en.wikipedia.org/wiki/Power_iteration - +`power iteration method `_. Here the computation is achieved thanks to Martinsson's Randomized SVD algorithm implemented in scikit-learn. The graph data is fetched from the DBpedia dumps. DBpedia is an extraction of the latent structured data of the Wikipedia content. + """ -# Author: Olivier Grisel -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -from bz2 import BZ2File import os +from bz2 import BZ2File from datetime import datetime from pprint import pprint from time import time +from urllib.request import urlopen import numpy as np - from scipy import sparse from sklearn.decomposition import randomized_svd -from urllib.request import urlopen - - -print(__doc__) -# ############################################################################# -# Where to download the data, if not already on disk +# %% +# Download data, if not already on disk +# ------------------------------------- redirects_url = "http://downloads.dbpedia.org/3.5.1/en/redirects_en.nt.bz2" redirects_filename = redirects_url.rsplit("/", 1)[1] @@ -65,14 +57,14 @@ if not os.path.exists(filename): print("Downloading data from '%s', please wait..." % url) opener = urlopen(url) - open(filename, 'wb').write(opener.read()) + with open(filename, "wb") as f: + f.write(opener.read()) print() -# ############################################################################# +# %% # Loading the redirect files - - +# -------------------------- def index(redirects, index_map, k): """Find the index of an article name after redirect resolution""" k = redirects.get(k, k) @@ -120,6 +112,9 @@ def get_redirects(redirects_filename): return redirects +# %% +# Computing the Adjacency matrix +# ------------------------------ def get_adjacency_matrix(redirects_filename, page_links_filename, limit=None): """Extract the adjacency graph as a scipy sparse matrix @@ -163,9 +158,14 @@ def get_adjacency_matrix(redirects_filename, page_links_filename, limit=None): # stop after 5M links to make it possible to work in RAM X, redirects, index_map = get_adjacency_matrix( - redirects_filename, page_links_filename, limit=5000000) + redirects_filename, page_links_filename, limit=5000000 +) names = {i: name for name, i in index_map.items()} + +# %% +# Computing Principal Singular Vector using Randomized SVD +# -------------------------------------------------------- print("Computing the principal singular vectors using randomized_svd") t0 = time() U, s, V = randomized_svd(X, 5, n_iter=3) @@ -178,6 +178,9 @@ def get_adjacency_matrix(redirects_filename, page_links_filename, limit=None): pprint([names[i] for i in np.abs(V[0]).argsort()[-10:]]) +# %% +# Computing Centrality scores +# --------------------------- def centrality_scores(X, alpha=0.85, max_iter=100, tol=1e-10): """Power iteration computation of the principal eigenvector @@ -195,16 +198,17 @@ def centrality_scores(X, alpha=0.85, max_iter=100, tol=1e-10): print("Normalizing the graph") for i in incoming_counts.nonzero()[0]: - X.data[X.indptr[i]:X.indptr[i + 1]] *= 1.0 / incoming_counts[i] - dangle = np.asarray(np.where(np.isclose(X.sum(axis=1), 0), - 1.0 / n, 0)).ravel() + X.data[X.indptr[i] : X.indptr[i + 1]] *= 1.0 / incoming_counts[i] + dangle = np.asarray(np.where(np.isclose(X.sum(axis=1), 0), 1.0 / n, 0)).ravel() - scores = np.full(n, 1. / n, dtype=np.float32) # initial guess + scores = np.full(n, 1.0 / n, dtype=np.float32) # initial guess for i in range(max_iter): print("power iteration #%d" % i) prev_scores = scores - scores = (alpha * (scores * X + np.dot(dangle, prev_scores)) - + (1 - alpha) * prev_scores.sum() / n) + scores = ( + alpha * (scores * X + np.dot(dangle, prev_scores)) + + (1 - alpha) * prev_scores.sum() / n + ) # check convergence: normalized l_inf norm scores_max = np.abs(scores).max() if scores_max == 0.0: @@ -216,6 +220,7 @@ def centrality_scores(X, alpha=0.85, max_iter=100, tol=1e-10): return scores + print("Computing principal eigenvector score using a power iteration method") t0 = time() scores = centrality_scores(X, max_iter=100) diff --git a/examples/bicluster/README.txt b/examples/bicluster/README.txt index 468e2524eb310..0b2bda2522b63 100644 --- a/examples/bicluster/README.txt +++ b/examples/bicluster/README.txt @@ -3,4 +3,4 @@ Biclustering ------------ -Examples concerning the :mod:`sklearn.cluster.bicluster` module. +Examples concerning biclustering techniques. diff --git a/examples/bicluster/plot_bicluster_newsgroups.py b/examples/bicluster/plot_bicluster_newsgroups.py index 250c22e78f796..054fb0ba399e1 100644 --- a/examples/bicluster/plot_bicluster_newsgroups.py +++ b/examples/bicluster/plot_bicluster_newsgroups.py @@ -22,23 +22,22 @@ achieve a better V-measure than clusters found by MiniBatchKMeans. """ -from collections import defaultdict -import operator + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +from collections import Counter from time import time import numpy as np -from sklearn.cluster import SpectralCoclustering -from sklearn.cluster import MiniBatchKMeans +from sklearn.cluster import MiniBatchKMeans, SpectralCoclustering from sklearn.datasets import fetch_20newsgroups from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.metrics.cluster import v_measure_score -print(__doc__) - def number_normalizer(tokens): - """ Map all numeric tokens to a placeholder. + """Map all numeric tokens to a placeholder. For many applications, tokens that begin with a number are not directly useful, but the fact that such a token exists can be relevant. By applying @@ -54,22 +53,37 @@ def build_tokenizer(self): # exclude 'comp.os.ms-windows.misc' -categories = ['alt.atheism', 'comp.graphics', - 'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware', - 'comp.windows.x', 'misc.forsale', 'rec.autos', - 'rec.motorcycles', 'rec.sport.baseball', - 'rec.sport.hockey', 'sci.crypt', 'sci.electronics', - 'sci.med', 'sci.space', 'soc.religion.christian', - 'talk.politics.guns', 'talk.politics.mideast', - 'talk.politics.misc', 'talk.religion.misc'] +categories = [ + "alt.atheism", + "comp.graphics", + "comp.sys.ibm.pc.hardware", + "comp.sys.mac.hardware", + "comp.windows.x", + "misc.forsale", + "rec.autos", + "rec.motorcycles", + "rec.sport.baseball", + "rec.sport.hockey", + "sci.crypt", + "sci.electronics", + "sci.med", + "sci.space", + "soc.religion.christian", + "talk.politics.guns", + "talk.politics.mideast", + "talk.politics.misc", + "talk.religion.misc", +] newsgroups = fetch_20newsgroups(categories=categories) y_true = newsgroups.target -vectorizer = NumberNormalizingVectorizer(stop_words='english', min_df=5) -cocluster = SpectralCoclustering(n_clusters=len(categories), - svd_method='arpack', random_state=0) -kmeans = MiniBatchKMeans(n_clusters=len(categories), batch_size=20000, - random_state=0) +vectorizer = NumberNormalizingVectorizer(stop_words="english", min_df=5) +cocluster = SpectralCoclustering( + n_clusters=len(categories), svd_method="arpack", random_state=0 +) +kmeans = MiniBatchKMeans( + n_clusters=len(categories), batch_size=20000, random_state=0, n_init=3 +) print("Vectorizing...") X = vectorizer.fit_transform(newsgroups.data) @@ -78,18 +92,22 @@ def build_tokenizer(self): start_time = time() cocluster.fit(X) y_cocluster = cocluster.row_labels_ -print("Done in {:.2f}s. V-measure: {:.4f}".format( - time() - start_time, - v_measure_score(y_cocluster, y_true))) +print( + f"Done in {time() - start_time:.2f}s. V-measure: \ +{v_measure_score(y_cocluster, y_true):.4f}" +) + print("MiniBatchKMeans...") start_time = time() y_kmeans = kmeans.fit_predict(X) -print("Done in {:.2f}s. V-measure: {:.4f}".format( - time() - start_time, - v_measure_score(y_kmeans, y_true))) +print( + f"Done in {time() - start_time:.2f}s. V-measure: \ +{v_measure_score(y_kmeans, y_true):.4f}" +) -feature_names = vectorizer.get_feature_names() + +feature_names = vectorizer.get_feature_names_out() document_names = list(newsgroups.target_names[i] for i in newsgroups.target) @@ -97,27 +115,18 @@ def bicluster_ncut(i): rows, cols = cocluster.get_indices(i) if not (np.any(rows) and np.any(cols)): import sys + return sys.float_info.max row_complement = np.nonzero(np.logical_not(cocluster.rows_[i]))[0] col_complement = np.nonzero(np.logical_not(cocluster.columns_[i]))[0] # Note: the following is identical to X[rows[:, np.newaxis], # cols].sum() but much faster in scipy <= 0.16 weight = X[rows][:, cols].sum() - cut = (X[row_complement][:, cols].sum() + - X[rows][:, col_complement].sum()) + cut = X[row_complement][:, cols].sum() + X[rows][:, col_complement].sum() return cut / weight -def most_common(d): - """Items of a defaultdict(int) with the highest values. - - Like Counter.most_common in Python >=2.7. - """ - return sorted(d.items(), key=operator.itemgetter(1), reverse=True) - - -bicluster_ncuts = list(bicluster_ncut(i) - for i in range(len(newsgroups.target_names))) +bicluster_ncuts = list(bicluster_ncut(i) for i in range(len(newsgroups.target_names))) best_idx = np.argsort(bicluster_ncuts)[:5] print() @@ -130,23 +139,25 @@ def most_common(d): continue # categories - counter = defaultdict(int) - for i in cluster_docs: - counter[document_names[i]] += 1 - cat_string = ", ".join("{:.0f}% {}".format(float(c) / n_rows * 100, name) - for name, c in most_common(counter)[:3]) + counter = Counter(document_names[doc] for doc in cluster_docs) + + cat_string = ", ".join( + f"{(c / n_rows * 100):.0f}% {name}" for name, c in counter.most_common(3) + ) # words out_of_cluster_docs = cocluster.row_labels_ != cluster - out_of_cluster_docs = np.where(out_of_cluster_docs)[0] + out_of_cluster_docs = out_of_cluster_docs.nonzero()[0] word_col = X[:, cluster_words] - word_scores = np.array(word_col[cluster_docs, :].sum(axis=0) - - word_col[out_of_cluster_docs, :].sum(axis=0)) + word_scores = np.array( + word_col[cluster_docs, :].sum(axis=0) + - word_col[out_of_cluster_docs, :].sum(axis=0) + ) word_scores = word_scores.ravel() - important_words = list(feature_names[cluster_words[i]] - for i in word_scores.argsort()[:-11:-1]) + important_words = list( + feature_names[cluster_words[i]] for i in word_scores.argsort()[:-11:-1] + ) - print("bicluster {} : {} documents, {} words".format( - idx, n_rows, n_cols)) - print("categories : {}".format(cat_string)) - print("words : {}\n".format(', '.join(important_words))) + print(f"bicluster {idx} : {n_rows} documents, {n_cols} words") + print(f"categories : {cat_string}") + print(f"words : {', '.join(important_words)}\n") diff --git a/examples/bicluster/plot_spectral_biclustering.py b/examples/bicluster/plot_spectral_biclustering.py index abc63879a8420..86245325ae493 100644 --- a/examples/bicluster/plot_spectral_biclustering.py +++ b/examples/bicluster/plot_spectral_biclustering.py @@ -3,65 +3,121 @@ A demo of the Spectral Biclustering algorithm ============================================= -This example demonstrates how to generate a checkerboard dataset and -bicluster it using the Spectral Biclustering algorithm. - -The data is generated with the ``make_checkerboard`` function, then -shuffled and passed to the Spectral Biclustering algorithm. The rows -and columns of the shuffled matrix are rearranged to show the -biclusters found by the algorithm. - -The outer product of the row and column label vectors shows a -representation of the checkerboard structure. - +This example demonstrates how to generate a checkerboard dataset and bicluster +it using the :class:`~sklearn.cluster.SpectralBiclustering` algorithm. The +spectral biclustering algorithm is specifically designed to cluster data by +simultaneously considering both the rows (samples) and columns (features) of a +matrix. It aims to identify patterns not only between samples but also within +subsets of samples, allowing for the detection of localized structure within the +data. This makes spectral biclustering particularly well-suited for datasets +where the order or arrangement of features is fixed, such as in images, time +series, or genomes. + +The data is generated, then shuffled and passed to the spectral biclustering +algorithm. The rows and columns of the shuffled matrix are then rearranged to +plot the biclusters found. """ -print(__doc__) -# Author: Kemal Eren -# License: BSD 3 clause - -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Generate sample data +# -------------------- +# We generate the sample data using the +# :func:`~sklearn.datasets.make_checkerboard` function. Each pixel within +# `shape=(300, 300)` represents with its color a value from a uniform +# distribution. The noise is added from a normal distribution, where the value +# chosen for `noise` is the standard deviation. +# +# As you can see, the data is distributed over 12 cluster cells and is +# relatively well distinguishable. from matplotlib import pyplot as plt from sklearn.datasets import make_checkerboard -from sklearn.cluster import SpectralBiclustering -from sklearn.metrics import consensus_score - n_clusters = (4, 3) data, rows, columns = make_checkerboard( - shape=(300, 300), n_clusters=n_clusters, noise=10, - shuffle=False, random_state=0) + shape=(300, 300), n_clusters=n_clusters, noise=10, shuffle=False, random_state=42 +) plt.matshow(data, cmap=plt.cm.Blues) plt.title("Original dataset") +_ = plt.show() + +# %% +# We shuffle the data and the goal is to reconstruct it afterwards using +# :class:`~sklearn.cluster.SpectralBiclustering`. +import numpy as np -# shuffle clusters +# Creating lists of shuffled row and column indices rng = np.random.RandomState(0) -row_idx = rng.permutation(data.shape[0]) -col_idx = rng.permutation(data.shape[1]) -data = data[row_idx][:, col_idx] +row_idx_shuffled = rng.permutation(data.shape[0]) +col_idx_shuffled = rng.permutation(data.shape[1]) + +# %% +# We redefine the shuffled data and plot it. We observe that we lost the +# structure of original data matrix. +data = data[row_idx_shuffled][:, col_idx_shuffled] plt.matshow(data, cmap=plt.cm.Blues) plt.title("Shuffled dataset") +_ = plt.show() + +# %% +# Fitting `SpectralBiclustering` +# ------------------------------ +# We fit the model and compare the obtained clusters with the ground truth. Note +# that when creating the model we specify the same number of clusters that we +# used to create the dataset (`n_clusters = (4, 3)`), which will contribute to +# obtain a good result. +from sklearn.cluster import SpectralBiclustering +from sklearn.metrics import consensus_score -model = SpectralBiclustering(n_clusters=n_clusters, method='log', - random_state=0) +model = SpectralBiclustering(n_clusters=n_clusters, method="log", random_state=0) model.fit(data) -score = consensus_score(model.biclusters_, - (rows[:, row_idx], columns[:, col_idx])) - -print("consensus score: {:.1f}".format(score)) -fit_data = data[np.argsort(model.row_labels_)] -fit_data = fit_data[:, np.argsort(model.column_labels_)] - -plt.matshow(fit_data, cmap=plt.cm.Blues) +# Compute the similarity of two sets of biclusters +score = consensus_score( + model.biclusters_, (rows[:, row_idx_shuffled], columns[:, col_idx_shuffled]) +) +print(f"consensus score: {score:.1f}") + +# %% +# The score is between 0 and 1, where 1 corresponds to a perfect matching. It +# shows the quality of the biclustering. + +# %% +# Plotting results +# ---------------- +# Now, we rearrange the data based on the row and column labels assigned by the +# :class:`~sklearn.cluster.SpectralBiclustering` model in ascending order and +# plot again. The `row_labels_` range from 0 to 3, while the `column_labels_` +# range from 0 to 2, representing a total of 4 clusters per row and 3 clusters +# per column. + +# Reordering first the rows and then the columns. +reordered_rows = data[np.argsort(model.row_labels_)] +reordered_data = reordered_rows[:, np.argsort(model.column_labels_)] + +plt.matshow(reordered_data, cmap=plt.cm.Blues) plt.title("After biclustering; rearranged to show biclusters") - -plt.matshow(np.outer(np.sort(model.row_labels_) + 1, - np.sort(model.column_labels_) + 1), - cmap=plt.cm.Blues) +_ = plt.show() + +# %% +# As a last step, we want to demonstrate the relationships between the row +# and column labels assigned by the model. Therefore, we create a grid with +# :func:`numpy.outer`, which takes the sorted `row_labels_` and `column_labels_` +# and adds 1 to each to ensure that the labels start from 1 instead of 0 for +# better visualization. +plt.matshow( + np.outer(np.sort(model.row_labels_) + 1, np.sort(model.column_labels_) + 1), + cmap=plt.cm.Blues, +) plt.title("Checkerboard structure of rearranged data") - plt.show() + +# %% +# The outer product of the row and column label vectors shows a representation +# of the checkerboard structure, where different combinations of row and column +# labels are represented by different shades of blue. diff --git a/examples/bicluster/plot_spectral_coclustering.py b/examples/bicluster/plot_spectral_coclustering.py index 0681d51e0bfd3..a0edaceafeb66 100644 --- a/examples/bicluster/plot_spectral_coclustering.py +++ b/examples/bicluster/plot_spectral_coclustering.py @@ -14,21 +14,20 @@ the biclusters. """ -print(__doc__) -# Author: Kemal Eren -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import numpy as np from matplotlib import pyplot as plt -from sklearn.datasets import make_biclusters from sklearn.cluster import SpectralCoclustering +from sklearn.datasets import make_biclusters from sklearn.metrics import consensus_score data, rows, columns = make_biclusters( - shape=(300, 300), n_clusters=5, noise=5, - shuffle=False, random_state=0) + shape=(300, 300), n_clusters=5, noise=5, shuffle=False, random_state=0 +) plt.matshow(data, cmap=plt.cm.Blues) plt.title("Original dataset") @@ -44,8 +43,7 @@ model = SpectralCoclustering(n_clusters=5, random_state=0) model.fit(data) -score = consensus_score(model.biclusters_, - (rows[:, row_idx], columns[:, col_idx])) +score = consensus_score(model.biclusters_, (rows[:, row_idx], columns[:, col_idx])) print("consensus score: {:.3f}".format(score)) diff --git a/examples/calibration/plot_calibration.py b/examples/calibration/plot_calibration.py index e8d8d5184a178..e4826ea33b1d8 100644 --- a/examples/calibration/plot_calibration.py +++ b/examples/calibration/plot_calibration.py @@ -20,56 +20,56 @@ to the expected 0.5 for most of the samples belonging to the middle cluster with heterogeneous labels. This results in a significantly improved Brier score. + """ -print(__doc__) -# Author: Mathieu Blondel -# Alexandre Gramfort -# Balazs Kegl -# Jan Hendrik Metzen -# License: BSD Style. +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +# %% +# Generate synthetic dataset +# -------------------------- import numpy as np -import matplotlib.pyplot as plt -from matplotlib import cm from sklearn.datasets import make_blobs -from sklearn.naive_bayes import GaussianNB -from sklearn.metrics import brier_score_loss -from sklearn.calibration import CalibratedClassifierCV from sklearn.model_selection import train_test_split - n_samples = 50000 -n_bins = 3 # use 3 bins for calibration_curve as we have 3 clusters here # Generate 3 blobs with 2 classes where the second blob contains # half positive samples and half negative samples. Probability in this # blob is therefore 0.5. centers = [(-5, -5), (0, 0), (5, 5)] -X, y = make_blobs(n_samples=n_samples, centers=centers, shuffle=False, - random_state=42) +X, y = make_blobs(n_samples=n_samples, centers=centers, shuffle=False, random_state=42) -y[:n_samples // 2] = 0 -y[n_samples // 2:] = 1 +y[: n_samples // 2] = 0 +y[n_samples // 2 :] = 1 sample_weight = np.random.RandomState(42).rand(y.shape[0]) # split train, test for calibration -X_train, X_test, y_train, y_test, sw_train, sw_test = \ - train_test_split(X, y, sample_weight, test_size=0.9, random_state=42) +X_train, X_test, y_train, y_test, sw_train, sw_test = train_test_split( + X, y, sample_weight, test_size=0.9, random_state=42 +) -# Gaussian Naive-Bayes with no calibration +# %% +# Gaussian Naive-Bayes +# -------------------- +from sklearn.calibration import CalibratedClassifierCV +from sklearn.metrics import brier_score_loss +from sklearn.naive_bayes import GaussianNB + +# With no calibration clf = GaussianNB() clf.fit(X_train, y_train) # GaussianNB itself does not support sample-weights prob_pos_clf = clf.predict_proba(X_test)[:, 1] -# Gaussian Naive-Bayes with isotonic calibration -clf_isotonic = CalibratedClassifierCV(clf, cv=2, method='isotonic') +# With isotonic calibration +clf_isotonic = CalibratedClassifierCV(clf, cv=2, method="isotonic") clf_isotonic.fit(X_train, y_train, sample_weight=sw_train) prob_pos_isotonic = clf_isotonic.predict_proba(X_test)[:, 1] -# Gaussian Naive-Bayes with sigmoid calibration -clf_sigmoid = CalibratedClassifierCV(clf, cv=2, method='sigmoid') +# With sigmoid calibration +clf_sigmoid = CalibratedClassifierCV(clf, cv=2, method="sigmoid") clf_sigmoid.fit(X_train, y_train, sample_weight=sw_train) prob_pos_sigmoid = clf_sigmoid.predict_proba(X_test)[:, 1] @@ -78,42 +78,61 @@ clf_score = brier_score_loss(y_test, prob_pos_clf, sample_weight=sw_test) print("No calibration: %1.3f" % clf_score) -clf_isotonic_score = brier_score_loss(y_test, prob_pos_isotonic, - sample_weight=sw_test) +clf_isotonic_score = brier_score_loss(y_test, prob_pos_isotonic, sample_weight=sw_test) print("With isotonic calibration: %1.3f" % clf_isotonic_score) -clf_sigmoid_score = brier_score_loss(y_test, prob_pos_sigmoid, - sample_weight=sw_test) +clf_sigmoid_score = brier_score_loss(y_test, prob_pos_sigmoid, sample_weight=sw_test) print("With sigmoid calibration: %1.3f" % clf_sigmoid_score) -# ############################################################################# -# Plot the data and the predicted probabilities +# %% +# Plot data and the predicted probabilities +# ----------------------------------------- +import matplotlib.pyplot as plt +from matplotlib import cm + plt.figure() y_unique = np.unique(y) colors = cm.rainbow(np.linspace(0.0, 1.0, y_unique.size)) for this_y, color in zip(y_unique, colors): this_X = X_train[y_train == this_y] this_sw = sw_train[y_train == this_y] - plt.scatter(this_X[:, 0], this_X[:, 1], s=this_sw * 50, - c=color[np.newaxis, :], - alpha=0.5, edgecolor='k', - label="Class %s" % this_y) + plt.scatter( + this_X[:, 0], + this_X[:, 1], + s=this_sw * 50, + c=color[np.newaxis, :], + alpha=0.5, + edgecolor="k", + label="Class %s" % this_y, + ) plt.legend(loc="best") plt.title("Data") plt.figure() -order = np.lexsort((prob_pos_clf, )) -plt.plot(prob_pos_clf[order], 'r', label='No calibration (%1.3f)' % clf_score) -plt.plot(prob_pos_isotonic[order], 'g', linewidth=3, - label='Isotonic calibration (%1.3f)' % clf_isotonic_score) -plt.plot(prob_pos_sigmoid[order], 'b', linewidth=3, - label='Sigmoid calibration (%1.3f)' % clf_sigmoid_score) -plt.plot(np.linspace(0, y_test.size, 51)[1::2], - y_test[order].reshape(25, -1).mean(1), - 'k', linewidth=3, label=r'Empirical') + +order = np.lexsort((prob_pos_clf,)) +plt.plot(prob_pos_clf[order], "r", label="No calibration (%1.3f)" % clf_score) +plt.plot( + prob_pos_isotonic[order], + "g", + linewidth=3, + label="Isotonic calibration (%1.3f)" % clf_isotonic_score, +) +plt.plot( + prob_pos_sigmoid[order], + "b", + linewidth=3, + label="Sigmoid calibration (%1.3f)" % clf_sigmoid_score, +) +plt.plot( + np.linspace(0, y_test.size, 51)[1::2], + y_test[order].reshape(25, -1).mean(1), + "k", + linewidth=3, + label=r"Empirical", +) plt.ylim([-0.05, 1.05]) -plt.xlabel("Instances sorted according to predicted probability " - "(uncalibrated GNB)") +plt.xlabel("Instances sorted according to predicted probability (uncalibrated GNB)") plt.ylabel("P(y=1)") plt.legend(loc="upper left") plt.title("Gaussian naive Bayes probabilities") diff --git a/examples/calibration/plot_calibration_curve.py b/examples/calibration/plot_calibration_curve.py index 3218e5c223343..1c5e297026ae7 100644 --- a/examples/calibration/plot_calibration_curve.py +++ b/examples/calibration/plot_calibration_curve.py @@ -5,130 +5,332 @@ When performing classification one often wants to predict not only the class label, but also the associated probability. This probability gives some -kind of confidence on the prediction. This example demonstrates how to display -how well calibrated the predicted probabilities are and how to calibrate an -uncalibrated classifier. - -The experiment is performed on an artificial dataset for binary classification -with 100,000 samples (1,000 of them are used for model fitting) with 20 -features. Of the 20 features, only 2 are informative and 10 are redundant. The -first figure shows the estimated probabilities obtained with logistic -regression, Gaussian naive Bayes, and Gaussian naive Bayes with both isotonic -calibration and sigmoid calibration. The calibration performance is evaluated -with Brier score, reported in the legend (the smaller the better). One can -observe here that logistic regression is well calibrated while raw Gaussian -naive Bayes performs very badly. This is because of the redundant features -which violate the assumption of feature-independence and result in an overly -confident classifier, which is indicated by the typical transposed-sigmoid -curve. - -Calibration of the probabilities of Gaussian naive Bayes with isotonic -regression can fix this issue as can be seen from the nearly diagonal -calibration curve. Sigmoid calibration also improves the brier score slightly, -albeit not as strongly as the non-parametric isotonic regression. This can be -attributed to the fact that we have plenty of calibration data such that the -greater flexibility of the non-parametric model can be exploited. - -The second figure shows the calibration curve of a linear support-vector -classifier (LinearSVC). LinearSVC shows the opposite behavior as Gaussian -naive Bayes: the calibration curve has a sigmoid curve, which is typical for -an under-confident classifier. In the case of LinearSVC, this is caused by the -margin property of the hinge loss, which lets the model focus on hard samples -that are close to the decision boundary (the support vectors). - -Both kinds of calibration can fix this issue and yield nearly identical -results. This shows that sigmoid calibration can deal with situations where -the calibration curve of the base classifier is sigmoid (e.g., for LinearSVC) -but not where it is transposed-sigmoid (e.g., Gaussian naive Bayes). +kind of confidence on the prediction. This example demonstrates how to +visualize how well calibrated the predicted probabilities are using calibration +curves, also known as reliability diagrams. Calibration of an uncalibrated +classifier will also be demonstrated. + """ -print(__doc__) -# Author: Alexandre Gramfort -# Jan Hendrik Metzen -# License: BSD Style. +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Dataset +# ------- +# +# We will use a synthetic binary classification dataset with 100,000 samples +# and 20 features. Of the 20 features, only 2 are informative, 10 are +# redundant (random combinations of the informative features) and the +# remaining 8 are uninformative (random numbers). Of the 100,000 samples, 1,000 +# will be used for model fitting and the rest for testing. + +from sklearn.datasets import make_classification +from sklearn.model_selection import train_test_split + +X, y = make_classification( + n_samples=100_000, n_features=20, n_informative=2, n_redundant=10, random_state=42 +) + +X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.99, random_state=42 +) + +# %% +# Calibration curves +# ------------------ +# +# Gaussian Naive Bayes +# ^^^^^^^^^^^^^^^^^^^^ +# +# First, we will compare: +# +# * :class:`~sklearn.linear_model.LogisticRegression` (used as baseline +# since very often, properly regularized logistic regression is well +# calibrated by default thanks to the use of the log-loss) +# * Uncalibrated :class:`~sklearn.naive_bayes.GaussianNB` +# * :class:`~sklearn.naive_bayes.GaussianNB` with isotonic and sigmoid +# calibration (see :ref:`User Guide `) +# +# Calibration curves for all 4 conditions are plotted below, with the average +# predicted probability for each bin on the x-axis and the fraction of positive +# classes in each bin on the y-axis. import matplotlib.pyplot as plt +from matplotlib.gridspec import GridSpec -from sklearn import datasets -from sklearn.naive_bayes import GaussianNB -from sklearn.svm import LinearSVC +from sklearn.calibration import CalibratedClassifierCV, CalibrationDisplay from sklearn.linear_model import LogisticRegression -from sklearn.metrics import (brier_score_loss, precision_score, recall_score, - f1_score) -from sklearn.calibration import CalibratedClassifierCV, calibration_curve -from sklearn.model_selection import train_test_split +from sklearn.naive_bayes import GaussianNB + +lr = LogisticRegression(C=1.0) +gnb = GaussianNB() +gnb_isotonic = CalibratedClassifierCV(gnb, cv=2, method="isotonic") +gnb_sigmoid = CalibratedClassifierCV(gnb, cv=2, method="sigmoid") +clf_list = [ + (lr, "Logistic"), + (gnb, "Naive Bayes"), + (gnb_isotonic, "Naive Bayes + Isotonic"), + (gnb_sigmoid, "Naive Bayes + Sigmoid"), +] -# Create dataset of classification task with many redundant and few -# informative features -X, y = datasets.make_classification(n_samples=100000, n_features=20, - n_informative=2, n_redundant=10, - random_state=42) +# %% +fig = plt.figure(figsize=(10, 10)) +gs = GridSpec(4, 2) +colors = plt.get_cmap("Dark2") -X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.99, - random_state=42) +ax_calibration_curve = fig.add_subplot(gs[:2, :2]) +calibration_displays = {} +for i, (clf, name) in enumerate(clf_list): + clf.fit(X_train, y_train) + display = CalibrationDisplay.from_estimator( + clf, + X_test, + y_test, + n_bins=10, + name=name, + ax=ax_calibration_curve, + color=colors(i), + ) + calibration_displays[name] = display +ax_calibration_curve.grid() +ax_calibration_curve.set_title("Calibration plots (Naive Bayes)") -def plot_calibration_curve(est, name, fig_index): - """Plot calibration curve for est w/o and with calibration. """ - # Calibrated with isotonic calibration - isotonic = CalibratedClassifierCV(est, cv=2, method='isotonic') +# Add histogram +grid_positions = [(2, 0), (2, 1), (3, 0), (3, 1)] +for i, (_, name) in enumerate(clf_list): + row, col = grid_positions[i] + ax = fig.add_subplot(gs[row, col]) - # Calibrated with sigmoid calibration - sigmoid = CalibratedClassifierCV(est, cv=2, method='sigmoid') + ax.hist( + calibration_displays[name].y_prob, + range=(0, 1), + bins=10, + label=name, + color=colors(i), + ) + ax.set(title=name, xlabel="Mean predicted probability", ylabel="Count") + +plt.tight_layout() +plt.show() - # Logistic regression with no calibration as baseline - lr = LogisticRegression(C=1.) +# %% +# Uncalibrated :class:`~sklearn.naive_bayes.GaussianNB` is poorly calibrated +# because of +# the redundant features which violate the assumption of feature-independence +# and result in an overly confident classifier, which is indicated by the +# typical transposed-sigmoid curve. Calibration of the probabilities of +# :class:`~sklearn.naive_bayes.GaussianNB` with :ref:`isotonic` can fix +# this issue as can be seen from the nearly diagonal calibration curve. +# :ref:`Sigmoid regression ` also improves calibration +# slightly, +# albeit not as strongly as the non-parametric isotonic regression. This can be +# attributed to the fact that we have plenty of calibration data such that the +# greater flexibility of the non-parametric model can be exploited. +# +# Below we will make a quantitative analysis considering several classification +# metrics: :ref:`brier_score_loss`, :ref:`log_loss`, +# :ref:`precision, recall, F1 score ` and +# :ref:`ROC AUC `. - fig = plt.figure(fig_index, figsize=(10, 10)) - ax1 = plt.subplot2grid((3, 1), (0, 0), rowspan=2) - ax2 = plt.subplot2grid((3, 1), (2, 0)) +from collections import defaultdict - ax1.plot([0, 1], [0, 1], "k:", label="Perfectly calibrated") - for clf, name in [(lr, 'Logistic'), - (est, name), - (isotonic, name + ' + Isotonic'), - (sigmoid, name + ' + Sigmoid')]: - clf.fit(X_train, y_train) - y_pred = clf.predict(X_test) - if hasattr(clf, "predict_proba"): - prob_pos = clf.predict_proba(X_test)[:, 1] - else: # use decision function - prob_pos = clf.decision_function(X_test) - prob_pos = \ - (prob_pos - prob_pos.min()) / (prob_pos.max() - prob_pos.min()) +import pandas as pd - clf_score = brier_score_loss(y_test, prob_pos, pos_label=y.max()) - print("%s:" % name) - print("\tBrier: %1.3f" % (clf_score)) - print("\tPrecision: %1.3f" % precision_score(y_test, y_pred)) - print("\tRecall: %1.3f" % recall_score(y_test, y_pred)) - print("\tF1: %1.3f\n" % f1_score(y_test, y_pred)) +from sklearn.metrics import ( + brier_score_loss, + f1_score, + log_loss, + precision_score, + recall_score, + roc_auc_score, +) - fraction_of_positives, mean_predicted_value = \ - calibration_curve(y_test, prob_pos, n_bins=10) +scores = defaultdict(list) +for i, (clf, name) in enumerate(clf_list): + clf.fit(X_train, y_train) + y_prob = clf.predict_proba(X_test) + y_pred = clf.predict(X_test) + scores["Classifier"].append(name) - ax1.plot(mean_predicted_value, fraction_of_positives, "s-", - label="%s (%1.3f)" % (name, clf_score)) + for metric in [brier_score_loss, log_loss, roc_auc_score]: + score_name = metric.__name__.replace("_", " ").replace("score", "").capitalize() + scores[score_name].append(metric(y_test, y_prob[:, 1])) - ax2.hist(prob_pos, range=(0, 1), bins=10, label=name, - histtype="step", lw=2) + for metric in [precision_score, recall_score, f1_score]: + score_name = metric.__name__.replace("_", " ").replace("score", "").capitalize() + scores[score_name].append(metric(y_test, y_pred)) - ax1.set_ylabel("Fraction of positives") - ax1.set_ylim([-0.05, 1.05]) - ax1.legend(loc="lower right") - ax1.set_title('Calibration plots (reliability curve)') + score_df = pd.DataFrame(scores).set_index("Classifier") + score_df.round(decimals=3) - ax2.set_xlabel("Mean predicted value") - ax2.set_ylabel("Count") - ax2.legend(loc="upper center", ncol=2) +score_df + +# %% +# Notice that although calibration improves the :ref:`brier_score_loss` (a +# metric composed +# of calibration term and refinement term) and :ref:`log_loss`, it does not +# significantly alter the prediction accuracy measures (precision, recall and +# F1 score). +# This is because calibration should not significantly change prediction +# probabilities at the location of the decision threshold (at x = 0.5 on the +# graph). Calibration should however, make the predicted probabilities more +# accurate and thus more useful for making allocation decisions under +# uncertainty. +# Further, ROC AUC, should not change at all because calibration is a +# monotonic transformation. Indeed, no rank metrics are affected by +# calibration. +# +# Linear support vector classifier +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# Next, we will compare: +# +# * :class:`~sklearn.linear_model.LogisticRegression` (baseline) +# * Uncalibrated :class:`~sklearn.svm.LinearSVC`. Since SVC does not output +# probabilities by default, we naively scale the output of the +# :term:`decision_function` into [0, 1] by applying min-max scaling. +# * :class:`~sklearn.svm.LinearSVC` with isotonic and sigmoid +# calibration (see :ref:`User Guide `) + +import numpy as np + +from sklearn.svm import LinearSVC - plt.tight_layout() -# Plot calibration curve for Gaussian Naive Bayes -plot_calibration_curve(GaussianNB(), "Naive Bayes", 1) +class NaivelyCalibratedLinearSVC(LinearSVC): + """LinearSVC with `predict_proba` method that naively scales + `decision_function` output for binary classification.""" -# Plot calibration curve for Linear SVC -plot_calibration_curve(LinearSVC(max_iter=10000), "SVC", 2) + def fit(self, X, y): + super().fit(X, y) + df = self.decision_function(X) + self.df_min_ = df.min() + self.df_max_ = df.max() + def predict_proba(self, X): + """Min-max scale output of `decision_function` to [0, 1].""" + df = self.decision_function(X) + calibrated_df = (df - self.df_min_) / (self.df_max_ - self.df_min_) + proba_pos_class = np.clip(calibrated_df, 0, 1) + proba_neg_class = 1 - proba_pos_class + proba = np.c_[proba_neg_class, proba_pos_class] + return proba + + +# %% + +lr = LogisticRegression(C=1.0) +svc = NaivelyCalibratedLinearSVC(max_iter=10_000) +svc_isotonic = CalibratedClassifierCV(svc, cv=2, method="isotonic") +svc_sigmoid = CalibratedClassifierCV(svc, cv=2, method="sigmoid") + +clf_list = [ + (lr, "Logistic"), + (svc, "SVC"), + (svc_isotonic, "SVC + Isotonic"), + (svc_sigmoid, "SVC + Sigmoid"), +] + +# %% +fig = plt.figure(figsize=(10, 10)) +gs = GridSpec(4, 2) + +ax_calibration_curve = fig.add_subplot(gs[:2, :2]) +calibration_displays = {} +for i, (clf, name) in enumerate(clf_list): + clf.fit(X_train, y_train) + display = CalibrationDisplay.from_estimator( + clf, + X_test, + y_test, + n_bins=10, + name=name, + ax=ax_calibration_curve, + color=colors(i), + ) + calibration_displays[name] = display + +ax_calibration_curve.grid() +ax_calibration_curve.set_title("Calibration plots (SVC)") + +# Add histogram +grid_positions = [(2, 0), (2, 1), (3, 0), (3, 1)] +for i, (_, name) in enumerate(clf_list): + row, col = grid_positions[i] + ax = fig.add_subplot(gs[row, col]) + + ax.hist( + calibration_displays[name].y_prob, + range=(0, 1), + bins=10, + label=name, + color=colors(i), + ) + ax.set(title=name, xlabel="Mean predicted probability", ylabel="Count") + +plt.tight_layout() plt.show() + +# %% +# :class:`~sklearn.svm.LinearSVC` shows the opposite +# behavior to :class:`~sklearn.naive_bayes.GaussianNB`; the calibration +# curve has a sigmoid shape, which is typical for an under-confident +# classifier. In the case of :class:`~sklearn.svm.LinearSVC`, this is caused +# by the margin property of the hinge loss, which focuses on samples that are +# close to the decision boundary (support vectors). Samples that are far +# away from the decision boundary do not impact the hinge loss. It thus makes +# sense that :class:`~sklearn.svm.LinearSVC` does not try to separate samples +# in the high confidence region regions. This leads to flatter calibration +# curves near 0 and 1 and is empirically shown with a variety of datasets +# in Niculescu-Mizil & Caruana [1]_. +# +# Both kinds of calibration (sigmoid and isotonic) can fix this issue and +# yield similar results. +# +# As before, we show the :ref:`brier_score_loss`, :ref:`log_loss`, +# :ref:`precision, recall, F1 score ` and +# :ref:`ROC AUC `. + +scores = defaultdict(list) +for i, (clf, name) in enumerate(clf_list): + clf.fit(X_train, y_train) + y_prob = clf.predict_proba(X_test) + y_pred = clf.predict(X_test) + scores["Classifier"].append(name) + + for metric in [brier_score_loss, log_loss, roc_auc_score]: + score_name = metric.__name__.replace("_", " ").replace("score", "").capitalize() + scores[score_name].append(metric(y_test, y_prob[:, 1])) + + for metric in [precision_score, recall_score, f1_score]: + score_name = metric.__name__.replace("_", " ").replace("score", "").capitalize() + scores[score_name].append(metric(y_test, y_pred)) + + score_df = pd.DataFrame(scores).set_index("Classifier") + score_df.round(decimals=3) + +score_df + +# %% +# As with :class:`~sklearn.naive_bayes.GaussianNB` above, calibration improves +# both :ref:`brier_score_loss` and :ref:`log_loss` but does not alter the +# prediction accuracy measures (precision, recall and F1 score) much. +# +# Summary +# ------- +# +# Parametric sigmoid calibration can deal with situations where the calibration +# curve of the base classifier is sigmoid (e.g., for +# :class:`~sklearn.svm.LinearSVC`) but not where it is transposed-sigmoid +# (e.g., :class:`~sklearn.naive_bayes.GaussianNB`). Non-parametric +# isotonic calibration can deal with both situations but may require more +# data to produce good results. +# +# References +# ---------- +# +# .. [1] `Predicting Good Probabilities with Supervised Learning +# `_, +# A. Niculescu-Mizil & R. Caruana, ICML 2005 diff --git a/examples/calibration/plot_calibration_multiclass.py b/examples/calibration/plot_calibration_multiclass.py index 5aaeedbb079fe..782a59133fcca 100644 --- a/examples/calibration/plot_calibration_multiclass.py +++ b/examples/calibration/plot_calibration_multiclass.py @@ -10,6 +10,7 @@ classifier to the probability vectors predicted by the same classifier after sigmoid calibration on a hold-out validation set. Colors indicate the true class of an instance (red: class 1, green: class 2, blue: class 3). + """ # %% @@ -26,16 +27,18 @@ class of an instance (red: class 1, green: class 2, blue: class 3). # of both the train and valid subsets. This is used when we only want to train # the classifier but not calibrate the predicted probabilities. -# Author: Jan Hendrik Metzen -# License: BSD Style. +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import numpy as np + from sklearn.datasets import make_blobs np.random.seed(0) -X, y = make_blobs(n_samples=2000, n_features=2, centers=3, random_state=42, - cluster_std=5.0) +X, y = make_blobs( + n_samples=2000, n_features=2, centers=3, random_state=42, cluster_std=5.0 +) X_train, y_train = X[:600], y[:600] X_valid, y_valid = X[600:1000], y[600:1000] X_train_valid, y_train_valid = X[:1000], y[:1000] @@ -61,10 +64,11 @@ class of an instance (red: class 1, green: class 2, blue: class 3). # using the valid data subset (400 samples) in a 2-stage process. from sklearn.calibration import CalibratedClassifierCV +from sklearn.frozen import FrozenEstimator clf = RandomForestClassifier(n_estimators=25) clf.fit(X_train, y_train) -cal_clf = CalibratedClassifierCV(clf, method="sigmoid", cv="prefit") +cal_clf = CalibratedClassifierCV(FrozenEstimator(clf), method="sigmoid") cal_clf.fit(X_valid, y_valid) # %% @@ -82,58 +86,96 @@ class of an instance (red: class 1, green: class 2, blue: class 3). cal_clf_probs = cal_clf.predict_proba(X_test) # Plot arrows for i in range(clf_probs.shape[0]): - plt.arrow(clf_probs[i, 0], clf_probs[i, 1], - cal_clf_probs[i, 0] - clf_probs[i, 0], - cal_clf_probs[i, 1] - clf_probs[i, 1], - color=colors[y_test[i]], head_width=1e-2) + plt.arrow( + clf_probs[i, 0], + clf_probs[i, 1], + cal_clf_probs[i, 0] - clf_probs[i, 0], + cal_clf_probs[i, 1] - clf_probs[i, 1], + color=colors[y_test[i]], + head_width=1e-2, + ) # Plot perfect predictions, at each vertex -plt.plot([1.0], [0.0], 'ro', ms=20, label="Class 1") -plt.plot([0.0], [1.0], 'go', ms=20, label="Class 2") -plt.plot([0.0], [0.0], 'bo', ms=20, label="Class 3") +plt.plot([1.0], [0.0], "ro", ms=20, label="Class 1") +plt.plot([0.0], [1.0], "go", ms=20, label="Class 2") +plt.plot([0.0], [0.0], "bo", ms=20, label="Class 3") # Plot boundaries of unit simplex -plt.plot([0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], 'k', label="Simplex") +plt.plot([0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], "k", label="Simplex") # Annotate points 6 points around the simplex, and mid point inside simplex -plt.annotate(r'($\frac{1}{3}$, $\frac{1}{3}$, $\frac{1}{3}$)', - xy=(1.0/3, 1.0/3), xytext=(1.0/3, .23), xycoords='data', - arrowprops=dict(facecolor='black', shrink=0.05), - horizontalalignment='center', verticalalignment='center') -plt.plot([1.0/3], [1.0/3], 'ko', ms=5) -plt.annotate(r'($\frac{1}{2}$, $0$, $\frac{1}{2}$)', - xy=(.5, .0), xytext=(.5, .1), xycoords='data', - arrowprops=dict(facecolor='black', shrink=0.05), - horizontalalignment='center', verticalalignment='center') -plt.annotate(r'($0$, $\frac{1}{2}$, $\frac{1}{2}$)', - xy=(.0, .5), xytext=(.1, .5), xycoords='data', - arrowprops=dict(facecolor='black', shrink=0.05), - horizontalalignment='center', verticalalignment='center') -plt.annotate(r'($\frac{1}{2}$, $\frac{1}{2}$, $0$)', - xy=(.5, .5), xytext=(.6, .6), xycoords='data', - arrowprops=dict(facecolor='black', shrink=0.05), - horizontalalignment='center', verticalalignment='center') -plt.annotate(r'($0$, $0$, $1$)', - xy=(0, 0), xytext=(.1, .1), xycoords='data', - arrowprops=dict(facecolor='black', shrink=0.05), - horizontalalignment='center', verticalalignment='center') -plt.annotate(r'($1$, $0$, $0$)', - xy=(1, 0), xytext=(1, .1), xycoords='data', - arrowprops=dict(facecolor='black', shrink=0.05), - horizontalalignment='center', verticalalignment='center') -plt.annotate(r'($0$, $1$, $0$)', - xy=(0, 1), xytext=(.1, 1), xycoords='data', - arrowprops=dict(facecolor='black', shrink=0.05), - horizontalalignment='center', verticalalignment='center') +plt.annotate( + r"($\frac{1}{3}$, $\frac{1}{3}$, $\frac{1}{3}$)", + xy=(1.0 / 3, 1.0 / 3), + xytext=(1.0 / 3, 0.23), + xycoords="data", + arrowprops=dict(facecolor="black", shrink=0.05), + horizontalalignment="center", + verticalalignment="center", +) +plt.plot([1.0 / 3], [1.0 / 3], "ko", ms=5) +plt.annotate( + r"($\frac{1}{2}$, $0$, $\frac{1}{2}$)", + xy=(0.5, 0.0), + xytext=(0.5, 0.1), + xycoords="data", + arrowprops=dict(facecolor="black", shrink=0.05), + horizontalalignment="center", + verticalalignment="center", +) +plt.annotate( + r"($0$, $\frac{1}{2}$, $\frac{1}{2}$)", + xy=(0.0, 0.5), + xytext=(0.1, 0.5), + xycoords="data", + arrowprops=dict(facecolor="black", shrink=0.05), + horizontalalignment="center", + verticalalignment="center", +) +plt.annotate( + r"($\frac{1}{2}$, $\frac{1}{2}$, $0$)", + xy=(0.5, 0.5), + xytext=(0.6, 0.6), + xycoords="data", + arrowprops=dict(facecolor="black", shrink=0.05), + horizontalalignment="center", + verticalalignment="center", +) +plt.annotate( + r"($0$, $0$, $1$)", + xy=(0, 0), + xytext=(0.1, 0.1), + xycoords="data", + arrowprops=dict(facecolor="black", shrink=0.05), + horizontalalignment="center", + verticalalignment="center", +) +plt.annotate( + r"($1$, $0$, $0$)", + xy=(1, 0), + xytext=(1, 0.1), + xycoords="data", + arrowprops=dict(facecolor="black", shrink=0.05), + horizontalalignment="center", + verticalalignment="center", +) +plt.annotate( + r"($0$, $1$, $0$)", + xy=(0, 1), + xytext=(0.1, 1), + xycoords="data", + arrowprops=dict(facecolor="black", shrink=0.05), + horizontalalignment="center", + verticalalignment="center", +) # Add grid plt.grid(False) for x in [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]: - plt.plot([0, x], [x, 0], 'k', alpha=0.2) - plt.plot([0, 0 + (1-x)/2], [x, x + (1-x)/2], 'k', alpha=0.2) - plt.plot([x, x + (1-x)/2], [0, 0 + (1-x)/2], 'k', alpha=0.2) + plt.plot([0, x], [x, 0], "k", alpha=0.2) + plt.plot([0, 0 + (1 - x) / 2], [x, x + (1 - x) / 2], "k", alpha=0.2) + plt.plot([x, x + (1 - x) / 2], [0, 0 + (1 - x) / 2], "k", alpha=0.2) -plt.title("Change of predicted probabilities on test samples " - "after sigmoid calibration") +plt.title("Change of predicted probabilities on test samples after sigmoid calibration") plt.xlabel("Probability class 1") plt.ylabel("Probability class 2") plt.xlim(-0.05, 1.05) @@ -157,8 +199,8 @@ class of an instance (red: class 1, green: class 2, blue: class 3). # of the arrows point towards the true class, e.g., green arrows (samples where # the true class is 'green') generally point towards the green vertex. This # results in fewer over-confident, 0 predicted probabilities and at the same -# time an increase in the the predicted probabilities of the correct class. -# Thus, the calibrated classifier produces more accurate predicted probablities +# time an increase in the predicted probabilities of the correct class. +# Thus, the calibrated classifier produces more accurate predicted probabilities # that incur a lower :ref:`log loss ` # # We can show this objectively by comparing the :ref:`log loss ` of @@ -170,15 +212,31 @@ class of an instance (red: class 1, green: class 2, blue: class 3). from sklearn.metrics import log_loss -score = log_loss(y_test, clf_probs) -cal_score = log_loss(y_test, cal_clf_probs) +loss = log_loss(y_test, clf_probs) +cal_loss = log_loss(y_test, cal_clf_probs) -print("Log-loss of") -print(f" * uncalibrated classifier: {score:.3f}") -print(f" * calibrated classifier: {cal_score:.3f}") +print("Log-loss of:") +print(f" - uncalibrated classifier: {loss:.3f}") +print(f" - calibrated classifier: {cal_loss:.3f}") # %% -# Finally we generate a grid of possibile uncalibrated probabilities over +# We can also assess calibration with the Brier score for probabilistics predictions +# (lower is better, possible range is [0, 2]): + +from sklearn.metrics import brier_score_loss + +loss = brier_score_loss(y_test, clf_probs) +cal_loss = brier_score_loss(y_test, cal_clf_probs) + +print("Brier score of") +print(f" - uncalibrated classifier: {loss:.3f}") +print(f" - calibrated classifier: {cal_loss:.3f}") + +# %% +# According to the Brier score, the calibrated classifier is not better than +# the original model. +# +# Finally we generate a grid of possible uncalibrated probabilities over # the 2-simplex, compute the corresponding calibrated probabilities and # plot arrows for each. The arrows are colored according the highest # uncalibrated probability. This illustrates the learned calibration map: @@ -193,9 +251,12 @@ class of an instance (red: class 1, green: class 2, blue: class 3). # Use the three class-wise calibrators to compute calibrated probabilities calibrated_classifier = cal_clf.calibrated_classifiers_[0] -prediction = np.vstack([calibrator.predict(this_p) - for calibrator, this_p in - zip(calibrated_classifier.calibrators, p.T)]).T +prediction = np.vstack( + [ + calibrator.predict(this_p) + for calibrator, this_p in zip(calibrated_classifier.calibrators, p.T) + ] +).T # Re-normalize the calibrated predictions to make sure they stay inside the # simplex. This same renormalization step is performed internally by the @@ -204,18 +265,23 @@ class of an instance (red: class 1, green: class 2, blue: class 3). # Plot changes in predicted probabilities induced by the calibrators for i in range(prediction.shape[0]): - plt.arrow(p[i, 0], p[i, 1], - prediction[i, 0] - p[i, 0], prediction[i, 1] - p[i, 1], - head_width=1e-2, color=colors[np.argmax(p[i])]) + plt.arrow( + p[i, 0], + p[i, 1], + prediction[i, 0] - p[i, 0], + prediction[i, 1] - p[i, 1], + head_width=1e-2, + color=colors[np.argmax(p[i])], + ) # Plot the boundaries of the unit simplex -plt.plot([0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], 'k', label="Simplex") +plt.plot([0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], "k", label="Simplex") plt.grid(False) for x in [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]: - plt.plot([0, x], [x, 0], 'k', alpha=0.2) - plt.plot([0, 0 + (1-x)/2], [x, x + (1-x)/2], 'k', alpha=0.2) - plt.plot([x, x + (1-x)/2], [0, 0 + (1-x)/2], 'k', alpha=0.2) + plt.plot([0, x], [x, 0], "k", alpha=0.2) + plt.plot([0, 0 + (1 - x) / 2], [x, x + (1 - x) / 2], "k", alpha=0.2) + plt.plot([x, x + (1 - x) / 2], [0, 0 + (1 - x) / 2], "k", alpha=0.2) plt.title("Learned sigmoid calibration map") plt.xlabel("Probability class 1") @@ -224,3 +290,15 @@ class of an instance (red: class 1, green: class 2, blue: class 3). plt.ylim(-0.05, 1.05) plt.show() + +# %% +# One can observe that, on average, the calibrator is pushing highly confident +# predictions away from the boundaries of the simplex while simultaneously +# moving uncertain predictions towards one of three modes, one for each class. +# We can also observe that the mapping is not symmetric. Furthermore some +# arrows seems to cross class assignment boundaries which is not necessarily +# what one would expect from a calibration map as it means that some predicted +# classes will change after calibration. +# +# All in all, the One-vs-Rest multiclass-calibration strategy implemented in +# `CalibratedClassifierCV` should not be trusted blindly. diff --git a/examples/calibration/plot_compare_calibration.py b/examples/calibration/plot_compare_calibration.py index a8599aecc16af..aa60de1032765 100644 --- a/examples/calibration/plot_compare_calibration.py +++ b/examples/calibration/plot_compare_calibration.py @@ -4,119 +4,279 @@ ======================================== Well calibrated classifiers are probabilistic classifiers for which the output -of the predict_proba method can be directly interpreted as a confidence level. -For instance a well calibrated (binary) classifier should classify the samples -such that among the samples to which it gave a predict_proba value close to -0.8, approx. 80% actually belong to the positive class. - -LogisticRegression returns well calibrated predictions as it directly -optimizes log-loss. In contrast, the other methods return biased probabilities, -with different biases per method: - -* GaussianNaiveBayes tends to push probabilities to 0 or 1 (note the counts in - the histograms). This is mainly because it makes the assumption that features - are conditionally independent given the class, which is not the case in this - dataset which contains 2 redundant features. - -* RandomForestClassifier shows the opposite behavior: the histograms show - peaks at approx. 0.2 and 0.9 probability, while probabilities close to 0 or 1 - are very rare. An explanation for this is given by Niculescu-Mizil and Caruana - [1]_: "Methods such as bagging and random forests that average predictions - from a base set of models can have difficulty making predictions near 0 and 1 - because variance in the underlying base models will bias predictions that - should be near zero or one away from these values. Because predictions are - restricted to the interval [0,1], errors caused by variance tend to be one- - sided near zero and one. For example, if a model should predict p = 0 for a - case, the only way bagging can achieve this is if all bagged trees predict - zero. If we add noise to the trees that bagging is averaging over, this noise - will cause some trees to predict values larger than 0 for this case, thus - moving the average prediction of the bagged ensemble away from 0. We observe - this effect most strongly with random forests because the base-level trees - trained with random forests have relatively high variance due to feature - subsetting." As a result, the calibration curve shows a characteristic - sigmoid shape, indicating that the classifier could trust its "intuition" - more and return probabilities closer to 0 or 1 typically. - -* Support Vector Classification (SVC) shows an even more sigmoid curve as - the RandomForestClassifier, which is typical for maximum-margin methods - (compare Niculescu-Mizil and Caruana [1]_), which focus on hard samples - that are close to the decision boundary (the support vectors). - -.. topic:: References: - - .. [1] Predicting Good Probabilities with Supervised Learning, - A. Niculescu-Mizil & R. Caruana, ICML 2005 +of :term:`predict_proba` can be directly interpreted as a confidence level. +For instance, a well calibrated (binary) classifier should classify the samples +such that for the samples to which it gave a :term:`predict_proba` value close +to 0.8, approximately 80% actually belong to the positive class. + +In this example we will compare the calibration of four different +models: :ref:`Logistic_regression`, :ref:`gaussian_naive_bayes`, +:ref:`Random Forest Classifier ` and :ref:`Linear SVM +`. + """ -print(__doc__) -# Author: Jan Hendrik Metzen -# License: BSD Style. +# %% +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np -np.random.seed(0) +# +# Dataset +# ------- +# +# We will use a synthetic binary classification dataset with 100,000 samples +# and 20 features. Of the 20 features, only 2 are informative, 2 are +# redundant (random combinations of the informative features) and the +# remaining 16 are uninformative (random numbers). +# +# Of the 100,000 samples, 100 will be used for model fitting and the remaining +# for testing. Note that this split is quite unusual: the goal is to obtain +# stable calibration curve estimates for models that are potentially prone to +# overfitting. In practice, one should rather use cross-validation with more +# balanced splits but this would make the code of this example more complicated +# to follow. -import matplotlib.pyplot as plt +from sklearn.datasets import make_classification +from sklearn.model_selection import train_test_split + +X, y = make_classification( + n_samples=100_000, n_features=20, n_informative=2, n_redundant=2, random_state=42 +) + +train_samples = 100 # Samples used for training the models +X_train, X_test, y_train, y_test = train_test_split( + X, + y, + shuffle=False, + test_size=100_000 - train_samples, +) + +# %% +# Calibration curves +# ------------------ +# +# Below, we train each of the four models with the small training dataset, then +# plot calibration curves (also known as reliability diagrams) using +# predicted probabilities of the test dataset. Calibration curves are created +# by binning predicted probabilities, then plotting the mean predicted +# probability in each bin against the observed frequency ('fraction of +# positives'). Below the calibration curve, we plot a histogram showing +# the distribution of the predicted probabilities or more specifically, +# the number of samples in each predicted probability bin. + +import numpy as np -from sklearn import datasets -from sklearn.naive_bayes import GaussianNB -from sklearn.linear_model import LogisticRegression -from sklearn.ensemble import RandomForestClassifier from sklearn.svm import LinearSVC -from sklearn.calibration import calibration_curve -X, y = datasets.make_classification(n_samples=100000, n_features=20, - n_informative=2, n_redundant=2) -train_samples = 100 # Samples used for training the models +class NaivelyCalibratedLinearSVC(LinearSVC): + """LinearSVC with `predict_proba` method that naively scales + `decision_function` output.""" -X_train = X[:train_samples] -X_test = X[train_samples:] -y_train = y[:train_samples] -y_test = y[train_samples:] + def fit(self, X, y): + super().fit(X, y) + df = self.decision_function(X) + self.df_min_ = df.min() + self.df_max_ = df.max() -# Create classifiers -lr = LogisticRegression() + def predict_proba(self, X): + """Min-max scale output of `decision_function` to [0,1].""" + df = self.decision_function(X) + calibrated_df = (df - self.df_min_) / (self.df_max_ - self.df_min_) + proba_pos_class = np.clip(calibrated_df, 0, 1) + proba_neg_class = 1 - proba_pos_class + proba = np.c_[proba_neg_class, proba_pos_class] + return proba + + +# %% + +from sklearn.calibration import CalibrationDisplay +from sklearn.ensemble import RandomForestClassifier +from sklearn.linear_model import LogisticRegressionCV +from sklearn.naive_bayes import GaussianNB + +# Define the classifiers to be compared in the study. +# +# Note that we use a variant of the logistic regression model that can +# automatically tune its regularization parameter. +# +# For a fair comparison, we should run a hyper-parameter search for all the +# classifiers but we don't do it here for the sake of keeping the example code +# concise and fast to execute. +lr = LogisticRegressionCV( + Cs=np.logspace(-6, 6, 101), cv=10, scoring="neg_log_loss", max_iter=1_000 +) gnb = GaussianNB() -svc = LinearSVC(C=1.0) -rfc = RandomForestClassifier() +svc = NaivelyCalibratedLinearSVC(C=1.0) +rfc = RandomForestClassifier(random_state=42) +clf_list = [ + (lr, "Logistic Regression"), + (gnb, "Naive Bayes"), + (svc, "SVC"), + (rfc, "Random forest"), +] -# ############################################################################# -# Plot calibration plots +# %% -plt.figure(figsize=(10, 10)) -ax1 = plt.subplot2grid((3, 1), (0, 0), rowspan=2) -ax2 = plt.subplot2grid((3, 1), (2, 0)) +import matplotlib.pyplot as plt +from matplotlib.gridspec import GridSpec + +fig = plt.figure(figsize=(10, 10)) +gs = GridSpec(4, 2) +colors = plt.get_cmap("Dark2") -ax1.plot([0, 1], [0, 1], "k:", label="Perfectly calibrated") -for clf, name in [(lr, 'Logistic'), - (gnb, 'Naive Bayes'), - (svc, 'Support Vector Classification'), - (rfc, 'Random Forest')]: +ax_calibration_curve = fig.add_subplot(gs[:2, :2]) +calibration_displays = {} +markers = ["^", "v", "s", "o"] +for i, (clf, name) in enumerate(clf_list): clf.fit(X_train, y_train) - if hasattr(clf, "predict_proba"): - prob_pos = clf.predict_proba(X_test)[:, 1] - else: # use decision function - prob_pos = clf.decision_function(X_test) - prob_pos = \ - (prob_pos - prob_pos.min()) / (prob_pos.max() - prob_pos.min()) - fraction_of_positives, mean_predicted_value = \ - calibration_curve(y_test, prob_pos, n_bins=10) - - ax1.plot(mean_predicted_value, fraction_of_positives, "s-", - label="%s" % (name, )) - - ax2.hist(prob_pos, range=(0, 1), bins=10, label=name, - histtype="step", lw=2) - -ax1.set_ylabel("Fraction of positives") -ax1.set_ylim([-0.05, 1.05]) -ax1.legend(loc="lower right") -ax1.set_title('Calibration plots (reliability curve)') - -ax2.set_xlabel("Mean predicted value") -ax2.set_ylabel("Count") -ax2.legend(loc="upper center", ncol=2) + display = CalibrationDisplay.from_estimator( + clf, + X_test, + y_test, + n_bins=10, + name=name, + ax=ax_calibration_curve, + color=colors(i), + marker=markers[i], + ) + calibration_displays[name] = display + +ax_calibration_curve.grid() +ax_calibration_curve.set_title("Calibration plots") + +# Add histogram +grid_positions = [(2, 0), (2, 1), (3, 0), (3, 1)] +for i, (_, name) in enumerate(clf_list): + row, col = grid_positions[i] + ax = fig.add_subplot(gs[row, col]) + + ax.hist( + calibration_displays[name].y_prob, + range=(0, 1), + bins=10, + label=name, + color=colors(i), + ) + ax.set(title=name, xlabel="Mean predicted probability", ylabel="Count") plt.tight_layout() plt.show() + +# %% +# +# Analysis of the results +# ----------------------- +# +# :class:`~sklearn.linear_model.LogisticRegressionCV` returns reasonably well +# calibrated predictions despite the small training set size: its reliability +# curve is the closest to the diagonal among the four models. +# +# Logistic regression is trained by minimizing the log-loss which is a strictly +# proper scoring rule: in the limit of infinite training data, strictly proper +# scoring rules are minimized by the model that predicts the true conditional +# probabilities. That (hypothetical) model would therefore be perfectly +# calibrated. However, using a proper scoring rule as training objective is not +# sufficient to guarantee a well-calibrated model by itself: even with a very +# large training set, logistic regression could still be poorly calibrated, if +# it was too strongly regularized or if the choice and preprocessing of input +# features made this model mis-specified (e.g. if the true decision boundary of +# the dataset is a highly non-linear function of the input features). +# +# In this example the training set was intentionally kept very small. In this +# setting, optimizing the log-loss can still lead to poorly calibrated models +# because of overfitting. To mitigate this, the +# :class:`~sklearn.linear_model.LogisticRegressionCV` class was configured to +# tune the `C` regularization parameter to also minimize the log-loss via inner +# cross-validation so as to find the best compromise for this model in the +# small training set setting. +# +# Because of the finite training set size and the lack of guarantee for +# well-specification, we observe that the calibration curve of the logistic +# regression model is close but not perfectly on the diagonal. The shape of the +# calibration curve of this model can be interpreted as slightly +# under-confident: the predicted probabilities are a bit too close to 0.5 +# compared to the true fraction of positive samples. +# +# The other methods all output less well calibrated probabilities: +# +# * :class:`~sklearn.naive_bayes.GaussianNB` tends to push probabilities to 0 +# or 1 (see histogram) on this particular dataset (over-confidence). This is +# mainly because the naive Bayes equation only provides correct estimate of +# probabilities when the assumption that features are conditionally +# independent holds [2]_. However, features can be correlated and this is the case +# with this dataset, which contains 2 features generated as random linear +# combinations of the informative features. These correlated features are +# effectively being 'counted twice', resulting in pushing the predicted +# probabilities towards 0 and 1 [3]_. Note, however, that changing the seed +# used to generate the dataset can lead to widely varying results for the +# naive Bayes estimator. +# +# * :class:`~sklearn.svm.LinearSVC` is not a natural probabilistic classifier. +# In order to interpret its prediction as such, we naively scaled the output +# of the :term:`decision_function` into [0, 1] by applying min-max scaling in +# the `NaivelyCalibratedLinearSVC` wrapper class defined above. This +# estimator shows a typical sigmoid-shaped calibration curve on this data: +# predictions larger than 0.5 correspond to samples with an even larger +# effective positive class fraction (above the diagonal), while predictions +# below 0.5 corresponds to even lower positive class fractions (below the +# diagonal). This under-confident predictions are typical for maximum-margin +# methods [1]_. +# +# * :class:`~sklearn.ensemble.RandomForestClassifier`'s prediction histogram +# shows peaks at approx. 0.2 and 0.9 probability, while probabilities close to +# 0 or 1 are very rare. An explanation for this is given by [1]_: +# "Methods such as bagging and random forests that average +# predictions from a base set of models can have difficulty making +# predictions near 0 and 1 because variance in the underlying base models +# will bias predictions that should be near zero or one away from these +# values. Because predictions are restricted to the interval [0, 1], errors +# caused by variance tend to be one-sided near zero and one. For example, if +# a model should predict p = 0 for a case, the only way bagging can achieve +# this is if all bagged trees predict zero. If we add noise to the trees that +# bagging is averaging over, this noise will cause some trees to predict +# values larger than 0 for this case, thus moving the average prediction of +# the bagged ensemble away from 0. We observe this effect most strongly with +# random forests because the base-level trees trained with random forests +# have relatively high variance due to feature subsetting." This effect can +# make random forests under-confident. Despite this possible bias, note that +# the trees themselves are fit by minimizing either the Gini or Entropy +# criterion, both of which lead to splits that minimize proper scoring rules: +# the Brier score or the log-loss respectively. See :ref:`the user guide +# ` for more details. This can explain why +# this model shows a good enough calibration curve on this particular example +# dataset. Indeed the Random Forest model is not significantly more +# under-confident than the Logistic Regression model. +# +# Feel free to re-run this example with different random seeds and other +# dataset generation parameters to see how different the calibration plots can +# look. In general, Logistic Regression and Random Forest will tend to be the +# best calibrated classifiers, while SVC will often display the typical +# under-confident miscalibration. The naive Bayes model is also often poorly +# calibrated but the general shape of its calibration curve can vary widely +# depending on the dataset. +# +# Finally, note that for some dataset seeds, all models are poorly calibrated, +# even when tuning the regularization parameter as above. This is bound to +# happen when the training size is too small or when the model is severely +# misspecified. +# +# References +# ---------- +# +# .. [1] `Predicting Good Probabilities with Supervised Learning +# `_, A. +# Niculescu-Mizil & R. Caruana, ICML 2005 +# +# .. [2] `Beyond independence: Conditions for the optimality of the simple +# bayesian classifier +# `_ +# Domingos, P., & Pazzani, M., Proc. 13th Intl. Conf. Machine Learning. +# 1996. +# +# .. [3] `Obtaining calibrated probability estimates from decision trees and +# naive Bayesian classifiers +# `_ +# Zadrozny, Bianca, and Charles Elkan. Icml. Vol. 1. 2001. diff --git a/examples/classification/plot_classification_probability.py b/examples/classification/plot_classification_probability.py index ea4df9e6fb583..7ea706d8c307c 100644 --- a/examples/classification/plot_classification_probability.py +++ b/examples/classification/plot_classification_probability.py @@ -3,96 +3,239 @@ Plot classification probability =============================== -Plot the classification probability for different classifiers. We use a 3 class -dataset, and we classify it with a Support Vector classifier, L1 and L2 -penalized logistic regression with either a One-Vs-Rest or multinomial setting, -and Gaussian process classification. - -Linear SVC is not a probabilistic classifier by default but it has a built-in -calibration option enabled in this example (`probability=True`). - -The logistic regression with One-Vs-Rest is not a multiclass classifier out of -the box. As a result it has more trouble in separating class 2 and 3 than the -other estimators. +This example illustrates the use of +:class:`sklearn.inspection.DecisionBoundaryDisplay` to plot the predicted class +probabilities of various classifiers in a 2D feature space, mostly for didactic +purposes. + +The first three columns shows the predicted probability for varying values of +the two features. Round markers represent the test data that was predicted to +belong to that class. + +In the last column, all three classes are represented on each plot; the class +with the highest predicted probability at each point is plotted. The round +markers show the test data and are colored by their true label. """ -print(__doc__) -# Author: Alexandre Gramfort -# License: BSD 3 clause +# %% +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +import matplotlib as mpl import matplotlib.pyplot as plt import numpy as np +import pandas as pd +from matplotlib import cm -from sklearn.metrics import accuracy_score -from sklearn.linear_model import LogisticRegression -from sklearn.svm import SVC +from sklearn import datasets +from sklearn.ensemble import HistGradientBoostingClassifier from sklearn.gaussian_process import GaussianProcessClassifier from sklearn.gaussian_process.kernels import RBF -from sklearn import datasets - +from sklearn.inspection import DecisionBoundaryDisplay +from sklearn.kernel_approximation import Nystroem +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import accuracy_score, log_loss, roc_auc_score +from sklearn.model_selection import train_test_split +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import ( + KBinsDiscretizer, + PolynomialFeatures, + SplineTransformer, +) + +# %% +# Data: 2D projection of the iris dataset +# --------------------------------------- iris = datasets.load_iris() X = iris.data[:, 0:2] # we only take the first two features for visualization y = iris.target -n_features = X.shape[1] +X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.5, random_state=42 +) + -C = 10 -kernel = 1.0 * RBF([1.0, 1.0]) # for GPC +# %% +# Probabilistic classifiers +# ------------------------- +# +# We will plot the decision boundaries of several classifiers that have a +# `predict_proba` method. This will allow us to visualize the uncertainty of +# the classifier in regions where it is not certain of its prediction. -# Create different classifiers. classifiers = { - 'L1 logistic': LogisticRegression(C=C, penalty='l1', - solver='saga', - multi_class='multinomial', - max_iter=10000), - 'L2 logistic (Multinomial)': LogisticRegression(C=C, penalty='l2', - solver='saga', - multi_class='multinomial', - max_iter=10000), - 'L2 logistic (OvR)': LogisticRegression(C=C, penalty='l2', - solver='saga', - multi_class='ovr', - max_iter=10000), - 'Linear SVC': SVC(kernel='linear', C=C, probability=True, - random_state=0), - 'GPC': GaussianProcessClassifier(kernel) + "Logistic regression\n(C=0.01)": LogisticRegression(C=0.1), + "Logistic regression\n(C=1)": LogisticRegression(C=100), + "Gaussian Process": GaussianProcessClassifier(kernel=1.0 * RBF([1.0, 1.0])), + "Logistic regression\n(RBF features)": make_pipeline( + Nystroem(kernel="rbf", gamma=5e-1, n_components=50, random_state=1), + LogisticRegression(C=10), + ), + "Gradient Boosting": HistGradientBoostingClassifier(), + "Logistic regression\n(binned features)": make_pipeline( + KBinsDiscretizer(n_bins=5, quantile_method="averaged_inverted_cdf"), + PolynomialFeatures(interaction_only=True), + LogisticRegression(C=10), + ), + "Logistic regression\n(spline features)": make_pipeline( + SplineTransformer(n_knots=5), + PolynomialFeatures(interaction_only=True), + LogisticRegression(C=10), + ), } -n_classifiers = len(classifiers) +# %% +# Plotting the decision boundaries +# -------------------------------- +# +# For each classifier, we plot the per-class probabilities on the first three +# columns and the probabilities of the most likely class on the last column. -plt.figure(figsize=(3 * 2, n_classifiers * 2)) -plt.subplots_adjust(bottom=.2, top=.95) - -xx = np.linspace(3, 9, 100) -yy = np.linspace(1, 5, 100).T -xx, yy = np.meshgrid(xx, yy) -Xfull = np.c_[xx.ravel(), yy.ravel()] - -for index, (name, classifier) in enumerate(classifiers.items()): - classifier.fit(X, y) - - y_pred = classifier.predict(X) - accuracy = accuracy_score(y, y_pred) - print("Accuracy (train) for %s: %0.1f%% " % (name, accuracy * 100)) - - # View probabilities: - probas = classifier.predict_proba(Xfull) - n_classes = np.unique(y_pred).size - for k in range(n_classes): - plt.subplot(n_classifiers, n_classes, index * n_classes + k + 1) - plt.title("Class %d" % k) - if k == 0: - plt.ylabel(name) - imshow_handle = plt.imshow(probas[:, k].reshape((100, 100)), - extent=(3, 9, 1, 5), origin='lower') - plt.xticks(()) - plt.yticks(()) - idx = (y_pred == k) - if idx.any(): - plt.scatter(X[idx, 0], X[idx, 1], marker='o', c='w', edgecolor='k') - -ax = plt.axes([0.15, 0.04, 0.7, 0.05]) +n_classifiers = len(classifiers) +scatter_kwargs = { + "s": 25, + "marker": "o", + "linewidths": 0.8, + "edgecolor": "k", + "alpha": 0.7, +} +y_unique = np.unique(y) + +# Ensure legend not cut off +mpl.rcParams["savefig.bbox"] = "tight" +fig, axes = plt.subplots( + nrows=n_classifiers, + ncols=len(iris.target_names) + 1, + figsize=(4 * 2.2, n_classifiers * 2.2), +) +evaluation_results = [] +levels = 100 +for classifier_idx, (name, classifier) in enumerate(classifiers.items()): + y_pred = classifier.fit(X_train, y_train).predict(X_test) + y_pred_proba = classifier.predict_proba(X_test) + accuracy_test = accuracy_score(y_test, y_pred) + roc_auc_test = roc_auc_score(y_test, y_pred_proba, multi_class="ovr") + log_loss_test = log_loss(y_test, y_pred_proba) + evaluation_results.append( + { + "name": name.replace("\n", " "), + "accuracy": accuracy_test, + "roc_auc": roc_auc_test, + "log_loss": log_loss_test, + } + ) + for label in y_unique: + # plot the probability estimate provided by the classifier + disp = DecisionBoundaryDisplay.from_estimator( + classifier, + X_train, + response_method="predict_proba", + class_of_interest=label, + ax=axes[classifier_idx, label], + vmin=0, + vmax=1, + cmap="Blues", + levels=levels, + ) + axes[classifier_idx, label].set_title(f"Class {label}") + # plot data predicted to belong to given class + mask_y_pred = y_pred == label + axes[classifier_idx, label].scatter( + X_test[mask_y_pred, 0], X_test[mask_y_pred, 1], c="w", **scatter_kwargs + ) + + axes[classifier_idx, label].set(xticks=(), yticks=()) + # add column that shows all classes by plotting class with max 'predict_proba' + max_class_disp = DecisionBoundaryDisplay.from_estimator( + classifier, + X_train, + response_method="predict_proba", + class_of_interest=None, + ax=axes[classifier_idx, len(y_unique)], + vmin=0, + vmax=1, + levels=levels, + ) + for label in y_unique: + mask_label = y_test == label + axes[classifier_idx, 3].scatter( + X_test[mask_label, 0], + X_test[mask_label, 1], + c=max_class_disp.multiclass_colors_[[label], :], + **scatter_kwargs, + ) + + axes[classifier_idx, 3].set(xticks=(), yticks=()) + axes[classifier_idx, 3].set_title("Max class") + axes[classifier_idx, 0].set_ylabel(name) + +# colorbar for single class plots +ax_single = fig.add_axes([0.15, 0.01, 0.5, 0.02]) plt.title("Probability") -plt.colorbar(imshow_handle, cax=ax, orientation='horizontal') - -plt.show() +_ = plt.colorbar( + cm.ScalarMappable(norm=None, cmap=disp.surface_.cmap), + cax=ax_single, + orientation="horizontal", +) + +# colorbars for max probability class column +max_class_cmaps = [s.cmap for s in max_class_disp.surface_] + +for label in y_unique: + ax_max = fig.add_axes([0.73, (0.06 - (label * 0.04)), 0.16, 0.015]) + plt.title(f"Probability class {label}", fontsize=10) + _ = plt.colorbar( + cm.ScalarMappable(norm=None, cmap=max_class_cmaps[label]), + cax=ax_max, + orientation="horizontal", + ) + if label in (0, 1): + ax_max.set(xticks=(), yticks=()) + + +# %% +# Quantitative evaluation +# ----------------------- +pd.DataFrame(evaluation_results).round(2) + + +# %% +# Analysis +# -------- +# +# The two logistic regression models fitted on the original features display +# linear decision boundaries as expected. For this particular problem, this +# does not seem to be detrimental as both models are competitive with the +# non-linear models when quantitatively evaluated on the test set. We can +# observe that the amount of regularization influences the model confidence: +# lighter colors for the strongly regularized model with a lower value of `C`. +# Regularization also impacts the orientation of decision boundary leading to +# slightly different ROC AUC. +# +# The log-loss on the other hand evaluates both sharpness and calibration and +# as a result strongly favors the weakly regularized logistic-regression model, +# probably because the strongly regularized model is under-confident. This +# could be confirmed by looking at the calibration curve using +# :class:`sklearn.calibration.CalibrationDisplay`. +# +# The logistic regression model with RBF features has a "blobby" decision +# boundary that is non-linear in the original feature space and is quite +# similar to the decision boundary of the Gaussian process classifier which is +# configured to use an RBF kernel. +# +# The logistic regression model fitted on binned features with interactions has +# a decision boundary that is non-linear in the original feature space and is +# quite similar to the decision boundary of the gradient boosting classifier: +# both models favor axis-aligned decisions when extrapolating to unseen region +# of the feature space. +# +# The logistic regression model fitted on spline features with interactions +# has a similar axis-aligned extrapolation behavior but a smoother decision +# boundary in the dense region of the feature space than the two previous +# models. +# +# To conclude, it is interesting to observe that feature engineering for +# logistic regression models can be used to mimic some of the inductive bias of +# various non-linear models. However, for this particular dataset, using the +# raw features is enough to train a competitive model. This would not +# necessarily the case for other datasets. diff --git a/examples/classification/plot_classifier_comparison.py b/examples/classification/plot_classifier_comparison.py index 83019e821dae5..7028eaa70e029 100644 --- a/examples/classification/plot_classifier_comparison.py +++ b/examples/classification/plot_classifier_comparison.py @@ -1,12 +1,9 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - """ ===================== Classifier comparison ===================== -A comparison of a several classifiers in scikit-learn on synthetic datasets. +A comparison of several classifiers in scikit-learn on synthetic datasets. The point of this example is to illustrate the nature of decision boundaries of different classifiers. This should be taken with a grain of salt, as the intuition conveyed by @@ -19,59 +16,71 @@ The plots show training points in solid colors and testing points semi-transparent. The lower right shows the classification accuracy on the test set. -""" -print(__doc__) +""" -# Code source: GaÃĢl Varoquaux -# Andreas MÃŧller -# Modified for documentation by Jaques Grobler -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt +import numpy as np from matplotlib.colors import ListedColormap + +from sklearn.datasets import make_circles, make_classification, make_moons +from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis +from sklearn.ensemble import AdaBoostClassifier, RandomForestClassifier +from sklearn.gaussian_process import GaussianProcessClassifier +from sklearn.gaussian_process.kernels import RBF +from sklearn.inspection import DecisionBoundaryDisplay from sklearn.model_selection import train_test_split -from sklearn.preprocessing import StandardScaler -from sklearn.datasets import make_moons, make_circles, make_classification -from sklearn.neural_network import MLPClassifier +from sklearn.naive_bayes import GaussianNB from sklearn.neighbors import KNeighborsClassifier +from sklearn.neural_network import MLPClassifier +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import StandardScaler from sklearn.svm import SVC -from sklearn.gaussian_process import GaussianProcessClassifier -from sklearn.gaussian_process.kernels import RBF from sklearn.tree import DecisionTreeClassifier -from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier -from sklearn.naive_bayes import GaussianNB -from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis -h = .02 # step size in the mesh - -names = ["Nearest Neighbors", "Linear SVM", "RBF SVM", "Gaussian Process", - "Decision Tree", "Random Forest", "Neural Net", "AdaBoost", - "Naive Bayes", "QDA"] +names = [ + "Nearest Neighbors", + "Linear SVM", + "RBF SVM", + "Gaussian Process", + "Decision Tree", + "Random Forest", + "Neural Net", + "AdaBoost", + "Naive Bayes", + "QDA", +] classifiers = [ KNeighborsClassifier(3), - SVC(kernel="linear", C=0.025), - SVC(gamma=2, C=1), - GaussianProcessClassifier(1.0 * RBF(1.0)), - DecisionTreeClassifier(max_depth=5), - RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1), - MLPClassifier(alpha=1, max_iter=1000), - AdaBoostClassifier(), + SVC(kernel="linear", C=0.025, random_state=42), + SVC(gamma=2, C=1, random_state=42), + GaussianProcessClassifier(1.0 * RBF(1.0), random_state=42), + DecisionTreeClassifier(max_depth=5, random_state=42), + RandomForestClassifier( + max_depth=5, n_estimators=10, max_features=1, random_state=42 + ), + MLPClassifier(alpha=1, max_iter=1000, random_state=42), + AdaBoostClassifier(random_state=42), GaussianNB(), - QuadraticDiscriminantAnalysis()] + QuadraticDiscriminantAnalysis(), +] -X, y = make_classification(n_features=2, n_redundant=0, n_informative=2, - random_state=1, n_clusters_per_class=1) +X, y = make_classification( + n_features=2, n_redundant=0, n_informative=2, random_state=1, n_clusters_per_class=1 +) rng = np.random.RandomState(2) X += 2 * rng.uniform(size=X.shape) linearly_separable = (X, y) -datasets = [make_moons(noise=0.3, random_state=0), - make_circles(noise=0.2, factor=0.5, random_state=1), - linearly_separable - ] +datasets = [ + make_moons(noise=0.3, random_state=0), + make_circles(noise=0.2, factor=0.5, random_state=1), + linearly_separable, +] figure = plt.figure(figsize=(27, 9)) i = 1 @@ -79,29 +88,27 @@ for ds_cnt, ds in enumerate(datasets): # preprocess dataset, split into training and test part X, y = ds - X = StandardScaler().fit_transform(X) - X_train, X_test, y_train, y_test = \ - train_test_split(X, y, test_size=.4, random_state=42) + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.4, random_state=42 + ) - x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5 - y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5 - xx, yy = np.meshgrid(np.arange(x_min, x_max, h), - np.arange(y_min, y_max, h)) + x_min, x_max = X[:, 0].min() - 0.5, X[:, 0].max() + 0.5 + y_min, y_max = X[:, 1].min() - 0.5, X[:, 1].max() + 0.5 # just plot the dataset first cm = plt.cm.RdBu - cm_bright = ListedColormap(['#FF0000', '#0000FF']) + cm_bright = ListedColormap(["#FF0000", "#0000FF"]) ax = plt.subplot(len(datasets), len(classifiers) + 1, i) if ds_cnt == 0: ax.set_title("Input data") # Plot the training points - ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright, - edgecolors='k') + ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright, edgecolors="k") # Plot the testing points - ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright, alpha=0.6, - edgecolors='k') - ax.set_xlim(xx.min(), xx.max()) - ax.set_ylim(yy.min(), yy.max()) + ax.scatter( + X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright, alpha=0.6, edgecolors="k" + ) + ax.set_xlim(x_min, x_max) + ax.set_ylim(y_min, y_max) ax.set_xticks(()) ax.set_yticks(()) i += 1 @@ -109,35 +116,41 @@ # iterate over classifiers for name, clf in zip(names, classifiers): ax = plt.subplot(len(datasets), len(classifiers) + 1, i) + + clf = make_pipeline(StandardScaler(), clf) clf.fit(X_train, y_train) score = clf.score(X_test, y_test) - - # Plot the decision boundary. For that, we will assign a color to each - # point in the mesh [x_min, x_max]x[y_min, y_max]. - if hasattr(clf, "decision_function"): - Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()]) - else: - Z = clf.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:, 1] - - # Put the result into a color plot - Z = Z.reshape(xx.shape) - ax.contourf(xx, yy, Z, cmap=cm, alpha=.8) + DecisionBoundaryDisplay.from_estimator( + clf, X, cmap=cm, alpha=0.8, ax=ax, eps=0.5 + ) # Plot the training points - ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright, - edgecolors='k') + ax.scatter( + X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright, edgecolors="k" + ) # Plot the testing points - ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright, - edgecolors='k', alpha=0.6) - - ax.set_xlim(xx.min(), xx.max()) - ax.set_ylim(yy.min(), yy.max()) + ax.scatter( + X_test[:, 0], + X_test[:, 1], + c=y_test, + cmap=cm_bright, + edgecolors="k", + alpha=0.6, + ) + + ax.set_xlim(x_min, x_max) + ax.set_ylim(y_min, y_max) ax.set_xticks(()) ax.set_yticks(()) if ds_cnt == 0: ax.set_title(name) - ax.text(xx.max() - .3, yy.min() + .3, ('%.2f' % score).lstrip('0'), - size=15, horizontalalignment='right') + ax.text( + x_max - 0.3, + y_min + 0.3, + ("%.2f" % score).lstrip("0"), + size=15, + horizontalalignment="right", + ) i += 1 plt.tight_layout() diff --git a/examples/classification/plot_digits_classification.py b/examples/classification/plot_digits_classification.py index 8cb61df2c9736..58b514161d570 100644 --- a/examples/classification/plot_digits_classification.py +++ b/examples/classification/plot_digits_classification.py @@ -5,18 +5,17 @@ This example shows how scikit-learn can be used to recognize images of hand-written digits, from 0-9. -""" -print(__doc__) +""" -# Author: Gael Varoquaux -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause # Standard scientific Python imports import matplotlib.pyplot as plt # Import datasets, classifiers and performance metrics -from sklearn import datasets, svm, metrics +from sklearn import datasets, metrics, svm from sklearn.model_selection import train_test_split ############################################################################### @@ -38,8 +37,8 @@ _, axes = plt.subplots(nrows=1, ncols=4, figsize=(10, 3)) for ax, image, label in zip(axes, digits.images, digits.target): ax.set_axis_off() - ax.imshow(image, cmap=plt.cm.gray_r, interpolation='nearest') - ax.set_title('Training: %i' % label) + ax.imshow(image, cmap=plt.cm.gray_r, interpolation="nearest") + ax.set_title("Training: %i" % label) ############################################################################### # Classification @@ -65,7 +64,8 @@ # Split data into 50% train and 50% test subsets X_train, X_test, y_train, y_test = train_test_split( - data, digits.target, test_size=0.5, shuffle=False) + data, digits.target, test_size=0.5, shuffle=False +) # Learn the digits on the train subset clf.fit(X_train, y_train) @@ -81,15 +81,17 @@ for ax, image, prediction in zip(axes, X_test, predicted): ax.set_axis_off() image = image.reshape(8, 8) - ax.imshow(image, cmap=plt.cm.gray_r, interpolation='nearest') - ax.set_title(f'Prediction: {prediction}') + ax.imshow(image, cmap=plt.cm.gray_r, interpolation="nearest") + ax.set_title(f"Prediction: {prediction}") ############################################################################### # :func:`~sklearn.metrics.classification_report` builds a text report showing # the main classification metrics. -print(f"Classification report for classifier {clf}:\n" - f"{metrics.classification_report(y_test, predicted)}\n") +print( + f"Classification report for classifier {clf}:\n" + f"{metrics.classification_report(y_test, predicted)}\n" +) ############################################################################### # We can also plot a :ref:`confusion matrix ` of the @@ -100,3 +102,27 @@ print(f"Confusion matrix:\n{disp.confusion_matrix}") plt.show() + +############################################################################### +# If the results from evaluating a classifier are stored in the form of a +# :ref:`confusion matrix ` and not in terms of `y_true` and +# `y_pred`, one can still build a :func:`~sklearn.metrics.classification_report` +# as follows: + + +# The ground truth and predicted lists +y_true = [] +y_pred = [] +cm = disp.confusion_matrix + +# For each cell in the confusion matrix, add the corresponding ground truths +# and predictions to the lists +for gt in range(len(cm)): + for pred in range(len(cm)): + y_true += [gt] * cm[gt][pred] + y_pred += [pred] * cm[gt][pred] + +print( + "Classification report rebuilt from confusion matrix:\n" + f"{metrics.classification_report(y_true, y_pred)}\n" +) diff --git a/examples/classification/plot_lda.py b/examples/classification/plot_lda.py index ad16e7b0d2efa..f85f3fc6043f7 100644 --- a/examples/classification/plot_lda.py +++ b/examples/classification/plot_lda.py @@ -3,16 +3,20 @@ Normal, Ledoit-Wolf and OAS Linear Discriminant Analysis for classification =========================================================================== -This example illustrates how the Ledoit-Wolf and Oracle Shrinkage -Approximating (OAS) estimators of covariance can improve classification. +This example illustrates how the Ledoit-Wolf and Oracle Approximating +Shrinkage (OAS) estimators of covariance can improve classification. + """ -import numpy as np + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt +import numpy as np +from sklearn.covariance import OAS from sklearn.datasets import make_blobs from sklearn.discriminant_analysis import LinearDiscriminantAnalysis -from sklearn.covariance import OAS - n_train = 20 # samples for training n_test = 200 # samples for testing @@ -45,13 +49,12 @@ def generate_data(n_samples, n_features): for _ in range(n_averages): X, y = generate_data(n_train, n_features) - clf1 = LinearDiscriminantAnalysis(solver='lsqr', - shrinkage='auto').fit(X, y) - clf2 = LinearDiscriminantAnalysis(solver='lsqr', - shrinkage=None).fit(X, y) + clf1 = LinearDiscriminantAnalysis(solver="lsqr", shrinkage=None).fit(X, y) + clf2 = LinearDiscriminantAnalysis(solver="lsqr", shrinkage="auto").fit(X, y) oa = OAS(store_precision=False, assume_centered=False) - clf3 = LinearDiscriminantAnalysis(solver='lsqr', - covariance_estimator=oa).fit(X, y) + clf3 = LinearDiscriminantAnalysis(solver="lsqr", covariance_estimator=oa).fit( + X, y + ) X, y = generate_data(n_test, n_features) score_clf1 += clf1.score(X, y) @@ -64,18 +67,41 @@ def generate_data(n_samples, n_features): features_samples_ratio = np.array(n_features_range) / n_train -plt.plot(features_samples_ratio, acc_clf1, linewidth=2, - label="Linear Discriminant Analysis with Ledoit Wolf", color='navy') -plt.plot(features_samples_ratio, acc_clf2, linewidth=2, - label="Linear Discriminant Analysis", color='gold') -plt.plot(features_samples_ratio, acc_clf3, linewidth=2, - label="Linear Discriminant Analysis with OAS", color='red') - -plt.xlabel('n_features / n_samples') -plt.ylabel('Classification accuracy') - -plt.legend(loc=3, prop={'size': 12}) -plt.suptitle('Linear Discriminant Analysis vs. ' + '\n' - + 'Shrinkage Linear Discriminant Analysis vs. ' + '\n' - + 'OAS Linear Discriminant Analysis (1 discriminative feature)') +plt.plot( + features_samples_ratio, + acc_clf1, + linewidth=2, + label="LDA", + color="gold", + linestyle="solid", +) +plt.plot( + features_samples_ratio, + acc_clf2, + linewidth=2, + label="LDA with Ledoit Wolf", + color="navy", + linestyle="dashed", +) +plt.plot( + features_samples_ratio, + acc_clf3, + linewidth=2, + label="LDA with OAS", + color="red", + linestyle="dotted", +) + +plt.xlabel("n_features / n_samples") +plt.ylabel("Classification accuracy") + +plt.legend(loc="lower left") +plt.ylim((0.65, 1.0)) +plt.suptitle( + "LDA (Linear Discriminant Analysis) vs." + "\n" + "LDA with Ledoit Wolf vs." + "\n" + "LDA with OAS (1 discriminative feature)" +) plt.show() diff --git a/examples/classification/plot_lda_qda.py b/examples/classification/plot_lda_qda.py index d02adb03e6028..599659fdac2dc 100644 --- a/examples/classification/plot_lda_qda.py +++ b/examples/classification/plot_lda_qda.py @@ -3,149 +3,226 @@ Linear and Quadratic Discriminant Analysis with covariance ellipsoid ==================================================================== -This example plots the covariance ellipsoids of each class and -decision boundary learned by LDA and QDA. The ellipsoids display -the double standard deviation for each class. With LDA, the -standard deviation is the same for all the classes, while each -class has its own standard deviation with QDA. +This example plots the covariance ellipsoids of each class and the decision boundary +learned by :class:`~sklearn.discriminant_analysis.LinearDiscriminantAnalysis` (LDA) and +:class:`~sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis` (QDA). The +ellipsoids display the double standard deviation for each class. With LDA, the standard +deviation is the same for all the classes, while each class has its own standard +deviation with QDA. """ -print(__doc__) -from scipy import linalg +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Data generation +# --------------- +# +# First, we define a function to generate synthetic data. It creates two blobs centered +# at `(0, 0)` and `(1, 1)`. Each blob is assigned a specific class. The dispersion of +# the blob is controlled by the parameters `cov_class_1` and `cov_class_2`, that are the +# covariance matrices used when generating the samples from the Gaussian distributions. import numpy as np -import matplotlib.pyplot as plt -import matplotlib as mpl -from matplotlib import colors -from sklearn.discriminant_analysis import LinearDiscriminantAnalysis -from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis - -# ############################################################################# -# Colormap -cmap = colors.LinearSegmentedColormap( - 'red_blue_classes', - {'red': [(0, 1, 1), (1, 0.7, 0.7)], - 'green': [(0, 0.7, 0.7), (1, 0.7, 0.7)], - 'blue': [(0, 0.7, 0.7), (1, 1, 1)]}) -plt.cm.register_cmap(cmap=cmap) - - -# ############################################################################# -# Generate datasets -def dataset_fixed_cov(): - '''Generate 2 Gaussians samples with the same covariance matrix''' - n, dim = 300, 2 - np.random.seed(0) - C = np.array([[0., -0.23], [0.83, .23]]) - X = np.r_[np.dot(np.random.randn(n, dim), C), - np.dot(np.random.randn(n, dim), C) + np.array([1, 1])] - y = np.hstack((np.zeros(n), np.ones(n))) + +def make_data(n_samples, n_features, cov_class_1, cov_class_2, seed=0): + rng = np.random.RandomState(seed) + X = np.concatenate( + [ + rng.randn(n_samples, n_features) @ cov_class_1, + rng.randn(n_samples, n_features) @ cov_class_2 + np.array([1, 1]), + ] + ) + y = np.concatenate([np.zeros(n_samples), np.ones(n_samples)]) return X, y -def dataset_cov(): - '''Generate 2 Gaussians samples with different covariance matrices''' - n, dim = 300, 2 - np.random.seed(0) - C = np.array([[0., -1.], [2.5, .7]]) * 2. - X = np.r_[np.dot(np.random.randn(n, dim), C), - np.dot(np.random.randn(n, dim), C.T) + np.array([1, 4])] - y = np.hstack((np.zeros(n), np.ones(n))) - return X, y +# %% +# We generate three datasets. In the first dataset, the two classes share the same +# covariance matrix, and this covariance matrix has the specificity of being spherical +# (isotropic). The second dataset is similar to the first one but does not enforce the +# covariance to be spherical. Finally, the third dataset has a non-spherical covariance +# matrix for each class. +covariance = np.array([[1, 0], [0, 1]]) +X_isotropic_covariance, y_isotropic_covariance = make_data( + n_samples=1_000, + n_features=2, + cov_class_1=covariance, + cov_class_2=covariance, + seed=0, +) +covariance = np.array([[0.0, -0.23], [0.83, 0.23]]) +X_shared_covariance, y_shared_covariance = make_data( + n_samples=300, + n_features=2, + cov_class_1=covariance, + cov_class_2=covariance, + seed=0, +) +cov_class_1 = np.array([[0.0, -1.0], [2.5, 0.7]]) * 2.0 +cov_class_2 = cov_class_1.T +X_different_covariance, y_different_covariance = make_data( + n_samples=300, + n_features=2, + cov_class_1=cov_class_1, + cov_class_2=cov_class_2, + seed=0, +) + + +# %% +# Plotting Functions +# ------------------ +# +# The code below is used to plot several pieces of information from the estimators used, +# i.e., :class:`~sklearn.discriminant_analysis.LinearDiscriminantAnalysis` (LDA) and +# :class:`~sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis` (QDA). The +# displayed information includes: +# +# - the decision boundary based on the probability estimate of the estimator; +# - a scatter plot with circles representing the well-classified samples; +# - a scatter plot with crosses representing the misclassified samples; +# - the mean of each class, estimated by the estimator, marked with a star; +# - the estimated covariance represented by an ellipse at 2 standard deviations from the +# mean. +import matplotlib as mpl +from matplotlib import colors + +from sklearn.inspection import DecisionBoundaryDisplay -# ############################################################################# -# Plot functions -def plot_data(lda, X, y, y_pred, fig_index): - splot = plt.subplot(2, 2, fig_index) - if fig_index == 1: - plt.title('Linear Discriminant Analysis') - plt.ylabel('Data with\n fixed covariance') - elif fig_index == 2: - plt.title('Quadratic Discriminant Analysis') - elif fig_index == 3: - plt.ylabel('Data with\n varying covariances') - - tp = (y == y_pred) # True Positive - tp0, tp1 = tp[y == 0], tp[y == 1] - X0, X1 = X[y == 0], X[y == 1] - X0_tp, X0_fp = X0[tp0], X0[~tp0] - X1_tp, X1_fp = X1[tp1], X1[~tp1] - - # class 0: dots - plt.scatter(X0_tp[:, 0], X0_tp[:, 1], marker='.', color='red') - plt.scatter(X0_fp[:, 0], X0_fp[:, 1], marker='x', - s=20, color='#990000') # dark red - - # class 1: dots - plt.scatter(X1_tp[:, 0], X1_tp[:, 1], marker='.', color='blue') - plt.scatter(X1_fp[:, 0], X1_fp[:, 1], marker='x', - s=20, color='#000099') # dark blue - - # class 0 and 1 : areas - nx, ny = 200, 100 - x_min, x_max = plt.xlim() - y_min, y_max = plt.ylim() - xx, yy = np.meshgrid(np.linspace(x_min, x_max, nx), - np.linspace(y_min, y_max, ny)) - Z = lda.predict_proba(np.c_[xx.ravel(), yy.ravel()]) - Z = Z[:, 1].reshape(xx.shape) - plt.pcolormesh(xx, yy, Z, cmap='red_blue_classes', - norm=colors.Normalize(0., 1.), zorder=0) - plt.contour(xx, yy, Z, [0.5], linewidths=2., colors='white') - - # means - plt.plot(lda.means_[0][0], lda.means_[0][1], - '*', color='yellow', markersize=15, markeredgecolor='grey') - plt.plot(lda.means_[1][0], lda.means_[1][1], - '*', color='yellow', markersize=15, markeredgecolor='grey') - - return splot - - -def plot_ellipse(splot, mean, cov, color): - v, w = linalg.eigh(cov) - u = w[0] / linalg.norm(w[0]) +def plot_ellipse(mean, cov, color, ax): + v, w = np.linalg.eigh(cov) + u = w[0] / np.linalg.norm(w[0]) angle = np.arctan(u[1] / u[0]) angle = 180 * angle / np.pi # convert to degrees # filled Gaussian at 2 standard deviation - ell = mpl.patches.Ellipse(mean, 2 * v[0] ** 0.5, 2 * v[1] ** 0.5, - 180 + angle, facecolor=color, - edgecolor='black', linewidth=2) - ell.set_clip_box(splot.bbox) - ell.set_alpha(0.2) - splot.add_artist(ell) - splot.set_xticks(()) - splot.set_yticks(()) - - -def plot_lda_cov(lda, splot): - plot_ellipse(splot, lda.means_[0], lda.covariance_, 'red') - plot_ellipse(splot, lda.means_[1], lda.covariance_, 'blue') - - -def plot_qda_cov(qda, splot): - plot_ellipse(splot, qda.means_[0], qda.covariance_[0], 'red') - plot_ellipse(splot, qda.means_[1], qda.covariance_[1], 'blue') - - -plt.figure(figsize=(10, 8), facecolor='white') -plt.suptitle('Linear Discriminant Analysis vs Quadratic Discriminant Analysis', - y=0.98, fontsize=15) -for i, (X, y) in enumerate([dataset_fixed_cov(), dataset_cov()]): - # Linear Discriminant Analysis - lda = LinearDiscriminantAnalysis(solver="svd", store_covariance=True) - y_pred = lda.fit(X, y).predict(X) - splot = plot_data(lda, X, y, y_pred, fig_index=2 * i + 1) - plot_lda_cov(lda, splot) - plt.axis('tight') - - # Quadratic Discriminant Analysis - qda = QuadraticDiscriminantAnalysis(store_covariance=True) - y_pred = qda.fit(X, y).predict(X) - splot = plot_data(qda, X, y, y_pred, fig_index=2 * i + 2) - plot_qda_cov(qda, splot) - plt.axis('tight') -plt.tight_layout() -plt.subplots_adjust(top=0.92) + ell = mpl.patches.Ellipse( + mean, + 2 * v[0] ** 0.5, + 2 * v[1] ** 0.5, + angle=180 + angle, + facecolor=color, + edgecolor="black", + linewidth=2, + ) + ell.set_clip_box(ax.bbox) + ell.set_alpha(0.4) + ax.add_artist(ell) + + +def plot_result(estimator, X, y, ax): + cmap = colors.ListedColormap(["tab:red", "tab:blue"]) + DecisionBoundaryDisplay.from_estimator( + estimator, + X, + response_method="predict_proba", + plot_method="pcolormesh", + ax=ax, + cmap="RdBu", + alpha=0.3, + ) + DecisionBoundaryDisplay.from_estimator( + estimator, + X, + response_method="predict_proba", + plot_method="contour", + ax=ax, + alpha=1.0, + levels=[0.5], + ) + y_pred = estimator.predict(X) + X_right, y_right = X[y == y_pred], y[y == y_pred] + X_wrong, y_wrong = X[y != y_pred], y[y != y_pred] + ax.scatter(X_right[:, 0], X_right[:, 1], c=y_right, s=20, cmap=cmap, alpha=0.5) + ax.scatter( + X_wrong[:, 0], + X_wrong[:, 1], + c=y_wrong, + s=30, + cmap=cmap, + alpha=0.9, + marker="x", + ) + ax.scatter( + estimator.means_[:, 0], + estimator.means_[:, 1], + c="yellow", + s=200, + marker="*", + edgecolor="black", + ) + + if isinstance(estimator, LinearDiscriminantAnalysis): + covariance = [estimator.covariance_] * 2 + else: + covariance = estimator.covariance_ + plot_ellipse(estimator.means_[0], covariance[0], "tab:red", ax) + plot_ellipse(estimator.means_[1], covariance[1], "tab:blue", ax) + + ax.set_box_aspect(1) + ax.spines["top"].set_visible(False) + ax.spines["bottom"].set_visible(False) + ax.spines["left"].set_visible(False) + ax.spines["right"].set_visible(False) + ax.set(xticks=[], yticks=[]) + + +# %% +# Comparison of LDA and QDA +# ------------------------- +# +# We compare the two estimators LDA and QDA on all three datasets. +import matplotlib.pyplot as plt + +from sklearn.discriminant_analysis import ( + LinearDiscriminantAnalysis, + QuadraticDiscriminantAnalysis, +) + +fig, axs = plt.subplots(nrows=3, ncols=2, sharex="row", sharey="row", figsize=(8, 12)) + +lda = LinearDiscriminantAnalysis(solver="svd", store_covariance=True) +qda = QuadraticDiscriminantAnalysis(store_covariance=True) + +for ax_row, X, y in zip( + axs, + (X_isotropic_covariance, X_shared_covariance, X_different_covariance), + (y_isotropic_covariance, y_shared_covariance, y_different_covariance), +): + lda.fit(X, y) + plot_result(lda, X, y, ax_row[0]) + qda.fit(X, y) + plot_result(qda, X, y, ax_row[1]) + +axs[0, 0].set_title("Linear Discriminant Analysis") +axs[0, 0].set_ylabel("Data with fixed and spherical covariance") +axs[1, 0].set_ylabel("Data with fixed covariance") +axs[0, 1].set_title("Quadratic Discriminant Analysis") +axs[2, 0].set_ylabel("Data with varying covariances") +fig.suptitle( + "Linear Discriminant Analysis vs Quadratic Discriminant Analysis", + y=0.94, + fontsize=15, +) plt.show() + +# %% +# The first important thing to notice is that LDA and QDA are equivalent for the +# first and second datasets. Indeed, the major difference is that LDA assumes +# that the covariance matrix of each class is equal, while QDA estimates a +# covariance matrix per class. Since in these cases the data generative process +# has the same covariance matrix for both classes, QDA estimates two covariance +# matrices that are (almost) equal and therefore equivalent to the covariance +# matrix estimated by LDA. +# +# In the first dataset the covariance matrix used to generate the dataset is +# spherical, which results in a discriminant boundary that aligns with the +# perpendicular bisector between the two means. This is no longer the case for +# the second dataset. The discriminant boundary only passes through the middle +# of the two means. +# +# Finally, in the third dataset, we observe the real difference between LDA and +# QDA. QDA fits two covariance matrices and provides a non-linear discriminant +# boundary, whereas LDA underfits since it assumes that both classes share a +# single covariance matrix. diff --git a/examples/cluster/plot_adjusted_for_chance_measures.py b/examples/cluster/plot_adjusted_for_chance_measures.py index f9d31dbd4a6ac..088a42771ed95 100644 --- a/examples/cluster/plot_adjusted_for_chance_measures.py +++ b/examples/cluster/plot_adjusted_for_chance_measures.py @@ -2,124 +2,228 @@ ========================================================== Adjustment for chance in clustering performance evaluation ========================================================== +This notebook explores the impact of uniformly-distributed random labeling on +the behavior of some clustering evaluation metrics. For such purpose, the +metrics are computed with a fixed number of samples and as a function of the number +of clusters assigned by the estimator. The example is divided into two +experiments: + +- a first experiment with fixed "ground truth labels" (and therefore fixed + number of classes) and randomly "predicted labels"; +- a second experiment with varying "ground truth labels", randomly "predicted + labels". The "predicted labels" have the same number of classes and clusters + as the "ground truth labels". +""" -The following plots demonstrate the impact of the number of clusters and -number of samples on various clustering performance evaluation metrics. +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Defining the list of metrics to evaluate +# ---------------------------------------- +# +# Clustering algorithms are fundamentally unsupervised learning methods. +# However, since we assign class labels for the synthetic clusters in this +# example, it is possible to use evaluation metrics that leverage this +# "supervised" ground truth information to quantify the quality of the resulting +# clusters. Examples of such metrics are the following: +# +# - V-measure, the harmonic mean of completeness and homogeneity; +# +# - Rand index, which measures how frequently pairs of data points are grouped +# consistently according to the result of the clustering algorithm and the +# ground truth class assignment; +# +# - Adjusted Rand index (ARI), a chance-adjusted Rand index such that a random +# cluster assignment has an ARI of 0.0 in expectation; +# +# - Mutual Information (MI) is an information theoretic measure that quantifies +# how dependent are the two labelings. Note that the maximum value of MI for +# perfect labelings depends on the number of clusters and samples; +# +# - Normalized Mutual Information (NMI), a Mutual Information defined between 0 +# (no mutual information) in the limit of large number of data points and 1 +# (perfectly matching label assignments, up to a permutation of the labels). +# It is not adjusted for chance: then the number of clustered data points is +# not large enough, the expected values of MI or NMI for random labelings can +# be significantly non-zero; +# +# - Adjusted Mutual Information (AMI), a chance-adjusted Mutual Information. +# Similarly to ARI, random cluster assignment has an AMI of 0.0 in +# expectation. +# +# For more information, see the :ref:`clustering_evaluation` module. -Non-adjusted measures such as the V-Measure show a dependency between -the number of clusters and the number of samples: the mean V-Measure -of random labeling increases significantly as the number of clusters is -closer to the total number of samples used to compute the measure. +from sklearn import metrics -Adjusted for chance measure such as ARI display some random variations -centered around a mean score of 0.0 for any number of samples and -clusters. +score_funcs = [ + ("V-measure", metrics.v_measure_score), + ("Rand index", metrics.rand_score), + ("ARI", metrics.adjusted_rand_score), + ("MI", metrics.mutual_info_score), + ("NMI", metrics.normalized_mutual_info_score), + ("AMI", metrics.adjusted_mutual_info_score), +] -Only adjusted measures can hence safely be used as a consensus index -to evaluate the average stability of clustering algorithms for a given -value of k on various overlapping sub-samples of the dataset. +# %% +# First experiment: fixed ground truth labels and growing number of clusters +# -------------------------------------------------------------------------- +# +# We first define a function that creates uniformly-distributed random labeling. -""" -print(__doc__) +import numpy as np -# Author: Olivier Grisel -# License: BSD 3 clause +rng = np.random.RandomState(0) -import numpy as np -import matplotlib.pyplot as plt -from time import time -from sklearn import metrics -def uniform_labelings_scores(score_func, n_samples, n_clusters_range, - fixed_n_classes=None, n_runs=5, seed=42): - """Compute score for 2 random uniform cluster labelings. +def random_labels(n_samples, n_classes): + return rng.randint(low=0, high=n_classes, size=n_samples) - Both random labelings have the same number of clusters for each value - possible value in ``n_clusters_range``. - When fixed_n_classes is not None the first labeling is considered a ground - truth class assignment with fixed number of classes. - """ - random_labels = np.random.RandomState(seed).randint - scores = np.zeros((len(n_clusters_range), n_runs)) +# %% +# Another function will use the `random_labels` function to create a fixed set +# of ground truth labels (`labels_a`) distributed in `n_classes` and then score +# several sets of randomly "predicted" labels (`labels_b`) to assess the +# variability of a given metric at a given `n_clusters`. + - if fixed_n_classes is not None: - labels_a = random_labels(low=0, high=fixed_n_classes, size=n_samples) +def fixed_classes_uniform_labelings_scores( + score_func, n_samples, n_clusters_range, n_classes, n_runs=5 +): + scores = np.zeros((len(n_clusters_range), n_runs)) + labels_a = random_labels(n_samples=n_samples, n_classes=n_classes) - for i, k in enumerate(n_clusters_range): + for i, n_clusters in enumerate(n_clusters_range): for j in range(n_runs): - if fixed_n_classes is None: - labels_a = random_labels(low=0, high=k, size=n_samples) - labels_b = random_labels(low=0, high=k, size=n_samples) + labels_b = random_labels(n_samples=n_samples, n_classes=n_clusters) scores[i, j] = score_func(labels_a, labels_b) return scores -def ami_score(U, V): - return metrics.adjusted_mutual_info_score(U, V) +# %% +# In this first example we set the number of classes (true number of clusters) to +# `n_classes=10`. The number of clusters varies over the values provided by +# `n_clusters_range`. -score_funcs = [ - metrics.adjusted_rand_score, - metrics.v_measure_score, - ami_score, - metrics.mutual_info_score, -] - -# 2 independent random clusterings with equal cluster number - -n_samples = 100 -n_clusters_range = np.linspace(2, n_samples, 10).astype(int) - -plt.figure(1) +import matplotlib.pyplot as plt +import seaborn as sns +n_samples = 1000 +n_classes = 10 +n_clusters_range = np.linspace(2, 100, 10).astype(int) plots = [] names = [] -for score_func in score_funcs: - print("Computing %s for %d values of n_clusters and n_samples=%d" - % (score_func.__name__, len(n_clusters_range), n_samples)) - t0 = time() - scores = uniform_labelings_scores(score_func, n_samples, n_clusters_range) - print("done in %0.3fs" % (time() - t0)) - plots.append(plt.errorbar( - n_clusters_range, np.median(scores, axis=1), scores.std(axis=1))[0]) - names.append(score_func.__name__) - -plt.title("Clustering measures for 2 random uniform labelings\n" - "with equal number of clusters") -plt.xlabel('Number of clusters (Number of samples is fixed to %d)' % n_samples) -plt.ylabel('Score value') -plt.legend(plots, names) +sns.color_palette("colorblind") +plt.figure(1) + +for marker, (score_name, score_func) in zip("d^vx.,", score_funcs): + scores = fixed_classes_uniform_labelings_scores( + score_func, n_samples, n_clusters_range, n_classes=n_classes + ) + plots.append( + plt.errorbar( + n_clusters_range, + scores.mean(axis=1), + scores.std(axis=1), + alpha=0.8, + linewidth=1, + marker=marker, + )[0] + ) + names.append(score_name) + +plt.title( + "Clustering measures for random uniform labeling\n" + f"against reference assignment with {n_classes} classes" +) +plt.xlabel(f"Number of clusters (Number of samples is fixed to {n_samples})") +plt.ylabel("Score value") plt.ylim(bottom=-0.05, top=1.05) +plt.legend(plots, names, bbox_to_anchor=(0.5, 0.5)) +plt.show() +# %% +# The Rand index saturates for `n_clusters` > `n_classes`. Other non-adjusted +# measures such as the V-Measure show a linear dependency between the number of +# clusters and the number of samples. +# +# Adjusted for chance measure, such as ARI and AMI, display some random +# variations centered around a mean score of 0.0, independently of the number of +# samples and clusters. +# +# Second experiment: varying number of classes and clusters +# --------------------------------------------------------- +# +# In this section we define a similar function that uses several metrics to +# score 2 uniformly-distributed random labelings. In this case, the number of +# classes and assigned number of clusters are matched for each possible value in +# `n_clusters_range`. + + +def uniform_labelings_scores(score_func, n_samples, n_clusters_range, n_runs=5): + scores = np.zeros((len(n_clusters_range), n_runs)) -# Random labeling with varying n_clusters against ground class labels -# with fixed number of clusters + for i, n_clusters in enumerate(n_clusters_range): + for j in range(n_runs): + labels_a = random_labels(n_samples=n_samples, n_classes=n_clusters) + labels_b = random_labels(n_samples=n_samples, n_classes=n_clusters) + scores[i, j] = score_func(labels_a, labels_b) + return scores -n_samples = 1000 -n_clusters_range = np.linspace(2, 100, 10).astype(int) -n_classes = 10 + +# %% +# In this case, we use `n_samples=100` to show the effect of having a number of +# clusters similar or equal to the number of samples. + +n_samples = 100 +n_clusters_range = np.linspace(2, n_samples, 10).astype(int) plt.figure(2) plots = [] names = [] -for score_func in score_funcs: - print("Computing %s for %d values of n_clusters and n_samples=%d" - % (score_func.__name__, len(n_clusters_range), n_samples)) - - t0 = time() - scores = uniform_labelings_scores(score_func, n_samples, n_clusters_range, - fixed_n_classes=n_classes) - print("done in %0.3fs" % (time() - t0)) - plots.append(plt.errorbar( - n_clusters_range, scores.mean(axis=1), scores.std(axis=1))[0]) - names.append(score_func.__name__) - -plt.title("Clustering measures for random uniform labeling\n" - "against reference assignment with %d classes" % n_classes) -plt.xlabel('Number of clusters (Number of samples is fixed to %d)' % n_samples) -plt.ylabel('Score value') -plt.ylim(bottom=-0.05, top=1.05) + +for marker, (score_name, score_func) in zip("d^vx.,", score_funcs): + scores = uniform_labelings_scores(score_func, n_samples, n_clusters_range) + plots.append( + plt.errorbar( + n_clusters_range, + np.median(scores, axis=1), + scores.std(axis=1), + alpha=0.8, + linewidth=2, + marker=marker, + )[0] + ) + names.append(score_name) + +plt.title( + "Clustering measures for 2 random uniform labelings\nwith equal number of clusters" +) +plt.xlabel(f"Number of clusters (Number of samples is fixed to {n_samples})") +plt.ylabel("Score value") plt.legend(plots, names) +plt.ylim(bottom=-0.05, top=1.05) plt.show() + +# %% +# We observe similar results as for the first experiment: adjusted for chance +# metrics stay constantly near zero while other metrics tend to get larger with +# finer-grained labelings. The mean V-measure of random labeling increases +# significantly as the number of clusters is closer to the total number of +# samples used to compute the measure. Furthermore, raw Mutual Information is +# unbounded from above and its scale depends on the dimensions of the clustering +# problem and the cardinality of the ground truth classes. This is why the +# curve goes off the chart. +# +# Only adjusted measures can hence be safely used as a consensus index to +# evaluate the average stability of clustering algorithms for a given value of k +# on various overlapping sub-samples of the dataset. +# +# Non-adjusted clustering evaluation metric can therefore be misleading as they +# output large values for fine-grained labelings, one could be lead to think +# that the labeling has captured meaningful groups while they can be totally +# random. In particular, such non-adjusted metrics should not be used to compare +# the results of different clustering algorithms that output a different number +# of clusters. diff --git a/examples/cluster/plot_affinity_propagation.py b/examples/cluster/plot_affinity_propagation.py index 9ca5f17dda83d..2066212abea5d 100644 --- a/examples/cluster/plot_affinity_propagation.py +++ b/examples/cluster/plot_affinity_propagation.py @@ -8,55 +8,71 @@ Between Data Points", Science Feb. 2007 """ -print(__doc__) -from sklearn.cluster import AffinityPropagation +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import numpy as np + from sklearn import metrics +from sklearn.cluster import AffinityPropagation from sklearn.datasets import make_blobs -# ############################################################################# +# %% # Generate sample data +# -------------------- centers = [[1, 1], [-1, -1], [1, -1]] -X, labels_true = make_blobs(n_samples=300, centers=centers, cluster_std=0.5, - random_state=0) +X, labels_true = make_blobs( + n_samples=300, centers=centers, cluster_std=0.5, random_state=0 +) -# ############################################################################# +# %% # Compute Affinity Propagation -af = AffinityPropagation(preference=-50).fit(X) +# ---------------------------- +af = AffinityPropagation(preference=-50, random_state=0).fit(X) cluster_centers_indices = af.cluster_centers_indices_ labels = af.labels_ n_clusters_ = len(cluster_centers_indices) -print('Estimated number of clusters: %d' % n_clusters_) +print("Estimated number of clusters: %d" % n_clusters_) print("Homogeneity: %0.3f" % metrics.homogeneity_score(labels_true, labels)) print("Completeness: %0.3f" % metrics.completeness_score(labels_true, labels)) print("V-measure: %0.3f" % metrics.v_measure_score(labels_true, labels)) -print("Adjusted Rand Index: %0.3f" - % metrics.adjusted_rand_score(labels_true, labels)) -print("Adjusted Mutual Information: %0.3f" - % metrics.adjusted_mutual_info_score(labels_true, labels)) -print("Silhouette Coefficient: %0.3f" - % metrics.silhouette_score(X, labels, metric='sqeuclidean')) - -# ############################################################################# +print("Adjusted Rand Index: %0.3f" % metrics.adjusted_rand_score(labels_true, labels)) +print( + "Adjusted Mutual Information: %0.3f" + % metrics.adjusted_mutual_info_score(labels_true, labels) +) +print( + "Silhouette Coefficient: %0.3f" + % metrics.silhouette_score(X, labels, metric="sqeuclidean") +) + +# %% # Plot result +# ----------- import matplotlib.pyplot as plt -from itertools import cycle -plt.close('all') +plt.close("all") plt.figure(1) plt.clf() -colors = cycle('bgrcmykbgrcmykbgrcmykbgrcmyk') +colors = plt.cycler("color", plt.cm.viridis(np.linspace(0, 1, 4))) + for k, col in zip(range(n_clusters_), colors): class_members = labels == k cluster_center = X[cluster_centers_indices[k]] - plt.plot(X[class_members, 0], X[class_members, 1], col + '.') - plt.plot(cluster_center[0], cluster_center[1], 'o', markerfacecolor=col, - markeredgecolor='k', markersize=14) + plt.scatter( + X[class_members, 0], X[class_members, 1], color=col["color"], marker="." + ) + plt.scatter( + cluster_center[0], cluster_center[1], s=14, color=col["color"], marker="o" + ) for x in X[class_members]: - plt.plot([cluster_center[0], x[0]], [cluster_center[1], x[1]], col) + plt.plot( + [cluster_center[0], x[0]], [cluster_center[1], x[1]], color=col["color"] + ) -plt.title('Estimated number of clusters: %d' % n_clusters_) +plt.title("Estimated number of clusters: %d" % n_clusters_) plt.show() diff --git a/examples/cluster/plot_agglomerative_clustering.py b/examples/cluster/plot_agglomerative_clustering.py index b5fbce9e49b0d..f6165266206aa 100644 --- a/examples/cluster/plot_agglomerative_clustering.py +++ b/examples/cluster/plot_agglomerative_clustering.py @@ -6,8 +6,8 @@ local structure in the data. The graph is simply the graph of 20 nearest neighbors. -Two consequences of imposing a connectivity can be seen. First clustering -with a connectivity matrix is much faster. +There are two advantages of imposing a connectivity. First, clustering +with sparse connectivity matrices is faster in general. Second, when using a connectivity matrix, single, average and complete linkage are unstable and tend to create a few clusters that grow very @@ -20,11 +20,15 @@ (try decreasing the number of neighbors in kneighbors_graph) and with complete linkage. In particular, having a very small number of neighbors in the graph, imposes a geometry that is close to that of single linkage, -which is well known to have this percolation instability. """ -# Authors: Gael Varoquaux, Nelle Varoquaux -# License: BSD 3 clause +which is well known to have this percolation instability. + +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import time + import matplotlib.pyplot as plt import numpy as np @@ -40,7 +44,7 @@ X = np.concatenate((x, y)) -X += .7 * np.random.randn(2, n_samples) +X += 0.7 * np.random.randn(2, n_samples) X = X.T # Create a graph capturing local connectivity. Larger number of neighbors @@ -53,28 +57,28 @@ for connectivity in (None, knn_graph): for n_clusters in (30, 3): plt.figure(figsize=(10, 4)) - for index, linkage in enumerate(('average', - 'complete', - 'ward', - 'single')): + for index, linkage in enumerate(("average", "complete", "ward", "single")): plt.subplot(1, 4, index + 1) - model = AgglomerativeClustering(linkage=linkage, - connectivity=connectivity, - n_clusters=n_clusters) + model = AgglomerativeClustering( + linkage=linkage, connectivity=connectivity, n_clusters=n_clusters + ) t0 = time.time() model.fit(X) elapsed_time = time.time() - t0 - plt.scatter(X[:, 0], X[:, 1], c=model.labels_, - cmap=plt.cm.nipy_spectral) - plt.title('linkage=%s\n(time %.2fs)' % (linkage, elapsed_time), - fontdict=dict(verticalalignment='top')) - plt.axis('equal') - plt.axis('off') - - plt.subplots_adjust(bottom=0, top=.83, wspace=0, - left=0, right=1) - plt.suptitle('n_cluster=%i, connectivity=%r' % - (n_clusters, connectivity is not None), size=17) + plt.scatter(X[:, 0], X[:, 1], c=model.labels_, cmap=plt.cm.nipy_spectral) + plt.title( + "linkage=%s\n(time %.2fs)" % (linkage, elapsed_time), + fontdict=dict(verticalalignment="top"), + ) + plt.axis("equal") + plt.axis("off") + + plt.subplots_adjust(bottom=0, top=0.83, wspace=0, left=0, right=1) + plt.suptitle( + "n_cluster=%i, connectivity=%r" + % (n_clusters, connectivity is not None), + size=17, + ) plt.show() diff --git a/examples/cluster/plot_agglomerative_clustering_metrics.py b/examples/cluster/plot_agglomerative_clustering_metrics.py index be3bbad36764e..dbf929d9576e1 100644 --- a/examples/cluster/plot_agglomerative_clustering_metrics.py +++ b/examples/cluster/plot_agglomerative_clustering_metrics.py @@ -18,7 +18,7 @@ We add observation noise to these waveforms. We generate very sparse noise: only 6% of the time points contain noise. As a result, the -l1 norm of this noise (ie "cityblock" distance) is much smaller than it's +l1 norm of this noise (ie "cityblock" distance) is much smaller than its l2 norm ("euclidean" distance). This can be seen on the inter-class distance matrices: the values on the diagonal, that characterize the spread of the class, are much bigger for the Euclidean distance than for @@ -31,10 +31,13 @@ distance, the separation is good and the waveform classes are recovered. Finally, the cosine distance does not separate at all waveform 1 and 2, thus the clustering puts them in the same cluster. + """ -# Author: Gael Varoquaux -# License: BSD 3-Clause or CC-0 +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import matplotlib.patheffects as PathEffects import matplotlib.pyplot as plt import numpy as np @@ -51,19 +54,24 @@ def sqr(x): return np.sign(np.cos(x)) + X = list() y = list() -for i, (phi, a) in enumerate([(.5, .15), (.5, .6), (.3, .2)]): +for i, (phi, a) in enumerate([(0.5, 0.15), (0.5, 0.6), (0.3, 0.2)]): for _ in range(30): - phase_noise = .01 * np.random.normal() - amplitude_noise = .04 * np.random.normal() + phase_noise = 0.01 * np.random.normal() + amplitude_noise = 0.04 * np.random.normal() additional_noise = 1 - 2 * np.random.rand(n_features) # Make the noise sparse - additional_noise[np.abs(additional_noise) < .997] = 0 - - X.append(12 * ((a + amplitude_noise) - * (sqr(6 * (t + phi + phase_noise))) - + additional_noise)) + additional_noise[np.abs(additional_noise) < 0.997] = 0 + + X.append( + 12 + * ( + (a + amplitude_noise) * (sqr(6 * (t + phi + phase_noise))) + + additional_noise + ) + ) y.append(i) X = np.array(X) @@ -71,21 +79,22 @@ def sqr(x): n_clusters = 3 -labels = ('Waveform 1', 'Waveform 2', 'Waveform 3') +labels = ("Waveform 1", "Waveform 2", "Waveform 3") + +colors = ["#f7bd01", "#377eb8", "#f781bf"] # Plot the ground-truth labelling plt.figure() plt.axes([0, 0, 1, 1]) -for l, c, n in zip(range(n_clusters), 'rgb', - labels): - lines = plt.plot(X[y == l].T, c=c, alpha=.5) +for l, color, n in zip(range(n_clusters), colors, labels): + lines = plt.plot(X[y == l].T, c=color, alpha=0.5) lines[0].set_label(n) -plt.legend(loc='best') +plt.legend(loc="best") -plt.axis('tight') -plt.axis('off') -plt.suptitle("Ground truth", size=20) +plt.axis("tight") +plt.axis("off") +plt.suptitle("Ground truth", size=20, y=1) # Plot the distances @@ -94,36 +103,44 @@ def sqr(x): plt.figure(figsize=(5, 4.5)) for i in range(n_clusters): for j in range(n_clusters): - avg_dist[i, j] = pairwise_distances(X[y == i], X[y == j], - metric=metric).mean() + avg_dist[i, j] = pairwise_distances( + X[y == i], X[y == j], metric=metric + ).mean() avg_dist /= avg_dist.max() for i in range(n_clusters): for j in range(n_clusters): - plt.text(i, j, '%5.3f' % avg_dist[i, j], - verticalalignment='center', - horizontalalignment='center') - - plt.imshow(avg_dist, interpolation='nearest', cmap=plt.cm.gnuplot2, - vmin=0) + t = plt.text( + i, + j, + "%5.3f" % avg_dist[i, j], + verticalalignment="center", + horizontalalignment="center", + ) + t.set_path_effects( + [PathEffects.withStroke(linewidth=5, foreground="w", alpha=0.5)] + ) + + plt.imshow(avg_dist, interpolation="nearest", cmap="cividis", vmin=0) plt.xticks(range(n_clusters), labels, rotation=45) plt.yticks(range(n_clusters), labels) plt.colorbar() - plt.suptitle("Interclass %s distances" % metric, size=18) + plt.suptitle("Interclass %s distances" % metric, size=18, y=1) plt.tight_layout() # Plot clustering results for index, metric in enumerate(["cosine", "euclidean", "cityblock"]): - model = AgglomerativeClustering(n_clusters=n_clusters, - linkage="average", affinity=metric) + model = AgglomerativeClustering( + n_clusters=n_clusters, linkage="average", metric=metric + ) model.fit(X) plt.figure() plt.axes([0, 0, 1, 1]) - for l, c in zip(np.arange(model.n_clusters), 'rgbk'): - plt.plot(X[model.labels_ == l].T, c=c, alpha=.5) - plt.axis('tight') - plt.axis('off') - plt.suptitle("AgglomerativeClustering(affinity=%s)" % metric, size=20) + for l, color in zip(np.arange(model.n_clusters), colors): + plt.plot(X[model.labels_ == l].T, c=color, alpha=0.5) + plt.axis("tight") + plt.axis("off") + plt.suptitle("AgglomerativeClustering(metric=%s)" % metric, size=20, y=1) plt.show() diff --git a/examples/cluster/plot_agglomerative_dendrogram.py b/examples/cluster/plot_agglomerative_dendrogram.py index 401d28803b946..bea2a5e84653a 100644 --- a/examples/cluster/plot_agglomerative_dendrogram.py +++ b/examples/cluster/plot_agglomerative_dendrogram.py @@ -1,19 +1,21 @@ -# Authors: Mathew Kallada, Andreas Mueller -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + """ ========================================= Plot Hierarchical Clustering Dendrogram ========================================= This example plots the corresponding dendrogram of a hierarchical clustering using AgglomerativeClustering and the dendrogram method available in scipy. + """ import numpy as np - from matplotlib import pyplot as plt from scipy.cluster.hierarchy import dendrogram -from sklearn.datasets import load_iris + from sklearn.cluster import AgglomerativeClustering +from sklearn.datasets import load_iris def plot_dendrogram(model, **kwargs): @@ -31,8 +33,9 @@ def plot_dendrogram(model, **kwargs): current_count += counts[child_idx - n_samples] counts[i] = current_count - linkage_matrix = np.column_stack([model.children_, model.distances_, - counts]).astype(float) + linkage_matrix = np.column_stack( + [model.children_, model.distances_, counts] + ).astype(float) # Plot the corresponding dendrogram dendrogram(linkage_matrix, **kwargs) @@ -45,8 +48,8 @@ def plot_dendrogram(model, **kwargs): model = AgglomerativeClustering(distance_threshold=0, n_clusters=None) model = model.fit(X) -plt.title('Hierarchical Clustering Dendrogram') +plt.title("Hierarchical Clustering Dendrogram") # plot the top three levels of the dendrogram -plot_dendrogram(model, truncate_mode='level', p=3) +plot_dendrogram(model, truncate_mode="level", p=3) plt.xlabel("Number of points in node (or index of point if no parenthesis).") plt.show() diff --git a/examples/cluster/plot_birch_vs_minibatchkmeans.py b/examples/cluster/plot_birch_vs_minibatchkmeans.py index c4648ee5bd795..347a28cbc95b7 100644 --- a/examples/cluster/plot_birch_vs_minibatchkmeans.py +++ b/examples/cluster/plot_birch_vs_minibatchkmeans.py @@ -5,39 +5,44 @@ This example compares the timing of BIRCH (with and without the global clustering step) and MiniBatchKMeans on a synthetic dataset having -100,000 samples and 2 features generated using make_blobs. +25,000 samples and 2 features generated using make_blobs. -If ``n_clusters`` is set to None, the data is reduced from 100,000 +Both ``MiniBatchKMeans`` and ``BIRCH`` are very scalable algorithms and could +run efficiently on hundreds of thousands or even millions of datapoints. We +chose to limit the dataset size of this example in the interest of keeping +our Continuous Integration resource usage reasonable but the interested +reader might enjoy editing this script to rerun it with a larger value for +`n_samples`. + +If ``n_clusters`` is set to None, the data is reduced from 25,000 samples to a set of 158 clusters. This can be viewed as a preprocessing step before the final (global) clustering step that further reduces these 158 clusters to 100 clusters. -""" -# Authors: Manoj Kumar -# License: BSD 3 clause +""" -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause from itertools import cycle from time import time -import numpy as np -import matplotlib.pyplot as plt + import matplotlib.colors as colors +import matplotlib.pyplot as plt +import numpy as np +from joblib import cpu_count from sklearn.cluster import Birch, MiniBatchKMeans from sklearn.datasets import make_blobs - # Generate centers for the blobs so that it forms a 10 X 10 grid. xx = np.linspace(-22, 22, 10) yy = np.linspace(-22, 22, 10) xx, yy = np.meshgrid(xx, yy) -n_centres = np.hstack((np.ravel(xx)[:, np.newaxis], - np.ravel(yy)[:, np.newaxis])) +n_centers = np.hstack((np.ravel(xx)[:, np.newaxis], np.ravel(yy)[:, np.newaxis])) # Generate blobs to do a comparison between MiniBatchKMeans and BIRCH. -X, y = make_blobs(n_samples=100000, centers=n_centres, random_state=0) +X, y = make_blobs(n_samples=25000, centers=n_centers, random_state=0) # Use all colors that matplotlib provides by default. colors_ = cycle(colors.cnames.keys()) @@ -47,16 +52,16 @@ # Compute clustering with BIRCH with and without the final clustering step # and plot. -birch_models = [Birch(threshold=1.7, n_clusters=None), - Birch(threshold=1.7, n_clusters=100)] -final_step = ['without global clustering', 'with global clustering'] +birch_models = [ + Birch(threshold=1.7, n_clusters=None), + Birch(threshold=1.7, n_clusters=100), +] +final_step = ["without global clustering", "with global clustering"] for ind, (birch_model, info) in enumerate(zip(birch_models, final_step)): t = time() birch_model.fit(X) - time_ = time() - t - print("BIRCH %s as the final step took %0.2f seconds" % ( - info, (time() - t))) + print("BIRCH %s as the final step took %0.2f seconds" % (info, (time() - t))) # Plot result labels = birch_model.labels_ @@ -67,20 +72,24 @@ ax = fig.add_subplot(1, 3, ind + 1) for this_centroid, k, col in zip(centroids, range(n_clusters), colors_): mask = labels == k - ax.scatter(X[mask, 0], X[mask, 1], - c='w', edgecolor=col, marker='.', alpha=0.5) + ax.scatter(X[mask, 0], X[mask, 1], c="w", edgecolor=col, marker=".", alpha=0.5) if birch_model.n_clusters is None: - ax.scatter(this_centroid[0], this_centroid[1], marker='+', - c='k', s=25) + ax.scatter(this_centroid[0], this_centroid[1], marker="+", c="k", s=25) ax.set_ylim([-25, 25]) ax.set_xlim([-25, 25]) ax.set_autoscaley_on(False) - ax.set_title('BIRCH %s' % info) + ax.set_title("BIRCH %s" % info) # Compute clustering with MiniBatchKMeans. -mbk = MiniBatchKMeans(init='k-means++', n_clusters=100, batch_size=100, - n_init=10, max_no_improvement=10, verbose=0, - random_state=0) +mbk = MiniBatchKMeans( + init="k-means++", + n_clusters=100, + batch_size=256 * cpu_count(), + n_init=10, + max_no_improvement=10, + verbose=0, + random_state=0, +) t0 = time() mbk.fit(X) t_mini_batch = time() - t0 @@ -88,13 +97,10 @@ mbk_means_labels_unique = np.unique(mbk.labels_) ax = fig.add_subplot(1, 3, 3) -for this_centroid, k, col in zip(mbk.cluster_centers_, - range(n_clusters), colors_): +for this_centroid, k, col in zip(mbk.cluster_centers_, range(n_clusters), colors_): mask = mbk.labels_ == k - ax.scatter(X[mask, 0], X[mask, 1], marker='.', - c='w', edgecolor=col, alpha=0.5) - ax.scatter(this_centroid[0], this_centroid[1], marker='+', - c='k', s=25) + ax.scatter(X[mask, 0], X[mask, 1], marker=".", c="w", edgecolor=col, alpha=0.5) + ax.scatter(this_centroid[0], this_centroid[1], marker="+", c="k", s=25) ax.set_xlim([-25, 25]) ax.set_ylim([-25, 25]) ax.set_title("MiniBatchKMeans") diff --git a/examples/cluster/plot_bisect_kmeans.py b/examples/cluster/plot_bisect_kmeans.py new file mode 100644 index 0000000000000..7fc738bf08218 --- /dev/null +++ b/examples/cluster/plot_bisect_kmeans.py @@ -0,0 +1,68 @@ +""" +============================================================= +Bisecting K-Means and Regular K-Means Performance Comparison +============================================================= + +This example shows differences between Regular K-Means algorithm and Bisecting K-Means. + +While K-Means clusterings are different when increasing n_clusters, +Bisecting K-Means clustering builds on top of the previous ones. As a result, it +tends to create clusters that have a more regular large-scale structure. This +difference can be visually observed: for all numbers of clusters, there is a +dividing line cutting the overall data cloud in two for BisectingKMeans, which is not +present for regular K-Means. + +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import matplotlib.pyplot as plt + +from sklearn.cluster import BisectingKMeans, KMeans +from sklearn.datasets import make_blobs + +print(__doc__) + + +# Generate sample data +n_samples = 10000 +random_state = 0 + +X, _ = make_blobs(n_samples=n_samples, centers=2, random_state=random_state) + +# Number of cluster centers for KMeans and BisectingKMeans +n_clusters_list = [4, 8, 16] + +# Algorithms to compare +clustering_algorithms = { + "Bisecting K-Means": BisectingKMeans, + "K-Means": KMeans, +} + +# Make subplots for each variant +fig, axs = plt.subplots( + len(clustering_algorithms), len(n_clusters_list), figsize=(12, 5) +) + +axs = axs.T + +for i, (algorithm_name, Algorithm) in enumerate(clustering_algorithms.items()): + for j, n_clusters in enumerate(n_clusters_list): + algo = Algorithm(n_clusters=n_clusters, random_state=random_state, n_init=3) + algo.fit(X) + centers = algo.cluster_centers_ + + axs[j, i].scatter(X[:, 0], X[:, 1], s=10, c=algo.labels_) + axs[j, i].scatter(centers[:, 0], centers[:, 1], c="r", s=20) + + axs[j, i].set_title(f"{algorithm_name} : {n_clusters} clusters") + + +# Hide x labels and tick labels for top plots and y ticks for right plots. +for ax in axs.flat: + ax.label_outer() + ax.set_xticks([]) + ax.set_yticks([]) + +plt.show() diff --git a/examples/cluster/plot_cluster_comparison.py b/examples/cluster/plot_cluster_comparison.py index 0eea5ee1e27e1..ce45ee2f7e99a 100644 --- a/examples/cluster/plot_cluster_comparison.py +++ b/examples/cluster/plot_cluster_comparison.py @@ -21,32 +21,36 @@ While these examples give some intuition about the algorithms, this intuition might not apply to very high dimensional data. + """ -print(__doc__) + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import time import warnings +from itertools import cycle, islice -import numpy as np import matplotlib.pyplot as plt +import numpy as np from sklearn import cluster, datasets, mixture from sklearn.neighbors import kneighbors_graph from sklearn.preprocessing import StandardScaler -from itertools import cycle, islice - -np.random.seed(0) # ============ # Generate datasets. We choose the size big enough to see the scalability # of the algorithms, but not too big to avoid too long running times # ============ -n_samples = 1500 -noisy_circles = datasets.make_circles(n_samples=n_samples, factor=.5, - noise=.05) -noisy_moons = datasets.make_moons(n_samples=n_samples, noise=.05) -blobs = datasets.make_blobs(n_samples=n_samples, random_state=8) -no_structure = np.random.rand(n_samples, 2), None +n_samples = 500 +seed = 30 +noisy_circles = datasets.make_circles( + n_samples=n_samples, factor=0.5, noise=0.05, random_state=seed +) +noisy_moons = datasets.make_moons(n_samples=n_samples, noise=0.05, random_state=seed) +blobs = datasets.make_blobs(n_samples=n_samples, random_state=seed) +rng = np.random.RandomState(seed) +no_structure = rng.rand(n_samples, 2), None # Anisotropicly distributed data random_state = 170 @@ -56,40 +60,81 @@ aniso = (X_aniso, y) # blobs with varied variances -varied = datasets.make_blobs(n_samples=n_samples, - cluster_std=[1.0, 2.5, 0.5], - random_state=random_state) +varied = datasets.make_blobs( + n_samples=n_samples, cluster_std=[1.0, 2.5, 0.5], random_state=random_state +) # ============ # Set up cluster parameters # ============ plt.figure(figsize=(9 * 2 + 3, 13)) -plt.subplots_adjust(left=.02, right=.98, bottom=.001, top=.95, wspace=.05, - hspace=.01) +plt.subplots_adjust( + left=0.02, right=0.98, bottom=0.001, top=0.95, wspace=0.05, hspace=0.01 +) plot_num = 1 -default_base = {'quantile': .3, - 'eps': .3, - 'damping': .9, - 'preference': -200, - 'n_neighbors': 10, - 'n_clusters': 3, - 'min_samples': 20, - 'xi': 0.05, - 'min_cluster_size': 0.1} +default_base = { + "quantile": 0.3, + "eps": 0.3, + "damping": 0.9, + "preference": -200, + "n_neighbors": 3, + "n_clusters": 3, + "min_samples": 7, + "xi": 0.05, + "min_cluster_size": 0.1, + "allow_single_cluster": True, + "hdbscan_min_cluster_size": 15, + "hdbscan_min_samples": 3, + "random_state": 42, +} datasets = [ - (noisy_circles, {'damping': .77, 'preference': -240, - 'quantile': .2, 'n_clusters': 2, - 'min_samples': 20, 'xi': 0.25}), - (noisy_moons, {'damping': .75, 'preference': -220, 'n_clusters': 2}), - (varied, {'eps': .18, 'n_neighbors': 2, - 'min_samples': 5, 'xi': 0.035, 'min_cluster_size': .2}), - (aniso, {'eps': .15, 'n_neighbors': 2, - 'min_samples': 20, 'xi': 0.1, 'min_cluster_size': .2}), - (blobs, {}), - (no_structure, {})] + ( + noisy_circles, + { + "damping": 0.77, + "preference": -240, + "quantile": 0.2, + "n_clusters": 2, + "min_samples": 7, + "xi": 0.08, + }, + ), + ( + noisy_moons, + { + "damping": 0.75, + "preference": -220, + "n_clusters": 2, + "min_samples": 7, + "xi": 0.1, + }, + ), + ( + varied, + { + "eps": 0.18, + "n_neighbors": 2, + "min_samples": 7, + "xi": 0.01, + "min_cluster_size": 0.2, + }, + ), + ( + aniso, + { + "eps": 0.15, + "n_neighbors": 2, + "min_samples": 7, + "xi": 0.1, + "min_cluster_size": 0.2, + }, + ), + (blobs, {"min_samples": 7, "xi": 0.1, "min_cluster_size": 0.2}), + (no_structure, {}), +] for i_dataset, (dataset, algo_params) in enumerate(datasets): # update parameters with dataset-specific values @@ -102,11 +147,12 @@ X = StandardScaler().fit_transform(X) # estimate bandwidth for mean shift - bandwidth = cluster.estimate_bandwidth(X, quantile=params['quantile']) + bandwidth = cluster.estimate_bandwidth(X, quantile=params["quantile"]) # connectivity matrix for structured Ward connectivity = kneighbors_graph( - X, n_neighbors=params['n_neighbors'], include_self=False) + X, n_neighbors=params["n_neighbors"], include_self=False + ) # make connectivity symmetric connectivity = 0.5 * (connectivity + connectivity.T) @@ -114,37 +160,60 @@ # Create cluster objects # ============ ms = cluster.MeanShift(bandwidth=bandwidth, bin_seeding=True) - two_means = cluster.MiniBatchKMeans(n_clusters=params['n_clusters']) + two_means = cluster.MiniBatchKMeans( + n_clusters=params["n_clusters"], + random_state=params["random_state"], + ) ward = cluster.AgglomerativeClustering( - n_clusters=params['n_clusters'], linkage='ward', - connectivity=connectivity) + n_clusters=params["n_clusters"], linkage="ward", connectivity=connectivity + ) spectral = cluster.SpectralClustering( - n_clusters=params['n_clusters'], eigen_solver='arpack', - affinity="nearest_neighbors") - dbscan = cluster.DBSCAN(eps=params['eps']) - optics = cluster.OPTICS(min_samples=params['min_samples'], - xi=params['xi'], - min_cluster_size=params['min_cluster_size']) + n_clusters=params["n_clusters"], + eigen_solver="arpack", + affinity="nearest_neighbors", + random_state=params["random_state"], + ) + dbscan = cluster.DBSCAN(eps=params["eps"]) + hdbscan = cluster.HDBSCAN( + min_samples=params["hdbscan_min_samples"], + min_cluster_size=params["hdbscan_min_cluster_size"], + allow_single_cluster=params["allow_single_cluster"], + ) + optics = cluster.OPTICS( + min_samples=params["min_samples"], + xi=params["xi"], + min_cluster_size=params["min_cluster_size"], + ) affinity_propagation = cluster.AffinityPropagation( - damping=params['damping'], preference=params['preference']) + damping=params["damping"], + preference=params["preference"], + random_state=params["random_state"], + ) average_linkage = cluster.AgglomerativeClustering( - linkage="average", affinity="cityblock", - n_clusters=params['n_clusters'], connectivity=connectivity) - birch = cluster.Birch(n_clusters=params['n_clusters']) + linkage="average", + metric="cityblock", + n_clusters=params["n_clusters"], + connectivity=connectivity, + ) + birch = cluster.Birch(n_clusters=params["n_clusters"]) gmm = mixture.GaussianMixture( - n_components=params['n_clusters'], covariance_type='full') + n_components=params["n_clusters"], + covariance_type="full", + random_state=params["random_state"], + ) clustering_algorithms = ( - ('MiniBatch\nKMeans', two_means), - ('Affinity\nPropagation', affinity_propagation), - ('MeanShift', ms), - ('Spectral\nClustering', spectral), - ('Ward', ward), - ('Agglomerative\nClustering', average_linkage), - ('DBSCAN', dbscan), - ('OPTICS', optics), - ('BIRCH', birch), - ('Gaussian\nMixture', gmm) + ("MiniBatch\nKMeans", two_means), + ("Affinity\nPropagation", affinity_propagation), + ("MeanShift", ms), + ("Spectral\nClustering", spectral), + ("Ward", ward), + ("Agglomerative\nClustering", average_linkage), + ("DBSCAN", dbscan), + ("HDBSCAN", hdbscan), + ("OPTICS", optics), + ("BIRCH", birch), + ("Gaussian\nMixture", gmm), ) for name, algorithm in clustering_algorithms: @@ -154,19 +223,21 @@ with warnings.catch_warnings(): warnings.filterwarnings( "ignore", - message="the number of connected components of the " + - "connectivity matrix is [0-9]{1,2}" + + message="the number of connected components of the " + "connectivity matrix is [0-9]{1,2}" " > 1. Completing it to avoid stopping the tree early.", - category=UserWarning) + category=UserWarning, + ) warnings.filterwarnings( "ignore", - message="Graph is not fully connected, spectral embedding" + + message="Graph is not fully connected, spectral embedding" " may not work as expected.", - category=UserWarning) + category=UserWarning, + ) algorithm.fit(X) t1 = time.time() - if hasattr(algorithm, 'labels_'): + if hasattr(algorithm, "labels_"): y_pred = algorithm.labels_.astype(int) else: y_pred = algorithm.predict(X) @@ -175,10 +246,26 @@ if i_dataset == 0: plt.title(name, size=18) - colors = np.array(list(islice(cycle(['#377eb8', '#ff7f00', '#4daf4a', - '#f781bf', '#a65628', '#984ea3', - '#999999', '#e41a1c', '#dede00']), - int(max(y_pred) + 1)))) + colors = np.array( + list( + islice( + cycle( + [ + "#377eb8", + "#ff7f00", + "#4daf4a", + "#f781bf", + "#a65628", + "#984ea3", + "#999999", + "#e41a1c", + "#dede00", + ] + ), + int(max(y_pred) + 1), + ) + ) + ) # add black color for outliers (if any) colors = np.append(colors, ["#000000"]) plt.scatter(X[:, 0], X[:, 1], s=10, color=colors[y_pred]) @@ -187,9 +274,14 @@ plt.ylim(-2.5, 2.5) plt.xticks(()) plt.yticks(()) - plt.text(.99, .01, ('%.2fs' % (t1 - t0)).lstrip('0'), - transform=plt.gca().transAxes, size=15, - horizontalalignment='right') + plt.text( + 0.99, + 0.01, + ("%.2fs" % (t1 - t0)).lstrip("0"), + transform=plt.gca().transAxes, + size=15, + horizontalalignment="right", + ) plot_num += 1 plt.show() diff --git a/examples/cluster/plot_cluster_iris.py b/examples/cluster/plot_cluster_iris.py deleted file mode 100755 index f20d1a2374008..0000000000000 --- a/examples/cluster/plot_cluster_iris.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -""" -========================================================= -K-means Clustering -========================================================= - -The plots display firstly what a K-means algorithm would yield -using three clusters. It is then shown what the effect of a bad -initialization is on the classification process: -By setting n_init to only 1 (default is 10), the amount of -times that the algorithm will be run with different centroid -seeds is reduced. -The next plot displays what using eight clusters would deliver -and finally the ground truth. - -""" -print(__doc__) - - -# Code source: GaÃĢl Varoquaux -# Modified for documentation by Jaques Grobler -# License: BSD 3 clause - -import numpy as np -import matplotlib.pyplot as plt -# Though the following import is not directly being used, it is required -# for 3D projection to work -from mpl_toolkits.mplot3d import Axes3D - -from sklearn.cluster import KMeans -from sklearn import datasets - -np.random.seed(5) - -iris = datasets.load_iris() -X = iris.data -y = iris.target - -estimators = [('k_means_iris_8', KMeans(n_clusters=8)), - ('k_means_iris_3', KMeans(n_clusters=3)), - ('k_means_iris_bad_init', KMeans(n_clusters=3, n_init=1, - init='random'))] - -fignum = 1 -titles = ['8 clusters', '3 clusters', '3 clusters, bad initialization'] -for name, est in estimators: - fig = plt.figure(fignum, figsize=(4, 3)) - ax = Axes3D(fig, rect=[0, 0, .95, 1], elev=48, azim=134) - est.fit(X) - labels = est.labels_ - - ax.scatter(X[:, 3], X[:, 0], X[:, 2], - c=labels.astype(float), edgecolor='k') - - ax.w_xaxis.set_ticklabels([]) - ax.w_yaxis.set_ticklabels([]) - ax.w_zaxis.set_ticklabels([]) - ax.set_xlabel('Petal width') - ax.set_ylabel('Sepal length') - ax.set_zlabel('Petal length') - ax.set_title(titles[fignum - 1]) - ax.dist = 12 - fignum = fignum + 1 - -# Plot the ground truth -fig = plt.figure(fignum, figsize=(4, 3)) -ax = Axes3D(fig, rect=[0, 0, .95, 1], elev=48, azim=134) - -for name, label in [('Setosa', 0), - ('Versicolour', 1), - ('Virginica', 2)]: - ax.text3D(X[y == label, 3].mean(), - X[y == label, 0].mean(), - X[y == label, 2].mean() + 2, name, - horizontalalignment='center', - bbox=dict(alpha=.2, edgecolor='w', facecolor='w')) -# Reorder the labels to have colors matching the cluster results -y = np.choose(y, [1, 2, 0]).astype(float) -ax.scatter(X[:, 3], X[:, 0], X[:, 2], c=y, edgecolor='k') - -ax.w_xaxis.set_ticklabels([]) -ax.w_yaxis.set_ticklabels([]) -ax.w_zaxis.set_ticklabels([]) -ax.set_xlabel('Petal width') -ax.set_ylabel('Sepal length') -ax.set_zlabel('Petal length') -ax.set_title('Ground Truth') -ax.dist = 12 - -fig.show() diff --git a/examples/cluster/plot_coin_segmentation.py b/examples/cluster/plot_coin_segmentation.py index 9fb9b11be2753..304ba35bf68bd 100644 --- a/examples/cluster/plot_coin_segmentation.py +++ b/examples/cluster/plot_coin_segmentation.py @@ -10,36 +10,29 @@ This procedure (spectral clustering on an image) is an efficient approximate solution for finding normalized graph cuts. -There are two options to assign labels: +There are three options to assign labels: -* with 'kmeans' spectral clustering will cluster samples in the embedding space +* 'kmeans' spectral clustering clusters samples in the embedding space using a kmeans algorithm -* whereas 'discrete' will iteratively search for the closest partition - space to the embedding space. +* 'discrete' iteratively searches for the closest partition + space to the embedding space of spectral clustering. +* 'cluster_qr' assigns labels using the QR factorization with pivoting + that directly determines the partition in the embedding space. """ -print(__doc__) -# Author: Gael Varoquaux , Brian Cheung -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import time -import numpy as np -from scipy.ndimage.filters import gaussian_filter import matplotlib.pyplot as plt -import skimage +import numpy as np +from scipy.ndimage import gaussian_filter from skimage.data import coins from skimage.transform import rescale -from sklearn.feature_extraction import image from sklearn.cluster import spectral_clustering -from sklearn.utils.fixes import parse_version - -# these were introduced in skimage-0.14 -if parse_version(skimage.__version__) >= parse_version('0.14'): - rescale_params = {'anti_aliasing': False, 'multichannel': False} -else: - rescale_params = {} +from sklearn.feature_extraction import image # load the coins as a numpy array orig_coins = coins() @@ -48,8 +41,7 @@ # Applying a Gaussian filter for smoothing prior to down-scaling # reduces aliasing artifacts. smoothened_coins = gaussian_filter(orig_coins, sigma=2) -rescaled_coins = rescale(smoothened_coins, 0.2, mode="reflect", - **rescale_params) +rescaled_coins = rescale(smoothened_coins, 0.2, mode="reflect", anti_aliasing=False) # Convert the image into a graph with the value of the gradient on the # edges. @@ -62,28 +54,51 @@ eps = 1e-6 graph.data = np.exp(-beta * graph.data / graph.data.std()) + eps -# Apply spectral clustering (this step goes much faster if you have pyamg -# installed) -N_REGIONS = 25 +# The number of segmented regions to display needs to be chosen manually. +# The current version of 'spectral_clustering' does not support determining +# the number of good quality clusters automatically. +n_regions = 26 # %% -# Visualize the resulting regions - -for assign_labels in ('kmeans', 'discretize'): +# Compute and visualize the resulting regions + +# Computing a few extra eigenvectors may speed up the eigen_solver. +# The spectral clustering quality may also benefit from requesting +# extra regions for segmentation. +n_regions_plus = 3 + +# Apply spectral clustering using the default eigen_solver='arpack'. +# Any implemented solver can be used: eigen_solver='arpack', 'lobpcg', or 'amg'. +# Choosing eigen_solver='amg' requires an extra package called 'pyamg'. +# The quality of segmentation and the speed of calculations is mostly determined +# by the choice of the solver and the value of the tolerance 'eigen_tol'. +# TODO: varying eigen_tol seems to have no effect for 'lobpcg' and 'amg' #21243. +for assign_labels in ("kmeans", "discretize", "cluster_qr"): t0 = time.time() - labels = spectral_clustering(graph, n_clusters=N_REGIONS, - assign_labels=assign_labels, random_state=42) + labels = spectral_clustering( + graph, + n_clusters=(n_regions + n_regions_plus), + eigen_tol=1e-7, + assign_labels=assign_labels, + random_state=42, + ) + t1 = time.time() labels = labels.reshape(rescaled_coins.shape) - plt.figure(figsize=(5, 5)) plt.imshow(rescaled_coins, cmap=plt.cm.gray) - for l in range(N_REGIONS): - plt.contour(labels == l, - colors=[plt.cm.nipy_spectral(l / float(N_REGIONS))]) + plt.xticks(()) plt.yticks(()) - title = 'Spectral clustering: %s, %.2fs' % (assign_labels, (t1 - t0)) + title = "Spectral clustering: %s, %.2fs" % (assign_labels, (t1 - t0)) print(title) plt.title(title) + for l in range(n_regions): + colors = [plt.cm.nipy_spectral((l + 4) / float(n_regions + 4))] + plt.contour(labels == l, colors=colors) + # To view individual segments as appear comment in plt.pause(0.5) plt.show() + +# TODO: After #21194 is merged and #21243 is fixed, check which eigen_solver +# is the best and set eigen_solver='arpack', 'lobpcg', or 'amg' and eigen_tol +# explicitly in this example. diff --git a/examples/cluster/plot_coin_ward_segmentation.py b/examples/cluster/plot_coin_ward_segmentation.py index 218e6fb4c8d86..431829af3bac8 100644 --- a/examples/cluster/plot_coin_ward_segmentation.py +++ b/examples/cluster/plot_coin_ward_segmentation.py @@ -6,72 +6,87 @@ Compute the segmentation of a 2D image with Ward hierarchical clustering. The clustering is spatially constrained in order for each segmented region to be in one piece. -""" - -# Author : Vincent Michel, 2010 -# Alexandre Gramfort, 2011 -# License: BSD 3 clause -print(__doc__) - -import time as time +""" -import numpy as np -from scipy.ndimage.filters import gaussian_filter +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import matplotlib.pyplot as plt +# %% +# Generate data +# ------------- -import skimage from skimage.data import coins -from skimage.transform import rescale - -from sklearn.feature_extraction.image import grid_to_graph -from sklearn.cluster import AgglomerativeClustering -from sklearn.utils.fixes import parse_version -# these were introduced in skimage-0.14 -if parse_version(skimage.__version__) >= parse_version('0.14'): - rescale_params = {'anti_aliasing': False, 'multichannel': False} -else: - rescale_params = {} - -# ############################################################################# -# Generate data orig_coins = coins() +# %% # Resize it to 20% of the original size to speed up the processing # Applying a Gaussian filter for smoothing prior to down-scaling # reduces aliasing artifacts. + +import numpy as np +from scipy.ndimage import gaussian_filter +from skimage.transform import rescale + smoothened_coins = gaussian_filter(orig_coins, sigma=2) -rescaled_coins = rescale(smoothened_coins, 0.2, mode="reflect", - **rescale_params) +rescaled_coins = rescale( + smoothened_coins, + 0.2, + mode="reflect", + anti_aliasing=False, +) X = np.reshape(rescaled_coins, (-1, 1)) -# ############################################################################# -# Define the structure A of the data. Pixels connected to their neighbors. +# %% +# Define structure of the data +# ---------------------------- +# +# Pixels are connected to their neighbors. + +from sklearn.feature_extraction.image import grid_to_graph + connectivity = grid_to_graph(*rescaled_coins.shape) -# ############################################################################# +# %% # Compute clustering +# ------------------ + +import time as time + +from sklearn.cluster import AgglomerativeClustering + print("Compute structured hierarchical clustering...") st = time.time() n_clusters = 27 # number of regions -ward = AgglomerativeClustering(n_clusters=n_clusters, linkage='ward', - connectivity=connectivity) +ward = AgglomerativeClustering( + n_clusters=n_clusters, linkage="ward", connectivity=connectivity +) ward.fit(X) label = np.reshape(ward.labels_, rescaled_coins.shape) -print("Elapsed time: ", time.time() - st) -print("Number of pixels: ", label.size) -print("Number of clusters: ", np.unique(label).size) +print(f"Elapsed time: {time.time() - st:.3f}s") +print(f"Number of pixels: {label.size}") +print(f"Number of clusters: {np.unique(label).size}") -# ############################################################################# +# %% # Plot the results on an image +# ---------------------------- +# +# Agglomerative clustering is able to segment each coin however, we have had to +# use a ``n_cluster`` larger than the number of coins because the segmentation +# is finding a large in the background. + +import matplotlib.pyplot as plt + plt.figure(figsize=(5, 5)) plt.imshow(rescaled_coins, cmap=plt.cm.gray) for l in range(n_clusters): - plt.contour(label == l, - colors=[plt.cm.nipy_spectral(l / float(n_clusters)), ]) -plt.xticks(()) -plt.yticks(()) + plt.contour( + label == l, + colors=[ + plt.cm.nipy_spectral(l / float(n_clusters)), + ], + ) +plt.axis("off") plt.show() diff --git a/examples/cluster/plot_color_quantization.py b/examples/cluster/plot_color_quantization.py deleted file mode 100644 index ccc45eff73306..0000000000000 --- a/examples/cluster/plot_color_quantization.py +++ /dev/null @@ -1,102 +0,0 @@ -# -*- coding: utf-8 -*- -""" -================================== -Color Quantization using K-Means -================================== - -Performs a pixel-wise Vector Quantization (VQ) of an image of the summer palace -(China), reducing the number of colors required to show the image from 96,615 -unique colors to 64, while preserving the overall appearance quality. - -In this example, pixels are represented in a 3D-space and K-means is used to -find 64 color clusters. In the image processing literature, the codebook -obtained from K-means (the cluster centers) is called the color palette. Using -a single byte, up to 256 colors can be addressed, whereas an RGB encoding -requires 3 bytes per pixel. The GIF file format, for example, uses such a -palette. - -For comparison, a quantized image using a random codebook (colors picked up -randomly) is also shown. -""" -# Authors: Robert Layton -# Olivier Grisel -# Mathieu Blondel -# -# License: BSD 3 clause - -print(__doc__) -import numpy as np -import matplotlib.pyplot as plt -from sklearn.cluster import KMeans -from sklearn.metrics import pairwise_distances_argmin -from sklearn.datasets import load_sample_image -from sklearn.utils import shuffle -from time import time - -n_colors = 64 - -# Load the Summer Palace photo -china = load_sample_image("china.jpg") - -# Convert to floats instead of the default 8 bits integer coding. Dividing by -# 255 is important so that plt.imshow behaves works well on float data (need to -# be in the range [0-1]) -china = np.array(china, dtype=np.float64) / 255 - -# Load Image and transform to a 2D numpy array. -w, h, d = original_shape = tuple(china.shape) -assert d == 3 -image_array = np.reshape(china, (w * h, d)) - -print("Fitting model on a small sub-sample of the data") -t0 = time() -image_array_sample = shuffle(image_array, random_state=0)[:1000] -kmeans = KMeans(n_clusters=n_colors, random_state=0).fit(image_array_sample) -print("done in %0.3fs." % (time() - t0)) - -# Get labels for all points -print("Predicting color indices on the full image (k-means)") -t0 = time() -labels = kmeans.predict(image_array) -print("done in %0.3fs." % (time() - t0)) - - -codebook_random = shuffle(image_array, random_state=0)[:n_colors] -print("Predicting color indices on the full image (random)") -t0 = time() -labels_random = pairwise_distances_argmin(codebook_random, - image_array, - axis=0) -print("done in %0.3fs." % (time() - t0)) - - -def recreate_image(codebook, labels, w, h): - """Recreate the (compressed) image from the code book & labels""" - d = codebook.shape[1] - image = np.zeros((w, h, d)) - label_idx = 0 - for i in range(w): - for j in range(h): - image[i][j] = codebook[labels[label_idx]] - label_idx += 1 - return image - -# Display all results, alongside original image -plt.figure(1) -plt.clf() -plt.axis('off') -plt.title('Original image (96,615 colors)') -plt.imshow(china) - -plt.figure(2) -plt.clf() -plt.axis('off') -plt.title('Quantized image (64 colors, K-Means)') -plt.imshow(recreate_image(kmeans.cluster_centers_, labels, w, h)) - -plt.figure(3) -plt.clf() -plt.axis('off') -plt.title('Quantized image (64 colors, Random)') -plt.imshow(recreate_image(codebook_random, labels_random, w, h)) -plt.show() diff --git a/examples/cluster/plot_dbscan.py b/examples/cluster/plot_dbscan.py index d7cfc3ec524b3..27a5db29c4191 100644 --- a/examples/cluster/plot_dbscan.py +++ b/examples/cluster/plot_dbscan.py @@ -1,75 +1,132 @@ -# -*- coding: utf-8 -*- """ =================================== Demo of DBSCAN clustering algorithm =================================== -Finds core samples of high density and expands clusters from them. +DBSCAN (Density-Based Spatial Clustering of Applications with Noise) finds core +samples in regions of high density and expands clusters from them. This +algorithm is good for data which contains clusters of similar density. + +See the :ref:`sphx_glr_auto_examples_cluster_plot_cluster_comparison.py` example +for a demo of different clustering algorithms on 2D datasets. """ -print(__doc__) -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Data generation +# --------------- +# +# We use :class:`~sklearn.datasets.make_blobs` to create 3 synthetic clusters. -from sklearn.cluster import DBSCAN -from sklearn import metrics from sklearn.datasets import make_blobs from sklearn.preprocessing import StandardScaler - -# ############################################################################# -# Generate sample data centers = [[1, 1], [-1, -1], [1, -1]] -X, labels_true = make_blobs(n_samples=750, centers=centers, cluster_std=0.4, - random_state=0) +X, labels_true = make_blobs( + n_samples=750, centers=centers, cluster_std=0.4, random_state=0 +) X = StandardScaler().fit_transform(X) -# ############################################################################# +# %% +# We can visualize the resulting data: + +import matplotlib.pyplot as plt + +plt.scatter(X[:, 0], X[:, 1]) +plt.show() + +# %% # Compute DBSCAN +# -------------- +# +# One can access the labels assigned by :class:`~sklearn.cluster.DBSCAN` using +# the `labels_` attribute. Noisy samples are given the label :math:`-1`. + +import numpy as np + +from sklearn import metrics +from sklearn.cluster import DBSCAN + db = DBSCAN(eps=0.3, min_samples=10).fit(X) -core_samples_mask = np.zeros_like(db.labels_, dtype=bool) -core_samples_mask[db.core_sample_indices_] = True labels = db.labels_ # Number of clusters in labels, ignoring noise if present. n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0) n_noise_ = list(labels).count(-1) -print('Estimated number of clusters: %d' % n_clusters_) -print('Estimated number of noise points: %d' % n_noise_) -print("Homogeneity: %0.3f" % metrics.homogeneity_score(labels_true, labels)) -print("Completeness: %0.3f" % metrics.completeness_score(labels_true, labels)) -print("V-measure: %0.3f" % metrics.v_measure_score(labels_true, labels)) -print("Adjusted Rand Index: %0.3f" - % metrics.adjusted_rand_score(labels_true, labels)) -print("Adjusted Mutual Information: %0.3f" - % metrics.adjusted_mutual_info_score(labels_true, labels)) -print("Silhouette Coefficient: %0.3f" - % metrics.silhouette_score(X, labels)) - -# ############################################################################# -# Plot result -import matplotlib.pyplot as plt +print("Estimated number of clusters: %d" % n_clusters_) +print("Estimated number of noise points: %d" % n_noise_) + +# %% +# Clustering algorithms are fundamentally unsupervised learning methods. +# However, since :class:`~sklearn.datasets.make_blobs` gives access to the true +# labels of the synthetic clusters, it is possible to use evaluation metrics +# that leverage this "supervised" ground truth information to quantify the +# quality of the resulting clusters. Examples of such metrics are the +# homogeneity, completeness, V-measure, Rand-Index, Adjusted Rand-Index and +# Adjusted Mutual Information (AMI). +# +# If the ground truth labels are not known, evaluation can only be performed +# using the model results itself. In that case, the Silhouette Coefficient comes +# in handy. +# +# For more information, see the +# :ref:`sphx_glr_auto_examples_cluster_plot_adjusted_for_chance_measures.py` +# example or the :ref:`clustering_evaluation` module. + +print(f"Homogeneity: {metrics.homogeneity_score(labels_true, labels):.3f}") +print(f"Completeness: {metrics.completeness_score(labels_true, labels):.3f}") +print(f"V-measure: {metrics.v_measure_score(labels_true, labels):.3f}") +print(f"Adjusted Rand Index: {metrics.adjusted_rand_score(labels_true, labels):.3f}") +print( + "Adjusted Mutual Information:" + f" {metrics.adjusted_mutual_info_score(labels_true, labels):.3f}" +) +print(f"Silhouette Coefficient: {metrics.silhouette_score(X, labels):.3f}") + +# %% +# Plot results +# ------------ +# +# Core samples (large dots) and non-core samples (small dots) are color-coded +# according to the assigned cluster. Samples tagged as noise are represented in +# black. -# Black removed and is used for noise instead. unique_labels = set(labels) -colors = [plt.cm.Spectral(each) - for each in np.linspace(0, 1, len(unique_labels))] +core_samples_mask = np.zeros_like(labels, dtype=bool) +core_samples_mask[db.core_sample_indices_] = True + +colors = [plt.cm.Spectral(each) for each in np.linspace(0, 1, len(unique_labels))] for k, col in zip(unique_labels, colors): if k == -1: # Black used for noise. col = [0, 0, 0, 1] - class_member_mask = (labels == k) + class_member_mask = labels == k xy = X[class_member_mask & core_samples_mask] - plt.plot(xy[:, 0], xy[:, 1], 'o', markerfacecolor=tuple(col), - markeredgecolor='k', markersize=14) + plt.plot( + xy[:, 0], + xy[:, 1], + "o", + markerfacecolor=tuple(col), + markeredgecolor="k", + markersize=14, + ) xy = X[class_member_mask & ~core_samples_mask] - plt.plot(xy[:, 0], xy[:, 1], 'o', markerfacecolor=tuple(col), - markeredgecolor='k', markersize=6) - -plt.title('Estimated number of clusters: %d' % n_clusters_) + plt.plot( + xy[:, 0], + xy[:, 1], + "o", + markerfacecolor=tuple(col), + markeredgecolor="k", + markersize=6, + ) + +plt.title(f"Estimated number of clusters: {n_clusters_}") plt.show() diff --git a/examples/cluster/plot_dict_face_patches.py b/examples/cluster/plot_dict_face_patches.py index ac144e7213cc5..27eeb07ec7867 100644 --- a/examples/cluster/plot_dict_face_patches.py +++ b/examples/cluster/plot_dict_face_patches.py @@ -1,6 +1,6 @@ """ Online learning of a dictionary of parts of faces -================================================== +================================================= This example uses a large dataset of faces to learn a set of 20 x 20 images patches that constitute faces. @@ -18,27 +18,34 @@ partial-fit. This is because the number of patches that they represent has become too low, and it is better to choose a random new cluster. -""" -print(__doc__) -import time +""" -import matplotlib.pyplot as plt -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +# %% +# Load the data +# ------------- from sklearn import datasets -from sklearn.cluster import MiniBatchKMeans -from sklearn.feature_extraction.image import extract_patches_2d faces = datasets.fetch_olivetti_faces() -# ############################################################################# +# %% # Learn the dictionary of images +# ------------------------------ + +import time -print('Learning the dictionary... ') +import numpy as np + +from sklearn.cluster import MiniBatchKMeans +from sklearn.feature_extraction.image import extract_patches_2d + +print("Learning the dictionary... ") rng = np.random.RandomState(0) -kmeans = MiniBatchKMeans(n_clusters=81, random_state=rng, verbose=True) +kmeans = MiniBatchKMeans(n_clusters=81, random_state=rng, verbose=True, n_init=3) patch_size = (20, 20) buffer = [] @@ -48,8 +55,7 @@ index = 0 for _ in range(6): for img in faces.images: - data = extract_patches_2d(img, patch_size, max_patches=50, - random_state=rng) + data = extract_patches_2d(img, patch_size, max_patches=50, random_state=rng) data = np.reshape(data, (len(data), -1)) buffer.append(data) index += 1 @@ -60,25 +66,29 @@ kmeans.partial_fit(data) buffer = [] if index % 100 == 0: - print('Partial fit of %4i out of %i' - % (index, 6 * len(faces.images))) + print("Partial fit of %4i out of %i" % (index, 6 * len(faces.images))) dt = time.time() - t0 -print('done in %.2fs.' % dt) +print("done in %.2fs." % dt) -# ############################################################################# +# %% # Plot the results +# ---------------- + +import matplotlib.pyplot as plt + plt.figure(figsize=(4.2, 4)) for i, patch in enumerate(kmeans.cluster_centers_): plt.subplot(9, 9, i + 1) - plt.imshow(patch.reshape(patch_size), cmap=plt.cm.gray, - interpolation='nearest') + plt.imshow(patch.reshape(patch_size), cmap=plt.cm.gray, interpolation="nearest") plt.xticks(()) plt.yticks(()) -plt.suptitle('Patches of faces\nTrain time %.1fs on %d patches' % - (dt, 8 * len(faces.images)), fontsize=16) +plt.suptitle( + "Patches of faces\nTrain time %.1fs on %d patches" % (dt, 8 * len(faces.images)), + fontsize=16, +) plt.subplots_adjust(0.08, 0.02, 0.92, 0.85, 0.08, 0.23) plt.show() diff --git a/examples/cluster/plot_digits_agglomeration.py b/examples/cluster/plot_digits_agglomeration.py index ce8bd8daf3bf0..8de14b0729f53 100644 --- a/examples/cluster/plot_digits_agglomeration.py +++ b/examples/cluster/plot_digits_agglomeration.py @@ -1,24 +1,20 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - """ ========================================================= Feature agglomeration ========================================================= -These images how similar features are merged together using +These images show how similar features are merged together using feature agglomeration. + """ -print(__doc__) -# Code source: GaÃĢl Varoquaux -# Modified for documentation by Jaques Grobler -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt +import numpy as np -from sklearn import datasets, cluster +from sklearn import cluster, datasets from sklearn.feature_extraction.image import grid_to_graph digits = datasets.load_digits() @@ -26,8 +22,7 @@ X = np.reshape(images, (len(images), -1)) connectivity = grid_to_graph(*images[0].shape) -agglo = cluster.FeatureAgglomeration(connectivity=connectivity, - n_clusters=32) +agglo = cluster.FeatureAgglomeration(connectivity=connectivity, n_clusters=32) agglo.fit(X) X_reduced = agglo.transform(X) @@ -36,26 +31,28 @@ images_restored = np.reshape(X_restored, images.shape) plt.figure(1, figsize=(4, 3.5)) plt.clf() -plt.subplots_adjust(left=.01, right=.99, bottom=.01, top=.91) +plt.subplots_adjust(left=0.01, right=0.99, bottom=0.01, top=0.91) for i in range(4): plt.subplot(3, 4, i + 1) - plt.imshow(images[i], cmap=plt.cm.gray, vmax=16, interpolation='nearest') + plt.imshow(images[i], cmap=plt.cm.gray, vmax=16, interpolation="nearest") plt.xticks(()) plt.yticks(()) if i == 1: - plt.title('Original data') + plt.title("Original data") plt.subplot(3, 4, 4 + i + 1) - plt.imshow(images_restored[i], cmap=plt.cm.gray, vmax=16, - interpolation='nearest') + plt.imshow(images_restored[i], cmap=plt.cm.gray, vmax=16, interpolation="nearest") if i == 1: - plt.title('Agglomerated data') + plt.title("Agglomerated data") plt.xticks(()) plt.yticks(()) plt.subplot(3, 4, 10) -plt.imshow(np.reshape(agglo.labels_, images[0].shape), - interpolation='nearest', cmap=plt.cm.nipy_spectral) +plt.imshow( + np.reshape(agglo.labels_, images[0].shape), + interpolation="nearest", + cmap=plt.cm.nipy_spectral, +) plt.xticks(()) plt.yticks(()) -plt.title('Labels') +plt.title("Labels") plt.show() diff --git a/examples/cluster/plot_digits_linkage.py b/examples/cluster/plot_digits_linkage.py index 790dc638b44f4..05d71d5c94172 100644 --- a/examples/cluster/plot_digits_linkage.py +++ b/examples/cluster/plot_digits_linkage.py @@ -12,66 +12,63 @@ What this example shows us is the behavior "rich getting richer" of agglomerative clustering that tends to create uneven cluster sizes. + This behavior is pronounced for the average linkage strategy, -that ends up with a couple of singleton clusters, while in the case -of single linkage we get a single central cluster with all other clusters -being drawn from noise points around the fringes. +that ends up with a couple of clusters with few datapoints. + +The case of single linkage is even more pathologic with a very +large cluster covering most digits, an intermediate size (clean) +cluster with most zero digits and all other clusters being drawn +from noise points around the fringes. + +The other linkage strategies lead to more evenly distributed +clusters that are therefore likely to be less sensible to a +random resampling of the dataset. + """ -# Authors: Gael Varoquaux -# License: BSD 3 clause (C) INRIA 2014 +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -print(__doc__) from time import time import numpy as np -from scipy import ndimage from matplotlib import pyplot as plt -from sklearn import manifold, datasets +from sklearn import datasets, manifold -X, y = datasets.load_digits(return_X_y=True) +digits = datasets.load_digits() +X, y = digits.data, digits.target n_samples, n_features = X.shape np.random.seed(0) -def nudge_images(X, y): - # Having a larger dataset shows more clearly the behavior of the - # methods, but we multiply the size of the dataset only by 2, as the - # cost of the hierarchical clustering methods are strongly - # super-linear in n_samples - shift = lambda x: ndimage.shift(x.reshape((8, 8)), - .3 * np.random.normal(size=2), - mode='constant', - ).ravel() - X = np.concatenate([X, np.apply_along_axis(shift, 1, X)]) - Y = np.concatenate([y, y], axis=0) - return X, Y - -X, y = nudge_images(X, y) - - -#---------------------------------------------------------------------- +# ---------------------------------------------------------------------- # Visualize the clustering def plot_clustering(X_red, labels, title=None): x_min, x_max = np.min(X_red, axis=0), np.max(X_red, axis=0) X_red = (X_red - x_min) / (x_max - x_min) plt.figure(figsize=(6, 4)) - for i in range(X_red.shape[0]): - plt.text(X_red[i, 0], X_red[i, 1], str(y[i]), - color=plt.cm.nipy_spectral(labels[i] / 10.), - fontdict={'weight': 'bold', 'size': 9}) + for digit in digits.target_names: + plt.scatter( + *X_red[y == digit].T, + marker=f"${digit}$", + s=50, + c=plt.cm.nipy_spectral(labels[y == digit] / 10), + alpha=0.5, + ) plt.xticks([]) plt.yticks([]) if title is not None: plt.title(title, size=17) - plt.axis('off') + plt.axis("off") plt.tight_layout(rect=[0, 0.03, 1, 0.95]) -#---------------------------------------------------------------------- + +# ---------------------------------------------------------------------- # 2D embedding of the digits dataset print("Computing embedding") X_red = manifold.SpectralEmbedding(n_components=2).fit_transform(X) @@ -79,7 +76,7 @@ def plot_clustering(X_red, labels, title=None): from sklearn.cluster import AgglomerativeClustering -for linkage in ('ward', 'average', 'complete', 'single'): +for linkage in ("ward", "average", "complete", "single"): clustering = AgglomerativeClustering(linkage=linkage, n_clusters=10) t0 = time() clustering.fit(X_red) diff --git a/examples/cluster/plot_face_compress.py b/examples/cluster/plot_face_compress.py index 4eed00d623f9b..4e248a0fc65b2 100644 --- a/examples/cluster/plot_face_compress.py +++ b/examples/cluster/plot_face_compress.py @@ -1,81 +1,190 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - """ -========================================================= +=========================== Vector Quantization Example -========================================================= - -Face, a 1024 x 768 size image of a raccoon face, -is used here to illustrate how `k`-means is -used for vector quantization. +=========================== +This example shows how one can use :class:`~sklearn.preprocessing.KBinsDiscretizer` +to perform vector quantization on a set of toy image, the raccoon face. """ -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Original image +# -------------- +# +# We start by loading the raccoon face image from SciPy. We will additionally check +# a couple of information regarding the image, such as the shape and data type used +# to store the image. +# +# Note that depending of the SciPy version, we have to adapt the import since the +# function returning the image is not located in the same module. Also, SciPy >= 1.10 +# requires the package `pooch` to be installed. +try: # Scipy >= 1.10 + from scipy.datasets import face +except ImportError: + from scipy.misc import face -# Code source: GaÃĢl Varoquaux -# Modified for documentation by Jaques Grobler -# License: BSD 3 clause +raccoon_face = face(gray=True) -import numpy as np -import scipy as sp -import matplotlib.pyplot as plt - -from sklearn import cluster +print(f"The dimension of the image is {raccoon_face.shape}") +print(f"The data used to encode the image is of type {raccoon_face.dtype}") +print(f"The number of bytes taken in RAM is {raccoon_face.nbytes}") +# %% +# Thus the image is a 2D array of 768 pixels in height and 1024 pixels in width. Each +# value is a 8-bit unsigned integer, which means that the image is encoded using 8 +# bits per pixel. The total memory usage of the image is 786 kilobytes (1 byte equals +# 8 bits). +# +# Using 8-bit unsigned integer means that the image is encoded using 256 different +# shades of gray, at most. We can check the distribution of these values. +import matplotlib.pyplot as plt -try: # SciPy >= 0.16 have face in misc - from scipy.misc import face - face = face(gray=True) -except ImportError: - face = sp.face(gray=True) - -n_clusters = 5 -np.random.seed(0) - -X = face.reshape((-1, 1)) # We need an (n_sample, n_feature) array -k_means = cluster.KMeans(n_clusters=n_clusters, n_init=4) -k_means.fit(X) -values = k_means.cluster_centers_.squeeze() -labels = k_means.labels_ - -# create an array from labels and values -face_compressed = np.choose(labels, values) -face_compressed.shape = face.shape - -vmin = face.min() -vmax = face.max() - -# original face -plt.figure(1, figsize=(3, 2.2)) -plt.imshow(face, cmap=plt.cm.gray, vmin=vmin, vmax=256) - -# compressed face -plt.figure(2, figsize=(3, 2.2)) -plt.imshow(face_compressed, cmap=plt.cm.gray, vmin=vmin, vmax=vmax) - -# equal bins face -regular_values = np.linspace(0, 256, n_clusters + 1) -regular_labels = np.searchsorted(regular_values, face) - 1 -regular_values = .5 * (regular_values[1:] + regular_values[:-1]) # mean -regular_face = np.choose(regular_labels.ravel(), regular_values, mode="clip") -regular_face.shape = face.shape -plt.figure(3, figsize=(3, 2.2)) -plt.imshow(regular_face, cmap=plt.cm.gray, vmin=vmin, vmax=vmax) - -# histogram -plt.figure(4, figsize=(3, 2.2)) -plt.clf() -plt.axes([.01, .01, .98, .98]) -plt.hist(X, bins=256, color='.5', edgecolor='.5') -plt.yticks(()) -plt.xticks(regular_values) -values = np.sort(values) -for center_1, center_2 in zip(values[:-1], values[1:]): - plt.axvline(.5 * (center_1 + center_2), color='b') - -for center_1, center_2 in zip(regular_values[:-1], regular_values[1:]): - plt.axvline(.5 * (center_1 + center_2), color='b', linestyle='--') - -plt.show() +fig, ax = plt.subplots(ncols=2, figsize=(12, 4)) + +ax[0].imshow(raccoon_face, cmap=plt.cm.gray) +ax[0].axis("off") +ax[0].set_title("Rendering of the image") +ax[1].hist(raccoon_face.ravel(), bins=256) +ax[1].set_xlabel("Pixel value") +ax[1].set_ylabel("Count of pixels") +ax[1].set_title("Distribution of the pixel values") +_ = fig.suptitle("Original image of a raccoon face") + +# %% +# Compression via vector quantization +# ----------------------------------- +# +# The idea behind compression via vector quantization is to reduce the number of +# gray levels to represent an image. For instance, we can use 8 values instead +# of 256 values. Therefore, it means that we could efficiently use 3 bits instead +# of 8 bits to encode a single pixel and therefore reduce the memory usage by a +# factor of approximately 2.5. We will later discuss about this memory usage. +# +# Encoding strategy +# """"""""""""""""" +# +# The compression can be done using a +# :class:`~sklearn.preprocessing.KBinsDiscretizer`. We need to choose a strategy +# to define the 8 gray values to sub-sample. The simplest strategy is to define +# them equally spaced, which correspond to setting `strategy="uniform"`. From +# the previous histogram, we know that this strategy is certainly not optimal. + +from sklearn.preprocessing import KBinsDiscretizer + +n_bins = 8 +encoder = KBinsDiscretizer( + n_bins=n_bins, + encode="ordinal", + strategy="uniform", + random_state=0, +) +compressed_raccoon_uniform = encoder.fit_transform(raccoon_face.reshape(-1, 1)).reshape( + raccoon_face.shape +) + +fig, ax = plt.subplots(ncols=2, figsize=(12, 4)) +ax[0].imshow(compressed_raccoon_uniform, cmap=plt.cm.gray) +ax[0].axis("off") +ax[0].set_title("Rendering of the image") +ax[1].hist(compressed_raccoon_uniform.ravel(), bins=256) +ax[1].set_xlabel("Pixel value") +ax[1].set_ylabel("Count of pixels") +ax[1].set_title("Sub-sampled distribution of the pixel values") +_ = fig.suptitle("Raccoon face compressed using 3 bits and a uniform strategy") + +# %% +# Qualitatively, we can spot some small regions where we see the effect of the +# compression (e.g. leaves on the bottom right corner). But after all, the resulting +# image is still looking good. +# +# We observe that the distribution of pixels values have been mapped to 8 +# different values. We can check the correspondence between such values and the +# original pixel values. + +bin_edges = encoder.bin_edges_[0] +bin_center = bin_edges[:-1] + (bin_edges[1:] - bin_edges[:-1]) / 2 +bin_center + +# %% +_, ax = plt.subplots() +ax.hist(raccoon_face.ravel(), bins=256) +color = "tab:orange" +for center in bin_center: + ax.axvline(center, color=color) + ax.text(center - 10, ax.get_ybound()[1] + 100, f"{center:.1f}", color=color) + +# %% +# As previously stated, the uniform sampling strategy is not optimal. Notice for +# instance that the pixels mapped to the value 7 will encode a rather small +# amount of information, whereas the mapped value 3 will represent a large +# amount of counts. We can instead use a clustering strategy such as k-means to +# find a more optimal mapping. + +encoder = KBinsDiscretizer( + n_bins=n_bins, + encode="ordinal", + strategy="kmeans", + random_state=0, +) +compressed_raccoon_kmeans = encoder.fit_transform(raccoon_face.reshape(-1, 1)).reshape( + raccoon_face.shape +) + +fig, ax = plt.subplots(ncols=2, figsize=(12, 4)) +ax[0].imshow(compressed_raccoon_kmeans, cmap=plt.cm.gray) +ax[0].axis("off") +ax[0].set_title("Rendering of the image") +ax[1].hist(compressed_raccoon_kmeans.ravel(), bins=256) +ax[1].set_xlabel("Pixel value") +ax[1].set_ylabel("Number of pixels") +ax[1].set_title("Distribution of the pixel values") +_ = fig.suptitle("Raccoon face compressed using 3 bits and a K-means strategy") + +# %% +bin_edges = encoder.bin_edges_[0] +bin_center = bin_edges[:-1] + (bin_edges[1:] - bin_edges[:-1]) / 2 +bin_center + +# %% +_, ax = plt.subplots() +ax.hist(raccoon_face.ravel(), bins=256) +color = "tab:orange" +for center in bin_center: + ax.axvline(center, color=color) + ax.text(center - 10, ax.get_ybound()[1] + 100, f"{center:.1f}", color=color) + +# %% +# The counts in the bins are now more balanced and their centers are no longer +# equally spaced. Note that we could enforce the same number of pixels per bin +# by using the `strategy="quantile"` instead of `strategy="kmeans"`. +# +# Memory footprint +# """""""""""""""" +# +# We previously stated that we should save 8 times less memory. Let's verify it. + +print(f"The number of bytes taken in RAM is {compressed_raccoon_kmeans.nbytes}") +print(f"Compression ratio: {compressed_raccoon_kmeans.nbytes / raccoon_face.nbytes}") + +# %% +# It is quite surprising to see that our compressed image is taking x8 more +# memory than the original image. This is indeed the opposite of what we +# expected. The reason is mainly due to the type of data used to encode the +# image. + +print(f"Type of the compressed image: {compressed_raccoon_kmeans.dtype}") + +# %% +# Indeed, the output of the :class:`~sklearn.preprocessing.KBinsDiscretizer` is +# an array of 64-bit float. It means that it takes x8 more memory. However, we +# use this 64-bit float representation to encode 8 values. Indeed, we will save +# memory only if we cast the compressed image into an array of 3-bits integers. We +# could use the method `numpy.ndarray.astype`. However, a 3-bits integer +# representation does not exist and to encode the 8 values, we would need to use +# the 8-bit unsigned integer representation as well. +# +# In practice, observing a memory gain would require the original image to be in +# a 64-bit float representation. diff --git a/examples/cluster/plot_feature_agglomeration_vs_univariate_selection.py b/examples/cluster/plot_feature_agglomeration_vs_univariate_selection.py index e148647446613..fbad9c0ad7a31 100644 --- a/examples/cluster/plot_feature_agglomeration_vs_univariate_selection.py +++ b/examples/cluster/plot_feature_agglomeration_vs_univariate_selection.py @@ -11,86 +11,90 @@ Both methods are compared in a regression problem using a BayesianRidge as supervised estimator. -""" -# Author: Alexandre Gramfort -# License: BSD 3 clause +""" -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +# %% import shutil import tempfile -import numpy as np import matplotlib.pyplot as plt -from scipy import linalg, ndimage +import numpy as np from joblib import Memory +from scipy import linalg, ndimage -from sklearn.feature_extraction.image import grid_to_graph from sklearn import feature_selection from sklearn.cluster import FeatureAgglomeration +from sklearn.feature_extraction.image import grid_to_graph from sklearn.linear_model import BayesianRidge +from sklearn.model_selection import GridSearchCV, KFold from sklearn.pipeline import Pipeline -from sklearn.model_selection import GridSearchCV -from sklearn.model_selection import KFold -# ############################################################################# -# Generate data +# %% +# Set parameters n_samples = 200 size = 40 # image size roi_size = 15 -snr = 5. +snr = 5.0 np.random.seed(0) -mask = np.ones([size, size], dtype=bool) +# %% +# Generate data coef = np.zeros((size, size)) -coef[0:roi_size, 0:roi_size] = -1. -coef[-roi_size:, -roi_size:] = 1. +coef[0:roi_size, 0:roi_size] = -1.0 +coef[-roi_size:, -roi_size:] = 1.0 -X = np.random.randn(n_samples, size ** 2) +X = np.random.randn(n_samples, size**2) for x in X: # smooth data x[:] = ndimage.gaussian_filter(x.reshape(size, size), sigma=1.0).ravel() X -= X.mean(axis=0) X /= X.std(axis=0) y = np.dot(X, coef.ravel()) + +# %% +# add noise noise = np.random.randn(y.shape[0]) -noise_coef = (linalg.norm(y, 2) / np.exp(snr / 20.)) / linalg.norm(noise, 2) -y += noise_coef * noise # add noise +noise_coef = (linalg.norm(y, 2) / np.exp(snr / 20.0)) / linalg.norm(noise, 2) +y += noise_coef * noise -# ############################################################################# +# %% # Compute the coefs of a Bayesian Ridge with GridSearch cv = KFold(2) # cross-validation generator for model selection ridge = BayesianRidge() cachedir = tempfile.mkdtemp() mem = Memory(location=cachedir, verbose=1) +# %% # Ward agglomeration followed by BayesianRidge connectivity = grid_to_graph(n_x=size, n_y=size) -ward = FeatureAgglomeration(n_clusters=10, connectivity=connectivity, - memory=mem) -clf = Pipeline([('ward', ward), ('ridge', ridge)]) +ward = FeatureAgglomeration(n_clusters=10, connectivity=connectivity, memory=mem) +clf = Pipeline([("ward", ward), ("ridge", ridge)]) # Select the optimal number of parcels with grid search -clf = GridSearchCV(clf, {'ward__n_clusters': [10, 20, 30]}, n_jobs=1, cv=cv) +clf = GridSearchCV(clf, {"ward__n_clusters": [10, 20, 30]}, n_jobs=1, cv=cv) clf.fit(X, y) # set the best parameters coef_ = clf.best_estimator_.steps[-1][1].coef_ coef_ = clf.best_estimator_.steps[0][1].inverse_transform(coef_) coef_agglomeration_ = coef_.reshape(size, size) +# %% # Anova univariate feature selection followed by BayesianRidge f_regression = mem.cache(feature_selection.f_regression) # caching function anova = feature_selection.SelectPercentile(f_regression) -clf = Pipeline([('anova', anova), ('ridge', ridge)]) +clf = Pipeline([("anova", anova), ("ridge", ridge)]) # Select the optimal percentage of features with grid search -clf = GridSearchCV(clf, {'anova__percentile': [5, 10, 20]}, cv=cv) +clf = GridSearchCV(clf, {"anova__percentile": [5, 10, 20]}, cv=cv) clf.fit(X, y) # set the best parameters coef_ = clf.best_estimator_.steps[-1][1].coef_ coef_ = clf.best_estimator_.steps[0][1].inverse_transform(coef_.reshape(1, -1)) coef_selection_ = coef_.reshape(size, size) -# ############################################################################# +# %% # Inverse the transformation to plot the results on an image -plt.close('all') +plt.close("all") plt.figure(figsize=(7.3, 2.7)) plt.subplot(1, 3, 1) plt.imshow(coef, interpolation="nearest", cmap=plt.cm.RdBu_r) @@ -104,5 +108,6 @@ plt.subplots_adjust(0.04, 0.0, 0.98, 0.94, 0.16, 0.26) plt.show() +# %% # Attempt to remove the temporary cachedir, but don't worry if it fails shutil.rmtree(cachedir, ignore_errors=True) diff --git a/examples/cluster/plot_hdbscan.py b/examples/cluster/plot_hdbscan.py new file mode 100644 index 0000000000000..eee221d578ca3 --- /dev/null +++ b/examples/cluster/plot_hdbscan.py @@ -0,0 +1,249 @@ +# -*- coding: utf-8 -*- +""" +==================================== +Demo of HDBSCAN clustering algorithm +==================================== +.. currentmodule:: sklearn + +In this demo we will take a look at :class:`cluster.HDBSCAN` from the +perspective of generalizing the :class:`cluster.DBSCAN` algorithm. +We'll compare both algorithms on specific datasets. Finally we'll evaluate +HDBSCAN's sensitivity to certain hyperparameters. + +We first define a couple utility functions for convenience. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +import matplotlib.pyplot as plt +import numpy as np + +from sklearn.cluster import DBSCAN, HDBSCAN +from sklearn.datasets import make_blobs + + +def plot(X, labels, probabilities=None, parameters=None, ground_truth=False, ax=None): + if ax is None: + _, ax = plt.subplots(figsize=(10, 4)) + labels = labels if labels is not None else np.ones(X.shape[0]) + probabilities = probabilities if probabilities is not None else np.ones(X.shape[0]) + # Black removed and is used for noise instead. + unique_labels = set(labels) + colors = [plt.cm.Spectral(each) for each in np.linspace(0, 1, len(unique_labels))] + # The probability of a point belonging to its labeled cluster determines + # the size of its marker + proba_map = {idx: probabilities[idx] for idx in range(len(labels))} + for k, col in zip(unique_labels, colors): + if k == -1: + # Black used for noise. + col = [0, 0, 0, 1] + + class_index = (labels == k).nonzero()[0] + for ci in class_index: + ax.plot( + X[ci, 0], + X[ci, 1], + "x" if k == -1 else "o", + markerfacecolor=tuple(col), + markeredgecolor="k", + markersize=4 if k == -1 else 1 + 5 * proba_map[ci], + ) + n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0) + preamble = "True" if ground_truth else "Estimated" + title = f"{preamble} number of clusters: {n_clusters_}" + if parameters is not None: + parameters_str = ", ".join(f"{k}={v}" for k, v in parameters.items()) + title += f" | {parameters_str}" + ax.set_title(title) + plt.tight_layout() + + +# %% +# Generate sample data +# -------------------- +# One of the greatest advantages of HDBSCAN over DBSCAN is its out-of-the-box +# robustness. It's especially remarkable on heterogeneous mixtures of data. +# Like DBSCAN, it can model arbitrary shapes and distributions, however unlike +# DBSCAN it does not require specification of an arbitrary and sensitive +# `eps` hyperparameter. +# +# For example, below we generate a dataset from a mixture of three bi-dimensional +# and isotropic Gaussian distributions. +centers = [[1, 1], [-1, -1], [1.5, -1.5]] +X, labels_true = make_blobs( + n_samples=750, centers=centers, cluster_std=[0.4, 0.1, 0.75], random_state=0 +) +plot(X, labels=labels_true, ground_truth=True) +# %% +# Scale Invariance +# ----------------- +# It's worth remembering that, while DBSCAN provides a default value for `eps` +# parameter, it hardly has a proper default value and must be tuned for the +# specific dataset at use. +# +# As a simple demonstration, consider the clustering for a `eps` value tuned +# for one dataset, and clustering obtained with the same value but applied to +# rescaled versions of the dataset. +fig, axes = plt.subplots(3, 1, figsize=(10, 12)) +dbs = DBSCAN(eps=0.3) +for idx, scale in enumerate([1, 0.5, 3]): + dbs.fit(X * scale) + plot(X * scale, dbs.labels_, parameters={"scale": scale, "eps": 0.3}, ax=axes[idx]) + +# %% +# Indeed, in order to maintain the same results we would have to scale `eps` by +# the same factor. +fig, axis = plt.subplots(1, 1, figsize=(12, 5)) +dbs = DBSCAN(eps=0.9).fit(3 * X) +plot(3 * X, dbs.labels_, parameters={"scale": 3, "eps": 0.9}, ax=axis) +# %% +# While standardizing data (e.g. using +# :class:`sklearn.preprocessing.StandardScaler`) helps mitigate this problem, +# great care must be taken to select the appropriate value for `eps`. +# +# HDBSCAN is much more robust in this sense: HDBSCAN can be seen as +# clustering over all possible values of `eps` and extracting the best +# clusters from all possible clusters (see :ref:`User Guide `). +# One immediate advantage is that HDBSCAN is scale-invariant. +fig, axes = plt.subplots(3, 1, figsize=(10, 12)) +hdb = HDBSCAN() +for idx, scale in enumerate([1, 0.5, 3]): + hdb.fit(X * scale) + plot( + X * scale, + hdb.labels_, + hdb.probabilities_, + ax=axes[idx], + parameters={"scale": scale}, + ) +# %% +# Multi-Scale Clustering +# ---------------------- +# HDBSCAN is much more than scale invariant though -- it is capable of +# multi-scale clustering, which accounts for clusters with varying density. +# Traditional DBSCAN assumes that any potential clusters are homogeneous in +# density. HDBSCAN is free from such constraints. To demonstrate this we +# consider the following dataset +centers = [[-0.85, -0.85], [-0.85, 0.85], [3, 3], [3, -3]] +X, labels_true = make_blobs( + n_samples=750, centers=centers, cluster_std=[0.2, 0.35, 1.35, 1.35], random_state=0 +) +plot(X, labels=labels_true, ground_truth=True) + +# %% +# This dataset is more difficult for DBSCAN due to the varying densities and +# spatial separation: +# +# - If `eps` is too large then we risk falsely clustering the two dense +# clusters as one since their mutual reachability will extend +# clusters. +# - If `eps` is too small, then we risk fragmenting the sparser clusters +# into many false clusters. +# +# Not to mention this requires manually tuning choices of `eps` until we +# find a tradeoff that we are comfortable with. +fig, axes = plt.subplots(2, 1, figsize=(10, 8)) +params = {"eps": 0.7} +dbs = DBSCAN(**params).fit(X) +plot(X, dbs.labels_, parameters=params, ax=axes[0]) +params = {"eps": 0.3} +dbs = DBSCAN(**params).fit(X) +plot(X, dbs.labels_, parameters=params, ax=axes[1]) + +# %% +# To properly cluster the two dense clusters, we would need a smaller value of +# epsilon, however at `eps=0.3` we are already fragmenting the sparse clusters, +# which would only become more severe as we decrease epsilon. Indeed it seems +# that DBSCAN is incapable of simultaneously separating the two dense clusters +# while preventing the sparse clusters from fragmenting. Let's compare with +# HDBSCAN. +hdb = HDBSCAN().fit(X) +plot(X, hdb.labels_, hdb.probabilities_) + +# %% +# HDBSCAN is able to adapt to the multi-scale structure of the dataset without +# requiring parameter tuning. While any sufficiently interesting dataset will +# require tuning, this case demonstrates that HDBSCAN can yield qualitatively +# better classes of clusterings without users' intervention which are +# inaccessible via DBSCAN. + +# %% +# Hyperparameter Robustness +# ------------------------- +# Ultimately tuning will be an important step in any real world application, so +# let's take a look at some of the most important hyperparameters for HDBSCAN. +# While HDBSCAN is free from the `eps` parameter of DBSCAN, it does still have +# some hyperparameters like `min_cluster_size` and `min_samples` which tune its +# results regarding density. We will however see that HDBSCAN is relatively robust +# to various real world examples thanks to those parameters whose clear meaning +# helps tuning them. +# +# `min_cluster_size` +# ^^^^^^^^^^^^^^^^^^ +# `min_cluster_size` is the minimum number of samples in a group for that +# group to be considered a cluster. +# +# Clusters smaller than the ones of this size will be left as noise. +# The default value is 5. This parameter is generally tuned to +# larger values as needed. Smaller values will likely to lead to results with +# fewer points labeled as noise. However values which too small will lead to +# false sub-clusters being picked up and preferred. Larger values tend to be +# more robust with respect to noisy datasets, e.g. high-variance clusters with +# significant overlap. + +PARAM = ({"min_cluster_size": 5}, {"min_cluster_size": 3}, {"min_cluster_size": 25}) +fig, axes = plt.subplots(3, 1, figsize=(10, 12)) +for i, param in enumerate(PARAM): + hdb = HDBSCAN(**param).fit(X) + labels = hdb.labels_ + + plot(X, labels, hdb.probabilities_, param, ax=axes[i]) + +# %% +# `min_samples` +# ^^^^^^^^^^^^^ +# `min_samples` is the number of samples in a neighborhood for a point to +# be considered as a core point, including the point itself. +# `min_samples` defaults to `min_cluster_size`. +# Similarly to `min_cluster_size`, larger values for `min_samples` increase +# the model's robustness to noise, but risks ignoring or discarding +# potentially valid but small clusters. +# `min_samples` better be tuned after finding a good value for `min_cluster_size`. + +PARAM = ( + {"min_cluster_size": 20, "min_samples": 5}, + {"min_cluster_size": 20, "min_samples": 3}, + {"min_cluster_size": 20, "min_samples": 25}, +) +fig, axes = plt.subplots(3, 1, figsize=(10, 12)) +for i, param in enumerate(PARAM): + hdb = HDBSCAN(**param).fit(X) + labels = hdb.labels_ + + plot(X, labels, hdb.probabilities_, param, ax=axes[i]) + +# %% +# `dbscan_clustering` +# ^^^^^^^^^^^^^^^^^^^ +# During `fit`, `HDBSCAN` builds a single-linkage tree which encodes the +# clustering of all points across all values of :class:`~cluster.DBSCAN`'s +# `eps` parameter. +# We can thus plot and evaluate these clusterings efficiently without fully +# recomputing intermediate values such as core-distances, mutual-reachability, +# and the minimum spanning tree. All we need to do is specify the `cut_distance` +# (equivalent to `eps`) we want to cluster with. + +PARAM = ( + {"cut_distance": 0.1}, + {"cut_distance": 0.5}, + {"cut_distance": 1.0}, +) +hdb = HDBSCAN() +hdb.fit(X) +fig, axes = plt.subplots(len(PARAM), 1, figsize=(10, 12)) +for i, param in enumerate(PARAM): + labels = hdb.dbscan_clustering(**param) + + plot(X, labels, hdb.probabilities_, param, ax=axes[i]) diff --git a/examples/cluster/plot_inductive_clustering.py b/examples/cluster/plot_inductive_clustering.py index c5a51db5ef577..29846b15cdb60 100644 --- a/examples/cluster/plot_inductive_clustering.py +++ b/examples/cluster/plot_inductive_clustering.py @@ -1,7 +1,7 @@ """ -============================================== +==================== Inductive Clustering -============================================== +==================== Clustering can be expensive, especially when our dataset contains millions of datapoints. Many clustering algorithms are not :term:`inductive` and so @@ -17,24 +17,39 @@ This example illustrates a generic implementation of a meta-estimator which extends clustering by inducing a classifier from the cluster labels. + """ -# Authors: Chirag Nagpal -# Christos Aridas -print(__doc__) -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt + from sklearn.base import BaseEstimator, clone from sklearn.cluster import AgglomerativeClustering from sklearn.datasets import make_blobs from sklearn.ensemble import RandomForestClassifier -from sklearn.utils.metaestimators import if_delegate_has_method - +from sklearn.inspection import DecisionBoundaryDisplay +from sklearn.utils.metaestimators import available_if +from sklearn.utils.validation import check_is_fitted N_SAMPLES = 5000 RANDOM_STATE = 42 +def _classifier_has(attr): + """Check if we can delegate a method to the underlying classifier. + + First, we check the first fitted classifier if available, otherwise we + check the unfitted classifier. + """ + return lambda estimator: ( + hasattr(estimator.classifier_, attr) + if hasattr(estimator, "classifier_") + else hasattr(estimator.classifier, attr) + ) + + class InductiveClusterer(BaseEstimator): def __init__(self, clusterer, classifier): self.clusterer = clusterer @@ -47,28 +62,28 @@ def fit(self, X, y=None): self.classifier_.fit(X, y) return self - @if_delegate_has_method(delegate='classifier_') + @available_if(_classifier_has("predict")) def predict(self, X): + check_is_fitted(self) return self.classifier_.predict(X) - @if_delegate_has_method(delegate='classifier_') + @available_if(_classifier_has("decision_function")) def decision_function(self, X): + check_is_fitted(self) return self.classifier_.decision_function(X) -def plot_scatter(X, color, alpha=0.5): - return plt.scatter(X[:, 0], - X[:, 1], - c=color, - alpha=alpha, - edgecolor='k') +def plot_scatter(X, color, alpha=0.5): + return plt.scatter(X[:, 0], X[:, 1], c=color, alpha=alpha, edgecolor="k") # Generate some training data from clustering -X, y = make_blobs(n_samples=N_SAMPLES, - cluster_std=[1.0, 1.0, 0.5], - centers=[(-5, -5), (0, 0), (5, 5)], - random_state=RANDOM_STATE) +X, y = make_blobs( + n_samples=N_SAMPLES, + cluster_std=[1.0, 1.0, 0.5], + centers=[(-5, -5), (0, 0), (5, 5)], + random_state=RANDOM_STATE, +) # Train a clustering algorithm on the training data and get the cluster labels @@ -83,13 +98,13 @@ def plot_scatter(X, color, alpha=0.5): # Generate new samples and plot them along with the original dataset -X_new, y_new = make_blobs(n_samples=10, - centers=[(-7, -1), (-2, 4), (3, 6)], - random_state=RANDOM_STATE) +X_new, y_new = make_blobs( + n_samples=10, centers=[(-7, -1), (-2, 4), (3, 6)], random_state=RANDOM_STATE +) plt.subplot(132) plot_scatter(X, cluster_labels) -plot_scatter(X_new, 'black', 1) +plot_scatter(X_new, "black", 1) plt.title("Unknown instances") @@ -101,20 +116,14 @@ def plot_scatter(X, color, alpha=0.5): probable_clusters = inductive_learner.predict(X_new) -plt.subplot(133) +ax = plt.subplot(133) plot_scatter(X, cluster_labels) plot_scatter(X_new, probable_clusters) # Plotting decision regions -x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1 -y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1 -xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.1), - np.arange(y_min, y_max, 0.1)) - -Z = inductive_learner.predict(np.c_[xx.ravel(), yy.ravel()]) -Z = Z.reshape(xx.shape) - -plt.contourf(xx, yy, Z, alpha=0.4) +DecisionBoundaryDisplay.from_estimator( + inductive_learner, X, response_method="predict", alpha=0.4, ax=ax +) plt.title("Classify unknown instances") plt.show() diff --git a/examples/cluster/plot_kmeans_assumptions.py b/examples/cluster/plot_kmeans_assumptions.py index 2ff04d523855a..25dcff9570314 100644 --- a/examples/cluster/plot_kmeans_assumptions.py +++ b/examples/cluster/plot_kmeans_assumptions.py @@ -3,62 +3,176 @@ Demonstration of k-means assumptions ==================================== -This example is meant to illustrate situations where k-means will produce -unintuitive and possibly unexpected clusters. In the first three plots, the -input data does not conform to some implicit assumption that k-means makes and -undesirable clusters are produced as a result. In the last plot, k-means -returns intuitive clusters despite unevenly sized blobs. +This example is meant to illustrate situations where k-means produces +unintuitive and possibly undesirable clusters. + """ -print(__doc__) -# Author: Phil Roth -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Data generation +# --------------- +# +# The function :func:`~sklearn.datasets.make_blobs` generates isotropic +# (spherical) gaussian blobs. To obtain anisotropic (elliptical) gaussian blobs +# one has to define a linear `transformation`. import numpy as np -import matplotlib.pyplot as plt -from sklearn.cluster import KMeans from sklearn.datasets import make_blobs -plt.figure(figsize=(12, 12)) - n_samples = 1500 random_state = 170 +transformation = [[0.60834549, -0.63667341], [-0.40887718, 0.85253229]] + X, y = make_blobs(n_samples=n_samples, random_state=random_state) +X_aniso = np.dot(X, transformation) # Anisotropic blobs +X_varied, y_varied = make_blobs( + n_samples=n_samples, cluster_std=[1.0, 2.5, 0.5], random_state=random_state +) # Unequal variance +X_filtered = np.vstack( + (X[y == 0][:500], X[y == 1][:100], X[y == 2][:10]) +) # Unevenly sized blobs +y_filtered = [0] * 500 + [1] * 100 + [2] * 10 -# Incorrect number of clusters -y_pred = KMeans(n_clusters=2, random_state=random_state).fit_predict(X) +# %% +# We can visualize the resulting data: -plt.subplot(221) -plt.scatter(X[:, 0], X[:, 1], c=y_pred) -plt.title("Incorrect Number of Blobs") +import matplotlib.pyplot as plt -# Anisotropicly distributed data -transformation = [[0.60834549, -0.63667341], [-0.40887718, 0.85253229]] -X_aniso = np.dot(X, transformation) -y_pred = KMeans(n_clusters=3, random_state=random_state).fit_predict(X_aniso) +fig, axs = plt.subplots(nrows=2, ncols=2, figsize=(12, 12)) + +axs[0, 0].scatter(X[:, 0], X[:, 1], c=y) +axs[0, 0].set_title("Mixture of Gaussian Blobs") + +axs[0, 1].scatter(X_aniso[:, 0], X_aniso[:, 1], c=y) +axs[0, 1].set_title("Anisotropically Distributed Blobs") + +axs[1, 0].scatter(X_varied[:, 0], X_varied[:, 1], c=y_varied) +axs[1, 0].set_title("Unequal Variance") + +axs[1, 1].scatter(X_filtered[:, 0], X_filtered[:, 1], c=y_filtered) +axs[1, 1].set_title("Unevenly Sized Blobs") -plt.subplot(222) -plt.scatter(X_aniso[:, 0], X_aniso[:, 1], c=y_pred) -plt.title("Anisotropicly Distributed Blobs") +plt.suptitle("Ground truth clusters").set_y(0.95) +plt.show() + +# %% +# Fit models and plot results +# --------------------------- +# +# The previously generated data is now used to show how +# :class:`~sklearn.cluster.KMeans` behaves in the following scenarios: +# +# - Non-optimal number of clusters: in a real setting there is no uniquely +# defined **true** number of clusters. An appropriate number of clusters has +# to be decided from data-based criteria and knowledge of the intended goal. +# - Anisotropically distributed blobs: k-means consists of minimizing sample's +# euclidean distances to the centroid of the cluster they are assigned to. As +# a consequence, k-means is more appropriate for clusters that are isotropic +# and normally distributed (i.e. spherical gaussians). +# - Unequal variance: k-means is equivalent to taking the maximum likelihood +# estimator for a "mixture" of k gaussian distributions with the same +# variances but with possibly different means. +# - Unevenly sized blobs: there is no theoretical result about k-means that +# states that it requires similar cluster sizes to perform well, yet +# minimizing euclidean distances does mean that the more sparse and +# high-dimensional the problem is, the higher is the need to run the algorithm +# with different centroid seeds to ensure a global minimal inertia. + +from sklearn.cluster import KMeans -# Different variance -X_varied, y_varied = make_blobs(n_samples=n_samples, - cluster_std=[1.0, 2.5, 0.5], - random_state=random_state) -y_pred = KMeans(n_clusters=3, random_state=random_state).fit_predict(X_varied) +common_params = { + "n_init": "auto", + "random_state": random_state, +} -plt.subplot(223) -plt.scatter(X_varied[:, 0], X_varied[:, 1], c=y_pred) -plt.title("Unequal Variance") +fig, axs = plt.subplots(nrows=2, ncols=2, figsize=(12, 12)) -# Unevenly sized blobs -X_filtered = np.vstack((X[y == 0][:500], X[y == 1][:100], X[y == 2][:10])) -y_pred = KMeans(n_clusters=3, - random_state=random_state).fit_predict(X_filtered) +y_pred = KMeans(n_clusters=2, **common_params).fit_predict(X) +axs[0, 0].scatter(X[:, 0], X[:, 1], c=y_pred) +axs[0, 0].set_title("Non-optimal Number of Clusters") -plt.subplot(224) +y_pred = KMeans(n_clusters=3, **common_params).fit_predict(X_aniso) +axs[0, 1].scatter(X_aniso[:, 0], X_aniso[:, 1], c=y_pred) +axs[0, 1].set_title("Anisotropically Distributed Blobs") + +y_pred = KMeans(n_clusters=3, **common_params).fit_predict(X_varied) +axs[1, 0].scatter(X_varied[:, 0], X_varied[:, 1], c=y_pred) +axs[1, 0].set_title("Unequal Variance") + +y_pred = KMeans(n_clusters=3, **common_params).fit_predict(X_filtered) +axs[1, 1].scatter(X_filtered[:, 0], X_filtered[:, 1], c=y_pred) +axs[1, 1].set_title("Unevenly Sized Blobs") + +plt.suptitle("Unexpected KMeans clusters").set_y(0.95) +plt.show() + +# %% +# Possible solutions +# ------------------ +# +# For an example on how to find a correct number of blobs, see +# :ref:`sphx_glr_auto_examples_cluster_plot_kmeans_silhouette_analysis.py`. +# In this case it suffices to set `n_clusters=3`. + +y_pred = KMeans(n_clusters=3, **common_params).fit_predict(X) +plt.scatter(X[:, 0], X[:, 1], c=y_pred) +plt.title("Optimal Number of Clusters") +plt.show() + +# %% +# To deal with unevenly sized blobs one can increase the number of random +# initializations. In this case we set `n_init=10` to avoid finding a +# sub-optimal local minimum. For more details see :ref:`kmeans_sparse_high_dim`. + +y_pred = KMeans(n_clusters=3, n_init=10, random_state=random_state).fit_predict( + X_filtered +) plt.scatter(X_filtered[:, 0], X_filtered[:, 1], c=y_pred) -plt.title("Unevenly Sized Blobs") +plt.title("Unevenly Sized Blobs \nwith several initializations") +plt.show() + +# %% +# As anisotropic and unequal variances are real limitations of the k-means +# algorithm, here we propose instead the use of +# :class:`~sklearn.mixture.GaussianMixture`, which also assumes gaussian +# clusters but does not impose any constraints on their variances. Notice that +# one still has to find the correct number of blobs (see +# :ref:`sphx_glr_auto_examples_mixture_plot_gmm_selection.py`). +# +# For an example on how other clustering methods deal with anisotropic or +# unequal variance blobs, see the example +# :ref:`sphx_glr_auto_examples_cluster_plot_cluster_comparison.py`. + +from sklearn.mixture import GaussianMixture +fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, figsize=(12, 6)) + +y_pred = GaussianMixture(n_components=3).fit_predict(X_aniso) +ax1.scatter(X_aniso[:, 0], X_aniso[:, 1], c=y_pred) +ax1.set_title("Anisotropically Distributed Blobs") + +y_pred = GaussianMixture(n_components=3).fit_predict(X_varied) +ax2.scatter(X_varied[:, 0], X_varied[:, 1], c=y_pred) +ax2.set_title("Unequal Variance") + +plt.suptitle("Gaussian mixture clusters").set_y(0.95) plt.show() + +# %% +# Final remarks +# ------------- +# +# In high-dimensional spaces, Euclidean distances tend to become inflated +# (not shown in this example). Running a dimensionality reduction algorithm +# prior to k-means clustering can alleviate this problem and speed up the +# computations (see the example +# :ref:`sphx_glr_auto_examples_text_plot_document_clustering.py`). +# +# In the case where clusters are known to be isotropic, have similar variance +# and are not too sparse, the k-means algorithm is quite effective and is one of +# the fastest clustering algorithms available. This advantage is lost if one has +# to restart it several times to avoid convergence to a local minimum. diff --git a/examples/cluster/plot_kmeans_digits.py b/examples/cluster/plot_kmeans_digits.py index 9fc0abebc4464..3e7c70b9d08a9 100644 --- a/examples/cluster/plot_kmeans_digits.py +++ b/examples/cluster/plot_kmeans_digits.py @@ -22,8 +22,11 @@ AMI adjusted mutual information silhouette silhouette coefficient =========== ======================================================== + """ -print(__doc__) + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause # %% # Load the dataset @@ -34,14 +37,13 @@ # to group images such that the handwritten digits on the image are the same. import numpy as np + from sklearn.datasets import load_digits data, labels = load_digits(return_X_y=True) (n_samples, n_features), n_digits = data.shape, np.unique(labels).size -print( - f"# digits: {n_digits}; # samples: {n_samples}; # features {n_features}" -) +print(f"# digits: {n_digits}; # samples: {n_samples}; # features {n_features}") # %% # Define our evaluation benchmark @@ -55,6 +57,7 @@ # * train and time the pipeline fitting; # * measure the performance of the clustering obtained via different metrics. from time import time + from sklearn import metrics from sklearn.pipeline import make_pipeline from sklearn.preprocessing import StandardScaler @@ -95,13 +98,18 @@ def bench_k_means(kmeans, name, data, labels): # The silhouette score requires the full dataset results += [ - metrics.silhouette_score(data, estimator[-1].labels_, - metric="euclidean", sample_size=300,) + metrics.silhouette_score( + data, + estimator[-1].labels_, + metric="euclidean", + sample_size=300, + ) ] # Show the results - formatter_result = ("{:9s}\t{:.3f}s\t{:.0f}\t{:.3f}\t{:.3f}" - "\t{:.3f}\t{:.3f}\t{:.3f}\t{:.3f}") + formatter_result = ( + "{:9s}\t{:.3f}s\t{:.0f}\t{:.3f}\t{:.3f}\t{:.3f}\t{:.3f}\t{:.3f}\t{:.3f}" + ) print(formatter_result.format(*results)) @@ -111,7 +119,7 @@ def bench_k_means(kmeans, name, data, labels): # # We will compare three approaches: # -# * an initialization using `kmeans++`. This method is stochastic and we will +# * an initialization using `k-means++`. This method is stochastic and we will # run the initialization 4 times; # * a random initialization. This method is stochastic as well and we will run # the initialization 4 times; @@ -122,11 +130,10 @@ def bench_k_means(kmeans, name, data, labels): from sklearn.cluster import KMeans from sklearn.decomposition import PCA -print(82 * '_') -print('init\t\ttime\tinertia\thomo\tcompl\tv-meas\tARI\tAMI\tsilhouette') +print(82 * "_") +print("init\t\ttime\tinertia\thomo\tcompl\tv-meas\tARI\tAMI\tsilhouette") -kmeans = KMeans(init="k-means++", n_clusters=n_digits, n_init=4, - random_state=0) +kmeans = KMeans(init="k-means++", n_clusters=n_digits, n_init=4, random_state=0) bench_k_means(kmeans=kmeans, name="k-means++", data=data, labels=labels) kmeans = KMeans(init="random", n_clusters=n_digits, n_init=4, random_state=0) @@ -136,7 +143,7 @@ def bench_k_means(kmeans, name, data, labels): kmeans = KMeans(init=pca.components_, n_clusters=n_digits, n_init=1) bench_k_means(kmeans=kmeans, name="PCA-based", data=data, labels=labels) -print(82 * '_') +print(82 * "_") # %% # Visualize the results on PCA-reduced data @@ -153,7 +160,7 @@ def bench_k_means(kmeans, name, data, labels): kmeans.fit(reduced_data) # Step size of the mesh. Decrease to increase the quality of the VQ. -h = .02 # point in the mesh [x_min, x_max]x[y_min, y_max]. +h = 0.02 # point in the mesh [x_min, x_max]x[y_min, y_max]. # Plot the decision boundary. For that, we will assign a color to each x_min, x_max = reduced_data[:, 0].min() - 1, reduced_data[:, 0].max() + 1 @@ -167,17 +174,31 @@ def bench_k_means(kmeans, name, data, labels): Z = Z.reshape(xx.shape) plt.figure(1) plt.clf() -plt.imshow(Z, interpolation="nearest", - extent=(xx.min(), xx.max(), yy.min(), yy.max()), - cmap=plt.cm.Paired, aspect="auto", origin="lower") +plt.imshow( + Z, + interpolation="nearest", + extent=(xx.min(), xx.max(), yy.min(), yy.max()), + cmap=plt.cm.Paired, + aspect="auto", + origin="lower", +) -plt.plot(reduced_data[:, 0], reduced_data[:, 1], 'k.', markersize=2) +plt.plot(reduced_data[:, 0], reduced_data[:, 1], "k.", markersize=2) # Plot the centroids as a white X centroids = kmeans.cluster_centers_ -plt.scatter(centroids[:, 0], centroids[:, 1], marker="x", s=169, linewidths=3, - color="w", zorder=10) -plt.title("K-means clustering on the digits dataset (PCA-reduced data)\n" - "Centroids are marked with white cross") +plt.scatter( + centroids[:, 0], + centroids[:, 1], + marker="x", + s=169, + linewidths=3, + color="w", + zorder=10, +) +plt.title( + "K-means clustering on the digits dataset (PCA-reduced data)\n" + "Centroids are marked with white cross" +) plt.xlim(x_min, x_max) plt.ylim(y_min, y_max) plt.xticks(()) diff --git a/examples/cluster/plot_kmeans_plusplus.py b/examples/cluster/plot_kmeans_plusplus.py index d9821db2d452e..355426a2a4872 100644 --- a/examples/cluster/plot_kmeans_plusplus.py +++ b/examples/cluster/plot_kmeans_plusplus.py @@ -9,36 +9,36 @@ K-Means++ is used as the default initialization for :ref:`k_means`. """ -print(__doc__) + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import matplotlib.pyplot as plt from sklearn.cluster import kmeans_plusplus from sklearn.datasets import make_blobs -import matplotlib.pyplot as plt # Generate sample data n_samples = 4000 n_components = 4 -X, y_true = make_blobs(n_samples=n_samples, - centers=n_components, - cluster_std=0.60, - random_state=0) +X, y_true = make_blobs( + n_samples=n_samples, centers=n_components, cluster_std=0.60, random_state=0 +) X = X[:, ::-1] -# Calculate seeds from kmeans++ -centers_init, indices = kmeans_plusplus(X, n_clusters=4, - random_state=0) +# Calculate seeds from k-means++ +centers_init, indices = kmeans_plusplus(X, n_clusters=4, random_state=0) # Plot init seeds along side sample data plt.figure(1) -colors = ['#4EACC5', '#FF9C34', '#4E9A06', 'm'] +colors = ["#4EACC5", "#FF9C34", "#4E9A06", "m"] for k, col in enumerate(colors): cluster_data = y_true == k - plt.scatter(X[cluster_data, 0], X[cluster_data, 1], - c=col, marker='.', s=10) + plt.scatter(X[cluster_data, 0], X[cluster_data, 1], c=col, marker=".", s=10) -plt.scatter(centers_init[:, 0], centers_init[:, 1], c='b', s=50) +plt.scatter(centers_init[:, 0], centers_init[:, 1], c="b", s=50) plt.title("K-Means++ Initialization") plt.xticks([]) plt.yticks([]) diff --git a/examples/cluster/plot_kmeans_silhouette_analysis.py b/examples/cluster/plot_kmeans_silhouette_analysis.py index 8cd65cfb6a865..4b5c8d2c6d66d 100644 --- a/examples/cluster/plot_kmeans_silhouette_analysis.py +++ b/examples/cluster/plot_kmeans_silhouette_analysis.py @@ -28,27 +28,32 @@ cluster. However when the ``n_clusters`` is equal to 4, all the plots are more or less of similar thickness and hence are of similar sizes as can be also verified from the labelled scatter plot on the right. + """ -from sklearn.datasets import make_blobs -from sklearn.cluster import KMeans -from sklearn.metrics import silhouette_samples, silhouette_score -import matplotlib.pyplot as plt +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.cm as cm +import matplotlib.pyplot as plt import numpy as np -print(__doc__) +from sklearn.cluster import KMeans +from sklearn.datasets import make_blobs +from sklearn.metrics import silhouette_samples, silhouette_score # Generating the sample data from make_blobs # This particular setting has one distinct cluster and 3 clusters placed close # together. -X, y = make_blobs(n_samples=500, - n_features=2, - centers=4, - cluster_std=1, - center_box=(-10.0, 10.0), - shuffle=True, - random_state=1) # For reproducibility +X, y = make_blobs( + n_samples=500, + n_features=2, + centers=4, + cluster_std=1, + center_box=(-10.0, 10.0), + shuffle=True, + random_state=1, +) # For reproducibility range_n_clusters = [2, 3, 4, 5, 6] @@ -74,8 +79,12 @@ # This gives a perspective into the density and separation of the formed # clusters silhouette_avg = silhouette_score(X, cluster_labels) - print("For n_clusters =", n_clusters, - "The average silhouette_score is :", silhouette_avg) + print( + "For n_clusters =", + n_clusters, + "The average silhouette_score is :", + silhouette_avg, + ) # Compute the silhouette scores for each sample sample_silhouette_values = silhouette_samples(X, cluster_labels) @@ -84,8 +93,7 @@ for i in range(n_clusters): # Aggregate the silhouette scores for samples belonging to # cluster i, and sort them - ith_cluster_silhouette_values = \ - sample_silhouette_values[cluster_labels == i] + ith_cluster_silhouette_values = sample_silhouette_values[cluster_labels == i] ith_cluster_silhouette_values.sort() @@ -93,9 +101,14 @@ y_upper = y_lower + size_cluster_i color = cm.nipy_spectral(float(i) / n_clusters) - ax1.fill_betweenx(np.arange(y_lower, y_upper), - 0, ith_cluster_silhouette_values, - facecolor=color, edgecolor=color, alpha=0.7) + ax1.fill_betweenx( + np.arange(y_lower, y_upper), + 0, + ith_cluster_silhouette_values, + facecolor=color, + edgecolor=color, + alpha=0.7, + ) # Label the silhouette plots with their cluster numbers at the middle ax1.text(-0.05, y_lower + 0.5 * size_cluster_i, str(i)) @@ -115,25 +128,35 @@ # 2nd Plot showing the actual clusters formed colors = cm.nipy_spectral(cluster_labels.astype(float) / n_clusters) - ax2.scatter(X[:, 0], X[:, 1], marker='.', s=30, lw=0, alpha=0.7, - c=colors, edgecolor='k') + ax2.scatter( + X[:, 0], X[:, 1], marker=".", s=30, lw=0, alpha=0.7, c=colors, edgecolor="k" + ) # Labeling the clusters centers = clusterer.cluster_centers_ # Draw white circles at cluster centers - ax2.scatter(centers[:, 0], centers[:, 1], marker='o', - c="white", alpha=1, s=200, edgecolor='k') + ax2.scatter( + centers[:, 0], + centers[:, 1], + marker="o", + c="white", + alpha=1, + s=200, + edgecolor="k", + ) for i, c in enumerate(centers): - ax2.scatter(c[0], c[1], marker='$%d$' % i, alpha=1, - s=50, edgecolor='k') + ax2.scatter(c[0], c[1], marker="$%d$" % i, alpha=1, s=50, edgecolor="k") ax2.set_title("The visualization of the clustered data.") ax2.set_xlabel("Feature space for the 1st feature") ax2.set_ylabel("Feature space for the 2nd feature") - plt.suptitle(("Silhouette analysis for KMeans clustering on sample data " - "with n_clusters = %d" % n_clusters), - fontsize=14, fontweight='bold') + plt.suptitle( + "Silhouette analysis for KMeans clustering on sample data with n_clusters = %d" + % n_clusters, + fontsize=14, + fontweight="bold", + ) plt.show() diff --git a/examples/cluster/plot_kmeans_stability_low_dim_dense.py b/examples/cluster/plot_kmeans_stability_low_dim_dense.py index 3f8a4e8e5233d..e56fc81eb6e88 100644 --- a/examples/cluster/plot_kmeans_stability_low_dim_dense.py +++ b/examples/cluster/plot_kmeans_stability_low_dim_dense.py @@ -4,36 +4,34 @@ ============================================================ Evaluate the ability of k-means initializations strategies to make -the algorithm convergence robust as measured by the relative standard +the algorithm convergence robust, as measured by the relative standard deviation of the inertia of the clustering (i.e. the sum of squared distances to the nearest cluster center). The first plot shows the best inertia reached for each combination -of the model (``KMeans`` or ``MiniBatchKMeans``) and the init method -(``init="random"`` or ``init="kmeans++"``) for increasing values of the +of the model (``KMeans`` or ``MiniBatchKMeans``), and the init method +(``init="random"`` or ``init="k-means++"``) for increasing values of the ``n_init`` parameter that controls the number of initializations. -The second plot demonstrate one single run of the ``MiniBatchKMeans`` +The second plot demonstrates one single run of the ``MiniBatchKMeans`` estimator using a ``init="random"`` and ``n_init=1``. This run leads to -a bad convergence (local optimum) with estimated centers stuck +a bad convergence (local optimum), with estimated centers stuck between ground truth clusters. The dataset used for evaluation is a 2D grid of isotropic Gaussian clusters widely spaced. + """ -print(__doc__) -# Author: Olivier Grisel -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np -import matplotlib.pyplot as plt import matplotlib.cm as cm +import matplotlib.pyplot as plt +import numpy as np -from sklearn.utils import shuffle -from sklearn.utils import check_random_state -from sklearn.cluster import MiniBatchKMeans -from sklearn.cluster import KMeans +from sklearn.cluster import KMeans, MiniBatchKMeans +from sklearn.utils import check_random_state, shuffle random_state = np.random.RandomState(0) @@ -49,71 +47,86 @@ n_samples_per_center = 100 grid_size = 3 scale = 0.1 -n_clusters = grid_size ** 2 +n_clusters = grid_size**2 def make_data(random_state, n_samples_per_center, grid_size, scale): random_state = check_random_state(random_state) - centers = np.array([[i, j] - for i in range(grid_size) - for j in range(grid_size)]) + centers = np.array([[i, j] for i in range(grid_size) for j in range(grid_size)]) n_clusters_true, n_features = centers.shape noise = random_state.normal( - scale=scale, size=(n_samples_per_center, centers.shape[1])) + scale=scale, size=(n_samples_per_center, centers.shape[1]) + ) X = np.concatenate([c + noise for c in centers]) - y = np.concatenate([[i] * n_samples_per_center - for i in range(n_clusters_true)]) + y = np.concatenate([[i] * n_samples_per_center for i in range(n_clusters_true)]) return shuffle(X, y, random_state=random_state) + # Part 1: Quantitative evaluation of various init methods + plt.figure() plots = [] legends = [] cases = [ - (KMeans, 'k-means++', {}), - (KMeans, 'random', {}), - (MiniBatchKMeans, 'k-means++', {'max_no_improvement': 3}), - (MiniBatchKMeans, 'random', {'max_no_improvement': 3, 'init_size': 500}), + (KMeans, "k-means++", {}, "^-"), + (KMeans, "random", {}, "o-"), + (MiniBatchKMeans, "k-means++", {"max_no_improvement": 3}, "x-"), + (MiniBatchKMeans, "random", {"max_no_improvement": 3, "init_size": 500}, "d-"), ] -for factory, init, params in cases: +for factory, init, params, format in cases: print("Evaluation of %s with %s init" % (factory.__name__, init)) inertia = np.empty((len(n_init_range), n_runs)) for run_id in range(n_runs): X, y = make_data(run_id, n_samples_per_center, grid_size, scale) for i, n_init in enumerate(n_init_range): - km = factory(n_clusters=n_clusters, init=init, random_state=run_id, - n_init=n_init, **params).fit(X) + km = factory( + n_clusters=n_clusters, + init=init, + random_state=run_id, + n_init=n_init, + **params, + ).fit(X) inertia[i, run_id] = km.inertia_ - p = plt.errorbar(n_init_range, inertia.mean(axis=1), inertia.std(axis=1)) + p = plt.errorbar( + n_init_range, inertia.mean(axis=1), inertia.std(axis=1), fmt=format + ) plots.append(p[0]) legends.append("%s with %s init" % (factory.__name__, init)) -plt.xlabel('n_init') -plt.ylabel('inertia') +plt.xlabel("n_init") +plt.ylabel("inertia") plt.legend(plots, legends) plt.title("Mean inertia for various k-means init across %d runs" % n_runs) # Part 2: Qualitative visual inspection of the convergence X, y = make_data(random_state, n_samples_per_center, grid_size, scale) -km = MiniBatchKMeans(n_clusters=n_clusters, init='random', n_init=1, - random_state=random_state).fit(X) +km = MiniBatchKMeans( + n_clusters=n_clusters, init="random", n_init=1, random_state=random_state +).fit(X) plt.figure() for k in range(n_clusters): my_members = km.labels_ == k color = cm.nipy_spectral(float(k) / n_clusters, 1) - plt.plot(X[my_members, 0], X[my_members, 1], 'o', marker='.', c=color) + plt.plot(X[my_members, 0], X[my_members, 1], ".", c=color) cluster_center = km.cluster_centers_[k] - plt.plot(cluster_center[0], cluster_center[1], 'o', - markerfacecolor=color, markeredgecolor='k', markersize=6) - plt.title("Example cluster allocation with a single random init\n" - "with MiniBatchKMeans") + plt.plot( + cluster_center[0], + cluster_center[1], + "o", + markerfacecolor=color, + markeredgecolor="k", + markersize=6, + ) + plt.title( + "Example cluster allocation with a single random init\nwith MiniBatchKMeans" + ) plt.show() diff --git a/examples/cluster/plot_linkage_comparison.py b/examples/cluster/plot_linkage_comparison.py index 390aa1fe889d2..359d02e88041a 100644 --- a/examples/cluster/plot_linkage_comparison.py +++ b/examples/cluster/plot_linkage_comparison.py @@ -20,64 +20,67 @@ While these examples give some intuition about the algorithms, this intuition might not apply to very high dimensional data. + """ -print(__doc__) + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import time import warnings +from itertools import cycle, islice -import numpy as np import matplotlib.pyplot as plt +import numpy as np from sklearn import cluster, datasets from sklearn.preprocessing import StandardScaler -from itertools import cycle, islice - -np.random.seed(0) # %% # Generate datasets. We choose the size big enough to see the scalability # of the algorithms, but not too big to avoid too long running times n_samples = 1500 -noisy_circles = datasets.make_circles(n_samples=n_samples, factor=.5, - noise=.05) -noisy_moons = datasets.make_moons(n_samples=n_samples, noise=.05) -blobs = datasets.make_blobs(n_samples=n_samples, random_state=8) -no_structure = np.random.rand(n_samples, 2), None +noisy_circles = datasets.make_circles( + n_samples=n_samples, factor=0.5, noise=0.05, random_state=170 +) +noisy_moons = datasets.make_moons(n_samples=n_samples, noise=0.05, random_state=170) +blobs = datasets.make_blobs(n_samples=n_samples, random_state=170) +rng = np.random.RandomState(170) +no_structure = rng.rand(n_samples, 2), None # Anisotropicly distributed data -random_state = 170 -X, y = datasets.make_blobs(n_samples=n_samples, random_state=random_state) +X, y = datasets.make_blobs(n_samples=n_samples, random_state=170) transformation = [[0.6, -0.6], [-0.4, 0.8]] X_aniso = np.dot(X, transformation) aniso = (X_aniso, y) # blobs with varied variances -varied = datasets.make_blobs(n_samples=n_samples, - cluster_std=[1.0, 2.5, 0.5], - random_state=random_state) +varied = datasets.make_blobs( + n_samples=n_samples, cluster_std=[1.0, 2.5, 0.5], random_state=170 +) # %% # Run the clustering and plot # Set up cluster parameters plt.figure(figsize=(9 * 1.3 + 2, 14.5)) -plt.subplots_adjust(left=.02, right=.98, bottom=.001, top=.96, wspace=.05, - hspace=.01) +plt.subplots_adjust( + left=0.02, right=0.98, bottom=0.001, top=0.96, wspace=0.05, hspace=0.01 +) plot_num = 1 -default_base = {'n_neighbors': 10, - 'n_clusters': 3} +default_base = {"n_neighbors": 10, "n_clusters": 3} datasets = [ - (noisy_circles, {'n_clusters': 2}), - (noisy_moons, {'n_clusters': 2}), - (varied, {'n_neighbors': 2}), - (aniso, {'n_neighbors': 2}), + (noisy_circles, {"n_clusters": 2}), + (noisy_moons, {"n_clusters": 2}), + (varied, {"n_neighbors": 2}), + (aniso, {"n_neighbors": 2}), (blobs, {}), - (no_structure, {})] + (no_structure, {}), +] for i_dataset, (dataset, algo_params) in enumerate(datasets): # update parameters with dataset-specific values @@ -93,19 +96,23 @@ # Create cluster objects # ============ ward = cluster.AgglomerativeClustering( - n_clusters=params['n_clusters'], linkage='ward') + n_clusters=params["n_clusters"], linkage="ward" + ) complete = cluster.AgglomerativeClustering( - n_clusters=params['n_clusters'], linkage='complete') + n_clusters=params["n_clusters"], linkage="complete" + ) average = cluster.AgglomerativeClustering( - n_clusters=params['n_clusters'], linkage='average') + n_clusters=params["n_clusters"], linkage="average" + ) single = cluster.AgglomerativeClustering( - n_clusters=params['n_clusters'], linkage='single') + n_clusters=params["n_clusters"], linkage="single" + ) clustering_algorithms = ( - ('Single Linkage', single), - ('Average Linkage', average), - ('Complete Linkage', complete), - ('Ward Linkage', ward), + ("Single Linkage", single), + ("Average Linkage", average), + ("Complete Linkage", complete), + ("Ward Linkage", ward), ) for name, algorithm in clustering_algorithms: @@ -115,14 +122,15 @@ with warnings.catch_warnings(): warnings.filterwarnings( "ignore", - message="the number of connected components of the " + - "connectivity matrix is [0-9]{1,2}" + + message="the number of connected components of the " + "connectivity matrix is [0-9]{1,2}" " > 1. Completing it to avoid stopping the tree early.", - category=UserWarning) + category=UserWarning, + ) algorithm.fit(X) t1 = time.time() - if hasattr(algorithm, 'labels_'): + if hasattr(algorithm, "labels_"): y_pred = algorithm.labels_.astype(int) else: y_pred = algorithm.predict(X) @@ -131,19 +139,40 @@ if i_dataset == 0: plt.title(name, size=18) - colors = np.array(list(islice(cycle(['#377eb8', '#ff7f00', '#4daf4a', - '#f781bf', '#a65628', '#984ea3', - '#999999', '#e41a1c', '#dede00']), - int(max(y_pred) + 1)))) + colors = np.array( + list( + islice( + cycle( + [ + "#377eb8", + "#ff7f00", + "#4daf4a", + "#f781bf", + "#a65628", + "#984ea3", + "#999999", + "#e41a1c", + "#dede00", + ] + ), + int(max(y_pred) + 1), + ) + ) + ) plt.scatter(X[:, 0], X[:, 1], s=10, color=colors[y_pred]) plt.xlim(-2.5, 2.5) plt.ylim(-2.5, 2.5) plt.xticks(()) plt.yticks(()) - plt.text(.99, .01, ('%.2fs' % (t1 - t0)).lstrip('0'), - transform=plt.gca().transAxes, size=15, - horizontalalignment='right') + plt.text( + 0.99, + 0.01, + ("%.2fs" % (t1 - t0)).lstrip("0"), + transform=plt.gca().transAxes, + size=15, + horizontalalignment="right", + ) plot_num += 1 plt.show() diff --git a/examples/cluster/plot_mean_shift.py b/examples/cluster/plot_mean_shift.py index dce7b35503232..456a1c4ac2020 100644 --- a/examples/cluster/plot_mean_shift.py +++ b/examples/cluster/plot_mean_shift.py @@ -10,19 +10,24 @@ Machine Intelligence. 2002. pp. 603-619. """ -print(__doc__) + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import numpy as np + from sklearn.cluster import MeanShift, estimate_bandwidth from sklearn.datasets import make_blobs -# ############################################################################# +# %% # Generate sample data +# -------------------- centers = [[1, 1], [-1, -1], [1, -1]] X, _ = make_blobs(n_samples=10000, centers=centers, cluster_std=0.6) -# ############################################################################# +# %% # Compute clustering with MeanShift +# --------------------------------- # The following bandwidth can be automatically detected using bandwidth = estimate_bandwidth(X, quantile=0.2, n_samples=500) @@ -37,20 +42,28 @@ print("number of estimated clusters : %d" % n_clusters_) -# ############################################################################# +# %% # Plot result +# ----------- import matplotlib.pyplot as plt -from itertools import cycle plt.figure(1) plt.clf() -colors = cycle('bgrcmykbgrcmykbgrcmykbgrcmyk') +colors = ["#dede00", "#377eb8", "#f781bf"] +markers = ["x", "o", "^"] + for k, col in zip(range(n_clusters_), colors): my_members = labels == k cluster_center = cluster_centers[k] - plt.plot(X[my_members, 0], X[my_members, 1], col + '.') - plt.plot(cluster_center[0], cluster_center[1], 'o', markerfacecolor=col, - markeredgecolor='k', markersize=14) -plt.title('Estimated number of clusters: %d' % n_clusters_) + plt.plot(X[my_members, 0], X[my_members, 1], markers[k], color=col) + plt.plot( + cluster_center[0], + cluster_center[1], + markers[k], + markerfacecolor=col, + markeredgecolor="k", + markersize=14, + ) +plt.title("Estimated number of clusters: %d" % n_clusters_) plt.show() diff --git a/examples/cluster/plot_mini_batch_kmeans.py b/examples/cluster/plot_mini_batch_kmeans.py index e8f78556f80d2..d189ed0e02a5c 100644 --- a/examples/cluster/plot_mini_batch_kmeans.py +++ b/examples/cluster/plot_mini_batch_kmeans.py @@ -11,20 +11,22 @@ MiniBatchKMeans, and plot the results. We will also plot the points that are labelled differently between the two algorithms. + """ -print(__doc__) -import time +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Generate the data +# ----------------- +# +# We start by generating the blobs of data to be clustered. import numpy as np -import matplotlib.pyplot as plt -from sklearn.cluster import MiniBatchKMeans, KMeans -from sklearn.metrics.pairwise import pairwise_distances_argmin from sklearn.datasets import make_blobs -# ############################################################################# -# Generate sample data np.random.seed(0) batch_size = 45 @@ -32,84 +34,113 @@ n_clusters = len(centers) X, labels_true = make_blobs(n_samples=3000, centers=centers, cluster_std=0.7) -# ############################################################################# -# Compute clustering with Means +# %% +# Compute clustering with KMeans +# ------------------------------ + +import time + +from sklearn.cluster import KMeans -k_means = KMeans(init='k-means++', n_clusters=3, n_init=10) +k_means = KMeans(init="k-means++", n_clusters=3, n_init=10) t0 = time.time() k_means.fit(X) t_batch = time.time() - t0 -# ############################################################################# +# %% # Compute clustering with MiniBatchKMeans - -mbk = MiniBatchKMeans(init='k-means++', n_clusters=3, batch_size=batch_size, - n_init=10, max_no_improvement=10, verbose=0) +# --------------------------------------- + +from sklearn.cluster import MiniBatchKMeans + +mbk = MiniBatchKMeans( + init="k-means++", + n_clusters=3, + batch_size=batch_size, + n_init=10, + max_no_improvement=10, + verbose=0, +) t0 = time.time() mbk.fit(X) t_mini_batch = time.time() - t0 -# ############################################################################# -# Plot result - -fig = plt.figure(figsize=(8, 3)) -fig.subplots_adjust(left=0.02, right=0.98, bottom=0.05, top=0.9) -colors = ['#4EACC5', '#FF9C34', '#4E9A06'] - -# We want to have the same colors for the same cluster from the +# %% +# Establishing parity between clusters +# ------------------------------------ +# +# We want to have the same color for the same cluster from both the # MiniBatchKMeans and the KMeans algorithm. Let's pair the cluster centers per # closest one. + +from sklearn.metrics.pairwise import pairwise_distances_argmin + k_means_cluster_centers = k_means.cluster_centers_ -order = pairwise_distances_argmin(k_means.cluster_centers_, - mbk.cluster_centers_) +order = pairwise_distances_argmin(k_means.cluster_centers_, mbk.cluster_centers_) mbk_means_cluster_centers = mbk.cluster_centers_[order] k_means_labels = pairwise_distances_argmin(X, k_means_cluster_centers) mbk_means_labels = pairwise_distances_argmin(X, mbk_means_cluster_centers) +# %% +# Plotting the results +# -------------------- + +import matplotlib.pyplot as plt + +fig = plt.figure(figsize=(8, 3)) +fig.subplots_adjust(left=0.02, right=0.98, bottom=0.05, top=0.9) +colors = ["#4EACC5", "#FF9C34", "#4E9A06"] + # KMeans ax = fig.add_subplot(1, 3, 1) for k, col in zip(range(n_clusters), colors): my_members = k_means_labels == k cluster_center = k_means_cluster_centers[k] - ax.plot(X[my_members, 0], X[my_members, 1], 'w', - markerfacecolor=col, marker='.') - ax.plot(cluster_center[0], cluster_center[1], 'o', markerfacecolor=col, - markeredgecolor='k', markersize=6) -ax.set_title('KMeans') + ax.plot(X[my_members, 0], X[my_members, 1], "w", markerfacecolor=col, marker=".") + ax.plot( + cluster_center[0], + cluster_center[1], + "o", + markerfacecolor=col, + markeredgecolor="k", + markersize=6, + ) +ax.set_title("KMeans") ax.set_xticks(()) ax.set_yticks(()) -plt.text(-3.5, 1.8, 'train time: %.2fs\ninertia: %f' % ( - t_batch, k_means.inertia_)) +plt.text(-3.5, 1.8, "train time: %.2fs\ninertia: %f" % (t_batch, k_means.inertia_)) # MiniBatchKMeans ax = fig.add_subplot(1, 3, 2) for k, col in zip(range(n_clusters), colors): my_members = mbk_means_labels == k cluster_center = mbk_means_cluster_centers[k] - ax.plot(X[my_members, 0], X[my_members, 1], 'w', - markerfacecolor=col, marker='.') - ax.plot(cluster_center[0], cluster_center[1], 'o', markerfacecolor=col, - markeredgecolor='k', markersize=6) -ax.set_title('MiniBatchKMeans') + ax.plot(X[my_members, 0], X[my_members, 1], "w", markerfacecolor=col, marker=".") + ax.plot( + cluster_center[0], + cluster_center[1], + "o", + markerfacecolor=col, + markeredgecolor="k", + markersize=6, + ) +ax.set_title("MiniBatchKMeans") ax.set_xticks(()) ax.set_yticks(()) -plt.text(-3.5, 1.8, 'train time: %.2fs\ninertia: %f' % - (t_mini_batch, mbk.inertia_)) +plt.text(-3.5, 1.8, "train time: %.2fs\ninertia: %f" % (t_mini_batch, mbk.inertia_)) -# Initialise the different array to all False -different = (mbk_means_labels == 4) +# Initialize the different array to all False +different = mbk_means_labels == 4 ax = fig.add_subplot(1, 3, 3) for k in range(n_clusters): - different += ((k_means_labels == k) != (mbk_means_labels == k)) - -identic = np.logical_not(different) -ax.plot(X[identic, 0], X[identic, 1], 'w', - markerfacecolor='#bbbbbb', marker='.') -ax.plot(X[different, 0], X[different, 1], 'w', - markerfacecolor='m', marker='.') -ax.set_title('Difference') + different += (k_means_labels == k) != (mbk_means_labels == k) + +identical = np.logical_not(different) +ax.plot(X[identical, 0], X[identical, 1], "w", markerfacecolor="#bbbbbb", marker=".") +ax.plot(X[different, 0], X[different, 1], "w", markerfacecolor="m", marker=".") +ax.set_title("Difference") ax.set_xticks(()) ax.set_yticks(()) diff --git a/examples/cluster/plot_optics.py b/examples/cluster/plot_optics.py index 211fb84aede30..26218302542d9 100644 --- a/examples/cluster/plot_optics.py +++ b/examples/cluster/plot_optics.py @@ -8,47 +8,54 @@ Finds core samples of high density and expands clusters from them. This example uses data that is generated so that the clusters have different densities. + The :class:`~cluster.OPTICS` is first used with its Xi cluster detection method, and then setting specific thresholds on the reachability, which corresponds to :class:`~cluster.DBSCAN`. We can see that the different clusters of OPTICS's Xi method can be recovered with different choices of thresholds in DBSCAN. -""" -# Authors: Shane Grigsby -# Adrin Jalali -# License: BSD 3 clause +""" +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -from sklearn.cluster import OPTICS, cluster_optics_dbscan import matplotlib.gridspec as gridspec import matplotlib.pyplot as plt import numpy as np +from sklearn.cluster import OPTICS, cluster_optics_dbscan + # Generate sample data np.random.seed(0) n_points_per_cluster = 250 -C1 = [-5, -2] + .8 * np.random.randn(n_points_per_cluster, 2) -C2 = [4, -1] + .1 * np.random.randn(n_points_per_cluster, 2) -C3 = [1, -2] + .2 * np.random.randn(n_points_per_cluster, 2) -C4 = [-2, 3] + .3 * np.random.randn(n_points_per_cluster, 2) +C1 = [-5, -2] + 0.8 * np.random.randn(n_points_per_cluster, 2) +C2 = [4, -1] + 0.1 * np.random.randn(n_points_per_cluster, 2) +C3 = [1, -2] + 0.2 * np.random.randn(n_points_per_cluster, 2) +C4 = [-2, 3] + 0.3 * np.random.randn(n_points_per_cluster, 2) C5 = [3, -2] + 1.6 * np.random.randn(n_points_per_cluster, 2) C6 = [5, 6] + 2 * np.random.randn(n_points_per_cluster, 2) X = np.vstack((C1, C2, C3, C4, C5, C6)) -clust = OPTICS(min_samples=50, xi=.05, min_cluster_size=.05) +clust = OPTICS(min_samples=50, xi=0.05, min_cluster_size=0.05) # Run the fit clust.fit(X) -labels_050 = cluster_optics_dbscan(reachability=clust.reachability_, - core_distances=clust.core_distances_, - ordering=clust.ordering_, eps=0.5) -labels_200 = cluster_optics_dbscan(reachability=clust.reachability_, - core_distances=clust.core_distances_, - ordering=clust.ordering_, eps=2) +labels_050 = cluster_optics_dbscan( + reachability=clust.reachability_, + core_distances=clust.core_distances_, + ordering=clust.ordering_, + eps=0.5, +) +labels_200 = cluster_optics_dbscan( + reachability=clust.reachability_, + core_distances=clust.core_distances_, + ordering=clust.ordering_, + eps=2, +) space = np.arange(len(X)) reachability = clust.reachability_[clust.ordering_] @@ -62,40 +69,40 @@ ax4 = plt.subplot(G[1, 2]) # Reachability plot -colors = ['g.', 'r.', 'b.', 'y.', 'c.'] -for klass, color in zip(range(0, 5), colors): +colors = ["g.", "r.", "b.", "y.", "c."] +for klass, color in enumerate(colors): Xk = space[labels == klass] Rk = reachability[labels == klass] ax1.plot(Xk, Rk, color, alpha=0.3) -ax1.plot(space[labels == -1], reachability[labels == -1], 'k.', alpha=0.3) -ax1.plot(space, np.full_like(space, 2., dtype=float), 'k-', alpha=0.5) -ax1.plot(space, np.full_like(space, 0.5, dtype=float), 'k-.', alpha=0.5) -ax1.set_ylabel('Reachability (epsilon distance)') -ax1.set_title('Reachability Plot') +ax1.plot(space[labels == -1], reachability[labels == -1], "k.", alpha=0.3) +ax1.plot(space, np.full_like(space, 2.0, dtype=float), "k-", alpha=0.5) +ax1.plot(space, np.full_like(space, 0.5, dtype=float), "k-.", alpha=0.5) +ax1.set_ylabel("Reachability (epsilon distance)") +ax1.set_title("Reachability Plot") # OPTICS -colors = ['g.', 'r.', 'b.', 'y.', 'c.'] -for klass, color in zip(range(0, 5), colors): +colors = ["g.", "r.", "b.", "y.", "c."] +for klass, color in enumerate(colors): Xk = X[clust.labels_ == klass] ax2.plot(Xk[:, 0], Xk[:, 1], color, alpha=0.3) -ax2.plot(X[clust.labels_ == -1, 0], X[clust.labels_ == -1, 1], 'k+', alpha=0.1) -ax2.set_title('Automatic Clustering\nOPTICS') +ax2.plot(X[clust.labels_ == -1, 0], X[clust.labels_ == -1, 1], "k+", alpha=0.1) +ax2.set_title("Automatic Clustering\nOPTICS") # DBSCAN at 0.5 -colors = ['g', 'greenyellow', 'olive', 'r', 'b', 'c'] -for klass, color in zip(range(0, 6), colors): +colors = ["g.", "r.", "b.", "c."] +for klass, color in enumerate(colors): Xk = X[labels_050 == klass] - ax3.plot(Xk[:, 0], Xk[:, 1], color, alpha=0.3, marker='.') -ax3.plot(X[labels_050 == -1, 0], X[labels_050 == -1, 1], 'k+', alpha=0.1) -ax3.set_title('Clustering at 0.5 epsilon cut\nDBSCAN') + ax3.plot(Xk[:, 0], Xk[:, 1], color, alpha=0.3) +ax3.plot(X[labels_050 == -1, 0], X[labels_050 == -1, 1], "k+", alpha=0.1) +ax3.set_title("Clustering at 0.5 epsilon cut\nDBSCAN") # DBSCAN at 2. -colors = ['g.', 'm.', 'y.', 'c.'] -for klass, color in zip(range(0, 4), colors): +colors = ["g.", "m.", "y.", "c."] +for klass, color in enumerate(colors): Xk = X[labels_200 == klass] ax4.plot(Xk[:, 0], Xk[:, 1], color, alpha=0.3) -ax4.plot(X[labels_200 == -1, 0], X[labels_200 == -1, 1], 'k+', alpha=0.1) -ax4.set_title('Clustering at 2.0 epsilon cut\nDBSCAN') +ax4.plot(X[labels_200 == -1, 0], X[labels_200 == -1, 1], "k+", alpha=0.1) +ax4.set_title("Clustering at 2.0 epsilon cut\nDBSCAN") plt.tight_layout() plt.show() diff --git a/examples/cluster/plot_segmentation_toy.py b/examples/cluster/plot_segmentation_toy.py index a6980c5f271ef..5cd239bd39572 100644 --- a/examples/cluster/plot_segmentation_toy.py +++ b/examples/cluster/plot_segmentation_toy.py @@ -23,18 +23,16 @@ In addition, we use the mask of the objects to restrict the graph to the outline of the objects. In this example, we are interested in separating the objects one from the other, and not from the background. + """ -print(__doc__) -# Authors: Emmanuelle Gouillart -# Gael Varoquaux -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +# %% +# Generate the data +# ----------------- import numpy as np -import matplotlib.pyplot as plt - -from sklearn.feature_extraction import image -from sklearn.cluster import spectral_clustering l = 100 x, y = np.indices((l, l)) @@ -46,13 +44,14 @@ radius1, radius2, radius3, radius4 = 16, 14, 15, 14 -circle1 = (x - center1[0]) ** 2 + (y - center1[1]) ** 2 < radius1 ** 2 -circle2 = (x - center2[0]) ** 2 + (y - center2[1]) ** 2 < radius2 ** 2 -circle3 = (x - center3[0]) ** 2 + (y - center3[1]) ** 2 < radius3 ** 2 -circle4 = (x - center4[0]) ** 2 + (y - center4[1]) ** 2 < radius4 ** 2 +circle1 = (x - center1[0]) ** 2 + (y - center1[1]) ** 2 < radius1**2 +circle2 = (x - center2[0]) ** 2 + (y - center2[1]) ** 2 < radius2**2 +circle3 = (x - center3[0]) ** 2 + (y - center3[1]) ** 2 < radius3**2 +circle4 = (x - center4[0]) ** 2 + (y - center4[1]) ** 2 < radius4**2 -# ############################################################################# -# 4 circles +# %% +# Plotting four circles +# --------------------- img = circle1 + circle2 + circle3 + circle4 # We use a mask that limits to the foreground: the problem that we are @@ -63,25 +62,42 @@ img = img.astype(float) img += 1 + 0.2 * np.random.randn(*img.shape) +# %% # Convert the image into a graph with the value of the gradient on the # edges. +from sklearn.feature_extraction import image + graph = image.img_to_graph(img, mask=mask) -# Take a decreasing function of the gradient: we take it weakly -# dependent from the gradient the segmentation is close to a voronoi +# %% +# Take a decreasing function of the gradient resulting in a segmentation +# that is close to a Voronoi partition graph.data = np.exp(-graph.data / graph.data.std()) -# Force the solver to be arpack, since amg is numerically -# unstable on this example -labels = spectral_clustering(graph, n_clusters=4, eigen_solver='arpack') -label_im = np.full(mask.shape, -1.) +# %% +# Here we perform spectral clustering using the arpack solver since amg is +# numerically unstable on this example. We then plot the results. +import matplotlib.pyplot as plt + +from sklearn.cluster import spectral_clustering + +labels = spectral_clustering(graph, n_clusters=4, eigen_solver="arpack") +label_im = np.full(mask.shape, -1.0) label_im[mask] = labels -plt.matshow(img) -plt.matshow(label_im) +fig, axs = plt.subplots(nrows=1, ncols=2, figsize=(10, 5)) +axs[0].matshow(img) +axs[1].matshow(label_im) + +plt.show() + +# %% +# Plotting two circles +# -------------------- +# Here we repeat the above process but only consider the first two circles +# we generated. Note that this results in a cleaner separation between the +# circles as the region sizes are easier to balance in this case. -# ############################################################################# -# 2 circles img = circle1 + circle2 mask = img.astype(bool) img = img.astype(float) @@ -91,11 +107,12 @@ graph = image.img_to_graph(img, mask=mask) graph.data = np.exp(-graph.data / graph.data.std()) -labels = spectral_clustering(graph, n_clusters=2, eigen_solver='arpack') -label_im = np.full(mask.shape, -1.) +labels = spectral_clustering(graph, n_clusters=2, eigen_solver="arpack") +label_im = np.full(mask.shape, -1.0) label_im[mask] = labels -plt.matshow(img) -plt.matshow(label_im) +fig, axs = plt.subplots(nrows=1, ncols=2, figsize=(10, 5)) +axs[0].matshow(img) +axs[1].matshow(label_im) plt.show() diff --git a/examples/cluster/plot_ward_structured_vs_unstructured.py b/examples/cluster/plot_ward_structured_vs_unstructured.py index 671bdd2735280..5f8d416aaf51f 100644 --- a/examples/cluster/plot_ward_structured_vs_unstructured.py +++ b/examples/cluster/plot_ward_structured_vs_unstructured.py @@ -17,77 +17,112 @@ respect the structure of the swiss roll and extend across different folds of the manifolds. On the opposite, when opposing connectivity constraints, the clusters form a nice parcellation of the swiss roll. -""" -# Authors : Vincent Michel, 2010 -# Alexandre Gramfort, 2010 -# Gael Varoquaux, 2010 -# License: BSD 3 clause +""" -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import time as time + +# The following import is required +# for 3D projection to work with matplotlib < 3.2 +import mpl_toolkits.mplot3d # noqa: F401 import numpy as np -import matplotlib.pyplot as plt -import mpl_toolkits.mplot3d.axes3d as p3 -from sklearn.cluster import AgglomerativeClustering + +# %% +# Generate data +# ------------- +# +# We start by generating the Swiss Roll dataset. from sklearn.datasets import make_swiss_roll -# ############################################################################# -# Generate data (swiss roll dataset) n_samples = 1500 noise = 0.05 X, _ = make_swiss_roll(n_samples, noise=noise) # Make it thinner -X[:, 1] *= .5 +X[:, 1] *= 0.5 -# ############################################################################# +# %% # Compute clustering +# ------------------ +# +# We perform AgglomerativeClustering which comes under Hierarchical Clustering +# without any connectivity constraints. + +from sklearn.cluster import AgglomerativeClustering + print("Compute unstructured hierarchical clustering...") st = time.time() -ward = AgglomerativeClustering(n_clusters=6, linkage='ward').fit(X) +ward = AgglomerativeClustering(n_clusters=6, linkage="ward").fit(X) elapsed_time = time.time() - st label = ward.labels_ -print("Elapsed time: %.2fs" % elapsed_time) -print("Number of points: %i" % label.size) +print(f"Elapsed time: {elapsed_time:.2f}s") +print(f"Number of points: {label.size}") -# ############################################################################# +# %% # Plot result -fig = plt.figure() -ax = p3.Axes3D(fig) -ax.view_init(7, -80) +# ----------- +# Plotting the unstructured hierarchical clusters. + +import matplotlib.pyplot as plt + +fig1 = plt.figure() +ax1 = fig1.add_subplot(111, projection="3d", elev=7, azim=-80) +ax1.set_position([0, 0, 0.95, 1]) for l in np.unique(label): - ax.scatter(X[label == l, 0], X[label == l, 1], X[label == l, 2], - color=plt.cm.jet(float(l) / np.max(label + 1)), - s=20, edgecolor='k') -plt.title('Without connectivity constraints (time %.2fs)' % elapsed_time) + ax1.scatter( + X[label == l, 0], + X[label == l, 1], + X[label == l, 2], + color=plt.cm.jet(float(l) / np.max(label + 1)), + s=20, + edgecolor="k", + ) +_ = fig1.suptitle(f"Without connectivity constraints (time {elapsed_time:.2f}s)") +# %% +# We are defining k-Nearest Neighbors with 10 neighbors +# ----------------------------------------------------- -# ############################################################################# -# Define the structure A of the data. Here a 10 nearest neighbors from sklearn.neighbors import kneighbors_graph + connectivity = kneighbors_graph(X, n_neighbors=10, include_self=False) -# ############################################################################# +# %% # Compute clustering +# ------------------ +# +# We perform AgglomerativeClustering again with connectivity constraints. + print("Compute structured hierarchical clustering...") st = time.time() -ward = AgglomerativeClustering(n_clusters=6, connectivity=connectivity, - linkage='ward').fit(X) +ward = AgglomerativeClustering( + n_clusters=6, connectivity=connectivity, linkage="ward" +).fit(X) elapsed_time = time.time() - st label = ward.labels_ -print("Elapsed time: %.2fs" % elapsed_time) -print("Number of points: %i" % label.size) +print(f"Elapsed time: {elapsed_time:.2f}s") +print(f"Number of points: {label.size}") -# ############################################################################# +# %% # Plot result -fig = plt.figure() -ax = p3.Axes3D(fig) -ax.view_init(7, -80) +# ----------- +# +# Plotting the structured hierarchical clusters. + +fig2 = plt.figure() +ax2 = fig2.add_subplot(121, projection="3d", elev=7, azim=-80) +ax2.set_position([0, 0, 0.95, 1]) for l in np.unique(label): - ax.scatter(X[label == l, 0], X[label == l, 1], X[label == l, 2], - color=plt.cm.jet(float(l) / np.max(label + 1)), - s=20, edgecolor='k') -plt.title('With connectivity constraints (time %.2fs)' % elapsed_time) + ax2.scatter( + X[label == l, 0], + X[label == l, 1], + X[label == l, 2], + color=plt.cm.jet(float(l) / np.max(label + 1)), + s=20, + edgecolor="k", + ) +fig2.suptitle(f"With connectivity constraints (time {elapsed_time:.2f}s)") plt.show() diff --git a/examples/compose/plot_column_transformer.py b/examples/compose/plot_column_transformer.py index f6144ed2c491b..8f779d085614a 100644 --- a/examples/compose/plot_column_transformer.py +++ b/examples/compose/plot_column_transformer.py @@ -15,22 +15,22 @@ :class:`~sklearn.compose.ColumnTransformer` on a dataset containing different types of features. The choice of features is not particularly helpful, but serves to illustrate the technique. + """ -# Author: Matt Terry -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import numpy as np -from sklearn.preprocessing import FunctionTransformer +from sklearn.compose import ColumnTransformer from sklearn.datasets import fetch_20newsgroups -from sklearn.decomposition import TruncatedSVD +from sklearn.decomposition import PCA from sklearn.feature_extraction import DictVectorizer from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.metrics import classification_report from sklearn.pipeline import Pipeline -from sklearn.compose import ColumnTransformer +from sklearn.preprocessing import FunctionTransformer from sklearn.svm import LinearSVC ############################################################################## @@ -43,17 +43,21 @@ # a specific date. We will only use posts from 2 categories to speed up running # time. -categories = ['sci.med', 'sci.space'] -X_train, y_train = fetch_20newsgroups(random_state=1, - subset='train', - categories=categories, - remove=('footers', 'quotes'), - return_X_y=True) -X_test, y_test = fetch_20newsgroups(random_state=1, - subset='test', - categories=categories, - remove=('footers', 'quotes'), - return_X_y=True) +categories = ["sci.med", "sci.space"] +X_train, y_train = fetch_20newsgroups( + random_state=1, + subset="train", + categories=categories, + remove=("footers", "quotes"), + return_X_y=True, +) +X_test, y_test = fetch_20newsgroups( + random_state=1, + subset="test", + categories=categories, + remove=("footers", "quotes"), + return_X_y=True, +) ############################################################################## # Each feature comprises meta information about that post, such as the subject, @@ -79,16 +83,16 @@ def subject_body_extractor(posts): features = np.empty(shape=(len(posts), 2), dtype=object) for i, text in enumerate(posts): # temporary variable `_` stores '\n\n' - headers, _, body = text.partition('\n\n') + headers, _, body = text.partition("\n\n") # store body text in second column features[i, 1] = body - prefix = 'Subject:' - sub = '' + prefix = "Subject:" + sub = "" # save text after 'Subject:' in first column - for line in headers.split('\n'): + for line in headers.split("\n"): if line.startswith(prefix): - sub = line[len(prefix):] + sub = line[len(prefix) :] break features[i, 0] = sub @@ -103,9 +107,7 @@ def subject_body_extractor(posts): def text_stats(posts): - return [{'length': len(text), - 'num_sentences': text.count('.')} - for text in posts] + return [{"length": len(text), "num_sentences": text.count(".")} for text in posts] text_stats_transformer = FunctionTransformer(text_stats) @@ -121,35 +123,59 @@ def text_stats(posts): # ``ColumnTransformer``. We combine them, with weights, then train a # classifier on the combined set of features. -pipeline = Pipeline([ - # Extract subject & body - ('subjectbody', subject_body_transformer), - # Use ColumnTransformer to combine the subject and body features - ('union', ColumnTransformer( - [ - # bag-of-words for subject (col 0) - ('subject', TfidfVectorizer(min_df=50), 0), - # bag-of-words with decomposition for body (col 1) - ('body_bow', Pipeline([ - ('tfidf', TfidfVectorizer()), - ('best', TruncatedSVD(n_components=50)), - ]), 1), - # Pipeline for pulling text stats from post's body - ('body_stats', Pipeline([ - ('stats', text_stats_transformer), # returns a list of dicts - ('vect', DictVectorizer()), # list of dicts -> feature matrix - ]), 1), - ], - # weight above ColumnTransformer features - transformer_weights={ - 'subject': 0.8, - 'body_bow': 0.5, - 'body_stats': 1.0, - } - )), - # Use a SVC classifier on the combined features - ('svc', LinearSVC(dual=False)), -], verbose=True) +pipeline = Pipeline( + [ + # Extract subject & body + ("subjectbody", subject_body_transformer), + # Use ColumnTransformer to combine the subject and body features + ( + "union", + ColumnTransformer( + [ + # bag-of-words for subject (col 0) + ("subject", TfidfVectorizer(min_df=50), 0), + # bag-of-words with decomposition for body (col 1) + ( + "body_bow", + Pipeline( + [ + ("tfidf", TfidfVectorizer()), + ("best", PCA(n_components=50, svd_solver="arpack")), + ] + ), + 1, + ), + # Pipeline for pulling text stats from post's body + ( + "body_stats", + Pipeline( + [ + ( + "stats", + text_stats_transformer, + ), # returns a list of dicts + ( + "vect", + DictVectorizer(), + ), # list of dicts -> feature matrix + ] + ), + 1, + ), + ], + # weight above ColumnTransformer features + transformer_weights={ + "subject": 0.8, + "body_bow": 0.5, + "body_stats": 1.0, + }, + ), + ), + # Use a SVC classifier on the combined features + ("svc", LinearSVC(dual=False)), + ], + verbose=True, +) ############################################################################## # Finally, we fit our pipeline on the training data and use it to predict @@ -157,6 +183,4 @@ def text_stats(posts): pipeline.fit(X_train, y_train) y_pred = pipeline.predict(X_test) -print('Classification report:\n\n{}'.format( - classification_report(y_test, y_pred)) -) +print("Classification report:\n\n{}".format(classification_report(y_test, y_pred))) diff --git a/examples/compose/plot_column_transformer_mixed_types.py b/examples/compose/plot_column_transformer_mixed_types.py index a2937e041f186..91768e261f271 100644 --- a/examples/compose/plot_column_transformer_mixed_types.py +++ b/examples/compose/plot_column_transformer_mixed_types.py @@ -11,9 +11,10 @@ case of datasets that contain heterogeneous data types, since we may want to scale the numeric features and one-hot encode the categorical ones. -In this example, the numeric data is standard-scaled after mean-imputation, -while the categorical data is one-hot encoded after imputing missing values -with a new category (``'missing'``). +In this example, the numeric data is standard-scaled after mean-imputation. The +categorical data is one-hot encoded via ``OneHotEncoder``, which +creates a new category for missing values. We further reduce the dimensionality +by selecting categories using a chi-squared test. In addition, we show two different ways to dispatch the columns to the particular pre-processor: by column names and by column data types. @@ -21,24 +22,27 @@ Finally, the preprocessing pipeline is integrated in a full prediction pipeline using :class:`~pipeline.Pipeline`, together with a simple classification model. + """ -# Author: Pedro Morales -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +# %% import numpy as np from sklearn.compose import ColumnTransformer from sklearn.datasets import fetch_openml -from sklearn.pipeline import Pipeline +from sklearn.feature_selection import SelectPercentile, chi2 from sklearn.impute import SimpleImputer -from sklearn.preprocessing import StandardScaler, OneHotEncoder from sklearn.linear_model import LogisticRegression -from sklearn.model_selection import train_test_split, GridSearchCV +from sklearn.model_selection import RandomizedSearchCV, train_test_split +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import OneHotEncoder, StandardScaler np.random.seed(0) +# %% # Load data from https://www.openml.org/d/40945 X, y = fetch_openml("titanic", version=1, as_frame=True, return_X_y=True) @@ -48,7 +52,7 @@ # %% # Use ``ColumnTransformer`` by selecting column by names -############################################################################### +# # We will train our classifier with the following features: # # Numeric Features: @@ -66,43 +70,47 @@ # Note that ``pclass`` could either be treated as a categorical or numeric # feature. -numeric_features = ['age', 'fare'] -numeric_transformer = Pipeline(steps=[ - ('imputer', SimpleImputer(strategy='median')), - ('scaler', StandardScaler())]) - -categorical_features = ['embarked', 'sex', 'pclass'] -categorical_transformer = OneHotEncoder(handle_unknown='ignore') - +numeric_features = ["age", "fare"] +numeric_transformer = Pipeline( + steps=[("imputer", SimpleImputer(strategy="median")), ("scaler", StandardScaler())] +) + +categorical_features = ["embarked", "sex", "pclass"] +categorical_transformer = Pipeline( + steps=[ + ("encoder", OneHotEncoder(handle_unknown="ignore")), + ("selector", SelectPercentile(chi2, percentile=50)), + ] +) preprocessor = ColumnTransformer( transformers=[ - ('num', numeric_transformer, numeric_features), - ('cat', categorical_transformer, categorical_features)]) + ("num", numeric_transformer, numeric_features), + ("cat", categorical_transformer, categorical_features), + ] +) +# %% # Append classifier to preprocessing pipeline. # Now we have a full prediction pipeline. -clf = Pipeline(steps=[('preprocessor', preprocessor), - ('classifier', LogisticRegression())]) +clf = Pipeline( + steps=[("preprocessor", preprocessor), ("classifier", LogisticRegression())] +) -X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, - random_state=0) +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0) clf.fit(X_train, y_train) print("model score: %.3f" % clf.score(X_test, y_test)) # %% -# HTML representation of ``Pipeline`` -############################################################################### +# HTML representation of ``Pipeline`` (display diagram) +# # When the ``Pipeline`` is printed out in a jupyter notebook an HTML -# representation of the estimator is displayed as follows: -from sklearn import set_config - -set_config(display='diagram') +# representation of the estimator is displayed: clf # %% # Use ``ColumnTransformer`` by selecting column by data types -############################################################################### +# # When dealing with a cleaned dataset, the preprocessing can be automatic by # using the data types of the column to decide whether to treat a column as a # numerical or categorical feature. @@ -110,7 +118,7 @@ # First, let's only select a subset of columns to simplify our # example. -subset_feature = ['embarked', 'sex', 'pclass', 'age', 'fare'] +subset_feature = ["embarked", "sex", "pclass", "age", "fare"] X_train, X_test = X_train[subset_feature], X_test[subset_feature] # %% @@ -134,21 +142,25 @@ from sklearn.compose import make_column_selector as selector -preprocessor = ColumnTransformer(transformers=[ - ('num', numeric_transformer, selector(dtype_exclude="category")), - ('cat', categorical_transformer, selector(dtype_include="category")) -]) -clf = Pipeline(steps=[('preprocessor', preprocessor), - ('classifier', LogisticRegression())]) +preprocessor = ColumnTransformer( + transformers=[ + ("num", numeric_transformer, selector(dtype_exclude="category")), + ("cat", categorical_transformer, selector(dtype_include="category")), + ] +) +clf = Pipeline( + steps=[("preprocessor", preprocessor), ("classifier", LogisticRegression())] +) clf.fit(X_train, y_train) print("model score: %.3f" % clf.score(X_test, y_test)) +clf # %% # The resulting score is not exactly the same as the one from the previous -# pipeline becase the dtype-based selector treats the ``pclass`` columns as -# a numeric features instead of a categorical feature as previously: +# pipeline because the dtype-based selector treats the ``pclass`` column as +# a numeric feature instead of a categorical feature as previously: selector(dtype_exclude="category")(X_train) @@ -158,50 +170,62 @@ # %% # Using the prediction pipeline in a grid search -############################################################################## +# # Grid search can also be performed on the different preprocessing steps # defined in the ``ColumnTransformer`` object, together with the classifier's # hyperparameters as part of the ``Pipeline``. # We will search for both the imputer strategy of the numeric preprocessing # and the regularization parameter of the logistic regression using -# :class:`~sklearn.model_selection.GridSearchCV`. +# :class:`~sklearn.model_selection.RandomizedSearchCV`. This +# hyperparameter search randomly selects a fixed number of parameter +# settings configured by `n_iter`. Alternatively, one can use +# :class:`~sklearn.model_selection.GridSearchCV` but the cartesian product of +# the parameter space will be evaluated. param_grid = { - 'preprocessor__num__imputer__strategy': ['mean', 'median'], - 'classifier__C': [0.1, 1.0, 10, 100], + "preprocessor__num__imputer__strategy": ["mean", "median"], + "preprocessor__cat__selector__percentile": [10, 30, 50, 70], + "classifier__C": [0.1, 1.0, 10, 100], } -grid_search = GridSearchCV(clf, param_grid, cv=10) -grid_search +search_cv = RandomizedSearchCV(clf, param_grid, n_iter=10, random_state=0) +search_cv # %% # Calling 'fit' triggers the cross-validated search for the best # hyper-parameters combination: # -grid_search.fit(X_train, y_train) +search_cv.fit(X_train, y_train) -print(f"Best params:") -print(grid_search.best_params_) +print("Best params:") +print(search_cv.best_params_) # %% # The internal cross-validation scores obtained by those parameters is: -print(f"Internal CV score: {grid_search.best_score_:.3f}") +print(f"Internal CV score: {search_cv.best_score_:.3f}") # %% # We can also introspect the top grid search results as a pandas dataframe: import pandas as pd -cv_results = pd.DataFrame(grid_search.cv_results_) +cv_results = pd.DataFrame(search_cv.cv_results_) cv_results = cv_results.sort_values("mean_test_score", ascending=False) -cv_results[["mean_test_score", "std_test_score", - "param_preprocessor__num__imputer__strategy", - "param_classifier__C" - ]].head(5) +cv_results[ + [ + "mean_test_score", + "std_test_score", + "param_preprocessor__num__imputer__strategy", + "param_preprocessor__cat__selector__percentile", + "param_classifier__C", + ] +].head(5) # %% # The best hyper-parameters have be used to re-fit a final model on the full # training set. We can evaluate that final model on held out test data that was -# not used for hyparameter tuning. +# not used for hyperparameter tuning. # -print(("best logistic regression from grid search: %.3f" - % grid_search.score(X_test, y_test))) +print( + "accuracy of the best model from randomized search: " + f"{search_cv.score(X_test, y_test):.3f}" +) diff --git a/examples/compose/plot_compare_reduction.py b/examples/compose/plot_compare_reduction.py old mode 100755 new mode 100644 index f7425b52a9b64..da6cfbb651d44 --- a/examples/compose/plot_compare_reduction.py +++ b/examples/compose/plot_compare_reduction.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- """ ================================================================= Selecting dimensionality reduction with Pipeline and GridSearchCV @@ -20,78 +18,82 @@ Note that the use of ``memory`` to enable caching becomes interesting when the fitting of a transformer is costly. -# %% -Illustration of ``Pipeline`` and ``GridSearchCV`` -############################################################################### - -This section illustrates the use of a ``Pipeline`` with ``GridSearchCV`` """ -# Authors: Robert McGibbon, Joel Nothman, Guillaume Lemaitre +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +# %% +# Illustration of ``Pipeline`` and ``GridSearchCV`` +############################################################################### -import numpy as np import matplotlib.pyplot as plt +import numpy as np + from sklearn.datasets import load_digits +from sklearn.decomposition import NMF, PCA +from sklearn.feature_selection import SelectKBest, mutual_info_classif from sklearn.model_selection import GridSearchCV from sklearn.pipeline import Pipeline +from sklearn.preprocessing import MinMaxScaler from sklearn.svm import LinearSVC -from sklearn.decomposition import PCA, NMF -from sklearn.feature_selection import SelectKBest, chi2 -print(__doc__) +X, y = load_digits(return_X_y=True) -pipe = Pipeline([ - # the reduce_dim stage is populated by the param_grid - ('reduce_dim', 'passthrough'), - ('classify', LinearSVC(dual=False, max_iter=10000)) -]) +pipe = Pipeline( + [ + ("scaling", MinMaxScaler()), + # the reduce_dim stage is populated by the param_grid + ("reduce_dim", "passthrough"), + ("classify", LinearSVC(dual=False, max_iter=10000)), + ] +) N_FEATURES_OPTIONS = [2, 4, 8] C_OPTIONS = [1, 10, 100, 1000] param_grid = [ { - 'reduce_dim': [PCA(iterated_power=7), NMF()], - 'reduce_dim__n_components': N_FEATURES_OPTIONS, - 'classify__C': C_OPTIONS + "reduce_dim": [PCA(iterated_power=7), NMF(max_iter=1_000)], + "reduce_dim__n_components": N_FEATURES_OPTIONS, + "classify__C": C_OPTIONS, }, { - 'reduce_dim': [SelectKBest(chi2)], - 'reduce_dim__k': N_FEATURES_OPTIONS, - 'classify__C': C_OPTIONS + "reduce_dim": [SelectKBest(mutual_info_classif)], + "reduce_dim__k": N_FEATURES_OPTIONS, + "classify__C": C_OPTIONS, }, ] -reducer_labels = ['PCA', 'NMF', 'KBest(chi2)'] +reducer_labels = ["PCA", "NMF", "KBest(mutual_info_classif)"] grid = GridSearchCV(pipe, n_jobs=1, param_grid=param_grid) -X, y = load_digits(return_X_y=True) grid.fit(X, y) -mean_scores = np.array(grid.cv_results_['mean_test_score']) +# %% +import pandas as pd + +mean_scores = np.array(grid.cv_results_["mean_test_score"]) # scores are in the order of param_grid iteration, which is alphabetical mean_scores = mean_scores.reshape(len(C_OPTIONS), -1, len(N_FEATURES_OPTIONS)) # select score for best C mean_scores = mean_scores.max(axis=0) -bar_offsets = (np.arange(len(N_FEATURES_OPTIONS)) * - (len(reducer_labels) + 1) + .5) - -plt.figure() -COLORS = 'bgrcmyk' -for i, (label, reducer_scores) in enumerate(zip(reducer_labels, mean_scores)): - plt.bar(bar_offsets + i, reducer_scores, label=label, color=COLORS[i]) - -plt.title("Comparing feature reduction techniques") -plt.xlabel('Reduced number of features') -plt.xticks(bar_offsets + len(reducer_labels) / 2, N_FEATURES_OPTIONS) -plt.ylabel('Digit classification accuracy') -plt.ylim((0, 1)) -plt.legend(loc='upper left') +# create a dataframe to ease plotting +mean_scores = pd.DataFrame( + mean_scores.T, index=N_FEATURES_OPTIONS, columns=reducer_labels +) + +ax = mean_scores.plot.bar() +ax.set_title("Comparing feature reduction techniques") +ax.set_xlabel("Reduced number of features") +ax.set_ylabel("Digit classification accuracy") +ax.set_ylim((0, 1)) +ax.legend(loc="upper left") plt.show() # %% # Caching transformers within a ``Pipeline`` -############################################################################### +# ########################################## +# # It is sometimes worthwhile storing the state of a specific transformer # since it could be used again. Using a pipeline in ``GridSearchCV`` triggers # such situations. Therefore, we use the argument ``memory`` to enable caching. @@ -102,15 +104,17 @@ # cache. Hence, use the ``memory`` constructor parameter when the fitting # of a transformer is costly. -from joblib import Memory from shutil import rmtree +from joblib import Memory + # Create a temporary folder to store the transformers of the pipeline -location = 'cachedir' +location = "cachedir" memory = Memory(location=location, verbose=10) -cached_pipe = Pipeline([('reduce_dim', PCA()), - ('classify', LinearSVC(dual=False, max_iter=10000))], - memory=memory) +cached_pipe = Pipeline( + [("reduce_dim", PCA()), ("classify", LinearSVC(dual=False, max_iter=10000))], + memory=memory, +) # This time, a cached pipeline will be used within the grid search diff --git a/examples/compose/plot_digits_pipe.py b/examples/compose/plot_digits_pipe.py index 7c7ddf938a14f..8a202bb5bd74c 100644 --- a/examples/compose/plot_digits_pipe.py +++ b/examples/compose/plot_digits_pipe.py @@ -1,6 +1,3 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - """ ========================================================= Pipelining: chaining a PCA and a logistic regression @@ -12,40 +9,38 @@ We use a GridSearchCV to set the dimensionality of the PCA """ -print(__doc__) - - -# Code source: GaÃĢl Varoquaux -# Modified for documentation by Jaques Grobler -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt -import pandas as pd +import numpy as np +import polars as pl from sklearn import datasets from sklearn.decomposition import PCA from sklearn.linear_model import LogisticRegression -from sklearn.pipeline import Pipeline from sklearn.model_selection import GridSearchCV - +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import StandardScaler # Define a pipeline to search for the best combination of PCA truncation # and classifier regularization. pca = PCA() +# Define a Standard Scaler to normalize inputs +scaler = StandardScaler() + # set the tolerance to a large value to make the example faster logistic = LogisticRegression(max_iter=10000, tol=0.1) -pipe = Pipeline(steps=[('pca', pca), ('logistic', logistic)]) +pipe = Pipeline(steps=[("scaler", scaler), ("pca", pca), ("logistic", logistic)]) X_digits, y_digits = datasets.load_digits(return_X_y=True) - -# Parameters of pipelines can be set using ‘__’ separated parameter names: +# Parameters of pipelines can be set using '__' separated parameter names: param_grid = { - 'pca__n_components': [5, 15, 30, 45, 64], - 'logistic__C': np.logspace(-4, 4, 4), + "pca__n_components": [5, 15, 30, 45, 60], + "logistic__C": np.logspace(-4, 4, 4), } -search = GridSearchCV(pipe, param_grid, n_jobs=-1) +search = GridSearchCV(pipe, param_grid, n_jobs=2) search.fit(X_digits, y_digits) print("Best parameter (CV score=%0.3f):" % search.best_score_) print(search.best_params_) @@ -54,24 +49,35 @@ pca.fit(X_digits) fig, (ax0, ax1) = plt.subplots(nrows=2, sharex=True, figsize=(6, 6)) -ax0.plot(np.arange(1, pca.n_components_ + 1), - pca.explained_variance_ratio_, '+', linewidth=2) -ax0.set_ylabel('PCA explained variance ratio') - -ax0.axvline(search.best_estimator_.named_steps['pca'].n_components, - linestyle=':', label='n_components chosen') +ax0.plot( + np.arange(1, pca.n_components_ + 1), pca.explained_variance_ratio_, "+", linewidth=2 +) +ax0.set_ylabel("PCA explained variance ratio") + +ax0.axvline( + search.best_estimator_.named_steps["pca"].n_components, + linestyle=":", + label="n_components chosen", +) ax0.legend(prop=dict(size=12)) # For each number of components, find the best classifier results -results = pd.DataFrame(search.cv_results_) -components_col = 'param_pca__n_components' -best_clfs = results.groupby(components_col).apply( - lambda g: g.nlargest(1, 'mean_test_score')) - -best_clfs.plot(x=components_col, y='mean_test_score', yerr='std_test_score', - legend=False, ax=ax1) -ax1.set_ylabel('Classification accuracy (val)') -ax1.set_xlabel('n_components') +components_col = "param_pca__n_components" +is_max_test_score = pl.col("mean_test_score") == pl.col("mean_test_score").max() +best_clfs = ( + pl.LazyFrame(search.cv_results_) + .filter(is_max_test_score.over(components_col)) + .unique(components_col) + .sort(components_col) + .collect() +) +ax1.errorbar( + best_clfs[components_col], + best_clfs["mean_test_score"], + yerr=best_clfs["std_test_score"], +) +ax1.set_ylabel("Classification accuracy (val)") +ax1.set_xlabel("n_components") plt.xlim(-1, 70) diff --git a/examples/compose/plot_feature_union.py b/examples/compose/plot_feature_union.py index c76548a2d376f..ef04a50846a48 100644 --- a/examples/compose/plot_feature_union.py +++ b/examples/compose/plot_feature_union.py @@ -13,18 +13,18 @@ The combination used in this example is not particularly helpful on this dataset and is only used to illustrate the usage of FeatureUnion. + """ -# Author: Andreas Mueller -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -from sklearn.pipeline import Pipeline, FeatureUnion -from sklearn.model_selection import GridSearchCV -from sklearn.svm import SVC from sklearn.datasets import load_iris from sklearn.decomposition import PCA from sklearn.feature_selection import SelectKBest +from sklearn.model_selection import GridSearchCV +from sklearn.pipeline import FeatureUnion, Pipeline +from sklearn.svm import SVC iris = load_iris() @@ -50,9 +50,11 @@ pipeline = Pipeline([("features", combined_features), ("svm", svm)]) -param_grid = dict(features__pca__n_components=[1, 2, 3], - features__univ_select__k=[1, 2], - svm__C=[0.1, 1, 10]) +param_grid = dict( + features__pca__n_components=[1, 2, 3], + features__univ_select__k=[1, 2], + svm__C=[0.1, 1, 10], +) grid_search = GridSearchCV(pipeline, param_grid=param_grid, verbose=10) grid_search.fit(X, y) diff --git a/examples/compose/plot_transformed_target.py b/examples/compose/plot_transformed_target.py old mode 100755 new mode 100644 index eac10a926ee46..e4d0e1e108fb6 --- a/examples/compose/plot_transformed_target.py +++ b/examples/compose/plot_transformed_target.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - """ ====================================================== Effect of transforming the targets in regression model @@ -11,68 +8,59 @@ to illustrate the benefit of transforming the targets before learning a linear regression model. The first example uses synthetic data while the second example is based on the Ames housing data set. -""" -# Author: Guillaume Lemaitre -# License: BSD 3 clause +""" -import numpy as np -import matplotlib -import matplotlib.pyplot as plt +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -from sklearn.datasets import make_regression -from sklearn.model_selection import train_test_split -from sklearn.linear_model import RidgeCV -from sklearn.compose import TransformedTargetRegressor -from sklearn.metrics import median_absolute_error, r2_score -from sklearn.utils.fixes import parse_version +print(__doc__) # %% # Synthetic example -############################################################################## - -# `normed` is being deprecated in favor of `density` in histograms -if parse_version(matplotlib.__version__) >= parse_version('2.1'): - density_param = {'density': True} -else: - density_param = {'normed': True} - -# %% +# ################# +# # A synthetic random regression dataset is generated. The targets ``y`` are # modified by: # -# 1. translating all targets such that all entries are -# non-negative (by adding the absolute value of the lowest ``y``) and -# 2. applying an exponential function to obtain non-linear -# targets which cannot be fitted using a simple linear model. +# 1. translating all targets such that all entries are +# non-negative (by adding the absolute value of the lowest ``y``) and +# 2. applying an exponential function to obtain non-linear +# targets which cannot be fitted using a simple linear model. # # Therefore, a logarithmic (`np.log1p`) and an exponential function # (`np.expm1`) will be used to transform the targets before training a linear # regression model and using it for prediction. +import numpy as np -X, y = make_regression(n_samples=10000, noise=100, random_state=0) +from sklearn.datasets import make_regression + +X, y = make_regression(n_samples=10_000, noise=100, random_state=0) y = np.expm1((y + abs(y.min())) / 200) y_trans = np.log1p(y) # %% # Below we plot the probability density functions of the target # before and after applying the logarithmic functions. +import matplotlib.pyplot as plt + +from sklearn.model_selection import train_test_split f, (ax0, ax1) = plt.subplots(1, 2) -ax0.hist(y, bins=100, **density_param) +ax0.hist(y, bins=100, density=True) ax0.set_xlim([0, 2000]) -ax0.set_ylabel('Probability') -ax0.set_xlabel('Target') -ax0.set_title('Target distribution') +ax0.set_ylabel("Probability") +ax0.set_xlabel("Target") +ax0.set_title("Target distribution") -ax1.hist(y_trans, bins=100, **density_param) -ax1.set_ylabel('Probability') -ax1.set_xlabel('Target') -ax1.set_title('Transformed target distribution') +ax1.hist(y_trans, bins=100, density=True) +ax1.set_ylabel("Probability") +ax1.set_xlabel("Target") +ax1.set_title("Transformed target distribution") -f.suptitle("Synthetic data", y=0.06, x=0.53) -f.tight_layout(rect=[0.05, 0.05, 0.95, 0.95]) +f.suptitle("Synthetic data", y=1.05) +plt.tight_layout() X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0) @@ -81,143 +69,165 @@ # non-linearity, the model trained will not be precise during # prediction. Subsequently, a logarithmic function is used to linearize the # targets, allowing better prediction even with a similar linear model as -# reported by the median absolute error (MAE). +# reported by the median absolute error (MedAE). +from sklearn.metrics import median_absolute_error, r2_score + + +def compute_score(y_true, y_pred): + return { + "R2": f"{r2_score(y_true, y_pred):.3f}", + "MedAE": f"{median_absolute_error(y_true, y_pred):.3f}", + } + + +# %% +from sklearn.compose import TransformedTargetRegressor +from sklearn.linear_model import RidgeCV +from sklearn.metrics import PredictionErrorDisplay f, (ax0, ax1) = plt.subplots(1, 2, sharey=True) -# Use linear model -regr = RidgeCV() -regr.fit(X_train, y_train) -y_pred = regr.predict(X_test) -# Plot results -ax0.scatter(y_test, y_pred) -ax0.plot([0, 2000], [0, 2000], '--k') -ax0.set_ylabel('Target predicted') -ax0.set_xlabel('True Target') -ax0.set_title('Ridge regression \n without target transformation') -ax0.text(100, 1750, r'$R^2$=%.2f, MAE=%.2f' % ( - r2_score(y_test, y_pred), median_absolute_error(y_test, y_pred))) -ax0.set_xlim([0, 2000]) -ax0.set_ylim([0, 2000]) -# Transform targets and use same linear model -regr_trans = TransformedTargetRegressor(regressor=RidgeCV(), - func=np.log1p, - inverse_func=np.expm1) -regr_trans.fit(X_train, y_train) -y_pred = regr_trans.predict(X_test) - -ax1.scatter(y_test, y_pred) -ax1.plot([0, 2000], [0, 2000], '--k') -ax1.set_ylabel('Target predicted') -ax1.set_xlabel('True Target') -ax1.set_title('Ridge regression \n with target transformation') -ax1.text(100, 1750, r'$R^2$=%.2f, MAE=%.2f' % ( - r2_score(y_test, y_pred), median_absolute_error(y_test, y_pred))) -ax1.set_xlim([0, 2000]) -ax1.set_ylim([0, 2000]) - -f.suptitle("Synthetic data", y=0.035) -f.tight_layout(rect=[0.05, 0.05, 0.95, 0.95]) + +ridge_cv = RidgeCV().fit(X_train, y_train) +y_pred_ridge = ridge_cv.predict(X_test) + +ridge_cv_with_trans_target = TransformedTargetRegressor( + regressor=RidgeCV(), func=np.log1p, inverse_func=np.expm1 +).fit(X_train, y_train) +y_pred_ridge_with_trans_target = ridge_cv_with_trans_target.predict(X_test) + +PredictionErrorDisplay.from_predictions( + y_test, + y_pred_ridge, + kind="actual_vs_predicted", + ax=ax0, + scatter_kwargs={"alpha": 0.5}, +) +PredictionErrorDisplay.from_predictions( + y_test, + y_pred_ridge_with_trans_target, + kind="actual_vs_predicted", + ax=ax1, + scatter_kwargs={"alpha": 0.5}, +) + +# Add the score in the legend of each axis +for ax, y_pred in zip([ax0, ax1], [y_pred_ridge, y_pred_ridge_with_trans_target]): + for name, score in compute_score(y_test, y_pred).items(): + ax.plot([], [], " ", label=f"{name}={score}") + ax.legend(loc="upper left") + +ax0.set_title("Ridge regression \n without target transformation") +ax1.set_title("Ridge regression \n with target transformation") +f.suptitle("Synthetic data", y=1.05) +plt.tight_layout() # %% # Real-world data set -############################################################################### +# ################### # # In a similar manner, the Ames housing data set is used to show the impact # of transforming the targets before learning a model. In this example, the # target to be predicted is the selling price of each house. - from sklearn.datasets import fetch_openml -from sklearn.preprocessing import QuantileTransformer, quantile_transform +from sklearn.preprocessing import quantile_transform ames = fetch_openml(name="house_prices", as_frame=True) # Keep only numeric columns X = ames.data.select_dtypes(np.number) # Remove columns with NaN or Inf values -X = X.drop(columns=['LotFrontage', 'GarageYrBlt', 'MasVnrArea']) -y = ames.target -y_trans = quantile_transform(y.to_frame(), - n_quantiles=900, - output_distribution='normal', - copy=True).squeeze() +X = X.drop(columns=["LotFrontage", "GarageYrBlt", "MasVnrArea"]) +# Let the price be in k$ +y = ames.target / 1000 +y_trans = quantile_transform( + y.to_frame(), n_quantiles=900, output_distribution="normal", copy=True +).squeeze() + # %% # A :class:`~sklearn.preprocessing.QuantileTransformer` is used to normalize # the target distribution before applying a # :class:`~sklearn.linear_model.RidgeCV` model. - f, (ax0, ax1) = plt.subplots(1, 2) -ax0.hist(y, bins=100, **density_param) -ax0.set_ylabel('Probability') -ax0.set_xlabel('Target') -ax0.text(s='Target distribution', x=1.2e5, y=9.8e-6, fontsize=12) -ax0.ticklabel_format(axis="both", style="sci", scilimits=(0, 0)) +ax0.hist(y, bins=100, density=True) +ax0.set_ylabel("Probability") +ax0.set_xlabel("Target") +ax0.set_title("Target distribution") -ax1.hist(y_trans, bins=100, **density_param) -ax1.set_ylabel('Probability') -ax1.set_xlabel('Target') -ax1.text(s='Transformed target distribution', x=-6.8, y=0.479, fontsize=12) +ax1.hist(y_trans, bins=100, density=True) +ax1.set_ylabel("Probability") +ax1.set_xlabel("Target") +ax1.set_title("Transformed target distribution") -f.suptitle("Ames housing data: selling price", y=0.04) -f.tight_layout(rect=[0.05, 0.05, 0.95, 0.95]) +f.suptitle("Ames housing data: selling price", y=1.05) +plt.tight_layout() +# %% X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1) # %% # The effect of the transformer is weaker than on the synthetic data. However, # the transformation results in an increase in :math:`R^2` and large decrease -# of the MAE. The residual plot (predicted target - true target vs predicted +# of the MedAE. The residual plot (predicted target - true target vs predicted # target) without target transformation takes on a curved, 'reverse smile' # shape due to residual values that vary depending on the value of predicted # target. With target transformation, the shape is more linear indicating # better model fit. +from sklearn.preprocessing import QuantileTransformer -f, (ax0, ax1) = plt.subplots(2, 2, sharey='row', figsize=(6.5, 8)) - -regr = RidgeCV() -regr.fit(X_train, y_train) -y_pred = regr.predict(X_test) - -ax0[0].scatter(y_pred, y_test, s=8) -ax0[0].plot([0, 7e5], [0, 7e5], '--k') -ax0[0].set_ylabel('True target') -ax0[0].set_xlabel('Predicted target') -ax0[0].text(s='Ridge regression \n without target transformation', x=-5e4, - y=8e5, fontsize=12, multialignment='center') -ax0[0].text(3e4, 64e4, r'$R^2$=%.2f, MAE=%.2f' % ( - r2_score(y_test, y_pred), median_absolute_error(y_test, y_pred))) -ax0[0].set_xlim([0, 7e5]) -ax0[0].set_ylim([0, 7e5]) -ax0[0].ticklabel_format(axis="both", style="sci", scilimits=(0, 0)) - -ax1[0].scatter(y_pred, (y_pred - y_test), s=8) -ax1[0].set_ylabel('Residual') -ax1[0].set_xlabel('Predicted target') -ax1[0].ticklabel_format(axis="both", style="sci", scilimits=(0, 0)) - -regr_trans = TransformedTargetRegressor( - regressor=RidgeCV(), - transformer=QuantileTransformer(n_quantiles=900, - output_distribution='normal')) -regr_trans.fit(X_train, y_train) -y_pred = regr_trans.predict(X_test) - -ax0[1].scatter(y_pred, y_test, s=8) -ax0[1].plot([0, 7e5], [0, 7e5], '--k') -ax0[1].set_ylabel('True target') -ax0[1].set_xlabel('Predicted target') -ax0[1].text(s='Ridge regression \n with target transformation', x=-5e4, - y=8e5, fontsize=12, multialignment='center') -ax0[1].text(3e4, 64e4, r'$R^2$=%.2f, MAE=%.2f' % ( - r2_score(y_test, y_pred), median_absolute_error(y_test, y_pred))) -ax0[1].set_xlim([0, 7e5]) -ax0[1].set_ylim([0, 7e5]) -ax0[1].ticklabel_format(axis="both", style="sci", scilimits=(0, 0)) - -ax1[1].scatter(y_pred, (y_pred - y_test), s=8) -ax1[1].set_ylabel('Residual') -ax1[1].set_xlabel('Predicted target') -ax1[1].ticklabel_format(axis="both", style="sci", scilimits=(0, 0)) - -f.suptitle("Ames housing data: selling price", y=0.035) +f, (ax0, ax1) = plt.subplots(2, 2, sharey="row", figsize=(6.5, 8)) +ridge_cv = RidgeCV().fit(X_train, y_train) +y_pred_ridge = ridge_cv.predict(X_test) + +ridge_cv_with_trans_target = TransformedTargetRegressor( + regressor=RidgeCV(), + transformer=QuantileTransformer(n_quantiles=900, output_distribution="normal"), +).fit(X_train, y_train) +y_pred_ridge_with_trans_target = ridge_cv_with_trans_target.predict(X_test) + +# plot the actual vs predicted values +PredictionErrorDisplay.from_predictions( + y_test, + y_pred_ridge, + kind="actual_vs_predicted", + ax=ax0[0], + scatter_kwargs={"alpha": 0.5}, +) +PredictionErrorDisplay.from_predictions( + y_test, + y_pred_ridge_with_trans_target, + kind="actual_vs_predicted", + ax=ax0[1], + scatter_kwargs={"alpha": 0.5}, +) + +# Add the score in the legend of each axis +for ax, y_pred in zip([ax0[0], ax0[1]], [y_pred_ridge, y_pred_ridge_with_trans_target]): + for name, score in compute_score(y_test, y_pred).items(): + ax.plot([], [], " ", label=f"{name}={score}") + ax.legend(loc="upper left") + +ax0[0].set_title("Ridge regression \n without target transformation") +ax0[1].set_title("Ridge regression \n with target transformation") + +# plot the residuals vs the predicted values +PredictionErrorDisplay.from_predictions( + y_test, + y_pred_ridge, + kind="residual_vs_predicted", + ax=ax1[0], + scatter_kwargs={"alpha": 0.5}, +) +PredictionErrorDisplay.from_predictions( + y_test, + y_pred_ridge_with_trans_target, + kind="residual_vs_predicted", + ax=ax1[1], + scatter_kwargs={"alpha": 0.5}, +) +ax1[0].set_title("Ridge regression \n without target transformation") +ax1[1].set_title("Ridge regression \n with target transformation") + +f.suptitle("Ames housing data: selling price", y=1.05) +plt.tight_layout() plt.show() diff --git a/examples/covariance/plot_covariance_estimation.py b/examples/covariance/plot_covariance_estimation.py index c0540a75f9e67..1fdede5364eec 100644 --- a/examples/covariance/plot_covariance_estimation.py +++ b/examples/covariance/plot_covariance_estimation.py @@ -13,47 +13,17 @@ :ref:`shrunk_covariance` estimators. In particular, it focuses on how to set the amount of regularization, i.e. how to choose the bias-variance trade-off. - -Here we compare 3 approaches: - -* Setting the parameter by cross-validating the likelihood on three folds - according to a grid of potential shrinkage parameters. - -* A close formula proposed by Ledoit and Wolf to compute - the asymptotically optimal regularization parameter (minimizing a MSE - criterion), yielding the :class:`~sklearn.covariance.LedoitWolf` - covariance estimate. - -* An improvement of the Ledoit-Wolf shrinkage, the - :class:`~sklearn.covariance.OAS`, proposed by Chen et al. Its - convergence is significantly better under the assumption that the data - are Gaussian, in particular for small samples. - -To quantify estimation error, we plot the likelihood of unseen data for -different values of the shrinkage parameter. We also show the choices by -cross-validation, or with the LedoitWolf and OAS estimates. - -Note that the maximum likelihood estimate corresponds to no shrinkage, -and thus performs poorly. The Ledoit-Wolf estimate performs really well, -as it is close to the optimal and is computational not costly. In this -example, the OAS estimate is a bit further away. Interestingly, both -approaches outperform cross-validation, which is significantly most -computationally costly. - """ -print(__doc__) -import numpy as np -import matplotlib.pyplot as plt -from scipy import linalg +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -from sklearn.covariance import LedoitWolf, OAS, ShrunkCovariance, \ - log_likelihood, empirical_covariance -from sklearn.model_selection import GridSearchCV +# %% +# Generate sample data +# -------------------- +import numpy as np -# ############################################################################# -# Generate sample data n_features, n_samples = 40, 20 np.random.seed(42) base_X_train = np.random.normal(size=(n_samples, n_features)) @@ -64,13 +34,20 @@ X_train = np.dot(base_X_train, coloring_matrix) X_test = np.dot(base_X_test, coloring_matrix) -# ############################################################################# + +# %% # Compute the likelihood on test data +# ----------------------------------- + +from scipy import linalg + +from sklearn.covariance import ShrunkCovariance, empirical_covariance, log_likelihood # spanning a range of possible shrinkage coefficient values shrinkages = np.logspace(-2, 0, 30) -negative_logliks = [-ShrunkCovariance(shrinkage=s).fit(X_train).score(X_test) - for s in shrinkages] +negative_logliks = [ + -ShrunkCovariance(shrinkage=s).fit(X_train).score(X_test) for s in shrinkages +] # under the ground-truth model, which we would not have access to in real # settings @@ -78,11 +55,32 @@ emp_cov = empirical_covariance(X_train) loglik_real = -log_likelihood(emp_cov, linalg.inv(real_cov)) -# ############################################################################# -# Compare different approaches to setting the parameter + +# %% +# Compare different approaches to setting the regularization parameter +# -------------------------------------------------------------------- +# +# Here we compare 3 approaches: +# +# * Setting the parameter by cross-validating the likelihood on three folds +# according to a grid of potential shrinkage parameters. +# +# * A close formula proposed by Ledoit and Wolf to compute +# the asymptotically optimal regularization parameter (minimizing a MSE +# criterion), yielding the :class:`~sklearn.covariance.LedoitWolf` +# covariance estimate. +# +# * An improvement of the Ledoit-Wolf shrinkage, the +# :class:`~sklearn.covariance.OAS`, proposed by Chen et al. Its +# convergence is significantly better under the assumption that the data +# are Gaussian, in particular for small samples. + + +from sklearn.covariance import OAS, LedoitWolf +from sklearn.model_selection import GridSearchCV # GridSearch for an optimal shrinkage coefficient -tuned_parameters = [{'shrinkage': shrinkages}] +tuned_parameters = [{"shrinkage": shrinkages}] cv = GridSearchCV(ShrunkCovariance(), tuned_parameters) cv.fit(X_train) @@ -94,38 +92,68 @@ oa = OAS() loglik_oa = oa.fit(X_train).score(X_test) -# ############################################################################# +# %% # Plot results +# ------------ +# +# +# To quantify estimation error, we plot the likelihood of unseen data for +# different values of the shrinkage parameter. We also show the choices by +# cross-validation, or with the LedoitWolf and OAS estimates. + +import matplotlib.pyplot as plt + fig = plt.figure() plt.title("Regularized covariance: likelihood and shrinkage coefficient") -plt.xlabel('Regularization parameter: shrinkage coefficient') -plt.ylabel('Error: negative log-likelihood on test data') +plt.xlabel("Regularization parameter: shrinkage coefficient") +plt.ylabel("Error: negative log-likelihood on test data") # range shrinkage curve plt.loglog(shrinkages, negative_logliks, label="Negative log-likelihood") -plt.plot(plt.xlim(), 2 * [loglik_real], '--r', - label="Real covariance likelihood") +plt.plot(plt.xlim(), 2 * [loglik_real], "--r", label="Real covariance likelihood") # adjust view lik_max = np.amax(negative_logliks) lik_min = np.amin(negative_logliks) -ymin = lik_min - 6. * np.log((plt.ylim()[1] - plt.ylim()[0])) -ymax = lik_max + 10. * np.log(lik_max - lik_min) +ymin = lik_min - 6.0 * np.log((plt.ylim()[1] - plt.ylim()[0])) +ymax = lik_max + 10.0 * np.log(lik_max - lik_min) xmin = shrinkages[0] xmax = shrinkages[-1] # LW likelihood -plt.vlines(lw.shrinkage_, ymin, -loglik_lw, color='magenta', - linewidth=3, label='Ledoit-Wolf estimate') +plt.vlines( + lw.shrinkage_, + ymin, + -loglik_lw, + color="magenta", + linewidth=3, + label="Ledoit-Wolf estimate", +) # OAS likelihood -plt.vlines(oa.shrinkage_, ymin, -loglik_oa, color='purple', - linewidth=3, label='OAS estimate') +plt.vlines( + oa.shrinkage_, ymin, -loglik_oa, color="purple", linewidth=3, label="OAS estimate" +) # best CV estimator likelihood -plt.vlines(cv.best_estimator_.shrinkage, ymin, - -cv.best_estimator_.score(X_test), color='cyan', - linewidth=3, label='Cross-validation best estimate') +plt.vlines( + cv.best_estimator_.shrinkage, + ymin, + -cv.best_estimator_.score(X_test), + color="cyan", + linewidth=3, + label="Cross-validation best estimate", +) plt.ylim(ymin, ymax) plt.xlim(xmin, xmax) plt.legend() plt.show() + +# %% +# .. note:: +# +# The maximum likelihood estimate corresponds to no shrinkage, +# and thus performs poorly. The Ledoit-Wolf estimate performs really well, +# as it is close to the optimal and is not computationally costly. In this +# example, the OAS estimate is a bit further away. Interestingly, both +# approaches outperform cross-validation, which is significantly most +# computationally costly. diff --git a/examples/covariance/plot_lw_vs_oas.py b/examples/covariance/plot_lw_vs_oas.py index f9cd11bbb81b4..c1c41bc811a85 100644 --- a/examples/covariance/plot_lw_vs_oas.py +++ b/examples/covariance/plot_lw_vs_oas.py @@ -20,13 +20,15 @@ Chen et al., IEEE Trans. on Sign. Proc., Volume 58, Issue 10, October 2010. """ -print(__doc__) -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt -from scipy.linalg import toeplitz, cholesky +import numpy as np +from scipy.linalg import cholesky, toeplitz -from sklearn.covariance import LedoitWolf, OAS +from sklearn.covariance import OAS, LedoitWolf np.random.seed(0) # %% @@ -44,8 +46,7 @@ oa_shrinkage = np.zeros((n_samples_range.size, repeat)) for i, n_samples in enumerate(n_samples_range): for j in range(repeat): - X = np.dot( - np.random.normal(size=(n_samples, n_features)), coloring_matrix.T) + X = np.dot(np.random.normal(size=(n_samples, n_features)), coloring_matrix.T) lw = LedoitWolf(store_precision=False, assume_centered=True) lw.fit(X) @@ -59,10 +60,22 @@ # plot MSE plt.subplot(2, 1, 1) -plt.errorbar(n_samples_range, lw_mse.mean(1), yerr=lw_mse.std(1), - label='Ledoit-Wolf', color='navy', lw=2) -plt.errorbar(n_samples_range, oa_mse.mean(1), yerr=oa_mse.std(1), - label='OAS', color='darkorange', lw=2) +plt.errorbar( + n_samples_range, + lw_mse.mean(1), + yerr=lw_mse.std(1), + label="Ledoit-Wolf", + color="navy", + lw=2, +) +plt.errorbar( + n_samples_range, + oa_mse.mean(1), + yerr=oa_mse.std(1), + label="OAS", + color="darkorange", + lw=2, +) plt.ylabel("Squared error") plt.legend(loc="upper right") plt.title("Comparison of covariance estimators") @@ -70,14 +83,26 @@ # plot shrinkage coefficient plt.subplot(2, 1, 2) -plt.errorbar(n_samples_range, lw_shrinkage.mean(1), yerr=lw_shrinkage.std(1), - label='Ledoit-Wolf', color='navy', lw=2) -plt.errorbar(n_samples_range, oa_shrinkage.mean(1), yerr=oa_shrinkage.std(1), - label='OAS', color='darkorange', lw=2) +plt.errorbar( + n_samples_range, + lw_shrinkage.mean(1), + yerr=lw_shrinkage.std(1), + label="Ledoit-Wolf", + color="navy", + lw=2, +) +plt.errorbar( + n_samples_range, + oa_shrinkage.mean(1), + yerr=oa_shrinkage.std(1), + label="OAS", + color="darkorange", + lw=2, +) plt.xlabel("n_samples") plt.ylabel("Shrinkage") plt.legend(loc="lower right") -plt.ylim(plt.ylim()[0], 1. + (plt.ylim()[1] - plt.ylim()[0]) / 10.) +plt.ylim(plt.ylim()[0], 1.0 + (plt.ylim()[1] - plt.ylim()[0]) / 10.0) plt.xlim(5, 31) plt.show() diff --git a/examples/covariance/plot_mahalanobis_distances.py b/examples/covariance/plot_mahalanobis_distances.py index b517db5fa9613..a1507c3ef162e 100644 --- a/examples/covariance/plot_mahalanobis_distances.py +++ b/examples/covariance/plot_mahalanobis_distances.py @@ -50,18 +50,21 @@ See also :ref:`sphx_glr_auto_examples_covariance_plot_robust_vs_empirical_covariance.py` -.. topic:: References: +.. rubric:: References - .. [1] P. J. Rousseeuw. `Least median of squares regression - `_. J. Am - Stat Ass, 79:871, 1984. - .. [2] Wilson, E. B., & Hilferty, M. M. (1931). `The distribution of chi-square. - `_ - Proceedings of the National Academy of Sciences of the United States - of America, 17, 684-688. +.. [1] P. J. Rousseeuw. `Least median of squares regression + `_. J. Am + Stat Ass, 79:871, 1984. +.. [2] Wilson, E. B., & Hilferty, M. M. (1931). `The distribution of chi-square. + `_ + Proceedings of the National Academy of Sciences of the United States + of America, 17, 684-688. """ # noqa: E501 +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + # %% # Generate data # -------------- @@ -70,7 +73,7 @@ # are Gaussian distributed with mean of 0 but feature 1 has a standard # deviation equal to 2 and feature 2 has a standard deviation equal to 1. Next, # 25 samples are replaced with Gaussian outlier samples where feature 1 has -# a standard devation equal to 1 and feature 2 has a standard deviation equal +# a standard deviation equal to 1 and feature 2 has a standard deviation equal # to 7. import numpy as np @@ -84,11 +87,11 @@ # generate Gaussian data of shape (125, 2) gen_cov = np.eye(n_features) -gen_cov[0, 0] = 2. +gen_cov[0, 0] = 2.0 X = np.dot(np.random.randn(n_samples, n_features), gen_cov) # add some outliers outliers_cov = np.eye(n_features) -outliers_cov[np.arange(1, n_features), np.arange(1, n_features)] = 7. +outliers_cov[np.arange(1, n_features), np.arange(1, n_features)] = 7.0 X[-n_outliers:] = np.dot(np.random.randn(n_outliers, n_features), outliers_cov) # %% @@ -103,15 +106,18 @@ # designed to have a much larger variance in feature 2. import matplotlib.pyplot as plt + from sklearn.covariance import EmpiricalCovariance, MinCovDet # fit a MCD robust estimator to data robust_cov = MinCovDet().fit(X) # fit a MLE estimator to data emp_cov = EmpiricalCovariance().fit(X) -print('Estimated covariance matrix:\n' - 'MCD (Robust):\n{}\n' - 'MLE:\n{}'.format(robust_cov.covariance_, emp_cov.covariance_)) +print( + "Estimated covariance matrix:\nMCD (Robust):\n{}\nMLE:\n{}".format( + robust_cov.covariance_, emp_cov.covariance_ + ) +) # %% # To better visualize the difference, we plot contours of the @@ -119,36 +125,48 @@ # MCD based Mahalanobis distances fit the inlier black points much better, # whereas the MLE based distances are more influenced by the outlier # red points. +import matplotlib.lines as mlines fig, ax = plt.subplots(figsize=(10, 5)) # Plot data set -inlier_plot = ax.scatter(X[:, 0], X[:, 1], - color='black', label='inliers') -outlier_plot = ax.scatter(X[:, 0][-n_outliers:], X[:, 1][-n_outliers:], - color='red', label='outliers') -ax.set_xlim(ax.get_xlim()[0], 10.) +inlier_plot = ax.scatter(X[:, 0], X[:, 1], color="black", label="inliers") +outlier_plot = ax.scatter( + X[:, 0][-n_outliers:], X[:, 1][-n_outliers:], color="red", label="outliers" +) +ax.set_xlim(ax.get_xlim()[0], 10.0) ax.set_title("Mahalanobis distances of a contaminated data set") # Create meshgrid of feature 1 and feature 2 values -xx, yy = np.meshgrid(np.linspace(plt.xlim()[0], plt.xlim()[1], 100), - np.linspace(plt.ylim()[0], plt.ylim()[1], 100)) +xx, yy = np.meshgrid( + np.linspace(plt.xlim()[0], plt.xlim()[1], 100), + np.linspace(plt.ylim()[0], plt.ylim()[1], 100), +) zz = np.c_[xx.ravel(), yy.ravel()] # Calculate the MLE based Mahalanobis distances of the meshgrid mahal_emp_cov = emp_cov.mahalanobis(zz) mahal_emp_cov = mahal_emp_cov.reshape(xx.shape) -emp_cov_contour = plt.contour(xx, yy, np.sqrt(mahal_emp_cov), - cmap=plt.cm.PuBu_r, linestyles='dashed') +emp_cov_contour = plt.contour( + xx, yy, np.sqrt(mahal_emp_cov), cmap=plt.cm.PuBu_r, linestyles="dashed" +) # Calculate the MCD based Mahalanobis distances mahal_robust_cov = robust_cov.mahalanobis(zz) mahal_robust_cov = mahal_robust_cov.reshape(xx.shape) -robust_contour = ax.contour(xx, yy, np.sqrt(mahal_robust_cov), - cmap=plt.cm.YlOrBr_r, linestyles='dotted') +robust_contour = ax.contour( + xx, yy, np.sqrt(mahal_robust_cov), cmap=plt.cm.YlOrBr_r, linestyles="dotted" +) # Add legend -ax.legend([emp_cov_contour.collections[1], robust_contour.collections[1], - inlier_plot, outlier_plot], - ['MLE dist', 'MCD dist', 'inliers', 'outliers'], - loc="upper right", borderaxespad=0) +ax.legend( + [ + mlines.Line2D([], [], color="tab:blue", linestyle="dashed"), + mlines.Line2D([], [], color="tab:orange", linestyle="dotted"), + inlier_plot, + outlier_plot, + ], + ["MLE dist", "MCD dist", "inliers", "outliers"], + loc="upper right", + borderaxespad=0, +) plt.show() @@ -161,32 +179,37 @@ # distribution of inlier samples for robust MCD based Mahalanobis distances. fig, (ax1, ax2) = plt.subplots(1, 2) -plt.subplots_adjust(wspace=.6) +plt.subplots_adjust(wspace=0.6) # Calculate cubic root of MLE Mahalanobis distances for samples emp_mahal = emp_cov.mahalanobis(X - np.mean(X, 0)) ** (0.33) # Plot boxplots -ax1.boxplot([emp_mahal[:-n_outliers], emp_mahal[-n_outliers:]], widths=.25) +ax1.boxplot([emp_mahal[:-n_outliers], emp_mahal[-n_outliers:]], widths=0.25) # Plot individual samples -ax1.plot(np.full(n_samples - n_outliers, 1.26), emp_mahal[:-n_outliers], - '+k', markeredgewidth=1) -ax1.plot(np.full(n_outliers, 2.26), emp_mahal[-n_outliers:], - '+k', markeredgewidth=1) -ax1.axes.set_xticklabels(('inliers', 'outliers'), size=15) +ax1.plot( + np.full(n_samples - n_outliers, 1.26), + emp_mahal[:-n_outliers], + "+k", + markeredgewidth=1, +) +ax1.plot(np.full(n_outliers, 2.26), emp_mahal[-n_outliers:], "+k", markeredgewidth=1) +ax1.axes.set_xticklabels(("inliers", "outliers"), size=15) ax1.set_ylabel(r"$\sqrt[3]{\rm{(Mahal. dist.)}}$", size=16) ax1.set_title("Using non-robust estimates\n(Maximum Likelihood)") # Calculate cubic root of MCD Mahalanobis distances for samples robust_mahal = robust_cov.mahalanobis(X - robust_cov.location_) ** (0.33) # Plot boxplots -ax2.boxplot([robust_mahal[:-n_outliers], robust_mahal[-n_outliers:]], - widths=.25) +ax2.boxplot([robust_mahal[:-n_outliers], robust_mahal[-n_outliers:]], widths=0.25) # Plot individual samples -ax2.plot(np.full(n_samples - n_outliers, 1.26), robust_mahal[:-n_outliers], - '+k', markeredgewidth=1) -ax2.plot(np.full(n_outliers, 2.26), robust_mahal[-n_outliers:], - '+k', markeredgewidth=1) -ax2.axes.set_xticklabels(('inliers', 'outliers'), size=15) +ax2.plot( + np.full(n_samples - n_outliers, 1.26), + robust_mahal[:-n_outliers], + "+k", + markeredgewidth=1, +) +ax2.plot(np.full(n_outliers, 2.26), robust_mahal[-n_outliers:], "+k", markeredgewidth=1) +ax2.axes.set_xticklabels(("inliers", "outliers"), size=15) ax2.set_ylabel(r"$\sqrt[3]{\rm{(Mahal. dist.)}}$", size=16) ax2.set_title("Using robust estimates\n(Minimum Covariance Determinant)") diff --git a/examples/covariance/plot_robust_vs_empirical_covariance.py b/examples/covariance/plot_robust_vs_empirical_covariance.py index 8ca0a49d66642..2be2d0a21a4f7 100644 --- a/examples/covariance/plot_robust_vs_empirical_covariance.py +++ b/examples/covariance/plot_robust_vs_empirical_covariance.py @@ -52,11 +52,13 @@ Statistical Ass., 79:871, 1984. """ -print(__doc__) -import numpy as np -import matplotlib.pyplot as plt +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.font_manager +import matplotlib.pyplot as plt +import numpy as np from sklearn.covariance import EmpiricalCovariance, MinCovDet @@ -66,8 +68,11 @@ repeat = 10 range_n_outliers = np.concatenate( - (np.linspace(0, n_samples / 8, 5), - np.linspace(n_samples / 8, n_samples / 2, 5)[1:-1])).astype(int) + ( + np.linspace(0, n_samples / 8, 5), + np.linspace(n_samples / 8, n_samples / 2, 5)[1:-1], + ) +).astype(int) # definition of arrays to store results err_loc_mcd = np.zeros((range_n_outliers.size, repeat)) @@ -80,15 +85,15 @@ # computation for i, n_outliers in enumerate(range_n_outliers): for j in range(repeat): - rng = np.random.RandomState(i * j) # generate data X = rng.randn(n_samples, n_features) # add some outliers outliers_index = rng.permutation(n_samples)[:n_outliers] - outliers_offset = 10. * \ - (np.random.randint(2, size=(n_outliers, n_features)) - 0.5) + outliers_offset = 10.0 * ( + np.random.randint(2, size=(n_outliers, n_features)) - 0.5 + ) X[outliers_index] += outliers_offset inliers_mask = np.ones(n_samples).astype(bool) inliers_mask[outliers_index] = False @@ -96,58 +101,89 @@ # fit a Minimum Covariance Determinant (MCD) robust estimator to data mcd = MinCovDet().fit(X) # compare raw robust estimates with the true location and covariance - err_loc_mcd[i, j] = np.sum(mcd.location_ ** 2) + err_loc_mcd[i, j] = np.sum(mcd.location_**2) err_cov_mcd[i, j] = mcd.error_norm(np.eye(n_features)) # compare estimators learned from the full data set with true # parameters err_loc_emp_full[i, j] = np.sum(X.mean(0) ** 2) - err_cov_emp_full[i, j] = EmpiricalCovariance().fit(X).error_norm( - np.eye(n_features)) + err_cov_emp_full[i, j] = ( + EmpiricalCovariance().fit(X).error_norm(np.eye(n_features)) + ) # compare with an empirical covariance learned from a pure data set # (i.e. "perfect" mcd) pure_X = X[inliers_mask] pure_location = pure_X.mean(0) pure_emp_cov = EmpiricalCovariance().fit(pure_X) - err_loc_emp_pure[i, j] = np.sum(pure_location ** 2) + err_loc_emp_pure[i, j] = np.sum(pure_location**2) err_cov_emp_pure[i, j] = pure_emp_cov.error_norm(np.eye(n_features)) # Display results font_prop = matplotlib.font_manager.FontProperties(size=11) plt.subplot(2, 1, 1) lw = 2 -plt.errorbar(range_n_outliers, err_loc_mcd.mean(1), - yerr=err_loc_mcd.std(1) / np.sqrt(repeat), - label="Robust location", lw=lw, color='m') -plt.errorbar(range_n_outliers, err_loc_emp_full.mean(1), - yerr=err_loc_emp_full.std(1) / np.sqrt(repeat), - label="Full data set mean", lw=lw, color='green') -plt.errorbar(range_n_outliers, err_loc_emp_pure.mean(1), - yerr=err_loc_emp_pure.std(1) / np.sqrt(repeat), - label="Pure data set mean", lw=lw, color='black') +plt.errorbar( + range_n_outliers, + err_loc_mcd.mean(1), + yerr=err_loc_mcd.std(1) / np.sqrt(repeat), + label="Robust location", + lw=lw, + color="m", +) +plt.errorbar( + range_n_outliers, + err_loc_emp_full.mean(1), + yerr=err_loc_emp_full.std(1) / np.sqrt(repeat), + label="Full data set mean", + lw=lw, + color="green", +) +plt.errorbar( + range_n_outliers, + err_loc_emp_pure.mean(1), + yerr=err_loc_emp_pure.std(1) / np.sqrt(repeat), + label="Pure data set mean", + lw=lw, + color="black", +) plt.title("Influence of outliers on the location estimation") plt.ylabel(r"Error ($||\mu - \hat{\mu}||_2^2$)") plt.legend(loc="upper left", prop=font_prop) plt.subplot(2, 1, 2) x_size = range_n_outliers.size -plt.errorbar(range_n_outliers, err_cov_mcd.mean(1), - yerr=err_cov_mcd.std(1), - label="Robust covariance (mcd)", color='m') -plt.errorbar(range_n_outliers[:(x_size // 5 + 1)], - err_cov_emp_full.mean(1)[:(x_size // 5 + 1)], - yerr=err_cov_emp_full.std(1)[:(x_size // 5 + 1)], - label="Full data set empirical covariance", color='green') -plt.plot(range_n_outliers[(x_size // 5):(x_size // 2 - 1)], - err_cov_emp_full.mean(1)[(x_size // 5):(x_size // 2 - 1)], - color='green', ls='--') -plt.errorbar(range_n_outliers, err_cov_emp_pure.mean(1), - yerr=err_cov_emp_pure.std(1), - label="Pure data set empirical covariance", color='black') +plt.errorbar( + range_n_outliers, + err_cov_mcd.mean(1), + yerr=err_cov_mcd.std(1), + label="Robust covariance (mcd)", + color="m", +) +plt.errorbar( + range_n_outliers[: (x_size // 5 + 1)], + err_cov_emp_full.mean(1)[: (x_size // 5 + 1)], + yerr=err_cov_emp_full.std(1)[: (x_size // 5 + 1)], + label="Full data set empirical covariance", + color="green", +) +plt.plot( + range_n_outliers[(x_size // 5) : (x_size // 2 - 1)], + err_cov_emp_full.mean(1)[(x_size // 5) : (x_size // 2 - 1)], + color="green", + ls="--", +) +plt.errorbar( + range_n_outliers, + err_cov_emp_pure.mean(1), + yerr=err_cov_emp_pure.std(1), + label="Pure data set empirical covariance", + color="black", +) plt.title("Influence of outliers on the covariance estimation") plt.xlabel("Amount of contamination (%)") plt.ylabel("RMSE") -plt.legend(loc="upper center", prop=font_prop) +plt.legend(loc="center", prop=font_prop) +plt.tight_layout() plt.show() diff --git a/examples/covariance/plot_sparse_cov.py b/examples/covariance/plot_sparse_cov.py index c595e7d3d9661..868f1f3d49a6c 100644 --- a/examples/covariance/plot_sparse_cov.py +++ b/examples/covariance/plot_sparse_cov.py @@ -47,28 +47,27 @@ set by internal cross-validation in the GraphicalLassoCV. As can be seen on figure 2, the grid to compute the cross-validation score is iteratively refined in the neighborhood of the maximum. + """ -print(__doc__) -# author: Gael Varoquaux -# License: BSD 3 clause -# Copyright: INRIA +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Generate the data +# ----------------- import numpy as np from scipy import linalg + from sklearn.datasets import make_sparse_spd_matrix -from sklearn.covariance import GraphicalLassoCV, ledoit_wolf -import matplotlib.pyplot as plt -# ############################################################################# -# Generate the data n_samples = 60 n_features = 20 prng = np.random.RandomState(1) -prec = make_sparse_spd_matrix(n_features, alpha=.98, - smallest_coef=.4, - largest_coef=.7, - random_state=prng) +prec = make_sparse_spd_matrix( + n_features, alpha=0.98, smallest_coef=0.4, largest_coef=0.7, random_state=prng +) cov = linalg.inv(prec) d = np.sqrt(np.diag(cov)) cov /= d @@ -79,8 +78,11 @@ X -= X.mean(axis=0) X /= X.std(axis=0) -# ############################################################################# +# %% # Estimate the covariance +# ----------------------- +from sklearn.covariance import GraphicalLassoCV, ledoit_wolf + emp_cov = np.dot(X.T, X) / n_samples model = GraphicalLassoCV() @@ -91,48 +93,66 @@ lw_cov_, _ = ledoit_wolf(X) lw_prec_ = linalg.inv(lw_cov_) -# ############################################################################# +# %% # Plot the results +# ---------------- +import matplotlib.pyplot as plt + plt.figure(figsize=(10, 6)) plt.subplots_adjust(left=0.02, right=0.98) # plot the covariances -covs = [('Empirical', emp_cov), ('Ledoit-Wolf', lw_cov_), - ('GraphicalLassoCV', cov_), ('True', cov)] +covs = [ + ("Empirical", emp_cov), + ("Ledoit-Wolf", lw_cov_), + ("GraphicalLassoCV", cov_), + ("True", cov), +] vmax = cov_.max() for i, (name, this_cov) in enumerate(covs): plt.subplot(2, 4, i + 1) - plt.imshow(this_cov, interpolation='nearest', vmin=-vmax, vmax=vmax, - cmap=plt.cm.RdBu_r) + plt.imshow( + this_cov, interpolation="nearest", vmin=-vmax, vmax=vmax, cmap=plt.cm.RdBu_r + ) plt.xticks(()) plt.yticks(()) - plt.title('%s covariance' % name) + plt.title("%s covariance" % name) # plot the precisions -precs = [('Empirical', linalg.inv(emp_cov)), ('Ledoit-Wolf', lw_prec_), - ('GraphicalLasso', prec_), ('True', prec)] -vmax = .9 * prec_.max() +precs = [ + ("Empirical", linalg.inv(emp_cov)), + ("Ledoit-Wolf", lw_prec_), + ("GraphicalLasso", prec_), + ("True", prec), +] +vmax = 0.9 * prec_.max() for i, (name, this_prec) in enumerate(precs): ax = plt.subplot(2, 4, i + 5) - plt.imshow(np.ma.masked_equal(this_prec, 0), - interpolation='nearest', vmin=-vmax, vmax=vmax, - cmap=plt.cm.RdBu_r) + plt.imshow( + np.ma.masked_equal(this_prec, 0), + interpolation="nearest", + vmin=-vmax, + vmax=vmax, + cmap=plt.cm.RdBu_r, + ) plt.xticks(()) plt.yticks(()) - plt.title('%s precision' % name) - if hasattr(ax, 'set_facecolor'): - ax.set_facecolor('.7') + plt.title("%s precision" % name) + if hasattr(ax, "set_facecolor"): + ax.set_facecolor(".7") else: - ax.set_axis_bgcolor('.7') + ax.set_axis_bgcolor(".7") + +# %% # plot the model selection metric plt.figure(figsize=(4, 3)) -plt.axes([.2, .15, .75, .7]) -plt.plot(model.cv_results_["alphas"], model.cv_results_["mean_score"], 'o-') -plt.axvline(model.alpha_, color='.5') -plt.title('Model selection') -plt.ylabel('Cross-validation score') -plt.xlabel('alpha') +plt.axes([0.2, 0.15, 0.75, 0.7]) +plt.plot(model.cv_results_["alphas"], model.cv_results_["mean_test_score"], "o-") +plt.axvline(model.alpha_, color=".5") +plt.title("Model selection") +plt.ylabel("Cross-validation score") +plt.xlabel("alpha") plt.show() diff --git a/examples/cross_decomposition/README.txt b/examples/cross_decomposition/README.txt index 07649ffbb6960..a63e7f9159182 100644 --- a/examples/cross_decomposition/README.txt +++ b/examples/cross_decomposition/README.txt @@ -4,4 +4,3 @@ Cross decomposition ------------------- Examples concerning the :mod:`sklearn.cross_decomposition` module. - diff --git a/examples/cross_decomposition/plot_compare_cross_decomposition.py b/examples/cross_decomposition/plot_compare_cross_decomposition.py index 2f1f1d21f30d5..1fce2f70bc42a 100644 --- a/examples/cross_decomposition/plot_compare_cross_decomposition.py +++ b/examples/cross_decomposition/plot_compare_cross_decomposition.py @@ -4,6 +4,7 @@ =================================== Simple usage of various cross decomposition algorithms: + - PLSCanonical - PLSRegression, with multivariate response, a.k.a. PLS2 - PLSRegression, with univariate response, a.k.a. PLS1 @@ -17,15 +18,17 @@ first diagonal). This is also true for components 2 in both dataset, however, the correlation across datasets for different components is weak: the point cloud is very spherical. + """ -print(__doc__) -import numpy as np -import matplotlib.pyplot as plt -from sklearn.cross_decomposition import PLSCanonical, PLSRegression, CCA +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -# ############################################################################# +# %% # Dataset based latent variables model +# ------------------------------------ + +import numpy as np n = 500 # 2 latents vars: @@ -36,86 +39,97 @@ X = latents + np.random.normal(size=4 * n).reshape((n, 4)) Y = latents + np.random.normal(size=4 * n).reshape((n, 4)) -X_train = X[:n // 2] -Y_train = Y[:n // 2] -X_test = X[n // 2:] -Y_test = Y[n // 2:] +X_train = X[: n // 2] +Y_train = Y[: n // 2] +X_test = X[n // 2 :] +Y_test = Y[n // 2 :] print("Corr(X)") print(np.round(np.corrcoef(X.T), 2)) print("Corr(Y)") print(np.round(np.corrcoef(Y.T), 2)) -# ############################################################################# +# %% # Canonical (symmetric) PLS - +# ------------------------- +# # Transform data # ~~~~~~~~~~~~~~ + +from sklearn.cross_decomposition import PLSCanonical + plsca = PLSCanonical(n_components=2) plsca.fit(X_train, Y_train) X_train_r, Y_train_r = plsca.transform(X_train, Y_train) X_test_r, Y_test_r = plsca.transform(X_test, Y_test) +# %% # Scatter plot of scores # ~~~~~~~~~~~~~~~~~~~~~~ -# 1) On diagonal plot X vs Y scores on each components + +import matplotlib.pyplot as plt + +# On diagonal plot X vs Y scores on each components plt.figure(figsize=(12, 8)) plt.subplot(221) -plt.scatter(X_train_r[:, 0], Y_train_r[:, 0], label="train", - marker="o", s=25) -plt.scatter(X_test_r[:, 0], Y_test_r[:, 0], label="test", - marker="o", s=25) +plt.scatter(X_train_r[:, 0], Y_train_r[:, 0], label="train", marker="o", s=25) +plt.scatter(X_test_r[:, 0], Y_test_r[:, 0], label="test", marker="o", s=25) plt.xlabel("x scores") plt.ylabel("y scores") -plt.title('Comp. 1: X vs Y (test corr = %.2f)' % - np.corrcoef(X_test_r[:, 0], Y_test_r[:, 0])[0, 1]) +plt.title( + "Comp. 1: X vs Y (test corr = %.2f)" + % np.corrcoef(X_test_r[:, 0], Y_test_r[:, 0])[0, 1] +) plt.xticks(()) plt.yticks(()) plt.legend(loc="best") plt.subplot(224) -plt.scatter(X_train_r[:, 1], Y_train_r[:, 1], label="train", - marker="o", s=25) -plt.scatter(X_test_r[:, 1], Y_test_r[:, 1], label="test", - marker="o", s=25) +plt.scatter(X_train_r[:, 1], Y_train_r[:, 1], label="train", marker="o", s=25) +plt.scatter(X_test_r[:, 1], Y_test_r[:, 1], label="test", marker="o", s=25) plt.xlabel("x scores") plt.ylabel("y scores") -plt.title('Comp. 2: X vs Y (test corr = %.2f)' % - np.corrcoef(X_test_r[:, 1], Y_test_r[:, 1])[0, 1]) +plt.title( + "Comp. 2: X vs Y (test corr = %.2f)" + % np.corrcoef(X_test_r[:, 1], Y_test_r[:, 1])[0, 1] +) plt.xticks(()) plt.yticks(()) plt.legend(loc="best") -# 2) Off diagonal plot components 1 vs 2 for X and Y +# Off diagonal plot components 1 vs 2 for X and Y plt.subplot(222) -plt.scatter(X_train_r[:, 0], X_train_r[:, 1], label="train", - marker="*", s=50) -plt.scatter(X_test_r[:, 0], X_test_r[:, 1], label="test", - marker="*", s=50) +plt.scatter(X_train_r[:, 0], X_train_r[:, 1], label="train", marker="*", s=50) +plt.scatter(X_test_r[:, 0], X_test_r[:, 1], label="test", marker="*", s=50) plt.xlabel("X comp. 1") plt.ylabel("X comp. 2") -plt.title('X comp. 1 vs X comp. 2 (test corr = %.2f)' - % np.corrcoef(X_test_r[:, 0], X_test_r[:, 1])[0, 1]) +plt.title( + "X comp. 1 vs X comp. 2 (test corr = %.2f)" + % np.corrcoef(X_test_r[:, 0], X_test_r[:, 1])[0, 1] +) plt.legend(loc="best") plt.xticks(()) plt.yticks(()) plt.subplot(223) -plt.scatter(Y_train_r[:, 0], Y_train_r[:, 1], label="train", - marker="*", s=50) -plt.scatter(Y_test_r[:, 0], Y_test_r[:, 1], label="test", - marker="*", s=50) +plt.scatter(Y_train_r[:, 0], Y_train_r[:, 1], label="train", marker="*", s=50) +plt.scatter(Y_test_r[:, 0], Y_test_r[:, 1], label="test", marker="*", s=50) plt.xlabel("Y comp. 1") plt.ylabel("Y comp. 2") -plt.title('Y comp. 1 vs Y comp. 2 , (test corr = %.2f)' - % np.corrcoef(Y_test_r[:, 0], Y_test_r[:, 1])[0, 1]) +plt.title( + "Y comp. 1 vs Y comp. 2 , (test corr = %.2f)" + % np.corrcoef(Y_test_r[:, 0], Y_test_r[:, 1])[0, 1] +) plt.legend(loc="best") plt.xticks(()) plt.yticks(()) plt.show() -# ############################################################################# +# %% # PLS regression, with multivariate response, a.k.a. PLS2 +# ------------------------------------------------------- + +from sklearn.cross_decomposition import PLSRegression n = 1000 q = 3 @@ -134,7 +148,9 @@ print(np.round(pls2.coef_, 1)) pls2.predict(X) +# %% # PLS regression, with univariate response, a.k.a. PLS1 +# ----------------------------------------------------- n = 1000 p = 10 @@ -146,8 +162,11 @@ print("Estimated betas") print(np.round(pls1.coef_, 1)) -# ############################################################################# +# %% # CCA (PLS mode B with symmetric deflation) +# ----------------------------------------- + +from sklearn.cross_decomposition import CCA cca = CCA(n_components=2) cca.fit(X_train, Y_train) diff --git a/examples/cross_decomposition/plot_pcr_vs_pls.py b/examples/cross_decomposition/plot_pcr_vs_pls.py index 17a9cce651a73..e0606a5c3dd42 100644 --- a/examples/cross_decomposition/plot_pcr_vs_pls.py +++ b/examples/cross_decomposition/plot_pcr_vs_pls.py @@ -30,9 +30,11 @@ linear regressor to the transformed data. The main difference with PCR is that the PLS transformation is supervised. Therefore, as we will see in this example, it does not suffer from the issue we just mentioned. + """ -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause # %% # The data @@ -42,26 +44,34 @@ # into PCR and PLS, we fit a PCA estimator to display the two principal # components of this dataset, i.e. the two directions that explain the most # variance in the data. -import numpy as np import matplotlib.pyplot as plt +import numpy as np + from sklearn.decomposition import PCA rng = np.random.RandomState(0) n_samples = 500 -cov = [[3, 3], - [3, 4]] +cov = [[3, 3], [3, 4]] X = rng.multivariate_normal(mean=[0, 0], cov=cov, size=n_samples) pca = PCA(n_components=2).fit(X) -plt.scatter(X[:, 0], X[:, 1], alpha=.3, label='samples') +plt.scatter(X[:, 0], X[:, 1], alpha=0.3, label="samples") for i, (comp, var) in enumerate(zip(pca.components_, pca.explained_variance_)): comp = comp * var # scale component by its variance explanation power - plt.plot([0, comp[0]], [0, comp[1]], label=f"Component {i}", linewidth=5, - color=f"C{i + 2}") -plt.gca().set(aspect='equal', - title="2-dimensional dataset with principal components", - xlabel='first feature', ylabel='second feature') + plt.plot( + [0, comp[0]], + [0, comp[1]], + label=f"Component {i}", + linewidth=5, + color=f"C{i + 2}", + ) +plt.gca().set( + aspect="equal", + title="2-dimensional dataset with principal components", + xlabel="first feature", + ylabel="second feature", +) plt.legend() plt.show() @@ -74,10 +84,10 @@ fig, axes = plt.subplots(1, 2, figsize=(10, 3)) -axes[0].scatter(X.dot(pca.components_[0]), y, alpha=.3) -axes[0].set(xlabel='Projected data onto first PCA component', ylabel='y') -axes[1].scatter(X.dot(pca.components_[1]), y, alpha=.3) -axes[1].set(xlabel='Projected data onto second PCA component', ylabel='y') +axes[0].scatter(X.dot(pca.components_[0]), y, alpha=0.3) +axes[0].set(xlabel="Projected data onto first PCA component", ylabel="y") +axes[1].scatter(X.dot(pca.components_[1]), y, alpha=0.3) +axes[1].set(xlabel="Projected data onto second PCA component", ylabel="y") plt.tight_layout() plt.show() @@ -93,34 +103,36 @@ # For both models, we plot the projected data onto the first component against # the target. In both cases, this projected data is what the regressors will # use as training data. +from sklearn.cross_decomposition import PLSRegression +from sklearn.decomposition import PCA +from sklearn.linear_model import LinearRegression from sklearn.model_selection import train_test_split from sklearn.pipeline import make_pipeline -from sklearn.linear_model import LinearRegression from sklearn.preprocessing import StandardScaler -from sklearn.decomposition import PCA -from sklearn.cross_decomposition import PLSRegression X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=rng) pcr = make_pipeline(StandardScaler(), PCA(n_components=1), LinearRegression()) pcr.fit(X_train, y_train) -pca = pcr.named_steps['pca'] # retrieve the PCA step of the pipeline +pca = pcr.named_steps["pca"] # retrieve the PCA step of the pipeline pls = PLSRegression(n_components=1) pls.fit(X_train, y_train) fig, axes = plt.subplots(1, 2, figsize=(10, 3)) -axes[0].scatter(pca.transform(X_test), y_test, alpha=.3, label='ground truth') -axes[0].scatter(pca.transform(X_test), pcr.predict(X_test), alpha=.3, - label='predictions') -axes[0].set(xlabel='Projected data onto first PCA component', - ylabel='y', title='PCR / PCA') +axes[0].scatter(pca.transform(X_test), y_test, alpha=0.3, label="ground truth") +axes[0].scatter( + pca.transform(X_test), pcr.predict(X_test), alpha=0.3, label="predictions" +) +axes[0].set( + xlabel="Projected data onto first PCA component", ylabel="y", title="PCR / PCA" +) axes[0].legend() -axes[1].scatter(pls.transform(X_test), y_test, alpha=.3, label='ground truth') -axes[1].scatter(pls.transform(X_test), pls.predict(X_test), alpha=.3, - label='predictions') -axes[1].set(xlabel='Projected data onto first PLS component', - ylabel='y', title='PLS') +axes[1].scatter(pls.transform(X_test), y_test, alpha=0.3, label="ground truth") +axes[1].scatter( + pls.transform(X_test), pls.predict(X_test), alpha=0.3, label="predictions" +) +axes[1].set(xlabel="Projected data onto first PLS component", ylabel="y", title="PLS") axes[1].legend() plt.tight_layout() plt.show() @@ -134,7 +146,7 @@ # # On the other hand, the PLS regressor manages to capture the effect of the # direction with the lowest variance, thanks to its use of target information -# during the transformation: it can recogize that this direction is actually +# during the transformation: it can recognize that this direction is actually # the most predictive. We note that the first PLS component is negatively # correlated with the target, which comes from the fact that the signs of # eigenvectors are arbitrary. diff --git a/examples/datasets/plot_digits_last_image.py b/examples/datasets/plot_digits_last_image.py deleted file mode 100644 index 7105b62880bdd..0000000000000 --- a/examples/datasets/plot_digits_last_image.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -""" -========================================================= -The Digit Dataset -========================================================= - -This dataset is made up of 1797 8x8 images. Each image, -like the one shown below, is of a hand-written digit. -In order to utilize an 8x8 figure like this, we'd have to -first transform it into a feature vector with length 64. - -See `here -`_ -for more information about this dataset. -""" -print(__doc__) - - -# Code source: GaÃĢl Varoquaux -# Modified for documentation by Jaques Grobler -# License: BSD 3 clause - -from sklearn import datasets - -import matplotlib.pyplot as plt - -#Load the digits dataset -digits = datasets.load_digits() - -#Display the first digit -plt.figure(1, figsize=(3, 3)) -plt.imshow(digits.images[-1], cmap=plt.cm.gray_r, interpolation='nearest') -plt.show() diff --git a/examples/datasets/plot_iris_dataset.py b/examples/datasets/plot_iris_dataset.py deleted file mode 100644 index d6e23253aa53e..0000000000000 --- a/examples/datasets/plot_iris_dataset.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -""" -========================================================= -The Iris Dataset -========================================================= -This data sets consists of 3 different types of irises' -(Setosa, Versicolour, and Virginica) petal and sepal -length, stored in a 150x4 numpy.ndarray - -The rows being the samples and the columns being: -Sepal Length, Sepal Width, Petal Length and Petal Width. - -The below plot uses the first two features. -See `here `_ for more -information on this dataset. -""" -print(__doc__) - - -# Code source: GaÃĢl Varoquaux -# Modified for documentation by Jaques Grobler -# License: BSD 3 clause - -import matplotlib.pyplot as plt -from mpl_toolkits.mplot3d import Axes3D -from sklearn import datasets -from sklearn.decomposition import PCA - -# import some data to play with -iris = datasets.load_iris() -X = iris.data[:, :2] # we only take the first two features. -y = iris.target - -x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5 -y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5 - -plt.figure(2, figsize=(8, 6)) -plt.clf() - -# Plot the training points -plt.scatter(X[:, 0], X[:, 1], c=y, cmap=plt.cm.Set1, - edgecolor='k') -plt.xlabel('Sepal length') -plt.ylabel('Sepal width') - -plt.xlim(x_min, x_max) -plt.ylim(y_min, y_max) -plt.xticks(()) -plt.yticks(()) - -# To getter a better understanding of interaction of the dimensions -# plot the first three PCA dimensions -fig = plt.figure(1, figsize=(8, 6)) -ax = Axes3D(fig, elev=-150, azim=110) -X_reduced = PCA(n_components=3).fit_transform(iris.data) -ax.scatter(X_reduced[:, 0], X_reduced[:, 1], X_reduced[:, 2], c=y, - cmap=plt.cm.Set1, edgecolor='k', s=40) -ax.set_title("First three PCA directions") -ax.set_xlabel("1st eigenvector") -ax.w_xaxis.set_ticklabels([]) -ax.set_ylabel("2nd eigenvector") -ax.w_yaxis.set_ticklabels([]) -ax.set_zlabel("3rd eigenvector") -ax.w_zaxis.set_ticklabels([]) - -plt.show() diff --git a/examples/datasets/plot_random_dataset.py b/examples/datasets/plot_random_dataset.py deleted file mode 100644 index 2f8d4be8ac383..0000000000000 --- a/examples/datasets/plot_random_dataset.py +++ /dev/null @@ -1,68 +0,0 @@ -""" -============================================== -Plot randomly generated classification dataset -============================================== - -This example plots several randomly generated classification datasets. -For easy visualization, all datasets have 2 features, plotted on the x and y -axis. The color of each point represents its class label. - -The first 4 plots use the :func:`~sklearn.datasets.make_classification` with -different numbers of informative features, clusters per class and classes. -The final 2 plots use :func:`~sklearn.datasets.make_blobs` and -:func:`~sklearn.datasets.make_gaussian_quantiles`. -""" - -print(__doc__) - -import matplotlib.pyplot as plt - -from sklearn.datasets import make_classification -from sklearn.datasets import make_blobs -from sklearn.datasets import make_gaussian_quantiles - -plt.figure(figsize=(8, 8)) -plt.subplots_adjust(bottom=.05, top=.9, left=.05, right=.95) - -plt.subplot(321) -plt.title("One informative feature, one cluster per class", fontsize='small') -X1, Y1 = make_classification(n_features=2, n_redundant=0, n_informative=1, - n_clusters_per_class=1) -plt.scatter(X1[:, 0], X1[:, 1], marker='o', c=Y1, - s=25, edgecolor='k') - -plt.subplot(322) -plt.title("Two informative features, one cluster per class", fontsize='small') -X1, Y1 = make_classification(n_features=2, n_redundant=0, n_informative=2, - n_clusters_per_class=1) -plt.scatter(X1[:, 0], X1[:, 1], marker='o', c=Y1, - s=25, edgecolor='k') - -plt.subplot(323) -plt.title("Two informative features, two clusters per class", - fontsize='small') -X2, Y2 = make_classification(n_features=2, n_redundant=0, n_informative=2) -plt.scatter(X2[:, 0], X2[:, 1], marker='o', c=Y2, - s=25, edgecolor='k') - -plt.subplot(324) -plt.title("Multi-class, two informative features, one cluster", - fontsize='small') -X1, Y1 = make_classification(n_features=2, n_redundant=0, n_informative=2, - n_clusters_per_class=1, n_classes=3) -plt.scatter(X1[:, 0], X1[:, 1], marker='o', c=Y1, - s=25, edgecolor='k') - -plt.subplot(325) -plt.title("Three blobs", fontsize='small') -X1, Y1 = make_blobs(n_features=2, centers=3) -plt.scatter(X1[:, 0], X1[:, 1], marker='o', c=Y1, - s=25, edgecolor='k') - -plt.subplot(326) -plt.title("Gaussian divided into three quantiles", fontsize='small') -X1, Y1 = make_gaussian_quantiles(n_features=2, n_classes=3) -plt.scatter(X1[:, 0], X1[:, 1], marker='o', c=Y1, - s=25, edgecolor='k') - -plt.show() diff --git a/examples/datasets/plot_random_multilabel_dataset.py b/examples/datasets/plot_random_multilabel_dataset.py index 5cb54689d64be..ae7ff483e65b4 100644 --- a/examples/datasets/plot_random_multilabel_dataset.py +++ b/examples/datasets/plot_random_multilabel_dataset.py @@ -9,17 +9,17 @@ Points are labeled as follows, where Y means the class is present: - ===== ===== ===== ====== - 1 2 3 Color - ===== ===== ===== ====== - Y N N Red - N Y N Blue - N N Y Yellow - Y Y N Purple - Y N Y Orange - Y Y N Green - Y Y Y Brown - ===== ===== ===== ====== +===== ===== ===== ====== + 1 2 3 Color +===== ===== ===== ====== + Y N N Red + N Y N Blue + N N Y Yellow + Y Y N Purple + Y N Y Orange + Y Y N Green + Y Y Y Brown +===== ===== ===== ====== A star marks the expected sample for each class; its size reflects the probability of selecting that class label. @@ -32,63 +32,78 @@ "document length", while here we have much larger documents than vocabulary. Similarly, with ``n_classes > n_features``, it is much less likely that a feature distinguishes a particular class. + """ -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt +import numpy as np from sklearn.datasets import make_multilabel_classification as make_ml_clf -print(__doc__) - -COLORS = np.array(['!', - '#FF3333', # red - '#0198E1', # blue - '#BF5FFF', # purple - '#FCD116', # yellow - '#FF7216', # orange - '#4DBD33', # green - '#87421F' # brown - ]) +COLORS = np.array( + [ + "!", + "#FF3333", # red + "#0198E1", # blue + "#BF5FFF", # purple + "#FCD116", # yellow + "#FF7216", # orange + "#4DBD33", # green + "#87421F", # brown + ] +) # Use same random seed for multiple calls to make_multilabel_classification to # ensure same distributions -RANDOM_SEED = np.random.randint(2 ** 10) +RANDOM_SEED = np.random.randint(2**10) def plot_2d(ax, n_labels=1, n_classes=3, length=50): - X, Y, p_c, p_w_c = make_ml_clf(n_samples=150, n_features=2, - n_classes=n_classes, n_labels=n_labels, - length=length, allow_unlabeled=False, - return_distributions=True, - random_state=RANDOM_SEED) - - ax.scatter(X[:, 0], X[:, 1], color=COLORS.take((Y * [1, 2, 4] - ).sum(axis=1)), - marker='.') - ax.scatter(p_w_c[0] * length, p_w_c[1] * length, - marker='*', linewidth=.5, edgecolor='black', - s=20 + 1500 * p_c ** 2, - color=COLORS.take([1, 2, 4])) - ax.set_xlabel('Feature 0 count') + X, Y, p_c, p_w_c = make_ml_clf( + n_samples=150, + n_features=2, + n_classes=n_classes, + n_labels=n_labels, + length=length, + allow_unlabeled=False, + return_distributions=True, + random_state=RANDOM_SEED, + ) + + ax.scatter( + X[:, 0], X[:, 1], color=COLORS.take((Y * [1, 2, 4]).sum(axis=1)), marker="." + ) + ax.scatter( + p_w_c[0] * length, + p_w_c[1] * length, + marker="*", + linewidth=0.5, + edgecolor="black", + s=20 + 1500 * p_c**2, + color=COLORS.take([1, 2, 4]), + ) + ax.set_xlabel("Feature 0 count") return p_c, p_w_c -_, (ax1, ax2) = plt.subplots(1, 2, sharex='row', sharey='row', figsize=(8, 4)) -plt.subplots_adjust(bottom=.15) +_, (ax1, ax2) = plt.subplots(1, 2, sharex="row", sharey="row", figsize=(8, 4)) +plt.subplots_adjust(bottom=0.15) p_c, p_w_c = plot_2d(ax1, n_labels=1) -ax1.set_title('n_labels=1, length=50') -ax1.set_ylabel('Feature 1 count') +ax1.set_title("n_labels=1, length=50") +ax1.set_ylabel("Feature 1 count") plot_2d(ax2, n_labels=3) -ax2.set_title('n_labels=3, length=50') +ax2.set_title("n_labels=3, length=50") ax2.set_xlim(left=0, auto=True) ax2.set_ylim(bottom=0, auto=True) plt.show() -print('The data was generated from (random_state=%d):' % RANDOM_SEED) -print('Class', 'P(C)', 'P(w0|C)', 'P(w1|C)', sep='\t') -for k, p, p_w in zip(['red', 'blue', 'yellow'], p_c, p_w_c.T): - print('%s\t%0.2f\t%0.2f\t%0.2f' % (k, p, p_w[0], p_w[1])) +print("The data was generated from (random_state=%d):" % RANDOM_SEED) +print("Class", "P(C)", "P(w0|C)", "P(w1|C)", sep="\t") +for k, p, p_w in zip(["red", "blue", "yellow"], p_c, p_w_c.T): + print("%s\t%0.2f\t%0.2f\t%0.2f" % (k, p, p_w[0], p_w[1])) diff --git a/examples/decomposition/README.txt b/examples/decomposition/README.txt index 73014f768ff9f..40fc716bb0a1f 100644 --- a/examples/decomposition/README.txt +++ b/examples/decomposition/README.txt @@ -4,4 +4,3 @@ Decomposition ------------- Examples concerning the :mod:`sklearn.decomposition` module. - diff --git a/examples/decomposition/plot_beta_divergence.py b/examples/decomposition/plot_beta_divergence.py deleted file mode 100644 index 8f39039446e88..0000000000000 --- a/examples/decomposition/plot_beta_divergence.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -============================== -Beta-divergence loss functions -============================== - -A plot that compares the various Beta-divergence loss functions supported by -the Multiplicative-Update ('mu') solver in :class:`~sklearn.decomposition.NMF`. -""" -import numpy as np -import matplotlib.pyplot as plt -from sklearn.decomposition._nmf import _beta_divergence - -print(__doc__) - -x = np.linspace(0.001, 4, 1000) -y = np.zeros(x.shape) - -colors = 'mbgyr' -for j, beta in enumerate((0., 0.5, 1., 1.5, 2.)): - for i, xi in enumerate(x): - y[i] = _beta_divergence(1, xi, 1, beta) - name = "beta = %1.1f" % beta - plt.plot(x, y, label=name, color=colors[j]) - -plt.xlabel("x") -plt.title("beta-divergence(1, x)") -plt.legend(loc=0) -plt.axis([0, 4, 0, 3]) -plt.show() diff --git a/examples/decomposition/plot_faces_decomposition.py b/examples/decomposition/plot_faces_decomposition.py index d8b1120d3590b..8eb124015009d 100644 --- a/examples/decomposition/plot_faces_decomposition.py +++ b/examples/decomposition/plot_faces_decomposition.py @@ -5,189 +5,331 @@ This example applies to :ref:`olivetti_faces_dataset` different unsupervised matrix decomposition (dimension reduction) methods from the module -:py:mod:`sklearn.decomposition` (see the documentation chapter -:ref:`decompositions`) . - +:mod:`sklearn.decomposition` (see the documentation chapter +:ref:`decompositions`). """ -print(__doc__) -# Authors: Vlad Niculae, Alexandre Gramfort -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Dataset preparation +# ------------------- +# +# Loading and preprocessing the Olivetti faces dataset. import logging -from time import time -from numpy.random import RandomState import matplotlib.pyplot as plt +from numpy.random import RandomState +from sklearn import cluster, decomposition from sklearn.datasets import fetch_olivetti_faces -from sklearn.cluster import MiniBatchKMeans -from sklearn import decomposition -# Display progress logs on stdout -logging.basicConfig(level=logging.INFO, - format='%(asctime)s %(levelname)s %(message)s') -n_row, n_col = 2, 3 -n_components = n_row * n_col -image_shape = (64, 64) rng = RandomState(0) -# ############################################################################# -# Load faces data -faces, _ = fetch_olivetti_faces(return_X_y=True, shuffle=True, - random_state=rng) +# Display progress logs on stdout +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + +faces, _ = fetch_olivetti_faces(return_X_y=True, shuffle=True, random_state=rng) n_samples, n_features = faces.shape -# global centering +# Global centering (focus on one feature, centering all samples) faces_centered = faces - faces.mean(axis=0) -# local centering +# Local centering (focus on one sample, centering all features) faces_centered -= faces_centered.mean(axis=1).reshape(n_samples, -1) print("Dataset consists of %d faces" % n_samples) +# %% +# Define a base function to plot the gallery of faces. -def plot_gallery(title, images, n_col=n_col, n_row=n_row, cmap=plt.cm.gray): - plt.figure(figsize=(2. * n_col, 2.26 * n_row)) - plt.suptitle(title, size=16) - for i, comp in enumerate(images): - plt.subplot(n_row, n_col, i + 1) - vmax = max(comp.max(), -comp.min()) - plt.imshow(comp.reshape(image_shape), cmap=cmap, - interpolation='nearest', - vmin=-vmax, vmax=vmax) - plt.xticks(()) - plt.yticks(()) - plt.subplots_adjust(0.01, 0.05, 0.99, 0.93, 0.04, 0.) - -# ############################################################################# -# List of the different estimators, whether to center and transpose the -# problem, and whether the transformer uses the clustering API. -estimators = [ - ('Eigenfaces - PCA using randomized SVD', - decomposition.PCA(n_components=n_components, svd_solver='randomized', - whiten=True), - True), - - ('Non-negative components - NMF', - decomposition.NMF(n_components=n_components, init='nndsvda', tol=5e-3), - False), - - ('Independent components - FastICA', - decomposition.FastICA(n_components=n_components, whiten=True), - True), - - ('Sparse comp. - MiniBatchSparsePCA', - decomposition.MiniBatchSparsePCA(n_components=n_components, alpha=0.8, - n_iter=100, batch_size=3, - random_state=rng), - True), - - ('MiniBatchDictionaryLearning', - decomposition.MiniBatchDictionaryLearning(n_components=15, alpha=0.1, - n_iter=50, batch_size=3, - random_state=rng), - True), - - ('Cluster centers - MiniBatchKMeans', - MiniBatchKMeans(n_clusters=n_components, tol=1e-3, batch_size=20, - max_iter=50, random_state=rng), - True), - - ('Factor Analysis components - FA', - decomposition.FactorAnalysis(n_components=n_components, max_iter=20), - True), -] - - -# ############################################################################# -# Plot a sample of the input data - -plot_gallery("First centered Olivetti faces", faces_centered[:n_components]) - -# ############################################################################# -# Do the estimation and plot it - -for name, estimator, center in estimators: - print("Extracting the top %d %s..." % (n_components, name)) - t0 = time() - data = faces - if center: - data = faces_centered - estimator.fit(data) - train_time = (time() - t0) - print("done in %0.3fs" % train_time) - if hasattr(estimator, 'cluster_centers_'): - components_ = estimator.cluster_centers_ - else: - components_ = estimator.components_ - - # Plot an image representing the pixelwise variance provided by the - # estimator e.g its noise_variance_ attribute. The Eigenfaces estimator, - # via the PCA decomposition, also provides a scalar noise_variance_ - # (the mean of pixelwise variance) that cannot be displayed as an image - # so we skip it. - if (hasattr(estimator, 'noise_variance_') and - estimator.noise_variance_.ndim > 0): # Skip the Eigenfaces case - plot_gallery("Pixelwise variance", - estimator.noise_variance_.reshape(1, -1), n_col=1, - n_row=1) - plot_gallery('%s - Train time %.1fs' % (name, train_time), - components_[:n_components]) - -plt.show() +n_row, n_col = 2, 3 +n_components = n_row * n_col +image_shape = (64, 64) -# ############################################################################# -# Various positivity constraints applied to dictionary learning. -estimators = [ - ('Dictionary learning', - decomposition.MiniBatchDictionaryLearning(n_components=15, alpha=0.1, - n_iter=50, batch_size=3, - random_state=rng), - True), - ('Dictionary learning - positive dictionary', - decomposition.MiniBatchDictionaryLearning(n_components=15, alpha=0.1, - n_iter=50, batch_size=3, - random_state=rng, - positive_dict=True), - True), - ('Dictionary learning - positive code', - decomposition.MiniBatchDictionaryLearning(n_components=15, alpha=0.1, - n_iter=50, batch_size=3, - fit_algorithm='cd', - random_state=rng, - positive_code=True), - True), - ('Dictionary learning - positive dictionary & code', - decomposition.MiniBatchDictionaryLearning(n_components=15, alpha=0.1, - n_iter=50, batch_size=3, - fit_algorithm='cd', - random_state=rng, - positive_dict=True, - positive_code=True), - True), -] - - -# ############################################################################# -# Plot a sample of the input data - -plot_gallery("First centered Olivetti faces", faces_centered[:n_components], - cmap=plt.cm.RdBu) - -# ############################################################################# -# Do the estimation and plot it - -for name, estimator, center in estimators: - print("Extracting the top %d %s..." % (n_components, name)) - t0 = time() - data = faces - if center: - data = faces_centered - estimator.fit(data) - train_time = (time() - t0) - print("done in %0.3fs" % train_time) - components_ = estimator.components_ - plot_gallery(name, components_[:n_components], cmap=plt.cm.RdBu) +def plot_gallery(title, images, n_col=n_col, n_row=n_row, cmap=plt.cm.gray): + fig, axs = plt.subplots( + nrows=n_row, + ncols=n_col, + figsize=(2.0 * n_col, 2.3 * n_row), + facecolor="white", + constrained_layout=True, + ) + fig.set_constrained_layout_pads(w_pad=0.01, h_pad=0.02, hspace=0, wspace=0) + fig.set_edgecolor("black") + fig.suptitle(title, size=16) + for ax, vec in zip(axs.flat, images): + vmax = max(vec.max(), -vec.min()) + im = ax.imshow( + vec.reshape(image_shape), + cmap=cmap, + interpolation="nearest", + vmin=-vmax, + vmax=vmax, + ) + ax.axis("off") + + fig.colorbar(im, ax=axs, orientation="horizontal", shrink=0.99, aspect=40, pad=0.01) + plt.show() + + +# %% +# Let's take a look at our data. Gray color indicates negative values, +# white indicates positive values. + +plot_gallery("Faces from dataset", faces_centered[:n_components]) + +# %% +# Decomposition +# ------------- +# +# Initialise different estimators for decomposition and fit each +# of them on all images and plot some results. Each estimator extracts +# 6 components as vectors :math:`h \in \mathbb{R}^{4096}`. +# We just displayed these vectors in human-friendly visualisation as 64x64 pixel images. +# +# Read more in the :ref:`User Guide `. + +# %% +# Eigenfaces - PCA using randomized SVD +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# Linear dimensionality reduction using Singular Value Decomposition (SVD) of the data +# to project it to a lower dimensional space. +# +# +# .. note:: +# +# The Eigenfaces estimator, via the :py:mod:`sklearn.decomposition.PCA`, +# also provides a scalar `noise_variance_` (the mean of pixelwise variance) +# that cannot be displayed as an image. + +# %% +pca_estimator = decomposition.PCA( + n_components=n_components, svd_solver="randomized", whiten=True +) +pca_estimator.fit(faces_centered) +plot_gallery( + "Eigenfaces - PCA using randomized SVD", pca_estimator.components_[:n_components] +) + +# %% +# Non-negative components - NMF +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# Estimate non-negative original data as production of two non-negative matrices. + +# %% +nmf_estimator = decomposition.NMF(n_components=n_components, tol=5e-3) +nmf_estimator.fit(faces) # original non- negative dataset +plot_gallery("Non-negative components - NMF", nmf_estimator.components_[:n_components]) + +# %% +# Independent components - FastICA +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# Independent component analysis separates a multivariate vectors into additive +# subcomponents that are maximally independent. + +# %% +ica_estimator = decomposition.FastICA( + n_components=n_components, max_iter=400, whiten="arbitrary-variance", tol=15e-5 +) +ica_estimator.fit(faces_centered) +plot_gallery( + "Independent components - FastICA", ica_estimator.components_[:n_components] +) + +# %% +# Sparse components - MiniBatchSparsePCA +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# Mini-batch sparse PCA (:class:`~sklearn.decomposition.MiniBatchSparsePCA`) +# extracts the set of sparse components that best reconstruct the data. This +# variant is faster but less accurate than the similar +# :class:`~sklearn.decomposition.SparsePCA`. + +# %% +batch_pca_estimator = decomposition.MiniBatchSparsePCA( + n_components=n_components, alpha=0.1, max_iter=100, batch_size=3, random_state=rng +) +batch_pca_estimator.fit(faces_centered) +plot_gallery( + "Sparse components - MiniBatchSparsePCA", + batch_pca_estimator.components_[:n_components], +) + +# %% +# Dictionary learning +# ^^^^^^^^^^^^^^^^^^^ +# +# By default, :class:`~sklearn.decomposition.MiniBatchDictionaryLearning` +# divides the data into mini-batches and optimizes in an online manner by +# cycling over the mini-batches for the specified number of iterations. + +# %% +batch_dict_estimator = decomposition.MiniBatchDictionaryLearning( + n_components=n_components, alpha=0.1, max_iter=50, batch_size=3, random_state=rng +) +batch_dict_estimator.fit(faces_centered) +plot_gallery("Dictionary learning", batch_dict_estimator.components_[:n_components]) + +# %% +# Cluster centers - MiniBatchKMeans +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# :class:`sklearn.cluster.MiniBatchKMeans` is computationally efficient and +# implements on-line learning with a +# :meth:`~sklearn.cluster.MiniBatchKMeans.partial_fit` method. That is +# why it could be beneficial to enhance some time-consuming algorithms with +# :class:`~sklearn.cluster.MiniBatchKMeans`. + +# %% +kmeans_estimator = cluster.MiniBatchKMeans( + n_clusters=n_components, + tol=1e-3, + batch_size=20, + max_iter=50, + random_state=rng, +) +kmeans_estimator.fit(faces_centered) +plot_gallery( + "Cluster centers - MiniBatchKMeans", + kmeans_estimator.cluster_centers_[:n_components], +) + + +# %% +# Factor Analysis components - FA +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# :class:`~sklearn.decomposition.FactorAnalysis` is similar to +# :class:`~sklearn.decomposition.PCA` but has the advantage of modelling the +# variance in every direction of the input space independently (heteroscedastic +# noise). Read more in the :ref:`User Guide `. + +# %% +fa_estimator = decomposition.FactorAnalysis(n_components=n_components, max_iter=20) +fa_estimator.fit(faces_centered) +plot_gallery("Factor Analysis (FA)", fa_estimator.components_[:n_components]) + +# --- Pixelwise variance +plt.figure(figsize=(3.2, 3.6), facecolor="white", tight_layout=True) +vec = fa_estimator.noise_variance_ +vmax = max(vec.max(), -vec.min()) +plt.imshow( + vec.reshape(image_shape), + cmap=plt.cm.gray, + interpolation="nearest", + vmin=-vmax, + vmax=vmax, +) +plt.axis("off") +plt.title("Pixelwise variance from \n Factor Analysis (FA)", size=16, wrap=True) +plt.colorbar(orientation="horizontal", shrink=0.8, pad=0.03) plt.show() + +# %% +# Decomposition: Dictionary learning +# ---------------------------------- +# +# In the further section, let's consider :ref:`DictionaryLearning` more precisely. +# Dictionary learning is a problem that amounts to finding a sparse representation +# of the input data as a combination of simple elements. These simple elements form +# a dictionary. It is possible to constrain the dictionary and/or coding coefficients +# to be positive to match constraints that may be present in the data. +# +# :class:`~sklearn.decomposition.MiniBatchDictionaryLearning` implements a +# faster, but less accurate version of the dictionary learning algorithm that +# is better suited for large datasets. Read more in the :ref:`User Guide +# `. + +# %% +# Plot the same samples from our dataset but with another colormap. +# Red indicates negative values, blue indicates positive values, +# and white represents zeros. + +plot_gallery("Faces from dataset", faces_centered[:n_components], cmap=plt.cm.RdBu) + +# %% +# Similar to the previous examples, we change parameters and train +# :class:`~sklearn.decomposition.MiniBatchDictionaryLearning` estimator on all +# images. Generally, the dictionary learning and sparse encoding decompose +# input data into the dictionary and the coding coefficients matrices. :math:`X +# \approx UV`, where :math:`X = [x_1, . . . , x_n]`, :math:`X \in +# \mathbb{R}^{m×n}`, dictionary :math:`U \in \mathbb{R}^{m×k}`, coding +# coefficients :math:`V \in \mathbb{R}^{k×n}`. +# +# Also below are the results when the dictionary and coding +# coefficients are positively constrained. + +# %% +# Dictionary learning - positive dictionary +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# In the following section we enforce positivity when finding the dictionary. + +# %% +dict_pos_dict_estimator = decomposition.MiniBatchDictionaryLearning( + n_components=n_components, + alpha=0.1, + max_iter=50, + batch_size=3, + random_state=rng, + positive_dict=True, +) +dict_pos_dict_estimator.fit(faces_centered) +plot_gallery( + "Dictionary learning - positive dictionary", + dict_pos_dict_estimator.components_[:n_components], + cmap=plt.cm.RdBu, +) + +# %% +# Dictionary learning - positive code +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# Below we constrain the coding coefficients as a positive matrix. + +# %% +dict_pos_code_estimator = decomposition.MiniBatchDictionaryLearning( + n_components=n_components, + alpha=0.1, + max_iter=50, + batch_size=3, + fit_algorithm="cd", + random_state=rng, + positive_code=True, +) +dict_pos_code_estimator.fit(faces_centered) +plot_gallery( + "Dictionary learning - positive code", + dict_pos_code_estimator.components_[:n_components], + cmap=plt.cm.RdBu, +) + +# %% +# Dictionary learning - positive dictionary & code +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# Also below are the results if the dictionary values and coding +# coefficients are positively constrained. + +# %% +dict_pos_estimator = decomposition.MiniBatchDictionaryLearning( + n_components=n_components, + alpha=0.1, + max_iter=50, + batch_size=3, + fit_algorithm="cd", + random_state=rng, + positive_dict=True, + positive_code=True, +) +dict_pos_estimator.fit(faces_centered) +plot_gallery( + "Dictionary learning - positive dictionary & code", + dict_pos_estimator.components_[:n_components], + cmap=plt.cm.RdBu, +) diff --git a/examples/decomposition/plot_ica_blind_source_separation.py b/examples/decomposition/plot_ica_blind_source_separation.py index b405b1770cd34..e8d571d814a1b 100644 --- a/examples/decomposition/plot_ica_blind_source_separation.py +++ b/examples/decomposition/plot_ica_blind_source_separation.py @@ -13,16 +13,17 @@ non-Gaussian processes. """ -print(__doc__) + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Generate sample data +# -------------------- import numpy as np -import matplotlib.pyplot as plt from scipy import signal -from sklearn.decomposition import FastICA, PCA - -# ############################################################################# -# Generate sample data np.random.seed(0) n_samples = 2000 time = np.linspace(0, 8, n_samples) @@ -39,8 +40,14 @@ A = np.array([[1, 1, 1], [0.5, 2, 1.0], [1.5, 1.0, 2.0]]) # Mixing matrix X = np.dot(S, A.T) # Generate observations +# %% +# Fit ICA and PCA models +# ---------------------- + +from sklearn.decomposition import PCA, FastICA + # Compute ICA -ica = FastICA(n_components=3) +ica = FastICA(n_components=3, whiten="arbitrary-variance") S_ = ica.fit_transform(X) # Reconstruct signals A_ = ica.mixing_ # Get estimated mixing matrix @@ -51,17 +58,22 @@ pca = PCA(n_components=3) H = pca.fit_transform(X) # Reconstruct signals based on orthogonal components -# ############################################################################# +# %% # Plot results +# ------------ + +import matplotlib.pyplot as plt plt.figure() models = [X, S, S_, H] -names = ['Observations (mixed signal)', - 'True Sources', - 'ICA recovered signals', - 'PCA recovered signals'] -colors = ['red', 'steelblue', 'orange'] +names = [ + "Observations (mixed signal)", + "True Sources", + "ICA recovered signals", + "PCA recovered signals", +] +colors = ["red", "steelblue", "orange"] for ii, (model, name) in enumerate(zip(models, names), 1): plt.subplot(4, 1, ii) diff --git a/examples/decomposition/plot_ica_vs_pca.py b/examples/decomposition/plot_ica_vs_pca.py index 21afb6b74e906..eb6ef9aad35c1 100644 --- a/examples/decomposition/plot_ica_vs_pca.py +++ b/examples/decomposition/plot_ica_vs_pca.py @@ -26,22 +26,22 @@ after whitening by the variance corresponding to the PCA vectors (lower left). Running ICA corresponds to finding a rotation in this space to identify the directions of largest non-Gaussianity (lower right). + """ -print(__doc__) -# Authors: Alexandre Gramfort, Gael Varoquaux -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +# %% +# Generate sample data +# -------------------- import numpy as np -import matplotlib.pyplot as plt from sklearn.decomposition import PCA, FastICA -# ############################################################################# -# Generate sample data rng = np.random.RandomState(42) S = rng.standard_t(1.5, size=(20000, 2)) -S[:, 0] *= 2. +S[:, 0] *= 2.0 # Mix data A = np.array([[1, 1], [0, 2]]) # Mixing matrix @@ -51,55 +51,65 @@ pca = PCA() S_pca_ = pca.fit(X).transform(X) -ica = FastICA(random_state=rng) +ica = FastICA(random_state=rng, whiten="arbitrary-variance") S_ica_ = ica.fit(X).transform(X) # Estimate the sources -S_ica_ /= S_ica_.std(axis=0) - -# ############################################################################# +# %% # Plot results +# ------------ +import matplotlib.pyplot as plt + def plot_samples(S, axis_list=None): - plt.scatter(S[:, 0], S[:, 1], s=2, marker='o', zorder=10, - color='steelblue', alpha=0.5) + plt.scatter( + S[:, 0], S[:, 1], s=2, marker="o", zorder=10, color="steelblue", alpha=0.5 + ) if axis_list is not None: - colors = ['orange', 'red'] - for color, axis in zip(colors, axis_list): - axis /= axis.std() - x_axis, y_axis = axis - # Trick to get legend to work - plt.plot(0.1 * x_axis, 0.1 * y_axis, linewidth=2, color=color) - plt.quiver((0, 0), (0, 0), x_axis, y_axis, zorder=11, width=0.01, - scale=6, color=color) - - plt.hlines(0, -3, 3) - plt.vlines(0, -3, 3) - plt.xlim(-3, 3) + for axis, color, label in axis_list: + x_axis, y_axis = axis / axis.std() + plt.quiver( + (0, 0), + (0, 0), + x_axis, + y_axis, + zorder=11, + width=0.01, + scale=6, + color=color, + label=label, + ) + + plt.hlines(0, -5, 5, color="black", linewidth=0.5) + plt.vlines(0, -3, 3, color="black", linewidth=0.5) + plt.xlim(-5, 5) plt.ylim(-3, 3) - plt.xlabel('x') - plt.ylabel('y') + plt.gca().set_aspect("equal") + plt.xlabel("x") + plt.ylabel("y") + plt.figure() plt.subplot(2, 2, 1) plot_samples(S / S.std()) -plt.title('True Independent Sources') +plt.title("True Independent Sources") -axis_list = [pca.components_.T, ica.mixing_] +axis_list = [(pca.components_.T, "orange", "PCA"), (ica.mixing_, "red", "ICA")] plt.subplot(2, 2, 2) plot_samples(X / np.std(X), axis_list=axis_list) -legend = plt.legend(['PCA', 'ICA'], loc='upper right') +legend = plt.legend(loc="upper left") legend.set_zorder(100) -plt.title('Observations') +plt.title("Observations") plt.subplot(2, 2, 3) -plot_samples(S_pca_ / np.std(S_pca_, axis=0)) -plt.title('PCA recovered signals') +plot_samples(S_pca_ / np.std(S_pca_)) +plt.title("PCA recovered signals") plt.subplot(2, 2, 4) plot_samples(S_ica_ / np.std(S_ica_)) -plt.title('ICA recovered signals') +plt.title("ICA recovered signals") plt.subplots_adjust(0.09, 0.04, 0.94, 0.94, 0.26, 0.36) +plt.tight_layout() plt.show() diff --git a/examples/decomposition/plot_image_denoising.py b/examples/decomposition/plot_image_denoising.py index 41c285772e0e0..5248fdff5a8ca 100644 --- a/examples/decomposition/plot_image_denoising.py +++ b/examples/decomposition/plot_image_denoising.py @@ -31,121 +31,147 @@ necessarily related to visualisation. """ -print(__doc__) -from time import time +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import matplotlib.pyplot as plt +# %% +# Generate distorted image +# ------------------------ import numpy as np -import scipy as sp - -from sklearn.decomposition import MiniBatchDictionaryLearning -from sklearn.feature_extraction.image import extract_patches_2d -from sklearn.feature_extraction.image import reconstruct_from_patches_2d - -try: # SciPy >= 0.16 have face in misc - from scipy.misc import face - face = face(gray=True) +try: # Scipy >= 1.10 + from scipy.datasets import face except ImportError: - face = sp.face(gray=True) + from scipy.misc import face + +raccoon_face = face(gray=True) # Convert from uint8 representation with values between 0 and 255 to # a floating point representation with values between 0 and 1. -face = face / 255. +raccoon_face = raccoon_face / 255.0 # downsample for higher speed -face = face[::4, ::4] + face[1::4, ::4] + face[::4, 1::4] + face[1::4, 1::4] -face /= 4.0 -height, width = face.shape +raccoon_face = ( + raccoon_face[::4, ::4] + + raccoon_face[1::4, ::4] + + raccoon_face[::4, 1::4] + + raccoon_face[1::4, 1::4] +) +raccoon_face /= 4.0 +height, width = raccoon_face.shape # Distort the right half of the image -print('Distorting image...') -distorted = face.copy() -distorted[:, width // 2:] += 0.075 * np.random.randn(height, width // 2) +print("Distorting image...") +distorted = raccoon_face.copy() +distorted[:, width // 2 :] += 0.075 * np.random.randn(height, width // 2) + + +# %% +# Display the distorted image +# --------------------------- +import matplotlib.pyplot as plt + + +def show_with_diff(image, reference, title): + """Helper function to display denoising""" + plt.figure(figsize=(5, 3.3)) + plt.subplot(1, 2, 1) + plt.title("Image") + plt.imshow(image, vmin=0, vmax=1, cmap=plt.cm.gray, interpolation="nearest") + plt.xticks(()) + plt.yticks(()) + plt.subplot(1, 2, 2) + difference = image - reference + + plt.title("Difference (norm: %.2f)" % np.sqrt(np.sum(difference**2))) + plt.imshow( + difference, vmin=-0.5, vmax=0.5, cmap=plt.cm.PuOr, interpolation="nearest" + ) + plt.xticks(()) + plt.yticks(()) + plt.suptitle(title, size=16) + plt.subplots_adjust(0.02, 0.02, 0.98, 0.79, 0.02, 0.2) + + +show_with_diff(distorted, raccoon_face, "Distorted image") + + +# %% +# Extract reference patches +# ---------------------------- +from time import time + +from sklearn.feature_extraction.image import extract_patches_2d # Extract all reference patches from the left half of the image -print('Extracting reference patches...') +print("Extracting reference patches...") t0 = time() patch_size = (7, 7) -data = extract_patches_2d(distorted[:, :width // 2], patch_size) +data = extract_patches_2d(distorted[:, : width // 2], patch_size) data = data.reshape(data.shape[0], -1) data -= np.mean(data, axis=0) data /= np.std(data, axis=0) -print('done in %.2fs.' % (time() - t0)) +print(f"{data.shape[0]} patches extracted in %.2fs." % (time() - t0)) + -# ############################################################################# +# %% # Learn the dictionary from reference patches +# ------------------------------------------- +from sklearn.decomposition import MiniBatchDictionaryLearning -print('Learning the dictionary...') +print("Learning the dictionary...") t0 = time() -dico = MiniBatchDictionaryLearning(n_components=100, alpha=1, n_iter=500) +dico = MiniBatchDictionaryLearning( + # increase to 300 for higher quality results at the cost of slower + # training times. + n_components=50, + batch_size=200, + alpha=1.0, + max_iter=10, +) V = dico.fit(data).components_ dt = time() - t0 -print('done in %.2fs.' % dt) +print(f"{dico.n_iter_} iterations / {dico.n_steps_} steps in {dt:.2f}.") plt.figure(figsize=(4.2, 4)) for i, comp in enumerate(V[:100]): plt.subplot(10, 10, i + 1) - plt.imshow(comp.reshape(patch_size), cmap=plt.cm.gray_r, - interpolation='nearest') + plt.imshow(comp.reshape(patch_size), cmap=plt.cm.gray_r, interpolation="nearest") plt.xticks(()) plt.yticks(()) -plt.suptitle('Dictionary learned from face patches\n' + - 'Train time %.1fs on %d patches' % (dt, len(data)), - fontsize=16) +plt.suptitle( + "Dictionary learned from face patches\n" + + "Train time %.1fs on %d patches" % (dt, len(data)), + fontsize=16, +) plt.subplots_adjust(0.08, 0.02, 0.92, 0.85, 0.08, 0.23) -# ############################################################################# -# Display the distorted image - -def show_with_diff(image, reference, title): - """Helper function to display denoising""" - plt.figure(figsize=(5, 3.3)) - plt.subplot(1, 2, 1) - plt.title('Image') - plt.imshow(image, vmin=0, vmax=1, cmap=plt.cm.gray, - interpolation='nearest') - plt.xticks(()) - plt.yticks(()) - plt.subplot(1, 2, 2) - difference = image - reference - - plt.title('Difference (norm: %.2f)' % np.sqrt(np.sum(difference ** 2))) - plt.imshow(difference, vmin=-0.5, vmax=0.5, cmap=plt.cm.PuOr, - interpolation='nearest') - plt.xticks(()) - plt.yticks(()) - plt.suptitle(title, size=16) - plt.subplots_adjust(0.02, 0.02, 0.98, 0.79, 0.02, 0.2) - -show_with_diff(distorted, face, 'Distorted image') - -# ############################################################################# +# %% # Extract noisy patches and reconstruct them using the dictionary +# --------------------------------------------------------------- +from sklearn.feature_extraction.image import reconstruct_from_patches_2d -print('Extracting noisy patches... ') +print("Extracting noisy patches... ") t0 = time() -data = extract_patches_2d(distorted[:, width // 2:], patch_size) +data = extract_patches_2d(distorted[:, width // 2 :], patch_size) data = data.reshape(data.shape[0], -1) intercept = np.mean(data, axis=0) data -= intercept -print('done in %.2fs.' % (time() - t0)) +print("done in %.2fs." % (time() - t0)) transform_algorithms = [ - ('Orthogonal Matching Pursuit\n1 atom', 'omp', - {'transform_n_nonzero_coefs': 1}), - ('Orthogonal Matching Pursuit\n2 atoms', 'omp', - {'transform_n_nonzero_coefs': 2}), - ('Least-angle regression\n5 atoms', 'lars', - {'transform_n_nonzero_coefs': 5}), - ('Thresholding\n alpha=0.1', 'threshold', {'transform_alpha': .1})] + ("Orthogonal Matching Pursuit\n1 atom", "omp", {"transform_n_nonzero_coefs": 1}), + ("Orthogonal Matching Pursuit\n2 atoms", "omp", {"transform_n_nonzero_coefs": 2}), + ("Least-angle regression\n4 atoms", "lars", {"transform_n_nonzero_coefs": 4}), + ("Thresholding\n alpha=0.1", "threshold", {"transform_alpha": 0.1}), +] reconstructions = {} for title, transform_algorithm, kwargs in transform_algorithms: - print(title + '...') - reconstructions[title] = face.copy() + print(title + "...") + reconstructions[title] = raccoon_face.copy() t0 = time() dico.set_params(transform_algorithm=transform_algorithm, **kwargs) code = dico.transform(data) @@ -153,14 +179,14 @@ def show_with_diff(image, reference, title): patches += intercept patches = patches.reshape(len(data), *patch_size) - if transform_algorithm == 'threshold': + if transform_algorithm == "threshold": patches -= patches.min() patches /= patches.max() - reconstructions[title][:, width // 2:] = reconstruct_from_patches_2d( - patches, (height, width // 2)) + reconstructions[title][:, width // 2 :] = reconstruct_from_patches_2d( + patches, (height, width // 2) + ) dt = time() - t0 - print('done in %.2fs.' % dt) - show_with_diff(reconstructions[title], face, - title + ' (time: %.1fs)' % dt) + print("done in %.2fs." % dt) + show_with_diff(reconstructions[title], raccoon_face, title + " (time: %.1fs)" % dt) plt.show() diff --git a/examples/decomposition/plot_incremental_pca.py b/examples/decomposition/plot_incremental_pca.py index 980f9d019ea1c..b10618dcddf00 100644 --- a/examples/decomposition/plot_incremental_pca.py +++ b/examples/decomposition/plot_incremental_pca.py @@ -18,13 +18,12 @@ incremental approaches. """ -print(__doc__) -# Authors: Kyle Kastner -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt +import numpy as np from sklearn.datasets import load_iris from sklearn.decomposition import PCA, IncrementalPCA @@ -40,18 +39,22 @@ pca = PCA(n_components=n_components) X_pca = pca.fit_transform(X) -colors = ['navy', 'turquoise', 'darkorange'] +colors = ["navy", "turquoise", "darkorange"] for X_transformed, title in [(X_ipca, "Incremental PCA"), (X_pca, "PCA")]: plt.figure(figsize=(8, 8)) for color, i, target_name in zip(colors, [0, 1, 2], iris.target_names): - plt.scatter(X_transformed[y == i, 0], X_transformed[y == i, 1], - color=color, lw=2, label=target_name) + plt.scatter( + X_transformed[y == i, 0], + X_transformed[y == i, 1], + color=color, + lw=2, + label=target_name, + ) if "Incremental" in title: err = np.abs(np.abs(X_pca) - np.abs(X_ipca)).mean() - plt.title(title + " of iris dataset\nMean absolute unsigned error " - "%.6f" % err) + plt.title(title + " of iris dataset\nMean absolute unsigned error %.6f" % err) else: plt.title(title + " of iris dataset") plt.legend(loc="best", shadow=False, scatterpoints=1) diff --git a/examples/decomposition/plot_kernel_pca.py b/examples/decomposition/plot_kernel_pca.py index cfec4f4ec8b1d..0c3a148c7c753 100644 --- a/examples/decomposition/plot_kernel_pca.py +++ b/examples/decomposition/plot_kernel_pca.py @@ -3,78 +3,161 @@ Kernel PCA ========== -This example shows that Kernel PCA is able to find a projection of the data -that makes data linearly separable. +This example shows the difference between the Principal Components Analysis +(:class:`~sklearn.decomposition.PCA`) and its kernelized version +(:class:`~sklearn.decomposition.KernelPCA`). + +On the one hand, we show that :class:`~sklearn.decomposition.KernelPCA` is able +to find a projection of the data which linearly separates them while it is not the case +with :class:`~sklearn.decomposition.PCA`. + +Finally, we show that inverting this projection is an approximation with +:class:`~sklearn.decomposition.KernelPCA`, while it is exact with +:class:`~sklearn.decomposition.PCA`. """ -print(__doc__) -# Authors: Mathieu Blondel -# Andreas Mueller -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Projecting data: `PCA` vs. `KernelPCA` +# -------------------------------------- +# +# In this section, we show the advantages of using a kernel when +# projecting data using a Principal Component Analysis (PCA). We create a +# dataset made of two nested circles. +from sklearn.datasets import make_circles +from sklearn.model_selection import train_test_split + +X, y = make_circles(n_samples=1_000, factor=0.3, noise=0.05, random_state=0) +X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) -import numpy as np +# %% +# Let's have a quick first look at the generated dataset. import matplotlib.pyplot as plt +_, (train_ax, test_ax) = plt.subplots(ncols=2, sharex=True, sharey=True, figsize=(8, 4)) + +train_ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train) +train_ax.set_ylabel("Feature #1") +train_ax.set_xlabel("Feature #0") +train_ax.set_title("Training data") + +test_ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test) +test_ax.set_xlabel("Feature #0") +_ = test_ax.set_title("Testing data") + +# %% +# The samples from each class cannot be linearly separated: there is no +# straight line that can split the samples of the inner set from the outer +# set. +# +# Now, we will use PCA with and without a kernel to see what is the effect of +# using such a kernel. The kernel used here is a radial basis function (RBF) +# kernel. from sklearn.decomposition import PCA, KernelPCA -from sklearn.datasets import make_circles -np.random.seed(0) - -X, y = make_circles(n_samples=400, factor=.3, noise=.05) - -kpca = KernelPCA(kernel="rbf", fit_inverse_transform=True, gamma=10) -X_kpca = kpca.fit_transform(X) -X_back = kpca.inverse_transform(X_kpca) -pca = PCA() -X_pca = pca.fit_transform(X) - -# Plot results - -plt.figure() -plt.subplot(2, 2, 1, aspect='equal') -plt.title("Original space") -reds = y == 0 -blues = y == 1 - -plt.scatter(X[reds, 0], X[reds, 1], c="red", - s=20, edgecolor='k') -plt.scatter(X[blues, 0], X[blues, 1], c="blue", - s=20, edgecolor='k') -plt.xlabel("$x_1$") -plt.ylabel("$x_2$") - -X1, X2 = np.meshgrid(np.linspace(-1.5, 1.5, 50), np.linspace(-1.5, 1.5, 50)) -X_grid = np.array([np.ravel(X1), np.ravel(X2)]).T -# projection on the first principal component (in the phi space) -Z_grid = kpca.transform(X_grid)[:, 0].reshape(X1.shape) -plt.contour(X1, X2, Z_grid, colors='grey', linewidths=1, origin='lower') - -plt.subplot(2, 2, 2, aspect='equal') -plt.scatter(X_pca[reds, 0], X_pca[reds, 1], c="red", - s=20, edgecolor='k') -plt.scatter(X_pca[blues, 0], X_pca[blues, 1], c="blue", - s=20, edgecolor='k') -plt.title("Projection by PCA") -plt.xlabel("1st principal component") -plt.ylabel("2nd component") - -plt.subplot(2, 2, 3, aspect='equal') -plt.scatter(X_kpca[reds, 0], X_kpca[reds, 1], c="red", - s=20, edgecolor='k') -plt.scatter(X_kpca[blues, 0], X_kpca[blues, 1], c="blue", - s=20, edgecolor='k') -plt.title("Projection by KPCA") -plt.xlabel(r"1st principal component in space induced by $\phi$") -plt.ylabel("2nd component") - -plt.subplot(2, 2, 4, aspect='equal') -plt.scatter(X_back[reds, 0], X_back[reds, 1], c="red", - s=20, edgecolor='k') -plt.scatter(X_back[blues, 0], X_back[blues, 1], c="blue", - s=20, edgecolor='k') -plt.title("Original space after inverse transform") -plt.xlabel("$x_1$") -plt.ylabel("$x_2$") - -plt.tight_layout() -plt.show() +pca = PCA(n_components=2) +kernel_pca = KernelPCA( + n_components=None, kernel="rbf", gamma=10, fit_inverse_transform=True, alpha=0.1 +) + +X_test_pca = pca.fit(X_train).transform(X_test) +X_test_kernel_pca = kernel_pca.fit(X_train).transform(X_test) + +# %% +fig, (orig_data_ax, pca_proj_ax, kernel_pca_proj_ax) = plt.subplots( + ncols=3, figsize=(14, 4) +) + +orig_data_ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test) +orig_data_ax.set_ylabel("Feature #1") +orig_data_ax.set_xlabel("Feature #0") +orig_data_ax.set_title("Testing data") + +pca_proj_ax.scatter(X_test_pca[:, 0], X_test_pca[:, 1], c=y_test) +pca_proj_ax.set_ylabel("Principal component #1") +pca_proj_ax.set_xlabel("Principal component #0") +pca_proj_ax.set_title("Projection of testing data\n using PCA") + +kernel_pca_proj_ax.scatter(X_test_kernel_pca[:, 0], X_test_kernel_pca[:, 1], c=y_test) +kernel_pca_proj_ax.set_ylabel("Principal component #1") +kernel_pca_proj_ax.set_xlabel("Principal component #0") +_ = kernel_pca_proj_ax.set_title("Projection of testing data\n using KernelPCA") + +# %% +# We recall that PCA transforms the data linearly. Intuitively, it means that +# the coordinate system will be centered, rescaled on each component +# with respected to its variance and finally be rotated. +# The obtained data from this transformation is isotropic and can now be +# projected on its *principal components*. +# +# Thus, looking at the projection made using PCA (i.e. the middle figure), we +# see that there is no change regarding the scaling; indeed the data being two +# concentric circles centered in zero, the original data is already isotropic. +# However, we can see that the data have been rotated. As a +# conclusion, we see that such a projection would not help if define a linear +# classifier to distinguish samples from both classes. +# +# Using a kernel allows to make a non-linear projection. Here, by using an RBF +# kernel, we expect that the projection will unfold the dataset while keeping +# approximately preserving the relative distances of pairs of data points that +# are close to one another in the original space. +# +# We observe such behaviour in the figure on the right: the samples of a given +# class are closer to each other than the samples from the opposite class, +# untangling both sample sets. Now, we can use a linear classifier to separate +# the samples from the two classes. +# +# Projecting into the original feature space +# ------------------------------------------ +# +# One particularity to have in mind when using +# :class:`~sklearn.decomposition.KernelPCA` is related to the reconstruction +# (i.e. the back projection in the original feature space). With +# :class:`~sklearn.decomposition.PCA`, the reconstruction will be exact if +# `n_components` is the same than the number of original features. +# This is the case in this example. +# +# We can investigate if we get the original dataset when back projecting with +# :class:`~sklearn.decomposition.KernelPCA`. +X_reconstructed_pca = pca.inverse_transform(pca.transform(X_test)) +X_reconstructed_kernel_pca = kernel_pca.inverse_transform(kernel_pca.transform(X_test)) + +# %% +fig, (orig_data_ax, pca_back_proj_ax, kernel_pca_back_proj_ax) = plt.subplots( + ncols=3, sharex=True, sharey=True, figsize=(13, 4) +) + +orig_data_ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test) +orig_data_ax.set_ylabel("Feature #1") +orig_data_ax.set_xlabel("Feature #0") +orig_data_ax.set_title("Original test data") + +pca_back_proj_ax.scatter(X_reconstructed_pca[:, 0], X_reconstructed_pca[:, 1], c=y_test) +pca_back_proj_ax.set_xlabel("Feature #0") +pca_back_proj_ax.set_title("Reconstruction via PCA") + +kernel_pca_back_proj_ax.scatter( + X_reconstructed_kernel_pca[:, 0], X_reconstructed_kernel_pca[:, 1], c=y_test +) +kernel_pca_back_proj_ax.set_xlabel("Feature #0") +_ = kernel_pca_back_proj_ax.set_title("Reconstruction via KernelPCA") + +# %% +# While we see a perfect reconstruction with +# :class:`~sklearn.decomposition.PCA` we observe a different result for +# :class:`~sklearn.decomposition.KernelPCA`. +# +# Indeed, :meth:`~sklearn.decomposition.KernelPCA.inverse_transform` cannot +# rely on an analytical back-projection and thus an exact reconstruction. +# Instead, a :class:`~sklearn.kernel_ridge.KernelRidge` is internally trained +# to learn a mapping from the kernalized PCA basis to the original feature +# space. This method therefore comes with an approximation introducing small +# differences when back projecting in the original feature space. +# +# To improve the reconstruction using +# :meth:`~sklearn.decomposition.KernelPCA.inverse_transform`, one can tune +# `alpha` in :class:`~sklearn.decomposition.KernelPCA`, the regularization term +# which controls the reliance on the training data during the training of +# the mapping. diff --git a/examples/decomposition/plot_pca_3d.py b/examples/decomposition/plot_pca_3d.py deleted file mode 100644 index f932d698adc8b..0000000000000 --- a/examples/decomposition/plot_pca_3d.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -""" -========================================================= -Principal components analysis (PCA) -========================================================= - -These figures aid in illustrating how a point cloud -can be very flat in one direction--which is where PCA -comes in to choose a direction that is not flat. - -""" -print(__doc__) - -# Authors: Gael Varoquaux -# Jaques Grobler -# Kevin Hughes -# License: BSD 3 clause - -from sklearn.decomposition import PCA - -from mpl_toolkits.mplot3d import Axes3D -import numpy as np -import matplotlib.pyplot as plt -from scipy import stats - - -# ############################################################################# -# Create the data - -e = np.exp(1) -np.random.seed(4) - - -def pdf(x): - return 0.5 * (stats.norm(scale=0.25 / e).pdf(x) - + stats.norm(scale=4 / e).pdf(x)) - -y = np.random.normal(scale=0.5, size=(30000)) -x = np.random.normal(scale=0.5, size=(30000)) -z = np.random.normal(scale=0.1, size=len(x)) - -density = pdf(x) * pdf(y) -pdf_z = pdf(5 * z) - -density *= pdf_z - -a = x + y -b = 2 * y -c = a - b + z - -norm = np.sqrt(a.var() + b.var()) -a /= norm -b /= norm - - -# ############################################################################# -# Plot the figures -def plot_figs(fig_num, elev, azim): - fig = plt.figure(fig_num, figsize=(4, 3)) - plt.clf() - ax = Axes3D(fig, rect=[0, 0, .95, 1], elev=elev, azim=azim) - - ax.scatter(a[::10], b[::10], c[::10], c=density[::10], marker='+', alpha=.4) - Y = np.c_[a, b, c] - - # Using SciPy's SVD, this would be: - # _, pca_score, Vt = scipy.linalg.svd(Y, full_matrices=False) - - pca = PCA(n_components=3) - pca.fit(Y) - V = pca.components_.T - - x_pca_axis, y_pca_axis, z_pca_axis = 3 * V - x_pca_plane = np.r_[x_pca_axis[:2], - x_pca_axis[1::-1]] - y_pca_plane = np.r_[y_pca_axis[:2], - y_pca_axis[1::-1]] - z_pca_plane = np.r_[z_pca_axis[:2], - z_pca_axis[1::-1]] - x_pca_plane.shape = (2, 2) - y_pca_plane.shape = (2, 2) - z_pca_plane.shape = (2, 2) - ax.plot_surface(x_pca_plane, y_pca_plane, z_pca_plane) - ax.w_xaxis.set_ticklabels([]) - ax.w_yaxis.set_ticklabels([]) - ax.w_zaxis.set_ticklabels([]) - - -elev = -40 -azim = -80 -plot_figs(1, elev, azim) - -elev = 30 -azim = 20 -plot_figs(2, elev, azim) - -plt.show() diff --git a/examples/decomposition/plot_pca_iris.py b/examples/decomposition/plot_pca_iris.py index 77a871ec9af84..e6e61341c0f8a 100644 --- a/examples/decomposition/plot_pca_iris.py +++ b/examples/decomposition/plot_pca_iris.py @@ -1,60 +1,105 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - """ -========================================================= -PCA example with Iris Data-set -========================================================= - -Principal Component Analysis applied to the Iris dataset. +================================================== +Principal Component Analysis (PCA) on Iris Dataset +================================================== -See `here `_ for more -information on this dataset. +This example shows a well known decomposition technique known as Principal Component +Analysis (PCA) on the +`Iris dataset `_. +This dataset is made of 4 features: sepal length, sepal width, petal length, petal +width. We use PCA to project this 4 feature space into a 3-dimensional space. """ -print(__doc__) - -# Code source: GaÃĢl Varoquaux -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Loading the Iris dataset +# ------------------------ +# +# The Iris dataset is directly available as part of scikit-learn. It can be loaded +# using the :func:`~sklearn.datasets.load_iris` function. With the default parameters, +# a :class:`~sklearn.utils.Bunch` object is returned, containing the data, the +# target values, the feature names, and the target names. +from sklearn.datasets import load_iris + +iris = load_iris(as_frame=True) +print(iris.keys()) + +# %% +# Plot of pairs of features of the Iris dataset +# --------------------------------------------- +# +# Let's first plot the pairs of features of the Iris dataset. +import seaborn as sns + +# Rename classes using the iris target names +iris.frame["target"] = iris.target_names[iris.target] +_ = sns.pairplot(iris.frame, hue="target") + +# %% +# Each data point on each scatter plot refers to one of the 150 iris flowers +# in the dataset, with the color indicating their respective type +# (Setosa, Versicolor, and Virginica). +# +# You can already see a pattern regarding the Setosa type, which is +# easily identifiable based on its short and wide sepal. Only +# considering these two dimensions, sepal width and length, there's still +# overlap between the Versicolor and Virginica types. +# +# The diagonal of the plot shows the distribution of each feature. We observe +# that the petal width and the petal length are the most discriminant features +# for the three types. +# +# Plot a PCA representation +# ------------------------- +# Let's apply a Principal Component Analysis (PCA) to the iris dataset +# and then plot the irises across the first three PCA dimensions. +# This will allow us to better differentiate among the three types! -import numpy as np import matplotlib.pyplot as plt -from mpl_toolkits.mplot3d import Axes3D - - -from sklearn import decomposition -from sklearn import datasets - -np.random.seed(5) -centers = [[1, 1], [-1, -1], [1, -1]] -iris = datasets.load_iris() -X = iris.data -y = iris.target - -fig = plt.figure(1, figsize=(4, 3)) -plt.clf() -ax = Axes3D(fig, rect=[0, 0, .95, 1], elev=48, azim=134) - -plt.cla() -pca = decomposition.PCA(n_components=3) -pca.fit(X) -X = pca.transform(X) - -for name, label in [('Setosa', 0), ('Versicolour', 1), ('Virginica', 2)]: - ax.text3D(X[y == label, 0].mean(), - X[y == label, 1].mean() + 1.5, - X[y == label, 2].mean(), name, - horizontalalignment='center', - bbox=dict(alpha=.5, edgecolor='w', facecolor='w')) -# Reorder the labels to have colors matching the cluster results -y = np.choose(y, [1, 2, 0]).astype(float) -ax.scatter(X[:, 0], X[:, 1], X[:, 2], c=y, cmap=plt.cm.nipy_spectral, - edgecolor='k') - -ax.w_xaxis.set_ticklabels([]) -ax.w_yaxis.set_ticklabels([]) -ax.w_zaxis.set_ticklabels([]) +# unused but required import for doing 3d projections with matplotlib < 3.2 +import mpl_toolkits.mplot3d # noqa: F401 + +from sklearn.decomposition import PCA + +fig = plt.figure(1, figsize=(8, 6)) +ax = fig.add_subplot(111, projection="3d", elev=-150, azim=110) + +X_reduced = PCA(n_components=3).fit_transform(iris.data) +scatter = ax.scatter( + X_reduced[:, 0], + X_reduced[:, 1], + X_reduced[:, 2], + c=iris.target, + s=40, +) + +ax.set( + title="First three PCA dimensions", + xlabel="1st Eigenvector", + ylabel="2nd Eigenvector", + zlabel="3rd Eigenvector", +) +ax.xaxis.set_ticklabels([]) +ax.yaxis.set_ticklabels([]) +ax.zaxis.set_ticklabels([]) + +# Add a legend +legend1 = ax.legend( + scatter.legend_elements()[0], + iris.target_names.tolist(), + loc="upper right", + title="Classes", +) +ax.add_artist(legend1) plt.show() + +# %% +# PCA will create 3 new features that are a linear combination of the 4 original +# features. In addition, this transformation maximizes the variance. With this +# transformation, we see that we can identify each species using only the first feature +# (i.e., first eigenvector). diff --git a/examples/decomposition/plot_pca_vs_fa_model_selection.py b/examples/decomposition/plot_pca_vs_fa_model_selection.py index b858434d910e3..865d69989255a 100644 --- a/examples/decomposition/plot_pca_vs_fa_model_selection.py +++ b/examples/decomposition/plot_pca_vs_fa_model_selection.py @@ -16,7 +16,8 @@ in recovering the size of the low rank subspace. The likelihood with PCA is higher than FA in this case. However PCA fails and overestimates the rank when heteroscedastic noise is present. Under appropriate -circumstances the low rank models are more likely than shrinkage models. +circumstances (choice of the number of components), the held-out +data is more likely for low rank models than for shrinkage models. The automatic estimation from Automatic Choice of Dimensionality for PCA. NIPS 2000: 598-604 @@ -24,26 +25,18 @@ """ -# Authors: Alexandre Gramfort -# Denis A. Engemann -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Create the data +# --------------- import numpy as np -import matplotlib.pyplot as plt from scipy import linalg -from sklearn.decomposition import PCA, FactorAnalysis -from sklearn.covariance import ShrunkCovariance, LedoitWolf -from sklearn.model_selection import cross_val_score -from sklearn.model_selection import GridSearchCV - -print(__doc__) - -# ############################################################################# -# Create the data - -n_samples, n_features, rank = 1000, 50, 10 -sigma = 1. +n_samples, n_features, rank = 500, 25, 5 +sigma = 1.0 rng = np.random.RandomState(42) U, _, _ = linalg.svd(rng.randn(n_features, n_features)) X = np.dot(rng.randn(n_samples, rank), U[:, :rank].T) @@ -52,17 +45,24 @@ X_homo = X + sigma * rng.randn(n_samples, n_features) # Adding heteroscedastic noise -sigmas = sigma * rng.rand(n_features) + sigma / 2. +sigmas = sigma * rng.rand(n_features) + sigma / 2.0 X_hetero = X + rng.randn(n_samples, n_features) * sigmas -# ############################################################################# +# %% # Fit the models +# -------------- + +import matplotlib.pyplot as plt + +from sklearn.covariance import LedoitWolf, ShrunkCovariance +from sklearn.decomposition import PCA, FactorAnalysis +from sklearn.model_selection import GridSearchCV, cross_val_score n_components = np.arange(0, n_features, 5) # options for n_components def compute_scores(X): - pca = PCA(svd_solver='full') + pca = PCA(svd_solver="full") fa = FactorAnalysis() pca_scores, fa_scores = [], [] @@ -77,7 +77,7 @@ def compute_scores(X): def shrunk_cov_score(X): shrinkages = np.logspace(-2, 0, 30) - cv = GridSearchCV(ShrunkCovariance(), {'shrinkage': shrinkages}) + cv = GridSearchCV(ShrunkCovariance(), {"shrinkage": shrinkages}) return np.mean(cross_val_score(cv.fit(X).best_estimator_, X)) @@ -85,13 +85,12 @@ def lw_score(X): return np.mean(cross_val_score(LedoitWolf(), X)) -for X, title in [(X_homo, 'Homoscedastic Noise'), - (X_hetero, 'Heteroscedastic Noise')]: +for X, title in [(X_homo, "Homoscedastic Noise"), (X_hetero, "Heteroscedastic Noise")]: pca_scores, fa_scores = compute_scores(X) n_components_pca = n_components[np.argmax(pca_scores)] n_components_fa = n_components[np.argmax(fa_scores)] - pca = PCA(svd_solver='full', n_components='mle') + pca = PCA(svd_solver="full", n_components="mle") pca.fit(X) n_components_pca_mle = pca.n_components_ @@ -100,26 +99,45 @@ def lw_score(X): print("best n_components by PCA MLE = %d" % n_components_pca_mle) plt.figure() - plt.plot(n_components, pca_scores, 'b', label='PCA scores') - plt.plot(n_components, fa_scores, 'r', label='FA scores') - plt.axvline(rank, color='g', label='TRUTH: %d' % rank, linestyle='-') - plt.axvline(n_components_pca, color='b', - label='PCA CV: %d' % n_components_pca, linestyle='--') - plt.axvline(n_components_fa, color='r', - label='FactorAnalysis CV: %d' % n_components_fa, - linestyle='--') - plt.axvline(n_components_pca_mle, color='k', - label='PCA MLE: %d' % n_components_pca_mle, linestyle='--') + plt.plot(n_components, pca_scores, "b", label="PCA scores") + plt.plot(n_components, fa_scores, "r", label="FA scores") + plt.axvline(rank, color="g", label="TRUTH: %d" % rank, linestyle="-") + plt.axvline( + n_components_pca, + color="b", + label="PCA CV: %d" % n_components_pca, + linestyle="--", + ) + plt.axvline( + n_components_fa, + color="r", + label="FactorAnalysis CV: %d" % n_components_fa, + linestyle="--", + ) + plt.axvline( + n_components_pca_mle, + color="k", + label="PCA MLE: %d" % n_components_pca_mle, + linestyle="--", + ) # compare with other covariance estimators - plt.axhline(shrunk_cov_score(X), color='violet', - label='Shrunk Covariance MLE', linestyle='-.') - plt.axhline(lw_score(X), color='orange', - label='LedoitWolf MLE' % n_components_pca_mle, linestyle='-.') - - plt.xlabel('nb of components') - plt.ylabel('CV scores') - plt.legend(loc='lower right') + plt.axhline( + shrunk_cov_score(X), + color="violet", + label="Shrunk Covariance MLE", + linestyle="-.", + ) + plt.axhline( + lw_score(X), + color="orange", + label="LedoitWolf MLE" % n_components_pca_mle, + linestyle="-.", + ) + + plt.xlabel("nb of components") + plt.ylabel("CV scores") + plt.legend(loc="lower right") plt.title(title) plt.show() diff --git a/examples/decomposition/plot_pca_vs_lda.py b/examples/decomposition/plot_pca_vs_lda.py index 051b96ffedf2c..4679a410af76a 100644 --- a/examples/decomposition/plot_pca_vs_lda.py +++ b/examples/decomposition/plot_pca_vs_lda.py @@ -15,8 +15,11 @@ Linear Discriminant Analysis (LDA) tries to identify attributes that account for the most variance *between classes*. In particular, LDA, in contrast to PCA, is a supervised method, using known class labels. + """ -print(__doc__) + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import matplotlib.pyplot as plt @@ -37,24 +40,28 @@ X_r2 = lda.fit(X, y).transform(X) # Percentage of variance explained for each components -print('explained variance ratio (first two components): %s' - % str(pca.explained_variance_ratio_)) +print( + "explained variance ratio (first two components): %s" + % str(pca.explained_variance_ratio_) +) plt.figure() -colors = ['navy', 'turquoise', 'darkorange'] +colors = ["navy", "turquoise", "darkorange"] lw = 2 for color, i, target_name in zip(colors, [0, 1, 2], target_names): - plt.scatter(X_r[y == i, 0], X_r[y == i, 1], color=color, alpha=.8, lw=lw, - label=target_name) -plt.legend(loc='best', shadow=False, scatterpoints=1) -plt.title('PCA of IRIS dataset') + plt.scatter( + X_r[y == i, 0], X_r[y == i, 1], color=color, alpha=0.8, lw=lw, label=target_name + ) +plt.legend(loc="best", shadow=False, scatterpoints=1) +plt.title("PCA of IRIS dataset") plt.figure() for color, i, target_name in zip(colors, [0, 1, 2], target_names): - plt.scatter(X_r2[y == i, 0], X_r2[y == i, 1], alpha=.8, color=color, - label=target_name) -plt.legend(loc='best', shadow=False, scatterpoints=1) -plt.title('LDA of IRIS dataset') + plt.scatter( + X_r2[y == i, 0], X_r2[y == i, 1], alpha=0.8, color=color, label=target_name + ) +plt.legend(loc="best", shadow=False, scatterpoints=1) +plt.title("LDA of IRIS dataset") plt.show() diff --git a/examples/decomposition/plot_sparse_coding.py b/examples/decomposition/plot_sparse_coding.py index 144401073a7fe..a3456b553486c 100644 --- a/examples/decomposition/plot_sparse_coding.py +++ b/examples/decomposition/plot_sparse_coding.py @@ -13,22 +13,26 @@ The richer dictionary on the right is not larger in size, heavier subsampling is performed in order to stay on the same order of magnitude. + """ -print(__doc__) -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt +import numpy as np from sklearn.decomposition import SparseCoder -from sklearn.utils.fixes import np_version, parse_version def ricker_function(resolution, center, width): """Discrete sub-sampled Ricker (Mexican hat) wavelet""" x = np.linspace(0, resolution - 1, resolution) - x = ((2 / (np.sqrt(3 * width) * np.pi ** .25)) - * (1 - (x - center) ** 2 / width ** 2) - * np.exp(-(x - center) ** 2 / (2 * width ** 2))) + x = ( + (2 / (np.sqrt(3 * width) * np.pi**0.25)) + * (1 - (x - center) ** 2 / width**2) + * np.exp(-((x - center) ** 2) / (2 * width**2)) + ) return x @@ -38,7 +42,7 @@ def ricker_matrix(width, resolution, n_components): D = np.empty((n_components, resolution)) for i, center in enumerate(centers): D[i] = ricker_function(resolution, center, width) - D /= np.sqrt(np.sum(D ** 2, axis=1))[:, np.newaxis] + D /= np.sqrt(np.sum(D**2, axis=1))[:, np.newaxis] return D @@ -48,57 +52,72 @@ def ricker_matrix(width, resolution, n_components): n_components = resolution // subsampling # Compute a wavelet dictionary -D_fixed = ricker_matrix(width=width, resolution=resolution, - n_components=n_components) -D_multi = np.r_[tuple(ricker_matrix(width=w, resolution=resolution, - n_components=n_components // 5) - for w in (10, 50, 100, 500, 1000))] +D_fixed = ricker_matrix(width=width, resolution=resolution, n_components=n_components) +D_multi = np.r_[ + tuple( + ricker_matrix(width=w, resolution=resolution, n_components=n_components // 5) + for w in (10, 50, 100, 500, 1000) + ) +] # Generate a signal y = np.linspace(0, resolution - 1, resolution) first_quarter = y < resolution / 4 -y[first_quarter] = 3. -y[np.logical_not(first_quarter)] = -1. +y[first_quarter] = 3.0 +y[np.logical_not(first_quarter)] = -1.0 # List the different sparse coding methods in the following format: # (title, transform_algorithm, transform_alpha, # transform_n_nozero_coefs, color) -estimators = [('OMP', 'omp', None, 15, 'navy'), - ('Lasso', 'lasso_lars', 2, None, 'turquoise'), ] +estimators = [ + ("OMP", "omp", None, 15, "navy"), + ("Lasso", "lasso_lars", 2, None, "turquoise"), +] lw = 2 -# Avoid FutureWarning about default value change when numpy >= 1.14 -lstsq_rcond = None if np_version >= parse_version('1.14') else -1 plt.figure(figsize=(13, 6)) -for subplot, (D, title) in enumerate(zip((D_fixed, D_multi), - ('fixed width', 'multiple widths'))): +for subplot, (D, title) in enumerate( + zip((D_fixed, D_multi), ("fixed width", "multiple widths")) +): plt.subplot(1, 2, subplot + 1) - plt.title('Sparse coding against %s dictionary' % title) - plt.plot(y, lw=lw, linestyle='--', label='Original signal') + plt.title("Sparse coding against %s dictionary" % title) + plt.plot(y, lw=lw, linestyle="--", label="Original signal") # Do a wavelet approximation for title, algo, alpha, n_nonzero, color in estimators: - coder = SparseCoder(dictionary=D, transform_n_nonzero_coefs=n_nonzero, - transform_alpha=alpha, transform_algorithm=algo) + coder = SparseCoder( + dictionary=D, + transform_n_nonzero_coefs=n_nonzero, + transform_alpha=alpha, + transform_algorithm=algo, + ) x = coder.transform(y.reshape(1, -1)) density = len(np.flatnonzero(x)) x = np.ravel(np.dot(x, D)) squared_error = np.sum((y - x) ** 2) - plt.plot(x, color=color, lw=lw, - label='%s: %s nonzero coefs,\n%.2f error' - % (title, density, squared_error)) + plt.plot( + x, + color=color, + lw=lw, + label="%s: %s nonzero coefs,\n%.2f error" % (title, density, squared_error), + ) # Soft thresholding debiasing - coder = SparseCoder(dictionary=D, transform_algorithm='threshold', - transform_alpha=20) + coder = SparseCoder( + dictionary=D, transform_algorithm="threshold", transform_alpha=20 + ) x = coder.transform(y.reshape(1, -1)) - _, idx = np.where(x != 0) - x[0, idx], _, _, _ = np.linalg.lstsq(D[idx, :].T, y, rcond=lstsq_rcond) + _, idx = (x != 0).nonzero() + x[0, idx], _, _, _ = np.linalg.lstsq(D[idx, :].T, y, rcond=None) x = np.ravel(np.dot(x, D)) squared_error = np.sum((y - x) ** 2) - plt.plot(x, color='darkorange', lw=lw, - label='Thresholding w/ debiasing:\n%d nonzero coefs, %.2f error' - % (len(idx), squared_error)) - plt.axis('tight') - plt.legend(shadow=False, loc='best') -plt.subplots_adjust(.04, .07, .97, .90, .09, .2) + plt.plot( + x, + color="darkorange", + lw=lw, + label="Thresholding w/ debiasing:\n%d nonzero coefs, %.2f error" + % (len(idx), squared_error), + ) + plt.axis("tight") + plt.legend(shadow=False, loc="best") +plt.subplots_adjust(0.04, 0.07, 0.97, 0.90, 0.09, 0.2) plt.show() diff --git a/examples/decomposition/plot_varimax_fa.py b/examples/decomposition/plot_varimax_fa.py index 3f850fd3b545d..4289464af5eea 100644 --- a/examples/decomposition/plot_varimax_fa.py +++ b/examples/decomposition/plot_varimax_fa.py @@ -7,25 +7,24 @@ length and petal width are highly correlated. Sepal width is less redundant. Matrix decomposition techniques can uncover these latent patterns. Applying rotations to the resulting -components does not inherently improve the predictve value +components does not inherently improve the predictive value of the derived latent space, but can help visualise their structure; here, for example, the varimax rotation, which is found by maximizing the squared variances of the weights, finds a structure where the second component only loads positively on sepal width. + """ -# Authors: Jona Sassenhagen -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import matplotlib.pyplot as plt import numpy as np -from sklearn.decomposition import FactorAnalysis, PCA -from sklearn.preprocessing import StandardScaler from sklearn.datasets import load_iris - -print(__doc__) +from sklearn.decomposition import PCA, FactorAnalysis +from sklearn.preprocessing import StandardScaler # %% # Load Iris data @@ -52,10 +51,12 @@ # Run factor analysis with Varimax rotation n_comps = 2 -methods = [('PCA', PCA()), - ('Unrotated FA', FactorAnalysis()), - ('Varimax FA', FactorAnalysis(rotation='varimax'))] -fig, axes = plt.subplots(ncols=len(methods), figsize=(10, 8)) +methods = [ + ("PCA", PCA()), + ("Unrotated FA", FactorAnalysis()), + ("Varimax FA", FactorAnalysis(rotation="varimax")), +] +fig, axes = plt.subplots(ncols=len(methods), figsize=(10, 8), sharey=True) for ax, (method, fa) in zip(axes, methods): fa.set_params(n_components=n_comps) @@ -68,10 +69,7 @@ vmax = np.abs(components).max() ax.imshow(components, cmap="RdBu_r", vmax=vmax, vmin=-vmax) ax.set_yticks(np.arange(len(feature_names))) - if ax.is_first_col(): - ax.set_yticklabels(feature_names) - else: - ax.set_yticklabels([]) + ax.set_yticklabels(feature_names) ax.set_title(str(method)) ax.set_xticks([0, 1]) ax.set_xticklabels(["Comp. 1", "Comp. 2"]) diff --git a/examples/developing_estimators/README.txt b/examples/developing_estimators/README.txt new file mode 100644 index 0000000000000..c9ec204812057 --- /dev/null +++ b/examples/developing_estimators/README.txt @@ -0,0 +1,6 @@ +.. _developing_estimator_examples: + +Developing Estimators +--------------------- + +Examples concerning the development of Custom Estimator. diff --git a/examples/developing_estimators/sklearn_is_fitted.py b/examples/developing_estimators/sklearn_is_fitted.py new file mode 100644 index 0000000000000..e5ebea7cd8641 --- /dev/null +++ b/examples/developing_estimators/sklearn_is_fitted.py @@ -0,0 +1,75 @@ +""" +======================================== +`__sklearn_is_fitted__` as Developer API +======================================== + +The `__sklearn_is_fitted__` method is a convention used in scikit-learn for +checking whether an estimator object has been fitted or not. This method is +typically implemented in custom estimator classes that are built on top of +scikit-learn's base classes like `BaseEstimator` or its subclasses. + +Developers should use :func:`~sklearn.utils.validation.check_is_fitted` +at the beginning of all methods except `fit`. If they need to customize or +speed-up the check, they can implement the `__sklearn_is_fitted__` method as +shown below. + +In this example the custom estimator showcases the usage of the +`__sklearn_is_fitted__` method and the `check_is_fitted` utility function +as developer APIs. The `__sklearn_is_fitted__` method checks fitted status +by verifying the presence of the `_is_fitted` attribute. +""" + +# %% +# An example custom estimator implementing a simple classifier +# ------------------------------------------------------------ +# This code snippet defines a custom estimator class called `CustomEstimator` +# that extends both the `BaseEstimator` and `ClassifierMixin` classes from +# scikit-learn and showcases the usage of the `__sklearn_is_fitted__` method +# and the `check_is_fitted` utility function. + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from sklearn.base import BaseEstimator, ClassifierMixin +from sklearn.utils.validation import check_is_fitted + + +class CustomEstimator(BaseEstimator, ClassifierMixin): + def __init__(self, parameter=1): + self.parameter = parameter + + def fit(self, X, y): + """ + Fit the estimator to the training data. + """ + self.classes_ = sorted(set(y)) + # Custom attribute to track if the estimator is fitted + self._is_fitted = True + return self + + def predict(self, X): + """ + Perform Predictions + + If the estimator is not fitted, then raise NotFittedError + """ + check_is_fitted(self) + # Perform prediction logic + predictions = [self.classes_[0]] * len(X) + return predictions + + def score(self, X, y): + """ + Calculate Score + + If the estimator is not fitted, then raise NotFittedError + """ + check_is_fitted(self) + # Perform scoring logic + return 0.5 + + def __sklearn_is_fitted__(self): + """ + Check fitted status and return a Boolean value. + """ + return hasattr(self, "_is_fitted") and self._is_fitted diff --git a/examples/ensemble/plot_adaboost_hastie_10_2.py b/examples/ensemble/plot_adaboost_hastie_10_2.py deleted file mode 100644 index 4d48d13dd24f2..0000000000000 --- a/examples/ensemble/plot_adaboost_hastie_10_2.py +++ /dev/null @@ -1,112 +0,0 @@ -""" -============================= -Discrete versus Real AdaBoost -============================= - -This example is based on Figure 10.2 from Hastie et al 2009 [1]_ and -illustrates the difference in performance between the discrete SAMME [2]_ -boosting algorithm and real SAMME.R boosting algorithm. Both algorithms are -evaluated on a binary classification task where the target Y is a non-linear -function of 10 input features. - -Discrete SAMME AdaBoost adapts based on errors in predicted class labels -whereas real SAMME.R uses the predicted class probabilities. - -.. [1] T. Hastie, R. Tibshirani and J. Friedman, "Elements of Statistical - Learning Ed. 2", Springer, 2009. - -.. [2] J. Zhu, H. Zou, S. Rosset, T. Hastie, "Multi-class AdaBoost", 2009. - -""" -print(__doc__) - -# Author: Peter Prettenhofer , -# Noel Dawe -# -# License: BSD 3 clause - -import numpy as np -import matplotlib.pyplot as plt - -from sklearn import datasets -from sklearn.tree import DecisionTreeClassifier -from sklearn.metrics import zero_one_loss -from sklearn.ensemble import AdaBoostClassifier - - -n_estimators = 400 -# A learning rate of 1. may not be optimal for both SAMME and SAMME.R -learning_rate = 1. - -X, y = datasets.make_hastie_10_2(n_samples=12000, random_state=1) - -X_test, y_test = X[2000:], y[2000:] -X_train, y_train = X[:2000], y[:2000] - -dt_stump = DecisionTreeClassifier(max_depth=1, min_samples_leaf=1) -dt_stump.fit(X_train, y_train) -dt_stump_err = 1.0 - dt_stump.score(X_test, y_test) - -dt = DecisionTreeClassifier(max_depth=9, min_samples_leaf=1) -dt.fit(X_train, y_train) -dt_err = 1.0 - dt.score(X_test, y_test) - -ada_discrete = AdaBoostClassifier( - base_estimator=dt_stump, - learning_rate=learning_rate, - n_estimators=n_estimators, - algorithm="SAMME") -ada_discrete.fit(X_train, y_train) - -ada_real = AdaBoostClassifier( - base_estimator=dt_stump, - learning_rate=learning_rate, - n_estimators=n_estimators, - algorithm="SAMME.R") -ada_real.fit(X_train, y_train) - -fig = plt.figure() -ax = fig.add_subplot(111) - -ax.plot([1, n_estimators], [dt_stump_err] * 2, 'k-', - label='Decision Stump Error') -ax.plot([1, n_estimators], [dt_err] * 2, 'k--', - label='Decision Tree Error') - -ada_discrete_err = np.zeros((n_estimators,)) -for i, y_pred in enumerate(ada_discrete.staged_predict(X_test)): - ada_discrete_err[i] = zero_one_loss(y_pred, y_test) - -ada_discrete_err_train = np.zeros((n_estimators,)) -for i, y_pred in enumerate(ada_discrete.staged_predict(X_train)): - ada_discrete_err_train[i] = zero_one_loss(y_pred, y_train) - -ada_real_err = np.zeros((n_estimators,)) -for i, y_pred in enumerate(ada_real.staged_predict(X_test)): - ada_real_err[i] = zero_one_loss(y_pred, y_test) - -ada_real_err_train = np.zeros((n_estimators,)) -for i, y_pred in enumerate(ada_real.staged_predict(X_train)): - ada_real_err_train[i] = zero_one_loss(y_pred, y_train) - -ax.plot(np.arange(n_estimators) + 1, ada_discrete_err, - label='Discrete AdaBoost Test Error', - color='red') -ax.plot(np.arange(n_estimators) + 1, ada_discrete_err_train, - label='Discrete AdaBoost Train Error', - color='blue') -ax.plot(np.arange(n_estimators) + 1, ada_real_err, - label='Real AdaBoost Test Error', - color='orange') -ax.plot(np.arange(n_estimators) + 1, ada_real_err_train, - label='Real AdaBoost Train Error', - color='green') - -ax.set_ylim((0.0, 0.5)) -ax.set_xlabel('n_estimators') -ax.set_ylabel('error rate') - -leg = ax.legend(loc='upper right', fancybox=True) -leg.get_frame().set_alpha(0.7) - -plt.show() diff --git a/examples/ensemble/plot_adaboost_multiclass.py b/examples/ensemble/plot_adaboost_multiclass.py index 0ee08c5ed322e..e0c30ae1586b6 100644 --- a/examples/ensemble/plot_adaboost_multiclass.py +++ b/examples/ensemble/plot_adaboost_multiclass.py @@ -1,118 +1,252 @@ -r""" +""" ===================================== Multi-class AdaBoosted Decision Trees ===================================== -This example reproduces Figure 1 of Zhu et al [1]_ and shows how boosting can -improve prediction accuracy on a multi-class problem. The classification -dataset is constructed by taking a ten-dimensional standard normal distribution -and defining three classes separated by nested concentric ten-dimensional -spheres such that roughly equal numbers of samples are in each class (quantiles -of the :math:`\chi^2` distribution). - -The performance of the SAMME and SAMME.R [1]_ algorithms are compared. SAMME.R -uses the probability estimates to update the additive model, while SAMME uses -the classifications only. As the example illustrates, the SAMME.R algorithm -typically converges faster than SAMME, achieving a lower test error with fewer -boosting iterations. The error of each algorithm on the test set after each -boosting iteration is shown on the left, the classification error on the test -set of each tree is shown in the middle, and the boost weight of each tree is -shown on the right. All trees have a weight of one in the SAMME.R algorithm and -therefore are not shown. - -.. [1] J. Zhu, H. Zou, S. Rosset, T. Hastie, "Multi-class AdaBoost", 2009. +This example shows how boosting can improve the prediction accuracy on a +multi-label classification problem. It reproduces a similar experiment as +depicted by Figure 1 in Zhu et al [1]_. -""" -print(__doc__) +The core principle of AdaBoost (Adaptive Boosting) is to fit a sequence of weak +learners (e.g. Decision Trees) on repeatedly re-sampled versions of the data. +Each sample carries a weight that is adjusted after each training step, such +that misclassified samples will be assigned higher weights. The re-sampling +process with replacement takes into account the weights assigned to each sample. +Samples with higher weights have a greater chance of being selected multiple +times in the new data set, while samples with lower weights are less likely to +be selected. This ensures that subsequent iterations of the algorithm focus on +the difficult-to-classify samples. -# Author: Noel Dawe -# -# License: BSD 3 clause +.. rubric:: References -import matplotlib.pyplot as plt +.. [1] :doi:`J. Zhu, H. Zou, S. Rosset, T. Hastie, "Multi-class adaboost." + Statistics and its Interface 2.3 (2009): 349-360. + <10.4310/SII.2009.v2.n3.a8>` +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Creating the dataset +# -------------------- +# The classification dataset is constructed by taking a ten-dimensional standard +# normal distribution (:math:`x` in :math:`R^{10}`) and defining three classes +# separated by nested concentric ten-dimensional spheres such that roughly equal +# numbers of samples are in each class (quantiles of the :math:`\chi^2` +# distribution). from sklearn.datasets import make_gaussian_quantiles + +X, y = make_gaussian_quantiles( + n_samples=2_000, n_features=10, n_classes=3, random_state=1 +) + +# %% +# We split the dataset into 2 sets: 70 percent of the samples are used for +# training and the remaining 30 percent for testing. +from sklearn.model_selection import train_test_split + +X_train, X_test, y_train, y_test = train_test_split( + X, y, train_size=0.7, random_state=42 +) + +# %% +# Training the `AdaBoostClassifier` +# --------------------------------- +# We train the :class:`~sklearn.ensemble.AdaBoostClassifier`. The estimator +# utilizes boosting to improve the classification accuracy. Boosting is a method +# designed to train weak learners (i.e. `estimator`) that learn from their +# predecessor's mistakes. +# +# Here, we define the weak learner as a +# :class:`~sklearn.tree.DecisionTreeClassifier` and set the maximum number of +# leaves to 8. In a real setting, this parameter should be tuned. We set it to a +# rather low value to limit the runtime of the example. +# +# The `SAMME` algorithm build into the +# :class:`~sklearn.ensemble.AdaBoostClassifier` then uses the correct or +# incorrect predictions made be the current weak learner to update the sample +# weights used for training the consecutive weak learners. Also, the weight of +# the weak learner itself is calculated based on its accuracy in classifying the +# training examples. The weight of the weak learner determines its influence on +# the final ensemble prediction. from sklearn.ensemble import AdaBoostClassifier -from sklearn.metrics import accuracy_score from sklearn.tree import DecisionTreeClassifier +weak_learner = DecisionTreeClassifier(max_leaf_nodes=8) +n_estimators = 300 + +adaboost_clf = AdaBoostClassifier( + estimator=weak_learner, + n_estimators=n_estimators, + random_state=42, +).fit(X_train, y_train) + +# %% +# Analysis +# -------- +# Convergence of the `AdaBoostClassifier` +# *************************************** +# To demonstrate the effectiveness of boosting in improving accuracy, we +# evaluate the misclassification error of the boosted trees in comparison to two +# baseline scores. The first baseline score is the `misclassification_error` +# obtained from a single weak-learner (i.e. +# :class:`~sklearn.tree.DecisionTreeClassifier`), which serves as a reference +# point. The second baseline score is obtained from the +# :class:`~sklearn.dummy.DummyClassifier`, which predicts the most prevalent +# class in a dataset. +from sklearn.dummy import DummyClassifier +from sklearn.metrics import accuracy_score + +dummy_clf = DummyClassifier() + + +def misclassification_error(y_true, y_pred): + return 1 - accuracy_score(y_true, y_pred) + + +weak_learners_misclassification_error = misclassification_error( + y_test, weak_learner.fit(X_train, y_train).predict(X_test) +) + +dummy_classifiers_misclassification_error = misclassification_error( + y_test, dummy_clf.fit(X_train, y_train).predict(X_test) +) -X, y = make_gaussian_quantiles(n_samples=13000, n_features=10, - n_classes=3, random_state=1) - -n_split = 3000 - -X_train, X_test = X[:n_split], X[n_split:] -y_train, y_test = y[:n_split], y[n_split:] - -bdt_real = AdaBoostClassifier( - DecisionTreeClassifier(max_depth=2), - n_estimators=600, - learning_rate=1) - -bdt_discrete = AdaBoostClassifier( - DecisionTreeClassifier(max_depth=2), - n_estimators=600, - learning_rate=1.5, - algorithm="SAMME") - -bdt_real.fit(X_train, y_train) -bdt_discrete.fit(X_train, y_train) - -real_test_errors = [] -discrete_test_errors = [] - -for real_test_predict, discrete_train_predict in zip( - bdt_real.staged_predict(X_test), bdt_discrete.staged_predict(X_test)): - real_test_errors.append( - 1. - accuracy_score(real_test_predict, y_test)) - discrete_test_errors.append( - 1. - accuracy_score(discrete_train_predict, y_test)) - -n_trees_discrete = len(bdt_discrete) -n_trees_real = len(bdt_real) - -# Boosting might terminate early, but the following arrays are always -# n_estimators long. We crop them to the actual number of trees here: -discrete_estimator_errors = bdt_discrete.estimator_errors_[:n_trees_discrete] -real_estimator_errors = bdt_real.estimator_errors_[:n_trees_real] -discrete_estimator_weights = bdt_discrete.estimator_weights_[:n_trees_discrete] - -plt.figure(figsize=(15, 5)) - -plt.subplot(131) -plt.plot(range(1, n_trees_discrete + 1), - discrete_test_errors, c='black', label='SAMME') -plt.plot(range(1, n_trees_real + 1), - real_test_errors, c='black', - linestyle='dashed', label='SAMME.R') -plt.legend() -plt.ylim(0.18, 0.62) -plt.ylabel('Test Error') -plt.xlabel('Number of Trees') - -plt.subplot(132) -plt.plot(range(1, n_trees_discrete + 1), discrete_estimator_errors, - "b", label='SAMME', alpha=.5) -plt.plot(range(1, n_trees_real + 1), real_estimator_errors, - "r", label='SAMME.R', alpha=.5) -plt.legend() -plt.ylabel('Error') -plt.xlabel('Number of Trees') -plt.ylim((.2, - max(real_estimator_errors.max(), - discrete_estimator_errors.max()) * 1.2)) -plt.xlim((-20, len(bdt_discrete) + 20)) - -plt.subplot(133) -plt.plot(range(1, n_trees_discrete + 1), discrete_estimator_weights, - "b", label='SAMME') -plt.legend() -plt.ylabel('Weight') -plt.xlabel('Number of Trees') -plt.ylim((0, discrete_estimator_weights.max() * 1.2)) -plt.xlim((-20, n_trees_discrete + 20)) - -# prevent overlapping y-axis labels -plt.subplots_adjust(wspace=0.25) +print( + "DecisionTreeClassifier's misclassification_error: " + f"{weak_learners_misclassification_error:.3f}" +) +print( + "DummyClassifier's misclassification_error: " + f"{dummy_classifiers_misclassification_error:.3f}" +) + +# %% +# After training the :class:`~sklearn.tree.DecisionTreeClassifier` model, the +# achieved error surpasses the expected value that would have been obtained by +# guessing the most frequent class label, as the +# :class:`~sklearn.dummy.DummyClassifier` does. +# +# Now, we calculate the `misclassification_error`, i.e. `1 - accuracy`, of the +# additive model (:class:`~sklearn.tree.DecisionTreeClassifier`) at each +# boosting iteration on the test set to assess its performance. +# +# We use :meth:`~sklearn.ensemble.AdaBoostClassifier.staged_predict` that makes +# as many iterations as the number of fitted estimator (i.e. corresponding to +# `n_estimators`). At iteration `n`, the predictions of AdaBoost only use the +# `n` first weak learners. We compare these predictions with the true +# predictions `y_test` and we, therefore, conclude on the benefit (or not) of adding a +# new weak learner into the chain. +# +# We plot the misclassification error for the different stages: +import matplotlib.pyplot as plt +import pandas as pd + +boosting_errors = pd.DataFrame( + { + "Number of trees": range(1, n_estimators + 1), + "AdaBoost": [ + misclassification_error(y_test, y_pred) + for y_pred in adaboost_clf.staged_predict(X_test) + ], + } +).set_index("Number of trees") +ax = boosting_errors.plot() +ax.set_ylabel("Misclassification error on test set") +ax.set_title("Convergence of AdaBoost algorithm") + +plt.plot( + [boosting_errors.index.min(), boosting_errors.index.max()], + [weak_learners_misclassification_error, weak_learners_misclassification_error], + color="tab:orange", + linestyle="dashed", +) +plt.plot( + [boosting_errors.index.min(), boosting_errors.index.max()], + [ + dummy_classifiers_misclassification_error, + dummy_classifiers_misclassification_error, + ], + color="c", + linestyle="dotted", +) +plt.legend(["AdaBoost", "DecisionTreeClassifier", "DummyClassifier"], loc=1) plt.show() + +# %% +# The plot shows the missclassification error on the test set after each +# boosting iteration. We see that the error of the boosted trees converges to an +# error of around 0.3 after 50 iterations, indicating a significantly higher +# accuracy compared to a single tree, as illustrated by the dashed line in the +# plot. +# +# The misclassification error jitters because the `SAMME` algorithm uses the +# discrete outputs of the weak learners to train the boosted model. +# +# The convergence of :class:`~sklearn.ensemble.AdaBoostClassifier` is mainly +# influenced by the learning rate (i.e. `learning_rate`), the number of weak +# learners used (`n_estimators`), and the expressivity of the weak learners +# (e.g. `max_leaf_nodes`). + +# %% +# Errors and weights of the Weak Learners +# *************************************** +# As previously mentioned, AdaBoost is a forward stagewise additive model. We +# now focus on understanding the relationship between the attributed weights of +# the weak learners and their statistical performance. +# +# We use the fitted :class:`~sklearn.ensemble.AdaBoostClassifier`'s attributes +# `estimator_errors_` and `estimator_weights_` to investigate this link. +weak_learners_info = pd.DataFrame( + { + "Number of trees": range(1, n_estimators + 1), + "Errors": adaboost_clf.estimator_errors_, + "Weights": adaboost_clf.estimator_weights_, + } +).set_index("Number of trees") + +axs = weak_learners_info.plot( + subplots=True, layout=(1, 2), figsize=(10, 4), legend=False, color="tab:blue" +) +axs[0, 0].set_ylabel("Train error") +axs[0, 0].set_title("Weak learner's training error") +axs[0, 1].set_ylabel("Weight") +axs[0, 1].set_title("Weak learner's weight") +fig = axs[0, 0].get_figure() +fig.suptitle("Weak learner's errors and weights for the AdaBoostClassifier") +fig.tight_layout() + +# %% +# On the left plot, we show the weighted error of each weak learner on the +# reweighted training set at each boosting iteration. On the right plot, we show +# the weights associated with each weak learner later used to make the +# predictions of the final additive model. +# +# We see that the error of the weak learner is the inverse of the weights. It +# means that our additive model will trust more a weak learner that makes +# smaller errors (on the training set) by increasing its impact on the final +# decision. Indeed, this exactly is the formulation of updating the base +# estimators' weights after each iteration in AdaBoost. +# +# .. dropdown:: Mathematical details +# +# The weight associated with a weak learner trained at the stage :math:`m` is +# inversely associated with its misclassification error such that: +# +# .. math:: \alpha^{(m)} = \log \frac{1 - err^{(m)}}{err^{(m)}} + \log (K - 1), +# +# where :math:`\alpha^{(m)}` and :math:`err^{(m)}` are the weight and the error +# of the :math:`m` th weak learner, respectively, and :math:`K` is the number of +# classes in our classification problem. +# +# Another interesting observation boils down to the fact that the first weak +# learners of the model make fewer errors than later weak learners of the +# boosting chain. +# +# The intuition behind this observation is the following: due to the sample +# reweighting, later classifiers are forced to try to classify more difficult or +# noisy samples and to ignore already well classified samples. Therefore, the +# overall error on the training set will increase. That's why the weak learner's +# weights are built to counter-balance the worse performing weak learners. diff --git a/examples/ensemble/plot_adaboost_regression.py b/examples/ensemble/plot_adaboost_regression.py index 0c76ac6af3ae9..bed8e8ee30b56 100644 --- a/examples/ensemble/plot_adaboost_regression.py +++ b/examples/ensemble/plot_adaboost_regression.py @@ -9,44 +9,69 @@ regressor. As the number of boosts is increased the regressor can fit more detail. -.. [1] H. Drucker, "Improving Regressors using Boosting Techniques", 1997. +See :ref:`sphx_glr_auto_examples_ensemble_plot_hgbt_regression.py` for an +example showcasing the benefits of using more efficient regression models such +as :class:`~ensemble.HistGradientBoostingRegressor`. + +.. [1] `H. Drucker, "Improving Regressors using Boosting Techniques", 1997. + `_ """ -print(__doc__) -# Author: Noel Dawe -# -# License: BSD 3 clause +# %% +# Preparing the data +# ------------------ +# First, we prepare dummy data with a sinusoidal relationship and some gaussian noise. + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -# importing necessary libraries import numpy as np -import matplotlib.pyplot as plt -from sklearn.tree import DecisionTreeRegressor -from sklearn.ensemble import AdaBoostRegressor -# Create the dataset rng = np.random.RandomState(1) X = np.linspace(0, 6, 100)[:, np.newaxis] y = np.sin(X).ravel() + np.sin(6 * X).ravel() + rng.normal(0, 0.1, X.shape[0]) -# Fit regression model +# %% +# Training and prediction with DecisionTree and AdaBoost Regressors +# ----------------------------------------------------------------- +# Now, we define the classifiers and fit them to the data. +# Then we predict on that same data to see how well they could fit it. +# The first regressor is a `DecisionTreeRegressor` with `max_depth=4`. +# The second regressor is an `AdaBoostRegressor` with a `DecisionTreeRegressor` +# of `max_depth=4` as base learner and will be built with `n_estimators=300` +# of those base learners. + +from sklearn.ensemble import AdaBoostRegressor +from sklearn.tree import DecisionTreeRegressor + regr_1 = DecisionTreeRegressor(max_depth=4) -regr_2 = AdaBoostRegressor(DecisionTreeRegressor(max_depth=4), - n_estimators=300, random_state=rng) +regr_2 = AdaBoostRegressor( + DecisionTreeRegressor(max_depth=4), n_estimators=300, random_state=rng +) regr_1.fit(X, y) regr_2.fit(X, y) -# Predict y_1 = regr_1.predict(X) y_2 = regr_2.predict(X) -# Plot the results +# %% +# Plotting the results +# -------------------- +# Finally, we plot how well our two regressors, +# single decision tree regressor and AdaBoost regressor, could fit the data. + +import matplotlib.pyplot as plt +import seaborn as sns + +colors = sns.color_palette("colorblind") + plt.figure() -plt.scatter(X, y, c="k", label="training samples") -plt.plot(X, y_1, c="g", label="n_estimators=1", linewidth=2) -plt.plot(X, y_2, c="r", label="n_estimators=300", linewidth=2) +plt.scatter(X, y, color=colors[0], label="training samples") +plt.plot(X, y_1, color=colors[1], label="n_estimators=1", linewidth=2) +plt.plot(X, y_2, color=colors[2], label="n_estimators=300", linewidth=2) plt.xlabel("data") plt.ylabel("target") plt.title("Boosted Decision Tree Regression") diff --git a/examples/ensemble/plot_adaboost_twoclass.py b/examples/ensemble/plot_adaboost_twoclass.py index edb4cbb1a97b3..18a2a10841c1c 100644 --- a/examples/ensemble/plot_adaboost_twoclass.py +++ b/examples/ensemble/plot_adaboost_twoclass.py @@ -16,35 +16,30 @@ with a decision score above some value. """ -print(__doc__) -# Author: Noel Dawe -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt +import numpy as np +from sklearn.datasets import make_gaussian_quantiles from sklearn.ensemble import AdaBoostClassifier +from sklearn.inspection import DecisionBoundaryDisplay from sklearn.tree import DecisionTreeClassifier -from sklearn.datasets import make_gaussian_quantiles - # Construct dataset -X1, y1 = make_gaussian_quantiles(cov=2., - n_samples=200, n_features=2, - n_classes=2, random_state=1) -X2, y2 = make_gaussian_quantiles(mean=(3, 3), cov=1.5, - n_samples=300, n_features=2, - n_classes=2, random_state=1) +X1, y1 = make_gaussian_quantiles( + cov=2.0, n_samples=200, n_features=2, n_classes=2, random_state=1 +) +X2, y2 = make_gaussian_quantiles( + mean=(3, 3), cov=1.5, n_samples=300, n_features=2, n_classes=2, random_state=1 +) X = np.concatenate((X1, X2)) -y = np.concatenate((y1, - y2 + 1)) +y = np.concatenate((y1, -y2 + 1)) # Create and fit an AdaBoosted decision tree -bdt = AdaBoostClassifier(DecisionTreeClassifier(max_depth=1), - algorithm="SAMME", - n_estimators=200) - +bdt = AdaBoostClassifier(DecisionTreeClassifier(max_depth=1), n_estimators=200) bdt.fit(X, y) plot_colors = "br" @@ -54,49 +49,57 @@ plt.figure(figsize=(10, 5)) # Plot the decision boundaries -plt.subplot(121) -x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1 -y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1 -xx, yy = np.meshgrid(np.arange(x_min, x_max, plot_step), - np.arange(y_min, y_max, plot_step)) - -Z = bdt.predict(np.c_[xx.ravel(), yy.ravel()]) -Z = Z.reshape(xx.shape) -cs = plt.contourf(xx, yy, Z, cmap=plt.cm.Paired) +ax = plt.subplot(121) +disp = DecisionBoundaryDisplay.from_estimator( + bdt, + X, + cmap=plt.cm.Paired, + response_method="predict", + ax=ax, + xlabel="x", + ylabel="y", +) +x_min, x_max = disp.xx0.min(), disp.xx0.max() +y_min, y_max = disp.xx1.min(), disp.xx1.max() plt.axis("tight") # Plot the training points for i, n, c in zip(range(2), class_names, plot_colors): - idx = np.where(y == i) - plt.scatter(X[idx, 0], X[idx, 1], - c=c, cmap=plt.cm.Paired, - s=20, edgecolor='k', - label="Class %s" % n) + idx = (y == i).nonzero() + plt.scatter( + X[idx, 0], + X[idx, 1], + c=c, + s=20, + edgecolor="k", + label="Class %s" % n, + ) plt.xlim(x_min, x_max) plt.ylim(y_min, y_max) -plt.legend(loc='upper right') -plt.xlabel('x') -plt.ylabel('y') -plt.title('Decision Boundary') +plt.legend(loc="upper right") + +plt.title("Decision Boundary") # Plot the two-class decision scores twoclass_output = bdt.decision_function(X) plot_range = (twoclass_output.min(), twoclass_output.max()) plt.subplot(122) for i, n, c in zip(range(2), class_names, plot_colors): - plt.hist(twoclass_output[y == i], - bins=10, - range=plot_range, - facecolor=c, - label='Class %s' % n, - alpha=.5, - edgecolor='k') + plt.hist( + twoclass_output[y == i], + bins=10, + range=plot_range, + facecolor=c, + label="Class %s" % n, + alpha=0.5, + edgecolor="k", + ) x1, x2, y1, y2 = plt.axis() plt.axis((x1, x2, y1, y2 * 1.2)) -plt.legend(loc='upper right') -plt.ylabel('Samples') -plt.xlabel('Score') -plt.title('Decision Scores') +plt.legend(loc="upper right") +plt.ylabel("Samples") +plt.xlabel("Score") +plt.title("Decision Scores") plt.tight_layout() plt.subplots_adjust(wspace=0.35) diff --git a/examples/ensemble/plot_bias_variance.py b/examples/ensemble/plot_bias_variance.py index 0af239e197cf0..72134841c78ea 100644 --- a/examples/ensemble/plot_bias_variance.py +++ b/examples/ensemble/plot_bias_variance.py @@ -12,7 +12,8 @@ predictions of the estimator differ from the predictions of the best possible estimator for the problem (i.e., the Bayes model). The variance term measures the variability of the predictions of the estimator when fit over different -instances LS of the problem. Finally, the noise measures the irreducible part +random instances of the same problem. Each problem instance is noted "LS", for +"Learning Sample", in the following. Finally, the noise measures the irreducible part of the error which is due the variability in the data. The upper left figure illustrates the predictions (in dark red) of a single @@ -42,8 +43,8 @@ curve is also slightly higher than in the lower left figure. In terms of variance however, the beam of predictions is narrower, which suggests that the variance is lower. Indeed, as the lower right figure confirms, the variance -term (in green) is lower than for single decision trees. Overall, the bias- -variance decomposition is therefore no longer the same. The tradeoff is better +term (in green) is lower than for single decision trees. Overall, the bias-variance +decomposition is therefore no longer the same. The tradeoff is better for bagging: averaging several decision trees fit on bootstrap copies of the dataset slightly increases the bias term but allows for a larger reduction of the variance, which results in a lower overall mean squared error (compare the @@ -61,30 +62,31 @@ "Elements of Statistical Learning", Springer, 2009. """ -print(__doc__) -# Author: Gilles Louppe -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt +import numpy as np from sklearn.ensemble import BaggingRegressor from sklearn.tree import DecisionTreeRegressor # Settings -n_repeat = 50 # Number of iterations for computing expectations -n_train = 50 # Size of the training set -n_test = 1000 # Size of the test set -noise = 0.1 # Standard deviation of the noise +n_repeat = 50 # Number of iterations for computing expectations +n_train = 50 # Size of the training set +n_test = 1000 # Size of the test set +noise = 0.1 # Standard deviation of the noise np.random.seed(0) # Change this for exploring the bias-variance decomposition of other # estimators. This should work well for estimators with high variance (e.g., # decision trees or KNN), but poorly for estimators with low variance (e.g., # linear models). -estimators = [("Tree", DecisionTreeRegressor()), - ("Bagging(Tree)", BaggingRegressor(DecisionTreeRegressor()))] +estimators = [ + ("Tree", DecisionTreeRegressor()), + ("Bagging(Tree)", BaggingRegressor(DecisionTreeRegressor())), +] n_estimators = len(estimators) @@ -93,7 +95,7 @@ def f(x): x = x.ravel() - return np.exp(-x ** 2) + 1.5 * np.exp(-(x - 2) ** 2) + return np.exp(-(x**2)) + 1.5 * np.exp(-((x - 2) ** 2)) def generate(n_samples, noise, n_repeat=1): @@ -141,18 +143,18 @@ def generate(n_samples, noise, n_repeat=1): for j in range(n_repeat): y_error += (y_test[:, j] - y_predict[:, i]) ** 2 - y_error /= (n_repeat * n_repeat) + y_error /= n_repeat * n_repeat y_noise = np.var(y_test, axis=1) y_bias = (f(X_test) - np.mean(y_predict, axis=1)) ** 2 y_var = np.var(y_predict, axis=1) - print("{0}: {1:.4f} (error) = {2:.4f} (bias^2) " - " + {3:.4f} (var) + {4:.4f} (noise)".format(name, - np.mean(y_error), - np.mean(y_bias), - np.mean(y_var), - np.mean(y_noise))) + print( + "{0}: {1:.4f} (error) = {2:.4f} (bias^2) " + " + {3:.4f} (var) + {4:.4f} (noise)".format( + name, np.mean(y_error), np.mean(y_bias), np.mean(y_var), np.mean(y_noise) + ) + ) # Plot figures plt.subplot(2, n_estimators, n + 1) @@ -165,27 +167,25 @@ def generate(n_samples, noise, n_repeat=1): else: plt.plot(X_test, y_predict[:, i], "r", alpha=0.05) - plt.plot(X_test, np.mean(y_predict, axis=1), "c", - label=r"$\mathbb{E}_{LS} \^y(x)$") + plt.plot(X_test, np.mean(y_predict, axis=1), "c", label=r"$\mathbb{E}_{LS} \^y(x)$") plt.xlim([-5, 5]) plt.title(name) if n == n_estimators - 1: - plt.legend(loc=(1.1, .5)) + plt.legend(loc=(1.1, 0.5)) plt.subplot(2, n_estimators, n_estimators + n + 1) plt.plot(X_test, y_error, "r", label="$error(x)$") - plt.plot(X_test, y_bias, "b", label="$bias^2(x)$"), - plt.plot(X_test, y_var, "g", label="$variance(x)$"), + plt.plot(X_test, y_bias, "b", label="$bias^2(x)$") + plt.plot(X_test, y_var, "g", label="$variance(x)$") plt.plot(X_test, y_noise, "c", label="$noise(x)$") plt.xlim([-5, 5]) plt.ylim([0, 0.1]) if n == n_estimators - 1: + plt.legend(loc=(1.1, 0.5)) - plt.legend(loc=(1.1, .5)) - -plt.subplots_adjust(right=.75) +plt.subplots_adjust(right=0.75) plt.show() diff --git a/examples/ensemble/plot_ensemble_oob.py b/examples/ensemble/plot_ensemble_oob.py index e7b37f212177c..2c89660b69d42 100644 --- a/examples/ensemble/plot_ensemble_oob.py +++ b/examples/ensemble/plot_ensemble_oob.py @@ -19,43 +19,59 @@ Learning Ed. 2", p592-593, Springer, 2009. """ -import matplotlib.pyplot as plt + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause from collections import OrderedDict -from sklearn.datasets import make_classification -from sklearn.ensemble import RandomForestClassifier -# Author: Kian Ho -# Gilles Louppe -# Andreas Mueller -# -# License: BSD 3 Clause +import matplotlib.pyplot as plt -print(__doc__) +from sklearn.datasets import make_classification +from sklearn.ensemble import RandomForestClassifier RANDOM_STATE = 123 # Generate a binary classification dataset. -X, y = make_classification(n_samples=500, n_features=25, - n_clusters_per_class=1, n_informative=15, - random_state=RANDOM_STATE) +X, y = make_classification( + n_samples=500, + n_features=25, + n_clusters_per_class=1, + n_informative=15, + random_state=RANDOM_STATE, +) # NOTE: Setting the `warm_start` construction parameter to `True` disables # support for parallelized ensembles but is necessary for tracking the OOB # error trajectory during training. ensemble_clfs = [ - ("RandomForestClassifier, max_features='sqrt'", - RandomForestClassifier(warm_start=True, oob_score=True, - max_features="sqrt", - random_state=RANDOM_STATE)), - ("RandomForestClassifier, max_features='log2'", - RandomForestClassifier(warm_start=True, max_features='log2', - oob_score=True, - random_state=RANDOM_STATE)), - ("RandomForestClassifier, max_features=None", - RandomForestClassifier(warm_start=True, max_features=None, - oob_score=True, - random_state=RANDOM_STATE)) + ( + "RandomForestClassifier, max_features='sqrt'", + RandomForestClassifier( + warm_start=True, + oob_score=True, + max_features="sqrt", + random_state=RANDOM_STATE, + ), + ), + ( + "RandomForestClassifier, max_features='log2'", + RandomForestClassifier( + warm_start=True, + max_features="log2", + oob_score=True, + random_state=RANDOM_STATE, + ), + ), + ( + "RandomForestClassifier, max_features=None", + RandomForestClassifier( + warm_start=True, + max_features=None, + oob_score=True, + random_state=RANDOM_STATE, + ), + ), ] # Map a classifier name to a list of (, ) pairs. @@ -63,10 +79,10 @@ # Range of `n_estimators` values to explore. min_estimators = 15 -max_estimators = 175 +max_estimators = 150 for label, clf in ensemble_clfs: - for i in range(min_estimators, max_estimators + 1): + for i in range(min_estimators, max_estimators + 1, 5): clf.set_params(n_estimators=i) clf.fit(X, y) diff --git a/examples/ensemble/plot_feature_transformation.py b/examples/ensemble/plot_feature_transformation.py index f7e374c9d1c13..ef0d66fa3eda4 100644 --- a/examples/ensemble/plot_feature_transformation.py +++ b/examples/ensemble/plot_feature_transformation.py @@ -17,16 +17,11 @@ The resulting transformer has then learned a supervised, sparse, high-dimensional categorical embedding of the data. -""" - -# Author: Tim Head -# -# License: BSD 3 clause -print(__doc__) +""" -from sklearn import set_config -set_config(display='diagram') +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause # %% # First, we will create a large dataset and split it into three sets: @@ -42,13 +37,14 @@ from sklearn.datasets import make_classification from sklearn.model_selection import train_test_split -X, y = make_classification(n_samples=80000, random_state=10) +X, y = make_classification(n_samples=80_000, random_state=10) X_full_train, X_test, y_full_train, y_test = train_test_split( - X, y, test_size=0.5, random_state=10) -X_train_ensemble, X_train_linear, y_train_ensemble, y_train_linear = \ - train_test_split(X_full_train, y_full_train, test_size=0.5, - random_state=10) + X, y, test_size=0.5, random_state=10 +) +X_train_ensemble, X_train_linear, y_train_ensemble, y_train_linear = train_test_split( + X_full_train, y_full_train, test_size=0.5, random_state=10 +) # %% # For each of the ensemble methods, we will use 10 estimators and a maximum @@ -61,24 +57,32 @@ # First, we will start by training the random forest and gradient boosting on # the separated training set -from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier +from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier random_forest = RandomForestClassifier( - n_estimators=n_estimators, max_depth=max_depth, random_state=10) + n_estimators=n_estimators, max_depth=max_depth, random_state=10 +) random_forest.fit(X_train_ensemble, y_train_ensemble) gradient_boosting = GradientBoostingClassifier( - n_estimators=n_estimators, max_depth=max_depth, random_state=10) + n_estimators=n_estimators, max_depth=max_depth, random_state=10 +) _ = gradient_boosting.fit(X_train_ensemble, y_train_ensemble) # %% +# Notice that :class:`~sklearn.ensemble.HistGradientBoostingClassifier` is much +# faster than :class:`~sklearn.ensemble.GradientBoostingClassifier` starting +# with intermediate datasets (`n_samples >= 10_000`), which is not the case of +# the present example. +# # The :class:`~sklearn.ensemble.RandomTreesEmbedding` is an unsupervised method # and thus does not required to be trained independently. from sklearn.ensemble import RandomTreesEmbedding random_tree_embedding = RandomTreesEmbedding( - n_estimators=n_estimators, max_depth=max_depth, random_state=0) + n_estimators=n_estimators, max_depth=max_depth, random_state=0 +) # %% # Now, we will create three pipelines that will use the above embedding as @@ -90,8 +94,7 @@ from sklearn.linear_model import LogisticRegression from sklearn.pipeline import make_pipeline -rt_model = make_pipeline( - random_tree_embedding, LogisticRegression(max_iter=1000)) +rt_model = make_pipeline(random_tree_embedding, LogisticRegression(max_iter=1000)) rt_model.fit(X_train_linear, y_train_linear) # %% @@ -100,20 +103,20 @@ # method `apply`. The pipeline in scikit-learn expects a call to `transform`. # Therefore, we wrapped the call to `apply` within a `FunctionTransformer`. -from sklearn.preprocessing import FunctionTransformer -from sklearn.preprocessing import OneHotEncoder +from sklearn.preprocessing import FunctionTransformer, OneHotEncoder def rf_apply(X, model): return model.apply(X) -rf_leaves_yielder = FunctionTransformer( - rf_apply, kw_args={"model": random_forest}) +rf_leaves_yielder = FunctionTransformer(rf_apply, kw_args={"model": random_forest}) rf_model = make_pipeline( - rf_leaves_yielder, OneHotEncoder(handle_unknown="ignore"), - LogisticRegression(max_iter=1000)) + rf_leaves_yielder, + OneHotEncoder(handle_unknown="ignore"), + LogisticRegression(max_iter=1000), +) rf_model.fit(X_train_linear, y_train_linear) @@ -123,20 +126,24 @@ def gbdt_apply(X, model): gbdt_leaves_yielder = FunctionTransformer( - gbdt_apply, kw_args={"model": gradient_boosting}) + gbdt_apply, kw_args={"model": gradient_boosting} +) gbdt_model = make_pipeline( - gbdt_leaves_yielder, OneHotEncoder(handle_unknown="ignore"), - LogisticRegression(max_iter=1000)) + gbdt_leaves_yielder, + OneHotEncoder(handle_unknown="ignore"), + LogisticRegression(max_iter=1000), +) gbdt_model.fit(X_train_linear, y_train_linear) # %% # We can finally show the different ROC curves for all the models. import matplotlib.pyplot as plt -from sklearn.metrics import plot_roc_curve -fig, ax = plt.subplots() +from sklearn.metrics import RocCurveDisplay + +_, ax = plt.subplots() models = [ ("RT embedding -> LR", rt_model), @@ -148,15 +155,16 @@ def gbdt_apply(X, model): model_displays = {} for name, pipeline in models: - model_displays[name] = plot_roc_curve( - pipeline, X_test, y_test, ax=ax, name=name) -_ = ax.set_title('ROC curve') + model_displays[name] = RocCurveDisplay.from_estimator( + pipeline, X_test, y_test, ax=ax, name=name + ) +_ = ax.set_title("ROC curve") # %% -fig, ax = plt.subplots() +_, ax = plt.subplots() for name, pipeline in models: model_displays[name].plot(ax=ax) ax.set_xlim(0, 0.2) ax.set_ylim(0.8, 1) -_ = ax.set_title('ROC curve (zoomed in at top left)') +_ = ax.set_title("ROC curve (zoomed in at top left)") diff --git a/examples/ensemble/plot_forest_hist_grad_boosting_comparison.py b/examples/ensemble/plot_forest_hist_grad_boosting_comparison.py new file mode 100644 index 0000000000000..85e73a2298d36 --- /dev/null +++ b/examples/ensemble/plot_forest_hist_grad_boosting_comparison.py @@ -0,0 +1,226 @@ +""" +=============================================================== +Comparing Random Forests and Histogram Gradient Boosting models +=============================================================== + +In this example we compare the performance of Random Forest (RF) and Histogram +Gradient Boosting (HGBT) models in terms of score and computation time for a +regression dataset, though **all the concepts here presented apply to +classification as well**. + +The comparison is made by varying the parameters that control the number of +trees according to each estimator: + +- `n_estimators` controls the number of trees in the forest. It's a fixed number. +- `max_iter` is the maximum number of iterations in a gradient boosting + based model. The number of iterations corresponds to the number of trees for + regression and binary classification problems. Furthermore, the actual number + of trees required by the model depends on the stopping criteria. + +HGBT uses gradient boosting to iteratively improve the model's performance by +fitting each tree to the negative gradient of the loss function with respect to +the predicted value. RFs, on the other hand, are based on bagging and use a +majority vote to predict the outcome. + +See the :ref:`User Guide ` for more information on ensemble models or +see :ref:`sphx_glr_auto_examples_ensemble_plot_hgbt_regression.py` for an +example showcasing some other features of HGBT models. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Load dataset +# ------------ + +from sklearn.datasets import fetch_california_housing + +X, y = fetch_california_housing(return_X_y=True, as_frame=True) +n_samples, n_features = X.shape + +# %% +# HGBT uses a histogram-based algorithm on binned feature values that can +# efficiently handle large datasets (tens of thousands of samples or more) with +# a high number of features (see :ref:`Why_it's_faster`). The scikit-learn +# implementation of RF does not use binning and relies on exact splitting, which +# can be computationally expensive. + +print(f"The dataset consists of {n_samples} samples and {n_features} features") + +# %% +# Compute score and computation times +# ----------------------------------- +# +# Notice that many parts of the implementation of +# :class:`~sklearn.ensemble.HistGradientBoostingClassifier` and +# :class:`~sklearn.ensemble.HistGradientBoostingRegressor` are parallelized by +# default. +# +# The implementation of :class:`~sklearn.ensemble.RandomForestRegressor` and +# :class:`~sklearn.ensemble.RandomForestClassifier` can also be run on multiple +# cores by using the `n_jobs` parameter, here set to match the number of +# physical cores on the host machine. See :ref:`parallelism` for more +# information. + +import joblib + +N_CORES = joblib.cpu_count(only_physical_cores=True) +print(f"Number of physical cores: {N_CORES}") + +# %% +# Unlike RF, HGBT models offer an early-stopping option (see +# :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_early_stopping.py`) +# to avoid adding new unnecessary trees. Internally, the algorithm uses an +# out-of-sample set to compute the generalization performance of the model at +# each addition of a tree. Thus, if the generalization performance is not +# improving for more than `n_iter_no_change` iterations, it stops adding trees. +# +# The other parameters of both models were tuned but the procedure is not shown +# here to keep the example simple. + +import pandas as pd + +from sklearn.ensemble import HistGradientBoostingRegressor, RandomForestRegressor +from sklearn.model_selection import GridSearchCV, KFold + +models = { + "Random Forest": RandomForestRegressor( + min_samples_leaf=5, random_state=0, n_jobs=N_CORES + ), + "Hist Gradient Boosting": HistGradientBoostingRegressor( + max_leaf_nodes=15, random_state=0, early_stopping=False + ), +} +param_grids = { + "Random Forest": {"n_estimators": [10, 20, 50, 100]}, + "Hist Gradient Boosting": {"max_iter": [10, 20, 50, 100, 300, 500]}, +} +cv = KFold(n_splits=4, shuffle=True, random_state=0) + +results = [] +for name, model in models.items(): + grid_search = GridSearchCV( + estimator=model, + param_grid=param_grids[name], + return_train_score=True, + cv=cv, + ).fit(X, y) + result = {"model": name, "cv_results": pd.DataFrame(grid_search.cv_results_)} + results.append(result) + +# %% +# .. Note:: +# Tuning the `n_estimators` for RF generally results in a waste of computer +# power. In practice one just needs to ensure that it is large enough so that +# doubling its value does not lead to a significant improvement of the testing +# score. +# +# Plot results +# ------------ +# We can use a `plotly.express.scatter +# `_ +# to visualize the trade-off between elapsed computing time and mean test score. +# Passing the cursor over a given point displays the corresponding parameters. +# Error bars correspond to one standard deviation as computed in the different +# folds of the cross-validation. + +import plotly.colors as colors +import plotly.express as px +from plotly.subplots import make_subplots + +fig = make_subplots( + rows=1, + cols=2, + shared_yaxes=True, + subplot_titles=["Train time vs score", "Predict time vs score"], +) +model_names = [result["model"] for result in results] +colors_list = colors.qualitative.Plotly * ( + len(model_names) // len(colors.qualitative.Plotly) + 1 +) + +for idx, result in enumerate(results): + cv_results = result["cv_results"].round(3) + model_name = result["model"] + param_name = next(iter(param_grids[model_name].keys())) + cv_results[param_name] = cv_results["param_" + param_name] + cv_results["model"] = model_name + + scatter_fig = px.scatter( + cv_results, + x="mean_fit_time", + y="mean_test_score", + error_x="std_fit_time", + error_y="std_test_score", + hover_data=param_name, + color="model", + ) + line_fig = px.line( + cv_results, + x="mean_fit_time", + y="mean_test_score", + ) + + scatter_trace = scatter_fig["data"][0] + line_trace = line_fig["data"][0] + scatter_trace.update(marker=dict(color=colors_list[idx])) + line_trace.update(line=dict(color=colors_list[idx])) + fig.add_trace(scatter_trace, row=1, col=1) + fig.add_trace(line_trace, row=1, col=1) + + scatter_fig = px.scatter( + cv_results, + x="mean_score_time", + y="mean_test_score", + error_x="std_score_time", + error_y="std_test_score", + hover_data=param_name, + ) + line_fig = px.line( + cv_results, + x="mean_score_time", + y="mean_test_score", + ) + + scatter_trace = scatter_fig["data"][0] + line_trace = line_fig["data"][0] + scatter_trace.update(marker=dict(color=colors_list[idx])) + line_trace.update(line=dict(color=colors_list[idx])) + fig.add_trace(scatter_trace, row=1, col=2) + fig.add_trace(line_trace, row=1, col=2) + +fig.update_layout( + xaxis=dict(title="Train time (s) - lower is better"), + yaxis=dict(title="Test R2 score - higher is better"), + xaxis2=dict(title="Predict time (s) - lower is better"), + legend=dict(x=0.72, y=0.05, traceorder="normal", borderwidth=1), + title=dict(x=0.5, text="Speed-score trade-off of tree-based ensembles"), +) + +# %% +# Both HGBT and RF models improve when increasing the number of trees in the +# ensemble. However, the scores reach a plateau where adding new trees just +# makes fitting and scoring slower. The RF model reaches such plateau earlier +# and can never reach the test score of the largest HGBDT model. +# +# Note that the results shown on the above plot can change slightly across runs +# and even more significantly when running on other machines: try to run this +# example on your own local machine. +# +# Overall, one should often observe that the Histogram-based gradient boosting +# models uniformly dominate the Random Forest models in the "test score vs +# training speed trade-off" (the HGBDT curve should be on the top left of the RF +# curve, without ever crossing). The "test score vs prediction speed" trade-off +# can also be more disputed, but it's most often favorable to HGBDT. It's always +# a good idea to check both kinds of model (with hyper-parameter tuning) and +# compare their performance on your specific problem to determine which model is +# the best fit but **HGBT almost always offers a more favorable speed-accuracy +# trade-off than RF**, either with the default hyper-parameters or including the +# hyper-parameter tuning cost. +# +# There is one exception to this rule of thumb though: when training a +# multiclass classification model with a large number of possible classes, HGBDT +# fits internally one-tree per class at each boosting iteration while the trees +# used by the RF models are naturally multiclass which should improve the speed +# accuracy trade-off of the RF models in this case. diff --git a/examples/ensemble/plot_forest_importances.py b/examples/ensemble/plot_forest_importances.py index 7b75b92a1f0e0..b77e30c327fb4 100644 --- a/examples/ensemble/plot_forest_importances.py +++ b/examples/ensemble/plot_forest_importances.py @@ -10,8 +10,12 @@ As expected, the plot suggests that 3 features are informative, while the remaining are not. + """ -print(__doc__) + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt # %% @@ -25,16 +29,22 @@ from sklearn.model_selection import train_test_split X, y = make_classification( - n_samples=1000, n_features=10, n_informative=3, n_redundant=0, - n_repeated=0, n_classes=2, random_state=0, shuffle=False) -X_train, X_test, y_train, y_test = train_test_split( - X, y, stratify=y, random_state=42) + n_samples=1000, + n_features=10, + n_informative=3, + n_redundant=0, + n_repeated=0, + n_classes=2, + random_state=0, + shuffle=False, +) +X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=42) # %% # A random forest classifier will be fitted to compute the feature importances. from sklearn.ensemble import RandomForestClassifier -feature_names = [f'feature {i}' for i in range(X.shape[1])] +feature_names = [f"feature {i}" for i in range(X.shape[1])] forest = RandomForestClassifier(random_state=0) forest.fit(X_train, y_train) @@ -46,24 +56,24 @@ # deviation of accumulation of the impurity decrease within each tree. # # .. warning:: -# Impurity-based feature importances can be misleading for high cardinality -# features (many unique values). See :ref:`permutation_importance` as -# an alternative below. +# Impurity-based feature importances can be misleading for **high +# cardinality** features (many unique values). See +# :ref:`permutation_importance` as an alternative below. import time + import numpy as np start_time = time.time() importances = forest.feature_importances_ -std = np.std([ - tree.feature_importances_ for tree in forest.estimators_], axis=0) +std = np.std([tree.feature_importances_ for tree in forest.estimators_], axis=0) elapsed_time = time.time() - start_time -print(f"Elapsed time to compute the importances: " - f"{elapsed_time:.3f} seconds") +print(f"Elapsed time to compute the importances: {elapsed_time:.3f} seconds") # %% # Let's plot the impurity-based importance. import pandas as pd + forest_importances = pd.Series(importances, index=feature_names) fig, ax = plt.subplots() @@ -84,10 +94,10 @@ start_time = time.time() result = permutation_importance( - forest, X_test, y_test, n_repeats=10, random_state=42, n_jobs=2) + forest, X_test, y_test, n_repeats=10, random_state=42, n_jobs=2 +) elapsed_time = time.time() - start_time -print(f"Elapsed time to compute the importances: " - f"{elapsed_time:.3f} seconds") +print(f"Elapsed time to compute the importances: {elapsed_time:.3f} seconds") forest_importances = pd.Series(result.importances_mean, index=feature_names) diff --git a/examples/ensemble/plot_forest_importances_faces.py b/examples/ensemble/plot_forest_importances_faces.py deleted file mode 100644 index 6cea84ca4744c..0000000000000 --- a/examples/ensemble/plot_forest_importances_faces.py +++ /dev/null @@ -1,48 +0,0 @@ -""" -================================================= -Pixel importances with a parallel forest of trees -================================================= - -This example shows the use of forests of trees to evaluate the impurity-based -importance of the pixels in an image classification task (faces). -The hotter the pixel, the more important. - -The code below also illustrates how the construction and the computation -of the predictions can be parallelized within multiple jobs. -""" -print(__doc__) - -from time import time -import matplotlib.pyplot as plt - -from sklearn.datasets import fetch_olivetti_faces -from sklearn.ensemble import ExtraTreesClassifier - -# Number of cores to use to perform parallel fitting of the forest model -n_jobs = 1 - -# Load the faces dataset -data = fetch_olivetti_faces() -X, y = data.data, data.target - -mask = y < 5 # Limit to 5 classes -X = X[mask] -y = y[mask] - -# Build a forest and compute the pixel importances -print("Fitting ExtraTreesClassifier on faces data with %d cores..." % n_jobs) -t0 = time() -forest = ExtraTreesClassifier(n_estimators=1000, - max_features=128, - n_jobs=n_jobs, - random_state=0) - -forest.fit(X, y) -print("done in %0.3fs" % (time() - t0)) -importances = forest.feature_importances_ -importances = importances.reshape(data.images[0].shape) - -# Plot pixel importances -plt.matshow(importances, cmap=plt.cm.hot) -plt.title("Pixel importances with forests of trees") -plt.show() diff --git a/examples/ensemble/plot_forest_iris.py b/examples/ensemble/plot_forest_iris.py index 81cd54a9bb4d3..c3fefdcb60d7e 100644 --- a/examples/ensemble/plot_forest_iris.py +++ b/examples/ensemble/plot_forest_iris.py @@ -7,8 +7,8 @@ features of the iris dataset. This plot compares the decision surfaces learned by a decision tree classifier -(first column), by a random forest classifier (second column), by an extra- -trees classifier (third column) and by an AdaBoost classifier (fourth column). +(first column), by a random forest classifier (second column), by an extra-trees +classifier (third column) and by an AdaBoost classifier (fourth column). In the first row, the classifiers are built using the sepal width and the sepal length features only, on the second row using the petal length and @@ -39,16 +39,22 @@ It is worth noting that RandomForests and ExtraTrees can be fitted in parallel on many cores as each tree is built independently of the others. AdaBoost's samples are built sequentially and so do not use multiple cores. + """ -print(__doc__) -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt +import numpy as np from matplotlib.colors import ListedColormap from sklearn.datasets import load_iris -from sklearn.ensemble import (RandomForestClassifier, ExtraTreesClassifier, - AdaBoostClassifier) +from sklearn.ensemble import ( + AdaBoostClassifier, + ExtraTreesClassifier, + RandomForestClassifier, +) from sklearn.tree import DecisionTreeClassifier # Parameters @@ -64,11 +70,12 @@ plot_idx = 1 -models = [DecisionTreeClassifier(max_depth=None), - RandomForestClassifier(n_estimators=n_estimators), - ExtraTreesClassifier(n_estimators=n_estimators), - AdaBoostClassifier(DecisionTreeClassifier(max_depth=3), - n_estimators=n_estimators)] +models = [ + DecisionTreeClassifier(max_depth=None), + RandomForestClassifier(n_estimators=n_estimators), + ExtraTreesClassifier(n_estimators=n_estimators), + AdaBoostClassifier(DecisionTreeClassifier(max_depth=3), n_estimators=n_estimators), +] for pair in ([0, 1], [0, 2], [2, 3]): for model in models: @@ -94,15 +101,12 @@ scores = model.score(X, y) # Create a title for each column and the console by using str() and # slicing away useless parts of the string - model_title = str(type(model)).split( - ".")[-1][:-2][:-len("Classifier")] + model_title = str(type(model)).split(".")[-1][:-2][: -len("Classifier")] model_details = model_title if hasattr(model, "estimators_"): - model_details += " with {} estimators".format( - len(model.estimators_)) - print(model_details + " with features", pair, - "has a score of", scores) + model_details += " with {} estimators".format(len(model.estimators_)) + print(model_details + " with features", pair, "has a score of", scores) plt.subplot(3, 4, plot_idx) if plot_idx <= len(models): @@ -113,8 +117,9 @@ # filled contour plot x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1 y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1 - xx, yy = np.meshgrid(np.arange(x_min, x_max, plot_step), - np.arange(y_min, y_max, plot_step)) + xx, yy = np.meshgrid( + np.arange(x_min, x_max, plot_step), np.arange(y_min, y_max, plot_step) + ) # Plot either a single DecisionTreeClassifier or alpha blend the # decision surfaces of the ensemble of classifiers @@ -139,19 +144,30 @@ # black outline xx_coarser, yy_coarser = np.meshgrid( np.arange(x_min, x_max, plot_step_coarser), - np.arange(y_min, y_max, plot_step_coarser)) - Z_points_coarser = model.predict(np.c_[xx_coarser.ravel(), - yy_coarser.ravel()] - ).reshape(xx_coarser.shape) - cs_points = plt.scatter(xx_coarser, yy_coarser, s=15, - c=Z_points_coarser, cmap=cmap, - edgecolors="none") + np.arange(y_min, y_max, plot_step_coarser), + ) + Z_points_coarser = model.predict( + np.c_[xx_coarser.ravel(), yy_coarser.ravel()] + ).reshape(xx_coarser.shape) + cs_points = plt.scatter( + xx_coarser, + yy_coarser, + s=15, + c=Z_points_coarser, + cmap=cmap, + edgecolors="none", + ) # Plot the training points, these are clustered together and have a # black outline - plt.scatter(X[:, 0], X[:, 1], c=y, - cmap=ListedColormap(['r', 'y', 'b']), - edgecolor='k', s=20) + plt.scatter( + X[:, 0], + X[:, 1], + c=y, + cmap=ListedColormap(["r", "y", "b"]), + edgecolor="k", + s=20, + ) plot_idx += 1 # move on to the next plot in sequence plt.suptitle("Classifiers on feature subsets of the Iris dataset", fontsize=12) diff --git a/examples/ensemble/plot_gradient_boosting_categorical.py b/examples/ensemble/plot_gradient_boosting_categorical.py index 820a508f4de3c..e80c0fb6fdc6e 100644 --- a/examples/ensemble/plot_gradient_boosting_categorical.py +++ b/examples/ensemble/plot_gradient_boosting_categorical.py @@ -18,22 +18,61 @@ category support ` of the :class:`~ensemble.HistGradientBoostingRegressor` estimator. -We will work with the Ames Lowa Housing dataset which consists of numerical +We will work with the Ames Iowa Housing dataset which consists of numerical and categorical features, where the houses' sales prices is the target. + +See :ref:`sphx_glr_auto_examples_ensemble_plot_hgbt_regression.py` for an +example showcasing some other features of +:class:`~ensemble.HistGradientBoostingRegressor`. + """ -print(__doc__) + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause # %% # Load Ames Housing dataset # ------------------------- -# First, we load the ames housing data as a pandas dataframe. The features +# First, we load the Ames Housing data as a pandas dataframe. The features # are either categorical or numerical: from sklearn.datasets import fetch_openml -X, y = fetch_openml(data_id=41211, as_frame=True, return_X_y=True) +X, y = fetch_openml(data_id=42165, as_frame=True, return_X_y=True) + +# Select only a subset of features of X to make the example faster to run +categorical_columns_subset = [ + "BldgType", + "GarageFinish", + "LotConfig", + "Functional", + "MasVnrType", + "HouseStyle", + "FireplaceQu", + "ExterCond", + "ExterQual", + "PoolQC", +] + +numerical_columns_subset = [ + "3SsnPorch", + "Fireplaces", + "BsmtHalfBath", + "HalfBath", + "GarageCars", + "TotRmsAbvGrd", + "BsmtFinSF1", + "BsmtFinSF2", + "GrLivArea", + "ScreenPorch", +] + +X = X[categorical_columns_subset + numerical_columns_subset] +X[categorical_columns_subset] = X[categorical_columns_subset].astype("category") + +categorical_columns = X.select_dtypes(include="category").columns +n_categorical_features = len(categorical_columns) +n_numerical_features = X.select_dtypes(include="number").shape[1] -n_categorical_features = (X.dtypes == 'category').sum() -n_numerical_features = (X.dtypes == 'float').sum() print(f"Number of samples: {X.shape[0]}") print(f"Number of features: {X.shape[1]}") print(f"Number of categorical features: {n_categorical_features}") @@ -45,17 +84,14 @@ # As a baseline, we create an estimator where the categorical features are # dropped: -from sklearn.experimental import enable_hist_gradient_boosting # noqa +from sklearn.compose import make_column_selector, make_column_transformer from sklearn.ensemble import HistGradientBoostingRegressor from sklearn.pipeline import make_pipeline -from sklearn.compose import make_column_transformer -from sklearn.compose import make_column_selector dropper = make_column_transformer( - ('drop', make_column_selector(dtype_include='category')), - remainder='passthrough') -hist_dropped = make_pipeline(dropper, - HistGradientBoostingRegressor(random_state=42)) + ("drop", make_column_selector(dtype_include="category")), remainder="passthrough" +) +hist_dropped = make_pipeline(dropper, HistGradientBoostingRegressor(random_state=42)) # %% # Gradient boosting estimator with one-hot encoding @@ -66,12 +102,16 @@ from sklearn.preprocessing import OneHotEncoder one_hot_encoder = make_column_transformer( - (OneHotEncoder(sparse=False, handle_unknown='ignore'), - make_column_selector(dtype_include='category')), - remainder='passthrough') + ( + OneHotEncoder(sparse_output=False, handle_unknown="ignore"), + make_column_selector(dtype_include="category"), + ), + remainder="passthrough", +) -hist_one_hot = make_pipeline(one_hot_encoder, - HistGradientBoostingRegressor(random_state=42)) +hist_one_hot = make_pipeline( + one_hot_encoder, HistGradientBoostingRegressor(random_state=42) +) # %% # Gradient boosting estimator with ordinal encoding @@ -80,42 +120,42 @@ # were ordered quantities, i.e. the categories will be encoded as 0, 1, 2, # etc., and treated as continuous features. -from sklearn.preprocessing import OrdinalEncoder import numpy as np +from sklearn.preprocessing import OrdinalEncoder + ordinal_encoder = make_column_transformer( - (OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=np.nan), - make_column_selector(dtype_include='category')), - remainder='passthrough') + ( + OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=np.nan), + make_column_selector(dtype_include="category"), + ), + remainder="passthrough", + # Use short feature names to make it easier to specify the categorical + # variables in the HistGradientBoostingRegressor in the next step + # of the pipeline. + verbose_feature_names_out=False, +) -hist_ordinal = make_pipeline(ordinal_encoder, - HistGradientBoostingRegressor(random_state=42)) +hist_ordinal = make_pipeline( + ordinal_encoder, HistGradientBoostingRegressor(random_state=42) +) # %% # Gradient boosting estimator with native categorical support # ----------------------------------------------------------- # We now create a :class:`~ensemble.HistGradientBoostingRegressor` estimator # that will natively handle categorical features. This estimator will not treat -# categorical features as ordered quantities. +# categorical features as ordered quantities. We set +# `categorical_features="from_dtype"` such that features with categorical dtype +# are considered categorical features. # -# Since the :class:`~ensemble.HistGradientBoostingRegressor` requires category -# values to be encoded in `[0, n_unique_categories - 1]`, we still rely on an -# :class:`~preprocessing.OrdinalEncoder` to pre-process the data. -# -# The main difference between this pipeline and the previous one is that in -# this one, we let the :class:`~ensemble.HistGradientBoostingRegressor` know -# which features are categorical. - -# The ordinal encoder will first output the categorical features, and then the -# continuous (passed-through) features -categorical_mask = ([True] * n_categorical_features + - [False] * n_numerical_features) -hist_native = make_pipeline( - ordinal_encoder, - HistGradientBoostingRegressor(random_state=42, - categorical_features=categorical_mask) -) +# The main difference between this estimator and the previous one is that in +# this one, we let the :class:`~ensemble.HistGradientBoostingRegressor` detect +# which features are categorical from the DataFrame columns' dtypes. +hist_native = HistGradientBoostingRegressor( + random_state=42, categorical_features="from_dtype" +) # %% # Model comparison @@ -124,37 +164,57 @@ # models performance in terms of # :func:`~metrics.mean_absolute_percentage_error` and fit times. -from sklearn.model_selection import cross_validate import matplotlib.pyplot as plt +from sklearn.model_selection import cross_validate + scoring = "neg_mean_absolute_percentage_error" -dropped_result = cross_validate(hist_dropped, X, y, cv=3, scoring=scoring) -one_hot_result = cross_validate(hist_one_hot, X, y, cv=3, scoring=scoring) -ordinal_result = cross_validate(hist_ordinal, X, y, cv=3, scoring=scoring) -native_result = cross_validate(hist_native, X, y, cv=3, scoring=scoring) +n_cv_folds = 3 + +dropped_result = cross_validate(hist_dropped, X, y, cv=n_cv_folds, scoring=scoring) +one_hot_result = cross_validate(hist_one_hot, X, y, cv=n_cv_folds, scoring=scoring) +ordinal_result = cross_validate(hist_ordinal, X, y, cv=n_cv_folds, scoring=scoring) +native_result = cross_validate(hist_native, X, y, cv=n_cv_folds, scoring=scoring) def plot_results(figure_title): fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 8)) - plot_info = [('fit_time', 'Fit times (s)', ax1, None), - ('test_score', 'Mean Absolute Percentage Error', ax2, - (0, 0.20))] + plot_info = [ + ("fit_time", "Fit times (s)", ax1, None), + ("test_score", "Mean Absolute Percentage Error", ax2, None), + ] x, width = np.arange(4), 0.9 for key, title, ax, y_limit in plot_info: - items = [dropped_result[key], one_hot_result[key], ordinal_result[key], - native_result[key]] - ax.bar(x, [np.mean(np.abs(item)) for item in items], - width, yerr=[np.std(item) for item in items], - color=['C0', 'C1', 'C2', 'C3']) - ax.set(xlabel='Model', title=title, xticks=x, - xticklabels=["Dropped", "One Hot", "Ordinal", "Native"], - ylim=y_limit) + items = [ + dropped_result[key], + one_hot_result[key], + ordinal_result[key], + native_result[key], + ] + + mape_cv_mean = [np.mean(np.abs(item)) for item in items] + mape_cv_std = [np.std(item) for item in items] + + ax.bar( + x=x, + height=mape_cv_mean, + width=width, + yerr=mape_cv_std, + color=["C0", "C1", "C2", "C3"], + ) + ax.set( + xlabel="Model", + title=title, + xticks=x, + xticklabels=["Dropped", "One Hot", "Ordinal", "Native"], + ylim=y_limit, + ) fig.suptitle(figure_title) -plot_results("Gradient Boosting on Adult Census") +plot_results("Gradient Boosting on Ames Housing") # %% # We see that the model with one-hot-encoded data is by far the slowest. This @@ -172,9 +232,8 @@ def plot_results(figure_title): # comparable error rates, with a slight edge for the native handling. # %% -# Limitting the number of splits -# ------------------------------ -# +# Limiting the number of splits +# ----------------------------- # In general, one can expect poorer predictions from one-hot-encoded data, # especially when the tree depths or the number of nodes are limited: with # one-hot-encoded data, one needs more split points, i.e. more depth, in order @@ -191,19 +250,26 @@ def plot_results(figure_title): # dataset and on the flexibility of the trees. # # To see this, let us re-run the same analysis with under-fitting models where -# we artificially limit the total number of splits by both limitting the number +# we artificially limit the total number of splits by both limiting the number # of trees and the depth of each tree. for pipe in (hist_dropped, hist_one_hot, hist_ordinal, hist_native): - pipe.set_params(histgradientboostingregressor__max_depth=3, - histgradientboostingregressor__max_iter=15) - -dropped_result = cross_validate(hist_dropped, X, y, cv=3, scoring=scoring) -one_hot_result = cross_validate(hist_one_hot, X, y, cv=3, scoring=scoring) -ordinal_result = cross_validate(hist_ordinal, X, y, cv=3, scoring=scoring) -native_result = cross_validate(hist_native, X, y, cv=3, scoring=scoring) - -plot_results("Gradient Boosting on Adult Census (few and small trees)") + if pipe is hist_native: + # The native model does not use a pipeline so, we can set the parameters + # directly. + pipe.set_params(max_depth=3, max_iter=15) + else: + pipe.set_params( + histgradientboostingregressor__max_depth=3, + histgradientboostingregressor__max_iter=15, + ) + +dropped_result = cross_validate(hist_dropped, X, y, cv=n_cv_folds, scoring=scoring) +one_hot_result = cross_validate(hist_one_hot, X, y, cv=n_cv_folds, scoring=scoring) +ordinal_result = cross_validate(hist_ordinal, X, y, cv=n_cv_folds, scoring=scoring) +native_result = cross_validate(hist_native, X, y, cv=n_cv_folds, scoring=scoring) + +plot_results("Gradient Boosting on Ames Housing (few and small trees)") plt.show() diff --git a/examples/ensemble/plot_gradient_boosting_early_stopping.py b/examples/ensemble/plot_gradient_boosting_early_stopping.py index ab3acdf6ce08f..5949ebc9ebe9f 100644 --- a/examples/ensemble/plot_gradient_boosting_early_stopping.py +++ b/examples/ensemble/plot_gradient_boosting_early_stopping.py @@ -1,158 +1,182 @@ """ =================================== -Early stopping of Gradient Boosting +Early stopping in Gradient Boosting =================================== -Gradient boosting is an ensembling technique where several weak learners -(regression trees) are combined to yield a powerful single model, in an -iterative fashion. - -Early stopping support in Gradient Boosting enables us to find the least number -of iterations which is sufficient to build a model that generalizes well to -unseen data. - -The concept of early stopping is simple. We specify a ``validation_fraction`` -which denotes the fraction of the whole dataset that will be kept aside from -training to assess the validation loss of the model. The gradient boosting -model is trained using the training set and evaluated using the validation set. -When each additional stage of regression tree is added, the validation set is -used to score the model. This is continued until the scores of the model in -the last ``n_iter_no_change`` stages do not improve by atleast `tol`. After -that the model is considered to have converged and further addition of stages -is "stopped early". - -The number of stages of the final model is available at the attribute -``n_estimators_``. - -This example illustrates how the early stopping can used in the -:class:`~sklearn.ensemble.GradientBoostingClassifier` model to achieve -almost the same accuracy as compared to a model built without early stopping -using many fewer estimators. This can significantly reduce training time, -memory usage and prediction latency. +Gradient Boosting is an ensemble technique that combines multiple weak +learners, typically decision trees, to create a robust and powerful +predictive model. It does so in an iterative fashion, where each new stage +(tree) corrects the errors of the previous ones. + +Early stopping is a technique in Gradient Boosting that allows us to find +the optimal number of iterations required to build a model that generalizes +well to unseen data and avoids overfitting. The concept is simple: we set +aside a portion of our dataset as a validation set (specified using +`validation_fraction`) to assess the model's performance during training. +As the model is iteratively built with additional stages (trees), its +performance on the validation set is monitored as a function of the +number of steps. + +Early stopping becomes effective when the model's performance on the +validation set plateaus or worsens (within deviations specified by `tol`) +over a certain number of consecutive stages (specified by `n_iter_no_change`). +This signals that the model has reached a point where further iterations may +lead to overfitting, and it's time to stop training. + +The number of estimators (trees) in the final model, when early stopping is +applied, can be accessed using the `n_estimators_` attribute. Overall, early +stopping is a valuable tool to strike a balance between model performance and +efficiency in gradient boosting. """ -# Authors: Vighnesh Birodkar -# Raghav RV -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Data Preparation +# ---------------- +# First we load and prepares the California Housing Prices dataset for +# training and evaluation. It subsets the dataset, splits it into training +# and validation sets. import time -import numpy as np import matplotlib.pyplot as plt -from sklearn import ensemble -from sklearn import datasets +from sklearn.datasets import fetch_california_housing +from sklearn.ensemble import GradientBoostingRegressor +from sklearn.metrics import mean_squared_error from sklearn.model_selection import train_test_split -print(__doc__) - -data_list = [datasets.load_iris(), datasets.load_digits()] -data_list = [(d.data, d.target) for d in data_list] -data_list += [datasets.make_hastie_10_2()] -names = ['Iris Data', 'Digits Data', 'Hastie Data'] - -n_gb = [] -score_gb = [] -time_gb = [] -n_gbes = [] -score_gbes = [] -time_gbes = [] - -n_estimators = 500 - -for X, y in data_list: - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, - random_state=0) - - # We specify that if the scores don't improve by atleast 0.01 for the last - # 10 stages, stop fitting additional stages - gbes = ensemble.GradientBoostingClassifier(n_estimators=n_estimators, - validation_fraction=0.2, - n_iter_no_change=5, tol=0.01, - random_state=0) - gb = ensemble.GradientBoostingClassifier(n_estimators=n_estimators, - random_state=0) - start = time.time() - gb.fit(X_train, y_train) - time_gb.append(time.time() - start) - - start = time.time() - gbes.fit(X_train, y_train) - time_gbes.append(time.time() - start) - - score_gb.append(gb.score(X_test, y_test)) - score_gbes.append(gbes.score(X_test, y_test)) - - n_gb.append(gb.n_estimators_) - n_gbes.append(gbes.n_estimators_) - -bar_width = 0.2 -n = len(data_list) -index = np.arange(0, n * bar_width, bar_width) * 2.5 -index = index[0:n] - -# %% -# Compare scores with and without early stopping -# ---------------------------------------------- - -plt.figure(figsize=(9, 5)) - -bar1 = plt.bar(index, score_gb, bar_width, label='Without early stopping', - color='crimson') -bar2 = plt.bar(index + bar_width, score_gbes, bar_width, - label='With early stopping', color='coral') +data = fetch_california_housing() +X, y = data.data[:600], data.target[:600] -plt.xticks(index + bar_width, names) -plt.yticks(np.arange(0, 1.3, 0.1)) +X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42) +# %% +# Model Training and Comparison +# ----------------------------- +# Two :class:`~sklearn.ensemble.GradientBoostingRegressor` models are trained: +# one with and another without early stopping. The purpose is to compare their +# performance. It also calculates the training time and the `n_estimators_` +# used by both models. + +params = dict(n_estimators=1000, max_depth=5, learning_rate=0.1, random_state=42) + +gbm_full = GradientBoostingRegressor(**params) +gbm_early_stopping = GradientBoostingRegressor( + **params, + validation_fraction=0.1, + n_iter_no_change=10, +) + +start_time = time.time() +gbm_full.fit(X_train, y_train) +training_time_full = time.time() - start_time +n_estimators_full = gbm_full.n_estimators_ + +start_time = time.time() +gbm_early_stopping.fit(X_train, y_train) +training_time_early_stopping = time.time() - start_time +estimators_early_stopping = gbm_early_stopping.n_estimators_ -def autolabel(rects, n_estimators): - """ - Attach a text label above each bar displaying n_estimators of each model - """ - for i, rect in enumerate(rects): - plt.text(rect.get_x() + rect.get_width() / 2., - 1.05 * rect.get_height(), 'n_est=%d' % n_estimators[i], - ha='center', va='bottom') - - -autolabel(bar1, n_gb) -autolabel(bar2, n_gbes) - -plt.ylim([0, 1.3]) -plt.legend(loc='best') -plt.grid(True) - -plt.xlabel('Datasets') -plt.ylabel('Test score') +# %% +# Error Calculation +# ----------------- +# The code calculates the :func:`~sklearn.metrics.mean_squared_error` for both +# training and validation datasets for the models trained in the previous +# section. It computes the errors for each boosting iteration. The purpose is +# to assess the performance and convergence of the models. + +train_errors_without = [] +val_errors_without = [] + +train_errors_with = [] +val_errors_with = [] + +for i, (train_pred, val_pred) in enumerate( + zip( + gbm_full.staged_predict(X_train), + gbm_full.staged_predict(X_val), + ) +): + train_errors_without.append(mean_squared_error(y_train, train_pred)) + val_errors_without.append(mean_squared_error(y_val, val_pred)) + +for i, (train_pred, val_pred) in enumerate( + zip( + gbm_early_stopping.staged_predict(X_train), + gbm_early_stopping.staged_predict(X_val), + ) +): + train_errors_with.append(mean_squared_error(y_train, train_pred)) + val_errors_with.append(mean_squared_error(y_val, val_pred)) +# %% +# Visualize Comparison +# -------------------- +# It includes three subplots: +# +# 1. Plotting training errors of both models over boosting iterations. +# 2. Plotting validation errors of both models over boosting iterations. +# 3. Creating a bar chart to compare the training times and the estimator used +# of the models with and without early stopping. +# + +fig, axes = plt.subplots(ncols=3, figsize=(12, 4)) + +axes[0].plot(train_errors_without, label="gbm_full") +axes[0].plot(train_errors_with, label="gbm_early_stopping") +axes[0].set_xlabel("Boosting Iterations") +axes[0].set_ylabel("MSE (Training)") +axes[0].set_yscale("log") +axes[0].legend() +axes[0].set_title("Training Error") + +axes[1].plot(val_errors_without, label="gbm_full") +axes[1].plot(val_errors_with, label="gbm_early_stopping") +axes[1].set_xlabel("Boosting Iterations") +axes[1].set_ylabel("MSE (Validation)") +axes[1].set_yscale("log") +axes[1].legend() +axes[1].set_title("Validation Error") + +training_times = [training_time_full, training_time_early_stopping] +labels = ["gbm_full", "gbm_early_stopping"] +bars = axes[2].bar(labels, training_times) +axes[2].set_ylabel("Training Time (s)") + +for bar, n_estimators in zip(bars, [n_estimators_full, estimators_early_stopping]): + height = bar.get_height() + axes[2].text( + bar.get_x() + bar.get_width() / 2, + height + 0.001, + f"Estimators: {n_estimators}", + ha="center", + va="bottom", + ) + +plt.tight_layout() plt.show() - # %% -# Compare fit times with and without early stopping -# ------------------------------------------------- - -plt.figure(figsize=(9, 5)) - -bar1 = plt.bar(index, time_gb, bar_width, label='Without early stopping', - color='crimson') -bar2 = plt.bar(index + bar_width, time_gbes, bar_width, - label='With early stopping', color='coral') - -max_y = np.amax(np.maximum(time_gb, time_gbes)) - -plt.xticks(index + bar_width, names) -plt.yticks(np.linspace(0, 1.3 * max_y, 13)) +# The difference in training error between the `gbm_full` and the +# `gbm_early_stopping` stems from the fact that `gbm_early_stopping` sets +# aside `validation_fraction` of the training data as internal validation set. +# Early stopping is decided based on this internal validation score. -autolabel(bar1, n_gb) -autolabel(bar2, n_gbes) - -plt.ylim([0, 1.3 * max_y]) -plt.legend(loc='best') -plt.grid(True) - -plt.xlabel('Datasets') -plt.ylabel('Fit Time') - -plt.show() +# %% +# Summary +# ------- +# In our example with the :class:`~sklearn.ensemble.GradientBoostingRegressor` +# model on the California Housing Prices dataset, we have demonstrated the +# practical benefits of early stopping: +# +# - **Preventing Overfitting:** We showed how the validation error stabilizes +# or starts to increase after a certain point, indicating that the model +# generalizes better to unseen data. This is achieved by stopping the training +# process before overfitting occurs. +# - **Improving Training Efficiency:** We compared training times between +# models with and without early stopping. The model with early stopping +# achieved comparable accuracy while requiring significantly fewer +# estimators, resulting in faster training. diff --git a/examples/ensemble/plot_gradient_boosting_oob.py b/examples/ensemble/plot_gradient_boosting_oob.py index c3b9321f166be..8c4ff8d755ebe 100644 --- a/examples/ensemble/plot_gradient_boosting_oob.py +++ b/examples/ensemble/plot_gradient_boosting_oob.py @@ -2,7 +2,6 @@ ====================================== Gradient Boosting Out-of-Bag estimates ====================================== - Out-of-bag (OOB) estimates can be a useful heuristic to estimate the "optimal" number of boosting iterations. OOB estimates are almost identical to cross-validation estimates but @@ -14,7 +13,6 @@ (the so-called out-of-bag examples). The OOB estimator is a pessimistic estimator of the true test loss, but remains a fairly good approximation for a small number of trees. - The figure shows the cumulative sum of the negative OOB improvements as a function of the boosting iteration. As you can see, it tracks the test loss for the first hundred iterations but then diverges in a @@ -23,20 +21,17 @@ usually gives a better estimate of the test loss but is computationally more demanding. """ -print(__doc__) -# Author: Peter Prettenhofer -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt +import numpy as np +from scipy.special import expit from sklearn import ensemble -from sklearn.model_selection import KFold -from sklearn.model_selection import train_test_split - -from scipy.special import expit +from sklearn.metrics import log_loss +from sklearn.model_selection import KFold, train_test_split # Generate data (adapted from G. Ridgeway's gbm example) n_samples = 1000 @@ -51,27 +46,32 @@ X = np.c_[x1, x2, x3] X = X.astype(np.float32) -X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, - random_state=9) +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=9) # Fit classifier with out-of-bag estimates -params = {'n_estimators': 1200, 'max_depth': 3, 'subsample': 0.5, - 'learning_rate': 0.01, 'min_samples_leaf': 1, 'random_state': 3} +params = { + "n_estimators": 1200, + "max_depth": 3, + "subsample": 0.5, + "learning_rate": 0.01, + "min_samples_leaf": 1, + "random_state": 3, +} clf = ensemble.GradientBoostingClassifier(**params) clf.fit(X_train, y_train) acc = clf.score(X_test, y_test) print("Accuracy: {:.4f}".format(acc)) -n_estimators = params['n_estimators'] +n_estimators = params["n_estimators"] x = np.arange(n_estimators) + 1 def heldout_score(clf, X_test, y_test): - """compute deviance scores on ``X_test`` and ``y_test``. """ + """compute deviance scores on ``X_test`` and ``y_test``.""" score = np.zeros((n_estimators,), dtype=np.float64) - for i, y_pred in enumerate(clf.staged_decision_function(X_test)): - score[i] = clf.loss_(y_test, y_pred) + for i, y_proba in enumerate(clf.staged_predict_proba(X_test)): + score[i] = 2 * log_loss(y_test, y_proba[:, 1]) return score @@ -111,27 +111,33 @@ def cv_estimate(n_splits=None): test_color = list(map(lambda x: x / 256.0, (127, 201, 127))) cv_color = list(map(lambda x: x / 256.0, (253, 192, 134))) +# line type for the three curves +oob_line = "dashed" +test_line = "solid" +cv_line = "dashdot" + # plot curves and vertical lines for best iterations -plt.plot(x, cumsum, label='OOB loss', color=oob_color) -plt.plot(x, test_score, label='Test loss', color=test_color) -plt.plot(x, cv_score, label='CV loss', color=cv_color) -plt.axvline(x=oob_best_iter, color=oob_color) -plt.axvline(x=test_best_iter, color=test_color) -plt.axvline(x=cv_best_iter, color=cv_color) +plt.figure(figsize=(8, 4.8)) +plt.plot(x, cumsum, label="OOB loss", color=oob_color, linestyle=oob_line) +plt.plot(x, test_score, label="Test loss", color=test_color, linestyle=test_line) +plt.plot(x, cv_score, label="CV loss", color=cv_color, linestyle=cv_line) +plt.axvline(x=oob_best_iter, color=oob_color, linestyle=oob_line) +plt.axvline(x=test_best_iter, color=test_color, linestyle=test_line) +plt.axvline(x=cv_best_iter, color=cv_color, linestyle=cv_line) # add three vertical lines to xticks xticks = plt.xticks() -xticks_pos = np.array(xticks[0].tolist() + - [oob_best_iter, cv_best_iter, test_best_iter]) -xticks_label = np.array(list(map(lambda t: int(t), xticks[0])) + - ['OOB', 'CV', 'Test']) +xticks_pos = np.array( + xticks[0].tolist() + [oob_best_iter, cv_best_iter, test_best_iter] +) +xticks_label = np.array(list(map(lambda t: int(t), xticks[0])) + ["OOB", "CV", "Test"]) ind = np.argsort(xticks_pos) xticks_pos = xticks_pos[ind] xticks_label = xticks_label[ind] -plt.xticks(xticks_pos, xticks_label) +plt.xticks(xticks_pos, xticks_label, rotation=90) -plt.legend(loc='upper right') -plt.ylabel('normalized loss') -plt.xlabel('number of iterations') +plt.legend(loc="upper center") +plt.ylabel("normalized loss") +plt.xlabel("number of iterations") plt.show() diff --git a/examples/ensemble/plot_gradient_boosting_quantile.py b/examples/ensemble/plot_gradient_boosting_quantile.py index f29a87fe6cff7..dbe3a99b045dd 100644 --- a/examples/ensemble/plot_gradient_boosting_quantile.py +++ b/examples/ensemble/plot_gradient_boosting_quantile.py @@ -4,12 +4,20 @@ ===================================================== This example shows how quantile regression can be used to create prediction -intervals. +intervals. See :ref:`sphx_glr_auto_examples_ensemble_plot_hgbt_regression.py` +for an example showcasing some other features of +:class:`~ensemble.HistGradientBoostingRegressor`. + """ + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + # %% # Generate some data for a synthetic regression problem by applying the # function f to uniformly sampled random inputs. import numpy as np + from sklearn.model_selection import train_test_split @@ -33,7 +41,7 @@ def f(x): # The lognormal distribution is non-symmetric and long tailed: observing large # outliers is likely but it is impossible to observe small outliers. sigma = 0.5 + X.ravel() / 10 -noise = rng.lognormal(sigma=sigma) - np.exp(sigma ** 2 / 2) +noise = rng.lognormal(sigma=sigma) - np.exp(sigma**2 / 2) y = expected_y + noise # %% @@ -56,25 +64,28 @@ def f(x): from sklearn.ensemble import GradientBoostingRegressor from sklearn.metrics import mean_pinball_loss, mean_squared_error - all_models = {} common_params = dict( learning_rate=0.05, - n_estimators=250, + n_estimators=200, max_depth=2, min_samples_leaf=9, min_samples_split=9, ) for alpha in [0.05, 0.5, 0.95]: - gbr = GradientBoostingRegressor(loss='quantile', alpha=alpha, - **common_params) + gbr = GradientBoostingRegressor(loss="quantile", alpha=alpha, **common_params) all_models["q %1.2f" % alpha] = gbr.fit(X_train, y_train) # %% -# For the sake of comparison, also fit a baseline model trained with the usual -# least squares loss (ls), also known as the mean squared error (MSE). -gbr_ls = GradientBoostingRegressor(loss='ls', **common_params) -all_models["ls"] = gbr_ls.fit(X_train, y_train) +# Notice that :class:`~sklearn.ensemble.HistGradientBoostingRegressor` is much +# faster than :class:`~sklearn.ensemble.GradientBoostingRegressor` starting with +# intermediate datasets (`n_samples >= 10_000`), which is not the case of the +# present example. +# +# For the sake of comparison, we also fit a baseline model trained with the +# usual (mean) squared error (MSE). +gbr_ls = GradientBoostingRegressor(loss="squared_error", **common_params) +all_models["mse"] = gbr_ls.fit(X_train, y_train) # %% # Create an evenly spaced evaluation set of input values spanning the [0, 10] @@ -82,30 +93,28 @@ def f(x): xx = np.atleast_2d(np.linspace(0, 10, 1000)).T # %% -# Plot the true conditional mean function f, the prediction of the conditional -# mean (least squares loss), the conditional median and the conditional 90% -# interval (from 5th to 95th conditional percentiles). +# Plot the true conditional mean function f, the predictions of the conditional +# mean (loss equals squared error), the conditional median and the conditional +# 90% interval (from 5th to 95th conditional percentiles). import matplotlib.pyplot as plt - -y_pred = all_models['ls'].predict(xx) -y_lower = all_models['q 0.05'].predict(xx) -y_upper = all_models['q 0.95'].predict(xx) -y_med = all_models['q 0.50'].predict(xx) +y_pred = all_models["mse"].predict(xx) +y_lower = all_models["q 0.05"].predict(xx) +y_upper = all_models["q 0.95"].predict(xx) +y_med = all_models["q 0.50"].predict(xx) fig = plt.figure(figsize=(10, 10)) -plt.plot(xx, f(xx), 'g:', linewidth=3, label=r'$f(x) = x\,\sin(x)$') -plt.plot(X_test, y_test, 'b.', markersize=10, label='Test observations') -plt.plot(xx, y_med, 'r-', label='Predicted median', color="orange") -plt.plot(xx, y_pred, 'r-', label='Predicted mean') -plt.plot(xx, y_upper, 'k-') -plt.plot(xx, y_lower, 'k-') -plt.fill_between(xx.ravel(), y_lower, y_upper, alpha=0.4, - label='Predicted 90% interval') -plt.xlabel('$x$') -plt.ylabel('$f(x)$') +plt.plot(xx, f(xx), "black", linewidth=3, label=r"$f(x) = x\,\sin(x)$") +plt.plot(X_test, y_test, "b.", markersize=10, label="Test observations") +plt.plot(xx, y_med, "tab:orange", linewidth=3, label="Predicted median") +plt.plot(xx, y_pred, "tab:green", linewidth=3, label="Predicted mean") +plt.fill_between( + xx.ravel(), y_lower, y_upper, alpha=0.4, label="Predicted 90% interval" +) +plt.xlabel("$x$") +plt.ylabel("$f(x)$") plt.ylim(-10, 25) -plt.legend(loc='upper left') +plt.legend(loc="upper left") plt.show() # %% @@ -122,28 +131,26 @@ def f(x): # Analysis of the error metrics # ----------------------------- # -# Measure the models with :func:`mean_squared_error` and -# :func:`mean_pinball_loss` metrics on the training dataset. +# Measure the models with :func:`~sklearn.metrics.mean_squared_error` and +# :func:`~sklearn.metrics.mean_pinball_loss` metrics on the training dataset. import pandas as pd def highlight_min(x): x_min = x.min() - return ['font-weight: bold' if v == x_min else '' - for v in x] + return ["font-weight: bold" if v == x_min else "" for v in x] results = [] for name, gbr in sorted(all_models.items()): - metrics = {'model': name} + metrics = {"model": name} y_pred = gbr.predict(X_train) for alpha in [0.05, 0.5, 0.95]: - metrics["pbl=%1.2f" % alpha] = mean_pinball_loss( - y_train, y_pred, alpha=alpha) - metrics['MSE'] = mean_squared_error(y_train, y_pred) + metrics["pbl=%1.2f" % alpha] = mean_pinball_loss(y_train, y_pred, alpha=alpha) + metrics["MSE"] = mean_squared_error(y_train, y_pred) results.append(metrics) -pd.DataFrame(results).set_index('model').style.apply(highlight_min) +pd.DataFrame(results).set_index("model").style.apply(highlight_min) # %% # One column shows all models evaluated by the same metric. The minimum number @@ -152,8 +159,8 @@ def highlight_min(x): # training converged. # # Note that because the target distribution is asymmetric, the expected -# conditional mean and conditional median are signficiantly different and -# therefore one could not use the least squares model get a good estimation of +# conditional mean and conditional median are significantly different and +# therefore one could not use the squared error model get a good estimation of # the conditional median nor the converse. # # If the target distribution were symmetric and had no outliers (e.g. with a @@ -163,15 +170,14 @@ def highlight_min(x): # We then do the same on the test set. results = [] for name, gbr in sorted(all_models.items()): - metrics = {'model': name} + metrics = {"model": name} y_pred = gbr.predict(X_test) for alpha in [0.05, 0.5, 0.95]: - metrics["pbl=%1.2f" % alpha] = mean_pinball_loss( - y_test, y_pred, alpha=alpha) - metrics['MSE'] = mean_squared_error(y_test, y_pred) + metrics["pbl=%1.2f" % alpha] = mean_pinball_loss(y_test, y_pred, alpha=alpha) + metrics["MSE"] = mean_squared_error(y_test, y_pred) results.append(metrics) -pd.DataFrame(results).set_index('model').style.apply(highlight_min) +pd.DataFrame(results).set_index("model").style.apply(highlight_min) # %% @@ -179,19 +185,21 @@ def highlight_min(x): # shows that the best test metric is obtained when the model is trained by # minimizing this same metric. # -# Note that the conditional median estimator is competitive with the least -# squares estimator in terms of MSE on the test set: this can be explained by -# the fact the least squares estimator is very sensitive to large outliers +# Note that the conditional median estimator is competitive with the squared +# error estimator in terms of MSE on the test set: this can be explained by +# the fact the squared error estimator is very sensitive to large outliers # which can cause significant overfitting. This can be seen on the right hand # side of the previous plot. The conditional median estimator is biased -# (underestimation for this asymetric noise) but is also naturally robust to +# (underestimation for this asymmetric noise) but is also naturally robust to # outliers and overfits less. # +# .. _calibration-section: +# # Calibration of the confidence interval # -------------------------------------- # # We can also evaluate the ability of the two extreme quantile estimators at -# producing a well-calibrated conditational 90%-confidence interval. +# producing a well-calibrated conditional 90%-confidence interval. # # To do this we can compute the fraction of observations that fall between the # predictions: @@ -199,16 +207,18 @@ def coverage_fraction(y, y_low, y_high): return np.mean(np.logical_and(y >= y_low, y <= y_high)) -coverage_fraction(y_train, - all_models['q 0.05'].predict(X_train), - all_models['q 0.95'].predict(X_train)) +coverage_fraction( + y_train, + all_models["q 0.05"].predict(X_train), + all_models["q 0.95"].predict(X_train), +) # %% # On the training set the calibration is very close to the expected coverage # value for a 90% confidence interval. -coverage_fraction(y_test, - all_models['q 0.05'].predict(X_test), - all_models['q 0.95'].predict(X_test)) +coverage_fraction( + y_test, all_models["q 0.05"].predict(X_test), all_models["q 0.95"].predict(X_test) +) # %% @@ -223,7 +233,7 @@ def coverage_fraction(y, y_low, y_high): # underfit and could not adapt to sinusoidal shape of the signal. # # The hyper-parameters of the model were approximately hand-tuned for the -# median regressor and there is no reason than the same hyper-parameters are +# median regressor and there is no reason that the same hyper-parameters are # suitable for the 5th percentile regressor. # # To confirm this hypothesis, we tune the hyper-parameters of a new regressor @@ -231,17 +241,17 @@ def coverage_fraction(y, y_low, y_high): # cross-validation on the pinball loss with alpha=0.05: # %% -from sklearn.model_selection import RandomizedSearchCV -from sklearn.metrics import make_scorer from pprint import pprint +from sklearn.experimental import enable_halving_search_cv # noqa: F401 +from sklearn.metrics import make_scorer +from sklearn.model_selection import HalvingRandomSearchCV param_grid = dict( - learning_rate=[0.01, 0.05, 0.1], - n_estimators=[100, 150, 200, 250, 300], - max_depth=[2, 5, 10, 15, 20], - min_samples_leaf=[1, 5, 10, 20, 30, 50], - min_samples_split=[2, 5, 10, 20, 30, 50], + learning_rate=[0.05, 0.1, 0.2], + max_depth=[2, 5, 10], + min_samples_leaf=[1, 5, 10, 20], + min_samples_split=[5, 10, 20, 30, 50], ) alpha = 0.05 neg_mean_pinball_loss_05p_scorer = make_scorer( @@ -250,10 +260,12 @@ def coverage_fraction(y, y_low, y_high): greater_is_better=False, # maximize the negative loss ) gbr = GradientBoostingRegressor(loss="quantile", alpha=alpha, random_state=0) -search_05p = RandomizedSearchCV( +search_05p = HalvingRandomSearchCV( gbr, param_grid, - n_iter=10, # increase this if computational budget allows + resource="n_estimators", + max_resources=250, + min_resources=50, scoring=neg_mean_pinball_loss_05p_scorer, n_jobs=2, random_state=0, @@ -261,9 +273,9 @@ def coverage_fraction(y, y_low, y_high): pprint(search_05p.best_params_) # %% -# We observe that the search procedure identifies that deeper trees are needed -# to get a good fit for the 5th percentile regressor. Deeper trees are more -# expressive and less likely to underfit. +# We observe that the hyper-parameters that were hand-tuned for the median +# regressor are in the same range as the hyper-parameters suitable for the 5th +# percentile regressor. # # Let's now tune the hyper-parameters for the 95th percentile regressor. We # need to redefine the `scoring` metric used to select the best model, along @@ -285,29 +297,27 @@ def coverage_fraction(y, y_low, y_high): pprint(search_95p.best_params_) # %% -# This time, shallower trees are selected and lead to a more constant piecewise -# and therefore more robust estimation of the 95th percentile. This is -# beneficial as it avoids overfitting the large outliers of the log-normal -# additive noise. -# -# We can confirm this intuition by displaying the predicted 90% confidence -# interval comprised by the predictions of those two tuned quantile regressors: -# the prediction of the upper 95th percentile has a much coarser shape than the -# prediction of the lower 5th percentile: +# The result shows that the hyper-parameters for the 95th percentile regressor +# identified by the search procedure are roughly in the same range as the hand-tuned +# hyper-parameters for the median regressor and the hyper-parameters +# identified by the search procedure for the 5th percentile regressor. However, +# the hyper-parameter searches did lead to an improved 90% confidence interval +# that is comprised by the predictions of those two tuned quantile regressors. +# Note that the prediction of the upper 95th percentile has a much coarser shape +# than the prediction of the lower 5th percentile because of the outliers: y_lower = search_05p.predict(xx) y_upper = search_95p.predict(xx) fig = plt.figure(figsize=(10, 10)) -plt.plot(xx, f(xx), 'g:', linewidth=3, label=r'$f(x) = x\,\sin(x)$') -plt.plot(X_test, y_test, 'b.', markersize=10, label='Test observations') -plt.plot(xx, y_upper, 'k-') -plt.plot(xx, y_lower, 'k-') -plt.fill_between(xx.ravel(), y_lower, y_upper, alpha=0.4, - label='Predicted 90% interval') -plt.xlabel('$x$') -plt.ylabel('$f(x)$') +plt.plot(xx, f(xx), "black", linewidth=3, label=r"$f(x) = x\,\sin(x)$") +plt.plot(X_test, y_test, "b.", markersize=10, label="Test observations") +plt.fill_between( + xx.ravel(), y_lower, y_upper, alpha=0.4, label="Predicted 90% interval" +) +plt.xlabel("$x$") +plt.ylabel("$f(x)$") plt.ylim(-10, 25) -plt.legend(loc='upper left') +plt.legend(loc="upper left") plt.title("Prediction with tuned hyper-parameters") plt.show() @@ -317,13 +327,9 @@ def coverage_fraction(y, y_low, y_high): # # We now quantitatively evaluate the joint-calibration of the pair of # estimators: -coverage_fraction(y_train, - search_05p.predict(X_train), - search_95p.predict(X_train)) +coverage_fraction(y_train, search_05p.predict(X_train), search_95p.predict(X_train)) # %% -coverage_fraction(y_test, - search_05p.predict(X_test), - search_95p.predict(X_test)) +coverage_fraction(y_test, search_05p.predict(X_test), search_95p.predict(X_test)) # %% # The calibration of the tuned pair is sadly not better on the test set: the # width of the estimated confidence interval is still too narrow. diff --git a/examples/ensemble/plot_gradient_boosting_regression.py b/examples/ensemble/plot_gradient_boosting_regression.py index 92d35b26deb9c..68a50b7a27492 100644 --- a/examples/ensemble/plot_gradient_boosting_regression.py +++ b/examples/ensemble/plot_gradient_boosting_regression.py @@ -11,22 +11,25 @@ and 500 regression trees of depth 4. Note: For larger datasets (n_samples >= 10000), please refer to -:class:`~sklearn.ensemble.HistGradientBoostingRegressor`. +:class:`~sklearn.ensemble.HistGradientBoostingRegressor`. See +:ref:`sphx_glr_auto_examples_ensemble_plot_hgbt_regression.py` for an example +showcasing some other advantages of +:class:`~ensemble.HistGradientBoostingRegressor`. + """ -print(__doc__) -# Author: Peter Prettenhofer -# Maria Telenczuk -# Katrina Ni -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +import matplotlib import matplotlib.pyplot as plt import numpy as np + from sklearn import datasets, ensemble from sklearn.inspection import permutation_importance from sklearn.metrics import mean_squared_error from sklearn.model_selection import train_test_split +from sklearn.utils.fixes import parse_version # %% # Load the data @@ -45,33 +48,36 @@ # for testing. We will also set the regression model parameters. You can play # with these parameters to see how the results change. # -# n_estimators : the number of boosting stages that will be performed. +# `n_estimators` : the number of boosting stages that will be performed. # Later, we will plot deviance against boosting iterations. # -# max_depth : limits the number of nodes in the tree. +# `max_depth` : limits the number of nodes in the tree. # The best value depends on the interaction of the input variables. # -# min_samples_split : the minimum number of samples required to split an +# `min_samples_split` : the minimum number of samples required to split an # internal node. # -# learning_rate : how much the contribution of each tree will shrink. +# `learning_rate` : how much the contribution of each tree will shrink. # -# loss : loss function to optimize. The least squares function is used in this -# case however, there are many other options (see +# `loss` : loss function to optimize. The least squares function is used in +# this case however, there are many other options (see # :class:`~sklearn.ensemble.GradientBoostingRegressor` ). X_train, X_test, y_train, y_test = train_test_split( - X, y, test_size=0.1, random_state=13) + X, y, test_size=0.1, random_state=13 +) -params = {'n_estimators': 500, - 'max_depth': 4, - 'min_samples_split': 5, - 'learning_rate': 0.01, - 'loss': 'ls'} +params = { + "n_estimators": 500, + "max_depth": 4, + "min_samples_split": 5, + "learning_rate": 0.01, + "loss": "squared_error", +} # %% # Fit regression model -# ------------------------------------- +# -------------------- # # Now we will initiate the gradient boosting regressors and fit it with our # training data. Let's also look and the mean squared error on the test data. @@ -84,36 +90,42 @@ # %% # Plot training deviance -# ------------------------------------- +# ---------------------- # # Finally, we will visualize the results. To do that we will first compute the # test set deviance and then plot it against boosting iterations. -test_score = np.zeros((params['n_estimators'],), dtype=np.float64) +test_score = np.zeros((params["n_estimators"],), dtype=np.float64) for i, y_pred in enumerate(reg.staged_predict(X_test)): - test_score[i] = reg.loss_(y_test, y_pred) + test_score[i] = mean_squared_error(y_test, y_pred) fig = plt.figure(figsize=(6, 6)) plt.subplot(1, 1, 1) -plt.title('Deviance') -plt.plot(np.arange(params['n_estimators']) + 1, reg.train_score_, 'b-', - label='Training Set Deviance') -plt.plot(np.arange(params['n_estimators']) + 1, test_score, 'r-', - label='Test Set Deviance') -plt.legend(loc='upper right') -plt.xlabel('Boosting Iterations') -plt.ylabel('Deviance') +plt.title("Deviance") +plt.plot( + np.arange(params["n_estimators"]) + 1, + reg.train_score_, + "b-", + label="Training Set Deviance", +) +plt.plot( + np.arange(params["n_estimators"]) + 1, test_score, "r-", label="Test Set Deviance" +) +plt.legend(loc="upper right") +plt.xlabel("Boosting Iterations") +plt.ylabel("Deviance") fig.tight_layout() plt.show() # %% # Plot feature importance -# ------------------------------------- +# ----------------------- # -# Careful, impurity-based feature importances can be misleading for -# high cardinality features (many unique values). As an alternative, -# the permutation importances of ``reg`` can be computed on a -# held out test set. See :ref:`permutation_importance` for more details. +# .. warning:: +# Careful, impurity-based feature importances can be misleading for +# **high cardinality** features (many unique values). As an alternative, +# the permutation importances of ``reg`` can be computed on a +# held out test set. See :ref:`permutation_importance` for more details. # # For this example, the impurity-based and permutation methods identify the # same 2 strongly predictive features but not in the same order. The third most @@ -123,19 +135,32 @@ feature_importance = reg.feature_importances_ sorted_idx = np.argsort(feature_importance) -pos = np.arange(sorted_idx.shape[0]) + .5 +pos = np.arange(sorted_idx.shape[0]) + 0.5 fig = plt.figure(figsize=(12, 6)) plt.subplot(1, 2, 1) -plt.barh(pos, feature_importance[sorted_idx], align='center') +plt.barh(pos, feature_importance[sorted_idx], align="center") plt.yticks(pos, np.array(diabetes.feature_names)[sorted_idx]) -plt.title('Feature Importance (MDI)') +plt.title("Feature Importance (MDI)") -result = permutation_importance(reg, X_test, y_test, n_repeats=10, - random_state=42, n_jobs=2) +result = permutation_importance( + reg, X_test, y_test, n_repeats=10, random_state=42, n_jobs=2 +) sorted_idx = result.importances_mean.argsort() plt.subplot(1, 2, 2) -plt.boxplot(result.importances[sorted_idx].T, - vert=False, labels=np.array(diabetes.feature_names)[sorted_idx]) + +# `labels` argument in boxplot is deprecated in matplotlib 3.9 and has been +# renamed to `tick_labels`. The following code handles this, but as a +# scikit-learn user you probably can write simpler code by using `labels=...` +# (matplotlib < 3.9) or `tick_labels=...` (matplotlib >= 3.9). +tick_labels_parameter_name = ( + "tick_labels" + if parse_version(matplotlib.__version__) >= parse_version("3.9") + else "labels" +) +tick_labels_dict = { + tick_labels_parameter_name: np.array(diabetes.feature_names)[sorted_idx] +} +plt.boxplot(result.importances[sorted_idx].T, vert=False, **tick_labels_dict) plt.title("Permutation Importance (test set)") fig.tight_layout() plt.show() diff --git a/examples/ensemble/plot_gradient_boosting_regularization.py b/examples/ensemble/plot_gradient_boosting_regularization.py index 592dd40ca47cb..1aa01c7ba4ff4 100644 --- a/examples/ensemble/plot_gradient_boosting_regularization.py +++ b/examples/ensemble/plot_gradient_boosting_regularization.py @@ -18,44 +18,51 @@ .. [1] T. Hastie, R. Tibshirani and J. Friedman, "Elements of Statistical Learning Ed. 2", Springer, 2009. + """ -print(__doc__) -# Author: Peter Prettenhofer -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt +import numpy as np -from sklearn import ensemble -from sklearn import datasets - +from sklearn import datasets, ensemble +from sklearn.metrics import log_loss +from sklearn.model_selection import train_test_split -X, y = datasets.make_hastie_10_2(n_samples=12000, random_state=1) -X = X.astype(np.float32) +X, y = datasets.make_hastie_10_2(n_samples=4000, random_state=1) # map labels from {-1, 1} to {0, 1} labels, y = np.unique(y, return_inverse=True) -X_train, X_test = X[:2000], X[2000:] -y_train, y_test = y[:2000], y[2000:] +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.8, random_state=0) -original_params = {'n_estimators': 1000, 'max_leaf_nodes': 4, 'max_depth': None, 'random_state': 2, - 'min_samples_split': 5} +original_params = { + "n_estimators": 400, + "max_leaf_nodes": 4, + "max_depth": None, + "random_state": 2, + "min_samples_split": 5, +} plt.figure() -for label, color, setting in [('No shrinkage', 'orange', - {'learning_rate': 1.0, 'subsample': 1.0}), - ('learning_rate=0.1', 'turquoise', - {'learning_rate': 0.1, 'subsample': 1.0}), - ('subsample=0.5', 'blue', - {'learning_rate': 1.0, 'subsample': 0.5}), - ('learning_rate=0.1, subsample=0.5', 'gray', - {'learning_rate': 0.1, 'subsample': 0.5}), - ('learning_rate=0.1, max_features=2', 'magenta', - {'learning_rate': 0.1, 'max_features': 2})]: +for label, color, setting in [ + ("No shrinkage", "orange", {"learning_rate": 1.0, "subsample": 1.0}), + ("learning_rate=0.2", "turquoise", {"learning_rate": 0.2, "subsample": 1.0}), + ("subsample=0.5", "blue", {"learning_rate": 1.0, "subsample": 0.5}), + ( + "learning_rate=0.2, subsample=0.5", + "gray", + {"learning_rate": 0.2, "subsample": 0.5}, + ), + ( + "learning_rate=0.2, max_features=2", + "magenta", + {"learning_rate": 0.2, "max_features": 2}, + ), +]: params = dict(original_params) params.update(setting) @@ -63,17 +70,21 @@ clf.fit(X_train, y_train) # compute test set deviance - test_deviance = np.zeros((params['n_estimators'],), dtype=np.float64) - - for i, y_pred in enumerate(clf.staged_decision_function(X_test)): - # clf.loss_ assumes that y_test[i] in {0, 1} - test_deviance[i] = clf.loss_(y_test, y_pred) - - plt.plot((np.arange(test_deviance.shape[0]) + 1)[::5], test_deviance[::5], - '-', color=color, label=label) - -plt.legend(loc='upper left') -plt.xlabel('Boosting Iterations') -plt.ylabel('Test Set Deviance') + test_deviance = np.zeros((params["n_estimators"],), dtype=np.float64) + + for i, y_proba in enumerate(clf.staged_predict_proba(X_test)): + test_deviance[i] = 2 * log_loss(y_test, y_proba[:, 1]) + + plt.plot( + (np.arange(test_deviance.shape[0]) + 1)[::5], + test_deviance[::5], + "-", + color=color, + label=label, + ) + +plt.legend(loc="upper right") +plt.xlabel("Boosting Iterations") +plt.ylabel("Test Set Deviance") plt.show() diff --git a/examples/ensemble/plot_hgbt_regression.py b/examples/ensemble/plot_hgbt_regression.py new file mode 100644 index 0000000000000..dce97a6e0b700 --- /dev/null +++ b/examples/ensemble/plot_hgbt_regression.py @@ -0,0 +1,429 @@ +""" +============================================== +Features in Histogram Gradient Boosting Trees +============================================== + +:ref:`histogram_based_gradient_boosting` (HGBT) models may be one of the most +useful supervised learning models in scikit-learn. They are based on a modern +gradient boosting implementation comparable to LightGBM and XGBoost. As such, +HGBT models are more feature rich than and often outperform alternative models +like random forests, especially when the number of samples is larger than some +ten thousands (see +:ref:`sphx_glr_auto_examples_ensemble_plot_forest_hist_grad_boosting_comparison.py`). + +The top usability features of HGBT models are: + +1. Several available loss functions for mean and quantile regression tasks, see + :ref:`Quantile loss `. +2. :ref:`categorical_support_gbdt`, see + :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_categorical.py`. +3. Early stopping. +4. :ref:`nan_support_hgbt`, which avoids the need for an imputer. +5. :ref:`monotonic_cst_gbdt`. +6. :ref:`interaction_cst_hgbt`. + +This example aims at showcasing all points except 2 and 6 in a real life +setting. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Preparing the data +# ================== +# The `electricity dataset `_ consists of data +# collected from the Australian New South Wales Electricity Market. In this +# market, prices are not fixed and are affected by supply and demand. They are +# set every five minutes. Electricity transfers to/from the neighboring state of +# Victoria were done to alleviate fluctuations. +# +# The dataset, originally named ELEC2, contains 45,312 instances dated from 7 +# May 1996 to 5 December 1998. Each sample of the dataset refers to a period of +# 30 minutes, i.e. there are 48 instances for each time period of one day. Each +# sample on the dataset has 7 columns: +# +# - date: between 7 May 1996 to 5 December 1998. Normalized between 0 and 1; +# - day: day of week (1-7); +# - period: half hour intervals over 24 hours. Normalized between 0 and 1; +# - nswprice/nswdemand: electricity price/demand of New South Wales; +# - vicprice/vicdemand: electricity price/demand of Victoria. +# +# Originally, it is a classification task, but here we use it for the regression +# task to predict the scheduled electricity transfer between states. + +from sklearn.datasets import fetch_openml + +electricity = fetch_openml( + name="electricity", version=1, as_frame=True, parser="pandas" +) +df = electricity.frame + +# %% +# This particular dataset has a stepwise constant target for the first 17,760 +# samples: + +df["transfer"][:17_760].unique() + +# %% +# Let us drop those entries and explore the hourly electricity transfer over +# different days of the week: + +import matplotlib.pyplot as plt +import seaborn as sns + +df = electricity.frame.iloc[17_760:] +X = df.drop(columns=["transfer", "class"]) +y = df["transfer"] + +fig, ax = plt.subplots(figsize=(15, 10)) +pointplot = sns.lineplot(x=df["period"], y=df["transfer"], hue=df["day"], ax=ax) +handles, labels = ax.get_legend_handles_labels() +ax.set( + title="Hourly energy transfer for different days of the week", + xlabel="Normalized time of the day", + ylabel="Normalized energy transfer", +) +_ = ax.legend(handles, ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"]) + +# %% +# Notice that energy transfer increases systematically during weekends. +# +# Effect of number of trees and early stopping +# ============================================ +# For the sake of illustrating the effect of the (maximum) number of trees, we +# train a :class:`~sklearn.ensemble.HistGradientBoostingRegressor` over the +# daily electricity transfer using the whole dataset. Then we visualize its +# predictions depending on the `max_iter` parameter. Here we don't try to +# evaluate the performance of the model and its capacity to generalize but +# rather its capability to learn from the training data. + +from sklearn.ensemble import HistGradientBoostingRegressor +from sklearn.model_selection import train_test_split + +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, shuffle=False) + +print(f"Training sample size: {X_train.shape[0]}") +print(f"Test sample size: {X_test.shape[0]}") +print(f"Number of features: {X_train.shape[1]}") + +# %% +max_iter_list = [5, 50] +average_week_demand = ( + df.loc[X_test.index].groupby(["day", "period"], observed=False)["transfer"].mean() +) +colors = sns.color_palette("colorblind") +fig, ax = plt.subplots(figsize=(10, 5)) +average_week_demand.plot(color=colors[0], label="recorded average", linewidth=2, ax=ax) + +for idx, max_iter in enumerate(max_iter_list): + hgbt = HistGradientBoostingRegressor( + max_iter=max_iter, categorical_features=None, random_state=42 + ) + hgbt.fit(X_train, y_train) + + y_pred = hgbt.predict(X_test) + prediction_df = df.loc[X_test.index].copy() + prediction_df["y_pred"] = y_pred + average_pred = prediction_df.groupby(["day", "period"], observed=False)[ + "y_pred" + ].mean() + average_pred.plot( + color=colors[idx + 1], label=f"max_iter={max_iter}", linewidth=2, ax=ax + ) + +ax.set( + title="Predicted average energy transfer during the week", + xticks=[(i + 0.2) * 48 for i in range(7)], + xticklabels=["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"], + xlabel="Time of the week", + ylabel="Normalized energy transfer", +) +_ = ax.legend() + +# %% +# With just a few iterations, HGBT models can achieve convergence (see +# :ref:`sphx_glr_auto_examples_ensemble_plot_forest_hist_grad_boosting_comparison.py`), +# meaning that adding more trees does not improve the model anymore. In the +# figure above, 5 iterations are not enough to get good predictions. With 50 +# iterations, we are already able to do a good job. +# +# Setting `max_iter` too high might degrade the prediction quality and cost a lot of +# avoidable computing resources. Therefore, the HGBT implementation in scikit-learn +# provides an automatic **early stopping** strategy. With it, the model +# uses a fraction of the training data as internal validation set +# (`validation_fraction`) and stops training if the validation score does not +# improve (or degrades) after `n_iter_no_change` iterations up to a certain +# tolerance (`tol`). +# +# Notice that there is a trade-off between `learning_rate` and `max_iter`: +# Generally, smaller learning rates are preferable but require more iterations +# to converge to the minimum loss, while larger learning rates converge faster +# (less iterations/trees needed) but at the cost of a larger minimum loss. +# +# Because of this high correlation between the learning rate the number of iterations, +# a good practice is to tune the learning rate along with all (important) other +# hyperparameters, fit the HBGT on the training set with a large enough value +# for `max_iter` and determine the best `max_iter` via early stopping and some +# explicit `validation_fraction`. + +common_params = { + "max_iter": 1_000, + "learning_rate": 0.3, + "validation_fraction": 0.2, + "random_state": 42, + "categorical_features": None, + "scoring": "neg_root_mean_squared_error", +} + +hgbt = HistGradientBoostingRegressor(early_stopping=True, **common_params) +hgbt.fit(X_train, y_train) + +_, ax = plt.subplots() +plt.plot(-hgbt.validation_score_) +_ = ax.set( + xlabel="number of iterations", + ylabel="root mean squared error", + title=f"Loss of hgbt with early stopping (n_iter={hgbt.n_iter_})", +) + +# %% +# We can then overwrite the value for `max_iter` to a reasonable value and avoid +# the extra computational cost of the inner validation. Rounding up the number +# of iterations may account for variability of the training set: + +import math + +common_params["max_iter"] = math.ceil(hgbt.n_iter_ / 100) * 100 +common_params["early_stopping"] = False +hgbt = HistGradientBoostingRegressor(**common_params) + +# %% +# .. note:: The inner validation done during early stopping is not optimal for +# time series. +# +# Support for missing values +# ========================== +# HGBT models have native support of missing values. During training, the tree +# grower decides where samples with missing values should go (left or right +# child) at each split, based on the potential gain. When predicting, these +# samples are sent to the learnt child accordingly. If a feature had no missing +# values during training, then for prediction, samples with missing values for that +# feature are sent to the child with the most samples (as seen during fit). +# +# The present example shows how HGBT regressions deal with values missing +# completely at random (MCAR), i.e. the missingness does not depend on the +# observed data or the unobserved data. We can simulate such scenario by +# randomly replacing values from randomly selected features with `nan` values. + +import numpy as np + +from sklearn.metrics import root_mean_squared_error + +rng = np.random.RandomState(42) +first_week = slice(0, 336) # first week in the test set as 7 * 48 = 336 +missing_fraction_list = [0, 0.01, 0.03] + + +def generate_missing_values(X, missing_fraction): + total_cells = X.shape[0] * X.shape[1] + num_missing_cells = int(total_cells * missing_fraction) + row_indices = rng.choice(X.shape[0], num_missing_cells, replace=True) + col_indices = rng.choice(X.shape[1], num_missing_cells, replace=True) + X_missing = X.copy() + X_missing.iloc[row_indices, col_indices] = np.nan + return X_missing + + +fig, ax = plt.subplots(figsize=(12, 6)) +ax.plot(y_test.values[first_week], label="Actual transfer") + +for missing_fraction in missing_fraction_list: + X_train_missing = generate_missing_values(X_train, missing_fraction) + X_test_missing = generate_missing_values(X_test, missing_fraction) + hgbt.fit(X_train_missing, y_train) + y_pred = hgbt.predict(X_test_missing[first_week]) + rmse = root_mean_squared_error(y_test[first_week], y_pred) + ax.plot( + y_pred[first_week], + label=f"missing_fraction={missing_fraction}, RMSE={rmse:.3f}", + alpha=0.5, + ) +ax.set( + title="Daily energy transfer predictions on data with MCAR values", + xticks=[(i + 0.2) * 48 for i in range(7)], + xticklabels=["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"], + xlabel="Time of the week", + ylabel="Normalized energy transfer", +) +_ = ax.legend(loc="lower right") + +# %% +# As expected, the model degrades as the proportion of missing values increases. +# +# Support for quantile loss +# ========================= +# +# The quantile loss in regression enables a view of the variability or +# uncertainty of the target variable. For instance, predicting the 5th and 95th +# percentiles can provide a 90% prediction interval, i.e. the range within which +# we expect a new observed value to fall with 90% probability. + +from sklearn.metrics import mean_pinball_loss + +quantiles = [0.95, 0.05] +predictions = [] + +fig, ax = plt.subplots(figsize=(12, 6)) +ax.plot(y_test.values[first_week], label="Actual transfer") + +for quantile in quantiles: + hgbt_quantile = HistGradientBoostingRegressor( + loss="quantile", quantile=quantile, **common_params + ) + hgbt_quantile.fit(X_train, y_train) + y_pred = hgbt_quantile.predict(X_test[first_week]) + + predictions.append(y_pred) + score = mean_pinball_loss(y_test[first_week], y_pred) + ax.plot( + y_pred[first_week], + label=f"quantile={quantile}, pinball loss={score:.2f}", + alpha=0.5, + ) + +ax.fill_between( + range(len(predictions[0][first_week])), + predictions[0][first_week], + predictions[1][first_week], + color=colors[0], + alpha=0.1, +) +ax.set( + title="Daily energy transfer predictions with quantile loss", + xticks=[(i + 0.2) * 48 for i in range(7)], + xticklabels=["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"], + xlabel="Time of the week", + ylabel="Normalized energy transfer", +) +_ = ax.legend(loc="lower right") + +# %% +# We observe a tendence to over-estimate the energy transfer. This could be be +# quantitatively confirmed by computing empirical coverage numbers as done in +# the :ref:`calibration of confidence intervals section `. +# Keep in mind that those predicted percentiles are just estimations from a +# model. One can still improve the quality of such estimations by: +# +# - collecting more data-points; +# - better tuning of the model hyperparameters, see +# :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_quantile.py`; +# - engineering more predictive features from the same data, see +# :ref:`sphx_glr_auto_examples_applications_plot_cyclical_feature_engineering.py`. +# +# Monotonic constraints +# ===================== +# +# Given specific domain knowledge that requires the relationship between a +# feature and the target to be monotonically increasing or decreasing, one can +# enforce such behaviour in the predictions of a HGBT model using monotonic +# constraints. This makes the model more interpretable and can reduce its +# variance (and potentially mitigate overfitting) at the risk of increasing +# bias. Monotonic constraints can also be used to enforce specific regulatory +# requirements, ensure compliance and align with ethical considerations. +# +# In the present example, the policy of transferring energy from Victoria to New +# South Wales is meant to alleviate price fluctuations, meaning that the model +# predictions have to enforce such goal, i.e. transfer should increase with +# price and demand in New South Wales, but also decrease with price and demand +# in Victoria, in order to benefit both populations. +# +# If the training data has feature names, it’s possible to specify the monotonic +# constraints by passing a dictionary with the convention: +# +# - 1: monotonic increase +# - 0: no constraint +# - -1: monotonic decrease +# +# Alternatively, one can pass an array-like object encoding the above convention by +# position. + +from sklearn.inspection import PartialDependenceDisplay + +monotonic_cst = { + "date": 0, + "day": 0, + "period": 0, + "nswdemand": 1, + "nswprice": 1, + "vicdemand": -1, + "vicprice": -1, +} +hgbt_no_cst = HistGradientBoostingRegressor( + categorical_features=None, random_state=42 +).fit(X, y) +hgbt_cst = HistGradientBoostingRegressor( + monotonic_cst=monotonic_cst, categorical_features=None, random_state=42 +).fit(X, y) + +fig, ax = plt.subplots(nrows=2, figsize=(15, 10)) +disp = PartialDependenceDisplay.from_estimator( + hgbt_no_cst, + X, + features=["nswdemand", "nswprice"], + line_kw={"linewidth": 2, "label": "unconstrained", "color": "tab:blue"}, + ax=ax[0], +) +PartialDependenceDisplay.from_estimator( + hgbt_cst, + X, + features=["nswdemand", "nswprice"], + line_kw={"linewidth": 2, "label": "constrained", "color": "tab:orange"}, + ax=disp.axes_, +) +disp = PartialDependenceDisplay.from_estimator( + hgbt_no_cst, + X, + features=["vicdemand", "vicprice"], + line_kw={"linewidth": 2, "label": "unconstrained", "color": "tab:blue"}, + ax=ax[1], +) +PartialDependenceDisplay.from_estimator( + hgbt_cst, + X, + features=["vicdemand", "vicprice"], + line_kw={"linewidth": 2, "label": "constrained", "color": "tab:orange"}, + ax=disp.axes_, +) +_ = plt.legend() + +# %% +# Observe that `nswdemand` and `vicdemand` seem already monotonic without constraint. +# This is a good example to show that the model with monotonicity constraints is +# "overconstraining". +# +# Additionally, we can verify that the predictive quality of the model is not +# significantly degraded by introducing the monotonic constraints. For such +# purpose we use :class:`~sklearn.model_selection.TimeSeriesSplit` +# cross-validation to estimate the variance of the test score. By doing so we +# guarantee that the training data does not succeed the testing data, which is +# crucial when dealing with data that have a temporal relationship. + +from sklearn.metrics import make_scorer, root_mean_squared_error +from sklearn.model_selection import TimeSeriesSplit, cross_validate + +ts_cv = TimeSeriesSplit(n_splits=5, gap=48, test_size=336) # a week has 336 samples +scorer = make_scorer(root_mean_squared_error) + +cv_results = cross_validate(hgbt_no_cst, X, y, cv=ts_cv, scoring=scorer) +rmse = cv_results["test_score"] +print(f"RMSE without constraints = {rmse.mean():.3f} +/- {rmse.std():.3f}") + +cv_results = cross_validate(hgbt_cst, X, y, cv=ts_cv, scoring=scorer) +rmse = cv_results["test_score"] +print(f"RMSE with constraints = {rmse.mean():.3f} +/- {rmse.std():.3f}") + +# %% +# That being said, notice the comparison is between two different models that +# may be optimized by a different combination of hyperparameters. That is the +# reason why we do no use the `common_params` in this section as done before. diff --git a/examples/ensemble/plot_isolation_forest.py b/examples/ensemble/plot_isolation_forest.py index 5370f3af3ef97..2bd5bc9e99a0e 100644 --- a/examples/ensemble/plot_isolation_forest.py +++ b/examples/ensemble/plot_isolation_forest.py @@ -1,70 +1,126 @@ """ -========================================== +======================= IsolationForest example -========================================== +======================= An example using :class:`~sklearn.ensemble.IsolationForest` for anomaly detection. -The IsolationForest 'isolates' observations by randomly selecting a feature -and then randomly selecting a split value between the maximum and minimum -values of the selected feature. +The :ref:`isolation_forest` is an ensemble of "Isolation Trees" that "isolate" +observations by recursive random partitioning, which can be represented by a +tree structure. The number of splittings required to isolate a sample is lower +for outliers and higher for inliers. -Since recursive partitioning can be represented by a tree structure, the -number of splittings required to isolate a sample is equivalent to the path -length from the root node to the terminating node. +In the present example we demo two ways to visualize the decision boundary of an +Isolation Forest trained on a toy dataset. -This path length, averaged over a forest of such random trees, is a measure -of normality and our decision function. +""" -Random partitioning produces noticeable shorter paths for anomalies. -Hence, when a forest of random trees collectively produce shorter path lengths -for particular samples, they are highly likely to be anomalies. +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -""" -print(__doc__) +# %% +# Data generation +# --------------- +# +# We generate two clusters (each one containing `n_samples`) by randomly +# sampling the standard normal distribution as returned by +# :func:`numpy.random.randn`. One of them is spherical and the other one is +# slightly deformed. +# +# For consistency with the :class:`~sklearn.ensemble.IsolationForest` notation, +# the inliers (i.e. the gaussian clusters) are assigned a ground truth label `1` +# whereas the outliers (created with :func:`numpy.random.uniform`) are assigned +# the label `-1`. import numpy as np + +from sklearn.model_selection import train_test_split + +n_samples, n_outliers = 120, 40 +rng = np.random.RandomState(0) +covariance = np.array([[0.5, -0.1], [0.7, 0.4]]) +cluster_1 = 0.4 * rng.randn(n_samples, 2) @ covariance + np.array([2, 2]) # general +cluster_2 = 0.3 * rng.randn(n_samples, 2) + np.array([-2, -2]) # spherical +outliers = rng.uniform(low=-4, high=4, size=(n_outliers, 2)) + +X = np.concatenate([cluster_1, cluster_2, outliers]) +y = np.concatenate( + [np.ones((2 * n_samples), dtype=int), -np.ones((n_outliers), dtype=int)] +) + +X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=42) + +# %% +# We can visualize the resulting clusters: + import matplotlib.pyplot as plt -from sklearn.ensemble import IsolationForest -rng = np.random.RandomState(42) +scatter = plt.scatter(X[:, 0], X[:, 1], c=y, s=20, edgecolor="k") +handles, labels = scatter.legend_elements() +plt.axis("square") +plt.legend(handles=handles, labels=["outliers", "inliers"], title="true class") +plt.title("Gaussian inliers with \nuniformly distributed outliers") +plt.show() + +# %% +# Training of the model +# --------------------- -# Generate train data -X = 0.3 * rng.randn(100, 2) -X_train = np.r_[X + 2, X - 2] -# Generate some regular novel observations -X = 0.3 * rng.randn(20, 2) -X_test = np.r_[X + 2, X - 2] -# Generate some abnormal novel observations -X_outliers = rng.uniform(low=-4, high=4, size=(20, 2)) +from sklearn.ensemble import IsolationForest -# fit the model -clf = IsolationForest(max_samples=100, random_state=rng) +clf = IsolationForest(max_samples=100, random_state=0) clf.fit(X_train) -y_pred_train = clf.predict(X_train) -y_pred_test = clf.predict(X_test) -y_pred_outliers = clf.predict(X_outliers) - -# plot the line, the samples, and the nearest vectors to the plane -xx, yy = np.meshgrid(np.linspace(-5, 5, 50), np.linspace(-5, 5, 50)) -Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()]) -Z = Z.reshape(xx.shape) - -plt.title("IsolationForest") -plt.contourf(xx, yy, Z, cmap=plt.cm.Blues_r) - -b1 = plt.scatter(X_train[:, 0], X_train[:, 1], c='white', - s=20, edgecolor='k') -b2 = plt.scatter(X_test[:, 0], X_test[:, 1], c='green', - s=20, edgecolor='k') -c = plt.scatter(X_outliers[:, 0], X_outliers[:, 1], c='red', - s=20, edgecolor='k') -plt.axis('tight') -plt.xlim((-5, 5)) -plt.ylim((-5, 5)) -plt.legend([b1, b2, c], - ["training observations", - "new regular observations", "new abnormal observations"], - loc="upper left") + +# %% +# Plot discrete decision boundary +# ------------------------------- +# +# We use the class :class:`~sklearn.inspection.DecisionBoundaryDisplay` to +# visualize a discrete decision boundary. The background color represents +# whether a sample in that given area is predicted to be an outlier +# or not. The scatter plot displays the true labels. + +import matplotlib.pyplot as plt + +from sklearn.inspection import DecisionBoundaryDisplay + +disp = DecisionBoundaryDisplay.from_estimator( + clf, + X, + response_method="predict", + alpha=0.5, +) +disp.ax_.scatter(X[:, 0], X[:, 1], c=y, s=20, edgecolor="k") +disp.ax_.set_title("Binary decision boundary \nof IsolationForest") +plt.axis("square") +plt.legend(handles=handles, labels=["outliers", "inliers"], title="true class") +plt.show() + +# %% +# Plot path length decision boundary +# ---------------------------------- +# +# By setting the `response_method="decision_function"`, the background of the +# :class:`~sklearn.inspection.DecisionBoundaryDisplay` represents the measure of +# normality of an observation. Such score is given by the path length averaged +# over a forest of random trees, which itself is given by the depth of the leaf +# (or equivalently the number of splits) required to isolate a given sample. +# +# When a forest of random trees collectively produce short path lengths for +# isolating some particular samples, they are highly likely to be anomalies and +# the measure of normality is close to `0`. Similarly, large paths correspond to +# values close to `1` and are more likely to be inliers. + +disp = DecisionBoundaryDisplay.from_estimator( + clf, + X, + response_method="decision_function", + alpha=0.5, +) +disp.ax_.scatter(X[:, 0], X[:, 1], c=y, s=20, edgecolor="k") +disp.ax_.set_title("Path length decision boundary \nof IsolationForest") +plt.axis("square") +plt.legend(handles=handles, labels=["outliers", "inliers"], title="true class") +plt.colorbar(disp.ax_.collections[1]) plt.show() diff --git a/examples/ensemble/plot_monotonic_constraints.py b/examples/ensemble/plot_monotonic_constraints.py index 8b3f69f1d542e..40fb61eae19e9 100644 --- a/examples/ensemble/plot_monotonic_constraints.py +++ b/examples/ensemble/plot_monotonic_constraints.py @@ -11,59 +11,67 @@ non-random variations), and in general negatively correlated with the second feature. -By imposing a positive (increasing) or negative (decreasing) constraint on -the features during the learning process, the estimator is able to properly -follow the general trend instead of being subject to the variations. +By imposing a monotonic increase or a monotonic decrease constraint, respectively, +on the features during the learning process, the estimator is able to properly follow +the general trend instead of being subject to the variations. This example was inspired by the `XGBoost documentation `_. + """ -from sklearn.experimental import enable_hist_gradient_boosting # noqa -from sklearn.ensemble import HistGradientBoostingRegressor -from sklearn.inspection import plot_partial_dependence -import numpy as np -import matplotlib.pyplot as plt +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -print(__doc__) +# %% +import matplotlib.pyplot as plt +import numpy as np + +from sklearn.ensemble import HistGradientBoostingRegressor +from sklearn.inspection import PartialDependenceDisplay rng = np.random.RandomState(0) -n_samples = 5000 -f_0 = rng.rand(n_samples) # positive correlation with y -f_1 = rng.rand(n_samples) # negative correlation with y +n_samples = 1000 +f_0 = rng.rand(n_samples) +f_1 = rng.rand(n_samples) X = np.c_[f_0, f_1] noise = rng.normal(loc=0.0, scale=0.01, size=n_samples) -y = (5 * f_0 + np.sin(10 * np.pi * f_0) - - 5 * f_1 - np.cos(10 * np.pi * f_1) + - noise) -fig, ax = plt.subplots() +# y is positively correlated with f_0, and negatively correlated with f_1 +y = 5 * f_0 + np.sin(10 * np.pi * f_0) - 5 * f_1 - np.cos(10 * np.pi * f_1) + noise + + +# %% +# Fit a first model on this dataset without any constraints. +gbdt_no_cst = HistGradientBoostingRegressor() +gbdt_no_cst.fit(X, y) +# %% +# Fit a second model on this dataset with monotonic increase (1) +# and a monotonic decrease (-1) constraints, respectively. +gbdt_with_monotonic_cst = HistGradientBoostingRegressor(monotonic_cst=[1, -1]) +gbdt_with_monotonic_cst.fit(X, y) -# Without any constraint -gbdt = HistGradientBoostingRegressor() -gbdt.fit(X, y) -disp = plot_partial_dependence( - gbdt, + +# %% +# Let's display the partial dependence of the predictions on the two features. +fig, ax = plt.subplots() +disp = PartialDependenceDisplay.from_estimator( + gbdt_no_cst, X, features=[0, 1], + feature_names=( + "First feature", + "Second feature", + ), line_kw={"linewidth": 4, "label": "unconstrained", "color": "tab:blue"}, ax=ax, ) - -# With positive and negative constraints -gbdt = HistGradientBoostingRegressor(monotonic_cst=[1, -1]) -gbdt.fit(X, y) - -plot_partial_dependence( - gbdt, +PartialDependenceDisplay.from_estimator( + gbdt_with_monotonic_cst, X, features=[0, 1], - feature_names=( - "First feature\nPositive constraint", - "Second feature\nNegtive constraint", - ), line_kw={"linewidth": 4, "label": "constrained", "color": "tab:orange"}, ax=disp.axes_, ) @@ -75,6 +83,30 @@ disp.axes_[0, f_idx].set_ylim(-6, 6) plt.legend() -fig.suptitle("Monotonic constraints illustration") - +fig.suptitle("Monotonic constraints effect on partial dependences") plt.show() + +# %% +# We can see that the predictions of the unconstrained model capture the +# oscillations of the data while the constrained model follows the general +# trend and ignores the local variations. + +# %% +# .. _monotonic_cst_features_names: +# +# Using feature names to specify monotonic constraints +# ---------------------------------------------------- +# +# Note that if the training data has feature names, it's possible to specify the +# monotonic constraints by passing a dictionary: +import pandas as pd + +X_df = pd.DataFrame(X, columns=["f_0", "f_1"]) + +gbdt_with_monotonic_cst_df = HistGradientBoostingRegressor( + monotonic_cst={"f_0": 1, "f_1": -1} +).fit(X_df, y) + +np.allclose( + gbdt_with_monotonic_cst_df.predict(X_df), gbdt_with_monotonic_cst.predict(X) +) diff --git a/examples/ensemble/plot_random_forest_embedding.py b/examples/ensemble/plot_random_forest_embedding.py index 4d0ccd4502c31..d3d595df232a9 100644 --- a/examples/ensemble/plot_random_forest_embedding.py +++ b/examples/ensemble/plot_random_forest_embedding.py @@ -23,13 +23,18 @@ decision boundary obtained by BernoulliNB in the transformed space with an ExtraTreesClassifier forests learned on the original data. + """ -import numpy as np + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt +import numpy as np from sklearn.datasets import make_circles -from sklearn.ensemble import RandomTreesEmbedding, ExtraTreesClassifier from sklearn.decomposition import TruncatedSVD +from sklearn.ensemble import ExtraTreesClassifier, RandomTreesEmbedding from sklearn.naive_bayes import BernoulliNB # make a synthetic dataset @@ -57,23 +62,24 @@ fig = plt.figure(figsize=(9, 8)) ax = plt.subplot(221) -ax.scatter(X[:, 0], X[:, 1], c=y, s=50, edgecolor='k') +ax.scatter(X[:, 0], X[:, 1], c=y, s=50, edgecolor="k") ax.set_title("Original Data (2d)") ax.set_xticks(()) ax.set_yticks(()) ax = plt.subplot(222) -ax.scatter(X_reduced[:, 0], X_reduced[:, 1], c=y, s=50, edgecolor='k') -ax.set_title("Truncated SVD reduction (2d) of transformed data (%dd)" % - X_transformed.shape[1]) +ax.scatter(X_reduced[:, 0], X_reduced[:, 1], c=y, s=50, edgecolor="k") +ax.set_title( + "Truncated SVD reduction (2d) of transformed data (%dd)" % X_transformed.shape[1] +) ax.set_xticks(()) ax.set_yticks(()) # Plot the decision in original space. For that, we will assign a color # to each point in the mesh [x_min, x_max]x[y_min, y_max]. -h = .01 -x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5 -y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5 +h = 0.01 +x_min, x_max = X[:, 0].min() - 0.5, X[:, 0].max() + 0.5 +y_min, y_max = X[:, 1].min() - 0.5, X[:, 1].max() + 0.5 xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h)) # transform grid using RandomTreesEmbedding @@ -83,7 +89,7 @@ ax = plt.subplot(223) ax.set_title("Naive Bayes on Transformed data") ax.pcolormesh(xx, yy, y_grid_pred.reshape(xx.shape)) -ax.scatter(X[:, 0], X[:, 1], c=y, s=50, edgecolor='k') +ax.scatter(X[:, 0], X[:, 1], c=y, s=50, edgecolor="k") ax.set_ylim(-1.4, 1.4) ax.set_xlim(-1.4, 1.4) ax.set_xticks(()) @@ -95,7 +101,7 @@ ax = plt.subplot(224) ax.set_title("ExtraTrees predictions") ax.pcolormesh(xx, yy, y_grid_pred.reshape(xx.shape)) -ax.scatter(X[:, 0], X[:, 1], c=y, s=50, edgecolor='k') +ax.scatter(X[:, 0], X[:, 1], c=y, s=50, edgecolor="k") ax.set_ylim(-1.4, 1.4) ax.set_xlim(-1.4, 1.4) ax.set_xticks(()) diff --git a/examples/ensemble/plot_random_forest_regression_multioutput.py b/examples/ensemble/plot_random_forest_regression_multioutput.py index 8b7803361a60a..52df62dc19f41 100644 --- a/examples/ensemble/plot_random_forest_regression_multioutput.py +++ b/examples/ensemble/plot_random_forest_regression_multioutput.py @@ -20,36 +20,34 @@ x and y coordinate as output. """ -print(__doc__) -# Author: Tim Head -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt +import numpy as np + from sklearn.ensemble import RandomForestRegressor from sklearn.model_selection import train_test_split from sklearn.multioutput import MultiOutputRegressor - # Create a random dataset rng = np.random.RandomState(1) X = np.sort(200 * rng.rand(600, 1) - 100, axis=0) y = np.array([np.pi * np.sin(X).ravel(), np.pi * np.cos(X).ravel()]).T -y += (0.5 - rng.rand(*y.shape)) +y += 0.5 - rng.rand(*y.shape) X_train, X_test, y_train, y_test = train_test_split( - X, y, train_size=400, test_size=200, random_state=4) + X, y, train_size=400, test_size=200, random_state=4 +) max_depth = 30 -regr_multirf = MultiOutputRegressor(RandomForestRegressor(n_estimators=100, - max_depth=max_depth, - random_state=0)) +regr_multirf = MultiOutputRegressor( + RandomForestRegressor(n_estimators=100, max_depth=max_depth, random_state=0) +) regr_multirf.fit(X_train, y_train) -regr_rf = RandomForestRegressor(n_estimators=100, max_depth=max_depth, - random_state=2) +regr_rf = RandomForestRegressor(n_estimators=100, max_depth=max_depth, random_state=2) regr_rf.fit(X_train, y_train) # Predict on new data @@ -60,14 +58,35 @@ plt.figure() s = 50 a = 0.4 -plt.scatter(y_test[:, 0], y_test[:, 1], edgecolor='k', - c="navy", s=s, marker="s", alpha=a, label="Data") -plt.scatter(y_multirf[:, 0], y_multirf[:, 1], edgecolor='k', - c="cornflowerblue", s=s, alpha=a, - label="Multi RF score=%.2f" % regr_multirf.score(X_test, y_test)) -plt.scatter(y_rf[:, 0], y_rf[:, 1], edgecolor='k', - c="c", s=s, marker="^", alpha=a, - label="RF score=%.2f" % regr_rf.score(X_test, y_test)) +plt.scatter( + y_test[:, 0], + y_test[:, 1], + edgecolor="k", + c="navy", + s=s, + marker="s", + alpha=a, + label="Data", +) +plt.scatter( + y_multirf[:, 0], + y_multirf[:, 1], + edgecolor="k", + c="cornflowerblue", + s=s, + alpha=a, + label="Multi RF score=%.2f" % regr_multirf.score(X_test, y_test), +) +plt.scatter( + y_rf[:, 0], + y_rf[:, 1], + edgecolor="k", + c="c", + s=s, + marker="^", + alpha=a, + label="RF score=%.2f" % regr_rf.score(X_test, y_test), +) plt.xlim([-6, 6]) plt.ylim([-6, 6]) plt.xlabel("target 1") diff --git a/examples/ensemble/plot_stack_predictors.py b/examples/ensemble/plot_stack_predictors.py index c07068b060c57..bd37e8fb4fdfa 100644 --- a/examples/ensemble/plot_stack_predictors.py +++ b/examples/ensemble/plot_stack_predictors.py @@ -16,20 +16,14 @@ """ -# Authors: Guillaume Lemaitre -# Maria Telenczuk -# License: BSD 3 clause - -print(__doc__) - -from sklearn import set_config -set_config(display='diagram') +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause # %% # Download the dataset -############################################################################## +# #################### # -# We will use `Ames Housing`_ dataset which was first compiled by Dean De Cock +# We will use the `Ames Housing`_ dataset which was first compiled by Dean De Cock # and became better known after it was used in Kaggle challenge. It is a set # of 1460 residential homes in Ames, Iowa, each described by 80 features. We # will use it to predict the final logarithmic price of the houses. In this @@ -54,26 +48,42 @@ def load_ames_housing(): X = df.data y = df.target - features = ['YrSold', 'HeatingQC', 'Street', 'YearRemodAdd', 'Heating', - 'MasVnrType', 'BsmtUnfSF', 'Foundation', 'MasVnrArea', - 'MSSubClass', 'ExterQual', 'Condition2', 'GarageCars', - 'GarageType', 'OverallQual', 'TotalBsmtSF', 'BsmtFinSF1', - 'HouseStyle', 'MiscFeature', 'MoSold'] - - X = X[features] + features = [ + "YrSold", + "HeatingQC", + "Street", + "YearRemodAdd", + "Heating", + "MasVnrType", + "BsmtUnfSF", + "Foundation", + "MasVnrArea", + "MSSubClass", + "ExterQual", + "Condition2", + "GarageCars", + "GarageType", + "OverallQual", + "TotalBsmtSF", + "BsmtFinSF1", + "HouseStyle", + "MiscFeature", + "MoSold", + ] + + X = X.loc[:, features] X, y = shuffle(X, y, random_state=0) - X = X[:600] - y = y[:600] + X = X.iloc[:600] + y = y.iloc[:600] return X, np.log(y) X, y = load_ames_housing() - # %% # Make pipeline to preprocess the data -############################################################################## +# #################################### # # Before we can use Ames dataset we still need to do some preprocessing. # First, we will select the categorical and numerical columns of the dataset to @@ -105,31 +115,36 @@ def load_ames_housing(): from sklearn.preprocessing import OrdinalEncoder cat_tree_processor = OrdinalEncoder( - handle_unknown="use_encoded_value", unknown_value=-1) + handle_unknown="use_encoded_value", + unknown_value=-1, + encoded_missing_value=-2, +) num_tree_processor = SimpleImputer(strategy="mean", add_indicator=True) tree_preprocessor = make_column_transformer( - (num_tree_processor, num_selector), (cat_tree_processor, cat_selector)) + (num_tree_processor, num_selector), (cat_tree_processor, cat_selector) +) tree_preprocessor # %% # Then, we will now define the preprocessor used when the ending regressor # is a linear model. -from sklearn.preprocessing import OneHotEncoder -from sklearn.preprocessing import StandardScaler +from sklearn.preprocessing import OneHotEncoder, StandardScaler cat_linear_processor = OneHotEncoder(handle_unknown="ignore") num_linear_processor = make_pipeline( - StandardScaler(), SimpleImputer(strategy="mean", add_indicator=True)) + StandardScaler(), SimpleImputer(strategy="mean", add_indicator=True) +) linear_preprocessor = make_column_transformer( - (num_linear_processor, num_selector), (cat_linear_processor, cat_selector)) + (num_linear_processor, num_selector), (cat_linear_processor, cat_selector) +) linear_preprocessor # %% # Stack of predictors on a single data set -############################################################################## +# ######################################## # # It is sometimes tedious to find the model which will best perform on a given # dataset. Stacking provide an alternative by combining the outputs of several @@ -155,95 +170,84 @@ def load_ames_housing(): # %% from sklearn.ensemble import RandomForestRegressor -rf_pipeline = make_pipeline( - tree_preprocessor, RandomForestRegressor(random_state=42)) +rf_pipeline = make_pipeline(tree_preprocessor, RandomForestRegressor(random_state=42)) rf_pipeline # %% -from sklearn.experimental import enable_hist_gradient_boosting # noqa from sklearn.ensemble import HistGradientBoostingRegressor gbdt_pipeline = make_pipeline( - tree_preprocessor, HistGradientBoostingRegressor(random_state=0)) + tree_preprocessor, HistGradientBoostingRegressor(random_state=0) +) gbdt_pipeline # %% from sklearn.ensemble import StackingRegressor from sklearn.linear_model import RidgeCV -estimators = [('Random Forest', rf_pipeline), - ('Lasso', lasso_pipeline), - ('Gradient Boosting', gbdt_pipeline)] +estimators = [ + ("Random Forest", rf_pipeline), + ("Lasso", lasso_pipeline), + ("Gradient Boosting", gbdt_pipeline), +] -stacking_regressor = StackingRegressor( - estimators=estimators, final_estimator=RidgeCV()) +stacking_regressor = StackingRegressor(estimators=estimators, final_estimator=RidgeCV()) stacking_regressor # %% # Measure and plot the results -############################################################################## +# ############################ # # Now we can use Ames Housing dataset to make the predictions. We check the # performance of each individual predictor as well as of the stack of the # regressors. -# -# The function ``plot_regression_results`` is used to plot the predicted and -# true targets. import time + import matplotlib.pyplot as plt -from sklearn.model_selection import cross_validate, cross_val_predict - - -def plot_regression_results(ax, y_true, y_pred, title, scores, elapsed_time): - """Scatter plot of the predicted vs true targets.""" - ax.plot([y_true.min(), y_true.max()], - [y_true.min(), y_true.max()], - '--r', linewidth=2) - ax.scatter(y_true, y_pred, alpha=0.2) - - ax.spines['top'].set_visible(False) - ax.spines['right'].set_visible(False) - ax.get_xaxis().tick_bottom() - ax.get_yaxis().tick_left() - ax.spines['left'].set_position(('outward', 10)) - ax.spines['bottom'].set_position(('outward', 10)) - ax.set_xlim([y_true.min(), y_true.max()]) - ax.set_ylim([y_true.min(), y_true.max()]) - ax.set_xlabel('Measured') - ax.set_ylabel('Predicted') - extra = plt.Rectangle((0, 0), 0, 0, fc="w", fill=False, - edgecolor='none', linewidth=0) - ax.legend([extra], [scores], loc='upper left') - title = title + '\n Evaluation in {:.2f} seconds'.format(elapsed_time) - ax.set_title(title) +from sklearn.metrics import PredictionErrorDisplay +from sklearn.model_selection import cross_val_predict, cross_validate fig, axs = plt.subplots(2, 2, figsize=(9, 7)) axs = np.ravel(axs) -for ax, (name, est) in zip(axs, estimators + [('Stacking Regressor', - stacking_regressor)]): +for ax, (name, est) in zip( + axs, estimators + [("Stacking Regressor", stacking_regressor)] +): + scorers = {"R2": "r2", "MAE": "neg_mean_absolute_error"} + start_time = time.time() - score = cross_validate(est, X, y, - scoring=['r2', 'neg_mean_absolute_error'], - n_jobs=-1, verbose=0) + scores = cross_validate( + est, X, y, scoring=list(scorers.values()), n_jobs=-1, verbose=0 + ) elapsed_time = time.time() - start_time y_pred = cross_val_predict(est, X, y, n_jobs=-1, verbose=0) - - plot_regression_results( - ax, y, y_pred, - name, - (r'$R^2={:.2f} \pm {:.2f}$' + '\n' + r'$MAE={:.2f} \pm {:.2f}$') - .format(np.mean(score['test_r2']), - np.std(score['test_r2']), - -np.mean(score['test_neg_mean_absolute_error']), - np.std(score['test_neg_mean_absolute_error'])), - elapsed_time) - -plt.suptitle('Single predictors versus stacked predictors') + scores = { + key: ( + f"{np.abs(np.mean(scores[f'test_{value}'])):.2f} +- " + f"{np.std(scores[f'test_{value}']):.2f}" + ) + for key, value in scorers.items() + } + + display = PredictionErrorDisplay.from_predictions( + y_true=y, + y_pred=y_pred, + kind="actual_vs_predicted", + ax=ax, + scatter_kwargs={"alpha": 0.2, "color": "tab:blue"}, + line_kwargs={"color": "tab:red"}, + ) + ax.set_title(f"{name}\nEvaluation in {elapsed_time:.2f} seconds") + + for name, score in scores.items(): + ax.plot([], [], " ", label=f"{name}: {score}") + ax.legend(loc="upper left") + +plt.suptitle("Single predictors versus stacked predictors") plt.tight_layout() plt.subplots_adjust(top=0.9) plt.show() diff --git a/examples/ensemble/plot_voting_decision_regions.py b/examples/ensemble/plot_voting_decision_regions.py index fdfda74947f5f..57f3f4b22b947 100644 --- a/examples/ensemble/plot_voting_decision_regions.py +++ b/examples/ensemble/plot_voting_decision_regions.py @@ -1,77 +1,218 @@ """ -================================================== -Plot the decision boundaries of a VotingClassifier -================================================== +=============================================================== +Visualizing the probabilistic predictions of a VotingClassifier +=============================================================== .. currentmodule:: sklearn -Plot the decision boundaries of a :class:`~ensemble.VotingClassifier` for two -features of the Iris dataset. +Plot the predicted class probabilities in a toy dataset predicted by three +different classifiers and averaged by the :class:`~ensemble.VotingClassifier`. -Plot the class probabilities of the first sample in a toy dataset predicted by -three different classifiers and averaged by the -:class:`~ensemble.VotingClassifier`. +First, three linear classifiers are initialized. Two are spline models with +interaction terms, one using constant extrapolation and the other using periodic +extrapolation. The third classifier is a :class:`~kernel_approximation.Nystroem` +with the default "rbf" kernel. -First, three exemplary classifiers are initialized -(:class:`~tree.DecisionTreeClassifier`, -:class:`~neighbors.KNeighborsClassifier`, and :class:`~svm.SVC`) and used to -initialize a soft-voting :class:`~ensemble.VotingClassifier` with weights `[2, -1, 2]`, which means that the predicted probabilities of the -:class:`~tree.DecisionTreeClassifier` and :class:`~svm.SVC` each count 2 times -as much as the weights of the :class:`~neighbors.KNeighborsClassifier` -classifier when the averaged probability is calculated. +In the first part of this example, these three classifiers are used to +demonstrate soft-voting using :class:`~ensemble.VotingClassifier` with weighted +average. We set `weights=[2, 1, 3]`, meaning the constant extrapolation spline +model's predictions are weighted twice as much as the periodic spline model's, +and the Nystroem model's predictions are weighted three times as much as the +periodic spline. + +The second part demonstrates how soft predictions can be converted into hard +predictions. """ -print(__doc__) -from itertools import product +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np -import matplotlib.pyplot as plt +# %% +# We first generate a noisy XOR dataset, which is a binary classification task. -from sklearn import datasets -from sklearn.tree import DecisionTreeClassifier -from sklearn.neighbors import KNeighborsClassifier -from sklearn.svm import SVC -from sklearn.ensemble import VotingClassifier +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +from matplotlib.colors import ListedColormap + +n_samples = 500 +rng = np.random.default_rng(0) +feature_names = ["Feature #0", "Feature #1"] +common_scatter_plot_params = dict( + cmap=ListedColormap(["tab:red", "tab:blue"]), + edgecolor="white", + linewidth=1, +) + +xor = pd.DataFrame( + np.random.RandomState(0).uniform(low=-1, high=1, size=(n_samples, 2)), + columns=feature_names, +) +noise = rng.normal(loc=0, scale=0.1, size=(n_samples, 2)) +target_xor = np.logical_xor( + xor["Feature #0"] + noise[:, 0] > 0, xor["Feature #1"] + noise[:, 1] > 0 +) + +X = xor[feature_names] +y = target_xor.astype(np.int32) + +fig, ax = plt.subplots() +ax.scatter(X["Feature #0"], X["Feature #1"], c=y, **common_scatter_plot_params) +ax.set_title("The XOR dataset") +plt.show() -# Loading some example data -iris = datasets.load_iris() -X = iris.data[:, [0, 2]] -y = iris.target +# %% +# Due to the inherent non-linear separability of the XOR dataset, tree-based +# models would often be preferred. However, appropriate feature engineering +# combined with a linear model can yield effective results, with the added +# benefit of producing better-calibrated probabilities for samples located in +# the transition regions affected by noise. +# +# We define and fit the models on the whole dataset. -# Training classifiers -clf1 = DecisionTreeClassifier(max_depth=4) -clf2 = KNeighborsClassifier(n_neighbors=7) -clf3 = SVC(gamma=.1, kernel='rbf', probability=True) -eclf = VotingClassifier(estimators=[('dt', clf1), ('knn', clf2), - ('svc', clf3)], - voting='soft', weights=[2, 1, 2]) +from sklearn.ensemble import VotingClassifier +from sklearn.kernel_approximation import Nystroem +from sklearn.linear_model import LogisticRegression +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import PolynomialFeatures, SplineTransformer, StandardScaler + +clf1 = make_pipeline( + SplineTransformer(degree=2, n_knots=2), + PolynomialFeatures(interaction_only=True), + LogisticRegression(C=10), +) +clf2 = make_pipeline( + SplineTransformer( + degree=2, + n_knots=4, + extrapolation="periodic", + include_bias=True, + ), + PolynomialFeatures(interaction_only=True), + LogisticRegression(C=10), +) +clf3 = make_pipeline( + StandardScaler(), + Nystroem(gamma=2, random_state=0), + LogisticRegression(C=10), +) +weights = [2, 1, 3] +eclf = VotingClassifier( + estimators=[ + ("constant splines model", clf1), + ("periodic splines model", clf2), + ("nystroem model", clf3), + ], + voting="soft", + weights=weights, +) clf1.fit(X, y) clf2.fit(X, y) clf3.fit(X, y) eclf.fit(X, y) -# Plotting decision regions -x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1 -y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1 -xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.1), - np.arange(y_min, y_max, 0.1)) - -f, axarr = plt.subplots(2, 2, sharex='col', sharey='row', figsize=(10, 8)) +# %% +# Finally we use :class:`~inspection.DecisionBoundaryDisplay` to plot the +# predicted probabilities. By using a diverging colormap (such as `"RdBu"`), we +# can ensure that darker colors correspond to `predict_proba` close to either 0 +# or 1, and white corresponds to `predict_proba` of 0.5. -for idx, clf, tt in zip(product([0, 1], [0, 1]), - [clf1, clf2, clf3, eclf], - ['Decision Tree (depth=4)', 'KNN (k=7)', - 'Kernel SVM', 'Soft Voting']): - - Z = clf.predict(np.c_[xx.ravel(), yy.ravel()]) - Z = Z.reshape(xx.shape) +from itertools import product - axarr[idx[0], idx[1]].contourf(xx, yy, Z, alpha=0.4) - axarr[idx[0], idx[1]].scatter(X[:, 0], X[:, 1], c=y, - s=20, edgecolor='k') - axarr[idx[0], idx[1]].set_title(tt) +from sklearn.inspection import DecisionBoundaryDisplay + +fig, axarr = plt.subplots(2, 2, sharex="col", sharey="row", figsize=(10, 8)) +for idx, clf, title in zip( + product([0, 1], [0, 1]), + [clf1, clf2, clf3, eclf], + [ + "Splines with\nconstant extrapolation", + "Splines with\nperiodic extrapolation", + "RBF Nystroem", + "Soft Voting", + ], +): + disp = DecisionBoundaryDisplay.from_estimator( + clf, + X, + response_method="predict_proba", + plot_method="pcolormesh", + cmap="RdBu", + alpha=0.8, + ax=axarr[idx[0], idx[1]], + ) + axarr[idx[0], idx[1]].scatter( + X["Feature #0"], + X["Feature #1"], + c=y, + **common_scatter_plot_params, + ) + axarr[idx[0], idx[1]].set_title(title) + fig.colorbar(disp.surface_, ax=axarr[idx[0], idx[1]], label="Probability estimate") plt.show() + +# %% +# As a sanity check, we can verify for a given sample that the probability +# predicted by the :class:`~ensemble.VotingClassifier` is indeed the weighted +# average of the individual classifiers' soft-predictions. +# +# In the case of binary classification such as in the present example, the +# :term:`predict_proba` arrays contain the probability of belonging to class 0 +# (here in red) as the first entry, and the probability of belonging to class 1 +# (here in blue) as the second entry. + +test_sample = pd.DataFrame({"Feature #0": [-0.5], "Feature #1": [1.5]}) +predict_probas = [est.predict_proba(test_sample).ravel() for est in eclf.estimators_] +for (est_name, _), est_probas in zip(eclf.estimators, predict_probas): + print(f"{est_name}'s predicted probabilities: {est_probas}") + +# %% +print( + "Weighted average of soft-predictions: " + f"{np.dot(weights, predict_probas) / np.sum(weights)}" +) + +# %% +# We can see that manual calculation of predicted probabilities above is +# equivalent to that produced by the `VotingClassifier`: + +print( + "Predicted probability of VotingClassifier: " + f"{eclf.predict_proba(test_sample).ravel()}" +) + +# %% +# To convert soft predictions into hard predictions when weights are provided, +# the weighted average predicted probabilities are computed for each class. +# Then, the final class label is then derived from the class label with the +# highest average probability, which corresponds to the default threshold at +# `predict_proba=0.5` in the case of binary classification. + +print( + "Class with the highest weighted average of soft-predictions: " + f"{np.argmax(np.dot(weights, predict_probas) / np.sum(weights))}" +) + +# %% +# This is equivalent to the output of `VotingClassifier`'s `predict` method: + +print(f"Predicted class of VotingClassifier: {eclf.predict(test_sample).ravel()}") + +# %% +# Soft votes can be thresholded as for any other probabilistic classifier. This +# allows you to set a threshold probability at which the positive class will be +# predicted, instead of simply selecting the class with the highest predicted +# probability. + +from sklearn.model_selection import FixedThresholdClassifier + +eclf_other_threshold = FixedThresholdClassifier( + eclf, threshold=0.7, response_method="predict_proba" +).fit(X, y) +print( + "Predicted class of thresholded VotingClassifier: " + f"{eclf_other_threshold.predict(test_sample)}" +) diff --git a/examples/ensemble/plot_voting_probas.py b/examples/ensemble/plot_voting_probas.py deleted file mode 100644 index 1e65e7d725964..0000000000000 --- a/examples/ensemble/plot_voting_probas.py +++ /dev/null @@ -1,86 +0,0 @@ -""" -=========================================================== -Plot class probabilities calculated by the VotingClassifier -=========================================================== - -.. currentmodule:: sklearn - -Plot the class probabilities of the first sample in a toy dataset predicted by -three different classifiers and averaged by the -:class:`~ensemble.VotingClassifier`. - -First, three examplary classifiers are initialized -(:class:`~linear_model.LogisticRegression`, :class:`~naive_bayes.GaussianNB`, -and :class:`~ensemble.RandomForestClassifier`) and used to initialize a -soft-voting :class:`~ensemble.VotingClassifier` with weights `[1, 1, 5]`, which -means that the predicted probabilities of the -:class:`~ensemble.RandomForestClassifier` count 5 times as much as the weights -of the other classifiers when the averaged probability is calculated. - -To visualize the probability weighting, we fit each classifier on the training -set and plot the predicted class probabilities for the first sample in this -example dataset. - -""" -print(__doc__) - -import numpy as np -import matplotlib.pyplot as plt - -from sklearn.linear_model import LogisticRegression -from sklearn.naive_bayes import GaussianNB -from sklearn.ensemble import RandomForestClassifier -from sklearn.ensemble import VotingClassifier - -clf1 = LogisticRegression(max_iter=1000, random_state=123) -clf2 = RandomForestClassifier(n_estimators=100, random_state=123) -clf3 = GaussianNB() -X = np.array([[-1.0, -1.0], [-1.2, -1.4], [-3.4, -2.2], [1.1, 1.2]]) -y = np.array([1, 1, 2, 2]) - -eclf = VotingClassifier(estimators=[('lr', clf1), ('rf', clf2), ('gnb', clf3)], - voting='soft', - weights=[1, 1, 5]) - -# predict class probabilities for all classifiers -probas = [c.fit(X, y).predict_proba(X) for c in (clf1, clf2, clf3, eclf)] - -# get class probabilities for the first sample in the dataset -class1_1 = [pr[0, 0] for pr in probas] -class2_1 = [pr[0, 1] for pr in probas] - - -# plotting - -N = 4 # number of groups -ind = np.arange(N) # group positions -width = 0.35 # bar width - -fig, ax = plt.subplots() - -# bars for classifier 1-3 -p1 = ax.bar(ind, np.hstack(([class1_1[:-1], [0]])), width, - color='green', edgecolor='k') -p2 = ax.bar(ind + width, np.hstack(([class2_1[:-1], [0]])), width, - color='lightgreen', edgecolor='k') - -# bars for VotingClassifier -p3 = ax.bar(ind, [0, 0, 0, class1_1[-1]], width, - color='blue', edgecolor='k') -p4 = ax.bar(ind + width, [0, 0, 0, class2_1[-1]], width, - color='steelblue', edgecolor='k') - -# plot annotations -plt.axvline(2.8, color='k', linestyle='dashed') -ax.set_xticks(ind + width) -ax.set_xticklabels(['LogisticRegression\nweight 1', - 'GaussianNB\nweight 1', - 'RandomForestClassifier\nweight 5', - 'VotingClassifier\n(average probabilities)'], - rotation=40, - ha='right') -plt.ylim([0, 1]) -plt.title('Class probabilities for sample 1 by different classifiers') -plt.legend([p1[0], p2[0]], ['class 1', 'class 2'], loc='upper left') -plt.tight_layout() -plt.show() diff --git a/examples/ensemble/plot_voting_regressor.py b/examples/ensemble/plot_voting_regressor.py index 106efbac2461d..6ccc4e81b700a 100644 --- a/examples/ensemble/plot_voting_regressor.py +++ b/examples/ensemble/plot_voting_regressor.py @@ -22,15 +22,19 @@ measure of disease progression one year after baseline. """ -print(__doc__) + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import matplotlib.pyplot as plt from sklearn.datasets import load_diabetes -from sklearn.ensemble import GradientBoostingRegressor -from sklearn.ensemble import RandomForestRegressor +from sklearn.ensemble import ( + GradientBoostingRegressor, + RandomForestRegressor, + VotingRegressor, +) from sklearn.linear_model import LinearRegression -from sklearn.ensemble import VotingRegressor # %% # Training classifiers @@ -51,7 +55,7 @@ reg2.fit(X, y) reg3.fit(X, y) -ereg = VotingRegressor([('gb', reg1), ('rf', reg2), ('lr', reg3)]) +ereg = VotingRegressor([("gb", reg1), ("rf", reg2), ("lr", reg3)]) ereg.fit(X, y) # %% @@ -75,16 +79,15 @@ # prediction made by :class:`~ensemble.VotingRegressor`. plt.figure() -plt.plot(pred1, 'gd', label='GradientBoostingRegressor') -plt.plot(pred2, 'b^', label='RandomForestRegressor') -plt.plot(pred3, 'ys', label='LinearRegression') -plt.plot(pred4, 'r*', ms=10, label='VotingRegressor') - -plt.tick_params(axis='x', which='both', bottom=False, top=False, - labelbottom=False) -plt.ylabel('predicted') -plt.xlabel('training samples') +plt.plot(pred1, "gd", label="GradientBoostingRegressor") +plt.plot(pred2, "b^", label="RandomForestRegressor") +plt.plot(pred3, "ys", label="LinearRegression") +plt.plot(pred4, "r*", ms=10, label="VotingRegressor") + +plt.tick_params(axis="x", which="both", bottom=False, top=False, labelbottom=False) +plt.ylabel("predicted") +plt.xlabel("training samples") plt.legend(loc="best") -plt.title('Regressor predictions and their average') +plt.title("Regressor predictions and their average") plt.show() diff --git a/examples/exercises/README.txt b/examples/exercises/README.txt deleted file mode 100644 index 5f211eadfef5a..0000000000000 --- a/examples/exercises/README.txt +++ /dev/null @@ -1,4 +0,0 @@ -Tutorial exercises ------------------- - -Exercises for the tutorials diff --git a/examples/exercises/plot_cv_diabetes.py b/examples/exercises/plot_cv_diabetes.py deleted file mode 100644 index d71abd8fe6455..0000000000000 --- a/examples/exercises/plot_cv_diabetes.py +++ /dev/null @@ -1,79 +0,0 @@ -""" -=============================================== -Cross-validation on diabetes Dataset Exercise -=============================================== - -A tutorial exercise which uses cross-validation with linear models. - -This exercise is used in the :ref:`cv_estimators_tut` part of the -:ref:`model_selection_tut` section of the :ref:`stat_learn_tut_index`. -""" - -print(__doc__) - -import numpy as np -import matplotlib.pyplot as plt - -from sklearn import datasets -from sklearn.linear_model import LassoCV -from sklearn.linear_model import Lasso -from sklearn.model_selection import KFold -from sklearn.model_selection import GridSearchCV - -X, y = datasets.load_diabetes(return_X_y=True) -X = X[:150] -y = y[:150] - -lasso = Lasso(random_state=0, max_iter=10000) -alphas = np.logspace(-4, -0.5, 30) - -tuned_parameters = [{'alpha': alphas}] -n_folds = 5 - -clf = GridSearchCV(lasso, tuned_parameters, cv=n_folds, refit=False) -clf.fit(X, y) -scores = clf.cv_results_['mean_test_score'] -scores_std = clf.cv_results_['std_test_score'] -plt.figure().set_size_inches(8, 6) -plt.semilogx(alphas, scores) - -# plot error lines showing +/- std. errors of the scores -std_error = scores_std / np.sqrt(n_folds) - -plt.semilogx(alphas, scores + std_error, 'b--') -plt.semilogx(alphas, scores - std_error, 'b--') - -# alpha=0.2 controls the translucency of the fill color -plt.fill_between(alphas, scores + std_error, scores - std_error, alpha=0.2) - -plt.ylabel('CV score +/- std error') -plt.xlabel('alpha') -plt.axhline(np.max(scores), linestyle='--', color='.5') -plt.xlim([alphas[0], alphas[-1]]) - -# ############################################################################# -# Bonus: how much can you trust the selection of alpha? - -# To answer this question we use the LassoCV object that sets its alpha -# parameter automatically from the data by internal cross-validation (i.e. it -# performs cross-validation on the training data it receives). -# We use external cross-validation to see how much the automatically obtained -# alphas differ across different cross-validation folds. -lasso_cv = LassoCV(alphas=alphas, random_state=0, max_iter=10000) -k_fold = KFold(3) - -print("Answer to the bonus question:", - "how much can you trust the selection of alpha?") -print() -print("Alpha parameters maximising the generalization score on different") -print("subsets of the data:") -for k, (train, test) in enumerate(k_fold.split(X, y)): - lasso_cv.fit(X[train], y[train]) - print("[fold {0}] alpha: {1:.5f}, score: {2:.5f}". - format(k, lasso_cv.alpha_, lasso_cv.score(X[test], y[test]))) -print() -print("Answer: Not very much since we obtained different alphas for different") -print("subsets of the data and moreover, the scores for these alphas differ") -print("quite substantially.") - -plt.show() diff --git a/examples/exercises/plot_cv_digits.py b/examples/exercises/plot_cv_digits.py deleted file mode 100644 index 4f4ef8cc761e6..0000000000000 --- a/examples/exercises/plot_cv_digits.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -============================================= -Cross-validation on Digits Dataset Exercise -============================================= - -A tutorial exercise using Cross-validation with an SVM on the Digits dataset. - -This exercise is used in the :ref:`cv_generators_tut` part of the -:ref:`model_selection_tut` section of the :ref:`stat_learn_tut_index`. -""" -print(__doc__) - - -import numpy as np -from sklearn.model_selection import cross_val_score -from sklearn import datasets, svm - -X, y = datasets.load_digits(return_X_y=True) - -svc = svm.SVC(kernel='linear') -C_s = np.logspace(-10, 0, 10) - -scores = list() -scores_std = list() -for C in C_s: - svc.C = C - this_scores = cross_val_score(svc, X, y, n_jobs=1) - scores.append(np.mean(this_scores)) - scores_std.append(np.std(this_scores)) - -# Do the plotting -import matplotlib.pyplot as plt -plt.figure() -plt.semilogx(C_s, scores) -plt.semilogx(C_s, np.array(scores) + np.array(scores_std), 'b--') -plt.semilogx(C_s, np.array(scores) - np.array(scores_std), 'b--') -locs, labels = plt.yticks() -plt.yticks(locs, list(map(lambda x: "%g" % x, locs))) -plt.ylabel('CV score') -plt.xlabel('Parameter C') -plt.ylim(0, 1.1) -plt.show() diff --git a/examples/exercises/plot_digits_classification_exercise.py b/examples/exercises/plot_digits_classification_exercise.py deleted file mode 100644 index f5f01687d03eb..0000000000000 --- a/examples/exercises/plot_digits_classification_exercise.py +++ /dev/null @@ -1,32 +0,0 @@ -""" -================================ -Digits Classification Exercise -================================ - -A tutorial exercise regarding the use of classification techniques on -the Digits dataset. - -This exercise is used in the :ref:`clf_tut` part of the -:ref:`supervised_learning_tut` section of the -:ref:`stat_learn_tut_index`. -""" -print(__doc__) - -from sklearn import datasets, neighbors, linear_model - -X_digits, y_digits = datasets.load_digits(return_X_y=True) -X_digits = X_digits / X_digits.max() - -n_samples = len(X_digits) - -X_train = X_digits[:int(.9 * n_samples)] -y_train = y_digits[:int(.9 * n_samples)] -X_test = X_digits[int(.9 * n_samples):] -y_test = y_digits[int(.9 * n_samples):] - -knn = neighbors.KNeighborsClassifier() -logistic = linear_model.LogisticRegression(max_iter=1000) - -print('KNN score: %f' % knn.fit(X_train, y_train).score(X_test, y_test)) -print('LogisticRegression score: %f' - % logistic.fit(X_train, y_train).score(X_test, y_test)) diff --git a/examples/exercises/plot_iris_exercise.py b/examples/exercises/plot_iris_exercise.py deleted file mode 100644 index 39723a2271f5d..0000000000000 --- a/examples/exercises/plot_iris_exercise.py +++ /dev/null @@ -1,67 +0,0 @@ -""" -================================ -SVM Exercise -================================ - -A tutorial exercise for using different SVM kernels. - -This exercise is used in the :ref:`using_kernels_tut` part of the -:ref:`supervised_learning_tut` section of the :ref:`stat_learn_tut_index`. -""" -print(__doc__) - - -import numpy as np -import matplotlib.pyplot as plt -from sklearn import datasets, svm - -iris = datasets.load_iris() -X = iris.data -y = iris.target - -X = X[y != 0, :2] -y = y[y != 0] - -n_sample = len(X) - -np.random.seed(0) -order = np.random.permutation(n_sample) -X = X[order] -y = y[order].astype(float) - -X_train = X[:int(.9 * n_sample)] -y_train = y[:int(.9 * n_sample)] -X_test = X[int(.9 * n_sample):] -y_test = y[int(.9 * n_sample):] - -# fit the model -for kernel in ('linear', 'rbf', 'poly'): - clf = svm.SVC(kernel=kernel, gamma=10) - clf.fit(X_train, y_train) - - plt.figure() - plt.clf() - plt.scatter(X[:, 0], X[:, 1], c=y, zorder=10, cmap=plt.cm.Paired, - edgecolor='k', s=20) - - # Circle out the test data - plt.scatter(X_test[:, 0], X_test[:, 1], s=80, facecolors='none', - zorder=10, edgecolor='k') - - plt.axis('tight') - x_min = X[:, 0].min() - x_max = X[:, 0].max() - y_min = X[:, 1].min() - y_max = X[:, 1].max() - - XX, YY = np.mgrid[x_min:x_max:200j, y_min:y_max:200j] - Z = clf.decision_function(np.c_[XX.ravel(), YY.ravel()]) - - # Put the result into a color plot - Z = Z.reshape(XX.shape) - plt.pcolormesh(XX, YY, Z > 0, cmap=plt.cm.Paired) - plt.contour(XX, YY, Z, colors=['k', 'k', 'k'], - linestyles=['--', '-', '--'], levels=[-.5, 0, .5]) - - plt.title(kernel) -plt.show() diff --git a/examples/feature_selection/plot_f_test_vs_mi.py b/examples/feature_selection/plot_f_test_vs_mi.py index d9359380bfa96..e3c75d39e0a27 100644 --- a/examples/feature_selection/plot_f_test_vs_mi.py +++ b/examples/feature_selection/plot_f_test_vs_mi.py @@ -9,7 +9,7 @@ We consider 3 features x_1, x_2, x_3 distributed uniformly over [0, 1], the target depends on them as follows: -y = x_1 + sin(6 * pi * x_2) + 0.1 * N(0, 1), that is the third features is +y = x_1 + sin(6 * pi * x_2) + 0.1 * N(0, 1), that is the third feature is completely irrelevant. The code below plots the dependency of y against individual x_i and normalized @@ -19,12 +19,16 @@ discriminative feature. On the other hand, mutual information can capture any kind of dependency between variables and it rates x_2 as the most discriminative feature, which probably agrees better with our intuitive -perception for this example. Both methods correctly marks x_3 as irrelevant. +perception for this example. Both methods correctly mark x_3 as irrelevant. + """ -print(__doc__) -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt +import numpy as np + from sklearn.feature_selection import f_regression, mutual_info_regression np.random.seed(0) @@ -40,10 +44,9 @@ plt.figure(figsize=(15, 5)) for i in range(3): plt.subplot(1, 3, i + 1) - plt.scatter(X[:, i], y, edgecolor='black', s=20) + plt.scatter(X[:, i], y, edgecolor="black", s=20) plt.xlabel("$x_{}$".format(i + 1), fontsize=14) if i == 0: plt.ylabel("$y$", fontsize=14) - plt.title("F-test={:.2f}, MI={:.2f}".format(f_test[i], mi[i]), - fontsize=16) + plt.title("F-test={:.2f}, MI={:.2f}".format(f_test[i], mi[i]), fontsize=16) plt.show() diff --git a/examples/feature_selection/plot_feature_selection.py b/examples/feature_selection/plot_feature_selection.py index 1e4ef6a81bba8..520747f417db1 100644 --- a/examples/feature_selection/plot_feature_selection.py +++ b/examples/feature_selection/plot_feature_selection.py @@ -3,36 +3,30 @@ Univariate Feature Selection ============================ -An example showing univariate feature selection. - -Noisy (non informative) features are added to the iris data and -univariate feature selection is applied. For each feature, we plot the -p-values for the univariate feature selection and the corresponding -weights of an SVM. We can see that univariate feature selection -selects the informative features and that these have larger SVM weights. - -In the total set of features, only the 4 first ones are significant. We -can see that they have the highest score with univariate feature -selection. The SVM assigns a large weight to one of these features, but also -Selects many of the non-informative features. -Applying univariate feature selection before the SVM -increases the SVM weight attributed to the significant features, and will -thus improve classification. +This notebook is an example of using univariate feature selection +to improve classification accuracy on a noisy dataset. + +In this example, some noisy (non informative) features are added to +the iris dataset. Support vector machine (SVM) is used to classify the +dataset both before and after applying univariate feature selection. +For each feature, we plot the p-values for the univariate feature selection +and the corresponding weights of SVMs. With this, we will compare model +accuracy and examine the impact of univariate feature selection on model +weights. + """ -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Generate sample data +# -------------------- +# import numpy as np -import matplotlib.pyplot as plt from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split -from sklearn.preprocessing import MinMaxScaler -from sklearn.svm import LinearSVC -from sklearn.pipeline import make_pipeline -from sklearn.feature_selection import SelectKBest, f_classif - -# ############################################################################# -# Import some data to play with # The iris dataset X, y = load_iris(return_X_y=True) @@ -44,55 +38,96 @@ X = np.hstack((X, E)) # Split dataset to select feature and evaluate the classifier -X_train, X_test, y_train, y_test = train_test_split( - X, y, stratify=y, random_state=0 -) - -plt.figure(1) -plt.clf() - -X_indices = np.arange(X.shape[-1]) +X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) + +# %% +# Univariate feature selection +# ---------------------------- +# +# Univariate feature selection with F-test for feature scoring. +# We use the default selection function to select +# the four most significant features. +from sklearn.feature_selection import SelectKBest, f_classif -# ############################################################################# -# Univariate feature selection with F-test for feature scoring -# We use the default selection function to select the four -# most significant features selector = SelectKBest(f_classif, k=4) selector.fit(X_train, y_train) scores = -np.log10(selector.pvalues_) scores /= scores.max() -plt.bar(X_indices - .45, scores, width=.2, - label=r'Univariate score ($-Log(p_{value})$)') -# ############################################################################# -# Compare to the weights of an SVM +# %% +import matplotlib.pyplot as plt + +X_indices = np.arange(X.shape[-1]) +plt.figure(1) +plt.clf() +plt.bar(X_indices - 0.05, scores, width=0.2) +plt.title("Feature univariate score") +plt.xlabel("Feature number") +plt.ylabel(r"Univariate score ($-Log(p_{value})$)") +plt.show() + +# %% +# In the total set of features, only the 4 of the original features are significant. +# We can see that they have the highest score with univariate feature +# selection. + +# %% +# Compare with SVMs +# ----------------- +# +# Without univariate feature selection +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import MinMaxScaler +from sklearn.svm import LinearSVC + clf = make_pipeline(MinMaxScaler(), LinearSVC()) clf.fit(X_train, y_train) -print('Classification accuracy without selecting features: {:.3f}' - .format(clf.score(X_test, y_test))) +print( + "Classification accuracy without selecting features: {:.3f}".format( + clf.score(X_test, y_test) + ) +) svm_weights = np.abs(clf[-1].coef_).sum(axis=0) svm_weights /= svm_weights.sum() -plt.bar(X_indices - .25, svm_weights, width=.2, label='SVM weight') - -clf_selected = make_pipeline( - SelectKBest(f_classif, k=4), MinMaxScaler(), LinearSVC() -) +# %% +# After univariate feature selection +clf_selected = make_pipeline(SelectKBest(f_classif, k=4), MinMaxScaler(), LinearSVC()) clf_selected.fit(X_train, y_train) -print('Classification accuracy after univariate feature selection: {:.3f}' - .format(clf_selected.score(X_test, y_test))) +print( + "Classification accuracy after univariate feature selection: {:.3f}".format( + clf_selected.score(X_test, y_test) + ) +) svm_weights_selected = np.abs(clf_selected[-1].coef_).sum(axis=0) svm_weights_selected /= svm_weights_selected.sum() -plt.bar(X_indices[selector.get_support()] - .05, svm_weights_selected, - width=.2, label='SVM weights after selection') +# %% +plt.bar( + X_indices - 0.45, scores, width=0.2, label=r"Univariate score ($-Log(p_{value})$)" +) + +plt.bar(X_indices - 0.25, svm_weights, width=0.2, label="SVM weight") +plt.bar( + X_indices[selector.get_support()] - 0.05, + svm_weights_selected, + width=0.2, + label="SVM weights after selection", +) plt.title("Comparing feature selection") -plt.xlabel('Feature number') +plt.xlabel("Feature number") plt.yticks(()) -plt.axis('tight') -plt.legend(loc='upper right') +plt.axis("tight") +plt.legend(loc="upper right") plt.show() + +# %% +# Without univariate feature selection, the SVM assigns a large weight +# to the first 4 original significant features, but also selects many of the +# non-informative features. Applying univariate feature selection before +# the SVM increases the SVM weight attributed to the significant features, +# and will thus improve classification. diff --git a/examples/feature_selection/plot_feature_selection_pipeline.py b/examples/feature_selection/plot_feature_selection_pipeline.py index 871c894ee0711..b33bfeda0a37a 100644 --- a/examples/feature_selection/plot_feature_selection_pipeline.py +++ b/examples/feature_selection/plot_feature_selection_pipeline.py @@ -6,12 +6,12 @@ This example shows how a feature selection can be easily integrated within a machine learning pipeline. -We also show that you can easily introspect part of the pipeline. +We also show that you can easily inspect part of the pipeline. + """ -print(__doc__) -from sklearn import set_config -set_config(display='diagram') +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause # %% # We will start by generating a binary classification dataset. Subsequently, we @@ -21,13 +21,18 @@ from sklearn.model_selection import train_test_split X, y = make_classification( - n_features=20, n_informative=3, n_redundant=0, n_classes=2, - n_clusters_per_class=2, random_state=42) + n_features=20, + n_informative=3, + n_redundant=0, + n_classes=2, + n_clusters_per_class=2, + random_state=42, +) X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42) # %% # A common mistake done with feature selection is to search a subset of -# discriminative features on the full dataset instead of only using the +# discriminative features on the full dataset, instead of only using the # training set. The usage of scikit-learn :func:`~sklearn.pipeline.Pipeline` # prevents to make such mistake. # @@ -36,7 +41,7 @@ # # When calling `fit` on the training data, a subset of feature will be selected # and the index of these selected features will be stored. The feature selector -# will subsequently reduce the number of feature and pass this subset to the +# will subsequently reduce the number of features, and pass this subset to the # classifier which will be trained. from sklearn.feature_selection import SelectKBest, f_classif @@ -49,12 +54,12 @@ anova_svm.fit(X_train, y_train) # %% -# Once the training accomplished, we can predict on new unseen samples. In this +# Once the training is complete, we can predict on new unseen samples. In this # case, the feature selector will only select the most discriminative features # based on the information stored during training. Then, the data will be # passed to the classifier which will make the prediction. # -# Here, we report the final metrics via a classification report. +# Here, we show the final metrics via a classification report. from sklearn.metrics import classification_report @@ -69,13 +74,13 @@ anova_svm[-1].coef_ # %% -# However, we do not know which features where selected from the original -# dataset. We could proceed by several manner. Here, we will inverse the +# However, we do not know which features were selected from the original +# dataset. We could proceed by several manners. Here, we will invert the # transformation of these coefficients to get information about the original # space. anova_svm[:-1].inverse_transform(anova_svm[-1].coef_) # %% -# We can see that the first three features where the selected features by -# the first step. +# We can see that the features with non-zero coefficients are the selected +# features by the first step. diff --git a/examples/feature_selection/plot_rfe_digits.py b/examples/feature_selection/plot_rfe_digits.py index 1c2ca03875f2f..360a9bd92837f 100644 --- a/examples/feature_selection/plot_rfe_digits.py +++ b/examples/feature_selection/plot_rfe_digits.py @@ -3,34 +3,55 @@ Recursive feature elimination ============================= -A recursive feature elimination example showing the relevance of pixels in -a digit classification task. +This example demonstrates how Recursive Feature Elimination +(:class:`~sklearn.feature_selection.RFE`) can be used to determine the +importance of individual pixels for classifying handwritten digits. +:class:`~sklearn.feature_selection.RFE` recursively removes the least +significant features, assigning ranks based on their importance, where higher +`ranking_` values denote lower importance. The ranking is visualized using both +shades of blue and pixel annotations for clarity. As expected, pixels positioned +at the center of the image tend to be more predictive than those near the edges. .. note:: See also :ref:`sphx_glr_auto_examples_feature_selection_plot_rfe_with_cross_validation.py` -""" -print(__doc__) +""" # noqa: E501 + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import matplotlib.pyplot as plt -from sklearn.svm import SVC from sklearn.datasets import load_digits from sklearn.feature_selection import RFE -import matplotlib.pyplot as plt +from sklearn.linear_model import LogisticRegression +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import MinMaxScaler # Load the digits dataset digits = load_digits() X = digits.images.reshape((len(digits.images), -1)) y = digits.target -# Create the RFE object and rank each pixel -svc = SVC(kernel="linear", C=1) -rfe = RFE(estimator=svc, n_features_to_select=1, step=1) -rfe.fit(X, y) -ranking = rfe.ranking_.reshape(digits.images[0].shape) +pipe = Pipeline( + [ + ("scaler", MinMaxScaler()), + ("rfe", RFE(estimator=LogisticRegression(), n_features_to_select=1, step=1)), + ] +) + +pipe.fit(X, y) +ranking = pipe.named_steps["rfe"].ranking_.reshape(digits.images[0].shape) # Plot pixel ranking plt.matshow(ranking, cmap=plt.cm.Blues) + +# Add annotations for pixel numbers +for i in range(ranking.shape[0]): + for j in range(ranking.shape[1]): + plt.text(j, i, str(ranking[i, j]), ha="center", va="center", color="black") + plt.colorbar() -plt.title("Ranking of pixels with RFE") +plt.title("Ranking of pixels with RFE\n(Logistic Regression)") plt.show() diff --git a/examples/feature_selection/plot_rfe_with_cross_validation.py b/examples/feature_selection/plot_rfe_with_cross_validation.py index e63f3edb551af..951b82bffa46d 100644 --- a/examples/feature_selection/plot_rfe_with_cross_validation.py +++ b/examples/feature_selection/plot_rfe_with_cross_validation.py @@ -3,40 +3,113 @@ Recursive feature elimination with cross-validation =================================================== -A recursive feature elimination example with automatic tuning of the +A Recursive Feature Elimination (RFE) example with automatic tuning of the number of features selected with cross-validation. + """ -print(__doc__) -import matplotlib.pyplot as plt -from sklearn.svm import SVC -from sklearn.model_selection import StratifiedKFold -from sklearn.feature_selection import RFECV +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Data generation +# --------------- +# +# We build a classification task using 3 informative features. The introduction +# of 2 additional redundant (i.e. correlated) features has the effect that the +# selected features vary depending on the cross-validation fold. The remaining +# features are non-informative as they are drawn at random. + from sklearn.datasets import make_classification -# Build a classification task using 3 informative features -X, y = make_classification(n_samples=1000, n_features=25, n_informative=3, - n_redundant=2, n_repeated=0, n_classes=8, - n_clusters_per_class=1, random_state=0) +n_features = 15 +feat_names = [f"feature_{i}" for i in range(15)] + +X, y = make_classification( + n_samples=500, + n_features=n_features, + n_informative=3, + n_redundant=2, + n_repeated=0, + n_classes=8, + n_clusters_per_class=1, + class_sep=0.8, + random_state=0, +) -# Create the RFE object and compute a cross-validated score. -svc = SVC(kernel="linear") -# The "accuracy" scoring is proportional to the number of correct -# classifications +# %% +# Model training and selection +# ---------------------------- +# +# We create the RFE object and compute the cross-validated scores. The scoring +# strategy "accuracy" optimizes the proportion of correctly classified samples. + +from sklearn.feature_selection import RFECV +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import StratifiedKFold min_features_to_select = 1 # Minimum number of features to consider -rfecv = RFECV(estimator=svc, step=1, cv=StratifiedKFold(2), - scoring='accuracy', - min_features_to_select=min_features_to_select) +clf = LogisticRegression() +cv = StratifiedKFold(5) + +rfecv = RFECV( + estimator=clf, + step=1, + cv=cv, + scoring="accuracy", + min_features_to_select=min_features_to_select, + n_jobs=2, +) rfecv.fit(X, y) -print("Optimal number of features : %d" % rfecv.n_features_) +print(f"Optimal number of features: {rfecv.n_features_}") +# %% +# In the present case, the model with 3 features (which corresponds to the true +# generative model) is found to be the most optimal. +# # Plot number of features VS. cross-validation scores +# --------------------------------------------------- + +import matplotlib.pyplot as plt +import pandas as pd + +data = { + key: value + for key, value in rfecv.cv_results_.items() + if key in ["n_features", "mean_test_score", "std_test_score"] +} +cv_results = pd.DataFrame(data) plt.figure() plt.xlabel("Number of features selected") -plt.ylabel("Cross validation score (nb of correct classifications)") -plt.plot(range(min_features_to_select, - len(rfecv.grid_scores_) + min_features_to_select), - rfecv.grid_scores_) +plt.ylabel("Mean test accuracy") +plt.errorbar( + x=cv_results["n_features"], + y=cv_results["mean_test_score"], + yerr=cv_results["std_test_score"], +) +plt.title("Recursive Feature Elimination \nwith correlated features") plt.show() + +# %% +# From the plot above one can further notice a plateau of equivalent scores +# (similar mean value and overlapping errorbars) for 3 to 5 selected features. +# This is the result of introducing correlated features. Indeed, the optimal +# model selected by the RFE can lie within this range, depending on the +# cross-validation technique. The test accuracy decreases above 5 selected +# features, this is, keeping non-informative features leads to over-fitting and +# is therefore detrimental for the statistical performance of the models. + +# %% +import numpy as np + +for i in range(cv.n_splits): + mask = rfecv.cv_results_[f"split{i}_support"][ + rfecv.n_features_ + ] # mask of features selected by the RFE + features_selected = np.ma.compressed(np.ma.masked_array(feat_names, mask=1 - mask)) + print(f"Features selected in fold {i}: {features_selected}") +# %% +# In the five folds, the selected features are consistent. This is good news, +# it means that the selection is stable across folds, and it confirms that +# these features are the most informative ones. diff --git a/examples/feature_selection/plot_select_from_model_diabetes.py b/examples/feature_selection/plot_select_from_model_diabetes.py index fdc57dc867f9e..793a6916e8969 100644 --- a/examples/feature_selection/plot_select_from_model_diabetes.py +++ b/examples/feature_selection/plot_select_from_model_diabetes.py @@ -6,20 +6,15 @@ This example illustrates and compares two approaches for feature selection: :class:`~sklearn.feature_selection.SelectFromModel` which is based on feature importance, and -:class:`~sklearn.feature_selection.SequentialFeatureSelection` which relies +:class:`~sklearn.feature_selection.SequentialFeatureSelector` which relies on a greedy approach. We use the Diabetes dataset, which consists of 10 features collected from 442 diabetes patients. - -Authors: `Manoj Kumar `_, -`Maria Telenczuk `_, Nicolas Hug. - -License: BSD 3 clause """ -print(__doc__) - +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause # %% # Loading the data @@ -38,20 +33,21 @@ # ------------------------------------ # # To get an idea of the importance of the features, we are going to use the -# :class:`~sklearn.linear_model.LassoCV` estimator. The features with the +# :class:`~sklearn.linear_model.RidgeCV` estimator. The features with the # highest absolute `coef_` value are considered the most important. # We can observe the coefficients directly without needing to scale them (or # scale the data) because from the description above, we know that the features # were already standardized. # For a more complete example on the interpretations of the coefficients of # linear models, you may refer to -# :ref:`sphx_glr_auto_examples_inspection_plot_linear_model_coefficient_interpretation.py`. +# :ref:`sphx_glr_auto_examples_inspection_plot_linear_model_coefficient_interpretation.py`. # noqa: E501 import matplotlib.pyplot as plt import numpy as np -from sklearn.linear_model import LassoCV -lasso = LassoCV().fit(X, y) -importance = np.abs(lasso.coef_) +from sklearn.linear_model import RidgeCV + +ridge = RidgeCV(alphas=np.logspace(-6, 6, num=5)).fit(X, y) +importance = np.abs(ridge.coef_) feature_names = np.array(diabetes.feature_names) plt.bar(height=importance, x=feature_names) plt.title("Feature importances via coefficients") @@ -69,16 +65,16 @@ # # Since we want to select only 2 features, we will set this threshold slightly # above the coefficient of third most important feature. -from sklearn.feature_selection import SelectFromModel from time import time +from sklearn.feature_selection import SelectFromModel + threshold = np.sort(importance)[-3] + 0.01 tic = time() -sfm = SelectFromModel(lasso, threshold=threshold).fit(X, y) +sfm = SelectFromModel(ridge, threshold=threshold).fit(X, y) toc = time() -print("Features selected by SelectFromModel: " - f"{feature_names[sfm.get_support()]}") +print(f"Features selected by SelectFromModel: {feature_names[sfm.get_support()]}") print(f"Done in {toc - tic:.3f}s") # %% @@ -100,26 +96,29 @@ from sklearn.feature_selection import SequentialFeatureSelector tic_fwd = time() -sfs_forward = SequentialFeatureSelector(lasso, n_features_to_select=2, - direction='forward').fit(X, y) +sfs_forward = SequentialFeatureSelector( + ridge, n_features_to_select=2, direction="forward" +).fit(X, y) toc_fwd = time() tic_bwd = time() -sfs_backward = SequentialFeatureSelector(lasso, n_features_to_select=2, - direction='backward').fit(X, y) +sfs_backward = SequentialFeatureSelector( + ridge, n_features_to_select=2, direction="backward" +).fit(X, y) toc_bwd = time() -print("Features selected by forward sequential selection: " - f"{feature_names[sfs_forward.get_support()]}") +print( + "Features selected by forward sequential selection: " + f"{feature_names[sfs_forward.get_support()]}" +) print(f"Done in {toc_fwd - tic_fwd:.3f}s") -print("Features selected by backward sequential selection: " - f"{feature_names[sfs_backward.get_support()]}") +print( + "Features selected by backward sequential selection: " + f"{feature_names[sfs_backward.get_support()]}" +) print(f"Done in {toc_bwd - tic_bwd:.3f}s") # %% -# Discussion -# ---------- -# # Interestingly, forward and backward selection have selected the same set of # features. In general, this isn't the case and the two methods would lead to # different results. @@ -140,3 +139,54 @@ # attribute. The forward SFS is faster than the backward SFS because it only # needs to perform `n_features_to_select = 2` iterations, while the backward # SFS needs to perform `n_features - n_features_to_select = 8` iterations. +# +# Using negative tolerance values +# ------------------------------- +# +# :class:`~sklearn.feature_selection.SequentialFeatureSelector` can be used +# to remove features present in the dataset and return a +# smaller subset of the original features with `direction="backward"` +# and a negative value of `tol`. +# +# We begin by loading the Breast Cancer dataset, consisting of 30 different +# features and 569 samples. +import numpy as np + +from sklearn.datasets import load_breast_cancer + +breast_cancer_data = load_breast_cancer() +X, y = breast_cancer_data.data, breast_cancer_data.target +feature_names = np.array(breast_cancer_data.feature_names) +print(breast_cancer_data.DESCR) + +# %% +# We will make use of the :class:`~sklearn.linear_model.LogisticRegression` +# estimator with :class:`~sklearn.feature_selection.SequentialFeatureSelector` +# to perform the feature selection. +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import roc_auc_score +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import StandardScaler + +for tol in [-1e-2, -1e-3, -1e-4]: + start = time() + feature_selector = SequentialFeatureSelector( + LogisticRegression(), + n_features_to_select="auto", + direction="backward", + scoring="roc_auc", + tol=tol, + n_jobs=2, + ) + model = make_pipeline(StandardScaler(), feature_selector, LogisticRegression()) + model.fit(X, y) + end = time() + print(f"\ntol: {tol}") + print(f"Features selected: {feature_names[model[1].get_support()]}") + print(f"ROC AUC score: {roc_auc_score(y, model.predict_proba(X)[:, 1]):.3f}") + print(f"Done in {end - start:.3f}s") + +# %% +# We can see that the number of features selected tend to increase as negative +# values of `tol` approach to zero. The time taken for feature selection also +# decreases as the values of `tol` come closer to zero. diff --git a/examples/frozen/README.txt b/examples/frozen/README.txt new file mode 100644 index 0000000000000..b0468dcae04d5 --- /dev/null +++ b/examples/frozen/README.txt @@ -0,0 +1,6 @@ +.. _frozen_examples: + +Frozen Estimators +----------------- + +Examples concerning the :mod:`sklearn.frozen` module. diff --git a/examples/frozen/plot_frozen_examples.py b/examples/frozen/plot_frozen_examples.py new file mode 100644 index 0000000000000..7237003090d13 --- /dev/null +++ b/examples/frozen/plot_frozen_examples.py @@ -0,0 +1,98 @@ +""" +=================================== +Examples of Using `FrozenEstimator` +=================================== + +This example showcases some use cases of :class:`~sklearn.frozen.FrozenEstimator`. + +:class:`~sklearn.frozen.FrozenEstimator` is a utility class that allows to freeze a +fitted estimator. This is useful, for instance, when we want to pass a fitted estimator +to a meta-estimator, such as :class:`~sklearn.model_selection.FixedThresholdClassifier` +without letting the meta-estimator refit the estimator. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Setting a decision threshold for a pre-fitted classifier +# -------------------------------------------------------- +# Fitted classifiers in scikit-learn use an arbitrary decision threshold to decide +# which class the given sample belongs to. The decision threshold is either `0.0` on the +# value returned by :term:`decision_function`, or `0.5` on the probability returned by +# :term:`predict_proba`. +# +# However, one might want to set a custom decision threshold. We can do this by +# using :class:`~sklearn.model_selection.FixedThresholdClassifier` and wrapping the +# classifier with :class:`~sklearn.frozen.FrozenEstimator`. +from sklearn.datasets import make_classification +from sklearn.frozen import FrozenEstimator +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import FixedThresholdClassifier, train_test_split + +X, y = make_classification(n_samples=1000, random_state=0) +X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0) +classifier = LogisticRegression().fit(X_train, y_train) + +print( + "Probability estimates for three data points:\n" + f"{classifier.predict_proba(X_test[-3:]).round(3)}" +) +print( + "Predicted class for the same three data points:\n" + f"{classifier.predict(X_test[-3:])}" +) + +# %% +# Now imagine you'd want to set a different decision threshold on the probability +# estimates. We can do this by wrapping the classifier with +# :class:`~sklearn.frozen.FrozenEstimator` and passing it to +# :class:`~sklearn.model_selection.FixedThresholdClassifier`. + +threshold_classifier = FixedThresholdClassifier( + estimator=FrozenEstimator(classifier), threshold=0.9 +) + +# %% +# Note that in the above piece of code, calling `fit` on +# :class:`~sklearn.model_selection.FixedThresholdClassifier` does not refit the +# underlying classifier. +# +# Now, let's see how the predictions changed with respect to the probability +# threshold. +print( + "Probability estimates for three data points with FixedThresholdClassifier:\n" + f"{threshold_classifier.predict_proba(X_test[-3:]).round(3)}" +) +print( + "Predicted class for the same three data points with FixedThresholdClassifier:\n" + f"{threshold_classifier.predict(X_test[-3:])}" +) + +# %% +# We see that the probability estimates stay the same, but since a different decision +# threshold is used, the predicted classes are different. +# +# Please refer to +# :ref:`sphx_glr_auto_examples_model_selection_plot_cost_sensitive_learning.py` +# to learn about cost-sensitive learning and decision threshold tuning. + +# %% +# Calibration of a pre-fitted classifier +# -------------------------------------- +# You can use :class:`~sklearn.frozen.FrozenEstimator` to calibrate a pre-fitted +# classifier using :class:`~sklearn.calibration.CalibratedClassifierCV`. +from sklearn.calibration import CalibratedClassifierCV +from sklearn.metrics import brier_score_loss + +calibrated_classifier = CalibratedClassifierCV( + estimator=FrozenEstimator(classifier) +).fit(X_train, y_train) + +prob_pos_clf = classifier.predict_proba(X_test)[:, 1] +clf_score = brier_score_loss(y_test, prob_pos_clf) +print(f"No calibration: {clf_score:.3f}") + +prob_pos_calibrated = calibrated_classifier.predict_proba(X_test)[:, 1] +calibrated_score = brier_score_loss(y_test, prob_pos_calibrated) +print(f"With calibration: {calibrated_score:.3f}") diff --git a/examples/gaussian_process/README.txt b/examples/gaussian_process/README.txt index 5ee038e015639..a6aab882c540f 100644 --- a/examples/gaussian_process/README.txt +++ b/examples/gaussian_process/README.txt @@ -4,4 +4,3 @@ Gaussian Process for Machine Learning ------------------------------------- Examples concerning the :mod:`sklearn.gaussian_process` module. - diff --git a/examples/gaussian_process/plot_compare_gpr_krr.py b/examples/gaussian_process/plot_compare_gpr_krr.py index 1eb771673b0d6..52375a9c4a267 100644 --- a/examples/gaussian_process/plot_compare_gpr_krr.py +++ b/examples/gaussian_process/plot_compare_gpr_krr.py @@ -3,119 +3,394 @@ Comparison of kernel ridge and Gaussian process regression ========================================================== -Both kernel ridge regression (KRR) and Gaussian process regression (GPR) learn -a target function by employing internally the "kernel trick". KRR learns a -linear function in the space induced by the respective kernel which corresponds -to a non-linear function in the original space. The linear function in the -kernel space is chosen based on the mean-squared error loss with -ridge regularization. GPR uses the kernel to define the covariance of -a prior distribution over the target functions and uses the observed training -data to define a likelihood function. Based on Bayes theorem, a (Gaussian) -posterior distribution over target functions is defined, whose mean is used -for prediction. - -A major difference is that GPR can choose the kernel's hyperparameters based -on gradient-ascent on the marginal likelihood function while KRR needs to -perform a grid search on a cross-validated loss function (mean-squared error -loss). A further difference is that GPR learns a generative, probabilistic -model of the target function and can thus provide meaningful confidence -intervals and posterior samples along with the predictions while KRR only -provides predictions. - -This example illustrates both methods on an artificial dataset, which -consists of a sinusoidal target function and strong noise. The figure compares -the learned model of KRR and GPR based on a ExpSineSquared kernel, which is -suited for learning periodic functions. The kernel's hyperparameters control -the smoothness (l) and periodicity of the kernel (p). Moreover, the noise level -of the data is learned explicitly by GPR by an additional WhiteKernel component -in the kernel and by the regularization parameter alpha of KRR. - -The figure shows that both methods learn reasonable models of the target -function. GPR correctly identifies the periodicity of the function to be -roughly 2*pi (6.28), while KRR chooses the doubled periodicity 4*pi. Besides -that, GPR provides reasonable confidence bounds on the prediction which are not -available for KRR. A major difference between the two methods is the time -required for fitting and predicting: while fitting KRR is fast in principle, -the grid-search for hyperparameter optimization scales exponentially with the -number of hyperparameters ("curse of dimensionality"). The gradient-based -optimization of the parameters in GPR does not suffer from this exponential -scaling and is thus considerable faster on this example with 3-dimensional -hyperparameter space. The time for predicting is similar; however, generating -the variance of the predictive distribution of GPR takes considerable longer -than just predicting the mean. -""" -print(__doc__) +This example illustrates differences between a kernel ridge regression and a +Gaussian process regression. -# Authors: Jan Hendrik Metzen -# License: BSD 3 clause +Both kernel ridge regression and Gaussian process regression are using a +so-called "kernel trick" to make their models expressive enough to fit +the training data. However, the machine learning problems solved by the two +methods are drastically different. +Kernel ridge regression will find the target function that minimizes a loss +function (the mean squared error). -import time +Instead of finding a single target function, the Gaussian process regression +employs a probabilistic approach : a Gaussian posterior distribution over +target functions is defined based on the Bayes' theorem, Thus prior +probabilities on target functions are being combined with a likelihood function +defined by the observed training data to provide estimates of the posterior +distributions. + +We will illustrate these differences with an example and we will also focus on +tuning the kernel hyperparameters. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +# %% +# Generating a dataset +# -------------------- +# +# We create a synthetic dataset. The true generative process will take a 1-D +# vector and compute its sine. Note that the period of this sine is thus +# :math:`2 \pi`. We will reuse this information later in this example. import numpy as np +rng = np.random.RandomState(0) +data = np.linspace(0, 30, num=1_000).reshape(-1, 1) +target = np.sin(data).ravel() + +# %% +# Now, we can imagine a scenario where we get observations from this true +# process. However, we will add some challenges: +# +# - the measurements will be noisy; +# - only samples from the beginning of the signal will be available. +training_sample_indices = rng.choice(np.arange(0, 400), size=40, replace=False) +training_data = data[training_sample_indices] +training_noisy_target = target[training_sample_indices] + 0.5 * rng.randn( + len(training_sample_indices) +) + +# %% +# Let's plot the true signal and the noisy measurements available for training. import matplotlib.pyplot as plt +plt.plot(data, target, label="True signal", linewidth=2) +plt.scatter( + training_data, + training_noisy_target, + color="black", + label="Noisy measurements", +) +plt.legend() +plt.xlabel("data") +plt.ylabel("target") +_ = plt.title( + "Illustration of the true generative process and \n" + "noisy measurements available during training" +) + +# %% +# Limitations of a simple linear model +# ------------------------------------ +# +# First, we would like to highlight the limitations of a linear model given +# our dataset. We fit a :class:`~sklearn.linear_model.Ridge` and check the +# predictions of this model on our dataset. +from sklearn.linear_model import Ridge + +ridge = Ridge().fit(training_data, training_noisy_target) + +plt.plot(data, target, label="True signal", linewidth=2) +plt.scatter( + training_data, + training_noisy_target, + color="black", + label="Noisy measurements", +) +plt.plot(data, ridge.predict(data), label="Ridge regression") +plt.legend() +plt.xlabel("data") +plt.ylabel("target") +_ = plt.title("Limitation of a linear model such as ridge") + +# %% +# Such a ridge regressor underfits data since it is not expressive enough. +# +# Kernel methods: kernel ridge and Gaussian process +# ------------------------------------------------- +# +# Kernel ridge +# ............ +# +# We can make the previous linear model more expressive by using a so-called +# kernel. A kernel is an embedding from the original feature space to another +# one. Simply put, it is used to map our original data into a newer and more +# complex feature space. This new space is explicitly defined by the choice of +# kernel. +# +# In our case, we know that the true generative process is a periodic function. +# We can use a :class:`~sklearn.gaussian_process.kernels.ExpSineSquared` kernel +# which allows recovering the periodicity. The class +# :class:`~sklearn.kernel_ridge.KernelRidge` will accept such a kernel. +# +# Using this model together with a kernel is equivalent to embed the data +# using the mapping function of the kernel and then apply a ridge regression. +# In practice, the data are not mapped explicitly; instead the dot product +# between samples in the higher dimensional feature space is computed using the +# "kernel trick". +# +# Thus, let's use such a :class:`~sklearn.kernel_ridge.KernelRidge`. +import time + +from sklearn.gaussian_process.kernels import ExpSineSquared from sklearn.kernel_ridge import KernelRidge -from sklearn.model_selection import GridSearchCV + +kernel_ridge = KernelRidge(kernel=ExpSineSquared()) + +start_time = time.time() +kernel_ridge.fit(training_data, training_noisy_target) +print( + f"Fitting KernelRidge with default kernel: {time.time() - start_time:.3f} seconds" +) + +# %% +plt.plot(data, target, label="True signal", linewidth=2, linestyle="dashed") +plt.scatter( + training_data, + training_noisy_target, + color="black", + label="Noisy measurements", +) +plt.plot( + data, + kernel_ridge.predict(data), + label="Kernel ridge", + linewidth=2, + linestyle="dashdot", +) +plt.legend(loc="lower right") +plt.xlabel("data") +plt.ylabel("target") +_ = plt.title( + "Kernel ridge regression with an exponential sine squared\n " + "kernel using default hyperparameters" +) + +# %% +# This fitted model is not accurate. Indeed, we did not set the parameters of +# the kernel and instead used the default ones. We can inspect them. +kernel_ridge.kernel + +# %% +# Our kernel has two parameters: the length-scale and the periodicity. For our +# dataset, we use `sin` as the generative process, implying a +# :math:`2 \pi`-periodicity for the signal. The default value of the parameter +# being :math:`1`, it explains the high frequency observed in the predictions of +# our model. +# Similar conclusions could be drawn with the length-scale parameter. Thus, it +# tell us that the kernel parameters need to be tuned. We will use a randomized +# search to tune the different parameters the kernel ridge model: the `alpha` +# parameter and the kernel parameters. + +# %% +from scipy.stats import loguniform + +from sklearn.model_selection import RandomizedSearchCV + +param_distributions = { + "alpha": loguniform(1e0, 1e3), + "kernel__length_scale": loguniform(1e-2, 1e2), + "kernel__periodicity": loguniform(1e0, 1e1), +} +kernel_ridge_tuned = RandomizedSearchCV( + kernel_ridge, + param_distributions=param_distributions, + n_iter=500, + random_state=0, +) +start_time = time.time() +kernel_ridge_tuned.fit(training_data, training_noisy_target) +print(f"Time for KernelRidge fitting: {time.time() - start_time:.3f} seconds") + +# %% +# Fitting the model is now more computationally expensive since we have to try +# several combinations of hyperparameters. We can have a look at the +# hyperparameters found to get some intuitions. +kernel_ridge_tuned.best_params_ + +# %% +# Looking at the best parameters, we see that they are different from the +# defaults. We also see that the periodicity is closer to the expected value: +# :math:`2 \pi`. We can now inspect the predictions of our tuned kernel ridge. +start_time = time.time() +predictions_kr = kernel_ridge_tuned.predict(data) +print(f"Time for KernelRidge predict: {time.time() - start_time:.3f} seconds") + +# %% +plt.plot(data, target, label="True signal", linewidth=2, linestyle="dashed") +plt.scatter( + training_data, + training_noisy_target, + color="black", + label="Noisy measurements", +) +plt.plot( + data, + predictions_kr, + label="Kernel ridge", + linewidth=2, + linestyle="dashdot", +) +plt.legend(loc="lower right") +plt.xlabel("data") +plt.ylabel("target") +_ = plt.title( + "Kernel ridge regression with an exponential sine squared\n " + "kernel using tuned hyperparameters" +) + +# %% +# We get a much more accurate model. We still observe some errors mainly due to +# the noise added to the dataset. +# +# Gaussian process regression +# ........................... +# +# Now, we will use a +# :class:`~sklearn.gaussian_process.GaussianProcessRegressor` to fit the same +# dataset. When training a Gaussian process, the hyperparameters of the kernel +# are optimized during the fitting process. There is no need for an external +# hyperparameter search. Here, we create a slightly more complex kernel than +# for the kernel ridge regressor: we add a +# :class:`~sklearn.gaussian_process.kernels.WhiteKernel` that is used to +# estimate the noise in the dataset. from sklearn.gaussian_process import GaussianProcessRegressor -from sklearn.gaussian_process.kernels import WhiteKernel, ExpSineSquared +from sklearn.gaussian_process.kernels import WhiteKernel -rng = np.random.RandomState(0) +kernel = 1.0 * ExpSineSquared(1.0, 5.0, periodicity_bounds=(1e-2, 1e1)) + WhiteKernel( + 1e-1 +) +gaussian_process = GaussianProcessRegressor(kernel=kernel) +start_time = time.time() +gaussian_process.fit(training_data, training_noisy_target) +print( + f"Time for GaussianProcessRegressor fitting: {time.time() - start_time:.3f} seconds" +) + +# %% +# The computation cost of training a Gaussian process is much less than the +# kernel ridge that uses a randomized search. We can check the parameters of +# the kernels that we computed. +gaussian_process.kernel_ + +# %% +# Indeed, we see that the parameters have been optimized. Looking at the +# `periodicity` parameter, we see that we found a period close to the +# theoretical value :math:`2 \pi`. We can have a look now at the predictions of +# our model. +start_time = time.time() +mean_predictions_gpr, std_predictions_gpr = gaussian_process.predict( + data, + return_std=True, +) +print( + f"Time for GaussianProcessRegressor predict: {time.time() - start_time:.3f} seconds" +) + +# %% +plt.plot(data, target, label="True signal", linewidth=2, linestyle="dashed") +plt.scatter( + training_data, + training_noisy_target, + color="black", + label="Noisy measurements", +) +# Plot the predictions of the kernel ridge +plt.plot( + data, + predictions_kr, + label="Kernel ridge", + linewidth=2, + linestyle="dashdot", +) +# Plot the predictions of the gaussian process regressor +plt.plot( + data, + mean_predictions_gpr, + label="Gaussian process regressor", + linewidth=2, + linestyle="dotted", +) +plt.fill_between( + data.ravel(), + mean_predictions_gpr - std_predictions_gpr, + mean_predictions_gpr + std_predictions_gpr, + color="tab:green", + alpha=0.2, +) +plt.legend(loc="lower right") +plt.xlabel("data") +plt.ylabel("target") +_ = plt.title("Comparison between kernel ridge and gaussian process regressor") + +# %% +# We observe that the results of the kernel ridge and the Gaussian process +# regressor are close. However, the Gaussian process regressor also provide +# an uncertainty information that is not available with a kernel ridge. +# Due to the probabilistic formulation of the target functions, the +# Gaussian process can output the standard deviation (or the covariance) +# together with the mean predictions of the target functions. +# +# However, it comes at a cost: the time to compute the predictions is higher +# with a Gaussian process. +# +# Final conclusion +# ---------------- +# +# We can give a final word regarding the possibility of the two models to +# extrapolate. Indeed, we only provided the beginning of the signal as a +# training set. Using a periodic kernel forces our model to repeat the pattern +# found on the training set. Using this kernel information together with the +# capacity of the both models to extrapolate, we observe that the models will +# continue to predict the sine pattern. +# +# Gaussian process allows to combine kernels together. Thus, we could associate +# the exponential sine squared kernel together with a radial basis function +# kernel. +from sklearn.gaussian_process.kernels import RBF + +kernel = 1.0 * ExpSineSquared(1.0, 5.0, periodicity_bounds=(1e-2, 1e1)) * RBF( + length_scale=15, length_scale_bounds="fixed" +) + WhiteKernel(1e-1) +gaussian_process = GaussianProcessRegressor(kernel=kernel) +gaussian_process.fit(training_data, training_noisy_target) +mean_predictions_gpr, std_predictions_gpr = gaussian_process.predict( + data, + return_std=True, +) + +# %% +plt.plot(data, target, label="True signal", linewidth=2, linestyle="dashed") +plt.scatter( + training_data, + training_noisy_target, + color="black", + label="Noisy measurements", +) +# Plot the predictions of the kernel ridge +plt.plot( + data, + predictions_kr, + label="Kernel ridge", + linewidth=2, + linestyle="dashdot", +) +# Plot the predictions of the gaussian process regressor +plt.plot( + data, + mean_predictions_gpr, + label="Gaussian process regressor", + linewidth=2, + linestyle="dotted", +) +plt.fill_between( + data.ravel(), + mean_predictions_gpr - std_predictions_gpr, + mean_predictions_gpr + std_predictions_gpr, + color="tab:green", + alpha=0.2, +) +plt.legend(loc="lower right") +plt.xlabel("data") +plt.ylabel("target") +_ = plt.title("Effect of using a radial basis function kernel") -# Generate sample data -X = 15 * rng.rand(100, 1) -y = np.sin(X).ravel() -y += 3 * (0.5 - rng.rand(X.shape[0])) # add noise - -# Fit KernelRidge with parameter selection based on 5-fold cross validation -param_grid = {"alpha": [1e0, 1e-1, 1e-2, 1e-3], - "kernel": [ExpSineSquared(l, p) - for l in np.logspace(-2, 2, 10) - for p in np.logspace(0, 2, 10)]} -kr = GridSearchCV(KernelRidge(), param_grid=param_grid) -stime = time.time() -kr.fit(X, y) -print("Time for KRR fitting: %.3f" % (time.time() - stime)) - -gp_kernel = ExpSineSquared(1.0, 5.0, periodicity_bounds=(1e-2, 1e1)) \ - + WhiteKernel(1e-1) -gpr = GaussianProcessRegressor(kernel=gp_kernel) -stime = time.time() -gpr.fit(X, y) -print("Time for GPR fitting: %.3f" % (time.time() - stime)) - -# Predict using kernel ridge -X_plot = np.linspace(0, 20, 10000)[:, None] -stime = time.time() -y_kr = kr.predict(X_plot) -print("Time for KRR prediction: %.3f" % (time.time() - stime)) - -# Predict using gaussian process regressor -stime = time.time() -y_gpr = gpr.predict(X_plot, return_std=False) -print("Time for GPR prediction: %.3f" % (time.time() - stime)) - -stime = time.time() -y_gpr, y_std = gpr.predict(X_plot, return_std=True) -print("Time for GPR prediction with standard-deviation: %.3f" - % (time.time() - stime)) - -# Plot results -plt.figure(figsize=(10, 5)) -lw = 2 -plt.scatter(X, y, c='k', label='data') -plt.plot(X_plot, np.sin(X_plot), color='navy', lw=lw, label='True') -plt.plot(X_plot, y_kr, color='turquoise', lw=lw, - label='KRR (%s)' % kr.best_params_) -plt.plot(X_plot, y_gpr, color='darkorange', lw=lw, - label='GPR (%s)' % gpr.kernel_) -plt.fill_between(X_plot[:, 0], y_gpr - y_std, y_gpr + y_std, color='darkorange', - alpha=0.2) -plt.xlabel('data') -plt.ylabel('target') -plt.xlim(0, 20) -plt.ylim(-4, 4) -plt.title('GPR versus Kernel Ridge') -plt.legend(loc="best", scatterpoints=1, prop={'size': 8}) -plt.show() +# %% +# The effect of using a radial basis function kernel will attenuate the +# periodicity effect once that no sample are available in the training. +# As testing samples get further away from the training ones, predictions +# are converging towards their mean and their standard deviation +# also increases. diff --git a/examples/gaussian_process/plot_gpc.py b/examples/gaussian_process/plot_gpc.py index 1d00b9f330400..e1d5d03586774 100644 --- a/examples/gaussian_process/plot_gpc.py +++ b/examples/gaussian_process/plot_gpc.py @@ -19,21 +19,18 @@ The second figure shows the log-marginal-likelihood for different choices of the kernel's hyperparameters, highlighting the two choices of the hyperparameters used in the first figure by black dots. + """ -print(__doc__) -# Authors: Jan Hendrik Metzen -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import numpy as np - from matplotlib import pyplot as plt -from sklearn.metrics import accuracy_score, log_loss from sklearn.gaussian_process import GaussianProcessClassifier from sklearn.gaussian_process.kernels import RBF - +from sklearn.metrics import accuracy_score, log_loss # Generate data train_size = 50 @@ -42,37 +39,58 @@ y = np.array(X[:, 0] > 2.5, dtype=int) # Specify Gaussian Processes with fixed and optimized hyperparameters -gp_fix = GaussianProcessClassifier(kernel=1.0 * RBF(length_scale=1.0), - optimizer=None) +gp_fix = GaussianProcessClassifier(kernel=1.0 * RBF(length_scale=1.0), optimizer=None) gp_fix.fit(X[:train_size], y[:train_size]) gp_opt = GaussianProcessClassifier(kernel=1.0 * RBF(length_scale=1.0)) gp_opt.fit(X[:train_size], y[:train_size]) -print("Log Marginal Likelihood (initial): %.3f" - % gp_fix.log_marginal_likelihood(gp_fix.kernel_.theta)) -print("Log Marginal Likelihood (optimized): %.3f" - % gp_opt.log_marginal_likelihood(gp_opt.kernel_.theta)) - -print("Accuracy: %.3f (initial) %.3f (optimized)" - % (accuracy_score(y[:train_size], gp_fix.predict(X[:train_size])), - accuracy_score(y[:train_size], gp_opt.predict(X[:train_size])))) -print("Log-loss: %.3f (initial) %.3f (optimized)" - % (log_loss(y[:train_size], gp_fix.predict_proba(X[:train_size])[:, 1]), - log_loss(y[:train_size], gp_opt.predict_proba(X[:train_size])[:, 1]))) +print( + "Log Marginal Likelihood (initial): %.3f" + % gp_fix.log_marginal_likelihood(gp_fix.kernel_.theta) +) +print( + "Log Marginal Likelihood (optimized): %.3f" + % gp_opt.log_marginal_likelihood(gp_opt.kernel_.theta) +) + +print( + "Accuracy: %.3f (initial) %.3f (optimized)" + % ( + accuracy_score(y[:train_size], gp_fix.predict(X[:train_size])), + accuracy_score(y[:train_size], gp_opt.predict(X[:train_size])), + ) +) +print( + "Log-loss: %.3f (initial) %.3f (optimized)" + % ( + log_loss(y[:train_size], gp_fix.predict_proba(X[:train_size])[:, 1]), + log_loss(y[:train_size], gp_opt.predict_proba(X[:train_size])[:, 1]), + ) +) # Plot posteriors plt.figure() -plt.scatter(X[:train_size, 0], y[:train_size], c='k', label="Train data", - edgecolors=(0, 0, 0)) -plt.scatter(X[train_size:, 0], y[train_size:], c='g', label="Test data", - edgecolors=(0, 0, 0)) +plt.scatter( + X[:train_size, 0], y[:train_size], c="k", label="Train data", edgecolors=(0, 0, 0) +) +plt.scatter( + X[train_size:, 0], y[train_size:], c="g", label="Test data", edgecolors=(0, 0, 0) +) X_ = np.linspace(0, 5, 100) -plt.plot(X_, gp_fix.predict_proba(X_[:, np.newaxis])[:, 1], 'r', - label="Initial kernel: %s" % gp_fix.kernel_) -plt.plot(X_, gp_opt.predict_proba(X_[:, np.newaxis])[:, 1], 'b', - label="Optimized kernel: %s" % gp_opt.kernel_) +plt.plot( + X_, + gp_fix.predict_proba(X_[:, np.newaxis])[:, 1], + "r", + label="Initial kernel: %s" % gp_fix.kernel_, +) +plt.plot( + X_, + gp_opt.predict_proba(X_[:, np.newaxis])[:, 1], + "b", + label="Optimized kernel: %s" % gp_opt.kernel_, +) plt.xlabel("Feature") plt.ylabel("Class 1 probability") plt.xlim(0, 5) @@ -84,13 +102,20 @@ theta0 = np.logspace(0, 8, 30) theta1 = np.logspace(-1, 1, 29) Theta0, Theta1 = np.meshgrid(theta0, theta1) -LML = [[gp_opt.log_marginal_likelihood(np.log([Theta0[i, j], Theta1[i, j]])) - for i in range(Theta0.shape[0])] for j in range(Theta0.shape[1])] +LML = [ + [ + gp_opt.log_marginal_likelihood(np.log([Theta0[i, j], Theta1[i, j]])) + for i in range(Theta0.shape[0]) + ] + for j in range(Theta0.shape[1]) +] LML = np.array(LML).T -plt.plot(np.exp(gp_fix.kernel_.theta)[0], np.exp(gp_fix.kernel_.theta)[1], - 'ko', zorder=10) -plt.plot(np.exp(gp_opt.kernel_.theta)[0], np.exp(gp_opt.kernel_.theta)[1], - 'ko', zorder=10) +plt.plot( + np.exp(gp_fix.kernel_.theta)[0], np.exp(gp_fix.kernel_.theta)[1], "ko", zorder=10 +) +plt.plot( + np.exp(gp_opt.kernel_.theta)[0], np.exp(gp_opt.kernel_.theta)[1], "ko", zorder=10 +) plt.pcolor(Theta0, Theta1, LML) plt.xscale("log") plt.yscale("log") diff --git a/examples/gaussian_process/plot_gpc_iris.py b/examples/gaussian_process/plot_gpc_iris.py index fe1030131709e..a01d9ac081d7e 100644 --- a/examples/gaussian_process/plot_gpc_iris.py +++ b/examples/gaussian_process/plot_gpc_iris.py @@ -7,11 +7,15 @@ and anisotropic RBF kernel on a two-dimensional version for the iris-dataset. The anisotropic RBF kernel obtains slightly higher log-marginal-likelihood by assigning different length-scales to the two feature dimensions. + """ -print(__doc__) -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt +import numpy as np + from sklearn import datasets from sklearn.gaussian_process import GaussianProcessClassifier from sklearn.gaussian_process.kernels import RBF @@ -21,7 +25,7 @@ X = iris.data[:, :2] # we only take the first two features. y = np.array(iris.target, dtype=int) -h = .02 # step size in the mesh +h = 0.02 # step size in the mesh kernel = 1.0 * RBF([1.0]) gpc_rbf_isotropic = GaussianProcessClassifier(kernel=kernel).fit(X, y) @@ -31,8 +35,7 @@ # create a mesh to plot in x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1 y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1 -xx, yy = np.meshgrid(np.arange(x_min, x_max, h), - np.arange(y_min, y_max, h)) +xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h)) titles = ["Isotropic RBF", "Anisotropic RBF"] plt.figure(figsize=(10, 5)) @@ -48,16 +51,16 @@ plt.imshow(Z, extent=(x_min, x_max, y_min, y_max), origin="lower") # Plot also the training points - plt.scatter(X[:, 0], X[:, 1], c=np.array(["r", "g", "b"])[y], - edgecolors=(0, 0, 0)) - plt.xlabel('Sepal length') - plt.ylabel('Sepal width') + plt.scatter(X[:, 0], X[:, 1], c=np.array(["r", "g", "b"])[y], edgecolors=(0, 0, 0)) + plt.xlabel("Sepal length") + plt.ylabel("Sepal width") plt.xlim(xx.min(), xx.max()) plt.ylim(yy.min(), yy.max()) plt.xticks(()) plt.yticks(()) - plt.title("%s, LML: %.3f" % - (titles[i], clf.log_marginal_likelihood(clf.kernel_.theta))) + plt.title( + "%s, LML: %.3f" % (titles[i], clf.log_marginal_likelihood(clf.kernel_.theta)) + ) plt.tight_layout() plt.show() diff --git a/examples/gaussian_process/plot_gpc_isoprobability.py b/examples/gaussian_process/plot_gpc_isoprobability.py index 2e0813120ea12..52a97e7fd2944 100644 --- a/examples/gaussian_process/plot_gpc_isoprobability.py +++ b/examples/gaussian_process/plot_gpc_isoprobability.py @@ -1,6 +1,3 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - """ ================================================================= Iso-probability lines for Gaussian Processes classification (GPC) @@ -8,21 +5,19 @@ A two-dimensional classification example showing iso-probability lines for the predicted probabilities. + """ -print(__doc__) -# Author: Vincent Dubourg -# Adapted to GaussianProcessClassifier: -# Jan Hendrik Metzen -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import numpy as np - -from matplotlib import pyplot as plt from matplotlib import cm +from matplotlib import pyplot as plt from sklearn.gaussian_process import GaussianProcessClassifier -from sklearn.gaussian_process.kernels import DotProduct, ConstantKernel as C +from sklearn.gaussian_process.kernels import ConstantKernel as C +from sklearn.gaussian_process.kernels import DotProduct # A few constants lim = 8 @@ -31,17 +26,22 @@ def g(x): """The function to predict (classification will then consist in predicting whether g(x) <= 0 or not)""" - return 5. - x[:, 1] - .5 * x[:, 0] ** 2. + return 5.0 - x[:, 1] - 0.5 * x[:, 0] ** 2.0 + # Design of experiments -X = np.array([[-4.61611719, -6.00099547], - [4.10469096, 5.32782448], - [0.00000000, -0.50000000], - [-6.17289014, -4.6984743], - [1.3109306, -6.93271427], - [-5.03823144, 3.10584743], - [-2.87600388, 6.74310541], - [5.21301203, 4.26386883]]) +X = np.array( + [ + [-4.61611719, -6.00099547], + [4.10469096, 5.32782448], + [0.00000000, -0.50000000], + [-6.17289014, -4.6984743], + [1.3109306, -6.93271427], + [-5.03823144, 3.10584743], + [-2.87600388, 6.74310541], + [5.21301203, 4.26386883], + ] +) # Observations y = np.array(g(X) > 0, dtype=int) @@ -54,8 +54,7 @@ def g(x): # Evaluate real function and the predicted probability res = 50 -x1, x2 = np.meshgrid(np.linspace(- lim, lim, res), - np.linspace(- lim, lim, res)) +x1, x2 = np.meshgrid(np.linspace(-lim, lim, res), np.linspace(-lim, lim, res)) xx = np.vstack([x1.reshape(x1.size), x2.reshape(x2.size)]).T y_true = g(xx) @@ -66,37 +65,33 @@ def g(x): # Plot the probabilistic classification iso-values fig = plt.figure(1) ax = fig.gca() -ax.axes.set_aspect('equal') +ax.axes.set_aspect("equal") plt.xticks([]) plt.yticks([]) ax.set_xticklabels([]) ax.set_yticklabels([]) -plt.xlabel('$x_1$') -plt.ylabel('$x_2$') - -cax = plt.imshow(y_prob, cmap=cm.gray_r, alpha=0.8, - extent=(-lim, lim, -lim, lim)) -norm = plt.matplotlib.colors.Normalize(vmin=0., vmax=0.9) -cb = plt.colorbar(cax, ticks=[0., 0.2, 0.4, 0.6, 0.8, 1.], norm=norm) -cb.set_label(r'${\rm \mathbb{P}}\left[\widehat{G}(\mathbf{x}) \leq 0\right]$') +plt.xlabel("$x_1$") +plt.ylabel("$x_2$") + +cax = plt.imshow(y_prob, cmap=cm.gray_r, alpha=0.8, extent=(-lim, lim, -lim, lim)) +norm = plt.matplotlib.colors.Normalize(vmin=0.0, vmax=0.9) +cb = plt.colorbar(cax, ticks=[0.0, 0.2, 0.4, 0.6, 0.8, 1.0], norm=norm) +cb.set_label(r"${\rm \mathbb{P}}\left[\widehat{G}(\mathbf{x}) \leq 0\right]$") plt.clim(0, 1) -plt.plot(X[y <= 0, 0], X[y <= 0, 1], 'r.', markersize=12) +plt.plot(X[y <= 0, 0], X[y <= 0, 1], "r.", markersize=12) -plt.plot(X[y > 0, 0], X[y > 0, 1], 'b.', markersize=12) +plt.plot(X[y > 0, 0], X[y > 0, 1], "b.", markersize=12) -plt.contour(x1, x2, y_true, [0.], colors='k', linestyles='dashdot') +plt.contour(x1, x2, y_true, [0.0], colors="k", linestyles="dashdot") -cs = plt.contour(x1, x2, y_prob, [0.666], colors='b', - linestyles='solid') +cs = plt.contour(x1, x2, y_prob, [0.666], colors="b", linestyles="solid") plt.clabel(cs, fontsize=11) -cs = plt.contour(x1, x2, y_prob, [0.5], colors='k', - linestyles='dashed') +cs = plt.contour(x1, x2, y_prob, [0.5], colors="k", linestyles="dashed") plt.clabel(cs, fontsize=11) -cs = plt.contour(x1, x2, y_prob, [0.334], colors='r', - linestyles='solid') +cs = plt.contour(x1, x2, y_prob, [0.334], colors="r", linestyles="solid") plt.clabel(cs, fontsize=11) plt.show() diff --git a/examples/gaussian_process/plot_gpc_xor.py b/examples/gaussian_process/plot_gpc_xor.py index 04f014e13e8ae..012b8f98ad337 100644 --- a/examples/gaussian_process/plot_gpc_xor.py +++ b/examples/gaussian_process/plot_gpc_xor.py @@ -8,29 +8,26 @@ dataset, the DotProduct kernel obtains considerably better results because the class-boundaries are linear and coincide with the coordinate axes. In general, stationary kernels often obtain better results. + """ -print(__doc__) -# Authors: Jan Hendrik Metzen -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt +import numpy as np from sklearn.gaussian_process import GaussianProcessClassifier from sklearn.gaussian_process.kernels import RBF, DotProduct - -xx, yy = np.meshgrid(np.linspace(-3, 3, 50), - np.linspace(-3, 3, 50)) +xx, yy = np.meshgrid(np.linspace(-3, 3, 50), np.linspace(-3, 3, 50)) rng = np.random.RandomState(0) X = rng.randn(200, 2) Y = np.logical_xor(X[:, 0] > 0, X[:, 1] > 0) # fit the model plt.figure(figsize=(10, 5)) -kernels = [1.0 * RBF(length_scale=1.0), 1.0 * DotProduct(sigma_0=1.0)**2] +kernels = [1.0 * RBF(length_scale=1.15), 1.0 * DotProduct(sigma_0=1.0) ** 2] for i, kernel in enumerate(kernels): clf = GaussianProcessClassifier(kernel=kernel, warm_start=True).fit(X, Y) @@ -39,20 +36,25 @@ Z = Z.reshape(xx.shape) plt.subplot(1, 2, i + 1) - image = plt.imshow(Z, interpolation='nearest', - extent=(xx.min(), xx.max(), yy.min(), yy.max()), - aspect='auto', origin='lower', cmap=plt.cm.PuOr_r) - contours = plt.contour(xx, yy, Z, levels=[0.5], linewidths=2, - colors=['k']) - plt.scatter(X[:, 0], X[:, 1], s=30, c=Y, cmap=plt.cm.Paired, - edgecolors=(0, 0, 0)) + image = plt.imshow( + Z, + interpolation="nearest", + extent=(xx.min(), xx.max(), yy.min(), yy.max()), + aspect="auto", + origin="lower", + cmap=plt.cm.PuOr_r, + ) + contours = plt.contour(xx, yy, Z, levels=[0.5], linewidths=2, colors=["k"]) + plt.scatter(X[:, 0], X[:, 1], s=30, c=Y, cmap=plt.cm.Paired, edgecolors=(0, 0, 0)) plt.xticks(()) plt.yticks(()) plt.axis([-3, 3, -3, 3]) plt.colorbar(image) - plt.title("%s\n Log-Marginal-Likelihood:%.3f" - % (clf.kernel_, clf.log_marginal_likelihood(clf.kernel_.theta)), - fontsize=12) + plt.title( + "%s\n Log-Marginal-Likelihood:%.3f" + % (clf.kernel_, clf.log_marginal_likelihood(clf.kernel_.theta)), + fontsize=12, + ) plt.tight_layout() plt.show() diff --git a/examples/gaussian_process/plot_gpr_co2.py b/examples/gaussian_process/plot_gpr_co2.py index 2cc751438cbd4..ae3d96aebc17f 100644 --- a/examples/gaussian_process/plot_gpr_co2.py +++ b/examples/gaussian_process/plot_gpr_co2.py @@ -1,156 +1,228 @@ """ -======================================================== -Gaussian process regression (GPR) on Mauna Loa CO2 data. -======================================================== +==================================================================================== +Forecasting of CO2 level on Mona Loa dataset using Gaussian process regression (GPR) +==================================================================================== This example is based on Section 5.4.3 of "Gaussian Processes for Machine -Learning" [RW2006]. It illustrates an example of complex kernel engineering and -hyperparameter optimization using gradient ascent on the +Learning" [1]_. It illustrates an example of complex kernel engineering +and hyperparameter optimization using gradient ascent on the log-marginal-likelihood. The data consists of the monthly average atmospheric -CO2 concentrations (in parts per million by volume (ppmv)) collected at the +CO2 concentrations (in parts per million by volume (ppm)) collected at the Mauna Loa Observatory in Hawaii, between 1958 and 2001. The objective is to -model the CO2 concentration as a function of the time t. - -The kernel is composed of several terms that are responsible for explaining -different properties of the signal: - -- a long term, smooth rising trend is to be explained by an RBF kernel. The - RBF kernel with a large length-scale enforces this component to be smooth; - it is not enforced that the trend is rising which leaves this choice to the - GP. The specific length-scale and the amplitude are free hyperparameters. - -- a seasonal component, which is to be explained by the periodic - ExpSineSquared kernel with a fixed periodicity of 1 year. The length-scale - of this periodic component, controlling its smoothness, is a free parameter. - In order to allow decaying away from exact periodicity, the product with an - RBF kernel is taken. The length-scale of this RBF component controls the - decay time and is a further free parameter. - -- smaller, medium term irregularities are to be explained by a - RationalQuadratic kernel component, whose length-scale and alpha parameter, - which determines the diffuseness of the length-scales, are to be determined. - According to [RW2006], these irregularities can better be explained by - a RationalQuadratic than an RBF kernel component, probably because it can - accommodate several length-scales. - -- a "noise" term, consisting of an RBF kernel contribution, which shall - explain the correlated noise components such as local weather phenomena, - and a WhiteKernel contribution for the white noise. The relative amplitudes - and the RBF's length scale are further free parameters. - -Maximizing the log-marginal-likelihood after subtracting the target's mean -yields the following kernel with an LML of -83.214:: - - 34.4**2 * RBF(length_scale=41.8) - + 3.27**2 * RBF(length_scale=180) * ExpSineSquared(length_scale=1.44, - periodicity=1) - + 0.446**2 * RationalQuadratic(alpha=17.7, length_scale=0.957) - + 0.197**2 * RBF(length_scale=0.138) + WhiteKernel(noise_level=0.0336) - -Thus, most of the target signal (34.4ppm) is explained by a long-term rising -trend (length-scale 41.8 years). The periodic component has an amplitude of -3.27ppm, a decay time of 180 years and a length-scale of 1.44. The long decay -time indicates that we have a locally very close to periodic seasonal -component. The correlated noise has an amplitude of 0.197ppm with a length -scale of 0.138 years and a white-noise contribution of 0.197ppm. Thus, the -overall noise level is very small, indicating that the data can be very well -explained by the model. The figure shows also that the model makes very -confident predictions until around 2015. +model the CO2 concentration as a function of the time :math:`t` and extrapolate +for years after 2001. + +.. rubric:: References + +.. [1] `Rasmussen, Carl Edward. "Gaussian processes in machine learning." + Summer school on machine learning. Springer, Berlin, Heidelberg, 2003 + `_. """ -# Authors: Jan Hendrik Metzen -# -# License: BSD 3 clause +print(__doc__) -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -from matplotlib import pyplot as plt +# %% +# Build the dataset +# ----------------- +# +# We will derive a dataset from the Mauna Loa Observatory that collected air +# samples. We are interested in estimating the concentration of CO2 and +# extrapolate it for further year. First, we load the original dataset available +# in OpenML as a pandas dataframe. This will be replaced with Polars +# once `fetch_openml` adds a native support for it. from sklearn.datasets import fetch_openml + +co2 = fetch_openml(data_id=41187, as_frame=True) +co2.frame.head() + +# %% +# First, we process the original dataframe to create a date column and select +# it along with the CO2 column. +import polars as pl + +co2_data = pl.DataFrame(co2.frame[["year", "month", "day", "co2"]]).select( + pl.date("year", "month", "day"), "co2" +) +co2_data.head() + +# %% +co2_data["date"].min(), co2_data["date"].max() + +# %% +# We see that we get CO2 concentration for some days from March, 1958 to +# December, 2001. We can plot these raw information to have a better +# understanding. +import matplotlib.pyplot as plt + +plt.plot(co2_data["date"], co2_data["co2"]) +plt.xlabel("date") +plt.ylabel("CO$_2$ concentration (ppm)") +_ = plt.title("Raw air samples measurements from the Mauna Loa Observatory") + +# %% +# We will preprocess the dataset by taking a monthly average and drop month +# for which no measurements were collected. Such a processing will have an +# smoothing effect on the data. + +co2_data = ( + co2_data.sort(by="date") + .group_by_dynamic("date", every="1mo") + .agg(pl.col("co2").mean()) + .drop_nulls() +) +plt.plot(co2_data["date"], co2_data["co2"]) +plt.xlabel("date") +plt.ylabel("Monthly average of CO$_2$ concentration (ppm)") +_ = plt.title( + "Monthly average of air samples measurements\nfrom the Mauna Loa Observatory" +) + +# %% +# The idea in this example will be to predict the CO2 concentration in function +# of the date. We are as well interested in extrapolating for upcoming year +# after 2001. +# +# As a first step, we will divide the data and the target to estimate. The data +# being a date, we will convert it into a numeric. +X = co2_data.select( + pl.col("date").dt.year() + pl.col("date").dt.month() / 12 +).to_numpy() +y = co2_data["co2"].to_numpy() + +# %% +# Design the proper kernel +# ------------------------ +# +# To design the kernel to use with our Gaussian process, we can make some +# assumption regarding the data at hand. We observe that they have several +# characteristics: we see a long term rising trend, a pronounced seasonal +# variation and some smaller irregularities. We can use different appropriate +# kernel that would capture these features. +# +# First, the long term rising trend could be fitted using a radial basis +# function (RBF) kernel with a large length-scale parameter. The RBF kernel +# with a large length-scale enforces this component to be smooth. An trending +# increase is not enforced as to give a degree of freedom to our model. The +# specific length-scale and the amplitude are free hyperparameters. +from sklearn.gaussian_process.kernels import RBF + +long_term_trend_kernel = 50.0**2 * RBF(length_scale=50.0) + +# %% +# The seasonal variation is explained by the periodic exponential sine squared +# kernel with a fixed periodicity of 1 year. The length-scale of this periodic +# component, controlling its smoothness, is a free parameter. In order to allow +# decaying away from exact periodicity, the product with an RBF kernel is +# taken. The length-scale of this RBF component controls the decay time and is +# a further free parameter. This type of kernel is also known as locally +# periodic kernel. +from sklearn.gaussian_process.kernels import ExpSineSquared + +seasonal_kernel = ( + 2.0**2 + * RBF(length_scale=100.0) + * ExpSineSquared(length_scale=1.0, periodicity=1.0, periodicity_bounds="fixed") +) + +# %% +# The small irregularities are to be explained by a rational quadratic kernel +# component, whose length-scale and alpha parameter, which quantifies the +# diffuseness of the length-scales, are to be determined. A rational quadratic +# kernel is equivalent to an RBF kernel with several length-scale and will +# better accommodate the different irregularities. +from sklearn.gaussian_process.kernels import RationalQuadratic + +irregularities_kernel = 0.5**2 * RationalQuadratic(length_scale=1.0, alpha=1.0) + +# %% +# Finally, the noise in the dataset can be accounted with a kernel consisting +# of an RBF kernel contribution, which shall explain the correlated noise +# components such as local weather phenomena, and a white kernel contribution +# for the white noise. The relative amplitudes and the RBF's length scale are +# further free parameters. +from sklearn.gaussian_process.kernels import WhiteKernel + +noise_kernel = 0.1**2 * RBF(length_scale=0.1) + WhiteKernel( + noise_level=0.1**2, noise_level_bounds=(1e-5, 1e5) +) + +# %% +# Thus, our final kernel is an addition of all previous kernel. +co2_kernel = ( + long_term_trend_kernel + seasonal_kernel + irregularities_kernel + noise_kernel +) +co2_kernel + +# %% +# Model fitting and extrapolation +# ------------------------------- +# +# Now, we are ready to use a Gaussian process regressor and fit the available +# data. To follow the example from the literature, we will subtract the mean +# from the target. We could have used `normalize_y=True`. However, doing so +# would have also scaled the target (dividing `y` by its standard deviation). +# Thus, the hyperparameters of the different kernel would have had different +# meaning since they would not have been expressed in ppm. from sklearn.gaussian_process import GaussianProcessRegressor -from sklearn.gaussian_process.kernels \ - import RBF, WhiteKernel, RationalQuadratic, ExpSineSquared -print(__doc__) +y_mean = y.mean() +gaussian_process = GaussianProcessRegressor(kernel=co2_kernel, normalize_y=False) +gaussian_process.fit(X, y - y_mean) +# %% +# Now, we will use the Gaussian process to predict on: +# +# - training data to inspect the goodness of fit; +# - future data to see the extrapolation done by the model. +# +# Thus, we create synthetic data from 1958 to the current month. In addition, +# we need to add the subtracted mean computed during training. +import datetime -def load_mauna_loa_atmospheric_co2(): - ml_data = fetch_openml(data_id=41187, as_frame=False) - months = [] - ppmv_sums = [] - counts = [] - - y = ml_data.data[:, 0] - m = ml_data.data[:, 1] - month_float = y + (m - 1) / 12 - ppmvs = ml_data.target - - for month, ppmv in zip(month_float, ppmvs): - if not months or month != months[-1]: - months.append(month) - ppmv_sums.append(ppmv) - counts.append(1) - else: - # aggregate monthly sum to produce average - ppmv_sums[-1] += ppmv - counts[-1] += 1 - - months = np.asarray(months).reshape(-1, 1) - avg_ppmvs = np.asarray(ppmv_sums) / counts - return months, avg_ppmvs - - -X, y = load_mauna_loa_atmospheric_co2() - -# Kernel with parameters given in GPML book -k1 = 66.0**2 * RBF(length_scale=67.0) # long term smooth rising trend -k2 = 2.4**2 * RBF(length_scale=90.0) \ - * ExpSineSquared(length_scale=1.3, periodicity=1.0) # seasonal component -# medium term irregularity -k3 = 0.66**2 \ - * RationalQuadratic(length_scale=1.2, alpha=0.78) -k4 = 0.18**2 * RBF(length_scale=0.134) \ - + WhiteKernel(noise_level=0.19**2) # noise terms -kernel_gpml = k1 + k2 + k3 + k4 - -gp = GaussianProcessRegressor(kernel=kernel_gpml, alpha=0, - optimizer=None, normalize_y=True) -gp.fit(X, y) - -print("GPML kernel: %s" % gp.kernel_) -print("Log-marginal-likelihood: %.3f" - % gp.log_marginal_likelihood(gp.kernel_.theta)) - -# Kernel with optimized parameters -k1 = 50.0**2 * RBF(length_scale=50.0) # long term smooth rising trend -k2 = 2.0**2 * RBF(length_scale=100.0) \ - * ExpSineSquared(length_scale=1.0, periodicity=1.0, - periodicity_bounds="fixed") # seasonal component -# medium term irregularities -k3 = 0.5**2 * RationalQuadratic(length_scale=1.0, alpha=1.0) -k4 = 0.1**2 * RBF(length_scale=0.1) \ - + WhiteKernel(noise_level=0.1**2, - noise_level_bounds=(1e-5, np.inf)) # noise terms -kernel = k1 + k2 + k3 + k4 - -gp = GaussianProcessRegressor(kernel=kernel, alpha=0, - normalize_y=True) -gp.fit(X, y) - -print("\nLearned kernel: %s" % gp.kernel_) -print("Log-marginal-likelihood: %.3f" - % gp.log_marginal_likelihood(gp.kernel_.theta)) - -X_ = np.linspace(X.min(), X.max() + 30, 1000)[:, np.newaxis] -y_pred, y_std = gp.predict(X_, return_std=True) - -# Illustration -plt.scatter(X, y, c='k') -plt.plot(X_, y_pred) -plt.fill_between(X_[:, 0], y_pred - y_std, y_pred + y_std, - alpha=0.5, color='k') -plt.xlim(X_.min(), X_.max()) +import numpy as np + +today = datetime.datetime.now() +current_month = today.year + today.month / 12 +X_test = np.linspace(start=1958, stop=current_month, num=1_000).reshape(-1, 1) +mean_y_pred, std_y_pred = gaussian_process.predict(X_test, return_std=True) +mean_y_pred += y_mean + +# %% +plt.plot(X, y, color="black", linestyle="dashed", label="Measurements") +plt.plot(X_test, mean_y_pred, color="tab:blue", alpha=0.4, label="Gaussian process") +plt.fill_between( + X_test.ravel(), + mean_y_pred - std_y_pred, + mean_y_pred + std_y_pred, + color="tab:blue", + alpha=0.2, +) +plt.legend() plt.xlabel("Year") -plt.ylabel(r"CO$_2$ in ppm") -plt.title(r"Atmospheric CO$_2$ concentration at Mauna Loa") -plt.tight_layout() -plt.show() +plt.ylabel("Monthly average of CO$_2$ concentration (ppm)") +_ = plt.title( + "Monthly average of air samples measurements\nfrom the Mauna Loa Observatory" +) + +# %% +# Our fitted model is capable to fit previous data properly and extrapolate to +# future year with confidence. +# +# Interpretation of kernel hyperparameters +# ---------------------------------------- +# +# Now, we can have a look at the hyperparameters of the kernel. +gaussian_process.kernel_ + +# %% +# Thus, most of the target signal, with the mean subtracted, is explained by a +# long-term rising trend for ~45 ppm and a length-scale of ~52 years. The +# periodic component has an amplitude of ~2.6ppm, a decay time of ~90 years and +# a length-scale of ~1.5. The long decay time indicates that we have a +# component very close to a seasonal periodicity. The correlated noise has an +# amplitude of ~0.2 ppm with a length scale of ~0.12 years and a white-noise +# contribution of ~0.04 ppm. Thus, the overall noise level is very small, +# indicating that the data can be very well explained by the model. diff --git a/examples/gaussian_process/plot_gpr_noisy.py b/examples/gaussian_process/plot_gpr_noisy.py index 5f8ce2cd0fe96..8aa01a70fc64a 100644 --- a/examples/gaussian_process/plot_gpr_noisy.py +++ b/examples/gaussian_process/plot_gpr_noisy.py @@ -1,97 +1,227 @@ """ -============================================================= -Gaussian process regression (GPR) with noise-level estimation -============================================================= - -This example illustrates that GPR with a sum-kernel including a WhiteKernel can -estimate the noise level of data. An illustration of the -log-marginal-likelihood (LML) landscape shows that there exist two local -maxima of LML. The first corresponds to a model with a high noise level and a -large length scale, which explains all variations in the data by noise. The -second one has a smaller noise level and shorter length scale, which explains -most of the variation by the noise-free functional relationship. The second -model has a higher likelihood; however, depending on the initial value for the -hyperparameters, the gradient-based optimization might also converge to the -high-noise solution. It is thus important to repeat the optimization several -times for different initializations. +========================================================================= +Ability of Gaussian process regression (GPR) to estimate data noise-level +========================================================================= + +This example shows the ability of the +:class:`~sklearn.gaussian_process.kernels.WhiteKernel` to estimate the noise +level in the data. Moreover, we show the importance of kernel hyperparameters +initialization. """ -print(__doc__) -# Authors: Jan Hendrik Metzen -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +# %% +# Data generation +# --------------- +# +# We will work in a setting where `X` will contain a single feature. We create a +# function that will generate the target to be predicted. We will add an +# option to add some noise to the generated target. import numpy as np -from matplotlib import pyplot as plt -from matplotlib.colors import LogNorm +def target_generator(X, add_noise=False): + target = 0.5 + np.sin(3 * X) + if add_noise: + rng = np.random.RandomState(1) + target += rng.normal(0, 0.3, size=target.shape) + return target.squeeze() + + +# %% +# Let's have a look to the target generator where we will not add any noise to +# observe the signal that we would like to predict. +X = np.linspace(0, 5, num=80).reshape(-1, 1) +y = target_generator(X, add_noise=False) + +# %% +import matplotlib.pyplot as plt + +plt.plot(X, y, label="Expected signal") +plt.legend() +plt.xlabel("X") +_ = plt.ylabel("y") + +# %% +# The target is transforming the input `X` using a sine function. Now, we will +# generate few noisy training samples. To illustrate the noise level, we will +# plot the true signal together with the noisy training samples. +rng = np.random.RandomState(0) +X_train = rng.uniform(0, 5, size=20).reshape(-1, 1) +y_train = target_generator(X_train, add_noise=True) + +# %% +plt.plot(X, y, label="Expected signal") +plt.scatter( + x=X_train[:, 0], + y=y_train, + color="black", + alpha=0.4, + label="Observations", +) +plt.legend() +plt.xlabel("X") +_ = plt.ylabel("y") + +# %% +# Optimisation of kernel hyperparameters in GPR +# --------------------------------------------- +# +# Now, we will create a +# :class:`~sklearn.gaussian_process.GaussianProcessRegressor` +# using an additive kernel adding a +# :class:`~sklearn.gaussian_process.kernels.RBF` and +# :class:`~sklearn.gaussian_process.kernels.WhiteKernel` kernels. +# The :class:`~sklearn.gaussian_process.kernels.WhiteKernel` is a kernel that +# will able to estimate the amount of noise present in the data while the +# :class:`~sklearn.gaussian_process.kernels.RBF` will serve at fitting the +# non-linearity between the data and the target. +# +# However, we will show that the hyperparameter space contains several local +# minima. It will highlights the importance of initial hyperparameter values. +# +# We will create a model using a kernel with a high noise level and a large +# length scale, which will explain all variations in the data by noise. from sklearn.gaussian_process import GaussianProcessRegressor from sklearn.gaussian_process.kernels import RBF, WhiteKernel +kernel = 1.0 * RBF(length_scale=1e1, length_scale_bounds=(1e-2, 1e3)) + WhiteKernel( + noise_level=1, noise_level_bounds=(1e-10, 1e1) +) +gpr = GaussianProcessRegressor(kernel=kernel, alpha=0.0) +gpr.fit(X_train, y_train) +y_mean, y_std = gpr.predict(X, return_std=True) -rng = np.random.RandomState(0) -X = rng.uniform(0, 5, 20)[:, np.newaxis] -y = 0.5 * np.sin(3 * X[:, 0]) + rng.normal(0, 0.5, X.shape[0]) - -# First run -plt.figure() -kernel = 1.0 * RBF(length_scale=100.0, length_scale_bounds=(1e-2, 1e3)) \ - + WhiteKernel(noise_level=1, noise_level_bounds=(1e-10, 1e+1)) -gp = GaussianProcessRegressor(kernel=kernel, - alpha=0.0).fit(X, y) -X_ = np.linspace(0, 5, 100) -y_mean, y_cov = gp.predict(X_[:, np.newaxis], return_cov=True) -plt.plot(X_, y_mean, 'k', lw=3, zorder=9) -plt.fill_between(X_, y_mean - np.sqrt(np.diag(y_cov)), - y_mean + np.sqrt(np.diag(y_cov)), - alpha=0.5, color='k') -plt.plot(X_, 0.5*np.sin(3*X_), 'r', lw=3, zorder=9) -plt.scatter(X[:, 0], y, c='r', s=50, zorder=10, edgecolors=(0, 0, 0)) -plt.title("Initial: %s\nOptimum: %s\nLog-Marginal-Likelihood: %s" - % (kernel, gp.kernel_, - gp.log_marginal_likelihood(gp.kernel_.theta))) -plt.tight_layout() - -# Second run -plt.figure() -kernel = 1.0 * RBF(length_scale=1.0, length_scale_bounds=(1e-2, 1e3)) \ - + WhiteKernel(noise_level=1e-5, noise_level_bounds=(1e-10, 1e+1)) -gp = GaussianProcessRegressor(kernel=kernel, - alpha=0.0).fit(X, y) -X_ = np.linspace(0, 5, 100) -y_mean, y_cov = gp.predict(X_[:, np.newaxis], return_cov=True) -plt.plot(X_, y_mean, 'k', lw=3, zorder=9) -plt.fill_between(X_, y_mean - np.sqrt(np.diag(y_cov)), - y_mean + np.sqrt(np.diag(y_cov)), - alpha=0.5, color='k') -plt.plot(X_, 0.5*np.sin(3*X_), 'r', lw=3, zorder=9) -plt.scatter(X[:, 0], y, c='r', s=50, zorder=10, edgecolors=(0, 0, 0)) -plt.title("Initial: %s\nOptimum: %s\nLog-Marginal-Likelihood: %s" - % (kernel, gp.kernel_, - gp.log_marginal_likelihood(gp.kernel_.theta))) -plt.tight_layout() - -# Plot LML landscape -plt.figure() -theta0 = np.logspace(-2, 3, 49) -theta1 = np.logspace(-2, 0, 50) -Theta0, Theta1 = np.meshgrid(theta0, theta1) -LML = [[gp.log_marginal_likelihood(np.log([0.36, Theta0[i, j], Theta1[i, j]])) - for i in range(Theta0.shape[0])] for j in range(Theta0.shape[1])] -LML = np.array(LML).T - -vmin, vmax = (-LML).min(), (-LML).max() -vmax = 50 -level = np.around(np.logspace(np.log10(vmin), np.log10(vmax), 50), decimals=1) -plt.contour(Theta0, Theta1, -LML, - levels=level, norm=LogNorm(vmin=vmin, vmax=vmax)) +# %% +plt.plot(X, y, label="Expected signal") +plt.scatter(x=X_train[:, 0], y=y_train, color="black", alpha=0.4, label="Observations") +plt.errorbar(X, y_mean, y_std, label="Posterior mean Âą std") +plt.legend() +plt.xlabel("X") +plt.ylabel("y") +_ = plt.title( + ( + f"Initial: {kernel}\nOptimum: {gpr.kernel_}\nLog-Marginal-Likelihood: " + f"{gpr.log_marginal_likelihood(gpr.kernel_.theta)}" + ), + fontsize=8, +) +# %% +# We see that the optimum kernel found still has a high noise level and an even +# larger length scale. The length scale reaches the maximum bound that we +# allowed for this parameter and we got a warning as a result. +# +# More importantly, we observe that the model does not provide useful +# predictions: the mean prediction seems to be constant: it does not follow the +# expected noise-free signal. +# +# Now, we will initialize the :class:`~sklearn.gaussian_process.kernels.RBF` +# with a larger `length_scale` initial value and the +# :class:`~sklearn.gaussian_process.kernels.WhiteKernel` with a smaller initial +# noise level lower while keeping the parameter bounds unchanged. +kernel = 1.0 * RBF(length_scale=1e-1, length_scale_bounds=(1e-2, 1e3)) + WhiteKernel( + noise_level=1e-2, noise_level_bounds=(1e-10, 1e1) +) +gpr = GaussianProcessRegressor(kernel=kernel, alpha=0.0) +gpr.fit(X_train, y_train) +y_mean, y_std = gpr.predict(X, return_std=True) + +# %% +plt.plot(X, y, label="Expected signal") +plt.scatter(x=X_train[:, 0], y=y_train, color="black", alpha=0.4, label="Observations") +plt.errorbar(X, y_mean, y_std, label="Posterior mean Âą std") +plt.legend() +plt.xlabel("X") +plt.ylabel("y") +_ = plt.title( + ( + f"Initial: {kernel}\nOptimum: {gpr.kernel_}\nLog-Marginal-Likelihood: " + f"{gpr.log_marginal_likelihood(gpr.kernel_.theta)}" + ), + fontsize=8, +) + +# %% +# First, we see that the model's predictions are more precise than the +# previous model's: this new model is able to estimate the noise-free +# functional relationship. +# +# Looking at the kernel hyperparameters, we see that the best combination found +# has a smaller noise level and shorter length scale than the first model. +# +# We can inspect the negative Log-Marginal-Likelihood (LML) of +# :class:`~sklearn.gaussian_process.GaussianProcessRegressor` +# for different hyperparameters to get a sense of the local minima. +from matplotlib.colors import LogNorm + +length_scale = np.logspace(-2, 4, num=80) +noise_level = np.logspace(-2, 1, num=80) +length_scale_grid, noise_level_grid = np.meshgrid(length_scale, noise_level) + +log_marginal_likelihood = [ + gpr.log_marginal_likelihood(theta=np.log([0.36, scale, noise])) + for scale, noise in zip(length_scale_grid.ravel(), noise_level_grid.ravel()) +] +log_marginal_likelihood = np.reshape(log_marginal_likelihood, noise_level_grid.shape) + +# %% +vmin, vmax = (-log_marginal_likelihood).min(), 50 +level = np.around(np.logspace(np.log10(vmin), np.log10(vmax), num=20), decimals=1) +plt.contour( + length_scale_grid, + noise_level_grid, + -log_marginal_likelihood, + levels=level, + norm=LogNorm(vmin=vmin, vmax=vmax), +) plt.colorbar() plt.xscale("log") plt.yscale("log") plt.xlabel("Length-scale") plt.ylabel("Noise-level") -plt.title("Log-marginal-likelihood") -plt.tight_layout() - +plt.title("Negative log-marginal-likelihood") plt.show() + +# %% +# +# We see that there are two local minima that correspond to the combination of +# hyperparameters previously found. Depending on the initial values for the +# hyperparameters, the gradient-based optimization might or might not +# converge to the best model. It is thus important to repeat the optimization +# several times for different initializations. This can be done by setting the +# `n_restarts_optimizer` parameter of the +# :class:`~sklearn.gaussian_process.GaussianProcessRegressor` class. +# +# Let's try again to fit our model with the bad initial values but this time +# with 10 random restarts. + +kernel = 1.0 * RBF(length_scale=1e1, length_scale_bounds=(1e-2, 1e3)) + WhiteKernel( + noise_level=1, noise_level_bounds=(1e-10, 1e1) +) +gpr = GaussianProcessRegressor( + kernel=kernel, alpha=0.0, n_restarts_optimizer=10, random_state=0 +) +gpr.fit(X_train, y_train) +y_mean, y_std = gpr.predict(X, return_std=True) + +# %% +plt.plot(X, y, label="Expected signal") +plt.scatter(x=X_train[:, 0], y=y_train, color="black", alpha=0.4, label="Observations") +plt.errorbar(X, y_mean, y_std, label="Posterior mean Âą std") +plt.legend() +plt.xlabel("X") +plt.ylabel("y") +_ = plt.title( + ( + f"Initial: {kernel}\nOptimum: {gpr.kernel_}\nLog-Marginal-Likelihood: " + f"{gpr.log_marginal_likelihood(gpr.kernel_.theta)}" + ), + fontsize=8, +) + +# %% +# +# As we hoped, random restarts allow the optimization to find the best set +# of hyperparameters despite the bad initial values. diff --git a/examples/gaussian_process/plot_gpr_noisy_targets.py b/examples/gaussian_process/plot_gpr_noisy_targets.py index 50992c19337b3..48160f48f6f32 100644 --- a/examples/gaussian_process/plot_gpr_noisy_targets.py +++ b/examples/gaussian_process/plot_gpr_noisy_targets.py @@ -11,103 +11,140 @@ In both cases, the kernel's parameters are estimated using the maximum likelihood principle. -The figures illustrate the interpolating property of the Gaussian Process -model as well as its probabilistic nature in the form of a pointwise 95% -confidence interval. +The figures illustrate the interpolating property of the Gaussian Process model +as well as its probabilistic nature in the form of a pointwise 95% confidence +interval. -Note that the parameter ``alpha`` is applied as a Tikhonov -regularization of the assumed covariance between the training points. +Note that `alpha` is a parameter to control the strength of the Tikhonov +regularization on the assumed training points' covariance matrix. """ -print(__doc__) -# Author: Vincent Dubourg -# Jake Vanderplas -# Jan Hendrik Metzen s -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +# %% +# Dataset generation +# ------------------ +# +# We will start by generating a synthetic dataset. The true generative process +# is defined as :math:`f(x) = x \sin(x)`. import numpy as np -from matplotlib import pyplot as plt +X = np.linspace(start=0, stop=10, num=1_000).reshape(-1, 1) +y = np.squeeze(X * np.sin(X)) + +# %% +import matplotlib.pyplot as plt + +plt.plot(X, y, label=r"$f(x) = x \sin(x)$", linestyle="dotted") +plt.legend() +plt.xlabel("$x$") +plt.ylabel("$f(x)$") +_ = plt.title("True generative process") + +# %% +# We will use this dataset in the next experiment to illustrate how Gaussian +# Process regression is working. +# +# Example with noise-free target +# ------------------------------ +# +# In this first example, we will use the true generative process without +# adding any noise. For training the Gaussian Process regression, we will only +# select few samples. +rng = np.random.RandomState(1) +training_indices = rng.choice(np.arange(y.size), size=6, replace=False) +X_train, y_train = X[training_indices], y[training_indices] + +# %% +# Now, we fit a Gaussian process on these few training data samples. We will +# use a radial basis function (RBF) kernel and a constant parameter to fit the +# amplitude. from sklearn.gaussian_process import GaussianProcessRegressor -from sklearn.gaussian_process.kernels import RBF, ConstantKernel as C - -np.random.seed(1) - - -def f(x): - """The function to predict.""" - return x * np.sin(x) - -# ---------------------------------------------------------------------- -# First the noiseless case -X = np.atleast_2d([1., 3., 5., 6., 7., 8.]).T - -# Observations -y = f(X).ravel() - -# Mesh the input space for evaluations of the real function, the prediction and -# its MSE -x = np.atleast_2d(np.linspace(0, 10, 1000)).T - -# Instantiate a Gaussian Process model -kernel = C(1.0, (1e-3, 1e3)) * RBF(10, (1e-2, 1e2)) -gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=9) - -# Fit to data using Maximum Likelihood Estimation of the parameters -gp.fit(X, y) - -# Make the prediction on the meshed x-axis (ask for MSE as well) -y_pred, sigma = gp.predict(x, return_std=True) - -# Plot the function, the prediction and the 95% confidence interval based on -# the MSE -plt.figure() -plt.plot(x, f(x), 'r:', label=r'$f(x) = x\,\sin(x)$') -plt.plot(X, y, 'r.', markersize=10, label='Observations') -plt.plot(x, y_pred, 'b-', label='Prediction') -plt.fill(np.concatenate([x, x[::-1]]), - np.concatenate([y_pred - 1.9600 * sigma, - (y_pred + 1.9600 * sigma)[::-1]]), - alpha=.5, fc='b', ec='None', label='95% confidence interval') -plt.xlabel('$x$') -plt.ylabel('$f(x)$') -plt.ylim(-10, 20) -plt.legend(loc='upper left') - -# ---------------------------------------------------------------------- -# now the noisy case -X = np.linspace(0.1, 9.9, 20) -X = np.atleast_2d(X).T - -# Observations and noise -y = f(X).ravel() -dy = 0.5 + 1.0 * np.random.random(y.shape) -noise = np.random.normal(0, dy) -y += noise - -# Instantiate a Gaussian Process model -gp = GaussianProcessRegressor(kernel=kernel, alpha=dy ** 2, - n_restarts_optimizer=10) - -# Fit to data using Maximum Likelihood Estimation of the parameters -gp.fit(X, y) - -# Make the prediction on the meshed x-axis (ask for MSE as well) -y_pred, sigma = gp.predict(x, return_std=True) - -# Plot the function, the prediction and the 95% confidence interval based on -# the MSE -plt.figure() -plt.plot(x, f(x), 'r:', label=r'$f(x) = x\,\sin(x)$') -plt.errorbar(X.ravel(), y, dy, fmt='r.', markersize=10, label='Observations') -plt.plot(x, y_pred, 'b-', label='Prediction') -plt.fill(np.concatenate([x, x[::-1]]), - np.concatenate([y_pred - 1.9600 * sigma, - (y_pred + 1.9600 * sigma)[::-1]]), - alpha=.5, fc='b', ec='None', label='95% confidence interval') -plt.xlabel('$x$') -plt.ylabel('$f(x)$') -plt.ylim(-10, 20) -plt.legend(loc='upper left') - -plt.show() +from sklearn.gaussian_process.kernels import RBF + +kernel = 1 * RBF(length_scale=1.0, length_scale_bounds=(1e-2, 1e2)) +gaussian_process = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=9) +gaussian_process.fit(X_train, y_train) +gaussian_process.kernel_ + +# %% +# After fitting our model, we see that the hyperparameters of the kernel have +# been optimized. Now, we will use our kernel to compute the mean prediction +# of the full dataset and plot the 95% confidence interval. +mean_prediction, std_prediction = gaussian_process.predict(X, return_std=True) + +plt.plot(X, y, label=r"$f(x) = x \sin(x)$", linestyle="dotted") +plt.scatter(X_train, y_train, label="Observations") +plt.plot(X, mean_prediction, label="Mean prediction") +plt.fill_between( + X.ravel(), + mean_prediction - 1.96 * std_prediction, + mean_prediction + 1.96 * std_prediction, + alpha=0.5, + label=r"95% confidence interval", +) +plt.legend() +plt.xlabel("$x$") +plt.ylabel("$f(x)$") +_ = plt.title("Gaussian process regression on noise-free dataset") + +# %% +# We see that for a prediction made on a data point close to the one from the +# training set, the 95% confidence has a small amplitude. Whenever a sample +# falls far from training data, our model's prediction is less accurate and the +# model prediction is less precise (higher uncertainty). +# +# Example with noisy targets +# -------------------------- +# +# We can repeat a similar experiment adding an additional noise to the target +# this time. It will allow seeing the effect of the noise on the fitted model. +# +# We add some random Gaussian noise to the target with an arbitrary +# standard deviation. +noise_std = 0.75 +y_train_noisy = y_train + rng.normal(loc=0.0, scale=noise_std, size=y_train.shape) + +# %% +# We create a similar Gaussian process model. In addition to the kernel, this +# time, we specify the parameter `alpha` which can be interpreted as the +# variance of a Gaussian noise. +gaussian_process = GaussianProcessRegressor( + kernel=kernel, alpha=noise_std**2, n_restarts_optimizer=9 +) +gaussian_process.fit(X_train, y_train_noisy) +mean_prediction, std_prediction = gaussian_process.predict(X, return_std=True) + +# %% +# Let's plot the mean prediction and the uncertainty region as before. +plt.plot(X, y, label=r"$f(x) = x \sin(x)$", linestyle="dotted") +plt.errorbar( + X_train, + y_train_noisy, + noise_std, + linestyle="None", + color="tab:blue", + marker=".", + markersize=10, + label="Observations", +) +plt.plot(X, mean_prediction, label="Mean prediction") +plt.fill_between( + X.ravel(), + mean_prediction - 1.96 * std_prediction, + mean_prediction + 1.96 * std_prediction, + color="tab:orange", + alpha=0.5, + label=r"95% confidence interval", +) +plt.legend() +plt.xlabel("$x$") +plt.ylabel("$f(x)$") +_ = plt.title("Gaussian process regression on a noisy dataset") + +# %% +# The noise affects the predictions close to the training samples: the +# predictive uncertainty near to the training samples is larger because we +# explicitly model a given level target noise independent of the input +# variable. diff --git a/examples/gaussian_process/plot_gpr_on_structured_data.py b/examples/gaussian_process/plot_gpr_on_structured_data.py index 64a84ab38647a..f3a8de5d018ef 100644 --- a/examples/gaussian_process/plot_gpr_on_structured_data.py +++ b/examples/gaussian_process/plot_gpr_on_structured_data.py @@ -35,57 +35,58 @@ .. [1] Haussler, D. (1999). Convolution kernels on discrete structures (Vol. 646). Technical report, Department of Computer Science, University of California at Santa Cruz. + """ -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% import numpy as np -import matplotlib.pyplot as plt -from sklearn.gaussian_process.kernels import Kernel, Hyperparameter -from sklearn.gaussian_process.kernels import GenericKernelMixin -from sklearn.gaussian_process import GaussianProcessRegressor -from sklearn.gaussian_process import GaussianProcessClassifier + from sklearn.base import clone +from sklearn.gaussian_process import GaussianProcessClassifier, GaussianProcessRegressor +from sklearn.gaussian_process.kernels import GenericKernelMixin, Hyperparameter, Kernel class SequenceKernel(GenericKernelMixin, Kernel): - ''' + """ A minimal (but valid) convolutional kernel for sequences of variable - lengths.''' - def __init__(self, - baseline_similarity=0.5, - baseline_similarity_bounds=(1e-5, 1)): + lengths.""" + + def __init__(self, baseline_similarity=0.5, baseline_similarity_bounds=(1e-5, 1)): self.baseline_similarity = baseline_similarity self.baseline_similarity_bounds = baseline_similarity_bounds @property def hyperparameter_baseline_similarity(self): - return Hyperparameter("baseline_similarity", - "numeric", - self.baseline_similarity_bounds) + return Hyperparameter( + "baseline_similarity", "numeric", self.baseline_similarity_bounds + ) def _f(self, s1, s2): - ''' + """ kernel value between a pair of sequences - ''' - return sum([1.0 if c1 == c2 else self.baseline_similarity - for c1 in s1 - for c2 in s2]) + """ + return sum( + [1.0 if c1 == c2 else self.baseline_similarity for c1 in s1 for c2 in s2] + ) def _g(self, s1, s2): - ''' + """ kernel derivative between a pair of sequences - ''' - return sum([0.0 if c1 == c2 else 1.0 - for c1 in s1 - for c2 in s2]) + """ + return sum([0.0 if c1 == c2 else 1.0 for c1 in s1 for c2 in s2]) def __call__(self, X, Y=None, eval_gradient=False): if Y is None: Y = X if eval_gradient: - return (np.array([[self._f(x, y) for y in Y] for x in X]), - np.array([[[self._g(x, y)] for y in Y] for x in X])) + return ( + np.array([[self._f(x, y) for y in Y] for x in X]), + np.array([[[self._g(x, y)] for y in Y] for x in X]), + ) else: return np.array([[self._f(x, y) for y in Y] for x in X]) @@ -103,12 +104,13 @@ def clone_with_theta(self, theta): kernel = SequenceKernel() -''' -Sequence similarity matrix under the kernel -=========================================== -''' +# %% +# Sequence similarity matrix under the kernel +# =========================================== -X = np.array(['AGCT', 'AGC', 'AACT', 'TAA', 'AAA', 'GAACA']) +import matplotlib.pyplot as plt + +X = np.array(["AGCT", "AGC", "AACT", "TAA", "AAA", "GAACA"]) K = kernel(X) D = kernel.diag(X) @@ -117,14 +119,14 @@ def clone_with_theta(self, theta): plt.imshow(np.diag(D**-0.5).dot(K).dot(np.diag(D**-0.5))) plt.xticks(np.arange(len(X)), X) plt.yticks(np.arange(len(X)), X) -plt.title('Sequence similarity under the kernel') +plt.title("Sequence similarity under the kernel") +plt.show() -''' -Regression -========== -''' +# %% +# Regression +# ========== -X = np.array(['AGCT', 'AGC', 'AACT', 'TAA', 'AAA', 'GAACA']) +X = np.array(["AGCT", "AGC", "AACT", "TAA", "AAA", "GAACA"]) Y = np.array([1.0, 1.0, 2.0, 2.0, 3.0, 3.0]) training_idx = [0, 1, 3, 4] @@ -132,43 +134,57 @@ def clone_with_theta(self, theta): gp.fit(X[training_idx], Y[training_idx]) plt.figure(figsize=(8, 5)) -plt.bar(np.arange(len(X)), gp.predict(X), color='b', label='prediction') -plt.bar(training_idx, Y[training_idx], width=0.2, color='r', - alpha=1, label='training') +plt.bar(np.arange(len(X)), gp.predict(X), color="b", label="prediction") +plt.bar(training_idx, Y[training_idx], width=0.2, color="r", alpha=1, label="training") plt.xticks(np.arange(len(X)), X) -plt.title('Regression on sequences') +plt.title("Regression on sequences") plt.legend() +plt.show() -''' -Classification -============== -''' +# %% +# Classification +# ============== -X_train = np.array(['AGCT', 'CGA', 'TAAC', 'TCG', 'CTTT', 'TGCT']) +X_train = np.array(["AGCT", "CGA", "TAAC", "TCG", "CTTT", "TGCT"]) # whether there are 'A's in the sequence Y_train = np.array([True, True, True, False, False, False]) gp = GaussianProcessClassifier(kernel) gp.fit(X_train, Y_train) -X_test = ['AAA', 'ATAG', 'CTC', 'CT', 'C'] +X_test = ["AAA", "ATAG", "CTC", "CT", "C"] Y_test = [True, True, False, False, False] plt.figure(figsize=(8, 5)) -plt.scatter(np.arange(len(X_train)), [1.0 if c else -1.0 for c in Y_train], - s=100, marker='o', edgecolor='none', facecolor=(1, 0.75, 0), - label='training') -plt.scatter(len(X_train) + np.arange(len(X_test)), - [1.0 if c else -1.0 for c in Y_test], - s=100, marker='o', edgecolor='none', facecolor='r', label='truth') -plt.scatter(len(X_train) + np.arange(len(X_test)), - [1.0 if c else -1.0 for c in gp.predict(X_test)], - s=100, marker='x', edgecolor=(0, 1.0, 0.3), linewidth=2, - label='prediction') -plt.xticks(np.arange(len(X_train) + len(X_test)), - np.concatenate((X_train, X_test))) +plt.scatter( + np.arange(len(X_train)), + [1.0 if c else -1.0 for c in Y_train], + s=100, + marker="o", + edgecolor="none", + facecolor=(1, 0.75, 0), + label="training", +) +plt.scatter( + len(X_train) + np.arange(len(X_test)), + [1.0 if c else -1.0 for c in Y_test], + s=100, + marker="o", + edgecolor="none", + facecolor="r", + label="truth", +) +plt.scatter( + len(X_train) + np.arange(len(X_test)), + [1.0 if c else -1.0 for c in gp.predict(X_test)], + s=100, + marker="x", + facecolor="b", + linewidth=2, + label="prediction", +) +plt.xticks(np.arange(len(X_train) + len(X_test)), np.concatenate((X_train, X_test))) plt.yticks([-1, 1], [False, True]) -plt.title('Classification on sequences') +plt.title("Classification on sequences") plt.legend() - plt.show() diff --git a/examples/gaussian_process/plot_gpr_prior_posterior.py b/examples/gaussian_process/plot_gpr_prior_posterior.py index 78eb22df48f6a..df4ab89719678 100644 --- a/examples/gaussian_process/plot_gpr_prior_posterior.py +++ b/examples/gaussian_process/plot_gpr_prior_posterior.py @@ -3,76 +3,255 @@ Illustration of prior and posterior Gaussian process for different kernels ========================================================================== -This example illustrates the prior and posterior of a GPR with different -kernels. Mean, standard deviation, and 10 samples are shown for both prior -and posterior. +This example illustrates the prior and posterior of a +:class:`~sklearn.gaussian_process.GaussianProcessRegressor` with different +kernels. Mean, standard deviation, and 5 samples are shown for both prior +and posterior distributions. + +Here, we only give some illustration. To know more about kernels' formulation, +refer to the :ref:`User Guide `. + """ -print(__doc__) -# Authors: Jan Hendrik Metzen -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +# %% +# Helper function +# --------------- +# +# Before presenting each individual kernel available for Gaussian processes, +# we will define an helper function allowing us plotting samples drawn from +# the Gaussian process. +# +# This function will take a +# :class:`~sklearn.gaussian_process.GaussianProcessRegressor` model and will +# drawn sample from the Gaussian process. If the model was not fit, the samples +# are drawn from the prior distribution while after model fitting, the samples are +# drawn from the posterior distribution. +import matplotlib.pyplot as plt import numpy as np -from matplotlib import pyplot as plt +def plot_gpr_samples(gpr_model, n_samples, ax): + """Plot samples drawn from the Gaussian process model. + + If the Gaussian process model is not trained then the drawn samples are + drawn from the prior distribution. Otherwise, the samples are drawn from + the posterior distribution. Be aware that a sample here corresponds to a + function. + + Parameters + ---------- + gpr_model : `GaussianProcessRegressor` + A :class:`~sklearn.gaussian_process.GaussianProcessRegressor` model. + n_samples : int + The number of samples to draw from the Gaussian process distribution. + ax : matplotlib axis + The matplotlib axis where to plot the samples. + """ + x = np.linspace(0, 5, 100) + X = x.reshape(-1, 1) + + y_mean, y_std = gpr_model.predict(X, return_std=True) + y_samples = gpr_model.sample_y(X, n_samples) + + for idx, single_prior in enumerate(y_samples.T): + ax.plot( + x, + single_prior, + linestyle="--", + alpha=0.7, + label=f"Sampled function #{idx + 1}", + ) + ax.plot(x, y_mean, color="black", label="Mean") + ax.fill_between( + x, + y_mean - y_std, + y_mean + y_std, + alpha=0.1, + color="black", + label=r"$\pm$ 1 std. dev.", + ) + ax.set_xlabel("x") + ax.set_ylabel("y") + ax.set_ylim([-3, 3]) + + +# %% +# Dataset and Gaussian process generation +# --------------------------------------- +# We will create a training dataset that we will use in the different sections. +rng = np.random.RandomState(4) +X_train = rng.uniform(0, 5, 10).reshape(-1, 1) +y_train = np.sin((X_train[:, 0] - 2.5) ** 2) +n_samples = 5 + +# %% +# Kernel cookbook +# --------------- +# +# In this section, we illustrate some samples drawn from the prior and posterior +# distributions of the Gaussian process with different kernels. +# +# Radial Basis Function kernel +# ............................ from sklearn.gaussian_process import GaussianProcessRegressor -from sklearn.gaussian_process.kernels import (RBF, Matern, RationalQuadratic, - ExpSineSquared, DotProduct, - ConstantKernel) - - -kernels = [1.0 * RBF(length_scale=1.0, length_scale_bounds=(1e-1, 10.0)), - 1.0 * RationalQuadratic(length_scale=1.0, alpha=0.1), - 1.0 * ExpSineSquared(length_scale=1.0, periodicity=3.0, - length_scale_bounds=(0.1, 10.0), - periodicity_bounds=(1.0, 10.0)), - ConstantKernel(0.1, (0.01, 10.0)) - * (DotProduct(sigma_0=1.0, sigma_0_bounds=(0.1, 10.0)) ** 2), - 1.0 * Matern(length_scale=1.0, length_scale_bounds=(1e-1, 10.0), - nu=1.5)] - -for kernel in kernels: - # Specify Gaussian Process - gp = GaussianProcessRegressor(kernel=kernel) - - # Plot prior - plt.figure(figsize=(8, 8)) - plt.subplot(2, 1, 1) - X_ = np.linspace(0, 5, 100) - y_mean, y_std = gp.predict(X_[:, np.newaxis], return_std=True) - plt.plot(X_, y_mean, 'k', lw=3, zorder=9) - plt.fill_between(X_, y_mean - y_std, y_mean + y_std, - alpha=0.2, color='k') - y_samples = gp.sample_y(X_[:, np.newaxis], 10) - plt.plot(X_, y_samples, lw=1) - plt.xlim(0, 5) - plt.ylim(-3, 3) - plt.title("Prior (kernel: %s)" % kernel, fontsize=12) - - # Generate data and fit GP - rng = np.random.RandomState(4) - X = rng.uniform(0, 5, 10)[:, np.newaxis] - y = np.sin((X[:, 0] - 2.5) ** 2) - gp.fit(X, y) - - # Plot posterior - plt.subplot(2, 1, 2) - X_ = np.linspace(0, 5, 100) - y_mean, y_std = gp.predict(X_[:, np.newaxis], return_std=True) - plt.plot(X_, y_mean, 'k', lw=3, zorder=9) - plt.fill_between(X_, y_mean - y_std, y_mean + y_std, - alpha=0.2, color='k') - - y_samples = gp.sample_y(X_[:, np.newaxis], 10) - plt.plot(X_, y_samples, lw=1) - plt.scatter(X[:, 0], y, c='r', s=50, zorder=10, edgecolors=(0, 0, 0)) - plt.xlim(0, 5) - plt.ylim(-3, 3) - plt.title("Posterior (kernel: %s)\n Log-Likelihood: %.3f" - % (gp.kernel_, gp.log_marginal_likelihood(gp.kernel_.theta)), - fontsize=12) - plt.tight_layout() - -plt.show() +from sklearn.gaussian_process.kernels import RBF + +kernel = 1.0 * RBF(length_scale=1.0, length_scale_bounds=(1e-1, 10.0)) +gpr = GaussianProcessRegressor(kernel=kernel, random_state=0) + +fig, axs = plt.subplots(nrows=2, sharex=True, sharey=True, figsize=(10, 8)) + +# plot prior +plot_gpr_samples(gpr, n_samples=n_samples, ax=axs[0]) +axs[0].set_title("Samples from prior distribution") + +# plot posterior +gpr.fit(X_train, y_train) +plot_gpr_samples(gpr, n_samples=n_samples, ax=axs[1]) +axs[1].scatter(X_train[:, 0], y_train, color="red", zorder=10, label="Observations") +axs[1].legend(bbox_to_anchor=(1.05, 1.5), loc="upper left") +axs[1].set_title("Samples from posterior distribution") + +fig.suptitle("Radial Basis Function kernel", fontsize=18) +plt.tight_layout() + +# %% +print(f"Kernel parameters before fit:\n{kernel})") +print( + f"Kernel parameters after fit: \n{gpr.kernel_} \n" + f"Log-likelihood: {gpr.log_marginal_likelihood(gpr.kernel_.theta):.3f}" +) + +# %% +# Rational Quadratic kernel +# ......................... +from sklearn.gaussian_process.kernels import RationalQuadratic + +kernel = 1.0 * RationalQuadratic(length_scale=1.0, alpha=0.1, alpha_bounds=(1e-5, 1e15)) +gpr = GaussianProcessRegressor(kernel=kernel, random_state=0) + +fig, axs = plt.subplots(nrows=2, sharex=True, sharey=True, figsize=(10, 8)) + +# plot prior +plot_gpr_samples(gpr, n_samples=n_samples, ax=axs[0]) +axs[0].set_title("Samples from prior distribution") + +# plot posterior +gpr.fit(X_train, y_train) +plot_gpr_samples(gpr, n_samples=n_samples, ax=axs[1]) +axs[1].scatter(X_train[:, 0], y_train, color="red", zorder=10, label="Observations") +axs[1].legend(bbox_to_anchor=(1.05, 1.5), loc="upper left") +axs[1].set_title("Samples from posterior distribution") + +fig.suptitle("Rational Quadratic kernel", fontsize=18) +plt.tight_layout() + +# %% +print(f"Kernel parameters before fit:\n{kernel})") +print( + f"Kernel parameters after fit: \n{gpr.kernel_} \n" + f"Log-likelihood: {gpr.log_marginal_likelihood(gpr.kernel_.theta):.3f}" +) + +# %% +# Exp-Sine-Squared kernel +# ....................... +from sklearn.gaussian_process.kernels import ExpSineSquared + +kernel = 1.0 * ExpSineSquared( + length_scale=1.0, + periodicity=3.0, + length_scale_bounds=(0.1, 10.0), + periodicity_bounds=(1.0, 10.0), +) +gpr = GaussianProcessRegressor(kernel=kernel, random_state=0) + +fig, axs = plt.subplots(nrows=2, sharex=True, sharey=True, figsize=(10, 8)) + +# plot prior +plot_gpr_samples(gpr, n_samples=n_samples, ax=axs[0]) +axs[0].set_title("Samples from prior distribution") + +# plot posterior +gpr.fit(X_train, y_train) +plot_gpr_samples(gpr, n_samples=n_samples, ax=axs[1]) +axs[1].scatter(X_train[:, 0], y_train, color="red", zorder=10, label="Observations") +axs[1].legend(bbox_to_anchor=(1.05, 1.5), loc="upper left") +axs[1].set_title("Samples from posterior distribution") + +fig.suptitle("Exp-Sine-Squared kernel", fontsize=18) +plt.tight_layout() + +# %% +print(f"Kernel parameters before fit:\n{kernel})") +print( + f"Kernel parameters after fit: \n{gpr.kernel_} \n" + f"Log-likelihood: {gpr.log_marginal_likelihood(gpr.kernel_.theta):.3f}" +) + +# %% +# Dot-product kernel +# .................. +from sklearn.gaussian_process.kernels import ConstantKernel, DotProduct + +kernel = ConstantKernel(0.1, (0.01, 10.0)) * ( + DotProduct(sigma_0=1.0, sigma_0_bounds=(0.1, 10.0)) ** 2 +) +gpr = GaussianProcessRegressor(kernel=kernel, random_state=0, normalize_y=True) + +fig, axs = plt.subplots(nrows=2, sharex=True, sharey=True, figsize=(10, 8)) + +# plot prior +plot_gpr_samples(gpr, n_samples=n_samples, ax=axs[0]) +axs[0].set_title("Samples from prior distribution") + +# plot posterior +gpr.fit(X_train, y_train) +plot_gpr_samples(gpr, n_samples=n_samples, ax=axs[1]) +axs[1].scatter(X_train[:, 0], y_train, color="red", zorder=10, label="Observations") +axs[1].legend(bbox_to_anchor=(1.05, 1.5), loc="upper left") +axs[1].set_title("Samples from posterior distribution") + +fig.suptitle("Dot-product kernel", fontsize=18) +plt.tight_layout() + +# %% +print(f"Kernel parameters before fit:\n{kernel})") +print( + f"Kernel parameters after fit: \n{gpr.kernel_} \n" + f"Log-likelihood: {gpr.log_marginal_likelihood(gpr.kernel_.theta):.3f}" +) + +# %% +# MatÊrn kernel +# .............. +from sklearn.gaussian_process.kernels import Matern + +kernel = 1.0 * Matern(length_scale=1.0, length_scale_bounds=(1e-1, 10.0), nu=1.5) +gpr = GaussianProcessRegressor(kernel=kernel, random_state=0) + +fig, axs = plt.subplots(nrows=2, sharex=True, sharey=True, figsize=(10, 8)) + +# plot prior +plot_gpr_samples(gpr, n_samples=n_samples, ax=axs[0]) +axs[0].set_title("Samples from prior distribution") + +# plot posterior +gpr.fit(X_train, y_train) +plot_gpr_samples(gpr, n_samples=n_samples, ax=axs[1]) +axs[1].scatter(X_train[:, 0], y_train, color="red", zorder=10, label="Observations") +axs[1].legend(bbox_to_anchor=(1.05, 1.5), loc="upper left") +axs[1].set_title("Samples from posterior distribution") + +fig.suptitle("MatÊrn kernel", fontsize=18) +plt.tight_layout() + +# %% +print(f"Kernel parameters before fit:\n{kernel})") +print( + f"Kernel parameters after fit: \n{gpr.kernel_} \n" + f"Log-likelihood: {gpr.log_marginal_likelihood(gpr.kernel_.theta):.3f}" +) diff --git a/examples/impute/plot_iterative_imputer_variants_comparison.py b/examples/impute/plot_iterative_imputer_variants_comparison.py index 877cfd37c0067..d2a68d351ce8a 100644 --- a/examples/impute/plot_iterative_imputer_variants_comparison.py +++ b/examples/impute/plot_iterative_imputer_variants_comparison.py @@ -13,17 +13,16 @@ imputation with :class:`~impute.IterativeImputer`: * :class:`~linear_model.BayesianRidge`: regularized linear regression -* :class:`~tree.DecisionTreeRegressor`: non-linear regression -* :class:`~ensemble.ExtraTreesRegressor`: similar to missForest in R +* :class:`~ensemble.RandomForestRegressor`: Forests of randomized trees regression +* :func:`~pipeline.make_pipeline` (:class:`~kernel_approximation.Nystroem`, + :class:`~linear_model.Ridge`): a pipeline with the expansion of a degree 2 + polynomial kernel and regularized linear regression * :class:`~neighbors.KNeighborsRegressor`: comparable to other KNN imputation approaches Of particular interest is the ability of :class:`~impute.IterativeImputer` to mimic the behavior of missForest, a -popular imputation package for R. In this example, we have chosen to use -:class:`~ensemble.ExtraTreesRegressor` instead of -:class:`~ensemble.RandomForestRegressor` (as in missForest) due to its -increased speed. +popular imputation package for R. Note that :class:`~neighbors.KNeighborsRegressor` is different from KNN imputation, which learns from samples with missing values by using a distance @@ -35,26 +34,34 @@ dataset with a single value randomly removed from each row. For this particular pattern of missing values we see that -:class:`~ensemble.ExtraTreesRegressor` and -:class:`~linear_model.BayesianRidge` give the best results. +:class:`~linear_model.BayesianRidge` and +:class:`~ensemble.RandomForestRegressor` give the best results. + +It should be noted that some estimators such as +:class:`~ensemble.HistGradientBoostingRegressor` can natively deal with +missing features and are often recommended over building pipelines with +complex and costly missing values imputation strategies. + """ -print(__doc__) -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt +import numpy as np import pandas as pd -# To use this experimental feature, we need to explicitly ask for it: -from sklearn.experimental import enable_iterative_imputer # noqa from sklearn.datasets import fetch_california_housing -from sklearn.impute import SimpleImputer -from sklearn.impute import IterativeImputer -from sklearn.linear_model import BayesianRidge -from sklearn.tree import DecisionTreeRegressor -from sklearn.ensemble import ExtraTreesRegressor +from sklearn.ensemble import RandomForestRegressor + +# To use this experimental feature, we need to explicitly ask for it: +from sklearn.experimental import enable_iterative_imputer # noqa: F401 +from sklearn.impute import IterativeImputer, SimpleImputer +from sklearn.kernel_approximation import Nystroem +from sklearn.linear_model import BayesianRidge, Ridge +from sklearn.model_selection import cross_val_score from sklearn.neighbors import KNeighborsRegressor from sklearn.pipeline import make_pipeline -from sklearn.model_selection import cross_val_score N_SPLITS = 5 @@ -71,10 +78,9 @@ br_estimator = BayesianRidge() score_full_data = pd.DataFrame( cross_val_score( - br_estimator, X_full, y_full, scoring='neg_mean_squared_error', - cv=N_SPLITS + br_estimator, X_full, y_full, scoring="neg_mean_squared_error", cv=N_SPLITS ), - columns=['Full Data'] + columns=["Full Data"], ) # Add a single missing value to each row @@ -86,39 +92,55 @@ # Estimate the score after imputation (mean and median strategies) score_simple_imputer = pd.DataFrame() -for strategy in ('mean', 'median'): +for strategy in ("mean", "median"): estimator = make_pipeline( - SimpleImputer(missing_values=np.nan, strategy=strategy), - br_estimator + SimpleImputer(missing_values=np.nan, strategy=strategy), br_estimator ) score_simple_imputer[strategy] = cross_val_score( - estimator, X_missing, y_missing, scoring='neg_mean_squared_error', - cv=N_SPLITS + estimator, X_missing, y_missing, scoring="neg_mean_squared_error", cv=N_SPLITS ) # Estimate the score after iterative imputation of the missing values # with different estimators estimators = [ BayesianRidge(), - DecisionTreeRegressor(max_features='sqrt', random_state=0), - ExtraTreesRegressor(n_estimators=10, random_state=0), - KNeighborsRegressor(n_neighbors=15) + RandomForestRegressor( + # We tuned the hyperparameters of the RandomForestRegressor to get a good + # enough predictive performance for a restricted execution time. + n_estimators=4, + max_depth=10, + bootstrap=True, + max_samples=0.5, + n_jobs=2, + random_state=0, + ), + make_pipeline( + Nystroem(kernel="polynomial", degree=2, random_state=0), Ridge(alpha=1e3) + ), + KNeighborsRegressor(n_neighbors=15), ] score_iterative_imputer = pd.DataFrame() -for impute_estimator in estimators: +# iterative imputer is sensible to the tolerance and +# dependent on the estimator used internally. +# we tuned the tolerance to keep this example run with limited computational +# resources while not changing the results too much compared to keeping the +# stricter default value for the tolerance parameter. +tolerances = (1e-3, 1e-1, 1e-1, 1e-2) +for impute_estimator, tol in zip(estimators, tolerances): estimator = make_pipeline( - IterativeImputer(random_state=0, estimator=impute_estimator), - br_estimator + IterativeImputer( + random_state=0, estimator=impute_estimator, max_iter=25, tol=tol + ), + br_estimator, + ) + score_iterative_imputer[impute_estimator.__class__.__name__] = cross_val_score( + estimator, X_missing, y_missing, scoring="neg_mean_squared_error", cv=N_SPLITS ) - score_iterative_imputer[impute_estimator.__class__.__name__] = \ - cross_val_score( - estimator, X_missing, y_missing, scoring='neg_mean_squared_error', - cv=N_SPLITS - ) scores = pd.concat( [score_full_data, score_simple_imputer, score_iterative_imputer], - keys=['Original', 'SimpleImputer', 'IterativeImputer'], axis=1 + keys=["Original", "SimpleImputer", "IterativeImputer"], + axis=1, ) # plot california housing results @@ -126,8 +148,8 @@ means = -scores.mean() errors = scores.std() means.plot.barh(xerr=errors, ax=ax) -ax.set_title('California Housing Regression with Different Imputation Methods') -ax.set_xlabel('MSE (smaller is better)') +ax.set_title("California Housing Regression with Different Imputation Methods") +ax.set_xlabel("MSE (smaller is better)") ax.set_yticks(np.arange(means.shape[0])) ax.set_yticklabels([" w/ ".join(label) for label in means.index.tolist()]) plt.tight_layout(pad=1) diff --git a/examples/impute/plot_missing_values.py b/examples/impute/plot_missing_values.py index f5d75f68c3d09..851bfd419453b 100644 --- a/examples/impute/plot_missing_values.py +++ b/examples/impute/plot_missing_values.py @@ -27,14 +27,13 @@ missing values imputed using different techniques. """ -print(__doc__) -# Authors: Maria Telenczuk -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause # %% # Download the data and make missing values sets -################################################ +# ############################################## # # First we download the two datasets. Diabetes dataset is shipped with # scikit-learn. It has 442 entries, each with 10 features. California Housing @@ -45,16 +44,16 @@ import numpy as np -from sklearn.datasets import fetch_california_housing -from sklearn.datasets import load_diabetes - +from sklearn.datasets import fetch_california_housing, load_diabetes rng = np.random.RandomState(42) X_diabetes, y_diabetes = load_diabetes(return_X_y=True) X_california, y_california = fetch_california_housing(return_X_y=True) -X_california = X_california[:400] -y_california = y_california[:400] +X_california = X_california[:300] +y_california = y_california[:300] +X_diabetes = X_diabetes[:300] +y_diabetes = y_diabetes[:300] def add_missing_values(X_full, y_full): @@ -65,7 +64,7 @@ def add_missing_values(X_full, y_full): n_missing_samples = int(n_samples * missing_rate) missing_samples = np.zeros(n_samples, dtype=bool) - missing_samples[: n_missing_samples] = True + missing_samples[:n_missing_samples] = True rng.shuffle(missing_samples) missing_features = rng.randint(0, n_features, n_missing_samples) @@ -76,11 +75,9 @@ def add_missing_values(X_full, y_full): return X_missing, y_missing -X_miss_california, y_miss_california = add_missing_values( - X_california, y_california) +X_miss_california, y_miss_california = add_missing_values(X_california, y_california) -X_miss_diabetes, y_miss_diabetes = add_missing_values( - X_diabetes, y_diabetes) +X_miss_diabetes, y_miss_diabetes = add_missing_values(X_diabetes, y_diabetes) # %% @@ -95,13 +92,12 @@ def add_missing_values(X_full, y_full): from sklearn.ensemble import RandomForestRegressor # To use the experimental IterativeImputer, we need to explicitly ask for it: -from sklearn.experimental import enable_iterative_imputer # noqa -from sklearn.impute import SimpleImputer, KNNImputer, IterativeImputer +from sklearn.experimental import enable_iterative_imputer # noqa: F401 +from sklearn.impute import IterativeImputer, KNNImputer, SimpleImputer from sklearn.model_selection import cross_val_score from sklearn.pipeline import make_pipeline - -N_SPLITS = 5 +N_SPLITS = 4 regressor = RandomForestRegressor(random_state=0) # %% @@ -115,17 +111,13 @@ def add_missing_values(X_full, y_full): def get_scores_for_imputer(imputer, X_missing, y_missing): estimator = make_pipeline(imputer, regressor) - impute_scores = cross_val_score(estimator, X_missing, y_missing, - scoring='neg_mean_squared_error', - cv=N_SPLITS) + impute_scores = cross_val_score( + estimator, X_missing, y_missing, scoring="neg_mean_squared_error", cv=N_SPLITS + ) return impute_scores -x_labels = ['Full data', - 'Zero imputation', - 'Mean Imputation', - 'KNN Imputation', - 'Iterative Imputation'] +x_labels = [] mses_california = np.zeros(5) stds_california = np.zeros(5) @@ -140,15 +132,15 @@ def get_scores_for_imputer(imputer, X_missing, y_missing): def get_full_score(X_full, y_full): - full_scores = cross_val_score(regressor, X_full, y_full, - scoring='neg_mean_squared_error', - cv=N_SPLITS) + full_scores = cross_val_score( + regressor, X_full, y_full, scoring="neg_mean_squared_error", cv=N_SPLITS + ) return full_scores.mean(), full_scores.std() -mses_california[0], stds_california[0] = get_full_score(X_california, - y_california) +mses_california[0], stds_california[0] = get_full_score(X_california, y_california) mses_diabetes[0], stds_diabetes[0] = get_full_score(X_diabetes, y_diabetes) +x_labels.append("Full data") # %% @@ -161,17 +153,20 @@ def get_full_score(X_full, y_full): def get_impute_zero_score(X_missing, y_missing): - - imputer = SimpleImputer(missing_values=np.nan, add_indicator=True, - strategy='constant', fill_value=0) + imputer = SimpleImputer( + missing_values=np.nan, add_indicator=True, strategy="constant", fill_value=0 + ) zero_impute_scores = get_scores_for_imputer(imputer, X_missing, y_missing) return zero_impute_scores.mean(), zero_impute_scores.std() mses_california[1], stds_california[1] = get_impute_zero_score( - X_miss_california, y_miss_california) -mses_diabetes[1], stds_diabetes[1] = get_impute_zero_score(X_miss_diabetes, - y_miss_diabetes) + X_miss_california, y_miss_california +) +mses_diabetes[1], stds_diabetes[1] = get_impute_zero_score( + X_miss_diabetes, y_miss_diabetes +) +x_labels.append("Zero imputation") # %% @@ -181,6 +176,7 @@ def get_impute_zero_score(X_missing, y_missing): # :class:`~sklearn.impute.KNNImputer` imputes missing values using the weighted # or unweighted mean of the desired number of nearest neighbors. + def get_impute_knn_score(X_missing, y_missing): imputer = KNNImputer(missing_values=np.nan, add_indicator=True) knn_impute_scores = get_scores_for_imputer(imputer, X_missing, y_missing) @@ -188,9 +184,12 @@ def get_impute_knn_score(X_missing, y_missing): mses_california[2], stds_california[2] = get_impute_knn_score( - X_miss_california, y_miss_california) -mses_diabetes[2], stds_diabetes[2] = get_impute_knn_score(X_miss_diabetes, - y_miss_diabetes) + X_miss_california, y_miss_california +) +mses_diabetes[2], stds_diabetes[2] = get_impute_knn_score( + X_miss_diabetes, y_miss_diabetes +) +x_labels.append("KNN Imputation") # %% @@ -198,17 +197,18 @@ def get_impute_knn_score(X_missing, y_missing): # ------------------------------- # + def get_impute_mean(X_missing, y_missing): - imputer = SimpleImputer(missing_values=np.nan, strategy="mean", - add_indicator=True) + imputer = SimpleImputer(missing_values=np.nan, strategy="mean", add_indicator=True) mean_impute_scores = get_scores_for_imputer(imputer, X_missing, y_missing) return mean_impute_scores.mean(), mean_impute_scores.std() -mses_california[3], stds_california[3] = get_impute_mean(X_miss_california, - y_miss_california) -mses_diabetes[3], stds_diabetes[3] = get_impute_mean(X_miss_diabetes, - y_miss_diabetes) +mses_california[3], stds_california[3] = get_impute_mean( + X_miss_california, y_miss_california +) +mses_diabetes[3], stds_diabetes[3] = get_impute_mean(X_miss_diabetes, y_miss_diabetes) +x_labels.append("Mean Imputation") # %% @@ -223,20 +223,27 @@ def get_impute_mean(X_missing, y_missing): # to potentially improve performance. # + def get_impute_iterative(X_missing, y_missing): - imputer = IterativeImputer(missing_values=np.nan, add_indicator=True, - random_state=0, n_nearest_features=5, - sample_posterior=True) - iterative_impute_scores = get_scores_for_imputer(imputer, - X_missing, - y_missing) + imputer = IterativeImputer( + missing_values=np.nan, + add_indicator=True, + random_state=0, + n_nearest_features=3, + max_iter=1, + sample_posterior=True, + ) + iterative_impute_scores = get_scores_for_imputer(imputer, X_missing, y_missing) return iterative_impute_scores.mean(), iterative_impute_scores.std() mses_california[4], stds_california[4] = get_impute_iterative( - X_miss_california, y_miss_california) -mses_diabetes[4], stds_diabetes[4] = get_impute_iterative(X_miss_diabetes, - y_miss_diabetes) + X_miss_california, y_miss_california +) +mses_diabetes[4], stds_diabetes[4] = get_impute_iterative( + X_miss_diabetes, y_miss_diabetes +) +x_labels.append("Iterative Imputation") mses_diabetes = mses_diabetes * -1 mses_california = mses_california * -1 @@ -250,41 +257,52 @@ def get_impute_iterative(X_missing, y_missing): import matplotlib.pyplot as plt - n_bars = len(mses_diabetes) xval = np.arange(n_bars) -colors = ['r', 'g', 'b', 'orange', 'black'] +colors = ["r", "g", "b", "orange", "black"] # plot diabetes results plt.figure(figsize=(12, 6)) ax1 = plt.subplot(121) for j in xval: - ax1.barh(j, mses_diabetes[j], xerr=stds_diabetes[j], - color=colors[j], alpha=0.6, align='center') - -ax1.set_title('Imputation Techniques with Diabetes Data') -ax1.set_xlim(left=np.min(mses_diabetes) * 0.9, - right=np.max(mses_diabetes) * 1.1) + ax1.barh( + j, + mses_diabetes[j], + xerr=stds_diabetes[j], + color=colors[j], + alpha=0.6, + align="center", + ) + +ax1.set_title("Imputation Techniques with Diabetes Data") +ax1.set_xlim(left=np.min(mses_diabetes) * 0.9, right=np.max(mses_diabetes) * 1.1) ax1.set_yticks(xval) -ax1.set_xlabel('MSE') +ax1.set_xlabel("MSE") ax1.invert_yaxis() ax1.set_yticklabels(x_labels) # plot california dataset results ax2 = plt.subplot(122) for j in xval: - ax2.barh(j, mses_california[j], xerr=stds_california[j], - color=colors[j], alpha=0.6, align='center') - -ax2.set_title('Imputation Techniques with California Data') + ax2.barh( + j, + mses_california[j], + xerr=stds_california[j], + color=colors[j], + alpha=0.6, + align="center", + ) + +ax2.set_title("Imputation Techniques with California Data") ax2.set_yticks(xval) -ax2.set_xlabel('MSE') +ax2.set_xlabel("MSE") ax2.invert_yaxis() -ax2.set_yticklabels([''] * n_bars) +ax2.set_yticklabels([""] * n_bars) plt.show() +# %% # You can also try different techniques. For instance, the median is a more # robust estimator for data with high magnitude variables which could dominate # results (otherwise known as a 'long tail'). diff --git a/examples/inspection/README.txt b/examples/inspection/README.txt index e64900d978e59..8d197dea20f71 100644 --- a/examples/inspection/README.txt +++ b/examples/inspection/README.txt @@ -4,4 +4,3 @@ Inspection ---------- Examples related to the :mod:`sklearn.inspection` module. - diff --git a/examples/inspection/plot_causal_interpretation.py b/examples/inspection/plot_causal_interpretation.py new file mode 100644 index 0000000000000..cf6c72f29951d --- /dev/null +++ b/examples/inspection/plot_causal_interpretation.py @@ -0,0 +1,189 @@ +""" +=================================================== +Failure of Machine Learning to infer causal effects +=================================================== + +Machine Learning models are great for measuring statistical associations. +Unfortunately, unless we're willing to make strong assumptions about the data, +those models are unable to infer causal effects. + +To illustrate this, we will simulate a situation in which we try to answer one +of the most important questions in economics of education: **what is the causal +effect of earning a college degree on hourly wages?** Although the answer to +this question is crucial to policy makers, `Omitted-Variable Biases +`_ (OVB) prevent us from +identifying that causal effect. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# The dataset: simulated hourly wages +# ----------------------------------- +# +# The data generating process is laid out in the code below. Work experience in +# years and a measure of ability are drawn from Normal distributions; the +# hourly wage of one of the parents is drawn from Beta distribution. We then +# create an indicator of college degree which is positively impacted by ability +# and parental hourly wage. Finally, we model hourly wages as a linear function +# of all the previous variables and a random component. Note that all variables +# have a positive effect on hourly wages. +import numpy as np +import pandas as pd + +n_samples = 10_000 +rng = np.random.RandomState(32) + +experiences = rng.normal(20, 10, size=n_samples).astype(int) +experiences[experiences < 0] = 0 +abilities = rng.normal(0, 0.15, size=n_samples) +parent_hourly_wages = 50 * rng.beta(2, 8, size=n_samples) +parent_hourly_wages[parent_hourly_wages < 0] = 0 +college_degrees = ( + 9 * abilities + 0.02 * parent_hourly_wages + rng.randn(n_samples) > 0.7 +).astype(int) + +true_coef = pd.Series( + { + "college degree": 2.0, + "ability": 5.0, + "experience": 0.2, + "parent hourly wage": 1.0, + } +) +hourly_wages = ( + true_coef["experience"] * experiences + + true_coef["parent hourly wage"] * parent_hourly_wages + + true_coef["college degree"] * college_degrees + + true_coef["ability"] * abilities + + rng.normal(0, 1, size=n_samples) +) + +hourly_wages[hourly_wages < 0] = 0 + +# %% +# Description of the simulated data +# --------------------------------- +# +# The following plot shows the distribution of each variable, and pairwise +# scatter plots. Key to our OVB story is the positive relationship between +# ability and college degree. +import seaborn as sns + +df = pd.DataFrame( + { + "college degree": college_degrees, + "ability": abilities, + "hourly wage": hourly_wages, + "experience": experiences, + "parent hourly wage": parent_hourly_wages, + } +) + +grid = sns.pairplot(df, diag_kind="kde", corner=True) + +# %% +# In the next section, we train predictive models and we therefore split the +# target column from over features and we split the data into a training and a +# testing set. +from sklearn.model_selection import train_test_split + +target_name = "hourly wage" +X, y = df.drop(columns=target_name), df[target_name] +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0) + +# %% +# Income prediction with fully observed variables +# ----------------------------------------------- +# +# First, we train a predictive model, a +# :class:`~sklearn.linear_model.LinearRegression` model. In this experiment, +# we assume that all variables used by the true generative model are available. +from sklearn.linear_model import LinearRegression +from sklearn.metrics import r2_score + +features_names = ["experience", "parent hourly wage", "college degree", "ability"] + +regressor_with_ability = LinearRegression() +regressor_with_ability.fit(X_train[features_names], y_train) +y_pred_with_ability = regressor_with_ability.predict(X_test[features_names]) +R2_with_ability = r2_score(y_test, y_pred_with_ability) + +print(f"R2 score with ability: {R2_with_ability:.3f}") + +# %% +# This model predicts well the hourly wages as shown by the high R2 score. We +# plot the model coefficients to show that we exactly recover the values of +# the true generative model. +import matplotlib.pyplot as plt + +model_coef = pd.Series(regressor_with_ability.coef_, index=features_names) +coef = pd.concat( + [true_coef[features_names], model_coef], + keys=["Coefficients of true generative model", "Model coefficients"], + axis=1, +) +ax = coef.plot.barh() +ax.set_xlabel("Coefficient values") +ax.set_title("Coefficients of the linear regression including the ability features") +_ = plt.tight_layout() + +# %% +# Income prediction with partial observations +# ------------------------------------------- +# +# In practice, intellectual abilities are not observed or are only estimated +# from proxies that inadvertently measure education as well (e.g. by IQ tests). +# But omitting the "ability" feature from a linear model inflates the estimate +# via a positive OVB. +features_names = ["experience", "parent hourly wage", "college degree"] + +regressor_without_ability = LinearRegression() +regressor_without_ability.fit(X_train[features_names], y_train) +y_pred_without_ability = regressor_without_ability.predict(X_test[features_names]) +R2_without_ability = r2_score(y_test, y_pred_without_ability) + +print(f"R2 score without ability: {R2_without_ability:.3f}") + +# %% +# The predictive power of our model is similar when we omit the ability feature +# in terms of R2 score. We now check if the coefficient of the model are +# different from the true generative model. + +model_coef = pd.Series(regressor_without_ability.coef_, index=features_names) +coef = pd.concat( + [true_coef[features_names], model_coef], + keys=["Coefficients of true generative model", "Model coefficients"], + axis=1, +) +ax = coef.plot.barh() +ax.set_xlabel("Coefficient values") +_ = ax.set_title("Coefficients of the linear regression excluding the ability feature") +plt.tight_layout() +plt.show() + +# %% +# To compensate for the omitted variable, the model inflates the coefficient of +# the college degree feature. Therefore, interpreting this coefficient value +# as a causal effect of the true generative model is incorrect. +# +# Lessons learned +# --------------- +# +# Machine learning models are not designed for the estimation of causal +# effects. While we showed this with a linear model, OVB can affect any type of +# model. +# +# Whenever interpreting a coefficient or a change in predictions brought about +# by a change in one of the features, it is important to keep in mind +# potentially unobserved variables that could be correlated with both the +# feature in question and the target variable. Such variables are called +# `Confounding Variables `_. In +# order to still estimate causal effect in the presence of confounding, +# researchers usually conduct experiments in which the treatment variable (e.g. +# college degree) is randomized. When an experiment is prohibitively expensive +# or unethical, researchers can sometimes use other causal inference techniques +# such as `Instrumental Variables +# `_ (IV) +# estimations. diff --git a/examples/inspection/plot_linear_model_coefficient_interpretation.py b/examples/inspection/plot_linear_model_coefficient_interpretation.py index 459b180f00e36..2510db7f077e6 100644 --- a/examples/inspection/plot_linear_model_coefficient_interpretation.py +++ b/examples/inspection/plot_linear_model_coefficient_interpretation.py @@ -1,39 +1,47 @@ """ -================================================================== -Common pitfalls in interpretation of coefficients of linear models -================================================================== - -In linear models, the target value is modeled as -a linear combination of the features (see the :ref:`linear_model` User Guide -section for a description of a set of linear models available in -scikit-learn). -Coefficients in multiple linear models represent the relationship between the -given feature, :math:`X_i` and the target, :math:`y`, assuming that all the -other features remain constant (`conditional dependence -`_). -This is different from plotting :math:`X_i` versus :math:`y` and fitting a -linear relationship: in that case all possible values of the other features are -taken into account in the estimation (marginal dependence). +====================================================================== +Common pitfalls in the interpretation of coefficients of linear models +====================================================================== + +In linear models, the target value is modeled as a linear combination of the +features (see the :ref:`linear_model` User Guide section for a description of a +set of linear models available in scikit-learn). Coefficients in multiple linear +models represent the relationship between the given feature, :math:`X_i` and the +target, :math:`y`, assuming that all the other features remain constant +(`conditional dependence +`_). This is different +from plotting :math:`X_i` versus :math:`y` and fitting a linear relationship: in +that case all possible values of the other features are taken into account in +the estimation (marginal dependence). This example will provide some hints in interpreting coefficient in linear models, pointing at problems that arise when either the linear model is not appropriate to describe the dataset, or when features are correlated. -We will use data from the `"Current Population Survey" -`_ from 1985 to predict -wage as a function of various features such as experience, age, or education. +.. note:: + + Keep in mind that the features :math:`X` and the outcome :math:`y` are in + general the result of a data generating process that is unknown to us. + Machine learning models are trained to approximate the unobserved + mathematical function that links :math:`X` to :math:`y` from sample data. As + a result, any interpretation made about a model may not necessarily + generalize to the true data generating process. This is especially true when + the model is of bad quality or when the sample data is not representative of + the population. -.. contents:: - :local: - :depth: 1 +We will use data from the `"Current Population Survey" +`_ from 1985 to predict wage as a function of +various features such as experience, age, or education. """ -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +# %% +import matplotlib.pyplot as plt import numpy as np -import scipy as sp import pandas as pd -import matplotlib.pyplot as plt +import scipy as sp import seaborn as sns # %% @@ -43,7 +51,6 @@ # We fetch the data from `OpenML `_. # Note that setting the parameter `as_frame` to True will retrieve the data # as a pandas dataframe. - from sklearn.datasets import fetch_openml survey = fetch_openml(data_id=534, as_frame=True) @@ -51,7 +58,7 @@ # %% # Then, we identify features `X` and targets `y`: the column WAGE is our # target variable (i.e., the variable which we want to predict). -# + X = survey.data[survey.feature_names] X.describe(include="all") @@ -65,6 +72,8 @@ # %% # Our target for prediction: the wage. # Wages are described as floating-point number in dollars per hour. + +# %% y = survey.target.values.ravel() survey.target.head() @@ -77,20 +86,18 @@ from sklearn.model_selection import train_test_split -X_train, X_test, y_train, y_test = train_test_split( - X, y, random_state=42 -) +X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42) # %% # First, let's get some insights by looking at the variable distributions and # at the pairwise relationships between them. Only numerical # variables will be used. In the following plot, each dot represents a sample. # -# .. _marginal_dependencies: +# .. _marginal_dependencies: train_dataset = X_train.copy() train_dataset.insert(0, "WAGE", y_train) -_ = sns.pairplot(train_dataset, kind='reg', diag_kind='kde') +_ = sns.pairplot(train_dataset, kind="reg", diag_kind="kde") # %% # Looking closely at the WAGE distribution reveals that it has a @@ -124,38 +131,35 @@ # Our pre-processor will # # - one-hot encode (i.e., generate a column by category) the categorical -# columns; +# columns, only for non-binary categorical variables; # - as a first approach (we will see after how the normalisation of numerical # values will affect our discussion), keep numerical values as they are. from sklearn.compose import make_column_transformer from sklearn.preprocessing import OneHotEncoder -categorical_columns = ['RACE', 'OCCUPATION', 'SECTOR', - 'MARR', 'UNION', 'SEX', 'SOUTH'] -numerical_columns = ['EDUCATION', 'EXPERIENCE', 'AGE'] +categorical_columns = ["RACE", "OCCUPATION", "SECTOR", "MARR", "UNION", "SEX", "SOUTH"] +numerical_columns = ["EDUCATION", "EXPERIENCE", "AGE"] preprocessor = make_column_transformer( - (OneHotEncoder(drop='if_binary'), categorical_columns), - remainder='passthrough' + (OneHotEncoder(drop="if_binary"), categorical_columns), + remainder="passthrough", + verbose_feature_names_out=False, # avoid to prepend the preprocessor names ) # %% # To describe the dataset as a linear model we use a ridge regressor # with a very small regularization and to model the logarithm of the WAGE. - -from sklearn.pipeline import make_pipeline -from sklearn.linear_model import Ridge from sklearn.compose import TransformedTargetRegressor +from sklearn.linear_model import Ridge +from sklearn.pipeline import make_pipeline model = make_pipeline( preprocessor, TransformedTargetRegressor( - regressor=Ridge(alpha=1e-10), - func=np.log10, - inverse_func=sp.special.exp10 - ) + regressor=Ridge(alpha=1e-10), func=np.log10, inverse_func=sp.special.exp10 + ), ) # %% @@ -164,36 +168,38 @@ # # First, we fit the model. -_ = model.fit(X_train, y_train) +model.fit(X_train, y_train) # %% # Then we check the performance of the computed model plotting its predictions # on the test set and computing, # for example, the median absolute error of the model. -from sklearn.metrics import median_absolute_error - -y_pred = model.predict(X_train) +from sklearn.metrics import PredictionErrorDisplay, median_absolute_error -mae = median_absolute_error(y_train, y_pred) -string_score = f'MAE on training set: {mae:.2f} $/hour' +mae_train = median_absolute_error(y_train, model.predict(X_train)) y_pred = model.predict(X_test) -mae = median_absolute_error(y_test, y_pred) -string_score += f'\nMAE on testing set: {mae:.2f} $/hour' -fig, ax = plt.subplots(figsize=(5, 5)) -plt.scatter(y_test, y_pred) -ax.plot([0, 1], [0, 1], transform=ax.transAxes, ls="--", c="red") -plt.text(3, 20, string_score) -plt.title('Ridge model, small regularization') -plt.ylabel('Model predictions') -plt.xlabel('Truths') -plt.xlim([0, 27]) -_ = plt.ylim([0, 27]) +mae_test = median_absolute_error(y_test, y_pred) +scores = { + "MedAE on training set": f"{mae_train:.2f} $/hour", + "MedAE on testing set": f"{mae_test:.2f} $/hour", +} + +# %% +_, ax = plt.subplots(figsize=(5, 5)) +display = PredictionErrorDisplay.from_predictions( + y_test, y_pred, kind="actual_vs_predicted", ax=ax, scatter_kwargs={"alpha": 0.5} +) +ax.set_title("Ridge model, small regularization") +for name, score in scores.items(): + ax.plot([], [], " ", label=f"{name}: {score}") +ax.legend(loc="upper left") +plt.tight_layout() # %% # The model learnt is far from being a good model making accurate predictions: # this is obvious when looking at the plot above, where good predictions -# should lie on the red line. +# should lie on the black dashed line. # # In the following section, we will interpret the coefficients of the model. # While we do so, we should keep in mind that any conclusion we draw is @@ -201,20 +207,16 @@ # generative process of the data. # # Interpreting coefficients: scale matters -# --------------------------------------------- +# ---------------------------------------- # # First of all, we can take a look to the values of the coefficients of the # regressor we have fitted. - -feature_names = (model.named_steps['columntransformer'] - .named_transformers_['onehotencoder'] - .get_feature_names(input_features=categorical_columns)) -feature_names = np.concatenate( - [feature_names, numerical_columns]) +feature_names = model[:-1].get_feature_names_out() coefs = pd.DataFrame( - model.named_steps['transformedtargetregressor'].regressor_.coef_, - columns=['Coefficients'], index=feature_names + model[-1].regressor_.coef_, + columns=["Coefficients"], + index=feature_names, ) coefs @@ -233,10 +235,11 @@ # hence value ranges, because of their different unit of measure. This is more # visible if we plot the coefficients. -coefs.plot(kind='barh', figsize=(9, 7)) -plt.title('Ridge model, small regularization') -plt.axvline(x=0, color='.5') -plt.subplots_adjust(left=.3) +coefs.plot.barh(figsize=(9, 7)) +plt.title("Ridge model, small regularization") +plt.axvline(x=0, color=".5") +plt.xlabel("Raw coefficient values") +plt.subplots_adjust(left=0.3) # %% # Indeed, from the plot above the most important factor in determining WAGE @@ -252,13 +255,13 @@ # features. X_train_preprocessed = pd.DataFrame( - model.named_steps['columntransformer'].transform(X_train), - columns=feature_names + model[:-1].transform(X_train), columns=feature_names ) -X_train_preprocessed.std(axis=0).plot(kind='barh', figsize=(9, 7)) -plt.title('Features std. dev.') -plt.subplots_adjust(left=.3) +X_train_preprocessed.std(axis=0).plot.barh(figsize=(9, 7)) +plt.title("Feature ranges") +plt.xlabel("Std. dev. of feature values") +plt.subplots_adjust(left=0.3) # %% # Multiplying the coefficients by the standard deviation of the related @@ -273,14 +276,15 @@ # coefficient on the output, all else being equal. coefs = pd.DataFrame( - model.named_steps['transformedtargetregressor'].regressor_.coef_ * - X_train_preprocessed.std(axis=0), - columns=['Coefficient importance'], index=feature_names + model[-1].regressor_.coef_ * X_train_preprocessed.std(axis=0), + columns=["Coefficient importance"], + index=feature_names, ) -coefs.plot(kind='barh', figsize=(9, 7)) -plt.title('Ridge model, small regularization') -plt.axvline(x=0, color='.5') -plt.subplots_adjust(left=.3) +coefs.plot(kind="barh", figsize=(9, 7)) +plt.xlabel("Coefficient values corrected by the feature's std. dev.") +plt.title("Ridge model, small regularization") +plt.axvline(x=0, color=".5") +plt.subplots_adjust(left=0.3) # %% # Now that the coefficients have been scaled, we can safely compare them. @@ -300,6 +304,34 @@ # Also, AGE, EXPERIENCE and EDUCATION are the three variables that most # influence the model. # +# Interpreting coefficients: being cautious about causality +# --------------------------------------------------------- +# +# Linear models are a great tool for measuring statistical association, but we +# should be cautious when making statements about causality, after all +# correlation doesn't always imply causation. This is particularly difficult in +# the social sciences because the variables we observe only function as proxies +# for the underlying causal process. +# +# In our particular case we can think of the EDUCATION of an individual as a +# proxy for their professional aptitude, the real variable we're interested in +# but can't observe. We'd certainly like to think that staying in school for +# longer would increase technical competency, but it's also quite possible that +# causality goes the other way too. That is, those who are technically +# competent tend to stay in school for longer. +# +# An employer is unlikely to care which case it is (or if it's a mix of both), +# as long as they remain convinced that a person with more EDUCATION is better +# suited for the job, they will be happy to pay out a higher WAGE. +# +# This confounding of effects becomes problematic when thinking about some +# form of intervention e.g. government subsidies of university degrees or +# promotional material encouraging individuals to take up higher education. +# The usefulness of these measures could end up being overstated, especially if +# the degree of confounding is strong. Our model predicts a :math:`0.054699` +# increase in hourly wage for each year of education. The actual causal effect +# might be lower because of this confounding. +# # Checking the variability of the coefficients # -------------------------------------------- # @@ -311,26 +343,35 @@ # their robustness is not guaranteed, and they should probably be interpreted # with caution. -from sklearn.model_selection import cross_validate -from sklearn.model_selection import RepeatedKFold +from sklearn.model_selection import RepeatedKFold, cross_validate +cv = RepeatedKFold(n_splits=5, n_repeats=5, random_state=0) cv_model = cross_validate( - model, X, y, cv=RepeatedKFold(n_splits=5, n_repeats=5), - return_estimator=True, n_jobs=-1 + model, + X, + y, + cv=cv, + return_estimator=True, + n_jobs=2, ) + coefs = pd.DataFrame( - [est.named_steps['transformedtargetregressor'].regressor_.coef_ * - X_train_preprocessed.std(axis=0) - for est in cv_model['estimator']], - columns=feature_names + [ + est[-1].regressor_.coef_ * est[:-1].transform(X.iloc[train_idx]).std(axis=0) + for est, (train_idx, _) in zip(cv_model["estimator"], cv.split(X, y)) + ], + columns=feature_names, ) + +# %% plt.figure(figsize=(9, 7)) -sns.stripplot(data=coefs, orient='h', color='k', alpha=0.5) -sns.boxplot(data=coefs, orient='h', color='cyan', saturation=0.5) -plt.axvline(x=0, color='.5') -plt.xlabel('Coefficient importance') -plt.title('Coefficient importance and its variability') -plt.subplots_adjust(left=.3) +sns.stripplot(data=coefs, orient="h", palette="dark:k", alpha=0.5) +sns.boxplot(data=coefs, orient="h", color="cyan", saturation=0.5, whis=10) +plt.axvline(x=0, color=".5") +plt.xlabel("Coefficient importance") +plt.title("Coefficient importance and its variability") +plt.suptitle("Ridge model, small regularization") +plt.subplots_adjust(left=0.3) # %% # The problem of correlated variables @@ -346,46 +387,55 @@ # # .. _covariation: -plt.ylabel('Age coefficient') -plt.xlabel('Experience coefficient') +plt.ylabel("Age coefficient") +plt.xlabel("Experience coefficient") plt.grid(True) plt.xlim(-0.4, 0.5) plt.ylim(-0.4, 0.5) plt.scatter(coefs["AGE"], coefs["EXPERIENCE"]) -_ = plt.title('Co-variations of coefficients for AGE and EXPERIENCE ' - 'across folds') +_ = plt.title("Co-variations of coefficients for AGE and EXPERIENCE across folds") # %% # Two regions are populated: when the EXPERIENCE coefficient is -# positive the AGE one is negative and viceversa. +# positive the AGE one is negative and vice-versa. # # To go further we remove one of the 2 features and check what is the impact # on the model stability. -column_to_drop = ['AGE'] +column_to_drop = ["AGE"] cv_model = cross_validate( - model, X.drop(columns=column_to_drop), y, - cv=RepeatedKFold(n_splits=5, n_repeats=5), - return_estimator=True, n_jobs=-1 + model, + X.drop(columns=column_to_drop), + y, + cv=cv, + return_estimator=True, + n_jobs=2, ) + coefs = pd.DataFrame( - [est.named_steps['transformedtargetregressor'].regressor_.coef_ * - X_train_preprocessed.drop(columns=column_to_drop).std(axis=0) - for est in cv_model['estimator']], - columns=feature_names[:-1] + [ + est[-1].regressor_.coef_ + * est[:-1].transform(X.drop(columns=column_to_drop).iloc[train_idx]).std(axis=0) + for est, (train_idx, _) in zip(cv_model["estimator"], cv.split(X, y)) + ], + columns=feature_names[:-1], ) + +# %% plt.figure(figsize=(9, 7)) -sns.stripplot(data=coefs, orient='h', color='k', alpha=0.5) -sns.boxplot(data=coefs, orient='h', color='cyan', saturation=0.5) -plt.axvline(x=0, color='.5') -plt.title('Coefficient importance and its variability') -plt.xlabel('Coefficient importance') -plt.subplots_adjust(left=.3) +sns.stripplot(data=coefs, orient="h", palette="dark:k", alpha=0.5) +sns.boxplot(data=coefs, orient="h", color="cyan", saturation=0.5) +plt.axvline(x=0, color=".5") +plt.title("Coefficient importance and its variability") +plt.xlabel("Coefficient importance") +plt.suptitle("Ridge model, small regularization, AGE dropped") +plt.subplots_adjust(left=0.3) # %% -# The estimation of the EXPERIENCE coefficient is now less variable and -# remain important for all models trained during cross-validation. +# The estimation of the EXPERIENCE coefficient now shows a much reduced +# variability. EXPERIENCE remains important for all models trained during +# cross-validation. # # .. _scaling_num: # @@ -394,17 +444,16 @@ # # As said above (see ":ref:`the-pipeline`"), we could also choose to scale # numerical values before training the model. -# This can be useful to apply a similar amount regularization to all of them -# in the Ridge. +# This can be useful when we apply a similar amount of regularization to all of them +# in the ridge. # The preprocessor is redefined in order to subtract the mean and scale # variables to unit variance. from sklearn.preprocessing import StandardScaler preprocessor = make_column_transformer( - (OneHotEncoder(drop='if_binary'), categorical_columns), + (OneHotEncoder(drop="if_binary"), categorical_columns), (StandardScaler(), numerical_columns), - remainder='passthrough' ) # %% @@ -413,67 +462,74 @@ model = make_pipeline( preprocessor, TransformedTargetRegressor( - regressor=Ridge(alpha=1e-10), - func=np.log10, - inverse_func=sp.special.exp10 - ) + regressor=Ridge(alpha=1e-10), func=np.log10, inverse_func=sp.special.exp10 + ), ) - -_ = model.fit(X_train, y_train) +model.fit(X_train, y_train) # %% # Again, we check the performance of the computed # model using, for example, the median absolute error of the model and the R # squared coefficient. -y_pred = model.predict(X_train) -mae = median_absolute_error(y_train, y_pred) -string_score = f'MAE on training set: {mae:.2f} $/hour' +mae_train = median_absolute_error(y_train, model.predict(X_train)) y_pred = model.predict(X_test) -mae = median_absolute_error(y_test, y_pred) -string_score += f'\nMAE on testing set: {mae:.2f} $/hour' -fig, ax = plt.subplots(figsize=(6, 6)) -plt.scatter(y_test, y_pred) -ax.plot([0, 1], [0, 1], transform=ax.transAxes, ls="--", c="red") - -plt.text(3, 20, string_score) - -plt.title('Ridge model, small regularization, normalized variables') -plt.ylabel('Model predictions') -plt.xlabel('Truths') -plt.xlim([0, 27]) -_ = plt.ylim([0, 27]) +mae_test = median_absolute_error(y_test, y_pred) +scores = { + "MedAE on training set": f"{mae_train:.2f} $/hour", + "MedAE on testing set": f"{mae_test:.2f} $/hour", +} + +_, ax = plt.subplots(figsize=(5, 5)) +display = PredictionErrorDisplay.from_predictions( + y_test, y_pred, kind="actual_vs_predicted", ax=ax, scatter_kwargs={"alpha": 0.5} +) +ax.set_title("Ridge model, small regularization") +for name, score in scores.items(): + ax.plot([], [], " ", label=f"{name}: {score}") +ax.legend(loc="upper left") +plt.tight_layout() # %% -# For the coefficient analysis, scaling is not needed this time. +# For the coefficient analysis, scaling is not needed this time because it +# was performed during the preprocessing step. coefs = pd.DataFrame( - model.named_steps['transformedtargetregressor'].regressor_.coef_, - columns=['Coefficients'], index=feature_names + model[-1].regressor_.coef_, + columns=["Coefficients importance"], + index=feature_names, ) -coefs.plot(kind='barh', figsize=(9, 7)) -plt.title('Ridge model, small regularization, normalized variables') -plt.axvline(x=0, color='.5') -plt.subplots_adjust(left=.3) +coefs.plot.barh(figsize=(9, 7)) +plt.title("Ridge model, small regularization, normalized variables") +plt.xlabel("Raw coefficient values") +plt.axvline(x=0, color=".5") +plt.subplots_adjust(left=0.3) # %% -# We now inspect the coefficients across several cross-validation folds. +# We now inspect the coefficients across several cross-validation folds. As in +# the above example, we do not need to scale the coefficients by the std. dev. +# of the feature values since this scaling was already +# done in the preprocessing step of the pipeline. cv_model = cross_validate( - model, X, y, cv=RepeatedKFold(n_splits=5, n_repeats=5), - return_estimator=True, n_jobs=-1 + model, + X, + y, + cv=cv, + return_estimator=True, + n_jobs=2, ) coefs = pd.DataFrame( - [est.named_steps['transformedtargetregressor'].regressor_.coef_ - for est in cv_model['estimator']], - columns=feature_names + [est[-1].regressor_.coef_ for est in cv_model["estimator"]], columns=feature_names ) + +# %% plt.figure(figsize=(9, 7)) -sns.stripplot(data=coefs, orient='h', color='k', alpha=0.5) -sns.boxplot(data=coefs, orient='h', color='cyan', saturation=0.5) -plt.axvline(x=0, color='.5') -plt.title('Coefficient variability') -plt.subplots_adjust(left=.3) +sns.stripplot(data=coefs, orient="h", palette="dark:k", alpha=0.5) +sns.boxplot(data=coefs, orient="h", color="cyan", saturation=0.5, whis=10) +plt.axvline(x=0, color=".5") +plt.title("Coefficient variability") +plt.subplots_adjust(left=0.3) # %% # The result is quite similar to the non-normalized case. @@ -481,27 +537,27 @@ # Linear models with regularization # --------------------------------- # -# In machine-learning practice, Ridge Regression is more often used with +# In machine-learning practice, ridge regression is more often used with # non-negligible regularization. # -# Above, we limited this regularization to a very little amount. -# Regularization improves the conditioning of the problem and reduces the -# variance of the estimates. RidgeCV applies cross validation in order to -# determine which value of the regularization parameter (`alpha`) is best -# suited for prediction. +# Above, we limited this regularization to a very little amount. Regularization +# improves the conditioning of the problem and reduces the variance of the +# estimates. :class:`~sklearn.linear_model.RidgeCV` applies cross validation +# in order to determine which value of the regularization parameter (`alpha`) +# is best suited for prediction. from sklearn.linear_model import RidgeCV +alphas = np.logspace(-10, 10, 21) # alpha values to be chosen from by cross-validation model = make_pipeline( preprocessor, TransformedTargetRegressor( - regressor=RidgeCV(alphas=np.logspace(-10, 10, 21)), + regressor=RidgeCV(alphas=alphas), func=np.log10, - inverse_func=sp.special.exp10 - ) + inverse_func=sp.special.exp10, + ), ) - -_ = model.fit(X_train, y_train) +model.fit(X_train, y_train) # %% # First we check which value of :math:`\alpha` has been selected. @@ -510,38 +566,38 @@ # %% # Then we check the quality of the predictions. - -y_pred = model.predict(X_train) -mae = median_absolute_error(y_train, y_pred) -string_score = f'MAE on training set: {mae:.2f} $/hour' +mae_train = median_absolute_error(y_train, model.predict(X_train)) y_pred = model.predict(X_test) -mae = median_absolute_error(y_test, y_pred) -string_score += f'\nMAE on testing set: {mae:.2f} $/hour' - -fig, ax = plt.subplots(figsize=(6, 6)) -plt.scatter(y_test, y_pred) -ax.plot([0, 1], [0, 1], transform=ax.transAxes, ls="--", c="red") - -plt.text(3, 20, string_score) - -plt.title('Ridge model, regularization, normalized variables') -plt.ylabel('Model predictions') -plt.xlabel('Truths') -plt.xlim([0, 27]) -_ = plt.ylim([0, 27]) +mae_test = median_absolute_error(y_test, y_pred) +scores = { + "MedAE on training set": f"{mae_train:.2f} $/hour", + "MedAE on testing set": f"{mae_test:.2f} $/hour", +} + +_, ax = plt.subplots(figsize=(5, 5)) +display = PredictionErrorDisplay.from_predictions( + y_test, y_pred, kind="actual_vs_predicted", ax=ax, scatter_kwargs={"alpha": 0.5} +) +ax.set_title("Ridge model, optimum regularization") +for name, score in scores.items(): + ax.plot([], [], " ", label=f"{name}: {score}") +ax.legend(loc="upper left") +plt.tight_layout() # %% # The ability to reproduce the data of the regularized model is similar to # the one of the non-regularized model. coefs = pd.DataFrame( - model.named_steps['transformedtargetregressor'].regressor_.coef_, - columns=['Coefficients'], index=feature_names + model[-1].regressor_.coef_, + columns=["Coefficients importance"], + index=feature_names, ) -coefs.plot(kind='barh', figsize=(9, 7)) -plt.title('Ridge model, regularization, normalized variables') -plt.axvline(x=0, color='.5') -plt.subplots_adjust(left=.3) +coefs.plot.barh(figsize=(9, 7)) +plt.title("Ridge model, with regularization, normalized variables") +plt.xlabel("Raw coefficient values") +plt.axvline(x=0, color=".5") +plt.subplots_adjust(left=0.3) # %% # The coefficients are significantly different. @@ -553,30 +609,31 @@ # predictive variables, so neither alone would have strong weights. # # On the other hand, the weights obtained with regularization are more -# stable (see the :ref:`ridge_regression` User Guide section). This +# stable (see the :ref:`ridge_regression` User Guide section). This # increased stability is visible from the plot, obtained from data -# perturbations, in a cross validation. This plot can be compared with +# perturbations, in a cross-validation. This plot can be compared with # the :ref:`previous one`. cv_model = cross_validate( - model, X, y, cv=RepeatedKFold(n_splits=5, n_repeats=5), - return_estimator=True, n_jobs=-1 + model, + X, + y, + cv=cv, + return_estimator=True, + n_jobs=2, ) coefs = pd.DataFrame( - [est.named_steps['transformedtargetregressor'].regressor_.coef_ * - X_train_preprocessed.std(axis=0) - for est in cv_model['estimator']], - columns=feature_names + [est[-1].regressor_.coef_ for est in cv_model["estimator"]], columns=feature_names ) -plt.ylabel('Age coefficient') -plt.xlabel('Experience coefficient') +# %% +plt.ylabel("Age coefficient") +plt.xlabel("Experience coefficient") plt.grid(True) plt.xlim(-0.4, 0.5) plt.ylim(-0.4, 0.5) plt.scatter(coefs["AGE"], coefs["EXPERIENCE"]) -_ = plt.title('Co-variations of coefficients for AGE and EXPERIENCE ' - 'across folds') +_ = plt.title("Co-variations of coefficients for AGE and EXPERIENCE across folds") # %% # Linear models with sparse coefficients @@ -584,22 +641,23 @@ # # Another possibility to take into account correlated variables in the dataset, # is to estimate sparse coefficients. In some way we already did it manually -# when we dropped the AGE column in a previous Ridge estimation. +# when we dropped the AGE column in a previous ridge estimation. # # Lasso models (see the :ref:`lasso` User Guide section) estimates sparse -# coefficients. LassoCV applies cross validation in order to -# determine which value of the regularization parameter (`alpha`) is best -# suited for the model estimation. +# coefficients. :class:`~sklearn.linear_model.LassoCV` applies cross +# validation in order to determine which value of the regularization parameter +# (`alpha`) is best suited for the model estimation. from sklearn.linear_model import LassoCV +alphas = np.logspace(-10, 10, 21) # alpha values to be chosen from by cross-validation model = make_pipeline( preprocessor, TransformedTargetRegressor( - regressor=LassoCV(alphas=np.logspace(-10, 10, 21), max_iter=100000), + regressor=LassoCV(alphas=alphas, max_iter=100_000), func=np.log10, - inverse_func=sp.special.exp10 - ) + inverse_func=sp.special.exp10, + ), ) _ = model.fit(X_train, y_train) @@ -612,36 +670,36 @@ # %% # Then we check the quality of the predictions. -y_pred = model.predict(X_train) -mae = median_absolute_error(y_train, y_pred) -string_score = f'MAE on training set: {mae:.2f} $/hour' +mae_train = median_absolute_error(y_train, model.predict(X_train)) y_pred = model.predict(X_test) -mae = median_absolute_error(y_test, y_pred) -string_score += f'\nMAE on testing set: {mae:.2f} $/hour' - -fig, ax = plt.subplots(figsize=(6, 6)) -plt.scatter(y_test, y_pred) -ax.plot([0, 1], [0, 1], transform=ax.transAxes, ls="--", c="red") - -plt.text(3, 20, string_score) - -plt.title('Lasso model, regularization, normalized variables') -plt.ylabel('Model predictions') -plt.xlabel('Truths') -plt.xlim([0, 27]) -_ = plt.ylim([0, 27]) +mae_test = median_absolute_error(y_test, y_pred) +scores = { + "MedAE on training set": f"{mae_train:.2f} $/hour", + "MedAE on testing set": f"{mae_test:.2f} $/hour", +} + +_, ax = plt.subplots(figsize=(6, 6)) +display = PredictionErrorDisplay.from_predictions( + y_test, y_pred, kind="actual_vs_predicted", ax=ax, scatter_kwargs={"alpha": 0.5} +) +ax.set_title("Lasso model, optimum regularization") +for name, score in scores.items(): + ax.plot([], [], " ", label=f"{name}: {score}") +ax.legend(loc="upper left") +plt.tight_layout() # %% # For our dataset, again the model is not very predictive. coefs = pd.DataFrame( - model.named_steps['transformedtargetregressor'].regressor_.coef_, - columns=['Coefficients'], index=feature_names + model[-1].regressor_.coef_, + columns=["Coefficients importance"], + index=feature_names, ) -coefs.plot(kind='barh', figsize=(9, 7)) -plt.title('Lasso model, regularization, normalized variables') -plt.axvline(x=0, color='.5') -plt.subplots_adjust(left=.3) +coefs.plot(kind="barh", figsize=(9, 7)) +plt.title("Lasso model, optimum regularization, normalized variables") +plt.axvline(x=0, color=".5") +plt.subplots_adjust(left=0.3) # %% # A Lasso model identifies the correlation between @@ -650,16 +708,69 @@ # It is important to keep in mind that the coefficients that have been # dropped may still be related to the outcome by themselves: the model # chose to suppress them because they bring little or no additional -# information on top of the other features. Additionnaly, this selection +# information on top of the other features. Additionally, this selection # is unstable for correlated features, and should be interpreted with # caution. # +# Indeed, we can check the variability of the coefficients across folds. +cv_model = cross_validate( + model, + X, + y, + cv=cv, + return_estimator=True, + n_jobs=2, +) +coefs = pd.DataFrame( + [est[-1].regressor_.coef_ for est in cv_model["estimator"]], columns=feature_names +) + +# %% +plt.figure(figsize=(9, 7)) +sns.stripplot(data=coefs, orient="h", palette="dark:k", alpha=0.5) +sns.boxplot(data=coefs, orient="h", color="cyan", saturation=0.5, whis=100) +plt.axvline(x=0, color=".5") +plt.title("Coefficient variability") +plt.subplots_adjust(left=0.3) + +# %% +# We observe that the AGE and EXPERIENCE coefficients are varying a lot +# depending of the fold. +# +# Wrong causal interpretation +# --------------------------- +# +# Policy makers might want to know the effect of education on wage to assess +# whether or not a certain policy designed to entice people to pursue more +# education would make economic sense. While Machine Learning models are great +# for measuring statistical associations, they are generally unable to infer +# causal effects. +# +# It might be tempting to look at the coefficient of education on wage from our +# last model (or any model for that matter) and conclude that it captures the +# true effect of a change in the standardized education variable on wages. +# +# Unfortunately there are likely unobserved confounding variables that either +# inflate or deflate that coefficient. A confounding variable is a variable that +# causes both EDUCATION and WAGE. One example of such variable is ability. +# Presumably, more able people are more likely to pursue education while at the +# same time being more likely to earn a higher hourly wage at any level of +# education. In this case, ability induces a positive `Omitted Variable Bias +# `_ (OVB) on the EDUCATION +# coefficient, thereby exaggerating the effect of education on wages. +# +# See the :ref:`sphx_glr_auto_examples_inspection_plot_causal_interpretation.py` +# for a simulated case of ability OVB. +# # Lessons learned # --------------- # # * Coefficients must be scaled to the same unit of measure to retrieve # feature importance. Scaling them with the standard-deviation of the # feature is a useful proxy. +# * Interpreting causality is difficult when there are confounding effects. If +# the relationship between two variables is also affected by something +# unobserved, we should be careful when making conclusions about causality. # * Coefficients in multivariate linear models represent the dependency # between a given feature and the target, **conditional** on the other # features. @@ -669,3 +780,7 @@ # coefficients could significantly vary from one another. # * Inspecting coefficients across the folds of a cross-validation loop # gives an idea of their stability. +# * Coefficients are unlikely to have any causal meaning. They tend +# to be biased by unobserved confounders. +# * Inspection tools may not necessarily provide insights on the true +# data generating process. diff --git a/examples/inspection/plot_partial_dependence.py b/examples/inspection/plot_partial_dependence.py index 927857d845f9e..d28388a001ea3 100644 --- a/examples/inspection/plot_partial_dependence.py +++ b/examples/inspection/plot_partial_dependence.py @@ -19,79 +19,221 @@ This example shows how to obtain partial dependence and ICE plots from a :class:`~sklearn.neural_network.MLPRegressor` and a :class:`~sklearn.ensemble.HistGradientBoostingRegressor` trained on the -California housing dataset. The example is taken from [1]_. +bike sharing dataset. The example is inspired by [1]_. -.. [1] T. Hastie, R. Tibshirani and J. Friedman, "Elements of Statistical - Learning Ed. 2", Springer, 2009. +.. [1] `Molnar, Christoph. "Interpretable machine learning. + A Guide for Making Black Box Models Explainable", + 2019. `_ .. [2] For classification you can think of it as the regression score before the link function. -.. [3] Goldstein, A., Kapelner, A., Bleich, J., and Pitkin, E., Peeking Inside - the Black Box: Visualizing Statistical Learning With Plots of - Individual Conditional Expectation. (2015) Journal of Computational and - Graphical Statistics, 24(1): 44-65 (https://arxiv.org/abs/1309.6392) +.. [3] :arxiv:`Goldstein, A., Kapelner, A., Bleich, J., and Pitkin, E. (2015). + "Peeking Inside the Black Box: Visualizing Statistical Learning With Plots of + Individual Conditional Expectation". Journal of Computational and + Graphical Statistics, 24(1): 44-65 <1309.6392>` """ -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause # %% -# California Housing data preprocessing -# ------------------------------------- +# Bike sharing dataset preprocessing +# ---------------------------------- # -# Center target to avoid gradient boosting init bias: gradient boosting -# with the 'recursion' method does not account for the initial estimator -# (here the average target, by default). +# We will use the bike sharing dataset. The goal is to predict the number of bike +# rentals using weather and season data as well as the datetime information. +from sklearn.datasets import fetch_openml -import pandas as pd -from sklearn.datasets import fetch_california_housing -from sklearn.model_selection import train_test_split +bikes = fetch_openml("Bike_Sharing_Demand", version=2, as_frame=True) +# Make an explicit copy to avoid "SettingWithCopyWarning" from pandas +X, y = bikes.data.copy(), bikes.target -cal_housing = fetch_california_housing() -X = pd.DataFrame(cal_housing.data, columns=cal_housing.feature_names) -y = cal_housing.target +# We use only a subset of the data to speed up the example. +X = X.iloc[::5, :] +y = y[::5] -y -= y.mean() +# %% +# The feature `"weather"` has a particularity: the category `"heavy_rain"` is a rare +# category. +X["weather"].value_counts() -X_train, X_test, y_train, y_test = train_test_split( - X, y, test_size=0.1, random_state=0 +# %% +# Because of this rare category, we collapse it into `"rain"`. +X["weather"] = ( + X["weather"] + .astype(object) + .replace(to_replace="heavy_rain", value="rain") + .astype("category") ) +# %% +# We now have a closer look at the `"year"` feature: +X["year"].value_counts() + +# %% +# We see that we have data from two years. We use the first year to train the +# model and the second year to test the model. +mask_training = X["year"] == 0.0 +X = X.drop(columns=["year"]) +X_train, y_train = X[mask_training], y[mask_training] +X_test, y_test = X[~mask_training], y[~mask_training] + +# %% +# We can check the dataset information to see that we have heterogeneous data types. We +# have to preprocess the different columns accordingly. +X_train.info() + +# %% +# From the previous information, we will consider the `category` columns as nominal +# categorical features. In addition, we will consider the date and time information as +# categorical features as well. +# +# We manually define the columns containing numerical and categorical +# features. +numerical_features = [ + "temp", + "feel_temp", + "humidity", + "windspeed", +] +categorical_features = X_train.columns.drop(numerical_features) + +# %% +# Before we go into the details regarding the preprocessing of the different machine +# learning pipelines, we will try to get some additional intuition regarding the dataset +# that will be helpful to understand the model's statistical performance and results of +# the partial dependence analysis. +# +# We plot the average number of bike rentals by grouping the data by season and +# by year. +from itertools import product + +import matplotlib.pyplot as plt +import numpy as np + +days = ("Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat") +hours = tuple(range(24)) +xticklabels = [f"{day}\n{hour}:00" for day, hour in product(days, hours)] +xtick_start, xtick_period = 6, 12 + +fig, axs = plt.subplots(nrows=2, figsize=(8, 6), sharey=True, sharex=True) +average_bike_rentals = bikes.frame.groupby( + ["year", "season", "weekday", "hour"], observed=True +).mean(numeric_only=True)["count"] +for ax, (idx, df) in zip(axs, average_bike_rentals.groupby("year")): + df.groupby("season", observed=True).plot(ax=ax, legend=True) + + # decorate the plot + ax.set_xticks( + np.linspace( + start=xtick_start, + stop=len(xticklabels), + num=len(xticklabels) // xtick_period, + ) + ) + ax.set_xticklabels(xticklabels[xtick_start::xtick_period]) + ax.set_xlabel("") + ax.set_ylabel("Average number of bike rentals") + ax.set_title( + f"Bike rental for {'2010 (train set)' if idx == 0.0 else '2011 (test set)'}" + ) + ax.set_ylim(0, 1_000) + ax.set_xlim(0, len(xticklabels)) + ax.legend(loc=2) + +# %% +# The first striking difference between the train and test set is that the number of +# bike rentals is higher in the test set. For this reason, it will not be surprising to +# get a machine learning model that underestimates the number of bike rentals. We +# also observe that the number of bike rentals is lower during the spring season. In +# addition, we see that during working days, there is a specific pattern around 6-7 +# am and 5-6 pm with some peaks of bike rentals. We can keep in mind these different +# insights and use them to understand the partial dependence plot. +# +# Preprocessor for machine-learning models +# ---------------------------------------- +# +# Since we later use two different models, a +# :class:`~sklearn.neural_network.MLPRegressor` and a +# :class:`~sklearn.ensemble.HistGradientBoostingRegressor`, we create two different +# preprocessors, specific for each model. +# +# Preprocessor for the neural network model +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# We will use a :class:`~sklearn.preprocessing.QuantileTransformer` to scale the +# numerical features and encode the categorical features with a +# :class:`~sklearn.preprocessing.OneHotEncoder`. +from sklearn.compose import ColumnTransformer +from sklearn.preprocessing import OneHotEncoder, QuantileTransformer + +mlp_preprocessor = ColumnTransformer( + transformers=[ + ("num", QuantileTransformer(n_quantiles=100), numerical_features), + ("cat", OneHotEncoder(handle_unknown="ignore"), categorical_features), + ] +) +mlp_preprocessor + +# %% +# Preprocessor for the gradient boosting model +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# For the gradient boosting model, we leave the numerical features as-is and only +# encode the categorical features using a +# :class:`~sklearn.preprocessing.OrdinalEncoder`. +from sklearn.preprocessing import OrdinalEncoder + +hgbdt_preprocessor = ColumnTransformer( + transformers=[ + ("cat", OrdinalEncoder(), categorical_features), + ("num", "passthrough", numerical_features), + ], + sparse_threshold=1, + verbose_feature_names_out=False, +).set_output(transform="pandas") +hgbdt_preprocessor + # %% # 1-way partial dependence with different models # ---------------------------------------------- # # In this section, we will compute 1-way partial dependence with two different # machine-learning models: (i) a multi-layer perceptron and (ii) a -# gradient-boosting. With these two models, we illustrate how to compute and -# interpret both partial dependence plot (PDP) and individual conditional -# expectation (ICE). +# gradient-boosting model. With these two models, we illustrate how to compute and +# interpret both partial dependence plot (PDP) for both numerical and categorical +# features and individual conditional expectation (ICE). # # Multi-layer perceptron -# ...................... +# ~~~~~~~~~~~~~~~~~~~~~~ # # Let's fit a :class:`~sklearn.neural_network.MLPRegressor` and compute # single-variable partial dependence plots. - from time import time -from sklearn.pipeline import make_pipeline -from sklearn.preprocessing import QuantileTransformer + from sklearn.neural_network import MLPRegressor +from sklearn.pipeline import make_pipeline print("Training MLPRegressor...") tic = time() -est = make_pipeline(QuantileTransformer(), - MLPRegressor(hidden_layer_sizes=(50, 50), - learning_rate_init=0.01, - early_stopping=True)) -est.fit(X_train, y_train) +mlp_model = make_pipeline( + mlp_preprocessor, + MLPRegressor( + hidden_layer_sizes=(30, 15), + learning_rate_init=0.01, + early_stopping=True, + random_state=0, + ), +) +mlp_model.fit(X_train, y_train) print(f"done in {time() - tic:.3f}s") -print(f"Test R2 score: {est.score(X_test, y_test):.2f}") +print(f"Test R2 score: {mlp_model.score(X_test, y_test):.2f}") # %% -# We configured a pipeline to scale the numerical input features and tuned the -# neural network size and learning rate to get a reasonable compromise between -# training time and predictive performance on a test set. +# We configured a pipeline using the preprocessor that we created specifically for the +# neural network and tuned the neural network size and learning rate to get a reasonable +# compromise between training time and predictive performance on a test set. # # Importantly, this tabular dataset has very different dynamic ranges for its # features. Neural networks tend to be very sensitive to features with varying @@ -104,45 +246,70 @@ # Note that it is important to check that the model is accurate enough on a # test set before plotting the partial dependence since there would be little # use in explaining the impact of a given feature on the prediction function of -# a poor model. +# a model with poor predictive performance. In this regard, our MLP model works +# reasonably well. # -# We will plot the partial dependence, both individual (ICE) and averaged one -# (PDP). We limit to only 50 ICE curves to not overcrowd the plot. - +# We will plot the averaged partial dependence. import matplotlib.pyplot as plt -from sklearn.inspection import partial_dependence -from sklearn.inspection import plot_partial_dependence -print('Computing partial dependence plots...') +from sklearn.inspection import PartialDependenceDisplay + +common_params = { + "subsample": 50, + "n_jobs": 2, + "grid_resolution": 20, + "random_state": 0, +} + +print("Computing partial dependence plots...") +features_info = { + # features of interest + "features": ["temp", "humidity", "windspeed", "season", "weather", "hour"], + # type of partial dependence plot + "kind": "average", + # information regarding categorical features + "categorical_features": categorical_features, +} tic = time() -features = ['MedInc', 'AveOccup', 'HouseAge', 'AveRooms'] -display = plot_partial_dependence( - est, X_train, features, kind="both", subsample=50, - n_jobs=3, grid_resolution=20, random_state=0 +_, ax = plt.subplots(ncols=3, nrows=2, figsize=(9, 8), constrained_layout=True) +display = PartialDependenceDisplay.from_estimator( + mlp_model, + X_train, + **features_info, + ax=ax, + **common_params, ) print(f"done in {time() - tic:.3f}s") -display.figure_.suptitle( - 'Partial dependence of house value on non-location features\n' - 'for the California housing dataset, with MLPRegressor' +_ = display.figure_.suptitle( + ( + "Partial dependence of the number of bike rentals\n" + "for the bike rental dataset with an MLPRegressor" + ), + fontsize=16, ) -display.figure_.subplots_adjust(hspace=0.3) # %% # Gradient boosting -# ................. +# ~~~~~~~~~~~~~~~~~ # # Let's now fit a :class:`~sklearn.ensemble.HistGradientBoostingRegressor` and -# compute the partial dependence on the same features. - -from sklearn.experimental import enable_hist_gradient_boosting # noqa +# compute the partial dependence on the same features. We also use the +# specific preprocessor we created for this model. from sklearn.ensemble import HistGradientBoostingRegressor print("Training HistGradientBoostingRegressor...") tic = time() -est = HistGradientBoostingRegressor() -est.fit(X_train, y_train) +hgbdt_model = make_pipeline( + hgbdt_preprocessor, + HistGradientBoostingRegressor( + categorical_features=categorical_features, + random_state=0, + max_iter=50, + ), +) +hgbdt_model.fit(X_train, y_train) print(f"done in {time() - tic:.3f}s") -print(f"Test R2 score: {est.score(X_test, y_test):.2f}") +print(f"Test R2 score: {hgbdt_model.score(X_test, y_test):.2f}") # %% # Here, we used the default hyperparameters for the gradient boosting model @@ -154,111 +321,298 @@ # also significantly cheaper to tune their hyperparameters (the defaults tend # to work well while this is not often the case for neural networks). # -# We will plot the partial dependence, both individual (ICE) and averaged one -# (PDP). We limit to only 50 ICE curves to not overcrowd the plot. - -print('Computing partial dependence plots...') +# We will plot the partial dependence for some of the numerical and categorical +# features. +print("Computing partial dependence plots...") tic = time() -display = plot_partial_dependence( - est, X_train, features, kind="both", subsample=50, - n_jobs=3, grid_resolution=20, random_state=0 +_, ax = plt.subplots(ncols=3, nrows=2, figsize=(9, 8), constrained_layout=True) +display = PartialDependenceDisplay.from_estimator( + hgbdt_model, + X_train, + **features_info, + ax=ax, + **common_params, ) print(f"done in {time() - tic:.3f}s") -display.figure_.suptitle( - 'Partial dependence of house value on non-location features\n' - 'for the California housing dataset, with Gradient Boosting' +_ = display.figure_.suptitle( + ( + "Partial dependence of the number of bike rentals\n" + "for the bike rental dataset with a gradient boosting" + ), + fontsize=16, ) -display.figure_.subplots_adjust(wspace=0.4, hspace=0.3) # %% # Analysis of the plots -# ..................... -# -# We can clearly see on the PDPs (thick blue line) that the median house price -# shows a linear relationship with the median income (top left) and that the -# house price drops when the average occupants per household increases (top -# middle). The top right plot shows that the house age in a district does not -# have a strong influence on the (median) house price; so does the average -# rooms per household. -# -# The ICE curves (light blue lines) complement the analysis: we can see that -# there are some exceptions, where the house price remain constant with median -# income and average occupants. On the other hand, while the house age (top -# right) does not have a strong influence on the median house price on average, -# there seems to be a number of exceptions where the house price increase when -# between the ages 15-25. Similar exceptions can be observed for the average -# number of rooms (bottom left). Therefore, ICE plots show some individual -# effect which are attenuated by taking the averages. -# -# In all plots, the tick marks on the x-axis represent the deciles of the -# feature values in the training data. -# -# We also observe that :class:`~sklearn.neural_network.MLPRegressor` has much -# smoother predictions than -# :class:`~sklearn.ensemble.HistGradientBoostingRegressor`. +# ~~~~~~~~~~~~~~~~~~~~~ +# +# We will first look at the PDPs for the numerical features. For both models, the +# general trend of the PDP of the temperature is that the number of bike rentals is +# increasing with temperature. We can make a similar analysis but with the opposite +# trend for the humidity features. The number of bike rentals is decreasing when the +# humidity increases. Finally, we see the same trend for the wind speed feature. The +# number of bike rentals is decreasing when the wind speed is increasing for both +# models. We also observe that :class:`~sklearn.neural_network.MLPRegressor` has much +# smoother predictions than :class:`~sklearn.ensemble.HistGradientBoostingRegressor`. +# +# Now, we will look at the partial dependence plots for the categorical features. +# +# We observe that the spring season is the lowest bar for the season feature. With the +# weather feature, the rain category is the lowest bar. Regarding the hour feature, +# we see two peaks around the 7 am and 6 pm. These findings are in line with the +# the observations we made earlier on the dataset. # # However, it is worth noting that we are creating potential meaningless # synthetic samples if features are correlated. +# +# .. _ice-vs-pdp: +# +# ICE vs. PDP +# ~~~~~~~~~~~ +# +# PDP is an average of the marginal effects of the features. We are averaging the +# response of all samples of the provided set. Thus, some effects could be hidden. In +# this regard, it is possible to plot each individual response. This representation is +# called the Individual Effect Plot (ICE). In the plot below, we plot 50 randomly +# selected ICEs for the temperature and humidity features. +print("Computing partial dependence plots and individual conditional expectation...") +tic = time() +_, ax = plt.subplots(ncols=2, figsize=(6, 4), sharey=True, constrained_layout=True) + +features_info = { + "features": ["temp", "humidity"], + "kind": "both", + "centered": True, +} + +display = PartialDependenceDisplay.from_estimator( + hgbdt_model, + X_train, + **features_info, + ax=ax, + **common_params, +) +print(f"done in {time() - tic:.3f}s") +_ = display.figure_.suptitle("ICE and PDP representations", fontsize=16) + +# %% +# We see that the ICE for the temperature feature gives us some additional information: +# Some of the ICE lines are flat while some others show a decrease of the dependence +# for temperature above 35 degrees Celsius. We observe a similar pattern for the +# humidity feature: some of the ICEs lines show a sharp decrease when the humidity is +# above 80%. +# +# Not all ICE lines are parallel, this indicates that the model finds +# interactions between features. We can repeat the experiment by constraining the +# gradient boosting model to not use any interactions between features using the +# parameter `interaction_cst`: +from sklearn.base import clone + +interaction_cst = [[i] for i in range(X_train.shape[1])] +hgbdt_model_without_interactions = ( + clone(hgbdt_model) + .set_params(histgradientboostingregressor__interaction_cst=interaction_cst) + .fit(X_train, y_train) +) +print(f"Test R2 score: {hgbdt_model_without_interactions.score(X_test, y_test):.2f}") + +# %% +_, ax = plt.subplots(ncols=2, figsize=(6, 4), sharey=True, constrained_layout=True) + +features_info["centered"] = False +display = PartialDependenceDisplay.from_estimator( + hgbdt_model_without_interactions, + X_train, + **features_info, + ax=ax, + **common_params, +) +_ = display.figure_.suptitle("ICE and PDP representations", fontsize=16) # %% # 2D interaction plots # -------------------- # -# PDPs with two features of interest enable us to visualize interactions among -# them. However, ICEs cannot be plotted in an easy manner and thus interpreted. -# Another consideration is linked to the performance to compute the PDPs. With -# the tree-based algorithm, when only PDPs are requested, they can be computed -# on an efficient way using the `'recursion'` method. +# PDPs with two features of interest enable us to visualize interactions among them. +# However, ICEs cannot be plotted in an easy manner and thus interpreted. We will show +# the representation of available in +# :meth:`~sklearn.inspection.PartialDependenceDisplay.from_estimator` that is a 2D +# heatmap. +print("Computing partial dependence plots...") +features_info = { + "features": ["temp", "humidity", ("temp", "humidity")], + "kind": "average", +} +_, ax = plt.subplots(ncols=3, figsize=(10, 4), constrained_layout=True) +tic = time() +display = PartialDependenceDisplay.from_estimator( + hgbdt_model, + X_train, + **features_info, + ax=ax, + **common_params, +) +print(f"done in {time() - tic:.3f}s") +_ = display.figure_.suptitle( + "1-way vs 2-way of numerical PDP using gradient boosting", fontsize=16 +) -features = ['AveOccup', 'HouseAge', ('AveOccup', 'HouseAge')] -print('Computing partial dependence plots...') +# %% +# The two-way partial dependence plot shows the dependence of the number of bike rentals +# on joint values of temperature and humidity. +# We clearly see an interaction between the two features. For a temperature higher than +# 20 degrees Celsius, the humidity has a impact on the number of bike rentals +# that seems independent on the temperature. +# +# On the other hand, for temperatures lower than 20 degrees Celsius, both the +# temperature and humidity continuously impact the number of bike rentals. +# +# Furthermore, the slope of the of the impact ridge of the 20 degrees Celsius +# threshold is very dependent on the humidity level: the ridge is steep under +# dry conditions but much smoother under wetter conditions above 70% of humidity. +# +# We now contrast those results with the same plots computed for the model +# constrained to learn a prediction function that does not depend on such +# non-linear feature interactions. +print("Computing partial dependence plots...") +features_info = { + "features": ["temp", "humidity", ("temp", "humidity")], + "kind": "average", +} +_, ax = plt.subplots(ncols=3, figsize=(10, 4), constrained_layout=True) tic = time() -_, ax = plt.subplots(ncols=3, figsize=(9, 4)) -display = plot_partial_dependence( - est, X_train, features, kind='average', n_jobs=3, grid_resolution=20, +display = PartialDependenceDisplay.from_estimator( + hgbdt_model_without_interactions, + X_train, + **features_info, ax=ax, + **common_params, ) print(f"done in {time() - tic:.3f}s") -display.figure_.suptitle( - 'Partial dependence of house value on non-location features\n' - 'for the California housing dataset, with Gradient Boosting' +_ = display.figure_.suptitle( + "1-way vs 2-way of numerical PDP using gradient boosting", fontsize=16 ) -display.figure_.subplots_adjust(wspace=0.4, hspace=0.3) # %% -# The two-way partial dependence plot shows the dependence of median house -# price on joint values of house age and average occupants per household. We -# can clearly see an interaction between the two features: for an average -# occupancy greater than two, the house price is nearly independent of the -# house age, whereas for values less than two there is a strong dependence on -# age. -# -# 3D interaction plots -# -------------------- +# The 1D partial dependence plots for the model constrained to not model feature +# interactions show local spikes for each features individually, in particular for +# for the "humidity" feature. Those spikes might be reflecting a degraded behavior +# of the model that attempts to somehow compensate for the forbidden interactions +# by overfitting particular training points. Note that the predictive performance +# of this model as measured on the test set is significantly worse than that of +# the original, unconstrained model. +# +# Also note that the number of local spikes visible on those plots is depends on +# the grid resolution parameter of the PD plot itself. +# +# Those local spikes result in a noisily gridded 2D PD plot. It is quite +# challenging to tell whether or not there are no interaction between those +# features because of the high frequency oscillations in the humidity feature. +# However it can clearly be seen that the simple interaction effect observed when +# the temperature crosses the 20 degrees boundary is no longer visible for this +# model. +# +# The partial dependence between categorical features will provide a discrete +# representation that can be shown as a heatmap. For instance the interaction between +# the season, the weather, and the target would be as follow: +print("Computing partial dependence plots...") +features_info = { + "features": ["season", "weather", ("season", "weather")], + "kind": "average", + "categorical_features": categorical_features, +} +_, ax = plt.subplots(ncols=3, figsize=(14, 6), constrained_layout=True) +tic = time() +display = PartialDependenceDisplay.from_estimator( + hgbdt_model, + X_train, + **features_info, + ax=ax, + **common_params, +) + +print(f"done in {time() - tic:.3f}s") +_ = display.figure_.suptitle( + "1-way vs 2-way PDP of categorical features using gradient boosting", fontsize=16 +) + +# %% +# 3D representation +# ~~~~~~~~~~~~~~~~~ # # Let's make the same partial dependence plot for the 2 features interaction, # this time in 3 dimensions. +# unused but required import for doing 3d projections with matplotlib < 3.2 +import mpl_toolkits.mplot3d # noqa: F401 import numpy as np -from mpl_toolkits.mplot3d import Axes3D -fig = plt.figure() -features = ('AveOccup', 'HouseAge') +from sklearn.inspection import partial_dependence + +fig = plt.figure(figsize=(5.5, 5)) + +features = ("temp", "humidity") pdp = partial_dependence( - est, X_train, features=features, kind='average', grid_resolution=20 + hgbdt_model, X_train, features=features, kind="average", grid_resolution=10 ) -XX, YY = np.meshgrid(pdp["values"][0], pdp["values"][1]) +XX, YY = np.meshgrid(pdp["grid_values"][0], pdp["grid_values"][1]) Z = pdp.average[0].T -ax = Axes3D(fig) -surf = ax.plot_surface(XX, YY, Z, rstride=1, cstride=1, - cmap=plt.cm.BuPu, edgecolor='k') +ax = fig.add_subplot(projection="3d") +fig.add_axes(ax) + +surf = ax.plot_surface(XX, YY, Z, rstride=1, cstride=1, cmap=plt.cm.BuPu, edgecolor="k") ax.set_xlabel(features[0]) ax.set_ylabel(features[1]) -ax.set_zlabel('Partial dependence') +fig.suptitle( + "PD of number of bike rentals on\nthe temperature and humidity GBDT model", + fontsize=16, +) # pretty init view ax.view_init(elev=22, azim=122) -plt.colorbar(surf) -plt.suptitle('Partial dependence of house value on median\n' - 'age and average occupancy, with Gradient Boosting') -plt.subplots_adjust(top=0.9) +clb = plt.colorbar(surf, pad=0.08, shrink=0.6, aspect=10) +clb.ax.set_title("Partial\ndependence") plt.show() + +# %% +# .. _plt_partial_dependence_custom_values: +# +# Custom Inspection Points +# ~~~~~~~~~~~~~~~~~~~~~~~~ +# +# None of the examples so far specify _which_ points are evaluated to create the +# partial dependence plots. By default we use percentiles defined by the input dataset. +# In some cases it can be helpful to specify the exact points where you would like the +# model evaluated. For instance, if a user wants to test the model behavior on +# out-of-distribution data or compare two models that were fit on slightly different +# data. The `custom_values` parameter allows the user to pass in the values that they +# want the model to be evaluated on. This overrides the `grid_resolution` and +# `percentiles` parameters. Let's return to our gradient boosting example above +# but with custom values + +print("Computing partial dependence plots with custom evaluation values...") +tic = time() +_, ax = plt.subplots(ncols=2, figsize=(6, 4), sharey=True, constrained_layout=True) + +features_info = { + "features": ["temp", "humidity"], + "kind": "both", +} + +display = PartialDependenceDisplay.from_estimator( + hgbdt_model, + X_train, + **features_info, + ax=ax, + **common_params, + # we set custom values for temp feature - + # all other features are evaluated based on the data + custom_values={"temp": np.linspace(0, 40, 10)}, +) +print(f"done in {time() - tic:.3f}s") +_ = display.figure_.suptitle( + ( + "Partial dependence of the number of bike rentals\n" + "for the bike rental dataset with a gradient boosting" + ), + fontsize=16, +) diff --git a/examples/inspection/plot_permutation_importance.py b/examples/inspection/plot_permutation_importance.py index daa75453a0b69..529e82302e61c 100644 --- a/examples/inspection/plot_permutation_importance.py +++ b/examples/inspection/plot_permutation_importance.py @@ -18,24 +18,15 @@ This example shows how to use Permutation Importances as an alternative that can mitigate those limitations. -.. topic:: References: +.. rubric:: References - [1] L. Breiman, "Random Forests", Machine Learning, 45(1), 5-32, - 2001. https://doi.org/10.1023/A:1010933404324 -""" -print(__doc__) -import matplotlib.pyplot as plt -import numpy as np +* :doi:`L. Breiman, "Random Forests", Machine Learning, 45(1), 5-32, + 2001. <10.1023/A:1010933404324>` -from sklearn.datasets import fetch_openml -from sklearn.ensemble import RandomForestClassifier -from sklearn.impute import SimpleImputer -from sklearn.inspection import permutation_importance -from sklearn.compose import ColumnTransformer -from sklearn.model_selection import train_test_split -from sklearn.pipeline import Pipeline -from sklearn.preprocessing import OneHotEncoder +""" +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause # %% # Data Loading and Feature Engineering @@ -50,42 +41,69 @@ # values as records). # - ``random_cat`` is a low cardinality categorical variable (3 possible # values). +import numpy as np + +from sklearn.datasets import fetch_openml +from sklearn.model_selection import train_test_split + X, y = fetch_openml("titanic", version=1, as_frame=True, return_X_y=True) rng = np.random.RandomState(seed=42) -X['random_cat'] = rng.randint(3, size=X.shape[0]) -X['random_num'] = rng.randn(X.shape[0]) +X["random_cat"] = rng.randint(3, size=X.shape[0]) +X["random_num"] = rng.randn(X.shape[0]) -categorical_columns = ['pclass', 'sex', 'embarked', 'random_cat'] -numerical_columns = ['age', 'sibsp', 'parch', 'fare', 'random_num'] +categorical_columns = ["pclass", "sex", "embarked", "random_cat"] +numerical_columns = ["age", "sibsp", "parch", "fare", "random_num"] X = X[categorical_columns + numerical_columns] +X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=42) -X_train, X_test, y_train, y_test = train_test_split( - X, y, stratify=y, random_state=42) +# %% +# We define a predictive model based on a random forest. Therefore, we will make +# the following preprocessing steps: +# +# - use :class:`~sklearn.preprocessing.OrdinalEncoder` to encode the +# categorical features; +# - use :class:`~sklearn.impute.SimpleImputer` to fill missing values for +# numerical features using a mean strategy. +from sklearn.compose import ColumnTransformer +from sklearn.ensemble import RandomForestClassifier +from sklearn.impute import SimpleImputer +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import OrdinalEncoder -categorical_encoder = OneHotEncoder(handle_unknown='ignore') -numerical_pipe = Pipeline([ - ('imputer', SimpleImputer(strategy='mean')) -]) +categorical_encoder = OrdinalEncoder( + handle_unknown="use_encoded_value", unknown_value=-1, encoded_missing_value=-1 +) +numerical_pipe = SimpleImputer(strategy="mean") preprocessing = ColumnTransformer( - [('cat', categorical_encoder, categorical_columns), - ('num', numerical_pipe, numerical_columns)]) - -rf = Pipeline([ - ('preprocess', preprocessing), - ('classifier', RandomForestClassifier(random_state=42)) -]) + [ + ("cat", categorical_encoder, categorical_columns), + ("num", numerical_pipe, numerical_columns), + ], + verbose_feature_names_out=False, +) + +rf = Pipeline( + [ + ("preprocess", preprocessing), + ("classifier", RandomForestClassifier(random_state=42)), + ] +) rf.fit(X_train, y_train) # %% # Accuracy of the Model # --------------------- -# Prior to inspecting the feature importances, it is important to check that -# the model predictive performance is high enough. Indeed there would be little -# interest of inspecting the important features of a non-predictive model. -# -# Here one can observe that the train accuracy is very high (the forest model +# Before inspecting the feature importances, it is important to check that +# the model predictive performance is high enough. Indeed, there would be little +# interest in inspecting the important features of a non-predictive model. + +print(f"RF train accuracy: {rf.score(X_train, y_train):.3f}") +print(f"RF test accuracy: {rf.score(X_test, y_test):.3f}") + +# %% +# Here, one can observe that the train accuracy is very high (the forest model # has enough capacity to completely memorize the training set) but it can still # generalize well enough to the test set thanks to the built-in bagging of # random forests. @@ -96,19 +114,16 @@ # ``min_samples_leaf=10``) so as to limit overfitting while not introducing too # much underfitting. # -# However let's keep our high capacity random forest model for now so as to -# illustrate some pitfalls with feature importance on variables with many +# However, let us keep our high capacity random forest model for now so that we can +# illustrate some pitfalls about feature importance on variables with many # unique values. -print("RF train accuracy: %0.3f" % rf.score(X_train, y_train)) -print("RF test accuracy: %0.3f" % rf.score(X_test, y_test)) - # %% # Tree's Feature Importance from Mean Decrease in Impurity (MDI) # -------------------------------------------------------------- # The impurity-based feature importance ranks the numerical features to be the # most important features. As a result, the non-predictive ``random_num`` -# variable is ranked the most important! +# variable is ranked as one of the most important features! # # This problem stems from two limitations of impurity-based feature # importances: @@ -118,57 +133,114 @@ # therefore do not reflect the ability of feature to be useful to make # predictions that generalize to the test set (when the model has enough # capacity). -ohe = (rf.named_steps['preprocess'] - .named_transformers_['cat']) -feature_names = ohe.get_feature_names(input_features=categorical_columns) -feature_names = np.r_[feature_names, numerical_columns] - -tree_feature_importances = ( - rf.named_steps['classifier'].feature_importances_) -sorted_idx = tree_feature_importances.argsort() - -y_ticks = np.arange(0, len(feature_names)) -fig, ax = plt.subplots() -ax.barh(y_ticks, tree_feature_importances[sorted_idx]) -ax.set_yticks(y_ticks) -ax.set_yticklabels(feature_names[sorted_idx]) -ax.set_title("Random Forest Feature Importances (MDI)") -fig.tight_layout() -plt.show() +# +# The bias towards high cardinality features explains why the `random_num` has +# a really large importance in comparison with `random_cat` while we would +# expect that both random features have a null importance. +# +# The fact that we use training set statistics explains why both the +# `random_num` and `random_cat` features have a non-null importance. +import pandas as pd +feature_names = rf[:-1].get_feature_names_out() + +mdi_importances = pd.Series( + rf[-1].feature_importances_, index=feature_names +).sort_values(ascending=True) + +# %% +ax = mdi_importances.plot.barh() +ax.set_title("Random Forest Feature Importances (MDI)") +ax.figure.tight_layout() # %% # As an alternative, the permutation importances of ``rf`` are computed on a # held out test set. This shows that the low cardinality categorical feature, -# ``sex`` is the most important feature. +# `sex` and `pclass` are the most important features. Indeed, permuting the +# values of these features will lead to the most decrease in accuracy score of the +# model on the test set. # -# Also note that both random features have very low importances (close to 0) as +# Also, note that both random features have very low importances (close to 0) as # expected. -result = permutation_importance(rf, X_test, y_test, n_repeats=10, - random_state=42, n_jobs=2) -sorted_idx = result.importances_mean.argsort() +from sklearn.inspection import permutation_importance + +result = permutation_importance( + rf, X_test, y_test, n_repeats=10, random_state=42, n_jobs=2 +) -fig, ax = plt.subplots() -ax.boxplot(result.importances[sorted_idx].T, - vert=False, labels=X_test.columns[sorted_idx]) +sorted_importances_idx = result.importances_mean.argsort() +importances = pd.DataFrame( + result.importances[sorted_importances_idx].T, + columns=X.columns[sorted_importances_idx], +) +ax = importances.plot.box(vert=False, whis=10) ax.set_title("Permutation Importances (test set)") -fig.tight_layout() -plt.show() +ax.axvline(x=0, color="k", linestyle="--") +ax.set_xlabel("Decrease in accuracy score") +ax.figure.tight_layout() # %% # It is also possible to compute the permutation importances on the training -# set. This reveals that ``random_num`` gets a significantly higher importance -# ranking than when computed on the test set. The difference between those two -# plots is a confirmation that the RF model has enough capacity to use that -# random numerical feature to overfit. You can further confirm this by -# re-running this example with constrained RF with min_samples_leaf=10. -result = permutation_importance(rf, X_train, y_train, n_repeats=10, - random_state=42, n_jobs=2) -sorted_idx = result.importances_mean.argsort() - -fig, ax = plt.subplots() -ax.boxplot(result.importances[sorted_idx].T, - vert=False, labels=X_train.columns[sorted_idx]) +# set. This reveals that `random_num` and `random_cat` get a significantly +# higher importance ranking than when computed on the test set. The difference +# between those two plots is a confirmation that the RF model has enough +# capacity to use that random numerical and categorical features to overfit. +result = permutation_importance( + rf, X_train, y_train, n_repeats=10, random_state=42, n_jobs=2 +) + +sorted_importances_idx = result.importances_mean.argsort() +importances = pd.DataFrame( + result.importances[sorted_importances_idx].T, + columns=X.columns[sorted_importances_idx], +) +ax = importances.plot.box(vert=False, whis=10) ax.set_title("Permutation Importances (train set)") -fig.tight_layout() -plt.show() +ax.axvline(x=0, color="k", linestyle="--") +ax.set_xlabel("Decrease in accuracy score") +ax.figure.tight_layout() + +# %% +# We can further retry the experiment by limiting the capacity of the trees +# to overfit by setting `min_samples_leaf` at 20 data points. +rf.set_params(classifier__min_samples_leaf=20).fit(X_train, y_train) + +# %% +# Observing the accuracy score on the training and testing set, we observe that +# the two metrics are very similar now. Therefore, our model is not overfitting +# anymore. We can then check the permutation importances with this new model. +print(f"RF train accuracy: {rf.score(X_train, y_train):.3f}") +print(f"RF test accuracy: {rf.score(X_test, y_test):.3f}") + +# %% +train_result = permutation_importance( + rf, X_train, y_train, n_repeats=10, random_state=42, n_jobs=2 +) +test_results = permutation_importance( + rf, X_test, y_test, n_repeats=10, random_state=42, n_jobs=2 +) +sorted_importances_idx = train_result.importances_mean.argsort() + +# %% +train_importances = pd.DataFrame( + train_result.importances[sorted_importances_idx].T, + columns=X.columns[sorted_importances_idx], +) +test_importances = pd.DataFrame( + test_results.importances[sorted_importances_idx].T, + columns=X.columns[sorted_importances_idx], +) + +# %% +for name, importances in zip(["train", "test"], [train_importances, test_importances]): + ax = importances.plot.box(vert=False, whis=10) + ax.set_title(f"Permutation Importances ({name} set)") + ax.set_xlabel("Decrease in accuracy score") + ax.axvline(x=0, color="k", linestyle="--") + ax.figure.tight_layout() + +# %% +# Now, we can observe that on both sets, the `random_num` and `random_cat` +# features have a lower importance compared to the overfitting random forest. +# However, the conclusions regarding the importance of the other features are +# still valid. diff --git a/examples/inspection/plot_permutation_importance_multicollinear.py b/examples/inspection/plot_permutation_importance_multicollinear.py index bde3710519667..2924021281035 100644 --- a/examples/inspection/plot_permutation_importance_multicollinear.py +++ b/examples/inspection/plot_permutation_importance_multicollinear.py @@ -3,112 +3,190 @@ Permutation Importance with Multicollinear or Correlated Features ================================================================= -In this example, we compute the permutation importance on the Wisconsin -breast cancer dataset using :func:`~sklearn.inspection.permutation_importance`. -The :class:`~sklearn.ensemble.RandomForestClassifier` can easily get about 97% -accuracy on a test dataset. Because this dataset contains multicollinear -features, the permutation importance will show that none of the features are -important. One approach to handling multicollinearity is by performing +In this example, we compute the +:func:`~sklearn.inspection.permutation_importance` of the features to a trained +:class:`~sklearn.ensemble.RandomForestClassifier` using the +:ref:`breast_cancer_dataset`. The model can easily get about 97% accuracy on a +test dataset. Because this dataset contains multicollinear features, the +permutation importance shows that none of the features are important, in +contradiction with the high test accuracy. + +We demo a possible approach to handling multicollinearity, which consists of hierarchical clustering on the features' Spearman rank-order correlations, picking a threshold, and keeping a single feature from each cluster. .. note:: See also :ref:`sphx_glr_auto_examples_inspection_plot_permutation_importance.py` + """ -print(__doc__) -from collections import defaultdict -import matplotlib.pyplot as plt -import numpy as np -from scipy.stats import spearmanr -from scipy.cluster import hierarchy +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Random Forest Feature Importance on Breast Cancer Data +# ------------------------------------------------------ +# +# First, we define a function to ease the plotting: +import matplotlib +from sklearn.inspection import permutation_importance +from sklearn.utils.fixes import parse_version + + +def plot_permutation_importance(clf, X, y, ax): + result = permutation_importance(clf, X, y, n_repeats=10, random_state=42, n_jobs=2) + perm_sorted_idx = result.importances_mean.argsort() + + # `labels` argument in boxplot is deprecated in matplotlib 3.9 and has been + # renamed to `tick_labels`. The following code handles this, but as a + # scikit-learn user you probably can write simpler code by using `labels=...` + # (matplotlib < 3.9) or `tick_labels=...` (matplotlib >= 3.9). + tick_labels_parameter_name = ( + "tick_labels" + if parse_version(matplotlib.__version__) >= parse_version("3.9") + else "labels" + ) + tick_labels_dict = {tick_labels_parameter_name: X.columns[perm_sorted_idx]} + ax.boxplot(result.importances[perm_sorted_idx].T, vert=False, **tick_labels_dict) + ax.axvline(x=0, color="k", linestyle="--") + return ax + + +# %% +# We then train a :class:`~sklearn.ensemble.RandomForestClassifier` on the +# :ref:`breast_cancer_dataset` and evaluate its accuracy on a test set: from sklearn.datasets import load_breast_cancer from sklearn.ensemble import RandomForestClassifier -from sklearn.inspection import permutation_importance from sklearn.model_selection import train_test_split -# %% -# Random Forest Feature Importance on Breast Cancer Data -# ------------------------------------------------------ -# First, we train a random forest on the breast cancer dataset and evaluate -# its accuracy on a test set: -data = load_breast_cancer() -X, y = data.data, data.target +X, y = load_breast_cancer(return_X_y=True, as_frame=True) X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42) clf = RandomForestClassifier(n_estimators=100, random_state=42) clf.fit(X_train, y_train) -print("Accuracy on test data: {:.2f}".format(clf.score(X_test, y_test))) +print(f"Baseline accuracy on test data: {clf.score(X_test, y_test):.2}") # %% # Next, we plot the tree based feature importance and the permutation -# importance. The permutation importance plot shows that permuting a feature -# drops the accuracy by at most `0.012`, which would suggest that none of the -# features are important. This is in contradiction with the high test accuracy -# computed above: some feature must be important. The permutation importance -# is calculated on the training set to show how much the model relies on each -# feature during training. -result = permutation_importance(clf, X_train, y_train, n_repeats=10, - random_state=42) -perm_sorted_idx = result.importances_mean.argsort() +# importance. The permutation importance is calculated on the training set to +# show how much the model relies on each feature during training. +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +mdi_importances = pd.Series(clf.feature_importances_, index=X_train.columns) tree_importance_sorted_idx = np.argsort(clf.feature_importances_) -tree_indices = np.arange(0, len(clf.feature_importances_)) + 0.5 fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 8)) -ax1.barh(tree_indices, - clf.feature_importances_[tree_importance_sorted_idx], height=0.7) -ax1.set_yticks(tree_indices) -ax1.set_yticklabels(data.feature_names[tree_importance_sorted_idx]) -ax1.set_ylim((0, len(clf.feature_importances_))) -ax2.boxplot(result.importances[perm_sorted_idx].T, vert=False, - labels=data.feature_names[perm_sorted_idx]) -fig.tight_layout() -plt.show() +mdi_importances.sort_values().plot.barh(ax=ax1) +ax1.set_xlabel("Gini importance") +plot_permutation_importance(clf, X_train, y_train, ax2) +ax2.set_xlabel("Decrease in accuracy score") +fig.suptitle( + "Impurity-based vs. permutation importances on multicollinear features (train set)" +) +_ = fig.tight_layout() + +# %% +# The plot on the left shows the Gini importance of the model. As the +# scikit-learn implementation of +# :class:`~sklearn.ensemble.RandomForestClassifier` uses a random subsets of +# :math:`\sqrt{n_\text{features}}` features at each split, it is able to dilute +# the dominance of any single correlated feature. As a result, the individual +# feature importance may be distributed more evenly among the correlated +# features. Since the features have large cardinality and the classifier is +# non-overfitted, we can relatively trust those values. +# +# The permutation importance on the right plot shows that permuting a feature +# drops the accuracy by at most `0.012`, which would suggest that none of the +# features are important. This is in contradiction with the high test accuracy +# computed as baseline: some feature must be important. +# +# Similarly, the change in accuracy score computed on the test set appears to be +# driven by chance: + +fig, ax = plt.subplots(figsize=(7, 6)) +plot_permutation_importance(clf, X_test, y_test, ax) +ax.set_title("Permutation Importances on multicollinear features\n(test set)") +ax.set_xlabel("Decrease in accuracy score") +_ = ax.figure.tight_layout() # %% +# Nevertheless, one can still compute a meaningful permutation importance in the +# presence of correlated features, as demonstrated in the following section. +# # Handling Multicollinear Features # -------------------------------- -# When features are collinear, permutating one feature will have little -# effect on the models performance because it can get the same information -# from a correlated feature. One way to handle multicollinear features is by -# performing hierarchical clustering on the Spearman rank-order correlations, -# picking a threshold, and keeping a single feature from each cluster. First, -# we plot a heatmap of the correlated features: +# When features are collinear, permuting one feature has little effect on the +# models performance because it can get the same information from a correlated +# feature. Note that this is not the case for all predictive models and depends +# on their underlying implementation. +# +# One way to handle multicollinear features is by performing hierarchical +# clustering on the Spearman rank-order correlations, picking a threshold, and +# keeping a single feature from each cluster. First, we plot a heatmap of the +# correlated features: +from scipy.cluster import hierarchy +from scipy.spatial.distance import squareform +from scipy.stats import spearmanr + fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 8)) corr = spearmanr(X).correlation -corr_linkage = hierarchy.ward(corr) + +# Ensure the correlation matrix is symmetric +corr = (corr + corr.T) / 2 +np.fill_diagonal(corr, 1) + +# We convert the correlation matrix to a distance matrix before performing +# hierarchical clustering using Ward's linkage. +distance_matrix = 1 - np.abs(corr) +dist_linkage = hierarchy.ward(squareform(distance_matrix)) dendro = hierarchy.dendrogram( - corr_linkage, labels=data.feature_names.tolist(), ax=ax1, leaf_rotation=90 + dist_linkage, labels=X.columns.to_list(), ax=ax1, leaf_rotation=90 ) -dendro_idx = np.arange(0, len(dendro['ivl'])) +dendro_idx = np.arange(0, len(dendro["ivl"])) -ax2.imshow(corr[dendro['leaves'], :][:, dendro['leaves']]) +ax2.imshow(corr[dendro["leaves"], :][:, dendro["leaves"]]) ax2.set_xticks(dendro_idx) ax2.set_yticks(dendro_idx) -ax2.set_xticklabels(dendro['ivl'], rotation='vertical') -ax2.set_yticklabels(dendro['ivl']) -fig.tight_layout() -plt.show() +ax2.set_xticklabels(dendro["ivl"], rotation="vertical") +ax2.set_yticklabels(dendro["ivl"]) +_ = fig.tight_layout() # %% -# Next, we manually pick a threshold by visual inspection of the dendrogram -# to group our features into clusters and choose a feature from each cluster to +# Next, we manually pick a threshold by visual inspection of the dendrogram to +# group our features into clusters and choose a feature from each cluster to # keep, select those features from our dataset, and train a new random forest. -# The test accuracy of the new random forest did not change much compared to -# the random forest trained on the complete dataset. -cluster_ids = hierarchy.fcluster(corr_linkage, 1, criterion='distance') +# The test accuracy of the new random forest did not change much compared to the +# random forest trained on the complete dataset. +from collections import defaultdict + +cluster_ids = hierarchy.fcluster(dist_linkage, 1, criterion="distance") cluster_id_to_feature_ids = defaultdict(list) for idx, cluster_id in enumerate(cluster_ids): cluster_id_to_feature_ids[cluster_id].append(idx) selected_features = [v[0] for v in cluster_id_to_feature_ids.values()] +selected_features_names = X.columns[selected_features] -X_train_sel = X_train[:, selected_features] -X_test_sel = X_test[:, selected_features] +X_train_sel = X_train[selected_features_names] +X_test_sel = X_test[selected_features_names] clf_sel = RandomForestClassifier(n_estimators=100, random_state=42) clf_sel.fit(X_train_sel, y_train) -print("Accuracy on test data with features removed: {:.2f}".format( - clf_sel.score(X_test_sel, y_test))) +print( + "Baseline accuracy on test data with features removed:" + f" {clf_sel.score(X_test_sel, y_test):.2}" +) + +# %% +# We can finally explore the permutation importance of the selected subset of +# features: + +fig, ax = plt.subplots(figsize=(7, 6)) +plot_permutation_importance(clf_sel, X_test_sel, y_test, ax) +ax.set_title("Permutation Importances on selected subset of features\n(test set)") +ax.set_xlabel("Decrease in accuracy score") +ax.figure.tight_layout() +plt.show() diff --git a/examples/kernel_approximation/plot_scalable_poly_kernels.py b/examples/kernel_approximation/plot_scalable_poly_kernels.py index 845ba1fdf3050..c589755a259eb 100644 --- a/examples/kernel_approximation/plot_scalable_poly_kernels.py +++ b/examples/kernel_approximation/plot_scalable_poly_kernels.py @@ -1,15 +1,15 @@ """ -======================================================= -Scalable learning with polynomial kernel aproximation -======================================================= +====================================================== +Scalable learning with polynomial kernel approximation +====================================================== + +.. currentmodule:: sklearn.kernel_approximation This example illustrates the use of :class:`PolynomialCountSketch` to efficiently generate polynomial kernel feature-space approximations. This is used to train linear classifiers that approximate the accuracy of kernelized ones. -.. currentmodule:: sklearn.kernel_approximation - We use the Covtype dataset [2], trying to reproduce the experiments on the original paper of Tensor Sketch [1], i.e. the algorithm implemented by :class:`PolynomialCountSketch`. @@ -18,21 +18,16 @@ features. Then, we train linear classifiers on different numbers of features (`n_components`) generated by :class:`PolynomialCountSketch`, approximating the accuracy of a kernelized classifier in a scalable manner. + """ -print(__doc__) -# Author: Daniel Lopez-Sanchez -# License: BSD 3 clause -import matplotlib.pyplot as plt -from sklearn.datasets import fetch_covtype -from sklearn.model_selection import train_test_split -from sklearn.preprocessing import MinMaxScaler, Normalizer -from sklearn.svm import LinearSVC -from sklearn.kernel_approximation import PolynomialCountSketch -from sklearn.pipeline import Pipeline, make_pipeline -import time +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause # %% +# Preparing the data +# ------------------ +# # Load the Covtype dataset, which contains 581,012 samples # with 54 features each, distributed among 6 classes. The goal of this dataset # is to predict forest cover type from cartographic variables only @@ -40,34 +35,53 @@ # classification problem to match the version of the dataset in the # LIBSVM webpage [2], which was the one used in [1]. +from sklearn.datasets import fetch_covtype + X, y = fetch_covtype(return_X_y=True) y[y != 2] = 0 y[y == 2] = 1 # We will try to separate class 2 from the other 6 classes. # %% +# Partitioning the data +# --------------------- +# # Here we select 5,000 samples for training and 10,000 for testing. # To actually reproduce the results in the original Tensor Sketch paper, # select 100,000 for training. -X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=5_000, - test_size=10_000, - random_state=42) +from sklearn.model_selection import train_test_split + +X_train, X_test, y_train, y_test = train_test_split( + X, y, train_size=5_000, test_size=10_000, random_state=42 +) # %% +# Feature normalization +# --------------------- +# # Now scale features to the range [0, 1] to match the format of the dataset in # the LIBSVM webpage, and then normalize to unit length as done in the # original Tensor Sketch paper [1]. +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import MinMaxScaler, Normalizer + mm = make_pipeline(MinMaxScaler(), Normalizer()) X_train = mm.fit_transform(X_train) X_test = mm.transform(X_test) - # %% +# Establishing a baseline model +# ----------------------------- +# # As a baseline, train a linear SVM on the original features and print the # accuracy. We also measure and store accuracies and training times to -# plot them latter. +# plot them later. + +import time + +from sklearn.svm import LinearSVC results = {} @@ -81,6 +95,9 @@ print(f"Linear SVM score on raw features: {lsvm_score:.2f}%") # %% +# Establishing the kernel approximation model +# ------------------------------------------- +# # Then we train linear SVMs on the features generated by # :class:`PolynomialCountSketch` with different values for `n_components`, # showing that these kernel feature approximations improve the accuracy @@ -94,21 +111,23 @@ # polynomial kernel of degree four would have approximately 8.5 million # features (precisely, 54^4). Thanks to :class:`PolynomialCountSketch`, we can # condense most of the discriminative information of that feature space into a -# much more compact representation. We repeat the experiment 5 times to -# compensate for the stochastic nature of :class:`PolynomialCountSketch`. +# much more compact representation. While we run the experiment only a single time +# (`n_runs` = 1) in this example, in practice one should repeat the experiment several +# times to compensate for the stochastic nature of :class:`PolynomialCountSketch`. -n_runs = 3 -for n_components in [250, 500, 1000, 2000]: +from sklearn.kernel_approximation import PolynomialCountSketch +n_runs = 1 +N_COMPONENTS = [250, 500, 1000, 2000] + +for n_components in N_COMPONENTS: ps_lsvm_time = 0 ps_lsvm_score = 0 for _ in range(n_runs): - - pipeline = Pipeline(steps=[("kernel_approximator", - PolynomialCountSketch( - n_components=n_components, - degree=4)), - ("linear_classifier", LinearSVC())]) + pipeline = make_pipeline( + PolynomialCountSketch(n_components=n_components, degree=4), + LinearSVC(), + ) start = time.time() pipeline.fit(X_train, y_train) @@ -119,12 +138,18 @@ ps_lsvm_score /= n_runs results[f"LSVM + PS({n_components})"] = { - "time": ps_lsvm_time, "score": ps_lsvm_score + "time": ps_lsvm_time, + "score": ps_lsvm_score, } - print(f"Linear SVM score on {n_components} PolynomialCountSketch " + - f"features: {ps_lsvm_score:.2f}%") + print( + f"Linear SVM score on {n_components} PolynomialCountSketch " + f"features: {ps_lsvm_score:.2f}%" + ) # %% +# Establishing the kernelized SVM model +# ------------------------------------- +# # Train a kernelized SVM to see how well :class:`PolynomialCountSketch` # is approximating the performance of the kernel. This, of course, may take # some time, as the SVC class has a relatively poor scalability. This is the @@ -132,7 +157,7 @@ from sklearn.svm import SVC -ksvm = SVC(C=500., kernel="poly", degree=4, coef0=0, gamma=1.) +ksvm = SVC(C=500.0, kernel="poly", degree=4, coef0=0, gamma=1.0) start = time.time() ksvm.fit(X_train, y_train) @@ -140,37 +165,77 @@ ksvm_score = 100 * ksvm.score(X_test, y_test) results["KSVM"] = {"time": ksvm_time, "score": ksvm_score} -print(f"Kernel-SVM score on raw featrues: {ksvm_score:.2f}%") +print(f"Kernel-SVM score on raw features: {ksvm_score:.2f}%") # %% -# Finally, plot the resuts of the different methods against their training +# Comparing the results +# --------------------- +# +# Finally, plot the results of the different methods against their training # times. As we can see, the kernelized SVM achieves a higher accuracy, # but its training time is much larger and, most importantly, will grow # much faster if the number of training samples increases. -N_COMPONENTS = [250, 500, 1000, 2000] +import matplotlib.pyplot as plt fig, ax = plt.subplots(figsize=(7, 7)) -ax.scatter([results["LSVM"]["time"], ], [results["LSVM"]["score"], ], - label="Linear SVM", c="green", marker="^") +ax.scatter( + [ + results["LSVM"]["time"], + ], + [ + results["LSVM"]["score"], + ], + label="Linear SVM", + c="green", + marker="^", +) + +ax.scatter( + [ + results["LSVM + PS(250)"]["time"], + ], + [ + results["LSVM + PS(250)"]["score"], + ], + label="Linear SVM + PolynomialCountSketch", + c="blue", +) -ax.scatter([results["LSVM + PS(250)"]["time"], ], - [results["LSVM + PS(250)"]["score"], ], - label="Linear SVM + PolynomialCountSketch", c="blue") for n_components in N_COMPONENTS: - ax.scatter([results[f"LSVM + PS({n_components})"]["time"], ], - [results[f"LSVM + PS({n_components})"]["score"], ], - c="blue") - ax.annotate(f"n_comp.={n_components}", - (results[f"LSVM + PS({n_components})"]["time"], - results[f"LSVM + PS({n_components})"]["score"]), - xytext=(-30, 10), textcoords="offset pixels") - -ax.scatter([results["KSVM"]["time"], ], [results["KSVM"]["score"], ], - label="Kernel SVM", c="red", marker="x") + ax.scatter( + [ + results[f"LSVM + PS({n_components})"]["time"], + ], + [ + results[f"LSVM + PS({n_components})"]["score"], + ], + c="blue", + ) + ax.annotate( + f"n_comp.={n_components}", + ( + results[f"LSVM + PS({n_components})"]["time"], + results[f"LSVM + PS({n_components})"]["score"], + ), + xytext=(-30, 10), + textcoords="offset pixels", + ) + +ax.scatter( + [ + results["KSVM"]["time"], + ], + [ + results["KSVM"]["score"], + ], + label="Kernel SVM", + c="red", + marker="x", +) ax.set_xlabel("Training time (s)") -ax.set_ylabel("Accurary (%)") +ax.set_ylabel("Accuracy (%)") ax.legend() plt.show() diff --git a/examples/linear_model/plot_ard.py b/examples/linear_model/plot_ard.py index 177bd8ce24ad1..475350e7cd73e 100644 --- a/examples/linear_model/plot_ard.py +++ b/examples/linear_model/plot_ard.py @@ -1,116 +1,212 @@ """ -================================================== -Automatic Relevance Determination Regression (ARD) -================================================== +==================================== +Comparing Linear Bayesian Regressors +==================================== -Fit regression model with Bayesian Ridge Regression. +This example compares two different bayesian regressors: -See :ref:`bayesian_ridge_regression` for more information on the regressor. +- a :ref:`automatic_relevance_determination` +- a :ref:`bayesian_ridge_regression` -Compared to the OLS (ordinary least squares) estimator, the coefficient -weights are slightly shifted toward zeros, which stabilises them. +In the first part, we use an :ref:`ordinary_least_squares` (OLS) model as a +baseline for comparing the models' coefficients with respect to the true +coefficients. Thereafter, we show that the estimation of such models is done by +iteratively maximizing the marginal log-likelihood of the observations. -The histogram of the estimated weights is very peaked, as a sparsity-inducing -prior is implied on the weights. +In the last section we plot predictions and uncertainties for the ARD and the +Bayesian Ridge regressions using a polynomial feature expansion to fit a +non-linear relationship between `X` and `y`. -The estimation of the model is done by iteratively maximizing the -marginal log-likelihood of the observations. - -We also plot predictions and uncertainties for ARD -for one dimensional regression using polynomial feature expansion. -Note the uncertainty starts going up on the right side of the plot. -This is because these test samples are outside of the range of the training -samples. """ -print(__doc__) -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Models robustness to recover the ground truth weights +# ===================================================== +# +# Generate synthetic dataset +# -------------------------- +# +# We generate a dataset where `X` and `y` are linearly linked: 10 of the +# features of `X` will be used to generate `y`. The other features are not +# useful at predicting `y`. In addition, we generate a dataset where `n_samples +# == n_features`. Such a setting is challenging for an OLS model and leads +# potentially to arbitrary large weights. Having a prior on the weights and a +# penalty alleviates the problem. Finally, gaussian noise is added. + +from sklearn.datasets import make_regression + +X, y, true_weights = make_regression( + n_samples=100, + n_features=100, + n_informative=10, + noise=8, + coef=True, + random_state=42, +) + +# %% +# Fit the regressors +# ------------------ +# +# We now fit both Bayesian models and the OLS to later compare the models' +# coefficients. + +import pandas as pd + +from sklearn.linear_model import ARDRegression, BayesianRidge, LinearRegression + +olr = LinearRegression().fit(X, y) +brr = BayesianRidge(compute_score=True, max_iter=30).fit(X, y) +ard = ARDRegression(compute_score=True, max_iter=30).fit(X, y) +df = pd.DataFrame( + { + "Weights of true generative process": true_weights, + "ARDRegression": ard.coef_, + "BayesianRidge": brr.coef_, + "LinearRegression": olr.coef_, + } +) + +# %% +# Plot the true and estimated coefficients +# ---------------------------------------- +# +# Now we compare the coefficients of each model with the weights of +# the true generative model. import matplotlib.pyplot as plt -from scipy import stats - -from sklearn.linear_model import ARDRegression, LinearRegression - -# ############################################################################# -# Generating simulated data with Gaussian weights - -# Parameters of the example -np.random.seed(0) -n_samples, n_features = 100, 100 -# Create Gaussian data -X = np.random.randn(n_samples, n_features) -# Create weights with a precision lambda_ of 4. -lambda_ = 4. -w = np.zeros(n_features) -# Only keep 10 weights of interest -relevant_features = np.random.randint(0, n_features, 10) -for i in relevant_features: - w[i] = stats.norm.rvs(loc=0, scale=1. / np.sqrt(lambda_)) -# Create noise with a precision alpha of 50. -alpha_ = 50. -noise = stats.norm.rvs(loc=0, scale=1. / np.sqrt(alpha_), size=n_samples) -# Create the target -y = np.dot(X, w) + noise - -# ############################################################################# -# Fit the ARD Regression -clf = ARDRegression(compute_score=True) -clf.fit(X, y) - -ols = LinearRegression() -ols.fit(X, y) - -# ############################################################################# -# Plot the true weights, the estimated weights, the histogram of the -# weights, and predictions with standard deviations -plt.figure(figsize=(6, 5)) -plt.title("Weights of the model") -plt.plot(clf.coef_, color='darkblue', linestyle='-', linewidth=2, - label="ARD estimate") -plt.plot(ols.coef_, color='yellowgreen', linestyle=':', linewidth=2, - label="OLS estimate") -plt.plot(w, color='orange', linestyle='-', linewidth=2, label="Ground truth") -plt.xlabel("Features") -plt.ylabel("Values of the weights") -plt.legend(loc=1) - -plt.figure(figsize=(6, 5)) -plt.title("Histogram of the weights") -plt.hist(clf.coef_, bins=n_features, color='navy', log=True) -plt.scatter(clf.coef_[relevant_features], np.full(len(relevant_features), 5.), - color='gold', marker='o', label="Relevant features") -plt.ylabel("Features") -plt.xlabel("Values of the weights") -plt.legend(loc=1) - -plt.figure(figsize=(6, 5)) -plt.title("Marginal log-likelihood") -plt.plot(clf.scores_, color='navy', linewidth=2) -plt.ylabel("Score") -plt.xlabel("Iterations") - +import seaborn as sns +from matplotlib.colors import SymLogNorm + +plt.figure(figsize=(10, 6)) +ax = sns.heatmap( + df.T, + norm=SymLogNorm(linthresh=10e-4, vmin=-80, vmax=80), + cbar_kws={"label": "coefficients' values"}, + cmap="seismic_r", +) +plt.ylabel("linear model") +plt.xlabel("coefficients") +plt.tight_layout(rect=(0, 0, 1, 0.95)) +_ = plt.title("Models' coefficients") + +# %% +# Due to the added noise, none of the models recover the true weights. Indeed, +# all models always have more than 10 non-zero coefficients. Compared to the OLS +# estimator, the coefficients using a Bayesian Ridge regression are slightly +# shifted toward zero, which stabilises them. The ARD regression provides a +# sparser solution: some of the non-informative coefficients are set exactly to +# zero, while shifting others closer to zero. Some non-informative coefficients +# are still present and retain large values. + +# %% +# Plot the marginal log-likelihood +# -------------------------------- +import numpy as np -# Plotting some predictions for polynomial regression -def f(x, noise_amount): - y = np.sqrt(x) * np.sin(x) - noise = np.random.normal(0, 1, len(x)) - return y + noise_amount * noise - - -degree = 10 -X = np.linspace(0, 10, 100) -y = f(X, noise_amount=1) -clf_poly = ARDRegression(threshold_lambda=1e5) -clf_poly.fit(np.vander(X, degree), y) - -X_plot = np.linspace(0, 11, 25) -y_plot = f(X_plot, noise_amount=0) -y_mean, y_std = clf_poly.predict(np.vander(X_plot, degree), return_std=True) -plt.figure(figsize=(6, 5)) -plt.errorbar(X_plot, y_mean, y_std, color='navy', - label="Polynomial ARD", linewidth=2) -plt.plot(X_plot, y_plot, color='gold', linewidth=2, - label="Ground Truth") -plt.ylabel("Output y") -plt.xlabel("Feature X") -plt.legend(loc="lower left") -plt.show() +ard_scores = -np.array(ard.scores_) +brr_scores = -np.array(brr.scores_) +plt.plot(ard_scores, color="navy", label="ARD") +plt.plot(brr_scores, color="red", label="BayesianRidge") +plt.ylabel("Log-likelihood") +plt.xlabel("Iterations") +plt.xlim(1, 30) +plt.legend() +_ = plt.title("Models log-likelihood") + +# %% +# Indeed, both models minimize the log-likelihood up to an arbitrary cutoff +# defined by the `max_iter` parameter. +# +# Bayesian regressions with polynomial feature expansion +# ====================================================== +# Generate synthetic dataset +# -------------------------- +# We create a target that is a non-linear function of the input feature. +# Noise following a standard uniform distribution is added. + +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import PolynomialFeatures, StandardScaler + +rng = np.random.RandomState(0) +n_samples = 110 + +# sort the data to make plotting easier later +X = np.sort(-10 * rng.rand(n_samples) + 10) +noise = rng.normal(0, 1, n_samples) * 1.35 +y = np.sqrt(X) * np.sin(X) + noise +full_data = pd.DataFrame({"input_feature": X, "target": y}) +X = X.reshape((-1, 1)) + +# extrapolation +X_plot = np.linspace(10, 10.4, 10) +y_plot = np.sqrt(X_plot) * np.sin(X_plot) +X_plot = np.concatenate((X, X_plot.reshape((-1, 1)))) +y_plot = np.concatenate((y - noise, y_plot)) + +# %% +# Fit the regressors +# ------------------ +# +# Here we try a degree 10 polynomial to potentially overfit, though the bayesian +# linear models regularize the size of the polynomial coefficients. As +# `fit_intercept=True` by default for +# :class:`~sklearn.linear_model.ARDRegression` and +# :class:`~sklearn.linear_model.BayesianRidge`, then +# :class:`~sklearn.preprocessing.PolynomialFeatures` should not introduce an +# additional bias feature. By setting `return_std=True`, the bayesian regressors +# return the standard deviation of the posterior distribution for the model +# parameters. + +ard_poly = make_pipeline( + PolynomialFeatures(degree=10, include_bias=False), + StandardScaler(), + ARDRegression(), +).fit(X, y) +brr_poly = make_pipeline( + PolynomialFeatures(degree=10, include_bias=False), + StandardScaler(), + BayesianRidge(), +).fit(X, y) + +y_ard, y_ard_std = ard_poly.predict(X_plot, return_std=True) +y_brr, y_brr_std = brr_poly.predict(X_plot, return_std=True) + +# %% +# Plotting polynomial regressions with std errors of the scores +# ------------------------------------------------------------- + +ax = sns.scatterplot( + data=full_data, x="input_feature", y="target", color="black", alpha=0.75 +) +ax.plot(X_plot, y_plot, color="black", label="Ground Truth") +ax.plot(X_plot, y_brr, color="red", label="BayesianRidge with polynomial features") +ax.plot(X_plot, y_ard, color="navy", label="ARD with polynomial features") +ax.fill_between( + X_plot.ravel(), + y_ard - y_ard_std, + y_ard + y_ard_std, + color="navy", + alpha=0.3, +) +ax.fill_between( + X_plot.ravel(), + y_brr - y_brr_std, + y_brr + y_brr_std, + color="red", + alpha=0.3, +) +ax.legend() +_ = ax.set_title("Polynomial fit of a non-linear feature") + +# %% +# The error bars represent one standard deviation of the predicted gaussian +# distribution of the query points. Notice that the ARD regression captures the +# ground truth the best when using the default parameters in both models, but +# further reducing the `lambda_init` hyperparameter of the Bayesian Ridge can +# reduce its bias (see example +# :ref:`sphx_glr_auto_examples_linear_model_plot_bayesian_ridge_curvefit.py`). +# Finally, due to the intrinsic limitations of a polynomial regression, both +# models fail when extrapolating. diff --git a/examples/linear_model/plot_bayesian_ridge.py b/examples/linear_model/plot_bayesian_ridge.py deleted file mode 100644 index 43925e72c591c..0000000000000 --- a/examples/linear_model/plot_bayesian_ridge.py +++ /dev/null @@ -1,114 +0,0 @@ -""" -========================= -Bayesian Ridge Regression -========================= - -Computes a Bayesian Ridge Regression on a synthetic dataset. - -See :ref:`bayesian_ridge_regression` for more information on the regressor. - -Compared to the OLS (ordinary least squares) estimator, the coefficient -weights are slightly shifted toward zeros, which stabilises them. - -As the prior on the weights is a Gaussian prior, the histogram of the -estimated weights is Gaussian. - -The estimation of the model is done by iteratively maximizing the -marginal log-likelihood of the observations. - -We also plot predictions and uncertainties for Bayesian Ridge Regression -for one dimensional regression using polynomial feature expansion. -Note the uncertainty starts going up on the right side of the plot. -This is because these test samples are outside of the range of the training -samples. -""" -print(__doc__) - -import numpy as np -import matplotlib.pyplot as plt -from scipy import stats - -from sklearn.linear_model import BayesianRidge, LinearRegression - -# ############################################################################# -# Generating simulated data with Gaussian weights -np.random.seed(0) -n_samples, n_features = 100, 100 -X = np.random.randn(n_samples, n_features) # Create Gaussian data -# Create weights with a precision lambda_ of 4. -lambda_ = 4. -w = np.zeros(n_features) -# Only keep 10 weights of interest -relevant_features = np.random.randint(0, n_features, 10) -for i in relevant_features: - w[i] = stats.norm.rvs(loc=0, scale=1. / np.sqrt(lambda_)) -# Create noise with a precision alpha of 50. -alpha_ = 50. -noise = stats.norm.rvs(loc=0, scale=1. / np.sqrt(alpha_), size=n_samples) -# Create the target -y = np.dot(X, w) + noise - -# ############################################################################# -# Fit the Bayesian Ridge Regression and an OLS for comparison -clf = BayesianRidge(compute_score=True) -clf.fit(X, y) - -ols = LinearRegression() -ols.fit(X, y) - -# ############################################################################# -# Plot true weights, estimated weights, histogram of the weights, and -# predictions with standard deviations -lw = 2 -plt.figure(figsize=(6, 5)) -plt.title("Weights of the model") -plt.plot(clf.coef_, color='lightgreen', linewidth=lw, - label="Bayesian Ridge estimate") -plt.plot(w, color='gold', linewidth=lw, label="Ground truth") -plt.plot(ols.coef_, color='navy', linestyle='--', label="OLS estimate") -plt.xlabel("Features") -plt.ylabel("Values of the weights") -plt.legend(loc="best", prop=dict(size=12)) - -plt.figure(figsize=(6, 5)) -plt.title("Histogram of the weights") -plt.hist(clf.coef_, bins=n_features, color='gold', log=True, - edgecolor='black') -plt.scatter(clf.coef_[relevant_features], np.full(len(relevant_features), 5.), - color='navy', label="Relevant features") -plt.ylabel("Features") -plt.xlabel("Values of the weights") -plt.legend(loc="upper left") - -plt.figure(figsize=(6, 5)) -plt.title("Marginal log-likelihood") -plt.plot(clf.scores_, color='navy', linewidth=lw) -plt.ylabel("Score") -plt.xlabel("Iterations") - - -# Plotting some predictions for polynomial regression -def f(x, noise_amount): - y = np.sqrt(x) * np.sin(x) - noise = np.random.normal(0, 1, len(x)) - return y + noise_amount * noise - - -degree = 10 -X = np.linspace(0, 10, 100) -y = f(X, noise_amount=0.1) -clf_poly = BayesianRidge() -clf_poly.fit(np.vander(X, degree), y) - -X_plot = np.linspace(0, 11, 25) -y_plot = f(X_plot, noise_amount=0) -y_mean, y_std = clf_poly.predict(np.vander(X_plot, degree), return_std=True) -plt.figure(figsize=(6, 5)) -plt.errorbar(X_plot, y_mean, y_std, color='navy', - label="Polynomial Bayesian Ridge Regression", linewidth=lw) -plt.plot(X_plot, y_plot, color='gold', linewidth=lw, - label="Ground Truth") -plt.ylabel("Output y") -plt.xlabel("Feature X") -plt.legend(loc="lower left") -plt.show() diff --git a/examples/linear_model/plot_bayesian_ridge_curvefit.py b/examples/linear_model/plot_bayesian_ridge_curvefit.py index 2f4a36d47d9a6..45679580dbf34 100644 --- a/examples/linear_model/plot_bayesian_ridge_curvefit.py +++ b/examples/linear_model/plot_bayesian_ridge_curvefit.py @@ -23,45 +23,51 @@ Also, by evaluating log marginal likelihood (L) of these models, we can determine which one is better. It can be concluded that the model with larger L is more likely. + """ -print(__doc__) -# Author: Yoshihiro Uchida +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +# %% +# Generate sinusoidal data with noise +# ----------------------------------- import numpy as np -import matplotlib.pyplot as plt - -from sklearn.linear_model import BayesianRidge -def func(x): return np.sin(2*np.pi*x) +def func(x): + return np.sin(2 * np.pi * x) -# ############################################################################# -# Generate sinusoidal data with noise size = 25 rng = np.random.RandomState(1234) -x_train = rng.uniform(0., 1., size) +x_train = rng.uniform(0.0, 1.0, size) y_train = func(x_train) + rng.normal(scale=0.1, size=size) -x_test = np.linspace(0., 1., 100) +x_test = np.linspace(0.0, 1.0, 100) -# ############################################################################# +# %% # Fit by cubic polynomial +# ----------------------- +from sklearn.linear_model import BayesianRidge + n_order = 3 X_train = np.vander(x_train, n_order + 1, increasing=True) X_test = np.vander(x_test, n_order + 1, increasing=True) +reg = BayesianRidge(tol=1e-6, fit_intercept=False, compute_score=True) -# ############################################################################# +# %% # Plot the true and predicted curves with log marginal likelihood (L) -reg = BayesianRidge(tol=1e-6, fit_intercept=False, compute_score=True) +# ------------------------------------------------------------------- +import matplotlib.pyplot as plt + fig, axes = plt.subplots(1, 2, figsize=(8, 4)) for i, ax in enumerate(axes): # Bayesian ridge regression with different initial value pairs if i == 0: - init = [1 / np.var(y_train), 1.] # Default values + init = [1 / np.var(y_train), 1.0] # Default values elif i == 1: - init = [1., 1e-3] + init = [1.0, 1e-3] reg.set_params(alpha_init=init[0], lambda_init=init[1]) reg.fit(X_train, y_train) ymean, ystd = reg.predict(X_test, return_std=True) @@ -69,17 +75,18 @@ def func(x): return np.sin(2*np.pi*x) ax.plot(x_test, func(x_test), color="blue", label="sin($2\\pi x$)") ax.scatter(x_train, y_train, s=50, alpha=0.5, label="observation") ax.plot(x_test, ymean, color="red", label="predict mean") - ax.fill_between(x_test, ymean-ystd, ymean+ystd, - color="pink", alpha=0.5, label="predict std") + ax.fill_between( + x_test, ymean - ystd, ymean + ystd, color="pink", alpha=0.5, label="predict std" + ) ax.set_ylim(-1.3, 1.3) ax.legend() - title = "$\\alpha$_init$={:.2f},\\ \\lambda$_init$={}$".format( - init[0], init[1]) + title = "$\\alpha$_init$={:.2f},\\ \\lambda$_init$={}$".format(init[0], init[1]) if i == 0: title += " (Default)" ax.set_title(title, fontsize=12) text = "$\\alpha={:.1f}$\n$\\lambda={:.3f}$\n$L={:.1f}$".format( - reg.alpha_, reg.lambda_, reg.scores_[-1]) + reg.alpha_, reg.lambda_, reg.scores_[-1] + ) ax.text(0.05, -1.0, text, fontsize=12) plt.tight_layout() diff --git a/examples/linear_model/plot_elastic_net_precomputed_gram_matrix_with_weighted_samples.py b/examples/linear_model/plot_elastic_net_precomputed_gram_matrix_with_weighted_samples.py index 852ea545c5fd6..e118847a8737c 100644 --- a/examples/linear_model/plot_elastic_net_precomputed_gram_matrix_with_weighted_samples.py +++ b/examples/linear_model/plot_elastic_net_precomputed_gram_matrix_with_weighted_samples.py @@ -4,7 +4,7 @@ ========================================================================== The following example shows how to precompute the gram matrix -while using weighted samples with an ElasticNet. +while using weighted samples with an :class:`~sklearn.linear_model.ElasticNet`. If weighted samples are used, the design matrix must be centered and then rescaled by the square root of the weight vector before the gram matrix @@ -13,15 +13,17 @@ .. note:: `sample_weight` vector is also rescaled to sum to `n_samples`, see the documentation for the `sample_weight` parameter to - :func:`linear_model.ElasticNet.fit`. + :meth:`~sklearn.linear_model.ElasticNet.fit`. """ -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause # %% # Let's start by loading the dataset and creating some sample weights. import numpy as np + from sklearn.datasets import make_regression rng = np.random.RandomState(0) @@ -38,7 +40,7 @@ # weights, we must first center the design matrix, and rescale it by the # normalized weights prior to computing the gram matrix. X_offset = np.average(X, axis=0, weights=normalized_weights) -X_centered = (X - np.average(X, axis=0, weights=normalized_weights)) +X_centered = X - np.average(X, axis=0, weights=normalized_weights) X_scaled = X_centered * np.sqrt(normalized_weights)[:, np.newaxis] gram = np.dot(X_scaled.T, X_scaled) diff --git a/examples/linear_model/plot_huber_vs_ridge.py b/examples/linear_model/plot_huber_vs_ridge.py index 63abffe6be4ba..e4dd6b502881e 100644 --- a/examples/linear_model/plot_huber_vs_ridge.py +++ b/examples/linear_model/plot_huber_vs_ridge.py @@ -10,40 +10,40 @@ influenced by the outliers since the model uses the linear loss for these. As the parameter epsilon is increased for the Huber regressor, the decision function approaches that of the ridge. -""" -# Authors: Manoj Kumar mks542@nyu.edu -# License: BSD 3 clause +""" -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt +import numpy as np from sklearn.datasets import make_regression from sklearn.linear_model import HuberRegressor, Ridge # Generate toy data. rng = np.random.RandomState(0) -X, y = make_regression(n_samples=20, n_features=1, random_state=0, noise=4.0, - bias=100.0) +X, y = make_regression( + n_samples=20, n_features=1, random_state=0, noise=4.0, bias=100.0 +) # Add four strong outliers to the dataset. X_outliers = rng.normal(0, 0.5, size=(4, 1)) y_outliers = rng.normal(0, 2.0, size=4) -X_outliers[:2, :] += X.max() + X.mean() / 4. -X_outliers[2:, :] += X.min() - X.mean() / 4. -y_outliers[:2] += y.min() - y.mean() / 4. -y_outliers[2:] += y.max() + y.mean() / 4. +X_outliers[:2, :] += X.max() + X.mean() / 4.0 +X_outliers[2:, :] += X.min() - X.mean() / 4.0 +y_outliers[:2] += y.min() - y.mean() / 4.0 +y_outliers[2:] += y.max() + y.mean() / 4.0 X = np.vstack((X, X_outliers)) y = np.concatenate((y, y_outliers)) -plt.plot(X, y, 'b.') +plt.plot(X, y, "b.") # Fit the huber regressor over a series of epsilon values. -colors = ['r-', 'b-', 'y-', 'm-'] +colors = ["r-", "b-", "y-", "m-"] x = np.linspace(X.min(), X.max(), 7) -epsilon_values = [1.35, 1.5, 1.75, 1.9] +epsilon_values = [1, 1.5, 1.75, 1.9] for k, epsilon in enumerate(epsilon_values): huber = HuberRegressor(alpha=0.0, epsilon=epsilon) huber.fit(X, y) @@ -51,11 +51,11 @@ plt.plot(x, coef_, colors[k], label="huber loss, %s" % epsilon) # Fit a ridge regressor to compare it to huber regressor. -ridge = Ridge(alpha=0.0, random_state=0, normalize=True) +ridge = Ridge(alpha=0.0, random_state=0) ridge.fit(X, y) coef_ridge = ridge.coef_ coef_ = ridge.coef_ * x + ridge.intercept_ -plt.plot(x, coef_, 'g-', label="ridge regression") +plt.plot(x, coef_, "g-", label="ridge regression") plt.title("Comparison of HuberRegressor vs Ridge") plt.xlabel("X") diff --git a/examples/linear_model/plot_iris_logistic.py b/examples/linear_model/plot_iris_logistic.py deleted file mode 100644 index b04094a647e90..0000000000000 --- a/examples/linear_model/plot_iris_logistic.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -""" -========================================================= -Logistic Regression 3-class Classifier -========================================================= - -Show below is a logistic-regression classifiers decision boundaries on the -first two dimensions (sepal length and width) of the `iris -`_ dataset. The datapoints -are colored according to their labels. - -""" -print(__doc__) - -# Code source: GaÃĢl Varoquaux -# Modified for documentation by Jaques Grobler -# License: BSD 3 clause - -import numpy as np -import matplotlib.pyplot as plt -from sklearn.linear_model import LogisticRegression -from sklearn import datasets - -# import some data to play with -iris = datasets.load_iris() -X = iris.data[:, :2] # we only take the first two features. -Y = iris.target - -# Create an instance of Logistic Regression Classifier and fit the data. -logreg = LogisticRegression(C=1e5) -logreg.fit(X, Y) - -# Plot the decision boundary. For that, we will assign a color to each -# point in the mesh [x_min, x_max]x[y_min, y_max]. -x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5 -y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5 -h = .02 # step size in the mesh -xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h)) -Z = logreg.predict(np.c_[xx.ravel(), yy.ravel()]) - -# Put the result into a color plot -Z = Z.reshape(xx.shape) -plt.figure(1, figsize=(4, 3)) -plt.pcolormesh(xx, yy, Z, cmap=plt.cm.Paired) - -# Plot also the training points -plt.scatter(X[:, 0], X[:, 1], c=Y, edgecolors='k', cmap=plt.cm.Paired) -plt.xlabel('Sepal length') -plt.ylabel('Sepal width') - -plt.xlim(xx.min(), xx.max()) -plt.ylim(yy.min(), yy.max()) -plt.xticks(()) -plt.yticks(()) - -plt.show() diff --git a/examples/linear_model/plot_lasso_and_elasticnet.py b/examples/linear_model/plot_lasso_and_elasticnet.py index 8934ddc76395c..1b1a495c1a7f7 100644 --- a/examples/linear_model/plot_lasso_and_elasticnet.py +++ b/examples/linear_model/plot_lasso_and_elasticnet.py @@ -1,76 +1,249 @@ """ -======================================== -Lasso and Elastic Net for Sparse Signals -======================================== +================================== +L1-based models for Sparse Signals +================================== -Estimates Lasso and Elastic-Net regression models on a manually generated -sparse signal corrupted with an additive noise. Estimated coefficients are -compared with the ground-truth. +The present example compares three l1-based regression models on a synthetic +signal obtained from sparse and correlated features that are further corrupted +with additive gaussian noise: + +- a :ref:`lasso`; +- an :ref:`automatic_relevance_determination`; +- an :ref:`elastic_net`. +It is known that the Lasso estimates turn to be close to the model selection +estimates when the data dimensions grow, given that the irrelevant variables are +not too correlated with the relevant ones. In the presence of correlated +features, Lasso itself cannot select the correct sparsity pattern [1]_. + +Here we compare the performance of the three models in terms of the :math:`R^2` +score, the fitting time and the sparsity of the estimated coefficients when +compared with the ground-truth. """ -print(__doc__) -import numpy as np -import matplotlib.pyplot as plt +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Generate synthetic dataset +# -------------------------- +# +# We generate a dataset where the number of samples is lower than the total +# number of features. This leads to an underdetermined system, i.e. the solution +# is not unique, and thus we cannot apply an :ref:`ordinary_least_squares` by +# itself. Regularization introduces a penalty term to the objective function, +# which modifies the optimization problem and can help alleviate the +# underdetermined nature of the system. +# +# The target `y` is a linear combination with alternating signs of sinusoidal +# signals. Only the 10 lowest out of the 100 frequencies in `X` are used to +# generate `y`, while the rest of the features are not informative. This results +# in a high dimensional sparse feature space, where some degree of +# l1-penalization is necessary. -from sklearn.metrics import r2_score +import numpy as np -# ############################################################################# -# Generate some sparse data to play with -np.random.seed(42) +rng = np.random.RandomState(0) +n_samples, n_features, n_informative = 50, 100, 10 +time_step = np.linspace(-2, 2, n_samples) +freqs = 2 * np.pi * np.sort(rng.rand(n_features)) / 0.01 +X = np.zeros((n_samples, n_features)) -n_samples, n_features = 50, 100 -X = np.random.randn(n_samples, n_features) +for i in range(n_features): + X[:, i] = np.sin(freqs[i] * time_step) -# Decreasing coef w. alternated signs for visualization idx = np.arange(n_features) -coef = (-1) ** idx * np.exp(-idx / 10) -coef[10:] = 0 # sparsify coef -y = np.dot(X, coef) +true_coef = (-1) ** idx * np.exp(-idx / 10) +true_coef[n_informative:] = 0 # sparsify coef +y = np.dot(X, true_coef) + +# %% +# Some of the informative features have close frequencies to induce +# (anti-)correlations. + +freqs[:n_informative] -# Add noise -y += 0.01 * np.random.normal(size=n_samples) +# %% +# A random phase is introduced using :func:`numpy.random.random_sample` +# and some gaussian noise (implemented by :func:`numpy.random.normal`) +# is added to both the features and the target. -# Split data in train set and test set -n_samples = X.shape[0] -X_train, y_train = X[:n_samples // 2], y[:n_samples // 2] -X_test, y_test = X[n_samples // 2:], y[n_samples // 2:] +for i in range(n_features): + X[:, i] = np.sin(freqs[i] * time_step + 2 * (rng.random_sample() - 0.5)) + X[:, i] += 0.2 * rng.normal(0, 1, n_samples) -# ############################################################################# +y += 0.2 * rng.normal(0, 1, n_samples) + +# %% +# Such sparse, noisy and correlated features can be obtained, for instance, from +# sensor nodes monitoring some environmental variables, as they typically register +# similar values depending on their positions (spatial correlations). +# We can visualize the target. + +import matplotlib.pyplot as plt + +plt.plot(time_step, y) +plt.ylabel("target signal") +plt.xlabel("time") +_ = plt.title("Superposition of sinusoidal signals") + +# %% +# We split the data into train and test sets for simplicity. In practice one +# should use a :class:`~sklearn.model_selection.TimeSeriesSplit` +# cross-validation to estimate the variance of the test score. Here we set +# `shuffle="False"` as we must not use training data that succeed the testing +# data when dealing with data that have a temporal relationship. + +from sklearn.model_selection import train_test_split + +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, shuffle=False) + +# %% +# In the following, we compute the performance of three l1-based models in terms +# of the goodness of fit :math:`R^2` score and the fitting time. Then we make a +# plot to compare the sparsity of the estimated coefficients with respect to the +# ground-truth coefficients and finally we analyze the previous results. +# # Lasso +# ----- +# +# In this example, we demo a :class:`~sklearn.linear_model.Lasso` with a fixed +# value of the regularization parameter `alpha`. In practice, the optimal +# parameter `alpha` should be selected by passing a +# :class:`~sklearn.model_selection.TimeSeriesSplit` cross-validation strategy to a +# :class:`~sklearn.linear_model.LassoCV`. To keep the example simple and fast to +# execute, we directly set the optimal value for alpha here. +from time import time + from sklearn.linear_model import Lasso +from sklearn.metrics import r2_score -alpha = 0.1 -lasso = Lasso(alpha=alpha) +t0 = time() +lasso = Lasso(alpha=0.14).fit(X_train, y_train) +print(f"Lasso fit done in {(time() - t0):.3f}s") -y_pred_lasso = lasso.fit(X_train, y_train).predict(X_test) +y_pred_lasso = lasso.predict(X_test) r2_score_lasso = r2_score(y_test, y_pred_lasso) -print(lasso) -print("r^2 on test data : %f" % r2_score_lasso) - -# ############################################################################# +print(f"Lasso r^2 on test data : {r2_score_lasso:.3f}") + +# %% +# Automatic Relevance Determination (ARD) +# --------------------------------------- +# +# An ARD regression is the bayesian version of the Lasso. It can produce +# interval estimates for all of the parameters, including the error variance, if +# required. It is a suitable option when the signals have gaussian noise. See +# the example :ref:`sphx_glr_auto_examples_linear_model_plot_ard.py` for a +# comparison of :class:`~sklearn.linear_model.ARDRegression` and +# :class:`~sklearn.linear_model.BayesianRidge` regressors. + +from sklearn.linear_model import ARDRegression + +t0 = time() +ard = ARDRegression().fit(X_train, y_train) +print(f"ARD fit done in {(time() - t0):.3f}s") + +y_pred_ard = ard.predict(X_test) +r2_score_ard = r2_score(y_test, y_pred_ard) +print(f"ARD r^2 on test data : {r2_score_ard:.3f}") + +# %% # ElasticNet +# ---------- +# +# :class:`~sklearn.linear_model.ElasticNet` is a middle ground between +# :class:`~sklearn.linear_model.Lasso` and :class:`~sklearn.linear_model.Ridge`, +# as it combines a L1 and a L2-penalty. The amount of regularization is +# controlled by the two hyperparameters `l1_ratio` and `alpha`. For `l1_ratio = +# 0` the penalty is pure L2 and the model is equivalent to a +# :class:`~sklearn.linear_model.Ridge`. Similarly, `l1_ratio = 1` is a pure L1 +# penalty and the model is equivalent to a :class:`~sklearn.linear_model.Lasso`. +# For `0 < l1_ratio < 1`, the penalty is a combination of L1 and L2. +# +# As done before, we train the model with fix values for `alpha` and `l1_ratio`. +# To select their optimal value we used an +# :class:`~sklearn.linear_model.ElasticNetCV`, not shown here to keep the +# example simple. + from sklearn.linear_model import ElasticNet -enet = ElasticNet(alpha=alpha, l1_ratio=0.7) +t0 = time() +enet = ElasticNet(alpha=0.08, l1_ratio=0.5).fit(X_train, y_train) +print(f"ElasticNet fit done in {(time() - t0):.3f}s") -y_pred_enet = enet.fit(X_train, y_train).predict(X_test) +y_pred_enet = enet.predict(X_test) r2_score_enet = r2_score(y_test, y_pred_enet) -print(enet) -print("r^2 on test data : %f" % r2_score_enet) - -m, s, _ = plt.stem(np.where(enet.coef_)[0], enet.coef_[enet.coef_ != 0], - markerfmt='x', label='Elastic net coefficients', - use_line_collection=True) -plt.setp([m, s], color="#2ca02c") -m, s, _ = plt.stem(np.where(lasso.coef_)[0], lasso.coef_[lasso.coef_ != 0], - markerfmt='x', label='Lasso coefficients', - use_line_collection=True) -plt.setp([m, s], color='#ff7f0e') -plt.stem(np.where(coef)[0], coef[coef != 0], label='true coefficients', - markerfmt='bx', use_line_collection=True) - -plt.legend(loc='best') -plt.title("Lasso $R^2$: %.3f, Elastic Net $R^2$: %.3f" - % (r2_score_lasso, r2_score_enet)) -plt.show() +print(f"ElasticNet r^2 on test data : {r2_score_enet:.3f}") + +# %% +# Plot and analysis of the results +# -------------------------------- +# +# In this section, we use a heatmap to visualize the sparsity of the true +# and estimated coefficients of the respective linear models. + +import matplotlib.pyplot as plt +import pandas as pd +import seaborn as sns +from matplotlib.colors import SymLogNorm + +df = pd.DataFrame( + { + "True coefficients": true_coef, + "Lasso": lasso.coef_, + "ARDRegression": ard.coef_, + "ElasticNet": enet.coef_, + } +) + +plt.figure(figsize=(10, 6)) +ax = sns.heatmap( + df.T, + norm=SymLogNorm(linthresh=10e-4, vmin=-1, vmax=1), + cbar_kws={"label": "coefficients' values"}, + cmap="seismic_r", +) +plt.ylabel("linear model") +plt.xlabel("coefficients") +plt.title( + f"Models' coefficients\nLasso $R^2$: {r2_score_lasso:.3f}, " + f"ARD $R^2$: {r2_score_ard:.3f}, " + f"ElasticNet $R^2$: {r2_score_enet:.3f}" +) +plt.tight_layout() + +# %% +# In the present example :class:`~sklearn.linear_model.ElasticNet` yields the +# best score and captures the most of the predictive features, yet still fails +# at finding all the true components. Notice that both +# :class:`~sklearn.linear_model.ElasticNet` and +# :class:`~sklearn.linear_model.ARDRegression` result in a less sparse model +# than a :class:`~sklearn.linear_model.Lasso`. +# +# Conclusions +# ----------- +# +# :class:`~sklearn.linear_model.Lasso` is known to recover sparse data +# effectively but does not perform well with highly correlated features. Indeed, +# if several correlated features contribute to the target, +# :class:`~sklearn.linear_model.Lasso` would end up selecting a single one of +# them. In the case of sparse yet non-correlated features, a +# :class:`~sklearn.linear_model.Lasso` model would be more suitable. +# +# :class:`~sklearn.linear_model.ElasticNet` introduces some sparsity on the +# coefficients and shrinks their values to zero. Thus, in the presence of +# correlated features that contribute to the target, the model is still able to +# reduce their weights without setting them exactly to zero. This results in a +# less sparse model than a pure :class:`~sklearn.linear_model.Lasso` and may +# capture non-predictive features as well. +# +# :class:`~sklearn.linear_model.ARDRegression` is better when handling gaussian +# noise, but is still unable to handle correlated features and requires a larger +# amount of time due to fitting a prior. +# +# References +# ---------- +# +# .. [1] :doi:`"Lasso-type recovery of sparse representations for +# high-dimensional data" N. Meinshausen, B. Yu - The Annals of Statistics +# 2009, Vol. 37, No. 1, 246-270 <10.1214/07-AOS582>` diff --git a/examples/linear_model/plot_lasso_coordinate_descent_path.py b/examples/linear_model/plot_lasso_coordinate_descent_path.py deleted file mode 100644 index 1a217bb863625..0000000000000 --- a/examples/linear_model/plot_lasso_coordinate_descent_path.py +++ /dev/null @@ -1,89 +0,0 @@ -""" -===================== -Lasso and Elastic Net -===================== - -Lasso and elastic net (L1 and L2 penalisation) implemented using a -coordinate descent. - -The coefficients can be forced to be positive. -""" -print(__doc__) - -# Author: Alexandre Gramfort -# License: BSD 3 clause - -from itertools import cycle -import numpy as np -import matplotlib.pyplot as plt - -from sklearn.linear_model import lasso_path, enet_path -from sklearn import datasets - - -X, y = datasets.load_diabetes(return_X_y=True) - - -X /= X.std(axis=0) # Standardize data (easier to set the l1_ratio parameter) - -# Compute paths - -eps = 5e-3 # the smaller it is the longer is the path - -print("Computing regularization path using the lasso...") -alphas_lasso, coefs_lasso, _ = lasso_path(X, y, eps=eps, fit_intercept=False) - -print("Computing regularization path using the positive lasso...") -alphas_positive_lasso, coefs_positive_lasso, _ = lasso_path( - X, y, eps=eps, positive=True, fit_intercept=False) -print("Computing regularization path using the elastic net...") -alphas_enet, coefs_enet, _ = enet_path( - X, y, eps=eps, l1_ratio=0.8, fit_intercept=False) - -print("Computing regularization path using the positive elastic net...") -alphas_positive_enet, coefs_positive_enet, _ = enet_path( - X, y, eps=eps, l1_ratio=0.8, positive=True, fit_intercept=False) - -# Display results - -plt.figure(1) -colors = cycle(['b', 'r', 'g', 'c', 'k']) -neg_log_alphas_lasso = -np.log10(alphas_lasso) -neg_log_alphas_enet = -np.log10(alphas_enet) -for coef_l, coef_e, c in zip(coefs_lasso, coefs_enet, colors): - l1 = plt.plot(neg_log_alphas_lasso, coef_l, c=c) - l2 = plt.plot(neg_log_alphas_enet, coef_e, linestyle='--', c=c) - -plt.xlabel('-Log(alpha)') -plt.ylabel('coefficients') -plt.title('Lasso and Elastic-Net Paths') -plt.legend((l1[-1], l2[-1]), ('Lasso', 'Elastic-Net'), loc='lower left') -plt.axis('tight') - - -plt.figure(2) -neg_log_alphas_positive_lasso = -np.log10(alphas_positive_lasso) -for coef_l, coef_pl, c in zip(coefs_lasso, coefs_positive_lasso, colors): - l1 = plt.plot(neg_log_alphas_lasso, coef_l, c=c) - l2 = plt.plot(neg_log_alphas_positive_lasso, coef_pl, linestyle='--', c=c) - -plt.xlabel('-Log(alpha)') -plt.ylabel('coefficients') -plt.title('Lasso and positive Lasso') -plt.legend((l1[-1], l2[-1]), ('Lasso', 'positive Lasso'), loc='lower left') -plt.axis('tight') - - -plt.figure(3) -neg_log_alphas_positive_enet = -np.log10(alphas_positive_enet) -for (coef_e, coef_pe, c) in zip(coefs_enet, coefs_positive_enet, colors): - l1 = plt.plot(neg_log_alphas_enet, coef_e, c=c) - l2 = plt.plot(neg_log_alphas_positive_enet, coef_pe, linestyle='--', c=c) - -plt.xlabel('-Log(alpha)') -plt.ylabel('coefficients') -plt.title('Elastic-Net and positive Elastic-Net') -plt.legend((l1[-1], l2[-1]), ('Elastic-Net', 'positive Elastic-Net'), - loc='lower left') -plt.axis('tight') -plt.show() diff --git a/examples/linear_model/plot_lasso_dense_vs_sparse_data.py b/examples/linear_model/plot_lasso_dense_vs_sparse_data.py index ba8f4f7bb4089..920994da1ffb5 100644 --- a/examples/linear_model/plot_lasso_dense_vs_sparse_data.py +++ b/examples/linear_model/plot_lasso_dense_vs_sparse_data.py @@ -7,21 +7,31 @@ data and that in the case of sparse data the speed is improved. """ -print(__doc__) + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause from time import time -from scipy import sparse -from scipy import linalg + +from scipy import linalg, sparse from sklearn.datasets import make_regression from sklearn.linear_model import Lasso - -# ############################################################################# -# The two Lasso implementations on Dense data -print("--- Dense matrices") +# %% +# Comparing the two Lasso implementations on Dense data +# ----------------------------------------------------- +# +# We create a linear regression problem that is suitable for the Lasso, +# that is to say, with more features than samples. We then store the data +# matrix in both dense (the usual) and sparse format, and train a Lasso on +# each. We compute the runtime of both and check that they learned the +# same model by computing the Euclidean norm of the difference between the +# coefficients they learned. Because the data is dense, we expect better +# runtime with a dense data format. X, y = make_regression(n_samples=200, n_features=5000, random_state=0) +# create a copy of X in sparse format X_sp = sparse.coo_matrix(X) alpha = 1 @@ -30,37 +40,50 @@ t0 = time() sparse_lasso.fit(X_sp, y) -print("Sparse Lasso done in %fs" % (time() - t0)) +print(f"Sparse Lasso done in {(time() - t0):.3f}s") t0 = time() dense_lasso.fit(X, y) -print("Dense Lasso done in %fs" % (time() - t0)) - -print("Distance between coefficients : %s" - % linalg.norm(sparse_lasso.coef_ - dense_lasso.coef_)) - -# ############################################################################# -# The two Lasso implementations on Sparse data -print("--- Sparse matrices") - +print(f"Dense Lasso done in {(time() - t0):.3f}s") + +# compare the regression coefficients +coeff_diff = linalg.norm(sparse_lasso.coef_ - dense_lasso.coef_) +print(f"Distance between coefficients : {coeff_diff:.2e}") + +# +# %% +# Comparing the two Lasso implementations on Sparse data +# ------------------------------------------------------ +# +# We make the previous problem sparse by replacing all small values with 0 +# and run the same comparisons as above. Because the data is now sparse, we +# expect the implementation that uses the sparse data format to be faster. + +# make a copy of the previous data Xs = X.copy() +# make Xs sparse by replacing the values lower than 2.5 with 0s Xs[Xs < 2.5] = 0.0 -Xs = sparse.coo_matrix(Xs) -Xs = Xs.tocsc() +# create a copy of Xs in sparse format +Xs_sp = sparse.coo_matrix(Xs) +Xs_sp = Xs_sp.tocsc() -print("Matrix density : %s %%" % (Xs.nnz / float(X.size) * 100)) +# compute the proportion of non-zero coefficient in the data matrix +print(f"Matrix density : {(Xs_sp.nnz / float(X.size) * 100):.3f}%") alpha = 0.1 sparse_lasso = Lasso(alpha=alpha, fit_intercept=False, max_iter=10000) dense_lasso = Lasso(alpha=alpha, fit_intercept=False, max_iter=10000) t0 = time() -sparse_lasso.fit(Xs, y) -print("Sparse Lasso done in %fs" % (time() - t0)) +sparse_lasso.fit(Xs_sp, y) +print(f"Sparse Lasso done in {(time() - t0):.3f}s") t0 = time() -dense_lasso.fit(Xs.toarray(), y) -print("Dense Lasso done in %fs" % (time() - t0)) +dense_lasso.fit(Xs, y) +print(f"Dense Lasso done in {(time() - t0):.3f}s") + +# compare the regression coefficients +coeff_diff = linalg.norm(sparse_lasso.coef_ - dense_lasso.coef_) +print(f"Distance between coefficients : {coeff_diff:.2e}") -print("Distance between coefficients : %s" - % linalg.norm(sparse_lasso.coef_ - dense_lasso.coef_)) +# %% diff --git a/examples/linear_model/plot_lasso_lars.py b/examples/linear_model/plot_lasso_lars.py deleted file mode 100644 index 06d4c94bbed70..0000000000000 --- a/examples/linear_model/plot_lasso_lars.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -""" -===================== -Lasso path using LARS -===================== - -Computes Lasso Path along the regularization parameter using the LARS -algorithm on the diabetes dataset. Each color represents a different -feature of the coefficient vector, and this is displayed as a function -of the regularization parameter. - -""" -print(__doc__) - -# Author: Fabian Pedregosa -# Alexandre Gramfort -# License: BSD 3 clause - -import numpy as np -import matplotlib.pyplot as plt - -from sklearn import linear_model -from sklearn import datasets - -X, y = datasets.load_diabetes(return_X_y=True) - -print("Computing regularization path using the LARS ...") -_, _, coefs = linear_model.lars_path(X, y, method='lasso', verbose=True) - -xx = np.sum(np.abs(coefs.T), axis=1) -xx /= xx[-1] - -plt.plot(xx, coefs.T) -ymin, ymax = plt.ylim() -plt.vlines(xx, ymin, ymax, linestyle='dashed') -plt.xlabel('|coef| / max|coef|') -plt.ylabel('Coefficients') -plt.title('LASSO Path') -plt.axis('tight') -plt.show() diff --git a/examples/linear_model/plot_lasso_lars_ic.py b/examples/linear_model/plot_lasso_lars_ic.py new file mode 100644 index 0000000000000..3e1d268c05e03 --- /dev/null +++ b/examples/linear_model/plot_lasso_lars_ic.py @@ -0,0 +1,115 @@ +""" +============================================== +Lasso model selection via information criteria +============================================== + +This example reproduces the example of Fig. 2 of [ZHT2007]_. A +:class:`~sklearn.linear_model.LassoLarsIC` estimator is fit on a +diabetes dataset and the AIC and the BIC criteria are used to select +the best model. + +.. note:: + It is important to note that the optimization to find `alpha` with + :class:`~sklearn.linear_model.LassoLarsIC` relies on the AIC or BIC + criteria that are computed in-sample, thus on the training set directly. + This approach differs from the cross-validation procedure. For a comparison + of the two approaches, you can refer to the following example: + :ref:`sphx_glr_auto_examples_linear_model_plot_lasso_model_selection.py`. + +.. rubric:: References + +.. [ZHT2007] :arxiv:`Zou, Hui, Trevor Hastie, and Robert Tibshirani. + "On the degrees of freedom of the lasso." + The Annals of Statistics 35.5 (2007): 2173-2192. + <0712.0881>` +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# We will use the diabetes dataset. +from sklearn.datasets import load_diabetes + +X, y = load_diabetes(return_X_y=True, as_frame=True) +n_samples = X.shape[0] +X.head() + +# %% +# Scikit-learn provides an estimator called +# :class:`~sklearn.linear_model.LassoLarsIC` that uses either Akaike's +# information criterion (AIC) or the Bayesian information criterion (BIC) to +# select the best model. Before fitting +# this model, we will scale the dataset. +# +# In the following, we are going to fit two models to compare the values +# reported by AIC and BIC. +from sklearn.linear_model import LassoLarsIC +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import StandardScaler + +lasso_lars_ic = make_pipeline(StandardScaler(), LassoLarsIC(criterion="aic")).fit(X, y) + + +# %% +# To be in line with the definition in [ZHT2007]_, we need to rescale the +# AIC and the BIC. Indeed, Zou et al. are ignoring some constant terms +# compared to the original definition of AIC derived from the maximum +# log-likelihood of a linear model. You can refer to +# :ref:`mathematical detail section for the User Guide `. +def zou_et_al_criterion_rescaling(criterion, n_samples, noise_variance): + """Rescale the information criterion to follow the definition of Zou et al.""" + return criterion - n_samples * np.log(2 * np.pi * noise_variance) - n_samples + + +# %% +import numpy as np + +aic_criterion = zou_et_al_criterion_rescaling( + lasso_lars_ic[-1].criterion_, + n_samples, + lasso_lars_ic[-1].noise_variance_, +) + +index_alpha_path_aic = np.flatnonzero( + lasso_lars_ic[-1].alphas_ == lasso_lars_ic[-1].alpha_ +)[0] + +# %% +lasso_lars_ic.set_params(lassolarsic__criterion="bic").fit(X, y) + +bic_criterion = zou_et_al_criterion_rescaling( + lasso_lars_ic[-1].criterion_, + n_samples, + lasso_lars_ic[-1].noise_variance_, +) + +index_alpha_path_bic = np.flatnonzero( + lasso_lars_ic[-1].alphas_ == lasso_lars_ic[-1].alpha_ +)[0] + +# %% +# Now that we collected the AIC and BIC, we can as well check that the minima +# of both criteria happen at the same alpha. Then, we can simplify the +# following plot. +index_alpha_path_aic == index_alpha_path_bic + +# %% +# Finally, we can plot the AIC and BIC criterion and the subsequent selected +# regularization parameter. +import matplotlib.pyplot as plt + +plt.plot(aic_criterion, color="tab:blue", marker="o", label="AIC criterion") +plt.plot(bic_criterion, color="tab:orange", marker="o", label="BIC criterion") +plt.vlines( + index_alpha_path_bic, + aic_criterion.min(), + aic_criterion.max(), + color="black", + linestyle="--", + label="Selected alpha", +) +plt.legend() +plt.ylabel("Information criterion") +plt.xlabel("Lasso model sequence") +_ = plt.title("Lasso model selection via AIC and BIC") diff --git a/examples/linear_model/plot_lasso_lasso_lars_elasticnet_path.py b/examples/linear_model/plot_lasso_lasso_lars_elasticnet_path.py new file mode 100644 index 0000000000000..44ae64c4c2811 --- /dev/null +++ b/examples/linear_model/plot_lasso_lasso_lars_elasticnet_path.py @@ -0,0 +1,136 @@ +""" +======================================== +Lasso, Lasso-LARS, and Elastic Net paths +======================================== + +This example shows how to compute the "paths" of coefficients along the Lasso, +Lasso-LARS, and Elastic Net regularization paths. In other words, it shows the +relationship between the regularization parameter (alpha) and the coefficients. + +Lasso and Lasso-LARS impose a sparsity constraint on the coefficients, +encouraging some of them to be zero. Elastic Net is a generalization of +Lasso that adds an L2 penalty term to the L1 penalty term. This allows for +some coefficients to be non-zero while still encouraging sparsity. + +Lasso and Elastic Net use a coordinate descent method to compute the paths, while +Lasso-LARS uses the LARS algorithm to compute the paths. + +The paths are computed using :func:`~sklearn.linear_model.lasso_path`, +:func:`~sklearn.linear_model.lars_path`, and :func:`~sklearn.linear_model.enet_path`. + +The results show different comparison plots: + +- Compare Lasso and Lasso-LARS +- Compare Lasso and Elastic Net +- Compare Lasso with positive Lasso +- Compare LARS and Positive LARS +- Compare Elastic Net and positive Elastic Net + +Each plot shows how the model coefficients vary as the regularization strength changes, +offering insight into the behavior of these models +under different constraints. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from itertools import cycle + +import matplotlib.pyplot as plt + +from sklearn.datasets import load_diabetes +from sklearn.linear_model import enet_path, lars_path, lasso_path + +X, y = load_diabetes(return_X_y=True) +X /= X.std(axis=0) # Standardize data (easier to set the l1_ratio parameter) + +# Compute paths + +eps = 5e-3 # the smaller it is the longer is the path + +print("Computing regularization path using the lasso...") +alphas_lasso, coefs_lasso, _ = lasso_path(X, y, eps=eps) + +print("Computing regularization path using the positive lasso...") +alphas_positive_lasso, coefs_positive_lasso, _ = lasso_path( + X, y, eps=eps, positive=True +) + +print("Computing regularization path using the LARS...") +alphas_lars, _, coefs_lars = lars_path(X, y, method="lasso") + +print("Computing regularization path using the positive LARS...") +alphas_positive_lars, _, coefs_positive_lars = lars_path( + X, y, method="lasso", positive=True +) + +print("Computing regularization path using the elastic net...") +alphas_enet, coefs_enet, _ = enet_path(X, y, eps=eps, l1_ratio=0.8) + +print("Computing regularization path using the positive elastic net...") +alphas_positive_enet, coefs_positive_enet, _ = enet_path( + X, y, eps=eps, l1_ratio=0.8, positive=True +) + +# Display results + +plt.figure(1) +colors = cycle(["b", "r", "g", "c", "k"]) +for coef_lasso, coef_lars, c in zip(coefs_lasso, coefs_lars, colors): + l1 = plt.semilogx(alphas_lasso, coef_lasso, c=c) + l2 = plt.semilogx(alphas_lars, coef_lars, linestyle="--", c=c) + +plt.xlabel("alpha") +plt.ylabel("coefficients") +plt.title("Lasso and LARS Paths") +plt.legend((l1[-1], l2[-1]), ("Lasso", "LARS"), loc="lower right") +plt.axis("tight") + +plt.figure(2) +colors = cycle(["b", "r", "g", "c", "k"]) +for coef_l, coef_e, c in zip(coefs_lasso, coefs_enet, colors): + l1 = plt.semilogx(alphas_lasso, coef_l, c=c) + l2 = plt.semilogx(alphas_enet, coef_e, linestyle="--", c=c) + +plt.xlabel("alpha") +plt.ylabel("coefficients") +plt.title("Lasso and Elastic-Net Paths") +plt.legend((l1[-1], l2[-1]), ("Lasso", "Elastic-Net"), loc="lower right") +plt.axis("tight") + + +plt.figure(3) +for coef_l, coef_pl, c in zip(coefs_lasso, coefs_positive_lasso, colors): + l1 = plt.semilogy(alphas_lasso, coef_l, c=c) + l2 = plt.semilogy(alphas_positive_lasso, coef_pl, linestyle="--", c=c) + +plt.xlabel("alpha") +plt.ylabel("coefficients") +plt.title("Lasso and positive Lasso") +plt.legend((l1[-1], l2[-1]), ("Lasso", "positive Lasso"), loc="lower right") +plt.axis("tight") + + +plt.figure(4) +colors = cycle(["b", "r", "g", "c", "k"]) +for coef_lars, coef_positive_lars, c in zip(coefs_lars, coefs_positive_lars, colors): + l1 = plt.semilogx(alphas_lars, coef_lars, c=c) + l2 = plt.semilogx(alphas_positive_lars, coef_positive_lars, linestyle="--", c=c) + +plt.xlabel("alpha") +plt.ylabel("coefficients") +plt.title("LARS and Positive LARS") +plt.legend((l1[-1], l2[-1]), ("LARS", "Positive LARS"), loc="lower right") +plt.axis("tight") + +plt.figure(5) +for coef_e, coef_pe, c in zip(coefs_enet, coefs_positive_enet, colors): + l1 = plt.semilogx(alphas_enet, coef_e, c=c) + l2 = plt.semilogx(alphas_positive_enet, coef_pe, linestyle="--", c=c) + +plt.xlabel("alpha") +plt.ylabel("coefficients") +plt.title("Elastic-Net and positive Elastic-Net") +plt.legend((l1[-1], l2[-1]), ("Elastic-Net", "positive Elastic-Net"), loc="lower right") +plt.axis("tight") +plt.show() diff --git a/examples/linear_model/plot_lasso_model_selection.py b/examples/linear_model/plot_lasso_model_selection.py index 73fc94fb94600..3f8baa901f399 100644 --- a/examples/linear_model/plot_lasso_model_selection.py +++ b/examples/linear_model/plot_lasso_model_selection.py @@ -1,151 +1,249 @@ """ -=================================================== -Lasso model selection: Cross-Validation / AIC / BIC -=================================================== - -Use the Akaike information criterion (AIC), the Bayes Information -criterion (BIC) and cross-validation to select an optimal value -of the regularization parameter alpha of the :ref:`lasso` estimator. - -Results obtained with LassoLarsIC are based on AIC/BIC criteria. - -Information-criterion based model selection is very fast, but it -relies on a proper estimation of degrees of freedom, are -derived for large samples (asymptotic results) and assume the model -is correct, i.e. that the data are actually generated by this model. -They also tend to break when the problem is badly conditioned -(more features than samples). - -For cross-validation, we use 20-fold with 2 algorithms to compute the -Lasso path: coordinate descent, as implemented by the LassoCV class, and -Lars (least angle regression) as implemented by the LassoLarsCV class. -Both algorithms give roughly the same results. They differ with regards -to their execution speed and sources of numerical errors. - -Lars computes a path solution only for each kink in the path. As a -result, it is very efficient when there are only of few kinks, which is -the case if there are few features or samples. Also, it is able to -compute the full path without setting any meta parameter. On the -opposite, coordinate descent compute the path points on a pre-specified -grid (here we use the default). Thus it is more efficient if the number -of grid points is smaller than the number of kinks in the path. Such a -strategy can be interesting if the number of features is really large -and there are enough samples to select a large amount. In terms of -numerical errors, for heavily correlated variables, Lars will accumulate -more errors, while the coordinate descent algorithm will only sample the -path on a grid. - -Note how the optimal value of alpha varies for each fold. This -illustrates why nested-cross validation is necessary when trying to -evaluate the performance of a method for which a parameter is chosen by -cross-validation: this choice of parameter may not be optimal for unseen -data. -""" -print(__doc__) - -# Author: Olivier Grisel, Gael Varoquaux, Alexandre Gramfort -# License: BSD 3 clause - -import time - -import numpy as np -import matplotlib.pyplot as plt - -from sklearn.linear_model import LassoCV, LassoLarsCV, LassoLarsIC -from sklearn import datasets - -# This is to avoid division by zero while doing np.log10 -EPSILON = 1e-4 - -X, y = datasets.load_diabetes(return_X_y=True) - -rng = np.random.RandomState(42) -X = np.c_[X, rng.randn(X.shape[0], 14)] # add some bad features +================================================= +Lasso model selection: AIC-BIC / cross-validation +================================================= -# normalize data as done by Lars to allow for comparison -X /= np.sqrt(np.sum(X ** 2, axis=0)) +This example focuses on model selection for Lasso models that are +linear models with an L1 penalty for regression problems. -# ############################################################################# -# LassoLarsIC: least angle regression with BIC/AIC criterion +Indeed, several strategies can be used to select the value of the +regularization parameter: via cross-validation or using an information +criterion, namely AIC or BIC. -model_bic = LassoLarsIC(criterion='bic') -t1 = time.time() -model_bic.fit(X, y) -t_bic = time.time() - t1 -alpha_bic_ = model_bic.alpha_ - -model_aic = LassoLarsIC(criterion='aic') -model_aic.fit(X, y) -alpha_aic_ = model_aic.alpha_ +In what follows, we will discuss in details the different strategies. +""" +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -def plot_ic_criterion(model, name, color): - criterion_ = model.criterion_ - plt.semilogx(model.alphas_ + EPSILON, criterion_, '--', color=color, - linewidth=3, label='%s criterion' % name) - plt.axvline(model.alpha_ + EPSILON, color=color, linewidth=3, - label='alpha: %s estimate' % name) - plt.xlabel(r'$\alpha$') - plt.ylabel('criterion') +# %% +# Dataset +# ------- +# In this example, we will use the diabetes dataset. +from sklearn.datasets import load_diabetes +X, y = load_diabetes(return_X_y=True, as_frame=True) +X.head() -plt.figure() -plot_ic_criterion(model_aic, 'AIC', 'b') -plot_ic_criterion(model_bic, 'BIC', 'r') -plt.legend() -plt.title('Information-criterion for model selection (training time %.3fs)' - % t_bic) +# %% +# In addition, we add some random features to the original data to +# better illustrate the feature selection performed by the Lasso model. +import numpy as np +import pandas as pd -# ############################################################################# -# LassoCV: coordinate descent +rng = np.random.RandomState(42) +n_random_features = 14 +X_random = pd.DataFrame( + rng.randn(X.shape[0], n_random_features), + columns=[f"random_{i:02d}" for i in range(n_random_features)], +) +X = pd.concat([X, X_random], axis=1) +# Show only a subset of the columns +X[X.columns[::3]].head() + +# %% +# Selecting Lasso via an information criterion +# -------------------------------------------- +# :class:`~sklearn.linear_model.LassoLarsIC` provides a Lasso estimator that +# uses the Akaike information criterion (AIC) or the Bayes information +# criterion (BIC) to select the optimal value of the regularization +# parameter alpha. +# +# Before fitting the model, we will standardize the data with a +# :class:`~sklearn.preprocessing.StandardScaler`. In addition, we will +# measure the time to fit and tune the hyperparameter alpha in order to +# compare with the cross-validation strategy. +# +# We will first fit a Lasso model with the AIC criterion. +import time -# Compute paths -print("Computing regularization path using the coordinate descent lasso...") -t1 = time.time() -model = LassoCV(cv=20).fit(X, y) -t_lasso_cv = time.time() - t1 +from sklearn.linear_model import LassoLarsIC +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import StandardScaler + +start_time = time.time() +lasso_lars_ic = make_pipeline(StandardScaler(), LassoLarsIC(criterion="aic")).fit(X, y) +fit_time = time.time() - start_time + +# %% +# We store the AIC metric for each value of alpha used during `fit`. +results = pd.DataFrame( + { + "alphas": lasso_lars_ic[-1].alphas_, + "AIC criterion": lasso_lars_ic[-1].criterion_, + } +).set_index("alphas") +alpha_aic = lasso_lars_ic[-1].alpha_ + +# %% +# Now, we perform the same analysis using the BIC criterion. +lasso_lars_ic.set_params(lassolarsic__criterion="bic").fit(X, y) +results["BIC criterion"] = lasso_lars_ic[-1].criterion_ +alpha_bic = lasso_lars_ic[-1].alpha_ + + +# %% +# We can check which value of `alpha` leads to the minimum AIC and BIC. +def highlight_min(x): + x_min = x.min() + return ["font-weight: bold" if v == x_min else "" for v in x] + + +results.style.apply(highlight_min) + +# %% +# Finally, we can plot the AIC and BIC values for the different alpha values. +# The vertical lines in the plot correspond to the alpha chosen for each +# criterion. The selected alpha corresponds to the minimum of the AIC or BIC +# criterion. +ax = results.plot() +ax.vlines( + alpha_aic, + results["AIC criterion"].min(), + results["AIC criterion"].max(), + label="alpha: AIC estimate", + linestyles="--", + color="tab:blue", +) +ax.vlines( + alpha_bic, + results["BIC criterion"].min(), + results["BIC criterion"].max(), + label="alpha: BIC estimate", + linestyle="--", + color="tab:orange", +) +ax.set_xlabel(r"$\alpha$") +ax.set_ylabel("criterion") +ax.set_xscale("log") +ax.legend() +_ = ax.set_title( + f"Information-criterion for model selection (training time {fit_time:.2f}s)" +) + +# %% +# Model selection with an information-criterion is very fast. It relies on +# computing the criterion on the in-sample set provided to `fit`. Both criteria +# estimate the model generalization error based on the training set error and +# penalize this overly optimistic error. However, this penalty relies on a +# proper estimation of the degrees of freedom and the noise variance. Both are +# derived for large samples (asymptotic results) and assume the model is +# correct, i.e. that the data are actually generated by this model. +# +# These models also tend to break when the problem is badly conditioned (more +# features than samples). It is then required to provide an estimate of the +# noise variance. +# +# Selecting Lasso via cross-validation +# ------------------------------------ +# The Lasso estimator can be implemented with different solvers: coordinate +# descent and least angle regression. They differ with regards to their +# execution speed and sources of numerical errors. +# +# In scikit-learn, two different estimators are available with integrated +# cross-validation: :class:`~sklearn.linear_model.LassoCV` and +# :class:`~sklearn.linear_model.LassoLarsCV` that respectively solve the +# problem with coordinate descent and least angle regression. +# +# In the remainder of this section, we will present both approaches. For both +# algorithms, we will use a 20-fold cross-validation strategy. +# +# Lasso via coordinate descent +# ............................ +# Let's start by making the hyperparameter tuning using +# :class:`~sklearn.linear_model.LassoCV`. +from sklearn.linear_model import LassoCV + +start_time = time.time() +model = make_pipeline(StandardScaler(), LassoCV(cv=20)).fit(X, y) +fit_time = time.time() - start_time + +# %% +import matplotlib.pyplot as plt -# Display results -plt.figure() ymin, ymax = 2300, 3800 -plt.semilogx(model.alphas_ + EPSILON, model.mse_path_, ':') -plt.plot(model.alphas_ + EPSILON, model.mse_path_.mean(axis=-1), 'k', - label='Average across the folds', linewidth=2) -plt.axvline(model.alpha_ + EPSILON, linestyle='--', color='k', - label='alpha: CV estimate') - -plt.legend() +lasso = model[-1] +plt.semilogx(lasso.alphas_, lasso.mse_path_, linestyle=":") +plt.plot( + lasso.alphas_, + lasso.mse_path_.mean(axis=-1), + color="black", + label="Average across the folds", + linewidth=2, +) +plt.axvline(lasso.alpha_, linestyle="--", color="black", label="alpha: CV estimate") -plt.xlabel(r'$\alpha$') -plt.ylabel('Mean square error') -plt.title('Mean square error on each fold: coordinate descent ' - '(train time: %.2fs)' % t_lasso_cv) -plt.axis('tight') plt.ylim(ymin, ymax) - -# ############################################################################# -# LassoLarsCV: least angle regression - -# Compute paths -print("Computing regularization path using the Lars lasso...") -t1 = time.time() -model = LassoLarsCV(cv=20).fit(X, y) -t_lasso_lars_cv = time.time() - t1 - -# Display results -plt.figure() -plt.semilogx(model.cv_alphas_ + EPSILON, model.mse_path_, ':') -plt.semilogx(model.cv_alphas_ + EPSILON, model.mse_path_.mean(axis=-1), 'k', - label='Average across the folds', linewidth=2) -plt.axvline(model.alpha_, linestyle='--', color='k', - label='alpha CV') +plt.xlabel(r"$\alpha$") +plt.ylabel("Mean square error") plt.legend() +_ = plt.title( + f"Mean square error on each fold: coordinate descent (train time: {fit_time:.2f}s)" +) + +# %% +# Lasso via least angle regression +# ................................ +# Let's start by making the hyperparameter tuning using +# :class:`~sklearn.linear_model.LassoLarsCV`. +from sklearn.linear_model import LassoLarsCV + +start_time = time.time() +model = make_pipeline(StandardScaler(), LassoLarsCV(cv=20)).fit(X, y) +fit_time = time.time() - start_time + +# %% +lasso = model[-1] +plt.semilogx(lasso.cv_alphas_, lasso.mse_path_, ":") +plt.semilogx( + lasso.cv_alphas_, + lasso.mse_path_.mean(axis=-1), + color="black", + label="Average across the folds", + linewidth=2, +) +plt.axvline(lasso.alpha_, linestyle="--", color="black", label="alpha CV") -plt.xlabel(r'$\alpha$') -plt.ylabel('Mean square error') -plt.title('Mean square error on each fold: Lars (train time: %.2fs)' - % t_lasso_lars_cv) -plt.axis('tight') plt.ylim(ymin, ymax) - -plt.show() +plt.xlabel(r"$\alpha$") +plt.ylabel("Mean square error") +plt.legend() +_ = plt.title(f"Mean square error on each fold: Lars (train time: {fit_time:.2f}s)") + +# %% +# Summary of cross-validation approach +# .................................... +# Both algorithms give roughly the same results. +# +# Lars computes a solution path only for each kink in the path. As a result, it +# is very efficient when there are only of few kinks, which is the case if +# there are few features or samples. Also, it is able to compute the full path +# without setting any hyperparameter. On the opposite, coordinate descent +# computes the path points on a pre-specified grid (here we use the default). +# Thus it is more efficient if the number of grid points is smaller than the +# number of kinks in the path. Such a strategy can be interesting if the number +# of features is really large and there are enough samples to be selected in +# each of the cross-validation fold. In terms of numerical errors, for heavily +# correlated variables, Lars will accumulate more errors, while the coordinate +# descent algorithm will only sample the path on a grid. +# +# Note how the optimal value of alpha varies for each fold. This illustrates +# why nested-cross validation is a good strategy when trying to evaluate the +# performance of a method for which a parameter is chosen by cross-validation: +# this choice of parameter may not be optimal for a final evaluation on +# unseen test set only. +# +# Conclusion +# ---------- +# In this tutorial, we presented two approaches for selecting the best +# hyperparameter `alpha`: one strategy finds the optimal value of `alpha` +# by only using the training set and some information criterion, and another +# strategy is based on cross-validation. +# +# In this example, both approaches are working similarly. The in-sample +# hyperparameter selection even shows its efficacy in terms of computational +# performance. However, it can only be used when the number of samples is large +# enough compared to the number of features. +# +# That's why hyperparameter optimization via cross-validation is a safe +# strategy: it works in different settings. diff --git a/examples/linear_model/plot_logistic.py b/examples/linear_model/plot_logistic.py index 6b843007624a9..b54c1fbf1340d 100644 --- a/examples/linear_model/plot_logistic.py +++ b/examples/linear_model/plot_logistic.py @@ -1,7 +1,3 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - - """ ========================================================= Logistic function @@ -12,54 +8,59 @@ i.e. class one or two, using the logistic curve. """ -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -# Code source: Gael Varoquaux -# License: BSD 3 clause - -import numpy as np import matplotlib.pyplot as plt - -from sklearn import linear_model +import numpy as np from scipy.special import expit -# General a toy dataset:s it's just a straight line with some Gaussian noise: +from sklearn.linear_model import LinearRegression, LogisticRegression + +# Generate a toy dataset, it's just a straight line with some Gaussian noise: xmin, xmax = -5, 5 n_samples = 100 np.random.seed(0) X = np.random.normal(size=n_samples) y = (X > 0).astype(float) X[X > 0] *= 4 -X += .3 * np.random.normal(size=n_samples) +X += 0.3 * np.random.normal(size=n_samples) X = X[:, np.newaxis] # Fit the classifier -clf = linear_model.LogisticRegression(C=1e5) +clf = LogisticRegression(C=1e5) clf.fit(X, y) # and plot the result plt.figure(1, figsize=(4, 3)) plt.clf() -plt.scatter(X.ravel(), y, color='black', zorder=20) +plt.scatter(X.ravel(), y, label="example data", color="black", zorder=20) X_test = np.linspace(-5, 10, 300) loss = expit(X_test * clf.coef_ + clf.intercept_).ravel() -plt.plot(X_test, loss, color='red', linewidth=3) +plt.plot(X_test, loss, label="Logistic Regression Model", color="red", linewidth=3) -ols = linear_model.LinearRegression() +ols = LinearRegression() ols.fit(X, y) -plt.plot(X_test, ols.coef_ * X_test + ols.intercept_, linewidth=1) -plt.axhline(.5, color='.5') +plt.plot( + X_test, + ols.coef_ * X_test + ols.intercept_, + label="Linear Regression Model", + linewidth=1, +) +plt.axhline(0.5, color=".5") -plt.ylabel('y') -plt.xlabel('X') +plt.ylabel("y") +plt.xlabel("X") plt.xticks(range(-5, 10)) plt.yticks([0, 0.5, 1]) -plt.ylim(-.25, 1.25) +plt.ylim(-0.25, 1.25) plt.xlim(-4, 10) -plt.legend(('Logistic Regression Model', 'Linear Regression Model'), - loc="lower right", fontsize='small') +plt.legend( + loc="lower right", + fontsize="small", +) plt.tight_layout() plt.show() diff --git a/examples/linear_model/plot_logistic_l1_l2_sparsity.py b/examples/linear_model/plot_logistic_l1_l2_sparsity.py index 3e518e8ec1e7a..f642dfade5db8 100644 --- a/examples/linear_model/plot_logistic_l1_l2_sparsity.py +++ b/examples/linear_model/plot_logistic_l1_l2_sparsity.py @@ -12,20 +12,17 @@ We classify 8x8 images of digits into two classes: 0-4 against 5-9. The visualization shows coefficients of the models for varying C. -""" -print(__doc__) +""" -# Authors: Alexandre Gramfort -# Mathieu Blondel -# Andreas Mueller -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt +import numpy as np -from sklearn.linear_model import LogisticRegression from sklearn import datasets +from sklearn.linear_model import LogisticRegression from sklearn.preprocessing import StandardScaler X, y = datasets.load_digits(return_X_y=True) @@ -41,11 +38,12 @@ # Set regularization parameter for i, (C, axes_row) in enumerate(zip((1, 0.1, 0.01), axes)): - # turn down tolerance for short training time - clf_l1_LR = LogisticRegression(C=C, penalty='l1', tol=0.01, solver='saga') - clf_l2_LR = LogisticRegression(C=C, penalty='l2', tol=0.01, solver='saga') - clf_en_LR = LogisticRegression(C=C, penalty='elasticnet', solver='saga', - l1_ratio=l1_ratio, tol=0.01) + # Increase tolerance for short training time + clf_l1_LR = LogisticRegression(C=C, penalty="l1", tol=0.01, solver="saga") + clf_l2_LR = LogisticRegression(C=C, penalty="l2", tol=0.01, solver="saga") + clf_en_LR = LogisticRegression( + C=C, penalty="elasticnet", solver="saga", l1_ratio=l1_ratio, tol=0.01 + ) clf_l1_LR.fit(X, y) clf_l2_LR.fit(X, y) clf_en_LR.fit(X, y) @@ -61,17 +59,13 @@ sparsity_l2_LR = np.mean(coef_l2_LR == 0) * 100 sparsity_en_LR = np.mean(coef_en_LR == 0) * 100 - print("C=%.2f" % C) - print("{:<40} {:.2f}%".format("Sparsity with L1 penalty:", sparsity_l1_LR)) - print("{:<40} {:.2f}%".format("Sparsity with Elastic-Net penalty:", - sparsity_en_LR)) - print("{:<40} {:.2f}%".format("Sparsity with L2 penalty:", sparsity_l2_LR)) - print("{:<40} {:.2f}".format("Score with L1 penalty:", - clf_l1_LR.score(X, y))) - print("{:<40} {:.2f}".format("Score with Elastic-Net penalty:", - clf_en_LR.score(X, y))) - print("{:<40} {:.2f}".format("Score with L2 penalty:", - clf_l2_LR.score(X, y))) + print(f"C={C:.2f}") + print(f"{'Sparsity with L1 penalty:':<40} {sparsity_l1_LR:.2f}%") + print(f"{'Sparsity with Elastic-Net penalty:':<40} {sparsity_en_LR:.2f}%") + print(f"{'Sparsity with L2 penalty:':<40} {sparsity_l2_LR:.2f}%") + print(f"{'Score with L1 penalty:':<40} {clf_l1_LR.score(X, y):.2f}") + print(f"{'Score with Elastic-Net penalty:':<40} {clf_en_LR.score(X, y):.2f}") + print(f"{'Score with L2 penalty:':<40} {clf_l2_LR.score(X, y):.2f}") if i == 0: axes_row[0].set_title("L1 penalty") @@ -79,11 +73,16 @@ axes_row[2].set_title("L2 penalty") for ax, coefs in zip(axes_row, [coef_l1_LR, coef_en_LR, coef_l2_LR]): - ax.imshow(np.abs(coefs.reshape(8, 8)), interpolation='nearest', - cmap='binary', vmax=1, vmin=0) + ax.imshow( + np.abs(coefs.reshape(8, 8)), + interpolation="nearest", + cmap="binary", + vmax=1, + vmin=0, + ) ax.set_xticks(()) ax.set_yticks(()) - axes_row[0].set_ylabel('C = %s' % C) + axes_row[0].set_ylabel(f"C = {C}") plt.show() diff --git a/examples/linear_model/plot_logistic_multinomial.py b/examples/linear_model/plot_logistic_multinomial.py index 518a2aeade61c..c12229c81c7f1 100644 --- a/examples/linear_model/plot_logistic_multinomial.py +++ b/examples/linear_model/plot_logistic_multinomial.py @@ -1,71 +1,193 @@ """ -==================================================== -Plot multinomial and One-vs-Rest Logistic Regression -==================================================== +====================================================================== +Decision Boundaries of Multinomial and One-vs-Rest Logistic Regression +====================================================================== -Plot decision surface of multinomial and One-vs-Rest Logistic Regression. -The hyperplanes corresponding to the three One-vs-Rest (OVR) classifiers -are represented by the dashed lines. +This example compares decision boundaries of multinomial and one-vs-rest +logistic regression on a 2D dataset with three classes. + +We make a comparison of the decision boundaries of both methods that is equivalent +to call the method `predict`. In addition, we plot the hyperplanes that correspond to +the line when the probability estimate for a class is of 0.5. """ -print(__doc__) -# Authors: Tom Dupre la Tour -# License: BSD 3 clause -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Dataset Generation +# ------------------ +# +# We generate a synthetic dataset using :func:`~sklearn.datasets.make_blobs` function. +# The dataset consists of 1,000 samples from three different classes, +# centered around [-5, 0], [0, 1.5], and [5, -1]. After generation, we apply a linear +# transformation to introduce some correlation between features and make the problem +# more challenging. This results in a 2D dataset with three overlapping classes, +# suitable for demonstrating the differences between multinomial and one-vs-rest +# logistic regression. import matplotlib.pyplot as plt +import numpy as np + from sklearn.datasets import make_blobs -from sklearn.linear_model import LogisticRegression -# make 3-class dataset for classification centers = [[-5, 0], [0, 1.5], [5, -1]] -X, y = make_blobs(n_samples=1000, centers=centers, random_state=40) +X, y = make_blobs(n_samples=1_000, centers=centers, random_state=40) transformation = [[0.4, 0.2], [-0.4, 1.2]] X = np.dot(X, transformation) -for multi_class in ('multinomial', 'ovr'): - clf = LogisticRegression(solver='sag', max_iter=100, random_state=42, - multi_class=multi_class).fit(X, y) - - # print the training scores - print("training score : %.3f (%s)" % (clf.score(X, y), multi_class)) - - # create a mesh to plot in - h = .02 # step size in the mesh - x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1 - y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1 - xx, yy = np.meshgrid(np.arange(x_min, x_max, h), - np.arange(y_min, y_max, h)) - - # Plot the decision boundary. For that, we will assign a color to each - # point in the mesh [x_min, x_max]x[y_min, y_max]. - Z = clf.predict(np.c_[xx.ravel(), yy.ravel()]) - # Put the result into a color plot - Z = Z.reshape(xx.shape) - plt.figure() - plt.contourf(xx, yy, Z, cmap=plt.cm.Paired) - plt.title("Decision surface of LogisticRegression (%s)" % multi_class) - plt.axis('tight') - - # Plot also the training points - colors = "bry" - for i, color in zip(clf.classes_, colors): - idx = np.where(y == i) - plt.scatter(X[idx, 0], X[idx, 1], c=color, cmap=plt.cm.Paired, - edgecolor='black', s=20) - - # Plot the three one-against-all classifiers - xmin, xmax = plt.xlim() - ymin, ymax = plt.ylim() - coef = clf.coef_ - intercept = clf.intercept_ - - def plot_hyperplane(c, color): - def line(x0): - return (-(x0 * coef[c, 0]) - intercept[c]) / coef[c, 1] - plt.plot([xmin, xmax], [line(xmin), line(xmax)], - ls="--", color=color) - - for i, color in zip(clf.classes_, colors): - plot_hyperplane(i, color) +fig, ax = plt.subplots(figsize=(6, 4)) + +scatter = ax.scatter(X[:, 0], X[:, 1], c=y, edgecolor="black") +ax.set(title="Synthetic Dataset", xlabel="Feature 1", ylabel="Feature 2") +_ = ax.legend(*scatter.legend_elements(), title="Classes") + + +# %% +# Classifier Training +# ------------------- +# +# We train two different logistic regression classifiers: multinomial and one-vs-rest. +# The multinomial classifier handles all classes simultaneously, while the one-vs-rest +# approach trains a binary classifier for each class against all others. +from sklearn.linear_model import LogisticRegression +from sklearn.multiclass import OneVsRestClassifier + +logistic_regression_multinomial = LogisticRegression().fit(X, y) +logistic_regression_ovr = OneVsRestClassifier(LogisticRegression()).fit(X, y) + +accuracy_multinomial = logistic_regression_multinomial.score(X, y) +accuracy_ovr = logistic_regression_ovr.score(X, y) + +# %% +# Decision Boundaries Visualization +# --------------------------------- +# +# Let's visualize the decision boundaries of both models that is provided by the +# method `predict` of the classifiers. +from sklearn.inspection import DecisionBoundaryDisplay + +fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 5), sharex=True, sharey=True) + +for model, title, ax in [ + ( + logistic_regression_multinomial, + f"Multinomial Logistic Regression\n(Accuracy: {accuracy_multinomial:.3f})", + ax1, + ), + ( + logistic_regression_ovr, + f"One-vs-Rest Logistic Regression\n(Accuracy: {accuracy_ovr:.3f})", + ax2, + ), +]: + DecisionBoundaryDisplay.from_estimator( + model, + X, + ax=ax, + response_method="predict", + alpha=0.8, + ) + scatter = ax.scatter(X[:, 0], X[:, 1], c=y, edgecolor="k") + legend = ax.legend(*scatter.legend_elements(), title="Classes") + ax.add_artist(legend) + ax.set_title(title) + + +# %% +# We see that the decision boundaries are different. This difference stems from their +# approaches: +# +# - Multinomial logistic regression considers all classes simultaneously during +# optimization. +# - One-vs-rest logistic regression fits each class independently against all others. +# +# These distinct strategies can lead to varying decision boundaries, especially in +# complex multi-class problems. +# +# Hyperplanes Visualization +# -------------------------- +# +# We also visualize the hyperplanes that correspond to the line when the probability +# estimate for a class is of 0.5. +def plot_hyperplanes(classifier, X, ax): + xmin, xmax = X[:, 0].min(), X[:, 0].max() + ymin, ymax = X[:, 1].min(), X[:, 1].max() + ax.set(xlim=(xmin, xmax), ylim=(ymin, ymax)) + + if isinstance(classifier, OneVsRestClassifier): + coef = np.concatenate([est.coef_ for est in classifier.estimators_]) + intercept = np.concatenate([est.intercept_ for est in classifier.estimators_]) + else: + coef = classifier.coef_ + intercept = classifier.intercept_ + + for i in range(coef.shape[0]): + w = coef[i] + a = -w[0] / w[1] + xx = np.linspace(xmin, xmax) + yy = a * xx - (intercept[i]) / w[1] + ax.plot(xx, yy, "--", linewidth=3, label=f"Class {i}") + + return ax.get_legend_handles_labels() + + +# %% +fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 5), sharex=True, sharey=True) + +for model, title, ax in [ + ( + logistic_regression_multinomial, + "Multinomial Logistic Regression Hyperplanes", + ax1, + ), + (logistic_regression_ovr, "One-vs-Rest Logistic Regression Hyperplanes", ax2), +]: + hyperplane_handles, hyperplane_labels = plot_hyperplanes(model, X, ax) + scatter = ax.scatter(X[:, 0], X[:, 1], c=y, edgecolor="k") + scatter_handles, scatter_labels = scatter.legend_elements() + + all_handles = hyperplane_handles + scatter_handles + all_labels = hyperplane_labels + scatter_labels + + ax.legend(all_handles, all_labels, title="Classes") + ax.set_title(title) plt.show() + +# %% +# While the hyperplanes for classes 0 and 2 are quite similar between the two methods, +# we observe that the hyperplane for class 1 is notably different. This difference stems +# from the fundamental approaches of one-vs-rest and multinomial logistic regression: +# +# For one-vs-rest logistic regression: +# +# - Each hyperplane is determined independently by considering one class against all +# others. +# - For class 1, the hyperplane represents the decision boundary that best separates +# class 1 from the combined classes 0 and 2. +# - This binary approach can lead to simpler decision boundaries but may not capture +# complex relationships between all classes simultaneously. +# - There is no possible interpretation of the conditional class probabilities. +# +# For multinomial logistic regression: +# +# - All hyperplanes are determined simultaneously, considering the relationships between +# all classes at once. +# - The loss minimized by the model is a proper scoring rule, which means that the model +# is optimized to estimate the conditional class probabilities that are, therefore, +# meaningful. +# - Each hyperplane represents the decision boundary where the probability of one class +# becomes higher than the others, based on the overall probability distribution. +# - This approach can capture more nuanced relationships between classes, potentially +# leading to more accurate classification in multi-class problems. +# +# The difference in hyperplanes, especially for class 1, highlights how these methods +# can produce different decision boundaries despite similar overall accuracy. +# +# In practice, using multinomial logistic regression is recommended since it minimizes a +# well-formulated loss function, leading to better-calibrated class probabilities and +# thus more interpretable results. When it comes to decision boundaries, one should +# formulate a utility function to transform the class probabilities into a meaningful +# quantity for the problem at hand. One-vs-rest allows for different decision boundaries +# but does not allow for fine-grained control over the trade-off between the classes as +# a utility function would. diff --git a/examples/linear_model/plot_logistic_path.py b/examples/linear_model/plot_logistic_path.py index 7aead065f3445..46608f683740e 100644 --- a/examples/linear_model/plot_logistic_path.py +++ b/examples/linear_model/plot_logistic_path.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ ============================================== Regularization path of L1- Logistic Regression @@ -25,52 +24,80 @@ full-path. """ -print(__doc__) -# Author: Alexandre Gramfort -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -from time import time -import numpy as np -import matplotlib.pyplot as plt +# %% +# Load data +# --------- -from sklearn import linear_model from sklearn import datasets -from sklearn.svm import l1_min_c iris = datasets.load_iris() X = iris.data y = iris.target +feature_names = iris.feature_names +# %% +# Here we remove the third class to make the problem a binary classification X = X[y != 2] y = y[y != 2] -X /= X.max() # Normalize X to speed-up convergence +# %% +# Compute regularization path +# --------------------------- -# ############################################################################# -# Demo path functions - -cs = l1_min_c(X, y, loss='log') * np.logspace(0, 7, 16) +import numpy as np +from sklearn.linear_model import LogisticRegression +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import StandardScaler +from sklearn.svm import l1_min_c -print("Computing regularization path ...") -start = time() -clf = linear_model.LogisticRegression(penalty='l1', solver='liblinear', - tol=1e-6, max_iter=int(1e6), - warm_start=True, - intercept_scaling=10000.) +cs = l1_min_c(X, y, loss="log") * np.logspace(0, 1, 16) + +# %% +# Create a pipeline with `StandardScaler` and `LogisticRegression`, to normalize +# the data before fitting a linear model, in order to speed-up convergence and +# make the coefficients comparable. Also, as a side effect, since the data is now +# centered around 0, we don't need to fit an intercept. +clf = make_pipeline( + StandardScaler(), + LogisticRegression( + penalty="l1", + solver="liblinear", + tol=1e-6, + max_iter=int(1e6), + warm_start=True, + fit_intercept=False, + ), +) coefs_ = [] for c in cs: - clf.set_params(C=c) + clf.set_params(logisticregression__C=c) clf.fit(X, y) - coefs_.append(clf.coef_.ravel().copy()) -print("This took %0.3fs" % (time() - start)) + coefs_.append(clf["logisticregression"].coef_.ravel().copy()) coefs_ = np.array(coefs_) -plt.plot(np.log10(cs), coefs_, marker='o') + +# %% +# Plot regularization path +# ------------------------ + +import matplotlib.pyplot as plt + +# Colorblind-friendly palette (IBM Color Blind Safe palette) +colors = ["#648FFF", "#785EF0", "#DC267F", "#FE6100"] + +plt.figure(figsize=(10, 6)) +for i in range(coefs_.shape[1]): + plt.semilogx(cs, coefs_[:, i], marker="o", color=colors[i], label=feature_names[i]) + ymin, ymax = plt.ylim() -plt.xlabel('log(C)') -plt.ylabel('Coefficients') -plt.title('Logistic Regression Path') -plt.axis('tight') +plt.xlabel("C") +plt.ylabel("Coefficients") +plt.title("Logistic Regression Path") +plt.legend() +plt.axis("tight") plt.show() diff --git a/examples/linear_model/plot_multi_task_lasso_support.py b/examples/linear_model/plot_multi_task_lasso_support.py index c7a9536383bc2..433176145e414 100644 --- a/examples/linear_model/plot_multi_task_lasso_support.py +++ b/examples/linear_model/plot_multi_task_lasso_support.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ ============================================= Joint feature selection with multi-task Lasso @@ -13,15 +12,15 @@ point. This makes feature selection by the Lasso more stable. """ -print(__doc__) -# Author: Alexandre Gramfort -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import matplotlib.pyplot as plt -import numpy as np +# %% +# Generate data +# ------------- -from sklearn.linear_model import MultiTaskLasso, Lasso +import numpy as np rng = np.random.RandomState(42) @@ -31,39 +30,53 @@ coef = np.zeros((n_tasks, n_features)) times = np.linspace(0, 2 * np.pi, n_tasks) for k in range(n_relevant_features): - coef[:, k] = np.sin((1. + rng.randn(1)) * times + 3 * rng.randn(1)) + coef[:, k] = np.sin((1.0 + rng.randn(1)) * times + 3 * rng.randn(1)) X = rng.randn(n_samples, n_features) Y = np.dot(X, coef.T) + rng.randn(n_samples, n_tasks) +# %% +# Fit models +# ---------- + +from sklearn.linear_model import Lasso, MultiTaskLasso + coef_lasso_ = np.array([Lasso(alpha=0.5).fit(X, y).coef_ for y in Y.T]) -coef_multi_task_lasso_ = MultiTaskLasso(alpha=1.).fit(X, Y).coef_ +coef_multi_task_lasso_ = MultiTaskLasso(alpha=1.0).fit(X, Y).coef_ -# ############################################################################# +# %% # Plot support and time series +# ---------------------------- + +import matplotlib.pyplot as plt + fig = plt.figure(figsize=(8, 5)) plt.subplot(1, 2, 1) plt.spy(coef_lasso_) -plt.xlabel('Feature') -plt.ylabel('Time (or Task)') -plt.text(10, 5, 'Lasso') +plt.xlabel("Feature") +plt.ylabel("Time (or Task)") +plt.text(10, 5, "Lasso") plt.subplot(1, 2, 2) plt.spy(coef_multi_task_lasso_) -plt.xlabel('Feature') -plt.ylabel('Time (or Task)') -plt.text(10, 5, 'MultiTaskLasso') -fig.suptitle('Coefficient non-zero location') +plt.xlabel("Feature") +plt.ylabel("Time (or Task)") +plt.text(10, 5, "MultiTaskLasso") +fig.suptitle("Coefficient non-zero location") feature_to_plot = 0 plt.figure() lw = 2 -plt.plot(coef[:, feature_to_plot], color='seagreen', linewidth=lw, - label='Ground truth') -plt.plot(coef_lasso_[:, feature_to_plot], color='cornflowerblue', linewidth=lw, - label='Lasso') -plt.plot(coef_multi_task_lasso_[:, feature_to_plot], color='gold', linewidth=lw, - label='MultiTaskLasso') -plt.legend(loc='upper center') -plt.axis('tight') +plt.plot(coef[:, feature_to_plot], color="seagreen", linewidth=lw, label="Ground truth") +plt.plot( + coef_lasso_[:, feature_to_plot], color="cornflowerblue", linewidth=lw, label="Lasso" +) +plt.plot( + coef_multi_task_lasso_[:, feature_to_plot], + color="gold", + linewidth=lw, + label="MultiTaskLasso", +) +plt.legend(loc="upper center") +plt.axis("tight") plt.ylim([-1.1, 1.1]) plt.show() diff --git a/examples/linear_model/plot_nnls.py b/examples/linear_model/plot_nnls.py index 56f357c4214a6..9ab19e19a1882 100644 --- a/examples/linear_model/plot_nnls.py +++ b/examples/linear_model/plot_nnls.py @@ -6,10 +6,15 @@ In this example, we fit a linear model with positive constraints on the regression coefficients and compare the estimated coefficients to a classic linear regression. + """ -print(__doc__) -import numpy as np + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt +import numpy as np + from sklearn.metrics import r2_score # %% @@ -24,7 +29,7 @@ y = np.dot(X, true_coef) # Add some noise -y += 5 * np.random.normal(size=(n_samples, )) +y += 5 * np.random.normal(size=(n_samples,)) # %% # Split the data in train set and test set @@ -62,6 +67,6 @@ low_y, high_y = ax.get_ylim() low = max(low_x, low_y) high = min(high_x, high_y) -ax.plot([low, high], [low, high], ls="--", c=".3", alpha=.5) +ax.plot([low, high], [low, high], ls="--", c=".3", alpha=0.5) ax.set_xlabel("OLS regression coefficients", fontweight="bold") ax.set_ylabel("NNLS regression coefficients", fontweight="bold") diff --git a/examples/linear_model/plot_ols.py b/examples/linear_model/plot_ols.py deleted file mode 100644 index 5f8c0079c3582..0000000000000 --- a/examples/linear_model/plot_ols.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -""" -========================================================= -Linear Regression Example -========================================================= -The example below uses only the first feature of the `diabetes` dataset, -in order to illustrate the data points within the two-dimensional plot. -The straight line can be seen in the plot, showing how linear regression -attempts to draw a straight line that will best minimize the -residual sum of squares between the observed responses in the dataset, -and the responses predicted by the linear approximation. - -The coefficients, residual sum of squares and the coefficient of -determination are also calculated. -""" -print(__doc__) - - -# Code source: Jaques Grobler -# License: BSD 3 clause - - -import matplotlib.pyplot as plt -import numpy as np -from sklearn import datasets, linear_model -from sklearn.metrics import mean_squared_error, r2_score - -# Load the diabetes dataset -diabetes_X, diabetes_y = datasets.load_diabetes(return_X_y=True) - -# Use only one feature -diabetes_X = diabetes_X[:, np.newaxis, 2] - -# Split the data into training/testing sets -diabetes_X_train = diabetes_X[:-20] -diabetes_X_test = diabetes_X[-20:] - -# Split the targets into training/testing sets -diabetes_y_train = diabetes_y[:-20] -diabetes_y_test = diabetes_y[-20:] - -# Create linear regression object -regr = linear_model.LinearRegression() - -# Train the model using the training sets -regr.fit(diabetes_X_train, diabetes_y_train) - -# Make predictions using the testing set -diabetes_y_pred = regr.predict(diabetes_X_test) - -# The coefficients -print('Coefficients: \n', regr.coef_) -# The mean squared error -print('Mean squared error: %.2f' - % mean_squared_error(diabetes_y_test, diabetes_y_pred)) -# The coefficient of determination: 1 is perfect prediction -print('Coefficient of determination: %.2f' - % r2_score(diabetes_y_test, diabetes_y_pred)) - -# Plot outputs -plt.scatter(diabetes_X_test, diabetes_y_test, color='black') -plt.plot(diabetes_X_test, diabetes_y_pred, color='blue', linewidth=3) - -plt.xticks(()) -plt.yticks(()) - -plt.show() diff --git a/examples/linear_model/plot_ols_3d.py b/examples/linear_model/plot_ols_3d.py deleted file mode 100644 index aa34b3b680202..0000000000000 --- a/examples/linear_model/plot_ols_3d.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -""" -========================================================= -Sparsity Example: Fitting only features 1 and 2 -========================================================= - -Features 1 and 2 of the diabetes-dataset are fitted and -plotted below. It illustrates that although feature 2 -has a strong coefficient on the full model, it does not -give us much regarding `y` when compared to just feature 1 - -""" -print(__doc__) - - -# Code source: GaÃĢl Varoquaux -# Modified for documentation by Jaques Grobler -# License: BSD 3 clause - -import matplotlib.pyplot as plt -import numpy as np -from mpl_toolkits.mplot3d import Axes3D - -from sklearn import datasets, linear_model - -X, y = datasets.load_diabetes(return_X_y=True) -indices = (0, 1) - -X_train = X[:-20, indices] -X_test = X[-20:, indices] -y_train = y[:-20] -y_test = y[-20:] - -ols = linear_model.LinearRegression() -ols.fit(X_train, y_train) - - -# ############################################################################# -# Plot the figure -def plot_figs(fig_num, elev, azim, X_train, clf): - fig = plt.figure(fig_num, figsize=(4, 3)) - plt.clf() - ax = Axes3D(fig, elev=elev, azim=azim) - - ax.scatter(X_train[:, 0], X_train[:, 1], y_train, c='k', marker='+') - ax.plot_surface(np.array([[-.1, -.1], [.15, .15]]), - np.array([[-.1, .15], [-.1, .15]]), - clf.predict(np.array([[-.1, -.1, .15, .15], - [-.1, .15, -.1, .15]]).T - ).reshape((2, 2)), - alpha=.5) - ax.set_xlabel('X_1') - ax.set_ylabel('X_2') - ax.set_zlabel('Y') - ax.w_xaxis.set_ticklabels([]) - ax.w_yaxis.set_ticklabels([]) - ax.w_zaxis.set_ticklabels([]) - - -# Generate the three different figures from different views -elev = 43.5 -azim = -110 -plot_figs(1, elev, azim, X_train, ols) - -elev = -.5 -azim = 0 -plot_figs(2, elev, azim, X_train, ols) - -elev = -.5 -azim = 90 -plot_figs(3, elev, azim, X_train, ols) - -plt.show() diff --git a/examples/linear_model/plot_ols_ridge.py b/examples/linear_model/plot_ols_ridge.py new file mode 100644 index 0000000000000..d94d767de1736 --- /dev/null +++ b/examples/linear_model/plot_ols_ridge.py @@ -0,0 +1,167 @@ +""" +=========================================== +Ordinary Least Squares and Ridge Regression +=========================================== + +1. Ordinary Least Squares: + We illustrate how to use the ordinary least squares (OLS) model, + :class:`~sklearn.linear_model.LinearRegression`, on a single feature of + the diabetes dataset. We train on a subset of the data, evaluate on a + test set, and visualize the predictions. + +2. Ordinary Least Squares and Ridge Regression Variance: + We then show how OLS can have high variance when the data is sparse or + noisy, by fitting on a very small synthetic sample repeatedly. Ridge + regression, :class:`~sklearn.linear_model.Ridge`, reduces this variance + by penalizing (shrinking) the coefficients, leading to more stable + predictions. + +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Data Loading and Preparation +# ---------------------------- +# +# Load the diabetes dataset. For simplicity, we only keep a single feature in the data. +# Then, we split the data and target into training and test sets. +from sklearn.datasets import load_diabetes +from sklearn.model_selection import train_test_split + +X, y = load_diabetes(return_X_y=True) +X = X[:, [2]] # Use only one feature +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=20, shuffle=False) + +# %% +# Linear regression model +# ----------------------- +# +# We create a linear regression model and fit it on the training data. Note that by +# default, an intercept is added to the model. We can control this behavior by setting +# the `fit_intercept` parameter. +from sklearn.linear_model import LinearRegression + +regressor = LinearRegression().fit(X_train, y_train) + +# %% +# Model evaluation +# ---------------- +# +# We evaluate the model's performance on the test set using the mean squared error +# and the coefficient of determination. +from sklearn.metrics import mean_squared_error, r2_score + +y_pred = regressor.predict(X_test) + +print(f"Mean squared error: {mean_squared_error(y_test, y_pred):.2f}") +print(f"Coefficient of determination: {r2_score(y_test, y_pred):.2f}") + +# %% +# Plotting the results +# -------------------- +# +# Finally, we visualize the results on the train and test data. +import matplotlib.pyplot as plt + +fig, ax = plt.subplots(ncols=2, figsize=(10, 5), sharex=True, sharey=True) + +ax[0].scatter(X_train, y_train, label="Train data points") +ax[0].plot( + X_train, + regressor.predict(X_train), + linewidth=3, + color="tab:orange", + label="Model predictions", +) +ax[0].set(xlabel="Feature", ylabel="Target", title="Train set") +ax[0].legend() + +ax[1].scatter(X_test, y_test, label="Test data points") +ax[1].plot(X_test, y_pred, linewidth=3, color="tab:orange", label="Model predictions") +ax[1].set(xlabel="Feature", ylabel="Target", title="Test set") +ax[1].legend() + +fig.suptitle("Linear Regression") + +plt.show() + +# %% +# +# OLS on this single-feature subset learns a linear function that minimizes +# the mean squared error on the training data. We can see how well (or poorly) +# it generalizes by looking at the R^2 score and mean squared error on the +# test set. In higher dimensions, pure OLS often overfits, especially if the +# data is noisy. Regularization techniques (like Ridge or Lasso) can help +# reduce that. + +# %% +# Ordinary Least Squares and Ridge Regression Variance +# ---------------------------------------------------------- +# +# Next, we illustrate the problem of high variance more clearly by using +# a tiny synthetic dataset. We sample only two data points, then repeatedly +# add small Gaussian noise to them and refit both OLS and Ridge. We plot +# each new line to see how much OLS can jump around, whereas Ridge remains +# more stable thanks to its penalty term. + + +import matplotlib.pyplot as plt +import numpy as np + +from sklearn import linear_model + +X_train = np.c_[0.5, 1].T +y_train = [0.5, 1] +X_test = np.c_[0, 2].T + +np.random.seed(0) + +classifiers = dict( + ols=linear_model.LinearRegression(), ridge=linear_model.Ridge(alpha=0.1) +) + +for name, clf in classifiers.items(): + fig, ax = plt.subplots(figsize=(4, 3)) + + for _ in range(6): + this_X = 0.1 * np.random.normal(size=(2, 1)) + X_train + clf.fit(this_X, y_train) + + ax.plot(X_test, clf.predict(X_test), color="gray") + ax.scatter(this_X, y_train, s=3, c="gray", marker="o", zorder=10) + + clf.fit(X_train, y_train) + ax.plot(X_test, clf.predict(X_test), linewidth=2, color="blue") + ax.scatter(X_train, y_train, s=30, c="red", marker="+", zorder=10) + + ax.set_title(name) + ax.set_xlim(0, 2) + ax.set_ylim((0, 1.6)) + ax.set_xlabel("X") + ax.set_ylabel("y") + + fig.tight_layout() + +plt.show() + + +# %% +# Conclusion +# ---------- +# +# - In the first example, we applied OLS to a real dataset, showing +# how a plain linear model can fit the data by minimizing the squared error +# on the training set. +# +# - In the second example, OLS lines varied drastically each time noise +# was added, reflecting its high variance when data is sparse or noisy. By +# contrast, **Ridge** regression introduces a regularization term that shrinks +# the coefficients, stabilizing predictions. +# +# Techniques like :class:`~sklearn.linear_model.Ridge` or +# :class:`~sklearn.linear_model.Lasso` (which applies an L1 penalty) are both +# common ways to improve generalization and reduce overfitting. A well-tuned +# Ridge or Lasso often outperforms pure OLS when features are correlated, data +# is noisy, or sample size is small. diff --git a/examples/linear_model/plot_ols_ridge_variance.py b/examples/linear_model/plot_ols_ridge_variance.py deleted file mode 100644 index ba5f65575f927..0000000000000 --- a/examples/linear_model/plot_ols_ridge_variance.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -""" -========================================================= -Ordinary Least Squares and Ridge Regression Variance -========================================================= -Due to the few points in each dimension and the straight -line that linear regression uses to follow these points -as well as it can, noise on the observations will cause -great variance as shown in the first plot. Every line's slope -can vary quite a bit for each prediction due to the noise -induced in the observations. - -Ridge regression is basically minimizing a penalised version -of the least-squared function. The penalising `shrinks` the -value of the regression coefficients. -Despite the few data points in each dimension, the slope -of the prediction is much more stable and the variance -in the line itself is greatly reduced, in comparison to that -of the standard linear regression -""" -print(__doc__) - - -# Code source: GaÃĢl Varoquaux -# Modified for documentation by Jaques Grobler -# License: BSD 3 clause - - -import numpy as np -import matplotlib.pyplot as plt - -from sklearn import linear_model - -X_train = np.c_[.5, 1].T -y_train = [.5, 1] -X_test = np.c_[0, 2].T - -np.random.seed(0) - -classifiers = dict(ols=linear_model.LinearRegression(), - ridge=linear_model.Ridge(alpha=.1)) - -for name, clf in classifiers.items(): - fig, ax = plt.subplots(figsize=(4, 3)) - - for _ in range(6): - this_X = .1 * np.random.normal(size=(2, 1)) + X_train - clf.fit(this_X, y_train) - - ax.plot(X_test, clf.predict(X_test), color='gray') - ax.scatter(this_X, y_train, s=3, c='gray', marker='o', zorder=10) - - clf.fit(X_train, y_train) - ax.plot(X_test, clf.predict(X_test), linewidth=2, color='blue') - ax.scatter(X_train, y_train, s=30, c='red', marker='+', zorder=10) - - ax.set_title(name) - ax.set_xlim(0, 2) - ax.set_ylim((0, 1.6)) - ax.set_xlabel('X') - ax.set_ylabel('y') - - fig.tight_layout() - -plt.show() diff --git a/examples/linear_model/plot_omp.py b/examples/linear_model/plot_omp.py index 731a76a76e13c..815b3c9425fdf 100644 --- a/examples/linear_model/plot_omp.py +++ b/examples/linear_model/plot_omp.py @@ -5,14 +5,17 @@ Using orthogonal matching pursuit for recovering a sparse signal from a noisy measurement encoded with a dictionary + """ -print(__doc__) + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import matplotlib.pyplot as plt import numpy as np -from sklearn.linear_model import OrthogonalMatchingPursuit -from sklearn.linear_model import OrthogonalMatchingPursuitCV + from sklearn.datasets import make_sparse_coded_signal +from sklearn.linear_model import OrthogonalMatchingPursuit, OrthogonalMatchingPursuitCV n_components, n_features = 512, 100 n_nonzero_coefs = 17 @@ -22,13 +25,16 @@ # y = Xw # |x|_0 = n_nonzero_coefs -y, X, w = make_sparse_coded_signal(n_samples=1, - n_components=n_components, - n_features=n_features, - n_nonzero_coefs=n_nonzero_coefs, - random_state=0) +y, X, w = make_sparse_coded_signal( + n_samples=1, + n_components=n_components, + n_features=n_features, + n_nonzero_coefs=n_nonzero_coefs, + random_state=0, +) +X = X.T -idx, = w.nonzero() +(idx,) = w.nonzero() # distort the clean signal y_noisy = y + 0.05 * np.random.randn(len(y)) @@ -38,38 +44,37 @@ plt.subplot(4, 1, 1) plt.xlim(0, 512) plt.title("Sparse signal") -plt.stem(idx, w[idx], use_line_collection=True) +plt.stem(idx, w[idx]) # plot the noise-free reconstruction omp = OrthogonalMatchingPursuit(n_nonzero_coefs=n_nonzero_coefs) omp.fit(X, y) coef = omp.coef_ -idx_r, = coef.nonzero() +(idx_r,) = coef.nonzero() plt.subplot(4, 1, 2) plt.xlim(0, 512) plt.title("Recovered signal from noise-free measurements") -plt.stem(idx_r, coef[idx_r], use_line_collection=True) +plt.stem(idx_r, coef[idx_r]) # plot the noisy reconstruction omp.fit(X, y_noisy) coef = omp.coef_ -idx_r, = coef.nonzero() +(idx_r,) = coef.nonzero() plt.subplot(4, 1, 3) plt.xlim(0, 512) plt.title("Recovered signal from noisy measurements") -plt.stem(idx_r, coef[idx_r], use_line_collection=True) +plt.stem(idx_r, coef[idx_r]) # plot the noisy reconstruction with number of non-zeros set by CV omp_cv = OrthogonalMatchingPursuitCV() omp_cv.fit(X, y_noisy) coef = omp_cv.coef_ -idx_r, = coef.nonzero() +(idx_r,) = coef.nonzero() plt.subplot(4, 1, 4) plt.xlim(0, 512) plt.title("Recovered signal from noisy measurements with CV") -plt.stem(idx_r, coef[idx_r], use_line_collection=True) +plt.stem(idx_r, coef[idx_r]) plt.subplots_adjust(0.06, 0.04, 0.94, 0.90, 0.20, 0.38) -plt.suptitle('Sparse signal recovery with Orthogonal Matching Pursuit', - fontsize=16) +plt.suptitle("Sparse signal recovery with Orthogonal Matching Pursuit", fontsize=16) plt.show() diff --git a/examples/linear_model/plot_poisson_regression_non_normal_loss.py b/examples/linear_model/plot_poisson_regression_non_normal_loss.py index 9541be1f62b24..a1f7a699b71c9 100644 --- a/examples/linear_model/plot_poisson_regression_non_normal_loss.py +++ b/examples/linear_model/plot_poisson_regression_non_normal_loss.py @@ -1,3 +1,6 @@ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + """ ====================================== Poisson regression and non-normal loss @@ -32,29 +35,22 @@ .. [1] A. Noll, R. Salzmann and M.V. Wuthrich, Case Study: French Motor Third-Party Liability Claims (November 8, 2018). `doi:10.2139/ssrn.3164764 - `_ + `_ """ -print(__doc__) -# Authors: Christian Lorentzen -# Roman Yurchak -# Olivier Grisel -# License: BSD 3 clause -import numpy as np + import matplotlib.pyplot as plt +import numpy as np import pandas as pd - ############################################################################## # The French Motor Third-Party Liability Claims dataset # ----------------------------------------------------- # # Let's load the motor claim dataset from OpenML: # https://www.openml.org/d/41214 - from sklearn.datasets import fetch_openml - df = fetch_openml(data_id=41214, as_frame=True).frame df @@ -70,12 +66,15 @@ df["Frequency"] = df["ClaimNb"] / df["Exposure"] -print("Average Frequency = {}" - .format(np.average(df["Frequency"], weights=df["Exposure"]))) +print( + "Average Frequency = {}".format(np.average(df["Frequency"], weights=df["Exposure"])) +) -print("Fraction of exposure with zero claims = {0:.1%}" - .format(df.loc[df["ClaimNb"] == 0, "Exposure"].sum() / - df["Exposure"].sum())) +print( + "Fraction of exposure with zero claims = {0:.1%}".format( + df.loc[df["ClaimNb"] == 0, "Exposure"].sum() / df["Exposure"].sum() + ) +) fig, (ax0, ax1, ax2) = plt.subplots(ncols=3, figsize=(16, 4)) ax0.set_title("Number of claims") @@ -93,27 +92,35 @@ # In order to fit linear models with those predictors it is therefore # necessary to perform standard feature transformations as follows: -from sklearn.pipeline import make_pipeline -from sklearn.preprocessing import FunctionTransformer, OneHotEncoder -from sklearn.preprocessing import StandardScaler, KBinsDiscretizer from sklearn.compose import ColumnTransformer - +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import ( + FunctionTransformer, + KBinsDiscretizer, + OneHotEncoder, + StandardScaler, +) log_scale_transformer = make_pipeline( - FunctionTransformer(np.log, validate=False), - StandardScaler() + FunctionTransformer(np.log, validate=False), StandardScaler() ) linear_model_preprocessor = ColumnTransformer( [ - ("passthrough_numeric", "passthrough", - ["BonusMalus"]), - ("binned_numeric", KBinsDiscretizer(n_bins=10), - ["VehAge", "DrivAge"]), - ("log_scaled_numeric", log_scale_transformer, - ["Density"]), - ("onehot_categorical", OneHotEncoder(), - ["VehBrand", "VehPower", "VehGas", "Region", "Area"]), + ("passthrough_numeric", "passthrough", ["BonusMalus"]), + ( + "binned_numeric", + KBinsDiscretizer( + n_bins=10, quantile_method="averaged_inverted_cdf", random_state=0 + ), + ["VehAge", "DrivAge"], + ), + ("log_scaled_numeric", log_scale_transformer, ["Density"]), + ( + "onehot_categorical", + OneHotEncoder(), + ["VehBrand", "VehPower", "VehGas", "Region", "Area"], + ), ], remainder="drop", ) @@ -132,51 +139,66 @@ # the training sample. from sklearn.dummy import DummyRegressor -from sklearn.pipeline import Pipeline from sklearn.model_selection import train_test_split +from sklearn.pipeline import Pipeline df_train, df_test = train_test_split(df, test_size=0.33, random_state=0) -dummy = Pipeline([ - ("preprocessor", linear_model_preprocessor), - ("regressor", DummyRegressor(strategy='mean')), -]).fit(df_train, df_train["Frequency"], - regressor__sample_weight=df_train["Exposure"]) +dummy = Pipeline( + [ + ("preprocessor", linear_model_preprocessor), + ("regressor", DummyRegressor(strategy="mean")), + ] +).fit(df_train, df_train["Frequency"], regressor__sample_weight=df_train["Exposure"]) ############################################################################## # Let's compute the performance of this constant prediction baseline with 3 # different regression metrics: -from sklearn.metrics import mean_squared_error -from sklearn.metrics import mean_absolute_error -from sklearn.metrics import mean_poisson_deviance +from sklearn.metrics import ( + mean_absolute_error, + mean_poisson_deviance, + mean_squared_error, +) def score_estimator(estimator, df_test): """Score an estimator on the test set.""" y_pred = estimator.predict(df_test) - print("MSE: %.3f" % - mean_squared_error(df_test["Frequency"], y_pred, - sample_weight=df_test["Exposure"])) - print("MAE: %.3f" % - mean_absolute_error(df_test["Frequency"], y_pred, - sample_weight=df_test["Exposure"])) + print( + "MSE: %.3f" + % mean_squared_error( + df_test["Frequency"], y_pred, sample_weight=df_test["Exposure"] + ) + ) + print( + "MAE: %.3f" + % mean_absolute_error( + df_test["Frequency"], y_pred, sample_weight=df_test["Exposure"] + ) + ) # Ignore non-positive predictions, as they are invalid for # the Poisson deviance. mask = y_pred > 0 if (~mask).any(): n_masked, n_samples = (~mask).sum(), mask.shape[0] - print(f"WARNING: Estimator yields invalid, non-positive predictions " - f" for {n_masked} samples out of {n_samples}. These predictions " - f"are ignored when computing the Poisson deviance.") + print( + "WARNING: Estimator yields invalid, non-positive predictions " + f" for {n_masked} samples out of {n_samples}. These predictions " + "are ignored when computing the Poisson deviance." + ) - print("mean Poisson deviance: %.3f" % - mean_poisson_deviance(df_test["Frequency"][mask], - y_pred[mask], - sample_weight=df_test["Exposure"][mask])) + print( + "mean Poisson deviance: %.3f" + % mean_poisson_deviance( + df_test["Frequency"][mask], + y_pred[mask], + sample_weight=df_test["Exposure"][mask], + ) + ) print("Constant mean frequency evaluation:") @@ -187,18 +209,18 @@ def score_estimator(estimator, df_test): # --------------------------- # # We start by modeling the target variable with the (l2 penalized) least -# squares linear regression model, more comonly known as Ridge regression. We +# squares linear regression model, more commonly known as Ridge regression. We # use a low penalization `alpha`, as we expect such a linear model to under-fit # on such a large dataset. from sklearn.linear_model import Ridge - -ridge_glm = Pipeline([ - ("preprocessor", linear_model_preprocessor), - ("regressor", Ridge(alpha=1e-6)), -]).fit(df_train, df_train["Frequency"], - regressor__sample_weight=df_train["Exposure"]) +ridge_glm = Pipeline( + [ + ("preprocessor", linear_model_preprocessor), + ("regressor", Ridge(alpha=1e-6)), + ] +).fit(df_train, df_train["Frequency"], regressor__sample_weight=df_train["Exposure"]) # %% # The Poisson deviance cannot be computed on non-positive values predicted by @@ -227,12 +249,15 @@ def score_estimator(estimator, df_test): n_samples = df_train.shape[0] -poisson_glm = Pipeline([ - ("preprocessor", linear_model_preprocessor), - ("regressor", PoissonRegressor(alpha=1e-12, max_iter=300)) -]) -poisson_glm.fit(df_train, df_train["Frequency"], - regressor__sample_weight=df_train["Exposure"]) +poisson_glm = Pipeline( + [ + ("preprocessor", linear_model_preprocessor), + ("regressor", PoissonRegressor(alpha=1e-12, solver="newton-cholesky")), + ] +) +poisson_glm.fit( + df_train, df_train["Frequency"], regressor__sample_weight=df_train["Exposure"] +) print("PoissonRegressor evaluation:") score_estimator(poisson_glm, df_test) @@ -258,27 +283,32 @@ def score_estimator(estimator, df_test): # least-squares loss. Here we only fit trees with the Poisson loss to keep this # example concise. -from sklearn.experimental import enable_hist_gradient_boosting # noqa from sklearn.ensemble import HistGradientBoostingRegressor from sklearn.preprocessing import OrdinalEncoder - tree_preprocessor = ColumnTransformer( [ - ("categorical", OrdinalEncoder(), - ["VehBrand", "VehPower", "VehGas", "Region", "Area"]), - ("numeric", "passthrough", - ["VehAge", "DrivAge", "BonusMalus", "Density"]), + ( + "categorical", + OrdinalEncoder(), + ["VehBrand", "VehPower", "VehGas", "Region", "Area"], + ), + ("numeric", "passthrough", ["VehAge", "DrivAge", "BonusMalus", "Density"]), ], remainder="drop", ) -poisson_gbrt = Pipeline([ - ("preprocessor", tree_preprocessor), - ("regressor", HistGradientBoostingRegressor(loss="poisson", - max_leaf_nodes=128)), -]) -poisson_gbrt.fit(df_train, df_train["Frequency"], - regressor__sample_weight=df_train["Exposure"]) +poisson_gbrt = Pipeline( + [ + ("preprocessor", tree_preprocessor), + ( + "regressor", + HistGradientBoostingRegressor(loss="poisson", max_leaf_nodes=128), + ), + ] +) +poisson_gbrt.fit( + df_train, df_train["Frequency"], regressor__sample_weight=df_train["Exposure"] +) print("Poisson Gradient Boosted Trees evaluation:") score_estimator(poisson_gbrt, df_test) @@ -299,14 +329,11 @@ def score_estimator(estimator, df_test): fig, axes = plt.subplots(nrows=2, ncols=4, figsize=(16, 6), sharey=True) fig.subplots_adjust(bottom=0.2) n_bins = 20 -for row_idx, label, df in zip(range(2), - ["train", "test"], - [df_train, df_test]): - df["Frequency"].hist(bins=np.linspace(-1, 30, n_bins), - ax=axes[row_idx, 0]) +for row_idx, label, df in zip(range(2), ["train", "test"], [df_train, df_test]): + df["Frequency"].hist(bins=np.linspace(-1, 30, n_bins), ax=axes[row_idx, 0]) axes[row_idx, 0].set_title("Data") - axes[row_idx, 0].set_yscale('log') + axes[row_idx, 0].set_yscale("log") axes[row_idx, 0].set_xlabel("y (observed Frequency)") axes[row_idx, 0].set_ylim([1e1, 5e5]) axes[row_idx, 0].set_ylabel(label + " samples") @@ -314,12 +341,13 @@ def score_estimator(estimator, df_test): for idx, model in enumerate([ridge_glm, poisson_glm, poisson_gbrt]): y_pred = model.predict(df) - pd.Series(y_pred).hist(bins=np.linspace(-1, 4, n_bins), - ax=axes[row_idx, idx+1]) + pd.Series(y_pred).hist( + bins=np.linspace(-1, 4, n_bins), ax=axes[row_idx, idx + 1] + ) axes[row_idx, idx + 1].set( title=model[-1].__class__.__name__, - yscale='log', - xlabel="y_pred (predicted expected Frequency)" + yscale="log", + xlabel="y_pred (predicted expected Frequency)", ) plt.tight_layout() @@ -362,8 +390,7 @@ def score_estimator(estimator, df_test): from sklearn.utils import gen_even_slices -def _mean_frequency_by_risk_group(y_true, y_pred, sample_weight=None, - n_bins=100): +def _mean_frequency_by_risk_group(y_true, y_pred, sample_weight=None, n_bins=100): """Compare predictions and observations for bins ordered by y_pred. We order the samples by ``y_pred`` and split it in bins. @@ -390,19 +417,14 @@ def _mean_frequency_by_risk_group(y_true, y_pred, sample_weight=None, average y_pred for each bin """ idx_sort = np.argsort(y_pred) - bin_centers = np.arange(0, 1, 1/n_bins) + 0.5/n_bins + bin_centers = np.arange(0, 1, 1 / n_bins) + 0.5 / n_bins y_pred_bin = np.zeros(n_bins) y_true_bin = np.zeros(n_bins) for n, sl in enumerate(gen_even_slices(len(y_true), n_bins)): weights = sample_weight[idx_sort][sl] - y_pred_bin[n] = np.average( - y_pred[idx_sort][sl], weights=weights - ) - y_true_bin[n] = np.average( - y_true[idx_sort][sl], - weights=weights - ) + y_pred_bin[n] = np.average(y_pred[idx_sort][sl], weights=weights) + y_true_bin[n] = np.average(y_true[idx_sort][sl], weights=weights) return bin_centers, y_true_bin, y_pred_bin @@ -410,27 +432,26 @@ def _mean_frequency_by_risk_group(y_true, y_pred, sample_weight=None, fig, ax = plt.subplots(nrows=2, ncols=2, figsize=(12, 8)) plt.subplots_adjust(wspace=0.3) -for axi, model in zip(ax.ravel(), [ridge_glm, poisson_glm, poisson_gbrt, - dummy]): +for axi, model in zip(ax.ravel(), [ridge_glm, poisson_glm, poisson_gbrt, dummy]): y_pred = model.predict(df_test) y_true = df_test["Frequency"].values exposure = df_test["Exposure"].values q, y_true_seg, y_pred_seg = _mean_frequency_by_risk_group( - y_true, y_pred, sample_weight=exposure, n_bins=10) + y_true, y_pred, sample_weight=exposure, n_bins=10 + ) # Name of the model after the estimator used in the last step of the # pipeline. - print(f"Predicted number of claims by {model[-1]}: " - f"{np.sum(y_pred * exposure):.1f}") + print(f"Predicted number of claims by {model[-1]}: {np.sum(y_pred * exposure):.1f}") - axi.plot(q, y_pred_seg, marker='x', linestyle="--", label="predictions") - axi.plot(q, y_true_seg, marker='o', linestyle="--", label="observations") + axi.plot(q, y_pred_seg, marker="x", linestyle="--", label="predictions") + axi.plot(q, y_true_seg, marker="o", linestyle="--", label="observations") axi.set_xlim(0, 1.0) axi.set_ylim(0, 0.5) axi.set( title=model[-1], - xlabel='Fraction of samples sorted by y_pred', - ylabel='Mean Frequency (y_pred)' + xlabel="Fraction of samples sorted by y_pred", + ylabel="Mean Frequency (y_pred)", ) axi.legend() plt.tight_layout() @@ -441,7 +462,7 @@ def _mean_frequency_by_risk_group(y_true, y_pred, sample_weight=None, # well calibrated (to estimate the mean frequency of the entire population). # # The ``Ridge`` regression model can predict very low expected frequencies that -# do not match the data. It can therefore severly under-estimate the risk for +# do not match the data. It can therefore severely under-estimate the risk for # some policyholders. # # ``PoissonRegressor`` and ``HistGradientBoostingRegressor`` show better @@ -490,27 +511,27 @@ def lorenz_curve(y_true, y_pred, exposure): for model in [dummy, ridge_glm, poisson_glm, poisson_gbrt]: y_pred = model.predict(df_test) - cum_exposure, cum_claims = lorenz_curve(df_test["Frequency"], y_pred, - df_test["Exposure"]) + cum_exposure, cum_claims = lorenz_curve( + df_test["Frequency"], y_pred, df_test["Exposure"] + ) gini = 1 - 2 * auc(cum_exposure, cum_claims) label = "{} (Gini: {:.2f})".format(model[-1], gini) ax.plot(cum_exposure, cum_claims, linestyle="-", label=label) # Oracle model: y_pred == y_test -cum_exposure, cum_claims = lorenz_curve(df_test["Frequency"], - df_test["Frequency"], - df_test["Exposure"]) +cum_exposure, cum_claims = lorenz_curve( + df_test["Frequency"], df_test["Frequency"], df_test["Exposure"] +) gini = 1 - 2 * auc(cum_exposure, cum_claims) label = "Oracle (Gini: {:.2f})".format(gini) ax.plot(cum_exposure, cum_claims, linestyle="-.", color="gray", label=label) # Random Baseline -ax.plot([0, 1], [0, 1], linestyle="--", color="black", - label="Random baseline") +ax.plot([0, 1], [0, 1], linestyle="--", color="black", label="Random baseline") ax.set( title="Lorenz curves by model", - xlabel='Cumulative proportion of exposure (from safest to riskiest)', - ylabel='Cumulative proportion of claims' + xlabel="Cumulative proportion of exposure (from safest to riskiest)", + ylabel="Cumulative proportion of claims", ) ax.legend(loc="upper left") diff --git a/examples/linear_model/plot_polynomial_interpolation.py b/examples/linear_model/plot_polynomial_interpolation.py index cfa684ffd79ca..e14fd61ee3d75 100644 --- a/examples/linear_model/plot_polynomial_interpolation.py +++ b/examples/linear_model/plot_polynomial_interpolation.py @@ -33,26 +33,24 @@ non-linear effects with a linear model, using a pipeline to add non-linear features. Kernel methods extend this idea and can induce very high (even infinite) dimensional feature spaces. + """ -print(__doc__) -# Author: Mathieu Blondel -# Jake Vanderplas -# Christian Lorentzen -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt +import numpy as np from sklearn.linear_model import Ridge -from sklearn.preprocessing import PolynomialFeatures, SplineTransformer from sklearn.pipeline import make_pipeline - +from sklearn.preprocessing import PolynomialFeatures, SplineTransformer # %% -# We start by defining a function that we intent to approximate and prepare +# We start by defining a function that we intend to approximate and prepare # plotting it. + def f(x): """Function to be approximated by polynomial interpolation.""" return x * np.sin(x) @@ -80,9 +78,9 @@ def f(x): # plot function lw = 2 fig, ax = plt.subplots() -ax.set_prop_cycle(color=[ - "black", "teal", "yellowgreen", "gold", "darkorange", "tomato" -]) +ax.set_prop_cycle( + color=["black", "teal", "yellowgreen", "gold", "darkorange", "tomato"] +) ax.plot(x_plot, f(x_plot), linewidth=lw, label="ground truth") # plot training points @@ -96,13 +94,12 @@ def f(x): ax.plot(x_plot, y_plot, label=f"degree {degree}") # B-spline with 4 + 3 - 1 = 6 basis functions -model = make_pipeline(SplineTransformer(n_knots=4, degree=3), - Ridge(alpha=1e-3)) +model = make_pipeline(SplineTransformer(n_knots=4, degree=3), Ridge(alpha=1e-3)) model.fit(X_train, y_train) y_plot = model.predict(X_plot) ax.plot(x_plot, y_plot, label="B-spline") -ax.legend(loc='lower center') +ax.legend(loc="lower center") ax.set_ylim(-20, 10) plt.show() @@ -127,17 +124,17 @@ def f(x): splt = SplineTransformer(n_knots=4, degree=3).fit(X_train) axes[1].plot(x_plot, splt.transform(X_plot)) -axes[1].legend(axes[1].lines, [f"spline {n}" for n in range(4)]) +axes[1].legend(axes[1].lines, [f"spline {n}" for n in range(6)]) axes[1].set_title("SplineTransformer") # plot knots of spline knots = splt.bsplines_[0].t -axes[1].vlines(knots[3:-3], ymin=0, ymax=0.8, linestyles='dashed') +axes[1].vlines(knots[3:-3], ymin=0, ymax=0.8, linestyles="dashed") plt.show() # %% # In the left plot, we recognize the lines corresponding to simple monomials -# from ``x**0`` to ``x**3``. In the right figure, we see the four B-spline +# from ``x**0`` to ``x**3``. In the right figure, we see the six B-spline # basis functions of ``degree=3`` and also the four knot positions that were # chosen during ``fit``. Note that there are ``degree`` number of additional # knots each to the left and to the right of the fitted interval. These are @@ -145,3 +142,71 @@ def f(x): # function has local support and is continued as a constant beyond the fitted # range. This extrapolating behaviour could be changed by the argument # ``extrapolation``. + +# %% +# Periodic Splines +# ---------------- +# In the previous example we saw the limitations of polynomials and splines for +# extrapolation beyond the range of the training observations. In some +# settings, e.g. with seasonal effects, we expect a periodic continuation of +# the underlying signal. Such effects can be modelled using periodic splines, +# which have equal function value and equal derivatives at the first and last +# knot. In the following case we show how periodic splines provide a better fit +# both within and outside of the range of training data given the additional +# information of periodicity. The splines period is the distance between +# the first and last knot, which we specify manually. +# +# Periodic splines can also be useful for naturally periodic features (such as +# day of the year), as the smoothness at the boundary knots prevents a jump in +# the transformed values (e.g. from Dec 31st to Jan 1st). For such naturally +# periodic features or more generally features where the period is known, it is +# advised to explicitly pass this information to the `SplineTransformer` by +# setting the knots manually. + + +# %% +def g(x): + """Function to be approximated by periodic spline interpolation.""" + return np.sin(x) - 0.7 * np.cos(x * 3) + + +y_train = g(x_train) + +# Extend the test data into the future: +x_plot_ext = np.linspace(-1, 21, 200) +X_plot_ext = x_plot_ext[:, np.newaxis] + +lw = 2 +fig, ax = plt.subplots() +ax.set_prop_cycle(color=["black", "tomato", "teal"]) +ax.plot(x_plot_ext, g(x_plot_ext), linewidth=lw, label="ground truth") +ax.scatter(x_train, y_train, label="training points") + +for transformer, label in [ + (SplineTransformer(degree=3, n_knots=10), "spline"), + ( + SplineTransformer( + degree=3, + knots=np.linspace(0, 2 * np.pi, 10)[:, None], + extrapolation="periodic", + ), + "periodic spline", + ), +]: + model = make_pipeline(transformer, Ridge(alpha=1e-3)) + model.fit(X_train, y_train) + y_plot_ext = model.predict(X_plot_ext) + ax.plot(x_plot_ext, y_plot_ext, label=label) + +ax.legend() +fig.show() + +# %% We again plot the underlying splines. +fig, ax = plt.subplots() +knots = np.linspace(0, 2 * np.pi, 4) +splt = SplineTransformer(knots=knots[:, None], degree=3, extrapolation="periodic").fit( + X_train +) +ax.plot(x_plot_ext, splt.transform(X_plot_ext)) +ax.legend(ax.lines, [f"spline {n}" for n in range(3)]) +plt.show() diff --git a/examples/linear_model/plot_quantile_regression.py b/examples/linear_model/plot_quantile_regression.py new file mode 100644 index 0000000000000..2cf1b5eabf5a5 --- /dev/null +++ b/examples/linear_model/plot_quantile_regression.py @@ -0,0 +1,314 @@ +""" +=================== +Quantile regression +=================== + +This example illustrates how quantile regression can predict non-trivial +conditional quantiles. + +The left figure shows the case when the error distribution is normal, +but has non-constant variance, i.e. with heteroscedasticity. + +The right figure shows an example of an asymmetric error distribution, +namely the Pareto distribution. + +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Dataset generation +# ------------------ +# +# To illustrate the behaviour of quantile regression, we will generate two +# synthetic datasets. The true generative random processes for both datasets +# will be composed by the same expected value with a linear relationship with a +# single feature `x`. +import numpy as np + +rng = np.random.RandomState(42) +x = np.linspace(start=0, stop=10, num=100) +X = x[:, np.newaxis] +y_true_mean = 10 + 0.5 * x + +# %% +# We will create two subsequent problems by changing the distribution of the +# target `y` while keeping the same expected value: +# +# - in the first case, a heteroscedastic Normal noise is added; +# - in the second case, an asymmetric Pareto noise is added. +y_normal = y_true_mean + rng.normal(loc=0, scale=0.5 + 0.5 * x, size=x.shape[0]) +a = 5 +y_pareto = y_true_mean + 10 * (rng.pareto(a, size=x.shape[0]) - 1 / (a - 1)) + +# %% +# Let's first visualize the datasets as well as the distribution of the +# residuals `y - mean(y)`. +import matplotlib.pyplot as plt + +_, axs = plt.subplots(nrows=2, ncols=2, figsize=(15, 11), sharex="row", sharey="row") + +axs[0, 0].plot(x, y_true_mean, label="True mean") +axs[0, 0].scatter(x, y_normal, color="black", alpha=0.5, label="Observations") +axs[1, 0].hist(y_true_mean - y_normal, edgecolor="black") + + +axs[0, 1].plot(x, y_true_mean, label="True mean") +axs[0, 1].scatter(x, y_pareto, color="black", alpha=0.5, label="Observations") +axs[1, 1].hist(y_true_mean - y_pareto, edgecolor="black") + +axs[0, 0].set_title("Dataset with heteroscedastic Normal distributed targets") +axs[0, 1].set_title("Dataset with asymmetric Pareto distributed target") +axs[1, 0].set_title( + "Residuals distribution for heteroscedastic Normal distributed targets" +) +axs[1, 1].set_title("Residuals distribution for asymmetric Pareto distributed target") +axs[0, 0].legend() +axs[0, 1].legend() +axs[0, 0].set_ylabel("y") +axs[1, 0].set_ylabel("Counts") +axs[0, 1].set_xlabel("x") +axs[0, 0].set_xlabel("x") +axs[1, 0].set_xlabel("Residuals") +_ = axs[1, 1].set_xlabel("Residuals") + +# %% +# With the heteroscedastic Normal distributed target, we observe that the +# variance of the noise is increasing when the value of the feature `x` is +# increasing. +# +# With the asymmetric Pareto distributed target, we observe that the positive +# residuals are bounded. +# +# These types of noisy targets make the estimation via +# :class:`~sklearn.linear_model.LinearRegression` less efficient, i.e. we need +# more data to get stable results and, in addition, large outliers can have a +# huge impact on the fitted coefficients. (Stated otherwise: in a setting with +# constant variance, ordinary least squares estimators converge much faster to +# the *true* coefficients with increasing sample size.) +# +# In this asymmetric setting, the median or different quantiles give additional +# insights. On top of that, median estimation is much more robust to outliers +# and heavy tailed distributions. But note that extreme quantiles are estimated +# by very few data points. 95% quantile are more or less estimated by the 5% +# largest values and thus also a bit sensitive outliers. +# +# In the remainder of this tutorial, we will show how +# :class:`~sklearn.linear_model.QuantileRegressor` can be used in practice and +# give the intuition into the properties of the fitted models. Finally, +# we will compare the both :class:`~sklearn.linear_model.QuantileRegressor` +# and :class:`~sklearn.linear_model.LinearRegression`. +# +# Fitting a `QuantileRegressor` +# ----------------------------- +# +# In this section, we want to estimate the conditional median as well as +# a low and high quantile fixed at 5% and 95%, respectively. Thus, we will get +# three linear models, one for each quantile. +# +# We will use the quantiles at 5% and 95% to find the outliers in the training +# sample beyond the central 90% interval. + +# %% +from sklearn.linear_model import QuantileRegressor + +quantiles = [0.05, 0.5, 0.95] +predictions = {} +out_bounds_predictions = np.zeros_like(y_true_mean, dtype=np.bool_) +for quantile in quantiles: + qr = QuantileRegressor(quantile=quantile, alpha=0) + y_pred = qr.fit(X, y_normal).predict(X) + predictions[quantile] = y_pred + + if quantile == min(quantiles): + out_bounds_predictions = np.logical_or( + out_bounds_predictions, y_pred >= y_normal + ) + elif quantile == max(quantiles): + out_bounds_predictions = np.logical_or( + out_bounds_predictions, y_pred <= y_normal + ) + +# %% +# Now, we can plot the three linear models and the distinguished samples that +# are within the central 90% interval from samples that are outside this +# interval. +plt.plot(X, y_true_mean, color="black", linestyle="dashed", label="True mean") + +for quantile, y_pred in predictions.items(): + plt.plot(X, y_pred, label=f"Quantile: {quantile}") + +plt.scatter( + x[out_bounds_predictions], + y_normal[out_bounds_predictions], + color="black", + marker="+", + alpha=0.5, + label="Outside interval", +) +plt.scatter( + x[~out_bounds_predictions], + y_normal[~out_bounds_predictions], + color="black", + alpha=0.5, + label="Inside interval", +) + +plt.legend() +plt.xlabel("x") +plt.ylabel("y") +_ = plt.title("Quantiles of heteroscedastic Normal distributed target") + +# %% +# Since the noise is still Normally distributed, in particular is symmetric, +# the true conditional mean and the true conditional median coincide. Indeed, +# we see that the estimated median almost hits the true mean. We observe the +# effect of having an increasing noise variance on the 5% and 95% quantiles: +# the slopes of those quantiles are very different and the interval between +# them becomes wider with increasing `x`. +# +# To get an additional intuition regarding the meaning of the 5% and 95% +# quantiles estimators, one can count the number of samples above and below the +# predicted quantiles (represented by a cross on the above plot), considering +# that we have a total of 100 samples. +# +# We can repeat the same experiment using the asymmetric Pareto distributed +# target. +quantiles = [0.05, 0.5, 0.95] +predictions = {} +out_bounds_predictions = np.zeros_like(y_true_mean, dtype=np.bool_) +for quantile in quantiles: + qr = QuantileRegressor(quantile=quantile, alpha=0) + y_pred = qr.fit(X, y_pareto).predict(X) + predictions[quantile] = y_pred + + if quantile == min(quantiles): + out_bounds_predictions = np.logical_or( + out_bounds_predictions, y_pred >= y_pareto + ) + elif quantile == max(quantiles): + out_bounds_predictions = np.logical_or( + out_bounds_predictions, y_pred <= y_pareto + ) + +# %% +plt.plot(X, y_true_mean, color="black", linestyle="dashed", label="True mean") + +for quantile, y_pred in predictions.items(): + plt.plot(X, y_pred, label=f"Quantile: {quantile}") + +plt.scatter( + x[out_bounds_predictions], + y_pareto[out_bounds_predictions], + color="black", + marker="+", + alpha=0.5, + label="Outside interval", +) +plt.scatter( + x[~out_bounds_predictions], + y_pareto[~out_bounds_predictions], + color="black", + alpha=0.5, + label="Inside interval", +) + +plt.legend() +plt.xlabel("x") +plt.ylabel("y") +_ = plt.title("Quantiles of asymmetric Pareto distributed target") + + +# %% +# Due to the asymmetry of the distribution of the noise, we observe that the +# true mean and estimated conditional median are different. We also observe +# that each quantile model has different parameters to better fit the desired +# quantile. Note that ideally, all quantiles would be parallel in this case, +# which would become more visible with more data points or less extreme +# quantiles, e.g. 10% and 90%. +# +# Comparing `QuantileRegressor` and `LinearRegression` +# ---------------------------------------------------- +# +# In this section, we will linger on the difference regarding the loss functions that +# :class:`~sklearn.linear_model.QuantileRegressor` and +# :class:`~sklearn.linear_model.LinearRegression` are minimizing. +# +# Indeed, :class:`~sklearn.linear_model.LinearRegression` is a least squares +# approach minimizing the mean squared error (MSE) between the training and +# predicted targets. In contrast, +# :class:`~sklearn.linear_model.QuantileRegressor` with `quantile=0.5` +# minimizes the mean absolute error (MAE) instead. +# +# Why does it matter? The loss functions specify what exactly the model is aiming +# to predict, see +# :ref:`user guide on the choice of scoring function`. +# In short, a model minimizing MSE predicts the mean (expectation) and a model +# minimizing MAE predicts the median. +# +# Let's compute the training errors of such models in terms of mean +# squared error and mean absolute error. We will use the asymmetric Pareto +# distributed target to make it more interesting as mean and median are not +# equal. +from sklearn.linear_model import LinearRegression +from sklearn.metrics import mean_absolute_error, mean_squared_error + +linear_regression = LinearRegression() +quantile_regression = QuantileRegressor(quantile=0.5, alpha=0) + +y_pred_lr = linear_regression.fit(X, y_pareto).predict(X) +y_pred_qr = quantile_regression.fit(X, y_pareto).predict(X) + +print( + "Training error (in-sample performance)\n" + f"{'model':<20} MAE MSE\n" + f"{linear_regression.__class__.__name__:<20} " + f"{mean_absolute_error(y_pareto, y_pred_lr):5.3f} " + f"{mean_squared_error(y_pareto, y_pred_lr):5.3f}\n" + f"{quantile_regression.__class__.__name__:<20} " + f"{mean_absolute_error(y_pareto, y_pred_qr):5.3f} " + f"{mean_squared_error(y_pareto, y_pred_qr):5.3f}" +) + +# %% +# On the training set, we see that MAE is lower for +# :class:`~sklearn.linear_model.QuantileRegressor` than +# :class:`~sklearn.linear_model.LinearRegression`. In contrast to that, MSE is +# lower for :class:`~sklearn.linear_model.LinearRegression` than +# :class:`~sklearn.linear_model.QuantileRegressor`. These results confirms that +# MAE is the loss minimized by :class:`~sklearn.linear_model.QuantileRegressor` +# while MSE is the loss minimized +# :class:`~sklearn.linear_model.LinearRegression`. +# +# We can make a similar evaluation by looking at the test error obtained by +# cross-validation. +from sklearn.model_selection import cross_validate + +cv_results_lr = cross_validate( + linear_regression, + X, + y_pareto, + cv=3, + scoring=["neg_mean_absolute_error", "neg_mean_squared_error"], +) +cv_results_qr = cross_validate( + quantile_regression, + X, + y_pareto, + cv=3, + scoring=["neg_mean_absolute_error", "neg_mean_squared_error"], +) +print( + "Test error (cross-validated performance)\n" + f"{'model':<20} MAE MSE\n" + f"{linear_regression.__class__.__name__:<20} " + f"{-cv_results_lr['test_neg_mean_absolute_error'].mean():5.3f} " + f"{-cv_results_lr['test_neg_mean_squared_error'].mean():5.3f}\n" + f"{quantile_regression.__class__.__name__:<20} " + f"{-cv_results_qr['test_neg_mean_absolute_error'].mean():5.3f} " + f"{-cv_results_qr['test_neg_mean_squared_error'].mean():5.3f}" +) + +# %% +# We reach similar conclusions on the out-of-sample evaluation. diff --git a/examples/linear_model/plot_ransac.py b/examples/linear_model/plot_ransac.py index 0bafe4ee4a394..ecef43e79f9bf 100644 --- a/examples/linear_model/plot_ransac.py +++ b/examples/linear_model/plot_ransac.py @@ -3,23 +3,38 @@ Robust linear model estimation using RANSAC =========================================== -In this example we see how to robustly fit a linear model to faulty data using -the RANSAC algorithm. +In this example, we see how to robustly fit a linear model to faulty data using +the :ref:`RANSAC ` algorithm. + +The ordinary linear regressor is sensitive to outliers, and the fitted line can +easily be skewed away from the true underlying relationship of data. + +The RANSAC regressor automatically splits the data into inliers and outliers, +and the fitted line is determined only by the identified inliers. + """ + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import numpy as np from matplotlib import pyplot as plt -from sklearn import linear_model, datasets - +from sklearn import datasets, linear_model n_samples = 1000 n_outliers = 50 -X, y, coef = datasets.make_regression(n_samples=n_samples, n_features=1, - n_informative=1, noise=10, - coef=True, random_state=0) +X, y, coef = datasets.make_regression( + n_samples=n_samples, + n_features=1, + n_informative=1, + noise=10, + coef=True, + random_state=0, +) # Add outlier data np.random.seed(0) @@ -46,14 +61,21 @@ print(coef, lr.coef_, ransac.estimator_.coef_) lw = 2 -plt.scatter(X[inlier_mask], y[inlier_mask], color='yellowgreen', marker='.', - label='Inliers') -plt.scatter(X[outlier_mask], y[outlier_mask], color='gold', marker='.', - label='Outliers') -plt.plot(line_X, line_y, color='navy', linewidth=lw, label='Linear regressor') -plt.plot(line_X, line_y_ransac, color='cornflowerblue', linewidth=lw, - label='RANSAC regressor') -plt.legend(loc='lower right') +plt.scatter( + X[inlier_mask], y[inlier_mask], color="yellowgreen", marker=".", label="Inliers" +) +plt.scatter( + X[outlier_mask], y[outlier_mask], color="gold", marker=".", label="Outliers" +) +plt.plot(line_X, line_y, color="navy", linewidth=lw, label="Linear regressor") +plt.plot( + line_X, + line_y_ransac, + color="cornflowerblue", + linewidth=lw, + label="RANSAC regressor", +) +plt.legend(loc="lower right") plt.xlabel("Input") plt.ylabel("Response") plt.show() diff --git a/examples/linear_model/plot_ridge_coeffs.py b/examples/linear_model/plot_ridge_coeffs.py index 3f3e574708d48..1ad7962f8bfa3 100644 --- a/examples/linear_model/plot_ridge_coeffs.py +++ b/examples/linear_model/plot_ridge_coeffs.py @@ -1,89 +1,181 @@ """ -============================================================== -Plot Ridge coefficients as a function of the L2 regularization -============================================================== - -.. currentmodule:: sklearn.linear_model - -:class:`Ridge` Regression is the estimator used in this example. -Each color in the left plot represents one different dimension of the -coefficient vector, and this is displayed as a function of the -regularization parameter. The right plot shows how exact the solution -is. This example illustrates how a well defined solution is -found by Ridge regression and how regularization affects the -coefficients and their values. The plot on the right shows how -the difference of the coefficients from the estimator changes -as a function of regularization. - -In this example the dependent variable Y is set as a function -of the input features: y = X*w + c. The coefficient vector w is -randomly sampled from a normal distribution, whereas the bias term c is -set to a constant. - -As alpha tends toward zero the coefficients found by Ridge -regression stabilize towards the randomly sampled vector w. -For big alpha (strong regularisation) the coefficients -are smaller (eventually converging at 0) leading to a -simpler and biased solution. -These dependencies can be observed on the left plot. - -The right plot shows the mean squared error between the -coefficients found by the model and the chosen vector w. -Less regularised models retrieve the exact -coefficients (error is equal to 0), stronger regularised -models increase the error. - -Please note that in this example the data is non-noisy, hence -it is possible to extract the exact coefficients. +========================================================= +Ridge coefficients as a function of the L2 Regularization +========================================================= + +A model that overfits learns the training data too well, capturing both the +underlying patterns and the noise in the data. However, when applied to unseen +data, the learned associations may not hold. We normally detect this when we +apply our trained predictions to the test data and see the statistical +performance drop significantly compared to the training data. + +One way to overcome overfitting is through regularization, which can be done by +penalizing large weights (coefficients) in linear models, forcing the model to +shrink all coefficients. Regularization reduces a model's reliance on specific +information obtained from the training samples. + +This example illustrates how L2 regularization in a +:class:`~sklearn.linear_model.Ridge` regression affects a model's performance by +adding a penalty term to the loss that increases with the coefficients +:math:`\\beta`. + +The regularized loss function is given by: :math:`\\mathcal{L}(X, y, \\beta) = +\\| y - X \\beta \\|^{2}_{2} + \\alpha \\| \\beta \\|^{2}_{2}` + +where :math:`X` is the input data, :math:`y` is the target variable, +:math:`\\beta` is the vector of coefficients associated with the features, and +:math:`\\alpha` is the regularization strength. + +The regularized loss function aims to balance the trade-off between accurately +predicting the training set and to prevent overfitting. + +In this regularized loss, the left-hand side (e.g. :math:`\\|y - +X\\beta\\|^{2}_{2}`) measures the squared difference between the actual target +variable, :math:`y`, and the predicted values. Minimizing this term alone could +lead to overfitting, as the model may become too complex and sensitive to noise +in the training data. + +To address overfitting, Ridge regularization adds a constraint, called a penalty +term, (:math:`\\alpha \\| \\beta\\|^{2}_{2}`) to the loss function. This penalty +term is the sum of the squares of the model's coefficients, multiplied by the +regularization strength :math:`\\alpha`. By introducing this constraint, Ridge +regularization discourages any single coefficient :math:`\\beta_{i}` from taking +an excessively large value and encourages smaller and more evenly distributed +coefficients. Higher values of :math:`\\alpha` force the coefficients towards +zero. However, an excessively high :math:`\\alpha` can result in an underfit +model that fails to capture important patterns in the data. + +Therefore, the regularized loss function combines the prediction accuracy term +and the penalty term. By adjusting the regularization strength, practitioners +can fine-tune the degree of constraint imposed on the weights, training a model +capable of generalizing well to unseen data while avoiding overfitting. """ -# Author: Kornel Kielczewski -- - -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Purpose of this example +# ----------------------- +# For the purpose of showing how Ridge regularization works, we will create a +# non-noisy data set. Then we will train a regularized model on a range of +# regularization strengths (:math:`\alpha`) and plot how the trained +# coefficients and the mean squared error between those and the original values +# behave as functions of the regularization strength. +# +# Creating a non-noisy data set +# ***************************** +# We make a toy data set with 100 samples and 10 features, that's suitable to +# detect regression. Out of the 10 features, 8 are informative and contribute to +# the regression, while the remaining 2 features do not have any effect on the +# target variable (their true coefficients are 0). Please note that in this +# example the data is non-noisy, hence we can expect our regression model to +# recover exactly the true coefficients w. +from sklearn.datasets import make_regression -import matplotlib.pyplot as plt +X, y, w = make_regression( + n_samples=100, n_features=10, n_informative=8, coef=True, random_state=1 +) + +# Obtain the true coefficients +print(f"The true coefficient of this regression problem are:\n{w}") + +# %% +# Training the Ridge Regressor +# **************************** +# We use :class:`~sklearn.linear_model.Ridge`, a linear model with L2 +# regularization. We train several models, each with a different value for the +# model parameter `alpha`, which is a positive constant that multiplies the +# penalty term, controlling the regularization strength. For each trained model +# we then compute the error between the true coefficients `w` and the +# coefficients found by the model `clf`. We store the identified coefficients +# and the calculated errors for the corresponding coefficients in lists, which +# makes it convenient for us to plot them. import numpy as np -from sklearn.datasets import make_regression from sklearn.linear_model import Ridge from sklearn.metrics import mean_squared_error clf = Ridge() -X, y, w = make_regression(n_samples=10, n_features=10, coef=True, - random_state=1, bias=3.5) - +# Generate values for `alpha` that are evenly distributed on a logarithmic scale +alphas = np.logspace(-3, 4, 200) coefs = [] -errors = [] - -alphas = np.logspace(-6, 6, 200) +errors_coefs = [] # Train the model with different regularisation strengths for a in alphas: - clf.set_params(alpha=a) - clf.fit(X, y) + clf.set_params(alpha=a).fit(X, y) coefs.append(clf.coef_) - errors.append(mean_squared_error(clf.coef_, w)) - -# Display results -plt.figure(figsize=(20, 6)) - -plt.subplot(121) -ax = plt.gca() -ax.plot(alphas, coefs) -ax.set_xscale('log') -plt.xlabel('alpha') -plt.ylabel('weights') -plt.title('Ridge coefficients as a function of the regularization') -plt.axis('tight') - -plt.subplot(122) -ax = plt.gca() -ax.plot(alphas, errors) -ax.set_xscale('log') -plt.xlabel('alpha') -plt.ylabel('error') -plt.title('Coefficient error as a function of the regularization') -plt.axis('tight') - -plt.show() + errors_coefs.append(mean_squared_error(clf.coef_, w)) + +# %% +# Plotting trained Coefficients and Mean Squared Errors +# ***************************************************** +# We now plot the 10 different regularized coefficients as a function of the +# regularization parameter `alpha` where each color represents a different +# coefficient. +# +# On the right-hand-side, we plot how the errors of the coefficients from the +# estimator change as a function of regularization. +import matplotlib.pyplot as plt +import pandas as pd + +alphas = pd.Index(alphas, name="alpha") +coefs = pd.DataFrame(coefs, index=alphas, columns=[f"Feature {i}" for i in range(10)]) +errors = pd.Series(errors_coefs, index=alphas, name="Mean squared error") + +fig, axs = plt.subplots(1, 2, figsize=(20, 6)) + +coefs.plot( + ax=axs[0], + logx=True, + title="Ridge coefficients as a function of the regularization strength", +) +axs[0].set_ylabel("Ridge coefficient values") +errors.plot( + ax=axs[1], + logx=True, + title="Coefficient error as a function of the regularization strength", +) +_ = axs[1].set_ylabel("Mean squared error") +# %% +# Interpreting the plots +# ********************** +# The plot on the left-hand side shows how the regularization strength (`alpha`) +# affects the Ridge regression coefficients. Smaller values of `alpha` (weak +# regularization), allow the coefficients to closely resemble the true +# coefficients (`w`) used to generate the data set. This is because no +# additional noise was added to our artificial data set. As `alpha` increases, +# the coefficients shrink towards zero, gradually reducing the impact of the +# features that were formerly more significant. +# +# The right-hand side plot shows the mean squared error (MSE) between the +# coefficients found by the model and the true coefficients (`w`). It provides a +# measure that relates to how exact our ridge model is in comparison to the true +# generative model. A low error means that it found coefficients closer to the +# ones of the true generative model. In this case, since our toy data set was +# non-noisy, we can see that the least regularized model retrieves coefficients +# closest to the true coefficients (`w`) (error is close to 0). +# +# When `alpha` is small, the model captures the intricate details of the +# training data, whether those were caused by noise or by actual information. As +# `alpha` increases, the highest coefficients shrink more rapidly, rendering +# their corresponding features less influential in the training process. This +# can enhance a model's ability to generalize to unseen data (if there was a lot +# of noise to capture), but it also poses the risk of losing performance if the +# regularization becomes too strong compared to the amount of noise the data +# contained (as in this example). +# +# In real-world scenarios where data typically includes noise, selecting an +# appropriate `alpha` value becomes crucial in striking a balance between an +# overfitting and an underfitting model. +# +# Here, we saw that :class:`~sklearn.linear_model.Ridge` adds a penalty to the +# coefficients to fight overfitting. Another problem that occurs is linked to +# the presence of outliers in the training dataset. An outlier is a data point +# that differs significantly from other observations. Concretely, these outliers +# impact the left-hand side term of the loss function that we showed earlier. +# Some other linear models are formulated to be robust to outliers such as the +# :class:`~sklearn.linear_model.HuberRegressor`. You can learn more about it in +# the :ref:`sphx_glr_auto_examples_linear_model_plot_huber_vs_ridge.py` example. diff --git a/examples/linear_model/plot_ridge_path.py b/examples/linear_model/plot_ridge_path.py index b16212cbd3718..d3c19acd9e18c 100644 --- a/examples/linear_model/plot_ridge_path.py +++ b/examples/linear_model/plot_ridge_path.py @@ -24,23 +24,24 @@ and the solution tends towards the ordinary least squares, coefficients exhibit big oscillations. In practise it is necessary to tune alpha in such a way that a balance is maintained between both. -""" -# Author: Fabian Pedregosa -- -# License: BSD 3 clause +""" -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt +import numpy as np + from sklearn import linear_model # X is the 10x10 Hilbert matrix -X = 1. / (np.arange(1, 11) + np.arange(0, 10)[:, np.newaxis]) +X = 1.0 / (np.arange(1, 11) + np.arange(0, 10)[:, np.newaxis]) y = np.ones(10) -# ############################################################################# +# %% # Compute paths +# ------------- n_alphas = 200 alphas = np.logspace(-10, -2, n_alphas) @@ -51,16 +52,17 @@ ridge.fit(X, y) coefs.append(ridge.coef_) -# ############################################################################# +# %% # Display results +# --------------- ax = plt.gca() ax.plot(alphas, coefs) -ax.set_xscale('log') +ax.set_xscale("log") ax.set_xlim(ax.get_xlim()[::-1]) # reverse axis -plt.xlabel('alpha') -plt.ylabel('weights') -plt.title('Ridge coefficients as a function of the regularization') -plt.axis('tight') +plt.xlabel("alpha") +plt.ylabel("weights") +plt.title("Ridge coefficients as a function of the regularization") +plt.axis("tight") plt.show() diff --git a/examples/linear_model/plot_robust_fit.py b/examples/linear_model/plot_robust_fit.py index 8b4ce05c773cf..874a21fb87a22 100644 --- a/examples/linear_model/plot_robust_fit.py +++ b/examples/linear_model/plot_robust_fit.py @@ -5,7 +5,7 @@ Here a sine function is fit with a polynomial of order 3, for values close to zero. -Robust fitting is demoed in different situations: +Robust fitting is demonstrated in different situations: - No measurement errors, only modelling errors (fitting a sine with a polynomial) @@ -30,14 +30,21 @@ """ -from matplotlib import pyplot as plt +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import numpy as np +from matplotlib import pyplot as plt from sklearn.linear_model import ( - LinearRegression, TheilSenRegressor, RANSACRegressor, HuberRegressor) + HuberRegressor, + LinearRegression, + RANSACRegressor, + TheilSenRegressor, +) from sklearn.metrics import mean_squared_error -from sklearn.preprocessing import PolynomialFeatures from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import PolynomialFeatures np.random.seed(42) @@ -62,35 +69,50 @@ X_errors_large = X.copy() X_errors_large[::3] = 10 -estimators = [('OLS', LinearRegression()), - ('Theil-Sen', TheilSenRegressor(random_state=42)), - ('RANSAC', RANSACRegressor(random_state=42)), - ('HuberRegressor', HuberRegressor())] -colors = {'OLS': 'turquoise', 'Theil-Sen': 'gold', 'RANSAC': 'lightgreen', 'HuberRegressor': 'black'} -linestyle = {'OLS': '-', 'Theil-Sen': '-.', 'RANSAC': '--', 'HuberRegressor': '--'} +estimators = [ + ("OLS", LinearRegression()), + ("Theil-Sen", TheilSenRegressor(random_state=42)), + ("RANSAC", RANSACRegressor(random_state=42)), + ("HuberRegressor", HuberRegressor()), +] +colors = { + "OLS": "turquoise", + "Theil-Sen": "gold", + "RANSAC": "lightgreen", + "HuberRegressor": "black", +} +linestyle = {"OLS": "-", "Theil-Sen": "-.", "RANSAC": "--", "HuberRegressor": "--"} lw = 3 x_plot = np.linspace(X.min(), X.max()) for title, this_X, this_y in [ - ('Modeling Errors Only', X, y), - ('Corrupt X, Small Deviants', X_errors, y), - ('Corrupt y, Small Deviants', X, y_errors), - ('Corrupt X, Large Deviants', X_errors_large, y), - ('Corrupt y, Large Deviants', X, y_errors_large)]: + ("Modeling Errors Only", X, y), + ("Corrupt X, Small Deviants", X_errors, y), + ("Corrupt y, Small Deviants", X, y_errors), + ("Corrupt X, Large Deviants", X_errors_large, y), + ("Corrupt y, Large Deviants", X, y_errors_large), +]: plt.figure(figsize=(5, 4)) - plt.plot(this_X[:, 0], this_y, 'b+') + plt.plot(this_X[:, 0], this_y, "b+") for name, estimator in estimators: model = make_pipeline(PolynomialFeatures(3), estimator) model.fit(this_X, this_y) mse = mean_squared_error(model.predict(X_test), y_test) y_plot = model.predict(x_plot[:, np.newaxis]) - plt.plot(x_plot, y_plot, color=colors[name], linestyle=linestyle[name], - linewidth=lw, label='%s: error = %.3f' % (name, mse)) - - legend_title = 'Error of Mean\nAbsolute Deviation\nto Non-corrupt Data' - legend = plt.legend(loc='upper right', frameon=False, title=legend_title, - prop=dict(size='x-small')) + plt.plot( + x_plot, + y_plot, + color=colors[name], + linestyle=linestyle[name], + linewidth=lw, + label="%s: error = %.3f" % (name, mse), + ) + + legend_title = "Error of Mean\nAbsolute Deviation\nto Non-corrupt Data" + legend = plt.legend( + loc="upper right", frameon=False, title=legend_title, prop=dict(size="x-small") + ) plt.xlim(-4, 10.2) plt.ylim(-2, 10.2) plt.title(title) diff --git a/examples/linear_model/plot_sgd_comparison.py b/examples/linear_model/plot_sgd_comparison.py deleted file mode 100644 index 346627c933d85..0000000000000 --- a/examples/linear_model/plot_sgd_comparison.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -================================== -Comparing various online solvers -================================== - -An example showing how different online solvers perform -on the hand-written digits dataset. - -""" -# Author: Rob Zinkov -# License: BSD 3 clause - -import numpy as np -import matplotlib.pyplot as plt -from sklearn import datasets - -from sklearn.model_selection import train_test_split -from sklearn.linear_model import SGDClassifier, Perceptron -from sklearn.linear_model import PassiveAggressiveClassifier -from sklearn.linear_model import LogisticRegression - -heldout = [0.95, 0.90, 0.75, 0.50, 0.01] -rounds = 20 -X, y = datasets.load_digits(return_X_y=True) - -classifiers = [ - ("SGD", SGDClassifier(max_iter=100)), - ("ASGD", SGDClassifier(average=True)), - ("Perceptron", Perceptron()), - ("Passive-Aggressive I", PassiveAggressiveClassifier(loss='hinge', - C=1.0, tol=1e-4)), - ("Passive-Aggressive II", PassiveAggressiveClassifier(loss='squared_hinge', - C=1.0, tol=1e-4)), - ("SAG", LogisticRegression(solver='sag', tol=1e-1, C=1.e4 / X.shape[0])) -] - -xx = 1. - np.array(heldout) - -for name, clf in classifiers: - print("training %s" % name) - rng = np.random.RandomState(42) - yy = [] - for i in heldout: - yy_ = [] - for r in range(rounds): - X_train, X_test, y_train, y_test = \ - train_test_split(X, y, test_size=i, random_state=rng) - clf.fit(X_train, y_train) - y_pred = clf.predict(X_test) - yy_.append(1 - np.mean(y_pred == y_test)) - yy.append(np.mean(yy_)) - plt.plot(xx, yy, label=name) - -plt.legend(loc="upper right") -plt.xlabel("Proportion train") -plt.ylabel("Test Error Rate") -plt.show() diff --git a/examples/linear_model/plot_sgd_early_stopping.py b/examples/linear_model/plot_sgd_early_stopping.py index bd7461dfb5f92..a9a5f111dbc18 100644 --- a/examples/linear_model/plot_sgd_early_stopping.py +++ b/examples/linear_model/plot_sgd_early_stopping.py @@ -34,31 +34,31 @@ significantly reduce training time. Note that scores differ between the stopping criteria even from early iterations because some of the training data is held out with the validation stopping criterion. + """ -# Authors: Tom Dupre la Tour -# -# License: BSD 3 clause -import time + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import sys +import time -import pandas as pd -import numpy as np import matplotlib.pyplot as plt +import numpy as np +import pandas as pd from sklearn import linear_model from sklearn.datasets import fetch_openml -from sklearn.model_selection import train_test_split -from sklearn.utils._testing import ignore_warnings from sklearn.exceptions import ConvergenceWarning +from sklearn.model_selection import train_test_split from sklearn.utils import shuffle - -print(__doc__) +from sklearn.utils._testing import ignore_warnings -def load_mnist(n_samples=None, class_0='0', class_1='8'): +def load_mnist(n_samples=None, class_0="0", class_1="8"): """Load MNIST, select two classes, shuffle and return only n_samples.""" # Load data from http://openml.org/d/554 - mnist = fetch_openml('mnist_784', version=1) + mnist = fetch_openml("mnist_784", version=1, as_frame=False) # take only two classes for binary classification mask = np.logical_or(mnist.target == class_0, mnist.target == class_1) @@ -88,62 +88,67 @@ def fit_and_score(estimator, max_iter, X_train, X_test, y_train, y_test): # Define the estimators to compare estimator_dict = { - 'No stopping criterion': - linear_model.SGDClassifier(n_iter_no_change=3), - 'Training loss': - linear_model.SGDClassifier(early_stopping=False, n_iter_no_change=3, - tol=0.1), - 'Validation score': - linear_model.SGDClassifier(early_stopping=True, n_iter_no_change=3, - tol=0.0001, validation_fraction=0.2) + "No stopping criterion": linear_model.SGDClassifier(n_iter_no_change=3), + "Training loss": linear_model.SGDClassifier( + early_stopping=False, n_iter_no_change=3, tol=0.1 + ), + "Validation score": linear_model.SGDClassifier( + early_stopping=True, n_iter_no_change=3, tol=0.0001, validation_fraction=0.2 + ), } # Load the dataset X, y = load_mnist(n_samples=10000) -X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, - random_state=0) +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0) results = [] for estimator_name, estimator in estimator_dict.items(): - print(estimator_name + ': ', end='') + print(estimator_name + ": ", end="") for max_iter in range(1, 50): - print('.', end='') + print(".", end="") sys.stdout.flush() fit_time, n_iter, train_score, test_score = fit_and_score( - estimator, max_iter, X_train, X_test, y_train, y_test) + estimator, max_iter, X_train, X_test, y_train, y_test + ) - results.append((estimator_name, max_iter, fit_time, n_iter, - train_score, test_score)) - print('') + results.append( + (estimator_name, max_iter, fit_time, n_iter, train_score, test_score) + ) + print("") # Transform the results in a pandas dataframe for easy plotting columns = [ - 'Stopping criterion', 'max_iter', 'Fit time (sec)', 'n_iter_', - 'Train score', 'Test score' + "Stopping criterion", + "max_iter", + "Fit time (sec)", + "n_iter_", + "Train score", + "Test score", ] results_df = pd.DataFrame(results, columns=columns) -# Define what to plot (x_axis, y_axis) -lines = 'Stopping criterion' -plot_list = [ - ('max_iter', 'Train score'), - ('max_iter', 'Test score'), - ('max_iter', 'n_iter_'), - ('max_iter', 'Fit time (sec)'), -] - -nrows = 2 -ncols = int(np.ceil(len(plot_list) / 2.)) -fig, axes = plt.subplots(nrows=nrows, ncols=ncols, figsize=(6 * ncols, - 4 * nrows)) -axes[0, 0].get_shared_y_axes().join(axes[0, 0], axes[0, 1]) +# Define what to plot +lines = "Stopping criterion" +x_axis = "max_iter" +styles = ["-.", "--", "-"] -for ax, (x_axis, y_axis) in zip(axes.ravel(), plot_list): - for criterion, group_df in results_df.groupby(lines): - group_df.plot(x=x_axis, y=y_axis, label=criterion, ax=ax) +# First plot: train and test scores +fig, axes = plt.subplots(nrows=1, ncols=2, sharey=True, figsize=(12, 4)) +for ax, y_axis in zip(axes, ["Train score", "Test score"]): + for style, (criterion, group_df) in zip(styles, results_df.groupby(lines)): + group_df.plot(x=x_axis, y=y_axis, label=criterion, ax=ax, style=style) ax.set_title(y_axis) ax.legend(title=lines) +fig.tight_layout() +# Second plot: n_iter and fit time +fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(12, 4)) +for ax, y_axis in zip(axes, ["n_iter_", "Fit time (sec)"]): + for style, (criterion, group_df) in zip(styles, results_df.groupby(lines)): + group_df.plot(x=x_axis, y=y_axis, label=criterion, ax=ax, style=style) + ax.set_title(y_axis) + ax.legend(title=lines) fig.tight_layout() + plt.show() diff --git a/examples/linear_model/plot_sgd_iris.py b/examples/linear_model/plot_sgd_iris.py index 0dddf7475728d..e8aaf3a2e13a2 100644 --- a/examples/linear_model/plot_sgd_iris.py +++ b/examples/linear_model/plot_sgd_iris.py @@ -8,11 +8,15 @@ are represented by the dashed lines. """ -print(__doc__) -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt +import numpy as np + from sklearn import datasets +from sklearn.inspection import DecisionBoundaryDisplay from sklearn.linear_model import SGDClassifier # import some data to play with @@ -36,31 +40,32 @@ std = X.std(axis=0) X = (X - mean) / std -h = .02 # step size in the mesh - clf = SGDClassifier(alpha=0.001, max_iter=100).fit(X, y) - -# create a mesh to plot in -x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1 -y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1 -xx, yy = np.meshgrid(np.arange(x_min, x_max, h), - np.arange(y_min, y_max, h)) - -# Plot the decision boundary. For that, we will assign a color to each -# point in the mesh [x_min, x_max]x[y_min, y_max]. -Z = clf.predict(np.c_[xx.ravel(), yy.ravel()]) -# Put the result into a color plot -Z = Z.reshape(xx.shape) -cs = plt.contourf(xx, yy, Z, cmap=plt.cm.Paired) -plt.axis('tight') +ax = plt.gca() +DecisionBoundaryDisplay.from_estimator( + clf, + X, + cmap=plt.cm.Paired, + ax=ax, + response_method="predict", + xlabel=iris.feature_names[0], + ylabel=iris.feature_names[1], +) +plt.axis("tight") # Plot also the training points for i, color in zip(clf.classes_, colors): - idx = np.where(y == i) - plt.scatter(X[idx, 0], X[idx, 1], c=color, label=iris.target_names[i], - cmap=plt.cm.Paired, edgecolor='black', s=20) + idx = (y == i).nonzero() + plt.scatter( + X[idx, 0], + X[idx, 1], + c=color, + label=iris.target_names[i], + edgecolor="black", + s=20, + ) plt.title("Decision surface of multi-class SGD") -plt.axis('tight') +plt.axis("tight") # Plot the three one-against-all classifiers xmin, xmax = plt.xlim() @@ -73,8 +78,7 @@ def plot_hyperplane(c, color): def line(x0): return (-(x0 * coef[c, 0]) - intercept[c]) / coef[c, 1] - plt.plot([xmin, xmax], [line(xmin), line(xmax)], - ls="--", color=color) + plt.plot([xmin, xmax], [line(xmin), line(xmax)], ls="--", color=color) for i, color in zip(clf.classes_, colors): diff --git a/examples/linear_model/plot_sgd_loss_functions.py b/examples/linear_model/plot_sgd_loss_functions.py index 4a7ad9ce9f0be..b0c61da6ddcc1 100644 --- a/examples/linear_model/plot_sgd_loss_functions.py +++ b/examples/linear_model/plot_sgd_loss_functions.py @@ -5,36 +5,46 @@ A plot that compares the various convex loss functions supported by :class:`~sklearn.linear_model.SGDClassifier` . + """ -print(__doc__) -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt +import numpy as np def modified_huber_loss(y_true, y_pred): z = y_pred * y_true loss = -4 * z loss[z >= -1] = (1 - z[z >= -1]) ** 2 - loss[z >= 1.] = 0 + loss[z >= 1.0] = 0 return loss xmin, xmax = -4, 4 xx = np.linspace(xmin, xmax, 100) lw = 2 -plt.plot([xmin, 0, 0, xmax], [1, 1, 0, 0], color='gold', lw=lw, - label="Zero-one loss") -plt.plot(xx, np.where(xx < 1, 1 - xx, 0), color='teal', lw=lw, - label="Hinge loss") -plt.plot(xx, -np.minimum(xx, 0), color='yellowgreen', lw=lw, - label="Perceptron loss") -plt.plot(xx, np.log2(1 + np.exp(-xx)), color='cornflowerblue', lw=lw, - label="Log loss") -plt.plot(xx, np.where(xx < 1, 1 - xx, 0) ** 2, color='orange', lw=lw, - label="Squared hinge loss") -plt.plot(xx, modified_huber_loss(xx, 1), color='darkorchid', lw=lw, - linestyle='--', label="Modified Huber loss") +plt.plot([xmin, 0, 0, xmax], [1, 1, 0, 0], color="gold", lw=lw, label="Zero-one loss") +plt.plot(xx, np.where(xx < 1, 1 - xx, 0), color="teal", lw=lw, label="Hinge loss") +plt.plot(xx, -np.minimum(xx, 0), color="yellowgreen", lw=lw, label="Perceptron loss") +plt.plot(xx, np.log2(1 + np.exp(-xx)), color="cornflowerblue", lw=lw, label="Log loss") +plt.plot( + xx, + np.where(xx < 1, 1 - xx, 0) ** 2, + color="orange", + lw=lw, + label="Squared hinge loss", +) +plt.plot( + xx, + modified_huber_loss(xx, 1), + color="darkorchid", + lw=lw, + linestyle="--", + label="Modified Huber loss", +) plt.ylim((0, 8)) plt.legend(loc="upper right") plt.xlabel(r"Decision function $f(x)$") diff --git a/examples/linear_model/plot_sgd_penalties.py b/examples/linear_model/plot_sgd_penalties.py index 0307fb0e8ed94..6f8830b52fe7a 100644 --- a/examples/linear_model/plot_sgd_penalties.py +++ b/examples/linear_model/plot_sgd_penalties.py @@ -10,10 +10,12 @@ and :class:`~sklearn.linear_model.SGDRegressor`. """ -print(__doc__) -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt +import numpy as np l1_color = "navy" l2_color = "c" @@ -22,7 +24,7 @@ line = np.linspace(-1.5, 1.5, 1001) xx, yy = np.meshgrid(line, line) -l2 = xx ** 2 + yy ** 2 +l2 = xx**2 + yy**2 l1 = np.abs(xx) + np.abs(yy) rho = 0.5 elastic_net = rho * l1 + (1 - rho) * l2 @@ -30,22 +32,26 @@ plt.figure(figsize=(10, 10), dpi=100) ax = plt.gca() -elastic_net_contour = plt.contour(xx, yy, elastic_net, levels=[1], - colors=elastic_net_color) +elastic_net_contour = plt.contour( + xx, yy, elastic_net, levels=[1], colors=elastic_net_color +) l2_contour = plt.contour(xx, yy, l2, levels=[1], colors=l2_color) l1_contour = plt.contour(xx, yy, l1, levels=[1], colors=l1_color) ax.set_aspect("equal") -ax.spines['left'].set_position('center') -ax.spines['right'].set_color('none') -ax.spines['bottom'].set_position('center') -ax.spines['top'].set_color('none') - -plt.clabel(elastic_net_contour, inline=1, fontsize=18, - fmt={1.0: 'elastic-net'}, manual=[(-1, -1)]) -plt.clabel(l2_contour, inline=1, fontsize=18, - fmt={1.0: 'L2'}, manual=[(-1, -1)]) -plt.clabel(l1_contour, inline=1, fontsize=18, - fmt={1.0: 'L1'}, manual=[(-1, -1)]) +ax.spines["left"].set_position("center") +ax.spines["right"].set_color("none") +ax.spines["bottom"].set_position("center") +ax.spines["top"].set_color("none") + +plt.clabel( + elastic_net_contour, + inline=1, + fontsize=18, + fmt={1.0: "elastic-net"}, + manual=[(-1, -1)], +) +plt.clabel(l2_contour, inline=1, fontsize=18, fmt={1.0: "L2"}, manual=[(-1, -1)]) +plt.clabel(l1_contour, inline=1, fontsize=18, fmt={1.0: "L1"}, manual=[(-1, -1)]) plt.tight_layout() plt.show() diff --git a/examples/linear_model/plot_sgd_separating_hyperplane.py b/examples/linear_model/plot_sgd_separating_hyperplane.py index e7263e4ecd347..90f7502900291 100644 --- a/examples/linear_model/plot_sgd_separating_hyperplane.py +++ b/examples/linear_model/plot_sgd_separating_hyperplane.py @@ -6,13 +6,17 @@ Plot the maximum margin separating hyperplane within a two-class separable dataset using a linear Support Vector Machines classifier trained using SGD. + """ -print(__doc__) -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt -from sklearn.linear_model import SGDClassifier +import numpy as np + from sklearn.datasets import make_blobs +from sklearn.linear_model import SGDClassifier # we create 50 separable points X, Y = make_blobs(n_samples=50, centers=2, random_state=0, cluster_std=0.60) @@ -34,11 +38,10 @@ p = clf.decision_function([[x1, x2]]) Z[i, j] = p[0] levels = [-1.0, 0.0, 1.0] -linestyles = ['dashed', 'solid', 'dashed'] -colors = 'k' +linestyles = ["dashed", "solid", "dashed"] +colors = "k" plt.contour(X1, X2, Z, levels, colors=colors, linestyles=linestyles) -plt.scatter(X[:, 0], X[:, 1], c=Y, cmap=plt.cm.Paired, - edgecolor='black', s=20) +plt.scatter(X[:, 0], X[:, 1], c=Y, cmap=plt.cm.Paired, edgecolor="black", s=20) -plt.axis('tight') +plt.axis("tight") plt.show() diff --git a/examples/linear_model/plot_sgd_weighted_samples.py b/examples/linear_model/plot_sgd_weighted_samples.py index 3617d81b0a063..e9e6587004e70 100644 --- a/examples/linear_model/plot_sgd_weighted_samples.py +++ b/examples/linear_model/plot_sgd_weighted_samples.py @@ -5,11 +5,15 @@ Plot decision function of a weighted dataset, where the size of points is proportional to its weight. + """ -print(__doc__) -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt +import numpy as np + from sklearn import linear_model # we create 20 points @@ -22,27 +26,38 @@ # plot the weighted data points xx, yy = np.meshgrid(np.linspace(-4, 5, 500), np.linspace(-4, 5, 500)) -plt.figure() -plt.scatter(X[:, 0], X[:, 1], c=y, s=sample_weight, alpha=0.9, - cmap=plt.cm.bone, edgecolor='black') +fig, ax = plt.subplots() +ax.scatter( + X[:, 0], + X[:, 1], + c=y, + s=sample_weight, + alpha=0.9, + cmap=plt.cm.bone, + edgecolor="black", +) # fit the unweighted model clf = linear_model.SGDClassifier(alpha=0.01, max_iter=100) clf.fit(X, y) Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()]) Z = Z.reshape(xx.shape) -no_weights = plt.contour(xx, yy, Z, levels=[0], linestyles=['solid']) +no_weights = ax.contour(xx, yy, Z, levels=[0], linestyles=["solid"]) # fit the weighted model clf = linear_model.SGDClassifier(alpha=0.01, max_iter=100) clf.fit(X, y, sample_weight=sample_weight) Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()]) Z = Z.reshape(xx.shape) -samples_weights = plt.contour(xx, yy, Z, levels=[0], linestyles=['dashed']) +samples_weights = ax.contour(xx, yy, Z, levels=[0], linestyles=["dashed"]) -plt.legend([no_weights.collections[0], samples_weights.collections[0]], - ["no weights", "with weights"], loc="lower left") +no_weights_handles, _ = no_weights.legend_elements() +weights_handles, _ = samples_weights.legend_elements() +ax.legend( + [no_weights_handles[0], weights_handles[0]], + ["no weights", "with weights"], + loc="lower left", +) -plt.xticks(()) -plt.yticks(()) +ax.set(xticks=(), yticks=()) plt.show() diff --git a/examples/linear_model/plot_sgdocsvm_vs_ocsvm.py b/examples/linear_model/plot_sgdocsvm_vs_ocsvm.py new file mode 100644 index 0000000000000..4829e87bfda0b --- /dev/null +++ b/examples/linear_model/plot_sgdocsvm_vs_ocsvm.py @@ -0,0 +1,200 @@ +""" +==================================================================== +One-Class SVM versus One-Class SVM using Stochastic Gradient Descent +==================================================================== + +This example shows how to approximate the solution of +:class:`sklearn.svm.OneClassSVM` in the case of an RBF kernel with +:class:`sklearn.linear_model.SGDOneClassSVM`, a Stochastic Gradient Descent +(SGD) version of the One-Class SVM. A kernel approximation is first used in +order to apply :class:`sklearn.linear_model.SGDOneClassSVM` which implements a +linear One-Class SVM using SGD. + +Note that :class:`sklearn.linear_model.SGDOneClassSVM` scales linearly with +the number of samples whereas the complexity of a kernelized +:class:`sklearn.svm.OneClassSVM` is at best quadratic with respect to the +number of samples. It is not the purpose of this example to illustrate the +benefits of such an approximation in terms of computation time but rather to +show that we obtain similar results on a toy dataset. + +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +import matplotlib +import matplotlib.lines as mlines +import matplotlib.pyplot as plt +import numpy as np + +from sklearn.kernel_approximation import Nystroem +from sklearn.linear_model import SGDOneClassSVM +from sklearn.pipeline import make_pipeline +from sklearn.svm import OneClassSVM + +font = {"weight": "normal", "size": 15} + +matplotlib.rc("font", **font) + +random_state = 42 +rng = np.random.RandomState(random_state) + +# Generate train data +X = 0.3 * rng.randn(500, 2) +X_train = np.r_[X + 2, X - 2] +# Generate some regular novel observations +X = 0.3 * rng.randn(20, 2) +X_test = np.r_[X + 2, X - 2] +# Generate some abnormal novel observations +X_outliers = rng.uniform(low=-4, high=4, size=(20, 2)) + +# OCSVM hyperparameters +nu = 0.05 +gamma = 2.0 + +# Fit the One-Class SVM +clf = OneClassSVM(gamma=gamma, kernel="rbf", nu=nu) +clf.fit(X_train) +y_pred_train = clf.predict(X_train) +y_pred_test = clf.predict(X_test) +y_pred_outliers = clf.predict(X_outliers) +n_error_train = y_pred_train[y_pred_train == -1].size +n_error_test = y_pred_test[y_pred_test == -1].size +n_error_outliers = y_pred_outliers[y_pred_outliers == 1].size + +# Fit the One-Class SVM using a kernel approximation and SGD +transform = Nystroem(gamma=gamma, random_state=random_state) +clf_sgd = SGDOneClassSVM( + nu=nu, shuffle=True, fit_intercept=True, random_state=random_state, tol=1e-4 +) +pipe_sgd = make_pipeline(transform, clf_sgd) +pipe_sgd.fit(X_train) +y_pred_train_sgd = pipe_sgd.predict(X_train) +y_pred_test_sgd = pipe_sgd.predict(X_test) +y_pred_outliers_sgd = pipe_sgd.predict(X_outliers) +n_error_train_sgd = y_pred_train_sgd[y_pred_train_sgd == -1].size +n_error_test_sgd = y_pred_test_sgd[y_pred_test_sgd == -1].size +n_error_outliers_sgd = y_pred_outliers_sgd[y_pred_outliers_sgd == 1].size + + +# %% +from sklearn.inspection import DecisionBoundaryDisplay + +_, ax = plt.subplots(figsize=(9, 6)) + +xx, yy = np.meshgrid(np.linspace(-4.5, 4.5, 50), np.linspace(-4.5, 4.5, 50)) +X = np.concatenate([xx.ravel().reshape(-1, 1), yy.ravel().reshape(-1, 1)], axis=1) +DecisionBoundaryDisplay.from_estimator( + clf, + X, + response_method="decision_function", + plot_method="contourf", + ax=ax, + cmap="PuBu", +) +DecisionBoundaryDisplay.from_estimator( + clf, + X, + response_method="decision_function", + plot_method="contour", + ax=ax, + linewidths=2, + colors="darkred", + levels=[0], +) +DecisionBoundaryDisplay.from_estimator( + clf, + X, + response_method="decision_function", + plot_method="contourf", + ax=ax, + colors="palevioletred", + levels=[0, clf.decision_function(X).max()], +) + +s = 20 +b1 = plt.scatter(X_train[:, 0], X_train[:, 1], c="white", s=s, edgecolors="k") +b2 = plt.scatter(X_test[:, 0], X_test[:, 1], c="blueviolet", s=s, edgecolors="k") +c = plt.scatter(X_outliers[:, 0], X_outliers[:, 1], c="gold", s=s, edgecolors="k") + +ax.set( + title="One-Class SVM", + xlim=(-4.5, 4.5), + ylim=(-4.5, 4.5), + xlabel=( + f"error train: {n_error_train}/{X_train.shape[0]}; " + f"errors novel regular: {n_error_test}/{X_test.shape[0]}; " + f"errors novel abnormal: {n_error_outliers}/{X_outliers.shape[0]}" + ), +) +_ = ax.legend( + [mlines.Line2D([], [], color="darkred", label="learned frontier"), b1, b2, c], + [ + "learned frontier", + "training observations", + "new regular observations", + "new abnormal observations", + ], + loc="upper left", +) + +# %% +_, ax = plt.subplots(figsize=(9, 6)) + +xx, yy = np.meshgrid(np.linspace(-4.5, 4.5, 50), np.linspace(-4.5, 4.5, 50)) +X = np.concatenate([xx.ravel().reshape(-1, 1), yy.ravel().reshape(-1, 1)], axis=1) +DecisionBoundaryDisplay.from_estimator( + pipe_sgd, + X, + response_method="decision_function", + plot_method="contourf", + ax=ax, + cmap="PuBu", +) +DecisionBoundaryDisplay.from_estimator( + pipe_sgd, + X, + response_method="decision_function", + plot_method="contour", + ax=ax, + linewidths=2, + colors="darkred", + levels=[0], +) +DecisionBoundaryDisplay.from_estimator( + pipe_sgd, + X, + response_method="decision_function", + plot_method="contourf", + ax=ax, + colors="palevioletred", + levels=[0, pipe_sgd.decision_function(X).max()], +) + +s = 20 +b1 = plt.scatter(X_train[:, 0], X_train[:, 1], c="white", s=s, edgecolors="k") +b2 = plt.scatter(X_test[:, 0], X_test[:, 1], c="blueviolet", s=s, edgecolors="k") +c = plt.scatter(X_outliers[:, 0], X_outliers[:, 1], c="gold", s=s, edgecolors="k") + +ax.set( + title="Online One-Class SVM", + xlim=(-4.5, 4.5), + ylim=(-4.5, 4.5), + xlabel=( + f"error train: {n_error_train_sgd}/{X_train.shape[0]}; " + f"errors novel regular: {n_error_test_sgd}/{X_test.shape[0]}; " + f"errors novel abnormal: {n_error_outliers_sgd}/{X_outliers.shape[0]}" + ), +) +ax.legend( + [mlines.Line2D([], [], color="darkred", label="learned frontier"), b1, b2, c], + [ + "learned frontier", + "training observations", + "new regular observations", + "new abnormal observations", + ], + loc="upper left", +) +plt.show() diff --git a/examples/linear_model/plot_sparse_logistic_regression_20newsgroups.py b/examples/linear_model/plot_sparse_logistic_regression_20newsgroups.py index bab97be5acd8b..fdf914f3a7ab2 100644 --- a/examples/linear_model/plot_sparse_logistic_regression_20newsgroups.py +++ b/examples/linear_model/plot_sparse_logistic_regression_20newsgroups.py @@ -17,7 +17,12 @@ A more traditional (and possibly better) way to predict on a sparse subset of input features would be to use univariate feature selection followed by a traditional (l2-penalised) logistic regression model. + """ + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import timeit import warnings @@ -25,39 +30,39 @@ import numpy as np from sklearn.datasets import fetch_20newsgroups_vectorized +from sklearn.exceptions import ConvergenceWarning from sklearn.linear_model import LogisticRegression from sklearn.model_selection import train_test_split -from sklearn.exceptions import ConvergenceWarning - -print(__doc__) -# Author: Arthur Mensch +from sklearn.multiclass import OneVsRestClassifier -warnings.filterwarnings("ignore", category=ConvergenceWarning, - module="sklearn") +warnings.filterwarnings("ignore", category=ConvergenceWarning, module="sklearn") t0 = timeit.default_timer() # We use SAGA solver -solver = 'saga' +solver = "saga" # Turn down for faster run time -n_samples = 10000 +n_samples = 5000 -X, y = fetch_20newsgroups_vectorized(subset='all', return_X_y=True) +X, y = fetch_20newsgroups_vectorized(subset="all", return_X_y=True) X = X[:n_samples] y = y[:n_samples] -X_train, X_test, y_train, y_test = train_test_split(X, y, - random_state=42, - stratify=y, - test_size=0.1) +X_train, X_test, y_train, y_test = train_test_split( + X, y, random_state=42, stratify=y, test_size=0.1 +) train_samples, n_features = X_train.shape n_classes = np.unique(y).shape[0] -print('Dataset 20newsgroup, train_samples=%i, n_features=%i, n_classes=%i' - % (train_samples, n_features, n_classes)) +print( + "Dataset 20newsgroup, train_samples=%i, n_features=%i, n_classes=%i" + % (train_samples, n_features, n_classes) +) -models = {'ovr': {'name': 'One versus Rest', 'iters': [1, 2, 4]}, - 'multinomial': {'name': 'Multinomial', 'iters': [1, 3, 7]}} +models = { + "ovr": {"name": "One versus Rest", "iters": [1, 2, 3]}, + "multinomial": {"name": "Multinomial", "iters": [1, 2, 5]}, +} for model in models: # Add initial chance-level values for plotting purpose @@ -68,50 +73,60 @@ model_params = models[model] # Small number of epochs for fast runtime - for this_max_iter in model_params['iters']: - print('[model=%s, solver=%s] Number of epochs: %s' % - (model_params['name'], solver, this_max_iter)) - lr = LogisticRegression(solver=solver, - multi_class=model, - penalty='l1', - max_iter=this_max_iter, - random_state=42, - ) + for this_max_iter in model_params["iters"]: + print( + "[model=%s, solver=%s] Number of epochs: %s" + % (model_params["name"], solver, this_max_iter) + ) + clf = LogisticRegression( + solver=solver, + penalty="l1", + max_iter=this_max_iter, + random_state=42, + ) + if model == "ovr": + clf = OneVsRestClassifier(clf) t1 = timeit.default_timer() - lr.fit(X_train, y_train) + clf.fit(X_train, y_train) train_time = timeit.default_timer() - t1 - y_pred = lr.predict(X_test) + y_pred = clf.predict(X_test) accuracy = np.sum(y_pred == y_test) / y_test.shape[0] - density = np.mean(lr.coef_ != 0, axis=1) * 100 + if model == "ovr": + coef = np.concatenate([est.coef_ for est in clf.estimators_]) + else: + coef = clf.coef_ + density = np.mean(coef != 0, axis=1) * 100 accuracies.append(accuracy) densities.append(density) times.append(train_time) - models[model]['times'] = times - models[model]['densities'] = densities - models[model]['accuracies'] = accuracies - print('Test accuracy for model %s: %.4f' % (model, accuracies[-1])) - print('%% non-zero coefficients for model %s, ' - 'per class:\n %s' % (model, densities[-1])) - print('Run time (%i epochs) for model %s:' - '%.2f' % (model_params['iters'][-1], model, times[-1])) + models[model]["times"] = times + models[model]["densities"] = densities + models[model]["accuracies"] = accuracies + print("Test accuracy for model %s: %.4f" % (model, accuracies[-1])) + print( + "%% non-zero coefficients for model %s, per class:\n %s" + % (model, densities[-1]) + ) + print( + "Run time (%i epochs) for model %s:%.2f" + % (model_params["iters"][-1], model, times[-1]) + ) fig = plt.figure() ax = fig.add_subplot(111) for model in models: - name = models[model]['name'] - times = models[model]['times'] - accuracies = models[model]['accuracies'] - ax.plot(times, accuracies, marker='o', - label='Model: %s' % name) - ax.set_xlabel('Train time (s)') - ax.set_ylabel('Test accuracy') + name = models[model]["name"] + times = models[model]["times"] + accuracies = models[model]["accuracies"] + ax.plot(times, accuracies, marker="o", label="Model: %s" % name) + ax.set_xlabel("Train time (s)") + ax.set_ylabel("Test accuracy") ax.legend() -fig.suptitle('Multinomial vs One-vs-Rest Logistic L1\n' - 'Dataset %s' % '20newsgroups') +fig.suptitle("Multinomial vs One-vs-Rest Logistic L1\nDataset %s" % "20newsgroups") fig.tight_layout() fig.subplots_adjust(top=0.85) run_time = timeit.default_timer() - t0 -print('Example run in %.3f s' % run_time) +print("Example run in %.3f s" % run_time) plt.show() diff --git a/examples/linear_model/plot_sparse_logistic_regression_mnist.py b/examples/linear_model/plot_sparse_logistic_regression_mnist.py index 968d597a5cac7..e4a44e989b565 100644 --- a/examples/linear_model/plot_sparse_logistic_regression_mnist.py +++ b/examples/linear_model/plot_sparse_logistic_regression_mnist.py @@ -16,7 +16,12 @@ multi-layer perceptron model on this dataset. """ + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import time + import matplotlib.pyplot as plt import numpy as np @@ -26,17 +31,12 @@ from sklearn.preprocessing import StandardScaler from sklearn.utils import check_random_state -print(__doc__) - -# Author: Arthur Mensch -# License: BSD 3 clause - # Turn down for faster convergence t0 = time.time() train_samples = 5000 # Load data from https://www.openml.org/d/554 -X, y = fetch_openml('mnist_784', version=1, return_X_y=True, as_frame=False) +X, y = fetch_openml("mnist_784", version=1, return_X_y=True, as_frame=False) random_state = check_random_state(0) permutation = random_state.permutation(X.shape[0]) @@ -45,16 +45,15 @@ X = X.reshape((X.shape[0], -1)) X_train, X_test, y_train, y_test = train_test_split( - X, y, train_size=train_samples, test_size=10000) + X, y, train_size=train_samples, test_size=10000 +) scaler = StandardScaler() X_train = scaler.fit_transform(X_train) X_test = scaler.transform(X_test) # Turn up tolerance for faster convergence -clf = LogisticRegression( - C=50. / train_samples, penalty='l1', solver='saga', tol=0.1 -) +clf = LogisticRegression(C=50.0 / train_samples, penalty="l1", solver="saga", tol=0.1) clf.fit(X_train, y_train) sparsity = np.mean(clf.coef_ == 0) * 100 score = clf.score(X_test, y_test) @@ -67,13 +66,18 @@ scale = np.abs(coef).max() for i in range(10): l1_plot = plt.subplot(2, 5, i + 1) - l1_plot.imshow(coef[i].reshape(28, 28), interpolation='nearest', - cmap=plt.cm.RdBu, vmin=-scale, vmax=scale) + l1_plot.imshow( + coef[i].reshape(28, 28), + interpolation="nearest", + cmap=plt.cm.RdBu, + vmin=-scale, + vmax=scale, + ) l1_plot.set_xticks(()) l1_plot.set_yticks(()) - l1_plot.set_xlabel('Class %i' % i) -plt.suptitle('Classification vector for...') + l1_plot.set_xlabel(f"Class {i}") +plt.suptitle("Classification vector for...") run_time = time.time() - t0 -print('Example run in %.3f s' % run_time) +print("Example run in %.3f s" % run_time) plt.show() diff --git a/examples/linear_model/plot_theilsen.py b/examples/linear_model/plot_theilsen.py index c80b4a409937b..486317ffc81eb 100644 --- a/examples/linear_model/plot_theilsen.py +++ b/examples/linear_model/plot_theilsen.py @@ -32,56 +32,65 @@ therefore also limits the runtime. Therefore, Theil-Sen is applicable to larger problems with the drawback of losing some of its mathematical properties since it then works on a random subset. + """ -# Author: Florian Wilhelm -- -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import time -import numpy as np + import matplotlib.pyplot as plt -from sklearn.linear_model import LinearRegression, TheilSenRegressor -from sklearn.linear_model import RANSACRegressor +import numpy as np -print(__doc__) +from sklearn.linear_model import LinearRegression, RANSACRegressor, TheilSenRegressor -estimators = [('OLS', LinearRegression()), - ('Theil-Sen', TheilSenRegressor(random_state=42)), - ('RANSAC', RANSACRegressor(random_state=42)), ] -colors = {'OLS': 'turquoise', 'Theil-Sen': 'gold', 'RANSAC': 'lightgreen'} +estimators = [ + ("OLS", LinearRegression()), + ("Theil-Sen", TheilSenRegressor(random_state=42)), + ("RANSAC", RANSACRegressor(random_state=42)), +] +colors = {"OLS": "turquoise", "Theil-Sen": "gold", "RANSAC": "lightgreen"} lw = 2 -# ############################################################################# +# %% # Outliers only in the y direction +# -------------------------------- np.random.seed(0) n_samples = 200 # Linear model y = 3*x + N(2, 0.1**2) x = np.random.randn(n_samples) -w = 3. -c = 2. +w = 3.0 +c = 2.0 noise = 0.1 * np.random.randn(n_samples) y = w * x + c + noise # 10% outliers y[-20:] += -20 * x[-20:] X = x[:, np.newaxis] -plt.scatter(x, y, color='indigo', marker='x', s=40) +plt.scatter(x, y, color="indigo", marker="x", s=40) line_x = np.array([-3, 3]) for name, estimator in estimators: t0 = time.time() estimator.fit(X, y) elapsed_time = time.time() - t0 y_pred = estimator.predict(line_x.reshape(2, 1)) - plt.plot(line_x, y_pred, color=colors[name], linewidth=lw, - label='%s (fit time: %.2fs)' % (name, elapsed_time)) - -plt.axis('tight') -plt.legend(loc='upper left') -plt.title("Corrupt y") - -# ############################################################################# + plt.plot( + line_x, + y_pred, + color=colors[name], + linewidth=lw, + label="%s (fit time: %.2fs)" % (name, elapsed_time), + ) + +plt.axis("tight") +plt.legend(loc="upper right") +_ = plt.title("Corrupt y") + +# %% # Outliers in the X direction +# --------------------------- np.random.seed(0) # Linear model y = 3*x + N(2, 0.1**2) @@ -94,7 +103,7 @@ X = x[:, np.newaxis] plt.figure() -plt.scatter(x, y, color='indigo', marker='x', s=40) +plt.scatter(x, y, color="indigo", marker="x", s=40) line_x = np.array([-3, 10]) for name, estimator in estimators: @@ -102,10 +111,15 @@ estimator.fit(X, y) elapsed_time = time.time() - t0 y_pred = estimator.predict(line_x.reshape(2, 1)) - plt.plot(line_x, y_pred, color=colors[name], linewidth=lw, - label='%s (fit time: %.2fs)' % (name, elapsed_time)) - -plt.axis('tight') -plt.legend(loc='upper left') + plt.plot( + line_x, + y_pred, + color=colors[name], + linewidth=lw, + label="%s (fit time: %.2fs)" % (name, elapsed_time), + ) + +plt.axis("tight") +plt.legend(loc="upper left") plt.title("Corrupt x") plt.show() diff --git a/examples/linear_model/plot_tweedie_regression_insurance_claims.py b/examples/linear_model/plot_tweedie_regression_insurance_claims.py index 35ced1a58983c..ea2365a71d48a 100644 --- a/examples/linear_model/plot_tweedie_regression_insurance_claims.py +++ b/examples/linear_model/plot_tweedie_regression_insurance_claims.py @@ -1,3 +1,6 @@ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + """ ====================================== Tweedie regression on insurance claims @@ -34,65 +37,65 @@ .. [1] A. Noll, R. Salzmann and M.V. Wuthrich, Case Study: French Motor Third-Party Liability Claims (November 8, 2018). `doi:10.2139/ssrn.3164764 - `_ - + `_ """ -print(__doc__) -# Authors: Christian Lorentzen -# Roman Yurchak -# Olivier Grisel -# License: BSD 3 clause +# %% + from functools import partial -import numpy as np import matplotlib.pyplot as plt +import numpy as np import pandas as pd from sklearn.datasets import fetch_openml -from sklearn.compose import ColumnTransformer -from sklearn.linear_model import PoissonRegressor, GammaRegressor -from sklearn.linear_model import TweedieRegressor -from sklearn.metrics import mean_tweedie_deviance -from sklearn.model_selection import train_test_split -from sklearn.pipeline import make_pipeline -from sklearn.preprocessing import FunctionTransformer, OneHotEncoder -from sklearn.preprocessing import StandardScaler, KBinsDiscretizer - -from sklearn.metrics import mean_absolute_error, mean_squared_error, auc +from sklearn.metrics import ( + mean_absolute_error, + mean_squared_error, + mean_tweedie_deviance, +) -def load_mtpl2(n_samples=100000): +def load_mtpl2(n_samples=None): """Fetch the French Motor Third-Party Liability Claims dataset. Parameters ---------- - n_samples: int, default=100000 + n_samples: int, default=None number of samples to select (for faster run time). Full dataset has 678013 samples. """ # freMTPL2freq dataset from https://www.openml.org/d/41214 - df_freq = fetch_openml(data_id=41214, as_frame=True)['data'] - df_freq['IDpol'] = df_freq['IDpol'].astype(int) - df_freq.set_index('IDpol', inplace=True) + df_freq = fetch_openml(data_id=41214, as_frame=True).data + df_freq["IDpol"] = df_freq["IDpol"].astype(int) + df_freq.set_index("IDpol", inplace=True) # freMTPL2sev dataset from https://www.openml.org/d/41215 - df_sev = fetch_openml(data_id=41215, as_frame=True)['data'] + df_sev = fetch_openml(data_id=41215, as_frame=True).data # sum ClaimAmount over identical IDs - df_sev = df_sev.groupby('IDpol').sum() + df_sev = df_sev.groupby("IDpol").sum() df = df_freq.join(df_sev, how="left") - df["ClaimAmount"].fillna(0, inplace=True) + df["ClaimAmount"] = df["ClaimAmount"].fillna(0) # unquote string fields - for column_name in df.columns[df.dtypes.values == object]: + for column_name in df.columns[[t is object for t in df.dtypes.values]]: df[column_name] = df[column_name].str.strip("'") return df.iloc[:n_samples] -def plot_obs_pred(df, feature, weight, observed, predicted, y_label=None, - title=None, ax=None, fill_legend=False): +def plot_obs_pred( + df, + feature, + weight, + observed, + predicted, + y_label=None, + title=None, + ax=None, + fill_legend=False, +): """Plot observed and predicted - aggregated per feature level. Parameters @@ -139,21 +142,30 @@ def plot_obs_pred(df, feature, weight, observed, predicted, y_label=None, def score_estimator( - estimator, X_train, X_test, df_train, df_test, target, weights, + estimator, + X_train, + X_test, + df_train, + df_test, + target, + weights, tweedie_powers=None, ): """Evaluate an estimator on train and test sets with different metrics""" metrics = [ - ("D² explained", None), # Use default scorer if it exists + ("D² explained", None), # Use default scorer if it exists ("mean abs. error", mean_absolute_error), ("mean squared error", mean_squared_error), ] if tweedie_powers: - metrics += [( - "mean Tweedie dev p={:.4f}".format(power), - partial(mean_tweedie_deviance, power=power) - ) for power in tweedie_powers] + metrics += [ + ( + "mean Tweedie dev p={:.4f}".format(power), + partial(mean_tweedie_deviance, power=power), + ) + for power in tweedie_powers + ] res = [] for subset_label, X, df in [ @@ -177,16 +189,14 @@ def score_estimator( else: score = metric(y, y_pred, sample_weight=_weights) - res.append( - {"subset": subset_label, "metric": score_label, "score": score} - ) + res.append({"subset": subset_label, "metric": score_label, "score": score}) res = ( pd.DataFrame(res) .set_index(["metric", "subset"]) .score.unstack(-1) .round(4) - .loc[:, ['train', 'test']] + .loc[:, ["train", "test"]] ) return res @@ -199,34 +209,48 @@ def score_estimator( # containing the number of claims (``ClaimNb``), with the freMTPL2sev table, # containing the claim amount (``ClaimAmount``) for the same policy ids # (``IDpol``). +from sklearn.compose import ColumnTransformer +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import ( + FunctionTransformer, + KBinsDiscretizer, + OneHotEncoder, + StandardScaler, +) -df = load_mtpl2(n_samples=60000) +df = load_mtpl2() -# Note: filter out claims with zero amount, as the severity model -# requires strictly positive target values. -df.loc[(df["ClaimAmount"] == 0) & (df["ClaimNb"] >= 1), "ClaimNb"] = 0 # Correct for unreasonable observations (that might be data error) # and a few exceptionally large claim amounts df["ClaimNb"] = df["ClaimNb"].clip(upper=4) df["Exposure"] = df["Exposure"].clip(upper=1) df["ClaimAmount"] = df["ClaimAmount"].clip(upper=200000) +# If the claim amount is 0, then we do not count it as a claim. The loss function +# used by the severity model needs strictly positive claim amounts. This way +# frequency and severity are more consistent with each other. +df.loc[(df["ClaimAmount"] == 0) & (df["ClaimNb"] >= 1), "ClaimNb"] = 0 log_scale_transformer = make_pipeline( - FunctionTransformer(func=np.log), - StandardScaler() + FunctionTransformer(func=np.log), StandardScaler() ) column_trans = ColumnTransformer( [ - ("binned_numeric", KBinsDiscretizer(n_bins=10), - ["VehAge", "DrivAge"]), - ("onehot_categorical", OneHotEncoder(), - ["VehBrand", "VehPower", "VehGas", "Region", "Area"]), - ("passthrough_numeric", "passthrough", - ["BonusMalus"]), - ("log_scaled_numeric", log_scale_transformer, - ["Density"]), + ( + "binned_numeric", + KBinsDiscretizer( + n_bins=10, quantile_method="averaged_inverted_cdf", random_state=0 + ), + ["VehAge", "DrivAge"], + ), + ( + "onehot_categorical", + OneHotEncoder(), + ["VehBrand", "VehPower", "VehGas", "Region", "Area"], + ), + ("passthrough_numeric", "passthrough", ["BonusMalus"]), + ("log_scaled_numeric", log_scale_transformer, ["Density"]), ], remainder="drop", ) @@ -252,19 +276,36 @@ def score_estimator( # # The number of claims (``ClaimNb``) is a positive integer (0 included). # Thus, this target can be modelled by a Poisson distribution. -# It is then assumed to be the number of discrete events occuring with a +# It is then assumed to be the number of discrete events occurring with a # constant rate in a given time interval (``Exposure``, in units of years). # Here we model the frequency ``y = ClaimNb / Exposure``, which is still a # (scaled) Poisson distribution, and use ``Exposure`` as `sample_weight`. +from sklearn.linear_model import PoissonRegressor +from sklearn.model_selection import train_test_split df_train, df_test, X_train, X_test = train_test_split(df, X, random_state=0) +# %% +# +# Let us keep in mind that despite the seemingly large number of data points in +# this dataset, the number of evaluation points where the claim amount is +# non-zero is quite small: +len(df_test) + +# %% +len(df_test[df_test["ClaimAmount"] > 0]) + +# %% +# +# As a consequence, we expect a significant variability in our +# evaluation upon random resampling of the train test split. +# # The parameters of the model are estimated by minimizing the Poisson deviance -# on the training set via a quasi-Newton solver: l-BFGS. Some of the features -# are collinear, we use a weak penalization to avoid numerical issues. -glm_freq = PoissonRegressor(alpha=1e-3, max_iter=400) -glm_freq.fit(X_train, df_train["Frequency"], - sample_weight=df_train["Exposure"]) +# on the training set via a Newton solver. Some of the features are collinear +# (e.g. because we did not drop any categorical level in the `OneHotEncoder`), +# we use a weak L2 penalization to avoid numerical issues. +glm_freq = PoissonRegressor(alpha=1e-4, solver="newton-cholesky") +glm_freq.fit(X_train, df_train["Frequency"], sample_weight=df_train["Exposure"]) scores = score_estimator( glm_freq, @@ -279,6 +320,12 @@ def score_estimator( print(scores) # %% +# +# Note that the score measured on the test set is surprisingly better than on +# the training set. This might be specific to this random train-test split. +# Proper cross-validation could help us to assess the sampling variability of +# these results. +# # We can visually compare observed and predicted values, aggregated by the # drivers age (``DrivAge``), vehicle age (``VehAge``) and the insurance # bonus/malus (``BonusMalus``). @@ -306,7 +353,7 @@ def score_estimator( y_label="Claim Frequency", title="test data", ax=ax[0, 1], - fill_legend=True + fill_legend=True, ) plot_obs_pred( @@ -318,7 +365,7 @@ def score_estimator( y_label="Claim Frequency", title="test data", ax=ax[1, 0], - fill_legend=True + fill_legend=True, ) plot_obs_pred( @@ -330,7 +377,7 @@ def score_estimator( y_label="Claim Frequency", title="test data", ax=ax[1, 1], - fill_legend=True + fill_legend=True, ) @@ -352,11 +399,12 @@ def score_estimator( # on :math:`(0, \infty)`, not :math:`[0, \infty)`. # - We use ``ClaimNb`` as `sample_weight` to account for policies that contain # more than one claim. +from sklearn.linear_model import GammaRegressor mask_train = df_train["ClaimAmount"] > 0 mask_test = df_test["ClaimAmount"] > 0 -glm_sev = GammaRegressor(alpha=10., max_iter=10000) +glm_sev = GammaRegressor(alpha=10.0, solver="newton-cholesky") glm_sev.fit( X_train[mask_train.values], @@ -377,21 +425,61 @@ def score_estimator( print(scores) # %% -# Here, the scores for the test data call for caution as they are -# significantly worse than for the training data indicating an overfit despite -# the strong regularization. # -# Note that the resulting model is the average claim amount per claim. As -# such, it is conditional on having at least one claim, and cannot be used to -# predict the average claim amount per policy in general. +# Those values of the metrics are not necessarily easy to interpret. It can be +# insightful to compare them with a model that does not use any input +# features and always predicts a constant value, i.e. the average claim +# amount, in the same setting: -print("Mean AvgClaim Amount per policy: %.2f " - % df_train["AvgClaimAmount"].mean()) -print("Mean AvgClaim Amount | NbClaim > 0: %.2f" - % df_train["AvgClaimAmount"][df_train["AvgClaimAmount"] > 0].mean()) -print("Predicted Mean AvgClaim Amount | NbClaim > 0: %.2f" - % glm_sev.predict(X_train).mean()) +from sklearn.dummy import DummyRegressor +dummy_sev = DummyRegressor(strategy="mean") +dummy_sev.fit( + X_train[mask_train.values], + df_train.loc[mask_train, "AvgClaimAmount"], + sample_weight=df_train.loc[mask_train, "ClaimNb"], +) + +scores = score_estimator( + dummy_sev, + X_train[mask_train.values], + X_test[mask_test.values], + df_train[mask_train], + df_test[mask_test], + target="AvgClaimAmount", + weights="ClaimNb", +) +print("Evaluation of a mean predictor on target AvgClaimAmount") +print(scores) + +# %% +# +# We conclude that the claim amount is very challenging to predict. Still, the +# :class:`~sklearn.linear_model.GammaRegressor` is able to leverage some +# information from the input features to slightly improve upon the mean +# baseline in terms of D². +# +# Note that the resulting model is the average claim amount per claim. As such, +# it is conditional on having at least one claim, and cannot be used to predict +# the average claim amount per policy. For this, it needs to be combined with +# a claims frequency model. + +print( + "Mean AvgClaim Amount per policy: %.2f " + % df_train["AvgClaimAmount"].mean() +) +print( + "Mean AvgClaim Amount | NbClaim > 0: %.2f" + % df_train["AvgClaimAmount"][df_train["AvgClaimAmount"] > 0].mean() +) +print( + "Predicted Mean AvgClaim Amount | NbClaim > 0: %.2f" + % glm_sev.predict(X_train).mean() +) +print( + "Predicted Mean AvgClaim Amount (dummy) | NbClaim > 0: %.2f" + % dummy_sev.predict(X_train).mean() +) # %% # We can visually compare observed and predicted values, aggregated for @@ -419,7 +507,7 @@ def score_estimator( y_label="Average Claim Severity", title="test data", ax=ax[1], - fill_legend=True + fill_legend=True, ) plt.tight_layout() @@ -454,10 +542,12 @@ def score_estimator( # models side by side, i.e. we compare them at identical values of `power`. # Ideally, we hope that one model will be consistently better than the other, # regardless of `power`. +from sklearn.linear_model import TweedieRegressor -glm_pure_premium = TweedieRegressor(power=1.9, alpha=.1, max_iter=10000) -glm_pure_premium.fit(X_train, df_train["PurePremium"], - sample_weight=df_train["Exposure"]) +glm_pure_premium = TweedieRegressor(power=1.9, alpha=0.1, solver="newton-cholesky") +glm_pure_premium.fit( + X_train, df_train["PurePremium"], sample_weight=df_train["Exposure"] +) tweedie_powers = [1.5, 1.7, 1.8, 1.9, 1.99, 1.999, 1.9999] @@ -480,15 +570,17 @@ def score_estimator( df_test, target="PurePremium", weights="Exposure", - tweedie_powers=tweedie_powers + tweedie_powers=tweedie_powers, ) -scores = pd.concat([scores_product_model, scores_glm_pure_premium], - axis=1, sort=True, - keys=('Product Model', 'TweedieRegressor')) -print("Evaluation of the Product Model and the Tweedie Regressor " - "on target PurePremium") -with pd.option_context('display.expand_frame_repr', False): +scores = pd.concat( + [scores_product_model, scores_glm_pure_premium], + axis=1, + sort=True, + keys=("Product Model", "TweedieRegressor"), +) +print("Evaluation of the Product Model and the Tweedie Regressor on target PurePremium") +with pd.option_context("display.expand_frame_repr", False): print(scores) # %% @@ -514,38 +606,46 @@ def score_estimator( "predicted, frequency*severity model": np.sum( exposure * glm_freq.predict(X) * glm_sev.predict(X) ), - "predicted, tweedie, power=%.2f" - % glm_pure_premium.power: np.sum( - exposure * glm_pure_premium.predict(X)), + "predicted, tweedie, power=%.2f" % glm_pure_premium.power: np.sum( + exposure * glm_pure_premium.predict(X) + ), } ) print(pd.DataFrame(res).set_index("subset").T) # %% -# Finally, we can compare the two models using a plot of cumulated claims: for -# each model, the policyholders are ranked from safest to riskiest and the -# fraction of observed total cumulated claims is plotted on the y axis. This -# plot is often called the ordered Lorenz curve of the model. # -# The Gini coefficient (based on the area under the curve) can be used as a -# model selection metric to quantify the ability of the model to rank -# policyholders. Note that this metric does not reflect the ability of the -# models to make accurate predictions in terms of absolute value of total -# claim amounts but only in terms of relative amounts as a ranking metric. +# Finally, we can compare the two models using a plot of cumulative claims: for +# each model, the policyholders are ranked from safest to riskiest based on the +# model predictions and the cumulative proportion of claim amounts is plotted +# against the cumulative proportion of exposure. This plot is often called +# the ordered Lorenz curve of the model. # -# Both models are able to rank policyholders by risky-ness significantly -# better than chance although they are also both far from perfect due to the -# natural difficulty of the prediction problem from few features. +# The Gini coefficient (based on the area between the curve and the diagonal) +# can be used as a model selection metric to quantify the ability of the model +# to rank policyholders. Note that this metric does not reflect the ability of +# the models to make accurate predictions in terms of absolute value of total +# claim amounts but only in terms of relative amounts as a ranking metric. The +# Gini coefficient is upper bounded by 1.0 but even an oracle model that ranks +# the policyholders by the observed claim amounts cannot reach a score of 1.0. # -# Note that the Gini index only characterize the ranking performance of the -# model but not its calibration: any monotonic transformation of the -# predictions leaves the Gini index of the model unchanged. +# We observe that both models are able to rank policyholders by riskiness +# significantly better than chance although they are also both far from the +# oracle model due to the natural difficulty of the prediction problem from a +# few features: most accidents are not predictable and can be caused by +# environmental circumstances that are not described at all by the input +# features of the models. # -# Finally one should highlight that the Compound Poisson Gamma model that -# is directly fit on the pure premium is operationally simpler to develop and -# maintain as it consists in a single scikit-learn estimator instead of a -# pair of models, each with its own set of hyperparameters. +# Note that the Gini index only characterizes the ranking performance of the +# model but not its calibration: any monotonic transformation of the predictions +# leaves the Gini index of the model unchanged. +# +# Finally one should highlight that the Compound Poisson Gamma model that is +# directly fit on the pure premium is operationally simpler to develop and +# maintain as it consists of a single scikit-learn estimator instead of a pair +# of models, each with its own set of hyperparameters. +from sklearn.metrics import auc def lorenz_curve(y_true, y_pred, exposure): @@ -556,10 +656,11 @@ def lorenz_curve(y_true, y_pred, exposure): ranking = np.argsort(y_pred) ranked_exposure = exposure[ranking] ranked_pure_premium = y_true[ranking] - cumulated_claim_amount = np.cumsum(ranked_pure_premium * ranked_exposure) - cumulated_claim_amount /= cumulated_claim_amount[-1] - cumulated_samples = np.linspace(0, 1, len(cumulated_claim_amount)) - return cumulated_samples, cumulated_claim_amount + cumulative_claim_amount = np.cumsum(ranked_pure_premium * ranked_exposure) + cumulative_claim_amount /= cumulative_claim_amount[-1] + cumulative_exposure = np.cumsum(ranked_exposure) + cumulative_exposure /= cumulative_exposure[-1] + return cumulative_exposure, cumulative_claim_amount fig, ax = plt.subplots(figsize=(8, 8)) @@ -567,30 +668,33 @@ def lorenz_curve(y_true, y_pred, exposure): y_pred_product = glm_freq.predict(X_test) * glm_sev.predict(X_test) y_pred_total = glm_pure_premium.predict(X_test) -for label, y_pred in [("Frequency * Severity model", y_pred_product), - ("Compound Poisson Gamma", y_pred_total)]: - ordered_samples, cum_claims = lorenz_curve( - df_test["PurePremium"], y_pred, df_test["Exposure"]) - gini = 1 - 2 * auc(ordered_samples, cum_claims) +for label, y_pred in [ + ("Frequency * Severity model", y_pred_product), + ("Compound Poisson Gamma", y_pred_total), +]: + cum_exposure, cum_claims = lorenz_curve( + df_test["PurePremium"], y_pred, df_test["Exposure"] + ) + gini = 1 - 2 * auc(cum_exposure, cum_claims) label += " (Gini index: {:.3f})".format(gini) - ax.plot(ordered_samples, cum_claims, linestyle="-", label=label) + ax.plot(cum_exposure, cum_claims, linestyle="-", label=label) # Oracle model: y_pred == y_test -ordered_samples, cum_claims = lorenz_curve( - df_test["PurePremium"], df_test["PurePremium"], df_test["Exposure"]) -gini = 1 - 2 * auc(ordered_samples, cum_claims) +cum_exposure, cum_claims = lorenz_curve( + df_test["PurePremium"], df_test["PurePremium"], df_test["Exposure"] +) +gini = 1 - 2 * auc(cum_exposure, cum_claims) label = "Oracle (Gini index: {:.3f})".format(gini) -ax.plot(ordered_samples, cum_claims, linestyle="-.", color="gray", - label=label) +ax.plot(cum_exposure, cum_claims, linestyle="-.", color="gray", label=label) # Random baseline -ax.plot([0, 1], [0, 1], linestyle="--", color="black", - label="Random baseline") +ax.plot([0, 1], [0, 1], linestyle="--", color="black", label="Random baseline") ax.set( title="Lorenz Curves", - xlabel=('Fraction of policyholders\n' - '(ordered by model from safest to riskiest)'), - ylabel='Fraction of total claim amount' + xlabel=( + "Cumulative proportion of exposure\n(ordered by model from safest to riskiest)" + ), + ylabel="Cumulative proportion of claim amounts", ) ax.legend(loc="upper left") plt.plot() diff --git a/examples/manifold/README.txt b/examples/manifold/README.txt index bf12be84b21ab..7a62a67150b69 100644 --- a/examples/manifold/README.txt +++ b/examples/manifold/README.txt @@ -4,4 +4,3 @@ Manifold learning ----------------------- Examples concerning the :mod:`sklearn.manifold` module. - diff --git a/examples/manifold/plot_compare_methods.py b/examples/manifold/plot_compare_methods.py index ed01e8ac19b89..6203a4afc436d 100644 --- a/examples/manifold/plot_compare_methods.py +++ b/examples/manifold/plot_compare_methods.py @@ -17,67 +17,197 @@ the distances in the original high-dimensional space, unlike other manifold-learning algorithms, it does not seeks an isotropic representation of the data in the low-dimensional space. -""" -# Author: Jake Vanderplas -- +""" -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -from collections import OrderedDict -from functools import partial -from time import time +# %% +# Dataset preparation +# ------------------- +# +# We start by generating the S-curve dataset. import matplotlib.pyplot as plt -from mpl_toolkits.mplot3d import Axes3D -from matplotlib.ticker import NullFormatter - -from sklearn import manifold, datasets - -# Next line to silence pyflakes. This import is needed. -Axes3D - -n_points = 1000 -X, color = datasets.make_s_curve(n_points, random_state=0) -n_neighbors = 10 -n_components = 2 - -# Create figure -fig = plt.figure(figsize=(15, 8)) -fig.suptitle("Manifold Learning with %i points, %i neighbors" - % (1000, n_neighbors), fontsize=14) - -# Add 3d scatter plot -ax = fig.add_subplot(251, projection='3d') -ax.scatter(X[:, 0], X[:, 1], X[:, 2], c=color, cmap=plt.cm.Spectral) -ax.view_init(4, -72) - -# Set-up manifold methods -LLE = partial(manifold.LocallyLinearEmbedding, - n_neighbors, n_components, eigen_solver='auto') - -methods = OrderedDict() -methods['LLE'] = LLE(method='standard') -methods['LTSA'] = LLE(method='ltsa') -methods['Hessian LLE'] = LLE(method='hessian') -methods['Modified LLE'] = LLE(method='modified') -methods['Isomap'] = manifold.Isomap(n_neighbors, n_components) -methods['MDS'] = manifold.MDS(n_components, max_iter=100, n_init=1) -methods['SE'] = manifold.SpectralEmbedding(n_components=n_components, - n_neighbors=n_neighbors) -methods['t-SNE'] = manifold.TSNE(n_components=n_components, init='pca', - random_state=0) - -# Plot results -for i, (label, method) in enumerate(methods.items()): - t0 = time() - Y = method.fit_transform(X) - t1 = time() - print("%s: %.2g sec" % (label, t1 - t0)) - ax = fig.add_subplot(2, 5, 2 + i + (i > 3)) - ax.scatter(Y[:, 0], Y[:, 1], c=color, cmap=plt.cm.Spectral) - ax.set_title("%s (%.2g sec)" % (label, t1 - t0)) - ax.xaxis.set_major_formatter(NullFormatter()) - ax.yaxis.set_major_formatter(NullFormatter()) - ax.axis('tight') + +# unused but required import for doing 3d projections with matplotlib < 3.2 +import mpl_toolkits.mplot3d # noqa: F401 +from matplotlib import ticker + +from sklearn import datasets, manifold + +n_samples = 1500 +S_points, S_color = datasets.make_s_curve(n_samples, random_state=0) + +# %% +# Let's look at the original data. Also define some helping +# functions, which we will use further on. + + +def plot_3d(points, points_color, title): + x, y, z = points.T + + fig, ax = plt.subplots( + figsize=(6, 6), + facecolor="white", + tight_layout=True, + subplot_kw={"projection": "3d"}, + ) + fig.suptitle(title, size=16) + col = ax.scatter(x, y, z, c=points_color, s=50, alpha=0.8) + ax.view_init(azim=-60, elev=9) + ax.xaxis.set_major_locator(ticker.MultipleLocator(1)) + ax.yaxis.set_major_locator(ticker.MultipleLocator(1)) + ax.zaxis.set_major_locator(ticker.MultipleLocator(1)) + + fig.colorbar(col, ax=ax, orientation="horizontal", shrink=0.6, aspect=60, pad=0.01) + plt.show() + + +def plot_2d(points, points_color, title): + fig, ax = plt.subplots(figsize=(3, 3), facecolor="white", constrained_layout=True) + fig.suptitle(title, size=16) + add_2d_scatter(ax, points, points_color) + plt.show() + + +def add_2d_scatter(ax, points, points_color, title=None): + x, y = points.T + ax.scatter(x, y, c=points_color, s=50, alpha=0.8) + ax.set_title(title) + ax.xaxis.set_major_formatter(ticker.NullFormatter()) + ax.yaxis.set_major_formatter(ticker.NullFormatter()) + + +plot_3d(S_points, S_color, "Original S-curve samples") + +# %% +# Define algorithms for the manifold learning +# ------------------------------------------- +# +# Manifold learning is an approach to non-linear dimensionality reduction. +# Algorithms for this task are based on the idea that the dimensionality of +# many data sets is only artificially high. +# +# Read more in the :ref:`User Guide `. + +n_neighbors = 12 # neighborhood which is used to recover the locally linear structure +n_components = 2 # number of coordinates for the manifold + +# %% +# Locally Linear Embeddings +# ^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# Locally linear embedding (LLE) can be thought of as a series of local +# Principal Component Analyses which are globally compared to find the +# best non-linear embedding. +# Read more in the :ref:`User Guide `. + +params = { + "n_neighbors": n_neighbors, + "n_components": n_components, + "eigen_solver": "auto", + "random_state": 0, +} + +lle_standard = manifold.LocallyLinearEmbedding(method="standard", **params) +S_standard = lle_standard.fit_transform(S_points) + +lle_ltsa = manifold.LocallyLinearEmbedding(method="ltsa", **params) +S_ltsa = lle_ltsa.fit_transform(S_points) + +lle_hessian = manifold.LocallyLinearEmbedding(method="hessian", **params) +S_hessian = lle_hessian.fit_transform(S_points) + +lle_mod = manifold.LocallyLinearEmbedding(method="modified", **params) +S_mod = lle_mod.fit_transform(S_points) + +# %% +fig, axs = plt.subplots( + nrows=2, ncols=2, figsize=(7, 7), facecolor="white", constrained_layout=True +) +fig.suptitle("Locally Linear Embeddings", size=16) + +lle_methods = [ + ("Standard locally linear embedding", S_standard), + ("Local tangent space alignment", S_ltsa), + ("Hessian eigenmap", S_hessian), + ("Modified locally linear embedding", S_mod), +] +for ax, method in zip(axs.flat, lle_methods): + name, points = method + add_2d_scatter(ax, points, S_color, name) plt.show() + +# %% +# Isomap Embedding +# ^^^^^^^^^^^^^^^^ +# +# Non-linear dimensionality reduction through Isometric Mapping. +# Isomap seeks a lower-dimensional embedding which maintains geodesic +# distances between all points. Read more in the :ref:`User Guide `. + +isomap = manifold.Isomap(n_neighbors=n_neighbors, n_components=n_components, p=1) +S_isomap = isomap.fit_transform(S_points) + +plot_2d(S_isomap, S_color, "Isomap Embedding") + +# %% +# Multidimensional scaling +# ^^^^^^^^^^^^^^^^^^^^^^^^ +# +# Multidimensional scaling (MDS) seeks a low-dimensional representation +# of the data in which the distances respect well the distances in the +# original high-dimensional space. +# Read more in the :ref:`User Guide `. + +md_scaling = manifold.MDS( + n_components=n_components, + max_iter=50, + n_init=1, + random_state=0, + normalized_stress=False, +) +S_scaling = md_scaling.fit_transform(S_points) + +plot_2d(S_scaling, S_color, "Multidimensional scaling") + +# %% +# Spectral embedding for non-linear dimensionality reduction +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# This implementation uses Laplacian Eigenmaps, which finds a low dimensional +# representation of the data using a spectral decomposition of the graph Laplacian. +# Read more in the :ref:`User Guide `. + +spectral = manifold.SpectralEmbedding( + n_components=n_components, n_neighbors=n_neighbors, random_state=42 +) +S_spectral = spectral.fit_transform(S_points) + +plot_2d(S_spectral, S_color, "Spectral Embedding") + +# %% +# T-distributed Stochastic Neighbor Embedding +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# It converts similarities between data points to joint probabilities and +# tries to minimize the Kullback-Leibler divergence between the joint probabilities +# of the low-dimensional embedding and the high-dimensional data. t-SNE has a cost +# function that is not convex, i.e. with different initializations we can get +# different results. Read more in the :ref:`User Guide `. + +t_sne = manifold.TSNE( + n_components=n_components, + perplexity=30, + init="random", + max_iter=250, + random_state=0, +) +S_t_sne = t_sne.fit_transform(S_points) + +plot_2d(S_t_sne, S_color, "T-distributed Stochastic \n Neighbor Embedding") + +# %% diff --git a/examples/manifold/plot_lle_digits.py b/examples/manifold/plot_lle_digits.py index a01c7b01f988b..d53816536158f 100644 --- a/examples/manifold/plot_lle_digits.py +++ b/examples/manifold/plot_lle_digits.py @@ -3,246 +3,176 @@ Manifold learning on handwritten digits: Locally Linear Embedding, Isomap... ============================================================================= -An illustration of various embeddings on the digits dataset. - -The RandomTreesEmbedding, from the :mod:`sklearn.ensemble` module, is not -technically a manifold embedding method, as it learn a high-dimensional -representation on which we apply a dimensionality reduction method. -However, it is often useful to cast a dataset into a representation in -which the classes are linearly-separable. - -t-SNE will be initialized with the embedding that is generated by PCA in -this example, which is not the default setting. It ensures global stability -of the embedding, i.e., the embedding does not depend on random -initialization. - -Linear Discriminant Analysis, from the :mod:`sklearn.discriminant_analysis` -module, and Neighborhood Components Analysis, from the :mod:`sklearn.neighbors` -module, are supervised dimensionality reduction method, i.e. they make use of -the provided labels, contrary to other methods. +We illustrate various embedding techniques on the digits dataset. + """ -# Authors: Fabian Pedregosa -# Olivier Grisel -# Mathieu Blondel -# Gael Varoquaux -# License: BSD 3 clause (C) INRIA 2011 +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -from time import time -import numpy as np -import matplotlib.pyplot as plt -from matplotlib import offsetbox -from sklearn import (manifold, datasets, decomposition, ensemble, - discriminant_analysis, random_projection, neighbors) -print(__doc__) +# %% +# Load digits dataset +# ------------------- +# We will load the digits dataset and only use six first of the ten available classes. +from sklearn.datasets import load_digits -digits = datasets.load_digits(n_class=6) -X = digits.data -y = digits.target +digits = load_digits(n_class=6) +X, y = digits.data, digits.target n_samples, n_features = X.shape n_neighbors = 30 +# %% +# We can plot the first hundred digits from this data set. +import matplotlib.pyplot as plt + +fig, axs = plt.subplots(nrows=10, ncols=10, figsize=(6, 6)) +for idx, ax in enumerate(axs.ravel()): + ax.imshow(X[idx].reshape((8, 8)), cmap=plt.cm.binary) + ax.axis("off") +_ = fig.suptitle("A selection from the 64-dimensional digits dataset", fontsize=16) + +# %% +# Helper function to plot embedding +# --------------------------------- +# Below, we will use different techniques to embed the digits dataset. We will plot +# the projection of the original data onto each embedding. It will allow us to +# check whether or digits are grouped together in the embedding space, or +# scattered across it. +import numpy as np +from matplotlib import offsetbox + +from sklearn.preprocessing import MinMaxScaler -# ---------------------------------------------------------------------- -# Scale and visualize the embedding vectors -def plot_embedding(X, title=None): - x_min, x_max = np.min(X, 0), np.max(X, 0) - X = (X - x_min) / (x_max - x_min) - plt.figure() - ax = plt.subplot(111) +def plot_embedding(X, title): + _, ax = plt.subplots() + X = MinMaxScaler().fit_transform(X) + + for digit in digits.target_names: + ax.scatter( + *X[y == digit].T, + marker=f"${digit}$", + s=60, + color=plt.cm.Dark2(digit), + alpha=0.425, + zorder=2, + ) + shown_images = np.array([[1.0, 1.0]]) # just something big for i in range(X.shape[0]): - plt.text(X[i, 0], X[i, 1], str(y[i]), - color=plt.cm.Set1(y[i] / 10.), - fontdict={'weight': 'bold', 'size': 9}) - - if hasattr(offsetbox, 'AnnotationBbox'): - # only print thumbnails with matplotlib > 1.0 - shown_images = np.array([[1., 1.]]) # just something big - for i in range(X.shape[0]): - dist = np.sum((X[i] - shown_images) ** 2, 1) - if np.min(dist) < 4e-3: - # don't show points that are too close - continue - shown_images = np.r_[shown_images, [X[i]]] - imagebox = offsetbox.AnnotationBbox( - offsetbox.OffsetImage(digits.images[i], cmap=plt.cm.gray_r), - X[i]) - ax.add_artist(imagebox) - plt.xticks([]), plt.yticks([]) - if title is not None: - plt.title(title) - - -# ---------------------------------------------------------------------- -# Plot images of the digits -n_img_per_row = 20 -img = np.zeros((10 * n_img_per_row, 10 * n_img_per_row)) -for i in range(n_img_per_row): - ix = 10 * i + 1 - for j in range(n_img_per_row): - iy = 10 * j + 1 - img[ix:ix + 8, iy:iy + 8] = X[i * n_img_per_row + j].reshape((8, 8)) - -plt.imshow(img, cmap=plt.cm.binary) -plt.xticks([]) -plt.yticks([]) -plt.title('A selection from the 64-dimensional digits dataset') - - -# ---------------------------------------------------------------------- -# Random 2D projection using a random unitary matrix -print("Computing random projection") -rp = random_projection.SparseRandomProjection(n_components=2, random_state=42) -X_projected = rp.fit_transform(X) -plot_embedding(X_projected, "Random Projection of the digits") - - -# ---------------------------------------------------------------------- -# Projection on to the first 2 principal components - -print("Computing PCA projection") -t0 = time() -X_pca = decomposition.TruncatedSVD(n_components=2).fit_transform(X) -plot_embedding(X_pca, - "Principal Components projection of the digits (time %.2fs)" % - (time() - t0)) - -# ---------------------------------------------------------------------- -# Projection on to the first 2 linear discriminant components - -print("Computing Linear Discriminant Analysis projection") -X2 = X.copy() -X2.flat[::X.shape[1] + 1] += 0.01 # Make X invertible -t0 = time() -X_lda = discriminant_analysis.LinearDiscriminantAnalysis(n_components=2 - ).fit_transform(X2, y) -plot_embedding(X_lda, - "Linear Discriminant projection of the digits (time %.2fs)" % - (time() - t0)) - - -# ---------------------------------------------------------------------- -# Isomap projection of the digits dataset -print("Computing Isomap projection") -t0 = time() -X_iso = manifold.Isomap(n_neighbors=n_neighbors, n_components=2 - ).fit_transform(X) -print("Done.") -plot_embedding(X_iso, - "Isomap projection of the digits (time %.2fs)" % - (time() - t0)) - - -# ---------------------------------------------------------------------- -# Locally linear embedding of the digits dataset -print("Computing LLE embedding") -clf = manifold.LocallyLinearEmbedding(n_neighbors=n_neighbors, n_components=2, - method='standard') -t0 = time() -X_lle = clf.fit_transform(X) -print("Done. Reconstruction error: %g" % clf.reconstruction_error_) -plot_embedding(X_lle, - "Locally Linear Embedding of the digits (time %.2fs)" % - (time() - t0)) - - -# ---------------------------------------------------------------------- -# Modified Locally linear embedding of the digits dataset -print("Computing modified LLE embedding") -clf = manifold.LocallyLinearEmbedding(n_neighbors=n_neighbors, n_components=2, - method='modified') -t0 = time() -X_mlle = clf.fit_transform(X) -print("Done. Reconstruction error: %g" % clf.reconstruction_error_) -plot_embedding(X_mlle, - "Modified Locally Linear Embedding of the digits (time %.2fs)" % - (time() - t0)) - - -# ---------------------------------------------------------------------- -# HLLE embedding of the digits dataset -print("Computing Hessian LLE embedding") -clf = manifold.LocallyLinearEmbedding(n_neighbors=n_neighbors, n_components=2, - method='hessian') -t0 = time() -X_hlle = clf.fit_transform(X) -print("Done. Reconstruction error: %g" % clf.reconstruction_error_) -plot_embedding(X_hlle, - "Hessian Locally Linear Embedding of the digits (time %.2fs)" % - (time() - t0)) - - -# ---------------------------------------------------------------------- -# LTSA embedding of the digits dataset -print("Computing LTSA embedding") -clf = manifold.LocallyLinearEmbedding(n_neighbors=n_neighbors, n_components=2, - method='ltsa') -t0 = time() -X_ltsa = clf.fit_transform(X) -print("Done. Reconstruction error: %g" % clf.reconstruction_error_) -plot_embedding(X_ltsa, - "Local Tangent Space Alignment of the digits (time %.2fs)" % - (time() - t0)) - -# ---------------------------------------------------------------------- -# MDS embedding of the digits dataset -print("Computing MDS embedding") -clf = manifold.MDS(n_components=2, n_init=1, max_iter=100) -t0 = time() -X_mds = clf.fit_transform(X) -print("Done. Stress: %f" % clf.stress_) -plot_embedding(X_mds, - "MDS embedding of the digits (time %.2fs)" % - (time() - t0)) - -# ---------------------------------------------------------------------- -# Random Trees embedding of the digits dataset -print("Computing Totally Random Trees embedding") -hasher = ensemble.RandomTreesEmbedding(n_estimators=200, random_state=0, - max_depth=5) -t0 = time() -X_transformed = hasher.fit_transform(X) -pca = decomposition.TruncatedSVD(n_components=2) -X_reduced = pca.fit_transform(X_transformed) - -plot_embedding(X_reduced, - "Random forest embedding of the digits (time %.2fs)" % - (time() - t0)) - -# ---------------------------------------------------------------------- -# Spectral embedding of the digits dataset -print("Computing Spectral embedding") -embedder = manifold.SpectralEmbedding(n_components=2, random_state=0, - eigen_solver="arpack") -t0 = time() -X_se = embedder.fit_transform(X) - -plot_embedding(X_se, - "Spectral embedding of the digits (time %.2fs)" % - (time() - t0)) - -# ---------------------------------------------------------------------- -# t-SNE embedding of the digits dataset -print("Computing t-SNE embedding") -tsne = manifold.TSNE(n_components=2, init='pca', random_state=0) -t0 = time() -X_tsne = tsne.fit_transform(X) - -plot_embedding(X_tsne, - "t-SNE embedding of the digits (time %.2fs)" % - (time() - t0)) - -# ---------------------------------------------------------------------- -# NCA projection of the digits dataset -print("Computing NCA projection") -nca = neighbors.NeighborhoodComponentsAnalysis(init='random', - n_components=2, random_state=0) -t0 = time() -X_nca = nca.fit_transform(X, y) - -plot_embedding(X_nca, - "NCA embedding of the digits (time %.2fs)" % - (time() - t0)) + # plot every digit on the embedding + # show an annotation box for a group of digits + dist = np.sum((X[i] - shown_images) ** 2, 1) + if np.min(dist) < 4e-3: + # don't show points that are too close + continue + shown_images = np.concatenate([shown_images, [X[i]]], axis=0) + imagebox = offsetbox.AnnotationBbox( + offsetbox.OffsetImage(digits.images[i], cmap=plt.cm.gray_r), X[i] + ) + imagebox.set(zorder=1) + ax.add_artist(imagebox) + + ax.set_title(title) + ax.axis("off") + + +# %% +# Embedding techniques comparison +# ------------------------------- +# +# Below, we compare different techniques. However, there are a couple of things +# to note: +# +# * the :class:`~sklearn.ensemble.RandomTreesEmbedding` is not +# technically a manifold embedding method, as it learn a high-dimensional +# representation on which we apply a dimensionality reduction method. +# However, it is often useful to cast a dataset into a representation in +# which the classes are linearly-separable. +# * the :class:`~sklearn.discriminant_analysis.LinearDiscriminantAnalysis` and +# the :class:`~sklearn.neighbors.NeighborhoodComponentsAnalysis`, are supervised +# dimensionality reduction method, i.e. they make use of the provided labels, +# contrary to other methods. +# * the :class:`~sklearn.manifold.TSNE` is initialized with the embedding that is +# generated by PCA in this example. It ensures global stability of the embedding, +# i.e., the embedding does not depend on random initialization. +from sklearn.decomposition import TruncatedSVD +from sklearn.discriminant_analysis import LinearDiscriminantAnalysis +from sklearn.ensemble import RandomTreesEmbedding +from sklearn.manifold import ( + MDS, + TSNE, + Isomap, + LocallyLinearEmbedding, + SpectralEmbedding, +) +from sklearn.neighbors import NeighborhoodComponentsAnalysis +from sklearn.pipeline import make_pipeline +from sklearn.random_projection import SparseRandomProjection + +embeddings = { + "Random projection embedding": SparseRandomProjection( + n_components=2, random_state=42 + ), + "Truncated SVD embedding": TruncatedSVD(n_components=2), + "Linear Discriminant Analysis embedding": LinearDiscriminantAnalysis( + n_components=2 + ), + "Isomap embedding": Isomap(n_neighbors=n_neighbors, n_components=2), + "Standard LLE embedding": LocallyLinearEmbedding( + n_neighbors=n_neighbors, n_components=2, method="standard" + ), + "Modified LLE embedding": LocallyLinearEmbedding( + n_neighbors=n_neighbors, n_components=2, method="modified" + ), + "Hessian LLE embedding": LocallyLinearEmbedding( + n_neighbors=n_neighbors, n_components=2, method="hessian" + ), + "LTSA LLE embedding": LocallyLinearEmbedding( + n_neighbors=n_neighbors, n_components=2, method="ltsa" + ), + "MDS embedding": MDS(n_components=2, n_init=1, max_iter=120, eps=1e-6), + "Random Trees embedding": make_pipeline( + RandomTreesEmbedding(n_estimators=200, max_depth=5, random_state=0), + TruncatedSVD(n_components=2), + ), + "Spectral embedding": SpectralEmbedding( + n_components=2, random_state=0, eigen_solver="arpack" + ), + "t-SNE embedding": TSNE( + n_components=2, + max_iter=500, + n_iter_without_progress=150, + n_jobs=2, + random_state=0, + ), + "NCA embedding": NeighborhoodComponentsAnalysis( + n_components=2, init="pca", random_state=0 + ), +} + +# %% +# Once we declared all the methods of interest, we can run and perform the projection +# of the original data. We will store the projected data as well as the computational +# time needed to perform each projection. +from time import time + +projections, timing = {}, {} +for name, transformer in embeddings.items(): + if name.startswith("Linear Discriminant Analysis"): + data = X.copy() + data.flat[:: X.shape[1] + 1] += 0.01 # Make X invertible + else: + data = X + + print(f"Computing {name}...") + start_time = time() + projections[name] = transformer.fit_transform(data, y) + timing[name] = time() - start_time + +# %% +# Finally, we can plot the resulting projection given by each method. +for name in timing: + title = f"{name} (time {timing[name]:.3f}s)" + plot_embedding(projections[name], title) plt.show() diff --git a/examples/manifold/plot_manifold_sphere.py b/examples/manifold/plot_manifold_sphere.py index 2b6566c4ecd92..d52d99be4d087 100644 --- a/examples/manifold/plot_manifold_sphere.py +++ b/examples/manifold/plot_manifold_sphere.py @@ -1,6 +1,3 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - """ ============================================= Manifold Learning methods on a severed sphere @@ -25,26 +22,24 @@ the low-dimensional space. Here the manifold problem matches fairly that of representing a flat map of the Earth, as with `map projection `_ -""" -# Author: Jaques Grobler -# License: BSD 3 clause +""" -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause from time import time -import numpy as np import matplotlib.pyplot as plt -from mpl_toolkits.mplot3d import Axes3D + +# Unused but required import for doing 3d projections with matplotlib < 3.2 +import mpl_toolkits.mplot3d # noqa: F401 +import numpy as np from matplotlib.ticker import NullFormatter from sklearn import manifold from sklearn.utils import check_random_state -# Next line to silence pyflakes. -Axes3D - # Variables for manifold learning. n_neighbors = 10 n_samples = 1000 @@ -55,32 +50,39 @@ t = random_state.rand(n_samples) * np.pi # Sever the poles from the sphere. -indices = ((t < (np.pi - (np.pi / 8))) & (t > ((np.pi / 8)))) +indices = (t < (np.pi - (np.pi / 8))) & (t > (np.pi / 8)) colors = p[indices] -x, y, z = np.sin(t[indices]) * np.cos(p[indices]), \ - np.sin(t[indices]) * np.sin(p[indices]), \ - np.cos(t[indices]) +x, y, z = ( + np.sin(t[indices]) * np.cos(p[indices]), + np.sin(t[indices]) * np.sin(p[indices]), + np.cos(t[indices]), +) # Plot our dataset. fig = plt.figure(figsize=(15, 8)) -plt.suptitle("Manifold Learning with %i points, %i neighbors" - % (1000, n_neighbors), fontsize=14) +plt.suptitle( + "Manifold Learning with %i points, %i neighbors" % (1000, n_neighbors), fontsize=14 +) -ax = fig.add_subplot(251, projection='3d') +ax = fig.add_subplot(251, projection="3d") ax.scatter(x, y, z, c=p[indices], cmap=plt.cm.rainbow) ax.view_init(40, -10) sphere_data = np.array([x, y, z]).T # Perform Locally Linear Embedding Manifold learning -methods = ['standard', 'ltsa', 'hessian', 'modified'] -labels = ['LLE', 'LTSA', 'Hessian LLE', 'Modified LLE'] +methods = ["standard", "ltsa", "hessian", "modified"] +labels = ["LLE", "LTSA", "Hessian LLE", "Modified LLE"] for i, method in enumerate(methods): t0 = time() - trans_data = manifold\ - .LocallyLinearEmbedding(n_neighbors, 2, - method=method).fit_transform(sphere_data).T + trans_data = ( + manifold.LocallyLinearEmbedding( + n_neighbors=n_neighbors, n_components=2, method=method, random_state=42 + ) + .fit_transform(sphere_data) + .T + ) t1 = time() print("%s: %.2g sec" % (methods[i], t1 - t0)) @@ -89,25 +91,28 @@ plt.title("%s (%.2g sec)" % (labels[i], t1 - t0)) ax.xaxis.set_major_formatter(NullFormatter()) ax.yaxis.set_major_formatter(NullFormatter()) - plt.axis('tight') + plt.axis("tight") # Perform Isomap Manifold learning. t0 = time() -trans_data = manifold.Isomap(n_neighbors, n_components=2)\ - .fit_transform(sphere_data).T +trans_data = ( + manifold.Isomap(n_neighbors=n_neighbors, n_components=2) + .fit_transform(sphere_data) + .T +) t1 = time() -print("%s: %.2g sec" % ('ISO', t1 - t0)) +print("%s: %.2g sec" % ("ISO", t1 - t0)) ax = fig.add_subplot(257) plt.scatter(trans_data[0], trans_data[1], c=colors, cmap=plt.cm.rainbow) -plt.title("%s (%.2g sec)" % ('Isomap', t1 - t0)) +plt.title("%s (%.2g sec)" % ("Isomap", t1 - t0)) ax.xaxis.set_major_formatter(NullFormatter()) ax.yaxis.set_major_formatter(NullFormatter()) -plt.axis('tight') +plt.axis("tight") # Perform Multi-dimensional scaling. t0 = time() -mds = manifold.MDS(2, max_iter=100, n_init=1) +mds = manifold.MDS(2, max_iter=100, n_init=1, random_state=42) trans_data = mds.fit_transform(sphere_data).T t1 = time() print("MDS: %.2g sec" % (t1 - t0)) @@ -117,12 +122,13 @@ plt.title("MDS (%.2g sec)" % (t1 - t0)) ax.xaxis.set_major_formatter(NullFormatter()) ax.yaxis.set_major_formatter(NullFormatter()) -plt.axis('tight') +plt.axis("tight") # Perform Spectral Embedding. t0 = time() -se = manifold.SpectralEmbedding(n_components=2, - n_neighbors=n_neighbors) +se = manifold.SpectralEmbedding( + n_components=2, n_neighbors=n_neighbors, random_state=42 +) trans_data = se.fit_transform(sphere_data).T t1 = time() print("Spectral Embedding: %.2g sec" % (t1 - t0)) @@ -132,11 +138,11 @@ plt.title("Spectral Embedding (%.2g sec)" % (t1 - t0)) ax.xaxis.set_major_formatter(NullFormatter()) ax.yaxis.set_major_formatter(NullFormatter()) -plt.axis('tight') +plt.axis("tight") # Perform t-distributed stochastic neighbor embedding. t0 = time() -tsne = manifold.TSNE(n_components=2, init='pca', random_state=0) +tsne = manifold.TSNE(n_components=2, random_state=0) trans_data = tsne.fit_transform(sphere_data).T t1 = time() print("t-SNE: %.2g sec" % (t1 - t0)) @@ -146,6 +152,6 @@ plt.title("t-SNE (%.2g sec)" % (t1 - t0)) ax.xaxis.set_major_formatter(NullFormatter()) ax.yaxis.set_major_formatter(NullFormatter()) -plt.axis('tight') +plt.axis("tight") plt.show() diff --git a/examples/manifold/plot_mds.py b/examples/manifold/plot_mds.py index aa2218a399a34..9d9828fc448f5 100644 --- a/examples/manifold/plot_mds.py +++ b/examples/manifold/plot_mds.py @@ -5,83 +5,124 @@ An illustration of the metric and non-metric MDS on generated noisy data. -The reconstructed points using the metric MDS and non metric MDS are slightly -shifted to avoid overlapping. """ -# Author: Nelle Varoquaux -# License: BSD +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -print(__doc__) -import numpy as np +# %% +# Dataset preparation +# ------------------- +# +# We start by uniformly generating 20 points in a 2D space. +import numpy as np from matplotlib import pyplot as plt from matplotlib.collections import LineCollection from sklearn import manifold -from sklearn.metrics import euclidean_distances from sklearn.decomposition import PCA +from sklearn.metrics import euclidean_distances +# Generate the data EPSILON = np.finfo(np.float32).eps n_samples = 20 -seed = np.random.RandomState(seed=3) -X_true = seed.randint(0, 20, 2 * n_samples).astype(float) +rng = np.random.RandomState(seed=3) +X_true = rng.randint(0, 20, 2 * n_samples).astype(float) X_true = X_true.reshape((n_samples, 2)) + # Center the data X_true -= X_true.mean() -similarities = euclidean_distances(X_true) +# %% +# Now we compute pairwise distances between all points and add +# a small amount of noise to the distance matrix. We make sure +# to keep the noisy distance matrix symmetric. -# Add noise to the similarities -noise = np.random.rand(n_samples, n_samples) -noise = noise + noise.T -noise[np.arange(noise.shape[0]), np.arange(noise.shape[0])] = 0 -similarities += noise - -mds = manifold.MDS(n_components=2, max_iter=3000, eps=1e-9, random_state=seed, - dissimilarity="precomputed", n_jobs=1) -pos = mds.fit(similarities).embedding_ - -nmds = manifold.MDS(n_components=2, metric=False, max_iter=3000, eps=1e-12, - dissimilarity="precomputed", random_state=seed, n_jobs=1, - n_init=1) -npos = nmds.fit_transform(similarities, init=pos) +# Compute pairwise Euclidean distances +distances = euclidean_distances(X_true) -# Rescale the data -pos *= np.sqrt((X_true ** 2).sum()) / np.sqrt((pos ** 2).sum()) -npos *= np.sqrt((X_true ** 2).sum()) / np.sqrt((npos ** 2).sum()) +# Add noise to the distances +noise = rng.rand(n_samples, n_samples) +noise = noise + noise.T +np.fill_diagonal(noise, 0) +distances += noise + +# %% +# Here we compute metric and non-metric MDS of the noisy distance matrix. + +mds = manifold.MDS( + n_components=2, + max_iter=3000, + eps=1e-9, + n_init=1, + random_state=42, + dissimilarity="precomputed", + n_jobs=1, +) +X_mds = mds.fit(distances).embedding_ + +nmds = manifold.MDS( + n_components=2, + metric=False, + max_iter=3000, + eps=1e-12, + dissimilarity="precomputed", + random_state=42, + n_jobs=1, + n_init=1, +) +X_nmds = nmds.fit_transform(distances) + +# %% +# Rescaling the non-metric MDS solution to match the spread of the original data. + +X_nmds *= np.sqrt((X_true**2).sum()) / np.sqrt((X_nmds**2).sum()) + +# %% +# To make the visual comparisons easier, we rotate the original data and both MDS +# solutions to their PCA axes. And flip horizontal and vertical MDS axes, if needed, +# to match the original data orientation. # Rotate the data -clf = PCA(n_components=2) -X_true = clf.fit_transform(X_true) +pca = PCA(n_components=2) +X_true = pca.fit_transform(X_true) +X_mds = pca.fit_transform(X_mds) +X_nmds = pca.fit_transform(X_nmds) -pos = clf.fit_transform(pos) +# Align the sign of PCs +for i in [0, 1]: + if np.corrcoef(X_mds[:, i], X_true[:, i])[0, 1] < 0: + X_mds[:, i] *= -1 + if np.corrcoef(X_nmds[:, i], X_true[:, i])[0, 1] < 0: + X_nmds[:, i] *= -1 -npos = clf.fit_transform(npos) +# %% +# Finally, we plot the original data and both MDS reconstructions. fig = plt.figure(1) -ax = plt.axes([0., 0., 1., 1.]) +ax = plt.axes([0.0, 0.0, 1.0, 1.0]) s = 100 -plt.scatter(X_true[:, 0], X_true[:, 1], color='navy', s=s, lw=0, - label='True Position') -plt.scatter(pos[:, 0], pos[:, 1], color='turquoise', s=s, lw=0, label='MDS') -plt.scatter(npos[:, 0], npos[:, 1], color='darkorange', s=s, lw=0, label='NMDS') -plt.legend(scatterpoints=1, loc='best', shadow=False) - -similarities = similarities.max() / (similarities + EPSILON) * 100 -np.fill_diagonal(similarities, 0) +plt.scatter(X_true[:, 0], X_true[:, 1], color="navy", s=s, lw=0, label="True Position") +plt.scatter(X_mds[:, 0], X_mds[:, 1], color="turquoise", s=s, lw=0, label="MDS") +plt.scatter(X_nmds[:, 0], X_nmds[:, 1], color="darkorange", s=s, lw=0, label="NMDS") +plt.legend(scatterpoints=1, loc="best", shadow=False) + # Plot the edges -start_idx, end_idx = np.where(pos) +start_idx, end_idx = X_mds.nonzero() # a sequence of (*line0*, *line1*, *line2*), where:: # linen = (x0, y0), (x1, y1), ... (xm, ym) -segments = [[X_true[i, :], X_true[j, :]] - for i in range(len(pos)) for j in range(len(pos))] -values = np.abs(similarities) -lc = LineCollection(segments, - zorder=0, cmap=plt.cm.Blues, - norm=plt.Normalize(0, values.max())) -lc.set_array(similarities.flatten()) +segments = [ + [X_true[i, :], X_true[j, :]] for i in range(len(X_true)) for j in range(len(X_true)) +] +edges = distances.max() / (distances + EPSILON) * 100 +np.fill_diagonal(edges, 0) +edges = np.abs(edges) +lc = LineCollection( + segments, zorder=0, cmap=plt.cm.Blues, norm=plt.Normalize(0, edges.max()) +) +lc.set_array(edges.flatten()) lc.set_linewidths(np.full(len(segments), 0.5)) ax.add_collection(lc) diff --git a/examples/manifold/plot_swissroll.py b/examples/manifold/plot_swissroll.py index 1edfed8da3fa0..803dc391ba4c2 100644 --- a/examples/manifold/plot_swissroll.py +++ b/examples/manifold/plot_swissroll.py @@ -1,46 +1,123 @@ """ =================================== -Swiss Roll reduction with LLE +Swiss Roll And Swiss-Hole Reduction =================================== - -An illustration of Swiss Roll reduction -with locally linear embedding +This notebook seeks to compare two popular non-linear dimensionality +techniques, T-distributed Stochastic Neighbor Embedding (t-SNE) and +Locally Linear Embedding (LLE), on the classic Swiss Roll dataset. +Then, we will explore how they both deal with the addition of a hole +in the data. """ -# Author: Fabian Pedregosa -- -# License: BSD 3 clause (C) INRIA 2011 +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -print(__doc__) +# %% +# Swiss Roll +# --------------------------------------------------- +# +# We start by generating the Swiss Roll dataset. import matplotlib.pyplot as plt -# This import is needed to modify the way figure behaves -from mpl_toolkits.mplot3d import Axes3D -Axes3D +from sklearn import datasets, manifold + +sr_points, sr_color = datasets.make_swiss_roll(n_samples=1500, random_state=0) + +# %% +# Now, let's take a look at our data: + +fig = plt.figure(figsize=(8, 6)) +ax = fig.add_subplot(111, projection="3d") +fig.add_axes(ax) +ax.scatter( + sr_points[:, 0], sr_points[:, 1], sr_points[:, 2], c=sr_color, s=50, alpha=0.8 +) +ax.set_title("Swiss Roll in Ambient Space") +ax.view_init(azim=-66, elev=12) +_ = ax.text2D(0.8, 0.05, s="n_samples=1500", transform=ax.transAxes) + +# %% +# Computing the LLE and t-SNE embeddings, we find that LLE seems to unroll the +# Swiss Roll pretty effectively. t-SNE on the other hand, is able +# to preserve the general structure of the data, but, poorly represents the +# continuous nature of our original data. Instead, it seems to unnecessarily +# clump sections of points together. + +sr_lle, sr_err = manifold.locally_linear_embedding( + sr_points, n_neighbors=12, n_components=2 +) + +sr_tsne = manifold.TSNE(n_components=2, perplexity=40, random_state=0).fit_transform( + sr_points +) + +fig, axs = plt.subplots(figsize=(8, 8), nrows=2) +axs[0].scatter(sr_lle[:, 0], sr_lle[:, 1], c=sr_color) +axs[0].set_title("LLE Embedding of Swiss Roll") +axs[1].scatter(sr_tsne[:, 0], sr_tsne[:, 1], c=sr_color) +_ = axs[1].set_title("t-SNE Embedding of Swiss Roll") + +# %% +# .. note:: +# +# LLE seems to be stretching the points from the center (purple) +# of the swiss roll. However, we observe that this is simply a byproduct +# of how the data was generated. There is a higher density of points near the +# center of the roll, which ultimately affects how LLE reconstructs the +# data in a lower dimension. + +# %% +# Swiss-Hole +# --------------------------------------------------- +# +# Now let's take a look at how both algorithms deal with us adding a hole to +# the data. First, we generate the Swiss-Hole dataset and plot it: + +sh_points, sh_color = datasets.make_swiss_roll( + n_samples=1500, hole=True, random_state=0 +) -#---------------------------------------------------------------------- -# Locally linear embedding of the swiss roll +fig = plt.figure(figsize=(8, 6)) +ax = fig.add_subplot(111, projection="3d") +fig.add_axes(ax) +ax.scatter( + sh_points[:, 0], sh_points[:, 1], sh_points[:, 2], c=sh_color, s=50, alpha=0.8 +) +ax.set_title("Swiss-Hole in Ambient Space") +ax.view_init(azim=-66, elev=12) +_ = ax.text2D(0.8, 0.05, s="n_samples=1500", transform=ax.transAxes) -from sklearn import manifold, datasets -X, color = datasets.make_swiss_roll(n_samples=1500) +# %% +# Computing the LLE and t-SNE embeddings, we obtain similar results to the +# Swiss Roll. LLE very capably unrolls the data and even preserves +# the hole. t-SNE, again seems to clump sections of points together, but, we +# note that it preserves the general topology of the original data. -print("Computing LLE embedding") -X_r, err = manifold.locally_linear_embedding(X, n_neighbors=12, - n_components=2) -print("Done. Reconstruction error: %g" % err) -#---------------------------------------------------------------------- -# Plot result +sh_lle, sh_err = manifold.locally_linear_embedding( + sh_points, n_neighbors=12, n_components=2 +) -fig = plt.figure() +sh_tsne = manifold.TSNE( + n_components=2, perplexity=40, init="random", random_state=0 +).fit_transform(sh_points) -ax = fig.add_subplot(211, projection='3d') -ax.scatter(X[:, 0], X[:, 1], X[:, 2], c=color, cmap=plt.cm.Spectral) +fig, axs = plt.subplots(figsize=(8, 8), nrows=2) +axs[0].scatter(sh_lle[:, 0], sh_lle[:, 1], c=sh_color) +axs[0].set_title("LLE Embedding of Swiss-Hole") +axs[1].scatter(sh_tsne[:, 0], sh_tsne[:, 1], c=sh_color) +_ = axs[1].set_title("t-SNE Embedding of Swiss-Hole") -ax.set_title("Original data") -ax = fig.add_subplot(212) -ax.scatter(X_r[:, 0], X_r[:, 1], c=color, cmap=plt.cm.Spectral) -plt.axis('tight') -plt.xticks([]), plt.yticks([]) -plt.title('Projected data') -plt.show() +# %% +# +# Concluding remarks +# ------------------ +# +# We note that t-SNE benefits from testing more combinations of parameters. +# Better results could probably have been obtained by better tuning these +# parameters. +# +# We observe that, as seen in the "Manifold learning on +# handwritten digits" example, t-SNE generally performs better than LLE +# on real world data. diff --git a/examples/manifold/plot_t_sne_perplexity.py b/examples/manifold/plot_t_sne_perplexity.py index dd7b4d1f21a09..0a4ecd4897a56 100644 --- a/examples/manifold/plot_t_sne_perplexity.py +++ b/examples/manifold/plot_t_sne_perplexity.py @@ -21,26 +21,28 @@ https://distill.pub/2016/misread-tsne/ provides a good discussion of the effects of various parameters, as well as interactive plots to explore those effects. + """ -# Author: Narine Kokhlikyan -# License: BSD +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -print(__doc__) +from time import time -import numpy as np import matplotlib.pyplot as plt - +import numpy as np from matplotlib.ticker import NullFormatter -from sklearn import manifold, datasets -from time import time -n_samples = 300 +from sklearn import datasets, manifold + +n_samples = 150 n_components = 2 (fig, subplots) = plt.subplots(3, 5, figsize=(15, 8)) perplexities = [5, 30, 50, 100] -X, y = datasets.make_circles(n_samples=n_samples, factor=.5, noise=.05) +X, y = datasets.make_circles( + n_samples=n_samples, factor=0.5, noise=0.05, random_state=0 +) red = y == 0 green = y == 1 @@ -50,14 +52,19 @@ ax.scatter(X[green, 0], X[green, 1], c="g") ax.xaxis.set_major_formatter(NullFormatter()) ax.yaxis.set_major_formatter(NullFormatter()) -plt.axis('tight') +plt.axis("tight") for i, perplexity in enumerate(perplexities): ax = subplots[0][i + 1] t0 = time() - tsne = manifold.TSNE(n_components=n_components, init='random', - random_state=0, perplexity=perplexity) + tsne = manifold.TSNE( + n_components=n_components, + init="random", + random_state=0, + perplexity=perplexity, + max_iter=300, + ) Y = tsne.fit_transform(X) t1 = time() print("circles, perplexity=%d in %.2g sec" % (perplexity, t1 - t0)) @@ -66,7 +73,7 @@ ax.scatter(Y[green, 0], Y[green, 1], c="g") ax.xaxis.set_major_formatter(NullFormatter()) ax.yaxis.set_major_formatter(NullFormatter()) - ax.axis('tight') + ax.axis("tight") # Another example using s-curve X, color = datasets.make_s_curve(n_samples, random_state=0) @@ -80,8 +87,14 @@ ax = subplots[1][i + 1] t0 = time() - tsne = manifold.TSNE(n_components=n_components, init='random', - random_state=0, perplexity=perplexity) + tsne = manifold.TSNE( + n_components=n_components, + init="random", + random_state=0, + perplexity=perplexity, + learning_rate="auto", + max_iter=300, + ) Y = tsne.fit_transform(X) t1 = time() print("S-curve, perplexity=%d in %.2g sec" % (perplexity, t1 - t0)) @@ -90,16 +103,18 @@ ax.scatter(Y[:, 0], Y[:, 1], c=color) ax.xaxis.set_major_formatter(NullFormatter()) ax.yaxis.set_major_formatter(NullFormatter()) - ax.axis('tight') + ax.axis("tight") # Another example using a 2D uniform grid x = np.linspace(0, 1, int(np.sqrt(n_samples))) xx, yy = np.meshgrid(x, x) -X = np.hstack([ - xx.ravel().reshape(-1, 1), - yy.ravel().reshape(-1, 1), -]) +X = np.hstack( + [ + xx.ravel().reshape(-1, 1), + yy.ravel().reshape(-1, 1), + ] +) color = xx.ravel() ax = subplots[2][0] ax.scatter(X[:, 0], X[:, 1], c=color) @@ -110,8 +125,13 @@ ax = subplots[2][i + 1] t0 = time() - tsne = manifold.TSNE(n_components=n_components, init='random', - random_state=0, perplexity=perplexity) + tsne = manifold.TSNE( + n_components=n_components, + init="random", + random_state=0, + perplexity=perplexity, + max_iter=400, + ) Y = tsne.fit_transform(X) t1 = time() print("uniform grid, perplexity=%d in %.2g sec" % (perplexity, t1 - t0)) @@ -120,7 +140,7 @@ ax.scatter(Y[:, 0], Y[:, 1], c=color) ax.xaxis.set_major_formatter(NullFormatter()) ax.yaxis.set_major_formatter(NullFormatter()) - ax.axis('tight') + ax.axis("tight") plt.show() diff --git a/examples/miscellaneous/README.txt b/examples/miscellaneous/README.txt index 4e44ceee95809..bef5239bb9cb9 100644 --- a/examples/miscellaneous/README.txt +++ b/examples/miscellaneous/README.txt @@ -4,4 +4,3 @@ Miscellaneous ------------- Miscellaneous and introductory examples for scikit-learn. - diff --git a/examples/miscellaneous/plot_anomaly_comparison.py b/examples/miscellaneous/plot_anomaly_comparison.py index b5ebd96bd8815..819a775724e64 100644 --- a/examples/miscellaneous/plot_anomaly_comparison.py +++ b/examples/miscellaneous/plot_anomaly_comparison.py @@ -22,7 +22,17 @@ One-class SVM might give useful results in these situations depending on the value of its hyperparameters. -:class:`~sklearn.covariance.EllipticEnvelope` assumes the data is Gaussian and +The :class:`sklearn.linear_model.SGDOneClassSVM` is an implementation of the +One-Class SVM based on stochastic gradient descent (SGD). Combined with kernel +approximation, this estimator can be used to approximate the solution +of a kernelized :class:`sklearn.svm.OneClassSVM`. We note that, although not +identical, the decision boundaries of the +:class:`sklearn.linear_model.SGDOneClassSVM` and the ones of +:class:`sklearn.svm.OneClassSVM` are very similar. The main advantage of using +:class:`sklearn.linear_model.SGDOneClassSVM` is that it scales linearly with +the number of samples. + +:class:`sklearn.covariance.EllipticEnvelope` assumes the data is Gaussian and learns an ellipse. It thus degrades when the data is not unimodal. Notice however that this estimator is robust to outliers. @@ -49,27 +59,28 @@ Finally, note that parameters of the models have been here handpicked but that in practice they need to be adjusted. In the absence of labelled data, the problem is completely unsupervised so model selection can be a challenge. + """ -# Author: Alexandre Gramfort -# Albert Thomas -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import time -import numpy as np import matplotlib import matplotlib.pyplot as plt +import numpy as np from sklearn import svm -from sklearn.datasets import make_moons, make_blobs from sklearn.covariance import EllipticEnvelope +from sklearn.datasets import make_blobs, make_moons from sklearn.ensemble import IsolationForest +from sklearn.kernel_approximation import Nystroem +from sklearn.linear_model import SGDOneClassSVM from sklearn.neighbors import LocalOutlierFactor +from sklearn.pipeline import make_pipeline -print(__doc__) - -matplotlib.rcParams['contour.negative_linestyle'] = 'solid' +matplotlib.rcParams["contour.negative_linestyle"] = "solid" # Example settings n_samples = 300 @@ -77,44 +88,66 @@ n_outliers = int(outliers_fraction * n_samples) n_inliers = n_samples - n_outliers -# define outlier/anomaly detection methods to be compared +# define outlier/anomaly detection methods to be compared. +# the SGDOneClassSVM must be used in a pipeline with a kernel approximation +# to give similar results to the OneClassSVM anomaly_algorithms = [ - ("Robust covariance", EllipticEnvelope(contamination=outliers_fraction)), - ("One-Class SVM", svm.OneClassSVM(nu=outliers_fraction, kernel="rbf", - gamma=0.1)), - ("Isolation Forest", IsolationForest(contamination=outliers_fraction, - random_state=42)), - ("Local Outlier Factor", LocalOutlierFactor( - n_neighbors=35, contamination=outliers_fraction))] + ( + "Robust covariance", + EllipticEnvelope(contamination=outliers_fraction, random_state=42), + ), + ("One-Class SVM", svm.OneClassSVM(nu=outliers_fraction, kernel="rbf", gamma=0.1)), + ( + "One-Class SVM (SGD)", + make_pipeline( + Nystroem(gamma=0.1, random_state=42, n_components=150), + SGDOneClassSVM( + nu=outliers_fraction, + shuffle=True, + fit_intercept=True, + random_state=42, + tol=1e-6, + ), + ), + ), + ( + "Isolation Forest", + IsolationForest(contamination=outliers_fraction, random_state=42), + ), + ( + "Local Outlier Factor", + LocalOutlierFactor(n_neighbors=35, contamination=outliers_fraction), + ), +] # Define datasets blobs_params = dict(random_state=0, n_samples=n_inliers, n_features=2) datasets = [ - make_blobs(centers=[[0, 0], [0, 0]], cluster_std=0.5, - **blobs_params)[0], - make_blobs(centers=[[2, 2], [-2, -2]], cluster_std=[0.5, 0.5], - **blobs_params)[0], - make_blobs(centers=[[2, 2], [-2, -2]], cluster_std=[1.5, .3], - **blobs_params)[0], - 4. * (make_moons(n_samples=n_samples, noise=.05, random_state=0)[0] - - np.array([0.5, 0.25])), - 14. * (np.random.RandomState(42).rand(n_samples, 2) - 0.5)] + make_blobs(centers=[[0, 0], [0, 0]], cluster_std=0.5, **blobs_params)[0], + make_blobs(centers=[[2, 2], [-2, -2]], cluster_std=[0.5, 0.5], **blobs_params)[0], + make_blobs(centers=[[2, 2], [-2, -2]], cluster_std=[1.5, 0.3], **blobs_params)[0], + 4.0 + * ( + make_moons(n_samples=n_samples, noise=0.05, random_state=0)[0] + - np.array([0.5, 0.25]) + ), + 14.0 * (np.random.RandomState(42).rand(n_samples, 2) - 0.5), +] # Compare given classifiers under given settings -xx, yy = np.meshgrid(np.linspace(-7, 7, 150), - np.linspace(-7, 7, 150)) +xx, yy = np.meshgrid(np.linspace(-7, 7, 150), np.linspace(-7, 7, 150)) -plt.figure(figsize=(len(anomaly_algorithms) * 2 + 3, 12.5)) -plt.subplots_adjust(left=.02, right=.98, bottom=.001, top=.96, wspace=.05, - hspace=.01) +plt.figure(figsize=(len(anomaly_algorithms) * 2 + 4, 12.5)) +plt.subplots_adjust( + left=0.02, right=0.98, bottom=0.001, top=0.96, wspace=0.05, hspace=0.01 +) plot_num = 1 rng = np.random.RandomState(42) for i_dataset, X in enumerate(datasets): # Add outliers - X = np.concatenate([X, rng.uniform(low=-6, high=6, - size=(n_outliers, 2))], axis=0) + X = np.concatenate([X, rng.uniform(low=-6, high=6, size=(n_outliers, 2))], axis=0) for name, algorithm in anomaly_algorithms: t0 = time.time() @@ -134,18 +167,23 @@ if name != "Local Outlier Factor": # LOF does not implement predict Z = algorithm.predict(np.c_[xx.ravel(), yy.ravel()]) Z = Z.reshape(xx.shape) - plt.contour(xx, yy, Z, levels=[0], linewidths=2, colors='black') + plt.contour(xx, yy, Z, levels=[0], linewidths=2, colors="black") - colors = np.array(['#377eb8', '#ff7f00']) + colors = np.array(["#377eb8", "#ff7f00"]) plt.scatter(X[:, 0], X[:, 1], s=10, color=colors[(y_pred + 1) // 2]) plt.xlim(-7, 7) plt.ylim(-7, 7) plt.xticks(()) plt.yticks(()) - plt.text(.99, .01, ('%.2fs' % (t1 - t0)).lstrip('0'), - transform=plt.gca().transAxes, size=15, - horizontalalignment='right') + plt.text( + 0.99, + 0.01, + ("%.2fs" % (t1 - t0)).lstrip("0"), + transform=plt.gca().transAxes, + size=15, + horizontalalignment="right", + ) plot_num += 1 plt.show() diff --git a/examples/miscellaneous/plot_changed_only_pprint_parameter.py b/examples/miscellaneous/plot_changed_only_pprint_parameter.py deleted file mode 100644 index a35471105b6c1..0000000000000 --- a/examples/miscellaneous/plot_changed_only_pprint_parameter.py +++ /dev/null @@ -1,30 +0,0 @@ -""" -================================= -Compact estimator representations -================================= - -This example illustrates the use of the print_changed_only global parameter. - -Setting print_changed_only to True will alternate the representation of -estimators to only show the parameters that have been set to non-default -values. This can be used to have more compact representations. -""" -print(__doc__) - -from sklearn.linear_model import LogisticRegression -from sklearn import set_config - - -lr = LogisticRegression(penalty='l1') -print('Default representation:') -print(lr) -# LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True, -# intercept_scaling=1, l1_ratio=None, max_iter=100, -# multi_class='auto', n_jobs=None, penalty='l1', -# random_state=None, solver='warn', tol=0.0001, verbose=0, -# warm_start=False) - -set_config(print_changed_only=True) -print('\nWith changed_only option:') -print(lr) -# LogisticRegression(penalty='l1') diff --git a/examples/miscellaneous/plot_display_object_visualization.py b/examples/miscellaneous/plot_display_object_visualization.py index 3a16dfa7d1a68..ec54d909d1c37 100644 --- a/examples/miscellaneous/plot_display_object_visualization.py +++ b/examples/miscellaneous/plot_display_object_visualization.py @@ -12,22 +12,25 @@ a model's predictions are already computed or expensive to compute. Note that this is advanced usage, and in general we recommend using their respective plot functions. + """ -print(__doc__) + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause # %% # Load Data and train model # ------------------------- # For this example, we load a blood transfusion service center data set from -# `OpenML `. This is a binary classification +# `OpenML `_. This is a binary classification # problem where the target is whether an individual donated blood. Then the # data is split into a train and test dataset and a logistic regression is -# fitted wtih the train dataset. +# fitted with the train dataset. from sklearn.datasets import fetch_openml -from sklearn.preprocessing import StandardScaler -from sklearn.pipeline import make_pipeline from sklearn.linear_model import LogisticRegression from sklearn.model_selection import train_test_split +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import StandardScaler X, y = fetch_openml(data_id=1464, return_X_y=True) X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y) @@ -37,12 +40,11 @@ # %% # Create :class:`ConfusionMatrixDisplay` -############################################################################## +# ###################################### # With the fitted model, we compute the predictions of the model on the test -# dataset. These predictions are used to compute the confustion matrix which +# dataset. These predictions are used to compute the confusion matrix which # is plotted with the :class:`ConfusionMatrixDisplay` -from sklearn.metrics import confusion_matrix -from sklearn.metrics import ConfusionMatrixDisplay +from sklearn.metrics import ConfusionMatrixDisplay, confusion_matrix y_pred = clf.predict(X_test) cm = confusion_matrix(y_test, y_pred) @@ -52,12 +54,12 @@ # %% # Create :class:`RocCurveDisplay` -############################################################################## +# ############################### # The roc curve requires either the probabilities or the non-thresholded # decision values from the estimator. Since the logistic regression provides # a decision function, we will use it to plot the roc curve: -from sklearn.metrics import roc_curve -from sklearn.metrics import RocCurveDisplay +from sklearn.metrics import RocCurveDisplay, roc_curve + y_score = clf.decision_function(X_test) fpr, tpr, _ = roc_curve(y_test, y_score, pos_label=clf.classes_[1]) @@ -65,19 +67,17 @@ # %% # Create :class:`PrecisionRecallDisplay` -############################################################################## +# ###################################### # Similarly, the precision recall curve can be plotted using `y_score` from # the prevision sections. -from sklearn.metrics import precision_recall_curve -from sklearn.metrics import PrecisionRecallDisplay +from sklearn.metrics import PrecisionRecallDisplay, precision_recall_curve -prec, recall, _ = precision_recall_curve(y_test, y_score, - pos_label=clf.classes_[1]) +prec, recall, _ = precision_recall_curve(y_test, y_score, pos_label=clf.classes_[1]) pr_display = PrecisionRecallDisplay(precision=prec, recall=recall).plot() # %% # Combining the display objects into a single plot -############################################################################## +# ################################################ # The display objects store the computed values that were passed as arguments. # This allows for the visualizations to be easliy combined using matplotlib's # API. In the following example, we place the displays next to each other in a @@ -85,6 +85,7 @@ # sphinx_gallery_thumbnail_number = 4 import matplotlib.pyplot as plt + fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 8)) roc_display.plot(ax=ax1) diff --git a/examples/miscellaneous/plot_estimator_representation.py b/examples/miscellaneous/plot_estimator_representation.py new file mode 100644 index 0000000000000..683f0c5785f20 --- /dev/null +++ b/examples/miscellaneous/plot_estimator_representation.py @@ -0,0 +1,52 @@ +""" +=========================================== +Displaying estimators and complex pipelines +=========================================== + +This example illustrates different ways estimators and pipelines can be +displayed. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from sklearn.compose import make_column_transformer +from sklearn.impute import SimpleImputer +from sklearn.linear_model import LogisticRegression +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import OneHotEncoder, StandardScaler + +# %% +# Compact text representation +# --------------------------- +# +# Estimators will only show the parameters that have been set to non-default +# values when displayed as a string. This reduces the visual noise and makes it +# easier to spot what the differences are when comparing instances. + +lr = LogisticRegression(penalty="l1") +print(lr) + +# %% +# Rich HTML representation +# ------------------------ +# In notebooks estimators and pipelines will use a rich HTML representation. +# This is particularly useful to summarise the +# structure of pipelines and other composite estimators, with interactivity to +# provide detail. Click on the example image below to expand Pipeline +# elements. See :ref:`visualizing_composite_estimators` for how you can use +# this feature. + +num_proc = make_pipeline(SimpleImputer(strategy="median"), StandardScaler()) + +cat_proc = make_pipeline( + SimpleImputer(strategy="constant", fill_value="missing"), + OneHotEncoder(handle_unknown="ignore"), +) + +preprocessor = make_column_transformer( + (num_proc, ("feat1", "feat3")), (cat_proc, ("feat0", "feat2")) +) + +clf = make_pipeline(preprocessor, LogisticRegression()) +clf diff --git a/examples/miscellaneous/plot_isotonic_regression.py b/examples/miscellaneous/plot_isotonic_regression.py index 8848b1151e1fa..4ca352e882f36 100644 --- a/examples/miscellaneous/plot_isotonic_regression.py +++ b/examples/miscellaneous/plot_isotonic_regression.py @@ -18,24 +18,22 @@ target values are computed by the isotonic non-parametric fit. """ -print(__doc__) -# Author: Nelle Varoquaux -# Alexandre Gramfort -# License: BSD +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt +import numpy as np from matplotlib.collections import LineCollection -from sklearn.linear_model import LinearRegression from sklearn.isotonic import IsotonicRegression +from sklearn.linear_model import LinearRegression from sklearn.utils import check_random_state n = 100 x = np.arange(n) rs = check_random_state(0) -y = rs.randint(-50, 50, size=(n,)) + 50. * np.log1p(np.arange(n)) +y = rs.randint(-50, 50, size=(n,)) + 50.0 * np.log1p(np.arange(n)) # %% # Fit IsotonicRegression and LinearRegression models: @@ -56,16 +54,16 @@ fig, (ax0, ax1) = plt.subplots(ncols=2, figsize=(12, 6)) -ax0.plot(x, y, 'C0.', markersize=12) -ax0.plot(x, y_, 'C1.-', markersize=12) -ax0.plot(x, lr.predict(x[:, np.newaxis]), 'C2-') +ax0.plot(x, y, "C0.", markersize=12) +ax0.plot(x, y_, "C1.-", markersize=12) +ax0.plot(x, lr.predict(x[:, np.newaxis]), "C2-") ax0.add_collection(lc) -ax0.legend(('Training data', 'Isotonic fit', 'Linear fit'), loc='lower right') -ax0.set_title('Isotonic regression fit on noisy data (n=%d)' % n) +ax0.legend(("Training data", "Isotonic fit", "Linear fit"), loc="lower right") +ax0.set_title("Isotonic regression fit on noisy data (n=%d)" % n) x_test = np.linspace(-10, 110, 1000) -ax1.plot(x_test, ir.predict(x_test), 'C1-') -ax1.plot(ir.X_thresholds_, ir.y_thresholds_, 'C1.', markersize=12) +ax1.plot(x_test, ir.predict(x_test), "C1-") +ax1.plot(ir.X_thresholds_, ir.y_thresholds_, "C1.", markersize=12) ax1.set_title("Prediction function (%d thresholds)" % len(ir.X_thresholds_)) plt.show() diff --git a/examples/miscellaneous/plot_johnson_lindenstrauss_bound.py b/examples/miscellaneous/plot_johnson_lindenstrauss_bound.py index 433f5a7f05d37..5528eada1ed4a 100644 --- a/examples/miscellaneous/plot_johnson_lindenstrauss_bound.py +++ b/examples/miscellaneous/plot_johnson_lindenstrauss_bound.py @@ -10,27 +10,24 @@ .. _`Johnson-Lindenstrauss lemma`: https://en.wikipedia.org/wiki/\ Johnson%E2%80%93Lindenstrauss_lemma + """ -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import sys from time import time -import numpy as np -import matplotlib + import matplotlib.pyplot as plt -from sklearn.random_projection import johnson_lindenstrauss_min_dim -from sklearn.random_projection import SparseRandomProjection -from sklearn.datasets import fetch_20newsgroups_vectorized -from sklearn.datasets import load_digits -from sklearn.metrics.pairwise import euclidean_distances -from sklearn.utils.fixes import parse_version +import numpy as np -# `normed` is being deprecated in favor of `density` in histograms -if parse_version(matplotlib.__version__) >= parse_version('2.1'): - density_param = {'density': True} -else: - density_param = {'normed': True} +from sklearn.datasets import fetch_20newsgroups_vectorized, load_digits +from sklearn.metrics.pairwise import euclidean_distances +from sklearn.random_projection import ( + SparseRandomProjection, + johnson_lindenstrauss_min_dim, +) # %% # Theoretical bounds @@ -42,9 +39,9 @@ # .. math:: # (1 - eps) \|u - v\|^2 < \|p(u) - p(v)\|^2 < (1 + eps) \|u - v\|^2 # -# Where u and v are any rows taken from a dataset of shape (n_samples, -# n_features) and p is a projection by a random Gaussian N(0, 1) matrix -# of shape (n_components, n_features) (or a sparse Achlioptas matrix). +# Where `u` and `v` are any rows taken from a dataset of shape `(n_samples, +# n_features)` and `p` is a projection by a random Gaussian `N(0, 1)` matrix +# of shape `(n_components, n_features)` (or a sparse Achlioptas matrix). # # The minimum number of components to guarantees the eps-embedding is # given by: @@ -69,7 +66,7 @@ min_n_components = johnson_lindenstrauss_min_dim(n_samples_range, eps=eps) plt.loglog(n_samples_range, min_n_components, color=color) -plt.legend(["eps = %0.1f" % eps for eps in eps_range], loc="lower right") +plt.legend([f"eps = {eps:0.1f}" for eps in eps_range], loc="lower right") plt.xlabel("Number of observations to eps-embed") plt.ylabel("Minimum number of dimensions") plt.title("Johnson-Lindenstrauss bounds:\nn_samples vs n_components") @@ -93,7 +90,7 @@ min_n_components = johnson_lindenstrauss_min_dim(n_samples, eps=eps_range) plt.semilogy(eps_range, min_n_components, color=color) -plt.legend(["n_samples = %d" % n for n in n_samples_range], loc="upper right") +plt.legend([f"n_samples = {n}" for n in n_samples_range], loc="upper right") plt.xlabel("Distortion eps") plt.ylabel("Minimum number of dimensions") plt.title("Johnson-Lindenstrauss bounds:\nn_components vs eps") @@ -106,12 +103,12 @@ # We validate the above bounds on the 20 newsgroups text document # (TF-IDF word frequencies) dataset or on the digits dataset: # -# - for the 20 newsgroups dataset some 500 documents with 100k +# - for the 20 newsgroups dataset some 300 documents with 100k # features in total are projected using a sparse random matrix to smaller # euclidean spaces with various values for the target number of dimensions # ``n_components``. # -# - for the digits dataset, some 8x8 gray level pixels data for 500 +# - for the digits dataset, some 8x8 gray level pixels data for 300 # handwritten digits pictures are randomly projected to spaces for various # larger number of dimensions ``n_components``. # @@ -119,24 +116,26 @@ # digits dataset, pass the ``--use-digits-dataset`` command line argument to # this script. -if '--use-digits-dataset' in sys.argv: - data = load_digits().data[:500] +if "--use-digits-dataset" in sys.argv: + data = load_digits().data[:300] else: - data = fetch_20newsgroups_vectorized().data[:500] + data = fetch_20newsgroups_vectorized().data[:300] # %% # For each value of ``n_components``, we plot: # # - 2D distribution of sample pairs with pairwise distances in original -# and projected spaces as x and y axis respectively. +# and projected spaces as x- and y-axis respectively. # # - 1D histogram of the ratio of those distances (projected / original). n_samples, n_features = data.shape -print("Embedding %d samples with dim %d using various random projections" - % (n_samples, n_features)) +print( + f"Embedding {n_samples} samples with dim {n_features} using various " + "random projections" +) -n_components_range = np.array([300, 1000, 10000]) +n_components_range = np.array([300, 1_000, 10_000]) dists = euclidean_distances(data, squared=True).ravel() # select only non-identical samples pairs @@ -147,38 +146,41 @@ t0 = time() rp = SparseRandomProjection(n_components=n_components) projected_data = rp.fit_transform(data) - print("Projected %d samples from %d to %d in %0.3fs" - % (n_samples, n_features, n_components, time() - t0)) - if hasattr(rp, 'components_'): + print( + f"Projected {n_samples} samples from {n_features} to {n_components} in " + f"{time() - t0:0.3f}s" + ) + if hasattr(rp, "components_"): n_bytes = rp.components_.data.nbytes n_bytes += rp.components_.indices.nbytes - print("Random matrix with size: %0.3fMB" % (n_bytes / 1e6)) + print(f"Random matrix with size: {n_bytes / 1e6:0.3f} MB") - projected_dists = euclidean_distances( - projected_data, squared=True).ravel()[nonzero] + projected_dists = euclidean_distances(projected_data, squared=True).ravel()[nonzero] plt.figure() min_dist = min(projected_dists.min(), dists.min()) max_dist = max(projected_dists.max(), dists.max()) - plt.hexbin(dists, projected_dists, gridsize=100, cmap=plt.cm.PuBu, - extent=[min_dist, max_dist, min_dist, max_dist]) + plt.hexbin( + dists, + projected_dists, + gridsize=100, + cmap=plt.cm.PuBu, + extent=[min_dist, max_dist, min_dist, max_dist], + ) plt.xlabel("Pairwise squared distances in original space") plt.ylabel("Pairwise squared distances in projected space") - plt.title("Pairwise distances distribution for n_components=%d" % - n_components) + plt.title("Pairwise distances distribution for n_components=%d" % n_components) cb = plt.colorbar() - cb.set_label('Sample pairs counts') + cb.set_label("Sample pairs counts") rates = projected_dists / dists - print("Mean distances rate: %0.2f (%0.2f)" - % (np.mean(rates), np.std(rates))) + print(f"Mean distances rate: {np.mean(rates):.2f} ({np.std(rates):.2f})") plt.figure() - plt.hist(rates, bins=50, range=(0., 2.), edgecolor='k', **density_param) + plt.hist(rates, bins=50, range=(0.0, 2.0), edgecolor="k", density=True) plt.xlabel("Squared distances rate: projected / original") plt.ylabel("Distribution of samples pairs") - plt.title("Histogram of pairwise distance rates for n_components=%d" % - n_components) + plt.title("Histogram of pairwise distance rates for n_components=%d" % n_components) # TODO: compute the expected value of eps and add them to the previous plot # as vertical lines / region @@ -190,15 +192,13 @@ # We can see that for low values of ``n_components`` the distribution is wide # with many distorted pairs and a skewed distribution (due to the hard # limit of zero ratio on the left as distances are always positives) -# while for larger values of n_components the distortion is controlled +# while for larger values of `n_components` the distortion is controlled # and the distances are well preserved by the random projection. - - -# %% +# # Remarks # ======= # -# According to the JL lemma, projecting 500 samples without too much distortion +# According to the JL lemma, projecting 300 samples without too much distortion # will require at least several thousands dimensions, irrespective of the # number of features of the original dataset. # @@ -207,5 +207,5 @@ # for dimensionality reduction in this case. # # On the twenty newsgroups on the other hand the dimensionality can be -# decreased from 56436 down to 10000 while reasonably preserving +# decreased from 56,436 down to 10,000 while reasonably preserving # pairwise distances. diff --git a/examples/miscellaneous/plot_kernel_approximation.py b/examples/miscellaneous/plot_kernel_approximation.py index 80b4582a45670..4c994af033080 100644 --- a/examples/miscellaneous/plot_kernel_approximation.py +++ b/examples/miscellaneous/plot_kernel_approximation.py @@ -34,22 +34,19 @@ # --------------------------------------------------- -# Author: Gael Varoquaux -# Andreas Mueller -# License: BSD 3 clause - -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause # Standard scientific Python imports +from time import time + import matplotlib.pyplot as plt import numpy as np -from time import time # Import datasets, classifiers and performance metrics -from sklearn import datasets, svm, pipeline -from sklearn.kernel_approximation import (RBFSampler, - Nystroem) +from sklearn import datasets, pipeline, svm from sklearn.decomposition import PCA +from sklearn.kernel_approximation import Nystroem, RBFSampler # The digits dataset digits = datasets.load_digits(n_class=9) @@ -61,32 +58,38 @@ # To apply an classifier on this data, we need to flatten the image, to # turn the data in a (samples, feature) matrix: n_samples = len(digits.data) -data = digits.data / 16. +data = digits.data / 16.0 data -= data.mean(axis=0) # We learn the digits on the first half of the digits -data_train, targets_train = (data[:n_samples // 2], - digits.target[:n_samples // 2]) +data_train, targets_train = (data[: n_samples // 2], digits.target[: n_samples // 2]) # Now predict the value of the digit on the second half: -data_test, targets_test = (data[n_samples // 2:], - digits.target[n_samples // 2:]) +data_test, targets_test = (data[n_samples // 2 :], digits.target[n_samples // 2 :]) # data_test = scaler.transform(data_test) # Create a classifier: a support vector classifier -kernel_svm = svm.SVC(gamma=.2) -linear_svm = svm.LinearSVC() +kernel_svm = svm.SVC(gamma=0.2) +linear_svm = svm.LinearSVC(random_state=42) # create pipeline from kernel approximation # and linear svm -feature_map_fourier = RBFSampler(gamma=.2, random_state=1) -feature_map_nystroem = Nystroem(gamma=.2, random_state=1) -fourier_approx_svm = pipeline.Pipeline([("feature_map", feature_map_fourier), - ("svm", svm.LinearSVC())]) - -nystroem_approx_svm = pipeline.Pipeline([("feature_map", feature_map_nystroem), - ("svm", svm.LinearSVC())]) +feature_map_fourier = RBFSampler(gamma=0.2, random_state=1) +feature_map_nystroem = Nystroem(gamma=0.2, random_state=1) +fourier_approx_svm = pipeline.Pipeline( + [ + ("feature_map", feature_map_fourier), + ("svm", svm.LinearSVC(random_state=42)), + ] +) + +nystroem_approx_svm = pipeline.Pipeline( + [ + ("feature_map", feature_map_nystroem), + ("svm", svm.LinearSVC(random_state=42)), + ] +) # fit and predict using linear and kernel svm: @@ -129,23 +132,35 @@ timescale = plt.subplot(122) accuracy.plot(sample_sizes, nystroem_scores, label="Nystroem approx. kernel") -timescale.plot(sample_sizes, nystroem_times, '--', - label='Nystroem approx. kernel') +timescale.plot(sample_sizes, nystroem_times, "--", label="Nystroem approx. kernel") accuracy.plot(sample_sizes, fourier_scores, label="Fourier approx. kernel") -timescale.plot(sample_sizes, fourier_times, '--', - label='Fourier approx. kernel') +timescale.plot(sample_sizes, fourier_times, "--", label="Fourier approx. kernel") # horizontal lines for exact rbf and linear kernels: -accuracy.plot([sample_sizes[0], sample_sizes[-1]], - [linear_svm_score, linear_svm_score], label="linear svm") -timescale.plot([sample_sizes[0], sample_sizes[-1]], - [linear_svm_time, linear_svm_time], '--', label='linear svm') - -accuracy.plot([sample_sizes[0], sample_sizes[-1]], - [kernel_svm_score, kernel_svm_score], label="rbf svm") -timescale.plot([sample_sizes[0], sample_sizes[-1]], - [kernel_svm_time, kernel_svm_time], '--', label='rbf svm') +accuracy.plot( + [sample_sizes[0], sample_sizes[-1]], + [linear_svm_score, linear_svm_score], + label="linear svm", +) +timescale.plot( + [sample_sizes[0], sample_sizes[-1]], + [linear_svm_time, linear_svm_time], + "--", + label="linear svm", +) + +accuracy.plot( + [sample_sizes[0], sample_sizes[-1]], + [kernel_svm_score, kernel_svm_score], + label="rbf svm", +) +timescale.plot( + [sample_sizes[0], sample_sizes[-1]], + [kernel_svm_time, kernel_svm_time], + "--", + label="rbf svm", +) # vertical line for dataset dimensionality = 64 accuracy.plot([64, 64], [0.7, 1], label="n_features") @@ -159,8 +174,8 @@ timescale.set_xlabel("Sampling steps = transformed feature dimension") accuracy.set_ylabel("Classification accuracy") timescale.set_ylabel("Training time in seconds") -accuracy.legend(loc='best') -timescale.legend(loc='best') +accuracy.legend(loc="best") +timescale.legend(loc="best") plt.tight_layout() plt.show() @@ -182,7 +197,7 @@ # visualize the decision surface, projected down to the first # two principal components of the dataset -pca = PCA(n_components=8).fit(data_train) +pca = PCA(n_components=8, random_state=42).fit(data_train) X = pca.transform(data_train) @@ -197,17 +212,16 @@ flat_grid = grid.reshape(-1, data.shape[1]) # title for the plots -titles = ['SVC with rbf kernel', - 'SVC (linear kernel)\n with Fourier rbf feature map\n' - 'n_components=100', - 'SVC (linear kernel)\n with Nystroem rbf feature map\n' - 'n_components=100'] +titles = [ + "SVC with rbf kernel", + "SVC (linear kernel)\n with Fourier rbf feature map\nn_components=100", + "SVC (linear kernel)\n with Nystroem rbf feature map\nn_components=100", +] plt.figure(figsize=(18, 7.5)) -plt.rcParams.update({'font.size': 14}) +plt.rcParams.update({"font.size": 14}) # predict and plot -for i, clf in enumerate((kernel_svm, nystroem_approx_svm, - fourier_approx_svm)): +for i, clf in enumerate((kernel_svm, nystroem_approx_svm, fourier_approx_svm)): # Plot the decision boundary. For that, we will assign a color to each # point in the mesh [x_min, x_max]x[y_min, y_max]. plt.subplot(1, 3, i + 1) @@ -215,12 +229,30 @@ # Put the result into a color plot Z = Z.reshape(grid.shape[:-1]) - plt.contourf(multiples, multiples, Z, cmap=plt.cm.Paired) - plt.axis('off') + levels = np.arange(10) + lv_eps = 0.01 # Adjust a mapping from calculated contour levels to color. + plt.contourf( + multiples, + multiples, + Z, + levels=levels - lv_eps, + cmap=plt.cm.tab10, + vmin=0, + vmax=10, + alpha=0.7, + ) + plt.axis("off") # Plot also the training points - plt.scatter(X[:, 0], X[:, 1], c=targets_train, cmap=plt.cm.Paired, - edgecolors=(0, 0, 0)) + plt.scatter( + X[:, 0], + X[:, 1], + c=targets_train, + cmap=plt.cm.tab10, + edgecolors=(0, 0, 0), + vmin=0, + vmax=10, + ) plt.title(titles[i]) plt.tight_layout() diff --git a/examples/miscellaneous/plot_kernel_ridge_regression.py b/examples/miscellaneous/plot_kernel_ridge_regression.py index 1eb84d8fdac81..13c2b184c2d30 100644 --- a/examples/miscellaneous/plot_kernel_ridge_regression.py +++ b/examples/miscellaneous/plot_kernel_ridge_regression.py @@ -14,41 +14,21 @@ This example illustrates both methods on an artificial dataset, which consists of a sinusoidal target function and strong noise added to every fifth -datapoint. The first figure compares the learned model of KRR and SVR when both -complexity/regularization and bandwidth of the RBF kernel are optimized using -grid-search. The learned functions are very similar; however, fitting KRR is -approx. seven times faster than fitting SVR (both with grid-search). However, -prediction of 100000 target values is more than tree times faster with SVR -since it has learned a sparse model using only approx. 1/3 of the 100 training -datapoints as support vectors. - -The next figure compares the time for fitting and prediction of KRR and SVR for -different sizes of the training set. Fitting KRR is faster than SVR for medium- -sized training sets (less than 1000 samples); however, for larger training sets -SVR scales better. With regard to prediction time, SVR is faster than -KRR for all sizes of the training set because of the learned sparse -solution. Note that the degree of sparsity and thus the prediction time depends -on the parameters epsilon and C of the SVR. -""" +datapoint. -# Authors: Jan Hendrik Metzen -# License: BSD 3 clause +""" +# %% +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import time - +# %% +# Generate sample data +# -------------------- import numpy as np -from sklearn.svm import SVR -from sklearn.model_selection import GridSearchCV -from sklearn.model_selection import learning_curve -from sklearn.kernel_ridge import KernelRidge -import matplotlib.pyplot as plt +rng = np.random.RandomState(42) -rng = np.random.RandomState(0) - -# ############################################################################# -# Generate sample data X = 5 * rng.rand(10000, 1) y = np.sin(X).ravel() @@ -57,28 +37,43 @@ X_plot = np.linspace(0, 5, 100000)[:, None] -# ############################################################################# -# Fit regression model +# %% +# Construct the kernel-based regression models +# -------------------------------------------- + +from sklearn.kernel_ridge import KernelRidge +from sklearn.model_selection import GridSearchCV +from sklearn.svm import SVR + train_size = 100 -svr = GridSearchCV(SVR(kernel='rbf', gamma=0.1), - param_grid={"C": [1e0, 1e1, 1e2, 1e3], - "gamma": np.logspace(-2, 2, 5)}) -kr = GridSearchCV(KernelRidge(kernel='rbf', gamma=0.1), - param_grid={"alpha": [1e0, 0.1, 1e-2, 1e-3], - "gamma": np.logspace(-2, 2, 5)}) +svr = GridSearchCV( + SVR(kernel="rbf", gamma=0.1), + param_grid={"C": [1e0, 1e1, 1e2, 1e3], "gamma": np.logspace(-2, 2, 5)}, +) + +kr = GridSearchCV( + KernelRidge(kernel="rbf", gamma=0.1), + param_grid={"alpha": [1e0, 0.1, 1e-2, 1e-3], "gamma": np.logspace(-2, 2, 5)}, +) + +# %% +# Compare times of SVR and Kernel Ridge Regression +# ------------------------------------------------ + +import time t0 = time.time() svr.fit(X[:train_size], y[:train_size]) svr_fit = time.time() - t0 -print("SVR complexity and bandwidth selected and model fitted in %.3f s" - % svr_fit) +print(f"Best SVR with params: {svr.best_params_} and R2 score: {svr.best_score_:.3f}") +print("SVR complexity and bandwidth selected and model fitted in %.3f s" % svr_fit) t0 = time.time() kr.fit(X[:train_size], y[:train_size]) kr_fit = time.time() - t0 -print("KRR complexity and bandwidth selected and model fitted in %.3f s" - % kr_fit) +print(f"Best KRR with params: {kr.best_params_} and R2 score: {kr.best_score_:.3f}") +print("KRR complexity and bandwidth selected and model fitted in %.3f s" % kr_fit) sv_ratio = svr.best_estimator_.support_.shape[0] / train_size print("Support vector ratio: %.3f" % sv_ratio) @@ -86,43 +81,69 @@ t0 = time.time() y_svr = svr.predict(X_plot) svr_predict = time.time() - t0 -print("SVR prediction for %d inputs in %.3f s" - % (X_plot.shape[0], svr_predict)) +print("SVR prediction for %d inputs in %.3f s" % (X_plot.shape[0], svr_predict)) t0 = time.time() y_kr = kr.predict(X_plot) kr_predict = time.time() - t0 -print("KRR prediction for %d inputs in %.3f s" - % (X_plot.shape[0], kr_predict)) - +print("KRR prediction for %d inputs in %.3f s" % (X_plot.shape[0], kr_predict)) -# ############################################################################# +# %% # Look at the results +# ------------------- + +import matplotlib.pyplot as plt + sv_ind = svr.best_estimator_.support_ -plt.scatter(X[sv_ind], y[sv_ind], c='r', s=50, label='SVR support vectors', - zorder=2, edgecolors=(0, 0, 0)) -plt.scatter(X[:100], y[:100], c='k', label='data', zorder=1, - edgecolors=(0, 0, 0)) -plt.plot(X_plot, y_svr, c='r', - label='SVR (fit: %.3fs, predict: %.3fs)' % (svr_fit, svr_predict)) -plt.plot(X_plot, y_kr, c='g', - label='KRR (fit: %.3fs, predict: %.3fs)' % (kr_fit, kr_predict)) -plt.xlabel('data') -plt.ylabel('target') -plt.title('SVR versus Kernel Ridge') -plt.legend() - -# Visualize training and prediction time +plt.scatter( + X[sv_ind], + y[sv_ind], + c="r", + s=50, + label="SVR support vectors", + zorder=2, + edgecolors=(0, 0, 0), +) +plt.scatter(X[:100], y[:100], c="k", label="data", zorder=1, edgecolors=(0, 0, 0)) +plt.plot( + X_plot, + y_svr, + c="r", + label="SVR (fit: %.3fs, predict: %.3fs)" % (svr_fit, svr_predict), +) +plt.plot( + X_plot, y_kr, c="g", label="KRR (fit: %.3fs, predict: %.3fs)" % (kr_fit, kr_predict) +) +plt.xlabel("data") +plt.ylabel("target") +plt.title("SVR versus Kernel Ridge") +_ = plt.legend() + +# %% +# The previous figure compares the learned model of KRR and SVR when both +# complexity/regularization and bandwidth of the RBF kernel are optimized using +# grid-search. The learned functions are very similar; however, fitting KRR is +# approximately 3-4 times faster than fitting SVR (both with grid-search). +# +# Prediction of 100000 target values could be in theory approximately three +# times faster with SVR since it has learned a sparse model using only +# approximately 1/3 of the training datapoints as support vectors. However, in +# practice, this is not necessarily the case because of implementation details +# in the way the kernel function is computed for each model that can make the +# KRR model as fast or even faster despite computing more arithmetic +# operations. + +# %% +# Visualize training and prediction times +# --------------------------------------- + plt.figure() -# Generate sample data -X = 5 * rng.rand(10000, 1) -y = np.sin(X).ravel() -y[::5] += 3 * (0.5 - rng.rand(X.shape[0] // 5)) -sizes = np.logspace(1, 4, 7).astype(int) -for name, estimator in {"KRR": KernelRidge(kernel='rbf', alpha=0.1, - gamma=10), - "SVR": SVR(kernel='rbf', C=1e1, gamma=10)}.items(): +sizes = np.logspace(1, 3.8, 7).astype(int) +for name, estimator in { + "KRR": KernelRidge(kernel="rbf", alpha=0.01, gamma=10), + "SVR": SVR(kernel="rbf", C=1e2, gamma=10), +}.items(): train_time = [] test_time = [] for train_test_size in sizes: @@ -134,37 +155,63 @@ estimator.predict(X_plot[:1000]) test_time.append(time.time() - t0) - plt.plot(sizes, train_time, 'o-', color="r" if name == "SVR" else "g", - label="%s (train)" % name) - plt.plot(sizes, test_time, 'o--', color="r" if name == "SVR" else "g", - label="%s (test)" % name) + plt.plot( + sizes, + train_time, + "o-", + color="r" if name == "SVR" else "g", + label="%s (train)" % name, + ) + plt.plot( + sizes, + test_time, + "o--", + color="r" if name == "SVR" else "g", + label="%s (test)" % name, + ) plt.xscale("log") plt.yscale("log") plt.xlabel("Train size") plt.ylabel("Time (seconds)") -plt.title('Execution Time') -plt.legend(loc="best") - -# Visualize learning curves -plt.figure() - -svr = SVR(kernel='rbf', C=1e1, gamma=0.1) -kr = KernelRidge(kernel='rbf', alpha=0.1, gamma=0.1) -train_sizes, train_scores_svr, test_scores_svr = \ - learning_curve(svr, X[:100], y[:100], train_sizes=np.linspace(0.1, 1, 10), - scoring="neg_mean_squared_error", cv=10) -train_sizes_abs, train_scores_kr, test_scores_kr = \ - learning_curve(kr, X[:100], y[:100], train_sizes=np.linspace(0.1, 1, 10), - scoring="neg_mean_squared_error", cv=10) - -plt.plot(train_sizes, -test_scores_svr.mean(1), 'o-', color="r", - label="SVR") -plt.plot(train_sizes, -test_scores_kr.mean(1), 'o-', color="g", - label="KRR") -plt.xlabel("Train size") -plt.ylabel("Mean Squared Error") -plt.title('Learning curves') -plt.legend(loc="best") +plt.title("Execution Time") +_ = plt.legend(loc="best") + +# %% +# This figure compares the time for fitting and prediction of KRR and SVR for +# different sizes of the training set. Fitting KRR is faster than SVR for +# medium-sized training sets (less than a few thousand samples); however, for +# larger training sets SVR scales better. With regard to prediction time, SVR +# should be faster than KRR for all sizes of the training set because of the +# learned sparse solution, however this is not necessarily the case in practice +# because of implementation details. Note that the degree of sparsity and thus +# the prediction time depends on the parameters epsilon and C of the SVR. + +# %% +# Visualize the learning curves +# ----------------------------- +from sklearn.model_selection import LearningCurveDisplay + +_, ax = plt.subplots() + +svr = SVR(kernel="rbf", C=1e1, gamma=0.1) +kr = KernelRidge(kernel="rbf", alpha=0.1, gamma=0.1) + +common_params = { + "X": X[:100], + "y": y[:100], + "train_sizes": np.linspace(0.1, 1, 10), + "scoring": "neg_mean_squared_error", + "negate_score": True, + "score_name": "Mean Squared Error", + "score_type": "test", + "std_display_style": None, + "ax": ax, +} + +LearningCurveDisplay.from_estimator(svr, **common_params) +LearningCurveDisplay.from_estimator(kr, **common_params) +ax.set_title("Learning curves") +ax.legend(handles=ax.get_legend_handles_labels()[0], labels=["SVR", "KRR"]) plt.show() diff --git a/examples/miscellaneous/plot_metadata_routing.py b/examples/miscellaneous/plot_metadata_routing.py new file mode 100644 index 0000000000000..634ca304d125d --- /dev/null +++ b/examples/miscellaneous/plot_metadata_routing.py @@ -0,0 +1,719 @@ +""" +================ +Metadata Routing +================ + +.. currentmodule:: sklearn + +This document shows how you can use the :ref:`metadata routing mechanism +` in scikit-learn to route metadata to the estimators, +scorers, and CV splitters consuming them. + +To better understand the following document, we need to introduce two concepts: +routers and consumers. A router is an object which forwards some given data and +metadata to other objects. In most cases, a router is a :term:`meta-estimator`, +i.e. an estimator which takes another estimator as a parameter. A function such +as :func:`sklearn.model_selection.cross_validate` which takes an estimator as a +parameter and forwards data and metadata, is also a router. + +A consumer, on the other hand, is an object which accepts and uses some given +metadata. For instance, an estimator taking into account ``sample_weight`` in +its :term:`fit` method is a consumer of ``sample_weight``. + +It is possible for an object to be both a router and a consumer. For instance, +a meta-estimator may take into account ``sample_weight`` in certain +calculations, but it may also route it to the underlying estimator. + +First a few imports and some random data for the rest of the script. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% + +import warnings +from pprint import pprint + +import numpy as np + +from sklearn import set_config +from sklearn.base import ( + BaseEstimator, + ClassifierMixin, + MetaEstimatorMixin, + RegressorMixin, + TransformerMixin, + clone, +) +from sklearn.linear_model import LinearRegression +from sklearn.utils import metadata_routing +from sklearn.utils.metadata_routing import ( + MetadataRouter, + MethodMapping, + get_routing_for_object, + process_routing, +) +from sklearn.utils.validation import check_is_fitted + +n_samples, n_features = 100, 4 +rng = np.random.RandomState(42) +X = rng.rand(n_samples, n_features) +y = rng.randint(0, 2, size=n_samples) +my_groups = rng.randint(0, 10, size=n_samples) +my_weights = rng.rand(n_samples) +my_other_weights = rng.rand(n_samples) + +# %% +# Metadata routing is only available if explicitly enabled: +set_config(enable_metadata_routing=True) + + +# %% +# This utility function is a dummy to check if a metadata is passed: +def check_metadata(obj, **kwargs): + for key, value in kwargs.items(): + if value is not None: + print( + f"Received {key} of length = {len(value)} in {obj.__class__.__name__}." + ) + else: + print(f"{key} is None in {obj.__class__.__name__}.") + + +# %% +# A utility function to nicely print the routing information of an object: +def print_routing(obj): + pprint(obj.get_metadata_routing()._serialize()) + + +# %% +# Consuming Estimator +# ------------------- +# Here we demonstrate how an estimator can expose the required API to support +# metadata routing as a consumer. Imagine a simple classifier accepting +# ``sample_weight`` as a metadata on its ``fit`` and ``groups`` in its +# ``predict`` method: + + +class ExampleClassifier(ClassifierMixin, BaseEstimator): + def fit(self, X, y, sample_weight=None): + check_metadata(self, sample_weight=sample_weight) + # all classifiers need to expose a classes_ attribute once they're fit. + self.classes_ = np.array([0, 1]) + return self + + def predict(self, X, groups=None): + check_metadata(self, groups=groups) + # return a constant value of 1, not a very smart classifier! + return np.ones(len(X)) + + +# %% +# The above estimator now has all it needs to consume metadata. This is +# accomplished by some magic done in :class:`~base.BaseEstimator`. There are +# now three methods exposed by the above class: ``set_fit_request``, +# ``set_predict_request``, and ``get_metadata_routing``. There is also a +# ``set_score_request`` for ``sample_weight`` which is present since +# :class:`~base.ClassifierMixin` implements a ``score`` method accepting +# ``sample_weight``. The same applies to regressors which inherit from +# :class:`~base.RegressorMixin`. +# +# By default, no metadata is requested, which we can see as: + +print_routing(ExampleClassifier()) + +# %% +# The above output means that ``sample_weight`` and ``groups`` are not +# requested by `ExampleClassifier`, and if a router is given those metadata, it +# should raise an error, since the user has not explicitly set whether they are +# required or not. The same is true for ``sample_weight`` in the ``score`` +# method, which is inherited from :class:`~base.ClassifierMixin`. In order to +# explicitly set request values for those metadata, we can use these methods: + +est = ( + ExampleClassifier() + .set_fit_request(sample_weight=False) + .set_predict_request(groups=True) + .set_score_request(sample_weight=False) +) +print_routing(est) + +# %% +# .. note :: +# Please note that as long as the above estimator is not used in a +# meta-estimator, the user does not need to set any requests for the +# metadata and the set values are ignored, since a consumer does not +# validate or route given metadata. A simple usage of the above estimator +# would work as expected. + +est = ExampleClassifier() +est.fit(X, y, sample_weight=my_weights) +est.predict(X[:3, :], groups=my_groups) + +# %% +# Routing Meta-Estimator +# ---------------------- +# Now, we show how to design a meta-estimator to be a router. As a simplified +# example, here is a meta-estimator, which doesn't do much other than routing +# the metadata. + + +class MetaClassifier(MetaEstimatorMixin, ClassifierMixin, BaseEstimator): + def __init__(self, estimator): + self.estimator = estimator + + def get_metadata_routing(self): + # This method defines the routing for this meta-estimator. + # In order to do so, a `MetadataRouter` instance is created, and the + # routing is added to it. More explanations follow below. + router = MetadataRouter(owner=self.__class__.__name__).add( + estimator=self.estimator, + method_mapping=MethodMapping() + .add(caller="fit", callee="fit") + .add(caller="predict", callee="predict") + .add(caller="score", callee="score"), + ) + return router + + def fit(self, X, y, **fit_params): + # `get_routing_for_object` returns a copy of the `MetadataRouter` + # constructed by the above `get_metadata_routing` method, that is + # internally called. + request_router = get_routing_for_object(self) + # Meta-estimators are responsible for validating the given metadata. + # `method` refers to the parent's method, i.e. `fit` in this example. + request_router.validate_metadata(params=fit_params, method="fit") + # `MetadataRouter.route_params` maps the given metadata to the metadata + # required by the underlying estimator based on the routing information + # defined by the MetadataRouter. The output of type `Bunch` has a key + # for each consuming object and those hold keys for their consuming + # methods, which then contain key for the metadata which should be + # routed to them. + routed_params = request_router.route_params(params=fit_params, caller="fit") + + # A sub-estimator is fitted and its classes are attributed to the + # meta-estimator. + self.estimator_ = clone(self.estimator).fit(X, y, **routed_params.estimator.fit) + self.classes_ = self.estimator_.classes_ + return self + + def predict(self, X, **predict_params): + check_is_fitted(self) + # As in `fit`, we get a copy of the object's MetadataRouter, + request_router = get_routing_for_object(self) + # then we validate the given metadata, + request_router.validate_metadata(params=predict_params, method="predict") + # and then prepare the input to the underlying `predict` method. + routed_params = request_router.route_params( + params=predict_params, caller="predict" + ) + return self.estimator_.predict(X, **routed_params.estimator.predict) + + +# %% +# Let's break down different parts of the above code. +# +# First, the :meth:`~utils.metadata_routing.get_routing_for_object` takes our +# meta-estimator (``self``) and returns a +# :class:`~utils.metadata_routing.MetadataRouter` or, a +# :class:`~utils.metadata_routing.MetadataRequest` if the object is a consumer, +# based on the output of the estimator's ``get_metadata_routing`` method. +# +# Then in each method, we use the ``route_params`` method to construct a +# dictionary of the form ``{"object_name": {"method_name": {"metadata": +# value}}}`` to pass to the underlying estimator's method. The ``object_name`` +# (``estimator`` in the above ``routed_params.estimator.fit`` example) is the +# same as the one added in the ``get_metadata_routing``. ``validate_metadata`` +# makes sure all given metadata are requested to avoid silent bugs. +# +# Next, we illustrate the different behaviors and notably the type of errors +# raised. + +meta_est = MetaClassifier( + estimator=ExampleClassifier().set_fit_request(sample_weight=True) +) +meta_est.fit(X, y, sample_weight=my_weights) + +# %% +# Note that the above example is calling our utility function +# `check_metadata()` via the `ExampleClassifier`. It checks that +# ``sample_weight`` is correctly passed to it. If it is not, like in the +# following example, it would print that ``sample_weight`` is ``None``: + +meta_est.fit(X, y) + +# %% +# If we pass an unknown metadata, an error is raised: +try: + meta_est.fit(X, y, test=my_weights) +except TypeError as e: + print(e) + +# %% +# And if we pass a metadata which is not explicitly requested: +try: + meta_est.fit(X, y, sample_weight=my_weights).predict(X, groups=my_groups) +except ValueError as e: + print(e) + +# %% +# Also, if we explicitly set it as not requested, but it is provided: +meta_est = MetaClassifier( + estimator=ExampleClassifier() + .set_fit_request(sample_weight=True) + .set_predict_request(groups=False) +) +try: + meta_est.fit(X, y, sample_weight=my_weights).predict(X[:3, :], groups=my_groups) +except TypeError as e: + print(e) + +# %% +# Another concept to introduce is **aliased metadata**. This is when an +# estimator requests a metadata with a different variable name than the default +# variable name. For instance, in a setting where there are two estimators in a +# pipeline, one could request ``sample_weight1`` and the other +# ``sample_weight2``. Note that this doesn't change what the estimator expects, +# it only tells the meta-estimator how to map the provided metadata to what is +# required. Here's an example, where we pass ``aliased_sample_weight`` to the +# meta-estimator, but the meta-estimator understands that +# ``aliased_sample_weight`` is an alias for ``sample_weight``, and passes it as +# ``sample_weight`` to the underlying estimator: +meta_est = MetaClassifier( + estimator=ExampleClassifier().set_fit_request(sample_weight="aliased_sample_weight") +) +meta_est.fit(X, y, aliased_sample_weight=my_weights) + +# %% +# Passing ``sample_weight`` here will fail since it is requested with an +# alias and ``sample_weight`` with that name is not requested: +try: + meta_est.fit(X, y, sample_weight=my_weights) +except TypeError as e: + print(e) + +# %% +# This leads us to the ``get_metadata_routing``. The way routing works in +# scikit-learn is that consumers request what they need, and routers pass that +# along. Additionally, a router exposes what it requires itself so that it can +# be used inside another router, e.g. a pipeline inside a grid search object. +# The output of the ``get_metadata_routing`` which is a dictionary +# representation of a :class:`~utils.metadata_routing.MetadataRouter`, includes +# the complete tree of requested metadata by all nested objects and their +# corresponding method routings, i.e. which method of a sub-estimator is used +# in which method of a meta-estimator: + +print_routing(meta_est) + +# %% +# As you can see, the only metadata requested for method ``fit`` is +# ``"sample_weight"`` with ``"aliased_sample_weight"`` as the alias. The +# ``~utils.metadata_routing.MetadataRouter`` class enables us to easily create +# the routing object which would create the output we need for our +# ``get_metadata_routing``. +# +# In order to understand how aliases work in meta-estimators, imagine our +# meta-estimator inside another one: + +meta_meta_est = MetaClassifier(estimator=meta_est).fit( + X, y, aliased_sample_weight=my_weights +) + +# %% +# In the above example, this is how the ``fit`` method of `meta_meta_est` +# will call their sub-estimator's ``fit`` methods:: +# +# # user feeds `my_weights` as `aliased_sample_weight` into `meta_meta_est`: +# meta_meta_est.fit(X, y, aliased_sample_weight=my_weights): +# ... +# +# # the first sub-estimator (`meta_est`) expects `aliased_sample_weight` +# self.estimator_.fit(X, y, aliased_sample_weight=aliased_sample_weight): +# ... +# +# # the second sub-estimator (`est`) expects `sample_weight` +# self.estimator_.fit(X, y, sample_weight=aliased_sample_weight): +# ... + +# %% +# Consuming and routing Meta-Estimator +# ------------------------------------ +# For a slightly more complex example, consider a meta-estimator that routes +# metadata to an underlying estimator as before, but it also uses some metadata +# in its own methods. This meta-estimator is a consumer and a router at the +# same time. Implementing one is very similar to what we had before, but with a +# few tweaks. + + +class RouterConsumerClassifier(MetaEstimatorMixin, ClassifierMixin, BaseEstimator): + def __init__(self, estimator): + self.estimator = estimator + + def get_metadata_routing(self): + router = ( + MetadataRouter(owner=self.__class__.__name__) + # defining metadata routing request values for usage in the meta-estimator + .add_self_request(self) + # defining metadata routing request values for usage in the sub-estimator + .add( + estimator=self.estimator, + method_mapping=MethodMapping() + .add(caller="fit", callee="fit") + .add(caller="predict", callee="predict") + .add(caller="score", callee="score"), + ) + ) + return router + + # Since `sample_weight` is used and consumed here, it should be defined as + # an explicit argument in the method's signature. All other metadata which + # are only routed, will be passed as `**fit_params`: + def fit(self, X, y, sample_weight, **fit_params): + if self.estimator is None: + raise ValueError("estimator cannot be None!") + + check_metadata(self, sample_weight=sample_weight) + + # We add `sample_weight` to the `fit_params` dictionary. + if sample_weight is not None: + fit_params["sample_weight"] = sample_weight + + request_router = get_routing_for_object(self) + request_router.validate_metadata(params=fit_params, method="fit") + routed_params = request_router.route_params(params=fit_params, caller="fit") + self.estimator_ = clone(self.estimator).fit(X, y, **routed_params.estimator.fit) + self.classes_ = self.estimator_.classes_ + return self + + def predict(self, X, **predict_params): + check_is_fitted(self) + # As in `fit`, we get a copy of the object's MetadataRouter, + request_router = get_routing_for_object(self) + # we validate the given metadata, + request_router.validate_metadata(params=predict_params, method="predict") + # and then prepare the input to the underlying ``predict`` method. + routed_params = request_router.route_params( + params=predict_params, caller="predict" + ) + return self.estimator_.predict(X, **routed_params.estimator.predict) + + +# %% +# The key parts where the above meta-estimator differs from our previous +# meta-estimator is accepting ``sample_weight`` explicitly in ``fit`` and +# including it in ``fit_params``. Since ``sample_weight`` is an explicit +# argument, we can be sure that ``set_fit_request(sample_weight=...)`` is +# present for this method. The meta-estimator is both a consumer, as well as a +# router of ``sample_weight``. +# +# In ``get_metadata_routing``, we add ``self`` to the routing using +# ``add_self_request`` to indicate this estimator is consuming +# ``sample_weight`` as well as being a router; which also adds a +# ``$self_request`` key to the routing info as illustrated below. Now let's +# look at some examples: + +# %% +# - No metadata requested +meta_est = RouterConsumerClassifier(estimator=ExampleClassifier()) +print_routing(meta_est) + + +# %% +# - ``sample_weight`` requested by sub-estimator +meta_est = RouterConsumerClassifier( + estimator=ExampleClassifier().set_fit_request(sample_weight=True) +) +print_routing(meta_est) + +# %% +# - ``sample_weight`` requested by meta-estimator +meta_est = RouterConsumerClassifier(estimator=ExampleClassifier()).set_fit_request( + sample_weight=True +) +print_routing(meta_est) + +# %% +# Note the difference in the requested metadata representations above. +# +# - We can also alias the metadata to pass different values to the fit methods +# of the meta- and the sub-estimator: + +meta_est = RouterConsumerClassifier( + estimator=ExampleClassifier().set_fit_request(sample_weight="clf_sample_weight"), +).set_fit_request(sample_weight="meta_clf_sample_weight") +print_routing(meta_est) + +# %% +# However, ``fit`` of the meta-estimator only needs the alias for the +# sub-estimator and addresses their own sample weight as `sample_weight`, since +# it doesn't validate and route its own required metadata: +meta_est.fit(X, y, sample_weight=my_weights, clf_sample_weight=my_other_weights) + +# %% +# - Alias only on the sub-estimator: +# +# This is useful when we don't want the meta-estimator to use the metadata, but +# the sub-estimator should. +meta_est = RouterConsumerClassifier( + estimator=ExampleClassifier().set_fit_request(sample_weight="aliased_sample_weight") +) +print_routing(meta_est) +# %% +# The meta-estimator cannot use `aliased_sample_weight`, because it expects +# it passed as `sample_weight`. This would apply even if +# `set_fit_request(sample_weight=True)` was set on it. + +# %% +# Simple Pipeline +# --------------- +# A slightly more complicated use-case is a meta-estimator resembling a +# :class:`~pipeline.Pipeline`. Here is a meta-estimator, which accepts a +# transformer and a classifier. When calling its `fit` method, it applies the +# transformer's `fit` and `transform` before running the classifier on the +# transformed data. Upon `predict`, it applies the transformer's `transform` +# before predicting with the classifier's `predict` method on the transformed +# new data. + + +class SimplePipeline(ClassifierMixin, BaseEstimator): + def __init__(self, transformer, classifier): + self.transformer = transformer + self.classifier = classifier + + def get_metadata_routing(self): + router = ( + MetadataRouter(owner=self.__class__.__name__) + # We add the routing for the transformer. + .add( + transformer=self.transformer, + method_mapping=MethodMapping() + # The metadata is routed such that it retraces how + # `SimplePipeline` internally calls the transformer's `fit` and + # `transform` methods in its own methods (`fit` and `predict`). + .add(caller="fit", callee="fit") + .add(caller="fit", callee="transform") + .add(caller="predict", callee="transform"), + ) + # We add the routing for the classifier. + .add( + classifier=self.classifier, + method_mapping=MethodMapping() + .add(caller="fit", callee="fit") + .add(caller="predict", callee="predict"), + ) + ) + return router + + def fit(self, X, y, **fit_params): + routed_params = process_routing(self, "fit", **fit_params) + + self.transformer_ = clone(self.transformer).fit( + X, y, **routed_params.transformer.fit + ) + X_transformed = self.transformer_.transform( + X, **routed_params.transformer.transform + ) + + self.classifier_ = clone(self.classifier).fit( + X_transformed, y, **routed_params.classifier.fit + ) + return self + + def predict(self, X, **predict_params): + routed_params = process_routing(self, "predict", **predict_params) + + X_transformed = self.transformer_.transform( + X, **routed_params.transformer.transform + ) + return self.classifier_.predict( + X_transformed, **routed_params.classifier.predict + ) + + +# %% +# Note the usage of :class:`~utils.metadata_routing.MethodMapping` to +# declare which methods of the child estimator (callee) are used in which +# methods of the meta estimator (caller). As you can see, `SimplePipeline` uses +# the transformer's ``transform`` and ``fit`` methods in ``fit``, and its +# ``transform`` method in ``predict``, and that's what you see implemented in +# the routing structure of the pipeline class. +# +# Another difference in the above example with the previous ones is the usage +# of :func:`~utils.metadata_routing.process_routing`, which processes the input +# parameters, does the required validation, and returns the `routed_params` +# which we had created in previous examples. This reduces the boilerplate code +# a developer needs to write in each meta-estimator's method. Developers are +# strongly recommended to use this function unless there is a good reason +# against it. +# +# In order to test the above pipeline, let's add an example transformer. + + +class ExampleTransformer(TransformerMixin, BaseEstimator): + def fit(self, X, y, sample_weight=None): + check_metadata(self, sample_weight=sample_weight) + return self + + def transform(self, X, groups=None): + check_metadata(self, groups=groups) + return X + + def fit_transform(self, X, y, sample_weight=None, groups=None): + return self.fit(X, y, sample_weight).transform(X, groups) + + +# %% +# Note that in the above example, we have implemented ``fit_transform`` which +# calls ``fit`` and ``transform`` with the appropriate metadata. This is only +# required if ``transform`` accepts metadata, since the default ``fit_transform`` +# implementation in :class:`~base.TransformerMixin` doesn't pass metadata to +# ``transform``. +# +# Now we can test our pipeline, and see if metadata is correctly passed around. +# This example uses our `SimplePipeline`, our `ExampleTransformer`, and our +# `RouterConsumerClassifier` which uses our `ExampleClassifier`. + +pipe = SimplePipeline( + transformer=ExampleTransformer() + # we set transformer's fit to receive sample_weight + .set_fit_request(sample_weight=True) + # we set transformer's transform to receive groups + .set_transform_request(groups=True), + classifier=RouterConsumerClassifier( + estimator=ExampleClassifier() + # we want this sub-estimator to receive sample_weight in fit + .set_fit_request(sample_weight=True) + # but not groups in predict + .set_predict_request(groups=False), + ) + # and we want the meta-estimator to receive sample_weight as well + .set_fit_request(sample_weight=True), +) +pipe.fit(X, y, sample_weight=my_weights, groups=my_groups).predict( + X[:3], groups=my_groups +) + +# %% +# Deprecation / Default Value Change +# ---------------------------------- +# In this section we show how one should handle the case where a router becomes +# also a consumer, especially when it consumes the same metadata as its +# sub-estimator, or a consumer starts consuming a metadata which it wasn't in +# an older release. In this case, a warning should be raised for a while, to +# let users know the behavior is changed from previous versions. + + +class MetaRegressor(MetaEstimatorMixin, RegressorMixin, BaseEstimator): + def __init__(self, estimator): + self.estimator = estimator + + def fit(self, X, y, **fit_params): + routed_params = process_routing(self, "fit", **fit_params) + self.estimator_ = clone(self.estimator).fit(X, y, **routed_params.estimator.fit) + + def get_metadata_routing(self): + router = MetadataRouter(owner=self.__class__.__name__).add( + estimator=self.estimator, + method_mapping=MethodMapping().add(caller="fit", callee="fit"), + ) + return router + + +# %% +# As explained above, this is a valid usage if `my_weights` aren't supposed +# to be passed as `sample_weight` to `MetaRegressor`: + +reg = MetaRegressor(estimator=LinearRegression().set_fit_request(sample_weight=True)) +reg.fit(X, y, sample_weight=my_weights) + + +# %% +# Now imagine we further develop ``MetaRegressor`` and it now also *consumes* +# ``sample_weight``: + + +class WeightedMetaRegressor(MetaEstimatorMixin, RegressorMixin, BaseEstimator): + # show warning to remind user to explicitly set the value with + # `.set_{method}_request(sample_weight={boolean})` + __metadata_request__fit = {"sample_weight": metadata_routing.WARN} + + def __init__(self, estimator): + self.estimator = estimator + + def fit(self, X, y, sample_weight=None, **fit_params): + routed_params = process_routing( + self, "fit", sample_weight=sample_weight, **fit_params + ) + check_metadata(self, sample_weight=sample_weight) + self.estimator_ = clone(self.estimator).fit(X, y, **routed_params.estimator.fit) + + def get_metadata_routing(self): + router = ( + MetadataRouter(owner=self.__class__.__name__) + .add_self_request(self) + .add( + estimator=self.estimator, + method_mapping=MethodMapping().add(caller="fit", callee="fit"), + ) + ) + return router + + +# %% +# The above implementation is almost the same as ``MetaRegressor``, and +# because of the default request value defined in ``__metadata_request__fit`` +# there is a warning raised when fitted. + +with warnings.catch_warnings(record=True) as record: + WeightedMetaRegressor( + estimator=LinearRegression().set_fit_request(sample_weight=False) + ).fit(X, y, sample_weight=my_weights) +for w in record: + print(w.message) + + +# %% +# When an estimator consumes a metadata which it didn't consume before, the +# following pattern can be used to warn the users about it. + + +class ExampleRegressor(RegressorMixin, BaseEstimator): + __metadata_request__fit = {"sample_weight": metadata_routing.WARN} + + def fit(self, X, y, sample_weight=None): + check_metadata(self, sample_weight=sample_weight) + return self + + def predict(self, X): + return np.zeros(shape=(len(X))) + + +with warnings.catch_warnings(record=True) as record: + MetaRegressor(estimator=ExampleRegressor()).fit(X, y, sample_weight=my_weights) +for w in record: + print(w.message) + +# %% +# At the end we disable the configuration flag for metadata routing: + +set_config(enable_metadata_routing=False) + +# %% +# Third Party Development and scikit-learn Dependency +# --------------------------------------------------- +# +# As seen above, information is communicated between classes using +# :class:`~utils.metadata_routing.MetadataRequest` and +# :class:`~utils.metadata_routing.MetadataRouter`. It is strongly not advised, +# but possible to vendor the tools related to metadata-routing if you strictly +# want to have a scikit-learn compatible estimator, without depending on the +# scikit-learn package. If all of the following conditions are met, you do NOT +# need to modify your code at all: +# +# - your estimator inherits from :class:`~base.BaseEstimator` +# - the parameters consumed by your estimator's methods, e.g. ``fit``, are +# explicitly defined in the method's signature, as opposed to being +# ``*args`` or ``*kwargs``. +# - your estimator does not route any metadata to the underlying objects, i.e. +# it's not a *router*. diff --git a/examples/miscellaneous/plot_multilabel.py b/examples/miscellaneous/plot_multilabel.py index 828ca17fafa23..4c88dbe1838f2 100644 --- a/examples/miscellaneous/plot_multilabel.py +++ b/examples/miscellaneous/plot_multilabel.py @@ -1,5 +1,3 @@ -# Authors: Vlad Niculae, Mathieu Blondel -# License: BSD 3 clause """ ========================= Multilabel classification @@ -8,10 +6,10 @@ This example simulates a multi-label document classification problem. The dataset is generated randomly based on the following process: - - pick the number of labels: n ~ Poisson(n_labels) - - n times, choose a class c: c ~ Multinomial(theta) - - pick the document length: k ~ Poisson(length) - - k times, choose a word: w ~ Multinomial(theta_c) +- pick the number of labels: n ~ Poisson(n_labels) +- n times, choose a class c: c ~ Multinomial(theta) +- pick the document length: k ~ Poisson(length) +- k times, choose a word: w ~ Multinomial(theta_c) In the above process, rejection sampling is used to make sure that n is more than 2, and that the document length is never zero. Likewise, we reject classes @@ -28,17 +26,20 @@ Note: in the plot, "unlabeled samples" does not mean that we don't know the labels (as in semi-supervised learning) but that the samples simply do *not* have a label. + """ -print(__doc__) -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt +import numpy as np +from sklearn.cross_decomposition import CCA from sklearn.datasets import make_multilabel_classification +from sklearn.decomposition import PCA from sklearn.multiclass import OneVsRestClassifier from sklearn.svm import SVC -from sklearn.decomposition import PCA -from sklearn.cross_decomposition import CCA def plot_hyperplane(clf, min_x, max_x, linestyle, label): @@ -64,50 +65,66 @@ def plot_subfigure(X, Y, subplot, title, transform): min_y = np.min(X[:, 1]) max_y = np.max(X[:, 1]) - classif = OneVsRestClassifier(SVC(kernel='linear')) + classif = OneVsRestClassifier(SVC(kernel="linear")) classif.fit(X, Y) plt.subplot(2, 2, subplot) plt.title(title) - zero_class = np.where(Y[:, 0]) - one_class = np.where(Y[:, 1]) - plt.scatter(X[:, 0], X[:, 1], s=40, c='gray', edgecolors=(0, 0, 0)) - plt.scatter(X[zero_class, 0], X[zero_class, 1], s=160, edgecolors='b', - facecolors='none', linewidths=2, label='Class 1') - plt.scatter(X[one_class, 0], X[one_class, 1], s=80, edgecolors='orange', - facecolors='none', linewidths=2, label='Class 2') - - plot_hyperplane(classif.estimators_[0], min_x, max_x, 'k--', - 'Boundary\nfor class 1') - plot_hyperplane(classif.estimators_[1], min_x, max_x, 'k-.', - 'Boundary\nfor class 2') + zero_class = (Y[:, 0]).nonzero() + one_class = (Y[:, 1]).nonzero() + plt.scatter(X[:, 0], X[:, 1], s=40, c="gray", edgecolors=(0, 0, 0)) + plt.scatter( + X[zero_class, 0], + X[zero_class, 1], + s=160, + edgecolors="b", + facecolors="none", + linewidths=2, + label="Class 1", + ) + plt.scatter( + X[one_class, 0], + X[one_class, 1], + s=80, + edgecolors="orange", + facecolors="none", + linewidths=2, + label="Class 2", + ) + + plot_hyperplane( + classif.estimators_[0], min_x, max_x, "k--", "Boundary\nfor class 1" + ) + plot_hyperplane( + classif.estimators_[1], min_x, max_x, "k-.", "Boundary\nfor class 2" + ) plt.xticks(()) plt.yticks(()) - plt.xlim(min_x - .5 * max_x, max_x + .5 * max_x) - plt.ylim(min_y - .5 * max_y, max_y + .5 * max_y) + plt.xlim(min_x - 0.5 * max_x, max_x + 0.5 * max_x) + plt.ylim(min_y - 0.5 * max_y, max_y + 0.5 * max_y) if subplot == 2: - plt.xlabel('First principal component') - plt.ylabel('Second principal component') + plt.xlabel("First principal component") + plt.ylabel("Second principal component") plt.legend(loc="upper left") plt.figure(figsize=(8, 6)) -X, Y = make_multilabel_classification(n_classes=2, n_labels=1, - allow_unlabeled=True, - random_state=1) +X, Y = make_multilabel_classification( + n_classes=2, n_labels=1, allow_unlabeled=True, random_state=1 +) plot_subfigure(X, Y, 1, "With unlabeled samples + CCA", "cca") plot_subfigure(X, Y, 2, "With unlabeled samples + PCA", "pca") -X, Y = make_multilabel_classification(n_classes=2, n_labels=1, - allow_unlabeled=False, - random_state=1) +X, Y = make_multilabel_classification( + n_classes=2, n_labels=1, allow_unlabeled=False, random_state=1 +) plot_subfigure(X, Y, 3, "Without unlabeled samples + CCA", "cca") plot_subfigure(X, Y, 4, "Without unlabeled samples + PCA", "pca") -plt.subplots_adjust(.04, .02, .97, .94, .09, .2) +plt.subplots_adjust(0.04, 0.02, 0.97, 0.94, 0.09, 0.2) plt.show() diff --git a/examples/miscellaneous/plot_multioutput_face_completion.py b/examples/miscellaneous/plot_multioutput_face_completion.py index 62fd20d24645f..a924da0d2b4a5 100644 --- a/examples/miscellaneous/plot_multioutput_face_completion.py +++ b/examples/miscellaneous/plot_multioutput_face_completion.py @@ -11,18 +11,18 @@ regression and ridge regression complete the lower half of those faces. """ -print(__doc__) -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt +import numpy as np from sklearn.datasets import fetch_olivetti_faces -from sklearn.utils.validation import check_random_state - from sklearn.ensemble import ExtraTreesRegressor +from sklearn.linear_model import LinearRegression, RidgeCV from sklearn.neighbors import KNeighborsRegressor -from sklearn.linear_model import LinearRegression -from sklearn.linear_model import RidgeCV +from sklearn.utils.validation import check_random_state # Load the faces datasets data, targets = fetch_olivetti_faces(return_X_y=True) @@ -33,21 +33,22 @@ # Test on a subset of people n_faces = 5 rng = check_random_state(4) -face_ids = rng.randint(test.shape[0], size=(n_faces, )) +face_ids = rng.randint(test.shape[0], size=(n_faces,)) test = test[face_ids, :] n_pixels = data.shape[1] # Upper half of the faces -X_train = train[:, :(n_pixels + 1) // 2] +X_train = train[:, : (n_pixels + 1) // 2] # Lower half of the faces -y_train = train[:, n_pixels // 2:] -X_test = test[:, :(n_pixels + 1) // 2] -y_test = test[:, n_pixels // 2:] +y_train = train[:, n_pixels // 2 :] +X_test = test[:, : (n_pixels + 1) // 2] +y_test = test[:, n_pixels // 2 :] # Fit estimators ESTIMATORS = { - "Extra trees": ExtraTreesRegressor(n_estimators=10, max_features=32, - random_state=0), + "Extra trees": ExtraTreesRegressor( + n_estimators=10, max_features=32, random_state=0 + ), "K-nn": KNeighborsRegressor(), "Linear regression": LinearRegression(), "Ridge": RidgeCV(), @@ -62,7 +63,7 @@ image_shape = (64, 64) n_cols = 1 + len(ESTIMATORS) -plt.figure(figsize=(2. * n_cols, 2.26 * n_faces)) +plt.figure(figsize=(2.0 * n_cols, 2.26 * n_faces)) plt.suptitle("Face completion with multi-output estimators", size=16) for i in range(n_faces): @@ -71,13 +72,12 @@ if i: sub = plt.subplot(n_faces, n_cols, i * n_cols + 1) else: - sub = plt.subplot(n_faces, n_cols, i * n_cols + 1, - title="true faces") + sub = plt.subplot(n_faces, n_cols, i * n_cols + 1, title="true faces") sub.axis("off") - sub.imshow(true_face.reshape(image_shape), - cmap=plt.cm.gray, - interpolation="nearest") + sub.imshow( + true_face.reshape(image_shape), cmap=plt.cm.gray, interpolation="nearest" + ) for j, est in enumerate(sorted(ESTIMATORS)): completed_face = np.hstack((X_test[i], y_test_predict[est][i])) @@ -86,12 +86,13 @@ sub = plt.subplot(n_faces, n_cols, i * n_cols + 2 + j) else: - sub = plt.subplot(n_faces, n_cols, i * n_cols + 2 + j, - title=est) + sub = plt.subplot(n_faces, n_cols, i * n_cols + 2 + j, title=est) sub.axis("off") - sub.imshow(completed_face.reshape(image_shape), - cmap=plt.cm.gray, - interpolation="nearest") + sub.imshow( + completed_face.reshape(image_shape), + cmap=plt.cm.gray, + interpolation="nearest", + ) plt.show() diff --git a/examples/miscellaneous/plot_outlier_detection_bench.py b/examples/miscellaneous/plot_outlier_detection_bench.py new file mode 100644 index 0000000000000..933902500ef8b --- /dev/null +++ b/examples/miscellaneous/plot_outlier_detection_bench.py @@ -0,0 +1,453 @@ +""" +========================================== +Evaluation of outlier detection estimators +========================================== + +This example compares two outlier detection algorithms, namely +:ref:`local_outlier_factor` (LOF) and :ref:`isolation_forest` (IForest), on +real-world datasets available in :class:`sklearn.datasets`. The goal is to show +that different algorithms perform well on different datasets and contrast their +training speed and sensitivity to hyperparameters. + +The algorithms are trained (without labels) on the whole dataset assumed to +contain outliers. + +1. The ROC curves are computed using knowledge of the ground-truth labels +and displayed using :class:`~sklearn.metrics.RocCurveDisplay`. + +2. The performance is assessed in terms of the ROC-AUC. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Dataset preprocessing and model training +# ======================================== +# +# Different outlier detection models require different preprocessing. In the +# presence of categorical variables, +# :class:`~sklearn.preprocessing.OrdinalEncoder` is often a good strategy for +# tree-based models such as :class:`~sklearn.ensemble.IsolationForest`, whereas +# neighbors-based models such as :class:`~sklearn.neighbors.LocalOutlierFactor` +# would be impacted by the ordering induced by ordinal encoding. To avoid +# inducing an ordering, on should rather use +# :class:`~sklearn.preprocessing.OneHotEncoder`. +# +# Neighbors-based models may also require scaling of the numerical features (see +# for instance :ref:`neighbors_scaling`). In the presence of outliers, a good +# option is to use a :class:`~sklearn.preprocessing.RobustScaler`. + +from sklearn.compose import ColumnTransformer +from sklearn.ensemble import IsolationForest +from sklearn.neighbors import LocalOutlierFactor +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import ( + OneHotEncoder, + OrdinalEncoder, + RobustScaler, +) + + +def make_estimator(name, categorical_columns=None, iforest_kw=None, lof_kw=None): + """Create an outlier detection estimator based on its name.""" + if name == "LOF": + outlier_detector = LocalOutlierFactor(**(lof_kw or {})) + if categorical_columns is None: + preprocessor = RobustScaler() + else: + preprocessor = ColumnTransformer( + transformers=[("categorical", OneHotEncoder(), categorical_columns)], + remainder=RobustScaler(), + ) + else: # name == "IForest" + outlier_detector = IsolationForest(**(iforest_kw or {})) + if categorical_columns is None: + preprocessor = None + else: + ordinal_encoder = OrdinalEncoder( + handle_unknown="use_encoded_value", unknown_value=-1 + ) + preprocessor = ColumnTransformer( + transformers=[ + ("categorical", ordinal_encoder, categorical_columns), + ], + remainder="passthrough", + ) + + return make_pipeline(preprocessor, outlier_detector) + + +# %% +# The following `fit_predict` function returns the average outlier score of X. + +from time import perf_counter + + +def fit_predict(estimator, X): + tic = perf_counter() + if estimator[-1].__class__.__name__ == "LocalOutlierFactor": + estimator.fit(X) + y_score = estimator[-1].negative_outlier_factor_ + else: # "IsolationForest" + y_score = estimator.fit(X).decision_function(X) + toc = perf_counter() + print(f"Duration for {model_name}: {toc - tic:.2f} s") + return y_score + + +# %% +# On the rest of the example we process one dataset per section. After loading +# the data, the targets are modified to consist of two classes: 0 representing +# inliers and 1 representing outliers. Due to computational constraints of the +# scikit-learn documentation, the sample size of some datasets is reduced using +# a stratified :class:`~sklearn.model_selection.train_test_split`. +# +# Furthermore, we set `n_neighbors` to match the expected number of anomalies +# `expected_n_anomalies = n_samples * expected_anomaly_fraction`. This is a good +# heuristic as long as the proportion of outliers is not very low, the reason +# being that `n_neighbors` should be at least greater than the number of samples +# in the less populated cluster (see +# :ref:`sphx_glr_auto_examples_neighbors_plot_lof_outlier_detection.py`). +# +# KDDCup99 - SA dataset +# --------------------- +# +# The :ref:`kddcup99_dataset` was generated using a closed network and +# hand-injected attacks. The SA dataset is a subset of it obtained by simply +# selecting all the normal data and an anomaly proportion of around 3%. + +# %% +import numpy as np + +from sklearn.datasets import fetch_kddcup99 +from sklearn.model_selection import train_test_split + +X, y = fetch_kddcup99( + subset="SA", percent10=True, random_state=42, return_X_y=True, as_frame=True +) +y = (y != b"normal.").astype(np.int32) +X, _, y, _ = train_test_split(X, y, train_size=0.1, stratify=y, random_state=42) + +n_samples, anomaly_frac = X.shape[0], y.mean() +print(f"{n_samples} datapoints with {y.sum()} anomalies ({anomaly_frac:.02%})") + +# %% +# The SA dataset contains 41 features out of which 3 are categorical: +# "protocol_type", "service" and "flag". + +# %% +y_true = {} +y_score = {"LOF": {}, "IForest": {}} +model_names = ["LOF", "IForest"] +cat_columns = ["protocol_type", "service", "flag"] + +y_true["KDDCup99 - SA"] = y +for model_name in model_names: + model = make_estimator( + name=model_name, + categorical_columns=cat_columns, + lof_kw={"n_neighbors": int(n_samples * anomaly_frac)}, + iforest_kw={"random_state": 42}, + ) + y_score[model_name]["KDDCup99 - SA"] = fit_predict(model, X) + +# %% +# Forest covertypes dataset +# ------------------------- +# +# The :ref:`covtype_dataset` is a multiclass dataset where the target is the +# dominant species of tree in a given patch of forest. It contains 54 features, +# some of which ("Wilderness_Area" and "Soil_Type") are already binary encoded. +# Though originally meant as a classification task, one can regard inliers as +# samples encoded with label 2 and outliers as those with label 4. + +# %% +from sklearn.datasets import fetch_covtype + +X, y = fetch_covtype(return_X_y=True, as_frame=True) +s = (y == 2) + (y == 4) +X = X.loc[s] +y = y.loc[s] +y = (y != 2).astype(np.int32) + +X, _, y, _ = train_test_split(X, y, train_size=0.05, stratify=y, random_state=42) +X_forestcover = X # save X for later use + +n_samples, anomaly_frac = X.shape[0], y.mean() +print(f"{n_samples} datapoints with {y.sum()} anomalies ({anomaly_frac:.02%})") + +# %% +y_true["forestcover"] = y +for model_name in model_names: + model = make_estimator( + name=model_name, + lof_kw={"n_neighbors": int(n_samples * anomaly_frac)}, + iforest_kw={"random_state": 42}, + ) + y_score[model_name]["forestcover"] = fit_predict(model, X) + +# %% +# Ames Housing dataset +# -------------------- +# +# The `Ames housing dataset `_ is originally a +# regression dataset where the target are sales prices of houses in Ames, Iowa. +# Here we convert it into an outlier detection problem by regarding houses with +# price over 70 USD/sqft. To make the problem easier, we drop intermediate +# prices between 40 and 70 USD/sqft. + +# %% +import matplotlib.pyplot as plt + +from sklearn.datasets import fetch_openml + +X, y = fetch_openml(name="ames_housing", version=1, return_X_y=True, as_frame=True) +y = y.div(X["Lot_Area"]) + +# None values in pandas 1.5.1 were mapped to np.nan in pandas 2.0.1 +X["Misc_Feature"] = X["Misc_Feature"].cat.add_categories("NoInfo").fillna("NoInfo") +X["Mas_Vnr_Type"] = X["Mas_Vnr_Type"].cat.add_categories("NoInfo").fillna("NoInfo") + +X.drop(columns="Lot_Area", inplace=True) +mask = (y < 40) | (y > 70) +X = X.loc[mask] +y = y.loc[mask] +y.hist(bins=20, edgecolor="black") +plt.xlabel("House price in USD/sqft") +_ = plt.title("Distribution of house prices in Ames") + +# %% +y = (y > 70).astype(np.int32) + +n_samples, anomaly_frac = X.shape[0], y.mean() +print(f"{n_samples} datapoints with {y.sum()} anomalies ({anomaly_frac:.02%})") + +# %% +# The dataset contains 46 categorical features. In this case it is easier use a +# :class:`~sklearn.compose.make_column_selector` to find them instead of passing +# a list made by hand. + +# %% +from sklearn.compose import make_column_selector as selector + +categorical_columns_selector = selector(dtype_include="category") +cat_columns = categorical_columns_selector(X) + +y_true["ames_housing"] = y +for model_name in model_names: + model = make_estimator( + name=model_name, + categorical_columns=cat_columns, + lof_kw={"n_neighbors": int(n_samples * anomaly_frac)}, + iforest_kw={"random_state": 42}, + ) + y_score[model_name]["ames_housing"] = fit_predict(model, X) + +# %% +# Cardiotocography dataset +# ------------------------ +# +# The `Cardiotocography dataset `_ is a multiclass +# dataset of fetal cardiotocograms, the classes being the fetal heart rate (FHR) +# pattern encoded with labels from 1 to 10. Here we set class 3 (the minority +# class) to represent the outliers. It contains 30 numerical features, some of +# which are binary encoded and some are continuous. + +# %% +X, y = fetch_openml(name="cardiotocography", version=1, return_X_y=True, as_frame=False) +X_cardiotocography = X # save X for later use +s = y == "3" +y = s.astype(np.int32) + +n_samples, anomaly_frac = X.shape[0], y.mean() +print(f"{n_samples} datapoints with {y.sum()} anomalies ({anomaly_frac:.02%})") + +# %% +y_true["cardiotocography"] = y +for model_name in model_names: + model = make_estimator( + name=model_name, + lof_kw={"n_neighbors": int(n_samples * anomaly_frac)}, + iforest_kw={"random_state": 42}, + ) + y_score[model_name]["cardiotocography"] = fit_predict(model, X) + +# %% +# Plot and interpret results +# ========================== +# +# The algorithm performance relates to how good the true positive rate (TPR) is +# at low value of the false positive rate (FPR). The best algorithms have the +# curve on the top-left of the plot and the area under curve (AUC) close to 1. +# The diagonal dashed line represents a random classification of outliers and +# inliers. + +# %% +import math + +from sklearn.metrics import RocCurveDisplay + +cols = 2 +pos_label = 0 # mean 0 belongs to positive class +datasets_names = y_true.keys() +rows = math.ceil(len(datasets_names) / cols) + +fig, axs = plt.subplots(nrows=rows, ncols=cols, squeeze=False, figsize=(10, rows * 4)) + +for ax, dataset_name in zip(axs.ravel(), datasets_names): + for model_idx, model_name in enumerate(model_names): + display = RocCurveDisplay.from_predictions( + y_true[dataset_name], + y_score[model_name][dataset_name], + pos_label=pos_label, + name=model_name, + ax=ax, + plot_chance_level=(model_idx == len(model_names) - 1), + chance_level_kw={"linestyle": ":"}, + ) + ax.set_title(dataset_name) +_ = plt.tight_layout(pad=2.0) # spacing between subplots + +# %% +# We observe that once the number of neighbors is tuned, LOF and IForest perform +# similarly in terms of ROC AUC for the forestcover and cardiotocography +# datasets. The score for IForest is slightly better for the SA dataset and LOF +# performs considerably better on the Ames housing dataset than IForest. +# +# Recall however that Isolation Forest tends to train much faster than LOF on +# datasets with a large number of samples. LOF needs to compute pairwise +# distances to find nearest neighbors, which has a quadratic complexity with respect +# to the number of observations. This can make this method prohibitive on large +# datasets. +# +# Ablation study +# ============== +# +# In this section we explore the impact of the hyperparameter `n_neighbors` and +# the choice of scaling the numerical variables on the LOF model. Here we use +# the :ref:`covtype_dataset` dataset as the binary encoded categories introduce +# a natural scale of euclidean distances between 0 and 1. We then want a scaling +# method to avoid granting a privilege to non-binary features and that is robust +# enough to outliers so that the task of finding them does not become too +# difficult. + +# %% +X = X_forestcover +y = y_true["forestcover"] + +n_samples = X.shape[0] +n_neighbors_list = (n_samples * np.array([0.2, 0.02, 0.01, 0.001])).astype(np.int32) +model = make_pipeline(RobustScaler(), LocalOutlierFactor()) + +linestyles = ["solid", "dashed", "dashdot", ":", (5, (10, 3))] + +fig, ax = plt.subplots() +for model_idx, (linestyle, n_neighbors) in enumerate(zip(linestyles, n_neighbors_list)): + model.set_params(localoutlierfactor__n_neighbors=n_neighbors) + model.fit(X) + y_score = model[-1].negative_outlier_factor_ + display = RocCurveDisplay.from_predictions( + y, + y_score, + pos_label=pos_label, + name=f"n_neighbors = {n_neighbors}", + ax=ax, + plot_chance_level=(model_idx == len(n_neighbors_list) - 1), + chance_level_kw={"linestyle": (0, (1, 10))}, + curve_kwargs=dict(linestyle=linestyle, linewidth=2), + ) +_ = ax.set_title("RobustScaler with varying n_neighbors\non forestcover dataset") + +# %% +# We observe that the number of neighbors has a big impact on the performance of +# the model. If one has access to (at least some) ground truth labels, it is +# then important to tune `n_neighbors` accordingly. A convenient way to do so is +# to explore values for `n_neighbors` of the order of magnitud of the expected +# contamination. + +# %% +from sklearn.preprocessing import MinMaxScaler, SplineTransformer, StandardScaler + +preprocessor_list = [ + None, + RobustScaler(), + StandardScaler(), + MinMaxScaler(), + SplineTransformer(), +] +expected_anomaly_fraction = 0.02 +lof = LocalOutlierFactor(n_neighbors=int(n_samples * expected_anomaly_fraction)) + +fig, ax = plt.subplots() +for model_idx, (linestyle, preprocessor) in enumerate( + zip(linestyles, preprocessor_list) +): + model = make_pipeline(preprocessor, lof) + model.fit(X) + y_score = model[-1].negative_outlier_factor_ + display = RocCurveDisplay.from_predictions( + y, + y_score, + pos_label=pos_label, + name=str(preprocessor).split("(")[0], + ax=ax, + plot_chance_level=(model_idx == len(preprocessor_list) - 1), + chance_level_kw={"linestyle": (0, (1, 10))}, + curve_kwargs=dict(linestyle=linestyle, linewidth=2), + ) +_ = ax.set_title("Fixed n_neighbors with varying preprocessing\non forestcover dataset") + +# %% +# On the one hand, :class:`~sklearn.preprocessing.RobustScaler` scales each +# feature independently by using the interquartile range (IQR) by default, which +# is the range between the 25th and 75th percentiles of the data. It centers the +# data by subtracting the median and then scale it by dividing by the IQR. The +# IQR is robust to outliers: the median and interquartile range are less +# affected by extreme values than the range, the mean and the standard +# deviation. Furthermore, :class:`~sklearn.preprocessing.RobustScaler` does not +# squash marginal outlier values, contrary to +# :class:`~sklearn.preprocessing.StandardScaler`. +# +# On the other hand, :class:`~sklearn.preprocessing.MinMaxScaler` scales each +# feature individually such that its range maps into the range between zero and +# one. If there are outliers in the data, they can skew it towards either the +# minimum or maximum values, leading to a completely different distribution of +# data with large marginal outliers: all non-outlier values can be collapsed +# almost together as a result. +# +# We also evaluated no preprocessing at all (by passing `None` to the pipeline), +# :class:`~sklearn.preprocessing.StandardScaler` and +# :class:`~sklearn.preprocessing.SplineTransformer`. Please refer to their +# respective documentation for more details. +# +# Note that the optimal preprocessing depends on the dataset, as shown below: + +# %% +X = X_cardiotocography +y = y_true["cardiotocography"] + +n_samples, expected_anomaly_fraction = X.shape[0], 0.025 +lof = LocalOutlierFactor(n_neighbors=int(n_samples * expected_anomaly_fraction)) + +fig, ax = plt.subplots() +for model_idx, (linestyle, preprocessor) in enumerate( + zip(linestyles, preprocessor_list) +): + model = make_pipeline(preprocessor, lof) + model.fit(X) + y_score = model[-1].negative_outlier_factor_ + display = RocCurveDisplay.from_predictions( + y, + y_score, + pos_label=pos_label, + name=str(preprocessor).split("(")[0], + ax=ax, + plot_chance_level=(model_idx == len(preprocessor_list) - 1), + chance_level_kw={"linestyle": (0, (1, 10))}, + curve_kwargs=dict(linestyle=linestyle, linewidth=2), + ) +ax.set_title( + "Fixed n_neighbors with varying preprocessing\non cardiotocography dataset" +) +plt.show() diff --git a/examples/miscellaneous/plot_partial_dependence_visualization_api.py b/examples/miscellaneous/plot_partial_dependence_visualization_api.py index 84658c74ed213..8c98b40816496 100644 --- a/examples/miscellaneous/plot_partial_dependence_visualization_api.py +++ b/examples/miscellaneous/plot_partial_dependence_visualization_api.py @@ -2,8 +2,7 @@ ========================================= Advanced Plotting With Partial Dependence ========================================= -The :func:`~sklearn.inspection.plot_partial_dependence` function returns a -:class:`~sklearn.inspection.PartialDependenceDisplay` object that can be used +The :class:`~sklearn.inspection.PartialDependenceDisplay` object can be used for plotting without needing to recalculate the partial dependence. In this example, we show how to plot partial dependence plots and how to quickly customize the plot with the visualization API. @@ -12,18 +11,20 @@ See also :ref:`sphx_glr_auto_examples_miscellaneous_plot_roc_curve_visualization_api.py` -""" # noqa -print(__doc__) +""" # noqa: E501 + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import pandas as pd import matplotlib.pyplot as plt +import pandas as pd + from sklearn.datasets import load_diabetes +from sklearn.inspection import PartialDependenceDisplay from sklearn.neural_network import MLPRegressor -from sklearn.preprocessing import StandardScaler from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import StandardScaler from sklearn.tree import DecisionTreeRegressor -from sklearn.inspection import plot_partial_dependence - # %% # Train models on the diabetes dataset @@ -37,9 +38,10 @@ y = diabetes.target tree = DecisionTreeRegressor() -mlp = make_pipeline(StandardScaler(), - MLPRegressor(hidden_layer_sizes=(100, 100), - tol=1e-2, max_iter=500, random_state=0)) +mlp = make_pipeline( + StandardScaler(), + MLPRegressor(hidden_layer_sizes=(100, 100), tol=1e-2, max_iter=500, random_state=0), +) tree.fit(X, y) mlp.fit(X, y) @@ -49,22 +51,23 @@ # # We plot partial dependence curves for features "age" and "bmi" (body mass # index) for the decision tree. With two features, -# :func:`~sklearn.inspection.plot_partial_dependence` expects to plot two -# curves. Here the plot function place a grid of two plots using the space +# :func:`~sklearn.inspection.PartialDependenceDisplay.from_estimator` expects to plot +# two curves. Here the plot function place a grid of two plots using the space # defined by `ax` . fig, ax = plt.subplots(figsize=(12, 6)) ax.set_title("Decision Tree") -tree_disp = plot_partial_dependence(tree, X, ["age", "bmi"], ax=ax) +tree_disp = PartialDependenceDisplay.from_estimator(tree, X, ["age", "bmi"], ax=ax) # %% -# The partial depdendence curves can be plotted for the multi-layer perceptron. +# The partial dependence curves can be plotted for the multi-layer perceptron. # In this case, `line_kw` is passed to -# :func:`~sklearn.inspection.plot_partial_dependence` to change the color of -# the curve. +# :func:`~sklearn.inspection.PartialDependenceDisplay.from_estimator` to change the +# color of the curve. fig, ax = plt.subplots(figsize=(12, 6)) ax.set_title("Multi-layer Perceptron") -mlp_disp = plot_partial_dependence(mlp, X, ["age", "bmi"], ax=ax, - line_kw={"color": "red"}) +mlp_disp = PartialDependenceDisplay.from_estimator( + mlp, X, ["age", "bmi"], ax=ax, line_kw={"color": "red"} +) # %% # Plotting partial dependence of the two models together @@ -101,8 +104,9 @@ # sphinx_gallery_thumbnail_number = 4 fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 6)) tree_disp.plot(ax=[ax1, ax2], line_kw={"label": "Decision Tree"}) -mlp_disp.plot(ax=[ax1, ax2], line_kw={"label": "Multi-layer Perceptron", - "color": "red"}) +mlp_disp.plot( + ax=[ax1, ax2], line_kw={"label": "Multi-layer Perceptron", "color": "red"} +) ax1.legend() ax2.legend() @@ -115,8 +119,9 @@ # `plot` will only show the y label and y ticks on the left most plot. tree_disp.plot(line_kw={"label": "Decision Tree"}) -mlp_disp.plot(line_kw={"label": "Multi-layer Perceptron", "color": "red"}, - ax=tree_disp.axes_) +mlp_disp.plot( + line_kw={"label": "Multi-layer Perceptron", "color": "red"}, ax=tree_disp.axes_ +) tree_disp.figure_.set_size_inches(10, 6) tree_disp.axes_[0, 0].legend() tree_disp.axes_[0, 1].legend() @@ -129,7 +134,7 @@ # Here, we plot the partial dependence curves for a single feature, "age", on # the same axes. In this case, `tree_disp.axes_` is passed into the second # plot function. -tree_disp = plot_partial_dependence(tree, X, ["age"]) -mlp_disp = plot_partial_dependence(mlp, X, ["age"], - ax=tree_disp.axes_, - line_kw={"color": "red"}) +tree_disp = PartialDependenceDisplay.from_estimator(tree, X, ["age"]) +mlp_disp = PartialDependenceDisplay.from_estimator( + mlp, X, ["age"], ax=tree_disp.axes_, line_kw={"color": "red"} +) diff --git a/examples/miscellaneous/plot_pipeline_display.py b/examples/miscellaneous/plot_pipeline_display.py new file mode 100644 index 0000000000000..b14dd1576f6c7 --- /dev/null +++ b/examples/miscellaneous/plot_pipeline_display.py @@ -0,0 +1,183 @@ +""" +================================================================= +Displaying Pipelines +================================================================= + +The default configuration for displaying a pipeline in a Jupyter Notebook is +`'diagram'` where `set_config(display='diagram')`. To deactivate HTML representation, +use `set_config(display='text')`. + +To see more detailed steps in the visualization of the pipeline, click on the +steps in the pipeline. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Displaying a Pipeline with a Preprocessing Step and Classifier +# ############################################################## +# This section constructs a :class:`~sklearn.pipeline.Pipeline` with a preprocessing +# step, :class:`~sklearn.preprocessing.StandardScaler`, and classifier, +# :class:`~sklearn.linear_model.LogisticRegression`, and displays its visual +# representation. + +from sklearn import set_config +from sklearn.linear_model import LogisticRegression +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import StandardScaler + +steps = [ + ("preprocessing", StandardScaler()), + ("classifier", LogisticRegression()), +] +pipe = Pipeline(steps) + +# %% +# To visualize the diagram, the default is `display='diagram'`. +set_config(display="diagram") +pipe # click on the diagram below to see the details of each step + +# %% +# To view the text pipeline, change to `display='text'`. +set_config(display="text") +pipe + +# %% +# Put back the default display +set_config(display="diagram") + +# %% +# Displaying a Pipeline Chaining Multiple Preprocessing Steps & Classifier +# ######################################################################## +# This section constructs a :class:`~sklearn.pipeline.Pipeline` with multiple +# preprocessing steps, :class:`~sklearn.preprocessing.PolynomialFeatures` and +# :class:`~sklearn.preprocessing.StandardScaler`, and a classifier step, +# :class:`~sklearn.linear_model.LogisticRegression`, and displays its visual +# representation. + +from sklearn.linear_model import LogisticRegression +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import PolynomialFeatures, StandardScaler + +steps = [ + ("standard_scaler", StandardScaler()), + ("polynomial", PolynomialFeatures(degree=3)), + ("classifier", LogisticRegression(C=2.0)), +] +pipe = Pipeline(steps) +pipe # click on the diagram below to see the details of each step + +# %% +# Displaying a Pipeline and Dimensionality Reduction and Classifier +# ################################################################# +# This section constructs a :class:`~sklearn.pipeline.Pipeline` with a +# dimensionality reduction step, :class:`~sklearn.decomposition.PCA`, +# a classifier, :class:`~sklearn.svm.SVC`, and displays its visual +# representation. + +from sklearn.decomposition import PCA +from sklearn.pipeline import Pipeline +from sklearn.svm import SVC + +steps = [("reduce_dim", PCA(n_components=4)), ("classifier", SVC(kernel="linear"))] +pipe = Pipeline(steps) +pipe # click on the diagram below to see the details of each step + +# %% +# Displaying a Complex Pipeline Chaining a Column Transformer +# ########################################################### +# This section constructs a complex :class:`~sklearn.pipeline.Pipeline` with a +# :class:`~sklearn.compose.ColumnTransformer` and a classifier, +# :class:`~sklearn.linear_model.LogisticRegression`, and displays its visual +# representation. + +import numpy as np + +from sklearn.compose import ColumnTransformer +from sklearn.impute import SimpleImputer +from sklearn.linear_model import LogisticRegression +from sklearn.pipeline import Pipeline, make_pipeline +from sklearn.preprocessing import OneHotEncoder, StandardScaler + +numeric_preprocessor = Pipeline( + steps=[ + ("imputation_mean", SimpleImputer(missing_values=np.nan, strategy="mean")), + ("scaler", StandardScaler()), + ] +) + +categorical_preprocessor = Pipeline( + steps=[ + ( + "imputation_constant", + SimpleImputer(fill_value="missing", strategy="constant"), + ), + ("onehot", OneHotEncoder(handle_unknown="ignore")), + ] +) + +preprocessor = ColumnTransformer( + [ + ("categorical", categorical_preprocessor, ["state", "gender"]), + ("numerical", numeric_preprocessor, ["age", "weight"]), + ] +) + +pipe = make_pipeline(preprocessor, LogisticRegression(max_iter=500)) +pipe # click on the diagram below to see the details of each step + +# %% +# Displaying a Grid Search over a Pipeline with a Classifier +# ########################################################## +# This section constructs a :class:`~sklearn.model_selection.GridSearchCV` +# over a :class:`~sklearn.pipeline.Pipeline` with +# :class:`~sklearn.ensemble.RandomForestClassifier` and displays its visual +# representation. + +import numpy as np + +from sklearn.compose import ColumnTransformer +from sklearn.ensemble import RandomForestClassifier +from sklearn.impute import SimpleImputer +from sklearn.model_selection import GridSearchCV +from sklearn.pipeline import Pipeline, make_pipeline +from sklearn.preprocessing import OneHotEncoder, StandardScaler + +numeric_preprocessor = Pipeline( + steps=[ + ("imputation_mean", SimpleImputer(missing_values=np.nan, strategy="mean")), + ("scaler", StandardScaler()), + ] +) + +categorical_preprocessor = Pipeline( + steps=[ + ( + "imputation_constant", + SimpleImputer(fill_value="missing", strategy="constant"), + ), + ("onehot", OneHotEncoder(handle_unknown="ignore")), + ] +) + +preprocessor = ColumnTransformer( + [ + ("categorical", categorical_preprocessor, ["state", "gender"]), + ("numerical", numeric_preprocessor, ["age", "weight"]), + ] +) + +pipe = Pipeline( + steps=[("preprocessor", preprocessor), ("classifier", RandomForestClassifier())] +) + +param_grid = { + "classifier__n_estimators": [200, 500], + "classifier__max_features": ["auto", "sqrt", "log2"], + "classifier__max_depth": [4, 5, 6, 7, 8], + "classifier__criterion": ["gini", "entropy"], +} + +grid_search = GridSearchCV(pipe, param_grid=param_grid, n_jobs=1) +grid_search # click on the diagram below to see the details of each step diff --git a/examples/miscellaneous/plot_roc_curve_visualization_api.py b/examples/miscellaneous/plot_roc_curve_visualization_api.py index 90fc064931ea7..1aacbd9de3631 100644 --- a/examples/miscellaneous/plot_roc_curve_visualization_api.py +++ b/examples/miscellaneous/plot_roc_curve_visualization_api.py @@ -6,8 +6,11 @@ learning. The key features of this API is to allow for quick plotting and visual adjustments without recalculation. In this example, we will demonstrate how to use the visualization API by comparing ROC curves. + """ -print(__doc__) + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause # %% # Load Data and Train a SVC @@ -15,11 +18,12 @@ # First, we load the wine dataset and convert it to a binary classification # problem. Then, we train a support vector classifier on a training dataset. import matplotlib.pyplot as plt -from sklearn.svm import SVC -from sklearn.ensemble import RandomForestClassifier -from sklearn.metrics import plot_roc_curve + from sklearn.datasets import load_wine +from sklearn.ensemble import RandomForestClassifier +from sklearn.metrics import RocCurveDisplay from sklearn.model_selection import train_test_split +from sklearn.svm import SVC X, y = load_wine(return_X_y=True) y = y == 2 @@ -32,15 +36,15 @@ # Plotting the ROC Curve # ---------------------- # Next, we plot the ROC curve with a single call to -# :func:`sklearn.metrics.plot_roc_curve`. The returned `svc_disp` object allows -# us to continue using the already computed ROC curve for the SVC in future -# plots. -svc_disp = plot_roc_curve(svc, X_test, y_test) +# :func:`sklearn.metrics.RocCurveDisplay.from_estimator`. The returned +# `svc_disp` object allows us to continue using the already computed ROC curve +# for the SVC in future plots. +svc_disp = RocCurveDisplay.from_estimator(svc, X_test, y_test) plt.show() # %% # Training a Random Forest and Plotting the ROC Curve -# -------------------------------------------------------- +# --------------------------------------------------- # We train a random forest classifier and create a plot comparing it to the SVC # ROC curve. Notice how `svc_disp` uses # :func:`~sklearn.metrics.RocCurveDisplay.plot` to plot the SVC ROC curve @@ -50,6 +54,8 @@ rfc = RandomForestClassifier(n_estimators=10, random_state=42) rfc.fit(X_train, y_train) ax = plt.gca() -rfc_disp = plot_roc_curve(rfc, X_test, y_test, ax=ax, alpha=0.8) -svc_disp.plot(ax=ax, alpha=0.8) +rfc_disp = RocCurveDisplay.from_estimator( + rfc, X_test, y_test, ax=ax, curve_kwargs=dict(alpha=0.8) +) +svc_disp.plot(ax=ax, curve_kwargs=dict(alpha=0.8)) plt.show() diff --git a/examples/miscellaneous/plot_set_output.py b/examples/miscellaneous/plot_set_output.py new file mode 100644 index 0000000000000..f3e5be13f5182 --- /dev/null +++ b/examples/miscellaneous/plot_set_output.py @@ -0,0 +1,151 @@ +""" +================================ +Introducing the `set_output` API +================================ + +.. currentmodule:: sklearn + +This example will demonstrate the `set_output` API to configure transformers to +output pandas DataFrames. `set_output` can be configured per estimator by calling +the `set_output` method or globally by setting `set_config(transform_output="pandas")`. +For details, see +`SLEP018 `__. +""" # noqa: CPY001 + +# %% +# First, we load the iris dataset as a DataFrame to demonstrate the `set_output` API. +from sklearn.datasets import load_iris +from sklearn.model_selection import train_test_split + +X, y = load_iris(as_frame=True, return_X_y=True) +X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) +X_train.head() + +# %% +# To configure an estimator such as :class:`preprocessing.StandardScaler` to return +# DataFrames, call `set_output`. This feature requires pandas to be installed. + +from sklearn.preprocessing import StandardScaler + +scaler = StandardScaler().set_output(transform="pandas") + +scaler.fit(X_train) +X_test_scaled = scaler.transform(X_test) +X_test_scaled.head() + +# %% +# `set_output` can be called after `fit` to configure `transform` after the fact. +scaler2 = StandardScaler() + +scaler2.fit(X_train) +X_test_np = scaler2.transform(X_test) +print(f"Default output type: {type(X_test_np).__name__}") + +scaler2.set_output(transform="pandas") +X_test_df = scaler2.transform(X_test) +print(f"Configured pandas output type: {type(X_test_df).__name__}") + +# %% +# In a :class:`pipeline.Pipeline`, `set_output` configures all steps to output +# DataFrames. +from sklearn.feature_selection import SelectPercentile +from sklearn.linear_model import LogisticRegression +from sklearn.pipeline import make_pipeline + +clf = make_pipeline( + StandardScaler(), SelectPercentile(percentile=75), LogisticRegression() +) +clf.set_output(transform="pandas") +clf.fit(X_train, y_train) + +# %% +# Each transformer in the pipeline is configured to return DataFrames. This +# means that the final logistic regression step contains the feature names of the input. +clf[-1].feature_names_in_ + +# %% +# .. note:: If one uses the method `set_params`, the transformer will be +# replaced by a new one with the default output format. +clf.set_params(standardscaler=StandardScaler()) +clf.fit(X_train, y_train) +clf[-1].feature_names_in_ + +# %% +# To keep the intended behavior, use `set_output` on the new transformer +# beforehand +scaler = StandardScaler().set_output(transform="pandas") +clf.set_params(standardscaler=scaler) +clf.fit(X_train, y_train) +clf[-1].feature_names_in_ + +# %% +# Next we load the titanic dataset to demonstrate `set_output` with +# :class:`compose.ColumnTransformer` and heterogeneous data. +from sklearn.datasets import fetch_openml + +X, y = fetch_openml("titanic", version=1, as_frame=True, return_X_y=True) +X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y) + +# %% +# The `set_output` API can be configured globally by using :func:`set_config` and +# setting `transform_output` to `"pandas"`. +from sklearn import set_config +from sklearn.compose import ColumnTransformer +from sklearn.impute import SimpleImputer +from sklearn.preprocessing import OneHotEncoder, StandardScaler + +set_config(transform_output="pandas") + +num_pipe = make_pipeline(SimpleImputer(), StandardScaler()) +num_cols = ["age", "fare"] +ct = ColumnTransformer( + ( + ("numerical", num_pipe, num_cols), + ( + "categorical", + OneHotEncoder( + sparse_output=False, drop="if_binary", handle_unknown="ignore" + ), + ["embarked", "sex", "pclass"], + ), + ), + verbose_feature_names_out=False, +) +clf = make_pipeline(ct, SelectPercentile(percentile=50), LogisticRegression()) +clf.fit(X_train, y_train) +clf.score(X_test, y_test) + +# %% +# With the global configuration, all transformers output DataFrames. This allows us to +# easily plot the logistic regression coefficients with the corresponding feature names. +import pandas as pd + +log_reg = clf[-1] +coef = pd.Series(log_reg.coef_.ravel(), index=log_reg.feature_names_in_) +_ = coef.sort_values().plot.barh() + +# %% +# In order to demonstrate the :func:`config_context` functionality below, let +# us first reset `transform_output` to its default value. +set_config(transform_output="default") + +# %% +# When configuring the output type with :func:`config_context` the +# configuration at the time when `transform` or `fit_transform` are +# called is what counts. Setting these only when you construct or fit +# the transformer has no effect. +from sklearn import config_context + +scaler = StandardScaler() +scaler.fit(X_train[num_cols]) + +# %% +with config_context(transform_output="pandas"): + # the output of transform will be a Pandas DataFrame + X_test_scaled = scaler.transform(X_test[num_cols]) +X_test_scaled.head() + +# %% +# outside of the context manager, the output will be a NumPy array +X_test_scaled = scaler.transform(X_test[num_cols]) +X_test_scaled[:5] diff --git a/examples/mixture/plot_concentration_prior.py b/examples/mixture/plot_concentration_prior.py index ad0b3e14da0ce..9b21bcd91db22 100644 --- a/examples/mixture/plot_concentration_prior.py +++ b/examples/mixture/plot_concentration_prior.py @@ -26,19 +26,19 @@ On the contrary the classical finite mixture model with a Dirichlet distribution prior will favor more uniformly weighted components and therefore tends to divide natural clusters into unnecessary sub-components. + """ -# Author: Thierry Guillemot -# License: BSD 3 clause -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib as mpl -import matplotlib.pyplot as plt import matplotlib.gridspec as gridspec +import matplotlib.pyplot as plt +import numpy as np from sklearn.mixture import BayesianGaussianMixture -print(__doc__) - def plot_ellipses(ax, weights, means, covars): for n in range(means.shape[0]): @@ -49,88 +49,116 @@ def plot_ellipses(ax, weights, means, covars): angle = 180 * angle / np.pi # eigenvector normalization eig_vals = 2 * np.sqrt(2) * np.sqrt(eig_vals) - ell = mpl.patches.Ellipse(means[n], eig_vals[0], eig_vals[1], - 180 + angle, edgecolor='black') + ell = mpl.patches.Ellipse( + means[n], eig_vals[0], eig_vals[1], angle=180 + angle, edgecolor="black" + ) ell.set_clip_box(ax.bbox) ell.set_alpha(weights[n]) - ell.set_facecolor('#56B4E9') + ell.set_facecolor("#56B4E9") ax.add_artist(ell) def plot_results(ax1, ax2, estimator, X, y, title, plot_title=False): ax1.set_title(title) - ax1.scatter(X[:, 0], X[:, 1], s=5, marker='o', color=colors[y], alpha=0.8) - ax1.set_xlim(-2., 2.) - ax1.set_ylim(-3., 3.) + ax1.scatter(X[:, 0], X[:, 1], s=5, marker="o", color=colors[y], alpha=0.8) + ax1.set_xlim(-2.0, 2.0) + ax1.set_ylim(-3.0, 3.0) ax1.set_xticks(()) ax1.set_yticks(()) - plot_ellipses(ax1, estimator.weights_, estimator.means_, - estimator.covariances_) + plot_ellipses(ax1, estimator.weights_, estimator.means_, estimator.covariances_) - ax2.get_xaxis().set_tick_params(direction='out') + ax2.get_xaxis().set_tick_params(direction="out") ax2.yaxis.grid(True, alpha=0.7) for k, w in enumerate(estimator.weights_): - ax2.bar(k, w, width=0.9, color='#56B4E9', zorder=3, - align='center', edgecolor='black') - ax2.text(k, w + 0.007, "%.1f%%" % (w * 100.), - horizontalalignment='center') - ax2.set_xlim(-.6, 2 * n_components - .4) - ax2.set_ylim(0., 1.1) - ax2.tick_params(axis='y', which='both', left=False, - right=False, labelleft=False) - ax2.tick_params(axis='x', which='both', top=False) + ax2.bar( + k, + w, + width=0.9, + color="#56B4E9", + zorder=3, + align="center", + edgecolor="black", + ) + ax2.text(k, w + 0.007, "%.1f%%" % (w * 100.0), horizontalalignment="center") + ax2.set_xlim(-0.6, 2 * n_components - 0.4) + ax2.set_ylim(0.0, 1.1) + ax2.tick_params(axis="y", which="both", left=False, right=False, labelleft=False) + ax2.tick_params(axis="x", which="both", top=False) if plot_title: - ax1.set_ylabel('Estimated Mixtures') - ax2.set_ylabel('Weight of each component') + ax1.set_ylabel("Estimated Mixtures") + ax2.set_ylabel("Weight of each component") + # Parameters of the dataset random_state, n_components, n_features = 2, 3, 2 -colors = np.array(['#0072B2', '#F0E442', '#D55E00']) +colors = np.array(["#0072B2", "#F0E442", "#D55E00"]) -covars = np.array([[[.7, .0], [.0, .1]], - [[.5, .0], [.0, .1]], - [[.5, .0], [.0, .1]]]) +covars = np.array( + [[[0.7, 0.0], [0.0, 0.1]], [[0.5, 0.0], [0.0, 0.1]], [[0.5, 0.0], [0.0, 0.1]]] +) samples = np.array([200, 500, 200]) -means = np.array([[.0, -.70], - [.0, .0], - [.0, .70]]) +means = np.array([[0.0, -0.70], [0.0, 0.0], [0.0, 0.70]]) # mean_precision_prior= 0.8 to minimize the influence of the prior estimators = [ - ("Finite mixture with a Dirichlet distribution\nprior and " - r"$\gamma_0=$", BayesianGaussianMixture( - weight_concentration_prior_type="dirichlet_distribution", - n_components=2 * n_components, reg_covar=0, init_params='random', - max_iter=1500, mean_precision_prior=.8, - random_state=random_state), [0.001, 1, 1000]), - ("Infinite mixture with a Dirichlet process\n prior and" r"$\gamma_0=$", - BayesianGaussianMixture( - weight_concentration_prior_type="dirichlet_process", - n_components=2 * n_components, reg_covar=0, init_params='random', - max_iter=1500, mean_precision_prior=.8, - random_state=random_state), [1, 1000, 100000])] + ( + "Finite mixture with a Dirichlet distribution\n" r"prior and $\gamma_0=$", + BayesianGaussianMixture( + weight_concentration_prior_type="dirichlet_distribution", + n_components=2 * n_components, + reg_covar=0, + init_params="random", + max_iter=1500, + mean_precision_prior=0.8, + random_state=random_state, + ), + [0.001, 1, 1000], + ), + ( + "Infinite mixture with a Dirichlet process\n" r"prior and $\gamma_0=$", + BayesianGaussianMixture( + weight_concentration_prior_type="dirichlet_process", + n_components=2 * n_components, + reg_covar=0, + init_params="random", + max_iter=1500, + mean_precision_prior=0.8, + random_state=random_state, + ), + [1, 1000, 100000], + ), +] # Generate data rng = np.random.RandomState(random_state) -X = np.vstack([ - rng.multivariate_normal(means[j], covars[j], samples[j]) - for j in range(n_components)]) -y = np.concatenate([np.full(samples[j], j, dtype=int) - for j in range(n_components)]) +X = np.vstack( + [ + rng.multivariate_normal(means[j], covars[j], samples[j]) + for j in range(n_components) + ] +) +y = np.concatenate([np.full(samples[j], j, dtype=int) for j in range(n_components)]) # Plot results in two different figures -for (title, estimator, concentrations_prior) in estimators: +for title, estimator, concentrations_prior in estimators: plt.figure(figsize=(4.7 * 3, 8)) - plt.subplots_adjust(bottom=.04, top=0.90, hspace=.05, wspace=.05, - left=.03, right=.99) + plt.subplots_adjust( + bottom=0.04, top=0.90, hspace=0.05, wspace=0.05, left=0.03, right=0.99 + ) gs = gridspec.GridSpec(3, len(concentrations_prior)) for k, concentration in enumerate(concentrations_prior): estimator.weight_concentration_prior = concentration estimator.fit(X) - plot_results(plt.subplot(gs[0:2, k]), plt.subplot(gs[2, k]), estimator, - X, y, r"%s$%.1e$" % (title, concentration), - plot_title=k == 0) + plot_results( + plt.subplot(gs[0:2, k]), + plt.subplot(gs[2, k]), + estimator, + X, + y, + r"%s$%.1e$" % (title, concentration), + plot_title=k == 0, + ) plt.show() diff --git a/examples/mixture/plot_gmm.py b/examples/mixture/plot_gmm.py index 5f2f8596d4bbe..9a27b1c42f81a 100644 --- a/examples/mixture/plot_gmm.py +++ b/examples/mixture/plot_gmm.py @@ -21,45 +21,47 @@ full covariance matrices effectively even when there are less examples per cluster than there are dimensions in the data, due to regularization properties of the inference algorithm. + """ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import itertools +import matplotlib as mpl +import matplotlib.pyplot as plt import numpy as np from scipy import linalg -import matplotlib.pyplot as plt -import matplotlib as mpl from sklearn import mixture -color_iter = itertools.cycle(['navy', 'c', 'cornflowerblue', 'gold', - 'darkorange']) +color_iter = itertools.cycle(["navy", "c", "cornflowerblue", "gold", "darkorange"]) def plot_results(X, Y_, means, covariances, index, title): splot = plt.subplot(2, 1, 1 + index) - for i, (mean, covar, color) in enumerate(zip( - means, covariances, color_iter)): + for i, (mean, covar, color) in enumerate(zip(means, covariances, color_iter)): v, w = linalg.eigh(covar) - v = 2. * np.sqrt(2.) * np.sqrt(v) + v = 2.0 * np.sqrt(2.0) * np.sqrt(v) u = w[0] / linalg.norm(w[0]) # as the DP will not use every component it has access to # unless it needs it, we shouldn't plot the redundant # components. if not np.any(Y_ == i): continue - plt.scatter(X[Y_ == i, 0], X[Y_ == i, 1], .8, color=color) + plt.scatter(X[Y_ == i, 0], X[Y_ == i, 1], 0.8, color=color) # Plot an ellipse to show the Gaussian component angle = np.arctan(u[1] / u[0]) - angle = 180. * angle / np.pi # convert to degrees - ell = mpl.patches.Ellipse(mean, v[0], v[1], 180. + angle, color=color) + angle = 180.0 * angle / np.pi # convert to degrees + ell = mpl.patches.Ellipse(mean, v[0], v[1], angle=180.0 + angle, color=color) ell.set_clip_box(splot.bbox) ell.set_alpha(0.5) splot.add_artist(ell) - plt.xlim(-9., 5.) - plt.ylim(-3., 6.) + plt.xlim(-9.0, 5.0) + plt.ylim(-3.0, 6.0) plt.xticks(()) plt.yticks(()) plt.title(title) @@ -70,19 +72,25 @@ def plot_results(X, Y_, means, covariances, index, title): # Generate random sample, two components np.random.seed(0) -C = np.array([[0., -0.1], [1.7, .4]]) -X = np.r_[np.dot(np.random.randn(n_samples, 2), C), - .7 * np.random.randn(n_samples, 2) + np.array([-6, 3])] +C = np.array([[0.0, -0.1], [1.7, 0.4]]) +X = np.r_[ + np.dot(np.random.randn(n_samples, 2), C), + 0.7 * np.random.randn(n_samples, 2) + np.array([-6, 3]), +] # Fit a Gaussian mixture with EM using five components -gmm = mixture.GaussianMixture(n_components=5, covariance_type='full').fit(X) -plot_results(X, gmm.predict(X), gmm.means_, gmm.covariances_, 0, - 'Gaussian Mixture') +gmm = mixture.GaussianMixture(n_components=5, covariance_type="full").fit(X) +plot_results(X, gmm.predict(X), gmm.means_, gmm.covariances_, 0, "Gaussian Mixture") # Fit a Dirichlet process Gaussian mixture using five components -dpgmm = mixture.BayesianGaussianMixture(n_components=5, - covariance_type='full').fit(X) -plot_results(X, dpgmm.predict(X), dpgmm.means_, dpgmm.covariances_, 1, - 'Bayesian Gaussian Mixture with a Dirichlet process prior') +dpgmm = mixture.BayesianGaussianMixture(n_components=5, covariance_type="full").fit(X) +plot_results( + X, + dpgmm.predict(X), + dpgmm.means_, + dpgmm.covariances_, + 1, + "Bayesian Gaussian Mixture with a Dirichlet process prior", +) plt.show() diff --git a/examples/mixture/plot_gmm_covariances.py b/examples/mixture/plot_gmm_covariances.py index b9704ee435db2..91a26f518f332 100644 --- a/examples/mixture/plot_gmm_covariances.py +++ b/examples/mixture/plot_gmm_covariances.py @@ -24,47 +24,46 @@ crosses. The iris dataset is four-dimensional. Only the first two dimensions are shown here, and thus some points are separated in other dimensions. + """ -# Author: Ron Weiss , Gael Varoquaux -# Modified by Thierry Guillemot -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import matplotlib as mpl import matplotlib.pyplot as plt - import numpy as np from sklearn import datasets from sklearn.mixture import GaussianMixture from sklearn.model_selection import StratifiedKFold -print(__doc__) - -colors = ['navy', 'turquoise', 'darkorange'] +colors = ["navy", "turquoise", "darkorange"] def make_ellipses(gmm, ax): for n, color in enumerate(colors): - if gmm.covariance_type == 'full': + if gmm.covariance_type == "full": covariances = gmm.covariances_[n][:2, :2] - elif gmm.covariance_type == 'tied': + elif gmm.covariance_type == "tied": covariances = gmm.covariances_[:2, :2] - elif gmm.covariance_type == 'diag': + elif gmm.covariance_type == "diag": covariances = np.diag(gmm.covariances_[n][:2]) - elif gmm.covariance_type == 'spherical': + elif gmm.covariance_type == "spherical": covariances = np.eye(gmm.means_.shape[1]) * gmm.covariances_[n] v, w = np.linalg.eigh(covariances) u = w[0] / np.linalg.norm(w[0]) angle = np.arctan2(u[1], u[0]) angle = 180 * angle / np.pi # convert to degrees - v = 2. * np.sqrt(2.) * np.sqrt(v) - ell = mpl.patches.Ellipse(gmm.means_[n, :2], v[0], v[1], - 180 + angle, color=color) + v = 2.0 * np.sqrt(2.0) * np.sqrt(v) + ell = mpl.patches.Ellipse( + gmm.means_[n, :2], v[0], v[1], angle=180 + angle, color=color + ) ell.set_clip_box(ax.bbox) ell.set_alpha(0.5) ax.add_artist(ell) - ax.set_aspect('equal', 'datalim') + ax.set_aspect("equal", "datalim") + iris = datasets.load_iris() @@ -83,22 +82,27 @@ def make_ellipses(gmm, ax): n_classes = len(np.unique(y_train)) # Try GMMs using different types of covariances. -estimators = {cov_type: GaussianMixture(n_components=n_classes, - covariance_type=cov_type, max_iter=20, random_state=0) - for cov_type in ['spherical', 'diag', 'tied', 'full']} +estimators = { + cov_type: GaussianMixture( + n_components=n_classes, covariance_type=cov_type, max_iter=20, random_state=0 + ) + for cov_type in ["spherical", "diag", "tied", "full"] +} n_estimators = len(estimators) plt.figure(figsize=(3 * n_estimators // 2, 6)) -plt.subplots_adjust(bottom=.01, top=0.95, hspace=.15, wspace=.05, - left=.01, right=.99) +plt.subplots_adjust( + bottom=0.01, top=0.95, hspace=0.15, wspace=0.05, left=0.01, right=0.99 +) for index, (name, estimator) in enumerate(estimators.items()): # Since we have class labels for the training data, we can # initialize the GMM parameters in a supervised manner. - estimator.means_init = np.array([X_train[y_train == i].mean(axis=0) - for i in range(n_classes)]) + estimator.means_init = np.array( + [X_train[y_train == i].mean(axis=0) for i in range(n_classes)] + ) # Train the other parameters using the EM algorithm. estimator.fit(X_train) @@ -108,28 +112,27 @@ def make_ellipses(gmm, ax): for n, color in enumerate(colors): data = iris.data[iris.target == n] - plt.scatter(data[:, 0], data[:, 1], s=0.8, color=color, - label=iris.target_names[n]) + plt.scatter( + data[:, 0], data[:, 1], s=0.8, color=color, label=iris.target_names[n] + ) # Plot the test data with crosses for n, color in enumerate(colors): data = X_test[y_test == n] - plt.scatter(data[:, 0], data[:, 1], marker='x', color=color) + plt.scatter(data[:, 0], data[:, 1], marker="x", color=color) y_train_pred = estimator.predict(X_train) train_accuracy = np.mean(y_train_pred.ravel() == y_train.ravel()) * 100 - plt.text(0.05, 0.9, 'Train accuracy: %.1f' % train_accuracy, - transform=h.transAxes) + plt.text(0.05, 0.9, "Train accuracy: %.1f" % train_accuracy, transform=h.transAxes) y_test_pred = estimator.predict(X_test) test_accuracy = np.mean(y_test_pred.ravel() == y_test.ravel()) * 100 - plt.text(0.05, 0.8, 'Test accuracy: %.1f' % test_accuracy, - transform=h.transAxes) + plt.text(0.05, 0.8, "Test accuracy: %.1f" % test_accuracy, transform=h.transAxes) plt.xticks(()) plt.yticks(()) plt.title(name) -plt.legend(scatterpoints=1, loc='lower right', prop=dict(size=12)) +plt.legend(scatterpoints=1, loc="lower right", prop=dict(size=12)) plt.show() diff --git a/examples/mixture/plot_gmm_init.py b/examples/mixture/plot_gmm_init.py new file mode 100644 index 0000000000000..0178d4a07af11 --- /dev/null +++ b/examples/mixture/plot_gmm_init.py @@ -0,0 +1,110 @@ +""" +========================== +GMM Initialization Methods +========================== + +Examples of the different methods of initialization in Gaussian Mixture Models + +See :ref:`gmm` for more information on the estimator. + +Here we generate some sample data with four easy to identify clusters. The +purpose of this example is to show the four different methods for the +initialization parameter *init_param*. + +The four initializations are *kmeans* (default), *random*, *random_from_data* and +*k-means++*. + +Orange diamonds represent the initialization centers for the gmm generated by +the *init_param*. The rest of the data is represented as crosses and the +colouring represents the eventual associated classification after the GMM has +finished. + +The numbers in the top right of each subplot represent the number of +iterations taken for the GaussianMixture to converge and the relative time +taken for the initialization part of the algorithm to run. The shorter +initialization times tend to have a greater number of iterations to converge. + +The initialization time is the ratio of the time taken for that method versus +the time taken for the default *kmeans* method. As you can see all three +alternative methods take less time to initialize when compared to *kmeans*. + +In this example, when initialized with *random_from_data* or *random* the model takes +more iterations to converge. Here *k-means++* does a good job of both low +time to initialize and low number of GaussianMixture iterations to converge. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from timeit import default_timer as timer + +import matplotlib.pyplot as plt +import numpy as np + +from sklearn.datasets._samples_generator import make_blobs +from sklearn.mixture import GaussianMixture +from sklearn.utils.extmath import row_norms + +print(__doc__) + +# Generate some data + +X, y_true = make_blobs(n_samples=4000, centers=4, cluster_std=0.60, random_state=0) +X = X[:, ::-1] + +n_samples = 4000 +n_components = 4 +x_squared_norms = row_norms(X, squared=True) + + +def get_initial_means(X, init_params, r): + # Run a GaussianMixture with max_iter=0 to output the initialization means + gmm = GaussianMixture( + n_components=4, init_params=init_params, tol=1e-9, max_iter=0, random_state=r + ).fit(X) + return gmm.means_ + + +methods = ["kmeans", "random_from_data", "k-means++", "random"] +colors = ["navy", "turquoise", "cornflowerblue", "darkorange"] +times_init = {} +relative_times = {} + +plt.figure(figsize=(4 * len(methods) // 2, 6)) +plt.subplots_adjust( + bottom=0.1, top=0.9, hspace=0.15, wspace=0.05, left=0.05, right=0.95 +) + +for n, method in enumerate(methods): + r = np.random.RandomState(seed=1234) + plt.subplot(2, len(methods) // 2, n + 1) + + start = timer() + ini = get_initial_means(X, method, r) + end = timer() + init_time = end - start + + gmm = GaussianMixture( + n_components=4, means_init=ini, tol=1e-9, max_iter=2000, random_state=r + ).fit(X) + + times_init[method] = init_time + for i, color in enumerate(colors): + data = X[gmm.predict(X) == i] + plt.scatter(data[:, 0], data[:, 1], color=color, marker="x") + + plt.scatter( + ini[:, 0], ini[:, 1], s=75, marker="D", c="orange", lw=1.5, edgecolors="black" + ) + relative_times[method] = times_init[method] / times_init[methods[0]] + + plt.xticks(()) + plt.yticks(()) + plt.title(method, loc="left", fontsize=12) + plt.title( + "Iter %i | Init Time %.2fx" % (gmm.n_iter_, relative_times[method]), + loc="right", + fontsize=10, + ) +plt.suptitle("GMM iterations and relative time taken to initialize") +plt.show() diff --git a/examples/mixture/plot_gmm_pdf.py b/examples/mixture/plot_gmm_pdf.py index 4469c36a89625..be70578402f55 100644 --- a/examples/mixture/plot_gmm_pdf.py +++ b/examples/mixture/plot_gmm_pdf.py @@ -6,11 +6,16 @@ Plot the density estimation of a mixture of two Gaussians. Data is generated from two Gaussians with different centers and covariance matrices. + """ -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt +import numpy as np from matplotlib.colors import LogNorm + from sklearn import mixture n_samples = 300 @@ -22,29 +27,30 @@ shifted_gaussian = np.random.randn(n_samples, 2) + np.array([20, 20]) # generate zero centered stretched Gaussian data -C = np.array([[0., -0.7], [3.5, .7]]) +C = np.array([[0.0, -0.7], [3.5, 0.7]]) stretched_gaussian = np.dot(np.random.randn(n_samples, 2), C) # concatenate the two datasets into the final training set X_train = np.vstack([shifted_gaussian, stretched_gaussian]) # fit a Gaussian Mixture Model with two components -clf = mixture.GaussianMixture(n_components=2, covariance_type='full') +clf = mixture.GaussianMixture(n_components=2, covariance_type="full") clf.fit(X_train) # display predicted scores by the model as a contour plot -x = np.linspace(-20., 30.) -y = np.linspace(-20., 40.) +x = np.linspace(-20.0, 30.0) +y = np.linspace(-20.0, 40.0) X, Y = np.meshgrid(x, y) XX = np.array([X.ravel(), Y.ravel()]).T Z = -clf.score_samples(XX) Z = Z.reshape(X.shape) -CS = plt.contour(X, Y, Z, norm=LogNorm(vmin=1.0, vmax=1000.0), - levels=np.logspace(0, 3, 10)) -CB = plt.colorbar(CS, shrink=0.8, extend='both') -plt.scatter(X_train[:, 0], X_train[:, 1], .8) +CS = plt.contour( + X, Y, Z, norm=LogNorm(vmin=1.0, vmax=1000.0), levels=np.logspace(0, 3, 10) +) +CB = plt.colorbar(CS, shrink=0.8, extend="both") +plt.scatter(X_train[:, 0], X_train[:, 1], 0.8) -plt.title('Negative log-likelihood predicted by a GMM') -plt.axis('tight') +plt.title("Negative log-likelihood predicted by a GMM") +plt.axis("tight") plt.show() diff --git a/examples/mixture/plot_gmm_selection.py b/examples/mixture/plot_gmm_selection.py index 3340ea93ea965..ef256aa4f8e0f 100644 --- a/examples/mixture/plot_gmm_selection.py +++ b/examples/mixture/plot_gmm_selection.py @@ -3,98 +3,174 @@ Gaussian Mixture Model Selection ================================ -This example shows that model selection can be performed with -Gaussian Mixture Models using information-theoretic criteria (BIC). -Model selection concerns both the covariance type -and the number of components in the model. -In that case, AIC also provides the right result (not shown to save time), -but BIC is better suited if the problem is to identify the right model. -Unlike Bayesian procedures, such inferences are prior-free. - -In that case, the model with 2 components and full covariance -(which corresponds to the true generative model) is selected. +This example shows that model selection can be performed with Gaussian Mixture +Models (GMM) using :ref:`information-theory criteria `. Model selection +concerns both the covariance type and the number of components in the model. + +In this case, both the Akaike Information Criterion (AIC) and the Bayes +Information Criterion (BIC) provide the right result, but we only demo the +latter as BIC is better suited to identify the true model among a set of +candidates. Unlike Bayesian procedures, such inferences are prior-free. + """ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Data generation +# --------------- +# +# We generate two components (each one containing `n_samples`) by randomly +# sampling the standard normal distribution as returned by `numpy.random.randn`. +# One component is kept spherical yet shifted and re-scaled. The other one is +# deformed to have a more general covariance matrix. + import numpy as np -import itertools -from scipy import linalg +n_samples = 500 +np.random.seed(0) +C = np.array([[0.0, -0.1], [1.7, 0.4]]) +component_1 = np.dot(np.random.randn(n_samples, 2), C) # general +component_2 = 0.7 * np.random.randn(n_samples, 2) + np.array([-4, 1]) # spherical + +X = np.concatenate([component_1, component_2]) + +# %% +# We can visualize the different components: + import matplotlib.pyplot as plt -import matplotlib as mpl -from sklearn import mixture +plt.scatter(component_1[:, 0], component_1[:, 1], s=0.8) +plt.scatter(component_2[:, 0], component_2[:, 1], s=0.8) +plt.title("Gaussian Mixture components") +plt.axis("equal") +plt.show() -print(__doc__) +# %% +# Model training and selection +# ---------------------------- +# +# We vary the number of components from 1 to 6 and the type of covariance +# parameters to use: +# +# - `"full"`: each component has its own general covariance matrix. +# - `"tied"`: all components share the same general covariance matrix. +# - `"diag"`: each component has its own diagonal covariance matrix. +# - `"spherical"`: each component has its own single variance. +# +# We score the different models and keep the best model (the lowest BIC). This +# is done by using :class:`~sklearn.model_selection.GridSearchCV` and a +# user-defined score function which returns the negative BIC score, as +# :class:`~sklearn.model_selection.GridSearchCV` is designed to **maximize** a +# score (maximizing the negative BIC is equivalent to minimizing the BIC). +# +# The best set of parameters and estimator are stored in `best_parameters_` and +# `best_estimator_`, respectively. -# Number of samples per component -n_samples = 500 +from sklearn.mixture import GaussianMixture +from sklearn.model_selection import GridSearchCV -# Generate random sample, two components -np.random.seed(0) -C = np.array([[0., -0.1], [1.7, .4]]) -X = np.r_[np.dot(np.random.randn(n_samples, 2), C), - .7 * np.random.randn(n_samples, 2) + np.array([-6, 3])] - -lowest_bic = np.infty -bic = [] -n_components_range = range(1, 7) -cv_types = ['spherical', 'tied', 'diag', 'full'] -for cv_type in cv_types: - for n_components in n_components_range: - # Fit a Gaussian mixture with EM - gmm = mixture.GaussianMixture(n_components=n_components, - covariance_type=cv_type) - gmm.fit(X) - bic.append(gmm.bic(X)) - if bic[-1] < lowest_bic: - lowest_bic = bic[-1] - best_gmm = gmm - -bic = np.array(bic) -color_iter = itertools.cycle(['navy', 'turquoise', 'cornflowerblue', - 'darkorange']) -clf = best_gmm -bars = [] +def gmm_bic_score(estimator, X): + """Callable to pass to GridSearchCV that will use the BIC score.""" + # Make it negative since GridSearchCV expects a score to maximize + return -estimator.bic(X) + + +param_grid = { + "n_components": range(1, 7), + "covariance_type": ["spherical", "tied", "diag", "full"], +} +grid_search = GridSearchCV( + GaussianMixture(), param_grid=param_grid, scoring=gmm_bic_score +) +grid_search.fit(X) + +# %% # Plot the BIC scores -plt.figure(figsize=(8, 6)) -spl = plt.subplot(2, 1, 1) -for i, (cv_type, color) in enumerate(zip(cv_types, color_iter)): - xpos = np.array(n_components_range) + .2 * (i - 2) - bars.append(plt.bar(xpos, bic[i * len(n_components_range): - (i + 1) * len(n_components_range)], - width=.2, color=color)) -plt.xticks(n_components_range) -plt.ylim([bic.min() * 1.01 - .01 * bic.max(), bic.max()]) -plt.title('BIC score per model') -xpos = np.mod(bic.argmin(), len(n_components_range)) + .65 +\ - .2 * np.floor(bic.argmin() / len(n_components_range)) -plt.text(xpos, bic.min() * 0.97 + .03 * bic.max(), '*', fontsize=14) -spl.set_xlabel('Number of components') -spl.legend([b[0] for b in bars], cv_types) - -# Plot the winner -splot = plt.subplot(2, 1, 2) -Y_ = clf.predict(X) -for i, (mean, cov, color) in enumerate(zip(clf.means_, clf.covariances_, - color_iter)): +# ------------------- +# +# To ease the plotting we can create a `pandas.DataFrame` from the results of +# the cross-validation done by the grid search. We re-inverse the sign of the +# BIC score to show the effect of minimizing it. + +import pandas as pd + +df = pd.DataFrame(grid_search.cv_results_)[ + ["param_n_components", "param_covariance_type", "mean_test_score"] +] +df["mean_test_score"] = -df["mean_test_score"] +df = df.rename( + columns={ + "param_n_components": "Number of components", + "param_covariance_type": "Type of covariance", + "mean_test_score": "BIC score", + } +) +df.sort_values(by="BIC score").head() + +# %% +import seaborn as sns + +sns.catplot( + data=df, + kind="bar", + x="Number of components", + y="BIC score", + hue="Type of covariance", +) +plt.show() + +# %% +# In the present case, the model with 2 components and full covariance (which +# corresponds to the true generative model) has the lowest BIC score and is +# therefore selected by the grid search. +# +# Plot the best model +# ------------------- +# +# We plot an ellipse to show each Gaussian component of the selected model. For +# such purpose, one needs to find the eigenvalues of the covariance matrices as +# returned by the `covariances_` attribute. The shape of such matrices depends +# on the `covariance_type`: +# +# - `"full"`: (`n_components`, `n_features`, `n_features`) +# - `"tied"`: (`n_features`, `n_features`) +# - `"diag"`: (`n_components`, `n_features`) +# - `"spherical"`: (`n_components`,) + +from matplotlib.patches import Ellipse +from scipy import linalg + +color_iter = sns.color_palette("tab10", 2)[::-1] +Y_ = grid_search.predict(X) + +fig, ax = plt.subplots() + +for i, (mean, cov, color) in enumerate( + zip( + grid_search.best_estimator_.means_, + grid_search.best_estimator_.covariances_, + color_iter, + ) +): v, w = linalg.eigh(cov) if not np.any(Y_ == i): continue - plt.scatter(X[Y_ == i, 0], X[Y_ == i, 1], .8, color=color) + plt.scatter(X[Y_ == i, 0], X[Y_ == i, 1], 0.8, color=color) - # Plot an ellipse to show the Gaussian component angle = np.arctan2(w[0][1], w[0][0]) - angle = 180. * angle / np.pi # convert to degrees - v = 2. * np.sqrt(2.) * np.sqrt(v) - ell = mpl.patches.Ellipse(mean, v[0], v[1], 180. + angle, color=color) - ell.set_clip_box(splot.bbox) - ell.set_alpha(.5) - splot.add_artist(ell) - -plt.xticks(()) -plt.yticks(()) -plt.title(f'Selected GMM: {best_gmm.covariance_type} model, ' - f'{best_gmm.n_components} components') -plt.subplots_adjust(hspace=.35, bottom=.02) + angle = 180.0 * angle / np.pi # convert to degrees + v = 2.0 * np.sqrt(2.0) * np.sqrt(v) + ellipse = Ellipse(mean, v[0], v[1], angle=180.0 + angle, color=color) + ellipse.set_clip_box(fig.bbox) + ellipse.set_alpha(0.5) + ax.add_artist(ellipse) + +plt.title( + f"Selected GMM: {grid_search.best_params_['covariance_type']} model, " + f"{grid_search.best_params_['n_components']} components" +) +plt.axis("equal") plt.show() diff --git a/examples/mixture/plot_gmm_sin.py b/examples/mixture/plot_gmm_sin.py index 1d436b93d15cc..fe9c12bbe5adc 100644 --- a/examples/mixture/plot_gmm_sin.py +++ b/examples/mixture/plot_gmm_sin.py @@ -39,45 +39,44 @@ """ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import itertools +import matplotlib as mpl +import matplotlib.pyplot as plt import numpy as np from scipy import linalg -import matplotlib.pyplot as plt -import matplotlib as mpl from sklearn import mixture -print(__doc__) - -color_iter = itertools.cycle(['navy', 'c', 'cornflowerblue', 'gold', - 'darkorange']) +color_iter = itertools.cycle(["navy", "c", "cornflowerblue", "gold", "darkorange"]) def plot_results(X, Y, means, covariances, index, title): splot = plt.subplot(5, 1, 1 + index) - for i, (mean, covar, color) in enumerate(zip( - means, covariances, color_iter)): + for i, (mean, covar, color) in enumerate(zip(means, covariances, color_iter)): v, w = linalg.eigh(covar) - v = 2. * np.sqrt(2.) * np.sqrt(v) + v = 2.0 * np.sqrt(2.0) * np.sqrt(v) u = w[0] / linalg.norm(w[0]) # as the DP will not use every component it has access to # unless it needs it, we shouldn't plot the redundant # components. if not np.any(Y == i): continue - plt.scatter(X[Y == i, 0], X[Y == i, 1], .8, color=color) + plt.scatter(X[Y == i, 0], X[Y == i, 1], 0.8, color=color) # Plot an ellipse to show the Gaussian component angle = np.arctan(u[1] / u[0]) - angle = 180. * angle / np.pi # convert to degrees - ell = mpl.patches.Ellipse(mean, v[0], v[1], 180. + angle, color=color) + angle = 180.0 * angle / np.pi # convert to degrees + ell = mpl.patches.Ellipse(mean, v[0], v[1], angle=180.0 + angle, color=color) ell.set_clip_box(splot.bbox) ell.set_alpha(0.5) splot.add_artist(ell) - plt.xlim(-6., 4. * np.pi - 6.) - plt.ylim(-5., 5.) + plt.xlim(-6.0, 4.0 * np.pi - 6.0) + plt.ylim(-5.0, 5.0) plt.title(title) plt.xticks(()) plt.yticks(()) @@ -91,10 +90,10 @@ def plot_samples(X, Y, n_components, index, title): # components. if not np.any(Y == i): continue - plt.scatter(X[Y == i, 0], X[Y == i, 1], .8, color=color) + plt.scatter(X[Y == i, 0], X[Y == i, 1], 0.8, color=color) - plt.xlim(-6., 4. * np.pi - 6.) - plt.ylim(-5., 5.) + plt.xlim(-6.0, 4.0 * np.pi - 6.0) + plt.ylim(-5.0, 5.0) plt.title(title) plt.xticks(()) plt.yticks(()) @@ -106,49 +105,86 @@ def plot_samples(X, Y, n_components, index, title): # Generate random sample following a sine curve np.random.seed(0) X = np.zeros((n_samples, 2)) -step = 4. * np.pi / n_samples +step = 4.0 * np.pi / n_samples for i in range(X.shape[0]): - x = i * step - 6. + x = i * step - 6.0 X[i, 0] = x + np.random.normal(0, 0.1) - X[i, 1] = 3. * (np.sin(x) + np.random.normal(0, .2)) + X[i, 1] = 3.0 * (np.sin(x) + np.random.normal(0, 0.2)) plt.figure(figsize=(10, 10)) -plt.subplots_adjust(bottom=.04, top=0.95, hspace=.2, wspace=.05, - left=.03, right=.97) +plt.subplots_adjust( + bottom=0.04, top=0.95, hspace=0.2, wspace=0.05, left=0.03, right=0.97 +) # Fit a Gaussian mixture with EM using ten components -gmm = mixture.GaussianMixture(n_components=10, covariance_type='full', - max_iter=100).fit(X) -plot_results(X, gmm.predict(X), gmm.means_, gmm.covariances_, 0, - 'Expectation-maximization') +gmm = mixture.GaussianMixture( + n_components=10, covariance_type="full", max_iter=100 +).fit(X) +plot_results( + X, gmm.predict(X), gmm.means_, gmm.covariances_, 0, "Expectation-maximization" +) dpgmm = mixture.BayesianGaussianMixture( - n_components=10, covariance_type='full', weight_concentration_prior=1e-2, - weight_concentration_prior_type='dirichlet_process', - mean_precision_prior=1e-2, covariance_prior=1e0 * np.eye(2), - init_params="random", max_iter=100, random_state=2).fit(X) -plot_results(X, dpgmm.predict(X), dpgmm.means_, dpgmm.covariances_, 1, - "Bayesian Gaussian mixture models with a Dirichlet process prior " - r"for $\gamma_0=0.01$.") + n_components=10, + covariance_type="full", + weight_concentration_prior=1e-2, + weight_concentration_prior_type="dirichlet_process", + mean_precision_prior=1e-2, + covariance_prior=1e0 * np.eye(2), + init_params="random", + max_iter=100, + random_state=2, +).fit(X) +plot_results( + X, + dpgmm.predict(X), + dpgmm.means_, + dpgmm.covariances_, + 1, + "Bayesian Gaussian mixture models with a Dirichlet process prior " + r"for $\gamma_0=0.01$.", +) X_s, y_s = dpgmm.sample(n_samples=2000) -plot_samples(X_s, y_s, dpgmm.n_components, 0, - "Gaussian mixture with a Dirichlet process prior " - r"for $\gamma_0=0.01$ sampled with $2000$ samples.") +plot_samples( + X_s, + y_s, + dpgmm.n_components, + 0, + "Gaussian mixture with a Dirichlet process prior " + r"for $\gamma_0=0.01$ sampled with $2000$ samples.", +) dpgmm = mixture.BayesianGaussianMixture( - n_components=10, covariance_type='full', weight_concentration_prior=1e+2, - weight_concentration_prior_type='dirichlet_process', - mean_precision_prior=1e-2, covariance_prior=1e0 * np.eye(2), - init_params="kmeans", max_iter=100, random_state=2).fit(X) -plot_results(X, dpgmm.predict(X), dpgmm.means_, dpgmm.covariances_, 2, - "Bayesian Gaussian mixture models with a Dirichlet process prior " - r"for $\gamma_0=100$") + n_components=10, + covariance_type="full", + weight_concentration_prior=1e2, + weight_concentration_prior_type="dirichlet_process", + mean_precision_prior=1e-2, + covariance_prior=1e0 * np.eye(2), + init_params="kmeans", + max_iter=100, + random_state=2, +).fit(X) +plot_results( + X, + dpgmm.predict(X), + dpgmm.means_, + dpgmm.covariances_, + 2, + "Bayesian Gaussian mixture models with a Dirichlet process prior " + r"for $\gamma_0=100$", +) X_s, y_s = dpgmm.sample(n_samples=2000) -plot_samples(X_s, y_s, dpgmm.n_components, 1, - "Gaussian mixture with a Dirichlet process prior " - r"for $\gamma_0=100$ sampled with $2000$ samples.") +plot_samples( + X_s, + y_s, + dpgmm.n_components, + 1, + "Gaussian mixture with a Dirichlet process prior " + r"for $\gamma_0=100$ sampled with $2000$ samples.", +) plt.show() diff --git a/examples/model_selection/grid_search_text_feature_extraction.py b/examples/model_selection/grid_search_text_feature_extraction.py deleted file mode 100644 index 79bc88a68d1c2..0000000000000 --- a/examples/model_selection/grid_search_text_feature_extraction.py +++ /dev/null @@ -1,128 +0,0 @@ - -""" -========================================================== -Sample pipeline for text feature extraction and evaluation -========================================================== - -The dataset used in this example is the 20 newsgroups dataset which will be -automatically downloaded and then cached and reused for the document -classification example. - -You can adjust the number of categories by giving their names to the dataset -loader or setting them to None to get the 20 of them. - -Here is a sample output of a run on a quad-core machine:: - - Loading 20 newsgroups dataset for categories: - ['alt.atheism', 'talk.religion.misc'] - 1427 documents - 2 categories - - Performing grid search... - pipeline: ['vect', 'tfidf', 'clf'] - parameters: - {'clf__alpha': (1.0000000000000001e-05, 9.9999999999999995e-07), - 'clf__max_iter': (10, 50, 80), - 'clf__penalty': ('l2', 'elasticnet'), - 'tfidf__use_idf': (True, False), - 'vect__max_n': (1, 2), - 'vect__max_df': (0.5, 0.75, 1.0), - 'vect__max_features': (None, 5000, 10000, 50000)} - done in 1737.030s - - Best score: 0.940 - Best parameters set: - clf__alpha: 9.9999999999999995e-07 - clf__max_iter: 50 - clf__penalty: 'elasticnet' - tfidf__use_idf: True - vect__max_n: 2 - vect__max_df: 0.75 - vect__max_features: 50000 - -""" - -# Author: Olivier Grisel -# Peter Prettenhofer -# Mathieu Blondel -# License: BSD 3 clause -from pprint import pprint -from time import time -import logging - -from sklearn.datasets import fetch_20newsgroups -from sklearn.feature_extraction.text import CountVectorizer -from sklearn.feature_extraction.text import TfidfTransformer -from sklearn.linear_model import SGDClassifier -from sklearn.model_selection import GridSearchCV -from sklearn.pipeline import Pipeline - -print(__doc__) - -# Display progress logs on stdout -logging.basicConfig(level=logging.INFO, - format='%(asctime)s %(levelname)s %(message)s') - - -# ############################################################################# -# Load some categories from the training set -categories = [ - 'alt.atheism', - 'talk.religion.misc', -] -# Uncomment the following to do the analysis on all the categories -#categories = None - -print("Loading 20 newsgroups dataset for categories:") -print(categories) - -data = fetch_20newsgroups(subset='train', categories=categories) -print("%d documents" % len(data.filenames)) -print("%d categories" % len(data.target_names)) -print() - -# ############################################################################# -# Define a pipeline combining a text feature extractor with a simple -# classifier -pipeline = Pipeline([ - ('vect', CountVectorizer()), - ('tfidf', TfidfTransformer()), - ('clf', SGDClassifier()), -]) - -# uncommenting more parameters will give better exploring power but will -# increase processing time in a combinatorial way -parameters = { - 'vect__max_df': (0.5, 0.75, 1.0), - # 'vect__max_features': (None, 5000, 10000, 50000), - 'vect__ngram_range': ((1, 1), (1, 2)), # unigrams or bigrams - # 'tfidf__use_idf': (True, False), - # 'tfidf__norm': ('l1', 'l2'), - 'clf__max_iter': (20,), - 'clf__alpha': (0.00001, 0.000001), - 'clf__penalty': ('l2', 'elasticnet'), - # 'clf__max_iter': (10, 50, 80), -} - -if __name__ == "__main__": - # multiprocessing requires the fork to happen in a __main__ protected - # block - - # find the best parameters for both the feature extraction and the - # classifier - grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1) - - print("Performing grid search...") - print("pipeline:", [name for name, _ in pipeline.steps]) - print("parameters:") - pprint(parameters) - t0 = time() - grid_search.fit(data.data, data.target) - print("done in %0.3fs" % (time() - t0)) - print() - - print("Best score: %0.3f" % grid_search.best_score_) - print("Best parameters set:") - best_parameters = grid_search.best_estimator_.get_params() - for param_name in sorted(parameters.keys()): - print("\t%s: %r" % (param_name, best_parameters[param_name])) diff --git a/examples/model_selection/plot_confusion_matrix.py b/examples/model_selection/plot_confusion_matrix.py index d54d9747a8cf3..9a0312d34f005 100644 --- a/examples/model_selection/plot_confusion_matrix.py +++ b/examples/model_selection/plot_confusion_matrix.py @@ -24,14 +24,15 @@ """ -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt +import numpy as np -from sklearn import svm, datasets -from sklearn.model_selection import train_test_split +from sklearn import datasets, svm from sklearn.metrics import ConfusionMatrixDisplay +from sklearn.model_selection import train_test_split # import some data to play with iris = datasets.load_iris() @@ -44,17 +45,23 @@ # Run classifier, using a model that is too regularized (C too low) to see # the impact on the results -classifier = svm.SVC(kernel='linear', C=0.01).fit(X_train, y_train) +classifier = svm.SVC(kernel="linear", C=0.01).fit(X_train, y_train) np.set_printoptions(precision=2) # Plot non-normalized confusion matrix -titles_options = [("Confusion matrix, without normalization", None), - ("Normalized confusion matrix", 'true')] +titles_options = [ + ("Confusion matrix, without normalization", None), + ("Normalized confusion matrix", "true"), +] for title, normalize in titles_options: disp = ConfusionMatrixDisplay.from_estimator( - classifier, X_test, y_test, display_labels=class_names, - cmap=plt.cm.Blues, normalize=normalize + classifier, + X_test, + y_test, + display_labels=class_names, + cmap=plt.cm.Blues, + normalize=normalize, ) disp.ax_.set_title(title) diff --git a/examples/model_selection/plot_cost_sensitive_learning.py b/examples/model_selection/plot_cost_sensitive_learning.py new file mode 100644 index 0000000000000..6b5b651463b05 --- /dev/null +++ b/examples/model_selection/plot_cost_sensitive_learning.py @@ -0,0 +1,693 @@ +""" +============================================================== +Post-tuning the decision threshold for cost-sensitive learning +============================================================== + +Once a classifier is trained, the output of the :term:`predict` method outputs class +label predictions corresponding to a thresholding of either the +:term:`decision_function` or the :term:`predict_proba` output. For a binary classifier, +the default threshold is defined as a posterior probability estimate of 0.5 or a +decision score of 0.0. + +However, this default strategy is most likely not optimal for the task at hand. +Here, we use the "Statlog" German credit dataset [1]_ to illustrate a use case. +In this dataset, the task is to predict whether a person has a "good" or "bad" credit. +In addition, a cost-matrix is provided that specifies the cost of +misclassification. Specifically, misclassifying a "bad" credit as "good" is five +times more costly on average than misclassifying a "good" credit as "bad". + +We use the :class:`~sklearn.model_selection.TunedThresholdClassifierCV` to select the +cut-off point of the decision function that minimizes the provided business +cost. + +In the second part of the example, we further extend this approach by +considering the problem of fraud detection in credit card transactions: in this +case, the business metric depends on the amount of each individual transaction. + +.. rubric :: References + +.. [1] "Statlog (German Credit Data) Data Set", UCI Machine Learning Repository, + `Link `_. + +.. [2] `Charles Elkan, "The Foundations of Cost-Sensitive Learning", + International joint conference on artificial intelligence. + Vol. 17. No. 1. Lawrence Erlbaum Associates Ltd, 2001. + `_ +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Cost-sensitive learning with constant gains and costs +# ----------------------------------------------------- +# +# In this first section, we illustrate the use of the +# :class:`~sklearn.model_selection.TunedThresholdClassifierCV` in a setting of +# cost-sensitive learning when the gains and costs associated to each entry of the +# confusion matrix are constant. We use the problematic presented in [2]_ using the +# "Statlog" German credit dataset [1]_. +# +# "Statlog" German credit dataset +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# We fetch the German credit dataset from OpenML. +import sklearn +from sklearn.datasets import fetch_openml + +sklearn.set_config(transform_output="pandas") + +german_credit = fetch_openml(data_id=31, as_frame=True, parser="pandas") +X, y = german_credit.data, german_credit.target + +# %% +# We check the feature types available in `X`. +X.info() + +# %% +# Many features are categorical and usually string-encoded. We need to encode +# these categories when we develop our predictive model. Let's check the targets. +y.value_counts() + +# %% +# Another observation is that the dataset is imbalanced. We would need to be careful +# when evaluating our predictive model and use a family of metrics that are adapted +# to this setting. +# +# In addition, we observe that the target is string-encoded. Some metrics +# (e.g. precision and recall) require to provide the label of interest also called +# the "positive label". Here, we define that our goal is to predict whether or not +# a sample is a "bad" credit. +pos_label, neg_label = "bad", "good" + +# %% +# To carry our analysis, we split our dataset using a single stratified split. +from sklearn.model_selection import train_test_split + +X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) + +# %% +# We are ready to design our predictive model and the associated evaluation strategy. +# +# Evaluation metrics +# ^^^^^^^^^^^^^^^^^^ +# +# In this section, we define a set of metrics that we use later. To see +# the effect of tuning the cut-off point, we evaluate the predictive model using +# the Receiver Operating Characteristic (ROC) curve and the Precision-Recall curve. +# The values reported on these plots are therefore the true positive rate (TPR), +# also known as the recall or the sensitivity, and the false positive rate (FPR), +# also known as the specificity, for the ROC curve and the precision and recall for +# the Precision-Recall curve. +# +# From these four metrics, scikit-learn does not provide a scorer for the FPR. We +# therefore need to define a small custom function to compute it. +from sklearn.metrics import confusion_matrix + + +def fpr_score(y, y_pred, neg_label, pos_label): + cm = confusion_matrix(y, y_pred, labels=[neg_label, pos_label]) + tn, fp, _, _ = cm.ravel() + tnr = tn / (tn + fp) + return 1 - tnr + + +# %% +# As previously stated, the "positive label" is not defined as the value "1" and calling +# some of the metrics with this non-standard value raise an error. We need to +# provide the indication of the "positive label" to the metrics. +# +# We therefore need to define a scikit-learn scorer using +# :func:`~sklearn.metrics.make_scorer` where the information is passed. We store all +# the custom scorers in a dictionary. To use them, we need to pass the fitted model, +# the data and the target on which we want to evaluate the predictive model. +from sklearn.metrics import make_scorer, precision_score, recall_score + +tpr_score = recall_score # TPR and recall are the same metric +scoring = { + "precision": make_scorer(precision_score, pos_label=pos_label), + "recall": make_scorer(recall_score, pos_label=pos_label), + "fpr": make_scorer(fpr_score, neg_label=neg_label, pos_label=pos_label), + "tpr": make_scorer(tpr_score, pos_label=pos_label), +} + +# %% +# In addition, the original research [1]_ defines a custom business metric. We +# call a "business metric" any metric function that aims at quantifying how the +# predictions (correct or wrong) might impact the business value of deploying a +# given machine learning model in a specific application context. For our +# credit prediction task, the authors provide a custom cost-matrix which +# encodes that classifying a a "bad" credit as "good" is 5 times more costly on +# average than the opposite: it is less costly for the financing institution to +# not grant a credit to a potential customer that will not default (and +# therefore miss a good customer that would have otherwise both reimbursed the +# credit and paid interests) than to grant a credit to a customer that will +# default. +# +# We define a python function that weight the confusion matrix and return the +# overall cost. +import numpy as np + + +def credit_gain_score(y, y_pred, neg_label, pos_label): + cm = confusion_matrix(y, y_pred, labels=[neg_label, pos_label]) + # The rows of the confusion matrix hold the counts of observed classes + # while the columns hold counts of predicted classes. Recall that here we + # consider "bad" as the positive class (second row and column). + # Scikit-learn model selection tools expect that we follow a convention + # that "higher" means "better", hence the following gain matrix assigns + # negative gains (costs) to the two kinds of prediction errors: + # - a gain of -1 for each false positive ("good" credit labeled as "bad"), + # - a gain of -5 for each false negative ("bad" credit labeled as "good"), + # The true positives and true negatives are assigned null gains in this + # metric. + # + # Note that theoretically, given that our model is calibrated and our data + # set representative and large enough, we do not need to tune the + # threshold, but can safely set it to the cost ration 1/5, as stated by Eq. + # (2) in Elkan paper [2]_. + gain_matrix = np.array( + [ + [0, -1], # -1 gain for false positives + [-5, 0], # -5 gain for false negatives + ] + ) + return np.sum(cm * gain_matrix) + + +scoring["credit_gain"] = make_scorer( + credit_gain_score, neg_label=neg_label, pos_label=pos_label +) +# %% +# Vanilla predictive model +# ^^^^^^^^^^^^^^^^^^^^^^^^ +# +# We use :class:`~sklearn.ensemble.HistGradientBoostingClassifier` as a predictive model +# that natively handles categorical features and missing values. +from sklearn.ensemble import HistGradientBoostingClassifier + +model = HistGradientBoostingClassifier( + categorical_features="from_dtype", random_state=0 +).fit(X_train, y_train) +model + +# %% +# We evaluate the performance of our predictive model using the ROC and Precision-Recall +# curves. +import matplotlib.pyplot as plt + +from sklearn.metrics import PrecisionRecallDisplay, RocCurveDisplay + +fig, axs = plt.subplots(nrows=1, ncols=2, figsize=(14, 6)) + +PrecisionRecallDisplay.from_estimator( + model, X_test, y_test, pos_label=pos_label, ax=axs[0], name="GBDT" +) +axs[0].plot( + scoring["recall"](model, X_test, y_test), + scoring["precision"](model, X_test, y_test), + marker="o", + markersize=10, + color="tab:blue", + label="Default cut-off point at a probability of 0.5", +) +axs[0].set_title("Precision-Recall curve") +axs[0].legend() + +RocCurveDisplay.from_estimator( + model, + X_test, + y_test, + pos_label=pos_label, + ax=axs[1], + name="GBDT", + plot_chance_level=True, +) +axs[1].plot( + scoring["fpr"](model, X_test, y_test), + scoring["tpr"](model, X_test, y_test), + marker="o", + markersize=10, + color="tab:blue", + label="Default cut-off point at a probability of 0.5", +) +axs[1].set_title("ROC curve") +axs[1].legend() +_ = fig.suptitle("Evaluation of the vanilla GBDT model") + +# %% +# We recall that these curves give insights on the statistical performance of the +# predictive model for different cut-off points. For the Precision-Recall curve, the +# reported metrics are the precision and recall and for the ROC curve, the reported +# metrics are the TPR (same as recall) and FPR. +# +# Here, the different cut-off points correspond to different levels of posterior +# probability estimates ranging between 0 and 1. By default, `model.predict` uses a +# cut-off point at a probability estimate of 0.5. The metrics for such a cut-off point +# are reported with the blue dot on the curves: it corresponds to the statistical +# performance of the model when using `model.predict`. +# +# However, we recall that the original aim was to minimize the cost (or maximize the +# gain) as defined by the business metric. We can compute the value of the business +# metric: +print(f"Business defined metric: {scoring['credit_gain'](model, X_test, y_test)}") + +# %% +# At this stage we don't know if any other cut-off can lead to a greater gain. To find +# the optimal one, we need to compute the cost-gain using the business metric for all +# possible cut-off points and choose the best. This strategy can be quite tedious to +# implement by hand, but the +# :class:`~sklearn.model_selection.TunedThresholdClassifierCV` class is here to help us. +# It automatically computes the cost-gain for all possible cut-off points and optimizes +# for the `scoring`. +# +# .. _cost_sensitive_learning_example: +# +# Tuning the cut-off point +# ^^^^^^^^^^^^^^^^^^^^^^^^ +# +# We use :class:`~sklearn.model_selection.TunedThresholdClassifierCV` to tune the +# cut-off point. We need to provide the business metric to optimize as well as the +# positive label. Internally, the optimum cut-off point is chosen such that it maximizes +# the business metric via cross-validation. By default a 5-fold stratified +# cross-validation is used. +from sklearn.model_selection import TunedThresholdClassifierCV + +tuned_model = TunedThresholdClassifierCV( + estimator=model, + scoring=scoring["credit_gain"], + store_cv_results=True, # necessary to inspect all results +) +tuned_model.fit(X_train, y_train) +print(f"{tuned_model.best_threshold_=:0.2f}") + +# %% +# We plot the ROC and Precision-Recall curves for the vanilla model and the tuned model. +# Also we plot the cut-off points that would be used by each model. Because, we are +# reusing the same code later, we define a function that generates the plots. + + +def plot_roc_pr_curves(vanilla_model, tuned_model, *, title): + fig, axs = plt.subplots(nrows=1, ncols=3, figsize=(21, 6)) + + linestyles = ("dashed", "dotted") + markerstyles = ("o", ">") + colors = ("tab:blue", "tab:orange") + names = ("Vanilla GBDT", "Tuned GBDT") + for idx, (est, linestyle, marker, color, name) in enumerate( + zip((vanilla_model, tuned_model), linestyles, markerstyles, colors, names) + ): + decision_threshold = getattr(est, "best_threshold_", 0.5) + PrecisionRecallDisplay.from_estimator( + est, + X_test, + y_test, + pos_label=pos_label, + linestyle=linestyle, + color=color, + ax=axs[0], + name=name, + ) + axs[0].plot( + scoring["recall"](est, X_test, y_test), + scoring["precision"](est, X_test, y_test), + marker, + markersize=10, + color=color, + label=f"Cut-off point at probability of {decision_threshold:.2f}", + ) + RocCurveDisplay.from_estimator( + est, + X_test, + y_test, + pos_label=pos_label, + curve_kwargs=dict(linestyle=linestyle, color=color), + ax=axs[1], + name=name, + plot_chance_level=idx == 1, + ) + axs[1].plot( + scoring["fpr"](est, X_test, y_test), + scoring["tpr"](est, X_test, y_test), + marker, + markersize=10, + color=color, + label=f"Cut-off point at probability of {decision_threshold:.2f}", + ) + + axs[0].set_title("Precision-Recall curve") + axs[0].legend() + axs[1].set_title("ROC curve") + axs[1].legend() + + axs[2].plot( + tuned_model.cv_results_["thresholds"], + tuned_model.cv_results_["scores"], + color="tab:orange", + ) + axs[2].plot( + tuned_model.best_threshold_, + tuned_model.best_score_, + "o", + markersize=10, + color="tab:orange", + label="Optimal cut-off point for the business metric", + ) + axs[2].legend() + axs[2].set_xlabel("Decision threshold (probability)") + axs[2].set_ylabel("Objective score (using cost-matrix)") + axs[2].set_title("Objective score as a function of the decision threshold") + fig.suptitle(title) + + +# %% +title = "Comparison of the cut-off point for the vanilla and tuned GBDT model" +plot_roc_pr_curves(model, tuned_model, title=title) + +# %% +# The first remark is that both classifiers have exactly the same ROC and +# Precision-Recall curves. It is expected because by default, the classifier is fitted +# on the same training data. In a later section, we discuss more in detail the +# available options regarding model refitting and cross-validation. +# +# The second remark is that the cut-off points of the vanilla and tuned model are +# different. To understand why the tuned model has chosen this cut-off point, we can +# look at the right-hand side plot that plots the objective score that is our exactly +# the same as our business metric. We see that the optimum threshold corresponds to the +# maximum of the objective score. This maximum is reached for a decision threshold +# much lower than 0.5: the tuned model enjoys a much higher recall at the cost of +# of significantly lower precision: the tuned model is much more eager to +# predict the "bad" class label to larger fraction of individuals. +# +# We can now check if choosing this cut-off point leads to a better score on the testing +# set: +print(f"Business defined metric: {scoring['credit_gain'](tuned_model, X_test, y_test)}") + +# %% +# We observe that tuning the decision threshold almost improves our business gains +# by factor of 2. +# +# .. _TunedThresholdClassifierCV_no_cv: +# +# Consideration regarding model refitting and cross-validation +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# In the above experiment, we used the default setting of the +# :class:`~sklearn.model_selection.TunedThresholdClassifierCV`. In particular, the +# cut-off point is tuned using a 5-fold stratified cross-validation. Also, the +# underlying predictive model is refitted on the entire training data once the cut-off +# point is chosen. +# +# These two strategies can be changed by providing the `refit` and `cv` parameters. +# For instance, one could provide a fitted `estimator` and set `cv="prefit"`, in which +# case the cut-off point is found on the entire dataset provided at fitting time. +# Also, the underlying classifier is not be refitted by setting `refit=False`. Here, we +# can try to do such experiment. +model.fit(X_train, y_train) +tuned_model.set_params(cv="prefit", refit=False).fit(X_train, y_train) +print(f"{tuned_model.best_threshold_=:0.2f}") + + +# %% +# Then, we evaluate our model with the same approach as before: +title = "Tuned GBDT model without refitting and using the entire dataset" +plot_roc_pr_curves(model, tuned_model, title=title) + +# %% +# We observe the that the optimum cut-off point is different from the one found +# in the previous experiment. If we look at the right-hand side plot, we +# observe that the business gain has large plateau of near-optimal 0 gain for a +# large span of decision thresholds. This behavior is symptomatic of an +# overfitting. Because we disable cross-validation, we tuned the cut-off point +# on the same set as the model was trained on, and this is the reason for the +# observed overfitting. +# +# This option should therefore be used with caution. One needs to make sure that the +# data provided at fitting time to the +# :class:`~sklearn.model_selection.TunedThresholdClassifierCV` is not the same as the +# data used to train the underlying classifier. This could happen sometimes when the +# idea is just to tune the predictive model on a completely new validation set without a +# costly complete refit. +# +# When cross-validation is too costly, a potential alternative is to use a +# single train-test split by providing a floating number in range `[0, 1]` to the `cv` +# parameter. It splits the data into a training and testing set. Let's explore this +# option: +tuned_model.set_params(cv=0.75).fit(X_train, y_train) + +# %% +title = "Tuned GBDT model without refitting and using the entire dataset" +plot_roc_pr_curves(model, tuned_model, title=title) + +# %% +# Regarding the cut-off point, we observe that the optimum is similar to the multiple +# repeated cross-validation case. However, be aware that a single split does not account +# for the variability of the fit/predict process and thus we are unable to know if there +# is any variance in the cut-off point. The repeated cross-validation averages out +# this effect. +# +# Another observation concerns the ROC and Precision-Recall curves of the tuned model. +# As expected, these curves differ from those of the vanilla model, given that we +# trained the underlying classifier on a subset of the data provided during fitting and +# reserved a validation set for tuning the cut-off point. +# +# Cost-sensitive learning when gains and costs are not constant +# ------------------------------------------------------------- +# +# As stated in [2]_, gains and costs are generally not constant in real-world problems. +# In this section, we use a similar example as in [2]_ for the problem of +# detecting fraud in credit card transaction records. +# +# The credit card dataset +# ^^^^^^^^^^^^^^^^^^^^^^^ +credit_card = fetch_openml(data_id=1597, as_frame=True, parser="pandas") +credit_card.frame.info() + +# %% +# The dataset contains information about credit card records from which some are +# fraudulent and others are legitimate. The goal is therefore to predict whether or +# not a credit card record is fraudulent. +columns_to_drop = ["Class"] +data = credit_card.frame.drop(columns=columns_to_drop) +target = credit_card.frame["Class"].astype(int) + +# %% +# First, we check the class distribution of the datasets. +target.value_counts(normalize=True) + +# %% +# The dataset is highly imbalanced with fraudulent transaction representing only 0.17% +# of the data. Since we are interested in training a machine learning model, we should +# also make sure that we have enough samples in the minority class to train the model. +target.value_counts() + +# %% +# We observe that we have around 500 samples that is on the low end of the number of +# samples required to train a machine learning model. In addition of the target +# distribution, we check the distribution of the amount of the +# fraudulent transactions. +fraud = target == 1 +amount_fraud = data["Amount"][fraud] +_, ax = plt.subplots() +ax.hist(amount_fraud, bins=30) +ax.set_title("Amount of fraud transaction") +_ = ax.set_xlabel("Amount (â‚Ŧ)") + +# %% +# Addressing the problem with a business metric +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# Now, we create the business metric that depends on the amount of each transaction. We +# define the cost matrix similarly to [2]_. Accepting a legitimate transaction provides +# a gain of 2% of the amount of the transaction. However, accepting a fraudulent +# transaction result in a loss of the amount of the transaction. As stated in [2]_, the +# gain and loss related to refusals (of fraudulent and legitimate transactions) are not +# trivial to define. Here, we define that a refusal of a legitimate transaction +# is estimated to a loss of 5â‚Ŧ while the refusal of a fraudulent transaction is +# estimated to a gain of 50â‚Ŧ. Therefore, we define the following function to +# compute the total benefit of a given decision: + + +def business_metric(y_true, y_pred, amount): + mask_true_positive = (y_true == 1) & (y_pred == 1) + mask_true_negative = (y_true == 0) & (y_pred == 0) + mask_false_positive = (y_true == 0) & (y_pred == 1) + mask_false_negative = (y_true == 1) & (y_pred == 0) + fraudulent_refuse = mask_true_positive.sum() * 50 + fraudulent_accept = -amount[mask_false_negative].sum() + legitimate_refuse = mask_false_positive.sum() * -5 + legitimate_accept = (amount[mask_true_negative] * 0.02).sum() + return fraudulent_refuse + fraudulent_accept + legitimate_refuse + legitimate_accept + + +# %% +# From this business metric, we create a scikit-learn scorer that given a fitted +# classifier and a test set compute the business metric. In this regard, we use +# the :func:`~sklearn.metrics.make_scorer` factory. The variable `amount` is an +# additional metadata to be passed to the scorer and we need to use +# :ref:`metadata routing ` to take into account this information. +sklearn.set_config(enable_metadata_routing=True) +business_scorer = make_scorer(business_metric).set_score_request(amount=True) + +# %% +# So at this stage, we observe that the amount of the transaction is used twice: once +# as a feature to train our predictive model and once as a metadata to compute the +# the business metric and thus the statistical performance of our model. When used as a +# feature, we are only required to have a column in `data` that contains the amount of +# each transaction. To use this information as metadata, we need to have an external +# variable that we can pass to the scorer or the model that internally routes this +# metadata to the scorer. So let's create this variable. +amount = credit_card.frame["Amount"].to_numpy() + +# %% +from sklearn.model_selection import train_test_split + +data_train, data_test, target_train, target_test, amount_train, amount_test = ( + train_test_split( + data, target, amount, stratify=target, test_size=0.5, random_state=42 + ) +) + +# %% +# We first evaluate some baseline policies to serve as reference. Recall that +# class "0" is the legitimate class and class "1" is the fraudulent class. +from sklearn.dummy import DummyClassifier + +always_accept_policy = DummyClassifier(strategy="constant", constant=0) +always_accept_policy.fit(data_train, target_train) +benefit = business_scorer( + always_accept_policy, data_test, target_test, amount=amount_test +) +print(f"Benefit of the 'always accept' policy: {benefit:,.2f}â‚Ŧ") + +# %% +# A policy that considers all transactions as legitimate would create a profit of +# around 220,000â‚Ŧ. We make the same evaluation for a classifier that predicts all +# transactions as fraudulent. +always_reject_policy = DummyClassifier(strategy="constant", constant=1) +always_reject_policy.fit(data_train, target_train) +benefit = business_scorer( + always_reject_policy, data_test, target_test, amount=amount_test +) +print(f"Benefit of the 'always reject' policy: {benefit:,.2f}â‚Ŧ") + + +# %% +# Such a policy would entail a catastrophic loss: around 670,000â‚Ŧ. This is +# expected since the vast majority of the transactions are legitimate and the +# policy would refuse them at a non-trivial cost. +# +# A predictive model that adapts the accept/reject decisions on a per +# transaction basis should ideally allow us to make a profit larger than the +# 220,000â‚Ŧ of the best of our constant baseline policies. +# +# We start with a logistic regression model with the default decision threshold +# at 0.5. Here we tune the hyperparameter `C` of the logistic regression with a +# proper scoring rule (the log loss) to ensure that the model's probabilistic +# predictions returned by its `predict_proba` method are as accurate as +# possible, irrespectively of the choice of the value of the decision +# threshold. +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import GridSearchCV +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import StandardScaler + +logistic_regression = make_pipeline(StandardScaler(), LogisticRegression()) +param_grid = {"logisticregression__C": np.logspace(-6, 6, 13)} +model = GridSearchCV(logistic_regression, param_grid, scoring="neg_log_loss").fit( + data_train, target_train +) +model + +# %% +print( + "Benefit of logistic regression with default threshold: " + f"{business_scorer(model, data_test, target_test, amount=amount_test):,.2f}â‚Ŧ" +) + +# %% +# The business metric shows that our predictive model with a default decision +# threshold is already winning over the baseline in terms of profit and it would be +# already beneficial to use it to accept or reject transactions instead of +# accepting all transactions. +# +# Tuning the decision threshold +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# Now the question is: is our model optimum for the type of decision that we want to do? +# Up to now, we did not optimize the decision threshold. We use the +# :class:`~sklearn.model_selection.TunedThresholdClassifierCV` to optimize the decision +# given our business scorer. To avoid a nested cross-validation, we will use the +# best estimator found during the previous grid-search. +tuned_model = TunedThresholdClassifierCV( + estimator=model.best_estimator_, + scoring=business_scorer, + thresholds=100, + n_jobs=2, +) + +# %% +# Since our business scorer requires the amount of each transaction, we need to pass +# this information in the `fit` method. The +# :class:`~sklearn.model_selection.TunedThresholdClassifierCV` is in charge of +# automatically dispatching this metadata to the underlying scorer. +tuned_model.fit(data_train, target_train, amount=amount_train) + +# %% +# We observe that the tuned decision threshold is far away from the default 0.5: +print(f"Tuned decision threshold: {tuned_model.best_threshold_:.2f}") + +# %% +print( + "Benefit of logistic regression with a tuned threshold: " + f"{business_scorer(tuned_model, data_test, target_test, amount=amount_test):,.2f}â‚Ŧ" +) + +# %% +# We observe that tuning the decision threshold increases the expected profit +# when deploying our model - as indicated by the business metric. It is therefore +# valuable, whenever possible, to optimize the decision threshold with respect +# to the business metric. +# +# Manually setting the decision threshold instead of tuning it +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# In the previous example, we used the +# :class:`~sklearn.model_selection.TunedThresholdClassifierCV` to find the optimal +# decision threshold. However, in some cases, we might have some prior knowledge about +# the problem at hand and we might be happy to set the decision threshold manually. +# +# The class :class:`~sklearn.model_selection.FixedThresholdClassifier` allows us to +# manually set the decision threshold. At prediction time, it behave as the previous +# tuned model but no search is performed during the fitting process. Note that here +# we use :class:`~sklearn.frozen.FrozenEstimator` to wrap the predictive model to +# avoid any refitting. +# +# Here, we will reuse the decision threshold found in the previous section to create a +# new model and check that it gives the same results. +from sklearn.frozen import FrozenEstimator +from sklearn.model_selection import FixedThresholdClassifier + +model_fixed_threshold = FixedThresholdClassifier( + estimator=FrozenEstimator(model), threshold=tuned_model.best_threshold_ +) + +# %% +business_score = business_scorer( + model_fixed_threshold, data_test, target_test, amount=amount_test +) +print(f"Benefit of logistic regression with a tuned threshold: {business_score:,.2f}â‚Ŧ") + +# %% +# We observe that we obtained the exact same results but the fitting process +# was much faster since we did not perform any hyper-parameter search. +# +# Finally, the estimate of the (average) business metric itself can be unreliable, in +# particular when the number of data points in the minority class is very small. +# Any business impact estimated by cross-validation of a business metric on +# historical data (offline evaluation) should ideally be confirmed by A/B testing +# on live data (online evaluation). Note however that A/B testing models is +# beyond the scope of the scikit-learn library itself. + +# At the end, we disable the configuration flag for metadata routing:: +sklearn.set_config(enable_metadata_routing=False) diff --git a/examples/model_selection/plot_cv_indices.py b/examples/model_selection/plot_cv_indices.py index 91f71b0451cb2..b922fc75d7473 100644 --- a/examples/model_selection/plot_cv_indices.py +++ b/examples/model_selection/plot_cv_indices.py @@ -9,15 +9,28 @@ This example visualizes the behavior of several common scikit-learn objects for comparison. + """ -from sklearn.model_selection import (TimeSeriesSplit, KFold, ShuffleSplit, - StratifiedKFold, GroupShuffleSplit, - GroupKFold, StratifiedShuffleSplit) -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt +import numpy as np from matplotlib.patches import Patch -np.random.seed(1338) + +from sklearn.model_selection import ( + GroupKFold, + GroupShuffleSplit, + KFold, + ShuffleSplit, + StratifiedGroupKFold, + StratifiedKFold, + StratifiedShuffleSplit, + TimeSeriesSplit, +) + +rng = np.random.RandomState(1338) cmap_data = plt.cm.Paired cmap_cv = plt.cm.coolwarm n_splits = 4 @@ -38,28 +51,44 @@ # Generate the class/group data n_points = 100 -X = np.random.randn(100, 10) +X = rng.randn(100, 10) -percentiles_classes = [.1, .3, .6] -y = np.hstack([[ii] * int(100 * perc) - for ii, perc in enumerate(percentiles_classes)]) +percentiles_classes = [0.1, 0.3, 0.6] +y = np.hstack([[ii] * int(100 * perc) for ii, perc in enumerate(percentiles_classes)]) -# Evenly spaced groups repeated once -groups = np.hstack([[ii] * 10 for ii in range(10)]) +# Generate uneven groups +group_prior = rng.dirichlet([2] * 10) +groups = np.repeat(np.arange(10), rng.multinomial(100, group_prior)) def visualize_groups(classes, groups, name): # Visualize dataset groups fig, ax = plt.subplots() - ax.scatter(range(len(groups)), [.5] * len(groups), c=groups, marker='_', - lw=50, cmap=cmap_data) - ax.scatter(range(len(groups)), [3.5] * len(groups), c=classes, marker='_', - lw=50, cmap=cmap_data) - ax.set(ylim=[-1, 5], yticks=[.5, 3.5], - yticklabels=['Data\ngroup', 'Data\nclass'], xlabel="Sample index") - - -visualize_groups(y, groups, 'no groups') + ax.scatter( + range(len(groups)), + [0.5] * len(groups), + c=groups, + marker="_", + lw=50, + cmap=cmap_data, + ) + ax.scatter( + range(len(groups)), + [3.5] * len(groups), + c=classes, + marker="_", + lw=50, + cmap=cmap_data, + ) + ax.set( + ylim=[-1, 5], + yticks=[0.5, 3.5], + yticklabels=["Data\ngroup", "Data\nclass"], + xlabel="Sample index", + ) + + +visualize_groups(y, groups, "no groups") # %% # Define a function to visualize cross-validation behavior @@ -73,32 +102,47 @@ def visualize_groups(classes, groups, name): def plot_cv_indices(cv, X, y, group, ax, n_splits, lw=10): """Create a sample plot for indices of a cross-validation object.""" - + use_groups = "Group" in type(cv).__name__ + groups = group if use_groups else None # Generate the training/testing visualizations for each CV split - for ii, (tr, tt) in enumerate(cv.split(X=X, y=y, groups=group)): + for ii, (tr, tt) in enumerate(cv.split(X=X, y=y, groups=groups)): # Fill in indices with the training/test groups indices = np.array([np.nan] * len(X)) indices[tt] = 1 indices[tr] = 0 # Visualize the results - ax.scatter(range(len(indices)), [ii + .5] * len(indices), - c=indices, marker='_', lw=lw, cmap=cmap_cv, - vmin=-.2, vmax=1.2) + ax.scatter( + range(len(indices)), + [ii + 0.5] * len(indices), + c=indices, + marker="_", + lw=lw, + cmap=cmap_cv, + vmin=-0.2, + vmax=1.2, + ) # Plot the data classes and groups at the end - ax.scatter(range(len(X)), [ii + 1.5] * len(X), - c=y, marker='_', lw=lw, cmap=cmap_data) + ax.scatter( + range(len(X)), [ii + 1.5] * len(X), c=y, marker="_", lw=lw, cmap=cmap_data + ) - ax.scatter(range(len(X)), [ii + 2.5] * len(X), - c=group, marker='_', lw=lw, cmap=cmap_data) + ax.scatter( + range(len(X)), [ii + 2.5] * len(X), c=group, marker="_", lw=lw, cmap=cmap_data + ) # Formatting - yticklabels = list(range(n_splits)) + ['class', 'group'] - ax.set(yticks=np.arange(n_splits+2) + .5, yticklabels=yticklabels, - xlabel='Sample index', ylabel="CV iteration", - ylim=[n_splits+2.2, -.2], xlim=[0, 100]) - ax.set_title('{}'.format(type(cv).__name__), fontsize=15) + yticklabels = list(range(n_splits)) + ["class", "group"] + ax.set( + yticks=np.arange(n_splits + 2) + 0.5, + yticklabels=yticklabels, + xlabel="Sample index", + ylabel="CV iteration", + ylim=[n_splits + 2.2, -0.2], + xlim=[0, 100], + ) + ax.set_title("{}".format(type(cv).__name__), fontsize=15) return ax @@ -113,16 +157,29 @@ def plot_cv_indices(cv, X, y, group, ax, n_splits, lw=10): # %% # As you can see, by default the KFold cross-validation iterator does not # take either datapoint class or group into consideration. We can change this -# by using the ``StratifiedKFold`` like so. +# by using either: +# +# - ``StratifiedKFold`` to preserve the percentage of samples for each class. +# - ``GroupKFold`` to ensure that the same group will not appear in two +# different folds. +# - ``StratifiedGroupKFold`` to keep the constraint of ``GroupKFold`` while +# attempting to return stratified folds. +cvs = [StratifiedKFold, GroupKFold, StratifiedGroupKFold] -fig, ax = plt.subplots() -cv = StratifiedKFold(n_splits) -plot_cv_indices(cv, X, y, groups, ax, n_splits) +for cv in cvs: + fig, ax = plt.subplots(figsize=(6, 3)) + plot_cv_indices(cv(n_splits), X, y, groups, ax, n_splits) + ax.legend( + [Patch(color=cmap_cv(0.8)), Patch(color=cmap_cv(0.02))], + ["Testing set", "Training set"], + loc=(1.02, 0.8), + ) + # Make the legend fit + plt.tight_layout() + fig.subplots_adjust(right=0.7) # %% -# In this case, the cross-validation retained the same ratio of classes across -# each CV split. Next we'll visualize this behavior for a number of CV -# iterators. +# Next we'll visualize this behavior for a number of CV iterators. # # Visualize cross-validation indices for many CV objects # ------------------------------------------------------ @@ -133,8 +190,16 @@ def plot_cv_indices(cv, X, y, group, ax, n_splits, lw=10): # # Note how some use the group/class information while others do not. -cvs = [KFold, GroupKFold, ShuffleSplit, StratifiedKFold, - GroupShuffleSplit, StratifiedShuffleSplit, TimeSeriesSplit] +cvs = [ + KFold, + GroupKFold, + ShuffleSplit, + StratifiedKFold, + StratifiedGroupKFold, + GroupShuffleSplit, + StratifiedShuffleSplit, + TimeSeriesSplit, +] for cv in cvs: @@ -142,9 +207,12 @@ def plot_cv_indices(cv, X, y, group, ax, n_splits, lw=10): fig, ax = plt.subplots(figsize=(6, 3)) plot_cv_indices(this_cv, X, y, groups, ax, n_splits) - ax.legend([Patch(color=cmap_cv(.8)), Patch(color=cmap_cv(.02))], - ['Testing set', 'Training set'], loc=(1.02, .8)) + ax.legend( + [Patch(color=cmap_cv(0.8)), Patch(color=cmap_cv(0.02))], + ["Testing set", "Training set"], + loc=(1.02, 0.8), + ) # Make the legend fit plt.tight_layout() - fig.subplots_adjust(right=.7) + fig.subplots_adjust(right=0.7) plt.show() diff --git a/examples/model_selection/plot_cv_predict.py b/examples/model_selection/plot_cv_predict.py index ee3e82f42cba1..fa77749020d2b 100644 --- a/examples/model_selection/plot_cv_predict.py +++ b/examples/model_selection/plot_cv_predict.py @@ -4,25 +4,79 @@ ==================================== This example shows how to use -:func:`~sklearn.model_selection.cross_val_predict` to visualize prediction +:func:`~sklearn.model_selection.cross_val_predict` together with +:class:`~sklearn.metrics.PredictionErrorDisplay` to visualize prediction errors. - """ -from sklearn import datasets + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# We will load the diabetes dataset and create an instance of a linear +# regression model. +from sklearn.datasets import load_diabetes +from sklearn.linear_model import LinearRegression + +X, y = load_diabetes(return_X_y=True) +lr = LinearRegression() + +# %% +# :func:`~sklearn.model_selection.cross_val_predict` returns an array of the +# same size of `y` where each entry is a prediction obtained by cross +# validation. from sklearn.model_selection import cross_val_predict -from sklearn import linear_model -import matplotlib.pyplot as plt -lr = linear_model.LinearRegression() -X, y = datasets.load_diabetes(return_X_y=True) +y_pred = cross_val_predict(lr, X, y, cv=10) + +# %% +# Since `cv=10`, it means that we trained 10 models and each model was +# used to predict on one of the 10 folds. We can now use the +# :class:`~sklearn.metrics.PredictionErrorDisplay` to visualize the +# prediction errors. +# +# On the left axis, we plot the observed values :math:`y` vs. the predicted +# values :math:`\hat{y}` given by the models. On the right axis, we plot the +# residuals (i.e. the difference between the observed values and the predicted +# values) vs. the predicted values. +import matplotlib.pyplot as plt -# cross_val_predict returns an array of the same size as `y` where each entry -# is a prediction obtained by cross validation: -predicted = cross_val_predict(lr, X, y, cv=10) +from sklearn.metrics import PredictionErrorDisplay -fig, ax = plt.subplots() -ax.scatter(y, predicted, edgecolors=(0, 0, 0)) -ax.plot([y.min(), y.max()], [y.min(), y.max()], 'k--', lw=4) -ax.set_xlabel('Measured') -ax.set_ylabel('Predicted') +fig, axs = plt.subplots(ncols=2, figsize=(8, 4)) +PredictionErrorDisplay.from_predictions( + y, + y_pred=y_pred, + kind="actual_vs_predicted", + subsample=100, + ax=axs[0], + random_state=0, +) +axs[0].set_title("Actual vs. Predicted values") +PredictionErrorDisplay.from_predictions( + y, + y_pred=y_pred, + kind="residual_vs_predicted", + subsample=100, + ax=axs[1], + random_state=0, +) +axs[1].set_title("Residuals vs. Predicted Values") +fig.suptitle("Plotting cross-validated predictions") +plt.tight_layout() plt.show() + +# %% +# It is important to note that we used +# :func:`~sklearn.model_selection.cross_val_predict` for visualization +# purpose only in this example. +# +# It would be problematic to +# quantitatively assess the model performance by computing a single +# performance metric from the concatenated predictions returned by +# :func:`~sklearn.model_selection.cross_val_predict` +# when the different CV folds vary by size and distributions. +# +# It is recommended to compute per-fold performance metrics using: +# :func:`~sklearn.model_selection.cross_val_score` or +# :func:`~sklearn.model_selection.cross_validate` instead. diff --git a/examples/model_selection/plot_det.py b/examples/model_selection/plot_det.py index 2e27dd07ee684..4a22cdcd44eb8 100644 --- a/examples/model_selection/plot_det.py +++ b/examples/model_selection/plot_det.py @@ -3,36 +3,18 @@ Detection error tradeoff (DET) curve ==================================== -In this example, we compare receiver operating characteristic (ROC) and -detection error tradeoff (DET) curves for different classification algorithms -for the same classification task. - -DET curves are commonly plotted in normal deviate scale. -To achieve this `plot_det_curve` transforms the error rates as returned by the -:func:`~sklearn.metrics.det_curve` and the axis scale using -:func:`scipy.stats.norm`. - -The point of this example is to demonstrate two properties of DET curves, -namely: - -1. It might be easier to visually assess the overall performance of different - classification algorithms using DET curves over ROC curves. - Due to the linear scale used for plotting ROC curves, different classifiers - usually only differ in the top left corner of the graph and appear similar - for a large part of the plot. On the other hand, because DET curves - represent straight lines in normal deviate scale. As such, they tend to be - distinguishable as a whole and the area of interest spans a large part of - the plot. -2. DET curves give the user direct feedback of the detection error tradeoff to - aid in operating point analysis. - The user can deduct directly from the DET-curve plot at which rate - false-negative error rate will improve when willing to accept an increase in - false-positive error rate (or vice-versa). - -The plots in this example compare ROC curves on the left side to corresponding -DET curves on the right. -There is no particular reason why these classifiers have been chosen for the -example plot over other classifiers available in scikit-learn. +In this example, we compare two binary classification multi-threshold metrics: +the Receiver Operating Characteristic (ROC) and the Detection Error Tradeoff +(DET). For such purpose, we evaluate two different classifiers for the same +classification task. + +ROC curves feature true positive rate (TPR) on the Y axis, and false positive +rate (FPR) on the X axis. This means that the top left corner of the plot is the +"ideal" point - a FPR of zero, and a TPR of one. + +DET curves are a variation of ROC curves where False Negative Rate (FNR) is +plotted on the y-axis instead of the TPR. In this case the origin (bottom left +corner) is the "ideal" point. .. note:: @@ -46,48 +28,136 @@ :ref:`sphx_glr_auto_examples_classification_plot_classifier_comparison.py` example. + - See :ref:`sphx_glr_auto_examples_model_selection_plot_roc_crossval.py` for + an example estimating the variance of the ROC curves and ROC-AUC. + """ -import matplotlib.pyplot as plt + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Generate synthetic data +# ----------------------- from sklearn.datasets import make_classification -from sklearn.ensemble import RandomForestClassifier -from sklearn.metrics import plot_det_curve -from sklearn.metrics import plot_roc_curve from sklearn.model_selection import train_test_split -from sklearn.pipeline import make_pipeline from sklearn.preprocessing import StandardScaler -from sklearn.svm import LinearSVC -N_SAMPLES = 1000 +X, y = make_classification( + n_samples=1_000, + n_features=2, + n_redundant=0, + n_informative=2, + random_state=1, + n_clusters_per_class=1, +) + +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, random_state=0) + +# %% +# Define the classifiers +# ---------------------- +# +# Here we define two different classifiers. The goal is to visually compare their +# statistical performance across thresholds using the ROC and DET curves. + +from sklearn.dummy import DummyClassifier +from sklearn.ensemble import RandomForestClassifier +from sklearn.pipeline import make_pipeline +from sklearn.svm import LinearSVC classifiers = { "Linear SVM": make_pipeline(StandardScaler(), LinearSVC(C=0.025)), "Random Forest": RandomForestClassifier( - max_depth=5, n_estimators=10, max_features=1 + max_depth=5, n_estimators=10, max_features=1, random_state=0 ), + "Non-informative baseline": DummyClassifier(), } -X, y = make_classification( - n_samples=N_SAMPLES, n_features=2, n_redundant=0, n_informative=2, - random_state=1, n_clusters_per_class=1) +# %% +# Compare ROC and DET curves +# -------------------------- +# +# DET curves are commonly plotted in normal deviate scale. To achieve this the +# DET display transforms the error rates as returned by the +# :func:`~sklearn.metrics.det_curve` and the axis scale using +# `scipy.stats.norm`. -X_train, X_test, y_train, y_test = train_test_split( - X, y, test_size=.4, random_state=0) +import matplotlib.pyplot as plt -# prepare plots -fig, [ax_roc, ax_det] = plt.subplots(1, 2, figsize=(11, 5)) +from sklearn.dummy import DummyClassifier +from sklearn.metrics import DetCurveDisplay, RocCurveDisplay -for name, clf in classifiers.items(): - clf.fit(X_train, y_train) +fig, [ax_roc, ax_det] = plt.subplots(1, 2, figsize=(11, 5)) - plot_roc_curve(clf, X_test, y_test, ax=ax_roc, name=name) - plot_det_curve(clf, X_test, y_test, ax=ax_det, name=name) +ax_roc.set_title("Receiver Operating Characteristic (ROC) curves") +ax_det.set_title("Detection Error Tradeoff (DET) curves") -ax_roc.set_title('Receiver Operating Characteristic (ROC) curves') -ax_det.set_title('Detection Error Tradeoff (DET) curves') +ax_roc.grid(linestyle="--") +ax_det.grid(linestyle="--") -ax_roc.grid(linestyle='--') -ax_det.grid(linestyle='--') +for name, clf in classifiers.items(): + (color, linestyle) = ( + ("black", "--") if name == "Non-informative baseline" else (None, None) + ) + clf.fit(X_train, y_train) + RocCurveDisplay.from_estimator( + clf, + X_test, + y_test, + ax=ax_roc, + name=name, + curve_kwargs=dict(color=color, linestyle=linestyle), + ) + DetCurveDisplay.from_estimator( + clf, X_test, y_test, ax=ax_det, name=name, color=color, linestyle=linestyle + ) plt.legend() plt.show() + +# %% +# Notice that it is easier to visually assess the overall performance of +# different classification algorithms using DET curves than using ROC curves. As +# ROC curves are plot in a linear scale, different classifiers usually appear +# similar for a large part of the plot and differ the most in the top left +# corner of the graph. On the other hand, because DET curves represent straight +# lines in normal deviate scale, they tend to be distinguishable as a whole and +# the area of interest spans a large part of the plot. +# +# DET curves give direct feedback of the detection error tradeoff to aid in +# operating point analysis. The user can then decide the FNR they are willing to +# accept at the expense of the FPR (or vice-versa). +# +# Non-informative classifier baseline for the ROC and DET curves +# -------------------------------------------------------------- +# +# The diagonal black-dotted lines in the plots above correspond to a +# :class:`~sklearn.dummy.DummyClassifier` using the default "prior" strategy, to +# serve as baseline for comparison with other classifiers. This classifier makes +# constant predictions, independent of the input features in `X`, making it a +# non-informative classifier. +# +# To further understand the non-informative baseline of the ROC and DET curves, +# we recall the following mathematical definitions: +# +# :math:`\text{FPR} = \frac{\text{FP}}{\text{FP} + \text{TN}}` +# +# :math:`\text{FNR} = \frac{\text{FN}}{\text{TP} + \text{FN}}` +# +# :math:`\text{TPR} = \frac{\text{TP}}{\text{TP} + \text{FN}}` +# +# A classifier that always predict the positive class would have no true +# negatives nor false negatives, giving :math:`\text{FPR} = \text{TPR} = 1` and +# :math:`\text{FNR} = 0`, i.e.: +# +# - a single point in the upper right corner of the ROC plane, +# - a single point in the lower right corner of the DET plane. +# +# Similarly, a classifier that always predict the negative class would have no +# true positives nor false positives, thus :math:`\text{FPR} = \text{TPR} = 0` +# and :math:`\text{FNR} = 1`, i.e.: +# +# - a single point in the lower left corner of the ROC plane, +# - a single point in the upper left corner of the DET plane. diff --git a/examples/model_selection/plot_grid_search_digits.py b/examples/model_selection/plot_grid_search_digits.py index 498b00082b7c1..f9d7adc2a404b 100644 --- a/examples/model_selection/plot_grid_search_digits.py +++ b/examples/model_selection/plot_grid_search_digits.py @@ -1,6 +1,6 @@ """ ============================================================ -Parameter estimation using grid search with cross-validation +Custom refit strategy of a grid search with cross-validation ============================================================ This examples shows how a classifier is optimized by cross-validation, @@ -13,66 +13,199 @@ More details on tools available for model selection can be found in the sections on :ref:`cross_validation` and :ref:`grid_search`. - """ -from sklearn import datasets -from sklearn.model_selection import train_test_split -from sklearn.model_selection import GridSearchCV -from sklearn.metrics import classification_report -from sklearn.svm import SVC -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# The dataset +# ----------- +# +# We will work with the `digits` dataset. The goal is to classify handwritten +# digits images. +# We transform the problem into a binary classification for easier +# understanding: the goal is to identify whether a digit is `8` or not. +from sklearn import datasets -# Loading the Digits dataset digits = datasets.load_digits() -# To apply an classifier on this data, we need to flatten the image, to -# turn the data in a (samples, feature) matrix: +# %% +# In order to train a classifier on images, we need to flatten them into vectors. +# Each image of 8 by 8 pixels needs to be transformed to a vector of 64 pixels. +# Thus, we will get a final data array of shape `(n_images, n_pixels)`. n_samples = len(digits.images) X = digits.images.reshape((n_samples, -1)) -y = digits.target +y = digits.target == 8 +print( + f"The number of images is {X.shape[0]} and each image contains {X.shape[1]} pixels" +) + +# %% +# As presented in the introduction, the data will be split into a training +# and a testing set of equal size. +from sklearn.model_selection import train_test_split + +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0) -# Split the dataset in two equal parts -X_train, X_test, y_train, y_test = train_test_split( - X, y, test_size=0.5, random_state=0) +# %% +# Define our grid-search strategy +# ------------------------------- +# +# We will select a classifier by searching the best hyper-parameters on folds +# of the training set. To do this, we need to define +# the scores to select the best candidate. -# Set the parameters by cross-validation -tuned_parameters = [{'kernel': ['rbf'], 'gamma': [1e-3, 1e-4], - 'C': [1, 10, 100, 1000]}, - {'kernel': ['linear'], 'C': [1, 10, 100, 1000]}] +scores = ["precision", "recall"] -scores = ['precision', 'recall'] +# %% +# We can also define a function to be passed to the `refit` parameter of the +# :class:`~sklearn.model_selection.GridSearchCV` instance. It will implement the +# custom strategy to select the best candidate from the `cv_results_` attribute +# of the :class:`~sklearn.model_selection.GridSearchCV`. Once the candidate is +# selected, it is automatically refitted by the +# :class:`~sklearn.model_selection.GridSearchCV` instance. +# +# Here, the strategy is to short-list the models which are the best in terms of +# precision and recall. From the selected models, we finally select the fastest +# model at predicting. Notice that these custom choices are completely +# arbitrary. -for score in scores: - print("# Tuning hyper-parameters for %s" % score) +import pandas as pd + + +def print_dataframe(filtered_cv_results): + """Pretty print for filtered dataframe""" + for mean_precision, std_precision, mean_recall, std_recall, params in zip( + filtered_cv_results["mean_test_precision"], + filtered_cv_results["std_test_precision"], + filtered_cv_results["mean_test_recall"], + filtered_cv_results["std_test_recall"], + filtered_cv_results["params"], + ): + print( + f"precision: {mean_precision:0.3f} (Âą{std_precision:0.03f})," + f" recall: {mean_recall:0.3f} (Âą{std_recall:0.03f})," + f" for {params}" + ) print() - clf = GridSearchCV( - SVC(), tuned_parameters, scoring='%s_macro' % score + +def refit_strategy(cv_results): + """Define the strategy to select the best estimator. + + The strategy defined here is to filter-out all results below a precision threshold + of 0.98, rank the remaining by recall and keep all models with one standard + deviation of the best by recall. Once these models are selected, we can select the + fastest model to predict. + + Parameters + ---------- + cv_results : dict of numpy (masked) ndarrays + CV results as returned by the `GridSearchCV`. + + Returns + ------- + best_index : int + The index of the best estimator as it appears in `cv_results`. + """ + # print the info about the grid-search for the different scores + precision_threshold = 0.98 + + cv_results_ = pd.DataFrame(cv_results) + print("All grid-search results:") + print_dataframe(cv_results_) + + # Filter-out all results below the threshold + high_precision_cv_results = cv_results_[ + cv_results_["mean_test_precision"] > precision_threshold + ] + + print(f"Models with a precision higher than {precision_threshold}:") + print_dataframe(high_precision_cv_results) + + high_precision_cv_results = high_precision_cv_results[ + [ + "mean_score_time", + "mean_test_recall", + "std_test_recall", + "mean_test_precision", + "std_test_precision", + "rank_test_recall", + "rank_test_precision", + "params", + ] + ] + + # Select the most performant models in terms of recall + # (within 1 sigma from the best) + best_recall_std = high_precision_cv_results["mean_test_recall"].std() + best_recall = high_precision_cv_results["mean_test_recall"].max() + best_recall_threshold = best_recall - best_recall_std + + high_recall_cv_results = high_precision_cv_results[ + high_precision_cv_results["mean_test_recall"] > best_recall_threshold + ] + print( + "Out of the previously selected high precision models, we keep all the\n" + "the models within one standard deviation of the highest recall model:" ) - clf.fit(X_train, y_train) + print_dataframe(high_recall_cv_results) - print("Best parameters set found on development set:") - print() - print(clf.best_params_) - print() - print("Grid scores on development set:") - print() - means = clf.cv_results_['mean_test_score'] - stds = clf.cv_results_['std_test_score'] - for mean, std, params in zip(means, stds, clf.cv_results_['params']): - print("%0.3f (+/-%0.03f) for %r" - % (mean, std * 2, params)) - print() + # From the best candidates, select the fastest model to predict + fastest_top_recall_high_precision_index = high_recall_cv_results[ + "mean_score_time" + ].idxmin() - print("Detailed classification report:") - print() - print("The model is trained on the full development set.") - print("The scores are computed on the full evaluation set.") - print() - y_true, y_pred = y_test, clf.predict(X_test) - print(classification_report(y_true, y_pred)) - print() + print( + "\nThe selected final model is the fastest to predict out of the previously\n" + "selected subset of best models based on precision and recall.\n" + "Its scoring time is:\n\n" + f"{high_recall_cv_results.loc[fastest_top_recall_high_precision_index]}" + ) + + return fastest_top_recall_high_precision_index + + +# %% +# +# Tuning hyper-parameters +# ----------------------- +# +# Once we defined our strategy to select the best model, we define the values +# of the hyper-parameters and create the grid-search instance: +from sklearn.model_selection import GridSearchCV +from sklearn.svm import SVC + +tuned_parameters = [ + {"kernel": ["rbf"], "gamma": [1e-3, 1e-4], "C": [1, 10, 100, 1000]}, + {"kernel": ["linear"], "C": [1, 10, 100, 1000]}, +] + +grid_search = GridSearchCV( + SVC(), tuned_parameters, scoring=scores, refit=refit_strategy +) +grid_search.fit(X_train, y_train) + +# %% +# +# The parameters selected by the grid-search with our custom strategy are: +grid_search.best_params_ + +# %% +# +# Finally, we evaluate the fine-tuned model on the left-out evaluation set: the +# `grid_search` object **has automatically been refit** on the full training +# set with the parameters selected by our custom refit strategy. +# +# We can use the classification report to compute standard classification +# metrics on the left-out set: +from sklearn.metrics import classification_report + +y_pred = grid_search.predict(X_test) +print(classification_report(y_test, y_pred)) -# Note the problem is too easy: the hyperparameter plateau is too flat and the -# output model is the same for precision and recall with ties in quality. +# %% +# .. note:: +# The problem is too easy: the hyperparameter plateau is too flat and the +# output model is the same for precision and recall with ties in quality. diff --git a/examples/model_selection/plot_grid_search_refit_callable.py b/examples/model_selection/plot_grid_search_refit_callable.py index b2e0cf9d8dcc7..945daf32b41ff 100644 --- a/examples/model_selection/plot_grid_search_refit_callable.py +++ b/examples/model_selection/plot_grid_search_refit_callable.py @@ -3,12 +3,14 @@ Balance model complexity and cross-validated score ================================================== -This example balances model complexity and cross-validated score by -finding a decent accuracy within 1 standard deviation of the best accuracy -score while minimising the number of PCA components [1]. +This example demonstrates how to balance model complexity and cross-validated score by +finding a decent accuracy within 1 standard deviation of the best accuracy score while +minimising the number of :class:`~sklearn.decomposition.PCA` components [1]. It uses +:class:`~sklearn.model_selection.GridSearchCV` with a custom refit callable to select +the optimal model. The figure shows the trade-off between cross-validated score and the number -of PCA components. The balanced case is when n_components=10 and accuracy=0.88, +of PCA components. The balanced case is when `n_components=10` and `accuracy=0.88`, which falls into the range within 1 standard deviation of the best accuracy score. @@ -16,18 +18,39 @@ Selection. The Elements of Statistical Learning (pp. 219-260). New York, NY, USA: Springer New York Inc.. """ -# Author: Wenhao Zhang -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt +import numpy as np +import polars as pl from sklearn.datasets import load_digits from sklearn.decomposition import PCA -from sklearn.model_selection import GridSearchCV +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import GridSearchCV, ShuffleSplit from sklearn.pipeline import Pipeline -from sklearn.svm import LinearSVC + +# %% +# Introduction +# ------------ +# +# When tuning hyperparameters, we often want to balance model complexity and +# performance. The "one-standard-error" rule is a common approach: select the simplest +# model whose performance is within one standard error of the best model's performance. +# This helps to avoid overfitting by preferring simpler models when their performance is +# statistically comparable to more complex ones. + +# %% +# Helper functions +# ---------------- +# +# We define two helper functions: +# 1. `lower_bound`: Calculates the threshold for acceptable performance +# (best score - 1 std) +# 2. `best_low_complexity`: Selects the model with the fewest PCA components that +# exceeds this threshold def lower_bound(cv_results): @@ -46,10 +69,12 @@ def lower_bound(cv_results): Lower bound within 1 standard deviation of the best `mean_test_score`. """ - best_score_idx = np.argmax(cv_results['mean_test_score']) + best_score_idx = np.argmax(cv_results["mean_test_score"]) - return (cv_results['mean_test_score'][best_score_idx] - - cv_results['std_test_score'][best_score_idx]) + return ( + cv_results["mean_test_score"][best_score_idx] + - cv_results["std_test_score"][best_score_idx] + ) def best_low_complexity(cv_results): @@ -69,48 +94,287 @@ def best_low_complexity(cv_results): `mean_test_score`. """ threshold = lower_bound(cv_results) - candidate_idx = np.flatnonzero(cv_results['mean_test_score'] >= threshold) - best_idx = candidate_idx[cv_results['param_reduce_dim__n_components'] - [candidate_idx].argmin()] + candidate_idx = np.flatnonzero(cv_results["mean_test_score"] >= threshold) + best_idx = candidate_idx[ + cv_results["param_reduce_dim__n_components"][candidate_idx].argmin() + ] return best_idx -pipe = Pipeline([ - ('reduce_dim', PCA(random_state=42)), - ('classify', LinearSVC(random_state=42, C=0.01)), -]) +# %% +# Set up the pipeline and parameter grid +# -------------------------------------- +# +# We create a pipeline with two steps: +# 1. Dimensionality reduction using PCA +# 2. Classification using LogisticRegression +# +# We'll search over different numbers of PCA components to find the optimal complexity. + +pipe = Pipeline( + [ + ("reduce_dim", PCA(random_state=42)), + ("classify", LogisticRegression(random_state=42, C=0.01, max_iter=1000)), + ] +) + +param_grid = {"reduce_dim__n_components": [6, 8, 10, 15, 20, 25, 35, 45, 55]} + +# %% +# Perform the search with GridSearchCV +# ------------------------------------ +# +# We use `GridSearchCV` with our custom `best_low_complexity` function as the refit +# parameter. This function will select the model with the fewest PCA components that +# still performs within one standard deviation of the best model. + +grid = GridSearchCV( + pipe, + # Use a non-stratified CV strategy to make sure that the inter-fold + # standard deviation of the test scores is informative. + cv=ShuffleSplit(n_splits=30, random_state=0), + n_jobs=1, # increase this on your machine to use more physical cores + param_grid=param_grid, + scoring="accuracy", + refit=best_low_complexity, + return_train_score=True, +) -param_grid = { - 'reduce_dim__n_components': [6, 8, 10, 12, 14] -} +# %% +# Load the digits dataset and fit the model +# ----------------------------------------- -grid = GridSearchCV(pipe, cv=10, n_jobs=1, param_grid=param_grid, - scoring='accuracy', refit=best_low_complexity) X, y = load_digits(return_X_y=True) grid.fit(X, y) -n_components = grid.cv_results_['param_reduce_dim__n_components'] -test_scores = grid.cv_results_['mean_test_score'] +# %% +# Visualize the results +# --------------------- +# +# We'll create a bar chart showing the test scores for different numbers of PCA +# components, along with horizontal lines indicating the best score and the +# one-standard-deviation threshold. -plt.figure() -plt.bar(n_components, test_scores, width=1.3, color='b') +n_components = grid.cv_results_["param_reduce_dim__n_components"] +test_scores = grid.cv_results_["mean_test_score"] -lower = lower_bound(grid.cv_results_) -plt.axhline(np.max(test_scores), linestyle='--', color='y', - label='Best score') -plt.axhline(lower, linestyle='--', color='.5', label='Best score - 1 std') +# Create a polars DataFrame for better data manipulation and visualization +results_df = pl.DataFrame( + { + "n_components": n_components, + "mean_test_score": test_scores, + "std_test_score": grid.cv_results_["std_test_score"], + "mean_train_score": grid.cv_results_["mean_train_score"], + "std_train_score": grid.cv_results_["std_train_score"], + "mean_fit_time": grid.cv_results_["mean_fit_time"], + "rank_test_score": grid.cv_results_["rank_test_score"], + } +) + +# Sort by number of components +results_df = results_df.sort("n_components") -plt.title("Balance model complexity and cross-validated score") -plt.xlabel('Number of PCA components used') -plt.ylabel('Digit classification accuracy') -plt.xticks(n_components.tolist()) -plt.ylim((0, 1.0)) -plt.legend(loc='upper left') +# Calculate the lower bound threshold +lower = lower_bound(grid.cv_results_) +# Get the best model information best_index_ = grid.best_index_ +best_components = n_components[best_index_] +best_score = grid.cv_results_["mean_test_score"][best_index_] + +# Add a column to mark the selected model +results_df = results_df.with_columns( + pl.when(pl.col("n_components") == best_components) + .then(pl.lit("Selected")) + .otherwise(pl.lit("Regular")) + .alias("model_type") +) + +# Get the number of CV splits from the results +n_splits = sum( + 1 + for key in grid.cv_results_.keys() + if key.startswith("split") and key.endswith("test_score") +) + +# Extract individual scores for each split +test_scores = np.array( + [ + [grid.cv_results_[f"split{i}_test_score"][j] for i in range(n_splits)] + for j in range(len(n_components)) + ] +) +train_scores = np.array( + [ + [grid.cv_results_[f"split{i}_train_score"][j] for i in range(n_splits)] + for j in range(len(n_components)) + ] +) + +# Calculate mean and std of test scores +mean_test_scores = np.mean(test_scores, axis=1) +std_test_scores = np.std(test_scores, axis=1) + +# Find best score and threshold +best_mean_score = np.max(mean_test_scores) +threshold = best_mean_score - std_test_scores[np.argmax(mean_test_scores)] + +# Create a single figure for visualization +fig, ax = plt.subplots(figsize=(12, 8)) + +# Plot individual points +for i, comp in enumerate(n_components): + # Plot individual test points + plt.scatter( + [comp] * n_splits, + test_scores[i], + alpha=0.2, + color="blue", + s=20, + label="Individual test scores" if i == 0 else "", + ) + # Plot individual train points + plt.scatter( + [comp] * n_splits, + train_scores[i], + alpha=0.2, + color="green", + s=20, + label="Individual train scores" if i == 0 else "", + ) + +# Plot mean lines with error bands +plt.plot( + n_components, + np.mean(test_scores, axis=1), + "-", + color="blue", + linewidth=2, + label="Mean test score", +) +plt.fill_between( + n_components, + np.mean(test_scores, axis=1) - np.std(test_scores, axis=1), + np.mean(test_scores, axis=1) + np.std(test_scores, axis=1), + alpha=0.15, + color="blue", +) + +plt.plot( + n_components, + np.mean(train_scores, axis=1), + "-", + color="green", + linewidth=2, + label="Mean train score", +) +plt.fill_between( + n_components, + np.mean(train_scores, axis=1) - np.std(train_scores, axis=1), + np.mean(train_scores, axis=1) + np.std(train_scores, axis=1), + alpha=0.15, + color="green", +) + +# Add threshold lines +plt.axhline( + best_mean_score, + color="#9b59b6", # Purple + linestyle="--", + label="Best score", + linewidth=2, +) +plt.axhline( + threshold, + color="#e67e22", # Orange + linestyle="--", + label="Best score - 1 std", + linewidth=2, +) + +# Highlight selected model +plt.axvline( + best_components, + color="#9b59b6", # Purple + alpha=0.2, + linewidth=8, + label="Selected model", +) + +# Set titles and labels +plt.xlabel("Number of PCA components", fontsize=12) +plt.ylabel("Score", fontsize=12) +plt.title("Model Selection: Balancing Complexity and Performance", fontsize=14) +plt.grid(True, linestyle="--", alpha=0.7) +plt.legend( + bbox_to_anchor=(1.02, 1), + loc="upper left", + borderaxespad=0, +) + +# Set axis properties +plt.xticks(n_components) +plt.ylim((0.85, 1.0)) + +# # Adjust layout +plt.tight_layout() + +# %% +# Print the results +# ----------------- +# +# We print information about the selected model, including its complexity and +# performance. We also show a summary table of all models using polars. + +print("Best model selected by the one-standard-error rule:") +print(f"Number of PCA components: {best_components}") +print(f"Accuracy score: {best_score:.4f}") +print(f"Best possible accuracy: {np.max(test_scores):.4f}") +print(f"Accuracy threshold (best - 1 std): {lower:.4f}") + +# Create a summary table with polars +summary_df = results_df.select( + pl.col("n_components"), + pl.col("mean_test_score").round(4).alias("test_score"), + pl.col("std_test_score").round(4).alias("test_std"), + pl.col("mean_train_score").round(4).alias("train_score"), + pl.col("std_train_score").round(4).alias("train_std"), + pl.col("mean_fit_time").round(3).alias("fit_time"), + pl.col("rank_test_score").alias("rank"), +) + +# Add a column to mark the selected model +summary_df = summary_df.with_columns( + pl.when(pl.col("n_components") == best_components) + .then(pl.lit("*")) + .otherwise(pl.lit("")) + .alias("selected") +) + +print("\nModel comparison table:") +print(summary_df) + +# %% +# Conclusion +# ---------- +# +# The one-standard-error rule helps us select a simpler model (fewer PCA components) +# while maintaining performance statistically comparable to the best model. +# This approach can help prevent overfitting and improve model interpretability +# and efficiency. +# +# In this example, we've seen how to implement this rule using a custom refit +# callable with :class:`~sklearn.model_selection.GridSearchCV`. +# +# Key takeaways: +# 1. The one-standard-error rule provides a good rule of thumb to select simpler models +# 2. Custom refit callables in :class:`~sklearn.model_selection.GridSearchCV` allow for +# flexible model selection strategies +# 3. Visualizing both train and test scores helps identify potential overfitting +# +# This approach can be applied to other model selection scenarios where balancing +# complexity and performance is important, or in cases where a use-case specific +# selection of the "best" model is desired. -print("The best_index_ is %d" % best_index_) -print("The n_components selected is %d" % n_components[best_index_]) -print("The corresponding accuracy score is %.2f" - % grid.cv_results_['mean_test_score'][best_index_]) +# Display the figure plt.show() diff --git a/examples/model_selection/plot_grid_search_stats.py b/examples/model_selection/plot_grid_search_stats.py index 23fe009c6aa50..2fa0daa008ee9 100644 --- a/examples/model_selection/plot_grid_search_stats.py +++ b/examples/model_selection/plot_grid_search_stats.py @@ -5,24 +5,27 @@ This example illustrates how to statistically compare the performance of models trained and evaluated using :class:`~sklearn.model_selection.GridSearchCV`. + """ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + # %% # We will start by simulating moon shaped data (where the ideal separation # between classes is non-linear), adding to it a moderate degree of noise. # Datapoints will belong to one of two possible classes to be predicted by two # features. We will simulate 50 samples for each class: -print(__doc__) import matplotlib.pyplot as plt import seaborn as sns + from sklearn.datasets import make_moons X, y = make_moons(noise=0.352, random_state=1, n_samples=100) sns.scatterplot( - x=X[:, 0], y=X[:, 1], hue=y, - marker='o', s=25, edgecolor='k', legend=False + x=X[:, 0], y=X[:, 1], hue=y, marker="o", s=25, edgecolor="k", legend=False ).set_title("Data") plt.show() @@ -40,21 +43,16 @@ from sklearn.svm import SVC param_grid = [ - {'kernel': ['linear']}, - {'kernel': ['poly'], 'degree': [2, 3]}, - {'kernel': ['rbf']} + {"kernel": ["linear"]}, + {"kernel": ["poly"], "degree": [2, 3]}, + {"kernel": ["rbf"]}, ] svc = SVC(random_state=0) -cv = RepeatedStratifiedKFold( - n_splits=10, n_repeats=10, random_state=0 -) +cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=10, random_state=0) -search = GridSearchCV( - estimator=svc, param_grid=param_grid, - scoring='roc_auc', cv=cv -) +search = GridSearchCV(estimator=svc, param_grid=param_grid, scoring="roc_auc", cv=cv) search.fit(X, y) # %% @@ -64,23 +62,17 @@ import pandas as pd results_df = pd.DataFrame(search.cv_results_) -results_df = results_df.sort_values(by=['rank_test_score']) -results_df = ( - results_df - .set_index(results_df["params"].apply( - lambda x: "_".join(str(val) for val in x.values())) - ) - .rename_axis('kernel') -) -results_df[ - ['params', 'rank_test_score', 'mean_test_score', 'std_test_score'] -] +results_df = results_df.sort_values(by=["rank_test_score"]) +results_df = results_df.set_index( + results_df["params"].apply(lambda x: "_".join(str(val) for val in x.values())) +).rename_axis("kernel") +results_df[["params", "rank_test_score", "mean_test_score", "std_test_score"]] # %% # We can see that the estimator using the `'rbf'` kernel performed best, # closely followed by `'linear'`. Both estimators with a `'poly'` kernel # performed worse, with the one using a two-degree polynomial achieving a much -# lower perfomance than all other models. +# lower performance than all other models. # # Usually, the analysis just ends here, but half the story is missing. The # output of :class:`~sklearn.model_selection.GridSearchCV` does not provide @@ -101,14 +93,18 @@ # Let's inspect this partition effect by plotting the performance of all models # in each fold, and calculating the correlation between models across folds: -# create df of model scores ordered by perfomance -model_scores = results_df.filter(regex=r'split\d*_test_score') +# create df of model scores ordered by performance +model_scores = results_df.filter(regex=r"split\d*_test_score") # plot 30 examples of dependency between cv fold and AUC scores fig, ax = plt.subplots() sns.lineplot( data=model_scores.transpose().iloc[:30], - dashes=False, palette='Set1', marker='o', alpha=.5, ax=ax + dashes=False, + palette="Set1", + marker="o", + alpha=0.5, + ax=ax, ) ax.set_xlabel("CV test fold", size=12, labelpad=10) ax.set_ylabel("Model AUC", size=12) @@ -146,7 +142,7 @@ # described in the previous section. We will use the one proven to obtain the # highest replicability scores (which rate how similar the performance of a # model is when evaluating it on different random partitions of the same -# dataset) while mantaining a low rate of false postitives and false negatives: +# dataset) while maintaining a low rate of false positives and false negatives: # the Nadeau and Bengio's corrected t-test [2]_ that uses a 10 times repeated # 10-fold cross validation [3]_. # @@ -178,7 +174,7 @@ def corrected_std(differences, n_train, n_test): Parameters ---------- - differences : ndarray of shape (n_samples, 1) + differences : ndarray of shape (n_samples,) Vector containing the differences in the score metrics of two models. n_train : int Number of samples in the training set. @@ -187,13 +183,13 @@ def corrected_std(differences, n_train, n_test): Returns ------- - corrected_std : int + corrected_std : float Variance-corrected standard deviation of the set of differences. """ - n = n_train + n_test - corrected_var = ( - np.var(differences, ddof=1) * ((1 / n) + (n_test / n_train)) - ) + # kr = k times r, r times repeated k-fold crossvalidation, + # kr equals the number of times the model was evaluated + kr = len(differences) + corrected_var = np.var(differences, ddof=1) * (1 / kr + n_test / n_train) corrected_std = np.sqrt(corrected_var) return corrected_std @@ -203,7 +199,7 @@ def compute_corrected_ttest(differences, df, n_train, n_test): Parameters ---------- - differences : array-like of shape (n_samples, 1) + differences : array-like of shape (n_samples,) Vector containing the differences in the score metrics of two models. df : int Degrees of freedom. @@ -234,23 +230,22 @@ def compute_corrected_ttest(differences, df, n_train, n_test): n = differences.shape[0] # number of test sets df = n - 1 -n_train = len(list(cv.split(X, y))[0][0]) -n_test = len(list(cv.split(X, y))[0][1]) +n_train = len(next(iter(cv.split(X, y)))[0]) +n_test = len(next(iter(cv.split(X, y)))[1]) t_stat, p_val = compute_corrected_ttest(differences, df, n_train, n_test) -print(f"Corrected t-value: {t_stat:.3f}\n" - f"Corrected p-value: {p_val:.3f}") +print(f"Corrected t-value: {t_stat:.3f}\nCorrected p-value: {p_val:.3f}") # %% # We can compare the corrected t- and p-values with the uncorrected ones: -t_stat_uncorrected = ( - np.mean(differences) / np.sqrt(np.var(differences, ddof=1) / n) -) +t_stat_uncorrected = np.mean(differences) / np.sqrt(np.var(differences, ddof=1) / n) p_val_uncorrected = t.sf(np.abs(t_stat_uncorrected), df) -print(f"Uncorrected t-value: {t_stat_uncorrected:.3f}\n" - f"Uncorrected p-value: {p_val_uncorrected:.3f}") +print( + f"Uncorrected t-value: {t_stat_uncorrected:.3f}\n" + f"Uncorrected p-value: {p_val_uncorrected:.3f}" +) # %% # Using the conventional significance alpha level at `p=0.05`, we observe that @@ -279,10 +274,10 @@ def compute_corrected_ttest(differences, df, n_train, n_test): # # Bayesian estimation can be carried out in many forms to answer our question, # but in this example we will implement the approach suggested by Benavoli and -# collegues [4]_. +# colleagues [4]_. # # One way of defining our posterior using a closed-form expression is to select -# a prior conjugate to the likelihood function. Benavoli and collegues [4]_ +# a prior conjugate to the likelihood function. Benavoli and colleagues [4]_ # show that when comparing the performance of two classifiers we can model the # prior as a Normal-Gamma distribution (with both mean and variance unknown) # conjugate to a normal likelihood, to thus express the posterior as a normal @@ -306,10 +301,9 @@ def compute_corrected_ttest(differences, df, n_train, n_test): # # Let's compute and plot the posterior: -# intitialize random variable +# initialize random variable t_post = t( - df, loc=np.mean(differences), - scale=corrected_std(differences, n_train, n_test) + df, loc=np.mean(differences), scale=corrected_std(differences, n_train, n_test) ) # %% @@ -319,7 +313,7 @@ def compute_corrected_ttest(differences, df, n_train, n_test): plt.plot(x, t_post.pdf(x)) plt.xticks(np.arange(-0.04, 0.06, 0.01)) -plt.fill_between(x, t_post.pdf(x), 0, facecolor='blue', alpha=.2) +plt.fill_between(x, t_post.pdf(x), 0, facecolor="blue", alpha=0.2) plt.ylabel("Probability density") plt.xlabel(r"Mean difference ($\mu$)") plt.title("Posterior distribution") @@ -327,17 +321,21 @@ def compute_corrected_ttest(differences, df, n_train, n_test): # %% # We can calculate the probability that the first model is better than the -# second by computing the area under the curve of the posterior distirbution +# second by computing the area under the curve of the posterior distribution # from zero to infinity. And also the reverse: we can calculate the probability # that the second model is better than the first by computing the area under # the curve from minus infinity to zero. better_prob = 1 - t_post.cdf(0) -print(f"Probability of {model_scores.index[0]} being more accurate than " - f"{model_scores.index[1]}: {better_prob:.3f}") -print(f"Probability of {model_scores.index[1]} being more accurate than " - f"{model_scores.index[0]}: {1 - better_prob:.3f}") +print( + f"Probability of {model_scores.index[0]} being more accurate than " + f"{model_scores.index[1]}: {better_prob:.3f}" +) +print( + f"Probability of {model_scores.index[1]} being more accurate than " + f"{model_scores.index[0]}: {1 - better_prob:.3f}" +) # %% # In contrast with the frequentist approach, we can compute the probability @@ -371,8 +369,10 @@ def compute_corrected_ttest(differences, df, n_train, n_test): rope_interval = [-0.01, 0.01] rope_prob = t_post.cdf(rope_interval[1]) - t_post.cdf(rope_interval[0]) -print(f"Probability of {model_scores.index[0]} and {model_scores.index[1]} " - f"being practically equivalent: {rope_prob:.3f}") +print( + f"Probability of {model_scores.index[0]} and {model_scores.index[1]} " + f"being practically equivalent: {rope_prob:.3f}" +) # %% # We can plot how the posterior is distributed over the ROPE interval: @@ -382,7 +382,7 @@ def compute_corrected_ttest(differences, df, n_train, n_test): plt.plot(x, t_post.pdf(x)) plt.xticks(np.arange(-0.04, 0.06, 0.01)) plt.vlines([-0.01, 0.01], ymin=0, ymax=(np.max(t_post.pdf(x)) + 1)) -plt.fill_between(x_rope, t_post.pdf(x_rope), 0, facecolor='blue', alpha=.2) +plt.fill_between(x_rope, t_post.pdf(x_rope), 0, facecolor="blue", alpha=0.2) plt.ylabel("Probability density") plt.xlabel(r"Mean difference ($\mu$)") plt.title("Posterior distribution under the ROPE") @@ -414,16 +414,15 @@ def compute_corrected_ttest(differences, df, n_train, n_test): cred_intervals.append([interval, cred_interval[0], cred_interval[1]]) cred_int_df = pd.DataFrame( - cred_intervals, - columns=['interval', 'lower value', 'upper value'] -).set_index('interval') + cred_intervals, columns=["interval", "lower value", "upper value"] +).set_index("interval") cred_int_df # %% # As shown in the table, there is a 50% probability that the true mean # difference between models will be between 0.000977 and 0.019023, 70% # probability that it will be between -0.005422 and 0.025422, and 95% -# probability that it will be between -0.016445 and 0.036445. +# probability that it will be between -0.016445 and 0.036445. # %% # Pairwise comparison of all models: frequentist approach @@ -446,9 +445,8 @@ def compute_corrected_ttest(differences, df, n_train, n_test): from itertools import combinations from math import factorial -n_comparisons = ( - factorial(len(model_scores)) - / (factorial(2) * factorial(len(model_scores) - 2)) +n_comparisons = factorial(len(model_scores)) / ( + factorial(2) * factorial(len(model_scores) - 2) ) pairwise_t_test = [] @@ -456,20 +454,16 @@ def compute_corrected_ttest(differences, df, n_train, n_test): model_i_scores = model_scores.iloc[model_i].values model_k_scores = model_scores.iloc[model_k].values differences = model_i_scores - model_k_scores - t_stat, p_val = compute_corrected_ttest( - differences, df, n_train, n_test - ) + t_stat, p_val = compute_corrected_ttest(differences, df, n_train, n_test) p_val *= n_comparisons # implement Bonferroni correction # Bonferroni can output p-values higher than 1 p_val = 1 if p_val > 1 else p_val pairwise_t_test.append( - [model_scores.index[model_i], model_scores.index[model_k], - t_stat, p_val] + [model_scores.index[model_i], model_scores.index[model_k], t_stat, p_val] ) pairwise_comp_df = pd.DataFrame( - pairwise_t_test, - columns=['model_1', 'model_2', 't_stat', 'p_val'] + pairwise_t_test, columns=["model_1", "model_2", "t_stat", "p_val"] ).round(3) pairwise_comp_df @@ -497,8 +491,7 @@ def compute_corrected_ttest(differences, df, n_train, n_test): model_k_scores = model_scores.iloc[model_k].values differences = model_i_scores - model_k_scores t_post = t( - df, loc=np.mean(differences), - scale=corrected_std(differences, n_train, n_test) + df, loc=np.mean(differences), scale=corrected_std(differences, n_train, n_test) ) worse_prob = t_post.cdf(rope_interval[0]) better_prob = 1 - t_post.cdf(rope_interval[1]) @@ -506,10 +499,9 @@ def compute_corrected_ttest(differences, df, n_train, n_test): pairwise_bayesian.append([worse_prob, better_prob, rope_prob]) -pairwise_bayesian_df = (pd.DataFrame( - pairwise_bayesian, - columns=['worse_prob', 'better_prob', 'rope_prob'] -).round(3)) +pairwise_bayesian_df = pd.DataFrame( + pairwise_bayesian, columns=["worse_prob", "better_prob", "rope_prob"] +).round(3) pairwise_comp_df = pairwise_comp_df.join(pairwise_bayesian_df) pairwise_comp_df @@ -551,27 +543,27 @@ def compute_corrected_ttest(differences, df, n_train, n_test): # correction is needed when using the frequentist approach. # %% -# .. topic:: References -# -# .. [1] Dietterich, T. G. (1998). `Approximate statistical tests for -# comparing supervised classification learning algorithms -# `_. -# Neural computation, 10(7). -# .. [2] Nadeau, C., & Bengio, Y. (2000). `Inference for the generalization -# error -# `_. -# In Advances in neural information processing systems. -# .. [3] Bouckaert, R. R., & Frank, E. (2004). `Evaluating the replicability -# of significance tests for comparing learning algorithms -# `_. -# In Pacific-Asia Conference on Knowledge Discovery and Data Mining. -# .. [4] Benavoli, A., Corani, G., DemÅĄar, J., & Zaffalon, M. (2017). `Time -# for a change: a tutorial for comparing multiple classifiers through -# Bayesian analysis -# `_. -# The Journal of Machine Learning Research, 18(1). See the Python -# library that accompanies this paper `here -# `_. -# .. [5] Diebold, F.X. & Mariano R.S. (1995). `Comparing predictive accuracy -# `_ -# Journal of Business & economic statistics, 20(1), 134-144. +# .. rubric:: References +# +# .. [1] Dietterich, T. G. (1998). `Approximate statistical tests for +# comparing supervised classification learning algorithms +# `_. +# Neural computation, 10(7). +# .. [2] Nadeau, C., & Bengio, Y. (2000). `Inference for the generalization +# error +# `_. +# In Advances in neural information processing systems. +# .. [3] Bouckaert, R. R., & Frank, E. (2004). `Evaluating the replicability +# of significance tests for comparing learning algorithms +# `_. +# In Pacific-Asia Conference on Knowledge Discovery and Data Mining. +# .. [4] Benavoli, A., Corani, G., DemÅĄar, J., & Zaffalon, M. (2017). `Time +# for a change: a tutorial for comparing multiple classifiers through +# Bayesian analysis +# `_. +# The Journal of Machine Learning Research, 18(1). See the Python +# library that accompanies this paper `here +# `_. +# .. [5] Diebold, F.X. & Mariano R.S. (1995). `Comparing predictive accuracy +# `_ +# Journal of Business & economic statistics, 20(1), 134-144. diff --git a/examples/model_selection/plot_grid_search_text_feature_extraction.py b/examples/model_selection/plot_grid_search_text_feature_extraction.py new file mode 100644 index 0000000000000..8eb7b2f424896 --- /dev/null +++ b/examples/model_selection/plot_grid_search_text_feature_extraction.py @@ -0,0 +1,263 @@ +""" +========================================================== +Sample pipeline for text feature extraction and evaluation +========================================================== + +The dataset used in this example is :ref:`20newsgroups_dataset` which will be +automatically downloaded, cached and reused for the document classification +example. + +In this example, we tune the hyperparameters of a particular classifier using a +:class:`~sklearn.model_selection.RandomizedSearchCV`. For a demo on the +performance of some other classifiers, see the +:ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py` +notebook. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Data loading +# ------------ +# We load two categories from the training set. You can adjust the number of +# categories by adding their names to the list or setting `categories=None` when +# calling the dataset loader :func:`~sklearn.datasets.fetch_20newsgroups` to get +# the 20 of them. + +from sklearn.datasets import fetch_20newsgroups + +categories = [ + "alt.atheism", + "talk.religion.misc", +] + +data_train = fetch_20newsgroups( + subset="train", + categories=categories, + shuffle=True, + random_state=42, + remove=("headers", "footers", "quotes"), +) + +data_test = fetch_20newsgroups( + subset="test", + categories=categories, + shuffle=True, + random_state=42, + remove=("headers", "footers", "quotes"), +) + +print(f"Loading 20 newsgroups dataset for {len(data_train.target_names)} categories:") +print(data_train.target_names) +print(f"{len(data_train.data)} documents") + +# %% +# Pipeline with hyperparameter tuning +# ----------------------------------- +# +# We define a pipeline combining a text feature vectorizer with a simple +# classifier yet effective for text classification. + +from sklearn.feature_extraction.text import TfidfVectorizer +from sklearn.naive_bayes import ComplementNB +from sklearn.pipeline import Pipeline + +pipeline = Pipeline( + [ + ("vect", TfidfVectorizer()), + ("clf", ComplementNB()), + ] +) +pipeline + +# %% +# We define a grid of hyperparameters to be explored by the +# :class:`~sklearn.model_selection.RandomizedSearchCV`. Using a +# :class:`~sklearn.model_selection.GridSearchCV` instead would explore all the +# possible combinations on the grid, which can be costly to compute, whereas the +# parameter `n_iter` of the :class:`~sklearn.model_selection.RandomizedSearchCV` +# controls the number of different random combination that are evaluated. Notice +# that setting `n_iter` larger than the number of possible combinations in a +# grid would lead to repeating already-explored combinations. We search for the +# best parameter combination for both the feature extraction (`vect__`) and the +# classifier (`clf__`). + +import numpy as np + +parameter_grid = { + "vect__max_df": (0.2, 0.4, 0.6, 0.8, 1.0), + "vect__min_df": (1, 3, 5, 10), + "vect__ngram_range": ((1, 1), (1, 2)), # unigrams or bigrams + "vect__norm": ("l1", "l2"), + "clf__alpha": np.logspace(-6, 6, 13), +} + +# %% +# In this case `n_iter=40` is not an exhaustive search of the hyperparameters' +# grid. In practice it would be interesting to increase the parameter `n_iter` +# to get a more informative analysis. As a consequence, the computional time +# increases. We can reduce it by taking advantage of the parallelisation over +# the parameter combinations evaluation by increasing the number of CPUs used +# via the parameter `n_jobs`. + +from pprint import pprint + +from sklearn.model_selection import RandomizedSearchCV + +random_search = RandomizedSearchCV( + estimator=pipeline, + param_distributions=parameter_grid, + n_iter=40, + random_state=0, + n_jobs=2, + verbose=1, +) + +print("Performing grid search...") +print("Hyperparameters to be evaluated:") +pprint(parameter_grid) + +# %% +from time import time + +t0 = time() +random_search.fit(data_train.data, data_train.target) +print(f"Done in {time() - t0:.3f}s") + +# %% +print("Best parameters combination found:") +best_parameters = random_search.best_estimator_.get_params() +for param_name in sorted(parameter_grid.keys()): + print(f"{param_name}: {best_parameters[param_name]}") + +# %% +test_accuracy = random_search.score(data_test.data, data_test.target) +print( + "Accuracy of the best parameters using the inner CV of " + f"the random search: {random_search.best_score_:.3f}" +) +print(f"Accuracy on test set: {test_accuracy:.3f}") + +# %% +# The prefixes `vect` and `clf` are required to avoid possible ambiguities in +# the pipeline, but are not necessary for visualizing the results. Because of +# this, we define a function that will rename the tuned hyperparameters and +# improve the readability. + +import pandas as pd + + +def shorten_param(param_name): + """Remove components' prefixes in param_name.""" + if "__" in param_name: + return param_name.rsplit("__", 1)[1] + return param_name + + +cv_results = pd.DataFrame(random_search.cv_results_) +cv_results = cv_results.rename(shorten_param, axis=1) + +# %% +# We can use a `plotly.express.scatter +# `_ +# to visualize the trade-off between scoring time and mean test score (i.e. "CV +# score"). Passing the cursor over a given point displays the corresponding +# parameters. Error bars correspond to one standard deviation as computed in the +# different folds of the cross-validation. + +import plotly.express as px + +param_names = [shorten_param(name) for name in parameter_grid.keys()] +labels = { + "mean_score_time": "CV Score time (s)", + "mean_test_score": "CV score (accuracy)", +} +fig = px.scatter( + cv_results, + x="mean_score_time", + y="mean_test_score", + error_x="std_score_time", + error_y="std_test_score", + hover_data=param_names, + labels=labels, +) +fig.update_layout( + title={ + "text": "trade-off between scoring time and mean test score", + "y": 0.95, + "x": 0.5, + "xanchor": "center", + "yanchor": "top", + } +) +fig + +# %% +# Notice that the cluster of models in the upper-left corner of the plot have +# the best trade-off between accuracy and scoring time. In this case, using +# bigrams increases the required scoring time without improving considerably the +# accuracy of the pipeline. +# +# .. note:: For more information on how to customize an automated tuning to +# maximize score and minimize scoring time, see the example notebook +# :ref:`sphx_glr_auto_examples_model_selection_plot_grid_search_digits.py`. +# +# We can also use a `plotly.express.parallel_coordinates +# `_ +# to further visualize the mean test score as a function of the tuned +# hyperparameters. This helps finding interactions between more than two +# hyperparameters and provide intuition on their relevance for improving the +# performance of a pipeline. +# +# We apply a `math.log10` transformation on the `alpha` axis to spread the +# active range and improve the readability of the plot. A value :math:`x` on +# said axis is to be understood as :math:`10^x`. + +import math + +column_results = param_names + ["mean_test_score", "mean_score_time"] + +transform_funcs = dict.fromkeys(column_results, lambda x: x) +# Using a logarithmic scale for alpha +transform_funcs["alpha"] = math.log10 +# L1 norms are mapped to index 1, and L2 norms to index 2 +transform_funcs["norm"] = lambda x: 2 if x == "l2" else 1 +# Unigrams are mapped to index 1 and bigrams to index 2 +transform_funcs["ngram_range"] = lambda x: x[1] + +fig = px.parallel_coordinates( + cv_results[column_results].apply(transform_funcs), + color="mean_test_score", + color_continuous_scale=px.colors.sequential.Viridis_r, + labels=labels, +) +fig.update_layout( + title={ + "text": "Parallel coordinates plot of text classifier pipeline", + "y": 0.99, + "x": 0.5, + "xanchor": "center", + "yanchor": "top", + } +) +fig + +# %% +# The parallel coordinates plot displays the values of the hyperparameters on +# different columns while the performance metric is color coded. It is possible +# to select a range of results by clicking and holding on any axis of the +# parallel coordinate plot. You can then slide (move) the range selection and +# cross two selections to see the intersections. You can undo a selection by +# clicking once again on the same axis. +# +# In particular for this hyperparameter search, it is interesting to notice that +# the top performing models do not seem to depend on the regularization `norm`, +# but they do depend on a trade-off between `max_df`, `min_df` and the +# regularization strength `alpha`. The reason is that including noisy features +# (i.e. `max_df` close to :math:`1.0` or `min_df` close to :math:`0`) tend to +# overfit and therefore require a stronger regularization to compensate. Having +# less features require less regularization and less scoring time. +# +# The best accuracy scores are obtained when `alpha` is between :math:`10^{-6}` +# and :math:`10^0`, regardless of the hyperparameter `norm`. diff --git a/examples/model_selection/plot_learning_curve.py b/examples/model_selection/plot_learning_curve.py index 71cc565c3528c..d8060c67cbe15 100644 --- a/examples/model_selection/plot_learning_curve.py +++ b/examples/model_selection/plot_learning_curve.py @@ -1,165 +1,187 @@ """ -======================== -Plotting Learning Curves -======================== -In the first column, first row the learning curve of a naive Bayes classifier -is shown for the digits dataset. Note that the training score and the -cross-validation score are both not very good at the end. However, the shape -of the curve can be found in more complex datasets very often: the training -score is very high at the beginning and decreases and the cross-validation -score is very low at the beginning and increases. In the second column, first -row we see the learning curve of an SVM with RBF kernel. We can see clearly -that the training score is still around the maximum and the validation score -could be increased with more training samples. The plots in the second row -show the times required by the models to train with various sizes of training -dataset. The plots in the third row show how much time was required to train -the models for each training sizes. +========================================================= +Plotting Learning Curves and Checking Models' Scalability +========================================================= + +In this example, we show how to use the class +:class:`~sklearn.model_selection.LearningCurveDisplay` to easily plot learning +curves. In addition, we give an interpretation to the learning curves obtained +for a naive Bayes and SVM classifiers. + +Then, we explore and draw some conclusions about the scalability of these predictive +models by looking at their computational cost and not only at their statistical +accuracy. """ -print(__doc__) -import numpy as np -import matplotlib.pyplot as plt +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Learning Curve +# ============== +# +# Learning curves show the effect of adding more samples during the training +# process. The effect is depicted by checking the statistical performance of +# the model in terms of training score and testing score. +# +# Here, we compute the learning curve of a naive Bayes classifier and a SVM +# classifier with a RBF kernel using the digits dataset. +from sklearn.datasets import load_digits from sklearn.naive_bayes import GaussianNB from sklearn.svm import SVC -from sklearn.datasets import load_digits -from sklearn.model_selection import learning_curve -from sklearn.model_selection import ShuffleSplit - - -def plot_learning_curve(estimator, title, X, y, axes=None, ylim=None, cv=None, - n_jobs=None, train_sizes=np.linspace(.1, 1.0, 5)): - """ - Generate 3 plots: the test and training learning curve, the training - samples vs fit times curve, the fit times vs score curve. - - Parameters - ---------- - estimator : estimator instance - An estimator instance implementing `fit` and `predict` methods which - will be cloned for each validation. - - title : str - Title for the chart. - - X : array-like of shape (n_samples, n_features) - Training vector, where ``n_samples`` is the number of samples and - ``n_features`` is the number of features. - - y : array-like of shape (n_samples) or (n_samples, n_features) - Target relative to ``X`` for classification or regression; - None for unsupervised learning. - - axes : array-like of shape (3,), default=None - Axes to use for plotting the curves. - - ylim : tuple of shape (2,), default=None - Defines minimum and maximum y-values plotted, e.g. (ymin, ymax). - - cv : int, cross-validation generator or an iterable, default=None - Determines the cross-validation splitting strategy. - Possible inputs for cv are: - - - None, to use the default 5-fold cross-validation, - - integer, to specify the number of folds. - - :term:`CV splitter`, - - An iterable yielding (train, test) splits as arrays of indices. - - For integer/None inputs, if ``y`` is binary or multiclass, - :class:`StratifiedKFold` used. If the estimator is not a classifier - or if ``y`` is neither binary nor multiclass, :class:`KFold` is used. - - Refer :ref:`User Guide ` for the various - cross-validators that can be used here. - - n_jobs : int or None, default=None - Number of jobs to run in parallel. - ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. - ``-1`` means using all processors. See :term:`Glossary ` - for more details. - - train_sizes : array-like of shape (n_ticks,) - Relative or absolute numbers of training examples that will be used to - generate the learning curve. If the ``dtype`` is float, it is regarded - as a fraction of the maximum size of the training set (that is - determined by the selected validation method), i.e. it has to be within - (0, 1]. Otherwise it is interpreted as absolute sizes of the training - sets. Note that for classification the number of samples usually have - to be big enough to contain at least one sample from each class. - (default: np.linspace(0.1, 1.0, 5)) - """ - if axes is None: - _, axes = plt.subplots(1, 3, figsize=(20, 5)) - - axes[0].set_title(title) - if ylim is not None: - axes[0].set_ylim(*ylim) - axes[0].set_xlabel("Training examples") - axes[0].set_ylabel("Score") - - train_sizes, train_scores, test_scores, fit_times, _ = \ - learning_curve(estimator, X, y, cv=cv, n_jobs=n_jobs, - train_sizes=train_sizes, - return_times=True) - train_scores_mean = np.mean(train_scores, axis=1) - train_scores_std = np.std(train_scores, axis=1) - test_scores_mean = np.mean(test_scores, axis=1) - test_scores_std = np.std(test_scores, axis=1) - fit_times_mean = np.mean(fit_times, axis=1) - fit_times_std = np.std(fit_times, axis=1) - - # Plot learning curve - axes[0].grid() - axes[0].fill_between(train_sizes, train_scores_mean - train_scores_std, - train_scores_mean + train_scores_std, alpha=0.1, - color="r") - axes[0].fill_between(train_sizes, test_scores_mean - test_scores_std, - test_scores_mean + test_scores_std, alpha=0.1, - color="g") - axes[0].plot(train_sizes, train_scores_mean, 'o-', color="r", - label="Training score") - axes[0].plot(train_sizes, test_scores_mean, 'o-', color="g", - label="Cross-validation score") - axes[0].legend(loc="best") - - # Plot n_samples vs fit_times - axes[1].grid() - axes[1].plot(train_sizes, fit_times_mean, 'o-') - axes[1].fill_between(train_sizes, fit_times_mean - fit_times_std, - fit_times_mean + fit_times_std, alpha=0.1) - axes[1].set_xlabel("Training examples") - axes[1].set_ylabel("fit_times") - axes[1].set_title("Scalability of the model") - - # Plot fit_time vs score - axes[2].grid() - axes[2].plot(fit_times_mean, test_scores_mean, 'o-') - axes[2].fill_between(fit_times_mean, test_scores_mean - test_scores_std, - test_scores_mean + test_scores_std, alpha=0.1) - axes[2].set_xlabel("fit_times") - axes[2].set_ylabel("Score") - axes[2].set_title("Performance of the model") - - return plt - - -fig, axes = plt.subplots(3, 2, figsize=(10, 15)) X, y = load_digits(return_X_y=True) +naive_bayes = GaussianNB() +svc = SVC(kernel="rbf", gamma=0.001) + +# %% +# The :meth:`~sklearn.model_selection.LearningCurveDisplay.from_estimator` +# displays the learning curve given the dataset and the predictive model to +# analyze. To get an estimate of the scores uncertainty, this method uses +# a cross-validation procedure. +import matplotlib.pyplot as plt +import numpy as np -title = "Learning Curves (Naive Bayes)" -# Cross validation with 100 iterations to get smoother mean test and train -# score curves, each time with 20% data randomly selected as a validation set. -cv = ShuffleSplit(n_splits=100, test_size=0.2, random_state=0) - -estimator = GaussianNB() -plot_learning_curve(estimator, title, X, y, axes=axes[:, 0], ylim=(0.7, 1.01), - cv=cv, n_jobs=4) +from sklearn.model_selection import LearningCurveDisplay, ShuffleSplit + +fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(10, 6), sharey=True) + +common_params = { + "X": X, + "y": y, + "train_sizes": np.linspace(0.1, 1.0, 5), + "cv": ShuffleSplit(n_splits=50, test_size=0.2, random_state=0), + "score_type": "both", + "n_jobs": 4, + "line_kw": {"marker": "o"}, + "std_display_style": "fill_between", + "score_name": "Accuracy", +} + +for ax_idx, estimator in enumerate([naive_bayes, svc]): + LearningCurveDisplay.from_estimator(estimator, **common_params, ax=ax[ax_idx]) + handles, label = ax[ax_idx].get_legend_handles_labels() + ax[ax_idx].legend(handles[:2], ["Training Score", "Test Score"]) + ax[ax_idx].set_title(f"Learning Curve for {estimator.__class__.__name__}") + +# %% +# We first analyze the learning curve of the naive Bayes classifier. Its shape +# can be found in more complex datasets very often: the training score is very +# high when using few samples for training and decreases when increasing the +# number of samples, whereas the test score is very low at the beginning and +# then increases when adding samples. The training and test scores become more +# realistic when all the samples are used for training. +# +# We see another typical learning curve for the SVM classifier with RBF kernel. +# The training score remains high regardless of the size of the training set. +# On the other hand, the test score increases with the size of the training +# dataset. Indeed, it increases up to a point where it reaches a plateau. +# Observing such a plateau is an indication that it might not be useful to +# acquire new data to train the model since the generalization performance of +# the model will not increase anymore. +# +# Complexity analysis +# =================== +# +# In addition to these learning curves, it is also possible to look at the +# scalability of the predictive models in terms of training and scoring times. +# +# The :class:`~sklearn.model_selection.LearningCurveDisplay` class does not +# provide such information. We need to resort to the +# :func:`~sklearn.model_selection.learning_curve` function instead and make +# the plot manually. + +# %% +from sklearn.model_selection import learning_curve -title = r"Learning Curves (SVM, RBF kernel, $\gamma=0.001$)" -# SVC is more expensive so we do a lower number of CV iterations: -cv = ShuffleSplit(n_splits=10, test_size=0.2, random_state=0) -estimator = SVC(gamma=0.001) -plot_learning_curve(estimator, title, X, y, axes=axes[:, 1], ylim=(0.7, 1.01), - cv=cv, n_jobs=4) +common_params = { + "X": X, + "y": y, + "train_sizes": np.linspace(0.1, 1.0, 5), + "cv": ShuffleSplit(n_splits=50, test_size=0.2, random_state=0), + "n_jobs": 4, + "return_times": True, +} + +train_sizes, _, test_scores_nb, fit_times_nb, score_times_nb = learning_curve( + naive_bayes, **common_params +) +train_sizes, _, test_scores_svm, fit_times_svm, score_times_svm = learning_curve( + svc, **common_params +) + +# %% +fig, ax = plt.subplots(nrows=2, ncols=2, figsize=(16, 12), sharex=True) + +for ax_idx, (fit_times, score_times, estimator) in enumerate( + zip( + [fit_times_nb, fit_times_svm], + [score_times_nb, score_times_svm], + [naive_bayes, svc], + ) +): + # scalability regarding the fit time + ax[0, ax_idx].plot(train_sizes, fit_times.mean(axis=1), "o-") + ax[0, ax_idx].fill_between( + train_sizes, + fit_times.mean(axis=1) - fit_times.std(axis=1), + fit_times.mean(axis=1) + fit_times.std(axis=1), + alpha=0.3, + ) + ax[0, ax_idx].set_ylabel("Fit time (s)") + ax[0, ax_idx].set_title( + f"Scalability of the {estimator.__class__.__name__} classifier" + ) + + # scalability regarding the score time + ax[1, ax_idx].plot(train_sizes, score_times.mean(axis=1), "o-") + ax[1, ax_idx].fill_between( + train_sizes, + score_times.mean(axis=1) - score_times.std(axis=1), + score_times.mean(axis=1) + score_times.std(axis=1), + alpha=0.3, + ) + ax[1, ax_idx].set_ylabel("Score time (s)") + ax[1, ax_idx].set_xlabel("Number of training samples") + +# %% +# We see that the scalability of the SVM and naive Bayes classifiers is very +# different. The SVM classifier complexity at fit and score time increases +# rapidly with the number of samples. Indeed, it is known that the fit time +# complexity of this classifier is more than quadratic with the number of +# samples which makes it hard to scale to dataset with more than a few +# 10,000 samples. In contrast, the naive Bayes classifier scales much better +# with a lower complexity at fit and score time. +# +# Subsequently, we can check the trade-off between increased training time and +# the cross-validation score. + +# %% +fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(16, 6)) + +for ax_idx, (fit_times, test_scores, estimator) in enumerate( + zip( + [fit_times_nb, fit_times_svm], + [test_scores_nb, test_scores_svm], + [naive_bayes, svc], + ) +): + ax[ax_idx].plot(fit_times.mean(axis=1), test_scores.mean(axis=1), "o-") + ax[ax_idx].fill_between( + fit_times.mean(axis=1), + test_scores.mean(axis=1) - test_scores.std(axis=1), + test_scores.mean(axis=1) + test_scores.std(axis=1), + alpha=0.3, + ) + ax[ax_idx].set_ylabel("Accuracy") + ax[ax_idx].set_xlabel("Fit time (s)") + ax[ax_idx].set_title( + f"Performance of the {estimator.__class__.__name__} classifier" + ) plt.show() + +# %% +# In these plots, we can look for the inflection point for which the +# cross-validation score does not increase anymore and only the training time +# increases. diff --git a/examples/model_selection/plot_likelihood_ratios.py b/examples/model_selection/plot_likelihood_ratios.py new file mode 100644 index 0000000000000..e4c1a6662ffa3 --- /dev/null +++ b/examples/model_selection/plot_likelihood_ratios.py @@ -0,0 +1,326 @@ +""" +============================================================= +Class Likelihood Ratios to measure classification performance +============================================================= + +This example demonstrates the :func:`~sklearn.metrics.class_likelihood_ratios` +function, which computes the positive and negative likelihood ratios (`LR+`, +`LR-`) to assess the predictive power of a binary classifier. As we will see, +these metrics are independent of the proportion between classes in the test set, +which makes them very useful when the available data for a study has a different +class proportion than the target application. + +A typical use is a case-control study in medicine, which has nearly balanced +classes while the general population has large class imbalance. In such +application, the pre-test probability of an individual having the target +condition can be chosen to be the prevalence, i.e. the proportion of a +particular population found to be affected by a medical condition. The post-test +probabilities represent then the probability that the condition is truly present +given a positive test result. + +In this example we first discuss the link between pre-test and post-test odds +given by the :ref:`class_likelihood_ratios`. Then we evaluate their behavior in +some controlled scenarios. In the last section we plot them as a function of the +prevalence of the positive class. + +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Pre-test vs. post-test analysis +# =============================== +# +# Suppose we have a population of subjects with physiological measurements `X` +# that can hopefully serve as indirect bio-markers of the disease and actual +# disease indicators `y` (ground truth). Most of the people in the population do +# not carry the disease but a minority (in this case around 10%) does: + +from sklearn.datasets import make_classification + +X, y = make_classification(n_samples=10_000, weights=[0.9, 0.1], random_state=0) +print(f"Percentage of people carrying the disease: {100 * y.mean():.2f}%") + +# %% +# A machine learning model is built to diagnose if a person with some given +# physiological measurements is likely to carry the disease of interest. To +# evaluate the model, we need to assess its performance on a held-out test set: + +from sklearn.model_selection import train_test_split + +X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0) + +# %% +# Then we can fit our diagnosis model and compute the positive likelihood +# ratio to evaluate the usefulness of this classifier as a disease diagnosis +# tool: + +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import class_likelihood_ratios + +estimator = LogisticRegression().fit(X_train, y_train) +y_pred = estimator.predict(X_test) +pos_LR, neg_LR = class_likelihood_ratios(y_test, y_pred, replace_undefined_by=1.0) +print(f"LR+: {pos_LR:.3f}") + +# %% +# Since the positive class likelihood ratio is much larger than 1.0, it means +# that the machine learning-based diagnosis tool is useful: the post-test odds +# that the condition is truly present given a positive test result are more than +# 12 times larger than the pre-test odds. +# +# Cross-validation of likelihood ratios +# ===================================== +# +# We assess the variability of the measurements for the class likelihood ratios +# in some particular cases. + +import pandas as pd + + +def scoring(estimator, X, y): + y_pred = estimator.predict(X) + pos_lr, neg_lr = class_likelihood_ratios(y, y_pred, replace_undefined_by=1.0) + return {"positive_likelihood_ratio": pos_lr, "negative_likelihood_ratio": neg_lr} + + +def extract_score(cv_results): + lr = pd.DataFrame( + { + "positive": cv_results["test_positive_likelihood_ratio"], + "negative": cv_results["test_negative_likelihood_ratio"], + } + ) + return lr.aggregate(["mean", "std"]) + + +# %% +# We first validate the :class:`~sklearn.linear_model.LogisticRegression` model +# with default hyperparameters as used in the previous section. + +from sklearn.model_selection import cross_validate + +estimator = LogisticRegression() +extract_score(cross_validate(estimator, X, y, scoring=scoring, cv=10)) + +# %% +# We confirm that the model is useful: the post-test odds are between 12 and 20 +# times larger than the pre-test odds. +# +# On the contrary, let's consider a dummy model that will output random +# predictions with similar odds as the average disease prevalence in the +# training set: + +from sklearn.dummy import DummyClassifier + +estimator = DummyClassifier(strategy="stratified", random_state=1234) +extract_score(cross_validate(estimator, X, y, scoring=scoring, cv=10)) + +# %% +# Here both class likelihood ratios are compatible with 1.0 which makes this +# classifier useless as a diagnostic tool to improve disease detection. +# +# Another option for the dummy model is to always predict the most frequent +# class, which in this case is "no-disease". + +estimator = DummyClassifier(strategy="most_frequent") +extract_score(cross_validate(estimator, X, y, scoring=scoring, cv=10)) + +# %% +# The absence of positive predictions means there will be no true positives nor +# false positives, leading to an undefined `LR+` that by no means should be +# interpreted as an infinite `LR+` (the classifier perfectly identifying +# positive cases). In such situation the +# :func:`~sklearn.metrics.class_likelihood_ratios` function returns `nan` and +# raises a warning by default. Indeed, the value of `LR-` helps us discard this +# model. +# +# A similar scenario may arise when cross-validating highly imbalanced data with +# few samples: some folds will have no samples with the disease and therefore +# they will output no true positives nor false negatives when used for testing. +# Mathematically this leads to an infinite `LR+`, which should also not be +# interpreted as the model perfectly identifying positive cases. Such event +# leads to a higher variance of the estimated likelihood ratios, but can still +# be interpreted as an increment of the post-test odds of having the condition. + +estimator = LogisticRegression() +X, y = make_classification(n_samples=300, weights=[0.9, 0.1], random_state=0) +extract_score(cross_validate(estimator, X, y, scoring=scoring, cv=10)) + +# %% +# Invariance with respect to prevalence +# ===================================== +# +# The likelihood ratios are independent of the disease prevalence and can be +# extrapolated between populations regardless of any possible class imbalance, +# **as long as the same model is applied to all of them**. Notice that in the +# plots below **the decision boundary is constant** (see +# :ref:`sphx_glr_auto_examples_svm_plot_separating_hyperplane_unbalanced.py` for +# a study of the boundary decision for unbalanced classes). +# +# Here we train a :class:`~sklearn.linear_model.LogisticRegression` base model +# on a case-control study with a prevalence of 50%. It is then evaluated over +# populations with varying prevalence. We use the +# :func:`~sklearn.datasets.make_classification` function to ensure the +# data-generating process is always the same as shown in the plots below. The +# label `1` corresponds to the positive class "disease", whereas the label `0` +# stands for "no-disease". + +from collections import defaultdict + +import matplotlib.pyplot as plt +import numpy as np + +from sklearn.inspection import DecisionBoundaryDisplay + +populations = defaultdict(list) +common_params = { + "n_samples": 10_000, + "n_features": 2, + "n_informative": 2, + "n_redundant": 0, + "random_state": 0, +} +weights = np.linspace(0.1, 0.8, 6) +weights = weights[::-1] + +# fit and evaluate base model on balanced classes +X, y = make_classification(**common_params, weights=[0.5, 0.5]) +estimator = LogisticRegression().fit(X, y) +lr_base = extract_score(cross_validate(estimator, X, y, scoring=scoring, cv=10)) +pos_lr_base, pos_lr_base_std = lr_base["positive"].values +neg_lr_base, neg_lr_base_std = lr_base["negative"].values + +# %% +# We will now show the decision boundary for each level of prevalence. Note that +# we only plot a subset of the original data to better assess the linear model +# decision boundary. + +fig, axs = plt.subplots(nrows=3, ncols=2, figsize=(15, 12)) + +for ax, (n, weight) in zip(axs.ravel(), enumerate(weights)): + X, y = make_classification( + **common_params, + weights=[weight, 1 - weight], + ) + prevalence = y.mean() + populations["prevalence"].append(prevalence) + populations["X"].append(X) + populations["y"].append(y) + + # down-sample for plotting + rng = np.random.RandomState(1) + plot_indices = rng.choice(np.arange(X.shape[0]), size=500, replace=True) + X_plot, y_plot = X[plot_indices], y[plot_indices] + + # plot fixed decision boundary of base model with varying prevalence + disp = DecisionBoundaryDisplay.from_estimator( + estimator, + X_plot, + response_method="predict", + alpha=0.5, + ax=ax, + ) + scatter = disp.ax_.scatter(X_plot[:, 0], X_plot[:, 1], c=y_plot, edgecolor="k") + disp.ax_.set_title(f"prevalence = {y_plot.mean():.2f}") + disp.ax_.legend(*scatter.legend_elements()) + +# %% +# We define a function for bootstrapping. + + +def scoring_on_bootstrap(estimator, X, y, rng, n_bootstrap=100): + results_for_prevalence = defaultdict(list) + for _ in range(n_bootstrap): + bootstrap_indices = rng.choice( + np.arange(X.shape[0]), size=X.shape[0], replace=True + ) + for key, value in scoring( + estimator, X[bootstrap_indices], y[bootstrap_indices] + ).items(): + results_for_prevalence[key].append(value) + return pd.DataFrame(results_for_prevalence) + + +# %% +# We score the base model for each prevalence using bootstrapping. + +results = defaultdict(list) +n_bootstrap = 100 +rng = np.random.default_rng(seed=0) + +for prevalence, X, y in zip( + populations["prevalence"], populations["X"], populations["y"] +): + results_for_prevalence = scoring_on_bootstrap( + estimator, X, y, rng, n_bootstrap=n_bootstrap + ) + results["prevalence"].append(prevalence) + results["metrics"].append( + results_for_prevalence.aggregate(["mean", "std"]).unstack() + ) + +results = pd.DataFrame(results["metrics"], index=results["prevalence"]) +results.index.name = "prevalence" +results + +# %% +# In the plots below we observe that the class likelihood ratios re-computed +# with different prevalences are indeed constant within one standard deviation +# of those computed with on balanced classes. + +fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, figsize=(15, 6)) +results["positive_likelihood_ratio"]["mean"].plot( + ax=ax1, color="r", label="extrapolation through populations" +) +ax1.axhline(y=pos_lr_base + pos_lr_base_std, color="r", linestyle="--") +ax1.axhline( + y=pos_lr_base - pos_lr_base_std, + color="r", + linestyle="--", + label="base model confidence band", +) +ax1.fill_between( + results.index, + results["positive_likelihood_ratio"]["mean"] + - results["positive_likelihood_ratio"]["std"], + results["positive_likelihood_ratio"]["mean"] + + results["positive_likelihood_ratio"]["std"], + color="r", + alpha=0.3, +) +ax1.set( + title="Positive likelihood ratio", + ylabel="LR+", + ylim=[0, 5], +) +ax1.legend(loc="lower right") + +ax2 = results["negative_likelihood_ratio"]["mean"].plot( + ax=ax2, color="b", label="extrapolation through populations" +) +ax2.axhline(y=neg_lr_base + neg_lr_base_std, color="b", linestyle="--") +ax2.axhline( + y=neg_lr_base - neg_lr_base_std, + color="b", + linestyle="--", + label="base model confidence band", +) +ax2.fill_between( + results.index, + results["negative_likelihood_ratio"]["mean"] + - results["negative_likelihood_ratio"]["std"], + results["negative_likelihood_ratio"]["mean"] + + results["negative_likelihood_ratio"]["std"], + color="b", + alpha=0.3, +) +ax2.set( + title="Negative likelihood ratio", + ylabel="LR-", + ylim=[0, 0.5], +) +ax2.legend(loc="lower right") + +plt.show() diff --git a/examples/model_selection/plot_multi_metric_evaluation.py b/examples/model_selection/plot_multi_metric_evaluation.py index 775d0af9817a8..14d8c52dd82a9 100644 --- a/examples/model_selection/plot_multi_metric_evaluation.py +++ b/examples/model_selection/plot_multi_metric_evaluation.py @@ -13,22 +13,20 @@ The ``best_estimator_``, ``best_index_``, ``best_score_`` and ``best_params_`` correspond to the scorer (key) that is set to the ``refit`` attribute. + """ -# Author: Raghav RV -# License: BSD +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import numpy as np from matplotlib import pyplot as plt from sklearn.datasets import make_hastie_10_2 +from sklearn.metrics import accuracy_score, make_scorer from sklearn.model_selection import GridSearchCV -from sklearn.metrics import make_scorer -from sklearn.metrics import accuracy_score from sklearn.tree import DecisionTreeClassifier -print(__doc__) - # %% # Running ``GridSearchCV`` using multiple evaluation metrics # ---------------------------------------------------------- @@ -36,18 +34,23 @@ X, y = make_hastie_10_2(n_samples=8000, random_state=42) -# The scorers can be either be one of the predefined metric strings or a scorer +# The scorers can be either one of the predefined metric strings or a scorer # callable, like the one returned by make_scorer -scoring = {'AUC': 'roc_auc', 'Accuracy': make_scorer(accuracy_score)} +scoring = {"AUC": "roc_auc", "Accuracy": make_scorer(accuracy_score)} # Setting refit='AUC', refits an estimator on the whole dataset with the # parameter setting that has the best cross-validated AUC score. # That estimator is made available at ``gs.best_estimator_`` along with # parameters like ``gs.best_score_``, ``gs.best_params_`` and # ``gs.best_index_`` -gs = GridSearchCV(DecisionTreeClassifier(random_state=42), - param_grid={'min_samples_split': range(2, 403, 10)}, - scoring=scoring, refit='AUC', return_train_score=True) +gs = GridSearchCV( + DecisionTreeClassifier(random_state=42), + param_grid={"min_samples_split": range(2, 403, 20)}, + scoring=scoring, + refit="AUC", + n_jobs=2, + return_train_score=True, +) gs.fit(X, y) results = gs.cv_results_ @@ -56,8 +59,7 @@ # ------------------- plt.figure(figsize=(13, 13)) -plt.title("GridSearchCV evaluating using multiple scorers simultaneously", - fontsize=16) +plt.title("GridSearchCV evaluating using multiple scorers simultaneously", fontsize=16) plt.xlabel("min_samples_split") plt.ylabel("Score") @@ -67,29 +69,47 @@ ax.set_ylim(0.73, 1) # Get the regular numpy array from the MaskedArray -X_axis = np.array(results['param_min_samples_split'].data, dtype=float) - -for scorer, color in zip(sorted(scoring), ['g', 'k']): - for sample, style in (('train', '--'), ('test', '-')): - sample_score_mean = results['mean_%s_%s' % (sample, scorer)] - sample_score_std = results['std_%s_%s' % (sample, scorer)] - ax.fill_between(X_axis, sample_score_mean - sample_score_std, - sample_score_mean + sample_score_std, - alpha=0.1 if sample == 'test' else 0, color=color) - ax.plot(X_axis, sample_score_mean, style, color=color, - alpha=1 if sample == 'test' else 0.7, - label="%s (%s)" % (scorer, sample)) - - best_index = np.nonzero(results['rank_test_%s' % scorer] == 1)[0][0] - best_score = results['mean_test_%s' % scorer][best_index] +X_axis = np.array(results["param_min_samples_split"].data, dtype=float) + +for scorer, color in zip(sorted(scoring), ["g", "k"]): + for sample, style in (("train", "--"), ("test", "-")): + sample_score_mean = results["mean_%s_%s" % (sample, scorer)] + sample_score_std = results["std_%s_%s" % (sample, scorer)] + ax.fill_between( + X_axis, + sample_score_mean - sample_score_std, + sample_score_mean + sample_score_std, + alpha=0.1 if sample == "test" else 0, + color=color, + ) + ax.plot( + X_axis, + sample_score_mean, + style, + color=color, + alpha=1 if sample == "test" else 0.7, + label="%s (%s)" % (scorer, sample), + ) + + best_index = np.nonzero(results["rank_test_%s" % scorer] == 1)[0][0] + best_score = results["mean_test_%s" % scorer][best_index] # Plot a dotted vertical line at the best score for that scorer marked by x - ax.plot([X_axis[best_index], ] * 2, [0, best_score], - linestyle='-.', color=color, marker='x', markeredgewidth=3, ms=8) + ax.plot( + [ + X_axis[best_index], + ] + * 2, + [0, best_score], + linestyle="-.", + color=color, + marker="x", + markeredgewidth=3, + ms=8, + ) # Annotate the best score for that scorer - ax.annotate("%0.2f" % best_score, - (X_axis[best_index], best_score + 0.005)) + ax.annotate("%0.2f" % best_score, (X_axis[best_index], best_score + 0.005)) plt.legend(loc="best") plt.grid(False) diff --git a/examples/model_selection/plot_nested_cross_validation_iris.py b/examples/model_selection/plot_nested_cross_validation_iris.py index d6aea44e6c546..15082123761af 100644 --- a/examples/model_selection/plot_nested_cross_validation_iris.py +++ b/examples/model_selection/plot_nested_cross_validation_iris.py @@ -30,26 +30,29 @@ performance of non-nested and nested CV strategies by taking the difference between their scores. -.. topic:: See Also: +.. seealso:: - :ref:`cross_validation` - :ref:`grid_search` -.. topic:: References: +.. rubric:: References - .. [1] `Cawley, G.C.; Talbot, N.L.C. On over-fitting in model selection and - subsequent selection bias in performance evaluation. - J. Mach. Learn. Res 2010,11, 2079-2107. - `_ +.. [1] `Cawley, G.C.; Talbot, N.L.C. On over-fitting in model selection and + subsequent selection bias in performance evaluation. + J. Mach. Learn. Res 2010,11, 2079-2107. + `_ """ -from sklearn.datasets import load_iris -from matplotlib import pyplot as plt -from sklearn.svm import SVC -from sklearn.model_selection import GridSearchCV, cross_val_score, KFold + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import numpy as np +from matplotlib import pyplot as plt -print(__doc__) +from sklearn.datasets import load_iris +from sklearn.model_selection import GridSearchCV, KFold, cross_val_score +from sklearn.svm import SVC # Number of random trials NUM_TRIALS = 30 @@ -60,8 +63,7 @@ y_iris = iris.target # Set up possible values of parameters to optimize over -p_grid = {"C": [1, 10, 100], - "gamma": [.01, .1]} +p_grid = {"C": [1, 10, 100], "gamma": [0.01, 0.1]} # We will use a Support Vector Classifier with "rbf" kernel svm = SVC(kernel="rbf") @@ -72,7 +74,6 @@ # Loop for each trial for i in range(NUM_TRIALS): - # Choose cross-validation techniques for the inner and outer loops, # independently of the dataset. # E.g "GroupKFold", "LeaveOneOut", "LeaveOneGroupOut", etc. @@ -80,38 +81,50 @@ outer_cv = KFold(n_splits=4, shuffle=True, random_state=i) # Non_nested parameter search and scoring - clf = GridSearchCV(estimator=svm, param_grid=p_grid, cv=inner_cv) + clf = GridSearchCV(estimator=svm, param_grid=p_grid, cv=outer_cv) clf.fit(X_iris, y_iris) non_nested_scores[i] = clf.best_score_ # Nested CV with parameter optimization + clf = GridSearchCV(estimator=svm, param_grid=p_grid, cv=inner_cv) nested_score = cross_val_score(clf, X=X_iris, y=y_iris, cv=outer_cv) nested_scores[i] = nested_score.mean() score_difference = non_nested_scores - nested_scores -print("Average difference of {:6f} with std. dev. of {:6f}." - .format(score_difference.mean(), score_difference.std())) +print( + "Average difference of {:6f} with std. dev. of {:6f}.".format( + score_difference.mean(), score_difference.std() + ) +) # Plot scores on each trial for nested and non-nested CV plt.figure() plt.subplot(211) -non_nested_scores_line, = plt.plot(non_nested_scores, color='r') -nested_line, = plt.plot(nested_scores, color='b') +(non_nested_scores_line,) = plt.plot(non_nested_scores, color="r") +(nested_line,) = plt.plot(nested_scores, color="b") plt.ylabel("score", fontsize="14") -plt.legend([non_nested_scores_line, nested_line], - ["Non-Nested CV", "Nested CV"], - bbox_to_anchor=(0, .4, .5, 0)) -plt.title("Non-Nested and Nested Cross Validation on Iris Dataset", - x=.5, y=1.1, fontsize="15") +plt.legend( + [non_nested_scores_line, nested_line], + ["Non-Nested CV", "Nested CV"], + bbox_to_anchor=(0, 0.4, 0.5, 0), +) +plt.title( + "Non-Nested and Nested Cross Validation on Iris Dataset", + x=0.5, + y=1.1, + fontsize="15", +) # Plot bar chart of the difference. plt.subplot(212) difference_plot = plt.bar(range(NUM_TRIALS), score_difference) plt.xlabel("Individual Trial #") -plt.legend([difference_plot], - ["Non-Nested CV - Nested CV Score"], - bbox_to_anchor=(0, 1, .8, 0)) +plt.legend( + [difference_plot], + ["Non-Nested CV - Nested CV Score"], + bbox_to_anchor=(0, 1, 0.8, 0), +) plt.ylabel("score difference", fontsize="14") plt.show() diff --git a/examples/feature_selection/plot_permutation_test_for_classification.py b/examples/model_selection/plot_permutation_tests_for_classification.py similarity index 51% rename from examples/feature_selection/plot_permutation_test_for_classification.py rename to examples/model_selection/plot_permutation_tests_for_classification.py index 6468ad2dfdca8..77afd2aca89ce 100644 --- a/examples/feature_selection/plot_permutation_test_for_classification.py +++ b/examples/model_selection/plot_permutation_tests_for_classification.py @@ -5,18 +5,20 @@ This example demonstrates the use of :func:`~sklearn.model_selection.permutation_test_score` to evaluate the -significance of a cross-valdiated score using permutations. +significance of a cross-validated score using permutations. + """ -# Authors: Alexandre Gramfort -# Lucy Liu -# License: BSD 3 clause -# +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% # Dataset # ------- # # We will use the :ref:`iris_dataset`, which consists of measurements taken -# from 3 types of irises. +# from 3 Iris species. Our model will use the measurements to predict +# the iris species. from sklearn.datasets import load_iris @@ -25,14 +27,14 @@ y = iris.target # %% -# We will also generate some random feature data (i.e., 2200 features), +# For comparison, we also generate some random feature data (i.e., 20 features), # uncorrelated with the class labels in the iris dataset. import numpy as np -n_uncorrelated_features = 2200 +n_uncorrelated_features = 20 rng = np.random.RandomState(seed=0) -# Use same number of samples as in iris and 2200 features +# Use same number of samples as in iris and 20 features X_rand = rng.normal(size=(X.shape[0], n_uncorrelated_features)) # %% @@ -40,34 +42,36 @@ # ---------------------- # # Next, we calculate the -# :func:`~sklearn.model_selection.permutation_test_score` using the original -# iris dataset, which strongly predict the labels and -# the randomly generated features and iris labels, which should have -# no dependency between features and labels. We use the +# :func:`~sklearn.model_selection.permutation_test_score` for both, the original +# iris dataset (where there's a strong relationship between features and labels) and +# the randomly generated features with iris labels (where no dependency between features +# and labels is expected). We use the # :class:`~sklearn.svm.SVC` classifier and :ref:`accuracy_score` to evaluate # the model at each round. # # :func:`~sklearn.model_selection.permutation_test_score` generates a null # distribution by calculating the accuracy of the classifier # on 1000 different permutations of the dataset, where features -# remain the same but labels undergo different permutations. This is the +# remain the same but labels undergo different random permutations. This is the # distribution for the null hypothesis which states there is no dependency # between the features and labels. An empirical p-value is then calculated as -# the percentage of permutations for which the score obtained is greater -# that the score obtained using the original data. +# the proportion of permutations, for which the score obtained by the model trained on +# the permutation, is greater than or equal to the score obtained using the original +# data. +from sklearn.model_selection import StratifiedKFold, permutation_test_score from sklearn.svm import SVC -from sklearn.model_selection import StratifiedKFold -from sklearn.model_selection import permutation_test_score -clf = SVC(kernel='linear', random_state=7) -cv = StratifiedKFold(2, shuffle=True, random_state=0) +clf = SVC(kernel="linear", random_state=7) +cv = StratifiedKFold(n_splits=2, shuffle=True, random_state=0) score_iris, perm_scores_iris, pvalue_iris = permutation_test_score( - clf, X, y, scoring="accuracy", cv=cv, n_permutations=1000) + clf, X, y, scoring="accuracy", cv=cv, n_permutations=1000 +) score_rand, perm_scores_rand, pvalue_rand = permutation_test_score( - clf, X_rand, y, scoring="accuracy", cv=cv, n_permutations=1000) + clf, X_rand, y, scoring="accuracy", cv=cv, n_permutations=1000 +) # %% # Original data @@ -75,24 +79,25 @@ # # Below we plot a histogram of the permutation scores (the null # distribution). The red line indicates the score obtained by the classifier -# on the original data. The score is much better than those obtained by -# using permuted data and the p-value is thus very low. This indicates that +# on the original data (without permuted labels). The score is much better than those +# obtained by using permuted data and the p-value is thus very low. This indicates that # there is a low likelihood that this good score would be obtained by chance # alone. It provides evidence that the iris dataset contains real dependency # between features and labels and the classifier was able to utilize this -# to obtain good results. +# to obtain good results. The low p-value can lead us to reject the null hypothesis. import matplotlib.pyplot as plt fig, ax = plt.subplots() ax.hist(perm_scores_iris, bins=20, density=True) -ax.axvline(score_iris, ls='--', color='r') -score_label = (f"Score on original\ndata: {score_iris:.2f}\n" - f"(p-value: {pvalue_iris:.3f})") -ax.text(0.7, 260, score_label, fontsize=12) +ax.axvline(score_iris, ls="--", color="r") +score_label = ( + f"Score on original\niris data: {score_iris:.2f}\n(p-value: {pvalue_iris:.3f})" +) +ax.text(0.7, 10, score_label, fontsize=12) ax.set_xlabel("Accuracy score") -_ = ax.set_ylabel("Probability") +_ = ax.set_ylabel("Probability density") # %% # Random data @@ -100,37 +105,40 @@ # # Below we plot the null distribution for the randomized data. The permutation # scores are similar to those obtained using the original iris dataset -# because the permutation always destroys any feature label dependency present. -# The score obtained on the original randomized data in this case though, is -# very poor. This results in a large p-value, confirming that there was no -# feature label dependency in the original data. +# because the permutation always destroys any feature-label dependency present. +# The score obtained on the randomized data in this case +# though, is very poor. This results in a large p-value, confirming that there was no +# feature-label dependency in the randomized data. fig, ax = plt.subplots() ax.hist(perm_scores_rand, bins=20, density=True) ax.set_xlim(0.13) -ax.axvline(score_rand, ls='--', color='r') -score_label = (f"Score on original\ndata: {score_rand:.2f}\n" - f"(p-value: {pvalue_rand:.3f})") -ax.text(0.14, 125, score_label, fontsize=12) +ax.axvline(score_rand, ls="--", color="r") +score_label = ( + f"Score on original\nrandom data: {score_rand:.2f}\n(p-value: {pvalue_rand:.3f})" +) +ax.text(0.14, 7.5, score_label, fontsize=12) ax.set_xlabel("Accuracy score") -ax.set_ylabel("Probability") +ax.set_ylabel("Probability density") plt.show() # %% -# Another possible reason for obtaining a high p-value is that the classifier +# Another possible reason for obtaining a high p-value could be that the classifier # was not able to use the structure in the data. In this case, the p-value # would only be low for classifiers that are able to utilize the dependency # present. In our case above, where the data is random, all classifiers would -# have a high p-value as there is no structure present in the data. +# have a high p-value as there is no structure present in the data. We might or might +# not fail to reject the null hypothesis depending on whether the p-value is high on a +# more appropriate estimator as well. # # Finally, note that this test has been shown to produce low p-values even # if there is only weak structure in the data [1]_. # -# .. topic:: References: +# .. rubric:: References # -# .. [1] Ojala and Garriga. `Permutation Tests for Studying Classifier -# Performance -# `_. The -# Journal of Machine Learning Research (2010) vol. 11 +# .. [1] Ojala and Garriga. `Permutation Tests for Studying Classifier +# Performance +# `_. The +# Journal of Machine Learning Research (2010) vol. 11 # diff --git a/examples/model_selection/plot_precision_recall.py b/examples/model_selection/plot_precision_recall.py index 620e1d404a2fd..c7ff06d3f8fcb 100644 --- a/examples/model_selection/plot_precision_recall.py +++ b/examples/model_selection/plot_precision_recall.py @@ -7,43 +7,42 @@ Precision-Recall is a useful measure of success of prediction when the classes are very imbalanced. In information retrieval, precision is a -measure of result relevancy, while recall is a measure of how many truly -relevant results are returned. - -The precision-recall curve shows the tradeoff between precision and -recall for different threshold. A high area under the curve represents -both high recall and high precision, where high precision relates to a -low false positive rate, and high recall relates to a low false negative -rate. High scores for both show that the classifier is returning accurate -results (high precision), as well as returning a majority of all positive -results (high recall). - -A system with high recall but low precision returns many results, but most of -its predicted labels are incorrect when compared to the training labels. A -system with high precision but low recall is just the opposite, returning very -few results, but most of its predicted labels are correct when compared to the -training labels. An ideal system with high precision and high recall will -return many results, with all results labeled correctly. +measure of the fraction of relevant items among actually returned items while recall +is a measure of the fraction of items that were returned among all items that should +have been returned. 'Relevancy' here refers to items that are +positively labeled, i.e., true positives and false negatives. Precision (:math:`P`) is defined as the number of true positives (:math:`T_p`) over the number of true positives plus the number of false positives (:math:`F_p`). -:math:`P = \\frac{T_p}{T_p+F_p}` +.. math:: + P = \\frac{T_p}{T_p+F_p} Recall (:math:`R`) is defined as the number of true positives (:math:`T_p`) over the number of true positives plus the number of false negatives (:math:`F_n`). -:math:`R = \\frac{T_p}{T_p + F_n}` +.. math:: + R = \\frac{T_p}{T_p + F_n} -These quantities are also related to the (:math:`F_1`) score, which is defined -as the harmonic mean of precision and recall. +The precision-recall curve shows the tradeoff between precision and +recall for different thresholds. A high area under the curve represents +both high recall and high precision. High precision is achieved by having +few false positives in the returned results, and high recall is achieved by +having few false negatives in the relevant results. +High scores for both show that the classifier is returning +accurate results (high precision), as well as returning a majority of all relevant +results (high recall). -:math:`F1 = 2\\frac{P \\times R}{P+R}` +A system with high recall but low precision returns most of the relevant items, but +the proportion of returned results that are incorrectly labeled is high. A +system with high precision but low recall is just the opposite, returning very +few of the relevant items, but most of its predicted labels are correct when compared +to the actual labels. An ideal system with high precision and high recall will +return most of the relevant items, with most results labeled correctly. -Note that the precision may not decrease with recall. The -definition of precision (:math:`\\frac{T_p}{T_p + F_p}`) shows that lowering +The definition of precision (:math:`\\frac{T_p}{T_p + F_p}`) shows that lowering the threshold of a classifier may increase the denominator, by increasing the number of results returned. If the threshold was previously set too high, the new results may all be true positives, which will increase precision. If the @@ -51,10 +50,12 @@ will introduce false positives, decreasing precision. Recall is defined as :math:`\\frac{T_p}{T_p+F_n}`, where :math:`T_p+F_n` does -not depend on the classifier threshold. This means that lowering the classifier +not depend on the classifier threshold. Changing the classifier threshold can only +change the numerator, :math:`T_p`. Lowering the classifier threshold may increase recall, by increasing the number of true positive results. It is also possible that lowering the threshold may leave recall -unchanged, while the precision fluctuates. +unchanged, while the precision fluctuates. Thus, precision does not necessarily +decrease with recall. The relationship between recall and precision can be observed in the stairstep area of the plot - at the edges of these steps a small change @@ -81,7 +82,7 @@ average precision to multi-class or multi-label classification, it is necessary to binarize the output. One curve can be drawn per label, but one can also draw a precision-recall curve by considering each element of the label indicator -matrix as a binary prediction (micro-averaging). +matrix as a binary prediction (:ref:`micro-averaging `). .. note:: @@ -90,66 +91,89 @@ :func:`sklearn.metrics.precision_score`, :func:`sklearn.metrics.f1_score` """ + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + # %% # In binary classification settings -# -------------------------------------------------------- +# --------------------------------- # -# Create simple data -# .................. +# Dataset and model +# ................. # -# Try to differentiate the two first classes of the iris data -from sklearn import svm, datasets -from sklearn.model_selection import train_test_split +# We will use a Linear SVC classifier to differentiate two types of irises. import numpy as np -iris = datasets.load_iris() -X = iris.data -y = iris.target +from sklearn.datasets import load_iris +from sklearn.model_selection import train_test_split + +X, y = load_iris(return_X_y=True) # Add noisy features random_state = np.random.RandomState(0) n_samples, n_features = X.shape -X = np.c_[X, random_state.randn(n_samples, 200 * n_features)] +X = np.concatenate([X, random_state.randn(n_samples, 200 * n_features)], axis=1) # Limit to the two first classes, and split into training and test -X_train, X_test, y_train, y_test = train_test_split(X[y < 2], y[y < 2], - test_size=.5, - random_state=random_state) +X_train, X_test, y_train, y_test = train_test_split( + X[y < 2], y[y < 2], test_size=0.5, random_state=random_state +) -# Create a simple classifier -classifier = svm.LinearSVC(random_state=random_state) +# %% +# Linear SVC will expect each feature to have a similar range of values. Thus, +# we will first scale the data using a +# :class:`~sklearn.preprocessing.StandardScaler`. +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import StandardScaler +from sklearn.svm import LinearSVC + +classifier = make_pipeline(StandardScaler(), LinearSVC(random_state=random_state)) classifier.fit(X_train, y_train) -y_score = classifier.decision_function(X_test) # %% -# Compute the average precision score -# ................................... -from sklearn.metrics import average_precision_score -average_precision = average_precision_score(y_test, y_score) +# Plot the Precision-Recall curve +# ............................... +# +# To plot the precision-recall curve, you should use +# :class:`~sklearn.metrics.PrecisionRecallDisplay`. Indeed, there is two +# methods available depending if you already computed the predictions of the +# classifier or not. +# +# Let's first plot the precision-recall curve without the classifier +# predictions. We use +# :func:`~sklearn.metrics.PrecisionRecallDisplay.from_estimator` that +# computes the predictions for us before plotting the curve. +from sklearn.metrics import PrecisionRecallDisplay -print('Average precision-recall score: {0:0.2f}'.format( - average_precision)) +display = PrecisionRecallDisplay.from_estimator( + classifier, X_test, y_test, name="LinearSVC", plot_chance_level=True, despine=True +) +_ = display.ax_.set_title("2-class Precision-Recall curve") # %% -# Plot the Precision-Recall curve -# ................................ -from sklearn.metrics import precision_recall_curve -from sklearn.metrics import plot_precision_recall_curve -import matplotlib.pyplot as plt +# If we already got the estimated probabilities or scores for +# our model, then we can use +# :func:`~sklearn.metrics.PrecisionRecallDisplay.from_predictions`. +y_score = classifier.decision_function(X_test) -disp = plot_precision_recall_curve(classifier, X_test, y_test) -disp.ax_.set_title('2-class Precision-Recall curve: ' - 'AP={0:0.2f}'.format(average_precision)) +display = PrecisionRecallDisplay.from_predictions( + y_test, y_score, name="LinearSVC", plot_chance_level=True, despine=True +) +_ = display.ax_.set_title("2-class Precision-Recall curve") # %% # In multi-label settings -# ------------------------ +# ----------------------- +# +# The precision-recall curve does not support the multilabel setting. However, +# one can decide how to handle this case. We show such an example below. # # Create multi-label data, fit, and predict -# ........................................... +# ......................................... # # We create a multi-label dataset, to illustrate the precision-recall in -# multi-label settings +# multi-label settings. from sklearn.preprocessing import label_binarize @@ -158,96 +182,98 @@ n_classes = Y.shape[1] # Split into training and test -X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=.5, - random_state=random_state) +X_train, X_test, Y_train, Y_test = train_test_split( + X, Y, test_size=0.5, random_state=random_state +) -# We use OneVsRestClassifier for multi-label prediction +# %% +# We use :class:`~sklearn.multiclass.OneVsRestClassifier` for multi-label +# prediction. from sklearn.multiclass import OneVsRestClassifier -# Run classifier -classifier = OneVsRestClassifier(svm.LinearSVC(random_state=random_state)) +classifier = OneVsRestClassifier( + make_pipeline(StandardScaler(), LinearSVC(random_state=random_state)) +) classifier.fit(X_train, Y_train) y_score = classifier.decision_function(X_test) # %% # The average precision score in multi-label settings -# .................................................... -from sklearn.metrics import precision_recall_curve -from sklearn.metrics import average_precision_score +# ................................................... +from sklearn.metrics import average_precision_score, precision_recall_curve # For each class precision = dict() recall = dict() average_precision = dict() for i in range(n_classes): - precision[i], recall[i], _ = precision_recall_curve(Y_test[:, i], - y_score[:, i]) + precision[i], recall[i], _ = precision_recall_curve(Y_test[:, i], y_score[:, i]) average_precision[i] = average_precision_score(Y_test[:, i], y_score[:, i]) # A "micro-average": quantifying score on all classes jointly -precision["micro"], recall["micro"], _ = precision_recall_curve(Y_test.ravel(), - y_score.ravel()) -average_precision["micro"] = average_precision_score(Y_test, y_score, - average="micro") -print('Average precision score, micro-averaged over all classes: {0:0.2f}' - .format(average_precision["micro"])) +precision["micro"], recall["micro"], _ = precision_recall_curve( + Y_test.ravel(), y_score.ravel() +) +average_precision["micro"] = average_precision_score(Y_test, y_score, average="micro") # %% # Plot the micro-averaged Precision-Recall curve -# ............................................... -# - -plt.figure() -plt.step(recall['micro'], precision['micro'], where='post') - -plt.xlabel('Recall') -plt.ylabel('Precision') -plt.ylim([0.0, 1.05]) -plt.xlim([0.0, 1.0]) -plt.title( - 'Average precision score, micro-averaged over all classes: AP={0:0.2f}' - .format(average_precision["micro"])) +# .............................................. +from collections import Counter + +display = PrecisionRecallDisplay( + recall=recall["micro"], + precision=precision["micro"], + average_precision=average_precision["micro"], + prevalence_pos_label=Counter(Y_test.ravel())[1] / Y_test.size, +) +display.plot(plot_chance_level=True, despine=True) +_ = display.ax_.set_title("Micro-averaged over all classes") # %% # Plot Precision-Recall curve for each class and iso-f1 curves -# ............................................................. -# +# ............................................................ from itertools import cycle + +import matplotlib.pyplot as plt + # setup plot details -colors = cycle(['navy', 'turquoise', 'darkorange', 'cornflowerblue', 'teal']) +colors = cycle(["navy", "turquoise", "darkorange", "cornflowerblue", "teal"]) + +_, ax = plt.subplots(figsize=(7, 8)) -plt.figure(figsize=(7, 8)) f_scores = np.linspace(0.2, 0.8, num=4) -lines = [] -labels = [] +lines, labels = [], [] for f_score in f_scores: x = np.linspace(0.01, 1) y = f_score * x / (2 * x - f_score) - l, = plt.plot(x[y >= 0], y[y >= 0], color='gray', alpha=0.2) - plt.annotate('f1={0:0.1f}'.format(f_score), xy=(0.9, y[45] + 0.02)) + (l,) = plt.plot(x[y >= 0], y[y >= 0], color="gray", alpha=0.2) + plt.annotate("f1={0:0.1f}".format(f_score), xy=(0.9, y[45] + 0.02)) -lines.append(l) -labels.append('iso-f1 curves') -l, = plt.plot(recall["micro"], precision["micro"], color='gold', lw=2) -lines.append(l) -labels.append('micro-average Precision-recall (area = {0:0.2f})' - ''.format(average_precision["micro"])) +display = PrecisionRecallDisplay( + recall=recall["micro"], + precision=precision["micro"], + average_precision=average_precision["micro"], +) +display.plot(ax=ax, name="Micro-average precision-recall", color="gold") for i, color in zip(range(n_classes), colors): - l, = plt.plot(recall[i], precision[i], color=color, lw=2) - lines.append(l) - labels.append('Precision-recall for class {0} (area = {1:0.2f})' - ''.format(i, average_precision[i])) - -fig = plt.gcf() -fig.subplots_adjust(bottom=0.25) -plt.xlim([0.0, 1.0]) -plt.ylim([0.0, 1.05]) -plt.xlabel('Recall') -plt.ylabel('Precision') -plt.title('Extension of Precision-Recall curve to multi-class') -plt.legend(lines, labels, loc=(0, -.38), prop=dict(size=14)) - + display = PrecisionRecallDisplay( + recall=recall[i], + precision=precision[i], + average_precision=average_precision[i], + ) + display.plot( + ax=ax, name=f"Precision-recall for class {i}", color=color, despine=True + ) + +# add the legend for the iso-f1 curves +handles, labels = display.ax_.get_legend_handles_labels() +handles.extend([l]) +labels.extend(["iso-f1 curves"]) +# set the legend and the axes +ax.legend(handles=handles, labels=labels, loc="best") +ax.set_title("Extension of Precision-Recall curve to multi-class") plt.show() diff --git a/examples/model_selection/plot_randomized_search.py b/examples/model_selection/plot_randomized_search.py index ff5b51837ed2a..7acd3a5550acf 100644 --- a/examples/model_selection/plot_randomized_search.py +++ b/examples/model_selection/plot_randomized_search.py @@ -17,66 +17,79 @@ Note that in practice, one would not search over this many different parameters simultaneously using grid search, but pick only the ones deemed most important. + """ -print(__doc__) -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause from time import time + +import numpy as np import scipy.stats as stats -from sklearn.utils.fixes import loguniform -from sklearn.model_selection import GridSearchCV, RandomizedSearchCV from sklearn.datasets import load_digits from sklearn.linear_model import SGDClassifier +from sklearn.model_selection import GridSearchCV, RandomizedSearchCV # get some data -X, y = load_digits(return_X_y=True) +X, y = load_digits(return_X_y=True, n_class=3) # build a classifier -clf = SGDClassifier(loss='hinge', penalty='elasticnet', - fit_intercept=True) +clf = SGDClassifier(loss="hinge", penalty="elasticnet", fit_intercept=True) # Utility function to report best scores def report(results, n_top=3): for i in range(1, n_top + 1): - candidates = np.flatnonzero(results['rank_test_score'] == i) + candidates = np.flatnonzero(results["rank_test_score"] == i) for candidate in candidates: print("Model with rank: {0}".format(i)) - print("Mean validation score: {0:.3f} (std: {1:.3f})" - .format(results['mean_test_score'][candidate], - results['std_test_score'][candidate])) - print("Parameters: {0}".format(results['params'][candidate])) + print( + "Mean validation score: {0:.3f} (std: {1:.3f})".format( + results["mean_test_score"][candidate], + results["std_test_score"][candidate], + ) + ) + print("Parameters: {0}".format(results["params"][candidate])) print("") # specify parameters and distributions to sample from -param_dist = {'average': [True, False], - 'l1_ratio': stats.uniform(0, 1), - 'alpha': loguniform(1e-4, 1e0)} +param_dist = { + "average": [True, False], + "l1_ratio": stats.uniform(0, 1), + "alpha": stats.loguniform(1e-2, 1e0), +} # run randomized search -n_iter_search = 20 -random_search = RandomizedSearchCV(clf, param_distributions=param_dist, - n_iter=n_iter_search) +n_iter_search = 15 +random_search = RandomizedSearchCV( + clf, param_distributions=param_dist, n_iter=n_iter_search +) start = time() random_search.fit(X, y) -print("RandomizedSearchCV took %.2f seconds for %d candidates" - " parameter settings." % ((time() - start), n_iter_search)) +print( + "RandomizedSearchCV took %.2f seconds for %d candidates parameter settings." + % ((time() - start), n_iter_search) +) report(random_search.cv_results_) # use a full grid over all parameters -param_grid = {'average': [True, False], - 'l1_ratio': np.linspace(0, 1, num=10), - 'alpha': np.power(10, np.arange(-4, 1, dtype=float))} +param_grid = { + "average": [True, False], + "l1_ratio": np.linspace(0, 1, num=10), + "alpha": np.power(10, np.arange(-2, 1, dtype=float)), +} # run grid search grid_search = GridSearchCV(clf, param_grid=param_grid) start = time() grid_search.fit(X, y) -print("GridSearchCV took %.2f seconds for %d candidate parameter settings." - % (time() - start, len(grid_search.cv_results_['params']))) +print( + "GridSearchCV took %.2f seconds for %d candidate parameter settings." + % (time() - start, len(grid_search.cv_results_["params"])) +) report(grid_search.cv_results_) diff --git a/examples/model_selection/plot_roc.py b/examples/model_selection/plot_roc.py index 86ca06eb00445..9e659b9a2aa64 100644 --- a/examples/model_selection/plot_roc.py +++ b/examples/model_selection/plot_roc.py @@ -1,172 +1,463 @@ """ -======================================= -Receiver Operating Characteristic (ROC) -======================================= +================================================== +Multiclass Receiver Operating Characteristic (ROC) +================================================== -Example of Receiver Operating Characteristic (ROC) metric to evaluate -classifier output quality. +This example describes the use of the Receiver Operating Characteristic (ROC) +metric to evaluate the quality of multiclass classifiers. -ROC curves typically feature true positive rate on the Y axis, and false -positive rate on the X axis. This means that the top left corner of the plot is -the "ideal" point - a false positive rate of zero, and a true positive rate of -one. This is not very realistic, but it does mean that a larger area under the -curve (AUC) is usually better. +ROC curves typically feature true positive rate (TPR) on the Y axis, and false +positive rate (FPR) on the X axis. This means that the top left corner of the +plot is the "ideal" point - a FPR of zero, and a TPR of one. This is not very +realistic, but it does mean that a larger area under the curve (AUC) is usually +better. The "steepness" of ROC curves is also important, since it is ideal to +maximize the TPR while minimizing the FPR. -The "steepness" of ROC curves is also important, since it is ideal to maximize -the true positive rate while minimizing the false positive rate. +ROC curves are typically used in binary classification, where the TPR and FPR +can be defined unambiguously. In the case of multiclass classification, a notion +of TPR or FPR is obtained only after binarizing the output. This can be done in +2 different ways: -ROC curves are typically used in binary classification to study the output of -a classifier. In order to extend ROC curve and ROC area to multi-label -classification, it is necessary to binarize the output. One ROC -curve can be drawn per label, but one can also draw a ROC curve by considering -each element of the label indicator matrix as a binary prediction -(micro-averaging). +- the One-vs-Rest scheme compares each class against all the others (assumed as + one); +- the One-vs-One scheme compares every unique pairwise combination of classes. -Another evaluation measure for multi-label classification is -macro-averaging, which gives equal weight to the classification of each -label. +In this example we explore both schemes and demo the concepts of micro and macro +averaging as different ways of summarizing the information of the multiclass ROC +curves. .. note:: - See also :func:`sklearn.metrics.roc_auc_score`, - :ref:`sphx_glr_auto_examples_model_selection_plot_roc_crossval.py` - + See :ref:`sphx_glr_auto_examples_model_selection_plot_roc_crossval.py` for + an extension of the present example estimating the variance of the ROC + curves and their respective AUC. """ -print(__doc__) + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Load and prepare data +# ===================== +# +# We import the :ref:`iris_dataset` which contains 3 classes, each one +# corresponding to a type of iris plant. One class is linearly separable from +# the other 2; the latter are **not** linearly separable from each other. +# +# Here we binarize the output and add noisy features to make the problem harder. import numpy as np -import matplotlib.pyplot as plt -from itertools import cycle -from sklearn import svm, datasets -from sklearn.metrics import roc_curve, auc +from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split -from sklearn.preprocessing import label_binarize -from sklearn.multiclass import OneVsRestClassifier -from scipy import interp -from sklearn.metrics import roc_auc_score - -# Import some data to play with -iris = datasets.load_iris() -X = iris.data -y = iris.target -# Binarize the output -y = label_binarize(y, classes=[0, 1, 2]) -n_classes = y.shape[1] +iris = load_iris() +target_names = iris.target_names +X, y = iris.data, iris.target +y = iris.target_names[y] -# Add noisy features to make the problem harder random_state = np.random.RandomState(0) n_samples, n_features = X.shape -X = np.c_[X, random_state.randn(n_samples, 200 * n_features)] +n_classes = len(np.unique(y)) +X = np.concatenate([X, random_state.randn(n_samples, 200 * n_features)], axis=1) +( + X_train, + X_test, + y_train, + y_test, +) = train_test_split(X, y, test_size=0.5, stratify=y, random_state=0) -# shuffle and split training and test sets -X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.5, - random_state=0) +# %% +# We train a :class:`~sklearn.linear_model.LogisticRegression` model which can +# naturally handle multiclass problems, thanks to the use of the multinomial +# formulation. -# Learn to predict each class against the other -classifier = OneVsRestClassifier(svm.SVC(kernel='linear', probability=True, - random_state=random_state)) -y_score = classifier.fit(X_train, y_train).decision_function(X_test) +from sklearn.linear_model import LogisticRegression -# Compute ROC curve and ROC area for each class -fpr = dict() -tpr = dict() -roc_auc = dict() -for i in range(n_classes): - fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i]) - roc_auc[i] = auc(fpr[i], tpr[i]) +classifier = LogisticRegression() +y_score = classifier.fit(X_train, y_train).predict_proba(X_test) + +# %% +# One-vs-Rest multiclass ROC +# ========================== +# +# The One-vs-the-Rest (OvR) multiclass strategy, also known as one-vs-all, +# consists in computing a ROC curve per each of the `n_classes`. In each step, a +# given class is regarded as the positive class and the remaining classes are +# regarded as the negative class as a bulk. +# +# .. note:: One should not confuse the OvR strategy used for the **evaluation** +# of multiclass classifiers with the OvR strategy used to **train** a +# multiclass classifier by fitting a set of binary classifiers (for instance +# via the :class:`~sklearn.multiclass.OneVsRestClassifier` meta-estimator). +# The OvR ROC evaluation can be used to scrutinize any kind of classification +# models irrespectively of how they were trained (see :ref:`multiclass`). +# +# In this section we use a :class:`~sklearn.preprocessing.LabelBinarizer` to +# binarize the target by one-hot-encoding in a OvR fashion. This means that the +# target of shape (`n_samples`,) is mapped to a target of shape (`n_samples`, +# `n_classes`). + +from sklearn.preprocessing import LabelBinarizer + +label_binarizer = LabelBinarizer().fit(y_train) +y_onehot_test = label_binarizer.transform(y_test) +y_onehot_test.shape # (n_samples, n_classes) + +# %% +# We can as well easily check the encoding of a specific class: + +label_binarizer.transform(["virginica"]) + +# %% +# ROC curve showing a specific class +# ---------------------------------- +# +# In the following plot we show the resulting ROC curve when regarding the iris +# flowers as either "virginica" (`class_id=2`) or "non-virginica" (the rest). + +class_of_interest = "virginica" +class_id = np.flatnonzero(label_binarizer.classes_ == class_of_interest)[0] +class_id + +# %% +import matplotlib.pyplot as plt + +from sklearn.metrics import RocCurveDisplay + +display = RocCurveDisplay.from_predictions( + y_onehot_test[:, class_id], + y_score[:, class_id], + name=f"{class_of_interest} vs the rest", + curve_kwargs=dict(color="darkorange"), + plot_chance_level=True, + despine=True, +) +_ = display.ax_.set( + xlabel="False Positive Rate", + ylabel="True Positive Rate", + title="One-vs-Rest ROC curves:\nVirginica vs (Setosa & Versicolor)", +) + +# %% +# ROC curve using micro-averaged OvR +# ---------------------------------- +# +# Micro-averaging aggregates the contributions from all the classes (using +# :func:`numpy.ravel`) to compute the average metrics as follows: +# +# :math:`TPR=\frac{\sum_{c}TP_c}{\sum_{c}(TP_c + FN_c)}` ; +# +# :math:`FPR=\frac{\sum_{c}FP_c}{\sum_{c}(FP_c + TN_c)}` . +# +# We can briefly demo the effect of :func:`numpy.ravel`: + +print(f"y_score:\n{y_score[0:2, :]}") +print() +print(f"y_score.ravel():\n{y_score[0:2, :].ravel()}") + +# %% +# In a multi-class classification setup with highly imbalanced classes, +# micro-averaging is preferable over macro-averaging. In such cases, one can +# alternatively use a weighted macro-averaging, not demonstrated here. + +display = RocCurveDisplay.from_predictions( + y_onehot_test.ravel(), + y_score.ravel(), + name="micro-average OvR", + curve_kwargs=dict(color="darkorange"), + plot_chance_level=True, + despine=True, +) +_ = display.ax_.set( + xlabel="False Positive Rate", + ylabel="True Positive Rate", + title="Micro-averaged One-vs-Rest\nReceiver Operating Characteristic", +) + +# %% +# In the case where the main interest is not the plot but the ROC-AUC score +# itself, we can reproduce the value shown in the plot using +# :class:`~sklearn.metrics.roc_auc_score`. + +from sklearn.metrics import roc_auc_score + +micro_roc_auc_ovr = roc_auc_score( + y_test, + y_score, + multi_class="ovr", + average="micro", +) + +print(f"Micro-averaged One-vs-Rest ROC AUC score:\n{micro_roc_auc_ovr:.2f}") + +# %% +# This is equivalent to computing the ROC curve with +# :class:`~sklearn.metrics.roc_curve` and then the area under the curve with +# :class:`~sklearn.metrics.auc` for the raveled true and predicted classes. + +from sklearn.metrics import auc, roc_curve +# store the fpr, tpr, and roc_auc for all averaging strategies +fpr, tpr, roc_auc = dict(), dict(), dict() # Compute micro-average ROC curve and ROC area -fpr["micro"], tpr["micro"], _ = roc_curve(y_test.ravel(), y_score.ravel()) +fpr["micro"], tpr["micro"], _ = roc_curve(y_onehot_test.ravel(), y_score.ravel()) roc_auc["micro"] = auc(fpr["micro"], tpr["micro"]) +print(f"Micro-averaged One-vs-Rest ROC AUC score:\n{roc_auc['micro']:.2f}") # %% -# Plot of a ROC curve for a specific class -plt.figure() -lw = 2 -plt.plot(fpr[2], tpr[2], color='darkorange', - lw=lw, label='ROC curve (area = %0.2f)' % roc_auc[2]) -plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') -plt.xlim([0.0, 1.0]) -plt.ylim([0.0, 1.05]) -plt.xlabel('False Positive Rate') -plt.ylabel('True Positive Rate') -plt.title('Receiver operating characteristic example') -plt.legend(loc="lower right") -plt.show() +# .. note:: By default, the computation of the ROC curve adds a single point at +# the maximal false positive rate by using linear interpolation and the +# McClish correction [:doi:`Analyzing a portion of the ROC curve Med Decis +# Making. 1989 Jul-Sep; 9(3):190-5.<10.1177/0272989x8900900307>`]. +# +# ROC curve using the OvR macro-average +# ------------------------------------- +# +# Obtaining the macro-average requires computing the metric independently for +# each class and then taking the average over them, hence treating all classes +# equally a priori. We first aggregate the true/false positive rates per class: +# +# :math:`TPR=\frac{1}{C}\sum_{c}\frac{TP_c}{TP_c + FN_c}` ; +# +# :math:`FPR=\frac{1}{C}\sum_{c}\frac{FP_c}{FP_c + TN_c}` . +# +# where `C` is the total number of classes. +for i in range(n_classes): + fpr[i], tpr[i], _ = roc_curve(y_onehot_test[:, i], y_score[:, i]) + roc_auc[i] = auc(fpr[i], tpr[i]) -# %% -# Plot ROC curves for the multilabel problem -# .......................................... -# Compute macro-average ROC curve and ROC area +fpr_grid = np.linspace(0.0, 1.0, 1000) -# First aggregate all false positive rates -all_fpr = np.unique(np.concatenate([fpr[i] for i in range(n_classes)])) +# Interpolate all ROC curves at these points +mean_tpr = np.zeros_like(fpr_grid) -# Then interpolate all ROC curves at this points -mean_tpr = np.zeros_like(all_fpr) for i in range(n_classes): - mean_tpr += interp(all_fpr, fpr[i], tpr[i]) + mean_tpr += np.interp(fpr_grid, fpr[i], tpr[i]) # linear interpolation -# Finally average it and compute AUC +# Average it and compute AUC mean_tpr /= n_classes -fpr["macro"] = all_fpr +fpr["macro"] = fpr_grid tpr["macro"] = mean_tpr roc_auc["macro"] = auc(fpr["macro"], tpr["macro"]) -# Plot all ROC curves -plt.figure() -plt.plot(fpr["micro"], tpr["micro"], - label='micro-average ROC curve (area = {0:0.2f})' - ''.format(roc_auc["micro"]), - color='deeppink', linestyle=':', linewidth=4) - -plt.plot(fpr["macro"], tpr["macro"], - label='macro-average ROC curve (area = {0:0.2f})' - ''.format(roc_auc["macro"]), - color='navy', linestyle=':', linewidth=4) - -colors = cycle(['aqua', 'darkorange', 'cornflowerblue']) -for i, color in zip(range(n_classes), colors): - plt.plot(fpr[i], tpr[i], color=color, lw=lw, - label='ROC curve of class {0} (area = {1:0.2f})' - ''.format(i, roc_auc[i])) - -plt.plot([0, 1], [0, 1], 'k--', lw=lw) -plt.xlim([0.0, 1.0]) -plt.ylim([0.0, 1.05]) -plt.xlabel('False Positive Rate') -plt.ylabel('True Positive Rate') -plt.title('Some extension of Receiver operating characteristic to multi-class') -plt.legend(loc="lower right") -plt.show() - - -# %% -# Area under ROC for the multiclass problem -# ......................................... -# The :func:`sklearn.metrics.roc_auc_score` function can be used for -# multi-class classification. The multi-class One-vs-One scheme compares every -# unique pairwise combination of classes. In this section, we calculate the AUC -# using the OvR and OvO schemes. We report a macro average, and a -# prevalence-weighted average. -y_prob = classifier.predict_proba(X_test) - -macro_roc_auc_ovo = roc_auc_score(y_test, y_prob, multi_class="ovo", - average="macro") -weighted_roc_auc_ovo = roc_auc_score(y_test, y_prob, multi_class="ovo", - average="weighted") -macro_roc_auc_ovr = roc_auc_score(y_test, y_prob, multi_class="ovr", - average="macro") -weighted_roc_auc_ovr = roc_auc_score(y_test, y_prob, multi_class="ovr", - average="weighted") -print("One-vs-One ROC AUC scores:\n{:.6f} (macro),\n{:.6f} " - "(weighted by prevalence)" - .format(macro_roc_auc_ovo, weighted_roc_auc_ovo)) -print("One-vs-Rest ROC AUC scores:\n{:.6f} (macro),\n{:.6f} " - "(weighted by prevalence)" - .format(macro_roc_auc_ovr, weighted_roc_auc_ovr)) +print(f"Macro-averaged One-vs-Rest ROC AUC score:\n{roc_auc['macro']:.2f}") + +# %% +# This computation is equivalent to simply calling + +macro_roc_auc_ovr = roc_auc_score( + y_test, + y_score, + multi_class="ovr", + average="macro", +) + +print(f"Macro-averaged One-vs-Rest ROC AUC score:\n{macro_roc_auc_ovr:.2f}") + +# %% +# Plot all OvR ROC curves together +# -------------------------------- + +from itertools import cycle + +fig, ax = plt.subplots(figsize=(6, 6)) + +plt.plot( + fpr["micro"], + tpr["micro"], + label=f"micro-average ROC curve (AUC = {roc_auc['micro']:.2f})", + color="deeppink", + linestyle=":", + linewidth=4, +) + +plt.plot( + fpr["macro"], + tpr["macro"], + label=f"macro-average ROC curve (AUC = {roc_auc['macro']:.2f})", + color="navy", + linestyle=":", + linewidth=4, +) + +colors = cycle(["aqua", "darkorange", "cornflowerblue"]) +for class_id, color in zip(range(n_classes), colors): + RocCurveDisplay.from_predictions( + y_onehot_test[:, class_id], + y_score[:, class_id], + name=f"ROC curve for {target_names[class_id]}", + curve_kwargs=dict(color=color), + ax=ax, + plot_chance_level=(class_id == 2), + despine=True, + ) + +_ = ax.set( + xlabel="False Positive Rate", + ylabel="True Positive Rate", + title="Extension of Receiver Operating Characteristic\nto One-vs-Rest multiclass", +) + +# %% +# One-vs-One multiclass ROC +# ========================= +# +# The One-vs-One (OvO) multiclass strategy consists in fitting one classifier +# per class pair. Since it requires to train `n_classes` * (`n_classes` - 1) / 2 +# classifiers, this method is usually slower than One-vs-Rest due to its +# O(`n_classes` ^2) complexity. +# +# In this section, we demonstrate the macro-averaged AUC using the OvO scheme +# for the 3 possible combinations in the :ref:`iris_dataset`: "setosa" vs +# "versicolor", "versicolor" vs "virginica" and "virginica" vs "setosa". Notice +# that micro-averaging is not defined for the OvO scheme. +# +# ROC curve using the OvO macro-average +# ------------------------------------- +# +# In the OvO scheme, the first step is to identify all possible unique +# combinations of pairs. The computation of scores is done by treating one of +# the elements in a given pair as the positive class and the other element as +# the negative class, then re-computing the score by inversing the roles and +# taking the mean of both scores. + +from itertools import combinations + +pair_list = list(combinations(np.unique(y), 2)) +print(pair_list) + +# %% +pair_scores = [] +mean_tpr = dict() + +for ix, (label_a, label_b) in enumerate(pair_list): + a_mask = y_test == label_a + b_mask = y_test == label_b + ab_mask = np.logical_or(a_mask, b_mask) + + a_true = a_mask[ab_mask] + b_true = b_mask[ab_mask] + + idx_a = np.flatnonzero(label_binarizer.classes_ == label_a)[0] + idx_b = np.flatnonzero(label_binarizer.classes_ == label_b)[0] + + fpr_a, tpr_a, _ = roc_curve(a_true, y_score[ab_mask, idx_a]) + fpr_b, tpr_b, _ = roc_curve(b_true, y_score[ab_mask, idx_b]) + + mean_tpr[ix] = np.zeros_like(fpr_grid) + mean_tpr[ix] += np.interp(fpr_grid, fpr_a, tpr_a) + mean_tpr[ix] += np.interp(fpr_grid, fpr_b, tpr_b) + mean_tpr[ix] /= 2 + mean_score = auc(fpr_grid, mean_tpr[ix]) + pair_scores.append(mean_score) + + fig, ax = plt.subplots(figsize=(6, 6)) + plt.plot( + fpr_grid, + mean_tpr[ix], + label=f"Mean {label_a} vs {label_b} (AUC = {mean_score:.2f})", + linestyle=":", + linewidth=4, + ) + RocCurveDisplay.from_predictions( + a_true, + y_score[ab_mask, idx_a], + ax=ax, + name=f"{label_a} as positive class", + ) + RocCurveDisplay.from_predictions( + b_true, + y_score[ab_mask, idx_b], + ax=ax, + name=f"{label_b} as positive class", + plot_chance_level=True, + despine=True, + ) + ax.set( + xlabel="False Positive Rate", + ylabel="True Positive Rate", + title=f"{target_names[idx_a]} vs {label_b} ROC curves", + ) + +print(f"Macro-averaged One-vs-One ROC AUC score:\n{np.average(pair_scores):.2f}") + +# %% +# One can also assert that the macro-average we computed "by hand" is equivalent +# to the implemented `average="macro"` option of the +# :class:`~sklearn.metrics.roc_auc_score` function. + +macro_roc_auc_ovo = roc_auc_score( + y_test, + y_score, + multi_class="ovo", + average="macro", +) + +print(f"Macro-averaged One-vs-One ROC AUC score:\n{macro_roc_auc_ovo:.2f}") + +# %% +# Plot all OvO ROC curves together +# -------------------------------- + +ovo_tpr = np.zeros_like(fpr_grid) + +fig, ax = plt.subplots(figsize=(6, 6)) +for ix, (label_a, label_b) in enumerate(pair_list): + ovo_tpr += mean_tpr[ix] + ax.plot( + fpr_grid, + mean_tpr[ix], + label=f"Mean {label_a} vs {label_b} (AUC = {pair_scores[ix]:.2f})", + ) + +ovo_tpr /= sum(1 for pair in enumerate(pair_list)) + +ax.plot( + fpr_grid, + ovo_tpr, + label=f"One-vs-One macro-average (AUC = {macro_roc_auc_ovo:.2f})", + linestyle=":", + linewidth=4, +) +ax.plot([0, 1], [0, 1], "k--", label="Chance level (AUC = 0.5)") +_ = ax.set( + xlabel="False Positive Rate", + ylabel="True Positive Rate", + title="Extension of Receiver Operating Characteristic\nto One-vs-One multiclass", + aspect="equal", + xlim=(-0.01, 1.01), + ylim=(-0.01, 1.01), +) + +# %% +# We confirm that the classes "versicolor" and "virginica" are not well +# identified by a linear classifier. Notice that the "virginica"-vs-the-rest +# ROC-AUC score (0.77) is between the OvO ROC-AUC scores for "versicolor" vs +# "virginica" (0.64) and "setosa" vs "virginica" (0.90). Indeed, the OvO +# strategy gives additional information on the confusion between a pair of +# classes, at the expense of computational cost when the number of classes +# is large. +# +# The OvO strategy is recommended if the user is mainly interested in correctly +# identifying a particular class or subset of classes, whereas evaluating the +# global performance of a classifier can still be summarized via a given +# averaging strategy. +# +# When dealing with imbalanced datasets, choosing the appropriate metric based on +# the business context or problem you are addressing is crucial. +# It is also essential to select an appropriate averaging method (micro vs. macro) +# depending on the desired outcome: +# +# - Micro-averaging aggregates metrics across all instances, treating each +# individual instance equally, regardless of its class. This approach is useful +# when evaluating overall performance, but note that it can be dominated by +# the majority class in imbalanced datasets. +# +# - Macro-averaging calculates metrics for each class independently and then +# averages them, giving equal weight to each class. This is particularly useful +# when you want under-represented classes to be considered as important as highly +# populated classes. diff --git a/examples/model_selection/plot_roc_crossval.py b/examples/model_selection/plot_roc_crossval.py index 24633f10b93b3..3c5c3fc9119b7 100644 --- a/examples/model_selection/plot_roc_crossval.py +++ b/examples/model_selection/plot_roc_crossval.py @@ -3,97 +3,140 @@ Receiver Operating Characteristic (ROC) with cross validation ============================================================= -Example of Receiver Operating Characteristic (ROC) metric to evaluate -classifier output quality using cross-validation. +This example presents how to estimate and visualize the variance of the Receiver +Operating Characteristic (ROC) metric using cross-validation. -ROC curves typically feature true positive rate on the Y axis, and false -positive rate on the X axis. This means that the top left corner of the plot is -the "ideal" point - a false positive rate of zero, and a true positive rate of -one. This is not very realistic, but it does mean that a larger area under the -curve (AUC) is usually better. - -The "steepness" of ROC curves is also important, since it is ideal to maximize -the true positive rate while minimizing the false positive rate. +ROC curves typically feature true positive rate (TPR) on the Y axis, and false +positive rate (FPR) on the X axis. This means that the top left corner of the +plot is the "ideal" point - a FPR of zero, and a TPR of one. This is not very +realistic, but it does mean that a larger Area Under the Curve (AUC) is usually +better. The "steepness" of ROC curves is also important, since it is ideal to +maximize the TPR while minimizing the FPR. This example shows the ROC response of different datasets, created from K-fold cross-validation. Taking all of these curves, it is possible to calculate the -mean area under curve, and see the variance of the curve when the +mean AUC, and see the variance of the curve when the training set is split into different subsets. This roughly shows how the -classifier output is affected by changes in the training data, and how -different the splits generated by K-fold cross-validation are from one another. +classifier output is affected by changes in the training data, and how different +the splits generated by K-fold cross-validation are from one another. .. note:: - See also :func:`sklearn.metrics.roc_auc_score`, - :func:`sklearn.model_selection.cross_val_score`, - :ref:`sphx_glr_auto_examples_model_selection_plot_roc.py`, - + See :ref:`sphx_glr_auto_examples_model_selection_plot_roc.py` for a + complement of the present example explaining the averaging strategies to + generalize the metrics for multiclass classifiers. """ -print(__doc__) -import numpy as np -import matplotlib.pyplot as plt +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Load and prepare data +# ===================== +# +# We import the :ref:`iris_dataset` which contains 3 classes, each one +# corresponding to a type of iris plant. One class is linearly separable from +# the other 2; the latter are **not** linearly separable from each other. +# +# In the following we binarize the dataset by dropping the "virginica" class +# (`class_id=2`). This means that the "versicolor" class (`class_id=1`) is +# regarded as the positive class and "setosa" as the negative class +# (`class_id=0`). -from sklearn import svm, datasets -from sklearn.metrics import auc -from sklearn.metrics import plot_roc_curve -from sklearn.model_selection import StratifiedKFold +import numpy as np -# ############################################################################# -# Data IO and generation +from sklearn.datasets import load_iris -# Import some data to play with -iris = datasets.load_iris() -X = iris.data -y = iris.target +iris = load_iris() +target_names = iris.target_names +X, y = iris.data, iris.target X, y = X[y != 2], y[y != 2] n_samples, n_features = X.shape -# Add noisy features +# %% +# We also add noisy features to make the problem harder. random_state = np.random.RandomState(0) -X = np.c_[X, random_state.randn(n_samples, 200 * n_features)] +X = np.concatenate([X, random_state.randn(n_samples, 200 * n_features)], axis=1) -# ############################################################################# +# %% # Classification and ROC analysis +# ------------------------------- +# +# Here we run :func:`~sklearn.model_selection.cross_validate` on a +# :class:`~sklearn.svm.SVC` classifier, then use the computed cross-validation results +# to plot the ROC curves fold-wise. Notice that the baseline to define the chance +# level (dashed ROC curve) is a classifier that would always predict the most +# frequent class. -# Run classifier with cross-validation and plot ROC curves -cv = StratifiedKFold(n_splits=6) -classifier = svm.SVC(kernel='linear', probability=True, - random_state=random_state) +import matplotlib.pyplot as plt -tprs = [] -aucs = [] -mean_fpr = np.linspace(0, 1, 100) +from sklearn import svm +from sklearn.metrics import RocCurveDisplay, auc +from sklearn.model_selection import StratifiedKFold, cross_validate -fig, ax = plt.subplots() -for i, (train, test) in enumerate(cv.split(X, y)): - classifier.fit(X[train], y[train]) - viz = plot_roc_curve(classifier, X[test], y[test], - name='ROC fold {}'.format(i), - alpha=0.3, lw=1, ax=ax) - interp_tpr = np.interp(mean_fpr, viz.fpr, viz.tpr) - interp_tpr[0] = 0.0 - tprs.append(interp_tpr) - aucs.append(viz.roc_auc) +n_splits = 6 +cv = StratifiedKFold(n_splits=n_splits) +classifier = svm.SVC(kernel="linear", probability=True, random_state=random_state) +cv_results = cross_validate( + classifier, X, y, cv=cv, return_estimator=True, return_indices=True +) + +prop_cycle = plt.rcParams["axes.prop_cycle"] +colors = prop_cycle.by_key()["color"] +curve_kwargs_list = [ + dict(alpha=0.3, lw=1, color=colors[fold % len(colors)]) for fold in range(n_splits) +] +names = [f"ROC fold {idx}" for idx in range(n_splits)] -ax.plot([0, 1], [0, 1], linestyle='--', lw=2, color='r', - label='Chance', alpha=.8) +mean_fpr = np.linspace(0, 1, 100) +interp_tprs = [] + +_, ax = plt.subplots(figsize=(6, 6)) +viz = RocCurveDisplay.from_cv_results( + cv_results, + X, + y, + ax=ax, + name=names, + curve_kwargs=curve_kwargs_list, + plot_chance_level=True, +) + +for idx in range(n_splits): + interp_tpr = np.interp(mean_fpr, viz.fpr[idx], viz.tpr[idx]) + interp_tpr[0] = 0.0 + interp_tprs.append(interp_tpr) -mean_tpr = np.mean(tprs, axis=0) +mean_tpr = np.mean(interp_tprs, axis=0) mean_tpr[-1] = 1.0 mean_auc = auc(mean_fpr, mean_tpr) -std_auc = np.std(aucs) -ax.plot(mean_fpr, mean_tpr, color='b', - label=r'Mean ROC (AUC = %0.2f $\pm$ %0.2f)' % (mean_auc, std_auc), - lw=2, alpha=.8) - -std_tpr = np.std(tprs, axis=0) +std_auc = np.std(viz.roc_auc) + +ax.plot( + mean_fpr, + mean_tpr, + color="b", + label=r"Mean ROC (AUC = %0.2f $\pm$ %0.2f)" % (mean_auc, std_auc), + lw=2, + alpha=0.8, +) + +std_tpr = np.std(interp_tprs, axis=0) tprs_upper = np.minimum(mean_tpr + std_tpr, 1) tprs_lower = np.maximum(mean_tpr - std_tpr, 0) -ax.fill_between(mean_fpr, tprs_lower, tprs_upper, color='grey', alpha=.2, - label=r'$\pm$ 1 std. dev.') - -ax.set(xlim=[-0.05, 1.05], ylim=[-0.05, 1.05], - title="Receiver operating characteristic example") +ax.fill_between( + mean_fpr, + tprs_lower, + tprs_upper, + color="grey", + alpha=0.2, + label=r"$\pm$ 1 std. dev.", +) + +ax.set( + xlabel="False Positive Rate", + ylabel="True Positive Rate", + title=f"Mean ROC curve with variability\n(Positive label '{target_names[1]}')", +) ax.legend(loc="lower right") plt.show() diff --git a/examples/model_selection/plot_successive_halving_heatmap.py b/examples/model_selection/plot_successive_halving_heatmap.py index 04a2d19df89f7..c46068532e52e 100644 --- a/examples/model_selection/plot_successive_halving_heatmap.py +++ b/examples/model_selection/plot_successive_halving_heatmap.py @@ -7,20 +7,20 @@ :class:`~sklearn.model_selection.GridSearchCV`. """ + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + from time import time import matplotlib.pyplot as plt import numpy as np import pandas as pd -from sklearn.svm import SVC from sklearn import datasets -from sklearn.model_selection import GridSearchCV -from sklearn.experimental import enable_halving_search_cv # noqa -from sklearn.model_selection import HalvingGridSearchCV - - -print(__doc__) +from sklearn.experimental import enable_halving_search_cv # noqa: F401 +from sklearn.model_selection import GridSearchCV, HalvingGridSearchCV +from sklearn.svm import SVC # %% # We first define the parameter space for an :class:`~sklearn.svm.SVC` @@ -33,13 +33,14 @@ gammas = [1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-6, 1e-7] Cs = [1, 10, 100, 1e3, 1e4, 1e5] -param_grid = {'gamma': gammas, 'C': Cs} +param_grid = {"gamma": gammas, "C": Cs} clf = SVC(random_state=rng) tic = time() -gsh = HalvingGridSearchCV(estimator=clf, param_grid=param_grid, factor=2, - random_state=rng) +gsh = HalvingGridSearchCV( + estimator=clf, param_grid=param_grid, factor=2, random_state=rng +) gsh.fit(X, y) gsh_time = time() - tic @@ -54,47 +55,57 @@ def make_heatmap(ax, gs, is_sh=False, make_cbar=False): """Helper to make a heatmap.""" - results = pd.DataFrame.from_dict(gs.cv_results_) - results['params_str'] = results.params.apply(str) + results = pd.DataFrame(gs.cv_results_) + results[["param_C", "param_gamma"]] = results[["param_C", "param_gamma"]].astype( + np.float64 + ) if is_sh: # SH dataframe: get mean_test_score values for the highest iter - scores_matrix = results.sort_values('iter').pivot_table( - index='param_gamma', columns='param_C', - values='mean_test_score', aggfunc='last' + scores_matrix = results.sort_values("iter").pivot_table( + index="param_gamma", + columns="param_C", + values="mean_test_score", + aggfunc="last", ) else: - scores_matrix = results.pivot(index='param_gamma', columns='param_C', - values='mean_test_score') + scores_matrix = results.pivot( + index="param_gamma", columns="param_C", values="mean_test_score" + ) im = ax.imshow(scores_matrix) ax.set_xticks(np.arange(len(Cs))) - ax.set_xticklabels(['{:.0E}'.format(x) for x in Cs]) - ax.set_xlabel('C', fontsize=15) + ax.set_xticklabels(["{:.0E}".format(x) for x in Cs]) + ax.set_xlabel("C", fontsize=15) ax.set_yticks(np.arange(len(gammas))) - ax.set_yticklabels(['{:.0E}'.format(x) for x in gammas]) - ax.set_ylabel('gamma', fontsize=15) + ax.set_yticklabels(["{:.0E}".format(x) for x in gammas]) + ax.set_ylabel("gamma", fontsize=15) # Rotate the tick labels and set their alignment. - plt.setp(ax.get_xticklabels(), rotation=45, ha="right", - rotation_mode="anchor") + plt.setp(ax.get_xticklabels(), rotation=45, ha="right", rotation_mode="anchor") if is_sh: - iterations = results.pivot_table(index='param_gamma', - columns='param_C', values='iter', - aggfunc='max').values + iterations = results.pivot_table( + index="param_gamma", columns="param_C", values="iter", aggfunc="max" + ).values for i in range(len(gammas)): for j in range(len(Cs)): - ax.text(j, i, iterations[i, j], - ha="center", va="center", color="w", fontsize=20) + ax.text( + j, + i, + iterations[i, j], + ha="center", + va="center", + color="w", + fontsize=20, + ) if make_cbar: fig.subplots_adjust(right=0.8) cbar_ax = fig.add_axes([0.85, 0.15, 0.05, 0.7]) fig.colorbar(im, cax=cbar_ax) - cbar_ax.set_ylabel('mean_test_score', rotation=-90, va="bottom", - fontsize=15) + cbar_ax.set_ylabel("mean_test_score", rotation=-90, va="bottom", fontsize=15) fig, axes = plt.subplots(ncols=2, sharey=True) @@ -103,9 +114,8 @@ def make_heatmap(ax, gs, is_sh=False, make_cbar=False): make_heatmap(ax1, gsh, is_sh=True) make_heatmap(ax2, gs, make_cbar=True) -ax1.set_title('Successive Halving\ntime = {:.3f}s'.format(gsh_time), - fontsize=15) -ax2.set_title('GridSearch\ntime = {:.3f}s'.format(gs_time), fontsize=15) +ax1.set_title("Successive Halving\ntime = {:.3f}s".format(gsh_time), fontsize=15) +ax2.set_title("GridSearch\ntime = {:.3f}s".format(gs_time), fontsize=15) plt.show() diff --git a/examples/model_selection/plot_successive_halving_iterations.py b/examples/model_selection/plot_successive_halving_iterations.py index 53d33849e9801..986be49ac0bef 100644 --- a/examples/model_selection/plot_successive_halving_iterations.py +++ b/examples/model_selection/plot_successive_halving_iterations.py @@ -9,18 +9,19 @@ multiple candidates. """ -import pandas as pd -from sklearn import datasets + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt -from scipy.stats import randint import numpy as np +import pandas as pd +from scipy.stats import randint -from sklearn.experimental import enable_halving_search_cv # noqa -from sklearn.model_selection import HalvingRandomSearchCV +from sklearn import datasets from sklearn.ensemble import RandomForestClassifier - - -print(__doc__) +from sklearn.experimental import enable_halving_search_cv # noqa: F401 +from sklearn.model_selection import HalvingRandomSearchCV # %% # We first define the parameter space and train a @@ -28,21 +29,21 @@ rng = np.random.RandomState(0) -X, y = datasets.make_classification(n_samples=700, random_state=rng) +X, y = datasets.make_classification(n_samples=400, n_features=12, random_state=rng) clf = RandomForestClassifier(n_estimators=20, random_state=rng) -param_dist = {"max_depth": [3, None], - "max_features": randint(1, 11), - "min_samples_split": randint(2, 11), - "bootstrap": [True, False], - "criterion": ["gini", "entropy"]} +param_dist = { + "max_depth": [3, None], + "max_features": randint(1, 6), + "min_samples_split": randint(2, 11), + "bootstrap": [True, False], + "criterion": ["gini", "entropy"], +} rsh = HalvingRandomSearchCV( - estimator=clf, - param_distributions=param_dist, - factor=2, - random_state=rng) + estimator=clf, param_distributions=param_dist, factor=2, random_state=rng +) rsh.fit(X, y) # %% @@ -50,23 +51,23 @@ # and plot the evolution of the search. results = pd.DataFrame(rsh.cv_results_) -results['params_str'] = results.params.apply(str) -results.drop_duplicates(subset=('params_str', 'iter'), inplace=True) -mean_scores = results.pivot(index='iter', columns='params_str', - values='mean_test_score') -ax = mean_scores.plot(legend=False, alpha=.6) +results["params_str"] = results.params.apply(str) +results.drop_duplicates(subset=("params_str", "iter"), inplace=True) +mean_scores = results.pivot( + index="iter", columns="params_str", values="mean_test_score" +) +ax = mean_scores.plot(legend=False, alpha=0.6) labels = [ - f'iter={i}\nn_samples={rsh.n_resources_[i]}\n' - f'n_candidates={rsh.n_candidates_[i]}' + f"iter={i}\nn_samples={rsh.n_resources_[i]}\nn_candidates={rsh.n_candidates_[i]}" for i in range(rsh.n_iterations_) ] ax.set_xticks(range(rsh.n_iterations_)) -ax.set_xticklabels(labels, rotation=45, multialignment='left') -ax.set_title('Scores of candidates over iterations') -ax.set_ylabel('mean test score', fontsize=15) -ax.set_xlabel('iterations', fontsize=15) +ax.set_xticklabels(labels, rotation=45, multialignment="left") +ax.set_title("Scores of candidates over iterations") +ax.set_ylabel("mean test score", fontsize=15) +ax.set_xlabel("iterations", fontsize=15) plt.tight_layout() plt.show() diff --git a/examples/model_selection/plot_train_error_vs_test_error.py b/examples/model_selection/plot_train_error_vs_test_error.py index f32d6e46f9933..a64b4ca94846e 100644 --- a/examples/model_selection/plot_train_error_vs_test_error.py +++ b/examples/model_selection/plot_train_error_vs_test_error.py @@ -1,75 +1,166 @@ """ -========================= -Train error vs Test error -========================= +========================================================= +Effect of model regularization on training and test error +========================================================= -Illustration of how the performance of an estimator on unseen data (test data) -is not the same as the performance on training data. As the regularization -increases the performance on train decreases while the performance on test -is optimal within a range of values of the regularization parameter. -The example with an Elastic-Net regression model and the performance is -measured using the explained variance a.k.a. R^2. +In this example, we evaluate the impact of the regularization parameter in a +linear model called :class:`~sklearn.linear_model.ElasticNet`. To carry out this +evaluation, we use a validation curve using +:class:`~sklearn.model_selection.ValidationCurveDisplay`. This curve shows the +training and test scores of the model for different values of the regularization +parameter. +Once we identify the optimal regularization parameter, we compare the true and +estimated coefficients of the model to determine if the model is able to recover +the coefficients from the noisy input data. """ -print(__doc__) -# Author: Alexandre Gramfort -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +# %% +# Generate sample data +# -------------------- +# +# We generate a regression dataset that contains many features relative to the +# number of samples. However, only 10% of the features are informative. In this context, +# linear models exposing L1 penalization are commonly used to recover a sparse +# set of coefficients. +from sklearn.datasets import make_regression +from sklearn.model_selection import train_test_split + +n_samples_train, n_samples_test, n_features = 150, 300, 500 +X, y, true_coef = make_regression( + n_samples=n_samples_train + n_samples_test, + n_features=n_features, + n_informative=50, + shuffle=False, + noise=1.0, + coef=True, + random_state=42, +) +X_train, X_test, y_train, y_test = train_test_split( + X, y, train_size=n_samples_train, test_size=n_samples_test, shuffle=False +) + +# %% +# Model definition +# ---------------- +# +# Here, we do not use a model that only exposes an L1 penalty. Instead, we use +# an :class:`~sklearn.linear_model.ElasticNet` model that exposes both L1 and L2 +# penalties. +# +# We fix the `l1_ratio` parameter such that the solution found by the model is still +# sparse. Therefore, this type of model tries to find a sparse solution but at the same +# time also tries to shrink all coefficients towards zero. +# +# In addition, we force the coefficients of the model to be positive since we know that +# `make_regression` generates a response with a positive signal. So we use this +# pre-knowledge to get a better model. + +from sklearn.linear_model import ElasticNet + +enet = ElasticNet(l1_ratio=0.9, positive=True, max_iter=10_000) + + +# %% +# Evaluate the impact of the regularization parameter +# --------------------------------------------------- +# +# To evaluate the impact of the regularization parameter, we use a validation +# curve. This curve shows the training and test scores of the model for different +# values of the regularization parameter. +# +# The regularization `alpha` is a parameter applied to the coefficients of the model: +# when it tends to zero, no regularization is applied and the model tries to fit the +# training data with the least amount of error. However, it leads to overfitting when +# features are noisy. When `alpha` increases, the model coefficients are constrained, +# and thus the model cannot fit the training data as closely, avoiding overfitting. +# However, if too much regularization is applied, the model underfits the data and +# is not able to properly capture the signal. +# +# The validation curve helps in finding a good trade-off between both extremes: the +# model is not regularized and thus flexible enough to fit the signal, but not too +# flexible to overfit. The :class:`~sklearn.model_selection.ValidationCurveDisplay` +# allows us to display the training and validation scores across a range of alpha +# values. import numpy as np -from sklearn import linear_model -# ############################################################################# -# Generate sample data -n_samples_train, n_samples_test, n_features = 75, 150, 500 -np.random.seed(0) -coef = np.random.randn(n_features) -coef[50:] = 0.0 # only the top 10 features are impacting the model -X = np.random.randn(n_samples_train + n_samples_test, n_features) -y = np.dot(X, coef) - -# Split train and test data -X_train, X_test = X[:n_samples_train], X[n_samples_train:] -y_train, y_test = y[:n_samples_train], y[n_samples_train:] - -# ############################################################################# -# Compute train and test errors +from sklearn.model_selection import ValidationCurveDisplay + alphas = np.logspace(-5, 1, 60) -enet = linear_model.ElasticNet(l1_ratio=0.7, max_iter=10000) -train_errors = list() -test_errors = list() -for alpha in alphas: - enet.set_params(alpha=alpha) - enet.fit(X_train, y_train) - train_errors.append(enet.score(X_train, y_train)) - test_errors.append(enet.score(X_test, y_test)) - -i_alpha_optim = np.argmax(test_errors) -alpha_optim = alphas[i_alpha_optim] -print("Optimal regularization parameter : %s" % alpha_optim) - -# Estimate the coef_ on full data with optimal regularization parameter -enet.set_params(alpha=alpha_optim) -coef_ = enet.fit(X, y).coef_ - -# ############################################################################# -# Plot results functions +disp = ValidationCurveDisplay.from_estimator( + enet, + X_train, + y_train, + param_name="alpha", + param_range=alphas, + scoring="r2", + n_jobs=2, + score_type="both", +) +disp.ax_.set( + title=r"Validation Curve for ElasticNet (R$^2$ Score)", + xlabel=r"alpha (regularization strength)", + ylabel="R$^2$ Score", +) + +test_scores_mean = disp.test_scores.mean(axis=1) +idx_avg_max_test_score = np.argmax(test_scores_mean) +disp.ax_.vlines( + alphas[idx_avg_max_test_score], + disp.ax_.get_ylim()[0], + test_scores_mean[idx_avg_max_test_score], + color="k", + linewidth=2, + linestyle="--", + label=f"Optimum on test\n$\\alpha$ = {alphas[idx_avg_max_test_score]:.2e}", +) +_ = disp.ax_.legend(loc="lower right") + +# %% +# To find the optimal regularization parameter, we can select the value of `alpha` +# that maximizes the validation score. +# +# Coefficients comparison +# ----------------------- +# +# Now that we have identified the optimal regularization parameter, we can compare the +# true coefficients and the estimated coefficients. +# +# First, let's set the regularization parameter to the optimal value and fit the +# model on the training data. In addition, we'll show the test score for this model. +enet.set_params(alpha=alphas[idx_avg_max_test_score]).fit(X_train, y_train) +print( + f"Test score: {enet.score(X_test, y_test):.3f}", +) +# %% +# Now, we plot the true coefficients and the estimated coefficients. import matplotlib.pyplot as plt -plt.subplot(2, 1, 1) -plt.semilogx(alphas, train_errors, label='Train') -plt.semilogx(alphas, test_errors, label='Test') -plt.vlines(alpha_optim, plt.ylim()[0], np.max(test_errors), color='k', - linewidth=3, label='Optimum on test') -plt.legend(loc='lower left') -plt.ylim([0, 1.2]) -plt.xlabel('Regularization parameter') -plt.ylabel('Performance') - -# Show estimated coef_ vs true coef -plt.subplot(2, 1, 2) -plt.plot(coef, label='True coef') -plt.plot(coef_, label='Estimated coef') -plt.legend() -plt.subplots_adjust(0.09, 0.04, 0.94, 0.94, 0.26, 0.26) + +fig, axs = plt.subplots(ncols=2, figsize=(12, 6), sharex=True, sharey=True) +for ax, coef, title in zip(axs, [true_coef, enet.coef_], ["True", "Model"]): + ax.stem(coef) + ax.set( + title=f"{title} Coefficients", + xlabel="Feature Index", + ylabel="Coefficient Value", + ) +fig.suptitle( + "Comparison of the coefficients of the true generative model and \n" + "the estimated elastic net coefficients" +) + plt.show() + +# %% +# While the original coefficients are sparse, the estimated coefficients are not +# as sparse. The reason is that we fixed the `l1_ratio` parameter to 0.9. We could +# force the model to get a sparser solution by increasing the `l1_ratio` parameter. +# +# However, we observed that for the estimated coefficients that are close to zero in +# the true generative model, our model shrinks them towards zero. So we don't recover +# the true coefficients, but we get a sensible outcome in line with the performance +# obtained on the test set. diff --git a/examples/model_selection/plot_tuned_decision_threshold.py b/examples/model_selection/plot_tuned_decision_threshold.py new file mode 100644 index 0000000000000..59986a3910d00 --- /dev/null +++ b/examples/model_selection/plot_tuned_decision_threshold.py @@ -0,0 +1,187 @@ +""" +====================================================== +Post-hoc tuning the cut-off point of decision function +====================================================== + +Once a binary classifier is trained, the :term:`predict` method outputs class label +predictions corresponding to a thresholding of either the :term:`decision_function` or +the :term:`predict_proba` output. The default threshold is defined as a posterior +probability estimate of 0.5 or a decision score of 0.0. However, this default strategy +may not be optimal for the task at hand. + +This example shows how to use the +:class:`~sklearn.model_selection.TunedThresholdClassifierCV` to tune the decision +threshold, depending on a metric of interest. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# The diabetes dataset +# -------------------- +# +# To illustrate the tuning of the decision threshold, we will use the diabetes dataset. +# This dataset is available on OpenML: https://www.openml.org/d/37. We use the +# :func:`~sklearn.datasets.fetch_openml` function to fetch this dataset. +from sklearn.datasets import fetch_openml + +diabetes = fetch_openml(data_id=37, as_frame=True, parser="pandas") +data, target = diabetes.data, diabetes.target + +# %% +# We look at the target to understand the type of problem we are dealing with. +target.value_counts() + +# %% +# We can see that we are dealing with a binary classification problem. Since the +# labels are not encoded as 0 and 1, we make it explicit that we consider the class +# labeled "tested_negative" as the negative class (which is also the most frequent) +# and the class labeled "tested_positive" the positive as the positive class: +neg_label, pos_label = target.value_counts().index + +# %% +# We can also observe that this binary problem is slightly imbalanced where we have +# around twice more samples from the negative class than from the positive class. When +# it comes to evaluation, we should consider this aspect to interpret the results. +# +# Our vanilla classifier +# ---------------------- +# +# We define a basic predictive model composed of a scaler followed by a logistic +# regression classifier. +from sklearn.linear_model import LogisticRegression +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import StandardScaler + +model = make_pipeline(StandardScaler(), LogisticRegression()) +model + +# %% +# We evaluate our model using cross-validation. We use the accuracy and the balanced +# accuracy to report the performance of our model. The balanced accuracy is a metric +# that is less sensitive to class imbalance and will allow us to put the accuracy +# score in perspective. +# +# Cross-validation allows us to study the variance of the decision threshold across +# different splits of the data. However, the dataset is rather small and it would be +# detrimental to use more than 5 folds to evaluate the dispersion. Therefore, we use +# a :class:`~sklearn.model_selection.RepeatedStratifiedKFold` where we apply several +# repetitions of 5-fold cross-validation. +import pandas as pd + +from sklearn.model_selection import RepeatedStratifiedKFold, cross_validate + +scoring = ["accuracy", "balanced_accuracy"] +cv_scores = [ + "train_accuracy", + "test_accuracy", + "train_balanced_accuracy", + "test_balanced_accuracy", +] +cv = RepeatedStratifiedKFold(n_splits=5, n_repeats=10, random_state=42) +cv_results_vanilla_model = pd.DataFrame( + cross_validate( + model, + data, + target, + scoring=scoring, + cv=cv, + return_train_score=True, + return_estimator=True, + ) +) +cv_results_vanilla_model[cv_scores].aggregate(["mean", "std"]).T + +# %% +# Our predictive model succeeds to grasp the relationship between the data and the +# target. The training and testing scores are close to each other, meaning that our +# predictive model is not overfitting. We can also observe that the balanced accuracy is +# lower than the accuracy, due to the class imbalance previously mentioned. +# +# For this classifier, we let the decision threshold, used convert the probability of +# the positive class into a class prediction, to its default value: 0.5. However, this +# threshold might not be optimal. If our interest is to maximize the balanced accuracy, +# we should select another threshold that would maximize this metric. +# +# The :class:`~sklearn.model_selection.TunedThresholdClassifierCV` meta-estimator allows +# to tune the decision threshold of a classifier given a metric of interest. +# +# Tuning the decision threshold +# ----------------------------- +# +# We create a :class:`~sklearn.model_selection.TunedThresholdClassifierCV` and +# configure it to maximize the balanced accuracy. We evaluate the model using the same +# cross-validation strategy as previously. +from sklearn.model_selection import TunedThresholdClassifierCV + +tuned_model = TunedThresholdClassifierCV(estimator=model, scoring="balanced_accuracy") +cv_results_tuned_model = pd.DataFrame( + cross_validate( + tuned_model, + data, + target, + scoring=scoring, + cv=cv, + return_train_score=True, + return_estimator=True, + ) +) +cv_results_tuned_model[cv_scores].aggregate(["mean", "std"]).T + +# %% +# In comparison with the vanilla model, we observe that the balanced accuracy score +# increased. Of course, it comes at the cost of a lower accuracy score. It means that +# our model is now more sensitive to the positive class but makes more mistakes on the +# negative class. +# +# However, it is important to note that this tuned predictive model is internally the +# same model as the vanilla model: they have the same fitted coefficients. +import matplotlib.pyplot as plt + +vanilla_model_coef = pd.DataFrame( + [est[-1].coef_.ravel() for est in cv_results_vanilla_model["estimator"]], + columns=diabetes.feature_names, +) +tuned_model_coef = pd.DataFrame( + [est.estimator_[-1].coef_.ravel() for est in cv_results_tuned_model["estimator"]], + columns=diabetes.feature_names, +) + +fig, ax = plt.subplots(ncols=2, figsize=(12, 4), sharex=True, sharey=True) +vanilla_model_coef.boxplot(ax=ax[0]) +ax[0].set_ylabel("Coefficient value") +ax[0].set_title("Vanilla model") +tuned_model_coef.boxplot(ax=ax[1]) +ax[1].set_title("Tuned model") +_ = fig.suptitle("Coefficients of the predictive models") + +# %% +# Only the decision threshold of each model was changed during the cross-validation. +decision_threshold = pd.Series( + [est.best_threshold_ for est in cv_results_tuned_model["estimator"]], +) +ax = decision_threshold.plot.kde() +ax.axvline( + decision_threshold.mean(), + color="k", + linestyle="--", + label=f"Mean decision threshold: {decision_threshold.mean():.2f}", +) +ax.set_xlabel("Decision threshold") +ax.legend(loc="upper right") +_ = ax.set_title( + "Distribution of the decision threshold \nacross different cross-validation folds" +) + +# %% +# In average, a decision threshold around 0.32 maximizes the balanced accuracy, which is +# different from the default decision threshold of 0.5. Thus tuning the decision +# threshold is particularly important when the output of the predictive model +# is used to make decisions. Besides, the metric used to tune the decision threshold +# should be chosen carefully. Here, we used the balanced accuracy but it might not be +# the most appropriate metric for the problem at hand. The choice of the "right" metric +# is usually problem-dependent and might require some domain knowledge. Refer to the +# example entitled, +# :ref:`sphx_glr_auto_examples_model_selection_plot_cost_sensitive_learning.py`, +# for more details. diff --git a/examples/model_selection/plot_underfitting_overfitting.py b/examples/model_selection/plot_underfitting_overfitting.py index 6538a501e78bf..a6151cd6b3c20 100644 --- a/examples/model_selection/plot_underfitting_overfitting.py +++ b/examples/model_selection/plot_underfitting_overfitting.py @@ -18,21 +18,25 @@ cross-validation. We calculate the mean squared error (MSE) on the validation set, the higher, the less likely the model generalizes correctly from the training data. + """ -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt -from sklearn.pipeline import Pipeline -from sklearn.preprocessing import PolynomialFeatures +import numpy as np + from sklearn.linear_model import LinearRegression from sklearn.model_selection import cross_val_score +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import PolynomialFeatures def true_fun(X): return np.cos(1.5 * np.pi * X) + np.random.seed(0) n_samples = 30 @@ -46,26 +50,33 @@ def true_fun(X): ax = plt.subplot(1, len(degrees), i + 1) plt.setp(ax, xticks=(), yticks=()) - polynomial_features = PolynomialFeatures(degree=degrees[i], - include_bias=False) + polynomial_features = PolynomialFeatures(degree=degrees[i], include_bias=False) linear_regression = LinearRegression() - pipeline = Pipeline([("polynomial_features", polynomial_features), - ("linear_regression", linear_regression)]) + pipeline = Pipeline( + [ + ("polynomial_features", polynomial_features), + ("linear_regression", linear_regression), + ] + ) pipeline.fit(X[:, np.newaxis], y) # Evaluate the models using crossvalidation - scores = cross_val_score(pipeline, X[:, np.newaxis], y, - scoring="neg_mean_squared_error", cv=10) + scores = cross_val_score( + pipeline, X[:, np.newaxis], y, scoring="neg_mean_squared_error", cv=10 + ) X_test = np.linspace(0, 1, 100) plt.plot(X_test, pipeline.predict(X_test[:, np.newaxis]), label="Model") plt.plot(X_test, true_fun(X_test), label="True function") - plt.scatter(X, y, edgecolor='b', s=20, label="Samples") + plt.scatter(X, y, edgecolor="b", s=20, label="Samples") plt.xlabel("x") plt.ylabel("y") plt.xlim((0, 1)) plt.ylim((-2, 2)) plt.legend(loc="best") - plt.title("Degree {}\nMSE = {:.2e}(+/- {:.2e})".format( - degrees[i], -scores.mean(), scores.std())) + plt.title( + "Degree {}\nMSE = {:.2e}(+/- {:.2e})".format( + degrees[i], -scores.mean(), scores.std() + ) + ) plt.show() diff --git a/examples/model_selection/plot_validation_curve.py b/examples/model_selection/plot_validation_curve.py deleted file mode 100644 index 0eb3850787c53..0000000000000 --- a/examples/model_selection/plot_validation_curve.py +++ /dev/null @@ -1,50 +0,0 @@ -""" -========================== -Plotting Validation Curves -========================== - -In this plot you can see the training scores and validation scores of an SVM -for different values of the kernel parameter gamma. For very low values of -gamma, you can see that both the training score and the validation score are -low. This is called underfitting. Medium values of gamma will result in high -values for both scores, i.e. the classifier is performing fairly well. If gamma -is too high, the classifier will overfit, which means that the training score -is good but the validation score is poor. -""" -print(__doc__) - -import matplotlib.pyplot as plt -import numpy as np - -from sklearn.datasets import load_digits -from sklearn.svm import SVC -from sklearn.model_selection import validation_curve - -X, y = load_digits(return_X_y=True) - -param_range = np.logspace(-6, -1, 5) -train_scores, test_scores = validation_curve( - SVC(), X, y, param_name="gamma", param_range=param_range, - scoring="accuracy", n_jobs=1) -train_scores_mean = np.mean(train_scores, axis=1) -train_scores_std = np.std(train_scores, axis=1) -test_scores_mean = np.mean(test_scores, axis=1) -test_scores_std = np.std(test_scores, axis=1) - -plt.title("Validation Curve with SVM") -plt.xlabel(r"$\gamma$") -plt.ylabel("Score") -plt.ylim(0.0, 1.1) -lw = 2 -plt.semilogx(param_range, train_scores_mean, label="Training score", - color="darkorange", lw=lw) -plt.fill_between(param_range, train_scores_mean - train_scores_std, - train_scores_mean + train_scores_std, alpha=0.2, - color="darkorange", lw=lw) -plt.semilogx(param_range, test_scores_mean, label="Cross-validation score", - color="navy", lw=lw) -plt.fill_between(param_range, test_scores_mean - test_scores_std, - test_scores_mean + test_scores_std, alpha=0.2, - color="navy", lw=lw) -plt.legend(loc="best") -plt.show() diff --git a/examples/multiclass/README.txt b/examples/multiclass/README.txt new file mode 100644 index 0000000000000..60a85711e6b1a --- /dev/null +++ b/examples/multiclass/README.txt @@ -0,0 +1,6 @@ +.. _multiclass_examples: + +Multiclass methods +------------------ + +Examples concerning the :mod:`sklearn.multiclass` module. diff --git a/examples/multiclass/plot_multiclass_overview.py b/examples/multiclass/plot_multiclass_overview.py new file mode 100644 index 0000000000000..6e18f84b9d222 --- /dev/null +++ b/examples/multiclass/plot_multiclass_overview.py @@ -0,0 +1,202 @@ +""" +=============================================== +Overview of multiclass training meta-estimators +=============================================== + +In this example, we discuss the problem of classification when the target +variable is composed of more than two classes. This is called multiclass +classification. + +In scikit-learn, all estimators support multiclass classification out of the +box: the most sensible strategy was implemented for the end-user. The +:mod:`sklearn.multiclass` module implements various strategies that one can use +for experimenting or developing third-party estimators that only support binary +classification. + +:mod:`sklearn.multiclass` includes OvO/OvR strategies used to train a +multiclass classifier by fitting a set of binary classifiers (the +:class:`~sklearn.multiclass.OneVsOneClassifier` and +:class:`~sklearn.multiclass.OneVsRestClassifier` meta-estimators). This example +will review them. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# The Yeast UCI dataset +# --------------------- +# +# In this example, we use a UCI dataset [1]_, generally referred as the Yeast +# dataset. We use the :func:`sklearn.datasets.fetch_openml` function to load +# the dataset from OpenML. +from sklearn.datasets import fetch_openml + +X, y = fetch_openml(data_id=181, as_frame=True, return_X_y=True) + +# %% +# To know the type of data science problem we are dealing with, we can check +# the target for which we want to build a predictive model. +y.value_counts().sort_index() + +# %% +# We see that the target is discrete and composed of 10 classes. We therefore +# deal with a multiclass classification problem. +# +# Strategies comparison +# --------------------- +# +# In the following experiment, we use a +# :class:`~sklearn.tree.DecisionTreeClassifier` and a +# :class:`~sklearn.model_selection.RepeatedStratifiedKFold` cross-validation +# with 3 splits and 5 repetitions. +# +# We compare the following strategies: +# +# * :class:`~sklearn.tree.DecisionTreeClassifier` can handle multiclass +# classification without needing any special adjustments. It works by breaking +# down the training data into smaller subsets and focusing on the most common +# class in each subset. By repeating this process, the model can accurately +# classify input data into multiple different classes. +# * :class:`~sklearn.multiclass.OneVsOneClassifier` trains a set of binary +# classifiers where each classifier is trained to distinguish between +# two classes. +# * :class:`~sklearn.multiclass.OneVsRestClassifier`: trains a set of binary +# classifiers where each classifier is trained to distinguish between +# one class and the rest of the classes. +# * :class:`~sklearn.multiclass.OutputCodeClassifier`: trains a set of binary +# classifiers where each classifier is trained to distinguish between +# a set of classes from the rest of the classes. The set of classes is +# defined by a codebook, which is randomly generated in scikit-learn. This +# method exposes a parameter `code_size` to control the size of the codebook. +# We set it above one since we are not interested in compressing the class +# representation. +import pandas as pd + +from sklearn.model_selection import RepeatedStratifiedKFold, cross_validate +from sklearn.multiclass import ( + OneVsOneClassifier, + OneVsRestClassifier, + OutputCodeClassifier, +) +from sklearn.tree import DecisionTreeClassifier + +cv = RepeatedStratifiedKFold(n_splits=3, n_repeats=5, random_state=0) + +tree = DecisionTreeClassifier(random_state=0) +ovo_tree = OneVsOneClassifier(tree) +ovr_tree = OneVsRestClassifier(tree) +ecoc = OutputCodeClassifier(tree, code_size=2) + +cv_results_tree = cross_validate(tree, X, y, cv=cv, n_jobs=2) +cv_results_ovo = cross_validate(ovo_tree, X, y, cv=cv, n_jobs=2) +cv_results_ovr = cross_validate(ovr_tree, X, y, cv=cv, n_jobs=2) +cv_results_ecoc = cross_validate(ecoc, X, y, cv=cv, n_jobs=2) + +# %% +# We can now compare the statistical performance of the different strategies. +# We plot the score distribution of the different strategies. +from matplotlib import pyplot as plt + +scores = pd.DataFrame( + { + "DecisionTreeClassifier": cv_results_tree["test_score"], + "OneVsOneClassifier": cv_results_ovo["test_score"], + "OneVsRestClassifier": cv_results_ovr["test_score"], + "OutputCodeClassifier": cv_results_ecoc["test_score"], + } +) +ax = scores.plot.kde(legend=True) +ax.set_xlabel("Accuracy score") +ax.set_xlim([0, 0.7]) +_ = ax.set_title( + "Density of the accuracy scores for the different multiclass strategies" +) + +# %% +# At a first glance, we can see that the built-in strategy of the decision +# tree classifier is working quite well. One-vs-one and the error-correcting +# output code strategies are working even better. However, the +# one-vs-rest strategy is not working as well as the other strategies. +# +# Indeed, these results reproduce something reported in the literature +# as in [2]_. However, the story is not as simple as it seems. +# +# The importance of hyperparameters search +# ---------------------------------------- +# +# It was later shown in [3]_ that the multiclass strategies would show similar +# scores if the hyperparameters of the base classifiers are first optimized. +# +# Here we try to reproduce such result by at least optimizing the depth of the +# base decision tree. +from sklearn.model_selection import GridSearchCV + +param_grid = {"max_depth": [3, 5, 8]} +tree_optimized = GridSearchCV(tree, param_grid=param_grid, cv=3) +ovo_tree = OneVsOneClassifier(tree_optimized) +ovr_tree = OneVsRestClassifier(tree_optimized) +ecoc = OutputCodeClassifier(tree_optimized, code_size=2) + +cv_results_tree = cross_validate(tree_optimized, X, y, cv=cv, n_jobs=2) +cv_results_ovo = cross_validate(ovo_tree, X, y, cv=cv, n_jobs=2) +cv_results_ovr = cross_validate(ovr_tree, X, y, cv=cv, n_jobs=2) +cv_results_ecoc = cross_validate(ecoc, X, y, cv=cv, n_jobs=2) + +scores = pd.DataFrame( + { + "DecisionTreeClassifier": cv_results_tree["test_score"], + "OneVsOneClassifier": cv_results_ovo["test_score"], + "OneVsRestClassifier": cv_results_ovr["test_score"], + "OutputCodeClassifier": cv_results_ecoc["test_score"], + } +) +ax = scores.plot.kde(legend=True) +ax.set_xlabel("Accuracy score") +ax.set_xlim([0, 0.7]) +_ = ax.set_title( + "Density of the accuracy scores for the different multiclass strategies" +) + +plt.show() + +# %% +# We can see that once the hyperparameters are optimized, all multiclass +# strategies have similar performance as discussed in [3]_. +# +# Conclusion +# ---------- +# +# We can get some intuition behind those results. +# +# First, the reason for which one-vs-one and error-correcting output code are +# outperforming the tree when the hyperparameters are not optimized relies on +# fact that they ensemble a larger number of classifiers. The ensembling +# improves the generalization performance. This is a bit similar why a bagging +# classifier generally performs better than a single decision tree if no care +# is taken to optimize the hyperparameters. +# +# Then, we see the importance of optimizing the hyperparameters. Indeed, it +# should be regularly explored when developing predictive models even if +# techniques such as ensembling help at reducing this impact. +# +# Finally, it is important to recall that the estimators in scikit-learn +# are developed with a specific strategy to handle multiclass classification +# out of the box. So for these estimators, it means that there is no need to +# use different strategies. These strategies are mainly useful for third-party +# estimators supporting only binary classification. In all cases, we also show +# that the hyperparameters should be optimized. +# +# References +# ---------- +# +# .. [1] https://archive.ics.uci.edu/ml/datasets/Yeast +# +# .. [2] `"Reducing multiclass to binary: A unifying approach for margin classifiers." +# Allwein, Erin L., Robert E. Schapire, and Yoram Singer. +# Journal of machine learning research. 1 Dec (2000): 113-141. +# `_ +# +# .. [3] `"In defense of one-vs-all classification." +# Journal of Machine Learning Research. 5 Jan (2004): 101-141. +# `_ diff --git a/examples/multioutput/plot_classifier_chain_yeast.py b/examples/multioutput/plot_classifier_chain_yeast.py index 7ae80af3fdab3..9d6e7d411c289 100644 --- a/examples/multioutput/plot_classifier_chain_yeast.py +++ b/examples/multioutput/plot_classifier_chain_yeast.py @@ -1,113 +1,153 @@ """ -============================ -Classifier Chain -============================ -Example of using classifier chain on a multilabel dataset. - -For this example we will use the `yeast -`_ dataset which contains -2417 datapoints each with 103 features and 14 possible labels. Each -data point has at least one label. As a baseline we first train a logistic -regression classifier for each of the 14 labels. To evaluate the performance of -these classifiers we predict on a held-out test set and calculate the -:ref:`jaccard score ` for each sample. - -Next we create 10 classifier chains. Each classifier chain contains a -logistic regression model for each of the 14 labels. The models in each -chain are ordered randomly. In addition to the 103 features in the dataset, -each model gets the predictions of the preceding models in the chain as -features (note that by default at training time each model gets the true -labels as features). These additional features allow each chain to exploit -correlations among the classes. The Jaccard similarity score for each chain -tends to be greater than that of the set independent logistic models. - -Because the models in each chain are arranged randomly there is significant -variation in performance among the chains. Presumably there is an optimal -ordering of the classes in a chain that will yield the best performance. -However we do not know that ordering a priori. Instead we can construct an -voting ensemble of classifier chains by averaging the binary predictions of -the chains and apply a threshold of 0.5. The Jaccard similarity score of the -ensemble is greater than that of the independent models and tends to exceed -the score of each chain in the ensemble (although this is not guaranteed -with randomly ordered chains). +================================================== +Multilabel classification using a classifier chain +================================================== +This example shows how to use :class:`~sklearn.multioutput.ClassifierChain` to solve +a multilabel classification problem. + +The most naive strategy to solve such a task is to independently train a binary +classifier on each label (i.e. each column of the target variable). At prediction +time, the ensemble of binary classifiers is used to assemble multitask prediction. + +This strategy does not allow to model relationship between different tasks. The +:class:`~sklearn.multioutput.ClassifierChain` is the meta-estimator (i.e. an estimator +taking an inner estimator) that implements a more advanced strategy. The ensemble +of binary classifiers are used as a chain where the prediction of a classifier in the +chain is used as a feature for training the next classifier on a new label. Therefore, +these additional features allow each chain to exploit correlations among labels. + +The :ref:`Jaccard similarity ` score for chain tends to be +greater than that of the set independent base models. """ -# Author: Adam Kleczewski -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Loading a dataset +# ----------------- +# For this example, we use the `yeast +# `_ dataset which contains +# 2,417 datapoints each with 103 features and 14 possible labels. Each +# data point has at least one label. As a baseline we first train a logistic +# regression classifier for each of the 14 labels. To evaluate the performance of +# these classifiers we predict on a held-out test set and calculate the +# Jaccard similarity for each sample. -import numpy as np import matplotlib.pyplot as plt +import numpy as np + from sklearn.datasets import fetch_openml -from sklearn.multioutput import ClassifierChain from sklearn.model_selection import train_test_split -from sklearn.multiclass import OneVsRestClassifier -from sklearn.metrics import jaccard_score -from sklearn.linear_model import LogisticRegression - -print(__doc__) # Load a multi-label dataset from https://www.openml.org/d/40597 -X, Y = fetch_openml('yeast', version=4, return_X_y=True) -Y = Y == 'TRUE' -X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=.2, - random_state=0) +X, Y = fetch_openml("yeast", version=4, return_X_y=True) +Y = Y == "TRUE" +X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=0) + +# %% +# Fit models +# ---------- +# We fit :class:`~sklearn.linear_model.LogisticRegression` wrapped by +# :class:`~sklearn.multiclass.OneVsRestClassifier` and ensemble of multiple +# :class:`~sklearn.multioutput.ClassifierChain`. +# +# LogisticRegression wrapped by OneVsRestClassifier +# ************************************************** +# Since by default :class:`~sklearn.linear_model.LogisticRegression` can't +# handle data with multiple targets, we need to use +# :class:`~sklearn.multiclass.OneVsRestClassifier`. +# After fitting the model we calculate Jaccard similarity. + +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import jaccard_score +from sklearn.multiclass import OneVsRestClassifier -# Fit an independent logistic regression model for each class using the -# OneVsRestClassifier wrapper. base_lr = LogisticRegression() ovr = OneVsRestClassifier(base_lr) ovr.fit(X_train, Y_train) Y_pred_ovr = ovr.predict(X_test) -ovr_jaccard_score = jaccard_score(Y_test, Y_pred_ovr, average='samples') +ovr_jaccard_score = jaccard_score(Y_test, Y_pred_ovr, average="samples") + +# %% +# Chain of binary classifiers +# *************************** +# Because the models in each chain are arranged randomly there is significant +# variation in performance among the chains. Presumably there is an optimal +# ordering of the classes in a chain that will yield the best performance. +# However, we do not know that ordering a priori. Instead, we can build a +# voting ensemble of classifier chains by averaging the binary predictions of +# the chains and apply a threshold of 0.5. The Jaccard similarity score of the +# ensemble is greater than that of the independent models and tends to exceed +# the score of each chain in the ensemble (although this is not guaranteed +# with randomly ordered chains). -# Fit an ensemble of logistic regression classifier chains and take the -# take the average prediction of all the chains. -chains = [ClassifierChain(base_lr, order='random', random_state=i) - for i in range(10)] +from sklearn.multioutput import ClassifierChain + +chains = [ClassifierChain(base_lr, order="random", random_state=i) for i in range(10)] for chain in chains: chain.fit(X_train, Y_train) -Y_pred_chains = np.array([chain.predict(X_test) for chain in - chains]) -chain_jaccard_scores = [jaccard_score(Y_test, Y_pred_chain >= .5, - average='samples') - for Y_pred_chain in Y_pred_chains] +Y_pred_chains = np.array([chain.predict_proba(X_test) for chain in chains]) +chain_jaccard_scores = [ + jaccard_score(Y_test, Y_pred_chain >= 0.5, average="samples") + for Y_pred_chain in Y_pred_chains +] Y_pred_ensemble = Y_pred_chains.mean(axis=0) -ensemble_jaccard_score = jaccard_score(Y_test, - Y_pred_ensemble >= .5, - average='samples') - -model_scores = [ovr_jaccard_score] + chain_jaccard_scores -model_scores.append(ensemble_jaccard_score) - -model_names = ('Independent', - 'Chain 1', - 'Chain 2', - 'Chain 3', - 'Chain 4', - 'Chain 5', - 'Chain 6', - 'Chain 7', - 'Chain 8', - 'Chain 9', - 'Chain 10', - 'Ensemble') - -x_pos = np.arange(len(model_names)) +ensemble_jaccard_score = jaccard_score( + Y_test, Y_pred_ensemble >= 0.5, average="samples" +) +# %% +# Plot results +# ------------ # Plot the Jaccard similarity scores for the independent model, each of the # chains, and the ensemble (note that the vertical axis on this plot does # not begin at 0). +model_scores = [ovr_jaccard_score] + chain_jaccard_scores + [ensemble_jaccard_score] + +model_names = ( + "Independent", + "Chain 1", + "Chain 2", + "Chain 3", + "Chain 4", + "Chain 5", + "Chain 6", + "Chain 7", + "Chain 8", + "Chain 9", + "Chain 10", + "Ensemble", +) + +x_pos = np.arange(len(model_names)) + fig, ax = plt.subplots(figsize=(7, 4)) ax.grid(True) -ax.set_title('Classifier Chain Ensemble Performance Comparison') +ax.set_title("Classifier Chain Ensemble Performance Comparison") ax.set_xticks(x_pos) -ax.set_xticklabels(model_names, rotation='vertical') -ax.set_ylabel('Jaccard Similarity Score') -ax.set_ylim([min(model_scores) * .9, max(model_scores) * 1.1]) -colors = ['r'] + ['b'] * len(chain_jaccard_scores) + ['g'] +ax.set_xticklabels(model_names, rotation="vertical") +ax.set_ylabel("Jaccard Similarity Score") +ax.set_ylim([min(model_scores) * 0.9, max(model_scores) * 1.1]) +colors = ["r"] + ["b"] * len(chain_jaccard_scores) + ["g"] ax.bar(x_pos, model_scores, alpha=0.5, color=colors) plt.tight_layout() plt.show() + +# %% +# Results interpretation +# ---------------------- +# There are three main takeaways from this plot: +# +# - Independent model wrapped by :class:`~sklearn.multiclass.OneVsRestClassifier` +# performs worse than the ensemble of classifier chains and some of individual chains. +# This is caused by the fact that the logistic regression doesn't model relationship +# between the labels. +# - :class:`~sklearn.multioutput.ClassifierChain` takes advantage of correlation +# among labels but due to random nature of labels ordering, it could yield worse +# result than an independent model. +# - An ensemble of chains performs better because it not only captures relationship +# between labels but also does not make strong assumptions about their correct order. diff --git a/examples/neighbors/approximate_nearest_neighbors.py b/examples/neighbors/approximate_nearest_neighbors.py index b7f09d3127b98..a2da69f62fb10 100644 --- a/examples/neighbors/approximate_nearest_neighbors.py +++ b/examples/neighbors/approximate_nearest_neighbors.py @@ -3,83 +3,53 @@ Approximate nearest neighbors in TSNE ===================================== -This example presents how to chain KNeighborsTransformer and TSNE in a -pipeline. It also shows how to wrap the packages `annoy` and `nmslib` to -replace KNeighborsTransformer and perform approximate nearest neighbors. -These packages can be installed with `pip install annoy nmslib`. - -Note: Currently `TSNE(metric='precomputed')` does not modify the precomputed -distances, and thus assumes that precomputed euclidean distances are squared. -In future versions, a parameter in TSNE will control the optional squaring of -precomputed distances (see #12401). +This example presents how to chain KNeighborsTransformer and TSNE in a pipeline. +It also shows how to wrap the packages `nmslib` and `pynndescent` to replace +KNeighborsTransformer and perform approximate nearest neighbors. These packages +can be installed with `pip install nmslib pynndescent`. Note: In KNeighborsTransformer we use the definition which includes each training point as its own neighbor in the count of `n_neighbors`, and for -compatibility reasons, one extra neighbor is computed when -`mode == 'distance'`. Please note that we do the same in the proposed wrappers. - -Sample output:: - - Benchmarking on MNIST_2000: - --------------------------- - AnnoyTransformer: 0.583 sec - NMSlibTransformer: 0.321 sec - KNeighborsTransformer: 1.225 sec - TSNE with AnnoyTransformer: 4.903 sec - TSNE with NMSlibTransformer: 5.009 sec - TSNE with KNeighborsTransformer: 6.210 sec - TSNE with internal NearestNeighbors: 6.365 sec - - Benchmarking on MNIST_10000: - ---------------------------- - AnnoyTransformer: 4.457 sec - NMSlibTransformer: 2.080 sec - KNeighborsTransformer: 30.680 sec - TSNE with AnnoyTransformer: 30.225 sec - TSNE with NMSlibTransformer: 43.295 sec - TSNE with KNeighborsTransformer: 64.845 sec - TSNE with internal NearestNeighbors: 64.984 sec - +compatibility reasons, one extra neighbor is computed when `mode == 'distance'`. +Please note that we do the same in the proposed `nmslib` wrapper. """ -# Author: Tom Dupre la Tour -# -# License: BSD 3 clause -import time + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# First we try to import the packages and warn the user in case they are +# missing. import sys try: - import annoy + import nmslib except ImportError: - print("The package 'annoy' is required to run this example.") + print("The package 'nmslib' is required to run this example.") sys.exit() try: - import nmslib + from pynndescent import PyNNDescentTransformer except ImportError: - print("The package 'nmslib' is required to run this example.") + print("The package 'pynndescent' is required to run this example.") sys.exit() +# %% +# We define a wrapper class for implementing the scikit-learn API to the +# `nmslib`, as well as a loading function. +import joblib import numpy as np -import matplotlib.pyplot as plt -from matplotlib.ticker import NullFormatter from scipy.sparse import csr_matrix from sklearn.base import BaseEstimator, TransformerMixin -from sklearn.neighbors import KNeighborsTransformer -from sklearn.utils._testing import assert_array_almost_equal from sklearn.datasets import fetch_openml -from sklearn.pipeline import make_pipeline -from sklearn.manifold import TSNE from sklearn.utils import shuffle -print(__doc__) - class NMSlibTransformer(TransformerMixin, BaseEstimator): """Wrapper for using nmslib as sklearn's KNeighborsTransformer""" - def __init__(self, n_neighbors=5, metric='euclidean', method='sw-graph', - n_jobs=1): + def __init__(self, n_neighbors=5, metric="euclidean", method="sw-graph", n_jobs=-1): self.n_neighbors = n_neighbors self.method = method self.metric = metric @@ -91,15 +61,14 @@ def fit(self, X): # see more metric in the manual # https://github.com/nmslib/nmslib/tree/master/manual space = { - 'sqeuclidean': 'l2', - 'euclidean': 'l2', - 'cosine': 'cosinesimil', - 'l1': 'l1', - 'l2': 'l2', + "euclidean": "l2", + "cosine": "cosinesimil", + "l1": "l1", + "l2": "l2", }[self.metric] self.nmslib_ = nmslib.init(method=self.method, space=space) - self.nmslib_.addDataPointBatch(X) + self.nmslib_.addDataPointBatch(X.copy()) self.nmslib_.createIndex() return self @@ -110,185 +79,236 @@ def transform(self, X): # neighbor, one extra neighbor will be computed. n_neighbors = self.n_neighbors + 1 - results = self.nmslib_.knnQueryBatch(X, k=n_neighbors, - num_threads=self.n_jobs) - indices, distances = zip(*results) - indices, distances = np.vstack(indices), np.vstack(distances) - - if self.metric == 'sqeuclidean': - distances **= 2 - - indptr = np.arange(0, n_samples_transform * n_neighbors + 1, - n_neighbors) - kneighbors_graph = csr_matrix((distances.ravel(), indices.ravel(), - indptr), shape=(n_samples_transform, - self.n_samples_fit_)) - - return kneighbors_graph - - -class AnnoyTransformer(TransformerMixin, BaseEstimator): - """Wrapper for using annoy.AnnoyIndex as sklearn's KNeighborsTransformer""" - - def __init__(self, n_neighbors=5, metric='euclidean', n_trees=10, - search_k=-1): - self.n_neighbors = n_neighbors - self.n_trees = n_trees - self.search_k = search_k - self.metric = metric - - def fit(self, X): - self.n_samples_fit_ = X.shape[0] - metric = self.metric if self.metric != 'sqeuclidean' else 'euclidean' - self.annoy_ = annoy.AnnoyIndex(X.shape[1], metric=metric) - for i, x in enumerate(X): - self.annoy_.add_item(i, x.tolist()) - self.annoy_.build(self.n_trees) - return self - - def transform(self, X): - return self._transform(X) - - def fit_transform(self, X, y=None): - return self.fit(X)._transform(X=None) - - def _transform(self, X): - """As `transform`, but handles X is None for faster `fit_transform`.""" - - n_samples_transform = self.n_samples_fit_ if X is None else X.shape[0] - - # For compatibility reasons, as each sample is considered as its own - # neighbor, one extra neighbor will be computed. - n_neighbors = self.n_neighbors + 1 - - indices = np.empty((n_samples_transform, n_neighbors), - dtype=int) - distances = np.empty((n_samples_transform, n_neighbors)) - - if X is None: - for i in range(self.annoy_.get_n_items()): - ind, dist = self.annoy_.get_nns_by_item( - i, n_neighbors, self.search_k, include_distances=True) - - indices[i], distances[i] = ind, dist + if self.n_jobs < 0: + # Same handling as done in joblib for negative values of n_jobs: + # in particular, `n_jobs == -1` means "as many threads as CPUs". + num_threads = joblib.cpu_count() + self.n_jobs + 1 else: - for i, x in enumerate(X): - indices[i], distances[i] = self.annoy_.get_nns_by_vector( - x.tolist(), n_neighbors, self.search_k, - include_distances=True) + num_threads = self.n_jobs - if self.metric == 'sqeuclidean': - distances **= 2 + results = self.nmslib_.knnQueryBatch( + X.copy(), k=n_neighbors, num_threads=num_threads + ) + indices, distances = zip(*results) + indices, distances = np.vstack(indices), np.vstack(distances) - indptr = np.arange(0, n_samples_transform * n_neighbors + 1, - n_neighbors) - kneighbors_graph = csr_matrix((distances.ravel(), indices.ravel(), - indptr), shape=(n_samples_transform, - self.n_samples_fit_)) + indptr = np.arange(0, n_samples_transform * n_neighbors + 1, n_neighbors) + kneighbors_graph = csr_matrix( + (distances.ravel(), indices.ravel(), indptr), + shape=(n_samples_transform, self.n_samples_fit_), + ) return kneighbors_graph -def test_transformers(): - """Test that AnnoyTransformer and KNeighborsTransformer give same results - """ - X = np.random.RandomState(42).randn(10, 2) - - knn = KNeighborsTransformer() - Xt0 = knn.fit_transform(X) - - ann = AnnoyTransformer() - Xt1 = ann.fit_transform(X) - - nms = NMSlibTransformer() - Xt2 = nms.fit_transform(X) - - assert_array_almost_equal(Xt0.toarray(), Xt1.toarray(), decimal=5) - assert_array_almost_equal(Xt0.toarray(), Xt2.toarray(), decimal=5) - - def load_mnist(n_samples): """Load MNIST, shuffle the data, and return only n_samples.""" - mnist = fetch_openml("mnist_784") + mnist = fetch_openml("mnist_784", as_frame=False) X, y = shuffle(mnist.data, mnist.target, random_state=2) return X[:n_samples] / 255, y[:n_samples] -def run_benchmark(): - datasets = [ - ('MNIST_2000', load_mnist(n_samples=2000)), - ('MNIST_10000', load_mnist(n_samples=10000)), - ] - - n_iter = 500 - perplexity = 30 - # TSNE requires a certain number of neighbors which depends on the - # perplexity parameter. - # Add one since we include each sample as its own neighbor. - n_neighbors = int(3. * perplexity + 1) + 1 - - transformers = [ - ('AnnoyTransformer', AnnoyTransformer(n_neighbors=n_neighbors, - metric='sqeuclidean')), - ('NMSlibTransformer', NMSlibTransformer(n_neighbors=n_neighbors, - metric='sqeuclidean')), - ('KNeighborsTransformer', KNeighborsTransformer( - n_neighbors=n_neighbors, mode='distance', metric='sqeuclidean')), - ('TSNE with AnnoyTransformer', make_pipeline( - AnnoyTransformer(n_neighbors=n_neighbors, metric='sqeuclidean'), - TSNE(metric='precomputed', perplexity=perplexity, - method="barnes_hut", random_state=42, n_iter=n_iter), )), - ('TSNE with NMSlibTransformer', make_pipeline( - NMSlibTransformer(n_neighbors=n_neighbors, metric='sqeuclidean'), - TSNE(metric='precomputed', perplexity=perplexity, - method="barnes_hut", random_state=42, n_iter=n_iter), )), - ('TSNE with KNeighborsTransformer', make_pipeline( - KNeighborsTransformer(n_neighbors=n_neighbors, mode='distance', - metric='sqeuclidean'), - TSNE(metric='precomputed', perplexity=perplexity, - method="barnes_hut", random_state=42, n_iter=n_iter), )), - ('TSNE with internal NearestNeighbors', - TSNE(metric='sqeuclidean', perplexity=perplexity, method="barnes_hut", - random_state=42, n_iter=n_iter)), - ] - - # init the plot - nrows = len(datasets) - ncols = np.sum([1 for name, model in transformers if 'TSNE' in name]) - fig, axes = plt.subplots(nrows=nrows, ncols=ncols, squeeze=False, - figsize=(5 * ncols, 4 * nrows)) - axes = axes.ravel() - i_ax = 0 - - for dataset_name, (X, y) in datasets: - - msg = 'Benchmarking on %s:' % dataset_name - print('\n%s\n%s' % (msg, '-' * len(msg))) - - for transformer_name, transformer in transformers: +# %% +# We benchmark the different exact/approximate nearest neighbors transformers. +import time + +from sklearn.manifold import TSNE +from sklearn.neighbors import KNeighborsTransformer +from sklearn.pipeline import make_pipeline + +datasets = [ + ("MNIST_10000", load_mnist(n_samples=10_000)), + ("MNIST_20000", load_mnist(n_samples=20_000)), +] + +n_iter = 500 +perplexity = 30 +metric = "euclidean" +# TSNE requires a certain number of neighbors which depends on the +# perplexity parameter. +# Add one since we include each sample as its own neighbor. +n_neighbors = int(3.0 * perplexity + 1) + 1 + +tsne_params = dict( + init="random", # pca not supported for sparse matrices + perplexity=perplexity, + method="barnes_hut", + random_state=42, + n_iter=n_iter, + learning_rate="auto", +) + +transformers = [ + ( + "KNeighborsTransformer", + KNeighborsTransformer(n_neighbors=n_neighbors, mode="distance", metric=metric), + ), + ( + "NMSlibTransformer", + NMSlibTransformer(n_neighbors=n_neighbors, metric=metric), + ), + ( + "PyNNDescentTransformer", + PyNNDescentTransformer( + n_neighbors=n_neighbors, metric=metric, parallel_batch_queries=True + ), + ), +] + +for dataset_name, (X, y) in datasets: + msg = f"Benchmarking on {dataset_name}:" + print(f"\n{msg}\n" + str("-" * len(msg))) + + for transformer_name, transformer in transformers: + longest = np.max([len(name) for name, model in transformers]) + start = time.time() + transformer.fit(X) + fit_duration = time.time() - start + print(f"{transformer_name:<{longest}} {fit_duration:.3f} sec (fit)") + start = time.time() + Xt = transformer.transform(X) + transform_duration = time.time() - start + print(f"{transformer_name:<{longest}} {transform_duration:.3f} sec (transform)") + if transformer_name == "PyNNDescentTransformer": start = time.time() - Xt = transformer.fit_transform(X) - duration = time.time() - start - - # print the duration report - longest = np.max([len(name) for name, model in transformers]) - whitespaces = ' ' * (longest - len(transformer_name)) - print('%s: %s%.3f sec' % (transformer_name, whitespaces, duration)) - - # plot TSNE embedding which should be very similar across methods - if 'TSNE' in transformer_name: - axes[i_ax].set_title(transformer_name + '\non ' + dataset_name) - axes[i_ax].scatter(Xt[:, 0], Xt[:, 1], c=y.astype(np.int32), - alpha=0.2, cmap=plt.cm.viridis) - axes[i_ax].xaxis.set_major_formatter(NullFormatter()) - axes[i_ax].yaxis.set_major_formatter(NullFormatter()) - axes[i_ax].axis('tight') - i_ax += 1 - - fig.tight_layout() - plt.show() - - -if __name__ == '__main__': - test_transformers() - run_benchmark() + Xt = transformer.transform(X) + transform_duration = time.time() - start + print( + f"{transformer_name:<{longest}} {transform_duration:.3f} sec" + " (transform)" + ) + +# %% +# Sample output:: +# +# Benchmarking on MNIST_10000: +# ---------------------------- +# KNeighborsTransformer 0.007 sec (fit) +# KNeighborsTransformer 1.139 sec (transform) +# NMSlibTransformer 0.208 sec (fit) +# NMSlibTransformer 0.315 sec (transform) +# PyNNDescentTransformer 4.823 sec (fit) +# PyNNDescentTransformer 4.884 sec (transform) +# PyNNDescentTransformer 0.744 sec (transform) +# +# Benchmarking on MNIST_20000: +# ---------------------------- +# KNeighborsTransformer 0.011 sec (fit) +# KNeighborsTransformer 5.769 sec (transform) +# NMSlibTransformer 0.733 sec (fit) +# NMSlibTransformer 1.077 sec (transform) +# PyNNDescentTransformer 14.448 sec (fit) +# PyNNDescentTransformer 7.103 sec (transform) +# PyNNDescentTransformer 1.759 sec (transform) +# +# Notice that the `PyNNDescentTransformer` takes more time during the first +# `fit` and the first `transform` due to the overhead of the numba just in time +# compiler. But after the first call, the compiled Python code is kept in a +# cache by numba and subsequent calls do not suffer from this initial overhead. +# Both :class:`~sklearn.neighbors.KNeighborsTransformer` and `NMSlibTransformer` +# are only run once here as they would show more stable `fit` and `transform` +# times (they don't have the cold start problem of PyNNDescentTransformer). + +# %% +import matplotlib.pyplot as plt +from matplotlib.ticker import NullFormatter + +transformers = [ + ("TSNE with internal NearestNeighbors", TSNE(metric=metric, **tsne_params)), + ( + "TSNE with KNeighborsTransformer", + make_pipeline( + KNeighborsTransformer( + n_neighbors=n_neighbors, mode="distance", metric=metric + ), + TSNE(metric="precomputed", **tsne_params), + ), + ), + ( + "TSNE with NMSlibTransformer", + make_pipeline( + NMSlibTransformer(n_neighbors=n_neighbors, metric=metric), + TSNE(metric="precomputed", **tsne_params), + ), + ), +] + +# init the plot +nrows = len(datasets) +ncols = np.sum([1 for name, model in transformers if "TSNE" in name]) +fig, axes = plt.subplots( + nrows=nrows, ncols=ncols, squeeze=False, figsize=(5 * ncols, 4 * nrows) +) +axes = axes.ravel() +i_ax = 0 + +for dataset_name, (X, y) in datasets: + msg = f"Benchmarking on {dataset_name}:" + print(f"\n{msg}\n" + str("-" * len(msg))) + + for transformer_name, transformer in transformers: + longest = np.max([len(name) for name, model in transformers]) + start = time.time() + Xt = transformer.fit_transform(X) + transform_duration = time.time() - start + print( + f"{transformer_name:<{longest}} {transform_duration:.3f} sec" + " (fit_transform)" + ) + + # plot TSNE embedding which should be very similar across methods + axes[i_ax].set_title(transformer_name + "\non " + dataset_name) + axes[i_ax].scatter( + Xt[:, 0], + Xt[:, 1], + c=y.astype(np.int32), + alpha=0.2, + cmap=plt.cm.viridis, + ) + axes[i_ax].xaxis.set_major_formatter(NullFormatter()) + axes[i_ax].yaxis.set_major_formatter(NullFormatter()) + axes[i_ax].axis("tight") + i_ax += 1 + +fig.tight_layout() +plt.show() + +# %% +# Sample output:: +# +# Benchmarking on MNIST_10000: +# ---------------------------- +# TSNE with internal NearestNeighbors 24.828 sec (fit_transform) +# TSNE with KNeighborsTransformer 20.111 sec (fit_transform) +# TSNE with NMSlibTransformer 21.757 sec (fit_transform) +# +# Benchmarking on MNIST_20000: +# ---------------------------- +# TSNE with internal NearestNeighbors 51.955 sec (fit_transform) +# TSNE with KNeighborsTransformer 50.994 sec (fit_transform) +# TSNE with NMSlibTransformer 43.536 sec (fit_transform) +# +# We can observe that the default :class:`~sklearn.manifold.TSNE` estimator with +# its internal :class:`~sklearn.neighbors.NearestNeighbors` implementation is +# roughly equivalent to the pipeline with :class:`~sklearn.manifold.TSNE` and +# :class:`~sklearn.neighbors.KNeighborsTransformer` in terms of performance. +# This is expected because both pipelines rely internally on the same +# :class:`~sklearn.neighbors.NearestNeighbors` implementation that performs +# exacts neighbors search. The approximate `NMSlibTransformer` is already +# slightly faster than the exact search on the smallest dataset but this speed +# difference is expected to become more significant on datasets with a larger +# number of samples. +# +# Notice however that not all approximate search methods are guaranteed to +# improve the speed of the default exact search method: indeed the exact search +# implementation significantly improved since scikit-learn 1.1. Furthermore, the +# brute-force exact search method does not require building an index at `fit` +# time. So, to get an overall performance improvement in the context of the +# :class:`~sklearn.manifold.TSNE` pipeline, the gains of the approximate search +# at `transform` need to be larger than the extra time spent to build the +# approximate search index at `fit` time. +# +# Finally, the TSNE algorithm itself is also computationally intensive, +# irrespective of the nearest neighbors search. So speeding-up the nearest +# neighbors search step by a factor of 5 would not result in a speed up by a +# factor of 5 for the overall pipeline. diff --git a/examples/neighbors/plot_caching_nearest_neighbors.py b/examples/neighbors/plot_caching_nearest_neighbors.py index a97ed3f2983a4..f3a7468871b26 100644 --- a/examples/neighbors/plot_caching_nearest_neighbors.py +++ b/examples/neighbors/plot_caching_nearest_neighbors.py @@ -3,7 +3,7 @@ Caching nearest neighbors ========================= -This examples demonstrates how to precompute the k nearest neighbors before +This example demonstrates how to precompute the k nearest neighbors before using them in KNeighborsClassifier. KNeighborsClassifier can compute the nearest neighbors internally, but precomputing them can have several benefits, such as finer parameter control, caching for multiple use, or custom @@ -11,54 +11,60 @@ Here we use the caching property of pipelines to cache the nearest neighbors graph between multiple fits of KNeighborsClassifier. The first call is slow -since it computes the neighbors graph, while subsequent call are faster as they +since it computes the neighbors graph, while subsequent calls are faster as they do not need to recompute the graph. Here the durations are small since the dataset is small, but the gain can be more substantial when the dataset grows larger, or when the grid of parameter to search is large. + """ -# Author: Tom Dupre la Tour -# -# License: BSD 3 clause + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + from tempfile import TemporaryDirectory + import matplotlib.pyplot as plt -from sklearn.neighbors import KNeighborsTransformer, KNeighborsClassifier -from sklearn.model_selection import GridSearchCV from sklearn.datasets import load_digits +from sklearn.model_selection import GridSearchCV +from sklearn.neighbors import KNeighborsClassifier, KNeighborsTransformer from sklearn.pipeline import Pipeline -print(__doc__) - X, y = load_digits(return_X_y=True) n_neighbors_list = [1, 2, 3, 4, 5, 6, 7, 8, 9] # The transformer computes the nearest neighbors graph using the maximum number # of neighbors necessary in the grid search. The classifier model filters the # nearest neighbors graph as required by its own n_neighbors parameter. -graph_model = KNeighborsTransformer(n_neighbors=max(n_neighbors_list), - mode='distance') -classifier_model = KNeighborsClassifier(metric='precomputed') +graph_model = KNeighborsTransformer(n_neighbors=max(n_neighbors_list), mode="distance") +classifier_model = KNeighborsClassifier(metric="precomputed") # Note that we give `memory` a directory to cache the graph computation # that will be used several times when tuning the hyperparameters of the # classifier. with TemporaryDirectory(prefix="sklearn_graph_cache_") as tmpdir: full_model = Pipeline( - steps=[('graph', graph_model), ('classifier', classifier_model)], - memory=tmpdir) + steps=[("graph", graph_model), ("classifier", classifier_model)], memory=tmpdir + ) - param_grid = {'classifier__n_neighbors': n_neighbors_list} + param_grid = {"classifier__n_neighbors": n_neighbors_list} grid_model = GridSearchCV(full_model, param_grid) grid_model.fit(X, y) # Plot the results of the grid search. fig, axes = plt.subplots(1, 2, figsize=(8, 4)) -axes[0].errorbar(x=n_neighbors_list, - y=grid_model.cv_results_['mean_test_score'], - yerr=grid_model.cv_results_['std_test_score']) -axes[0].set(xlabel='n_neighbors', title='Classification accuracy') -axes[1].errorbar(x=n_neighbors_list, y=grid_model.cv_results_['mean_fit_time'], - yerr=grid_model.cv_results_['std_fit_time'], color='r') -axes[1].set(xlabel='n_neighbors', title='Fit time (with caching)') +axes[0].errorbar( + x=n_neighbors_list, + y=grid_model.cv_results_["mean_test_score"], + yerr=grid_model.cv_results_["std_test_score"], +) +axes[0].set(xlabel="n_neighbors", title="Classification accuracy") +axes[1].errorbar( + x=n_neighbors_list, + y=grid_model.cv_results_["mean_fit_time"], + yerr=grid_model.cv_results_["std_fit_time"], + color="r", +) +axes[1].set(xlabel="n_neighbors", title="Fit time (with caching)") fig.tight_layout() plt.show() diff --git a/examples/neighbors/plot_classification.py b/examples/neighbors/plot_classification.py index 0b8d828225f39..1754869943ac7 100644 --- a/examples/neighbors/plot_classification.py +++ b/examples/neighbors/plot_classification.py @@ -3,59 +3,95 @@ Nearest Neighbors Classification ================================ -Sample usage of Nearest Neighbors classification. -It will plot the decision boundaries for each class. +This example shows how to use :class:`~sklearn.neighbors.KNeighborsClassifier`. +We train such a classifier on the iris dataset and observe the difference of the +decision boundary obtained with regards to the parameter `weights`. """ -print(__doc__) -import numpy as np -import matplotlib.pyplot as plt -import seaborn as sns -from matplotlib.colors import ListedColormap -from sklearn import neighbors, datasets - -n_neighbors = 15 +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -# import some data to play with -iris = datasets.load_iris() +# %% +# Load the data +# ------------- +# +# In this example, we use the iris dataset. We split the data into a train and test +# dataset. +from sklearn.datasets import load_iris +from sklearn.model_selection import train_test_split -# we only take the first two features. We could avoid this ugly -# slicing by using a two-dim dataset -X = iris.data[:, :2] +iris = load_iris(as_frame=True) +X = iris.data[["sepal length (cm)", "sepal width (cm)"]] y = iris.target +X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) + +# %% +# K-nearest neighbors classifier +# ------------------------------ +# +# We want to use a k-nearest neighbors classifier considering a neighborhood of 11 data +# points. Since our k-nearest neighbors model uses euclidean distance to find the +# nearest neighbors, it is therefore important to scale the data beforehand. Refer to +# the example entitled +# :ref:`sphx_glr_auto_examples_preprocessing_plot_scaling_importance.py` for more +# detailed information. +# +# Thus, we use a :class:`~sklearn.pipeline.Pipeline` to chain a scaler before to use +# our classifier. +from sklearn.neighbors import KNeighborsClassifier +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import StandardScaler -h = .02 # step size in the mesh - -# Create color maps -cmap_light = ListedColormap(['orange', 'cyan', 'cornflowerblue']) -cmap_bold = ['darkorange', 'c', 'darkblue'] - -for weights in ['uniform', 'distance']: - # we create an instance of Neighbours Classifier and fit the data. - clf = neighbors.KNeighborsClassifier(n_neighbors, weights=weights) - clf.fit(X, y) - - # Plot the decision boundary. For that, we will assign a color to each - # point in the mesh [x_min, x_max]x[y_min, y_max]. - x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1 - y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1 - xx, yy = np.meshgrid(np.arange(x_min, x_max, h), - np.arange(y_min, y_max, h)) - Z = clf.predict(np.c_[xx.ravel(), yy.ravel()]) - - # Put the result into a color plot - Z = Z.reshape(xx.shape) - plt.figure(figsize=(8, 6)) - plt.contourf(xx, yy, Z, cmap=cmap_light) - - # Plot also the training points - sns.scatterplot(x=X[:, 0], y=X[:, 1], hue=iris.target_names[y], - palette=cmap_bold, alpha=1.0, edgecolor="black") - plt.xlim(xx.min(), xx.max()) - plt.ylim(yy.min(), yy.max()) - plt.title("3-Class classification (k = %i, weights = '%s')" - % (n_neighbors, weights)) - plt.xlabel(iris.feature_names[0]) - plt.ylabel(iris.feature_names[1]) +clf = Pipeline( + steps=[("scaler", StandardScaler()), ("knn", KNeighborsClassifier(n_neighbors=11))] +) + +# %% +# Decision boundary +# ----------------- +# +# Now, we fit two classifiers with different values of the parameter +# `weights`. We plot the decision boundary of each classifier as well as the original +# dataset to observe the difference. +import matplotlib.pyplot as plt + +from sklearn.inspection import DecisionBoundaryDisplay + +_, axs = plt.subplots(ncols=2, figsize=(12, 5)) + +for ax, weights in zip(axs, ("uniform", "distance")): + clf.set_params(knn__weights=weights).fit(X_train, y_train) + disp = DecisionBoundaryDisplay.from_estimator( + clf, + X_test, + response_method="predict", + plot_method="pcolormesh", + xlabel=iris.feature_names[0], + ylabel=iris.feature_names[1], + shading="auto", + alpha=0.5, + ax=ax, + ) + scatter = disp.ax_.scatter(X.iloc[:, 0], X.iloc[:, 1], c=y, edgecolors="k") + disp.ax_.legend( + scatter.legend_elements()[0], + iris.target_names, + loc="lower left", + title="Classes", + ) + _ = disp.ax_.set_title( + f"3-Class classification\n(k={clf[-1].n_neighbors}, weights={weights!r})" + ) plt.show() + +# %% +# Conclusion +# ---------- +# +# We observe that the parameter `weights` has an impact on the decision boundary. When +# `weights="unifom"` all nearest neighbors will have the same impact on the decision. +# Whereas when `weights="distance"` the weight given to each neighbor is proportional +# to the inverse of the distance from that neighbor to the query point. +# +# In some cases, taking the distance into account might improve the model. diff --git a/examples/neighbors/plot_digits_kde_sampling.py b/examples/neighbors/plot_digits_kde_sampling.py index 8367d16b955fe..d4860f117e4e9 100644 --- a/examples/neighbors/plot_digits_kde_sampling.py +++ b/examples/neighbors/plot_digits_kde_sampling.py @@ -8,15 +8,19 @@ a generative model for a dataset. With this generative model in place, new samples can be drawn. These new samples reflect the underlying model of the data. + """ -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt +import numpy as np from sklearn.datasets import load_digits -from sklearn.neighbors import KernelDensity from sklearn.decomposition import PCA from sklearn.model_selection import GridSearchCV +from sklearn.neighbors import KernelDensity # load the data digits = load_digits() @@ -26,7 +30,7 @@ data = pca.fit_transform(digits.data) # use grid search cross-validation to optimize the bandwidth -params = {'bandwidth': np.logspace(-1, 1, 20)} +params = {"bandwidth": np.logspace(-1, 1, 20)} grid = GridSearchCV(KernelDensity(), params) grid.fit(data) @@ -48,14 +52,16 @@ for j in range(11): ax[4, j].set_visible(False) for i in range(4): - im = ax[i, j].imshow(real_data[i, j].reshape((8, 8)), - cmap=plt.cm.binary, interpolation='nearest') + im = ax[i, j].imshow( + real_data[i, j].reshape((8, 8)), cmap=plt.cm.binary, interpolation="nearest" + ) im.set_clim(0, 16) - im = ax[i + 5, j].imshow(new_data[i, j].reshape((8, 8)), - cmap=plt.cm.binary, interpolation='nearest') + im = ax[i + 5, j].imshow( + new_data[i, j].reshape((8, 8)), cmap=plt.cm.binary, interpolation="nearest" + ) im.set_clim(0, 16) -ax[0, 5].set_title('Selection from the input data') +ax[0, 5].set_title("Selection from the input data") ax[5, 5].set_title('"New" digits drawn from the kernel density model') plt.show() diff --git a/examples/neighbors/plot_kde_1d.py b/examples/neighbors/plot_kde_1d.py index 27ce99c25d3f3..ed5a454e476ad 100644 --- a/examples/neighbors/plot_kde_1d.py +++ b/examples/neighbors/plot_kde_1d.py @@ -25,28 +25,25 @@ samples in 1 dimension. Though this example uses 1D distributions, kernel density estimation is easily and efficiently extensible to higher dimensions as well. + """ -# Author: Jake Vanderplas -# -import numpy as np -import matplotlib + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt +import numpy as np from scipy.stats import norm -from sklearn.neighbors import KernelDensity -from sklearn.utils.fixes import parse_version -# `normed` is being deprecated in favor of `density` in histograms -if parse_version(matplotlib.__version__) >= parse_version('2.1'): - density_param = {'density': True} -else: - density_param = {'normed': True} +from sklearn.neighbors import KernelDensity # ---------------------------------------------------------------------- # Plot the progression of histograms to kernels np.random.seed(1) N = 20 -X = np.concatenate((np.random.normal(0, 1, int(0.3 * N)), - np.random.normal(5, 1, int(0.7 * N))))[:, np.newaxis] +X = np.concatenate( + (np.random.normal(0, 1, int(0.3 * N)), np.random.normal(5, 1, int(0.7 * N))) +)[:, np.newaxis] X_plot = np.linspace(-5, 10, 1000)[:, np.newaxis] bins = np.linspace(-5, 10, 10) @@ -54,35 +51,35 @@ fig.subplots_adjust(hspace=0.05, wspace=0.05) # histogram 1 -ax[0, 0].hist(X[:, 0], bins=bins, fc='#AAAAFF', **density_param) +ax[0, 0].hist(X[:, 0], bins=bins, fc="#AAAAFF", density=True) ax[0, 0].text(-3.5, 0.31, "Histogram") # histogram 2 -ax[0, 1].hist(X[:, 0], bins=bins + 0.75, fc='#AAAAFF', **density_param) +ax[0, 1].hist(X[:, 0], bins=bins + 0.75, fc="#AAAAFF", density=True) ax[0, 1].text(-3.5, 0.31, "Histogram, bins shifted") # tophat KDE -kde = KernelDensity(kernel='tophat', bandwidth=0.75).fit(X) +kde = KernelDensity(kernel="tophat", bandwidth=0.75).fit(X) log_dens = kde.score_samples(X_plot) -ax[1, 0].fill(X_plot[:, 0], np.exp(log_dens), fc='#AAAAFF') +ax[1, 0].fill(X_plot[:, 0], np.exp(log_dens), fc="#AAAAFF") ax[1, 0].text(-3.5, 0.31, "Tophat Kernel Density") # Gaussian KDE -kde = KernelDensity(kernel='gaussian', bandwidth=0.75).fit(X) +kde = KernelDensity(kernel="gaussian", bandwidth=0.75).fit(X) log_dens = kde.score_samples(X_plot) -ax[1, 1].fill(X_plot[:, 0], np.exp(log_dens), fc='#AAAAFF') +ax[1, 1].fill(X_plot[:, 0], np.exp(log_dens), fc="#AAAAFF") ax[1, 1].text(-3.5, 0.31, "Gaussian Kernel Density") for axi in ax.ravel(): - axi.plot(X[:, 0], np.full(X.shape[0], -0.01), '+k') + axi.plot(X[:, 0], np.full(X.shape[0], -0.01), "+k") axi.set_xlim(-4, 9) axi.set_ylim(-0.02, 0.34) for axi in ax[:, 0]: - axi.set_ylabel('Normalized Density') + axi.set_ylabel("Normalized Density") for axi in ax[1, :]: - axi.set_xlabel('x') + axi.set_xlabel("x") # ---------------------------------------------------------------------- # Plot all available kernels @@ -95,19 +92,21 @@ def format_func(x, loc): if x == 0: - return '0' + return "0" elif x == 1: - return 'h' + return "h" elif x == -1: - return '-h' + return "-h" else: - return '%ih' % x + return "%ih" % x + -for i, kernel in enumerate(['gaussian', 'tophat', 'epanechnikov', - 'exponential', 'linear', 'cosine']): +for i, kernel in enumerate( + ["gaussian", "tophat", "epanechnikov", "exponential", "linear", "cosine"] +): axi = ax.ravel()[i] log_dens = KernelDensity(kernel=kernel).fit(X_src).score_samples(X_plot) - axi.fill(X_plot[:, 0], np.exp(log_dens), '-k', fc='#AAAAFF') + axi.fill(X_plot[:, 0], np.exp(log_dens), "-k", fc="#AAAAFF") axi.text(-2.6, 0.95, kernel) axi.xaxis.set_major_formatter(plt.FuncFormatter(format_func)) @@ -117,37 +116,42 @@ def format_func(x, loc): axi.set_ylim(0, 1.05) axi.set_xlim(-2.9, 2.9) -ax[0, 1].set_title('Available Kernels') +ax[0, 1].set_title("Available Kernels") # ---------------------------------------------------------------------- # Plot a 1D density example N = 100 np.random.seed(1) -X = np.concatenate((np.random.normal(0, 1, int(0.3 * N)), - np.random.normal(5, 1, int(0.7 * N))))[:, np.newaxis] +X = np.concatenate( + (np.random.normal(0, 1, int(0.3 * N)), np.random.normal(5, 1, int(0.7 * N))) +)[:, np.newaxis] X_plot = np.linspace(-5, 10, 1000)[:, np.newaxis] -true_dens = (0.3 * norm(0, 1).pdf(X_plot[:, 0]) - + 0.7 * norm(5, 1).pdf(X_plot[:, 0])) +true_dens = 0.3 * norm(0, 1).pdf(X_plot[:, 0]) + 0.7 * norm(5, 1).pdf(X_plot[:, 0]) fig, ax = plt.subplots() -ax.fill(X_plot[:, 0], true_dens, fc='black', alpha=0.2, - label='input distribution') -colors = ['navy', 'cornflowerblue', 'darkorange'] -kernels = ['gaussian', 'tophat', 'epanechnikov'] +ax.fill(X_plot[:, 0], true_dens, fc="black", alpha=0.2, label="input distribution") +colors = ["navy", "cornflowerblue", "darkorange"] +kernels = ["gaussian", "tophat", "epanechnikov"] lw = 2 for color, kernel in zip(colors, kernels): kde = KernelDensity(kernel=kernel, bandwidth=0.5).fit(X) log_dens = kde.score_samples(X_plot) - ax.plot(X_plot[:, 0], np.exp(log_dens), color=color, lw=lw, - linestyle='-', label="kernel = '{0}'".format(kernel)) + ax.plot( + X_plot[:, 0], + np.exp(log_dens), + color=color, + lw=lw, + linestyle="-", + label="kernel = '{0}'".format(kernel), + ) ax.text(6, 0.38, "N={0} points".format(N)) -ax.legend(loc='upper left') -ax.plot(X[:, 0], -0.005 - 0.01 * np.random.random(X.shape[0]), '+k') +ax.legend(loc="upper left") +ax.plot(X[:, 0], -0.005 - 0.01 * np.random.random(X.shape[0]), "+k") ax.set_xlim(-4, 9) ax.set_ylim(-0.02, 0.4) diff --git a/examples/neighbors/plot_lof_novelty_detection.py b/examples/neighbors/plot_lof_novelty_detection.py index 71c0736a256a5..85fb0c4f7ba75 100644 --- a/examples/neighbors/plot_lof_novelty_detection.py +++ b/examples/neighbors/plot_lof_novelty_detection.py @@ -20,16 +20,20 @@ so that other samples can be local outliers relative to this cluster, and 2) smaller than the maximum number of close by samples that can potentially be local outliers. -In practice, such informations are generally not available, and taking +In practice, such information is generally not available, and taking n_neighbors=20 appears to work well in general. + """ -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib +import matplotlib.lines as mlines import matplotlib.pyplot as plt -from sklearn.neighbors import LocalOutlierFactor +import numpy as np -print(__doc__) +from sklearn.neighbors import LocalOutlierFactor np.random.seed(42) @@ -60,24 +64,30 @@ plt.title("Novelty Detection with LOF") plt.contourf(xx, yy, Z, levels=np.linspace(Z.min(), 0, 7), cmap=plt.cm.PuBu) -a = plt.contour(xx, yy, Z, levels=[0], linewidths=2, colors='darkred') -plt.contourf(xx, yy, Z, levels=[0, Z.max()], colors='palevioletred') +a = plt.contour(xx, yy, Z, levels=[0], linewidths=2, colors="darkred") +plt.contourf(xx, yy, Z, levels=[0, Z.max()], colors="palevioletred") s = 40 -b1 = plt.scatter(X_train[:, 0], X_train[:, 1], c='white', s=s, edgecolors='k') -b2 = plt.scatter(X_test[:, 0], X_test[:, 1], c='blueviolet', s=s, - edgecolors='k') -c = plt.scatter(X_outliers[:, 0], X_outliers[:, 1], c='gold', s=s, - edgecolors='k') -plt.axis('tight') +b1 = plt.scatter(X_train[:, 0], X_train[:, 1], c="white", s=s, edgecolors="k") +b2 = plt.scatter(X_test[:, 0], X_test[:, 1], c="blueviolet", s=s, edgecolors="k") +c = plt.scatter(X_outliers[:, 0], X_outliers[:, 1], c="gold", s=s, edgecolors="k") +plt.axis("tight") plt.xlim((-5, 5)) plt.ylim((-5, 5)) -plt.legend([a.collections[0], b1, b2, c], - ["learned frontier", "training observations", - "new regular observations", "new abnormal observations"], - loc="upper left", - prop=matplotlib.font_manager.FontProperties(size=11)) +plt.legend( + [mlines.Line2D([], [], color="darkred"), b1, b2, c], + [ + "learned frontier", + "training observations", + "new regular observations", + "new abnormal observations", + ], + loc=(1.05, 0.4), + prop=matplotlib.font_manager.FontProperties(size=11), +) plt.xlabel( "errors novel regular: %d/40 ; errors novel abnormal: %d/40" - % (n_error_test, n_error_outliers)) + % (n_error_test, n_error_outliers) +) +plt.tight_layout() plt.show() diff --git a/examples/neighbors/plot_lof_outlier_detection.py b/examples/neighbors/plot_lof_outlier_detection.py index 6f0e5bb490b9b..9b5e92579625b 100644 --- a/examples/neighbors/plot_lof_outlier_detection.py +++ b/examples/neighbors/plot_lof_outlier_detection.py @@ -6,36 +6,36 @@ The Local Outlier Factor (LOF) algorithm is an unsupervised anomaly detection method which computes the local density deviation of a given data point with respect to its neighbors. It considers as outliers the samples that have a -substantially lower density than their neighbors. This example shows how to -use LOF for outlier detection which is the default use case of this estimator -in scikit-learn. Note that when LOF is used for outlier detection it has no -predict, decision_function and score_samples methods. See -:ref:`User Guide `: for details on the difference between -outlier detection and novelty detection and how to use LOF for novelty -detection. - -The number of neighbors considered (parameter n_neighbors) is typically -set 1) greater than the minimum number of samples a cluster has to contain, -so that other samples can be local outliers relative to this cluster, and 2) -smaller than the maximum number of close by samples that can potentially be -local outliers. -In practice, such informations are generally not available, and taking -n_neighbors=20 appears to work well in general. +substantially lower density than their neighbors. This example shows how to use +LOF for outlier detection which is the default use case of this estimator in +scikit-learn. Note that when LOF is used for outlier detection it has no +`predict`, `decision_function` and `score_samples` methods. See the :ref:`User +Guide ` for details on the difference between outlier +detection and novelty detection and how to use LOF for novelty detection. + +The number of neighbors considered (parameter `n_neighbors`) is typically set 1) +greater than the minimum number of samples a cluster has to contain, so that +other samples can be local outliers relative to this cluster, and 2) smaller +than the maximum number of close by samples that can potentially be local +outliers. In practice, such information is generally not available, and taking +`n_neighbors=20` appears to work well in general. + """ -import numpy as np -import matplotlib.pyplot as plt -from sklearn.neighbors import LocalOutlierFactor +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -print(__doc__) +# %% +# Generate data with outliers +# --------------------------- + +# %% +import numpy as np np.random.seed(42) -# Generate train data X_inliers = 0.3 * np.random.randn(100, 2) X_inliers = np.r_[X_inliers + 2, X_inliers - 2] - -# Generate some outliers X_outliers = np.random.uniform(low=-4, high=4, size=(20, 2)) X = np.r_[X_inliers, X_outliers] @@ -43,26 +43,53 @@ ground_truth = np.ones(len(X), dtype=int) ground_truth[-n_outliers:] = -1 -# fit the model for outlier detection (default) +# %% +# Fit the model for outlier detection (default) +# --------------------------------------------- +# +# Use `fit_predict` to compute the predicted labels of the training samples +# (when LOF is used for outlier detection, the estimator has no `predict`, +# `decision_function` and `score_samples` methods). + +from sklearn.neighbors import LocalOutlierFactor + clf = LocalOutlierFactor(n_neighbors=20, contamination=0.1) -# use fit_predict to compute the predicted labels of the training samples -# (when LOF is used for outlier detection, the estimator has no predict, -# decision_function and score_samples methods). y_pred = clf.fit_predict(X) n_errors = (y_pred != ground_truth).sum() X_scores = clf.negative_outlier_factor_ -plt.title("Local Outlier Factor (LOF)") -plt.scatter(X[:, 0], X[:, 1], color='k', s=3., label='Data points') +# %% +# Plot results +# ------------ + +# %% +import matplotlib.pyplot as plt +from matplotlib.legend_handler import HandlerPathCollection + + +def update_legend_marker_size(handle, orig): + "Customize size of the legend marker" + handle.update_from(orig) + handle.set_sizes([20]) + + +plt.scatter(X[:, 0], X[:, 1], color="k", s=3.0, label="Data points") # plot circles with radius proportional to the outlier scores radius = (X_scores.max() - X_scores) / (X_scores.max() - X_scores.min()) -plt.scatter(X[:, 0], X[:, 1], s=1000 * radius, edgecolors='r', - facecolors='none', label='Outlier scores') -plt.axis('tight') +scatter = plt.scatter( + X[:, 0], + X[:, 1], + s=1000 * radius, + edgecolors="r", + facecolors="none", + label="Outlier scores", +) +plt.axis("tight") plt.xlim((-5, 5)) plt.ylim((-5, 5)) plt.xlabel("prediction errors: %d" % (n_errors)) -legend = plt.legend(loc='upper left') -legend.legendHandles[0]._sizes = [10] -legend.legendHandles[1]._sizes = [20] +plt.legend( + handler_map={scatter: HandlerPathCollection(update_func=update_legend_marker_size)} +) +plt.title("Local Outlier Factor (LOF)") plt.show() diff --git a/examples/neighbors/plot_nca_classification.py b/examples/neighbors/plot_nca_classification.py index 5536e8eb69e89..b8d69b82fec42 100644 --- a/examples/neighbors/plot_nca_classification.py +++ b/examples/neighbors/plot_nca_classification.py @@ -12,22 +12,21 @@ Components Analysis. The latter aims to find a linear transformation that maximises the (stochastic) nearest neighbor classification accuracy on the training set. + """ -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt from matplotlib.colors import ListedColormap + from sklearn import datasets +from sklearn.inspection import DecisionBoundaryDisplay from sklearn.model_selection import train_test_split -from sklearn.preprocessing import StandardScaler -from sklearn.neighbors import (KNeighborsClassifier, - NeighborhoodComponentsAnalysis) +from sklearn.neighbors import KNeighborsClassifier, NeighborhoodComponentsAnalysis from sklearn.pipeline import Pipeline - - -print(__doc__) +from sklearn.preprocessing import StandardScaler n_neighbors = 1 @@ -38,51 +37,61 @@ # slicing by using a two-dim dataset X = X[:, [0, 2]] -X_train, X_test, y_train, y_test = \ - train_test_split(X, y, stratify=y, test_size=0.7, random_state=42) +X_train, X_test, y_train, y_test = train_test_split( + X, y, stratify=y, test_size=0.7, random_state=42 +) -h = .01 # step size in the mesh +h = 0.05 # step size in the mesh # Create color maps -cmap_light = ListedColormap(['#FFAAAA', '#AAFFAA', '#AAAAFF']) -cmap_bold = ListedColormap(['#FF0000', '#00FF00', '#0000FF']) - -names = ['KNN', 'NCA, KNN'] - -classifiers = [Pipeline([('scaler', StandardScaler()), - ('knn', KNeighborsClassifier(n_neighbors=n_neighbors)) - ]), - Pipeline([('scaler', StandardScaler()), - ('nca', NeighborhoodComponentsAnalysis()), - ('knn', KNeighborsClassifier(n_neighbors=n_neighbors)) - ]) - ] - -x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1 -y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1 -xx, yy = np.meshgrid(np.arange(x_min, x_max, h), - np.arange(y_min, y_max, h)) +cmap_light = ListedColormap(["#FFAAAA", "#AAFFAA", "#AAAAFF"]) +cmap_bold = ListedColormap(["#FF0000", "#00FF00", "#0000FF"]) + +names = ["KNN", "NCA, KNN"] + +classifiers = [ + Pipeline( + [ + ("scaler", StandardScaler()), + ("knn", KNeighborsClassifier(n_neighbors=n_neighbors)), + ] + ), + Pipeline( + [ + ("scaler", StandardScaler()), + ("nca", NeighborhoodComponentsAnalysis()), + ("knn", KNeighborsClassifier(n_neighbors=n_neighbors)), + ] + ), +] for name, clf in zip(names, classifiers): - clf.fit(X_train, y_train) score = clf.score(X_test, y_test) - # Plot the decision boundary. For that, we will assign a color to each - # point in the mesh [x_min, x_max]x[y_min, y_max]. - Z = clf.predict(np.c_[xx.ravel(), yy.ravel()]) - - # Put the result into a color plot - Z = Z.reshape(xx.shape) - plt.figure() - plt.pcolormesh(xx, yy, Z, cmap=cmap_light, alpha=.8) + _, ax = plt.subplots() + DecisionBoundaryDisplay.from_estimator( + clf, + X, + cmap=cmap_light, + alpha=0.8, + ax=ax, + response_method="predict", + plot_method="pcolormesh", + shading="auto", + ) # Plot also the training and testing points - plt.scatter(X[:, 0], X[:, 1], c=y, cmap=cmap_bold, edgecolor='k', s=20) - plt.xlim(xx.min(), xx.max()) - plt.ylim(yy.min(), yy.max()) + plt.scatter(X[:, 0], X[:, 1], c=y, cmap=cmap_bold, edgecolor="k", s=20) plt.title("{} (k = {})".format(name, n_neighbors)) - plt.text(0.9, 0.1, '{:.2f}'.format(score), size=15, - ha='center', va='center', transform=plt.gca().transAxes) + plt.text( + 0.9, + 0.1, + "{:.2f}".format(score), + size=15, + ha="center", + va="center", + transform=plt.gca().transAxes, + ) plt.show() diff --git a/examples/neighbors/plot_nca_dim_reduction.py b/examples/neighbors/plot_nca_dim_reduction.py index 64135f76ee58e..fcf2b0f602d20 100644 --- a/examples/neighbors/plot_nca_dim_reduction.py +++ b/examples/neighbors/plot_nca_dim_reduction.py @@ -25,23 +25,23 @@ One can see that NCA enforces a clustering of the data that is visually meaningful despite the large reduction in dimension. + """ -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt +import numpy as np + from sklearn import datasets -from sklearn.model_selection import train_test_split from sklearn.decomposition import PCA from sklearn.discriminant_analysis import LinearDiscriminantAnalysis -from sklearn.neighbors import (KNeighborsClassifier, - NeighborhoodComponentsAnalysis) +from sklearn.model_selection import train_test_split +from sklearn.neighbors import KNeighborsClassifier, NeighborhoodComponentsAnalysis from sklearn.pipeline import make_pipeline from sklearn.preprocessing import StandardScaler -print(__doc__) - n_neighbors = 3 random_state = 0 @@ -49,31 +49,30 @@ X, y = datasets.load_digits(return_X_y=True) # Split into train/test -X_train, X_test, y_train, y_test = \ - train_test_split(X, y, test_size=0.5, stratify=y, - random_state=random_state) +X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.5, stratify=y, random_state=random_state +) dim = len(X[0]) n_classes = len(np.unique(y)) # Reduce dimension to 2 with PCA -pca = make_pipeline(StandardScaler(), - PCA(n_components=2, random_state=random_state)) +pca = make_pipeline(StandardScaler(), PCA(n_components=2, random_state=random_state)) # Reduce dimension to 2 with LinearDiscriminantAnalysis -lda = make_pipeline(StandardScaler(), - LinearDiscriminantAnalysis(n_components=2)) +lda = make_pipeline(StandardScaler(), LinearDiscriminantAnalysis(n_components=2)) # Reduce dimension to 2 with NeighborhoodComponentAnalysis -nca = make_pipeline(StandardScaler(), - NeighborhoodComponentsAnalysis(n_components=2, - random_state=random_state)) +nca = make_pipeline( + StandardScaler(), + NeighborhoodComponentsAnalysis(n_components=2, random_state=random_state), +) # Use a nearest neighbor classifier to evaluate the methods knn = KNeighborsClassifier(n_neighbors=n_neighbors) # Make a list of the methods to be compared -dim_reduction_methods = [('PCA', pca), ('LDA', lda), ('NCA', nca)] +dim_reduction_methods = [("PCA", pca), ("LDA", lda), ("NCA", nca)] # plt.figure() for i, (name, model) in enumerate(dim_reduction_methods): @@ -93,8 +92,8 @@ X_embedded = model.transform(X) # Plot the projected points and show the evaluation score - plt.scatter(X_embedded[:, 0], X_embedded[:, 1], c=y, s=30, cmap='Set1') - plt.title("{}, KNN (k={})\nTest accuracy = {:.2f}".format(name, - n_neighbors, - acc_knn)) + plt.scatter(X_embedded[:, 0], X_embedded[:, 1], c=y, s=30, cmap="Set1") + plt.title( + "{}, KNN (k={})\nTest accuracy = {:.2f}".format(name, n_neighbors, acc_knn) + ) plt.show() diff --git a/examples/neighbors/plot_nca_illustration.py b/examples/neighbors/plot_nca_illustration.py index 1e1f435e7e57a..e67bdb4b2d4d7 100644 --- a/examples/neighbors/plot_nca_illustration.py +++ b/examples/neighbors/plot_nca_illustration.py @@ -7,18 +7,19 @@ the nearest neighbors classification accuracy. It provides a visual representation of this metric compared to the original point space. Please refer to the :ref:`User Guide ` for more information. + """ -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt -from sklearn.datasets import make_classification -from sklearn.neighbors import NeighborhoodComponentsAnalysis +import numpy as np from matplotlib import cm from scipy.special import logsumexp -print(__doc__) +from sklearn.datasets import make_classification +from sklearn.neighbors import NeighborhoodComponentsAnalysis # %% # Original points @@ -28,32 +29,37 @@ # point no. 3. The thickness of a link between point no. 3 and another point # is proportional to their distance. -X, y = make_classification(n_samples=9, n_features=2, n_informative=2, - n_redundant=0, n_classes=3, n_clusters_per_class=1, - class_sep=1.0, random_state=0) +X, y = make_classification( + n_samples=9, + n_features=2, + n_informative=2, + n_redundant=0, + n_classes=3, + n_clusters_per_class=1, + class_sep=1.0, + random_state=0, +) plt.figure(1) ax = plt.gca() for i in range(X.shape[0]): - ax.text(X[i, 0], X[i, 1], str(i), va='center', ha='center') + ax.text(X[i, 0], X[i, 1], str(i), va="center", ha="center") ax.scatter(X[i, 0], X[i, 1], s=300, c=cm.Set1(y[[i]]), alpha=0.4) ax.set_title("Original points") ax.axes.get_xaxis().set_visible(False) ax.axes.get_yaxis().set_visible(False) -ax.axis('equal') # so that boundaries are displayed correctly as circles +ax.axis("equal") # so that boundaries are displayed correctly as circles def link_thickness_i(X, i): diff_embedded = X[i] - X - dist_embedded = np.einsum('ij,ij->i', diff_embedded, - diff_embedded) + dist_embedded = np.einsum("ij,ij->i", diff_embedded, diff_embedded) dist_embedded[i] = np.inf # compute exponentiated distances (use the log-sum-exp trick to # avoid numerical instabilities - exp_dist_embedded = np.exp(-dist_embedded - - logsumexp(-dist_embedded)) + exp_dist_embedded = np.exp(-dist_embedded - logsumexp(-dist_embedded)) return exp_dist_embedded @@ -63,8 +69,7 @@ def relate_point(X, i, ax): thickness = link_thickness_i(X, i) if i != j: line = ([pt_i[0], pt_j[0]], [pt_i[1], pt_j[1]]) - ax.plot(*line, c=cm.Set1(y[j]), - linewidth=5*thickness[j]) + ax.plot(*line, c=cm.Set1(y[j]), linewidth=5 * thickness[j]) i = 3 @@ -87,13 +92,11 @@ def relate_point(X, i, ax): relate_point(X_embedded, i, ax2) for i in range(len(X)): - ax2.text(X_embedded[i, 0], X_embedded[i, 1], str(i), - va='center', ha='center') - ax2.scatter(X_embedded[i, 0], X_embedded[i, 1], s=300, c=cm.Set1(y[[i]]), - alpha=0.4) + ax2.text(X_embedded[i, 0], X_embedded[i, 1], str(i), va="center", ha="center") + ax2.scatter(X_embedded[i, 0], X_embedded[i, 1], s=300, c=cm.Set1(y[[i]]), alpha=0.4) ax2.set_title("NCA embedding") ax2.axes.get_xaxis().set_visible(False) ax2.axes.get_yaxis().set_visible(False) -ax2.axis('equal') +ax2.axis("equal") plt.show() diff --git a/examples/neighbors/plot_nearest_centroid.py b/examples/neighbors/plot_nearest_centroid.py index 04a105c0e07fd..1718e213f9252 100644 --- a/examples/neighbors/plot_nearest_centroid.py +++ b/examples/neighbors/plot_nearest_centroid.py @@ -5,17 +5,20 @@ Sample usage of Nearest Centroid classification. It will plot the decision boundaries for each class. + """ -print(__doc__) -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt +import numpy as np from matplotlib.colors import ListedColormap + from sklearn import datasets +from sklearn.inspection import DecisionBoundaryDisplay from sklearn.neighbors import NearestCentroid -n_neighbors = 15 - # import some data to play with iris = datasets.load_iris() # we only take the first two features. We could avoid this ugly @@ -23,36 +26,25 @@ X = iris.data[:, :2] y = iris.target -h = .02 # step size in the mesh - # Create color maps -cmap_light = ListedColormap(['orange', 'cyan', 'cornflowerblue']) -cmap_bold = ListedColormap(['darkorange', 'c', 'darkblue']) +cmap_light = ListedColormap(["orange", "cyan", "cornflowerblue"]) +cmap_bold = ListedColormap(["darkorange", "c", "darkblue"]) -for shrinkage in [None, .2]: - # we create an instance of Neighbours Classifier and fit the data. +for shrinkage in [None, 0.2]: + # we create an instance of Nearest Centroid Classifier and fit the data. clf = NearestCentroid(shrink_threshold=shrinkage) clf.fit(X, y) y_pred = clf.predict(X) print(shrinkage, np.mean(y == y_pred)) - # Plot the decision boundary. For that, we will assign a color to each - # point in the mesh [x_min, x_max]x[y_min, y_max]. - x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1 - y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1 - xx, yy = np.meshgrid(np.arange(x_min, x_max, h), - np.arange(y_min, y_max, h)) - Z = clf.predict(np.c_[xx.ravel(), yy.ravel()]) - - # Put the result into a color plot - Z = Z.reshape(xx.shape) - plt.figure() - plt.pcolormesh(xx, yy, Z, cmap=cmap_light) + + _, ax = plt.subplots() + DecisionBoundaryDisplay.from_estimator( + clf, X, cmap=cmap_light, ax=ax, response_method="predict" + ) # Plot also the training points - plt.scatter(X[:, 0], X[:, 1], c=y, cmap=cmap_bold, - edgecolor='k', s=20) - plt.title("3-Class classification (shrink_threshold=%r)" - % shrinkage) - plt.axis('tight') + plt.scatter(X[:, 0], X[:, 1], c=y, cmap=cmap_bold, edgecolor="k", s=20) + plt.title("3-Class classification (shrink_threshold=%r)" % shrinkage) + plt.axis("tight") plt.show() diff --git a/examples/neighbors/plot_regression.py b/examples/neighbors/plot_regression.py index 9625e205009aa..431540e81761b 100644 --- a/examples/neighbors/plot_regression.py +++ b/examples/neighbors/plot_regression.py @@ -6,45 +6,47 @@ Demonstrate the resolution of a regression problem using a k-Nearest Neighbor and the interpolation of the target using both barycenter and constant weights. - """ -print(__doc__) - -# Author: Alexandre Gramfort -# Fabian Pedregosa -# -# License: BSD 3 clause (C) INRIA +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -# ############################################################################# +# %% # Generate sample data -import numpy as np +# -------------------- +# Here we generate a few data points to use to train the model. We also generate +# data in the whole range of the training data to visualize how the model would +# react in that whole region. import matplotlib.pyplot as plt +import numpy as np + from sklearn import neighbors -np.random.seed(0) -X = np.sort(5 * np.random.rand(40, 1), axis=0) -T = np.linspace(0, 5, 500)[:, np.newaxis] -y = np.sin(X).ravel() +rng = np.random.RandomState(0) +X_train = np.sort(5 * rng.rand(40, 1), axis=0) +X_test = np.linspace(0, 5, 500)[:, np.newaxis] +y = np.sin(X_train).ravel() # Add noise to targets y[::5] += 1 * (0.5 - np.random.rand(8)) -# ############################################################################# +# %% # Fit regression model +# -------------------- +# Here we train a model and visualize how `uniform` and `distance` +# weights in prediction effect predicted values. n_neighbors = 5 -for i, weights in enumerate(['uniform', 'distance']): +for i, weights in enumerate(["uniform", "distance"]): knn = neighbors.KNeighborsRegressor(n_neighbors, weights=weights) - y_ = knn.fit(X, y).predict(T) + y_ = knn.fit(X_train, y).predict(X_test) plt.subplot(2, 1, i + 1) - plt.scatter(X, y, color='darkorange', label='data') - plt.plot(T, y_, color='navy', label='prediction') - plt.axis('tight') + plt.scatter(X_train, y, color="darkorange", label="data") + plt.plot(X_test, y_, color="navy", label="prediction") + plt.axis("tight") plt.legend() - plt.title("KNeighborsRegressor (k = %i, weights = '%s')" % (n_neighbors, - weights)) + plt.title("KNeighborsRegressor (k = %i, weights = '%s')" % (n_neighbors, weights)) plt.tight_layout() plt.show() diff --git a/examples/neighbors/plot_species_kde.py b/examples/neighbors/plot_species_kde.py index 5a5bbf400c0d8..a6c6808476673 100644 --- a/examples/neighbors/plot_species_kde.py +++ b/examples/neighbors/plot_species_kde.py @@ -10,37 +10,37 @@ `basemap `_ to plot the coast lines and national boundaries of South America. -This example does not perform any learning over the data -(see :ref:`sphx_glr_auto_examples_applications_plot_species_distribution_modeling.py` for -an example of classification based on the attributes in this dataset). It -simply shows the kernel density estimate of observed data points in -geospatial coordinates. +This example does not perform any learning over the data (see +:ref:`sphx_glr_auto_examples_applications_plot_species_distribution_modeling.py` for an +example of classification based on the attributes in this dataset). It simply shows the +kernel density estimate of observed data points in geospatial coordinates. The two species are: - - `"Bradypus variegatus" - `_ , - the Brown-throated Sloth. +- `"Bradypus variegatus" + `_ , + the Brown-throated Sloth. - - `"Microryzomys minutus" - `_ , - also known as the Forest Small Rice Rat, a rodent that lives in Peru, - Colombia, Ecuador, Peru, and Venezuela. +- `"Microryzomys minutus" + `_ , + also known as the Forest Small Rice Rat, a rodent that lives in Peru, + Colombia, Ecuador, Peru, and Venezuela. References ---------- - * `"Maximum entropy modeling of species geographic distributions" - `_ - S. J. Phillips, R. P. Anderson, R. E. Schapire - Ecological Modelling, - 190:231-259, 2006. +- `"Maximum entropy modeling of species geographic distributions" + `_ + S. J. Phillips, R. P. Anderson, R. E. Schapire - Ecological Modelling, + 190:231-259, 2006. """ -# Author: Jake Vanderplas -# -# License: BSD 3 clause -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt +import numpy as np + from sklearn.datasets import fetch_species_distributions from sklearn.neighbors import KernelDensity @@ -48,6 +48,7 @@ # otherwise, we'll improvise later... try: from mpl_toolkits.basemap import Basemap + basemap = True except ImportError: basemap = False @@ -82,13 +83,14 @@ def construct_grids(batch): # Get matrices/arrays of species IDs and locations data = fetch_species_distributions() -species_names = ['Bradypus Variegatus', 'Microryzomys Minutus'] +species_names = ["Bradypus Variegatus", "Microryzomys Minutus"] -Xtrain = np.vstack([data['train']['dd lat'], - data['train']['dd long']]).T -ytrain = np.array([d.decode('ascii').startswith('micro') - for d in data['train']['species']], dtype='int') -Xtrain *= np.pi / 180. # Convert lat/long to radians +Xtrain = np.vstack([data["train"]["dd lat"], data["train"]["dd long"]]).T +ytrain = np.array( + [d.decode("ascii").startswith("micro") for d in data["train"]["species"]], + dtype="int", +) +Xtrain *= np.pi / 180.0 # Convert lat/long to radians # Set up the data grid for the contour plot xgrid, ygrid = construct_grids(data) @@ -98,7 +100,7 @@ def construct_grids(batch): xy = np.vstack([Y.ravel(), X.ravel()]).T xy = xy[land_mask] -xy *= np.pi / 180. +xy *= np.pi / 180.0 # Plot map of South America with distributions of each species fig = plt.figure() @@ -109,12 +111,13 @@ def construct_grids(batch): # construct a kernel density estimate of the distribution print(" - computing KDE in spherical coordinates") - kde = KernelDensity(bandwidth=0.04, metric='haversine', - kernel='gaussian', algorithm='ball_tree') + kde = KernelDensity( + bandwidth=0.04, metric="haversine", kernel="gaussian", algorithm="ball_tree" + ) kde.fit(Xtrain[ytrain == i]) # evaluate only on the land: -9999 indicates ocean - Z = np.full(land_mask.shape[0], -9999, dtype='int') + Z = np.full(land_mask.shape[0], -9999, dtype="int") Z[land_mask] = np.exp(kde.score_samples(xy)) Z = Z.reshape(X.shape) @@ -124,16 +127,21 @@ def construct_grids(batch): if basemap: print(" - plot coastlines using basemap") - m = Basemap(projection='cyl', llcrnrlat=Y.min(), - urcrnrlat=Y.max(), llcrnrlon=X.min(), - urcrnrlon=X.max(), resolution='c') + m = Basemap( + projection="cyl", + llcrnrlat=Y.min(), + urcrnrlat=Y.max(), + llcrnrlon=X.min(), + urcrnrlon=X.max(), + resolution="c", + ) m.drawcoastlines() m.drawcountries() else: print(" - plot coastlines from coverage") - plt.contour(X, Y, land_reference, - levels=[-9998], colors="k", - linestyles="solid") + plt.contour( + X, Y, land_reference, levels=[-9998], colors="k", linestyles="solid" + ) plt.xticks([]) plt.yticks([]) diff --git a/examples/neural_networks/plot_mlp_alpha.py b/examples/neural_networks/plot_mlp_alpha.py index 15cd8e490efca..d5f35ea88ff96 100644 --- a/examples/neural_networks/plot_mlp_alpha.py +++ b/examples/neural_networks/plot_mlp_alpha.py @@ -14,63 +14,73 @@ Similarly, decreasing alpha may fix high bias (a sign of underfitting) by encouraging larger weights, potentially resulting in a more complicated decision boundary. -""" -print(__doc__) +""" -# Author: Issam H. Laradji -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import numpy as np from matplotlib import pyplot as plt from matplotlib.colors import ListedColormap + +from sklearn.datasets import make_circles, make_classification, make_moons from sklearn.model_selection import train_test_split -from sklearn.preprocessing import StandardScaler -from sklearn.datasets import make_moons, make_circles, make_classification from sklearn.neural_network import MLPClassifier from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import StandardScaler -h = .02 # step size in the mesh +h = 0.02 # step size in the mesh alphas = np.logspace(-1, 1, 5) classifiers = [] names = [] for alpha in alphas: - classifiers.append(make_pipeline( - StandardScaler(), - MLPClassifier( - solver='lbfgs', alpha=alpha, random_state=1, max_iter=2000, - early_stopping=True, hidden_layer_sizes=[100, 100], + classifiers.append( + make_pipeline( + StandardScaler(), + MLPClassifier( + solver="lbfgs", + alpha=alpha, + random_state=1, + max_iter=2000, + early_stopping=True, + hidden_layer_sizes=[10, 10], + ), ) - )) + ) names.append(f"alpha {alpha:.2f}") -X, y = make_classification(n_features=2, n_redundant=0, n_informative=2, - random_state=0, n_clusters_per_class=1) +X, y = make_classification( + n_features=2, n_redundant=0, n_informative=2, random_state=0, n_clusters_per_class=1 +) rng = np.random.RandomState(2) X += 2 * rng.uniform(size=X.shape) linearly_separable = (X, y) -datasets = [make_moons(noise=0.3, random_state=0), - make_circles(noise=0.2, factor=0.5, random_state=1), - linearly_separable] +datasets = [ + make_moons(noise=0.3, random_state=0), + make_circles(noise=0.2, factor=0.5, random_state=1), + linearly_separable, +] figure = plt.figure(figsize=(17, 9)) i = 1 # iterate over datasets for X, y in datasets: # split into training and test part - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.4) + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.4, random_state=42 + ) - x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5 - y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5 - xx, yy = np.meshgrid(np.arange(x_min, x_max, h), - np.arange(y_min, y_max, h)) + x_min, x_max = X[:, 0].min() - 0.5, X[:, 0].max() + 0.5 + y_min, y_max = X[:, 1].min() - 0.5, X[:, 1].max() + 0.5 + xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h)) # just plot the dataset first cm = plt.cm.RdBu - cm_bright = ListedColormap(['#FF0000', '#0000FF']) + cm_bright = ListedColormap(["#FF0000", "#0000FF"]) ax = plt.subplot(len(datasets), len(classifiers) + 1, i) # Plot the training points ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright) @@ -91,29 +101,47 @@ # Plot the decision boundary. For that, we will assign a color to each # point in the mesh [x_min, x_max] x [y_min, y_max]. if hasattr(clf, "decision_function"): - Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()]) + Z = clf.decision_function(np.column_stack([xx.ravel(), yy.ravel()])) else: - Z = clf.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:, 1] + Z = clf.predict_proba(np.column_stack([xx.ravel(), yy.ravel()]))[:, 1] # Put the result into a color plot Z = Z.reshape(xx.shape) - ax.contourf(xx, yy, Z, cmap=cm, alpha=.8) + ax.contourf(xx, yy, Z, cmap=cm, alpha=0.8) # Plot also the training points - ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright, - edgecolors='black', s=25) + ax.scatter( + X_train[:, 0], + X_train[:, 1], + c=y_train, + cmap=cm_bright, + edgecolors="black", + s=25, + ) # and testing points - ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright, - alpha=0.6, edgecolors='black', s=25) + ax.scatter( + X_test[:, 0], + X_test[:, 1], + c=y_test, + cmap=cm_bright, + alpha=0.6, + edgecolors="black", + s=25, + ) ax.set_xlim(xx.min(), xx.max()) ax.set_ylim(yy.min(), yy.max()) ax.set_xticks(()) ax.set_yticks(()) ax.set_title(name) - ax.text(xx.max() - .3, yy.min() + .3, ('%.2f' % score).lstrip('0'), - size=15, horizontalalignment='right') + ax.text( + xx.max() - 0.3, + yy.min() + 0.3, + f"{score:.3f}".lstrip("0"), + size=15, + horizontalalignment="right", + ) i += 1 -figure.subplots_adjust(left=.02, right=.98) +figure.subplots_adjust(left=0.02, right=0.98) plt.show() diff --git a/examples/neural_networks/plot_mlp_training_curves.py b/examples/neural_networks/plot_mlp_training_curves.py index 608db0f75ad98..7b63d0de1adfe 100644 --- a/examples/neural_networks/plot_mlp_training_curves.py +++ b/examples/neural_networks/plot_mlp_training_curves.py @@ -11,46 +11,85 @@ Note that those results can be highly dependent on the value of ``learning_rate_init``. + """ -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import warnings import matplotlib.pyplot as plt -from sklearn.neural_network import MLPClassifier -from sklearn.preprocessing import MinMaxScaler from sklearn import datasets from sklearn.exceptions import ConvergenceWarning +from sklearn.neural_network import MLPClassifier +from sklearn.preprocessing import MinMaxScaler # different learning rate schedules and momentum parameters -params = [{'solver': 'sgd', 'learning_rate': 'constant', 'momentum': 0, - 'learning_rate_init': 0.2}, - {'solver': 'sgd', 'learning_rate': 'constant', 'momentum': .9, - 'nesterovs_momentum': False, 'learning_rate_init': 0.2}, - {'solver': 'sgd', 'learning_rate': 'constant', 'momentum': .9, - 'nesterovs_momentum': True, 'learning_rate_init': 0.2}, - {'solver': 'sgd', 'learning_rate': 'invscaling', 'momentum': 0, - 'learning_rate_init': 0.2}, - {'solver': 'sgd', 'learning_rate': 'invscaling', 'momentum': .9, - 'nesterovs_momentum': True, 'learning_rate_init': 0.2}, - {'solver': 'sgd', 'learning_rate': 'invscaling', 'momentum': .9, - 'nesterovs_momentum': False, 'learning_rate_init': 0.2}, - {'solver': 'adam', 'learning_rate_init': 0.01}] - -labels = ["constant learning-rate", "constant with momentum", - "constant with Nesterov's momentum", - "inv-scaling learning-rate", "inv-scaling with momentum", - "inv-scaling with Nesterov's momentum", "adam"] - -plot_args = [{'c': 'red', 'linestyle': '-'}, - {'c': 'green', 'linestyle': '-'}, - {'c': 'blue', 'linestyle': '-'}, - {'c': 'red', 'linestyle': '--'}, - {'c': 'green', 'linestyle': '--'}, - {'c': 'blue', 'linestyle': '--'}, - {'c': 'black', 'linestyle': '-'}] +params = [ + { + "solver": "sgd", + "learning_rate": "constant", + "momentum": 0, + "learning_rate_init": 0.2, + }, + { + "solver": "sgd", + "learning_rate": "constant", + "momentum": 0.9, + "nesterovs_momentum": False, + "learning_rate_init": 0.2, + }, + { + "solver": "sgd", + "learning_rate": "constant", + "momentum": 0.9, + "nesterovs_momentum": True, + "learning_rate_init": 0.2, + }, + { + "solver": "sgd", + "learning_rate": "invscaling", + "momentum": 0, + "learning_rate_init": 0.2, + }, + { + "solver": "sgd", + "learning_rate": "invscaling", + "momentum": 0.9, + "nesterovs_momentum": False, + "learning_rate_init": 0.2, + }, + { + "solver": "sgd", + "learning_rate": "invscaling", + "momentum": 0.9, + "nesterovs_momentum": True, + "learning_rate_init": 0.2, + }, + {"solver": "adam", "learning_rate_init": 0.01}, +] + +labels = [ + "constant learning-rate", + "constant with momentum", + "constant with Nesterov's momentum", + "inv-scaling learning-rate", + "inv-scaling with momentum", + "inv-scaling with Nesterov's momentum", + "adam", +] + +plot_args = [ + {"c": "red", "linestyle": "-"}, + {"c": "green", "linestyle": "-"}, + {"c": "blue", "linestyle": "-"}, + {"c": "red", "linestyle": "--"}, + {"c": "green", "linestyle": "--"}, + {"c": "blue", "linestyle": "--"}, + {"c": "black", "linestyle": "-"}, +] def plot_on_dataset(X, y, ax, name): @@ -68,14 +107,14 @@ def plot_on_dataset(X, y, ax, name): for label, param in zip(labels, params): print("training: %s" % label) - mlp = MLPClassifier(random_state=0, - max_iter=max_iter, **param) + mlp = MLPClassifier(random_state=0, max_iter=max_iter, **param) # some parameter combinations will not converge as can be seen on the # plots so they are ignored here with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=ConvergenceWarning, - module="sklearn") + warnings.filterwarnings( + "ignore", category=ConvergenceWarning, module="sklearn" + ) mlp.fit(X, y) mlps.append(mlp) @@ -89,13 +128,16 @@ def plot_on_dataset(X, y, ax, name): # load / generate some toy datasets iris = datasets.load_iris() X_digits, y_digits = datasets.load_digits(return_X_y=True) -data_sets = [(iris.data, iris.target), - (X_digits, y_digits), - datasets.make_circles(noise=0.2, factor=0.5, random_state=1), - datasets.make_moons(noise=0.3, random_state=0)] - -for ax, data, name in zip(axes.ravel(), data_sets, ['iris', 'digits', - 'circles', 'moons']): +data_sets = [ + (iris.data, iris.target), + (X_digits, y_digits), + datasets.make_circles(noise=0.2, factor=0.5, random_state=1), + datasets.make_moons(noise=0.3, random_state=0), +] + +for ax, data, name in zip( + axes.ravel(), data_sets, ["iris", "digits", "circles", "moons"] +): plot_on_dataset(*data, ax=ax, name=name) fig.legend(ax.get_lines(), labels, ncol=3, loc="upper center") diff --git a/examples/neural_networks/plot_mnist_filters.py b/examples/neural_networks/plot_mnist_filters.py index 33f421a226c33..889e78e2e5e5b 100644 --- a/examples/neural_networks/plot_mnist_filters.py +++ b/examples/neural_networks/plot_mnist_filters.py @@ -12,43 +12,52 @@ MLPClassifier trained on the MNIST dataset. The input data consists of 28x28 pixel handwritten digits, leading to 784 -features in the dataset. Therefore the first layer weight matrix have the shape +features in the dataset. Therefore the first layer weight matrix has the shape (784, hidden_layer_sizes[0]). We can therefore visualize a single column of the weight matrix as a 28x28 pixel image. To make the example run faster, we use very few hidden units, and train only for a very short time. Training longer would result in weights with a much smoother spatial appearance. The example will throw a warning because it -doesn't converge, in this case this is what we want because of CI's time -constraints. +doesn't converge, in this case this is what we want because of resource +usage constraints on our Continuous Integration infrastructure that is used +to build this documentation on a regular basis. """ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import warnings import matplotlib.pyplot as plt + from sklearn.datasets import fetch_openml from sklearn.exceptions import ConvergenceWarning +from sklearn.model_selection import train_test_split from sklearn.neural_network import MLPClassifier -print(__doc__) - # Load data from https://www.openml.org/d/554 -X, y = fetch_openml('mnist_784', version=1, return_X_y=True) -X = X / 255. +X, y = fetch_openml("mnist_784", version=1, return_X_y=True, as_frame=False) +X = X / 255.0 -# rescale the data, use the traditional train/test split -X_train, X_test = X[:60000], X[60000:] -y_train, y_test = y[:60000], y[60000:] +# Split data into train partition and test partition +X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0, test_size=0.7) -mlp = MLPClassifier(hidden_layer_sizes=(50,), max_iter=10, alpha=1e-4, - solver='sgd', verbose=10, random_state=1, - learning_rate_init=.1) +mlp = MLPClassifier( + hidden_layer_sizes=(40,), + max_iter=8, + alpha=1e-4, + solver="sgd", + verbose=10, + random_state=1, + learning_rate_init=0.2, +) -# this example won't converge because of CI's time constraints, so we catch the -# warning and are ignore it here +# this example won't converge because of resource usage constraints on +# our Continuous Integration infrastructure, so we catch the warning and +# ignore it here with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=ConvergenceWarning, - module="sklearn") + warnings.filterwarnings("ignore", category=ConvergenceWarning, module="sklearn") mlp.fit(X_train, y_train) print("Training set score: %f" % mlp.score(X_train, y_train)) @@ -58,8 +67,7 @@ # use global min / max to ensure all weights are shown on the same scale vmin, vmax = mlp.coefs_[0].min(), mlp.coefs_[0].max() for coef, ax in zip(mlp.coefs_[0].T, axes.ravel()): - ax.matshow(coef.reshape(28, 28), cmap=plt.cm.gray, vmin=.5 * vmin, - vmax=.5 * vmax) + ax.matshow(coef.reshape(28, 28), cmap=plt.cm.gray, vmin=0.5 * vmin, vmax=0.5 * vmax) ax.set_xticks(()) ax.set_yticks(()) diff --git a/examples/neural_networks/plot_rbm_logistic_classification.py b/examples/neural_networks/plot_rbm_logistic_classification.py index 6994d3850f2f5..c42735bea8684 100644 --- a/examples/neural_networks/plot_rbm_logistic_classification.py +++ b/examples/neural_networks/plot_rbm_logistic_classification.py @@ -9,39 +9,26 @@ `) can perform effective non-linear feature extraction. -In order to learn good latent representations from a small dataset, we -artificially generate more labeled data by perturbing the training data with -linear shifts of 1 pixel in each direction. - -This example shows how to build a classification pipeline with a BernoulliRBM -feature extractor and a :class:`LogisticRegression -` classifier. The hyperparameters -of the entire model (learning rate, hidden layer size, regularization) -were optimized by grid search, but the search is not reproduced here because -of runtime constraints. - -Logistic regression on raw pixel values is presented for comparison. The -example shows that the features extracted by the BernoulliRBM help improve the -classification accuracy. """ -print(__doc__) -# Authors: Yann N. Dauphin, Vlad Niculae, Gabriel Synnaeve -# License: BSD +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np -import matplotlib.pyplot as plt +# %% +# Generate data +# ------------- +# +# In order to learn good latent representations from a small dataset, we +# artificially generate more labeled data by perturbing the training data with +# linear shifts of 1 pixel in each direction. +import numpy as np from scipy.ndimage import convolve -from sklearn import linear_model, datasets, metrics -from sklearn.model_selection import train_test_split -from sklearn.neural_network import BernoulliRBM -from sklearn.pipeline import Pipeline -from sklearn.base import clone +from sklearn import datasets +from sklearn.model_selection import train_test_split +from sklearn.preprocessing import minmax_scale -# ############################################################################# -# Setting up def nudge_dataset(X, Y): """ @@ -49,56 +36,62 @@ def nudge_dataset(X, Y): by moving the 8x8 images in X around by 1px to left, right, down, up """ direction_vectors = [ - [[0, 1, 0], - [0, 0, 0], - [0, 0, 0]], - - [[0, 0, 0], - [1, 0, 0], - [0, 0, 0]], - - [[0, 0, 0], - [0, 0, 1], - [0, 0, 0]], - - [[0, 0, 0], - [0, 0, 0], - [0, 1, 0]]] + [[0, 1, 0], [0, 0, 0], [0, 0, 0]], + [[0, 0, 0], [1, 0, 0], [0, 0, 0]], + [[0, 0, 0], [0, 0, 1], [0, 0, 0]], + [[0, 0, 0], [0, 0, 0], [0, 1, 0]], + ] def shift(x, w): - return convolve(x.reshape((8, 8)), mode='constant', weights=w).ravel() + return convolve(x.reshape((8, 8)), mode="constant", weights=w).ravel() - X = np.concatenate([X] + - [np.apply_along_axis(shift, 1, X, vector) - for vector in direction_vectors]) + X = np.concatenate( + [X] + [np.apply_along_axis(shift, 1, X, vector) for vector in direction_vectors] + ) Y = np.concatenate([Y for _ in range(5)], axis=0) return X, Y -# Load Data X, y = datasets.load_digits(return_X_y=True) -X = np.asarray(X, 'float32') +X = np.asarray(X, "float32") X, Y = nudge_dataset(X, y) -X = (X - np.min(X, 0)) / (np.max(X, 0) + 0.0001) # 0-1 scaling +X = minmax_scale(X, feature_range=(0, 1)) # 0-1 scaling -X_train, X_test, Y_train, Y_test = train_test_split( - X, Y, test_size=0.2, random_state=0) +X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=0) -# Models we will use -logistic = linear_model.LogisticRegression(solver='newton-cg', tol=1) +# %% +# Models definition +# ----------------- +# +# We build a classification pipeline with a BernoulliRBM feature extractor and +# a :class:`LogisticRegression ` +# classifier. + +from sklearn import linear_model +from sklearn.neural_network import BernoulliRBM +from sklearn.pipeline import Pipeline + +logistic = linear_model.LogisticRegression(solver="newton-cg", tol=1) rbm = BernoulliRBM(random_state=0, verbose=True) -rbm_features_classifier = Pipeline( - steps=[('rbm', rbm), ('logistic', logistic)]) +rbm_features_classifier = Pipeline(steps=[("rbm", rbm), ("logistic", logistic)]) -# ############################################################################# +# %% # Training +# -------- +# +# The hyperparameters of the entire model (learning rate, hidden layer size, +# regularization) were optimized by grid search, but the search is not +# reproduced here because of runtime constraints. + +from sklearn.base import clone # Hyper-parameters. These were set by cross-validation, # using a GridSearchCV. Here we are not performing cross-validation to # save time. rbm.learning_rate = 0.06 rbm.n_iter = 10 + # More components tend to give better prediction performance, but larger # fitting time rbm.n_components = 100 @@ -109,31 +102,45 @@ def shift(x, w): # Training the Logistic regression classifier directly on the pixel raw_pixel_classifier = clone(logistic) -raw_pixel_classifier.C = 100. +raw_pixel_classifier.C = 100.0 raw_pixel_classifier.fit(X_train, Y_train) -# ############################################################################# +# %% # Evaluation +# ---------- + +from sklearn import metrics Y_pred = rbm_features_classifier.predict(X_test) -print("Logistic regression using RBM features:\n%s\n" % ( - metrics.classification_report(Y_test, Y_pred))) +print( + "Logistic regression using RBM features:\n%s\n" + % (metrics.classification_report(Y_test, Y_pred)) +) +# %% Y_pred = raw_pixel_classifier.predict(X_test) -print("Logistic regression using raw pixel features:\n%s\n" % ( - metrics.classification_report(Y_test, Y_pred))) +print( + "Logistic regression using raw pixel features:\n%s\n" + % (metrics.classification_report(Y_test, Y_pred)) +) + +# %% +# The features extracted by the BernoulliRBM help improve the classification +# accuracy with respect to the logistic regression on raw pixels. -# ############################################################################# +# %% # Plotting +# -------- + +import matplotlib.pyplot as plt plt.figure(figsize=(4.2, 4)) for i, comp in enumerate(rbm.components_): plt.subplot(10, 10, i + 1) - plt.imshow(comp.reshape((8, 8)), cmap=plt.cm.gray_r, - interpolation='nearest') + plt.imshow(comp.reshape((8, 8)), cmap=plt.cm.gray_r, interpolation="nearest") plt.xticks(()) plt.yticks(()) -plt.suptitle('100 components extracted by RBM', fontsize=16) +plt.suptitle("100 components extracted by RBM", fontsize=16) plt.subplots_adjust(0.08, 0.02, 0.92, 0.85, 0.08, 0.23) plt.show() diff --git a/examples/preprocessing/plot_all_scaling.py b/examples/preprocessing/plot_all_scaling.py old mode 100755 new mode 100644 index 56492a3efc092..09b099067da14 --- a/examples/preprocessing/plot_all_scaling.py +++ b/examples/preprocessing/plot_all_scaling.py @@ -1,12 +1,9 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - """ ============================================================= Compare the effect of different scalers on data with outliers ============================================================= -Feature 0 (median income in a block) and feature 5 (number of households) of +Feature 0 (median income in a block) and feature 5 (average house occupancy) of the :ref:`california_housing_dataset` have very different scales and contain some very large outliers. These two characteristics lead to difficulties to visualize the data and, more @@ -43,68 +40,85 @@ """ -# Author: Raghav RV -# Guillaume Lemaitre -# Thomas Unterthiner -# License: BSD 3 clause - -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import matplotlib as mpl -from matplotlib import pyplot as plt +import numpy as np from matplotlib import cm - -from sklearn.preprocessing import MinMaxScaler -from sklearn.preprocessing import minmax_scale -from sklearn.preprocessing import MaxAbsScaler -from sklearn.preprocessing import StandardScaler -from sklearn.preprocessing import RobustScaler -from sklearn.preprocessing import Normalizer -from sklearn.preprocessing import QuantileTransformer -from sklearn.preprocessing import PowerTransformer +from matplotlib import pyplot as plt from sklearn.datasets import fetch_california_housing - -print(__doc__) +from sklearn.preprocessing import ( + MaxAbsScaler, + MinMaxScaler, + Normalizer, + PowerTransformer, + QuantileTransformer, + RobustScaler, + StandardScaler, + minmax_scale, +) dataset = fetch_california_housing() X_full, y_full = dataset.data, dataset.target +feature_names = dataset.feature_names + +feature_mapping = { + "MedInc": "Median income in block", + "HouseAge": "Median house age in block", + "AveRooms": "Average number of rooms", + "AveBedrms": "Average number of bedrooms", + "Population": "Block population", + "AveOccup": "Average house occupancy", + "Latitude": "House block latitude", + "Longitude": "House block longitude", +} # Take only 2 features to make visualization easier -# Feature of 0 has a long tail distribution. -# Feature 5 has a few but very large outliers. - -X = X_full[:, [0, 5]] - +# Feature MedInc has a long tail distribution. +# Feature AveOccup has a few but very large outliers. +features = ["MedInc", "AveOccup"] +features_idx = [feature_names.index(feature) for feature in features] +X = X_full[:, features_idx] distributions = [ - ('Unscaled data', X), - ('Data after standard scaling', - StandardScaler().fit_transform(X)), - ('Data after min-max scaling', - MinMaxScaler().fit_transform(X)), - ('Data after max-abs scaling', - MaxAbsScaler().fit_transform(X)), - ('Data after robust scaling', - RobustScaler(quantile_range=(25, 75)).fit_transform(X)), - ('Data after power transformation (Yeo-Johnson)', - PowerTransformer(method='yeo-johnson').fit_transform(X)), - ('Data after power transformation (Box-Cox)', - PowerTransformer(method='box-cox').fit_transform(X)), - ('Data after quantile transformation (uniform pdf)', - QuantileTransformer(output_distribution='uniform') - .fit_transform(X)), - ('Data after quantile transformation (gaussian pdf)', - QuantileTransformer(output_distribution='normal') - .fit_transform(X)), - ('Data after sample-wise L2 normalizing', - Normalizer().fit_transform(X)), + ("Unscaled data", X), + ("Data after standard scaling", StandardScaler().fit_transform(X)), + ("Data after min-max scaling", MinMaxScaler().fit_transform(X)), + ("Data after max-abs scaling", MaxAbsScaler().fit_transform(X)), + ( + "Data after robust scaling", + RobustScaler(quantile_range=(25, 75)).fit_transform(X), + ), + ( + "Data after power transformation (Yeo-Johnson)", + PowerTransformer(method="yeo-johnson").fit_transform(X), + ), + ( + "Data after power transformation (Box-Cox)", + PowerTransformer(method="box-cox").fit_transform(X), + ), + ( + "Data after quantile transformation (uniform pdf)", + QuantileTransformer( + output_distribution="uniform", random_state=42 + ).fit_transform(X), + ), + ( + "Data after quantile transformation (gaussian pdf)", + QuantileTransformer( + output_distribution="normal", random_state=42 + ).fit_transform(X), + ), + ("Data after sample-wise L2 normalizing", Normalizer().fit_transform(X)), ] # scale the output between 0 and 1 for the colorbar y = minmax_scale(y_full) # plasma does not exist in matplotlib < 1.5 -cmap = getattr(cm, 'plasma_r', cm.hot_r) +cmap = getattr(cm, "plasma_r", cm.hot_r) + def create_axes(title, figsize=(16, 6)): fig = plt.figure(figsize=figsize) @@ -142,13 +156,14 @@ def create_axes(title, figsize=(16, 6)): rect_colorbar = [left, bottom, width, height] ax_colorbar = plt.axes(rect_colorbar) - return ((ax_scatter, ax_histy, ax_histx), - (ax_scatter_zoom, ax_histy_zoom, ax_histx_zoom), - ax_colorbar) + return ( + (ax_scatter, ax_histy, ax_histx), + (ax_scatter_zoom, ax_histy_zoom, ax_histx_zoom), + ax_colorbar, + ) -def plot_distribution(axes, X, y, hist_nbins=50, title="", - x0_label="", x1_label=""): +def plot_distribution(axes, X, y, hist_nbins=50, title="", x0_label="", x1_label=""): ax, hist_X1, hist_X0 = axes ax.set_title(title) @@ -157,28 +172,31 @@ def plot_distribution(axes, X, y, hist_nbins=50, title="", # The scatter plot colors = cmap(y) - ax.scatter(X[:, 0], X[:, 1], alpha=0.5, marker='o', s=5, lw=0, c=colors) + ax.scatter(X[:, 0], X[:, 1], alpha=0.5, marker="o", s=5, lw=0, c=colors) # Removing the top and the right spine for aesthetics # make nice axis layout - ax.spines['top'].set_visible(False) - ax.spines['right'].set_visible(False) + ax.spines["top"].set_visible(False) + ax.spines["right"].set_visible(False) ax.get_xaxis().tick_bottom() ax.get_yaxis().tick_left() - ax.spines['left'].set_position(('outward', 10)) - ax.spines['bottom'].set_position(('outward', 10)) + ax.spines["left"].set_position(("outward", 10)) + ax.spines["bottom"].set_position(("outward", 10)) # Histogram for axis X1 (feature 5) hist_X1.set_ylim(ax.get_ylim()) - hist_X1.hist(X[:, 1], bins=hist_nbins, orientation='horizontal', - color='grey', ec='grey') - hist_X1.axis('off') + hist_X1.hist( + X[:, 1], bins=hist_nbins, orientation="horizontal", color="grey", ec="grey" + ) + hist_X1.axis("off") # Histogram for axis X0 (feature 0) hist_X0.set_xlim(ax.get_xlim()) - hist_X0.hist(X[:, 0], bins=hist_nbins, orientation='vertical', - color='grey', ec='grey') - hist_X0.axis('off') + hist_X0.hist( + X[:, 0], bins=hist_nbins, orientation="vertical", color="grey", ec="grey" + ) + hist_X0.axis("off") + # %% # Two plots will be shown for each scaler/normalizer/transformer. The left @@ -192,29 +210,42 @@ def make_plot(item_idx): title, X = distributions[item_idx] ax_zoom_out, ax_zoom_in, ax_colorbar = create_axes(title) axarr = (ax_zoom_out, ax_zoom_in) - plot_distribution(axarr[0], X, y, hist_nbins=200, - x0_label="Median Income", - x1_label="Number of households", - title="Full data") + plot_distribution( + axarr[0], + X, + y, + hist_nbins=200, + x0_label=feature_mapping[features[0]], + x1_label=feature_mapping[features[1]], + title="Full data", + ) # zoom-in zoom_in_percentile_range = (0, 99) cutoffs_X0 = np.percentile(X[:, 0], zoom_in_percentile_range) cutoffs_X1 = np.percentile(X[:, 1], zoom_in_percentile_range) - non_outliers_mask = ( - np.all(X > [cutoffs_X0[0], cutoffs_X1[0]], axis=1) & - np.all(X < [cutoffs_X0[1], cutoffs_X1[1]], axis=1)) - plot_distribution(axarr[1], X[non_outliers_mask], y[non_outliers_mask], - hist_nbins=50, - x0_label="Median Income", - x1_label="Number of households", - title="Zoom-in") + non_outliers_mask = np.all(X > [cutoffs_X0[0], cutoffs_X1[0]], axis=1) & np.all( + X < [cutoffs_X0[1], cutoffs_X1[1]], axis=1 + ) + plot_distribution( + axarr[1], + X[non_outliers_mask], + y[non_outliers_mask], + hist_nbins=50, + x0_label=feature_mapping[features[0]], + x1_label=feature_mapping[features[1]], + title="Zoom-in", + ) norm = mpl.colors.Normalize(y_full.min(), y_full.max()) - mpl.colorbar.ColorbarBase(ax_colorbar, cmap=cmap, - norm=norm, orientation='vertical', - label='Color mapping for values of y') + mpl.colorbar.ColorbarBase( + ax_colorbar, + cmap=cmap, + norm=norm, + orientation="vertical", + label="Color mapping for values of y", + ) # %% @@ -227,15 +258,17 @@ def make_plot(item_idx): # left plot showing the entire dataset, and the right zoomed-in to show the # dataset without the marginal outliers. A large majority of the samples are # compacted to a specific range, [0, 10] for the median income and [0, 6] for -# the number of households. Note that there are some marginal outliers (some -# blocks have more than 1200 households). Therefore, a specific pre-processing -# can be very beneficial depending of the application. In the following, we -# present some insights and behaviors of those pre-processing methods in the -# presence of marginal outliers. +# the average house occupancy. Note that there are some marginal outliers (some +# blocks have average occupancy of more than 1200). Therefore, a specific +# pre-processing can be very beneficial depending of the application. In the +# following, we present some insights and behaviors of those pre-processing +# methods in the presence of marginal outliers. make_plot(0) # %% +# .. _plot_all_scaling_standard_scaler_section: +# # StandardScaler # -------------- # @@ -247,7 +280,7 @@ def make_plot(item_idx): # feature have different magnitudes, the spread of the transformed data on # each feature is very different: most of the data lie in the [-2, 4] range for # the transformed median income feature while the same data is squeezed in the -# smaller [-0.2, 0.2] range for the transformed number of households. +# smaller [-0.2, 0.2] range for the transformed average house occupancy. # # :class:`~sklearn.preprocessing.StandardScaler` therefore cannot guarantee # balanced feature scales in the @@ -256,6 +289,8 @@ def make_plot(item_idx): make_plot(1) # %% +# .. _plot_all_scaling_minmax_scaler_section: +# # MinMaxScaler # ------------ # @@ -263,7 +298,7 @@ def make_plot(item_idx): # all feature values are in # the range [0, 1] as shown in the right panel below. However, this scaling # compresses all inliers into the narrow range [0, 0.005] for the transformed -# number of households. +# average house occupancy. # # Both :class:`~sklearn.preprocessing.StandardScaler` and # :class:`~sklearn.preprocessing.MinMaxScaler` are very sensitive to the @@ -272,25 +307,33 @@ def make_plot(item_idx): make_plot(2) # %% +# .. _plot_all_scaling_max_abs_scaler_section: +# # MaxAbsScaler # ------------ # # :class:`~sklearn.preprocessing.MaxAbsScaler` is similar to # :class:`~sklearn.preprocessing.MinMaxScaler` except that the -# values are mapped in the range [0, 1]. On positive only data, both scalers -# behave similarly. +# values are mapped across several ranges depending on whether negative +# OR positive values are present. If only positive values are present, the +# range is [0, 1]. If only negative values are present, the range is [-1, 0]. +# If both negative and positive values are present, the range is [-1, 1]. +# On positive only data, both :class:`~sklearn.preprocessing.MinMaxScaler` +# and :class:`~sklearn.preprocessing.MaxAbsScaler` behave similarly. # :class:`~sklearn.preprocessing.MaxAbsScaler` therefore also suffers from # the presence of large outliers. make_plot(3) # %% +# .. _plot_all_scaling_robust_scaler_section: +# # RobustScaler # ------------ # # Unlike the previous scalers, the centering and scaling statistics of # :class:`~sklearn.preprocessing.RobustScaler` -# is based on percentiles and are therefore not influenced by a few +# are based on percentiles and are therefore not influenced by a small # number of very large marginal outliers. Consequently, the resulting range of # the transformed feature values is larger than for the previous scalers and, # more importantly, are approximately similar: for both features most of the @@ -302,6 +345,8 @@ def make_plot(item_idx): make_plot(4) # %% +# .. _plot_all_scaling_power_transformer_section: +# # PowerTransformer # ---------------- # @@ -312,14 +357,16 @@ def make_plot(item_idx): # scaling factor is determined via maximum likelihood estimation in both # methods. By default, :class:`~sklearn.preprocessing.PowerTransformer` applies # zero-mean, unit variance normalization. Note that -# Box-Cox can only be applied to strictly positive data. Income and number of -# households happen to be strictly positive, but if negative values are present -# the Yeo-Johnson transformed is preferred. +# Box-Cox can only be applied to strictly positive data. Income and average +# house occupancy happen to be strictly positive, but if negative values are +# present the Yeo-Johnson transformed is preferred. make_plot(5) make_plot(6) # %% +# .. _plot_all_scaling_quantile_transformer_section: +# # QuantileTransformer (uniform output) # ------------------------------------ # @@ -351,6 +398,8 @@ def make_plot(item_idx): make_plot(8) # %% +# .. _plot_all_scaling_normalizer_section: +# # Normalizer # ---------- # diff --git a/examples/preprocessing/plot_discretization.py b/examples/preprocessing/plot_discretization.py index 9cfcb30e6fdd7..833d456f5b5f6 100644 --- a/examples/preprocessing/plot_discretization.py +++ b/examples/preprocessing/plot_discretization.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """ ================================================================ Using KBinsDiscretizer to discretize continuous features @@ -29,19 +27,16 @@ """ -# Author: Andreas MÃŧller -# Hanmin Qin -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt +import numpy as np from sklearn.linear_model import LinearRegression from sklearn.preprocessing import KBinsDiscretizer from sklearn.tree import DecisionTreeRegressor -print(__doc__) - # construct the dataset rnd = np.random.RandomState(42) X = rnd.uniform(-3, 3, size=100) @@ -49,19 +44,19 @@ X = X.reshape(-1, 1) # transform the dataset with KBinsDiscretizer -enc = KBinsDiscretizer(n_bins=10, encode='onehot') +enc = KBinsDiscretizer( + n_bins=10, encode="onehot", quantile_method="averaged_inverted_cdf" +) X_binned = enc.fit_transform(X) # predict with original dataset fig, (ax1, ax2) = plt.subplots(ncols=2, sharey=True, figsize=(10, 4)) line = np.linspace(-3, 3, 1000, endpoint=False).reshape(-1, 1) reg = LinearRegression().fit(X, y) -ax1.plot(line, reg.predict(line), linewidth=2, color='green', - label="linear regression") +ax1.plot(line, reg.predict(line), linewidth=2, color="green", label="linear regression") reg = DecisionTreeRegressor(min_samples_split=3, random_state=0).fit(X, y) -ax1.plot(line, reg.predict(line), linewidth=2, color='red', - label="decision tree") -ax1.plot(X[:, 0], y, 'o', c='k') +ax1.plot(line, reg.predict(line), linewidth=2, color="red", label="decision tree") +ax1.plot(X[:, 0], y, "o", c="k") ax1.legend(loc="best") ax1.set_ylabel("Regression output") ax1.set_xlabel("Input feature") @@ -70,14 +65,25 @@ # predict with transformed dataset line_binned = enc.transform(line) reg = LinearRegression().fit(X_binned, y) -ax2.plot(line, reg.predict(line_binned), linewidth=2, color='green', - linestyle='-', label='linear regression') -reg = DecisionTreeRegressor(min_samples_split=3, - random_state=0).fit(X_binned, y) -ax2.plot(line, reg.predict(line_binned), linewidth=2, color='red', - linestyle=':', label='decision tree') -ax2.plot(X[:, 0], y, 'o', c='k') -ax2.vlines(enc.bin_edges_[0], *plt.gca().get_ylim(), linewidth=1, alpha=.2) +ax2.plot( + line, + reg.predict(line_binned), + linewidth=2, + color="green", + linestyle="-", + label="linear regression", +) +reg = DecisionTreeRegressor(min_samples_split=3, random_state=0).fit(X_binned, y) +ax2.plot( + line, + reg.predict(line_binned), + linewidth=2, + color="red", + linestyle=":", + label="decision tree", +) +ax2.plot(X[:, 0], y, "o", c="k") +ax2.vlines(enc.bin_edges_[0], *plt.gca().get_ylim(), linewidth=1, alpha=0.2) ax2.legend(loc="best") ax2.set_xlabel("Input feature") ax2.set_title("Result after discretization") diff --git a/examples/preprocessing/plot_discretization_classification.py b/examples/preprocessing/plot_discretization_classification.py index e55e7cb500eb1..9f1dccb6a0275 100644 --- a/examples/preprocessing/plot_discretization_classification.py +++ b/examples/preprocessing/plot_discretization_classification.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- """ ====================== Feature discretization @@ -27,152 +25,190 @@ The plots show training points in solid colors and testing points semi-transparent. The lower right shows the classification accuracy on the test set. + """ -# Code source: Tom DuprÊ la Tour -# Adapted from plot_classifier_comparison by GaÃĢl Varoquaux and Andreas MÃŧller -# -# License: BSD 3 clause -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt +import numpy as np from matplotlib.colors import ListedColormap -from sklearn.model_selection import train_test_split -from sklearn.preprocessing import StandardScaler -from sklearn.datasets import make_moons, make_circles, make_classification + +from sklearn.datasets import make_circles, make_classification, make_moons +from sklearn.ensemble import GradientBoostingClassifier +from sklearn.exceptions import ConvergenceWarning from sklearn.linear_model import LogisticRegression -from sklearn.model_selection import GridSearchCV +from sklearn.model_selection import GridSearchCV, train_test_split from sklearn.pipeline import make_pipeline -from sklearn.preprocessing import KBinsDiscretizer +from sklearn.preprocessing import KBinsDiscretizer, StandardScaler from sklearn.svm import SVC, LinearSVC -from sklearn.ensemble import GradientBoostingClassifier from sklearn.utils._testing import ignore_warnings -from sklearn.exceptions import ConvergenceWarning - -print(__doc__) -h = .02 # step size in the mesh +h = 0.02 # step size in the mesh def get_name(estimator): name = estimator.__class__.__name__ - if name == 'Pipeline': + if name == "Pipeline": name = [get_name(est[1]) for est in estimator.steps] - name = ' + '.join(name) + name = " + ".join(name) return name # list of (estimator, param_grid), where param_grid is used in GridSearchCV +# The parameter spaces in this example are limited to a narrow band to reduce +# its runtime. In a real use case, a broader search space for the algorithms +# should be used. classifiers = [ - (LogisticRegression(random_state=0), { - 'C': np.logspace(-2, 7, 10) - }), - (LinearSVC(random_state=0), { - 'C': np.logspace(-2, 7, 10) - }), - (make_pipeline( - KBinsDiscretizer(encode='onehot'), - LogisticRegression(random_state=0)), { - 'kbinsdiscretizer__n_bins': np.arange(2, 10), - 'logisticregression__C': np.logspace(-2, 7, 10), - }), - (make_pipeline( - KBinsDiscretizer(encode='onehot'), LinearSVC(random_state=0)), { - 'kbinsdiscretizer__n_bins': np.arange(2, 10), - 'linearsvc__C': np.logspace(-2, 7, 10), - }), - (GradientBoostingClassifier(n_estimators=50, random_state=0), { - 'learning_rate': np.logspace(-4, 0, 10) - }), - (SVC(random_state=0), { - 'C': np.logspace(-2, 7, 10) - }), + ( + make_pipeline(StandardScaler(), LogisticRegression(random_state=0)), + {"logisticregression__C": np.logspace(-1, 1, 3)}, + ), + ( + make_pipeline(StandardScaler(), LinearSVC(random_state=0)), + {"linearsvc__C": np.logspace(-1, 1, 3)}, + ), + ( + make_pipeline( + StandardScaler(), + KBinsDiscretizer( + encode="onehot", quantile_method="averaged_inverted_cdf", random_state=0 + ), + LogisticRegression(random_state=0), + ), + { + "kbinsdiscretizer__n_bins": np.arange(5, 8), + "logisticregression__C": np.logspace(-1, 1, 3), + }, + ), + ( + make_pipeline( + StandardScaler(), + KBinsDiscretizer( + encode="onehot", quantile_method="averaged_inverted_cdf", random_state=0 + ), + LinearSVC(random_state=0), + ), + { + "kbinsdiscretizer__n_bins": np.arange(5, 8), + "linearsvc__C": np.logspace(-1, 1, 3), + }, + ), + ( + make_pipeline( + StandardScaler(), GradientBoostingClassifier(n_estimators=5, random_state=0) + ), + {"gradientboostingclassifier__learning_rate": np.logspace(-2, 0, 5)}, + ), + ( + make_pipeline(StandardScaler(), SVC(random_state=0)), + {"svc__C": np.logspace(-1, 1, 3)}, + ), ] -names = [get_name(e) for e, g in classifiers] +names = [get_name(e).replace("StandardScaler + ", "") for e, _ in classifiers] n_samples = 100 datasets = [ make_moons(n_samples=n_samples, noise=0.2, random_state=0), make_circles(n_samples=n_samples, noise=0.2, factor=0.5, random_state=1), - make_classification(n_samples=n_samples, n_features=2, n_redundant=0, - n_informative=2, random_state=2, - n_clusters_per_class=1) + make_classification( + n_samples=n_samples, + n_features=2, + n_redundant=0, + n_informative=2, + random_state=2, + n_clusters_per_class=1, + ), ] -fig, axes = plt.subplots(nrows=len(datasets), ncols=len(classifiers) + 1, - figsize=(21, 9)) +fig, axes = plt.subplots( + nrows=len(datasets), ncols=len(classifiers) + 1, figsize=(21, 9) +) -cm = plt.cm.PiYG -cm_bright = ListedColormap(['#b30065', '#178000']) +cm_piyg = plt.cm.PiYG +cm_bright = ListedColormap(["#b30065", "#178000"]) # iterate over datasets for ds_cnt, (X, y) in enumerate(datasets): - print('\ndataset %d\n---------' % ds_cnt) + print(f"\ndataset {ds_cnt}\n---------") - # preprocess dataset, split into training and test part - X = StandardScaler().fit_transform(X) + # split into training and test part X_train, X_test, y_train, y_test = train_test_split( - X, y, test_size=.5, random_state=42) + X, y, test_size=0.5, random_state=42 + ) # create the grid for background colors - x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5 - y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5 - xx, yy = np.meshgrid( - np.arange(x_min, x_max, h), np.arange(y_min, y_max, h)) + x_min, x_max = X[:, 0].min() - 0.5, X[:, 0].max() + 0.5 + y_min, y_max = X[:, 1].min() - 0.5, X[:, 1].max() + 0.5 + xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h)) # plot the dataset first ax = axes[ds_cnt, 0] if ds_cnt == 0: ax.set_title("Input data") # plot the training points - ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright, - edgecolors='k') + ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright, edgecolors="k") # and testing points - ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright, alpha=0.6, - edgecolors='k') + ax.scatter( + X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright, alpha=0.6, edgecolors="k" + ) ax.set_xlim(xx.min(), xx.max()) ax.set_ylim(yy.min(), yy.max()) ax.set_xticks(()) ax.set_yticks(()) # iterate over classifiers - for est_idx, (name, (estimator, param_grid)) in \ - enumerate(zip(names, classifiers)): + for est_idx, (name, (estimator, param_grid)) in enumerate(zip(names, classifiers)): ax = axes[ds_cnt, est_idx + 1] clf = GridSearchCV(estimator=estimator, param_grid=param_grid) with ignore_warnings(category=ConvergenceWarning): clf.fit(X_train, y_train) score = clf.score(X_test, y_test) - print('%s: %.2f' % (name, score)) + print(f"{name}: {score:.2f}") # plot the decision boundary. For that, we will assign a color to each # point in the mesh [x_min, x_max]*[y_min, y_max]. if hasattr(clf, "decision_function"): - Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()]) + Z = clf.decision_function(np.column_stack([xx.ravel(), yy.ravel()])) else: - Z = clf.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:, 1] + Z = clf.predict_proba(np.column_stack([xx.ravel(), yy.ravel()]))[:, 1] # put the result into a color plot Z = Z.reshape(xx.shape) - ax.contourf(xx, yy, Z, cmap=cm, alpha=.8) + ax.contourf(xx, yy, Z, cmap=cm_piyg, alpha=0.8) # plot the training points - ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright, - edgecolors='k') + ax.scatter( + X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright, edgecolors="k" + ) # and testing points - ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright, - edgecolors='k', alpha=0.6) + ax.scatter( + X_test[:, 0], + X_test[:, 1], + c=y_test, + cmap=cm_bright, + edgecolors="k", + alpha=0.6, + ) ax.set_xlim(xx.min(), xx.max()) ax.set_ylim(yy.min(), yy.max()) ax.set_xticks(()) ax.set_yticks(()) if ds_cnt == 0: - ax.set_title(name.replace(' + ', '\n')) - ax.text(0.95, 0.06, ('%.2f' % score).lstrip('0'), size=15, - bbox=dict(boxstyle='round', alpha=0.8, facecolor='white'), - transform=ax.transAxes, horizontalalignment='right') + ax.set_title(name.replace(" + ", "\n")) + ax.text( + 0.95, + 0.06, + (f"{score:.2f}").lstrip("0"), + size=15, + bbox=dict(boxstyle="round", alpha=0.8, facecolor="white"), + transform=ax.transAxes, + horizontalalignment="right", + ) plt.tight_layout() @@ -180,12 +216,18 @@ def get_name(estimator): # Add suptitles above the figure plt.subplots_adjust(top=0.90) suptitles = [ - 'Linear classifiers', - 'Feature discretization and linear classifiers', - 'Non-linear classifiers', + "Linear classifiers", + "Feature discretization and linear classifiers", + "Non-linear classifiers", ] for i, suptitle in zip([1, 3, 5], suptitles): ax = axes[0, i] - ax.text(1.05, 1.25, suptitle, transform=ax.transAxes, - horizontalalignment='center', size='x-large') + ax.text( + 1.05, + 1.25, + suptitle, + transform=ax.transAxes, + horizontalalignment="center", + size="x-large", + ) plt.show() diff --git a/examples/preprocessing/plot_discretization_strategies.py b/examples/preprocessing/plot_discretization_strategies.py index 9ef211a83ccf3..6a201b642d3c3 100644 --- a/examples/preprocessing/plot_discretization_strategies.py +++ b/examples/preprocessing/plot_discretization_strategies.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ ========================================================== Demonstrating the different strategies of KBinsDiscretizer @@ -14,20 +13,19 @@ procedure. The plot shows the regions where the discretized encoding is constant. + """ -# Author: Tom DuprÊ la Tour -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt +import numpy as np -from sklearn.preprocessing import KBinsDiscretizer from sklearn.datasets import make_blobs +from sklearn.preprocessing import KBinsDiscretizer -print(__doc__) - -strategies = ['uniform', 'quantile', 'kmeans'] +strategies = ["uniform", "quantile", "kmeans"] n_samples = 200 centers_0 = np.array([[0, 0], [0, 5], [2, 4], [8, 8]]) @@ -37,27 +35,37 @@ random_state = 42 X_list = [ np.random.RandomState(random_state).uniform(-3, 3, size=(n_samples, 2)), - make_blobs(n_samples=[n_samples // 10, n_samples * 4 // 10, - n_samples // 10, n_samples * 4 // 10], - cluster_std=0.5, centers=centers_0, - random_state=random_state)[0], - make_blobs(n_samples=[n_samples // 5, n_samples * 4 // 5], - cluster_std=0.5, centers=centers_1, - random_state=random_state)[0], + make_blobs( + n_samples=[ + n_samples // 10, + n_samples * 4 // 10, + n_samples // 10, + n_samples * 4 // 10, + ], + cluster_std=0.5, + centers=centers_0, + random_state=random_state, + )[0], + make_blobs( + n_samples=[n_samples // 5, n_samples * 4 // 5], + cluster_std=0.5, + centers=centers_1, + random_state=random_state, + )[0], ] figure = plt.figure(figsize=(14, 9)) i = 1 for ds_cnt, X in enumerate(X_list): - ax = plt.subplot(len(X_list), len(strategies) + 1, i) - ax.scatter(X[:, 0], X[:, 1], edgecolors='k') + ax.scatter(X[:, 0], X[:, 1], edgecolors="k") if ds_cnt == 0: ax.set_title("Input data", size=14) xx, yy = np.meshgrid( np.linspace(X[:, 0].min(), X[:, 0].max(), 300), - np.linspace(X[:, 1].min(), X[:, 1].max(), 300)) + np.linspace(X[:, 1].min(), X[:, 1].max(), 300), + ) grid = np.c_[xx.ravel(), yy.ravel()] ax.set_xlim(xx.min(), xx.max()) @@ -68,7 +76,12 @@ i += 1 # transform the dataset with KBinsDiscretizer for strategy in strategies: - enc = KBinsDiscretizer(n_bins=4, encode='ordinal', strategy=strategy) + enc = KBinsDiscretizer( + n_bins=4, + encode="ordinal", + quantile_method="averaged_inverted_cdf", + strategy=strategy, + ) enc.fit(X) grid_encoded = enc.transform(grid) @@ -76,18 +89,18 @@ # horizontal stripes horizontal = grid_encoded[:, 0].reshape(xx.shape) - ax.contourf(xx, yy, horizontal, alpha=.5) + ax.contourf(xx, yy, horizontal, alpha=0.5) # vertical stripes vertical = grid_encoded[:, 1].reshape(xx.shape) - ax.contourf(xx, yy, vertical, alpha=.5) + ax.contourf(xx, yy, vertical, alpha=0.5) - ax.scatter(X[:, 0], X[:, 1], edgecolors='k') + ax.scatter(X[:, 0], X[:, 1], edgecolors="k") ax.set_xlim(xx.min(), xx.max()) ax.set_ylim(yy.min(), yy.max()) ax.set_xticks(()) ax.set_yticks(()) if ds_cnt == 0: - ax.set_title("strategy='%s'" % (strategy, ), size=14) + ax.set_title("strategy='%s'" % (strategy,), size=14) i += 1 diff --git a/examples/preprocessing/plot_map_data_to_normal.py b/examples/preprocessing/plot_map_data_to_normal.py index ff465df78b0df..399c528a69f46 100644 --- a/examples/preprocessing/plot_map_data_to_normal.py +++ b/examples/preprocessing/plot_map_data_to_normal.py @@ -31,21 +31,17 @@ On "small" datasets (less than a few hundred points), the quantile transformer is prone to overfitting. The use of the power transform is then recommended. + """ -# Author: Eric Chang -# Nicolas Hug -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt +import numpy as np -from sklearn.preprocessing import PowerTransformer -from sklearn.preprocessing import QuantileTransformer from sklearn.model_selection import train_test_split - -print(__doc__) - +from sklearn.preprocessing import PowerTransformer, QuantileTransformer N_SAMPLES = 1000 FONT_SIZE = 6 @@ -53,12 +49,13 @@ rng = np.random.RandomState(304) -bc = PowerTransformer(method='box-cox') -yj = PowerTransformer(method='yeo-johnson') +bc = PowerTransformer(method="box-cox") +yj = PowerTransformer(method="yeo-johnson") # n_quantiles is set to the training set size rather than the default value # to avoid a warning being raised by this example -qt = QuantileTransformer(n_quantiles=500, output_distribution='normal', - random_state=rng) +qt = QuantileTransformer( + n_quantiles=500, output_distribution="normal", random_state=rng +) size = (N_SAMPLES, 1) @@ -88,28 +85,32 @@ # create plots distributions = [ - ('Lognormal', X_lognormal), - ('Chi-squared', X_chisq), - ('Weibull', X_weibull), - ('Gaussian', X_gaussian), - ('Uniform', X_uniform), - ('Bimodal', X_bimodal) + ("Lognormal", X_lognormal), + ("Chi-squared", X_chisq), + ("Weibull", X_weibull), + ("Gaussian", X_gaussian), + ("Uniform", X_uniform), + ("Bimodal", X_bimodal), ] -colors = ['#D81B60', '#0188FF', '#FFC107', - '#B7A2FF', '#000000', '#2EC5AC'] +colors = ["#D81B60", "#0188FF", "#FFC107", "#B7A2FF", "#000000", "#2EC5AC"] fig, axes = plt.subplots(nrows=8, ncols=3, figsize=plt.figaspect(2)) axes = axes.flatten() -axes_idxs = [(0, 3, 6, 9), (1, 4, 7, 10), (2, 5, 8, 11), (12, 15, 18, 21), - (13, 16, 19, 22), (14, 17, 20, 23)] -axes_list = [(axes[i], axes[j], axes[k], axes[l]) - for (i, j, k, l) in axes_idxs] +axes_idxs = [ + (0, 3, 6, 9), + (1, 4, 7, 10), + (2, 5, 8, 11), + (12, 15, 18, 21), + (13, 16, 19, 22), + (14, 17, 20, 23), +] +axes_list = [(axes[i], axes[j], axes[k], axes[l]) for (i, j, k, l) in axes_idxs] for distribution, color, axes in zip(distributions, colors, axes_list): name, X = distribution - X_train, X_test = train_test_split(X, test_size=.5) + X_train, X_test = train_test_split(X, test_size=0.5) # perform power transforms and quantile transform X_trans_bc = bc.fit(X_train).transform(X_test) @@ -122,19 +123,20 @@ ax_original.hist(X_train, color=color, bins=BINS) ax_original.set_title(name, fontsize=FONT_SIZE) - ax_original.tick_params(axis='both', which='major', labelsize=FONT_SIZE) + ax_original.tick_params(axis="both", which="major", labelsize=FONT_SIZE) for ax, X_trans, meth_name, lmbda in zip( - (ax_bc, ax_yj, ax_qt), - (X_trans_bc, X_trans_yj, X_trans_qt), - ('Box-Cox', 'Yeo-Johnson', 'Quantile transform'), - (lmbda_bc, lmbda_yj, None)): + (ax_bc, ax_yj, ax_qt), + (X_trans_bc, X_trans_yj, X_trans_qt), + ("Box-Cox", "Yeo-Johnson", "Quantile transform"), + (lmbda_bc, lmbda_yj, None), + ): ax.hist(X_trans, color=color, bins=BINS) - title = 'After {}'.format(meth_name) + title = "After {}".format(meth_name) if lmbda is not None: - title += r'\n$\lambda$ = {}'.format(lmbda) + title += "\n$\\lambda$ = {}".format(lmbda) ax.set_title(title, fontsize=FONT_SIZE) - ax.tick_params(axis='both', which='major', labelsize=FONT_SIZE) + ax.tick_params(axis="both", which="major", labelsize=FONT_SIZE) ax.set_xlim([-3.5, 3.5]) diff --git a/examples/preprocessing/plot_scaling_importance.py b/examples/preprocessing/plot_scaling_importance.py index b24786e1a018d..6432a1c48ec69 100644 --- a/examples/preprocessing/plot_scaling_importance.py +++ b/examples/preprocessing/plot_scaling_importance.py @@ -1,133 +1,256 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- """ -========================================================= +============================= Importance of Feature Scaling -========================================================= - -Feature scaling through standardization (or Z-score normalization) -can be an important preprocessing step for many machine learning -algorithms. Standardization involves rescaling the features such -that they have the properties of a standard normal distribution -with a mean of zero and a standard deviation of one. - -While many algorithms (such as SVM, K-nearest neighbors, and logistic -regression) require features to be normalized, intuitively we can -think of Principle Component Analysis (PCA) as being a prime example -of when normalization is important. In PCA we are interested in the -components that maximize the variance. If one component (e.g. human -height) varies less than another (e.g. weight) because of their -respective scales (meters vs. kilos), PCA might determine that the -direction of maximal variance more closely corresponds with the -'weight' axis, if those features are not scaled. As a change in -height of one meter can be considered much more important than the -change in weight of one kilogram, this is clearly incorrect. - -To illustrate this, PCA is performed comparing the use of data with -:class:`StandardScaler ` applied, -to unscaled data. The results are visualized and a clear difference noted. -The 1st principal component in the unscaled set can be seen. It can be seen -that feature #13 dominates the direction, being a whole two orders of -magnitude above the other features. This is contrasted when observing -the principal component for the scaled version of the data. In the scaled -version, the orders of magnitude are roughly the same across all the features. - -The dataset used is the Wine Dataset available at UCI. This dataset -has continuous features that are heterogeneous in scale due to differing -properties that they measure (i.e alcohol content, and malic acid). - -The transformed data is then used to train a naive Bayes classifier, and a -clear difference in prediction accuracies is observed wherein the dataset -which is scaled before PCA vastly outperforms the unscaled version. +============================= + +Feature scaling through standardization, also called Z-score normalization, is +an important preprocessing step for many machine learning algorithms. It +involves rescaling each feature such that it has a standard deviation of 1 and a +mean of 0. + +Even if tree based models are (almost) not affected by scaling, many other +algorithms require features to be normalized, often for different reasons: to +ease the convergence (such as a non-penalized logistic regression), to create a +completely different model fit compared to the fit with unscaled data (such as +KNeighbors models). The latter is demonstrated on the first part of the present +example. + +On the second part of the example we show how Principal Component Analysis (PCA) +is impacted by normalization of features. To illustrate this, we compare the +principal components found using :class:`~sklearn.decomposition.PCA` on unscaled +data with those obtained when using a +:class:`~sklearn.preprocessing.StandardScaler` to scale data first. + +In the last part of the example we show the effect of the normalization on the +accuracy of a model trained on PCA-reduced data. """ + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Load and prepare data +# ===================== +# +# The dataset used is the :ref:`wine_dataset` available at UCI. This dataset has +# continuous features that are heterogeneous in scale due to differing +# properties that they measure (e.g. alcohol content and malic acid). + +from sklearn.datasets import load_wine from sklearn.model_selection import train_test_split from sklearn.preprocessing import StandardScaler -from sklearn.decomposition import PCA -from sklearn.naive_bayes import GaussianNB -from sklearn import metrics -import matplotlib.pyplot as plt -from sklearn.datasets import load_wine -from sklearn.pipeline import make_pipeline -print(__doc__) -# Code source: Tyler Lanigan -# Sebastian Raschka +X, y = load_wine(return_X_y=True, as_frame=True) +scaler = StandardScaler().set_output(transform="pandas") -# License: BSD 3 clause +X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.30, random_state=42 +) +scaled_X_train = scaler.fit_transform(X_train) -RANDOM_STATE = 42 -FIG_SIZE = (10, 7) +# %% +# .. _neighbors_scaling: +# +# Effect of rescaling on a k-neighbors models +# =========================================== +# +# For the sake of visualizing the decision boundary of a +# :class:`~sklearn.neighbors.KNeighborsClassifier`, in this section we select a +# subset of 2 features that have values with different orders of magnitude. +# +# Keep in mind that using a subset of the features to train the model may likely +# leave out feature with high predictive impact, resulting in a decision +# boundary that is much worse in comparison to a model trained on the full set +# of features. +import matplotlib.pyplot as plt -features, target = load_wine(return_X_y=True) +from sklearn.inspection import DecisionBoundaryDisplay +from sklearn.neighbors import KNeighborsClassifier -# Make a train/test split using 30% test size -X_train, X_test, y_train, y_test = train_test_split(features, target, - test_size=0.30, - random_state=RANDOM_STATE) +X_plot = X[["proline", "hue"]] +X_plot_scaled = scaler.fit_transform(X_plot) +clf = KNeighborsClassifier(n_neighbors=20) -# Fit to data and predict using pipelined GNB and PCA. -unscaled_clf = make_pipeline(PCA(n_components=2), GaussianNB()) -unscaled_clf.fit(X_train, y_train) -pred_test = unscaled_clf.predict(X_test) -# Fit to data and predict using pipelined scaling, GNB and PCA. -std_clf = make_pipeline(StandardScaler(), PCA(n_components=2), GaussianNB()) -std_clf.fit(X_train, y_train) -pred_test_std = std_clf.predict(X_test) +def fit_and_plot_model(X_plot, y, clf, ax): + clf.fit(X_plot, y) + disp = DecisionBoundaryDisplay.from_estimator( + clf, + X_plot, + response_method="predict", + alpha=0.5, + ax=ax, + ) + disp.ax_.scatter(X_plot["proline"], X_plot["hue"], c=y, s=20, edgecolor="k") + disp.ax_.set_xlim((X_plot["proline"].min(), X_plot["proline"].max())) + disp.ax_.set_ylim((X_plot["hue"].min(), X_plot["hue"].max())) + return disp.ax_ + -# Show prediction accuracies in scaled and unscaled data. -print('\nPrediction accuracy for the normal test dataset with PCA') -print('{:.2%}\n'.format(metrics.accuracy_score(y_test, pred_test))) +fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=(12, 6)) -print('\nPrediction accuracy for the standardized test dataset with PCA') -print('{:.2%}\n'.format(metrics.accuracy_score(y_test, pred_test_std))) +fit_and_plot_model(X_plot, y, clf, ax1) +ax1.set_title("KNN without scaling") -# Extract PCA from pipeline -pca = unscaled_clf.named_steps['pca'] -pca_std = std_clf.named_steps['pca'] +fit_and_plot_model(X_plot_scaled, y, clf, ax2) +ax2.set_xlabel("scaled proline") +ax2.set_ylabel("scaled hue") +_ = ax2.set_title("KNN with scaling") -# Show first principal components -print('\nPC 1 without scaling:\n', pca.components_[0]) -print('\nPC 1 with scaling:\n', pca_std.components_[0]) +# %% +# Here the decision boundary shows that fitting scaled or non-scaled data lead +# to completely different models. The reason is that the variable "proline" has +# values which vary between 0 and 1,000; whereas the variable "hue" varies +# between 1 and 10. Because of this, distances between samples are mostly +# impacted by differences in values of "proline", while values of the "hue" will +# be comparatively ignored. If one uses +# :class:`~sklearn.preprocessing.StandardScaler` to normalize this database, +# both scaled values lay approximately between -3 and 3 and the neighbors +# structure will be impacted more or less equivalently by both variables. +# +# Effect of rescaling on a PCA dimensional reduction +# ================================================== +# +# Dimensional reduction using :class:`~sklearn.decomposition.PCA` consists of +# finding the features that maximize the variance. If one feature varies more +# than the others only because of their respective scales, +# :class:`~sklearn.decomposition.PCA` would determine that such feature +# dominates the direction of the principal components. +# +# We can inspect the first principal components using all the original features: -# Use PCA without and with scale on X_train data for visualization. +import pandas as pd + +from sklearn.decomposition import PCA + +pca = PCA(n_components=2).fit(X_train) +scaled_pca = PCA(n_components=2).fit(scaled_X_train) X_train_transformed = pca.transform(X_train) -scaler = std_clf.named_steps['standardscaler'] -X_train_std_transformed = pca_std.transform(scaler.transform(X_train)) - -# visualize standardized vs. untouched dataset with PCA performed -fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=FIG_SIZE) - - -for l, c, m in zip(range(0, 3), ('blue', 'red', 'green'), ('^', 's', 'o')): - ax1.scatter(X_train_transformed[y_train == l, 0], - X_train_transformed[y_train == l, 1], - color=c, - label='class %s' % l, - alpha=0.5, - marker=m - ) - -for l, c, m in zip(range(0, 3), ('blue', 'red', 'green'), ('^', 's', 'o')): - ax2.scatter(X_train_std_transformed[y_train == l, 0], - X_train_std_transformed[y_train == l, 1], - color=c, - label='class %s' % l, - alpha=0.5, - marker=m - ) - -ax1.set_title('Training dataset after PCA') -ax2.set_title('Standardized training dataset after PCA') +X_train_std_transformed = scaled_pca.transform(scaled_X_train) + +first_pca_component = pd.DataFrame( + pca.components_[0], index=X.columns, columns=["without scaling"] +) +first_pca_component["with scaling"] = scaled_pca.components_[0] +first_pca_component.plot.bar( + title="Weights of the first principal component", figsize=(6, 8) +) + +_ = plt.tight_layout() + +# %% +# Indeed we find that the "proline" feature dominates the direction of the first +# principal component without scaling, being about two orders of magnitude above +# the other features. This is contrasted when observing the first principal +# component for the scaled version of the data, where the orders of magnitude +# are roughly the same across all the features. +# +# We can visualize the distribution of the principal components in both cases: + +fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, figsize=(10, 5)) + +target_classes = range(0, 3) +colors = ("blue", "red", "green") +markers = ("^", "s", "o") + +for target_class, color, marker in zip(target_classes, colors, markers): + ax1.scatter( + x=X_train_transformed[y_train == target_class, 0], + y=X_train_transformed[y_train == target_class, 1], + color=color, + label=f"class {target_class}", + alpha=0.5, + marker=marker, + ) + + ax2.scatter( + x=X_train_std_transformed[y_train == target_class, 0], + y=X_train_std_transformed[y_train == target_class, 1], + color=color, + label=f"class {target_class}", + alpha=0.5, + marker=marker, + ) + +ax1.set_title("Unscaled training dataset after PCA") +ax2.set_title("Standardized training dataset after PCA") for ax in (ax1, ax2): - ax.set_xlabel('1st principal component') - ax.set_ylabel('2nd principal component') - ax.legend(loc='upper right') + ax.set_xlabel("1st principal component") + ax.set_ylabel("2nd principal component") + ax.legend(loc="upper right") ax.grid() -plt.tight_layout() +_ = plt.tight_layout() + +# %% +# From the plot above we observe that scaling the features before reducing the +# dimensionality results in components with the same order of magnitude. In this +# case it also improves the separability of the classes. Indeed, in the next +# section we confirm that a better separability has a good repercussion on the +# overall model's performance. +# +# Effect of rescaling on model's performance +# ========================================== +# +# First we show how the optimal regularization of a +# :class:`~sklearn.linear_model.LogisticRegressionCV` depends on the scaling or +# non-scaling of the data: + +import numpy as np + +from sklearn.linear_model import LogisticRegressionCV +from sklearn.pipeline import make_pipeline + +Cs = np.logspace(-5, 5, 20) + +unscaled_clf = make_pipeline(pca, LogisticRegressionCV(Cs=Cs)) +unscaled_clf.fit(X_train, y_train) + +scaled_clf = make_pipeline(scaler, pca, LogisticRegressionCV(Cs=Cs)) +scaled_clf.fit(X_train, y_train) + +print(f"Optimal C for the unscaled PCA: {unscaled_clf[-1].C_[0]:.4f}\n") +print(f"Optimal C for the standardized data with PCA: {scaled_clf[-1].C_[0]:.2f}") + +# %% +# The need for regularization is higher (lower values of `C`) for the data that +# was not scaled before applying PCA. We now evaluate the effect of scaling on +# the accuracy and the mean log-loss of the optimal models: + +from sklearn.metrics import accuracy_score, log_loss + +y_pred = unscaled_clf.predict(X_test) +y_pred_scaled = scaled_clf.predict(X_test) +y_proba = unscaled_clf.predict_proba(X_test) +y_proba_scaled = scaled_clf.predict_proba(X_test) + +print("Test accuracy for the unscaled PCA") +print(f"{accuracy_score(y_test, y_pred):.2%}\n") +print("Test accuracy for the standardized data with PCA") +print(f"{accuracy_score(y_test, y_pred_scaled):.2%}\n") +print("Log-loss for the unscaled PCA") +print(f"{log_loss(y_test, y_proba):.3}\n") +print("Log-loss for the standardized data with PCA") +print(f"{log_loss(y_test, y_proba_scaled):.3}") -plt.show() +# %% +# A clear difference in prediction accuracies is observed when the data is +# scaled before :class:`~sklearn.decomposition.PCA`, as it vastly outperforms +# the unscaled version. This corresponds to the intuition obtained from the plot +# in the previous section, where the components become linearly separable when +# scaling before using :class:`~sklearn.decomposition.PCA`. +# +# Notice that in this case the models with scaled features perform better than +# the models with non-scaled features because all the variables are expected to +# be predictive and we rather avoid some of them being comparatively ignored. +# +# If the variables in lower scales were not predictive, one may experience a +# decrease of the performance after scaling the features: noisy features would +# contribute more to the prediction after scaling and therefore scaling would +# increase overfitting. +# +# Last but not least, we observe that one achieves a lower log-loss by means of +# the scaling step. diff --git a/examples/preprocessing/plot_target_encoder.py b/examples/preprocessing/plot_target_encoder.py new file mode 100644 index 0000000000000..04f3222d4e512 --- /dev/null +++ b/examples/preprocessing/plot_target_encoder.py @@ -0,0 +1,228 @@ +""" +============================================ +Comparing Target Encoder with Other Encoders +============================================ + +.. currentmodule:: sklearn.preprocessing + +The :class:`TargetEncoder` uses the value of the target to encode each +categorical feature. In this example, we will compare three different approaches +for handling categorical features: :class:`TargetEncoder`, +:class:`OrdinalEncoder`, :class:`OneHotEncoder` and dropping the category. + +.. note:: + `fit(X, y).transform(X)` does not equal `fit_transform(X, y)` because a + cross fitting scheme is used in `fit_transform` for encoding. See the + :ref:`User Guide `. for details. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Loading Data from OpenML +# ======================== +# First, we load the wine reviews dataset, where the target is the points given +# be a reviewer: +from sklearn.datasets import fetch_openml + +wine_reviews = fetch_openml(data_id=42074, as_frame=True) + +df = wine_reviews.frame +df.head() + +# %% +# For this example, we use the following subset of numerical and categorical +# features in the data. The target are continuous values from 80 to 100: +numerical_features = ["price"] +categorical_features = [ + "country", + "province", + "region_1", + "region_2", + "variety", + "winery", +] +target_name = "points" + +X = df[numerical_features + categorical_features] +y = df[target_name] + +_ = y.hist() + +# %% +# Training and Evaluating Pipelines with Different Encoders +# ========================================================= +# In this section, we will evaluate pipelines with +# :class:`~sklearn.ensemble.HistGradientBoostingRegressor` with different encoding +# strategies. First, we list out the encoders we will be using to preprocess +# the categorical features: +from sklearn.compose import ColumnTransformer +from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder, TargetEncoder + +categorical_preprocessors = [ + ("drop", "drop"), + ("ordinal", OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=-1)), + ( + "one_hot", + OneHotEncoder(handle_unknown="ignore", max_categories=20, sparse_output=False), + ), + ("target", TargetEncoder(target_type="continuous")), +] + +# %% +# Next, we evaluate the models using cross validation and record the results: +from sklearn.ensemble import HistGradientBoostingRegressor +from sklearn.model_selection import cross_validate +from sklearn.pipeline import make_pipeline + +n_cv_folds = 3 +max_iter = 20 +results = [] + + +def evaluate_model_and_store(name, pipe): + result = cross_validate( + pipe, + X, + y, + scoring="neg_root_mean_squared_error", + cv=n_cv_folds, + return_train_score=True, + ) + rmse_test_score = -result["test_score"] + rmse_train_score = -result["train_score"] + results.append( + { + "preprocessor": name, + "rmse_test_mean": rmse_test_score.mean(), + "rmse_test_std": rmse_train_score.std(), + "rmse_train_mean": rmse_train_score.mean(), + "rmse_train_std": rmse_train_score.std(), + } + ) + + +for name, categorical_preprocessor in categorical_preprocessors: + preprocessor = ColumnTransformer( + [ + ("numerical", "passthrough", numerical_features), + ("categorical", categorical_preprocessor, categorical_features), + ] + ) + pipe = make_pipeline( + preprocessor, HistGradientBoostingRegressor(random_state=0, max_iter=max_iter) + ) + evaluate_model_and_store(name, pipe) + + +# %% +# Native Categorical Feature Support +# ================================== +# In this section, we build and evaluate a pipeline that uses native categorical +# feature support in :class:`~sklearn.ensemble.HistGradientBoostingRegressor`, +# which only supports up to 255 unique categories. In our dataset, the most of +# the categorical features have more than 255 unique categories: +n_unique_categories = df[categorical_features].nunique().sort_values(ascending=False) +n_unique_categories + +# %% +# To workaround the limitation above, we group the categorical features into +# low cardinality and high cardinality features. The high cardinality features +# will be target encoded and the low cardinality features will use the native +# categorical feature in gradient boosting. +high_cardinality_features = n_unique_categories[n_unique_categories > 255].index +low_cardinality_features = n_unique_categories[n_unique_categories <= 255].index +mixed_encoded_preprocessor = ColumnTransformer( + [ + ("numerical", "passthrough", numerical_features), + ( + "high_cardinality", + TargetEncoder(target_type="continuous"), + high_cardinality_features, + ), + ( + "low_cardinality", + OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=-1), + low_cardinality_features, + ), + ], + verbose_feature_names_out=False, +) + +# The output of the of the preprocessor must be set to pandas so the +# gradient boosting model can detect the low cardinality features. +mixed_encoded_preprocessor.set_output(transform="pandas") +mixed_pipe = make_pipeline( + mixed_encoded_preprocessor, + HistGradientBoostingRegressor( + random_state=0, max_iter=max_iter, categorical_features=low_cardinality_features + ), +) +mixed_pipe + +# %% +# Finally, we evaluate the pipeline using cross validation and record the results: +evaluate_model_and_store("mixed_target", mixed_pipe) + +# %% +# Plotting the Results +# ==================== +# In this section, we display the results by plotting the test and train scores: +import matplotlib.pyplot as plt +import pandas as pd + +results_df = ( + pd.DataFrame(results).set_index("preprocessor").sort_values("rmse_test_mean") +) + +fig, (ax1, ax2) = plt.subplots( + 1, 2, figsize=(12, 8), sharey=True, constrained_layout=True +) +xticks = range(len(results_df)) +name_to_color = dict( + zip((r["preprocessor"] for r in results), ["C0", "C1", "C2", "C3", "C4"]) +) + +for subset, ax in zip(["test", "train"], [ax1, ax2]): + mean, std = f"rmse_{subset}_mean", f"rmse_{subset}_std" + data = results_df[[mean, std]].sort_values(mean) + ax.bar( + x=xticks, + height=data[mean], + yerr=data[std], + width=0.9, + color=[name_to_color[name] for name in data.index], + ) + ax.set( + title=f"RMSE ({subset.title()})", + xlabel="Encoding Scheme", + xticks=xticks, + xticklabels=data.index, + ) + +# %% +# When evaluating the predictive performance on the test set, dropping the +# categories perform the worst and the target encoders performs the best. This +# can be explained as follows: +# +# - Dropping the categorical features makes the pipeline less expressive and +# underfitting as a result; +# - Due to the high cardinality and to reduce the training time, the one-hot +# encoding scheme uses `max_categories=20` which prevents the features from +# expanding too much, which can result in underfitting. +# - If we had not set `max_categories=20`, the one-hot encoding scheme would have +# likely made the pipeline overfitting as the number of features explodes with rare +# category occurrences that are correlated with the target by chance (on the training +# set only); +# - The ordinal encoding imposes an arbitrary order to the features which are then +# treated as numerical values by the +# :class:`~sklearn.ensemble.HistGradientBoostingRegressor`. Since this +# model groups numerical features in 256 bins per feature, many unrelated categories +# can be grouped together and as a result overall pipeline can underfit; +# - When using the target encoder, the same binning happens, but since the encoded +# values are statistically ordered by marginal association with the target variable, +# the binning use by the :class:`~sklearn.ensemble.HistGradientBoostingRegressor` +# makes sense and leads to good results: the combination of smoothed target +# encoding and binning works as a good regularizing strategy against +# overfitting while not limiting the expressiveness of the pipeline too much. diff --git a/examples/preprocessing/plot_target_encoder_cross_val.py b/examples/preprocessing/plot_target_encoder_cross_val.py new file mode 100644 index 0000000000000..3d51664710096 --- /dev/null +++ b/examples/preprocessing/plot_target_encoder_cross_val.py @@ -0,0 +1,194 @@ +""" +======================================= +Target Encoder's Internal Cross fitting +======================================= + +.. currentmodule:: sklearn.preprocessing + +The :class:`TargetEncoder` replaces each category of a categorical feature with +the shrunk mean of the target variable for that category. This method is useful +in cases where there is a strong relationship between the categorical feature +and the target. To prevent overfitting, :meth:`TargetEncoder.fit_transform` uses +an internal :term:`cross fitting` scheme to encode the training data to be used +by a downstream model. This scheme involves splitting the data into *k* folds +and encoding each fold using the encodings learnt using the other *k-1* folds. +In this example, we demonstrate the importance of the cross +fitting procedure to prevent overfitting. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Create Synthetic Dataset +# ======================== +# For this example, we build a dataset with three categorical features: +# +# * an informative feature with medium cardinality ("informative") +# * an uninformative feature with medium cardinality ("shuffled") +# * an uninformative feature with high cardinality ("near_unique") +# +# First, we generate the informative feature: +import numpy as np + +from sklearn.preprocessing import KBinsDiscretizer + +n_samples = 50_000 + +rng = np.random.RandomState(42) +y = rng.randn(n_samples) +noise = 0.5 * rng.randn(n_samples) +n_categories = 100 + +kbins = KBinsDiscretizer( + n_bins=n_categories, + encode="ordinal", + strategy="uniform", + random_state=rng, + subsample=None, +) +X_informative = kbins.fit_transform((y + noise).reshape(-1, 1)) + +# Remove the linear relationship between y and the bin index by permuting the +# values of X_informative: +permuted_categories = rng.permutation(n_categories) +X_informative = permuted_categories[X_informative.astype(np.int32)] + +# %% +# The uninformative feature with medium cardinality is generated by permuting the +# informative feature and removing the relationship with the target: +X_shuffled = rng.permutation(X_informative) + +# %% +# The uninformative feature with high cardinality is generated so that it is +# independent of the target variable. We will show that target encoding without +# :term:`cross fitting` will cause catastrophic overfitting for the downstream +# regressor. These high cardinality features are basically unique identifiers +# for samples which should generally be removed from machine learning datasets. +# In this example, we generate them to show how :class:`TargetEncoder`'s default +# :term:`cross fitting` behavior mitigates the overfitting issue automatically. +X_near_unique_categories = rng.choice( + int(0.9 * n_samples), size=n_samples, replace=True +).reshape(-1, 1) + +# %% +# Finally, we assemble the dataset and perform a train test split: +import pandas as pd + +from sklearn.model_selection import train_test_split + +X = pd.DataFrame( + np.concatenate( + [X_informative, X_shuffled, X_near_unique_categories], + axis=1, + ), + columns=["informative", "shuffled", "near_unique"], +) +X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0) + +# %% +# Training a Ridge Regressor +# ========================== +# In this section, we train a ridge regressor on the dataset with and without +# encoding and explore the influence of target encoder with and without the +# internal :term:`cross fitting`. First, we see the Ridge model trained on the +# raw features will have low performance. This is because we permuted the order +# of the informative feature meaning `X_informative` is not informative when +# raw: +import sklearn +from sklearn.linear_model import Ridge + +# Configure transformers to always output DataFrames +sklearn.set_config(transform_output="pandas") + +ridge = Ridge(alpha=1e-6, solver="lsqr", fit_intercept=False) + +raw_model = ridge.fit(X_train, y_train) +print("Raw Model score on training set: ", raw_model.score(X_train, y_train)) +print("Raw Model score on test set: ", raw_model.score(X_test, y_test)) + +# %% +# Next, we create a pipeline with the target encoder and ridge model. The pipeline +# uses :meth:`TargetEncoder.fit_transform` which uses :term:`cross fitting`. We +# see that the model fits the data well and generalizes to the test set: +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import TargetEncoder + +model_with_cf = make_pipeline(TargetEncoder(random_state=0), ridge) +model_with_cf.fit(X_train, y_train) +print("Model with CF on train set: ", model_with_cf.score(X_train, y_train)) +print("Model with CF on test set: ", model_with_cf.score(X_test, y_test)) + +# %% +# The coefficients of the linear model shows that most of the weight is on the +# feature at column index 0, which is the informative feature +import matplotlib.pyplot as plt +import pandas as pd + +plt.rcParams["figure.constrained_layout.use"] = True + +coefs_cf = pd.Series( + model_with_cf[-1].coef_, index=model_with_cf[-1].feature_names_in_ +).sort_values() +ax = coefs_cf.plot(kind="barh") +_ = ax.set( + title="Target encoded with cross fitting", + xlabel="Ridge coefficient", + ylabel="Feature", +) + +# %% +# While :meth:`TargetEncoder.fit_transform` uses an internal +# :term:`cross fitting` scheme to learn encodings for the training set, +# :meth:`TargetEncoder.transform` itself does not. +# It uses the complete training set to learn encodings and to transform the +# categorical features. Thus, we can use :meth:`TargetEncoder.fit` followed by +# :meth:`TargetEncoder.transform` to disable the :term:`cross fitting`. This +# encoding is then passed to the ridge model. +target_encoder = TargetEncoder(random_state=0) +target_encoder.fit(X_train, y_train) +X_train_no_cf_encoding = target_encoder.transform(X_train) +X_test_no_cf_encoding = target_encoder.transform(X_test) + +model_no_cf = ridge.fit(X_train_no_cf_encoding, y_train) + +# %% +# We evaluate the model that did not use :term:`cross fitting` when encoding and +# see that it overfits: +print( + "Model without CF on training set: ", + model_no_cf.score(X_train_no_cf_encoding, y_train), +) +print( + "Model without CF on test set: ", + model_no_cf.score( + X_test_no_cf_encoding, + y_test, + ), +) + +# %% +# The ridge model overfits because it assigns much more weight to the +# uninformative extremely high cardinality ("near_unique") and medium +# cardinality ("shuffled") features than when the model used +# :term:`cross fitting` to encode the features. +coefs_no_cf = pd.Series( + model_no_cf.coef_, index=model_no_cf.feature_names_in_ +).sort_values() +ax = coefs_no_cf.plot(kind="barh") +_ = ax.set( + title="Target encoded without cross fitting", + xlabel="Ridge coefficient", + ylabel="Feature", +) + +# %% +# Conclusion +# ========== +# This example demonstrates the importance of :class:`TargetEncoder`'s internal +# :term:`cross fitting`. It is important to use +# :meth:`TargetEncoder.fit_transform` to encode training data before passing it +# to a machine learning model. When a :class:`TargetEncoder` is a part of a +# :class:`~sklearn.pipeline.Pipeline` and the pipeline is fitted, the pipeline +# will correctly call :meth:`TargetEncoder.fit_transform` and use +# :term:`cross fitting` when encoding the training data. diff --git a/examples/release_highlights/plot_release_highlights_0_22_0.py b/examples/release_highlights/plot_release_highlights_0_22_0.py index d9efc9a520af1..8d5648188f0fe 100644 --- a/examples/release_highlights/plot_release_highlights_0_22_0.py +++ b/examples/release_highlights/plot_release_highlights_0_22_0.py @@ -8,7 +8,7 @@ We are pleased to announce the release of scikit-learn 0.22, which comes with many bug fixes and new features! We detail below a few of the major features of this release. For an exhaustive list of all the changes, please -refer to the :ref:`release notes `. +refer to the :ref:`release notes `. To install the latest version (with pip):: @@ -17,8 +17,12 @@ or with conda:: conda install -c conda-forge scikit-learn + """ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + # %% # New plotting API # ---------------- @@ -26,19 +30,24 @@ # A new plotting API is available for creating visualizations. This new API # allows for quickly adjusting the visuals of a plot without involving any # recomputation. It is also possible to add different plots to the same -# figure. The following example illustrates :class:`~metrics.plot_roc_curve`, +# figure. The following example illustrates `plot_roc_curve`, # but other plots utilities are supported like -# :class:`~inspection.plot_partial_dependence`, -# :class:`~metrics.plot_precision_recall_curve`, and -# :class:`~metrics.plot_confusion_matrix`. Read more about this new API in the +# `plot_partial_dependence`, +# `plot_precision_recall_curve`, and +# `plot_confusion_matrix`. Read more about this new API in the # :ref:`User Guide `. +import matplotlib +import matplotlib.pyplot as plt + +from sklearn.datasets import make_classification +from sklearn.ensemble import RandomForestClassifier + +# from sklearn.metrics import plot_roc_curve +from sklearn.metrics import RocCurveDisplay from sklearn.model_selection import train_test_split from sklearn.svm import SVC -from sklearn.metrics import plot_roc_curve -from sklearn.ensemble import RandomForestClassifier -from sklearn.datasets import make_classification -import matplotlib.pyplot as plt +from sklearn.utils.fixes import parse_version X, y = make_classification(random_state=0) X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42) @@ -48,8 +57,11 @@ rfc = RandomForestClassifier(random_state=42) rfc.fit(X_train, y_train) -svc_disp = plot_roc_curve(svc, X_test, y_test) -rfc_disp = plot_roc_curve(rfc, X_test, y_test, ax=svc_disp.ax_) +# plot_roc_curve has been removed in version 1.2. From 1.2, use RocCurveDisplay instead. +# svc_disp = plot_roc_curve(svc, X_test, y_test) +# rfc_disp = plot_roc_curve(rfc, X_test, y_test, ax=svc_disp.ax_) +svc_disp = RocCurveDisplay.from_estimator(svc, X_test, y_test) +rfc_disp = RocCurveDisplay.from_estimator(rfc, X_test, y_test, ax=svc_disp.ax_) rfc_disp.figure_.suptitle("ROC curve comparison") plt.show() @@ -72,25 +84,20 @@ # Read more in the :ref:`User Guide `. from sklearn.datasets import load_iris -from sklearn.svm import LinearSVC -from sklearn.linear_model import LogisticRegression -from sklearn.preprocessing import StandardScaler -from sklearn.pipeline import make_pipeline from sklearn.ensemble import StackingClassifier +from sklearn.linear_model import LogisticRegression from sklearn.model_selection import train_test_split +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import StandardScaler +from sklearn.svm import LinearSVC X, y = load_iris(return_X_y=True) estimators = [ - ('rf', RandomForestClassifier(n_estimators=10, random_state=42)), - ('svr', make_pipeline(StandardScaler(), - LinearSVC(random_state=42))) + ("rf", RandomForestClassifier(n_estimators=10, random_state=42)), + ("svr", make_pipeline(StandardScaler(), LinearSVC(dual="auto", random_state=42))), ] -clf = StackingClassifier( - estimators=estimators, final_estimator=LogisticRegression() -) -X_train, X_test, y_train, y_test = train_test_split( - X, y, stratify=y, random_state=42 -) +clf = StackingClassifier(estimators=estimators, final_estimator=LogisticRegression()) +X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=42) clf.fit(X_train, y_train).score(X_test, y_test) # %% @@ -100,23 +107,33 @@ # The :func:`inspection.permutation_importance` can be used to get an # estimate of the importance of each feature, for any fitted estimator: -import numpy as np import matplotlib.pyplot as plt +import numpy as np + from sklearn.datasets import make_classification from sklearn.ensemble import RandomForestClassifier from sklearn.inspection import permutation_importance X, y = make_classification(random_state=0, n_features=5, n_informative=3) -feature_names = np.array([f'x_{i}' for i in range(X.shape[1])]) +feature_names = np.array([f"x_{i}" for i in range(X.shape[1])]) rf = RandomForestClassifier(random_state=0).fit(X, y) -result = permutation_importance(rf, X, y, n_repeats=10, random_state=0, - n_jobs=-1) +result = permutation_importance(rf, X, y, n_repeats=10, random_state=0, n_jobs=2) fig, ax = plt.subplots() sorted_idx = result.importances_mean.argsort() -ax.boxplot(result.importances[sorted_idx].T, - vert=False, labels=feature_names[sorted_idx]) + +# `labels` argument in boxplot is deprecated in matplotlib 3.9 and has been +# renamed to `tick_labels`. The following code handles this, but as a +# scikit-learn user you probably can write simpler code by using `labels=...` +# (matplotlib < 3.9) or `tick_labels=...` (matplotlib >= 3.9). +tick_labels_parameter_name = ( + "tick_labels" + if parse_version(matplotlib.__version__) >= parse_version("3.9") + else "labels" +) +tick_labels_dict = {tick_labels_parameter_name: feature_names[sorted_idx]} +ax.boxplot(result.importances[sorted_idx].T, vert=False, **tick_labels_dict) ax.set_title("Permutation Importance of each feature") ax.set_ylabel("Features") fig.tight_layout() @@ -131,7 +148,6 @@ # support for missing values (NaNs). This means that there is no need for # imputing data when training or predicting. -from sklearn.experimental import enable_hist_gradient_boosting # noqa from sklearn.ensemble import HistGradientBoostingClassifier X = np.array([0, 1, 2, np.nan]).reshape(-1, 1) @@ -154,17 +170,19 @@ # See more details in the :ref:`User Guide `. from tempfile import TemporaryDirectory -from sklearn.neighbors import KNeighborsTransformer + from sklearn.manifold import Isomap +from sklearn.neighbors import KNeighborsTransformer from sklearn.pipeline import make_pipeline X, y = make_classification(random_state=0) with TemporaryDirectory(prefix="sklearn_cache_") as tmpdir: estimator = make_pipeline( - KNeighborsTransformer(n_neighbors=10, mode='distance'), - Isomap(n_neighbors=10, metric='precomputed'), - memory=tmpdir) + KNeighborsTransformer(n_neighbors=10, mode="distance"), + Isomap(n_neighbors=10, metric="precomputed"), + memory=tmpdir, + ) estimator.fit(X) # We can decrease the number of neighbors and the graph will not be @@ -183,7 +201,7 @@ # close if the features that neither is missing are close. # By default, a euclidean distance metric # that supports missing values, -# :func:`~metrics.nan_euclidean_distances`, is used to find the nearest +# :func:`~sklearn.metrics.pairwise.nan_euclidean_distances`, is used to find the nearest # neighbors. # # Read more in the :ref:`User Guide `. @@ -205,12 +223,18 @@ X, y = make_classification(random_state=0) rf = RandomForestClassifier(random_state=0, ccp_alpha=0).fit(X, y) -print("Average number of nodes without pruning {:.1f}".format( - np.mean([e.tree_.node_count for e in rf.estimators_]))) +print( + "Average number of nodes without pruning {:.1f}".format( + np.mean([e.tree_.node_count for e in rf.estimators_]) + ) +) rf = RandomForestClassifier(random_state=0, ccp_alpha=0.05).fit(X, y) -print("Average number of nodes with pruning {:.1f}".format( - np.mean([e.tree_.node_count for e in rf.estimators_]))) +print( + "Average number of nodes with pruning {:.1f}".format( + np.mean([e.tree_.node_count for e in rf.estimators_]) + ) +) # %% # Retrieve dataframes from OpenML @@ -220,8 +244,8 @@ from sklearn.datasets import fetch_openml -titanic = fetch_openml('titanic', version=1, as_frame=True) -print(titanic.data.head()[['pclass', 'embarked']]) +titanic = fetch_openml("titanic", version=1, as_frame=True, parser="pandas") +print(titanic.data.head()[["pclass", "embarked"]]) # %% # Checking scikit-learn compatibility of an estimator @@ -246,10 +270,11 @@ def test_sklearn_compatible_estimator(estimator, check): check(estimator) + # %% # ROC AUC now supports multiclass classification # ---------------------------------------------- -# The :func:`roc_auc_score` function can also be used in multi-class +# The :func:`~sklearn.metrics.roc_auc_score` function can also be used in multi-class # classification. Two averaging strategies are currently supported: the # one-vs-one algorithm computes the average of the pairwise ROC AUC scores, and # the one-vs-rest algorithm computes the average of the ROC AUC scores for each @@ -263,9 +288,9 @@ def test_sklearn_compatible_estimator(estimator, check): from sklearn.datasets import make_classification -from sklearn.svm import SVC from sklearn.metrics import roc_auc_score +from sklearn.svm import SVC X, y = make_classification(n_classes=4, n_informative=16) -clf = SVC(decision_function_shape='ovo', probability=True).fit(X, y) -print(roc_auc_score(y, clf.predict_proba(X), multi_class='ovo')) +clf = SVC(decision_function_shape="ovo", probability=True).fit(X, y) +print(roc_auc_score(y, clf.predict_proba(X), multi_class="ovo")) diff --git a/examples/release_highlights/plot_release_highlights_0_23_0.py b/examples/release_highlights/plot_release_highlights_0_23_0.py index a34c23b4912be..00c36969ec18b 100644 --- a/examples/release_highlights/plot_release_highlights_0_23_0.py +++ b/examples/release_highlights/plot_release_highlights_0_23_0.py @@ -1,4 +1,4 @@ -# flake8: noqa +# ruff: noqa: CPY001 """ ======================================== Release Highlights for scikit-learn 0.23 @@ -9,7 +9,7 @@ We are pleased to announce the release of scikit-learn 0.23! Many bug fixes and improvements were added, as well as some new key features. We detail below a few of the major features of this release. **For an exhaustive list of -all the changes**, please refer to the :ref:`release notes `. +all the changes**, please refer to the :ref:`release notes `. To install the latest version (with pip):: @@ -18,6 +18,7 @@ or with conda:: conda install -c conda-forge scikit-learn + """ ############################################################################## @@ -34,10 +35,10 @@ # 'poisson' loss as well. import numpy as np -from sklearn.model_selection import train_test_split -from sklearn.linear_model import PoissonRegressor -from sklearn.experimental import enable_hist_gradient_boosting # noqa + from sklearn.ensemble import HistGradientBoostingRegressor +from sklearn.linear_model import PoissonRegressor +from sklearn.model_selection import train_test_split n_samples, n_features = 1000, 20 rng = np.random.RandomState(0) @@ -46,7 +47,7 @@ y = rng.poisson(lam=np.exp(X[:, 5]) / 2) X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=rng) glm = PoissonRegressor() -gbdt = HistGradientBoostingRegressor(loss='poisson', learning_rate=.01) +gbdt = HistGradientBoostingRegressor(loss="poisson", learning_rate=0.01) glm.fit(X_train, y_train) gbdt.fit(X_train, y_train) print(glm.score(X_test, y_test)) @@ -63,21 +64,24 @@ # this feature. from sklearn import set_config -from sklearn.pipeline import make_pipeline -from sklearn.preprocessing import OneHotEncoder, StandardScaler -from sklearn.impute import SimpleImputer from sklearn.compose import make_column_transformer +from sklearn.impute import SimpleImputer from sklearn.linear_model import LogisticRegression -set_config(display='diagram') +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import OneHotEncoder, StandardScaler -num_proc = make_pipeline(SimpleImputer(strategy='median'), StandardScaler()) +set_config(display="diagram") + +num_proc = make_pipeline(SimpleImputer(strategy="median"), StandardScaler()) cat_proc = make_pipeline( - SimpleImputer(strategy='constant', fill_value='missing'), - OneHotEncoder(handle_unknown='ignore')) + SimpleImputer(strategy="constant", fill_value="missing"), + OneHotEncoder(handle_unknown="ignore"), +) -preprocessor = make_column_transformer((num_proc, ('feat1', 'feat3')), - (cat_proc, ('feat0', 'feat2'))) +preprocessor = make_column_transformer( + (num_proc, ("feat1", "feat3")), (cat_proc, ("feat0", "feat2")) +) clf = make_pipeline(preprocessor, LogisticRegression()) clf @@ -91,18 +95,19 @@ # parallelism instead of relying on joblib, so the `n_jobs` parameter has no # effect anymore. For more details on how to control the number of threads, # please refer to our :ref:`parallelism` notes. -import scipy import numpy as np -from sklearn.model_selection import train_test_split +import scipy + from sklearn.cluster import KMeans from sklearn.datasets import make_blobs from sklearn.metrics import completeness_score +from sklearn.model_selection import train_test_split rng = np.random.RandomState(0) X, y = make_blobs(random_state=rng) X = scipy.sparse.csr_matrix(X) X_train, X_test, _, y_test = train_test_split(X, y, random_state=rng) -kmeans = KMeans(algorithm='elkan').fit(X_train) +kmeans = KMeans(n_init="auto").fit(X_train) print(completeness_score(kmeans.predict(X_test), y_test)) ############################################################################## @@ -111,7 +116,7 @@ # Various improvements were made to # :class:`~sklearn.ensemble.HistGradientBoostingClassifier` and # :class:`~sklearn.ensemble.HistGradientBoostingRegressor`. On top of the -# Poisson loss mentionned above, these estimators now support :ref:`sample +# Poisson loss mentioned above, these estimators now support :ref:`sample # weights `. Also, an automatic early-stopping criterion was added: # early-stopping is enabled by default when the number of samples exceeds 10k. # Finally, users can now define :ref:`monotonic constraints @@ -119,33 +124,49 @@ # specific features. In the following example, we construct a target that is # generally positively correlated with the first feature, with some noise. # Applying monotoinc constraints allows the prediction to capture the global -# effect of the first feature, instead of fitting the noise. +# effect of the first feature, instead of fitting the noise. For a usecase +# example, see :ref:`sphx_glr_auto_examples_ensemble_plot_hgbt_regression.py`. import numpy as np from matplotlib import pyplot as plt -from sklearn.model_selection import train_test_split -from sklearn.inspection import plot_partial_dependence -from sklearn.experimental import enable_hist_gradient_boosting # noqa + from sklearn.ensemble import HistGradientBoostingRegressor +# from sklearn.inspection import plot_partial_dependence +from sklearn.inspection import PartialDependenceDisplay +from sklearn.model_selection import train_test_split + n_samples = 500 rng = np.random.RandomState(0) X = rng.randn(n_samples, 2) noise = rng.normal(loc=0.0, scale=0.01, size=n_samples) -y = (5 * X[:, 0] + np.sin(10 * np.pi * X[:, 0]) - noise) +y = 5 * X[:, 0] + np.sin(10 * np.pi * X[:, 0]) - noise gbdt_no_cst = HistGradientBoostingRegressor().fit(X, y) gbdt_cst = HistGradientBoostingRegressor(monotonic_cst=[1, 0]).fit(X, y) -disp = plot_partial_dependence( - gbdt_no_cst, X, features=[0], feature_names=['feature 0'], - line_kw={'linewidth': 4, 'label': 'unconstrained', "color": "tab:blue"}) -plot_partial_dependence(gbdt_cst, X, features=[0], - line_kw={'linewidth': 4, 'label': 'constrained', "color": "tab:orange"}, - ax=disp.axes_) +# plot_partial_dependence has been removed in version 1.2. From 1.2, use +# PartialDependenceDisplay instead. +# disp = plot_partial_dependence( +disp = PartialDependenceDisplay.from_estimator( + gbdt_no_cst, + X, + features=[0], + feature_names=["feature 0"], + line_kw={"linewidth": 4, "label": "unconstrained", "color": "tab:blue"}, +) +# plot_partial_dependence( +PartialDependenceDisplay.from_estimator( + gbdt_cst, + X, + features=[0], + line_kw={"linewidth": 4, "label": "constrained", "color": "tab:orange"}, + ax=disp.axes_, +) disp.axes_[0, 0].plot( - X[:, 0], y, 'o', alpha=.5, zorder=-1, label='samples', color="tab:green" + X[:, 0], y, "o", alpha=0.5, zorder=-1, label="samples", color="tab:green" ) -disp.axes_[0, 0].set_ylim(-3, 3); disp.axes_[0, 0].set_xlim(-1, 1) +disp.axes_[0, 0].set_ylim(-3, 3) +disp.axes_[0, 0].set_xlim(-1, 1) plt.legend() plt.show() @@ -155,17 +176,19 @@ # The two linear regressors :class:`~sklearn.linear_model.Lasso` and # :class:`~sklearn.linear_model.ElasticNet` now support sample weights. -from sklearn.model_selection import train_test_split +import numpy as np + from sklearn.datasets import make_regression from sklearn.linear_model import Lasso -import numpy as np +from sklearn.model_selection import train_test_split n_samples, n_features = 1000, 20 rng = np.random.RandomState(0) X, y = make_regression(n_samples, n_features, random_state=rng) sample_weight = rng.rand(n_samples) X_train, X_test, y_train, y_test, sw_train, sw_test = train_test_split( - X, y, sample_weight, random_state=rng) + X, y, sample_weight, random_state=rng +) reg = Lasso() reg.fit(X_train, y_train, sample_weight=sw_train) print(reg.score(X_test, y_test, sw_test)) diff --git a/examples/release_highlights/plot_release_highlights_0_24_0.py b/examples/release_highlights/plot_release_highlights_0_24_0.py index f5b10dfb21acc..d09250ba6ff64 100644 --- a/examples/release_highlights/plot_release_highlights_0_24_0.py +++ b/examples/release_highlights/plot_release_highlights_0_24_0.py @@ -1,4 +1,4 @@ -# flake8: noqa +# ruff: noqa: CPY001, E501 """ ======================================== Release Highlights for scikit-learn 0.24 @@ -9,7 +9,7 @@ We are pleased to announce the release of scikit-learn 0.24! Many bug fixes and improvements were added, as well as some new key features. We detail below a few of the major features of this release. **For an exhaustive list of -all the changes**, please refer to the :ref:`release notes `. +all the changes**, please refer to the :ref:`release notes `. To install the latest version (with pip):: @@ -18,6 +18,7 @@ or with conda:: conda install -c conda-forge scikit-learn + """ ############################################################################## @@ -43,17 +44,18 @@ # Read more in the :ref:`User Guide ` (note: # the Successive Halving estimators are still :term:`experimental # `). -# +# # .. figure:: ../model_selection/images/sphx_glr_plot_successive_halving_iterations_001.png # :target: ../model_selection/plot_successive_halving_iterations.html # :align: center import numpy as np from scipy.stats import randint -from sklearn.experimental import enable_halving_search_cv # noqa -from sklearn.model_selection import HalvingRandomSearchCV -from sklearn.ensemble import RandomForestClassifier + from sklearn.datasets import make_classification +from sklearn.ensemble import RandomForestClassifier +from sklearn.experimental import enable_halving_search_cv # noqa: F401 +from sklearn.model_selection import HalvingRandomSearchCV rng = np.random.RandomState(0) @@ -61,14 +63,17 @@ clf = RandomForestClassifier(n_estimators=10, random_state=rng) -param_dist = {"max_depth": [3, None], - "max_features": randint(1, 11), - "min_samples_split": randint(2, 11), - "bootstrap": [True, False], - "criterion": ["gini", "entropy"]} +param_dist = { + "max_depth": [3, None], + "max_features": randint(1, 11), + "min_samples_split": randint(2, 11), + "bootstrap": [True, False], + "criterion": ["gini", "entropy"], +} -rsh = HalvingRandomSearchCV(estimator=clf, param_distributions=param_dist, - factor=2, random_state=rng) +rsh = HalvingRandomSearchCV( + estimator=clf, param_distributions=param_dist, factor=2, random_state=rng +) rsh.fit(X, y) rsh.best_params_ @@ -114,6 +119,7 @@ # Read more in the :ref:`User guide `. import numpy as np + from sklearn import datasets from sklearn.semi_supervised import SelfTrainingClassifier from sklearn.svm import SVC @@ -136,17 +142,19 @@ # (backward selection), based on a cross-validated score maximization. # See the :ref:`User Guide `. +from sklearn.datasets import load_iris from sklearn.feature_selection import SequentialFeatureSelector from sklearn.neighbors import KNeighborsClassifier -from sklearn.datasets import load_iris X, y = load_iris(return_X_y=True, as_frame=True) feature_names = X.columns knn = KNeighborsClassifier(n_neighbors=3) sfs = SequentialFeatureSelector(knn, n_features_to_select=2) sfs.fit(X, y) -print("Features selected by forward sequential selection: " - f"{feature_names[sfs.get_support()].tolist()}") +print( + "Features selected by forward sequential selection: " + f"{feature_names[sfs.get_support()].tolist()}" +) ############################################################################## # New PolynomialCountSketch kernel approximation function @@ -157,26 +165,27 @@ # :class:`~sklearn.preprocessing.PolynomialFeatures`. from sklearn.datasets import fetch_covtype -from sklearn.pipeline import make_pipeline -from sklearn.model_selection import train_test_split -from sklearn.preprocessing import MinMaxScaler from sklearn.kernel_approximation import PolynomialCountSketch from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import train_test_split +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import MinMaxScaler X, y = fetch_covtype(return_X_y=True) -pipe = make_pipeline(MinMaxScaler(), - PolynomialCountSketch(degree=2, n_components=300), - LogisticRegression(max_iter=1000)) -X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=5000, - test_size=10000, - random_state=42) +pipe = make_pipeline( + MinMaxScaler(), + PolynomialCountSketch(degree=2, n_components=300), + LogisticRegression(max_iter=1000), +) +X_train, X_test, y_train, y_test = train_test_split( + X, y, train_size=5000, test_size=10000, random_state=42 +) pipe.fit(X_train, y_train).score(X_test, y_test) ############################################################################## # For comparison, here is the score of a linear baseline for the same data: -linear_baseline = make_pipeline(MinMaxScaler(), - LogisticRegression(max_iter=1000)) +linear_baseline = make_pipeline(MinMaxScaler(), LogisticRegression(max_iter=1000)) linear_baseline.fit(X_train, y_train).score(X_test, y_test) ############################################################################## @@ -187,21 +196,33 @@ # prediction on a feature for each sample separately, with one line per sample. # See the :ref:`User Guide ` -from sklearn.ensemble import RandomForestRegressor from sklearn.datasets import fetch_california_housing -from sklearn.inspection import plot_partial_dependence +from sklearn.ensemble import RandomForestRegressor + +# from sklearn.inspection import plot_partial_dependence +from sklearn.inspection import PartialDependenceDisplay X, y = fetch_california_housing(return_X_y=True, as_frame=True) -features = ['MedInc', 'AveOccup', 'HouseAge', 'AveRooms'] +features = ["MedInc", "AveOccup", "HouseAge", "AveRooms"] est = RandomForestRegressor(n_estimators=10) est.fit(X, y) -display = plot_partial_dependence( - est, X, features, kind="individual", subsample=50, - n_jobs=3, grid_resolution=20, random_state=0 + +# plot_partial_dependence has been removed in version 1.2. From 1.2, use +# PartialDependenceDisplay instead. +# display = plot_partial_dependence( +display = PartialDependenceDisplay.from_estimator( + est, + X, + features, + kind="individual", + subsample=50, + n_jobs=3, + grid_resolution=20, + random_state=0, ) display.figure_.suptitle( - 'Partial dependence of house value on non-location features\n' - 'for the California housing dataset, with BayesianRidge' + "Partial dependence of house value on non-location features\n" + "for the California housing dataset, with BayesianRidge" ) display.figure_.subplots_adjust(hspace=0.3) @@ -213,17 +234,18 @@ # splitting criterion. Setting `criterion="poisson"` might be a good choice # if your target is a count or a frequency. -from sklearn.tree import DecisionTreeRegressor -from sklearn.model_selection import train_test_split import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.tree import DecisionTreeRegressor + n_samples, n_features = 1000, 20 rng = np.random.RandomState(0) X = rng.randn(n_samples, n_features) # positive integer target correlated with X[:, 5] with many zeros: y = rng.poisson(lam=np.exp(X[:, 5]) / 2) X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=rng) -regressor = DecisionTreeRegressor(criterion='poisson', random_state=0) +regressor = DecisionTreeRegressor(criterion="poisson", random_state=0) regressor.fit(X_train, y_train) ############################################################################## diff --git a/examples/release_highlights/plot_release_highlights_1_0_0.py b/examples/release_highlights/plot_release_highlights_1_0_0.py new file mode 100644 index 0000000000000..03213076b326e --- /dev/null +++ b/examples/release_highlights/plot_release_highlights_1_0_0.py @@ -0,0 +1,246 @@ +# ruff: noqa: CPY001 +""" +======================================= +Release Highlights for scikit-learn 1.0 +======================================= + +.. currentmodule:: sklearn + +We are very pleased to announce the release of scikit-learn 1.0! The library +has been stable for quite some time, releasing version 1.0 is recognizing that +and signalling it to our users. This release does not include any breaking +changes apart from the usual two-release deprecation cycle. For the future, we +do our best to keep this pattern. + +This release includes some new key features as well as many improvements and +bug fixes. We detail below a few of the major features of this release. **For +an exhaustive list of all the changes**, please refer to the :ref:`release +notes `. + +To install the latest version (with pip):: + + pip install --upgrade scikit-learn + +or with conda:: + + conda install -c conda-forge scikit-learn + +""" + +############################################################################## +# Keyword and positional arguments +# --------------------------------------------------------- +# The scikit-learn API exposes many functions and methods which have many input +# parameters. For example, before this release, one could instantiate a +# :class:`~ensemble.HistGradientBoostingRegressor` as:: +# +# HistGradientBoostingRegressor("squared_error", 0.1, 100, 31, None, +# 20, 0.0, 255, None, None, False, "auto", "loss", 0.1, 10, 1e-7, +# 0, None) +# +# Understanding the above code requires the reader to go to the API +# documentation and to check each and every parameter for its position and +# its meaning. To improve the readability of code written based on scikit-learn, +# now users have to provide most parameters with their names, as keyword +# arguments, instead of positional arguments. For example, the above code would +# be:: +# +# HistGradientBoostingRegressor( +# loss="squared_error", +# learning_rate=0.1, +# max_iter=100, +# max_leaf_nodes=31, +# max_depth=None, +# min_samples_leaf=20, +# l2_regularization=0.0, +# max_bins=255, +# categorical_features=None, +# monotonic_cst=None, +# warm_start=False, +# early_stopping="auto", +# scoring="loss", +# validation_fraction=0.1, +# n_iter_no_change=10, +# tol=1e-7, +# verbose=0, +# random_state=None, +# ) +# +# which is much more readable. Positional arguments have been deprecated since +# version 0.23 and will now raise a ``TypeError``. A limited number of +# positional arguments are still allowed in some cases, for example in +# :class:`~decomposition.PCA`, where ``PCA(10)`` is still allowed, but ``PCA(10, +# False)`` is not allowed. + +############################################################################## +# Spline Transformers +# --------------------------------------------------------- +# One way to add nonlinear terms to a dataset's feature set is to generate +# spline basis functions for continuous/numerical features with the new +# :class:`~preprocessing.SplineTransformer`. Splines are piecewise polynomials, +# parametrized by their polynomial degree and the positions of the knots. The +# :class:`~preprocessing.SplineTransformer` implements a B-spline basis. +# +# .. figure:: ../linear_model/images/sphx_glr_plot_polynomial_interpolation_001.png +# :target: ../linear_model/plot_polynomial_interpolation.html +# :align: center +# +# The following code shows splines in action, for more information, please +# refer to the :ref:`User Guide `. + +import numpy as np + +from sklearn.preprocessing import SplineTransformer + +X = np.arange(5).reshape(5, 1) +spline = SplineTransformer(degree=2, n_knots=3) +spline.fit_transform(X) + + +############################################################################## +# Quantile Regressor +# -------------------------------------------------------------------------- +# Quantile regression estimates the median or other quantiles of :math:`y` +# conditional on :math:`X`, while ordinary least squares (OLS) estimates the +# conditional mean. +# +# As a linear model, the new :class:`~linear_model.QuantileRegressor` gives +# linear predictions :math:`\hat{y}(w, X) = Xw` for the :math:`q`-th quantile, +# :math:`q \in (0, 1)`. The weights or coefficients :math:`w` are then found by +# the following minimization problem: +# +# .. math:: +# \min_{w} {\frac{1}{n_{\text{samples}}} +# \sum_i PB_q(y_i - X_i w) + \alpha ||w||_1}. +# +# This consists of the pinball loss (also known as linear loss), +# see also :class:`~sklearn.metrics.mean_pinball_loss`, +# +# .. math:: +# PB_q(t) = q \max(t, 0) + (1 - q) \max(-t, 0) = +# \begin{cases} +# q t, & t > 0, \\ +# 0, & t = 0, \\ +# (1-q) t, & t < 0 +# \end{cases} +# +# and the L1 penalty controlled by parameter ``alpha``, similar to +# :class:`linear_model.Lasso`. +# +# Please check the following example to see how it works, and the :ref:`User +# Guide ` for more details. +# +# .. figure:: ../linear_model/images/sphx_glr_plot_quantile_regression_002.png +# :target: ../linear_model/plot_quantile_regression.html +# :align: center +# :scale: 50% + +############################################################################## +# Feature Names Support +# -------------------------------------------------------------------------- +# When an estimator is passed a `pandas' dataframe +# `_ during +# :term:`fit`, the estimator will set a `feature_names_in_` attribute +# containing the feature names. This is a part of +# `SLEP007 `__. +# Note that feature names support is only enabled +# when the column names in the dataframe are all strings. `feature_names_in_` +# is used to check that the column names of the dataframe passed in +# non-:term:`fit`, such as :term:`predict`, are consistent with features in +# :term:`fit`: +import pandas as pd + +from sklearn.preprocessing import StandardScaler + +X = pd.DataFrame([[1, 2, 3], [4, 5, 6]], columns=["a", "b", "c"]) +scalar = StandardScaler().fit(X) +scalar.feature_names_in_ + +# %% +# The support of :term:`get_feature_names_out` is available for transformers +# that already had `get_feature_names` and transformers with a one-to-one +# correspondence between input and output such as +# :class:`~preprocessing.StandardScaler`. :term:`get_feature_names_out` support +# will be added to all other transformers in future releases. Additionally, +# :meth:`compose.ColumnTransformer.get_feature_names_out` is available to +# combine feature names of its transformers: +import pandas as pd + +from sklearn.compose import ColumnTransformer +from sklearn.preprocessing import OneHotEncoder + +X = pd.DataFrame({"pet": ["dog", "cat", "fish"], "age": [3, 7, 1]}) +preprocessor = ColumnTransformer( + [ + ("numerical", StandardScaler(), ["age"]), + ("categorical", OneHotEncoder(), ["pet"]), + ], + verbose_feature_names_out=False, +).fit(X) + +preprocessor.get_feature_names_out() + +# %% +# When this ``preprocessor`` is used with a pipeline, the feature names used +# by the classifier are obtained by slicing and calling +# :term:`get_feature_names_out`: +from sklearn.linear_model import LogisticRegression +from sklearn.pipeline import make_pipeline + +y = [1, 0, 1] +pipe = make_pipeline(preprocessor, LogisticRegression()) +pipe.fit(X, y) +pipe[:-1].get_feature_names_out() + + +############################################################################## +# A more flexible plotting API +# -------------------------------------------------------------------------- +# :class:`metrics.ConfusionMatrixDisplay`, +# :class:`metrics.PrecisionRecallDisplay`, :class:`metrics.DetCurveDisplay`, +# and :class:`inspection.PartialDependenceDisplay` now expose two class +# methods: `from_estimator` and `from_predictions` which allow users to create +# a plot given the predictions or an estimator. This means the corresponding +# `plot_*` functions are deprecated. Please check :ref:`example one +# ` and +# :ref:`example two +# ` for +# how to use the new plotting functionalities. + +############################################################################## +# Online One-Class SVM +# -------------------------------------------------------------------------- +# The new class :class:`~linear_model.SGDOneClassSVM` implements an online +# linear version of the One-Class SVM using a stochastic gradient descent. +# Combined with kernel approximation techniques, +# :class:`~linear_model.SGDOneClassSVM` can be used to approximate the solution +# of a kernelized One-Class SVM, implemented in :class:`~svm.OneClassSVM`, with +# a fit time complexity linear in the number of samples. Note that the +# complexity of a kernelized One-Class SVM is at best quadratic in the number +# of samples. :class:`~linear_model.SGDOneClassSVM` is thus well suited for +# datasets with a large number of training samples (> 10,000) for which the SGD +# variant can be several orders of magnitude faster. Please check this +# :ref:`example +# ` to see how +# it's used, and the :ref:`User Guide ` for more +# details. +# +# .. figure:: ../miscellaneous/images/sphx_glr_plot_anomaly_comparison_001.png +# :target: ../miscellaneous/plot_anomaly_comparison.html +# :align: center + +############################################################################## +# Histogram-based Gradient Boosting Models are now stable +# -------------------------------------------------------------------------- +# :class:`~sklearn.ensemble.HistGradientBoostingRegressor` and +# :class:`~ensemble.HistGradientBoostingClassifier` are no longer experimental +# and can simply be imported and used as:: +# +# from sklearn.ensemble import HistGradientBoostingClassifier + +############################################################################## +# New documentation improvements +# ------------------------------ +# This release includes many documentation improvements. Out of over 2100 +# merged pull requests, about 800 of them are improvements to our +# documentation. diff --git a/examples/release_highlights/plot_release_highlights_1_1_0.py b/examples/release_highlights/plot_release_highlights_1_1_0.py new file mode 100644 index 0000000000000..fdb11f887f3db --- /dev/null +++ b/examples/release_highlights/plot_release_highlights_1_1_0.py @@ -0,0 +1,240 @@ +# ruff: noqa: CPY001 +""" +======================================= +Release Highlights for scikit-learn 1.1 +======================================= + +.. currentmodule:: sklearn + +We are pleased to announce the release of scikit-learn 1.1! Many bug fixes +and improvements were added, as well as some new key features. We detail +below a few of the major features of this release. **For an exhaustive list of +all the changes**, please refer to the :ref:`release notes `. + +To install the latest version (with pip):: + + pip install --upgrade scikit-learn + +or with conda:: + + conda install -c conda-forge scikit-learn + +""" + +# %% +# .. _quantile_support_hgbdt: +# +# Quantile loss in :class:`~ensemble.HistGradientBoostingRegressor` +# ----------------------------------------------------------------- +# :class:`~ensemble.HistGradientBoostingRegressor` can model quantiles with +# `loss="quantile"` and the new parameter `quantile`. +import matplotlib.pyplot as plt +import numpy as np + +from sklearn.ensemble import HistGradientBoostingRegressor + +# Simple regression function for X * cos(X) +rng = np.random.RandomState(42) +X_1d = np.linspace(0, 10, num=2000) +X = X_1d.reshape(-1, 1) +y = X_1d * np.cos(X_1d) + rng.normal(scale=X_1d / 3) + +quantiles = [0.95, 0.5, 0.05] +parameters = dict(loss="quantile", max_bins=32, max_iter=50) +hist_quantiles = { + f"quantile={quantile:.2f}": HistGradientBoostingRegressor( + **parameters, quantile=quantile + ).fit(X, y) + for quantile in quantiles +} + +fig, ax = plt.subplots() +ax.plot(X_1d, y, "o", alpha=0.5, markersize=1) +for quantile, hist in hist_quantiles.items(): + ax.plot(X_1d, hist.predict(X), label=quantile) +_ = ax.legend(loc="lower left") + +# %% +# For a usecase example, see +# :ref:`sphx_glr_auto_examples_ensemble_plot_hgbt_regression.py` + +# %% +# `get_feature_names_out` Available in all Transformers +# ----------------------------------------------------- +# :term:`get_feature_names_out` is now available in all transformers, thereby +# concluding the implementation of +# `SLEP007 `__. +# This enables :class:`~pipeline.Pipeline` to construct the output feature names for +# more complex pipelines: +from sklearn.compose import ColumnTransformer +from sklearn.datasets import fetch_openml +from sklearn.feature_selection import SelectKBest +from sklearn.impute import SimpleImputer +from sklearn.linear_model import LogisticRegression +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import OneHotEncoder, StandardScaler + +X, y = fetch_openml( + "titanic", version=1, as_frame=True, return_X_y=True, parser="pandas" +) +numeric_features = ["age", "fare"] +numeric_transformer = make_pipeline(SimpleImputer(strategy="median"), StandardScaler()) +categorical_features = ["embarked", "pclass"] + +preprocessor = ColumnTransformer( + [ + ("num", numeric_transformer, numeric_features), + ( + "cat", + OneHotEncoder(handle_unknown="ignore", sparse_output=False), + categorical_features, + ), + ], + verbose_feature_names_out=False, +) +log_reg = make_pipeline(preprocessor, SelectKBest(k=7), LogisticRegression()) +log_reg.fit(X, y) + + +# %% +# Here we slice the pipeline to include all the steps but the last one. The output +# feature names of this pipeline slice are the features put into logistic +# regression. These names correspond directly to the coefficients in the logistic +# regression: +import pandas as pd + +log_reg_input_features = log_reg[:-1].get_feature_names_out() +pd.Series(log_reg[-1].coef_.ravel(), index=log_reg_input_features).plot.bar() +plt.tight_layout() + + +# %% +# Grouping infrequent categories in :class:`~preprocessing.OneHotEncoder` +# ----------------------------------------------------------------------- +# :class:`~preprocessing.OneHotEncoder` supports aggregating infrequent +# categories into a single output for each feature. The parameters to enable +# the gathering of infrequent categories are `min_frequency` and +# `max_categories`. See the :ref:`User Guide ` +# for more details. +import numpy as np + +from sklearn.preprocessing import OneHotEncoder + +X = np.array( + [["dog"] * 5 + ["cat"] * 20 + ["rabbit"] * 10 + ["snake"] * 3], dtype=object +).T +enc = OneHotEncoder(min_frequency=6, sparse_output=False).fit(X) +enc.infrequent_categories_ + +# %% +# Since dog and snake are infrequent categories, they are grouped together when +# transformed: +encoded = enc.transform(np.array([["dog"], ["snake"], ["cat"], ["rabbit"]])) +pd.DataFrame(encoded, columns=enc.get_feature_names_out()) + +# %% +# Performance improvements +# ------------------------ +# Reductions on pairwise distances for dense float64 datasets has been refactored +# to better take advantage of non-blocking thread parallelism. For example, +# :meth:`neighbors.NearestNeighbors.kneighbors` and +# :meth:`neighbors.NearestNeighbors.radius_neighbors` can respectively be up to ×20 and +# ×5 faster than previously. In summary, the following functions and estimators +# now benefit from improved performance: +# +# - :func:`metrics.pairwise_distances_argmin` +# - :func:`metrics.pairwise_distances_argmin_min` +# - :class:`cluster.AffinityPropagation` +# - :class:`cluster.Birch` +# - :class:`cluster.MeanShift` +# - :class:`cluster.OPTICS` +# - :class:`cluster.SpectralClustering` +# - :func:`feature_selection.mutual_info_regression` +# - :class:`neighbors.KNeighborsClassifier` +# - :class:`neighbors.KNeighborsRegressor` +# - :class:`neighbors.RadiusNeighborsClassifier` +# - :class:`neighbors.RadiusNeighborsRegressor` +# - :class:`neighbors.LocalOutlierFactor` +# - :class:`neighbors.NearestNeighbors` +# - :class:`manifold.Isomap` +# - :class:`manifold.LocallyLinearEmbedding` +# - :class:`manifold.TSNE` +# - :func:`manifold.trustworthiness` +# - :class:`semi_supervised.LabelPropagation` +# - :class:`semi_supervised.LabelSpreading` +# +# To know more about the technical details of this work, you can read +# `this suite of blog posts `_. +# +# Moreover, the computation of loss functions has been refactored using +# Cython resulting in performance improvements for the following estimators: +# +# - :class:`linear_model.LogisticRegression` +# - :class:`linear_model.GammaRegressor` +# - :class:`linear_model.PoissonRegressor` +# - :class:`linear_model.TweedieRegressor` + +# %% +# :class:`~decomposition.MiniBatchNMF`: an online version of NMF +# -------------------------------------------------------------- +# The new class :class:`~decomposition.MiniBatchNMF` implements a faster but +# less accurate version of non-negative matrix factorization +# (:class:`~decomposition.NMF`). :class:`~decomposition.MiniBatchNMF` divides the +# data into mini-batches and optimizes the NMF model in an online manner by +# cycling over the mini-batches, making it better suited for large datasets. In +# particular, it implements `partial_fit`, which can be used for online +# learning when the data is not readily available from the start, or when the +# data does not fit into memory. +import numpy as np + +from sklearn.decomposition import MiniBatchNMF + +rng = np.random.RandomState(0) +n_samples, n_features, n_components = 10, 10, 5 +true_W = rng.uniform(size=(n_samples, n_components)) +true_H = rng.uniform(size=(n_components, n_features)) +X = true_W @ true_H + +nmf = MiniBatchNMF(n_components=n_components, random_state=0) + +for _ in range(10): + nmf.partial_fit(X) + +W = nmf.transform(X) +H = nmf.components_ +X_reconstructed = W @ H + +print( + "relative reconstruction error: ", + f"{np.sum((X - X_reconstructed) ** 2) / np.sum(X**2):.5f}", +) + +# %% +# :class:`~cluster.BisectingKMeans`: divide and cluster +# ----------------------------------------------------- +# The new class :class:`~cluster.BisectingKMeans` is a variant of +# :class:`~cluster.KMeans`, using divisive hierarchical clustering. Instead of +# creating all centroids at once, centroids are picked progressively based on a +# previous clustering: a cluster is split into two new clusters repeatedly +# until the target number of clusters is reached, giving a hierarchical +# structure to the clustering. +import matplotlib.pyplot as plt + +from sklearn.cluster import BisectingKMeans, KMeans +from sklearn.datasets import make_blobs + +X, _ = make_blobs(n_samples=1000, centers=2, random_state=0) + +km = KMeans(n_clusters=5, random_state=0, n_init="auto").fit(X) +bisect_km = BisectingKMeans(n_clusters=5, random_state=0).fit(X) + +fig, ax = plt.subplots(1, 2, figsize=(10, 5)) +ax[0].scatter(X[:, 0], X[:, 1], s=10, c=km.labels_) +ax[0].scatter(km.cluster_centers_[:, 0], km.cluster_centers_[:, 1], s=20, c="r") +ax[0].set_title("KMeans") + +ax[1].scatter(X[:, 0], X[:, 1], s=10, c=bisect_km.labels_) +ax[1].scatter( + bisect_km.cluster_centers_[:, 0], bisect_km.cluster_centers_[:, 1], s=20, c="r" +) +_ = ax[1].set_title("BisectingKMeans") diff --git a/examples/release_highlights/plot_release_highlights_1_2_0.py b/examples/release_highlights/plot_release_highlights_1_2_0.py new file mode 100644 index 0000000000000..ee5316229dd9a --- /dev/null +++ b/examples/release_highlights/plot_release_highlights_1_2_0.py @@ -0,0 +1,172 @@ +# ruff: noqa: CPY001, E501 +""" +======================================= +Release Highlights for scikit-learn 1.2 +======================================= + +.. currentmodule:: sklearn + +We are pleased to announce the release of scikit-learn 1.2! Many bug fixes +and improvements were added, as well as some new key features. We detail +below a few of the major features of this release. **For an exhaustive list of +all the changes**, please refer to the :ref:`release notes `. + +To install the latest version (with pip):: + + pip install --upgrade scikit-learn + +or with conda:: + + conda install -c conda-forge scikit-learn + +""" + +# %% +# Pandas output with `set_output` API +# ----------------------------------- +# scikit-learn's transformers now support pandas output with the `set_output` API. +# To learn more about the `set_output` API see the example: +# :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` and +# # this `video, pandas DataFrame output for scikit-learn transformers +# (some examples) `__. + +import numpy as np + +from sklearn.compose import ColumnTransformer +from sklearn.datasets import load_iris +from sklearn.preprocessing import KBinsDiscretizer, StandardScaler + +X, y = load_iris(as_frame=True, return_X_y=True) +sepal_cols = ["sepal length (cm)", "sepal width (cm)"] +petal_cols = ["petal length (cm)", "petal width (cm)"] + +preprocessor = ColumnTransformer( + [ + ("scaler", StandardScaler(), sepal_cols), + ( + "kbin", + KBinsDiscretizer(encode="ordinal", quantile_method="averaged_inverted_cdf"), + petal_cols, + ), + ], + verbose_feature_names_out=False, +).set_output(transform="pandas") + +X_out = preprocessor.fit_transform(X) +X_out.sample(n=5, random_state=0) + +# %% +# Interaction constraints in Histogram-based Gradient Boosting Trees +# ------------------------------------------------------------------ +# :class:`~ensemble.HistGradientBoostingRegressor` and +# :class:`~ensemble.HistGradientBoostingClassifier` now supports interaction constraints +# with the `interaction_cst` parameter. For details, see the +# :ref:`User Guide `. In the following example, features are not +# allowed to interact. +from sklearn.datasets import load_diabetes +from sklearn.ensemble import HistGradientBoostingRegressor + +X, y = load_diabetes(return_X_y=True, as_frame=True) + +hist_no_interact = HistGradientBoostingRegressor( + interaction_cst=[[i] for i in range(X.shape[1])], random_state=0 +) +hist_no_interact.fit(X, y) + +# %% +# New and enhanced displays +# ------------------------- +# :class:`~metrics.PredictionErrorDisplay` provides a way to analyze regression +# models in a qualitative manner. +import matplotlib.pyplot as plt + +from sklearn.metrics import PredictionErrorDisplay + +fig, axs = plt.subplots(nrows=1, ncols=2, figsize=(12, 5)) +_ = PredictionErrorDisplay.from_estimator( + hist_no_interact, X, y, kind="actual_vs_predicted", ax=axs[0] +) +_ = PredictionErrorDisplay.from_estimator( + hist_no_interact, X, y, kind="residual_vs_predicted", ax=axs[1] +) + +# %% +# :class:`~model_selection.LearningCurveDisplay` is now available to plot +# results from :func:`~model_selection.learning_curve`. +from sklearn.model_selection import LearningCurveDisplay + +_ = LearningCurveDisplay.from_estimator( + hist_no_interact, X, y, cv=5, n_jobs=2, train_sizes=np.linspace(0.1, 1, 5) +) + +# %% +# :class:`~inspection.PartialDependenceDisplay` exposes a new parameter +# `categorical_features` to display partial dependence for categorical features +# using bar plots and heatmaps. +from sklearn.datasets import fetch_openml + +X, y = fetch_openml( + "titanic", version=1, as_frame=True, return_X_y=True, parser="pandas" +) +X = X.select_dtypes(["number", "category"]).drop(columns=["body"]) + +# %% +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import OrdinalEncoder + +categorical_features = ["pclass", "sex", "embarked"] +model = make_pipeline( + ColumnTransformer( + transformers=[("cat", OrdinalEncoder(), categorical_features)], + remainder="passthrough", + ), + HistGradientBoostingRegressor(random_state=0), +).fit(X, y) + +# %% +from sklearn.inspection import PartialDependenceDisplay + +fig, ax = plt.subplots(figsize=(14, 4), constrained_layout=True) +_ = PartialDependenceDisplay.from_estimator( + model, + X, + features=["age", "sex", ("pclass", "sex")], + categorical_features=categorical_features, + ax=ax, +) + +# %% +# Faster parser in :func:`~datasets.fetch_openml` +# ----------------------------------------------- +# :func:`~datasets.fetch_openml` now supports a new `"pandas"` parser that is +# more memory and CPU efficient. In v1.4, the default will change to +# `parser="auto"` which will automatically use the `"pandas"` parser for dense +# data and `"liac-arff"` for sparse data. +X, y = fetch_openml( + "titanic", version=1, as_frame=True, return_X_y=True, parser="pandas" +) +X.head() + +# %% +# Experimental Array API support in :class:`~discriminant_analysis.LinearDiscriminantAnalysis` +# -------------------------------------------------------------------------------------------- +# Experimental support for the `Array API `_ +# specification was added to :class:`~discriminant_analysis.LinearDiscriminantAnalysis`. +# The estimator can now run on any Array API compliant libraries such as +# `CuPy `__, a GPU-accelerated array +# library. For details, see the :ref:`User Guide `. + +# %% +# Improved efficiency of many estimators +# -------------------------------------- +# In version 1.1 the efficiency of many estimators relying on the computation of +# pairwise distances (essentially estimators related to clustering, manifold +# learning and neighbors search algorithms) was greatly improved for float64 +# dense input. Efficiency improvement especially were a reduced memory footprint +# and a much better scalability on multi-core machines. +# In version 1.2, the efficiency of these estimators was further improved for all +# combinations of dense and sparse inputs on float32 and float64 datasets, except +# the sparse-dense and dense-sparse combinations for the Euclidean and Squared +# Euclidean Distance metrics. +# A detailed list of the impacted estimators can be found in the +# :ref:`changelog `. diff --git a/examples/release_highlights/plot_release_highlights_1_3_0.py b/examples/release_highlights/plot_release_highlights_1_3_0.py new file mode 100644 index 0000000000000..f7faad08c9b1e --- /dev/null +++ b/examples/release_highlights/plot_release_highlights_1_3_0.py @@ -0,0 +1,163 @@ +# ruff: noqa: CPY001 +""" +======================================= +Release Highlights for scikit-learn 1.3 +======================================= + +.. currentmodule:: sklearn + +We are pleased to announce the release of scikit-learn 1.3! Many bug fixes +and improvements were added, as well as some new key features. We detail +below a few of the major features of this release. **For an exhaustive list of +all the changes**, please refer to the :ref:`release notes `. + +To install the latest version (with pip):: + + pip install --upgrade scikit-learn + +or with conda:: + + conda install -c conda-forge scikit-learn + +""" + +# %% +# Metadata Routing +# ---------------- +# We are in the process of introducing a new way to route metadata such as +# ``sample_weight`` throughout the codebase, which would affect how +# meta-estimators such as :class:`pipeline.Pipeline` and +# :class:`model_selection.GridSearchCV` route metadata. While the +# infrastructure for this feature is already included in this release, the work +# is ongoing and not all meta-estimators support this new feature. You can read +# more about this feature in the :ref:`Metadata Routing User Guide +# `. Note that this feature is still under development and +# not implemented for most meta-estimators. +# +# Third party developers can already start incorporating this into their +# meta-estimators. For more details, see +# :ref:`metadata routing developer guide +# `. + +# %% +# HDBSCAN: hierarchical density-based clustering +# ---------------------------------------------- +# Originally hosted in the scikit-learn-contrib repository, :class:`cluster.HDBSCAN` +# has been adpoted into scikit-learn. It's missing a few features from the original +# implementation which will be added in future releases. +# By performing a modified version of :class:`cluster.DBSCAN` over multiple epsilon +# values simultaneously, :class:`cluster.HDBSCAN` finds clusters of varying densities +# making it more robust to parameter selection than :class:`cluster.DBSCAN`. +# More details in the :ref:`User Guide `. +import numpy as np + +from sklearn.cluster import HDBSCAN +from sklearn.datasets import load_digits +from sklearn.metrics import v_measure_score + +X, true_labels = load_digits(return_X_y=True) +print(f"number of digits: {len(np.unique(true_labels))}") + +hdbscan = HDBSCAN(min_cluster_size=15).fit(X) +non_noisy_labels = hdbscan.labels_[hdbscan.labels_ != -1] +print(f"number of clusters found: {len(np.unique(non_noisy_labels))}") + +print(v_measure_score(true_labels[hdbscan.labels_ != -1], non_noisy_labels)) + +# %% +# TargetEncoder: a new category encoding strategy +# ----------------------------------------------- +# Well suited for categorical features with high cardinality, +# :class:`preprocessing.TargetEncoder` encodes the categories based on a shrunk +# estimate of the average target values for observations belonging to that category. +# More details in the :ref:`User Guide `. +import numpy as np + +from sklearn.preprocessing import TargetEncoder + +X = np.array([["cat"] * 30 + ["dog"] * 20 + ["snake"] * 38], dtype=object).T +y = [90.3] * 30 + [20.4] * 20 + [21.2] * 38 + +enc = TargetEncoder(random_state=0) +X_trans = enc.fit_transform(X, y) + +enc.encodings_ + +# %% +# Missing values support in decision trees +# ---------------------------------------- +# The classes :class:`tree.DecisionTreeClassifier` and +# :class:`tree.DecisionTreeRegressor` now support missing values. For each potential +# threshold on the non-missing data, the splitter will evaluate the split with all the +# missing values going to the left node or the right node. +# See more details in the :ref:`User Guide ` or see +# :ref:`sphx_glr_auto_examples_ensemble_plot_hgbt_regression.py` for a usecase +# example of this feature in :class:`~ensemble.HistGradientBoostingRegressor`. +import numpy as np + +from sklearn.tree import DecisionTreeClassifier + +X = np.array([0, 1, 6, np.nan]).reshape(-1, 1) +y = [0, 0, 1, 1] + +tree = DecisionTreeClassifier(random_state=0).fit(X, y) +tree.predict(X) + +# %% +# New display :class:`~model_selection.ValidationCurveDisplay` +# ------------------------------------------------------------ +# :class:`model_selection.ValidationCurveDisplay` is now available to plot results +# from :func:`model_selection.validation_curve`. +from sklearn.datasets import make_classification +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import ValidationCurveDisplay + +X, y = make_classification(1000, 10, random_state=0) + +_ = ValidationCurveDisplay.from_estimator( + LogisticRegression(), + X, + y, + param_name="C", + param_range=np.geomspace(1e-5, 1e3, num=9), + score_type="both", + score_name="Accuracy", +) + +# %% +# Gamma loss for gradient boosting +# -------------------------------- +# The class :class:`ensemble.HistGradientBoostingRegressor` supports the +# Gamma deviance loss function via `loss="gamma"`. This loss function is useful for +# modeling strictly positive targets with a right-skewed distribution. +import numpy as np + +from sklearn.datasets import make_low_rank_matrix +from sklearn.ensemble import HistGradientBoostingRegressor +from sklearn.model_selection import cross_val_score + +n_samples, n_features = 500, 10 +rng = np.random.RandomState(0) +X = make_low_rank_matrix(n_samples, n_features, random_state=rng) +coef = rng.uniform(low=-10, high=20, size=n_features) +y = rng.gamma(shape=2, scale=np.exp(X @ coef) / 2) +gbdt = HistGradientBoostingRegressor(loss="gamma") +cross_val_score(gbdt, X, y).mean() + +# %% +# Grouping infrequent categories in :class:`~preprocessing.OrdinalEncoder` +# ------------------------------------------------------------------------ +# Similarly to :class:`preprocessing.OneHotEncoder`, the class +# :class:`preprocessing.OrdinalEncoder` now supports aggregating infrequent categories +# into a single output for each feature. The parameters to enable the gathering of +# infrequent categories are `min_frequency` and `max_categories`. +# See the :ref:`User Guide ` for more details. +import numpy as np + +from sklearn.preprocessing import OrdinalEncoder + +X = np.array( + [["dog"] * 5 + ["cat"] * 20 + ["rabbit"] * 10 + ["snake"] * 3], dtype=object +).T +enc = OrdinalEncoder(min_frequency=6).fit(X) +enc.infrequent_categories_ diff --git a/examples/release_highlights/plot_release_highlights_1_4_0.py b/examples/release_highlights/plot_release_highlights_1_4_0.py new file mode 100644 index 0000000000000..5ce256b065e48 --- /dev/null +++ b/examples/release_highlights/plot_release_highlights_1_4_0.py @@ -0,0 +1,238 @@ +# ruff: noqa: CPY001 +""" +======================================= +Release Highlights for scikit-learn 1.4 +======================================= + +.. currentmodule:: sklearn + +We are pleased to announce the release of scikit-learn 1.4! Many bug fixes +and improvements were added, as well as some new key features. We detail +below a few of the major features of this release. **For an exhaustive list of +all the changes**, please refer to the :ref:`release notes `. + +To install the latest version (with pip):: + + pip install --upgrade scikit-learn + +or with conda:: + + conda install -c conda-forge scikit-learn + +""" + +# %% +# HistGradientBoosting Natively Supports Categorical DTypes in DataFrames +# ----------------------------------------------------------------------- +# :class:`ensemble.HistGradientBoostingClassifier` and +# :class:`ensemble.HistGradientBoostingRegressor` now directly supports dataframes with +# categorical features. Here we have a dataset with a mixture of +# categorical and numerical features: +from sklearn.datasets import fetch_openml + +X_adult, y_adult = fetch_openml("adult", version=2, return_X_y=True) + +# Remove redundant and non-feature columns +X_adult = X_adult.drop(["education-num", "fnlwgt"], axis="columns") +X_adult.dtypes + +# %% +# By setting `categorical_features="from_dtype"`, the gradient boosting classifier +# treats the columns with categorical dtypes as categorical features in the +# algorithm: +from sklearn.ensemble import HistGradientBoostingClassifier +from sklearn.metrics import roc_auc_score +from sklearn.model_selection import train_test_split + +X_train, X_test, y_train, y_test = train_test_split(X_adult, y_adult, random_state=0) +hist = HistGradientBoostingClassifier(categorical_features="from_dtype") + +hist.fit(X_train, y_train) +y_decision = hist.decision_function(X_test) +print(f"ROC AUC score is {roc_auc_score(y_test, y_decision)}") + +# %% +# Polars output in `set_output` +# ----------------------------- +# scikit-learn's transformers now support polars output with the `set_output` API. +import polars as pl + +from sklearn.compose import ColumnTransformer +from sklearn.preprocessing import OneHotEncoder, StandardScaler + +df = pl.DataFrame( + {"height": [120, 140, 150, 110, 100], "pet": ["dog", "cat", "dog", "cat", "cat"]} +) +preprocessor = ColumnTransformer( + [ + ("numerical", StandardScaler(), ["height"]), + ("categorical", OneHotEncoder(sparse_output=False), ["pet"]), + ], + verbose_feature_names_out=False, +) +preprocessor.set_output(transform="polars") + +df_out = preprocessor.fit_transform(df) +df_out + +# %% +print(f"Output type: {type(df_out)}") + +# %% +# Missing value support for Random Forest +# --------------------------------------- +# The classes :class:`ensemble.RandomForestClassifier` and +# :class:`ensemble.RandomForestRegressor` now support missing values. When training +# every individual tree, the splitter evaluates each potential threshold with the +# missing values going to the left and right nodes. More details in the +# :ref:`User Guide `. +import numpy as np + +from sklearn.ensemble import RandomForestClassifier + +X = np.array([0, 1, 6, np.nan]).reshape(-1, 1) +y = [0, 0, 1, 1] + +forest = RandomForestClassifier(random_state=0).fit(X, y) +forest.predict(X) + +# %% +# Add support for monotonic constraints in tree-based models +# ---------------------------------------------------------- +# While we added support for monotonic constraints in histogram-based gradient boosting +# in scikit-learn 0.23, we now support this feature for all other tree-based models as +# trees, random forests, extra-trees, and exact gradient boosting. Here, we show this +# feature for random forest on a regression problem. +import matplotlib.pyplot as plt + +from sklearn.ensemble import RandomForestRegressor +from sklearn.inspection import PartialDependenceDisplay + +n_samples = 500 +rng = np.random.RandomState(0) +X = rng.randn(n_samples, 2) +noise = rng.normal(loc=0.0, scale=0.01, size=n_samples) +y = 5 * X[:, 0] + np.sin(10 * np.pi * X[:, 0]) - noise + +rf_no_cst = RandomForestRegressor().fit(X, y) +rf_cst = RandomForestRegressor(monotonic_cst=[1, 0]).fit(X, y) + +disp = PartialDependenceDisplay.from_estimator( + rf_no_cst, + X, + features=[0], + feature_names=["feature 0"], + line_kw={"linewidth": 4, "label": "unconstrained", "color": "tab:blue"}, +) +PartialDependenceDisplay.from_estimator( + rf_cst, + X, + features=[0], + line_kw={"linewidth": 4, "label": "constrained", "color": "tab:orange"}, + ax=disp.axes_, +) +disp.axes_[0, 0].plot( + X[:, 0], y, "o", alpha=0.5, zorder=-1, label="samples", color="tab:green" +) +disp.axes_[0, 0].set_ylim(-3, 3) +disp.axes_[0, 0].set_xlim(-1, 1) +disp.axes_[0, 0].legend() +plt.show() + +# %% +# Enriched estimator displays +# --------------------------- +# Estimators displays have been enriched: if we look at `forest`, defined above: +forest + +# %% +# One can access the documentation of the estimator by clicking on the icon "?" on +# the top right corner of the diagram. +# +# In addition, the display changes color, from orange to blue, when the estimator is +# fitted. You can also get this information by hovering on the icon "i". +from sklearn.base import clone + +clone(forest) # the clone is not fitted + +# %% +# Metadata Routing Support +# ------------------------ +# Many meta-estimators and cross-validation routines now support metadata +# routing, which are listed in the :ref:`user guide +# `. For instance, this is how you can do a nested +# cross-validation with sample weights and :class:`~model_selection.GroupKFold`: +import sklearn +from sklearn.datasets import make_regression +from sklearn.linear_model import Lasso +from sklearn.metrics import get_scorer +from sklearn.model_selection import GridSearchCV, GroupKFold, cross_validate + +# For now by default metadata routing is disabled, and need to be explicitly +# enabled. +sklearn.set_config(enable_metadata_routing=True) + +n_samples = 100 +X, y = make_regression(n_samples=n_samples, n_features=5, noise=0.5) +rng = np.random.RandomState(7) +groups = rng.randint(0, 10, size=n_samples) +sample_weights = rng.rand(n_samples) +estimator = Lasso().set_fit_request(sample_weight=True) +hyperparameter_grid = {"alpha": [0.1, 0.5, 1.0, 2.0]} +scoring_inner_cv = get_scorer("neg_mean_squared_error").set_score_request( + sample_weight=True +) +inner_cv = GroupKFold(n_splits=5) + +grid_search = GridSearchCV( + estimator=estimator, + param_grid=hyperparameter_grid, + cv=inner_cv, + scoring=scoring_inner_cv, +) + +outer_cv = GroupKFold(n_splits=5) +scorers = { + "mse": get_scorer("neg_mean_squared_error").set_score_request(sample_weight=True) +} +results = cross_validate( + grid_search, + X, + y, + cv=outer_cv, + scoring=scorers, + return_estimator=True, + params={"sample_weight": sample_weights, "groups": groups}, +) +print("cv error on test sets:", results["test_mse"]) + +# Setting the flag to the default `False` to avoid interference with other +# scripts. +sklearn.set_config(enable_metadata_routing=False) + +# %% +# Improved memory and runtime efficiency for PCA on sparse data +# ------------------------------------------------------------- +# PCA is now able to handle sparse matrices natively for the `arpack` +# solver by levaraging `scipy.sparse.linalg.LinearOperator` to avoid +# materializing large sparse matrices when performing the +# eigenvalue decomposition of the data set covariance matrix. +# +from time import time + +import scipy.sparse as sp + +from sklearn.decomposition import PCA + +X_sparse = sp.random(m=1000, n=1000, random_state=0) +X_dense = X_sparse.toarray() + +t0 = time() +PCA(n_components=10, svd_solver="arpack").fit(X_sparse) +time_sparse = time() - t0 + +t0 = time() +PCA(n_components=10, svd_solver="arpack").fit(X_dense) +time_dense = time() - t0 + +print(f"Speedup: {time_dense / time_sparse:.1f}x") diff --git a/examples/release_highlights/plot_release_highlights_1_5_0.py b/examples/release_highlights/plot_release_highlights_1_5_0.py new file mode 100644 index 0000000000000..ef389a5db290b --- /dev/null +++ b/examples/release_highlights/plot_release_highlights_1_5_0.py @@ -0,0 +1,230 @@ +# ruff: noqa: CPY001 +""" +======================================= +Release Highlights for scikit-learn 1.5 +======================================= + +.. currentmodule:: sklearn + +We are pleased to announce the release of scikit-learn 1.5! Many bug fixes +and improvements were added, as well as some key new features. Below we +detail the highlights of this release. **For an exhaustive list of +all the changes**, please refer to the :ref:`release notes `. + +To install the latest version (with pip):: + + pip install --upgrade scikit-learn + +or with conda:: + + conda install -c conda-forge scikit-learn + +""" + +# %% +# FixedThresholdClassifier: Setting the decision threshold of a binary classifier +# ------------------------------------------------------------------------------- +# All binary classifiers of scikit-learn use a fixed decision threshold of 0.5 +# to convert probability estimates (i.e. output of `predict_proba`) into class +# predictions. However, 0.5 is almost never the desired threshold for a given +# problem. :class:`~model_selection.FixedThresholdClassifier` allows wrapping any +# binary classifier and setting a custom decision threshold. +from sklearn.datasets import make_classification +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import ConfusionMatrixDisplay +from sklearn.model_selection import train_test_split + +X, y = make_classification(n_samples=10_000, weights=[0.9, 0.1], random_state=0) +X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0) + +classifier_05 = LogisticRegression(C=1e6, random_state=0).fit(X_train, y_train) +_ = ConfusionMatrixDisplay.from_estimator(classifier_05, X_test, y_test) + +# %% +# Lowering the threshold, i.e. allowing more samples to be classified as the positive +# class, increases the number of true positives at the cost of more false positives +# (as is well known from the concavity of the ROC curve). +from sklearn.model_selection import FixedThresholdClassifier + +classifier_01 = FixedThresholdClassifier(classifier_05, threshold=0.1) +classifier_01.fit(X_train, y_train) +_ = ConfusionMatrixDisplay.from_estimator(classifier_01, X_test, y_test) + +# %% +# TunedThresholdClassifierCV: Tuning the decision threshold of a binary classifier +# -------------------------------------------------------------------------------- +# The decision threshold of a binary classifier can be tuned to optimize a +# given metric, using :class:`~model_selection.TunedThresholdClassifierCV`. +# +# It is particularly useful to find the best decision threshold when the model +# is meant to be deployed in a specific application context where we can assign +# different gains or costs for true positives, true negatives, false positives, +# and false negatives. +# +# Let's illustrate this by considering an arbitrary case where: +# +# - each true positive gains 1 unit of profit, e.g. euro, year of life in good +# health, etc.; +# - true negatives gain or cost nothing; +# - each false negative costs 2; +# - each false positive costs 0.1. +# +# Our metric quantifies the average profit per sample, which is defined by the +# following Python function: +from sklearn.metrics import confusion_matrix + + +def custom_score(y_observed, y_pred): + tn, fp, fn, tp = confusion_matrix(y_observed, y_pred, normalize="all").ravel() + return tp - 2 * fn - 0.1 * fp + + +print("Untuned decision threshold: 0.5") +print(f"Custom score: {custom_score(y_test, classifier_05.predict(X_test)):.2f}") + +# %% +# It is interesting to observe that the average gain per prediction is negative +# which means that this decision system is making a loss on average. +# +# Tuning the threshold to optimize this custom metric gives a smaller threshold +# that allows more samples to be classified as the positive class. As a result, +# the average gain per prediction improves. +from sklearn.metrics import make_scorer +from sklearn.model_selection import TunedThresholdClassifierCV + +custom_scorer = make_scorer( + custom_score, response_method="predict", greater_is_better=True +) +tuned_classifier = TunedThresholdClassifierCV( + classifier_05, cv=5, scoring=custom_scorer +).fit(X, y) + +print(f"Tuned decision threshold: {tuned_classifier.best_threshold_:.3f}") +print(f"Custom score: {custom_score(y_test, tuned_classifier.predict(X_test)):.2f}") + +# %% +# We observe that tuning the decision threshold can turn a machine +# learning-based system that makes a loss on average into a beneficial one. +# +# In practice, defining a meaningful application-specific metric might involve +# making those costs for bad predictions and gains for good predictions depend on +# auxiliary metadata specific to each individual data point such as the amount +# of a transaction in a fraud detection system. +# +# To achieve this, :class:`~model_selection.TunedThresholdClassifierCV` +# leverages metadata routing support (:ref:`Metadata Routing User +# Guide`) allowing to optimize complex business metrics as +# detailed in :ref:`Post-tuning the decision threshold for cost-sensitive +# learning +# `. + +# %% +# Performance improvements in PCA +# ------------------------------- +# :class:`~decomposition.PCA` has a new solver, `"covariance_eigh"`, which is +# up to an order of magnitude faster and more memory efficient than the other +# solvers for datasets with many data points and few features. +from sklearn.datasets import make_low_rank_matrix +from sklearn.decomposition import PCA + +X = make_low_rank_matrix( + n_samples=10_000, n_features=100, tail_strength=0.1, random_state=0 +) + +pca = PCA(n_components=10, svd_solver="covariance_eigh").fit(X) +print(f"Explained variance: {pca.explained_variance_ratio_.sum():.2f}") + + +# %% +# The new solver also accepts sparse input data: +from scipy.sparse import random + +X = random(10_000, 100, format="csr", random_state=0) + +pca = PCA(n_components=10, svd_solver="covariance_eigh").fit(X) +print(f"Explained variance: {pca.explained_variance_ratio_.sum():.2f}") + +# %% +# The `"full"` solver has also been improved to use less memory and allows +# faster transformation. The default `svd_solver="auto"` option takes +# advantage of the new solver and is now able to select an appropriate solver +# for sparse datasets. +# +# Similarly to most other PCA solvers, the new `"covariance_eigh"` solver can leverage +# GPU computation if the input data is passed as a PyTorch or CuPy array by +# enabling the experimental support for :ref:`Array API `. + +# %% +# ColumnTransformer is subscriptable +# ---------------------------------- +# The transformers of a :class:`~compose.ColumnTransformer` can now be directly +# accessed using indexing by name. +import numpy as np + +from sklearn.compose import ColumnTransformer +from sklearn.preprocessing import OneHotEncoder, StandardScaler + +X = np.array([[0, 1, 2], [3, 4, 5]]) +column_transformer = ColumnTransformer( + [("std_scaler", StandardScaler(), [0]), ("one_hot", OneHotEncoder(), [1, 2])] +) + +column_transformer.fit(X) + +print(column_transformer["std_scaler"]) +print(column_transformer["one_hot"]) + +# %% +# Custom imputation strategies for the SimpleImputer +# -------------------------------------------------- +# :class:`~impute.SimpleImputer` now supports custom strategies for imputation, +# using a callable that computes a scalar value from the non missing values of +# a column vector. +from sklearn.impute import SimpleImputer + +X = np.array( + [ + [-1.1, 1.1, 1.1], + [3.9, -1.2, np.nan], + [np.nan, 1.3, np.nan], + [-0.1, -1.4, -1.4], + [-4.9, 1.5, -1.5], + [np.nan, 1.6, 1.6], + ] +) + + +def smallest_abs(arr): + """Return the smallest absolute value of a 1D array.""" + return np.min(np.abs(arr)) + + +imputer = SimpleImputer(strategy=smallest_abs) + +imputer.fit_transform(X) + +# %% +# Pairwise distances with non-numeric arrays +# ------------------------------------------ +# :func:`~metrics.pairwise_distances` can now compute distances between +# non-numeric arrays using a callable metric. +from sklearn.metrics import pairwise_distances + +X = ["cat", "dog"] +Y = ["cat", "fox"] + + +def levenshtein_distance(x, y): + """Return the Levenshtein distance between two strings.""" + if x == "" or y == "": + return max(len(x), len(y)) + if x[0] == y[0]: + return levenshtein_distance(x[1:], y[1:]) + return 1 + min( + levenshtein_distance(x[1:], y), + levenshtein_distance(x, y[1:]), + levenshtein_distance(x[1:], y[1:]), + ) + + +pairwise_distances(X, Y, metric=levenshtein_distance) diff --git a/examples/release_highlights/plot_release_highlights_1_6_0.py b/examples/release_highlights/plot_release_highlights_1_6_0.py new file mode 100644 index 0000000000000..503af8c076fbb --- /dev/null +++ b/examples/release_highlights/plot_release_highlights_1_6_0.py @@ -0,0 +1,214 @@ +# ruff: noqa: CPY001, E501 +""" +======================================= +Release Highlights for scikit-learn 1.6 +======================================= + +.. currentmodule:: sklearn + +We are pleased to announce the release of scikit-learn 1.6! Many bug fixes +and improvements were added, as well as some key new features. Below we +detail the highlights of this release. **For an exhaustive list of +all the changes**, please refer to the :ref:`release notes `. + +To install the latest version (with pip):: + + pip install --upgrade scikit-learn + +or with conda:: + + conda install -c conda-forge scikit-learn + +""" + +# %% +# FrozenEstimator: Freezing an estimator +# -------------------------------------- +# +# This meta-estimator allows you to take an estimator and freeze its fit method, meaning +# that calling `fit` does not perform any operations; also, `fit_predict` and +# `fit_transform` call `predict` and `transform` respectively without calling `fit`. The +# original estimator's other methods and properties are left unchanged. An interesting +# use case for this is to use a pre-fitted model as a transformer step in a pipeline +# or to pass a pre-fitted model to some of the meta-estimators. Here's a short example: + +import time + +from sklearn.datasets import make_classification +from sklearn.frozen import FrozenEstimator +from sklearn.linear_model import SGDClassifier +from sklearn.model_selection import FixedThresholdClassifier + +X, y = make_classification(n_samples=1000, random_state=0) + +start = time.time() +classifier = SGDClassifier().fit(X, y) +print(f"Fitting the classifier took {(time.time() - start) * 1_000:.2f} milliseconds") + +start = time.time() +threshold_classifier = FixedThresholdClassifier( + estimator=FrozenEstimator(classifier), threshold=0.9 +).fit(X, y) +print( + f"Fitting the threshold classifier took {(time.time() - start) * 1_000:.2f} " + "milliseconds" +) + +# %% +# Fitting the threshold classifier skipped fitting the inner `SGDClassifier`. For more +# details refer to the example :ref:`sphx_glr_auto_examples_frozen_plot_frozen_examples.py`. + +# %% +# Transforming data other than X in a Pipeline +# -------------------------------------------- +# +# The :class:`~pipeline.Pipeline` now supports transforming passed data other than `X` +# if necessary. This can be done by setting the new `transform_input` parameter. This +# is particularly useful when passing a validation set through the pipeline. +# +# As an example, imagine `EstimatorWithValidationSet` is an estimator which accepts +# a validation set. We can now have a pipeline which will transform the validation set +# and pass it to the estimator:: +# +# with sklearn.config_context(enable_metadata_routing=True): +# est_gs = GridSearchCV( +# Pipeline( +# ( +# StandardScaler(), +# EstimatorWithValidationSet(...).set_fit_request(X_val=True, y_val=True), +# ), +# # telling pipeline to transform these inputs up to the step which is +# # requesting them. +# transform_input=["X_val"], +# ), +# param_grid={"estimatorwithvalidationset__param_to_optimize": list(range(5))}, +# cv=5, +# ).fit(X, y, X_val=X_val, y_val=y_val) +# +# In the above code, the key parts are the call to `set_fit_request` to specify that +# `X_val` and `y_val` are required by the `EstimatorWithValidationSet.fit` method, and +# the `transform_input` parameter to tell the pipeline to transform `X_val` before +# passing it to `EstimatorWithValidationSet.fit`. +# +# Note that at this time scikit-learn estimators have not yet been extended to accept +# user specified validation sets. This feature is released early to collect feedback +# from third-party libraries who might benefit from it. + +# %% +# Multiclass support for `LogisticRegression(solver="newton-cholesky")` +# --------------------------------------------------------------------- +# +# The `"newton-cholesky"` solver (originally introduced in scikit-learn version +# 1.2) was previously limited to binary +# :class:`~linear_model.LogisticRegression` and some other generalized linear +# regression estimators (namely :class:`~linear_model.PoissonRegressor`, +# :class:`~linear_model.GammaRegressor` and +# :class:`~linear_model.TweedieRegressor`). +# +# This new release includes support for multiclass (multinomial) +# :class:`~linear_model.LogisticRegression`. +# +# This solver is particularly useful when the number of features is small to +# medium. It has been empirically shown to converge more reliably and faster +# than other solvers on some medium sized datasets with one-hot encoded +# categorical features as can be seen in the `benchmark results of the +# pull-request +# `_. + +# %% +# Missing value support for Extra Trees +# ------------------------------------- +# +# The classes :class:`ensemble.ExtraTreesClassifier` and +# :class:`ensemble.ExtraTreesRegressor` now support missing values. More details in the +# :ref:`User Guide `. +import numpy as np + +from sklearn.ensemble import ExtraTreesClassifier + +X = np.array([0, 1, 6, np.nan]).reshape(-1, 1) +y = [0, 0, 1, 1] + +forest = ExtraTreesClassifier(random_state=0).fit(X, y) +forest.predict(X) + +# %% +# Download any dataset from the web +# --------------------------------- +# +# The function :func:`datasets.fetch_file` allows downloading a file from any given URL. +# This convenience function provides built-in local disk caching, sha256 digest +# integrity check and an automated retry mechanism on network error. +# +# The goal is to provide the same convenience and reliability as dataset fetchers while +# giving the flexibility to work with data from arbitrary online sources and file +# formats. +# +# The downloaded file can then be loaded with generic or domain specific functions such +# as `pandas.read_csv`, `pandas.read_parquet`, etc. + +# %% +# Array API support +# ----------------- +# +# Many more estimators and functions have been updated to support array API compatible +# inputs since version 1.5, in particular the meta-estimators for hyperparameter tuning +# from the :mod:`sklearn.model_selection` module and the metrics from the +# :mod:`sklearn.metrics` module. +# +# Please refer to the :ref:`array API support` page for instructions to use +# scikit-learn with array API compatible libraries such as PyTorch or CuPy. + +# %% +# Almost complete Metadata Routing support +# ---------------------------------------- +# +# Support for routing metadata has been added to all remaining estimators and +# functions except AdaBoost. See :ref:`Metadata Routing User Guide ` +# for more details. + +# %% +# Free-threaded CPython 3.13 support +# ---------------------------------- +# +# scikit-learn has preliminary support for free-threaded CPython, in particular +# free-threaded wheels are available for all of our supported platforms. +# +# Free-threaded (also known as nogil) CPython 3.13 is an experimental version of +# CPython 3.13 which aims at enabling efficient multi-threaded use cases by +# removing the Global Interpreter Lock (GIL). +# +# For more details about free-threaded CPython see `py-free-threading doc `_, +# in particular `how to install a free-threaded CPython `_ +# and `Ecosystem compatibility tracking `_. +# +# Feel free to try free-threaded CPython on your use case and report any issues! + +# %% +# Improvements to the developer API for third party libraries +# ----------------------------------------------------------- +# +# We have been working on improving the developer API for third party libraries. +# This is still a work in progress, but a fair amount of work has been done in this +# release. This release includes: +# +# - :func:`sklearn.utils.validation.validate_data` is introduced and replaces the +# previously private `BaseEstimator._validate_data` method. This function extends +# :func:`~sklearn.utils.validation.check_array` and adds support for remembering +# input feature counts and names. +# - Estimator tags are now revamped and a part of the public API via +# :class:`sklearn.utils.Tags`. Estimators should now override the +# :meth:`BaseEstimator.__sklearn_tags__` method instead of implementing a `_more_tags` +# method. If you'd like to support multiple scikit-learn versions, you can implement +# both methods in your class. +# - As a consequence of developing a public tag API, we've removed the `_xfail_checks` +# tag and tests which are expected to fail are directly passed to +# :func:`~sklearn.utils.estimator_checks.check_estimator` and +# :func:`~sklearn.utils.estimator_checks.parametrize_with_checks`. See their +# corresponding API docs for more details. +# - Many tests in the common test suite are updated and raise more helpful error +# messages. We've also added some new tests, which should help you more easily fix +# potential issues with your estimators. +# +# An updated version of our :ref:`develop` is also available, which we recommend you +# check out. diff --git a/examples/release_highlights/plot_release_highlights_1_7_0.py b/examples/release_highlights/plot_release_highlights_1_7_0.py new file mode 100644 index 0000000000000..06c2f10e70b28 --- /dev/null +++ b/examples/release_highlights/plot_release_highlights_1_7_0.py @@ -0,0 +1,115 @@ +# ruff: noqa: CPY001 +""" +======================================= +Release Highlights for scikit-learn 1.7 +======================================= + +.. currentmodule:: sklearn + +We are pleased to announce the release of scikit-learn 1.7! Many bug fixes +and improvements were added, as well as some key new features. Below we +detail the highlights of this release. **For an exhaustive list of +all the changes**, please refer to the :ref:`release notes `. + +To install the latest version (with pip):: + + pip install --upgrade scikit-learn + +or with conda:: + + conda install -c conda-forge scikit-learn + +""" + +# %% +# Improved estimator's HTML representation +# ---------------------------------------- +# The HTML representation of estimators now includes a section containing the list of +# parameters and their values. Non-default parameters are highlighted in orange. A copy +# button is also available to copy the "fully-qualified" parameter name without the +# need to call the `get_params` method. It is particularly useful when defining a +# parameter grid for a grid-search or a randomized-search with a complex pipeline. +# +# See the example below and click on the different estimator's blocks to see the +# improved HTML representation. + +from sklearn.linear_model import LogisticRegression +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import StandardScaler + +model = make_pipeline(StandardScaler(with_std=False), LogisticRegression(C=2.0)) +model + +# %% +# Custom validation set for histogram-based Gradient Boosting estimators +# ---------------------------------------------------------------------- +# The :class:`ensemble.HistGradientBoostingClassifier` and +# :class:`ensemble.HistGradientBoostingRegressor` now support directly passing a custom +# validation set for early stopping to the `fit` method, using the `X_val`, `y_val`, and +# `sample_weight_val` parameters. +# In a :class:`pipeline.Pipeline`, the validation set `X_val` can be transformed along +# with `X` using the `transform_input` parameter. + +import sklearn +from sklearn.datasets import make_classification +from sklearn.ensemble import HistGradientBoostingClassifier +from sklearn.model_selection import train_test_split +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import StandardScaler + +sklearn.set_config(enable_metadata_routing=True) + +X, y = make_classification(random_state=0) +X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=0) + +clf = HistGradientBoostingClassifier() +clf.set_fit_request(X_val=True, y_val=True) + +model = Pipeline([("sc", StandardScaler()), ("clf", clf)], transform_input=["X_val"]) +model.fit(X, y, X_val=X_val, y_val=y_val) + +# %% +# Plotting ROC curves from cross-validation results +# ------------------------------------------------- +# The class :class:`metrics.RocCurveDisplay` has a new class method `from_cv_results` +# that allows to easily plot multiple ROC curves from the results of +# :func:`model_selection.cross_validate`. + +from sklearn.datasets import make_classification +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import RocCurveDisplay +from sklearn.model_selection import cross_validate + +X, y = make_classification(n_samples=150, random_state=0) +clf = LogisticRegression(random_state=0) +cv_results = cross_validate(clf, X, y, cv=5, return_estimator=True, return_indices=True) +_ = RocCurveDisplay.from_cv_results(cv_results, X, y) + +# %% +# Array API support +# ----------------- +# Several functions have been updated to support array API compatible inputs since +# version 1.6, especially metrics from the :mod:`sklearn.metrics` module. +# +# In addition, it is no longer required to install the `array-api-compat` package to use +# the experimental array API support in scikit-learn. +# +# Please refer to the :ref:`array API support` page for instructions to use +# scikit-learn with array API compatible libraries such as PyTorch or CuPy. + +# %% +# Improved API consistency of Multi-layer Perceptron +# -------------------------------------------------- +# The :class:`neural_network.MLPRegressor` has a new parameter `loss` and now supports +# the "poisson" loss in addition to the default "squared_error" loss. +# Moreover, the :class:`neural_network.MLPClassifier` and +# :class:`neural_network.MLPRegressor` estimators now support sample weights. +# These improvements have been made to improve the consistency of these estimators +# with regard to the other estimators in scikit-learn. + +# %% +# Migration toward sparse arrays +# ------------------------------ +# In order to prepare `SciPy migration from sparse matrices to sparse arrays `_, +# all scikit-learn estimators that accept sparse matrices as input now also accept +# sparse arrays. diff --git a/examples/semi_supervised/plot_label_propagation_digits.py b/examples/semi_supervised/plot_label_propagation_digits.py index 715546f78ab25..b8b544005c1b2 100644 --- a/examples/semi_supervised/plot_label_propagation_digits.py +++ b/examples/semi_supervised/plot_label_propagation_digits.py @@ -13,27 +13,31 @@ class will be very good. At the end, the top 10 most uncertain predictions will be shown. + """ -print(__doc__) -# Authors: Clay Woolam -# License: BSD +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +# %% +# Data generation +# --------------- +# +# We use the digits dataset. We only use a subset of randomly selected samples. import numpy as np -import matplotlib.pyplot as plt - -from scipy import stats from sklearn import datasets -from sklearn.semi_supervised import LabelSpreading - -from sklearn.metrics import confusion_matrix, classification_report digits = datasets.load_digits() rng = np.random.RandomState(2) indices = np.arange(len(digits.data)) rng.shuffle(indices) +# %% +# +# We selected 340 samples of which only 40 will be associated with a known label. +# Therefore, we store the indices of the 300 other samples for which we are not +# supposed to know their labels. X = digits.data[indices[:340]] y = digits.target[indices[:340]] images = digits.images[indices[:340]] @@ -45,38 +49,59 @@ class will be very good. unlabeled_set = indices[n_labeled_points:] -# ############################################################################# +# %% # Shuffle everything around y_train = np.copy(y) y_train[unlabeled_set] = -1 -# ############################################################################# -# Learn with LabelSpreading -lp_model = LabelSpreading(gamma=.25, max_iter=20) +# %% +# Semi-supervised learning +# ------------------------ +# +# We fit a :class:`~sklearn.semi_supervised.LabelSpreading` and use it to predict +# the unknown labels. +from sklearn.metrics import classification_report +from sklearn.semi_supervised import LabelSpreading + +lp_model = LabelSpreading(gamma=0.25, max_iter=20) lp_model.fit(X, y_train) predicted_labels = lp_model.transduction_[unlabeled_set] true_labels = y[unlabeled_set] -cm = confusion_matrix(true_labels, predicted_labels, labels=lp_model.classes_) - -print("Label Spreading model: %d labeled & %d unlabeled points (%d total)" % - (n_labeled_points, n_total_samples - n_labeled_points, n_total_samples)) +print( + "Label Spreading model: %d labeled & %d unlabeled points (%d total)" + % (n_labeled_points, n_total_samples - n_labeled_points, n_total_samples) +) +# %% +# Classification report print(classification_report(true_labels, predicted_labels)) -print("Confusion matrix") -print(cm) +# %% +# Confusion matrix +from sklearn.metrics import ConfusionMatrixDisplay + +ConfusionMatrixDisplay.from_predictions( + true_labels, predicted_labels, labels=lp_model.classes_ +) + +# %% +# Plot the most uncertain predictions +# ----------------------------------- +# +# Here, we will pick and show the 10 most uncertain predictions. +from scipy import stats -# ############################################################################# -# Calculate uncertainty values for each transduced distribution pred_entropies = stats.distributions.entropy(lp_model.label_distributions_.T) -# ############################################################################# +# %% # Pick the top 10 most uncertain labels uncertainty_index = np.argsort(pred_entropies)[-10:] -# ############################################################################# +# %% # Plot +import matplotlib.pyplot as plt + f = plt.figure(figsize=(7, 5)) for index, image_index in enumerate(uncertainty_index): image = images[image_index] @@ -85,8 +110,9 @@ class will be very good. sub.imshow(image, cmap=plt.cm.gray_r) plt.xticks([]) plt.yticks([]) - sub.set_title('predict: %i\ntrue: %i' % ( - lp_model.transduction_[image_index], y[image_index])) + sub.set_title( + "predict: %i\ntrue: %i" % (lp_model.transduction_[image_index], y[image_index]) + ) -f.suptitle('Learning with small amount of labeled data') +f.suptitle("Learning with small amount of labeled data") plt.show() diff --git a/examples/semi_supervised/plot_label_propagation_digits_active_learning.py b/examples/semi_supervised/plot_label_propagation_digits_active_learning.py index a05fa07d755d7..eda6804fe3863 100644 --- a/examples/semi_supervised/plot_label_propagation_digits_active_learning.py +++ b/examples/semi_supervised/plot_label_propagation_digits_active_learning.py @@ -1,7 +1,7 @@ """ -======================================== -Label Propagation digits active learning -======================================== +========================================= +Label Propagation digits: Active learning +========================================= Demonstrates an active learning technique to learn handwritten digits using label propagation. @@ -17,19 +17,19 @@ A plot will appear showing the top 5 most uncertain digits for each iteration of training. These may or may not contain mistakes, but we will train the next model with their true labels. + """ -print(__doc__) -# Authors: Clay Woolam -# License: BSD +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt +import numpy as np from scipy import stats from sklearn import datasets -from sklearn.semi_supervised import LabelSpreading from sklearn.metrics import classification_report, confusion_matrix +from sklearn.semi_supervised import LabelSpreading digits = datasets.load_digits() rng = np.random.RandomState(0) @@ -60,13 +60,13 @@ predicted_labels = lp_model.transduction_[unlabeled_indices] true_labels = y[unlabeled_indices] - cm = confusion_matrix(true_labels, predicted_labels, - labels=lp_model.classes_) + cm = confusion_matrix(true_labels, predicted_labels, labels=lp_model.classes_) print("Iteration %i %s" % (i, 70 * "_")) - print("Label Spreading model: %d labeled & %d unlabeled (%d total)" - % (n_labeled_points, n_total_samples - n_labeled_points, - n_total_samples)) + print( + "Label Spreading model: %d labeled & %d unlabeled (%d total)" + % (n_labeled_points, n_total_samples - n_labeled_points, n_total_samples) + ) print(classification_report(true_labels, predicted_labels)) @@ -74,42 +74,52 @@ print(cm) # compute the entropies of transduced label distributions - pred_entropies = stats.distributions.entropy( - lp_model.label_distributions_.T) + pred_entropies = stats.distributions.entropy(lp_model.label_distributions_.T) # select up to 5 digit examples that the classifier is most uncertain about uncertainty_index = np.argsort(pred_entropies)[::-1] uncertainty_index = uncertainty_index[ - np.in1d(uncertainty_index, unlabeled_indices)][:5] + np.isin(uncertainty_index, unlabeled_indices) + ][:5] # keep track of indices that we get labels for delete_indices = np.array([], dtype=int) # for more than 5 iterations, visualize the gain only on the first 5 if i < 5: - f.text(.05, (1 - (i + 1) * .183), - "model %d\n\nfit with\n%d labels" % - ((i + 1), i * 5 + 10), size=10) + f.text( + 0.05, + (1 - (i + 1) * 0.183), + "model %d\n\nfit with\n%d labels" % ((i + 1), i * 5 + 10), + size=10, + ) for index, image_index in enumerate(uncertainty_index): image = images[image_index] # for more than 5 iterations, visualize the gain only on the first 5 if i < 5: sub = f.add_subplot(5, 5, index + 1 + (5 * i)) - sub.imshow(image, cmap=plt.cm.gray_r, interpolation='none') - sub.set_title("predict: %i\ntrue: %i" % ( - lp_model.transduction_[image_index], y[image_index]), size=10) - sub.axis('off') + sub.imshow(image, cmap=plt.cm.gray_r, interpolation="none") + sub.set_title( + "predict: %i\ntrue: %i" + % (lp_model.transduction_[image_index], y[image_index]), + size=10, + ) + sub.axis("off") # labeling 5 points, remote from labeled set - delete_index, = np.where(unlabeled_indices == image_index) + (delete_index,) = (unlabeled_indices == image_index).nonzero() delete_indices = np.concatenate((delete_indices, delete_index)) unlabeled_indices = np.delete(unlabeled_indices, delete_indices) n_labeled_points += len(uncertainty_index) -f.suptitle("Active learning with Label Propagation.\nRows show 5 most " - "uncertain labels to learn with the next model.", y=1.15) -plt.subplots_adjust(left=0.2, bottom=0.03, right=0.9, top=0.9, wspace=0.2, - hspace=0.85) +f.suptitle( + ( + "Active learning with Label Propagation.\nRows show 5 most " + "uncertain labels to learn with the next model." + ), + y=1.15, +) +plt.subplots_adjust(left=0.2, bottom=0.03, right=0.9, top=0.9, wspace=0.2, hspace=0.85) plt.show() diff --git a/examples/semi_supervised/plot_label_propagation_structure.py b/examples/semi_supervised/plot_label_propagation_structure.py index f0145bcd53ccb..323cfb2a110cf 100644 --- a/examples/semi_supervised/plot_label_propagation_structure.py +++ b/examples/semi_supervised/plot_label_propagation_structure.py @@ -1,62 +1,105 @@ """ -============================================== -Label Propagation learning a complex structure -============================================== +======================================================= +Label Propagation circles: Learning a complex structure +======================================================= Example of LabelPropagation learning a complex internal structure to demonstrate "manifold learning". The outer circle should be labeled "red" and the inner circle "blue". Because both label groups lie inside their own distinct shape, we can see that the labels propagate correctly around the circle. + """ -print(__doc__) -# Authors: Clay Woolam -# Andreas Mueller -# License: BSD +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# We generate a dataset with two concentric circles. In addition, a label +# is associated with each sample of the dataset that is: 0 (belonging to +# the outer circle), 1 (belonging to the inner circle), and -1 (unknown). +# Here, all labels but two are tagged as unknown. import numpy as np -import matplotlib.pyplot as plt -from sklearn.semi_supervised import LabelSpreading + from sklearn.datasets import make_circles -# generate ring with inner box n_samples = 200 X, y = make_circles(n_samples=n_samples, shuffle=False) outer, inner = 0, 1 -labels = np.full(n_samples, -1.) +labels = np.full(n_samples, -1.0) labels[0] = outer labels[-1] = inner -# ############################################################################# -# Learn with LabelSpreading -label_spread = LabelSpreading(kernel='knn', alpha=0.8) +# %% +# Plot raw data +import matplotlib.pyplot as plt + +plt.figure(figsize=(4, 4)) +plt.scatter( + X[labels == outer, 0], + X[labels == outer, 1], + color="navy", + marker="s", + lw=0, + label="outer labeled", + s=10, +) +plt.scatter( + X[labels == inner, 0], + X[labels == inner, 1], + color="c", + marker="s", + lw=0, + label="inner labeled", + s=10, +) +plt.scatter( + X[labels == -1, 0], + X[labels == -1, 1], + color="darkorange", + marker=".", + label="unlabeled", +) +plt.legend(scatterpoints=1, shadow=False, loc="center") +_ = plt.title("Raw data (2 classes=outer and inner)") + +# %% +# +# The aim of :class:`~sklearn.semi_supervised.LabelSpreading` is to associate +# a label to sample where the label is initially unknown. +from sklearn.semi_supervised import LabelSpreading + +label_spread = LabelSpreading(kernel="knn", alpha=0.8) label_spread.fit(X, labels) -# ############################################################################# -# Plot output labels +# %% +# Now, we can check which labels have been associated with each sample +# when the label was unknown. output_labels = label_spread.transduction_ -plt.figure(figsize=(8.5, 4)) -plt.subplot(1, 2, 1) -plt.scatter(X[labels == outer, 0], X[labels == outer, 1], color='navy', - marker='s', lw=0, label="outer labeled", s=10) -plt.scatter(X[labels == inner, 0], X[labels == inner, 1], color='c', - marker='s', lw=0, label='inner labeled', s=10) -plt.scatter(X[labels == -1, 0], X[labels == -1, 1], color='darkorange', - marker='.', label='unlabeled') -plt.legend(scatterpoints=1, shadow=False, loc='upper right') -plt.title("Raw data (2 classes=outer and inner)") - -plt.subplot(1, 2, 2) output_label_array = np.asarray(output_labels) -outer_numbers = np.where(output_label_array == outer)[0] -inner_numbers = np.where(output_label_array == inner)[0] -plt.scatter(X[outer_numbers, 0], X[outer_numbers, 1], color='navy', - marker='s', lw=0, s=10, label="outer learned") -plt.scatter(X[inner_numbers, 0], X[inner_numbers, 1], color='c', - marker='s', lw=0, s=10, label="inner learned") -plt.legend(scatterpoints=1, shadow=False, loc='upper right') -plt.title("Labels learned with Label Spreading (KNN)") +outer_numbers = (output_label_array == outer).nonzero()[0] +inner_numbers = (output_label_array == inner).nonzero()[0] -plt.subplots_adjust(left=0.07, bottom=0.07, right=0.93, top=0.92) +plt.figure(figsize=(4, 4)) +plt.scatter( + X[outer_numbers, 0], + X[outer_numbers, 1], + color="navy", + marker="s", + lw=0, + s=10, + label="outer learned", +) +plt.scatter( + X[inner_numbers, 0], + X[inner_numbers, 1], + color="c", + marker="s", + lw=0, + s=10, + label="inner learned", +) +plt.legend(scatterpoints=1, shadow=False, loc="center") +plt.title("Labels learned with Label Spreading (KNN)") plt.show() diff --git a/examples/semi_supervised/plot_self_training_varying_threshold.py b/examples/semi_supervised/plot_self_training_varying_threshold.py index c2c89a36b5e8c..bbdaeb634f570 100644 --- a/examples/semi_supervised/plot_self_training_varying_threshold.py +++ b/examples/semi_supervised/plot_self_training_varying_threshold.py @@ -26,19 +26,20 @@ The optimal accuracy lies in between both of these extremes at a threshold of around 0.7. + """ -print(__doc__) -# Authors: Oliver Rausch -# License: BSD +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt +import numpy as np + from sklearn import datasets -from sklearn.svm import SVC +from sklearn.metrics import accuracy_score from sklearn.model_selection import StratifiedKFold from sklearn.semi_supervised import SelfTrainingClassifier -from sklearn.metrics import accuracy_score +from sklearn.svm import SVC from sklearn.utils import shuffle n_splits = 3 @@ -57,9 +58,8 @@ amount_labeled = np.empty((x_values.shape[0], n_splits)) amount_iterations = np.empty((x_values.shape[0], n_splits)) -for (i, threshold) in enumerate(x_values): - self_training_clf = SelfTrainingClassifier(base_classifier, - threshold=threshold) +for i, threshold in enumerate(x_values): + self_training_clf = SelfTrainingClassifier(base_classifier, threshold=threshold) # We need manual cross validation so that we don't treat -1 as a separate # class when computing accuracy @@ -74,8 +74,10 @@ self_training_clf.fit(X_train, y_train) # The amount of labeled samples that at the end of fitting - amount_labeled[i, fold] = total_samples - np.unique( - self_training_clf.labeled_iter_, return_counts=True)[1][0] + amount_labeled[i, fold] = ( + total_samples + - np.unique(self_training_clf.labeled_iter_, return_counts=True)[1][0] + ) # The last iteration the classifier labeled a sample in amount_iterations[i, fold] = np.max(self_training_clf.labeled_iter_) @@ -84,26 +86,34 @@ ax1 = plt.subplot(211) -ax1.errorbar(x_values, scores.mean(axis=1), - yerr=scores.std(axis=1), - capsize=2, color='b') -ax1.set_ylabel('Accuracy', color='b') -ax1.tick_params('y', colors='b') +ax1.errorbar( + x_values, scores.mean(axis=1), yerr=scores.std(axis=1), capsize=2, color="b" +) +ax1.set_ylabel("Accuracy", color="b") +ax1.tick_params("y", colors="b") ax2 = ax1.twinx() -ax2.errorbar(x_values, amount_labeled.mean(axis=1), - yerr=amount_labeled.std(axis=1), - capsize=2, color='g') +ax2.errorbar( + x_values, + amount_labeled.mean(axis=1), + yerr=amount_labeled.std(axis=1), + capsize=2, + color="g", +) ax2.set_ylim(bottom=0) -ax2.set_ylabel('Amount of labeled samples', color='g') -ax2.tick_params('y', colors='g') +ax2.set_ylabel("Amount of labeled samples", color="g") +ax2.tick_params("y", colors="g") ax3 = plt.subplot(212, sharex=ax1) -ax3.errorbar(x_values, amount_iterations.mean(axis=1), - yerr=amount_iterations.std(axis=1), - capsize=2, color='b') +ax3.errorbar( + x_values, + amount_iterations.mean(axis=1), + yerr=amount_iterations.std(axis=1), + capsize=2, + color="b", +) ax3.set_ylim(bottom=0) -ax3.set_ylabel('Amount of iterations') -ax3.set_xlabel('Threshold') +ax3.set_ylabel("Amount of iterations") +ax3.set_xlabel("Threshold") plt.show() diff --git a/examples/semi_supervised/plot_semi_supervised_newsgroups.py b/examples/semi_supervised/plot_semi_supervised_newsgroups.py index 72815471f54b9..1ad7bf85953e7 100644 --- a/examples/semi_supervised/plot_semi_supervised_newsgroups.py +++ b/examples/semi_supervised/plot_semi_supervised_newsgroups.py @@ -8,61 +8,79 @@ You can adjust the number of categories by giving their names to the dataset loader or setting them to `None` to get all 20 of them. + """ -import os + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import numpy as np from sklearn.datasets import fetch_20newsgroups -from sklearn.feature_extraction.text import CountVectorizer -from sklearn.feature_extraction.text import TfidfTransformer -from sklearn.preprocessing import FunctionTransformer +from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer from sklearn.linear_model import SGDClassifier +from sklearn.metrics import f1_score from sklearn.model_selection import train_test_split from sklearn.pipeline import Pipeline -from sklearn.semi_supervised import SelfTrainingClassifier -from sklearn.semi_supervised import LabelSpreading -from sklearn.metrics import f1_score - -data = fetch_20newsgroups(subset='train', categories=None) +from sklearn.preprocessing import FunctionTransformer +from sklearn.semi_supervised import LabelSpreading, SelfTrainingClassifier + +# Loading dataset containing first five categories +data = fetch_20newsgroups( + subset="train", + categories=[ + "alt.atheism", + "comp.graphics", + "comp.os.ms-windows.misc", + "comp.sys.ibm.pc.hardware", + "comp.sys.mac.hardware", + ], +) print("%d documents" % len(data.filenames)) print("%d categories" % len(data.target_names)) print() # Parameters -sdg_params = dict(alpha=1e-5, penalty='l2', loss='log') +sdg_params = dict(alpha=1e-5, penalty="l2", loss="log_loss") vectorizer_params = dict(ngram_range=(1, 2), min_df=5, max_df=0.8) # Supervised Pipeline -pipeline = Pipeline([ - ('vect', CountVectorizer(**vectorizer_params)), - ('tfidf', TfidfTransformer()), - ('clf', SGDClassifier(**sdg_params)), -]) +pipeline = Pipeline( + [ + ("vect", CountVectorizer(**vectorizer_params)), + ("tfidf", TfidfTransformer()), + ("clf", SGDClassifier(**sdg_params)), + ] +) # SelfTraining Pipeline -st_pipeline = Pipeline([ - ('vect', CountVectorizer(**vectorizer_params)), - ('tfidf', TfidfTransformer()), - ('clf', SelfTrainingClassifier(SGDClassifier(**sdg_params), verbose=True)), -]) +st_pipeline = Pipeline( + [ + ("vect", CountVectorizer(**vectorizer_params)), + ("tfidf", TfidfTransformer()), + ("clf", SelfTrainingClassifier(SGDClassifier(**sdg_params), verbose=True)), + ] +) # LabelSpreading Pipeline -ls_pipeline = Pipeline([ - ('vect', CountVectorizer(**vectorizer_params)), - ('tfidf', TfidfTransformer()), - # LabelSpreading does not support dense matrices - ('todense', FunctionTransformer(lambda x: x.todense())), - ('clf', LabelSpreading()), -]) +ls_pipeline = Pipeline( + [ + ("vect", CountVectorizer(**vectorizer_params)), + ("tfidf", TfidfTransformer()), + # LabelSpreading does not support dense matrices + ("toarray", FunctionTransformer(lambda x: x.toarray())), + ("clf", LabelSpreading()), + ] +) def eval_and_print_metrics(clf, X_train, y_train, X_test, y_test): print("Number of training samples:", len(X_train)) - print("Unlabeled samples in training set:", - sum(1 for x in y_train if x == -1)) + print("Unlabeled samples in training set:", sum(1 for x in y_train if x == -1)) clf.fit(X_train, y_train) y_pred = clf.predict(X_test) - print("Micro-averaged F1 score on test set: " - "%0.3f" % f1_score(y_test, y_pred, average='micro')) + print( + "Micro-averaged F1 score on test set: %0.3f" + % f1_score(y_test, y_pred, average="micro") + ) print("-" * 10) print() @@ -78,18 +96,16 @@ def eval_and_print_metrics(clf, X_train, y_train, X_test, y_test): y_mask = np.random.rand(len(y_train)) < 0.2 # X_20 and y_20 are the subset of the train dataset indicated by the mask - X_20, y_20 = map(list, zip(*((x, y) - for x, y, m in zip(X_train, y_train, y_mask) if m))) + X_20, y_20 = map( + list, zip(*((x, y) for x, y, m in zip(X_train, y_train, y_mask) if m)) + ) print("Supervised SGDClassifier on 20% of the training data:") eval_and_print_metrics(pipeline, X_20, y_20, X_test, y_test) # set the non-masked subset to be unlabeled y_train[~y_mask] = -1 - print("SelfTrainingClassifier on 20% of the training data (rest " - "is unlabeled):") + print("SelfTrainingClassifier on 20% of the training data (rest is unlabeled):") eval_and_print_metrics(st_pipeline, X_train, y_train, X_test, y_test) - if 'CI' not in os.environ: - # LabelSpreading takes too long to run in the online documentation - print("LabelSpreading on 20% of the data (rest is unlabeled):") - eval_and_print_metrics(ls_pipeline, X_train, y_train, X_test, y_test) + print("LabelSpreading on 20% of the data (rest is unlabeled):") + eval_and_print_metrics(ls_pipeline, X_train, y_train, X_test, y_test) diff --git a/examples/semi_supervised/plot_semi_supervised_versus_svm_iris.py b/examples/semi_supervised/plot_semi_supervised_versus_svm_iris.py index f93dc2f28370e..3872a59377cab 100644 --- a/examples/semi_supervised/plot_semi_supervised_versus_svm_iris.py +++ b/examples/semi_supervised/plot_semi_supervised_versus_svm_iris.py @@ -13,19 +13,16 @@ identical to training the SVC on 100% of the data. """ -print(__doc__) -# Authors: Clay Woolam -# Oliver Rausch -# License: BSD +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt +import numpy as np + from sklearn import datasets +from sklearn.semi_supervised import LabelSpreading, SelfTrainingClassifier from sklearn.svm import SVC -from sklearn.semi_supervised import LabelSpreading -from sklearn.semi_supervised import SelfTrainingClassifier - iris = datasets.load_iris() @@ -33,7 +30,7 @@ y = iris.target # step size in the mesh -h = .02 +h = 0.02 rng = np.random.RandomState(0) y_rand = rng.rand(y.shape[0]) @@ -43,26 +40,31 @@ y_50[y_rand < 0.5] = -1 # we create an instance of SVM and fit out data. We do not scale our # data since we want to plot the support vectors -ls30 = (LabelSpreading().fit(X, y_30), y_30, 'Label Spreading 30% data') -ls50 = (LabelSpreading().fit(X, y_50), y_50, 'Label Spreading 50% data') -ls100 = (LabelSpreading().fit(X, y), y, 'Label Spreading 100% data') +ls30 = (LabelSpreading().fit(X, y_30), y_30, "Label Spreading 30% data") +ls50 = (LabelSpreading().fit(X, y_50), y_50, "Label Spreading 50% data") +ls100 = (LabelSpreading().fit(X, y), y, "Label Spreading 100% data") # the base classifier for self-training is identical to the SVC -base_classifier = SVC(kernel='rbf', gamma=.5, probability=True) -st30 = (SelfTrainingClassifier(base_classifier).fit(X, y_30), - y_30, 'Self-training 30% data') -st50 = (SelfTrainingClassifier(base_classifier).fit(X, y_50), - y_50, 'Self-training 50% data') - -rbf_svc = (SVC(kernel='rbf', gamma=.5).fit(X, y), y, 'SVC with rbf kernel') +base_classifier = SVC(kernel="rbf", gamma=0.5, probability=True) +st30 = ( + SelfTrainingClassifier(base_classifier).fit(X, y_30), + y_30, + "Self-training 30% data", +) +st50 = ( + SelfTrainingClassifier(base_classifier).fit(X, y_50), + y_50, + "Self-training 50% data", +) + +rbf_svc = (SVC(kernel="rbf", gamma=0.5).fit(X, y), y, "SVC with rbf kernel") # create a mesh to plot in x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1 y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1 -xx, yy = np.meshgrid(np.arange(x_min, x_max, h), - np.arange(y_min, y_max, h)) +xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h)) -color_map = {-1: (1, 1, 1), 0: (0, 0, .9), 1: (1, 0, 0), 2: (.8, .6, 0)} +color_map = {-1: (1, 1, 1), 0: (0, 0, 0.9), 1: (1, 0, 0), 2: (0.8, 0.6, 0)} classifiers = (ls30, st30, ls50, st50, ls100, rbf_svc) for i, (clf, y_train, title) in enumerate(classifiers): @@ -74,11 +76,11 @@ # Put the result into a color plot Z = Z.reshape(xx.shape) plt.contourf(xx, yy, Z, cmap=plt.cm.Paired) - plt.axis('off') + plt.axis("off") # Plot also the training points colors = [color_map[y] for y in y_train] - plt.scatter(X[:, 0], X[:, 1], c=colors, edgecolors='black') + plt.scatter(X[:, 0], X[:, 1], c=colors, edgecolors="black") plt.title(title) diff --git a/examples/svm/plot_custom_kernel.py b/examples/svm/plot_custom_kernel.py index 28641cd35f8cb..d3816849f73b8 100644 --- a/examples/svm/plot_custom_kernel.py +++ b/examples/svm/plot_custom_kernel.py @@ -7,16 +7,20 @@ plot the decision surface and the support vectors. """ -print(__doc__) -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt -from sklearn import svm, datasets +import numpy as np + +from sklearn import datasets, svm +from sklearn.inspection import DecisionBoundaryDisplay # import some data to play with iris = datasets.load_iris() X = iris.data[:, :2] # we only take the first two features. We could - # avoid this ugly slicing by using a two-dim dataset +# avoid this ugly slicing by using a two-dim dataset Y = iris.target @@ -32,26 +36,25 @@ def my_kernel(X, Y): return np.dot(np.dot(X, M), Y.T) -h = .02 # step size in the mesh +h = 0.02 # step size in the mesh # we create an instance of SVM and fit out data. clf = svm.SVC(kernel=my_kernel) clf.fit(X, Y) -# Plot the decision boundary. For that, we will assign a color to each -# point in the mesh [x_min, x_max]x[y_min, y_max]. -x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1 -y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1 -xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h)) -Z = clf.predict(np.c_[xx.ravel(), yy.ravel()]) - -# Put the result into a color plot -Z = Z.reshape(xx.shape) -plt.pcolormesh(xx, yy, Z, cmap=plt.cm.Paired) +ax = plt.gca() +DecisionBoundaryDisplay.from_estimator( + clf, + X, + cmap=plt.cm.Paired, + ax=ax, + response_method="predict", + plot_method="pcolormesh", + shading="auto", +) # Plot also the training points -plt.scatter(X[:, 0], X[:, 1], c=Y, cmap=plt.cm.Paired, edgecolors='k') -plt.title('3-Class classification using Support Vector Machine with custom' - ' kernel') -plt.axis('tight') +plt.scatter(X[:, 0], X[:, 1], c=Y, cmap=plt.cm.Paired, edgecolors="k") +plt.title("3-Class classification using Support Vector Machine with custom kernel") +plt.axis("tight") plt.show() diff --git a/examples/svm/plot_iris_svc.py b/examples/svm/plot_iris_svc.py index ab7860296985c..77259f9d1ea2c 100644 --- a/examples/svm/plot_iris_svc.py +++ b/examples/svm/plot_iris_svc.py @@ -33,49 +33,14 @@ more realistic high-dimensional problems. """ -print(__doc__) -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt -from sklearn import svm, datasets - - -def make_meshgrid(x, y, h=.02): - """Create a mesh of points to plot in - - Parameters - ---------- - x: data to base x-axis meshgrid on - y: data to base y-axis meshgrid on - h: stepsize for meshgrid, optional - - Returns - ------- - xx, yy : ndarray - """ - x_min, x_max = x.min() - 1, x.max() + 1 - y_min, y_max = y.min() - 1, y.max() + 1 - xx, yy = np.meshgrid(np.arange(x_min, x_max, h), - np.arange(y_min, y_max, h)) - return xx, yy - - -def plot_contours(ax, clf, xx, yy, **params): - """Plot the decision boundaries for a classifier. - - Parameters - ---------- - ax: matplotlib axes object - clf: a classifier - xx: meshgrid ndarray - yy: meshgrid ndarray - params: dictionary of params to pass to contourf, optional - """ - Z = clf.predict(np.c_[xx.ravel(), yy.ravel()]) - Z = Z.reshape(xx.shape) - out = ax.contourf(xx, yy, Z, **params) - return out +from sklearn import datasets, svm +from sklearn.inspection import DecisionBoundaryDisplay # import some data to play with iris = datasets.load_iris() @@ -86,33 +51,40 @@ def plot_contours(ax, clf, xx, yy, **params): # we create an instance of SVM and fit out data. We do not scale our # data since we want to plot the support vectors C = 1.0 # SVM regularization parameter -models = (svm.SVC(kernel='linear', C=C), - svm.LinearSVC(C=C, max_iter=10000), - svm.SVC(kernel='rbf', gamma=0.7, C=C), - svm.SVC(kernel='poly', degree=3, gamma='auto', C=C)) +models = ( + svm.SVC(kernel="linear", C=C), + svm.LinearSVC(C=C, max_iter=10000), + svm.SVC(kernel="rbf", gamma=0.7, C=C), + svm.SVC(kernel="poly", degree=3, gamma="auto", C=C), +) models = (clf.fit(X, y) for clf in models) # title for the plots -titles = ('SVC with linear kernel', - 'LinearSVC (linear kernel)', - 'SVC with RBF kernel', - 'SVC with polynomial (degree 3) kernel') +titles = ( + "SVC with linear kernel", + "LinearSVC (linear kernel)", + "SVC with RBF kernel", + "SVC with polynomial (degree 3) kernel", +) # Set-up 2x2 grid for plotting. fig, sub = plt.subplots(2, 2) plt.subplots_adjust(wspace=0.4, hspace=0.4) X0, X1 = X[:, 0], X[:, 1] -xx, yy = make_meshgrid(X0, X1) for clf, title, ax in zip(models, titles, sub.flatten()): - plot_contours(ax, clf, xx, yy, - cmap=plt.cm.coolwarm, alpha=0.8) - ax.scatter(X0, X1, c=y, cmap=plt.cm.coolwarm, s=20, edgecolors='k') - ax.set_xlim(xx.min(), xx.max()) - ax.set_ylim(yy.min(), yy.max()) - ax.set_xlabel('Sepal length') - ax.set_ylabel('Sepal width') + disp = DecisionBoundaryDisplay.from_estimator( + clf, + X, + response_method="predict", + cmap=plt.cm.coolwarm, + alpha=0.8, + ax=ax, + xlabel=iris.feature_names[0], + ylabel=iris.feature_names[1], + ) + ax.scatter(X0, X1, c=y, cmap=plt.cm.coolwarm, s=20, edgecolors="k") ax.set_xticks(()) ax.set_yticks(()) ax.set_title(title) diff --git a/examples/svm/plot_linearsvc_support_vectors.py b/examples/svm/plot_linearsvc_support_vectors.py index cc7e9caa5cda8..370f826d11a64 100644 --- a/examples/svm/plot_linearsvc_support_vectors.py +++ b/examples/svm/plot_linearsvc_support_vectors.py @@ -9,9 +9,14 @@ """ -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt +import numpy as np + from sklearn.datasets import make_blobs +from sklearn.inspection import DecisionBoundaryDisplay from sklearn.svm import LinearSVC X, y = make_blobs(n_samples=40, centers=2, random_state=0) @@ -26,23 +31,31 @@ # decision_function = np.dot(X, clf.coef_[0]) + clf.intercept_[0] # The support vectors are the samples that lie within the margin # boundaries, whose size is conventionally constrained to 1 - support_vector_indices = np.where( - np.abs(decision_function) <= 1 + 1e-15)[0] + support_vector_indices = (np.abs(decision_function) <= 1 + 1e-15).nonzero()[0] support_vectors = X[support_vector_indices] plt.subplot(1, 2, i + 1) plt.scatter(X[:, 0], X[:, 1], c=y, s=30, cmap=plt.cm.Paired) ax = plt.gca() - xlim = ax.get_xlim() - ylim = ax.get_ylim() - xx, yy = np.meshgrid(np.linspace(xlim[0], xlim[1], 50), - np.linspace(ylim[0], ylim[1], 50)) - Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()]) - Z = Z.reshape(xx.shape) - plt.contour(xx, yy, Z, colors='k', levels=[-1, 0, 1], alpha=0.5, - linestyles=['--', '-', '--']) - plt.scatter(support_vectors[:, 0], support_vectors[:, 1], s=100, - linewidth=1, facecolors='none', edgecolors='k') + DecisionBoundaryDisplay.from_estimator( + clf, + X, + ax=ax, + grid_resolution=50, + plot_method="contour", + colors="k", + levels=[-1, 0, 1], + alpha=0.5, + linestyles=["--", "-", "--"], + ) + plt.scatter( + support_vectors[:, 0], + support_vectors[:, 1], + s=100, + linewidth=1, + facecolors="none", + edgecolors="k", + ) plt.title("C=" + str(C)) plt.tight_layout() plt.show() diff --git a/examples/svm/plot_oneclass.py b/examples/svm/plot_oneclass.py index 3f04537ca1f00..0db71966db6a9 100644 --- a/examples/svm/plot_oneclass.py +++ b/examples/svm/plot_oneclass.py @@ -8,15 +8,17 @@ :ref:`One-class SVM ` is an unsupervised algorithm that learns a decision function for novelty detection: classifying new data as similar or different to the training set. + """ -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% import numpy as np -import matplotlib.pyplot as plt -import matplotlib.font_manager + from sklearn import svm -xx, yy = np.meshgrid(np.linspace(-5, 5, 500), np.linspace(-5, 5, 500)) # Generate train data X = 0.3 * np.random.randn(100, 2) X_train = np.r_[X + 2, X - 2] @@ -36,31 +38,68 @@ n_error_test = y_pred_test[y_pred_test == -1].size n_error_outliers = y_pred_outliers[y_pred_outliers == 1].size -# plot the line, the points, and the nearest vectors to the plane -Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()]) -Z = Z.reshape(xx.shape) +# %% +import matplotlib.font_manager +import matplotlib.lines as mlines +import matplotlib.pyplot as plt + +from sklearn.inspection import DecisionBoundaryDisplay + +_, ax = plt.subplots() -plt.title("Novelty Detection") -plt.contourf(xx, yy, Z, levels=np.linspace(Z.min(), 0, 7), cmap=plt.cm.PuBu) -a = plt.contour(xx, yy, Z, levels=[0], linewidths=2, colors='darkred') -plt.contourf(xx, yy, Z, levels=[0, Z.max()], colors='palevioletred') +# generate grid for the boundary display +xx, yy = np.meshgrid(np.linspace(-5, 5, 10), np.linspace(-5, 5, 10)) +X = np.concatenate([xx.reshape(-1, 1), yy.reshape(-1, 1)], axis=1) +DecisionBoundaryDisplay.from_estimator( + clf, + X, + response_method="decision_function", + plot_method="contourf", + ax=ax, + cmap="PuBu", +) +DecisionBoundaryDisplay.from_estimator( + clf, + X, + response_method="decision_function", + plot_method="contourf", + ax=ax, + levels=[0, 10000], + colors="palevioletred", +) +DecisionBoundaryDisplay.from_estimator( + clf, + X, + response_method="decision_function", + plot_method="contour", + ax=ax, + levels=[0], + colors="darkred", + linewidths=2, +) s = 40 -b1 = plt.scatter(X_train[:, 0], X_train[:, 1], c='white', s=s, edgecolors='k') -b2 = plt.scatter(X_test[:, 0], X_test[:, 1], c='blueviolet', s=s, - edgecolors='k') -c = plt.scatter(X_outliers[:, 0], X_outliers[:, 1], c='gold', s=s, - edgecolors='k') -plt.axis('tight') -plt.xlim((-5, 5)) -plt.ylim((-5, 5)) -plt.legend([a.collections[0], b1, b2, c], - ["learned frontier", "training observations", - "new regular observations", "new abnormal observations"], - loc="upper left", - prop=matplotlib.font_manager.FontProperties(size=11)) -plt.xlabel( - "error train: %d/200 ; errors novel regular: %d/40 ; " - "errors novel abnormal: %d/40" - % (n_error_train, n_error_test, n_error_outliers)) +b1 = ax.scatter(X_train[:, 0], X_train[:, 1], c="white", s=s, edgecolors="k") +b2 = ax.scatter(X_test[:, 0], X_test[:, 1], c="blueviolet", s=s, edgecolors="k") +c = ax.scatter(X_outliers[:, 0], X_outliers[:, 1], c="gold", s=s, edgecolors="k") +plt.legend( + [mlines.Line2D([], [], color="darkred"), b1, b2, c], + [ + "learned frontier", + "training observations", + "new regular observations", + "new abnormal observations", + ], + loc="upper left", + prop=matplotlib.font_manager.FontProperties(size=11), +) +ax.set( + xlabel=( + f"error train: {n_error_train}/200 ; errors novel regular: {n_error_test}/40 ;" + f" errors novel abnormal: {n_error_outliers}/40" + ), + title="Novelty Detection", + xlim=(-5, 5), + ylim=(-5, 5), +) plt.show() diff --git a/examples/svm/plot_rbf_parameters.py b/examples/svm/plot_rbf_parameters.py index 61d647d16325b..356707e2d72b2 100644 --- a/examples/svm/plot_rbf_parameters.py +++ b/examples/svm/plot_rbf_parameters.py @@ -1,4 +1,4 @@ -''' +""" ================== RBF SVM parameters ================== @@ -73,25 +73,20 @@ ``gamma_range`` steps will increase the resolution of the hyper-parameter heat map. -''' -print(__doc__) +""" -import numpy as np -import matplotlib.pyplot as plt -from matplotlib.colors import Normalize +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -from sklearn.svm import SVC -from sklearn.preprocessing import StandardScaler -from sklearn.datasets import load_iris -from sklearn.model_selection import StratifiedShuffleSplit -from sklearn.model_selection import GridSearchCV +# %% +# Utility class to move the midpoint of a colormap to be around +# the values of interest. +import numpy as np +from matplotlib.colors import Normalize -# Utility function to move the midpoint of a colormap to be around -# the values of interest. class MidpointNormalize(Normalize): - def __init__(self, vmin=None, vmax=None, midpoint=None, clip=False): self.midpoint = midpoint Normalize.__init__(self, vmin, vmax, clip) @@ -100,15 +95,20 @@ def __call__(self, value, clip=None): x, y = [self.vmin, self.midpoint, self.vmax], [0, 0.5, 1] return np.ma.masked_array(np.interp(value, x, y)) -# ############################################################################# + +# %% # Load and prepare data set +# ------------------------- # # dataset for grid search +from sklearn.datasets import load_iris + iris = load_iris() X = iris.data y = iris.target +# %% # Dataset for decision function visualization: we only keep the first two # features in X and sub-sample the dataset to keep only 2 classes and # make it a binary classification problem. @@ -118,22 +118,29 @@ def __call__(self, value, clip=None): y_2d = y[y > 0] y_2d -= 1 +# %% # It is usually a good idea to scale the data for SVM training. # We are cheating a bit in this example in scaling all of the data, # instead of fitting the transformation on the training set and # just applying it on the test set. +from sklearn.preprocessing import StandardScaler + scaler = StandardScaler() X = scaler.fit_transform(X) X_2d = scaler.fit_transform(X_2d) -# ############################################################################# +# %% # Train classifiers +# ----------------- # # For an initial search, a logarithmic grid with basis # 10 is often helpful. Using a basis of 2, a finer # tuning can be achieved but at a much higher cost. +from sklearn.model_selection import GridSearchCV, StratifiedShuffleSplit +from sklearn.svm import SVC + C_range = np.logspace(-2, 10, 13) gamma_range = np.logspace(-9, 3, 13) param_grid = dict(gamma=gamma_range, C=C_range) @@ -141,9 +148,12 @@ def __call__(self, value, clip=None): grid = GridSearchCV(SVC(), param_grid=param_grid, cv=cv) grid.fit(X, y) -print("The best parameters are %s with a score of %0.2f" - % (grid.best_params_, grid.best_score_)) +print( + "The best parameters are %s with a score of %0.2f" + % (grid.best_params_, grid.best_score_) +) +# %% # Now we need to fit a classifier for all parameters in the 2d version # (we use a smaller set of parameters here because it takes a while to train) @@ -156,34 +166,35 @@ def __call__(self, value, clip=None): clf.fit(X_2d, y_2d) classifiers.append((C, gamma, clf)) -# ############################################################################# +# %% # Visualization +# ------------- # # draw visualization of parameter effects +import matplotlib.pyplot as plt + plt.figure(figsize=(8, 6)) xx, yy = np.meshgrid(np.linspace(-3, 3, 200), np.linspace(-3, 3, 200)) -for (k, (C, gamma, clf)) in enumerate(classifiers): +for k, (C, gamma, clf) in enumerate(classifiers): # evaluate decision function in a grid Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()]) Z = Z.reshape(xx.shape) # visualize decision function for these parameters plt.subplot(len(C_2d_range), len(gamma_2d_range), k + 1) - plt.title("gamma=10^%d, C=10^%d" % (np.log10(gamma), np.log10(C)), - size='medium') + plt.title("gamma=10^%d, C=10^%d" % (np.log10(gamma), np.log10(C)), size="medium") # visualize parameter's effect on decision function plt.pcolormesh(xx, yy, -Z, cmap=plt.cm.RdBu) - plt.scatter(X_2d[:, 0], X_2d[:, 1], c=y_2d, cmap=plt.cm.RdBu_r, - edgecolors='k') + plt.scatter(X_2d[:, 0], X_2d[:, 1], c=y_2d, cmap=plt.cm.RdBu_r, edgecolors="k") plt.xticks(()) plt.yticks(()) - plt.axis('tight') + plt.axis("tight") -scores = grid.cv_results_['mean_test_score'].reshape(len(C_range), - len(gamma_range)) +scores = grid.cv_results_["mean_test_score"].reshape(len(C_range), len(gamma_range)) +# %% # Draw heatmap of the validation accuracy as a function of gamma and C # # The score are encoded as colors with the hot colormap which varies from dark @@ -194,13 +205,17 @@ def __call__(self, value, clip=None): # the same color. plt.figure(figsize=(8, 6)) -plt.subplots_adjust(left=.2, right=0.95, bottom=0.15, top=0.95) -plt.imshow(scores, interpolation='nearest', cmap=plt.cm.hot, - norm=MidpointNormalize(vmin=0.2, midpoint=0.92)) -plt.xlabel('gamma') -plt.ylabel('C') +plt.subplots_adjust(left=0.2, right=0.95, bottom=0.15, top=0.95) +plt.imshow( + scores, + interpolation="nearest", + cmap=plt.cm.hot, + norm=MidpointNormalize(vmin=0.2, midpoint=0.92), +) +plt.xlabel("gamma") +plt.ylabel("C") plt.colorbar() plt.xticks(np.arange(len(gamma_range)), gamma_range, rotation=45) plt.yticks(np.arange(len(C_range)), C_range) -plt.title('Validation accuracy') +plt.title("Validation accuracy") plt.show() diff --git a/examples/svm/plot_separating_hyperplane.py b/examples/svm/plot_separating_hyperplane.py index cbd61abad53e6..842da314feb1a 100644 --- a/examples/svm/plot_separating_hyperplane.py +++ b/examples/svm/plot_separating_hyperplane.py @@ -6,40 +6,46 @@ Plot the maximum margin separating hyperplane within a two-class separable dataset using a Support Vector Machine classifier with linear kernel. + """ -print(__doc__) -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt + from sklearn import svm from sklearn.datasets import make_blobs - +from sklearn.inspection import DecisionBoundaryDisplay # we create 40 separable points X, y = make_blobs(n_samples=40, centers=2, random_state=6) # fit the model, don't regularize for illustration purposes -clf = svm.SVC(kernel='linear', C=1000) +clf = svm.SVC(kernel="linear", C=1000) clf.fit(X, y) plt.scatter(X[:, 0], X[:, 1], c=y, s=30, cmap=plt.cm.Paired) # plot the decision function ax = plt.gca() -xlim = ax.get_xlim() -ylim = ax.get_ylim() - -# create grid to evaluate model -xx = np.linspace(xlim[0], xlim[1], 30) -yy = np.linspace(ylim[0], ylim[1], 30) -YY, XX = np.meshgrid(yy, xx) -xy = np.vstack([XX.ravel(), YY.ravel()]).T -Z = clf.decision_function(xy).reshape(XX.shape) - -# plot decision boundary and margins -ax.contour(XX, YY, Z, colors='k', levels=[-1, 0, 1], alpha=0.5, - linestyles=['--', '-', '--']) +DecisionBoundaryDisplay.from_estimator( + clf, + X, + plot_method="contour", + colors="k", + levels=[-1, 0, 1], + alpha=0.5, + linestyles=["--", "-", "--"], + ax=ax, +) # plot support vectors -ax.scatter(clf.support_vectors_[:, 0], clf.support_vectors_[:, 1], s=100, - linewidth=1, facecolors='none', edgecolors='k') +ax.scatter( + clf.support_vectors_[:, 0], + clf.support_vectors_[:, 1], + s=100, + linewidth=1, + facecolors="none", + edgecolors="k", +) plt.show() diff --git a/examples/svm/plot_separating_hyperplane_unbalanced.py b/examples/svm/plot_separating_hyperplane_unbalanced.py index 2a0540fead310..d0814e1af065f 100644 --- a/examples/svm/plot_separating_hyperplane_unbalanced.py +++ b/examples/svm/plot_separating_hyperplane_unbalanced.py @@ -24,57 +24,72 @@ clf = SGDClassifier(n_iter=100, alpha=0.01) """ -print(__doc__) -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import matplotlib.lines as mlines import matplotlib.pyplot as plt + from sklearn import svm from sklearn.datasets import make_blobs +from sklearn.inspection import DecisionBoundaryDisplay # we create two clusters of random points n_samples_1 = 1000 n_samples_2 = 100 centers = [[0.0, 0.0], [2.0, 2.0]] clusters_std = [1.5, 0.5] -X, y = make_blobs(n_samples=[n_samples_1, n_samples_2], - centers=centers, - cluster_std=clusters_std, - random_state=0, shuffle=False) +X, y = make_blobs( + n_samples=[n_samples_1, n_samples_2], + centers=centers, + cluster_std=clusters_std, + random_state=0, + shuffle=False, +) # fit the model and get the separating hyperplane -clf = svm.SVC(kernel='linear', C=1.0) +clf = svm.SVC(kernel="linear", C=1.0) clf.fit(X, y) # fit the model and get the separating hyperplane using weighted classes -wclf = svm.SVC(kernel='linear', class_weight={1: 10}) +wclf = svm.SVC(kernel="linear", class_weight={1: 10}) wclf.fit(X, y) # plot the samples -plt.scatter(X[:, 0], X[:, 1], c=y, cmap=plt.cm.Paired, edgecolors='k') +plt.scatter(X[:, 0], X[:, 1], c=y, cmap=plt.cm.Paired, edgecolors="k") # plot the decision functions for both classifiers ax = plt.gca() -xlim = ax.get_xlim() -ylim = ax.get_ylim() - -# create grid to evaluate model -xx = np.linspace(xlim[0], xlim[1], 30) -yy = np.linspace(ylim[0], ylim[1], 30) -YY, XX = np.meshgrid(yy, xx) -xy = np.vstack([XX.ravel(), YY.ravel()]).T - -# get the separating hyperplane -Z = clf.decision_function(xy).reshape(XX.shape) - -# plot decision boundary and margins -a = ax.contour(XX, YY, Z, colors='k', levels=[0], alpha=0.5, linestyles=['-']) - -# get the separating hyperplane for weighted classes -Z = wclf.decision_function(xy).reshape(XX.shape) +disp = DecisionBoundaryDisplay.from_estimator( + clf, + X, + plot_method="contour", + colors="k", + levels=[0], + alpha=0.5, + linestyles=["-"], + ax=ax, +) # plot decision boundary and margins for weighted classes -b = ax.contour(XX, YY, Z, colors='r', levels=[0], alpha=0.5, linestyles=['-']) - -plt.legend([a.collections[0], b.collections[0]], ["non weighted", "weighted"], - loc="upper right") +wdisp = DecisionBoundaryDisplay.from_estimator( + wclf, + X, + plot_method="contour", + colors="r", + levels=[0], + alpha=0.5, + linestyles=["-"], + ax=ax, +) + +plt.legend( + [ + mlines.Line2D([], [], color="k", label="non weighted"), + mlines.Line2D([], [], color="r", label="weighted"), + ], + ["non weighted", "weighted"], + loc="upper right", +) plt.show() diff --git a/examples/svm/plot_svm_anova.py b/examples/svm/plot_svm_anova.py index b0392b1c00361..1c2a78e79fdb9 100644 --- a/examples/svm/plot_svm_anova.py +++ b/examples/svm/plot_svm_anova.py @@ -7,35 +7,51 @@ SVC (support vector classifier) to improve the classification scores. We use the iris dataset (4 features) and add 36 non-informative features. We can find that our model achieves best performance when we select around 10% of features. + """ -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Load some data to play with +# --------------------------- import numpy as np -import matplotlib.pyplot as plt -from sklearn.datasets import load_iris -from sklearn.feature_selection import SelectPercentile, chi2 -from sklearn.model_selection import cross_val_score -from sklearn.pipeline import Pipeline -from sklearn.preprocessing import StandardScaler -from sklearn.svm import SVC +from sklearn.datasets import load_iris -# ############################################################################# -# Import some data to play with X, y = load_iris(return_X_y=True) + # Add non-informative features -np.random.seed(0) -X = np.hstack((X, 2 * np.random.random((X.shape[0], 36)))) +rng = np.random.RandomState(0) +X = np.hstack((X, 2 * rng.random((X.shape[0], 36)))) + +# %% +# Create the pipeline +# ------------------- +from sklearn.feature_selection import SelectPercentile, f_classif +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import StandardScaler +from sklearn.svm import SVC -# ############################################################################# # Create a feature-selection transform, a scaler and an instance of SVM that we # combine together to have a full-blown estimator -clf = Pipeline([('anova', SelectPercentile(chi2)), - ('scaler', StandardScaler()), - ('svc', SVC(gamma="auto"))]) -# ############################################################################# +clf = Pipeline( + [ + ("anova", SelectPercentile(f_classif)), + ("scaler", StandardScaler()), + ("svc", SVC(gamma="auto")), + ] +) + +# %% # Plot the cross-validation score as a function of percentile of features +# ----------------------------------------------------------------------- +import matplotlib.pyplot as plt + +from sklearn.model_selection import cross_val_score + score_means = list() score_stds = list() percentiles = (1, 3, 6, 10, 15, 20, 30, 40, 60, 80, 100) @@ -47,10 +63,9 @@ score_stds.append(this_scores.std()) plt.errorbar(percentiles, score_means, np.array(score_stds)) -plt.title( - 'Performance of the SVM-Anova varying the percentile of features selected') +plt.title("Performance of the SVM-Anova varying the percentile of features selected") plt.xticks(np.linspace(0, 100, 11, endpoint=True)) -plt.xlabel('Percentile') -plt.ylabel('Accuracy Score') -plt.axis('tight') +plt.xlabel("Percentile") +plt.ylabel("Accuracy Score") +plt.axis("tight") plt.show() diff --git a/examples/svm/plot_svm_kernels.py b/examples/svm/plot_svm_kernels.py index dbad4e0b725e2..d01f049dbe0b4 100644 --- a/examples/svm/plot_svm_kernels.py +++ b/examples/svm/plot_svm_kernels.py @@ -1,85 +1,304 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - """ ========================================================= -SVM-Kernels +Plot classification boundaries with different SVM Kernels ========================================================= +This example shows how different kernels in a :class:`~sklearn.svm.SVC` (Support Vector +Classifier) influence the classification boundaries in a binary, two-dimensional +classification problem. -Three different types of SVM-Kernels are displayed below. -The polynomial and RBF are especially useful when the -data-points are not linearly separable. +SVCs aim to find a hyperplane that effectively separates the classes in their training +data by maximizing the margin between the outermost data points of each class. This is +achieved by finding the best weight vector :math:`w` that defines the decision boundary +hyperplane and minimizes the sum of hinge losses for misclassified samples, as measured +by the :func:`~sklearn.metrics.hinge_loss` function. By default, regularization is +applied with the parameter `C=1`, which allows for a certain degree of misclassification +tolerance. +If the data is not linearly separable in the original feature space, a non-linear kernel +parameter can be set. Depending on the kernel, the process involves adding new features +or transforming existing features to enrich and potentially add meaning to the data. +When a kernel other than `"linear"` is set, the SVC applies the `kernel trick +`__, which +computes the similarity between pairs of data points using the kernel function without +explicitly transforming the entire dataset. The kernel trick surpasses the otherwise +necessary matrix transformation of the whole dataset by only considering the relations +between all pairs of data points. The kernel function maps two vectors (each pair of +observations) to their similarity using their dot product. -""" -print(__doc__) +The hyperplane can then be calculated using the kernel function as if the dataset were +represented in a higher-dimensional space. Using a kernel function instead of an +explicit matrix transformation improves performance, as the kernel function has a time +complexity of :math:`O({n}^2)`, whereas matrix transformation scales according to the +specific transformation being applied. +In this example, we compare the most common kernel types of Support Vector Machines: the +linear kernel (`"linear"`), the polynomial kernel (`"poly"`), the radial basis function +kernel (`"rbf"`) and the sigmoid kernel (`"sigmoid"`). +""" -# Code source: GaÃĢl Varoquaux -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np +# %% +# Creating a dataset +# ------------------ +# We create a two-dimensional classification dataset with 16 samples and two classes. We +# plot the samples with the colors matching their respective targets. import matplotlib.pyplot as plt +import numpy as np + +X = np.array( + [ + [0.4, -0.7], + [-1.5, -1.0], + [-1.4, -0.9], + [-1.3, -1.2], + [-1.1, -0.2], + [-1.2, -0.4], + [-0.5, 1.2], + [-1.5, 2.1], + [1.0, 1.0], + [1.3, 0.8], + [1.2, 0.5], + [0.2, -2.0], + [0.5, -2.4], + [0.2, -2.3], + [0.0, -2.7], + [1.3, 2.1], + ] +) + +y = np.array([0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]) + +# Plotting settings +fig, ax = plt.subplots(figsize=(4, 3)) +x_min, x_max, y_min, y_max = -3, 3, -3, 3 +ax.set(xlim=(x_min, x_max), ylim=(y_min, y_max)) + +# Plot samples by color and add legend +scatter = ax.scatter(X[:, 0], X[:, 1], s=150, c=y, label=y, edgecolors="k") +ax.legend(*scatter.legend_elements(), loc="upper right", title="Classes") +ax.set_title("Samples in two-dimensional feature space") +_ = plt.show() + +# %% +# We can see that the samples are not clearly separable by a straight line. +# +# Training SVC model and plotting decision boundaries +# --------------------------------------------------- +# We define a function that fits a :class:`~sklearn.svm.SVC` classifier, +# allowing the `kernel` parameter as an input, and then plots the decision +# boundaries learned by the model using +# :class:`~sklearn.inspection.DecisionBoundaryDisplay`. +# +# Notice that for the sake of simplicity, the `C` parameter is set to its +# default value (`C=1`) in this example and the `gamma` parameter is set to +# `gamma=2` across all kernels, although it is automatically ignored for the +# linear kernel. In a real classification task, where performance matters, +# parameter tuning (by using :class:`~sklearn.model_selection.GridSearchCV` for +# instance) is highly recommended to capture different structures within the +# data. +# +# Setting `response_method="predict"` in +# :class:`~sklearn.inspection.DecisionBoundaryDisplay` colors the areas based +# on their predicted class. Using `response_method="decision_function"` allows +# us to also plot the decision boundary and the margins to both sides of it. +# Finally the support vectors used during training (which always lay on the +# margins) are identified by means of the `support_vectors_` attribute of +# the trained SVCs, and plotted as well. from sklearn import svm +from sklearn.inspection import DecisionBoundaryDisplay + + +def plot_training_data_with_decision_boundary( + kernel, ax=None, long_title=True, support_vectors=True +): + # Train the SVC + clf = svm.SVC(kernel=kernel, gamma=2).fit(X, y) + + # Settings for plotting + if ax is None: + _, ax = plt.subplots(figsize=(4, 3)) + x_min, x_max, y_min, y_max = -3, 3, -3, 3 + ax.set(xlim=(x_min, x_max), ylim=(y_min, y_max)) + # Plot decision boundary and margins + common_params = {"estimator": clf, "X": X, "ax": ax} + DecisionBoundaryDisplay.from_estimator( + **common_params, + response_method="predict", + plot_method="pcolormesh", + alpha=0.3, + ) + DecisionBoundaryDisplay.from_estimator( + **common_params, + response_method="decision_function", + plot_method="contour", + levels=[-1, 0, 1], + colors=["k", "k", "k"], + linestyles=["--", "-", "--"], + ) -# Our dataset and targets -X = np.c_[(.4, -.7), - (-1.5, -1), - (-1.4, -.9), - (-1.3, -1.2), - (-1.1, -.2), - (-1.2, -.4), - (-.5, 1.2), - (-1.5, 2.1), - (1, 1), - # -- - (1.3, .8), - (1.2, .5), - (.2, -2), - (.5, -2.4), - (.2, -2.3), - (0, -2.7), - (1.3, 2.1)].T -Y = [0] * 8 + [1] * 8 - -# figure number -fignum = 1 - -# fit the model -for kernel in ('linear', 'poly', 'rbf'): - clf = svm.SVC(kernel=kernel, gamma=2) - clf.fit(X, Y) - - # plot the line, the points, and the nearest vectors to the plane - plt.figure(fignum, figsize=(4, 3)) - plt.clf() - - plt.scatter(clf.support_vectors_[:, 0], clf.support_vectors_[:, 1], s=80, - facecolors='none', zorder=10, edgecolors='k') - plt.scatter(X[:, 0], X[:, 1], c=Y, zorder=10, cmap=plt.cm.Paired, - edgecolors='k') - - plt.axis('tight') - x_min = -3 - x_max = 3 - y_min = -3 - y_max = 3 - - XX, YY = np.mgrid[x_min:x_max:200j, y_min:y_max:200j] - Z = clf.decision_function(np.c_[XX.ravel(), YY.ravel()]) - - # Put the result into a color plot - Z = Z.reshape(XX.shape) - plt.figure(fignum, figsize=(4, 3)) - plt.pcolormesh(XX, YY, Z > 0, cmap=plt.cm.Paired) - plt.contour(XX, YY, Z, colors=['k', 'k', 'k'], linestyles=['--', '-', '--'], - levels=[-.5, 0, .5]) - - plt.xlim(x_min, x_max) - plt.ylim(y_min, y_max) - - plt.xticks(()) - plt.yticks(()) - fignum = fignum + 1 + if support_vectors: + # Plot bigger circles around samples that serve as support vectors + ax.scatter( + clf.support_vectors_[:, 0], + clf.support_vectors_[:, 1], + s=150, + facecolors="none", + edgecolors="k", + ) + + # Plot samples by color and add legend + ax.scatter(X[:, 0], X[:, 1], c=y, s=30, edgecolors="k") + ax.legend(*scatter.legend_elements(), loc="upper right", title="Classes") + if long_title: + ax.set_title(f" Decision boundaries of {kernel} kernel in SVC") + else: + ax.set_title(kernel) + + if ax is None: + plt.show() + + +# %% +# Linear kernel +# ************* +# Linear kernel is the dot product of the input samples: +# +# .. math:: K(\mathbf{x}_1, \mathbf{x}_2) = \mathbf{x}_1^\top \mathbf{x}_2 +# +# It is then applied to any combination of two data points (samples) in the +# dataset. The dot product of the two points determines the +# :func:`~sklearn.metrics.pairwise.cosine_similarity` between both points. The +# higher the value, the more similar the points are. +plot_training_data_with_decision_boundary("linear") + +# %% +# Training a :class:`~sklearn.svm.SVC` on a linear kernel results in an +# untransformed feature space, where the hyperplane and the margins are +# straight lines. Due to the lack of expressivity of the linear kernel, the +# trained classes do not perfectly capture the training data. +# +# Polynomial kernel +# ***************** +# The polynomial kernel changes the notion of similarity. The kernel function +# is defined as: +# +# .. math:: +# K(\mathbf{x}_1, \mathbf{x}_2) = (\gamma \cdot \ +# \mathbf{x}_1^\top\mathbf{x}_2 + r)^d +# +# where :math:`{d}` is the degree (`degree`) of the polynomial, :math:`{\gamma}` +# (`gamma`) controls the influence of each individual training sample on the +# decision boundary and :math:`{r}` is the bias term (`coef0`) that shifts the +# data up or down. Here, we use the default value for the degree of the +# polynomial in the kernel function (`degree=3`). When `coef0=0` (the default), +# the data is only transformed, but no additional dimension is added. Using a +# polynomial kernel is equivalent to creating +# :class:`~sklearn.preprocessing.PolynomialFeatures` and then fitting a +# :class:`~sklearn.svm.SVC` with a linear kernel on the transformed data, +# although this alternative approach would be computationally expensive for most +# datasets. +plot_training_data_with_decision_boundary("poly") + +# %% +# The polynomial kernel with `gamma=2` adapts well to the training data, +# causing the margins on both sides of the hyperplane to bend accordingly. +# +# RBF kernel +# ********** +# The radial basis function (RBF) kernel, also known as the Gaussian kernel, is +# the default kernel for Support Vector Machines in scikit-learn. It measures +# similarity between two data points in infinite dimensions and then approaches +# classification by majority vote. The kernel function is defined as: +# +# .. math:: +# K(\mathbf{x}_1, \mathbf{x}_2) = \exp\left(-\gamma \cdot +# {\|\mathbf{x}_1 - \mathbf{x}_2\|^2}\right) +# +# where :math:`{\gamma}` (`gamma`) controls the influence of each individual +# training sample on the decision boundary. +# +# The larger the euclidean distance between two points +# :math:`\|\mathbf{x}_1 - \mathbf{x}_2\|^2` +# the closer the kernel function is to zero. This means that two points far away +# are more likely to be dissimilar. +plot_training_data_with_decision_boundary("rbf") + +# %% +# In the plot we can see how the decision boundaries tend to contract around +# data points that are close to each other. +# +# Sigmoid kernel +# ************** +# The sigmoid kernel function is defined as: +# +# .. math:: +# K(\mathbf{x}_1, \mathbf{x}_2) = \tanh(\gamma \cdot +# \mathbf{x}_1^\top\mathbf{x}_2 + r) +# +# where the kernel coefficient :math:`{\gamma}` (`gamma`) controls the influence +# of each individual training sample on the decision boundary and :math:`{r}` is +# the bias term (`coef0`) that shifts the data up or down. +# +# In the sigmoid kernel, the similarity between two data points is computed +# using the hyperbolic tangent function (:math:`\tanh`). The kernel function +# scales and possibly shifts the dot product of the two points +# (:math:`\mathbf{x}_1` and :math:`\mathbf{x}_2`). +plot_training_data_with_decision_boundary("sigmoid") + +# %% +# We can see that the decision boundaries obtained with the sigmoid kernel +# appear curved and irregular. The decision boundary tries to separate the +# classes by fitting a sigmoid-shaped curve, resulting in a complex boundary +# that may not generalize well to unseen data. From this example it becomes +# obvious, that the sigmoid kernel has very specific use cases, when dealing +# with data that exhibits a sigmoidal shape. In this example, careful fine +# tuning might find more generalizable decision boundaries. Because of its +# specificity, the sigmoid kernel is less commonly used in practice compared to +# other kernels. +# +# Conclusion +# ---------- +# In this example, we have visualized the decision boundaries trained with the +# provided dataset. The plots serve as an intuitive demonstration of how +# different kernels utilize the training data to determine the classification +# boundaries. +# +# The hyperplanes and margins, although computed indirectly, can be imagined as +# planes in the transformed feature space. However, in the plots, they are +# represented relative to the original feature space, resulting in curved +# decision boundaries for the polynomial, RBF, and sigmoid kernels. +# +# Please note that the plots do not evaluate the individual kernel's accuracy or +# quality. They are intended to provide a visual understanding of how the +# different kernels use the training data. +# +# For a comprehensive evaluation, fine-tuning of :class:`~sklearn.svm.SVC` +# parameters using techniques such as +# :class:`~sklearn.model_selection.GridSearchCV` is recommended to capture the +# underlying structures within the data. + +# %% +# XOR dataset +# ----------- +# A classical example of a dataset which is not linearly separable is the XOR +# pattern. HEre we demonstrate how different kernels work on such a dataset. + +xx, yy = np.meshgrid(np.linspace(-3, 3, 500), np.linspace(-3, 3, 500)) +np.random.seed(0) +X = np.random.randn(300, 2) +y = np.logical_xor(X[:, 0] > 0, X[:, 1] > 0) + +_, ax = plt.subplots(2, 2, figsize=(8, 8)) +args = dict(long_title=False, support_vectors=False) +plot_training_data_with_decision_boundary("linear", ax[0, 0], **args) +plot_training_data_with_decision_boundary("poly", ax[0, 1], **args) +plot_training_data_with_decision_boundary("rbf", ax[1, 0], **args) +plot_training_data_with_decision_boundary("sigmoid", ax[1, 1], **args) plt.show() + +# %% +# As you can see from the plots above, only the `rbf` kernel can find a +# reasonable decision boundary for the above dataset. diff --git a/examples/svm/plot_svm_margin.py b/examples/svm/plot_svm_margin.py index 5b267957677f8..f38858bb714a9 100644 --- a/examples/svm/plot_svm_margin.py +++ b/examples/svm/plot_svm_margin.py @@ -1,6 +1,3 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - """ ========================================================= SVM Margins Example @@ -15,16 +12,13 @@ the margins to be calculated using all the data in the area. """ -print(__doc__) - -# Code source: GaÃĢl Varoquaux -# Modified for documentation by Jaques Grobler -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt -from matplotlib import cm +import numpy as np + from sklearn import svm # we create 40 separable points @@ -36,9 +30,8 @@ fignum = 1 # fit the model -for name, penalty in (('unreg', 1), ('reg', 0.05)): - - clf = svm.SVC(kernel='linear', C=penalty) +for name, penalty in (("unreg", 1), ("reg", 0.05)): + clf = svm.SVC(kernel="linear", C=penalty) clf.fit(X, Y) # get the separating hyperplane @@ -51,24 +44,30 @@ # support vectors (margin away from hyperplane in direction # perpendicular to hyperplane). This is sqrt(1+a^2) away vertically in # 2-d. - margin = 1 / np.sqrt(np.sum(clf.coef_ ** 2)) - yy_down = yy - np.sqrt(1 + a ** 2) * margin - yy_up = yy + np.sqrt(1 + a ** 2) * margin + margin = 1 / np.sqrt(np.sum(clf.coef_**2)) + yy_down = yy - np.sqrt(1 + a**2) * margin + yy_up = yy + np.sqrt(1 + a**2) * margin # plot the line, the points, and the nearest vectors to the plane plt.figure(fignum, figsize=(4, 3)) plt.clf() - plt.plot(xx, yy, 'k-') - plt.plot(xx, yy_down, 'k--') - plt.plot(xx, yy_up, 'k--') - - plt.scatter(clf.support_vectors_[:, 0], clf.support_vectors_[:, 1], s=80, - facecolors='none', zorder=10, edgecolors='k', - cmap=cm.get_cmap('RdBu')) - plt.scatter(X[:, 0], X[:, 1], c=Y, zorder=10, cmap=cm.get_cmap('RdBu'), - edgecolors='k') - - plt.axis('tight') + plt.plot(xx, yy, "k-") + plt.plot(xx, yy_down, "k--") + plt.plot(xx, yy_up, "k--") + + plt.scatter( + clf.support_vectors_[:, 0], + clf.support_vectors_[:, 1], + s=80, + facecolors="none", + zorder=10, + edgecolors="k", + ) + plt.scatter( + X[:, 0], X[:, 1], c=Y, zorder=10, cmap=plt.get_cmap("RdBu"), edgecolors="k" + ) + + plt.axis("tight") x_min = -4.8 x_max = 4.2 y_min = -6 @@ -79,8 +78,7 @@ Z = clf.decision_function(xy).reshape(XX.shape) # Put the result into a contour plot - plt.contourf(XX, YY, Z, cmap=cm.get_cmap('RdBu'), - alpha=0.5, linestyles=['-']) + plt.contourf(XX, YY, Z, cmap=plt.get_cmap("RdBu"), alpha=0.5, linestyles=["-"]) plt.xlim(x_min, x_max) plt.ylim(y_min, y_max) diff --git a/examples/svm/plot_svm_nonlinear.py b/examples/svm/plot_svm_nonlinear.py deleted file mode 100644 index 47575d992a63b..0000000000000 --- a/examples/svm/plot_svm_nonlinear.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -============== -Non-linear SVM -============== - -Perform binary classification using non-linear SVC -with RBF kernel. The target to predict is a XOR of the -inputs. - -The color map illustrates the decision function learned by the SVC. -""" -print(__doc__) - -import numpy as np -import matplotlib.pyplot as plt -from sklearn import svm - -xx, yy = np.meshgrid(np.linspace(-3, 3, 500), - np.linspace(-3, 3, 500)) -np.random.seed(0) -X = np.random.randn(300, 2) -Y = np.logical_xor(X[:, 0] > 0, X[:, 1] > 0) - -# fit the model -clf = svm.NuSVC(gamma='auto') -clf.fit(X, Y) - -# plot the decision function for each datapoint on the grid -Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()]) -Z = Z.reshape(xx.shape) - -plt.imshow(Z, interpolation='nearest', - extent=(xx.min(), xx.max(), yy.min(), yy.max()), aspect='auto', - origin='lower', cmap=plt.cm.PuOr_r) -contours = plt.contour(xx, yy, Z, levels=[0], linewidths=2, - linestyles='dashed') -plt.scatter(X[:, 0], X[:, 1], s=30, c=Y, cmap=plt.cm.Paired, - edgecolors='k') -plt.xticks(()) -plt.yticks(()) -plt.axis([-3, 3, -3, 3]) -plt.show() diff --git a/examples/svm/plot_svm_regression.py b/examples/svm/plot_svm_regression.py index a91b588a15f63..5da00ef1f88b7 100644 --- a/examples/svm/plot_svm_regression.py +++ b/examples/svm/plot_svm_regression.py @@ -6,51 +6,74 @@ Toy example of 1D regression using linear, polynomial and RBF kernels. """ -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import matplotlib.pyplot as plt import numpy as np + from sklearn.svm import SVR -import matplotlib.pyplot as plt -# ############################################################################# +# %% # Generate sample data +# -------------------- X = np.sort(5 * np.random.rand(40, 1), axis=0) y = np.sin(X).ravel() -# ############################################################################# -# Add noise to targets +# add noise to targets y[::5] += 3 * (0.5 - np.random.rand(8)) -# ############################################################################# +# %% # Fit regression model -svr_rbf = SVR(kernel='rbf', C=100, gamma=0.1, epsilon=.1) -svr_lin = SVR(kernel='linear', C=100, gamma='auto') -svr_poly = SVR(kernel='poly', C=100, gamma='auto', degree=3, epsilon=.1, - coef0=1) +# -------------------- +svr_rbf = SVR(kernel="rbf", C=100, gamma=0.1, epsilon=0.1) +svr_lin = SVR(kernel="linear", C=100, gamma="auto") +svr_poly = SVR(kernel="poly", C=100, gamma="auto", degree=3, epsilon=0.1, coef0=1) -# ############################################################################# +# %% # Look at the results +# ------------------- lw = 2 svrs = [svr_rbf, svr_lin, svr_poly] -kernel_label = ['RBF', 'Linear', 'Polynomial'] -model_color = ['m', 'c', 'g'] +kernel_label = ["RBF", "Linear", "Polynomial"] +model_color = ["m", "c", "g"] fig, axes = plt.subplots(nrows=1, ncols=3, figsize=(15, 10), sharey=True) for ix, svr in enumerate(svrs): - axes[ix].plot(X, svr.fit(X, y).predict(X), color=model_color[ix], lw=lw, - label='{} model'.format(kernel_label[ix])) - axes[ix].scatter(X[svr.support_], y[svr.support_], facecolor="none", - edgecolor=model_color[ix], s=50, - label='{} support vectors'.format(kernel_label[ix])) - axes[ix].scatter(X[np.setdiff1d(np.arange(len(X)), svr.support_)], - y[np.setdiff1d(np.arange(len(X)), svr.support_)], - facecolor="none", edgecolor="k", s=50, - label='other training data') - axes[ix].legend(loc='upper center', bbox_to_anchor=(0.5, 1.1), - ncol=1, fancybox=True, shadow=True) - -fig.text(0.5, 0.04, 'data', ha='center', va='center') -fig.text(0.06, 0.5, 'target', ha='center', va='center', rotation='vertical') + axes[ix].plot( + X, + svr.fit(X, y).predict(X), + color=model_color[ix], + lw=lw, + label="{} model".format(kernel_label[ix]), + ) + axes[ix].scatter( + X[svr.support_], + y[svr.support_], + facecolor="none", + edgecolor=model_color[ix], + s=50, + label="{} support vectors".format(kernel_label[ix]), + ) + axes[ix].scatter( + X[np.setdiff1d(np.arange(len(X)), svr.support_)], + y[np.setdiff1d(np.arange(len(X)), svr.support_)], + facecolor="none", + edgecolor="k", + s=50, + label="other training data", + ) + axes[ix].legend( + loc="upper center", + bbox_to_anchor=(0.5, 1.1), + ncol=1, + fancybox=True, + shadow=True, + ) + +fig.text(0.5, 0.04, "data", ha="center", va="center") +fig.text(0.06, 0.5, "target", ha="center", va="center", rotation="vertical") fig.suptitle("Support Vector Regression", fontsize=14) plt.show() diff --git a/examples/svm/plot_svm_scale_c.py b/examples/svm/plot_svm_scale_c.py index 5208519172824..09cde25983ba1 100644 --- a/examples/svm/plot_svm_scale_c.py +++ b/examples/svm/plot_svm_scale_c.py @@ -3,9 +3,8 @@ Scaling the regularization parameter for SVCs ============================================== -The following example illustrates the effect of scaling the -regularization parameter when using :ref:`svm` for -:ref:`classification `. +The following example illustrates the effect of scaling the regularization +parameter when using :ref:`svm` for :ref:`classification `. For SVC classification, we are interested in a risk minimization for the equation: @@ -16,135 +15,194 @@ where - - :math:`C` is used to set the amount of regularization - - :math:`\mathcal{L}` is a `loss` function of our samples - and our model parameters. - - :math:`\Omega` is a `penalty` function of our model parameters - -If we consider the loss function to be the individual error per -sample, then the data-fit term, or the sum of the error for each sample, will -increase as we add more samples. The penalization term, however, will not -increase. - -When using, for example, :ref:`cross validation `, to -set the amount of regularization with `C`, there will be a -different amount of samples between the main problem and the smaller problems -within the folds of the cross validation. - -Since our loss function is dependent on the amount of samples, the latter -will influence the selected value of `C`. -The question that arises is `How do we optimally adjust C to -account for the different amount of training samples?` - -The figures below are used to illustrate the effect of scaling our -`C` to compensate for the change in the number of samples, in the -case of using an `l1` penalty, as well as the `l2` penalty. - -l1-penalty case ------------------ -In the `l1` case, theory says that prediction consistency -(i.e. that under given hypothesis, the estimator -learned predicts as well as a model knowing the true distribution) -is not possible because of the bias of the `l1`. It does say, however, -that model consistency, in terms of finding the right set of non-zero -parameters as well as their signs, can be achieved by scaling -`C1`. - -l2-penalty case ------------------ -The theory says that in order to achieve prediction consistency, the -penalty parameter should be kept constant -as the number of samples grow. - -Simulations ------------- - -The two figures below plot the values of `C` on the `x-axis` and the -corresponding cross-validation scores on the `y-axis`, for several different -fractions of a generated data-set. - -In the `l1` penalty case, the cross-validation-error correlates best with -the test-error, when scaling our `C` with the number of samples, `n`, -which can be seen in the first figure. - -For the `l2` penalty case, the best result comes from the case where `C` -is not scaled. - -.. topic:: Note: - - Two separate datasets are used for the two different plots. The reason - behind this is the `l1` case works better on sparse data, while `l2` - is better suited to the non-sparse case. +- :math:`C` is used to set the amount of regularization +- :math:`\mathcal{L}` is a `loss` function of our samples and our model parameters. +- :math:`\Omega` is a `penalty` function of our model parameters + +If we consider the loss function to be the individual error per sample, then the +data-fit term, or the sum of the error for each sample, increases as we add more +samples. The penalization term, however, does not increase. + +When using, for example, :ref:`cross validation `, to set the +amount of regularization with `C`, there would be a different amount of samples +between the main problem and the smaller problems within the folds of the cross +validation. + +Since the loss function dependens on the amount of samples, the latter +influences the selected value of `C`. The question that arises is "How do we +optimally adjust C to account for the different amount of training samples?" """ -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Data generation +# --------------- +# +# In this example we investigate the effect of reparametrizing the regularization +# parameter `C` to account for the number of samples when using either L1 or L2 +# penalty. For such purpose we create a synthetic dataset with a large number of +# features, out of which only a few are informative. We therefore expect the +# regularization to shrink the coefficients towards zero (L2 penalty) or exactly +# zero (L1 penalty). + +from sklearn.datasets import make_classification + +n_samples, n_features = 100, 300 +X, y = make_classification( + n_samples=n_samples, n_features=n_features, n_informative=5, random_state=1 +) + +# %% +# L1-penalty case +# --------------- +# In the L1 case, theory says that provided a strong regularization, the +# estimator cannot predict as well as a model knowing the true distribution +# (even in the limit where the sample size grows to infinity) as it may set some +# weights of otherwise predictive features to zero, which induces a bias. It does +# say, however, that it is possible to find the right set of non-zero parameters +# as well as their signs by tuning `C`. +# +# We define a linear SVC with the L1 penalty. -# Author: Andreas Mueller -# Jaques Grobler -# License: BSD 3 clause +from sklearn.svm import LinearSVC +model_l1 = LinearSVC(penalty="l1", loss="squared_hinge", dual=False, tol=1e-3) + +# %% +# We compute the mean test score for different values of `C` via +# cross-validation. import numpy as np +import pandas as pd + +from sklearn.model_selection import ShuffleSplit, validation_curve + +Cs = np.logspace(-2.3, -1.3, 10) +train_sizes = np.linspace(0.3, 0.7, 3) +labels = [f"fraction: {train_size}" for train_size in train_sizes] +shuffle_params = { + "test_size": 0.3, + "n_splits": 150, + "random_state": 1, +} + +results = {"C": Cs} +for label, train_size in zip(labels, train_sizes): + cv = ShuffleSplit(train_size=train_size, **shuffle_params) + train_scores, test_scores = validation_curve( + model_l1, + X, + y, + param_name="C", + param_range=Cs, + cv=cv, + n_jobs=2, + ) + results[label] = test_scores.mean(axis=1) +results = pd.DataFrame(results) + +# %% import matplotlib.pyplot as plt -from sklearn.svm import LinearSVC -from sklearn.model_selection import ShuffleSplit -from sklearn.model_selection import GridSearchCV -from sklearn.utils import check_random_state -from sklearn import datasets - -rnd = check_random_state(1) - -# set up dataset -n_samples = 100 -n_features = 300 - -# l1 data (only 5 informative features) -X_1, y_1 = datasets.make_classification(n_samples=n_samples, - n_features=n_features, n_informative=5, - random_state=1) - -# l2 data: non sparse, but less features -y_2 = np.sign(.5 - rnd.rand(n_samples)) -X_2 = rnd.randn(n_samples, n_features // 5) + y_2[:, np.newaxis] -X_2 += 5 * rnd.randn(n_samples, n_features // 5) - -clf_sets = [(LinearSVC(penalty='l1', loss='squared_hinge', dual=False, - tol=1e-3), - np.logspace(-2.3, -1.3, 10), X_1, y_1), - (LinearSVC(penalty='l2', loss='squared_hinge', dual=True), - np.logspace(-4.5, -2, 10), X_2, y_2)] - -colors = ['navy', 'cyan', 'darkorange'] -lw = 2 - -for clf, cs, X, y in clf_sets: - # set up the plot for each regressor - fig, axes = plt.subplots(nrows=2, sharey=True, figsize=(9, 10)) - - for k, train_size in enumerate(np.linspace(0.3, 0.7, 3)[::-1]): - param_grid = dict(C=cs) - # To get nice curve, we need a large number of iterations to - # reduce the variance - grid = GridSearchCV(clf, refit=False, param_grid=param_grid, - cv=ShuffleSplit(train_size=train_size, - test_size=.3, - n_splits=250, random_state=1)) - grid.fit(X, y) - scores = grid.cv_results_['mean_test_score'] - - scales = [(1, 'No scaling'), - ((n_samples * train_size), '1/n_samples'), - ] - - for ax, (scaler, name) in zip(axes, scales): - ax.set_xlabel('C') - ax.set_ylabel('CV Score') - grid_cs = cs * float(scaler) # scale the C's - ax.semilogx(grid_cs, scores, label="fraction %.2f" % - train_size, color=colors[k], lw=lw) - ax.set_title('scaling=%s, penalty=%s, loss=%s' % - (name, clf.penalty, clf.loss)) - - plt.legend(loc="best") +fig, axes = plt.subplots(nrows=1, ncols=2, sharey=True, figsize=(12, 6)) + +# plot results without scaling C +results.plot(x="C", ax=axes[0], logx=True) +axes[0].set_ylabel("CV score") +axes[0].set_title("No scaling") + +for label in labels: + best_C = results.loc[results[label].idxmax(), "C"] + axes[0].axvline(x=best_C, linestyle="--", color="grey", alpha=0.7) + +# plot results by scaling C +for train_size_idx, label in enumerate(labels): + train_size = train_sizes[train_size_idx] + results_scaled = results[[label]].assign( + C_scaled=Cs * float(n_samples * np.sqrt(train_size)) + ) + results_scaled.plot(x="C_scaled", ax=axes[1], logx=True, label=label) + best_C_scaled = results_scaled["C_scaled"].loc[results[label].idxmax()] + axes[1].axvline(x=best_C_scaled, linestyle="--", color="grey", alpha=0.7) + +axes[1].set_title("Scaling C by sqrt(1 / n_samples)") + +_ = fig.suptitle("Effect of scaling C with L1 penalty") + +# %% +# In the region of small `C` (strong regularization) all the coefficients +# learned by the models are zero, leading to severe underfitting. Indeed, the +# accuracy in this region is at the chance level. +# +# Using the default scale results in a somewhat stable optimal value of `C`, +# whereas the transition out of the underfitting region depends on the number of +# training samples. The reparametrization leads to even more stable results. +# +# See e.g. theorem 3 of :arxiv:`On the prediction performance of the Lasso +# <1402.1700>` or :arxiv:`Simultaneous analysis of Lasso and Dantzig selector +# <0801.1095>` where the regularization parameter is always assumed to be +# proportional to 1 / sqrt(n_samples). +# +# L2-penalty case +# --------------- +# We can do a similar experiment with the L2 penalty. In this case, the +# theory says that in order to achieve prediction consistency, the penalty +# parameter should be kept constant as the number of samples grow. + +model_l2 = LinearSVC(penalty="l2", loss="squared_hinge", dual=True) +Cs = np.logspace(-8, 4, 11) + +labels = [f"fraction: {train_size}" for train_size in train_sizes] +results = {"C": Cs} +for label, train_size in zip(labels, train_sizes): + cv = ShuffleSplit(train_size=train_size, **shuffle_params) + train_scores, test_scores = validation_curve( + model_l2, + X, + y, + param_name="C", + param_range=Cs, + cv=cv, + n_jobs=2, + ) + results[label] = test_scores.mean(axis=1) +results = pd.DataFrame(results) + +# %% +import matplotlib.pyplot as plt + +fig, axes = plt.subplots(nrows=1, ncols=2, sharey=True, figsize=(12, 6)) + +# plot results without scaling C +results.plot(x="C", ax=axes[0], logx=True) +axes[0].set_ylabel("CV score") +axes[0].set_title("No scaling") + +for label in labels: + best_C = results.loc[results[label].idxmax(), "C"] + axes[0].axvline(x=best_C, linestyle="--", color="grey", alpha=0.8) + +# plot results by scaling C +for train_size_idx, label in enumerate(labels): + results_scaled = results[[label]].assign( + C_scaled=Cs * float(n_samples * np.sqrt(train_sizes[train_size_idx])) + ) + results_scaled.plot(x="C_scaled", ax=axes[1], logx=True, label=label) + best_C_scaled = results_scaled["C_scaled"].loc[results[label].idxmax()] + axes[1].axvline(x=best_C_scaled, linestyle="--", color="grey", alpha=0.8) +axes[1].set_title("Scaling C by sqrt(1 / n_samples)") + +fig.suptitle("Effect of scaling C with L2 penalty") plt.show() + +# %% +# For the L2 penalty case, the reparametrization seems to have a smaller impact +# on the stability of the optimal value for the regularization. The transition +# out of the overfitting region occurs in a more spread range and the accuracy +# does not seem to be degraded up to chance level. +# +# Try increasing the value to `n_splits=1_000` for better results in the L2 +# case, which is not shown here due to the limitations on the documentation +# builder. diff --git a/examples/svm/plot_svm_tie_breaking.py b/examples/svm/plot_svm_tie_breaking.py index 76eabfa1e35be..b5f4fb8dd18c3 100644 --- a/examples/svm/plot_svm_tie_breaking.py +++ b/examples/svm/plot_svm_tie_breaking.py @@ -11,29 +11,27 @@ tied. If ``break_ties=False``, all input in that area would be classified as one class, whereas if ``break_ties=True``, the tie-breaking mechanism will create a non-convex decision boundary in that area. -""" -print(__doc__) - -# Code source: Andreas Mueller, Adrin Jalali -# License: BSD 3 clause +""" +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import matplotlib.pyplot as plt -from sklearn.svm import SVC +import numpy as np + from sklearn.datasets import make_blobs +from sklearn.svm import SVC X, y = make_blobs(random_state=27) fig, sub = plt.subplots(2, 1, figsize=(5, 8)) -titles = ("break_ties = False", - "break_ties = True") +titles = ("break_ties = False", "break_ties = True") for break_ties, title, ax in zip((False, True), titles, sub.flatten()): - - svm = SVC(kernel="linear", C=1, break_ties=break_ties, - decision_function_shape='ovr').fit(X, y) + svm = SVC( + kernel="linear", C=1, break_ties=break_ties, decision_function_shape="ovr" + ).fit(X, y) xlim = [X[:, 0].min(), X[:, 0].max()] ylim = [X[:, 1].min(), X[:, 1].max()] @@ -49,8 +47,12 @@ points = ax.scatter(X[:, 0], X[:, 1], c=y, cmap="Accent") classes = [(0, 1), (0, 2), (1, 2)] line = np.linspace(X[:, 1].min() - 5, X[:, 1].max() + 5) - ax.imshow(-pred.reshape(xx.shape), cmap="Accent", alpha=.2, - extent=(xlim[0], xlim[1], ylim[1], ylim[0])) + ax.imshow( + -pred.reshape(xx.shape), + cmap="Accent", + alpha=0.2, + extent=(xlim[0], xlim[1], ylim[1], ylim[0]), + ) for coef, intercept, col in zip(svm.coef_, svm.intercept_, classes): line2 = -(line * coef[1] + intercept) / coef[0] diff --git a/examples/svm/plot_weighted_samples.py b/examples/svm/plot_weighted_samples.py index 0549da7a38084..6bdcbf24af8d6 100644 --- a/examples/svm/plot_weighted_samples.py +++ b/examples/svm/plot_weighted_samples.py @@ -9,55 +9,81 @@ The sample weighting rescales the C parameter, which means that the classifier puts more emphasis on getting these points right. The effect might often be subtle. -To emphasize the effect here, we particularly weight outliers, making the -deformation of the decision boundary very visible. +To emphasize the effect here, we particularly increase the weight of the positive +class, making the deformation of the decision boundary more visible. + """ -print(__doc__) -import numpy as np -import matplotlib.pyplot as plt -from sklearn import svm +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +import matplotlib.pyplot as plt +import numpy as np -def plot_decision_function(classifier, sample_weight, axis, title): - # plot the decision function - xx, yy = np.meshgrid(np.linspace(-4, 5, 500), np.linspace(-4, 5, 500)) +from sklearn.datasets import make_classification +from sklearn.inspection import DecisionBoundaryDisplay +from sklearn.svm import SVC - Z = classifier.decision_function(np.c_[xx.ravel(), yy.ravel()]) - Z = Z.reshape(xx.shape) +X, y = make_classification( + n_samples=1_000, + n_features=2, + n_informative=2, + n_redundant=0, + n_clusters_per_class=1, + class_sep=1.1, + weights=[0.9, 0.1], + random_state=0, +) +# down-sample for plotting +rng = np.random.RandomState(0) +plot_indices = rng.choice(np.arange(X.shape[0]), size=100, replace=True) +X_plot, y_plot = X[plot_indices], y[plot_indices] - # plot the line, the points, and the nearest vectors to the plane - axis.contourf(xx, yy, Z, alpha=0.75, cmap=plt.cm.bone) - axis.scatter(X[:, 0], X[:, 1], c=y, s=100 * sample_weight, alpha=0.9, - cmap=plt.cm.bone, edgecolors='black') - axis.axis('off') +def plot_decision_function(classifier, sample_weight, axis, title): + """Plot the synthetic data and the classifier decision function. Points with + larger sample_weight are mapped to larger circles in the scatter plot.""" + axis.scatter( + X_plot[:, 0], + X_plot[:, 1], + c=y_plot, + s=100 * sample_weight[plot_indices], + alpha=0.9, + cmap=plt.cm.bone, + edgecolors="black", + ) + DecisionBoundaryDisplay.from_estimator( + classifier, + X_plot, + response_method="decision_function", + alpha=0.75, + ax=axis, + cmap=plt.cm.bone, + ) + axis.axis("off") axis.set_title(title) -# we create 20 points -np.random.seed(0) -X = np.r_[np.random.randn(10, 2) + [1, 1], np.random.randn(10, 2)] -y = [1] * 10 + [-1] * 10 -sample_weight_last_ten = abs(np.random.randn(len(X))) +# we define constant weights as expected by the plotting function sample_weight_constant = np.ones(len(X)) -# and bigger weights to some outliers -sample_weight_last_ten[15:] *= 5 -sample_weight_last_ten[9] *= 15 - -# for reference, first fit without sample weights +# assign random weights to all points +sample_weight_modified = abs(rng.randn(len(X))) +# assign bigger weights to the positive class +positive_class_indices = np.asarray(y == 1).nonzero()[0] +sample_weight_modified[positive_class_indices] *= 15 -# fit the model -clf_weights = svm.SVC(gamma=1) -clf_weights.fit(X, y, sample_weight=sample_weight_last_ten) - -clf_no_weights = svm.SVC(gamma=1) +# This model does not include sample weights. +clf_no_weights = SVC(gamma=1) clf_no_weights.fit(X, y) +# This other model includes sample weights. +clf_weights = SVC(gamma=1) +clf_weights.fit(X, y, sample_weight=sample_weight_modified) + fig, axes = plt.subplots(1, 2, figsize=(14, 6)) -plot_decision_function(clf_no_weights, sample_weight_constant, axes[0], - "Constant weights") -plot_decision_function(clf_weights, sample_weight_last_ten, axes[1], - "Modified weights") +plot_decision_function( + clf_no_weights, sample_weight_constant, axes[0], "Constant weights" +) +plot_decision_function(clf_weights, sample_weight_modified, axes[1], "Modified weights") plt.show() diff --git a/examples/text/plot_document_classification_20newsgroups.py b/examples/text/plot_document_classification_20newsgroups.py index 1fe4d1624e8b8..aa80b7c1b630b 100644 --- a/examples/text/plot_document_classification_20newsgroups.py +++ b/examples/text/plot_document_classification_20newsgroups.py @@ -3,326 +3,448 @@ Classification of text documents using sparse features ====================================================== -This is an example showing how scikit-learn can be used to classify documents -by topics using a bag-of-words approach. This example uses a scipy.sparse -matrix to store the features and demonstrates various classifiers that can -efficiently handle sparse matrices. +This is an example showing how scikit-learn can be used to classify documents by +topics using a `Bag of Words approach +`_. This example uses a +Tf-idf-weighted document-term sparse matrix to encode the features and +demonstrates various classifiers that can efficiently handle sparse matrices. -The dataset used in this example is the 20 newsgroups dataset. It will be -automatically downloaded, then cached. +For document analysis via an unsupervised learning approach, see the example +script :ref:`sphx_glr_auto_examples_text_plot_document_clustering.py`. """ -# Author: Peter Prettenhofer -# Olivier Grisel -# Mathieu Blondel -# Lars Buitinck -# License: BSD 3 clause -import logging -import numpy as np -from optparse import OptionParser -import sys +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + + +# %% +# Loading and vectorizing the 20 newsgroups text dataset +# ====================================================== +# +# We define a function to load data from :ref:`20newsgroups_dataset`, which +# comprises around 18,000 newsgroups posts on 20 topics split in two subsets: +# one for training (or development) and the other one for testing (or for +# performance evaluation). Note that, by default, the text samples contain some +# message metadata such as `'headers'`, `'footers'` (signatures) and `'quotes'` +# to other posts. The `fetch_20newsgroups` function therefore accepts a +# parameter named `remove` to attempt stripping such information that can make +# the classification problem "too easy". This is achieved using simple +# heuristics that are neither perfect nor standard, hence disabled by default. + from time import time -import matplotlib.pyplot as plt from sklearn.datasets import fetch_20newsgroups from sklearn.feature_extraction.text import TfidfVectorizer -from sklearn.feature_extraction.text import HashingVectorizer -from sklearn.feature_selection import SelectFromModel -from sklearn.feature_selection import SelectKBest, chi2 -from sklearn.linear_model import RidgeClassifier -from sklearn.pipeline import Pipeline -from sklearn.svm import LinearSVC -from sklearn.linear_model import SGDClassifier -from sklearn.linear_model import Perceptron -from sklearn.linear_model import PassiveAggressiveClassifier -from sklearn.naive_bayes import BernoulliNB, ComplementNB, MultinomialNB -from sklearn.neighbors import KNeighborsClassifier -from sklearn.neighbors import NearestCentroid -from sklearn.ensemble import RandomForestClassifier -from sklearn.utils.extmath import density -from sklearn import metrics + +categories = [ + "alt.atheism", + "talk.religion.misc", + "comp.graphics", + "sci.space", +] -# Display progress logs on stdout -logging.basicConfig(level=logging.INFO, - format='%(asctime)s %(levelname)s %(message)s') - -op = OptionParser() -op.add_option("--report", - action="store_true", dest="print_report", - help="Print a detailed classification report.") -op.add_option("--chi2_select", - action="store", type="int", dest="select_chi2", - help="Select some number of features using a chi-squared test") -op.add_option("--confusion_matrix", - action="store_true", dest="print_cm", - help="Print the confusion matrix.") -op.add_option("--top10", - action="store_true", dest="print_top10", - help="Print ten most discriminative terms per class" - " for every classifier.") -op.add_option("--all_categories", - action="store_true", dest="all_categories", - help="Whether to use all categories or not.") -op.add_option("--use_hashing", - action="store_true", - help="Use a hashing vectorizer.") -op.add_option("--n_features", - action="store", type=int, default=2 ** 16, - help="n_features when using the hashing vectorizer.") -op.add_option("--filtered", - action="store_true", - help="Remove newsgroup information that is easily overfit: " - "headers, signatures, and quoting.") - - -def is_interactive(): - return not hasattr(sys.modules['__main__'], '__file__') - - -# work-around for Jupyter notebook and IPython console -argv = [] if is_interactive() else sys.argv[1:] -(opts, args) = op.parse_args(argv) -if len(args) > 0: - op.error("this script takes no arguments.") - sys.exit(1) - -print(__doc__) -op.print_help() -print() +def size_mb(docs): + return sum(len(s.encode("utf-8")) for s in docs) / 1e6 -# %% -# Load data from the training set -# ------------------------------------ -# Let's load data from the newsgroups dataset which comprises around 18000 -# newsgroups posts on 20 topics split in two subsets: one for training (or -# development) and the other one for testing (or for performance evaluation). -if opts.all_categories: - categories = None -else: - categories = [ - 'alt.atheism', - 'talk.religion.misc', - 'comp.graphics', - 'sci.space', - ] - -if opts.filtered: - remove = ('headers', 'footers', 'quotes') -else: - remove = () - -print("Loading 20 newsgroups dataset for categories:") -print(categories if categories else "all") - -data_train = fetch_20newsgroups(subset='train', categories=categories, - shuffle=True, random_state=42, - remove=remove) - -data_test = fetch_20newsgroups(subset='test', categories=categories, - shuffle=True, random_state=42, - remove=remove) -print('data loaded') - -# order of labels in `target_names` can be different from `categories` -target_names = data_train.target_names +def load_dataset(verbose=False, remove=()): + """Load and vectorize the 20 newsgroups dataset.""" + data_train = fetch_20newsgroups( + subset="train", + categories=categories, + shuffle=True, + random_state=42, + remove=remove, + ) -def size_mb(docs): - return sum(len(s.encode('utf-8')) for s in docs) / 1e6 - - -data_train_size_mb = size_mb(data_train.data) -data_test_size_mb = size_mb(data_test.data) - -print("%d documents - %0.3fMB (training set)" % ( - len(data_train.data), data_train_size_mb)) -print("%d documents - %0.3fMB (test set)" % ( - len(data_test.data), data_test_size_mb)) -print("%d categories" % len(target_names)) -print() - -# split a training set and a test set -y_train, y_test = data_train.target, data_test.target - -print("Extracting features from the training data using a sparse vectorizer") -t0 = time() -if opts.use_hashing: - vectorizer = HashingVectorizer(stop_words='english', alternate_sign=False, - n_features=opts.n_features) - X_train = vectorizer.transform(data_train.data) -else: - vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.5, - stop_words='english') + data_test = fetch_20newsgroups( + subset="test", + categories=categories, + shuffle=True, + random_state=42, + remove=remove, + ) + + # order of labels in `target_names` can be different from `categories` + target_names = data_train.target_names + + # split target in a training set and a test set + y_train, y_test = data_train.target, data_test.target + + # Extracting features from the training data using a sparse vectorizer + t0 = time() + vectorizer = TfidfVectorizer( + sublinear_tf=True, max_df=0.5, min_df=5, stop_words="english" + ) X_train = vectorizer.fit_transform(data_train.data) -duration = time() - t0 -print("done in %fs at %0.3fMB/s" % (duration, data_train_size_mb / duration)) -print("n_samples: %d, n_features: %d" % X_train.shape) -print() - -print("Extracting features from the test data using the same vectorizer") -t0 = time() -X_test = vectorizer.transform(data_test.data) -duration = time() - t0 -print("done in %fs at %0.3fMB/s" % (duration, data_test_size_mb / duration)) -print("n_samples: %d, n_features: %d" % X_test.shape) -print() - -# mapping from integer feature name to original token string -if opts.use_hashing: - feature_names = None -else: - feature_names = vectorizer.get_feature_names() - -if opts.select_chi2: - print("Extracting %d best features by a chi-squared test" % - opts.select_chi2) + duration_train = time() - t0 + + # Extracting features from the test data using the same vectorizer t0 = time() - ch2 = SelectKBest(chi2, k=opts.select_chi2) - X_train = ch2.fit_transform(X_train, y_train) - X_test = ch2.transform(X_test) - if feature_names: - # keep selected feature names - feature_names = [feature_names[i] for i - in ch2.get_support(indices=True)] - print("done in %fs" % (time() - t0)) - print() + X_test = vectorizer.transform(data_test.data) + duration_test = time() - t0 + + feature_names = vectorizer.get_feature_names_out() + + if verbose: + # compute size of loaded data + data_train_size_mb = size_mb(data_train.data) + data_test_size_mb = size_mb(data_test.data) + + print( + f"{len(data_train.data)} documents - " + f"{data_train_size_mb:.2f}MB (training set)" + ) + print(f"{len(data_test.data)} documents - {data_test_size_mb:.2f}MB (test set)") + print(f"{len(target_names)} categories") + print( + f"vectorize training done in {duration_train:.3f}s " + f"at {data_train_size_mb / duration_train:.3f}MB/s" + ) + print(f"n_samples: {X_train.shape[0]}, n_features: {X_train.shape[1]}") + print( + f"vectorize testing done in {duration_test:.3f}s " + f"at {data_test_size_mb / duration_test:.3f}MB/s" + ) + print(f"n_samples: {X_test.shape[0]}, n_features: {X_test.shape[1]}") + + return X_train, X_test, y_train, y_test, feature_names, target_names + -if feature_names: - feature_names = np.asarray(feature_names) +# %% +# Analysis of a bag-of-words document classifier +# ============================================== +# +# We will now train a classifier twice, once on the text samples including +# metadata and once after stripping the metadata. For both cases we will analyze +# the classification errors on a test set using a confusion matrix and inspect +# the coefficients that define the classification function of the trained +# models. +# +# Model without metadata stripping +# -------------------------------- +# +# We start by using the custom function `load_dataset` to load the data without +# metadata stripping. + +X_train, X_test, y_train, y_test, feature_names, target_names = load_dataset( + verbose=True +) +# %% +# Our first model is an instance of the +# :class:`~sklearn.linear_model.RidgeClassifier` class. This is a linear +# classification model that uses the mean squared error on {-1, 1} encoded +# targets, one for each possible class. Contrary to +# :class:`~sklearn.linear_model.LogisticRegression`, +# :class:`~sklearn.linear_model.RidgeClassifier` does not +# provide probabilistic predictions (no `predict_proba` method), +# but it is often faster to train. + +from sklearn.linear_model import RidgeClassifier + +clf = RidgeClassifier(tol=1e-2, solver="sparse_cg") +clf.fit(X_train, y_train) +pred = clf.predict(X_test) + +# %% +# We plot the confusion matrix of this classifier to find if there is a pattern +# in the classification errors. + +import matplotlib.pyplot as plt -def trim(s): - """Trim string to fit on terminal (assuming 80-column display)""" - return s if len(s) <= 80 else s[:77] + "..." +from sklearn.metrics import ConfusionMatrixDisplay +fig, ax = plt.subplots(figsize=(10, 5)) +ConfusionMatrixDisplay.from_predictions(y_test, pred, ax=ax) +ax.xaxis.set_ticklabels(target_names) +ax.yaxis.set_ticklabels(target_names) +_ = ax.set_title( + f"Confusion Matrix for {clf.__class__.__name__}\non the original documents" +) # %% -# Benchmark classifiers -# ------------------------------------ -# We train and test the datasets with 15 different classification models -# and get performance results for each model. -def benchmark(clf): - print('_' * 80) +# The confusion matrix highlights that documents of the `alt.atheism` class are +# often confused with documents with the class `talk.religion.misc` class and +# vice-versa which is expected since the topics are semantically related. +# +# We also observe that some documents of the `sci.space` class can be misclassified as +# `comp.graphics` while the converse is much rarer. A manual inspection of those +# badly classified documents would be required to get some insights on this +# asymmetry. It could be the case that the vocabulary of the space topic could +# be more specific than the vocabulary for computer graphics. +# +# We can gain a deeper understanding of how this classifier makes its decisions +# by looking at the words with the highest average feature effects: + +import numpy as np +import pandas as pd + + +def plot_feature_effects(): + # learned coefficients weighted by frequency of appearance + average_feature_effects = clf.coef_ * np.asarray(X_train.mean(axis=0)).ravel() + + for i, label in enumerate(target_names): + top5 = np.argsort(average_feature_effects[i])[-5:][::-1] + if i == 0: + top = pd.DataFrame(feature_names[top5], columns=[label]) + top_indices = top5 + else: + top[label] = feature_names[top5] + top_indices = np.concatenate((top_indices, top5), axis=None) + top_indices = np.unique(top_indices) + predictive_words = feature_names[top_indices] + + # plot feature effects + bar_size = 0.25 + padding = 0.75 + y_locs = np.arange(len(top_indices)) * (4 * bar_size + padding) + + fig, ax = plt.subplots(figsize=(10, 8)) + for i, label in enumerate(target_names): + ax.barh( + y_locs + (i - 2) * bar_size, + average_feature_effects[i, top_indices], + height=bar_size, + label=label, + ) + ax.set( + yticks=y_locs, + yticklabels=predictive_words, + ylim=[ + 0 - 4 * bar_size, + len(top_indices) * (4 * bar_size + padding) - 4 * bar_size, + ], + ) + ax.legend(loc="lower right") + + print("top 5 keywords per class:") + print(top) + + return ax + + +_ = plot_feature_effects().set_title("Average feature effect on the original data") + +# %% +# We can observe that the most predictive words are often strongly positively +# associated with a single class and negatively associated with all the other +# classes. Most of those positive associations are quite easy to interpret. +# However, some words such as `"god"` and `"people"` are positively associated to +# both `"talk.misc.religion"` and `"alt.atheism"` as those two classes expectedly +# share some common vocabulary. Notice however that there are also words such as +# `"christian"` and `"morality"` that are only positively associated with +# `"talk.misc.religion"`. Furthermore, in this version of the dataset, the word +# `"caltech"` is one of the top predictive features for atheism due to pollution +# in the dataset coming from some sort of metadata such as the email addresses +# of the sender of previous emails in the discussion as can be seen below: + +data_train = fetch_20newsgroups( + subset="train", categories=categories, shuffle=True, random_state=42 +) + +for doc in data_train.data: + if "caltech" in doc: + print(doc) + break + +# %% +# Such headers, signature footers (and quoted metadata from previous messages) +# can be considered side information that artificially reveals the newsgroup by +# identifying the registered members and one would rather want our text +# classifier to only learn from the "main content" of each text document instead +# of relying on the leaked identity of the writers. +# +# Model with metadata stripping +# ----------------------------- +# +# The `remove` option of the 20 newsgroups dataset loader in scikit-learn allows +# to heuristically attempt to filter out some of this unwanted metadata that +# makes the classification problem artificially easier. Be aware that such +# filtering of the text contents is far from perfect. +# +# Let us try to leverage this option to train a text classifier that does not +# rely too much on this kind of metadata to make its decisions: +( + X_train, + X_test, + y_train, + y_test, + feature_names, + target_names, +) = load_dataset(remove=("headers", "footers", "quotes")) + +clf = RidgeClassifier(tol=1e-2, solver="sparse_cg") +clf.fit(X_train, y_train) +pred = clf.predict(X_test) + +fig, ax = plt.subplots(figsize=(10, 5)) +ConfusionMatrixDisplay.from_predictions(y_test, pred, ax=ax) +ax.xaxis.set_ticklabels(target_names) +ax.yaxis.set_ticklabels(target_names) +_ = ax.set_title( + f"Confusion Matrix for {clf.__class__.__name__}\non filtered documents" +) + +# %% +# By looking at the confusion matrix, it is more evident that the scores of the +# model trained with metadata were over-optimistic. The classification problem +# without access to the metadata is less accurate but more representative of the +# intended text classification problem. + +_ = plot_feature_effects().set_title("Average feature effects on filtered documents") + +# %% +# In the next section we keep the dataset without metadata to compare several +# classifiers. + +# %% +# Benchmarking classifiers +# ======================== +# +# Scikit-learn provides many different kinds of classification algorithms. In +# this section we will train a selection of those classifiers on the same text +# classification problem and measure both their generalization performance +# (accuracy on the test set) and their computation performance (speed), both at +# training time and testing time. For such purpose we define the following +# benchmarking utilities: + +from sklearn import metrics +from sklearn.utils.extmath import density + + +def benchmark(clf, custom_name=False): + print("_" * 80) print("Training: ") print(clf) t0 = time() clf.fit(X_train, y_train) train_time = time() - t0 - print("train time: %0.3fs" % train_time) + print(f"train time: {train_time:.3}s") t0 = time() pred = clf.predict(X_test) test_time = time() - t0 - print("test time: %0.3fs" % test_time) + print(f"test time: {test_time:.3}s") score = metrics.accuracy_score(y_test, pred) - print("accuracy: %0.3f" % score) - - if hasattr(clf, 'coef_'): - print("dimensionality: %d" % clf.coef_.shape[1]) - print("density: %f" % density(clf.coef_)) + print(f"accuracy: {score:.3}") - if opts.print_top10 and feature_names is not None: - print("top 10 keywords per class:") - for i, label in enumerate(target_names): - top10 = np.argsort(clf.coef_[i])[-10:] - print(trim("%s: %s" % (label, " ".join(feature_names[top10])))) + if hasattr(clf, "coef_"): + print(f"dimensionality: {clf.coef_.shape[1]}") + print(f"density: {density(clf.coef_)}") print() - if opts.print_report: - print("classification report:") - print(metrics.classification_report(y_test, pred, - target_names=target_names)) - - if opts.print_cm: - print("confusion matrix:") - print(metrics.confusion_matrix(y_test, pred)) - print() - clf_descr = str(clf).split('(')[0] + if custom_name: + clf_descr = str(custom_name) + else: + clf_descr = clf.__class__.__name__ return clf_descr, score, train_time, test_time +# %% +# We now train and test the datasets with 8 different classification models and +# get performance results for each model. The goal of this study is to highlight +# the computation/accuracy tradeoffs of different types of classifiers for +# such a multi-class text classification problem. +# +# Notice that the most important hyperparameters values were tuned using a grid +# search procedure not shown in this notebook for the sake of simplicity. See +# the example script +# :ref:`sphx_glr_auto_examples_model_selection_plot_grid_search_text_feature_extraction.py` # noqa: E501 +# for a demo on how such tuning can be done. + +from sklearn.ensemble import RandomForestClassifier +from sklearn.linear_model import LogisticRegression, SGDClassifier +from sklearn.naive_bayes import ComplementNB +from sklearn.neighbors import KNeighborsClassifier, NearestCentroid +from sklearn.svm import LinearSVC + results = [] for clf, name in ( - (RidgeClassifier(tol=1e-2, solver="sag"), "Ridge Classifier"), - (Perceptron(max_iter=50), "Perceptron"), - (PassiveAggressiveClassifier(max_iter=50), - "Passive-Aggressive"), - (KNeighborsClassifier(n_neighbors=10), "kNN"), - (RandomForestClassifier(), "Random forest")): - print('=' * 80) + (LogisticRegression(C=5, max_iter=1000), "Logistic Regression"), + (RidgeClassifier(alpha=1.0, solver="sparse_cg"), "Ridge Classifier"), + (KNeighborsClassifier(n_neighbors=100), "kNN"), + (RandomForestClassifier(), "Random Forest"), + # L2 penalty Linear SVC + (LinearSVC(C=0.1, dual=False, max_iter=1000), "Linear SVC"), + # L2 penalty Linear SGD + ( + SGDClassifier( + loss="log_loss", alpha=1e-4, n_iter_no_change=3, early_stopping=True + ), + "log-loss SGD", + ), + # NearestCentroid (aka Rocchio classifier) + (NearestCentroid(), "NearestCentroid"), + # Sparse naive Bayes classifier + (ComplementNB(alpha=0.1), "Complement naive Bayes"), +): + print("=" * 80) print(name) - results.append(benchmark(clf)) - -for penalty in ["l2", "l1"]: - print('=' * 80) - print("%s penalty" % penalty.upper()) - # Train Liblinear model - results.append(benchmark(LinearSVC(penalty=penalty, dual=False, - tol=1e-3))) - - # Train SGD model - results.append(benchmark(SGDClassifier(alpha=.0001, max_iter=50, - penalty=penalty))) - -# Train SGD with Elastic Net penalty -print('=' * 80) -print("Elastic-Net penalty") -results.append(benchmark(SGDClassifier(alpha=.0001, max_iter=50, - penalty="elasticnet"))) - -# Train NearestCentroid without threshold -print('=' * 80) -print("NearestCentroid (aka Rocchio classifier)") -results.append(benchmark(NearestCentroid())) - -# Train sparse Naive Bayes classifiers -print('=' * 80) -print("Naive Bayes") -results.append(benchmark(MultinomialNB(alpha=.01))) -results.append(benchmark(BernoulliNB(alpha=.01))) -results.append(benchmark(ComplementNB(alpha=.1))) - -print('=' * 80) -print("LinearSVC with L1-based feature selection") -# The smaller C, the stronger the regularization. -# The more regularization, the more sparsity. -results.append(benchmark(Pipeline([ - ('feature_selection', SelectFromModel(LinearSVC(penalty="l1", dual=False, - tol=1e-3))), - ('classification', LinearSVC(penalty="l2"))]))) - + results.append(benchmark(clf, name)) # %% -# Add plots -# ------------------------------------ -# The bar plot indicates the accuracy, training time (normalized) and test time -# (normalized) of each classifier. +# Plot accuracy, training and test time of each classifier +# ======================================================== +# +# The scatter plots show the trade-off between the test accuracy and the +# training and testing time of each classifier. + indices = np.arange(len(results)) results = [[x[i] for x in results] for i in range(4)] clf_names, score, training_time, test_time = results -training_time = np.array(training_time) / np.max(training_time) -test_time = np.array(test_time) / np.max(test_time) - -plt.figure(figsize=(12, 8)) -plt.title("Score") -plt.barh(indices, score, .2, label="score", color='navy') -plt.barh(indices + .3, training_time, .2, label="training time", - color='c') -plt.barh(indices + .6, test_time, .2, label="test time", color='darkorange') -plt.yticks(()) -plt.legend(loc='best') -plt.subplots_adjust(left=.25) -plt.subplots_adjust(top=.95) -plt.subplots_adjust(bottom=.05) - -for i, c in zip(indices, clf_names): - plt.text(-.3, i, c) - -plt.show() +training_time = np.array(training_time) +test_time = np.array(test_time) + +fig, ax1 = plt.subplots(figsize=(10, 8)) +ax1.scatter(score, training_time, s=60) +ax1.set( + title="Score-training time trade-off", + yscale="log", + xlabel="test accuracy", + ylabel="training time (s)", +) +fig, ax2 = plt.subplots(figsize=(10, 8)) +ax2.scatter(score, test_time, s=60) +ax2.set( + title="Score-test time trade-off", + yscale="log", + xlabel="test accuracy", + ylabel="test time (s)", +) + +for i, txt in enumerate(clf_names): + ax1.annotate(txt, (score[i], training_time[i])) + ax2.annotate(txt, (score[i], test_time[i])) + +# %% +# The naive Bayes model has the best trade-off between score and +# training/testing time, while Random Forest is both slow to train, expensive to +# predict and has a comparatively bad accuracy. This is expected: for +# high-dimensional prediction problems, linear models are often better suited as +# most problems become linearly separable when the feature space has 10,000 +# dimensions or more. +# +# The difference in training speed and accuracy of the linear models can be +# explained by the choice of the loss function they optimize and the kind of +# regularization they use. Be aware that some linear models with the same loss +# but a different solver or regularization configuration may yield different +# fitting times and test accuracy. We can observe on the second plot that once +# trained, all linear models have approximately the same prediction speed which +# is expected because they all implement the same prediction function. +# +# KNeighborsClassifier has a relatively low accuracy and has the highest testing +# time. The long prediction time is also expected: for each prediction the model +# has to compute the pairwise distances between the testing sample and each +# document in the training set, which is computationally expensive. Furthermore, +# the "curse of dimensionality" harms the ability of this model to yield +# competitive accuracy in the high dimensional feature space of text +# classification problems. diff --git a/examples/text/plot_document_clustering.py b/examples/text/plot_document_clustering.py index bfcb7e6a5acf4..43dcd4f443bf5 100644 --- a/examples/text/plot_document_clustering.py +++ b/examples/text/plot_document_clustering.py @@ -3,223 +3,446 @@ Clustering text documents using k-means ======================================= -This is an example showing how the scikit-learn can be used to cluster -documents by topics using a bag-of-words approach. This example uses -a scipy.sparse matrix to store the features instead of standard numpy arrays. - -Two feature extraction methods can be used in this example: - - - TfidfVectorizer uses a in-memory vocabulary (a python dict) to map the most - frequent words to features indices and hence compute a word occurrence - frequency (sparse) matrix. The word frequencies are then reweighted using - the Inverse Document Frequency (IDF) vector collected feature-wise over - the corpus. - - - HashingVectorizer hashes word occurrences to a fixed dimensional space, - possibly with collisions. The word count vectors are then normalized to - each have l2-norm equal to one (projected to the euclidean unit-ball) which - seems to be important for k-means to work in high dimensional space. - - HashingVectorizer does not provide IDF weighting as this is a stateless - model (the fit method does nothing). When IDF weighting is needed it can - be added by pipelining its output to a TfidfTransformer instance. - -Two algorithms are demoed: ordinary k-means and its more scalable cousin -minibatch k-means. - -Additionally, latent semantic analysis can also be used to reduce -dimensionality and discover latent patterns in the data. - -It can be noted that k-means (and minibatch k-means) are very sensitive to -feature scaling and that in this case the IDF weighting helps improve the -quality of the clustering by quite a lot as measured against the "ground truth" -provided by the class label assignments of the 20 newsgroups dataset. - -This improvement is not visible in the Silhouette Coefficient which is small -for both as this measure seem to suffer from the phenomenon called -"Concentration of Measure" or "Curse of Dimensionality" for high dimensional -datasets such as text data. Other measures such as V-measure and Adjusted Rand -Index are information theoretic based evaluation scores: as they are only based -on cluster assignments rather than distances, hence not affected by the curse -of dimensionality. - -Note: as k-means is optimizing a non-convex objective function, it will likely -end up in a local optimum. Several runs with independent random init might be -necessary to get a good convergence. +This is an example showing how the scikit-learn API can be used to cluster +documents by topics using a `Bag of Words approach +`_. -""" +Two algorithms are demonstrated, namely :class:`~sklearn.cluster.KMeans` and its more +scalable variant, :class:`~sklearn.cluster.MiniBatchKMeans`. Additionally, +latent semantic analysis is used to reduce dimensionality and discover latent +patterns in the data. -# Author: Peter Prettenhofer -# Lars Buitinck -# License: BSD 3 clause -from sklearn.datasets import fetch_20newsgroups -from sklearn.decomposition import TruncatedSVD -from sklearn.feature_extraction.text import TfidfVectorizer -from sklearn.feature_extraction.text import HashingVectorizer -from sklearn.feature_extraction.text import TfidfTransformer -from sklearn.pipeline import make_pipeline -from sklearn.preprocessing import Normalizer -from sklearn import metrics +This example uses two different text vectorizers: a +:class:`~sklearn.feature_extraction.text.TfidfVectorizer` and a +:class:`~sklearn.feature_extraction.text.HashingVectorizer`. See the example +notebook :ref:`sphx_glr_auto_examples_text_plot_hashing_vs_dict_vectorizer.py` +for more information on vectorizers and a comparison of their processing times. -from sklearn.cluster import KMeans, MiniBatchKMeans +For document analysis via a supervised learning approach, see the example script +:ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py`. -import logging -from optparse import OptionParser -import sys -from time import time +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Loading text data +# ================= +# +# We load data from :ref:`20newsgroups_dataset`, which comprises around 18,000 +# newsgroups posts on 20 topics. For illustrative purposes and to reduce the +# computational cost, we select a subset of 4 topics only accounting for around +# 3,400 documents. See the example +# :ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py` +# to gain intuition on the overlap of such topics. +# +# Notice that, by default, the text samples contain some message metadata such +# as `"headers"`, `"footers"` (signatures) and `"quotes"` to other posts. We use +# the `remove` parameter from :func:`~sklearn.datasets.fetch_20newsgroups` to +# strip those features and have a more sensible clustering problem. import numpy as np +from sklearn.datasets import fetch_20newsgroups -# Display progress logs on stdout -logging.basicConfig(level=logging.INFO, - format='%(asctime)s %(levelname)s %(message)s') - -# parse commandline arguments -op = OptionParser() -op.add_option("--lsa", - dest="n_components", type="int", - help="Preprocess documents with latent semantic analysis.") -op.add_option("--no-minibatch", - action="store_false", dest="minibatch", default=True, - help="Use ordinary k-means algorithm (in batch mode).") -op.add_option("--no-idf", - action="store_false", dest="use_idf", default=True, - help="Disable Inverse Document Frequency feature weighting.") -op.add_option("--use-hashing", - action="store_true", default=False, - help="Use a hashing feature vectorizer") -op.add_option("--n-features", type=int, default=10000, - help="Maximum number of features (dimensions)" - " to extract from text.") -op.add_option("--verbose", - action="store_true", dest="verbose", default=False, - help="Print progress reports inside k-means algorithm.") - -print(__doc__) -op.print_help() - - -def is_interactive(): - return not hasattr(sys.modules['__main__'], '__file__') - - -# work-around for Jupyter notebook and IPython console -argv = [] if is_interactive() else sys.argv[1:] -(opts, args) = op.parse_args(argv) -if len(args) > 0: - op.error("this script takes no arguments.") - sys.exit(1) - - -# ############################################################################# -# Load some categories from the training set categories = [ - 'alt.atheism', - 'talk.religion.misc', - 'comp.graphics', - 'sci.space', + "alt.atheism", + "talk.religion.misc", + "comp.graphics", + "sci.space", ] -# Uncomment the following to do the analysis on all the categories -# categories = None - -print("Loading 20 newsgroups dataset for categories:") -print(categories) -dataset = fetch_20newsgroups(subset='all', categories=categories, - shuffle=True, random_state=42) - -print("%d documents" % len(dataset.data)) -print("%d categories" % len(dataset.target_names)) -print() +dataset = fetch_20newsgroups( + remove=("headers", "footers", "quotes"), + subset="all", + categories=categories, + shuffle=True, + random_state=42, +) labels = dataset.target -true_k = np.unique(labels).shape[0] - -print("Extracting features from the training dataset " - "using a sparse vectorizer") -t0 = time() -if opts.use_hashing: - if opts.use_idf: - # Perform an IDF normalization on the output of HashingVectorizer - hasher = HashingVectorizer(n_features=opts.n_features, - stop_words='english', alternate_sign=False, - norm=None) - vectorizer = make_pipeline(hasher, TfidfTransformer()) - else: - vectorizer = HashingVectorizer(n_features=opts.n_features, - stop_words='english', - alternate_sign=False, norm='l2') -else: - vectorizer = TfidfVectorizer(max_df=0.5, max_features=opts.n_features, - min_df=2, stop_words='english', - use_idf=opts.use_idf) -X = vectorizer.fit_transform(dataset.data) - -print("done in %fs" % (time() - t0)) -print("n_samples: %d, n_features: %d" % X.shape) -print() - -if opts.n_components: - print("Performing dimensionality reduction using LSA") - t0 = time() - # Vectorizer results are normalized, which makes KMeans behave as - # spherical k-means for better results. Since LSA/SVD results are - # not normalized, we have to redo the normalization. - svd = TruncatedSVD(opts.n_components) - normalizer = Normalizer(copy=False) - lsa = make_pipeline(svd, normalizer) - - X = lsa.fit_transform(X) - - print("done in %fs" % (time() - t0)) - - explained_variance = svd.explained_variance_ratio_.sum() - print("Explained variance of the SVD step: {}%".format( - int(explained_variance * 100))) - - print() +unique_labels, category_sizes = np.unique(labels, return_counts=True) +true_k = unique_labels.shape[0] + +print(f"{len(dataset.data)} documents - {true_k} categories") + +# %% +# Quantifying the quality of clustering results +# ============================================= +# +# In this section we define a function to score different clustering pipelines +# using several metrics. +# +# Clustering algorithms are fundamentally unsupervised learning methods. +# However, since we happen to have class labels for this specific dataset, it is +# possible to use evaluation metrics that leverage this "supervised" ground +# truth information to quantify the quality of the resulting clusters. Examples +# of such metrics are the following: +# +# - homogeneity, which quantifies how much clusters contain only members of a +# single class; +# +# - completeness, which quantifies how much members of a given class are +# assigned to the same clusters; +# +# - V-measure, the harmonic mean of completeness and homogeneity; +# +# - Rand-Index, which measures how frequently pairs of data points are grouped +# consistently according to the result of the clustering algorithm and the +# ground truth class assignment; +# +# - Adjusted Rand-Index, a chance-adjusted Rand-Index such that random cluster +# assignment have an ARI of 0.0 in expectation. +# +# If the ground truth labels are not known, evaluation can only be performed +# using the model results itself. In that case, the Silhouette Coefficient comes in +# handy. See :ref:`sphx_glr_auto_examples_cluster_plot_kmeans_silhouette_analysis.py` +# for an example on how to do it. +# +# For more reference, see :ref:`clustering_evaluation`. + +from collections import defaultdict +from time import time +from sklearn import metrics -# ############################################################################# -# Do the actual clustering +evaluations = [] +evaluations_std = [] + + +def fit_and_evaluate(km, X, name=None, n_runs=5): + name = km.__class__.__name__ if name is None else name + + train_times = [] + scores = defaultdict(list) + for seed in range(n_runs): + km.set_params(random_state=seed) + t0 = time() + km.fit(X) + train_times.append(time() - t0) + scores["Homogeneity"].append(metrics.homogeneity_score(labels, km.labels_)) + scores["Completeness"].append(metrics.completeness_score(labels, km.labels_)) + scores["V-measure"].append(metrics.v_measure_score(labels, km.labels_)) + scores["Adjusted Rand-Index"].append( + metrics.adjusted_rand_score(labels, km.labels_) + ) + scores["Silhouette Coefficient"].append( + metrics.silhouette_score(X, km.labels_, sample_size=2000) + ) + train_times = np.asarray(train_times) + + print(f"clustering done in {train_times.mean():.2f} Âą {train_times.std():.2f} s ") + evaluation = { + "estimator": name, + "train_time": train_times.mean(), + } + evaluation_std = { + "estimator": name, + "train_time": train_times.std(), + } + for score_name, score_values in scores.items(): + mean_score, std_score = np.mean(score_values), np.std(score_values) + print(f"{score_name}: {mean_score:.3f} Âą {std_score:.3f}") + evaluation[score_name] = mean_score + evaluation_std[score_name] = std_score + evaluations.append(evaluation) + evaluations_std.append(evaluation_std) + + +# %% +# K-means clustering on text features +# =================================== +# +# Two feature extraction methods are used in this example: +# +# - :class:`~sklearn.feature_extraction.text.TfidfVectorizer` uses an in-memory +# vocabulary (a Python dict) to map the most frequent words to features +# indices and hence compute a word occurrence frequency (sparse) matrix. The +# word frequencies are then reweighted using the Inverse Document Frequency +# (IDF) vector collected feature-wise over the corpus. +# +# - :class:`~sklearn.feature_extraction.text.HashingVectorizer` hashes word +# occurrences to a fixed dimensional space, possibly with collisions. The word +# count vectors are then normalized to each have l2-norm equal to one +# (projected to the euclidean unit-sphere) which seems to be important for +# k-means to work in high dimensional space. +# +# Furthermore it is possible to post-process those extracted features using +# dimensionality reduction. We will explore the impact of those choices on the +# clustering quality in the following. +# +# Feature Extraction using TfidfVectorizer +# ---------------------------------------- +# +# We first benchmark the estimators using a dictionary vectorizer along with an +# IDF normalization as provided by +# :class:`~sklearn.feature_extraction.text.TfidfVectorizer`. -if opts.minibatch: - km = MiniBatchKMeans(n_clusters=true_k, init='k-means++', n_init=1, - init_size=1000, batch_size=1000, verbose=opts.verbose) -else: - km = KMeans(n_clusters=true_k, init='k-means++', max_iter=100, n_init=1, - verbose=opts.verbose) +from sklearn.feature_extraction.text import TfidfVectorizer -print("Clustering sparse data with %s" % km) +vectorizer = TfidfVectorizer( + max_df=0.5, + min_df=5, + stop_words="english", +) t0 = time() -km.fit(X) -print("done in %0.3fs" % (time() - t0)) -print() - -print("Homogeneity: %0.3f" % metrics.homogeneity_score(labels, km.labels_)) -print("Completeness: %0.3f" % metrics.completeness_score(labels, km.labels_)) -print("V-measure: %0.3f" % metrics.v_measure_score(labels, km.labels_)) -print("Adjusted Rand-Index: %.3f" - % metrics.adjusted_rand_score(labels, km.labels_)) -print("Silhouette Coefficient: %0.3f" - % metrics.silhouette_score(X, km.labels_, sample_size=1000)) - +X_tfidf = vectorizer.fit_transform(dataset.data) + +print(f"vectorization done in {time() - t0:.3f} s") +print(f"n_samples: {X_tfidf.shape[0]}, n_features: {X_tfidf.shape[1]}") + +# %% +# After ignoring terms that appear in more than 50% of the documents (as set by +# `max_df=0.5`) and terms that are not present in at least 5 documents (set by +# `min_df=5`), the resulting number of unique terms `n_features` is around +# 8,000. We can additionally quantify the sparsity of the `X_tfidf` matrix as +# the fraction of non-zero entries divided by the total number of elements. + +print(f"{X_tfidf.nnz / np.prod(X_tfidf.shape):.3f}") + +# %% +# We find that around 0.7% of the entries of the `X_tfidf` matrix are non-zero. +# +# .. _kmeans_sparse_high_dim: +# +# Clustering sparse data with k-means +# ----------------------------------- +# +# As both :class:`~sklearn.cluster.KMeans` and +# :class:`~sklearn.cluster.MiniBatchKMeans` optimize a non-convex objective +# function, their clustering is not guaranteed to be optimal for a given random +# init. Even further, on sparse high-dimensional data such as text vectorized +# using the Bag of Words approach, k-means can initialize centroids on extremely +# isolated data points. Those data points can stay their own centroids all +# along. +# +# The following code illustrates how the previous phenomenon can sometimes lead +# to highly imbalanced clusters, depending on the random initialization: + +from sklearn.cluster import KMeans + +for seed in range(5): + kmeans = KMeans( + n_clusters=true_k, + max_iter=100, + n_init=1, + random_state=seed, + ).fit(X_tfidf) + cluster_ids, cluster_sizes = np.unique(kmeans.labels_, return_counts=True) + print(f"Number of elements assigned to each cluster: {cluster_sizes}") print() +print( + "True number of documents in each category according to the class labels: " + f"{category_sizes}" +) + +# %% +# To avoid this problem, one possibility is to increase the number of runs with +# independent random initiations `n_init`. In such case the clustering with the +# best inertia (objective function of k-means) is chosen. + +kmeans = KMeans( + n_clusters=true_k, + max_iter=100, + n_init=5, +) + +fit_and_evaluate(kmeans, X_tfidf, name="KMeans\non tf-idf vectors") + +# %% +# All those clustering evaluation metrics have a maximum value of 1.0 (for a +# perfect clustering result). Higher values are better. Values of the Adjusted +# Rand-Index close to 0.0 correspond to a random labeling. Notice from the +# scores above that the cluster assignment is indeed well above chance level, +# but the overall quality can certainly improve. +# +# Keep in mind that the class labels may not reflect accurately the document +# topics and therefore metrics that use labels are not necessarily the best to +# evaluate the quality of our clustering pipeline. +# +# Performing dimensionality reduction using LSA +# --------------------------------------------- +# +# A `n_init=1` can still be used as long as the dimension of the vectorized +# space is reduced first to make k-means more stable. For such purpose we use +# :class:`~sklearn.decomposition.TruncatedSVD`, which works on term count/tf-idf +# matrices. Since SVD results are not normalized, we redo the normalization to +# improve the :class:`~sklearn.cluster.KMeans` result. Using SVD to reduce the +# dimensionality of TF-IDF document vectors is often known as `latent semantic +# analysis `_ (LSA) in +# the information retrieval and text mining literature. +from sklearn.decomposition import TruncatedSVD +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import Normalizer -if not opts.use_hashing: - print("Top terms per cluster:") +lsa = make_pipeline(TruncatedSVD(n_components=100), Normalizer(copy=False)) +t0 = time() +X_lsa = lsa.fit_transform(X_tfidf) +explained_variance = lsa[0].explained_variance_ratio_.sum() + +print(f"LSA done in {time() - t0:.3f} s") +print(f"Explained variance of the SVD step: {explained_variance * 100:.1f}%") + +# %% +# Using a single initialization means the processing time will be reduced for +# both :class:`~sklearn.cluster.KMeans` and +# :class:`~sklearn.cluster.MiniBatchKMeans`. + +kmeans = KMeans( + n_clusters=true_k, + max_iter=100, + n_init=1, +) + +fit_and_evaluate(kmeans, X_lsa, name="KMeans\nwith LSA on tf-idf vectors") + +# %% +# We can observe that clustering on the LSA representation of the document is +# significantly faster (both because of `n_init=1` and because the +# dimensionality of the LSA feature space is much smaller). Furthermore, all the +# clustering evaluation metrics have improved. We repeat the experiment with +# :class:`~sklearn.cluster.MiniBatchKMeans`. + +from sklearn.cluster import MiniBatchKMeans + +minibatch_kmeans = MiniBatchKMeans( + n_clusters=true_k, + n_init=1, + init_size=1000, + batch_size=1000, +) + +fit_and_evaluate( + minibatch_kmeans, + X_lsa, + name="MiniBatchKMeans\nwith LSA on tf-idf vectors", +) + +# %% +# Top terms per cluster +# --------------------- +# +# Since :class:`~sklearn.feature_extraction.text.TfidfVectorizer` can be +# inverted we can identify the cluster centers, which provide an intuition of +# the most influential words **for each cluster**. See the example script +# :ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py` +# for a comparison with the most predictive words **for each target class**. + +original_space_centroids = lsa[0].inverse_transform(kmeans.cluster_centers_) +order_centroids = original_space_centroids.argsort()[:, ::-1] +terms = vectorizer.get_feature_names_out() + +for i in range(true_k): + print(f"Cluster {i}: ", end="") + for ind in order_centroids[i, :10]: + print(f"{terms[ind]} ", end="") + print() - if opts.n_components: - original_space_centroids = svd.inverse_transform(km.cluster_centers_) - order_centroids = original_space_centroids.argsort()[:, ::-1] - else: - order_centroids = km.cluster_centers_.argsort()[:, ::-1] +# %% +# HashingVectorizer +# ----------------- +# An alternative vectorization can be done using a +# :class:`~sklearn.feature_extraction.text.HashingVectorizer` instance, which +# does not provide IDF weighting as this is a stateless model (the fit method +# does nothing). When IDF weighting is needed it can be added by pipelining the +# :class:`~sklearn.feature_extraction.text.HashingVectorizer` output to a +# :class:`~sklearn.feature_extraction.text.TfidfTransformer` instance. In this +# case we also add LSA to the pipeline to reduce the dimension and sparcity of +# the hashed vector space. + +from sklearn.feature_extraction.text import HashingVectorizer, TfidfTransformer + +lsa_vectorizer = make_pipeline( + HashingVectorizer(stop_words="english", n_features=50_000), + TfidfTransformer(), + TruncatedSVD(n_components=100, random_state=0), + Normalizer(copy=False), +) - terms = vectorizer.get_feature_names() - for i in range(true_k): - print("Cluster %d:" % i, end='') - for ind in order_centroids[i, :10]: - print(' %s' % terms[ind], end='') - print() +t0 = time() +X_hashed_lsa = lsa_vectorizer.fit_transform(dataset.data) +print(f"vectorization done in {time() - t0:.3f} s") + +# %% +# One can observe that the LSA step takes a relatively long time to fit, +# especially with hashed vectors. The reason is that a hashed space is typically +# large (set to `n_features=50_000` in this example). One can try lowering the +# number of features at the expense of having a larger fraction of features with +# hash collisions as shown in the example notebook +# :ref:`sphx_glr_auto_examples_text_plot_hashing_vs_dict_vectorizer.py`. +# +# We now fit and evaluate the `kmeans` and `minibatch_kmeans` instances on this +# hashed-lsa-reduced data: + +fit_and_evaluate(kmeans, X_hashed_lsa, name="KMeans\nwith LSA on hashed vectors") + +# %% +fit_and_evaluate( + minibatch_kmeans, + X_hashed_lsa, + name="MiniBatchKMeans\nwith LSA on hashed vectors", +) + +# %% +# Both methods lead to good results that are similar to running the same models +# on the traditional LSA vectors (without hashing). +# +# Clustering evaluation summary +# ============================== + +import matplotlib.pyplot as plt +import pandas as pd + +fig, (ax0, ax1) = plt.subplots(ncols=2, figsize=(16, 6), sharey=True) + +df = pd.DataFrame(evaluations[::-1]).set_index("estimator") +df_std = pd.DataFrame(evaluations_std[::-1]).set_index("estimator") + +df.drop( + ["train_time"], + axis="columns", +).plot.barh(ax=ax0, xerr=df_std) +ax0.set_xlabel("Clustering scores") +ax0.set_ylabel("") + +df["train_time"].plot.barh(ax=ax1, xerr=df_std["train_time"]) +ax1.set_xlabel("Clustering time (s)") +plt.tight_layout() + +# %% +# :class:`~sklearn.cluster.KMeans` and :class:`~sklearn.cluster.MiniBatchKMeans` +# suffer from the phenomenon called the `Curse of Dimensionality +# `_ for high dimensional +# datasets such as text data. That is the reason why the overall scores improve +# when using LSA. Using LSA reduced data also improves the stability and +# requires lower clustering time, though keep in mind that the LSA step itself +# takes a long time, especially with hashed vectors. +# +# The Silhouette Coefficient is defined between 0 and 1. In all cases we obtain +# values close to 0 (even if they improve a bit after using LSA) because its +# definition requires measuring distances, in contrast with other evaluation +# metrics such as the V-measure and the Adjusted Rand Index which are only based +# on cluster assignments rather than distances. Notice that strictly speaking, +# one should not compare the Silhouette Coefficient between spaces of different +# dimension, due to the different notions of distance they imply. +# +# The homogeneity, completeness and hence v-measure metrics do not yield a +# baseline with regards to random labeling: this means that depending on the +# number of samples, clusters and ground truth classes, a completely random +# labeling will not always yield the same values. In particular random labeling +# won't yield zero scores, especially when the number of clusters is large. This +# problem can safely be ignored when the number of samples is more than a +# thousand and the number of clusters is less than 10, which is the case of the +# present example. For smaller sample sizes or larger number of clusters it is +# safer to use an adjusted index such as the Adjusted Rand Index (ARI). See the +# example +# :ref:`sphx_glr_auto_examples_cluster_plot_adjusted_for_chance_measures.py` for +# a demo on the effect of random labeling. +# +# The size of the error bars show that :class:`~sklearn.cluster.MiniBatchKMeans` +# is less stable than :class:`~sklearn.cluster.KMeans` for this relatively small +# dataset. It is more interesting to use when the number of samples is much +# bigger, but it can come at the expense of a small degradation in clustering +# quality compared to the traditional k-means algorithm. diff --git a/examples/text/plot_hashing_vs_dict_vectorizer.py b/examples/text/plot_hashing_vs_dict_vectorizer.py index 34673990c15be..4c59c7045bb19 100644 --- a/examples/text/plot_hashing_vs_dict_vectorizer.py +++ b/examples/text/plot_hashing_vs_dict_vectorizer.py @@ -3,108 +3,381 @@ FeatureHasher and DictVectorizer Comparison =========================================== -Compares FeatureHasher and DictVectorizer by using both to vectorize -text documents. +In this example we illustrate text vectorization, which is the process of +representing non-numerical input data (such as dictionaries or text documents) +as vectors of real numbers. -The example demonstrates syntax and speed only; it doesn't actually do -anything useful with the extracted vectors. See the example scripts -{document_classification_20newsgroups,clustering}.py for actual learning -on text documents. +We first compare :func:`~sklearn.feature_extraction.FeatureHasher` and +:func:`~sklearn.feature_extraction.DictVectorizer` by using both methods to +vectorize text documents that are preprocessed (tokenized) with the help of a +custom Python function. + +Later we introduce and analyze the text-specific vectorizers +:func:`~sklearn.feature_extraction.text.HashingVectorizer`, +:func:`~sklearn.feature_extraction.text.CountVectorizer` and +:func:`~sklearn.feature_extraction.text.TfidfVectorizer` that handle both the +tokenization and the assembling of the feature matrix within a single class. + +The objective of the example is to demonstrate the usage of text vectorization +API and to compare their processing time. See the example scripts +:ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py` +and :ref:`sphx_glr_auto_examples_text_plot_document_clustering.py` for actual +learning on text documents. -A discrepancy between the number of terms reported for DictVectorizer and -for FeatureHasher is to be expected due to hash collisions. """ -# Author: Lars Buitinck -# License: BSD 3 clause -from collections import defaultdict -import re -import sys -from time import time +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np +# %% +# Load Data +# --------- +# +# We load data from :ref:`20newsgroups_dataset`, which comprises around +# 18000 newsgroups posts on 20 topics split in two subsets: one for training and +# one for testing. For the sake of simplicity and reducing the computational +# cost, we select a subset of 7 topics and use the training set only. from sklearn.datasets import fetch_20newsgroups -from sklearn.feature_extraction import DictVectorizer, FeatureHasher +categories = [ + "alt.atheism", + "comp.graphics", + "comp.sys.ibm.pc.hardware", + "misc.forsale", + "rec.autos", + "sci.space", + "talk.religion.misc", +] -def n_nonzero_columns(X): - """Returns the number of non-zero columns in a CSR matrix X.""" - return len(np.unique(X.nonzero()[1])) +print("Loading 20 newsgroups training data") +raw_data, _ = fetch_20newsgroups(subset="train", categories=categories, return_X_y=True) +data_size_mb = sum(len(s.encode("utf-8")) for s in raw_data) / 1e6 +print(f"{len(raw_data)} documents - {data_size_mb:.3f}MB") + +# %% +# Define preprocessing functions +# ------------------------------ +# +# A token may be a word, part of a word or anything comprised between spaces or +# symbols in a string. Here we define a function that extracts the tokens using +# a simple regular expression (regex) that matches Unicode word characters. This +# includes most characters that can be part of a word in any language, as well +# as numbers and the underscore: + +import re -def tokens(doc): +def tokenize(doc): """Extract tokens from doc. - This uses a simple regex to break strings into tokens. For a more - principled approach, see CountVectorizer or TfidfVectorizer. + This uses a simple regex that matches word characters to break strings + into tokens. For a more principled approach, see CountVectorizer or + TfidfVectorizer. """ return (tok.lower() for tok in re.findall(r"\w+", doc)) +list(tokenize("This is a simple example, isn't it?")) + +# %% +# We define an additional function that counts the (frequency of) occurrence of +# each token in a given document. It returns a frequency dictionary to be used +# by the vectorizers. + +from collections import defaultdict + + def token_freqs(doc): - """Extract a dict mapping tokens from doc to their frequencies.""" + """Extract a dict mapping tokens from doc to their occurrences.""" + freq = defaultdict(int) - for tok in tokens(doc): + for tok in tokenize(doc): freq[tok] += 1 return freq -categories = [ - 'alt.atheism', - 'comp.graphics', - 'comp.sys.ibm.pc.hardware', - 'misc.forsale', - 'rec.autos', - 'sci.space', - 'talk.religion.misc', -] -# Uncomment the following line to use a larger set (11k+ documents) -# categories = None +token_freqs("That is one example, but this is another one") -print(__doc__) -print("Usage: %s [n_features_for_hashing]" % sys.argv[0]) -print(" The default number of features is 2**18.") -print() +# %% +# Observe in particular that the repeated token `"is"` is counted twice for +# instance. +# +# Breaking a text document into word tokens, potentially losing the order +# information between the words in a sentence is often called a `Bag of Words +# representation `_. -try: - n_features = int(sys.argv[1]) -except IndexError: - n_features = 2 ** 18 -except ValueError: - print("not a valid number of features: %r" % sys.argv[1]) - sys.exit(1) +# %% +# DictVectorizer +# -------------- +# +# First we benchmark the :func:`~sklearn.feature_extraction.DictVectorizer`, +# then we compare it to :func:`~sklearn.feature_extraction.FeatureHasher` as +# both of them receive dictionaries as input. +from time import time -print("Loading 20 newsgroups training data") -raw_data, _ = fetch_20newsgroups(subset='train', categories=categories, - return_X_y=True) -data_size_mb = sum(len(s.encode('utf-8')) for s in raw_data) / 1e6 -print("%d documents - %0.3fMB" % (len(raw_data), data_size_mb)) -print() +from sklearn.feature_extraction import DictVectorizer + +dict_count_vectorizers = defaultdict(list) -print("DictVectorizer") t0 = time() vectorizer = DictVectorizer() vectorizer.fit_transform(token_freqs(d) for d in raw_data) duration = time() - t0 -print("done in %fs at %0.3fMB/s" % (duration, data_size_mb / duration)) -print("Found %d unique terms" % len(vectorizer.get_feature_names())) -print() +dict_count_vectorizers["vectorizer"].append( + vectorizer.__class__.__name__ + "\non freq dicts" +) +dict_count_vectorizers["speed"].append(data_size_mb / duration) +print(f"done in {duration:.3f} s at {data_size_mb / duration:.1f} MB/s") +print(f"Found {len(vectorizer.get_feature_names_out())} unique terms") + +# %% +# The actual mapping from text token to column index is explicitly stored in +# the `.vocabulary_` attribute which is a potentially very large Python +# dictionary: +type(vectorizer.vocabulary_) + +# %% +len(vectorizer.vocabulary_) + +# %% +vectorizer.vocabulary_["example"] + +# %% +# FeatureHasher +# ------------- +# +# Dictionaries take up a large amount of storage space and grow in size as the +# training set grows. Instead of growing the vectors along with a dictionary, +# feature hashing builds a vector of pre-defined length by applying a hash +# function `h` to the features (e.g., tokens), then using the hash values +# directly as feature indices and updating the resulting vector at those +# indices. When the feature space is not large enough, hashing functions tend to +# map distinct values to the same hash code (hash collisions). As a result, it +# is impossible to determine what object generated any particular hash code. +# +# Because of the above it is impossible to recover the original tokens from the +# feature matrix and the best approach to estimate the number of unique terms in +# the original dictionary is to count the number of active columns in the +# encoded feature matrix. For such a purpose we define the following function: + +import numpy as np + + +def n_nonzero_columns(X): + """Number of columns with at least one non-zero value in a CSR matrix. + + This is useful to count the number of features columns that are effectively + active when using the FeatureHasher. + """ + return len(np.unique(X.nonzero()[1])) + + +# %% +# The default number of features for the +# :func:`~sklearn.feature_extraction.FeatureHasher` is 2**20. Here we set +# `n_features = 2**18` to illustrate hash collisions. +# +# **FeatureHasher on frequency dictionaries** + +from sklearn.feature_extraction import FeatureHasher + +t0 = time() +hasher = FeatureHasher(n_features=2**18) +X = hasher.transform(token_freqs(d) for d in raw_data) +duration = time() - t0 +dict_count_vectorizers["vectorizer"].append( + hasher.__class__.__name__ + "\non freq dicts" +) +dict_count_vectorizers["speed"].append(data_size_mb / duration) +print(f"done in {duration:.3f} s at {data_size_mb / duration:.1f} MB/s") +print(f"Found {n_nonzero_columns(X)} unique tokens") + +# %% +# The number of unique tokens when using the +# :func:`~sklearn.feature_extraction.FeatureHasher` is lower than those obtained +# using the :func:`~sklearn.feature_extraction.DictVectorizer`. This is due to +# hash collisions. +# +# The number of collisions can be reduced by increasing the feature space. +# Notice that the speed of the vectorizer does not change significantly when +# setting a large number of features, though it causes larger coefficient +# dimensions and then requires more memory usage to store them, even if a +# majority of them is inactive. -print("FeatureHasher on frequency dicts") t0 = time() -hasher = FeatureHasher(n_features=n_features) +hasher = FeatureHasher(n_features=2**22) X = hasher.transform(token_freqs(d) for d in raw_data) duration = time() - t0 -print("done in %fs at %0.3fMB/s" % (duration, data_size_mb / duration)) -print("Found %d unique terms" % n_nonzero_columns(X)) -print() -print("FeatureHasher on raw tokens") +print(f"done in {duration:.3f} s at {data_size_mb / duration:.1f} MB/s") +print(f"Found {n_nonzero_columns(X)} unique tokens") + +# %% +# We confirm that the number of unique tokens gets closer to the number of +# unique terms found by the :func:`~sklearn.feature_extraction.DictVectorizer`. +# +# **FeatureHasher on raw tokens** +# +# Alternatively, one can set `input_type="string"` in the +# :func:`~sklearn.feature_extraction.FeatureHasher` to vectorize the strings +# output directly from the customized `tokenize` function. This is equivalent to +# passing a dictionary with an implied frequency of 1 for each feature name. + t0 = time() -hasher = FeatureHasher(n_features=n_features, input_type="string") -X = hasher.transform(tokens(d) for d in raw_data) +hasher = FeatureHasher(n_features=2**18, input_type="string") +X = hasher.transform(tokenize(d) for d in raw_data) duration = time() - t0 -print("done in %fs at %0.3fMB/s" % (duration, data_size_mb / duration)) -print("Found %d unique terms" % n_nonzero_columns(X)) +dict_count_vectorizers["vectorizer"].append( + hasher.__class__.__name__ + "\non raw tokens" +) +dict_count_vectorizers["speed"].append(data_size_mb / duration) +print(f"done in {duration:.3f} s at {data_size_mb / duration:.1f} MB/s") +print(f"Found {n_nonzero_columns(X)} unique tokens") + +# %% +# We now plot the speed of the above methods for vectorizing. + +import matplotlib.pyplot as plt + +fig, ax = plt.subplots(figsize=(12, 6)) + +y_pos = np.arange(len(dict_count_vectorizers["vectorizer"])) +ax.barh(y_pos, dict_count_vectorizers["speed"], align="center") +ax.set_yticks(y_pos) +ax.set_yticklabels(dict_count_vectorizers["vectorizer"]) +ax.invert_yaxis() +_ = ax.set_xlabel("speed (MB/s)") + +# %% +# In both cases :func:`~sklearn.feature_extraction.FeatureHasher` is +# approximately twice as fast as +# :func:`~sklearn.feature_extraction.DictVectorizer`. This is handy when dealing +# with large amounts of data, with the downside of losing the invertibility of +# the transformation, which in turn makes the interpretation of a model a more +# complex task. +# +# The `FeatureHeasher` with `input_type="string"` is slightly faster than the +# variant that works on frequency dict because it does not count repeated +# tokens: each token is implicitly counted once, even if it was repeated. +# Depending on the downstream machine learning task, it can be a limitation or +# not. +# +# Comparison with special purpose text vectorizers +# ------------------------------------------------ +# +# :func:`~sklearn.feature_extraction.text.CountVectorizer` accepts raw data as +# it internally implements tokenization and occurrence counting. It is similar +# to the :func:`~sklearn.feature_extraction.DictVectorizer` when used along with +# the customized function `token_freqs` as done in the previous section. The +# difference being that :func:`~sklearn.feature_extraction.text.CountVectorizer` +# is more flexible. In particular it accepts various regex patterns through the +# `token_pattern` parameter. + +from sklearn.feature_extraction.text import CountVectorizer + +t0 = time() +vectorizer = CountVectorizer() +vectorizer.fit_transform(raw_data) +duration = time() - t0 +dict_count_vectorizers["vectorizer"].append(vectorizer.__class__.__name__) +dict_count_vectorizers["speed"].append(data_size_mb / duration) +print(f"done in {duration:.3f} s at {data_size_mb / duration:.1f} MB/s") +print(f"Found {len(vectorizer.get_feature_names_out())} unique terms") + +# %% +# We see that using the :func:`~sklearn.feature_extraction.text.CountVectorizer` +# implementation is approximately twice as fast as using the +# :func:`~sklearn.feature_extraction.DictVectorizer` along with the simple +# function we defined for mapping the tokens. The reason is that +# :func:`~sklearn.feature_extraction.text.CountVectorizer` is optimized by +# reusing a compiled regular expression for the full training set instead of +# creating one per document as done in our naive tokenize function. +# +# Now we make a similar experiment with the +# :func:`~sklearn.feature_extraction.text.HashingVectorizer`, which is +# equivalent to combining the "hashing trick" implemented by the +# :func:`~sklearn.feature_extraction.FeatureHasher` class and the text +# preprocessing and tokenization of the +# :func:`~sklearn.feature_extraction.text.CountVectorizer`. + +from sklearn.feature_extraction.text import HashingVectorizer + +t0 = time() +vectorizer = HashingVectorizer(n_features=2**18) +vectorizer.fit_transform(raw_data) +duration = time() - t0 +dict_count_vectorizers["vectorizer"].append(vectorizer.__class__.__name__) +dict_count_vectorizers["speed"].append(data_size_mb / duration) +print(f"done in {duration:.3f} s at {data_size_mb / duration:.1f} MB/s") + +# %% +# We can observe that this is the fastest text tokenization strategy so far, +# assuming that the downstream machine learning task can tolerate a few +# collisions. +# +# TfidfVectorizer +# --------------- +# +# In a large text corpus, some words appear with higher frequency (e.g. "the", +# "a", "is" in English) and do not carry meaningful information about the actual +# contents of a document. If we were to feed the word count data directly to a +# classifier, those very common terms would shadow the frequencies of rarer yet +# more informative terms. In order to re-weight the count features into floating +# point values suitable for usage by a classifier it is very common to use the +# tf-idf transform as implemented by the +# :func:`~sklearn.feature_extraction.text.TfidfTransformer`. TF stands for +# "term-frequency" while "tf-idf" means term-frequency times inverse +# document-frequency. +# +# We now benchmark the :func:`~sklearn.feature_extraction.text.TfidfVectorizer`, +# which is equivalent to combining the tokenization and occurrence counting of +# the :func:`~sklearn.feature_extraction.text.CountVectorizer` along with the +# normalizing and weighting from a +# :func:`~sklearn.feature_extraction.text.TfidfTransformer`. + +from sklearn.feature_extraction.text import TfidfVectorizer + +t0 = time() +vectorizer = TfidfVectorizer() +vectorizer.fit_transform(raw_data) +duration = time() - t0 +dict_count_vectorizers["vectorizer"].append(vectorizer.__class__.__name__) +dict_count_vectorizers["speed"].append(data_size_mb / duration) +print(f"done in {duration:.3f} s at {data_size_mb / duration:.1f} MB/s") +print(f"Found {len(vectorizer.get_feature_names_out())} unique terms") + +# %% +# Summary +# ------- +# Let's conclude this notebook by summarizing all the recorded processing speeds +# in a single plot: + +fig, ax = plt.subplots(figsize=(12, 6)) + +y_pos = np.arange(len(dict_count_vectorizers["vectorizer"])) +ax.barh(y_pos, dict_count_vectorizers["speed"], align="center") +ax.set_yticks(y_pos) +ax.set_yticklabels(dict_count_vectorizers["vectorizer"]) +ax.invert_yaxis() +_ = ax.set_xlabel("speed (MB/s)") + +# %% +# Notice from the plot that +# :func:`~sklearn.feature_extraction.text.TfidfVectorizer` is slightly slower +# than :func:`~sklearn.feature_extraction.text.CountVectorizer` because of the +# extra operation induced by the +# :func:`~sklearn.feature_extraction.text.TfidfTransformer`. +# +# Also notice that, by setting the number of features `n_features = 2**18`, the +# :func:`~sklearn.feature_extraction.text.HashingVectorizer` performs better +# than the :func:`~sklearn.feature_extraction.text.CountVectorizer` at the +# expense of inversibility of the transformation due to hash collisions. +# +# We highlight that :func:`~sklearn.feature_extraction.text.CountVectorizer` and +# :func:`~sklearn.feature_extraction.text.HashingVectorizer` perform better than +# their equivalent :func:`~sklearn.feature_extraction.DictVectorizer` and +# :func:`~sklearn.feature_extraction.FeatureHasher` on manually tokenized +# documents since the internal tokenization step of the former vectorizers +# compiles a regular expression once and then reuses it for all the documents. diff --git a/examples/tree/plot_cost_complexity_pruning.py b/examples/tree/plot_cost_complexity_pruning.py index 822d7a206f842..bdd1a2b0c358f 100644 --- a/examples/tree/plot_cost_complexity_pruning.py +++ b/examples/tree/plot_cost_complexity_pruning.py @@ -17,10 +17,13 @@ See also :ref:`minimal_cost_complexity_pruning` for details on pruning. """ -print(__doc__) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import matplotlib.pyplot as plt -from sklearn.model_selection import train_test_split + from sklearn.datasets import load_breast_cancer +from sklearn.model_selection import train_test_split from sklearn.tree import DecisionTreeClassifier # %% @@ -45,7 +48,7 @@ # In the following plot, the maximum effective alpha value is removed, because # it is the trivial tree with only one node. fig, ax = plt.subplots() -ax.plot(ccp_alphas[:-1], impurities[:-1], marker='o', drawstyle="steps-post") +ax.plot(ccp_alphas[:-1], impurities[:-1], marker="o", drawstyle="steps-post") ax.set_xlabel("effective alpha") ax.set_ylabel("total impurity of leaves") ax.set_title("Total Impurity vs effective alpha for training set") @@ -59,8 +62,11 @@ clf = DecisionTreeClassifier(random_state=0, ccp_alpha=ccp_alpha) clf.fit(X_train, y_train) clfs.append(clf) -print("Number of nodes in the last tree is: {} with ccp_alpha: {}".format( - clfs[-1].tree_.node_count, ccp_alphas[-1])) +print( + "Number of nodes in the last tree is: {} with ccp_alpha: {}".format( + clfs[-1].tree_.node_count, ccp_alphas[-1] + ) +) # %% # For the remainder of this example, we remove the last element in @@ -73,11 +79,11 @@ node_counts = [clf.tree_.node_count for clf in clfs] depth = [clf.tree_.max_depth for clf in clfs] fig, ax = plt.subplots(2, 1) -ax[0].plot(ccp_alphas, node_counts, marker='o', drawstyle="steps-post") +ax[0].plot(ccp_alphas, node_counts, marker="o", drawstyle="steps-post") ax[0].set_xlabel("alpha") ax[0].set_ylabel("number of nodes") ax[0].set_title("Number of nodes vs alpha") -ax[1].plot(ccp_alphas, depth, marker='o', drawstyle="steps-post") +ax[1].plot(ccp_alphas, depth, marker="o", drawstyle="steps-post") ax[1].set_xlabel("alpha") ax[1].set_ylabel("depth of tree") ax[1].set_title("Depth vs alpha") @@ -98,9 +104,7 @@ ax.set_xlabel("alpha") ax.set_ylabel("accuracy") ax.set_title("Accuracy vs alpha for training and testing sets") -ax.plot(ccp_alphas, train_scores, marker='o', label="train", - drawstyle="steps-post") -ax.plot(ccp_alphas, test_scores, marker='o', label="test", - drawstyle="steps-post") +ax.plot(ccp_alphas, train_scores, marker="o", label="train", drawstyle="steps-post") +ax.plot(ccp_alphas, test_scores, marker="o", label="test", drawstyle="steps-post") ax.legend() plt.show() diff --git a/examples/tree/plot_iris_dtc.py b/examples/tree/plot_iris_dtc.py index 60328c4f90d4f..349f4a893511e 100644 --- a/examples/tree/plot_iris_dtc.py +++ b/examples/tree/plot_iris_dtc.py @@ -1,7 +1,7 @@ """ -================================================================ -Plot the decision surface of a decision tree on the iris dataset -================================================================ +======================================================================= +Plot the decision surface of decision trees trained on the iris dataset +======================================================================= Plot the decision surface of a decision tree trained on pairs of features of the iris dataset. @@ -14,24 +14,33 @@ We also show the tree structure of a model built on all of the features. """ -print(__doc__) -import numpy as np +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# First load the copy of the Iris dataset shipped with scikit-learn: +from sklearn.datasets import load_iris + +iris = load_iris() + + +# %% +# Display the decision functions of trees trained on all pairs of features. import matplotlib.pyplot as plt +import numpy as np from sklearn.datasets import load_iris -from sklearn.tree import DecisionTreeClassifier, plot_tree +from sklearn.inspection import DecisionBoundaryDisplay +from sklearn.tree import DecisionTreeClassifier # Parameters n_classes = 3 plot_colors = "ryb" plot_step = 0.02 -# Load data -iris = load_iris() -for pairidx, pair in enumerate([[0, 1], [0, 2], [0, 3], - [1, 2], [1, 3], [2, 3]]): +for pairidx, pair in enumerate([[0, 1], [0, 2], [0, 3], [1, 2], [1, 3], [2, 3]]): # We only take the two corresponding features X = iris.data[:, pair] y = iris.target @@ -40,32 +49,41 @@ clf = DecisionTreeClassifier().fit(X, y) # Plot the decision boundary - plt.subplot(2, 3, pairidx + 1) - - x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1 - y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1 - xx, yy = np.meshgrid(np.arange(x_min, x_max, plot_step), - np.arange(y_min, y_max, plot_step)) + ax = plt.subplot(2, 3, pairidx + 1) plt.tight_layout(h_pad=0.5, w_pad=0.5, pad=2.5) - - Z = clf.predict(np.c_[xx.ravel(), yy.ravel()]) - Z = Z.reshape(xx.shape) - cs = plt.contourf(xx, yy, Z, cmap=plt.cm.RdYlBu) - - plt.xlabel(iris.feature_names[pair[0]]) - plt.ylabel(iris.feature_names[pair[1]]) + DecisionBoundaryDisplay.from_estimator( + clf, + X, + cmap=plt.cm.RdYlBu, + response_method="predict", + ax=ax, + xlabel=iris.feature_names[pair[0]], + ylabel=iris.feature_names[pair[1]], + ) # Plot the training points for i, color in zip(range(n_classes), plot_colors): - idx = np.where(y == i) - plt.scatter(X[idx, 0], X[idx, 1], c=color, label=iris.target_names[i], - cmap=plt.cm.RdYlBu, edgecolor='black', s=15) - -plt.suptitle("Decision surface of a decision tree using paired features") -plt.legend(loc='lower right', borderpad=0, handletextpad=0) -plt.axis("tight") + idx = np.asarray(y == i).nonzero() + plt.scatter( + X[idx, 0], + X[idx, 1], + c=color, + label=iris.target_names[i], + edgecolor="black", + s=15, + ) + +plt.suptitle("Decision surface of decision trees trained on pairs of features") +plt.legend(loc="lower right", borderpad=0, handletextpad=0) +_ = plt.axis("tight") + +# %% +# Display the structure of a single decision tree trained on all the features +# together. +from sklearn.tree import plot_tree plt.figure() clf = DecisionTreeClassifier().fit(iris.data, iris.target) plot_tree(clf, filled=True) +plt.title("Decision tree trained on all the iris features") plt.show() diff --git a/examples/tree/plot_tree_regression.py b/examples/tree/plot_tree_regression.py index 717de4ab72e4e..63abb8946e27a 100644 --- a/examples/tree/plot_tree_regression.py +++ b/examples/tree/plot_tree_regression.py @@ -1,51 +1,144 @@ """ -=================================================================== +======================== Decision Tree Regression -=================================================================== - -A 1D regression with decision tree. - -The :ref:`decision trees ` is -used to fit a sine curve with addition noisy observation. As a result, it -learns local linear regressions approximating the sine curve. - -We can see that if the maximum depth of the tree (controlled by the -`max_depth` parameter) is set too high, the decision trees learn too fine -details of the training data and learn from the noise, i.e. they overfit. +======================== +In this example, we demonstrate the effect of changing the maximum depth of a +decision tree on how it fits to the data. We perform this once on a 1D regression +task and once on a multi-output regression task. """ -print(__doc__) -# Import the necessary modules and libraries +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# %% +# Decision Tree on a 1D Regression Task +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# Here we fit a tree on a 1D regression task. +# +# The :ref:`decision trees ` is +# used to fit a sine curve with addition noisy observation. As a result, it +# learns local linear regressions approximating the sine curve. +# +# We can see that if the maximum depth of the tree (controlled by the +# `max_depth` parameter) is set too high, the decision trees learn too fine +# details of the training data and learn from the noise, i.e. they overfit. +# +# Create a random 1D dataset +# -------------------------- import numpy as np -from sklearn.tree import DecisionTreeRegressor -import matplotlib.pyplot as plt -# Create a random dataset rng = np.random.RandomState(1) X = np.sort(5 * rng.rand(80, 1), axis=0) y = np.sin(X).ravel() y[::5] += 3 * (0.5 - rng.rand(16)) +# %% # Fit regression model +# -------------------- +# Here we fit two models with different maximum depths +from sklearn.tree import DecisionTreeRegressor + regr_1 = DecisionTreeRegressor(max_depth=2) regr_2 = DecisionTreeRegressor(max_depth=5) regr_1.fit(X, y) regr_2.fit(X, y) +# %% # Predict +# ------- +# Get predictions on the test set X_test = np.arange(0.0, 5.0, 0.01)[:, np.newaxis] y_1 = regr_1.predict(X_test) y_2 = regr_2.predict(X_test) +# %% # Plot the results +# ---------------- +import matplotlib.pyplot as plt + plt.figure() -plt.scatter(X, y, s=20, edgecolor="black", - c="darkorange", label="data") -plt.plot(X_test, y_1, color="cornflowerblue", - label="max_depth=2", linewidth=2) +plt.scatter(X, y, s=20, edgecolor="black", c="darkorange", label="data") +plt.plot(X_test, y_1, color="cornflowerblue", label="max_depth=2", linewidth=2) plt.plot(X_test, y_2, color="yellowgreen", label="max_depth=5", linewidth=2) plt.xlabel("data") plt.ylabel("target") plt.title("Decision Tree Regression") plt.legend() plt.show() + +# %% +# As you can see, the model with a depth of 5 (yellow) learns the details of the +# training data to the point that it overfits to the noise. On the other hand, +# the model with a depth of 2 (blue) learns the major tendencies in the data well +# and does not overfit. In real use cases, you need to make sure that the tree +# is not overfitting the training data, which can be done using cross-validation. + +# %% +# Decision Tree Regression with Multi-Output Targets +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# Here the :ref:`decision trees ` +# is used to predict simultaneously the noisy `x` and `y` observations of a circle +# given a single underlying feature. As a result, it learns local linear +# regressions approximating the circle. +# +# We can see that if the maximum depth of the tree (controlled by the +# `max_depth` parameter) is set too high, the decision trees learn too fine +# details of the training data and learn from the noise, i.e. they overfit. + +# %% +# Create a random dataset +# ----------------------- +rng = np.random.RandomState(1) +X = np.sort(200 * rng.rand(100, 1) - 100, axis=0) +y = np.array([np.pi * np.sin(X).ravel(), np.pi * np.cos(X).ravel()]).T +y[::5, :] += 0.5 - rng.rand(20, 2) + +# %% +# Fit regression model +# -------------------- +regr_1 = DecisionTreeRegressor(max_depth=2) +regr_2 = DecisionTreeRegressor(max_depth=5) +regr_3 = DecisionTreeRegressor(max_depth=8) +regr_1.fit(X, y) +regr_2.fit(X, y) +regr_3.fit(X, y) + +# %% +# Predict +# ------- +# Get predictions on the test set +X_test = np.arange(-100.0, 100.0, 0.01)[:, np.newaxis] +y_1 = regr_1.predict(X_test) +y_2 = regr_2.predict(X_test) +y_3 = regr_3.predict(X_test) + +# %% +# Plot the results +# ---------------- +plt.figure() +s = 25 +plt.scatter(y[:, 0], y[:, 1], c="yellow", s=s, edgecolor="black", label="data") +plt.scatter( + y_1[:, 0], + y_1[:, 1], + c="cornflowerblue", + s=s, + edgecolor="black", + label="max_depth=2", +) +plt.scatter(y_2[:, 0], y_2[:, 1], c="red", s=s, edgecolor="black", label="max_depth=5") +plt.scatter(y_3[:, 0], y_3[:, 1], c="blue", s=s, edgecolor="black", label="max_depth=8") +plt.xlim([-6, 6]) +plt.ylim([-6, 6]) +plt.xlabel("target 1") +plt.ylabel("target 2") +plt.title("Multi-output Decision Tree Regression") +plt.legend(loc="best") +plt.show() + +# %% +# As you can see, the higher the value of `max_depth`, the more details of the data +# are caught by the model. However, the model also overfits to the data and is +# influenced by the noise. diff --git a/examples/tree/plot_tree_regression_multioutput.py b/examples/tree/plot_tree_regression_multioutput.py deleted file mode 100644 index b47bfcd80e49a..0000000000000 --- a/examples/tree/plot_tree_regression_multioutput.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -=================================================================== -Multi-output Decision Tree Regression -=================================================================== - -An example to illustrate multi-output regression with decision tree. - -The :ref:`decision trees ` -is used to predict simultaneously the noisy x and y observations of a circle -given a single underlying feature. As a result, it learns local linear -regressions approximating the circle. - -We can see that if the maximum depth of the tree (controlled by the -`max_depth` parameter) is set too high, the decision trees learn too fine -details of the training data and learn from the noise, i.e. they overfit. -""" -print(__doc__) - -import numpy as np -import matplotlib.pyplot as plt -from sklearn.tree import DecisionTreeRegressor - -# Create a random dataset -rng = np.random.RandomState(1) -X = np.sort(200 * rng.rand(100, 1) - 100, axis=0) -y = np.array([np.pi * np.sin(X).ravel(), np.pi * np.cos(X).ravel()]).T -y[::5, :] += (0.5 - rng.rand(20, 2)) - -# Fit regression model -regr_1 = DecisionTreeRegressor(max_depth=2) -regr_2 = DecisionTreeRegressor(max_depth=5) -regr_3 = DecisionTreeRegressor(max_depth=8) -regr_1.fit(X, y) -regr_2.fit(X, y) -regr_3.fit(X, y) - -# Predict -X_test = np.arange(-100.0, 100.0, 0.01)[:, np.newaxis] -y_1 = regr_1.predict(X_test) -y_2 = regr_2.predict(X_test) -y_3 = regr_3.predict(X_test) - -# Plot the results -plt.figure() -s = 25 -plt.scatter(y[:, 0], y[:, 1], c="navy", s=s, - edgecolor="black", label="data") -plt.scatter(y_1[:, 0], y_1[:, 1], c="cornflowerblue", s=s, - edgecolor="black", label="max_depth=2") -plt.scatter(y_2[:, 0], y_2[:, 1], c="red", s=s, - edgecolor="black", label="max_depth=5") -plt.scatter(y_3[:, 0], y_3[:, 1], c="orange", s=s, - edgecolor="black", label="max_depth=8") -plt.xlim([-6, 6]) -plt.ylim([-6, 6]) -plt.xlabel("target 1") -plt.ylabel("target 2") -plt.title("Multi-output Decision Tree Regression") -plt.legend(loc="best") -plt.show() diff --git a/examples/tree/plot_unveil_tree_structure.py b/examples/tree/plot_unveil_tree_structure.py index 81f67c615c94c..46b89c1d098a6 100644 --- a/examples/tree/plot_unveil_tree_structure.py +++ b/examples/tree/plot_unveil_tree_structure.py @@ -15,13 +15,17 @@ - the decision path shared by a group of samples. """ + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import numpy as np from matplotlib import pyplot as plt -from sklearn.model_selection import train_test_split +from sklearn import tree from sklearn.datasets import load_iris +from sklearn.model_selection import train_test_split from sklearn.tree import DecisionTreeClassifier -from sklearn import tree ############################################################################## # Train tree classifier @@ -43,25 +47,29 @@ # # The decision classifier has an attribute called ``tree_`` which allows access # to low level attributes such as ``node_count``, the total number of nodes, -# and ``max_depth``, the maximal depth of the tree. It also stores the -# entire binary tree structure, represented as a number of parallel arrays. The -# i-th element of each array holds information about the node ``i``. Node 0 is -# the tree's root. Some of the arrays only apply to either leaves or split -# nodes. In this case the values of the nodes of the other type is arbitrary. -# For example, the arrays ``feature`` and ``threshold`` only apply to split -# nodes. The values for leaf nodes in these arrays are therefore arbitrary. +# and ``max_depth``, the maximal depth of the tree. The +# ``tree_.compute_node_depths()`` method computes the depth of each node in the +# tree. `tree_` also stores the entire binary tree structure, represented as a +# number of parallel arrays. The i-th element of each array holds information +# about the node ``i``. Node 0 is the tree's root. Some of the arrays only +# apply to either leaves or split nodes. In this case the values of the nodes +# of the other type is arbitrary. For example, the arrays ``feature`` and +# ``threshold`` only apply to split nodes. The values for leaf nodes in these +# arrays are therefore arbitrary. # # Among these arrays, we have: # -# - ``children_left[i]``: id of the left child of node ``i`` or -1 if leaf -# node -# - ``children_right[i]``: id of the right child of node ``i`` or -1 if leaf -# node -# - ``feature[i]``: feature used for splitting node ``i`` -# - ``threshold[i]``: threshold value at node ``i`` -# - ``n_node_samples[i]``: the number of of training samples reaching node -# ``i`` -# - ``impurity[i]``: the impurity at node ``i`` +# - ``children_left[i]``: id of the left child of node ``i`` or -1 if leaf node +# - ``children_right[i]``: id of the right child of node ``i`` or -1 if leaf node +# - ``feature[i]``: feature used for splitting node ``i`` +# - ``threshold[i]``: threshold value at node ``i`` +# - ``n_node_samples[i]``: the number of training samples reaching node ``i`` +# - ``impurity[i]``: the impurity at node ``i`` +# - ``weighted_n_node_samples[i]``: the weighted number of training samples +# reaching node ``i`` +# - ``value[i, j, k]``: the summary of the training samples that reached node i for +# output j and class k (for regression tree, class is set to 1). See below +# for more information about ``value``. # # Using the arrays, we can traverse the tree structure to compute various # properties. Below, we will compute the depth of each node and whether or not @@ -72,6 +80,7 @@ children_right = clf.tree_.children_right feature = clf.tree_.feature threshold = clf.tree_.threshold +values = clf.tree_.value node_depth = np.zeros(shape=n_nodes, dtype=np.int64) is_leaves = np.zeros(shape=n_nodes, dtype=bool) @@ -92,27 +101,70 @@ else: is_leaves[node_id] = True -print("The binary tree structure has {n} nodes and has " - "the following tree structure:\n".format(n=n_nodes)) +print( + "The binary tree structure has {n} nodes and has " + "the following tree structure:\n".format(n=n_nodes) +) for i in range(n_nodes): if is_leaves[i]: - print("{space}node={node} is a leaf node.".format( - space=node_depth[i] * "\t", node=i)) + print( + "{space}node={node} is a leaf node with value={value}.".format( + space=node_depth[i] * "\t", node=i, value=np.around(values[i], 3) + ) + ) else: - print("{space}node={node} is a split node: " - "go to node {left} if X[:, {feature}] <= {threshold} " - "else to node {right}.".format( - space=node_depth[i] * "\t", - node=i, - left=children_left[i], - feature=feature[i], - threshold=threshold[i], - right=children_right[i])) + print( + "{space}node={node} is a split node with value={value}: " + "go to node {left} if X[:, {feature}] <= {threshold} " + "else to node {right}.".format( + space=node_depth[i] * "\t", + node=i, + left=children_left[i], + feature=feature[i], + threshold=threshold[i], + right=children_right[i], + value=np.around(values[i], 3), + ) + ) + +# %% +# What is the values array used here? +# ----------------------------------- +# The `tree_.value` array is a 3D array of shape +# [``n_nodes``, ``n_classes``, ``n_outputs``] which provides the proportion of samples +# reaching a node for each class and for each output. +# Each node has a ``value`` array which is the proportion of weighted samples reaching +# this node for each output and class with respect to the parent node. +# +# One could convert this to the absolute weighted number of samples reaching a node, +# by multiplying this number by `tree_.weighted_n_node_samples[node_idx]` for the +# given node. Note sample weights are not used in this example, so the weighted +# number of samples is the number of samples reaching the node because each sample +# has a weight of 1 by default. +# +# For example, in the above tree built on the iris dataset, the root node has +# ``value = [0.33, 0.304, 0.366]`` indicating there are 33% of class 0 samples, +# 30.4% of class 1 samples, and 36.6% of class 2 samples at the root node. One can +# convert this to the absolute number of samples by multiplying by the number of +# samples reaching the root node, which is `tree_.weighted_n_node_samples[0]`. +# Then the root node has ``value = [37, 34, 41]``, indicating there are 37 samples +# of class 0, 34 samples of class 1, and 41 samples of class 2 at the root node. +# +# Traversing the tree, the samples are split and as a result, the ``value`` array +# reaching each node changes. The left child of the root node has ``value = [1., 0, 0]`` +# (or ``value = [37, 0, 0]`` when converted to the absolute number of samples) +# because all 37 samples in the left child node are from class 0. +# +# Note: In this example, `n_outputs=1`, but the tree classifier can also handle +# multi-output problems. The `value` array at each node would just be a 2D +# array instead. ############################################################################## # We can compare the above output to the plot of the decision tree. +# Here, we show the proportions of samples of each class that reach each +# node corresponding to the actual elements of `tree_.value` array. -tree.plot_tree(clf) +tree.plot_tree(clf, proportion=True) plt.show() ############################################################################## @@ -139,29 +191,33 @@ sample_id = 0 # obtain ids of the nodes `sample_id` goes through, i.e., row `sample_id` -node_index = node_indicator.indices[node_indicator.indptr[sample_id]: - node_indicator.indptr[sample_id + 1]] +node_index = node_indicator.indices[ + node_indicator.indptr[sample_id] : node_indicator.indptr[sample_id + 1] +] -print('Rules used to predict sample {id}:\n'.format(id=sample_id)) +print("Rules used to predict sample {id}:\n".format(id=sample_id)) for node_id in node_index: # continue to the next node if it is a leaf node if leaf_id[sample_id] == node_id: continue # check if value of the split feature for sample 0 is below threshold - if (X_test[sample_id, feature[node_id]] <= threshold[node_id]): + if X_test[sample_id, feature[node_id]] <= threshold[node_id]: threshold_sign = "<=" else: threshold_sign = ">" - print("decision node {node} : (X_test[{sample}, {feature}] = {value}) " - "{inequality} {threshold})".format( - node=node_id, - sample=sample_id, - feature=feature[node_id], - value=X_test[sample_id, feature[node_id]], - inequality=threshold_sign, - threshold=threshold[node_id])) + print( + "decision node {node} : (X_test[{sample}, {feature}] = {value}) " + "{inequality} {threshold})".format( + node=node_id, + sample=sample_id, + feature=feature[node_id], + value=X_test[sample_id, feature[node_id]], + inequality=threshold_sign, + threshold=threshold[node_id], + ) + ) ############################################################################## # For a group of samples, we can determine the common nodes the samples go @@ -169,12 +225,13 @@ sample_ids = [0, 1] # boolean array indicating the nodes both samples go through -common_nodes = (node_indicator.toarray()[sample_ids].sum(axis=0) == - len(sample_ids)) +common_nodes = node_indicator.toarray()[sample_ids].sum(axis=0) == len(sample_ids) # obtain node ids using position in array common_node_id = np.arange(n_nodes)[common_nodes] -print("\nThe following samples {samples} share the node(s) {nodes} in the " - "tree.".format(samples=sample_ids, nodes=common_node_id)) -print("This is {prop}% of all nodes.".format( - prop=100 * len(common_node_id) / n_nodes)) +print( + "\nThe following samples {samples} share the node(s) {nodes} in the tree.".format( + samples=sample_ids, nodes=common_node_id + ) +) +print("This is {prop}% of all nodes.".format(prop=100 * len(common_node_id) / n_nodes)) diff --git a/lgtm.yml b/lgtm.yml deleted file mode 100644 index 5394cc1664bc9..0000000000000 --- a/lgtm.yml +++ /dev/null @@ -1,8 +0,0 @@ -extraction: - cpp: - before_index: - - pip3 install numpy==1.16.3 - - pip3 install --no-deps scipy Cython - index: - build_command: - - python3 setup.py build_ext -i diff --git a/maint_tools/bump-dependencies-versions.py b/maint_tools/bump-dependencies-versions.py new file mode 100644 index 0000000000000..58be1816f71a3 --- /dev/null +++ b/maint_tools/bump-dependencies-versions.py @@ -0,0 +1,171 @@ +import re +import subprocess +import sys +from datetime import datetime +from pathlib import Path + +import pandas as pd +import requests +from packaging import version + +df_list = pd.read_html("https://devguide.python.org/versions/") +df = pd.concat(df_list).astype({"Branch": str}) +release_dates = {} +python_version_info = { + version: release_date + for version, release_date in zip(df["Branch"], df["First release"]) +} +python_version_info = { + version: pd.to_datetime(release_date) + for version, release_date in python_version_info.items() +} + + +def get_min_version_with_wheel(package_name, python_version): + # For compiled dependencies we want the oldest minor version that has + # wheels for 'python_version' + url = f"https://pypi.org/pypi/{package_name}/json" + response = requests.get(url) + if response.status_code != 200: + return None + + data = response.json() + releases = data["releases"] + + compatible_versions = [] + # We want only minor X.Y.0 and not bugfix X.Y.Z + minor_releases = [ + (ver, release_info) + for ver, release_info in releases.items() + if re.match(r"^\d+\.\d+\.0$", ver) + ] + for ver, release_info in minor_releases: + for file_info in release_info: + if ( + file_info["packagetype"] == "bdist_wheel" + and f"cp{python_version.replace('.', '')}" in file_info["filename"] + and not file_info["yanked"] + ): + compatible_versions.append(ver) + break + + if not compatible_versions: + return None + + return min(compatible_versions, key=version.parse) + + +def get_min_python_version(scikit_learn_release_date_str="today"): + # min Python version is the most recent Python release at least 3 years old + # at the time of the scikit-learn release + if scikit_learn_release_date_str == "today": + scikit_learn_release_date = pd.to_datetime(datetime.now().date()) + else: + scikit_learn_release_date = datetime.strptime( + scikit_learn_release_date_str, "%Y-%m-%d" + ) + version_and_releases = [ + {"python_version": python_version, "python_release_date": python_release_date} + for python_version, python_release_date in python_version_info.items() + if (scikit_learn_release_date - python_release_date).days > 365 * 3 + ] + return max(version_and_releases, key=lambda each: each["python_release_date"])[ + "python_version" + ] + + +def get_min_version_pure_python(package_name, scikit_learn_release_date_str="today"): + # for pure Python dependencies we want the most recent minor release that + # is at least 2 years old + if scikit_learn_release_date_str == "today": + scikit_learn_release_date = pd.to_datetime(datetime.now().date()) + else: + scikit_learn_release_date = datetime.strptime( + scikit_learn_release_date_str, "%Y-%m-%d" + ) + + url = f"https://pypi.org/pypi/{package_name}/json" + response = requests.get(url) + if response.status_code != 200: + return None + + data = response.json() + releases = data["releases"] + + compatible_versions = [] + # We want only minor X.Y.0 and not bugfix X.Y.Z + releases = [ + (ver, release_info) + for ver, release_info in releases.items() + if re.match(r"^\d+\.\d+\.0$", ver) + ] + for ver, release_info in releases: + for file_info in release_info: + if ( + file_info["packagetype"] == "bdist_wheel" + and not file_info["yanked"] + and ( + scikit_learn_release_date - pd.to_datetime(file_info["upload_time"]) + ).days + > 365 * 2 + ): + compatible_versions.append(ver) + break + + if not compatible_versions: + return None + + return max(compatible_versions, key=version.parse) + + +def get_current_dependencies_version(dep): + return ( + subprocess.check_output([sys.executable, "sklearn/_min_dependencies.py", dep]) + .decode() + .strip() + ) + + +def get_current_min_python_version(): + content = Path("pyproject.toml").read_text() + min_python = re.findall(r'requires-python\s*=\s*">=(\d+\.\d+)"', content)[0] + + return min_python + + +def show_versions_update(scikit_learn_release_date="today"): + future_versions = {"python": get_min_python_version(scikit_learn_release_date)} + + compiled_dependencies = ["numpy", "scipy", "pandas", "matplotlib", "pyamg"] + future_versions.update( + { + dep: get_min_version_with_wheel(dep, future_versions["python"]) + for dep in compiled_dependencies + } + ) + + pure_python_dependencies = ["joblib", "threadpoolctl"] + future_versions.update( + { + dep: get_min_version_pure_python(dep, scikit_learn_release_date) + for dep in pure_python_dependencies + } + ) + + current_versions = {"python": get_current_min_python_version()} + current_versions.update( + { + dep: get_current_dependencies_version(dep) + for dep in compiled_dependencies + pure_python_dependencies + } + ) + + print(f"For future release at date {scikit_learn_release_date}") + for k in future_versions: + if future_versions[k] != current_versions[k]: + print(f"- {k}: {current_versions[k]} -> {future_versions[k]}") + + +if __name__ == "__main__": + scikit_learn_release_date = sys.argv[1] if len(sys.argv) > 1 else "today" + show_versions_update(scikit_learn_release_date) diff --git a/maint_tools/check_pxd_in_installation.py b/maint_tools/check_pxd_in_installation.py deleted file mode 100644 index 83c4b706294ad..0000000000000 --- a/maint_tools/check_pxd_in_installation.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Utility for testing presence and usability of .pxd files in the installation - -Usage: ------- -python check_pxd_in_installation.py path/to/install_dir/of/scikit-learn -""" - -import os -import sys -import pathlib -import tempfile -import textwrap -import subprocess - - -sklearn_dir = pathlib.Path(sys.argv[1]) -pxd_files = list(sklearn_dir.glob("**/*.pxd")) - -print("> Found pxd files:") -for pxd_file in pxd_files: - print(' -', pxd_file) - -print("\n> Trying to compile a cython extension cimporting all corresponding " - "modules\n") -with tempfile.TemporaryDirectory() as tmpdir: - tmpdir = pathlib.Path(tmpdir) - # A cython test file which cimports all modules corresponding to found - # pxd files. - # e.g. sklearn/tree/_utils.pxd becomes `cimport sklearn.tree._utils` - with open(tmpdir / 'tst.pyx', 'w') as f: - for pxd_file in pxd_files: - to_import = str(pxd_file.relative_to(sklearn_dir)) - to_import = to_import.replace(os.path.sep, '.') - to_import = to_import.replace('.pxd', '') - f.write('cimport sklearn.' + to_import + '\n') - - # A basic setup file to build the test file. - # We set the language to c++ and we use numpy.get_include() because - # some modules require it. - with open(tmpdir / 'setup_tst.py', 'w') as f: - f.write(textwrap.dedent( - """ - from distutils.core import setup - from distutils.extension import Extension - from Cython.Build import cythonize - import numpy - - extensions = [Extension("tst", - sources=["tst.pyx"], - language="c++", - include_dirs=[numpy.get_include()])] - - setup(ext_modules=cythonize(extensions)) - """)) - - subprocess.run(["python", "setup_tst.py", "build_ext", "-i"], - check=True, cwd=tmpdir) - - print("\n> Compilation succeeded !") diff --git a/maint_tools/check_xfailed_checks.py b/maint_tools/check_xfailed_checks.py new file mode 100644 index 0000000000000..d1108c6ab51a5 --- /dev/null +++ b/maint_tools/check_xfailed_checks.py @@ -0,0 +1,37 @@ +# This script checks that the common tests marked with xfail are actually +# failing. +# Note that in some cases, a test might be marked with xfail because it is +# failing on certain machines, and might not be triggered by this script. + +import contextlib +import io + +from sklearn.utils._test_common.instance_generator import ( + _get_expected_failed_checks, + _tested_estimators, +) +from sklearn.utils.estimator_checks import check_estimator + +for estimator in _tested_estimators(): + # calling check_estimator w/o passing expected_failed_checks will find + # all the failing tests in your environment. + # suppress stdout/stderr while running checks + with ( + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()), + ): + check_results = check_estimator(estimator, on_skip=None, on_fail=None) + failed_tests = [e for e in check_results if e["status"] == "failed"] + failed_test_names = set(e["check_name"] for e in failed_tests) + expected_failed_tests = set(_get_expected_failed_checks(estimator).keys()) + unexpected_failures = failed_test_names - expected_failed_tests + if unexpected_failures: + print(f"{estimator.__class__.__name__} failed with unexpected failures:") + for failure in unexpected_failures: + print(f" {failure}") + + expected_but_not_raised = expected_failed_tests - failed_test_names + if expected_but_not_raised: + print(f"{estimator.__class__.__name__} did not fail expected failures:") + for failure in expected_but_not_raised: + print(f" {failure}") diff --git a/maint_tools/sort_whats_new.py b/maint_tools/sort_whats_new.py index d977c14c248c0..aae5f8067a21e 100755 --- a/maint_tools/sort_whats_new.py +++ b/maint_tools/sort_whats_new.py @@ -2,44 +2,42 @@ # Sorts what's new entries with per-module headings. # Pass what's new entries on stdin. -import sys import re +import sys from collections import defaultdict -LABEL_ORDER = ['MajorFeature', 'Feature', 'Enhancement', 'Efficiency', - 'Fix', 'API'] +LABEL_ORDER = ["MajorFeature", "Feature", "Efficiency", "Enhancement", "Fix", "API"] def entry_sort_key(s): - if s.startswith('- |'): - return LABEL_ORDER.index(s.split('|')[1]) + if s.startswith("- |"): + return LABEL_ORDER.index(s.split("|")[1]) else: return -1 # discard headings and other non-entry lines -text = ''.join(l for l in sys.stdin - if l.startswith('- ') or l.startswith(' ')) +text = "".join(l for l in sys.stdin if l.startswith("- ") or l.startswith(" ")) bucketed = defaultdict(list) -for entry in re.split('\n(?=- )', text.strip()): - modules = re.findall(r':(?:func|meth|mod|class):' - r'`(?:[^<`]*<|~)?(?:sklearn.)?([a-z]\w+)', - entry) +for entry in re.split("\n(?=- )", text.strip()): + modules = re.findall( + r":(?:func|meth|mod|class):`(?:[^<`]*<|~)?(?:sklearn.)?([a-z]\w+)", entry + ) modules = set(modules) if len(modules) > 1: - key = 'Multiple modules' + key = "Multiple modules" elif modules: - key = ':mod:`sklearn.%s`' % next(iter(modules)) + key = ":mod:`sklearn.%s`" % next(iter(modules)) else: - key = 'Miscellaneous' + key = "Miscellaneous" bucketed[key].append(entry) - entry = entry.strip() + '\n' + entry = entry.strip() + "\n" everything = [] for key, bucket in sorted(bucketed.items()): - everything.append(key + '\n' + '.' * len(key)) + everything.append(key + "\n" + "." * len(key)) bucket.sort(key=entry_sort_key) everything.extend(bucket) -print('\n\n'.join(everything)) +print("\n\n".join(everything)) diff --git a/maint_tools/test_docstrings.py b/maint_tools/test_docstrings.py deleted file mode 100644 index f2d38596d4dcd..0000000000000 --- a/maint_tools/test_docstrings.py +++ /dev/null @@ -1,222 +0,0 @@ -import re -from inspect import signature -from typing import Optional - -import pytest -from sklearn.utils._testing import all_estimators - -numpydoc_validation = pytest.importorskip("numpydoc.validate") - -# List of whitelisted modules and methods; regexp are supported. -DOCSTRING_WHITELIST = [ - "LogisticRegression$", - "LogisticRegression.fit", - "LogisticRegression.decision_function", - "Birch.predict", - "Birch.transform", - "GradientBoostingClassifier", - "GradientBoostingRegressor", - "LinearDiscriminantAnalysis.decision_function", - "LinearSVC.decision_function", - "LogisticRegressionCV.decision_function", - "OPTICS", - "OPTICS.fit", - "PassiveAggressiveClassifier.decision_function", - "Perceptron.decision_function", - "RidgeClassifier.decision_function", - "RidgeClassifier.fit", - "RidgeClassifierCV.decision_function", - "KernelDensity", - "KernelDensity.fit", - "KernelDensity.score", - "DecisionTreeClassifier", - "DecisionTreeRegressor", - "LinearRegression$", - "SGDClassifier.decision_function", - "SGDClassifier.set_params", - "SGDClassifier.get_params", - "SGDClassifier.fit", - "SGDClassifier.partial_fit", - "SGDClassifier.predict$", # $ to avoid match w/ predict_proba (regex) - "SGDClassifier.score", - "SGDClassifier.sparsify", - "SGDClassifier.densify", - "VotingClassifier.fit", - "VotingClassifier.transform", - "VotingClassifier.predict", - "VotingClassifier.score", - "VotingClassifier.predict_proba", - "VotingClassifier.set_params", - "VotingClassifier.get_params", - "VotingClassifier.named_estimators", - "VotingClassifier$", -] - - -def get_all_methods(): - estimators = all_estimators() - for name, Estimator in estimators: - if name.startswith("_"): - # skip private classes - continue - methods = [] - for name in dir(Estimator): - if name.startswith("_"): - continue - method_obj = getattr(Estimator, name) - if (hasattr(method_obj, '__call__') - or isinstance(method_obj, property)): - methods.append(name) - methods.append(None) - - for method in sorted(methods, key=lambda x: str(x)): - yield Estimator, method - - -def filter_errors(errors, method): - """ - Ignore some errors based on the method type. - - These rules are specific for scikit-learn.""" - for code, message in errors: - # We ignore following error code, - # - RT02: The first line of the Returns section - # should contain only the type, .. - # (as we may need refer to the name of the returned - # object) - # - GL01: Docstring text (summary) should start in the line - # immediately after the opening quotes (not in the same line, - # or leaving a blank line in between) - - if code in ["RT02", "GL01"]: - continue - - # Following codes are only taken into account for the - # top level class docstrings: - # - ES01: No extended summary found - # - SA01: See Also section not found - # - EX01: No examples section found - - if method is not None and code in ["EX01", "SA01", "ES01"]: - continue - yield code, message - - -def repr_errors(res, estimator=None, method: Optional[str] = None) -> str: - """Pretty print original docstring and the obtained errors - - Parameters - ---------- - res : dict - result of numpydoc.validate.validate - estimator : {estimator, None} - estimator object or None - method : str - if estimator is not None, either the method name or None. - - Returns - ------- - str - String representation of the error. - """ - if method is None: - if hasattr(estimator, "__init__"): - method = "__init__" - elif estimator is None: - raise ValueError( - "At least one of estimator, method should be provided" - ) - else: - raise NotImplementedError - - if estimator is not None: - obj = getattr(estimator, method) - try: - obj_signature = signature(obj) - except TypeError: - # In particular we can't parse the signature of properties - obj_signature = ( - "\nParsing of the method signature failed, " - "possibly because this is a property." - ) - - obj_name = estimator.__name__ + "." + method - else: - obj_signature = "" - obj_name = method - - msg = "\n\n" + "\n\n".join( - [ - str(res["file"]), - obj_name + str(obj_signature), - res["docstring"], - "# Errors", - "\n".join( - " - {}: {}".format(code, message) - for code, message in res["errors"] - ), - ] - ) - return msg - - -@pytest.mark.parametrize("Estimator, method", get_all_methods()) -def test_docstring(Estimator, method, request): - base_import_path = Estimator.__module__ - import_path = [base_import_path, Estimator.__name__] - if method is not None: - import_path.append(method) - - import_path = ".".join(import_path) - - if not any(re.search(regex, import_path) for regex in DOCSTRING_WHITELIST): - request.applymarker( - pytest.mark.xfail( - run=False, reason="TODO pass numpydoc validation" - ) - ) - - res = numpydoc_validation.validate(import_path) - - res["errors"] = list(filter_errors(res["errors"], method)) - - if res["errors"]: - msg = repr_errors(res, Estimator, method) - - raise ValueError(msg) - - -if __name__ == "__main__": - import sys - import argparse - - parser = argparse.ArgumentParser( - description="Validate docstring with numpydoc." - ) - parser.add_argument("import_path", help="Import path to validate") - - args = parser.parse_args() - - res = numpydoc_validation.validate(args.import_path) - - import_path_sections = args.import_path.split(".") - # When applied to classes, detect class method. For functions - # method = None. - # TODO: this detection can be improved. Currently we assume that we have - # class # methods if the second path element before last is in camel case. - if len(import_path_sections) >= 2 and re.match( - r"(?:[A-Z][a-z]*)+", import_path_sections[-2] - ): - method = import_path_sections[-1] - else: - method = None - - res["errors"] = list(filter_errors(res["errors"], method)) - - if res["errors"]: - msg = repr_errors(res, method=args.import_path) - - print(msg) - sys.exit(1) - else: - print("All docstring checks passed for {}!".format(args.import_path)) diff --git a/maint_tools/update_tracking_issue.py b/maint_tools/update_tracking_issue.py new file mode 100644 index 0000000000000..b40e8222fefae --- /dev/null +++ b/maint_tools/update_tracking_issue.py @@ -0,0 +1,163 @@ +"""Creates or updates an issue if the CI fails. This is useful to keep track of +scheduled jobs that are failing repeatedly. + +This script depends on: +- `defusedxml` for safer parsing for xml +- `PyGithub` for interacting with GitHub + +The GitHub token only requires the `repo:public_repo` scope are described in +https://docs.github.com/en/developers/apps/building-oauth-apps/scopes-for-oauth-apps#available-scopes. +This scope allows the bot to create and edit its own issues. It is best to use a +github account that does **not** have commit access to the public repo. +""" + +import argparse +import sys +from datetime import datetime, timezone +from pathlib import Path + +import defusedxml.ElementTree as ET +from github import Github + +parser = argparse.ArgumentParser( + description="Create or update issue from JUnit test results from pytest" +) +parser.add_argument( + "bot_github_token", help="Github token for creating or updating an issue" +) +parser.add_argument("ci_name", help="Name of CI run instance") +parser.add_argument("issue_repo", help="Repo to track issues") +parser.add_argument("link_to_ci_run", help="URL to link to") +parser.add_argument("--junit-file", help="JUnit file to determine if tests passed") +parser.add_argument( + "--tests-passed", + help=( + "If --tests-passed is true, then the original issue is closed if the issue " + "exists. If tests-passed is false, then the an issue is updated or created." + ), +) +parser.add_argument( + "--auto-close", + help=( + "If --auto-close is false, then issues will not auto close even if the tests" + " pass." + ), + default="true", +) + +args = parser.parse_args() + +if args.junit_file is not None and args.tests_passed is not None: + print("--junit-file and --test-passed can not be set together") + sys.exit(1) + +if args.junit_file is None and args.tests_passed is None: + print("Either --junit-file or --test-passed must be passed in") + sys.exit(1) + +gh = Github(args.bot_github_token) +issue_repo = gh.get_repo(args.issue_repo) +dt_now = datetime.now(tz=timezone.utc) +date_str = dt_now.strftime("%b %d, %Y") +title_query = f"CI failed on {args.ci_name}" +title = f"⚠ī¸ {title_query} (last failure: {date_str}) ⚠ī¸" + + +def get_issue(): + login = gh.get_user().login + issues = gh.search_issues( + f"repo:{args.issue_repo} {title_query} in:title state:open author:{login}" + ) + first_page = issues.get_page(0) + # Return issue if it exist + return first_page[0] if first_page else None + + +def create_or_update_issue(body=""): + # Interact with GitHub API to create issue + link = f"[{args.ci_name}]({args.link_to_ci_run})" + issue = get_issue() + + max_body_length = 60_000 + original_body_length = len(body) + # Avoid "body is too long (maximum is 65536 characters)" error from github REST API + if original_body_length > max_body_length: + body = ( + f"{body[:max_body_length]}\n...\n" + f"Body was too long ({original_body_length} characters) and was shortened" + ) + + if issue is None: + # Create new issue + header = f"**CI failed on {link}** ({date_str})" + issue = issue_repo.create_issue(title=title, body=f"{header}\n{body}") + print(f"Created issue in {args.issue_repo}#{issue.number}") + sys.exit() + else: + # Update existing issue + header = f"**CI is still failing on {link}** ({date_str})" + issue.edit(title=title, body=f"{header}\n{body}") + print(f"Commented on issue: {args.issue_repo}#{issue.number}") + sys.exit() + + +def close_issue_if_opened(): + print("Test has no failures!") + issue = get_issue() + if issue is not None: + header_str = "## CI is no longer failing!" + comment_str = ( + f"{header_str} ✅\n\n[Successful run]({args.link_to_ci_run}) on {date_str}" + ) + + print(f"Commented on issue #{issue.number}") + # New comment if "## CI is no longer failing!" comment does not exist + # If it does exist update the original comment which includes the new date + for comment in issue.get_comments(): + if comment.body.startswith(header_str): + comment.edit(body=comment_str) + break + else: # no break + issue.create_comment(body=comment_str) + + if args.auto_close.lower() == "true": + print(f"Closing issue #{issue.number}") + issue.edit(state="closed") + sys.exit() + + +if args.tests_passed is not None: + if args.tests_passed.lower() == "true": + close_issue_if_opened() + else: + create_or_update_issue() + +junit_path = Path(args.junit_file) +if not junit_path.exists(): + body = "Unable to find junit file. Please see link for details." + create_or_update_issue(body) + +# Find failures in junit file +tree = ET.parse(args.junit_file) +failure_cases = [] + +# Check if test collection failed +error = tree.find("./testsuite/testcase/error") +if error is not None: + # Get information for test collection error + failure_cases.append("Test Collection Failure") + +for item in tree.iter("testcase"): + failure = item.find("failure") + if failure is None: + continue + + failure_cases.append(item.attrib["name"]) + +if not failure_cases: + close_issue_if_opened() + +# Create content for issue +body_list = [f"- {case}" for case in failure_cases] +body = "\n".join(body_list) +create_or_update_issue(body) diff --git a/maint_tools/vendor_array_api_compat.sh b/maint_tools/vendor_array_api_compat.sh new file mode 100755 index 0000000000000..51056ce477cbb --- /dev/null +++ b/maint_tools/vendor_array_api_compat.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Vendors https://github.com/data-apis/array-api-compat/ into sklearn/externals + +set -o nounset +set -o errexit + +URL="https://github.com/data-apis/array-api-compat.git" +VERSION="1.12" + +ROOT_DIR=sklearn/externals/array_api_compat + +rm -rf $ROOT_DIR +mkdir $ROOT_DIR +mkdir $ROOT_DIR/.tmp +git clone $URL $ROOT_DIR/.tmp +pushd $ROOT_DIR/.tmp +git checkout $VERSION +popd +mv -v $ROOT_DIR/.tmp/array_api_compat/* $ROOT_DIR/ +mv -v $ROOT_DIR/.tmp/LICENSE $ROOT_DIR/ +rm -rf $ROOT_DIR/.tmp + +echo "Update this directory using maint_tools/vendor_array_api_compat.sh" >$ROOT_DIR/README.md diff --git a/maint_tools/vendor_array_api_extra.sh b/maint_tools/vendor_array_api_extra.sh new file mode 100755 index 0000000000000..ead6e2e62c43f --- /dev/null +++ b/maint_tools/vendor_array_api_extra.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Vendors https://github.com/data-apis/array-api-extra/ into sklearn/externals + +set -o nounset +set -o errexit + +URL="https://github.com/data-apis/array-api-extra.git" +VERSION="v0.7.1" + +ROOT_DIR=sklearn/externals/array_api_extra + +rm -rf $ROOT_DIR +mkdir $ROOT_DIR +mkdir $ROOT_DIR/.tmp +git clone $URL $ROOT_DIR/.tmp +pushd $ROOT_DIR/.tmp +git checkout $VERSION +popd +mv -v $ROOT_DIR/.tmp/src/array_api_extra/* $ROOT_DIR/ +mv -v $ROOT_DIR/.tmp/LICENSE $ROOT_DIR/ +rm -rf $ROOT_DIR/.tmp + +echo "Update this directory using maint_tools/vendor_array_api_extra.sh" >$ROOT_DIR/README.md diff --git a/meson.build b/meson.build new file mode 100644 index 0000000000000..f843a1ff8f45c --- /dev/null +++ b/meson.build @@ -0,0 +1,53 @@ +project( + 'scikit-learn', + 'c', 'cpp', 'cython', + version: run_command('sklearn/_build_utils/version.py', check: true).stdout().strip(), + license: 'BSD-3', + meson_version: '>= 1.1.0', + default_options: [ + 'c_std=c11', + 'cpp_std=c++14', + ], +) + +cc = meson.get_compiler('c') +cpp = meson.get_compiler('cpp') +cython = meson.get_compiler('cython') + +# Check compiler is recent enough (see "Toolchain Roadmap" for details) +if cc.get_id() == 'gcc' + if not cc.version().version_compare('>=8.0') + error('scikit-learn requires GCC >= 8.0') + endif +elif cc.get_id() == 'msvc' + if not cc.version().version_compare('>=19.20') + error('scikit-learn requires at least vc142 (default with Visual Studio 2019) ' + \ + 'when building with MSVC') + endif +endif + +_global_c_args = cc.get_supported_arguments( + '-Wno-unused-but-set-variable', + '-Wno-unused-function', + '-Wno-conversion', + '-Wno-misleading-indentation', +) +add_project_arguments(_global_c_args, language : 'c') + +# We need -lm for all C code (assuming it uses math functions, which is safe to +# assume for scikit-learn). For C++ it isn't needed, because libstdc++/libc++ is +# guaranteed to depend on it. +m_dep = cc.find_library('m', required : false) +if m_dep.found() + add_project_link_arguments('-lm', language : 'c') +endif + +tempita = find_program('sklearn/_build_utils/tempita.py') + +py = import('python').find_installation(pure: false) + +# Copy all the .py files to the install dir, rather than using +# py.install_sources and needing to list them explicitly one by one +install_subdir('sklearn', install_dir: py.get_install_dir()) + +subdir('sklearn') diff --git a/pyproject.toml b/pyproject.toml index c55c68b3182b8..b793bd43dd5df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,15 +1,500 @@ +[project] +name = "scikit-learn" +dynamic = ["version"] +description = "A set of python modules for machine learning and data mining" +readme = "README.rst" +maintainers = [ + {name = "scikit-learn developers", email="scikit-learn@python.org"}, +] +dependencies = [ + "numpy>=1.22.0", + "scipy>=1.8.0", + "joblib>=1.2.0", + "threadpoolctl>=3.1.0", +] +requires-python = ">=3.10" +license = {file = "COPYING"} +classifiers=[ + "Intended Audience :: Science/Research", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Programming Language :: C", + "Programming Language :: Python", + "Topic :: Software Development", + "Topic :: Scientific/Engineering", + "Development Status :: 5 - Production/Stable", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX", + "Operating System :: Unix", + "Operating System :: MacOS", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: Implementation :: CPython", +] + +[project.urls] +homepage = "https://scikit-learn.org" +source = "https://github.com/scikit-learn/scikit-learn" +download = "https://pypi.org/project/scikit-learn/#files" +tracker = "https://github.com/scikit-learn/scikit-learn/issues" +"release notes" = "https://scikit-learn.org/stable/whats_new" + +[project.optional-dependencies] +build = ["numpy>=1.22.0", "scipy>=1.8.0", "cython>=3.0.10", "meson-python>=0.16.0"] +install = ["numpy>=1.22.0", "scipy>=1.8.0", "joblib>=1.2.0", "threadpoolctl>=3.1.0"] +benchmark = ["matplotlib>=3.5.0", "pandas>=1.4.0", "memory_profiler>=0.57.0"] +docs = [ + "matplotlib>=3.5.0", + "scikit-image>=0.19.0", + "pandas>=1.4.0", + "seaborn>=0.9.0", + "memory_profiler>=0.57.0", + "sphinx>=7.3.7", + "sphinx-copybutton>=0.5.2", + "sphinx-gallery>=0.17.1", + "numpydoc>=1.2.0", + "Pillow>=8.4.0", + "pooch>=1.6.0", + "sphinx-prompt>=1.4.0", + "sphinxext-opengraph>=0.9.1", + "plotly>=5.14.0", + "polars>=0.20.30", + "sphinx-design>=0.5.0", + "sphinx-design>=0.6.0", + "sphinxcontrib-sass>=0.3.4", + "pydata-sphinx-theme>=0.15.3", + "sphinx-remove-toctrees>=1.0.0.post1", + "towncrier>=24.8.0", +] +examples = [ + "matplotlib>=3.5.0", + "scikit-image>=0.19.0", + "pandas>=1.4.0", + "seaborn>=0.9.0", + "pooch>=1.6.0", + "plotly>=5.14.0", +] +tests = [ + "matplotlib>=3.5.0", + "scikit-image>=0.19.0", + "pandas>=1.4.0", + "pytest>=7.1.2", + "pytest-cov>=2.9.0", + "ruff>=0.11.7", + "mypy>=1.15", + "pyamg>=4.2.1", + "polars>=0.20.30", + "pyarrow>=12.0.0", + "numpydoc>=1.2.0", + "pooch>=1.6.0", +] +maintenance = ["conda-lock==3.0.1"] + [build-system] +build-backend = "mesonpy" # Minimum requirements for the build system to execute. requires = [ - "setuptools", - "wheel", - "Cython>=0.28.5", + "meson-python>=0.16.0", + "Cython>=3.0.10", + "numpy>=2", + "scipy>=1.8.0", +] - # use oldest-supported-numpy which provides the oldest numpy version with - # wheels on PyPI - # - # see: https://github.com/scipy/oldest-supported-numpy/blob/master/setup.cfg - "oldest-supported-numpy", +[tool.pytest.ini_options] +doctest_optionflags = "NORMALIZE_WHITESPACE ELLIPSIS" +testpaths = "sklearn" +addopts = [ + "--disable-pytest-warnings", + "--color=yes", + "--import-mode=importlib", +] + +[tool.ruff] +line-length = 88 +exclude=[ + ".eggs", + ".git", + ".mypy_cache", + ".vscode", + "__pycache__", + "build", + "dist", + "sklearn/externals", + "doc/_build", + "doc/auto_examples", + "asv_benchmarks/env", + "asv_benchmarks/html", + "asv_benchmarks/results", + "asv_benchmarks/benchmarks/cache", +] + +[tool.ruff.lint] +# This enables us to use CPY001: copyright header check +preview = true +# This enables us to use the explicit preview rules that we want only +explicit-preview-rules = true +# all rules can be found here: https://docs.astral.sh/ruff/rules/ +extend-select = ["E501", "W", "I", "CPY001", "PGH", "RUF"] +ignore=[ + # do not assign a lambda expression, use a def + "E731", + # do not use variables named 'l', 'O', or 'I' + "E741", + # E721 gives many false positives. + # Use `is` and `is not` for type comparisons, or `isinstance()` for + # isinstance checks + "E721", + # We don't care much about F841. + # Local variable ... is assigned to but never used + "F841", + # some RUF rules trigger too many changes + "RUF002", + "RUF003", + "RUF005", + "RUF012", + "RUF015", + "RUF021", + # https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules + "W191", + "E111", + "E114", + "E117", + "D206", + "D300", + "Q000", + "Q001", + "Q002", + "Q003", + "COM812", + "COM819", +] - "scipy>=0.19.1", +[tool.ruff.lint.flake8-copyright] +notice-rgx = "\\#\\ Authors:\\ The\\ scikit\\-learn\\ developers\\\r?\\\n\\#\\ SPDX\\-License\\-Identifier:\\ BSD\\-3\\-Clause" + +[tool.ruff.lint.per-file-ignores] +# It's fine not to put the import at the top of the file in the examples +# folder. +"examples/*"=["E402"] +"doc/conf.py"=["E402"] +"**/tests/*"=["CPY001"] +"asv_benchmarks/*"=["CPY001"] +"benchmarks/*"=["CPY001"] +"doc/*"=["CPY001"] +"build_tools/*"=["CPY001"] +"sklearn/_build_utils/*"=["CPY001"] +"maint_tools/*"=["CPY001"] +".spin/*"=["CPY001"] +".github/*"=["CPY001"] +# __doc__ is too long (>4096 chars) and therefore false positive on copyright check +"examples/model_selection/plot_precision_recall.py"=["CPY001"] +"examples/svm/plot_rbf_parameters.py"=["CPY001"] +# __all__ has un-imported names +"sklearn/__init__.py"=["F822"] + +[tool.mypy] +ignore_missing_imports = true +allow_redefinition = true +exclude = "^sklearn/externals" + +[[tool.mypy.overrides]] +module = ["joblib.*", "sklearn.externals.*"] +follow_imports = "skip" + +[tool.cython-lint] +# Ignore the same error codes as ruff +# + E501 (line too long) because keeping it < 88 in cython +# often makes code less readable. +ignore = [ + # multiple spaces/tab after comma + 'E24', + # line too long + 'E501', + # do not assign a lambda expression, use a def + 'E731', + # do not use variables named 'l', 'O', or 'I' + 'E741', + # line break before binary operator + 'W503', + # line break after binary operator + 'W504', ] +# Exclude files are generated from tempita templates +exclude= ''' +( + asv_benchmarks/ + | sklearn/_loss/_loss.pyx + | sklearn/linear_model/_sag_fast.pyx + | sklearn/linear_model/_sgd_fast.pyx + | sklearn/utils/_seq_dataset.pyx + | sklearn/utils/_seq_dataset.pxd + | sklearn/utils/_weight_vector.pyx + | sklearn/utils/_weight_vector.pxd + | sklearn/metrics/_dist_metrics.pyx + | sklearn/metrics/_dist_metrics.pxd + | sklearn/metrics/_pairwise_distances_reduction/_argkmin.pxd + | sklearn/metrics/_pairwise_distances_reduction/_argkmin.pyx + | sklearn/metrics/_pairwise_distances_reduction/_argkmin_classmode.pyx + | sklearn/metrics/_pairwise_distances_reduction/_base.pxd + | sklearn/metrics/_pairwise_distances_reduction/_base.pyx + | sklearn/metrics/_pairwise_distances_reduction/_datasets_pair.pxd + | sklearn/metrics/_pairwise_distances_reduction/_datasets_pair.pyx + | sklearn/metrics/_pairwise_distances_reduction/_middle_term_computer.pxd + | sklearn/metrics/_pairwise_distances_reduction/_middle_term_computer.pyx + | sklearn/metrics/_pairwise_distances_reduction/_radius_neighbors.pxd + | sklearn/metrics/_pairwise_distances_reduction/_radius_neighbors.pyx +) +''' + +[tool.check-sdist] +# These settings should match .gitattributes +sdist-only = [] +git-only = [".*", "asv_benchmarks", "azure-pipelines.yml", "benchmarks", "build_tools", "maint_tools"] +default-ignore = false + +[tool.spin] +package = "sklearn" # name of your package + +[tool.spin.commands] +"Build" = [ + "spin.cmds.pip.install", + "spin.cmds.meson.test", + ".spin/cmds.py:clean", +] +"Documentation" = [ + "spin.cmds.meson.docs" +] + +[tool.changelog-bot] + [tool.changelog-bot.towncrier_changelog] + enabled = true + verify_pr_number = true + changelog_noop_label = "No Changelog Needed" + whatsnew_pattern = 'doc/whatsnew/upcoming_changes/[^/]+/\d+\.[^.]+\.rst' + +[tool.codespell] +skip = ["./.git", "*.svg", "./.mypy_cache", "./sklearn/feature_extraction/_stop_words.py", "./sklearn/feature_extraction/tests/test_text.py", "./build_tools/wheels/LICENSE_windows.txt", "./doc/_build", "./doc/auto_examples", "./doc/modules/generated"] +ignore-words = "build_tools/codespell_ignore_words.txt" + +[tool.towncrier] + package = "sklearn" + filename = "doc/whats_new/v1.8.rst" + single_file = true + directory = "doc/whats_new/upcoming_changes" + issue_format = ":pr:`{issue}`" + template = "doc/whats_new/upcoming_changes/towncrier_template.rst.jinja2" + all_bullets = false + + [[tool.towncrier.type]] + directory = "major-feature" + name = "|MajorFeature|" + showcontent = true + + [[tool.towncrier.type]] + directory = "feature" + name = "|Feature|" + showcontent = true + + [[tool.towncrier.type]] + directory = "efficiency" + name = "|Efficiency|" + showcontent = true + + [[tool.towncrier.type]] + directory = "enhancement" + name = "|Enhancement|" + showcontent = true + + [[tool.towncrier.type]] + directory = "fix" + name = "|Fix|" + showcontent = true + + [[tool.towncrier.type]] + directory = "api" + name = "|API|" + showcontent = true + + [[tool.towncrier.type]] + directory = "other" + name = "" + showcontent = true + + [[tool.towncrier.section]] + name = "Security" + path = "security" + + [[tool.towncrier.section]] + name = "Changed models" + path = "changed-models" + + [[tool.towncrier.section]] + name = "Changes impacting many modules" + path = "many-modules" + + [[tool.towncrier.section]] + name = "Support for Array API" + path = "array-api" + + [[tool.towncrier.section]] + name = "Metadata routing" + path = "metadata-routing" + + [[tool.towncrier.section]] + name = "custom-top-level" + path = "custom-top-level" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.base`" + path = "sklearn.base" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.calibration`" + path = "sklearn.calibration" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.cluster`" + path = "sklearn.cluster" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.compose`" + path = "sklearn.compose" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.covariance`" + path = "sklearn.covariance" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.cross_decomposition`" + path = "sklearn.cross_decomposition" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.datasets`" + path = "sklearn.datasets" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.decomposition`" + path = "sklearn.decomposition" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.discriminant_analysis`" + path = "sklearn.discriminant_analysis" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.dummy`" + path = "sklearn.dummy" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.ensemble`" + path = "sklearn.ensemble" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.exceptions`" + path = "sklearn.exceptions" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.feature_extraction`" + path = "sklearn.feature_extraction" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.feature_selection`" + path = "sklearn.feature_selection" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.frozen`" + path = "sklearn.frozen" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.gaussian_process`" + path = "sklearn.gaussian_process" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.impute`" + path = "sklearn.impute" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.inspection`" + path = "sklearn.inspection" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.isotonic`" + path = "sklearn.isotonic" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.kernel_approximation`" + path = "sklearn.kernel_approximation" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.kernel_ridge`" + path = "sklearn.kernel_ridge" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.linear_model`" + path = "sklearn.linear_model" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.manifold`" + path = "sklearn.manifold" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.metrics`" + path = "sklearn.metrics" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.mixture`" + path = "sklearn.mixture" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.model_selection`" + path = "sklearn.model_selection" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.multiclass`" + path = "sklearn.multiclass" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.multioutput`" + path = "sklearn.multioutput" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.naive_bayes`" + path = "sklearn.naive_bayes" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.neighbors`" + path = "sklearn.neighbors" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.neural_network`" + path = "sklearn.neural_network" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.pipeline`" + path = "sklearn.pipeline" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.preprocessing`" + path = "sklearn.preprocessing" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.random_projection`" + path = "sklearn.random_projection" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.semi_supervised`" + path = "sklearn.semi_supervised" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.svm`" + path = "sklearn.svm" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.tree`" + path = "sklearn.tree" + + [[tool.towncrier.section]] + name = ":mod:`sklearn.utils`" + path = "sklearn.utils" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 1296273f721c4..0000000000000 --- a/setup.cfg +++ /dev/null @@ -1,45 +0,0 @@ -[aliases] -test = pytest - -[tool:pytest] -# disable-pytest-warnings should be removed once we rewrite tests -# using yield with parametrize -doctest_optionflags = NORMALIZE_WHITESPACE ELLIPSIS -addopts = - --ignore build_tools - --ignore benchmarks - --ignore doc - --ignore examples - --ignore maint_tools - --ignore asv_benchmarks - --doctest-modules - --disable-pytest-warnings - -rxXs - -filterwarnings = - ignore:the matrix subclass:PendingDeprecationWarning - -[wheelhouse_uploader] -artifact_indexes= - # Wheels built by the "Wheel builder" workflow in GitHub actions: - # https://github.com/scikit-learn/scikit-learn/actions?query=workflow%3A%22Wheel+builder%22 - https://pypi.anaconda.org/scikit-learn-wheels-staging/simple/scikit-learn/ - -[flake8] -# Default flake8 3.5 ignored flags -ignore=E121,E123,E126,E226,E24,E704,W503,W504 -# It's fine not to put the import at the top of the file in the examples -# folder. -per-file-ignores = - examples/*: E402 - -[mypy] -ignore_missing_imports = True -allow_redefinition = True - -[check-manifest] -# ignore files missing in VCS -ignore = - sklearn/linear_model/_sag_fast.pyx - sklearn/utils/_seq_dataset.pxd - sklearn/utils/_seq_dataset.pyx diff --git a/setup.py b/setup.py deleted file mode 100755 index e44f941e0a114..0000000000000 --- a/setup.py +++ /dev/null @@ -1,301 +0,0 @@ -#! /usr/bin/env python -# -# Copyright (C) 2007-2009 Cournapeau David -# 2010 Fabian Pedregosa -# License: 3-clause BSD - -import sys -import os -import platform -import shutil - -# We need to import setuptools before because it monkey-patches distutils -import setuptools # noqa -from distutils.command.clean import clean as Clean -from distutils.command.sdist import sdist - -from pkg_resources import parse_version -import traceback -import importlib -try: - import builtins -except ImportError: - # Python 2 compat: just to be able to declare that Python >=3.6 is needed. - import __builtin__ as builtins - -# This is a bit (!) hackish: we are setting a global variable so that the -# main sklearn __init__ can detect if it is being loaded by the setup -# routine, to avoid attempting to load components that aren't built yet: -# the numpy distutils extensions that are used by scikit-learn to -# recursively build the compiled extensions in sub-packages is based on the -# Python import machinery. -builtins.__SKLEARN_SETUP__ = True - - -DISTNAME = 'scikit-learn' -DESCRIPTION = 'A set of python modules for machine learning and data mining' -with open('README.rst') as f: - LONG_DESCRIPTION = f.read() -MAINTAINER = 'Andreas Mueller' -MAINTAINER_EMAIL = 'amueller@ais.uni-bonn.de' -URL = 'http://scikit-learn.org' -DOWNLOAD_URL = 'https://pypi.org/project/scikit-learn/#files' -LICENSE = 'new BSD' -PROJECT_URLS = { - 'Bug Tracker': 'https://github.com/scikit-learn/scikit-learn/issues', - 'Documentation': 'https://scikit-learn.org/stable/documentation.html', - 'Source Code': 'https://github.com/scikit-learn/scikit-learn' -} - -# We can actually import a restricted version of sklearn that -# does not need the compiled code -import sklearn -import sklearn._min_dependencies as min_deps # noqa - - -VERSION = sklearn.__version__ - - -# For some commands, use setuptools -SETUPTOOLS_COMMANDS = { - 'develop', 'release', 'bdist_egg', 'bdist_rpm', - 'bdist_wininst', 'install_egg_info', 'build_sphinx', - 'egg_info', 'easy_install', 'upload', 'bdist_wheel', - '--single-version-externally-managed', -} -if SETUPTOOLS_COMMANDS.intersection(sys.argv): - extra_setuptools_args = dict( - zip_safe=False, # the package can run out of an .egg file - include_package_data=True, - extras_require={ - key: min_deps.tag_to_packages[key] for - key in ['examples', 'docs', 'tests', 'benchmark'] - }, - ) -else: - extra_setuptools_args = dict() - - -# Custom clean command to remove build artifacts - -class CleanCommand(Clean): - description = "Remove build artifacts from the source tree" - - def run(self): - Clean.run(self) - # Remove c files if we are not within a sdist package - cwd = os.path.abspath(os.path.dirname(__file__)) - remove_c_files = not os.path.exists(os.path.join(cwd, 'PKG-INFO')) - if remove_c_files: - print('Will remove generated .c files') - if os.path.exists('build'): - shutil.rmtree('build') - for dirpath, dirnames, filenames in os.walk('sklearn'): - for filename in filenames: - if any(filename.endswith(suffix) for suffix in - (".so", ".pyd", ".dll", ".pyc")): - os.unlink(os.path.join(dirpath, filename)) - continue - extension = os.path.splitext(filename)[1] - if remove_c_files and extension in ['.c', '.cpp']: - pyx_file = str.replace(filename, extension, '.pyx') - if os.path.exists(os.path.join(dirpath, pyx_file)): - os.unlink(os.path.join(dirpath, filename)) - for dirname in dirnames: - if dirname == '__pycache__': - shutil.rmtree(os.path.join(dirpath, dirname)) - - -cmdclass = {'clean': CleanCommand, 'sdist': sdist} - -# Custom build_ext command to set OpenMP compile flags depending on os and -# compiler. Also makes it possible to set the parallelism level via -# and environment variable (useful for the wheel building CI). -# build_ext has to be imported after setuptools -try: - from numpy.distutils.command.build_ext import build_ext # noqa - - class build_ext_subclass(build_ext): - - def finalize_options(self): - super().finalize_options() - if self.parallel is None: - # Do not override self.parallel if already defined by - # command-line flag (--parallel or -j) - - parallel = os.environ.get("SKLEARN_BUILD_PARALLEL") - if parallel: - self.parallel = int(parallel) - if self.parallel: - print("setting parallel=%d " % self.parallel) - - def build_extensions(self): - from sklearn._build_utils.openmp_helpers import get_openmp_flag - - if sklearn._OPENMP_SUPPORTED: - openmp_flag = get_openmp_flag(self.compiler) - - for e in self.extensions: - e.extra_compile_args += openmp_flag - e.extra_link_args += openmp_flag - - build_ext.build_extensions(self) - - cmdclass['build_ext'] = build_ext_subclass - -except ImportError: - # Numpy should not be a dependency just to be able to introspect - # that python 3.6 is required. - pass - - -# Optional wheelhouse-uploader features -# To automate release of binary packages for scikit-learn we need a tool -# to download the packages generated by travis and appveyor workers (with -# version number matching the current release) and upload them all at once -# to PyPI at release time. -# The URL of the artifact repositories are configured in the setup.cfg file. - -WHEELHOUSE_UPLOADER_COMMANDS = {'fetch_artifacts', 'upload_all'} -if WHEELHOUSE_UPLOADER_COMMANDS.intersection(sys.argv): - import wheelhouse_uploader.cmd - - cmdclass.update(vars(wheelhouse_uploader.cmd)) - - -def configuration(parent_package='', top_path=None): - if os.path.exists('MANIFEST'): - os.remove('MANIFEST') - - from numpy.distutils.misc_util import Configuration - from sklearn._build_utils import _check_cython_version - - config = Configuration(None, parent_package, top_path) - - # Avoid non-useful msg: - # "Ignoring attempt to set 'name' (from ... " - config.set_options(ignore_setup_xxx_py=True, - assume_default_configuration=True, - delegate_options_to_subpackages=True, - quiet=True) - - # Cython is required by config.add_subpackage for templated extensions - # that need the tempita sub-submodule. So check that we have the correct - # version of Cython so as to be able to raise a more informative error - # message from the start if it's not the case. - _check_cython_version() - - config.add_subpackage('sklearn') - - return config - - -def check_package_status(package, min_version): - """ - Returns a dictionary containing a boolean specifying whether given package - is up-to-date, along with the version string (empty string if - not installed). - """ - package_status = {} - try: - module = importlib.import_module(package) - package_version = module.__version__ - package_status['up_to_date'] = parse_version( - package_version) >= parse_version(min_version) - package_status['version'] = package_version - except ImportError: - traceback.print_exc() - package_status['up_to_date'] = False - package_status['version'] = "" - - req_str = "scikit-learn requires {} >= {}.\n".format( - package, min_version) - - instructions = ("Installation instructions are available on the " - "scikit-learn website: " - "http://scikit-learn.org/stable/install.html\n") - - if package_status['up_to_date'] is False: - if package_status['version']: - raise ImportError("Your installation of {} " - "{} is out-of-date.\n{}{}" - .format(package, package_status['version'], - req_str, instructions)) - else: - raise ImportError("{} is not " - "installed.\n{}{}" - .format(package, req_str, instructions)) - - -def setup_package(): - metadata = dict(name=DISTNAME, - maintainer=MAINTAINER, - maintainer_email=MAINTAINER_EMAIL, - description=DESCRIPTION, - license=LICENSE, - url=URL, - download_url=DOWNLOAD_URL, - project_urls=PROJECT_URLS, - version=VERSION, - long_description=LONG_DESCRIPTION, - classifiers=['Intended Audience :: Science/Research', - 'Intended Audience :: Developers', - 'License :: OSI Approved', - 'Programming Language :: C', - 'Programming Language :: Python', - 'Topic :: Software Development', - 'Topic :: Scientific/Engineering', - 'Development Status :: 5 - Production/Stable', - 'Operating System :: Microsoft :: Windows', - 'Operating System :: POSIX', - 'Operating System :: Unix', - 'Operating System :: MacOS', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - ('Programming Language :: Python :: ' - 'Implementation :: CPython'), - ('Programming Language :: Python :: ' - 'Implementation :: PyPy') - ], - cmdclass=cmdclass, - python_requires=">=3.6", - install_requires=min_deps.tag_to_packages['install'], - package_data={'': ['*.pxd']}, - **extra_setuptools_args) - - commands = [arg for arg in sys.argv[1:] if not arg.startswith('-')] - if all(command in ('egg_info', 'dist_info', 'clean', 'check') - for command in commands): - # These actions are required to succeed without Numpy for example when - # pip is used to install Scikit-learn when Numpy is not yet present in - # the system. - - # These commands use setup from setuptools - from setuptools import setup - - metadata['version'] = VERSION - else: - if sys.version_info < (3, 6): - raise RuntimeError( - "Scikit-learn requires Python 3.6 or later. The current" - " Python version is %s installed in %s." - % (platform.python_version(), sys.executable)) - - check_package_status('numpy', min_deps.NUMPY_MIN_VERSION) - - check_package_status('scipy', min_deps.SCIPY_MIN_VERSION) - - # These commands require the setup from numpy.distutils because they - # may use numpy.distutils compiler classes. - from numpy.distutils.core import setup - - metadata['configuration'] = configuration - - setup(**metadata) - - -if __name__ == "__main__": - setup_package() diff --git a/sklearn/__check_build/__init__.py b/sklearn/__check_build/__init__.py index 6c1cdfd9fc7b2..6e06d16bd4d50 100644 --- a/sklearn/__check_build/__init__.py +++ b/sklearn/__check_build/__init__.py @@ -1,6 +1,10 @@ -""" Module to give helpful messages to the user that did not +"""Module to give helpful messages to the user that did not compile scikit-learn properly. """ + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import os INPLACE_MSG = """ @@ -24,11 +28,12 @@ def raise_build_error(e): msg = INPLACE_MSG dir_content = list() for i, filename in enumerate(os.listdir(local_dir)): - if ((i + 1) % 3): + if (i + 1) % 3: dir_content.append(filename.ljust(26)) else: - dir_content.append(filename + '\n') - raise ImportError("""%s + dir_content.append(filename + "\n") + raise ImportError( + """%s ___________________________________________________________________________ Contents of %s: %s @@ -36,11 +41,14 @@ def raise_build_error(e): It seems that scikit-learn has not been built correctly. If you have installed scikit-learn from source, please do not forget -to build the package before using it: run `python setup.py install` or -`make` in the source directory. -%s""" % (e, local_dir, ''.join(dir_content).strip(), msg)) +to build the package before using it. For detailed instructions, see: +https://scikit-learn.org/dev/developers/advanced_installation.html#building-from-source +%s""" + % (e, local_dir, "".join(dir_content).strip(), msg) + ) + try: - from ._check_build import check_build # noqa + from ._check_build import check_build # noqa: F401 except ImportError as e: raise_build_error(e) diff --git a/sklearn/__check_build/meson.build b/sklearn/__check_build/meson.build new file mode 100644 index 0000000000000..5f6115d976549 --- /dev/null +++ b/sklearn/__check_build/meson.build @@ -0,0 +1,6 @@ +py.extension_module( + '_check_build', + cython_gen.process('_check_build.pyx'), + install: true, + subdir: 'sklearn/__check_build', +) diff --git a/sklearn/__check_build/setup.py b/sklearn/__check_build/setup.py deleted file mode 100644 index b8c30d9c83dff..0000000000000 --- a/sklearn/__check_build/setup.py +++ /dev/null @@ -1,18 +0,0 @@ -# Author: Virgile Fritsch -# License: BSD 3 clause - -import numpy - - -def configuration(parent_package='', top_path=None): - from numpy.distutils.misc_util import Configuration - config = Configuration('__check_build', parent_package, top_path) - config.add_extension('_check_build', - sources=['_check_build.pyx'], - include_dirs=[numpy.get_include()]) - - return config - -if __name__ == '__main__': - from numpy.distutils.core import setup - setup(**configuration(top_path='').todict()) diff --git a/sklearn/__init__.py b/sklearn/__init__.py index 30022d33af0c6..597cc364a105b 100644 --- a/sklearn/__init__.py +++ b/sklearn/__init__.py @@ -1,24 +1,27 @@ -""" -Machine learning module for Python -================================== +"""Configure global settings and get information about the working environment.""" -sklearn is a Python module integrating classical machine -learning algorithms in the tightly-knit world of scientific Python -packages (numpy, scipy, matplotlib). +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -It aims to provide simple and efficient solutions to learning problems -that are accessible to everybody and reusable in various contexts: -machine-learning as a versatile tool for science and engineering. +# Machine learning module for Python +# ================================== +# +# sklearn is a Python module integrating classical machine +# learning algorithms in the tightly-knit world of scientific Python +# packages (numpy, scipy, matplotlib). +# +# It aims to provide simple and efficient solutions to learning problems +# that are accessible to everybody and reusable in various contexts: +# machine-learning as a versatile tool for science and engineering. +# +# See https://scikit-learn.org for complete documentation. -See http://scikit-learn.org for complete documentation. -""" -import sys +import importlib as _importlib import logging import os import random - -from ._config import get_config, set_config, config_context +from ._config import config_context, get_config, set_config logger = logging.getLogger(__name__) @@ -39,7 +42,7 @@ # Dev branch marker is: 'X.Y.dev' or 'X.Y.devN' where N is an integer. # 'X.Y.dev0' is the canonical version of 'X.Y.dev' # -__version__ = '1.0.dev0' +__version__ = "1.8.dev0" # On OSX, we can get a runtime error due to multiple OpenMP libraries loaded @@ -56,45 +59,92 @@ # https://github.com/ContinuumIO/anaconda-issues/issues/11294 os.environ.setdefault("KMP_INIT_AT_FORK", "FALSE") +# `_distributor_init` allows distributors to run custom init code. +# For instance, for the Windows wheel, this is used to pre-load the +# vcomp shared library runtime for OpenMP embedded in the sklearn/.libs +# sub-folder. +# It is necessary to do this prior to importing show_versions as the +# later is linked to the OpenMP runtime to make it possible to introspect +# it and importing it first would fail if the OpenMP dll cannot be found. +from . import ( # noqa: F401 E402 + __check_build, + _distributor_init, +) +from .base import clone # noqa: E402 +from .utils._show_versions import show_versions # noqa: E402 + +_submodules = [ + "calibration", + "cluster", + "covariance", + "cross_decomposition", + "datasets", + "decomposition", + "dummy", + "ensemble", + "exceptions", + "experimental", + "externals", + "feature_extraction", + "feature_selection", + "frozen", + "gaussian_process", + "inspection", + "isotonic", + "kernel_approximation", + "kernel_ridge", + "linear_model", + "manifold", + "metrics", + "mixture", + "model_selection", + "multiclass", + "multioutput", + "naive_bayes", + "neighbors", + "neural_network", + "pipeline", + "preprocessing", + "random_projection", + "semi_supervised", + "svm", + "tree", + "discriminant_analysis", + "impute", + "compose", +] + +__all__ = _submodules + [ + # Non-modules: + "clone", + "get_config", + "set_config", + "config_context", + "show_versions", +] + + +def __dir__(): + return __all__ + + +def __getattr__(name): + if name in _submodules: + return _importlib.import_module(f"sklearn.{name}") + else: + try: + return globals()[name] + except KeyError: + raise AttributeError(f"Module 'sklearn' has no attribute '{name}'") + + +_BUILT_WITH_MESON = False try: - # This variable is injected in the __builtins__ by the build - # process. It is used to enable importing subpackages of sklearn when - # the binaries are not built - # mypy error: Cannot determine type of '__SKLEARN_SETUP__' - __SKLEARN_SETUP__ # type: ignore -except NameError: - __SKLEARN_SETUP__ = False - -if __SKLEARN_SETUP__: - sys.stderr.write('Partial import of sklearn during the build process.\n') - # We are not importing the rest of scikit-learn during the build - # process, as it may not be compiled yet -else: - # `_distributor_init` allows distributors to run custom init code. - # For instance, for the Windows wheel, this is used to pre-load the - # vcomp shared library runtime for OpenMP embedded in the sklearn/.libs - # sub-folder. - # It is necessary to do this prior to importing show_versions as the - # later is linked to the OpenMP runtime to make it possible to introspect - # it and importing it first would fail if the OpenMP dll cannot be found. - from . import _distributor_init # noqa: F401 - from . import __check_build # noqa: F401 - from .base import clone - from .utils._show_versions import show_versions - - __all__ = ['calibration', 'cluster', 'covariance', 'cross_decomposition', - 'datasets', 'decomposition', 'dummy', 'ensemble', 'exceptions', - 'experimental', 'externals', 'feature_extraction', - 'feature_selection', 'gaussian_process', 'inspection', - 'isotonic', 'kernel_approximation', 'kernel_ridge', - 'linear_model', 'manifold', 'metrics', 'mixture', - 'model_selection', 'multiclass', 'multioutput', - 'naive_bayes', 'neighbors', 'neural_network', 'pipeline', - 'preprocessing', 'random_projection', 'semi_supervised', - 'svm', 'tree', 'discriminant_analysis', 'impute', 'compose', - # Non-modules: - 'clone', 'get_config', 'set_config', 'config_context', - 'show_versions'] + import sklearn._built_with_meson # noqa: F401 + + _BUILT_WITH_MESON = True +except ModuleNotFoundError: + pass def setup_module(module): @@ -103,7 +153,7 @@ def setup_module(module): import numpy as np # Check if a random seed exists in the environment, if not create one. - _random_seed = os.environ.get('SKLEARN_SEED', None) + _random_seed = os.environ.get("SKLEARN_SEED", None) if _random_seed is None: _random_seed = np.random.uniform() * np.iinfo(np.int32).max _random_seed = int(_random_seed) diff --git a/sklearn/_build_utils/__init__.py b/sklearn/_build_utils/__init__.py index b89a2f0b5f6bf..e69de29bb2d1d 100644 --- a/sklearn/_build_utils/__init__.py +++ b/sklearn/_build_utils/__init__.py @@ -1,96 +0,0 @@ -""" -Utilities useful during the build. -""" -# author: Andy Mueller, Gael Varoquaux -# license: BSD - - -import os -import sklearn -import contextlib - -from distutils.version import LooseVersion - -from .pre_build_helpers import basic_check_build -from .openmp_helpers import check_openmp_support -from .._min_dependencies import CYTHON_MIN_VERSION - - -DEFAULT_ROOT = 'sklearn' - - -def _check_cython_version(): - message = ('Please install Cython with a version >= {0} in order ' - 'to build a scikit-learn from source.').format( - CYTHON_MIN_VERSION) - try: - import Cython - except ModuleNotFoundError as e: - # Re-raise with more informative error message instead: - raise ModuleNotFoundError(message) from e - - if LooseVersion(Cython.__version__) < CYTHON_MIN_VERSION: - message += (' The current version of Cython is {} installed in {}.' - .format(Cython.__version__, Cython.__path__)) - raise ValueError(message) - - -def cythonize_extensions(top_path, config): - """Check that a recent Cython is available and cythonize extensions""" - _check_cython_version() - from Cython.Build import cythonize - - # Fast fail before cythonization if compiler fails compiling basic test - # code even without OpenMP - basic_check_build() - - # check simple compilation with OpenMP. If it fails scikit-learn will be - # built without OpenMP and the test test_openmp_supported in the test suite - # will fail. - # `check_openmp_support` compiles a small test program to see if the - # compilers are properly configured to build with OpenMP. This is expensive - # and we only want to call this function once. - # The result of this check is cached as a private attribute on the sklearn - # module (only at build-time) to be used twice: - # - First to set the value of SKLEARN_OPENMP_PARALLELISM_ENABLED, the - # cython build-time variable passed to the cythonize() call. - # - Then in the build_ext subclass defined in the top-level setup.py file - # to actually build the compiled extensions with OpenMP flags if needed. - sklearn._OPENMP_SUPPORTED = check_openmp_support() - - n_jobs = 1 - with contextlib.suppress(ImportError): - import joblib - if LooseVersion(joblib.__version__) > LooseVersion("0.13.0"): - # earlier joblib versions don't account for CPU affinity - # constraints, and may over-estimate the number of available - # CPU particularly in CI (cf loky#114) - n_jobs = joblib.cpu_count() - - config.ext_modules = cythonize( - config.ext_modules, - nthreads=n_jobs, - compile_time_env={ - 'SKLEARN_OPENMP_PARALLELISM_ENABLED': sklearn._OPENMP_SUPPORTED}, - compiler_directives={'language_level': 3}) - - -def gen_from_templates(templates, top_path): - """Generate cython files from a list of templates""" - # Lazy import because cython is not a runtime dependency. - from Cython import Tempita - - for template in templates: - outfile = template.replace('.tp', '') - - # if the template is not updated, no need to output the cython file - if not (os.path.exists(outfile) and - os.stat(template).st_mtime < os.stat(outfile).st_mtime): - - with open(template, "r") as f: - tmpl = f.read() - - tmpl_ = Tempita.sub(tmpl) - - with open(outfile, "w") as f: - f.write(tmpl_) diff --git a/sklearn/_build_utils/openmp_helpers.py b/sklearn/_build_utils/openmp_helpers.py deleted file mode 100644 index d98962b3c2a86..0000000000000 --- a/sklearn/_build_utils/openmp_helpers.py +++ /dev/null @@ -1,122 +0,0 @@ -"""Helpers for OpenMP support during the build.""" - -# This code is adapted for a large part from the astropy openmp helpers, which -# can be found at: https://github.com/astropy/extension-helpers/blob/master/extension_helpers/_openmp_helpers.py # noqa - - -import os -import sys -import textwrap -import warnings -import subprocess - -from distutils.errors import CompileError, LinkError - -from .pre_build_helpers import compile_test_program - - -def get_openmp_flag(compiler): - if hasattr(compiler, 'compiler'): - compiler = compiler.compiler[0] - else: - compiler = compiler.__class__.__name__ - - if sys.platform == "win32" and ('icc' in compiler or 'icl' in compiler): - return ['/Qopenmp'] - elif sys.platform == "win32": - return ['/openmp'] - elif sys.platform in ("darwin", "linux") and "icc" in compiler: - return ['-qopenmp'] - elif sys.platform == "darwin" and 'openmp' in os.getenv('CPPFLAGS', ''): - # -fopenmp can't be passed as compile flag when using Apple-clang. - # OpenMP support has to be enabled during preprocessing. - # - # For example, our macOS wheel build jobs use the following environment - # variables to build with Apple-clang and the brew installed "libomp": - # - # export CPPFLAGS="$CPPFLAGS -Xpreprocessor -fopenmp" - # export CFLAGS="$CFLAGS -I/usr/local/opt/libomp/include" - # export CXXFLAGS="$CXXFLAGS -I/usr/local/opt/libomp/include" - # export LDFLAGS="$LDFLAGS -Wl,-rpath,/usr/local/opt/libomp/lib - # -L/usr/local/opt/libomp/lib -lomp" - return [] - # Default flag for GCC and clang: - return ['-fopenmp'] - - -def check_openmp_support(): - """Check whether OpenMP test code can be compiled and run""" - if "PYODIDE_PACKAGE_ABI" in os.environ: - # Pyodide doesn't support OpenMP - return False - code = textwrap.dedent( - """\ - #include - #include - int main(void) { - #pragma omp parallel - printf("nthreads=%d\\n", omp_get_num_threads()); - return 0; - } - """) - - extra_preargs = os.getenv('LDFLAGS', None) - if extra_preargs is not None: - extra_preargs = extra_preargs.strip().split(" ") - extra_preargs = [ - flag for flag in extra_preargs - if flag.startswith(('-L', '-Wl,-rpath', '-l'))] - - extra_postargs = get_openmp_flag - - try: - output = compile_test_program(code, - extra_preargs=extra_preargs, - extra_postargs=extra_postargs) - - if output and 'nthreads=' in output[0]: - nthreads = int(output[0].strip().split('=')[1]) - openmp_supported = len(output) == nthreads - elif "PYTHON_CROSSENV" in os.environ: - # Since we can't run the test program when cross-compiling - # assume that openmp is supported if the program can be - # compiled. - openmp_supported = True - else: - openmp_supported = False - - except (CompileError, LinkError, subprocess.CalledProcessError): - openmp_supported = False - - if not openmp_supported: - if os.getenv("SKLEARN_FAIL_NO_OPENMP"): - raise CompileError("Failed to build with OpenMP") - else: - message = textwrap.dedent( - """ - - *********** - * WARNING * - *********** - - It seems that scikit-learn cannot be built with OpenMP. - - - Make sure you have followed the installation instructions: - - https://scikit-learn.org/dev/developers/advanced_installation.html - - - If your compiler supports OpenMP but you still see this - message, please submit a bug report at: - - https://github.com/scikit-learn/scikit-learn/issues - - - The build will continue with OpenMP-based parallelism - disabled. Note however that some estimators will run in - sequential mode instead of leveraging thread-based - parallelism. - - *** - """) - warnings.warn(message) - - return openmp_supported diff --git a/sklearn/_build_utils/pre_build_helpers.py b/sklearn/_build_utils/pre_build_helpers.py deleted file mode 100644 index 1041f4fab454b..0000000000000 --- a/sklearn/_build_utils/pre_build_helpers.py +++ /dev/null @@ -1,106 +0,0 @@ -"""Helpers to check build environment before actual build of scikit-learn""" - -import os -import sys -import glob -import tempfile -import textwrap -import setuptools # noqa -import subprocess - -from distutils.dist import Distribution -from distutils.sysconfig import customize_compiler -from numpy.distutils.ccompiler import new_compiler -from numpy.distutils.command.config_compiler import config_cc - - -def _get_compiler(): - """Get a compiler equivalent to the one that will be used to build sklearn - - Handles compiler specified as follows: - - python setup.py build_ext --compiler= - - CC= python setup.py build_ext - """ - dist = Distribution({'script_name': os.path.basename(sys.argv[0]), - 'script_args': sys.argv[1:], - 'cmdclass': {'config_cc': config_cc}}) - dist.parse_config_files() - dist.parse_command_line() - - cmd_opts = dist.command_options.get('build_ext') - if cmd_opts is not None and 'compiler' in cmd_opts: - compiler = cmd_opts['compiler'][1] - else: - compiler = None - - ccompiler = new_compiler(compiler=compiler) - customize_compiler(ccompiler) - - return ccompiler - - -def compile_test_program(code, extra_preargs=[], extra_postargs=[]): - """Check that some C code can be compiled and run""" - ccompiler = _get_compiler() - - # extra_(pre/post)args can be a callable to make it possible to get its - # value from the compiler - if callable(extra_preargs): - extra_preargs = extra_preargs(ccompiler) - if callable(extra_postargs): - extra_postargs = extra_postargs(ccompiler) - - start_dir = os.path.abspath('.') - - with tempfile.TemporaryDirectory() as tmp_dir: - try: - os.chdir(tmp_dir) - - # Write test program - with open('test_program.c', 'w') as f: - f.write(code) - - os.mkdir('objects') - - # Compile, test program - ccompiler.compile(['test_program.c'], output_dir='objects', - extra_postargs=extra_postargs) - - # Link test program - objects = glob.glob( - os.path.join('objects', '*' + ccompiler.obj_extension)) - ccompiler.link_executable(objects, 'test_program', - extra_preargs=extra_preargs, - extra_postargs=extra_postargs) - - if "PYTHON_CROSSENV" not in os.environ: - # Run test program if not cross compiling - # will raise a CalledProcessError if return code was non-zero - output = subprocess.check_output('./test_program') - output = output.decode( - sys.stdout.encoding or 'utf-8').splitlines() - else: - # Return an empty output if we are cross compiling - # as we cannot run the test_program - output = [] - except Exception: - raise - finally: - os.chdir(start_dir) - - return output - - -def basic_check_build(): - """Check basic compilation and linking of C code""" - if "PYODIDE_PACKAGE_ABI" in os.environ: - # The following check won't work in pyodide - return - code = textwrap.dedent( - """\ - #include - int main(void) { - return 0; - } - """) - compile_test_program(code) diff --git a/sklearn/_build_utils/tempita.py b/sklearn/_build_utils/tempita.py new file mode 100644 index 0000000000000..c8a7a35a62fee --- /dev/null +++ b/sklearn/_build_utils/tempita.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import argparse +import os + +from Cython import Tempita as tempita + +# XXX: If this import ever fails (does it really?), vendor either +# cython.tempita or numpy/npy_tempita. + + +def process_tempita(fromfile, outfile=None): + """Process tempita templated file and write out the result. + + The template file is expected to end in `.c.tp` or `.pyx.tp`: + E.g. processing `template.c.in` generates `template.c`. + + """ + with open(fromfile, "r", encoding="utf-8") as f: + template_content = f.read() + + template = tempita.Template(template_content) + content = template.substitute() + + with open(outfile, "w", encoding="utf-8") as f: + f.write(content) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("infile", type=str, help="Path to the input file") + parser.add_argument("-o", "--outdir", type=str, help="Path to the output directory") + parser.add_argument( + "-i", + "--ignore", + type=str, + help=( + "An ignored input - may be useful to add a " + "dependency between custom targets" + ), + ) + args = parser.parse_args() + + if not args.infile.endswith(".tp"): + raise ValueError(f"Unexpected extension: {args.infile}") + + if not args.outdir: + raise ValueError("Missing `--outdir` argument to tempita.py") + + outdir_abs = os.path.join(os.getcwd(), args.outdir) + outfile = os.path.join( + outdir_abs, os.path.splitext(os.path.split(args.infile)[1])[0] + ) + + process_tempita(args.infile, outfile) + + +if __name__ == "__main__": + main() diff --git a/sklearn/_build_utils/version.py b/sklearn/_build_utils/version.py new file mode 100755 index 0000000000000..922a14917bf3f --- /dev/null +++ b/sklearn/_build_utils/version.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +"""Extract version number from __init__.py""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import os + +sklearn_init = os.path.join(os.path.dirname(__file__), "../__init__.py") + +data = open(sklearn_init).readlines() +version_line = next(line for line in data if line.startswith("__version__")) + +version = version_line.strip().split(" = ")[1].replace('"', "").replace("'", "") + +print(version) diff --git a/sklearn/_config.py b/sklearn/_config.py index feb5e86287c38..05549c88a9ddc 100644 --- a/sklearn/_config.py +++ b/sklearn/_config.py @@ -1,18 +1,39 @@ -"""Global configuration state and functions for management -""" +"""Global configuration state and functions for management""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import os +import threading from contextlib import contextmanager as contextmanager _global_config = { - 'assume_finite': bool(os.environ.get('SKLEARN_ASSUME_FINITE', False)), - 'working_memory': int(os.environ.get('SKLEARN_WORKING_MEMORY', 1024)), - 'print_changed_only': True, - 'display': 'text', + "assume_finite": bool(os.environ.get("SKLEARN_ASSUME_FINITE", False)), + "working_memory": int(os.environ.get("SKLEARN_WORKING_MEMORY", 1024)), + "print_changed_only": True, + "display": "diagram", + "pairwise_dist_chunk_size": int( + os.environ.get("SKLEARN_PAIRWISE_DIST_CHUNK_SIZE", 256) + ), + "enable_cython_pairwise_dist": True, + "array_api_dispatch": False, + "transform_output": "default", + "enable_metadata_routing": False, + "skip_parameter_validation": False, } +_threadlocal = threading.local() + + +def _get_threadlocal_config(): + """Get a threadlocal **mutable** configuration. If the configuration + does not exist, copy the default global configuration.""" + if not hasattr(_threadlocal, "global_config"): + _threadlocal.global_config = _global_config.copy() + return _threadlocal.global_config def get_config(): - """Retrieve current values for configuration set by :func:`set_config` + """Retrieve current values for configuration set by :func:`set_config`. Returns ------- @@ -23,13 +44,32 @@ def get_config(): -------- config_context : Context manager for global scikit-learn configuration. set_config : Set global scikit-learn configuration. + + Examples + -------- + >>> import sklearn + >>> config = sklearn.get_config() + >>> config.keys() + dict_keys([...]) """ - return _global_config.copy() + # Return a copy of the threadlocal configuration so that users will + # not be able to modify the configuration with the returned dict. + return _get_threadlocal_config().copy() -def set_config(assume_finite=None, working_memory=None, - print_changed_only=None, display=None): - """Set global scikit-learn configuration +def set_config( + assume_finite=None, + working_memory=None, + print_changed_only=None, + display=None, + pairwise_dist_chunk_size=None, + enable_cython_pairwise_dist=None, + array_api_dispatch=None, + transform_output=None, + enable_metadata_routing=None, + skip_parameter_validation=None, +): + """Set global scikit-learn configuration. .. versionadded:: 0.19 @@ -63,65 +103,245 @@ def set_config(assume_finite=None, working_memory=None, display : {'text', 'diagram'}, default=None If 'diagram', estimators will be displayed as a diagram in a Jupyter lab or notebook context. If 'text', estimators will be displayed as - text. Default is 'text'. + text. Default is 'diagram'. .. versionadded:: 0.23 + pairwise_dist_chunk_size : int, default=None + The number of row vectors per chunk for the accelerated pairwise- + distances reduction backend. Default is 256 (suitable for most of + modern laptops' caches and architectures). + + Intended for easier benchmarking and testing of scikit-learn internals. + End users are not expected to benefit from customizing this configuration + setting. + + .. versionadded:: 1.1 + + enable_cython_pairwise_dist : bool, default=None + Use the accelerated pairwise-distances reduction backend when + possible. Global default: True. + + Intended for easier benchmarking and testing of scikit-learn internals. + End users are not expected to benefit from customizing this configuration + setting. + + .. versionadded:: 1.1 + + array_api_dispatch : bool, default=None + Use Array API dispatching when inputs follow the Array API standard. + Default is False. + + See the :ref:`User Guide ` for more details. + + .. versionadded:: 1.2 + + transform_output : str, default=None + Configure output of `transform` and `fit_transform`. + + See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` + for an example on how to use the API. + + - `"default"`: Default output format of a transformer + - `"pandas"`: DataFrame output + - `"polars"`: Polars output + - `None`: Transform configuration is unchanged + + .. versionadded:: 1.2 + .. versionadded:: 1.4 + `"polars"` option was added. + + enable_metadata_routing : bool, default=None + Enable metadata routing. By default this feature is disabled. + + Refer to :ref:`metadata routing user guide ` for more + details. + + - `True`: Metadata routing is enabled + - `False`: Metadata routing is disabled, use the old syntax. + - `None`: Configuration is unchanged + + .. versionadded:: 1.3 + + skip_parameter_validation : bool, default=None + If `True`, disable the validation of the hyper-parameters' types and values in + the fit method of estimators and for arguments passed to public helper + functions. It can save time in some situations but can lead to low level + crashes and exceptions with confusing error messages. + + Note that for data parameters, such as `X` and `y`, only type validation is + skipped but validation with `check_array` will continue to run. + + .. versionadded:: 1.3 + See Also -------- config_context : Context manager for global scikit-learn configuration. get_config : Retrieve current values of the global configuration. + + Examples + -------- + >>> from sklearn import set_config + >>> set_config(display='diagram') # doctest: +SKIP """ + local_config = _get_threadlocal_config() + if assume_finite is not None: - _global_config['assume_finite'] = assume_finite + local_config["assume_finite"] = assume_finite if working_memory is not None: - _global_config['working_memory'] = working_memory + local_config["working_memory"] = working_memory if print_changed_only is not None: - _global_config['print_changed_only'] = print_changed_only + local_config["print_changed_only"] = print_changed_only if display is not None: - _global_config['display'] = display + local_config["display"] = display + if pairwise_dist_chunk_size is not None: + local_config["pairwise_dist_chunk_size"] = pairwise_dist_chunk_size + if enable_cython_pairwise_dist is not None: + local_config["enable_cython_pairwise_dist"] = enable_cython_pairwise_dist + if array_api_dispatch is not None: + from .utils._array_api import _check_array_api_dispatch + + _check_array_api_dispatch(array_api_dispatch) + local_config["array_api_dispatch"] = array_api_dispatch + if transform_output is not None: + local_config["transform_output"] = transform_output + if enable_metadata_routing is not None: + local_config["enable_metadata_routing"] = enable_metadata_routing + if skip_parameter_validation is not None: + local_config["skip_parameter_validation"] = skip_parameter_validation @contextmanager -def config_context(**new_config): - """Context manager for global scikit-learn configuration +def config_context( + *, + assume_finite=None, + working_memory=None, + print_changed_only=None, + display=None, + pairwise_dist_chunk_size=None, + enable_cython_pairwise_dist=None, + array_api_dispatch=None, + transform_output=None, + enable_metadata_routing=None, + skip_parameter_validation=None, +): + """Context manager for global scikit-learn configuration. Parameters ---------- - assume_finite : bool, default=False + assume_finite : bool, default=None If True, validation for finiteness will be skipped, saving time, but leading to potential crashes. If False, validation for finiteness will be performed, - avoiding error. Global default: False. + avoiding error. If None, the existing value won't change. + The default value is False. - working_memory : int, default=1024 + working_memory : int, default=None If set, scikit-learn will attempt to limit the size of temporary arrays to this number of MiB (per job when parallelised), often saving both computation time and memory on expensive operations that can be - performed in chunks. Global default: 1024. + performed in chunks. If None, the existing value won't change. + The default value is 1024. - print_changed_only : bool, default=True + print_changed_only : bool, default=None If True, only the parameters that were set to non-default values will be printed when printing an estimator. For example, ``print(SVC())`` while True will only print 'SVC()', but would print 'SVC(C=1.0, cache_size=200, ...)' with all the non-changed parameters - when False. Default is True. + when False. If None, the existing value won't change. + The default value is True. .. versionchanged:: 0.23 Default changed from False to True. - display : {'text', 'diagram'}, default='text' + display : {'text', 'diagram'}, default=None If 'diagram', estimators will be displayed as a diagram in a Jupyter lab or notebook context. If 'text', estimators will be displayed as - text. Default is 'text'. + text. If None, the existing value won't change. + The default value is 'diagram'. .. versionadded:: 0.23 + pairwise_dist_chunk_size : int, default=None + The number of row vectors per chunk for the accelerated pairwise- + distances reduction backend. Default is 256 (suitable for most of + modern laptops' caches and architectures). + + Intended for easier benchmarking and testing of scikit-learn internals. + End users are not expected to benefit from customizing this configuration + setting. + + .. versionadded:: 1.1 + + enable_cython_pairwise_dist : bool, default=None + Use the accelerated pairwise-distances reduction backend when + possible. Global default: True. + + Intended for easier benchmarking and testing of scikit-learn internals. + End users are not expected to benefit from customizing this configuration + setting. + + .. versionadded:: 1.1 + + array_api_dispatch : bool, default=None + Use Array API dispatching when inputs follow the Array API standard. + Default is False. + + See the :ref:`User Guide ` for more details. + + .. versionadded:: 1.2 + + transform_output : str, default=None + Configure output of `transform` and `fit_transform`. + + See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` + for an example on how to use the API. + + - `"default"`: Default output format of a transformer + - `"pandas"`: DataFrame output + - `"polars"`: Polars output + - `None`: Transform configuration is unchanged + + .. versionadded:: 1.2 + .. versionadded:: 1.4 + `"polars"` option was added. + + enable_metadata_routing : bool, default=None + Enable metadata routing. By default this feature is disabled. + + Refer to :ref:`metadata routing user guide ` for more + details. + + - `True`: Metadata routing is enabled + - `False`: Metadata routing is disabled, use the old syntax. + - `None`: Configuration is unchanged + + .. versionadded:: 1.3 + + skip_parameter_validation : bool, default=None + If `True`, disable the validation of the hyper-parameters' types and values in + the fit method of estimators and for arguments passed to public helper + functions. It can save time in some situations but can lead to low level + crashes and exceptions with confusing error messages. + + Note that for data parameters, such as `X` and `y`, only type validation is + skipped but validation with `check_array` will continue to run. + + .. versionadded:: 1.3 + + Yields + ------ + None. + + See Also + -------- + set_config : Set global scikit-learn configuration. + get_config : Retrieve current values of the global configuration. + Notes ----- All settings, not just those presently modified, will be returned to - their previous values when the context manager is exited. This is not - thread-safe. + their previous values when the context manager is exited. Examples -------- @@ -134,15 +354,21 @@ def config_context(**new_config): ... assert_all_finite([float('nan')]) Traceback (most recent call last): ... - ValueError: Input contains NaN, ... - - See Also - -------- - set_config : Set global scikit-learn configuration. - get_config : Retrieve current values of the global configuration. + ValueError: Input contains NaN... """ - old_config = get_config().copy() - set_config(**new_config) + old_config = get_config() + set_config( + assume_finite=assume_finite, + working_memory=working_memory, + print_changed_only=print_changed_only, + display=display, + pairwise_dist_chunk_size=pairwise_dist_chunk_size, + enable_cython_pairwise_dist=enable_cython_pairwise_dist, + array_api_dispatch=array_api_dispatch, + transform_output=transform_output, + enable_metadata_routing=enable_metadata_routing, + skip_parameter_validation=skip_parameter_validation, + ) try: yield diff --git a/sklearn/_distributor_init.py b/sklearn/_distributor_init.py index a0142ac80878f..d66d5d36955c1 100644 --- a/sklearn/_distributor_init.py +++ b/sklearn/_distributor_init.py @@ -1,4 +1,4 @@ -""" Distributor init file +"""Distributor init file Distributors: you can add custom code here to support particular distributions of scikit-learn. @@ -8,3 +8,6 @@ The scikit-learn standard source distribution will not put code in this file, so you can safely replace this file with your own version. """ + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause diff --git a/sklearn/_isotonic.pyx b/sklearn/_isotonic.pyx index 7f60b889fa284..3dfb0421f0c19 100644 --- a/sklearn/_isotonic.pyx +++ b/sklearn/_isotonic.pyx @@ -1,18 +1,13 @@ -# Author: Nelle Varoquaux, Andrew Tulloch, Antony Lee +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause # Uses the pool adjacent violators algorithm (PAVA), with the # enhancement of searching for the longest decreasing subsequence to # pool at each step. -# -# cython: boundscheck=False, wraparound=False, cdivision=True import numpy as np -cimport numpy as np -cimport cython from cython cimport floating -np.import_array() - def _inplace_contiguous_isotonic_regression(floating[::1] y, floating[::1] w): cdef: @@ -65,9 +60,9 @@ def _inplace_contiguous_isotonic_regression(floating[::1] y, floating[::1] w): i = k -def _make_unique(np.ndarray[dtype=floating] X, - np.ndarray[dtype=floating] y, - np.ndarray[dtype=floating] sample_weights): +def _make_unique(const floating[::1] X, + const floating[::1] y, + const floating[::1] sample_weights): """Average targets for duplicate X, drop duplicates. Aggregates duplicate X values into a single X value where @@ -78,20 +73,23 @@ def _make_unique(np.ndarray[dtype=floating] X, """ unique_values = len(np.unique(X)) - cdef np.ndarray[dtype=floating] y_out = np.empty(unique_values, - dtype=X.dtype) - cdef np.ndarray[dtype=floating] x_out = np.empty_like(y_out) - cdef np.ndarray[dtype=floating] weights_out = np.empty_like(y_out) + if floating is float: + dtype = np.float32 + else: + dtype = np.float64 + + cdef floating[::1] y_out = np.empty(unique_values, dtype=dtype) + cdef floating[::1] x_out = np.empty_like(y_out) + cdef floating[::1] weights_out = np.empty_like(y_out) cdef floating current_x = X[0] cdef floating current_y = 0 cdef floating current_weight = 0 - cdef floating y_old = 0 cdef int i = 0 cdef int j cdef floating x cdef int n_samples = len(X) - cdef floating eps = np.finfo(X.dtype).resolution + cdef floating eps = np.finfo(dtype).resolution for j in range(n_samples): x = X[j] @@ -111,4 +109,8 @@ def _make_unique(np.ndarray[dtype=floating] X, x_out[i] = current_x weights_out[i] = current_weight y_out[i] = current_y / current_weight - return x_out[:i+1], y_out[:i+1], weights_out[:i+1] + return( + np.asarray(x_out[:i+1]), + np.asarray(y_out[:i+1]), + np.asarray(weights_out[:i+1]), + ) diff --git a/sklearn/_loss/__init__.py b/sklearn/_loss/__init__.py index e69de29bb2d1d..97fdd884e517c 100644 --- a/sklearn/_loss/__init__.py +++ b/sklearn/_loss/__init__.py @@ -0,0 +1,33 @@ +""" +The :mod:`sklearn._loss` module includes loss function classes suitable for +fitting classification and regression tasks. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from .loss import ( + AbsoluteError, + HalfBinomialLoss, + HalfGammaLoss, + HalfMultinomialLoss, + HalfPoissonLoss, + HalfSquaredError, + HalfTweedieLoss, + HalfTweedieLossIdentity, + HuberLoss, + PinballLoss, +) + +__all__ = [ + "AbsoluteError", + "HalfBinomialLoss", + "HalfGammaLoss", + "HalfMultinomialLoss", + "HalfPoissonLoss", + "HalfSquaredError", + "HalfTweedieLoss", + "HalfTweedieLossIdentity", + "HuberLoss", + "PinballLoss", +] diff --git a/sklearn/_loss/_loss.pxd b/sklearn/_loss/_loss.pxd new file mode 100644 index 0000000000000..ac01b122a0941 --- /dev/null +++ b/sklearn/_loss/_loss.pxd @@ -0,0 +1,101 @@ +# Fused types for input like y_true, raw_prediction, sample_weights. +ctypedef fused floating_in: + double + float + + +# Fused types for output like gradient and hessian +# We use a different fused types for input (floating_in) and output (floating_out), such +# that input and output can have different dtypes in the same function call. A single +# fused type can only take on one single value (type) for all arguments in one function +# call. +ctypedef fused floating_out: + double + float + + +# Struct to return 2 doubles +ctypedef struct double_pair: + double val1 + double val2 + + +# C base class for loss functions +cdef class CyLossFunction: + cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil + cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil + cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil + + +cdef class CyHalfSquaredError(CyLossFunction): + cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil + cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil + cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil + + +cdef class CyAbsoluteError(CyLossFunction): + cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil + cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil + cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil + + +cdef class CyPinballLoss(CyLossFunction): + cdef readonly double quantile # readonly makes it accessible from Python + cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil + cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil + cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil + + +cdef class CyHuberLoss(CyLossFunction): + cdef public double delta # public makes it accessible from Python + cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil + cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil + cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil + + +cdef class CyHalfPoissonLoss(CyLossFunction): + cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil + cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil + cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil + + +cdef class CyHalfGammaLoss(CyLossFunction): + cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil + cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil + cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil + + +cdef class CyHalfTweedieLoss(CyLossFunction): + cdef readonly double power # readonly makes it accessible from Python + cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil + cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil + cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil + + +cdef class CyHalfTweedieLossIdentity(CyLossFunction): + cdef readonly double power # readonly makes it accessible from Python + cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil + cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil + cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil + + +cdef class CyHalfBinomialLoss(CyLossFunction): + cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil + cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil + cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil + + +cdef class CyExponentialLoss(CyLossFunction): + cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil + cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil + cdef double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil + + +cdef class CyHalfMultinomialLoss(): + cdef void cy_gradient( + self, + const floating_in y_true, + const floating_in[::1] raw_prediction, + const floating_in sample_weight, + floating_out[::1] gradient_out, + ) noexcept nogil diff --git a/sklearn/_loss/_loss.pyx.tp b/sklearn/_loss/_loss.pyx.tp new file mode 100644 index 0000000000000..44d5acd530a7f --- /dev/null +++ b/sklearn/_loss/_loss.pyx.tp @@ -0,0 +1,1505 @@ +{{py: + +""" +Template file to easily generate loops over samples using Tempita +(https://github.com/cython/cython/blob/master/Cython/Tempita/_tempita.py). + +Generated file: _loss.pyx + +Each loss class is generated by a cdef functions on single samples. +The keywords between double braces are substituted during the build. +""" + +doc_HalfSquaredError = ( + """Half Squared Error with identity link. + + Domain: + y_true and y_pred all real numbers + + Link: + y_pred = raw_prediction + """ +) + +doc_AbsoluteError = ( + """Absolute Error with identity link. + + Domain: + y_true and y_pred all real numbers + + Link: + y_pred = raw_prediction + """ +) + +doc_PinballLoss = ( + """Quantile Loss aka Pinball Loss with identity link. + + Domain: + y_true and y_pred all real numbers + quantile in (0, 1) + + Link: + y_pred = raw_prediction + + Note: 2 * cPinballLoss(quantile=0.5) equals cAbsoluteError() + """ +) + +doc_HuberLoss = ( + """Huber Loss with identity link. + + Domain: + y_true and y_pred all real numbers + delta in positive real numbers + + Link: + y_pred = raw_prediction + """ +) + +doc_HalfPoissonLoss = ( + """Half Poisson deviance loss with log-link. + + Domain: + y_true in non-negative real numbers + y_pred in positive real numbers + + Link: + y_pred = exp(raw_prediction) + + Half Poisson deviance with log-link is + y_true * log(y_true/y_pred) + y_pred - y_true + = y_true * log(y_true) - y_true * raw_prediction + + exp(raw_prediction) - y_true + + Dropping constant terms, this gives: + exp(raw_prediction) - y_true * raw_prediction + """ +) + +doc_HalfGammaLoss = ( + """Half Gamma deviance loss with log-link. + + Domain: + y_true and y_pred in positive real numbers + + Link: + y_pred = exp(raw_prediction) + + Half Gamma deviance with log-link is + log(y_pred/y_true) + y_true/y_pred - 1 + = raw_prediction - log(y_true) + y_true * exp(-raw_prediction) - 1 + + Dropping constant terms, this gives: + raw_prediction + y_true * exp(-raw_prediction) + """ +) + +doc_HalfTweedieLoss = ( + """Half Tweedie deviance loss with log-link. + + Domain: + y_true in real numbers if p <= 0 + y_true in non-negative real numbers if 0 < p < 2 + y_true in positive real numbers if p >= 2 + y_pred and power in positive real numbers + + Link: + y_pred = exp(raw_prediction) + + Half Tweedie deviance with log-link and p=power is + max(y_true, 0)**(2-p) / (1-p) / (2-p) + - y_true * y_pred**(1-p) / (1-p) + + y_pred**(2-p) / (2-p) + = max(y_true, 0)**(2-p) / (1-p) / (2-p) + - y_true * exp((1-p) * raw_prediction) / (1-p) + + exp((2-p) * raw_prediction) / (2-p) + + Dropping constant terms, this gives: + exp((2-p) * raw_prediction) / (2-p) + - y_true * exp((1-p) * raw_prediction) / (1-p) + + Notes: + - Poisson with p=1 and Gamma with p=2 have different terms dropped such + that cHalfTweedieLoss is not continuous in p=power at p=1 and p=2. + - While the Tweedie distribution only exists for p<=0 or p>=1, the range + 0= 2 + y_pred and power in positive real numbers, y_pred may be negative for p=0. + + Link: + y_pred = raw_prediction + + Half Tweedie deviance with identity link and p=power is + max(y_true, 0)**(2-p) / (1-p) / (2-p) + - y_true * y_pred**(1-p) / (1-p) + + y_pred**(2-p) / (2-p) + + Notes: + - Here, we do not drop constant terms in contrast to the version with log-link. + """ +) + +doc_HalfBinomialLoss = ( + """Half Binomial deviance loss with logit link. + + Domain: + y_true in [0, 1] + y_pred in (0, 1), i.e. boundaries excluded + + Link: + y_pred = expit(raw_prediction) + """ +) + +doc_ExponentialLoss = ( + """"Exponential loss with (half) logit link + + Domain: + y_true in [0, 1] + y_pred in (0, 1), i.e. boundaries excluded + + Link: + y_pred = expit(2 * raw_prediction) + """ +) + +# loss class name, docstring, param, +# cy_loss, cy_loss_grad, +# cy_grad, cy_grad_hess, +class_list = [ + ("CyHalfSquaredError", doc_HalfSquaredError, None, + "closs_half_squared_error", None, + "cgradient_half_squared_error", "cgrad_hess_half_squared_error"), + ("CyAbsoluteError", doc_AbsoluteError, None, + "closs_absolute_error", None, + "cgradient_absolute_error", "cgrad_hess_absolute_error"), + ("CyPinballLoss", doc_PinballLoss, "quantile", + "closs_pinball_loss", None, + "cgradient_pinball_loss", "cgrad_hess_pinball_loss"), + ("CyHuberLoss", doc_HuberLoss, "delta", + "closs_huber_loss", None, + "cgradient_huber_loss", "cgrad_hess_huber_loss"), + ("CyHalfPoissonLoss", doc_HalfPoissonLoss, None, + "closs_half_poisson", "closs_grad_half_poisson", + "cgradient_half_poisson", "cgrad_hess_half_poisson"), + ("CyHalfGammaLoss", doc_HalfGammaLoss, None, + "closs_half_gamma", "closs_grad_half_gamma", + "cgradient_half_gamma", "cgrad_hess_half_gamma"), + ("CyHalfTweedieLoss", doc_HalfTweedieLoss, "power", + "closs_half_tweedie", "closs_grad_half_tweedie", + "cgradient_half_tweedie", "cgrad_hess_half_tweedie"), + ("CyHalfTweedieLossIdentity", doc_HalfTweedieLossIdentity, "power", + "closs_half_tweedie_identity", "closs_grad_half_tweedie_identity", + "cgradient_half_tweedie_identity", "cgrad_hess_half_tweedie_identity"), + ("CyHalfBinomialLoss", doc_HalfBinomialLoss, None, + "closs_half_binomial", "closs_grad_half_binomial", + "cgradient_half_binomial", "cgrad_hess_half_binomial"), + ("CyExponentialLoss", doc_ExponentialLoss, None, + "closs_exponential", "closs_grad_exponential", + "cgradient_exponential", "cgrad_hess_exponential"), +] +}} + +# Design: +# See https://github.com/scikit-learn/scikit-learn/issues/15123 for reasons. +# a) Merge link functions into loss functions for speed and numerical +# stability, i.e. use raw_prediction instead of y_pred in signature. +# b) Pure C functions (nogil) calculate single points (single sample) +# c) Wrap C functions in a loop to get Python functions operating on ndarrays. +# - Write loops manually---use Tempita for this. +# Reason: There is still some performance overhead when using a wrapper +# function "wrap" that carries out the loop and gets as argument a function +# pointer to one of the C functions from b), e.g. +# wrap(closs_half_poisson, y_true, ...) +# - Pass n_threads as argument to prange and propagate option to all callers. +# d) Provide classes (Cython extension types) per loss (names start with Cy) in +# order to have semantical structured objects. +# - Member functions for single points just call the C function from b). +# These are used e.g. in SGD `_plain_sgd`. +# - Member functions operating on ndarrays, see c), looping over calls to C +# functions from b). +# e) Provide convenience Python classes that compose from these extension types +# elsewhere (see loss.py) +# - Example: loss.gradient calls CyLoss.gradient but does some input +# checking like None -> np.empty(). +# +# Note: We require 1-dim ndarrays to be contiguous. + +from cython.parallel import parallel, prange +import numpy as np + +from libc.math cimport exp, fabs, log, log1p, pow +from libc.stdlib cimport malloc, free + + +# ------------------------------------- +# Helper functions +# ------------------------------------- +# Numerically stable version of log(1 + exp(x)) for double precision, see Eq. (10) of +# https://cran.r-project.org/web/packages/Rmpfr/vignettes/log1mexp-note.pdf +# Note: The only important cutoff is at x = 18. All others are to save computation +# time. Compared to the reference, we add the additional case distinction x <= -2 in +# order to use log instead of log1p for improved performance. As with the other +# cutoffs, this is accurate within machine precision of double. +cdef inline double log1pexp(double x) noexcept nogil: + if x <= -37: + return exp(x) + elif x <= -2: + return log1p(exp(x)) + elif x <= 18: + return log(1. + exp(x)) + elif x <= 33.3: + return x + exp(-x) + else: + return x + + +cdef inline double_pair sum_exp_minus_max( + const int i, + const floating_in[:, :] raw_prediction, # IN + floating_out *p # OUT +) noexcept nogil: + # Thread local buffers are used to store part of the results via p. + # The results are stored as follows: + # p[k] = exp(raw_prediction_i_k - max_value) for k = 0 to n_classes-1 + # return.val1 = max_value = max(raw_prediction_i_k, k = 0 to n_classes-1) + # return.val2 = sum_exps = sum(p[k], k = 0 to n_classes-1) = sum of exponentials + # len(p) must be n_classes + # Notes: + # - We return the max value and sum of exps (stored in p) as a double_pair. + # - i needs to be passed (and stays constant) because otherwise Cython does + # not generate optimal code, see + # https://github.com/scikit-learn/scikit-learn/issues/17299 + # - We do not normalize p by calculating p[k] = p[k] / sum_exps. + # This helps to save one loop over k. + cdef: + int k + int n_classes = raw_prediction.shape[1] + double_pair max_value_and_sum_exps # val1 = max_value, val2 = sum_exps + + max_value_and_sum_exps.val1 = raw_prediction[i, 0] + max_value_and_sum_exps.val2 = 0 + for k in range(1, n_classes): + # Compute max value of array for numerical stability + if max_value_and_sum_exps.val1 < raw_prediction[i, k]: + max_value_and_sum_exps.val1 = raw_prediction[i, k] + + for k in range(n_classes): + p[k] = exp(raw_prediction[i, k] - max_value_and_sum_exps.val1) + max_value_and_sum_exps.val2 += p[k] + + return max_value_and_sum_exps + + +# ------------------------------------- +# Single point inline C functions +# ------------------------------------- +# Half Squared Error +cdef inline double closs_half_squared_error( + double y_true, + double raw_prediction +) noexcept nogil: + return 0.5 * (raw_prediction - y_true) * (raw_prediction - y_true) + + +cdef inline double cgradient_half_squared_error( + double y_true, + double raw_prediction +) noexcept nogil: + return raw_prediction - y_true + + +cdef inline double_pair cgrad_hess_half_squared_error( + double y_true, + double raw_prediction +) noexcept nogil: + cdef double_pair gh + gh.val1 = raw_prediction - y_true # gradient + gh.val2 = 1. # hessian + return gh + + +# Absolute Error +cdef inline double closs_absolute_error( + double y_true, + double raw_prediction +) noexcept nogil: + return fabs(raw_prediction - y_true) + + +cdef inline double cgradient_absolute_error( + double y_true, + double raw_prediction +) noexcept nogil: + return 1. if raw_prediction > y_true else -1. + + +cdef inline double_pair cgrad_hess_absolute_error( + double y_true, + double raw_prediction +) noexcept nogil: + cdef double_pair gh + # Note that exact hessian = 0 almost everywhere. Optimization routines like + # in HGBT, however, need a hessian > 0. Therefore, we assign 1. + gh.val1 = 1. if raw_prediction > y_true else -1. # gradient + gh.val2 = 1. # hessian + return gh + + +# Quantile Loss / Pinball Loss +cdef inline double closs_pinball_loss( + double y_true, + double raw_prediction, + double quantile +) noexcept nogil: + return (quantile * (y_true - raw_prediction) if y_true >= raw_prediction + else (1. - quantile) * (raw_prediction - y_true)) + + +cdef inline double cgradient_pinball_loss( + double y_true, + double raw_prediction, + double quantile +) noexcept nogil: + return -quantile if y_true >=raw_prediction else 1. - quantile + + +cdef inline double_pair cgrad_hess_pinball_loss( + double y_true, + double raw_prediction, + double quantile +) noexcept nogil: + cdef double_pair gh + # Note that exact hessian = 0 almost everywhere. Optimization routines like + # in HGBT, however, need a hessian > 0. Therefore, we assign 1. + gh.val1 = -quantile if y_true >=raw_prediction else 1. - quantile # gradient + gh.val2 = 1. # hessian + return gh + + +# Huber Loss +cdef inline double closs_huber_loss( + double y_true, + double raw_prediction, + double delta, +) noexcept nogil: + cdef double abserr = fabs(y_true - raw_prediction) + if abserr <= delta: + return 0.5 * abserr**2 + else: + return delta * (abserr - 0.5 * delta) + + +cdef inline double cgradient_huber_loss( + double y_true, + double raw_prediction, + double delta, +) noexcept nogil: + cdef double res = raw_prediction - y_true + if fabs(res) <= delta: + return res + else: + return delta if res >=0 else -delta + + +cdef inline double_pair cgrad_hess_huber_loss( + double y_true, + double raw_prediction, + double delta, +) noexcept nogil: + cdef double_pair gh + gh.val2 = raw_prediction - y_true # used as temporary + if fabs(gh.val2) <= delta: + gh.val1 = gh.val2 # gradient + gh.val2 = 1 # hessian + else: + gh.val1 = delta if gh.val2 >=0 else -delta # gradient + gh.val2 = 0 # hessian + return gh + + +# Half Poisson Deviance with Log-Link, dropping constant terms +cdef inline double closs_half_poisson( + double y_true, + double raw_prediction +) noexcept nogil: + return exp(raw_prediction) - y_true * raw_prediction + + +cdef inline double cgradient_half_poisson( + double y_true, + double raw_prediction +) noexcept nogil: + # y_pred - y_true + return exp(raw_prediction) - y_true + + +cdef inline double_pair closs_grad_half_poisson( + double y_true, + double raw_prediction +) noexcept nogil: + cdef double_pair lg + lg.val2 = exp(raw_prediction) # used as temporary + lg.val1 = lg.val2 - y_true * raw_prediction # loss + lg.val2 -= y_true # gradient + return lg + + +cdef inline double_pair cgrad_hess_half_poisson( + double y_true, + double raw_prediction +) noexcept nogil: + cdef double_pair gh + gh.val2 = exp(raw_prediction) # hessian + gh.val1 = gh.val2 - y_true # gradient + return gh + + +# Half Gamma Deviance with Log-Link, dropping constant terms +cdef inline double closs_half_gamma( + double y_true, + double raw_prediction +) noexcept nogil: + return raw_prediction + y_true * exp(-raw_prediction) + + +cdef inline double cgradient_half_gamma( + double y_true, + double raw_prediction +) noexcept nogil: + return 1. - y_true * exp(-raw_prediction) + + +cdef inline double_pair closs_grad_half_gamma( + double y_true, + double raw_prediction +) noexcept nogil: + cdef double_pair lg + lg.val2 = exp(-raw_prediction) # used as temporary + lg.val1 = raw_prediction + y_true * lg.val2 # loss + lg.val2 = 1. - y_true * lg.val2 # gradient + return lg + + +cdef inline double_pair cgrad_hess_half_gamma( + double y_true, + double raw_prediction +) noexcept nogil: + cdef double_pair gh + gh.val2 = exp(-raw_prediction) # used as temporary + gh.val1 = 1. - y_true * gh.val2 # gradient + gh.val2 *= y_true # hessian + return gh + + +# Half Tweedie Deviance with Log-Link, dropping constant terms +# Note that by dropping constants this is no longer continuous in parameter power. +cdef inline double closs_half_tweedie( + double y_true, + double raw_prediction, + double power +) noexcept nogil: + if power == 0.: + return closs_half_squared_error(y_true, exp(raw_prediction)) + elif power == 1.: + return closs_half_poisson(y_true, raw_prediction) + elif power == 2.: + return closs_half_gamma(y_true, raw_prediction) + else: + return (exp((2. - power) * raw_prediction) / (2. - power) + - y_true * exp((1. - power) * raw_prediction) / (1. - power)) + + +cdef inline double cgradient_half_tweedie( + double y_true, + double raw_prediction, + double power +) noexcept nogil: + cdef double exp1 + if power == 0.: + exp1 = exp(raw_prediction) + return exp1 * (exp1 - y_true) + elif power == 1.: + return cgradient_half_poisson(y_true, raw_prediction) + elif power == 2.: + return cgradient_half_gamma(y_true, raw_prediction) + else: + return (exp((2. - power) * raw_prediction) + - y_true * exp((1. - power) * raw_prediction)) + + +cdef inline double_pair closs_grad_half_tweedie( + double y_true, + double raw_prediction, + double power +) noexcept nogil: + cdef double_pair lg + cdef double exp1, exp2 + if power == 0.: + exp1 = exp(raw_prediction) + lg.val1 = closs_half_squared_error(y_true, exp1) # loss + lg.val2 = exp1 * (exp1 - y_true) # gradient + elif power == 1.: + return closs_grad_half_poisson(y_true, raw_prediction) + elif power == 2.: + return closs_grad_half_gamma(y_true, raw_prediction) + else: + exp1 = exp((1. - power) * raw_prediction) + exp2 = exp((2. - power) * raw_prediction) + lg.val1 = exp2 / (2. - power) - y_true * exp1 / (1. - power) # loss + lg.val2 = exp2 - y_true * exp1 # gradient + return lg + + +cdef inline double_pair cgrad_hess_half_tweedie( + double y_true, + double raw_prediction, + double power +) noexcept nogil: + cdef double_pair gh + cdef double exp1, exp2 + if power == 0.: + exp1 = exp(raw_prediction) + gh.val1 = exp1 * (exp1 - y_true) # gradient + gh.val2 = exp1 * (2 * exp1 - y_true) # hessian + elif power == 1.: + return cgrad_hess_half_poisson(y_true, raw_prediction) + elif power == 2.: + return cgrad_hess_half_gamma(y_true, raw_prediction) + else: + exp1 = exp((1. - power) * raw_prediction) + exp2 = exp((2. - power) * raw_prediction) + gh.val1 = exp2 - y_true * exp1 # gradient + gh.val2 = (2. - power) * exp2 - (1. - power) * y_true * exp1 # hessian + return gh + + +# Half Tweedie Deviance with identity link, without dropping constant terms! +# Therefore, best loss value is zero. +cdef inline double closs_half_tweedie_identity( + double y_true, + double raw_prediction, + double power +) noexcept nogil: + cdef double tmp + if power == 0.: + return closs_half_squared_error(y_true, raw_prediction) + elif power == 1.: + if y_true == 0: + return raw_prediction + else: + return y_true * log(y_true/raw_prediction) + raw_prediction - y_true + elif power == 2.: + return log(raw_prediction/y_true) + y_true/raw_prediction - 1. + else: + tmp = pow(raw_prediction, 1. - power) + tmp = raw_prediction * tmp / (2. - power) - y_true * tmp / (1. - power) + if y_true > 0: + tmp += pow(y_true, 2. - power) / ((1. - power) * (2. - power)) + return tmp + + +cdef inline double cgradient_half_tweedie_identity( + double y_true, + double raw_prediction, + double power +) noexcept nogil: + if power == 0.: + return raw_prediction - y_true + elif power == 1.: + return 1. - y_true / raw_prediction + elif power == 2.: + return (raw_prediction - y_true) / (raw_prediction * raw_prediction) + else: + return pow(raw_prediction, -power) * (raw_prediction - y_true) + + +cdef inline double_pair closs_grad_half_tweedie_identity( + double y_true, + double raw_prediction, + double power +) noexcept nogil: + cdef double_pair lg + cdef double tmp + if power == 0.: + lg.val2 = raw_prediction - y_true # gradient + lg.val1 = 0.5 * lg.val2 * lg.val2 # loss + elif power == 1.: + if y_true == 0: + lg.val1 = raw_prediction + else: + lg.val1 = (y_true * log(y_true/raw_prediction) # loss + + raw_prediction - y_true) + lg.val2 = 1. - y_true / raw_prediction # gradient + elif power == 2.: + lg.val1 = log(raw_prediction/y_true) + y_true/raw_prediction - 1. # loss + tmp = raw_prediction * raw_prediction + lg.val2 = (raw_prediction - y_true) / tmp # gradient + else: + tmp = pow(raw_prediction, 1. - power) + lg.val1 = (raw_prediction * tmp / (2. - power) # loss + - y_true * tmp / (1. - power)) + if y_true > 0: + lg.val1 += (pow(y_true, 2. - power) + / ((1. - power) * (2. - power))) + lg.val2 = tmp * (1. - y_true / raw_prediction) # gradient + return lg + + +cdef inline double_pair cgrad_hess_half_tweedie_identity( + double y_true, + double raw_prediction, + double power +) noexcept nogil: + cdef double_pair gh + cdef double tmp + if power == 0.: + gh.val1 = raw_prediction - y_true # gradient + gh.val2 = 1. # hessian + elif power == 1.: + gh.val1 = 1. - y_true / raw_prediction # gradient + gh.val2 = y_true / (raw_prediction * raw_prediction) # hessian + elif power == 2.: + tmp = raw_prediction * raw_prediction + gh.val1 = (raw_prediction - y_true) / tmp # gradient + gh.val2 = (-1. + 2. * y_true / raw_prediction) / tmp # hessian + else: + tmp = pow(raw_prediction, -power) + gh.val1 = tmp * (raw_prediction - y_true) # gradient + gh.val2 = tmp * ((1. - power) + power * y_true / raw_prediction) # hessian + return gh + + +# Half Binomial deviance with logit-link, aka log-loss or binary cross entropy +cdef inline double closs_half_binomial( + double y_true, + double raw_prediction +) noexcept nogil: + # log1p(exp(raw_prediction)) - y_true * raw_prediction + return log1pexp(raw_prediction) - y_true * raw_prediction + + +cdef inline double cgradient_half_binomial( + double y_true, + double raw_prediction +) noexcept nogil: + # gradient = y_pred - y_true = expit(raw_prediction) - y_true + # Numerically more stable, see http://fa.bianp.net/blog/2019/evaluate_logistic/ + # if raw_prediction < 0: + # exp_tmp = exp(raw_prediction) + # return ((1 - y_true) * exp_tmp - y_true) / (1 + exp_tmp) + # else: + # exp_tmp = exp(-raw_prediction) + # return ((1 - y_true) - y_true * exp_tmp) / (1 + exp_tmp) + # Note that optimal speed would be achieved, at the cost of precision, by + # return expit(raw_prediction) - y_true + # i.e. no "if else" and an own inline implementation of expit instead of + # from scipy.special.cython_special cimport expit + # The case distinction raw_prediction < 0 in the stable implementation does not + # provide significant better precision apart from protecting overflow of exp(..). + # The branch (if else), however, can incur runtime costs of up to 30%. + # Instead, we help branch prediction by almost always ending in the first if clause + # and making the second branch (else) a bit simpler. This has the exact same + # precision but is faster than the stable implementation. + # As branching criteria, we use the same cutoff as in log1pexp. Note that the + # maximal value to get gradient = -1 with y_true = 1 is -37.439198610162731 + # (based on mpmath), and scipy.special.logit(np.finfo(float).eps) ~ -36.04365. + cdef double exp_tmp + if raw_prediction > -37: + exp_tmp = exp(-raw_prediction) + return ((1 - y_true) - y_true * exp_tmp) / (1 + exp_tmp) + else: + # expit(raw_prediction) = exp(raw_prediction) for raw_prediction <= -37 + return exp(raw_prediction) - y_true + + +cdef inline double_pair closs_grad_half_binomial( + double y_true, + double raw_prediction +) noexcept nogil: + cdef double_pair lg + # Same if else conditions as in log1pexp. + if raw_prediction <= -37: + lg.val2 = exp(raw_prediction) # used as temporary + lg.val1 = lg.val2 - y_true * raw_prediction # loss + lg.val2 -= y_true # gradient + elif raw_prediction <= -2: + lg.val2 = exp(raw_prediction) # used as temporary + lg.val1 = log1p(lg.val2) - y_true * raw_prediction # loss + lg.val2 = ((1 - y_true) * lg.val2 - y_true) / (1 + lg.val2) # gradient + elif raw_prediction <= 18: + lg.val2 = exp(-raw_prediction) # used as temporary + # log1p(exp(x)) = log(1 + exp(x)) = x + log1p(exp(-x)) + lg.val1 = log1p(lg.val2) + (1 - y_true) * raw_prediction # loss + lg.val2 = ((1 - y_true) - y_true * lg.val2) / (1 + lg.val2) # gradient + else: + lg.val2 = exp(-raw_prediction) # used as temporary + lg.val1 = lg.val2 + (1 - y_true) * raw_prediction # loss + lg.val2 = ((1 - y_true) - y_true * lg.val2) / (1 + lg.val2) # gradient + return lg + + +cdef inline double_pair cgrad_hess_half_binomial( + double y_true, + double raw_prediction +) noexcept nogil: + # with y_pred = expit(raw) + # hessian = y_pred * (1 - y_pred) = exp( raw) / (1 + exp( raw))**2 + # = exp(-raw) / (1 + exp(-raw))**2 + cdef double_pair gh + # See comment in cgradient_half_binomial. + if raw_prediction > -37: + gh.val2 = exp(-raw_prediction) # used as temporary + gh.val1 = ((1 - y_true) - y_true * gh.val2) / (1 + gh.val2) # gradient + gh.val2 = gh.val2 / (1 + gh.val2)**2 # hessian + else: + gh.val2 = exp(raw_prediction) # = 1. order Taylor in exp(raw_prediction) + gh.val1 = gh.val2 - y_true + return gh + + +# Exponential loss with (half) logit-link, aka boosting loss +cdef inline double closs_exponential( + double y_true, + double raw_prediction +) noexcept nogil: + cdef double tmp = exp(raw_prediction) + return y_true / tmp + (1 - y_true) * tmp + + +cdef inline double cgradient_exponential( + double y_true, + double raw_prediction +) noexcept nogil: + cdef double tmp = exp(raw_prediction) + return -y_true / tmp + (1 - y_true) * tmp + + +cdef inline double_pair closs_grad_exponential( + double y_true, + double raw_prediction +) noexcept nogil: + cdef double_pair lg + lg.val2 = exp(raw_prediction) # used as temporary + + lg.val1 = y_true / lg.val2 + (1 - y_true) * lg.val2 # loss + lg.val2 = -y_true / lg.val2 + (1 - y_true) * lg.val2 # gradient + return lg + + +cdef inline double_pair cgrad_hess_exponential( + double y_true, + double raw_prediction +) noexcept nogil: + # Note that hessian = loss + cdef double_pair gh + gh.val2 = exp(raw_prediction) # used as temporary + + gh.val1 = -y_true / gh.val2 + (1 - y_true) * gh.val2 # gradient + gh.val2 = y_true / gh.val2 + (1 - y_true) * gh.val2 # hessian + return gh + + +# --------------------------------------------------- +# Extension Types for Loss Functions of 1-dim targets +# --------------------------------------------------- +cdef class CyLossFunction: + """Base class for convex loss functions.""" + + def __reduce__(self): + return (self.__class__, ()) + + cdef double cy_loss(self, double y_true, double raw_prediction) noexcept nogil: + """Compute the loss for a single sample. + + Parameters + ---------- + y_true : double + Observed, true target value. + raw_prediction : double + Raw prediction value (in link space). + + Returns + ------- + double + The loss evaluated at `y_true` and `raw_prediction`. + """ + pass + + cdef double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil: + """Compute gradient of loss w.r.t. raw_prediction for a single sample. + + Parameters + ---------- + y_true : double + Observed, true target value. + raw_prediction : double + Raw prediction value (in link space). + + Returns + ------- + double + The derivative of the loss function w.r.t. `raw_prediction`. + """ + pass + + cdef double_pair cy_grad_hess( + self, double y_true, double raw_prediction + ) noexcept nogil: + """Compute gradient and hessian. + + Gradient and hessian of loss w.r.t. raw_prediction for a single sample. + + This is usually diagonal in raw_prediction_i and raw_prediction_j. + Therefore, we return the diagonal element i=j. + + For a loss with a non-canonical link, this might implement the diagonal + of the Fisher matrix (=expected hessian) instead of the hessian. + + Parameters + ---------- + y_true : double + Observed, true target value. + raw_prediction : double + Raw prediction value (in link space). + + Returns + ------- + double_pair + Gradient and hessian of the loss function w.r.t. `raw_prediction`. + """ + pass + + def loss( + self, + const floating_in[::1] y_true, # IN + const floating_in[::1] raw_prediction, # IN + const floating_in[::1] sample_weight, # IN + floating_out[::1] loss_out, # OUT + int n_threads=1 + ): + """Compute the point-wise loss value for each input. + + The point-wise loss is written to `loss_out` and no array is returned. + + Parameters + ---------- + y_true : array of shape (n_samples,) + Observed, true target values. + raw_prediction : array of shape (n_samples,) + Raw prediction values (in link space). + sample_weight : array of shape (n_samples,) or None + Sample weights. + loss_out : array of shape (n_samples,) + A location into which the result is stored. + n_threads : int + Number of threads used by OpenMP (if any). + """ + pass + + def gradient( + self, + const floating_in[::1] y_true, # IN + const floating_in[::1] raw_prediction, # IN + const floating_in[::1] sample_weight, # IN + floating_out[::1] gradient_out, # OUT + int n_threads=1 + ): + """Compute gradient of loss w.r.t raw_prediction for each input. + + The gradient is written to `gradient_out` and no array is returned. + + Parameters + ---------- + y_true : array of shape (n_samples,) + Observed, true target values. + raw_prediction : array of shape (n_samples,) + Raw prediction values (in link space). + sample_weight : array of shape (n_samples,) or None + Sample weights. + gradient_out : array of shape (n_samples,) + A location into which the result is stored. + n_threads : int + Number of threads used by OpenMP (if any). + """ + pass + + def loss_gradient( + self, + const floating_in[::1] y_true, # IN + const floating_in[::1] raw_prediction, # IN + const floating_in[::1] sample_weight, # IN + floating_out[::1] loss_out, # OUT + floating_out[::1] gradient_out, # OUT + int n_threads=1 + ): + """Compute loss and gradient of loss w.r.t raw_prediction. + + The loss and gradient are written to `loss_out` and `gradient_out` and no arrays + are returned. + + Parameters + ---------- + y_true : array of shape (n_samples,) + Observed, true target values. + raw_prediction : array of shape (n_samples,) + Raw prediction values (in link space). + sample_weight : array of shape (n_samples,) or None + Sample weights. + loss_out : array of shape (n_samples,) or None + A location into which the element-wise loss is stored. + gradient_out : array of shape (n_samples,) + A location into which the gradient is stored. + n_threads : int + Number of threads used by OpenMP (if any). + """ + self.loss(y_true, raw_prediction, sample_weight, loss_out, n_threads) + self.gradient(y_true, raw_prediction, sample_weight, gradient_out, n_threads) + + def gradient_hessian( + self, + const floating_in[::1] y_true, # IN + const floating_in[::1] raw_prediction, # IN + const floating_in[::1] sample_weight, # IN + floating_out[::1] gradient_out, # OUT + floating_out[::1] hessian_out, # OUT + int n_threads=1 + ): + """Compute gradient and hessian of loss w.r.t raw_prediction. + + The gradient and hessian are written to `gradient_out` and `hessian_out` and no + arrays are returned. + + Parameters + ---------- + y_true : array of shape (n_samples,) + Observed, true target values. + raw_prediction : array of shape (n_samples,) + Raw prediction values (in link space). + sample_weight : array of shape (n_samples,) or None + Sample weights. + gradient_out : array of shape (n_samples,) + A location into which the gradient is stored. + hessian_out : array of shape (n_samples,) + A location into which the hessian is stored. + n_threads : int + Number of threads used by OpenMP (if any). + """ + pass + + +{{for name, docstring, param, closs, closs_grad, cgrad, cgrad_hess, in class_list}} +{{py: +if param is None: + with_param = "" +else: + with_param = ", self." + param +}} + +cdef class {{name}}(CyLossFunction): + """{{docstring}}""" + + {{if param is not None}} + def __init__(self, {{param}}): + self.{{param}} = {{param}} + {{endif}} + + {{if param is not None}} + def __reduce__(self): + return (self.__class__, (self.{{param}},)) + {{endif}} + + cdef inline double cy_loss(self, double y_true, double raw_prediction) noexcept nogil: + return {{closs}}(y_true, raw_prediction{{with_param}}) + + cdef inline double cy_gradient(self, double y_true, double raw_prediction) noexcept nogil: + return {{cgrad}}(y_true, raw_prediction{{with_param}}) + + cdef inline double_pair cy_grad_hess(self, double y_true, double raw_prediction) noexcept nogil: + return {{cgrad_hess}}(y_true, raw_prediction{{with_param}}) + + def loss( + self, + const floating_in[::1] y_true, # IN + const floating_in[::1] raw_prediction, # IN + const floating_in[::1] sample_weight, # IN + floating_out[::1] loss_out, # OUT + int n_threads=1 + ): + cdef: + int i + int n_samples = y_true.shape[0] + + if sample_weight is None: + for i in prange( + n_samples, schedule='static', nogil=True, num_threads=n_threads + ): + loss_out[i] = {{closs}}(y_true[i], raw_prediction[i]{{with_param}}) + else: + for i in prange( + n_samples, schedule='static', nogil=True, num_threads=n_threads + ): + loss_out[i] = sample_weight[i] * {{closs}}(y_true[i], raw_prediction[i]{{with_param}}) + + {{if closs_grad is not None}} + def loss_gradient( + self, + const floating_in[::1] y_true, # IN + const floating_in[::1] raw_prediction, # IN + const floating_in[::1] sample_weight, # IN + floating_out[::1] loss_out, # OUT + floating_out[::1] gradient_out, # OUT + int n_threads=1 + ): + cdef: + int i + int n_samples = y_true.shape[0] + double_pair dbl2 + + if sample_weight is None: + for i in prange( + n_samples, schedule='static', nogil=True, num_threads=n_threads + ): + dbl2 = {{closs_grad}}(y_true[i], raw_prediction[i]{{with_param}}) + loss_out[i] = dbl2.val1 + gradient_out[i] = dbl2.val2 + else: + for i in prange( + n_samples, schedule='static', nogil=True, num_threads=n_threads + ): + dbl2 = {{closs_grad}}(y_true[i], raw_prediction[i]{{with_param}}) + loss_out[i] = sample_weight[i] * dbl2.val1 + gradient_out[i] = sample_weight[i] * dbl2.val2 + + {{endif}} + + def gradient( + self, + const floating_in[::1] y_true, # IN + const floating_in[::1] raw_prediction, # IN + const floating_in[::1] sample_weight, # IN + floating_out[::1] gradient_out, # OUT + int n_threads=1 + ): + cdef: + int i + int n_samples = y_true.shape[0] + + if sample_weight is None: + for i in prange( + n_samples, schedule='static', nogil=True, num_threads=n_threads + ): + gradient_out[i] = {{cgrad}}(y_true[i], raw_prediction[i]{{with_param}}) + else: + for i in prange( + n_samples, schedule='static', nogil=True, num_threads=n_threads + ): + gradient_out[i] = sample_weight[i] * {{cgrad}}(y_true[i], raw_prediction[i]{{with_param}}) + + def gradient_hessian( + self, + const floating_in[::1] y_true, # IN + const floating_in[::1] raw_prediction, # IN + const floating_in[::1] sample_weight, # IN + floating_out[::1] gradient_out, # OUT + floating_out[::1] hessian_out, # OUT + int n_threads=1 + ): + cdef: + int i + int n_samples = y_true.shape[0] + double_pair dbl2 + + if sample_weight is None: + for i in prange( + n_samples, schedule='static', nogil=True, num_threads=n_threads + ): + dbl2 = {{cgrad_hess}}(y_true[i], raw_prediction[i]{{with_param}}) + gradient_out[i] = dbl2.val1 + hessian_out[i] = dbl2.val2 + else: + for i in prange( + n_samples, schedule='static', nogil=True, num_threads=n_threads + ): + dbl2 = {{cgrad_hess}}(y_true[i], raw_prediction[i]{{with_param}}) + gradient_out[i] = sample_weight[i] * dbl2.val1 + hessian_out[i] = sample_weight[i] * dbl2.val2 + +{{endfor}} + + +# The multinomial deviance loss is also known as categorical cross-entropy or +# multinomial log-likelihood. +# Here, we do not inherit from CyLossFunction as its cy_gradient method deviates +# from the API. +cdef class CyHalfMultinomialLoss(): + """Half Multinomial deviance loss with multinomial logit link. + + Domain: + y_true in {0, 1, 2, 3, .., n_classes - 1} + y_pred in (0, 1)**n_classes, i.e. interval with boundaries excluded + + Link: + y_pred = softmax(raw_prediction) + + Note: Label encoding is built-in, i.e. {0, 1, 2, 3, .., n_classes - 1} is + mapped to (y_true == k) for k = 0 .. n_classes - 1 which is either 0 or 1. + """ + + # Here we deviate from the CyLossFunction API. SAG/SAGA needs direct access to + # sample-wise gradients which we provide here. + cdef inline void cy_gradient( + self, + const floating_in y_true, + const floating_in[::1] raw_prediction, # IN + const floating_in sample_weight, + floating_out[::1] gradient_out, # OUT + ) noexcept nogil: + """Compute gradient of loss w.r.t. `raw_prediction` for a single sample. + + The gradient of the multinomial logistic loss with respect to a class k, + and for one sample is: + grad_k = - sw * (p[k] - (y==k)) + + where: + p[k] = proba[k] = exp(raw_prediction[k] - logsumexp(raw_prediction)) + sw = sample_weight + + Parameters + ---------- + y_true : double + Observed, true target value. + raw_prediction : array of shape (n_classes,) + Raw prediction values (in link space). + sample_weight : double + Sample weight. + gradient_out : array of shape (n_classs,) + A location into which the gradient is stored. + + Returns + ------- + gradient : double + The derivative of the loss function w.r.t. `raw_prediction`. + """ + cdef: + int k + int n_classes = raw_prediction.shape[0] + double_pair max_value_and_sum_exps + const floating_in[:, :] raw = raw_prediction[None, :] + + max_value_and_sum_exps = sum_exp_minus_max(0, raw, &gradient_out[0]) + for k in range(n_classes): + # gradient_out[k] = p_k = y_pred_k = prob of class k + gradient_out[k] /= max_value_and_sum_exps.val2 + # gradient_k = (p_k - (y_true == k)) * sw + gradient_out[k] = (gradient_out[k] - (y_true == k)) * sample_weight + + def _test_cy_gradient( + self, + const floating_in[::1] y_true, # IN + const floating_in[:, ::1] raw_prediction, # IN + const floating_in[::1] sample_weight, # IN + ): + """For testing only.""" + cdef: + int i, k + int n_samples = y_true.shape[0] + int n_classes = raw_prediction.shape[1] + floating_in [:, ::1] gradient_out + gradient = np.empty((n_samples, n_classes), dtype=np.float64) + gradient_out = gradient + + for i in range(n_samples): + self.cy_gradient( + y_true=y_true[i], + raw_prediction=raw_prediction[i, :], + sample_weight=1.0 if sample_weight is None else sample_weight[i], + gradient_out=gradient_out[i, :], + ) + return gradient + + # Note that we do not assume memory alignment/contiguity of 2d arrays. + # There seems to be little benefit in doing so. Benchmarks proofing the + # opposite are welcome. + def loss( + self, + const floating_in[::1] y_true, # IN + const floating_in[:, :] raw_prediction, # IN + const floating_in[::1] sample_weight, # IN + floating_out[::1] loss_out, # OUT + int n_threads=1 + ): + cdef: + int i, k + int n_samples = y_true.shape[0] + int n_classes = raw_prediction.shape[1] + floating_in max_value, sum_exps + floating_in* p # temporary buffer + double_pair max_value_and_sum_exps + + # We assume n_samples > n_classes. In this case having the inner loop + # over n_classes is a good default. + # TODO: If every memoryview is contiguous and raw_prediction is + # f-contiguous, can we write a better algo (loops) to improve + # performance? + if sample_weight is None: + # inner loop over n_classes + with nogil, parallel(num_threads=n_threads): + # Define private buffer variables as each thread might use its + # own. + p = malloc(sizeof(floating_in) * (n_classes)) + + for i in prange(n_samples, schedule='static'): + max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p) + max_value = max_value_and_sum_exps.val1 + sum_exps = max_value_and_sum_exps.val2 + loss_out[i] = log(sum_exps) + max_value + + # label encoded y_true + k = int(y_true[i]) + loss_out[i] -= raw_prediction[i, k] + + free(p) + else: + with nogil, parallel(num_threads=n_threads): + p = malloc(sizeof(floating_in) * (n_classes)) + + for i in prange(n_samples, schedule='static'): + max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p) + max_value = max_value_and_sum_exps.val1 + sum_exps = max_value_and_sum_exps.val2 + loss_out[i] = log(sum_exps) + max_value + + # label encoded y_true + k = int(y_true[i]) + loss_out[i] -= raw_prediction[i, k] + + loss_out[i] *= sample_weight[i] + + free(p) + + def loss_gradient( + self, + const floating_in[::1] y_true, # IN + const floating_in[:, :] raw_prediction, # IN + const floating_in[::1] sample_weight, # IN + floating_out[::1] loss_out, # OUT + floating_out[:, :] gradient_out, # OUT + int n_threads=1 + ): + cdef: + int i, k + int n_samples = y_true.shape[0] + int n_classes = raw_prediction.shape[1] + floating_in max_value, sum_exps + floating_in* p # temporary buffer + double_pair max_value_and_sum_exps + + if sample_weight is None: + # inner loop over n_classes + with nogil, parallel(num_threads=n_threads): + # Define private buffer variables as each thread might use its + # own. + p = malloc(sizeof(floating_in) * (n_classes)) + + for i in prange(n_samples, schedule='static'): + max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p) + max_value = max_value_and_sum_exps.val1 + sum_exps = max_value_and_sum_exps.val2 + loss_out[i] = log(sum_exps) + max_value + + for k in range(n_classes): + # label decode y_true + if y_true[i] == k: + loss_out[i] -= raw_prediction[i, k] + p[k] /= sum_exps # p_k = y_pred_k = prob of class k + # gradient_k = p_k - (y_true == k) + gradient_out[i, k] = p[k] - (y_true[i] == k) + + free(p) + else: + with nogil, parallel(num_threads=n_threads): + p = malloc(sizeof(floating_in) * (n_classes)) + + for i in prange(n_samples, schedule='static'): + max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p) + max_value = max_value_and_sum_exps.val1 + sum_exps = max_value_and_sum_exps.val2 + loss_out[i] = log(sum_exps) + max_value + + for k in range(n_classes): + # label decode y_true + if y_true[i] == k: + loss_out[i] -= raw_prediction[i, k] + p[k] /= sum_exps # p_k = y_pred_k = prob of class k + # gradient_k = (p_k - (y_true == k)) * sw + gradient_out[i, k] = (p[k] - (y_true[i] == k)) * sample_weight[i] + + loss_out[i] *= sample_weight[i] + + free(p) + + def gradient( + self, + const floating_in[::1] y_true, # IN + const floating_in[:, :] raw_prediction, # IN + const floating_in[::1] sample_weight, # IN + floating_out[:, :] gradient_out, # OUT + int n_threads=1 + ): + cdef: + int i, k + int n_samples = y_true.shape[0] + int n_classes = raw_prediction.shape[1] + floating_in sum_exps + floating_in* p # temporary buffer + double_pair max_value_and_sum_exps + + if sample_weight is None: + # inner loop over n_classes + with nogil, parallel(num_threads=n_threads): + # Define private buffer variables as each thread might use its + # own. + p = malloc(sizeof(floating_in) * (n_classes)) + + for i in prange(n_samples, schedule='static'): + max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p) + sum_exps = max_value_and_sum_exps.val2 + + for k in range(n_classes): + p[k] /= sum_exps # p_k = y_pred_k = prob of class k + # gradient_k = y_pred_k - (y_true == k) + gradient_out[i, k] = p[k] - (y_true[i] == k) + + free(p) + else: + with nogil, parallel(num_threads=n_threads): + p = malloc(sizeof(floating_in) * (n_classes)) + + for i in prange(n_samples, schedule='static'): + max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p) + sum_exps = max_value_and_sum_exps.val2 + + for k in range(n_classes): + p[k] /= sum_exps # p_k = y_pred_k = prob of class k + # gradient_k = (p_k - (y_true == k)) * sw + gradient_out[i, k] = (p[k] - (y_true[i] == k)) * sample_weight[i] + + free(p) + + def gradient_hessian( + self, + const floating_in[::1] y_true, # IN + const floating_in[:, :] raw_prediction, # IN + const floating_in[::1] sample_weight, # IN + floating_out[:, :] gradient_out, # OUT + floating_out[:, :] hessian_out, # OUT + int n_threads=1 + ): + cdef: + int i, k + int n_samples = y_true.shape[0] + int n_classes = raw_prediction.shape[1] + floating_in sum_exps + floating_in* p # temporary buffer + double_pair max_value_and_sum_exps + + if sample_weight is None: + # inner loop over n_classes + with nogil, parallel(num_threads=n_threads): + # Define private buffer variables as each thread might use its + # own. + p = malloc(sizeof(floating_in) * (n_classes)) + + for i in prange(n_samples, schedule='static'): + max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p) + sum_exps = max_value_and_sum_exps.val2 + + for k in range(n_classes): + p[k] /= sum_exps # p_k = y_pred_k = prob of class k + # hessian_k = p_k * (1 - p_k) + # gradient_k = p_k - (y_true == k) + gradient_out[i, k] = p[k] - (y_true[i] == k) + hessian_out[i, k] = p[k] * (1. - p[k]) + + free(p) + else: + with nogil, parallel(num_threads=n_threads): + p = malloc(sizeof(floating_in) * (n_classes)) + + for i in prange(n_samples, schedule='static'): + max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p) + sum_exps = max_value_and_sum_exps.val2 + + for k in range(n_classes): + p[k] /= sum_exps # p_k = y_pred_k = prob of class k + # gradient_k = (p_k - (y_true == k)) * sw + # hessian_k = p_k * (1 - p_k) * sw + gradient_out[i, k] = (p[k] - (y_true[i] == k)) * sample_weight[i] + hessian_out[i, k] = (p[k] * (1. - p[k])) * sample_weight[i] + + free(p) + + # This method simplifies the implementation of hessp in linear models, + # i.e. the matrix-vector product of the full hessian, not only of the + # diagonal (in the classes) approximation as implemented above. + def gradient_proba( + self, + const floating_in[::1] y_true, # IN + const floating_in[:, :] raw_prediction, # IN + const floating_in[::1] sample_weight, # IN + floating_out[:, :] gradient_out, # OUT + floating_out[:, :] proba_out, # OUT + int n_threads=1 + ): + cdef: + int i, k + int n_samples = y_true.shape[0] + int n_classes = raw_prediction.shape[1] + floating_in sum_exps + floating_in* p # temporary buffer + double_pair max_value_and_sum_exps + + if sample_weight is None: + # inner loop over n_classes + with nogil, parallel(num_threads=n_threads): + # Define private buffer variables as each thread might use its + # own. + p = malloc(sizeof(floating_in) * (n_classes)) + + for i in prange(n_samples, schedule='static'): + max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p) + sum_exps = max_value_and_sum_exps.val2 + + for k in range(n_classes): + proba_out[i, k] = p[k] / sum_exps # y_pred_k = prob of class k + # gradient_k = y_pred_k - (y_true == k) + gradient_out[i, k] = proba_out[i, k] - (y_true[i] == k) + + free(p) + else: + with nogil, parallel(num_threads=n_threads): + p = malloc(sizeof(floating_in) * (n_classes)) + + for i in prange(n_samples, schedule='static'): + max_value_and_sum_exps = sum_exp_minus_max(i, raw_prediction, p) + sum_exps = max_value_and_sum_exps.val2 + + for k in range(n_classes): + proba_out[i, k] = p[k] / sum_exps # y_pred_k = prob of class k + # gradient_k = (p_k - (y_true == k)) * sw + gradient_out[i, k] = (proba_out[i, k] - (y_true[i] == k)) * sample_weight[i] + + free(p) diff --git a/sklearn/_loss/glm_distribution.py b/sklearn/_loss/glm_distribution.py deleted file mode 100644 index 1cea5ad878904..0000000000000 --- a/sklearn/_loss/glm_distribution.py +++ /dev/null @@ -1,355 +0,0 @@ -""" -Distribution functions used in GLM -""" - -# Author: Christian Lorentzen -# License: BSD 3 clause - -from abc import ABCMeta, abstractmethod -from collections import namedtuple -import numbers - -import numpy as np -from scipy.special import xlogy - - -DistributionBoundary = namedtuple("DistributionBoundary", - ("value", "inclusive")) - - -class ExponentialDispersionModel(metaclass=ABCMeta): - r"""Base class for reproductive Exponential Dispersion Models (EDM). - - The pdf of :math:`Y\sim \mathrm{EDM}(y_\textrm{pred}, \phi)` is given by - - .. math:: p(y| \theta, \phi) = c(y, \phi) - \exp\left(\frac{\theta y-A(\theta)}{\phi}\right) - = \tilde{c}(y, \phi) - \exp\left(-\frac{d(y, y_\textrm{pred})}{2\phi}\right) - - with mean :math:`\mathrm{E}[Y] = A'(\theta) = y_\textrm{pred}`, - variance :math:`\mathrm{Var}[Y] = \phi \cdot v(y_\textrm{pred})`, - unit variance :math:`v(y_\textrm{pred})` and - unit deviance :math:`d(y,y_\textrm{pred})`. - - Methods - ------- - deviance - deviance_derivative - in_y_range - unit_deviance - unit_deviance_derivative - unit_variance - - References - ---------- - https://en.wikipedia.org/wiki/Exponential_dispersion_model. - """ - - def in_y_range(self, y): - """Returns ``True`` if y is in the valid range of Y~EDM. - - Parameters - ---------- - y : array of shape (n_samples,) - Target values. - """ - # Note that currently supported distributions have +inf upper bound - - if not isinstance(self._lower_bound, DistributionBoundary): - raise TypeError('_lower_bound attribute must be of type ' - 'DistributionBoundary') - - if self._lower_bound.inclusive: - return np.greater_equal(y, self._lower_bound.value) - else: - return np.greater(y, self._lower_bound.value) - - @abstractmethod - def unit_variance(self, y_pred): - r"""Compute the unit variance function. - - The unit variance :math:`v(y_\textrm{pred})` determines the variance as - a function of the mean :math:`y_\textrm{pred}` by - :math:`\mathrm{Var}[Y_i] = \phi/s_i*v(y_\textrm{pred}_i)`. - It can also be derived from the unit deviance - :math:`d(y,y_\textrm{pred})` as - - .. math:: v(y_\textrm{pred}) = \frac{2}{ - \frac{\partial^2 d(y,y_\textrm{pred})}{ - \partialy_\textrm{pred}^2}}\big|_{y=y_\textrm{pred}} - - See also :func:`variance`. - - Parameters - ---------- - y_pred : array of shape (n_samples,) - Predicted mean. - """ - - @abstractmethod - def unit_deviance(self, y, y_pred, check_input=False): - r"""Compute the unit deviance. - - The unit_deviance :math:`d(y,y_\textrm{pred})` can be defined by the - log-likelihood as - :math:`d(y,y_\textrm{pred}) = -2\phi\cdot - \left(loglike(y,y_\textrm{pred},\phi) - loglike(y,y,\phi)\right).` - - Parameters - ---------- - y : array of shape (n_samples,) - Target values. - - y_pred : array of shape (n_samples,) - Predicted mean. - - check_input : bool, default=False - If True raise an exception on invalid y or y_pred values, otherwise - they will be propagated as NaN. - Returns - ------- - deviance: array of shape (n_samples,) - Computed deviance - """ - - def unit_deviance_derivative(self, y, y_pred): - r"""Compute the derivative of the unit deviance w.r.t. y_pred. - - The derivative of the unit deviance is given by - :math:`\frac{\partial}{\partialy_\textrm{pred}}d(y,y_\textrm{pred}) - = -2\frac{y-y_\textrm{pred}}{v(y_\textrm{pred})}` - with unit variance :math:`v(y_\textrm{pred})`. - - Parameters - ---------- - y : array of shape (n_samples,) - Target values. - - y_pred : array of shape (n_samples,) - Predicted mean. - """ - return -2 * (y - y_pred) / self.unit_variance(y_pred) - - def deviance(self, y, y_pred, weights=1): - r"""Compute the deviance. - - The deviance is a weighted sum of the per sample unit deviances, - :math:`D = \sum_i s_i \cdot d(y_i, y_\textrm{pred}_i)` - with weights :math:`s_i` and unit deviance - :math:`d(y,y_\textrm{pred})`. - In terms of the log-likelihood it is :math:`D = -2\phi\cdot - \left(loglike(y,y_\textrm{pred},\frac{phi}{s}) - - loglike(y,y,\frac{phi}{s})\right)`. - - Parameters - ---------- - y : array of shape (n_samples,) - Target values. - - y_pred : array of shape (n_samples,) - Predicted mean. - - weights : {int, array of shape (n_samples,)}, default=1 - Weights or exposure to which variance is inverse proportional. - """ - return np.sum(weights * self.unit_deviance(y, y_pred)) - - def deviance_derivative(self, y, y_pred, weights=1): - r"""Compute the derivative of the deviance w.r.t. y_pred. - - It gives :math:`\frac{\partial}{\partial y_\textrm{pred}} - D(y, \y_\textrm{pred}; weights)`. - - Parameters - ---------- - y : array, shape (n_samples,) - Target values. - - y_pred : array, shape (n_samples,) - Predicted mean. - - weights : {int, array of shape (n_samples,)}, default=1 - Weights or exposure to which variance is inverse proportional. - """ - return weights * self.unit_deviance_derivative(y, y_pred) - - -class TweedieDistribution(ExponentialDispersionModel): - r"""A class for the Tweedie distribution. - - A Tweedie distribution with mean :math:`y_\textrm{pred}=\mathrm{E}[Y]` - is uniquely defined by it's mean-variance relationship - :math:`\mathrm{Var}[Y] \propto y_\textrm{pred}^power`. - - Special cases are: - - ===== ================ - Power Distribution - ===== ================ - 0 Normal - 1 Poisson - (1,2) Compound Poisson - 2 Gamma - 3 Inverse Gaussian - - Parameters - ---------- - power : float, default=0 - The variance power of the `unit_variance` - :math:`v(y_\textrm{pred}) = y_\textrm{pred}^{power}`. - For ``0=1.') - elif 1 <= power < 2: - # Poisson or Compound Poisson distribution - self._lower_bound = DistributionBoundary(0, inclusive=True) - elif power >= 2: - # Gamma, Positive Stable, Inverse Gaussian distributions - self._lower_bound = DistributionBoundary(0, inclusive=False) - else: # pragma: no cover - # this branch should be unreachable. - raise ValueError - - self._power = power - - def unit_variance(self, y_pred): - """Compute the unit variance of a Tweedie distribution - v(y_\textrm{pred})=y_\textrm{pred}**power. - - Parameters - ---------- - y_pred : array of shape (n_samples,) - Predicted mean. - """ - return np.power(y_pred, self.power) - - def unit_deviance(self, y, y_pred, check_input=False): - r"""Compute the unit deviance. - - The unit_deviance :math:`d(y,y_\textrm{pred})` can be defined by the - log-likelihood as - :math:`d(y,y_\textrm{pred}) = -2\phi\cdot - \left(loglike(y,y_\textrm{pred},\phi) - loglike(y,y,\phi)\right).` - - Parameters - ---------- - y : array of shape (n_samples,) - Target values. - - y_pred : array of shape (n_samples,) - Predicted mean. - - check_input : bool, default=False - If True raise an exception on invalid y or y_pred values, otherwise - they will be propagated as NaN. - Returns - ------- - deviance: array of shape (n_samples,) - Computed deviance - """ - p = self.power - - if check_input: - message = ("Mean Tweedie deviance error with power={} can only be " - "used on ".format(p)) - if p < 0: - # 'Extreme stable', y any realy number, y_pred > 0 - if (y_pred <= 0).any(): - raise ValueError(message + "strictly positive y_pred.") - elif p == 0: - # Normal, y and y_pred can be any real number - pass - elif 0 < p < 1: - raise ValueError("Tweedie deviance is only defined for " - "power<=0 and power>=1.") - elif 1 <= p < 2: - # Poisson and Compount poisson distribution, y >= 0, y_pred > 0 - if (y < 0).any() or (y_pred <= 0).any(): - raise ValueError(message + "non-negative y and strictly " - "positive y_pred.") - elif p >= 2: - # Gamma and Extreme stable distribution, y and y_pred > 0 - if (y <= 0).any() or (y_pred <= 0).any(): - raise ValueError(message - + "strictly positive y and y_pred.") - else: # pragma: nocover - # Unreachable statement - raise ValueError - - if p < 0: - # 'Extreme stable', y any realy number, y_pred > 0 - dev = 2 * (np.power(np.maximum(y, 0), 2-p) / ((1-p) * (2-p)) - - y * np.power(y_pred, 1-p) / (1-p) - + np.power(y_pred, 2-p) / (2-p)) - - elif p == 0: - # Normal distribution, y and y_pred any real number - dev = (y - y_pred)**2 - elif p < 1: - raise ValueError("Tweedie deviance is only defined for power<=0 " - "and power>=1.") - elif p == 1: - # Poisson distribution - dev = 2 * (xlogy(y, y/y_pred) - y + y_pred) - elif p == 2: - # Gamma distribution - dev = 2 * (np.log(y_pred/y) + y/y_pred - 1) - else: - dev = 2 * (np.power(y, 2-p) / ((1-p) * (2-p)) - - y * np.power(y_pred, 1-p) / (1-p) - + np.power(y_pred, 2-p) / (2-p)) - return dev - - -class NormalDistribution(TweedieDistribution): - """Class for the Normal (aka Gaussian) distribution.""" - def __init__(self): - super().__init__(power=0) - - -class PoissonDistribution(TweedieDistribution): - """Class for the scaled Poisson distribution.""" - def __init__(self): - super().__init__(power=1) - - -class GammaDistribution(TweedieDistribution): - """Class for the Gamma distribution.""" - def __init__(self): - super().__init__(power=2) - - -class InverseGaussianDistribution(TweedieDistribution): - """Class for the scaled InverseGaussianDistribution distribution.""" - def __init__(self): - super().__init__(power=3) - - -EDM_DISTRIBUTIONS = { - 'normal': NormalDistribution, - 'poisson': PoissonDistribution, - 'gamma': GammaDistribution, - 'inverse-gaussian': InverseGaussianDistribution, -} diff --git a/sklearn/_loss/link.py b/sklearn/_loss/link.py new file mode 100644 index 0000000000000..53dff6c2e9285 --- /dev/null +++ b/sklearn/_loss/link.py @@ -0,0 +1,282 @@ +""" +Module contains classes for invertible (and differentiable) link functions. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from abc import ABC, abstractmethod +from dataclasses import dataclass + +import numpy as np +from scipy.special import expit, logit +from scipy.stats import gmean + +from ..utils.extmath import softmax + + +@dataclass +class Interval: + low: float + high: float + low_inclusive: bool + high_inclusive: bool + + def __post_init__(self): + """Check that low <= high""" + if self.low > self.high: + raise ValueError( + f"One must have low <= high; got low={self.low}, high={self.high}." + ) + + def includes(self, x): + """Test whether all values of x are in interval range. + + Parameters + ---------- + x : ndarray + Array whose elements are tested to be in interval range. + + Returns + ------- + result : bool + """ + if self.low_inclusive: + low = np.greater_equal(x, self.low) + else: + low = np.greater(x, self.low) + + if not np.all(low): + return False + + if self.high_inclusive: + high = np.less_equal(x, self.high) + else: + high = np.less(x, self.high) + + # Note: np.all returns numpy.bool_ + return bool(np.all(high)) + + +def _inclusive_low_high(interval, dtype=np.float64): + """Generate values low and high to be within the interval range. + + This is used in tests only. + + Returns + ------- + low, high : tuple + The returned values low and high lie within the interval. + """ + eps = 10 * np.finfo(dtype).eps + if interval.low == -np.inf: + low = -1e10 + elif interval.low < 0: + low = interval.low * (1 - eps) + eps + else: + low = interval.low * (1 + eps) + eps + + if interval.high == np.inf: + high = 1e10 + elif interval.high < 0: + high = interval.high * (1 + eps) - eps + else: + high = interval.high * (1 - eps) - eps + + return low, high + + +class BaseLink(ABC): + """Abstract base class for differentiable, invertible link functions. + + Convention: + - link function g: raw_prediction = g(y_pred) + - inverse link h: y_pred = h(raw_prediction) + + For (generalized) linear models, `raw_prediction = X @ coef` is the so + called linear predictor, and `y_pred = h(raw_prediction)` is the predicted + conditional (on X) expected value of the target `y_true`. + + The methods are not implemented as staticmethods in case a link function needs + parameters. + """ + + is_multiclass = False # used for testing only + + # Usually, raw_prediction may be any real number and y_pred is an open + # interval. + # interval_raw_prediction = Interval(-np.inf, np.inf, False, False) + interval_y_pred = Interval(-np.inf, np.inf, False, False) + + @abstractmethod + def link(self, y_pred, out=None): + """Compute the link function g(y_pred). + + The link function maps (predicted) target values to raw predictions, + i.e. `g(y_pred) = raw_prediction`. + + Parameters + ---------- + y_pred : array + Predicted target values. + out : array + A location into which the result is stored. If provided, it must + have a shape that the inputs broadcast to. If not provided or None, + a freshly-allocated array is returned. + + Returns + ------- + out : array + Output array, element-wise link function. + """ + + @abstractmethod + def inverse(self, raw_prediction, out=None): + """Compute the inverse link function h(raw_prediction). + + The inverse link function maps raw predictions to predicted target + values, i.e. `h(raw_prediction) = y_pred`. + + Parameters + ---------- + raw_prediction : array + Raw prediction values (in link space). + out : array + A location into which the result is stored. If provided, it must + have a shape that the inputs broadcast to. If not provided or None, + a freshly-allocated array is returned. + + Returns + ------- + out : array + Output array, element-wise inverse link function. + """ + + +class IdentityLink(BaseLink): + """The identity link function g(x)=x.""" + + def link(self, y_pred, out=None): + if out is not None: + np.copyto(out, y_pred) + return out + else: + return y_pred + + inverse = link + + +class LogLink(BaseLink): + """The log link function g(x)=log(x).""" + + interval_y_pred = Interval(0, np.inf, False, False) + + def link(self, y_pred, out=None): + return np.log(y_pred, out=out) + + def inverse(self, raw_prediction, out=None): + return np.exp(raw_prediction, out=out) + + +class LogitLink(BaseLink): + """The logit link function g(x)=logit(x).""" + + interval_y_pred = Interval(0, 1, False, False) + + def link(self, y_pred, out=None): + return logit(y_pred, out=out) + + def inverse(self, raw_prediction, out=None): + return expit(raw_prediction, out=out) + + +class HalfLogitLink(BaseLink): + """Half the logit link function g(x)=1/2 * logit(x). + + Used for the exponential loss. + """ + + interval_y_pred = Interval(0, 1, False, False) + + def link(self, y_pred, out=None): + out = logit(y_pred, out=out) + out *= 0.5 + return out + + def inverse(self, raw_prediction, out=None): + return expit(2 * raw_prediction, out) + + +class MultinomialLogit(BaseLink): + """The symmetric multinomial logit function. + + Convention: + - y_pred.shape = raw_prediction.shape = (n_samples, n_classes) + + Notes: + - The inverse link h is the softmax function. + - The sum is over the second axis, i.e. axis=1 (n_classes). + + We have to choose additional constraints in order to make + + y_pred[k] = exp(raw_pred[k]) / sum(exp(raw_pred[k]), k=0..n_classes-1) + + for n_classes classes identifiable and invertible. + We choose the symmetric side constraint where the geometric mean response + is set as reference category, see [2]: + + The symmetric multinomial logit link function for a single data point is + then defined as + + raw_prediction[k] = g(y_pred[k]) = log(y_pred[k]/gmean(y_pred)) + = log(y_pred[k]) - mean(log(y_pred)). + + Note that this is equivalent to the definition in [1] and implies mean + centered raw predictions: + + sum(raw_prediction[k], k=0..n_classes-1) = 0. + + For linear models with raw_prediction = X @ coef, this corresponds to + sum(coef[k], k=0..n_classes-1) = 0, i.e. the sum over classes for every + feature is zero. + + Reference + --------- + .. [1] Friedman, Jerome; Hastie, Trevor; Tibshirani, Robert. "Additive + logistic regression: a statistical view of boosting" Ann. Statist. + 28 (2000), no. 2, 337--407. doi:10.1214/aos/1016218223. + https://projecteuclid.org/euclid.aos/1016218223 + + .. [2] Zahid, Faisal Maqbool and Gerhard Tutz. "Ridge estimation for + multinomial logit models with symmetric side constraints." + Computational Statistics 28 (2013): 1017-1034. + http://epub.ub.uni-muenchen.de/11001/1/tr067.pdf + """ + + is_multiclass = True + interval_y_pred = Interval(0, 1, False, False) + + def symmetrize_raw_prediction(self, raw_prediction): + return raw_prediction - np.mean(raw_prediction, axis=1)[:, np.newaxis] + + def link(self, y_pred, out=None): + # geometric mean as reference category + gm = gmean(y_pred, axis=1) + return np.log(y_pred / gm[:, np.newaxis], out=out) + + def inverse(self, raw_prediction, out=None): + if out is None: + return softmax(raw_prediction, copy=True) + else: + np.copyto(out, raw_prediction) + softmax(out, copy=False) + return out + + +_LINKS = { + "identity": IdentityLink, + "log": LogLink, + "logit": LogitLink, + "half_logit": HalfLogitLink, + "multinomial_logit": MultinomialLogit, +} diff --git a/sklearn/_loss/loss.py b/sklearn/_loss/loss.py new file mode 100644 index 0000000000000..b45ff3322699a --- /dev/null +++ b/sklearn/_loss/loss.py @@ -0,0 +1,1181 @@ +""" +This module contains loss classes suitable for fitting. + +It is not part of the public API. +Specific losses are used for regression, binary classification or multiclass +classification. +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# Goals: +# - Provide a common private module for loss functions/classes. +# - To be used in: +# - LogisticRegression +# - PoissonRegressor, GammaRegressor, TweedieRegressor +# - HistGradientBoostingRegressor, HistGradientBoostingClassifier +# - GradientBoostingRegressor, GradientBoostingClassifier +# - SGDRegressor, SGDClassifier +# - Replace link module of GLMs. + +import numbers + +import numpy as np +from scipy.special import xlogy + +from ..utils import check_scalar +from ..utils.stats import _weighted_percentile +from ._loss import ( + CyAbsoluteError, + CyExponentialLoss, + CyHalfBinomialLoss, + CyHalfGammaLoss, + CyHalfMultinomialLoss, + CyHalfPoissonLoss, + CyHalfSquaredError, + CyHalfTweedieLoss, + CyHalfTweedieLossIdentity, + CyHuberLoss, + CyPinballLoss, +) +from .link import ( + HalfLogitLink, + IdentityLink, + Interval, + LogitLink, + LogLink, + MultinomialLogit, +) + + +# Note: The shape of raw_prediction for multiclass classifications are +# - GradientBoostingClassifier: (n_samples, n_classes) +# - HistGradientBoostingClassifier: (n_classes, n_samples) +# +# Note: Instead of inheritance like +# +# class BaseLoss(BaseLink, CyLossFunction): +# ... +# +# # Note: Naturally, we would inherit in the following order +# # class HalfSquaredError(IdentityLink, CyHalfSquaredError, BaseLoss) +# # But because of https://github.com/cython/cython/issues/4350 we set BaseLoss as +# # the last one. This, of course, changes the MRO. +# class HalfSquaredError(IdentityLink, CyHalfSquaredError, BaseLoss): +# +# we use composition. This way we improve maintainability by avoiding the above +# mentioned Cython edge case and have easier to understand code (which method calls +# which code). +class BaseLoss: + """Base class for a loss function of 1-dimensional targets. + + Conventions: + + - y_true.shape = sample_weight.shape = (n_samples,) + - y_pred.shape = raw_prediction.shape = (n_samples,) + - If is_multiclass is true (multiclass classification), then + y_pred.shape = raw_prediction.shape = (n_samples, n_classes) + Note that this corresponds to the return value of decision_function. + + y_true, y_pred, sample_weight and raw_prediction must either be all float64 + or all float32. + gradient and hessian must be either both float64 or both float32. + + Note that y_pred = link.inverse(raw_prediction). + + Specific loss classes can inherit specific link classes to satisfy + BaseLink's abstractmethods. + + Parameters + ---------- + sample_weight : {None, ndarray} + If sample_weight is None, the hessian might be constant. + n_classes : {None, int} + The number of classes for classification, else None. + + Attributes + ---------- + closs: CyLossFunction + link : BaseLink + interval_y_true : Interval + Valid interval for y_true + interval_y_pred : Interval + Valid Interval for y_pred + differentiable : bool + Indicates whether or not loss function is differentiable in + raw_prediction everywhere. + need_update_leaves_values : bool + Indicates whether decision trees in gradient boosting need to uptade + leave values after having been fit to the (negative) gradients. + approx_hessian : bool + Indicates whether the hessian is approximated or exact. If, + approximated, it should be larger or equal to the exact one. + constant_hessian : bool + Indicates whether the hessian is one for this loss. + is_multiclass : bool + Indicates whether n_classes > 2 is allowed. + """ + + # For gradient boosted decision trees: + # This variable indicates whether the loss requires the leaves values to + # be updated once the tree has been trained. The trees are trained to + # predict a Newton-Raphson step (see grower._finalize_leaf()). But for + # some losses (e.g. least absolute deviation) we need to adjust the tree + # values to account for the "line search" of the gradient descent + # procedure. See the original paper Greedy Function Approximation: A + # Gradient Boosting Machine by Friedman + # (https://statweb.stanford.edu/~jhf/ftp/trebst.pdf) for the theory. + differentiable = True + need_update_leaves_values = False + is_multiclass = False + + def __init__(self, closs, link, n_classes=None): + self.closs = closs + self.link = link + self.approx_hessian = False + self.constant_hessian = False + self.n_classes = n_classes + self.interval_y_true = Interval(-np.inf, np.inf, False, False) + self.interval_y_pred = self.link.interval_y_pred + + def in_y_true_range(self, y): + """Return True if y is in the valid range of y_true. + + Parameters + ---------- + y : ndarray + """ + return self.interval_y_true.includes(y) + + def in_y_pred_range(self, y): + """Return True if y is in the valid range of y_pred. + + Parameters + ---------- + y : ndarray + """ + return self.interval_y_pred.includes(y) + + def loss( + self, + y_true, + raw_prediction, + sample_weight=None, + loss_out=None, + n_threads=1, + ): + """Compute the pointwise loss value for each input. + + Parameters + ---------- + y_true : C-contiguous array of shape (n_samples,) + Observed, true target values. + raw_prediction : C-contiguous array of shape (n_samples,) or array of \ + shape (n_samples, n_classes) + Raw prediction values (in link space). + sample_weight : None or C-contiguous array of shape (n_samples,) + Sample weights. + loss_out : None or C-contiguous array of shape (n_samples,) + A location into which the result is stored. If None, a new array + might be created. + n_threads : int, default=1 + Might use openmp thread parallelism. + + Returns + ------- + loss : array of shape (n_samples,) + Element-wise loss function. + """ + if loss_out is None: + loss_out = np.empty_like(y_true) + # Be graceful to shape (n_samples, 1) -> (n_samples,) + if raw_prediction.ndim == 2 and raw_prediction.shape[1] == 1: + raw_prediction = raw_prediction.squeeze(1) + + self.closs.loss( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + loss_out=loss_out, + n_threads=n_threads, + ) + return loss_out + + def loss_gradient( + self, + y_true, + raw_prediction, + sample_weight=None, + loss_out=None, + gradient_out=None, + n_threads=1, + ): + """Compute loss and gradient w.r.t. raw_prediction for each input. + + Parameters + ---------- + y_true : C-contiguous array of shape (n_samples,) + Observed, true target values. + raw_prediction : C-contiguous array of shape (n_samples,) or array of \ + shape (n_samples, n_classes) + Raw prediction values (in link space). + sample_weight : None or C-contiguous array of shape (n_samples,) + Sample weights. + loss_out : None or C-contiguous array of shape (n_samples,) + A location into which the loss is stored. If None, a new array + might be created. + gradient_out : None or C-contiguous array of shape (n_samples,) or array \ + of shape (n_samples, n_classes) + A location into which the gradient is stored. If None, a new array + might be created. + n_threads : int, default=1 + Might use openmp thread parallelism. + + Returns + ------- + loss : array of shape (n_samples,) + Element-wise loss function. + + gradient : array of shape (n_samples,) or (n_samples, n_classes) + Element-wise gradients. + """ + if loss_out is None: + if gradient_out is None: + loss_out = np.empty_like(y_true) + gradient_out = np.empty_like(raw_prediction) + else: + loss_out = np.empty_like(y_true, dtype=gradient_out.dtype) + elif gradient_out is None: + gradient_out = np.empty_like(raw_prediction, dtype=loss_out.dtype) + + # Be graceful to shape (n_samples, 1) -> (n_samples,) + if raw_prediction.ndim == 2 and raw_prediction.shape[1] == 1: + raw_prediction = raw_prediction.squeeze(1) + if gradient_out.ndim == 2 and gradient_out.shape[1] == 1: + gradient_out = gradient_out.squeeze(1) + + self.closs.loss_gradient( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + loss_out=loss_out, + gradient_out=gradient_out, + n_threads=n_threads, + ) + return loss_out, gradient_out + + def gradient( + self, + y_true, + raw_prediction, + sample_weight=None, + gradient_out=None, + n_threads=1, + ): + """Compute gradient of loss w.r.t raw_prediction for each input. + + Parameters + ---------- + y_true : C-contiguous array of shape (n_samples,) + Observed, true target values. + raw_prediction : C-contiguous array of shape (n_samples,) or array of \ + shape (n_samples, n_classes) + Raw prediction values (in link space). + sample_weight : None or C-contiguous array of shape (n_samples,) + Sample weights. + gradient_out : None or C-contiguous array of shape (n_samples,) or array \ + of shape (n_samples, n_classes) + A location into which the result is stored. If None, a new array + might be created. + n_threads : int, default=1 + Might use openmp thread parallelism. + + Returns + ------- + gradient : array of shape (n_samples,) or (n_samples, n_classes) + Element-wise gradients. + """ + if gradient_out is None: + gradient_out = np.empty_like(raw_prediction) + + # Be graceful to shape (n_samples, 1) -> (n_samples,) + if raw_prediction.ndim == 2 and raw_prediction.shape[1] == 1: + raw_prediction = raw_prediction.squeeze(1) + if gradient_out.ndim == 2 and gradient_out.shape[1] == 1: + gradient_out = gradient_out.squeeze(1) + + self.closs.gradient( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + gradient_out=gradient_out, + n_threads=n_threads, + ) + return gradient_out + + def gradient_hessian( + self, + y_true, + raw_prediction, + sample_weight=None, + gradient_out=None, + hessian_out=None, + n_threads=1, + ): + """Compute gradient and hessian of loss w.r.t raw_prediction. + + Parameters + ---------- + y_true : C-contiguous array of shape (n_samples,) + Observed, true target values. + raw_prediction : C-contiguous array of shape (n_samples,) or array of \ + shape (n_samples, n_classes) + Raw prediction values (in link space). + sample_weight : None or C-contiguous array of shape (n_samples,) + Sample weights. + gradient_out : None or C-contiguous array of shape (n_samples,) or array \ + of shape (n_samples, n_classes) + A location into which the gradient is stored. If None, a new array + might be created. + hessian_out : None or C-contiguous array of shape (n_samples,) or array \ + of shape (n_samples, n_classes) + A location into which the hessian is stored. If None, a new array + might be created. + n_threads : int, default=1 + Might use openmp thread parallelism. + + Returns + ------- + gradient : arrays of shape (n_samples,) or (n_samples, n_classes) + Element-wise gradients. + + hessian : arrays of shape (n_samples,) or (n_samples, n_classes) + Element-wise hessians. + """ + if gradient_out is None: + if hessian_out is None: + gradient_out = np.empty_like(raw_prediction) + hessian_out = np.empty_like(raw_prediction) + else: + gradient_out = np.empty_like(hessian_out) + elif hessian_out is None: + hessian_out = np.empty_like(gradient_out) + + # Be graceful to shape (n_samples, 1) -> (n_samples,) + if raw_prediction.ndim == 2 and raw_prediction.shape[1] == 1: + raw_prediction = raw_prediction.squeeze(1) + if gradient_out.ndim == 2 and gradient_out.shape[1] == 1: + gradient_out = gradient_out.squeeze(1) + if hessian_out.ndim == 2 and hessian_out.shape[1] == 1: + hessian_out = hessian_out.squeeze(1) + + self.closs.gradient_hessian( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + gradient_out=gradient_out, + hessian_out=hessian_out, + n_threads=n_threads, + ) + return gradient_out, hessian_out + + def __call__(self, y_true, raw_prediction, sample_weight=None, n_threads=1): + """Compute the weighted average loss. + + Parameters + ---------- + y_true : C-contiguous array of shape (n_samples,) + Observed, true target values. + raw_prediction : C-contiguous array of shape (n_samples,) or array of \ + shape (n_samples, n_classes) + Raw prediction values (in link space). + sample_weight : None or C-contiguous array of shape (n_samples,) + Sample weights. + n_threads : int, default=1 + Might use openmp thread parallelism. + + Returns + ------- + loss : float + Mean or averaged loss function. + """ + return np.average( + self.loss( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=None, + loss_out=None, + n_threads=n_threads, + ), + weights=sample_weight, + ) + + def fit_intercept_only(self, y_true, sample_weight=None): + """Compute raw_prediction of an intercept-only model. + + This can be used as initial estimates of predictions, i.e. before the + first iteration in fit. + + Parameters + ---------- + y_true : array-like of shape (n_samples,) + Observed, true target values. + sample_weight : None or array of shape (n_samples,) + Sample weights. + + Returns + ------- + raw_prediction : numpy scalar or array of shape (n_classes,) + Raw predictions of an intercept-only model. + """ + # As default, take weighted average of the target over the samples + # axis=0 and then transform into link-scale (raw_prediction). + y_pred = np.average(y_true, weights=sample_weight, axis=0) + eps = 10 * np.finfo(y_pred.dtype).eps + + if self.interval_y_pred.low == -np.inf: + a_min = None + elif self.interval_y_pred.low_inclusive: + a_min = self.interval_y_pred.low + else: + a_min = self.interval_y_pred.low + eps + + if self.interval_y_pred.high == np.inf: + a_max = None + elif self.interval_y_pred.high_inclusive: + a_max = self.interval_y_pred.high + else: + a_max = self.interval_y_pred.high - eps + + if a_min is None and a_max is None: + return self.link.link(y_pred) + else: + return self.link.link(np.clip(y_pred, a_min, a_max)) + + def constant_to_optimal_zero(self, y_true, sample_weight=None): + """Calculate term dropped in loss. + + With this term added, the loss of perfect predictions is zero. + """ + return np.zeros_like(y_true) + + def init_gradient_and_hessian(self, n_samples, dtype=np.float64, order="F"): + """Initialize arrays for gradients and hessians. + + Unless hessians are constant, arrays are initialized with undefined values. + + Parameters + ---------- + n_samples : int + The number of samples, usually passed to `fit()`. + dtype : {np.float64, np.float32}, default=np.float64 + The dtype of the arrays gradient and hessian. + order : {'C', 'F'}, default='F' + Order of the arrays gradient and hessian. The default 'F' makes the arrays + contiguous along samples. + + Returns + ------- + gradient : C-contiguous array of shape (n_samples,) or array of shape \ + (n_samples, n_classes) + Empty array (allocated but not initialized) to be used as argument + gradient_out. + hessian : C-contiguous array of shape (n_samples,), array of shape + (n_samples, n_classes) or shape (1,) + Empty (allocated but not initialized) array to be used as argument + hessian_out. + If constant_hessian is True (e.g. `HalfSquaredError`), the array is + initialized to ``1``. + """ + if dtype not in (np.float32, np.float64): + raise ValueError( + "Valid options for 'dtype' are np.float32 and np.float64. " + f"Got dtype={dtype} instead." + ) + + if self.is_multiclass: + shape = (n_samples, self.n_classes) + else: + shape = (n_samples,) + gradient = np.empty(shape=shape, dtype=dtype, order=order) + + if self.constant_hessian: + # If the hessians are constant, we consider them equal to 1. + # - This is correct for HalfSquaredError + # - For AbsoluteError, hessians are actually 0, but they are + # always ignored anyway. + hessian = np.ones(shape=(1,), dtype=dtype) + else: + hessian = np.empty(shape=shape, dtype=dtype, order=order) + + return gradient, hessian + + +# Note: Naturally, we would inherit in the following order +# class HalfSquaredError(IdentityLink, CyHalfSquaredError, BaseLoss) +# But because of https://github.com/cython/cython/issues/4350 we +# set BaseLoss as the last one. This, of course, changes the MRO. +class HalfSquaredError(BaseLoss): + """Half squared error with identity link, for regression. + + Domain: + y_true and y_pred all real numbers + + Link: + y_pred = raw_prediction + + For a given sample x_i, half squared error is defined as:: + + loss(x_i) = 0.5 * (y_true_i - raw_prediction_i)**2 + + The factor of 0.5 simplifies the computation of gradients and results in a + unit hessian (and is consistent with what is done in LightGBM). It is also + half the Normal distribution deviance. + """ + + def __init__(self, sample_weight=None): + super().__init__(closs=CyHalfSquaredError(), link=IdentityLink()) + self.constant_hessian = sample_weight is None + + +class AbsoluteError(BaseLoss): + """Absolute error with identity link, for regression. + + Domain: + y_true and y_pred all real numbers + + Link: + y_pred = raw_prediction + + For a given sample x_i, the absolute error is defined as:: + + loss(x_i) = |y_true_i - raw_prediction_i| + + Note that the exact hessian = 0 almost everywhere (except at one point, therefore + differentiable = False). Optimization routines like in HGBT, however, need a + hessian > 0. Therefore, we assign 1. + """ + + differentiable = False + need_update_leaves_values = True + + def __init__(self, sample_weight=None): + super().__init__(closs=CyAbsoluteError(), link=IdentityLink()) + self.approx_hessian = True + self.constant_hessian = sample_weight is None + + def fit_intercept_only(self, y_true, sample_weight=None): + """Compute raw_prediction of an intercept-only model. + + This is the weighted median of the target, i.e. over the samples + axis=0. + """ + if sample_weight is None: + return np.median(y_true, axis=0) + else: + return _weighted_percentile(y_true, sample_weight, 50) + + +class PinballLoss(BaseLoss): + """Quantile loss aka pinball loss, for regression. + + Domain: + y_true and y_pred all real numbers + quantile in (0, 1) + + Link: + y_pred = raw_prediction + + For a given sample x_i, the pinball loss is defined as:: + + loss(x_i) = rho_{quantile}(y_true_i - raw_prediction_i) + + rho_{quantile}(u) = u * (quantile - 1_{u<0}) + = -u *(1 - quantile) if u < 0 + u * quantile if u >= 0 + + Note: 2 * PinballLoss(quantile=0.5) equals AbsoluteError(). + + Note that the exact hessian = 0 almost everywhere (except at one point, therefore + differentiable = False). Optimization routines like in HGBT, however, need a + hessian > 0. Therefore, we assign 1. + + Additional Attributes + --------------------- + quantile : float + The quantile level of the quantile to be estimated. Must be in range (0, 1). + """ + + differentiable = False + need_update_leaves_values = True + + def __init__(self, sample_weight=None, quantile=0.5): + check_scalar( + quantile, + "quantile", + target_type=numbers.Real, + min_val=0, + max_val=1, + include_boundaries="neither", + ) + super().__init__( + closs=CyPinballLoss(quantile=float(quantile)), + link=IdentityLink(), + ) + self.approx_hessian = True + self.constant_hessian = sample_weight is None + + def fit_intercept_only(self, y_true, sample_weight=None): + """Compute raw_prediction of an intercept-only model. + + This is the weighted median of the target, i.e. over the samples + axis=0. + """ + if sample_weight is None: + return np.percentile(y_true, 100 * self.closs.quantile, axis=0) + else: + return _weighted_percentile( + y_true, sample_weight, 100 * self.closs.quantile + ) + + +class HuberLoss(BaseLoss): + """Huber loss, for regression. + + Domain: + y_true and y_pred all real numbers + quantile in (0, 1) + + Link: + y_pred = raw_prediction + + For a given sample x_i, the Huber loss is defined as:: + + loss(x_i) = 1/2 * abserr**2 if abserr <= delta + delta * (abserr - delta/2) if abserr > delta + + abserr = |y_true_i - raw_prediction_i| + delta = quantile(abserr, self.quantile) + + Note: HuberLoss(quantile=1) equals HalfSquaredError and HuberLoss(quantile=0) + equals delta * (AbsoluteError() - delta/2). + + Additional Attributes + --------------------- + quantile : float + The quantile level which defines the breaking point `delta` to distinguish + between absolute error and squared error. Must be in range (0, 1). + + Reference + --------- + .. [1] Friedman, J.H. (2001). :doi:`Greedy function approximation: A gradient + boosting machine <10.1214/aos/1013203451>`. + Annals of Statistics, 29, 1189-1232. + """ + + differentiable = False + need_update_leaves_values = True + + def __init__(self, sample_weight=None, quantile=0.9, delta=0.5): + check_scalar( + quantile, + "quantile", + target_type=numbers.Real, + min_val=0, + max_val=1, + include_boundaries="neither", + ) + self.quantile = quantile # This is better stored outside of Cython. + super().__init__( + closs=CyHuberLoss(delta=float(delta)), + link=IdentityLink(), + ) + self.approx_hessian = True + self.constant_hessian = False + + def fit_intercept_only(self, y_true, sample_weight=None): + """Compute raw_prediction of an intercept-only model. + + This is the weighted median of the target, i.e. over the samples + axis=0. + """ + # See formula before algo 4 in Friedman (2001), but we apply it to y_true, + # not to the residual y_true - raw_prediction. An estimator like + # HistGradientBoostingRegressor might then call it on the residual, e.g. + # fit_intercept_only(y_true - raw_prediction). + if sample_weight is None: + median = np.percentile(y_true, 50, axis=0) + else: + median = _weighted_percentile(y_true, sample_weight, 50) + diff = y_true - median + term = np.sign(diff) * np.minimum(self.closs.delta, np.abs(diff)) + return median + np.average(term, weights=sample_weight) + + +class HalfPoissonLoss(BaseLoss): + """Half Poisson deviance loss with log-link, for regression. + + Domain: + y_true in non-negative real numbers + y_pred in positive real numbers + + Link: + y_pred = exp(raw_prediction) + + For a given sample x_i, half the Poisson deviance is defined as:: + + loss(x_i) = y_true_i * log(y_true_i/exp(raw_prediction_i)) + - y_true_i + exp(raw_prediction_i) + + Half the Poisson deviance is actually the negative log-likelihood up to + constant terms (not involving raw_prediction) and simplifies the + computation of the gradients. + We also skip the constant term `y_true_i * log(y_true_i) - y_true_i`. + """ + + def __init__(self, sample_weight=None): + super().__init__(closs=CyHalfPoissonLoss(), link=LogLink()) + self.interval_y_true = Interval(0, np.inf, True, False) + + def constant_to_optimal_zero(self, y_true, sample_weight=None): + term = xlogy(y_true, y_true) - y_true + if sample_weight is not None: + term *= sample_weight + return term + + +class HalfGammaLoss(BaseLoss): + """Half Gamma deviance loss with log-link, for regression. + + Domain: + y_true and y_pred in positive real numbers + + Link: + y_pred = exp(raw_prediction) + + For a given sample x_i, half Gamma deviance loss is defined as:: + + loss(x_i) = log(exp(raw_prediction_i)/y_true_i) + + y_true/exp(raw_prediction_i) - 1 + + Half the Gamma deviance is actually proportional to the negative log- + likelihood up to constant terms (not involving raw_prediction) and + simplifies the computation of the gradients. + We also skip the constant term `-log(y_true_i) - 1`. + """ + + def __init__(self, sample_weight=None): + super().__init__(closs=CyHalfGammaLoss(), link=LogLink()) + self.interval_y_true = Interval(0, np.inf, False, False) + + def constant_to_optimal_zero(self, y_true, sample_weight=None): + term = -np.log(y_true) - 1 + if sample_weight is not None: + term *= sample_weight + return term + + +class HalfTweedieLoss(BaseLoss): + """Half Tweedie deviance loss with log-link, for regression. + + Domain: + y_true in real numbers for power <= 0 + y_true in non-negative real numbers for 0 < power < 2 + y_true in positive real numbers for 2 <= power + y_pred in positive real numbers + power in real numbers + + Link: + y_pred = exp(raw_prediction) + + For a given sample x_i, half Tweedie deviance loss with p=power is defined + as:: + + loss(x_i) = max(y_true_i, 0)**(2-p) / (1-p) / (2-p) + - y_true_i * exp(raw_prediction_i)**(1-p) / (1-p) + + exp(raw_prediction_i)**(2-p) / (2-p) + + Taking the limits for p=0, 1, 2 gives HalfSquaredError with a log link, + HalfPoissonLoss and HalfGammaLoss. + + We also skip constant terms, but those are different for p=0, 1, 2. + Therefore, the loss is not continuous in `power`. + + Note furthermore that although no Tweedie distribution exists for + 0 < power < 1, it still gives a strictly consistent scoring function for + the expectation. + """ + + def __init__(self, sample_weight=None, power=1.5): + super().__init__( + closs=CyHalfTweedieLoss(power=float(power)), + link=LogLink(), + ) + if self.closs.power <= 0: + self.interval_y_true = Interval(-np.inf, np.inf, False, False) + elif self.closs.power < 2: + self.interval_y_true = Interval(0, np.inf, True, False) + else: + self.interval_y_true = Interval(0, np.inf, False, False) + + def constant_to_optimal_zero(self, y_true, sample_weight=None): + if self.closs.power == 0: + return HalfSquaredError().constant_to_optimal_zero( + y_true=y_true, sample_weight=sample_weight + ) + elif self.closs.power == 1: + return HalfPoissonLoss().constant_to_optimal_zero( + y_true=y_true, sample_weight=sample_weight + ) + elif self.closs.power == 2: + return HalfGammaLoss().constant_to_optimal_zero( + y_true=y_true, sample_weight=sample_weight + ) + else: + p = self.closs.power + term = np.power(np.maximum(y_true, 0), 2 - p) / (1 - p) / (2 - p) + if sample_weight is not None: + term *= sample_weight + return term + + +class HalfTweedieLossIdentity(BaseLoss): + """Half Tweedie deviance loss with identity link, for regression. + + Domain: + y_true in real numbers for power <= 0 + y_true in non-negative real numbers for 0 < power < 2 + y_true in positive real numbers for 2 <= power + y_pred in positive real numbers for power != 0 + y_pred in real numbers for power = 0 + power in real numbers + + Link: + y_pred = raw_prediction + + For a given sample x_i, half Tweedie deviance loss with p=power is defined + as:: + + loss(x_i) = max(y_true_i, 0)**(2-p) / (1-p) / (2-p) + - y_true_i * raw_prediction_i**(1-p) / (1-p) + + raw_prediction_i**(2-p) / (2-p) + + Note that the minimum value of this loss is 0. + + Note furthermore that although no Tweedie distribution exists for + 0 < power < 1, it still gives a strictly consistent scoring function for + the expectation. + """ + + def __init__(self, sample_weight=None, power=1.5): + super().__init__( + closs=CyHalfTweedieLossIdentity(power=float(power)), + link=IdentityLink(), + ) + if self.closs.power <= 0: + self.interval_y_true = Interval(-np.inf, np.inf, False, False) + elif self.closs.power < 2: + self.interval_y_true = Interval(0, np.inf, True, False) + else: + self.interval_y_true = Interval(0, np.inf, False, False) + + if self.closs.power == 0: + self.interval_y_pred = Interval(-np.inf, np.inf, False, False) + else: + self.interval_y_pred = Interval(0, np.inf, False, False) + + +class HalfBinomialLoss(BaseLoss): + """Half Binomial deviance loss with logit link, for binary classification. + + This is also know as binary cross entropy, log-loss and logistic loss. + + Domain: + y_true in [0, 1], i.e. regression on the unit interval + y_pred in (0, 1), i.e. boundaries excluded + + Link: + y_pred = expit(raw_prediction) + + For a given sample x_i, half Binomial deviance is defined as the negative + log-likelihood of the Binomial/Bernoulli distribution and can be expressed + as:: + + loss(x_i) = log(1 + exp(raw_pred_i)) - y_true_i * raw_pred_i + + See The Elements of Statistical Learning, by Hastie, Tibshirani, Friedman, + section 4.4.1 (about logistic regression). + + Note that the formulation works for classification, y = {0, 1}, as well as + logistic regression, y = [0, 1]. + If you add `constant_to_optimal_zero` to the loss, you get half the + Bernoulli/binomial deviance. + + More details: Inserting the predicted probability y_pred = expit(raw_prediction) + in the loss gives the well known:: + + loss(x_i) = - y_true_i * log(y_pred_i) - (1 - y_true_i) * log(1 - y_pred_i) + """ + + def __init__(self, sample_weight=None): + super().__init__( + closs=CyHalfBinomialLoss(), + link=LogitLink(), + n_classes=2, + ) + self.interval_y_true = Interval(0, 1, True, True) + + def constant_to_optimal_zero(self, y_true, sample_weight=None): + # This is non-zero only if y_true is neither 0 nor 1. + term = xlogy(y_true, y_true) + xlogy(1 - y_true, 1 - y_true) + if sample_weight is not None: + term *= sample_weight + return term + + def predict_proba(self, raw_prediction): + """Predict probabilities. + + Parameters + ---------- + raw_prediction : array of shape (n_samples,) or (n_samples, 1) + Raw prediction values (in link space). + + Returns + ------- + proba : array of shape (n_samples, 2) + Element-wise class probabilities. + """ + # Be graceful to shape (n_samples, 1) -> (n_samples,) + if raw_prediction.ndim == 2 and raw_prediction.shape[1] == 1: + raw_prediction = raw_prediction.squeeze(1) + proba = np.empty((raw_prediction.shape[0], 2), dtype=raw_prediction.dtype) + proba[:, 1] = self.link.inverse(raw_prediction) + proba[:, 0] = 1 - proba[:, 1] + return proba + + +class HalfMultinomialLoss(BaseLoss): + """Categorical cross-entropy loss, for multiclass classification. + + Domain: + y_true in {0, 1, 2, 3, .., n_classes - 1} + y_pred has n_classes elements, each element in (0, 1) + + Link: + y_pred = softmax(raw_prediction) + + Note: We assume y_true to be already label encoded. The inverse link is + softmax. But the full link function is the symmetric multinomial logit + function. + + For a given sample x_i, the categorical cross-entropy loss is defined as + the negative log-likelihood of the multinomial distribution, it + generalizes the binary cross-entropy to more than 2 classes:: + + loss_i = log(sum(exp(raw_pred_{i, k}), k=0..n_classes-1)) + - sum(y_true_{i, k} * raw_pred_{i, k}, k=0..n_classes-1) + + See [1]. + + Note that for the hessian, we calculate only the diagonal part in the + classes: If the full hessian for classes k and l and sample i is H_i_k_l, + we calculate H_i_k_k, i.e. k=l. + + Reference + --------- + .. [1] :arxiv:`Simon, Noah, J. Friedman and T. Hastie. + "A Blockwise Descent Algorithm for Group-penalized Multiresponse and + Multinomial Regression". + <1311.6529>` + """ + + is_multiclass = True + + def __init__(self, sample_weight=None, n_classes=3): + super().__init__( + closs=CyHalfMultinomialLoss(), + link=MultinomialLogit(), + n_classes=n_classes, + ) + self.interval_y_true = Interval(0, np.inf, True, False) + self.interval_y_pred = Interval(0, 1, False, False) + + def in_y_true_range(self, y): + """Return True if y is in the valid range of y_true. + + Parameters + ---------- + y : ndarray + """ + return self.interval_y_true.includes(y) and np.all(y.astype(int) == y) + + def fit_intercept_only(self, y_true, sample_weight=None): + """Compute raw_prediction of an intercept-only model. + + This is the softmax of the weighted average of the target, i.e. over + the samples axis=0. + """ + out = np.zeros(self.n_classes, dtype=y_true.dtype) + eps = np.finfo(y_true.dtype).eps + for k in range(self.n_classes): + out[k] = np.average(y_true == k, weights=sample_weight, axis=0) + out[k] = np.clip(out[k], eps, 1 - eps) + return self.link.link(out[None, :]).reshape(-1) + + def predict_proba(self, raw_prediction): + """Predict probabilities. + + Parameters + ---------- + raw_prediction : array of shape (n_samples, n_classes) + Raw prediction values (in link space). + + Returns + ------- + proba : array of shape (n_samples, n_classes) + Element-wise class probabilities. + """ + return self.link.inverse(raw_prediction) + + def gradient_proba( + self, + y_true, + raw_prediction, + sample_weight=None, + gradient_out=None, + proba_out=None, + n_threads=1, + ): + """Compute gradient and class probabilities fow raw_prediction. + + Parameters + ---------- + y_true : C-contiguous array of shape (n_samples,) + Observed, true target values. + raw_prediction : array of shape (n_samples, n_classes) + Raw prediction values (in link space). + sample_weight : None or C-contiguous array of shape (n_samples,) + Sample weights. + gradient_out : None or array of shape (n_samples, n_classes) + A location into which the gradient is stored. If None, a new array + might be created. + proba_out : None or array of shape (n_samples, n_classes) + A location into which the class probabilities are stored. If None, + a new array might be created. + n_threads : int, default=1 + Might use openmp thread parallelism. + + Returns + ------- + gradient : array of shape (n_samples, n_classes) + Element-wise gradients. + + proba : array of shape (n_samples, n_classes) + Element-wise class probabilities. + """ + if gradient_out is None: + if proba_out is None: + gradient_out = np.empty_like(raw_prediction) + proba_out = np.empty_like(raw_prediction) + else: + gradient_out = np.empty_like(proba_out) + elif proba_out is None: + proba_out = np.empty_like(gradient_out) + + self.closs.gradient_proba( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + gradient_out=gradient_out, + proba_out=proba_out, + n_threads=n_threads, + ) + return gradient_out, proba_out + + +class ExponentialLoss(BaseLoss): + """Exponential loss with (half) logit link, for binary classification. + + This is also know as boosting loss. + + Domain: + y_true in [0, 1], i.e. regression on the unit interval + y_pred in (0, 1), i.e. boundaries excluded + + Link: + y_pred = expit(2 * raw_prediction) + + For a given sample x_i, the exponential loss is defined as:: + + loss(x_i) = y_true_i * exp(-raw_pred_i)) + (1 - y_true_i) * exp(raw_pred_i) + + See: + - J. Friedman, T. Hastie, R. Tibshirani. + "Additive logistic regression: a statistical view of boosting (With discussion + and a rejoinder by the authors)." Ann. Statist. 28 (2) 337 - 407, April 2000. + https://doi.org/10.1214/aos/1016218223 + - A. Buja, W. Stuetzle, Y. Shen. (2005). + "Loss Functions for Binary Class Probability Estimation and Classification: + Structure and Applications." + + Note that the formulation works for classification, y = {0, 1}, as well as + "exponential logistic" regression, y = [0, 1]. + Note that this is a proper scoring rule, but without it's canonical link. + + More details: Inserting the predicted probability + y_pred = expit(2 * raw_prediction) in the loss gives:: + + loss(x_i) = y_true_i * sqrt((1 - y_pred_i) / y_pred_i) + + (1 - y_true_i) * sqrt(y_pred_i / (1 - y_pred_i)) + """ + + def __init__(self, sample_weight=None): + super().__init__( + closs=CyExponentialLoss(), + link=HalfLogitLink(), + n_classes=2, + ) + self.interval_y_true = Interval(0, 1, True, True) + + def constant_to_optimal_zero(self, y_true, sample_weight=None): + # This is non-zero only if y_true is neither 0 nor 1. + term = -2 * np.sqrt(y_true * (1 - y_true)) + if sample_weight is not None: + term *= sample_weight + return term + + def predict_proba(self, raw_prediction): + """Predict probabilities. + + Parameters + ---------- + raw_prediction : array of shape (n_samples,) or (n_samples, 1) + Raw prediction values (in link space). + + Returns + ------- + proba : array of shape (n_samples, 2) + Element-wise class probabilities. + """ + # Be graceful to shape (n_samples, 1) -> (n_samples,) + if raw_prediction.ndim == 2 and raw_prediction.shape[1] == 1: + raw_prediction = raw_prediction.squeeze(1) + proba = np.empty((raw_prediction.shape[0], 2), dtype=raw_prediction.dtype) + proba[:, 1] = self.link.inverse(raw_prediction) + proba[:, 0] = 1 - proba[:, 1] + return proba + + +_LOSSES = { + "squared_error": HalfSquaredError, + "absolute_error": AbsoluteError, + "pinball_loss": PinballLoss, + "huber_loss": HuberLoss, + "poisson_loss": HalfPoissonLoss, + "gamma_loss": HalfGammaLoss, + "tweedie_loss": HalfTweedieLoss, + "binomial_loss": HalfBinomialLoss, + "multinomial_loss": HalfMultinomialLoss, + "exponential_loss": ExponentialLoss, +} diff --git a/sklearn/_loss/meson.build b/sklearn/_loss/meson.build new file mode 100644 index 0000000000000..a4b3425a21cd2 --- /dev/null +++ b/sklearn/_loss/meson.build @@ -0,0 +1,23 @@ +# .pyx is generated, so this is needed to make Cython compilation work +_loss_cython_tree = [ + fs.copyfile('_loss.pxd') +] + +_loss_pyx = custom_target( + '_loss_pyx', + output: '_loss.pyx', + input: '_loss.pyx.tp', + command: [tempita, '@INPUT@', '-o', '@OUTDIR@'], + # TODO in principle this should go in py.exension_module below. This is + # temporary work-around for dependency issue with .pyx.tp files. For more + # details, see https://github.com/mesonbuild/meson/issues/13212 + depends: _loss_cython_tree, +) + +py.extension_module( + '_loss', + cython_gen.process(_loss_pyx), + dependencies: [openmp_dep], + install: true, + subdir: 'sklearn/_loss', +) diff --git a/sklearn/_loss/tests/test_glm_distribution.py b/sklearn/_loss/tests/test_glm_distribution.py deleted file mode 100644 index cb4c5ae07e4d1..0000000000000 --- a/sklearn/_loss/tests/test_glm_distribution.py +++ /dev/null @@ -1,112 +0,0 @@ -# Authors: Christian Lorentzen -# -# License: BSD 3 clause -import numpy as np -from numpy.testing import ( - assert_allclose, - assert_array_equal, -) -from scipy.optimize import check_grad -import pytest - -from sklearn._loss.glm_distribution import ( - TweedieDistribution, - NormalDistribution, PoissonDistribution, - GammaDistribution, InverseGaussianDistribution, - DistributionBoundary -) - - -@pytest.mark.parametrize( - 'family, expected', - [(NormalDistribution(), [True, True, True]), - (PoissonDistribution(), [False, True, True]), - (TweedieDistribution(power=1.5), [False, True, True]), - (GammaDistribution(), [False, False, True]), - (InverseGaussianDistribution(), [False, False, True]), - (TweedieDistribution(power=4.5), [False, False, True])]) -def test_family_bounds(family, expected): - """Test the valid range of distributions at -1, 0, 1.""" - result = family.in_y_range([-1, 0, 1]) - assert_array_equal(result, expected) - - -def test_invalid_distribution_bound(): - dist = TweedieDistribution() - dist._lower_bound = 0 - with pytest.raises(TypeError, - match="must be of type DistributionBoundary"): - dist.in_y_range([-1, 0, 1]) - - -def test_tweedie_distribution_power(): - msg = "distribution is only defined for power<=0 and power>=1" - with pytest.raises(ValueError, match=msg): - TweedieDistribution(power=0.5) - - with pytest.raises(TypeError, match="must be a real number"): - TweedieDistribution(power=1j) - - with pytest.raises(TypeError, match="must be a real number"): - dist = TweedieDistribution() - dist.power = 1j - - dist = TweedieDistribution() - assert isinstance(dist._lower_bound, DistributionBoundary) - - assert dist._lower_bound.inclusive is False - dist.power = 1 - assert dist._lower_bound.value == 0.0 - assert dist._lower_bound.inclusive is True - - -@pytest.mark.parametrize( - 'family, chk_values', - [(NormalDistribution(), [-1.5, -0.1, 0.1, 2.5]), - (PoissonDistribution(), [0.1, 1.5]), - (GammaDistribution(), [0.1, 1.5]), - (InverseGaussianDistribution(), [0.1, 1.5]), - (TweedieDistribution(power=-2.5), [0.1, 1.5]), - (TweedieDistribution(power=-1), [0.1, 1.5]), - (TweedieDistribution(power=1.5), [0.1, 1.5]), - (TweedieDistribution(power=2.5), [0.1, 1.5]), - (TweedieDistribution(power=-4), [0.1, 1.5])]) -def test_deviance_zero(family, chk_values): - """Test deviance(y,y) = 0 for different families.""" - for x in chk_values: - assert_allclose(family.deviance(x, x), 0, atol=1e-9) - - -@pytest.mark.parametrize( - 'family', - [NormalDistribution(), - PoissonDistribution(), - GammaDistribution(), - InverseGaussianDistribution(), - TweedieDistribution(power=-2.5), - TweedieDistribution(power=-1), - TweedieDistribution(power=1.5), - TweedieDistribution(power=2.5), - TweedieDistribution(power=-4)], - ids=lambda x: x.__class__.__name__ -) -def test_deviance_derivative(family): - """Test deviance derivative for different families.""" - rng = np.random.RandomState(0) - y_true = rng.rand(10) - # make data positive - y_true += np.abs(y_true.min()) + 1e-2 - - y_pred = y_true + np.fmax(rng.rand(10), 0.) - - dev = family.deviance(y_true, y_pred) - assert isinstance(dev, float) - dev_derivative = family.deviance_derivative(y_true, y_pred) - assert dev_derivative.shape == y_pred.shape - - err = check_grad( - lambda y_pred: family.deviance(y_true, y_pred), - lambda y_pred: family.deviance_derivative(y_true, y_pred), - y_pred, - ) / np.linalg.norm(dev_derivative) - assert abs(err) < 1e-6 diff --git a/sklearn/_loss/tests/test_link.py b/sklearn/_loss/tests/test_link.py new file mode 100644 index 0000000000000..e5a665f8d48ac --- /dev/null +++ b/sklearn/_loss/tests/test_link.py @@ -0,0 +1,111 @@ +import numpy as np +import pytest +from numpy.testing import assert_allclose, assert_array_equal + +from sklearn._loss.link import ( + _LINKS, + HalfLogitLink, + Interval, + MultinomialLogit, + _inclusive_low_high, +) + +LINK_FUNCTIONS = list(_LINKS.values()) + + +def test_interval_raises(): + """Test that interval with low > high raises ValueError.""" + with pytest.raises( + ValueError, match="One must have low <= high; got low=1, high=0." + ): + Interval(1, 0, False, False) + + +@pytest.mark.parametrize( + "interval", + [ + Interval(0, 1, False, False), + Interval(0, 1, False, True), + Interval(0, 1, True, False), + Interval(0, 1, True, True), + Interval(-np.inf, np.inf, False, False), + Interval(-np.inf, np.inf, False, True), + Interval(-np.inf, np.inf, True, False), + Interval(-np.inf, np.inf, True, True), + Interval(-10, -1, False, False), + Interval(-10, -1, False, True), + Interval(-10, -1, True, False), + Interval(-10, -1, True, True), + ], +) +def test_is_in_range(interval): + # make sure low and high are always within the interval, used for linspace + low, high = _inclusive_low_high(interval) + + x = np.linspace(low, high, num=10) + assert interval.includes(x) + + # x contains lower bound + assert interval.includes(np.r_[x, interval.low]) == interval.low_inclusive + + # x contains upper bound + assert interval.includes(np.r_[x, interval.high]) == interval.high_inclusive + + # x contains upper and lower bound + assert interval.includes(np.r_[x, interval.low, interval.high]) == ( + interval.low_inclusive and interval.high_inclusive + ) + + +@pytest.mark.parametrize("link", LINK_FUNCTIONS) +def test_link_inverse_identity(link, global_random_seed): + # Test that link of inverse gives identity. + rng = np.random.RandomState(global_random_seed) + link = link() + n_samples, n_classes = 100, None + # The values for `raw_prediction` are limited from -20 to 20 because in the + # class `LogitLink` the term `expit(x)` comes very close to 1 for large + # positive x and therefore loses precision. + if link.is_multiclass: + n_classes = 10 + raw_prediction = rng.uniform(low=-20, high=20, size=(n_samples, n_classes)) + if isinstance(link, MultinomialLogit): + raw_prediction = link.symmetrize_raw_prediction(raw_prediction) + elif isinstance(link, HalfLogitLink): + raw_prediction = rng.uniform(low=-10, high=10, size=(n_samples)) + else: + raw_prediction = rng.uniform(low=-20, high=20, size=(n_samples)) + + assert_allclose(link.link(link.inverse(raw_prediction)), raw_prediction) + y_pred = link.inverse(raw_prediction) + assert_allclose(link.inverse(link.link(y_pred)), y_pred) + + +@pytest.mark.parametrize("link", LINK_FUNCTIONS) +def test_link_out_argument(link): + # Test that out argument gets assigned the result. + rng = np.random.RandomState(42) + link = link() + n_samples, n_classes = 100, None + if link.is_multiclass: + n_classes = 10 + raw_prediction = rng.normal(loc=0, scale=10, size=(n_samples, n_classes)) + if isinstance(link, MultinomialLogit): + raw_prediction = link.symmetrize_raw_prediction(raw_prediction) + else: + # So far, the valid interval of raw_prediction is (-inf, inf) and + # we do not need to distinguish. + raw_prediction = rng.uniform(low=-10, high=10, size=(n_samples)) + + y_pred = link.inverse(raw_prediction, out=None) + out = np.empty_like(raw_prediction) + y_pred_2 = link.inverse(raw_prediction, out=out) + assert_allclose(y_pred, out) + assert_array_equal(out, y_pred_2) + assert np.shares_memory(out, y_pred_2) + + out = np.empty_like(y_pred) + raw_prediction_2 = link.link(y_pred, out=out) + assert_allclose(raw_prediction, out) + assert_array_equal(out, raw_prediction_2) + assert np.shares_memory(out, raw_prediction_2) diff --git a/sklearn/_loss/tests/test_loss.py b/sklearn/_loss/tests/test_loss.py new file mode 100644 index 0000000000000..4fea325729023 --- /dev/null +++ b/sklearn/_loss/tests/test_loss.py @@ -0,0 +1,1358 @@ +import pickle + +import numpy as np +import pytest +from numpy.testing import assert_allclose, assert_array_equal +from pytest import approx +from scipy.optimize import ( + LinearConstraint, + minimize, + minimize_scalar, + newton, +) +from scipy.special import logsumexp + +from sklearn._loss.link import IdentityLink, _inclusive_low_high +from sklearn._loss.loss import ( + _LOSSES, + AbsoluteError, + BaseLoss, + HalfBinomialLoss, + HalfGammaLoss, + HalfMultinomialLoss, + HalfPoissonLoss, + HalfSquaredError, + HalfTweedieLoss, + HalfTweedieLossIdentity, + HuberLoss, + PinballLoss, +) +from sklearn.utils import assert_all_finite +from sklearn.utils._testing import create_memmap_backed_data, skip_if_32bit + +ALL_LOSSES = list(_LOSSES.values()) + +LOSS_INSTANCES = [loss() for loss in ALL_LOSSES] +# HalfTweedieLoss(power=1.5) is already there as default +LOSS_INSTANCES += [ + PinballLoss(quantile=0.25), + HuberLoss(quantile=0.75), + HalfTweedieLoss(power=-1.5), + HalfTweedieLoss(power=0), + HalfTweedieLoss(power=1), + HalfTweedieLoss(power=2), + HalfTweedieLoss(power=3.0), + HalfTweedieLossIdentity(power=0), + HalfTweedieLossIdentity(power=1), + HalfTweedieLossIdentity(power=2), + HalfTweedieLossIdentity(power=3.0), +] + + +def loss_instance_name(param): + if isinstance(param, BaseLoss): + loss = param + name = loss.__class__.__name__ + if isinstance(loss, PinballLoss): + name += f"(quantile={loss.closs.quantile})" + elif isinstance(loss, HuberLoss): + name += f"(quantile={loss.quantile}" + elif hasattr(loss, "closs") and hasattr(loss.closs, "power"): + name += f"(power={loss.closs.power})" + return name + else: + return str(param) + + +def random_y_true_raw_prediction( + loss, n_samples, y_bound=(-100, 100), raw_bound=(-5, 5), seed=42 +): + """Random generate y_true and raw_prediction in valid range.""" + rng = np.random.RandomState(seed) + if loss.is_multiclass: + raw_prediction = np.empty((n_samples, loss.n_classes)) + raw_prediction.flat[:] = rng.uniform( + low=raw_bound[0], + high=raw_bound[1], + size=n_samples * loss.n_classes, + ) + y_true = np.arange(n_samples).astype(float) % loss.n_classes + else: + # If link is identity, we must respect the interval of y_pred: + if isinstance(loss.link, IdentityLink): + low, high = _inclusive_low_high(loss.interval_y_pred) + low = np.amax([low, raw_bound[0]]) + high = np.amin([high, raw_bound[1]]) + raw_bound = (low, high) + raw_prediction = rng.uniform( + low=raw_bound[0], high=raw_bound[1], size=n_samples + ) + # generate a y_true in valid range + low, high = _inclusive_low_high(loss.interval_y_true) + low = max(low, y_bound[0]) + high = min(high, y_bound[1]) + y_true = rng.uniform(low, high, size=n_samples) + # set some values at special boundaries + if loss.interval_y_true.low == 0 and loss.interval_y_true.low_inclusive: + y_true[:: (n_samples // 3)] = 0 + if loss.interval_y_true.high == 1 and loss.interval_y_true.high_inclusive: + y_true[1 :: (n_samples // 3)] = 1 + + return y_true, raw_prediction + + +def numerical_derivative(func, x, eps): + """Helper function for numerical (first) derivatives.""" + # For numerical derivatives, see + # https://en.wikipedia.org/wiki/Numerical_differentiation + # https://en.wikipedia.org/wiki/Finite_difference_coefficient + # We use central finite differences of accuracy 4. + h = np.full_like(x, fill_value=eps) + f_minus_2h = func(x - 2 * h) + f_minus_1h = func(x - h) + f_plus_1h = func(x + h) + f_plus_2h = func(x + 2 * h) + return (-f_plus_2h + 8 * f_plus_1h - 8 * f_minus_1h + f_minus_2h) / (12.0 * eps) + + +@pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name) +def test_loss_boundary(loss): + """Test interval ranges of y_true and y_pred in losses.""" + # make sure low and high are always within the interval, used for linspace + if loss.is_multiclass: + n_classes = 3 # default value + y_true = np.tile(np.linspace(0, n_classes - 1, num=n_classes), 3) + else: + low, high = _inclusive_low_high(loss.interval_y_true) + y_true = np.linspace(low, high, num=10) + + # add boundaries if they are included + if loss.interval_y_true.low_inclusive: + y_true = np.r_[y_true, loss.interval_y_true.low] + if loss.interval_y_true.high_inclusive: + y_true = np.r_[y_true, loss.interval_y_true.high] + + assert loss.in_y_true_range(y_true) + + n = y_true.shape[0] + low, high = _inclusive_low_high(loss.interval_y_pred) + if loss.is_multiclass: + y_pred = np.empty((n, n_classes)) + y_pred[:, 0] = np.linspace(low, high, num=n) + y_pred[:, 1] = 0.5 * (1 - y_pred[:, 0]) + y_pred[:, 2] = 0.5 * (1 - y_pred[:, 0]) + else: + y_pred = np.linspace(low, high, num=n) + + assert loss.in_y_pred_range(y_pred) + + # calculating losses should not fail + raw_prediction = loss.link.link(y_pred) + loss.loss(y_true=y_true, raw_prediction=raw_prediction) + + +# Fixture to test valid value ranges. +Y_COMMON_PARAMS = [ + # (loss, [y success], [y fail]) + (HalfSquaredError(), [-100, 0, 0.1, 100], [-np.inf, np.inf]), + (AbsoluteError(), [-100, 0, 0.1, 100], [-np.inf, np.inf]), + (PinballLoss(), [-100, 0, 0.1, 100], [-np.inf, np.inf]), + (HuberLoss(), [-100, 0, 0.1, 100], [-np.inf, np.inf]), + (HalfPoissonLoss(), [0.1, 100], [-np.inf, -3, -0.1, np.inf]), + (HalfGammaLoss(), [0.1, 100], [-np.inf, -3, -0.1, 0, np.inf]), + (HalfTweedieLoss(power=-3), [0.1, 100], [-np.inf, np.inf]), + (HalfTweedieLoss(power=0), [0.1, 100], [-np.inf, np.inf]), + (HalfTweedieLoss(power=1.5), [0.1, 100], [-np.inf, -3, -0.1, np.inf]), + (HalfTweedieLoss(power=2), [0.1, 100], [-np.inf, -3, -0.1, 0, np.inf]), + (HalfTweedieLoss(power=3), [0.1, 100], [-np.inf, -3, -0.1, 0, np.inf]), + (HalfTweedieLossIdentity(power=-3), [0.1, 100], [-np.inf, np.inf]), + (HalfTweedieLossIdentity(power=0), [-3, -0.1, 0, 0.1, 100], [-np.inf, np.inf]), + (HalfTweedieLossIdentity(power=1.5), [0.1, 100], [-np.inf, -3, -0.1, np.inf]), + (HalfTweedieLossIdentity(power=2), [0.1, 100], [-np.inf, -3, -0.1, 0, np.inf]), + (HalfTweedieLossIdentity(power=3), [0.1, 100], [-np.inf, -3, -0.1, 0, np.inf]), + (HalfBinomialLoss(), [0.1, 0.5, 0.9], [-np.inf, -1, 2, np.inf]), + (HalfMultinomialLoss(), [], [-np.inf, -1, 1.1, np.inf]), +] +# y_pred and y_true do not always have the same domain (valid value range). +# Hence, we define extra sets of parameters for each of them. +Y_TRUE_PARAMS = [ # type: ignore[var-annotated] + # (loss, [y success], [y fail]) + (HalfPoissonLoss(), [0], []), + (HuberLoss(), [0], []), + (HalfTweedieLoss(power=-3), [-100, -0.1, 0], []), + (HalfTweedieLoss(power=0), [-100, 0], []), + (HalfTweedieLoss(power=1.5), [0], []), + (HalfTweedieLossIdentity(power=-3), [-100, -0.1, 0], []), + (HalfTweedieLossIdentity(power=0), [-100, 0], []), + (HalfTweedieLossIdentity(power=1.5), [0], []), + (HalfBinomialLoss(), [0, 1], []), + (HalfMultinomialLoss(), [0.0, 1.0, 2], []), +] +Y_PRED_PARAMS = [ + # (loss, [y success], [y fail]) + (HalfPoissonLoss(), [], [0]), + (HalfTweedieLoss(power=-3), [], [-3, -0.1, 0]), + (HalfTweedieLoss(power=0), [], [-3, -0.1, 0]), + (HalfTweedieLoss(power=1.5), [], [0]), + (HalfTweedieLossIdentity(power=-3), [], [-3, -0.1, 0]), + (HalfTweedieLossIdentity(power=0), [-3, -0.1, 0], []), + (HalfTweedieLossIdentity(power=1.5), [], [0]), + (HalfBinomialLoss(), [], [0, 1]), + (HalfMultinomialLoss(), [0.1, 0.5], [0, 1]), +] + + +@pytest.mark.parametrize( + "loss, y_true_success, y_true_fail", + Y_COMMON_PARAMS + Y_TRUE_PARAMS, # type: ignore[operator] +) +def test_loss_boundary_y_true(loss, y_true_success, y_true_fail): + """Test boundaries of y_true for loss functions.""" + for y in y_true_success: + assert loss.in_y_true_range(np.array([y])) + for y in y_true_fail: + assert not loss.in_y_true_range(np.array([y])) + + +@pytest.mark.parametrize( + "loss, y_pred_success, y_pred_fail", + Y_COMMON_PARAMS + Y_PRED_PARAMS, # type: ignore[operator] +) +def test_loss_boundary_y_pred(loss, y_pred_success, y_pred_fail): + """Test boundaries of y_pred for loss functions.""" + for y in y_pred_success: + assert loss.in_y_pred_range(np.array([y])) + for y in y_pred_fail: + assert not loss.in_y_pred_range(np.array([y])) + + +@pytest.mark.parametrize( + "loss, y_true, raw_prediction, loss_true, gradient_true, hessian_true", + [ + (HalfSquaredError(), 1.0, 5.0, 8, 4, 1), + (AbsoluteError(), 1.0, 5.0, 4.0, 1.0, None), + (PinballLoss(quantile=0.5), 1.0, 5.0, 2, 0.5, None), + (PinballLoss(quantile=0.25), 1.0, 5.0, 4 * (1 - 0.25), 1 - 0.25, None), + (PinballLoss(quantile=0.25), 5.0, 1.0, 4 * 0.25, -0.25, None), + (HuberLoss(quantile=0.5, delta=3), 1.0, 5.0, 3 * (4 - 3 / 2), None, None), + (HuberLoss(quantile=0.5, delta=3), 1.0, 3.0, 0.5 * 2**2, None, None), + (HalfPoissonLoss(), 2.0, np.log(4), 4 - 2 * np.log(4), 4 - 2, 4), + (HalfGammaLoss(), 2.0, np.log(4), np.log(4) + 2 / 4, 1 - 2 / 4, 2 / 4), + (HalfTweedieLoss(power=3), 2.0, np.log(4), -1 / 4 + 1 / 4**2, None, None), + (HalfTweedieLossIdentity(power=1), 2.0, 4.0, 2 - 2 * np.log(2), None, None), + (HalfTweedieLossIdentity(power=2), 2.0, 4.0, np.log(2) - 1 / 2, None, None), + ( + HalfTweedieLossIdentity(power=3), + 2.0, + 4.0, + -1 / 4 + 1 / 4**2 + 1 / 2 / 2, + None, + None, + ), + ( + HalfBinomialLoss(), + 0.25, + np.log(4), + np.log1p(4) - 0.25 * np.log(4), + None, + None, + ), + # Extreme log loss cases, checked with mpmath: + # import mpmath as mp + # + # # Stolen from scipy + # def mpf2float(x): + # return float(mp.nstr(x, 17, min_fixed=0, max_fixed=0)) + # + # def mp_logloss(y_true, raw): + # with mp.workdps(100): + # y_true, raw = mp.mpf(float(y_true)), mp.mpf(float(raw)) + # out = mp.log1p(mp.exp(raw)) - y_true * raw + # return mpf2float(out) + # + # def mp_gradient(y_true, raw): + # with mp.workdps(100): + # y_true, raw = mp.mpf(float(y_true)), mp.mpf(float(raw)) + # out = mp.mpf(1) / (mp.mpf(1) + mp.exp(-raw)) - y_true + # return mpf2float(out) + # + # def mp_hessian(y_true, raw): + # with mp.workdps(100): + # y_true, raw = mp.mpf(float(y_true)), mp.mpf(float(raw)) + # p = mp.mpf(1) / (mp.mpf(1) + mp.exp(-raw)) + # out = p * (mp.mpf(1) - p) + # return mpf2float(out) + # + # y, raw = 0.0, 37. + # mp_logloss(y, raw), mp_gradient(y, raw), mp_hessian(y, raw) + (HalfBinomialLoss(), 0.0, -1e20, 0, 0, 0), + (HalfBinomialLoss(), 1.0, -1e20, 1e20, -1, 0), + (HalfBinomialLoss(), 0.0, -1e3, 0, 0, 0), + (HalfBinomialLoss(), 1.0, -1e3, 1e3, -1, 0), + (HalfBinomialLoss(), 1.0, -37.5, 37.5, -1, 0), + (HalfBinomialLoss(), 1.0, -37.0, 37, 1e-16 - 1, 8.533047625744065e-17), + (HalfBinomialLoss(), 0.0, -37.0, *[8.533047625744065e-17] * 3), + (HalfBinomialLoss(), 1.0, -36.9, 36.9, 1e-16 - 1, 9.430476078526806e-17), + (HalfBinomialLoss(), 0.0, -36.9, *[9.430476078526806e-17] * 3), + (HalfBinomialLoss(), 0.0, 37.0, 37, 1 - 1e-16, 8.533047625744065e-17), + (HalfBinomialLoss(), 1.0, 37.0, *[8.533047625744066e-17] * 3), + (HalfBinomialLoss(), 0.0, 37.5, 37.5, 1, 5.175555005801868e-17), + (HalfBinomialLoss(), 0.0, 232.8, 232.8, 1, 1.4287342391028437e-101), + (HalfBinomialLoss(), 1.0, 1e20, 0, 0, 0), + (HalfBinomialLoss(), 0.0, 1e20, 1e20, 1, 0), + ( + HalfBinomialLoss(), + 1.0, + 232.8, + 0, + -1.4287342391028437e-101, + 1.4287342391028437e-101, + ), + (HalfBinomialLoss(), 1.0, 232.9, 0, 0, 0), + (HalfBinomialLoss(), 1.0, 1e3, 0, 0, 0), + (HalfBinomialLoss(), 0.0, 1e3, 1e3, 1, 0), + ( + HalfMultinomialLoss(n_classes=3), + 0.0, + [0.2, 0.5, 0.3], + logsumexp([0.2, 0.5, 0.3]) - 0.2, + None, + None, + ), + ( + HalfMultinomialLoss(n_classes=3), + 1.0, + [0.2, 0.5, 0.3], + logsumexp([0.2, 0.5, 0.3]) - 0.5, + None, + None, + ), + ( + HalfMultinomialLoss(n_classes=3), + 2.0, + [0.2, 0.5, 0.3], + logsumexp([0.2, 0.5, 0.3]) - 0.3, + None, + None, + ), + ( + HalfMultinomialLoss(n_classes=3), + 2.0, + [1e4, 0, 7e-7], + logsumexp([1e4, 0, 7e-7]) - (7e-7), + None, + None, + ), + ], + ids=loss_instance_name, +) +def test_loss_on_specific_values( + loss, y_true, raw_prediction, loss_true, gradient_true, hessian_true +): + """Test losses, gradients and hessians at specific values.""" + loss1 = loss(y_true=np.array([y_true]), raw_prediction=np.array([raw_prediction])) + grad1 = loss.gradient( + y_true=np.array([y_true]), raw_prediction=np.array([raw_prediction]) + ) + loss2, grad2 = loss.loss_gradient( + y_true=np.array([y_true]), raw_prediction=np.array([raw_prediction]) + ) + grad3, hess = loss.gradient_hessian( + y_true=np.array([y_true]), raw_prediction=np.array([raw_prediction]) + ) + + assert loss1 == approx(loss_true, rel=1e-15, abs=1e-15) + assert loss2 == approx(loss_true, rel=1e-15, abs=1e-15) + + if gradient_true is not None: + assert grad1 == approx(gradient_true, rel=1e-15, abs=1e-15) + assert grad2 == approx(gradient_true, rel=1e-15, abs=1e-15) + assert grad3 == approx(gradient_true, rel=1e-15, abs=1e-15) + + if hessian_true is not None: + assert hess == approx(hessian_true, rel=1e-15, abs=1e-15) + + +@pytest.mark.parametrize("loss", ALL_LOSSES) +@pytest.mark.parametrize("readonly_memmap", [False, True]) +@pytest.mark.parametrize("dtype_in", [np.float32, np.float64]) +@pytest.mark.parametrize("dtype_out", [np.float32, np.float64]) +@pytest.mark.parametrize("sample_weight", [None, 1]) +@pytest.mark.parametrize("out1", [None, 1]) +@pytest.mark.parametrize("out2", [None, 1]) +@pytest.mark.parametrize("n_threads", [1, 2]) +def test_loss_dtype( + loss, readonly_memmap, dtype_in, dtype_out, sample_weight, out1, out2, n_threads +): + """Test acceptance of dtypes, readonly and writeable arrays in loss functions. + + Check that loss accepts if all input arrays are either all float32 or all + float64, and all output arrays are either all float32 or all float64. + + Also check that input arrays can be readonly, e.g. memory mapped. + """ + loss = loss() + # generate a y_true and raw_prediction in valid range + n_samples = 5 + y_true, raw_prediction = random_y_true_raw_prediction( + loss=loss, + n_samples=n_samples, + y_bound=(-100, 100), + raw_bound=(-10, 10), + seed=42, + ) + y_true = y_true.astype(dtype_in) + raw_prediction = raw_prediction.astype(dtype_in) + + if sample_weight is not None: + sample_weight = np.array([2.0] * n_samples, dtype=dtype_in) + if out1 is not None: + out1 = np.empty_like(y_true, dtype=dtype_out) + if out2 is not None: + out2 = np.empty_like(raw_prediction, dtype=dtype_out) + + if readonly_memmap: + y_true = create_memmap_backed_data(y_true) + raw_prediction = create_memmap_backed_data(raw_prediction) + if sample_weight is not None: + sample_weight = create_memmap_backed_data(sample_weight) + + l = loss.loss( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + loss_out=out1, + n_threads=n_threads, + ) + assert l is out1 if out1 is not None else True + g = loss.gradient( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + gradient_out=out2, + n_threads=n_threads, + ) + assert g is out2 if out2 is not None else True + l, g = loss.loss_gradient( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + loss_out=out1, + gradient_out=out2, + n_threads=n_threads, + ) + assert l is out1 if out1 is not None else True + assert g is out2 if out2 is not None else True + if out1 is not None and loss.is_multiclass: + out1 = np.empty_like(raw_prediction, dtype=dtype_out) + g, h = loss.gradient_hessian( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + gradient_out=out1, + hessian_out=out2, + n_threads=n_threads, + ) + assert g is out1 if out1 is not None else True + assert h is out2 if out2 is not None else True + loss(y_true=y_true, raw_prediction=raw_prediction, sample_weight=sample_weight) + loss.fit_intercept_only(y_true=y_true, sample_weight=sample_weight) + loss.constant_to_optimal_zero(y_true=y_true, sample_weight=sample_weight) + if hasattr(loss, "predict_proba"): + loss.predict_proba(raw_prediction=raw_prediction) + if hasattr(loss, "gradient_proba"): + g, p = loss.gradient_proba( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + gradient_out=out1, + proba_out=out2, + n_threads=n_threads, + ) + assert g is out1 if out1 is not None else True + assert p is out2 if out2 is not None else True + + +@pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name) +@pytest.mark.parametrize("sample_weight", [None, "range"]) +def test_loss_same_as_C_functions(loss, sample_weight): + """Test that Python and Cython functions return same results.""" + y_true, raw_prediction = random_y_true_raw_prediction( + loss=loss, + n_samples=20, + y_bound=(-100, 100), + raw_bound=(-10, 10), + seed=42, + ) + if sample_weight == "range": + sample_weight = np.linspace(1, y_true.shape[0], num=y_true.shape[0]) + + out_l1 = np.empty_like(y_true) + out_l2 = np.empty_like(y_true) + out_g1 = np.empty_like(raw_prediction) + out_g2 = np.empty_like(raw_prediction) + out_h1 = np.empty_like(raw_prediction) + out_h2 = np.empty_like(raw_prediction) + loss.loss( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + loss_out=out_l1, + ) + loss.closs.loss( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + loss_out=out_l2, + ) + assert_allclose(out_l1, out_l2) + loss.gradient( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + gradient_out=out_g1, + ) + loss.closs.gradient( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + gradient_out=out_g2, + ) + assert_allclose(out_g1, out_g2) + loss.closs.loss_gradient( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + loss_out=out_l1, + gradient_out=out_g1, + ) + loss.closs.loss_gradient( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + loss_out=out_l2, + gradient_out=out_g2, + ) + assert_allclose(out_l1, out_l2) + assert_allclose(out_g1, out_g2) + loss.gradient_hessian( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + gradient_out=out_g1, + hessian_out=out_h1, + ) + loss.closs.gradient_hessian( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + gradient_out=out_g2, + hessian_out=out_h2, + ) + assert_allclose(out_g1, out_g2) + assert_allclose(out_h1, out_h2) + + +@pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name) +@pytest.mark.parametrize("sample_weight", [None, "range"]) +def test_loss_gradients_are_the_same(loss, sample_weight, global_random_seed): + """Test that loss and gradient are the same across different functions. + + Also test that output arguments contain correct results. + """ + y_true, raw_prediction = random_y_true_raw_prediction( + loss=loss, + n_samples=20, + y_bound=(-100, 100), + raw_bound=(-10, 10), + seed=global_random_seed, + ) + if sample_weight == "range": + sample_weight = np.linspace(1, y_true.shape[0], num=y_true.shape[0]) + + out_l1 = np.empty_like(y_true) + out_l2 = np.empty_like(y_true) + out_g1 = np.empty_like(raw_prediction) + out_g2 = np.empty_like(raw_prediction) + out_g3 = np.empty_like(raw_prediction) + out_h3 = np.empty_like(raw_prediction) + + l1 = loss.loss( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + loss_out=out_l1, + ) + g1 = loss.gradient( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + gradient_out=out_g1, + ) + l2, g2 = loss.loss_gradient( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + loss_out=out_l2, + gradient_out=out_g2, + ) + g3, h3 = loss.gradient_hessian( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + gradient_out=out_g3, + hessian_out=out_h3, + ) + assert_allclose(l1, l2) + assert_array_equal(l1, out_l1) + assert np.shares_memory(l1, out_l1) + assert_array_equal(l2, out_l2) + assert np.shares_memory(l2, out_l2) + assert_allclose(g1, g2) + assert_allclose(g1, g3) + assert_array_equal(g1, out_g1) + assert np.shares_memory(g1, out_g1) + assert_array_equal(g2, out_g2) + assert np.shares_memory(g2, out_g2) + assert_array_equal(g3, out_g3) + assert np.shares_memory(g3, out_g3) + + if hasattr(loss, "gradient_proba"): + assert loss.is_multiclass # only for HalfMultinomialLoss + out_g4 = np.empty_like(raw_prediction) + out_proba = np.empty_like(raw_prediction) + g4, proba = loss.gradient_proba( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + gradient_out=out_g4, + proba_out=out_proba, + ) + assert_allclose(g1, out_g4) + assert_allclose(g1, g4) + assert_allclose(proba, out_proba) + assert_allclose(np.sum(proba, axis=1), 1, rtol=1e-11) + + +@pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name) +@pytest.mark.parametrize("sample_weight", ["ones", "random"]) +def test_sample_weight_multiplies(loss, sample_weight, global_random_seed): + """Test sample weights in loss, gradients and hessians. + + Make sure that passing sample weights to loss, gradient and hessian + computation methods is equivalent to multiplying by the weights. + """ + n_samples = 100 + y_true, raw_prediction = random_y_true_raw_prediction( + loss=loss, + n_samples=n_samples, + y_bound=(-100, 100), + raw_bound=(-5, 5), + seed=global_random_seed, + ) + + if sample_weight == "ones": + sample_weight = np.ones(shape=n_samples, dtype=np.float64) + else: + rng = np.random.RandomState(global_random_seed) + sample_weight = rng.normal(size=n_samples).astype(np.float64) + + assert_allclose( + loss.loss( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + ), + sample_weight + * loss.loss( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=None, + ), + ) + + losses, gradient = loss.loss_gradient( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=None, + ) + losses_sw, gradient_sw = loss.loss_gradient( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + ) + assert_allclose(losses * sample_weight, losses_sw) + if not loss.is_multiclass: + assert_allclose(gradient * sample_weight, gradient_sw) + else: + assert_allclose(gradient * sample_weight[:, None], gradient_sw) + + gradient, hessian = loss.gradient_hessian( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=None, + ) + gradient_sw, hessian_sw = loss.gradient_hessian( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + ) + if not loss.is_multiclass: + assert_allclose(gradient * sample_weight, gradient_sw) + assert_allclose(hessian * sample_weight, hessian_sw) + else: + assert_allclose(gradient * sample_weight[:, None], gradient_sw) + assert_allclose(hessian * sample_weight[:, None], hessian_sw) + + +@pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name) +def test_graceful_squeezing(loss): + """Test that reshaped raw_prediction gives same results.""" + y_true, raw_prediction = random_y_true_raw_prediction( + loss=loss, + n_samples=20, + y_bound=(-100, 100), + raw_bound=(-10, 10), + seed=42, + ) + + if raw_prediction.ndim == 1: + raw_prediction_2d = raw_prediction[:, None] + assert_allclose( + loss.loss(y_true=y_true, raw_prediction=raw_prediction_2d), + loss.loss(y_true=y_true, raw_prediction=raw_prediction), + ) + assert_allclose( + loss.loss_gradient(y_true=y_true, raw_prediction=raw_prediction_2d), + loss.loss_gradient(y_true=y_true, raw_prediction=raw_prediction), + ) + assert_allclose( + loss.gradient(y_true=y_true, raw_prediction=raw_prediction_2d), + loss.gradient(y_true=y_true, raw_prediction=raw_prediction), + ) + assert_allclose( + loss.gradient_hessian(y_true=y_true, raw_prediction=raw_prediction_2d), + loss.gradient_hessian(y_true=y_true, raw_prediction=raw_prediction), + ) + + +@pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name) +@pytest.mark.parametrize("sample_weight", [None, "range"]) +def test_loss_of_perfect_prediction(loss, sample_weight): + """Test value of perfect predictions. + + Loss of y_pred = y_true plus constant_to_optimal_zero should sums up to + zero. + """ + if not loss.is_multiclass: + # Use small values such that exp(value) is not nan. + raw_prediction = np.array([-10, -0.1, 0, 0.1, 3, 10]) + # If link is identity, we must respect the interval of y_pred: + if isinstance(loss.link, IdentityLink): + eps = 1e-10 + low = loss.interval_y_pred.low + if not loss.interval_y_pred.low_inclusive: + low = low + eps + high = loss.interval_y_pred.high + if not loss.interval_y_pred.high_inclusive: + high = high - eps + raw_prediction = np.clip(raw_prediction, low, high) + y_true = loss.link.inverse(raw_prediction) + else: + # HalfMultinomialLoss + y_true = np.arange(loss.n_classes).astype(float) + # raw_prediction with entries -exp(10), but +exp(10) on the diagonal + # this is close enough to np.inf which would produce nan + raw_prediction = np.full( + shape=(loss.n_classes, loss.n_classes), + fill_value=-np.exp(10), + dtype=float, + ) + raw_prediction.flat[:: loss.n_classes + 1] = np.exp(10) + + if sample_weight == "range": + sample_weight = np.linspace(1, y_true.shape[0], num=y_true.shape[0]) + + loss_value = loss.loss( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + ) + constant_term = loss.constant_to_optimal_zero( + y_true=y_true, sample_weight=sample_weight + ) + # Comparing loss_value + constant_term to zero would result in large + # round-off errors. + assert_allclose(loss_value, -constant_term, atol=1e-14, rtol=1e-15) + + +@pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name) +@pytest.mark.parametrize("sample_weight", [None, "range"]) +def test_gradients_hessians_numerically(loss, sample_weight, global_random_seed): + """Test gradients and hessians with numerical derivatives. + + Gradient should equal the numerical derivatives of the loss function. + Hessians should equal the numerical derivatives of gradients. + """ + n_samples = 20 + y_true, raw_prediction = random_y_true_raw_prediction( + loss=loss, + n_samples=n_samples, + y_bound=(-100, 100), + raw_bound=(-5, 5), + seed=global_random_seed, + ) + + if sample_weight == "range": + sample_weight = np.linspace(1, y_true.shape[0], num=y_true.shape[0]) + + g, h = loss.gradient_hessian( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + ) + + assert g.shape == raw_prediction.shape + assert h.shape == raw_prediction.shape + + if not loss.is_multiclass: + + def loss_func(x): + return loss.loss( + y_true=y_true, + raw_prediction=x, + sample_weight=sample_weight, + ) + + g_numeric = numerical_derivative(loss_func, raw_prediction, eps=1e-6) + assert_allclose(g, g_numeric, rtol=5e-6, atol=1e-10) + + def grad_func(x): + return loss.gradient( + y_true=y_true, + raw_prediction=x, + sample_weight=sample_weight, + ) + + h_numeric = numerical_derivative(grad_func, raw_prediction, eps=1e-6) + if loss.approx_hessian: + # TODO: What could we test if loss.approx_hessian? + pass + else: + assert_allclose(h, h_numeric, rtol=5e-6, atol=1e-10) + else: + # For multiclass loss, we should only change the predictions of the + # class for which the derivative is taken for, e.g. offset[:, k] = eps + # for class k. + # As a softmax is computed, offsetting the whole array by a constant + # would have no effect on the probabilities, and thus on the loss. + for k in range(loss.n_classes): + + def loss_func(x): + raw = raw_prediction.copy() + raw[:, k] = x + return loss.loss( + y_true=y_true, + raw_prediction=raw, + sample_weight=sample_weight, + ) + + g_numeric = numerical_derivative(loss_func, raw_prediction[:, k], eps=1e-5) + assert_allclose(g[:, k], g_numeric, rtol=5e-6, atol=1e-10) + + def grad_func(x): + raw = raw_prediction.copy() + raw[:, k] = x + return loss.gradient( + y_true=y_true, + raw_prediction=raw, + sample_weight=sample_weight, + )[:, k] + + h_numeric = numerical_derivative(grad_func, raw_prediction[:, k], eps=1e-6) + if loss.approx_hessian: + # TODO: What could we test if loss.approx_hessian? + pass + else: + assert_allclose(h[:, k], h_numeric, rtol=5e-6, atol=1e-10) + + +@pytest.mark.parametrize( + "loss, x0, y_true", + [ + ("squared_error", -2.0, 42), + ("squared_error", 117.0, 1.05), + ("squared_error", 0.0, 0.0), + # The argmin of binomial_loss for y_true=0 and y_true=1 is resp. + # -inf and +inf due to logit, cf. "complete separation". Therefore, we + # use 0 < y_true < 1. + ("binomial_loss", 0.3, 0.1), + ("binomial_loss", -12, 0.2), + ("binomial_loss", 30, 0.9), + ("poisson_loss", 12.0, 1.0), + ("poisson_loss", 0.0, 2.0), + ("poisson_loss", -22.0, 10.0), + ], +) +@skip_if_32bit +def test_derivatives(loss, x0, y_true): + """Test that gradients are zero at the minimum of the loss. + + We check this on a single value/sample using Halley's method with the + first and second order derivatives computed by the Loss instance. + Note that methods of Loss instances operate on arrays while the newton + root finder expects a scalar or a one-element array for this purpose. + """ + loss = _LOSSES[loss](sample_weight=None) + y_true = np.array([y_true], dtype=np.float64) + x0 = np.array([x0], dtype=np.float64) + + def func(x: np.ndarray) -> np.ndarray: + """Compute loss plus constant term. + + The constant term is such that the minimum function value is zero, + which is required by the Newton method. + """ + return loss.loss( + y_true=y_true, raw_prediction=x + ) + loss.constant_to_optimal_zero(y_true=y_true) + + def fprime(x: np.ndarray) -> np.ndarray: + return loss.gradient(y_true=y_true, raw_prediction=x) + + def fprime2(x: np.ndarray) -> np.ndarray: + return loss.gradient_hessian(y_true=y_true, raw_prediction=x)[1] + + optimum = newton( + func, + x0=x0, + fprime=fprime, + fprime2=fprime2, + maxiter=100, + tol=5e-8, + ) + + # Need to ravel arrays because assert_allclose requires matching + # dimensions. + y_true = y_true.ravel() + optimum = optimum.ravel() + assert_allclose(loss.link.inverse(optimum), y_true) + assert_allclose(func(optimum), 0, atol=1e-14) + assert_allclose(loss.gradient(y_true=y_true, raw_prediction=optimum), 0, atol=5e-7) + + +@pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name) +@pytest.mark.parametrize("sample_weight", [None, "range"]) +def test_loss_intercept_only(loss, sample_weight): + """Test that fit_intercept_only returns the argmin of the loss. + + Also test that the gradient is zero at the minimum. + """ + n_samples = 50 + if not loss.is_multiclass: + y_true = loss.link.inverse(np.linspace(-4, 4, num=n_samples)) + else: + y_true = np.arange(n_samples).astype(np.float64) % loss.n_classes + y_true[::5] = 0 # exceedance of class 0 + + if sample_weight == "range": + sample_weight = np.linspace(0.1, 2, num=n_samples) + + a = loss.fit_intercept_only(y_true=y_true, sample_weight=sample_weight) + + # find minimum by optimization + def fun(x): + if not loss.is_multiclass: + raw_prediction = np.full(shape=(n_samples), fill_value=x) + else: + raw_prediction = np.ascontiguousarray( + np.broadcast_to(x, shape=(n_samples, loss.n_classes)) + ) + return loss( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + ) + + if not loss.is_multiclass: + opt = minimize_scalar(fun, tol=1e-7, options={"maxiter": 100}) + grad = loss.gradient( + y_true=y_true, + raw_prediction=np.full_like(y_true, a), + sample_weight=sample_weight, + ) + assert a.shape == tuple() # scalar + assert a.dtype == y_true.dtype + assert_all_finite(a) + a == approx(opt.x, rel=1e-7) + grad.sum() == approx(0, abs=1e-12) + else: + # The constraint corresponds to sum(raw_prediction) = 0. Without it, we would + # need to apply loss.symmetrize_raw_prediction to opt.x before comparing. + opt = minimize( + fun, + np.zeros((loss.n_classes)), + tol=1e-13, + options={"maxiter": 100}, + method="SLSQP", + constraints=LinearConstraint(np.ones((1, loss.n_classes)), 0, 0), + ) + grad = loss.gradient( + y_true=y_true, + raw_prediction=np.tile(a, (n_samples, 1)), + sample_weight=sample_weight, + ) + assert a.dtype == y_true.dtype + assert_all_finite(a) + assert_allclose(a, opt.x, rtol=5e-6, atol=1e-12) + assert_allclose(grad.sum(axis=0), 0, atol=1e-12) + + +@pytest.mark.parametrize( + "loss, func, random_dist", + [ + (HalfSquaredError(), np.mean, "normal"), + (AbsoluteError(), np.median, "normal"), + (PinballLoss(quantile=0.25), lambda x: np.percentile(x, q=25), "normal"), + (HalfPoissonLoss(), np.mean, "poisson"), + (HalfGammaLoss(), np.mean, "exponential"), + (HalfTweedieLoss(), np.mean, "exponential"), + (HalfBinomialLoss(), np.mean, "binomial"), + ], +) +def test_specific_fit_intercept_only(loss, func, random_dist, global_random_seed): + """Test that fit_intercept_only returns the correct functional. + + We test the functional for specific, meaningful distributions, e.g. + squared error estimates the expectation of a probability distribution. + """ + rng = np.random.RandomState(global_random_seed) + if random_dist == "binomial": + y_train = rng.binomial(1, 0.5, size=100) + else: + y_train = getattr(rng, random_dist)(size=100) + baseline_prediction = loss.fit_intercept_only(y_true=y_train) + # Make sure baseline prediction is the expected functional=func, e.g. mean + # or median. + assert_all_finite(baseline_prediction) + assert baseline_prediction == approx(loss.link.link(func(y_train))) + assert loss.link.inverse(baseline_prediction) == approx(func(y_train)) + if isinstance(loss, IdentityLink): + assert_allclose(loss.link.inverse(baseline_prediction), baseline_prediction) + + # Test baseline at boundary + if loss.interval_y_true.low_inclusive: + y_train.fill(loss.interval_y_true.low) + baseline_prediction = loss.fit_intercept_only(y_true=y_train) + assert_all_finite(baseline_prediction) + if loss.interval_y_true.high_inclusive: + y_train.fill(loss.interval_y_true.high) + baseline_prediction = loss.fit_intercept_only(y_true=y_train) + assert_all_finite(baseline_prediction) + + +def test_multinomial_loss_fit_intercept_only(): + """Test that fit_intercept_only returns the mean functional for CCE.""" + rng = np.random.RandomState(0) + n_classes = 4 + loss = HalfMultinomialLoss(n_classes=n_classes) + # Same logic as test_specific_fit_intercept_only. Here inverse link + # function = softmax and link function = log - symmetry term. + y_train = rng.randint(0, n_classes + 1, size=100).astype(np.float64) + baseline_prediction = loss.fit_intercept_only(y_true=y_train) + assert baseline_prediction.shape == (n_classes,) + p = np.zeros(n_classes, dtype=y_train.dtype) + for k in range(n_classes): + p[k] = (y_train == k).mean() + assert_allclose(baseline_prediction, np.log(p) - np.mean(np.log(p))) + assert_allclose(baseline_prediction[None, :], loss.link.link(p[None, :])) + + for y_train in (np.zeros(shape=10), np.ones(shape=10)): + y_train = y_train.astype(np.float64) + baseline_prediction = loss.fit_intercept_only(y_true=y_train) + assert baseline_prediction.dtype == y_train.dtype + assert_all_finite(baseline_prediction) + + +def test_multinomial_cy_gradient(global_random_seed): + """Test that Multinomial cy_gradient gives the same result as gradient. + + CyHalfMultinomialLoss does not inherit from CyLossFunction and has a different API. + As a consequence, the functions like `loss` and `gradient` do not rely on `cy_loss` + and `cy_gradient`. + """ + n_samples = 100 + n_classes = 5 + loss = HalfMultinomialLoss(n_classes=n_classes) + y_true, raw_prediction = random_y_true_raw_prediction( + loss=loss, + n_samples=n_samples, + seed=global_random_seed, + ) + sample_weight = np.linspace(0.1, 2, num=n_samples) + + grad1 = loss.closs._test_cy_gradient( + y_true=y_true, + raw_prediction=raw_prediction, # needs to be C-contiguous + sample_weight=sample_weight, + ) + grad2 = loss.gradient( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + ) + assert_allclose(grad1, grad2) + + +def test_binomial_and_multinomial_loss(global_random_seed): + """Test that multinomial loss with n_classes = 2 is the same as binomial loss.""" + rng = np.random.RandomState(global_random_seed) + n_samples = 20 + binom = HalfBinomialLoss() + multinom = HalfMultinomialLoss(n_classes=2) + y_train = rng.randint(0, 2, size=n_samples).astype(np.float64) + raw_prediction = rng.normal(size=n_samples) + raw_multinom = np.empty((n_samples, 2)) + raw_multinom[:, 0] = -0.5 * raw_prediction + raw_multinom[:, 1] = 0.5 * raw_prediction + assert_allclose( + binom.loss(y_true=y_train, raw_prediction=raw_prediction), + multinom.loss(y_true=y_train, raw_prediction=raw_multinom), + ) + + +@pytest.mark.parametrize("y_true", (np.array([0.0, 0, 0]), np.array([1.0, 1, 1]))) +@pytest.mark.parametrize("y_pred", (np.array([-5.0, -5, -5]), np.array([3.0, 3, 3]))) +def test_binomial_vs_alternative_formulation(y_true, y_pred, global_dtype): + """Test that both formulations of the binomial deviance agree. + + Often, the binomial deviance or log loss is written in terms of a variable + z in {-1, +1}, but we use y in {0, 1}, hence z = 2 * y - 1. + ESL II Eq. (10.18): + + -loglike(z, f) = log(1 + exp(-2 * z * f)) + + Note: + - ESL 2*f = raw_prediction, hence the factor 2 of ESL disappears. + - Deviance = -2*loglike + .., but HalfBinomialLoss is half of the + deviance, hence the factor of 2 cancels in the comparison. + """ + + def alt_loss(y, raw_pred): + z = 2 * y - 1 + return np.mean(np.log(1 + np.exp(-z * raw_pred))) + + def alt_gradient(y, raw_pred): + # alternative gradient formula according to ESL + z = 2 * y - 1 + return -z / (1 + np.exp(z * raw_pred)) + + bin_loss = HalfBinomialLoss() + + y_true = y_true.astype(global_dtype) + y_pred = y_pred.astype(global_dtype) + datum = (y_true, y_pred) + + assert bin_loss(*datum) == approx(alt_loss(*datum)) + assert_allclose(bin_loss.gradient(*datum), alt_gradient(*datum)) + + +@pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name) +def test_predict_proba(loss, global_random_seed): + """Test that predict_proba and gradient_proba work as expected.""" + n_samples = 20 + y_true, raw_prediction = random_y_true_raw_prediction( + loss=loss, + n_samples=n_samples, + y_bound=(-100, 100), + raw_bound=(-5, 5), + seed=global_random_seed, + ) + + if hasattr(loss, "predict_proba"): + proba = loss.predict_proba(raw_prediction) + assert proba.shape == (n_samples, loss.n_classes) + assert np.sum(proba, axis=1) == approx(1, rel=1e-11) + + if hasattr(loss, "gradient_proba"): + for grad, proba in ( + (None, None), + (None, np.empty_like(raw_prediction)), + (np.empty_like(raw_prediction), None), + (np.empty_like(raw_prediction), np.empty_like(raw_prediction)), + ): + grad, proba = loss.gradient_proba( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=None, + gradient_out=grad, + proba_out=proba, + ) + assert proba.shape == (n_samples, loss.n_classes) + assert np.sum(proba, axis=1) == approx(1, rel=1e-11) + assert_allclose( + grad, + loss.gradient( + y_true=y_true, + raw_prediction=raw_prediction, + sample_weight=None, + gradient_out=None, + ), + ) + + +@pytest.mark.parametrize("loss", ALL_LOSSES) +@pytest.mark.parametrize("sample_weight", [None, "range"]) +@pytest.mark.parametrize("dtype", (np.float32, np.float64)) +@pytest.mark.parametrize("order", ("C", "F")) +def test_init_gradient_and_hessians(loss, sample_weight, dtype, order): + """Test that init_gradient_and_hessian works as expected. + + passing sample_weight to a loss correctly influences the constant_hessian + attribute, and consequently the shape of the hessian array. + """ + n_samples = 5 + if sample_weight == "range": + sample_weight = np.ones(n_samples) + loss = loss(sample_weight=sample_weight) + gradient, hessian = loss.init_gradient_and_hessian( + n_samples=n_samples, + dtype=dtype, + order=order, + ) + if loss.constant_hessian: + assert gradient.shape == (n_samples,) + assert hessian.shape == (1,) + elif loss.is_multiclass: + assert gradient.shape == (n_samples, loss.n_classes) + assert hessian.shape == (n_samples, loss.n_classes) + else: + assert hessian.shape == (n_samples,) + assert hessian.shape == (n_samples,) + + assert gradient.dtype == dtype + assert hessian.dtype == dtype + + if order == "C": + assert gradient.flags.c_contiguous + assert hessian.flags.c_contiguous + else: + assert gradient.flags.f_contiguous + assert hessian.flags.f_contiguous + + +@pytest.mark.parametrize("loss", ALL_LOSSES) +@pytest.mark.parametrize( + "params, err_msg", + [ + ( + {"dtype": np.int64}, + f"Valid options for 'dtype' are .* Got dtype={np.int64} instead.", + ), + ], +) +def test_init_gradient_and_hessian_raises(loss, params, err_msg): + """Test that init_gradient_and_hessian raises errors for invalid input.""" + loss = loss() + with pytest.raises((ValueError, TypeError), match=err_msg): + gradient, hessian = loss.init_gradient_and_hessian(n_samples=5, **params) + + +@pytest.mark.parametrize( + "loss, params, err_type, err_msg", + [ + ( + PinballLoss, + {"quantile": None}, + TypeError, + "quantile must be an instance of float, not NoneType.", + ), + ( + PinballLoss, + {"quantile": 0}, + ValueError, + "quantile == 0, must be > 0.", + ), + (PinballLoss, {"quantile": 1.1}, ValueError, "quantile == 1.1, must be < 1."), + ( + HuberLoss, + {"quantile": None}, + TypeError, + "quantile must be an instance of float, not NoneType.", + ), + ( + HuberLoss, + {"quantile": 0}, + ValueError, + "quantile == 0, must be > 0.", + ), + (HuberLoss, {"quantile": 1.1}, ValueError, "quantile == 1.1, must be < 1."), + ], +) +def test_loss_init_parameter_validation(loss, params, err_type, err_msg): + """Test that loss raises errors for invalid input.""" + with pytest.raises(err_type, match=err_msg): + loss(**params) + + +@pytest.mark.parametrize("loss", LOSS_INSTANCES, ids=loss_instance_name) +def test_loss_pickle(loss): + """Test that losses can be pickled.""" + n_samples = 20 + y_true, raw_prediction = random_y_true_raw_prediction( + loss=loss, + n_samples=n_samples, + y_bound=(-100, 100), + raw_bound=(-5, 5), + seed=42, + ) + pickled_loss = pickle.dumps(loss) + unpickled_loss = pickle.loads(pickled_loss) + assert loss(y_true=y_true, raw_prediction=raw_prediction) == approx( + unpickled_loss(y_true=y_true, raw_prediction=raw_prediction) + ) + + +@pytest.mark.parametrize("p", [-1.5, 0, 1, 1.5, 2, 3]) +def test_tweedie_log_identity_consistency(p): + """Test for identical losses when only the link function is different.""" + half_tweedie_log = HalfTweedieLoss(power=p) + half_tweedie_identity = HalfTweedieLossIdentity(power=p) + n_samples = 10 + y_true, raw_prediction = random_y_true_raw_prediction( + loss=half_tweedie_log, n_samples=n_samples, seed=42 + ) + y_pred = half_tweedie_log.link.inverse(raw_prediction) # exp(raw_prediction) + + # Let's compare the loss values, up to some constant term that is dropped + # in HalfTweedieLoss but not in HalfTweedieLossIdentity. + loss_log = half_tweedie_log.loss( + y_true=y_true, raw_prediction=raw_prediction + ) + half_tweedie_log.constant_to_optimal_zero(y_true) + loss_identity = half_tweedie_identity.loss( + y_true=y_true, raw_prediction=y_pred + ) + half_tweedie_identity.constant_to_optimal_zero(y_true) + # Note that HalfTweedieLoss ignores different constant terms than + # HalfTweedieLossIdentity. Constant terms means terms not depending on + # raw_prediction. By adding these terms, `constant_to_optimal_zero`, both losses + # give the same values. + assert_allclose(loss_log, loss_identity) + + # For gradients and hessians, the constant terms do not matter. We have, however, + # to account for the chain rule, i.e. with x=raw_prediction + # gradient_log(x) = d/dx loss_log(x) + # = d/dx loss_identity(exp(x)) + # = exp(x) * gradient_identity(exp(x)) + # Similarly, + # hessian_log(x) = exp(x) * gradient_identity(exp(x)) + # + exp(x)**2 * hessian_identity(x) + gradient_log, hessian_log = half_tweedie_log.gradient_hessian( + y_true=y_true, raw_prediction=raw_prediction + ) + gradient_identity, hessian_identity = half_tweedie_identity.gradient_hessian( + y_true=y_true, raw_prediction=y_pred + ) + assert_allclose(gradient_log, y_pred * gradient_identity) + assert_allclose( + hessian_log, y_pred * gradient_identity + y_pred**2 * hessian_identity + ) diff --git a/sklearn/_min_dependencies.py b/sklearn/_min_dependencies.py index 56d44586cdc6d..8d39075630437 100644 --- a/sklearn/_min_dependencies.py +++ b/sklearn/_min_dependencies.py @@ -1,65 +1,74 @@ """All minimum dependencies for scikit-learn.""" -import platform -import argparse +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -# numpy scipy and cython should by in sync with pyproject.toml -if platform.python_implementation() == 'PyPy': - SCIPY_MIN_VERSION = '1.1.0' - NUMPY_MIN_VERSION = '1.19.0' -else: - SCIPY_MIN_VERSION = '0.19.1' - NUMPY_MIN_VERSION = '1.13.3' +import argparse +from collections import defaultdict -JOBLIB_MIN_VERSION = '0.11' -THREADPOOLCTL_MIN_VERSION = '2.0.0' -PYTEST_MIN_VERSION = '5.0.1' -CYTHON_MIN_VERSION = '0.28.5' +# scipy and cython should by in sync with pyproject.toml +NUMPY_MIN_VERSION = "1.22.0" +SCIPY_MIN_VERSION = "1.8.0" +JOBLIB_MIN_VERSION = "1.2.0" +THREADPOOLCTL_MIN_VERSION = "3.1.0" +PYTEST_MIN_VERSION = "7.1.2" +CYTHON_MIN_VERSION = "3.0.10" # 'build' and 'install' is included to have structured metadata for CI. # It will NOT be included in setup's extras_require -# The values are (version_spec, comma seperated tags) +# The values are (version_spec, comma separated tags) dependent_packages = { - 'numpy': (NUMPY_MIN_VERSION, 'build, install'), - 'scipy': (SCIPY_MIN_VERSION, 'build, install'), - 'joblib': (JOBLIB_MIN_VERSION, 'install'), - 'threadpoolctl': (THREADPOOLCTL_MIN_VERSION, 'install'), - 'cython': (CYTHON_MIN_VERSION, 'build'), - 'matplotlib': ('2.1.1', 'benchmark, docs, examples, tests'), - 'scikit-image': ('0.13', 'docs, examples, tests'), - 'pandas': ('0.25.0', 'benchmark, docs, examples, tests'), - 'seaborn': ('0.9.0', 'docs, examples'), - 'memory_profiler': ('0.57.0', 'benchmark, docs'), - 'pytest': (PYTEST_MIN_VERSION, 'tests'), - 'pytest-cov': ('2.9.0', 'tests'), - 'flake8': ('3.8.2', 'tests'), - 'mypy': ('0.770', 'tests'), - 'pyamg': ('4.0.0', 'tests'), - 'sphinx': ('3.2.0', 'docs'), - 'sphinx-gallery': ('0.7.0', 'docs'), - 'numpydoc': ('1.0.0', 'docs'), - 'Pillow': ('7.1.2', 'docs'), - 'sphinx-prompt': ('1.3.0', 'docs'), + "numpy": (NUMPY_MIN_VERSION, "build, install"), + "scipy": (SCIPY_MIN_VERSION, "build, install"), + "joblib": (JOBLIB_MIN_VERSION, "install"), + "threadpoolctl": (THREADPOOLCTL_MIN_VERSION, "install"), + "cython": (CYTHON_MIN_VERSION, "build"), + "meson-python": ("0.16.0", "build"), + "matplotlib": ("3.5.0", "benchmark, docs, examples, tests"), + "scikit-image": ("0.19.0", "docs, examples, tests"), + "pandas": ("1.4.0", "benchmark, docs, examples, tests"), + "seaborn": ("0.9.0", "docs, examples"), + "memory_profiler": ("0.57.0", "benchmark, docs"), + "pytest": (PYTEST_MIN_VERSION, "tests"), + "pytest-cov": ("2.9.0", "tests"), + "ruff": ("0.11.7", "tests"), + "mypy": ("1.15", "tests"), + "pyamg": ("4.2.1", "tests"), + "polars": ("0.20.30", "docs, tests"), + "pyarrow": ("12.0.0", "tests"), + "sphinx": ("7.3.7", "docs"), + "sphinx-copybutton": ("0.5.2", "docs"), + "sphinx-gallery": ("0.17.1", "docs"), + "numpydoc": ("1.2.0", "docs, tests"), + "Pillow": ("8.4.0", "docs"), + "pooch": ("1.6.0", "docs, examples, tests"), + "sphinx-prompt": ("1.4.0", "docs"), + "sphinxext-opengraph": ("0.9.1", "docs"), + "plotly": ("5.14.0", "docs, examples"), + "sphinxcontrib-sass": ("0.3.4", "docs"), + "sphinx-remove-toctrees": ("1.0.0.post1", "docs"), + "sphinx-design": ("0.6.0", "docs"), + "pydata-sphinx-theme": ("0.15.3", "docs"), + "towncrier": ("24.8.0", "docs"), + # XXX: Pin conda-lock to the latest released version (needs manual update + # from time to time) + "conda-lock": ("3.0.1", "maintenance"), } # create inverse mapping for setuptools -tag_to_packages: dict = { - extra: [] for extra in ['build', 'install', 'docs', 'examples', - 'tests', 'benchmark'] -} +tag_to_packages: dict = defaultdict(list) for package, (min_version, extras) in dependent_packages.items(): - for extra in extras.split(', '): + for extra in extras.split(", "): tag_to_packages[extra].append("{}>={}".format(package, min_version)) # Used by CI to get the min dependencies -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description='Get min dependencies for a package') +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Get min dependencies for a package") - parser.add_argument('package', choices=dependent_packages) + parser.add_argument("package", choices=dependent_packages) args = parser.parse_args() min_version = dependent_packages[args.package][0] print(min_version) diff --git a/sklearn/base.py b/sklearn/base.py index 3626e931aa9cf..309b482357e12 100644 --- a/sklearn/base.py +++ b/sklearn/base.py @@ -1,80 +1,137 @@ -"""Base classes for all estimators.""" +"""Base classes for all estimators and various utility functions.""" -# Author: Gael Varoquaux -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import copy -import warnings -from collections import defaultdict -import platform +import functools import inspect +import platform import re +import warnings +from collections import defaultdict import numpy as np from . import __version__ -from ._config import get_config -from .utils import _IS_32BIT +from ._config import config_context, get_config +from .exceptions import InconsistentVersionWarning +from .utils._metadata_requests import _MetadataRequester, _routing_enabled +from .utils._missing import is_scalar_nan +from .utils._param_validation import validate_parameter_constraints +from .utils._repr_html.base import ReprHTMLMixin, _HTMLDocumentationLinkMixin +from .utils._repr_html.estimator import estimator_html_repr +from .utils._repr_html.params import ParamsDict +from .utils._set_output import _SetOutputMixin from .utils._tags import ( - _DEFAULT_TAGS, - _safe_tags, + ClassifierTags, + RegressorTags, + Tags, + TargetTags, + TransformerTags, + get_tags, +) +from .utils.fixes import _IS_32BIT +from .utils.validation import ( + _check_feature_names_in, + _generate_get_feature_names_out, + _is_fitted, + check_array, + check_is_fitted, ) -from .utils.validation import check_X_y -from .utils.validation import check_array -from .utils._estimator_html_repr import estimator_html_repr -from .utils.validation import _deprecate_positional_args -@_deprecate_positional_args def clone(estimator, *, safe=True): - """Constructs a new unfitted estimator with the same parameters. + """Construct a new unfitted estimator with the same parameters. Clone does a deep copy of the model in an estimator - without actually copying attached data. It yields a new estimator + without actually copying attached data. It returns a new estimator with the same parameters that has not been fitted on any data. - If the estimator's `random_state` parameter is an integer (or if the - estimator doesn't have a `random_state` parameter), an *exact clone* is - returned: the clone and the original estimator will give the exact same - results. Otherwise, *statistical clone* is returned: the clone might - yield different results from the original estimator. More details can be - found in :ref:`randomness`. + .. versionchanged:: 1.3 + Delegates to `estimator.__sklearn_clone__` if the method exists. Parameters ---------- estimator : {list, tuple, set} of estimator instance or a single \ estimator instance The estimator or group of estimators to be cloned. - safe : bool, default=True If safe is False, clone will fall back to a deep copy on objects - that are not estimators. + that are not estimators. Ignored if `estimator.__sklearn_clone__` + exists. + + Returns + ------- + estimator : object + The deep copy of the input, an estimator if input is an estimator. + + Notes + ----- + If the estimator's `random_state` parameter is an integer (or if the + estimator doesn't have a `random_state` parameter), an *exact clone* is + returned: the clone and the original estimator will give the exact same + results. Otherwise, *statistical clone* is returned: the clone might + return different results from the original estimator. More details can be + found in :ref:`randomness`. + Examples + -------- + >>> from sklearn.base import clone + >>> from sklearn.linear_model import LogisticRegression + >>> X = [[-1, 0], [0, 1], [0, -1], [1, 0]] + >>> y = [0, 0, 1, 1] + >>> classifier = LogisticRegression().fit(X, y) + >>> cloned_classifier = clone(classifier) + >>> hasattr(classifier, "classes_") + True + >>> hasattr(cloned_classifier, "classes_") + False + >>> classifier is cloned_classifier + False """ + if hasattr(estimator, "__sklearn_clone__") and not inspect.isclass(estimator): + return estimator.__sklearn_clone__() + return _clone_parametrized(estimator, safe=safe) + + +def _clone_parametrized(estimator, *, safe=True): + """Default implementation of clone. See :func:`sklearn.base.clone` for details.""" + estimator_type = type(estimator) - # XXX: not handling dictionaries - if estimator_type in (list, tuple, set, frozenset): + if estimator_type is dict: + return {k: clone(v, safe=safe) for k, v in estimator.items()} + elif estimator_type in (list, tuple, set, frozenset): return estimator_type([clone(e, safe=safe) for e in estimator]) - elif not hasattr(estimator, 'get_params') or isinstance(estimator, type): + elif not hasattr(estimator, "get_params") or isinstance(estimator, type): if not safe: return copy.deepcopy(estimator) else: if isinstance(estimator, type): - raise TypeError("Cannot clone object. " + - "You should provide an instance of " + - "scikit-learn estimator instead of a class.") + raise TypeError( + "Cannot clone object. " + "You should provide an instance of " + "scikit-learn estimator instead of a class." + ) else: - raise TypeError("Cannot clone object '%s' (type %s): " - "it does not seem to be a scikit-learn " - "estimator as it does not implement a " - "'get_params' method." - % (repr(estimator), type(estimator))) + raise TypeError( + "Cannot clone object '%s' (type %s): " + "it does not seem to be a scikit-learn " + "estimator as it does not implement a " + "'get_params' method." % (repr(estimator), type(estimator)) + ) klass = estimator.__class__ new_object_params = estimator.get_params(deep=False) for name, param in new_object_params.items(): new_object_params[name] = clone(param, safe=False) + new_object = klass(**new_object_params) + try: + new_object._metadata_request = copy.deepcopy(estimator._metadata_request) + except AttributeError: + pass + params_set = new_object.get_params(deep=False) # quick sanity check of the parameters of the clone @@ -82,78 +139,72 @@ def clone(estimator, *, safe=True): param1 = new_object_params[name] param2 = params_set[name] if param1 is not param2: - raise RuntimeError('Cannot clone object %s, as the constructor ' - 'either does not set or modifies parameter %s' % - (estimator, name)) - return new_object - - -def _pprint(params, offset=0, printer=repr): - """Pretty print the dictionary 'params' + raise RuntimeError( + "Cannot clone object %s, as the constructor " + "either does not set or modifies parameter %s" % (estimator, name) + ) - Parameters - ---------- - params : dict - The dictionary to pretty print + # _sklearn_output_config is used by `set_output` to configure the output + # container of an estimator. + if hasattr(estimator, "_sklearn_output_config"): + new_object._sklearn_output_config = copy.deepcopy( + estimator._sklearn_output_config + ) + return new_object - offset : int, default=0 - The offset in characters to add at the begin of each line. - printer : callable, default=repr - The function to convert entries to strings, typically - the builtin str or repr +class BaseEstimator(ReprHTMLMixin, _HTMLDocumentationLinkMixin, _MetadataRequester): + """Base class for all estimators in scikit-learn. - """ - # Do a multi-line justified repr: - options = np.get_printoptions() - np.set_printoptions(precision=5, threshold=64, edgeitems=2) - params_list = list() - this_line_length = offset - line_sep = ',\n' + (1 + offset // 2) * ' ' - for i, (k, v) in enumerate(sorted(params.items())): - if type(v) is float: - # use str for representing floating point numbers - # this way we get consistent representation across - # architectures and versions. - this_repr = '%s=%s' % (k, str(v)) - else: - # use repr of the rest - this_repr = '%s=%s' % (k, printer(v)) - if len(this_repr) > 500: - this_repr = this_repr[:300] + '...' + this_repr[-100:] - if i > 0: - if (this_line_length + len(this_repr) >= 75 or '\n' in this_repr): - params_list.append(line_sep) - this_line_length = len(line_sep) - else: - params_list.append(', ') - this_line_length += 2 - params_list.append(this_repr) - this_line_length += len(this_repr) + Inheriting from this class provides default implementations of: - np.set_printoptions(**options) - lines = ''.join(params_list) - # Strip trailing space to avoid nightmare in doctests - lines = '\n'.join(l.rstrip(' ') for l in lines.split('\n')) - return lines + - setting and getting parameters used by `GridSearchCV` and friends; + - textual and HTML representation displayed in terminals and IDEs; + - estimator serialization; + - parameters validation; + - data validation; + - feature names validation. + Read more in the :ref:`User Guide `. -class BaseEstimator: - """Base class for all estimators in scikit-learn. Notes ----- All estimators should specify all the parameters that can be set at the class level in their ``__init__`` as explicit keyword arguments (no ``*args`` or ``**kwargs``). + + Examples + -------- + >>> import numpy as np + >>> from sklearn.base import BaseEstimator + >>> class MyEstimator(BaseEstimator): + ... def __init__(self, *, param=1): + ... self.param = param + ... def fit(self, X, y=None): + ... self.is_fitted_ = True + ... return self + ... def predict(self, X): + ... return np.full(shape=X.shape[0], fill_value=self.param) + >>> estimator = MyEstimator(param=2) + >>> estimator.get_params() + {'param': 2} + >>> X = np.array([[1, 2], [2, 3], [3, 4]]) + >>> y = np.array([1, 0, 1]) + >>> estimator.fit(X, y).predict(X) + array([2, 2, 2]) + >>> estimator.set_params(param=3).fit(X, y).predict(X) + array([3, 3, 3]) """ + _html_repr = estimator_html_repr + @classmethod def _get_param_names(cls): """Get parameter names for the estimator""" # fetch the constructor or the original constructor before # deprecation wrapping if any - init = getattr(cls.__init__, 'deprecated_original', cls.__init__) + init = getattr(cls.__init__, "deprecated_original", cls.__init__) if init is object.__init__: # No explicit constructor to introspect return [] @@ -162,16 +213,20 @@ def _get_param_names(cls): # to represent init_signature = inspect.signature(init) # Consider the constructor parameters excluding 'self' - parameters = [p for p in init_signature.parameters.values() - if p.name != 'self' and p.kind != p.VAR_KEYWORD] + parameters = [ + p + for p in init_signature.parameters.values() + if p.name != "self" and p.kind != p.VAR_KEYWORD + ] for p in parameters: if p.kind == p.VAR_POSITIONAL: - raise RuntimeError("scikit-learn estimators should always " - "specify their parameters in the signature" - " of their __init__ (no varargs)." - " %s with constructor %s doesn't " - " follow this convention." - % (cls, init_signature)) + raise RuntimeError( + "scikit-learn estimators should always " + "specify their parameters in the signature" + " of their __init__ (no varargs)." + " %s with constructor %s doesn't " + " follow this convention." % (cls, init_signature) + ) # Extract and sort argument names excluding 'self' return sorted([p.name for p in parameters]) @@ -193,15 +248,72 @@ def get_params(self, deep=True): out = dict() for key in self._get_param_names(): value = getattr(self, key) - if deep and hasattr(value, 'get_params'): + if deep and hasattr(value, "get_params") and not isinstance(value, type): deep_items = value.get_params().items() - out.update((key + '__' + k, val) for k, val in deep_items) + out.update((key + "__" + k, val) for k, val in deep_items) out[key] = value return out - def set_params(self, **params): + def _get_params_html(self, deep=True): """ - Set the parameters of this estimator. + Get parameters for this estimator with a specific HTML representation. + + Parameters + ---------- + deep : bool, default=True + If True, will return the parameters for this estimator and + contained subobjects that are estimators. + + Returns + ------- + params : ParamsDict + Parameter names mapped to their values. We return a `ParamsDict` + dictionary, which renders a specific HTML representation in table + form. + """ + out = self.get_params(deep=deep) + + init_func = getattr(self.__init__, "deprecated_original", self.__init__) + init_default_params = inspect.signature(init_func).parameters + init_default_params = { + name: param.default for name, param in init_default_params.items() + } + + def is_non_default(param_name, param_value): + """Finds the parameters that have been set by the user.""" + if param_name not in init_default_params: + # happens if k is part of a **kwargs + return True + if init_default_params[param_name] == inspect._empty: + # k has no default value + return True + # avoid calling repr on nested estimators + if isinstance(param_value, BaseEstimator) and type(param_value) is not type( + init_default_params[param_name] + ): + return True + + if param_value != init_default_params[param_name] and not ( + is_scalar_nan(init_default_params[param_name]) + and is_scalar_nan(param_value) + ): + return True + return False + + # reorder the parameters from `self.get_params` using the `__init__` + # signature + remaining_params = [name for name in out if name not in init_default_params] + ordered_out = {name: out[name] for name in init_default_params if name in out} + ordered_out.update({name: out[name] for name in remaining_params}) + + non_default_ls = tuple( + [name for name, value in ordered_out.items() if is_non_default(name, value)] + ) + + return ParamsDict(ordered_out, non_default=non_default_ls) + + def set_params(self, **params): + """Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have @@ -225,12 +337,13 @@ def set_params(self, **params): nested_params = defaultdict(dict) # grouped by prefix for key, value in params.items(): - key, delim, sub_key = key.partition('__') + key, delim, sub_key = key.partition("__") if key not in valid_params: - raise ValueError('Invalid parameter %s for estimator %s. ' - 'Check the list of available parameters ' - 'with `estimator.get_params().keys()`.' % - (key, self)) + local_valid_params = self._get_param_names() + raise ValueError( + f"Invalid parameter {key!r} for estimator {self}. " + f"Valid parameters are: {local_valid_params!r}." + ) if delim: nested_params[key][sub_key] = value @@ -243,6 +356,9 @@ def set_params(self, **params): return self + def __sklearn_clone__(self): + return _clone_parametrized(self) + def __repr__(self, N_CHAR_MAX=700): # N_CHAR_MAX is the (approximate) maximum number of non-blank # characters to render. We pass it as an optional parameter to ease @@ -254,16 +370,19 @@ def __repr__(self, N_CHAR_MAX=700): # use ellipsis for sequences with a lot of elements pp = _EstimatorPrettyPrinter( - compact=True, indent=1, indent_at_name=True, - n_max_elements_to_show=N_MAX_ELEMENTS_TO_SHOW) + compact=True, + indent=1, + indent_at_name=True, + n_max_elements_to_show=N_MAX_ELEMENTS_TO_SHOW, + ) repr_ = pp.pformat(self) # Use bruteforce ellipsis when there are a lot of non-blank characters - n_nonblank = len(''.join(repr_.split())) + n_nonblank = len("".join(repr_.split())) if n_nonblank > N_CHAR_MAX: lim = N_CHAR_MAX // 2 # apprx number of chars to keep on both ends - regex = r'^(\s*\S){%d}' % lim + regex = r"^(\s*\S){%d}" % lim # The regex '^(\s*\S){%d}' % n # matches from the start of the string until the nth non-blank # character: @@ -273,7 +392,7 @@ def __repr__(self, N_CHAR_MAX=700): left_lim = re.match(regex, repr_).end() right_lim = re.match(regex, repr_[::-1]).end() - if '\n' in repr_[left_lim:-right_lim]: + if "\n" in repr_[left_lim:-right_lim]: # The left side and right side aren't on the same line. # To avoid weird cuts, e.g.: # categoric...ore', @@ -282,199 +401,125 @@ def __repr__(self, N_CHAR_MAX=700): # categoric... # handle_unknown='ignore', # so we add [^\n]*\n which matches until the next \n - regex += r'[^\n]*\n' + regex += r"[^\n]*\n" right_lim = re.match(regex, repr_[::-1]).end() - ellipsis = '...' + ellipsis = "..." if left_lim + len(ellipsis) < len(repr_) - right_lim: # Only add ellipsis if it results in a shorter repr - repr_ = repr_[:left_lim] + '...' + repr_[-right_lim:] + repr_ = repr_[:left_lim] + "..." + repr_[-right_lim:] return repr_ def __getstate__(self): + if getattr(self, "__slots__", None): + raise TypeError( + "You cannot use `__slots__` in objects inheriting from " + "`sklearn.base.BaseEstimator`." + ) + try: state = super().__getstate__() + if state is None: + # For Python 3.11+, empty instance (no `__slots__`, + # and `__dict__`) will return a state equal to `None`. + state = self.__dict__.copy() except AttributeError: + # Python < 3.11 state = self.__dict__.copy() - if type(self).__module__.startswith('sklearn.'): + if type(self).__module__.startswith("sklearn."): return dict(state.items(), _sklearn_version=__version__) else: return state def __setstate__(self, state): - if type(self).__module__.startswith('sklearn.'): + if type(self).__module__.startswith("sklearn."): pickle_version = state.pop("_sklearn_version", "pre-0.18") if pickle_version != __version__: warnings.warn( - "Trying to unpickle estimator {0} from version {1} when " - "using version {2}. This might lead to breaking code or " - "invalid results. Use at your own risk.".format( - self.__class__.__name__, pickle_version, __version__), - UserWarning) + InconsistentVersionWarning( + estimator_name=self.__class__.__name__, + current_sklearn_version=__version__, + original_sklearn_version=pickle_version, + ), + ) try: super().__setstate__(state) except AttributeError: self.__dict__.update(state) - def _more_tags(self): - return _DEFAULT_TAGS - - def _get_tags(self): - collected_tags = {} - for base_class in reversed(inspect.getmro(self.__class__)): - if hasattr(base_class, '_more_tags'): - # need the if because mixins might not have _more_tags - # but might do redundant work in estimators - # (i.e. calling more tags on BaseEstimator multiple times) - more_tags = base_class._more_tags(self) - collected_tags.update(more_tags) - return collected_tags - - def _check_n_features(self, X, reset): - """Set the `n_features_in_` attribute, or check against it. - - Parameters - ---------- - X : {ndarray, sparse matrix} of shape (n_samples, n_features) - The input samples. - reset : bool - If True, the `n_features_in_` attribute is set to `X.shape[1]`. - If False and the attribute exists, then check that it is equal to - `X.shape[1]`. If False and the attribute does *not* exist, then - the check is skipped. - .. note:: - It is recommended to call reset=True in `fit` and in the first - call to `partial_fit`. All other methods that validate `X` - should set `reset=False`. + def __sklearn_tags__(self): + return Tags( + estimator_type=None, + target_tags=TargetTags(required=False), + transformer_tags=None, + regressor_tags=None, + classifier_tags=None, + ) + + def _validate_params(self): + """Validate types and values of constructor parameters + + The expected type and values must be defined in the `_parameter_constraints` + class attribute, which is a dictionary `param_name: list of constraints`. See + the docstring of `validate_parameter_constraints` for a description of the + accepted constraints. """ - n_features = X.shape[1] - - if reset: - self.n_features_in_ = n_features - return - - if not hasattr(self, "n_features_in_"): - # Skip this check if the expected number of expected input features - # was not recorded by calling fit first. This is typically the case - # for stateless transformers. - return - - if n_features != self.n_features_in_: - raise ValueError( - f"X has {n_features} features, but {self.__class__.__name__} " - f"is expecting {self.n_features_in_} features as input.") - - def _validate_data(self, X, y='no_validation', reset=True, - validate_separately=False, **check_params): - """Validate input data and set or check the `n_features_in_` attribute. - - Parameters - ---------- - X : {array-like, sparse matrix, dataframe} of shape \ - (n_samples, n_features) - The input samples. - y : array-like of shape (n_samples,), default='no_validation' - The targets. - - - If `None`, `check_array` is called on `X`. If the estimator's - requires_y tag is True, then an error will be raised. - - If `'no_validation'`, `check_array` is called on `X` and the - estimator's requires_y tag is ignored. This is a default - placeholder and is never meant to be explicitly set. - - Otherwise, both `X` and `y` are checked with either `check_array` - or `check_X_y` depending on `validate_separately`. - - reset : bool, default=True - Whether to reset the `n_features_in_` attribute. - If False, the input will be checked for consistency with data - provided when reset was last True. - .. note:: - It is recommended to call reset=True in `fit` and in the first - call to `partial_fit`. All other methods that validate `X` - should set `reset=False`. - validate_separately : False or tuple of dicts, default=False - Only used if y is not None. - If False, call validate_X_y(). Else, it must be a tuple of kwargs - to be used for calling check_array() on X and y respectively. - **check_params : kwargs - Parameters passed to :func:`sklearn.utils.check_array` or - :func:`sklearn.utils.check_X_y`. Ignored if validate_separately - is not False. - - Returns - ------- - out : {ndarray, sparse matrix} or tuple of these - The validated input. A tuple is returned if `y` is not None. - """ - - if y is None: - if self._get_tags()['requires_y']: - raise ValueError( - f"This {self.__class__.__name__} estimator " - f"requires y to be passed, but the target y is None." - ) - X = check_array(X, **check_params) - out = X - elif isinstance(y, str) and y == 'no_validation': - X = check_array(X, **check_params) - out = X - else: - if validate_separately: - # We need this because some estimators validate X and y - # separately, and in general, separately calling check_array() - # on X and y isn't equivalent to just calling check_X_y() - # :( - check_X_params, check_y_params = validate_separately - X = check_array(X, **check_X_params) - y = check_array(y, **check_y_params) - else: - X, y = check_X_y(X, y, **check_params) - out = X, y - - if check_params.get('ensure_2d', True): - self._check_n_features(X, reset=reset) - - return out - - @property - def _repr_html_(self): - """HTML representation of estimator. - - This is redundant with the logic of `_repr_mimebundle_`. The latter - should be favorted in the long term, `_repr_html_` is only - implemented for consumers who do not interpret `_repr_mimbundle_`. - """ - if get_config()["display"] != 'diagram': - raise AttributeError("_repr_html_ is only defined when the " - "'display' configuration option is set to " - "'diagram'") - return self._repr_html_inner - - def _repr_html_inner(self): - """This function is returned by the @property `_repr_html_` to make - `hasattr(estimator, "_repr_html_") return `True` or `False` depending - on `get_config()["display"]`. - """ - return estimator_html_repr(self) - - def _repr_mimebundle_(self, **kwargs): - """Mime bundle used by jupyter kernels to display estimator""" - output = {"text/plain": repr(self)} - if get_config()["display"] == 'diagram': - output["text/html"] = estimator_html_repr(self) - return output + validate_parameter_constraints( + self._parameter_constraints, + self.get_params(deep=False), + caller_name=self.__class__.__name__, + ) class ClassifierMixin: - """Mixin class for all classifiers in scikit-learn.""" + """Mixin class for all classifiers in scikit-learn. + + This mixin defines the following functionality: + + - set estimator type to `"classifier"` through the `estimator_type` tag; + - `score` method that default to :func:`~sklearn.metrics.accuracy_score`. + - enforce that `fit` requires `y` to be passed through the `requires_y` tag, + which is done by setting the classifier type tag. + + Read more in the :ref:`User Guide `. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.base import BaseEstimator, ClassifierMixin + >>> # Mixin classes should always be on the left-hand side for a correct MRO + >>> class MyEstimator(ClassifierMixin, BaseEstimator): + ... def __init__(self, *, param=1): + ... self.param = param + ... def fit(self, X, y=None): + ... self.is_fitted_ = True + ... return self + ... def predict(self, X): + ... return np.full(shape=X.shape[0], fill_value=self.param) + >>> estimator = MyEstimator(param=1) + >>> X = np.array([[1, 2], [2, 3], [3, 4]]) + >>> y = np.array([1, 0, 1]) + >>> estimator.fit(X, y).predict(X) + array([1, 1, 1]) + >>> estimator.score(X, y) + 0.66... + """ + # TODO(1.8): Remove this attribute _estimator_type = "classifier" + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.estimator_type = "classifier" + tags.classifier_tags = ClassifierTags() + tags.target_tags.required = True + return tags + def score(self, X, y, sample_weight=None): """ - Return the mean accuracy on the given test data and labels. + Return :ref:`accuracy ` on provided data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that @@ -494,31 +539,68 @@ def score(self, X, y, sample_weight=None): Returns ------- score : float - Mean accuracy of ``self.predict(X)`` wrt. `y`. + Mean accuracy of ``self.predict(X)`` w.r.t. `y`. """ from .metrics import accuracy_score - return accuracy_score(y, self.predict(X), sample_weight=sample_weight) - def _more_tags(self): - return {'requires_y': True} + return accuracy_score(y, self.predict(X), sample_weight=sample_weight) class RegressorMixin: - """Mixin class for all regression estimators in scikit-learn.""" + """Mixin class for all regression estimators in scikit-learn. + + This mixin defines the following functionality: + + - set estimator type to `"regressor"` through the `estimator_type` tag; + - `score` method that default to :func:`~sklearn.metrics.r2_score`. + - enforce that `fit` requires `y` to be passed through the `requires_y` tag, + which is done by setting the regressor type tag. + + Read more in the :ref:`User Guide `. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.base import BaseEstimator, RegressorMixin + >>> # Mixin classes should always be on the left-hand side for a correct MRO + >>> class MyEstimator(RegressorMixin, BaseEstimator): + ... def __init__(self, *, param=1): + ... self.param = param + ... def fit(self, X, y=None): + ... self.is_fitted_ = True + ... return self + ... def predict(self, X): + ... return np.full(shape=X.shape[0], fill_value=self.param) + >>> estimator = MyEstimator(param=0) + >>> X = np.array([[1, 2], [2, 3], [3, 4]]) + >>> y = np.array([-1, 0, 1]) + >>> estimator.fit(X, y).predict(X) + array([0, 0, 0]) + >>> estimator.score(X, y) + 0.0 + """ + + # TODO(1.8): Remove this attribute _estimator_type = "regressor" + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.estimator_type = "regressor" + tags.regressor_tags = RegressorTags() + tags.target_tags.required = True + return tags + def score(self, X, y, sample_weight=None): - """Return the coefficient of determination :math:`R^2` of the - prediction. - - The coefficient :math:`R^2` is defined as :math:`(1 - \\frac{u}{v})`, - where :math:`u` is the residual sum of squares ``((y_true - y_pred) - ** 2).sum()`` and :math:`v` is the total sum of squares ``((y_true - - y_true.mean()) ** 2).sum()``. The best possible score is 1.0 and it - can be negative (because the model can be arbitrarily worse). A - constant model that always predicts the expected value of `y`, - disregarding the input features, would get a :math:`R^2` score of - 0.0. + """Return :ref:`coefficient of determination ` on test data. + + The coefficient of determination, :math:`R^2`, is defined as + :math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual + sum of squares ``((y_true - y_pred)** 2).sum()`` and :math:`v` + is the total sum of squares ``((y_true - y_true.mean()) ** 2).sum()``. + The best possible score is 1.0 and it can be negative (because the + model can be arbitrarily worse). A constant model that always predicts + the expected value of `y`, disregarding the input features, would get + a :math:`R^2` score of 0.0. Parameters ---------- @@ -537,7 +619,7 @@ def score(self, X, y, sample_weight=None): Returns ------- score : float - :math:`R^2` of ``self.predict(X)`` wrt. `y`. + :math:`R^2` of ``self.predict(X)`` w.r.t. `y`. Notes ----- @@ -550,18 +632,41 @@ def score(self, X, y, sample_weight=None): """ from .metrics import r2_score + y_pred = self.predict(X) return r2_score(y, y_pred, sample_weight=sample_weight) - def _more_tags(self): - return {'requires_y': True} - class ClusterMixin: - """Mixin class for all cluster estimators in scikit-learn.""" + """Mixin class for all cluster estimators in scikit-learn. + + - set estimator type to `"clusterer"` through the `estimator_type` tag; + - `fit_predict` method returning the cluster labels associated to each sample. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.base import BaseEstimator, ClusterMixin + >>> class MyClusterer(ClusterMixin, BaseEstimator): + ... def fit(self, X, y=None): + ... self.labels_ = np.ones(shape=(len(X),), dtype=np.int64) + ... return self + >>> X = [[1, 2], [2, 3], [3, 4]] + >>> MyClusterer().fit_predict(X) + array([1, 1, 1]) + """ + + # TODO(1.8): Remove this attribute _estimator_type = "clusterer" - def fit_predict(self, X, y=None): + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.estimator_type = "clusterer" + if tags.transformer_tags is not None: + tags.transformer_tags.preserves_dtype = [] + return tags + + def fit_predict(self, X, y=None, **kwargs): """ Perform clustering on `X` and returns cluster labels. @@ -573,6 +678,11 @@ def fit_predict(self, X, y=None): y : Ignored Not used, present for API consistency by convention. + **kwargs : dict + Arguments to be passed to ``fit``. + + .. versionadded:: 1.4 + Returns ------- labels : ndarray of shape (n_samples,), dtype=np.int64 @@ -580,15 +690,37 @@ def fit_predict(self, X, y=None): """ # non-optimized default implementation; override when a better # method is possible for a given clustering algorithm - self.fit(X) + self.fit(X, **kwargs) return self.labels_ - def _more_tags(self): - return {"preserves_dtype": []} - class BiclusterMixin: - """Mixin class for all bicluster estimators in scikit-learn.""" + """Mixin class for all bicluster estimators in scikit-learn. + + This mixin defines the following functionality: + + - `biclusters_` property that returns the row and column indicators; + - `get_indices` method that returns the row and column indices of a bicluster; + - `get_shape` method that returns the shape of a bicluster; + - `get_submatrix` method that returns the submatrix corresponding to a bicluster. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.base import BaseEstimator, BiclusterMixin + >>> class DummyBiClustering(BiclusterMixin, BaseEstimator): + ... def fit(self, X, y=None): + ... self.rows_ = np.ones(shape=(1, X.shape[0]), dtype=bool) + ... self.columns_ = np.ones(shape=(1, X.shape[1]), dtype=bool) + ... return self + >>> X = np.array([[1, 1], [2, 1], [1, 0], + ... [4, 7], [3, 5], [3, 6]]) + >>> bicluster = DummyBiClustering().fit(X) + >>> hasattr(bicluster, "biclusters_") + True + >>> bicluster.get_indices(0) + (array([0, 1, 2, 3, 4, 5]), array([0, 1])) + """ @property def biclusters_(self): @@ -614,7 +746,6 @@ def get_indices(self, i): Indices of rows in the dataset that belong to the bicluster. col_ind : ndarray, dtype=np.intp Indices of columns in the dataset that belong to the bicluster. - """ rows = self.rows_[i] columns = self.columns_[i] @@ -659,14 +790,49 @@ def get_submatrix(self, i, data): Works with sparse matrices. Only works if ``rows_`` and ``columns_`` attributes exist. """ - from .utils.validation import check_array - data = check_array(data, accept_sparse='csr') + + data = check_array(data, accept_sparse="csr") row_ind, col_ind = self.get_indices(i) return data[row_ind[:, np.newaxis], col_ind] -class TransformerMixin: - """Mixin class for all transformers in scikit-learn.""" +class TransformerMixin(_SetOutputMixin): + """Mixin class for all transformers in scikit-learn. + + This mixin defines the following functionality: + + - a `fit_transform` method that delegates to `fit` and `transform`; + - a `set_output` method to output `X` as a specific container type. + + If :term:`get_feature_names_out` is defined, then :class:`BaseEstimator` will + automatically wrap `transform` and `fit_transform` to follow the `set_output` + API. See the :ref:`developer_api_set_output` for details. + + :class:`OneToOneFeatureMixin` and + :class:`ClassNamePrefixFeaturesOutMixin` are helpful mixins for + defining :term:`get_feature_names_out`. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.base import BaseEstimator, TransformerMixin + >>> class MyTransformer(TransformerMixin, BaseEstimator): + ... def __init__(self, *, param=1): + ... self.param = param + ... def fit(self, X, y=None): + ... return self + ... def transform(self, X): + ... return np.full(shape=len(X), fill_value=self.param) + >>> transformer = MyTransformer() + >>> X = [[1, 2], [2, 3], [3, 4]] + >>> transformer.fit_transform(X) + array([1, 1, 1]) + """ + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.transformer_tags = TransformerTags() + return tags def fit_transform(self, X, y=None, **fit_params): """ @@ -694,6 +860,33 @@ def fit_transform(self, X, y=None, **fit_params): """ # non-optimized default implementation; override when a better # method is possible for a given clustering algorithm + + # we do not route parameters here, since consumers don't route. But + # since it's possible for a `transform` method to also consume + # metadata, we check if that's the case, and we raise a warning telling + # users that they should implement a custom `fit_transform` method + # to forward metadata to `transform` as well. + # + # For that, we calculate routing and check if anything would be routed + # to `transform` if we were to route them. + if _routing_enabled(): + transform_params = self.get_metadata_routing().consumes( + method="transform", params=fit_params.keys() + ) + if transform_params: + warnings.warn( + ( + f"This object ({self.__class__.__name__}) has a `transform`" + " method which consumes metadata, but `fit_transform` does not" + " forward metadata to `transform`. Please implement a custom" + " `fit_transform` method to forward metadata to `transform` as" + " well. Alternatively, you can explicitly do" + " `set_transform_request`and set all values to `False` to" + " disable metadata routed to `transform`, if that's an option." + ), + UserWarning, + ) + if y is None: # fit method of arity 1 (unsupervised transformation) return self.fit(X, **fit_params).transform(X) @@ -702,10 +895,128 @@ def fit_transform(self, X, y=None, **fit_params): return self.fit(X, y, **fit_params).transform(X) +class OneToOneFeatureMixin: + """Provides `get_feature_names_out` for simple transformers. + + This mixin assumes there's a 1-to-1 correspondence between input features + and output features, such as :class:`~sklearn.preprocessing.StandardScaler`. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.base import OneToOneFeatureMixin, BaseEstimator + >>> class MyEstimator(OneToOneFeatureMixin, BaseEstimator): + ... def fit(self, X, y=None): + ... self.n_features_in_ = X.shape[1] + ... return self + >>> X = np.array([[1, 2], [3, 4]]) + >>> MyEstimator().fit(X).get_feature_names_out() + array(['x0', 'x1'], dtype=object) + """ + + def get_feature_names_out(self, input_features=None): + """Get output feature names for transformation. + + Parameters + ---------- + input_features : array-like of str or None, default=None + Input features. + + - If `input_features` is `None`, then `feature_names_in_` is + used as feature names in. If `feature_names_in_` is not defined, + then the following input feature names are generated: + `["x0", "x1", ..., "x(n_features_in_ - 1)"]`. + - If `input_features` is an array-like, then `input_features` must + match `feature_names_in_` if `feature_names_in_` is defined. + + Returns + ------- + feature_names_out : ndarray of str objects + Same as input features. + """ + # Note that passing attributes="n_features_in_" forces check_is_fitted + # to check if the attribute is present. Otherwise it will pass on + # stateless estimators (requires_fit=False) + check_is_fitted(self, attributes="n_features_in_") + return _check_feature_names_in(self, input_features) + + +class ClassNamePrefixFeaturesOutMixin: + """Mixin class for transformers that generate their own names by prefixing. + + This mixin is useful when the transformer needs to generate its own feature + names out, such as :class:`~sklearn.decomposition.PCA`. For example, if + :class:`~sklearn.decomposition.PCA` outputs 3 features, then the generated feature + names out are: `["pca0", "pca1", "pca2"]`. + + This mixin assumes that a `_n_features_out` attribute is defined when the + transformer is fitted. `_n_features_out` is the number of output features + that the transformer will return in `transform` of `fit_transform`. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.base import ClassNamePrefixFeaturesOutMixin, BaseEstimator + >>> class MyEstimator(ClassNamePrefixFeaturesOutMixin, BaseEstimator): + ... def fit(self, X, y=None): + ... self._n_features_out = X.shape[1] + ... return self + >>> X = np.array([[1, 2], [3, 4]]) + >>> MyEstimator().fit(X).get_feature_names_out() + array(['myestimator0', 'myestimator1'], dtype=object) + """ + + def get_feature_names_out(self, input_features=None): + """Get output feature names for transformation. + + The feature names out will prefixed by the lowercased class name. For + example, if the transformer outputs 3 features, then the feature names + out are: `["class_name0", "class_name1", "class_name2"]`. + + Parameters + ---------- + input_features : array-like of str or None, default=None + Only used to validate feature names with the names seen in `fit`. + + Returns + ------- + feature_names_out : ndarray of str objects + Transformed feature names. + """ + check_is_fitted(self, "_n_features_out") + return _generate_get_feature_names_out( + self, self._n_features_out, input_features=input_features + ) + + class DensityMixin: - """Mixin class for all density estimators in scikit-learn.""" + """Mixin class for all density estimators in scikit-learn. + + This mixin defines the following functionality: + + - sets estimator type to `"density_estimator"` through the `estimator_type` tag; + - `score` method that default that do no-op. + + Examples + -------- + >>> from sklearn.base import DensityMixin + >>> class MyEstimator(DensityMixin): + ... def fit(self, X, y=None): + ... self.is_fitted_ = True + ... return self + >>> estimator = MyEstimator() + >>> hasattr(estimator, "score") + True + """ + + # TODO(1.8): Remove this attribute _estimator_type = "DensityEstimator" + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.estimator_type = "density_estimator" + return tags + def score(self, X, y=None): """Return the score of the model on the data `X`. @@ -725,47 +1036,139 @@ def score(self, X, y=None): class OutlierMixin: - """Mixin class for all outlier detection estimators in scikit-learn.""" + """Mixin class for all outlier detection estimators in scikit-learn. + + This mixin defines the following functionality: + + - set estimator type to `"outlier_detector"` through the `estimator_type` tag; + - `fit_predict` method that default to `fit` and `predict`. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.base import BaseEstimator, OutlierMixin + >>> class MyEstimator(OutlierMixin): + ... def fit(self, X, y=None): + ... self.is_fitted_ = True + ... return self + ... def predict(self, X): + ... return np.ones(shape=len(X)) + >>> estimator = MyEstimator() + >>> X = np.array([[1, 2], [2, 3], [3, 4]]) + >>> estimator.fit_predict(X) + array([1., 1., 1.]) + """ + + # TODO(1.8): Remove this attribute _estimator_type = "outlier_detector" - def fit_predict(self, X, y=None): + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.estimator_type = "outlier_detector" + return tags + + def fit_predict(self, X, y=None, **kwargs): """Perform fit on X and returns labels for X. Returns -1 for outliers and 1 for inliers. Parameters ---------- - X : {array-like, sparse matrix, dataframe} of shape \ - (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) + The input samples. y : Ignored Not used, present for API consistency by convention. + **kwargs : dict + Arguments to be passed to ``fit``. + + .. versionadded:: 1.4 + Returns ------- y : ndarray of shape (n_samples,) 1 for inliers, -1 for outliers. """ + # we do not route parameters here, since consumers don't route. But + # since it's possible for a `predict` method to also consume + # metadata, we check if that's the case, and we raise a warning telling + # users that they should implement a custom `fit_predict` method + # to forward metadata to `predict` as well. + # + # For that, we calculate routing and check if anything would be routed + # to `predict` if we were to route them. + if _routing_enabled(): + transform_params = self.get_metadata_routing().consumes( + method="predict", params=kwargs.keys() + ) + if transform_params: + warnings.warn( + ( + f"This object ({self.__class__.__name__}) has a `predict` " + "method which consumes metadata, but `fit_predict` does not " + "forward metadata to `predict`. Please implement a custom " + "`fit_predict` method to forward metadata to `predict` as well." + "Alternatively, you can explicitly do `set_predict_request`" + "and set all values to `False` to disable metadata routed to " + "`predict`, if that's an option." + ), + UserWarning, + ) + # override for transductive outlier detectors like LocalOulierFactor - return self.fit(X).predict(X) + return self.fit(X, **kwargs).predict(X) class MetaEstimatorMixin: - _required_parameters = ["estimator"] - """Mixin class for all meta estimators in scikit-learn.""" + """Mixin class for all meta estimators in scikit-learn. + + This mixin is empty, and only exists to indicate that the estimator is a + meta-estimator. + + .. versionchanged:: 1.6 + The `_required_parameters` is now removed and is unnecessary since tests are + refactored and don't use this anymore. + + Examples + -------- + >>> from sklearn.base import MetaEstimatorMixin + >>> from sklearn.datasets import load_iris + >>> from sklearn.linear_model import LogisticRegression + >>> class MyEstimator(MetaEstimatorMixin): + ... def __init__(self, *, estimator=None): + ... self.estimator = estimator + ... def fit(self, X, y=None): + ... if self.estimator is None: + ... self.estimator_ = LogisticRegression() + ... else: + ... self.estimator_ = self.estimator + ... return self + >>> X, y = load_iris(return_X_y=True) + >>> estimator = MyEstimator().fit(X, y) + >>> estimator.estimator_ + LogisticRegression() + """ class MultiOutputMixin: """Mixin to mark estimators that support multioutput.""" - def _more_tags(self): - return {'multioutput': True} + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.target_tags.multi_output = True + return tags class _UnstableArchMixin: """Mark estimators that are non-determinstic on 32bit or PowerPC""" - def _more_tags(self): - return {'non_deterministic': ( - _IS_32BIT or platform.machine().startswith(('ppc', 'powerpc')))} + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.non_deterministic = _IS_32BIT or platform.machine().startswith( + ("ppc", "powerpc") + ) + return tags def is_classifier(estimator): @@ -780,8 +1183,32 @@ def is_classifier(estimator): ------- out : bool True if estimator is a classifier and False otherwise. + + Examples + -------- + >>> from sklearn.base import is_classifier + >>> from sklearn.cluster import KMeans + >>> from sklearn.svm import SVC, SVR + >>> classifier = SVC() + >>> regressor = SVR() + >>> kmeans = KMeans() + >>> is_classifier(classifier) + True + >>> is_classifier(regressor) + False + >>> is_classifier(kmeans) + False """ - return getattr(estimator, "_estimator_type", None) == "classifier" + # TODO(1.8): Remove this check + if isinstance(estimator, type): + warnings.warn( + f"passing a class to {print(inspect.stack()[0][3])} is deprecated and " + "will be removed in 1.8. Use an instance of the class instead.", + FutureWarning, + ) + return getattr(estimator, "_estimator_type", None) == "classifier" + + return get_tags(estimator).estimator_type == "classifier" def is_regressor(estimator): @@ -796,8 +1223,74 @@ def is_regressor(estimator): ------- out : bool True if estimator is a regressor and False otherwise. + + Examples + -------- + >>> from sklearn.base import is_regressor + >>> from sklearn.cluster import KMeans + >>> from sklearn.svm import SVC, SVR + >>> classifier = SVC() + >>> regressor = SVR() + >>> kmeans = KMeans() + >>> is_regressor(classifier) + False + >>> is_regressor(regressor) + True + >>> is_regressor(kmeans) + False + """ + # TODO(1.8): Remove this check + if isinstance(estimator, type): + warnings.warn( + f"passing a class to {print(inspect.stack()[0][3])} is deprecated and " + "will be removed in 1.8. Use an instance of the class instead.", + FutureWarning, + ) + return getattr(estimator, "_estimator_type", None) == "regressor" + + return get_tags(estimator).estimator_type == "regressor" + + +def is_clusterer(estimator): + """Return True if the given estimator is (probably) a clusterer. + + .. versionadded:: 1.6 + + Parameters + ---------- + estimator : object + Estimator object to test. + + Returns + ------- + out : bool + True if estimator is a clusterer and False otherwise. + + Examples + -------- + >>> from sklearn.base import is_clusterer + >>> from sklearn.cluster import KMeans + >>> from sklearn.svm import SVC, SVR + >>> classifier = SVC() + >>> regressor = SVR() + >>> kmeans = KMeans() + >>> is_clusterer(classifier) + False + >>> is_clusterer(regressor) + False + >>> is_clusterer(kmeans) + True """ - return getattr(estimator, "_estimator_type", None) == "regressor" + # TODO(1.8): Remove this check + if isinstance(estimator, type): + warnings.warn( + f"passing a class to {print(inspect.stack()[0][3])} is deprecated and " + "will be removed in 1.8. Use an instance of the class instead.", + FutureWarning, + ) + return getattr(estimator, "_estimator_type", None) == "clusterer" + + return get_tags(estimator).estimator_type == "clusterer" def is_outlier_detector(estimator): @@ -813,45 +1306,62 @@ def is_outlier_detector(estimator): out : bool True if estimator is an outlier detector and False otherwise. """ - return getattr(estimator, "_estimator_type", None) == "outlier_detector" + # TODO(1.8): Remove this check + if isinstance(estimator, type): + warnings.warn( + f"passing a class to {print(inspect.stack()[0][3])} is deprecated and " + "will be removed in 1.8. Use an instance of the class instead.", + FutureWarning, + ) + return getattr(estimator, "_estimator_type", None) == "outlier_detector" + return get_tags(estimator).estimator_type == "outlier_detector" -def _is_pairwise(estimator): - """Returns True if estimator is pairwise. - - If the `_pairwise` attribute and the tag are present and consistent, - then use the value and not issue a warning. - - If the `_pairwise` attribute and the tag are present and not - consistent, use the `_pairwise` value and issue a deprecation - warning. - - If only the `_pairwise` attribute is present and it is not False, - issue a deprecation warning and use the `_pairwise` value. +def _fit_context(*, prefer_skip_nested_validation): + """Decorator to run the fit methods of estimators within context managers. Parameters ---------- - estimator : object - Estimator object to test. + prefer_skip_nested_validation : bool + If True, the validation of parameters of inner estimators or functions + called during fit will be skipped. + + This is useful to avoid validating many times the parameters passed by the + user from the public facing API. It's also useful to avoid validating + parameters that we pass internally to inner functions that are guaranteed to + be valid by the test suite. + + It should be set to True for most estimators, except for those that receive + non-validated objects as parameters, such as meta-estimators that are given + estimator objects. Returns ------- - out : bool - True if the estimator is pairwise and False otherwise. + decorated_fit : method + The decorated fit method. """ - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', category=FutureWarning) - has_pairwise_attribute = hasattr(estimator, '_pairwise') - pairwise_attribute = getattr(estimator, '_pairwise', False) - pairwise_tag = _safe_tags(estimator, key="pairwise") - - if has_pairwise_attribute: - if pairwise_attribute != pairwise_tag: - warnings.warn( - "_pairwise was deprecated in 0.24 and will be removed in 1.1 " - "(renaming of 0.26). Set the estimator tags of your estimator " - "instead", - FutureWarning + + def decorator(fit_method): + @functools.wraps(fit_method) + def wrapper(estimator, *args, **kwargs): + global_skip_validation = get_config()["skip_parameter_validation"] + + # we don't want to validate again for each call to partial_fit + partial_fit_and_fitted = ( + fit_method.__name__ == "partial_fit" and _is_fitted(estimator) ) - return pairwise_attribute - # use pairwise tag when the attribute is not present - return pairwise_tag + if not global_skip_validation and not partial_fit_and_fitted: + estimator._validate_params() + + with config_context( + skip_parameter_validation=( + prefer_skip_nested_validation or global_skip_validation + ) + ): + return fit_method(estimator, *args, **kwargs) + + return wrapper + + return decorator diff --git a/sklearn/calibration.py b/sklearn/calibration.py index b60a415b4419b..5b2bca2edfcc0 100644 --- a/sklearn/calibration.py +++ b/sklearn/calibration.py @@ -1,52 +1,67 @@ -"""Calibration of predicted probabilities.""" +"""Methods for calibrating predicted probabilities.""" -# Author: Alexandre Gramfort -# Balazs Kegl -# Jan Hendrik Metzen -# Mathieu Blondel -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import warnings from inspect import signature -from contextlib import suppress -from functools import partial - from math import log -import numpy as np -from joblib import Parallel +from numbers import Integral, Real +import numpy as np +from scipy.optimize import minimize from scipy.special import expit -from scipy.special import xlogy -from scipy.optimize import fmin_bfgs - -from .base import (BaseEstimator, ClassifierMixin, RegressorMixin, clone, - MetaEstimatorMixin) -from .preprocessing import label_binarize, LabelEncoder -from .utils import ( - column_or_1d, - deprecated, - indexable, -) -from .utils.multiclass import check_classification_targets -from .utils.fixes import delayed -from .utils.validation import check_is_fitted, check_consistent_length -from .utils.validation import _check_sample_weight, _num_samples -from .pipeline import Pipeline +from sklearn.utils import Bunch + +from ._loss import HalfBinomialLoss +from .base import ( + BaseEstimator, + ClassifierMixin, + MetaEstimatorMixin, + RegressorMixin, + _fit_context, + clone, +) +from .frozen import FrozenEstimator from .isotonic import IsotonicRegression +from .model_selection import LeaveOneOut, check_cv, cross_val_predict +from .preprocessing import LabelEncoder, label_binarize from .svm import LinearSVC -from .model_selection import check_cv, cross_val_predict -from .utils.validation import _deprecate_positional_args +from .utils import _safe_indexing, column_or_1d, get_tags, indexable +from .utils._param_validation import ( + HasMethods, + Hidden, + Interval, + StrOptions, + validate_params, +) +from .utils._plotting import _BinaryClassifierCurveDisplayMixin, _validate_style_kwargs +from .utils._response import _get_response_values, _process_predict_proba +from .utils.metadata_routing import ( + MetadataRouter, + MethodMapping, + _routing_enabled, + process_routing, +) +from .utils.multiclass import check_classification_targets +from .utils.parallel import Parallel, delayed +from .utils.validation import ( + _check_method_params, + _check_pos_label_consistency, + _check_response_method, + _check_sample_weight, + _num_samples, + check_consistent_length, + check_is_fitted, +) -class CalibratedClassifierCV(ClassifierMixin, - MetaEstimatorMixin, - BaseEstimator): +class CalibratedClassifierCV(ClassifierMixin, MetaEstimatorMixin, BaseEstimator): """Probability calibration with isotonic regression or logistic regression. This class uses cross-validation to both estimate the parameters of a - classifier and subsequently calibrate a classifier. With default + classifier and subsequently calibrate a classifier. With `ensemble=True`, for each cv split it fits a copy of the base estimator to the training subset, and calibrates it using the testing subset. For prediction, predicted probabilities are @@ -54,26 +69,34 @@ class CalibratedClassifierCV(ClassifierMixin, `ensemble=False`, cross-validation is used to obtain unbiased predictions, via :func:`~sklearn.model_selection.cross_val_predict`, which are then used for calibration. For prediction, the base estimator, trained using all - the data, is used. This is the method implemented when `probabilities=True` - for :mod:`sklearn.svm` estimators. + the data, is used. This is the prediction method implemented when + `probabilities=True` for :class:`~sklearn.svm.SVC` and :class:`~sklearn.svm.NuSVC` + estimators (see :ref:`User Guide ` for details). - Already fitted classifiers can be calibrated via the parameter - `cv="prefit"`. In this case, no cross-validation is used and all provided + Already fitted classifiers can be calibrated by wrapping the model in a + :class:`~sklearn.frozen.FrozenEstimator`. In this case all provided data is used for calibration. The user has to take care manually that data for model fitting and calibration are disjoint. The calibration is based on the :term:`decision_function` method of the - `base_estimator` if it exists, else on :term:`predict_proba`. + `estimator` if it exists, else on :term:`predict_proba`. Read more in the :ref:`User Guide `. + In order to learn more on the CalibratedClassifierCV class, see the + following calibration examples: + :ref:`sphx_glr_auto_examples_calibration_plot_calibration.py`, + :ref:`sphx_glr_auto_examples_calibration_plot_calibration_curve.py`, and + :ref:`sphx_glr_auto_examples_calibration_plot_calibration_multiclass.py`. Parameters ---------- - base_estimator : estimator instance, default=None + estimator : estimator instance, default=None The classifier whose output need to be calibrated to provide more accurate `predict_proba` outputs. The default classifier is a :class:`~sklearn.svm.LinearSVC`. + .. versionadded:: 1.2 + method : {'sigmoid', 'isotonic'}, default='sigmoid' The method to use for calibration. Can be 'sigmoid' which corresponds to Platt's method (i.e. a logistic regression model) or @@ -81,8 +104,7 @@ class CalibratedClassifierCV(ClassifierMixin, use isotonic calibration with too few calibration samples ``(<<1000)`` since it tends to overfit. - cv : int, cross-validation generator, iterable or "prefit", \ - default=None + cv : int, cross-validation generator, or iterable, default=None Determines the cross-validation splitting strategy. Possible inputs for cv are: @@ -99,12 +121,13 @@ class CalibratedClassifierCV(ClassifierMixin, Refer to the :ref:`User Guide ` for the various cross-validation strategies that can be used here. - If "prefit" is passed, it is assumed that `base_estimator` has been - fitted already and all data is used for calibration. - .. versionchanged:: 0.22 ``cv`` default value if None changed from 3-fold to 5-fold. + .. versionchanged:: 1.6 + `"prefit"` is deprecated. Use :class:`~sklearn.frozen.FrozenEstimator` + instead. + n_jobs : int, default=None Number of jobs to run in parallel. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. @@ -117,45 +140,77 @@ class CalibratedClassifierCV(ClassifierMixin, .. versionadded:: 0.24 - ensemble : bool, default=True - Determines how the calibrator is fitted when `cv` is not `'prefit'`. - Ignored if `cv='prefit'`. + ensemble : bool, or "auto", default="auto" + Determines how the calibrator is fitted. - If `True`, the `base_estimator` is fitted using training data and + "auto" will use `False` if the `estimator` is a + :class:`~sklearn.frozen.FrozenEstimator`, and `True` otherwise. + + If `True`, the `estimator` is fitted using training data, and calibrated using testing data, for each `cv` fold. The final estimator - is an ensemble of `n_cv` fitted classifer and calibrator pairs, where + is an ensemble of `n_cv` fitted classifier and calibrator pairs, where `n_cv` is the number of cross-validation folds. The output is the average predicted probabilities of all pairs. If `False`, `cv` is used to compute unbiased predictions, via :func:`~sklearn.model_selection.cross_val_predict`, which are then used for calibration. At prediction time, the classifier used is the - `base_estimator` trained on all the data. + `estimator` trained on all the data. Note that this method is also internally implemented in :mod:`sklearn.svm` estimators with the `probabilities=True` parameter. .. versionadded:: 0.24 + .. versionchanged:: 1.6 + `"auto"` option is added and is the default. + Attributes ---------- classes_ : ndarray of shape (n_classes,) The class labels. - calibrated_classifiers_ : list (len() equal to cv or 1 if `cv="prefit"` \ - or `ensemble=False`) + n_features_in_ : int + Number of features seen during :term:`fit`. Only defined if the + underlying estimator exposes such an attribute when fit. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Only defined if the + underlying estimator exposes such an attribute when fit. + + .. versionadded:: 1.0 + + calibrated_classifiers_ : list (len() equal to cv or 1 if `ensemble=False`) The list of classifier and calibrator pairs. - - When `cv="prefit"`, the fitted `base_estimator` and fitted + - When `ensemble=True`, `n_cv` fitted `estimator` and calibrator pairs. + `n_cv` is the number of cross-validation folds. + - When `ensemble=False`, the `estimator`, fitted on all the data, and fitted calibrator. - - When `cv` is not "prefit" and `ensemble=True`, `n_cv` fitted - `base_estimator` and calibrator pairs. `n_cv` is the number of - cross-validation folds. - - When `cv` is not "prefit" and `ensemble=False`, the `base_estimator`, - fitted on all the data, and fitted calibrator. .. versionchanged:: 0.24 Single calibrated classifier case when `ensemble=False`. + See Also + -------- + calibration_curve : Compute true and predicted probabilities + for a calibration curve. + + References + ---------- + .. [1] Obtaining calibrated probability estimates from decision trees + and naive Bayesian classifiers, B. Zadrozny & C. Elkan, ICML 2001 + + .. [2] Transforming Classifier Scores into Accurate Multiclass + Probability Estimates, B. Zadrozny & C. Elkan, (KDD 2002) + + .. [3] Probabilistic Outputs for Support Vector Machines and Comparisons to + Regularized Likelihood Methods, J. Platt, (1999) + + .. [4] Predicting Good Probabilities with Supervised Learning, + A. Niculescu-Mizil & R. Caruana, ICML 2005 + Examples -------- >>> from sklearn.datasets import make_classification @@ -164,18 +219,17 @@ class CalibratedClassifierCV(ClassifierMixin, >>> X, y = make_classification(n_samples=100, n_features=2, ... n_redundant=0, random_state=42) >>> base_clf = GaussianNB() - >>> calibrated_clf = CalibratedClassifierCV(base_estimator=base_clf, cv=3) + >>> calibrated_clf = CalibratedClassifierCV(base_clf, cv=3) >>> calibrated_clf.fit(X, y) - CalibratedClassifierCV(base_estimator=GaussianNB(), cv=3) + CalibratedClassifierCV(...) >>> len(calibrated_clf.calibrated_classifiers_) 3 >>> calibrated_clf.predict_proba(X)[:5, :] - array([[0.110..., 0.889...], - [0.072..., 0.927...], - [0.928..., 0.071...], - [0.928..., 0.071...], - [0.071..., 0.928...]]) - + array([[0.110, 0.889], + [0.072, 0.927], + [0.928, 0.072], + [0.928, 0.072], + [0.072, 0.928]]) >>> from sklearn.model_selection import train_test_split >>> X, y = make_classification(n_samples=100, n_features=2, ... n_redundant=0, random_state=42) @@ -185,41 +239,61 @@ class CalibratedClassifierCV(ClassifierMixin, >>> base_clf = GaussianNB() >>> base_clf.fit(X_train, y_train) GaussianNB() - >>> calibrated_clf = CalibratedClassifierCV( - ... base_estimator=base_clf, - ... cv="prefit" - ... ) + >>> from sklearn.frozen import FrozenEstimator + >>> calibrated_clf = CalibratedClassifierCV(FrozenEstimator(base_clf)) >>> calibrated_clf.fit(X_calib, y_calib) - CalibratedClassifierCV(base_estimator=GaussianNB(), cv='prefit') + CalibratedClassifierCV(...) >>> len(calibrated_clf.calibrated_classifiers_) 1 >>> calibrated_clf.predict_proba([[-0.5, 0.5]]) - array([[0.936..., 0.063...]]) - - References - ---------- - .. [1] Obtaining calibrated probability estimates from decision trees - and naive Bayesian classifiers, B. Zadrozny & C. Elkan, ICML 2001 - - .. [2] Transforming Classifier Scores into Accurate Multiclass - Probability Estimates, B. Zadrozny & C. Elkan, (KDD 2002) - - .. [3] Probabilistic Outputs for Support Vector Machines and Comparisons to - Regularized Likelihood Methods, J. Platt, (1999) - - .. [4] Predicting Good Probabilities with Supervised Learning, - A. Niculescu-Mizil & R. Caruana, ICML 2005 + array([[0.936, 0.063]]) """ - @_deprecate_positional_args - def __init__(self, base_estimator=None, *, method='sigmoid', - cv=None, n_jobs=None, ensemble=True): - self.base_estimator = base_estimator + + _parameter_constraints: dict = { + "estimator": [ + HasMethods(["fit", "predict_proba"]), + HasMethods(["fit", "decision_function"]), + None, + ], + "method": [StrOptions({"isotonic", "sigmoid"})], + "cv": ["cv_object", Hidden(StrOptions({"prefit"}))], + "n_jobs": [Integral, None], + "ensemble": ["boolean", StrOptions({"auto"})], + } + + def __init__( + self, + estimator=None, + *, + method="sigmoid", + cv=None, + n_jobs=None, + ensemble="auto", + ): + self.estimator = estimator self.method = method self.cv = cv self.n_jobs = n_jobs self.ensemble = ensemble - def fit(self, X, y, sample_weight=None): + def _get_estimator(self): + """Resolve which estimator to return (default is LinearSVC)""" + if self.estimator is None: + # we want all classifiers that don't expose a random_state + # to be deterministic (and we don't want to expose this one). + estimator = LinearSVC(random_state=0) + if _routing_enabled(): + estimator.set_fit_request(sample_weight=True) + else: + estimator = self.estimator + + return estimator + + @_fit_context( + # CalibratedClassifierCV.estimator is not validated yet + prefer_skip_nested_validation=False + ) + def fit(self, X, y, sample_weight=None, **fit_params): """Fit the calibrated model. Parameters @@ -233,6 +307,10 @@ def fit(self, X, y, sample_weight=None): sample_weight : array-like of shape (n_samples,), default=None Sample weights. If None, then samples are equally weighted. + **fit_params : dict + Parameters to pass to the `fit` method of the underlying + classifier. + Returns ------- self : object @@ -240,55 +318,83 @@ def fit(self, X, y, sample_weight=None): """ check_classification_targets(y) X, y = indexable(X, y) + estimator = self._get_estimator() - if self.base_estimator is None: - # we want all classifiers that don't expose a random_state - # to be deterministic (and we don't want to expose this one). - base_estimator = LinearSVC(random_state=0) - else: - base_estimator = self.base_estimator + _ensemble = self.ensemble + if _ensemble == "auto": + _ensemble = not isinstance(estimator, FrozenEstimator) self.calibrated_classifiers_ = [] if self.cv == "prefit": - # `classes_` and `n_features_in_` should be consistent with that - # of base_estimator - if isinstance(self.base_estimator, Pipeline): - check_is_fitted(self.base_estimator[-1]) - else: - check_is_fitted(self.base_estimator) - with suppress(AttributeError): - self.n_features_in_ = base_estimator.n_features_in_ - self.classes_ = self.base_estimator.classes_ + # TODO(1.8): Remove this code branch and cv='prefit' + warnings.warn( + "The `cv='prefit'` option is deprecated in 1.6 and will be removed in" + " 1.8. You can use CalibratedClassifierCV(FrozenEstimator(estimator))" + " instead.", + category=FutureWarning, + ) + # `classes_` should be consistent with that of estimator + check_is_fitted(self.estimator, attributes=["classes_"]) + self.classes_ = self.estimator.classes_ + + predictions, _ = _get_response_values( + estimator, + X, + response_method=["decision_function", "predict_proba"], + ) + if predictions.ndim == 1: + # Reshape binary output from `(n_samples,)` to `(n_samples, 1)` + predictions = predictions.reshape(-1, 1) - pred_method = _get_prediction_method(base_estimator) - n_classes = len(self.classes_) - predictions = _compute_predictions(pred_method, X, n_classes) + if sample_weight is not None: + # Check that the sample_weight dtype is consistent with the predictions + # to avoid unintentional upcasts. + sample_weight = _check_sample_weight( + sample_weight, predictions, dtype=predictions.dtype + ) calibrated_classifier = _fit_calibrator( - base_estimator, predictions, y, self.classes_, self.method, - sample_weight + estimator, + predictions, + y, + self.classes_, + self.method, + sample_weight, ) self.calibrated_classifiers_.append(calibrated_classifier) else: - X, y = self._validate_data( - X, y, accept_sparse=['csc', 'csr', 'coo'], - force_all_finite=False, allow_nd=True - ) # Set `classes_` using all `y` label_encoder_ = LabelEncoder().fit(y) self.classes_ = label_encoder_.classes_ - n_classes = len(self.classes_) - # sample_weight checks - fit_parameters = signature(base_estimator.fit).parameters - supports_sw = "sample_weight" in fit_parameters - if sample_weight is not None: - sample_weight = _check_sample_weight(sample_weight, X) - if not supports_sw: - estimator_name = type(base_estimator).__name__ - warnings.warn(f"Since {estimator_name} does not support " - "sample_weights, sample weights will only be" - " used for the calibration itself.") + if _routing_enabled(): + routed_params = process_routing( + self, + "fit", + sample_weight=sample_weight, + **fit_params, + ) + else: + # sample_weight checks + fit_parameters = signature(estimator.fit).parameters + supports_sw = "sample_weight" in fit_parameters + if sample_weight is not None and not supports_sw: + estimator_name = type(estimator).__name__ + warnings.warn( + f"Since {estimator_name} does not appear to accept" + " sample_weight, sample weights will only be used for the" + " calibration itself. This can be caused by a limitation of" + " the current scikit-learn API. See the following issue for" + " more details:" + " https://github.com/scikit-learn/scikit-learn/issues/21134." + " Be warned that the result of the calibration is likely to be" + " incorrect." + ) + routed_params = Bunch() + routed_params.splitter = Bunch(split={}) # no routing for splitter + routed_params.estimator = Bunch(fit=fit_params) + if sample_weight is not None and supports_sw: + routed_params.estimator.fit["sample_weight"] = sample_weight # Check that each cross-validation fold can have at least one # example per class @@ -298,42 +404,89 @@ def fit(self, X, y, sample_weight=None): n_folds = self.cv.n_splits else: n_folds = None - if n_folds and np.any([np.sum(y == class_) < n_folds - for class_ in self.classes_]): - raise ValueError(f"Requesting {n_folds}-fold " - "cross-validation but provided less than " - f"{n_folds} examples for at least one class.") + if n_folds and np.any(np.unique(y, return_counts=True)[1] < n_folds): + raise ValueError( + f"Requesting {n_folds}-fold " + "cross-validation but provided less than " + f"{n_folds} examples for at least one class." + ) + if isinstance(self.cv, LeaveOneOut): + raise ValueError( + "LeaveOneOut cross-validation does not allow" + "all classes to be present in test splits. " + "Please use a cross-validation generator that allows " + "all classes to appear in every test and train split." + ) cv = check_cv(self.cv, y, classifier=True) - if self.ensemble: + if _ensemble: parallel = Parallel(n_jobs=self.n_jobs) - self.calibrated_classifiers_ = parallel( delayed(_fit_classifier_calibrator_pair)( - clone(base_estimator), X, y, train=train, test=test, - method=self.method, classes=self.classes_, - supports_sw=supports_sw, sample_weight=sample_weight) - for train, test in cv.split(X, y) + clone(estimator), + X, + y, + train=train, + test=test, + method=self.method, + classes=self.classes_, + sample_weight=sample_weight, + fit_params=routed_params.estimator.fit, + ) + for train, test in cv.split(X, y, **routed_params.splitter.split) ) else: - this_estimator = clone(base_estimator) - method_name = _get_prediction_method(this_estimator).__name__ - pred_method = partial( - cross_val_predict, estimator=this_estimator, X=X, y=y, - cv=cv, method=method_name, n_jobs=self.n_jobs + this_estimator = clone(estimator) + method_name = _check_response_method( + this_estimator, + ["decision_function", "predict_proba"], + ).__name__ + predictions = cross_val_predict( + estimator=this_estimator, + X=X, + y=y, + cv=cv, + method=method_name, + n_jobs=self.n_jobs, + params=routed_params.estimator.fit, ) - predictions = _compute_predictions(pred_method, X, n_classes) - - if sample_weight is not None and supports_sw: - this_estimator.fit(X, y, sample_weight) - else: - this_estimator.fit(X, y) + if len(self.classes_) == 2: + # Ensure shape (n_samples, 1) in the binary case + if method_name == "predict_proba": + # Select the probability column of the positive class + predictions = _process_predict_proba( + y_pred=predictions, + target_type="binary", + classes=self.classes_, + pos_label=self.classes_[1], + ) + predictions = predictions.reshape(-1, 1) + + if sample_weight is not None: + # Check that the sample_weight dtype is consistent with the + # predictions to avoid unintentional upcasts. + sample_weight = _check_sample_weight( + sample_weight, predictions, dtype=predictions.dtype + ) + + this_estimator.fit(X, y, **routed_params.estimator.fit) + # Note: Here we don't pass on fit_params because the supported + # calibrators don't support fit_params anyway calibrated_classifier = _fit_calibrator( - this_estimator, predictions, y, self.classes_, self.method, - sample_weight + this_estimator, + predictions, + y, + self.classes_, + self.method, + sample_weight, ) self.calibrated_classifiers_.append(calibrated_classifier) + first_clf = self.calibrated_classifiers_[0].estimator + if hasattr(first_clf, "n_features_in_"): + self.n_features_in_ = first_clf.n_features_in_ + if hasattr(first_clf, "feature_names_in_"): + self.feature_names_in_ = first_clf.feature_names_in_ return self def predict_proba(self, X): @@ -344,7 +497,8 @@ def predict_proba(self, X): Parameters ---------- - X : The samples, as accepted by base_estimator.predict_proba + X : array-like of shape (n_samples, n_features) + The samples, as accepted by `estimator.predict_proba`. Returns ------- @@ -352,7 +506,6 @@ def predict_proba(self, X): The predicted probas. """ check_is_fitted(self) - # Compute the arithmetic mean of the predictions of the calibrated # classifiers mean_proba = np.zeros((_num_samples(X), len(self.classes_))) @@ -365,13 +518,15 @@ def predict_proba(self, X): return mean_proba def predict(self, X): - """Predict the target of new samples. The predicted class is the - class that has the highest probability, and can thus be different - from the prediction of the uncalibrated classifier. + """Predict the target of new samples. + + The predicted class is the class that has the highest probability, + and can thus be different from the prediction of the uncalibrated classifier. Parameters ---------- - X : The samples, as accepted by base_estimator.predict + X : array-like of shape (n_samples, n_features) + The samples, as accepted by `estimator.predict`. Returns ------- @@ -381,17 +536,49 @@ class that has the highest probability, and can thus be different check_is_fitted(self) return self.classes_[np.argmax(self.predict_proba(X), axis=1)] - def _more_tags(self): - return { - '_xfail_checks': { - 'check_sample_weights_invariance': - 'zero sample_weight is not equivalent to removing samples', - } - } + def get_metadata_routing(self): + """Get metadata routing of this object. + Please check :ref:`User Guide ` on how the routing + mechanism works. -def _fit_classifier_calibrator_pair(estimator, X, y, train, test, supports_sw, - method, classes, sample_weight=None): + Returns + ------- + routing : MetadataRouter + A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating + routing information. + """ + router = ( + MetadataRouter(owner=self.__class__.__name__) + .add_self_request(self) + .add( + estimator=self._get_estimator(), + method_mapping=MethodMapping().add(caller="fit", callee="fit"), + ) + .add( + splitter=self.cv, + method_mapping=MethodMapping().add(caller="fit", callee="split"), + ) + ) + return router + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.sparse = get_tags(self._get_estimator()).input_tags.sparse + return tags + + +def _fit_classifier_calibrator_pair( + estimator, + X, + y, + train, + test, + method, + classes, + sample_weight=None, + fit_params=None, +): """Fit a classifier/calibration pair on a given train/test split. Fit the classifier on the train set, compute its predictions on the test @@ -409,15 +596,12 @@ def _fit_classifier_calibrator_pair(estimator, X, y, train, test, supports_sw, y : array-like, shape (n_samples,) Targets. - train : ndarray, shape (n_train_indicies,) + train : ndarray, shape (n_train_indices,) Indices of the training subset. - test : ndarray, shape (n_test_indicies,) + test : ndarray, shape (n_test_indices,) Indices of the testing subset. - supports_sw : bool - Whether or not the `estimator` supports sample weights. - method : {'sigmoid', 'isotonic'} Method to use for calibration. @@ -427,92 +611,42 @@ def _fit_classifier_calibrator_pair(estimator, X, y, train, test, supports_sw, sample_weight : array-like, default=None Sample weights for `X`. + fit_params : dict, default=None + Parameters to pass to the `fit` method of the underlying + classifier. + Returns ------- calibrated_classifier : _CalibratedClassifier instance """ - if sample_weight is not None and supports_sw: - estimator.fit(X[train], y[train], - sample_weight=sample_weight[train]) - else: - estimator.fit(X[train], y[train]) + fit_params_train = _check_method_params(X, params=fit_params, indices=train) + X_train, y_train = _safe_indexing(X, train), _safe_indexing(y, train) + X_test, y_test = _safe_indexing(X, test), _safe_indexing(y, test) - n_classes = len(classes) - pred_method = _get_prediction_method(estimator) - predictions = _compute_predictions(pred_method, X[test], n_classes) + estimator.fit(X_train, y_train, **fit_params_train) - sw = None if sample_weight is None else sample_weight[test] + predictions, _ = _get_response_values( + estimator, + X_test, + response_method=["decision_function", "predict_proba"], + ) + if predictions.ndim == 1: + # Reshape binary output from `(n_samples,)` to `(n_samples, 1)` + predictions = predictions.reshape(-1, 1) + + if sample_weight is not None: + # Check that the sample_weight dtype is consistent with the predictions + # to avoid unintentional upcasts. + sample_weight = _check_sample_weight(sample_weight, X, dtype=predictions.dtype) + sw_test = _safe_indexing(sample_weight, test) + else: + sw_test = None calibrated_classifier = _fit_calibrator( - estimator, predictions, y[test], classes, method, sample_weight=sw + estimator, predictions, y_test, classes, method, sample_weight=sw_test ) return calibrated_classifier -def _get_prediction_method(clf): - """Return prediction method. - - `decision_function` method of `clf` returned, if it - exists, otherwise `predict_proba` method returned. - - Parameters - ---------- - clf : Estimator instance - Fitted classifier to obtain the prediction method from. - - Returns - ------- - prediction_method : callable - The prediction method. - """ - if hasattr(clf, 'decision_function'): - method = getattr(clf, 'decision_function') - elif hasattr(clf, 'predict_proba'): - method = getattr(clf, 'predict_proba') - else: - raise RuntimeError("'base_estimator' has no 'decision_function' or " - "'predict_proba' method.") - return method - - -def _compute_predictions(pred_method, X, n_classes): - """Return predictions for `X` and reshape binary outputs to shape - (n_samples, 1). - - Parameters - ---------- - pred_method : callable - Prediction method. - - X : array-like or None - Data used to obtain predictions. - - n_classes : int - Number of classes present. - - Returns - ------- - predictions : array-like, shape (X.shape[0], len(clf.classes_)) - The predictions. Note if there are 2 classes, array is of shape - (X.shape[0], 1). - """ - predictions = pred_method(X=X) - if hasattr(pred_method, '__name__'): - method_name = pred_method.__name__ - else: - method_name = signature(pred_method).parameters['method'].default - - if method_name == 'decision_function': - if predictions.ndim == 1: - predictions = predictions[:, np.newaxis] - elif method_name == 'predict_proba': - if n_classes == 2: - predictions = predictions[:, 1:] - else: # pragma: no cover - # this branch should be unreachable. - raise ValueError(f"Invalid prediction method: {method_name}") - return predictions - - def _fit_calibrator(clf, predictions, y, classes, method, sample_weight=None): """Fit calibrator(s) and return a `_CalibratedClassifier` instance. @@ -550,19 +684,14 @@ def _fit_calibrator(clf, predictions, y, classes, method, sample_weight=None): pos_class_indices = label_encoder.transform(clf.classes_) calibrators = [] for class_idx, this_pred in zip(pos_class_indices, predictions.T): - if method == 'isotonic': - calibrator = IsotonicRegression(out_of_bounds='clip') - elif method == 'sigmoid': + if method == "isotonic": + calibrator = IsotonicRegression(out_of_bounds="clip") + else: # "sigmoid" calibrator = _SigmoidCalibration() - else: - raise ValueError("'method' should be one of: 'sigmoid' or " - f"'isotonic'. Got {method}.") calibrator.fit(this_pred, Y[:, class_idx], sample_weight) calibrators.append(calibrator) - pipeline = _CalibratedClassifier( - clf, calibrators, method=method, classes=classes - ) + pipeline = _CalibratedClassifier(clf, calibrators, method=method, classes=classes) return pipeline @@ -571,7 +700,7 @@ class _CalibratedClassifier: Parameters ---------- - base_estimator : estimator instance + estimator : estimator instance Fitted classifier. calibrators : list of fitted estimator instances @@ -587,34 +716,14 @@ class _CalibratedClassifier: The method to use for calibration. Can be 'sigmoid' which corresponds to Platt's method or 'isotonic' which is a non-parametric approach based on isotonic regression. - - Attributes - ---------- - calibrators_ : list of fitted estimator instances - Same as `calibrators`. Exposed for backward-compatibility. Use - `calibrators` instead. - - .. deprecated:: 0.24 - `calibrators_` is deprecated from 0.24 and will be removed in - 1.1 (renaming of 0.26). Use `calibrators` instead. """ - def __init__(self, base_estimator, calibrators, *, classes, - method='sigmoid'): - self.base_estimator = base_estimator + + def __init__(self, estimator, calibrators, *, classes, method="sigmoid"): + self.estimator = estimator self.calibrators = calibrators self.classes = classes self.method = method - # TODO: Remove in 1.1 - # mypy error: Decorated property not supported - @deprecated( # type: ignore - "calibrators_ is deprecated in 0.24 and will be removed in 1.1" - "(renaming of 0.26). Use calibrators instead." - ) - @property - def calibrators_(self): - return self.calibrators - def predict_proba(self, X): """Calculate calibrated probabilities. @@ -631,18 +740,24 @@ def predict_proba(self, X): proba : array, shape (n_samples, n_classes) The predicted probabilities. Can be exact zeros. """ + predictions, _ = _get_response_values( + self.estimator, + X, + response_method=["decision_function", "predict_proba"], + ) + if predictions.ndim == 1: + # Reshape binary output from `(n_samples,)` to `(n_samples, 1)` + predictions = predictions.reshape(-1, 1) + n_classes = len(self.classes) - pred_method = _get_prediction_method(self.base_estimator) - predictions = _compute_predictions(pred_method, X, n_classes) label_encoder = LabelEncoder().fit(self.classes) - pos_class_indices = label_encoder.transform( - self.base_estimator.classes_ - ) + pos_class_indices = label_encoder.transform(self.estimator.classes_) proba = np.zeros((_num_samples(X), n_classes)) - for class_idx, this_pred, calibrator in \ - zip(pos_class_indices, predictions.T, self.calibrators): + for class_idx, this_pred, calibrator in zip( + pos_class_indices, predictions.T, self.calibrators + ): if n_classes == 2: # When binary, `predictions` consists only of predictions for # clf.classes_[1] but `pos_class_indices` = 0 @@ -651,15 +766,16 @@ def predict_proba(self, X): # Normalize the probabilities if n_classes == 2: - proba[:, 0] = 1. - proba[:, 1] + proba[:, 0] = 1.0 - proba[:, 1] else: denominator = np.sum(proba, axis=1)[:, np.newaxis] # In the edge case where for each class calibrator returns a null # probability for a given sample, use the uniform distribution # instead. uniform_proba = np.full_like(proba, 1 / n_classes) - proba = np.divide(proba, denominator, out=uniform_proba, - where=denominator != 0) + proba = np.divide( + proba, denominator, out=uniform_proba, where=denominator != 0 + ) # Deal with cases where the predicted probability minimally exceeds 1.0 proba[(1.0 < proba) & (proba <= 1.0 + 1e-5)] = 1.0 @@ -667,7 +783,11 @@ def predict_proba(self, X): return proba -def _sigmoid_calibration(predictions, y, sample_weight=None): +# The max_abs_prediction_threshold was approximated using +# logit(np.finfo(np.float64).eps) which is about -36 +def _sigmoid_calibration( + predictions, y, sample_weight=None, max_abs_prediction_threshold=30 +): """Probability Calibration with sigmoid method (Platt 2000) Parameters @@ -698,36 +818,73 @@ def _sigmoid_calibration(predictions, y, sample_weight=None): F = predictions # F follows Platt's notations - # Bayesian priors (see Platt end of section 2.2) - prior0 = float(np.sum(y <= 0)) - prior1 = y.shape[0] - prior0 - T = np.zeros(y.shape) - T[y > 0] = (prior1 + 1.) / (prior1 + 2.) - T[y <= 0] = 1. / (prior0 + 2.) - T1 = 1. - T - - def objective(AB): - # From Platt (beginning of Section 2.2) - P = expit(-(AB[0] * F + AB[1])) - loss = -(xlogy(T, P) + xlogy(T1, 1. - P)) - if sample_weight is not None: - return (sample_weight * loss).sum() - else: - return loss.sum() - - def grad(AB): - # gradient of the objective function - P = expit(-(AB[0] * F + AB[1])) - TEP_minus_T1P = T - P - if sample_weight is not None: - TEP_minus_T1P *= sample_weight - dA = np.dot(TEP_minus_T1P, F) - dB = np.sum(TEP_minus_T1P) - return np.array([dA, dB]) + scale_constant = 1.0 + max_prediction = np.max(np.abs(F)) + + # If the predictions have large values we scale them in order to bring + # them within a suitable range. This has no effect on the final + # (prediction) result because linear models like Logisitic Regression + # without a penalty are invariant to multiplying the features by a + # constant. + if max_prediction >= max_abs_prediction_threshold: + scale_constant = max_prediction + # We rescale the features in a copy: inplace rescaling could confuse + # the caller and make the code harder to reason about. + F = F / scale_constant + + # Bayesian priors (see Platt end of section 2.2): + # It corresponds to the number of samples, taking into account the + # `sample_weight`. + mask_negative_samples = y <= 0 + if sample_weight is not None: + prior0 = (sample_weight[mask_negative_samples]).sum() + prior1 = (sample_weight[~mask_negative_samples]).sum() + else: + prior0 = float(np.sum(mask_negative_samples)) + prior1 = y.shape[0] - prior0 + T = np.zeros_like(y, dtype=predictions.dtype) + T[y > 0] = (prior1 + 1.0) / (prior1 + 2.0) + T[y <= 0] = 1.0 / (prior0 + 2.0) + + bin_loss = HalfBinomialLoss() + + def loss_grad(AB): + # .astype below is needed to ensure y_true and raw_prediction have the + # same dtype. With result = np.float64(0) * np.array([1, 2], dtype=np.float32) + # - in Numpy 2, result.dtype is float64 + # - in Numpy<2, result.dtype is float32 + raw_prediction = -(AB[0] * F + AB[1]).astype(dtype=predictions.dtype) + l, g = bin_loss.loss_gradient( + y_true=T, + raw_prediction=raw_prediction, + sample_weight=sample_weight, + ) + loss = l.sum() + # TODO: Remove casting to np.float64 when minimum supported SciPy is 1.11.2 + # With SciPy >= 1.11.2, the LBFGS implementation will cast to float64 + # https://github.com/scipy/scipy/pull/18825. + # Here we cast to float64 to support SciPy < 1.11.2 + grad = np.asarray([-g @ F, -g.sum()], dtype=np.float64) + return loss, grad + + AB0 = np.array([0.0, log((prior0 + 1.0) / (prior1 + 1.0))]) + + opt_result = minimize( + loss_grad, + AB0, + method="L-BFGS-B", + jac=True, + options={ + "gtol": 1e-6, + "ftol": 64 * np.finfo(float).eps, + }, + ) + AB_ = opt_result.x - AB0 = np.array([0., log((prior0 + 1.) / (prior1 + 1.))]) - AB_ = fmin_bfgs(objective, AB0, fprime=grad, disp=False) - return AB_[0], AB_[1] + # The tuned multiplicative parameter is converted back to the original + # input feature scale. The offset parameter does not need rescaling since + # we did not rescale the outcome variable. + return AB_[0] / scale_constant, AB_[1] class _SigmoidCalibration(RegressorMixin, BaseEstimator): @@ -741,6 +898,7 @@ class _SigmoidCalibration(RegressorMixin, BaseEstimator): b_ : float The intercept. """ + def fit(self, X, y, sample_weight=None): """Fit the model using X, y as training data. @@ -784,9 +942,24 @@ def predict(self, T): return expit(-(self.a_ * T + self.b_)) -@_deprecate_positional_args -def calibration_curve(y_true, y_prob, *, normalize=False, n_bins=5, - strategy='uniform'): +@validate_params( + { + "y_true": ["array-like"], + "y_prob": ["array-like"], + "pos_label": [Real, str, "boolean", None], + "n_bins": [Interval(Integral, 1, None, closed="left")], + "strategy": [StrOptions({"uniform", "quantile"})], + }, + prefer_skip_nested_validation=True, +) +def calibration_curve( + y_true, + y_prob, + *, + pos_label=None, + n_bins=5, + strategy="uniform", +): """Compute true and predicted probabilities for a calibration curve. The method assumes the inputs come from a binary classifier, and @@ -804,10 +977,10 @@ def calibration_curve(y_true, y_prob, *, normalize=False, n_bins=5, y_prob : array-like of shape (n_samples,) Probabilities of the positive class. - normalize : bool, default=False - Whether y_prob needs to be normalized into the [0, 1] interval, i.e. - is not a proper probability. If True, the smallest value in y_prob - is linearly mapped onto 0 and the largest one onto 1. + pos_label : int, float, bool or str, default=None + The label of the positive class. + + .. versionadded:: 1.1 n_bins : int, default=5 Number of bins to discretize the [0, 1] interval. A bigger number @@ -832,6 +1005,13 @@ def calibration_curve(y_true, y_prob, *, normalize=False, n_bins=5, prob_pred : ndarray of shape (n_bins,) or smaller The mean predicted probability in each bin. + See Also + -------- + CalibrationDisplay.from_predictions : Plot calibration curve using true + and predicted labels. + CalibrationDisplay.from_estimator : Plot calibration curve using an + estimator and data. + References ---------- Alexandru Niculescu-Mizil and Rich Caruana (2005) Predicting Good @@ -854,30 +1034,30 @@ def calibration_curve(y_true, y_prob, *, normalize=False, n_bins=5, y_true = column_or_1d(y_true) y_prob = column_or_1d(y_prob) check_consistent_length(y_true, y_prob) + pos_label = _check_pos_label_consistency(pos_label, y_true) - if normalize: # Normalize predicted values into interval [0, 1] - y_prob = (y_prob - y_prob.min()) / (y_prob.max() - y_prob.min()) - elif y_prob.min() < 0 or y_prob.max() > 1: - raise ValueError("y_prob has values outside [0, 1] and normalize is " - "set to False.") + if y_prob.min() < 0 or y_prob.max() > 1: + raise ValueError("y_prob has values outside [0, 1].") labels = np.unique(y_true) if len(labels) > 2: - raise ValueError("Only binary classification is supported. " - "Provided labels %s." % labels) - y_true = label_binarize(y_true, classes=labels)[:, 0] + raise ValueError( + f"Only binary classification is supported. Provided labels {labels}." + ) + y_true = y_true == pos_label - if strategy == 'quantile': # Determine bin edges by distribution of data + if strategy == "quantile": # Determine bin edges by distribution of data quantiles = np.linspace(0, 1, n_bins + 1) bins = np.percentile(y_prob, quantiles * 100) - bins[-1] = bins[-1] + 1e-8 - elif strategy == 'uniform': - bins = np.linspace(0., 1. + 1e-8, n_bins + 1) + elif strategy == "uniform": + bins = np.linspace(0.0, 1.0, n_bins + 1) else: - raise ValueError("Invalid entry to 'strategy' input. Strategy " - "must be either 'quantile' or 'uniform'.") + raise ValueError( + "Invalid entry to 'strategy' input. Strategy " + "must be either 'quantile' or 'uniform'." + ) - binids = np.digitize(y_prob, bins) - 1 + binids = np.searchsorted(bins[1:-1], y_prob) bin_sums = np.bincount(binids, weights=y_prob, minlength=len(bins)) bin_true = np.bincount(binids, weights=y_true, minlength=len(bins)) @@ -888,3 +1068,381 @@ def calibration_curve(y_true, y_prob, *, normalize=False, n_bins=5, prob_pred = bin_sums[nonzero] / bin_total[nonzero] return prob_true, prob_pred + + +class CalibrationDisplay(_BinaryClassifierCurveDisplayMixin): + """Calibration curve (also known as reliability diagram) visualization. + + It is recommended to use + :func:`~sklearn.calibration.CalibrationDisplay.from_estimator` or + :func:`~sklearn.calibration.CalibrationDisplay.from_predictions` + to create a `CalibrationDisplay`. All parameters are stored as attributes. + + Read more about calibration in the :ref:`User Guide ` and + more about the scikit-learn visualization API in :ref:`visualizations`. + + For an example on how to use the visualization, see + :ref:`sphx_glr_auto_examples_calibration_plot_calibration_curve.py`. + + .. versionadded:: 1.0 + + Parameters + ---------- + prob_true : ndarray of shape (n_bins,) + The proportion of samples whose class is the positive class (fraction + of positives), in each bin. + + prob_pred : ndarray of shape (n_bins,) + The mean predicted probability in each bin. + + y_prob : ndarray of shape (n_samples,) + Probability estimates for the positive class, for each sample. + + estimator_name : str, default=None + Name of estimator. If None, the estimator name is not shown. + + pos_label : int, float, bool or str, default=None + The positive class when computing the calibration curve. + By default, `pos_label` is set to `estimators.classes_[1]` when using + `from_estimator` and set to 1 when using `from_predictions`. + + .. versionadded:: 1.1 + + Attributes + ---------- + line_ : matplotlib Artist + Calibration curve. + + ax_ : matplotlib Axes + Axes with calibration curve. + + figure_ : matplotlib Figure + Figure containing the curve. + + See Also + -------- + calibration_curve : Compute true and predicted probabilities for a + calibration curve. + CalibrationDisplay.from_predictions : Plot calibration curve using true + and predicted labels. + CalibrationDisplay.from_estimator : Plot calibration curve using an + estimator and data. + + Examples + -------- + >>> from sklearn.datasets import make_classification + >>> from sklearn.model_selection import train_test_split + >>> from sklearn.linear_model import LogisticRegression + >>> from sklearn.calibration import calibration_curve, CalibrationDisplay + >>> X, y = make_classification(random_state=0) + >>> X_train, X_test, y_train, y_test = train_test_split( + ... X, y, random_state=0) + >>> clf = LogisticRegression(random_state=0) + >>> clf.fit(X_train, y_train) + LogisticRegression(random_state=0) + >>> y_prob = clf.predict_proba(X_test)[:, 1] + >>> prob_true, prob_pred = calibration_curve(y_test, y_prob, n_bins=10) + >>> disp = CalibrationDisplay(prob_true, prob_pred, y_prob) + >>> disp.plot() + <...> + """ + + def __init__( + self, prob_true, prob_pred, y_prob, *, estimator_name=None, pos_label=None + ): + self.prob_true = prob_true + self.prob_pred = prob_pred + self.y_prob = y_prob + self.estimator_name = estimator_name + self.pos_label = pos_label + + def plot(self, *, ax=None, name=None, ref_line=True, **kwargs): + """Plot visualization. + + Extra keyword arguments will be passed to + :func:`matplotlib.pyplot.plot`. + + Parameters + ---------- + ax : Matplotlib Axes, default=None + Axes object to plot on. If `None`, a new figure and axes is + created. + + name : str, default=None + Name for labeling curve. If `None`, use `estimator_name` if + not `None`, otherwise no labeling is shown. + + ref_line : bool, default=True + If `True`, plots a reference line representing a perfectly + calibrated classifier. + + **kwargs : dict + Keyword arguments to be passed to :func:`matplotlib.pyplot.plot`. + + Returns + ------- + display : :class:`~sklearn.calibration.CalibrationDisplay` + Object that stores computed values. + """ + self.ax_, self.figure_, name = self._validate_plot_params(ax=ax, name=name) + + info_pos_label = ( + f"(Positive class: {self.pos_label})" if self.pos_label is not None else "" + ) + + default_line_kwargs = {"marker": "s", "linestyle": "-"} + if name is not None: + default_line_kwargs["label"] = name + line_kwargs = _validate_style_kwargs(default_line_kwargs, kwargs) + + ref_line_label = "Perfectly calibrated" + existing_ref_line = ref_line_label in self.ax_.get_legend_handles_labels()[1] + if ref_line and not existing_ref_line: + self.ax_.plot([0, 1], [0, 1], "k:", label=ref_line_label) + self.line_ = self.ax_.plot(self.prob_pred, self.prob_true, **line_kwargs)[0] + + # We always have to show the legend for at least the reference line + self.ax_.legend(loc="lower right") + + xlabel = f"Mean predicted probability {info_pos_label}" + ylabel = f"Fraction of positives {info_pos_label}" + self.ax_.set(xlabel=xlabel, ylabel=ylabel) + + return self + + @classmethod + def from_estimator( + cls, + estimator, + X, + y, + *, + n_bins=5, + strategy="uniform", + pos_label=None, + name=None, + ax=None, + ref_line=True, + **kwargs, + ): + """Plot calibration curve using a binary classifier and data. + + A calibration curve, also known as a reliability diagram, uses inputs + from a binary classifier and plots the average predicted probability + for each bin against the fraction of positive classes, on the + y-axis. + + Extra keyword arguments will be passed to + :func:`matplotlib.pyplot.plot`. + + Read more about calibration in the :ref:`User Guide ` and + more about the scikit-learn visualization API in :ref:`visualizations`. + + .. versionadded:: 1.0 + + Parameters + ---------- + estimator : estimator instance + Fitted classifier or a fitted :class:`~sklearn.pipeline.Pipeline` + in which the last estimator is a classifier. The classifier must + have a :term:`predict_proba` method. + + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Input values. + + y : array-like of shape (n_samples,) + Binary target values. + + n_bins : int, default=5 + Number of bins to discretize the [0, 1] interval into when + calculating the calibration curve. A bigger number requires more + data. + + strategy : {'uniform', 'quantile'}, default='uniform' + Strategy used to define the widths of the bins. + + - `'uniform'`: The bins have identical widths. + - `'quantile'`: The bins have the same number of samples and depend + on predicted probabilities. + + pos_label : int, float, bool or str, default=None + The positive class when computing the calibration curve. + By default, `estimators.classes_[1]` is considered as the + positive class. + + .. versionadded:: 1.1 + + name : str, default=None + Name for labeling curve. If `None`, the name of the estimator is + used. + + ax : matplotlib axes, default=None + Axes object to plot on. If `None`, a new figure and axes is + created. + + ref_line : bool, default=True + If `True`, plots a reference line representing a perfectly + calibrated classifier. + + **kwargs : dict + Keyword arguments to be passed to :func:`matplotlib.pyplot.plot`. + + Returns + ------- + display : :class:`~sklearn.calibration.CalibrationDisplay`. + Object that stores computed values. + + See Also + -------- + CalibrationDisplay.from_predictions : Plot calibration curve using true + and predicted labels. + + Examples + -------- + >>> import matplotlib.pyplot as plt + >>> from sklearn.datasets import make_classification + >>> from sklearn.model_selection import train_test_split + >>> from sklearn.linear_model import LogisticRegression + >>> from sklearn.calibration import CalibrationDisplay + >>> X, y = make_classification(random_state=0) + >>> X_train, X_test, y_train, y_test = train_test_split( + ... X, y, random_state=0) + >>> clf = LogisticRegression(random_state=0) + >>> clf.fit(X_train, y_train) + LogisticRegression(random_state=0) + >>> disp = CalibrationDisplay.from_estimator(clf, X_test, y_test) + >>> plt.show() + """ + y_prob, pos_label, name = cls._validate_and_get_response_values( + estimator, + X, + y, + response_method="predict_proba", + pos_label=pos_label, + name=name, + ) + + return cls.from_predictions( + y, + y_prob, + n_bins=n_bins, + strategy=strategy, + pos_label=pos_label, + name=name, + ref_line=ref_line, + ax=ax, + **kwargs, + ) + + @classmethod + def from_predictions( + cls, + y_true, + y_prob, + *, + n_bins=5, + strategy="uniform", + pos_label=None, + name=None, + ax=None, + ref_line=True, + **kwargs, + ): + """Plot calibration curve using true labels and predicted probabilities. + + Calibration curve, also known as reliability diagram, uses inputs + from a binary classifier and plots the average predicted probability + for each bin against the fraction of positive classes, on the + y-axis. + + Extra keyword arguments will be passed to + :func:`matplotlib.pyplot.plot`. + + Read more about calibration in the :ref:`User Guide ` and + more about the scikit-learn visualization API in :ref:`visualizations`. + + .. versionadded:: 1.0 + + Parameters + ---------- + y_true : array-like of shape (n_samples,) + True labels. + + y_prob : array-like of shape (n_samples,) + The predicted probabilities of the positive class. + + n_bins : int, default=5 + Number of bins to discretize the [0, 1] interval into when + calculating the calibration curve. A bigger number requires more + data. + + strategy : {'uniform', 'quantile'}, default='uniform' + Strategy used to define the widths of the bins. + + - `'uniform'`: The bins have identical widths. + - `'quantile'`: The bins have the same number of samples and depend + on predicted probabilities. + + pos_label : int, float, bool or str, default=None + The positive class when computing the calibration curve. + By default `pos_label` is set to 1. + + .. versionadded:: 1.1 + + name : str, default=None + Name for labeling curve. + + ax : matplotlib axes, default=None + Axes object to plot on. If `None`, a new figure and axes is + created. + + ref_line : bool, default=True + If `True`, plots a reference line representing a perfectly + calibrated classifier. + + **kwargs : dict + Keyword arguments to be passed to :func:`matplotlib.pyplot.plot`. + + Returns + ------- + display : :class:`~sklearn.calibration.CalibrationDisplay`. + Object that stores computed values. + + See Also + -------- + CalibrationDisplay.from_estimator : Plot calibration curve using an + estimator and data. + + Examples + -------- + >>> import matplotlib.pyplot as plt + >>> from sklearn.datasets import make_classification + >>> from sklearn.model_selection import train_test_split + >>> from sklearn.linear_model import LogisticRegression + >>> from sklearn.calibration import CalibrationDisplay + >>> X, y = make_classification(random_state=0) + >>> X_train, X_test, y_train, y_test = train_test_split( + ... X, y, random_state=0) + >>> clf = LogisticRegression(random_state=0) + >>> clf.fit(X_train, y_train) + LogisticRegression(random_state=0) + >>> y_prob = clf.predict_proba(X_test)[:, 1] + >>> disp = CalibrationDisplay.from_predictions(y_test, y_prob) + >>> plt.show() + """ + pos_label_validated, name = cls._validate_from_predictions_params( + y_true, y_prob, sample_weight=None, pos_label=pos_label, name=name + ) + + prob_true, prob_pred = calibration_curve( + y_true, y_prob, n_bins=n_bins, strategy=strategy, pos_label=pos_label + ) + + disp = cls( + prob_true=prob_true, + prob_pred=prob_pred, + y_prob=y_prob, + estimator_name=name, + pos_label=pos_label_validated, + ) + return disp.plot(ax=ax, ref_line=ref_line, **kwargs) diff --git a/sklearn/cluster/__init__.py b/sklearn/cluster/__init__.py index 714395d4fe469..de86a59e07113 100644 --- a/sklearn/cluster/__init__.py +++ b/sklearn/cluster/__init__.py @@ -1,43 +1,56 @@ -""" -The :mod:`sklearn.cluster` module gathers popular unsupervised clustering -algorithms. -""" +"""Popular unsupervised clustering algorithms.""" -from ._spectral import spectral_clustering, SpectralClustering -from ._mean_shift import (mean_shift, MeanShift, - estimate_bandwidth, get_bin_seeds) -from ._affinity_propagation import affinity_propagation, AffinityPropagation -from ._agglomerative import (ward_tree, AgglomerativeClustering, - linkage_tree, FeatureAgglomeration) -from ._kmeans import k_means, KMeans, MiniBatchKMeans, kmeans_plusplus -from ._dbscan import dbscan, DBSCAN -from ._optics import (OPTICS, cluster_optics_dbscan, compute_optics_graph, - cluster_optics_xi) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from ._affinity_propagation import AffinityPropagation, affinity_propagation +from ._agglomerative import ( + AgglomerativeClustering, + FeatureAgglomeration, + linkage_tree, + ward_tree, +) from ._bicluster import SpectralBiclustering, SpectralCoclustering from ._birch import Birch +from ._bisect_k_means import BisectingKMeans +from ._dbscan import DBSCAN, dbscan +from ._hdbscan.hdbscan import HDBSCAN +from ._kmeans import KMeans, MiniBatchKMeans, k_means, kmeans_plusplus +from ._mean_shift import MeanShift, estimate_bandwidth, get_bin_seeds, mean_shift +from ._optics import ( + OPTICS, + cluster_optics_dbscan, + cluster_optics_xi, + compute_optics_graph, +) +from ._spectral import SpectralClustering, spectral_clustering -__all__ = ['AffinityPropagation', - 'AgglomerativeClustering', - 'Birch', - 'DBSCAN', - 'OPTICS', - 'cluster_optics_dbscan', - 'cluster_optics_xi', - 'compute_optics_graph', - 'KMeans', - 'FeatureAgglomeration', - 'MeanShift', - 'MiniBatchKMeans', - 'SpectralClustering', - 'affinity_propagation', - 'dbscan', - 'estimate_bandwidth', - 'get_bin_seeds', - 'k_means', - 'kmeans_plusplus', - 'linkage_tree', - 'mean_shift', - 'spectral_clustering', - 'ward_tree', - 'SpectralBiclustering', - 'SpectralCoclustering'] +__all__ = [ + "DBSCAN", + "HDBSCAN", + "OPTICS", + "AffinityPropagation", + "AgglomerativeClustering", + "Birch", + "BisectingKMeans", + "FeatureAgglomeration", + "KMeans", + "MeanShift", + "MiniBatchKMeans", + "SpectralBiclustering", + "SpectralClustering", + "SpectralCoclustering", + "affinity_propagation", + "cluster_optics_dbscan", + "cluster_optics_xi", + "compute_optics_graph", + "dbscan", + "estimate_bandwidth", + "get_bin_seeds", + "k_means", + "kmeans_plusplus", + "linkage_tree", + "mean_shift", + "spectral_clustering", + "ward_tree", +] diff --git a/sklearn/cluster/_affinity_propagation.py b/sklearn/cluster/_affinity_propagation.py index cb9230cd2382f..c7ae6ed63580d 100644 --- a/sklearn/cluster/_affinity_propagation.py +++ b/sklearn/cluster/_affinity_propagation.py @@ -1,21 +1,20 @@ """Affinity Propagation clustering algorithm.""" -# Author: Alexandre Gramfort alexandre.gramfort@inria.fr -# Gael Varoquaux gael.varoquaux@normalesup.org +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -# License: BSD 3 clause +import warnings +from numbers import Integral, Real import numpy as np -import warnings -from ..exceptions import ConvergenceWarning -from ..base import BaseEstimator, ClusterMixin -from ..utils import as_float_array, check_random_state -from ..utils.deprecation import deprecated -from ..utils.validation import check_is_fitted, _deprecate_positional_args -from ..metrics import euclidean_distances -from ..metrics import pairwise_distances_argmin from .._config import config_context +from ..base import BaseEstimator, ClusterMixin, _fit_context +from ..exceptions import ConvergenceWarning +from ..metrics import euclidean_distances, pairwise_distances_argmin +from ..utils import check_random_state +from ..utils._param_validation import Interval, StrOptions, validate_params +from ..utils.validation import check_is_fitted, validate_data def _equal_similarities_and_preferences(S, preference): @@ -32,132 +31,41 @@ def all_equal_similarities(): return all_equal_preferences() and all_equal_similarities() -@_deprecate_positional_args -def affinity_propagation(S, *, preference=None, convergence_iter=15, - max_iter=200, damping=0.5, copy=True, verbose=False, - return_n_iter=False, random_state='warn'): - """Perform Affinity Propagation Clustering of data. - - Read more in the :ref:`User Guide `. - - Parameters - ---------- - - S : array-like of shape (n_samples, n_samples) - Matrix of similarities between points. - - preference : array-like of shape (n_samples,) or float, default=None - Preferences for each point - points with larger values of - preferences are more likely to be chosen as exemplars. The number of - exemplars, i.e. of clusters, is influenced by the input preferences - value. If the preferences are not passed as arguments, they will be - set to the median of the input similarities (resulting in a moderate - number of clusters). For a smaller amount of clusters, this can be set - to the minimum value of the similarities. - - convergence_iter : int, default=15 - Number of iterations with no change in the number - of estimated clusters that stops the convergence. - - max_iter : int, default=200 - Maximum number of iterations - - damping : float, default=0.5 - Damping factor between 0.5 and 1. - - copy : bool, default=True - If copy is False, the affinity matrix is modified inplace by the - algorithm, for memory efficiency. - - verbose : bool, default=False - The verbosity level. - - return_n_iter : bool, default=False - Whether or not to return the number of iterations. - - random_state : int, RandomState instance or None, default=0 - Pseudo-random number generator to control the starting state. - Use an int for reproducible results across function calls. - See the :term:`Glossary `. - - .. versionadded:: 0.23 - this parameter was previously hardcoded as 0. - - Returns - ------- - - cluster_centers_indices : ndarray of shape (n_clusters,) - Index of clusters centers. - - labels : ndarray of shape (n_samples,) - Cluster labels for each point. - - n_iter : int - Number of iterations run. Returned only if `return_n_iter` is - set to True. - - Notes - ----- - For an example, see :ref:`examples/cluster/plot_affinity_propagation.py - `. - - When the algorithm does not converge, it returns an empty array as - ``cluster_center_indices`` and ``-1`` as label for each training sample. - - When all training samples have equal similarities and equal preferences, - the assignment of cluster centers and labels depends on the preference. - If the preference is smaller than the similarities, a single cluster center - and label ``0`` for every sample will be returned. Otherwise, every - training sample becomes its own cluster center and is assigned a unique - label. - - References - ---------- - Brendan J. Frey and Delbert Dueck, "Clustering by Passing Messages - Between Data Points", Science Feb. 2007 - """ - S = as_float_array(S, copy=copy) +def _affinity_propagation( + S, + *, + preference, + convergence_iter, + max_iter, + damping, + verbose, + return_n_iter, + random_state, +): + """Main affinity propagation algorithm.""" n_samples = S.shape[0] - - if S.shape[0] != S.shape[1]: - raise ValueError("S must be a square array (shape=%s)" % repr(S.shape)) - - if preference is None: - preference = np.median(S) - if damping < 0.5 or damping >= 1: - raise ValueError('damping must be >= 0.5 and < 1') - - preference = np.array(preference) - - if (n_samples == 1 or - _equal_similarities_and_preferences(S, preference)): + if n_samples == 1 or _equal_similarities_and_preferences(S, preference): # It makes no sense to run the algorithm in this case, so return 1 or # n_samples clusters, depending on preferences - warnings.warn("All samples have mutually equal similarities. " - "Returning arbitrary cluster center(s).") - if preference.flat[0] >= S.flat[n_samples - 1]: - return ((np.arange(n_samples), np.arange(n_samples), 0) - if return_n_iter - else (np.arange(n_samples), np.arange(n_samples))) - else: - return ((np.array([0]), np.array([0] * n_samples), 0) - if return_n_iter - else (np.array([0]), np.array([0] * n_samples))) - - if random_state == 'warn': warnings.warn( - "'random_state' has been introduced in 0.23. It will be set to " - "None starting from 1.0 (renaming of 0.25) which means that " - "results will differ at every function call. Set 'random_state' " - "to None to silence this warning, or to 0 to keep the behavior of " - "versions <0.23.", - FutureWarning + "All samples have mutually equal similarities. " + "Returning arbitrary cluster center(s)." ) - random_state = 0 - random_state = check_random_state(random_state) + if preference.flat[0] > S.flat[n_samples - 1]: + return ( + (np.arange(n_samples), np.arange(n_samples), 0) + if return_n_iter + else (np.arange(n_samples), np.arange(n_samples)) + ) + else: + return ( + (np.array([0]), np.array([0] * n_samples), 0) + if return_n_iter + else (np.array([0]), np.array([0] * n_samples)) + ) # Place preference on the diagonal of S - S.flat[::(n_samples + 1)] = preference + S.flat[:: (n_samples + 1)] = preference A = np.zeros((n_samples, n_samples)) R = np.zeros((n_samples, n_samples)) # Initialize messages @@ -165,8 +73,9 @@ def affinity_propagation(S, *, preference=None, convergence_iter=15, tmp = np.zeros((n_samples, n_samples)) # Remove degeneracies - S += ((np.finfo(S.dtype).eps * S + np.finfo(S.dtype).tiny * 100) * - random_state.randn(n_samples, n_samples)) + S += ( + np.finfo(S.dtype).eps * S + np.finfo(S.dtype).tiny * 100 + ) * random_state.standard_normal(size=(n_samples, n_samples)) # Execute parallel affinity propagation updates e = np.zeros((n_samples, convergence_iter)) @@ -192,13 +101,13 @@ def affinity_propagation(S, *, preference=None, convergence_iter=15, # tmp = Rp; compute availabilities np.maximum(R, 0, tmp) - tmp.flat[::n_samples + 1] = R.flat[::n_samples + 1] + tmp.flat[:: n_samples + 1] = R.flat[:: n_samples + 1] # tmp = -Anew tmp -= np.sum(tmp, axis=0) dA = np.diag(tmp).copy() tmp.clip(0, np.inf, tmp) - tmp.flat[::n_samples + 1] = dA + tmp.flat[:: n_samples + 1] = dA # Damping tmp *= 1 - damping @@ -212,8 +121,7 @@ def affinity_propagation(S, *, preference=None, convergence_iter=15, if it >= convergence_iter: se = np.sum(e, axis=1) - unconverged = (np.sum((se == convergence_iter) + (se == 0)) - != n_samples) + unconverged = np.sum((se == convergence_iter) + (se == 0)) != n_samples if (not unconverged and (K > 0)) or (it == max_iter): never_converged = False if verbose: @@ -227,12 +135,20 @@ def affinity_propagation(S, *, preference=None, convergence_iter=15, I = np.flatnonzero(E) K = I.size # Identify exemplars - if K > 0 and not never_converged: + if K > 0: + if never_converged: + warnings.warn( + ( + "Affinity propagation did not converge, this model " + "may return degenerate cluster centers and labels." + ), + ConvergenceWarning, + ) c = np.argmax(S[:, I], axis=1) c[I] = np.arange(K) # Identify clusters # Refine the final set of exemplars and clusters and return results for k in range(K): - ii = np.where(c == k)[0] + ii = np.asarray(c == k).nonzero()[0] j = np.argmax(np.sum(S[ii[:, np.newaxis], ii], axis=0)) I[k] = ii[j] @@ -243,8 +159,13 @@ def affinity_propagation(S, *, preference=None, convergence_iter=15, cluster_centers_indices = np.unique(labels) labels = np.searchsorted(cluster_centers_indices, labels) else: - warnings.warn("Affinity propagation did not converge, this model " - "will not have any cluster centers.", ConvergenceWarning) + warnings.warn( + ( + "Affinity propagation did not converge and this model " + "will not have any cluster centers." + ), + ConvergenceWarning, + ) labels = np.array([-1] * n_samples) cluster_centers_indices = [] @@ -255,6 +176,138 @@ def affinity_propagation(S, *, preference=None, convergence_iter=15, ############################################################################### +# Public API + + +@validate_params( + { + "S": ["array-like"], + "return_n_iter": ["boolean"], + }, + prefer_skip_nested_validation=False, +) +def affinity_propagation( + S, + *, + preference=None, + convergence_iter=15, + max_iter=200, + damping=0.5, + copy=True, + verbose=False, + return_n_iter=False, + random_state=None, +): + """Perform Affinity Propagation Clustering of data. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + S : array-like of shape (n_samples, n_samples) + Matrix of similarities between points. + + preference : array-like of shape (n_samples,) or float, default=None + Preferences for each point - points with larger values of + preferences are more likely to be chosen as exemplars. The number of + exemplars, i.e. of clusters, is influenced by the input preferences + value. If the preferences are not passed as arguments, they will be + set to the median of the input similarities (resulting in a moderate + number of clusters). For a smaller amount of clusters, this can be set + to the minimum value of the similarities. + + convergence_iter : int, default=15 + Number of iterations with no change in the number + of estimated clusters that stops the convergence. + + max_iter : int, default=200 + Maximum number of iterations. + + damping : float, default=0.5 + Damping factor between 0.5 and 1. + + copy : bool, default=True + If copy is False, the affinity matrix is modified inplace by the + algorithm, for memory efficiency. + + verbose : bool, default=False + The verbosity level. + + return_n_iter : bool, default=False + Whether or not to return the number of iterations. + + random_state : int, RandomState instance or None, default=None + Pseudo-random number generator to control the starting state. + Use an int for reproducible results across function calls. + See the :term:`Glossary `. + + .. versionadded:: 0.23 + this parameter was previously hardcoded as 0. + + Returns + ------- + cluster_centers_indices : ndarray of shape (n_clusters,) + Index of clusters centers. + + labels : ndarray of shape (n_samples,) + Cluster labels for each point. + + n_iter : int + Number of iterations run. Returned only if `return_n_iter` is + set to True. + + Notes + ----- + For an example usage, + see :ref:`sphx_glr_auto_examples_cluster_plot_affinity_propagation.py`. + You may also check out, + :ref:`sphx_glr_auto_examples_applications_plot_stock_market.py` + + When the algorithm does not converge, it will still return a arrays of + ``cluster_center_indices`` and labels if there are any exemplars/clusters, + however they may be degenerate and should be used with caution. + + When all training samples have equal similarities and equal preferences, + the assignment of cluster centers and labels depends on the preference. + If the preference is smaller than the similarities, a single cluster center + and label ``0`` for every sample will be returned. Otherwise, every + training sample becomes its own cluster center and is assigned a unique + label. + + References + ---------- + Brendan J. Frey and Delbert Dueck, "Clustering by Passing Messages + Between Data Points", Science Feb. 2007 + + Examples + -------- + >>> import numpy as np + >>> from sklearn.cluster import affinity_propagation + >>> from sklearn.metrics.pairwise import euclidean_distances + >>> X = np.array([[1, 2], [1, 4], [1, 0], + ... [4, 2], [4, 4], [4, 0]]) + >>> S = -euclidean_distances(X, squared=True) + >>> cluster_centers_indices, labels = affinity_propagation(S, random_state=0) + >>> cluster_centers_indices + array([0, 3]) + >>> labels + array([0, 0, 0, 1, 1, 1]) + """ + estimator = AffinityPropagation( + damping=damping, + max_iter=max_iter, + convergence_iter=convergence_iter, + copy=copy, + preference=preference, + affinity="precomputed", + verbose=verbose, + random_state=random_state, + ).fit(S) + + if return_n_iter: + return estimator.cluster_centers_indices_, estimator.labels_, estimator.n_iter_ + return estimator.cluster_centers_indices_, estimator.labels_ + class AffinityPropagation(ClusterMixin, BaseEstimator): """Perform Affinity Propagation Clustering of data. @@ -264,7 +317,7 @@ class AffinityPropagation(ClusterMixin, BaseEstimator): Parameters ---------- damping : float, default=0.5 - Damping factor (between 0.5 and 1) is the extent to + Damping factor in the range `[0.5, 1.0)` is the extent to which the current value is maintained relative to incoming values (weighted 1 - damping). This in order to avoid numerical oscillations when updating these @@ -295,7 +348,7 @@ class AffinityPropagation(ClusterMixin, BaseEstimator): verbose : bool, default=False Whether to be verbose. - random_state : int, RandomState instance or None, default=0 + random_state : int, RandomState instance or None, default=None Pseudo-random number generator to control the starting state. Use an int for reproducible results across function calls. See the :term:`Glossary `. @@ -320,17 +373,42 @@ class AffinityPropagation(ClusterMixin, BaseEstimator): n_iter_ : int Number of iterations taken to converge. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + AgglomerativeClustering : Recursively merges the pair of + clusters that minimally increases a given linkage distance. + FeatureAgglomeration : Similar to AgglomerativeClustering, + but recursively merges features instead of samples. + KMeans : K-Means clustering. + MiniBatchKMeans : Mini-Batch K-Means clustering. + MeanShift : Mean shift clustering using a flat kernel. + SpectralClustering : Apply clustering to a projection + of the normalized Laplacian. + Notes ----- - For an example, see :ref:`examples/cluster/plot_affinity_propagation.py - `. - The algorithmic complexity of affinity propagation is quadratic in the number of points. - When ``fit`` does not converge, ``cluster_centers_`` becomes an empty - array and all training samples will be labelled as ``-1``. In addition, - ``predict`` will then label every sample as ``-1``. + When the algorithm does not converge, it will still return a arrays of + ``cluster_center_indices`` and labels if there are any exemplars/clusters, + however they may be degenerate and should be used with caution. + + When ``fit`` does not converge, ``cluster_centers_`` is still populated + however it may be degenerate. In such a case, proceed with caution. + If ``fit`` does not converge and fails to produce any ``cluster_centers_`` + then ``predict`` will label every sample as ``-1``. When all training samples have equal similarities and equal preferences, the assignment of cluster centers and labels depends on the preference. @@ -361,12 +439,41 @@ class AffinityPropagation(ClusterMixin, BaseEstimator): >>> clustering.cluster_centers_ array([[1, 2], [4, 2]]) + + For an example usage, + see :ref:`sphx_glr_auto_examples_cluster_plot_affinity_propagation.py`. + + For a comparison of Affinity Propagation with other clustering algorithms, see + :ref:`sphx_glr_auto_examples_cluster_plot_cluster_comparison.py` """ - @_deprecate_positional_args - def __init__(self, *, damping=.5, max_iter=200, convergence_iter=15, - copy=True, preference=None, affinity='euclidean', - verbose=False, random_state='warn'): + _parameter_constraints: dict = { + "damping": [Interval(Real, 0.5, 1.0, closed="left")], + "max_iter": [Interval(Integral, 1, None, closed="left")], + "convergence_iter": [Interval(Integral, 1, None, closed="left")], + "copy": ["boolean"], + "preference": [ + "array-like", + Interval(Real, None, None, closed="neither"), + None, + ], + "affinity": [StrOptions({"euclidean", "precomputed"})], + "verbose": ["verbose"], + "random_state": ["random_state"], + } + + def __init__( + self, + *, + damping=0.5, + max_iter=200, + convergence_iter=15, + copy=True, + preference=None, + affinity="euclidean", + verbose=False, + random_state=None, + ): self.damping = damping self.max_iter = max_iter self.convergence_iter = convergence_iter @@ -376,17 +483,13 @@ def __init__(self, *, damping=.5, max_iter=200, convergence_iter=15, self.affinity = affinity self.random_state = random_state - # TODO: Remove in 1.1 - # mypy error: Decorated property not supported - @deprecated("Attribute _pairwise was deprecated in " # type: ignore - "version 0.24 and will be removed in 1.1 (renaming of 0.26).") - @property - def _pairwise(self): - return self.affinity == "precomputed" - - def _more_tags(self): - return {'pairwise': self.affinity == 'precomputed'} + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.pairwise = self.affinity == "precomputed" + tags.input_tags.sparse = self.affinity != "precomputed" + return tags + @_fit_context(prefer_skip_nested_validation=True) def fit(self, X, y=None): """Fit the clustering from features, or affinity matrix. @@ -404,29 +507,43 @@ def fit(self, X, y=None): Returns ------- self - + Returns the instance itself. """ if self.affinity == "precomputed": - accept_sparse = False - else: - accept_sparse = 'csr' - X = self._validate_data(X, accept_sparse=accept_sparse) - if self.affinity == "precomputed": + X = validate_data(self, X, copy=self.copy, force_writeable=True) self.affinity_matrix_ = X - elif self.affinity == "euclidean": + else: # self.affinity == "euclidean" + X = validate_data(self, X, accept_sparse="csr") self.affinity_matrix_ = -euclidean_distances(X, squared=True) + + if self.affinity_matrix_.shape[0] != self.affinity_matrix_.shape[1]: + raise ValueError( + "The matrix of similarities must be a square array. " + f"Got {self.affinity_matrix_.shape} instead." + ) + + if self.preference is None: + preference = np.median(self.affinity_matrix_) else: - raise ValueError("Affinity must be 'precomputed' or " - "'euclidean'. Got %s instead" - % str(self.affinity)) - - self.cluster_centers_indices_, self.labels_, self.n_iter_ = \ - affinity_propagation( - self.affinity_matrix_, preference=self.preference, - max_iter=self.max_iter, - convergence_iter=self.convergence_iter, damping=self.damping, - copy=self.copy, verbose=self.verbose, return_n_iter=True, - random_state=self.random_state) + preference = self.preference + preference = np.asarray(preference) + + random_state = check_random_state(self.random_state) + + ( + self.cluster_centers_indices_, + self.labels_, + self.n_iter_, + ) = _affinity_propagation( + self.affinity_matrix_, + max_iter=self.max_iter, + convergence_iter=self.convergence_iter, + preference=preference, + damping=self.damping, + verbose=self.verbose, + return_n_iter=True, + random_state=random_state, + ) if self.affinity != "precomputed": self.cluster_centers_ = X[self.cluster_centers_indices_].copy() @@ -448,23 +565,28 @@ def predict(self, X): Cluster labels. """ check_is_fitted(self) - X = self._validate_data(X, reset=False) + X = validate_data(self, X, reset=False, accept_sparse="csr") if not hasattr(self, "cluster_centers_"): - raise ValueError("Predict method is not supported when " - "affinity='precomputed'.") + raise ValueError( + "Predict method is not supported when affinity='precomputed'." + ) if self.cluster_centers_.shape[0] > 0: with config_context(assume_finite=True): return pairwise_distances_argmin(X, self.cluster_centers_) else: - warnings.warn("This model does not have any cluster centers " - "because affinity propagation did not converge. " - "Labeling every sample as '-1'.", ConvergenceWarning) + warnings.warn( + ( + "This model does not have any cluster centers " + "because affinity propagation did not converge. " + "Labeling every sample as '-1'." + ), + ConvergenceWarning, + ) return np.array([-1] * X.shape[0]) def fit_predict(self, X, y=None): - """Fit the clustering from features or affinity matrix, and return - cluster labels. + """Fit clustering from features/affinity matrix; return cluster labels. Parameters ---------- diff --git a/sklearn/cluster/_agglomerative.py b/sklearn/cluster/_agglomerative.py index 66342797e33b5..f068dc934151d 100644 --- a/sklearn/cluster/_agglomerative.py +++ b/sklearn/cluster/_agglomerative.py @@ -2,28 +2,41 @@ These routines perform some hierarchical agglomerative clustering of some input data. - -Authors : Vincent Michel, Bertrand Thirion, Alexandre Gramfort, - Gael Varoquaux -License: BSD 3 clause """ + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import warnings from heapq import heapify, heappop, heappush, heappushpop +from numbers import Integral, Real import numpy as np from scipy import sparse from scipy.sparse.csgraph import connected_components -from ..base import BaseEstimator, ClusterMixin -from ..metrics.pairwise import paired_distances, pairwise_distances -from ..neighbors import DistanceMetric -from ..neighbors._dist_metrics import METRIC_MAPPING +from ..base import ( + BaseEstimator, + ClassNamePrefixFeaturesOutMixin, + ClusterMixin, + _fit_context, +) +from ..metrics import DistanceMetric +from ..metrics._dist_metrics import METRIC_MAPPING64 +from ..metrics.pairwise import _VALID_METRICS, paired_distances from ..utils import check_array from ..utils._fast_dict import IntFloatDict -from ..utils.fixes import _astype_copy_false -from ..utils.validation import _deprecate_positional_args, check_memory +from ..utils._param_validation import ( + HasMethods, + Interval, + StrOptions, + validate_params, +) +from ..utils.graph import _fix_connected_components +from ..utils.validation import check_memory, validate_data + # mypy error: Module 'sklearn.cluster' has no attribute '_hierarchical_fast' -from . import _hierarchical_fast as _hierarchical # type: ignore +from . import _hierarchical_fast as _hierarchical # type: ignore[attr-defined] from ._feature_agglomeration import AgglomerationTransform ############################################################################### @@ -32,56 +45,88 @@ def _fix_connectivity(X, connectivity, affinity): """ - Fixes the connectivity matrix + Fixes the connectivity matrix. + + The different steps are: + + - copies it + - makes it symmetric + - converts it to LIL if necessary + - completes it if necessary. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Feature matrix representing `n_samples` samples to be clustered. + + connectivity : sparse matrix, default=None + Connectivity matrix. Defines for each sample the neighboring samples + following a given structure of the data. The matrix is assumed to + be symmetric and only the upper triangular half is used. + Default is `None`, i.e, the Ward algorithm is unstructured. - - copies it - - makes it symmetric - - converts it to LIL if necessary - - completes it if necessary + affinity : {"euclidean", "precomputed"}, default="euclidean" + Which affinity to use. At the moment `precomputed` and + ``euclidean`` are supported. `euclidean` uses the + negative squared Euclidean distance between points. + + Returns + ------- + connectivity : sparse matrix + The fixed connectivity matrix. + + n_connected_components : int + The number of connected components in the graph. """ n_samples = X.shape[0] - if (connectivity.shape[0] != n_samples or - connectivity.shape[1] != n_samples): - raise ValueError('Wrong shape for connectivity matrix: %s ' - 'when X is %s' % (connectivity.shape, X.shape)) + if connectivity.shape[0] != n_samples or connectivity.shape[1] != n_samples: + raise ValueError( + "Wrong shape for connectivity matrix: %s when X is %s" + % (connectivity.shape, X.shape) + ) # Make the connectivity matrix symmetric: connectivity = connectivity + connectivity.T # Convert connectivity matrix to LIL - if not sparse.isspmatrix_lil(connectivity): - if not sparse.isspmatrix(connectivity): - connectivity = sparse.lil_matrix(connectivity) - else: - connectivity = connectivity.tolil() + if not sparse.issparse(connectivity): + connectivity = sparse.lil_matrix(connectivity) + + # `connectivity` is a sparse matrix at this point + if connectivity.format != "lil": + connectivity = connectivity.tolil() # Compute the number of nodes n_connected_components, labels = connected_components(connectivity) if n_connected_components > 1: - warnings.warn("the number of connected components of the " - "connectivity matrix is %d > 1. Completing it to avoid " - "stopping the tree early." % n_connected_components, - stacklevel=2) + warnings.warn( + "the number of connected components of the " + "connectivity matrix is %d > 1. Completing it to avoid " + "stopping the tree early." % n_connected_components, + stacklevel=2, + ) # XXX: Can we do without completing the matrix? - for i in range(n_connected_components): - idx_i = np.where(labels == i)[0] - Xi = X[idx_i] - for j in range(i): - idx_j = np.where(labels == j)[0] - Xj = X[idx_j] - D = pairwise_distances(Xi, Xj, metric=affinity) - ii, jj = np.where(D == np.min(D)) - ii = ii[0] - jj = jj[0] - connectivity[idx_i[ii], idx_j[jj]] = True - connectivity[idx_j[jj], idx_i[ii]] = True + connectivity = _fix_connected_components( + X=X, + graph=connectivity, + n_connected_components=n_connected_components, + component_labels=labels, + metric=affinity, + mode="connectivity", + ) return connectivity, n_connected_components -def _single_linkage_tree(connectivity, n_samples, n_nodes, n_clusters, - n_connected_components, return_distance): +def _single_linkage_tree( + connectivity, + n_samples, + n_nodes, + n_clusters, + n_connected_components, + return_distance, +): """ Perform single linkage clustering on sparse data via the minimum spanning tree from scipy.sparse.csgraph, then using union-find to label. @@ -90,8 +135,7 @@ def _single_linkage_tree(connectivity, n_samples, n_nodes, n_clusters, from scipy.sparse.csgraph import minimum_spanning_tree # explicitly cast connectivity to ensure safety - connectivity = connectivity.astype('float64', - **_astype_copy_false(connectivity)) + connectivity = connectivity.astype(np.float64, copy=False) # Ensure zero distances aren't ignored by setting them to "epsilon" epsilon_value = np.finfo(dtype=connectivity.data.dtype).eps @@ -109,7 +153,7 @@ def _single_linkage_tree(connectivity, n_samples, n_nodes, n_clusters, mst_array = np.vstack([mst.row, mst.col, mst.data]).T # Sort edges of the min_spanning_tree by weight - mst_array = mst_array[np.argsort(mst_array.T[2], kind='mergesort'), :] + mst_array = mst_array[np.argsort(mst_array.T[2], kind="mergesort"), :] # Convert edge list into standard hierarchical clustering format single_linkage_tree = _hierarchical._single_linkage_label(mst_array) @@ -134,7 +178,16 @@ def _single_linkage_tree(connectivity, n_samples, n_nodes, n_clusters, ############################################################################### # Hierarchical tree building functions -@_deprecate_positional_args + +@validate_params( + { + "X": ["array-like"], + "connectivity": ["array-like", "sparse matrix", None], + "n_clusters": [Interval(Integral, 1, None, closed="left"), None], + "return_distance": ["boolean"], + }, + prefer_skip_nested_validation=True, +) def ward_tree(X, *, connectivity=None, n_clusters=None, return_distance=False): """Ward clustering based on a Feature matrix. @@ -151,24 +204,25 @@ def ward_tree(X, *, connectivity=None, n_clusters=None, return_distance=False): Parameters ---------- X : array-like of shape (n_samples, n_features) - feature matrix representing n_samples samples to be clustered + Feature matrix representing `n_samples` samples to be clustered. - connectivity : sparse matrix, default=None - connectivity matrix. Defines for each sample the neighboring samples + connectivity : {array-like, sparse matrix}, default=None + Connectivity matrix. Defines for each sample the neighboring samples following a given structure of the data. The matrix is assumed to be symmetric and only the upper triangular half is used. Default is None, i.e, the Ward algorithm is unstructured. n_clusters : int, default=None - Stop early the construction of the tree at n_clusters. This is - useful to decrease computation time if the number of clusters is - not small compared to the number of samples. In this case, the - complete tree is not computed, thus the 'children' output is of - limited use, and the 'parents' output should rather be used. - This option is valid only when specifying a connectivity matrix. + `n_clusters` should be less than `n_samples`. Stop early the + construction of the tree at `n_clusters.` This is useful to decrease + computation time if the number of clusters is not small compared to the + number of samples. In this case, the complete tree is not computed, thus + the 'children' output is of limited use, and the 'parents' output should + rather be used. This option is valid only when specifying a connectivity + matrix. - return_distance : bool, default=None - If True, return the distance between the clusters. + return_distance : bool, default=False + If `True`, return the distance between the clusters. Returns ------- @@ -178,24 +232,24 @@ def ward_tree(X, *, connectivity=None, n_clusters=None, return_distance=False): A node `i` greater than or equal to `n_samples` is a non-leaf node and has children `children_[i - n_samples]`. Alternatively at the i-th iteration, children[i][0] and children[i][1] - are merged to form node `n_samples + i` + are merged to form node `n_samples + i`. n_connected_components : int The number of connected components in the graph. n_leaves : int - The number of leaves in the tree + The number of leaves in the tree. parents : ndarray of shape (n_nodes,) or None The parent of each node. Only returned when a connectivity matrix is specified, elsewhere 'None' is returned. distances : ndarray of shape (n_nodes-1,) - Only returned if return_distance is set to True (for compatibility). + Only returned if `return_distance` is set to `True` (for compatibility). The distances between the centers of the nodes. `distances[i]` - corresponds to a weighted euclidean distance between + corresponds to a weighted Euclidean distance between the nodes `children[i, 1]` and `children[i, 2]`. If the nodes refer to - leaves of the tree, then `distances[i]` is their unweighted euclidean + leaves of the tree, then `distances[i]` is their unweighted Euclidean distance. Distances are updated in the following way (from scipy.hierarchy.linkage): @@ -215,6 +269,24 @@ def ward_tree(X, *, connectivity=None, n_clusters=None, return_distance=False): cluster in the forest, :math:`T=|v|+|s|+|t|`, and :math:`|*|` is the cardinality of its argument. This is also known as the incremental algorithm. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.cluster import ward_tree + >>> X = np.array([[1, 2], [1, 4], [1, 0], + ... [4, 2], [4, 4], [4, 0]]) + >>> children, n_connected_components, n_leaves, parents = ward_tree(X) + >>> children + array([[0, 1], + [3, 5], + [2, 6], + [4, 7], + [8, 9]]) + >>> n_connected_components + 1 + >>> n_leaves + 6 """ X = np.asarray(X) if X.ndim == 1: @@ -225,13 +297,17 @@ def ward_tree(X, *, connectivity=None, n_clusters=None, return_distance=False): from scipy.cluster import hierarchy # imports PIL if n_clusters is not None: - warnings.warn('Partial build of the tree is implemented ' - 'only for structured clustering (i.e. with ' - 'explicit connectivity). The algorithm ' - 'will build the full tree and only ' - 'retain the lower branches required ' - 'for the specified number of clusters', - stacklevel=2) + warnings.warn( + ( + "Partial build of the tree is implemented " + "only for structured clustering (i.e. with " + "explicit connectivity). The algorithm " + "will build the full tree and only " + "retain the lower branches required " + "for the specified number of clusters" + ), + stacklevel=2, + ) X = np.require(X, requirements="W") out = hierarchy.ward(X) children_ = out[:, :2].astype(np.intp) @@ -243,15 +319,17 @@ def ward_tree(X, *, connectivity=None, n_clusters=None, return_distance=False): return children_, 1, n_samples, None connectivity, n_connected_components = _fix_connectivity( - X, connectivity, - affinity='euclidean') + X, connectivity, affinity="euclidean" + ) if n_clusters is None: n_nodes = 2 * n_samples - 1 else: if n_clusters > n_samples: - raise ValueError('Cannot provide more clusters than samples. ' - '%i n_clusters was asked, and there are %i ' - 'samples.' % (n_clusters, n_samples)) + raise ValueError( + "Cannot provide more clusters than samples. " + "%i n_clusters was asked, and there are %i " + "samples." % (n_clusters, n_samples) + ) n_nodes = 2 * n_samples - n_clusters # create inertia matrix @@ -263,20 +341,24 @@ def ward_tree(X, *, connectivity=None, n_clusters=None, return_distance=False): # We keep only the upper triangular for the moments # Generator expressions are faster than arrays on the following row = [i for i in row if i < ind] - coord_row.extend(len(row) * [ind, ]) + coord_row.extend( + len(row) + * [ + ind, + ] + ) coord_col.extend(row) - coord_row = np.array(coord_row, dtype=np.intp, order='C') - coord_col = np.array(coord_col, dtype=np.intp, order='C') + coord_row = np.array(coord_row, dtype=np.intp, order="C") + coord_col = np.array(coord_col, dtype=np.intp, order="C") # build moments as a list - moments_1 = np.zeros(n_nodes, order='C') + moments_1 = np.zeros(n_nodes, order="C") moments_1[:n_samples] = 1 - moments_2 = np.zeros((n_nodes, n_features), order='C') + moments_2 = np.zeros((n_nodes, n_features), order="C") moments_2[:n_samples] = X - inertia = np.empty(len(coord_row), dtype=np.float64, order='C') - _hierarchical.compute_ward_dist(moments_1, moments_2, coord_row, coord_col, - inertia) + inertia = np.empty(len(coord_row), dtype=np.float64, order="C") + _hierarchical.compute_ward_dist(moments_1, moments_2, coord_row, coord_col, inertia) inertia = list(zip(inertia, coord_row, coord_col)) heapify(inertia) @@ -287,7 +369,7 @@ def ward_tree(X, *, connectivity=None, n_clusters=None, return_distance=False): if return_distance: distances = np.empty(n_nodes - n_samples) - not_visited = np.empty(n_nodes, dtype=np.int8, order='C') + not_visited = np.empty(n_nodes, dtype=bool, order="C") # recursive merge loop for k in range(n_samples, n_nodes): @@ -315,18 +397,16 @@ def ward_tree(X, *, connectivity=None, n_clusters=None, return_distance=False): # List comprehension is faster than a for loop [A[col].append(k) for col in coord_col] A.append(coord_col) - coord_col = np.array(coord_col, dtype=np.intp, order='C') - coord_row = np.empty(coord_col.shape, dtype=np.intp, order='C') + coord_col = np.array(coord_col, dtype=np.intp, order="C") + coord_row = np.empty(coord_col.shape, dtype=np.intp, order="C") coord_row.fill(k) n_additions = len(coord_row) - ini = np.empty(n_additions, dtype=np.float64, order='C') + ini = np.empty(n_additions, dtype=np.float64, order="C") - _hierarchical.compute_ward_dist(moments_1, moments_2, - coord_row, coord_col, ini) + _hierarchical.compute_ward_dist(moments_1, moments_2, coord_row, coord_col, ini) # List comprehension is faster than a for loop - [heappush(inertia, (ini[idx], k, coord_col[idx])) - for idx in range(n_additions)] + [heappush(inertia, (ini[idx], k, coord_col[idx])) for idx in range(n_additions)] # Separate leaves in children (empty lists up to now) n_leaves = n_samples @@ -336,15 +416,21 @@ def ward_tree(X, *, connectivity=None, n_clusters=None, return_distance=False): if return_distance: # 2 is scaling factor to compare w/ unstructured version - distances = np.sqrt(2. * distances) + distances = np.sqrt(2.0 * distances) return children, n_connected_components, n_leaves, parent, distances else: return children, n_connected_components, n_leaves, parent # single average and complete linkage -def linkage_tree(X, connectivity=None, n_clusters=None, linkage='complete', - affinity="euclidean", return_distance=False): +def linkage_tree( + X, + connectivity=None, + n_clusters=None, + linkage="complete", + affinity="euclidean", + return_distance=False, +): """Linkage agglomerative clustering based on a Feature matrix. The inertia matrix uses a Heapq-based representation. @@ -357,16 +443,16 @@ def linkage_tree(X, connectivity=None, n_clusters=None, linkage='complete', Parameters ---------- X : array-like of shape (n_samples, n_features) - feature matrix representing n_samples samples to be clustered + Feature matrix representing `n_samples` samples to be clustered. connectivity : sparse matrix, default=None - connectivity matrix. Defines for each sample the neighboring samples + Connectivity matrix. Defines for each sample the neighboring samples following a given structure of the data. The matrix is assumed to be symmetric and only the upper triangular half is used. - Default is None, i.e, the Ward algorithm is unstructured. + Default is `None`, i.e, the Ward algorithm is unstructured. n_clusters : int, default=None - Stop early the construction of the tree at n_clusters. This is + Stop early the construction of the tree at `n_clusters`. This is useful to decrease computation time if the number of clusters is not small compared to the number of samples. In this case, the complete tree is not computed, thus the 'children' output is of @@ -376,19 +462,19 @@ def linkage_tree(X, connectivity=None, n_clusters=None, linkage='complete', linkage : {"average", "complete", "single"}, default="complete" Which linkage criteria to use. The linkage criterion determines which distance to use between sets of observation. - - average uses the average of the distances of each observation of - the two sets - - complete or maximum linkage uses the maximum distances between + - "average" uses the average of the distances of each observation of + the two sets. + - "complete" or maximum linkage uses the maximum distances between all observations of the two sets. - - single uses the minimum of the distances between all observations - of the two sets. + - "single" uses the minimum of the distances between all + observations of the two sets. - affinity : str or callable, default="euclidean". - which metric to use. Can be "euclidean", "manhattan", or any - distance know to paired distance (see metric.pairwise) + affinity : str or callable, default='euclidean' + Which metric to use. Can be 'euclidean', 'manhattan', or any + distance known to paired distance (see metric.pairwise). return_distance : bool, default=False - whether or not to return the distances between the clusters. + Whether or not to return the distances between the clusters. Returns ------- @@ -398,7 +484,7 @@ def linkage_tree(X, connectivity=None, n_clusters=None, linkage='complete', A node `i` greater than or equal to `n_samples` is a non-leaf node and has children `children_[i - n_samples]`. Alternatively at the i-th iteration, children[i][0] and children[i][1] - are merged to form node `n_samples + i` + are merged to form node `n_samples + i`. n_connected_components : int The number of connected components in the graph. @@ -411,7 +497,7 @@ def linkage_tree(X, connectivity=None, n_clusters=None, linkage='complete', is specified, elsewhere 'None' is returned. distances : ndarray of shape (n_nodes-1,) - Returned when return_distance is set to True. + Returned when `return_distance` is set to `True`. distances[i] refers to the distance between children[i][0] and children[i][1] when they are merged. @@ -425,58 +511,63 @@ def linkage_tree(X, connectivity=None, n_clusters=None, linkage='complete', X = np.reshape(X, (-1, 1)) n_samples, n_features = X.shape - linkage_choices = {'complete': _hierarchical.max_merge, - 'average': _hierarchical.average_merge, - 'single': None} # Single linkage is handled differently + linkage_choices = { + "complete": _hierarchical.max_merge, + "average": _hierarchical.average_merge, + "single": None, + } # Single linkage is handled differently try: join_func = linkage_choices[linkage] except KeyError as e: raise ValueError( - 'Unknown linkage option, linkage should be one ' - 'of %s, but %s was given' % (linkage_choices.keys(), linkage) + "Unknown linkage option, linkage should be one of %s, but %s was given" + % (linkage_choices.keys(), linkage) ) from e - if affinity == 'cosine' and np.any(~np.any(X, axis=1)): - raise ValueError( - 'Cosine affinity cannot be used when X contains zero vectors') + if affinity == "cosine" and np.any(~np.any(X, axis=1)): + raise ValueError("Cosine affinity cannot be used when X contains zero vectors") if connectivity is None: from scipy.cluster import hierarchy # imports PIL if n_clusters is not None: - warnings.warn('Partial build of the tree is implemented ' - 'only for structured clustering (i.e. with ' - 'explicit connectivity). The algorithm ' - 'will build the full tree and only ' - 'retain the lower branches required ' - 'for the specified number of clusters', - stacklevel=2) - - if affinity == 'precomputed': + warnings.warn( + ( + "Partial build of the tree is implemented " + "only for structured clustering (i.e. with " + "explicit connectivity). The algorithm " + "will build the full tree and only " + "retain the lower branches required " + "for the specified number of clusters" + ), + stacklevel=2, + ) + + if affinity == "precomputed": # for the linkage function of hierarchy to work on precomputed # data, provide as first argument an ndarray of the shape returned # by sklearn.metrics.pairwise_distances. if X.shape[0] != X.shape[1]: raise ValueError( - 'Distance matrix should be square, ' - 'Got matrix of shape {X.shape}' + f"Distance matrix should be square, got matrix of shape {X.shape}" ) i, j = np.triu_indices(X.shape[0], k=1) X = X[i, j] - elif affinity == 'l2': + elif affinity == "l2": # Translate to something understood by scipy - affinity = 'euclidean' - elif affinity in ('l1', 'manhattan'): - affinity = 'cityblock' + affinity = "euclidean" + elif affinity in ("l1", "manhattan"): + affinity = "cityblock" elif callable(affinity): X = affinity(X) i, j = np.triu_indices(X.shape[0], k=1) X = X[i, j] - if (linkage == 'single' - and affinity != 'precomputed' - and not callable(affinity) - and affinity in METRIC_MAPPING): - + if ( + linkage == "single" + and affinity != "precomputed" + and not callable(affinity) + and affinity in METRIC_MAPPING64 + ): # We need the fast cythonized metric from neighbors dist_metric = DistanceMetric.get_metric(affinity) @@ -485,7 +576,7 @@ def linkage_tree(X, connectivity=None, n_clusters=None, linkage='complete', mst = _hierarchical.mst_linkage_core(X, dist_metric) # Sort edges of the min_spanning_tree by weight - mst = mst[np.argsort(mst.T[2], kind='mergesort'), :] + mst = mst[np.argsort(mst.T[2], kind="mergesort"), :] # Convert edge list into standard hierarchical clustering format out = _hierarchical.single_linkage_label(mst) @@ -499,25 +590,24 @@ def linkage_tree(X, connectivity=None, n_clusters=None, linkage='complete', return children_, 1, n_samples, None connectivity, n_connected_components = _fix_connectivity( - X, connectivity, - affinity=affinity) + X, connectivity, affinity=affinity + ) connectivity = connectivity.tocoo() # Put the diagonal to zero - diag_mask = (connectivity.row != connectivity.col) + diag_mask = connectivity.row != connectivity.col connectivity.row = connectivity.row[diag_mask] connectivity.col = connectivity.col[diag_mask] connectivity.data = connectivity.data[diag_mask] del diag_mask - if affinity == 'precomputed': - distances = X[connectivity.row, connectivity.col].astype( - 'float64', **_astype_copy_false(X)) + if affinity == "precomputed": + distances = X[connectivity.row, connectivity.col].astype(np.float64, copy=False) else: # FIXME We compute all the distances, while we could have only computed # the "interesting" distances - distances = paired_distances(X[connectivity.row], - X[connectivity.col], - metric=affinity) + distances = paired_distances( + X[connectivity.row], X[connectivity.col], metric=affinity + ) connectivity.data = distances if n_clusters is None: @@ -526,10 +616,15 @@ def linkage_tree(X, connectivity=None, n_clusters=None, linkage='complete', assert n_clusters <= n_samples n_nodes = 2 * n_samples - n_clusters - if linkage == 'single': - return _single_linkage_tree(connectivity, n_samples, n_nodes, - n_clusters, n_connected_components, - return_distance) + if linkage == "single": + return _single_linkage_tree( + connectivity, + n_samples, + n_nodes, + n_clusters, + n_connected_components, + return_distance, + ) if return_distance: distances = np.empty(n_nodes - n_samples) @@ -541,14 +636,15 @@ def linkage_tree(X, connectivity=None, n_clusters=None, linkage='complete', # without the numpy overhead of slicing CSR indices and data. connectivity = connectivity.tolil() # We are storing the graph in a list of IntFloatDict - for ind, (data, row) in enumerate(zip(connectivity.data, - connectivity.rows)): - A[ind] = IntFloatDict(np.asarray(row, dtype=np.intp), - np.asarray(data, dtype=np.float64)) + for ind, (data, row) in enumerate(zip(connectivity.data, connectivity.rows)): + A[ind] = IntFloatDict( + np.asarray(row, dtype=np.intp), np.asarray(data, dtype=np.float64) + ) # We keep only the upper triangular for the heap # Generator expressions are faster than arrays on the following - inertia.extend(_hierarchical.WeightedEdge(d, ind, r) - for r, d in zip(row, data) if r < ind) + inertia.extend( + _hierarchical.WeightedEdge(d, ind, r) for r, d in zip(row, data) if r < ind + ) del connectivity heapify(inertia) @@ -605,17 +701,17 @@ def linkage_tree(X, connectivity=None, n_clusters=None, linkage='complete', # Matching names to tree-building strategies def _complete_linkage(*args, **kwargs): - kwargs['linkage'] = 'complete' + kwargs["linkage"] = "complete" return linkage_tree(*args, **kwargs) def _average_linkage(*args, **kwargs): - kwargs['linkage'] = 'average' + kwargs["linkage"] = "average" return linkage_tree(*args, **kwargs) def _single_linkage(*args, **kwargs): - kwargs['linkage'] = 'single' + kwargs["linkage"] = "single" return linkage_tree(*args, **kwargs) @@ -623,12 +719,13 @@ def _single_linkage(*args, **kwargs): ward=ward_tree, complete=_complete_linkage, average=_average_linkage, - single=_single_linkage) - + single=_single_linkage, +) ############################################################################### # Functions for cutting hierarchical clustering tree + def _hc_cut(n_clusters, children, n_leaves): """Function cutting the ward tree for a given number of clusters. @@ -643,7 +740,7 @@ def _hc_cut(n_clusters, children, n_leaves): A node `i` greater than or equal to `n_samples` is a non-leaf node and has children `children_[i - n_samples]`. Alternatively at the i-th iteration, children[i][0] and children[i][1] - are merged to form node `n_samples + i` + are merged to form node `n_samples + i`. n_leaves : int Number of leaves of the tree. @@ -651,13 +748,13 @@ def _hc_cut(n_clusters, children, n_leaves): Returns ------- labels : array [n_samples] - cluster labels for each point - + Cluster labels for each point. """ if n_clusters > n_leaves: - raise ValueError('Cannot extract more clusters than samples: ' - '%s clusters where given for a tree with %s leaves.' - % (n_clusters, n_leaves)) + raise ValueError( + "Cannot extract more clusters than samples: " + f"{n_clusters} clusters were given for a tree with {n_leaves} leaves." + ) # In this function, we store nodes as a heap to avoid recomputing # the max of the nodes: the first element is always the smallest # We use negated indices as heaps work on smallest elements, and we @@ -678,12 +775,12 @@ def _hc_cut(n_clusters, children, n_leaves): ############################################################################### + class AgglomerativeClustering(ClusterMixin, BaseEstimator): """ - Agglomerative Clustering + Agglomerative Clustering. - Recursively merges the pair of clusters that minimally increases - a given linkage distance. + Recursively merges pair of clusters of sample data; uses linkage distance. Read more in the :ref:`User Guide `. @@ -693,26 +790,36 @@ class AgglomerativeClustering(ClusterMixin, BaseEstimator): The number of clusters to find. It must be ``None`` if ``distance_threshold`` is not ``None``. - affinity : str or callable, default='euclidean' + metric : str or callable, default="euclidean" Metric used to compute the linkage. Can be "euclidean", "l1", "l2", - "manhattan", "cosine", or "precomputed". - If linkage is "ward", only "euclidean" is accepted. - If "precomputed", a distance matrix (instead of a similarity matrix) - is needed as input for the fit method. + "manhattan", "cosine", or "precomputed". If linkage is "ward", only + "euclidean" is accepted. If "precomputed", a distance matrix is needed + as input for the fit method. If connectivity is None, linkage is + "single" and affinity is not "precomputed" any valid pairwise distance + metric can be assigned. + + For an example of agglomerative clustering with different metrics, see + :ref:`sphx_glr_auto_examples_cluster_plot_agglomerative_clustering_metrics.py`. + + .. versionadded:: 1.2 memory : str or object with the joblib.Memory interface, default=None Used to cache the output of the computation of the tree. By default, no caching is done. If a string is given, it is the path to the caching directory. - connectivity : array-like or callable, default=None + connectivity : array-like, sparse matrix, or callable, default=None Connectivity matrix. Defines for each sample the neighboring samples following a given structure of the data. This can be a connectivity matrix itself or a callable that transforms the data into a connectivity matrix, such as derived from - kneighbors_graph. Default is ``None``, i.e, the + `kneighbors_graph`. Default is ``None``, i.e, the hierarchical clustering algorithm is unstructured. + For an example of connectivity matrix using + :class:`~sklearn.neighbors.kneighbors_graph`, see + :ref:`sphx_glr_auto_examples_cluster_plot_agglomerative_clustering.py`. + compute_full_tree : 'auto' or bool, default='auto' Stop early the construction of the tree at ``n_clusters``. This is useful to decrease computation time if the number of clusters is not @@ -741,8 +848,11 @@ class AgglomerativeClustering(ClusterMixin, BaseEstimator): .. versionadded:: 0.20 Added the 'single' option + For examples comparing different `linkage` criteria, see + :ref:`sphx_glr_auto_examples_cluster_plot_linkage_comparison.py`. + distance_threshold : float, default=None - The linkage distance threshold above which, clusters will not be + The linkage distance threshold at or above which clusters will not be merged. If not ``None``, ``n_clusters`` must be ``None`` and ``compute_full_tree`` must be ``True``. @@ -755,6 +865,9 @@ class AgglomerativeClustering(ClusterMixin, BaseEstimator): .. versionadded:: 0.24 + For an example of dendrogram visualization, see + :ref:`sphx_glr_auto_examples_cluster_plot_agglomerative_dendrogram.py`. + Attributes ---------- n_clusters_ : int @@ -763,7 +876,7 @@ class AgglomerativeClustering(ClusterMixin, BaseEstimator): ``n_clusters``. labels_ : ndarray of shape (n_samples) - cluster labels for each point + Cluster labels for each point. n_leaves_ : int Number of leaves in the hierarchical tree. @@ -774,19 +887,36 @@ class AgglomerativeClustering(ClusterMixin, BaseEstimator): .. versionadded:: 0.21 ``n_connected_components_`` was added to replace ``n_components_``. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + children_ : array-like of shape (n_samples-1, 2) The children of each non-leaf node. Values less than `n_samples` correspond to leaves of the tree which are the original samples. A node `i` greater than or equal to `n_samples` is a non-leaf node and has children `children_[i - n_samples]`. Alternatively at the i-th iteration, children[i][0] and children[i][1] - are merged to form node `n_samples + i` + are merged to form node `n_samples + i`. distances_ : array-like of shape (n_nodes-1,) Distances between nodes in the corresponding place in `children_`. Only computed if `distance_threshold` is used or `compute_distances` is set to `True`. + See Also + -------- + FeatureAgglomeration : Agglomerative clustering but for features instead of + samples. + ward_tree : Hierarchical clustering with ward linkage. + Examples -------- >>> from sklearn.cluster import AgglomerativeClustering @@ -799,64 +929,101 @@ class AgglomerativeClustering(ClusterMixin, BaseEstimator): >>> clustering.labels_ array([1, 1, 1, 0, 0, 0]) + For a comparison of Agglomerative clustering with other clustering algorithms, see + :ref:`sphx_glr_auto_examples_cluster_plot_cluster_comparison.py` """ - @_deprecate_positional_args - def __init__(self, n_clusters=2, *, affinity="euclidean", - memory=None, - connectivity=None, compute_full_tree='auto', - linkage='ward', distance_threshold=None, - compute_distances=False): + + _parameter_constraints: dict = { + "n_clusters": [Interval(Integral, 1, None, closed="left"), None], + "metric": [ + StrOptions(set(_VALID_METRICS) | {"precomputed"}), + callable, + ], + "memory": [str, HasMethods("cache"), None], + "connectivity": ["array-like", "sparse matrix", callable, None], + "compute_full_tree": [StrOptions({"auto"}), "boolean"], + "linkage": [StrOptions(set(_TREE_BUILDERS.keys()))], + "distance_threshold": [Interval(Real, 0, None, closed="left"), None], + "compute_distances": ["boolean"], + } + + def __init__( + self, + n_clusters=2, + *, + metric="euclidean", + memory=None, + connectivity=None, + compute_full_tree="auto", + linkage="ward", + distance_threshold=None, + compute_distances=False, + ): self.n_clusters = n_clusters self.distance_threshold = distance_threshold self.memory = memory self.connectivity = connectivity self.compute_full_tree = compute_full_tree self.linkage = linkage - self.affinity = affinity + self.metric = metric self.compute_distances = compute_distances + @_fit_context(prefer_skip_nested_validation=True) def fit(self, X, y=None): """Fit the hierarchical clustering from features, or distance matrix. Parameters ---------- - X : array-like, shape (n_samples, n_features) or (n_samples, n_samples) + X : array-like, shape (n_samples, n_features) or \ + (n_samples, n_samples) Training instances to cluster, or distances between instances if - ``affinity='precomputed'``. + ``metric='precomputed'``. y : Ignored Not used, present here for API consistency by convention. Returns ------- - self + self : object + Returns the fitted instance. """ - X = self._validate_data(X, ensure_min_samples=2, estimator=self) - memory = check_memory(self.memory) + X = validate_data(self, X, ensure_min_samples=2) + return self._fit(X) - if self.n_clusters is not None and self.n_clusters <= 0: - raise ValueError("n_clusters should be an integer greater than 0." - " %s was provided." % str(self.n_clusters)) + def _fit(self, X): + """Fit without validation + + Parameters + ---------- + X : ndarray of shape (n_samples, n_features) or (n_samples, n_samples) + Training instances to cluster, or distances between instances if + ``metric='precomputed'``. + + Returns + ------- + self : object + Returns the fitted instance. + """ + memory = check_memory(self.memory) if not ((self.n_clusters is None) ^ (self.distance_threshold is None)): - raise ValueError("Exactly one of n_clusters and " - "distance_threshold has to be set, and the other " - "needs to be None.") - - if (self.distance_threshold is not None - and not self.compute_full_tree): - raise ValueError("compute_full_tree must be True if " - "distance_threshold is set.") - - if self.linkage == "ward" and self.affinity != "euclidean": - raise ValueError("%s was provided as affinity. Ward can only " - "work with euclidean distances." % - (self.affinity, )) - - if self.linkage not in _TREE_BUILDERS: - raise ValueError("Unknown linkage type %s. " - "Valid options are %s" % (self.linkage, - _TREE_BUILDERS.keys())) + raise ValueError( + "Exactly one of n_clusters and " + "distance_threshold has to be set, and the other " + "needs to be None." + ) + + if self.distance_threshold is not None and not self.compute_full_tree: + raise ValueError( + "compute_full_tree must be True if distance_threshold is set." + ) + + if self.linkage == "ward" and self.metric != "euclidean": + raise ValueError( + f"{self.metric} was provided as metric. Ward can only " + "work with euclidean distances." + ) + tree_builder = _TREE_BUILDERS[self.linkage] connectivity = self.connectivity @@ -864,58 +1031,59 @@ def fit(self, X, y=None): if callable(self.connectivity): connectivity = self.connectivity(X) connectivity = check_array( - connectivity, accept_sparse=['csr', 'coo', 'lil']) + connectivity, accept_sparse=["csr", "coo", "lil"] + ) n_samples = len(X) compute_full_tree = self.compute_full_tree if self.connectivity is None: compute_full_tree = True - if compute_full_tree == 'auto': + if compute_full_tree == "auto": if self.distance_threshold is not None: compute_full_tree = True else: # Early stopping is likely to give a speed up only for # a large number of clusters. The actual threshold # implemented here is heuristic - compute_full_tree = self.n_clusters < max(100, .02 * n_samples) + compute_full_tree = self.n_clusters < max(100, 0.02 * n_samples) n_clusters = self.n_clusters if compute_full_tree: n_clusters = None # Construct the tree kwargs = {} - if self.linkage != 'ward': - kwargs['linkage'] = self.linkage - kwargs['affinity'] = self.affinity + if self.linkage != "ward": + kwargs["linkage"] = self.linkage + kwargs["affinity"] = self.metric distance_threshold = self.distance_threshold - return_distance = ( - (distance_threshold is not None) or self.compute_distances - ) + return_distance = (distance_threshold is not None) or self.compute_distances - out = memory.cache(tree_builder)(X, connectivity=connectivity, - n_clusters=n_clusters, - return_distance=return_distance, - **kwargs) - (self.children_, - self.n_connected_components_, - self.n_leaves_, - parents) = out[:4] + out = memory.cache(tree_builder)( + X, + connectivity=connectivity, + n_clusters=n_clusters, + return_distance=return_distance, + **kwargs, + ) + (self.children_, self.n_connected_components_, self.n_leaves_, parents) = out[ + :4 + ] if return_distance: self.distances_ = out[-1] if self.distance_threshold is not None: # distance_threshold is used - self.n_clusters_ = np.count_nonzero( - self.distances_ >= distance_threshold) + 1 + self.n_clusters_ = ( + np.count_nonzero(self.distances_ >= distance_threshold) + 1 + ) else: # n_clusters is used self.n_clusters_ = self.n_clusters # Cut the tree if compute_full_tree: - self.labels_ = _hc_cut(self.n_clusters_, self.children_, - self.n_leaves_) + self.labels_ = _hc_cut(self.n_clusters_, self.children_, self.n_leaves_) else: labels = _hierarchical.hc_get_heads(parents, copy=False) # copy to avoid holding a reference on the original array @@ -925,8 +1093,10 @@ def fit(self, X, y=None): return self def fit_predict(self, X, y=None): - """Fit the hierarchical clustering from features or distance matrix, - and return cluster labels. + """Fit and return the result of each sample's clustering assignment. + + In addition to fitting, this method also return the result of the + clustering assignment for each sample in the training set. Parameters ---------- @@ -946,40 +1116,49 @@ def fit_predict(self, X, y=None): return super().fit_predict(X, y) -class FeatureAgglomeration(AgglomerativeClustering, AgglomerationTransform): +class FeatureAgglomeration( + ClassNamePrefixFeaturesOutMixin, AgglomerationTransform, AgglomerativeClustering +): """Agglomerate features. - Similar to AgglomerativeClustering, but recursively merges features - instead of samples. + Recursively merges pair of clusters of features. + + Refer to + :ref:`sphx_glr_auto_examples_cluster_plot_feature_agglomeration_vs_univariate_selection.py` + for an example comparison of :class:`FeatureAgglomeration` strategy with a + univariate feature selection strategy (based on ANOVA). Read more in the :ref:`User Guide `. Parameters ---------- - n_clusters : int, default=2 + n_clusters : int or None, default=2 The number of clusters to find. It must be ``None`` if ``distance_threshold`` is not ``None``. - affinity : str or callable, default='euclidean' + metric : str or callable, default="euclidean" Metric used to compute the linkage. Can be "euclidean", "l1", "l2", - "manhattan", "cosine", or 'precomputed'. - If linkage is "ward", only "euclidean" is accepted. + "manhattan", "cosine", or "precomputed". If linkage is "ward", only + "euclidean" is accepted. If "precomputed", a distance matrix is needed + as input for the fit method. + + .. versionadded:: 1.2 memory : str or object with the joblib.Memory interface, default=None Used to cache the output of the computation of the tree. By default, no caching is done. If a string is given, it is the path to the caching directory. - connectivity : array-like or callable, default=None + connectivity : array-like, sparse matrix, or callable, default=None Connectivity matrix. Defines for each feature the neighboring features following a given structure of the data. This can be a connectivity matrix itself or a callable that transforms the data into a connectivity matrix, such as derived from - kneighbors_graph. Default is None, i.e, the + `kneighbors_graph`. Default is `None`, i.e, the hierarchical clustering algorithm is unstructured. compute_full_tree : 'auto' or bool, default='auto' - Stop early the construction of the tree at n_clusters. This is useful + Stop early the construction of the tree at `n_clusters`. This is useful to decrease computation time if the number of clusters is not small compared to the number of features. This option is useful only when specifying a connectivity matrix. Note also that when varying the @@ -990,17 +1169,17 @@ class FeatureAgglomeration(AgglomerativeClustering, AgglomerationTransform): is inferior to the maximum between 100 or `0.02 * n_samples`. Otherwise, "auto" is equivalent to `False`. - linkage : {'ward', 'complete', 'average', 'single'}, default='ward' + linkage : {"ward", "complete", "average", "single"}, default="ward" Which linkage criterion to use. The linkage criterion determines which distance to use between sets of features. The algorithm will merge the pairs of cluster that minimize this criterion. - - ward minimizes the variance of the clusters being merged. - - average uses the average of the distances of each feature of - the two sets. - - complete or maximum linkage uses the maximum distances between + - "ward" minimizes the variance of the clusters being merged. + - "complete" or maximum linkage uses the maximum distances between all features of the two sets. - - single uses the minimum of the distances between all observations + - "average" uses the average of the distances of each feature of + the two sets. + - "single" uses the minimum of the distances between all features of the two sets. pooling_func : callable, default=np.mean @@ -1009,7 +1188,7 @@ class FeatureAgglomeration(AgglomerativeClustering, AgglomerationTransform): argument `axis=1`, and reduce it to an array of size [M]. distance_threshold : float, default=None - The linkage distance threshold above which, clusters will not be + The linkage distance threshold at or above which clusters will not be merged. If not ``None``, ``n_clusters`` must be ``None`` and ``compute_full_tree`` must be ``True``. @@ -1030,7 +1209,7 @@ class FeatureAgglomeration(AgglomerativeClustering, AgglomerationTransform): ``n_clusters``. labels_ : array-like of (n_features,) - cluster labels for each feature. + Cluster labels for each feature. n_leaves_ : int Number of leaves in the hierarchical tree. @@ -1041,19 +1220,36 @@ class FeatureAgglomeration(AgglomerativeClustering, AgglomerationTransform): .. versionadded:: 0.21 ``n_connected_components_`` was added to replace ``n_components_``. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + children_ : array-like of shape (n_nodes-1, 2) The children of each non-leaf node. Values less than `n_features` correspond to leaves of the tree which are the original samples. A node `i` greater than or equal to `n_features` is a non-leaf node and has children `children_[i - n_features]`. Alternatively at the i-th iteration, children[i][0] and children[i][1] - are merged to form node `n_features + i` + are merged to form node `n_features + i`. distances_ : array-like of shape (n_nodes-1,) Distances between nodes in the corresponding place in `children_`. Only computed if `distance_threshold` is used or `compute_distances` is set to `True`. + See Also + -------- + AgglomerativeClustering : Agglomerative clustering samples instead of + features. + ward_tree : Hierarchical clustering with ward linkage. + Examples -------- >>> import numpy as np @@ -1068,42 +1264,70 @@ class FeatureAgglomeration(AgglomerativeClustering, AgglomerationTransform): >>> X_reduced.shape (1797, 32) """ - @_deprecate_positional_args - def __init__(self, n_clusters=2, *, affinity="euclidean", - memory=None, - connectivity=None, compute_full_tree='auto', - linkage='ward', pooling_func=np.mean, - distance_threshold=None, compute_distances=False): + + _parameter_constraints: dict = { + "n_clusters": [Interval(Integral, 1, None, closed="left"), None], + "metric": [ + StrOptions(set(_VALID_METRICS) | {"precomputed"}), + callable, + ], + "memory": [str, HasMethods("cache"), None], + "connectivity": ["array-like", "sparse matrix", callable, None], + "compute_full_tree": [StrOptions({"auto"}), "boolean"], + "linkage": [StrOptions(set(_TREE_BUILDERS.keys()))], + "pooling_func": [callable], + "distance_threshold": [Interval(Real, 0, None, closed="left"), None], + "compute_distances": ["boolean"], + } + + def __init__( + self, + n_clusters=2, + *, + metric="euclidean", + memory=None, + connectivity=None, + compute_full_tree="auto", + linkage="ward", + pooling_func=np.mean, + distance_threshold=None, + compute_distances=False, + ): super().__init__( - n_clusters=n_clusters, memory=memory, connectivity=connectivity, - compute_full_tree=compute_full_tree, linkage=linkage, - affinity=affinity, distance_threshold=distance_threshold, - compute_distances=compute_distances) + n_clusters=n_clusters, + memory=memory, + connectivity=connectivity, + compute_full_tree=compute_full_tree, + linkage=linkage, + metric=metric, + distance_threshold=distance_threshold, + compute_distances=compute_distances, + ) self.pooling_func = pooling_func - def fit(self, X, y=None, **params): - """Fit the hierarchical clustering on the data + @_fit_context(prefer_skip_nested_validation=True) + def fit(self, X, y=None): + """Fit the hierarchical clustering on the data. Parameters ---------- X : array-like of shape (n_samples, n_features) - The data + The data. y : Ignored + Not used, present here for API consistency by convention. Returns ------- - self + self : object + Returns the transformer. """ - X = self._validate_data(X, accept_sparse=['csr', 'csc', 'coo'], - ensure_min_features=2, estimator=self) - # save n_features_in_ attribute here to reset it after, because it will - # be overridden in AgglomerativeClustering since we passed it X.T. - n_features_in_ = self.n_features_in_ - AgglomerativeClustering.fit(self, X.T, **params) - self.n_features_in_ = n_features_in_ + X = validate_data(self, X, ensure_min_features=2) + super()._fit(X.T) + self._n_features_out = self.n_clusters_ return self @property def fit_predict(self): + """Fit and return the result of each sample's clustering assignment.""" raise AttributeError diff --git a/sklearn/cluster/_bicluster.py b/sklearn/cluster/_bicluster.py index 3bde33399a8e0..04a4e68024d33 100644 --- a/sklearn/cluster/_bicluster.py +++ b/sklearn/cluster/_bicluster.py @@ -1,28 +1,24 @@ """Spectral biclustering algorithms.""" -# Authors : Kemal Eren -# License: BSD 3 clause + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause from abc import ABCMeta, abstractmethod -import warnings +from numbers import Integral import numpy as np - from scipy.linalg import norm from scipy.sparse import dia_matrix, issparse from scipy.sparse.linalg import eigsh, svds -from . import KMeans, MiniBatchKMeans -from ..base import BaseEstimator, BiclusterMixin -from ..utils import check_random_state - -from ..utils.extmath import (make_nonnegative, randomized_svd, - safe_sparse_dot) - -from ..utils.validation import assert_all_finite, _deprecate_positional_args +from ..base import BaseEstimator, BiclusterMixin, _fit_context +from ..utils import check_random_state, check_scalar +from ..utils._param_validation import Interval, StrOptions +from ..utils.extmath import _randomized_svd, make_nonnegative, safe_sparse_dot +from ..utils.validation import assert_all_finite, validate_data +from ._kmeans import KMeans, MiniBatchKMeans - -__all__ = ['SpectralCoclustering', - 'SpectralBiclustering'] +__all__ = ["SpectralBiclustering", "SpectralCoclustering"] def _scale_normalize(X): @@ -30,7 +26,6 @@ def _scale_normalize(X): Returns the normalized matrix and the row and column scaling factors. - """ X = make_nonnegative(X) row_diag = np.asarray(1.0 / np.sqrt(X.sum(axis=1))).squeeze() @@ -41,7 +36,7 @@ def _scale_normalize(X): n_rows, n_cols = X.shape r = dia_matrix((row_diag, [0]), shape=(n_rows, n_rows)) c = dia_matrix((col_diag, [0]), shape=(n_cols, n_cols)) - an = r * X * c + an = r @ X @ c else: an = row_diag[:, np.newaxis] * X * col_diag return an, row_diag, col_diag @@ -51,7 +46,6 @@ def _bistochastic_normalize(X, max_iter=1000, tol=1e-5): """Normalize rows and columns of ``X`` simultaneously so that all rows sum to one constant and all columns sum to a different constant. - """ # According to paper, this can also be done more efficiently with # deviation reduction and balancing algorithms. @@ -73,9 +67,11 @@ def _log_normalize(X): """Normalize ``X`` according to Kluger's log-interactions scheme.""" X = make_nonnegative(X, min_value=1) if issparse(X): - raise ValueError("Cannot compute log of a sparse matrix," - " because log(x) diverges to -infinity as x" - " goes to 0.") + raise ValueError( + "Cannot compute log of a sparse matrix," + " because log(x) diverges to -infinity as x" + " goes to 0." + ) L = np.log(X) row_avg = L.mean(axis=1)[:, np.newaxis] col_avg = L.mean(axis=0) @@ -86,59 +82,73 @@ def _log_normalize(X): class BaseSpectral(BiclusterMixin, BaseEstimator, metaclass=ABCMeta): """Base class for spectral biclustering.""" + _parameter_constraints: dict = { + "svd_method": [StrOptions({"randomized", "arpack"})], + "n_svd_vecs": [Interval(Integral, 0, None, closed="left"), None], + "mini_batch": ["boolean"], + "init": [StrOptions({"k-means++", "random"}), np.ndarray], + "n_init": [Interval(Integral, 1, None, closed="left")], + "random_state": ["random_state"], + } + @abstractmethod - def __init__(self, n_clusters=3, svd_method="randomized", - n_svd_vecs=None, mini_batch=False, init="k-means++", - n_init=10, n_jobs='deprecated', random_state=None): + def __init__( + self, + n_clusters=3, + svd_method="randomized", + n_svd_vecs=None, + mini_batch=False, + init="k-means++", + n_init=10, + random_state=None, + ): self.n_clusters = n_clusters self.svd_method = svd_method self.n_svd_vecs = n_svd_vecs self.mini_batch = mini_batch self.init = init self.n_init = n_init - self.n_jobs = n_jobs self.random_state = random_state - def _check_parameters(self): - legal_svd_methods = ('randomized', 'arpack') - if self.svd_method not in legal_svd_methods: - raise ValueError("Unknown SVD method: '{0}'. svd_method must be" - " one of {1}.".format(self.svd_method, - legal_svd_methods)) + @abstractmethod + def _check_parameters(self, n_samples): + """Validate parameters depending on the input data.""" + @_fit_context(prefer_skip_nested_validation=True) def fit(self, X, y=None): - """Creates a biclustering for X. + """Create a biclustering for X. Parameters ---------- X : array-like of shape (n_samples, n_features) + Training data. y : Ignored + Not used, present for API consistency by convention. + Returns + ------- + self : object + SpectralBiclustering instance. """ - if self.n_jobs != 'deprecated': - warnings.warn("'n_jobs' was deprecated in version 0.23 and will be" - " removed in 1.0 (renaming of 0.25).", FutureWarning) - - X = self._validate_data(X, accept_sparse='csr', dtype=np.float64) - self._check_parameters() + X = validate_data(self, X, accept_sparse="csr", dtype=np.float64) + self._check_parameters(X.shape[0]) self._fit(X) return self def _svd(self, array, n_components, n_discard): """Returns first `n_components` left and right singular vectors u and v, discarding the first `n_discard`. - """ - if self.svd_method == 'randomized': + if self.svd_method == "randomized": kwargs = {} if self.n_svd_vecs is not None: - kwargs['n_oversamples'] = self.n_svd_vecs - u, _, vt = randomized_svd(array, n_components, - random_state=self.random_state, - **kwargs) + kwargs["n_oversamples"] = self.n_svd_vecs + u, _, vt = _randomized_svd( + array, n_components, random_state=self.random_state, **kwargs + ) - elif self.svd_method == 'arpack': + elif self.svd_method == "arpack": u, _, vt = svds(array, k=n_components, ncv=self.n_svd_vecs) if np.any(np.isnan(vt)): # some eigenvalues of A * A.T are negative, causing @@ -165,31 +175,28 @@ def _svd(self, array, n_components, n_discard): def _k_means(self, data, n_clusters): if self.mini_batch: - model = MiniBatchKMeans(n_clusters, - init=self.init, - n_init=self.n_init, - random_state=self.random_state) + model = MiniBatchKMeans( + n_clusters, + init=self.init, + n_init=self.n_init, + random_state=self.random_state, + ) else: - model = KMeans(n_clusters, init=self.init, - n_init=self.n_init, n_jobs=self.n_jobs, - random_state=self.random_state) + model = KMeans( + n_clusters, + init=self.init, + n_init=self.n_init, + random_state=self.random_state, + ) model.fit(data) centroid = model.cluster_centers_ labels = model.labels_ return centroid, labels - def _more_tags(self): - return { - "_xfail_checks": { - "check_estimators_dtypes": "raises nan error", - "check_fit2d_1sample": "_scale_normalize fails", - "check_fit2d_1feature": "raises apply_along_axis error", - "check_estimator_sparse_data": "does not fail gracefully", - "check_methods_subset_invariance": "empty array passed inside", - "check_dont_overwrite_parameters": "empty array passed inside", - "check_fit2d_predict1d": "emptry array passed inside", - } - } + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.sparse = True + return tags class SpectralCoclustering(BaseSpectral): @@ -229,7 +236,7 @@ class SpectralCoclustering(BaseSpectral): Whether to use mini-batch k-means, which is faster but may get different results. - init : {'k-means++', 'random', or ndarray of shape \ + init : {'k-means++', 'random'}, or ndarray of shape \ (n_clusters, n_features), default='k-means++' Method for initialization of k-means algorithm; defaults to 'k-means++'. @@ -242,19 +249,6 @@ class SpectralCoclustering(BaseSpectral): chosen and the algorithm runs once. Otherwise, the algorithm is run for each initialization and the best solution chosen. - n_jobs : int, default=None - The number of jobs to use for the computation. This works by breaking - down the pairwise matrix into n_jobs even slices and computing them in - parallel. - - ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. - ``-1`` means using all processors. See :term:`Glossary ` - for more details. - - .. deprecated:: 0.23 - ``n_jobs`` was deprecated in version 0.23 and will be removed in - 1.0 (renaming of 0.25). - random_state : int, RandomState instance, default=None Used for randomizing the singular value decomposition and the k-means initialization. Use an int to make the randomness deterministic. @@ -275,6 +269,31 @@ class SpectralCoclustering(BaseSpectral): column_labels_ : array-like of shape (n_cols,) The bicluster label of each column. + biclusters_ : tuple of two ndarrays + The tuple contains the `rows_` and `columns_` arrays. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + SpectralBiclustering : Partitions rows and columns under the assumption + that the data has an underlying checkerboard structure. + + References + ---------- + * :doi:`Dhillon, Inderjit S, 2001. Co-clustering documents and words using + bipartite spectral graph partitioning. + <10.1145/502512.502550>` + Examples -------- >>> from sklearn.cluster import SpectralCoclustering @@ -289,33 +308,42 @@ class SpectralCoclustering(BaseSpectral): >>> clustering SpectralCoclustering(n_clusters=2, random_state=0) - References - ---------- - - * Dhillon, Inderjit S, 2001. `Co-clustering documents and words using - bipartite spectral graph partitioning - `__. - + For a more detailed example, see the following: + :ref:`sphx_glr_auto_examples_bicluster_plot_spectral_coclustering.py`. """ - @_deprecate_positional_args - def __init__(self, n_clusters=3, *, svd_method='randomized', - n_svd_vecs=None, mini_batch=False, init='k-means++', - n_init=10, n_jobs='deprecated', random_state=None): - super().__init__(n_clusters, - svd_method, - n_svd_vecs, - mini_batch, - init, - n_init, - n_jobs, - random_state) + + _parameter_constraints: dict = { + **BaseSpectral._parameter_constraints, + "n_clusters": [Interval(Integral, 1, None, closed="left")], + } + + def __init__( + self, + n_clusters=3, + *, + svd_method="randomized", + n_svd_vecs=None, + mini_batch=False, + init="k-means++", + n_init=10, + random_state=None, + ): + super().__init__( + n_clusters, svd_method, n_svd_vecs, mini_batch, init, n_init, random_state + ) + + def _check_parameters(self, n_samples): + if self.n_clusters > n_samples: + raise ValueError( + f"n_clusters should be <= n_samples={n_samples}. Got" + f" {self.n_clusters} instead." + ) def _fit(self, X): normalized_data, row_diag, col_diag = _scale_normalize(X) n_sv = 1 + int(np.ceil(np.log2(self.n_clusters))) u, v = self._svd(normalized_data, n_sv, n_discard=1) - z = np.vstack((row_diag[:, np.newaxis] * u, - col_diag[:, np.newaxis] * v)) + z = np.vstack((row_diag[:, np.newaxis] * u, col_diag[:, np.newaxis] * v)) _, labels = self._k_means(z, self.n_clusters) @@ -323,10 +351,10 @@ def _fit(self, X): self.row_labels_ = labels[:n_rows] self.column_labels_ = labels[n_rows:] - self.rows_ = np.vstack([self.row_labels_ == c - for c in range(self.n_clusters)]) - self.columns_ = np.vstack([self.column_labels_ == c - for c in range(self.n_clusters)]) + self.rows_ = np.vstack([self.row_labels_ == c for c in range(self.n_clusters)]) + self.columns_ = np.vstack( + [self.column_labels_ == c for c in range(self.n_clusters)] + ) class SpectralBiclustering(BaseSpectral): @@ -355,7 +383,7 @@ class SpectralBiclustering(BaseSpectral): default is 'bistochastic'. .. warning:: - if `method='log'`, the data must be sparse. + if `method='log'`, the data must not be sparse. n_components : int, default=6 Number of singular vectors to check. @@ -381,7 +409,7 @@ class SpectralBiclustering(BaseSpectral): Whether to use mini-batch k-means, which is faster but may get different results. - init : {'k-means++', 'random'} or ndarray of (n_clusters, n_features), \ + init : {'k-means++', 'random'} or ndarray of shape (n_clusters, n_features), \ default='k-means++' Method for initialization of k-means algorithm; defaults to 'k-means++'. @@ -394,19 +422,6 @@ class SpectralBiclustering(BaseSpectral): chosen and the algorithm runs once. Otherwise, the algorithm is run for each initialization and the best solution chosen. - n_jobs : int, default=None - The number of jobs to use for the computation. This works by breaking - down the pairwise matrix into n_jobs even slices and computing them in - parallel. - - ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. - ``-1`` means using all processors. See :term:`Glossary ` - for more details. - - .. deprecated:: 0.23 - ``n_jobs`` was deprecated in version 0.23 and will be removed in - 1.0 (renaming of 0.25). - random_state : int, RandomState instance, default=None Used for randomizing the singular value decomposition and the k-means initialization. Use an int to make the randomness deterministic. @@ -427,6 +442,31 @@ class SpectralBiclustering(BaseSpectral): column_labels_ : array-like of shape (n_cols,) Column partition labels. + biclusters_ : tuple of two ndarrays + The tuple contains the `rows_` and `columns_` arrays. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + SpectralCoclustering : Spectral Co-Clustering algorithm (Dhillon, 2001). + + References + ---------- + + * :doi:`Kluger, Yuval, et. al., 2003. Spectral biclustering of microarray + data: coclustering genes and conditions. + <10.1101/gr.648603>` + Examples -------- >>> from sklearn.cluster import SpectralBiclustering @@ -437,75 +477,93 @@ class SpectralBiclustering(BaseSpectral): >>> clustering.row_labels_ array([1, 1, 1, 0, 0, 0], dtype=int32) >>> clustering.column_labels_ - array([0, 1], dtype=int32) + array([1, 0], dtype=int32) >>> clustering SpectralBiclustering(n_clusters=2, random_state=0) - References - ---------- - - * Kluger, Yuval, et. al., 2003. `Spectral biclustering of microarray - data: coclustering genes and conditions - `__. - + For a more detailed example, see + :ref:`sphx_glr_auto_examples_bicluster_plot_spectral_biclustering.py` """ - @_deprecate_positional_args - def __init__(self, n_clusters=3, *, method='bistochastic', - n_components=6, n_best=3, svd_method='randomized', - n_svd_vecs=None, mini_batch=False, init='k-means++', - n_init=10, n_jobs='deprecated', random_state=None): - super().__init__(n_clusters, - svd_method, - n_svd_vecs, - mini_batch, - init, - n_init, - n_jobs, - random_state) + + _parameter_constraints: dict = { + **BaseSpectral._parameter_constraints, + "n_clusters": [Interval(Integral, 1, None, closed="left"), tuple], + "method": [StrOptions({"bistochastic", "scale", "log"})], + "n_components": [Interval(Integral, 1, None, closed="left")], + "n_best": [Interval(Integral, 1, None, closed="left")], + } + + def __init__( + self, + n_clusters=3, + *, + method="bistochastic", + n_components=6, + n_best=3, + svd_method="randomized", + n_svd_vecs=None, + mini_batch=False, + init="k-means++", + n_init=10, + random_state=None, + ): + super().__init__( + n_clusters, svd_method, n_svd_vecs, mini_batch, init, n_init, random_state + ) self.method = method self.n_components = n_components self.n_best = n_best - def _check_parameters(self): - super()._check_parameters() - legal_methods = ('bistochastic', 'scale', 'log') - if self.method not in legal_methods: - raise ValueError("Unknown method: '{0}'. method must be" - " one of {1}.".format(self.method, legal_methods)) - try: - int(self.n_clusters) - except TypeError: + def _check_parameters(self, n_samples): + if isinstance(self.n_clusters, Integral): + if self.n_clusters > n_samples: + raise ValueError( + f"n_clusters should be <= n_samples={n_samples}. Got" + f" {self.n_clusters} instead." + ) + else: # tuple try: - r, c = self.n_clusters - int(r) - int(c) + n_row_clusters, n_column_clusters = self.n_clusters + check_scalar( + n_row_clusters, + "n_row_clusters", + target_type=Integral, + min_val=1, + max_val=n_samples, + ) + check_scalar( + n_column_clusters, + "n_column_clusters", + target_type=Integral, + min_val=1, + max_val=n_samples, + ) except (ValueError, TypeError) as e: - raise ValueError("Incorrect parameter n_clusters has value:" - " {}. It should either be a single integer" - " or an iterable with two integers:" - " (n_row_clusters, n_column_clusters)") from e - if self.n_components < 1: - raise ValueError("Parameter n_components must be greater than 0," - " but its value is {}".format(self.n_components)) - if self.n_best < 1: - raise ValueError("Parameter n_best must be greater than 0," - " but its value is {}".format(self.n_best)) + raise ValueError( + "Incorrect parameter n_clusters has value:" + f" {self.n_clusters}. It should either be a single integer" + " or an iterable with two integers:" + " (n_row_clusters, n_column_clusters)" + " And the values are should be in the" + " range: (1, n_samples)" + ) from e + if self.n_best > self.n_components: - raise ValueError("n_best cannot be larger than" - " n_components, but {} > {}" - "".format(self.n_best, self.n_components)) + raise ValueError( + f"n_best={self.n_best} must be <= n_components={self.n_components}." + ) def _fit(self, X): n_sv = self.n_components - if self.method == 'bistochastic': + if self.method == "bistochastic": normalized_data = _bistochastic_normalize(X) n_sv += 1 - elif self.method == 'scale': + elif self.method == "scale": normalized_data, _, _ = _scale_normalize(X) n_sv += 1 - elif self.method == 'log': + elif self.method == "log": normalized_data = _log_normalize(X) - n_discard = 0 if self.method == 'log' else 1 + n_discard = 0 if self.method == "log" else 1 u, v = self._svd(normalized_data, n_sv, n_discard) ut = u.T vt = v.T @@ -515,24 +573,28 @@ def _fit(self, X): except TypeError: n_row_clusters = n_col_clusters = self.n_clusters - best_ut = self._fit_best_piecewise(ut, self.n_best, - n_row_clusters) + best_ut = self._fit_best_piecewise(ut, self.n_best, n_row_clusters) - best_vt = self._fit_best_piecewise(vt, self.n_best, - n_col_clusters) + best_vt = self._fit_best_piecewise(vt, self.n_best, n_col_clusters) - self.row_labels_ = self._project_and_cluster(X, best_vt.T, - n_row_clusters) + self.row_labels_ = self._project_and_cluster(X, best_vt.T, n_row_clusters) - self.column_labels_ = self._project_and_cluster(X.T, best_ut.T, - n_col_clusters) + self.column_labels_ = self._project_and_cluster(X.T, best_ut.T, n_col_clusters) - self.rows_ = np.vstack([self.row_labels_ == label - for label in range(n_row_clusters) - for _ in range(n_col_clusters)]) - self.columns_ = np.vstack([self.column_labels_ == label - for _ in range(n_row_clusters) - for label in range(n_col_clusters)]) + self.rows_ = np.vstack( + [ + self.row_labels_ == label + for label in range(n_row_clusters) + for _ in range(n_col_clusters) + ] + ) + self.columns_ = np.vstack( + [ + self.column_labels_ == label + for _ in range(n_row_clusters) + for label in range(n_col_clusters) + ] + ) def _fit_best_piecewise(self, vectors, n_best, n_clusters): """Find the ``n_best`` vectors that are best approximated by piecewise @@ -542,13 +604,13 @@ def _fit_best_piecewise(self, vectors, n_best, n_clusters): according to Euclidean distance. """ + def make_piecewise(v): centroid, labels = self._k_means(v.reshape(-1, 1), n_clusters) return centroid[labels].ravel() - piecewise_vectors = np.apply_along_axis(make_piecewise, - axis=1, arr=vectors) - dists = np.apply_along_axis(norm, axis=1, - arr=(vectors - piecewise_vectors)) + + piecewise_vectors = np.apply_along_axis(make_piecewise, axis=1, arr=vectors) + dists = np.apply_along_axis(norm, axis=1, arr=(vectors - piecewise_vectors)) result = vectors[np.argsort(dists)[:n_best]] return result diff --git a/sklearn/cluster/_birch.py b/sklearn/cluster/_birch.py index 0587fe075a952..4c894a644c8bc 100644 --- a/sklearn/cluster/_birch.py +++ b/sklearn/cluster/_birch.py @@ -1,23 +1,28 @@ -# Authors: Manoj Kumar -# Alexandre Gramfort -# Joel Nothman -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import warnings -import numbers +from math import sqrt +from numbers import Integral, Real + import numpy as np from scipy import sparse -from math import sqrt +from .._config import config_context +from ..base import ( + BaseEstimator, + ClassNamePrefixFeaturesOutMixin, + ClusterMixin, + TransformerMixin, + _fit_context, +) +from ..exceptions import ConvergenceWarning from ..metrics import pairwise_distances_argmin from ..metrics.pairwise import euclidean_distances -from ..base import TransformerMixin, ClusterMixin, BaseEstimator +from ..utils._param_validation import Hidden, Interval, StrOptions from ..utils.extmath import row_norms -from ..utils import deprecated -from ..utils.validation import check_is_fitted, _deprecate_positional_args -from ..exceptions import ConvergenceWarning +from ..utils.validation import check_is_fitted, validate_data from . import AgglomerativeClustering -from .._config import config_context def _iterate_sparse_X(X): @@ -51,13 +56,19 @@ def _split_node(node, threshold, branching_factor): new_subcluster1 = _CFSubcluster() new_subcluster2 = _CFSubcluster() new_node1 = _CFNode( - threshold=threshold, branching_factor=branching_factor, + threshold=threshold, + branching_factor=branching_factor, is_leaf=node.is_leaf, - n_features=node.n_features) + n_features=node.n_features, + dtype=node.init_centroids_.dtype, + ) new_node2 = _CFNode( - threshold=threshold, branching_factor=branching_factor, + threshold=threshold, + branching_factor=branching_factor, is_leaf=node.is_leaf, - n_features=node.n_features) + n_features=node.n_features, + dtype=node.init_centroids_.dtype, + ) new_subcluster1.child_ = new_node1 new_subcluster2.child_ = new_node2 @@ -72,14 +83,19 @@ def _split_node(node, threshold, branching_factor): node.next_leaf_.prev_leaf_ = new_node2 dist = euclidean_distances( - node.centroids_, Y_norm_squared=node.squared_norm_, squared=True) + node.centroids_, Y_norm_squared=node.squared_norm_, squared=True + ) n_clusters = dist.shape[0] - farthest_idx = np.unravel_index( - dist.argmax(), (n_clusters, n_clusters)) + farthest_idx = np.unravel_index(dist.argmax(), (n_clusters, n_clusters)) node1_dist, node2_dist = dist[(farthest_idx,)] node1_closer = node1_dist < node2_dist + # make sure node1 is closest to itself even if all distances are equal. + # This can only happen when all node.centroids_ are duplicates leading to all + # distances between centroids being zero. + node1_closer[farthest_idx[0]] = True + for idx, subcluster in enumerate(node.subclusters_): if node1_closer[idx]: new_node1.append_subcluster(subcluster) @@ -137,7 +153,8 @@ class _CFNode: View of ``init_sq_norm_``. """ - def __init__(self, *, threshold, branching_factor, is_leaf, n_features): + + def __init__(self, *, threshold, branching_factor, is_leaf, n_features, dtype): self.threshold = threshold self.branching_factor = branching_factor self.is_leaf = is_leaf @@ -146,8 +163,8 @@ def __init__(self, *, threshold, branching_factor, is_leaf, n_features): # The list of subclusters, centroids and squared norms # to manipulate throughout. self.subclusters_ = [] - self.init_centroids_ = np.zeros((branching_factor + 1, n_features)) - self.init_sq_norm_ = np.zeros((branching_factor + 1)) + self.init_centroids_ = np.zeros((branching_factor + 1, n_features), dtype=dtype) + self.init_sq_norm_ = np.zeros((branching_factor + 1), dtype) self.squared_norm_ = [] self.prev_leaf_ = None self.next_leaf_ = None @@ -161,11 +178,10 @@ def append_subcluster(self, subcluster): # Keep centroids and squared norm as views. In this way # if we change init_centroids and init_sq_norm_, it is # sufficient, - self.centroids_ = self.init_centroids_[:n_samples + 1, :] - self.squared_norm_ = self.init_sq_norm_[:n_samples + 1] + self.centroids_ = self.init_centroids_[: n_samples + 1, :] + self.squared_norm_ = self.init_sq_norm_[: n_samples + 1] - def update_split_subclusters(self, subcluster, - new_subcluster1, new_subcluster2): + def update_split_subclusters(self, subcluster, new_subcluster1, new_subcluster2): """Remove a subcluster from a node and update it with the split subclusters. """ @@ -186,24 +202,25 @@ def insert_cf_subcluster(self, subcluster): # We need to find the closest subcluster among all the # subclusters so that we can insert our new subcluster. dist_matrix = np.dot(self.centroids_, subcluster.centroid_) - dist_matrix *= -2. + dist_matrix *= -2.0 dist_matrix += self.squared_norm_ closest_index = np.argmin(dist_matrix) closest_subcluster = self.subclusters_[closest_index] # If the subcluster has a child, we need a recursive strategy. if closest_subcluster.child_ is not None: - split_child = closest_subcluster.child_.insert_cf_subcluster( - subcluster) + split_child = closest_subcluster.child_.insert_cf_subcluster(subcluster) if not split_child: # If it is determined that the child need not be split, we # can just update the closest_subcluster closest_subcluster.update(subcluster) - self.init_centroids_[closest_index] = \ - self.subclusters_[closest_index].centroid_ - self.init_sq_norm_[closest_index] = \ - self.subclusters_[closest_index].sq_norm_ + self.init_centroids_[closest_index] = self.subclusters_[ + closest_index + ].centroid_ + self.init_sq_norm_[closest_index] = self.subclusters_[ + closest_index + ].sq_norm_ return False # things not too good. we need to redistribute the subclusters in @@ -211,9 +228,13 @@ def insert_cf_subcluster(self, subcluster): # subcluster to accommodate the new child. else: new_subcluster1, new_subcluster2 = _split_node( - closest_subcluster.child_, threshold, branching_factor) + closest_subcluster.child_, + threshold, + branching_factor, + ) self.update_split_subclusters( - closest_subcluster, new_subcluster1, new_subcluster2) + closest_subcluster, new_subcluster1, new_subcluster2 + ) if len(self.subclusters_) > self.branching_factor: return True @@ -221,13 +242,10 @@ def insert_cf_subcluster(self, subcluster): # good to go! else: - merged = closest_subcluster.merge_subcluster( - subcluster, self.threshold) + merged = closest_subcluster.merge_subcluster(subcluster, self.threshold) if merged: - self.init_centroids_[closest_index] = \ - closest_subcluster.centroid_ - self.init_sq_norm_[closest_index] = \ - closest_subcluster.sq_norm_ + self.init_centroids_[closest_index] = closest_subcluster.centroid_ + self.init_sq_norm_[closest_index] = closest_subcluster.sq_norm_ return False # not close to any other subclusters, and we still @@ -278,6 +296,7 @@ class _CFSubcluster: Squared norm of the subcluster. Used to prevent recomputing when pairwise minimum distances are computed. """ + def __init__(self, *, linear_sum=None): if linear_sum is None: self.n_samples_ = 0 @@ -287,7 +306,8 @@ def __init__(self, *, linear_sum=None): self.n_samples_ = 1 self.centroid_ = self.linear_sum_ = linear_sum self.squared_sum_ = self.sq_norm_ = np.dot( - self.linear_sum_, self.linear_sum_) + self.linear_sum_, self.linear_sum_ + ) self.child_ = None def update(self, subcluster): @@ -317,10 +337,14 @@ def merge_subcluster(self, nominee_cluster, threshold): # r^2 = sum_i ||x_i||^2 / n - ||c||^2 sq_radius = new_ss / new_n - new_sq_norm - if sq_radius <= threshold ** 2: - (self.n_samples_, self.linear_sum_, self.squared_sum_, - self.centroid_, self.sq_norm_) = \ - new_n, new_ls, new_ss, new_centroid, new_sq_norm + if sq_radius <= threshold**2: + ( + self.n_samples_, + self.linear_sum_, + self.squared_sum_, + self.centroid_, + self.sq_norm_, + ) = (new_n, new_ls, new_ss, new_centroid, new_sq_norm) return True return False @@ -332,7 +356,9 @@ def radius(self): return sqrt(max(0, sq_radius)) -class Birch(ClusterMixin, TransformerMixin, BaseEstimator): +class Birch( + ClassNamePrefixFeaturesOutMixin, ClusterMixin, TransformerMixin, BaseEstimator +): """Implements the BIRCH clustering algorithm. It is a memory-efficient, online-learning algorithm provided as an @@ -360,7 +386,7 @@ class Birch(ClusterMixin, TransformerMixin, BaseEstimator): in each. The parent subcluster of that node is removed and two new subclusters are added as parents of the 2 split nodes. - n_clusters : int, instance of sklearn.cluster model, default=3 + n_clusters : int, instance of sklearn.cluster model or None, default=3 Number of clusters after the final clustering step, which treats the subclusters from the leaves as new samples. @@ -381,6 +407,10 @@ class Birch(ClusterMixin, TransformerMixin, BaseEstimator): Whether or not to make a copy of the given data. If set to False, the initial data will be overwritten. + .. deprecated:: 1.6 + `copy` was deprecated in 1.6 and will be removed in 1.8. It has no effect + as the estimator does not perform in-place operations on the input data. + Attributes ---------- root_ : _CFNode @@ -401,6 +431,17 @@ class Birch(ClusterMixin, TransformerMixin, BaseEstimator): if partial_fit is used instead of fit, they are assigned to the last batch of data. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + See Also -------- MiniBatchKMeans : Alternative implementation that does incremental updates @@ -420,6 +461,9 @@ class Birch(ClusterMixin, TransformerMixin, BaseEstimator): subcluster are updated. This is done recursively till the properties of the leaf node are updated. + See :ref:`sphx_glr_auto_examples_cluster_plot_birch_vs_minibatchkmeans.py` for a + comparison with :class:`~sklearn.cluster.MiniBatchKMeans`. + References ---------- * Tian Zhang, Raghu Ramakrishnan, Maron Livny @@ -439,34 +483,35 @@ class Birch(ClusterMixin, TransformerMixin, BaseEstimator): Birch(n_clusters=None) >>> brc.predict(X) array([0, 0, 0, 1, 1, 1]) + + For a comparison of the BIRCH clustering algorithm with other clustering algorithms, + see :ref:`sphx_glr_auto_examples_cluster_plot_cluster_comparison.py` """ - @_deprecate_positional_args - def __init__(self, *, threshold=0.5, branching_factor=50, n_clusters=3, - compute_labels=True, copy=True): + + _parameter_constraints: dict = { + "threshold": [Interval(Real, 0.0, None, closed="neither")], + "branching_factor": [Interval(Integral, 1, None, closed="neither")], + "n_clusters": [None, ClusterMixin, Interval(Integral, 1, None, closed="left")], + "compute_labels": ["boolean"], + "copy": ["boolean", Hidden(StrOptions({"deprecated"}))], + } + + def __init__( + self, + *, + threshold=0.5, + branching_factor=50, + n_clusters=3, + compute_labels=True, + copy="deprecated", + ): self.threshold = threshold self.branching_factor = branching_factor self.n_clusters = n_clusters self.compute_labels = compute_labels self.copy = copy - # TODO: Remove in 1.2 - # mypy error: Decorated property not supported - @deprecated( # type: ignore - "fit_ is deprecated in 1.0 and will be removed in 1.2" - ) - @property - def fit_(self): - return self._deprecated_fit - - # TODO: Remove in 1.2 - # mypy error: Decorated property not supported - @deprecated( # type: ignore - "partial_fit_ is deprecated in 1.0 and will be removed in 1.2" - ) - @property - def partial_fit_(self): - return self._deprecated_partial_fit - + @_fit_context(prefer_skip_nested_validation=True) def fit(self, X, y=None): """ Build a CF Tree for the input data. @@ -484,36 +529,52 @@ def fit(self, X, y=None): self Fitted estimator. """ - # TODO: Remove deprected flags in 1.2 - self._deprecated_fit, self._deprecated_partial_fit = True, False return self._fit(X, partial=False) def _fit(self, X, partial): - has_root = getattr(self, 'root_', None) + has_root = getattr(self, "root_", None) first_call = not (partial and has_root) - X = self._validate_data(X, accept_sparse='csr', copy=self.copy, - reset=first_call) + if self.copy != "deprecated" and first_call: + warnings.warn( + "`copy` was deprecated in 1.6 and will be removed in 1.8 since it " + "has no effect internally. Simply leave this parameter to its default " + "value to avoid this warning.", + FutureWarning, + ) + + X = validate_data( + self, + X, + accept_sparse="csr", + reset=first_call, + dtype=[np.float64, np.float32], + ) threshold = self.threshold branching_factor = self.branching_factor - if branching_factor <= 1: - raise ValueError("Branching_factor should be greater than one.") n_samples, n_features = X.shape # If partial_fit is called for the first time or fit is called, we # start a new tree. if first_call: # The first root is the leaf. Manipulate this object throughout. - self.root_ = _CFNode(threshold=threshold, - branching_factor=branching_factor, - is_leaf=True, - n_features=n_features) + self.root_ = _CFNode( + threshold=threshold, + branching_factor=branching_factor, + is_leaf=True, + n_features=n_features, + dtype=X.dtype, + ) # To enable getting back subclusters. - self.dummy_leaf_ = _CFNode(threshold=threshold, - branching_factor=branching_factor, - is_leaf=True, n_features=n_features) + self.dummy_leaf_ = _CFNode( + threshold=threshold, + branching_factor=branching_factor, + is_leaf=True, + n_features=n_features, + dtype=X.dtype, + ) self.dummy_leaf_.next_leaf_ = self.root_ self.root_.prev_leaf_ = self.dummy_leaf_ @@ -529,18 +590,22 @@ def _fit(self, X, partial): if split: new_subcluster1, new_subcluster2 = _split_node( - self.root_, threshold, branching_factor) + self.root_, threshold, branching_factor + ) del self.root_ - self.root_ = _CFNode(threshold=threshold, - branching_factor=branching_factor, - is_leaf=False, - n_features=n_features) + self.root_ = _CFNode( + threshold=threshold, + branching_factor=branching_factor, + is_leaf=False, + n_features=n_features, + dtype=X.dtype, + ) self.root_.append_subcluster(new_subcluster1) self.root_.append_subcluster(new_subcluster2) - centroids = np.concatenate([ - leaf.centroids_ for leaf in self._get_leaves()]) + centroids = np.concatenate([leaf.centroids_ for leaf in self._get_leaves()]) self.subcluster_centers_ = centroids + self._n_features_out = self.subcluster_centers_.shape[0] self._global_clustering(X) return self @@ -561,6 +626,7 @@ def _get_leaves(self): leaf_ptr = leaf_ptr.next_leaf_ return leaves + @_fit_context(prefer_skip_nested_validation=True) def partial_fit(self, X=None, y=None): """ Online learning. Prevents rebuilding of CFTree from scratch. @@ -580,8 +646,6 @@ def partial_fit(self, X=None, y=None): self Fitted estimator. """ - # TODO: Remove deprected flags in 1.2 - self._deprecated_partial_fit, self._deprecated_fit = True, False if X is None: # Perform just the final global clustering step. self._global_clustering() @@ -589,15 +653,6 @@ def partial_fit(self, X=None, y=None): else: return self._fit(X, partial=True) - def _check_fit(self, X): - check_is_fitted(self) - - if (hasattr(self, 'subcluster_centers_') and - X.shape[1] != self.subcluster_centers_.shape[1]): - raise ValueError( - "Training data and predicted data do " - "not have same number of features.") - def predict(self, X): """ Predict data using the ``centroids_`` of subclusters. @@ -615,12 +670,17 @@ def predict(self, X): Labelled data. """ check_is_fitted(self) - X = self._validate_data(X, accept_sparse='csr', reset=False) - kwargs = {'Y_norm_squared': self._subcluster_norms} + X = validate_data(self, X, accept_sparse="csr", reset=False) + return self._predict(X) + + def _predict(self, X): + """Predict data using the ``centroids_`` of subclusters.""" + kwargs = {"Y_norm_squared": self._subcluster_norms} with config_context(assume_finite=True): - argmin = pairwise_distances_argmin(X, self.subcluster_centers_, - metric_kwargs=kwargs) + argmin = pairwise_distances_argmin( + X, self.subcluster_centers_, metric_kwargs=kwargs + ) return self.subcluster_labels_[argmin] def transform(self, X): @@ -641,7 +701,7 @@ def transform(self, X): Transformed data. """ check_is_fitted(self) - self._validate_data(X, accept_sparse='csr', reset=False) + X = validate_data(self, X, accept_sparse="csr", reset=False) with config_context(assume_finite=True): return euclidean_distances(X, self.subcluster_centers_) @@ -655,20 +715,14 @@ def _global_clustering(self, X=None): # Preprocessing for the global clustering. not_enough_centroids = False - if isinstance(clusterer, numbers.Integral): - clusterer = AgglomerativeClustering( - n_clusters=self.n_clusters) + if isinstance(clusterer, Integral): + clusterer = AgglomerativeClustering(n_clusters=self.n_clusters) # There is no need to perform the global clustering step. if len(centroids) < self.n_clusters: not_enough_centroids = True - elif (clusterer is not None and not - hasattr(clusterer, 'fit_predict')): - raise ValueError("n_clusters should be an instance of " - "ClusterMixin or an int") # To use in predict to avoid recalculation. - self._subcluster_norms = row_norms( - self.subcluster_centers_, squared=True) + self._subcluster_norms = row_norms(self.subcluster_centers_, squared=True) if clusterer is None or not_enough_centroids: self.subcluster_labels_ = np.arange(len(centroids)) @@ -676,13 +730,20 @@ def _global_clustering(self, X=None): warnings.warn( "Number of subclusters found (%d) by BIRCH is less " "than (%d). Decrease the threshold." - % (len(centroids), self.n_clusters), ConvergenceWarning) + % (len(centroids), self.n_clusters), + ConvergenceWarning, + ) else: # The global clustering step that clusters the subclusters of # the leaves. It assumes the centroids of the subclusters as # samples and finds the final centroids. - self.subcluster_labels_ = clusterer.fit_predict( - self.subcluster_centers_) + self.subcluster_labels_ = clusterer.fit_predict(self.subcluster_centers_) if compute_labels: - self.labels_ = self.predict(X) + self.labels_ = self._predict(X) + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.transformer_tags.preserves_dtype = ["float64", "float32"] + tags.input_tags.sparse = True + return tags diff --git a/sklearn/cluster/_bisect_k_means.py b/sklearn/cluster/_bisect_k_means.py new file mode 100644 index 0000000000000..77e24adbf8084 --- /dev/null +++ b/sklearn/cluster/_bisect_k_means.py @@ -0,0 +1,543 @@ +"""Bisecting K-means clustering.""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import warnings + +import numpy as np +import scipy.sparse as sp + +from ..base import _fit_context +from ..utils._openmp_helpers import _openmp_effective_n_threads +from ..utils._param_validation import Integral, Interval, StrOptions +from ..utils.extmath import row_norms +from ..utils.validation import ( + _check_sample_weight, + check_is_fitted, + check_random_state, + validate_data, +) +from ._k_means_common import _inertia_dense, _inertia_sparse +from ._kmeans import ( + _BaseKMeans, + _kmeans_single_elkan, + _kmeans_single_lloyd, + _labels_inertia_threadpool_limit, +) + + +class _BisectingTree: + """Tree structure representing the hierarchical clusters of BisectingKMeans.""" + + def __init__(self, center, indices, score): + """Create a new cluster node in the tree. + + The node holds the center of this cluster and the indices of the data points + that belong to it. + """ + self.center = center + self.indices = indices + self.score = score + + self.left = None + self.right = None + + def split(self, labels, centers, scores): + """Split the cluster node into two subclusters.""" + self.left = _BisectingTree( + indices=self.indices[labels == 0], center=centers[0], score=scores[0] + ) + self.right = _BisectingTree( + indices=self.indices[labels == 1], center=centers[1], score=scores[1] + ) + + # reset the indices attribute to save memory + self.indices = None + + def get_cluster_to_bisect(self): + """Return the cluster node to bisect next. + + It's based on the score of the cluster, which can be either the number of + data points assigned to that cluster or the inertia of that cluster + (see `bisecting_strategy` for details). + """ + max_score = None + + for cluster_leaf in self.iter_leaves(): + if max_score is None or cluster_leaf.score > max_score: + max_score = cluster_leaf.score + best_cluster_leaf = cluster_leaf + + return best_cluster_leaf + + def iter_leaves(self): + """Iterate over all the cluster leaves in the tree.""" + if self.left is None: + yield self + else: + yield from self.left.iter_leaves() + yield from self.right.iter_leaves() + + +class BisectingKMeans(_BaseKMeans): + """Bisecting K-Means clustering. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 1.1 + + Parameters + ---------- + n_clusters : int, default=8 + The number of clusters to form as well as the number of + centroids to generate. + + init : {'k-means++', 'random'} or callable, default='random' + Method for initialization: + + 'k-means++' : selects initial cluster centers for k-mean + clustering in a smart way to speed up convergence. See section + Notes in k_init for more details. + + 'random': choose `n_clusters` observations (rows) at random from data + for the initial centroids. + + If a callable is passed, it should take arguments X, n_clusters and a + random state and return an initialization. + + n_init : int, default=1 + Number of time the inner k-means algorithm will be run with different + centroid seeds in each bisection. + That will result producing for each bisection best output of n_init + consecutive runs in terms of inertia. + + random_state : int, RandomState instance or None, default=None + Determines random number generation for centroid initialization + in inner K-Means. Use an int to make the randomness deterministic. + See :term:`Glossary `. + + max_iter : int, default=300 + Maximum number of iterations of the inner k-means algorithm at each + bisection. + + verbose : int, default=0 + Verbosity mode. + + tol : float, default=1e-4 + Relative tolerance with regards to Frobenius norm of the difference + in the cluster centers of two consecutive iterations to declare + convergence. Used in inner k-means algorithm at each bisection to pick + best possible clusters. + + copy_x : bool, default=True + When pre-computing distances it is more numerically accurate to center + the data first. If copy_x is True (default), then the original data is + not modified. If False, the original data is modified, and put back + before the function returns, but small numerical differences may be + introduced by subtracting and then adding the data mean. Note that if + the original data is not C-contiguous, a copy will be made even if + copy_x is False. If the original data is sparse, but not in CSR format, + a copy will be made even if copy_x is False. + + algorithm : {"lloyd", "elkan"}, default="lloyd" + Inner K-means algorithm used in bisection. + The classical EM-style algorithm is `"lloyd"`. + The `"elkan"` variation can be more efficient on some datasets with + well-defined clusters, by using the triangle inequality. However it's + more memory intensive due to the allocation of an extra array of shape + `(n_samples, n_clusters)`. + + bisecting_strategy : {"biggest_inertia", "largest_cluster"},\ + default="biggest_inertia" + Defines how bisection should be performed: + + - "biggest_inertia" means that BisectingKMeans will always check + all calculated cluster for cluster with biggest SSE + (Sum of squared errors) and bisect it. This approach concentrates on + precision, but may be costly in terms of execution time (especially for + larger amount of data points). + + - "largest_cluster" - BisectingKMeans will always split cluster with + largest amount of points assigned to it from all clusters + previously calculated. That should work faster than picking by SSE + ('biggest_inertia') and may produce similar results in most cases. + + Attributes + ---------- + cluster_centers_ : ndarray of shape (n_clusters, n_features) + Coordinates of cluster centers. If the algorithm stops before fully + converging (see ``tol`` and ``max_iter``), these will not be + consistent with ``labels_``. + + labels_ : ndarray of shape (n_samples,) + Labels of each point. + + inertia_ : float + Sum of squared distances of samples to their closest cluster center, + weighted by the sample weights if provided. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + See Also + -------- + KMeans : Original implementation of K-Means algorithm. + + Notes + ----- + It might be inefficient when n_cluster is less than 3, due to unnecessary + calculations for that case. + + Examples + -------- + >>> from sklearn.cluster import BisectingKMeans + >>> import numpy as np + >>> X = np.array([[1, 1], [10, 1], [3, 1], + ... [10, 0], [2, 1], [10, 2], + ... [10, 8], [10, 9], [10, 10]]) + >>> bisect_means = BisectingKMeans(n_clusters=3, random_state=0).fit(X) + >>> bisect_means.labels_ + array([0, 2, 0, 2, 0, 2, 1, 1, 1], dtype=int32) + >>> bisect_means.predict([[0, 0], [12, 3]]) + array([0, 2], dtype=int32) + >>> bisect_means.cluster_centers_ + array([[ 2., 1.], + [10., 9.], + [10., 1.]]) + + For a comparison between BisectingKMeans and K-Means refer to example + :ref:`sphx_glr_auto_examples_cluster_plot_bisect_kmeans.py`. + """ + + _parameter_constraints: dict = { + **_BaseKMeans._parameter_constraints, + "init": [StrOptions({"k-means++", "random"}), callable], + "n_init": [Interval(Integral, 1, None, closed="left")], + "copy_x": ["boolean"], + "algorithm": [StrOptions({"lloyd", "elkan"})], + "bisecting_strategy": [StrOptions({"biggest_inertia", "largest_cluster"})], + } + + def __init__( + self, + n_clusters=8, + *, + init="random", + n_init=1, + random_state=None, + max_iter=300, + verbose=0, + tol=1e-4, + copy_x=True, + algorithm="lloyd", + bisecting_strategy="biggest_inertia", + ): + super().__init__( + n_clusters=n_clusters, + init=init, + max_iter=max_iter, + verbose=verbose, + random_state=random_state, + tol=tol, + n_init=n_init, + ) + + self.copy_x = copy_x + self.algorithm = algorithm + self.bisecting_strategy = bisecting_strategy + + def _warn_mkl_vcomp(self, n_active_threads): + """Warn when vcomp and mkl are both present""" + warnings.warn( + "BisectingKMeans is known to have a memory leak on Windows " + "with MKL, when there are less chunks than available " + "threads. You can avoid it by setting the environment" + f" variable OMP_NUM_THREADS={n_active_threads}." + ) + + def _inertia_per_cluster(self, X, centers, labels, sample_weight): + """Calculate the sum of squared errors (inertia) per cluster. + + Parameters + ---------- + X : {ndarray, csr_matrix} of shape (n_samples, n_features) + The input samples. + + centers : ndarray of shape (n_clusters=2, n_features) + The cluster centers. + + labels : ndarray of shape (n_samples,) + Index of the cluster each sample belongs to. + + sample_weight : ndarray of shape (n_samples,) + The weights for each observation in X. + + Returns + ------- + inertia_per_cluster : ndarray of shape (n_clusters=2,) + Sum of squared errors (inertia) for each cluster. + """ + n_clusters = centers.shape[0] # = 2 since centers comes from a bisection + _inertia = _inertia_sparse if sp.issparse(X) else _inertia_dense + + inertia_per_cluster = np.empty(n_clusters) + for label in range(n_clusters): + inertia_per_cluster[label] = _inertia( + X, sample_weight, centers, labels, self._n_threads, single_label=label + ) + + return inertia_per_cluster + + def _bisect(self, X, x_squared_norms, sample_weight, cluster_to_bisect): + """Split a cluster into 2 subsclusters. + + Parameters + ---------- + X : {ndarray, csr_matrix} of shape (n_samples, n_features) + Training instances to cluster. + + x_squared_norms : ndarray of shape (n_samples,) + Squared euclidean norm of each data point. + + sample_weight : ndarray of shape (n_samples,) + The weights for each observation in X. + + cluster_to_bisect : _BisectingTree node object + The cluster node to split. + """ + X = X[cluster_to_bisect.indices] + x_squared_norms = x_squared_norms[cluster_to_bisect.indices] + sample_weight = sample_weight[cluster_to_bisect.indices] + + best_inertia = None + + # Split samples in X into 2 clusters. + # Repeating `n_init` times to obtain best clusters + for _ in range(self.n_init): + centers_init = self._init_centroids( + X, + x_squared_norms=x_squared_norms, + init=self.init, + random_state=self._random_state, + n_centroids=2, + sample_weight=sample_weight, + ) + + labels, inertia, centers, _ = self._kmeans_single( + X, + sample_weight, + centers_init, + max_iter=self.max_iter, + verbose=self.verbose, + tol=self.tol, + n_threads=self._n_threads, + ) + + # allow small tolerance on the inertia to accommodate for + # non-deterministic rounding errors due to parallel computation + if best_inertia is None or inertia < best_inertia * (1 - 1e-6): + best_labels = labels + best_centers = centers + best_inertia = inertia + + if self.verbose: + print(f"New centroids from bisection: {best_centers}") + + if self.bisecting_strategy == "biggest_inertia": + scores = self._inertia_per_cluster( + X, best_centers, best_labels, sample_weight + ) + else: # bisecting_strategy == "largest_cluster" + # Using minlength to make sure that we have the counts for both labels even + # if all samples are labelled 0. + scores = np.bincount(best_labels, minlength=2) + + cluster_to_bisect.split(best_labels, best_centers, scores) + + @_fit_context(prefer_skip_nested_validation=True) + def fit(self, X, y=None, sample_weight=None): + """Compute bisecting k-means clustering. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + + Training instances to cluster. + + .. note:: The data will be converted to C ordering, + which will cause a memory copy + if the given data is not C-contiguous. + + y : Ignored + Not used, present here for API consistency by convention. + + sample_weight : array-like of shape (n_samples,), default=None + The weights for each observation in X. If None, all observations + are assigned equal weight. `sample_weight` is not used during + initialization if `init` is a callable. + + Returns + ------- + self + Fitted estimator. + """ + X = validate_data( + self, + X, + accept_sparse="csr", + dtype=[np.float64, np.float32], + order="C", + copy=self.copy_x, + accept_large_sparse=False, + ) + + self._check_params_vs_input(X) + + self._random_state = check_random_state(self.random_state) + sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype) + self._n_threads = _openmp_effective_n_threads() + + if self.algorithm == "lloyd" or self.n_clusters == 1: + self._kmeans_single = _kmeans_single_lloyd + self._check_mkl_vcomp(X, X.shape[0]) + else: + self._kmeans_single = _kmeans_single_elkan + + # Subtract of mean of X for more accurate distance computations + if not sp.issparse(X): + self._X_mean = X.mean(axis=0) + X -= self._X_mean + + # Initialize the hierarchical clusters tree + self._bisecting_tree = _BisectingTree( + indices=np.arange(X.shape[0]), + center=X.mean(axis=0), + score=0, + ) + + x_squared_norms = row_norms(X, squared=True) + + for _ in range(self.n_clusters - 1): + # Chose cluster to bisect + cluster_to_bisect = self._bisecting_tree.get_cluster_to_bisect() + + # Split this cluster into 2 subclusters + self._bisect(X, x_squared_norms, sample_weight, cluster_to_bisect) + + # Aggregate final labels and centers from the bisecting tree + self.labels_ = np.full(X.shape[0], -1, dtype=np.int32) + self.cluster_centers_ = np.empty((self.n_clusters, X.shape[1]), dtype=X.dtype) + + for i, cluster_node in enumerate(self._bisecting_tree.iter_leaves()): + self.labels_[cluster_node.indices] = i + self.cluster_centers_[i] = cluster_node.center + cluster_node.label = i # label final clusters for future prediction + cluster_node.indices = None # release memory + + # Restore original data + if not sp.issparse(X): + X += self._X_mean + self.cluster_centers_ += self._X_mean + + _inertia = _inertia_sparse if sp.issparse(X) else _inertia_dense + self.inertia_ = _inertia( + X, sample_weight, self.cluster_centers_, self.labels_, self._n_threads + ) + + self._n_features_out = self.cluster_centers_.shape[0] + + return self + + def predict(self, X): + """Predict which cluster each sample in X belongs to. + + Prediction is made by going down the hierarchical tree + in searching of closest leaf cluster. + + In the vector quantization literature, `cluster_centers_` is called + the code book and each value returned by `predict` is the index of + the closest code in the code book. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + New data to predict. + + Returns + ------- + labels : ndarray of shape (n_samples,) + Index of the cluster each sample belongs to. + """ + check_is_fitted(self) + + X = self._check_test_data(X) + x_squared_norms = row_norms(X, squared=True) + + # sample weights are unused but necessary in cython helpers + sample_weight = np.ones_like(x_squared_norms) + + labels = self._predict_recursive(X, sample_weight, self._bisecting_tree) + + return labels + + def _predict_recursive(self, X, sample_weight, cluster_node): + """Predict recursively by going down the hierarchical tree. + + Parameters + ---------- + X : {ndarray, csr_matrix} of shape (n_samples, n_features) + The data points, currently assigned to `cluster_node`, to predict between + the subclusters of this node. + + sample_weight : ndarray of shape (n_samples,) + The weights for each observation in X. + + cluster_node : _BisectingTree node object + The cluster node of the hierarchical tree. + + Returns + ------- + labels : ndarray of shape (n_samples,) + Index of the cluster each sample belongs to. + """ + if cluster_node.left is None: + # This cluster has no subcluster. Labels are just the label of the cluster. + return np.full(X.shape[0], cluster_node.label, dtype=np.int32) + + # Determine if data points belong to the left or right subcluster + centers = np.vstack((cluster_node.left.center, cluster_node.right.center)) + if hasattr(self, "_X_mean"): + centers += self._X_mean + + cluster_labels = _labels_inertia_threadpool_limit( + X, + sample_weight, + centers, + self._n_threads, + return_inertia=False, + ) + mask = cluster_labels == 0 + + # Compute the labels for each subset of the data points. + labels = np.full(X.shape[0], -1, dtype=np.int32) + + labels[mask] = self._predict_recursive( + X[mask], sample_weight[mask], cluster_node.left + ) + + labels[~mask] = self._predict_recursive( + X[~mask], sample_weight[~mask], cluster_node.right + ) + + return labels + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.sparse = True + tags.transformer_tags.preserves_dtype = ["float64", "float32"] + return tags diff --git a/sklearn/cluster/_dbscan.py b/sklearn/cluster/_dbscan.py index a841a9b7c213c..857a332cc2371 100644 --- a/sklearn/cluster/_dbscan.py +++ b/sklearn/cluster/_dbscan.py @@ -1,29 +1,44 @@ -# -*- coding: utf-8 -*- """ DBSCAN: Density-Based Spatial Clustering of Applications with Noise """ -# Author: Robert Layton -# Joel Nothman -# Lars Buitinck -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import warnings +from numbers import Integral, Real + +import numpy as np from scipy import sparse -from ..base import BaseEstimator, ClusterMixin -from ..utils.validation import _check_sample_weight, _deprecate_positional_args +from ..base import BaseEstimator, ClusterMixin, _fit_context +from ..metrics.pairwise import _VALID_METRICS from ..neighbors import NearestNeighbors - +from ..utils._param_validation import Interval, StrOptions, validate_params +from ..utils.validation import _check_sample_weight, validate_data from ._dbscan_inner import dbscan_inner -@_deprecate_positional_args -def dbscan(X, eps=0.5, *, min_samples=5, metric='minkowski', - metric_params=None, algorithm='auto', leaf_size=30, p=2, - sample_weight=None, n_jobs=None): +@validate_params( + { + "X": ["array-like", "sparse matrix"], + "sample_weight": ["array-like", None], + }, + prefer_skip_nested_validation=False, +) +def dbscan( + X, + eps=0.5, + *, + min_samples=5, + metric="minkowski", + metric_params=None, + algorithm="auto", + leaf_size=30, + p=2, + sample_weight=None, + n_jobs=None, +): """Perform DBSCAN clustering from vector array or distance matrix. Read more in the :ref:`User Guide `. @@ -105,8 +120,7 @@ def dbscan(X, eps=0.5, *, min_samples=5, metric='minkowski', Notes ----- - For an example, see :ref:`examples/cluster/plot_dbscan.py - `. + For an example, see :ref:`sphx_glr_auto_examples_cluster_plot_dbscan.py`. This implementation bulk-computes all neighborhood queries, which increases the memory complexity to O(n.d) where d is the average number of neighbors, @@ -123,24 +137,43 @@ def dbscan(X, eps=0.5, *, min_samples=5, metric='minkowski', Another way to reduce memory and computation time is to remove (near-)duplicate points and use ``sample_weight`` instead. - :func:`cluster.optics ` provides a similar - clustering with lower memory usage. + :class:`~sklearn.cluster.OPTICS` provides a similar clustering with lower + memory usage. References ---------- - Ester, M., H. P. Kriegel, J. Sander, and X. Xu, "A Density-Based - Algorithm for Discovering Clusters in Large Spatial Databases with Noise". + Ester, M., H. P. Kriegel, J. Sander, and X. Xu, `"A Density-Based + Algorithm for Discovering Clusters in Large Spatial Databases with Noise" + `_. In: Proceedings of the 2nd International Conference on Knowledge Discovery and Data Mining, Portland, OR, AAAI Press, pp. 226-231. 1996 Schubert, E., Sander, J., Ester, M., Kriegel, H. P., & Xu, X. (2017). - DBSCAN revisited, revisited: why and how you should (still) use DBSCAN. + :doi:`"DBSCAN revisited, revisited: why and how you should (still) use DBSCAN." + <10.1145/3068335>` ACM Transactions on Database Systems (TODS), 42(3), 19. + + Examples + -------- + >>> from sklearn.cluster import dbscan + >>> X = [[1, 2], [2, 2], [2, 3], [8, 7], [8, 8], [25, 80]] + >>> core_samples, labels = dbscan(X, eps=3, min_samples=2) + >>> core_samples + array([0, 1, 2, 3, 4]) + >>> labels + array([ 0, 0, 0, 1, 1, -1]) """ - est = DBSCAN(eps=eps, min_samples=min_samples, metric=metric, - metric_params=metric_params, algorithm=algorithm, - leaf_size=leaf_size, p=p, n_jobs=n_jobs) + est = DBSCAN( + eps=eps, + min_samples=min_samples, + metric=metric, + metric_params=metric_params, + algorithm=algorithm, + leaf_size=leaf_size, + p=p, + n_jobs=n_jobs, + ) est.fit(X, sample_weight=sample_weight) return est.core_sample_indices_, est.labels_ @@ -152,6 +185,11 @@ class DBSCAN(ClusterMixin, BaseEstimator): Finds core samples of high density and expands clusters from them. Good for data which contains clusters of similar density. + This implementation has a worst case memory complexity of :math:`O({n}^2)`, + which can occur when the `eps` param is large and `min_samples` is low, + while the original DBSCAN only uses linear memory. + For further details, see the Notes below. + Read more in the :ref:`User Guide `. Parameters @@ -164,16 +202,19 @@ class DBSCAN(ClusterMixin, BaseEstimator): and distance function. min_samples : int, default=5 - The number of samples (or total weight) in a neighborhood for a point - to be considered as a core point. This includes the point itself. + The number of samples (or total weight) in a neighborhood for a point to + be considered as a core point. This includes the point itself. If + `min_samples` is set to a higher value, DBSCAN will find denser clusters, + whereas if it is set to a lower value, the found clusters will be more + sparse. - metric : string, or callable, default='euclidean' + metric : str, or callable, default='euclidean' The metric to use when calculating distance between instances in a feature array. If metric is a string or callable, it must be one of the options allowed by :func:`sklearn.metrics.pairwise_distances` for its metric parameter. If metric is "precomputed", X is assumed to be a distance matrix and - must be square. X may be a :term:`Glossary `, in which + must be square. X may be a :term:`sparse graph`, in which case only "nonzero" elements may be considered neighbors for DBSCAN. .. versionadded:: 0.17 @@ -218,17 +259,16 @@ class DBSCAN(ClusterMixin, BaseEstimator): Cluster labels for each point in the dataset given to fit(). Noisy samples are given the label -1. - Examples - -------- - >>> from sklearn.cluster import DBSCAN - >>> import numpy as np - >>> X = np.array([[1, 2], [2, 2], [2, 3], - ... [8, 7], [8, 8], [25, 80]]) - >>> clustering = DBSCAN(eps=3, min_samples=2).fit(X) - >>> clustering.labels_ - array([ 0, 0, 0, 1, 1, -1]) - >>> clustering - DBSCAN(eps=3, min_samples=2) + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 See Also -------- @@ -237,9 +277,6 @@ class DBSCAN(ClusterMixin, BaseEstimator): Notes ----- - For an example, see :ref:`examples/cluster/plot_dbscan.py - `. - This implementation bulk-computes all neighborhood queries, which increases the memory complexity to O(n.d) where d is the average number of neighbors, while original DBSCAN had memory complexity O(n). It may attract a higher @@ -255,24 +292,67 @@ class DBSCAN(ClusterMixin, BaseEstimator): Another way to reduce memory and computation time is to remove (near-)duplicate points and use ``sample_weight`` instead. - :class:`cluster.OPTICS` provides a similar clustering with lower memory + :class:`~sklearn.cluster.OPTICS` provides a similar clustering with lower memory usage. References ---------- - Ester, M., H. P. Kriegel, J. Sander, and X. Xu, "A Density-Based - Algorithm for Discovering Clusters in Large Spatial Databases with Noise". + Ester, M., H. P. Kriegel, J. Sander, and X. Xu, `"A Density-Based + Algorithm for Discovering Clusters in Large Spatial Databases with Noise" + `_. In: Proceedings of the 2nd International Conference on Knowledge Discovery and Data Mining, Portland, OR, AAAI Press, pp. 226-231. 1996 Schubert, E., Sander, J., Ester, M., Kriegel, H. P., & Xu, X. (2017). - DBSCAN revisited, revisited: why and how you should (still) use DBSCAN. + :doi:`"DBSCAN revisited, revisited: why and how you should (still) use DBSCAN." + <10.1145/3068335>` ACM Transactions on Database Systems (TODS), 42(3), 19. + + Examples + -------- + >>> from sklearn.cluster import DBSCAN + >>> import numpy as np + >>> X = np.array([[1, 2], [2, 2], [2, 3], + ... [8, 7], [8, 8], [25, 80]]) + >>> clustering = DBSCAN(eps=3, min_samples=2).fit(X) + >>> clustering.labels_ + array([ 0, 0, 0, 1, 1, -1]) + >>> clustering + DBSCAN(eps=3, min_samples=2) + + For an example, see + :ref:`sphx_glr_auto_examples_cluster_plot_dbscan.py`. + + For a comparison of DBSCAN with other clustering algorithms, see + :ref:`sphx_glr_auto_examples_cluster_plot_cluster_comparison.py` """ - @_deprecate_positional_args - def __init__(self, eps=0.5, *, min_samples=5, metric='euclidean', - metric_params=None, algorithm='auto', leaf_size=30, p=None, - n_jobs=None): + + _parameter_constraints: dict = { + "eps": [Interval(Real, 0.0, None, closed="neither")], + "min_samples": [Interval(Integral, 1, None, closed="left")], + "metric": [ + StrOptions(set(_VALID_METRICS) | {"precomputed"}), + callable, + ], + "metric_params": [dict, None], + "algorithm": [StrOptions({"auto", "ball_tree", "kd_tree", "brute"})], + "leaf_size": [Interval(Integral, 1, None, closed="left")], + "p": [Interval(Real, 0.0, None, closed="left"), None], + "n_jobs": [Integral, None], + } + + def __init__( + self, + eps=0.5, + *, + min_samples=5, + metric="euclidean", + metric_params=None, + algorithm="auto", + leaf_size=30, + p=None, + n_jobs=None, + ): self.eps = eps self.min_samples = min_samples self.metric = metric @@ -282,6 +362,10 @@ def __init__(self, eps=0.5, *, min_samples=5, metric='euclidean', self.p = p self.n_jobs = n_jobs + @_fit_context( + # DBSCAN.metric is not validated yet + prefer_skip_nested_validation=False + ) def fit(self, X, y=None, sample_weight=None): """Perform DBSCAN clustering from features, or distance matrix. @@ -293,24 +377,21 @@ def fit(self, X, y=None, sample_weight=None): ``metric='precomputed'``. If a sparse matrix is provided, it will be converted into a sparse ``csr_matrix``. + y : Ignored + Not used, present here for API consistency by convention. + sample_weight : array-like of shape (n_samples,), default=None Weight of each sample, such that a sample with a weight of at least ``min_samples`` is by itself a core sample; a sample with a negative weight may inhibit its eps-neighbor from being core. Note that weights are absolute, and default to 1. - y : Ignored - Not used, present here for API consistency by convention. - Returns ------- - self - + self : object + Returns a fitted instance of self. """ - X = self._validate_data(X, accept_sparse='csr') - - if not self.eps > 0.0: - raise ValueError("eps must be positive.") + X = validate_data(self, X, accept_sparse="csr") if sample_weight is not None: sample_weight = _check_sample_weight(sample_weight, X) @@ -318,35 +399,39 @@ def fit(self, X, y=None, sample_weight=None): # Calculate neighborhood for all samples. This leaves the original # point in, which needs to be considered later (i.e. point i is in the # neighborhood of point i. While True, its useless information) - if self.metric == 'precomputed' and sparse.issparse(X): + if self.metric == "precomputed" and sparse.issparse(X): # set the diagonal to explicit values, as a point is its own # neighbor + X = X.copy() # copy to avoid in-place modification with warnings.catch_warnings(): - warnings.simplefilter('ignore', sparse.SparseEfficiencyWarning) - X.setdiag(X.diagonal()) # XXX: modifies X's internals in-place + warnings.simplefilter("ignore", sparse.SparseEfficiencyWarning) + X.setdiag(X.diagonal()) neighbors_model = NearestNeighbors( - radius=self.eps, algorithm=self.algorithm, - leaf_size=self.leaf_size, metric=self.metric, - metric_params=self.metric_params, p=self.p, n_jobs=self.n_jobs) + radius=self.eps, + algorithm=self.algorithm, + leaf_size=self.leaf_size, + metric=self.metric, + metric_params=self.metric_params, + p=self.p, + n_jobs=self.n_jobs, + ) neighbors_model.fit(X) # This has worst case O(n^2) memory complexity - neighborhoods = neighbors_model.radius_neighbors(X, - return_distance=False) + neighborhoods = neighbors_model.radius_neighbors(X, return_distance=False) if sample_weight is None: - n_neighbors = np.array([len(neighbors) - for neighbors in neighborhoods]) + n_neighbors = np.array([len(neighbors) for neighbors in neighborhoods]) else: - n_neighbors = np.array([np.sum(sample_weight[neighbors]) - for neighbors in neighborhoods]) + n_neighbors = np.array( + [np.sum(sample_weight[neighbors]) for neighbors in neighborhoods] + ) # Initially, all samples are noise. labels = np.full(X.shape[0], -1, dtype=np.intp) # A list of all core samples found. - core_samples = np.asarray(n_neighbors >= self.min_samples, - dtype=np.uint8) + core_samples = np.asarray(n_neighbors >= self.min_samples, dtype=np.uint8) dbscan_inner(core_samples, neighborhoods, labels) self.core_sample_indices_ = np.where(core_samples)[0] @@ -361,8 +446,7 @@ def fit(self, X, y=None, sample_weight=None): return self def fit_predict(self, X, y=None, sample_weight=None): - """Perform DBSCAN clustering from features or distance matrix, - and return cluster labels. + """Compute clusters from a data or distance matrix and predict labels. Parameters ---------- @@ -372,15 +456,15 @@ def fit_predict(self, X, y=None, sample_weight=None): ``metric='precomputed'``. If a sparse matrix is provided, it will be converted into a sparse ``csr_matrix``. + y : Ignored + Not used, present here for API consistency by convention. + sample_weight : array-like of shape (n_samples,), default=None Weight of each sample, such that a sample with a weight of at least ``min_samples`` is by itself a core sample; a sample with a negative weight may inhibit its eps-neighbor from being core. Note that weights are absolute, and default to 1. - y : Ignored - Not used, present here for API consistency by convention. - Returns ------- labels : ndarray of shape (n_samples,) @@ -388,3 +472,9 @@ def fit_predict(self, X, y=None, sample_weight=None): """ self.fit(X, sample_weight=sample_weight) return self.labels_ + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.pairwise = self.metric == "precomputed" + tags.input_tags.sparse = True + return tags diff --git a/sklearn/cluster/_dbscan_inner.pyx b/sklearn/cluster/_dbscan_inner.pyx index b9a80686a76f8..266b214bb269a 100644 --- a/sklearn/cluster/_dbscan_inner.pyx +++ b/sklearn/cluster/_dbscan_inner.pyx @@ -1,29 +1,19 @@ # Fast inner loop for DBSCAN. -# Author: Lars Buitinck -# License: 3-clause BSD -# -# cython: boundscheck=False, wraparound=False -cimport cython -from libcpp.vector cimport vector -cimport numpy as np -import numpy as np - -np.import_array() +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +from libcpp.vector cimport vector -# Work around Cython bug: C++ exceptions are not caught unless thrown within -# a cdef function with an "except +" declaration. -cdef inline void push(vector[np.npy_intp] &stack, np.npy_intp i) except +: - stack.push_back(i) +from ..utils._typedefs cimport uint8_t, intp_t -def dbscan_inner(np.ndarray[np.uint8_t, ndim=1, mode='c'] is_core, - np.ndarray[object, ndim=1] neighborhoods, - np.ndarray[np.npy_intp, ndim=1, mode='c'] labels): - cdef np.npy_intp i, label_num = 0, v - cdef np.ndarray[np.npy_intp, ndim=1] neighb - cdef vector[np.npy_intp] stack +def dbscan_inner(const uint8_t[::1] is_core, + object[:] neighborhoods, + intp_t[::1] labels): + cdef intp_t i, label_num = 0, v + cdef intp_t[:] neighb + cdef vector[intp_t] stack for i in range(labels.shape[0]): if labels[i] != -1 or not is_core[i]: @@ -41,7 +31,7 @@ def dbscan_inner(np.ndarray[np.uint8_t, ndim=1, mode='c'] is_core, for i in range(neighb.shape[0]): v = neighb[i] if labels[v] == -1: - push(stack, v) + stack.push_back(v) if stack.size() == 0: break diff --git a/sklearn/cluster/_feature_agglomeration.py b/sklearn/cluster/_feature_agglomeration.py index e27a048366401..32fcb85625f35 100644 --- a/sklearn/cluster/_feature_agglomeration.py +++ b/sklearn/cluster/_feature_agglomeration.py @@ -2,14 +2,15 @@ Feature agglomeration. Base classes and functions for performing feature agglomeration. """ -# Author: V. Michel, A. Gramfort -# License: BSD 3 clause + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import numpy as np +from scipy.sparse import issparse from ..base import TransformerMixin -from ..utils.validation import check_is_fitted -from scipy.sparse import issparse +from ..utils.validation import check_is_fitted, validate_data ############################################################################### # Mixin class for feature agglomeration. @@ -17,16 +18,17 @@ class AgglomerationTransform(TransformerMixin): """ - A class for feature agglomeration via the transform interface + A class for feature agglomeration via the transform interface. """ def transform(self, X): """ - Transform a new matrix using the built clustering + Transform a new matrix using the built clustering. Parameters ---------- - X : array-like of shape (n_samples, n_features) or (n_samples,) + X : array-like of shape (n_samples, n_features) or \ + (n_samples, n_samples) A M by N array of M observations in N dimensions or a length M array of M one-dimensional observations. @@ -37,37 +39,38 @@ def transform(self, X): """ check_is_fitted(self) - X = self._validate_data(X, reset=False) + X = validate_data(self, X, reset=False) if self.pooling_func == np.mean and not issparse(X): size = np.bincount(self.labels_) n_samples = X.shape[0] # a fast way to compute the mean of grouped features - nX = np.array([np.bincount(self.labels_, X[i, :]) / size - for i in range(n_samples)]) + nX = np.array( + [np.bincount(self.labels_, X[i, :]) / size for i in range(n_samples)] + ) else: - nX = [self.pooling_func(X[:, self.labels_ == l], axis=1) - for l in np.unique(self.labels_)] + nX = [ + self.pooling_func(X[:, self.labels_ == l], axis=1) + for l in np.unique(self.labels_) + ] nX = np.array(nX).T return nX - def inverse_transform(self, Xred): + def inverse_transform(self, X): """ - Inverse the transformation. - Return a vector of size nb_features with the values of Xred assigned - to each group of features + Inverse the transformation and return a vector of size `n_features`. Parameters ---------- - Xred : array-like of shape (n_samples, n_clusters) or (n_clusters,) - The values to be assigned to each cluster of samples + X : array-like of shape (n_samples, n_clusters) or (n_clusters,) + The values to be assigned to each cluster of samples. Returns ------- - X : ndarray of shape (n_samples, n_features) or (n_features,) - A vector of size n_samples with the values of Xred assigned to + X_original : ndarray of shape (n_samples, n_features) or (n_features,) + A vector of size `n_samples` with the values of `X` assigned to each of the cluster of samples. """ check_is_fitted(self) unil, inverse = np.unique(self.labels_, return_inverse=True) - return Xred[..., inverse] + return X[..., inverse] diff --git a/sklearn/cluster/_hdbscan/__init__.py b/sklearn/cluster/_hdbscan/__init__.py new file mode 100644 index 0000000000000..67dd18fb94b59 --- /dev/null +++ b/sklearn/cluster/_hdbscan/__init__.py @@ -0,0 +1,2 @@ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause diff --git a/sklearn/cluster/_hdbscan/_linkage.pyx b/sklearn/cluster/_hdbscan/_linkage.pyx new file mode 100644 index 0000000000000..5684193a13d40 --- /dev/null +++ b/sklearn/cluster/_hdbscan/_linkage.pyx @@ -0,0 +1,274 @@ +# Minimum spanning tree single linkage implementation for hdbscan + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +cimport numpy as cnp +from libc.float cimport DBL_MAX + +import numpy as np +from ...metrics._dist_metrics cimport DistanceMetric64 +from ...cluster._hierarchical_fast cimport UnionFind +from ...cluster._hdbscan._tree cimport HIERARCHY_t +from ...cluster._hdbscan._tree import HIERARCHY_dtype +from ...utils._typedefs cimport intp_t, float64_t, int64_t, uint8_t + +cnp.import_array() + +cdef extern from "numpy/arrayobject.h": + intp_t * PyArray_SHAPE(cnp.PyArrayObject *) + +# Numpy structured dtype representing a single ordered edge in Prim's algorithm +MST_edge_dtype = np.dtype([ + ("current_node", np.int64), + ("next_node", np.int64), + ("distance", np.float64), +]) + +# Packed shouldn't make a difference since they're all 8-byte quantities, +# but it's included just to be safe. +ctypedef packed struct MST_edge_t: + int64_t current_node + int64_t next_node + float64_t distance + +cpdef cnp.ndarray[MST_edge_t, ndim=1, mode='c'] mst_from_mutual_reachability( + cnp.ndarray[float64_t, ndim=2] mutual_reachability +): + """Compute the Minimum Spanning Tree (MST) representation of the mutual- + reachability graph using Prim's algorithm. + + Parameters + ---------- + mutual_reachability : ndarray of shape (n_samples, n_samples) + Array of mutual-reachabilities between samples. + + Returns + ------- + mst : ndarray of shape (n_samples - 1,), dtype=MST_edge_dtype + The MST representation of the mutual-reachability graph. The MST is + represented as a collection of edges. + """ + cdef: + # Note: we utilize ndarray's over memory-views to make use of numpy + # binary indexing and sub-selection below. + cnp.ndarray[int64_t, ndim=1, mode='c'] current_labels + cnp.ndarray[float64_t, ndim=1, mode='c'] min_reachability, left, right + cnp.ndarray[MST_edge_t, ndim=1, mode='c'] mst + + cnp.ndarray[uint8_t, mode='c'] label_filter + + int64_t n_samples = PyArray_SHAPE( mutual_reachability)[0] + int64_t current_node, new_node_index, new_node, i + + mst = np.empty(n_samples - 1, dtype=MST_edge_dtype) + current_labels = np.arange(n_samples, dtype=np.int64) + current_node = 0 + min_reachability = np.full(n_samples, fill_value=np.inf, dtype=np.float64) + for i in range(0, n_samples - 1): + label_filter = current_labels != current_node + current_labels = current_labels[label_filter] + left = min_reachability[label_filter] + right = mutual_reachability[current_node][current_labels] + min_reachability = np.minimum(left, right) + + new_node_index = np.argmin(min_reachability) + new_node = current_labels[new_node_index] + mst[i].current_node = current_node + mst[i].next_node = new_node + mst[i].distance = min_reachability[new_node_index] + current_node = new_node + + return mst + + +cpdef cnp.ndarray[MST_edge_t, ndim=1, mode='c'] mst_from_data_matrix( + const float64_t[:, ::1] raw_data, + const float64_t[::1] core_distances, + DistanceMetric64 dist_metric, + float64_t alpha=1.0 +): + """Compute the Minimum Spanning Tree (MST) representation of the mutual- + reachability graph generated from the provided `raw_data` and + `core_distances` using Prim's algorithm. + + Parameters + ---------- + raw_data : ndarray of shape (n_samples, n_features) + Input array of data samples. + + core_distances : ndarray of shape (n_samples,) + An array containing the core-distance calculated for each corresponding + sample. + + dist_metric : DistanceMetric + The distance metric to use when calculating pairwise distances for + determining mutual-reachability. + + Returns + ------- + mst : ndarray of shape (n_samples - 1,), dtype=MST_edge_dtype + The MST representation of the mutual-reachability graph. The MST is + represented as a collection of edges. + """ + + cdef: + uint8_t[::1] in_tree + float64_t[::1] min_reachability + int64_t[::1] current_sources + cnp.ndarray[MST_edge_t, ndim=1, mode='c'] mst + + int64_t current_node, source_node, new_node, next_node_source + int64_t i, j, n_samples, num_features + + float64_t current_node_core_dist, new_reachability, mutual_reachability_distance + float64_t next_node_min_reach, pair_distance, next_node_core_dist + + n_samples = raw_data.shape[0] + num_features = raw_data.shape[1] + + mst = np.empty(n_samples - 1, dtype=MST_edge_dtype) + + in_tree = np.zeros(n_samples, dtype=np.uint8) + min_reachability = np.full(n_samples, fill_value=np.inf, dtype=np.float64) + current_sources = np.ones(n_samples, dtype=np.int64) + + current_node = 0 + + # The following loop dynamically updates minimum reachability node-by-node, + # avoiding unnecessary computation where possible. + for i in range(0, n_samples - 1): + + in_tree[current_node] = 1 + + current_node_core_dist = core_distances[current_node] + + new_reachability = DBL_MAX + source_node = 0 + new_node = 0 + + for j in range(n_samples): + if in_tree[j]: + continue + + next_node_min_reach = min_reachability[j] + next_node_source = current_sources[j] + + pair_distance = dist_metric.dist( + &raw_data[current_node, 0], + &raw_data[j, 0], + num_features + ) + + pair_distance /= alpha + + next_node_core_dist = core_distances[j] + mutual_reachability_distance = max( + current_node_core_dist, + next_node_core_dist, + pair_distance + ) + + # If MRD(i, j) is smaller than node j's min_reachability, we update + # node j's min_reachability for future reference. + if mutual_reachability_distance < next_node_min_reach: + min_reachability[j] = mutual_reachability_distance + current_sources[j] = current_node + + # If MRD(i, j) is also smaller than node i's current + # min_reachability, we update and set their edge as the current + # MST edge candidate. + if mutual_reachability_distance < new_reachability: + new_reachability = mutual_reachability_distance + source_node = current_node + new_node = j + + # If the node j is closer to another node already in the tree, we + # make their edge the current MST candidate edge. + elif next_node_min_reach < new_reachability: + new_reachability = next_node_min_reach + source_node = next_node_source + new_node = j + + mst[i].current_node = source_node + mst[i].next_node = new_node + mst[i].distance = new_reachability + current_node = new_node + + return mst + +cpdef cnp.ndarray[HIERARCHY_t, ndim=1, mode="c"] make_single_linkage(const MST_edge_t[::1] mst): + """Construct a single-linkage tree from an MST. + + Parameters + ---------- + mst : ndarray of shape (n_samples - 1,), dtype=MST_edge_dtype + The MST representation of the mutual-reachability graph. The MST is + represented as a collection of edges. + + Returns + ------- + single_linkage : ndarray of shape (n_samples - 1,), dtype=HIERARCHY_dtype + The single-linkage tree tree (dendrogram) built from the MST. Each + of the array represents the following: + + - left node/cluster + - right node/cluster + - distance + - new cluster size + """ + cdef: + cnp.ndarray[HIERARCHY_t, ndim=1, mode="c"] single_linkage + + # Note mst.shape[0] is one fewer than the number of samples + int64_t n_samples = mst.shape[0] + 1 + intp_t current_node_cluster, next_node_cluster + int64_t current_node, next_node, i + float64_t distance + UnionFind U = UnionFind(n_samples) + + single_linkage = np.zeros(n_samples - 1, dtype=HIERARCHY_dtype) + + for i in range(n_samples - 1): + + current_node = mst[i].current_node + next_node = mst[i].next_node + distance = mst[i].distance + + current_node_cluster = U.fast_find(current_node) + next_node_cluster = U.fast_find(next_node) + + single_linkage[i].left_node = current_node_cluster + single_linkage[i].right_node = next_node_cluster + single_linkage[i].value = distance + single_linkage[i].cluster_size = U.size[current_node_cluster] + U.size[next_node_cluster] + + U.union(current_node_cluster, next_node_cluster) + + return single_linkage diff --git a/sklearn/cluster/_hdbscan/_reachability.pyx b/sklearn/cluster/_hdbscan/_reachability.pyx new file mode 100644 index 0000000000000..bff686ae0a636 --- /dev/null +++ b/sklearn/cluster/_hdbscan/_reachability.pyx @@ -0,0 +1,210 @@ +# mutual reachability distance computations + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +cimport numpy as cnp + +import numpy as np +from scipy.sparse import issparse +from cython cimport floating, integral +from libc.math cimport isfinite, INFINITY +from ...utils._typedefs cimport intp_t +cnp.import_array() + + +def mutual_reachability_graph( + distance_matrix, min_samples=5, max_distance=0.0 +): + """Compute the weighted adjacency matrix of the mutual reachability graph. + + The mutual reachability distance used to build the graph is defined as:: + + max(d_core(x_p), d_core(x_q), d(x_p, x_q)) + + and the core distance `d_core` is defined as the distance between a point + `x_p` and its k-th nearest neighbor. + + Note that all computations are done in-place. + + Parameters + ---------- + distance_matrix : {ndarray, sparse matrix} of shape (n_samples, n_samples) + Array of distances between samples. If sparse, the array must be in + `CSR` format. + + min_samples : int, default=5 + The parameter `k` used to calculate the distance between a point + `x_p` and its k-th nearest neighbor. + + max_distance : float, default=0.0 + The distance which `np.inf` is replaced with. When the true mutual- + reachability distance is measured to be infinite, it is instead + truncated to `max_dist`. Only used when `distance_matrix` is a sparse + matrix. + + Returns + ------- + mututal_reachability_graph: {ndarray, sparse matrix} of shape \ + (n_samples, n_samples) + Weighted adjacency matrix of the mutual reachability graph. + + References + ---------- + .. [1] Campello, R. J., Moulavi, D., & Sander, J. (2013, April). + Density-based clustering based on hierarchical density estimates. + In Pacific-Asia Conference on Knowledge Discovery and Data Mining + (pp. 160-172). Springer Berlin Heidelberg. + """ + further_neighbor_idx = min_samples - 1 + if issparse(distance_matrix): + if distance_matrix.format != "csr": + raise ValueError( + "Only sparse CSR matrices are supported for `distance_matrix`." + ) + _sparse_mutual_reachability_graph( + distance_matrix.data, + distance_matrix.indices, + distance_matrix.indptr, + distance_matrix.shape[0], + further_neighbor_idx=further_neighbor_idx, + max_distance=max_distance, + ) + else: + _dense_mutual_reachability_graph( + distance_matrix, further_neighbor_idx=further_neighbor_idx + ) + return distance_matrix + + +def _dense_mutual_reachability_graph( + floating[:, :] distance_matrix, + intp_t further_neighbor_idx, +): + """Dense implementation of mutual reachability graph. + + The computation is done in-place, i.e. the distance matrix is modified + directly. + + Parameters + ---------- + distance_matrix : ndarray of shape (n_samples, n_samples) + Array of distances between samples. + + further_neighbor_idx : int + The index of the furthest neighbor to use to define the core distances. + """ + cdef: + intp_t i, j, n_samples = distance_matrix.shape[0] + floating mutual_reachability_distance + floating[::1] core_distances + + # We assume that the distance matrix is symmetric. We choose to sort every + # row to have the same implementation than the sparse case that requires + # CSR matrix. + core_distances = np.ascontiguousarray( + np.partition( + distance_matrix, further_neighbor_idx, axis=1 + )[:, further_neighbor_idx] + ) + + with nogil: + # TODO: Update w/ prange with thread count based on + # _openmp_effective_n_threads + for i in range(n_samples): + for j in range(n_samples): + mutual_reachability_distance = max( + core_distances[i], + core_distances[j], + distance_matrix[i, j], + ) + distance_matrix[i, j] = mutual_reachability_distance + + +def _sparse_mutual_reachability_graph( + cnp.ndarray[floating, ndim=1, mode="c"] data, + cnp.ndarray[integral, ndim=1, mode="c"] indices, + cnp.ndarray[integral, ndim=1, mode="c"] indptr, + intp_t n_samples, + intp_t further_neighbor_idx, + floating max_distance, +): + """Sparse implementation of mutual reachability graph. + + The computation is done in-place, i.e. the distance matrix is modified + directly. This implementation only accepts `CSR` format sparse matrices. + + Parameters + ---------- + distance_matrix : sparse matrix of shape (n_samples, n_samples) + Sparse matrix of distances between samples. The sparse format should + be `CSR`. + + further_neighbor_idx : int + The index of the furthest neighbor to use to define the core distances. + + max_distance : float + The distance which `np.inf` is replaced with. When the true mutual- + reachability distance is measured to be infinite, it is instead + truncated to `max_dist`. Only used when `distance_matrix` is a sparse + matrix. + """ + cdef: + integral i, col_ind, row_ind + floating mutual_reachability_distance + floating[:] core_distances + floating[:] row_data + + if floating is float: + dtype = np.float32 + else: + dtype = np.float64 + + core_distances = np.empty(n_samples, dtype=dtype) + + for i in range(n_samples): + row_data = data[indptr[i]:indptr[i + 1]] + if further_neighbor_idx < row_data.size: + core_distances[i] = np.partition( + row_data, further_neighbor_idx + )[further_neighbor_idx] + else: + core_distances[i] = INFINITY + + with nogil: + for row_ind in range(n_samples): + for i in range(indptr[row_ind], indptr[row_ind + 1]): + col_ind = indices[i] + mutual_reachability_distance = max( + core_distances[row_ind], core_distances[col_ind], data[i] + ) + if isfinite(mutual_reachability_distance): + data[i] = mutual_reachability_distance + elif max_distance > 0: + data[i] = max_distance diff --git a/sklearn/cluster/_hdbscan/_tree.pxd b/sklearn/cluster/_hdbscan/_tree.pxd new file mode 100644 index 0000000000000..23708b9a38d07 --- /dev/null +++ b/sklearn/cluster/_hdbscan/_tree.pxd @@ -0,0 +1,49 @@ +# Copyright (c) 2015, Leland McInnes +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from ...utils._typedefs cimport intp_t, float64_t, uint8_t +cimport numpy as cnp + +# This corresponds to the scipy.cluster.hierarchy format +ctypedef packed struct HIERARCHY_t: + intp_t left_node + intp_t right_node + float64_t value + intp_t cluster_size + +# Effectively an edgelist encoding a parent/child pair, along with a value and +# the corresponding cluster_size in each row providing a tree structure. +ctypedef packed struct CONDENSED_t: + intp_t parent + intp_t child + float64_t value + intp_t cluster_size + +cdef extern from "numpy/arrayobject.h": + intp_t * PyArray_SHAPE(cnp.PyArrayObject *) diff --git a/sklearn/cluster/_hdbscan/_tree.pyx b/sklearn/cluster/_hdbscan/_tree.pyx new file mode 100644 index 0000000000000..161092033b915 --- /dev/null +++ b/sklearn/cluster/_hdbscan/_tree.pyx @@ -0,0 +1,799 @@ +# Tree handling (condensing, finding stable clusters) for hdbscan + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + + +cimport numpy as cnp +from libc.math cimport isinf +import cython + +import numpy as np + +cnp.import_array() + +cdef extern from "numpy/arrayobject.h": + intp_t * PyArray_SHAPE(cnp.PyArrayObject *) + +cdef cnp.float64_t INFTY = np.inf +cdef cnp.intp_t NOISE = -1 + +HIERARCHY_dtype = np.dtype([ + ("left_node", np.intp), + ("right_node", np.intp), + ("value", np.float64), + ("cluster_size", np.intp), +]) + +CONDENSED_dtype = np.dtype([ + ("parent", np.intp), + ("child", np.intp), + ("value", np.float64), + ("cluster_size", np.intp), +]) + +cpdef tuple tree_to_labels( + const HIERARCHY_t[::1] single_linkage_tree, + cnp.intp_t min_cluster_size=10, + cluster_selection_method="eom", + bint allow_single_cluster=False, + cnp.float64_t cluster_selection_epsilon=0.0, + max_cluster_size=None, +): + cdef: + cnp.ndarray[CONDENSED_t, ndim=1, mode='c'] condensed_tree + cnp.ndarray[cnp.intp_t, ndim=1, mode='c'] labels + cnp.ndarray[cnp.float64_t, ndim=1, mode='c'] probabilities + + condensed_tree = _condense_tree(single_linkage_tree, min_cluster_size) + labels, probabilities = _get_clusters( + condensed_tree, + _compute_stability(condensed_tree), + cluster_selection_method, + allow_single_cluster, + cluster_selection_epsilon, + max_cluster_size, + ) + + return (labels, probabilities) + +cdef list bfs_from_hierarchy( + const HIERARCHY_t[::1] hierarchy, + cnp.intp_t bfs_root +): + """ + Perform a breadth first search on a tree in scipy hclust format. + """ + + cdef list process_queue, next_queue, result + cdef cnp.intp_t n_samples = hierarchy.shape[0] + 1 + cdef cnp.intp_t node + process_queue = [bfs_root] + result = [] + + while process_queue: + result.extend(process_queue) + # By construction, node i is formed by the union of nodes + # hierarchy[i - n_samples, 0] and hierarchy[i - n_samples, 1] + process_queue = [ + x - n_samples + for x in process_queue + if x >= n_samples + ] + if process_queue: + next_queue = [] + for node in process_queue: + next_queue.extend( + [ + hierarchy[node].left_node, + hierarchy[node].right_node, + ] + ) + process_queue = next_queue + return result + + +cpdef cnp.ndarray[CONDENSED_t, ndim=1, mode='c'] _condense_tree( + const HIERARCHY_t[::1] hierarchy, + cnp.intp_t min_cluster_size=10 +): + """Condense a tree according to a minimum cluster size. This is akin + to the runt pruning procedure of Stuetzle. The result is a much simpler + tree that is easier to visualize. We include extra information on the + lambda value at which individual points depart clusters for later + analysis and computation. + + Parameters + ---------- + hierarchy : ndarray of shape (n_samples,), dtype=HIERARCHY_dtype + A single linkage hierarchy in scipy.cluster.hierarchy format. + + min_cluster_size : int, optional (default 10) + The minimum size of clusters to consider. Clusters smaller than this + are pruned from the tree. + + Returns + ------- + condensed_tree : ndarray of shape (n_samples,), dtype=CONDENSED_dtype + Effectively an edgelist encoding a parent/child pair, along with a + value and the corresponding cluster_size in each row providing a tree + structure. + """ + + cdef: + cnp.intp_t root = 2 * hierarchy.shape[0] + cnp.intp_t n_samples = hierarchy.shape[0] + 1 + cnp.intp_t next_label = n_samples + 1 + list result_list, node_list = bfs_from_hierarchy(hierarchy, root) + + cnp.intp_t[::1] relabel + cnp.uint8_t[::1] ignore + + cnp.intp_t node, sub_node, left, right + cnp.float64_t lambda_value, distance + cnp.intp_t left_count, right_count + HIERARCHY_t children + + relabel = np.empty(root + 1, dtype=np.intp) + relabel[root] = n_samples + result_list = [] + ignore = np.zeros(len(node_list), dtype=bool) + + for node in node_list: + if ignore[node] or node < n_samples: + continue + + children = hierarchy[node - n_samples] + left = children.left_node + right = children.right_node + distance = children.value + if distance > 0.0: + lambda_value = 1.0 / distance + else: + lambda_value = INFTY + + if left >= n_samples: + left_count = hierarchy[left - n_samples].cluster_size + else: + left_count = 1 + + if right >= n_samples: + right_count = hierarchy[right - n_samples].cluster_size + else: + right_count = 1 + + if left_count >= min_cluster_size and right_count >= min_cluster_size: + relabel[left] = next_label + next_label += 1 + result_list.append( + (relabel[node], relabel[left], lambda_value, left_count) + ) + + relabel[right] = next_label + next_label += 1 + result_list.append( + (relabel[node], relabel[right], lambda_value, right_count) + ) + + elif left_count < min_cluster_size and right_count < min_cluster_size: + for sub_node in bfs_from_hierarchy(hierarchy, left): + if sub_node < n_samples: + result_list.append( + (relabel[node], sub_node, lambda_value, 1) + ) + ignore[sub_node] = True + + for sub_node in bfs_from_hierarchy(hierarchy, right): + if sub_node < n_samples: + result_list.append( + (relabel[node], sub_node, lambda_value, 1) + ) + ignore[sub_node] = True + + elif left_count < min_cluster_size: + relabel[right] = relabel[node] + for sub_node in bfs_from_hierarchy(hierarchy, left): + if sub_node < n_samples: + result_list.append( + (relabel[node], sub_node, lambda_value, 1) + ) + ignore[sub_node] = True + + else: + relabel[left] = relabel[node] + for sub_node in bfs_from_hierarchy(hierarchy, right): + if sub_node < n_samples: + result_list.append( + (relabel[node], sub_node, lambda_value, 1) + ) + ignore[sub_node] = True + + return np.array(result_list, dtype=CONDENSED_dtype) + + +cdef dict _compute_stability( + cnp.ndarray[CONDENSED_t, ndim=1, mode='c'] condensed_tree +): + + cdef: + cnp.float64_t[::1] result, births + cnp.intp_t[:] parents = condensed_tree['parent'] + + cnp.intp_t parent, cluster_size, result_index, idx + cnp.float64_t lambda_val + CONDENSED_t condensed_node + cnp.intp_t largest_child = condensed_tree['child'].max() + cnp.intp_t smallest_cluster = np.min(parents) + cnp.intp_t num_clusters = np.max(parents) - smallest_cluster + 1 + dict stability_dict = {} + + largest_child = max(largest_child, smallest_cluster) + births = np.full(largest_child + 1, np.nan, dtype=np.float64) + + for idx in range(PyArray_SHAPE( condensed_tree)[0]): + condensed_node = condensed_tree[idx] + births[condensed_node.child] = condensed_node.value + + births[smallest_cluster] = 0.0 + + result = np.zeros(num_clusters, dtype=np.float64) + for idx in range(PyArray_SHAPE( condensed_tree)[0]): + condensed_node = condensed_tree[idx] + parent = condensed_node.parent + lambda_val = condensed_node.value + cluster_size = condensed_node.cluster_size + + result_index = parent - smallest_cluster + result[result_index] += (lambda_val - births[parent]) * cluster_size + + for idx in range(num_clusters): + stability_dict[idx + smallest_cluster] = result[idx] + + return stability_dict + + +cdef list bfs_from_cluster_tree( + cnp.ndarray[CONDENSED_t, ndim=1, mode='c'] condensed_tree, + cnp.intp_t bfs_root +): + + cdef: + list result = [] + cnp.ndarray[cnp.intp_t, ndim=1] process_queue = ( + np.array([bfs_root], dtype=np.intp) + ) + cnp.ndarray[cnp.intp_t, ndim=1] children = condensed_tree['child'] + cnp.intp_t[:] parents = condensed_tree['parent'] + + while len(process_queue) > 0: + result.extend(process_queue.tolist()) + process_queue = children[np.isin(parents, process_queue)] + + return result + + +cdef cnp.float64_t[::1] max_lambdas(cnp.ndarray[CONDENSED_t, ndim=1, mode='c'] condensed_tree): + + cdef: + cnp.intp_t parent, current_parent, idx + cnp.float64_t lambda_val, max_lambda + cnp.float64_t[::1] deaths + cnp.intp_t largest_parent = condensed_tree['parent'].max() + + deaths = np.zeros(largest_parent + 1, dtype=np.float64) + current_parent = condensed_tree[0].parent + max_lambda = condensed_tree[0].value + + for idx in range(1, PyArray_SHAPE( condensed_tree)[0]): + parent = condensed_tree[idx].parent + lambda_val = condensed_tree[idx].value + + if parent == current_parent: + max_lambda = max(max_lambda, lambda_val) + else: + deaths[current_parent] = max_lambda + current_parent = parent + max_lambda = lambda_val + + deaths[current_parent] = max_lambda # value for last parent + return deaths + + +@cython.final +cdef class TreeUnionFind: + + cdef cnp.intp_t[:, ::1] data + cdef cnp.uint8_t[::1] is_component + + def __init__(self, size): + cdef cnp.intp_t idx + self.data = np.zeros((size, 2), dtype=np.intp) + for idx in range(size): + self.data[idx, 0] = idx + self.is_component = np.ones(size, dtype=np.uint8) + + cdef void union(self, cnp.intp_t x, cnp.intp_t y): + cdef cnp.intp_t x_root = self.find(x) + cdef cnp.intp_t y_root = self.find(y) + + if self.data[x_root, 1] < self.data[y_root, 1]: + self.data[x_root, 0] = y_root + elif self.data[x_root, 1] > self.data[y_root, 1]: + self.data[y_root, 0] = x_root + else: + self.data[y_root, 0] = x_root + self.data[x_root, 1] += 1 + return + + cdef cnp.intp_t find(self, cnp.intp_t x): + if self.data[x, 0] != x: + self.data[x, 0] = self.find(self.data[x, 0]) + self.is_component[x] = False + return self.data[x, 0] + + +cpdef cnp.ndarray[cnp.intp_t, ndim=1, mode='c'] labelling_at_cut( + const HIERARCHY_t[::1] linkage, + cnp.float64_t cut, + cnp.intp_t min_cluster_size +): + """Given a single linkage tree and a cut value, return the + vector of cluster labels at that cut value. This is useful + for Robust Single Linkage, and extracting DBSCAN results + from a single HDBSCAN run. + + Parameters + ---------- + linkage : ndarray of shape (n_samples,), dtype=HIERARCHY_dtype + The single linkage tree in scipy.cluster.hierarchy format. + + cut : double + The cut value at which to find clusters. + + min_cluster_size : int + The minimum cluster size; clusters below this size at + the cut will be considered noise. + + Returns + ------- + labels : ndarray of shape (n_samples,) + The cluster labels for each point in the data set; + a label of -1 denotes a noise assignment. + """ + + cdef: + cnp.intp_t n, cluster, root, n_samples, cluster_label + cnp.intp_t[::1] unique_labels, cluster_size + cnp.ndarray[cnp.intp_t, ndim=1, mode='c'] result + TreeUnionFind union_find + dict cluster_label_map + HIERARCHY_t node + + root = 2 * linkage.shape[0] + n_samples = root // 2 + 1 + result = np.empty(n_samples, dtype=np.intp) + union_find = TreeUnionFind(root + 1) + + cluster = n_samples + for node in linkage: + if node.value < cut: + union_find.union(node.left_node, cluster) + union_find.union(node.right_node, cluster) + cluster += 1 + + cluster_size = np.zeros(cluster, dtype=np.intp) + for n in range(n_samples): + cluster = union_find.find(n) + cluster_size[cluster] += 1 + result[n] = cluster + + cluster_label_map = {-1: NOISE} + cluster_label = 0 + unique_labels = np.unique(result) + + for cluster in unique_labels: + if cluster_size[cluster] < min_cluster_size: + cluster_label_map[cluster] = NOISE + else: + cluster_label_map[cluster] = cluster_label + cluster_label += 1 + + for n in range(n_samples): + result[n] = cluster_label_map[result[n]] + + return result + + +cpdef cnp.ndarray[cnp.intp_t, ndim=1, mode='c'] _do_labelling( + cnp.ndarray[CONDENSED_t, ndim=1, mode='c'] condensed_tree, + set clusters, + dict cluster_label_map, + cnp.intp_t allow_single_cluster, + cnp.float64_t cluster_selection_epsilon +): + """Given a condensed tree, clusters and a labeling map for the clusters, + return an array containing the labels of each point based on cluster + membership. Note that this is where points may be marked as noisy + outliers. The determination of some points as noise is in large, single- + cluster datasets is controlled by the `allow_single_cluster` and + `cluster_selection_epsilon` parameters. + + Parameters + ---------- + condensed_tree : ndarray of shape (n_samples,), dtype=CONDENSED_dtype + Effectively an edgelist encoding a parent/child pair, along with a + value and the corresponding cluster_size in each row providing a tree + structure. + + clusters : set + The set of nodes corresponding to identified clusters. These node + values should be the same as those present in `condensed_tree`. + + cluster_label_map : dict + A mapping from the node values present in `clusters` to the labels + which will be returned. + + Returns + ------- + labels : ndarray of shape (n_samples,) + The cluster labels for each point in the data set; + a label of -1 denotes a noise assignment. + """ + + cdef: + cnp.intp_t root_cluster + cnp.ndarray[cnp.intp_t, ndim=1, mode='c'] result + cnp.ndarray[cnp.intp_t, ndim=1] parent_array, child_array + cnp.ndarray[cnp.float64_t, ndim=1] lambda_array + TreeUnionFind union_find + cnp.intp_t n, parent, child, cluster + cnp.float64_t threshold + + child_array = condensed_tree['child'] + parent_array = condensed_tree['parent'] + lambda_array = condensed_tree['value'] + + root_cluster = np.min(parent_array) + result = np.empty(root_cluster, dtype=np.intp) + union_find = TreeUnionFind(np.max(parent_array) + 1) + + for n in range(PyArray_SHAPE( condensed_tree)[0]): + child = child_array[n] + parent = parent_array[n] + if child not in clusters: + union_find.union(parent, child) + + for n in range(root_cluster): + cluster = union_find.find(n) + label = NOISE + if cluster != root_cluster: + label = cluster_label_map[cluster] + elif len(clusters) == 1 and allow_single_cluster: + # There can only be one edge with this particular child hence this + # expression extracts a unique, scalar lambda value. + parent_lambda = lambda_array[child_array == n] + if cluster_selection_epsilon != 0.0: + threshold = 1 / cluster_selection_epsilon + else: + # The threshold should be calculated per-sample based on the + # largest lambda of any simbling node. + threshold = lambda_array[parent_array == cluster].max() + if parent_lambda >= threshold: + label = cluster_label_map[cluster] + + result[n] = label + + return result + + +cdef cnp.ndarray[cnp.float64_t, ndim=1, mode='c'] get_probabilities( + cnp.ndarray[CONDENSED_t, ndim=1, mode='c'] condensed_tree, + dict cluster_map, + cnp.intp_t[::1] labels +): + + cdef: + cnp.ndarray[cnp.float64_t, ndim=1, mode='c'] result + cnp.float64_t[:] lambda_array + cnp.float64_t[::1] deaths + cnp.intp_t[:] child_array, parent_array + cnp.intp_t root_cluster, n, point, cluster_num, cluster + cnp.float64_t max_lambda, lambda_val + + child_array = condensed_tree['child'] + parent_array = condensed_tree['parent'] + lambda_array = condensed_tree['value'] + + result = np.zeros(labels.shape[0]) + deaths = max_lambdas(condensed_tree) + root_cluster = np.min(parent_array) + + for n in range(PyArray_SHAPE( condensed_tree)[0]): + point = child_array[n] + if point >= root_cluster: + continue + + cluster_num = labels[point] + if cluster_num == -1: + continue + + cluster = cluster_map[cluster_num] + max_lambda = deaths[cluster] + if max_lambda == 0.0 or isinf(lambda_array[n]): + result[point] = 1.0 + else: + lambda_val = min(lambda_array[n], max_lambda) + result[point] = lambda_val / max_lambda + + return result + + +cpdef list recurse_leaf_dfs( + cnp.ndarray[CONDENSED_t, ndim=1, mode='c'] cluster_tree, + cnp.intp_t current_node +): + cdef cnp.intp_t[:] children + cdef cnp.intp_t child + + children = cluster_tree[cluster_tree['parent'] == current_node]['child'] + if children.shape[0] == 0: + return [current_node,] + else: + return sum([recurse_leaf_dfs(cluster_tree, child) for child in children], []) + + +cpdef list get_cluster_tree_leaves(cnp.ndarray[CONDENSED_t, ndim=1, mode='c'] cluster_tree): + cdef cnp.intp_t root + if PyArray_SHAPE( cluster_tree)[0] == 0: + return [] + root = cluster_tree['parent'].min() + return recurse_leaf_dfs(cluster_tree, root) + +cdef cnp.intp_t traverse_upwards( + cnp.ndarray[CONDENSED_t, ndim=1, mode='c'] cluster_tree, + cnp.float64_t cluster_selection_epsilon, + cnp.intp_t leaf, + cnp.intp_t allow_single_cluster +): + cdef cnp.intp_t root, parent + cdef cnp.float64_t parent_eps + + root = cluster_tree['parent'].min() + parent = cluster_tree[cluster_tree['child'] == leaf]['parent'] + if parent == root: + if allow_single_cluster: + return parent + else: + return leaf # return node closest to root + + parent_eps = 1 / cluster_tree[cluster_tree['child'] == parent]['value'] + if parent_eps > cluster_selection_epsilon: + return parent + else: + return traverse_upwards( + cluster_tree, + cluster_selection_epsilon, + parent, + allow_single_cluster + ) + +cdef set epsilon_search( + set leaves, + cnp.ndarray[CONDENSED_t, ndim=1, mode='c'] cluster_tree, + cnp.float64_t cluster_selection_epsilon, + cnp.intp_t allow_single_cluster +): + cdef: + list selected_clusters = list() + list processed = list() + cnp.intp_t leaf, epsilon_child, sub_node + cnp.float64_t eps + cnp.uint8_t[:] leaf_nodes + cnp.ndarray[cnp.intp_t, ndim=1] children = cluster_tree['child'] + cnp.ndarray[cnp.float64_t, ndim=1] distances = cluster_tree['value'] + + for leaf in leaves: + leaf_nodes = children == leaf + eps = 1 / distances[leaf_nodes][0] + if eps < cluster_selection_epsilon: + if leaf not in processed: + epsilon_child = traverse_upwards( + cluster_tree, + cluster_selection_epsilon, + leaf, + allow_single_cluster + ) + selected_clusters.append(epsilon_child) + + for sub_node in bfs_from_cluster_tree(cluster_tree, epsilon_child): + if sub_node != epsilon_child: + processed.append(sub_node) + else: + selected_clusters.append(leaf) + + return set(selected_clusters) + + +@cython.wraparound(True) +cdef tuple _get_clusters( + cnp.ndarray[CONDENSED_t, ndim=1, mode='c'] condensed_tree, + dict stability, + cluster_selection_method='eom', + cnp.uint8_t allow_single_cluster=False, + cnp.float64_t cluster_selection_epsilon=0.0, + max_cluster_size=None +): + """Given a tree and stability dict, produce the cluster labels + (and probabilities) for a flat clustering based on the chosen + cluster selection method. + + Parameters + ---------- + condensed_tree : ndarray of shape (n_samples,), dtype=CONDENSED_dtype + Effectively an edgelist encoding a parent/child pair, along with a + value and the corresponding cluster_size in each row providing a tree + structure. + + stability : dict + A dictionary mapping cluster_ids to stability values + + cluster_selection_method : string, optional (default 'eom') + The method of selecting clusters. The default is the + Excess of Mass algorithm specified by 'eom'. The alternate + option is 'leaf'. + + allow_single_cluster : boolean, optional (default False) + Whether to allow a single cluster to be selected by the + Excess of Mass algorithm. + + cluster_selection_epsilon: double, optional (default 0.0) + A distance threshold for cluster splits. + + max_cluster_size: int, default=None + The maximum size for clusters located by the EOM clusterer. Can + be overridden by the cluster_selection_epsilon parameter in + rare cases. + + Returns + ------- + labels : ndarray of shape (n_samples,) + An integer array of cluster labels, with -1 denoting noise. + + probabilities : ndarray (n_samples,) + The cluster membership strength of each sample. + + stabilities : ndarray (n_clusters,) + The cluster coherence strengths of each cluster. + """ + cdef: + list node_list + cnp.ndarray[CONDENSED_t, ndim=1, mode='c'] cluster_tree + cnp.uint8_t[::1] child_selection + cnp.ndarray[cnp.intp_t, ndim=1, mode='c'] labels + dict is_cluster, cluster_sizes + cnp.float64_t subtree_stability + cnp.intp_t node, sub_node, cluster, n_samples + cnp.ndarray[cnp.float64_t, ndim=1, mode='c'] probs + + # Assume clusters are ordered by numeric id equivalent to + # a topological sort of the tree; This is valid given the + # current implementation above, so don't change that ... or + # if you do, change this accordingly! + if allow_single_cluster: + node_list = sorted(stability.keys(), reverse=True) + else: + node_list = sorted(stability.keys(), reverse=True)[:-1] + # (exclude root) + + cluster_tree = condensed_tree[condensed_tree['cluster_size'] > 1] + is_cluster = {cluster: True for cluster in node_list} + n_samples = np.max(condensed_tree[condensed_tree['cluster_size'] == 1]['child']) + 1 + + if max_cluster_size is None: + max_cluster_size = n_samples + 1 # Set to a value that will never be triggered + cluster_sizes = { + child: cluster_size for child, cluster_size + in zip(cluster_tree['child'], cluster_tree['cluster_size']) + } + if allow_single_cluster: + # Compute cluster size for the root node + cluster_sizes[node_list[-1]] = np.sum( + cluster_tree[cluster_tree['parent'] == node_list[-1]]['cluster_size']) + + if cluster_selection_method == 'eom': + for node in node_list: + child_selection = (cluster_tree['parent'] == node) + subtree_stability = np.sum([ + stability[child] for + child in cluster_tree['child'][child_selection]]) + if subtree_stability > stability[node] or cluster_sizes[node] > max_cluster_size: + is_cluster[node] = False + stability[node] = subtree_stability + else: + for sub_node in bfs_from_cluster_tree(cluster_tree, node): + if sub_node != node: + is_cluster[sub_node] = False + + if cluster_selection_epsilon != 0.0 and PyArray_SHAPE( cluster_tree)[0] > 0: + eom_clusters = [c for c in is_cluster if is_cluster[c]] + selected_clusters = [] + # first check if eom_clusters only has root node, which skips epsilon check. + if (len(eom_clusters) == 1 and eom_clusters[0] == cluster_tree['parent'].min()): + if allow_single_cluster: + selected_clusters = eom_clusters + else: + selected_clusters = epsilon_search( + set(eom_clusters), + cluster_tree, + cluster_selection_epsilon, + allow_single_cluster + ) + for c in is_cluster: + if c in selected_clusters: + is_cluster[c] = True + else: + is_cluster[c] = False + + elif cluster_selection_method == 'leaf': + leaves = set(get_cluster_tree_leaves(cluster_tree)) + if len(leaves) == 0: + for c in is_cluster: + is_cluster[c] = False + is_cluster[condensed_tree['parent'].min()] = True + + if cluster_selection_epsilon != 0.0: + selected_clusters = epsilon_search( + leaves, + cluster_tree, + cluster_selection_epsilon, + allow_single_cluster + ) + else: + selected_clusters = leaves + + for c in is_cluster: + if c in selected_clusters: + is_cluster[c] = True + else: + is_cluster[c] = False + + clusters = set([c for c in is_cluster if is_cluster[c]]) + cluster_map = {c: n for n, c in enumerate(sorted(list(clusters)))} + reverse_cluster_map = {n: c for c, n in cluster_map.items()} + + labels = _do_labelling( + condensed_tree, + clusters, + cluster_map, + allow_single_cluster, + cluster_selection_epsilon + ) + probs = get_probabilities(condensed_tree, reverse_cluster_map, labels) + + return (labels, probs) diff --git a/sklearn/cluster/_hdbscan/hdbscan.py b/sklearn/cluster/_hdbscan/hdbscan.py new file mode 100644 index 0000000000000..f292a1f65909b --- /dev/null +++ b/sklearn/cluster/_hdbscan/hdbscan.py @@ -0,0 +1,1000 @@ +""" +HDBSCAN: Hierarchical Density-Based Spatial Clustering + of Applications with Noise +""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from numbers import Integral, Real +from warnings import warn + +import numpy as np +from scipy.sparse import csgraph, issparse + +from ...base import BaseEstimator, ClusterMixin, _fit_context +from ...metrics import pairwise_distances +from ...metrics._dist_metrics import DistanceMetric +from ...metrics.pairwise import _VALID_METRICS +from ...neighbors import BallTree, KDTree, NearestNeighbors +from ...utils._param_validation import Interval, StrOptions +from ...utils.validation import ( + _allclose_dense_sparse, + _assert_all_finite, + validate_data, +) +from ._linkage import ( + MST_edge_dtype, + make_single_linkage, + mst_from_data_matrix, + mst_from_mutual_reachability, +) +from ._reachability import mutual_reachability_graph +from ._tree import HIERARCHY_dtype, labelling_at_cut, tree_to_labels + +FAST_METRICS = set(KDTree.valid_metrics + BallTree.valid_metrics) + +# Encodings are arbitrary but must be strictly negative. +# The current encodings are chosen as extensions to the -1 noise label. +# Avoided enums so that the end user only deals with simple labels. +_OUTLIER_ENCODING: dict = { + "infinite": { + "label": -2, + # The probability could also be 1, since infinite points are certainly + # infinite outliers, however 0 is convention from the HDBSCAN library + # implementation. + "prob": 0, + }, + "missing": { + "label": -3, + # A nan probability is chosen to emphasize the fact that the + # corresponding data was not considered in the clustering problem. + "prob": np.nan, + }, +} + + +def _brute_mst(mutual_reachability, min_samples): + """ + Builds a minimum spanning tree (MST) from the provided mutual-reachability + values. This function dispatches to a custom Cython implementation for + dense arrays, and `scipy.sparse.csgraph.minimum_spanning_tree` for sparse + arrays/matrices. + + Parameters + ---------- + mututal_reachability_graph: {ndarray, sparse matrix} of shape \ + (n_samples, n_samples) + Weighted adjacency matrix of the mutual reachability graph. + + min_samples : int, default=None + The number of samples in a neighborhood for a point + to be considered as a core point. This includes the point itself. + + Returns + ------- + mst : ndarray of shape (n_samples - 1,), dtype=MST_edge_dtype + The MST representation of the mutual-reachability graph. The MST is + represented as a collection of edges. + """ + if not issparse(mutual_reachability): + return mst_from_mutual_reachability(mutual_reachability) + + # Check if the mutual reachability matrix has any rows which have + # less than `min_samples` non-zero elements. + indptr = mutual_reachability.indptr + num_points = mutual_reachability.shape[0] + if any((indptr[i + 1] - indptr[i]) < min_samples for i in range(num_points)): + raise ValueError( + f"There exists points with fewer than {min_samples} neighbors. Ensure" + " your distance matrix has non-zero values for at least" + f" `min_sample`={min_samples} neighbors for each points (i.e. K-nn" + " graph), or specify a `max_distance` in `metric_params` to use when" + " distances are missing." + ) + # Check connected component on mutual reachability. + # If more than one connected component is present, + # it means that the graph is disconnected. + n_components = csgraph.connected_components( + mutual_reachability, directed=False, return_labels=False + ) + if n_components > 1: + raise ValueError( + f"Sparse mutual reachability matrix has {n_components} connected" + " components. HDBSCAN cannot be performed on a disconnected graph. Ensure" + " that the sparse distance matrix has only one connected component." + ) + + # Compute the minimum spanning tree for the sparse graph + sparse_min_spanning_tree = csgraph.minimum_spanning_tree(mutual_reachability) + rows, cols = sparse_min_spanning_tree.nonzero() + mst = np.rec.fromarrays( + [rows, cols, sparse_min_spanning_tree.data], + dtype=MST_edge_dtype, + ) + return mst + + +def _process_mst(min_spanning_tree): + """ + Builds a single-linkage tree (SLT) from the provided minimum spanning tree + (MST). The MST is first sorted then processed by a custom Cython routine. + + Parameters + ---------- + min_spanning_tree : ndarray of shape (n_samples - 1,), dtype=MST_edge_dtype + The MST representation of the mutual-reachability graph. The MST is + represented as a collection of edges. + + Returns + ------- + single_linkage : ndarray of shape (n_samples - 1,), dtype=HIERARCHY_dtype + The single-linkage tree tree (dendrogram) built from the MST. + """ + # Sort edges of the min_spanning_tree by weight + row_order = np.argsort(min_spanning_tree["distance"]) + min_spanning_tree = min_spanning_tree[row_order] + # Convert edge list into standard hierarchical clustering format + return make_single_linkage(min_spanning_tree) + + +def _hdbscan_brute( + X, + min_samples=5, + alpha=None, + metric="euclidean", + n_jobs=None, + copy=False, + **metric_params, +): + """ + Builds a single-linkage tree (SLT) from the input data `X`. If + `metric="precomputed"` then `X` must be a symmetric array of distances. + Otherwise, the pairwise distances are calculated directly and passed to + `mutual_reachability_graph`. + + Parameters + ---------- + X : ndarray of shape (n_samples, n_features) or (n_samples, n_samples) + Either the raw data from which to compute the pairwise distances, + or the precomputed distances. + + min_samples : int, default=None + The number of samples in a neighborhood for a point + to be considered as a core point. This includes the point itself. + + alpha : float, default=1.0 + A distance scaling parameter as used in robust single linkage. + + metric : str or callable, default='euclidean' + The metric to use when calculating distance between instances in a + feature array. + + - If metric is a string or callable, it must be one of + the options allowed by :func:`~sklearn.metrics.pairwise_distances` + for its metric parameter. + + - If metric is "precomputed", X is assumed to be a distance matrix and + must be square. + + n_jobs : int, default=None + The number of jobs to use for computing the pairwise distances. This + works by breaking down the pairwise matrix into n_jobs even slices and + computing them in parallel. This parameter is passed directly to + :func:`~sklearn.metrics.pairwise_distances`. + + ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. + ``-1`` means using all processors. See :term:`Glossary ` + for more details. + + copy : bool, default=False + If `copy=True` then any time an in-place modifications would be made + that would overwrite `X`, a copy will first be made, guaranteeing that + the original data will be unchanged. Currently, it only applies when + `metric="precomputed"`, when passing a dense array or a CSR sparse + array/matrix. + + metric_params : dict, default=None + Arguments passed to the distance metric. + + Returns + ------- + single_linkage : ndarray of shape (n_samples - 1,), dtype=HIERARCHY_dtype + The single-linkage tree tree (dendrogram) built from the MST. + """ + if metric == "precomputed": + if X.shape[0] != X.shape[1]: + raise ValueError( + "The precomputed distance matrix is expected to be symmetric, however" + f" it has shape {X.shape}. Please verify that the" + " distance matrix was constructed correctly." + ) + if not _allclose_dense_sparse(X, X.T): + raise ValueError( + "The precomputed distance matrix is expected to be symmetric, however" + " its values appear to be asymmetric. Please verify that the distance" + " matrix was constructed correctly." + ) + + distance_matrix = X.copy() if copy else X + else: + distance_matrix = pairwise_distances( + X, metric=metric, n_jobs=n_jobs, **metric_params + ) + distance_matrix /= alpha + + max_distance = metric_params.get("max_distance", 0.0) + if issparse(distance_matrix) and distance_matrix.format != "csr": + # we need CSR format to avoid a conversion in `_brute_mst` when calling + # `csgraph.connected_components` + distance_matrix = distance_matrix.tocsr() + + # Note that `distance_matrix` is manipulated in-place, however we do not + # need it for anything else past this point, hence the operation is safe. + mutual_reachability_ = mutual_reachability_graph( + distance_matrix, min_samples=min_samples, max_distance=max_distance + ) + min_spanning_tree = _brute_mst(mutual_reachability_, min_samples=min_samples) + # Warn if the MST couldn't be constructed around the missing distances + if np.isinf(min_spanning_tree["distance"]).any(): + warn( + ( + "The minimum spanning tree contains edge weights with value " + "infinity. Potentially, you are missing too many distances " + "in the initial distance matrix for the given neighborhood " + "size." + ), + UserWarning, + ) + return _process_mst(min_spanning_tree) + + +def _hdbscan_prims( + X, + algo, + min_samples=5, + alpha=1.0, + metric="euclidean", + leaf_size=40, + n_jobs=None, + **metric_params, +): + """ + Builds a single-linkage tree (SLT) from the input data `X`. If + `metric="precomputed"` then `X` must be a symmetric array of distances. + Otherwise, the pairwise distances are calculated directly and passed to + `mutual_reachability_graph`. + + Parameters + ---------- + X : ndarray of shape (n_samples, n_features) + The raw data. + + min_samples : int, default=None + The number of samples in a neighborhood for a point + to be considered as a core point. This includes the point itself. + + alpha : float, default=1.0 + A distance scaling parameter as used in robust single linkage. + + metric : str or callable, default='euclidean' + The metric to use when calculating distance between instances in a + feature array. `metric` must be one of the options allowed by + :func:`~sklearn.metrics.pairwise_distances` for its metric + parameter. + + n_jobs : int, default=None + The number of jobs to use for computing the pairwise distances. This + works by breaking down the pairwise matrix into n_jobs even slices and + computing them in parallel. This parameter is passed directly to + :func:`~sklearn.metrics.pairwise_distances`. + + ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. + ``-1`` means using all processors. See :term:`Glossary ` + for more details. + + copy : bool, default=False + If `copy=True` then any time an in-place modifications would be made + that would overwrite `X`, a copy will first be made, guaranteeing that + the original data will be unchanged. Currently, it only applies when + `metric="precomputed"`, when passing a dense array or a CSR sparse + array/matrix. + + metric_params : dict, default=None + Arguments passed to the distance metric. + + Returns + ------- + single_linkage : ndarray of shape (n_samples - 1,), dtype=HIERARCHY_dtype + The single-linkage tree tree (dendrogram) built from the MST. + """ + # The Cython routines used require contiguous arrays + X = np.asarray(X, order="C") + + # Get distance to kth nearest neighbour + nbrs = NearestNeighbors( + n_neighbors=min_samples, + algorithm=algo, + leaf_size=leaf_size, + metric=metric, + metric_params=metric_params, + n_jobs=n_jobs, + p=None, + ).fit(X) + + neighbors_distances, _ = nbrs.kneighbors(X, min_samples, return_distance=True) + core_distances = np.ascontiguousarray(neighbors_distances[:, -1]) + dist_metric = DistanceMetric.get_metric(metric, **metric_params) + + # Mutual reachability distance is implicit in mst_from_data_matrix + min_spanning_tree = mst_from_data_matrix(X, core_distances, dist_metric, alpha) + return _process_mst(min_spanning_tree) + + +def remap_single_linkage_tree(tree, internal_to_raw, non_finite): + """ + Takes an internal single_linkage_tree structure and adds back in a set of points + that were initially detected as non-finite and returns that new tree. + These points will all be merged into the final node at np.inf distance and + considered noise points. + + Parameters + ---------- + tree : ndarray of shape (n_samples - 1,), dtype=HIERARCHY_dtype + The single-linkage tree tree (dendrogram) built from the MST. + internal_to_raw: dict + A mapping from internal integer index to the raw integer index + non_finite : ndarray + Boolean array of which entries in the raw data are non-finite + """ + finite_count = len(internal_to_raw) + + outlier_count = len(non_finite) + for i, _ in enumerate(tree): + left = tree[i]["left_node"] + right = tree[i]["right_node"] + + if left < finite_count: + tree[i]["left_node"] = internal_to_raw[left] + else: + tree[i]["left_node"] = left + outlier_count + if right < finite_count: + tree[i]["right_node"] = internal_to_raw[right] + else: + tree[i]["right_node"] = right + outlier_count + + outlier_tree = np.zeros(len(non_finite), dtype=HIERARCHY_dtype) + last_cluster_id = max( + tree[tree.shape[0] - 1]["left_node"], tree[tree.shape[0] - 1]["right_node"] + ) + last_cluster_size = tree[tree.shape[0] - 1]["cluster_size"] + for i, outlier in enumerate(non_finite): + outlier_tree[i] = (outlier, last_cluster_id + 1, np.inf, last_cluster_size + 1) + last_cluster_id += 1 + last_cluster_size += 1 + tree = np.concatenate([tree, outlier_tree]) + return tree + + +def _get_finite_row_indices(matrix): + """ + Returns the indices of the purely finite rows of a + sparse matrix or dense ndarray + """ + if issparse(matrix): + row_indices = np.array( + [i for i, row in enumerate(matrix.tolil().data) if np.all(np.isfinite(row))] + ) + else: + (row_indices,) = np.isfinite(matrix.sum(axis=1)).nonzero() + return row_indices + + +class HDBSCAN(ClusterMixin, BaseEstimator): + """Cluster data using hierarchical density-based clustering. + + HDBSCAN - Hierarchical Density-Based Spatial Clustering of Applications + with Noise. Performs :class:`~sklearn.cluster.DBSCAN` over varying epsilon + values and integrates the result to find a clustering that gives the best + stability over epsilon. + This allows HDBSCAN to find clusters of varying densities (unlike + :class:`~sklearn.cluster.DBSCAN`), and be more robust to parameter selection. + Read more in the :ref:`User Guide `. + + .. versionadded:: 1.3 + + Parameters + ---------- + min_cluster_size : int, default=5 + The minimum number of samples in a group for that group to be + considered a cluster; groupings smaller than this size will be left + as noise. + + min_samples : int, default=None + The parameter `k` used to calculate the distance between a point + `x_p` and its k-th nearest neighbor. + When `None`, defaults to `min_cluster_size`. + + cluster_selection_epsilon : float, default=0.0 + A distance threshold. Clusters below this value will be merged. + See [5]_ for more information. + + max_cluster_size : int, default=None + A limit to the size of clusters returned by the `"eom"` cluster + selection algorithm. There is no limit when `max_cluster_size=None`. + Has no effect if `cluster_selection_method="leaf"`. + + metric : str or callable, default='euclidean' + The metric to use when calculating distance between instances in a + feature array. + + - If metric is a string or callable, it must be one of + the options allowed by :func:`~sklearn.metrics.pairwise_distances` + for its metric parameter. + + - If metric is "precomputed", X is assumed to be a distance matrix and + must be square. + + metric_params : dict, default=None + Arguments passed to the distance metric. + + alpha : float, default=1.0 + A distance scaling parameter as used in robust single linkage. + See [3]_ for more information. + + algorithm : {"auto", "brute", "kd_tree", "ball_tree"}, default="auto" + Exactly which algorithm to use for computing core distances; By default + this is set to `"auto"` which attempts to use a + :class:`~sklearn.neighbors.KDTree` tree if possible, otherwise it uses + a :class:`~sklearn.neighbors.BallTree` tree. Both `"kd_tree"` and + `"ball_tree"` algorithms use the + :class:`~sklearn.neighbors.NearestNeighbors` estimator. + + If the `X` passed during `fit` is sparse or `metric` is invalid for + both :class:`~sklearn.neighbors.KDTree` and + :class:`~sklearn.neighbors.BallTree`, then it resolves to use the + `"brute"` algorithm. + + leaf_size : int, default=40 + Leaf size for trees responsible for fast nearest neighbour queries when + a KDTree or a BallTree are used as core-distance algorithms. A large + dataset size and small `leaf_size` may induce excessive memory usage. + If you are running out of memory consider increasing the `leaf_size` + parameter. Ignored for `algorithm="brute"`. + + n_jobs : int, default=None + Number of jobs to run in parallel to calculate distances. + `None` means 1 unless in a :obj:`joblib.parallel_backend` context. + `-1` means using all processors. See :term:`Glossary ` + for more details. + + cluster_selection_method : {"eom", "leaf"}, default="eom" + The method used to select clusters from the condensed tree. The + standard approach for HDBSCAN* is to use an Excess of Mass (`"eom"`) + algorithm to find the most persistent clusters. Alternatively you can + instead select the clusters at the leaves of the tree -- this provides + the most fine grained and homogeneous clusters. + + allow_single_cluster : bool, default=False + By default HDBSCAN* will not produce a single cluster, setting this + to True will override this and allow single cluster results in + the case that you feel this is a valid result for your dataset. + + store_centers : str, default=None + Which, if any, cluster centers to compute and store. The options are: + + - `None` which does not compute nor store any centers. + - `"centroid"` which calculates the center by taking the weighted + average of their positions. Note that the algorithm uses the + euclidean metric and does not guarantee that the output will be + an observed data point. + - `"medoid"` which calculates the center by taking the point in the + fitted data which minimizes the distance to all other points in + the cluster. This is slower than "centroid" since it requires + computing additional pairwise distances between points of the + same cluster but guarantees the output is an observed data point. + The medoid is also well-defined for arbitrary metrics, and does not + depend on a euclidean metric. + - `"both"` which computes and stores both forms of centers. + + copy : bool, default=False + If `copy=True` then any time an in-place modifications would be made + that would overwrite data passed to :term:`fit`, a copy will first be + made, guaranteeing that the original data will be unchanged. + Currently, it only applies when `metric="precomputed"`, when passing + a dense array or a CSR sparse matrix and when `algorithm="brute"`. + + Attributes + ---------- + labels_ : ndarray of shape (n_samples,) + Cluster labels for each point in the dataset given to :term:`fit`. + Outliers are labeled as follows: + + - Noisy samples are given the label -1. + - Samples with infinite elements (+/- np.inf) are given the label -2. + - Samples with missing data are given the label -3, even if they + also have infinite elements. + + probabilities_ : ndarray of shape (n_samples,) + The strength with which each sample is a member of its assigned + cluster. + + - Clustered samples have probabilities proportional to the degree that + they persist as part of the cluster. + - Noisy samples have probability zero. + - Samples with infinite elements (+/- np.inf) have probability 0. + - Samples with missing data have probability `np.nan`. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + centroids_ : ndarray of shape (n_clusters, n_features) + A collection containing the centroid of each cluster calculated under + the standard euclidean metric. The centroids may fall "outside" their + respective clusters if the clusters themselves are non-convex. + + Note that `n_clusters` only counts non-outlier clusters. That is to + say, the `-1, -2, -3` labels for the outlier clusters are excluded. + + medoids_ : ndarray of shape (n_clusters, n_features) + A collection containing the medoid of each cluster calculated under + the whichever metric was passed to the `metric` parameter. The + medoids are points in the original cluster which minimize the average + distance to all other points in that cluster under the chosen metric. + These can be thought of as the result of projecting the `metric`-based + centroid back onto the cluster. + + Note that `n_clusters` only counts non-outlier clusters. That is to + say, the `-1, -2, -3` labels for the outlier clusters are excluded. + + See Also + -------- + DBSCAN : Density-Based Spatial Clustering of Applications + with Noise. + OPTICS : Ordering Points To Identify the Clustering Structure. + Birch : Memory-efficient, online-learning algorithm. + + Notes + ----- + The `min_samples` parameter includes the point itself, whereas the implementation in + `scikit-learn-contrib/hdbscan `_ + does not. To get the same results in both versions, the value of `min_samples` here + must be 1 greater than the value used in `scikit-learn-contrib/hdbscan + `_. + + References + ---------- + + .. [1] :doi:`Campello, R. J., Moulavi, D., & Sander, J. Density-based clustering + based on hierarchical density estimates. + <10.1007/978-3-642-37456-2_14>` + .. [2] :doi:`Campello, R. J., Moulavi, D., Zimek, A., & Sander, J. + Hierarchical density estimates for data clustering, visualization, + and outlier detection.<10.1145/2733381>` + + .. [3] `Chaudhuri, K., & Dasgupta, S. Rates of convergence for the + cluster tree. + `_ + + .. [4] `Moulavi, D., Jaskowiak, P.A., Campello, R.J., Zimek, A. and + Sander, J. Density-Based Clustering Validation. + `_ + + .. [5] :arxiv:`Malzer, C., & Baum, M. "A Hybrid Approach To Hierarchical + Density-based Cluster Selection."<1911.02282>`. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.cluster import HDBSCAN + >>> from sklearn.datasets import load_digits + >>> X, _ = load_digits(return_X_y=True) + >>> hdb = HDBSCAN(min_cluster_size=20) + >>> hdb.fit(X) + HDBSCAN(min_cluster_size=20) + >>> hdb.labels_.shape == (X.shape[0],) + True + >>> np.unique(hdb.labels_).tolist() + [-1, 0, 1, 2, 3, 4, 5, 6, 7] + """ + + _parameter_constraints = { + "min_cluster_size": [Interval(Integral, left=2, right=None, closed="left")], + "min_samples": [Interval(Integral, left=1, right=None, closed="left"), None], + "cluster_selection_epsilon": [ + Interval(Real, left=0, right=None, closed="left") + ], + "max_cluster_size": [ + None, + Interval(Integral, left=1, right=None, closed="left"), + ], + "metric": [ + StrOptions(FAST_METRICS | set(_VALID_METRICS) | {"precomputed"}), + callable, + ], + "metric_params": [dict, None], + "alpha": [Interval(Real, left=0, right=None, closed="neither")], + "algorithm": [StrOptions({"auto", "brute", "kd_tree", "ball_tree"})], + "leaf_size": [Interval(Integral, left=1, right=None, closed="left")], + "n_jobs": [Integral, None], + "cluster_selection_method": [StrOptions({"eom", "leaf"})], + "allow_single_cluster": ["boolean"], + "store_centers": [None, StrOptions({"centroid", "medoid", "both"})], + "copy": ["boolean"], + } + + def __init__( + self, + min_cluster_size=5, + min_samples=None, + cluster_selection_epsilon=0.0, + max_cluster_size=None, + metric="euclidean", + metric_params=None, + alpha=1.0, + algorithm="auto", + leaf_size=40, + n_jobs=None, + cluster_selection_method="eom", + allow_single_cluster=False, + store_centers=None, + copy=False, + ): + self.min_cluster_size = min_cluster_size + self.min_samples = min_samples + self.alpha = alpha + self.max_cluster_size = max_cluster_size + self.cluster_selection_epsilon = cluster_selection_epsilon + self.metric = metric + self.metric_params = metric_params + self.algorithm = algorithm + self.leaf_size = leaf_size + self.n_jobs = n_jobs + self.cluster_selection_method = cluster_selection_method + self.allow_single_cluster = allow_single_cluster + self.store_centers = store_centers + self.copy = copy + + @_fit_context( + # HDBSCAN.metric is not validated yet + prefer_skip_nested_validation=False + ) + def fit(self, X, y=None): + """Find clusters based on hierarchical density-based clustering. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features), or \ + ndarray of shape (n_samples, n_samples) + A feature array, or array of distances between samples if + `metric='precomputed'`. + + y : None + Ignored. + + Returns + ------- + self : object + Returns self. + """ + if self.metric == "precomputed" and self.store_centers is not None: + raise ValueError( + "Cannot store centers when using a precomputed distance matrix." + ) + + self._metric_params = self.metric_params or {} + if self.metric != "precomputed": + # Non-precomputed matrices may contain non-finite values. + X = validate_data( + self, + X, + accept_sparse=["csr", "lil"], + ensure_all_finite=False, + dtype=np.float64, + ) + self._raw_data = X + all_finite = True + try: + _assert_all_finite(X.data if issparse(X) else X) + except ValueError: + all_finite = False + + if not all_finite: + # Pass only the purely finite indices into hdbscan + # We will later assign all non-finite points their + # corresponding labels, as specified in `_OUTLIER_ENCODING` + + # Reduce X to make the checks for missing/outlier samples more + # convenient. + reduced_X = X.sum(axis=1) + + # Samples with missing data are denoted by the presence of + # `np.nan` + missing_index = np.isnan(reduced_X).nonzero()[0] + + # Outlier samples are denoted by the presence of `np.inf` + infinite_index = np.isinf(reduced_X).nonzero()[0] + + # Continue with only finite samples + finite_index = _get_finite_row_indices(X) + internal_to_raw = {x: y for x, y in enumerate(finite_index)} + X = X[finite_index] + elif issparse(X): + # Handle sparse precomputed distance matrices separately + X = validate_data( + self, + X, + accept_sparse=["csr", "lil"], + dtype=np.float64, + force_writeable=True, + ) + else: + # Only non-sparse, precomputed distance matrices are handled here + # and thereby allowed to contain numpy.inf for missing distances + + # Perform data validation after removing infinite values (numpy.inf) + # from the given distance matrix. + X = validate_data( + self, X, ensure_all_finite=False, dtype=np.float64, force_writeable=True + ) + if np.isnan(X).any(): + # TODO: Support np.nan in Cython implementation for precomputed + # dense HDBSCAN + raise ValueError("np.nan values found in precomputed-dense") + if X.shape[0] == 1: + raise ValueError("n_samples=1 while HDBSCAN requires more than one sample") + self._min_samples = ( + self.min_cluster_size if self.min_samples is None else self.min_samples + ) + + if self._min_samples > X.shape[0]: + raise ValueError( + f"min_samples ({self._min_samples}) must be at most the number of" + f" samples in X ({X.shape[0]})" + ) + + mst_func = None + kwargs = dict( + X=X, + min_samples=self._min_samples, + alpha=self.alpha, + metric=self.metric, + n_jobs=self.n_jobs, + **self._metric_params, + ) + if self.algorithm == "kd_tree" and self.metric not in KDTree.valid_metrics: + raise ValueError( + f"{self.metric} is not a valid metric for a KDTree-based algorithm." + " Please select a different metric." + ) + elif ( + self.algorithm == "ball_tree" and self.metric not in BallTree.valid_metrics + ): + raise ValueError( + f"{self.metric} is not a valid metric for a BallTree-based algorithm." + " Please select a different metric." + ) + + if self.algorithm != "auto": + if ( + self.metric != "precomputed" + and issparse(X) + and self.algorithm != "brute" + ): + raise ValueError("Sparse data matrices only support algorithm `brute`.") + + if self.algorithm == "brute": + mst_func = _hdbscan_brute + kwargs["copy"] = self.copy + elif self.algorithm == "kd_tree": + mst_func = _hdbscan_prims + kwargs["algo"] = "kd_tree" + kwargs["leaf_size"] = self.leaf_size + else: + mst_func = _hdbscan_prims + kwargs["algo"] = "ball_tree" + kwargs["leaf_size"] = self.leaf_size + else: + if issparse(X) or self.metric not in FAST_METRICS: + # We can't do much with sparse matrices ... + mst_func = _hdbscan_brute + kwargs["copy"] = self.copy + elif self.metric in KDTree.valid_metrics: + # TODO: Benchmark KD vs Ball Tree efficiency + mst_func = _hdbscan_prims + kwargs["algo"] = "kd_tree" + kwargs["leaf_size"] = self.leaf_size + else: + # Metric is a valid BallTree metric + mst_func = _hdbscan_prims + kwargs["algo"] = "ball_tree" + kwargs["leaf_size"] = self.leaf_size + + self._single_linkage_tree_ = mst_func(**kwargs) + + self.labels_, self.probabilities_ = tree_to_labels( + self._single_linkage_tree_, + self.min_cluster_size, + self.cluster_selection_method, + self.allow_single_cluster, + self.cluster_selection_epsilon, + self.max_cluster_size, + ) + if self.metric != "precomputed" and not all_finite: + # Remap indices to align with original data in the case of + # non-finite entries. Samples with np.inf are mapped to -1 and + # those with np.nan are mapped to -2. + self._single_linkage_tree_ = remap_single_linkage_tree( + self._single_linkage_tree_, + internal_to_raw, + # There may be overlap for points w/ both `np.inf` and `np.nan` + non_finite=set(np.hstack([infinite_index, missing_index])), + ) + new_labels = np.empty(self._raw_data.shape[0], dtype=np.int32) + new_labels[finite_index] = self.labels_ + new_labels[infinite_index] = _OUTLIER_ENCODING["infinite"]["label"] + new_labels[missing_index] = _OUTLIER_ENCODING["missing"]["label"] + self.labels_ = new_labels + + new_probabilities = np.zeros(self._raw_data.shape[0], dtype=np.float64) + new_probabilities[finite_index] = self.probabilities_ + # Infinite outliers have probability 0 by convention, though this + # is arbitrary. + new_probabilities[infinite_index] = _OUTLIER_ENCODING["infinite"]["prob"] + new_probabilities[missing_index] = _OUTLIER_ENCODING["missing"]["prob"] + self.probabilities_ = new_probabilities + + if self.store_centers: + self._weighted_cluster_center(X) + return self + + def fit_predict(self, X, y=None): + """Cluster X and return the associated cluster labels. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features), or \ + ndarray of shape (n_samples, n_samples) + A feature array, or array of distances between samples if + `metric='precomputed'`. + + y : None + Ignored. + + Returns + ------- + y : ndarray of shape (n_samples,) + Cluster labels. + """ + self.fit(X) + return self.labels_ + + def _weighted_cluster_center(self, X): + """Calculate and store the centroids/medoids of each cluster. + + This requires `X` to be a raw feature array, not precomputed + distances. Rather than return outputs directly, this helper method + instead stores them in the `self.{centroids, medoids}_` attributes. + The choice for which attributes are calculated and stored is mediated + by the value of `self.store_centers`. + + Parameters + ---------- + X : ndarray of shape (n_samples, n_features) + The feature array that the estimator was fit with. + + """ + # Number of non-noise clusters + n_clusters = len(set(self.labels_) - {-1, -2}) + mask = np.empty((X.shape[0],), dtype=np.bool_) + make_centroids = self.store_centers in ("centroid", "both") + make_medoids = self.store_centers in ("medoid", "both") + + if make_centroids: + self.centroids_ = np.empty((n_clusters, X.shape[1]), dtype=np.float64) + if make_medoids: + self.medoids_ = np.empty((n_clusters, X.shape[1]), dtype=np.float64) + + # Need to handle iteratively seen each cluster may have a different + # number of samples, hence we can't create a homogeneous 3D array. + for idx in range(n_clusters): + mask = self.labels_ == idx + data = X[mask] + strength = self.probabilities_[mask] + if make_centroids: + self.centroids_[idx] = np.average(data, weights=strength, axis=0) + if make_medoids: + # TODO: Implement weighted argmin PWD backend + dist_mat = pairwise_distances( + data, metric=self.metric, **self._metric_params + ) + dist_mat = dist_mat * strength + medoid_index = np.argmin(dist_mat.sum(axis=1)) + self.medoids_[idx] = data[medoid_index] + return + + def dbscan_clustering(self, cut_distance, min_cluster_size=5): + """Return clustering given by DBSCAN without border points. + + Return clustering that would be equivalent to running DBSCAN* for a + particular cut_distance (or epsilon) DBSCAN* can be thought of as + DBSCAN without the border points. As such these results may differ + slightly from `cluster.DBSCAN` due to the difference in implementation + over the non-core points. + + This can also be thought of as a flat clustering derived from constant + height cut through the single linkage tree. + + This represents the result of selecting a cut value for robust single linkage + clustering. The `min_cluster_size` allows the flat clustering to declare noise + points (and cluster smaller than `min_cluster_size`). + + Parameters + ---------- + cut_distance : float + The mutual reachability distance cut value to use to generate a + flat clustering. + + min_cluster_size : int, default=5 + Clusters smaller than this value with be called 'noise' and remain + unclustered in the resulting flat clustering. + + Returns + ------- + labels : ndarray of shape (n_samples,) + An array of cluster labels, one per datapoint. + Outliers are labeled as follows: + + - Noisy samples are given the label -1. + - Samples with infinite elements (+/- np.inf) are given the label -2. + - Samples with missing data are given the label -3, even if they + also have infinite elements. + """ + labels = labelling_at_cut( + self._single_linkage_tree_, cut_distance, min_cluster_size + ) + # Infer indices from labels generated during `fit` + infinite_index = self.labels_ == _OUTLIER_ENCODING["infinite"]["label"] + missing_index = self.labels_ == _OUTLIER_ENCODING["missing"]["label"] + + # Overwrite infinite/missing outlier samples (otherwise simple noise) + labels[infinite_index] = _OUTLIER_ENCODING["infinite"]["label"] + labels[missing_index] = _OUTLIER_ENCODING["missing"]["label"] + return labels + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.sparse = True + tags.input_tags.allow_nan = self.metric != "precomputed" + return tags diff --git a/sklearn/cluster/_hdbscan/meson.build b/sklearn/cluster/_hdbscan/meson.build new file mode 100644 index 0000000000000..8d880b39a4db5 --- /dev/null +++ b/sklearn/cluster/_hdbscan/meson.build @@ -0,0 +1,15 @@ +cluster_hdbscan_extension_metadata = { + '_linkage': {'sources': [cython_gen.process('_linkage.pyx'), metrics_cython_tree]}, + '_reachability': {'sources': [cython_gen.process('_reachability.pyx')]}, + '_tree': {'sources': [cython_gen.process('_tree.pyx')]} +} + +foreach ext_name, ext_dict : cluster_hdbscan_extension_metadata + py.extension_module( + ext_name, + ext_dict.get('sources'), + dependencies: [np_dep], + subdir: 'sklearn/cluster/_hdbscan', + install: true + ) +endforeach diff --git a/sklearn/cluster/_hdbscan/tests/__init__.py b/sklearn/cluster/_hdbscan/tests/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/sklearn/cluster/_hdbscan/tests/test_reachibility.py b/sklearn/cluster/_hdbscan/tests/test_reachibility.py new file mode 100644 index 0000000000000..a336e6be6116d --- /dev/null +++ b/sklearn/cluster/_hdbscan/tests/test_reachibility.py @@ -0,0 +1,63 @@ +import numpy as np +import pytest + +from sklearn.cluster._hdbscan._reachability import mutual_reachability_graph +from sklearn.utils._testing import ( + _convert_container, + assert_allclose, +) + + +def test_mutual_reachability_graph_error_sparse_format(): + """Check that we raise an error if the sparse format is not CSR.""" + rng = np.random.RandomState(0) + X = rng.randn(10, 10) + X = X.T @ X + np.fill_diagonal(X, 0.0) + X = _convert_container(X, "sparse_csc") + + err_msg = "Only sparse CSR matrices are supported" + with pytest.raises(ValueError, match=err_msg): + mutual_reachability_graph(X) + + +@pytest.mark.parametrize("array_type", ["array", "sparse_csr"]) +def test_mutual_reachability_graph_inplace(array_type): + """Check that the operation is happening inplace.""" + rng = np.random.RandomState(0) + X = rng.randn(10, 10) + X = X.T @ X + np.fill_diagonal(X, 0.0) + X = _convert_container(X, array_type) + + mr_graph = mutual_reachability_graph(X) + + assert id(mr_graph) == id(X) + + +def test_mutual_reachability_graph_equivalence_dense_sparse(): + """Check that we get the same results for dense and sparse implementation.""" + rng = np.random.RandomState(0) + X = rng.randn(5, 5) + X_dense = X.T @ X + X_sparse = _convert_container(X_dense, "sparse_csr") + + mr_graph_dense = mutual_reachability_graph(X_dense, min_samples=3) + mr_graph_sparse = mutual_reachability_graph(X_sparse, min_samples=3) + + assert_allclose(mr_graph_dense, mr_graph_sparse.toarray()) + + +@pytest.mark.parametrize("array_type", ["array", "sparse_csr"]) +@pytest.mark.parametrize("dtype", [np.float32, np.float64]) +def test_mutual_reachability_graph_preserves_dtype(array_type, dtype): + """Check that the computation preserve dtype thanks to fused types.""" + rng = np.random.RandomState(0) + X = rng.randn(10, 10) + X = (X.T @ X).astype(dtype) + np.fill_diagonal(X, 0.0) + X = _convert_container(X, array_type) + + assert X.dtype == dtype + mr_graph = mutual_reachability_graph(X) + assert mr_graph.dtype == dtype diff --git a/sklearn/cluster/_hierarchical_fast.pxd b/sklearn/cluster/_hierarchical_fast.pxd new file mode 100644 index 0000000000000..a10f8c12f3440 --- /dev/null +++ b/sklearn/cluster/_hierarchical_fast.pxd @@ -0,0 +1,9 @@ +from ..utils._typedefs cimport intp_t + +cdef class UnionFind: + cdef intp_t next_label + cdef intp_t[:] parent + cdef intp_t[:] size + + cdef void union(self, intp_t m, intp_t n) noexcept + cdef intp_t fast_find(self, intp_t n) noexcept diff --git a/sklearn/cluster/_hierarchical_fast.pyx b/sklearn/cluster/_hierarchical_fast.pyx index ec8c96410c25c..36ae0ab0d2414 100644 --- a/sklearn/cluster/_hierarchical_fast.pyx +++ b/sklearn/cluster/_hierarchical_fast.pyx @@ -1,49 +1,33 @@ -# Author: Gael Varoquaux +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import numpy as np -cimport numpy as np cimport cython -ctypedef np.float64_t DOUBLE -ctypedef np.npy_intp INTP -ctypedef np.int8_t INT8 - -# Numpy must be initialized. When using numpy from C or Cython you must -# _always_ do that, or you will have segfaults - -np.import_array() - -from ..neighbors._dist_metrics cimport DistanceMetric +from ..metrics._dist_metrics cimport DistanceMetric64 from ..utils._fast_dict cimport IntFloatDict +from ..utils._typedefs cimport float64_t, intp_t, uint8_t # C++ from cython.operator cimport dereference as deref, preincrement as inc from libcpp.map cimport map as cpp_map -from libc.math cimport fmax +from libc.math cimport fmax, INFINITY -DTYPE = np.float64 -ctypedef np.float64_t DTYPE_t - -ITYPE = np.intp -ctypedef np.intp_t ITYPE_t - -from numpy.math cimport INFINITY ############################################################################### # Utilities for computing the ward momentum -@cython.boundscheck(False) -@cython.wraparound(False) -@cython.cdivision(True) -def compute_ward_dist(np.ndarray[DOUBLE, ndim=1, mode='c'] m_1, - np.ndarray[DOUBLE, ndim=2, mode='c'] m_2, - np.ndarray[INTP, ndim=1, mode='c'] coord_row, - np.ndarray[INTP, ndim=1, mode='c'] coord_col, - np.ndarray[DOUBLE, ndim=1, mode='c'] res): - cdef INTP size_max = coord_row.shape[0] - cdef INTP n_features = m_2.shape[1] - cdef INTP i, j, row, col - cdef DOUBLE pa, n +def compute_ward_dist( + const float64_t[::1] m_1, + const float64_t[:, ::1] m_2, + const intp_t[::1] coord_row, + const intp_t[::1] coord_col, + float64_t[::1] res +): + cdef intp_t size_max = coord_row.shape[0] + cdef intp_t n_features = m_2.shape[1] + cdef intp_t i, j, row, col + cdef float64_t pa, n for i in range(size_max): row = coord_row[i] @@ -53,13 +37,12 @@ def compute_ward_dist(np.ndarray[DOUBLE, ndim=1, mode='c'] m_1, for j in range(n_features): pa += (m_2[row, j] / m_1[row] - m_2[col, j] / m_1[col]) ** 2 res[i] = pa * n - return res ############################################################################### # Utilities for cutting and exploring a hierarchical tree -def _hc_get_descendent(INTP node, children, INTP n_leaves): +def _hc_get_descendent(intp_t node, children, intp_t n_leaves): """ Function returning all the descendent leaves of a set of nodes in the tree. @@ -88,7 +71,7 @@ def _hc_get_descendent(INTP node, children, INTP n_leaves): # It is actually faster to do the accounting of the number of # elements is the list ourselves: len is a lengthy operation on a # chained list - cdef INTP i, n_indices = 1 + cdef intp_t i, n_indices = 1 while n_indices: i = ind.pop() @@ -101,9 +84,7 @@ def _hc_get_descendent(INTP node, children, INTP n_leaves): return descendent -@cython.boundscheck(False) -@cython.wraparound(False) -def hc_get_heads(np.ndarray[INTP, ndim=1] parents, copy=True): +def hc_get_heads(intp_t[:] parents, copy=True): """Returns the heads of the forest, as defined by parents. Parameters @@ -119,7 +100,7 @@ def hc_get_heads(np.ndarray[INTP, ndim=1] parents, copy=True): The indices in the 'parents' of the tree heads """ - cdef INTP parent, node0, node, size + cdef intp_t parent, node0, node, size if copy: parents = np.copy(parents) size = parents.size @@ -135,10 +116,12 @@ def hc_get_heads(np.ndarray[INTP, ndim=1] parents, copy=True): return parents -@cython.boundscheck(False) -@cython.wraparound(False) -def _get_parents(nodes, heads, np.ndarray[INTP, ndim=1] parents, - np.ndarray[INT8, ndim=1, mode='c'] not_visited): +def _get_parents( + nodes, + heads, + const intp_t[:] parents, + uint8_t[::1] not_visited +): """Returns the heads of the given nodes, as defined by parents. Modifies 'heads' and 'not_visited' in-place. @@ -155,7 +138,7 @@ def _get_parents(nodes, heads, np.ndarray[INTP, ndim=1] parents, The tree nodes to consider (modified inplace) """ - cdef INTP parent, node + cdef intp_t parent, node for node in nodes: parent = parents[node] @@ -165,7 +148,6 @@ def _get_parents(nodes, heads, np.ndarray[INTP, ndim=1] parents, if not_visited[node]: not_visited[node] = 0 heads.append(node) - return heads ############################################################################### @@ -176,11 +158,13 @@ def _get_parents(nodes, heads, np.ndarray[INTP, ndim=1] parents, # as keys and edge weights as values. -@cython.boundscheck(False) -@cython.wraparound(False) -def max_merge(IntFloatDict a, IntFloatDict b, - np.ndarray[ITYPE_t, ndim=1] mask, - ITYPE_t n_a, ITYPE_t n_b): +def max_merge( + IntFloatDict a, + IntFloatDict b, + const intp_t[:] mask, + intp_t n_a, + intp_t n_b +): """Merge two IntFloatDicts with the max strategy: when the same key is present in the two dicts, the max of the two values is used. @@ -201,10 +185,10 @@ def max_merge(IntFloatDict a, IntFloatDict b, The IntFloatDict resulting from the merge """ cdef IntFloatDict out_obj = IntFloatDict.__new__(IntFloatDict) - cdef cpp_map[ITYPE_t, DTYPE_t].iterator a_it = a.my_map.begin() - cdef cpp_map[ITYPE_t, DTYPE_t].iterator a_end = a.my_map.end() - cdef ITYPE_t key - cdef DTYPE_t value + cdef cpp_map[intp_t, float64_t].iterator a_it = a.my_map.begin() + cdef cpp_map[intp_t, float64_t].iterator a_end = a.my_map.end() + cdef intp_t key + cdef float64_t value # First copy a into out while a_it != a_end: key = deref(a_it).first @@ -213,10 +197,10 @@ def max_merge(IntFloatDict a, IntFloatDict b, inc(a_it) # Then merge b into out - cdef cpp_map[ITYPE_t, DTYPE_t].iterator out_it = out_obj.my_map.begin() - cdef cpp_map[ITYPE_t, DTYPE_t].iterator out_end = out_obj.my_map.end() - cdef cpp_map[ITYPE_t, DTYPE_t].iterator b_it = b.my_map.begin() - cdef cpp_map[ITYPE_t, DTYPE_t].iterator b_end = b.my_map.end() + cdef cpp_map[intp_t, float64_t].iterator out_it = out_obj.my_map.begin() + cdef cpp_map[intp_t, float64_t].iterator out_end = out_obj.my_map.end() + cdef cpp_map[intp_t, float64_t].iterator b_it = b.my_map.begin() + cdef cpp_map[intp_t, float64_t].iterator b_end = b.my_map.end() while b_it != b_end: key = deref(b_it).first value = deref(b_it).second @@ -231,13 +215,15 @@ def max_merge(IntFloatDict a, IntFloatDict b, return out_obj -@cython.boundscheck(False) -@cython.wraparound(False) -def average_merge(IntFloatDict a, IntFloatDict b, - np.ndarray[ITYPE_t, ndim=1] mask, - ITYPE_t n_a, ITYPE_t n_b): - """Merge two IntFloatDicts with the average strategy: when the - same key is present in the two dicts, the weighted average of the two +def average_merge( + IntFloatDict a, + IntFloatDict b, + const intp_t[:] mask, + intp_t n_a, + intp_t n_b +): + """Merge two IntFloatDicts with the average strategy: when the + same key is present in the two dicts, the weighted average of the two values is used. Parameters @@ -257,11 +243,11 @@ def average_merge(IntFloatDict a, IntFloatDict b, The IntFloatDict resulting from the merge """ cdef IntFloatDict out_obj = IntFloatDict.__new__(IntFloatDict) - cdef cpp_map[ITYPE_t, DTYPE_t].iterator a_it = a.my_map.begin() - cdef cpp_map[ITYPE_t, DTYPE_t].iterator a_end = a.my_map.end() - cdef ITYPE_t key - cdef DTYPE_t value - cdef DTYPE_t n_out = (n_a + n_b) + cdef cpp_map[intp_t, float64_t].iterator a_it = a.my_map.begin() + cdef cpp_map[intp_t, float64_t].iterator a_end = a.my_map.end() + cdef intp_t key + cdef float64_t value + cdef float64_t n_out = (n_a + n_b) # First copy a into out while a_it != a_end: key = deref(a_it).first @@ -270,10 +256,10 @@ def average_merge(IntFloatDict a, IntFloatDict b, inc(a_it) # Then merge b into out - cdef cpp_map[ITYPE_t, DTYPE_t].iterator out_it = out_obj.my_map.begin() - cdef cpp_map[ITYPE_t, DTYPE_t].iterator out_end = out_obj.my_map.end() - cdef cpp_map[ITYPE_t, DTYPE_t].iterator b_it = b.my_map.begin() - cdef cpp_map[ITYPE_t, DTYPE_t].iterator b_end = b.my_map.end() + cdef cpp_map[intp_t, float64_t].iterator out_it = out_obj.my_map.begin() + cdef cpp_map[intp_t, float64_t].iterator out_end = out_obj.my_map.end() + cdef cpp_map[intp_t, float64_t].iterator b_it = b.my_map.begin() + cdef cpp_map[intp_t, float64_t].iterator b_end = b.my_map.end() while b_it != b_end: key = deref(b_it).first value = deref(b_it).second @@ -290,19 +276,18 @@ def average_merge(IntFloatDict a, IntFloatDict b, ############################################################################### -# An edge object for fast comparisons +# An edge object for fast comparisons cdef class WeightedEdge: - cdef public ITYPE_t a - cdef public ITYPE_t b - cdef public DTYPE_t weight - - def __init__(self, DTYPE_t weight, ITYPE_t a, ITYPE_t b): + cdef public intp_t a + cdef public intp_t b + cdef public float64_t weight + + def __init__(self, float64_t weight, intp_t a, intp_t b): self.weight = weight self.a = a self.b = b - @cython.nonecheck(False) def __richcmp__(self, WeightedEdge other, int op): """Cython-specific comparison method. @@ -326,7 +311,7 @@ cdef class WeightedEdge: return self.weight > other.weight elif op == 5: return self.weight >= other.weight - + def __repr__(self): return "%s(weight=%f, a=%i, b=%i)" % (self.__class__.__name__, self.weight, @@ -338,30 +323,22 @@ cdef class WeightedEdge: cdef class UnionFind(object): - cdef ITYPE_t next_label - cdef ITYPE_t[:] parent - cdef ITYPE_t[:] size - def __init__(self, N): - self.parent = np.full(2 * N - 1, -1., dtype=ITYPE, order='C') + self.parent = np.full(2 * N - 1, -1., dtype=np.intp, order='C') self.next_label = N - self.size = np.hstack((np.ones(N, dtype=ITYPE), - np.zeros(N - 1, dtype=ITYPE))) + self.size = np.hstack((np.ones(N, dtype=np.intp), + np.zeros(N - 1, dtype=np.intp))) - @cython.boundscheck(False) - @cython.nonecheck(False) - cdef void union(self, ITYPE_t m, ITYPE_t n): + cdef void union(self, intp_t m, intp_t n) noexcept: self.parent[m] = self.next_label self.parent[n] = self.next_label self.size[self.next_label] = self.size[m] + self.size[n] self.next_label += 1 - return - @cython.boundscheck(False) - @cython.nonecheck(False) - cdef ITYPE_t fast_find(self, ITYPE_t n): - cdef ITYPE_t p + @cython.wraparound(True) + cdef intp_t fast_find(self, intp_t n) noexcept: + cdef intp_t p p = n # find the highest node in the linkage graph so far while self.parent[n] != -1: @@ -371,10 +348,8 @@ cdef class UnionFind(object): p, self.parent[p] = self.parent[p], n return n -@cython.boundscheck(False) -@cython.nonecheck(False) -cpdef np.ndarray[DTYPE_t, ndim=2] _single_linkage_label( - np.ndarray[DTYPE_t, ndim=2] L): + +def _single_linkage_label(const float64_t[:, :] L): """ Convert an linkage array or MST to a tree by labelling clusters at merges. This is done by using a Union find structure to keep track of merges @@ -394,35 +369,34 @@ cpdef np.ndarray[DTYPE_t, ndim=2] _single_linkage_label( A tree in the format used by scipy.cluster.hierarchy. """ - cdef np.ndarray[DTYPE_t, ndim=2] result_arr - cdef DTYPE_t[:, ::1] result + cdef float64_t[:, ::1] result_arr - cdef ITYPE_t left, left_cluster, right, right_cluster, index - cdef DTYPE_t delta + cdef intp_t left, left_cluster, right, right_cluster, index + cdef float64_t delta - result_arr = np.zeros((L.shape[0], 4), dtype=DTYPE) - result = result_arr + result_arr = np.zeros((L.shape[0], 4), dtype=np.float64) U = UnionFind(L.shape[0] + 1) for index in range(L.shape[0]): - left = L[index, 0] - right = L[index, 1] + left = L[index, 0] + right = L[index, 1] delta = L[index, 2] left_cluster = U.fast_find(left) right_cluster = U.fast_find(right) - result[index][0] = left_cluster - result[index][1] = right_cluster - result[index][2] = delta - result[index][3] = U.size[left_cluster] + U.size[right_cluster] + result_arr[index][0] = left_cluster + result_arr[index][1] = right_cluster + result_arr[index][2] = delta + result_arr[index][3] = U.size[left_cluster] + U.size[right_cluster] U.union(left_cluster, right_cluster) - return result_arr + return np.asarray(result_arr) +@cython.wraparound(True) def single_linkage_label(L): """ Convert an linkage array or MST to a tree by labelling clusters at merges. @@ -452,17 +426,15 @@ def single_linkage_label(L): # Implements MST-LINKAGE-CORE from https://arxiv.org/abs/1109.2378 -@cython.boundscheck(False) -@cython.nonecheck(False) def mst_linkage_core( - DTYPE_t [:, ::1] raw_data, - DistanceMetric dist_metric): + const float64_t [:, ::1] raw_data, + DistanceMetric64 dist_metric): """ Compute the necessary elements of a minimum spanning tree for computation of single linkage clustering. This represents the MST-LINKAGE-CORE algorithm (Figure 6) from - *Modern hierarchical, agglomerative clustering algorithms* - by Daniel Mullner (https://arxiv.org/abs/1109.2378). + :arxiv:`Daniel Mullner, "Modern hierarchical, agglomerative clustering + algorithms" <1109.2378>`. In contrast to the scipy implementation is never computes a full distance matrix, generating distances only as they @@ -473,9 +445,9 @@ def mst_linkage_core( raw_data: array of shape (n_samples, n_features) The array of feature data to be clustered. Must be C-aligned - dist_metric: DistanceMetric - A DistanceMetric object conforming to the API from - ``sklearn.neighbors._dist_metrics.pxd`` that will be + dist_metric: DistanceMetric64 + A DistanceMetric64 object conforming to the API from + ``sklearn.metrics._dist_metrics.pxd`` that will be used to compute distances. Returns @@ -483,27 +455,26 @@ def mst_linkage_core( mst_core_data: array of shape (n_samples, 3) An array providing information from which one can either compute an MST, or the linkage hierarchy - very efficiently. See https://arxiv.org/abs/1109.2378 - algorithm MST-LINKAGE-CORE for more details. + very efficiently. See :arxiv:`Daniel Mullner, "Modern hierarchical, + agglomerative clustering algorithms" <1109.2378>` algorithm + MST-LINKAGE-CORE for more details. """ cdef: - ITYPE_t n_samples = raw_data.shape[0] - np.int8_t[:] in_tree = np.zeros(n_samples, dtype=np.int8) - DTYPE_t[:, ::1] result = np.zeros((n_samples - 1, 3)) - - np.ndarray label_filter + intp_t n_samples = raw_data.shape[0] + uint8_t[:] in_tree = np.zeros(n_samples, dtype=bool) + float64_t[:, ::1] result = np.zeros((n_samples - 1, 3)) - ITYPE_t current_node = 0 - ITYPE_t new_node - ITYPE_t i - ITYPE_t j - ITYPE_t num_features = raw_data.shape[1] + intp_t current_node = 0 + intp_t new_node + intp_t i + intp_t j + intp_t num_features = raw_data.shape[1] - DTYPE_t right_value - DTYPE_t left_value - DTYPE_t new_distance + float64_t right_value + float64_t left_value + float64_t new_distance - DTYPE_t[:] current_distances = np.full(n_samples, INFINITY) + float64_t[:] current_distances = np.full(n_samples, INFINITY) for i in range(n_samples - 1): @@ -534,4 +505,3 @@ def mst_linkage_core( current_node = new_node return np.array(result) - diff --git a/sklearn/cluster/_k_means_common.pxd b/sklearn/cluster/_k_means_common.pxd new file mode 100644 index 0000000000000..9a41ea68d1baf --- /dev/null +++ b/sklearn/cluster/_k_means_common.pxd @@ -0,0 +1,48 @@ +from cython cimport floating + + +cdef floating _euclidean_dense_dense( + const floating*, + const floating*, + int, + bint +) noexcept nogil + +cdef floating _euclidean_sparse_dense( + const floating[::1], + const int[::1], + const floating[::1], + floating, + bint +) noexcept nogil + +cpdef void _relocate_empty_clusters_dense( + const floating[:, ::1], + const floating[::1], + const floating[:, ::1], + floating[:, ::1], + floating[::1], + const int[::1] +) + +cpdef void _relocate_empty_clusters_sparse( + const floating[::1], + const int[::1], + const int[::1], + const floating[::1], + const floating[:, ::1], + floating[:, ::1], + floating[::1], + const int[::1] +) + +cdef void _average_centers( + floating[:, ::1], + const floating[::1] +) + +cdef void _center_shift( + const floating[:, ::1], + const floating[:, ::1], + floating[::1] +) diff --git a/sklearn/cluster/_k_means_common.pyx b/sklearn/cluster/_k_means_common.pyx new file mode 100644 index 0000000000000..674d4026a6756 --- /dev/null +++ b/sklearn/cluster/_k_means_common.pyx @@ -0,0 +1,328 @@ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import numpy as np +from cython cimport floating +from cython.parallel cimport prange +from libc.math cimport sqrt + +from ..utils.extmath import row_norms + + +# Number of samples per data chunk defined as a global constant. +CHUNK_SIZE = 256 + + +cdef floating _euclidean_dense_dense( + const floating* a, # IN + const floating* b, # IN + int n_features, + bint squared +) noexcept nogil: + """Euclidean distance between a dense and b dense""" + cdef: + int i + int n = n_features // 4 + int rem = n_features % 4 + floating result = 0 + + # We manually unroll the loop for better cache optimization. + for i in range(n): + result += ( + (a[0] - b[0]) * (a[0] - b[0]) + + (a[1] - b[1]) * (a[1] - b[1]) + + (a[2] - b[2]) * (a[2] - b[2]) + + (a[3] - b[3]) * (a[3] - b[3]) + ) + a += 4 + b += 4 + + for i in range(rem): + result += (a[i] - b[i]) * (a[i] - b[i]) + + return result if squared else sqrt(result) + + +def _euclidean_dense_dense_wrapper( + const floating[::1] a, + const floating[::1] b, + bint squared +): + """Wrapper of _euclidean_dense_dense for testing purpose""" + return _euclidean_dense_dense(&a[0], &b[0], a.shape[0], squared) + + +cdef floating _euclidean_sparse_dense( + const floating[::1] a_data, # IN + const int[::1] a_indices, # IN + const floating[::1] b, # IN + floating b_squared_norm, + bint squared +) noexcept nogil: + """Euclidean distance between a sparse and b dense""" + cdef: + int nnz = a_indices.shape[0] + int i + floating tmp, bi + floating result = 0.0 + + for i in range(nnz): + bi = b[a_indices[i]] + tmp = a_data[i] - bi + result += tmp * tmp - bi * bi + + result += b_squared_norm + + if result < 0: + result = 0.0 + + return result if squared else sqrt(result) + + +def _euclidean_sparse_dense_wrapper( + const floating[::1] a_data, + const int[::1] a_indices, + const floating[::1] b, + floating b_squared_norm, + bint squared +): + """Wrapper of _euclidean_sparse_dense for testing purpose""" + return _euclidean_sparse_dense( + a_data, a_indices, b, b_squared_norm, squared) + + +cpdef floating _inertia_dense( + const floating[:, ::1] X, # IN + const floating[::1] sample_weight, # IN + const floating[:, ::1] centers, # IN + const int[::1] labels, # IN + int n_threads, + int single_label=-1, +): + """Compute inertia for dense input data + + Sum of squared distance between each sample and its assigned center. + + If single_label is >= 0, the inertia is computed only for that label. + """ + cdef: + int n_samples = X.shape[0] + int n_features = X.shape[1] + int i, j + + floating sq_dist = 0.0 + floating inertia = 0.0 + + for i in prange(n_samples, nogil=True, num_threads=n_threads, + schedule='static'): + j = labels[i] + if single_label < 0 or single_label == j: + sq_dist = _euclidean_dense_dense(&X[i, 0], ¢ers[j, 0], + n_features, True) + inertia += sq_dist * sample_weight[i] + + return inertia + + +cpdef floating _inertia_sparse( + X, # IN + const floating[::1] sample_weight, # IN + const floating[:, ::1] centers, # IN + const int[::1] labels, # IN + int n_threads, + int single_label=-1, +): + """Compute inertia for sparse input data + + Sum of squared distance between each sample and its assigned center. + + If single_label is >= 0, the inertia is computed only for that label. + """ + cdef: + floating[::1] X_data = X.data + int[::1] X_indices = X.indices + int[::1] X_indptr = X.indptr + + int n_samples = X.shape[0] + int i, j + + floating sq_dist = 0.0 + floating inertia = 0.0 + + floating[::1] centers_squared_norms = row_norms(centers, squared=True) + + for i in prange(n_samples, nogil=True, num_threads=n_threads, + schedule='static'): + j = labels[i] + if single_label < 0 or single_label == j: + sq_dist = _euclidean_sparse_dense( + X_data[X_indptr[i]: X_indptr[i + 1]], + X_indices[X_indptr[i]: X_indptr[i + 1]], + centers[j], centers_squared_norms[j], True) + inertia += sq_dist * sample_weight[i] + + return inertia + + +cpdef void _relocate_empty_clusters_dense( + const floating[:, ::1] X, # IN + const floating[::1] sample_weight, # IN + const floating[:, ::1] centers_old, # IN + floating[:, ::1] centers_new, # INOUT + floating[::1] weight_in_clusters, # INOUT + const int[::1] labels # IN +): + """Relocate centers which have no sample assigned to them.""" + cdef: + int[::1] empty_clusters = np.where(np.equal(weight_in_clusters, 0))[0].astype(np.int32) + int n_empty = empty_clusters.shape[0] + + if n_empty == 0: + return + + cdef: + int n_features = X.shape[1] + + floating[::1] distances = ((np.asarray(X) - np.asarray(centers_old)[labels])**2).sum(axis=1) + int[::1] far_from_centers = np.argpartition(distances, -n_empty)[:-n_empty-1:-1].astype(np.int32) + + int new_cluster_id, old_cluster_id, far_idx, idx, k + floating weight + + if np.max(distances) == 0: + # Happens when there are more clusters than non-duplicate samples. Relocating + # is pointless in this case. + return + + for idx in range(n_empty): + + new_cluster_id = empty_clusters[idx] + + far_idx = far_from_centers[idx] + weight = sample_weight[far_idx] + + old_cluster_id = labels[far_idx] + + for k in range(n_features): + centers_new[old_cluster_id, k] -= X[far_idx, k] * weight + centers_new[new_cluster_id, k] = X[far_idx, k] * weight + + weight_in_clusters[new_cluster_id] = weight + weight_in_clusters[old_cluster_id] -= weight + + +cpdef void _relocate_empty_clusters_sparse( + const floating[::1] X_data, # IN + const int[::1] X_indices, # IN + const int[::1] X_indptr, # IN + const floating[::1] sample_weight, # IN + const floating[:, ::1] centers_old, # IN + floating[:, ::1] centers_new, # INOUT + floating[::1] weight_in_clusters, # INOUT + const int[::1] labels # IN +): + """Relocate centers which have no sample assigned to them.""" + cdef: + int[::1] empty_clusters = np.where(np.equal(weight_in_clusters, 0))[0].astype(np.int32) + int n_empty = empty_clusters.shape[0] + + if n_empty == 0: + return + + cdef: + int n_samples = X_indptr.shape[0] - 1 + int i, j, k + + floating[::1] distances = np.zeros(n_samples, dtype=X_data.base.dtype) + floating[::1] centers_squared_norms = row_norms(centers_old, squared=True) + + for i in range(n_samples): + j = labels[i] + distances[i] = _euclidean_sparse_dense( + X_data[X_indptr[i]: X_indptr[i + 1]], + X_indices[X_indptr[i]: X_indptr[i + 1]], + centers_old[j], centers_squared_norms[j], True) + + if np.max(distances) == 0: + # Happens when there are more clusters than non-duplicate samples. Relocating + # is pointless in this case. + return + + cdef: + int[::1] far_from_centers = np.argpartition(distances, -n_empty)[:-n_empty-1:-1].astype(np.int32) + + int new_cluster_id, old_cluster_id, far_idx, idx + floating weight + + for idx in range(n_empty): + + new_cluster_id = empty_clusters[idx] + + far_idx = far_from_centers[idx] + weight = sample_weight[far_idx] + + old_cluster_id = labels[far_idx] + + for k in range(X_indptr[far_idx], X_indptr[far_idx + 1]): + centers_new[old_cluster_id, X_indices[k]] -= X_data[k] * weight + centers_new[new_cluster_id, X_indices[k]] = X_data[k] * weight + + weight_in_clusters[new_cluster_id] = weight + weight_in_clusters[old_cluster_id] -= weight + + +cdef void _average_centers( + floating[:, ::1] centers, # INOUT + const floating[::1] weight_in_clusters # IN +): + """Average new centers wrt weights.""" + cdef: + int n_clusters = centers.shape[0] + int n_features = centers.shape[1] + int j, k + floating alpha + int argmax_weight = np.argmax(weight_in_clusters) + + for j in range(n_clusters): + if weight_in_clusters[j] > 0: + alpha = 1.0 / weight_in_clusters[j] + for k in range(n_features): + centers[j, k] *= alpha + else: + # For convenience, we avoid setting empty clusters at the origin but place + # them at the location of the biggest cluster. + for k in range(n_features): + centers[j, k] = centers[argmax_weight, k] + + +cdef void _center_shift( + const floating[:, ::1] centers_old, # IN + const floating[:, ::1] centers_new, # IN + floating[::1] center_shift # OUT +): + """Compute shift between old and new centers.""" + cdef: + int n_clusters = centers_old.shape[0] + int n_features = centers_old.shape[1] + int j + + for j in range(n_clusters): + center_shift[j] = _euclidean_dense_dense( + ¢ers_new[j, 0], ¢ers_old[j, 0], n_features, False) + + +def _is_same_clustering( + const int[::1] labels1, + const int[::1] labels2, + n_clusters +): + """Check if two arrays of labels are the same up to a permutation of the labels""" + cdef int[::1] mapping = np.full(fill_value=-1, shape=(n_clusters,), dtype=np.int32) + cdef int i + + for i in range(labels1.shape[0]): + if mapping[labels1[i]] == -1: + mapping[labels1[i]] = labels2[i] + elif mapping[labels1[i]] != labels2[i]: + return False + return True diff --git a/sklearn/cluster/_k_means_elkan.pyx b/sklearn/cluster/_k_means_elkan.pyx index 1010e581f5e7f..564218a17f701 100644 --- a/sklearn/cluster/_k_means_elkan.pyx +++ b/sklearn/cluster/_k_means_elkan.pyx @@ -1,42 +1,34 @@ -# cython: profile=True, boundscheck=False, wraparound=False, cdivision=True -# -# Author: Andreas Mueller -# -# Licence: BSD 3 clause - -# TODO: We still need to use ndarrays instead of typed memoryviews when using -# fused types and when the array may be read-only (for instance when it's -# provided by the user). This is fixed in cython > 0.3. - -import numpy as np -cimport numpy as np -cimport cython +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + from cython cimport floating from cython.parallel import prange, parallel -from libc.math cimport sqrt from libc.stdlib cimport calloc, free -from libc.string cimport memset, memcpy +from libc.string cimport memset +from ..utils._openmp_helpers cimport omp_lock_t +from ..utils._openmp_helpers cimport omp_init_lock +from ..utils._openmp_helpers cimport omp_destroy_lock +from ..utils._openmp_helpers cimport omp_set_lock +from ..utils._openmp_helpers cimport omp_unset_lock from ..utils.extmath import row_norms -from ._k_means_fast import CHUNK_SIZE -from ._k_means_fast cimport _relocate_empty_clusters_dense -from ._k_means_fast cimport _relocate_empty_clusters_sparse -from ._k_means_fast cimport _euclidean_dense_dense -from ._k_means_fast cimport _euclidean_sparse_dense -from ._k_means_fast cimport _average_centers -from ._k_means_fast cimport _center_shift - - -np.import_array() +from ._k_means_common import CHUNK_SIZE +from ._k_means_common cimport _relocate_empty_clusters_dense +from ._k_means_common cimport _relocate_empty_clusters_sparse +from ._k_means_common cimport _euclidean_dense_dense +from ._k_means_common cimport _euclidean_sparse_dense +from ._k_means_common cimport _average_centers +from ._k_means_common cimport _center_shift def init_bounds_dense( - np.ndarray[floating, ndim=2, mode='c'] X, # IN - floating[:, ::1] centers, # IN - floating[:, ::1] center_half_distances, # IN - int[::1] labels, # OUT - floating[::1] upper_bounds, # OUT - floating[:, ::1] lower_bounds): # OUT + const floating[:, ::1] X, # IN + const floating[:, ::1] centers, # IN + const floating[:, ::1] center_half_distances, # IN + int[::1] labels, # OUT + floating[::1] upper_bounds, # OUT + floating[:, ::1] lower_bounds, # OUT + int n_threads): """Initialize upper and lower bounds for each sample for dense input data. Given X, centers and the pairwise distances divided by 2.0 between the @@ -74,6 +66,9 @@ def init_bounds_dense( lower_bounds : ndarray, of shape(n_samples, n_clusters), dtype=floating The lower bound on the distance between each sample and each cluster center. This array is modified in place. + + n_threads : int + The number of threads to be used by openmp. """ cdef: int n_samples = X.shape[0] @@ -83,7 +78,9 @@ def init_bounds_dense( floating min_dist, dist int best_cluster, i, j - for i in prange(n_samples, schedule='static', nogil=True): + for i in prange( + n_samples, num_threads=n_threads, schedule='static', nogil=True + ): best_cluster = 0 min_dist = _euclidean_dense_dense(&X[i, 0], ¢ers[0, 0], n_features, False) @@ -101,12 +98,13 @@ def init_bounds_dense( def init_bounds_sparse( - X, # IN - floating[:, ::1] centers, # IN - floating[:, ::1] center_half_distances, # IN - int[::1] labels, # OUT - floating[::1] upper_bounds, # OUT - floating[:, ::1] lower_bounds): # OUT + X, # IN + const floating[:, ::1] centers, # IN + const floating[:, ::1] center_half_distances, # IN + int[::1] labels, # OUT + floating[::1] upper_bounds, # OUT + floating[:, ::1] lower_bounds, # OUT + int n_threads): """Initialize upper and lower bounds for each sample for sparse input data. Given X, centers and the pairwise distances divided by 2.0 between the @@ -144,11 +142,13 @@ def init_bounds_sparse( lower_bounds : ndarray of shape(n_samples, n_clusters), dtype=floating The lower bound on the distance between each sample and each cluster center. This array is modified in place. + + n_threads : int + The number of threads to be used by openmp. """ cdef: int n_samples = X.shape[0] int n_clusters = centers.shape[0] - int n_features = X.shape[1] floating[::1] X_data = X.data int[::1] X_indices = X.indices @@ -159,7 +159,9 @@ def init_bounds_sparse( floating[::1] centers_squared_norms = row_norms(centers, squared=True) - for i in prange(n_samples, schedule='static', nogil=True): + for i in prange( + n_samples, num_threads=n_threads, schedule='static', nogil=True + ): best_cluster = 0 min_dist = _euclidean_sparse_dense( X_data[X_indptr[i]: X_indptr[i + 1]], @@ -182,17 +184,17 @@ def init_bounds_sparse( def elkan_iter_chunked_dense( - np.ndarray[floating, ndim=2, mode='c'] X, # IN - floating[::1] sample_weight, # IN - floating[:, ::1] centers_old, # IN - floating[:, ::1] centers_new, # OUT - floating[::1] weight_in_clusters, # OUT - floating[:, ::1] center_half_distances, # IN - floating[::1] distance_next_center, # IN - floating[::1] upper_bounds, # INOUT - floating[:, ::1] lower_bounds, # INOUT - int[::1] labels, # INOUT - floating[::1] center_shift, # OUT + const floating[:, ::1] X, # IN + const floating[::1] sample_weight, # IN + const floating[:, ::1] centers_old, # IN + floating[:, ::1] centers_new, # OUT + floating[::1] weight_in_clusters, # OUT + const floating[:, ::1] center_half_distances, # IN + const floating[::1] distance_next_center, # IN + floating[::1] upper_bounds, # INOUT + floating[:, ::1] lower_bounds, # INOUT + int[::1] labels, # INOUT + floating[::1] center_shift, # OUT int n_threads, bint update_centers=True): """Single iteration of K-means Elkan algorithm with dense input. @@ -256,12 +258,20 @@ def elkan_iter_chunked_dense( int n_features = X.shape[1] int n_clusters = centers_new.shape[0] + if n_samples == 0: + # An empty array was passed, do nothing and return early (before + # attempting to compute n_chunks). This can typically happen when + # calling the prediction function of a bisecting k-means model with a + # large fraction of outliers. + return + + cdef: # hard-coded number of samples per chunk. Splitting in chunks is - # necessary to get parallelism. Chunk size chosed to be same as lloyd's + # necessary to get parallelism. Chunk size chosen to be same as lloyd's int n_samples_chunk = CHUNK_SIZE if n_samples > CHUNK_SIZE else n_samples int n_chunks = n_samples // n_samples_chunk int n_samples_rem = n_samples % n_samples_chunk - int chunk_idx, n_samples_chunk_eff + int chunk_idx int start, end int i, j, k @@ -269,6 +279,8 @@ def elkan_iter_chunked_dense( floating *centers_new_chunk floating *weight_in_clusters_chunk + omp_lock_t lock + # count remainder chunk in total number of chunks n_chunks += n_samples != n_chunks * n_samples_chunk @@ -278,6 +290,7 @@ def elkan_iter_chunked_dense( if update_centers: memset(¢ers_new[0, 0], 0, n_clusters * n_features * sizeof(floating)) memset(&weight_in_clusters[0], 0, n_clusters * sizeof(floating)) + omp_init_lock(&lock) with nogil, parallel(num_threads=n_threads): # thread local buffers @@ -292,7 +305,7 @@ def elkan_iter_chunked_dense( end = start + n_samples_chunk _update_chunk_dense( - &X[start, 0], + X[start: end], sample_weight[start: end], centers_old, center_half_distances, @@ -304,19 +317,22 @@ def elkan_iter_chunked_dense( weight_in_clusters_chunk, update_centers) - # reduction from local buffers. The gil is necessary for that to avoid - # race conditions. + # reduction from local buffers. if update_centers: - with gil: - for j in range(n_clusters): - weight_in_clusters[j] += weight_in_clusters_chunk[j] - for k in range(n_features): - centers_new[j, k] += centers_new_chunk[j * n_features + k] + # The lock is necessary to avoid race conditions when aggregating + # info from different thread-local buffers. + omp_set_lock(&lock) + for j in range(n_clusters): + weight_in_clusters[j] += weight_in_clusters_chunk[j] + for k in range(n_features): + centers_new[j, k] += centers_new_chunk[j * n_features + k] + omp_unset_lock(&lock) free(centers_new_chunk) free(weight_in_clusters_chunk) if update_centers: + omp_destroy_lock(&lock) _relocate_empty_clusters_dense(X, sample_weight, centers_old, centers_new, weight_in_clusters, labels) @@ -334,20 +350,17 @@ def elkan_iter_chunked_dense( cdef void _update_chunk_dense( - floating *X, # IN - # expecting C alinged 2D array. XXX: Can be - # replaced by const memoryview when cython min - # version is >= 0.3 - floating[::1] sample_weight, # IN - floating[:, ::1] centers_old, # IN - floating[:, ::1] center_half_distances, # IN - floating[::1] distance_next_center, # IN - int[::1] labels, # INOUT - floating[::1] upper_bounds, # INOUT - floating[:, ::1] lower_bounds, # INOUT - floating *centers_new, # OUT - floating *weight_in_clusters, # OUT - bint update_centers) nogil: + const floating[:, ::1] X, # IN + const floating[::1] sample_weight, # IN + const floating[:, ::1] centers_old, # IN + const floating[:, ::1] center_half_distances, # IN + const floating[::1] distance_next_center, # IN + int[::1] labels, # INOUT + floating[::1] upper_bounds, # INOUT + floating[:, ::1] lower_bounds, # INOUT + floating *centers_new, # OUT + floating *weight_in_clusters, # OUT + bint update_centers) noexcept nogil: """K-means combined EM step for one dense data chunk. Compute the partial contribution of a single data chunk to the labels and @@ -375,26 +388,30 @@ cdef void _update_chunk_dense( # If this holds, then center_index is a good candidate for the # sample to be relabelled, and we need to confirm this by # recomputing the upper and lower bounds. - if (j != label + if ( + j != label and (upper_bound > lower_bounds[i, j]) - and (upper_bound > center_half_distances[label, j])): + and (upper_bound > center_half_distances[label, j]) + ): # Recompute upper bound by calculating the actual distance # between the sample and its current assigned center. if not bounds_tight: upper_bound = _euclidean_dense_dense( - X + i * n_features, ¢ers_old[label, 0], n_features, False) + &X[i, 0], ¢ers_old[label, 0], n_features, False) lower_bounds[i, label] = upper_bound bounds_tight = 1 # If the condition still holds, then compute the actual # distance between the sample and center. If this is less # than the previous distance, reassign label. - if (upper_bound > lower_bounds[i, j] - or (upper_bound > center_half_distances[label, j])): + if ( + upper_bound > lower_bounds[i, j] + or (upper_bound > center_half_distances[label, j]) + ): distance = _euclidean_dense_dense( - X + i * n_features, ¢ers_old[j, 0], n_features, False) + &X[i, 0], ¢ers_old[j, 0], n_features, False) lower_bounds[i, j] = distance if distance < upper_bound: label = j @@ -406,21 +423,21 @@ cdef void _update_chunk_dense( if update_centers: weight_in_clusters[label] += sample_weight[i] for k in range(n_features): - centers_new[label * n_features + k] += X[i * n_features + k] * sample_weight[i] + centers_new[label * n_features + k] += X[i, k] * sample_weight[i] def elkan_iter_chunked_sparse( - X, # IN - floating[::1] sample_weight, # IN - floating[:, ::1] centers_old, # IN - floating[:, ::1] centers_new, # OUT - floating[::1] weight_in_clusters, # OUT - floating[:, ::1] center_half_distances, # IN - floating[::1] distance_next_center, # IN - floating[::1] upper_bounds, # INOUT - floating[:, ::1] lower_bounds, # INOUT - int[::1] labels, # INOUT - floating[::1] center_shift, # OUT + X, # IN + const floating[::1] sample_weight, # IN + const floating[:, ::1] centers_old, # IN + floating[:, ::1] centers_new, # OUT + floating[::1] weight_in_clusters, # OUT + const floating[:, ::1] center_half_distances, # IN + const floating[::1] distance_next_center, # IN + floating[::1] upper_bounds, # INOUT + floating[:, ::1] lower_bounds, # INOUT + int[::1] labels, # INOUT + floating[::1] center_shift, # OUT int n_threads, bint update_centers=True): """Single iteration of K-means Elkan algorithm with sparse input. @@ -484,16 +501,24 @@ def elkan_iter_chunked_sparse( int n_features = X.shape[1] int n_clusters = centers_new.shape[0] + if n_samples == 0: + # An empty array was passed, do nothing and return early (before + # attempting to compute n_chunks). This can typically happen when + # calling the prediction function of a bisecting k-means model with a + # large fraction of outliers. + return + + cdef: floating[::1] X_data = X.data int[::1] X_indices = X.indices int[::1] X_indptr = X.indptr # hard-coded number of samples per chunk. Splitting in chunks is - # necessary to get parallelism. Chunk size chosed to be same as lloyd's + # necessary to get parallelism. Chunk size chosen to be same as lloyd's int n_samples_chunk = CHUNK_SIZE if n_samples > CHUNK_SIZE else n_samples int n_chunks = n_samples // n_samples_chunk int n_samples_rem = n_samples % n_samples_chunk - int chunk_idx, n_samples_chunk_eff + int chunk_idx int start, end int i, j, k @@ -503,6 +528,8 @@ def elkan_iter_chunked_sparse( floating *centers_new_chunk floating *weight_in_clusters_chunk + omp_lock_t lock + # count remainder chunk in total number of chunks n_chunks += n_samples != n_chunks * n_samples_chunk @@ -512,6 +539,7 @@ def elkan_iter_chunked_sparse( if update_centers: memset(¢ers_new[0, 0], 0, n_clusters * n_features * sizeof(floating)) memset(&weight_in_clusters[0], 0, n_clusters * sizeof(floating)) + omp_init_lock(&lock) with nogil, parallel(num_threads=n_threads): # thread local buffers @@ -528,7 +556,7 @@ def elkan_iter_chunked_sparse( _update_chunk_sparse( X_data[X_indptr[start]: X_indptr[end]], X_indices[X_indptr[start]: X_indptr[end]], - X_indptr[start: end], + X_indptr[start: end+1], sample_weight[start: end], centers_old, centers_squared_norms, @@ -541,19 +569,22 @@ def elkan_iter_chunked_sparse( weight_in_clusters_chunk, update_centers) - # reduction from local buffers. The gil is necessary for that to avoid - # race conditions. + # reduction from local buffers. if update_centers: - with gil: - for j in range(n_clusters): - weight_in_clusters[j] += weight_in_clusters_chunk[j] - for k in range(n_features): - centers_new[j, k] += centers_new_chunk[j * n_features + k] + # The lock is necessary to avoid race conditions when aggregating + # info from different thread-local buffers. + omp_set_lock(&lock) + for j in range(n_clusters): + weight_in_clusters[j] += weight_in_clusters_chunk[j] + for k in range(n_features): + centers_new[j, k] += centers_new_chunk[j * n_features + k] + omp_unset_lock(&lock) free(centers_new_chunk) free(weight_in_clusters_chunk) if update_centers: + omp_destroy_lock(&lock) _relocate_empty_clusters_sparse( X_data, X_indices, X_indptr, sample_weight, centers_old, centers_new, weight_in_clusters, labels) @@ -572,20 +603,20 @@ def elkan_iter_chunked_sparse( cdef void _update_chunk_sparse( - floating[::1] X_data, # IN - int[::1] X_indices, # IN - int[::1] X_indptr, # IN - floating[::1] sample_weight, # IN - floating[:, ::1] centers_old, # IN - floating[::1] centers_squared_norms, # IN - floating[:, ::1] center_half_distances, # IN - floating[::1] distance_next_center, # IN - int[::1] labels, # INOUT - floating[::1] upper_bounds, # INOUT - floating[:, ::1] lower_bounds, # INOUT - floating *centers_new, # OUT - floating *weight_in_clusters, # OUT - bint update_centers) nogil: + const floating[::1] X_data, # IN + const int[::1] X_indices, # IN + const int[::1] X_indptr, # IN + const floating[::1] sample_weight, # IN + const floating[:, ::1] centers_old, # IN + const floating[::1] centers_squared_norms, # IN + const floating[:, ::1] center_half_distances, # IN + const floating[::1] distance_next_center, # IN + int[::1] labels, # INOUT + floating[::1] upper_bounds, # INOUT + floating[:, ::1] lower_bounds, # INOUT + floating *centers_new, # OUT + floating *weight_in_clusters, # OUT + bint update_centers) noexcept nogil: """K-means combined EM step for one sparse data chunk. Compute the partial contribution of a single data chunk to the labels and @@ -614,9 +645,11 @@ cdef void _update_chunk_sparse( # If this holds, then center_index is a good candidate for the # sample to be relabelled, and we need to confirm this by # recomputing the upper and lower bounds. - if (j != label + if ( + j != label and (upper_bound > lower_bounds[i, j]) - and (upper_bound > center_half_distances[label, j])): + and (upper_bound > center_half_distances[label, j]) + ): # Recompute upper bound by calculating the actual distance # between the sample and its current assigned center. @@ -631,8 +664,10 @@ cdef void _update_chunk_sparse( # If the condition still holds, then compute the actual # distance between the sample and center. If this is less # than the previous distance, reassign label. - if (upper_bound > lower_bounds[i, j] - or (upper_bound > center_half_distances[label, j])): + if ( + upper_bound > lower_bounds[i, j] + or (upper_bound > center_half_distances[label, j]) + ): distance = _euclidean_sparse_dense( X_data[X_indptr[i] - s: X_indptr[i + 1] - s], X_indices[X_indptr[i] - s: X_indptr[i + 1] - s], diff --git a/sklearn/cluster/_k_means_fast.pxd b/sklearn/cluster/_k_means_fast.pxd deleted file mode 100644 index b8dcd947f92c6..0000000000000 --- a/sklearn/cluster/_k_means_fast.pxd +++ /dev/null @@ -1,23 +0,0 @@ -# cython: language_level=3 - - -from cython cimport floating -cimport numpy as np - - -cdef floating _euclidean_dense_dense(floating*, floating*, int, bint) nogil - -cdef floating _euclidean_sparse_dense(floating[::1], int[::1], floating[::1], - floating, bint) nogil - -cpdef void _relocate_empty_clusters_dense( - np.ndarray[floating, ndim=2, mode='c'], floating[::1], floating[:, ::1], - floating[:, ::1], floating[::1], int[::1]) - -cpdef void _relocate_empty_clusters_sparse( - floating[::1], int[::1], int[::1], floating[::1], floating[:, ::1], - floating[:, ::1], floating[::1], int[::1]) - -cdef void _average_centers(floating[:, ::1], floating[::1]) - -cdef void _center_shift(floating[:, ::1], floating[:, ::1], floating[::1]) diff --git a/sklearn/cluster/_k_means_fast.pyx b/sklearn/cluster/_k_means_fast.pyx deleted file mode 100644 index 21bf7dd9bf65a..0000000000000 --- a/sklearn/cluster/_k_means_fast.pyx +++ /dev/null @@ -1,389 +0,0 @@ -# cython: profile=True, boundscheck=False, wraparound=False, cdivision=True -# Profiling is enabled by default as the overhead does not seem to be -# measurable on this specific use case. - -# Author: Peter Prettenhofer -# Olivier Grisel -# Lars Buitinck -# -# License: BSD 3 clause - -# TODO: We still need to use ndarrays instead of typed memoryviews when using -# fused types and when the array may be read-only (for instance when it's -# provided by the user). This is fixed in cython > 0.3. - -import numpy as np -cimport numpy as np -cimport cython -from cython cimport floating -from libc.math cimport sqrt - -from ..utils.extmath import row_norms - - -np.import_array() - -ctypedef np.float64_t DOUBLE -ctypedef np.int32_t INT - - -# Number of samples per data chunk defined as a global constant. -CHUNK_SIZE = 256 - - -cdef floating _euclidean_dense_dense( - floating* a, # IN - floating* b, # IN - int n_features, - bint squared) nogil: - """Euclidean distance between a dense and b dense""" - cdef: - int i - int n = n_features // 4 - int rem = n_features % 4 - floating result = 0 - - # We manually unroll the loop for better cache optimization. - for i in range(n): - result += ((a[0] - b[0]) * (a[0] - b[0]) - +(a[1] - b[1]) * (a[1] - b[1]) - +(a[2] - b[2]) * (a[2] - b[2]) - +(a[3] - b[3]) * (a[3] - b[3])) - a += 4; b += 4 - - for i in range(rem): - result += (a[i] - b[i]) * (a[i] - b[i]) - - return result if squared else sqrt(result) - - -def _euclidean_dense_dense_wrapper(floating[::1] a, floating[::1] b, - bint squared): - """Wrapper of _euclidean_dense_dense for testing purpose""" - return _euclidean_dense_dense(&a[0], &b[0], a.shape[0], squared) - - -cdef floating _euclidean_sparse_dense( - floating[::1] a_data, # IN - int[::1] a_indices, # IN - floating[::1] b, # IN - floating b_squared_norm, - bint squared) nogil: - """Euclidean distance between a sparse and b dense""" - cdef: - int nnz = a_indices.shape[0] - int i - floating tmp, bi - floating result = 0.0 - - for i in range(nnz): - bi = b[a_indices[i]] - tmp = a_data[i] - bi - result += tmp * tmp - bi * bi - - result += b_squared_norm - - if result < 0: result = 0.0 - - return result if squared else sqrt(result) - - -def _euclidean_sparse_dense_wrapper( - floating[::1] a_data, - int[::1] a_indices, - floating[::1] b, - floating b_squared_norm, - bint squared): - """Wrapper of _euclidean_sparse_dense for testing purpose""" - return _euclidean_sparse_dense( - a_data, a_indices, b, b_squared_norm, squared) - - -cpdef floating _inertia_dense( - np.ndarray[floating, ndim=2, mode='c'] X, # IN - floating[::1] sample_weight, # IN - floating[:, ::1] centers, # IN - int[::1] labels): # IN - """Compute inertia for dense input data - - Sum of squared distance between each sample and its assigned center. - """ - cdef: - int n_samples = X.shape[0] - int n_features = X.shape[1] - int i, j - - floating sq_dist = 0.0 - floating inertia = 0.0 - - for i in range(n_samples): - j = labels[i] - sq_dist = _euclidean_dense_dense(&X[i, 0], ¢ers[j, 0], - n_features, True) - inertia += sq_dist * sample_weight[i] - - return inertia - - -cpdef floating _inertia_sparse( - X, # IN - floating[::1] sample_weight, # IN - floating[:, ::1] centers, # IN - int[::1] labels): # IN - """Compute inertia for sparse input data - - Sum of squared distance between each sample and its assigned center. - """ - cdef: - floating[::1] X_data = X.data - int[::1] X_indices = X.indices - int[::1] X_indptr = X.indptr - - int n_samples = X.shape[0] - int n_features = X.shape[1] - int i, j - - floating sq_dist = 0.0 - floating inertia = 0.0 - - floating[::1] centers_squared_norms = row_norms(centers, squared=True) - - for i in range(n_samples): - j = labels[i] - sq_dist = _euclidean_sparse_dense( - X_data[X_indptr[i]: X_indptr[i + 1]], - X_indices[X_indptr[i]: X_indptr[i + 1]], - centers[j], centers_squared_norms[j], True) - inertia += sq_dist * sample_weight[i] - - return inertia - - -cpdef void _relocate_empty_clusters_dense( - np.ndarray[floating, ndim=2, mode='c'] X, # IN - floating[::1] sample_weight, # IN - floating[:, ::1] centers_old, # IN - floating[:, ::1] centers_new, # INOUT - floating[::1] weight_in_clusters, # INOUT - int[::1] labels): # IN - """Relocate centers which have no sample assigned to them.""" - cdef: - int[::1] empty_clusters = np.where(np.equal(weight_in_clusters, 0))[0].astype(np.int32) - int n_empty = empty_clusters.shape[0] - - if n_empty == 0: - return - - cdef: - int n_features = X.shape[1] - - floating[::1] distances = ((np.asarray(X) - np.asarray(centers_old)[labels])**2).sum(axis=1) - int[::1] far_from_centers = np.argpartition(distances, -n_empty)[:-n_empty-1:-1].astype(np.int32) - - int new_cluster_id, old_cluster_id, far_idx, idx, k - floating weight - - for idx in range(n_empty): - - new_cluster_id = empty_clusters[idx] - - far_idx = far_from_centers[idx] - weight = sample_weight[far_idx] - - old_cluster_id = labels[far_idx] - - for k in range(n_features): - centers_new[old_cluster_id, k] -= X[far_idx, k] * weight - centers_new[new_cluster_id, k] = X[far_idx, k] * weight - - weight_in_clusters[new_cluster_id] = weight - weight_in_clusters[old_cluster_id] -= weight - - -cpdef void _relocate_empty_clusters_sparse( - floating[::1] X_data, # IN - int[::1] X_indices, # IN - int[::1] X_indptr, # IN - floating[::1] sample_weight, # IN - floating[:, ::1] centers_old, # IN - floating[:, ::1] centers_new, # INOUT - floating[::1] weight_in_clusters, # INOUT - int[::1] labels): # IN - """Relocate centers which have no sample assigned to them.""" - cdef: - int[::1] empty_clusters = np.where(np.equal(weight_in_clusters, 0))[0].astype(np.int32) - int n_empty = empty_clusters.shape[0] - - if n_empty == 0: - return - - cdef: - int n_samples = X_indptr.shape[0] - 1 - int n_features = centers_old.shape[1] - floating x - int i, j, k - - floating[::1] distances = np.zeros(n_samples, dtype=X_data.base.dtype) - floating[::1] centers_squared_norms = row_norms(centers_old, squared=True) - - for i in range(n_samples): - j = labels[i] - distances[i] = _euclidean_sparse_dense( - X_data[X_indptr[i]: X_indptr[i + 1]], - X_indices[X_indptr[i]: X_indptr[i + 1]], - centers_old[j], centers_squared_norms[j], True) - - cdef: - int[::1] far_from_centers = np.argpartition(distances, -n_empty)[:-n_empty-1:-1].astype(np.int32) - - int new_cluster_id, old_cluster_id, far_idx, idx - floating weight - - for idx in range(n_empty): - - new_cluster_id = empty_clusters[idx] - - far_idx = far_from_centers[idx] - weight = sample_weight[far_idx] - - old_cluster_id = labels[far_idx] - - for k in range(X_indptr[far_idx], X_indptr[far_idx + 1]): - centers_new[old_cluster_id, X_indices[k]] -= X_data[k] * weight - centers_new[new_cluster_id, X_indices[k]] = X_data[k] * weight - - weight_in_clusters[new_cluster_id] = weight - weight_in_clusters[old_cluster_id] -= weight - - -cdef void _average_centers( - floating[:, ::1] centers, # INOUT - floating[::1] weight_in_clusters): # IN - """Average new centers wrt weights.""" - cdef: - int n_clusters = centers.shape[0] - int n_features = centers.shape[1] - int j, k - floating alpha - - for j in range(n_clusters): - if weight_in_clusters[j] > 0: - alpha = 1.0 / weight_in_clusters[j] - for k in range(n_features): - centers[j, k] *= alpha - - -cdef void _center_shift( - floating[:, ::1] centers_old, # IN - floating[:, ::1] centers_new, # IN - floating[::1] center_shift): # OUT - """Compute shift between old and new centers.""" - cdef: - int n_clusters = centers_old.shape[0] - int n_features = centers_old.shape[1] - int j - - for j in range(n_clusters): - center_shift[j] = _euclidean_dense_dense( - ¢ers_new[j, 0], ¢ers_old[j, 0], n_features, False) - - -def _mini_batch_update_csr(X, np.ndarray[floating, ndim=1] sample_weight, - np.ndarray[floating, ndim=1] x_squared_norms, - np.ndarray[floating, ndim=2] centers, - np.ndarray[floating, ndim=1] weight_sums, - np.ndarray[INT, ndim=1] nearest_center, - np.ndarray[floating, ndim=1] old_center, - int compute_squared_diff): - """Incremental update of the centers for sparse MiniBatchKMeans. - - Parameters - ---------- - - X : CSR matrix, dtype float - The complete (pre allocated) training set as a CSR matrix. - - centers : array, shape (n_clusters, n_features) - The cluster centers - - counts : array, shape (n_clusters,) - The vector in which we keep track of the numbers of elements in a - cluster - - Returns - ------- - inertia : float - The inertia of the batch prior to centers update, i.e. the sum - of squared distances to the closest center for each sample. This - is the objective function being minimized by the k-means algorithm. - - squared_diff : float - The sum of squared update (squared norm of the centers position - change). If compute_squared_diff is 0, this computation is skipped and - 0.0 is returned instead. - - Both squared diff and inertia are commonly used to monitor the convergence - of the algorithm. - """ - cdef: - np.ndarray[floating, ndim=1] X_data = X.data - np.ndarray[int, ndim=1] X_indices = X.indices - np.ndarray[int, ndim=1] X_indptr = X.indptr - unsigned int n_samples = X.shape[0] - unsigned int n_clusters = centers.shape[0] - unsigned int n_features = centers.shape[1] - - unsigned int sample_idx, center_idx, feature_idx - unsigned int k - DOUBLE old_weight_sum, new_weight_sum - DOUBLE center_diff - DOUBLE squared_diff = 0.0 - - # move centers to the mean of both old and newly assigned samples - for center_idx in range(n_clusters): - old_weight_sum = weight_sums[center_idx] - new_weight_sum = old_weight_sum - - # count the number of samples assigned to this center - for sample_idx in range(n_samples): - if nearest_center[sample_idx] == center_idx: - new_weight_sum += sample_weight[sample_idx] - - if new_weight_sum == old_weight_sum: - # no new sample: leave this center as it stands - continue - - # rescale the old center to reflect it previous accumulated weight - # with regards to the new data that will be incrementally contributed - if compute_squared_diff: - old_center[:] = centers[center_idx] - centers[center_idx] *= old_weight_sum - - # iterate of over samples assigned to this cluster to move the center - # location by inplace summation - for sample_idx in range(n_samples): - if nearest_center[sample_idx] != center_idx: - continue - - # inplace sum with new samples that are members of this cluster - # and update of the incremental squared difference update of the - # center position - for k in range(X_indptr[sample_idx], X_indptr[sample_idx + 1]): - centers[center_idx, X_indices[k]] += X_data[k] - - # inplace rescale center with updated count - if new_weight_sum > old_weight_sum: - # update the count statistics for this center - weight_sums[center_idx] = new_weight_sum - - # re-scale the updated center with the total new counts - centers[center_idx] /= new_weight_sum - - # update the incremental computation of the squared total - # centers position change - if compute_squared_diff: - for feature_idx in range(n_features): - squared_diff += (old_center[feature_idx] - - centers[center_idx, feature_idx]) ** 2 - - return squared_diff diff --git a/sklearn/cluster/_k_means_lloyd.pyx b/sklearn/cluster/_k_means_lloyd.pyx index 255f4f470a648..a507a6239ab5f 100644 --- a/sklearn/cluster/_k_means_lloyd.pyx +++ b/sklearn/cluster/_k_means_lloyd.pyx @@ -1,40 +1,33 @@ -# cython: profile=True, boundscheck=False, wraparound=False, cdivision=True -# # Licence: BSD 3 clause -# TODO: We still need to use ndarrays instead of typed memoryviews when using -# fused types and when the array may be read-only (for instance when it's -# provided by the user). This is fixed in cython > 0.3. - -import numpy as np -cimport numpy as np from cython cimport floating from cython.parallel import prange, parallel from libc.stdlib cimport malloc, calloc, free -from libc.string cimport memset, memcpy +from libc.string cimport memset from libc.float cimport DBL_MAX, FLT_MAX +from ..utils._openmp_helpers cimport omp_lock_t +from ..utils._openmp_helpers cimport omp_init_lock +from ..utils._openmp_helpers cimport omp_destroy_lock +from ..utils._openmp_helpers cimport omp_set_lock +from ..utils._openmp_helpers cimport omp_unset_lock from ..utils.extmath import row_norms from ..utils._cython_blas cimport _gemm from ..utils._cython_blas cimport RowMajor, Trans, NoTrans -from ._k_means_fast import CHUNK_SIZE -from ._k_means_fast cimport _relocate_empty_clusters_dense -from ._k_means_fast cimport _relocate_empty_clusters_sparse -from ._k_means_fast cimport _average_centers, _center_shift - - -np.import_array() +from ._k_means_common import CHUNK_SIZE +from ._k_means_common cimport _relocate_empty_clusters_dense +from ._k_means_common cimport _relocate_empty_clusters_sparse +from ._k_means_common cimport _average_centers, _center_shift def lloyd_iter_chunked_dense( - np.ndarray[floating, ndim=2, mode='c'] X, # IN - floating[::1] sample_weight, # IN - floating[::1] x_squared_norms, # IN - floating[:, ::1] centers_old, # IN - floating[:, ::1] centers_new, # OUT - floating[::1] weight_in_clusters, # OUT - int[::1] labels, # OUT - floating[::1] center_shift, # OUT + const floating[:, ::1] X, # IN + const floating[::1] sample_weight, # IN + const floating[:, ::1] centers_old, # IN + floating[:, ::1] centers_new, # OUT + floating[::1] weight_in_clusters, # OUT + int[::1] labels, # OUT + floating[::1] center_shift, # OUT int n_threads, bint update_centers=True): """Single iteration of K-means lloyd algorithm with dense input. @@ -50,23 +43,19 @@ def lloyd_iter_chunked_dense( sample_weight : ndarray of shape (n_samples,), dtype=floating The weights for each observation in X. - x_squared_norms : ndarray of shape (n_samples,), dtype=floating - Squared L2 norm of X. - centers_old : ndarray of shape (n_clusters, n_features), dtype=floating Centers before previous iteration, placeholder for the centers after previous iteration. centers_new : ndarray of shape (n_clusters, n_features), dtype=floating Centers after previous iteration, placeholder for the new centers - computed during this iteration. - - centers_squared_norms : ndarray of shape (n_clusters,), dtype=floating - Squared L2 norm of the centers. + computed during this iteration. `centers_new` can be `None` if + `update_centers` is False. weight_in_clusters : ndarray of shape (n_clusters,), dtype=floating Placeholder for the sums of the weights of every observation assigned - to each center. + to each center. `weight_in_clusters` can be `None` if `update_centers` + is False. labels : ndarray of shape (n_samples,), dtype=int labels assignment. @@ -87,14 +76,22 @@ def lloyd_iter_chunked_dense( cdef: int n_samples = X.shape[0] int n_features = X.shape[1] - int n_clusters = centers_new.shape[0] + int n_clusters = centers_old.shape[0] + if n_samples == 0: + # An empty array was passed, do nothing and return early (before + # attempting to compute n_chunks). This can typically happen when + # calling the prediction function of a bisecting k-means model with a + # large fraction of outliers. + return + + cdef: # hard-coded number of samples per chunk. Appeared to be close to # optimal in all situations. int n_samples_chunk = CHUNK_SIZE if n_samples > CHUNK_SIZE else n_samples int n_chunks = n_samples // n_samples_chunk int n_samples_rem = n_samples % n_samples_chunk - int chunk_idx, n_samples_chunk_eff + int chunk_idx int start, end int j, k @@ -105,6 +102,8 @@ def lloyd_iter_chunked_dense( floating *weight_in_clusters_chunk floating *pairwise_distances_chunk + omp_lock_t lock + # count remainder chunk in total number of chunks n_chunks += n_samples != n_chunks * n_samples_chunk @@ -114,6 +113,7 @@ def lloyd_iter_chunked_dense( if update_centers: memset(¢ers_new[0, 0], 0, n_clusters * n_features * sizeof(floating)) memset(&weight_in_clusters[0], 0, n_clusters * sizeof(floating)) + omp_init_lock(&lock) with nogil, parallel(num_threads=n_threads): # thread local buffers @@ -129,9 +129,8 @@ def lloyd_iter_chunked_dense( end = start + n_samples_chunk _update_chunk_dense( - &X[start, 0], + X[start: end], sample_weight[start: end], - x_squared_norms[start: end], centers_old, centers_squared_norms, labels[start: end], @@ -140,41 +139,42 @@ def lloyd_iter_chunked_dense( pairwise_distances_chunk, update_centers) - # reduction from local buffers. The gil is necessary for that to avoid - # race conditions. + # reduction from local buffers. if update_centers: - with gil: - for j in range(n_clusters): - weight_in_clusters[j] += weight_in_clusters_chunk[j] - for k in range(n_features): - centers_new[j, k] += centers_new_chunk[j * n_features + k] + # The lock is necessary to avoid race conditions when aggregating + # info from different thread-local buffers. + omp_set_lock(&lock) + for j in range(n_clusters): + weight_in_clusters[j] += weight_in_clusters_chunk[j] + for k in range(n_features): + centers_new[j, k] += centers_new_chunk[j * n_features + k] + + omp_unset_lock(&lock) free(centers_new_chunk) free(weight_in_clusters_chunk) free(pairwise_distances_chunk) if update_centers: - _relocate_empty_clusters_dense(X, sample_weight, centers_old, - centers_new, weight_in_clusters, labels) + omp_destroy_lock(&lock) + _relocate_empty_clusters_dense( + X, sample_weight, centers_old, centers_new, weight_in_clusters, labels + ) _average_centers(centers_new, weight_in_clusters) _center_shift(centers_old, centers_new, center_shift) cdef void _update_chunk_dense( - floating *X, # IN - # expecting C alinged 2D array. XXX: Can be - # replaced by const memoryview when cython min - # version is >= 0.3 - floating[::1] sample_weight, # IN - floating[::1] x_squared_norms, # IN - floating[:, ::1] centers_old, # IN - floating[::1] centers_squared_norms, # IN - int[::1] labels, # OUT - floating *centers_new, # OUT - floating *weight_in_clusters, # OUT - floating *pairwise_distances, # OUT - bint update_centers) nogil: + const floating[:, ::1] X, # IN + const floating[::1] sample_weight, # IN + const floating[:, ::1] centers_old, # IN + const floating[::1] centers_squared_norms, # IN + int[::1] labels, # OUT + floating *centers_new, # OUT + floating *weight_in_clusters, # OUT + floating *pairwise_distances, # OUT + bint update_centers) noexcept nogil: """K-means combined EM step for one dense data chunk. Compute the partial contribution of a single data chunk to the labels and @@ -199,7 +199,7 @@ cdef void _update_chunk_dense( # pairwise_distances += -2 * X.dot(C.T) _gemm(RowMajor, NoTrans, Trans, n_samples, n_clusters, n_features, - -2.0, X, n_features, ¢ers_old[0, 0], n_features, + -2.0, &X[0, 0], n_features, ¢ers_old[0, 0], n_features, 1.0, pairwise_distances, n_clusters) for i in range(n_samples): @@ -215,18 +215,17 @@ cdef void _update_chunk_dense( if update_centers: weight_in_clusters[label] += sample_weight[i] for k in range(n_features): - centers_new[label * n_features + k] += X[i * n_features + k] * sample_weight[i] + centers_new[label * n_features + k] += X[i, k] * sample_weight[i] def lloyd_iter_chunked_sparse( - X, # IN - floating[::1] sample_weight, # IN - floating[::1] x_squared_norms, # IN - floating[:, ::1] centers_old, # IN - floating[:, ::1] centers_new, # OUT - floating[::1] weight_in_clusters, # OUT - int[::1] labels, # OUT - floating[::1] center_shift, # OUT + X, # IN + const floating[::1] sample_weight, # IN + const floating[:, ::1] centers_old, # IN + floating[:, ::1] centers_new, # OUT + floating[::1] weight_in_clusters, # OUT + int[::1] labels, # OUT + floating[::1] center_shift, # OUT int n_threads, bint update_centers=True): """Single iteration of K-means lloyd algorithm with sparse input. @@ -242,23 +241,19 @@ def lloyd_iter_chunked_sparse( sample_weight : ndarray of shape (n_samples,), dtype=floating The weights for each observation in X. - x_squared_norms : ndarray of shape (n_samples,), dtype=floating - Squared L2 norm of X. - centers_old : ndarray of shape (n_clusters, n_features), dtype=floating Centers before previous iteration, placeholder for the centers after previous iteration. centers_new : ndarray of shape (n_clusters, n_features), dtype=floating Centers after previous iteration, placeholder for the new centers - computed during this iteration. - - centers_squared_norms : ndarray of shape (n_clusters,), dtype=floating - Squared L2 norm of the centers. + computed during this iteration. `centers_new` can be `None` if + `update_centers` is False. weight_in_clusters : ndarray of shape (n_clusters,), dtype=floating Placeholder for the sums of the weights of every observation assigned - to each center. + to each center. `weight_in_clusters` can be `None` if `update_centers` + is False. labels : ndarray of shape (n_samples,), dtype=int labels assignment. @@ -276,19 +271,26 @@ def lloyd_iter_chunked_sparse( the algorithm. This is useful especially when calling predict on a fitted model. """ - # print(X.indices.dtype) cdef: int n_samples = X.shape[0] int n_features = X.shape[1] - int n_clusters = centers_new.shape[0] + int n_clusters = centers_old.shape[0] + + if n_samples == 0: + # An empty array was passed, do nothing and return early (before + # attempting to compute n_chunks). This can typically happen when + # calling the prediction function of a bisecting k-means model with a + # large fraction of outliers. + return - # Chosed same as for dense. Does not have the same impact since with + cdef: + # Choose same as for dense. Does not have the same impact since with # sparse data the pairwise distances matrix is not precomputed. # However, splitting in chunks is necessary to get parallelism. int n_samples_chunk = CHUNK_SIZE if n_samples > CHUNK_SIZE else n_samples int n_chunks = n_samples // n_samples_chunk int n_samples_rem = n_samples % n_samples_chunk - int chunk_idx, n_samples_chunk_eff = 0 + int chunk_idx int start = 0, end = 0 int j, k @@ -302,6 +304,8 @@ def lloyd_iter_chunked_sparse( floating *centers_new_chunk floating *weight_in_clusters_chunk + omp_lock_t lock + # count remainder chunk in total number of chunks n_chunks += n_samples != n_chunks * n_samples_chunk @@ -311,6 +315,7 @@ def lloyd_iter_chunked_sparse( if update_centers: memset(¢ers_new[0, 0], 0, n_clusters * n_features * sizeof(floating)) memset(&weight_in_clusters[0], 0, n_clusters * sizeof(floating)) + omp_init_lock(&lock) with nogil, parallel(num_threads=n_threads): # thread local buffers @@ -327,9 +332,8 @@ def lloyd_iter_chunked_sparse( _update_chunk_sparse( X_data[X_indptr[start]: X_indptr[end]], X_indices[X_indptr[start]: X_indptr[end]], - X_indptr[start: end], + X_indptr[start: end+1], sample_weight[start: end], - x_squared_norms[start: end], centers_old, centers_squared_norms, labels[start: end], @@ -337,19 +341,22 @@ def lloyd_iter_chunked_sparse( weight_in_clusters_chunk, update_centers) - # reduction from local buffers. The gil is necessary for that to avoid - # race conditions. + # reduction from local buffers. if update_centers: - with gil: - for j in range(n_clusters): - weight_in_clusters[j] += weight_in_clusters_chunk[j] - for k in range(n_features): - centers_new[j, k] += centers_new_chunk[j * n_features + k] + # The lock is necessary to avoid race conditions when aggregating + # info from different thread-local buffers. + omp_set_lock(&lock) + for j in range(n_clusters): + weight_in_clusters[j] += weight_in_clusters_chunk[j] + for k in range(n_features): + centers_new[j, k] += centers_new_chunk[j * n_features + k] + omp_unset_lock(&lock) free(centers_new_chunk) free(weight_in_clusters_chunk) if update_centers: + omp_destroy_lock(&lock) _relocate_empty_clusters_sparse( X_data, X_indices, X_indptr, sample_weight, centers_old, centers_new, weight_in_clusters, labels) @@ -359,17 +366,16 @@ def lloyd_iter_chunked_sparse( cdef void _update_chunk_sparse( - floating[::1] X_data, # IN - int[::1] X_indices, # IN - int[::1] X_indptr, # IN - floating[::1] sample_weight, # IN - floating[::1] x_squared_norms, # IN - floating[:, ::1] centers_old, # IN - floating[::1] centers_squared_norms, # IN - int[::1] labels, # OUT - floating *centers_new, # OUT - floating *weight_in_clusters, # OUT - bint update_centers) nogil: + const floating[::1] X_data, # IN + const int[::1] X_indices, # IN + const int[::1] X_indptr, # IN + const floating[::1] sample_weight, # IN + const floating[:, ::1] centers_old, # IN + const floating[::1] centers_squared_norms, # IN + int[::1] labels, # OUT + floating *centers_new, # OUT + floating *weight_in_clusters, # OUT + bint update_centers) noexcept nogil: """K-means combined EM step for one sparse data chunk. Compute the partial contribution of a single data chunk to the labels and diff --git a/sklearn/cluster/_k_means_minibatch.pyx b/sklearn/cluster/_k_means_minibatch.pyx new file mode 100644 index 0000000000000..22ca5255e3889 --- /dev/null +++ b/sklearn/cluster/_k_means_minibatch.pyx @@ -0,0 +1,218 @@ +from cython cimport floating +from cython.parallel cimport parallel, prange +from libc.stdlib cimport malloc, free + + +def _minibatch_update_dense( + const floating[:, ::1] X, # IN + const floating[::1] sample_weight, # IN + const floating[:, ::1] centers_old, # IN + floating[:, ::1] centers_new, # OUT + floating[::1] weight_sums, # INOUT + const int[::1] labels, # IN + int n_threads): + """Update of the centers for dense MiniBatchKMeans. + + Parameters + ---------- + X : ndarray of shape (n_samples, n_features), dtype=floating + The observations to cluster. + + sample_weight : ndarray of shape (n_samples,), dtype=floating + The weights for each observation in X. + + centers_old : ndarray of shape (n_clusters, n_features), dtype=floating + Centers before previous iteration, placeholder for the centers after + previous iteration. + + centers_new : ndarray of shape (n_clusters, n_features), dtype=floating + Centers after previous iteration, placeholder for the new centers + computed during this iteration. + + weight_sums : ndarray of shape (n_clusters,), dtype=floating + Current sums of the accumulated weights for each center. + + labels : ndarray of shape (n_samples,), dtype=int + labels assignment. + + n_threads : int + The number of threads to be used by openmp. + """ + cdef: + int n_samples = X.shape[0] + int n_clusters = centers_old.shape[0] + int cluster_idx + + int *indices + + with nogil, parallel(num_threads=n_threads): + indices = malloc(n_samples * sizeof(int)) + + for cluster_idx in prange(n_clusters, schedule="static"): + update_center_dense(cluster_idx, X, sample_weight, + centers_old, centers_new, weight_sums, labels, + indices) + + free(indices) + + +cdef void update_center_dense( + int cluster_idx, + const floating[:, ::1] X, # IN + const floating[::1] sample_weight, # IN + const floating[:, ::1] centers_old, # IN + floating[:, ::1] centers_new, # OUT + floating[::1] weight_sums, # INOUT + const int[::1] labels, # IN + int *indices) noexcept nogil: # TMP + """Update of a single center for dense MinibatchKMeans""" + cdef: + int n_samples = sample_weight.shape[0] + int n_features = centers_old.shape[1] + floating alpha + int n_indices + int k, sample_idx, feature_idx + + floating wsum = 0 + + # indices = np.where(labels == cluster_idx)[0] + k = 0 + for sample_idx in range(n_samples): + if labels[sample_idx] == cluster_idx: + indices[k] = sample_idx + wsum += sample_weight[sample_idx] + k += 1 + n_indices = k + + if wsum > 0: + # Undo the previous count-based scaling for this cluster center + for feature_idx in range(n_features): + centers_new[cluster_idx, feature_idx] = centers_old[cluster_idx, feature_idx] * weight_sums[cluster_idx] + + # Update cluster with new point members + for k in range(n_indices): + sample_idx = indices[k] + for feature_idx in range(n_features): + centers_new[cluster_idx, feature_idx] += X[sample_idx, feature_idx] * sample_weight[sample_idx] + + # Update the count statistics for this center + weight_sums[cluster_idx] += wsum + + # Rescale to compute mean of all points (old and new) + alpha = 1 / weight_sums[cluster_idx] + for feature_idx in range(n_features): + centers_new[cluster_idx, feature_idx] *= alpha + else: + # No sample was assigned to this cluster in this batch of data + for feature_idx in range(n_features): + centers_new[cluster_idx, feature_idx] = centers_old[cluster_idx, feature_idx] + + +def _minibatch_update_sparse( + X, # IN + const floating[::1] sample_weight, # IN + const floating[:, ::1] centers_old, # IN + floating[:, ::1] centers_new, # OUT + floating[::1] weight_sums, # INOUT + const int[::1] labels, # IN + int n_threads): + """Update of the centers for sparse MiniBatchKMeans. + + Parameters + ---------- + X : sparse matrix of shape (n_samples, n_features), dtype=floating + The observations to cluster. Must be in CSR format. + + sample_weight : ndarray of shape (n_samples,), dtype=floating + The weights for each observation in X. + + centers_old : ndarray of shape (n_clusters, n_features), dtype=floating + Centers before previous iteration, placeholder for the centers after + previous iteration. + + centers_new : ndarray of shape (n_clusters, n_features), dtype=floating + Centers after previous iteration, placeholder for the new centers + computed during this iteration. + + weight_sums : ndarray of shape (n_clusters,), dtype=floating + Current sums of the accumulated weights for each center. + + labels : ndarray of shape (n_samples,), dtype=int + labels assignment. + + n_threads : int + The number of threads to be used by openmp. + """ + cdef: + floating[::1] X_data = X.data + int[::1] X_indices = X.indices + int[::1] X_indptr = X.indptr + int n_samples = X.shape[0] + int n_clusters = centers_old.shape[0] + int cluster_idx + + int *indices + + with nogil, parallel(num_threads=n_threads): + indices = malloc(n_samples * sizeof(int)) + + for cluster_idx in prange(n_clusters, schedule="static"): + update_center_sparse(cluster_idx, X_data, X_indices, X_indptr, + sample_weight, centers_old, centers_new, + weight_sums, labels, indices) + + free(indices) + + +cdef void update_center_sparse( + int cluster_idx, + const floating[::1] X_data, # IN + const int[::1] X_indices, # IN + const int[::1] X_indptr, # IN + const floating[::1] sample_weight, # IN + const floating[:, ::1] centers_old, # IN + floating[:, ::1] centers_new, # OUT + floating[::1] weight_sums, # INOUT + const int[::1] labels, # IN + int *indices) noexcept nogil: # TMP + """Update of a single center for sparse MinibatchKMeans""" + cdef: + int n_samples = sample_weight.shape[0] + int n_features = centers_old.shape[1] + floating alpha + int n_indices + int k, sample_idx, feature_idx + + floating wsum = 0 + + # indices = np.where(labels == cluster_idx)[0] + k = 0 + for sample_idx in range(n_samples): + if labels[sample_idx] == cluster_idx: + indices[k] = sample_idx + wsum += sample_weight[sample_idx] + k += 1 + n_indices = k + + if wsum > 0: + # Undo the previous count-based scaling for this cluster center: + for feature_idx in range(n_features): + centers_new[cluster_idx, feature_idx] = centers_old[cluster_idx, feature_idx] * weight_sums[cluster_idx] + + # Update cluster with new point members + for k in range(n_indices): + sample_idx = indices[k] + for feature_idx in range(X_indptr[sample_idx], X_indptr[sample_idx + 1]): + centers_new[cluster_idx, X_indices[feature_idx]] += X_data[feature_idx] * sample_weight[sample_idx] + + # Update the count statistics for this center + weight_sums[cluster_idx] += wsum + + # Rescale to compute mean of all points (old and new) + alpha = 1 / weight_sums[cluster_idx] + for feature_idx in range(n_features): + centers_new[cluster_idx, feature_idx] *= alpha + else: + # No sample was assigned to this cluster in this batch of data + for feature_idx in range(n_features): + centers_new[cluster_idx, feature_idx] = centers_old[cluster_idx, feature_idx] diff --git a/sklearn/cluster/_kmeans.py b/sklearn/cluster/_kmeans.py index 17272858ae476..11c85610239cc 100644 --- a/sklearn/cluster/_kmeans.py +++ b/sklearn/cluster/_kmeans.py @@ -1,55 +1,80 @@ """K-means clustering.""" -# Authors: Gael Varoquaux -# Thomas Rueckstiess -# James Bergstra -# Jan Schlueter -# Nelle Varoquaux -# Peter Prettenhofer -# Olivier Grisel -# Mathieu Blondel -# Robert Layton -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import warnings +from abc import ABC, abstractmethod +from numbers import Integral, Real import numpy as np import scipy.sparse as sp -from threadpoolctl import threadpool_limits -from threadpoolctl import threadpool_info -from ..base import BaseEstimator, ClusterMixin, TransformerMixin -from ..metrics.pairwise import euclidean_distances -from ..metrics.pairwise import _euclidean_distances +from ..base import ( + BaseEstimator, + ClassNamePrefixFeaturesOutMixin, + ClusterMixin, + TransformerMixin, + _fit_context, +) +from ..exceptions import ConvergenceWarning +from ..metrics.pairwise import _euclidean_distances, euclidean_distances +from ..utils import check_array, check_random_state +from ..utils._openmp_helpers import _openmp_effective_n_threads +from ..utils._param_validation import Interval, StrOptions, validate_params from ..utils.extmath import row_norms, stable_cumsum -from ..utils.sparsefuncs_fast import assign_rows_csr +from ..utils.parallel import ( + _get_threadpool_controller, + _threadpool_controller_decorator, +) from ..utils.sparsefuncs import mean_variance_axis -from ..utils.validation import _deprecate_positional_args -from ..utils import check_array -from ..utils import gen_batches -from ..utils import check_random_state -from ..utils import deprecated -from ..utils.validation import check_is_fitted, _check_sample_weight -from ..utils._openmp_helpers import _openmp_effective_n_threads -from ..exceptions import ConvergenceWarning -from ._k_means_fast import CHUNK_SIZE -from ._k_means_fast import _inertia_dense -from ._k_means_fast import _inertia_sparse -from ._k_means_fast import _mini_batch_update_csr -from ._k_means_lloyd import lloyd_iter_chunked_dense -from ._k_means_lloyd import lloyd_iter_chunked_sparse -from ._k_means_elkan import init_bounds_dense -from ._k_means_elkan import init_bounds_sparse -from ._k_means_elkan import elkan_iter_chunked_dense -from ._k_means_elkan import elkan_iter_chunked_sparse - +from ..utils.sparsefuncs_fast import assign_rows_csr +from ..utils.validation import ( + _check_sample_weight, + _is_arraylike_not_scalar, + check_is_fitted, + validate_data, +) +from ._k_means_common import ( + CHUNK_SIZE, + _inertia_dense, + _inertia_sparse, + _is_same_clustering, +) +from ._k_means_elkan import ( + elkan_iter_chunked_dense, + elkan_iter_chunked_sparse, + init_bounds_dense, + init_bounds_sparse, +) +from ._k_means_lloyd import lloyd_iter_chunked_dense, lloyd_iter_chunked_sparse +from ._k_means_minibatch import _minibatch_update_dense, _minibatch_update_sparse ############################################################################### # Initialization heuristic -def kmeans_plusplus(X, n_clusters, *, x_squared_norms=None, - random_state=None, n_local_trials=None): - """Init n_clusters seeds according to k-means++ + +@validate_params( + { + "X": ["array-like", "sparse matrix"], + "n_clusters": [Interval(Integral, 1, None, closed="left")], + "sample_weight": ["array-like", None], + "x_squared_norms": ["array-like", None], + "random_state": ["random_state"], + "n_local_trials": [Interval(Integral, 1, None, closed="left"), None], + }, + prefer_skip_nested_validation=True, +) +def kmeans_plusplus( + X, + n_clusters, + *, + sample_weight=None, + x_squared_norms=None, + random_state=None, + n_local_trials=None, +): + """Init n_clusters seeds according to k-means++. .. versionadded:: 0.24 @@ -59,7 +84,14 @@ def kmeans_plusplus(X, n_clusters, *, x_squared_norms=None, The data to pick seeds from. n_clusters : int - The number of centroids to initialize + The number of centroids to initialize. + + sample_weight : array-like of shape (n_samples,), default=None + The weights for each observation in `X`. If `None`, all observations + are assigned equal weight. `sample_weight` is ignored if `init` + is a callable or a user provided array. + + .. versionadded:: 1.3 x_squared_norms : array-like of shape (n_samples,), default=None Squared Euclidean norm of each data point. @@ -73,12 +105,15 @@ def kmeans_plusplus(X, n_clusters, *, x_squared_norms=None, The number of seeding trials for each center (except the first), of which the one reducing inertia the most is greedily chosen. Set to None to make the number of trials depend logarithmically - on the number of seeds (2+log(k)). + on the number of seeds (2+log(k)) which is the recommended setting. + Setting to 1 disables the greedy cluster selection and recovers the + vanilla k-means++ algorithm which was empirically shown to work less + well than its greedy variant. Returns ------- centers : ndarray of shape (n_clusters, n_features) - The inital centers for k-means. + The initial centers for k-means. indices : ndarray of shape (n_clusters,) The index location of the chosen centers in the data array X. For a @@ -100,49 +135,45 @@ def kmeans_plusplus(X, n_clusters, *, x_squared_norms=None, ... [10, 2], [10, 4], [10, 0]]) >>> centers, indices = kmeans_plusplus(X, n_clusters=2, random_state=0) >>> centers - array([[10, 4], + array([[10, 2], [ 1, 0]]) >>> indices - array([4, 2]) + array([3, 2]) """ - # Check data - check_array(X, accept_sparse='csr', - dtype=[np.float64, np.float32]) + check_array(X, accept_sparse="csr", dtype=[np.float64, np.float32]) + sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype) if X.shape[0] < n_clusters: - raise ValueError(f"n_samples={X.shape[0]} should be >= " - f"n_clusters={n_clusters}.") + raise ValueError( + f"n_samples={X.shape[0]} should be >= n_clusters={n_clusters}." + ) # Check parameters if x_squared_norms is None: x_squared_norms = row_norms(X, squared=True) else: - x_squared_norms = check_array(x_squared_norms, - dtype=X.dtype, - ensure_2d=False) + x_squared_norms = check_array(x_squared_norms, dtype=X.dtype, ensure_2d=False) if x_squared_norms.shape[0] != X.shape[0]: raise ValueError( f"The length of x_squared_norms {x_squared_norms.shape[0]} should " - f"be equal to the length of n_samples {X.shape[0]}.") - - if n_local_trials is not None and n_local_trials < 1: - raise ValueError( - f"n_local_trials is set to {n_local_trials} but should be an " - f"integer value greater than zero.") + f"be equal to the length of n_samples {X.shape[0]}." + ) random_state = check_random_state(random_state) # Call private k-means++ - centers, indices = _kmeans_plusplus(X, n_clusters, x_squared_norms, - random_state, n_local_trials) + centers, indices = _kmeans_plusplus( + X, n_clusters, x_squared_norms, sample_weight, random_state, n_local_trials + ) return centers, indices -def _kmeans_plusplus(X, n_clusters, x_squared_norms, - random_state, n_local_trials=None): +def _kmeans_plusplus( + X, n_clusters, x_squared_norms, sample_weight, random_state, n_local_trials=None +): """Computational component for initialization of n_clusters by k-means++. Prior validation of data is assumed. @@ -154,6 +185,9 @@ def _kmeans_plusplus(X, n_clusters, x_squared_norms, n_clusters : int The number of seeds to choose. + sample_weight : ndarray of shape (n_samples,) + The weights for each observation in `X`. + x_squared_norms : ndarray of shape (n_samples,) Squared Euclidean norm of each data point. @@ -170,7 +204,7 @@ def _kmeans_plusplus(X, n_clusters, x_squared_norms, Returns ------- centers : ndarray of shape (n_clusters, n_features) - The inital centers for k-means. + The initial centers for k-means. indices : ndarray of shape (n_clusters,) The index location of the chosen centers in the data array X. For a @@ -188,39 +222,39 @@ def _kmeans_plusplus(X, n_clusters, x_squared_norms, n_local_trials = 2 + int(np.log(n_clusters)) # Pick first center randomly and track index of point - center_id = random_state.randint(n_samples) + center_id = random_state.choice(n_samples, p=sample_weight / sample_weight.sum()) indices = np.full(n_clusters, -1, dtype=int) if sp.issparse(X): - centers[0] = X[center_id].toarray() + centers[0] = X[[center_id]].toarray() else: centers[0] = X[center_id] indices[0] = center_id # Initialize list of closest distances and calculate current potential closest_dist_sq = _euclidean_distances( - centers[0, np.newaxis], X, Y_norm_squared=x_squared_norms, - squared=True) - current_pot = closest_dist_sq.sum() + centers[0, np.newaxis], X, Y_norm_squared=x_squared_norms, squared=True + ) + current_pot = closest_dist_sq @ sample_weight # Pick the remaining n_clusters-1 points for c in range(1, n_clusters): # Choose center candidates by sampling with probability proportional # to the squared distance to the closest existing center - rand_vals = random_state.random_sample(n_local_trials) * current_pot - candidate_ids = np.searchsorted(stable_cumsum(closest_dist_sq), - rand_vals) + rand_vals = random_state.uniform(size=n_local_trials) * current_pot + candidate_ids = np.searchsorted( + stable_cumsum(sample_weight * closest_dist_sq), rand_vals + ) # XXX: numerical imprecision can result in a candidate_id out of range - np.clip(candidate_ids, None, closest_dist_sq.size - 1, - out=candidate_ids) + np.clip(candidate_ids, None, closest_dist_sq.size - 1, out=candidate_ids) # Compute distances to center candidates distance_to_candidates = _euclidean_distances( - X[candidate_ids], X, Y_norm_squared=x_squared_norms, squared=True) + X[candidate_ids], X, Y_norm_squared=x_squared_norms, squared=True + ) # update closest distances squared and potential for each candidate - np.minimum(closest_dist_sq, distance_to_candidates, - out=distance_to_candidates) - candidates_pot = distance_to_candidates.sum(axis=1) + np.minimum(closest_dist_sq, distance_to_candidates, out=distance_to_candidates) + candidates_pot = distance_to_candidates @ sample_weight.reshape(-1, 1) # Decide which candidate is the best best_candidate = np.argmin(candidates_pot) @@ -230,7 +264,7 @@ def _kmeans_plusplus(X, n_clusters, x_squared_norms, # Permanently add best center candidate found in local tries if sp.issparse(X): - centers[c] = X[best_candidate].toarray() + centers[c] = X[[best_candidate]].toarray() else: centers[c] = X[best_candidate] indices[c] = best_candidate @@ -241,6 +275,7 @@ def _kmeans_plusplus(X, n_clusters, x_squared_norms, ############################################################################### # K-means batch estimation by EM (expectation maximization) + def _tolerance(X, tol): """Return a tolerance which is dependent on the dataset.""" if tol == 0: @@ -252,12 +287,30 @@ def _tolerance(X, tol): return np.mean(variances) * tol -@_deprecate_positional_args -def k_means(X, n_clusters, *, sample_weight=None, init='k-means++', - precompute_distances='deprecated', n_init=10, max_iter=300, - verbose=False, tol=1e-4, random_state=None, copy_x=True, - n_jobs='deprecated', algorithm="auto", return_n_iter=False): - """K-means clustering algorithm. +@validate_params( + { + "X": ["array-like", "sparse matrix"], + "sample_weight": ["array-like", None], + "return_n_iter": [bool], + }, + prefer_skip_nested_validation=False, +) +def k_means( + X, + n_clusters, + *, + sample_weight=None, + init="k-means++", + n_init="auto", + max_iter=300, + verbose=False, + tol=1e-4, + random_state=None, + copy_x=True, + algorithm="lloyd", + return_n_iter=False, +): + """Perform K-means clustering algorithm. Read more in the :ref:`User Guide `. @@ -273,46 +326,39 @@ def k_means(X, n_clusters, *, sample_weight=None, init='k-means++', centroids to generate. sample_weight : array-like of shape (n_samples,), default=None - The weights for each observation in X. If None, all observations - are assigned equal weight. + The weights for each observation in `X`. If `None`, all observations + are assigned equal weight. `sample_weight` is not used during + initialization if `init` is a callable or a user provided array. init : {'k-means++', 'random'}, callable or array-like of shape \ (n_clusters, n_features), default='k-means++' Method for initialization: - 'k-means++' : selects initial cluster centers for k-mean - clustering in a smart way to speed up convergence. See section - Notes in k_init for more details. - - 'random': choose `n_clusters` observations (rows) at random from data - for the initial centroids. - - If an array is passed, it should be of shape (n_clusters, n_features) - and gives the initial centers. - - If a callable is passed, it should take arguments X, n_clusters and a - random state and return an initialization. - - precompute_distances : {'auto', True, False} - Precompute distances (faster but takes more memory). - - 'auto' : do not precompute distances if n_samples * n_clusters > 12 - million. This corresponds to about 100MB overhead per job using - double precision. - - True : always precompute distances - - False : never precompute distances - - .. deprecated:: 0.23 - 'precompute_distances' was deprecated in version 0.23 and will be - removed in 1.0 (renaming of 0.25). It has no effect. - - n_init : int, default=10 + - `'k-means++'` : selects initial cluster centers for k-mean + clustering in a smart way to speed up convergence. See section + Notes in k_init for more details. + - `'random'`: choose `n_clusters` observations (rows) at random from data + for the initial centroids. + - If an array is passed, it should be of shape `(n_clusters, n_features)` + and gives the initial centers. + - If a callable is passed, it should take arguments `X`, `n_clusters` and a + random state and return an initialization. + + n_init : 'auto' or int, default="auto" Number of time the k-means algorithm will be run with different centroid seeds. The final results will be the best output of n_init consecutive runs in terms of inertia. + When `n_init='auto'`, the number of runs depends on the value of init: + 10 if using `init='random'` or `init` is a callable; + 1 if using `init='k-means++'` or `init` is an array-like. + + .. versionadded:: 1.2 + Added 'auto' option for `n_init`. + + .. versionchanged:: 1.4 + Default value for `n_init` changed to `'auto'`. + max_iter : int, default=300 Maximum number of iterations of the k-means algorithm to run. @@ -331,34 +377,27 @@ def k_means(X, n_clusters, *, sample_weight=None, init='k-means++', copy_x : bool, default=True When pre-computing distances it is more numerically accurate to center - the data first. If copy_x is True (default), then the original data is + the data first. If `copy_x` is True (default), then the original data is not modified. If False, the original data is modified, and put back before the function returns, but small numerical differences may be introduced by subtracting and then adding the data mean. Note that if the original data is not C-contiguous, a copy will be made even if - copy_x is False. If the original data is sparse, but not in CSR format, - a copy will be made even if copy_x is False. - - n_jobs : int, default=None - The number of OpenMP threads to use for the computation. Parallelism is - sample-wise on the main cython loop which assigns each sample to its - closest center. - - ``None`` or ``-1`` means using all processors. + `copy_x` is False. If the original data is sparse, but not in CSR format, + a copy will be made even if `copy_x` is False. - .. deprecated:: 0.23 - ``n_jobs`` was deprecated in version 0.23 and will be removed in - 1.0 (renaming of 0.25). + algorithm : {"lloyd", "elkan"}, default="lloyd" + K-means algorithm to use. The classical EM-style algorithm is `"lloyd"`. + The `"elkan"` variation can be more efficient on some datasets with + well-defined clusters, by using the triangle inequality. However it's + more memory intensive due to the allocation of an extra array of shape + `(n_samples, n_clusters)`. - algorithm : {"auto", "full", "elkan"}, default="auto" - K-means algorithm to use. The classical EM-style algorithm is "full". - The "elkan" variation is more efficient on data with well-defined - clusters, by using the triangle inequality. However it's more memory - intensive due to the allocation of an extra array of shape - (n_samples, n_clusters). + .. versionchanged:: 0.18 + Added Elkan algorithm - For now "auto" (kept for backward compatibility) chooses "elkan" but it - might change in the future for a better heuristic. + .. versionchanged:: 1.1 + Renamed "full" to "lloyd", and deprecated "auto" and "full". + Changed "auto" to use "lloyd" instead of "elkan". return_n_iter : bool, default=False Whether or not to return the number of iterations. @@ -369,7 +408,7 @@ def k_means(X, n_clusters, *, sample_weight=None, init='k-means++', Centroids found at the last iteration of k-means. label : ndarray of shape (n_samples,) - label[i] is the code or index of the centroid the + The `label[i]` is the code or index of the centroid the i'th observation is closest to. inertia : float @@ -379,12 +418,34 @@ def k_means(X, n_clusters, *, sample_weight=None, init='k-means++', best_n_iter : int Number of iterations corresponding to the best results. Returned only if `return_n_iter` is set to True. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.cluster import k_means + >>> X = np.array([[1, 2], [1, 4], [1, 0], + ... [10, 2], [10, 4], [10, 0]]) + >>> centroid, label, inertia = k_means( + ... X, n_clusters=2, n_init="auto", random_state=0 + ... ) + >>> centroid + array([[10., 2.], + [ 1., 2.]]) + >>> label + array([1, 1, 1, 0, 0, 0], dtype=int32) + >>> inertia + 16.0 """ est = KMeans( - n_clusters=n_clusters, init=init, n_init=n_init, max_iter=max_iter, - verbose=verbose, precompute_distances=precompute_distances, tol=tol, - random_state=random_state, copy_x=copy_x, n_jobs=n_jobs, - algorithm=algorithm + n_clusters=n_clusters, + init=init, + n_init=n_init, + max_iter=max_iter, + verbose=verbose, + tol=tol, + random_state=random_state, + copy_x=copy_x, + algorithm=algorithm, ).fit(X, sample_weight=sample_weight) if return_n_iter: return est.cluster_centers_, est.labels_, est.inertia_, est.n_iter_ @@ -392,9 +453,15 @@ def k_means(X, n_clusters, *, sample_weight=None, init='k-means++', return est.cluster_centers_, est.labels_, est.inertia_ -def _kmeans_single_elkan(X, sample_weight, centers_init, max_iter=300, - verbose=False, x_squared_norms=None, tol=1e-4, - n_threads=1): +def _kmeans_single_elkan( + X, + sample_weight, + centers_init, + max_iter=300, + verbose=False, + tol=1e-4, + n_threads=1, +): """A single run of k-means elkan, assumes preparation completed prior. Parameters @@ -414,9 +481,6 @@ def _kmeans_single_elkan(X, sample_weight, centers_init, max_iter=300, verbose : bool, default=False Verbosity mode. - x_squared_norms : array-like, default=None - Precomputed x_squared_norms. - tol : float, default=1e-4 Relative tolerance with regards to Frobenius norm of the difference in the cluster centers of two consecutive iterations to declare @@ -455,8 +519,9 @@ def _kmeans_single_elkan(X, sample_weight, centers_init, max_iter=300, labels = np.full(n_samples, -1, dtype=np.int32) labels_old = labels.copy() center_half_distances = euclidean_distances(centers) / 2 - distance_next_center = np.partition(np.asarray(center_half_distances), - kth=1, axis=0)[1] + distance_next_center = np.partition( + np.asarray(center_half_distances), kth=1, axis=0 + )[1] upper_bounds = np.zeros(n_samples, dtype=X.dtype) lower_bounds = np.zeros((n_samples, n_clusters), dtype=X.dtype) center_shift = np.zeros(n_clusters, dtype=X.dtype) @@ -470,25 +535,43 @@ def _kmeans_single_elkan(X, sample_weight, centers_init, max_iter=300, elkan_iter = elkan_iter_chunked_dense _inertia = _inertia_dense - init_bounds(X, centers, center_half_distances, - labels, upper_bounds, lower_bounds) + init_bounds( + X, + centers, + center_half_distances, + labels, + upper_bounds, + lower_bounds, + n_threads=n_threads, + ) strict_convergence = False for i in range(max_iter): - elkan_iter(X, sample_weight, centers, centers_new, - weight_in_clusters, center_half_distances, - distance_next_center, upper_bounds, lower_bounds, - labels, center_shift, n_threads) + elkan_iter( + X, + sample_weight, + centers, + centers_new, + weight_in_clusters, + center_half_distances, + distance_next_center, + upper_bounds, + lower_bounds, + labels, + center_shift, + n_threads, + ) # compute new pairwise distances between centers and closest other # center of each center for next iterations center_half_distances = euclidean_distances(centers_new) / 2 distance_next_center = np.partition( - np.asarray(center_half_distances), kth=1, axis=0)[1] + np.asarray(center_half_distances), kth=1, axis=0 + )[1] if verbose: - inertia = _inertia(X, sample_weight, centers, labels) + inertia = _inertia(X, sample_weight, centers, labels, n_threads) print(f"Iteration {i}, inertia {inertia}") centers, centers_new = centers_new, centers @@ -504,27 +587,49 @@ def _kmeans_single_elkan(X, sample_weight, centers_init, max_iter=300, center_shift_tot = (center_shift**2).sum() if center_shift_tot <= tol: if verbose: - print(f"Converged at iteration {i}: center shift " - f"{center_shift_tot} within tolerance {tol}.") + print( + f"Converged at iteration {i}: center shift " + f"{center_shift_tot} within tolerance {tol}." + ) break labels_old[:] = labels if not strict_convergence: # rerun E-step so that predicted labels match cluster centers - elkan_iter(X, sample_weight, centers, centers, weight_in_clusters, - center_half_distances, distance_next_center, - upper_bounds, lower_bounds, labels, center_shift, - n_threads, update_centers=False) - - inertia = _inertia(X, sample_weight, centers, labels) + elkan_iter( + X, + sample_weight, + centers, + centers, + weight_in_clusters, + center_half_distances, + distance_next_center, + upper_bounds, + lower_bounds, + labels, + center_shift, + n_threads, + update_centers=False, + ) + + inertia = _inertia(X, sample_weight, centers, labels, n_threads) return labels, inertia, centers, i + 1 -def _kmeans_single_lloyd(X, sample_weight, centers_init, max_iter=300, - verbose=False, x_squared_norms=None, tol=1e-4, - n_threads=1): +# Threadpoolctl context to limit the number of threads in second level of +# nested parallelism (i.e. BLAS) to avoid oversubscription. +@_threadpool_controller_decorator(limits=1, user_api="blas") +def _kmeans_single_lloyd( + X, + sample_weight, + centers_init, + max_iter=300, + verbose=False, + tol=1e-4, + n_threads=1, +): """A single run of k-means lloyd, assumes preparation completed prior. Parameters @@ -544,9 +649,6 @@ def _kmeans_single_lloyd(X, sample_weight, centers_init, max_iter=300, verbose : bool, default=False Verbosity mode - x_squared_norms : ndarray of shape (n_samples,), default=None - Precomputed x_squared_norms. - tol : float, default=1e-4 Relative tolerance with regards to Frobenius norm of the difference in the cluster centers of two consecutive iterations to declare @@ -594,49 +696,63 @@ def _kmeans_single_lloyd(X, sample_weight, centers_init, max_iter=300, strict_convergence = False - # Threadpoolctl context to limit the number of threads in second level of - # nested parallelism (i.e. BLAS) to avoid oversubsciption. - with threadpool_limits(limits=1, user_api="blas"): - for i in range(max_iter): - lloyd_iter(X, sample_weight, x_squared_norms, centers, centers_new, - weight_in_clusters, labels, center_shift, n_threads) + for i in range(max_iter): + lloyd_iter( + X, + sample_weight, + centers, + centers_new, + weight_in_clusters, + labels, + center_shift, + n_threads, + ) - if verbose: - inertia = _inertia(X, sample_weight, centers, labels) - print(f"Iteration {i}, inertia {inertia}.") + if verbose: + inertia = _inertia(X, sample_weight, centers, labels, n_threads) + print(f"Iteration {i}, inertia {inertia}.") - centers, centers_new = centers_new, centers + centers, centers_new = centers_new, centers - if np.array_equal(labels, labels_old): - # First check the labels for strict convergence. + if np.array_equal(labels, labels_old): + # First check the labels for strict convergence. + if verbose: + print(f"Converged at iteration {i}: strict convergence.") + strict_convergence = True + break + else: + # No strict convergence, check for tol based convergence. + center_shift_tot = (center_shift**2).sum() + if center_shift_tot <= tol: if verbose: - print(f"Converged at iteration {i}: strict convergence.") - strict_convergence = True + print( + f"Converged at iteration {i}: center shift " + f"{center_shift_tot} within tolerance {tol}." + ) break - else: - # No strict convergence, check for tol based convergence. - center_shift_tot = (center_shift**2).sum() - if center_shift_tot <= tol: - if verbose: - print(f"Converged at iteration {i}: center shift " - f"{center_shift_tot} within tolerance {tol}.") - break - - labels_old[:] = labels - if not strict_convergence: - # rerun E-step so that predicted labels match cluster centers - lloyd_iter(X, sample_weight, x_squared_norms, centers, centers, - weight_in_clusters, labels, center_shift, n_threads, - update_centers=False) + labels_old[:] = labels - inertia = _inertia(X, sample_weight, centers, labels) + if not strict_convergence: + # rerun E-step so that predicted labels match cluster centers + lloyd_iter( + X, + sample_weight, + centers, + centers, + weight_in_clusters, + labels, + center_shift, + n_threads, + update_centers=False, + ) + + inertia = _inertia(X, sample_weight, centers, labels, n_threads) return labels, inertia, centers, i + 1 -def _labels_inertia(X, sample_weight, x_squared_norms, centers, - n_threads=None): +def _labels_inertia(X, sample_weight, centers, n_threads=1, return_inertia=True): """E step of the K-means EM algorithm. Compute the labels and the inertia of the given samples and centers. @@ -657,11 +773,14 @@ def _labels_inertia(X, sample_weight, x_squared_norms, centers, centers : ndarray of shape (n_clusters, n_features) The cluster centers. - n_threads : int, default=None + n_threads : int, default=1 The number of OpenMP threads to use for the computation. Parallelism is sample-wise on the main cython loop which assigns each sample to its closest center. + return_inertia : bool, default=True + Whether to compute and return the inertia. + Returns ------- labels : ndarray of shape (n_samples,) @@ -669,15 +788,13 @@ def _labels_inertia(X, sample_weight, x_squared_norms, centers, inertia : float Sum of squared distances of samples to their closest cluster center. + Inertia is only returned if return_inertia is True. """ n_samples = X.shape[0] n_clusters = centers.shape[0] - n_threads = _openmp_effective_n_threads(n_threads) - labels = np.full(n_samples, -1, dtype=np.int32) - weight_in_clusters = np.zeros(n_clusters, dtype=centers.dtype) - center_shift = np.zeros_like(weight_in_clusters) + center_shift = np.zeros(n_clusters, dtype=centers.dtype) if sp.issparse(X): _labels = lloyd_iter_chunked_sparse @@ -686,276 +803,112 @@ def _labels_inertia(X, sample_weight, x_squared_norms, centers, _labels = lloyd_iter_chunked_dense _inertia = _inertia_dense - _labels(X, sample_weight, x_squared_norms, centers, centers, - weight_in_clusters, labels, center_shift, n_threads, - update_centers=False) - - inertia = _inertia(X, sample_weight, centers, labels) - - return labels, inertia - - -class KMeans(TransformerMixin, ClusterMixin, BaseEstimator): - """K-Means clustering. - - Read more in the :ref:`User Guide `. - - Parameters - ---------- - - n_clusters : int, default=8 - The number of clusters to form as well as the number of - centroids to generate. - - init : {'k-means++', 'random'}, callable or array-like of shape \ - (n_clusters, n_features), default='k-means++' - Method for initialization: - - 'k-means++' : selects initial cluster centers for k-mean - clustering in a smart way to speed up convergence. See section - Notes in k_init for more details. - - 'random': choose `n_clusters` observations (rows) at random from data - for the initial centroids. - - If an array is passed, it should be of shape (n_clusters, n_features) - and gives the initial centers. - - If a callable is passed, it should take arguments X, n_clusters and a - random state and return an initialization. - - n_init : int, default=10 - Number of time the k-means algorithm will be run with different - centroid seeds. The final results will be the best output of - n_init consecutive runs in terms of inertia. - - max_iter : int, default=300 - Maximum number of iterations of the k-means algorithm for a - single run. - - tol : float, default=1e-4 - Relative tolerance with regards to Frobenius norm of the difference - in the cluster centers of two consecutive iterations to declare - convergence. - - precompute_distances : {'auto', True, False}, default='auto' - Precompute distances (faster but takes more memory). - - 'auto' : do not precompute distances if n_samples * n_clusters > 12 - million. This corresponds to about 100MB overhead per job using - double precision. - - True : always precompute distances. - - False : never precompute distances. - - .. deprecated:: 0.23 - 'precompute_distances' was deprecated in version 0.22 and will be - removed in 1.0 (renaming of 0.25). It has no effect. - - verbose : int, default=0 - Verbosity mode. - - random_state : int, RandomState instance or None, default=None - Determines random number generation for centroid initialization. Use - an int to make the randomness deterministic. - See :term:`Glossary `. - - copy_x : bool, default=True - When pre-computing distances it is more numerically accurate to center - the data first. If copy_x is True (default), then the original data is - not modified. If False, the original data is modified, and put back - before the function returns, but small numerical differences may be - introduced by subtracting and then adding the data mean. Note that if - the original data is not C-contiguous, a copy will be made even if - copy_x is False. If the original data is sparse, but not in CSR format, - a copy will be made even if copy_x is False. - - n_jobs : int, default=None - The number of OpenMP threads to use for the computation. Parallelism is - sample-wise on the main cython loop which assigns each sample to its - closest center. - - ``None`` or ``-1`` means using all processors. - - .. deprecated:: 0.23 - ``n_jobs`` was deprecated in version 0.23 and will be removed in - 1.0 (renaming of 0.25). - - algorithm : {"auto", "full", "elkan"}, default="auto" - K-means algorithm to use. The classical EM-style algorithm is "full". - The "elkan" variation is more efficient on data with well-defined - clusters, by using the triangle inequality. However it's more memory - intensive due to the allocation of an extra array of shape - (n_samples, n_clusters). - - For now "auto" (kept for backward compatibiliy) chooses "elkan" but it - might change in the future for a better heuristic. - - .. versionchanged:: 0.18 - Added Elkan algorithm - - Attributes - ---------- - cluster_centers_ : ndarray of shape (n_clusters, n_features) - Coordinates of cluster centers. If the algorithm stops before fully - converging (see ``tol`` and ``max_iter``), these will not be - consistent with ``labels_``. - - labels_ : ndarray of shape (n_samples,) - Labels of each point - - inertia_ : float - Sum of squared distances of samples to their closest cluster center. - - n_iter_ : int - Number of iterations run. - - See Also - -------- - MiniBatchKMeans : Alternative online implementation that does incremental - updates of the centers positions using mini-batches. - For large scale learning (say n_samples > 10k) MiniBatchKMeans is - probably much faster than the default batch implementation. - - Notes - ----- - The k-means problem is solved using either Lloyd's or Elkan's algorithm. - - The average complexity is given by O(k n T), where n is the number of - samples and T is the number of iteration. - - The worst case complexity is given by O(n^(k+2/p)) with - n = n_samples, p = n_features. (D. Arthur and S. Vassilvitskii, - 'How slow is the k-means method?' SoCG2006) - - In practice, the k-means algorithm is very fast (one of the fastest - clustering algorithms available), but it falls in local minima. That's why - it can be useful to restart it several times. - - If the algorithm stops before fully converging (because of ``tol`` or - ``max_iter``), ``labels_`` and ``cluster_centers_`` will not be consistent, - i.e. the ``cluster_centers_`` will not be the means of the points in each - cluster. Also, the estimator will reassign ``labels_`` after the last - iteration to make ``labels_`` consistent with ``predict`` on the training - set. - - Examples - -------- - - >>> from sklearn.cluster import KMeans - >>> import numpy as np - >>> X = np.array([[1, 2], [1, 4], [1, 0], - ... [10, 2], [10, 4], [10, 0]]) - >>> kmeans = KMeans(n_clusters=2, random_state=0).fit(X) - >>> kmeans.labels_ - array([1, 1, 1, 0, 0, 0], dtype=int32) - >>> kmeans.predict([[0, 0], [12, 3]]) - array([1, 0], dtype=int32) - >>> kmeans.cluster_centers_ - array([[10., 2.], - [ 1., 2.]]) - """ - @_deprecate_positional_args - def __init__(self, n_clusters=8, *, init='k-means++', n_init=10, - max_iter=300, tol=1e-4, precompute_distances='deprecated', - verbose=0, random_state=None, copy_x=True, - n_jobs='deprecated', algorithm='auto'): - + _labels( + X, + sample_weight, + centers, + centers_new=None, + weight_in_clusters=None, + labels=labels, + center_shift=center_shift, + n_threads=n_threads, + update_centers=False, + ) + + if return_inertia: + inertia = _inertia(X, sample_weight, centers, labels, n_threads) + return labels, inertia + + return labels + + +# Same as _labels_inertia but in a threadpool_limits context. +_labels_inertia_threadpool_limit = _threadpool_controller_decorator( + limits=1, user_api="blas" +)(_labels_inertia) + + +class _BaseKMeans( + ClassNamePrefixFeaturesOutMixin, TransformerMixin, ClusterMixin, BaseEstimator, ABC +): + """Base class for KMeans and MiniBatchKMeans""" + + _parameter_constraints: dict = { + "n_clusters": [Interval(Integral, 1, None, closed="left")], + "init": [StrOptions({"k-means++", "random"}), callable, "array-like"], + "n_init": [ + StrOptions({"auto"}), + Interval(Integral, 1, None, closed="left"), + ], + "max_iter": [Interval(Integral, 1, None, closed="left")], + "tol": [Interval(Real, 0, None, closed="left")], + "verbose": ["verbose"], + "random_state": ["random_state"], + } + + def __init__( + self, + n_clusters, + *, + init, + n_init, + max_iter, + tol, + verbose, + random_state, + ): self.n_clusters = n_clusters self.init = init self.max_iter = max_iter self.tol = tol - self.precompute_distances = precompute_distances self.n_init = n_init self.verbose = verbose self.random_state = random_state - self.copy_x = copy_x - self.n_jobs = n_jobs - self.algorithm = algorithm - - def _check_params(self, X): - # precompute_distances - if self.precompute_distances != 'deprecated': - warnings.warn("'precompute_distances' was deprecated in version " - "0.23 and will be removed in 1.0 (renaming of 0.25)" - ". It has no effect", FutureWarning) - - # n_jobs - if self.n_jobs != 'deprecated': - warnings.warn("'n_jobs' was deprecated in version 0.23 and will be" - " removed in 1.0 (renaming of 0.25).", FutureWarning) - self._n_threads = self.n_jobs - else: - self._n_threads = None - self._n_threads = _openmp_effective_n_threads(self._n_threads) - - # n_init - if self.n_init <= 0: - raise ValueError( - f"n_init should be > 0, got {self.n_init} instead.") - self._n_init = self.n_init - - # max_iter - if self.max_iter <= 0: - raise ValueError( - f"max_iter should be > 0, got {self.max_iter} instead.") + def _check_params_vs_input(self, X, default_n_init=None): # n_clusters if X.shape[0] < self.n_clusters: - raise ValueError(f"n_samples={X.shape[0]} should be >= " - f"n_clusters={self.n_clusters}.") + raise ValueError( + f"n_samples={X.shape[0]} should be >= n_clusters={self.n_clusters}." + ) # tol self._tol = _tolerance(X, self.tol) - # algorithm - if self.algorithm not in ("auto", "full", "elkan"): - raise ValueError(f"Algorithm must be 'auto', 'full' or 'elkan', " - f"got {self.algorithm} instead.") - - self._algorithm = self.algorithm - if self._algorithm == "auto": - self._algorithm = "full" if self.n_clusters == 1 else "elkan" - if self._algorithm == "elkan" and self.n_clusters == 1: - warnings.warn("algorithm='elkan' doesn't make sense for a single " - "cluster. Using 'full' instead.", RuntimeWarning) - self._algorithm = "full" - - # init - if not (hasattr(self.init, '__array__') or callable(self.init) - or (isinstance(self.init, str) - and self.init in ["k-means++", "random"])): - raise ValueError( - f"init should be either 'k-means++', 'random', a ndarray or a " - f"callable, got '{self.init}' instead.") + # n-init + if self.n_init == "auto": + if isinstance(self.init, str) and self.init == "k-means++": + self._n_init = 1 + elif isinstance(self.init, str) and self.init == "random": + self._n_init = default_n_init + elif callable(self.init): + self._n_init = default_n_init + else: # array-like + self._n_init = 1 + else: + self._n_init = self.n_init - if hasattr(self.init, '__array__') and self._n_init != 1: + if _is_arraylike_not_scalar(self.init) and self._n_init != 1: warnings.warn( - f"Explicit initial center position passed: performing only" - f" one init in {self.__class__.__name__} instead of " - f"n_init={self._n_init}.", RuntimeWarning, stacklevel=2) + ( + "Explicit initial center position passed: performing only" + f" one init in {self.__class__.__name__} instead of " + f"n_init={self._n_init}." + ), + RuntimeWarning, + stacklevel=2, + ) self._n_init = 1 - def _validate_center_shape(self, X, centers): - """Check if centers is compatible with X and n_clusters.""" - if centers.shape[0] != self.n_clusters: - raise ValueError( - f"The shape of the initial centers {centers.shape} does not " - f"match the number of clusters {self.n_clusters}.") - if centers.shape[1] != X.shape[1]: - raise ValueError( - f"The shape of the initial centers {centers.shape} does not " - f"match the number of features of the data {X.shape[1]}.") + @abstractmethod + def _warn_mkl_vcomp(self, n_active_threads): + """Issue an estimator specific warning when vcomp and mkl are both present - def _check_test_data(self, X): - X = self._validate_data(X, accept_sparse='csr', reset=False, - dtype=[np.float64, np.float32], - order='C', accept_large_sparse=False) - return X + This method is called by `_check_mkl_vcomp`. + """ def _check_mkl_vcomp(self, X, n_samples): - """Warns when vcomp and mkl are both present""" + """Check when vcomp and mkl are both present""" # The BLAS call inside a prange in lloyd_iter_chunked_dense is known to # cause a small memory leak when there are less chunks than the number # of available threads. It only happens when the OpenMP library is @@ -963,31 +916,52 @@ def _check_mkl_vcomp(self, X, n_samples): if sp.issparse(X): return - active_threads = int(np.ceil(n_samples / CHUNK_SIZE)) - if active_threads < self._n_threads: - modules = threadpool_info() + n_active_threads = int(np.ceil(n_samples / CHUNK_SIZE)) + if n_active_threads < self._n_threads: + modules = _get_threadpool_controller().info() has_vcomp = "vcomp" in [module["prefix"] for module in modules] has_mkl = ("mkl", "intel") in [ (module["internal_api"], module.get("threading_layer", None)) - for module in modules] + for module in modules + ] if has_vcomp and has_mkl: - if not hasattr(self, "batch_size"): # KMeans - warnings.warn( - f"KMeans is known to have a memory leak on Windows " - f"with MKL, when there are less chunks than available " - f"threads. You can avoid it by setting the environment" - f" variable OMP_NUM_THREADS={active_threads}.") - else: # MiniBatchKMeans - warnings.warn( - f"MiniBatchKMeans is known to have a memory leak on " - f"Windows with MKL, when there are less chunks than " - f"available threads. You can prevent it by setting " - f"batch_size >= {self._n_threads * CHUNK_SIZE} or by " - f"setting the environment variable " - f"OMP_NUM_THREADS={active_threads}") - - def _init_centroids(self, X, x_squared_norms, init, random_state, - init_size=None): + self._warn_mkl_vcomp(n_active_threads) + + def _validate_center_shape(self, X, centers): + """Check if centers is compatible with X and n_clusters.""" + if centers.shape[0] != self.n_clusters: + raise ValueError( + f"The shape of the initial centers {centers.shape} does not " + f"match the number of clusters {self.n_clusters}." + ) + if centers.shape[1] != X.shape[1]: + raise ValueError( + f"The shape of the initial centers {centers.shape} does not " + f"match the number of features of the data {X.shape[1]}." + ) + + def _check_test_data(self, X): + X = validate_data( + self, + X, + accept_sparse="csr", + reset=False, + dtype=[np.float64, np.float32], + order="C", + accept_large_sparse=False, + ) + return X + + def _init_centroids( + self, + X, + x_squared_norms, + init, + random_state, + sample_weight, + init_size=None, + n_centroids=None, + ): """Compute the initial centroids. Parameters @@ -1007,36 +981,56 @@ def _init_centroids(self, X, x_squared_norms, init, random_state, Determines random number generation for centroid initialization. See :term:`Glossary `. + sample_weight : ndarray of shape (n_samples,) + The weights for each observation in X. `sample_weight` is not used + during initialization if `init` is a callable or a user provided + array. + init_size : int, default=None Number of samples to randomly sample for speeding up the initialization (sometimes at the expense of accuracy). + n_centroids : int, default=None + Number of centroids to initialize. + If left to 'None' the number of centroids will be equal to + number of clusters to form (self.n_clusters). + Returns ------- centers : ndarray of shape (n_clusters, n_features) + Initial centroids of clusters. """ n_samples = X.shape[0] - n_clusters = self.n_clusters + n_clusters = self.n_clusters if n_centroids is None else n_centroids if init_size is not None and init_size < n_samples: init_indices = random_state.randint(0, n_samples, init_size) X = X[init_indices] x_squared_norms = x_squared_norms[init_indices] n_samples = X.shape[0] + sample_weight = sample_weight[init_indices] - if isinstance(init, str) and init == 'k-means++': - centers, _ = _kmeans_plusplus(X, n_clusters, - random_state=random_state, - x_squared_norms=x_squared_norms) - elif isinstance(init, str) and init == 'random': - seeds = random_state.permutation(n_samples)[:n_clusters] + if isinstance(init, str) and init == "k-means++": + centers, _ = _kmeans_plusplus( + X, + n_clusters, + random_state=random_state, + x_squared_norms=x_squared_norms, + sample_weight=sample_weight, + ) + elif isinstance(init, str) and init == "random": + seeds = random_state.choice( + n_samples, + size=n_clusters, + replace=False, + p=sample_weight / sample_weight.sum(), + ) centers = X[seeds] - elif hasattr(init, '__array__'): + elif _is_arraylike_not_scalar(self.init): centers = init elif callable(init): centers = init(X, n_clusters, random_state=random_state) - centers = check_array( - centers, dtype=X.dtype, copy=False, order='C') + centers = check_array(centers, dtype=X.dtype, copy=False, order="C") self._validate_center_shape(X, centers) if sp.issparse(centers): @@ -1044,17 +1038,16 @@ def _init_centroids(self, X, x_squared_norms, init, random_state, return centers - def fit(self, X, y=None, sample_weight=None): - """Compute k-means clustering. + def fit_predict(self, X, y=None, sample_weight=None): + """Compute cluster centers and predict cluster index for each sample. + + Convenience method; equivalent to calling fit(X) followed by + predict(X). Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) - Training instances to cluster. It must be noted that the data - will be converted to C ordering, which will cause a memory - copy if the given data is not C-contiguous. - If a sparse matrix is passed, a copy will be made if it's not in - CSR format. + New data to transform. y : Ignored Not used, present here for API consistency by convention. @@ -1063,112 +1056,46 @@ def fit(self, X, y=None, sample_weight=None): The weights for each observation in X. If None, all observations are assigned equal weight. - .. versionadded:: 0.20 - Returns ------- - self - Fitted estimator. + labels : ndarray of shape (n_samples,) + Index of the cluster each sample belongs to. """ - X = self._validate_data(X, accept_sparse='csr', - dtype=[np.float64, np.float32], - order='C', copy=self.copy_x, - accept_large_sparse=False) - - self._check_params(X) - random_state = check_random_state(self.random_state) - sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype) - - # Validate init array - init = self.init - if hasattr(init, '__array__'): - init = check_array(init, dtype=X.dtype, copy=True, order='C') - self._validate_center_shape(X, init) - - # subtract of mean of x for more accurate distance computations - if not sp.issparse(X): - X_mean = X.mean(axis=0) - # The copy was already done above - X -= X_mean - - if hasattr(init, '__array__'): - init -= X_mean - - # precompute squared norms of data points - x_squared_norms = row_norms(X, squared=True) - - if self._algorithm == "full": - kmeans_single = _kmeans_single_lloyd - self._check_mkl_vcomp(X, X.shape[0]) - else: - kmeans_single = _kmeans_single_elkan - - best_inertia = None - - for i in range(self._n_init): - # Initialize centers - centers_init = self._init_centroids( - X, x_squared_norms=x_squared_norms, init=init, - random_state=random_state) - if self.verbose: - print("Initialization complete") - - # run a k-means once - labels, inertia, centers, n_iter_ = kmeans_single( - X, sample_weight, centers_init, max_iter=self.max_iter, - verbose=self.verbose, tol=self._tol, - x_squared_norms=x_squared_norms, n_threads=self._n_threads) - - # determine if these results are the best so far - if best_inertia is None or inertia < best_inertia: - best_labels = labels - best_centers = centers - best_inertia = inertia - best_n_iter = n_iter_ - - if not sp.issparse(X): - if not self.copy_x: - X += X_mean - best_centers += X_mean - - distinct_clusters = len(set(best_labels)) - if distinct_clusters < self.n_clusters: - warnings.warn( - "Number of distinct clusters ({}) found smaller than " - "n_clusters ({}). Possibly due to duplicate points " - "in X.".format(distinct_clusters, self.n_clusters), - ConvergenceWarning, stacklevel=2) - - self.cluster_centers_ = best_centers - self.labels_ = best_labels - self.inertia_ = best_inertia - self.n_iter_ = best_n_iter - return self + return self.fit(X, sample_weight=sample_weight).labels_ - def fit_predict(self, X, y=None, sample_weight=None): - """Compute cluster centers and predict cluster index for each sample. + def predict(self, X): + """Predict the closest cluster each sample in X belongs to. - Convenience method; equivalent to calling fit(X) followed by - predict(X). + In the vector quantization literature, `cluster_centers_` is called + the code book and each value returned by `predict` is the index of + the closest code in the code book. Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) - New data to transform. - - y : Ignored - Not used, present here for API consistency by convention. - - sample_weight : array-like of shape (n_samples,), default=None - The weights for each observation in X. If None, all observations - are assigned equal weight. + New data to predict. Returns ------- labels : ndarray of shape (n_samples,) Index of the cluster each sample belongs to. """ - return self.fit(X, sample_weight=sample_weight).labels_ + check_is_fitted(self) + + X = self._check_test_data(X) + + # sample weights are not used by predict but cython helpers expect an array + sample_weight = np.ones(X.shape[0], dtype=X.dtype) + + labels = _labels_inertia_threadpool_limit( + X, + sample_weight, + self.cluster_centers_, + n_threads=self._n_threads, + return_inertia=False, + ) + + return labels def fit_transform(self, X, y=None, sample_weight=None): """Compute clustering and transform X to cluster-distance space. @@ -1192,10 +1119,6 @@ def fit_transform(self, X, y=None, sample_weight=None): X_new : ndarray of shape (n_samples, n_clusters) X transformed in the new space. """ - # Currently, this just skips a copy of the data if it is not in - # np.array or CSR format already. - # XXX This skips _check_test_data, which may change the dtype; - # we should refactor the input validation. return self.fit(X, sample_weight=sample_weight)._transform(X) def transform(self, X): @@ -1224,17 +1147,16 @@ def _transform(self, X): """Guts of transform method; no input validation.""" return euclidean_distances(X, self.cluster_centers_) - def predict(self, X, sample_weight=None): - """Predict the closest cluster each sample in X belongs to. - - In the vector quantization literature, `cluster_centers_` is called - the code book and each value returned by `predict` is the index of - the closest code in the code book. + def score(self, X, y=None, sample_weight=None): + """Opposite of the value of X on the K-means objective. Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) - New data to predict. + New data. + + y : Ignored + Not used, present here for API consistency by convention. sample_weight : array-like of shape (n_samples,), default=None The weights for each observation in X. If None, all observations @@ -1242,100 +1164,441 @@ def predict(self, X, sample_weight=None): Returns ------- - labels : ndarray of shape (n_samples,) - Index of the cluster each sample belongs to. + score : float + Opposite of the value of X on the K-means objective. """ check_is_fitted(self) X = self._check_test_data(X) - x_squared_norms = row_norms(X, squared=True) sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype) - return _labels_inertia(X, sample_weight, x_squared_norms, - self.cluster_centers_, self._n_threads)[0] + _, scores = _labels_inertia_threadpool_limit( + X, sample_weight, self.cluster_centers_, self._n_threads + ) + return -scores - def score(self, X, y=None, sample_weight=None): - """Opposite of the value of X on the K-means objective. + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.sparse = True + return tags + + +class KMeans(_BaseKMeans): + """K-Means clustering. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + + n_clusters : int, default=8 + The number of clusters to form as well as the number of + centroids to generate. + + For an example of how to choose an optimal value for `n_clusters` refer to + :ref:`sphx_glr_auto_examples_cluster_plot_kmeans_silhouette_analysis.py`. + + init : {'k-means++', 'random'}, callable or array-like of shape \ + (n_clusters, n_features), default='k-means++' + Method for initialization: + + * 'k-means++' : selects initial cluster centroids using sampling \ + based on an empirical probability distribution of the points' \ + contribution to the overall inertia. This technique speeds up \ + convergence. The algorithm implemented is "greedy k-means++". It \ + differs from the vanilla k-means++ by making several trials at \ + each sampling step and choosing the best centroid among them. + + * 'random': choose `n_clusters` observations (rows) at random from \ + data for the initial centroids. + + * If an array is passed, it should be of shape (n_clusters, n_features)\ + and gives the initial centers. + + * If a callable is passed, it should take arguments X, n_clusters and a\ + random state and return an initialization. + + For an example of how to use the different `init` strategies, see + :ref:`sphx_glr_auto_examples_cluster_plot_kmeans_digits.py`. + + For an evaluation of the impact of initialization, see the example + :ref:`sphx_glr_auto_examples_cluster_plot_kmeans_stability_low_dim_dense.py`. + + n_init : 'auto' or int, default='auto' + Number of times the k-means algorithm is run with different centroid + seeds. The final results is the best output of `n_init` consecutive runs + in terms of inertia. Several runs are recommended for sparse + high-dimensional problems (see :ref:`kmeans_sparse_high_dim`). + + When `n_init='auto'`, the number of runs depends on the value of init: + 10 if using `init='random'` or `init` is a callable; + 1 if using `init='k-means++'` or `init` is an array-like. + + .. versionadded:: 1.2 + Added 'auto' option for `n_init`. + + .. versionchanged:: 1.4 + Default value for `n_init` changed to `'auto'`. + + max_iter : int, default=300 + Maximum number of iterations of the k-means algorithm for a + single run. + + tol : float, default=1e-4 + Relative tolerance with regards to Frobenius norm of the difference + in the cluster centers of two consecutive iterations to declare + convergence. + + verbose : int, default=0 + Verbosity mode. + + random_state : int, RandomState instance or None, default=None + Determines random number generation for centroid initialization. Use + an int to make the randomness deterministic. + See :term:`Glossary `. + + copy_x : bool, default=True + When pre-computing distances it is more numerically accurate to center + the data first. If copy_x is True (default), then the original data is + not modified. If False, the original data is modified, and put back + before the function returns, but small numerical differences may be + introduced by subtracting and then adding the data mean. Note that if + the original data is not C-contiguous, a copy will be made even if + copy_x is False. If the original data is sparse, but not in CSR format, + a copy will be made even if copy_x is False. + + algorithm : {"lloyd", "elkan"}, default="lloyd" + K-means algorithm to use. The classical EM-style algorithm is `"lloyd"`. + The `"elkan"` variation can be more efficient on some datasets with + well-defined clusters, by using the triangle inequality. However it's + more memory intensive due to the allocation of an extra array of shape + `(n_samples, n_clusters)`. + + .. versionchanged:: 0.18 + Added Elkan algorithm + + .. versionchanged:: 1.1 + Renamed "full" to "lloyd", and deprecated "auto" and "full". + Changed "auto" to use "lloyd" instead of "elkan". + + Attributes + ---------- + cluster_centers_ : ndarray of shape (n_clusters, n_features) + Coordinates of cluster centers. If the algorithm stops before fully + converging (see ``tol`` and ``max_iter``), these will not be + consistent with ``labels_``. + + labels_ : ndarray of shape (n_samples,) + Labels of each point + + inertia_ : float + Sum of squared distances of samples to their closest cluster center, + weighted by the sample weights if provided. + + n_iter_ : int + Number of iterations run. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + MiniBatchKMeans : Alternative online implementation that does incremental + updates of the centers positions using mini-batches. + For large scale learning (say n_samples > 10k) MiniBatchKMeans is + probably much faster than the default batch implementation. + + Notes + ----- + The k-means problem is solved using either Lloyd's or Elkan's algorithm. + + The average complexity is given by O(k n T), where n is the number of + samples and T is the number of iteration. + + The worst case complexity is given by O(n^(k+2/p)) with + n = n_samples, p = n_features. + Refer to :doi:`"How slow is the k-means method?" D. Arthur and S. Vassilvitskii - + SoCG2006.<10.1145/1137856.1137880>` for more details. + + In practice, the k-means algorithm is very fast (one of the fastest + clustering algorithms available), but it falls in local minima. That's why + it can be useful to restart it several times. + + If the algorithm stops before fully converging (because of ``tol`` or + ``max_iter``), ``labels_`` and ``cluster_centers_`` will not be consistent, + i.e. the ``cluster_centers_`` will not be the means of the points in each + cluster. Also, the estimator will reassign ``labels_`` after the last + iteration to make ``labels_`` consistent with ``predict`` on the training + set. + + Examples + -------- + + >>> from sklearn.cluster import KMeans + >>> import numpy as np + >>> X = np.array([[1, 2], [1, 4], [1, 0], + ... [10, 2], [10, 4], [10, 0]]) + >>> kmeans = KMeans(n_clusters=2, random_state=0, n_init="auto").fit(X) + >>> kmeans.labels_ + array([1, 1, 1, 0, 0, 0], dtype=int32) + >>> kmeans.predict([[0, 0], [12, 3]]) + array([1, 0], dtype=int32) + >>> kmeans.cluster_centers_ + array([[10., 2.], + [ 1., 2.]]) + + For examples of common problems with K-Means and how to address them see + :ref:`sphx_glr_auto_examples_cluster_plot_kmeans_assumptions.py`. + + For a demonstration of how K-Means can be used to cluster text documents see + :ref:`sphx_glr_auto_examples_text_plot_document_clustering.py`. + + For a comparison between K-Means and MiniBatchKMeans refer to example + :ref:`sphx_glr_auto_examples_cluster_plot_mini_batch_kmeans.py`. + + For a comparison between K-Means and BisectingKMeans refer to example + :ref:`sphx_glr_auto_examples_cluster_plot_bisect_kmeans.py`. + """ + + _parameter_constraints: dict = { + **_BaseKMeans._parameter_constraints, + "copy_x": ["boolean"], + "algorithm": [StrOptions({"lloyd", "elkan"})], + } + + def __init__( + self, + n_clusters=8, + *, + init="k-means++", + n_init="auto", + max_iter=300, + tol=1e-4, + verbose=0, + random_state=None, + copy_x=True, + algorithm="lloyd", + ): + super().__init__( + n_clusters=n_clusters, + init=init, + n_init=n_init, + max_iter=max_iter, + tol=tol, + verbose=verbose, + random_state=random_state, + ) + + self.copy_x = copy_x + self.algorithm = algorithm + + def _check_params_vs_input(self, X): + super()._check_params_vs_input(X, default_n_init=10) + + self._algorithm = self.algorithm + if self._algorithm == "elkan" and self.n_clusters == 1: + warnings.warn( + ( + "algorithm='elkan' doesn't make sense for a single " + "cluster. Using 'lloyd' instead." + ), + RuntimeWarning, + ) + self._algorithm = "lloyd" + + def _warn_mkl_vcomp(self, n_active_threads): + """Warn when vcomp and mkl are both present""" + warnings.warn( + "KMeans is known to have a memory leak on Windows " + "with MKL, when there are less chunks than available " + "threads. You can avoid it by setting the environment" + f" variable OMP_NUM_THREADS={n_active_threads}." + ) + + @_fit_context(prefer_skip_nested_validation=True) + def fit(self, X, y=None, sample_weight=None): + """Compute k-means clustering. Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) - New data. + Training instances to cluster. It must be noted that the data + will be converted to C ordering, which will cause a memory + copy if the given data is not C-contiguous. + If a sparse matrix is passed, a copy will be made if it's not in + CSR format. y : Ignored Not used, present here for API consistency by convention. sample_weight : array-like of shape (n_samples,), default=None The weights for each observation in X. If None, all observations - are assigned equal weight. + are assigned equal weight. `sample_weight` is not used during + initialization if `init` is a callable or a user provided array. + + .. versionadded:: 0.20 Returns ------- - score : float - Opposite of the value of X on the K-means objective. + self : object + Fitted estimator. """ - check_is_fitted(self) + X = validate_data( + self, + X, + accept_sparse="csr", + dtype=[np.float64, np.float32], + order="C", + copy=self.copy_x, + accept_large_sparse=False, + ) + + self._check_params_vs_input(X) - X = self._check_test_data(X) - x_squared_norms = row_norms(X, squared=True) + random_state = check_random_state(self.random_state) sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype) + self._n_threads = _openmp_effective_n_threads() - return -_labels_inertia(X, sample_weight, x_squared_norms, - self.cluster_centers_)[1] + # Validate init array + init = self.init + init_is_array_like = _is_arraylike_not_scalar(init) + if init_is_array_like: + init = check_array(init, dtype=X.dtype, copy=True, order="C") + self._validate_center_shape(X, init) + + # subtract of mean of x for more accurate distance computations + if not sp.issparse(X): + X_mean = X.mean(axis=0) + # The copy was already done above + X -= X_mean - def _more_tags(self): - return { - '_xfail_checks': { - 'check_sample_weights_invariance': - 'zero sample_weight is not equivalent to removing samples', - }, - } + if init_is_array_like: + init -= X_mean + # precompute squared norms of data points + x_squared_norms = row_norms(X, squared=True) -def _mini_batch_step(X, sample_weight, x_squared_norms, centers, weight_sums, - old_center_buffer, compute_squared_diff, - distances, random_reassign=False, - random_state=None, reassignment_ratio=.01, - verbose=False): + if self._algorithm == "elkan": + kmeans_single = _kmeans_single_elkan + else: + kmeans_single = _kmeans_single_lloyd + self._check_mkl_vcomp(X, X.shape[0]) + + best_inertia, best_labels = None, None + + for i in range(self._n_init): + # Initialize centers + centers_init = self._init_centroids( + X, + x_squared_norms=x_squared_norms, + init=init, + random_state=random_state, + sample_weight=sample_weight, + ) + if self.verbose: + print("Initialization complete") + + # run a k-means once + labels, inertia, centers, n_iter_ = kmeans_single( + X, + sample_weight, + centers_init, + max_iter=self.max_iter, + verbose=self.verbose, + tol=self._tol, + n_threads=self._n_threads, + ) + + # determine if these results are the best so far + # we chose a new run if it has a better inertia and the clustering is + # different from the best so far (it's possible that the inertia is + # slightly better even if the clustering is the same with potentially + # permuted labels, due to rounding errors) + if best_inertia is None or ( + inertia < best_inertia + and not _is_same_clustering(labels, best_labels, self.n_clusters) + ): + best_labels = labels + best_centers = centers + best_inertia = inertia + best_n_iter = n_iter_ + + if not sp.issparse(X): + if not self.copy_x: + X += X_mean + best_centers += X_mean + + distinct_clusters = len(set(best_labels)) + if distinct_clusters < self.n_clusters: + warnings.warn( + "Number of distinct clusters ({}) found smaller than " + "n_clusters ({}). Possibly due to duplicate points " + "in X.".format(distinct_clusters, self.n_clusters), + ConvergenceWarning, + stacklevel=2, + ) + + self.cluster_centers_ = best_centers + self._n_features_out = self.cluster_centers_.shape[0] + self.labels_ = best_labels + self.inertia_ = best_inertia + self.n_iter_ = best_n_iter + return self + + +def _mini_batch_step( + X, + sample_weight, + centers, + centers_new, + weight_sums, + random_state, + random_reassign=False, + reassignment_ratio=0.01, + verbose=False, + n_threads=1, +): """Incremental update of the centers for the Minibatch K-Means algorithm. Parameters ---------- - X : ndarray of shape (n_samples, n_features) - The original data array. - - sample_weight : array-like of shape (n_samples,) - The weights for each observation in X. + X : {ndarray, sparse matrix} of shape (n_samples, n_features) + The original data array. If sparse, must be in CSR format. x_squared_norms : ndarray of shape (n_samples,) Squared euclidean norm of each data point. - centers : ndarray of shape (k, n_features) - The cluster centers. This array is MODIFIED IN PLACE + sample_weight : ndarray of shape (n_samples,) + The weights for each observation in `X`. - old_center_buffer : int - Copy of old centers for monitoring convergence. + centers : ndarray of shape (n_clusters, n_features) + The cluster centers before the current iteration - compute_squared_diff : bool - If set to False, the squared diff computation is skipped. + centers_new : ndarray of shape (n_clusters, n_features) + The cluster centers after the current iteration. Modified in-place. - distances : ndarray of shape (n_samples,), dtype=float, default=None - If not None, should be a pre-allocated array that will be used to store - the distances of each sample to its closest center. - May not be None when random_reassign is True. + weight_sums : ndarray of shape (n_clusters,) + The vector in which we keep track of the numbers of points in a + cluster. This array is modified in place. - random_reassign : bool, default=False + random_state : RandomState instance + Determines random number generation for low count centers reassignment. + See :term:`Glossary `. + + random_reassign : boolean, default=False If True, centers with very low counts are randomly reassigned to observations. - random_state : int, RandomState instance or None, default=None - Determines random number generation for centroid initialization and to - pick new clusters amongst observations with uniform probability. Use - an int to make the randomness deterministic. - See :term:`Glossary `. - - reassignment_ratio : float, default=.01 + reassignment_ratio : float, default=0.01 Control the fraction of the maximum number of counts for a center to be reassigned. A higher value means that low count centers are more likely to be reassigned, which means that the @@ -1345,159 +1608,74 @@ def _mini_batch_step(X, sample_weight, x_squared_norms, centers, weight_sums, verbose : bool, default=False Controls the verbosity. + n_threads : int, default=1 + The number of OpenMP threads to use for the computation. + Returns ------- inertia : float Sum of squared distances of samples to their closest cluster center. - - squared_diff : ndarray of shape (n_clusters,) - Squared distances between previous and updated cluster centers. - + The inertia is computed after finding the labels and before updating + the centers. """ # Perform label assignment to nearest centers - nearest_center, inertia = _labels_inertia(X, sample_weight, - x_squared_norms, centers) + # For better efficiency, it's better to run _mini_batch_step in a + # threadpool_limit context than using _labels_inertia_threadpool_limit here + labels, inertia = _labels_inertia(X, sample_weight, centers, n_threads=n_threads) + # Update centers according to the labels + if sp.issparse(X): + _minibatch_update_sparse( + X, sample_weight, centers, centers_new, weight_sums, labels, n_threads + ) + else: + _minibatch_update_dense( + X, + sample_weight, + centers, + centers_new, + weight_sums, + labels, + n_threads, + ) + + # Reassign clusters that have very low weight if random_reassign and reassignment_ratio > 0: - random_state = check_random_state(random_state) - # Reassign clusters that have very low weight to_reassign = weight_sums < reassignment_ratio * weight_sums.max() + # pick at most .5 * batch_size samples as new centers - if to_reassign.sum() > .5 * X.shape[0]: - indices_dont_reassign = \ - np.argsort(weight_sums)[int(.5 * X.shape[0]):] + if to_reassign.sum() > 0.5 * X.shape[0]: + indices_dont_reassign = np.argsort(weight_sums)[int(0.5 * X.shape[0]) :] to_reassign[indices_dont_reassign] = False n_reassigns = to_reassign.sum() + if n_reassigns: # Pick new clusters amongst observations with uniform probability - new_centers = random_state.choice(X.shape[0], replace=False, - size=n_reassigns) + new_centers = random_state.choice( + X.shape[0], replace=False, size=n_reassigns + ) if verbose: - print("[MiniBatchKMeans] Reassigning %i cluster centers." - % n_reassigns) + print(f"[MiniBatchKMeans] Reassigning {n_reassigns} cluster centers.") - if sp.issparse(X) and not sp.issparse(centers): + if sp.issparse(X): assign_rows_csr( - X, new_centers.astype(np.intp, copy=False), - np.where(to_reassign)[0].astype(np.intp, copy=False), - centers) + X, + new_centers.astype(np.intp, copy=False), + np.where(to_reassign)[0].astype(np.intp, copy=False), + centers_new, + ) else: - centers[to_reassign] = X[new_centers] + centers_new[to_reassign] = X[new_centers] + # reset counts of reassigned centers, but don't reset them too small # to avoid instant reassignment. This is a pretty dirty hack as it # also modifies the learning rates. weight_sums[to_reassign] = np.min(weight_sums[~to_reassign]) - # implementation for the sparse CSR representation completely written in - # cython - if sp.issparse(X): - return inertia, _mini_batch_update_csr( - X, sample_weight, x_squared_norms, centers, weight_sums, - nearest_center, old_center_buffer, compute_squared_diff) - - # dense variant in mostly numpy (not as memory efficient though) - k = centers.shape[0] - squared_diff = 0.0 - for center_idx in range(k): - # find points from minibatch that are assigned to this center - center_mask = nearest_center == center_idx - wsum = sample_weight[center_mask].sum() - - if wsum > 0: - if compute_squared_diff: - old_center_buffer[:] = centers[center_idx] - - # inplace remove previous count scaling - centers[center_idx] *= weight_sums[center_idx] - - # inplace sum with new points members of this cluster - centers[center_idx] += \ - np.sum(X[center_mask] * - sample_weight[center_mask, np.newaxis], axis=0) - - # update the count statistics for this center - weight_sums[center_idx] += wsum - - # inplace rescale to compute mean of all points (old and new) - # Note: numpy >= 1.10 does not support '/=' for the following - # expression for a mixture of int and float (see numpy issue #6464) - centers[center_idx] = centers[center_idx] / weight_sums[center_idx] - - # update the squared diff if necessary - if compute_squared_diff: - diff = centers[center_idx].ravel() - old_center_buffer.ravel() - squared_diff += np.dot(diff, diff) - - return inertia, squared_diff - - -def _mini_batch_convergence(model, iteration_idx, n_iter, tol, - n_samples, centers_squared_diff, batch_inertia, - context, verbose=0): - """Helper function to encapsulate the early stopping logic.""" - # Normalize inertia to be able to compare values when - # batch_size changes - batch_inertia /= model.batch_size - centers_squared_diff /= model.batch_size - - # Compute an Exponentially Weighted Average of the squared - # diff to monitor the convergence while discarding - # minibatch-local stochastic variability: - # https://en.wikipedia.org/wiki/Moving_average - ewa_diff = context.get('ewa_diff') - ewa_inertia = context.get('ewa_inertia') - if ewa_diff is None: - ewa_diff = centers_squared_diff - ewa_inertia = batch_inertia - else: - alpha = float(model.batch_size) * 2.0 / (n_samples + 1) - alpha = 1.0 if alpha > 1.0 else alpha - ewa_diff = ewa_diff * (1 - alpha) + centers_squared_diff * alpha - ewa_inertia = ewa_inertia * (1 - alpha) + batch_inertia * alpha - - # Log progress to be able to monitor convergence - if verbose: - progress_msg = ( - 'Minibatch iteration %d/%d:' - ' mean batch inertia: %f, ewa inertia: %f ' % ( - iteration_idx + 1, n_iter, batch_inertia, - ewa_inertia)) - print(progress_msg) - - # Early stopping based on absolute tolerance on squared change of - # centers position (using EWA smoothing) - if tol > 0.0 and ewa_diff <= tol: - if verbose: - print('Converged (small centers change) at iteration %d/%d' - % (iteration_idx + 1, n_iter)) - return True - - # Early stopping heuristic due to lack of improvement on smoothed inertia - ewa_inertia_min = context.get('ewa_inertia_min') - no_improvement = context.get('no_improvement', 0) - if ewa_inertia_min is None or ewa_inertia < ewa_inertia_min: - no_improvement = 0 - ewa_inertia_min = ewa_inertia - else: - no_improvement += 1 - - if (model.max_no_improvement is not None - and no_improvement >= model.max_no_improvement): - if verbose: - print('Converged (lack of improvement in inertia)' - ' at iteration %d/%d' - % (iteration_idx + 1, n_iter)) - return True - - # update the convergence context to maintain state across successive calls: - context['ewa_diff'] = ewa_diff - context['ewa_inertia'] = ewa_inertia - context['ewa_inertia_min'] = ewa_inertia_min - context['no_improvement'] = no_improvement - return False + return inertia -class MiniBatchKMeans(KMeans): +class MiniBatchKMeans(_BaseKMeans): """ Mini-Batch K-Means clustering. @@ -1514,9 +1692,12 @@ class MiniBatchKMeans(KMeans): (n_clusters, n_features), default='k-means++' Method for initialization: - 'k-means++' : selects initial cluster centers for k-mean - clustering in a smart way to speed up convergence. See section - Notes in k_init for more details. + 'k-means++' : selects initial cluster centroids using sampling based on + an empirical probability distribution of the points' contribution to the + overall inertia. This technique speeds up convergence. The algorithm + implemented is "greedy k-means++". It differs from the vanilla k-means++ + by making several trials at each sampling step and choosing the best centroid + among them. 'random': choose `n_clusters` observations (rows) at random from data for the initial centroids. @@ -1527,12 +1708,20 @@ class MiniBatchKMeans(KMeans): If a callable is passed, it should take arguments X, n_clusters and a random state and return an initialization. + For an evaluation of the impact of initialization, see the example + :ref:`sphx_glr_auto_examples_cluster_plot_kmeans_stability_low_dim_dense.py`. + max_iter : int, default=100 Maximum number of iterations over the complete dataset before stopping independently of any early stopping criterion heuristics. - batch_size : int, default=100 + batch_size : int, default=1024 Size of the mini batches. + For faster computations, you can set the ``batch_size`` greater than + 256 * number of cores to enable parallelism on all cores. + + .. versionchanged:: 1.0 + `batch_size` default changed from 100 to 1024. verbose : int, default=0 Verbosity mode. @@ -1570,19 +1759,33 @@ class MiniBatchKMeans(KMeans): only algorithm is initialized by running a batch KMeans on a random subset of the data. This needs to be larger than n_clusters. - If `None`, `init_size= 3 * batch_size`. + If `None`, the heuristic is `init_size = 3 * batch_size` if + `3 * batch_size < n_clusters`, else `init_size = 3 * n_clusters`. - n_init : int, default=3 + n_init : 'auto' or int, default="auto" Number of random initializations that are tried. - In contrast to KMeans, the algorithm is only run once, using the - best of the ``n_init`` initializations as measured by inertia. + In contrast to KMeans, the algorithm is only run once, using the best of + the `n_init` initializations as measured by inertia. Several runs are + recommended for sparse high-dimensional problems (see + :ref:`kmeans_sparse_high_dim`). + + When `n_init='auto'`, the number of runs depends on the value of init: + 3 if using `init='random'` or `init` is a callable; + 1 if using `init='k-means++'` or `init` is an array-like. + + .. versionadded:: 1.2 + Added 'auto' option for `n_init`. + + .. versionchanged:: 1.4 + Default value for `n_init` changed to `'auto'` in version. reassignment_ratio : float, default=0.01 - Control the fraction of the maximum number of counts for a - center to be reassigned. A higher value means that low count - centers are more easily reassigned, which means that the - model will take longer to converge, but should converge in a - better clustering. + Control the fraction of the maximum number of counts for a center to + be reassigned. A higher value means that low count centers are more + easily reassigned, which means that the model will take longer to + converge, but should converge in a better clustering. However, too high + a value may cause convergence issues, especially with a small batch + size. Attributes ---------- @@ -1590,31 +1793,35 @@ class MiniBatchKMeans(KMeans): cluster_centers_ : ndarray of shape (n_clusters, n_features) Coordinates of cluster centers. - labels_ : int + labels_ : ndarray of shape (n_samples,) Labels of each point (if compute_labels is set to True). inertia_ : float The value of the inertia criterion associated with the chosen - partition (if compute_labels is set to True). The inertia is - defined as the sum of square distances of samples to their nearest - neighbor. + partition if compute_labels is set to True. If compute_labels is set to + False, it's an approximation of the inertia based on an exponentially + weighted average of the batch inertiae. + The inertia is defined as the sum of square distances of samples to + their cluster center, weighted by the sample weights if provided. n_iter_ : int - Number of batches processed. + Number of iterations over the full dataset. - counts_ : ndarray of shape (n_clusters,) - Weigth sum of each cluster. + n_steps_ : int + Number of minibatches processed. - .. deprecated:: 0.24 - This attribute is deprecated in 0.24 and will be removed in - 1.1 (renaming of 0.26). + .. versionadded:: 1.0 - init_size_ : int - The effective number of samples used for the initialization. + n_features_in_ : int + Number of features seen during :term:`fit`. - .. deprecated:: 0.24 - This attribute is deprecated in 0.24 and will be removed in - 1.1 (renaming of 0.26). + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 See Also -------- @@ -1626,6 +1833,15 @@ class MiniBatchKMeans(KMeans): ----- See https://www.eecs.tufts.edu/~dsculley/papers/fastkmeans.pdf + When there are too few points in the dataset, some centers may be + duplicated, which means that a proper clustering in terms of the number + of requesting clusters and the number of returned clusters will not + always match. One solution is to set `reassignment_ratio=0`, which + prevents reassignments of clusters that are too small. + + See :ref:`sphx_glr_auto_examples_cluster_plot_birch_vs_minibatchkmeans.py` for a + comparison with :class:`~sklearn.cluster.BIRCH`. + Examples -------- >>> from sklearn.cluster import MiniBatchKMeans @@ -1637,34 +1853,65 @@ class MiniBatchKMeans(KMeans): >>> # manually fit on batches >>> kmeans = MiniBatchKMeans(n_clusters=2, ... random_state=0, - ... batch_size=6) + ... batch_size=6, + ... n_init="auto") >>> kmeans = kmeans.partial_fit(X[0:6,:]) >>> kmeans = kmeans.partial_fit(X[6:12,:]) >>> kmeans.cluster_centers_ - array([[2. , 1. ], - [3.5, 4.5]]) + array([[3.375, 3. ], + [0.75 , 0.5 ]]) >>> kmeans.predict([[0, 0], [4, 4]]) - array([0, 1], dtype=int32) + array([1, 0], dtype=int32) >>> # fit on the whole data >>> kmeans = MiniBatchKMeans(n_clusters=2, ... random_state=0, ... batch_size=6, - ... max_iter=10).fit(X) + ... max_iter=10, + ... n_init="auto").fit(X) >>> kmeans.cluster_centers_ - array([[3.95918367, 2.40816327], - [1.12195122, 1.3902439 ]]) + array([[3.55102041, 2.48979592], + [1.06896552, 1. ]]) >>> kmeans.predict([[0, 0], [4, 4]]) array([1, 0], dtype=int32) + + For a comparison of Mini-Batch K-Means clustering with other clustering algorithms, + see :ref:`sphx_glr_auto_examples_cluster_plot_cluster_comparison.py` """ - @_deprecate_positional_args - def __init__(self, n_clusters=8, *, init='k-means++', max_iter=100, - batch_size=100, verbose=0, compute_labels=True, - random_state=None, tol=0.0, max_no_improvement=10, - init_size=None, n_init=3, reassignment_ratio=0.01): + _parameter_constraints: dict = { + **_BaseKMeans._parameter_constraints, + "batch_size": [Interval(Integral, 1, None, closed="left")], + "compute_labels": ["boolean"], + "max_no_improvement": [Interval(Integral, 0, None, closed="left"), None], + "init_size": [Interval(Integral, 1, None, closed="left"), None], + "reassignment_ratio": [Interval(Real, 0, None, closed="left")], + } + + def __init__( + self, + n_clusters=8, + *, + init="k-means++", + max_iter=100, + batch_size=1024, + verbose=0, + compute_labels=True, + random_state=None, + tol=0.0, + max_no_improvement=10, + init_size=None, + n_init="auto", + reassignment_ratio=0.01, + ): super().__init__( - n_clusters=n_clusters, init=init, max_iter=max_iter, - verbose=verbose, random_state=random_state, tol=tol, n_init=n_init) + n_clusters=n_clusters, + init=init, + max_iter=max_iter, + verbose=verbose, + random_state=random_state, + tol=tol, + n_init=n_init, + ) self.max_no_improvement = max_no_improvement self.batch_size = batch_size @@ -1672,62 +1919,130 @@ def __init__(self, n_clusters=8, *, init='k-means++', max_iter=100, self.init_size = init_size self.reassignment_ratio = reassignment_ratio - @deprecated("The attribute 'counts_' is deprecated in 0.24" # type: ignore - " and will be removed in 1.1 (renaming of 0.26).") - @property - def counts_(self): - return self._counts - - @deprecated("The attribute 'init_size_' is deprecated in " # type: ignore - "0.24 and will be removed in 1.1 (renaming of 0.26).") - @property - def init_size_(self): - return self._init_size - - @deprecated("The attribute 'random_state_' is deprecated " # type: ignore - "in 0.24 and will be removed in 1.1 (renaming of 0.26).") - @property - def random_state_(self): - return getattr(self, "_random_state", None) - - def _check_params(self, X): - super()._check_params(X) - - # max_no_improvement - if self.max_no_improvement is not None and self.max_no_improvement < 0: - raise ValueError( - f"max_no_improvement should be >= 0, got " - f"{self.max_no_improvement} instead.") + def _check_params_vs_input(self, X): + super()._check_params_vs_input(X, default_n_init=3) - # batch_size - if self.batch_size <= 0: - raise ValueError( - f"batch_size should be > 0, got {self.batch_size} instead.") + self._batch_size = min(self.batch_size, X.shape[0]) # init_size - if self.init_size is not None and self.init_size <= 0: - raise ValueError( - f"init_size should be > 0, got {self.init_size} instead.") self._init_size = self.init_size if self._init_size is None: - self._init_size = 3 * self.batch_size + self._init_size = 3 * self._batch_size if self._init_size < self.n_clusters: self._init_size = 3 * self.n_clusters elif self._init_size < self.n_clusters: warnings.warn( - f"init_size={self._init_size} should be larger than " - f"n_clusters={self.n_clusters}. Setting it to " - f"min(3*n_clusters, n_samples)", - RuntimeWarning, stacklevel=2) + ( + f"init_size={self._init_size} should be larger than " + f"n_clusters={self.n_clusters}. Setting it to " + "min(3*n_clusters, n_samples)" + ), + RuntimeWarning, + stacklevel=2, + ) self._init_size = 3 * self.n_clusters self._init_size = min(self._init_size, X.shape[0]) # reassignment_ratio if self.reassignment_ratio < 0: raise ValueError( - f"reassignment_ratio should be >= 0, got " - f"{self.reassignment_ratio} instead.") + "reassignment_ratio should be >= 0, got " + f"{self.reassignment_ratio} instead." + ) + + def _warn_mkl_vcomp(self, n_active_threads): + """Warn when vcomp and mkl are both present""" + warnings.warn( + "MiniBatchKMeans is known to have a memory leak on " + "Windows with MKL, when there are less chunks than " + "available threads. You can prevent it by setting " + f"batch_size >= {self._n_threads * CHUNK_SIZE} or by " + "setting the environment variable " + f"OMP_NUM_THREADS={n_active_threads}" + ) + + def _mini_batch_convergence( + self, step, n_steps, n_samples, centers_squared_diff, batch_inertia + ): + """Helper function to encapsulate the early stopping logic""" + # Normalize inertia to be able to compare values when + # batch_size changes + batch_inertia /= self._batch_size + + # count steps starting from 1 for user friendly verbose mode. + step = step + 1 + + # Ignore first iteration because it's inertia from initialization. + if step == 1: + if self.verbose: + print( + f"Minibatch step {step}/{n_steps}: mean batch " + f"inertia: {batch_inertia}" + ) + return False + + # Compute an Exponentially Weighted Average of the inertia to + # monitor the convergence while discarding minibatch-local stochastic + # variability: https://en.wikipedia.org/wiki/Moving_average + if self._ewa_inertia is None: + self._ewa_inertia = batch_inertia + else: + alpha = self._batch_size * 2.0 / (n_samples + 1) + alpha = min(alpha, 1) + self._ewa_inertia = self._ewa_inertia * (1 - alpha) + batch_inertia * alpha + # Log progress to be able to monitor convergence + if self.verbose: + print( + f"Minibatch step {step}/{n_steps}: mean batch inertia: " + f"{batch_inertia}, ewa inertia: {self._ewa_inertia}" + ) + + # Early stopping based on absolute tolerance on squared change of + # centers position + if self._tol > 0.0 and centers_squared_diff <= self._tol: + if self.verbose: + print(f"Converged (small centers change) at step {step}/{n_steps}") + return True + + # Early stopping heuristic due to lack of improvement on smoothed + # inertia + if self._ewa_inertia_min is None or self._ewa_inertia < self._ewa_inertia_min: + self._no_improvement = 0 + self._ewa_inertia_min = self._ewa_inertia + else: + self._no_improvement += 1 + + if ( + self.max_no_improvement is not None + and self._no_improvement >= self.max_no_improvement + ): + if self.verbose: + print( + "Converged (lack of improvement in inertia) at step " + f"{step}/{n_steps}" + ) + return True + + return False + + def _random_reassign(self): + """Check if a random reassignment needs to be done. + + Do random reassignments each time 10 * n_clusters samples have been + processed. + + If there are empty clusters we always want to reassign. + """ + self._n_since_last_reassign += self._batch_size + if (self._counts == 0).any() or self._n_since_last_reassign >= ( + 10 * self.n_clusters + ): + self._n_since_last_reassign = 0 + return True + return False + + @_fit_context(prefer_skip_nested_validation=True) def fit(self, X, y=None, sample_weight=None): """Compute the centroids on X by chunking it into mini-batches. @@ -1737,283 +2052,252 @@ def fit(self, X, y=None, sample_weight=None): Training instances to cluster. It must be noted that the data will be converted to C ordering, which will cause a memory copy if the given data is not C-contiguous. + If a sparse matrix is passed, a copy will be made if it's not in + CSR format. y : Ignored Not used, present here for API consistency by convention. sample_weight : array-like of shape (n_samples,), default=None The weights for each observation in X. If None, all observations - are assigned equal weight (default: None). + are assigned equal weight. `sample_weight` is not used during + initialization if `init` is a callable or a user provided array. .. versionadded:: 0.20 Returns ------- - self + self : object + Fitted estimator. """ - X = self._validate_data(X, accept_sparse='csr', - dtype=[np.float64, np.float32], - order='C', accept_large_sparse=False) - - self._check_params(X) + X = validate_data( + self, + X, + accept_sparse="csr", + dtype=[np.float64, np.float32], + order="C", + accept_large_sparse=False, + ) + + self._check_params_vs_input(X) random_state = check_random_state(self.random_state) sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype) + self._n_threads = _openmp_effective_n_threads() + n_samples, n_features = X.shape # Validate init array init = self.init - if hasattr(init, '__array__'): - init = check_array(init, dtype=X.dtype, copy=True, order='C') + if _is_arraylike_not_scalar(init): + init = check_array(init, dtype=X.dtype, copy=True, order="C") self._validate_center_shape(X, init) - n_samples, n_features = X.shape - x_squared_norms = row_norms(X, squared=True) - - if self.tol > 0.0: - tol = _tolerance(X, self.tol) - - # using tol-based early stopping needs the allocation of a - # dedicated before which can be expensive for high dim data: - # hence we allocate it outside of the main loop - old_center_buffer = np.zeros(n_features, dtype=X.dtype) - else: - tol = 0.0 - # no need for the center buffer if tol-based early stopping is - # disabled - old_center_buffer = np.zeros(0, dtype=X.dtype) - - distances = np.zeros(self.batch_size, dtype=X.dtype) - n_batches = int(np.ceil(float(n_samples) / self.batch_size)) - n_iter = int(self.max_iter * n_batches) + self._check_mkl_vcomp(X, self._batch_size) - self._check_mkl_vcomp(X, self.batch_size) + # precompute squared norms of data points + x_squared_norms = row_norms(X, squared=True) - validation_indices = random_state.randint(0, n_samples, - self._init_size) + # Validation set for the init + validation_indices = random_state.randint(0, n_samples, self._init_size) X_valid = X[validation_indices] sample_weight_valid = sample_weight[validation_indices] - x_squared_norms_valid = x_squared_norms[validation_indices] - # perform several inits with random sub-sets + # perform several inits with random subsets best_inertia = None for init_idx in range(self._n_init): if self.verbose: - print("Init %d/%d with method: %s" - % (init_idx + 1, self._n_init, init)) - weight_sums = np.zeros(self.n_clusters, dtype=sample_weight.dtype) - - # TODO: once the `k_means` function works with sparse input we - # should refactor the following init to use it instead. + print(f"Init {init_idx + 1}/{self._n_init} with method {init}") # Initialize the centers using only a fraction of the data as we - # expect n_samples to be very large when using MiniBatchKMeans + # expect n_samples to be very large when using MiniBatchKMeans. cluster_centers = self._init_centroids( - X, x_squared_norms=x_squared_norms, + X, + x_squared_norms=x_squared_norms, init=init, random_state=random_state, - init_size=self._init_size) + init_size=self._init_size, + sample_weight=sample_weight, + ) + + # Compute inertia on a validation set. + _, inertia = _labels_inertia_threadpool_limit( + X_valid, + sample_weight_valid, + cluster_centers, + n_threads=self._n_threads, + ) - # Compute the label assignment on the init dataset - _mini_batch_step( - X_valid, sample_weight_valid, - x_squared_norms[validation_indices], cluster_centers, - weight_sums, old_center_buffer, False, distances=None, - verbose=self.verbose) - - # Keep only the best cluster centers across independent inits on - # the common validation set - _, inertia = _labels_inertia(X_valid, sample_weight_valid, - x_squared_norms_valid, - cluster_centers) if self.verbose: - print("Inertia for init %d/%d: %f" - % (init_idx + 1, self._n_init, inertia)) + print(f"Inertia for init {init_idx + 1}/{self._n_init}: {inertia}") if best_inertia is None or inertia < best_inertia: - self.cluster_centers_ = cluster_centers - self._counts = weight_sums + init_centers = cluster_centers best_inertia = inertia - # Empty context to be used inplace by the convergence check routine - convergence_context = {} - - # Perform the iterative optimization until the final convergence - # criterion - for iteration_idx in range(n_iter): - # Sample a minibatch from the full dataset - minibatch_indices = random_state.randint( - 0, n_samples, self.batch_size) - - # Perform the actual update step on the minibatch data - batch_inertia, centers_squared_diff = _mini_batch_step( - X[minibatch_indices], sample_weight[minibatch_indices], - x_squared_norms[minibatch_indices], - self.cluster_centers_, self._counts, - old_center_buffer, tol > 0.0, distances=distances, - # Here we randomly choose whether to perform - # random reassignment: the choice is done as a function - # of the iteration index, and the minimum number of - # counts, in order to force this reassignment to happen - # every once in a while - random_reassign=((iteration_idx + 1) - % (10 + int(self._counts.min())) == 0), - random_state=random_state, - reassignment_ratio=self.reassignment_ratio, - verbose=self.verbose) + centers = init_centers + centers_new = np.empty_like(centers) + + # Initialize counts + self._counts = np.zeros(self.n_clusters, dtype=X.dtype) + + # Attributes to monitor the convergence + self._ewa_inertia = None + self._ewa_inertia_min = None + self._no_improvement = 0 + + # Initialize number of samples seen since last reassignment + self._n_since_last_reassign = 0 + + n_steps = (self.max_iter * n_samples) // self._batch_size + + with _get_threadpool_controller().limit(limits=1, user_api="blas"): + # Perform the iterative optimization until convergence + for i in range(n_steps): + # Sample a minibatch from the full dataset + minibatch_indices = random_state.randint(0, n_samples, self._batch_size) + + # Perform the actual update step on the minibatch data + batch_inertia = _mini_batch_step( + X=X[minibatch_indices], + sample_weight=sample_weight[minibatch_indices], + centers=centers, + centers_new=centers_new, + weight_sums=self._counts, + random_state=random_state, + random_reassign=self._random_reassign(), + reassignment_ratio=self.reassignment_ratio, + verbose=self.verbose, + n_threads=self._n_threads, + ) + + if self._tol > 0.0: + centers_squared_diff = np.sum((centers_new - centers) ** 2) + else: + centers_squared_diff = 0 + + centers, centers_new = centers_new, centers + + # Monitor convergence and do early stopping if necessary + if self._mini_batch_convergence( + i, n_steps, n_samples, centers_squared_diff, batch_inertia + ): + break - # Monitor convergence and do early stopping if necessary - if _mini_batch_convergence( - self, iteration_idx, n_iter, tol, n_samples, - centers_squared_diff, batch_inertia, convergence_context, - verbose=self.verbose): - break + self.cluster_centers_ = centers + self._n_features_out = self.cluster_centers_.shape[0] - self.n_iter_ = iteration_idx + 1 + self.n_steps_ = i + 1 + self.n_iter_ = int(np.ceil(((i + 1) * self._batch_size) / n_samples)) if self.compute_labels: - self.labels_, self.inertia_ = \ - self._labels_inertia_minibatch(X, sample_weight) + self.labels_, self.inertia_ = _labels_inertia_threadpool_limit( + X, + sample_weight, + self.cluster_centers_, + n_threads=self._n_threads, + ) + else: + self.inertia_ = self._ewa_inertia * n_samples return self - def _labels_inertia_minibatch(self, X, sample_weight): - """Compute labels and inertia using mini batches. - - This is slightly slower than doing everything at once but prevents - memory errors / segfaults. - - Parameters - ---------- - X : array-like of shape (n_samples, n_features) - Input data. - - sample_weight : array-like of shape (n_samples,) - The weights for each observation in X. - - Returns - ------- - labels : ndarray of shape (n_samples,) - Cluster labels for each point. - - inertia : float - Sum of squared distances of points to nearest cluster. - """ - if self.verbose: - print('Computing label assignment and total inertia') - sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype) - x_squared_norms = row_norms(X, squared=True) - slices = gen_batches(X.shape[0], self.batch_size) - results = [_labels_inertia(X[s], sample_weight[s], x_squared_norms[s], - self.cluster_centers_) for s in slices] - labels, inertia = zip(*results) - return np.hstack(labels), np.sum(inertia) - + @_fit_context(prefer_skip_nested_validation=True) def partial_fit(self, X, y=None, sample_weight=None): """Update k means estimate on a single mini-batch X. Parameters ---------- - X : array-like of shape (n_samples, n_features) - Coordinates of the data points to cluster. It must be noted that - X will be copied if it is not C-contiguous. + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training instances to cluster. It must be noted that the data + will be converted to C ordering, which will cause a memory copy + if the given data is not C-contiguous. + If a sparse matrix is passed, a copy will be made if it's not in + CSR format. y : Ignored Not used, present here for API consistency by convention. sample_weight : array-like of shape (n_samples,), default=None The weights for each observation in X. If None, all observations - are assigned equal weight (default: None). + are assigned equal weight. `sample_weight` is not used during + initialization if `init` is a callable or a user provided array. Returns ------- - self + self : object + Return updated estimator. """ - is_first_call_to_partial_fit = not hasattr(self, 'cluster_centers_') - - X = self._validate_data(X, accept_sparse='csr', - dtype=[np.float64, np.float32], - order='C', accept_large_sparse=False, - reset=is_first_call_to_partial_fit) - - self._random_state = getattr(self, "_random_state", - check_random_state(self.random_state)) + has_centers = hasattr(self, "cluster_centers_") + + X = validate_data( + self, + X, + accept_sparse="csr", + dtype=[np.float64, np.float32], + order="C", + accept_large_sparse=False, + reset=not has_centers, + ) + + self._random_state = getattr( + self, "_random_state", check_random_state(self.random_state) + ) sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype) + self.n_steps_ = getattr(self, "n_steps_", 0) + # precompute squared norms of data points x_squared_norms = row_norms(X, squared=True) - if is_first_call_to_partial_fit: - # this is the first call to partial_fit on this object - self._check_params(X) + if not has_centers: + # this instance has not been fitted yet (fit or partial_fit) + self._check_params_vs_input(X) + self._n_threads = _openmp_effective_n_threads() # Validate init array init = self.init - if hasattr(init, '__array__'): - init = check_array(init, dtype=X.dtype, copy=True, order='C') + if _is_arraylike_not_scalar(init): + init = check_array(init, dtype=X.dtype, copy=True, order="C") self._validate_center_shape(X, init) self._check_mkl_vcomp(X, X.shape[0]) # initialize the cluster centers self.cluster_centers_ = self._init_centroids( - X, x_squared_norms=x_squared_norms, + X, + x_squared_norms=x_squared_norms, init=init, random_state=self._random_state, - init_size=self._init_size) - - self._counts = np.zeros(self.n_clusters, - dtype=sample_weight.dtype) - random_reassign = False - distances = None - else: - # The lower the minimum count is, the more we do random - # reassignment, however, we don't want to do random - # reassignment too often, to allow for building up counts - random_reassign = self._random_state.randint( - 10 * (1 + self._counts.min())) == 0 - distances = np.zeros(X.shape[0], dtype=X.dtype) - - _mini_batch_step(X, sample_weight, x_squared_norms, - self.cluster_centers_, self._counts, - np.zeros(0, dtype=X.dtype), 0, - random_reassign=random_reassign, distances=distances, - random_state=self._random_state, - reassignment_ratio=self.reassignment_ratio, - verbose=self.verbose) - - if self.compute_labels: - self.labels_, self.inertia_ = _labels_inertia( - X, sample_weight, x_squared_norms, self.cluster_centers_) + init_size=self._init_size, + sample_weight=sample_weight, + ) - return self + # Initialize counts + self._counts = np.zeros(self.n_clusters, dtype=X.dtype) - def predict(self, X, sample_weight=None): - """Predict the closest cluster each sample in X belongs to. + # Initialize number of samples seen since last reassignment + self._n_since_last_reassign = 0 - In the vector quantization literature, `cluster_centers_` is called - the code book and each value returned by `predict` is the index of - the closest code in the code book. - - Parameters - ---------- - X : {array-like, sparse matrix} of shape (n_samples, n_features) - New data to predict. + with _get_threadpool_controller().limit(limits=1, user_api="blas"): + _mini_batch_step( + X, + sample_weight=sample_weight, + centers=self.cluster_centers_, + centers_new=self.cluster_centers_, + weight_sums=self._counts, + random_state=self._random_state, + random_reassign=self._random_reassign(), + reassignment_ratio=self.reassignment_ratio, + verbose=self.verbose, + n_threads=self._n_threads, + ) - sample_weight : array-like of shape (n_samples,), default=None - The weights for each observation in X. If None, all observations - are assigned equal weight (default: None). + if self.compute_labels: + self.labels_, self.inertia_ = _labels_inertia_threadpool_limit( + X, + sample_weight, + self.cluster_centers_, + n_threads=self._n_threads, + ) - Returns - ------- - labels : ndarray of shape (n_samples,) - Index of the cluster each sample belongs to. - """ - check_is_fitted(self) + self.n_steps_ += 1 + self._n_features_out = self.cluster_centers_.shape[0] - X = self._check_test_data(X) - return self._labels_inertia_minibatch(X, sample_weight)[0] - - def _more_tags(self): - return { - '_xfail_checks': { - 'check_sample_weights_invariance': - 'zero sample_weight is not equivalent to removing samples', - } - } + return self diff --git a/sklearn/cluster/_mean_shift.py b/sklearn/cluster/_mean_shift.py index fa62d2c8d9fe7..1ba4409d14698 100644 --- a/sklearn/cluster/_mean_shift.py +++ b/sklearn/cluster/_mean_shift.py @@ -9,32 +9,42 @@ Seeding is performed using a binning technique for scalability. """ -# Authors: Conrad Lee -# Alexandre Gramfort -# Gael Varoquaux -# Martino Sorbaro +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import warnings -from joblib import Parallel - from collections import defaultdict -from ..utils.validation import check_is_fitted, _deprecate_positional_args -from ..utils.fixes import delayed -from ..utils import check_random_state, gen_batches, check_array -from ..base import BaseEstimator, ClusterMixin -from ..neighbors import NearestNeighbors -from ..metrics.pairwise import pairwise_distances_argmin -from .._config import config_context +from numbers import Integral, Real +import numpy as np -@_deprecate_positional_args -def estimate_bandwidth(X, *, quantile=0.3, n_samples=None, random_state=0, - n_jobs=None): +from .._config import config_context +from ..base import BaseEstimator, ClusterMixin, _fit_context +from ..metrics.pairwise import pairwise_distances_argmin +from ..neighbors import NearestNeighbors +from ..utils import check_array, check_random_state, gen_batches +from ..utils._param_validation import Interval, validate_params +from ..utils.parallel import Parallel, delayed +from ..utils.validation import check_is_fitted, validate_data + + +@validate_params( + { + "X": ["array-like"], + "quantile": [Interval(Real, 0, 1, closed="both")], + "n_samples": [Interval(Integral, 1, None, closed="left"), None], + "random_state": ["random_state"], + "n_jobs": [Integral, None], + }, + prefer_skip_nested_validation=True, +) +def estimate_bandwidth(X, *, quantile=0.3, n_samples=None, random_state=0, n_jobs=None): """Estimate the bandwidth to use with the mean-shift algorithm. - That this function takes time at least quadratic in n_samples. For large - datasets, it's wise to set that parameter to a small value. + This function takes time at least quadratic in `n_samples`. For large + datasets, it is wise to subsample by setting `n_samples`. Alternatively, + the parameter `bandwidth` can be set to a small value without estimating + it. Parameters ---------- @@ -42,7 +52,7 @@ def estimate_bandwidth(X, *, quantile=0.3, n_samples=None, random_state=0, Input points. quantile : float, default=0.3 - should be between [0, 1] + Should be between [0, 1] 0.5 means that the median of all pairwise distances is used. n_samples : int, default=None @@ -64,6 +74,15 @@ def estimate_bandwidth(X, *, quantile=0.3, n_samples=None, random_state=0, ------- bandwidth : float The bandwidth parameter. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.cluster import estimate_bandwidth + >>> X = np.array([[1, 1], [2, 1], [1, 0], + ... [4, 7], [3, 5], [3, 6]]) + >>> estimate_bandwidth(X, quantile=0.5) + np.float64(1.61) """ X = check_array(X) @@ -74,11 +93,10 @@ def estimate_bandwidth(X, *, quantile=0.3, n_samples=None, random_state=0, n_neighbors = int(X.shape[0] * quantile) if n_neighbors < 1: # cannot fit NearestNeighbors with n_neighbors = 0 n_neighbors = 1 - nbrs = NearestNeighbors(n_neighbors=n_neighbors, - n_jobs=n_jobs) + nbrs = NearestNeighbors(n_neighbors=n_neighbors, n_jobs=n_jobs) nbrs.fit(X) - bandwidth = 0. + bandwidth = 0.0 for batch in gen_batches(len(X), 500): d, _ = nbrs.kneighbors(X[batch, :], return_distance=True) bandwidth += np.max(d, axis=1).sum() @@ -89,30 +107,42 @@ def estimate_bandwidth(X, *, quantile=0.3, n_samples=None, random_state=0, # separate function for each seed's iterative loop def _mean_shift_single_seed(my_mean, X, nbrs, max_iter): # For each seed, climb gradient until convergence or max_iter - bandwidth = nbrs.get_params()['radius'] + bandwidth = nbrs.get_params()["radius"] stop_thresh = 1e-3 * bandwidth # when mean has converged completed_iterations = 0 while True: # Find mean of points within bandwidth - i_nbrs = nbrs.radius_neighbors([my_mean], bandwidth, - return_distance=False)[0] + i_nbrs = nbrs.radius_neighbors([my_mean], bandwidth, return_distance=False)[0] points_within = X[i_nbrs] if len(points_within) == 0: break # Depending on seeding strategy this condition may occur my_old_mean = my_mean # save the old mean my_mean = np.mean(points_within, axis=0) # If converged or at max_iter, adds the cluster - if (np.linalg.norm(my_mean - my_old_mean) < stop_thresh or - completed_iterations == max_iter): + if ( + np.linalg.norm(my_mean - my_old_mean) <= stop_thresh + or completed_iterations == max_iter + ): break completed_iterations += 1 return tuple(my_mean), len(points_within), completed_iterations -@_deprecate_positional_args -def mean_shift(X, *, bandwidth=None, seeds=None, bin_seeding=False, - min_bin_freq=1, cluster_all=True, max_iter=300, - n_jobs=None): +@validate_params( + {"X": ["array-like"]}, + prefer_skip_nested_validation=False, +) +def mean_shift( + X, + *, + bandwidth=None, + seeds=None, + bin_seeding=False, + min_bin_freq=1, + cluster_all=True, + max_iter=300, + n_jobs=None, +): """Perform mean shift clustering of data using a flat kernel. Read more in the :ref:`User Guide `. @@ -124,9 +154,9 @@ def mean_shift(X, *, bandwidth=None, seeds=None, bin_seeding=False, Input data. bandwidth : float, default=None - Kernel bandwidth. + Kernel bandwidth. If not None, must be in the range [0, +inf). - If bandwidth is not given, it is determined using a heuristic based on + If None, the bandwidth is determined using a heuristic based on the median of all pairwise distances. This will take quadratic time in the number of samples. The sklearn.cluster.estimate_bandwidth function can be used to do this more efficiently. @@ -158,8 +188,15 @@ def mean_shift(X, *, bandwidth=None, seeds=None, bin_seeding=False, operation terminates (for that seed point), if has not converged yet. n_jobs : int, default=None - The number of jobs to use for the computation. This works by computing - each of the n_init runs in parallel. + The number of jobs to use for the computation. The following tasks benefit + from the parallelization: + + - The search of nearest neighbors for bandwidth estimation and label + assignments. See the details in the docstring of the + ``NearestNeighbors`` class. + - Hill-climbing optimization for all seeds. + + See :term:`Glossary ` for more details. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` @@ -179,20 +216,36 @@ def mean_shift(X, *, bandwidth=None, seeds=None, bin_seeding=False, Notes ----- - For an example, see :ref:`examples/cluster/plot_mean_shift.py - `. + For a usage example, see + :ref:`sphx_glr_auto_examples_cluster_plot_mean_shift.py`. + Examples + -------- + >>> import numpy as np + >>> from sklearn.cluster import mean_shift + >>> X = np.array([[1, 1], [2, 1], [1, 0], + ... [4, 7], [3, 5], [3, 6]]) + >>> cluster_centers, labels = mean_shift(X, bandwidth=2) + >>> cluster_centers + array([[3.33, 6. ], + [1.33, 0.66]]) + >>> labels + array([1, 1, 1, 0, 0, 0]) """ - model = MeanShift(bandwidth=bandwidth, seeds=seeds, - min_bin_freq=min_bin_freq, - bin_seeding=bin_seeding, - cluster_all=cluster_all, n_jobs=n_jobs, - max_iter=max_iter).fit(X) + model = MeanShift( + bandwidth=bandwidth, + seeds=seeds, + min_bin_freq=min_bin_freq, + bin_seeding=bin_seeding, + cluster_all=cluster_all, + n_jobs=n_jobs, + max_iter=max_iter, + ).fit(X) return model.cluster_centers_, model.labels_ def get_bin_seeds(X, bin_size, min_bin_freq=1): - """Finds seeds for mean_shift. + """Find seeds for mean_shift. Finds seeds by first binning data onto a grid whose lines are spaced bin_size apart, and then choosing those bins with at least @@ -230,11 +283,15 @@ def get_bin_seeds(X, bin_size, min_bin_freq=1): bin_sizes[tuple(binned_point)] += 1 # Select only those bins as seeds which have enough members - bin_seeds = np.array([point for point, freq in bin_sizes.items() if - freq >= min_bin_freq], dtype=np.float32) + bin_seeds = np.array( + [point for point, freq in bin_sizes.items() if freq >= min_bin_freq], + dtype=np.float32, + ) if len(bin_seeds) == len(X): - warnings.warn("Binning data failed with provided bin_size=%f," - " using data points as seeds." % bin_size) + warnings.warn( + "Binning data failed with provided bin_size=%f, using data points as seeds." + % bin_size + ) return X bin_seeds = bin_seeds * bin_size return bin_seeds @@ -251,12 +308,15 @@ class MeanShift(ClusterMixin, BaseEstimator): Seeding is performed using a binning technique for scalability. + For an example of how to use MeanShift clustering, refer to: + :ref:`sphx_glr_auto_examples_cluster_plot_mean_shift.py`. + Read more in the :ref:`User Guide `. Parameters ---------- bandwidth : float, default=None - Bandwidth used in the RBF kernel. + Bandwidth used in the flat kernel. If not given, the bandwidth is estimated using sklearn.cluster.estimate_bandwidth; see the documentation for that @@ -287,8 +347,15 @@ class MeanShift(ClusterMixin, BaseEstimator): If false, then orphans are given cluster label -1. n_jobs : int, default=None - The number of jobs to use for the computation. This works by computing - each of the n_init runs in parallel. + The number of jobs to use for the computation. The following tasks benefit + from the parallelization: + + - The search of nearest neighbors for bandwidth estimation and label + assignments. See the details in the docstring of the + ``NearestNeighbors`` class. + - Hill-climbing optimization for all seeds. + + See :term:`Glossary ` for more details. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` @@ -313,19 +380,20 @@ class MeanShift(ClusterMixin, BaseEstimator): .. versionadded:: 0.22 - Examples + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also -------- - >>> from sklearn.cluster import MeanShift - >>> import numpy as np - >>> X = np.array([[1, 1], [2, 1], [1, 0], - ... [4, 7], [3, 5], [3, 6]]) - >>> clustering = MeanShift(bandwidth=2).fit(X) - >>> clustering.labels_ - array([1, 1, 1, 0, 0, 0]) - >>> clustering.predict([[0, 0], [5, 5]]) - array([1, 0]) - >>> clustering - MeanShift(bandwidth=2) + KMeans : K-Means clustering. Notes ----- @@ -351,10 +419,45 @@ class MeanShift(ClusterMixin, BaseEstimator): feature space analysis". IEEE Transactions on Pattern Analysis and Machine Intelligence. 2002. pp. 603-619. + Examples + -------- + >>> from sklearn.cluster import MeanShift + >>> import numpy as np + >>> X = np.array([[1, 1], [2, 1], [1, 0], + ... [4, 7], [3, 5], [3, 6]]) + >>> clustering = MeanShift(bandwidth=2).fit(X) + >>> clustering.labels_ + array([1, 1, 1, 0, 0, 0]) + >>> clustering.predict([[0, 0], [5, 5]]) + array([1, 0]) + >>> clustering + MeanShift(bandwidth=2) + + For a comparison of Mean Shift clustering with other clustering algorithms, see + :ref:`sphx_glr_auto_examples_cluster_plot_cluster_comparison.py` """ - @_deprecate_positional_args - def __init__(self, *, bandwidth=None, seeds=None, bin_seeding=False, - min_bin_freq=1, cluster_all=True, n_jobs=None, max_iter=300): + + _parameter_constraints: dict = { + "bandwidth": [Interval(Real, 0, None, closed="neither"), None], + "seeds": ["array-like", None], + "bin_seeding": ["boolean"], + "min_bin_freq": [Interval(Integral, 1, None, closed="left")], + "cluster_all": ["boolean"], + "n_jobs": [Integral, None], + "max_iter": [Interval(Integral, 0, None, closed="left")], + } + + def __init__( + self, + *, + bandwidth=None, + seeds=None, + bin_seeding=False, + min_bin_freq=1, + cluster_all=True, + n_jobs=None, + max_iter=300, + ): self.bandwidth = bandwidth self.seeds = seeds self.bin_seeding = bin_seeding @@ -363,6 +466,7 @@ def __init__(self, *, bandwidth=None, seeds=None, bin_seeding=False, self.n_jobs = n_jobs self.max_iter = max_iter + @_fit_context(prefer_skip_nested_validation=True) def fit(self, X, y=None): """Perform clustering. @@ -372,15 +476,17 @@ def fit(self, X, y=None): Samples to cluster. y : Ignored + Not used, present for API consistency by convention. + Returns + ------- + self : object + Fitted instance. """ - X = self._validate_data(X) + X = validate_data(self, X) bandwidth = self.bandwidth if bandwidth is None: bandwidth = estimate_bandwidth(X, n_jobs=self.n_jobs) - elif bandwidth <= 0: - raise ValueError("bandwidth needs to be greater than zero or None," - " got %f" % bandwidth) seeds = self.seeds if seeds is None: @@ -398,8 +504,9 @@ def fit(self, X, y=None): # execute iterations on all seeds in parallel all_res = Parallel(n_jobs=self.n_jobs)( - delayed(_mean_shift_single_seed) - (seed, X, nbrs, self.max_iter) for seed in seeds) + delayed(_mean_shift_single_seed)(seed, X, nbrs, self.max_iter) + for seed in seeds + ) # copy results in a dictionary for i in range(len(seeds)): if all_res[i][1]: # i.e. len(points_within) > 0 @@ -409,34 +516,38 @@ def fit(self, X, y=None): if not center_intensity_dict: # nothing near seeds - raise ValueError("No point was within bandwidth=%f of any seed." - " Try a different seeding strategy \ - or increase the bandwidth." - % bandwidth) + raise ValueError( + "No point was within bandwidth=%f of any seed. Try a different seeding" + " strategy or increase the bandwidth." + % bandwidth + ) # POST PROCESSING: remove near duplicate points # If the distance between two kernels is less than the bandwidth, # then we have to remove one because it is a duplicate. Remove the # one with fewer points. - sorted_by_intensity = sorted(center_intensity_dict.items(), - key=lambda tup: (tup[1], tup[0]), - reverse=True) + sorted_by_intensity = sorted( + center_intensity_dict.items(), + key=lambda tup: (tup[1], tup[0]), + reverse=True, + ) sorted_centers = np.array([tup[0] for tup in sorted_by_intensity]) unique = np.ones(len(sorted_centers), dtype=bool) - nbrs = NearestNeighbors(radius=bandwidth, - n_jobs=self.n_jobs).fit(sorted_centers) + nbrs = NearestNeighbors(radius=bandwidth, n_jobs=self.n_jobs).fit( + sorted_centers + ) for i, center in enumerate(sorted_centers): if unique[i]: - neighbor_idxs = nbrs.radius_neighbors([center], - return_distance=False)[0] + neighbor_idxs = nbrs.radius_neighbors([center], return_distance=False)[ + 0 + ] unique[neighbor_idxs] = 0 unique[i] = 1 # leave the current point as unique cluster_centers = sorted_centers[unique] # ASSIGN LABELS: a point belongs to the cluster that it is closest to - nbrs = NearestNeighbors(n_neighbors=1, - n_jobs=self.n_jobs).fit(cluster_centers) + nbrs = NearestNeighbors(n_neighbors=1, n_jobs=self.n_jobs).fit(cluster_centers) labels = np.zeros(n_samples, dtype=int) distances, idxs = nbrs.kneighbors(X) if self.cluster_all: @@ -454,7 +565,7 @@ def predict(self, X): Parameters ---------- - X : {array-like, sparse matrix} of shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) New data to predict. Returns @@ -463,6 +574,6 @@ def predict(self, X): Index of the cluster each sample belongs to. """ check_is_fitted(self) - X = self._validate_data(X, reset=False) + X = validate_data(self, X, reset=False) with config_context(assume_finite=True): return pairwise_distances_argmin(X, self.cluster_centers_) diff --git a/sklearn/cluster/_optics.py b/sklearn/cluster/_optics.py old mode 100755 new mode 100644 index d0b94f43454b3..4a1a80c9065c2 --- a/sklearn/cluster/_optics.py +++ b/sklearn/cluster/_optics.py @@ -1,45 +1,54 @@ -# -*- coding: utf-8 -*- """Ordering Points To Identify the Clustering Structure (OPTICS) These routines execute the OPTICS algorithm, and implement various cluster extraction methods of the ordered list. - -Authors: Shane Grigsby - Adrin Jalali - Erich Schubert - Hanmin Qin -License: BSD 3 clause """ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import warnings +from numbers import Integral, Real + import numpy as np +from scipy.sparse import SparseEfficiencyWarning, issparse +from ..base import BaseEstimator, ClusterMixin, _fit_context from ..exceptions import DataConversionWarning -from ..metrics.pairwise import PAIRWISE_BOOLEAN_FUNCTIONS -from ..utils import gen_batches, get_chunk_n_rows -from ..utils.validation import _deprecate_positional_args -from ..neighbors import NearestNeighbors -from ..base import BaseEstimator, ClusterMixin from ..metrics import pairwise_distances +from ..metrics.pairwise import _VALID_METRICS, PAIRWISE_BOOLEAN_FUNCTIONS +from ..neighbors import NearestNeighbors +from ..utils import gen_batches +from ..utils._chunking import get_chunk_n_rows +from ..utils._param_validation import ( + HasMethods, + Interval, + RealNotInt, + StrOptions, + validate_params, +) +from ..utils.validation import check_memory, validate_data class OPTICS(ClusterMixin, BaseEstimator): """Estimate clustering structure from vector array. OPTICS (Ordering Points To Identify the Clustering Structure), closely - related to DBSCAN, finds core sample of high density and expands clusters - from them [1]_. Unlike DBSCAN, keeps cluster hierarchy for a variable + related to DBSCAN, finds core samples of high density and expands clusters + from them [1]_. Unlike DBSCAN, it keeps cluster hierarchy for a variable neighborhood radius. Better suited for usage on large datasets than the - current sklearn implementation of DBSCAN. + current scikit-learn implementation of DBSCAN. - Clusters are then extracted using a DBSCAN-like method - (cluster_method = 'dbscan') or an automatic + Clusters are then extracted from the cluster-order using a + DBSCAN-like method (cluster_method = 'dbscan') or an automatic technique proposed in [1]_ (cluster_method = 'xi'). This implementation deviates from the original OPTICS by first performing - k-nearest-neighborhood searches on all points to identify core sizes, then - computing only the distances to unprocessed points when constructing the - cluster order. Note that we do not employ a heap to manage the expansion + k-nearest-neighborhood searches on all points to identify core sizes of + all points (instead of computing neighbors while looping through points). + Reachability distances to only unprocessed points are then computed, to + construct the cluster order, similar to the original OPTICS. + Note that we do not employ a heap to manage the expansion candidates, so the time complexity will be O(n^2). Read more in the :ref:`User Guide `. @@ -61,14 +70,15 @@ class OPTICS(ClusterMixin, BaseEstimator): metric : str or callable, default='minkowski' Metric to use for distance computation. Any metric from scikit-learn - or scipy.spatial.distance can be used. + or :mod:`scipy.spatial.distance` can be used. - If metric is a callable function, it is called on each + If `metric` is a callable function, it is called on each pair of instances (rows) and the resulting value recorded. The callable should take two arrays as input and return one value indicating the distance between them. This works for Scipy's metrics, but is less efficient than passing the metric name as a string. If metric is - "precomputed", X is assumed to be a distance matrix and must be square. + "precomputed", `X` is assumed to be a distance matrix and must be + square. Valid values for metric are: @@ -81,10 +91,13 @@ class OPTICS(ClusterMixin, BaseEstimator): 'seuclidean', 'sokalmichener', 'sokalsneath', 'sqeuclidean', 'yule'] - See the documentation for scipy.spatial.distance for details on these - metrics. + Sparse matrices are only supported by scikit-learn metrics. + See :mod:`scipy.spatial.distance` for details on these metrics. + + .. note:: + `'kulsinski'` is deprecated from SciPy 1.9 and will be removed in SciPy 1.11. - p : int, default=2 + p : float, default=2 Parameter for the Minkowski metric from :class:`~sklearn.metrics.pairwise_distances`. When p = 1, this is equivalent to using manhattan_distance (l1), and euclidean_distance @@ -93,9 +106,9 @@ class OPTICS(ClusterMixin, BaseEstimator): metric_params : dict, default=None Additional keyword arguments for the metric function. - cluster_method : str, default='xi' + cluster_method : {'xi', 'dbscan'}, default='xi' The extraction method used to extract clusters using the calculated - reachability and ordering. Possible values are "xi" and "dbscan". + reachability and ordering. eps : float, default=None The maximum distance between two samples for one to be considered as @@ -124,20 +137,25 @@ class OPTICS(ClusterMixin, BaseEstimator): algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, default='auto' Algorithm used to compute the nearest neighbors: - - 'ball_tree' will use :class:`BallTree` - - 'kd_tree' will use :class:`KDTree` + - 'ball_tree' will use :class:`~sklearn.neighbors.BallTree`. + - 'kd_tree' will use :class:`~sklearn.neighbors.KDTree`. - 'brute' will use a brute-force search. - - 'auto' will attempt to decide the most appropriate algorithm - based on the values passed to :meth:`fit` method. (default) + - 'auto' (default) will attempt to decide the most appropriate + algorithm based on the values passed to :meth:`fit` method. Note: fitting on sparse input will override the setting of this parameter, using brute force. leaf_size : int, default=30 - Leaf size passed to :class:`BallTree` or :class:`KDTree`. This can - affect the speed of the construction and query, as well as the memory - required to store the tree. The optimal value depends on the - nature of the problem. + Leaf size passed to :class:`~sklearn.neighbors.BallTree` or + :class:`~sklearn.neighbors.KDTree`. This can affect the speed of the + construction and query, as well as the memory required to store the + tree. The optimal value depends on the nature of the problem. + + memory : str or object with the joblib.Memory interface, default=None + Used to cache the output of the computation of the tree. + By default, no caching is done. If a string is given, it is the + path to the caching directory. n_jobs : int, default=None The number of parallel jobs to run for neighbors search. @@ -179,6 +197,17 @@ class OPTICS(ClusterMixin, BaseEstimator): ``X[ordering_][start:end + 1]`` form a cluster. Only available when ``cluster_method='xi'``. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + See Also -------- DBSCAN : A similar clustering for a specified neighborhood radius (eps). @@ -203,12 +232,56 @@ class OPTICS(ClusterMixin, BaseEstimator): >>> clustering = OPTICS(min_samples=2).fit(X) >>> clustering.labels_ array([0, 0, 0, 1, 1, 1]) + + For a more detailed example see + :ref:`sphx_glr_auto_examples_cluster_plot_optics.py`. + + For a comparison of OPTICS with other clustering algorithms, see + :ref:`sphx_glr_auto_examples_cluster_plot_cluster_comparison.py` """ - @_deprecate_positional_args - def __init__(self, *, min_samples=5, max_eps=np.inf, metric='minkowski', - p=2, metric_params=None, cluster_method='xi', eps=None, - xi=0.05, predecessor_correction=True, min_cluster_size=None, - algorithm='auto', leaf_size=30, n_jobs=None): + + _parameter_constraints: dict = { + "min_samples": [ + Interval(Integral, 2, None, closed="left"), + Interval(RealNotInt, 0, 1, closed="both"), + ], + "max_eps": [Interval(Real, 0, None, closed="both")], + "metric": [StrOptions(set(_VALID_METRICS) | {"precomputed"}), callable], + "p": [Interval(Real, 1, None, closed="left")], + "metric_params": [dict, None], + "cluster_method": [StrOptions({"dbscan", "xi"})], + "eps": [Interval(Real, 0, None, closed="both"), None], + "xi": [Interval(Real, 0, 1, closed="both")], + "predecessor_correction": ["boolean"], + "min_cluster_size": [ + Interval(Integral, 2, None, closed="left"), + Interval(RealNotInt, 0, 1, closed="right"), + None, + ], + "algorithm": [StrOptions({"auto", "brute", "ball_tree", "kd_tree"})], + "leaf_size": [Interval(Integral, 1, None, closed="left")], + "memory": [str, HasMethods("cache"), None], + "n_jobs": [Integral, None], + } + + def __init__( + self, + *, + min_samples=5, + max_eps=np.inf, + metric="minkowski", + p=2, + metric_params=None, + cluster_method="xi", + eps=None, + xi=0.05, + predecessor_correction=True, + min_cluster_size=None, + algorithm="auto", + leaf_size=30, + memory=None, + n_jobs=None, + ): self.max_eps = max_eps self.min_samples = min_samples self.min_cluster_size = min_cluster_size @@ -221,8 +294,13 @@ def __init__(self, *, min_samples=5, max_eps=np.inf, metric='minkowski', self.eps = eps self.xi = xi self.predecessor_correction = predecessor_correction + self.memory = memory self.n_jobs = n_jobs + @_fit_context( + # Optics.metric is not validated yet + prefer_skip_nested_validation=False + ) def fit(self, X, y=None): """Perform OPTICS clustering. @@ -232,43 +310,58 @@ def fit(self, X, y=None): Parameters ---------- - X : ndarray of shape (n_samples, n_features), or \ - (n_samples, n_samples) if metric=’precomputed’ + X : {ndarray, sparse matrix} of shape (n_samples, n_features), or \ + (n_samples, n_samples) if metric='precomputed' A feature array, or array of distances between samples if - metric='precomputed'. + metric='precomputed'. If a sparse matrix is provided, it will be + converted into CSR format. - y : ignored - Ignored. + y : Ignored + Not used, present for API consistency by convention. Returns ------- - self : instance of OPTICS - The instance. + self : object + Returns a fitted instance of self. """ - dtype = bool if self.metric in PAIRWISE_BOOLEAN_FUNCTIONS else float - if dtype == bool and X.dtype != bool: - msg = (f"Data will be converted to boolean for" - f" metric {self.metric}, to avoid this warning," - f" you may convert the data prior to calling fit.") + if dtype is bool and X.dtype != bool: + msg = ( + "Data will be converted to boolean for" + f" metric {self.metric}, to avoid this warning," + " you may convert the data prior to calling fit." + ) warnings.warn(msg, DataConversionWarning) - X = self._validate_data(X, dtype=dtype) - - if self.cluster_method not in ['dbscan', 'xi']: - raise ValueError("cluster_method should be one of" - " 'dbscan' or 'xi' but is %s" % - self.cluster_method) - - (self.ordering_, self.core_distances_, self.reachability_, - self.predecessor_) = compute_optics_graph( - X=X, min_samples=self.min_samples, algorithm=self.algorithm, - leaf_size=self.leaf_size, metric=self.metric, - metric_params=self.metric_params, p=self.p, n_jobs=self.n_jobs, - max_eps=self.max_eps) + X = validate_data(self, X, dtype=dtype, accept_sparse="csr") + if self.metric == "precomputed" and issparse(X): + X = X.copy() # copy to avoid in-place modification + with warnings.catch_warnings(): + warnings.simplefilter("ignore", SparseEfficiencyWarning) + # Set each diagonal to an explicit value so each point is its + # own neighbor + X.setdiag(X.diagonal()) + memory = check_memory(self.memory) + + ( + self.ordering_, + self.core_distances_, + self.reachability_, + self.predecessor_, + ) = memory.cache(compute_optics_graph)( + X=X, + min_samples=self.min_samples, + algorithm=self.algorithm, + leaf_size=self.leaf_size, + metric=self.metric, + metric_params=self.metric_params, + p=self.p, + n_jobs=self.n_jobs, + max_eps=self.max_eps, + ) # Extract clusters from the calculated orders and reachability - if self.cluster_method == 'xi': + if self.cluster_method == "xi": labels_, clusters_ = cluster_optics_xi( reachability=self.reachability_, predecessor=self.predecessor_, @@ -276,43 +369,42 @@ def fit(self, X, y=None): min_samples=self.min_samples, min_cluster_size=self.min_cluster_size, xi=self.xi, - predecessor_correction=self.predecessor_correction) + predecessor_correction=self.predecessor_correction, + ) self.cluster_hierarchy_ = clusters_ - elif self.cluster_method == 'dbscan': + elif self.cluster_method == "dbscan": if self.eps is None: eps = self.max_eps else: eps = self.eps if eps > self.max_eps: - raise ValueError('Specify an epsilon smaller than %s. Got %s.' - % (self.max_eps, eps)) + raise ValueError( + "Specify an epsilon smaller than %s. Got %s." % (self.max_eps, eps) + ) labels_ = cluster_optics_dbscan( reachability=self.reachability_, core_distances=self.core_distances_, - ordering=self.ordering_, eps=eps) + ordering=self.ordering_, + eps=eps, + ) self.labels_ = labels_ return self def _validate_size(size, n_samples, param_name): - if size <= 0 or (size != - int(size) - and size > 1): - raise ValueError('%s must be a positive integer ' - 'or a float between 0 and 1. Got %r' % - (param_name, size)) - elif size > n_samples: - raise ValueError('%s must be no greater than the' - ' number of samples (%d). Got %d' % - (param_name, n_samples, size)) + if size > n_samples: + raise ValueError( + "%s must be no greater than the number of samples (%d). Got %d" + % (param_name, n_samples, size) + ) # OPTICS helper functions def _compute_core_distances_(X, neighbors, min_samples, working_memory): - """Compute the k-th nearest neighbor of each sample + """Compute the k-th nearest neighbor of each sample. Equivalent to neighbors.kneighbors(X, self.min_samples)[0][:, -1] but with more memory efficiency. @@ -338,29 +430,45 @@ def _compute_core_distances_(X, neighbors, min_samples, working_memory): core_distances = np.empty(n_samples) core_distances.fill(np.nan) - chunk_n_rows = get_chunk_n_rows(row_bytes=16 * min_samples, - max_n_rows=n_samples, - working_memory=working_memory) + chunk_n_rows = get_chunk_n_rows( + row_bytes=16 * min_samples, max_n_rows=n_samples, working_memory=working_memory + ) slices = gen_batches(n_samples, chunk_n_rows) for sl in slices: - core_distances[sl] = neighbors.kneighbors( - X[sl], min_samples)[0][:, -1] + core_distances[sl] = neighbors.kneighbors(X[sl], min_samples)[0][:, -1] return core_distances -@_deprecate_positional_args -def compute_optics_graph(X, *, min_samples, max_eps, metric, p, metric_params, - algorithm, leaf_size, n_jobs): - """Computes the OPTICS reachability graph. +@validate_params( + { + "X": [np.ndarray, "sparse matrix"], + "min_samples": [ + Interval(Integral, 2, None, closed="left"), + Interval(RealNotInt, 0, 1, closed="both"), + ], + "max_eps": [Interval(Real, 0, None, closed="both")], + "metric": [StrOptions(set(_VALID_METRICS) | {"precomputed"}), callable], + "p": [Interval(Real, 0, None, closed="right"), None], + "metric_params": [dict, None], + "algorithm": [StrOptions({"auto", "brute", "ball_tree", "kd_tree"})], + "leaf_size": [Interval(Integral, 1, None, closed="left")], + "n_jobs": [Integral, None], + }, + prefer_skip_nested_validation=False, # metric is not validated yet +) +def compute_optics_graph( + X, *, min_samples, max_eps, metric, p, metric_params, algorithm, leaf_size, n_jobs +): + """Compute the OPTICS reachability graph. Read more in the :ref:`User Guide `. Parameters ---------- - X : ndarray of shape (n_samples, n_features), or \ - (n_samples, n_samples) if metric=’precomputed’. + X : {ndarray, sparse matrix} of shape (n_samples, n_features), or \ + (n_samples, n_samples) if metric='precomputed' A feature array, or array of distances between samples if - metric='precomputed' + metric='precomputed'. min_samples : int > 1 or float between 0 and 1 The number of samples in a neighborhood for a point to be considered @@ -398,7 +506,10 @@ def compute_optics_graph(X, *, min_samples, max_eps, metric, p, metric_params, See the documentation for scipy.spatial.distance for details on these metrics. - p : int, default=2 + .. note:: + `'kulsinski'` is deprecated from SciPy 1.9 and will be removed in SciPy 1.11. + + p : float, default=2 Parameter for the Minkowski metric from :class:`~sklearn.metrics.pairwise_distances`. When p = 1, this is equivalent to using manhattan_distance (l1), and euclidean_distance @@ -410,20 +521,20 @@ def compute_optics_graph(X, *, min_samples, max_eps, metric, p, metric_params, algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, default='auto' Algorithm used to compute the nearest neighbors: - - 'ball_tree' will use :class:`BallTree` - - 'kd_tree' will use :class:`KDTree` + - 'ball_tree' will use :class:`~sklearn.neighbors.BallTree`. + - 'kd_tree' will use :class:`~sklearn.neighbors.KDTree`. - 'brute' will use a brute-force search. - 'auto' will attempt to decide the most appropriate algorithm - based on the values passed to :meth:`fit` method. (default) + based on the values passed to `fit` method. (default) Note: fitting on sparse input will override the setting of this parameter, using brute force. leaf_size : int, default=30 - Leaf size passed to :class:`BallTree` or :class:`KDTree`. This can - affect the speed of the construction and query, as well as the memory - required to store the tree. The optimal value depends on the - nature of the problem. + Leaf size passed to :class:`~sklearn.neighbors.BallTree` or + :class:`~sklearn.neighbors.KDTree`. This can affect the speed of the + construction and query, as well as the memory required to store the + tree. The optimal value depends on the nature of the problem. n_jobs : int, default=None The number of parallel jobs to run for neighbors search. @@ -454,9 +565,37 @@ def compute_optics_graph(X, *, min_samples, max_eps, metric, p, metric_params, .. [1] Ankerst, Mihael, Markus M. Breunig, Hans-Peter Kriegel, and JÃļrg Sander. "OPTICS: ordering points to identify the clustering structure." ACM SIGMOD Record 28, no. 2 (1999): 49-60. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.cluster import compute_optics_graph + >>> X = np.array([[1, 2], [2, 5], [3, 6], + ... [8, 7], [8, 8], [7, 3]]) + >>> ordering, core_distances, reachability, predecessor = compute_optics_graph( + ... X, + ... min_samples=2, + ... max_eps=np.inf, + ... metric="minkowski", + ... p=2, + ... metric_params=None, + ... algorithm="auto", + ... leaf_size=30, + ... n_jobs=None, + ... ) + >>> ordering + array([0, 1, 2, 5, 3, 4]) + >>> core_distances + array([3.16, 1.41, 1.41, 1. , 1. , + 4.12]) + >>> reachability + array([ inf, 3.16, 1.41, 4.12, 1. , + 5. ]) + >>> predecessor + array([-1, 0, 1, 5, 3, 2]) """ n_samples = X.shape[0] - _validate_size(min_samples, n_samples, 'min_samples') + _validate_size(min_samples, n_samples, "min_samples") if min_samples <= 1: min_samples = max(2, int(min_samples * n_samples)) @@ -466,26 +605,30 @@ def compute_optics_graph(X, *, min_samples, max_eps, metric, p, metric_params, predecessor_ = np.empty(n_samples, dtype=int) predecessor_.fill(-1) - nbrs = NearestNeighbors(n_neighbors=min_samples, - algorithm=algorithm, - leaf_size=leaf_size, - metric=metric, - metric_params=metric_params, - p=p, - n_jobs=n_jobs) + nbrs = NearestNeighbors( + n_neighbors=min_samples, + algorithm=algorithm, + leaf_size=leaf_size, + metric=metric, + metric_params=metric_params, + p=p, + n_jobs=n_jobs, + ) nbrs.fit(X) # Here we first do a kNN query for each point, this differs from # the original OPTICS that only used epsilon range queries. # TODO: handle working_memory somehow? - core_distances_ = _compute_core_distances_(X=X, neighbors=nbrs, - min_samples=min_samples, - working_memory=None) + core_distances_ = _compute_core_distances_( + X=X, neighbors=nbrs, min_samples=min_samples, working_memory=None + ) # OPTICS puts an upper limit on these, use inf for undefined. core_distances_[core_distances_ > max_eps] = np.inf - np.around(core_distances_, - decimals=np.finfo(core_distances_.dtype).precision, - out=core_distances_) + np.around( + core_distances_, + decimals=np.finfo(core_distances_.dtype).precision, + out=core_distances_, + ) # Main OPTICS loop. Not parallelizable. The order that entries are # written to the 'ordering_' list is important! @@ -502,29 +645,48 @@ def compute_optics_graph(X, *, min_samples, max_eps, metric, p, metric_params, processed[point] = True ordering[ordering_idx] = point if core_distances_[point] != np.inf: - _set_reach_dist(core_distances_=core_distances_, - reachability_=reachability_, - predecessor_=predecessor_, - point_index=point, - processed=processed, X=X, nbrs=nbrs, - metric=metric, metric_params=metric_params, - p=p, max_eps=max_eps) + _set_reach_dist( + core_distances_=core_distances_, + reachability_=reachability_, + predecessor_=predecessor_, + point_index=point, + processed=processed, + X=X, + nbrs=nbrs, + metric=metric, + metric_params=metric_params, + p=p, + max_eps=max_eps, + ) if np.all(np.isinf(reachability_)): - warnings.warn("All reachability values are inf. Set a larger" - " max_eps or all data will be considered outliers.", - UserWarning) + warnings.warn( + ( + "All reachability values are inf. Set a larger" + " max_eps or all data will be considered outliers." + ), + UserWarning, + ) return ordering, core_distances_, reachability_, predecessor_ -def _set_reach_dist(core_distances_, reachability_, predecessor_, - point_index, processed, X, nbrs, metric, metric_params, - p, max_eps): - P = X[point_index:point_index + 1] +def _set_reach_dist( + core_distances_, + reachability_, + predecessor_, + point_index, + processed, + X, + nbrs, + metric, + metric_params, + p, + max_eps, +): + P = X[point_index : point_index + 1] # Assume that radius_neighbors is faster without distances # and we don't need all distances, nevertheless, this means # we may be doing some work twice. - indices = nbrs.radius_neighbors(P, radius=max_eps, - return_distance=False)[0] + indices = nbrs.radius_neighbors(P, radius=max_eps, return_distance=False)[0] # Getting indices of neighbors that have not been processed unproc = np.compress(~np.take(processed, indices), indices) @@ -533,17 +695,18 @@ def _set_reach_dist(core_distances_, reachability_, predecessor_, return # Only compute distances to unprocessed neighbors: - if metric == 'precomputed': - dists = X[point_index, unproc] + if metric == "precomputed": + dists = X[[point_index], unproc] + if isinstance(dists, np.matrix): + dists = np.asarray(dists) + dists = dists.ravel() else: _params = dict() if metric_params is None else metric_params.copy() - if metric == 'minkowski' and 'p' not in _params: + if metric == "minkowski" and "p" not in _params: # the same logic as neighbors, p is ignored if explicitly set # in the dict params - _params['p'] = p - dists = pairwise_distances(P, np.take(X, unproc, axis=0), - metric=metric, n_jobs=None, - **_params).ravel() + _params["p"] = p + dists = pairwise_distances(P, X[unproc], metric, n_jobs=None, **_params).ravel() rdists = np.maximum(dists, core_distances_[point_index]) np.around(rdists, decimals=np.finfo(rdists.dtype).precision, out=rdists) @@ -552,9 +715,17 @@ def _set_reach_dist(core_distances_, reachability_, predecessor_, predecessor_[unproc[improved]] = point_index -@_deprecate_positional_args +@validate_params( + { + "reachability": [np.ndarray], + "core_distances": [np.ndarray], + "ordering": [np.ndarray], + "eps": [Interval(Real, 0, None, closed="both")], + }, + prefer_skip_nested_validation=True, +) def cluster_optics_dbscan(*, reachability, core_distances, ordering, eps): - """Performs DBSCAN extraction for an arbitrary epsilon. + """Perform DBSCAN extraction for an arbitrary epsilon. Extracting the clusters runs in linear time. Note that this results in ``labels_`` which are close to a :class:`~sklearn.cluster.DBSCAN` with @@ -562,14 +733,14 @@ def cluster_optics_dbscan(*, reachability, core_distances, ordering, eps): Parameters ---------- - reachability : array of shape (n_samples,) - Reachability distances calculated by OPTICS (``reachability_``) + reachability : ndarray of shape (n_samples,) + Reachability distances calculated by OPTICS (``reachability_``). - core_distances : array of shape (n_samples,) - Distances at which points become core (``core_distances_``) + core_distances : ndarray of shape (n_samples,) + Distances at which points become core (``core_distances_``). - ordering : array of shape (n_samples,) - OPTICS ordered point indices (``ordering_``) + ordering : ndarray of shape (n_samples,) + OPTICS ordered point indices (``ordering_``). eps : float DBSCAN ``eps`` parameter. Must be set to < ``max_eps``. Results @@ -581,6 +752,32 @@ def cluster_optics_dbscan(*, reachability, core_distances, ordering, eps): labels_ : array of shape (n_samples,) The estimated labels. + Examples + -------- + >>> import numpy as np + >>> from sklearn.cluster import cluster_optics_dbscan, compute_optics_graph + >>> X = np.array([[1, 2], [2, 5], [3, 6], + ... [8, 7], [8, 8], [7, 3]]) + >>> ordering, core_distances, reachability, predecessor = compute_optics_graph( + ... X, + ... min_samples=2, + ... max_eps=np.inf, + ... metric="minkowski", + ... p=2, + ... metric_params=None, + ... algorithm="auto", + ... leaf_size=30, + ... n_jobs=None, + ... ) + >>> eps = 4.5 + >>> labels = cluster_optics_dbscan( + ... reachability=reachability, + ... core_distances=core_distances, + ... ordering=ordering, + ... eps=eps, + ... ) + >>> labels + array([0, 0, 0, 1, 1, 1]) """ n_samples = len(core_distances) labels = np.zeros(n_samples, dtype=int) @@ -592,21 +789,47 @@ def cluster_optics_dbscan(*, reachability, core_distances, ordering, eps): return labels -def cluster_optics_xi(*, reachability, predecessor, ordering, min_samples, - min_cluster_size=None, xi=0.05, - predecessor_correction=True): +@validate_params( + { + "reachability": [np.ndarray], + "predecessor": [np.ndarray], + "ordering": [np.ndarray], + "min_samples": [ + Interval(Integral, 2, None, closed="left"), + Interval(RealNotInt, 0, 1, closed="both"), + ], + "min_cluster_size": [ + Interval(Integral, 2, None, closed="left"), + Interval(RealNotInt, 0, 1, closed="both"), + None, + ], + "xi": [Interval(Real, 0, 1, closed="both")], + "predecessor_correction": ["boolean"], + }, + prefer_skip_nested_validation=True, +) +def cluster_optics_xi( + *, + reachability, + predecessor, + ordering, + min_samples, + min_cluster_size=None, + xi=0.05, + predecessor_correction=True, +): """Automatically extract clusters according to the Xi-steep method. Parameters ---------- reachability : ndarray of shape (n_samples,) - Reachability distances calculated by OPTICS (`reachability_`) + Reachability distances calculated by OPTICS (`reachability_`). predecessor : ndarray of shape (n_samples,) Predecessors calculated by OPTICS. ordering : ndarray of shape (n_samples,) - OPTICS ordered point indices (`ordering_`) + OPTICS ordered point indices (`ordering_`). min_samples : int > 1 or float between 0 and 1 The same as the min_samples given to OPTICS. Up and down steep regions @@ -641,21 +864,57 @@ def cluster_optics_xi(*, reachability, predecessor, ordering, min_samples, clusters come after such nested smaller clusters. Since ``labels`` does not reflect the hierarchy, usually ``len(clusters) > np.unique(labels)``. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.cluster import cluster_optics_xi, compute_optics_graph + >>> X = np.array([[1, 2], [2, 5], [3, 6], + ... [8, 7], [8, 8], [7, 3]]) + >>> ordering, core_distances, reachability, predecessor = compute_optics_graph( + ... X, + ... min_samples=2, + ... max_eps=np.inf, + ... metric="minkowski", + ... p=2, + ... metric_params=None, + ... algorithm="auto", + ... leaf_size=30, + ... n_jobs=None + ... ) + >>> min_samples = 2 + >>> labels, clusters = cluster_optics_xi( + ... reachability=reachability, + ... predecessor=predecessor, + ... ordering=ordering, + ... min_samples=min_samples, + ... ) + >>> labels + array([0, 0, 0, 1, 1, 1]) + >>> clusters + array([[0, 2], + [3, 5], + [0, 5]]) """ n_samples = len(reachability) - _validate_size(min_samples, n_samples, 'min_samples') + _validate_size(min_samples, n_samples, "min_samples") if min_samples <= 1: min_samples = max(2, int(min_samples * n_samples)) if min_cluster_size is None: min_cluster_size = min_samples - _validate_size(min_cluster_size, n_samples, 'min_cluster_size') + _validate_size(min_cluster_size, n_samples, "min_cluster_size") if min_cluster_size <= 1: min_cluster_size = max(2, int(min_cluster_size * n_samples)) - clusters = _xi_cluster(reachability[ordering], predecessor[ordering], - ordering, xi, - min_samples, min_cluster_size, - predecessor_correction) + clusters = _xi_cluster( + reachability[ordering], + predecessor[ordering], + ordering, + xi, + min_samples, + min_cluster_size, + predecessor_correction, + ) labels = _extract_xi_labels(ordering, clusters) return labels, clusters @@ -729,10 +988,11 @@ def _update_filter_sdas(sdas, mib, xi_complement, reachability_plot): """ if np.isinf(mib): return [] - res = [sda for sda in sdas - if mib <= reachability_plot[sda['start']] * xi_complement] + res = [ + sda for sda in sdas if mib <= reachability_plot[sda["start"]] * xi_complement + ] for sda in res: - sda['mib'] = max(sda['mib'], mib) + sda["mib"] = max(sda["mib"], mib) return res @@ -750,7 +1010,7 @@ def _correct_predecessor(reachability_plot, predecessor_plot, ordering, s, e): while s < e: if reachability_plot[s] > reachability_plot[e]: return s, e - p_e = ordering[predecessor_plot[e]] + p_e = predecessor_plot[e] for i in range(s, e): if p_e == ordering[i]: return s, e @@ -758,8 +1018,15 @@ def _correct_predecessor(reachability_plot, predecessor_plot, ordering, s, e): return None, None -def _xi_cluster(reachability_plot, predecessor_plot, ordering, xi, min_samples, - min_cluster_size, predecessor_correction): +def _xi_cluster( + reachability_plot, + predecessor_plot, + ordering, + xi, + min_samples, + min_cluster_size, + predecessor_correction, +): """Automatically extract clusters according to the Xi-steep method. This is rouphly an implementation of Figure 19 of the OPTICS paper. @@ -807,20 +1074,20 @@ def _xi_cluster(reachability_plot, predecessor_plot, ordering, xi, min_samples, sdas = [] # steep down areas, introduced in section 4.3.2 of the paper clusters = [] index = 0 - mib = 0. # maximum in between, section 4.3.2 + mib = 0.0 # maximum in between, section 4.3.2 # Our implementation corrects a mistake in the original # paper, i.e., in Definition 9 steep downward point, # r(p) * (1 - x1) <= r(p + 1) should be # r(p) * (1 - x1) >= r(p + 1) - with np.errstate(invalid='ignore'): + with np.errstate(invalid="ignore"): ratio = reachability_plot[:-1] / reachability_plot[1:] steep_upward = ratio <= xi_complement steep_downward = ratio >= 1 / xi_complement downward = ratio > 1 upward = ratio < 1 - # the following loop is is almost exactly as Figure 19 of the paper. + # the following loop is almost exactly as Figure 19 of the paper. # it jumps over the areas which are not either steep down or up areas for steep_index in iter(np.flatnonzero(steep_upward | steep_downward)): # just continue if steep_index has been a part of a discovered xward @@ -828,47 +1095,44 @@ def _xi_cluster(reachability_plot, predecessor_plot, ordering, xi, min_samples, if steep_index < index: continue - mib = max(mib, np.max(reachability_plot[index:steep_index + 1])) + mib = max(mib, np.max(reachability_plot[index : steep_index + 1])) # steep downward areas if steep_downward[steep_index]: - sdas = _update_filter_sdas(sdas, mib, xi_complement, - reachability_plot) + sdas = _update_filter_sdas(sdas, mib, xi_complement, reachability_plot) D_start = steep_index - D_end = _extend_region(steep_downward, upward, - D_start, min_samples) - D = {'start': D_start, 'end': D_end, 'mib': 0.} + D_end = _extend_region(steep_downward, upward, D_start, min_samples) + D = {"start": D_start, "end": D_end, "mib": 0.0} sdas.append(D) index = D_end + 1 mib = reachability_plot[index] # steep upward areas else: - sdas = _update_filter_sdas(sdas, mib, xi_complement, - reachability_plot) + sdas = _update_filter_sdas(sdas, mib, xi_complement, reachability_plot) U_start = steep_index - U_end = _extend_region(steep_upward, downward, U_start, - min_samples) + U_end = _extend_region(steep_upward, downward, U_start, min_samples) index = U_end + 1 mib = reachability_plot[index] U_clusters = [] for D in sdas: - c_start = D['start'] + c_start = D["start"] c_end = U_end # line (**), sc2* - if reachability_plot[c_end + 1] * xi_complement < D['mib']: + if reachability_plot[c_end + 1] * xi_complement < D["mib"]: continue # Definition 11: criterion 4 - D_max = reachability_plot[D['start']] + D_max = reachability_plot[D["start"]] if D_max * xi_complement >= reachability_plot[c_end + 1]: # Find the first index from the left side which is almost # at the same level as the end of the detected cluster. - while (reachability_plot[c_start + 1] > - reachability_plot[c_end + 1] - and c_start < D['end']): + while ( + reachability_plot[c_start + 1] > reachability_plot[c_end + 1] + and c_start < D["end"] + ): c_start += 1 elif reachability_plot[c_end + 1] * xi_complement >= D_max: # Find the first index from the right side which is almost @@ -877,17 +1141,14 @@ def _xi_cluster(reachability_plot, predecessor_plot, ordering, xi, min_samples, # Our implementation corrects a mistake in the original # paper, i.e., in Definition 11 4c, r(x) < r(sD) should be # r(x) > r(sD). - while (reachability_plot[c_end - 1] > D_max - and c_end > U_start): + while reachability_plot[c_end - 1] > D_max and c_end > U_start: c_end -= 1 # predecessor correction if predecessor_correction: - c_start, c_end = _correct_predecessor(reachability_plot, - predecessor_plot, - ordering, - c_start, - c_end) + c_start, c_end = _correct_predecessor( + reachability_plot, predecessor_plot, ordering, c_start, c_end + ) if c_start is None: continue @@ -896,7 +1157,7 @@ def _xi_cluster(reachability_plot, predecessor_plot, ordering, xi, min_samples, continue # Definition 11: criterion 1 - if c_start > D['end']: + if c_start > D["end"]: continue # Definition 11: criterion 2 @@ -934,8 +1195,8 @@ def _extract_xi_labels(ordering, clusters): labels = np.full(len(ordering), -1, dtype=int) label = 0 for c in clusters: - if not np.any(labels[c[0]:(c[1] + 1)] != -1): - labels[c[0]:(c[1] + 1)] = label + if not np.any(labels[c[0] : (c[1] + 1)] != -1): + labels[c[0] : (c[1] + 1)] = label label += 1 labels[ordering] = labels.copy() return labels diff --git a/sklearn/cluster/_spectral.py b/sklearn/cluster/_spectral.py index b86d5870025c3..00d23437504e5 100644 --- a/sklearn/cluster/_spectral.py +++ b/sklearn/cluster/_spectral.py @@ -1,29 +1,63 @@ -# -*- coding: utf-8 -*- """Algorithms for spectral clustering""" -# Author: Gael Varoquaux gael.varoquaux@normalesup.org -# Brian Cheung -# Wei LI -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import warnings +from numbers import Integral, Real import numpy as np - -from ..base import BaseEstimator, ClusterMixin -from ..utils import check_random_state, as_float_array -from ..utils.validation import _deprecate_positional_args -from ..utils.deprecation import deprecated -from ..metrics.pairwise import pairwise_kernels -from ..neighbors import kneighbors_graph, NearestNeighbors -from ..manifold import spectral_embedding +from scipy.linalg import LinAlgError, qr, svd +from scipy.sparse import csc_matrix + +from ..base import BaseEstimator, ClusterMixin, _fit_context +from ..manifold._spectral_embedding import _spectral_embedding +from ..metrics.pairwise import KERNEL_PARAMS, pairwise_kernels +from ..neighbors import NearestNeighbors, kneighbors_graph +from ..utils import as_float_array, check_random_state +from ..utils._param_validation import Interval, StrOptions, validate_params +from ..utils.validation import validate_data from ._kmeans import k_means -@_deprecate_positional_args -def discretize(vectors, *, copy=True, max_svd_restarts=30, n_iter_max=20, - random_state=None): - """Search for a partition matrix (clustering) which is closest to the - eigenvector embedding. +def cluster_qr(vectors): + """Find the discrete partition closest to the eigenvector embedding. + + This implementation was proposed in [1]_. + + .. versionadded:: 1.1 + + Parameters + ---------- + vectors : array-like, shape: (n_samples, n_clusters) + The embedding space of the samples. + + Returns + ------- + labels : array of integers, shape: n_samples + The cluster labels of vectors. + + References + ---------- + .. [1] :doi:`Simple, direct, and efficient multi-way spectral clustering, 2019 + Anil Damle, Victor Minden, Lexing Ying + <10.1093/imaiai/iay008>` + + """ + + k = vectors.shape[1] + _, _, piv = qr(vectors.T, pivoting=True) + ut, _, v = svd(vectors[piv[:k], :].T) + vectors = abs(np.dot(vectors, np.dot(ut, v.conj()))) + return vectors.argmax(axis=1) + + +def discretize( + vectors, *, copy=True, max_svd_restarts=30, n_iter_max=20, random_state=None +): + """Search for a partition matrix which is closest to the eigenvector embedding. + + This implementation was proposed in [1]_. Parameters ---------- @@ -53,9 +87,9 @@ def discretize(vectors, *, copy=True, max_svd_restarts=30, n_iter_max=20, References ---------- - - Multiclass spectral clustering, 2003 - Stella X. Yu, Jianbo Shi - https://www1.icsi.berkeley.edu/~stellayu/publication/doc/2003kwayICCV.pdf + .. [1] `Multiclass spectral clustering, 2003 + Stella X. Yu, Jianbo Shi + `_ Notes ----- @@ -73,9 +107,6 @@ def discretize(vectors, *, copy=True, max_svd_restarts=30, n_iter_max=20, """ - from scipy.sparse import csc_matrix - from scipy.linalg import LinAlgError - random_state = check_random_state(random_state) vectors = as_float_array(vectors, copy=copy) @@ -90,15 +121,14 @@ def discretize(vectors, *, copy=True, max_svd_restarts=30, n_iter_max=20, # search easier. norm_ones = np.sqrt(n_samples) for i in range(vectors.shape[1]): - vectors[:, i] = (vectors[:, i] / np.linalg.norm(vectors[:, i])) \ - * norm_ones + vectors[:, i] = (vectors[:, i] / np.linalg.norm(vectors[:, i])) * norm_ones if vectors[0, i] != 0: vectors[:, i] = -1 * vectors[:, i] * np.sign(vectors[0, i]) # Normalize the rows of the eigenvectors. Samples should lie on the unit # hypersphere centered at the origin. This transforms the samples in the # embedding space to the space of partition matrices. - vectors = vectors / np.sqrt((vectors ** 2).sum(axis=1))[:, np.newaxis] + vectors = vectors / np.sqrt((vectors**2).sum(axis=1))[:, np.newaxis] svd_restarts = 0 has_converged = False @@ -106,7 +136,6 @@ def discretize(vectors, *, copy=True, max_svd_restarts=30, n_iter_max=20, # If there is an exception we try to randomize and rerun SVD again # do this max_svd_restarts times. while (svd_restarts < max_svd_restarts) and not has_converged: - # Initialize first column of rotation matrix with a row of the # eigenvectors rotation = np.zeros((n_components, n_components)) @@ -133,20 +162,20 @@ def discretize(vectors, *, copy=True, max_svd_restarts=30, n_iter_max=20, labels = t_discrete.argmax(axis=1) vectors_discrete = csc_matrix( (np.ones(len(labels)), (np.arange(0, n_samples), labels)), - shape=(n_samples, n_components)) + shape=(n_samples, n_components), + ) - t_svd = vectors_discrete.T * vectors + t_svd = vectors_discrete.T @ vectors try: U, S, Vh = np.linalg.svd(t_svd) - svd_restarts += 1 except LinAlgError: + svd_restarts += 1 print("SVD did not converge, randomizing and trying again") break ncut_value = 2.0 * (n_samples - S.sum()) - if ((abs(ncut_value - last_objective_value) < eps) or - (n_iter > n_iter_max)): + if (abs(ncut_value - last_objective_value) < eps) or (n_iter > n_iter_max): has_converged = True else: # otherwise calculate rotation and continue @@ -154,15 +183,26 @@ def discretize(vectors, *, copy=True, max_svd_restarts=30, n_iter_max=20, rotation = np.dot(Vh.T, U.T) if not has_converged: - raise LinAlgError('SVD did not converge') + raise LinAlgError("SVD did not converge") return labels -@_deprecate_positional_args -def spectral_clustering(affinity, *, n_clusters=8, n_components=None, - eigen_solver=None, random_state=None, n_init=10, - eigen_tol=0.0, assign_labels='kmeans', - verbose=False): +@validate_params( + {"affinity": ["array-like", "sparse matrix"]}, + prefer_skip_nested_validation=False, +) +def spectral_clustering( + affinity, + *, + n_clusters=8, + n_components=None, + eigen_solver=None, + random_state=None, + n_init=10, + eigen_tol="auto", + assign_labels="kmeans", + verbose=False, +): """Apply clustering to a projection of the normalized Laplacian. In practice Spectral Clustering is very useful when the structure of @@ -172,7 +212,7 @@ def spectral_clustering(affinity, *, n_clusters=8, n_components=None, nested circles on the 2D plane. If affinity is the adjacency matrix of a graph, this method can be - used to find normalized graph cuts. + used to find normalized graph cuts [1]_, [2]_. Read more in the :ref:`User Guide `. @@ -191,38 +231,68 @@ def spectral_clustering(affinity, *, n_clusters=8, n_components=None, Number of clusters to extract. n_components : int, default=n_clusters - Number of eigen vectors to use for the spectral embedding + Number of eigenvectors to use for the spectral embedding. eigen_solver : {None, 'arpack', 'lobpcg', or 'amg'} - The eigenvalue decomposition strategy to use. AMG requires pyamg - to be installed. It can be faster on very large, sparse problems, - but may also lead to instabilities. If None, then ``'arpack'`` is - used. + The eigenvalue decomposition method. If None then ``'arpack'`` is used. + See [4]_ for more details regarding ``'lobpcg'``. + Eigensolver ``'amg'`` runs ``'lobpcg'`` with optional + Algebraic MultiGrid preconditioning and requires pyamg to be installed. + It can be faster on very large sparse problems [6]_ and [7]_. random_state : int, RandomState instance, default=None - A pseudo random number generator used for the initialization of the - lobpcg eigen vectors decomposition when eigen_solver == 'amg' and by - the K-Means initialization. Use an int to make the randomness - deterministic. - See :term:`Glossary `. + A pseudo random number generator used for the initialization + of the lobpcg eigenvectors decomposition when `eigen_solver == + 'amg'`, and for the K-Means initialization. Use an int to make + the results deterministic across calls (See + :term:`Glossary `). + + .. note:: + When using `eigen_solver == 'amg'`, + it is necessary to also fix the global numpy seed with + `np.random.seed(int)` to get deterministic results. See + https://github.com/pyamg/pyamg/issues/139 for further + information. n_init : int, default=10 Number of time the k-means algorithm will be run with different - centroid seeds. The final results will be the best output of - n_init consecutive runs in terms of inertia. + centroid seeds. The final results will be the best output of n_init + consecutive runs in terms of inertia. Only used if + ``assign_labels='kmeans'``. + + eigen_tol : float, default="auto" + Stopping criterion for eigendecomposition of the Laplacian matrix. + If `eigen_tol="auto"` then the passed tolerance will depend on the + `eigen_solver`: - eigen_tol : float, default=0.0 - Stopping criterion for eigendecomposition of the Laplacian matrix - when using arpack eigen_solver. + - If `eigen_solver="arpack"`, then `eigen_tol=0.0`; + - If `eigen_solver="lobpcg"` or `eigen_solver="amg"`, then + `eigen_tol=None` which configures the underlying `lobpcg` solver to + automatically resolve the value according to their heuristics. See, + :func:`scipy.sparse.linalg.lobpcg` for details. - assign_labels : {'kmeans', 'discretize'}, default='kmeans' + Note that when using `eigen_solver="lobpcg"` or `eigen_solver="amg"` + values of `tol<1e-5` may lead to convergence issues and should be + avoided. + + .. versionadded:: 1.2 + Added 'auto' option. + + assign_labels : {'kmeans', 'discretize', 'cluster_qr'}, default='kmeans' The strategy to use to assign labels in the embedding - space. There are two ways to assign labels after the laplacian + space. There are three ways to assign labels after the Laplacian embedding. k-means can be applied and is a popular choice. But it can also be sensitive to initialization. Discretization is another - approach which is less sensitive to random initialization. See - the 'Multiclass spectral clustering' paper referenced below for - more details on the discretization approach. + approach which is less sensitive to random initialization [3]_. + The cluster_qr method [5]_ directly extracts clusters from eigenvectors + in spectral clustering. In contrast to k-means and discretization, cluster_qr + has no tuning parameters and is not an iterative method, yet may outperform + k-means and discretization in terms of both quality and speed. For a detailed + comparison of clustering strategies, refer to the following example: + :ref:`sphx_glr_auto_examples_cluster_plot_coin_segmentation.py`. + + .. versionchanged:: 1.1 + Added new labeling method 'cluster_qr'. verbose : bool, default=False Verbosity mode. @@ -234,78 +304,100 @@ def spectral_clustering(affinity, *, n_clusters=8, n_components=None, labels : array of integers, shape: n_samples The labels of the clusters. + Notes + ----- + The graph should contain only one connected component, elsewhere + the results make little sense. + + This algorithm solves the normalized cut for `k=2`: it is a + normalized spectral clustering. + References ---------- - - Normalized cuts and image segmentation, 2000 - Jianbo Shi, Jitendra Malik - http://citeseer.ist.psu.edu/viewdoc/summary?doi=10.1.1.160.2324 + .. [1] :doi:`Normalized cuts and image segmentation, 2000 + Jianbo Shi, Jitendra Malik + <10.1109/34.868688>` - - A Tutorial on Spectral Clustering, 2007 - Ulrike von Luxburg - http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.165.9323 + .. [2] :doi:`A Tutorial on Spectral Clustering, 2007 + Ulrike von Luxburg + <10.1007/s11222-007-9033-z>` - - Multiclass spectral clustering, 2003 - Stella X. Yu, Jianbo Shi - https://www1.icsi.berkeley.edu/~stellayu/publication/doc/2003kwayICCV.pdf + .. [3] `Multiclass spectral clustering, 2003 + Stella X. Yu, Jianbo Shi + `_ - Notes - ----- - The graph should contain only one connect component, elsewhere - the results make little sense. + .. [4] :doi:`Toward the Optimal Preconditioned Eigensolver: + Locally Optimal Block Preconditioned Conjugate Gradient Method, 2001 + A. V. Knyazev + SIAM Journal on Scientific Computing 23, no. 2, pp. 517-541. + <10.1137/S1064827500366124>` - This algorithm solves the normalized cut for k=2: it is a - normalized spectral clustering. + .. [5] :doi:`Simple, direct, and efficient multi-way spectral clustering, 2019 + Anil Damle, Victor Minden, Lexing Ying + <10.1093/imaiai/iay008>` + + .. [6] :doi:`Multiscale Spectral Image Segmentation Multiscale preconditioning + for computing eigenvalues of graph Laplacians in image segmentation, 2006 + Andrew Knyazev + <10.13140/RG.2.2.35280.02565>` + + .. [7] :doi:`Preconditioned spectral clustering for stochastic block partition + streaming graph challenge (Preliminary version at arXiv.) + David Zhuzhunashvili, Andrew Knyazev + <10.1109/HPEC.2017.8091045>` + + Examples + -------- + >>> import numpy as np + >>> from sklearn.metrics.pairwise import pairwise_kernels + >>> from sklearn.cluster import spectral_clustering + >>> X = np.array([[1, 1], [2, 1], [1, 0], + ... [4, 7], [3, 5], [3, 6]]) + >>> affinity = pairwise_kernels(X, metric='rbf') + >>> spectral_clustering( + ... affinity=affinity, n_clusters=2, assign_labels="discretize", random_state=0 + ... ) + array([1, 1, 1, 0, 0, 0]) """ - if assign_labels not in ('kmeans', 'discretize'): - raise ValueError("The 'assign_labels' parameter should be " - "'kmeans' or 'discretize', but '%s' was given" - % assign_labels) - random_state = check_random_state(random_state) - n_components = n_clusters if n_components is None else n_components - - # The first eigen vector is constant only for fully connected graphs - # and should be kept for spectral clustering (drop_first = False) - # See spectral_embedding documentation. - maps = spectral_embedding(affinity, n_components=n_components, - eigen_solver=eigen_solver, - random_state=random_state, - eigen_tol=eigen_tol, drop_first=False) - if verbose: - print(f'Computing label assignment using {assign_labels}') - - if assign_labels == 'kmeans': - _, labels, _ = k_means(maps, n_clusters, random_state=random_state, - n_init=n_init, verbose=verbose) - else: - labels = discretize(maps, random_state=random_state) + clusterer = SpectralClustering( + n_clusters=n_clusters, + n_components=n_components, + eigen_solver=eigen_solver, + random_state=random_state, + n_init=n_init, + affinity="precomputed", + eigen_tol=eigen_tol, + assign_labels=assign_labels, + verbose=verbose, + ).fit(affinity) - return labels + return clusterer.labels_ class SpectralClustering(ClusterMixin, BaseEstimator): """Apply clustering to a projection of the normalized Laplacian. In practice Spectral Clustering is very useful when the structure of - the individual clusters is highly non-convex or more generally when + the individual clusters is highly non-convex, or more generally when a measure of the center and spread of the cluster is not a suitable - description of the complete cluster. For instance when clusters are + description of the complete cluster, such as when clusters are nested circles on the 2D plane. - If affinity is the adjacency matrix of a graph, this method can be - used to find normalized graph cuts. + If the affinity matrix is the adjacency matrix of a graph, this method + can be used to find normalized graph cuts [1]_, [2]_. When calling ``fit``, an affinity matrix is constructed using either - kernel function such the Gaussian (aka RBF) kernel of the euclidean - distanced ``d(X, X)``:: + a kernel function such the Gaussian (aka RBF) kernel with Euclidean + distance ``d(X, X)``:: np.exp(-gamma * d(X,X) ** 2) or a k-nearest neighbors connectivity matrix. - Alternatively, using ``precomputed``, a user-provided affinity - matrix can be used. + Alternatively, a user-provided affinity matrix can be specified by + setting ``affinity='precomputed'``. Read more in the :ref:`User Guide `. @@ -318,39 +410,50 @@ class SpectralClustering(ClusterMixin, BaseEstimator): The eigenvalue decomposition strategy to use. AMG requires pyamg to be installed. It can be faster on very large, sparse problems, but may also lead to instabilities. If None, then ``'arpack'`` is - used. + used. See [4]_ for more details regarding `'lobpcg'`. - n_components : int, default=n_clusters - Number of eigen vectors to use for the spectral embedding + n_components : int, default=None + Number of eigenvectors to use for the spectral embedding. If None, + defaults to `n_clusters`. random_state : int, RandomState instance, default=None - A pseudo random number generator used for the initialization of the - lobpcg eigen vectors decomposition when ``eigen_solver='amg'`` and by - the K-Means initialization. Use an int to make the randomness - deterministic. - See :term:`Glossary `. + A pseudo random number generator used for the initialization + of the lobpcg eigenvectors decomposition when `eigen_solver == + 'amg'`, and for the K-Means initialization. Use an int to make + the results deterministic across calls (See + :term:`Glossary `). + + .. note:: + When using `eigen_solver == 'amg'`, + it is necessary to also fix the global numpy seed with + `np.random.seed(int)` to get deterministic results. See + https://github.com/pyamg/pyamg/issues/139 for further + information. n_init : int, default=10 Number of time the k-means algorithm will be run with different - centroid seeds. The final results will be the best output of - n_init consecutive runs in terms of inertia. + centroid seeds. The final results will be the best output of n_init + consecutive runs in terms of inertia. Only used if + ``assign_labels='kmeans'``. gamma : float, default=1.0 Kernel coefficient for rbf, poly, sigmoid, laplacian and chi2 kernels. - Ignored for ``affinity='nearest_neighbors'``. + Ignored for ``affinity='nearest_neighbors'``, ``affinity='precomputed'`` + or ``affinity='precomputed_nearest_neighbors'``. affinity : str or callable, default='rbf' How to construct the affinity matrix. - - 'nearest_neighbors' : construct the affinity matrix by computing a + - 'nearest_neighbors': construct the affinity matrix by computing a graph of nearest neighbors. - - 'rbf' : construct the affinity matrix using a radial basis function + - 'rbf': construct the affinity matrix using a radial basis function (RBF) kernel. - - 'precomputed' : interpret ``X`` as a precomputed affinity matrix. - - 'precomputed_nearest_neighbors' : interpret ``X`` as a sparse graph - of precomputed nearest neighbors, and constructs the affinity matrix - by selecting the ``n_neighbors`` nearest neighbors. + - 'precomputed': interpret ``X`` as a precomputed affinity matrix, + where larger values indicate greater similarity between instances. + - 'precomputed_nearest_neighbors': interpret ``X`` as a sparse graph + of precomputed distances, and construct a binary affinity matrix + from the ``n_neighbors`` nearest neighbors of each instance. - one of the kernels supported by - :func:`~sklearn.metrics.pairwise_kernels`. + :func:`~sklearn.metrics.pairwise.pairwise_kernels`. Only kernels that produce similarity scores (non-negative values that increase with similarity) should be used. This property is not checked @@ -360,16 +463,37 @@ class SpectralClustering(ClusterMixin, BaseEstimator): Number of neighbors to use when constructing the affinity matrix using the nearest neighbors method. Ignored for ``affinity='rbf'``. - eigen_tol : float, default=0.0 - Stopping criterion for eigendecomposition of the Laplacian matrix - when ``eigen_solver='arpack'``. - - assign_labels : {'kmeans', 'discretize'}, default='kmeans' - The strategy to use to assign labels in the embedding - space. There are two ways to assign labels after the laplacian - embedding. k-means can be applied and is a popular choice. But it can - also be sensitive to initialization. Discretization is another approach - which is less sensitive to random initialization. + eigen_tol : float, default="auto" + Stopping criterion for eigen decomposition of the Laplacian matrix. + If `eigen_tol="auto"` then the passed tolerance will depend on the + `eigen_solver`: + + - If `eigen_solver="arpack"`, then `eigen_tol=0.0`; + - If `eigen_solver="lobpcg"` or `eigen_solver="amg"`, then + `eigen_tol=None` which configures the underlying `lobpcg` solver to + automatically resolve the value according to their heuristics. See, + :func:`scipy.sparse.linalg.lobpcg` for details. + + Note that when using `eigen_solver="lobpcg"` or `eigen_solver="amg"` + values of `tol<1e-5` may lead to convergence issues and should be + avoided. + + .. versionadded:: 1.2 + Added 'auto' option. + + assign_labels : {'kmeans', 'discretize', 'cluster_qr'}, default='kmeans' + The strategy for assigning labels in the embedding space. There are two + ways to assign labels after the Laplacian embedding. k-means is a + popular choice, but it can be sensitive to initialization. + Discretization is another approach which is less sensitive to random + initialization [3]_. + The cluster_qr method [5]_ directly extract clusters from eigenvectors + in spectral clustering. In contrast to k-means and discretization, cluster_qr + has no tuning parameters and runs no iterations, yet may outperform + k-means and discretization in terms of both quality and speed. + + .. versionchanged:: 1.1 + Added new labeling method 'cluster_qr'. degree : float, default=3 Degree of the polynomial kernel. Ignored by other kernels. @@ -398,67 +522,136 @@ class SpectralClustering(ClusterMixin, BaseEstimator): Attributes ---------- affinity_matrix_ : array-like of shape (n_samples, n_samples) - Affinity matrix used for clustering. Available only if after calling + Affinity matrix used for clustering. Available only after calling ``fit``. labels_ : ndarray of shape (n_samples,) Labels of each point - Examples + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also -------- - >>> from sklearn.cluster import SpectralClustering - >>> import numpy as np - >>> X = np.array([[1, 1], [2, 1], [1, 0], - ... [4, 7], [3, 5], [3, 6]]) - >>> clustering = SpectralClustering(n_clusters=2, - ... assign_labels="discretize", - ... random_state=0).fit(X) - >>> clustering.labels_ - array([1, 1, 1, 0, 0, 0]) - >>> clustering - SpectralClustering(assign_labels='discretize', n_clusters=2, - random_state=0) + sklearn.cluster.KMeans : K-Means clustering. + sklearn.cluster.DBSCAN : Density-Based Spatial Clustering of + Applications with Noise. Notes ----- - If you have an affinity matrix, such as a distance matrix, - for which 0 means identical elements, and high values means - very dissimilar elements, it can be transformed in a - similarity matrix that is well suited for the algorithm by - applying the Gaussian (RBF, heat) kernel:: + A distance matrix for which 0 indicates identical elements and high values + indicate very dissimilar elements can be transformed into an affinity / + similarity matrix that is well-suited for the algorithm by + applying the Gaussian (aka RBF, heat) kernel:: np.exp(- dist_matrix ** 2 / (2. * delta ** 2)) - Where ``delta`` is a free parameter representing the width of the Gaussian + where ``delta`` is a free parameter representing the width of the Gaussian kernel. - Another alternative is to take a symmetric version of the k - nearest neighbors connectivity matrix of the points. + An alternative is to take a symmetric version of the k-nearest neighbors + connectivity matrix of the points. If the pyamg package is installed, it is used: this greatly speeds up computation. References ---------- + .. [1] :doi:`Normalized cuts and image segmentation, 2000 + Jianbo Shi, Jitendra Malik + <10.1109/34.868688>` - - Normalized cuts and image segmentation, 2000 - Jianbo Shi, Jitendra Malik - http://citeseer.ist.psu.edu/viewdoc/summary?doi=10.1.1.160.2324 + .. [2] :doi:`A Tutorial on Spectral Clustering, 2007 + Ulrike von Luxburg + <10.1007/s11222-007-9033-z>` - - A Tutorial on Spectral Clustering, 2007 - Ulrike von Luxburg - http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.165.9323 + .. [3] `Multiclass spectral clustering, 2003 + Stella X. Yu, Jianbo Shi + `_ - - Multiclass spectral clustering, 2003 - Stella X. Yu, Jianbo Shi - https://www1.icsi.berkeley.edu/~stellayu/publication/doc/2003kwayICCV.pdf + .. [4] :doi:`Toward the Optimal Preconditioned Eigensolver: + Locally Optimal Block Preconditioned Conjugate Gradient Method, 2001 + A. V. Knyazev + SIAM Journal on Scientific Computing 23, no. 2, pp. 517-541. + <10.1137/S1064827500366124>` + + .. [5] :doi:`Simple, direct, and efficient multi-way spectral clustering, 2019 + Anil Damle, Victor Minden, Lexing Ying + <10.1093/imaiai/iay008>` + + Examples + -------- + >>> from sklearn.cluster import SpectralClustering + >>> import numpy as np + >>> X = np.array([[1, 1], [2, 1], [1, 0], + ... [4, 7], [3, 5], [3, 6]]) + >>> clustering = SpectralClustering(n_clusters=2, + ... assign_labels='discretize', + ... random_state=0).fit(X) + >>> clustering.labels_ + array([1, 1, 1, 0, 0, 0]) + >>> clustering + SpectralClustering(assign_labels='discretize', n_clusters=2, + random_state=0) + + For a comparison of Spectral clustering with other clustering algorithms, see + :ref:`sphx_glr_auto_examples_cluster_plot_cluster_comparison.py` """ - @_deprecate_positional_args - def __init__(self, n_clusters=8, *, eigen_solver=None, n_components=None, - random_state=None, n_init=10, gamma=1., affinity='rbf', - n_neighbors=10, eigen_tol=0.0, assign_labels='kmeans', - degree=3, coef0=1, kernel_params=None, n_jobs=None, - verbose=False): + + _parameter_constraints: dict = { + "n_clusters": [Interval(Integral, 1, None, closed="left")], + "eigen_solver": [StrOptions({"arpack", "lobpcg", "amg"}), None], + "n_components": [Interval(Integral, 1, None, closed="left"), None], + "random_state": ["random_state"], + "n_init": [Interval(Integral, 1, None, closed="left")], + "gamma": [Interval(Real, 0, None, closed="left")], + "affinity": [ + callable, + StrOptions( + set(KERNEL_PARAMS) + | {"nearest_neighbors", "precomputed", "precomputed_nearest_neighbors"} + ), + ], + "n_neighbors": [Interval(Integral, 1, None, closed="left")], + "eigen_tol": [ + Interval(Real, 0.0, None, closed="left"), + StrOptions({"auto"}), + ], + "assign_labels": [StrOptions({"kmeans", "discretize", "cluster_qr"})], + "degree": [Interval(Real, 0, None, closed="left")], + "coef0": [Interval(Real, None, None, closed="neither")], + "kernel_params": [dict, None], + "n_jobs": [Integral, None], + "verbose": ["verbose"], + } + + def __init__( + self, + n_clusters=8, + *, + eigen_solver=None, + n_components=None, + random_state=None, + n_init=10, + gamma=1.0, + affinity="rbf", + n_neighbors=10, + eigen_tol="auto", + assign_labels="kmeans", + degree=3, + coef0=1, + kernel_params=None, + n_jobs=None, + verbose=False, + ): self.n_clusters = n_clusters self.eigen_solver = eigen_solver self.n_components = n_components @@ -475,87 +668,122 @@ def __init__(self, n_clusters=8, *, eigen_solver=None, n_components=None, self.n_jobs = n_jobs self.verbose = verbose + @_fit_context(prefer_skip_nested_validation=True) def fit(self, X, y=None): """Perform spectral clustering from features, or affinity matrix. Parameters ---------- - X : {array-like, sparse matrix} of shape (n_samples, n_features), or \ - array-like of shape (n_samples, n_samples) - Training instances to cluster, or similarities / affinities between - instances if ``affinity='precomputed'``. If a sparse matrix is - provided in a format other than ``csr_matrix``, ``csc_matrix``, - or ``coo_matrix``, it will be converted into a sparse - ``csr_matrix``. + X : {array-like, sparse matrix} of shape (n_samples, n_features) or \ + (n_samples, n_samples) + Training instances to cluster, similarities / affinities between + instances if ``affinity='precomputed'``, or distances between + instances if ``affinity='precomputed_nearest_neighbors``. If a + sparse matrix is provided in a format other than ``csr_matrix``, + ``csc_matrix``, or ``coo_matrix``, it will be converted into a + sparse ``csr_matrix``. y : Ignored Not used, present here for API consistency by convention. Returns ------- - self - + self : object + A fitted instance of the estimator. """ - X = self._validate_data(X, accept_sparse=['csr', 'csc', 'coo'], - dtype=np.float64, ensure_min_samples=2) - allow_squared = self.affinity in ["precomputed", - "precomputed_nearest_neighbors"] + X = validate_data( + self, + X, + accept_sparse=["csr", "csc", "coo"], + dtype=np.float64, + ensure_min_samples=2, + ) + allow_squared = self.affinity in [ + "precomputed", + "precomputed_nearest_neighbors", + ] if X.shape[0] == X.shape[1] and not allow_squared: - warnings.warn("The spectral clustering API has changed. ``fit``" - "now constructs an affinity matrix from data. To use" - " a custom affinity matrix, " - "set ``affinity=precomputed``.") - - if self.affinity == 'nearest_neighbors': - connectivity = kneighbors_graph(X, n_neighbors=self.n_neighbors, - include_self=True, - n_jobs=self.n_jobs) + warnings.warn( + "The spectral clustering API has changed. ``fit``" + "now constructs an affinity matrix from data. To use" + " a custom affinity matrix, " + "set ``affinity=precomputed``." + ) + + if self.affinity == "nearest_neighbors": + connectivity = kneighbors_graph( + X, n_neighbors=self.n_neighbors, include_self=True, n_jobs=self.n_jobs + ) self.affinity_matrix_ = 0.5 * (connectivity + connectivity.T) - elif self.affinity == 'precomputed_nearest_neighbors': - estimator = NearestNeighbors(n_neighbors=self.n_neighbors, - n_jobs=self.n_jobs, - metric="precomputed").fit(X) - connectivity = estimator.kneighbors_graph(X=X, mode='connectivity') + elif self.affinity == "precomputed_nearest_neighbors": + estimator = NearestNeighbors( + n_neighbors=self.n_neighbors, n_jobs=self.n_jobs, metric="precomputed" + ).fit(X) + connectivity = estimator.kneighbors_graph(X=X, mode="connectivity") self.affinity_matrix_ = 0.5 * (connectivity + connectivity.T) - elif self.affinity == 'precomputed': + elif self.affinity == "precomputed": self.affinity_matrix_ = X else: params = self.kernel_params if params is None: params = {} if not callable(self.affinity): - params['gamma'] = self.gamma - params['degree'] = self.degree - params['coef0'] = self.coef0 - self.affinity_matrix_ = pairwise_kernels(X, metric=self.affinity, - filter_params=True, - **params) + params["gamma"] = self.gamma + params["degree"] = self.degree + params["coef0"] = self.coef0 + self.affinity_matrix_ = pairwise_kernels( + X, metric=self.affinity, filter_params=True, **params + ) random_state = check_random_state(self.random_state) - self.labels_ = spectral_clustering(self.affinity_matrix_, - n_clusters=self.n_clusters, - n_components=self.n_components, - eigen_solver=self.eigen_solver, - random_state=random_state, - n_init=self.n_init, - eigen_tol=self.eigen_tol, - assign_labels=self.assign_labels, - verbose=self.verbose) + n_components = ( + self.n_clusters if self.n_components is None else self.n_components + ) + # We now obtain the real valued solution matrix to the + # relaxed Ncut problem, solving the eigenvalue problem + # L_sym x = lambda x and recovering u = D^-1/2 x. + # The first eigenvector is constant only for fully connected graphs + # and should be kept for spectral clustering (drop_first = False) + # See spectral_embedding documentation. + maps = _spectral_embedding( + self.affinity_matrix_, + n_components=n_components, + eigen_solver=self.eigen_solver, + random_state=random_state, + eigen_tol=self.eigen_tol, + drop_first=False, + ) + if self.verbose: + print(f"Computing label assignment using {self.assign_labels}") + + if self.assign_labels == "kmeans": + _, self.labels_, _ = k_means( + maps, + self.n_clusters, + random_state=random_state, + n_init=self.n_init, + verbose=self.verbose, + ) + elif self.assign_labels == "cluster_qr": + self.labels_ = cluster_qr(maps) + else: + self.labels_ = discretize(maps, random_state=random_state) + return self def fit_predict(self, X, y=None): - """Perform spectral clustering from features, or affinity matrix, - and return cluster labels. + """Perform spectral clustering on `X` and return cluster labels. Parameters ---------- - X : {array-like, sparse matrix} of shape (n_samples, n_features), or \ - array-like of shape (n_samples, n_samples) - Training instances to cluster, or similarities / affinities between - instances if ``affinity='precomputed'``. If a sparse matrix is - provided in a format other than ``csr_matrix``, ``csc_matrix``, - or ``coo_matrix``, it will be converted into a sparse - ``csr_matrix``. + X : {array-like, sparse matrix} of shape (n_samples, n_features) or \ + (n_samples, n_samples) + Training instances to cluster, similarities / affinities between + instances if ``affinity='precomputed'``, or distances between + instances if ``affinity='precomputed_nearest_neighbors``. If a + sparse matrix is provided in a format other than ``csr_matrix``, + ``csc_matrix``, or ``coo_matrix``, it will be converted into a + sparse ``csr_matrix``. y : Ignored Not used, present here for API consistency by convention. @@ -567,15 +795,11 @@ def fit_predict(self, X, y=None): """ return super().fit_predict(X, y) - def _more_tags(self): - return {'pairwise': self.affinity in ["precomputed", - "precomputed_nearest_neighbors"]} - - # TODO: Remove in 1.1 - # mypy error: Decorated property not supported - @deprecated("Attribute _pairwise was deprecated in " # type: ignore - "version 0.24 and will be removed in 1.1 (renaming of 0.26).") - @property - def _pairwise(self): - return self.affinity in ["precomputed", - "precomputed_nearest_neighbors"] + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.sparse = True + tags.input_tags.pairwise = self.affinity in [ + "precomputed", + "precomputed_nearest_neighbors", + ] + return tags diff --git a/sklearn/cluster/meson.build b/sklearn/cluster/meson.build new file mode 100644 index 0000000000000..6c11619f3ca55 --- /dev/null +++ b/sklearn/cluster/meson.build @@ -0,0 +1,26 @@ +cluster_extension_metadata = { + '_dbscan_inner': + {'sources': [cython_gen_cpp.process('_dbscan_inner.pyx')]}, + '_hierarchical_fast': + {'sources': [cython_gen_cpp.process('_hierarchical_fast.pyx'), metrics_cython_tree]}, + '_k_means_common': + {'sources': [cython_gen.process('_k_means_common.pyx')], 'dependencies': [openmp_dep]}, + '_k_means_lloyd': + {'sources': [cython_gen.process('_k_means_lloyd.pyx')], 'dependencies': [openmp_dep]}, + '_k_means_elkan': + {'sources': [cython_gen.process('_k_means_elkan.pyx')], 'dependencies': [openmp_dep]}, + '_k_means_minibatch': + {'sources': [cython_gen.process('_k_means_minibatch.pyx')], 'dependencies': [openmp_dep]}, +} + +foreach ext_name, ext_dict : cluster_extension_metadata + py.extension_module( + ext_name, + [ext_dict.get('sources'), utils_cython_tree], + dependencies: [np_dep] + ext_dict.get('dependencies', []), + subdir: 'sklearn/cluster', + install: true + ) +endforeach + +subdir('_hdbscan') diff --git a/sklearn/cluster/setup.py b/sklearn/cluster/setup.py deleted file mode 100644 index 48ed25c5c0eaf..0000000000000 --- a/sklearn/cluster/setup.py +++ /dev/null @@ -1,50 +0,0 @@ -# Author: Alexandre Gramfort -# License: BSD 3 clause -import os - -import numpy - - -def configuration(parent_package='', top_path=None): - from numpy.distutils.misc_util import Configuration - - libraries = [] - if os.name == 'posix': - libraries.append('m') - - config = Configuration('cluster', parent_package, top_path) - - config.add_extension('_dbscan_inner', - sources=['_dbscan_inner.pyx'], - include_dirs=[numpy.get_include()], - language="c++") - - config.add_extension('_hierarchical_fast', - sources=['_hierarchical_fast.pyx'], - language="c++", - include_dirs=[numpy.get_include()], - libraries=libraries) - - config.add_extension('_k_means_fast', - sources=['_k_means_fast.pyx'], - include_dirs=[numpy.get_include()], - libraries=libraries) - - config.add_extension('_k_means_lloyd', - sources=['_k_means_lloyd.pyx'], - include_dirs=[numpy.get_include()], - libraries=libraries) - - config.add_extension('_k_means_elkan', - sources=['_k_means_elkan.pyx'], - include_dirs=[numpy.get_include()], - libraries=libraries) - - config.add_subpackage('tests') - - return config - - -if __name__ == '__main__': - from numpy.distutils.core import setup - setup(**configuration(top_path='').todict()) diff --git a/sklearn/cluster/tests/common.py b/sklearn/cluster/tests/common.py index 957ebcf186596..b1fe047fe230a 100644 --- a/sklearn/cluster/tests/common.py +++ b/sklearn/cluster/tests/common.py @@ -5,24 +5,33 @@ import numpy as np - ############################################################################### # Generate sample data -def generate_clustered_data(seed=0, n_clusters=3, n_features=2, - n_samples_per_cluster=20, std=.4): + +def generate_clustered_data( + seed=0, n_clusters=3, n_features=2, n_samples_per_cluster=20, std=0.4 +): prng = np.random.RandomState(seed) # the data is voluntary shifted away from zero to check clustering # algorithm robustness with regards to non centered data - means = np.array([[1, 1, 1, 0], - [-1, -1, 0, 1], - [1, -1, 1, 1], - [-1, 1, 1, 0], - ]) + 10 + means = ( + np.array( + [ + [1, 1, 1, 0], + [-1, -1, 0, 1], + [1, -1, 1, 1], + [-1, 1, 1, 0], + ] + ) + + 10 + ) X = np.empty((0, n_features)) for i in range(n_clusters): - X = np.r_[X, means[i][:n_features] - + std * prng.randn(n_samples_per_cluster, n_features)] + X = np.r_[ + X, + means[i][:n_features] + std * prng.randn(n_samples_per_cluster, n_features), + ] return X diff --git a/sklearn/cluster/tests/test_affinity_propagation.py b/sklearn/cluster/tests/test_affinity_propagation.py index 51b4fd425349e..c3138e59111ed 100644 --- a/sklearn/cluster/tests/test_affinity_propagation.py +++ b/sklearn/cluster/tests/test_affinity_propagation.py @@ -3,46 +3,60 @@ """ +import warnings + import numpy as np import pytest -from scipy.sparse import csr_matrix - -from sklearn.exceptions import ConvergenceWarning -from sklearn.utils._testing import assert_array_equal -from sklearn.cluster import AffinityPropagation -from sklearn.cluster._affinity_propagation import ( - _equal_similarities_and_preferences -) -from sklearn.cluster import affinity_propagation +from sklearn.cluster import AffinityPropagation, affinity_propagation +from sklearn.cluster._affinity_propagation import _equal_similarities_and_preferences from sklearn.datasets import make_blobs +from sklearn.exceptions import ConvergenceWarning, NotFittedError from sklearn.metrics import euclidean_distances +from sklearn.utils._testing import assert_allclose, assert_array_equal +from sklearn.utils.fixes import CSR_CONTAINERS n_clusters = 3 centers = np.array([[1, 1], [-1, -1], [1, -1]]) + 10 -X, _ = make_blobs(n_samples=60, n_features=2, centers=centers, - cluster_std=0.4, shuffle=True, random_state=0) +X, _ = make_blobs( + n_samples=60, + n_features=2, + centers=centers, + cluster_std=0.4, + shuffle=True, + random_state=0, +) +# TODO: AffinityPropagation must preserve dtype for its fitted attributes +# and test must be created accordingly to this new behavior. +# For more details, see: https://github.com/scikit-learn/scikit-learn/issues/11000 -def test_affinity_propagation(): - # Affinity Propagation algorithm - # Compute similarities - S = -euclidean_distances(X, squared=True) + +def test_affinity_propagation(global_random_seed, global_dtype): + """Test consistency of the affinity propagations.""" + S = -euclidean_distances(X.astype(global_dtype, copy=False), squared=True) preference = np.median(S) * 10 - # Compute Affinity Propagation cluster_centers_indices, labels = affinity_propagation( - S, preference=preference, random_state=39) + S, preference=preference, random_state=global_random_seed + ) n_clusters_ = len(cluster_centers_indices) assert n_clusters == n_clusters_ - af = AffinityPropagation(preference=preference, affinity="precomputed", - random_state=28) + +def test_affinity_propagation_precomputed(): + """Check equality of precomputed affinity matrix to internally computed affinity + matrix. + """ + S = -euclidean_distances(X, squared=True) + preference = np.median(S) * 10 + af = AffinityPropagation( + preference=preference, affinity="precomputed", random_state=28 + ) labels_precomputed = af.fit(S).labels_ - af = AffinityPropagation(preference=preference, verbose=True, - random_state=37) + af = AffinityPropagation(preference=preference, verbose=True, random_state=37) labels = af.fit(X).labels_ assert_array_equal(labels, labels_precomputed) @@ -53,29 +67,56 @@ def test_affinity_propagation(): assert np.unique(labels).size == n_clusters_ assert n_clusters == n_clusters_ - # Test also with no copy - _, labels_no_copy = affinity_propagation(S, preference=preference, - copy=False, random_state=74) + +def test_affinity_propagation_no_copy(): + """Check behaviour of not copying the input data.""" + S = -euclidean_distances(X, squared=True) + S_original = S.copy() + preference = np.median(S) * 10 + assert not np.allclose(S.diagonal(), preference) + + # with copy=True S should not be modified + affinity_propagation(S, preference=preference, copy=True, random_state=0) + assert_allclose(S, S_original) + assert not np.allclose(S.diagonal(), preference) + assert_allclose(S.diagonal(), np.zeros(S.shape[0])) + + # with copy=False S will be modified inplace + affinity_propagation(S, preference=preference, copy=False, random_state=0) + assert_allclose(S.diagonal(), preference) + + # test that copy=True and copy=False lead to the same result + S = S_original.copy() + af = AffinityPropagation(preference=preference, verbose=True, random_state=0) + + labels = af.fit(X).labels_ + _, labels_no_copy = affinity_propagation( + S, preference=preference, copy=False, random_state=74 + ) assert_array_equal(labels, labels_no_copy) - # Test input validation - with pytest.raises(ValueError): + +def test_affinity_propagation_affinity_shape(): + """Check the shape of the affinity matrix when using `affinity_propagation.""" + S = -euclidean_distances(X, squared=True) + err_msg = "The matrix of similarities must be a square array" + with pytest.raises(ValueError, match=err_msg): affinity_propagation(S[:, :-1]) - with pytest.raises(ValueError): - affinity_propagation(S, damping=0) - af = AffinityPropagation(affinity="unknown", random_state=78) - with pytest.raises(ValueError): - af.fit(X) - af_2 = AffinityPropagation(affinity='precomputed', random_state=21) - with pytest.raises(TypeError): - af_2.fit(csr_matrix((3, 3))) -def test_affinity_propagation_predict(): +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_affinity_propagation_precomputed_with_sparse_input(csr_container): + err_msg = "Sparse data was passed for X, but dense data is required" + with pytest.raises(TypeError, match=err_msg): + AffinityPropagation(affinity="precomputed").fit(csr_container((3, 3))) + + +def test_affinity_propagation_predict(global_random_seed, global_dtype): # Test AffinityPropagation.predict - af = AffinityPropagation(affinity="euclidean", random_state=63) - labels = af.fit_predict(X) - labels2 = af.predict(X) + af = AffinityPropagation(affinity="euclidean", random_state=global_random_seed) + X_ = X.astype(global_dtype, copy=False) + labels = af.fit_predict(X_) + labels2 = af.predict(X_) assert_array_equal(labels, labels2) @@ -83,40 +124,39 @@ def test_affinity_propagation_predict_error(): # Test exception in AffinityPropagation.predict # Not fitted. af = AffinityPropagation(affinity="euclidean") - with pytest.raises(ValueError): + with pytest.raises(NotFittedError): af.predict(X) # Predict not supported when affinity="precomputed". S = np.dot(X, X.T) af = AffinityPropagation(affinity="precomputed", random_state=57) af.fit(S) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="expecting 60 features as input"): af.predict(X) -def test_affinity_propagation_fit_non_convergence(): +def test_affinity_propagation_fit_non_convergence(global_dtype): # In case of non-convergence of affinity_propagation(), the cluster # centers should be an empty array and training samples should be labelled # as noise (-1) - X = np.array([[0, 0], [1, 1], [-2, -2]]) + X = np.array([[0, 0], [1, 1], [-2, -2]], dtype=global_dtype) # Force non-convergence by allowing only a single iteration af = AffinityPropagation(preference=-10, max_iter=1, random_state=82) with pytest.warns(ConvergenceWarning): af.fit(X) - assert_array_equal(np.empty((0, 2)), af.cluster_centers_) + assert_allclose(np.empty((0, 2)), af.cluster_centers_) assert_array_equal(np.array([-1, -1, -1]), af.labels_) -def test_affinity_propagation_equal_mutual_similarities(): - X = np.array([[-1, 1], [1, -1]]) +def test_affinity_propagation_equal_mutual_similarities(global_dtype): + X = np.array([[-1, 1], [1, -1]], dtype=global_dtype) S = -euclidean_distances(X, squared=True) # setting preference > similarity with pytest.warns(UserWarning, match="mutually equal"): - cluster_center_indices, labels = affinity_propagation( - S, preference=0) + cluster_center_indices, labels = affinity_propagation(S, preference=0) # expect every sample to become an exemplar assert_array_equal([0, 1], cluster_center_indices) @@ -124,33 +164,32 @@ def test_affinity_propagation_equal_mutual_similarities(): # setting preference < similarity with pytest.warns(UserWarning, match="mutually equal"): - cluster_center_indices, labels = affinity_propagation( - S, preference=-10) + cluster_center_indices, labels = affinity_propagation(S, preference=-10) # expect one cluster, with arbitrary (first) sample as exemplar assert_array_equal([0], cluster_center_indices) assert_array_equal([0, 0], labels) # setting different preferences - with pytest.warns(None) as record: + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) cluster_center_indices, labels = affinity_propagation( - S, preference=[-20, -10], random_state=37) - assert not len(record) + S, preference=[-20, -10], random_state=37 + ) # expect one cluster, with highest-preference sample as exemplar assert_array_equal([1], cluster_center_indices) assert_array_equal([0, 0], labels) -def test_affinity_propagation_predict_non_convergence(): +def test_affinity_propagation_predict_non_convergence(global_dtype): # In case of non-convergence of affinity_propagation(), the cluster # centers should be an empty array - X = np.array([[0, 0], [1, 1], [-2, -2]]) + X = np.array([[0, 0], [1, 1], [-2, -2]], dtype=global_dtype) # Force non-convergence by allowing only a single iteration with pytest.warns(ConvergenceWarning): - af = AffinityPropagation(preference=-10, - max_iter=1, random_state=75).fit(X) + af = AffinityPropagation(preference=-10, max_iter=1, random_state=75).fit(X) # At prediction time, consider new samples as noise since there are no # clusters @@ -160,18 +199,24 @@ def test_affinity_propagation_predict_non_convergence(): assert_array_equal(np.array([-1, -1, -1]), y) -def test_affinity_propagation_non_convergence_regressiontest(): - X = np.array([[1, 0, 0, 0, 0, 0], - [0, 1, 1, 1, 0, 0], - [0, 0, 1, 0, 0, 1]]) - af = AffinityPropagation(affinity='euclidean', - max_iter=2, random_state=34).fit(X) - assert_array_equal(np.array([-1, -1, -1]), af.labels_) +def test_affinity_propagation_non_convergence_regressiontest(global_dtype): + X = np.array( + [[1, 0, 0, 0, 0, 0], [0, 1, 1, 1, 0, 0], [0, 0, 1, 0, 0, 1]], dtype=global_dtype + ) + af = AffinityPropagation(affinity="euclidean", max_iter=2, random_state=34) + msg = ( + "Affinity propagation did not converge, this model may return degenerate" + " cluster centers and labels." + ) + with pytest.warns(ConvergenceWarning, match=msg): + af.fit(X) + + assert_array_equal(np.array([0, 0, 0]), af.labels_) -def test_equal_similarities_and_preferences(): +def test_equal_similarities_and_preferences(global_dtype): # Unequal distances - X = np.array([[0, 0], [1, 1], [-2, -2]]) + X = np.array([[0, 0], [1, 1], [-2, -2]], dtype=global_dtype) S = -euclidean_distances(X, squared=True) assert not _equal_similarities_and_preferences(S, np.array(0)) @@ -179,7 +224,7 @@ def test_equal_similarities_and_preferences(): assert not _equal_similarities_and_preferences(S, np.array([0, 1])) # Equal distances - X = np.array([[0, 0], [1, 1]]) + X = np.array([[0, 0], [1, 1]], dtype=global_dtype) S = -euclidean_distances(X, squared=True) # Different preferences @@ -191,11 +236,13 @@ def test_equal_similarities_and_preferences(): def test_affinity_propagation_random_state(): - # Significance of random_state parameter - # Generate sample data + """Check that different random states lead to different initialisations + by looking at the center locations after two iterations. + """ centers = [[1, 1], [-1, -1], [1, -1]] - X, labels_true = make_blobs(n_samples=300, centers=centers, - cluster_std=0.5, random_state=0) + X, labels_true = make_blobs( + n_samples=300, centers=centers, cluster_std=0.5, random_state=0 + ) # random_state = 0 ap = AffinityPropagation(convergence_iter=1, max_iter=2, random_state=0) ap.fit(X) @@ -205,50 +252,70 @@ def test_affinity_propagation_random_state(): ap = AffinityPropagation(convergence_iter=1, max_iter=2, random_state=76) ap.fit(X) centers76 = ap.cluster_centers_ - + # check that the centers have not yet converged to the same solution assert np.mean((centers0 - centers76) ** 2) > 1 -# FIXME: to be removed in 1.0 -def test_affinity_propagation_random_state_warning(): - # test that a warning is raised when random_state is not defined. - X = np.array([[0, 0], [1, 1], [-2, -2]]) - match = "'random_state' has been introduced in 0.23." - with pytest.warns(FutureWarning, match=match): - AffinityPropagation().fit(X) - -@pytest.mark.parametrize('centers', [csr_matrix(np.zeros((1, 10))), - np.zeros((1, 10))]) -def test_affinity_propagation_convergence_warning_dense_sparse(centers): - """Non-regression, see #13334""" +@pytest.mark.parametrize("container", CSR_CONTAINERS + [np.array]) +def test_affinity_propagation_convergence_warning_dense_sparse(container, global_dtype): + """ + Check that having sparse or dense `centers` format should not + influence the convergence. + Non-regression test for gh-13334. + """ + centers = container(np.zeros((1, 10))) rng = np.random.RandomState(42) - X = rng.rand(40, 10) + X = rng.rand(40, 10).astype(global_dtype, copy=False) y = (4 * rng.rand(40)).astype(int) ap = AffinityPropagation(random_state=46) ap.fit(X, y) ap.cluster_centers_ = centers - with pytest.warns(None) as record: - assert_array_equal(ap.predict(X), - np.zeros(X.shape[0], dtype=int)) - assert len(record) == 0 + with warnings.catch_warnings(): + warnings.simplefilter("error", ConvergenceWarning) + assert_array_equal(ap.predict(X), np.zeros(X.shape[0], dtype=int)) -def test_affinity_propagation_float32(): +# FIXME; this test is broken with different random states, needs to be revisited +def test_correct_clusters(global_dtype): # Test to fix incorrect clusters due to dtype change # (non-regression test for issue #10832) - X = np.array([[1, 0, 0, 0], - [0, 1, 1, 0], - [0, 1, 1, 0], - [0, 0, 0, 1]], dtype='float32') - afp = AffinityPropagation(preference=1, affinity='precomputed', - random_state=0).fit(X) + X = np.array( + [[1, 0, 0, 0], [0, 1, 1, 0], [0, 1, 1, 0], [0, 0, 0, 1]], dtype=global_dtype + ) + afp = AffinityPropagation(preference=1, affinity="precomputed", random_state=0).fit( + X + ) expected = np.array([0, 1, 1, 2]) assert_array_equal(afp.labels_, expected) -# TODO: Remove in 1.1 -def test_affinity_propagation_pairwise_is_deprecated(): - afp = AffinityPropagation(affinity='precomputed') - msg = r"Attribute _pairwise was deprecated in version 0\.24" - with pytest.warns(FutureWarning, match=msg): - afp._pairwise +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_sparse_input_for_predict(csr_container): + # Test to make sure sparse inputs are accepted for predict + # (non-regression test for issue #20049) + af = AffinityPropagation(affinity="euclidean", random_state=42) + af.fit(X) + labels = af.predict(csr_container((2, 2))) + assert_array_equal(labels, (2, 2)) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_sparse_input_for_fit_predict(csr_container): + # Test to make sure sparse inputs are accepted for fit_predict + # (non-regression test for issue #20049) + af = AffinityPropagation(affinity="euclidean", random_state=42) + rng = np.random.RandomState(42) + X = csr_container(rng.randint(0, 2, size=(5, 5))) + labels = af.fit_predict(X) + assert_array_equal(labels, (0, 1, 1, 2, 3)) + + +def test_affinity_propagation_equal_points(): + """Make sure we do not assign multiple clusters to equal points. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/pull/20043 + """ + X = np.zeros((8, 1)) + af = AffinityPropagation(affinity="euclidean", damping=0.5, random_state=42).fit(X) + assert np.all(af.labels_ == 0) diff --git a/sklearn/cluster/tests/test_bicluster.py b/sklearn/cluster/tests/test_bicluster.py index 97ca3db0201b6..ebc845a7bf262 100644 --- a/sklearn/cluster/tests/test_bicluster.py +++ b/sklearn/cluster/tests/test_bicluster.py @@ -2,25 +2,24 @@ import numpy as np import pytest -from scipy.sparse import csr_matrix, issparse - -from sklearn.model_selection import ParameterGrid - -from sklearn.utils._testing import assert_almost_equal -from sklearn.utils._testing import assert_array_equal -from sklearn.utils._testing import assert_array_almost_equal +from scipy.sparse import issparse from sklearn.base import BaseEstimator, BiclusterMixin - -from sklearn.cluster import SpectralCoclustering -from sklearn.cluster import SpectralBiclustering -from sklearn.cluster._bicluster import _scale_normalize -from sklearn.cluster._bicluster import _bistochastic_normalize -from sklearn.cluster._bicluster import _log_normalize - -from sklearn.metrics import (consensus_score, v_measure_score) - +from sklearn.cluster import SpectralBiclustering, SpectralCoclustering +from sklearn.cluster._bicluster import ( + _bistochastic_normalize, + _log_normalize, + _scale_normalize, +) from sklearn.datasets import make_biclusters, make_checkerboard +from sklearn.metrics import consensus_score, v_measure_score +from sklearn.model_selection import ParameterGrid +from sklearn.utils._testing import ( + assert_almost_equal, + assert_array_almost_equal, + assert_array_equal, +) +from sklearn.utils.fixes import CSR_CONTAINERS class MockBiclustering(BiclusterMixin, BaseEstimator): @@ -30,21 +29,22 @@ def __init__(self): def get_indices(self, i): # Overridden to reproduce old get_submatrix test. - return (np.where([True, True, False, False, True])[0], - np.where([False, False, True, True])[0]) + return ( + np.where([True, True, False, False, True])[0], + np.where([False, False, True, True])[0], + ) -def test_get_submatrix(): +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_get_submatrix(csr_container): data = np.arange(20).reshape(5, 4) model = MockBiclustering() - for X in (data, csr_matrix(data), data.tolist()): + for X in (data, csr_container(data), data.tolist()): submatrix = model.get_submatrix(0, X) if issparse(submatrix): submatrix = submatrix.toarray() - assert_array_equal(submatrix, [[2, 3], - [6, 7], - [18, 19]]) + assert_array_equal(submatrix, [[2, 3], [6, 7], [18, 19]]) submatrix[:] = -1 if issparse(X): X = X.toarray() @@ -60,57 +60,62 @@ def _test_shape_indices(model): assert len(j_ind) == n -def test_spectral_coclustering(): +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_spectral_coclustering(global_random_seed, csr_container): # Test Dhillon's Spectral CoClustering on a simple problem. - param_grid = {'svd_method': ['randomized', 'arpack'], - 'n_svd_vecs': [None, 20], - 'mini_batch': [False, True], - 'init': ['k-means++'], - 'n_init': [10]} - random_state = 0 - S, rows, cols = make_biclusters((30, 30), 3, noise=0.5, - random_state=random_state) + param_grid = { + "svd_method": ["randomized", "arpack"], + "n_svd_vecs": [None, 20], + "mini_batch": [False, True], + "init": ["k-means++"], + "n_init": [10], + } + S, rows, cols = make_biclusters( + (30, 30), 3, noise=0.1, random_state=global_random_seed + ) S -= S.min() # needs to be nonnegative before making it sparse S = np.where(S < 1, 0, S) # threshold some values - for mat in (S, csr_matrix(S)): + for mat in (S, csr_container(S)): for kwargs in ParameterGrid(param_grid): - model = SpectralCoclustering(n_clusters=3, - random_state=random_state, - **kwargs) + model = SpectralCoclustering( + n_clusters=3, random_state=global_random_seed, **kwargs + ) model.fit(mat) assert model.rows_.shape == (3, 30) assert_array_equal(model.rows_.sum(axis=0), np.ones(30)) assert_array_equal(model.columns_.sum(axis=0), np.ones(30)) - assert consensus_score(model.biclusters_, - (rows, cols)) == 1 + assert consensus_score(model.biclusters_, (rows, cols)) == 1 _test_shape_indices(model) -def test_spectral_biclustering(): +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_spectral_biclustering(global_random_seed, csr_container): # Test Kluger methods on a checkerboard dataset. - S, rows, cols = make_checkerboard((30, 30), 3, noise=0.5, - random_state=0) - - non_default_params = {'method': ['scale', 'log'], - 'svd_method': ['arpack'], - 'n_svd_vecs': [20], - 'mini_batch': [True]} - - for mat in (S, csr_matrix(S)): + S, rows, cols = make_checkerboard( + (30, 30), 3, noise=0.5, random_state=global_random_seed + ) + + non_default_params = { + "method": ["scale", "log"], + "svd_method": ["arpack"], + "n_svd_vecs": [20], + "mini_batch": [True], + } + + for mat in (S, csr_container(S)): for param_name, param_values in non_default_params.items(): for param_value in param_values: - model = SpectralBiclustering( n_clusters=3, n_init=3, - init='k-means++', - random_state=0, + init="k-means++", + random_state=global_random_seed, ) model.set_params(**dict([(param_name, param_value)])) - if issparse(mat) and model.get_params().get('method') == 'log': + if issparse(mat) and model.get_params().get("method") == "log": # cannot take log of sparse matrix with pytest.raises(ValueError): model.fit(mat) @@ -120,12 +125,9 @@ def test_spectral_biclustering(): assert model.rows_.shape == (9, 30) assert model.columns_.shape == (9, 30) - assert_array_equal(model.rows_.sum(axis=0), - np.repeat(3, 30)) - assert_array_equal(model.columns_.sum(axis=0), - np.repeat(3, 30)) - assert consensus_score(model.biclusters_, - (rows, cols)) == 1 + assert_array_equal(model.rows_.sum(axis=0), np.repeat(3, 30)) + assert_array_equal(model.columns_.sum(axis=0), np.repeat(3, 30)) + assert consensus_score(model.biclusters_, (rows, cols)) == 1 _test_shape_indices(model) @@ -137,141 +139,126 @@ def _do_scale_test(scaled): if issparse(scaled): row_sum = np.asarray(row_sum).squeeze() col_sum = np.asarray(col_sum).squeeze() - assert_array_almost_equal(row_sum, np.tile(row_sum.mean(), 100), - decimal=1) - assert_array_almost_equal(col_sum, np.tile(col_sum.mean(), 100), - decimal=1) + assert_array_almost_equal(row_sum, np.tile(row_sum.mean(), 100), decimal=1) + assert_array_almost_equal(col_sum, np.tile(col_sum.mean(), 100), decimal=1) def _do_bistochastic_test(scaled): """Check that rows and columns sum to the same constant.""" _do_scale_test(scaled) - assert_almost_equal(scaled.sum(axis=0).mean(), - scaled.sum(axis=1).mean(), - decimal=1) + assert_almost_equal(scaled.sum(axis=0).mean(), scaled.sum(axis=1).mean(), decimal=1) -def test_scale_normalize(): - generator = np.random.RandomState(0) +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_scale_normalize(global_random_seed, csr_container): + generator = np.random.RandomState(global_random_seed) X = generator.rand(100, 100) - for mat in (X, csr_matrix(X)): + for mat in (X, csr_container(X)): scaled, _, _ = _scale_normalize(mat) _do_scale_test(scaled) if issparse(mat): assert issparse(scaled) -def test_bistochastic_normalize(): - generator = np.random.RandomState(0) +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_bistochastic_normalize(global_random_seed, csr_container): + generator = np.random.RandomState(global_random_seed) X = generator.rand(100, 100) - for mat in (X, csr_matrix(X)): + for mat in (X, csr_container(X)): scaled = _bistochastic_normalize(mat) _do_bistochastic_test(scaled) if issparse(mat): assert issparse(scaled) -def test_log_normalize(): +def test_log_normalize(global_random_seed): # adding any constant to a log-scaled matrix should make it # bistochastic - generator = np.random.RandomState(0) + generator = np.random.RandomState(global_random_seed) mat = generator.rand(100, 100) scaled = _log_normalize(mat) + 1 _do_bistochastic_test(scaled) -def test_fit_best_piecewise(): - model = SpectralBiclustering(random_state=0) - vectors = np.array([[0, 0, 0, 1, 1, 1], - [2, 2, 2, 3, 3, 3], - [0, 1, 2, 3, 4, 5]]) +def test_fit_best_piecewise(global_random_seed): + model = SpectralBiclustering(random_state=global_random_seed) + vectors = np.array([[0, 0, 0, 1, 1, 1], [2, 2, 2, 3, 3, 3], [0, 1, 2, 3, 4, 5]]) best = model._fit_best_piecewise(vectors, n_best=2, n_clusters=2) assert_array_equal(best, vectors[:2]) -def test_project_and_cluster(): - model = SpectralBiclustering(random_state=0) - data = np.array([[1, 1, 1], - [1, 1, 1], - [3, 6, 3], - [3, 6, 3]]) - vectors = np.array([[1, 0], - [0, 1], - [0, 0]]) - for mat in (data, csr_matrix(data)): - labels = model._project_and_cluster(mat, vectors, - n_clusters=2) +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_project_and_cluster(global_random_seed, csr_container): + model = SpectralBiclustering(random_state=global_random_seed) + data = np.array([[1, 1, 1], [1, 1, 1], [3, 6, 3], [3, 6, 3]]) + vectors = np.array([[1, 0], [0, 1], [0, 0]]) + for mat in (data, csr_container(data)): + labels = model._project_and_cluster(mat, vectors, n_clusters=2) assert_almost_equal(v_measure_score(labels, [0, 0, 1, 1]), 1.0) -def test_perfect_checkerboard(): +def test_perfect_checkerboard(global_random_seed): # XXX Previously failed on build bot (not reproducible) - model = SpectralBiclustering(3, svd_method="arpack", random_state=0) + model = SpectralBiclustering( + 3, svd_method="arpack", random_state=global_random_seed + ) - S, rows, cols = make_checkerboard((30, 30), 3, noise=0, - random_state=0) + S, rows, cols = make_checkerboard( + (30, 30), 3, noise=0, random_state=global_random_seed + ) model.fit(S) - assert consensus_score(model.biclusters_, - (rows, cols)) == 1 + assert consensus_score(model.biclusters_, (rows, cols)) == 1 - S, rows, cols = make_checkerboard((40, 30), 3, noise=0, - random_state=0) + S, rows, cols = make_checkerboard( + (40, 30), 3, noise=0, random_state=global_random_seed + ) model.fit(S) - assert consensus_score(model.biclusters_, - (rows, cols)) == 1 + assert consensus_score(model.biclusters_, (rows, cols)) == 1 - S, rows, cols = make_checkerboard((30, 40), 3, noise=0, - random_state=0) + S, rows, cols = make_checkerboard( + (30, 40), 3, noise=0, random_state=global_random_seed + ) model.fit(S) - assert consensus_score(model.biclusters_, - (rows, cols)) == 1 + assert consensus_score(model.biclusters_, (rows, cols)) == 1 @pytest.mark.parametrize( - "args", - [{'n_clusters': (3, 3, 3)}, - {'n_clusters': 'abc'}, - {'n_clusters': (3, 'abc')}, - {'method': 'unknown'}, - {'n_components': 0}, - {'n_best': 0}, - {'svd_method': 'unknown'}, - {'n_components': 3, 'n_best': 4}] + "params, type_err, err_msg", + [ + ( + {"n_clusters": 6}, + ValueError, + "n_clusters should be <= n_samples=5", + ), + ( + {"n_clusters": (3, 3, 3)}, + ValueError, + "Incorrect parameter n_clusters", + ), + ( + {"n_clusters": (3, 6)}, + ValueError, + "Incorrect parameter n_clusters", + ), + ( + {"n_components": 3, "n_best": 4}, + ValueError, + "n_best=4 must be <= n_components=3", + ), + ], ) -def test_errors(args): +def test_spectralbiclustering_parameter_validation(params, type_err, err_msg): + """Check parameters validation in `SpectralBiClustering`""" data = np.arange(25).reshape((5, 5)) - - model = SpectralBiclustering(**args) - with pytest.raises(ValueError): - model.fit(data) - - -def test_wrong_shape(): - model = SpectralBiclustering() - data = np.arange(27).reshape((3, 3, 3)) - with pytest.raises(ValueError): + model = SpectralBiclustering(**params) + with pytest.raises(type_err, match=err_msg): model.fit(data) -@pytest.mark.parametrize('est', - (SpectralBiclustering(), SpectralCoclustering())) +@pytest.mark.parametrize("est", (SpectralBiclustering(), SpectralCoclustering())) def test_n_features_in_(est): - X, _, _ = make_biclusters((3, 3), 3, random_state=0) - assert not hasattr(est, 'n_features_in_') + assert not hasattr(est, "n_features_in_") est.fit(X) assert est.n_features_in_ == 3 - - -@pytest.mark.parametrize("klass", [SpectralBiclustering, SpectralCoclustering]) -@pytest.mark.parametrize("n_jobs", [None, 1]) -def test_n_jobs_deprecated(klass, n_jobs): - # FIXME: remove in 1.0 - depr_msg = ("'n_jobs' was deprecated in version 0.23 and will be removed " - "in 1.0") - S, _, _ = make_biclusters((30, 30), 3, noise=0.5, random_state=0) - est = klass(random_state=0, n_jobs=n_jobs) - - with pytest.warns(FutureWarning, match=depr_msg): - est.fit(S) diff --git a/sklearn/cluster/tests/test_birch.py b/sklearn/cluster/tests/test_birch.py index e199c897f97ef..bc87934adaecd 100644 --- a/sklearn/cluster/tests/test_birch.py +++ b/sklearn/cluster/tests/test_birch.py @@ -2,45 +2,42 @@ Tests for the birch clustering algorithm. """ -from scipy import sparse import numpy as np import pytest +from sklearn.cluster import AgglomerativeClustering, Birch from sklearn.cluster.tests.common import generate_clustered_data -from sklearn.cluster import Birch -from sklearn.cluster import AgglomerativeClustering from sklearn.datasets import make_blobs from sklearn.exceptions import ConvergenceWarning -from sklearn.linear_model import ElasticNet from sklearn.metrics import pairwise_distances_argmin, v_measure_score +from sklearn.utils._testing import assert_allclose, assert_array_equal +from sklearn.utils.fixes import CSR_CONTAINERS -from sklearn.utils._testing import assert_almost_equal -from sklearn.utils._testing import assert_array_equal -from sklearn.utils._testing import assert_array_almost_equal - -def test_n_samples_leaves_roots(): +def test_n_samples_leaves_roots(global_random_seed, global_dtype): # Sanity check for the number of samples in leaves and roots - X, y = make_blobs(n_samples=10) + X, y = make_blobs(n_samples=10, random_state=global_random_seed) + X = X.astype(global_dtype, copy=False) brc = Birch() brc.fit(X) n_samples_root = sum([sc.n_samples_ for sc in brc.root_.subclusters_]) - n_samples_leaves = sum([sc.n_samples_ for leaf in brc._get_leaves() - for sc in leaf.subclusters_]) + n_samples_leaves = sum( + [sc.n_samples_ for leaf in brc._get_leaves() for sc in leaf.subclusters_] + ) assert n_samples_leaves == X.shape[0] assert n_samples_root == X.shape[0] -def test_partial_fit(): +def test_partial_fit(global_random_seed, global_dtype): # Test that fit is equivalent to calling partial_fit multiple times - X, y = make_blobs(n_samples=100) + X, y = make_blobs(n_samples=100, random_state=global_random_seed) + X = X.astype(global_dtype, copy=False) brc = Birch(n_clusters=3) brc.fit(X) brc_partial = Birch(n_clusters=None) brc_partial.partial_fit(X[:50]) brc_partial.partial_fit(X[50:]) - assert_array_almost_equal(brc_partial.subcluster_centers_, - brc.subcluster_centers_) + assert_allclose(brc_partial.subcluster_centers_, brc.subcluster_centers_) # Test that same global labels are obtained after calling partial_fit # with None @@ -49,27 +46,34 @@ def test_partial_fit(): assert_array_equal(brc_partial.subcluster_labels_, brc.subcluster_labels_) -def test_birch_predict(): +def test_birch_predict(global_random_seed, global_dtype): # Test the predict method predicts the nearest centroid. - rng = np.random.RandomState(0) - X = generate_clustered_data(n_clusters=3, n_features=3, - n_samples_per_cluster=10) + rng = np.random.RandomState(global_random_seed) + X = generate_clustered_data(n_clusters=3, n_features=3, n_samples_per_cluster=10) + X = X.astype(global_dtype, copy=False) # n_samples * n_samples_per_cluster shuffle_indices = np.arange(30) rng.shuffle(shuffle_indices) X_shuffle = X[shuffle_indices, :] - brc = Birch(n_clusters=4, threshold=1.) + brc = Birch(n_clusters=4, threshold=1.0) brc.fit(X_shuffle) - centroids = brc.subcluster_centers_ + + # Birch must preserve inputs' dtype + assert brc.subcluster_centers_.dtype == global_dtype + assert_array_equal(brc.labels_, brc.predict(X_shuffle)) - nearest_centroid = pairwise_distances_argmin(X_shuffle, centroids) - assert_almost_equal(v_measure_score(nearest_centroid, brc.labels_), 1.0) + centroids = brc.subcluster_centers_ + nearest_centroid = brc.subcluster_labels_[ + pairwise_distances_argmin(X_shuffle, centroids) + ] + assert_allclose(v_measure_score(nearest_centroid, brc.labels_), 1.0) -def test_n_clusters(): +def test_n_clusters(global_random_seed, global_dtype): # Test that n_clusters param works properly - X, y = make_blobs(n_samples=100, centers=10) + X, y = make_blobs(n_samples=100, centers=10, random_state=global_random_seed) + X = X.astype(global_dtype, copy=False) brc1 = Birch(n_clusters=10) brc1.fit(X) assert len(brc1.subcluster_centers_) > 10 @@ -83,31 +87,29 @@ def test_n_clusters(): assert_array_equal(brc1.subcluster_labels_, brc2.subcluster_labels_) assert_array_equal(brc1.labels_, brc2.labels_) - # Test that the wrong global clustering step raises an Error. - clf = ElasticNet() - brc3 = Birch(n_clusters=clf) - with pytest.raises(ValueError): - brc3.fit(X) - # Test that a small number of clusters raises a warning. - brc4 = Birch(threshold=10000.) + brc4 = Birch(threshold=10000.0) with pytest.warns(ConvergenceWarning): brc4.fit(X) -def test_sparse_X(): +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_sparse_X(global_random_seed, global_dtype, csr_container): # Test that sparse and dense data give same results - X, y = make_blobs(n_samples=100, centers=10) + X, y = make_blobs(n_samples=100, centers=10, random_state=global_random_seed) + X = X.astype(global_dtype, copy=False) brc = Birch(n_clusters=10) brc.fit(X) - csr = sparse.csr_matrix(X) + csr = csr_container(X) brc_sparse = Birch(n_clusters=10) brc_sparse.fit(csr) + # Birch must preserve inputs' dtype + assert brc_sparse.subcluster_centers_.dtype == global_dtype + assert_array_equal(brc.labels_, brc_sparse.labels_) - assert_array_almost_equal(brc.subcluster_centers_, - brc_sparse.subcluster_centers_) + assert_allclose(brc.subcluster_centers_, brc_sparse.subcluster_centers_) def test_partial_fit_second_call_error_checks(): @@ -130,26 +132,20 @@ def check_branching_factor(node, branching_factor): check_branching_factor(cluster.child_, branching_factor) -def test_branching_factor(): +def test_branching_factor(global_random_seed, global_dtype): # Test that nodes have at max branching_factor number of subclusters - X, y = make_blobs() + X, y = make_blobs(random_state=global_random_seed) + X = X.astype(global_dtype, copy=False) branching_factor = 9 # Purposefully set a low threshold to maximize the subclusters. - brc = Birch(n_clusters=None, branching_factor=branching_factor, - threshold=0.01) + brc = Birch(n_clusters=None, branching_factor=branching_factor, threshold=0.01) brc.fit(X) check_branching_factor(brc.root_, branching_factor) - brc = Birch(n_clusters=3, branching_factor=branching_factor, - threshold=0.01) + brc = Birch(n_clusters=3, branching_factor=branching_factor, threshold=0.01) brc.fit(X) check_branching_factor(brc.root_, branching_factor) - # Raises error when branching_factor is set to one. - brc = Birch(n_clusters=None, branching_factor=1, threshold=0.01) - with pytest.raises(ValueError): - brc.fit(X) - def check_threshold(birch_instance, threshold): """Use the leaf linked list for traversal""" @@ -161,16 +157,17 @@ def check_threshold(birch_instance, threshold): current_leaf = current_leaf.next_leaf_ -def test_threshold(): +def test_threshold(global_random_seed, global_dtype): # Test that the leaf subclusters have a threshold lesser than radius - X, y = make_blobs(n_samples=80, centers=4) + X, y = make_blobs(n_samples=80, centers=4, random_state=global_random_seed) + X = X.astype(global_dtype, copy=False) brc = Birch(threshold=0.5, n_clusters=None) brc.fit(X) check_threshold(brc, 0.5) brc = Birch(threshold=5.0, n_clusters=None) brc.fit(X) - check_threshold(brc, 5.) + check_threshold(brc, 5.0) def test_birch_n_clusters_long_int(): @@ -181,13 +178,73 @@ def test_birch_n_clusters_long_int(): Birch(n_clusters=n_clusters).fit(X) -# TODO: Remove in 1.2 -@pytest.mark.parametrize("attribute", ["fit_", "partial_fit_"]) -def test_birch_fit_attributes_deprecated(attribute): - """Test that fit_ and partial_fit_ attributes are deprecated.""" - msg = f"{attribute} is deprecated in 1.0 and will be removed in 1.2" - X, y = make_blobs(n_samples=10) - brc = Birch().fit(X, y) - - with pytest.warns(FutureWarning, match=msg): - getattr(brc, attribute) +def test_feature_names_out(): + """Check `get_feature_names_out` for `Birch`.""" + X, _ = make_blobs(n_samples=80, n_features=4, random_state=0) + brc = Birch(n_clusters=4) + brc.fit(X) + n_clusters = brc.subcluster_centers_.shape[0] + + names_out = brc.get_feature_names_out() + assert_array_equal([f"birch{i}" for i in range(n_clusters)], names_out) + + +def test_transform_match_across_dtypes(global_random_seed): + X, _ = make_blobs(n_samples=80, n_features=4, random_state=global_random_seed) + brc = Birch(n_clusters=4, threshold=1.1) + Y_64 = brc.fit_transform(X) + Y_32 = brc.fit_transform(X.astype(np.float32)) + + assert_allclose(Y_64, Y_32, atol=1e-6) + + +def test_subcluster_dtype(global_dtype): + X = make_blobs(n_samples=80, n_features=4, random_state=0)[0].astype( + global_dtype, copy=False + ) + brc = Birch(n_clusters=4) + assert brc.fit(X).subcluster_centers_.dtype == global_dtype + + +def test_both_subclusters_updated(): + """Check that both subclusters are updated when a node a split, even when there are + duplicated data points. Non-regression test for #23269. + """ + + X = np.array( + [ + [-2.6192791, -1.5053215], + [-2.9993038, -1.6863596], + [-2.3724914, -1.3438171], + [-2.336792, -1.3417323], + [-2.4089134, -1.3290224], + [-2.3724914, -1.3438171], + [-3.364009, -1.8846745], + [-2.3724914, -1.3438171], + [-2.617677, -1.5003285], + [-2.2960556, -1.3260119], + [-2.3724914, -1.3438171], + [-2.5459878, -1.4533926], + [-2.25979, -1.3003055], + [-2.4089134, -1.3290224], + [-2.3724914, -1.3438171], + [-2.4089134, -1.3290224], + [-2.5459878, -1.4533926], + [-2.3724914, -1.3438171], + [-2.9720619, -1.7058647], + [-2.336792, -1.3417323], + [-2.3724914, -1.3438171], + ], + dtype=np.float32, + ) + + # no error + Birch(branching_factor=5, threshold=1e-5, n_clusters=None).fit(X) + + +# TODO(1.8): Remove +def test_birch_copy_deprecated(): + X, _ = make_blobs(n_samples=80, n_features=4, random_state=0) + brc = Birch(n_clusters=4, copy=True) + with pytest.warns(FutureWarning, match="`copy` was deprecated"): + brc.fit(X) diff --git a/sklearn/cluster/tests/test_bisect_k_means.py b/sklearn/cluster/tests/test_bisect_k_means.py new file mode 100644 index 0000000000000..799ddbc086ce0 --- /dev/null +++ b/sklearn/cluster/tests/test_bisect_k_means.py @@ -0,0 +1,158 @@ +import numpy as np +import pytest + +from sklearn.cluster import BisectingKMeans +from sklearn.metrics import v_measure_score +from sklearn.utils._testing import assert_allclose, assert_array_equal +from sklearn.utils.fixes import CSR_CONTAINERS + + +@pytest.mark.parametrize("bisecting_strategy", ["biggest_inertia", "largest_cluster"]) +@pytest.mark.parametrize("init", ["k-means++", "random"]) +def test_three_clusters(bisecting_strategy, init): + """Tries to perform bisect k-means for three clusters to check + if splitting data is performed correctly. + """ + X = np.array( + [[1, 1], [10, 1], [3, 1], [10, 0], [2, 1], [10, 2], [10, 8], [10, 9], [10, 10]] + ) + bisect_means = BisectingKMeans( + n_clusters=3, + random_state=0, + bisecting_strategy=bisecting_strategy, + init=init, + ) + bisect_means.fit(X) + + expected_centers = [[2, 1], [10, 1], [10, 9]] + expected_labels = [0, 1, 0, 1, 0, 1, 2, 2, 2] + + assert_allclose( + sorted(expected_centers), sorted(bisect_means.cluster_centers_.tolist()) + ) + assert_allclose(v_measure_score(expected_labels, bisect_means.labels_), 1.0) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_sparse(csr_container): + """Test Bisecting K-Means with sparse data. + + Checks if labels and centers are the same between dense and sparse. + """ + + rng = np.random.RandomState(0) + + X = rng.rand(20, 2) + X[X < 0.8] = 0 + X_csr = csr_container(X) + + bisect_means = BisectingKMeans(n_clusters=3, random_state=0) + + bisect_means.fit(X_csr) + sparse_centers = bisect_means.cluster_centers_ + + bisect_means.fit(X) + normal_centers = bisect_means.cluster_centers_ + + # Check if results is the same for dense and sparse data + assert_allclose(normal_centers, sparse_centers, atol=1e-8) + + +@pytest.mark.parametrize("n_clusters", [4, 5]) +def test_n_clusters(n_clusters): + """Test if resulting labels are in range [0, n_clusters - 1].""" + + rng = np.random.RandomState(0) + X = rng.rand(10, 2) + + bisect_means = BisectingKMeans(n_clusters=n_clusters, random_state=0) + bisect_means.fit(X) + + assert_array_equal(np.unique(bisect_means.labels_), np.arange(n_clusters)) + + +def test_one_cluster(): + """Test single cluster.""" + + X = np.array([[1, 2], [10, 2], [10, 8]]) + + bisect_means = BisectingKMeans(n_clusters=1, random_state=0).fit(X) + + # All labels from fit or predict should be equal 0 + assert all(bisect_means.labels_ == 0) + assert all(bisect_means.predict(X) == 0) + + assert_allclose(bisect_means.cluster_centers_, X.mean(axis=0).reshape(1, -1)) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS + [None]) +def test_fit_predict(csr_container): + """Check if labels from fit(X) method are same as from fit(X).predict(X).""" + rng = np.random.RandomState(0) + + X = rng.rand(10, 2) + + if csr_container is not None: + X[X < 0.8] = 0 + X = csr_container(X) + + bisect_means = BisectingKMeans(n_clusters=3, random_state=0) + bisect_means.fit(X) + + assert_array_equal(bisect_means.labels_, bisect_means.predict(X)) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS + [None]) +def test_dtype_preserved(csr_container, global_dtype): + """Check that centers dtype is the same as input data dtype.""" + rng = np.random.RandomState(0) + X = rng.rand(10, 2).astype(global_dtype, copy=False) + + if csr_container is not None: + X[X < 0.8] = 0 + X = csr_container(X) + + km = BisectingKMeans(n_clusters=3, random_state=0) + km.fit(X) + + assert km.cluster_centers_.dtype == global_dtype + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS + [None]) +def test_float32_float64_equivalence(csr_container): + """Check that the results are the same between float32 and float64.""" + rng = np.random.RandomState(0) + X = rng.rand(10, 2) + + if csr_container is not None: + X[X < 0.8] = 0 + X = csr_container(X) + + km64 = BisectingKMeans(n_clusters=3, random_state=0).fit(X) + km32 = BisectingKMeans(n_clusters=3, random_state=0).fit(X.astype(np.float32)) + + assert_allclose(km32.cluster_centers_, km64.cluster_centers_) + assert_array_equal(km32.labels_, km64.labels_) + + +@pytest.mark.parametrize("algorithm", ("lloyd", "elkan")) +def test_no_crash_on_empty_bisections(algorithm): + # Non-regression test for: + # https://github.com/scikit-learn/scikit-learn/issues/27081 + rng = np.random.RandomState(0) + X_train = rng.rand(3000, 10) + bkm = BisectingKMeans(n_clusters=10, algorithm=algorithm).fit(X_train) + + # predict on scaled data to trigger pathologic case + # where the inner mask leads to empty bisections. + X_test = 50 * rng.rand(100, 10) + labels = bkm.predict(X_test) # should not crash with idiv by 0 + assert np.isin(np.unique(labels), np.arange(10)).all() + + +def test_one_feature(): + # Check that no error is raised when there is only one feature + # Non-regression test for: + # https://github.com/scikit-learn/scikit-learn/issues/27236 + X = np.random.normal(size=(128, 1)) + BisectingKMeans(bisecting_strategy="biggest_inertia", random_state=0).fit(X) diff --git a/sklearn/cluster/tests/test_dbscan.py b/sklearn/cluster/tests/test_dbscan.py index 3e59bf44d613e..556f89312d2fc 100644 --- a/sklearn/cluster/tests/test_dbscan.py +++ b/sklearn/cluster/tests/test_dbscan.py @@ -3,23 +3,18 @@ """ import pickle - -import numpy as np - import warnings -from scipy.spatial import distance -from scipy import sparse - +import numpy as np import pytest +from scipy.spatial import distance -from sklearn.utils._testing import assert_array_equal -from sklearn.neighbors import NearestNeighbors -from sklearn.cluster import DBSCAN -from sklearn.cluster import dbscan +from sklearn.cluster import DBSCAN, dbscan from sklearn.cluster.tests.common import generate_clustered_data from sklearn.metrics.pairwise import pairwise_distances - +from sklearn.neighbors import NearestNeighbors +from sklearn.utils._testing import assert_array_equal +from sklearn.utils.fixes import CSR_CONTAINERS, LIL_CONTAINERS n_clusters = 3 X = generate_clustered_data(n_clusters=n_clusters) @@ -34,8 +29,9 @@ def test_dbscan_similarity(): D = distance.squareform(distance.pdist(X)) D /= np.max(D) # Compute DBSCAN - core_samples, labels = dbscan(D, metric="precomputed", eps=eps, - min_samples=min_samples) + core_samples, labels = dbscan( + D, metric="precomputed", eps=eps, min_samples=min_samples + ) # number of clusters, ignoring noise if present n_clusters_1 = len(set(labels)) - (1 if -1 in labels else 0) @@ -54,11 +50,10 @@ def test_dbscan_feature(): # Different eps to other test, because distance is not normalised. eps = 0.8 min_samples = 10 - metric = 'euclidean' + metric = "euclidean" # Compute DBSCAN # parameters chosen for task - core_samples, labels = dbscan(X, metric=metric, eps=eps, - min_samples=min_samples) + core_samples, labels = dbscan(X, metric=metric, eps=eps, min_samples=min_samples) # number of clusters, ignoring noise if present n_clusters_1 = len(set(labels)) - int(-1 in labels) @@ -71,28 +66,26 @@ def test_dbscan_feature(): assert n_clusters_2 == n_clusters -def test_dbscan_sparse(): - core_sparse, labels_sparse = dbscan(sparse.lil_matrix(X), eps=.8, - min_samples=10) - core_dense, labels_dense = dbscan(X, eps=.8, min_samples=10) +@pytest.mark.parametrize("lil_container", LIL_CONTAINERS) +def test_dbscan_sparse(lil_container): + core_sparse, labels_sparse = dbscan(lil_container(X), eps=0.8, min_samples=10) + core_dense, labels_dense = dbscan(X, eps=0.8, min_samples=10) assert_array_equal(core_dense, core_sparse) assert_array_equal(labels_dense, labels_sparse) -@pytest.mark.parametrize('include_self', [False, True]) +@pytest.mark.parametrize("include_self", [False, True]) def test_dbscan_sparse_precomputed(include_self): D = pairwise_distances(X) - nn = NearestNeighbors(radius=.9).fit(X) + nn = NearestNeighbors(radius=0.9).fit(X) X_ = X if include_self else None - D_sparse = nn.radius_neighbors_graph(X=X_, mode='distance') + D_sparse = nn.radius_neighbors_graph(X=X_, mode="distance") # Ensure it is sparse not merely on diagonals: assert D_sparse.nnz < D.shape[0] * (D.shape[0] - 1) - core_sparse, labels_sparse = dbscan(D_sparse, - eps=.8, - min_samples=10, - metric='precomputed') - core_dense, labels_dense = dbscan(D, eps=.8, min_samples=10, - metric='precomputed') + core_sparse, labels_sparse = dbscan( + D_sparse, eps=0.8, min_samples=10, metric="precomputed" + ) + core_dense, labels_dense = dbscan(D, eps=0.8, min_samples=10, metric="precomputed") assert_array_equal(core_dense, core_sparse) assert_array_equal(labels_dense, labels_sparse) @@ -102,39 +95,62 @@ def test_dbscan_sparse_precomputed_different_eps(): # a radius larger than DBSCAN's eps. lower_eps = 0.2 nn = NearestNeighbors(radius=lower_eps).fit(X) - D_sparse = nn.radius_neighbors_graph(X, mode='distance') - dbscan_lower = dbscan(D_sparse, eps=lower_eps, metric='precomputed') + D_sparse = nn.radius_neighbors_graph(X, mode="distance") + dbscan_lower = dbscan(D_sparse, eps=lower_eps, metric="precomputed") higher_eps = lower_eps + 0.7 nn = NearestNeighbors(radius=higher_eps).fit(X) - D_sparse = nn.radius_neighbors_graph(X, mode='distance') - dbscan_higher = dbscan(D_sparse, eps=lower_eps, metric='precomputed') + D_sparse = nn.radius_neighbors_graph(X, mode="distance") + dbscan_higher = dbscan(D_sparse, eps=lower_eps, metric="precomputed") assert_array_equal(dbscan_lower[0], dbscan_higher[0]) assert_array_equal(dbscan_lower[1], dbscan_higher[1]) -@pytest.mark.parametrize('use_sparse', [True, False]) -@pytest.mark.parametrize('metric', ['precomputed', 'minkowski']) -def test_dbscan_input_not_modified(use_sparse, metric): +@pytest.mark.parametrize("metric", ["precomputed", "minkowski"]) +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS + [None]) +def test_dbscan_input_not_modified(metric, csr_container): # test that the input is not modified by dbscan X = np.random.RandomState(0).rand(10, 10) - X = sparse.csr_matrix(X) if use_sparse else X + X = csr_container(X) if csr_container is not None else X X_copy = X.copy() dbscan(X, metric=metric) - if use_sparse: + if csr_container is not None: assert_array_equal(X.toarray(), X_copy.toarray()) else: assert_array_equal(X, X_copy) -def test_dbscan_no_core_samples(): +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_dbscan_input_not_modified_precomputed_sparse_nodiag(csr_container): + """Check that we don't modify in-place the pre-computed sparse matrix. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/27508 + """ + X = np.random.RandomState(0).rand(10, 10) + # Add zeros on the diagonal that will be implicit when creating + # the sparse matrix. If `X` is modified in-place, the zeros from + # the diagonal will be made explicit. + np.fill_diagonal(X, 0) + X = csr_container(X) + assert all(row != col for row, col in zip(*X.nonzero())) + X_copy = X.copy() + dbscan(X, metric="precomputed") + # Make sure that we did not modify `X` in-place even by creating + # explicit 0s values. + assert X.nnz == X_copy.nnz + assert_array_equal(X.toarray(), X_copy.toarray()) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_dbscan_no_core_samples(csr_container): rng = np.random.RandomState(0) X = rng.rand(40, 10) - X[X < .8] = 0 + X[X < 0.8] = 0 - for X_ in [X, sparse.csr_matrix(X)]: + for X_ in [X, csr_container(X)]: db = DBSCAN(min_samples=6).fit(X_) assert_array_equal(db.components_, np.empty((0, X_.shape[1]))) assert_array_equal(db.labels_, -1) @@ -151,16 +167,15 @@ def test_dbscan_callable(): metric = distance.euclidean # Compute DBSCAN # parameters chosen for task - core_samples, labels = dbscan(X, metric=metric, eps=eps, - min_samples=min_samples, - algorithm='ball_tree') + core_samples, labels = dbscan( + X, metric=metric, eps=eps, min_samples=min_samples, algorithm="ball_tree" + ) # number of clusters, ignoring noise if present n_clusters_1 = len(set(labels)) - int(-1 in labels) assert n_clusters_1 == n_clusters - db = DBSCAN(metric=metric, eps=eps, min_samples=min_samples, - algorithm='ball_tree') + db = DBSCAN(metric=metric, eps=eps, min_samples=min_samples, algorithm="ball_tree") labels = db.fit(X).labels_ n_clusters_2 = len(set(labels)) - int(-1 in labels) @@ -177,23 +192,29 @@ def test_dbscan_metric_params(): with warnings.catch_warnings(record=True) as warns: db = DBSCAN( - metric='minkowski', metric_params={'p': p}, eps=eps, - p=None, min_samples=min_samples, algorithm='ball_tree' - ).fit(X) - assert not warns + metric="minkowski", + metric_params={"p": p}, + eps=eps, + p=None, + min_samples=min_samples, + algorithm="ball_tree", + ).fit(X) + assert not warns, warns[0].message core_sample_1, labels_1 = db.core_sample_indices_, db.labels_ # Test that sample labels are the same as passing Minkowski 'p' directly - db = DBSCAN(metric='minkowski', eps=eps, min_samples=min_samples, - algorithm='ball_tree', p=p).fit(X) + db = DBSCAN( + metric="minkowski", eps=eps, min_samples=min_samples, algorithm="ball_tree", p=p + ).fit(X) core_sample_2, labels_2 = db.core_sample_indices_, db.labels_ assert_array_equal(core_sample_1, core_sample_2) assert_array_equal(labels_1, labels_2) # Minkowski with p=1 should be equivalent to Manhattan distance - db = DBSCAN(metric='manhattan', eps=eps, min_samples=min_samples, - algorithm='ball_tree').fit(X) + db = DBSCAN( + metric="manhattan", eps=eps, min_samples=min_samples, algorithm="ball_tree" + ).fit(X) core_sample_3, labels_3 = db.core_sample_indices_, db.labels_ assert_array_equal(core_sample_1, core_sample_3) @@ -201,12 +222,21 @@ def test_dbscan_metric_params(): with pytest.warns( SyntaxWarning, - match="Parameter p is found in metric_params. " - "The corresponding parameter from __init__ " - "is ignored."): + match=( + "Parameter p is found in metric_params. " + "The corresponding parameter from __init__ " + "is ignored." + ), + ): # Test that checks p is ignored in favor of metric_params={'p': } - db = DBSCAN(metric='minkowski', metric_params={'p': p}, eps=eps, p=p+1, - min_samples=min_samples, algorithm='ball_tree').fit(X) + db = DBSCAN( + metric="minkowski", + metric_params={"p": p}, + eps=eps, + p=p + 1, + min_samples=min_samples, + algorithm="ball_tree", + ).fit(X) core_sample_4, labels_4 = db.core_sample_indices_, db.labels_ assert_array_equal(core_sample_1, core_sample_4) @@ -219,33 +249,33 @@ def test_dbscan_balltree(): min_samples = 10 D = pairwise_distances(X) - core_samples, labels = dbscan(D, metric="precomputed", eps=eps, - min_samples=min_samples) + core_samples, labels = dbscan( + D, metric="precomputed", eps=eps, min_samples=min_samples + ) # number of clusters, ignoring noise if present n_clusters_1 = len(set(labels)) - int(-1 in labels) assert n_clusters_1 == n_clusters - db = DBSCAN(p=2.0, eps=eps, min_samples=min_samples, algorithm='ball_tree') + db = DBSCAN(p=2.0, eps=eps, min_samples=min_samples, algorithm="ball_tree") labels = db.fit(X).labels_ n_clusters_2 = len(set(labels)) - int(-1 in labels) assert n_clusters_2 == n_clusters - db = DBSCAN(p=2.0, eps=eps, min_samples=min_samples, algorithm='kd_tree') + db = DBSCAN(p=2.0, eps=eps, min_samples=min_samples, algorithm="kd_tree") labels = db.fit(X).labels_ n_clusters_3 = len(set(labels)) - int(-1 in labels) assert n_clusters_3 == n_clusters - db = DBSCAN(p=1.0, eps=eps, min_samples=min_samples, algorithm='ball_tree') + db = DBSCAN(p=1.0, eps=eps, min_samples=min_samples, algorithm="ball_tree") labels = db.fit(X).labels_ n_clusters_4 = len(set(labels)) - int(-1 in labels) assert n_clusters_4 == n_clusters - db = DBSCAN(leaf_size=20, eps=eps, min_samples=min_samples, - algorithm='ball_tree') + db = DBSCAN(leaf_size=20, eps=eps, min_samples=min_samples, algorithm="ball_tree") labels = db.fit(X).labels_ n_clusters_5 = len(set(labels)) - int(-1 in labels) @@ -254,25 +284,14 @@ def test_dbscan_balltree(): def test_input_validation(): # DBSCAN.fit should accept a list of lists. - X = [[1., 2.], [3., 4.]] - DBSCAN().fit(X) # must not raise exception - - -@pytest.mark.parametrize( - "args", - [{'eps': -1.0}, {'algorithm': 'blah'}, {'metric': 'blah'}, - {'leaf_size': -1}, {'p': -1}] -) -def test_dbscan_badargs(args): - # Test bad argument values: these should all raise ValueErrors - with pytest.raises(ValueError): - dbscan(X, **args) + X = [[1.0, 2.0], [3.0, 4.0]] + DBSCAN().fit(X) # must not raise exception def test_pickle(): obj = DBSCAN() s = pickle.dumps(obj) - assert type(pickle.loads(s)) == obj.__class__ + assert type(pickle.loads(s)) is obj.__class__ def test_boundaries(): @@ -282,11 +301,11 @@ def test_boundaries(): # ensure eps is inclusive of circumference core, _ = dbscan([[0], [1], [1]], eps=1, min_samples=2) assert 0 in core - core, _ = dbscan([[0], [1], [1]], eps=.99, min_samples=2) + core, _ = dbscan([[0], [1], [1]], eps=0.99, min_samples=2) assert 0 not in core -def test_weighted_dbscan(): +def test_weighted_dbscan(global_random_seed): # ensure sample_weight is validated with pytest.raises(ValueError): dbscan([[0], [1]], sample_weight=[2]) @@ -294,30 +313,33 @@ def test_weighted_dbscan(): dbscan([[0], [1]], sample_weight=[2, 3, 4]) # ensure sample_weight has an effect - assert_array_equal([], dbscan([[0], [1]], sample_weight=None, - min_samples=6)[0]) - assert_array_equal([], dbscan([[0], [1]], sample_weight=[5, 5], - min_samples=6)[0]) - assert_array_equal([0], dbscan([[0], [1]], sample_weight=[6, 5], - min_samples=6)[0]) - assert_array_equal([0, 1], dbscan([[0], [1]], sample_weight=[6, 6], - min_samples=6)[0]) + assert_array_equal([], dbscan([[0], [1]], sample_weight=None, min_samples=6)[0]) + assert_array_equal([], dbscan([[0], [1]], sample_weight=[5, 5], min_samples=6)[0]) + assert_array_equal([0], dbscan([[0], [1]], sample_weight=[6, 5], min_samples=6)[0]) + assert_array_equal( + [0, 1], dbscan([[0], [1]], sample_weight=[6, 6], min_samples=6)[0] + ) # points within eps of each other: - assert_array_equal([0, 1], dbscan([[0], [1]], eps=1.5, - sample_weight=[5, 1], min_samples=6)[0]) + assert_array_equal( + [0, 1], dbscan([[0], [1]], eps=1.5, sample_weight=[5, 1], min_samples=6)[0] + ) # and effect of non-positive and non-integer sample_weight: - assert_array_equal([], dbscan([[0], [1]], sample_weight=[5, 0], - eps=1.5, min_samples=6)[0]) - assert_array_equal([0, 1], dbscan([[0], [1]], sample_weight=[5.9, 0.1], - eps=1.5, min_samples=6)[0]) - assert_array_equal([0, 1], dbscan([[0], [1]], sample_weight=[6, 0], - eps=1.5, min_samples=6)[0]) - assert_array_equal([], dbscan([[0], [1]], sample_weight=[6, -1], - eps=1.5, min_samples=6)[0]) + assert_array_equal( + [], dbscan([[0], [1]], sample_weight=[5, 0], eps=1.5, min_samples=6)[0] + ) + assert_array_equal( + [0, 1], dbscan([[0], [1]], sample_weight=[5.9, 0.1], eps=1.5, min_samples=6)[0] + ) + assert_array_equal( + [0, 1], dbscan([[0], [1]], sample_weight=[6, 0], eps=1.5, min_samples=6)[0] + ) + assert_array_equal( + [], dbscan([[0], [1]], sample_weight=[6, -1], eps=1.5, min_samples=6)[0] + ) # for non-negative sample_weight, cores should be identical to repetition - rng = np.random.RandomState(42) + rng = np.random.RandomState(global_random_seed) sample_weight = rng.randint(0, 5, X.shape[0]) core1, label1 = dbscan(X, sample_weight=sample_weight) assert len(label1) == len(X) @@ -332,8 +354,7 @@ def test_weighted_dbscan(): # sample_weight should work with precomputed distance matrix D = pairwise_distances(X) - core3, label3 = dbscan(D, sample_weight=sample_weight, - metric='precomputed') + core3, label3 = dbscan(D, sample_weight=sample_weight, metric="precomputed") assert_array_equal(core1, core3) assert_array_equal(label1, label3) @@ -352,64 +373,62 @@ def test_weighted_dbscan(): assert_array_equal(label1, est.labels_) -@pytest.mark.parametrize('algorithm', ['brute', 'kd_tree', 'ball_tree']) +@pytest.mark.parametrize("algorithm", ["brute", "kd_tree", "ball_tree"]) def test_dbscan_core_samples_toy(algorithm): X = [[0], [2], [3], [4], [6], [8], [10]] n_samples = len(X) # Degenerate case: every sample is a core sample, either with its own # cluster or including other close core samples. - core_samples, labels = dbscan(X, algorithm=algorithm, eps=1, - min_samples=1) + core_samples, labels = dbscan(X, algorithm=algorithm, eps=1, min_samples=1) assert_array_equal(core_samples, np.arange(n_samples)) assert_array_equal(labels, [0, 1, 1, 1, 2, 3, 4]) # With eps=1 and min_samples=2 only the 3 samples from the denser area # are core samples. All other points are isolated and considered noise. - core_samples, labels = dbscan(X, algorithm=algorithm, eps=1, - min_samples=2) + core_samples, labels = dbscan(X, algorithm=algorithm, eps=1, min_samples=2) assert_array_equal(core_samples, [1, 2, 3]) assert_array_equal(labels, [-1, 0, 0, 0, -1, -1, -1]) # Only the sample in the middle of the dense area is core. Its two # neighbors are edge samples. Remaining samples are noise. - core_samples, labels = dbscan(X, algorithm=algorithm, eps=1, - min_samples=3) + core_samples, labels = dbscan(X, algorithm=algorithm, eps=1, min_samples=3) assert_array_equal(core_samples, [2]) assert_array_equal(labels, [-1, 0, 0, 0, -1, -1, -1]) # It's no longer possible to extract core samples with eps=1: # everything is noise. - core_samples, labels = dbscan(X, algorithm=algorithm, eps=1, - min_samples=4) + core_samples, labels = dbscan(X, algorithm=algorithm, eps=1, min_samples=4) assert_array_equal(core_samples, []) - assert_array_equal(labels, np.full(n_samples, -1.)) + assert_array_equal(labels, np.full(n_samples, -1.0)) def test_dbscan_precomputed_metric_with_degenerate_input_arrays(): # see https://github.com/scikit-learn/scikit-learn/issues/4641 for # more details X = np.eye(10) - labels = DBSCAN(eps=0.5, metric='precomputed').fit(X).labels_ + labels = DBSCAN(eps=0.5, metric="precomputed").fit(X).labels_ assert len(set(labels)) == 1 X = np.zeros((10, 10)) - labels = DBSCAN(eps=0.5, metric='precomputed').fit(X).labels_ + labels = DBSCAN(eps=0.5, metric="precomputed").fit(X).labels_ assert len(set(labels)) == 1 -def test_dbscan_precomputed_metric_with_initial_rows_zero(): +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_dbscan_precomputed_metric_with_initial_rows_zero(csr_container): # sample matrix with initial two row all zero - ar = np.array([ - [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0, 0.1, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0, 0.1, 0.0, 0.0], - [0.0, 0.0, 0.1, 0.1, 0.0, 0.0, 0.3], - [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.1], - [0.0, 0.0, 0.0, 0.0, 0.3, 0.1, 0.0] - ]) - matrix = sparse.csr_matrix(ar) - labels = DBSCAN(eps=0.2, metric='precomputed', - min_samples=2).fit(matrix).labels_ - assert_array_equal(labels, [-1, -1, 0, 0, 0, 1, 1]) + ar = np.array( + [ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.1, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.1, 0.0, 0.0], + [0.0, 0.0, 0.1, 0.1, 0.0, 0.0, 0.3], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.1], + [0.0, 0.0, 0.0, 0.0, 0.3, 0.1, 0.0], + ] + ) + matrix = csr_container(ar) + labels = DBSCAN(eps=0.2, metric="precomputed", min_samples=2).fit(matrix).labels_ + assert_array_equal(labels, [-1, -1, 0, 0, 0, 1, 1]) diff --git a/sklearn/cluster/tests/test_feature_agglomeration.py b/sklearn/cluster/tests/test_feature_agglomeration.py index ebc2fe49d7a7f..80aa251c35815 100644 --- a/sklearn/cluster/tests/test_feature_agglomeration.py +++ b/sklearn/cluster/tests/test_feature_agglomeration.py @@ -1,10 +1,12 @@ """ Tests for sklearn.cluster._feature_agglomeration """ -# Authors: Sergul Aydore 2017 + import numpy as np -import pytest +from numpy.testing import assert_array_equal + from sklearn.cluster import FeatureAgglomeration +from sklearn.datasets import make_blobs from sklearn.utils._testing import assert_array_almost_equal @@ -12,16 +14,11 @@ def test_feature_agglomeration(): n_clusters = 1 X = np.array([0, 0, 1]).reshape(1, 3) # (n_samples, n_features) - agglo_mean = FeatureAgglomeration(n_clusters=n_clusters, - pooling_func=np.mean) - agglo_median = FeatureAgglomeration(n_clusters=n_clusters, - pooling_func=np.median) - with pytest.warns(None) as record: - agglo_mean.fit(X) - assert not len(record) - with pytest.warns(None) as record: - agglo_median.fit(X) - assert not len(record) + agglo_mean = FeatureAgglomeration(n_clusters=n_clusters, pooling_func=np.mean) + agglo_median = FeatureAgglomeration(n_clusters=n_clusters, pooling_func=np.median) + agglo_mean.fit(X) + agglo_median.fit(X) + assert np.size(np.unique(agglo_mean.labels_)) == n_clusters assert np.size(np.unique(agglo_median.labels_)) == n_clusters assert np.size(agglo_mean.labels_) == X.shape[1] @@ -32,8 +29,8 @@ def test_feature_agglomeration(): Xt_median = agglo_median.transform(X) assert Xt_mean.shape[1] == n_clusters assert Xt_median.shape[1] == n_clusters - assert Xt_mean == np.array([1 / 3.]) - assert Xt_median == np.array([0.]) + assert Xt_mean == np.array([1 / 3.0]) + assert Xt_median == np.array([0.0]) # Test inverse transform X_full_mean = agglo_mean.inverse_transform(Xt_mean) @@ -41,7 +38,18 @@ def test_feature_agglomeration(): assert np.unique(X_full_mean[0]).size == n_clusters assert np.unique(X_full_median[0]).size == n_clusters - assert_array_almost_equal(agglo_mean.transform(X_full_mean), - Xt_mean) - assert_array_almost_equal(agglo_median.transform(X_full_median), - Xt_median) + assert_array_almost_equal(agglo_mean.transform(X_full_mean), Xt_mean) + assert_array_almost_equal(agglo_median.transform(X_full_median), Xt_median) + + +def test_feature_agglomeration_feature_names_out(): + """Check `get_feature_names_out` for `FeatureAgglomeration`.""" + X, _ = make_blobs(n_features=6, random_state=0) + agglo = FeatureAgglomeration(n_clusters=3) + agglo.fit(X) + n_clusters = agglo.n_clusters_ + + names_out = agglo.get_feature_names_out() + assert_array_equal( + [f"featureagglomeration{i}" for i in range(n_clusters)], names_out + ) diff --git a/sklearn/cluster/tests/test_hdbscan.py b/sklearn/cluster/tests/test_hdbscan.py new file mode 100644 index 0000000000000..3b45d9d3cb7aa --- /dev/null +++ b/sklearn/cluster/tests/test_hdbscan.py @@ -0,0 +1,582 @@ +""" +Tests for HDBSCAN clustering algorithm +Based on the DBSCAN test code +""" + +import numpy as np +import pytest +from scipy import stats +from scipy.spatial import distance + +from sklearn.cluster import HDBSCAN +from sklearn.cluster._hdbscan._tree import ( + CONDENSED_dtype, + _condense_tree, + _do_labelling, +) +from sklearn.cluster._hdbscan.hdbscan import _OUTLIER_ENCODING +from sklearn.datasets import make_blobs +from sklearn.metrics import fowlkes_mallows_score +from sklearn.metrics.pairwise import _VALID_METRICS, euclidean_distances +from sklearn.neighbors import BallTree, KDTree +from sklearn.preprocessing import StandardScaler +from sklearn.utils import shuffle +from sklearn.utils._testing import assert_allclose, assert_array_equal +from sklearn.utils.fixes import CSC_CONTAINERS, CSR_CONTAINERS + +X, y = make_blobs(n_samples=200, random_state=10) +X, y = shuffle(X, y, random_state=7) +X = StandardScaler().fit_transform(X) + +ALGORITHMS = [ + "kd_tree", + "ball_tree", + "brute", + "auto", +] + +OUTLIER_SET = {-1} | {out["label"] for _, out in _OUTLIER_ENCODING.items()} + + +def check_label_quality(labels, threshold=0.99): + n_clusters = len(set(labels) - OUTLIER_SET) + assert n_clusters == 3 + assert fowlkes_mallows_score(labels, y) > threshold + + +@pytest.mark.parametrize("outlier_type", _OUTLIER_ENCODING) +def test_outlier_data(outlier_type): + """ + Tests if np.inf and np.nan data are each treated as special outliers. + """ + outlier = { + "infinite": np.inf, + "missing": np.nan, + }[outlier_type] + prob_check = { + "infinite": lambda x, y: x == y, + "missing": lambda x, y: np.isnan(x), + }[outlier_type] + label = _OUTLIER_ENCODING[outlier_type]["label"] + prob = _OUTLIER_ENCODING[outlier_type]["prob"] + + X_outlier = X.copy() + X_outlier[0] = [outlier, 1] + X_outlier[5] = [outlier, outlier] + model = HDBSCAN().fit(X_outlier) + + (missing_labels_idx,) = (model.labels_ == label).nonzero() + assert_array_equal(missing_labels_idx, [0, 5]) + + (missing_probs_idx,) = (prob_check(model.probabilities_, prob)).nonzero() + assert_array_equal(missing_probs_idx, [0, 5]) + + clean_indices = list(range(1, 5)) + list(range(6, 200)) + clean_model = HDBSCAN().fit(X_outlier[clean_indices]) + assert_array_equal(clean_model.labels_, model.labels_[clean_indices]) + + +def test_hdbscan_distance_matrix(): + """ + Tests that HDBSCAN works with precomputed distance matrices, and throws the + appropriate errors when needed. + """ + D = euclidean_distances(X) + D_original = D.copy() + labels = HDBSCAN(metric="precomputed", copy=True).fit_predict(D) + + assert_allclose(D, D_original) + check_label_quality(labels) + + msg = r"The precomputed distance matrix.*has shape" + with pytest.raises(ValueError, match=msg): + HDBSCAN(metric="precomputed", copy=True).fit_predict(X) + + msg = r"The precomputed distance matrix.*values" + # Ensure the matrix is not symmetric + D[0, 1] = 10 + D[1, 0] = 1 + with pytest.raises(ValueError, match=msg): + HDBSCAN(metric="precomputed").fit_predict(D) + + +@pytest.mark.parametrize("sparse_constructor", [*CSR_CONTAINERS, *CSC_CONTAINERS]) +def test_hdbscan_sparse_distance_matrix(sparse_constructor): + """ + Tests that HDBSCAN works with sparse distance matrices. + """ + D = distance.squareform(distance.pdist(X)) + D /= np.max(D) + + threshold = stats.scoreatpercentile(D.flatten(), 50) + + D[D >= threshold] = 0.0 + D = sparse_constructor(D) + D.eliminate_zeros() + + labels = HDBSCAN(metric="precomputed").fit_predict(D) + check_label_quality(labels) + + +def test_hdbscan_feature_array(): + """ + Tests that HDBSCAN works with feature array, including an arbitrary + goodness of fit check. Note that the check is a simple heuristic. + """ + labels = HDBSCAN().fit_predict(X) + + # Check that clustering is arbitrarily good + # This is a heuristic to guard against regression + check_label_quality(labels) + + +@pytest.mark.parametrize("algo", ALGORITHMS) +@pytest.mark.parametrize("metric", _VALID_METRICS) +def test_hdbscan_algorithms(algo, metric): + """ + Tests that HDBSCAN works with the expected combinations of algorithms and + metrics, or raises the expected errors. + """ + labels = HDBSCAN(algorithm=algo).fit_predict(X) + check_label_quality(labels) + + # Validation for brute is handled by `pairwise_distances` + if algo in ("brute", "auto"): + return + + ALGOS_TREES = { + "kd_tree": KDTree, + "ball_tree": BallTree, + } + metric_params = { + "mahalanobis": {"V": np.eye(X.shape[1])}, + "seuclidean": {"V": np.ones(X.shape[1])}, + "minkowski": {"p": 2}, + "wminkowski": {"p": 2, "w": np.ones(X.shape[1])}, + }.get(metric, None) + + hdb = HDBSCAN( + algorithm=algo, + metric=metric, + metric_params=metric_params, + ) + + if metric not in ALGOS_TREES[algo].valid_metrics: + with pytest.raises(ValueError): + hdb.fit(X) + elif metric == "wminkowski": + with pytest.warns(FutureWarning): + hdb.fit(X) + else: + hdb.fit(X) + + +def test_dbscan_clustering(): + """ + Tests that HDBSCAN can generate a sufficiently accurate dbscan clustering. + This test is more of a sanity check than a rigorous evaluation. + """ + clusterer = HDBSCAN().fit(X) + labels = clusterer.dbscan_clustering(0.3) + + # We use a looser threshold due to dbscan producing a more constrained + # clustering representation + check_label_quality(labels, threshold=0.92) + + +@pytest.mark.parametrize("cut_distance", (0.1, 0.5, 1)) +def test_dbscan_clustering_outlier_data(cut_distance): + """ + Tests if np.inf and np.nan data are each treated as special outliers. + """ + missing_label = _OUTLIER_ENCODING["missing"]["label"] + infinite_label = _OUTLIER_ENCODING["infinite"]["label"] + + X_outlier = X.copy() + X_outlier[0] = [np.inf, 1] + X_outlier[2] = [1, np.nan] + X_outlier[5] = [np.inf, np.nan] + model = HDBSCAN().fit(X_outlier) + labels = model.dbscan_clustering(cut_distance=cut_distance) + + missing_labels_idx = np.flatnonzero(labels == missing_label) + assert_array_equal(missing_labels_idx, [2, 5]) + + infinite_labels_idx = np.flatnonzero(labels == infinite_label) + assert_array_equal(infinite_labels_idx, [0]) + + clean_idx = list(set(range(200)) - set(missing_labels_idx + infinite_labels_idx)) + clean_model = HDBSCAN().fit(X_outlier[clean_idx]) + clean_labels = clean_model.dbscan_clustering(cut_distance=cut_distance) + assert_array_equal(clean_labels, labels[clean_idx]) + + +def test_hdbscan_best_balltree_metric(): + """ + Tests that HDBSCAN using `BallTree` works. + """ + labels = HDBSCAN( + metric="seuclidean", metric_params={"V": np.ones(X.shape[1])} + ).fit_predict(X) + check_label_quality(labels) + + +def test_hdbscan_no_clusters(): + """ + Tests that HDBSCAN correctly does not generate a valid cluster when the + `min_cluster_size` is too large for the data. + """ + labels = HDBSCAN(min_cluster_size=len(X) - 1).fit_predict(X) + assert set(labels).issubset(OUTLIER_SET) + + +def test_hdbscan_min_cluster_size(): + """ + Test that the smallest non-noise cluster has at least `min_cluster_size` + many points + """ + for min_cluster_size in range(2, len(X), 1): + labels = HDBSCAN(min_cluster_size=min_cluster_size).fit_predict(X) + true_labels = [label for label in labels if label != -1] + if len(true_labels) != 0: + assert np.min(np.bincount(true_labels)) >= min_cluster_size + + +def test_hdbscan_callable_metric(): + """ + Tests that HDBSCAN works when passed a callable metric. + """ + metric = distance.euclidean + labels = HDBSCAN(metric=metric).fit_predict(X) + check_label_quality(labels) + + +@pytest.mark.parametrize("tree", ["kd_tree", "ball_tree"]) +def test_hdbscan_precomputed_non_brute(tree): + """ + Tests that HDBSCAN correctly raises an error when passing precomputed data + while requesting a tree-based algorithm. + """ + hdb = HDBSCAN(metric="precomputed", algorithm=tree) + msg = "precomputed is not a valid metric for" + with pytest.raises(ValueError, match=msg): + hdb.fit(X) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_hdbscan_sparse(csr_container): + """ + Tests that HDBSCAN works correctly when passing sparse feature data. + Evaluates correctness by comparing against the same data passed as a dense + array. + """ + + dense_labels = HDBSCAN().fit(X).labels_ + check_label_quality(dense_labels) + + _X_sparse = csr_container(X) + X_sparse = _X_sparse.copy() + sparse_labels = HDBSCAN().fit(X_sparse).labels_ + assert_array_equal(dense_labels, sparse_labels) + + # Compare that the sparse and dense non-precomputed routines return the same labels + # where the 0th observation contains the outlier. + for outlier_val, outlier_type in ((np.inf, "infinite"), (np.nan, "missing")): + X_dense = X.copy() + X_dense[0, 0] = outlier_val + dense_labels = HDBSCAN().fit(X_dense).labels_ + check_label_quality(dense_labels) + assert dense_labels[0] == _OUTLIER_ENCODING[outlier_type]["label"] + + X_sparse = _X_sparse.copy() + X_sparse[0, 0] = outlier_val + sparse_labels = HDBSCAN().fit(X_sparse).labels_ + assert_array_equal(dense_labels, sparse_labels) + + msg = "Sparse data matrices only support algorithm `brute`." + with pytest.raises(ValueError, match=msg): + HDBSCAN(metric="euclidean", algorithm="ball_tree").fit(X_sparse) + + +@pytest.mark.parametrize("algorithm", ALGORITHMS) +def test_hdbscan_centers(algorithm): + """ + Tests that HDBSCAN centers are calculated and stored properly, and are + accurate to the data. + """ + centers = [(0.0, 0.0), (3.0, 3.0)] + H, _ = make_blobs(n_samples=2000, random_state=0, centers=centers, cluster_std=0.5) + hdb = HDBSCAN(store_centers="both").fit(H) + + for center, centroid, medoid in zip(centers, hdb.centroids_, hdb.medoids_): + assert_allclose(center, centroid, rtol=1, atol=0.05) + assert_allclose(center, medoid, rtol=1, atol=0.05) + + # Ensure that nothing is done for noise + hdb = HDBSCAN( + algorithm=algorithm, store_centers="both", min_cluster_size=X.shape[0] + ).fit(X) + assert hdb.centroids_.shape[0] == 0 + assert hdb.medoids_.shape[0] == 0 + + +def test_hdbscan_allow_single_cluster_with_epsilon(): + """ + Tests that HDBSCAN single-cluster selection with epsilon works correctly. + """ + rng = np.random.RandomState(0) + no_structure = rng.rand(150, 2) + # without epsilon we should see many noise points as children of root. + labels = HDBSCAN( + min_cluster_size=5, + cluster_selection_epsilon=0.0, + cluster_selection_method="eom", + allow_single_cluster=True, + ).fit_predict(no_structure) + unique_labels, counts = np.unique(labels, return_counts=True) + assert len(unique_labels) == 2 + + # Arbitrary heuristic. Would prefer something more precise. + assert counts[unique_labels == -1] > 30 + + # for this random seed an epsilon of 0.18 will produce exactly 2 noise + # points at that cut in single linkage. + labels = HDBSCAN( + min_cluster_size=5, + cluster_selection_epsilon=0.18, + cluster_selection_method="eom", + allow_single_cluster=True, + algorithm="kd_tree", + ).fit_predict(no_structure) + unique_labels, counts = np.unique(labels, return_counts=True) + assert len(unique_labels) == 2 + assert counts[unique_labels == -1] == 2 + + +def test_hdbscan_better_than_dbscan(): + """ + Validate that HDBSCAN can properly cluster this difficult synthetic + dataset. Note that DBSCAN fails on this (see HDBSCAN plotting + example) + """ + centers = [[-0.85, -0.85], [-0.85, 0.85], [3, 3], [3, -3]] + X, y = make_blobs( + n_samples=750, + centers=centers, + cluster_std=[0.2, 0.35, 1.35, 1.35], + random_state=0, + ) + labels = HDBSCAN().fit(X).labels_ + + n_clusters = len(set(labels)) - int(-1 in labels) + assert n_clusters == 4 + fowlkes_mallows_score(labels, y) > 0.99 + + +@pytest.mark.parametrize( + "kwargs, X", + [ + ({"metric": "precomputed"}, np.array([[1, np.inf], [np.inf, 1]])), + ({"metric": "precomputed"}, [[1, 2], [2, 1]]), + ({}, [[1, 2], [3, 4]]), + ], +) +def test_hdbscan_usable_inputs(X, kwargs): + """ + Tests that HDBSCAN works correctly for array-likes and precomputed inputs + with non-finite points. + """ + HDBSCAN(min_samples=1, **kwargs).fit(X) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_hdbscan_sparse_distances_too_few_nonzero(csr_container): + """ + Tests that HDBSCAN raises the correct error when there are too few + non-zero distances. + """ + X = csr_container(np.zeros((10, 10))) + + msg = "There exists points with fewer than" + with pytest.raises(ValueError, match=msg): + HDBSCAN(metric="precomputed").fit(X) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_hdbscan_sparse_distances_disconnected_graph(csr_container): + """ + Tests that HDBSCAN raises the correct error when the distance matrix + has multiple connected components. + """ + # Create symmetric sparse matrix with 2 connected components + X = np.zeros((20, 20)) + X[:5, :5] = 1 + X[5:, 15:] = 1 + X = X + X.T + X = csr_container(X) + msg = "HDBSCAN cannot be performed on a disconnected graph" + with pytest.raises(ValueError, match=msg): + HDBSCAN(metric="precomputed").fit(X) + + +def test_hdbscan_tree_invalid_metric(): + """ + Tests that HDBSCAN correctly raises an error for invalid metric choices. + """ + metric_callable = lambda x: x + msg = ( + ".* is not a valid metric for a .*-based algorithm\\. Please select a different" + " metric\\." + ) + + # Callables are not supported for either + with pytest.raises(ValueError, match=msg): + HDBSCAN(algorithm="kd_tree", metric=metric_callable).fit(X) + with pytest.raises(ValueError, match=msg): + HDBSCAN(algorithm="ball_tree", metric=metric_callable).fit(X) + + # The set of valid metrics for KDTree at the time of writing this test is a + # strict subset of those supported in BallTree + metrics_not_kd = list(set(BallTree.valid_metrics) - set(KDTree.valid_metrics)) + if len(metrics_not_kd) > 0: + with pytest.raises(ValueError, match=msg): + HDBSCAN(algorithm="kd_tree", metric=metrics_not_kd[0]).fit(X) + + +def test_hdbscan_too_many_min_samples(): + """ + Tests that HDBSCAN correctly raises an error when setting `min_samples` + larger than the number of samples. + """ + hdb = HDBSCAN(min_samples=len(X) + 1) + msg = r"min_samples (.*) must be at most" + with pytest.raises(ValueError, match=msg): + hdb.fit(X) + + +def test_hdbscan_precomputed_dense_nan(): + """ + Tests that HDBSCAN correctly raises an error when providing precomputed + distances with `np.nan` values. + """ + X_nan = X.copy() + X_nan[0, 0] = np.nan + msg = "np.nan values found in precomputed-dense" + hdb = HDBSCAN(metric="precomputed") + with pytest.raises(ValueError, match=msg): + hdb.fit(X_nan) + + +@pytest.mark.parametrize("allow_single_cluster", [True, False]) +@pytest.mark.parametrize("epsilon", [0, 0.1]) +def test_labelling_distinct(global_random_seed, allow_single_cluster, epsilon): + """ + Tests that the `_do_labelling` helper function correctly assigns labels. + """ + n_samples = 48 + X, y = make_blobs( + n_samples, + random_state=global_random_seed, + # Ensure the clusters are distinct with no overlap + centers=[ + [0, 0], + [10, 0], + [0, 10], + ], + ) + + est = HDBSCAN().fit(X) + condensed_tree = _condense_tree( + est._single_linkage_tree_, min_cluster_size=est.min_cluster_size + ) + clusters = {n_samples + 2, n_samples + 3, n_samples + 4} + cluster_label_map = {n_samples + 2: 0, n_samples + 3: 1, n_samples + 4: 2} + labels = _do_labelling( + condensed_tree=condensed_tree, + clusters=clusters, + cluster_label_map=cluster_label_map, + allow_single_cluster=allow_single_cluster, + cluster_selection_epsilon=epsilon, + ) + + first_with_label = {_y: np.where(y == _y)[0][0] for _y in list(set(y))} + y_to_labels = {_y: labels[first_with_label[_y]] for _y in list(set(y))} + aligned_target = np.vectorize(y_to_labels.get)(y) + assert_array_equal(labels, aligned_target) + + +def test_labelling_thresholding(): + """ + Tests that the `_do_labelling` helper function correctly thresholds the + incoming lambda values given various `cluster_selection_epsilon` values. + """ + n_samples = 5 + MAX_LAMBDA = 1.5 + condensed_tree = np.array( + [ + (5, 2, MAX_LAMBDA, 1), + (5, 1, 0.1, 1), + (5, 0, MAX_LAMBDA, 1), + (5, 3, 0.2, 1), + (5, 4, 0.3, 1), + ], + dtype=CONDENSED_dtype, + ) + labels = _do_labelling( + condensed_tree=condensed_tree, + clusters={n_samples}, + cluster_label_map={n_samples: 0, n_samples + 1: 1}, + allow_single_cluster=True, + cluster_selection_epsilon=1, + ) + num_noise = condensed_tree["value"] < 1 + assert sum(num_noise) == sum(labels == -1) + + labels = _do_labelling( + condensed_tree=condensed_tree, + clusters={n_samples}, + cluster_label_map={n_samples: 0, n_samples + 1: 1}, + allow_single_cluster=True, + cluster_selection_epsilon=0, + ) + # The threshold should be calculated per-sample based on the largest + # lambda of any simbling node. In this case, all points are siblings + # and the largest value is exactly MAX_LAMBDA. + num_noise = condensed_tree["value"] < MAX_LAMBDA + assert sum(num_noise) == sum(labels == -1) + + +@pytest.mark.parametrize("store_centers", ["centroid", "medoid"]) +def test_hdbscan_error_precomputed_and_store_centers(store_centers): + """Check that we raise an error if the centers are requested together with + a precomputed input matrix. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/27893 + """ + rng = np.random.RandomState(0) + X = rng.random((100, 2)) + X_dist = euclidean_distances(X) + err_msg = "Cannot store centers when using a precomputed distance matrix." + with pytest.raises(ValueError, match=err_msg): + HDBSCAN(metric="precomputed", store_centers=store_centers).fit(X_dist) + + +@pytest.mark.parametrize("valid_algo", ["auto", "brute"]) +def test_hdbscan_cosine_metric_valid_algorithm(valid_algo): + """Test that HDBSCAN works with the "cosine" metric when the algorithm is set + to "brute" or "auto". + + Non-regression test for issue #28631 + """ + HDBSCAN(metric="cosine", algorithm=valid_algo).fit_predict(X) + + +@pytest.mark.parametrize("invalid_algo", ["kd_tree", "ball_tree"]) +def test_hdbscan_cosine_metric_invalid_algorithm(invalid_algo): + """Test that HDBSCAN raises an informative error is raised when an unsupported + algorithm is used with the "cosine" metric. + """ + hdbscan = HDBSCAN(metric="cosine", algorithm=invalid_algo) + with pytest.raises(ValueError, match="cosine is not a valid metric"): + hdbscan.fit_predict(X) diff --git a/sklearn/cluster/tests/test_hierarchical.py b/sklearn/cluster/tests/test_hierarchical.py index 1f835a52f0105..222d4f6cd9264 100644 --- a/sklearn/cluster/tests/test_hierarchical.py +++ b/sklearn/cluster/tests/test_hierarchical.py @@ -2,48 +2,62 @@ Several basic tests for hierarchical clustering procedures """ -# Authors: Vincent Michel, 2010, Gael Varoquaux 2012, -# Matteo Visconti di Oleggio Castello 2014 -# License: BSD 3 clause -from tempfile import mkdtemp + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import itertools import shutil -import pytest from functools import partial +from tempfile import mkdtemp import numpy as np -from scipy import sparse +import pytest from scipy.cluster import hierarchy - -from sklearn.metrics.cluster import adjusted_rand_score -from sklearn.utils._testing import assert_almost_equal -from sklearn.utils._testing import assert_array_almost_equal -from sklearn.utils._testing import ignore_warnings - -from sklearn.cluster import ward_tree -from sklearn.cluster import AgglomerativeClustering, FeatureAgglomeration -from sklearn.cluster._agglomerative import (_hc_cut, _TREE_BUILDERS, - linkage_tree, - _fix_connectivity) +from scipy.sparse.csgraph import connected_components + +from sklearn.cluster import AgglomerativeClustering, FeatureAgglomeration, ward_tree +from sklearn.cluster._agglomerative import ( + _TREE_BUILDERS, + _fix_connectivity, + _hc_cut, + linkage_tree, +) +from sklearn.cluster._hierarchical_fast import ( + average_merge, + max_merge, + mst_linkage_core, +) +from sklearn.datasets import make_circles, make_moons from sklearn.feature_extraction.image import grid_to_graph -from sklearn.metrics.pairwise import PAIRED_DISTANCES, cosine_distances,\ - manhattan_distances, pairwise_distances -from sklearn.metrics.cluster import normalized_mutual_info_score +from sklearn.metrics import DistanceMetric +from sklearn.metrics.cluster import adjusted_rand_score, normalized_mutual_info_score +from sklearn.metrics.pairwise import ( + PAIRED_DISTANCES, + cosine_distances, + manhattan_distances, + pairwise_distances, +) +from sklearn.metrics.tests.test_dist_metrics import METRICS_DEFAULT_PARAMS from sklearn.neighbors import kneighbors_graph -from sklearn.cluster._hierarchical_fast import average_merge, max_merge from sklearn.utils._fast_dict import IntFloatDict -from sklearn.utils._testing import assert_array_equal -from sklearn.datasets import make_moons, make_circles +from sklearn.utils._testing import ( + assert_almost_equal, + assert_array_almost_equal, + assert_array_equal, + create_memmap_backed_data, + ignore_warnings, +) +from sklearn.utils.fixes import LIL_CONTAINERS def test_linkage_misc(): # Misc tests on linkage rng = np.random.RandomState(42) X = rng.normal(size=(5, 5)) - with pytest.raises(ValueError): - AgglomerativeClustering(linkage='foo').fit(X) with pytest.raises(ValueError): - linkage_tree(X, linkage='foo') + linkage_tree(X, linkage="foo") with pytest.raises(ValueError): linkage_tree(X, connectivity=np.ones((4, 4))) @@ -71,8 +85,9 @@ def test_structured_linkage_tree(): X = rng.randn(50, 100) connectivity = grid_to_graph(*mask.shape) for tree_builder in _TREE_BUILDERS.values(): - children, n_components, n_leaves, parent = \ - tree_builder(X.T, connectivity=connectivity) + children, n_components, n_leaves, parent = tree_builder( + X.T, connectivity=connectivity + ) n_nodes = 2 * X.shape[1] - 1 assert len(children) + n_leaves == n_nodes # Check that ward_tree raises a ValueError with a connectivity matrix @@ -93,8 +108,7 @@ def test_unstructured_linkage_tree(): # raising a warning and testing the warning code with ignore_warnings(): with pytest.warns(UserWarning): - children, n_nodes, n_leaves, parent = ward_tree( - this_X.T, n_clusters=10) + children, n_nodes, n_leaves, parent = ward_tree(this_X.T, n_clusters=10) n_nodes = 2 * X.shape[1] - 1 assert len(children) + n_leaves == n_nodes @@ -103,7 +117,8 @@ def test_unstructured_linkage_tree(): with ignore_warnings(): with pytest.warns(UserWarning): children, n_nodes, n_leaves, parent = tree_builder( - this_X.T, n_clusters=10) + this_X.T, n_clusters=10 + ) n_nodes = 2 * X.shape[1] - 1 assert len(children) + n_leaves == n_nodes @@ -116,41 +131,27 @@ def test_height_linkage_tree(): connectivity = grid_to_graph(*mask.shape) for linkage_func in _TREE_BUILDERS.values(): children, n_nodes, n_leaves, parent = linkage_func( - X.T, connectivity=connectivity) + X.T, connectivity=connectivity + ) n_nodes = 2 * X.shape[1] - 1 assert len(children) + n_leaves == n_nodes -def test_agglomerative_clustering_wrong_arg_memory(): - # Test either if an error is raised when memory is not - # either a str or a joblib.Memory instance - rng = np.random.RandomState(0) - n_samples = 100 - X = rng.randn(n_samples, 50) - memory = 5 - clustering = AgglomerativeClustering(memory=memory) - with pytest.raises(ValueError): - clustering.fit(X) - - def test_zero_cosine_linkage_tree(): # Check that zero vectors in X produce an error when # 'cosine' affinity is used - X = np.array([[0, 1], - [0, 0]]) - msg = 'Cosine affinity cannot be used when X contains zero vectors' + X = np.array([[0, 1], [0, 0]]) + msg = "Cosine affinity cannot be used when X contains zero vectors" with pytest.raises(ValueError, match=msg): - linkage_tree(X, affinity='cosine') + linkage_tree(X, affinity="cosine") -@pytest.mark.parametrize('n_clusters, distance_threshold', - [(None, 0.5), (10, None)]) -@pytest.mark.parametrize('compute_distances', [True, False]) -@pytest.mark.parametrize('linkage', ["ward", "complete", "average", "single"]) -def test_agglomerative_clustering_distances(n_clusters, - compute_distances, - distance_threshold, - linkage): +@pytest.mark.parametrize("n_clusters, distance_threshold", [(None, 0.5), (10, None)]) +@pytest.mark.parametrize("compute_distances", [True, False]) +@pytest.mark.parametrize("linkage", ["ward", "complete", "average", "single"]) +def test_agglomerative_clustering_distances( + n_clusters, compute_distances, distance_threshold, linkage +): # Check that when `compute_distances` is True or `distance_threshold` is # given, the fitted model has an attribute `distances_`. rng = np.random.RandomState(0) @@ -159,41 +160,46 @@ def test_agglomerative_clustering_distances(n_clusters, X = rng.randn(n_samples, 50) connectivity = grid_to_graph(*mask.shape) - clustering = AgglomerativeClustering(n_clusters=n_clusters, - connectivity=connectivity, - linkage=linkage, - distance_threshold=distance_threshold, - compute_distances=compute_distances) + clustering = AgglomerativeClustering( + n_clusters=n_clusters, + connectivity=connectivity, + linkage=linkage, + distance_threshold=distance_threshold, + compute_distances=compute_distances, + ) clustering.fit(X) if compute_distances or (distance_threshold is not None): - assert hasattr(clustering, 'distances_') + assert hasattr(clustering, "distances_") n_children = clustering.children_.shape[0] n_nodes = n_children + 1 - assert clustering.distances_.shape == (n_nodes-1, ) + assert clustering.distances_.shape == (n_nodes - 1,) else: - assert not hasattr(clustering, 'distances_') + assert not hasattr(clustering, "distances_") -def test_agglomerative_clustering(): +@pytest.mark.parametrize("lil_container", LIL_CONTAINERS) +def test_agglomerative_clustering(global_random_seed, lil_container): # Check that we obtain the correct number of clusters with # agglomerative clustering. - rng = np.random.RandomState(0) + rng = np.random.RandomState(global_random_seed) mask = np.ones([10, 10], dtype=bool) n_samples = 100 X = rng.randn(n_samples, 50) connectivity = grid_to_graph(*mask.shape) for linkage in ("ward", "complete", "average", "single"): - clustering = AgglomerativeClustering(n_clusters=10, - connectivity=connectivity, - linkage=linkage) + clustering = AgglomerativeClustering( + n_clusters=10, connectivity=connectivity, linkage=linkage + ) clustering.fit(X) # test caching try: tempdir = mkdtemp() clustering = AgglomerativeClustering( - n_clusters=10, connectivity=connectivity, + n_clusters=10, + connectivity=connectivity, memory=tempdir, - linkage=linkage) + linkage=linkage, + ) clustering.fit(X) labels = clustering.labels_ assert np.size(np.unique(labels)) == 10 @@ -201,22 +207,22 @@ def test_agglomerative_clustering(): shutil.rmtree(tempdir) # Turn caching off now clustering = AgglomerativeClustering( - n_clusters=10, connectivity=connectivity, linkage=linkage) + n_clusters=10, connectivity=connectivity, linkage=linkage + ) # Check that we obtain the same solution with early-stopping of the # tree building clustering.compute_full_tree = False clustering.fit(X) - assert_almost_equal(normalized_mutual_info_score(clustering.labels_, - labels), 1) + assert_almost_equal(normalized_mutual_info_score(clustering.labels_, labels), 1) clustering.connectivity = None clustering.fit(X) assert np.size(np.unique(clustering.labels_)) == 10 # Check that we raise a TypeError on dense matrices clustering = AgglomerativeClustering( n_clusters=10, - connectivity=sparse.lil_matrix( - connectivity.toarray()[:10, :10]), - linkage=linkage) + connectivity=lil_container(connectivity.toarray()[:10, :10]), + linkage=linkage, + ) with pytest.raises(ValueError): clustering.fit(X) @@ -225,48 +231,60 @@ def test_agglomerative_clustering(): clustering = AgglomerativeClustering( n_clusters=10, connectivity=connectivity.toarray(), - affinity="manhattan", - linkage="ward") + metric="manhattan", + linkage="ward", + ) with pytest.raises(ValueError): clustering.fit(X) # Test using another metric than euclidean works with linkage complete - for affinity in PAIRED_DISTANCES.keys(): + for metric in PAIRED_DISTANCES.keys(): # Compare our (structured) implementation to scipy clustering = AgglomerativeClustering( n_clusters=10, connectivity=np.ones((n_samples, n_samples)), - affinity=affinity, - linkage="complete") + metric=metric, + linkage="complete", + ) clustering.fit(X) clustering2 = AgglomerativeClustering( - n_clusters=10, - connectivity=None, - affinity=affinity, - linkage="complete") + n_clusters=10, connectivity=None, metric=metric, linkage="complete" + ) clustering2.fit(X) - assert_almost_equal(normalized_mutual_info_score(clustering2.labels_, - clustering.labels_), - 1) + assert_almost_equal( + normalized_mutual_info_score(clustering2.labels_, clustering.labels_), 1 + ) # Test that using a distance matrix (affinity = 'precomputed') has same # results (with connectivity constraints) - clustering = AgglomerativeClustering(n_clusters=10, - connectivity=connectivity, - linkage="complete") + clustering = AgglomerativeClustering( + n_clusters=10, connectivity=connectivity, linkage="complete" + ) clustering.fit(X) X_dist = pairwise_distances(X) - clustering2 = AgglomerativeClustering(n_clusters=10, - connectivity=connectivity, - affinity='precomputed', - linkage="complete") + clustering2 = AgglomerativeClustering( + n_clusters=10, + connectivity=connectivity, + metric="precomputed", + linkage="complete", + ) clustering2.fit(X_dist) assert_array_equal(clustering.labels_, clustering2.labels_) -def test_ward_agglomeration(): - # Check that we obtain the correct solution in a simplistic case +def test_agglomerative_clustering_memory_mapped(): + """AgglomerativeClustering must work on mem-mapped dataset. + + Non-regression test for issue #19875. + """ rng = np.random.RandomState(0) + Xmm = create_memmap_backed_data(rng.randn(50, 100)) + AgglomerativeClustering(metric="euclidean", linkage="single").fit(Xmm) + + +def test_ward_agglomeration(global_random_seed): + # Check that we obtain the correct solution in a simplistic case + rng = np.random.RandomState(global_random_seed) mask = np.ones([10, 10], dtype=bool) X = rng.randn(50, 100) connectivity = grid_to_graph(*mask.shape) @@ -288,17 +306,18 @@ def test_ward_agglomeration(): def test_single_linkage_clustering(): # Check that we get the correct result in two emblematic cases moons, moon_labels = make_moons(noise=0.05, random_state=42) - clustering = AgglomerativeClustering(n_clusters=2, linkage='single') + clustering = AgglomerativeClustering(n_clusters=2, linkage="single") clustering.fit(moons) - assert_almost_equal(normalized_mutual_info_score(clustering.labels_, - moon_labels), 1) + assert_almost_equal( + normalized_mutual_info_score(clustering.labels_, moon_labels), 1 + ) - circles, circle_labels = make_circles(factor=0.5, noise=0.025, - random_state=42) - clustering = AgglomerativeClustering(n_clusters=2, linkage='single') + circles, circle_labels = make_circles(factor=0.5, noise=0.025, random_state=42) + clustering = AgglomerativeClustering(n_clusters=2, linkage="single") clustering.fit(circles) - assert_almost_equal(normalized_mutual_info_score(clustering.labels_, - circle_labels), 1) + assert_almost_equal( + normalized_mutual_info_score(clustering.labels_, circle_labels), 1 + ) def assess_same_labelling(cut1, cut2): @@ -313,31 +332,34 @@ def assess_same_labelling(cut1, cut2): assert (co_clust[0] == co_clust[1]).all() -def test_sparse_scikit_vs_scipy(): +def test_sparse_scikit_vs_scipy(global_random_seed): # Test scikit linkage with full connectivity (i.e. unstructured) vs scipy n, p, k = 10, 5, 3 - rng = np.random.RandomState(0) + rng = np.random.RandomState(global_random_seed) # Not using a lil_matrix here, just to check that non sparse # matrices are well handled connectivity = np.ones((n, n)) for linkage in _TREE_BUILDERS.keys(): for i in range(5): - X = .1 * rng.normal(size=(n, p)) - X -= 4. * np.arange(n)[:, np.newaxis] + X = 0.1 * rng.normal(size=(n, p)) + X -= 4.0 * np.arange(n)[:, np.newaxis] X -= X.mean(axis=1)[:, np.newaxis] out = hierarchy.linkage(X, method=linkage) children_ = out[:, :2].astype(int, copy=False) children, _, n_leaves, _ = _TREE_BUILDERS[linkage]( - X, connectivity=connectivity) + X, connectivity=connectivity + ) # Sort the order of child nodes per row for consistency children.sort(axis=1) - assert_array_equal(children, children_, 'linkage tree differs' - ' from scipy impl for' - ' linkage: ' + linkage) + assert_array_equal( + children, + children_, + "linkage tree differs from scipy impl for linkage: " + linkage, + ) cut = _hc_cut(k, children, n_leaves) cut_ = _hc_cut(k, children_, n_leaves) @@ -350,82 +372,113 @@ def test_sparse_scikit_vs_scipy(): # Make sure our custom mst_linkage_core gives # the same results as scipy's builtin -@pytest.mark.parametrize('seed', range(5)) -def test_vector_scikit_single_vs_scipy_single(seed): +def test_vector_scikit_single_vs_scipy_single(global_random_seed): n_samples, n_features, n_clusters = 10, 5, 3 - rng = np.random.RandomState(seed) - X = .1 * rng.normal(size=(n_samples, n_features)) - X -= 4. * np.arange(n_samples)[:, np.newaxis] + rng = np.random.RandomState(global_random_seed) + X = 0.1 * rng.normal(size=(n_samples, n_features)) + X -= 4.0 * np.arange(n_samples)[:, np.newaxis] X -= X.mean(axis=1)[:, np.newaxis] - out = hierarchy.linkage(X, method='single') + out = hierarchy.linkage(X, method="single") children_scipy = out[:, :2].astype(int) - children, _, n_leaves, _ = _TREE_BUILDERS['single'](X) + children, _, n_leaves, _ = _TREE_BUILDERS["single"](X) # Sort the order of child nodes per row for consistency children.sort(axis=1) - assert_array_equal(children, children_scipy, - 'linkage tree differs' - ' from scipy impl for' - ' single linkage.') + assert_array_equal( + children, + children_scipy, + "linkage tree differs from scipy impl for single linkage.", + ) cut = _hc_cut(n_clusters, children, n_leaves) cut_scipy = _hc_cut(n_clusters, children_scipy, n_leaves) assess_same_labelling(cut, cut_scipy) +@pytest.mark.parametrize("metric_param_grid", METRICS_DEFAULT_PARAMS) +def test_mst_linkage_core_memory_mapped(metric_param_grid): + """The MST-LINKAGE-CORE algorithm must work on mem-mapped dataset. + + Non-regression test for issue #19875. + """ + rng = np.random.RandomState(seed=1) + X = rng.normal(size=(20, 4)) + Xmm = create_memmap_backed_data(X) + metric, param_grid = metric_param_grid + keys = param_grid.keys() + for vals in itertools.product(*param_grid.values()): + kwargs = dict(zip(keys, vals)) + distance_metric = DistanceMetric.get_metric(metric, **kwargs) + mst = mst_linkage_core(X, distance_metric) + mst_mm = mst_linkage_core(Xmm, distance_metric) + np.testing.assert_equal(mst, mst_mm) + + def test_identical_points(): # Ensure identical points are handled correctly when using mst with # a sparse connectivity matrix - X = np.array([[0, 0, 0], [0, 0, 0], - [1, 1, 1], [1, 1, 1], - [2, 2, 2], [2, 2, 2]]) + X = np.array([[0, 0, 0], [0, 0, 0], [1, 1, 1], [1, 1, 1], [2, 2, 2], [2, 2, 2]]) true_labels = np.array([0, 0, 1, 1, 2, 2]) connectivity = kneighbors_graph(X, n_neighbors=3, include_self=False) connectivity = 0.5 * (connectivity + connectivity.T) - connectivity, n_components = _fix_connectivity(X, - connectivity, - 'euclidean') - - for linkage in ('single', 'average', 'average', 'ward'): - clustering = AgglomerativeClustering(n_clusters=3, - linkage=linkage, - connectivity=connectivity) + connectivity, n_components = _fix_connectivity(X, connectivity, "euclidean") + + for linkage in ("single", "average", "average", "ward"): + clustering = AgglomerativeClustering( + n_clusters=3, linkage=linkage, connectivity=connectivity + ) clustering.fit(X) - assert_almost_equal(normalized_mutual_info_score(clustering.labels_, - true_labels), 1) + assert_almost_equal( + normalized_mutual_info_score(clustering.labels_, true_labels), 1 + ) def test_connectivity_propagation(): # Check that connectivity in the ward tree is propagated correctly during # merging. - X = np.array([(.014, .120), (.014, .099), (.014, .097), - (.017, .153), (.017, .153), (.018, .153), - (.018, .153), (.018, .153), (.018, .153), - (.018, .153), (.018, .153), (.018, .153), - (.018, .152), (.018, .149), (.018, .144)]) + X = np.array( + [ + (0.014, 0.120), + (0.014, 0.099), + (0.014, 0.097), + (0.017, 0.153), + (0.017, 0.153), + (0.018, 0.153), + (0.018, 0.153), + (0.018, 0.153), + (0.018, 0.153), + (0.018, 0.153), + (0.018, 0.153), + (0.018, 0.153), + (0.018, 0.152), + (0.018, 0.149), + (0.018, 0.144), + ] + ) connectivity = kneighbors_graph(X, 10, include_self=False) ward = AgglomerativeClustering( - n_clusters=4, connectivity=connectivity, linkage='ward') + n_clusters=4, connectivity=connectivity, linkage="ward" + ) # If changes are not propagated correctly, fit crashes with an # IndexError ward.fit(X) -def test_ward_tree_children_order(): +def test_ward_tree_children_order(global_random_seed): # Check that children are ordered in the same way for both structured and # unstructured versions of ward_tree. # test on five random datasets n, p = 10, 5 - rng = np.random.RandomState(0) + rng = np.random.RandomState(global_random_seed) connectivity = np.ones((n, n)) for i in range(5): - X = .1 * rng.normal(size=(n, p)) - X -= 4. * np.arange(n)[:, np.newaxis] + X = 0.1 * rng.normal(size=(n, p)) + X -= 4.0 * np.arange(n)[:, np.newaxis] X -= X.mean(axis=1)[:, np.newaxis] out_unstructured = ward_tree(X) @@ -434,23 +487,22 @@ def test_ward_tree_children_order(): assert_array_equal(out_unstructured[0], out_structured[0]) -def test_ward_linkage_tree_return_distance(): +def test_ward_linkage_tree_return_distance(global_random_seed): # Test return_distance option on linkage and ward trees # test that return_distance when set true, gives same # output on both structured and unstructured clustering. n, p = 10, 5 - rng = np.random.RandomState(0) + rng = np.random.RandomState(global_random_seed) connectivity = np.ones((n, n)) for i in range(5): - X = .1 * rng.normal(size=(n, p)) - X -= 4. * np.arange(n)[:, np.newaxis] + X = 0.1 * rng.normal(size=(n, p)) + X -= 4.0 * np.arange(n)[:, np.newaxis] X -= X.mean(axis=1)[:, np.newaxis] out_unstructured = ward_tree(X, return_distance=True) - out_structured = ward_tree(X, connectivity=connectivity, - return_distance=True) + out_structured = ward_tree(X, connectivity=connectivity, return_distance=True) # get children children_unstructured = out_unstructured[0] @@ -465,55 +517,68 @@ def test_ward_linkage_tree_return_distance(): assert_array_almost_equal(dist_unstructured, dist_structured) - for linkage in ['average', 'complete', 'single']: + for linkage in ["average", "complete", "single"]: structured_items = linkage_tree( - X, connectivity=connectivity, linkage=linkage, - return_distance=True)[-1] - unstructured_items = linkage_tree( - X, linkage=linkage, return_distance=True)[-1] + X, connectivity=connectivity, linkage=linkage, return_distance=True + )[-1] + unstructured_items = linkage_tree(X, linkage=linkage, return_distance=True)[ + -1 + ] structured_dist = structured_items[-1] unstructured_dist = unstructured_items[-1] structured_children = structured_items[0] unstructured_children = unstructured_items[0] assert_array_almost_equal(structured_dist, unstructured_dist) - assert_array_almost_equal( - structured_children, unstructured_children) + assert_array_almost_equal(structured_children, unstructured_children) # test on the following dataset where we know the truth # taken from scipy/cluster/tests/hierarchy_test_data.py - X = np.array([[1.43054825, -7.5693489], - [6.95887839, 6.82293382], - [2.87137846, -9.68248579], - [7.87974764, -6.05485803], - [8.24018364, -6.09495602], - [7.39020262, 8.54004355]]) + X = np.array( + [ + [1.43054825, -7.5693489], + [6.95887839, 6.82293382], + [2.87137846, -9.68248579], + [7.87974764, -6.05485803], + [8.24018364, -6.09495602], + [7.39020262, 8.54004355], + ] + ) # truth - linkage_X_ward = np.array([[3., 4., 0.36265956, 2.], - [1., 5., 1.77045373, 2.], - [0., 2., 2.55760419, 2.], - [6., 8., 9.10208346, 4.], - [7., 9., 24.7784379, 6.]]) + linkage_X_ward = np.array( + [ + [3.0, 4.0, 0.36265956, 2.0], + [1.0, 5.0, 1.77045373, 2.0], + [0.0, 2.0, 2.55760419, 2.0], + [6.0, 8.0, 9.10208346, 4.0], + [7.0, 9.0, 24.7784379, 6.0], + ] + ) linkage_X_complete = np.array( - [[3., 4., 0.36265956, 2.], - [1., 5., 1.77045373, 2.], - [0., 2., 2.55760419, 2.], - [6., 8., 6.96742194, 4.], - [7., 9., 18.77445997, 6.]]) + [ + [3.0, 4.0, 0.36265956, 2.0], + [1.0, 5.0, 1.77045373, 2.0], + [0.0, 2.0, 2.55760419, 2.0], + [6.0, 8.0, 6.96742194, 4.0], + [7.0, 9.0, 18.77445997, 6.0], + ] + ) linkage_X_average = np.array( - [[3., 4., 0.36265956, 2.], - [1., 5., 1.77045373, 2.], - [0., 2., 2.55760419, 2.], - [6., 8., 6.55832839, 4.], - [7., 9., 15.44089605, 6.]]) + [ + [3.0, 4.0, 0.36265956, 2.0], + [1.0, 5.0, 1.77045373, 2.0], + [0.0, 2.0, 2.55760419, 2.0], + [6.0, 8.0, 6.55832839, 4.0], + [7.0, 9.0, 15.44089605, 6.0], + ] + ) n_samples, n_features = np.shape(X) connectivity_X = np.ones((n_samples, n_samples)) out_X_unstructured = ward_tree(X, return_distance=True) - out_X_structured = ward_tree(X, connectivity=connectivity_X, - return_distance=True) + out_X_structured = ward_tree(X, connectivity=connectivity_X, return_distance=True) # check that the labels are the same assert_array_equal(linkage_X_ward[:, :2], out_X_unstructured[0]) @@ -523,14 +588,13 @@ def test_ward_linkage_tree_return_distance(): assert_array_almost_equal(linkage_X_ward[:, 2], out_X_unstructured[4]) assert_array_almost_equal(linkage_X_ward[:, 2], out_X_structured[4]) - linkage_options = ['complete', 'average', 'single'] + linkage_options = ["complete", "average", "single"] X_linkage_truth = [linkage_X_complete, linkage_X_average] - for (linkage, X_truth) in zip(linkage_options, X_linkage_truth): - out_X_unstructured = linkage_tree( - X, return_distance=True, linkage=linkage) + for linkage, X_truth in zip(linkage_options, X_linkage_truth): + out_X_unstructured = linkage_tree(X, return_distance=True, linkage=linkage) out_X_structured = linkage_tree( - X, connectivity=connectivity_X, linkage=linkage, - return_distance=True) + X, connectivity=connectivity_X, linkage=linkage, return_distance=True + ) # check that the labels are the same assert_array_equal(X_truth[:, :2], out_X_unstructured[0]) @@ -549,7 +613,7 @@ def test_connectivity_fixing_non_lil(): # create a mask with several components to force connectivity fixing m = np.array([[True, False], [False, True]]) c = grid_to_graph(n_x=2, n_y=2, mask=m) - w = AgglomerativeClustering(connectivity=c, linkage='ward') + w = AgglomerativeClustering(connectivity=c, linkage="ward") with pytest.warns(UserWarning): w.fit(x) @@ -577,8 +641,8 @@ def test_connectivity_callable(): connectivity = kneighbors_graph(X, 3, include_self=False) aglc1 = AgglomerativeClustering(connectivity=connectivity) aglc2 = AgglomerativeClustering( - connectivity=partial(kneighbors_graph, n_neighbors=3, - include_self=False)) + connectivity=partial(kneighbors_graph, n_neighbors=3, include_self=False) + ) aglc1.fit(X) aglc2.fit(X) assert_array_equal(aglc1.labels_, aglc2.labels_) @@ -615,8 +679,7 @@ def test_compute_full_tree(): n_clusters = 101 X = rng.randn(200, 2) connectivity = kneighbors_graph(X, 10, include_self=False) - agc = AgglomerativeClustering(n_clusters=n_clusters, - connectivity=connectivity) + agc = AgglomerativeClustering(n_clusters=n_clusters, connectivity=connectivity) agc.fit(X) n_samples = X.shape[0] n_nodes = agc.children_.shape[0] @@ -632,20 +695,7 @@ def test_n_components(): connectivity = np.eye(5) for linkage_func in _TREE_BUILDERS.values(): - assert ignore_warnings(linkage_func)(X, connectivity)[1] == 5 - - -def test_agg_n_clusters(): - # Test that an error is raised when n_clusters <= 0 - - rng = np.random.RandomState(0) - X = rng.rand(20, 10) - for n_clus in [-1, 0]: - agc = AgglomerativeClustering(n_clusters=n_clus) - msg = ("n_clusters should be an integer greater than 0." - " %s was provided." % str(agc.n_clusters)) - with pytest.raises(ValueError, match=msg): - agc.fit(X) + assert ignore_warnings(linkage_func)(X, connectivity=connectivity)[1] == 5 def test_affinity_passed_to_fix_connectivity(): @@ -657,8 +707,7 @@ def test_affinity_passed_to_fix_connectivity(): X = rng.randn(size, size) mask = np.array([True, False, False, True]) - connectivity = grid_to_graph(n_x=size, n_y=size, - mask=mask, return_as=np.ndarray) + connectivity = grid_to_graph(n_x=size, n_y=size, mask=mask, return_as=np.ndarray) class FakeAffinity: def __init__(self): @@ -675,11 +724,11 @@ def increment(self, *args, **kwargs): assert fa.counter == 3 -@pytest.mark.parametrize('linkage', ['ward', 'complete', 'average']) -def test_agglomerative_clustering_with_distance_threshold(linkage): +@pytest.mark.parametrize("linkage", ["ward", "complete", "average"]) +def test_agglomerative_clustering_with_distance_threshold(linkage, global_random_seed): # Check that we obtain the correct number of clusters with # agglomerative clustering with distance_threshold. - rng = np.random.RandomState(0) + rng = np.random.RandomState(global_random_seed) mask = np.ones([10, 10], dtype=bool) n_samples = 100 X = rng.randn(n_samples, 50) @@ -690,83 +739,86 @@ def test_agglomerative_clustering_with_distance_threshold(linkage): clustering = AgglomerativeClustering( n_clusters=None, distance_threshold=distance_threshold, - connectivity=conn, linkage=linkage) + connectivity=conn, + linkage=linkage, + ) clustering.fit(X) clusters_produced = clustering.labels_ num_clusters_produced = len(np.unique(clustering.labels_)) # test if the clusters produced match the point in the linkage tree # where the distance exceeds the threshold tree_builder = _TREE_BUILDERS[linkage] - children, n_components, n_leaves, parent, distances = \ - tree_builder(X, connectivity=conn, n_clusters=None, - return_distance=True) - num_clusters_at_threshold = np.count_nonzero( - distances >= distance_threshold) + 1 + children, n_components, n_leaves, parent, distances = tree_builder( + X, connectivity=conn, n_clusters=None, return_distance=True + ) + num_clusters_at_threshold = ( + np.count_nonzero(distances >= distance_threshold) + 1 + ) # test number of clusters produced assert num_clusters_at_threshold == num_clusters_produced # test clusters produced - clusters_at_threshold = _hc_cut(n_clusters=num_clusters_produced, - children=children, - n_leaves=n_leaves) - assert np.array_equiv(clusters_produced, - clusters_at_threshold) + clusters_at_threshold = _hc_cut( + n_clusters=num_clusters_produced, children=children, n_leaves=n_leaves + ) + assert np.array_equiv(clusters_produced, clusters_at_threshold) -def test_small_distance_threshold(): - rng = np.random.RandomState(0) +def test_small_distance_threshold(global_random_seed): + rng = np.random.RandomState(global_random_seed) n_samples = 10 X = rng.randint(-300, 300, size=(n_samples, 3)) # this should result in all data in their own clusters, given that # their pairwise distances are bigger than .1 (which may not be the case # with a different random seed). clustering = AgglomerativeClustering( - n_clusters=None, - distance_threshold=1., - linkage="single").fit(X) + n_clusters=None, distance_threshold=1.0, linkage="single" + ).fit(X) # check that the pairwise distances are indeed all larger than .1 - all_distances = pairwise_distances(X, metric='minkowski', p=2) + all_distances = pairwise_distances(X, metric="minkowski", p=2) np.fill_diagonal(all_distances, np.inf) - assert np.all(all_distances > .1) + assert np.all(all_distances > 0.1) assert clustering.n_clusters_ == n_samples -def test_cluster_distances_with_distance_threshold(): - rng = np.random.RandomState(0) +def test_cluster_distances_with_distance_threshold(global_random_seed): + rng = np.random.RandomState(global_random_seed) n_samples = 100 X = rng.randint(-10, 10, size=(n_samples, 3)) # check the distances within the clusters and with other clusters distance_threshold = 4 clustering = AgglomerativeClustering( - n_clusters=None, - distance_threshold=distance_threshold, - linkage="single").fit(X) + n_clusters=None, distance_threshold=distance_threshold, linkage="single" + ).fit(X) labels = clustering.labels_ D = pairwise_distances(X, metric="minkowski", p=2) # to avoid taking the 0 diagonal in min() np.fill_diagonal(D, np.inf) for label in np.unique(labels): in_cluster_mask = labels == label - max_in_cluster_distance = (D[in_cluster_mask][:, in_cluster_mask] - .min(axis=0).max()) - min_out_cluster_distance = (D[in_cluster_mask][:, ~in_cluster_mask] - .min(axis=0).min()) + max_in_cluster_distance = ( + D[in_cluster_mask][:, in_cluster_mask].min(axis=0).max() + ) + min_out_cluster_distance = ( + D[in_cluster_mask][:, ~in_cluster_mask].min(axis=0).min() + ) # single data point clusters only have that inf diagonal here if in_cluster_mask.sum() > 1: assert max_in_cluster_distance < distance_threshold assert min_out_cluster_distance >= distance_threshold -@pytest.mark.parametrize('linkage', ['ward', 'complete', 'average']) -@pytest.mark.parametrize(('threshold', 'y_true'), - [(0.5, [1, 0]), (1.0, [1, 0]), (1.5, [0, 0])]) +@pytest.mark.parametrize("linkage", ["ward", "complete", "average"]) +@pytest.mark.parametrize( + ("threshold", "y_true"), [(0.5, [1, 0]), (1.0, [1, 0]), (1.5, [0, 0])] +) def test_agglomerative_clustering_with_distance_threshold_edge_case( - linkage, threshold, y_true): + linkage, threshold, y_true +): # test boundary case of distance_threshold matching the distance X = [[0], [1]] clusterer = AgglomerativeClustering( - n_clusters=None, - distance_threshold=threshold, - linkage=linkage) + n_clusters=None, distance_threshold=threshold, linkage=linkage + ) y_pred = clusterer.fit_predict(X) assert adjusted_rand_score(y_true, y_pred) == 1 @@ -774,18 +826,16 @@ def test_agglomerative_clustering_with_distance_threshold_edge_case( def test_dist_threshold_invalid_parameters(): X = [[0], [1]] with pytest.raises(ValueError, match="Exactly one of "): - AgglomerativeClustering(n_clusters=None, - distance_threshold=None).fit(X) + AgglomerativeClustering(n_clusters=None, distance_threshold=None).fit(X) with pytest.raises(ValueError, match="Exactly one of "): - AgglomerativeClustering(n_clusters=2, - distance_threshold=1).fit(X) + AgglomerativeClustering(n_clusters=2, distance_threshold=1).fit(X) X = [[0], [1]] with pytest.raises(ValueError, match="compute_full_tree must be True if"): - AgglomerativeClustering(n_clusters=None, - distance_threshold=1, - compute_full_tree=False).fit(X) + AgglomerativeClustering( + n_clusters=None, distance_threshold=1, compute_full_tree=False + ).fit(X) def test_invalid_shape_precomputed_dist_matrix(): @@ -793,6 +843,47 @@ def test_invalid_shape_precomputed_dist_matrix(): # and a non square matrix is passed (PR #16257). rng = np.random.RandomState(0) X = rng.rand(5, 3) - with pytest.raises(ValueError, match="Distance matrix should be square, "): - AgglomerativeClustering(affinity='precomputed', - linkage='complete').fit(X) + with pytest.raises( + ValueError, + match=r"Distance matrix should be square, got matrix of shape \(5, 3\)", + ): + AgglomerativeClustering(metric="precomputed", linkage="complete").fit(X) + + +def test_precomputed_connectivity_metric_with_2_connected_components(): + """Check that connecting components works when connectivity and + affinity are both precomputed and the number of connected components is + greater than 1. Non-regression test for #16151. + """ + + connectivity_matrix = np.array( + [ + [0, 1, 1, 0, 0], + [0, 0, 1, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 1], + [0, 0, 0, 0, 0], + ] + ) + # ensure that connectivity_matrix has two connected components + assert connected_components(connectivity_matrix)[0] == 2 + + rng = np.random.RandomState(0) + X = rng.randn(5, 10) + + X_dist = pairwise_distances(X) + clusterer_precomputed = AgglomerativeClustering( + metric="precomputed", connectivity=connectivity_matrix, linkage="complete" + ) + msg = "Completing it to avoid stopping the tree early" + with pytest.warns(UserWarning, match=msg): + clusterer_precomputed.fit(X_dist) + + clusterer = AgglomerativeClustering( + connectivity=connectivity_matrix, linkage="complete" + ) + with pytest.warns(UserWarning, match=msg): + clusterer.fit(X) + + assert_array_equal(clusterer.labels_, clusterer_precomputed.labels_) + assert_array_equal(clusterer.children_, clusterer_precomputed.children_) diff --git a/sklearn/cluster/tests/test_k_means.py b/sklearn/cluster/tests/test_k_means.py index 341b00c5c137f..0ab602d32d133 100644 --- a/sklearn/cluster/tests/test_k_means.py +++ b/sklearn/cluster/tests/test_k_means.py @@ -1,55 +1,63 @@ """Testing for K-means""" + import re import sys +from io import StringIO import numpy as np -from scipy import sparse as sp -from threadpoolctl import threadpool_limits - import pytest +from scipy import sparse as sp -from sklearn.utils._testing import assert_array_equal -from sklearn.utils._testing import assert_array_almost_equal -from sklearn.utils._testing import assert_allclose -from sklearn.utils._testing import assert_almost_equal -from sklearn.utils.fixes import _astype_copy_false from sklearn.base import clone +from sklearn.cluster import KMeans, MiniBatchKMeans, k_means, kmeans_plusplus +from sklearn.cluster._k_means_common import ( + _euclidean_dense_dense_wrapper, + _euclidean_sparse_dense_wrapper, + _inertia_dense, + _inertia_sparse, + _is_same_clustering, + _relocate_empty_clusters_dense, + _relocate_empty_clusters_sparse, +) +from sklearn.cluster._kmeans import _labels_inertia, _mini_batch_step +from sklearn.datasets import make_blobs from sklearn.exceptions import ConvergenceWarning - -from sklearn.utils.extmath import row_norms -from sklearn.metrics import pairwise_distances -from sklearn.metrics import pairwise_distances_argmin +from sklearn.metrics import pairwise_distances, pairwise_distances_argmin from sklearn.metrics.cluster import v_measure_score -from sklearn.cluster import KMeans, k_means, kmeans_plusplus -from sklearn.cluster import MiniBatchKMeans -from sklearn.cluster._kmeans import _labels_inertia -from sklearn.cluster._kmeans import _mini_batch_step -from sklearn.cluster._k_means_fast import _relocate_empty_clusters_dense -from sklearn.cluster._k_means_fast import _relocate_empty_clusters_sparse -from sklearn.cluster._k_means_fast import _euclidean_dense_dense_wrapper -from sklearn.cluster._k_means_fast import _euclidean_sparse_dense_wrapper -from sklearn.cluster._k_means_fast import _inertia_dense -from sklearn.cluster._k_means_fast import _inertia_sparse -from sklearn.datasets import make_blobs -from io import StringIO - +from sklearn.metrics.pairwise import euclidean_distances +from sklearn.utils._testing import ( + assert_allclose, + assert_array_equal, + create_memmap_backed_data, +) +from sklearn.utils.extmath import row_norms +from sklearn.utils.fixes import CSR_CONTAINERS +from sklearn.utils.parallel import _get_threadpool_controller # non centered, sparse centers to check the -centers = np.array([ - [0.0, 5.0, 0.0, 0.0, 0.0], - [1.0, 1.0, 4.0, 0.0, 0.0], - [1.0, 0.0, 0.0, 5.0, 1.0], -]) +centers = np.array( + [ + [0.0, 5.0, 0.0, 0.0, 0.0], + [1.0, 1.0, 4.0, 0.0, 0.0], + [1.0, 0.0, 0.0, 5.0, 1.0], + ] +) n_samples = 100 n_clusters, n_features = centers.shape -X, true_labels = make_blobs(n_samples=n_samples, centers=centers, - cluster_std=1., random_state=42) -X_csr = sp.csr_matrix(X) +X, true_labels = make_blobs( + n_samples=n_samples, centers=centers, cluster_std=1.0, random_state=42 +) +X_as_any_csr = [container(X) for container in CSR_CONTAINERS] +data_containers = [np.array] + CSR_CONTAINERS +data_containers_ids = ( + ["dense", "sparse_matrix", "sparse_array"] + if len(X_as_any_csr) == 2 + else ["dense", "sparse_matrix"] +) -@pytest.mark.parametrize("array_constr", [np.array, sp.csr_matrix], - ids=["dense", "sparse"]) -@pytest.mark.parametrize("algo", ["full", "elkan"]) +@pytest.mark.parametrize("array_constr", data_containers, ids=data_containers_ids) +@pytest.mark.parametrize("algo", ["lloyd", "elkan"]) @pytest.mark.parametrize("dtype", [np.float32, np.float64]) def test_kmeans_results(array_constr, algo, dtype): # Checks that KMeans works as intended on toy dataset by comparing with @@ -72,9 +80,8 @@ def test_kmeans_results(array_constr, algo, dtype): assert kmeans.n_iter_ == expected_n_iter -@pytest.mark.parametrize("array_constr", [np.array, sp.csr_matrix], - ids=['dense', 'sparse']) -@pytest.mark.parametrize("algo", ['full', 'elkan']) +@pytest.mark.parametrize("array_constr", data_containers, ids=data_containers_ids) +@pytest.mark.parametrize("algo", ["lloyd", "elkan"]) def test_kmeans_relocated_clusters(array_constr, algo): # check that empty clusters are relocated as expected X = array_constr([[0, 0], [0.5, 0], [0.5, 1], [1, 1]]) @@ -82,49 +89,62 @@ def test_kmeans_relocated_clusters(array_constr, algo): # second center too far from others points will be empty at first iter init_centers = np.array([[0.5, 0.5], [3, 3]]) - expected_labels = [0, 0, 1, 1] - expected_inertia = 0.25 - expected_centers = [[0.25, 0], [0.75, 1]] - expected_n_iter = 3 - kmeans = KMeans(n_clusters=2, n_init=1, init=init_centers, algorithm=algo) kmeans.fit(X) - assert_array_equal(kmeans.labels_, expected_labels) + expected_n_iter = 3 + expected_inertia = 0.25 assert_allclose(kmeans.inertia_, expected_inertia) - assert_allclose(kmeans.cluster_centers_, expected_centers) assert kmeans.n_iter_ == expected_n_iter - -@pytest.mark.parametrize("array_constr", [np.array, sp.csr_matrix], - ids=["dense", "sparse"]) + # There are two acceptable ways of relocating clusters in this example, the output + # depends on how the argpartition strategy breaks ties. We accept both outputs. + try: + expected_labels = [0, 0, 1, 1] + expected_centers = [[0.25, 0], [0.75, 1]] + assert_array_equal(kmeans.labels_, expected_labels) + assert_allclose(kmeans.cluster_centers_, expected_centers) + except AssertionError: + expected_labels = [1, 1, 0, 0] + expected_centers = [[0.75, 1.0], [0.25, 0.0]] + assert_array_equal(kmeans.labels_, expected_labels) + assert_allclose(kmeans.cluster_centers_, expected_centers) + + +@pytest.mark.parametrize("array_constr", data_containers, ids=data_containers_ids) def test_relocate_empty_clusters(array_constr): # test for the _relocate_empty_clusters_(dense/sparse) helpers # Synthetic dataset with 3 obvious clusters of different sizes - X = np.array( - [-10., -9.5, -9, -8.5, -8, -1, 1, 9, 9.5, 10]).reshape(-1, 1) + X = np.array([-10.0, -9.5, -9, -8.5, -8, -1, 1, 9, 9.5, 10]).reshape(-1, 1) X = array_constr(X) sample_weight = np.ones(10) # centers all initialized to the first point of X - centers_old = np.array([-10., -10, -10]).reshape(-1, 1) + centers_old = np.array([-10.0, -10, -10]).reshape(-1, 1) # With this initialization, all points will be assigned to the first center # At this point a center in centers_new is the weighted sum of the points # it contains if it's not empty, otherwise it is the same as before. centers_new = np.array([-16.5, -10, -10]).reshape(-1, 1) - weight_in_clusters = np.array([10., 0, 0]) + weight_in_clusters = np.array([10.0, 0, 0]) labels = np.zeros(10, dtype=np.int32) if array_constr is np.array: - _relocate_empty_clusters_dense(X, sample_weight, centers_old, - centers_new, weight_in_clusters, labels) + _relocate_empty_clusters_dense( + X, sample_weight, centers_old, centers_new, weight_in_clusters, labels + ) else: - _relocate_empty_clusters_sparse(X.data, X.indices, X.indptr, - sample_weight, centers_old, - centers_new, weight_in_clusters, - labels) + _relocate_empty_clusters_sparse( + X.data, + X.indices, + X.indptr, + sample_weight, + centers_old, + centers_new, + weight_in_clusters, + labels, + ) # The relocation scheme will take the 2 points farthest from the center and # assign them to the 2 empty clusters, i.e. points at 10 and at 9.9. The @@ -134,12 +154,11 @@ def test_relocate_empty_clusters(array_constr): @pytest.mark.parametrize("distribution", ["normal", "blobs"]) -@pytest.mark.parametrize("array_constr", [np.array, sp.csr_matrix], - ids=["dense", "sparse"]) +@pytest.mark.parametrize("array_constr", data_containers, ids=data_containers_ids) @pytest.mark.parametrize("tol", [1e-2, 1e-8, 1e-100, 0]) -def test_kmeans_elkan_results(distribution, array_constr, tol): +def test_kmeans_elkan_results(distribution, array_constr, tol, global_random_seed): # Check that results are identical between lloyd and elkan algorithms - rnd = np.random.RandomState(0) + rnd = np.random.RandomState(global_random_seed) if distribution == "normal": X = rnd.normal(size=(5000, 10)) else: @@ -147,97 +166,104 @@ def test_kmeans_elkan_results(distribution, array_constr, tol): X[X < 0] = 0 X = array_constr(X) - km_full = KMeans(algorithm="full", n_clusters=5, - random_state=0, n_init=1, tol=tol) - km_elkan = KMeans(algorithm="elkan", n_clusters=5, - random_state=0, n_init=1, tol=tol) + km_lloyd = KMeans(n_clusters=5, random_state=global_random_seed, n_init=1, tol=tol) + km_elkan = KMeans( + algorithm="elkan", + n_clusters=5, + random_state=global_random_seed, + n_init=1, + tol=tol, + ) - km_full.fit(X) + km_lloyd.fit(X) km_elkan.fit(X) - assert_allclose(km_elkan.cluster_centers_, km_full.cluster_centers_) - assert_array_equal(km_elkan.labels_, km_full.labels_) - assert km_elkan.n_iter_ == km_full.n_iter_ - assert km_elkan.inertia_ == pytest.approx(km_full.inertia_, rel=1e-6) + assert_allclose(km_elkan.cluster_centers_, km_lloyd.cluster_centers_) + assert_array_equal(km_elkan.labels_, km_lloyd.labels_) + assert km_elkan.n_iter_ == km_lloyd.n_iter_ + assert km_elkan.inertia_ == pytest.approx(km_lloyd.inertia_, rel=1e-6) -@pytest.mark.parametrize("algorithm", ["full", "elkan"]) -def test_kmeans_convergence(algorithm): +@pytest.mark.parametrize("algorithm", ["lloyd", "elkan"]) +def test_kmeans_convergence(algorithm, global_random_seed): # Check that KMeans stops when convergence is reached when tol=0. (#16075) - rnd = np.random.RandomState(0) + rnd = np.random.RandomState(global_random_seed) X = rnd.normal(size=(5000, 10)) max_iter = 300 - km = KMeans(algorithm=algorithm, n_clusters=5, random_state=0, - n_init=1, tol=0, max_iter=max_iter).fit(X) + km = KMeans( + algorithm=algorithm, + n_clusters=5, + random_state=global_random_seed, + n_init=1, + tol=0, + max_iter=max_iter, + ).fit(X) assert km.n_iter_ < max_iter -def test_minibatch_update_consistency(): +@pytest.mark.parametrize("X_csr", X_as_any_csr) +def test_minibatch_update_consistency(X_csr, global_random_seed): # Check that dense and sparse minibatch update give the same results - rng = np.random.RandomState(42) - old_centers = centers + rng.normal(size=centers.shape) + rng = np.random.RandomState(global_random_seed) - new_centers = old_centers.copy() - new_centers_csr = old_centers.copy() + centers_old = centers + rng.normal(size=centers.shape) + centers_old_csr = centers_old.copy() - weight_sums = np.zeros(new_centers.shape[0], dtype=np.double) - weight_sums_csr = np.zeros(new_centers.shape[0], dtype=np.double) + centers_new = np.zeros_like(centers_old) + centers_new_csr = np.zeros_like(centers_old_csr) - x_squared_norms = (X ** 2).sum(axis=1) - x_squared_norms_csr = row_norms(X_csr, squared=True) + weight_sums = np.zeros(centers_old.shape[0], dtype=X.dtype) + weight_sums_csr = np.zeros(centers_old.shape[0], dtype=X.dtype) - buffer = np.zeros(centers.shape[1], dtype=np.double) - buffer_csr = np.zeros(centers.shape[1], dtype=np.double) + sample_weight = np.ones(X.shape[0], dtype=X.dtype) # extract a small minibatch X_mb = X[:10] X_mb_csr = X_csr[:10] - x_mb_squared_norms = x_squared_norms[:10] - x_mb_squared_norms_csr = x_squared_norms_csr[:10] - - sample_weight_mb = np.ones(X_mb.shape[0], dtype=np.double) + sample_weight_mb = sample_weight[:10] # step 1: compute the dense minibatch update - old_inertia, incremental_diff = _mini_batch_step( - X_mb, sample_weight_mb, x_mb_squared_norms, new_centers, weight_sums, - buffer, 1, None, random_reassign=False) + old_inertia = _mini_batch_step( + X_mb, + sample_weight_mb, + centers_old, + centers_new, + weight_sums, + np.random.RandomState(global_random_seed), + random_reassign=False, + ) assert old_inertia > 0.0 # compute the new inertia on the same batch to check that it decreased - labels, new_inertia = _labels_inertia( - X_mb, sample_weight_mb, x_mb_squared_norms, new_centers) + labels, new_inertia = _labels_inertia(X_mb, sample_weight_mb, centers_new) assert new_inertia > 0.0 assert new_inertia < old_inertia - # check that the incremental difference computation is matching the - # final observed value - effective_diff = np.sum((new_centers - old_centers) ** 2) - assert_almost_equal(incremental_diff, effective_diff) - # step 2: compute the sparse minibatch update - old_inertia_csr, incremental_diff_csr = _mini_batch_step( - X_mb_csr, sample_weight_mb, x_mb_squared_norms_csr, new_centers_csr, - weight_sums_csr, buffer_csr, 1, None, random_reassign=False) + old_inertia_csr = _mini_batch_step( + X_mb_csr, + sample_weight_mb, + centers_old_csr, + centers_new_csr, + weight_sums_csr, + np.random.RandomState(global_random_seed), + random_reassign=False, + ) assert old_inertia_csr > 0.0 # compute the new inertia on the same batch to check that it decreased labels_csr, new_inertia_csr = _labels_inertia( - X_mb_csr, sample_weight_mb, x_mb_squared_norms_csr, new_centers_csr) + X_mb_csr, sample_weight_mb, centers_new_csr + ) assert new_inertia_csr > 0.0 assert new_inertia_csr < old_inertia_csr - # check that the incremental difference computation is matching the - # final observed value - effective_diff = np.sum((new_centers_csr - old_centers) ** 2) - assert_almost_equal(incremental_diff_csr, effective_diff) - # step 3: check that sparse and dense updates lead to the same results assert_array_equal(labels, labels_csr) - assert_array_almost_equal(new_centers, new_centers_csr) - assert_almost_equal(incremental_diff, incremental_diff_csr) - assert_almost_equal(old_inertia, old_inertia_csr) - assert_almost_equal(new_inertia, new_inertia_csr) + assert_allclose(centers_new, centers_new_csr) + assert_allclose(old_inertia, old_inertia_csr) + assert_allclose(new_inertia, new_inertia_csr) def _check_fitted_model(km): @@ -250,93 +276,97 @@ def _check_fitted_model(km): assert np.unique(labels).shape[0] == n_clusters # check that the labels assignment are perfect (up to a permutation) - assert v_measure_score(true_labels, labels) == 1.0 + assert_allclose(v_measure_score(true_labels, labels), 1.0) assert km.inertia_ > 0.0 -@pytest.mark.parametrize("data", [X, X_csr], ids=["dense", "sparse"]) -@pytest.mark.parametrize("init", ["random", "k-means++", centers, - lambda X, k, random_state: centers], - ids=["random", "k-means++", "ndarray", "callable"]) +@pytest.mark.parametrize( + "input_data", + [X] + X_as_any_csr, + ids=data_containers_ids, +) +@pytest.mark.parametrize( + "init", + ["random", "k-means++", centers, lambda X, k, random_state: centers], + ids=["random", "k-means++", "ndarray", "callable"], +) @pytest.mark.parametrize("Estimator", [KMeans, MiniBatchKMeans]) -def test_all_init(Estimator, data, init): +def test_all_init(Estimator, input_data, init): # Check KMeans and MiniBatchKMeans with all possible init. n_init = 10 if isinstance(init, str) else 1 - km = Estimator(init=init, n_clusters=n_clusters, random_state=42, - n_init=n_init).fit(data) + km = Estimator( + init=init, n_clusters=n_clusters, random_state=42, n_init=n_init + ).fit(input_data) _check_fitted_model(km) -@pytest.mark.parametrize("init", ["random", "k-means++", centers, - lambda X, k, random_state: centers], - ids=["random", "k-means++", "ndarray", "callable"]) +@pytest.mark.parametrize( + "init", + ["random", "k-means++", centers, lambda X, k, random_state: centers], + ids=["random", "k-means++", "ndarray", "callable"], +) def test_minibatch_kmeans_partial_fit_init(init): # Check MiniBatchKMeans init with partial_fit n_init = 10 if isinstance(init, str) else 1 - km = MiniBatchKMeans(init=init, n_clusters=n_clusters, random_state=0, - n_init=n_init) + km = MiniBatchKMeans( + init=init, n_clusters=n_clusters, random_state=0, n_init=n_init + ) for i in range(100): # "random" init requires many batches to recover the true labels. km.partial_fit(X) _check_fitted_model(km) +@pytest.mark.parametrize( + "init, expected_n_init", + [ + ("k-means++", 1), + ("random", "default"), + ( + lambda X, n_clusters, random_state: random_state.uniform( + size=(n_clusters, X.shape[1]) + ), + "default", + ), + ("array-like", 1), + ], +) +@pytest.mark.parametrize("Estimator", [KMeans, MiniBatchKMeans]) +def test_kmeans_init_auto_with_initial_centroids(Estimator, init, expected_n_init): + """Check that `n_init="auto"` chooses the right number of initializations. + Non-regression test for #26657: + https://github.com/scikit-learn/scikit-learn/pull/26657 + """ + n_sample, n_features, n_clusters = 100, 10, 5 + X = np.random.randn(n_sample, n_features) + if init == "array-like": + init = np.random.randn(n_clusters, n_features) + if expected_n_init == "default": + expected_n_init = 3 if Estimator is MiniBatchKMeans else 10 + + kmeans = Estimator(n_clusters=n_clusters, init=init, n_init="auto").fit(X) + assert kmeans._n_init == expected_n_init + + @pytest.mark.parametrize("Estimator", [KMeans, MiniBatchKMeans]) -def test_fortran_aligned_data(Estimator): +def test_fortran_aligned_data(Estimator, global_random_seed): # Check that KMeans works with fortran-aligned data. X_fortran = np.asfortranarray(X) centers_fortran = np.asfortranarray(centers) - km_c = Estimator(n_clusters=n_clusters, init=centers, n_init=1, - random_state=42).fit(X) - km_f = Estimator(n_clusters=n_clusters, init=centers_fortran, n_init=1, - random_state=42).fit(X_fortran) + km_c = Estimator( + n_clusters=n_clusters, init=centers, n_init=1, random_state=global_random_seed + ).fit(X) + km_f = Estimator( + n_clusters=n_clusters, + init=centers_fortran, + n_init=1, + random_state=global_random_seed, + ).fit(X_fortran) assert_allclose(km_c.cluster_centers_, km_f.cluster_centers_) assert_array_equal(km_c.labels_, km_f.labels_) -@pytest.mark.parametrize('algo', ['full', 'elkan']) -@pytest.mark.parametrize('dtype', [np.float32, np.float64]) -@pytest.mark.parametrize('constructor', [np.asarray, sp.csr_matrix]) -@pytest.mark.parametrize('seed, max_iter, tol', [ - (0, 2, 1e-7), # strict non-convergence - (1, 2, 1e-1), # loose non-convergence - (3, 300, 1e-7), # strict convergence - (4, 300, 1e-1), # loose convergence -]) -def test_k_means_fit_predict(algo, dtype, constructor, seed, max_iter, tol): - # check that fit.predict gives same result as fit_predict - # There's a very small chance of failure with elkan on unstructured dataset - # because predict method uses fast euclidean distances computation which - # may cause small numerical instabilities. - # NB: This test is largely redundant with respect to test_predict and - # test_predict_equal_labels. This test has the added effect of - # testing idempotence of the fittng procesdure which appears to - # be where it fails on some MacOS setups. - if sys.platform == "darwin": - pytest.xfail( - "Known failures on MacOS, See " - "https://github.com/scikit-learn/scikit-learn/issues/12644") - - rng = np.random.RandomState(seed) - - X = make_blobs(n_samples=1000, n_features=10, centers=10, - random_state=rng)[0].astype(dtype, copy=False) - X = constructor(X) - - kmeans = KMeans(algorithm=algo, n_clusters=10, random_state=seed, - tol=tol, max_iter=max_iter) - - labels_1 = kmeans.fit(X).predict(X) - labels_2 = kmeans.fit_predict(X) - - # Due to randomness in the order in which chunks of data are processed when - # using more than one thread, the absolute values of the labels can be - # different between the 2 strategies but they should correspond to the same - # clustering. - assert v_measure_score(labels_1, labels_2) == pytest.approx(1, abs=1e-15) - - def test_minibatch_kmeans_verbose(): # Check verbose mode of MiniBatchKMeans for better coverage. km = MiniBatchKMeans(n_clusters=n_clusters, random_state=42, verbose=1) @@ -348,14 +378,21 @@ def test_minibatch_kmeans_verbose(): sys.stdout = old_stdout -@pytest.mark.parametrize("algorithm", ["full", "elkan"]) +@pytest.mark.parametrize("algorithm", ["lloyd", "elkan"]) @pytest.mark.parametrize("tol", [1e-2, 0]) def test_kmeans_verbose(algorithm, tol, capsys): # Check verbose mode of KMeans for better coverage. X = np.random.RandomState(0).normal(size=(5000, 10)) - KMeans(algorithm=algorithm, n_clusters=n_clusters, random_state=42, - init="random", n_init=1, tol=tol, verbose=1).fit(X) + KMeans( + algorithm=algorithm, + n_clusters=n_clusters, + random_state=42, + init="random", + n_init=1, + tol=tol, + verbose=1, + ).fit(X) captured = capsys.readouterr() @@ -370,8 +407,9 @@ def test_kmeans_verbose(algorithm, tol, capsys): def test_minibatch_kmeans_warning_init_size(): # Check that a warning is raised when init_size is smaller than n_clusters - with pytest.warns(RuntimeWarning, - match=r"init_size.* should be larger than n_clusters"): + with pytest.warns( + RuntimeWarning, match=r"init_size.* should be larger than n_clusters" + ): MiniBatchKMeans(init_size=10, n_clusters=20).fit(X) @@ -379,99 +417,109 @@ def test_minibatch_kmeans_warning_init_size(): def test_warning_n_init_precomputed_centers(Estimator): # Check that a warning is raised when n_init > 1 and an array is passed for # the init parameter. - with pytest.warns(RuntimeWarning, - match="Explicit initial center position passed: " - "performing only one init"): + with pytest.warns( + RuntimeWarning, + match="Explicit initial center position passed: performing only one init", + ): Estimator(init=centers, n_clusters=n_clusters, n_init=10).fit(X) -def test_minibatch_sensible_reassign(): +def test_minibatch_sensible_reassign(global_random_seed): # check that identical initial clusters are reassigned # also a regression test for when there are more desired reassignments than # samples. - zeroed_X, true_labels = make_blobs(n_samples=100, centers=5, - random_state=42) + zeroed_X, true_labels = make_blobs( + n_samples=100, centers=5, random_state=global_random_seed + ) zeroed_X[::2, :] = 0 - km = MiniBatchKMeans(n_clusters=20, batch_size=10, random_state=42, - init="random").fit(zeroed_X) + km = MiniBatchKMeans( + n_clusters=20, batch_size=10, random_state=global_random_seed, init="random" + ).fit(zeroed_X) # there should not be too many exact zero cluster centers - assert km.cluster_centers_.any(axis=1).sum() > 10 + num_non_zero_clusters = km.cluster_centers_.any(axis=1).sum() + assert num_non_zero_clusters > 9, f"{num_non_zero_clusters=} is too small" # do the same with batch-size > X.shape[0] (regression test) - km = MiniBatchKMeans(n_clusters=20, batch_size=200, random_state=42, - init="random").fit(zeroed_X) + km = MiniBatchKMeans( + n_clusters=20, batch_size=200, random_state=global_random_seed, init="random" + ).fit(zeroed_X) # there should not be too many exact zero cluster centers - assert km.cluster_centers_.any(axis=1).sum() > 10 + num_non_zero_clusters = km.cluster_centers_.any(axis=1).sum() + assert num_non_zero_clusters > 9, f"{num_non_zero_clusters=} is too small" # do the same with partial_fit API - km = MiniBatchKMeans(n_clusters=20, random_state=42, init="random") + km = MiniBatchKMeans(n_clusters=20, random_state=global_random_seed, init="random") for i in range(100): km.partial_fit(zeroed_X) # there should not be too many exact zero cluster centers - assert km.cluster_centers_.any(axis=1).sum() > 10 + num_non_zero_clusters = km.cluster_centers_.any(axis=1).sum() + assert num_non_zero_clusters > 9, f"{num_non_zero_clusters=} is too small" -def test_minibatch_reassign(): - # Give a perfect initialization, but a large reassignment_ratio, - # as a result all the centers should be reassigned and the model - # should no longer be good - sample_weight = np.ones(X.shape[0], dtype=X.dtype) - for this_X in (X, X_csr): - mb_k_means = MiniBatchKMeans(n_clusters=n_clusters, batch_size=100, - random_state=42) - mb_k_means.fit(this_X) - - score_before = mb_k_means.score(this_X) - try: - old_stdout = sys.stdout - sys.stdout = StringIO() - # Turn on verbosity to smoke test the display code - _mini_batch_step(this_X, sample_weight, (X ** 2).sum(axis=1), - mb_k_means.cluster_centers_, - mb_k_means._counts, - np.zeros(X.shape[1], np.double), - False, distances=np.zeros(X.shape[0]), - random_reassign=True, random_state=42, - reassignment_ratio=1, verbose=True) - finally: - sys.stdout = old_stdout - assert score_before > mb_k_means.score(this_X) +@pytest.mark.parametrize( + "input_data", + [X] + X_as_any_csr, + ids=data_containers_ids, +) +def test_minibatch_reassign(input_data, global_random_seed): + # Check the reassignment part of the minibatch step with very high or very + # low reassignment ratio. + perfect_centers = np.empty((n_clusters, n_features)) + for i in range(n_clusters): + perfect_centers[i] = X[true_labels == i].mean(axis=0) + + sample_weight = np.ones(n_samples) + centers_new = np.empty_like(perfect_centers) + + # Give a perfect initialization, but a large reassignment_ratio, as a + # result many centers should be reassigned and the model should no longer + # be good + score_before = -_labels_inertia(input_data, sample_weight, perfect_centers, 1)[1] + + _mini_batch_step( + input_data, + sample_weight, + perfect_centers, + centers_new, + np.zeros(n_clusters), + np.random.RandomState(global_random_seed), + random_reassign=True, + reassignment_ratio=1, + ) + + score_after = -_labels_inertia(input_data, sample_weight, centers_new, 1)[1] + + assert score_before > score_after # Give a perfect initialization, with a small reassignment_ratio, - # no center should be reassigned - for this_X in (X, X_csr): - mb_k_means = MiniBatchKMeans(n_clusters=n_clusters, batch_size=100, - init=centers.copy(), - random_state=42, n_init=1) - mb_k_means.fit(this_X) - clusters_before = mb_k_means.cluster_centers_ - # Turn on verbosity to smoke test the display code - _mini_batch_step(this_X, sample_weight, (X ** 2).sum(axis=1), - mb_k_means.cluster_centers_, - mb_k_means._counts, - np.zeros(X.shape[1], np.double), - False, distances=np.zeros(X.shape[0]), - random_reassign=True, random_state=42, - reassignment_ratio=1e-15) - assert_array_almost_equal(clusters_before, mb_k_means.cluster_centers_) + # no center should be reassigned. + _mini_batch_step( + input_data, + sample_weight, + perfect_centers, + centers_new, + np.zeros(n_clusters), + np.random.RandomState(global_random_seed), + random_reassign=True, + reassignment_ratio=1e-15, + ) + + assert_allclose(centers_new, perfect_centers) def test_minibatch_with_many_reassignments(): # Test for the case that the number of clusters to reassign is bigger - # than the batch_size - n_samples = 550 - rnd = np.random.RandomState(42) - X = rnd.uniform(size=(n_samples, 10)) - # Check that the fit works if n_clusters is bigger than the batch_size. - # Run the test with 550 clusters and 550 samples, because it turned out - # that this values ensure that the number of clusters to reassign - # is always bigger than the batch_size - n_clusters = 550 - MiniBatchKMeans(n_clusters=n_clusters, - batch_size=100, - init_size=n_samples, - random_state=42).fit(X) + # than the batch_size. Run the test with 100 clusters and a batch_size of + # 10 because it turned out that these values ensure that the number of + # clusters to reassign is always bigger than the batch_size. + MiniBatchKMeans( + n_clusters=100, + batch_size=10, + init_size=n_samples, + random_state=42, + verbose=True, + ).fit(X) def test_minibatch_kmeans_init_size(): @@ -486,11 +534,65 @@ def test_minibatch_kmeans_init_size(): assert km._init_size == 30 # it should not be larger than n_samples - km = MiniBatchKMeans(n_clusters=10, batch_size=5, n_init=1, - init_size=n_samples + 1).fit(X) + km = MiniBatchKMeans( + n_clusters=10, batch_size=5, n_init=1, init_size=n_samples + 1 + ).fit(X) assert km._init_size == n_samples +@pytest.mark.parametrize("tol, max_no_improvement", [(1e-4, None), (0, 10)]) +def test_minibatch_declared_convergence(capsys, tol, max_no_improvement): + # Check convergence detection based on ewa batch inertia or on + # small center change. + X, _, centers = make_blobs(centers=3, random_state=0, return_centers=True) + + km = MiniBatchKMeans( + n_clusters=3, + init=centers, + batch_size=20, + tol=tol, + random_state=0, + max_iter=10, + n_init=1, + verbose=1, + max_no_improvement=max_no_improvement, + ) + + km.fit(X) + assert 1 < km.n_iter_ < 10 + + captured = capsys.readouterr() + if max_no_improvement is None: + assert "Converged (small centers change)" in captured.out + if tol == 0: + assert "Converged (lack of improvement in inertia)" in captured.out + + +def test_minibatch_iter_steps(): + # Check consistency of n_iter_ and n_steps_ attributes. + batch_size = 30 + n_samples = X.shape[0] + km = MiniBatchKMeans(n_clusters=3, batch_size=batch_size, random_state=0).fit(X) + + # n_iter_ is the number of started epochs + assert km.n_iter_ == np.ceil((km.n_steps_ * batch_size) / n_samples) + assert isinstance(km.n_iter_, int) + + # without stopping condition, max_iter should be reached + km = MiniBatchKMeans( + n_clusters=3, + batch_size=batch_size, + random_state=0, + tol=0, + max_no_improvement=None, + max_iter=10, + ).fit(X) + + assert km.n_iter_ == 10 + assert km.n_steps_ == (10 * n_samples) // batch_size + assert isinstance(km.n_steps_, int) + + def test_kmeans_copyx(): # Check that copy_x=False returns nearly equal X after de-centering. my_X = X.copy() @@ -503,44 +605,41 @@ def test_kmeans_copyx(): @pytest.mark.parametrize("Estimator", [KMeans, MiniBatchKMeans]) -def test_score_max_iter(Estimator): +def test_score_max_iter(Estimator, global_random_seed): # Check that fitting KMeans or MiniBatchKMeans with more iterations gives # better score - X = np.random.RandomState(0).randn(100, 10) + X = np.random.RandomState(global_random_seed).randn(100, 10) - km1 = Estimator(n_init=1, random_state=42, max_iter=1) + km1 = Estimator(n_init=1, random_state=global_random_seed, max_iter=1) s1 = km1.fit(X).score(X) - km2 = Estimator(n_init=1, random_state=42, max_iter=10) + km2 = Estimator(n_init=1, random_state=global_random_seed, max_iter=10) s2 = km2.fit(X).score(X) assert s2 > s1 -@pytest.mark.parametrize("array_constr", [np.array, sp.csr_matrix], - ids=["dense", "sparse"]) -@pytest.mark.parametrize("dtype", [np.float32, np.float64]) -@pytest.mark.parametrize("init", ["random", "k-means++"]) -@pytest.mark.parametrize("Estimator, algorithm", [ - (KMeans, "full"), - (KMeans, "elkan"), - (MiniBatchKMeans, None) -]) -def test_predict(Estimator, algorithm, init, dtype, array_constr): +@pytest.mark.parametrize("array_constr", data_containers, ids=data_containers_ids) +@pytest.mark.parametrize( + "Estimator, algorithm", + [(KMeans, "lloyd"), (KMeans, "elkan"), (MiniBatchKMeans, None)], +) +@pytest.mark.parametrize("max_iter", [2, 100]) +def test_kmeans_predict( + Estimator, algorithm, array_constr, max_iter, global_dtype, global_random_seed +): # Check the predict method and the equivalence between fit.predict and # fit_predict. - - # There's a very small chance of failure with elkan on unstructured dataset - # because predict method uses fast euclidean distances computation which - # may cause small numerical instabilities. - if sys.platform == "darwin": - pytest.xfail( - "Known failures on MacOS, See " - "https://github.com/scikit-learn/scikit-learn/issues/12644") - - X, _ = make_blobs(n_samples=500, n_features=10, centers=10, random_state=0) - X = array_constr(X) - - # With n_init = 1 - km = Estimator(n_clusters=10, init=init, n_init=1, random_state=0) + X, _ = make_blobs( + n_samples=200, n_features=10, centers=10, random_state=global_random_seed + ) + X = array_constr(X, dtype=global_dtype) + + km = Estimator( + n_clusters=10, + init="random", + n_init=10, + max_iter=max_iter, + random_state=global_random_seed, + ) if algorithm is not None: km.set_params(algorithm=algorithm) km.fit(X) @@ -558,41 +657,37 @@ def test_predict(Estimator, algorithm, init, dtype, array_constr): pred = km.predict(km.cluster_centers_) assert_array_equal(pred, np.arange(10)) - # With n_init > 1 - # Due to randomness in the order in which chunks of data are processed when - # using more than one thread, there might be different rounding errors for - # the computation of the inertia between 2 runs. This might result in a - # different ranking of 2 inits, hence a different labeling, even if they - # give the same clustering. We only check the labels up to a permutation. - km = Estimator(n_clusters=10, init=init, n_init=10, random_state=0) - if algorithm is not None: - km.set_params(algorithm=algorithm) - km.fit(X) - labels = km.labels_ - - # re-predict labels for training set using predict - pred = km.predict(X) - assert_allclose(v_measure_score(pred, labels), 1) - - # re-predict labels for training set using fit_predict - pred = km.fit_predict(X) - assert_allclose(v_measure_score(pred, labels), 1) - - # predict centroid labels - pred = km.predict(km.cluster_centers_) - assert_allclose(v_measure_score(pred, np.arange(10)), 1) - - -@pytest.mark.parametrize("init", ["random", "k-means++", centers], - ids=["random", "k-means++", "ndarray"]) +@pytest.mark.parametrize("X_csr", X_as_any_csr) +@pytest.mark.parametrize("Estimator", [KMeans, MiniBatchKMeans]) +def test_dense_sparse(Estimator, X_csr, global_random_seed): + # Check that the results are the same for dense and sparse input. + sample_weight = np.random.RandomState(global_random_seed).random_sample( + (n_samples,) + ) + km_dense = Estimator( + n_clusters=n_clusters, random_state=global_random_seed, n_init=1 + ) + km_dense.fit(X, sample_weight=sample_weight) + km_sparse = Estimator( + n_clusters=n_clusters, random_state=global_random_seed, n_init=1 + ) + km_sparse.fit(X_csr, sample_weight=sample_weight) + + assert_array_equal(km_dense.labels_, km_sparse.labels_) + assert_allclose(km_dense.cluster_centers_, km_sparse.cluster_centers_) + + +@pytest.mark.parametrize("X_csr", X_as_any_csr) +@pytest.mark.parametrize( + "init", ["random", "k-means++", centers], ids=["random", "k-means++", "ndarray"] +) @pytest.mark.parametrize("Estimator", [KMeans, MiniBatchKMeans]) -def test_predict_dense_sparse(Estimator, init): +def test_predict_dense_sparse(Estimator, init, X_csr): # check that models trained on sparse input also works for dense input at # predict time and vice versa. n_init = 10 if isinstance(init, str) else 1 - km = Estimator(n_clusters=n_clusters, init=init, n_init=n_init, - random_state=0) + km = Estimator(n_clusters=n_clusters, init=init, n_init=n_init, random_state=0) km.fit(X_csr) assert_array_equal(km.predict(X), km.labels_) @@ -601,12 +696,11 @@ def test_predict_dense_sparse(Estimator, init): assert_array_equal(km.predict(X_csr), km.labels_) -@pytest.mark.parametrize("array_constr", [np.array, sp.csr_matrix], - ids=["dense", "sparse"]) +@pytest.mark.parametrize("array_constr", data_containers, ids=data_containers_ids) @pytest.mark.parametrize("dtype", [np.int32, np.int64]) @pytest.mark.parametrize("init", ["k-means++", "ndarray"]) @pytest.mark.parametrize("Estimator", [KMeans, MiniBatchKMeans]) -def test_integer_input(Estimator, array_constr, dtype, init): +def test_integer_input(Estimator, array_constr, dtype, init, global_random_seed): # Check that KMeans and MiniBatchKMeans work with integer input. X_dense = np.array([[0, 0], [10, 10], [12, 9], [-1, 1], [2, 0], [8, 10]]) X = array_constr(X_dense, dtype=dtype) @@ -614,7 +708,9 @@ def test_integer_input(Estimator, array_constr, dtype, init): n_init = 1 if init == "ndarray" else 10 init = X_dense[:2] if init == "ndarray" else init - km = Estimator(n_clusters=2, init=init, n_init=n_init, random_state=0) + km = Estimator( + n_clusters=2, init=init, n_init=n_init, random_state=global_random_seed + ) if Estimator is MiniBatchKMeans: km.set_params(batch_size=2) @@ -624,7 +720,7 @@ def test_integer_input(Estimator, array_constr, dtype, init): assert km.cluster_centers_.dtype == np.float64 expected_labels = [0, 1, 1, 0, 0, 1] - assert_allclose(v_measure_score(km.labels_, expected_labels), 1) + assert_allclose(v_measure_score(km.labels_, expected_labels), 1.0) # Same with partial_fit (#14314) if Estimator is MiniBatchKMeans: @@ -633,9 +729,9 @@ def test_integer_input(Estimator, array_constr, dtype, init): @pytest.mark.parametrize("Estimator", [KMeans, MiniBatchKMeans]) -def test_transform(Estimator): +def test_transform(Estimator, global_random_seed): # Check the transform method - km = Estimator(n_clusters=n_clusters).fit(X) + km = Estimator(n_clusters=n_clusters, random_state=global_random_seed).fit(X) # Transorfming cluster_centers_ should return the pairwise distances # between centers @@ -651,28 +747,34 @@ def test_transform(Estimator): @pytest.mark.parametrize("Estimator", [KMeans, MiniBatchKMeans]) -def test_fit_transform(Estimator): +def test_fit_transform(Estimator, global_random_seed): # Check equivalence between fit.transform and fit_transform - X1 = Estimator(random_state=0, n_init=1).fit(X).transform(X) - X2 = Estimator(random_state=0, n_init=1).fit_transform(X) + X1 = Estimator(random_state=global_random_seed, n_init=1).fit(X).transform(X) + X2 = Estimator(random_state=global_random_seed, n_init=1).fit_transform(X) assert_allclose(X1, X2) -def test_n_init(): +def test_n_init(global_random_seed): # Check that increasing the number of init increases the quality previous_inertia = np.inf for n_init in [1, 5, 10]: # set max_iter=1 to avoid finding the global minimum and get the same # inertia each time - km = KMeans(n_clusters=n_clusters, init="random", n_init=n_init, - random_state=0, max_iter=1).fit(X) + km = KMeans( + n_clusters=n_clusters, + init="random", + n_init=n_init, + random_state=global_random_seed, + max_iter=1, + ).fit(X) assert km.inertia_ <= previous_inertia -def test_k_means_function(): +def test_k_means_function(global_random_seed): # test calling the k_means function directly - cluster_centers, labels, inertia = k_means(X, n_clusters=n_clusters, - sample_weight=None) + cluster_centers, labels, inertia = k_means( + X, n_clusters=n_clusters, sample_weight=None, random_state=global_random_seed + ) assert cluster_centers.shape == (n_clusters, n_features) assert np.unique(labels).shape[0] == n_clusters @@ -682,11 +784,15 @@ def test_k_means_function(): assert inertia > 0.0 -@pytest.mark.parametrize("data", [X, X_csr], ids=["dense", "sparse"]) +@pytest.mark.parametrize( + "input_data", + [X] + X_as_any_csr, + ids=data_containers_ids, +) @pytest.mark.parametrize("Estimator", [KMeans, MiniBatchKMeans]) -def test_float_precision(Estimator, data): +def test_float_precision(Estimator, input_data, global_random_seed): # Check that the results are the same for single and double precision. - km = Estimator(n_init=1, random_state=0) + km = Estimator(n_init=1, random_state=global_random_seed) inertia = {} Xt = {} @@ -694,7 +800,7 @@ def test_float_precision(Estimator, data): labels = {} for dtype in [np.float64, np.float32]: - X = data.astype(dtype, **_astype_copy_false(data)) + X = input_data.astype(dtype, copy=False) km.fit(X) inertia[dtype] = km.inertia_ @@ -712,9 +818,11 @@ def test_float_precision(Estimator, data): # compare arrays with low precision since the difference between 32 and # 64 bit comes from an accumulation of rounding errors. - assert_allclose(inertia[np.float32], inertia[np.float64], rtol=1e-5) - assert_allclose(Xt[np.float32], Xt[np.float64], rtol=1e-5) - assert_allclose(centers[np.float32], centers[np.float64], rtol=1e-5) + assert_allclose(inertia[np.float32], inertia[np.float64], rtol=1e-4) + assert_allclose(Xt[np.float32], Xt[np.float64], atol=Xt[np.float64].max() * 1e-4) + assert_allclose( + centers[np.float32], centers[np.float64], atol=centers[np.float64].max() * 1e-4 + ) assert_array_equal(labels[np.float32], labels[np.float64]) @@ -733,28 +841,32 @@ def test_centers_not_mutated(Estimator, dtype): assert not np.may_share_memory(km.cluster_centers_, centers_new_type) -@pytest.mark.parametrize("data", [X, X_csr], ids=["dense", "sparse"]) -def test_kmeans_init_fitted_centers(data): +@pytest.mark.parametrize( + "input_data", + [X] + X_as_any_csr, + ids=data_containers_ids, +) +def test_kmeans_init_fitted_centers(input_data): # Check that starting fitting from a local optimum shouldn't change the # solution - km1 = KMeans(n_clusters=n_clusters).fit(data) - km2 = KMeans(n_clusters=n_clusters, init=km1.cluster_centers_, - n_init=1).fit(data) + km1 = KMeans(n_clusters=n_clusters).fit(input_data) + km2 = KMeans(n_clusters=n_clusters, init=km1.cluster_centers_, n_init=1).fit( + input_data + ) assert_allclose(km1.cluster_centers_, km2.cluster_centers_) -def test_kmeans_warns_less_centers_than_unique_points(): +def test_kmeans_warns_less_centers_than_unique_points(global_random_seed): # Check KMeans when the number of found clusters is smaller than expected - X = np.asarray([[0, 0], - [0, 1], - [1, 0], - [1, 0]]) # last point is duplicated - km = KMeans(n_clusters=4) + X = np.asarray([[0, 0], [0, 1], [1, 0], [1, 0]]) # last point is duplicated + km = KMeans(n_clusters=4, random_state=global_random_seed) # KMeans should warn that fewer labels than cluster centers have been used - msg = (r"Number of distinct clusters \(3\) found smaller than " - r"n_clusters \(4\). Possibly due to duplicate points in X.") + msg = ( + r"Number of distinct clusters \(3\) found smaller than " + r"n_clusters \(4\). Possibly due to duplicate points in X." + ) with pytest.warns(ConvergenceWarning, match=msg): km.fit(X) # only three distinct points, so only three clusters @@ -766,15 +878,19 @@ def _sort_centers(centers): return np.sort(centers, axis=0) -def test_weighted_vs_repeated(): +def test_weighted_vs_repeated(global_random_seed): # Check that a sample weight of N should yield the same result as an N-fold # repetition of the sample. Valid only if init is precomputed, otherwise # rng produces different results. Not valid for MinibatchKMeans due to rng # to extract minibatches. - sample_weight = np.random.RandomState(0).randint(1, 5, size=n_samples) + sample_weight = np.random.RandomState(global_random_seed).randint( + 1, 5, size=n_samples + ) X_repeat = np.repeat(X, sample_weight, axis=0) - km = KMeans(init=centers, n_init=1, n_clusters=n_clusters, random_state=0) + km = KMeans( + init=centers, n_init=1, n_clusters=n_clusters, random_state=global_random_seed + ) km_weighted = clone(km).fit(X, sample_weight=sample_weight) repeated_labels = np.repeat(km_weighted.labels_, sample_weight) @@ -782,36 +898,48 @@ def test_weighted_vs_repeated(): assert_array_equal(km_repeated.labels_, repeated_labels) assert_allclose(km_weighted.inertia_, km_repeated.inertia_) - assert_allclose(_sort_centers(km_weighted.cluster_centers_), - _sort_centers(km_repeated.cluster_centers_)) + assert_allclose( + _sort_centers(km_weighted.cluster_centers_), + _sort_centers(km_repeated.cluster_centers_), + ) -@pytest.mark.parametrize("data", [X, X_csr], ids=["dense", "sparse"]) +@pytest.mark.parametrize( + "input_data", + [X] + X_as_any_csr, + ids=data_containers_ids, +) @pytest.mark.parametrize("Estimator", [KMeans, MiniBatchKMeans]) -def test_unit_weights_vs_no_weights(Estimator, data): +def test_unit_weights_vs_no_weights(Estimator, input_data, global_random_seed): # Check that not passing sample weights should be equivalent to passing # sample weights all equal to one. sample_weight = np.ones(n_samples) - km = Estimator(n_clusters=n_clusters, random_state=42, n_init=1) - km_none = clone(km).fit(data, sample_weight=None) - km_ones = clone(km).fit(data, sample_weight=sample_weight) + km = Estimator(n_clusters=n_clusters, random_state=global_random_seed, n_init=1) + km_none = clone(km).fit(input_data, sample_weight=None) + km_ones = clone(km).fit(input_data, sample_weight=sample_weight) assert_array_equal(km_none.labels_, km_ones.labels_) assert_allclose(km_none.cluster_centers_, km_ones.cluster_centers_) -def test_scaled_weights(): - # scaling all sample weights by a common factor +@pytest.mark.parametrize( + "input_data", + [X] + X_as_any_csr, + ids=data_containers_ids, +) +@pytest.mark.parametrize("Estimator", [KMeans, MiniBatchKMeans]) +def test_scaled_weights(Estimator, input_data, global_random_seed): + # Check that scaling all sample weights by a common factor # shouldn't change the result - sample_weight = np.ones(n_samples) - for estimator in [KMeans(n_clusters=n_clusters, random_state=42), - MiniBatchKMeans(n_clusters=n_clusters, random_state=42)]: - est_1 = clone(estimator).fit(X) - est_2 = clone(estimator).fit(X, sample_weight=0.5*sample_weight) - assert_almost_equal(v_measure_score(est_1.labels_, est_2.labels_), 1.0) - assert_almost_equal(_sort_centers(est_1.cluster_centers_), - _sort_centers(est_2.cluster_centers_)) + sample_weight = np.random.RandomState(global_random_seed).uniform(size=n_samples) + + km = Estimator(n_clusters=n_clusters, random_state=global_random_seed, n_init=1) + km_orig = clone(km).fit(input_data, sample_weight=sample_weight) + km_scaled = clone(km).fit(input_data, sample_weight=0.5 * sample_weight) + + assert_array_equal(km_orig.labels_, km_scaled.labels_) + assert_allclose(km_orig.cluster_centers_, km_scaled.cluster_centers_) def test_kmeans_elkan_iter_attribute(): @@ -821,8 +949,7 @@ def test_kmeans_elkan_iter_attribute(): assert km.n_iter_ == 1 -@pytest.mark.parametrize("array_constr", [np.array, sp.csr_matrix], - ids=["dense", "sparse"]) +@pytest.mark.parametrize("array_constr", data_containers, ids=data_containers_ids) def test_kmeans_empty_cluster_relocated(array_constr): # check that empty clusters are correctly relocated when using sample # weights (#13486) @@ -837,75 +964,43 @@ def test_kmeans_empty_cluster_relocated(array_constr): assert_allclose(km.cluster_centers_, [[-1], [1]]) -def test_result_of_kmeans_equal_in_diff_n_threads(): - # Check that KMeans gives the same results in parallel mode than in - # sequential mode. - rnd = np.random.RandomState(0) +@pytest.mark.parametrize("Estimator", [KMeans, MiniBatchKMeans]) +def test_result_equal_in_diff_n_threads(Estimator, global_random_seed): + # Check that KMeans/MiniBatchKMeans give the same results in parallel mode + # than in sequential mode. + rnd = np.random.RandomState(global_random_seed) X = rnd.normal(size=(50, 10)) - with threadpool_limits(limits=1, user_api="openmp"): - result_1 = KMeans( - n_clusters=3, random_state=0).fit(X).labels_ - with threadpool_limits(limits=2, user_api="openmp"): - result_2 = KMeans( - n_clusters=3, random_state=0).fit(X).labels_ + with _get_threadpool_controller().limit(limits=1, user_api="openmp"): + result_1 = ( + Estimator(n_clusters=n_clusters, random_state=global_random_seed) + .fit(X) + .labels_ + ) + with _get_threadpool_controller().limit(limits=2, user_api="openmp"): + result_2 = ( + Estimator(n_clusters=n_clusters, random_state=global_random_seed) + .fit(X) + .labels_ + ) assert_array_equal(result_1, result_2) -@pytest.mark.parametrize("precompute_distances", ["auto", False, True]) -def test_precompute_distance_deprecated(precompute_distances): - # FIXME: remove in 1.0 - depr_msg = ("'precompute_distances' was deprecated in version 0.23 and " - "will be removed in 1.0") - X, _ = make_blobs(n_samples=10, n_features=2, centers=2, random_state=0) - kmeans = KMeans(n_clusters=2, n_init=1, init='random', random_state=0, - precompute_distances=precompute_distances) - - with pytest.warns(FutureWarning, match=depr_msg): - kmeans.fit(X) - - -@pytest.mark.parametrize("n_jobs", [None, 1]) -def test_n_jobs_deprecated(n_jobs): - # FIXME: remove in 1.0 - depr_msg = ("'n_jobs' was deprecated in version 0.23 and will be removed " - "in 1.0") - X, _ = make_blobs(n_samples=10, n_features=2, centers=2, random_state=0) - kmeans = KMeans(n_clusters=2, n_init=1, init='random', random_state=0, - n_jobs=n_jobs) - - with pytest.warns(FutureWarning, match=depr_msg): - kmeans.fit(X) - - -@pytest.mark.parametrize("attr", ["counts_", "init_size_", "random_state_"]) -def test_minibatch_kmeans_deprecated_attributes(attr): - # check that we raise a deprecation warning when accessing `init_size_` - # FIXME: remove in 1.1 - depr_msg = (f"The attribute '{attr}' is deprecated in 0.24 and will be " - f"removed in 1.1") - km = MiniBatchKMeans(n_clusters=2, n_init=1, init='random', random_state=0) - km.fit(X) - - with pytest.warns(FutureWarning, match=depr_msg): - getattr(km, attr) - - def test_warning_elkan_1_cluster(): # Check warning messages specific to KMeans - with pytest.warns(RuntimeWarning, - match="algorithm='elkan' doesn't make sense for a single" - " cluster"): + with pytest.warns( + RuntimeWarning, + match="algorithm='elkan' doesn't make sense for a single cluster", + ): KMeans(n_clusters=1, algorithm="elkan").fit(X) -@pytest.mark.parametrize("array_constr", [np.array, sp.csr_matrix], - ids=["dense", "sparse"]) -@pytest.mark.parametrize("algo", ["full", "elkan"]) -def test_k_means_1_iteration(array_constr, algo): +@pytest.mark.parametrize("array_constr", data_containers, ids=data_containers_ids) +@pytest.mark.parametrize("algo", ["lloyd", "elkan"]) +def test_k_means_1_iteration(array_constr, algo, global_random_seed): # check the results after a single iteration (E-step M-step E-step) by # comparing against a pure python implementation. - X = np.random.RandomState(0).uniform(size=(100, 5)) + X = np.random.RandomState(global_random_seed).uniform(size=(100, 5)) init_centers = X[:5] X = array_constr(X) @@ -919,8 +1014,9 @@ def py_kmeans(X, init): py_labels, py_centers = py_kmeans(X, init_centers) - cy_kmeans = KMeans(n_clusters=5, n_init=1, init=init_centers, - algorithm=algo, max_iter=1).fit(X) + cy_kmeans = KMeans( + n_clusters=5, n_init=1, init=init_centers, algorithm=algo, max_iter=1 + ).fit(X) cy_labels = cy_kmeans.labels_ cy_centers = cy_kmeans.cluster_centers_ @@ -930,47 +1026,83 @@ def py_kmeans(X, init): @pytest.mark.parametrize("dtype", [np.float32, np.float64]) @pytest.mark.parametrize("squared", [True, False]) -def test_euclidean_distance(dtype, squared): +def test_euclidean_distance(dtype, squared, global_random_seed): # Check that the _euclidean_(dense/sparse)_dense helpers produce correct # results - rng = np.random.RandomState(0) - a_sparse = sp.random(1, 100, density=0.5, format="csr", random_state=rng, - dtype=dtype) + rng = np.random.RandomState(global_random_seed) + a_sparse = sp.random( + 1, 100, density=0.5, format="csr", random_state=rng, dtype=dtype + ) a_dense = a_sparse.toarray().reshape(-1) b = rng.randn(100).astype(dtype, copy=False) b_squared_norm = (b**2).sum() - expected = ((a_dense - b)**2).sum() + expected = ((a_dense - b) ** 2).sum() expected = expected if squared else np.sqrt(expected) distance_dense_dense = _euclidean_dense_dense_wrapper(a_dense, b, squared) distance_sparse_dense = _euclidean_sparse_dense_wrapper( - a_sparse.data, a_sparse.indices, b, b_squared_norm, squared) + a_sparse.data, a_sparse.indices, b, b_squared_norm, squared + ) - assert_allclose(distance_dense_dense, distance_sparse_dense, rtol=1e-6) - assert_allclose(distance_dense_dense, expected, rtol=1e-6) - assert_allclose(distance_sparse_dense, expected, rtol=1e-6) + rtol = 1e-4 if dtype == np.float32 else 1e-7 + assert_allclose(distance_dense_dense, distance_sparse_dense, rtol=rtol) + assert_allclose(distance_dense_dense, expected, rtol=rtol) + assert_allclose(distance_sparse_dense, expected, rtol=rtol) @pytest.mark.parametrize("dtype", [np.float32, np.float64]) -def test_inertia(dtype): - rng = np.random.RandomState(0) - X_sparse = sp.random(100, 10, density=0.5, format="csr", random_state=rng, - dtype=dtype) +def test_inertia(dtype, global_random_seed): + # Check that the _inertia_(dense/sparse) helpers produce correct results. + rng = np.random.RandomState(global_random_seed) + X_sparse = sp.random( + 100, 10, density=0.5, format="csr", random_state=rng, dtype=dtype + ) X_dense = X_sparse.toarray() sample_weight = rng.randn(100).astype(dtype, copy=False) centers = rng.randn(5, 10).astype(dtype, copy=False) labels = rng.randint(5, size=100, dtype=np.int32) - distances = ((X_dense - centers[labels])**2).sum(axis=1) + distances = ((X_dense - centers[labels]) ** 2).sum(axis=1) expected = np.sum(distances * sample_weight) - inertia_dense = _inertia_dense(X_dense, sample_weight, centers, labels) - inertia_sparse = _inertia_sparse(X_sparse, sample_weight, centers, labels) + inertia_dense = _inertia_dense(X_dense, sample_weight, centers, labels, n_threads=1) + inertia_sparse = _inertia_sparse( + X_sparse, sample_weight, centers, labels, n_threads=1 + ) + + rtol = 1e-4 if dtype == np.float32 else 1e-6 + assert_allclose(inertia_dense, inertia_sparse, rtol=rtol) + assert_allclose(inertia_dense, expected, rtol=rtol) + assert_allclose(inertia_sparse, expected, rtol=rtol) + + # Check the single_label parameter. + label = 1 + mask = labels == label + distances = ((X_dense[mask] - centers[label]) ** 2).sum(axis=1) + expected = np.sum(distances * sample_weight[mask]) + + inertia_dense = _inertia_dense( + X_dense, sample_weight, centers, labels, n_threads=1, single_label=label + ) + inertia_sparse = _inertia_sparse( + X_sparse, sample_weight, centers, labels, n_threads=1, single_label=label + ) - assert_allclose(inertia_dense, inertia_sparse, rtol=1e-6) - assert_allclose(inertia_dense, expected, rtol=1e-6) - assert_allclose(inertia_sparse, expected, rtol=1e-6) + assert_allclose(inertia_dense, inertia_sparse, rtol=rtol) + assert_allclose(inertia_dense, expected, rtol=rtol) + assert_allclose(inertia_sparse, expected, rtol=rtol) + + +@pytest.mark.parametrize("Klass, default_n_init", [(KMeans, 10), (MiniBatchKMeans, 3)]) +def test_n_init_auto(Klass, default_n_init): + est = Klass(n_init="auto", init="k-means++") + est.fit(X) + assert est._n_init == 1 + + est = Klass(n_init="auto", init="random") + est.fit(X) + assert est._n_init == 10 if Klass.__name__ == "KMeans" else 3 @pytest.mark.parametrize("Estimator", [KMeans, MiniBatchKMeans]) @@ -984,25 +1116,31 @@ def test_sample_weight_unchanged(Estimator): @pytest.mark.parametrize("Estimator", [KMeans, MiniBatchKMeans]) -@pytest.mark.parametrize("param, match", [ - ({"n_init": 0}, r"n_init should be > 0"), - ({"max_iter": 0}, r"max_iter should be > 0"), - ({"n_clusters": n_samples + 1}, r"n_samples.* should be >= n_clusters"), - ({"init": X[:2]}, - r"The shape of the initial centers .* does not match " - r"the number of clusters"), - ({"init": lambda X_, k, random_state: X_[:2]}, - r"The shape of the initial centers .* does not match " - r"the number of clusters"), - ({"init": X[:8, :2]}, - r"The shape of the initial centers .* does not match " - r"the number of features of the data"), - ({"init": lambda X_, k, random_state: X_[:8, :2]}, - r"The shape of the initial centers .* does not match " - r"the number of features of the data"), - ({"init": "wrong"}, - r"init should be either 'k-means\+\+', 'random', " - r"a ndarray or a callable")] +@pytest.mark.parametrize( + "param, match", + [ + ({"n_clusters": n_samples + 1}, r"n_samples.* should be >= n_clusters"), + ( + {"init": X[:2]}, + r"The shape of the initial centers .* does not match " + r"the number of clusters", + ), + ( + {"init": lambda X_, k, random_state: X_[:2]}, + r"The shape of the initial centers .* does not match " + r"the number of clusters", + ), + ( + {"init": X[:8, :2]}, + r"The shape of the initial centers .* does not match " + r"the number of features of the data", + ), + ( + {"init": lambda X_, k, random_state: X_[:8, :2]}, + r"The shape of the initial centers .* does not match " + r"the number of features of the data", + ), + ], ) def test_wrong_params(Estimator, param, match): # Check that error are raised with clear error message when wrong values @@ -1013,48 +1151,32 @@ def test_wrong_params(Estimator, param, match): km.set_params(**param).fit(X) -@pytest.mark.parametrize("param, match", [ - ({"algorithm": "wrong"}, r"Algorithm must be 'auto', 'full' or 'elkan'")] -) -def test_kmeans_wrong_params(param, match): - # Check that error are raised with clear error message when wrong values - # are passed for the KMeans specific parameters - with pytest.raises(ValueError, match=match): - KMeans(**param).fit(X) - - -@pytest.mark.parametrize("param, match", [ - ({"max_no_improvement": -1}, r"max_no_improvement should be >= 0"), - ({"batch_size": -1}, r"batch_size should be > 0"), - ({"init_size": -1}, r"init_size should be > 0"), - ({"reassignment_ratio": -1}, r"reassignment_ratio should be >= 0")] -) -def test_minibatch_kmeans_wrong_params(param, match): - # Check that error are raised with clear error message when wrong values - # are passed for the MiniBatchKMeans specific parameters - with pytest.raises(ValueError, match=match): - MiniBatchKMeans(**param).fit(X) - - -@pytest.mark.parametrize("param, match", [ - ({"n_local_trials": 0}, - r"n_local_trials is set to 0 but should be an " - r"integer value greater than zero"), - ({"x_squared_norms": X[:2]}, - r"The length of x_squared_norms .* should " - r"be equal to the length of n_samples")] +@pytest.mark.parametrize( + "param, match", + [ + ( + {"x_squared_norms": X[:2]}, + r"The length of x_squared_norms .* should " + r"be equal to the length of n_samples", + ), + ], ) def test_kmeans_plusplus_wrong_params(param, match): with pytest.raises(ValueError, match=match): kmeans_plusplus(X, n_clusters, **param) -@pytest.mark.parametrize("data", [X, X_csr]) +@pytest.mark.parametrize( + "input_data", + [X] + X_as_any_csr, +) @pytest.mark.parametrize("dtype", [np.float64, np.float32]) -def test_kmeans_plusplus_output(data, dtype): +def test_kmeans_plusplus_output(input_data, dtype, global_random_seed): # Check for the correct number of seeds and all positive values - data = data.astype(dtype) - centers, indices = kmeans_plusplus(data, n_clusters) + data = input_data.astype(dtype) + centers, indices = kmeans_plusplus( + data, n_clusters, random_state=global_random_seed + ) # Check there are the correct number of indices and that all indices are # positive and within the number of samples @@ -1076,18 +1198,167 @@ def test_kmeans_plusplus_output(data, dtype): @pytest.mark.parametrize("x_squared_norms", [row_norms(X, squared=True), None]) def test_kmeans_plusplus_norms(x_squared_norms): # Check that defining x_squared_norms returns the same as default=None. - centers, indices = kmeans_plusplus(X, n_clusters, - x_squared_norms=x_squared_norms) + centers, indices = kmeans_plusplus(X, n_clusters, x_squared_norms=x_squared_norms) assert_allclose(X[indices], centers) -def test_kmeans_plusplus_dataorder(): +def test_kmeans_plusplus_dataorder(global_random_seed): # Check that memory layout does not effect result - centers_c, _ = kmeans_plusplus(X, n_clusters, random_state=0) + centers_c, _ = kmeans_plusplus(X, n_clusters, random_state=global_random_seed) X_fortran = np.asfortranarray(X) - centers_fortran, _ = kmeans_plusplus(X_fortran, n_clusters, random_state=0) + centers_fortran, _ = kmeans_plusplus( + X_fortran, n_clusters, random_state=global_random_seed + ) assert_allclose(centers_c, centers_fortran) + + +def test_is_same_clustering(): + # Sanity check for the _is_same_clustering utility function + labels1 = np.array([1, 0, 0, 1, 2, 0, 2, 1], dtype=np.int32) + assert _is_same_clustering(labels1, labels1, 3) + + # these other labels represent the same clustering since we can retrieve the first + # labels by simply renaming the labels: 0 -> 1, 1 -> 2, 2 -> 0. + labels2 = np.array([0, 2, 2, 0, 1, 2, 1, 0], dtype=np.int32) + assert _is_same_clustering(labels1, labels2, 3) + + # these other labels do not represent the same clustering since not all ones are + # mapped to a same value + labels3 = np.array([1, 0, 0, 2, 2, 0, 2, 1], dtype=np.int32) + assert not _is_same_clustering(labels1, labels3, 3) + + +@pytest.mark.parametrize( + "kwargs", ({"init": np.str_("k-means++")}, {"init": [[0, 0], [1, 1]], "n_init": 1}) +) +def test_kmeans_with_array_like_or_np_scalar_init(kwargs): + """Check that init works with numpy scalar strings. + + Non-regression test for #21964. + """ + X = np.asarray([[0, 0], [0.5, 0], [0.5, 1], [1, 1]], dtype=np.float64) + + clustering = KMeans(n_clusters=2, **kwargs) + # Does not raise + clustering.fit(X) + + +@pytest.mark.parametrize( + "Klass, method", + [(KMeans, "fit"), (MiniBatchKMeans, "fit"), (MiniBatchKMeans, "partial_fit")], +) +def test_feature_names_out(Klass, method): + """Check `feature_names_out` for `KMeans` and `MiniBatchKMeans`.""" + class_name = Klass.__name__.lower() + kmeans = Klass() + getattr(kmeans, method)(X) + n_clusters = kmeans.cluster_centers_.shape[0] + + names_out = kmeans.get_feature_names_out() + assert_array_equal([f"{class_name}{i}" for i in range(n_clusters)], names_out) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS + [None]) +def test_predict_does_not_change_cluster_centers(csr_container): + """Check that predict does not change cluster centers. + + Non-regression test for gh-24253. + """ + X, _ = make_blobs(n_samples=200, n_features=10, centers=10, random_state=0) + if csr_container is not None: + X = csr_container(X) + + kmeans = KMeans() + y_pred1 = kmeans.fit_predict(X) + # Make cluster_centers readonly + kmeans.cluster_centers_ = create_memmap_backed_data(kmeans.cluster_centers_) + kmeans.labels_ = create_memmap_backed_data(kmeans.labels_) + + y_pred2 = kmeans.predict(X) + assert_array_equal(y_pred1, y_pred2) + + +@pytest.mark.parametrize("init", ["k-means++", "random"]) +def test_sample_weight_init(init, global_random_seed): + """Check that sample weight is used during init. + + `_init_centroids` is shared across all classes inheriting from _BaseKMeans so + it's enough to check for KMeans. + """ + rng = np.random.RandomState(global_random_seed) + X, _ = make_blobs( + n_samples=200, n_features=10, centers=10, random_state=global_random_seed + ) + x_squared_norms = row_norms(X, squared=True) + + kmeans = KMeans() + clusters_weighted = kmeans._init_centroids( + X=X, + x_squared_norms=x_squared_norms, + init=init, + sample_weight=rng.uniform(size=X.shape[0]), + n_centroids=5, + random_state=np.random.RandomState(global_random_seed), + ) + clusters = kmeans._init_centroids( + X=X, + x_squared_norms=x_squared_norms, + init=init, + sample_weight=np.ones(X.shape[0]), + n_centroids=5, + random_state=np.random.RandomState(global_random_seed), + ) + with pytest.raises(AssertionError): + assert_allclose(clusters_weighted, clusters) + + +@pytest.mark.parametrize("init", ["k-means++", "random"]) +def test_sample_weight_zero(init, global_random_seed): + """Check that if sample weight is 0, this sample won't be chosen. + + `_init_centroids` is shared across all classes inheriting from _BaseKMeans so + it's enough to check for KMeans. + """ + rng = np.random.RandomState(global_random_seed) + X, _ = make_blobs( + n_samples=100, n_features=5, centers=5, random_state=global_random_seed + ) + sample_weight = rng.uniform(size=X.shape[0]) + sample_weight[::2] = 0 + x_squared_norms = row_norms(X, squared=True) + + kmeans = KMeans() + clusters_weighted = kmeans._init_centroids( + X=X, + x_squared_norms=x_squared_norms, + init=init, + sample_weight=sample_weight, + n_centroids=10, + random_state=np.random.RandomState(global_random_seed), + ) + # No center should be one of the 0 sample weight point + # (i.e. be at a distance=0 from it) + d = euclidean_distances(X[::2], clusters_weighted) + assert not np.any(np.isclose(d, 0)) + + +@pytest.mark.parametrize("array_constr", data_containers, ids=data_containers_ids) +@pytest.mark.parametrize("algorithm", ["lloyd", "elkan"]) +def test_relocating_with_duplicates(algorithm, array_constr): + """Check that kmeans stops when there are more centers than non-duplicate samples + + Non-regression test for issue: + https://github.com/scikit-learn/scikit-learn/issues/28055 + """ + X = np.array([[0, 0], [1, 1], [1, 1], [1, 0], [0, 1]]) + km = KMeans(n_clusters=5, init=X, algorithm=algorithm) + + msg = r"Number of distinct clusters \(4\) found smaller than n_clusters \(5\)" + with pytest.warns(ConvergenceWarning, match=msg): + km.fit(array_constr(X)) + + assert km.n_iter_ == 1 diff --git a/sklearn/cluster/tests/test_mean_shift.py b/sklearn/cluster/tests/test_mean_shift.py index 2feb5363c28c8..7216a064ccbc7 100644 --- a/sklearn/cluster/tests/test_mean_shift.py +++ b/sklearn/cluster/tests/test_mean_shift.py @@ -3,28 +3,35 @@ """ -import numpy as np import warnings -import pytest -from scipy import sparse - -from sklearn.utils._testing import assert_array_equal -from sklearn.utils._testing import assert_array_almost_equal -from sklearn.utils._testing import assert_allclose +import numpy as np +import pytest -from sklearn.cluster import MeanShift -from sklearn.cluster import mean_shift -from sklearn.cluster import estimate_bandwidth -from sklearn.cluster import get_bin_seeds +from sklearn.cluster import MeanShift, estimate_bandwidth, get_bin_seeds, mean_shift from sklearn.datasets import make_blobs from sklearn.metrics import v_measure_score - +from sklearn.utils._testing import assert_allclose, assert_array_equal n_clusters = 3 centers = np.array([[1, 1], [-1, -1], [1, -1]]) + 10 -X, _ = make_blobs(n_samples=300, n_features=2, centers=centers, - cluster_std=0.4, shuffle=True, random_state=11) +X, _ = make_blobs( + n_samples=300, + n_features=2, + centers=centers, + cluster_std=0.4, + shuffle=True, + random_state=11, +) + + +def test_convergence_of_1d_constant_data(): + # Test convergence using 1D constant data + # Non-regression test for: + # https://github.com/scikit-learn/scikit-learn/issues/28926 + model = MeanShift() + n_iter = model.fit(np.ones(10).reshape(-1, 1)).n_iter_ + assert n_iter < model.max_iter def test_estimate_bandwidth(): @@ -33,53 +40,56 @@ def test_estimate_bandwidth(): assert 0.9 <= bandwidth <= 1.5 -def test_estimate_bandwidth_1sample(): +def test_estimate_bandwidth_1sample(global_dtype): # Test estimate_bandwidth when n_samples=1 and quantile<1, so that # n_neighbors is set to 1. - bandwidth = estimate_bandwidth(X, n_samples=1, quantile=0.3) - assert bandwidth == pytest.approx(0., abs=1e-5) + bandwidth = estimate_bandwidth( + X.astype(global_dtype, copy=False), n_samples=1, quantile=0.3 + ) + assert bandwidth.dtype == X.dtype + assert bandwidth == pytest.approx(0.0, abs=1e-5) -@pytest.mark.parametrize("bandwidth, cluster_all, expected, " - "first_cluster_label", - [(1.2, True, 3, 0), (1.2, False, 4, -1)]) -def test_mean_shift(bandwidth, cluster_all, expected, first_cluster_label): + +@pytest.mark.parametrize( + "bandwidth, cluster_all, expected, first_cluster_label", + [(1.2, True, 3, 0), (1.2, False, 4, -1)], +) +def test_mean_shift( + global_dtype, bandwidth, cluster_all, expected, first_cluster_label +): # Test MeanShift algorithm + X_with_global_dtype = X.astype(global_dtype, copy=False) ms = MeanShift(bandwidth=bandwidth, cluster_all=cluster_all) - labels = ms.fit(X).labels_ + labels = ms.fit(X_with_global_dtype).labels_ labels_unique = np.unique(labels) n_clusters_ = len(labels_unique) assert n_clusters_ == expected assert labels_unique[0] == first_cluster_label + assert ms.cluster_centers_.dtype == global_dtype - cluster_centers, labels_mean_shift = mean_shift(X, cluster_all=cluster_all) + cluster_centers, labels_mean_shift = mean_shift( + X_with_global_dtype, cluster_all=cluster_all + ) labels_mean_shift_unique = np.unique(labels_mean_shift) n_clusters_mean_shift = len(labels_mean_shift_unique) assert n_clusters_mean_shift == expected assert labels_mean_shift_unique[0] == first_cluster_label + assert cluster_centers.dtype == global_dtype -def test_mean_shift_negative_bandwidth(): - bandwidth = -1 - ms = MeanShift(bandwidth=bandwidth) - msg = (r"bandwidth needs to be greater than zero or None," - r" got -1\.000000") - with pytest.raises(ValueError, match=msg): - ms.fit(X) - - -def test_estimate_bandwidth_with_sparse_matrix(): - # Test estimate_bandwidth with sparse matrix - X = sparse.lil_matrix((1000, 1000)) - msg = "A sparse matrix was passed, but dense data is required." - with pytest.raises(TypeError, match=msg): - estimate_bandwidth(X) - - -def test_parallel(): +def test_parallel(global_dtype, global_random_seed): centers = np.array([[1, 1], [-1, -1], [1, -1]]) + 10 - X, _ = make_blobs(n_samples=50, n_features=2, centers=centers, - cluster_std=0.4, shuffle=True, random_state=11) + X, _ = make_blobs( + n_samples=50, + n_features=2, + centers=centers, + cluster_std=0.4, + shuffle=True, + random_state=global_random_seed, + ) + + X = X.astype(global_dtype, copy=False) ms1 = MeanShift(n_jobs=2) ms1.fit(X) @@ -87,15 +97,17 @@ def test_parallel(): ms2 = MeanShift() ms2.fit(X) - assert_array_almost_equal(ms1.cluster_centers_, ms2.cluster_centers_) + assert_allclose(ms1.cluster_centers_, ms2.cluster_centers_) + assert ms1.cluster_centers_.dtype == ms2.cluster_centers_.dtype assert_array_equal(ms1.labels_, ms2.labels_) -def test_meanshift_predict(): +def test_meanshift_predict(global_dtype): # Test MeanShift.predict ms = MeanShift(bandwidth=1.2) - labels = ms.fit_predict(X) - labels2 = ms.predict(X) + X_with_global_dtype = X.astype(global_dtype, copy=False) + labels = ms.fit_predict(X_with_global_dtype) + labels2 = ms.predict(X_with_global_dtype) assert_array_equal(labels, labels2) @@ -104,7 +116,9 @@ def test_meanshift_all_orphans(): ms = MeanShift(bandwidth=0.1, seeds=[[-9, -9], [-10, -10]]) msg = "No point was within bandwidth=0.1" with pytest.raises(ValueError, match=msg): - ms.fit(X,) + ms.fit( + X, + ) def test_unfitted(): @@ -114,35 +128,35 @@ def test_unfitted(): assert not hasattr(ms, "labels_") -def test_cluster_intensity_tie(): - X = np.array([[1, 1], [2, 1], [1, 0], - [4, 7], [3, 5], [3, 6]]) +def test_cluster_intensity_tie(global_dtype): + X = np.array([[1, 1], [2, 1], [1, 0], [4, 7], [3, 5], [3, 6]], dtype=global_dtype) c1 = MeanShift(bandwidth=2).fit(X) - X = np.array([[4, 7], [3, 5], [3, 6], - [1, 1], [2, 1], [1, 0]]) + X = np.array([[4, 7], [3, 5], [3, 6], [1, 1], [2, 1], [1, 0]], dtype=global_dtype) c2 = MeanShift(bandwidth=2).fit(X) assert_array_equal(c1.labels_, [1, 1, 1, 0, 0, 0]) assert_array_equal(c2.labels_, [0, 0, 0, 1, 1, 1]) -def test_bin_seeds(): +def test_bin_seeds(global_dtype): # Test the bin seeding technique which can be used in the mean shift # algorithm # Data is just 6 points in the plane - X = np.array([[1., 1.], [1.4, 1.4], [1.8, 1.2], - [2., 1.], [2.1, 1.1], [0., 0.]]) + X = np.array( + [[1.0, 1.0], [1.4, 1.4], [1.8, 1.2], [2.0, 1.0], [2.1, 1.1], [0.0, 0.0]], + dtype=global_dtype, + ) # With a bin coarseness of 1.0 and min_bin_freq of 1, 3 bins should be # found - ground_truth = {(1., 1.), (2., 1.), (0., 0.)} + ground_truth = {(1.0, 1.0), (2.0, 1.0), (0.0, 0.0)} test_bins = get_bin_seeds(X, 1, 1) test_result = set(tuple(p) for p in test_bins) assert len(ground_truth.symmetric_difference(test_result)) == 0 # With a bin coarseness of 1.0 and min_bin_freq of 2, 2 bins should be # found - ground_truth = {(1., 1.), (2., 1.)} + ground_truth = {(1.0, 1.0), (2.0, 1.0)} test_bins = get_bin_seeds(X, 1, 2) test_result = set(tuple(p) for p in test_bins) assert len(ground_truth.symmetric_difference(test_result)) == 0 @@ -151,16 +165,22 @@ def test_bin_seeds(): # we bail and use the whole data here. with warnings.catch_warnings(record=True): test_bins = get_bin_seeds(X, 0.01, 1) - assert_array_almost_equal(test_bins, X) + assert_allclose(test_bins, X) # tight clusters around [0, 0] and [1, 1], only get two bins - X, _ = make_blobs(n_samples=100, n_features=2, centers=[[0, 0], [1, 1]], - cluster_std=0.1, random_state=0) + X, _ = make_blobs( + n_samples=100, + n_features=2, + centers=[[0, 0], [1, 1]], + cluster_std=0.1, + random_state=0, + ) + X = X.astype(global_dtype, copy=False) test_bins = get_bin_seeds(X, 1) assert_array_equal(test_bins, [[0, 0], [1, 1]]) -@pytest.mark.parametrize('max_iter', [1, 100]) +@pytest.mark.parametrize("max_iter", [1, 100]) def test_max_iter(max_iter): clusters1, _ = mean_shift(X, max_iter=max_iter) ms = MeanShift(max_iter=max_iter).fit(X) @@ -173,9 +193,9 @@ def test_max_iter(max_iter): assert np.allclose(c1, c2) -def test_mean_shift_zero_bandwidth(): +def test_mean_shift_zero_bandwidth(global_dtype): # Check that mean shift works when the estimated bandwidth is 0. - X = np.array([1, 1, 1, 2, 2, 2, 3, 3]).reshape(-1, 1) + X = np.array([1, 1, 1, 2, 2, 2, 3, 3], dtype=global_dtype).reshape(-1, 1) # estimate_bandwidth with default args returns 0 on this dataset bandwidth = estimate_bandwidth(X) @@ -190,6 +210,6 @@ def test_mean_shift_zero_bandwidth(): ms_nobinning = MeanShift(bin_seeding=False).fit(X) expected_labels = np.array([0, 0, 0, 1, 1, 1, 2, 2]) - assert v_measure_score(ms_binning.labels_, expected_labels) == 1 - assert v_measure_score(ms_nobinning.labels_, expected_labels) == 1 + assert v_measure_score(ms_binning.labels_, expected_labels) == pytest.approx(1) + assert v_measure_score(ms_nobinning.labels_, expected_labels) == pytest.approx(1) assert_allclose(ms_binning.cluster_centers_, ms_nobinning.cluster_centers_) diff --git a/sklearn/cluster/tests/test_optics.py b/sklearn/cluster/tests/test_optics.py index b253173c0b957..cf7d36f7848af 100644 --- a/sklearn/cluster/tests/test_optics.py +++ b/sklearn/cluster/tests/test_optics.py @@ -1,45 +1,46 @@ -# Authors: Shane Grigsby -# Adrin Jalali -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import warnings + import numpy as np import pytest -from sklearn.datasets import make_blobs -from sklearn.cluster import OPTICS +from sklearn.cluster import DBSCAN, OPTICS from sklearn.cluster._optics import _extend_region, _extract_xi_labels -from sklearn.exceptions import DataConversionWarning +from sklearn.cluster.tests.common import generate_clustered_data +from sklearn.datasets import make_blobs +from sklearn.exceptions import DataConversionWarning, EfficiencyWarning from sklearn.metrics.cluster import contingency_matrix from sklearn.metrics.pairwise import pairwise_distances -from sklearn.cluster import DBSCAN from sklearn.utils import shuffle -from sklearn.utils._testing import assert_array_equal -from sklearn.utils._testing import assert_allclose - -from sklearn.cluster.tests.common import generate_clustered_data - +from sklearn.utils._testing import assert_allclose, assert_array_equal +from sklearn.utils.fixes import CSR_CONTAINERS rng = np.random.RandomState(0) n_points_per_cluster = 10 -C1 = [-5, -2] + .8 * rng.randn(n_points_per_cluster, 2) -C2 = [4, -1] + .1 * rng.randn(n_points_per_cluster, 2) -C3 = [1, -2] + .2 * rng.randn(n_points_per_cluster, 2) -C4 = [-2, 3] + .3 * rng.randn(n_points_per_cluster, 2) +C1 = [-5, -2] + 0.8 * rng.randn(n_points_per_cluster, 2) +C2 = [4, -1] + 0.1 * rng.randn(n_points_per_cluster, 2) +C3 = [1, -2] + 0.2 * rng.randn(n_points_per_cluster, 2) +C4 = [-2, 3] + 0.3 * rng.randn(n_points_per_cluster, 2) C5 = [3, -2] + 1.6 * rng.randn(n_points_per_cluster, 2) C6 = [5, 6] + 2 * rng.randn(n_points_per_cluster, 2) X = np.vstack((C1, C2, C3, C4, C5, C6)) @pytest.mark.parametrize( - ('r_plot', 'end'), - [[[10, 8.9, 8.8, 8.7, 7, 10], 3], - [[10, 8.9, 8.8, 8.7, 8.6, 7, 10], 0], - [[10, 8.9, 8.8, 8.7, 7, 6, np.inf], 4], - [[10, 8.9, 8.8, 8.7, 7, 6, np.inf], 4], - ]) + ("r_plot", "end"), + [ + [[10, 8.9, 8.8, 8.7, 7, 10], 3], + [[10, 8.9, 8.8, 8.7, 8.6, 7, 10], 0], + [[10, 8.9, 8.8, 8.7, 7, 6, np.inf], 4], + [[10, 8.9, 8.8, 8.7, 7, 6, np.inf], 4], + ], +) def test_extend_downward(r_plot, end): r_plot = np.array(r_plot) ratio = r_plot[:-1] / r_plot[1:] - steep_downward = ratio >= 1 / .9 + steep_downward = ratio >= 1 / 0.9 upward = ratio < 1 e = _extend_region(steep_downward, upward, 0, 2) @@ -47,16 +48,18 @@ def test_extend_downward(r_plot, end): @pytest.mark.parametrize( - ('r_plot', 'end'), - [[[1, 2, 2.1, 2.2, 4, 8, 8, np.inf], 6], - [[1, 2, 2.1, 2.2, 2.3, 4, 8, 8, np.inf], 0], - [[1, 2, 2.1, 2, np.inf], 0], - [[1, 2, 2.1, np.inf], 2], - ]) + ("r_plot", "end"), + [ + [[1, 2, 2.1, 2.2, 4, 8, 8, np.inf], 6], + [[1, 2, 2.1, 2.2, 2.3, 4, 8, 8, np.inf], 0], + [[1, 2, 2.1, 2, np.inf], 0], + [[1, 2, 2.1, np.inf], 2], + ], +) def test_extend_upward(r_plot, end): r_plot = np.array(r_plot) ratio = r_plot[:-1] / r_plot[1:] - steep_upward = ratio <= .9 + steep_upward = ratio <= 0.9 downward = ratio > 1 e = _extend_region(steep_upward, downward, 0, 2) @@ -64,110 +67,130 @@ def test_extend_upward(r_plot, end): @pytest.mark.parametrize( - ('ordering', 'clusters', 'expected'), - [[[0, 1, 2, 3], [[0, 1], [2, 3]], [0, 0, 1, 1]], - [[0, 1, 2, 3], [[0, 1], [3, 3]], [0, 0, -1, 1]], - [[0, 1, 2, 3], [[0, 1], [3, 3], [0, 3]], [0, 0, -1, 1]], - [[3, 1, 2, 0], [[0, 1], [3, 3], [0, 3]], [1, 0, -1, 0]], - ]) + ("ordering", "clusters", "expected"), + [ + [[0, 1, 2, 3], [[0, 1], [2, 3]], [0, 0, 1, 1]], + [[0, 1, 2, 3], [[0, 1], [3, 3]], [0, 0, -1, 1]], + [[0, 1, 2, 3], [[0, 1], [3, 3], [0, 3]], [0, 0, -1, 1]], + [[3, 1, 2, 0], [[0, 1], [3, 3], [0, 3]], [1, 0, -1, 0]], + ], +) def test_the_extract_xi_labels(ordering, clusters, expected): labels = _extract_xi_labels(ordering, clusters) assert_array_equal(labels, expected) -def test_extract_xi(): +def test_extract_xi(global_dtype): # small and easy test (no clusters around other clusters) # but with a clear noise data. + # global_random_seed is not used here since the expected labels + # are hardcoded for these specific data. rng = np.random.RandomState(0) n_points_per_cluster = 5 - C1 = [-5, -2] + .8 * rng.randn(n_points_per_cluster, 2) - C2 = [4, -1] + .1 * rng.randn(n_points_per_cluster, 2) - C3 = [1, -2] + .2 * rng.randn(n_points_per_cluster, 2) - C4 = [-2, 3] + .3 * rng.randn(n_points_per_cluster, 2) - C5 = [3, -2] + .6 * rng.randn(n_points_per_cluster, 2) - C6 = [5, 6] + .2 * rng.randn(n_points_per_cluster, 2) - - X = np.vstack((C1, C2, C3, C4, C5, np.array([[100, 100]]), C6)) - expected_labels = np.r_[[2] * 5, [0] * 5, [1] * 5, [3] * 5, [1] * 5, - -1, [4] * 5] + C1 = [-5, -2] + 0.8 * rng.randn(n_points_per_cluster, 2) + C2 = [4, -1] + 0.1 * rng.randn(n_points_per_cluster, 2) + C3 = [1, -2] + 0.2 * rng.randn(n_points_per_cluster, 2) + C4 = [-2, 3] + 0.3 * rng.randn(n_points_per_cluster, 2) + C5 = [3, -2] + 0.6 * rng.randn(n_points_per_cluster, 2) + C6 = [5, 6] + 0.2 * rng.randn(n_points_per_cluster, 2) + + X = np.vstack((C1, C2, C3, C4, C5, np.array([[100, 100]]), C6)).astype( + global_dtype, copy=False + ) + expected_labels = np.r_[[2] * 5, [0] * 5, [1] * 5, [3] * 5, [1] * 5, -1, [4] * 5] X, expected_labels = shuffle(X, expected_labels, random_state=rng) - clust = OPTICS(min_samples=3, min_cluster_size=2, - max_eps=20, cluster_method='xi', - xi=0.4).fit(X) + clust = OPTICS( + min_samples=3, min_cluster_size=2, max_eps=20, cluster_method="xi", xi=0.4 + ).fit(X) assert_array_equal(clust.labels_, expected_labels) # check float min_samples and min_cluster_size - clust = OPTICS(min_samples=0.1, min_cluster_size=0.08, - max_eps=20, cluster_method='xi', - xi=0.4).fit(X) + clust = OPTICS( + min_samples=0.1, min_cluster_size=0.08, max_eps=20, cluster_method="xi", xi=0.4 + ).fit(X) assert_array_equal(clust.labels_, expected_labels) - X = np.vstack((C1, C2, C3, C4, C5, np.array([[100, 100]] * 2), C6)) - expected_labels = np.r_[[1] * 5, [3] * 5, [2] * 5, [0] * 5, [2] * 5, - -1, -1, [4] * 5] + X = np.vstack((C1, C2, C3, C4, C5, np.array([[100, 100]] * 2), C6)).astype( + global_dtype, copy=False + ) + expected_labels = np.r_[ + [1] * 5, [3] * 5, [2] * 5, [0] * 5, [2] * 5, -1, -1, [4] * 5 + ] X, expected_labels = shuffle(X, expected_labels, random_state=rng) - clust = OPTICS(min_samples=3, min_cluster_size=3, - max_eps=20, cluster_method='xi', - xi=0.3).fit(X) + clust = OPTICS( + min_samples=3, min_cluster_size=3, max_eps=20, cluster_method="xi", xi=0.3 + ).fit(X) # this may fail if the predecessor correction is not at work! assert_array_equal(clust.labels_, expected_labels) - C1 = [[0, 0], [0, 0.1], [0, -.1], [0.1, 0]] + C1 = [[0, 0], [0, 0.1], [0, -0.1], [0.1, 0]] C2 = [[10, 10], [10, 9], [10, 11], [9, 10]] C3 = [[100, 100], [100, 90], [100, 110], [90, 100]] - X = np.vstack((C1, C2, C3)) + X = np.vstack((C1, C2, C3)).astype(global_dtype, copy=False) expected_labels = np.r_[[0] * 4, [1] * 4, [2] * 4] X, expected_labels = shuffle(X, expected_labels, random_state=rng) - clust = OPTICS(min_samples=2, min_cluster_size=2, - max_eps=np.inf, cluster_method='xi', - xi=0.04).fit(X) + clust = OPTICS( + min_samples=2, min_cluster_size=2, max_eps=np.inf, cluster_method="xi", xi=0.04 + ).fit(X) assert_array_equal(clust.labels_, expected_labels) -def test_cluster_hierarchy_(): - rng = np.random.RandomState(0) +def test_cluster_hierarchy(global_dtype, global_random_seed): + rng = np.random.RandomState(global_random_seed) n_points_per_cluster = 100 - C1 = [0, 0] + 2 * rng.randn(n_points_per_cluster, 2) - C2 = [0, 0] + 50 * rng.randn(n_points_per_cluster, 2) + C1 = [0, 0] + 2 * rng.randn(n_points_per_cluster, 2).astype( + global_dtype, copy=False + ) + C2 = [0, 0] + 50 * rng.randn(n_points_per_cluster, 2).astype( + global_dtype, copy=False + ) X = np.vstack((C1, C2)) - X = shuffle(X, random_state=0) + X = shuffle(X, random_state=rng) - clusters = OPTICS(min_samples=20, xi=.1).fit(X).cluster_hierarchy_ + clusters = OPTICS(min_samples=20, xi=0.2).fit(X).cluster_hierarchy_ assert clusters.shape == (2, 2) - diff = np.sum(clusters - np.array([[0, 99], [0, 199]])) - assert diff / len(X) < 0.05 + # The first cluster should contain all point from C1 but due to how the data is + # generated, some points from C2 may end up in it. + assert 100 <= np.diff(clusters[0]) + 1 <= 115 + # The second cluster should contain all points from C1 and C2. + assert np.diff(clusters[-1]) + 1 == 200 -def test_correct_number_of_clusters(): + +@pytest.mark.parametrize( + "csr_container, metric", + [(None, "minkowski")] + [(container, "euclidean") for container in CSR_CONTAINERS], +) +def test_correct_number_of_clusters(metric, csr_container): # in 'auto' mode n_clusters = 3 X = generate_clustered_data(n_clusters=n_clusters) # Parameters chosen specifically for this task. # Compute OPTICS - clust = OPTICS(max_eps=5.0 * 6.0, min_samples=4, xi=.1) - clust.fit(X) + clust = OPTICS(max_eps=5.0 * 6.0, min_samples=4, xi=0.1, metric=metric) + clust.fit(csr_container(X) if csr_container is not None else X) # number of clusters, ignoring noise if present n_clusters_1 = len(set(clust.labels_)) - int(-1 in clust.labels_) assert n_clusters_1 == n_clusters # check attribute types and sizes assert clust.labels_.shape == (len(X),) - assert clust.labels_.dtype.kind == 'i' + assert clust.labels_.dtype.kind == "i" assert clust.reachability_.shape == (len(X),) - assert clust.reachability_.dtype.kind == 'f' + assert clust.reachability_.dtype.kind == "f" assert clust.core_distances_.shape == (len(X),) - assert clust.core_distances_.dtype.kind == 'f' + assert clust.core_distances_.dtype.kind == "f" assert clust.ordering_.shape == (len(X),) - assert clust.ordering_.dtype.kind == 'i' + assert clust.ordering_.dtype.kind == "i" assert set(clust.ordering_) == set(range(len(X))) @@ -177,7 +200,7 @@ def test_minimum_number_of_sample_check(): # Compute OPTICS X = [[1, 1]] - clust = OPTICS(max_eps=5.0 * 0.3, min_samples=10, min_cluster_size=1) + clust = OPTICS(max_eps=5.0 * 0.3, min_samples=10, min_cluster_size=1.0) # Run the fit with pytest.raises(ValueError, match=msg): @@ -188,13 +211,12 @@ def test_bad_extract(): # Test an extraction of eps too close to original eps msg = "Specify an epsilon smaller than 0.15. Got 0.3." centers = [[1, 1], [-1, -1], [1, -1]] - X, labels_true = make_blobs(n_samples=750, centers=centers, - cluster_std=0.4, random_state=0) + X, labels_true = make_blobs( + n_samples=750, centers=centers, cluster_std=0.4, random_state=0 + ) # Compute OPTICS - clust = OPTICS(max_eps=5.0 * 0.03, - cluster_method='dbscan', - eps=0.3, min_samples=10) + clust = OPTICS(max_eps=5.0 * 0.03, cluster_method="dbscan", eps=0.3, min_samples=10) with pytest.raises(ValueError, match=msg): clust.fit(X) @@ -202,8 +224,9 @@ def test_bad_extract(): def test_bad_reachability(): msg = "All reachability values are inf. Set a larger max_eps." centers = [[1, 1], [-1, -1], [1, -1]] - X, labels_true = make_blobs(n_samples=750, centers=centers, - cluster_std=0.4, random_state=0) + X, labels_true = make_blobs( + n_samples=750, centers=centers, cluster_std=0.4, random_state=0 + ) with pytest.warns(UserWarning, match=msg): clust = OPTICS(max_eps=5.0 * 0.003, min_samples=10, eps=0.015) @@ -215,12 +238,13 @@ def test_nowarn_if_metric_bool_data_bool(): # non-regression test for # https://github.com/scikit-learn/scikit-learn/issues/18996 - pairwise_metric = 'rogerstanimoto' + pairwise_metric = "rogerstanimoto" X = np.random.randint(2, size=(5, 2), dtype=bool) - with pytest.warns(None) as warn_record: + with warnings.catch_warnings(): + warnings.simplefilter("error", DataConversionWarning) + OPTICS(metric=pairwise_metric).fit(X) - assert len(warn_record) == 0 def test_warn_if_metric_bool_data_no_bool(): @@ -229,7 +253,7 @@ def test_warn_if_metric_bool_data_no_bool(): # non-regression test for # https://github.com/scikit-learn/scikit-learn/issues/18996 - pairwise_metric = 'rogerstanimoto' + pairwise_metric = "rogerstanimoto" X = np.random.randint(2, size=(5, 2), dtype=np.int32) msg = f"Data will be converted to boolean for metric {pairwise_metric}" @@ -241,51 +265,63 @@ def test_warn_if_metric_bool_data_no_bool(): def test_nowarn_if_metric_no_bool(): # make sure no conversion warning is raised if # metric isn't boolean, no matter what the data type is - pairwise_metric = 'minkowski' + pairwise_metric = "minkowski" X_bool = np.random.randint(2, size=(5, 2), dtype=bool) X_num = np.random.randint(2, size=(5, 2), dtype=np.int32) - with pytest.warns(None) as warn_record: + with warnings.catch_warnings(): + warnings.simplefilter("error", DataConversionWarning) + # fit boolean data OPTICS(metric=pairwise_metric).fit(X_bool) # fit numeric data OPTICS(metric=pairwise_metric).fit(X_num) - assert len(warn_record) == 0 def test_close_extract(): # Test extract where extraction eps is close to scaled max_eps centers = [[1, 1], [-1, -1], [1, -1]] - X, labels_true = make_blobs(n_samples=750, centers=centers, - cluster_std=0.4, random_state=0) + X, labels_true = make_blobs( + n_samples=750, centers=centers, cluster_std=0.4, random_state=0 + ) # Compute OPTICS - clust = OPTICS(max_eps=1.0, cluster_method='dbscan', - eps=0.3, min_samples=10).fit(X) + clust = OPTICS(max_eps=1.0, cluster_method="dbscan", eps=0.3, min_samples=10).fit(X) # Cluster ordering starts at 0; max cluster label = 2 is 3 clusters assert max(clust.labels_) == 2 -@pytest.mark.parametrize('eps', [0.1, .3, .5]) -@pytest.mark.parametrize('min_samples', [3, 10, 20]) -def test_dbscan_optics_parity(eps, min_samples): +@pytest.mark.parametrize("eps", [0.1, 0.3, 0.5]) +@pytest.mark.parametrize("min_samples", [3, 10, 20]) +@pytest.mark.parametrize( + "csr_container, metric", + [(None, "minkowski"), (None, "euclidean")] + + [(container, "euclidean") for container in CSR_CONTAINERS], +) +def test_dbscan_optics_parity(eps, min_samples, metric, global_dtype, csr_container): # Test that OPTICS clustering labels are <= 5% difference of DBSCAN centers = [[1, 1], [-1, -1], [1, -1]] - X, labels_true = make_blobs(n_samples=750, centers=centers, - cluster_std=0.4, random_state=0) + X, labels_true = make_blobs( + n_samples=150, centers=centers, cluster_std=0.4, random_state=0 + ) + X = csr_container(X) if csr_container is not None else X + + X = X.astype(global_dtype, copy=False) # calculate optics with dbscan extract at 0.3 epsilon - op = OPTICS(min_samples=min_samples, cluster_method='dbscan', - eps=eps).fit(X) + op = OPTICS( + min_samples=min_samples, cluster_method="dbscan", eps=eps, metric=metric + ).fit(X) # calculate dbscan labels db = DBSCAN(eps=eps, min_samples=min_samples).fit(X) contingency = contingency_matrix(db.labels_, op.labels_) - agree = min(np.sum(np.max(contingency, axis=0)), - np.sum(np.max(contingency, axis=1))) + agree = min( + np.sum(np.max(contingency, axis=0)), np.sum(np.max(contingency, axis=1)) + ) disagree = X.shape[0] - agree percent_mismatch = np.round((disagree - 1) / X.shape[0], 2) @@ -294,64 +330,59 @@ def test_dbscan_optics_parity(eps, min_samples): assert percent_mismatch <= 0.05 -def test_min_samples_edge_case(): - C1 = [[0, 0], [0, 0.1], [0, -.1]] +def test_min_samples_edge_case(global_dtype): + C1 = [[0, 0], [0, 0.1], [0, -0.1]] C2 = [[10, 10], [10, 9], [10, 11]] C3 = [[100, 100], [100, 96], [100, 106]] - X = np.vstack((C1, C2, C3)) + X = np.vstack((C1, C2, C3)).astype(global_dtype, copy=False) expected_labels = np.r_[[0] * 3, [1] * 3, [2] * 3] - clust = OPTICS(min_samples=3, - max_eps=7, cluster_method='xi', - xi=0.04).fit(X) + clust = OPTICS(min_samples=3, max_eps=7, cluster_method="xi", xi=0.04).fit(X) assert_array_equal(clust.labels_, expected_labels) expected_labels = np.r_[[0] * 3, [1] * 3, [-1] * 3] - clust = OPTICS(min_samples=3, - max_eps=3, cluster_method='xi', - xi=0.04).fit(X) + clust = OPTICS(min_samples=3, max_eps=3, cluster_method="xi", xi=0.04).fit(X) assert_array_equal(clust.labels_, expected_labels) expected_labels = np.r_[[-1] * 9] with pytest.warns(UserWarning, match="All reachability values"): - clust = OPTICS(min_samples=4, - max_eps=3, cluster_method='xi', - xi=0.04).fit(X) + clust = OPTICS(min_samples=4, max_eps=3, cluster_method="xi", xi=0.04).fit(X) assert_array_equal(clust.labels_, expected_labels) # try arbitrary minimum sizes -@pytest.mark.parametrize('min_cluster_size', range(2, X.shape[0] // 10, 23)) -def test_min_cluster_size(min_cluster_size): - redX = X[::2] # reduce for speed +@pytest.mark.parametrize("min_cluster_size", range(2, X.shape[0] // 10, 23)) +def test_min_cluster_size(min_cluster_size, global_dtype): + redX = X[::2].astype(global_dtype, copy=False) # reduce for speed clust = OPTICS(min_samples=9, min_cluster_size=min_cluster_size).fit(redX) cluster_sizes = np.bincount(clust.labels_[clust.labels_ != -1]) if cluster_sizes.size: assert min(cluster_sizes) >= min_cluster_size # check behaviour is the same when min_cluster_size is a fraction - clust_frac = OPTICS(min_samples=9, - min_cluster_size=min_cluster_size / redX.shape[0]) + clust_frac = OPTICS( + min_samples=9, + min_cluster_size=min_cluster_size / redX.shape[0], + ) clust_frac.fit(redX) assert_array_equal(clust.labels_, clust_frac.labels_) -@pytest.mark.parametrize('min_cluster_size', [0, -1, 1.1, 2.2]) -def test_min_cluster_size_invalid(min_cluster_size): - clust = OPTICS(min_cluster_size=min_cluster_size) - with pytest.raises(ValueError, match="must be a positive integer or a "): - clust.fit(X) - - -def test_min_cluster_size_invalid2(): +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_min_cluster_size_invalid2(csr_container): clust = OPTICS(min_cluster_size=len(X) + 1) with pytest.raises(ValueError, match="must be no greater than the "): clust.fit(X) + clust = OPTICS(min_cluster_size=len(X) + 1, metric="euclidean") + with pytest.raises(ValueError, match="must be no greater than the "): + clust.fit(csr_container(X)) + def test_processing_order(): # Ensure that we consider all unprocessed points, # not only direct neighbors. when picking the next point. Y = [[0], [10], [-10], [25]] + clust = OPTICS(min_samples=3, max_eps=15).fit(Y) assert_array_equal(clust.reachability_, [np.inf, 10, 10, 15]) assert_array_equal(clust.core_distances_, [10, 15, np.inf, np.inf]) @@ -363,34 +394,192 @@ def test_compare_to_ELKI(): # java -jar elki.jar cli -dbc.in csv -dbc.filter FixedDBIDsFilter # -algorithm clustering.optics.OPTICSHeap -optics.minpts 5 # where the FixedDBIDsFilter gives 0-indexed ids. - r1 = [np.inf, 1.0574896366427478, 0.7587934993548423, 0.7290174038973836, - 0.7290174038973836, 0.7290174038973836, 0.6861627576116127, - 0.7587934993548423, 0.9280118450166668, 1.1748022534146194, - 3.3355455741292257, 0.49618389254482587, 0.2552805046961355, - 0.2552805046961355, 0.24944622248445714, 0.24944622248445714, - 0.24944622248445714, 0.2552805046961355, 0.2552805046961355, - 0.3086779122185853, 4.163024452756142, 1.623152630340929, - 0.45315840475822655, 0.25468325192031926, 0.2254004358159971, - 0.18765711877083036, 0.1821471333893275, 0.1821471333893275, - 0.18765711877083036, 0.18765711877083036, 0.2240202988740153, - 1.154337614548715, 1.342604473837069, 1.323308536402633, - 0.8607514948648837, 0.27219111215810565, 0.13260875220533205, - 0.13260875220533205, 0.09890587675958984, 0.09890587675958984, - 0.13548790801634494, 0.1575483940837384, 0.17515137170530226, - 0.17575920159442388, 0.27219111215810565, 0.6101447895405373, - 1.3189208094864302, 1.323308536402633, 2.2509184159764577, - 2.4517810628594527, 3.675977064404973, 3.8264795626020365, - 2.9130735341510614, 2.9130735341510614, 2.9130735341510614, - 2.9130735341510614, 2.8459300127258036, 2.8459300127258036, - 2.8459300127258036, 3.0321982337972537] - o1 = [0, 3, 6, 4, 7, 8, 2, 9, 5, 1, 31, 30, 32, 34, 33, 38, 39, 35, 37, 36, - 44, 21, 23, 24, 22, 25, 27, 29, 26, 28, 20, 40, 45, 46, 10, 15, 11, - 13, 17, 19, 18, 12, 16, 14, 47, 49, 43, 48, 42, 41, 53, 57, 51, 52, - 56, 59, 54, 55, 58, 50] - p1 = [-1, 0, 3, 6, 6, 6, 8, 3, 7, 5, 1, 31, 30, 30, 34, 34, 34, 32, 32, 37, - 36, 44, 21, 23, 24, 22, 25, 25, 22, 22, 22, 21, 40, 45, 46, 10, 15, - 15, 13, 13, 15, 11, 19, 15, 10, 47, 12, 45, 14, 43, 42, 53, 57, 57, - 57, 57, 59, 59, 59, 58] + r1 = [ + np.inf, + 1.0574896366427478, + 0.7587934993548423, + 0.7290174038973836, + 0.7290174038973836, + 0.7290174038973836, + 0.6861627576116127, + 0.7587934993548423, + 0.9280118450166668, + 1.1748022534146194, + 3.3355455741292257, + 0.49618389254482587, + 0.2552805046961355, + 0.2552805046961355, + 0.24944622248445714, + 0.24944622248445714, + 0.24944622248445714, + 0.2552805046961355, + 0.2552805046961355, + 0.3086779122185853, + 4.163024452756142, + 1.623152630340929, + 0.45315840475822655, + 0.25468325192031926, + 0.2254004358159971, + 0.18765711877083036, + 0.1821471333893275, + 0.1821471333893275, + 0.18765711877083036, + 0.18765711877083036, + 0.2240202988740153, + 1.154337614548715, + 1.342604473837069, + 1.323308536402633, + 0.8607514948648837, + 0.27219111215810565, + 0.13260875220533205, + 0.13260875220533205, + 0.09890587675958984, + 0.09890587675958984, + 0.13548790801634494, + 0.1575483940837384, + 0.17515137170530226, + 0.17575920159442388, + 0.27219111215810565, + 0.6101447895405373, + 1.3189208094864302, + 1.323308536402633, + 2.2509184159764577, + 2.4517810628594527, + 3.675977064404973, + 3.8264795626020365, + 2.9130735341510614, + 2.9130735341510614, + 2.9130735341510614, + 2.9130735341510614, + 2.8459300127258036, + 2.8459300127258036, + 2.8459300127258036, + 3.0321982337972537, + ] + o1 = [ + 0, + 3, + 6, + 4, + 7, + 8, + 2, + 9, + 5, + 1, + 31, + 30, + 32, + 34, + 33, + 38, + 39, + 35, + 37, + 36, + 44, + 21, + 23, + 24, + 22, + 25, + 27, + 29, + 26, + 28, + 20, + 40, + 45, + 46, + 10, + 15, + 11, + 13, + 17, + 19, + 18, + 12, + 16, + 14, + 47, + 49, + 43, + 48, + 42, + 41, + 53, + 57, + 51, + 52, + 56, + 59, + 54, + 55, + 58, + 50, + ] + p1 = [ + -1, + 0, + 3, + 6, + 6, + 6, + 8, + 3, + 7, + 5, + 1, + 31, + 30, + 30, + 34, + 34, + 34, + 32, + 32, + 37, + 36, + 44, + 21, + 23, + 24, + 22, + 25, + 25, + 22, + 22, + 22, + 21, + 40, + 45, + 46, + 10, + 15, + 15, + 13, + 13, + 15, + 11, + 19, + 15, + 10, + 47, + 12, + 45, + 14, + 43, + 42, + 53, + 57, + 57, + 57, + 57, + 59, + 59, + 59, + 58, + ] # Tests against known extraction array # Does NOT work with metric='euclidean', because sklearn euclidean has @@ -403,32 +592,195 @@ def test_compare_to_ELKI(): # ELKI currently does not print the core distances (which are not used much # in literature, but we can at least ensure to have this consistency: for i in clust1.ordering_[1:]: - assert (clust1.reachability_[i] >= - clust1.core_distances_[clust1.predecessor_[i]]) + assert clust1.reachability_[i] >= clust1.core_distances_[clust1.predecessor_[i]] # Expected values, computed with (future) ELKI 0.7.5 using - r2 = [np.inf, np.inf, np.inf, np.inf, np.inf, np.inf, np.inf, np.inf, - np.inf, np.inf, np.inf, 0.27219111215810565, 0.13260875220533205, - 0.13260875220533205, 0.09890587675958984, 0.09890587675958984, - 0.13548790801634494, 0.1575483940837384, 0.17515137170530226, - 0.17575920159442388, 0.27219111215810565, 0.4928068613197889, - np.inf, 0.2666183922512113, 0.18765711877083036, 0.1821471333893275, - 0.1821471333893275, 0.1821471333893275, 0.18715928772277457, - 0.18765711877083036, 0.18765711877083036, 0.25468325192031926, - np.inf, 0.2552805046961355, 0.2552805046961355, 0.24944622248445714, - 0.24944622248445714, 0.24944622248445714, 0.2552805046961355, - 0.2552805046961355, 0.3086779122185853, 0.34466409325984865, - np.inf, np.inf, np.inf, np.inf, np.inf, np.inf, np.inf, np.inf, - np.inf, np.inf, np.inf, np.inf, np.inf, np.inf, np.inf, np.inf, - np.inf, np.inf] - o2 = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 11, 13, 17, 19, 18, 12, 16, 14, - 47, 46, 20, 22, 25, 23, 27, 29, 24, 26, 28, 21, 30, 32, 34, 33, 38, - 39, 35, 37, 36, 31, 40, 41, 42, 43, 44, 45, 48, 49, 50, 51, 52, 53, - 54, 55, 56, 57, 58, 59] - p2 = [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 10, 15, 15, 13, 13, 15, - 11, 19, 15, 10, 47, -1, 20, 22, 25, 25, 25, 25, 22, 22, 23, -1, 30, - 30, 34, 34, 34, 32, 32, 37, 38, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1] + r2 = [ + np.inf, + np.inf, + np.inf, + np.inf, + np.inf, + np.inf, + np.inf, + np.inf, + np.inf, + np.inf, + np.inf, + 0.27219111215810565, + 0.13260875220533205, + 0.13260875220533205, + 0.09890587675958984, + 0.09890587675958984, + 0.13548790801634494, + 0.1575483940837384, + 0.17515137170530226, + 0.17575920159442388, + 0.27219111215810565, + 0.4928068613197889, + np.inf, + 0.2666183922512113, + 0.18765711877083036, + 0.1821471333893275, + 0.1821471333893275, + 0.1821471333893275, + 0.18715928772277457, + 0.18765711877083036, + 0.18765711877083036, + 0.25468325192031926, + np.inf, + 0.2552805046961355, + 0.2552805046961355, + 0.24944622248445714, + 0.24944622248445714, + 0.24944622248445714, + 0.2552805046961355, + 0.2552805046961355, + 0.3086779122185853, + 0.34466409325984865, + np.inf, + np.inf, + np.inf, + np.inf, + np.inf, + np.inf, + np.inf, + np.inf, + np.inf, + np.inf, + np.inf, + np.inf, + np.inf, + np.inf, + np.inf, + np.inf, + np.inf, + np.inf, + ] + o2 = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 15, + 11, + 13, + 17, + 19, + 18, + 12, + 16, + 14, + 47, + 46, + 20, + 22, + 25, + 23, + 27, + 29, + 24, + 26, + 28, + 21, + 30, + 32, + 34, + 33, + 38, + 39, + 35, + 37, + 36, + 31, + 40, + 41, + 42, + 43, + 44, + 45, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + ] + p2 = [ + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + 10, + 15, + 15, + 13, + 13, + 15, + 11, + 19, + 15, + 10, + 47, + -1, + 20, + 22, + 25, + 25, + 25, + 25, + 22, + 22, + 23, + -1, + 30, + 30, + 34, + 34, + 34, + 32, + 32, + 37, + 38, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + ] clust2 = OPTICS(min_samples=5, max_eps=0.5).fit(X) assert_array_equal(clust2.ordering_, np.array(o2)) @@ -436,38 +788,81 @@ def test_compare_to_ELKI(): assert_allclose(clust2.reachability_[clust2.ordering_], np.array(r2)) index = np.where(clust1.core_distances_ <= 0.5)[0] - assert_allclose(clust1.core_distances_[index], - clust2.core_distances_[index]) + assert_allclose(clust1.core_distances_[index], clust2.core_distances_[index]) -def test_wrong_cluster_method(): - clust = OPTICS(cluster_method='superfancy') - with pytest.raises(ValueError, match="cluster_method should be one of "): - clust.fit(X) - - -def test_extract_dbscan(): +def test_extract_dbscan(global_dtype, global_random_seed): # testing an easy dbscan case. Not including clusters with different # densities. - rng = np.random.RandomState(0) + rng = np.random.RandomState(global_random_seed) n_points_per_cluster = 20 - C1 = [-5, -2] + .2 * rng.randn(n_points_per_cluster, 2) - C2 = [4, -1] + .2 * rng.randn(n_points_per_cluster, 2) - C3 = [1, 2] + .2 * rng.randn(n_points_per_cluster, 2) - C4 = [-2, 3] + .2 * rng.randn(n_points_per_cluster, 2) - X = np.vstack((C1, C2, C3, C4)) - - clust = OPTICS(cluster_method='dbscan', eps=.5).fit(X) - assert_array_equal(np.sort(np.unique(clust.labels_)), [0, 1, 2, 3]) - - -def test_precomputed_dists(): - redX = X[::2] - dists = pairwise_distances(redX, metric='euclidean') - clust1 = OPTICS(min_samples=10, algorithm='brute', - metric='precomputed').fit(dists) - clust2 = OPTICS(min_samples=10, algorithm='brute', - metric='euclidean').fit(redX) + C1 = [-5, -2] + 0.2 * rng.randn(n_points_per_cluster, 2) + C2 = [4, -1] + 0.2 * rng.randn(n_points_per_cluster, 2) + C3 = [1, 2] + 0.2 * rng.randn(n_points_per_cluster, 2) + C4 = [-2, 3] + 0.2 * rng.randn(n_points_per_cluster, 2) + X = np.vstack((C1, C2, C3, C4)).astype(global_dtype, copy=False) + + clust = OPTICS(cluster_method="dbscan", eps=0.5).fit(X) + assert_array_equal( + np.sort(np.unique(clust.labels_[clust.labels_ != -1])), [0, 1, 2, 3] + ) + + +@pytest.mark.parametrize("csr_container", [None] + CSR_CONTAINERS) +def test_precomputed_dists(global_dtype, csr_container): + redX = X[::2].astype(global_dtype, copy=False) + dists = pairwise_distances(redX, metric="euclidean") + dists = csr_container(dists) if csr_container is not None else dists + with warnings.catch_warnings(): + warnings.simplefilter("ignore", EfficiencyWarning) + clust1 = OPTICS(min_samples=10, algorithm="brute", metric="precomputed").fit( + dists + ) + clust2 = OPTICS(min_samples=10, algorithm="brute", metric="euclidean").fit(redX) assert_allclose(clust1.reachability_, clust2.reachability_) assert_array_equal(clust1.labels_, clust2.labels_) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_optics_input_not_modified_precomputed_sparse_nodiag( + csr_container, global_random_seed +): + """Check that we don't modify in-place the pre-computed sparse matrix. + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/27508 + """ + X = np.random.RandomState(global_random_seed).rand(6, 6) + # Add zeros on the diagonal that will be implicit when creating + # the sparse matrix. If `X` is modified in-place, the zeros from + # the diagonal will be made explicit. + np.fill_diagonal(X, 0) + X = csr_container(X) + assert all(row != col for row, col in zip(*X.nonzero())) + X_copy = X.copy() + OPTICS(metric="precomputed").fit(X) + # Make sure that we did not modify `X` in-place even by creating + # explicit 0s values. + assert X.nnz == X_copy.nnz + assert_array_equal(X.toarray(), X_copy.toarray()) + + +def test_optics_predecessor_correction_ordering(): + """Check that cluster correction using predecessor is working as expected. + + In the following example, the predecessor correction was not working properly + since it was not using the right indices. + + This non-regression test check that reordering the data does not change the results. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/26324 + """ + X_1 = np.array([1, 2, 3, 1, 8, 8, 7, 100]).reshape(-1, 1) + reorder = [0, 1, 2, 4, 5, 6, 7, 3] + X_2 = X_1[reorder] + + optics_1 = OPTICS(min_samples=3, metric="euclidean").fit(X_1) + optics_2 = OPTICS(min_samples=3, metric="euclidean").fit(X_2) + + assert_array_equal(optics_1.labels_[reorder], optics_2.labels_) diff --git a/sklearn/cluster/tests/test_spectral.py b/sklearn/cluster/tests/test_spectral.py index 6962e98917ed0..71b11c9fe151c 100644 --- a/sklearn/cluster/tests/test_spectral.py +++ b/sklearn/cluster/tests/test_spectral.py @@ -1,49 +1,67 @@ """Testing for Spectral Clustering methods""" + +import pickle import re import numpy as np -from scipy import sparse - import pytest - -import pickle - -from sklearn.utils import check_random_state -from sklearn.utils._testing import assert_array_equal +from scipy.linalg import LinAlgError from sklearn.cluster import SpectralClustering, spectral_clustering -from sklearn.cluster._spectral import discretize +from sklearn.cluster._spectral import cluster_qr, discretize +from sklearn.datasets import make_blobs from sklearn.feature_extraction import img_to_graph -from sklearn.metrics import pairwise_distances from sklearn.metrics import adjusted_rand_score from sklearn.metrics.pairwise import kernel_metrics, rbf_kernel from sklearn.neighbors import NearestNeighbors -from sklearn.datasets import make_blobs +from sklearn.utils import check_random_state +from sklearn.utils._testing import assert_array_equal +from sklearn.utils.fixes import COO_CONTAINERS, CSR_CONTAINERS try: - from pyamg import smoothed_aggregation_solver # noqa + from pyamg import smoothed_aggregation_solver # noqa: F401 + amg_loaded = True except ImportError: amg_loaded = False - -@pytest.mark.parametrize('eigen_solver', ('arpack', 'lobpcg')) -@pytest.mark.parametrize('assign_labels', ('kmeans', 'discretize')) -def test_spectral_clustering(eigen_solver, assign_labels): - S = np.array([[1.0, 1.0, 1.0, 0.2, 0.0, 0.0, 0.0], - [1.0, 1.0, 1.0, 0.2, 0.0, 0.0, 0.0], - [1.0, 1.0, 1.0, 0.2, 0.0, 0.0, 0.0], - [0.2, 0.2, 0.2, 1.0, 1.0, 1.0, 1.0], - [0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0], - [0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0], - [0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0]]) - - for mat in (S, sparse.csr_matrix(S)): - model = SpectralClustering(random_state=0, n_clusters=2, - affinity='precomputed', - eigen_solver=eigen_solver, - assign_labels=assign_labels - ).fit(mat) +centers = np.array([[1, 1], [-1, -1], [1, -1]]) + 10 +X, _ = make_blobs( + n_samples=60, + n_features=2, + centers=centers, + cluster_std=0.4, + shuffle=True, + random_state=0, +) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +@pytest.mark.parametrize("eigen_solver", ("arpack", "lobpcg")) +@pytest.mark.parametrize("assign_labels", ("kmeans", "discretize", "cluster_qr")) +def test_spectral_clustering( + eigen_solver, assign_labels, csr_container, global_random_seed +): + S = np.array( + [ + [1.0, 1.0, 1.0, 0.2, 0.0, 0.0, 0.0], + [1.0, 1.0, 1.0, 0.2, 0.0, 0.0, 0.0], + [1.0, 1.0, 1.0, 0.2, 0.0, 0.0, 0.0], + [0.2, 0.2, 0.2, 1.0, 1.0, 1.0, 1.0], + [0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0], + [0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0], + [0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0], + ] + ) + + for mat in (S, csr_container(S)): + model = SpectralClustering( + random_state=global_random_seed, + n_clusters=2, + affinity="precomputed", + eigen_solver=eigen_solver, + assign_labels=assign_labels, + ).fit(mat) labels = model.labels_ if labels[0] == 0: labels = 1 - labels @@ -56,86 +74,76 @@ def test_spectral_clustering(eigen_solver, assign_labels): assert_array_equal(model_copy.labels_, model.labels_) -def test_spectral_unknown_mode(): - # Test that SpectralClustering fails with an unknown mode set. - centers = np.array([ - [0., 0., 0.], - [10., 10., 10.], - [20., 20., 20.], - ]) - X, true_labels = make_blobs(n_samples=100, centers=centers, - cluster_std=1., random_state=42) - D = pairwise_distances(X) # Distance matrix - S = np.max(D) - D # Similarity matrix - S = sparse.coo_matrix(S) - with pytest.raises(ValueError): - spectral_clustering(S, n_clusters=2, random_state=0, - eigen_solver="") - - -def test_spectral_unknown_assign_labels(): - # Test that SpectralClustering fails with an unknown assign_labels set. - centers = np.array([ - [0., 0., 0.], - [10., 10., 10.], - [20., 20., 20.], - ]) - X, true_labels = make_blobs(n_samples=100, centers=centers, - cluster_std=1., random_state=42) - D = pairwise_distances(X) # Distance matrix - S = np.max(D) - D # Similarity matrix - S = sparse.coo_matrix(S) - with pytest.raises(ValueError): - spectral_clustering(S, n_clusters=2, random_state=0, - assign_labels="") - - -def test_spectral_clustering_sparse(): - X, y = make_blobs(n_samples=20, random_state=0, - centers=[[1, 1], [-1, -1]], cluster_std=0.01) +@pytest.mark.parametrize("coo_container", COO_CONTAINERS) +@pytest.mark.parametrize("assign_labels", ("kmeans", "discretize", "cluster_qr")) +def test_spectral_clustering_sparse(assign_labels, coo_container, global_random_seed): + X, y = make_blobs( + n_samples=20, + random_state=global_random_seed, + centers=[[1, 1], [-1, -1]], + cluster_std=0.01, + ) S = rbf_kernel(X, gamma=1) S = np.maximum(S - 1e-4, 0) - S = sparse.coo_matrix(S) - - labels = SpectralClustering(random_state=0, n_clusters=2, - affinity='precomputed').fit(S).labels_ + S = coo_container(S) + + labels = ( + SpectralClustering( + random_state=global_random_seed, + n_clusters=2, + affinity="precomputed", + assign_labels=assign_labels, + ) + .fit(S) + .labels_ + ) assert adjusted_rand_score(y, labels) == 1 -def test_precomputed_nearest_neighbors_filtering(): +def test_precomputed_nearest_neighbors_filtering(global_random_seed): # Test precomputed graph filtering when containing too many neighbors - X, y = make_blobs(n_samples=200, random_state=0, - centers=[[1, 1], [-1, -1]], cluster_std=0.01) + X, y = make_blobs( + n_samples=250, + random_state=global_random_seed, + centers=[[1, 1, 1], [-1, -1, -1]], + cluster_std=0.01, + ) n_neighbors = 2 results = [] for additional_neighbors in [0, 10]: - nn = NearestNeighbors( - n_neighbors=n_neighbors + additional_neighbors).fit(X) - graph = nn.kneighbors_graph(X, mode='connectivity') - labels = SpectralClustering(random_state=0, n_clusters=2, - affinity='precomputed_nearest_neighbors', - n_neighbors=n_neighbors).fit(graph).labels_ + nn = NearestNeighbors(n_neighbors=n_neighbors + additional_neighbors).fit(X) + graph = nn.kneighbors_graph(X, mode="distance") + labels = ( + SpectralClustering( + random_state=global_random_seed, + n_clusters=2, + affinity="precomputed_nearest_neighbors", + n_neighbors=n_neighbors, + ) + .fit(graph) + .labels_ + ) results.append(labels) assert_array_equal(results[0], results[1]) -def test_affinities(): +def test_affinities(global_random_seed): # Note: in the following, random_state has been selected to have # a dataset that yields a stable eigen decomposition both when built # on OSX and Linux - X, y = make_blobs(n_samples=20, random_state=0, - centers=[[1, 1], [-1, -1]], cluster_std=0.01) + X, y = make_blobs( + n_samples=20, random_state=0, centers=[[1, 1], [-1, -1]], cluster_std=0.01 + ) # nearest neighbors affinity - sp = SpectralClustering(n_clusters=2, affinity='nearest_neighbors', - random_state=0) - with pytest.warns(UserWarning, match='not fully connected'): + sp = SpectralClustering(n_clusters=2, affinity="nearest_neighbors", random_state=0) + with pytest.warns(UserWarning, match="not fully connected"): sp.fit(X) assert adjusted_rand_score(y, sp.labels_) == 1 - sp = SpectralClustering(n_clusters=2, gamma=2, random_state=0) + sp = SpectralClustering(n_clusters=2, gamma=2, random_state=global_random_seed) labels = sp.fit(X).labels_ assert adjusted_rand_score(y, labels) == 1 @@ -145,61 +153,77 @@ def test_affinities(): for kern in kernels_available: # Additive chi^2 gives a negative similarity matrix which # doesn't make sense for spectral clustering - if kern != 'additive_chi2': - sp = SpectralClustering(n_clusters=2, affinity=kern, - random_state=0) + if kern != "additive_chi2": + sp = SpectralClustering(n_clusters=2, affinity=kern, random_state=0) labels = sp.fit(X).labels_ assert (X.shape[0],) == labels.shape - sp = SpectralClustering(n_clusters=2, affinity=lambda x, y: 1, - random_state=0) + sp = SpectralClustering(n_clusters=2, affinity=lambda x, y: 1, random_state=0) labels = sp.fit(X).labels_ assert (X.shape[0],) == labels.shape def histogram(x, y, **kwargs): # Histogram kernel implemented as a callable. - assert kwargs == {} # no kernel_params that we didn't ask for + assert kwargs == {} # no kernel_params that we didn't ask for return np.minimum(x, y).sum() sp = SpectralClustering(n_clusters=2, affinity=histogram, random_state=0) labels = sp.fit(X).labels_ assert (X.shape[0],) == labels.shape - # raise error on unknown affinity - sp = SpectralClustering(n_clusters=2, affinity='') - with pytest.raises(ValueError): - sp.fit(X) - -@pytest.mark.parametrize('n_samples', [50, 100, 150, 500]) -def test_discretize(n_samples): +def test_cluster_qr(global_random_seed): + # cluster_qr by itself should not be used for clustering generic data + # other than the rows of the eigenvectors within spectral clustering, + # but cluster_qr must still preserve the labels for different dtypes + # of the generic fixed input even if the labels may be meaningless. + random_state = np.random.RandomState(seed=global_random_seed) + n_samples, n_components = 10, 5 + data = random_state.randn(n_samples, n_components) + labels_float64 = cluster_qr(data.astype(np.float64)) + # Each sample is assigned a cluster identifier + assert labels_float64.shape == (n_samples,) + # All components should be covered by the assignment + assert np.array_equal(np.unique(labels_float64), np.arange(n_components)) + # Single precision data should yield the same cluster assignments + labels_float32 = cluster_qr(data.astype(np.float32)) + assert np.array_equal(labels_float64, labels_float32) + + +def test_cluster_qr_permutation_invariance(global_random_seed): + # cluster_qr must be invariant to sample permutation. + random_state = np.random.RandomState(seed=global_random_seed) + n_samples, n_components = 100, 5 + data = random_state.randn(n_samples, n_components) + perm = random_state.permutation(n_samples) + assert np.array_equal( + cluster_qr(data)[perm], + cluster_qr(data[perm]), + ) + + +@pytest.mark.parametrize("coo_container", COO_CONTAINERS) +@pytest.mark.parametrize("n_samples", [50, 100, 150, 500]) +def test_discretize(n_samples, coo_container, global_random_seed): # Test the discretize using a noise assignment matrix - random_state = np.random.RandomState(seed=8) + random_state = np.random.RandomState(seed=global_random_seed) for n_class in range(2, 10): # random class labels y_true = random_state.randint(0, n_class + 1, n_samples) y_true = np.array(y_true, float) # noise class assignment matrix - y_indicator = sparse.coo_matrix((np.ones(n_samples), - (np.arange(n_samples), - y_true)), - shape=(n_samples, - n_class + 1)) - y_true_noisy = (y_indicator.toarray() - + 0.1 * random_state.randn(n_samples, - n_class + 1)) + y_indicator = coo_container( + (np.ones(n_samples), (np.arange(n_samples), y_true)), + shape=(n_samples, n_class + 1), + ) + y_true_noisy = y_indicator.toarray() + 0.1 * random_state.randn( + n_samples, n_class + 1 + ) y_pred = discretize(y_true_noisy, random_state=random_state) assert adjusted_rand_score(y_true, y_pred) > 0.8 -# TODO: Remove when pyamg does replaces sp.rand call with np.random.rand -# https://github.com/scikit-learn/scikit-learn/issues/15913 -@pytest.mark.filterwarnings( - "ignore:scipy.rand is deprecated:DeprecationWarning:pyamg.*") -# TODO: Remove when pyamg removes the use of np.float -@pytest.mark.filterwarnings( - "ignore:`np.float` is a deprecated alias:DeprecationWarning:pyamg.*") -def test_spectral_clustering_with_arpack_amg_solvers(): +def test_spectral_clustering_with_arpack_amg_solvers(global_random_seed): # Test that spectral_clustering is the same for arpack and amg solver # Based on toy example from plot_segmentation_toy.py @@ -209,8 +233,8 @@ def test_spectral_clustering_with_arpack_amg_solvers(): center1, center2 = (14, 12), (20, 25) radius1, radius2 = 8, 7 - circle1 = (x - center1[0]) ** 2 + (y - center1[1]) ** 2 < radius1 ** 2 - circle2 = (x - center2[0]) ** 2 + (y - center2[1]) ** 2 < radius2 ** 2 + circle1 = (x - center1[0]) ** 2 + (y - center1[1]) ** 2 < radius1**2 + circle2 = (x - center2[0]) ** 2 + (y - center2[1]) ** 2 < radius2**2 circles = circle1 | circle2 mask = circles.copy() @@ -220,45 +244,59 @@ def test_spectral_clustering_with_arpack_amg_solvers(): graph.data = np.exp(-graph.data / graph.data.std()) labels_arpack = spectral_clustering( - graph, n_clusters=2, eigen_solver='arpack', random_state=0) + graph, n_clusters=2, eigen_solver="arpack", random_state=global_random_seed + ) assert len(np.unique(labels_arpack)) == 2 if amg_loaded: labels_amg = spectral_clustering( - graph, n_clusters=2, eigen_solver='amg', random_state=0) + graph, n_clusters=2, eigen_solver="amg", random_state=global_random_seed + ) assert adjusted_rand_score(labels_arpack, labels_amg) == 1 else: with pytest.raises(ValueError): - spectral_clustering(graph, n_clusters=2, eigen_solver='amg', - random_state=0) + spectral_clustering(graph, n_clusters=2, eigen_solver="amg", random_state=0) -def test_n_components(): +def test_n_components(global_random_seed): # Test that after adding n_components, result is different and # n_components = n_clusters by default - X, y = make_blobs(n_samples=20, random_state=0, - centers=[[1, 1], [-1, -1]], cluster_std=0.01) - sp = SpectralClustering(n_clusters=2, random_state=0) + X, y = make_blobs( + n_samples=20, + random_state=global_random_seed, + centers=[[1, 1], [-1, -1]], + cluster_std=0.01, + ) + sp = SpectralClustering(n_clusters=2, random_state=global_random_seed) labels = sp.fit(X).labels_ # set n_components = n_cluster and test if result is the same - labels_same_ncomp = SpectralClustering(n_clusters=2, n_components=2, - random_state=0).fit(X).labels_ + labels_same_ncomp = ( + SpectralClustering( + n_clusters=2, n_components=2, random_state=global_random_seed + ) + .fit(X) + .labels_ + ) # test that n_components=n_clusters by default assert_array_equal(labels, labels_same_ncomp) # test that n_components affect result # n_clusters=8 by default, and set n_components=2 - labels_diff_ncomp = SpectralClustering(n_components=2, - random_state=0).fit(X).labels_ + labels_diff_ncomp = ( + SpectralClustering(n_components=2, random_state=global_random_seed) + .fit(X) + .labels_ + ) assert not np.array_equal(labels, labels_diff_ncomp) -@pytest.mark.parametrize('assign_labels', ('kmeans', 'discretize')) +@pytest.mark.parametrize("assign_labels", ("kmeans", "discretize", "cluster_qr")) def test_verbose(assign_labels, capsys): # Check verbose mode of KMeans for better coverage. - X, y = make_blobs(n_samples=20, random_state=0, - centers=[[1, 1], [-1, -1]], cluster_std=0.01) + X, y = make_blobs( + n_samples=20, random_state=0, centers=[[1, 1], [-1, -1]], cluster_std=0.01 + ) SpectralClustering(n_clusters=2, random_state=42, verbose=1).fit(X) @@ -271,11 +309,27 @@ def test_verbose(assign_labels, capsys): assert re.search(r"Iteration [0-9]+, inertia", captured.out) -# TODO: Remove in 1.1 -@pytest.mark.parametrize("affinity", ["precomputed", - "precomputed_nearest_neighbors"]) -def test_pairwise_is_deprecated(affinity): - sp = SpectralClustering(affinity=affinity) - msg = r"Attribute _pairwise was deprecated in version 0\.24" - with pytest.warns(FutureWarning, match=msg): - sp._pairwise +def test_spectral_clustering_np_matrix_raises(): + """Check that spectral_clustering raises an informative error when passed + a np.matrix. See #10993""" + X = np.matrix([[0.0, 2.0], [2.0, 0.0]]) + + msg = r"np\.matrix is not supported. Please convert to a numpy array" + with pytest.raises(TypeError, match=msg): + spectral_clustering(X) + + +def test_spectral_clustering_not_infinite_loop(capsys, monkeypatch): + """Check that discretize raises LinAlgError when svd never converges. + + Non-regression test for #21380 + """ + + def new_svd(*args, **kwargs): + raise LinAlgError() + + monkeypatch.setattr(np.linalg, "svd", new_svd) + vectors = np.ones((10, 4)) + + with pytest.raises(LinAlgError, match="SVD did not converge"): + discretize(vectors) diff --git a/sklearn/compose/__init__.py b/sklearn/compose/__init__.py index ea734aa230053..842a86ba21d9b 100644 --- a/sklearn/compose/__init__.py +++ b/sklearn/compose/__init__.py @@ -1,18 +1,23 @@ -"""Meta-estimators for building composite models with transformers +"""Meta-estimators for building composite models with transformers. In addition to its current contents, this module will eventually be home to -refurbished versions of Pipeline and FeatureUnion. - +refurbished versions of :class:`~sklearn.pipeline.Pipeline` and +:class:`~sklearn.pipeline.FeatureUnion`. """ -from ._column_transformer import (ColumnTransformer, make_column_transformer, - make_column_selector) -from ._target import TransformedTargetRegressor +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +from ._column_transformer import ( + ColumnTransformer, + make_column_selector, + make_column_transformer, +) +from ._target import TransformedTargetRegressor __all__ = [ - 'ColumnTransformer', - 'make_column_transformer', - 'TransformedTargetRegressor', - 'make_column_selector', + "ColumnTransformer", + "TransformedTargetRegressor", + "make_column_selector", + "make_column_transformer", ] diff --git a/sklearn/compose/_column_transformer.py b/sklearn/compose/_column_transformer.py index c0444fe2d6cda..2b9c32659e66e 100644 --- a/sklearn/compose/_column_transformer.py +++ b/sklearn/compose/_column_transformer.py @@ -3,36 +3,61 @@ to work with heterogeneous data and to apply different transformers to different columns. """ -# Author: Andreas Mueller -# Joris Van den Bossche -# License: BSD + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import warnings +from collections import Counter +from functools import partial from itertools import chain +from numbers import Integral, Real import numpy as np from scipy import sparse -from joblib import Parallel -from ..base import clone, TransformerMixin -from ..utils._estimator_html_repr import _VisualBlock -from ..pipeline import _fit_transform_one, _transform_one, _name_estimators +from ..base import TransformerMixin, _fit_context, clone +from ..pipeline import _fit_transform_one, _name_estimators, _transform_one from ..preprocessing import FunctionTransformer from ..utils import Bunch -from ..utils import _safe_indexing -from ..utils import _get_column_indices +from ..utils._indexing import _determine_key_type, _get_column_indices, _safe_indexing +from ..utils._metadata_requests import METHODS +from ..utils._param_validation import HasMethods, Hidden, Interval, StrOptions +from ..utils._repr_html.estimator import _VisualBlock +from ..utils._set_output import ( + _get_container_adapter, + _get_output_config, + _safe_set_output, +) +from ..utils._tags import get_tags +from ..utils.metadata_routing import ( + MetadataRouter, + MethodMapping, + _raise_for_params, + _routing_enabled, + process_routing, +) from ..utils.metaestimators import _BaseComposition -from ..utils.validation import check_array, check_is_fitted -from ..utils.validation import _deprecate_positional_args -from ..utils.fixes import delayed +from ..utils.parallel import Parallel, delayed +from ..utils.validation import ( + _check_feature_names, + _check_feature_names_in, + _check_n_features, + _get_feature_names, + _is_pandas_df, + _num_samples, + check_array, + check_is_fitted, +) +__all__ = ["ColumnTransformer", "make_column_selector", "make_column_transformer"] -__all__ = [ - 'ColumnTransformer', 'make_column_transformer', 'make_column_selector' -] - -_ERR_MSG_1DCOLUMN = ("1D data passed to a transformer that expects 2D data. " - "Try to specify the column selection as a list of one " - "item instead of a scalar.") +_ERR_MSG_1DCOLUMN = ( + "1D data passed to a transformer that expects 2D data. " + "Try to specify the column selection as a list of one " + "item instead of a scalar." +) class ColumnTransformer(TransformerMixin, _BaseComposition): @@ -79,9 +104,11 @@ class ColumnTransformer(TransformerMixin, _BaseComposition): transformed and combined in the output, and the non-specified columns are dropped. (default of ``'drop'``). By specifying ``remainder='passthrough'``, all remaining columns that - were not specified in `transformers` will be automatically passed - through. This subset of columns is concatenated with the output of - the transformers. + were not specified in `transformers`, but present in the data passed + to `fit` will be automatically passed through. This subset of columns + is concatenated with the output of the transformers. For dataframes, + extra columns not seen during `fit` will be excluded from the output + of `transform`. By setting ``remainder`` to be an estimator, the remaining non-specified columns will use the ``remainder`` estimator. The estimator must support :term:`fit` and :term:`transform`. @@ -110,20 +137,68 @@ class ColumnTransformer(TransformerMixin, _BaseComposition): If True, the time elapsed while fitting each transformer will be printed as it is completed. + verbose_feature_names_out : bool, str or Callable[[str, str], str], default=True + + - If True, :meth:`ColumnTransformer.get_feature_names_out` will prefix + all feature names with the name of the transformer that generated that + feature. It is equivalent to setting + `verbose_feature_names_out="{transformer_name}__{feature_name}"`. + - If False, :meth:`ColumnTransformer.get_feature_names_out` will not + prefix any feature names and will error if feature names are not + unique. + - If ``Callable[[str, str], str]``, + :meth:`ColumnTransformer.get_feature_names_out` will rename all the features + using the name of the transformer. The first argument of the callable is the + transformer name and the second argument is the feature name. The returned + string will be the new feature name. + - If ``str``, it must be a string ready for formatting. The given string will + be formatted using two field names: ``transformer_name`` and ``feature_name``. + e.g. ``"{feature_name}__{transformer_name}"``. See :meth:`str.format` method + from the standard library for more info. + + .. versionadded:: 1.0 + + .. versionchanged:: 1.6 + `verbose_feature_names_out` can be a callable or a string to be formatted. + + force_int_remainder_cols : bool, default=False + This parameter has no effect. + + .. note:: + If you do not access the list of columns for the remainder columns + in the `transformers_` fitted attribute, you do not need to set + this parameter. + + .. versionadded:: 1.5 + + .. versionchanged:: 1.7 + The default value for `force_int_remainder_cols` will change from + `True` to `False` in version 1.7. + + .. deprecated:: 1.7 + `force_int_remainder_cols` is deprecated and will be removed in 1.9. + Attributes ---------- transformers_ : list - The collection of fitted transformers as tuples of - (name, fitted_transformer, column). `fitted_transformer` can be an - estimator, 'drop', or 'passthrough'. In case there were no columns - selected, this will be the unfitted transformer. - If there are remaining columns, the final element is a tuple of the - form: + The collection of fitted transformers as tuples of (name, + fitted_transformer, column). `fitted_transformer` can be an estimator, + or `'drop'`; `'passthrough'` is replaced with an equivalent + :class:`~sklearn.preprocessing.FunctionTransformer`. In case there were + no columns selected, this will be the unfitted transformer. If there + are remaining columns, the final element is a tuple of the form: ('remainder', transformer, remaining_columns) corresponding to the ``remainder`` parameter. If there are remaining columns, then ``len(transformers_)==len(transformers)+1``, otherwise ``len(transformers_)==len(transformers)``. + .. versionadded:: 1.7 + The format of the remaining columns now attempts to match that of the other + transformers: if all columns were provided as column names (`str`), the + remaining columns are stored as column names; if all columns were provided + as mask arrays (`bool`), so are the remaining columns; in all other cases + the remaining columns are stored as indices (`int`). + named_transformers_ : :class:`~sklearn.utils.Bunch` Read-only attribute to access any transformer by given name. Keys are transformer names and values are the fitted transformer @@ -134,14 +209,25 @@ class ColumnTransformer(TransformerMixin, _BaseComposition): sparse matrix or a dense numpy array, which depends on the output of the individual transformers and the `sparse_threshold` keyword. - Notes - ----- - The order of the columns in the transformed feature matrix follows the - order of how the columns are specified in the `transformers` list. - Columns of the original feature matrix that are not specified are - dropped from the resulting transformed feature matrix, unless specified - in the `passthrough` keyword. Those columns specified with `passthrough` - are added at the right to the output of the transformers. + output_indices_ : dict + A dictionary from each transformer name to a slice, where the slice + corresponds to indices in the transformed output. This is useful to + inspect which transformer is responsible for which transformed + feature(s). + + .. versionadded:: 1.0 + + n_features_in_ : int + Number of features seen during :term:`fit`. Only defined if the + underlying transformers expose such an attribute when fit. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 See Also -------- @@ -151,6 +237,15 @@ class ColumnTransformer(TransformerMixin, _BaseComposition): make_column_selector : Convenience function for selecting columns based on datatype or the columns name with a regex pattern. + Notes + ----- + The order of the columns in the transformed feature matrix follows the + order of how the columns are specified in the `transformers` list. + Columns of the original feature matrix that are not specified are + dropped from the resulting transformed feature matrix, unless specified + in the `passthrough` keyword. Those columns specified with `passthrough` + are added at the right to the output of the transformers. + Examples -------- >>> import numpy as np @@ -168,39 +263,132 @@ class ColumnTransformer(TransformerMixin, _BaseComposition): array([[0. , 1. , 0.5, 0.5], [0.5, 0.5, 0. , 1. ]]) + :class:`ColumnTransformer` can be configured with a transformer that requires + a 1d array by setting the column to a string: + + >>> from sklearn.feature_extraction.text import CountVectorizer + >>> from sklearn.preprocessing import MinMaxScaler + >>> import pandas as pd # doctest: +SKIP + >>> X = pd.DataFrame({ + ... "documents": ["First item", "second one here", "Is this the last?"], + ... "width": [3, 4, 5], + ... }) # doctest: +SKIP + >>> # "documents" is a string which configures ColumnTransformer to + >>> # pass the documents column as a 1d array to the CountVectorizer + >>> ct = ColumnTransformer( + ... [("text_preprocess", CountVectorizer(), "documents"), + ... ("num_preprocess", MinMaxScaler(), ["width"])]) + >>> X_trans = ct.fit_transform(X) # doctest: +SKIP + + For a more detailed example of usage, see + :ref:`sphx_glr_auto_examples_compose_plot_column_transformer_mixed_types.py`. """ - _required_parameters = ['transformers'] - - @_deprecate_positional_args - def __init__(self, - transformers, *, - remainder='drop', - sparse_threshold=0.3, - n_jobs=None, - transformer_weights=None, - verbose=False): + + _parameter_constraints: dict = { + "transformers": [list, Hidden(tuple)], + "remainder": [ + StrOptions({"drop", "passthrough"}), + HasMethods(["fit", "transform"]), + HasMethods(["fit_transform", "transform"]), + ], + "sparse_threshold": [Interval(Real, 0, 1, closed="both")], + "n_jobs": [Integral, None], + "transformer_weights": [dict, None], + "verbose": ["verbose"], + "verbose_feature_names_out": ["boolean", str, callable], + "force_int_remainder_cols": ["boolean", Hidden(StrOptions({"deprecated"}))], + } + + def __init__( + self, + transformers, + *, + remainder="drop", + sparse_threshold=0.3, + n_jobs=None, + transformer_weights=None, + verbose=False, + verbose_feature_names_out=True, + force_int_remainder_cols="deprecated", + ): self.transformers = transformers self.remainder = remainder self.sparse_threshold = sparse_threshold self.n_jobs = n_jobs self.transformer_weights = transformer_weights self.verbose = verbose + self.verbose_feature_names_out = verbose_feature_names_out + self.force_int_remainder_cols = force_int_remainder_cols @property def _transformers(self): """ Internal list of transformer only containing the name and - transformers, dropping the columns. This is for the implementation - of get_params via BaseComposition._get_params which expects lists - of tuples of len 2. + transformers, dropping the columns. + + DO NOT USE: This is for the implementation of get_params via + BaseComposition._get_params which expects lists of tuples of len 2. + + To iterate through the transformers, use ``self._iter`` instead. """ - return [(name, trans) for name, trans, _ in self.transformers] + try: + return [(name, trans) for name, trans, _ in self.transformers] + except (TypeError, ValueError): + return self.transformers @_transformers.setter def _transformers(self, value): - self.transformers = [ - (name, trans, col) for ((name, trans), (_, _, col)) - in zip(value, self.transformers)] + """DO NOT USE: This is for the implementation of set_params via + BaseComposition._get_params which gives lists of tuples of len 2. + """ + try: + self.transformers = [ + (name, trans, col) + for ((name, trans), (_, _, col)) in zip(value, self.transformers) + ] + except (TypeError, ValueError): + self.transformers = value + + def set_output(self, *, transform=None): + """Set the output container when `"transform"` and `"fit_transform"` are called. + + Calling `set_output` will set the output of all estimators in `transformers` + and `transformers_`. + + Parameters + ---------- + transform : {"default", "pandas", "polars"}, default=None + Configure output of `transform` and `fit_transform`. + + - `"default"`: Default output format of a transformer + - `"pandas"`: DataFrame output + - `"polars"`: Polars output + - `None`: Transform configuration is unchanged + + .. versionadded:: 1.4 + `"polars"` option was added. + + Returns + ------- + self : estimator instance + Estimator instance. + """ + super().set_output(transform=transform) + + transformers = ( + trans + for _, trans, _ in chain( + self.transformers, getattr(self, "transformers_", []) + ) + if trans not in {"passthrough", "drop"} + ) + for trans in transformers: + _safe_set_output(trans, transform=transform) + + if self.remainder not in {"passthrough", "drop"}: + _safe_set_output(self.remainder, transform=transform) + + return self def get_params(self, deep=True): """Get parameters for this estimator. @@ -220,7 +408,7 @@ def get_params(self, deep=True): params : dict Parameter names mapped to their values. """ - return self._get_params('_transformers', deep=deep) + return self._get_params("_transformers", deep=deep) def set_params(self, **kwargs): """Set the parameters of this estimator. @@ -229,51 +417,89 @@ def set_params(self, **kwargs): can directly set the parameters of the estimators contained in `transformers` of `ColumnTransformer`. + Parameters + ---------- + **kwargs : dict + Estimator parameters. + Returns ------- - self + self : ColumnTransformer + This estimator. """ - self._set_params('_transformers', **kwargs) + self._set_params("_transformers", **kwargs) return self - def _iter(self, fitted=False, replace_strings=False): + def _iter(self, fitted, column_as_labels, skip_drop, skip_empty_columns): """ - Generate (name, trans, column, weight) tuples. + Generate (name, trans, columns, weight) tuples. - If fitted=True, use the fitted transformers, else use the - user specified transformers updated with converted column names - and potentially appended with transformer for remainder. + Parameters + ---------- + fitted : bool + If True, use the fitted transformers (``self.transformers_``) to + iterate through transformers, else use the transformers passed by + the user (``self.transformers``). + + column_as_labels : bool + If True, columns are returned as string labels. If False, columns + are returned as they were given by the user. This can only be True + if the ``ColumnTransformer`` is already fitted. + + skip_drop : bool + If True, 'drop' transformers are filtered out. + + skip_empty_columns : bool + If True, transformers with empty selected columns are filtered out. + + Yields + ------ + A generator of tuples containing: + - name : the name of the transformer + - transformer : the transformer object + - columns : the columns for that transformer + - weight : the weight of the transformer """ if fitted: transformers = self.transformers_ else: # interleave the validated column specifiers transformers = [ - (name, trans, column) for (name, trans, _), column - in zip(self.transformers, self._columns) + (name, trans, column) + for (name, trans, _), column in zip(self.transformers, self._columns) ] # add transformer tuple for remainder - if self._remainder[2] is not None: + if self._remainder[2]: transformers = chain(transformers, [self._remainder]) + get_weight = (self.transformer_weights or {}).get - for name, trans, column in transformers: - if replace_strings: - # replace 'passthrough' with identity transformer and - # skip in case of 'drop' - if trans == 'passthrough': - trans = FunctionTransformer( - accept_sparse=True, check_inverse=False - ) - elif trans == 'drop': - continue - elif _is_empty_column_selection(column): - continue + for name, trans, columns in transformers: + if skip_drop and trans == "drop": + continue + if skip_empty_columns and _is_empty_column_selection(columns): + continue + + if column_as_labels: + # Convert all columns to using their string labels + columns_is_scalar = np.isscalar(columns) + + indices = self._transformer_to_input_indices[name] + columns = self.feature_names_in_[indices] + + if columns_is_scalar: + # selection is done with one dimension + columns = columns[0] - yield (name, trans, column, get_weight(name)) + yield (name, trans, columns, get_weight(name)) def _validate_transformers(self): + """Validate names of transformers and the transformers themselves. + + This checks whether given transformers have the required methods, i.e. + `fit` or `fit_transform` and `transform` implemented. + """ if not self.transformers: return @@ -284,48 +510,68 @@ def _validate_transformers(self): # validate estimators for t in transformers: - if t in ('drop', 'passthrough'): + if t in ("drop", "passthrough"): continue - if (not (hasattr(t, "fit") or hasattr(t, "fit_transform")) or not - hasattr(t, "transform")): - raise TypeError("All estimators should implement fit and " - "transform, or can be 'drop' or 'passthrough' " - "specifiers. '%s' (type %s) doesn't." % - (t, type(t))) + if not (hasattr(t, "fit") or hasattr(t, "fit_transform")) or not hasattr( + t, "transform" + ): + # Used to validate the transformers in the `transformers` list + raise TypeError( + "All estimators should implement fit and " + "transform, or can be 'drop' or 'passthrough' " + "specifiers. '%s' (type %s) doesn't." % (t, type(t)) + ) def _validate_column_callables(self, X): """ Converts callable column specifications. + + This stores a dictionary of the form `{step_name: column_indices}` and + calls the `columns` on `X` if `columns` is a callable for a given + transformer. + + The results are then stored in `self._transformer_to_input_indices`. """ - columns = [] - for _, _, column in self.transformers: - if callable(column): - column = column(X) - columns.append(column) - self._columns = columns + all_columns = [] + transformer_to_input_indices = {} + for name, _, columns in self.transformers: + if callable(columns): + columns = columns(X) + all_columns.append(columns) + transformer_to_input_indices[name] = _get_column_indices(X, columns) + + self._columns = all_columns + self._transformer_to_input_indices = transformer_to_input_indices def _validate_remainder(self, X): """ Validates ``remainder`` and defines ``_remainder`` targeting the remaining columns. """ - is_transformer = ((hasattr(self.remainder, "fit") - or hasattr(self.remainder, "fit_transform")) - and hasattr(self.remainder, "transform")) - if (self.remainder not in ('drop', 'passthrough') - and not is_transformer): - raise ValueError( - "The remainder keyword needs to be one of 'drop', " - "'passthrough', or estimator. '%s' was passed instead" % - self.remainder) + cols = set(chain(*self._transformer_to_input_indices.values())) + remaining = sorted(set(range(self.n_features_in_)) - cols) + self._transformer_to_input_indices["remainder"] = remaining + remainder_cols = self._get_remainder_cols(remaining) + self._remainder = ("remainder", self.remainder, remainder_cols) - self._n_features = X.shape[1] - cols = [] - for columns in self._columns: - cols.extend(_get_column_indices(X, columns)) - - remaining_idx = sorted(set(range(self._n_features)) - set(cols)) - self._remainder = ('remainder', self.remainder, remaining_idx or None) + def _get_remainder_cols_dtype(self): + try: + all_dtypes = {_determine_key_type(c) for (*_, c) in self.transformers} + if len(all_dtypes) == 1: + return next(iter(all_dtypes)) + except ValueError: + # _determine_key_type raises a ValueError if some transformer + # columns are Callables + return "int" + return "int" + + def _get_remainder_cols(self, indices): + dtype = self._get_remainder_cols_dtype() + if dtype == "str": + return list(self.feature_names_in_[indices]) + if dtype == "bool": + return [i in indices for i in range(self.n_features_in_)] + return indices @property def named_transformers_(self): @@ -334,57 +580,158 @@ def named_transformers_(self): Read-only attribute to access any transformer by given name. Keys are transformer names and values are the fitted transformer objects. - """ # Use Bunch object to improve autocomplete - return Bunch(**{name: trans for name, trans, _ - in self.transformers_}) + return Bunch(**{name: trans for name, trans, _ in self.transformers_}) + + def _get_feature_name_out_for_transformer(self, name, trans, feature_names_in): + """Gets feature names of transformer. + + Used in conjunction with self._iter(fitted=True) in get_feature_names_out. + """ + column_indices = self._transformer_to_input_indices[name] + names = feature_names_in[column_indices] + # An actual transformer + if not hasattr(trans, "get_feature_names_out"): + raise AttributeError( + f"Transformer {name} (type {type(trans).__name__}) does " + "not provide get_feature_names_out." + ) + return trans.get_feature_names_out(names) - def get_feature_names(self): - """Get feature names from all transformers. + def get_feature_names_out(self, input_features=None): + """Get output feature names for transformation. + + Parameters + ---------- + input_features : array-like of str or None, default=None + Input features. + + - If `input_features` is `None`, then `feature_names_in_` is + used as feature names in. If `feature_names_in_` is not defined, + then the following input feature names are generated: + `["x0", "x1", ..., "x(n_features_in_ - 1)"]`. + - If `input_features` is an array-like, then `input_features` must + match `feature_names_in_` if `feature_names_in_` is defined. Returns ------- - feature_names : list of strings - Names of the features produced by transform. + feature_names_out : ndarray of str objects + Transformed feature names. """ check_is_fitted(self) - feature_names = [] - for name, trans, column, _ in self._iter(fitted=True): - if trans == 'drop' or _is_empty_column_selection(column): - continue - if trans == 'passthrough': - if self._feature_names_in is not None: - if ((not isinstance(column, slice)) - and all(isinstance(col, str) for col in column)): - feature_names.extend(column) - else: - feature_names.extend(self._feature_names_in[column]) - else: - indices = np.arange(self._n_features) - feature_names.extend(['x%d' % i for i in indices[column]]) + input_features = _check_feature_names_in(self, input_features) + + # List of tuples (name, feature_names_out) + transformer_with_feature_names_out = [] + for name, trans, *_ in self._iter( + fitted=True, + column_as_labels=False, + skip_empty_columns=True, + skip_drop=True, + ): + feature_names_out = self._get_feature_name_out_for_transformer( + name, trans, input_features + ) + if feature_names_out is None: continue - if not hasattr(trans, 'get_feature_names'): - raise AttributeError("Transformer %s (type %s) does not " - "provide get_feature_names." - % (str(name), type(trans).__name__)) - feature_names.extend([name + "__" + f for f in - trans.get_feature_names()]) - return feature_names + transformer_with_feature_names_out.append((name, feature_names_out)) + + if not transformer_with_feature_names_out: + # No feature names + return np.array([], dtype=object) + + return self._add_prefix_for_feature_names_out( + transformer_with_feature_names_out + ) + + def _add_prefix_for_feature_names_out(self, transformer_with_feature_names_out): + """Add prefix for feature names out that includes the transformer names. + + Parameters + ---------- + transformer_with_feature_names_out : list of tuples of (str, array-like of str) + The tuple consistent of the transformer's name and its feature names out. + + Returns + ------- + feature_names_out : ndarray of shape (n_features,), dtype=str + Transformed feature names. + """ + feature_names_out_callable = None + if callable(self.verbose_feature_names_out): + feature_names_out_callable = self.verbose_feature_names_out + elif isinstance(self.verbose_feature_names_out, str): + feature_names_out_callable = partial( + _feature_names_out_with_str_format, + str_format=self.verbose_feature_names_out, + ) + elif self.verbose_feature_names_out is True: + feature_names_out_callable = partial( + _feature_names_out_with_str_format, + str_format="{transformer_name}__{feature_name}", + ) + + if feature_names_out_callable is not None: + # Prefix the feature names out with the transformers name + names = list( + chain.from_iterable( + (feature_names_out_callable(name, i) for i in feature_names_out) + for name, feature_names_out in transformer_with_feature_names_out + ) + ) + return np.asarray(names, dtype=object) + + # verbose_feature_names_out is False + # Check that names are all unique without a prefix + feature_names_count = Counter( + chain.from_iterable(s for _, s in transformer_with_feature_names_out) + ) + top_6_overlap = [ + name for name, count in feature_names_count.most_common(6) if count > 1 + ] + top_6_overlap.sort() + if top_6_overlap: + if len(top_6_overlap) == 6: + # There are more than 5 overlapping names, we only show the 5 + # of the feature names + names_repr = str(top_6_overlap[:5])[:-1] + ", ...]" + else: + names_repr = str(top_6_overlap) + raise ValueError( + f"Output feature names: {names_repr} are not unique. Please set " + "verbose_feature_names_out=True to add prefixes to feature names" + ) + + return np.concatenate( + [name for _, name in transformer_with_feature_names_out], + ) def _update_fitted_transformers(self, transformers): + """Set self.transformers_ from given transformers. + + Parameters + ---------- + transformers : list of estimators + The fitted estimators as the output of + `self._call_func_on_transformers(func=_fit_transform_one, ...)`. + That function doesn't include 'drop' or transformers for which no + column is selected. 'drop' is kept as is, and for the no-column + transformers the unfitted transformer is put in + `self.transformers_`. + """ # transformers are fitted; excludes 'drop' cases fitted_transformers = iter(transformers) transformers_ = [] - for name, old, column, _ in self._iter(): - if old == 'drop': - trans = 'drop' - elif old == 'passthrough': - # FunctionTransformer is present in list of transformers, - # so get next transformer, but save original string - next(fitted_transformers) - trans = 'passthrough' + for name, old, column, _ in self._iter( + fitted=False, + column_as_labels=False, + skip_drop=False, + skip_empty_columns=False, + ): + if old == "drop": + trans = "drop" elif _is_empty_column_selection(column): trans = old else: @@ -400,47 +747,162 @@ def _validate_output(self, result): Ensure that the output of each transformer is 2D. Otherwise hstack can raise an error or produce incorrect results. """ - names = [name for name, _, _, _ in self._iter(fitted=True, - replace_strings=True)] + names = [ + name + for name, _, _, _ in self._iter( + fitted=True, + column_as_labels=False, + skip_drop=True, + skip_empty_columns=True, + ) + ] for Xs, name in zip(result, names): - if not getattr(Xs, 'ndim', 0) == 2: + if not getattr(Xs, "ndim", 0) == 2 and not hasattr(Xs, "__dataframe__"): raise ValueError( - "The output of the '{0}' transformer should be 2D (scipy " - "matrix, array, or pandas DataFrame).".format(name)) + "The output of the '{0}' transformer should be 2D (numpy array, " + "scipy sparse array, dataframe).".format(name) + ) + if _get_output_config("transform", self)["dense"] == "pandas": + return + try: + import pandas as pd + except ImportError: + return + for Xs, name in zip(result, names): + if not _is_pandas_df(Xs): + continue + for col_name, dtype in Xs.dtypes.to_dict().items(): + if getattr(dtype, "na_value", None) is not pd.NA: + continue + if pd.NA not in Xs[col_name].values: + continue + class_name = self.__class__.__name__ + raise ValueError( + f"The output of the '{name}' transformer for column" + f" '{col_name}' has dtype {dtype} and uses pandas.NA to" + " represent null values. Storing this output in a numpy array" + " can cause errors in downstream scikit-learn estimators, and" + " inefficiencies. To avoid this problem you can (i)" + " store the output in a pandas DataFrame by using" + f" {class_name}.set_output(transform='pandas') or (ii) modify" + f" the input data or the '{name}' transformer to avoid the" + " presence of pandas.NA (for example by using" + " pandas.DataFrame.astype)." + ) + + def _record_output_indices(self, Xs): + """ + Record which transformer produced which column. + """ + idx = 0 + self.output_indices_ = {} + + for transformer_idx, (name, _, _, _) in enumerate( + self._iter( + fitted=True, + column_as_labels=False, + skip_drop=True, + skip_empty_columns=True, + ) + ): + n_columns = Xs[transformer_idx].shape[1] + self.output_indices_[name] = slice(idx, idx + n_columns) + idx += n_columns + + # `_iter` only generates transformers that have a non empty + # selection. Here we set empty slices for transformers that + # generate no output, which are safe for indexing + all_names = [t[0] for t in self.transformers] + ["remainder"] + for name in all_names: + if name not in self.output_indices_: + self.output_indices_[name] = slice(0, 0) def _log_message(self, name, idx, total): if not self.verbose: return None - return '(%d of %d) Processing %s' % (idx, total, name) + return "(%d of %d) Processing %s" % (idx, total, name) - def _fit_transform(self, X, y, func, fitted=False): + def _call_func_on_transformers(self, X, y, func, column_as_labels, routed_params): """ Private function to fit and/or transform on demand. + Parameters + ---------- + X : {array-like, dataframe} of shape (n_samples, n_features) + The data to be used in fit and/or transform. + + y : array-like of shape (n_samples,) + Targets. + + func : callable + Function to call, which can be _fit_transform_one or + _transform_one. + + column_as_labels : bool + Used to iterate through transformers. If True, columns are returned + as strings. If False, columns are returned as they were given by + the user. Can be True only if the ``ColumnTransformer`` is already + fitted. + + routed_params : dict + The routed parameters as the output from ``process_routing``. + + Returns + ------- Return value (transformers and/or transformed X data) depends on the passed function. - ``fitted=True`` ensures the fitted transformers are used. """ + if func is _fit_transform_one: + fitted = False + else: # func is _transform_one + fitted = True + transformers = list( - self._iter(fitted=fitted, replace_strings=True)) + self._iter( + fitted=fitted, + column_as_labels=column_as_labels, + skip_drop=True, + skip_empty_columns=True, + ) + ) try: - return Parallel(n_jobs=self.n_jobs)( - delayed(func)( - transformer=clone(trans) if not fitted else trans, - X=_safe_indexing(X, column, axis=1), - y=y, - weight=weight, - message_clsname='ColumnTransformer', - message=self._log_message(name, idx, len(transformers))) - for idx, (name, trans, column, weight) in enumerate( - transformers, 1)) + jobs = [] + for idx, (name, trans, columns, weight) in enumerate(transformers, start=1): + if func is _fit_transform_one: + if trans == "passthrough": + output_config = _get_output_config("transform", self) + trans = FunctionTransformer( + accept_sparse=True, + check_inverse=False, + feature_names_out="one-to-one", + ).set_output(transform=output_config["dense"]) + + extra_args = dict( + message_clsname="ColumnTransformer", + message=self._log_message(name, idx, len(transformers)), + ) + else: # func is _transform_one + extra_args = {} + jobs.append( + delayed(func)( + transformer=clone(trans) if not fitted else trans, + X=_safe_indexing(X, columns, axis=1), + y=y, + weight=weight, + **extra_args, + params=routed_params[name], + ) + ) + + return Parallel(n_jobs=self.n_jobs)(jobs) + except ValueError as e: if "Expected 2D array, got 1D array instead" in str(e): raise ValueError(_ERR_MSG_1DCOLUMN) from e else: raise - def fit(self, X, y=None): + def fit(self, X, y=None, **params): """Fit all transformers using X. Parameters @@ -452,18 +914,31 @@ def fit(self, X, y=None): y : array-like of shape (n_samples,...), default=None Targets for supervised learning. + **params : dict, default=None + Parameters to be passed to the underlying transformers' ``fit`` and + ``transform`` methods. + + You can only pass this if metadata routing is enabled, which you + can enable using ``sklearn.set_config(enable_metadata_routing=True)``. + + .. versionadded:: 1.4 + Returns ------- self : ColumnTransformer - This estimator - + This estimator. """ + _raise_for_params(params, self, "fit") # we use fit_transform to make sure to set sparse_output_ (for which we # need the transformed data) to have consistent output type in predict - self.fit_transform(X, y=y) + self.fit_transform(X, y=y, **params) return self - def fit_transform(self, X, y=None): + @_fit_context( + # estimators in ColumnTransformer.transformers are not validated yet + prefer_skip_nested_validation=False + ) + def fit_transform(self, X, y=None, **params): """Fit all transformers, transform the data and concatenate results. Parameters @@ -475,42 +950,70 @@ def fit_transform(self, X, y=None): y : array-like of shape (n_samples,), default=None Targets for supervised learning. + **params : dict, default=None + Parameters to be passed to the underlying transformers' ``fit`` and + ``transform`` methods. + + You can only pass this if metadata routing is enabled, which you + can enable using ``sklearn.set_config(enable_metadata_routing=True)``. + + .. versionadded:: 1.4 + Returns ------- X_t : {array-like, sparse matrix} of \ shape (n_samples, sum_n_components) - hstack of results of transformers. sum_n_components is the + Horizontally stacked results of transformers. sum_n_components is the sum of n_components (output dimension) over transformers. If any result is a sparse matrix, everything will be converted to sparse matrices. - """ - # TODO: this should be `feature_names_in_` when we start having it - if hasattr(X, "columns"): - self._feature_names_in = np.asarray(X.columns) - else: - self._feature_names_in = None + _raise_for_params(params, self, "fit_transform") + _check_feature_names(self, X, reset=True) + + if self.force_int_remainder_cols != "deprecated": + warnings.warn( + "The parameter `force_int_remainder_cols` is deprecated and will be " + "removed in 1.9. It has no effect. Leave it to its default value to " + "avoid this warning.", + FutureWarning, + ) + X = _check_X(X) # set n_features_in_ attribute - self._check_n_features(X, reset=True) + _check_n_features(self, X, reset=True) self._validate_transformers() + n_samples = _num_samples(X) + self._validate_column_callables(X) self._validate_remainder(X) - result = self._fit_transform(X, y, _fit_transform_one) + if _routing_enabled(): + routed_params = process_routing(self, "fit_transform", **params) + else: + routed_params = self._get_empty_routing() + + result = self._call_func_on_transformers( + X, + y, + _fit_transform_one, + column_as_labels=False, + routed_params=routed_params, + ) if not result: self._update_fitted_transformers([]) # All transformers are None - return np.zeros((X.shape[0], 0)) + return np.zeros((n_samples, 0)) Xs, transformers = zip(*result) # determine if concatenated output will be sparse or not if any(sparse.issparse(X) for X in Xs): nnz = sum(X.nnz if sparse.issparse(X) else X.size for X in Xs) - total = sum(X.shape[0] * X.shape[1] if sparse.issparse(X) - else X.size for X in Xs) + total = sum( + X.shape[0] * X.shape[1] if sparse.issparse(X) else X.size for X in Xs + ) density = nnz / total self.sparse_output_ = density < self.sparse_threshold else: @@ -518,10 +1021,11 @@ def fit_transform(self, X, y=None): self._update_fitted_transformers(transformers) self._validate_output(Xs) + self._record_output_indices(Xs) - return self._hstack(list(Xs)) + return self._hstack(list(Xs), n_samples=n_samples) - def transform(self, X): + def transform(self, X, **params): """Transform X separately by each transformer, concatenate results. Parameters @@ -529,41 +1033,82 @@ def transform(self, X): X : {array-like, dataframe} of shape (n_samples, n_features) The data to be transformed by subset. + **params : dict, default=None + Parameters to be passed to the underlying transformers' ``transform`` + method. + + You can only pass this if metadata routing is enabled, which you + can enable using ``sklearn.set_config(enable_metadata_routing=True)``. + + .. versionadded:: 1.4 + Returns ------- X_t : {array-like, sparse matrix} of \ shape (n_samples, sum_n_components) - hstack of results of transformers. sum_n_components is the + Horizontally stacked results of transformers. sum_n_components is the sum of n_components (output dimension) over transformers. If any result is a sparse matrix, everything will be converted to sparse matrices. - """ + _raise_for_params(params, self, "transform") check_is_fitted(self) X = _check_X(X) - if hasattr(X, "columns"): - X_feature_names = np.asarray(X.columns) + + # If ColumnTransformer is fit using a dataframe, and now a dataframe is + # passed to be transformed, we select columns by name instead. This + # enables the user to pass X at transform time with extra columns which + # were not present in fit time, and the order of the columns doesn't + # matter. + fit_dataframe_and_transform_dataframe = hasattr(self, "feature_names_in_") and ( + _is_pandas_df(X) or hasattr(X, "__dataframe__") + ) + + n_samples = _num_samples(X) + column_names = _get_feature_names(X) + + if fit_dataframe_and_transform_dataframe: + named_transformers = self.named_transformers_ + # check that all names seen in fit are in transform, unless + # they were dropped + non_dropped_indices = [ + ind + for name, ind in self._transformer_to_input_indices.items() + if name in named_transformers and named_transformers[name] != "drop" + ] + + all_indices = set(chain(*non_dropped_indices)) + all_names = set(self.feature_names_in_[ind] for ind in all_indices) + + diff = all_names - set(column_names) + if diff: + raise ValueError(f"columns are missing: {diff}") else: - X_feature_names = None - - self._check_n_features(X, reset=False) - if (self._feature_names_in is not None and - X_feature_names is not None and - np.any(self._feature_names_in != X_feature_names)): - raise RuntimeError( - "Given feature/column names do not match the ones for the " - "data given during fit." - ) - Xs = self._fit_transform(X, None, _transform_one, fitted=True) + # ndarray was used for fitting or transforming, thus we only + # check that n_features_in_ is consistent + _check_n_features(self, X, reset=False) + + if _routing_enabled(): + routed_params = process_routing(self, "transform", **params) + else: + routed_params = self._get_empty_routing() + + Xs = self._call_func_on_transformers( + X, + None, + _transform_one, + column_as_labels=fit_dataframe_and_transform_dataframe, + routed_params=routed_params, + ) self._validate_output(Xs) if not Xs: # All transformers are None - return np.zeros((X.shape[0], 0)) + return np.zeros((n_samples, 0)) - return self._hstack(list(Xs)) + return self._hstack(list(Xs), n_samples=n_samples) - def _hstack(self, Xs): + def _hstack(self, Xs, *, n_samples): """Stacks Xs horizontally. This allows subclasses to control the stacking behavior, while reusing @@ -572,16 +1117,20 @@ def _hstack(self, Xs): Parameters ---------- Xs : list of {array-like, sparse matrix, dataframe} + The container to concatenate. + n_samples : int + The number of samples in the input data to checking the transformation + consistency. """ if self.sparse_output_: try: # since all columns should be numeric before stacking them # in a sparse matrix, `check_array` is used for the # dtype conversion if necessary. - converted_Xs = [check_array(X, - accept_sparse=True, - force_all_finite=False) - for X in Xs] + converted_Xs = [ + check_array(X, accept_sparse=True, ensure_all_finite=False) + for X in Xs + ] except ValueError as e: raise ValueError( "For a sparse output, all columns should " @@ -591,34 +1140,202 @@ def _hstack(self, Xs): return sparse.hstack(converted_Xs).tocsr() else: Xs = [f.toarray() if sparse.issparse(f) else f for f in Xs] + adapter = _get_container_adapter("transform", self) + if adapter and all(adapter.is_supported_container(X) for X in Xs): + # rename before stacking as it avoids to error on temporary duplicated + # columns + transformer_names = [ + t[0] + for t in self._iter( + fitted=True, + column_as_labels=False, + skip_drop=True, + skip_empty_columns=True, + ) + ] + feature_names_outs = [X.columns for X in Xs if X.shape[1] != 0] + if self.verbose_feature_names_out: + # `_add_prefix_for_feature_names_out` takes care about raising + # an error if there are duplicated columns. + feature_names_outs = self._add_prefix_for_feature_names_out( + list(zip(transformer_names, feature_names_outs)) + ) + else: + # check for duplicated columns and raise if any + feature_names_outs = list(chain.from_iterable(feature_names_outs)) + feature_names_count = Counter(feature_names_outs) + if any(count > 1 for count in feature_names_count.values()): + duplicated_feature_names = sorted( + name + for name, count in feature_names_count.items() + if count > 1 + ) + err_msg = ( + "Duplicated feature names found before concatenating the" + " outputs of the transformers:" + f" {duplicated_feature_names}.\n" + ) + for transformer_name, X in zip(transformer_names, Xs): + if X.shape[1] == 0: + continue + dup_cols_in_transformer = sorted( + set(X.columns).intersection(duplicated_feature_names) + ) + if len(dup_cols_in_transformer): + err_msg += ( + f"Transformer {transformer_name} has conflicting " + f"columns names: {dup_cols_in_transformer}.\n" + ) + raise ValueError( + err_msg + + "Either make sure that the transformers named above " + "do not generate columns with conflicting names or set " + "verbose_feature_names_out=True to automatically " + "prefix to the output feature names with the name " + "of the transformer to prevent any conflicting " + "names." + ) + + names_idx = 0 + for X in Xs: + if X.shape[1] == 0: + continue + names_out = feature_names_outs[names_idx : names_idx + X.shape[1]] + adapter.rename_columns(X, names_out) + names_idx += X.shape[1] + + output = adapter.hstack(Xs) + output_samples = output.shape[0] + if output_samples != n_samples: + raise ValueError( + "Concatenating DataFrames from the transformer's output lead to" + " an inconsistent number of samples. The output may have Pandas" + " Indexes that do not match, or that transformers are returning" + " number of samples which are not the same as the number input" + " samples." + ) + + return output + return np.hstack(Xs) def _sk_visual_block_(self): - if isinstance(self.remainder, str) and self.remainder == 'drop': + if isinstance(self.remainder, str) and self.remainder == "drop": transformers = self.transformers elif hasattr(self, "_remainder"): remainder_columns = self._remainder[2] - if self._feature_names_in is not None: - remainder_columns = ( - self._feature_names_in[remainder_columns].tolist() - ) - transformers = chain(self.transformers, - [('remainder', self.remainder, - remainder_columns)]) + if ( + hasattr(self, "feature_names_in_") + and remainder_columns + and not all(isinstance(col, str) for col in remainder_columns) + ): + remainder_columns = self.feature_names_in_[remainder_columns].tolist() + transformers = chain( + self.transformers, [("remainder", self.remainder, remainder_columns)] + ) else: - transformers = chain(self.transformers, - [('remainder', self.remainder, '')]) + transformers = chain(self.transformers, [("remainder", self.remainder, "")]) names, transformers, name_details = zip(*transformers) - return _VisualBlock('parallel', transformers, - names=names, name_details=name_details) + return _VisualBlock( + "parallel", transformers, names=names, name_details=name_details + ) + + def __getitem__(self, key): + try: + return self.named_transformers_[key] + except AttributeError as e: + raise TypeError( + "ColumnTransformer is subscriptable after it is fitted" + ) from e + except KeyError as e: + raise KeyError(f"'{key}' is not a valid transformer name") from e + + def _get_empty_routing(self): + """Return empty routing. + + Used while routing can be disabled. + + TODO: Remove when ``set_config(enable_metadata_routing=False)`` is no + more an option. + """ + return Bunch( + **{ + name: Bunch(**{method: {} for method in METHODS}) + for name, step, _, _ in self._iter( + fitted=False, + column_as_labels=False, + skip_drop=True, + skip_empty_columns=True, + ) + } + ) + + def get_metadata_routing(self): + """Get metadata routing of this object. + + Please check :ref:`User Guide ` on how the routing + mechanism works. + + .. versionadded:: 1.4 + + Returns + ------- + routing : MetadataRouter + A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating + routing information. + """ + router = MetadataRouter(owner=self.__class__.__name__) + # Here we don't care about which columns are used for which + # transformers, and whether or not a transformer is used at all, which + # might happen if no columns are selected for that transformer. We + # request all metadata requested by all transformers. + transformers = chain(self.transformers, [("remainder", self.remainder, None)]) + for name, step, _ in transformers: + method_mapping = MethodMapping() + if hasattr(step, "fit_transform"): + ( + method_mapping.add(caller="fit", callee="fit_transform").add( + caller="fit_transform", callee="fit_transform" + ) + ) + else: + ( + method_mapping.add(caller="fit", callee="fit") + .add(caller="fit", callee="transform") + .add(caller="fit_transform", callee="fit") + .add(caller="fit_transform", callee="transform") + ) + method_mapping.add(caller="transform", callee="transform") + router.add(method_mapping=method_mapping, **{name: step}) + + return router + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + try: + tags.input_tags.sparse = all( + get_tags(trans).input_tags.sparse + for name, trans, _ in self.transformers + if trans not in {"passthrough", "drop"} + ) + except Exception: + # If `transformers` does not comply with our API (list of tuples) + # then it will fail. In this case, we assume that `sparse` is False + # but the parameter validation will raise an error during `fit`. + pass # pragma: no cover + return tags def _check_X(X): - """Use check_array only on lists and other non-array-likes / sparse""" - if hasattr(X, '__array__') or sparse.issparse(X): + """Use check_array only when necessary, e.g. on lists and other non-array-likes.""" + if ( + (hasattr(X, "__array__") and hasattr(X, "shape")) + or hasattr(X, "__dataframe__") + or sparse.issparse(X) + ): return X - return check_array(X, force_all_finite='allow-nan', dtype=object) + return check_array(X, ensure_all_finite="allow-nan", dtype=object) def _is_empty_column_selection(column): @@ -627,12 +1344,12 @@ def _is_empty_column_selection(column): boolean array). """ - if hasattr(column, 'dtype') and np.issubdtype(column.dtype, np.bool_): + if hasattr(column, "dtype") and np.issubdtype(column.dtype, np.bool_): return not column.any() - elif hasattr(column, '__len__'): - return (len(column) == 0 or - all(isinstance(col, bool) for col in column) - and not any(column)) + elif hasattr(column, "__len__"): + return len(column) == 0 or ( + all(isinstance(col, bool) for col in column) and not any(column) + ) else: return False @@ -649,11 +1366,17 @@ def _get_transformer_list(estimators): return transformer_list -def make_column_transformer(*transformers, - remainder='drop', - sparse_threshold=0.3, - n_jobs=None, - verbose=False): +# This function is not validated using validate_params because +# it's just a factory for ColumnTransformer. +def make_column_transformer( + *transformers, + remainder="drop", + sparse_threshold=0.3, + n_jobs=None, + verbose=False, + verbose_feature_names_out=True, + force_int_remainder_cols="deprecated", +): """Construct a ColumnTransformer from the given transformers. This is a shorthand for the ColumnTransformer constructor; it does not @@ -715,9 +1438,37 @@ def make_column_transformer(*transformers, If True, the time elapsed while fitting each transformer will be printed as it is completed. + verbose_feature_names_out : bool, default=True + If True, :meth:`ColumnTransformer.get_feature_names_out` will prefix + all feature names with the name of the transformer that generated that + feature. + If False, :meth:`ColumnTransformer.get_feature_names_out` will not + prefix any feature names and will error if feature names are not + unique. + + .. versionadded:: 1.0 + + force_int_remainder_cols : bool, default=True + This parameter has no effect. + + .. note:: + If you do not access the list of columns for the remainder columns + in the :attr:`ColumnTransformer.transformers_` fitted attribute, + you do not need to set this parameter. + + .. versionadded:: 1.5 + + .. versionchanged:: 1.7 + The default value for `force_int_remainder_cols` will change from + `True` to `False` in version 1.7. + + .. deprecated:: 1.7 + `force_int_remainder_cols` is deprecated and will be removed in version 1.9. + Returns ------- ct : ColumnTransformer + Returns a :class:`ColumnTransformer` object. See Also -------- @@ -736,15 +1487,19 @@ def make_column_transformer(*transformers, ['numerical_column']), ('onehotencoder', OneHotEncoder(...), ['categorical_column'])]) - """ # transformer_weights keyword is not passed through because the user # would need to know the automatically generated names of the transformers transformer_list = _get_transformer_list(transformers) - return ColumnTransformer(transformer_list, n_jobs=n_jobs, - remainder=remainder, - sparse_threshold=sparse_threshold, - verbose=verbose) + return ColumnTransformer( + transformer_list, + n_jobs=n_jobs, + remainder=remainder, + sparse_threshold=sparse_threshold, + verbose=verbose, + verbose_feature_names_out=verbose_feature_names_out, + force_int_remainder_cols=force_int_remainder_cols, + ) class make_column_selector: @@ -755,6 +1510,11 @@ class make_column_selector: columns name with a regex. When using multiple selection criteria, **all** criteria must match for a column to be selected. + For an example of how to use :func:`make_column_selector` within a + :class:`ColumnTransformer` to select columns based on data type (i.e. + `dtype`), refer to + :ref:`sphx_glr_auto_examples_compose_plot_column_transformer_mixed_types.py`. + Parameters ---------- pattern : str, default=None @@ -786,6 +1546,7 @@ class make_column_selector: >>> from sklearn.preprocessing import StandardScaler, OneHotEncoder >>> from sklearn.compose import make_column_transformer >>> from sklearn.compose import make_column_selector + >>> import numpy as np >>> import pandas as pd # doctest: +SKIP >>> X = pd.DataFrame({'city': ['London', 'London', 'Paris', 'Sallisaw'], ... 'rating': [5, 3, 4, 5]}) # doctest: +SKIP @@ -800,9 +1561,8 @@ class make_column_selector: [-0.30151134, 0. , 1. , 0. ], [ 0.90453403, 0. , 0. , 1. ]]) """ - @_deprecate_positional_args - def __init__(self, pattern=None, *, dtype_include=None, - dtype_exclude=None): + + def __init__(self, pattern=None, *, dtype_include=None, dtype_exclude=None): self.pattern = pattern self.dtype_include = dtype_include self.dtype_exclude = dtype_exclude @@ -816,14 +1576,24 @@ def __call__(self, df): df : dataframe of shape (n_features, n_samples) DataFrame to select columns from. """ - if not hasattr(df, 'iloc'): - raise ValueError("make_column_selector can only be applied to " - "pandas dataframes") + if not hasattr(df, "iloc"): + raise ValueError( + "make_column_selector can only be applied to pandas dataframes" + ) df_row = df.iloc[:1] if self.dtype_include is not None or self.dtype_exclude is not None: - df_row = df_row.select_dtypes(include=self.dtype_include, - exclude=self.dtype_exclude) + df_row = df_row.select_dtypes( + include=self.dtype_include, exclude=self.dtype_exclude + ) cols = df_row.columns if self.pattern is not None: cols = cols[cols.str.contains(self.pattern, regex=True)] return cols.tolist() + + +def _feature_names_out_with_str_format( + transformer_name: str, feature_name: str, str_format: str +) -> str: + return str_format.format( + transformer_name=transformer_name, feature_name=feature_name + ) diff --git a/sklearn/compose/_target.py b/sklearn/compose/_target.py index 1d6695a808d81..86fc6294878b9 100644 --- a/sklearn/compose/_target.py +++ b/sklearn/compose/_target.py @@ -1,30 +1,37 @@ -# Authors: Andreas Mueller -# Guillaume Lemaitre -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import warnings import numpy as np -from ..base import BaseEstimator, RegressorMixin, clone -from ..utils.validation import check_is_fitted -from ..utils import check_array, _safe_indexing -from ..preprocessing import FunctionTransformer -from ..utils.validation import _deprecate_positional_args +from ..base import BaseEstimator, RegressorMixin, _fit_context, clone from ..exceptions import NotFittedError +from ..linear_model import LinearRegression +from ..preprocessing import FunctionTransformer +from ..utils import Bunch, _safe_indexing, check_array +from ..utils._metadata_requests import ( + MetadataRouter, + MethodMapping, + _routing_enabled, + process_routing, +) +from ..utils._param_validation import HasMethods +from ..utils._tags import get_tags +from ..utils.validation import check_is_fitted -__all__ = ['TransformedTargetRegressor'] +__all__ = ["TransformedTargetRegressor"] class TransformedTargetRegressor(RegressorMixin, BaseEstimator): """Meta-estimator to regress on a transformed target. - Useful for applying a non-linear transformation to the target ``y`` in + Useful for applying a non-linear transformation to the target `y` in regression problems. This transformation can be given as a Transformer - such as the QuantileTransformer or as a function and its inverse such as - ``log`` and ``exp``. + such as the :class:`~sklearn.preprocessing.QuantileTransformer` or as a + function and its inverse such as `np.log` and `np.exp`. - The computation during ``fit`` is:: + The computation during :meth:`fit` is:: regressor.fit(X, func(y)) @@ -32,7 +39,7 @@ class TransformedTargetRegressor(RegressorMixin, BaseEstimator): regressor.fit(X, transformer.transform(y)) - The computation during ``predict`` is:: + The computation during :meth:`predict` is:: inverse_func(regressor.predict(X)) @@ -47,33 +54,35 @@ class TransformedTargetRegressor(RegressorMixin, BaseEstimator): Parameters ---------- regressor : object, default=None - Regressor object such as derived from ``RegressorMixin``. This - regressor will automatically be cloned each time prior to fitting. - If regressor is ``None``, ``LinearRegression()`` is created and used. + Regressor object such as derived from + :class:`~sklearn.base.RegressorMixin`. This regressor will + automatically be cloned each time prior to fitting. If `regressor is + None`, :class:`~sklearn.linear_model.LinearRegression` is created and used. transformer : object, default=None - Estimator object such as derived from ``TransformerMixin``. Cannot be - set at the same time as ``func`` and ``inverse_func``. If - ``transformer`` is ``None`` as well as ``func`` and ``inverse_func``, - the transformer will be an identity transformer. Note that the - transformer will be cloned during fitting. Also, the transformer is - restricting ``y`` to be a numpy array. + Estimator object such as derived from + :class:`~sklearn.base.TransformerMixin`. Cannot be set at the same time + as `func` and `inverse_func`. If `transformer is None` as well as + `func` and `inverse_func`, the transformer will be an identity + transformer. Note that the transformer will be cloned during fitting. + Also, the transformer is restricting `y` to be a numpy array. func : function, default=None - Function to apply to ``y`` before passing to ``fit``. Cannot be set at - the same time as ``transformer``. The function needs to return a - 2-dimensional array. If ``func`` is ``None``, the function used will be - the identity function. + Function to apply to `y` before passing to :meth:`fit`. Cannot be set + at the same time as `transformer`. If `func is None`, the function used will be + the identity function. If `func` is set, `inverse_func` also needs to be + provided. The function needs to return a 2-dimensional array. inverse_func : function, default=None Function to apply to the prediction of the regressor. Cannot be set at - the same time as ``transformer`` as well. The function needs to return - a 2-dimensional array. The inverse function is used to return - predictions to the same space of the original training labels. + the same time as `transformer`. The inverse function is used to return + predictions to the same space of the original training labels. If + `inverse_func` is set, `func` also needs to be provided. The inverse + function needs to return a 2-dimensional array. check_inverse : bool, default=True - Whether to check that ``transform`` followed by ``inverse_transform`` - or ``func`` followed by ``inverse_func`` leads to the original targets. + Whether to check that `transform` followed by `inverse_transform` + or `func` followed by `inverse_func` leads to the original targets. Attributes ---------- @@ -81,7 +90,30 @@ class TransformedTargetRegressor(RegressorMixin, BaseEstimator): Fitted regressor. transformer_ : object - Transformer used in ``fit`` and ``predict``. + Transformer used in :meth:`fit` and :meth:`predict`. + + n_features_in_ : int + Number of features seen during :term:`fit`. Only defined if the + underlying regressor exposes such an attribute when fit. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + sklearn.preprocessing.FunctionTransformer : Construct a transformer from an + arbitrary callable. + + Notes + ----- + Internally, the target `y` is always converted into a 2-dimensional array + to be used by scikit-learn transformers. At the time of prediction, the + output will be reshaped to a have the same number of dimensions as `y`. Examples -------- @@ -99,19 +131,27 @@ class TransformedTargetRegressor(RegressorMixin, BaseEstimator): >>> tt.regressor_.coef_ array([2.]) - Notes - ----- - Internally, the target ``y`` is always converted into a 2-dimensional array - to be used by scikit-learn transformers. At the time of prediction, the - output will be reshaped to a have the same number of dimensions as ``y``. - - See :ref:`examples/compose/plot_transformed_target.py - `. - + For a more detailed example use case refer to + :ref:`sphx_glr_auto_examples_compose_plot_transformed_target.py`. """ - @_deprecate_positional_args - def __init__(self, regressor=None, *, transformer=None, - func=None, inverse_func=None, check_inverse=True): + + _parameter_constraints: dict = { + "regressor": [HasMethods(["fit", "predict"]), None], + "transformer": [HasMethods("transform"), None], + "func": [callable, None], + "inverse_func": [callable, None], + "check_inverse": ["boolean"], + } + + def __init__( + self, + regressor=None, + *, + transformer=None, + func=None, + inverse_func=None, + check_inverse=True, + ): self.regressor = regressor self.transformer = transformer self.func = func @@ -125,19 +165,38 @@ def _fit_transformer(self, y): check on a subset (optional). """ - if (self.transformer is not None and - (self.func is not None or self.inverse_func is not None)): - raise ValueError("'transformer' and functions 'func'/" - "'inverse_func' cannot both be set.") + if self.transformer is not None and ( + self.func is not None or self.inverse_func is not None + ): + raise ValueError( + "'transformer' and functions 'func'/'inverse_func' cannot both be set." + ) elif self.transformer is not None: self.transformer_ = clone(self.transformer) else: - if self.func is not None and self.inverse_func is None: - raise ValueError("When 'func' is provided, 'inverse_func' must" - " also be provided") + if (self.func is not None and self.inverse_func is None) or ( + self.func is None and self.inverse_func is not None + ): + lacking_param, existing_param = ( + ("func", "inverse_func") + if self.func is None + else ("inverse_func", "func") + ) + raise ValueError( + f"When '{existing_param}' is provided, '{lacking_param}' must also" + f" be provided. If {lacking_param} is supposed to be the default," + " you need to explicitly pass it the identity function." + ) self.transformer_ = FunctionTransformer( - func=self.func, inverse_func=self.inverse_func, validate=True, - check_inverse=self.check_inverse) + func=self.func, + inverse_func=self.inverse_func, + validate=True, + check_inverse=self.check_inverse, + ) + # We are transforming the target here and not the features, so we set the + # output of FunctionTransformer() to be a numpy array (default) and to not + # depend on the global configuration: + self.transformer_.set_output(transform="default") # XXX: sample_weight is not currently passed to the # transformer. However, if transformer starts using sample_weight, the # code should be modified accordingly. At the time to consider the @@ -147,36 +206,63 @@ def _fit_transformer(self, y): idx_selected = slice(None, None, max(1, y.shape[0] // 10)) y_sel = _safe_indexing(y, idx_selected) y_sel_t = self.transformer_.transform(y_sel) - if not np.allclose(y_sel, - self.transformer_.inverse_transform(y_sel_t)): - warnings.warn("The provided functions or transformer are" - " not strictly inverse of each other. If" - " you are sure you want to proceed regardless" - ", set 'check_inverse=False'", UserWarning) - + if not np.allclose(y_sel, self.transformer_.inverse_transform(y_sel_t)): + warnings.warn( + ( + "The provided functions or transformer are" + " not strictly inverse of each other. If" + " you are sure you want to proceed regardless" + ", set 'check_inverse=False'" + ), + UserWarning, + ) + + @_fit_context( + # TransformedTargetRegressor.regressor/transformer are not validated yet. + prefer_skip_nested_validation=False + ) def fit(self, X, y, **fit_params): """Fit the model according to the given training data. Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) - Training vector, where n_samples is the number of samples and - n_features is the number of features. + Training vector, where `n_samples` is the number of samples and + `n_features` is the number of features. y : array-like of shape (n_samples,) Target values. **fit_params : dict - Parameters passed to the ``fit`` method of the underlying - regressor. + - If `enable_metadata_routing=False` (default): Parameters directly passed + to the `fit` method of the underlying regressor. + + - If `enable_metadata_routing=True`: Parameters safely routed to the `fit` + method of the underlying regressor. + .. versionchanged:: 1.6 + See :ref:`Metadata Routing User Guide ` for + more details. Returns ------- self : object + Fitted estimator. """ - y = check_array(y, accept_sparse=False, force_all_finite=True, - ensure_2d=False, dtype='numeric') + if y is None: + raise ValueError( + f"This {self.__class__.__name__} estimator " + "requires y to be passed, but the target y is None." + ) + y = check_array( + y, + input_name="y", + accept_sparse=False, + ensure_all_finite=True, + ensure_2d=False, + dtype="numeric", + allow_nd=True, + ) # store the number of dimension of the target to predict an array of # similar shape at predict @@ -198,59 +284,114 @@ def fit(self, X, y, **fit_params): if y_trans.ndim == 2 and y_trans.shape[1] == 1: y_trans = y_trans.squeeze(axis=1) - if self.regressor is None: - from ..linear_model import LinearRegression - self.regressor_ = LinearRegression() + self.regressor_ = self._get_regressor(get_clone=True) + if _routing_enabled(): + routed_params = process_routing(self, "fit", **fit_params) else: - self.regressor_ = clone(self.regressor) + routed_params = Bunch(regressor=Bunch(fit=fit_params)) - self.regressor_.fit(X, y_trans, **fit_params) + self.regressor_.fit(X, y_trans, **routed_params.regressor.fit) + + if hasattr(self.regressor_, "feature_names_in_"): + self.feature_names_in_ = self.regressor_.feature_names_in_ return self - def predict(self, X): + def predict(self, X, **predict_params): """Predict using the base regressor, applying inverse. - The regressor is used to predict and the ``inverse_func`` or - ``inverse_transform`` is applied before returning the prediction. + The regressor is used to predict and the `inverse_func` or + `inverse_transform` is applied before returning the prediction. Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) Samples. + **predict_params : dict of str -> object + - If `enable_metadata_routing=False` (default): Parameters directly passed + to the `predict` method of the underlying regressor. + + - If `enable_metadata_routing=True`: Parameters safely routed to the + `predict` method of the underlying regressor. + + .. versionchanged:: 1.6 + See :ref:`Metadata Routing User Guide ` + for more details. + Returns ------- y_hat : ndarray of shape (n_samples,) Predicted values. - """ check_is_fitted(self) - pred = self.regressor_.predict(X) + if _routing_enabled(): + routed_params = process_routing(self, "predict", **predict_params) + else: + routed_params = Bunch(regressor=Bunch(predict=predict_params)) + + pred = self.regressor_.predict(X, **routed_params.regressor.predict) if pred.ndim == 1: - pred_trans = self.transformer_.inverse_transform( - pred.reshape(-1, 1)) + pred_trans = self.transformer_.inverse_transform(pred.reshape(-1, 1)) else: pred_trans = self.transformer_.inverse_transform(pred) - if (self._training_dim == 1 and - pred_trans.ndim == 2 and pred_trans.shape[1] == 1): + if ( + self._training_dim == 1 + and pred_trans.ndim == 2 + and pred_trans.shape[1] == 1 + ): pred_trans = pred_trans.squeeze(axis=1) return pred_trans - def _more_tags(self): - return {'poor_score': True, 'no_validation': True} + def __sklearn_tags__(self): + regressor = self._get_regressor() + tags = super().__sklearn_tags__() + tags.regressor_tags.poor_score = True + tags.input_tags.sparse = get_tags(regressor).input_tags.sparse + tags.target_tags.multi_output = get_tags(regressor).target_tags.multi_output + return tags @property def n_features_in_(self): + """Number of features seen during :term:`fit`.""" # For consistency with other estimators we raise a AttributeError so # that hasattr() returns False the estimator isn't fitted. try: check_is_fitted(self) except NotFittedError as nfe: raise AttributeError( - "{} object has no n_features_in_ attribute." - .format(self.__class__.__name__) + "{} object has no n_features_in_ attribute.".format( + self.__class__.__name__ + ) ) from nfe return self.regressor_.n_features_in_ + + def get_metadata_routing(self): + """Get metadata routing of this object. + + Please check :ref:`User Guide ` on how the routing + mechanism works. + + .. versionadded:: 1.6 + + Returns + ------- + routing : MetadataRouter + A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating + routing information. + """ + router = MetadataRouter(owner=self.__class__.__name__).add( + regressor=self._get_regressor(), + method_mapping=MethodMapping() + .add(caller="fit", callee="fit") + .add(caller="predict", callee="predict"), + ) + return router + + def _get_regressor(self, get_clone=False): + if self.regressor is None: + return LinearRegression() + + return clone(self.regressor) if get_clone else self.regressor diff --git a/sklearn/compose/tests/test_column_transformer.py b/sklearn/compose/tests/test_column_transformer.py index ae2e25b68210f..daa4111c9393d 100644 --- a/sklearn/compose/tests/test_column_transformer.py +++ b/sklearn/compose/tests/test_column_transformer.py @@ -1,40 +1,57 @@ """ Test the ColumnTransformer. """ -import re -import pickle +import pickle +import re import warnings + +import joblib import numpy as np -from scipy import sparse import pytest - from numpy.testing import assert_allclose -from sklearn.utils._testing import assert_raise_message -from sklearn.utils._testing import assert_array_equal -from sklearn.utils._testing import assert_allclose_dense_sparse -from sklearn.utils._testing import assert_almost_equal +from scipy import sparse -from sklearn.base import BaseEstimator +from sklearn import config_context +from sklearn.base import BaseEstimator, TransformerMixin from sklearn.compose import ( - ColumnTransformer, make_column_transformer, make_column_selector + ColumnTransformer, + make_column_selector, + make_column_transformer, ) from sklearn.exceptions import NotFittedError -from sklearn.preprocessing import FunctionTransformer -from sklearn.preprocessing import StandardScaler, Normalizer, OneHotEncoder -from sklearn.feature_extraction import DictVectorizer +from sklearn.feature_selection import VarianceThreshold +from sklearn.preprocessing import ( + FunctionTransformer, + Normalizer, + OneHotEncoder, + StandardScaler, +) +from sklearn.tests.metadata_routing_common import ( + ConsumingTransformer, + _Registry, + check_recorded_metadata, +) +from sklearn.utils._indexing import _safe_indexing +from sklearn.utils._testing import ( + _convert_container, + assert_allclose_dense_sparse, + assert_almost_equal, + assert_array_equal, +) +from sklearn.utils.fixes import CSR_CONTAINERS, parse_version -class Trans(BaseEstimator): +class Trans(TransformerMixin, BaseEstimator): def fit(self, X, y=None): return self def transform(self, X, y=None): # 1D Series -> 2D DataFrame - if hasattr(X, 'to_frame'): + if hasattr(X, "to_frame"): return X.to_frame() # 1D array -> 2D array - if X.ndim == 1: + if getattr(X, "ndim", 2) == 1: return np.atleast_2d(X).T return X @@ -44,16 +61,19 @@ def fit(self, X, y=None): return self def transform(self, X): - return 2*X + return 2 * X class SparseMatrixTrans(BaseEstimator): + def __init__(self, csr_container): + self.csr_container = csr_container + def fit(self, X, y=None): return self def transform(self, X, y=None): n_samples = len(X) - return sparse.eye(n_samples, n_samples).tocsr() + return self.csr_container(sparse.eye(n_samples, n_samples)) class TransNo2D(BaseEstimator): @@ -65,7 +85,6 @@ def transform(self, X, y=None): class TransRaise(BaseEstimator): - def fit(self, X, y=None): raise ValueError("specific message") @@ -99,65 +118,86 @@ def test_column_transformer(): ] for selection, res in cases: - ct = ColumnTransformer([('trans', Trans(), selection)], - remainder='drop') + ct = ColumnTransformer([("trans", Trans(), selection)], remainder="drop") assert_array_equal(ct.fit_transform(X_array), res) assert_array_equal(ct.fit(X_array).transform(X_array), res) # callable that returns any of the allowed specifiers - ct = ColumnTransformer([('trans', Trans(), lambda x: selection)], - remainder='drop') + ct = ColumnTransformer( + [("trans", Trans(), lambda x: selection)], remainder="drop" + ) assert_array_equal(ct.fit_transform(X_array), res) assert_array_equal(ct.fit(X_array).transform(X_array), res) - ct = ColumnTransformer([('trans1', Trans(), [0]), - ('trans2', Trans(), [1])]) + ct = ColumnTransformer([("trans1", Trans(), [0]), ("trans2", Trans(), [1])]) assert_array_equal(ct.fit_transform(X_array), X_res_both) assert_array_equal(ct.fit(X_array).transform(X_array), X_res_both) assert len(ct.transformers_) == 2 # test with transformer_weights - transformer_weights = {'trans1': .1, 'trans2': 10} - both = ColumnTransformer([('trans1', Trans(), [0]), - ('trans2', Trans(), [1])], - transformer_weights=transformer_weights) - res = np.vstack([transformer_weights['trans1'] * X_res_first1D, - transformer_weights['trans2'] * X_res_second1D]).T + transformer_weights = {"trans1": 0.1, "trans2": 10} + both = ColumnTransformer( + [("trans1", Trans(), [0]), ("trans2", Trans(), [1])], + transformer_weights=transformer_weights, + ) + res = np.vstack( + [ + transformer_weights["trans1"] * X_res_first1D, + transformer_weights["trans2"] * X_res_second1D, + ] + ).T assert_array_equal(both.fit_transform(X_array), res) assert_array_equal(both.fit(X_array).transform(X_array), res) assert len(both.transformers_) == 2 - both = ColumnTransformer([('trans', Trans(), [0, 1])], - transformer_weights={'trans': .1}) + both = ColumnTransformer( + [("trans", Trans(), [0, 1])], transformer_weights={"trans": 0.1} + ) assert_array_equal(both.fit_transform(X_array), 0.1 * X_res_both) assert_array_equal(both.fit(X_array).transform(X_array), 0.1 * X_res_both) assert len(both.transformers_) == 1 -def test_column_transformer_dataframe(): - pd = pytest.importorskip('pandas') +def test_column_transformer_tuple_transformers_parameter(): + X_array = np.array([[0, 1, 2], [2, 4, 6]]).T + + transformers = [("trans1", Trans(), [0]), ("trans2", Trans(), [1])] + + ct_with_list = ColumnTransformer(transformers) + ct_with_tuple = ColumnTransformer(tuple(transformers)) + + assert_array_equal( + ct_with_list.fit_transform(X_array), ct_with_tuple.fit_transform(X_array) + ) + assert_array_equal( + ct_with_list.fit(X_array).transform(X_array), + ct_with_tuple.fit(X_array).transform(X_array), + ) + + +@pytest.mark.parametrize("constructor_name", ["dataframe", "polars"]) +def test_column_transformer_dataframe(constructor_name): + if constructor_name == "dataframe": + dataframe_lib = pytest.importorskip("pandas") + else: + dataframe_lib = pytest.importorskip(constructor_name) X_array = np.array([[0, 1, 2], [2, 4, 6]]).T - X_df = pd.DataFrame(X_array, columns=['first', 'second']) + X_df = _convert_container( + X_array, constructor_name, columns_name=["first", "second"] + ) X_res_first = np.array([0, 1, 2]).reshape(-1, 1) X_res_both = X_array cases = [ # String keys: label based - - # scalar - ('first', X_res_first), # list - (['first'], X_res_first), - (['first', 'second'], X_res_both), + (["first"], X_res_first), + (["first", "second"], X_res_both), # slice - (slice('first', 'second'), X_res_both), - + (slice("first", "second"), X_res_both), # int keys: positional - - # scalar - (0, X_res_first), # list ([0], X_res_first), ([0, 1], X_res_both), @@ -165,213 +205,331 @@ def test_column_transformer_dataframe(): # slice (slice(0, 1), X_res_first), (slice(0, 2), X_res_both), - # boolean mask (np.array([True, False]), X_res_first), - (pd.Series([True, False], index=['first', 'second']), X_res_first), ([True, False], X_res_first), ] + if constructor_name == "dataframe": + # Scalars are only supported for pandas dataframes. + cases.extend( + [ + # scalar + (0, X_res_first), + ("first", X_res_first), + ( + dataframe_lib.Series([True, False], index=["first", "second"]), + X_res_first, + ), + ] + ) for selection, res in cases: - ct = ColumnTransformer([('trans', Trans(), selection)], - remainder='drop') + ct = ColumnTransformer([("trans", Trans(), selection)], remainder="drop") assert_array_equal(ct.fit_transform(X_df), res) assert_array_equal(ct.fit(X_df).transform(X_df), res) # callable that returns any of the allowed specifiers - ct = ColumnTransformer([('trans', Trans(), lambda X: selection)], - remainder='drop') + ct = ColumnTransformer( + [("trans", Trans(), lambda X: selection)], remainder="drop" + ) assert_array_equal(ct.fit_transform(X_df), res) assert_array_equal(ct.fit(X_df).transform(X_df), res) - ct = ColumnTransformer([('trans1', Trans(), ['first']), - ('trans2', Trans(), ['second'])]) + ct = ColumnTransformer( + [("trans1", Trans(), ["first"]), ("trans2", Trans(), ["second"])] + ) assert_array_equal(ct.fit_transform(X_df), X_res_both) assert_array_equal(ct.fit(X_df).transform(X_df), X_res_both) assert len(ct.transformers_) == 2 - assert ct.transformers_[-1][0] != 'remainder' + assert ct.transformers_[-1][0] != "remainder" - ct = ColumnTransformer([('trans1', Trans(), [0]), - ('trans2', Trans(), [1])]) + ct = ColumnTransformer([("trans1", Trans(), [0]), ("trans2", Trans(), [1])]) assert_array_equal(ct.fit_transform(X_df), X_res_both) assert_array_equal(ct.fit(X_df).transform(X_df), X_res_both) assert len(ct.transformers_) == 2 - assert ct.transformers_[-1][0] != 'remainder' + assert ct.transformers_[-1][0] != "remainder" # test with transformer_weights - transformer_weights = {'trans1': .1, 'trans2': 10} - both = ColumnTransformer([('trans1', Trans(), ['first']), - ('trans2', Trans(), ['second'])], - transformer_weights=transformer_weights) - res = np.vstack([transformer_weights['trans1'] * X_df['first'], - transformer_weights['trans2'] * X_df['second']]).T + transformer_weights = {"trans1": 0.1, "trans2": 10} + both = ColumnTransformer( + [("trans1", Trans(), ["first"]), ("trans2", Trans(), ["second"])], + transformer_weights=transformer_weights, + ) + res = np.vstack( + [ + transformer_weights["trans1"] * X_df["first"], + transformer_weights["trans2"] * X_df["second"], + ] + ).T assert_array_equal(both.fit_transform(X_df), res) assert_array_equal(both.fit(X_df).transform(X_df), res) assert len(both.transformers_) == 2 - assert both.transformers_[-1][0] != 'remainder' + assert both.transformers_[-1][0] != "remainder" # test multiple columns - both = ColumnTransformer([('trans', Trans(), ['first', 'second'])], - transformer_weights={'trans': .1}) + both = ColumnTransformer( + [("trans", Trans(), ["first", "second"])], transformer_weights={"trans": 0.1} + ) assert_array_equal(both.fit_transform(X_df), 0.1 * X_res_both) assert_array_equal(both.fit(X_df).transform(X_df), 0.1 * X_res_both) assert len(both.transformers_) == 1 - assert both.transformers_[-1][0] != 'remainder' + assert both.transformers_[-1][0] != "remainder" - both = ColumnTransformer([('trans', Trans(), [0, 1])], - transformer_weights={'trans': .1}) + both = ColumnTransformer( + [("trans", Trans(), [0, 1])], transformer_weights={"trans": 0.1} + ) assert_array_equal(both.fit_transform(X_df), 0.1 * X_res_both) assert_array_equal(both.fit(X_df).transform(X_df), 0.1 * X_res_both) assert len(both.transformers_) == 1 - assert both.transformers_[-1][0] != 'remainder' + assert both.transformers_[-1][0] != "remainder" - # ensure pandas object is passes through + # ensure pandas object is passed through class TransAssert(BaseEstimator): + def __init__(self, expected_type_transform): + self.expected_type_transform = expected_type_transform def fit(self, X, y=None): return self def transform(self, X, y=None): - assert isinstance(X, (pd.DataFrame, pd.Series)) - if isinstance(X, pd.Series): + assert isinstance(X, self.expected_type_transform) + if isinstance(X, dataframe_lib.Series): X = X.to_frame() return X - ct = ColumnTransformer([('trans', TransAssert(), 'first')], - remainder='drop') - ct.fit_transform(X_df) - ct = ColumnTransformer([('trans', TransAssert(), ['first', 'second'])]) + ct = ColumnTransformer( + [ + ( + "trans", + TransAssert(expected_type_transform=dataframe_lib.DataFrame), + ["first", "second"], + ) + ] + ) ct.fit_transform(X_df) - # integer column spec + integer column names -> still use positional - X_df2 = X_df.copy() - X_df2.columns = [1, 0] - ct = ColumnTransformer([('trans', Trans(), 0)], remainder='drop') - assert_array_equal(ct.fit_transform(X_df2), X_res_first) - assert_array_equal(ct.fit(X_df2).transform(X_df2), X_res_first) - - assert len(ct.transformers_) == 2 - assert ct.transformers_[-1][0] == 'remainder' - assert ct.transformers_[-1][1] == 'drop' - assert_array_equal(ct.transformers_[-1][2], [1]) - - -@pytest.mark.parametrize("pandas", [True, False], ids=['pandas', 'numpy']) -@pytest.mark.parametrize("column_selection", [[], np.array([False, False]), - [False, False]], - ids=['list', 'bool', 'bool_int']) + if constructor_name == "dataframe": + # DataFrame protocol does not have 1d columns, so we only test on Pandas + # dataframes. + ct = ColumnTransformer( + [ + ( + "trans", + TransAssert(expected_type_transform=dataframe_lib.Series), + "first", + ) + ], + remainder="drop", + ) + ct.fit_transform(X_df) + + # Only test on pandas because the dataframe protocol requires string column + # names + # integer column spec + integer column names -> still use positional + X_df2 = X_df.copy() + X_df2.columns = [1, 0] + ct = ColumnTransformer([("trans", Trans(), 0)], remainder="drop") + assert_array_equal(ct.fit_transform(X_df2), X_res_first) + assert_array_equal(ct.fit(X_df2).transform(X_df2), X_res_first) + + assert len(ct.transformers_) == 2 + assert ct.transformers_[-1][0] == "remainder" + assert ct.transformers_[-1][1] == "drop" + assert_array_equal(ct.transformers_[-1][2], [1]) + + +@pytest.mark.parametrize("pandas", [True, False], ids=["pandas", "numpy"]) +@pytest.mark.parametrize( + "column_selection", + [[], np.array([False, False]), [False, False]], + ids=["list", "bool", "bool_int"], +) @pytest.mark.parametrize("callable_column", [False, True]) -def test_column_transformer_empty_columns(pandas, column_selection, - callable_column): +def test_column_transformer_empty_columns(pandas, column_selection, callable_column): # test case that ensures that the column transformer does also work when # a given transformer doesn't have any columns to work on X_array = np.array([[0, 1, 2], [2, 4, 6]]).T X_res_both = X_array if pandas: - pd = pytest.importorskip('pandas') - X = pd.DataFrame(X_array, columns=['first', 'second']) + pd = pytest.importorskip("pandas") + X = pd.DataFrame(X_array, columns=["first", "second"]) else: X = X_array if callable_column: - column = lambda X: column_selection # noqa + column = lambda X: column_selection else: column = column_selection - ct = ColumnTransformer([('trans1', Trans(), [0, 1]), - ('trans2', TransRaise(), column)]) + ct = ColumnTransformer( + [("trans1", Trans(), [0, 1]), ("trans2", TransRaise(), column)] + ) assert_array_equal(ct.fit_transform(X), X_res_both) assert_array_equal(ct.fit(X).transform(X), X_res_both) assert len(ct.transformers_) == 2 assert isinstance(ct.transformers_[1][1], TransRaise) - ct = ColumnTransformer([('trans1', TransRaise(), column), - ('trans2', Trans(), [0, 1])]) + ct = ColumnTransformer( + [("trans1", TransRaise(), column), ("trans2", Trans(), [0, 1])] + ) assert_array_equal(ct.fit_transform(X), X_res_both) assert_array_equal(ct.fit(X).transform(X), X_res_both) assert len(ct.transformers_) == 2 assert isinstance(ct.transformers_[0][1], TransRaise) - ct = ColumnTransformer([('trans', TransRaise(), column)], - remainder='passthrough') + ct = ColumnTransformer([("trans", TransRaise(), column)], remainder="passthrough") assert_array_equal(ct.fit_transform(X), X_res_both) assert_array_equal(ct.fit(X).transform(X), X_res_both) assert len(ct.transformers_) == 2 # including remainder assert isinstance(ct.transformers_[0][1], TransRaise) fixture = np.array([[], [], []]) - ct = ColumnTransformer([('trans', TransRaise(), column)], - remainder='drop') + ct = ColumnTransformer([("trans", TransRaise(), column)], remainder="drop") assert_array_equal(ct.fit_transform(X), fixture) assert_array_equal(ct.fit(X).transform(X), fixture) assert len(ct.transformers_) == 2 # including remainder assert isinstance(ct.transformers_[0][1], TransRaise) -def test_column_transformer_sparse_array(): - X_sparse = sparse.eye(3, 2).tocsr() +def test_column_transformer_output_indices(): + # Checks for the output_indices_ attribute + X_array = np.arange(6).reshape(3, 2) + + ct = ColumnTransformer([("trans1", Trans(), [0]), ("trans2", Trans(), [1])]) + X_trans = ct.fit_transform(X_array) + assert ct.output_indices_ == { + "trans1": slice(0, 1), + "trans2": slice(1, 2), + "remainder": slice(0, 0), + } + assert_array_equal(X_trans[:, [0]], X_trans[:, ct.output_indices_["trans1"]]) + assert_array_equal(X_trans[:, [1]], X_trans[:, ct.output_indices_["trans2"]]) + + # test with transformer_weights and multiple columns + ct = ColumnTransformer( + [("trans", Trans(), [0, 1])], transformer_weights={"trans": 0.1} + ) + X_trans = ct.fit_transform(X_array) + assert ct.output_indices_ == {"trans": slice(0, 2), "remainder": slice(0, 0)} + assert_array_equal(X_trans[:, [0, 1]], X_trans[:, ct.output_indices_["trans"]]) + assert_array_equal(X_trans[:, []], X_trans[:, ct.output_indices_["remainder"]]) + + # test case that ensures that the attribute does also work when + # a given transformer doesn't have any columns to work on + ct = ColumnTransformer([("trans1", Trans(), [0, 1]), ("trans2", TransRaise(), [])]) + X_trans = ct.fit_transform(X_array) + assert ct.output_indices_ == { + "trans1": slice(0, 2), + "trans2": slice(0, 0), + "remainder": slice(0, 0), + } + assert_array_equal(X_trans[:, [0, 1]], X_trans[:, ct.output_indices_["trans1"]]) + assert_array_equal(X_trans[:, []], X_trans[:, ct.output_indices_["trans2"]]) + assert_array_equal(X_trans[:, []], X_trans[:, ct.output_indices_["remainder"]]) + + ct = ColumnTransformer([("trans", TransRaise(), [])], remainder="passthrough") + X_trans = ct.fit_transform(X_array) + assert ct.output_indices_ == {"trans": slice(0, 0), "remainder": slice(0, 2)} + assert_array_equal(X_trans[:, []], X_trans[:, ct.output_indices_["trans"]]) + assert_array_equal(X_trans[:, [0, 1]], X_trans[:, ct.output_indices_["remainder"]]) + + +def test_column_transformer_output_indices_df(): + # Checks for the output_indices_ attribute with data frames + pd = pytest.importorskip("pandas") + + X_df = pd.DataFrame(np.arange(6).reshape(3, 2), columns=["first", "second"]) + + ct = ColumnTransformer( + [("trans1", Trans(), ["first"]), ("trans2", Trans(), ["second"])] + ) + X_trans = ct.fit_transform(X_df) + assert ct.output_indices_ == { + "trans1": slice(0, 1), + "trans2": slice(1, 2), + "remainder": slice(0, 0), + } + assert_array_equal(X_trans[:, [0]], X_trans[:, ct.output_indices_["trans1"]]) + assert_array_equal(X_trans[:, [1]], X_trans[:, ct.output_indices_["trans2"]]) + assert_array_equal(X_trans[:, []], X_trans[:, ct.output_indices_["remainder"]]) + + ct = ColumnTransformer([("trans1", Trans(), [0]), ("trans2", Trans(), [1])]) + X_trans = ct.fit_transform(X_df) + assert ct.output_indices_ == { + "trans1": slice(0, 1), + "trans2": slice(1, 2), + "remainder": slice(0, 0), + } + assert_array_equal(X_trans[:, [0]], X_trans[:, ct.output_indices_["trans1"]]) + assert_array_equal(X_trans[:, [1]], X_trans[:, ct.output_indices_["trans2"]]) + assert_array_equal(X_trans[:, []], X_trans[:, ct.output_indices_["remainder"]]) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_column_transformer_sparse_array(csr_container): + X_sparse = csr_container(sparse.eye(3, 2)) # no distinction between 1D and 2D - X_res_first = X_sparse[:, 0] + X_res_first = X_sparse[:, [0]] X_res_both = X_sparse - for col in [0, [0], slice(0, 1)]: - for remainder, res in [('drop', X_res_first), - ('passthrough', X_res_both)]: - ct = ColumnTransformer([('trans', Trans(), col)], - remainder=remainder, - sparse_threshold=0.8) + for col in [(0,), [0], slice(0, 1)]: + for remainder, res in [("drop", X_res_first), ("passthrough", X_res_both)]: + ct = ColumnTransformer( + [("trans", Trans(), col)], remainder=remainder, sparse_threshold=0.8 + ) assert sparse.issparse(ct.fit_transform(X_sparse)) assert_allclose_dense_sparse(ct.fit_transform(X_sparse), res) - assert_allclose_dense_sparse(ct.fit(X_sparse).transform(X_sparse), - res) + assert_allclose_dense_sparse(ct.fit(X_sparse).transform(X_sparse), res) for col in [[0, 1], slice(0, 2)]: - ct = ColumnTransformer([('trans', Trans(), col)], - sparse_threshold=0.8) + ct = ColumnTransformer([("trans", Trans(), col)], sparse_threshold=0.8) assert sparse.issparse(ct.fit_transform(X_sparse)) assert_allclose_dense_sparse(ct.fit_transform(X_sparse), X_res_both) - assert_allclose_dense_sparse(ct.fit(X_sparse).transform(X_sparse), - X_res_both) + assert_allclose_dense_sparse(ct.fit(X_sparse).transform(X_sparse), X_res_both) def test_column_transformer_list(): - X_list = [ - [1, float('nan'), 'a'], - [0, 0, 'b'] - ] - expected_result = np.array([ - [1, float('nan'), 1, 0], - [-1, 0, 0, 1], - ]) + X_list = [[1, float("nan"), "a"], [0, 0, "b"]] + expected_result = np.array( + [ + [1, float("nan"), 1, 0], + [-1, 0, 0, 1], + ] + ) - ct = ColumnTransformer([ - ('numerical', StandardScaler(), [0, 1]), - ('categorical', OneHotEncoder(), [2]), - ]) + ct = ColumnTransformer( + [ + ("numerical", StandardScaler(), [0, 1]), + ("categorical", OneHotEncoder(), [2]), + ] + ) assert_array_equal(ct.fit_transform(X_list), expected_result) assert_array_equal(ct.fit(X_list).transform(X_list), expected_result) -def test_column_transformer_sparse_stacking(): +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_column_transformer_sparse_stacking(csr_container): X_array = np.array([[0, 1, 2], [2, 4, 6]]).T - col_trans = ColumnTransformer([('trans1', Trans(), [0]), - ('trans2', SparseMatrixTrans(), 1)], - sparse_threshold=0.8) + col_trans = ColumnTransformer( + [("trans1", Trans(), [0]), ("trans2", SparseMatrixTrans(csr_container), 1)], + sparse_threshold=0.8, + ) col_trans.fit(X_array) X_trans = col_trans.transform(X_array) assert sparse.issparse(X_trans) assert X_trans.shape == (X_trans.shape[0], X_trans.shape[0] + 1) assert_array_equal(X_trans.toarray()[:, 1:], np.eye(X_trans.shape[0])) assert len(col_trans.transformers_) == 2 - assert col_trans.transformers_[-1][0] != 'remainder' + assert col_trans.transformers_[-1][0] != "remainder" - col_trans = ColumnTransformer([('trans1', Trans(), [0]), - ('trans2', SparseMatrixTrans(), 1)], - sparse_threshold=0.1) + col_trans = ColumnTransformer( + [("trans1", Trans(), [0]), ("trans2", SparseMatrixTrans(csr_container), 1)], + sparse_threshold=0.1, + ) col_trans.fit(X_array) X_trans = col_trans.transform(X_array) assert not sparse.issparse(X_trans) @@ -380,43 +538,36 @@ def test_column_transformer_sparse_stacking(): def test_column_transformer_mixed_cols_sparse(): - df = np.array([['a', 1, True], - ['b', 2, False]], - dtype='O') + df = np.array([["a", 1, True], ["b", 2, False]], dtype="O") ct = make_column_transformer( - (OneHotEncoder(), [0]), - ('passthrough', [1, 2]), - sparse_threshold=1.0 + (OneHotEncoder(), [0]), ("passthrough", [1, 2]), sparse_threshold=1.0 ) # this shouldn't fail, since boolean can be coerced into a numeric # See: https://github.com/scikit-learn/scikit-learn/issues/11912 X_trans = ct.fit_transform(df) - assert X_trans.getformat() == 'csr' - assert_array_equal(X_trans.toarray(), np.array([[1, 0, 1, 1], - [0, 1, 2, 0]])) + assert X_trans.format == "csr" + assert_array_equal(X_trans.toarray(), np.array([[1, 0, 1, 1], [0, 1, 2, 0]])) ct = make_column_transformer( - (OneHotEncoder(), [0]), - ('passthrough', [0]), - sparse_threshold=1.0 + (OneHotEncoder(), [0]), ("passthrough", [0]), sparse_threshold=1.0 ) - with pytest.raises(ValueError, - match="For a sparse output, all columns should"): + with pytest.raises(ValueError, match="For a sparse output, all columns should"): # this fails since strings `a` and `b` cannot be # coerced into a numeric. ct.fit_transform(df) def test_column_transformer_sparse_threshold(): - X_array = np.array([['a', 'b'], ['A', 'B']], dtype=object).T + X_array = np.array([["a", "b"], ["A", "B"]], dtype=object).T # above data has sparsity of 4 / 8 = 0.5 # apply threshold even if all sparse - col_trans = ColumnTransformer([('trans1', OneHotEncoder(), [0]), - ('trans2', OneHotEncoder(), [1])], - sparse_threshold=0.2) + col_trans = ColumnTransformer( + [("trans1", OneHotEncoder(), [0]), ("trans2", OneHotEncoder(), [1])], + sparse_threshold=0.2, + ) res = col_trans.fit_transform(X_array) assert not sparse.issparse(res) assert not col_trans.sparse_output_ @@ -424,18 +575,24 @@ def test_column_transformer_sparse_threshold(): # mixed -> sparsity of (4 + 2) / 8 = 0.75 for thres in [0.75001, 1]: col_trans = ColumnTransformer( - [('trans1', OneHotEncoder(sparse=True), [0]), - ('trans2', OneHotEncoder(sparse=False), [1])], - sparse_threshold=thres) + [ + ("trans1", OneHotEncoder(sparse_output=True), [0]), + ("trans2", OneHotEncoder(sparse_output=False), [1]), + ], + sparse_threshold=thres, + ) res = col_trans.fit_transform(X_array) assert sparse.issparse(res) assert col_trans.sparse_output_ for thres in [0.75, 0]: col_trans = ColumnTransformer( - [('trans1', OneHotEncoder(sparse=True), [0]), - ('trans2', OneHotEncoder(sparse=False), [1])], - sparse_threshold=thres) + [ + ("trans1", OneHotEncoder(sparse_output=True), [0]), + ("trans2", OneHotEncoder(sparse_output=False), [1]), + ], + sparse_threshold=thres, + ) res = col_trans.fit_transform(X_array) assert not sparse.issparse(res) assert not col_trans.sparse_output_ @@ -443,90 +600,101 @@ def test_column_transformer_sparse_threshold(): # if nothing is sparse -> no sparse for thres in [0.33, 0, 1]: col_trans = ColumnTransformer( - [('trans1', OneHotEncoder(sparse=False), [0]), - ('trans2', OneHotEncoder(sparse=False), [1])], - sparse_threshold=thres) + [ + ("trans1", OneHotEncoder(sparse_output=False), [0]), + ("trans2", OneHotEncoder(sparse_output=False), [1]), + ], + sparse_threshold=thres, + ) res = col_trans.fit_transform(X_array) assert not sparse.issparse(res) assert not col_trans.sparse_output_ def test_column_transformer_error_msg_1D(): - X_array = np.array([[0., 1., 2.], [2., 4., 6.]]).T + X_array = np.array([[0.0, 1.0, 2.0], [2.0, 4.0, 6.0]]).T + + col_trans = ColumnTransformer([("trans", StandardScaler(), 0)]) + msg = "1D data passed to a transformer" + with pytest.raises(ValueError, match=msg): + col_trans.fit(X_array) - col_trans = ColumnTransformer([('trans', StandardScaler(), 0)]) - assert_raise_message(ValueError, "1D data passed to a transformer", - col_trans.fit, X_array) - assert_raise_message(ValueError, "1D data passed to a transformer", - col_trans.fit_transform, X_array) + with pytest.raises(ValueError, match=msg): + col_trans.fit_transform(X_array) - col_trans = ColumnTransformer([('trans', TransRaise(), 0)]) + col_trans = ColumnTransformer([("trans", TransRaise(), 0)]) for func in [col_trans.fit, col_trans.fit_transform]: - assert_raise_message(ValueError, "specific message", func, X_array) + with pytest.raises(ValueError, match="specific message"): + func(X_array) def test_2D_transformer_output(): X_array = np.array([[0, 1, 2], [2, 4, 6]]).T # if one transformer is dropped, test that name is still correct - ct = ColumnTransformer([('trans1', 'drop', 0), - ('trans2', TransNo2D(), 1)]) - assert_raise_message(ValueError, "the 'trans2' transformer should be 2D", - ct.fit_transform, X_array) + ct = ColumnTransformer([("trans1", "drop", 0), ("trans2", TransNo2D(), 1)]) + + msg = "the 'trans2' transformer should be 2D" + with pytest.raises(ValueError, match=msg): + ct.fit_transform(X_array) # because fit is also doing transform, this raises already on fit - assert_raise_message(ValueError, "the 'trans2' transformer should be 2D", - ct.fit, X_array) + with pytest.raises(ValueError, match=msg): + ct.fit(X_array) def test_2D_transformer_output_pandas(): - pd = pytest.importorskip('pandas') + pd = pytest.importorskip("pandas") X_array = np.array([[0, 1, 2], [2, 4, 6]]).T - X_df = pd.DataFrame(X_array, columns=['col1', 'col2']) + X_df = pd.DataFrame(X_array, columns=["col1", "col2"]) # if one transformer is dropped, test that name is still correct - ct = ColumnTransformer([('trans1', TransNo2D(), 'col1')]) - assert_raise_message(ValueError, "the 'trans1' transformer should be 2D", - ct.fit_transform, X_df) + ct = ColumnTransformer([("trans1", TransNo2D(), "col1")]) + msg = "the 'trans1' transformer should be 2D" + with pytest.raises(ValueError, match=msg): + ct.fit_transform(X_df) # because fit is also doing transform, this raises already on fit - assert_raise_message(ValueError, "the 'trans1' transformer should be 2D", - ct.fit, X_df) + with pytest.raises(ValueError, match=msg): + ct.fit(X_df) -@pytest.mark.parametrize("remainder", ['drop', 'passthrough']) +@pytest.mark.parametrize("remainder", ["drop", "passthrough"]) def test_column_transformer_invalid_columns(remainder): X_array = np.array([[0, 1, 2], [2, 4, 6]]).T # general invalid - for col in [1.5, ['string', 1], slice(1, 's'), np.array([1.])]: - ct = ColumnTransformer([('trans', Trans(), col)], remainder=remainder) - assert_raise_message(ValueError, "No valid specification", - ct.fit, X_array) + for col in [1.5, ["string", 1], slice(1, "s"), np.array([1.0])]: + ct = ColumnTransformer([("trans", Trans(), col)], remainder=remainder) + with pytest.raises(ValueError, match="No valid specification"): + ct.fit(X_array) # invalid for arrays - for col in ['string', ['string', 'other'], slice('a', 'b')]: - ct = ColumnTransformer([('trans', Trans(), col)], remainder=remainder) - assert_raise_message(ValueError, "Specifying the columns", - ct.fit, X_array) + for col in ["string", ["string", "other"], slice("a", "b")]: + ct = ColumnTransformer([("trans", Trans(), col)], remainder=remainder) + with pytest.raises(ValueError, match="Specifying the columns"): + ct.fit(X_array) # transformed n_features does not match fitted n_features col = [0, 1] - ct = ColumnTransformer([('trans', Trans(), col)], remainder=remainder) + ct = ColumnTransformer([("trans", Trans(), col)], remainder=remainder) ct.fit(X_array) X_array_more = np.array([[0, 1, 2], [2, 4, 6], [3, 6, 9]]).T - msg = ("X has 3 features, but ColumnTransformer is expecting 2 features " - "as input.") + msg = "X has 3 features, but ColumnTransformer is expecting 2 features as input." with pytest.raises(ValueError, match=msg): ct.transform(X_array_more) - X_array_fewer = np.array([[0, 1, 2], ]).T - err_msg = ("X has 1 features, but ColumnTransformer is expecting 2 " - "features as input.") + X_array_fewer = np.array( + [ + [0, 1, 2], + ] + ).T + err_msg = ( + "X has 1 features, but ColumnTransformer is expecting 2 features as input." + ) with pytest.raises(ValueError, match=err_msg): ct.transform(X_array_fewer) def test_column_transformer_invalid_transformer(): - class NoTrans(BaseEstimator): def fit(self, X, y=None): return self @@ -535,275 +703,195 @@ def predict(self, X): return X X_array = np.array([[0, 1, 2], [2, 4, 6]]).T - ct = ColumnTransformer([('trans', NoTrans(), [0])]) - assert_raise_message(TypeError, - "All estimators should implement fit and transform", - ct.fit, X_array) + ct = ColumnTransformer([("trans", NoTrans(), [0])]) + msg = "All estimators should implement fit and transform" + with pytest.raises(TypeError, match=msg): + ct.fit(X_array) def test_make_column_transformer(): scaler = StandardScaler() norm = Normalizer() - ct = make_column_transformer((scaler, 'first'), (norm, ['second'])) + ct = make_column_transformer((scaler, "first"), (norm, ["second"])) names, transformers, columns = zip(*ct.transformers) assert names == ("standardscaler", "normalizer") assert transformers == (scaler, norm) - assert columns == ('first', ['second']) + assert columns == ("first", ["second"]) def test_make_column_transformer_pandas(): - pd = pytest.importorskip('pandas') + pd = pytest.importorskip("pandas") X_array = np.array([[0, 1, 2], [2, 4, 6]]).T - X_df = pd.DataFrame(X_array, columns=['first', 'second']) + X_df = pd.DataFrame(X_array, columns=["first", "second"]) norm = Normalizer() - ct1 = ColumnTransformer([('norm', Normalizer(), X_df.columns)]) + ct1 = ColumnTransformer([("norm", Normalizer(), X_df.columns)]) ct2 = make_column_transformer((norm, X_df.columns)) - assert_almost_equal(ct1.fit_transform(X_df), - ct2.fit_transform(X_df)) + assert_almost_equal(ct1.fit_transform(X_df), ct2.fit_transform(X_df)) def test_make_column_transformer_kwargs(): scaler = StandardScaler() norm = Normalizer() - ct = make_column_transformer((scaler, 'first'), (norm, ['second']), - n_jobs=3, remainder='drop', - sparse_threshold=0.5) - assert ct.transformers == make_column_transformer( - (scaler, 'first'), (norm, ['second'])).transformers + ct = make_column_transformer( + (scaler, "first"), + (norm, ["second"]), + n_jobs=3, + remainder="drop", + sparse_threshold=0.5, + ) + assert ( + ct.transformers + == make_column_transformer((scaler, "first"), (norm, ["second"])).transformers + ) assert ct.n_jobs == 3 - assert ct.remainder == 'drop' + assert ct.remainder == "drop" assert ct.sparse_threshold == 0.5 # invalid keyword parameters should raise an error message - assert_raise_message( - TypeError, + msg = re.escape( "make_column_transformer() got an unexpected " - "keyword argument 'transformer_weights'", - make_column_transformer, (scaler, 'first'), (norm, ['second']), - transformer_weights={'pca': 10, 'Transf': 1} + "keyword argument 'transformer_weights'" ) + with pytest.raises(TypeError, match=msg): + make_column_transformer( + (scaler, "first"), + (norm, ["second"]), + transformer_weights={"pca": 10, "Transf": 1}, + ) def test_make_column_transformer_remainder_transformer(): scaler = StandardScaler() norm = Normalizer() remainder = StandardScaler() - ct = make_column_transformer((scaler, 'first'), (norm, ['second']), - remainder=remainder) + ct = make_column_transformer( + (scaler, "first"), (norm, ["second"]), remainder=remainder + ) assert ct.remainder == remainder def test_column_transformer_get_set_params(): - ct = ColumnTransformer([('trans1', StandardScaler(), [0]), - ('trans2', StandardScaler(), [1])]) - - exp = {'n_jobs': None, - 'remainder': 'drop', - 'sparse_threshold': 0.3, - 'trans1': ct.transformers[0][1], - 'trans1__copy': True, - 'trans1__with_mean': True, - 'trans1__with_std': True, - 'trans2': ct.transformers[1][1], - 'trans2__copy': True, - 'trans2__with_mean': True, - 'trans2__with_std': True, - 'transformers': ct.transformers, - 'transformer_weights': None, - 'verbose': False} + ct = ColumnTransformer( + [("trans1", StandardScaler(), [0]), ("trans2", StandardScaler(), [1])] + ) + + exp = { + "n_jobs": None, + "remainder": "drop", + "sparse_threshold": 0.3, + "trans1": ct.transformers[0][1], + "trans1__copy": True, + "trans1__with_mean": True, + "trans1__with_std": True, + "trans2": ct.transformers[1][1], + "trans2__copy": True, + "trans2__with_mean": True, + "trans2__with_std": True, + "transformers": ct.transformers, + "transformer_weights": None, + "verbose_feature_names_out": True, + "verbose": False, + "force_int_remainder_cols": "deprecated", + } assert ct.get_params() == exp ct.set_params(trans1__with_mean=False) - assert not ct.get_params()['trans1__with_mean'] - - ct.set_params(trans1='passthrough') - exp = {'n_jobs': None, - 'remainder': 'drop', - 'sparse_threshold': 0.3, - 'trans1': 'passthrough', - 'trans2': ct.transformers[1][1], - 'trans2__copy': True, - 'trans2__with_mean': True, - 'trans2__with_std': True, - 'transformers': ct.transformers, - 'transformer_weights': None, - 'verbose': False} + assert not ct.get_params()["trans1__with_mean"] + + ct.set_params(trans1="passthrough") + exp = { + "n_jobs": None, + "remainder": "drop", + "sparse_threshold": 0.3, + "trans1": "passthrough", + "trans2": ct.transformers[1][1], + "trans2__copy": True, + "trans2__with_mean": True, + "trans2__with_std": True, + "transformers": ct.transformers, + "transformer_weights": None, + "verbose_feature_names_out": True, + "verbose": False, + "force_int_remainder_cols": "deprecated", + } assert ct.get_params() == exp def test_column_transformer_named_estimators(): - X_array = np.array([[0., 1., 2.], [2., 4., 6.]]).T - ct = ColumnTransformer([('trans1', StandardScaler(), [0]), - ('trans2', StandardScaler(with_std=False), [1])]) - assert not hasattr(ct, 'transformers_') + X_array = np.array([[0.0, 1.0, 2.0], [2.0, 4.0, 6.0]]).T + ct = ColumnTransformer( + [ + ("trans1", StandardScaler(), [0]), + ("trans2", StandardScaler(with_std=False), [1]), + ] + ) + assert not hasattr(ct, "transformers_") ct.fit(X_array) - assert hasattr(ct, 'transformers_') - assert isinstance(ct.named_transformers_['trans1'], StandardScaler) + assert hasattr(ct, "transformers_") + assert isinstance(ct.named_transformers_["trans1"], StandardScaler) assert isinstance(ct.named_transformers_.trans1, StandardScaler) - assert isinstance(ct.named_transformers_['trans2'], StandardScaler) + assert isinstance(ct.named_transformers_["trans2"], StandardScaler) assert isinstance(ct.named_transformers_.trans2, StandardScaler) assert not ct.named_transformers_.trans2.with_std # check it are fitted transformers - assert ct.named_transformers_.trans1.mean_ == 1. + assert ct.named_transformers_.trans1.mean_ == 1.0 def test_column_transformer_cloning(): - X_array = np.array([[0., 1., 2.], [2., 4., 6.]]).T + X_array = np.array([[0.0, 1.0, 2.0], [2.0, 4.0, 6.0]]).T - ct = ColumnTransformer([('trans', StandardScaler(), [0])]) + ct = ColumnTransformer([("trans", StandardScaler(), [0])]) ct.fit(X_array) - assert not hasattr(ct.transformers[0][1], 'mean_') - assert hasattr(ct.transformers_[0][1], 'mean_') + assert not hasattr(ct.transformers[0][1], "mean_") + assert hasattr(ct.transformers_[0][1], "mean_") - ct = ColumnTransformer([('trans', StandardScaler(), [0])]) + ct = ColumnTransformer([("trans", StandardScaler(), [0])]) ct.fit_transform(X_array) - assert not hasattr(ct.transformers[0][1], 'mean_') - assert hasattr(ct.transformers_[0][1], 'mean_') + assert not hasattr(ct.transformers[0][1], "mean_") + assert hasattr(ct.transformers_[0][1], "mean_") def test_column_transformer_get_feature_names(): - X_array = np.array([[0., 1., 2.], [2., 4., 6.]]).T - ct = ColumnTransformer([('trans', Trans(), [0, 1])]) + X_array = np.array([[0.0, 1.0, 2.0], [2.0, 4.0, 6.0]]).T + ct = ColumnTransformer([("trans", Trans(), [0, 1])]) # raise correct error when not fitted with pytest.raises(NotFittedError): - ct.get_feature_names() + ct.get_feature_names_out() # raise correct error when no feature names are available ct.fit(X_array) - assert_raise_message(AttributeError, - "Transformer trans (type Trans) does not provide " - "get_feature_names", ct.get_feature_names) - - # working example - X = np.array([[{'a': 1, 'b': 2}, {'a': 3, 'b': 4}], - [{'c': 5}, {'c': 6}]], dtype=object).T - ct = ColumnTransformer( - [('col' + str(i), DictVectorizer(), i) for i in range(2)]) - ct.fit(X) - assert ct.get_feature_names() == ['col0__a', 'col0__b', 'col1__c'] - - # drop transformer - ct = ColumnTransformer( - [('col0', DictVectorizer(), 0), ('col1', 'drop', 1)]) - ct.fit(X) - assert ct.get_feature_names() == ['col0__a', 'col0__b'] - - # passthrough transformer - ct = ColumnTransformer([('trans', 'passthrough', [0, 1])]) - ct.fit(X) - assert ct.get_feature_names() == ['x0', 'x1'] - - ct = ColumnTransformer([('trans', DictVectorizer(), 0)], - remainder='passthrough') - ct.fit(X) - assert ct.get_feature_names() == ['trans__a', 'trans__b', 'x1'] - - ct = ColumnTransformer([('trans', 'passthrough', [1])], - remainder='passthrough') - ct.fit(X) - assert ct.get_feature_names() == ['x1', 'x0'] - - ct = ColumnTransformer([('trans', 'passthrough', lambda x: [1])], - remainder='passthrough') - ct.fit(X) - assert ct.get_feature_names() == ['x1', 'x0'] - - ct = ColumnTransformer([('trans', 'passthrough', np.array([False, True]))], - remainder='passthrough') - ct.fit(X) - assert ct.get_feature_names() == ['x1', 'x0'] - - ct = ColumnTransformer([('trans', 'passthrough', slice(1, 2))], - remainder='passthrough') - ct.fit(X) - assert ct.get_feature_names() == ['x1', 'x0'] - - -def test_column_transformer_get_feature_names_dataframe(): - # passthough transformer with a dataframe - pd = pytest.importorskip('pandas') - X = np.array([[{'a': 1, 'b': 2}, {'a': 3, 'b': 4}], - [{'c': 5}, {'c': 6}]], dtype=object).T - X_df = pd.DataFrame(X, columns=['col0', 'col1']) - - ct = ColumnTransformer([('trans', 'passthrough', ['col0', 'col1'])]) - ct.fit(X_df) - assert ct.get_feature_names() == ['col0', 'col1'] - - ct = ColumnTransformer([('trans', 'passthrough', [0, 1])]) - ct.fit(X_df) - assert ct.get_feature_names() == ['col0', 'col1'] - - ct = ColumnTransformer([('col0', DictVectorizer(), 0)], - remainder='passthrough') - ct.fit(X_df) - assert ct.get_feature_names() == ['col0__a', 'col0__b', 'col1'] - - ct = ColumnTransformer([('trans', 'passthrough', ['col1'])], - remainder='passthrough') - ct.fit(X_df) - assert ct.get_feature_names() == ['col1', 'col0'] - - ct = ColumnTransformer([('trans', 'passthrough', - lambda x: x[['col1']].columns)], - remainder='passthrough') - ct.fit(X_df) - assert ct.get_feature_names() == ['col1', 'col0'] - - ct = ColumnTransformer([('trans', 'passthrough', np.array([False, True]))], - remainder='passthrough') - ct.fit(X_df) - assert ct.get_feature_names() == ['col1', 'col0'] - - ct = ColumnTransformer([('trans', 'passthrough', slice(1, 2))], - remainder='passthrough') - ct.fit(X_df) - assert ct.get_feature_names() == ['col1', 'col0'] - - ct = ColumnTransformer([('trans', 'passthrough', [1])], - remainder='passthrough') - ct.fit(X_df) - assert ct.get_feature_names() == ['col1', 'col0'] + msg = re.escape( + "Transformer trans (type Trans) does not provide get_feature_names_out" + ) + with pytest.raises(AttributeError, match=msg): + ct.get_feature_names_out() def test_column_transformer_special_strings(): - # one 'drop' -> ignore - X_array = np.array([[0., 1., 2.], [2., 4., 6.]]).T - ct = ColumnTransformer( - [('trans1', Trans(), [0]), ('trans2', 'drop', [1])]) - exp = np.array([[0.], [1.], [2.]]) + X_array = np.array([[0.0, 1.0, 2.0], [2.0, 4.0, 6.0]]).T + ct = ColumnTransformer([("trans1", Trans(), [0]), ("trans2", "drop", [1])]) + exp = np.array([[0.0], [1.0], [2.0]]) assert_array_equal(ct.fit_transform(X_array), exp) assert_array_equal(ct.fit(X_array).transform(X_array), exp) assert len(ct.transformers_) == 2 - assert ct.transformers_[-1][0] != 'remainder' + assert ct.transformers_[-1][0] != "remainder" # all 'drop' -> return shape 0 array - ct = ColumnTransformer( - [('trans1', 'drop', [0]), ('trans2', 'drop', [1])]) + ct = ColumnTransformer([("trans1", "drop", [0]), ("trans2", "drop", [1])]) assert_array_equal(ct.fit(X_array).transform(X_array).shape, (3, 0)) assert_array_equal(ct.fit_transform(X_array).shape, (3, 0)) assert len(ct.transformers_) == 2 - assert ct.transformers_[-1][0] != 'remainder' + assert ct.transformers_[-1][0] != "remainder" # 'passthrough' - X_array = np.array([[0., 1., 2.], [2., 4., 6.]]).T - ct = ColumnTransformer( - [('trans1', Trans(), [0]), ('trans2', 'passthrough', [1])]) + X_array = np.array([[0.0, 1.0, 2.0], [2.0, 4.0, 6.0]]).T + ct = ColumnTransformer([("trans1", Trans(), [0]), ("trans2", "passthrough", [1])]) exp = X_array assert_array_equal(ct.fit_transform(X_array), exp) assert_array_equal(ct.fit(X_array).transform(X_array), exp) assert len(ct.transformers_) == 2 - assert ct.transformers_[-1][0] != 'remainder' - - # None itself / other string is not valid - for val in [None, 'other']: - ct = ColumnTransformer( - [('trans1', Trans(), [0]), ('trans2', None, [1])]) - assert_raise_message(TypeError, "All estimators should implement", - ct.fit_transform, X_array) - assert_raise_message(TypeError, "All estimators should implement", - ct.fit, X_array) + assert ct.transformers_[-1][0] != "remainder" def test_column_transformer_remainder(): @@ -814,163 +902,220 @@ def test_column_transformer_remainder(): X_res_both = X_array # default drop - ct = ColumnTransformer([('trans1', Trans(), [0])]) + ct = ColumnTransformer([("trans1", Trans(), [0])]) assert_array_equal(ct.fit_transform(X_array), X_res_first) assert_array_equal(ct.fit(X_array).transform(X_array), X_res_first) assert len(ct.transformers_) == 2 - assert ct.transformers_[-1][0] == 'remainder' - assert ct.transformers_[-1][1] == 'drop' + assert ct.transformers_[-1][0] == "remainder" + assert ct.transformers_[-1][1] == "drop" assert_array_equal(ct.transformers_[-1][2], [1]) # specify passthrough - ct = ColumnTransformer([('trans', Trans(), [0])], remainder='passthrough') + ct = ColumnTransformer([("trans", Trans(), [0])], remainder="passthrough") assert_array_equal(ct.fit_transform(X_array), X_res_both) assert_array_equal(ct.fit(X_array).transform(X_array), X_res_both) assert len(ct.transformers_) == 2 - assert ct.transformers_[-1][0] == 'remainder' - assert ct.transformers_[-1][1] == 'passthrough' + assert ct.transformers_[-1][0] == "remainder" + assert isinstance(ct.transformers_[-1][1], FunctionTransformer) assert_array_equal(ct.transformers_[-1][2], [1]) # column order is not preserved (passed through added to end) - ct = ColumnTransformer([('trans1', Trans(), [1])], - remainder='passthrough') + ct = ColumnTransformer([("trans1", Trans(), [1])], remainder="passthrough") assert_array_equal(ct.fit_transform(X_array), X_res_both[:, ::-1]) assert_array_equal(ct.fit(X_array).transform(X_array), X_res_both[:, ::-1]) assert len(ct.transformers_) == 2 - assert ct.transformers_[-1][0] == 'remainder' - assert ct.transformers_[-1][1] == 'passthrough' + assert ct.transformers_[-1][0] == "remainder" + assert isinstance(ct.transformers_[-1][1], FunctionTransformer) assert_array_equal(ct.transformers_[-1][2], [0]) # passthrough when all actual transformers are skipped - ct = ColumnTransformer([('trans1', 'drop', [0])], - remainder='passthrough') + ct = ColumnTransformer([("trans1", "drop", [0])], remainder="passthrough") assert_array_equal(ct.fit_transform(X_array), X_res_second) assert_array_equal(ct.fit(X_array).transform(X_array), X_res_second) assert len(ct.transformers_) == 2 - assert ct.transformers_[-1][0] == 'remainder' - assert ct.transformers_[-1][1] == 'passthrough' + assert ct.transformers_[-1][0] == "remainder" + assert isinstance(ct.transformers_[-1][1], FunctionTransformer) assert_array_equal(ct.transformers_[-1][2], [1]) - # error on invalid arg - ct = ColumnTransformer([('trans1', Trans(), [0])], remainder=1) - assert_raise_message( - ValueError, - "remainder keyword needs to be one of \'drop\', \'passthrough\', " - "or estimator.", ct.fit, X_array) - assert_raise_message( - ValueError, - "remainder keyword needs to be one of \'drop\', \'passthrough\', " - "or estimator.", ct.fit_transform, X_array) - # check default for make_column_transformer ct = make_column_transformer((Trans(), [0])) - assert ct.remainder == 'drop' + assert ct.remainder == "drop" + + +@pytest.mark.parametrize( + "cols1, cols2, expected_remainder_cols", + [ + ([0], [False, True, False], [2]), # mix types + ([0], [1], [2]), # ints + (lambda x: [0], lambda x: [1], [2]), # callables + (["A"], ["B"], ["C"]), # all strings + ([True, False, False], [False, True, False], [False, False, True]), # all bools + ], +) +def test_column_transformer_remainder_dtypes(cols1, cols2, expected_remainder_cols): + """Check that the remainder columns format matches the format of the other + columns when they're all strings or masks. + """ + X = np.ones((1, 3)) + + if isinstance(cols1, list) and isinstance(cols1[0], str): + pd = pytest.importorskip("pandas") + X = pd.DataFrame(X, columns=["A", "B", "C"]) + + # if inputs are column names store remainder columns as column names + ct = make_column_transformer( + (Trans(), cols1), + (Trans(), cols2), + remainder="passthrough", + ) + ct.fit_transform(X) + assert ct.transformers_[-1][-1] == expected_remainder_cols -@pytest.mark.parametrize("key", [[0], np.array([0]), slice(0, 1), - np.array([True, False])]) -def test_column_transformer_remainder_numpy(key): +# TODO(1.9): remove this test +@pytest.mark.parametrize("force_int_remainder_cols", [True, False]) +def test_force_int_remainder_cols_deprecation(force_int_remainder_cols): + """Check that ColumnTransformer raises a FutureWarning when + force_int_remainder_cols is set. + """ + X = np.ones((1, 3)) + ct = ColumnTransformer( + [("T1", Trans(), [0]), ("T2", Trans(), [1])], + remainder="passthrough", + force_int_remainder_cols=force_int_remainder_cols, + ) + + with pytest.warns(FutureWarning, match="`force_int_remainder_cols` is deprecated"): + ct.fit(X) + + +@pytest.mark.parametrize( + "key, expected_cols", + [ + ([0], [1]), + (np.array([0]), [1]), + (slice(0, 1), [1]), + (np.array([True, False]), [False, True]), + ], +) +def test_column_transformer_remainder_numpy(key, expected_cols): # test different ways that columns are specified with passthrough X_array = np.array([[0, 1, 2], [2, 4, 6]]).T X_res_both = X_array - ct = ColumnTransformer([('trans1', Trans(), key)], - remainder='passthrough') + ct = ColumnTransformer( + [("trans1", Trans(), key)], + remainder="passthrough", + ) assert_array_equal(ct.fit_transform(X_array), X_res_both) assert_array_equal(ct.fit(X_array).transform(X_array), X_res_both) assert len(ct.transformers_) == 2 - assert ct.transformers_[-1][0] == 'remainder' - assert ct.transformers_[-1][1] == 'passthrough' - assert_array_equal(ct.transformers_[-1][2], [1]) + assert ct.transformers_[-1][0] == "remainder" + assert isinstance(ct.transformers_[-1][1], FunctionTransformer) + assert ct.transformers_[-1][2] == expected_cols @pytest.mark.parametrize( - "key", [[0], slice(0, 1), np.array([True, False]), ['first'], 'pd-index', - np.array(['first']), np.array(['first'], dtype=object), - slice(None, 'first'), slice('first', 'first')]) -def test_column_transformer_remainder_pandas(key): + "key, expected_cols", + [ + ([0], [1]), + (slice(0, 1), [1]), + (np.array([True, False]), [False, True]), + (["first"], ["second"]), + ("pd-index", ["second"]), + (np.array(["first"]), ["second"]), + (np.array(["first"], dtype=object), ["second"]), + (slice(None, "first"), ["second"]), + (slice("first", "first"), ["second"]), + ], +) +def test_column_transformer_remainder_pandas(key, expected_cols): # test different ways that columns are specified with passthrough - pd = pytest.importorskip('pandas') - if isinstance(key, str) and key == 'pd-index': - key = pd.Index(['first']) + pd = pytest.importorskip("pandas") + if isinstance(key, str) and key == "pd-index": + key = pd.Index(["first"]) X_array = np.array([[0, 1, 2], [2, 4, 6]]).T - X_df = pd.DataFrame(X_array, columns=['first', 'second']) + X_df = pd.DataFrame(X_array, columns=["first", "second"]) X_res_both = X_array - ct = ColumnTransformer([('trans1', Trans(), key)], - remainder='passthrough') + ct = ColumnTransformer( + [("trans1", Trans(), key)], + remainder="passthrough", + ) assert_array_equal(ct.fit_transform(X_df), X_res_both) assert_array_equal(ct.fit(X_df).transform(X_df), X_res_both) assert len(ct.transformers_) == 2 - assert ct.transformers_[-1][0] == 'remainder' - assert ct.transformers_[-1][1] == 'passthrough' - assert_array_equal(ct.transformers_[-1][2], [1]) + assert ct.transformers_[-1][0] == "remainder" + assert isinstance(ct.transformers_[-1][1], FunctionTransformer) + assert ct.transformers_[-1][2] == expected_cols -@pytest.mark.parametrize("key", [[0], np.array([0]), slice(0, 1), - np.array([True, False, False])]) -def test_column_transformer_remainder_transformer(key): - X_array = np.array([[0, 1, 2], - [2, 4, 6], - [8, 6, 4]]).T +@pytest.mark.parametrize( + "key, expected_cols", + [ + ([0], [1, 2]), + (np.array([0]), [1, 2]), + (slice(0, 1), [1, 2]), + (np.array([True, False, False]), [False, True, True]), + ], +) +def test_column_transformer_remainder_transformer(key, expected_cols): + X_array = np.array([[0, 1, 2], [2, 4, 6], [8, 6, 4]]).T X_res_both = X_array.copy() # second and third columns are doubled when remainder = DoubleTrans X_res_both[:, 1:3] *= 2 - ct = ColumnTransformer([('trans1', Trans(), key)], - remainder=DoubleTrans()) + ct = ColumnTransformer( + [("trans1", Trans(), key)], + remainder=DoubleTrans(), + ) assert_array_equal(ct.fit_transform(X_array), X_res_both) assert_array_equal(ct.fit(X_array).transform(X_array), X_res_both) assert len(ct.transformers_) == 2 - assert ct.transformers_[-1][0] == 'remainder' + assert ct.transformers_[-1][0] == "remainder" assert isinstance(ct.transformers_[-1][1], DoubleTrans) - assert_array_equal(ct.transformers_[-1][2], [1, 2]) + assert ct.transformers_[-1][2] == expected_cols def test_column_transformer_no_remaining_remainder_transformer(): - X_array = np.array([[0, 1, 2], - [2, 4, 6], - [8, 6, 4]]).T + X_array = np.array([[0, 1, 2], [2, 4, 6], [8, 6, 4]]).T - ct = ColumnTransformer([('trans1', Trans(), [0, 1, 2])], - remainder=DoubleTrans()) + ct = ColumnTransformer([("trans1", Trans(), [0, 1, 2])], remainder=DoubleTrans()) assert_array_equal(ct.fit_transform(X_array), X_array) assert_array_equal(ct.fit(X_array).transform(X_array), X_array) assert len(ct.transformers_) == 1 - assert ct.transformers_[-1][0] != 'remainder' + assert ct.transformers_[-1][0] != "remainder" def test_column_transformer_drops_all_remainder_transformer(): - X_array = np.array([[0, 1, 2], - [2, 4, 6], - [8, 6, 4]]).T + X_array = np.array([[0, 1, 2], [2, 4, 6], [8, 6, 4]]).T # columns are doubled when remainder = DoubleTrans X_res_both = 2 * X_array.copy()[:, 1:3] - ct = ColumnTransformer([('trans1', 'drop', [0])], - remainder=DoubleTrans()) + ct = ColumnTransformer([("trans1", "drop", [0])], remainder=DoubleTrans()) assert_array_equal(ct.fit_transform(X_array), X_res_both) assert_array_equal(ct.fit(X_array).transform(X_array), X_res_both) assert len(ct.transformers_) == 2 - assert ct.transformers_[-1][0] == 'remainder' + assert ct.transformers_[-1][0] == "remainder" assert isinstance(ct.transformers_[-1][1], DoubleTrans) assert_array_equal(ct.transformers_[-1][2], [1, 2]) -def test_column_transformer_sparse_remainder_transformer(): - X_array = np.array([[0, 1, 2], - [2, 4, 6], - [8, 6, 4]]).T +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_column_transformer_sparse_remainder_transformer(csr_container): + X_array = np.array([[0, 1, 2], [2, 4, 6], [8, 6, 4]]).T - ct = ColumnTransformer([('trans1', Trans(), [0])], - remainder=SparseMatrixTrans(), - sparse_threshold=0.8) + ct = ColumnTransformer( + [("trans1", Trans(), [0])], + remainder=SparseMatrixTrans(csr_container), + sparse_threshold=0.8, + ) X_trans = ct.fit_transform(X_array) assert sparse.issparse(X_trans) @@ -978,22 +1123,22 @@ def test_column_transformer_sparse_remainder_transformer(): # one column in ``transformers``, thus: assert X_trans.shape == (3, 3 + 1) - exp_array = np.hstack( - (X_array[:, 0].reshape(-1, 1), np.eye(3))) + exp_array = np.hstack((X_array[:, 0].reshape(-1, 1), np.eye(3))) assert_array_equal(X_trans.toarray(), exp_array) assert len(ct.transformers_) == 2 - assert ct.transformers_[-1][0] == 'remainder' + assert ct.transformers_[-1][0] == "remainder" assert isinstance(ct.transformers_[-1][1], SparseMatrixTrans) assert_array_equal(ct.transformers_[-1][2], [1, 2]) -def test_column_transformer_drop_all_sparse_remainder_transformer(): - X_array = np.array([[0, 1, 2], - [2, 4, 6], - [8, 6, 4]]).T - ct = ColumnTransformer([('trans1', 'drop', [0])], - remainder=SparseMatrixTrans(), - sparse_threshold=0.8) +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_column_transformer_drop_all_sparse_remainder_transformer(csr_container): + X_array = np.array([[0, 1, 2], [2, 4, 6], [8, 6, 4]]).T + ct = ColumnTransformer( + [("trans1", "drop", [0])], + remainder=SparseMatrixTrans(csr_container), + sparse_threshold=0.8, + ) X_trans = ct.fit_transform(X_array) assert sparse.issparse(X_trans) @@ -1002,109 +1147,147 @@ def test_column_transformer_drop_all_sparse_remainder_transformer(): assert X_trans.shape == (3, 3) assert_array_equal(X_trans.toarray(), np.eye(3)) assert len(ct.transformers_) == 2 - assert ct.transformers_[-1][0] == 'remainder' + assert ct.transformers_[-1][0] == "remainder" assert isinstance(ct.transformers_[-1][1], SparseMatrixTrans) assert_array_equal(ct.transformers_[-1][2], [1, 2]) def test_column_transformer_get_set_params_with_remainder(): - ct = ColumnTransformer([('trans1', StandardScaler(), [0])], - remainder=StandardScaler()) - - exp = {'n_jobs': None, - 'remainder': ct.remainder, - 'remainder__copy': True, - 'remainder__with_mean': True, - 'remainder__with_std': True, - 'sparse_threshold': 0.3, - 'trans1': ct.transformers[0][1], - 'trans1__copy': True, - 'trans1__with_mean': True, - 'trans1__with_std': True, - 'transformers': ct.transformers, - 'transformer_weights': None, - 'verbose': False} + ct = ColumnTransformer( + [("trans1", StandardScaler(), [0])], remainder=StandardScaler() + ) + + exp = { + "n_jobs": None, + "remainder": ct.remainder, + "remainder__copy": True, + "remainder__with_mean": True, + "remainder__with_std": True, + "sparse_threshold": 0.3, + "trans1": ct.transformers[0][1], + "trans1__copy": True, + "trans1__with_mean": True, + "trans1__with_std": True, + "transformers": ct.transformers, + "transformer_weights": None, + "verbose_feature_names_out": True, + "verbose": False, + "force_int_remainder_cols": "deprecated", + } assert ct.get_params() == exp ct.set_params(remainder__with_std=False) - assert not ct.get_params()['remainder__with_std'] - - ct.set_params(trans1='passthrough') - exp = {'n_jobs': None, - 'remainder': ct.remainder, - 'remainder__copy': True, - 'remainder__with_mean': True, - 'remainder__with_std': False, - 'sparse_threshold': 0.3, - 'trans1': 'passthrough', - 'transformers': ct.transformers, - 'transformer_weights': None, - 'verbose': False} - + assert not ct.get_params()["remainder__with_std"] + + ct.set_params(trans1="passthrough") + exp = { + "n_jobs": None, + "remainder": ct.remainder, + "remainder__copy": True, + "remainder__with_mean": True, + "remainder__with_std": False, + "sparse_threshold": 0.3, + "trans1": "passthrough", + "transformers": ct.transformers, + "transformer_weights": None, + "verbose_feature_names_out": True, + "verbose": False, + "force_int_remainder_cols": "deprecated", + } assert ct.get_params() == exp def test_column_transformer_no_estimators(): - X_array = np.array([[0, 1, 2], - [2, 4, 6], - [8, 6, 4]]).astype('float').T + X_array = np.array([[0, 1, 2], [2, 4, 6], [8, 6, 4]]).astype("float").T ct = ColumnTransformer([], remainder=StandardScaler()) params = ct.get_params() - assert params['remainder__with_mean'] + assert params["remainder__with_mean"] X_trans = ct.fit_transform(X_array) assert X_trans.shape == X_array.shape assert len(ct.transformers_) == 1 - assert ct.transformers_[-1][0] == 'remainder' + assert ct.transformers_[-1][0] == "remainder" assert ct.transformers_[-1][2] == [0, 1, 2] @pytest.mark.parametrize( - ['est', 'pattern'], - [(ColumnTransformer([('trans1', Trans(), [0]), ('trans2', Trans(), [1])], - remainder=DoubleTrans()), - (r'\[ColumnTransformer\].*\(1 of 3\) Processing trans1.* total=.*\n' - r'\[ColumnTransformer\].*\(2 of 3\) Processing trans2.* total=.*\n' - r'\[ColumnTransformer\].*\(3 of 3\) Processing remainder.* total=.*\n$' - )), - (ColumnTransformer([('trans1', Trans(), [0]), ('trans2', Trans(), [1])], - remainder='passthrough'), - (r'\[ColumnTransformer\].*\(1 of 3\) Processing trans1.* total=.*\n' - r'\[ColumnTransformer\].*\(2 of 3\) Processing trans2.* total=.*\n' - r'\[ColumnTransformer\].*\(3 of 3\) Processing remainder.* total=.*\n$' - )), - (ColumnTransformer([('trans1', Trans(), [0]), ('trans2', 'drop', [1])], - remainder='passthrough'), - (r'\[ColumnTransformer\].*\(1 of 2\) Processing trans1.* total=.*\n' - r'\[ColumnTransformer\].*\(2 of 2\) Processing remainder.* total=.*\n$' - )), - (ColumnTransformer([('trans1', Trans(), [0]), - ('trans2', 'passthrough', [1])], - remainder='passthrough'), - (r'\[ColumnTransformer\].*\(1 of 3\) Processing trans1.* total=.*\n' - r'\[ColumnTransformer\].*\(2 of 3\) Processing trans2.* total=.*\n' - r'\[ColumnTransformer\].*\(3 of 3\) Processing remainder.* total=.*\n$' - )), - (ColumnTransformer([('trans1', Trans(), [0])], remainder='passthrough'), - (r'\[ColumnTransformer\].*\(1 of 2\) Processing trans1.* total=.*\n' - r'\[ColumnTransformer\].*\(2 of 2\) Processing remainder.* total=.*\n$' - )), - (ColumnTransformer([('trans1', Trans(), [0]), ('trans2', Trans(), [1])], - remainder='drop'), - (r'\[ColumnTransformer\].*\(1 of 2\) Processing trans1.* total=.*\n' - r'\[ColumnTransformer\].*\(2 of 2\) Processing trans2.* total=.*\n$')), - (ColumnTransformer([('trans1', Trans(), [0])], remainder='drop'), - (r'\[ColumnTransformer\].*\(1 of 1\) Processing trans1.* total=.*\n$'))]) -@pytest.mark.parametrize('method', ['fit', 'fit_transform']) + ["est", "pattern"], + [ + ( + ColumnTransformer( + [("trans1", Trans(), [0]), ("trans2", Trans(), [1])], + remainder=DoubleTrans(), + ), + ( + r"\[ColumnTransformer\].*\(1 of 3\) Processing trans1.* total=.*\n" + r"\[ColumnTransformer\].*\(2 of 3\) Processing trans2.* total=.*\n" + r"\[ColumnTransformer\].*\(3 of 3\) Processing remainder.* total=.*\n$" + ), + ), + ( + ColumnTransformer( + [("trans1", Trans(), [0]), ("trans2", Trans(), [1])], + remainder="passthrough", + ), + ( + r"\[ColumnTransformer\].*\(1 of 3\) Processing trans1.* total=.*\n" + r"\[ColumnTransformer\].*\(2 of 3\) Processing trans2.* total=.*\n" + r"\[ColumnTransformer\].*\(3 of 3\) Processing remainder.* total=.*\n$" + ), + ), + ( + ColumnTransformer( + [("trans1", Trans(), [0]), ("trans2", "drop", [1])], + remainder="passthrough", + ), + ( + r"\[ColumnTransformer\].*\(1 of 2\) Processing trans1.* total=.*\n" + r"\[ColumnTransformer\].*\(2 of 2\) Processing remainder.* total=.*\n$" + ), + ), + ( + ColumnTransformer( + [("trans1", Trans(), [0]), ("trans2", "passthrough", [1])], + remainder="passthrough", + ), + ( + r"\[ColumnTransformer\].*\(1 of 3\) Processing trans1.* total=.*\n" + r"\[ColumnTransformer\].*\(2 of 3\) Processing trans2.* total=.*\n" + r"\[ColumnTransformer\].*\(3 of 3\) Processing remainder.* total=.*\n$" + ), + ), + ( + ColumnTransformer([("trans1", Trans(), [0])], remainder="passthrough"), + ( + r"\[ColumnTransformer\].*\(1 of 2\) Processing trans1.* total=.*\n" + r"\[ColumnTransformer\].*\(2 of 2\) Processing remainder.* total=.*\n$" + ), + ), + ( + ColumnTransformer( + [("trans1", Trans(), [0]), ("trans2", Trans(), [1])], remainder="drop" + ), + ( + r"\[ColumnTransformer\].*\(1 of 2\) Processing trans1.* total=.*\n" + r"\[ColumnTransformer\].*\(2 of 2\) Processing trans2.* total=.*\n$" + ), + ), + ( + ColumnTransformer([("trans1", Trans(), [0])], remainder="drop"), + r"\[ColumnTransformer\].*\(1 of 1\) Processing trans1.* total=.*\n$", + ), + ], +) +@pytest.mark.parametrize("method", ["fit", "fit_transform"]) def test_column_transformer_verbose(est, pattern, method, capsys): X_array = np.array([[0, 1, 2], [2, 4, 6], [8, 6, 4]]).T func = getattr(est, method) est.set_params(verbose=False) func(X_array) - assert not capsys.readouterr().out, 'Got output for verbose=False' + assert not capsys.readouterr().out, "Got output for verbose=False" est.set_params(verbose=True) func(X_array) @@ -1125,8 +1308,7 @@ def func(X): assert_array_equal(X, X_array) return [0] - ct = ColumnTransformer([('trans', Trans(), func)], - remainder='drop') + ct = ColumnTransformer([("trans", Trans(), func)], remainder="drop") assert_array_equal(ct.fit_transform(X_array), X_res_first) assert_array_equal(ct.fit(X_array).transform(X_array), X_res_first) assert callable(ct.transformers[0][2]) @@ -1135,23 +1317,22 @@ def func(X): def test_column_transformer_callable_specifier_dataframe(): # assert that function gets the full dataframe - pd = pytest.importorskip('pandas') + pd = pytest.importorskip("pandas") X_array = np.array([[0, 1, 2], [2, 4, 6]]).T X_res_first = np.array([[0, 1, 2]]).T - X_df = pd.DataFrame(X_array, columns=['first', 'second']) + X_df = pd.DataFrame(X_array, columns=["first", "second"]) def func(X): assert_array_equal(X.columns, X_df.columns) assert_array_equal(X.values, X_df.values) - return ['first'] + return ["first"] - ct = ColumnTransformer([('trans', Trans(), func)], - remainder='drop') + ct = ColumnTransformer([("trans", Trans(), func)], remainder="drop") assert_array_equal(ct.fit_transform(X_df), X_res_first) assert_array_equal(ct.fit(X_df).transform(X_df), X_res_first) assert callable(ct.transformers[0][2]) - assert ct.transformers_[0][2] == ['first'] + assert ct.transformers_[0][2] == ["first"] def test_column_transformer_negative_column_indexes(): @@ -1161,96 +1342,19 @@ def test_column_transformer_negative_column_indexes(): ohe = OneHotEncoder() - tf_1 = ColumnTransformer([('ohe', ohe, [-1])], remainder='passthrough') - tf_2 = ColumnTransformer([('ohe', ohe, [2])], remainder='passthrough') + tf_1 = ColumnTransformer([("ohe", ohe, [-1])], remainder="passthrough") + tf_2 = ColumnTransformer([("ohe", ohe, [2])], remainder="passthrough") assert_array_equal(tf_1.fit_transform(X), tf_2.fit_transform(X)) -@pytest.mark.parametrize("explicit_colname", ['first', 'second']) -def test_column_transformer_reordered_column_names_remainder(explicit_colname): - """Regression test for issue #14223: 'Named col indexing fails with - ColumnTransformer remainder on changing DataFrame column ordering' - - Should raise error on changed order combined with remainder. - Should allow for added columns in `transform` input DataFrame - as long as all preceding columns match. - """ - pd = pytest.importorskip('pandas') - - X_fit_array = np.array([[0, 1, 2], [2, 4, 6]]).T - X_fit_df = pd.DataFrame(X_fit_array, columns=['first', 'second']) - - X_trans_array = np.array([[2, 4, 6], [0, 1, 2]]).T - X_trans_df = pd.DataFrame(X_trans_array, columns=['second', 'first']) - - tf = ColumnTransformer([('bycol', Trans(), explicit_colname)], - remainder=Trans()) - - tf.fit(X_fit_df) - err_msg = ("Given feature/column names do not match the ones for the " - "data given during fit.") - with pytest.raises(RuntimeError, match=err_msg): - tf.transform(X_trans_df) - - # ValueError for added columns - X_extended_df = X_fit_df.copy() - X_extended_df['third'] = [3, 6, 9] - err_msg = ("X has 3 features, but ColumnTransformer is expecting 2 " - "features as input.") - with pytest.raises(ValueError, match=err_msg): - tf.transform(X_extended_df) - - # No 'columns' AttributeError when transform input is a numpy array - X_array = X_fit_array.copy() - err_msg = 'Specifying the columns' - with pytest.raises(ValueError, match=err_msg): - tf.transform(X_array) - - -def test_feature_name_validation(): - """Tests if the proper warning/error is raised if the columns do not match - during fit and transform.""" - pd = pytest.importorskip("pandas") - - X = np.ones(shape=(3, 2)) - X_extra = np.ones(shape=(3, 3)) - df = pd.DataFrame(X, columns=['a', 'b']) - df_extra = pd.DataFrame(X_extra, columns=['a', 'b', 'c']) - - tf = ColumnTransformer([('bycol', Trans(), ['a', 'b'])]) - tf.fit(df) - - msg = ("X has 3 features, but ColumnTransformer is expecting 2 features " - "as input.") - with pytest.raises(ValueError, match=msg): - tf.transform(df_extra) - - tf = ColumnTransformer([('bycol', Trans(), [0])]) - tf.fit(df) - - with pytest.raises(ValueError, match=msg): - tf.transform(X_extra) - - with warnings.catch_warnings(record=True) as warns: - tf.transform(X) - assert not warns - - tf = ColumnTransformer([('bycol', Trans(), ['a'])], - remainder=Trans()) - tf.fit(df) - with pytest.raises(ValueError, match=msg): - tf.transform(df_extra) - - -@pytest.mark.parametrize("array_type", [np.asarray, sparse.csr_matrix]) +@pytest.mark.parametrize("array_type", [np.asarray, *CSR_CONTAINERS]) def test_column_transformer_mask_indexing(array_type): # Regression test for #14510 - # Boolean array-like does not behave as boolean array with NumPy < 1.12 - # and sparse matrices as well + # Boolean array-like does not behave as boolean array with sparse matrices. X = np.transpose([[1, 2, 3], [4, 5, 6], [5, 6, 7], [8, 9, 10]]) X = array_type(X) column_transformer = ColumnTransformer( - [('identity', FunctionTransformer(), [False, True, False, True])] + [("identity", FunctionTransformer(), [False, True, False, True])] ) X_trans = column_transformer.fit_transform(X) assert X_trans.shape == (3, 2) @@ -1261,65 +1365,73 @@ def test_n_features_in(): # transformer. X = [[1, 2], [3, 4], [5, 6]] - ct = ColumnTransformer([('a', DoubleTrans(), [0]), - ('b', DoubleTrans(), [1])]) - assert not hasattr(ct, 'n_features_in_') + ct = ColumnTransformer([("a", DoubleTrans(), [0]), ("b", DoubleTrans(), [1])]) + assert not hasattr(ct, "n_features_in_") ct.fit(X) assert ct.n_features_in_ == 2 -@pytest.mark.parametrize('cols, pattern, include, exclude', [ - (['col_int', 'col_float'], None, np.number, None), - (['col_int', 'col_float'], None, None, object), - (['col_int', 'col_float'], None, [int, float], None), - (['col_str'], None, [object], None), - (['col_str'], None, object, None), - (['col_float'], None, float, None), - (['col_float'], 'at$', [np.number], None), - (['col_int'], None, [int], None), - (['col_int'], '^col_int', [np.number], None), - (['col_float', 'col_str'], 'float|str', None, None), - (['col_str'], '^col_s', None, [int]), - ([], 'str$', float, None), - (['col_int', 'col_float', 'col_str'], None, [np.number, object], None), -]) -def test_make_column_selector_with_select_dtypes(cols, pattern, include, - exclude): - pd = pytest.importorskip('pandas') - - X_df = pd.DataFrame({ - 'col_int': np.array([0, 1, 2], dtype=int), - 'col_float': np.array([0.0, 1.0, 2.0], dtype=float), - 'col_str': ["one", "two", "three"], - }, columns=['col_int', 'col_float', 'col_str']) +@pytest.mark.parametrize( + "cols, pattern, include, exclude", + [ + (["col_int", "col_float"], None, np.number, None), + (["col_int", "col_float"], None, None, object), + (["col_int", "col_float"], None, [int, float], None), + (["col_str"], None, [object], None), + (["col_str"], None, object, None), + (["col_float"], None, float, None), + (["col_float"], "at$", [np.number], None), + (["col_int"], None, [int], None), + (["col_int"], "^col_int", [np.number], None), + (["col_float", "col_str"], "float|str", None, None), + (["col_str"], "^col_s", None, [int]), + ([], "str$", float, None), + (["col_int", "col_float", "col_str"], None, [np.number, object], None), + ], +) +def test_make_column_selector_with_select_dtypes(cols, pattern, include, exclude): + pd = pytest.importorskip("pandas") + + X_df = pd.DataFrame( + { + "col_int": np.array([0, 1, 2], dtype=int), + "col_float": np.array([0.0, 1.0, 2.0], dtype=float), + "col_str": ["one", "two", "three"], + }, + columns=["col_int", "col_float", "col_str"], + ) selector = make_column_selector( - dtype_include=include, dtype_exclude=exclude, pattern=pattern) + dtype_include=include, dtype_exclude=exclude, pattern=pattern + ) assert_array_equal(selector(X_df), cols) def test_column_transformer_with_make_column_selector(): # Functional test for column transformer + column selector - pd = pytest.importorskip('pandas') - X_df = pd.DataFrame({ - 'col_int': np.array([0, 1, 2], dtype=int), - 'col_float': np.array([0.0, 1.0, 2.0], dtype=float), - 'col_cat': ["one", "two", "one"], - 'col_str': ["low", "middle", "high"] - }, columns=['col_int', 'col_float', 'col_cat', 'col_str']) - X_df['col_str'] = X_df['col_str'].astype('category') - - cat_selector = make_column_selector(dtype_include=['category', object]) + pd = pytest.importorskip("pandas") + X_df = pd.DataFrame( + { + "col_int": np.array([0, 1, 2], dtype=int), + "col_float": np.array([0.0, 1.0, 2.0], dtype=float), + "col_cat": ["one", "two", "one"], + "col_str": ["low", "middle", "high"], + }, + columns=["col_int", "col_float", "col_cat", "col_str"], + ) + X_df["col_str"] = X_df["col_str"].astype("category") + + cat_selector = make_column_selector(dtype_include=["category", object]) num_selector = make_column_selector(dtype_include=np.number) ohe = OneHotEncoder() scaler = StandardScaler() - ct_selector = make_column_transformer((ohe, cat_selector), - (scaler, num_selector)) - ct_direct = make_column_transformer((ohe, ['col_cat', 'col_str']), - (scaler, ['col_float', 'col_int'])) + ct_selector = make_column_transformer((ohe, cat_selector), (scaler, num_selector)) + ct_direct = make_column_transformer( + (ohe, ["col_cat", "col_str"]), (scaler, ["col_float", "col_int"]) + ) X_selector = ct_selector.fit_transform(X_df) X_direct = ct_direct.fit_transform(X_df) @@ -1330,19 +1442,22 @@ def test_column_transformer_with_make_column_selector(): def test_make_column_selector_error(): selector = make_column_selector(dtype_include=np.number) X = np.array([[0.1, 0.2]]) - msg = ("make_column_selector can only be applied to pandas dataframes") + msg = "make_column_selector can only be applied to pandas dataframes" with pytest.raises(ValueError, match=msg): selector(X) def test_make_column_selector_pickle(): - pd = pytest.importorskip('pandas') + pd = pytest.importorskip("pandas") - X_df = pd.DataFrame({ - 'col_int': np.array([0, 1, 2], dtype=int), - 'col_float': np.array([0.0, 1.0, 2.0], dtype=float), - 'col_str': ["one", "two", "three"], - }, columns=['col_int', 'col_float', 'col_str']) + X_df = pd.DataFrame( + { + "col_int": np.array([0, 1, 2], dtype=int), + "col_float": np.array([0.0, 1.0, 2.0], dtype=float), + "col_str": ["one", "two", "three"], + }, + columns=["col_int", "col_float", "col_str"], + ) selector = make_column_selector(dtype_include=[object]) selector_picked = pickle.loads(pickle.dumps(selector)) @@ -1351,11 +1466,12 @@ def test_make_column_selector_pickle(): @pytest.mark.parametrize( - 'empty_col', [[], np.array([], dtype=int), lambda x: []], - ids=['list', 'array', 'callable'] + "empty_col", + [[], np.array([], dtype=int), lambda x: []], + ids=["list", "array", "callable"], ) def test_feature_names_empty_columns(empty_col): - pd = pytest.importorskip('pandas') + pd = pytest.importorskip("pandas") df = pd.DataFrame({"col1": ["a", "a", "b"], "col2": ["z", "z", "z"]}) @@ -1367,66 +1483,1293 @@ def test_feature_names_empty_columns(empty_col): ) ct.fit(df) - assert ct.get_feature_names() == ['ohe__x0_a', 'ohe__x0_b', 'ohe__x1_z'] + assert_array_equal( + ct.get_feature_names_out(), ["ohe__col1_a", "ohe__col1_b", "ohe__col2_z"] + ) + + +@pytest.mark.parametrize( + "selector", + [ + [1], + lambda x: [1], + ["col2"], + lambda x: ["col2"], + [False, True], + lambda x: [False, True], + ], +) +def test_feature_names_out_pandas(selector): + """Checks name when selecting only the second column""" + pd = pytest.importorskip("pandas") + df = pd.DataFrame({"col1": ["a", "a", "b"], "col2": ["z", "z", "z"]}) + ct = ColumnTransformer([("ohe", OneHotEncoder(), selector)]) + ct.fit(df) + + assert_array_equal(ct.get_feature_names_out(), ["ohe__col2_z"]) -@pytest.mark.parametrize('remainder', ["passthrough", StandardScaler()]) +@pytest.mark.parametrize( + "selector", [[1], lambda x: [1], [False, True], lambda x: [False, True]] +) +def test_feature_names_out_non_pandas(selector): + """Checks name when selecting the second column with numpy array""" + X = [["a", "z"], ["a", "z"], ["b", "z"]] + ct = ColumnTransformer([("ohe", OneHotEncoder(), selector)]) + ct.fit(X) + + assert_array_equal(ct.get_feature_names_out(), ["ohe__x1_z"]) + + +@pytest.mark.parametrize("remainder", ["passthrough", StandardScaler()]) def test_sk_visual_block_remainder(remainder): # remainder='passthrough' or an estimator will be shown in repr_html ohe = OneHotEncoder() - ct = ColumnTransformer(transformers=[('ohe', ohe, ["col1", "col2"])], - remainder=remainder) + ct = ColumnTransformer( + transformers=[("ohe", ohe, ["col1", "col2"])], remainder=remainder + ) visual_block = ct._sk_visual_block_() - assert visual_block.names == ('ohe', 'remainder') - assert visual_block.name_details == (['col1', 'col2'], '') + assert visual_block.names == ("ohe", "remainder") + assert visual_block.name_details == (["col1", "col2"], "") assert visual_block.estimators == (ohe, remainder) def test_sk_visual_block_remainder_drop(): # remainder='drop' is not shown in repr_html ohe = OneHotEncoder() - ct = ColumnTransformer(transformers=[('ohe', ohe, ["col1", "col2"])]) + ct = ColumnTransformer(transformers=[("ohe", ohe, ["col1", "col2"])]) visual_block = ct._sk_visual_block_() - assert visual_block.names == ('ohe',) - assert visual_block.name_details == (['col1', 'col2'],) + assert visual_block.names == ("ohe",) + assert visual_block.name_details == (["col1", "col2"],) assert visual_block.estimators == (ohe,) -@pytest.mark.parametrize('remainder', ["passthrough", StandardScaler()]) +@pytest.mark.parametrize("remainder", ["passthrough", StandardScaler()]) def test_sk_visual_block_remainder_fitted_pandas(remainder): # Remainder shows the columns after fitting - pd = pytest.importorskip('pandas') + pd = pytest.importorskip("pandas") ohe = OneHotEncoder() - ct = ColumnTransformer(transformers=[('ohe', ohe, ["col1", "col2"])], - remainder=remainder) - df = pd.DataFrame({"col1": ["a", "b", "c"], "col2": ["z", "z", "z"], - "col3": [1, 2, 3], "col4": [3, 4, 5]}) + ct = ColumnTransformer( + transformers=[("ohe", ohe, ["col1", "col2"])], + remainder=remainder, + ) + df = pd.DataFrame( + { + "col1": ["a", "b", "c"], + "col2": ["z", "z", "z"], + "col3": [1, 2, 3], + "col4": [3, 4, 5], + } + ) ct.fit(df) visual_block = ct._sk_visual_block_() - assert visual_block.names == ('ohe', 'remainder') - assert visual_block.name_details == (['col1', 'col2'], ['col3', 'col4']) + assert visual_block.names == ("ohe", "remainder") + assert visual_block.name_details == (["col1", "col2"], ["col3", "col4"]) assert visual_block.estimators == (ohe, remainder) -@pytest.mark.parametrize('remainder', ["passthrough", StandardScaler()]) +@pytest.mark.parametrize("remainder", ["passthrough", StandardScaler()]) def test_sk_visual_block_remainder_fitted_numpy(remainder): # Remainder shows the indices after fitting X = np.array([[1, 2, 3], [4, 5, 6]], dtype=float) scaler = StandardScaler() - ct = ColumnTransformer(transformers=[('scale', scaler, [0, 2])], - remainder=remainder) + ct = ColumnTransformer( + transformers=[("scale", scaler, [0, 2])], remainder=remainder + ) ct.fit(X) visual_block = ct._sk_visual_block_() - assert visual_block.names == ('scale', 'remainder') + assert visual_block.names == ("scale", "remainder") assert visual_block.name_details == ([0, 2], [1]) assert visual_block.estimators == (scaler, remainder) -@pytest.mark.parametrize("selector", [[], [False, False]]) -def test_get_feature_names_empty_selection(selector): - """Test that get_feature_names is only called for transformers that - were selected. Non-regression test for #19550. +@pytest.mark.parametrize("explicit_colname", ["first", "second", 0, 1]) +@pytest.mark.parametrize("remainder", [Trans(), "passthrough", "drop"]) +def test_column_transformer_reordered_column_names_remainder( + explicit_colname, remainder +): + """Test the interaction between remainder and column transformer""" + pd = pytest.importorskip("pandas") + + X_fit_array = np.array([[0, 1, 2], [2, 4, 6]]).T + X_fit_df = pd.DataFrame(X_fit_array, columns=["first", "second"]) + + X_trans_array = np.array([[2, 4, 6], [0, 1, 2]]).T + X_trans_df = pd.DataFrame(X_trans_array, columns=["second", "first"]) + + tf = ColumnTransformer([("bycol", Trans(), explicit_colname)], remainder=remainder) + + tf.fit(X_fit_df) + X_fit_trans = tf.transform(X_fit_df) + + # Changing the order still works + X_trans = tf.transform(X_trans_df) + assert_allclose(X_trans, X_fit_trans) + + # extra columns are ignored + X_extended_df = X_fit_df.copy() + X_extended_df["third"] = [3, 6, 9] + X_trans = tf.transform(X_extended_df) + assert_allclose(X_trans, X_fit_trans) + + if isinstance(explicit_colname, str): + # Raise error if columns are specified by names but input only allows + # to specify by position, e.g. numpy array instead of a pandas df. + X_array = X_fit_array.copy() + err_msg = "Specifying the columns" + with pytest.raises(ValueError, match=err_msg): + tf.transform(X_array) + + +def test_feature_name_validation_missing_columns_drop_passthough(): + """Test the interaction between {'drop', 'passthrough'} and + missing column names.""" + pd = pytest.importorskip("pandas") + + X = np.ones(shape=(3, 4)) + df = pd.DataFrame(X, columns=["a", "b", "c", "d"]) + + df_dropped = df.drop("c", axis=1) + + # with remainder='passthrough', all columns seen during `fit` must be + # present + tf = ColumnTransformer([("bycol", Trans(), [1])], remainder="passthrough") + tf.fit(df) + msg = r"columns are missing: {'c'}" + with pytest.raises(ValueError, match=msg): + tf.transform(df_dropped) + + # with remainder='drop', it is allowed to have column 'c' missing + tf = ColumnTransformer([("bycol", Trans(), [1])], remainder="drop") + tf.fit(df) + + df_dropped_trans = tf.transform(df_dropped) + df_fit_trans = tf.transform(df) + assert_allclose(df_dropped_trans, df_fit_trans) + + # bycol drops 'c', thus it is allowed for 'c' to be missing + tf = ColumnTransformer([("bycol", "drop", ["c"])], remainder="passthrough") + tf.fit(df) + df_dropped_trans = tf.transform(df_dropped) + df_fit_trans = tf.transform(df) + assert_allclose(df_dropped_trans, df_fit_trans) + + +def test_feature_names_in_(): + """Feature names are stored in column transformer. + + Column transformer deliberately does not check for column name consistency. + It only checks that the non-dropped names seen in `fit` are seen + in `transform`. This behavior is already tested in + `test_feature_name_validation_missing_columns_drop_passthough`""" + + pd = pytest.importorskip("pandas") + + feature_names = ["a", "c", "d"] + df = pd.DataFrame([[1, 2, 3]], columns=feature_names) + ct = ColumnTransformer([("bycol", Trans(), ["a", "d"])], remainder="passthrough") + + ct.fit(df) + assert_array_equal(ct.feature_names_in_, feature_names) + assert isinstance(ct.feature_names_in_, np.ndarray) + assert ct.feature_names_in_.dtype == object + + +class TransWithNames(Trans): + def __init__(self, feature_names_out=None): + self.feature_names_out = feature_names_out + + def get_feature_names_out(self, input_features=None): + if self.feature_names_out is not None: + return np.asarray(self.feature_names_out, dtype=object) + return input_features + + +@pytest.mark.parametrize( + "transformers, remainder, expected_names", + [ + ( + [ + ("bycol1", TransWithNames(), ["d", "c"]), + ("bycol2", "passthrough", ["d"]), + ], + "passthrough", + ["bycol1__d", "bycol1__c", "bycol2__d", "remainder__a", "remainder__b"], + ), + ( + [ + ("bycol1", TransWithNames(), ["d", "c"]), + ("bycol2", "passthrough", ["d"]), + ], + "drop", + ["bycol1__d", "bycol1__c", "bycol2__d"], + ), + ( + [ + ("bycol1", TransWithNames(), ["b"]), + ("bycol2", "drop", ["d"]), + ], + "passthrough", + ["bycol1__b", "remainder__a", "remainder__c"], + ), + ( + [ + ("bycol1", TransWithNames(["pca1", "pca2"]), ["a", "b", "d"]), + ], + "passthrough", + ["bycol1__pca1", "bycol1__pca2", "remainder__c"], + ), + ( + [ + ("bycol1", TransWithNames(["a", "b"]), ["d"]), + ("bycol2", "passthrough", ["b"]), + ], + "drop", + ["bycol1__a", "bycol1__b", "bycol2__b"], + ), + ( + [ + ("bycol1", TransWithNames([f"pca{i}" for i in range(2)]), ["b"]), + ("bycol2", TransWithNames([f"pca{i}" for i in range(2)]), ["b"]), + ], + "passthrough", + [ + "bycol1__pca0", + "bycol1__pca1", + "bycol2__pca0", + "bycol2__pca1", + "remainder__a", + "remainder__c", + "remainder__d", + ], + ), + ( + [ + ("bycol1", "drop", ["d"]), + ], + "drop", + [], + ), + ( + [ + ("bycol1", TransWithNames(), slice(1, 3)), + ], + "drop", + ["bycol1__b", "bycol1__c"], + ), + ( + [ + ("bycol1", TransWithNames(), ["b"]), + ("bycol2", "drop", slice(3, 4)), + ], + "passthrough", + ["bycol1__b", "remainder__a", "remainder__c"], + ), + ( + [ + ("bycol1", TransWithNames(), ["d", "c"]), + ("bycol2", "passthrough", slice(3, 4)), + ], + "passthrough", + ["bycol1__d", "bycol1__c", "bycol2__d", "remainder__a", "remainder__b"], + ), + ( + [ + ("bycol1", TransWithNames(), slice("b", "c")), + ], + "drop", + ["bycol1__b", "bycol1__c"], + ), + ( + [ + ("bycol1", TransWithNames(), ["b"]), + ("bycol2", "drop", slice("c", "d")), + ], + "passthrough", + ["bycol1__b", "remainder__a"], + ), + ( + [ + ("bycol1", TransWithNames(), ["d", "c"]), + ("bycol2", "passthrough", slice("c", "d")), + ], + "passthrough", + [ + "bycol1__d", + "bycol1__c", + "bycol2__c", + "bycol2__d", + "remainder__a", + "remainder__b", + ], + ), + ], +) +def test_verbose_feature_names_out_true(transformers, remainder, expected_names): + """Check feature_names_out for verbose_feature_names_out=True (default)""" + pd = pytest.importorskip("pandas") + df = pd.DataFrame([[1, 2, 3, 4]], columns=["a", "b", "c", "d"]) + ct = ColumnTransformer( + transformers, + remainder=remainder, + ) + ct.fit(df) + + names = ct.get_feature_names_out() + assert isinstance(names, np.ndarray) + assert names.dtype == object + assert_array_equal(names, expected_names) + + +def _feature_names_out_callable_name_clash(trans_name: str, feat_name: str): + return f"{trans_name[:2]}++{feat_name}" + + +def _feature_names_out_callable_upper(trans_name: str, feat_name: str): + return f"{trans_name.upper()}={feat_name.upper()}" + + +@pytest.mark.parametrize( + "transformers, remainder, verbose_feature_names_out, expected_names", + [ + ( + [ + ("bycol1", TransWithNames(), ["d", "c"]), + ("bycol2", "passthrough", ["d"]), + ], + "passthrough", + _feature_names_out_callable_name_clash, + ["by++d", "by++c", "by++d", "re++a", "re++b"], + ), + ( + [ + ("bycol1", TransWithNames(), ["d", "c"]), + ("bycol2", "passthrough", ["d"]), + ], + "drop", + "{feature_name}-{transformer_name}", + ["d-bycol1", "c-bycol1", "d-bycol2"], + ), + ( + [ + ("bycol1", TransWithNames(), ["d", "c"]), + ("bycol2", "passthrough", slice("c", "d")), + ], + "passthrough", + _feature_names_out_callable_upper, + [ + "BYCOL1=D", + "BYCOL1=C", + "BYCOL2=C", + "BYCOL2=D", + "REMAINDER=A", + "REMAINDER=B", + ], + ), + ], +) +def test_verbose_feature_names_out_callable_or_str( + transformers, remainder, verbose_feature_names_out, expected_names +): + """Check feature_names_out for verbose_feature_names_out=True (default)""" + pd = pytest.importorskip("pandas") + df = pd.DataFrame([[1, 2, 3, 4]], columns=["a", "b", "c", "d"]) + ct = ColumnTransformer( + transformers, + remainder=remainder, + verbose_feature_names_out=verbose_feature_names_out, + ) + ct.fit(df) + + names = ct.get_feature_names_out() + assert isinstance(names, np.ndarray) + assert names.dtype == object + assert_array_equal(names, expected_names) + + +@pytest.mark.parametrize( + "transformers, remainder, expected_names", + [ + ( + [ + ("bycol1", TransWithNames(), ["d", "c"]), + ("bycol2", "passthrough", ["a"]), + ], + "passthrough", + ["d", "c", "a", "b"], + ), + ( + [ + ("bycol1", TransWithNames(["a"]), ["d", "c"]), + ("bycol2", "passthrough", ["d"]), + ], + "drop", + ["a", "d"], + ), + ( + [ + ("bycol1", TransWithNames(), ["b"]), + ("bycol2", "drop", ["d"]), + ], + "passthrough", + ["b", "a", "c"], + ), + ( + [ + ("bycol1", TransWithNames(["pca1", "pca2"]), ["a", "b", "d"]), + ], + "passthrough", + ["pca1", "pca2", "c"], + ), + ( + [ + ("bycol1", TransWithNames(["a", "c"]), ["d"]), + ("bycol2", "passthrough", ["d"]), + ], + "drop", + ["a", "c", "d"], + ), + ( + [ + ("bycol1", TransWithNames([f"pca{i}" for i in range(2)]), ["b"]), + ("bycol2", TransWithNames([f"kpca{i}" for i in range(2)]), ["b"]), + ], + "passthrough", + ["pca0", "pca1", "kpca0", "kpca1", "a", "c", "d"], + ), + ( + [ + ("bycol1", "drop", ["d"]), + ], + "drop", + [], + ), + ( + [ + ("bycol1", TransWithNames(), slice(1, 2)), + ("bycol2", "drop", ["d"]), + ], + "passthrough", + ["b", "a", "c"], + ), + ( + [ + ("bycol1", TransWithNames(), ["b"]), + ("bycol2", "drop", slice(3, 4)), + ], + "passthrough", + ["b", "a", "c"], + ), + ( + [ + ("bycol1", TransWithNames(), ["d", "c"]), + ("bycol2", "passthrough", slice(0, 2)), + ], + "drop", + ["d", "c", "a", "b"], + ), + ( + [ + ("bycol1", TransWithNames(), slice("a", "b")), + ("bycol2", "drop", ["d"]), + ], + "passthrough", + ["a", "b", "c"], + ), + ( + [ + ("bycol1", TransWithNames(), ["b"]), + ("bycol2", "drop", slice("c", "d")), + ], + "passthrough", + ["b", "a"], + ), + ( + [ + ("bycol1", TransWithNames(), ["d", "c"]), + ("bycol2", "passthrough", slice("a", "b")), + ], + "drop", + ["d", "c", "a", "b"], + ), + ( + [ + ("bycol1", TransWithNames(), ["d", "c"]), + ("bycol2", "passthrough", slice("b", "b")), + ], + "drop", + ["d", "c", "b"], + ), + ], +) +def test_verbose_feature_names_out_false(transformers, remainder, expected_names): + """Check feature_names_out for verbose_feature_names_out=False""" + pd = pytest.importorskip("pandas") + df = pd.DataFrame([[1, 2, 3, 4]], columns=["a", "b", "c", "d"]) + ct = ColumnTransformer( + transformers, + remainder=remainder, + verbose_feature_names_out=False, + ) + ct.fit(df) + + names = ct.get_feature_names_out() + assert isinstance(names, np.ndarray) + assert names.dtype == object + assert_array_equal(names, expected_names) + + +@pytest.mark.parametrize( + "transformers, remainder, colliding_columns", + [ + ( + [ + ("bycol1", TransWithNames(), ["b"]), + ("bycol2", "passthrough", ["b"]), + ], + "drop", + "['b']", + ), + ( + [ + ("bycol1", TransWithNames(["c", "d"]), ["c"]), + ("bycol2", "passthrough", ["c"]), + ], + "drop", + "['c']", + ), + ( + [ + ("bycol1", TransWithNames(["a"]), ["b"]), + ("bycol2", "passthrough", ["b"]), + ], + "passthrough", + "['a']", + ), + ( + [ + ("bycol1", TransWithNames(["a"]), ["b"]), + ("bycol2", "drop", ["b"]), + ], + "passthrough", + "['a']", + ), + ( + [ + ("bycol1", TransWithNames(["c", "b"]), ["b"]), + ("bycol2", "passthrough", ["c", "b"]), + ], + "drop", + "['b', 'c']", + ), + ( + [ + ("bycol1", TransWithNames(["a"]), ["b"]), + ("bycol2", "passthrough", ["a"]), + ("bycol3", TransWithNames(["a"]), ["b"]), + ], + "passthrough", + "['a']", + ), + ( + [ + ("bycol1", TransWithNames(["a", "b"]), ["b"]), + ("bycol2", "passthrough", ["a"]), + ("bycol3", TransWithNames(["b"]), ["c"]), + ], + "passthrough", + "['a', 'b']", + ), + ( + [ + ("bycol1", TransWithNames([f"pca{i}" for i in range(6)]), ["b"]), + ("bycol2", TransWithNames([f"pca{i}" for i in range(6)]), ["b"]), + ], + "passthrough", + "['pca0', 'pca1', 'pca2', 'pca3', 'pca4', ...]", + ), + ( + [ + ("bycol1", TransWithNames(["a", "b"]), slice(1, 2)), + ("bycol2", "passthrough", ["a"]), + ("bycol3", TransWithNames(["b"]), ["c"]), + ], + "passthrough", + "['a', 'b']", + ), + ( + [ + ("bycol1", TransWithNames(["a", "b"]), ["b"]), + ("bycol2", "passthrough", slice(0, 1)), + ("bycol3", TransWithNames(["b"]), ["c"]), + ], + "passthrough", + "['a', 'b']", + ), + ( + [ + ("bycol1", TransWithNames(["a", "b"]), slice("b", "c")), + ("bycol2", "passthrough", ["a"]), + ("bycol3", TransWithNames(["b"]), ["c"]), + ], + "passthrough", + "['a', 'b']", + ), + ( + [ + ("bycol1", TransWithNames(["a", "b"]), ["b"]), + ("bycol2", "passthrough", slice("a", "a")), + ("bycol3", TransWithNames(["b"]), ["c"]), + ], + "passthrough", + "['a', 'b']", + ), + ], +) +def test_verbose_feature_names_out_false_errors( + transformers, remainder, colliding_columns +): + """Check feature_names_out for verbose_feature_names_out=False""" + + pd = pytest.importorskip("pandas") + df = pd.DataFrame([[1, 2, 3, 4]], columns=["a", "b", "c", "d"]) + ct = ColumnTransformer( + transformers, + remainder=remainder, + verbose_feature_names_out=False, + ) + ct.fit(df) + + msg = re.escape( + f"Output feature names: {colliding_columns} are not unique. Please set " + "verbose_feature_names_out=True to add prefixes to feature names" + ) + with pytest.raises(ValueError, match=msg): + ct.get_feature_names_out() + + +@pytest.mark.parametrize("verbose_feature_names_out", [True, False]) +@pytest.mark.parametrize("remainder", ["drop", "passthrough"]) +def test_column_transformer_set_output(verbose_feature_names_out, remainder): + """Check column transformer behavior with set_output.""" + pd = pytest.importorskip("pandas") + df = pd.DataFrame([[1, 2, 3, 4]], columns=["a", "b", "c", "d"], index=[10]) + ct = ColumnTransformer( + [("first", TransWithNames(), ["a", "c"]), ("second", TransWithNames(), ["d"])], + remainder=remainder, + verbose_feature_names_out=verbose_feature_names_out, + ) + X_trans = ct.fit_transform(df) + assert isinstance(X_trans, np.ndarray) + + ct.set_output(transform="pandas") + + df_test = pd.DataFrame([[1, 2, 3, 4]], columns=df.columns, index=[20]) + X_trans = ct.transform(df_test) + assert isinstance(X_trans, pd.DataFrame) + + feature_names_out = ct.get_feature_names_out() + assert_array_equal(X_trans.columns, feature_names_out) + assert_array_equal(X_trans.index, df_test.index) + + +@pytest.mark.parametrize("remainder", ["drop", "passthrough"]) +@pytest.mark.parametrize("fit_transform", [True, False]) +def test_column_transform_set_output_mixed(remainder, fit_transform): + """Check ColumnTransformer outputs mixed types correctly.""" + pd = pytest.importorskip("pandas") + df = pd.DataFrame( + { + "pet": pd.Series(["dog", "cat", "snake"], dtype="category"), + "color": pd.Series(["green", "blue", "red"], dtype="object"), + "age": [1.4, 2.1, 4.4], + "height": [20, 40, 10], + "distance": pd.Series([20, pd.NA, 100], dtype="Int32"), + } + ) + ct = ColumnTransformer( + [ + ( + "color_encode", + OneHotEncoder(sparse_output=False, dtype="int8"), + ["color"], + ), + ("age", StandardScaler(), ["age"]), + ], + remainder=remainder, + verbose_feature_names_out=False, + ).set_output(transform="pandas") + if fit_transform: + X_trans = ct.fit_transform(df) + else: + X_trans = ct.fit(df).transform(df) + + assert isinstance(X_trans, pd.DataFrame) + assert_array_equal(X_trans.columns, ct.get_feature_names_out()) + + expected_dtypes = { + "color_blue": "int8", + "color_green": "int8", + "color_red": "int8", + "age": "float64", + "pet": "category", + "height": "int64", + "distance": "Int32", + } + for col, dtype in X_trans.dtypes.items(): + assert dtype == expected_dtypes[col] + + +@pytest.mark.parametrize("remainder", ["drop", "passthrough"]) +def test_column_transform_set_output_after_fitting(remainder): + pd = pytest.importorskip("pandas") + df = pd.DataFrame( + { + "pet": pd.Series(["dog", "cat", "snake"], dtype="category"), + "age": [1.4, 2.1, 4.4], + "height": [20, 40, 10], + } + ) + ct = ColumnTransformer( + [ + ( + "color_encode", + OneHotEncoder(sparse_output=False, dtype="int16"), + ["pet"], + ), + ("age", StandardScaler(), ["age"]), + ], + remainder=remainder, + verbose_feature_names_out=False, + ) + + # fit without calling set_output + X_trans = ct.fit_transform(df) + assert isinstance(X_trans, np.ndarray) + assert X_trans.dtype == "float64" + + ct.set_output(transform="pandas") + X_trans_df = ct.transform(df) + expected_dtypes = { + "pet_cat": "int16", + "pet_dog": "int16", + "pet_snake": "int16", + "height": "int64", + "age": "float64", + } + for col, dtype in X_trans_df.dtypes.items(): + assert dtype == expected_dtypes[col] + + +# PandasOutTransformer that does not define get_feature_names_out and always expects +# the input to be a DataFrame. +class PandasOutTransformer(BaseEstimator): + def __init__(self, offset=1.0): + self.offset = offset + + def fit(self, X, y=None): + pd = pytest.importorskip("pandas") + assert isinstance(X, pd.DataFrame) + return self + + def transform(self, X, y=None): + pd = pytest.importorskip("pandas") + assert isinstance(X, pd.DataFrame) + return X - self.offset + + def set_output(self, transform=None): + # This transformer will always output a DataFrame regardless of the + # configuration. + return self + + +@pytest.mark.parametrize( + "trans_1, expected_verbose_names, expected_non_verbose_names", + [ + ( + PandasOutTransformer(offset=2.0), + ["trans_0__feat1", "trans_1__feat0"], + ["feat1", "feat0"], + ), + ( + "drop", + ["trans_0__feat1"], + ["feat1"], + ), + ( + "passthrough", + ["trans_0__feat1", "trans_1__feat0"], + ["feat1", "feat0"], + ), + ], +) +def test_transformers_with_pandas_out_but_not_feature_names_out( + trans_1, expected_verbose_names, expected_non_verbose_names +): + """Check that set_config(transform="pandas") is compatible with more transformers. + + Specifically, if transformers returns a DataFrame, but does not define + `get_feature_names_out`. + """ + pd = pytest.importorskip("pandas") + + X_df = pd.DataFrame({"feat0": [1.0, 2.0, 3.0], "feat1": [2.0, 3.0, 4.0]}) + ct = ColumnTransformer( + [ + ("trans_0", PandasOutTransformer(offset=3.0), ["feat1"]), + ("trans_1", trans_1, ["feat0"]), + ] + ) + X_trans_np = ct.fit_transform(X_df) + assert isinstance(X_trans_np, np.ndarray) + + # `ct` does not have `get_feature_names_out` because `PandasOutTransformer` does + # not define the method. + with pytest.raises(AttributeError, match="not provide get_feature_names_out"): + ct.get_feature_names_out() + + # The feature names are prefixed because verbose_feature_names_out=True is default + ct.set_output(transform="pandas") + X_trans_df0 = ct.fit_transform(X_df) + assert_array_equal(X_trans_df0.columns, expected_verbose_names) + + ct.set_params(verbose_feature_names_out=False) + X_trans_df1 = ct.fit_transform(X_df) + assert_array_equal(X_trans_df1.columns, expected_non_verbose_names) + + +@pytest.mark.parametrize( + "empty_selection", + [[], np.array([False, False]), [False, False]], + ids=["list", "bool", "bool_int"], +) +def test_empty_selection_pandas_output(empty_selection): + """Check that pandas output works when there is an empty selection. + + Non-regression test for gh-25487 + """ + pd = pytest.importorskip("pandas") + + X = pd.DataFrame([[1.0, 2.2], [3.0, 1.0]], columns=["a", "b"]) + ct = ColumnTransformer( + [ + ("categorical", "passthrough", empty_selection), + ("numerical", StandardScaler(), ["a", "b"]), + ], + verbose_feature_names_out=True, + ) + ct.set_output(transform="pandas") + X_out = ct.fit_transform(X) + assert_array_equal(X_out.columns, ["numerical__a", "numerical__b"]) + + ct.set_params(verbose_feature_names_out=False) + X_out = ct.fit_transform(X) + assert_array_equal(X_out.columns, ["a", "b"]) + + +def test_raise_error_if_index_not_aligned(): + """Check column transformer raises error if indices are not aligned. + + Non-regression test for gh-26210. + """ + pd = pytest.importorskip("pandas") + + X = pd.DataFrame([[1.0, 2.2], [3.0, 1.0]], columns=["a", "b"], index=[8, 3]) + reset_index_transformer = FunctionTransformer( + lambda x: x.reset_index(drop=True), feature_names_out="one-to-one" + ) + + ct = ColumnTransformer( + [ + ("num1", "passthrough", ["a"]), + ("num2", reset_index_transformer, ["b"]), + ], + ) + ct.set_output(transform="pandas") + msg = ( + "Concatenating DataFrames from the transformer's output lead to" + " an inconsistent number of samples. The output may have Pandas" + " Indexes that do not match." + ) + with pytest.raises(ValueError, match=msg): + ct.fit_transform(X) + + +def test_remainder_set_output(): + """Check that the output is set for the remainder. + + Non-regression test for #26306. + """ + + pd = pytest.importorskip("pandas") + df = pd.DataFrame({"a": [True, False, True], "b": [1, 2, 3]}) + + ct = make_column_transformer( + (VarianceThreshold(), make_column_selector(dtype_include=bool)), + remainder=VarianceThreshold(), + verbose_feature_names_out=False, + ) + ct.set_output(transform="pandas") + + out = ct.fit_transform(df) + pd.testing.assert_frame_equal(out, df) + + ct.set_output(transform="default") + out = ct.fit_transform(df) + assert isinstance(out, np.ndarray) + + +def test_transform_pd_na(): + """Check behavior when a tranformer's output contains pandas.NA + + It should raise an error unless the output config is set to 'pandas'. + """ + pd = pytest.importorskip("pandas") + if not hasattr(pd, "Float64Dtype"): + pytest.skip( + "The issue with pd.NA tested here does not happen in old versions that do" + " not have the extension dtypes" + ) + df = pd.DataFrame({"a": [1.5, None]}) + ct = make_column_transformer(("passthrough", ["a"])) + # No warning with non-extension dtypes and np.nan + with warnings.catch_warnings(): + warnings.simplefilter("error") + ct.fit_transform(df) + df = df.convert_dtypes() + + # Error with extension dtype and pd.NA + with pytest.raises(ValueError, match=r"set_output\(transform='pandas'\)"): + ct.fit_transform(df) + + # No error when output is set to pandas + ct.set_output(transform="pandas") + ct.fit_transform(df) + ct.set_output(transform="default") + + # No error when there are no pd.NA + ct.fit_transform(df.fillna(-1.0)) + + +def test_dataframe_different_dataframe_libraries(): + """Check fitting and transforming on pandas and polars dataframes.""" + pd = pytest.importorskip("pandas") + pl = pytest.importorskip("polars") + X_train_np = np.array([[0, 1], [2, 4], [4, 5]]) + X_test_np = np.array([[1, 2], [1, 3], [2, 3]]) + + # Fit on pandas and transform on polars + X_train_pd = pd.DataFrame(X_train_np, columns=["a", "b"]) + X_test_pl = pl.DataFrame(X_test_np, schema=["a", "b"]) + + ct = make_column_transformer((Trans(), [0, 1])) + ct.fit(X_train_pd) + + out_pl_in = ct.transform(X_test_pl) + assert_array_equal(out_pl_in, X_test_np) + + # Fit on polars and transform on pandas + X_train_pl = pl.DataFrame(X_train_np, schema=["a", "b"]) + X_test_pd = pd.DataFrame(X_test_np, columns=["a", "b"]) + ct.fit(X_train_pl) + + out_pd_in = ct.transform(X_test_pd) + assert_array_equal(out_pd_in, X_test_np) + + +def test_column_transformer__getitem__(): + """Check __getitem__ for ColumnTransformer.""" + X = np.array([[0, 1, 2], [3, 4, 5]]) + ct = ColumnTransformer([("t1", Trans(), [0, 1]), ("t2", Trans(), [1, 2])]) + + msg = "ColumnTransformer is subscriptable after it is fitted" + with pytest.raises(TypeError, match=msg): + ct["t1"] + + ct.fit(X) + assert ct["t1"] is ct.named_transformers_["t1"] + assert ct["t2"] is ct.named_transformers_["t2"] + + msg = "'does_not_exist' is not a valid transformer name" + with pytest.raises(KeyError, match=msg): + ct["does_not_exist"] + + +@pytest.mark.parametrize("transform_output", ["default", "pandas"]) +def test_column_transformer_remainder_passthrough_naming_consistency(transform_output): + """Check that when `remainder="passthrough"`, inconsistent naming is handled + correctly by the underlying `FunctionTransformer`. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/28232 + """ + pd = pytest.importorskip("pandas") + X = pd.DataFrame(np.random.randn(10, 4)) + + preprocessor = ColumnTransformer( + transformers=[("scaler", StandardScaler(), [0, 1])], + remainder="passthrough", + ).set_output(transform=transform_output) + X_trans = preprocessor.fit_transform(X) + assert X_trans.shape == X.shape + + expected_column_names = [ + "scaler__x0", + "scaler__x1", + "remainder__x2", + "remainder__x3", + ] + if hasattr(X_trans, "columns"): + assert X_trans.columns.tolist() == expected_column_names + assert preprocessor.get_feature_names_out().tolist() == expected_column_names + + +@pytest.mark.parametrize("dataframe_lib", ["pandas", "polars"]) +def test_column_transformer_column_renaming(dataframe_lib): + """Check that we properly rename columns when using `ColumnTransformer` and + selected columns are redundant between transformers. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/28260 + """ + lib = pytest.importorskip(dataframe_lib) + + df = lib.DataFrame({"x1": [1, 2, 3], "x2": [10, 20, 30], "x3": [100, 200, 300]}) + + transformer = ColumnTransformer( + transformers=[ + ("A", "passthrough", ["x1", "x2", "x3"]), + ("B", FunctionTransformer(), ["x1", "x2"]), + ("C", StandardScaler(), ["x1", "x3"]), + # special case of a transformer returning 0-columns, e.g feature selector + ( + "D", + FunctionTransformer(lambda x: _safe_indexing(x, [], axis=1)), + ["x1", "x2", "x3"], + ), + ], + verbose_feature_names_out=True, + ).set_output(transform=dataframe_lib) + df_trans = transformer.fit_transform(df) + assert list(df_trans.columns) == [ + "A__x1", + "A__x2", + "A__x3", + "B__x1", + "B__x2", + "C__x1", + "C__x3", + ] + + +@pytest.mark.parametrize("dataframe_lib", ["pandas", "polars"]) +def test_column_transformer_error_with_duplicated_columns(dataframe_lib): + """Check that we raise an error when using `ColumnTransformer` and + the columns names are duplicated between transformers.""" + lib = pytest.importorskip(dataframe_lib) + + df = lib.DataFrame({"x1": [1, 2, 3], "x2": [10, 20, 30], "x3": [100, 200, 300]}) + + transformer = ColumnTransformer( + transformers=[ + ("A", "passthrough", ["x1", "x2", "x3"]), + ("B", FunctionTransformer(), ["x1", "x2"]), + ("C", StandardScaler(), ["x1", "x3"]), + # special case of a transformer returning 0-columns, e.g feature selector + ( + "D", + FunctionTransformer(lambda x: _safe_indexing(x, [], axis=1)), + ["x1", "x2", "x3"], + ), + ], + verbose_feature_names_out=False, + ).set_output(transform=dataframe_lib) + err_msg = re.escape( + "Duplicated feature names found before concatenating the outputs of the " + "transformers: ['x1', 'x2', 'x3'].\n" + "Transformer A has conflicting columns names: ['x1', 'x2', 'x3'].\n" + "Transformer B has conflicting columns names: ['x1', 'x2'].\n" + "Transformer C has conflicting columns names: ['x1', 'x3'].\n" + ) + with pytest.raises(ValueError, match=err_msg): + transformer.fit_transform(df) + + +@pytest.mark.skipif( + parse_version(joblib.__version__) < parse_version("1.3"), + reason="requires joblib >= 1.3", +) +def test_column_transformer_auto_memmap(): + """Check that ColumnTransformer works in parallel with joblib's auto-memmapping. + + non-regression test for issue #28781 """ - ct = ColumnTransformer([('ohe', OneHotEncoder(drop='first'), selector)]) - ct.fit([[1, 2], [3, 4]]) - assert ct.get_feature_names() == [] + X = np.random.RandomState(0).uniform(size=(3, 4)) + + scaler = StandardScaler(copy=False) + + transformer = ColumnTransformer( + transformers=[("scaler", scaler, [0])], + n_jobs=2, + ) + + with joblib.parallel_backend("loky", max_nbytes=1): + Xt = transformer.fit_transform(X) + + assert_allclose(Xt, StandardScaler().fit_transform(X[:, [0]])) + + +# Metadata Routing Tests +# ====================== + + +@pytest.mark.parametrize("method", ["transform", "fit_transform", "fit"]) +def test_routing_passed_metadata_not_supported(method): + """Test that the right error message is raised when metadata is passed while + not supported when `enable_metadata_routing=False`.""" + + X = np.array([[0, 1, 2], [2, 4, 6]]).T + y = [1, 2, 3] + trs = ColumnTransformer([("trans", Trans(), [0])]).fit(X, y) + + with pytest.raises( + ValueError, match="is only supported if enable_metadata_routing=True" + ): + getattr(trs, method)([[1]], sample_weight=[1], prop="a") + + +@pytest.mark.parametrize("method", ["transform", "fit_transform", "fit"]) +@config_context(enable_metadata_routing=True) +def test_metadata_routing_for_column_transformer(method): + """Test that metadata is routed correctly for column transformer.""" + X = np.array([[0, 1, 2], [2, 4, 6]]).T + y = [1, 2, 3] + registry = _Registry() + sample_weight, metadata = [1], "a" + trs = ColumnTransformer( + [ + ( + "trans", + ConsumingTransformer(registry=registry) + .set_fit_request(sample_weight=True, metadata=True) + .set_transform_request(sample_weight=True, metadata=True), + [0], + ) + ] + ) + + if method == "transform": + trs.fit(X, y, sample_weight=sample_weight, metadata=metadata) + trs.transform(X, sample_weight=sample_weight, metadata=metadata) + else: + getattr(trs, method)(X, y, sample_weight=sample_weight, metadata=metadata) + + assert len(registry) + for _trs in registry: + check_recorded_metadata( + obj=_trs, + method=method, + parent=method, + sample_weight=sample_weight, + metadata=metadata, + ) + + +@config_context(enable_metadata_routing=True) +def test_metadata_routing_no_fit_transform(): + """Test metadata routing when the sub-estimator doesn't implement + ``fit_transform``.""" + + class NoFitTransform(BaseEstimator): + def fit(self, X, y=None, sample_weight=None, metadata=None): + assert sample_weight + assert metadata + return self + + def transform(self, X, sample_weight=None, metadata=None): + assert sample_weight + assert metadata + return X + + X = np.array([[0, 1, 2], [2, 4, 6]]).T + y = [1, 2, 3] + sample_weight, metadata = [1], "a" + trs = ColumnTransformer( + [ + ( + "trans", + NoFitTransform() + .set_fit_request(sample_weight=True, metadata=True) + .set_transform_request(sample_weight=True, metadata=True), + [0], + ) + ] + ) + + trs.fit(X, y, sample_weight=sample_weight, metadata=metadata) + trs.fit_transform(X, y, sample_weight=sample_weight, metadata=metadata) + + +@pytest.mark.parametrize("method", ["transform", "fit_transform", "fit"]) +@config_context(enable_metadata_routing=True) +def test_metadata_routing_error_for_column_transformer(method): + """Test that the right error is raised when metadata is not requested.""" + X = np.array([[0, 1, 2], [2, 4, 6]]).T + y = [1, 2, 3] + sample_weight, metadata = [1], "a" + trs = ColumnTransformer([("trans", ConsumingTransformer(), [0])]) + + error_message = ( + "[sample_weight, metadata] are passed but are not explicitly set as requested" + f" or not requested for ConsumingTransformer.{method}" + ) + with pytest.raises(ValueError, match=re.escape(error_message)): + if method == "transform": + trs.fit(X, y) + trs.transform(X, sample_weight=sample_weight, metadata=metadata) + else: + getattr(trs, method)(X, y, sample_weight=sample_weight, metadata=metadata) + + +@config_context(enable_metadata_routing=True) +def test_get_metadata_routing_works_without_fit(): + # Regression test for https://github.com/scikit-learn/scikit-learn/issues/28186 + # Make sure ct.get_metadata_routing() works w/o having called fit. + ct = ColumnTransformer([("trans", ConsumingTransformer(), [0])]) + ct.get_metadata_routing() + + +@config_context(enable_metadata_routing=True) +def test_remainder_request_always_present(): + # Test that remainder request is always present. + ct = ColumnTransformer( + [("trans", StandardScaler(), [0])], + remainder=ConsumingTransformer() + .set_fit_request(metadata=True) + .set_transform_request(metadata=True), + ) + router = ct.get_metadata_routing() + assert router.consumes("fit", ["metadata"]) == set(["metadata"]) + + +@config_context(enable_metadata_routing=True) +def test_unused_transformer_request_present(): + # Test that the request of a transformer is always present even when not + # used due to no selected columns. + ct = ColumnTransformer( + [ + ( + "trans", + ConsumingTransformer() + .set_fit_request(metadata=True) + .set_transform_request(metadata=True), + lambda X: [], + ) + ] + ) + router = ct.get_metadata_routing() + assert router.consumes("fit", ["metadata"]) == set(["metadata"]) + + +# End of Metadata Routing Tests +# ============================= diff --git a/sklearn/compose/tests/test_target.py b/sklearn/compose/tests/test_target.py index dc5d8d95743ef..e65b950f04007 100644 --- a/sklearn/compose/tests/test_target.py +++ b/sklearn/compose/tests/test_target.py @@ -1,25 +1,16 @@ +import warnings + import numpy as np import pytest -from sklearn.base import clone -from sklearn.base import BaseEstimator -from sklearn.base import TransformerMixin - +from sklearn import config_context, datasets +from sklearn.base import BaseEstimator, TransformerMixin, clone +from sklearn.compose import TransformedTargetRegressor from sklearn.dummy import DummyRegressor - -from sklearn.utils._testing import assert_allclose -from sklearn.utils._testing import assert_no_warnings - -from sklearn.preprocessing import FunctionTransformer -from sklearn.preprocessing import StandardScaler - -from sklearn.pipeline import Pipeline - from sklearn.linear_model import LinearRegression, OrthogonalMatchingPursuit - -from sklearn import datasets - -from sklearn.compose import TransformedTargetRegressor +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import FunctionTransformer, StandardScaler +from sklearn.utils._testing import assert_allclose friedman = datasets.make_friedman1(random_state=0) @@ -27,39 +18,65 @@ def test_transform_target_regressor_error(): X, y = friedman # provide a transformer and functions at the same time - regr = TransformedTargetRegressor(regressor=LinearRegression(), - transformer=StandardScaler(), - func=np.exp, inverse_func=np.log) - with pytest.raises(ValueError, - match="'transformer' and functions" - " 'func'/'inverse_func' cannot both be set."): + regr = TransformedTargetRegressor( + regressor=LinearRegression(), + transformer=StandardScaler(), + func=np.exp, + inverse_func=np.log, + ) + with pytest.raises( + ValueError, + match="'transformer' and functions 'func'/'inverse_func' cannot both be set.", + ): regr.fit(X, y) # fit with sample_weight with a regressor which does not support it sample_weight = np.ones((y.shape[0],)) - regr = TransformedTargetRegressor(regressor=OrthogonalMatchingPursuit(), - transformer=StandardScaler()) - with pytest.raises(TypeError, match=r"fit\(\) got an unexpected " - "keyword argument 'sample_weight'"): + regr = TransformedTargetRegressor( + regressor=OrthogonalMatchingPursuit(), transformer=StandardScaler() + ) + with pytest.raises( + TypeError, + match=r"fit\(\) got an unexpected keyword argument 'sample_weight'", + ): regr.fit(X, y, sample_weight=sample_weight) - # func is given but inverse_func is not + + # one of (func, inverse_func) is given but the other one is not regr = TransformedTargetRegressor(func=np.exp) - with pytest.raises(ValueError, match="When 'func' is provided, " - "'inverse_func' must also be provided"): + with pytest.raises( + ValueError, + match="When 'func' is provided, 'inverse_func' must also be provided", + ): + regr.fit(X, y) + + regr = TransformedTargetRegressor(inverse_func=np.log) + with pytest.raises( + ValueError, + match="When 'inverse_func' is provided, 'func' must also be provided", + ): regr.fit(X, y) def test_transform_target_regressor_invertible(): X, y = friedman - regr = TransformedTargetRegressor(regressor=LinearRegression(), - func=np.sqrt, inverse_func=np.log, - check_inverse=True) - with pytest.warns(UserWarning, match="The provided functions or" - " transformer are not strictly inverse of each other."): + regr = TransformedTargetRegressor( + regressor=LinearRegression(), + func=np.sqrt, + inverse_func=np.log, + check_inverse=True, + ) + with pytest.warns( + UserWarning, + match=(r"The provided functions.* are not strictly inverse of each other"), + ): regr.fit(X, y) - regr = TransformedTargetRegressor(regressor=LinearRegression(), - func=np.sqrt, inverse_func=np.log) + regr = TransformedTargetRegressor( + regressor=LinearRegression(), func=np.sqrt, inverse_func=np.log + ) regr.set_params(check_inverse=False) - assert_no_warnings(regr.fit, X, y) + + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + regr.fit(X, y) def _check_standard_scaled(y, y_pred): @@ -74,14 +91,16 @@ def _check_shifted_by_one(y, y_pred): def test_transform_target_regressor_functions(): X, y = friedman - regr = TransformedTargetRegressor(regressor=LinearRegression(), - func=np.log, inverse_func=np.exp) + regr = TransformedTargetRegressor( + regressor=LinearRegression(), func=np.log, inverse_func=np.exp + ) y_pred = regr.fit(X, y).predict(X) # check the transformer output y_tran = regr.transformer_.transform(y.reshape(-1, 1)).squeeze() assert_allclose(np.log(y), y_tran) - assert_allclose(y, regr.transformer_.inverse_transform( - y_tran.reshape(-1, 1)).squeeze()) + assert_allclose( + y, regr.transformer_.inverse_transform(y_tran.reshape(-1, 1)).squeeze() + ) assert y.shape == y_pred.shape assert_allclose(y_pred, regr.inverse_func(regr.regressor_.predict(X))) # check the regressor output @@ -92,8 +111,9 @@ def test_transform_target_regressor_functions(): def test_transform_target_regressor_functions_multioutput(): X = friedman[0] y = np.vstack((friedman[1], friedman[1] ** 2 + 1)).T - regr = TransformedTargetRegressor(regressor=LinearRegression(), - func=np.log, inverse_func=np.exp) + regr = TransformedTargetRegressor( + regressor=LinearRegression(), func=np.log, inverse_func=np.exp + ) y_pred = regr.fit(X, y).predict(X) # check the transformer output y_tran = regr.transformer_.transform(y) @@ -106,19 +126,20 @@ def test_transform_target_regressor_functions_multioutput(): assert_allclose(regr.regressor_.coef_.ravel(), lr.coef_.ravel()) -@pytest.mark.parametrize("X,y", [friedman, - (friedman[0], - np.vstack((friedman[1], - friedman[1] ** 2 + 1)).T)]) +@pytest.mark.parametrize( + "X,y", [friedman, (friedman[0], np.vstack((friedman[1], friedman[1] ** 2 + 1)).T)] +) def test_transform_target_regressor_1d_transformer(X, y): # All transformer in scikit-learn expect 2D data. FunctionTransformer with # validate=False lift this constraint without checking that the input is a # 2D vector. We check the consistency of the data shape using a 1D and 2D y # array. - transformer = FunctionTransformer(func=lambda x: x + 1, - inverse_func=lambda x: x - 1) - regr = TransformedTargetRegressor(regressor=LinearRegression(), - transformer=transformer) + transformer = FunctionTransformer( + func=lambda x: x + 1, inverse_func=lambda x: x - 1 + ) + regr = TransformedTargetRegressor( + regressor=LinearRegression(), transformer=transformer + ) y_pred = regr.fit(X, y).predict(X) assert y.shape == y_pred.shape # consistency forward transform @@ -126,8 +147,7 @@ def test_transform_target_regressor_1d_transformer(X, y): _check_shifted_by_one(y, y_tran) assert y.shape == y_pred.shape # consistency inverse transform - assert_allclose(y, regr.transformer_.inverse_transform( - y_tran).squeeze()) + assert_allclose(y, regr.transformer_.inverse_transform(y_tran).squeeze()) # consistency of the regressor lr = LinearRegression() transformer2 = clone(transformer) @@ -137,37 +157,40 @@ def test_transform_target_regressor_1d_transformer(X, y): assert_allclose(regr.regressor_.coef_, lr.coef_) -@pytest.mark.parametrize("X,y", [friedman, - (friedman[0], - np.vstack((friedman[1], - friedman[1] ** 2 + 1)).T)]) +@pytest.mark.parametrize( + "X,y", [friedman, (friedman[0], np.vstack((friedman[1], friedman[1] ** 2 + 1)).T)] +) def test_transform_target_regressor_2d_transformer(X, y): # Check consistency with transformer accepting only 2D array and a 1D/2D y # array. transformer = StandardScaler() - regr = TransformedTargetRegressor(regressor=LinearRegression(), - transformer=transformer) + regr = TransformedTargetRegressor( + regressor=LinearRegression(), transformer=transformer + ) y_pred = regr.fit(X, y).predict(X) assert y.shape == y_pred.shape # consistency forward transform if y.ndim == 1: # create a 2D array and squeeze results - y_tran = regr.transformer_.transform(y.reshape(-1, 1)).squeeze() + y_tran = regr.transformer_.transform(y.reshape(-1, 1)) else: y_tran = regr.transformer_.transform(y) - _check_standard_scaled(y, y_tran) + _check_standard_scaled(y, y_tran.squeeze()) assert y.shape == y_pred.shape # consistency inverse transform - assert_allclose(y, regr.transformer_.inverse_transform( - y_tran).squeeze()) + assert_allclose(y, regr.transformer_.inverse_transform(y_tran).squeeze()) # consistency of the regressor lr = LinearRegression() transformer2 = clone(transformer) if y.ndim == 1: # create a 2D array and squeeze results lr.fit(X, transformer2.fit_transform(y.reshape(-1, 1)).squeeze()) + y_lr_pred = lr.predict(X).reshape(-1, 1) + y_pred2 = transformer2.inverse_transform(y_lr_pred).squeeze() else: lr.fit(X, transformer2.fit_transform(y)) - y_lr_pred = lr.predict(X) - assert_allclose(y_pred, transformer2.inverse_transform(y_lr_pred)) + y_lr_pred = lr.predict(X) + y_pred2 = transformer2.inverse_transform(y_lr_pred) + + assert_allclose(y_pred, y_pred2) assert_allclose(regr.regressor_.coef_, lr.coef_) @@ -177,8 +200,9 @@ def test_transform_target_regressor_2d_transformer_multioutput(): X = friedman[0] y = np.vstack((friedman[1], friedman[1] ** 2 + 1)).T transformer = StandardScaler() - regr = TransformedTargetRegressor(regressor=LinearRegression(), - transformer=transformer) + regr = TransformedTargetRegressor( + regressor=LinearRegression(), transformer=transformer + ) y_pred = regr.fit(X, y).predict(X) assert y.shape == y_pred.shape # consistency forward transform @@ -186,8 +210,7 @@ def test_transform_target_regressor_2d_transformer_multioutput(): _check_standard_scaled(y, y_tran) assert y.shape == y_pred.shape # consistency inverse transform - assert_allclose(y, regr.transformer_.inverse_transform( - y_tran).squeeze()) + assert_allclose(y, regr.transformer_.inverse_transform(y_tran).squeeze()) # consistency of the regressor lr = LinearRegression() transformer2 = clone(transformer) @@ -197,6 +220,27 @@ def test_transform_target_regressor_2d_transformer_multioutput(): assert_allclose(regr.regressor_.coef_, lr.coef_) +def test_transform_target_regressor_3d_target(): + # Non-regression test for: + # https://github.com/scikit-learn/scikit-learn/issues/18866 + # Check with a 3D target with a transformer that reshapes the target + X = friedman[0] + y = np.tile(friedman[1].reshape(-1, 1, 1), [1, 3, 2]) + + def flatten_data(data): + return data.reshape(data.shape[0], -1) + + def unflatten_data(data): + return data.reshape(data.shape[0], -1, 2) + + transformer = FunctionTransformer(func=flatten_data, inverse_func=unflatten_data) + regr = TransformedTargetRegressor( + regressor=LinearRegression(), transformer=transformer + ) + y_pred = regr.fit(X, y).predict(X) + assert y.shape == y_pred.shape + + def test_transform_target_regressor_multi_to_single(): X = friedman[0] y = np.transpose([friedman[1], (friedman[1] ** 2 + 1)]) @@ -208,8 +252,9 @@ def func(y): def inverse_func(y): return y - tt = TransformedTargetRegressor(func=func, inverse_func=inverse_func, - check_inverse=False) + tt = TransformedTargetRegressor( + func=func, inverse_func=inverse_func, check_inverse=False + ) tt.fit(X, y) y_pred_2d_func = tt.predict(X) assert y_pred_2d_func.shape == (100, 1) @@ -218,8 +263,9 @@ def inverse_func(y): def func(y): return np.sqrt(y[:, 0] ** 2 + y[:, 1] ** 2) - tt = TransformedTargetRegressor(func=func, inverse_func=inverse_func, - check_inverse=False) + tt = TransformedTargetRegressor( + func=func, inverse_func=inverse_func, check_inverse=False + ) tt.fit(X, y) y_pred_1d_func = tt.predict(X) assert y_pred_1d_func.shape == (100, 1) @@ -228,7 +274,6 @@ def func(y): class DummyCheckerArrayTransformer(TransformerMixin, BaseEstimator): - def fit(self, X, y=None): assert isinstance(X, np.ndarray) return self @@ -243,7 +288,6 @@ def inverse_transform(self, X): class DummyCheckerListRegressor(DummyRegressor): - def fit(self, X, y, sample_weight=None): assert isinstance(X, list) return super().fit(X, y, sample_weight) @@ -258,9 +302,11 @@ def test_transform_target_regressor_ensure_y_array(): # numpy array. Similarly, if ``X`` is passed as a list, we check that the # predictor receive as it is. X, y = friedman - tt = TransformedTargetRegressor(transformer=DummyCheckerArrayTransformer(), - regressor=DummyCheckerListRegressor(), - check_inverse=False) + tt = TransformedTargetRegressor( + transformer=DummyCheckerArrayTransformer(), + regressor=DummyCheckerListRegressor(), + check_inverse=False, + ) tt.fit(X.tolist(), y.tolist()) tt.predict(X.tolist()) with pytest.raises(AssertionError): @@ -271,6 +317,7 @@ def test_transform_target_regressor_ensure_y_array(): class DummyTransformer(TransformerMixin, BaseEstimator): """Dummy transformer which count how many time fit was called.""" + def __init__(self, fit_counter=0): self.fit_counter = fit_counter @@ -308,8 +355,7 @@ def fit(self, X, y, sample_weight=None, check_input=True): def test_transform_target_regressor_pass_fit_parameters(): X, y = friedman regr = TransformedTargetRegressor( - regressor=DummyRegressorWithExtraFitParams(), - transformer=DummyTransformer() + regressor=DummyRegressorWithExtraFitParams(), transformer=DummyTransformer() ) regr.fit(X, y, check_input=False) @@ -320,14 +366,47 @@ def test_transform_target_regressor_route_pipeline(): X, y = friedman regr = TransformedTargetRegressor( - regressor=DummyRegressorWithExtraFitParams(), - transformer=DummyTransformer() + regressor=DummyRegressorWithExtraFitParams(), transformer=DummyTransformer() ) - estimators = [ - ('normalize', StandardScaler()), ('est', regr) - ] + estimators = [("normalize", StandardScaler()), ("est", regr)] pip = Pipeline(estimators) - pip.fit(X, y, **{'est__check_input': False}) + pip.fit(X, y, **{"est__check_input": False}) assert regr.transformer_.fit_counter == 1 + + +class DummyRegressorWithExtraPredictParams(DummyRegressor): + def predict(self, X, check_input=True): + # In the test below we make sure that the check input parameter is + # passed as false + self.predict_called = True + assert not check_input + return super().predict(X) + + +def test_transform_target_regressor_pass_extra_predict_parameters(): + # Checks that predict kwargs are passed to regressor. + X, y = friedman + regr = TransformedTargetRegressor( + regressor=DummyRegressorWithExtraPredictParams(), transformer=DummyTransformer() + ) + + regr.fit(X, y) + regr.predict(X, check_input=False) + assert regr.regressor_.predict_called + + +@pytest.mark.parametrize("output_format", ["pandas", "polars"]) +def test_transform_target_regressor_not_warns_with_global_output_set(output_format): + """Test that TransformedTargetRegressor will not raise warnings if + set_config(transform_output="pandas"/"polars") is set globally; regression test for + issue #29361.""" + X, y = datasets.make_regression() + y = np.abs(y) + 1 + with config_context(transform_output=output_format): + with warnings.catch_warnings(): + warnings.simplefilter("error") + TransformedTargetRegressor( + regressor=LinearRegression(), func=np.log, inverse_func=np.exp + ).fit(X, y) diff --git a/sklearn/conftest.py b/sklearn/conftest.py index 2978115e3091c..d5255ead1ffdc 100644 --- a/sklearn/conftest.py +++ b/sklearn/conftest.py @@ -1,54 +1,133 @@ -import os -from os import environ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import builtins +import faulthandler +import platform +import sys +from contextlib import suppress from functools import wraps +from os import environ +from unittest import SkipTest +import joblib +import numpy as np import pytest +from _pytest.doctest import DoctestItem from threadpoolctl import threadpool_limits -from sklearn.utils._openmp_helpers import _openmp_effective_n_threads -from sklearn.datasets import fetch_20newsgroups -from sklearn.datasets import fetch_20newsgroups_vectorized -from sklearn.datasets import fetch_california_housing -from sklearn.datasets import fetch_covtype -from sklearn.datasets import fetch_kddcup99 -from sklearn.datasets import fetch_olivetti_faces -from sklearn.datasets import fetch_rcv1 +from sklearn import set_config +from sklearn._min_dependencies import PYTEST_MIN_VERSION +from sklearn.datasets import ( + fetch_20newsgroups, + fetch_20newsgroups_vectorized, + fetch_california_housing, + fetch_covtype, + fetch_kddcup99, + fetch_lfw_pairs, + fetch_lfw_people, + fetch_olivetti_faces, + fetch_rcv1, + fetch_species_distributions, +) +from sklearn.utils._testing import get_pytest_filterwarning_lines +from sklearn.utils.fixes import ( + _IS_32BIT, + np_base_version, + parse_version, + sp_version, +) + +try: + from scipy_doctest.conftest import dt_config +except ModuleNotFoundError: + dt_config = None + +if parse_version(pytest.__version__) < parse_version(PYTEST_MIN_VERSION): + raise ImportError( + f"Your version of pytest is too old. Got version {pytest.__version__}, you" + f" should have pytest >= {PYTEST_MIN_VERSION} installed." + ) + +scipy_datasets_require_network = sp_version >= parse_version("1.10") + + +def raccoon_face_or_skip(): + # SciPy >= 1.10 requires network to access to get data + if scipy_datasets_require_network: + run_network_tests = environ.get("SKLEARN_SKIP_NETWORK_TESTS", "1") == "0" + if not run_network_tests: + raise SkipTest("test is enabled when SKLEARN_SKIP_NETWORK_TESTS=0") + + try: + import pooch # noqa: F401 + except ImportError: + raise SkipTest("test requires pooch to be installed") + + from scipy.datasets import face + else: + from scipy.misc import face + + return face(gray=True) dataset_fetchers = { - 'fetch_20newsgroups_fxt': fetch_20newsgroups, - 'fetch_20newsgroups_vectorized_fxt': fetch_20newsgroups_vectorized, - 'fetch_california_housing_fxt': fetch_california_housing, - 'fetch_covtype_fxt': fetch_covtype, - 'fetch_kddcup99_fxt': fetch_kddcup99, - 'fetch_olivetti_faces_fxt': fetch_olivetti_faces, - 'fetch_rcv1_fxt': fetch_rcv1, + "fetch_20newsgroups_fxt": fetch_20newsgroups, + "fetch_20newsgroups_vectorized_fxt": fetch_20newsgroups_vectorized, + "fetch_california_housing_fxt": fetch_california_housing, + "fetch_covtype_fxt": fetch_covtype, + "fetch_kddcup99_fxt": fetch_kddcup99, + "fetch_lfw_pairs_fxt": fetch_lfw_pairs, + "fetch_lfw_people_fxt": fetch_lfw_people, + "fetch_olivetti_faces_fxt": fetch_olivetti_faces, + "fetch_rcv1_fxt": fetch_rcv1, + "fetch_species_distributions_fxt": fetch_species_distributions, } +if scipy_datasets_require_network: + dataset_fetchers["raccoon_face_fxt"] = raccoon_face_or_skip + +_SKIP32_MARK = pytest.mark.skipif( + environ.get("SKLEARN_RUN_FLOAT32_TESTS", "0") != "1", + reason="Set SKLEARN_RUN_FLOAT32_TESTS=1 to run float32 dtype tests", +) + + +# Global fixtures +@pytest.fixture(params=[pytest.param(np.float32, marks=_SKIP32_MARK), np.float64]) +def global_dtype(request): + yield request.param + def _fetch_fixture(f): """Fetch dataset (download if missing and requested by environment).""" - download_if_missing = environ.get('SKLEARN_SKIP_NETWORK_TESTS', '1') == '0' + download_if_missing = environ.get("SKLEARN_SKIP_NETWORK_TESTS", "1") == "0" @wraps(f) def wrapped(*args, **kwargs): - kwargs['download_if_missing'] = download_if_missing + kwargs["download_if_missing"] = download_if_missing try: return f(*args, **kwargs) - except IOError: + except OSError as e: + if str(e) != "Data not found and `download_if_missing` is False": + raise pytest.skip("test is enabled when SKLEARN_SKIP_NETWORK_TESTS=0") + return pytest.fixture(lambda: wrapped) # Adds fixtures for fetching data fetch_20newsgroups_fxt = _fetch_fixture(fetch_20newsgroups) -fetch_20newsgroups_vectorized_fxt = \ - _fetch_fixture(fetch_20newsgroups_vectorized) +fetch_20newsgroups_vectorized_fxt = _fetch_fixture(fetch_20newsgroups_vectorized) fetch_california_housing_fxt = _fetch_fixture(fetch_california_housing) fetch_covtype_fxt = _fetch_fixture(fetch_covtype) fetch_kddcup99_fxt = _fetch_fixture(fetch_kddcup99) +fetch_lfw_pairs_fxt = _fetch_fixture(fetch_lfw_pairs) +fetch_lfw_people_fxt = _fetch_fixture(fetch_lfw_people) fetch_olivetti_faces_fxt = _fetch_fixture(fetch_olivetti_faces) fetch_rcv1_fxt = _fetch_fixture(fetch_rcv1) +fetch_species_distributions_fxt = _fetch_fixture(fetch_species_distributions) +raccoon_face_fxt = pytest.fixture(raccoon_face_or_skip) def pytest_collection_modifyitems(config, items): @@ -59,9 +138,10 @@ def pytest_collection_modifyitems(config, items): config : pytest config items : list of collected items """ - run_network_tests = environ.get('SKLEARN_SKIP_NETWORK_TESTS', '1') == '0' + run_network_tests = environ.get("SKLEARN_SKIP_NETWORK_TESTS", "1") == "0" skip_network = pytest.mark.skip( - reason="test is enabled when SKLEARN_SKIP_NETWORK_TESTS=0") + reason="test is enabled when SKLEARN_SKIP_NETWORK_TESTS=0" + ) # download datasets during collection to avoid thread unsafe behavior # when running pytest in parallel with pytest-xdist @@ -69,10 +149,16 @@ def pytest_collection_modifyitems(config, items): datasets_to_download = set() for item in items: - if not hasattr(item, "fixturenames"): + if isinstance(item, DoctestItem) and "fetch_" in item.name: + fetcher_function_name = item.name.split(".")[-1] + dataset_fetchers_key = f"{fetcher_function_name}_fxt" + dataset_to_fetch = set([dataset_fetchers_key]) & dataset_features_set + elif not hasattr(item, "fixturenames"): continue - item_fixtures = set(item.fixturenames) - dataset_to_fetch = item_fixtures & dataset_features_set + else: + item_fixtures = set(item.fixturenames) + dataset_to_fetch = item_fixtures & dataset_features_set + if not dataset_to_fetch: continue @@ -88,44 +174,202 @@ def pytest_collection_modifyitems(config, items): worker_id = environ.get("PYTEST_XDIST_WORKER", "gw0") if worker_id == "gw0" and run_network_tests: for name in datasets_to_download: - dataset_fetchers[name]() + with suppress(SkipTest): + dataset_fetchers[name]() + + for item in items: + # Known failure on with GradientBoostingClassifier on ARM64 + if ( + item.name.endswith("GradientBoostingClassifier") + and platform.machine() == "aarch64" + ): + marker = pytest.mark.xfail( + reason=( + "know failure. See " + "https://github.com/scikit-learn/scikit-learn/issues/17797" + ) + ) + item.add_marker(marker) + + skip_doctests = False + try: + import matplotlib # noqa: F401 + except ImportError: + skip_doctests = True + reason = "matplotlib is required to run the doctests" + + if _IS_32BIT: + reason = "doctest are only run when the default numpy int is 64 bits." + skip_doctests = True + elif sys.platform.startswith("win32"): + reason = ( + "doctests are not run for Windows because numpy arrays " + "repr is inconsistent across platforms." + ) + skip_doctests = True + + if np_base_version < parse_version("2"): + # TODO: configure numpy to output scalar arrays as regular Python scalars + # once possible to improve readability of the tests docstrings. + # https://numpy.org/neps/nep-0051-scalar-representation.html#implementation + reason = "Due to NEP 51 numpy scalar repr has changed in numpy 2" + skip_doctests = True + + if sp_version < parse_version("1.14"): + reason = "Scipy sparse matrix repr has changed in scipy 1.14" + skip_doctests = True + + # Normally doctest has the entire module's scope. Here we set globs to an empty dict + # to remove the module's scope: + # https://docs.python.org/3/library/doctest.html#what-s-the-execution-context + for item in items: + if isinstance(item, DoctestItem): + item.dtest.globs = {} + + if skip_doctests: + skip_marker = pytest.mark.skip(reason=reason) + + for item in items: + if isinstance(item, DoctestItem): + # work-around an internal error with pytest if adding a skip + # mark to a doctest in a contextmanager, see + # https://github.com/pytest-dev/pytest/issues/8796 for more + # details. + if item.name != "sklearn._config.config_context": + item.add_marker(skip_marker) + try: + import PIL # noqa: F401 + + pillow_installed = True + except ImportError: + pillow_installed = False + + if not pillow_installed: + skip_marker = pytest.mark.skip(reason="pillow (or PIL) not installed!") + for item in items: + if item.name in [ + "sklearn.feature_extraction.image.PatchExtractor", + "sklearn.feature_extraction.image.extract_patches_2d", + ]: + item.add_marker(skip_marker) -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def pyplot(): """Setup and teardown fixture for matplotlib. This fixture checks if we can import matplotlib. If not, the tests will be - skipped. Otherwise, we setup matplotlib backend and close the figures - after running the functions. + skipped. Otherwise, we close the figures before and after running the + functions. Returns ------- pyplot : module The ``matplotlib.pyplot`` module. """ - matplotlib = pytest.importorskip('matplotlib') - matplotlib.use('agg') - pyplot = pytest.importorskip('matplotlib.pyplot') + pyplot = pytest.importorskip("matplotlib.pyplot") + pyplot.close("all") yield pyplot - pyplot.close('all') + pyplot.close("all") -def pytest_runtest_setup(item): - """Set the number of openmp threads based on the number of workers - xdist is using to prevent oversubscription. +def pytest_generate_tests(metafunc): + """Parametrization of global_random_seed fixture + + based on the SKLEARN_TESTS_GLOBAL_RANDOM_SEED environment variable. + + The goal of this fixture is to prevent tests that use it to be sensitive + to a specific seed value while still being deterministic by default. + + See the documentation for the SKLEARN_TESTS_GLOBAL_RANDOM_SEED + variable for instructions on how to use this fixture. + + https://scikit-learn.org/dev/computing/parallelism.html#sklearn-tests-global-random-seed - Parameters - ---------- - item : pytest item - item to be processed """ + # When using pytest-xdist this function is called in the xdist workers. + # We rely on SKLEARN_TESTS_GLOBAL_RANDOM_SEED environment variable which is + # set in before running pytest and is available in xdist workers since they + # are subprocesses. + RANDOM_SEED_RANGE = list(range(100)) # All seeds in [0, 99] should be valid. + random_seed_var = environ.get("SKLEARN_TESTS_GLOBAL_RANDOM_SEED") + + default_random_seeds = [42] + + if random_seed_var is None: + random_seeds = default_random_seeds + elif random_seed_var == "all": + random_seeds = RANDOM_SEED_RANGE + else: + if "-" in random_seed_var: + start, stop = random_seed_var.split("-") + random_seeds = list(range(int(start), int(stop) + 1)) + else: + random_seeds = [int(random_seed_var)] + + if min(random_seeds) < 0 or max(random_seeds) > 99: + raise ValueError( + "The value(s) of the environment variable " + "SKLEARN_TESTS_GLOBAL_RANDOM_SEED must be in the range [0, 99] " + f"(or 'all'), got: {random_seed_var}" + ) + + if "global_random_seed" in metafunc.fixturenames: + metafunc.parametrize("global_random_seed", random_seeds) + + +def pytest_configure(config): + # Use matplotlib agg backend during the tests including doctests try: - xdist_worker_count = int(os.environ['PYTEST_XDIST_WORKER_COUNT']) - except KeyError: - # raises when pytest-xdist is not installed - return - - openmp_threads = _openmp_effective_n_threads() - threads_per_worker = max(openmp_threads // xdist_worker_count, 1) - threadpool_limits(threads_per_worker, user_api='openmp') + import matplotlib + + matplotlib.use("agg") + except ImportError: + pass + + allowed_parallelism = joblib.cpu_count(only_physical_cores=True) + xdist_worker_count = environ.get("PYTEST_XDIST_WORKER_COUNT") + if xdist_worker_count is not None: + # Set the number of OpenMP and BLAS threads based on the number of workers + # xdist is using to prevent oversubscription. + allowed_parallelism = max(allowed_parallelism // int(xdist_worker_count), 1) + threadpool_limits(allowed_parallelism) + + if environ.get("SKLEARN_WARNINGS_AS_ERRORS", "0") != "0": + # This seems like the only way to programmatically change the config + # filterwarnings. This was suggested in + # https://github.com/pytest-dev/pytest/issues/3311#issuecomment-373177592 + for line in get_pytest_filterwarning_lines(): + config.addinivalue_line("filterwarnings", line) + + faulthandler_timeout = int(environ.get("SKLEARN_FAULTHANDLER_TIMEOUT", "0")) + if faulthandler_timeout > 0: + faulthandler.enable() + faulthandler.dump_traceback_later(faulthandler_timeout, exit=True) + + +@pytest.fixture +def hide_available_pandas(monkeypatch): + """Pretend pandas was not installed.""" + import_orig = builtins.__import__ + + def mocked_import(name, *args, **kwargs): + if name == "pandas": + raise ImportError() + return import_orig(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", mocked_import) + + +@pytest.fixture +def print_changed_only_false(): + """Set `print_changed_only` to False for the duration of the test.""" + set_config(print_changed_only=False) + yield + set_config(print_changed_only=True) # reset to default + + +if dt_config is not None: + # Strict mode to differentiate between 3.14 and np.float64(3.14) + dt_config.strict_check = True + # dt_config.rtol = 0.01 diff --git a/sklearn/covariance/__init__.py b/sklearn/covariance/__init__.py index aec9c6b1462e9..65817ef7b977b 100644 --- a/sklearn/covariance/__init__.py +++ b/sklearn/covariance/__init__.py @@ -1,35 +1,46 @@ -""" -The :mod:`sklearn.covariance` module includes methods and algorithms to -robustly estimate the covariance of features given a set of points. The -precision matrix defined as the inverse of the covariance is also estimated. -Covariance estimation is closely related to the theory of Gaussian Graphical -Models. +"""Methods and algorithms to robustly estimate covariance. + +They estimate the covariance of features at given sets of points, as well as the +precision matrix defined as the inverse of the covariance. Covariance estimation is +closely related to the theory of Gaussian graphical models. """ -from ._empirical_covariance import (empirical_covariance, - EmpiricalCovariance, - log_likelihood) -from ._shrunk_covariance import (shrunk_covariance, ShrunkCovariance, - ledoit_wolf, ledoit_wolf_shrinkage, - LedoitWolf, oas, OAS) -from ._robust_covariance import fast_mcd, MinCovDet -from ._graph_lasso import graphical_lasso, GraphicalLasso, GraphicalLassoCV -from ._elliptic_envelope import EllipticEnvelope +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +from ._elliptic_envelope import EllipticEnvelope +from ._empirical_covariance import ( + EmpiricalCovariance, + empirical_covariance, + log_likelihood, +) +from ._graph_lasso import GraphicalLasso, GraphicalLassoCV, graphical_lasso +from ._robust_covariance import MinCovDet, fast_mcd +from ._shrunk_covariance import ( + OAS, + LedoitWolf, + ShrunkCovariance, + ledoit_wolf, + ledoit_wolf_shrinkage, + oas, + shrunk_covariance, +) -__all__ = ['EllipticEnvelope', - 'EmpiricalCovariance', - 'GraphicalLasso', - 'GraphicalLassoCV', - 'LedoitWolf', - 'MinCovDet', - 'OAS', - 'ShrunkCovariance', - 'empirical_covariance', - 'fast_mcd', - 'graphical_lasso', - 'ledoit_wolf', - 'ledoit_wolf_shrinkage', - 'log_likelihood', - 'oas', - 'shrunk_covariance'] +__all__ = [ + "OAS", + "EllipticEnvelope", + "EmpiricalCovariance", + "GraphicalLasso", + "GraphicalLassoCV", + "LedoitWolf", + "MinCovDet", + "ShrunkCovariance", + "empirical_covariance", + "fast_mcd", + "graphical_lasso", + "ledoit_wolf", + "ledoit_wolf_shrinkage", + "log_likelihood", + "oas", + "shrunk_covariance", +] diff --git a/sklearn/covariance/_elliptic_envelope.py b/sklearn/covariance/_elliptic_envelope.py index e599f0435f48c..71fb72ccd683d 100644 --- a/sklearn/covariance/_elliptic_envelope.py +++ b/sklearn/covariance/_elliptic_envelope.py @@ -1,13 +1,15 @@ -# Author: Virgile Fritsch -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from numbers import Real import numpy as np -from . import MinCovDet -from ..utils.validation import check_is_fitted -from ..utils.validation import _deprecate_positional_args + +from ..base import OutlierMixin, _fit_context from ..metrics import accuracy_score -from ..base import OutlierMixin +from ..utils._param_validation import Interval +from ..utils.validation import check_is_fitted +from ._robust_covariance import MinCovDet class EllipticEnvelope(OutlierMixin, MinCovDet): @@ -32,7 +34,7 @@ class EllipticEnvelope(OutlierMixin, MinCovDet): support_fraction : float, default=None The proportion of points to be included in the support of the raw MCD estimate. If None, the minimum value of support_fraction will - be used within the algorithm: `[n_sample + n_features + 1] / 2`. + be used within the algorithm: `(n_samples + n_features + 1) / 2 * n_samples`. Range is (0, 1). contamination : float, default=0.1 @@ -42,7 +44,7 @@ class EllipticEnvelope(OutlierMixin, MinCovDet): random_state : int, RandomState instance or None, default=None Determines the pseudo random number generator for shuffling the data. Pass an int for reproducible results across multiple function - calls. See :term: `Glossary `. + calls. See :term:`Glossary `. Attributes ---------- @@ -84,6 +86,40 @@ class EllipticEnvelope(OutlierMixin, MinCovDet): Mahalanobis distances of the training set (on which :meth:`fit` is called) observations. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + EmpiricalCovariance : Maximum likelihood covariance estimator. + GraphicalLasso : Sparse inverse covariance estimation + with an l1-penalized estimator. + LedoitWolf : LedoitWolf Estimator. + MinCovDet : Minimum Covariance Determinant + (robust estimator of covariance). + OAS : Oracle Approximating Shrinkage Estimator. + ShrunkCovariance : Covariance estimator with shrinkage. + + Notes + ----- + Outlier detection from covariance estimation may break or not + perform well in high-dimensional settings. In particular, one will + always take care to work with ``n_samples > n_features ** 2``. + + References + ---------- + .. [1] Rousseeuw, P.J., Van Driessen, K. "A fast algorithm for the + minimum covariance determinant estimator" Technometrics 41(3), 212 + (1999) + Examples -------- >>> import numpy as np @@ -99,56 +135,53 @@ class EllipticEnvelope(OutlierMixin, MinCovDet): ... [3, 3]]) array([ 1, -1]) >>> cov.covariance_ - array([[0.7411..., 0.2535...], - [0.2535..., 0.3053...]]) + array([[0.7411, 0.2535], + [0.2535, 0.3053]]) >>> cov.location_ - array([0.0813... , 0.0427...]) - - See Also - -------- - EmpiricalCovariance, MinCovDet - - Notes - ----- - Outlier detection from covariance estimation may break or not - perform well in high-dimensional settings. In particular, one will - always take care to work with ``n_samples > n_features ** 2``. - - References - ---------- - .. [1] Rousseeuw, P.J., Van Driessen, K. "A fast algorithm for the - minimum covariance determinant estimator" Technometrics 41(3), 212 - (1999) + array([0.0813 , 0.0427]) """ - @_deprecate_positional_args - def __init__(self, *, store_precision=True, assume_centered=False, - support_fraction=None, contamination=0.1, - random_state=None): + + _parameter_constraints: dict = { + **MinCovDet._parameter_constraints, + "contamination": [Interval(Real, 0, 0.5, closed="right")], + } + + def __init__( + self, + *, + store_precision=True, + assume_centered=False, + support_fraction=None, + contamination=0.1, + random_state=None, + ): super().__init__( store_precision=store_precision, assume_centered=assume_centered, support_fraction=support_fraction, - random_state=random_state) + random_state=random_state, + ) self.contamination = contamination + @_fit_context(prefer_skip_nested_validation=True) def fit(self, X, y=None): """Fit the EllipticEnvelope model. Parameters ---------- - X : {array-like, sparse matrix} of shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Training data. y : Ignored Not used, present for API consistency by convention. - """ - if self.contamination != 'auto': - if not(0. < self.contamination <= .5): - raise ValueError("contamination must be in (0, 0.5], " - "got: %f" % self.contamination) + Returns + ------- + self : object + Returns the instance itself. + """ super().fit(X) - self.offset_ = np.percentile(-self.dist_, 100. * self.contamination) + self.offset_ = np.percentile(-self.dist_, 100.0 * self.contamination) return self def decision_function(self, X): @@ -185,13 +218,11 @@ def score_samples(self, X): Opposite of the Mahalanobis distances. """ check_is_fitted(self) - X = self._validate_data(X, reset=False) return -self.mahalanobis(X) def predict(self, X): """ - Predict the labels (1 inlier, -1 outlier) of X according to the - fitted model. + Predict labels (1 inlier, -1 outlier) of X according to fitted model. Parameters ---------- @@ -210,7 +241,7 @@ def predict(self, X): return is_inlier def score(self, X, y, sample_weight=None): - """Returns the mean accuracy on the given test data and labels. + """Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that diff --git a/sklearn/covariance/_empirical_covariance.py b/sklearn/covariance/_empirical_covariance.py index fb1797e50f96a..7c4db63b4e363 100644 --- a/sklearn/covariance/_empirical_covariance.py +++ b/sklearn/covariance/_empirical_covariance.py @@ -3,31 +3,39 @@ """ -# Author: Alexandre Gramfort -# Gael Varoquaux -# Virgile Fritsch -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause # avoid division truncation import warnings + import numpy as np from scipy import linalg +from sklearn.utils import metadata_routing + from .. import config_context -from ..base import BaseEstimator +from ..base import BaseEstimator, _fit_context +from ..metrics.pairwise import pairwise_distances from ..utils import check_array +from ..utils._param_validation import validate_params from ..utils.extmath import fast_logdet -from ..metrics.pairwise import pairwise_distances -from ..utils.validation import _deprecate_positional_args +from ..utils.validation import validate_data +@validate_params( + { + "emp_cov": [np.ndarray], + "precision": [np.ndarray], + }, + prefer_skip_nested_validation=True, +) def log_likelihood(emp_cov, precision): - """Computes the sample mean of the log_likelihood under a covariance model + """Compute the sample mean of the log_likelihood under a covariance model. - computes the empirical expected log-likelihood (accounting for the - normalization terms and scaling), allowing for universal comparison (beyond - this software package) + Computes the empirical expected log-likelihood, allowing for universal + comparison (beyond this software package), and accounts for normalization + terms and scaling. Parameters ---------- @@ -43,27 +51,32 @@ def log_likelihood(emp_cov, precision): Sample mean of the log-likelihood. """ p = precision.shape[0] - log_likelihood_ = - np.sum(emp_cov * precision) + fast_logdet(precision) + log_likelihood_ = -np.sum(emp_cov * precision) + fast_logdet(precision) log_likelihood_ -= p * np.log(2 * np.pi) - log_likelihood_ /= 2. + log_likelihood_ /= 2.0 return log_likelihood_ -@_deprecate_positional_args +@validate_params( + { + "X": ["array-like"], + "assume_centered": ["boolean"], + }, + prefer_skip_nested_validation=True, +) def empirical_covariance(X, *, assume_centered=False): - """Computes the Maximum likelihood covariance estimator - + """Compute the Maximum likelihood covariance estimator. Parameters ---------- X : ndarray of shape (n_samples, n_features) - Data from which to compute the covariance estimate + Data from which to compute the covariance estimate. assume_centered : bool, default=False - If True, data will not be centered before computation. + If `True`, data will not be centered before computation. Useful when working with data whose mean is almost, but not exactly zero. - If False, data will be centered before computation. + If `False`, data will be centered before computation. Returns ------- @@ -80,14 +93,15 @@ def empirical_covariance(X, *, assume_centered=False): [0.25, 0.25, 0.25], [0.25, 0.25, 0.25]]) """ - X = np.asarray(X) + X = check_array(X, ensure_2d=False, ensure_all_finite=False) if X.ndim == 1: X = np.reshape(X, (1, -1)) if X.shape[0] == 1: - warnings.warn("Only one sample available. " - "You may want to reshape your data array") + warnings.warn( + "Only one sample available. You may want to reshape your data array" + ) if assume_centered: covariance = np.dot(X.T, X) / X.shape[0] @@ -100,7 +114,7 @@ def empirical_covariance(X, *, assume_centered=False): class EmpiricalCovariance(BaseEstimator): - """Maximum likelihood covariance estimator + """Maximum likelihood covariance estimator. Read more in the :ref:`User Guide `. @@ -127,6 +141,29 @@ class EmpiricalCovariance(BaseEstimator): Estimated pseudo-inverse matrix. (stored only if store_precision is True) + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + EllipticEnvelope : An object for detecting outliers in + a Gaussian distributed dataset. + GraphicalLasso : Sparse inverse covariance estimation + with an l1-penalized estimator. + LedoitWolf : LedoitWolf Estimator. + MinCovDet : Minimum Covariance Determinant + (robust estimator of covariance). + OAS : Oracle Approximating Shrinkage Estimator. + ShrunkCovariance : Covariance estimator with shrinkage. + Examples -------- >>> import numpy as np @@ -140,13 +177,20 @@ class EmpiricalCovariance(BaseEstimator): ... size=500) >>> cov = EmpiricalCovariance().fit(X) >>> cov.covariance_ - array([[0.7569..., 0.2818...], - [0.2818..., 0.3928...]]) + array([[0.7569, 0.2818], + [0.2818, 0.3928]]) >>> cov.location_ - array([0.0622..., 0.0193...]) - + array([0.0622, 0.0193]) """ - @_deprecate_positional_args + + # X_test should have been called X + __metadata_request__score = {"X_test": metadata_routing.UNUSED} + + _parameter_constraints: dict = { + "store_precision": ["boolean"], + "assume_centered": ["boolean"], + } + def __init__(self, *, store_precision=True, assume_centered=False): self.store_precision = store_precision self.assume_centered = assume_centered @@ -186,15 +230,15 @@ def get_precision(self): precision = linalg.pinvh(self.covariance_, check_finite=False) return precision + @_fit_context(prefer_skip_nested_validation=True) def fit(self, X, y=None): - """Fits the Maximum Likelihood Estimator covariance model - according to the given training data and parameters. + """Fit the maximum likelihood covariance estimator to X. Parameters ---------- X : array-like of shape (n_samples, n_features) - Training data, where n_samples is the number of samples and - n_features is the number of features. + Training data, where `n_samples` is the number of samples and + `n_features` is the number of features. y : Ignored Not used, present for API consistency by convention. @@ -202,28 +246,30 @@ def fit(self, X, y=None): Returns ------- self : object + Returns the instance itself. """ - X = self._validate_data(X) + X = validate_data(self, X) if self.assume_centered: self.location_ = np.zeros(X.shape[1]) else: self.location_ = X.mean(0) - covariance = empirical_covariance( - X, assume_centered=self.assume_centered) + covariance = empirical_covariance(X, assume_centered=self.assume_centered) self._set_covariance(covariance) return self def score(self, X_test, y=None): - """Computes the log-likelihood of a Gaussian data set with - `self.covariance_` as an estimator of its covariance matrix. + """Compute the log-likelihood of `X_test` under the estimated Gaussian model. + + The Gaussian model is defined by its mean and covariance matrix which are + represented respectively by `self.location_` and `self.covariance_`. Parameters ---------- X_test : array-like of shape (n_samples, n_features) - Test data of which we compute the likelihood, where n_samples is - the number of samples and n_features is the number of features. - X_test is assumed to be drawn from the same distribution than + Test data of which we compute the likelihood, where `n_samples` is + the number of samples and `n_features` is the number of features. + `X_test` is assumed to be drawn from the same distribution than the data used in fit (including centering). y : Ignored @@ -232,22 +278,19 @@ def score(self, X_test, y=None): Returns ------- res : float - The likelihood of the data set with `self.covariance_` as an - estimator of its covariance matrix. + The log-likelihood of `X_test` with `self.location_` and `self.covariance_` + as estimators of the Gaussian model mean and covariance matrix respectively. """ - X_test = self._validate_data(X_test, reset=False) + X_test = validate_data(self, X_test, reset=False) # compute empirical covariance of the test set - test_cov = empirical_covariance( - X_test - self.location_, assume_centered=True) + test_cov = empirical_covariance(X_test - self.location_, assume_centered=True) # compute log likelihood res = log_likelihood(test_cov, self.get_precision()) return res - def error_norm(self, comp_cov, norm='frobenius', scaling=True, - squared=True): - """Computes the Mean Squared Error between two covariance estimators. - (In the sense of the Frobenius norm). + def error_norm(self, comp_cov, norm="frobenius", scaling=True, squared=True): + """Compute the Mean Squared Error between two covariance estimators. Parameters ---------- @@ -279,12 +322,13 @@ def error_norm(self, comp_cov, norm='frobenius', scaling=True, error = comp_cov - self.covariance_ # compute the error norm if norm == "frobenius": - squared_norm = np.sum(error ** 2) + squared_norm = np.sum(error**2) elif norm == "spectral": squared_norm = np.amax(linalg.svdvals(np.dot(error.T, error))) else: raise NotImplementedError( - "Only spectral and frobenius norms are implemented") + "Only spectral and frobenius norms are implemented" + ) # optionally scale the error norm if scaling: squared_norm = squared_norm / error.shape[0] @@ -297,7 +341,7 @@ def error_norm(self, comp_cov, norm='frobenius', scaling=True, return result def mahalanobis(self, X): - """Computes the squared Mahalanobis distances of given observations. + """Compute the squared Mahalanobis distances of given observations. Parameters ---------- @@ -311,12 +355,13 @@ def mahalanobis(self, X): dist : ndarray of shape (n_samples,) Squared Mahalanobis distances of the observations. """ - X = self._validate_data(X, reset=False) + X = validate_data(self, X, reset=False) precision = self.get_precision() with config_context(assume_finite=True): # compute mahalanobis distances - dist = pairwise_distances(X, self.location_[np.newaxis, :], - metric='mahalanobis', VI=precision) + dist = pairwise_distances( + X, self.location_[np.newaxis, :], metric="mahalanobis", VI=precision + ) return np.reshape(dist, (len(X),)) ** 2 diff --git a/sklearn/covariance/_graph_lasso.py b/sklearn/covariance/_graph_lasso.py index 57167b81fe9e0..e94663120216d 100644 --- a/sklearn/covariance/_graph_lasso.py +++ b/sklearn/covariance/_graph_lasso.py @@ -2,30 +2,42 @@ estimator. """ -# Author: Gael Varoquaux -# License: BSD 3 clause -# Copyright: INRIA -from collections.abc import Sequence -import warnings +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import operator import sys import time +import warnings +from numbers import Integral, Real import numpy as np from scipy import linalg -from joblib import Parallel - -from . import empirical_covariance, EmpiricalCovariance, log_likelihood +from ..base import _fit_context from ..exceptions import ConvergenceWarning -from ..utils.validation import check_random_state -from ..utils.validation import _deprecate_positional_args -from ..utils.fixes import delayed + # mypy error: Module 'sklearn.linear_model' has no attribute '_cd_fast' -from ..linear_model import _cd_fast as cd_fast # type: ignore +from ..linear_model import _cd_fast as cd_fast # type: ignore[attr-defined] from ..linear_model import lars_path_gram from ..model_selection import check_cv, cross_val_score -from ..utils.deprecation import deprecated +from ..utils import Bunch +from ..utils._param_validation import Interval, StrOptions, validate_params +from ..utils.metadata_routing import ( + MetadataRouter, + MethodMapping, + _raise_for_params, + _routing_enabled, + process_routing, +) +from ..utils.parallel import Parallel, delayed +from ..utils.validation import ( + _is_arraylike_not_scalar, + check_random_state, + check_scalar, + validate_data, +) +from . import EmpiricalCovariance, empirical_covariance, log_likelihood # Helper functions to compute the objective and dual objective functions @@ -38,9 +50,8 @@ def _objective(mle, precision_, alpha): penalisation term to promote sparsity """ p = precision_.shape[0] - cost = - 2. * log_likelihood(mle, precision_) + p * np.log(2 * np.pi) - cost += alpha * (np.abs(precision_).sum() - - np.abs(np.diag(precision_)).sum()) + cost = -2.0 * log_likelihood(mle, precision_) + p * np.log(2 * np.pi) + cost += alpha * (np.abs(precision_).sum() - np.abs(np.diag(precision_)).sum()) return cost @@ -52,11 +63,144 @@ def _dual_gap(emp_cov, precision_, alpha): """ gap = np.sum(emp_cov * precision_) gap -= precision_.shape[0] - gap += alpha * (np.abs(precision_).sum() - - np.abs(np.diag(precision_)).sum()) + gap += alpha * (np.abs(precision_).sum() - np.abs(np.diag(precision_)).sum()) return gap +# The g-lasso algorithm +def _graphical_lasso( + emp_cov, + alpha, + *, + cov_init=None, + mode="cd", + tol=1e-4, + enet_tol=1e-4, + max_iter=100, + verbose=False, + eps=np.finfo(np.float64).eps, +): + _, n_features = emp_cov.shape + if alpha == 0: + # Early return without regularization + precision_ = linalg.inv(emp_cov) + cost = -2.0 * log_likelihood(emp_cov, precision_) + cost += n_features * np.log(2 * np.pi) + d_gap = np.sum(emp_cov * precision_) - n_features + return emp_cov, precision_, (cost, d_gap), 0 + + if cov_init is None: + covariance_ = emp_cov.copy() + else: + covariance_ = cov_init.copy() + # As a trivial regularization (Tikhonov like), we scale down the + # off-diagonal coefficients of our starting point: This is needed, as + # in the cross-validation the cov_init can easily be + # ill-conditioned, and the CV loop blows. Beside, this takes + # conservative stand-point on the initial conditions, and it tends to + # make the convergence go faster. + covariance_ *= 0.95 + diagonal = emp_cov.flat[:: n_features + 1] + covariance_.flat[:: n_features + 1] = diagonal + precision_ = linalg.pinvh(covariance_) + + indices = np.arange(n_features) + i = 0 # initialize the counter to be robust to `max_iter=0` + costs = list() + # The different l1 regression solver have different numerical errors + if mode == "cd": + errors = dict(over="raise", invalid="ignore") + else: + errors = dict(invalid="raise") + try: + # be robust to the max_iter=0 edge case, see: + # https://github.com/scikit-learn/scikit-learn/issues/4134 + d_gap = np.inf + # set a sub_covariance buffer + sub_covariance = np.copy(covariance_[1:, 1:], order="C") + for i in range(max_iter): + for idx in range(n_features): + # To keep the contiguous matrix `sub_covariance` equal to + # covariance_[indices != idx].T[indices != idx] + # we only need to update 1 column and 1 line when idx changes + if idx > 0: + di = idx - 1 + sub_covariance[di] = covariance_[di][indices != idx] + sub_covariance[:, di] = covariance_[:, di][indices != idx] + else: + sub_covariance[:] = covariance_[1:, 1:] + row = emp_cov[idx, indices != idx] + with np.errstate(**errors): + if mode == "cd": + # Use coordinate descent + coefs = -( + precision_[indices != idx, idx] + / (precision_[idx, idx] + 1000 * eps) + ) + coefs, _, _, _ = cd_fast.enet_coordinate_descent_gram( + coefs, + alpha, + 0, + sub_covariance, + row, + row, + max_iter, + enet_tol, + check_random_state(None), + False, + ) + else: # mode == "lars" + _, _, coefs = lars_path_gram( + Xy=row, + Gram=sub_covariance, + n_samples=row.size, + alpha_min=alpha / (n_features - 1), + copy_Gram=True, + eps=eps, + method="lars", + return_path=False, + ) + # Update the precision matrix + precision_[idx, idx] = 1.0 / ( + covariance_[idx, idx] + - np.dot(covariance_[indices != idx, idx], coefs) + ) + precision_[indices != idx, idx] = -precision_[idx, idx] * coefs + precision_[idx, indices != idx] = -precision_[idx, idx] * coefs + coefs = np.dot(sub_covariance, coefs) + covariance_[idx, indices != idx] = coefs + covariance_[indices != idx, idx] = coefs + if not np.isfinite(precision_.sum()): + raise FloatingPointError( + "The system is too ill-conditioned for this solver" + ) + d_gap = _dual_gap(emp_cov, precision_, alpha) + cost = _objective(emp_cov, precision_, alpha) + if verbose: + print( + "[graphical_lasso] Iteration % 3i, cost % 3.2e, dual gap %.3e" + % (i, cost, d_gap) + ) + costs.append((cost, d_gap)) + if np.abs(d_gap) < tol: + break + if not np.isfinite(cost) and i > 0: + raise FloatingPointError( + "Non SPD result: the system is too ill-conditioned for this solver" + ) + else: + warnings.warn( + "graphical_lasso: did not converge after %i iteration: dual gap: %.3e" + % (max_iter, d_gap), + ConvergenceWarning, + ) + except FloatingPointError as e: + e.args = (e.args[0] + ". The system is too ill-conditioned for this solver",) + raise e + + return covariance_, precision_, costs, i + 1 + + def alpha_max(emp_cov): """Find the maximum alpha for which there are some non-zeros off-diagonal. @@ -72,17 +216,32 @@ def alpha_max(emp_cov): bound for alpha is given by `max(abs(Xy))`, the result follows. """ A = np.copy(emp_cov) - A.flat[::A.shape[0] + 1] = 0 + A.flat[:: A.shape[0] + 1] = 0 return np.max(np.abs(A)) -# The g-lasso algorithm -@_deprecate_positional_args -def graphical_lasso(emp_cov, alpha, *, cov_init=None, mode='cd', tol=1e-4, - enet_tol=1e-4, max_iter=100, verbose=False, - return_costs=False, eps=np.finfo(np.float64).eps, - return_n_iter=False): - """l1-penalized covariance estimator +@validate_params( + { + "emp_cov": ["array-like"], + "return_costs": ["boolean"], + "return_n_iter": ["boolean"], + }, + prefer_skip_nested_validation=False, +) +def graphical_lasso( + emp_cov, + alpha, + *, + mode="cd", + tol=1e-4, + enet_tol=1e-4, + max_iter=100, + verbose=False, + return_costs=False, + eps=np.finfo(np.float64).eps, + return_n_iter=False, +): + """L1-penalized covariance estimator. Read more in the :ref:`User Guide `. @@ -91,7 +250,7 @@ def graphical_lasso(emp_cov, alpha, *, cov_init=None, mode='cd', tol=1e-4, Parameters ---------- - emp_cov : ndarray of shape (n_features, n_features) + emp_cov : array-like of shape (n_features, n_features) Empirical covariance from which to compute the covariance estimate. alpha : float @@ -99,10 +258,6 @@ def graphical_lasso(emp_cov, alpha, *, cov_init=None, mode='cd', tol=1e-4, regularization, the sparser the inverse covariance. Range is (0, inf]. - cov_init : array of shape (n_features, n_features), default=None - The initial guess for the covariance. If None, then the empirical - covariance is used. - mode : {'cd', 'lars'}, default='cd' The Lasso solver to use: coordinate descent or LARS. Use LARS for very sparse underlying graphs, where p > n. Elsewhere prefer cd @@ -125,7 +280,7 @@ def graphical_lasso(emp_cov, alpha, *, cov_init=None, mode='cd', tol=1e-4, If verbose is True, the objective function and dual gap are printed at each iteration. - return_costs : bool, default=Flase + return_costs : bool, default=False If return_costs is True, the objective function and dual gap at each iteration are returned. @@ -154,7 +309,10 @@ def graphical_lasso(emp_cov, alpha, *, cov_init=None, mode='cd', tol=1e-4, See Also -------- - GraphicalLasso, GraphicalLassoCV + GraphicalLasso : Sparse inverse covariance estimation + with an l1-penalized estimator. + GraphicalLassoCV : Sparse inverse covariance with + cross-validated choice of the l1 penalty. Notes ----- @@ -164,129 +322,79 @@ def graphical_lasso(emp_cov, alpha, *, cov_init=None, mode='cd', tol=1e-4, One possible difference with the `glasso` R package is that the diagonal coefficients are not penalized. - """ - _, n_features = emp_cov.shape - if alpha == 0: - if return_costs: - precision_ = linalg.inv(emp_cov) - cost = - 2. * log_likelihood(emp_cov, precision_) - cost += n_features * np.log(2 * np.pi) - d_gap = np.sum(emp_cov * precision_) - n_features - if return_n_iter: - return emp_cov, precision_, (cost, d_gap), 0 - else: - return emp_cov, precision_, (cost, d_gap) - else: - if return_n_iter: - return emp_cov, linalg.inv(emp_cov), 0 - else: - return emp_cov, linalg.inv(emp_cov) - if cov_init is None: - covariance_ = emp_cov.copy() - else: - covariance_ = cov_init.copy() - # As a trivial regularization (Tikhonov like), we scale down the - # off-diagonal coefficients of our starting point: This is needed, as - # in the cross-validation the cov_init can easily be - # ill-conditioned, and the CV loop blows. Beside, this takes - # conservative stand-point on the initial conditions, and it tends to - # make the convergence go faster. - covariance_ *= 0.95 - diagonal = emp_cov.flat[::n_features + 1] - covariance_.flat[::n_features + 1] = diagonal - precision_ = linalg.pinvh(covariance_) - - indices = np.arange(n_features) - costs = list() - # The different l1 regression solver have different numerical errors - if mode == 'cd': - errors = dict(over='raise', invalid='ignore') - else: - errors = dict(invalid='raise') - try: - # be robust to the max_iter=0 edge case, see: - # https://github.com/scikit-learn/scikit-learn/issues/4134 - d_gap = np.inf - # set a sub_covariance buffer - sub_covariance = np.copy(covariance_[1:, 1:], order='C') - for i in range(max_iter): - for idx in range(n_features): - # To keep the contiguous matrix `sub_covariance` equal to - # covariance_[indices != idx].T[indices != idx] - # we only need to update 1 column and 1 line when idx changes - if idx > 0: - di = idx - 1 - sub_covariance[di] = covariance_[di][indices != idx] - sub_covariance[:, di] = covariance_[:, di][indices != idx] - else: - sub_covariance[:] = covariance_[1:, 1:] - row = emp_cov[idx, indices != idx] - with np.errstate(**errors): - if mode == 'cd': - # Use coordinate descent - coefs = -(precision_[indices != idx, idx] - / (precision_[idx, idx] + 1000 * eps)) - coefs, _, _, _ = cd_fast.enet_coordinate_descent_gram( - coefs, alpha, 0, sub_covariance, - row, row, max_iter, enet_tol, - check_random_state(None), False) - else: - # Use LARS - _, _, coefs = lars_path_gram( - Xy=row, Gram=sub_covariance, n_samples=row.size, - alpha_min=alpha / (n_features - 1), copy_Gram=True, - eps=eps, method='lars', return_path=False) - # Update the precision matrix - precision_[idx, idx] = ( - 1. / (covariance_[idx, idx] - - np.dot(covariance_[indices != idx, idx], coefs))) - precision_[indices != idx, idx] = (- precision_[idx, idx] - * coefs) - precision_[idx, indices != idx] = (- precision_[idx, idx] - * coefs) - coefs = np.dot(sub_covariance, coefs) - covariance_[idx, indices != idx] = coefs - covariance_[indices != idx, idx] = coefs - if not np.isfinite(precision_.sum()): - raise FloatingPointError('The system is too ill-conditioned ' - 'for this solver') - d_gap = _dual_gap(emp_cov, precision_, alpha) - cost = _objective(emp_cov, precision_, alpha) - if verbose: - print('[graphical_lasso] Iteration ' - '% 3i, cost % 3.2e, dual gap %.3e' - % (i, cost, d_gap)) - if return_costs: - costs.append((cost, d_gap)) - if np.abs(d_gap) < tol: - break - if not np.isfinite(cost) and i > 0: - raise FloatingPointError('Non SPD result: the system is ' - 'too ill-conditioned for this solver') - else: - warnings.warn('graphical_lasso: did not converge after ' - '%i iteration: dual gap: %.3e' - % (max_iter, d_gap), ConvergenceWarning) - except FloatingPointError as e: - e.args = (e.args[0] - + '. The system is too ill-conditioned for this solver',) - raise e + Examples + -------- + >>> import numpy as np + >>> from sklearn.datasets import make_sparse_spd_matrix + >>> from sklearn.covariance import empirical_covariance, graphical_lasso + >>> true_cov = make_sparse_spd_matrix(n_dim=3,random_state=42) + >>> rng = np.random.RandomState(42) + >>> X = rng.multivariate_normal(mean=np.zeros(3), cov=true_cov, size=3) + >>> emp_cov = empirical_covariance(X, assume_centered=True) + >>> emp_cov, _ = graphical_lasso(emp_cov, alpha=0.05) + >>> emp_cov + array([[ 1.687, 0.212, -0.209], + [ 0.212, 0.221, -0.0817], + [-0.209, -0.0817, 0.232]]) + """ + model = GraphicalLasso( + alpha=alpha, + mode=mode, + covariance="precomputed", + tol=tol, + enet_tol=enet_tol, + max_iter=max_iter, + verbose=verbose, + eps=eps, + assume_centered=True, + ).fit(emp_cov) + + output = [model.covariance_, model.precision_] if return_costs: - if return_n_iter: - return covariance_, precision_, costs, i + 1 - else: - return covariance_, precision_, costs - else: - if return_n_iter: - return covariance_, precision_, i + 1 - else: - return covariance_, precision_ + output.append(model.costs_) + if return_n_iter: + output.append(model.n_iter_) + return tuple(output) + + +class BaseGraphicalLasso(EmpiricalCovariance): + _parameter_constraints: dict = { + **EmpiricalCovariance._parameter_constraints, + "tol": [Interval(Real, 0, None, closed="right")], + "enet_tol": [Interval(Real, 0, None, closed="right")], + "max_iter": [Interval(Integral, 0, None, closed="left")], + "mode": [StrOptions({"cd", "lars"})], + "verbose": ["verbose"], + "eps": [Interval(Real, 0, None, closed="both")], + } + _parameter_constraints.pop("store_precision") + + def __init__( + self, + tol=1e-4, + enet_tol=1e-4, + max_iter=100, + mode="cd", + verbose=False, + eps=np.finfo(np.float64).eps, + assume_centered=False, + ): + super().__init__(assume_centered=assume_centered) + self.tol = tol + self.enet_tol = enet_tol + self.max_iter = max_iter + self.mode = mode + self.verbose = verbose + self.eps = eps -class GraphicalLasso(EmpiricalCovariance): +class GraphicalLasso(BaseGraphicalLasso): """Sparse inverse covariance estimation with an l1-penalized estimator. + For a usage example see + :ref:`sphx_glr_auto_examples_applications_plot_stock_market.py`. + Read more in the :ref:`User Guide `. .. versionchanged:: v0.20 @@ -304,6 +412,13 @@ class GraphicalLasso(EmpiricalCovariance): very sparse underlying graphs, where p > n. Elsewhere prefer cd which is more numerically stable. + covariance : "precomputed", default=None + If covariance is "precomputed", the input data in `fit` is assumed + to be the covariance matrix. If `None`, the empirical covariance + is estimated from the data `X`. + + .. versionadded:: 1.3 + tol : float, default=1e-4 The tolerance to declare convergence: if the dual gap goes below this value, iterations are stopped. Range is (0, inf]. @@ -321,6 +436,13 @@ class GraphicalLasso(EmpiricalCovariance): If verbose is True, the objective function and dual gap are plotted at each iteration. + eps : float, default=eps + The machine-precision regularization in the computation of the + Cholesky diagonal factors. Increase this for very ill-conditioned + systems. Default is `np.finfo(np.float64).eps`. + + .. versionadded:: 1.3 + assume_centered : bool, default=False If True, data are not centered before computation. Useful when working with data whose mean is almost, but not exactly @@ -341,6 +463,29 @@ class GraphicalLasso(EmpiricalCovariance): n_iter_ : int Number of iterations run. + costs_ : list of (objective, dual_gap) pairs + The list of values of the objective function and the dual gap at + each iteration. Returned only if return_costs is True. + + .. versionadded:: 1.3 + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + graphical_lasso : L1-penalized covariance estimator. + GraphicalLassoCV : Sparse inverse covariance with + cross-validated choice of the l1 penalty. + Examples -------- >>> import numpy as np @@ -361,29 +506,47 @@ class GraphicalLasso(EmpiricalCovariance): [0.019, 0.034, 0.093, 0.69 ]]) >>> np.around(cov.location_, decimals=3) array([0.073, 0.04 , 0.038, 0.143]) - - See Also - -------- - graphical_lasso, GraphicalLassoCV """ - @_deprecate_positional_args - def __init__(self, alpha=.01, *, mode='cd', tol=1e-4, enet_tol=1e-4, - max_iter=100, verbose=False, assume_centered=False): - super().__init__(assume_centered=assume_centered) + + _parameter_constraints: dict = { + **BaseGraphicalLasso._parameter_constraints, + "alpha": [Interval(Real, 0, None, closed="both")], + "covariance": [StrOptions({"precomputed"}), None], + } + + def __init__( + self, + alpha=0.01, + *, + mode="cd", + covariance=None, + tol=1e-4, + enet_tol=1e-4, + max_iter=100, + verbose=False, + eps=np.finfo(np.float64).eps, + assume_centered=False, + ): + super().__init__( + tol=tol, + enet_tol=enet_tol, + max_iter=max_iter, + mode=mode, + verbose=verbose, + eps=eps, + assume_centered=assume_centered, + ) self.alpha = alpha - self.mode = mode - self.tol = tol - self.enet_tol = enet_tol - self.max_iter = max_iter - self.verbose = verbose + self.covariance = covariance + @_fit_context(prefer_skip_nested_validation=True) def fit(self, X, y=None): - """Fits the GraphicalLasso model to X. + """Fit the GraphicalLasso model to X. Parameters ---------- X : array-like of shape (n_samples, n_features) - Data from which to compute the covariance estimate + Data from which to compute the covariance estimate. y : Ignored Not used, present for API consistency by convention. @@ -391,27 +554,48 @@ def fit(self, X, y=None): Returns ------- self : object + Returns the instance itself. """ # Covariance does not make sense for a single feature - X = self._validate_data(X, ensure_min_features=2, ensure_min_samples=2, - estimator=self) + X = validate_data(self, X, ensure_min_features=2, ensure_min_samples=2) - if self.assume_centered: + if self.covariance == "precomputed": + emp_cov = X.copy() self.location_ = np.zeros(X.shape[1]) else: - self.location_ = X.mean(0) - emp_cov = empirical_covariance( - X, assume_centered=self.assume_centered) - self.covariance_, self.precision_, self.n_iter_ = graphical_lasso( - emp_cov, alpha=self.alpha, mode=self.mode, tol=self.tol, - enet_tol=self.enet_tol, max_iter=self.max_iter, - verbose=self.verbose, return_n_iter=True) + emp_cov = empirical_covariance(X, assume_centered=self.assume_centered) + if self.assume_centered: + self.location_ = np.zeros(X.shape[1]) + else: + self.location_ = X.mean(0) + + self.covariance_, self.precision_, self.costs_, self.n_iter_ = _graphical_lasso( + emp_cov, + alpha=self.alpha, + cov_init=None, + mode=self.mode, + tol=self.tol, + enet_tol=self.enet_tol, + max_iter=self.max_iter, + verbose=self.verbose, + eps=self.eps, + ) return self # Cross-validation with GraphicalLasso -def graphical_lasso_path(X, alphas, cov_init=None, X_test=None, mode='cd', - tol=1e-4, enet_tol=1e-4, max_iter=100, verbose=False): +def graphical_lasso_path( + X, + alphas, + cov_init=None, + X_test=None, + mode="cd", + tol=1e-4, + enet_tol=1e-4, + max_iter=100, + verbose=False, + eps=np.finfo(np.float64).eps, +): """l1-penalized covariance estimator along a path of decreasing alphas Read more in the :ref:`User Guide `. @@ -454,6 +638,13 @@ def graphical_lasso_path(X, alphas, cov_init=None, X_test=None, mode='cd', The higher the verbosity flag, the more information is printed during the fitting. + eps : float, default=eps + The machine-precision regularization in the computation of the + Cholesky diagonal factors. Increase this for very ill-conditioned + systems. Default is `np.finfo(np.float64).eps`. + + .. versionadded:: 1.3 + Returns ------- covariances_ : list of shape (n_alphas,) of ndarray of shape \ @@ -483,9 +674,17 @@ def graphical_lasso_path(X, alphas, cov_init=None, X_test=None, mode='cd', for alpha in alphas: try: # Capture the errors, and move on - covariance_, precision_ = graphical_lasso( - emp_cov, alpha=alpha, cov_init=covariance_, mode=mode, tol=tol, - enet_tol=enet_tol, max_iter=max_iter, verbose=inner_verbose) + covariance_, precision_, _, _ = _graphical_lasso( + emp_cov, + alpha=alpha, + cov_init=covariance_, + mode=mode, + tol=tol, + enet_tol=enet_tol, + max_iter=max_iter, + verbose=inner_verbose, + eps=eps, + ) covariances_.append(covariance_) precisions_.append(precision_) if X_test is not None: @@ -499,19 +698,21 @@ def graphical_lasso_path(X, alphas, cov_init=None, X_test=None, mode='cd', this_score = -np.inf scores_.append(this_score) if verbose == 1: - sys.stderr.write('.') + sys.stderr.write(".") elif verbose > 1: if X_test is not None: - print('[graphical_lasso_path] alpha: %.2e, score: %.2e' - % (alpha, this_score)) + print( + "[graphical_lasso_path] alpha: %.2e, score: %.2e" + % (alpha, this_score) + ) else: - print('[graphical_lasso_path] alpha: %.2e' % alpha) + print("[graphical_lasso_path] alpha: %.2e" % alpha) if X_test is not None: return covariances_, precisions_, scores_ return covariances_, precisions_ -class GraphicalLassoCV(GraphicalLasso): +class GraphicalLassoCV(BaseGraphicalLasso): """Sparse inverse covariance w/ cross-validated choice of the l1 penalty. See glossary entry for :term:`cross-validation estimator`. @@ -527,7 +728,8 @@ class GraphicalLassoCV(GraphicalLasso): If an integer is given, it fixes the number of points on the grids of alpha to be used. If a list is given, it gives the grid to be used. See the notes in the class docstring for - more details. Range is (0, inf] when floats given. + more details. Range is [1, inf) for an integer. + Range is (0, inf] for an array-like of floats. n_refinements : int, default=4 The number of times the grid is refined. Not used if explicit @@ -542,7 +744,7 @@ class GraphicalLassoCV(GraphicalLasso): - :term:`CV splitter`, - An iterable yielding (train, test) splits as arrays of indices. - For integer/None inputs :class:`KFold` is used. + For integer/None inputs :class:`~sklearn.model_selection.KFold` is used. Refer :ref:`User Guide ` for the various cross-validation strategies that can be used here. @@ -570,7 +772,7 @@ class GraphicalLassoCV(GraphicalLasso): stable. n_jobs : int, default=None - number of jobs to run in parallel. + Number of jobs to run in parallel. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. @@ -582,6 +784,13 @@ class GraphicalLassoCV(GraphicalLasso): If verbose is True, the objective function and duality gap are printed at each iteration. + eps : float, default=eps + The machine-precision regularization in the computation of the + Cholesky diagonal factors. Increase this for very ill-conditioned + systems. Default is `np.finfo(np.float64).eps`. + + .. versionadded:: 1.3 + assume_centered : bool, default=False If True, data are not centered before computation. Useful when working with data whose mean is almost, but not exactly @@ -599,24 +808,14 @@ class GraphicalLassoCV(GraphicalLasso): precision_ : ndarray of shape (n_features, n_features) Estimated precision matrix (inverse covariance). - alpha_ : float - Penalization parameter selected. - - cv_alphas_ : list of shape (n_alphas,), dtype=float - All penalization parameters explored. - - .. deprecated:: 0.24 - The `cv_alphas_` attribute is deprecated in version 0.24 in favor - of `cv_results_['alphas']` and will be removed in version - 1.1 (renaming of 0.26). + costs_ : list of (objective, dual_gap) pairs + The list of values of the objective function and the dual gap at + each iteration. Returned only if return_costs is True. - grid_scores_ : ndarray of shape (n_alphas, n_folds) - Log-likelihood score on left-out data across folds. + .. versionadded:: 1.3 - .. deprecated:: 0.24 - The `grid_scores_` attribute is deprecated in version 0.24 in favor - of `cv_results_` and will be removed in version - 1.1 (renaming of 0.26). + alpha_ : float + Penalization parameter selected. cv_results_ : dict of ndarrays A dict with keys: @@ -624,20 +823,56 @@ class GraphicalLassoCV(GraphicalLasso): alphas : ndarray of shape (n_alphas,) All penalization parameters explored. - split(k)_score : ndarray of shape (n_alphas,) + split(k)_test_score : ndarray of shape (n_alphas,) Log-likelihood score on left-out data across (k)th fold. - mean_score : ndarray of shape (n_alphas,) + .. versionadded:: 1.0 + + mean_test_score : ndarray of shape (n_alphas,) Mean of scores over the folds. - std_score : ndarray of shape (n_alphas,) + .. versionadded:: 1.0 + + std_test_score : ndarray of shape (n_alphas,) Standard deviation of scores over the folds. - .. versionadded:: 0.24 + .. versionadded:: 1.0 n_iter_ : int Number of iterations run for the optimal alpha. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + graphical_lasso : L1-penalized covariance estimator. + GraphicalLasso : Sparse inverse covariance estimation + with an l1-penalized estimator. + + Notes + ----- + The search for the optimal penalization parameter (`alpha`) is done on an + iteratively refined grid: first the cross-validated scores on a grid are + computed, then a new refined grid is centered around the maximum, and so + on. + + One of the challenges which is faced here is that the solvers can + fail to converge to a well-conditioned estimate. The corresponding + values of `alpha` then come out as missing values, but the optimum may + be close to these missing values. + + In `fit`, once the best parameter `alpha` is found through + cross-validation, the model is fit again using the entire training set. + Examples -------- >>> import numpy as np @@ -659,57 +894,86 @@ class GraphicalLassoCV(GraphicalLasso): >>> np.around(cov.location_, decimals=3) array([0.073, 0.04 , 0.038, 0.143]) - See Also - -------- - graphical_lasso, GraphicalLasso - - Notes - ----- - The search for the optimal penalization parameter (alpha) is done on an - iteratively refined grid: first the cross-validated scores on a grid are - computed, then a new refined grid is centered around the maximum, and so - on. - - One of the challenges which is faced here is that the solvers can - fail to converge to a well-conditioned estimate. The corresponding - values of alpha then come out as missing values, but the optimum may - be close to these missing values. + For an example comparing :class:`sklearn.covariance.GraphicalLassoCV`, + :func:`sklearn.covariance.ledoit_wolf` shrinkage and the empirical covariance + on high-dimensional gaussian data, see + :ref:`sphx_glr_auto_examples_covariance_plot_sparse_cov.py`. """ - @_deprecate_positional_args - def __init__(self, *, alphas=4, n_refinements=4, cv=None, tol=1e-4, - enet_tol=1e-4, max_iter=100, mode='cd', n_jobs=None, - verbose=False, assume_centered=False): + + _parameter_constraints: dict = { + **BaseGraphicalLasso._parameter_constraints, + "alphas": [Interval(Integral, 0, None, closed="left"), "array-like"], + "n_refinements": [Interval(Integral, 1, None, closed="left")], + "cv": ["cv_object"], + "n_jobs": [Integral, None], + } + + def __init__( + self, + *, + alphas=4, + n_refinements=4, + cv=None, + tol=1e-4, + enet_tol=1e-4, + max_iter=100, + mode="cd", + n_jobs=None, + verbose=False, + eps=np.finfo(np.float64).eps, + assume_centered=False, + ): super().__init__( - mode=mode, tol=tol, verbose=verbose, enet_tol=enet_tol, - max_iter=max_iter, assume_centered=assume_centered) + tol=tol, + enet_tol=enet_tol, + max_iter=max_iter, + mode=mode, + verbose=verbose, + eps=eps, + assume_centered=assume_centered, + ) self.alphas = alphas self.n_refinements = n_refinements self.cv = cv self.n_jobs = n_jobs - def fit(self, X, y=None): - """Fits the GraphicalLasso covariance model to X. + @_fit_context(prefer_skip_nested_validation=True) + def fit(self, X, y=None, **params): + """Fit the GraphicalLasso covariance model to X. Parameters ---------- X : array-like of shape (n_samples, n_features) - Data from which to compute the covariance estimate + Data from which to compute the covariance estimate. y : Ignored Not used, present for API consistency by convention. + **params : dict, default=None + Parameters to be passed to the CV splitter and the + cross_val_score function. + + .. versionadded:: 1.5 + Only available if `enable_metadata_routing=True`, + which can be set by using + ``sklearn.set_config(enable_metadata_routing=True)``. + See :ref:`Metadata Routing User Guide ` for + more details. + Returns ------- self : object + Returns the instance itself. """ # Covariance does not make sense for a single feature - X = self._validate_data(X, ensure_min_features=2, estimator=self) + _raise_for_params(params, self, "fit") + + X = validate_data(self, X, ensure_min_features=2) if self.assume_centered: self.location_ = np.zeros(X.shape[1]) else: self.location_ = X.mean(0) - emp_cov = empirical_covariance( - X, assume_centered=self.assume_centered) + emp_cov = empirical_covariance(X, assume_centered=self.assume_centered) cv = check_cv(self.cv, y, classifier=False) @@ -718,15 +982,28 @@ def fit(self, X, y=None): n_alphas = self.alphas inner_verbose = max(0, self.verbose - 1) - if isinstance(n_alphas, Sequence): + if _is_arraylike_not_scalar(n_alphas): + for alpha in self.alphas: + check_scalar( + alpha, + "alpha", + Real, + min_val=0, + max_val=np.inf, + include_boundaries="right", + ) alphas = self.alphas n_refinements = 1 else: n_refinements = self.n_refinements alpha_1 = alpha_max(emp_cov) alpha_0 = 1e-2 * alpha_1 - alphas = np.logspace(np.log10(alpha_0), np.log10(alpha_1), - n_alphas)[::-1] + alphas = np.logspace(np.log10(alpha_0), np.log10(alpha_1), n_alphas)[::-1] + + if _routing_enabled(): + routed_params = process_routing(self, "fit", **params) + else: + routed_params = Bunch(splitter=Bunch(split={})) t0 = time.time() for i in range(n_refinements): @@ -734,23 +1011,26 @@ def fit(self, X, y=None): # No need to see the convergence warnings on this grid: # they will always be points that will not converge # during the cross-validation - warnings.simplefilter('ignore', ConvergenceWarning) + warnings.simplefilter("ignore", ConvergenceWarning) # Compute the cross-validated loss on the current grid # NOTE: Warm-restarting graphical_lasso_path has been tried, # and this did not allow to gain anything # (same execution time with or without). - this_path = Parallel( - n_jobs=self.n_jobs, - verbose=self.verbose - )(delayed(graphical_lasso_path)(X[train], alphas=alphas, - X_test=X[test], mode=self.mode, - tol=self.tol, - enet_tol=self.enet_tol, - max_iter=int(.1 * - self.max_iter), - verbose=inner_verbose) - for train, test in cv.split(X, y)) + this_path = Parallel(n_jobs=self.n_jobs, verbose=self.verbose)( + delayed(graphical_lasso_path)( + X[train], + alphas=alphas, + X_test=X[test], + mode=self.mode, + tol=self.tol, + enet_tol=self.enet_tol, + max_iter=int(0.1 * self.max_iter), + verbose=inner_verbose, + eps=self.eps, + ) + for train, test in cv.split(X, y, **routed_params.splitter.split) + ) # Little danse to transform the list in what we need covs, _, scores = zip(*this_path) @@ -766,7 +1046,7 @@ def fit(self, X, y=None): last_finite_idx = 0 for index, (alpha, scores, _) in enumerate(path): this_score = np.mean(scores) - if this_score >= .1 / np.finfo(np.float64).eps: + if this_score >= 0.1 / np.finfo(np.float64).eps: this_score = np.nan if np.isfinite(this_score): last_finite_idx = index @@ -781,8 +1061,7 @@ def fit(self, X, y=None): # non-zero coefficients alpha_1 = path[0][0] alpha_0 = path[1][0] - elif (best_index == last_finite_idx - and not best_index == len(path) - 1): + elif best_index == last_finite_idx and not best_index == len(path) - 1: # We have non-converged models on the upper bound of the # grid, we need to refine the grid there alpha_1 = path[best_index][0] @@ -794,63 +1073,73 @@ def fit(self, X, y=None): alpha_1 = path[best_index - 1][0] alpha_0 = path[best_index + 1][0] - if not isinstance(n_alphas, Sequence): - alphas = np.logspace(np.log10(alpha_1), np.log10(alpha_0), - n_alphas + 2) + if not _is_arraylike_not_scalar(n_alphas): + alphas = np.logspace(np.log10(alpha_1), np.log10(alpha_0), n_alphas + 2) alphas = alphas[1:-1] if self.verbose and n_refinements > 1: - print('[GraphicalLassoCV] Done refinement % 2i out of' - ' %i: % 3is' % (i + 1, n_refinements, time.time() - t0)) + print( + "[GraphicalLassoCV] Done refinement % 2i out of %i: % 3is" + % (i + 1, n_refinements, time.time() - t0) + ) path = list(zip(*path)) grid_scores = list(path[1]) alphas = list(path[0]) # Finally, compute the score with alpha = 0 alphas.append(0) - grid_scores.append(cross_val_score(EmpiricalCovariance(), X, - cv=cv, n_jobs=self.n_jobs, - verbose=inner_verbose)) + grid_scores.append( + cross_val_score( + EmpiricalCovariance(), + X, + cv=cv, + n_jobs=self.n_jobs, + verbose=inner_verbose, + params=params, + ) + ) grid_scores = np.array(grid_scores) - self.cv_results_ = {'alphas': np.array(alphas)} + + self.cv_results_ = {"alphas": np.array(alphas)} + for i in range(grid_scores.shape[1]): - key = "split{}_score".format(i) - self.cv_results_[key] = grid_scores[:, i] + self.cv_results_[f"split{i}_test_score"] = grid_scores[:, i] - self.cv_results_["mean_score"] = np.mean(grid_scores, axis=1) - self.cv_results_["std_score"] = np.std(grid_scores, axis=1) + self.cv_results_["mean_test_score"] = np.mean(grid_scores, axis=1) + self.cv_results_["std_test_score"] = np.std(grid_scores, axis=1) best_alpha = alphas[best_index] self.alpha_ = best_alpha # Finally fit the model with the selected alpha - self.covariance_, self.precision_, self.n_iter_ = graphical_lasso( - emp_cov, alpha=best_alpha, mode=self.mode, tol=self.tol, - enet_tol=self.enet_tol, max_iter=self.max_iter, - verbose=inner_verbose, return_n_iter=True) + self.covariance_, self.precision_, self.costs_, self.n_iter_ = _graphical_lasso( + emp_cov, + alpha=best_alpha, + mode=self.mode, + tol=self.tol, + enet_tol=self.enet_tol, + max_iter=self.max_iter, + verbose=inner_verbose, + eps=self.eps, + ) return self - # TODO: Remove in 1.1 when grid_scores_ is deprecated - # mypy error: Decorated property not supported - @deprecated( # type: ignore - "The grid_scores_ attribute is deprecated in version 0.24 in favor " - "of cv_results_ and will be removed in version 1.1 (renaming of 0.26)." - ) - @property - def grid_scores_(self): - # remove 3 for mean_score, std_score, and alphas - n_alphas = len(self.cv_results_) - 3 - return np.asarray( - [self.cv_results_["split{}_score".format(i)] - for i in range(n_alphas)]).T - - # TODO: Remove in 1.1 when cv_alphas_ is deprecated - # mypy error: Decorated property not supported - @deprecated( # type: ignore - "The cv_alphas_ attribute is deprecated in version 0.24 in favor " - "of cv_results_['alpha'] and will be removed in version 1.1 " - "(renaming of 0.26)." - ) - @property - def cv_alphas_(self): - return self.cv_results_['alphas'].tolist() + def get_metadata_routing(self): + """Get metadata routing of this object. + + Please check :ref:`User Guide ` on how the routing + mechanism works. + + .. versionadded:: 1.5 + + Returns + ------- + routing : MetadataRouter + A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating + routing information. + """ + router = MetadataRouter(owner=self.__class__.__name__).add( + splitter=check_cv(self.cv), + method_mapping=MethodMapping().add(callee="split", caller="fit"), + ) + return router diff --git a/sklearn/covariance/_robust_covariance.py b/sklearn/covariance/_robust_covariance.py index d4331b591e43f..f386879e693fb 100644 --- a/sklearn/covariance/_robust_covariance.py +++ b/sklearn/covariance/_robust_covariance.py @@ -4,20 +4,23 @@ Here are implemented estimators that are resistant to outliers. """ -# Author: Virgile Fritsch -# -# License: BSD 3 clause + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import warnings -import numbers +from numbers import Integral, Real + import numpy as np from scipy import linalg from scipy.stats import chi2 -from . import empirical_covariance, EmpiricalCovariance +from ..base import _fit_context +from ..utils import check_array, check_random_state +from ..utils._param_validation import Interval from ..utils.extmath import fast_logdet -from ..utils import check_random_state, check_array -from ..utils.validation import _deprecate_positional_args +from ..utils.validation import validate_data +from ._empirical_covariance import EmpiricalCovariance, empirical_covariance # Minimum Covariance Determinant @@ -27,9 +30,15 @@ # for Quality, TECHNOMETRICS) # XXX Is this really a public function? It's not listed in the docs or # exported by sklearn.covariance. Deprecate? -def c_step(X, n_support, remaining_iterations=30, initial_estimates=None, - verbose=False, cov_computation_method=empirical_covariance, - random_state=None): +def c_step( + X, + n_support, + remaining_iterations=30, + initial_estimates=None, + verbose=False, + cov_computation_method=empirical_covariance, + random_state=None, +): """C_step procedure described in [Rouseeuw1984]_ aiming at computing MCD. Parameters @@ -66,7 +75,7 @@ def c_step(X, n_support, remaining_iterations=30, initial_estimates=None, random_state : int, RandomState instance or None, default=None Determines the pseudo random number generator for shuffling the data. Pass an int for reproducible results across multiple function calls. - See :term: `Glossary `. + See :term:`Glossary `. Returns ------- @@ -88,35 +97,45 @@ def c_step(X, n_support, remaining_iterations=30, initial_estimates=None, """ X = np.asarray(X) random_state = check_random_state(random_state) - return _c_step(X, n_support, remaining_iterations=remaining_iterations, - initial_estimates=initial_estimates, verbose=verbose, - cov_computation_method=cov_computation_method, - random_state=random_state) - - -def _c_step(X, n_support, random_state, remaining_iterations=30, - initial_estimates=None, verbose=False, - cov_computation_method=empirical_covariance): + return _c_step( + X, + n_support, + remaining_iterations=remaining_iterations, + initial_estimates=initial_estimates, + verbose=verbose, + cov_computation_method=cov_computation_method, + random_state=random_state, + ) + + +def _c_step( + X, + n_support, + random_state, + remaining_iterations=30, + initial_estimates=None, + verbose=False, + cov_computation_method=empirical_covariance, +): n_samples, n_features = X.shape dist = np.inf # Initialisation - support = np.zeros(n_samples, dtype=bool) if initial_estimates is None: # compute initial robust estimates from a random subset - support[random_state.permutation(n_samples)[:n_support]] = True + support_indices = random_state.permutation(n_samples)[:n_support] else: # get initial robust estimates from the function parameters location = initial_estimates[0] covariance = initial_estimates[1] - # run a special iteration for that case (to get an initial support) + # run a special iteration for that case (to get an initial support_indices) precision = linalg.pinvh(covariance) X_centered = X - location dist = (np.dot(X_centered, precision) * X_centered).sum(1) # compute new estimates - support[np.argsort(dist)[:n_support]] = True + support_indices = np.argpartition(dist, n_support - 1)[:n_support] - X_support = X[support] + X_support = X[support_indices] location = X_support.mean(0) covariance = cov_computation_method(X_support) @@ -128,21 +147,19 @@ def _c_step(X, n_support, random_state, remaining_iterations=30, precision = linalg.pinvh(covariance) previous_det = np.inf - while (det < previous_det and remaining_iterations > 0 - and not np.isinf(det)): + while det < previous_det and remaining_iterations > 0 and not np.isinf(det): # save old estimates values previous_location = location previous_covariance = covariance previous_det = det - previous_support = support - # compute a new support from the full data set mahalanobis distances + previous_support_indices = support_indices + # compute a new support_indices from the full data set mahalanobis distances precision = linalg.pinvh(covariance) X_centered = X - location dist = (np.dot(X_centered, precision) * X_centered).sum(axis=1) # compute new estimates - support = np.zeros(n_samples, dtype=bool) - support[np.argsort(dist)[:n_support]] = True - X_support = X[support] + support_indices = np.argpartition(dist, n_support - 1)[:n_support] + X_support = X[support_indices] location = X_support.mean(axis=0) covariance = cov_computation_method(X_support) det = fast_logdet(covariance) @@ -153,38 +170,56 @@ def _c_step(X, n_support, random_state, remaining_iterations=30, dist = (np.dot(X - location, precision) * (X - location)).sum(axis=1) # Check if best fit already found (det => 0, logdet => -inf) if np.isinf(det): - results = location, covariance, det, support, dist + results = location, covariance, det, support_indices, dist # Check convergence if np.allclose(det, previous_det): # c_step procedure converged if verbose: - print("Optimal couple (location, covariance) found before" - " ending iterations (%d left)" % (remaining_iterations)) - results = location, covariance, det, support, dist + print( + "Optimal couple (location, covariance) found before" + " ending iterations (%d left)" % (remaining_iterations) + ) + results = location, covariance, det, support_indices, dist elif det > previous_det: # determinant has increased (should not happen) - warnings.warn("Determinant has increased; this should not happen: " - "log(det) > log(previous_det) (%.15f > %.15f). " - "You may want to try with a higher value of " - "support_fraction (current value: %.3f)." - % (det, previous_det, n_support / n_samples), - RuntimeWarning) - results = previous_location, previous_covariance, \ - previous_det, previous_support, previous_dist + warnings.warn( + "Determinant has increased; this should not happen: " + "log(det) > log(previous_det) (%.15f > %.15f). " + "You may want to try with a higher value of " + "support_fraction (current value: %.3f)." + % (det, previous_det, n_support / n_samples), + RuntimeWarning, + ) + results = ( + previous_location, + previous_covariance, + previous_det, + previous_support_indices, + previous_dist, + ) # Check early stopping if remaining_iterations == 0: if verbose: - print('Maximum number of iterations reached') - results = location, covariance, det, support, dist - - return results - - -def select_candidates(X, n_support, n_trials, select=1, n_iter=30, - verbose=False, - cov_computation_method=empirical_covariance, - random_state=None): + print("Maximum number of iterations reached") + results = location, covariance, det, support_indices, dist + + location, covariance, det, support_indices, dist = results + # Convert from list of indices to boolean mask. + support = np.bincount(support_indices, minlength=n_samples).astype(bool) + return location, covariance, det, support, dist + + +def select_candidates( + X, + n_support, + n_trials, + select=1, + n_iter=30, + verbose=False, + cov_computation_method=empirical_covariance, + random_state=None, +): """Finds the best pure subset of observations to compute MCD from it. The purpose of this function is to find the best sets of n_support @@ -239,7 +274,7 @@ def select_candidates(X, n_support, n_trials, select=1, n_iter=30, random_state : int, RandomState instance or None, default=None Determines the pseudo random number generator for shuffling the data. Pass an int for reproducible results across multiple function calls. - See :term: `Glossary `. + See :term:`Glossary `. See Also --------- @@ -266,15 +301,17 @@ def select_candidates(X, n_support, n_trials, select=1, n_iter=30, """ random_state = check_random_state(random_state) - if isinstance(n_trials, numbers.Integral): + if isinstance(n_trials, Integral): run_from_estimates = False elif isinstance(n_trials, tuple): run_from_estimates = True estimates_list = n_trials n_trials = estimates_list[0].shape[0] else: - raise TypeError("Invalid 'n_trials' parameter, expected tuple or " - " integer, got %s (%s)" % (n_trials, type(n_trials))) + raise TypeError( + "Invalid 'n_trials' parameter, expected tuple or integer, got %s (%s)" + % (n_trials, type(n_trials)) + ) # compute `n_trials` location and shape estimates candidates in the subset all_estimates = [] @@ -283,20 +320,32 @@ def select_candidates(X, n_support, n_trials, select=1, n_iter=30, for j in range(n_trials): all_estimates.append( _c_step( - X, n_support, remaining_iterations=n_iter, verbose=verbose, + X, + n_support, + remaining_iterations=n_iter, + verbose=verbose, cov_computation_method=cov_computation_method, - random_state=random_state)) + random_state=random_state, + ) + ) else: # perform computations from every given initial estimates for j in range(n_trials): initial_estimates = (estimates_list[0][j], estimates_list[1][j]) - all_estimates.append(_c_step( - X, n_support, remaining_iterations=n_iter, - initial_estimates=initial_estimates, verbose=verbose, - cov_computation_method=cov_computation_method, - random_state=random_state)) - all_locs_sub, all_covs_sub, all_dets_sub, all_supports_sub, all_ds_sub = \ - zip(*all_estimates) + all_estimates.append( + _c_step( + X, + n_support, + remaining_iterations=n_iter, + initial_estimates=initial_estimates, + verbose=verbose, + cov_computation_method=cov_computation_method, + random_state=random_state, + ) + ) + all_locs_sub, all_covs_sub, all_dets_sub, all_supports_sub, all_ds_sub = zip( + *all_estimates + ) # find the `n_best` best results among the `n_trials` ones index_best = np.argsort(all_dets_sub)[:select] best_locations = np.asarray(all_locs_sub)[index_best] @@ -307,10 +356,13 @@ def select_candidates(X, n_support, n_trials, select=1, n_iter=30, return best_locations, best_covariances, best_supports, best_ds -def fast_mcd(X, support_fraction=None, - cov_computation_method=empirical_covariance, - random_state=None): - """Estimates the Minimum Covariance Determinant matrix. +def fast_mcd( + X, + support_fraction=None, + cov_computation_method=empirical_covariance, + random_state=None, +): + """Estimate the Minimum Covariance Determinant matrix. Read more in the :ref:`User Guide `. @@ -323,8 +375,8 @@ def fast_mcd(X, support_fraction=None, The proportion of points to be included in the support of the raw MCD estimate. Default is `None`, which implies that the minimum value of `support_fraction` will be used within the algorithm: - `(n_sample + n_features + 1) / 2`. This parameter must be in the - range (0, 1). + `(n_samples + n_features + 1) / 2 * n_samples`. This parameter must be + in the range (0, 1). cov_computation_method : callable, \ default=:func:`sklearn.covariance.empirical_covariance` @@ -334,7 +386,7 @@ def fast_mcd(X, support_fraction=None, random_state : int, RandomState instance or None, default=None Determines the pseudo random number generator for shuffling the data. Pass an int for reproducible results across multiple function calls. - See :term: `Glossary `. + See :term:`Glossary `. Returns ------- @@ -376,12 +428,12 @@ def fast_mcd(X, support_fraction=None, """ random_state = check_random_state(random_state) - X = check_array(X, ensure_min_samples=2, estimator='fast_mcd') + X = check_array(X, ensure_min_samples=2, estimator="fast_mcd") n_samples, n_features = X.shape # minimum breakdown value if support_fraction is None: - n_support = int(np.ceil(0.5 * (n_samples + n_features + 1))) + n_support = min(int(np.ceil(0.5 * (n_samples + n_features + 1))), n_samples) else: n_support = int(support_fraction * n_samples) @@ -392,11 +444,13 @@ def fast_mcd(X, support_fraction=None, if n_support < n_samples: # find the sample shortest halves X_sorted = np.sort(np.ravel(X)) - diff = X_sorted[n_support:] - X_sorted[:(n_samples - n_support)] + diff = X_sorted[n_support:] - X_sorted[: (n_samples - n_support)] halves_start = np.where(diff == np.min(diff))[0] # take the middle points' mean to get the robust location estimate - location = 0.5 * (X_sorted[n_support + halves_start] + - X_sorted[halves_start]).mean() + location = ( + 0.5 + * (X_sorted[n_support + halves_start] + X_sorted[halves_start]).mean() + ) support = np.zeros(n_samples, dtype=bool) X_centered = X - location support[np.argsort(np.abs(X_centered), 0)[:n_support]] = True @@ -420,8 +474,7 @@ def fast_mcd(X, support_fraction=None, n_subsets = n_samples // 300 n_samples_subsets = n_samples // n_subsets samples_shuffle = random_state.permutation(n_samples) - h_subset = int(np.ceil(n_samples_subsets * - (n_support / float(n_samples)))) + h_subset = int(np.ceil(n_samples_subsets * (n_support / float(n_samples)))) # b. perform a total of 500 trials n_trials_tot = 500 # c. select 10 best (location, covariance) for each subset @@ -430,45 +483,47 @@ def fast_mcd(X, support_fraction=None, n_best_tot = n_subsets * n_best_sub all_best_locations = np.zeros((n_best_tot, n_features)) try: - all_best_covariances = np.zeros((n_best_tot, n_features, - n_features)) + all_best_covariances = np.zeros((n_best_tot, n_features, n_features)) except MemoryError: # The above is too big. Let's try with something much small # (and less optimal) n_best_tot = 10 - all_best_covariances = np.zeros((n_best_tot, n_features, - n_features)) + all_best_covariances = np.zeros((n_best_tot, n_features, n_features)) n_best_sub = 2 for i in range(n_subsets): low_bound = i * n_samples_subsets high_bound = low_bound + n_samples_subsets current_subset = X[samples_shuffle[low_bound:high_bound]] best_locations_sub, best_covariances_sub, _, _ = select_candidates( - current_subset, h_subset, n_trials, - select=n_best_sub, n_iter=2, + current_subset, + h_subset, + n_trials, + select=n_best_sub, + n_iter=2, cov_computation_method=cov_computation_method, - random_state=random_state) + random_state=random_state, + ) subset_slice = np.arange(i * n_best_sub, (i + 1) * n_best_sub) all_best_locations[subset_slice] = best_locations_sub all_best_covariances[subset_slice] = best_covariances_sub # 2. Pool the candidate supports into a merged set # (possibly the full dataset) n_samples_merged = min(1500, n_samples) - h_merged = int(np.ceil(n_samples_merged * - (n_support / float(n_samples)))) + h_merged = int(np.ceil(n_samples_merged * (n_support / float(n_samples)))) if n_samples > 1500: n_best_merged = 10 else: n_best_merged = 1 # find the best couples (location, covariance) on the merged set selection = random_state.permutation(n_samples)[:n_samples_merged] - locations_merged, covariances_merged, supports_merged, d = \ - select_candidates( - X[selection], h_merged, - n_trials=(all_best_locations, all_best_covariances), - select=n_best_merged, - cov_computation_method=cov_computation_method, - random_state=random_state) + locations_merged, covariances_merged, supports_merged, d = select_candidates( + X[selection], + h_merged, + n_trials=(all_best_locations, all_best_covariances), + select=n_best_merged, + cov_computation_method=cov_computation_method, + random_state=random_state, + ) # 3. Finally get the overall best (locations, covariance) couple if n_samples < 1500: # directly get the best couple (location, covariance) @@ -480,13 +535,14 @@ def fast_mcd(X, support_fraction=None, dist[selection] = d[0] else: # select the best couple on the full dataset - locations_full, covariances_full, supports_full, d = \ - select_candidates( - X, n_support, - n_trials=(locations_merged, covariances_merged), - select=1, - cov_computation_method=cov_computation_method, - random_state=random_state) + locations_full, covariances_full, supports_full, d = select_candidates( + X, + n_support, + n_trials=(locations_merged, covariances_merged), + select=1, + cov_computation_method=cov_computation_method, + random_state=random_state, + ) location = locations_full[0] covariance = covariances_full[0] support = supports_full[0] @@ -497,14 +553,23 @@ def fast_mcd(X, support_fraction=None, n_trials = 30 n_best = 10 locations_best, covariances_best, _, _ = select_candidates( - X, n_support, n_trials=n_trials, select=n_best, n_iter=2, + X, + n_support, + n_trials=n_trials, + select=n_best, + n_iter=2, cov_computation_method=cov_computation_method, - random_state=random_state) + random_state=random_state, + ) # 2. Select the best couple on the full dataset amongst the 10 locations_full, covariances_full, supports_full, d = select_candidates( - X, n_support, n_trials=(locations_best, covariances_best), - select=1, cov_computation_method=cov_computation_method, - random_state=random_state) + X, + n_support, + n_trials=(locations_best, covariances_best), + select=1, + cov_computation_method=cov_computation_method, + random_state=random_state, + ) location = locations_full[0] covariance = covariances_full[0] support = supports_full[0] @@ -544,13 +609,13 @@ class MinCovDet(EmpiricalCovariance): The proportion of points to be included in the support of the raw MCD estimate. Default is None, which implies that the minimum value of support_fraction will be used within the algorithm: - `(n_sample + n_features + 1) / 2`. The parameter must be in the range - (0, 1). + `(n_samples + n_features + 1) / 2 * n_samples`. The parameter must be + in the range (0, 1]. random_state : int, RandomState instance or None, default=None Determines the pseudo random number generator for shuffling the data. Pass an int for reproducible results across multiple function calls. - See :term: `Glossary `. + See :term:`Glossary `. Attributes ---------- @@ -583,6 +648,42 @@ class MinCovDet(EmpiricalCovariance): Mahalanobis distances of the training set (on which :meth:`fit` is called) observations. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + EllipticEnvelope : An object for detecting outliers in + a Gaussian distributed dataset. + EmpiricalCovariance : Maximum likelihood covariance estimator. + GraphicalLasso : Sparse inverse covariance estimation + with an l1-penalized estimator. + GraphicalLassoCV : Sparse inverse covariance with cross-validated + choice of the l1 penalty. + LedoitWolf : LedoitWolf Estimator. + OAS : Oracle Approximating Shrinkage Estimator. + ShrunkCovariance : Covariance estimator with shrinkage. + + References + ---------- + + .. [Rouseeuw1984] P. J. Rousseeuw. Least median of squares regression. + J. Am Stat Ass, 79:871, 1984. + .. [Rousseeuw] A Fast Algorithm for the Minimum Covariance Determinant + Estimator, 1999, American Statistical Association and the American + Society for Quality, TECHNOMETRICS + .. [ButlerDavies] R. W. Butler, P. L. Davies and M. Jhun, + Asymptotics For The Minimum Covariance Determinant Estimator, + The Annals of Statistics, 1993, Vol. 21, No. 3, 1385-1400 + Examples -------- >>> import numpy as np @@ -596,35 +697,35 @@ class MinCovDet(EmpiricalCovariance): ... size=500) >>> cov = MinCovDet(random_state=0).fit(X) >>> cov.covariance_ - array([[0.7411..., 0.2535...], - [0.2535..., 0.3053...]]) + array([[0.7411, 0.2535], + [0.2535, 0.3053]]) >>> cov.location_ - array([0.0813... , 0.0427...]) - - References - ---------- - - .. [Rouseeuw1984] P. J. Rousseeuw. Least median of squares regression. - J. Am Stat Ass, 79:871, 1984. - .. [Rousseeuw] A Fast Algorithm for the Minimum Covariance Determinant - Estimator, 1999, American Statistical Association and the American - Society for Quality, TECHNOMETRICS - .. [ButlerDavies] R. W. Butler, P. L. Davies and M. Jhun, - Asymptotics For The Minimum Covariance Determinant Estimator, - The Annals of Statistics, 1993, Vol. 21, No. 3, 1385-1400 + array([0.0813 , 0.0427]) """ + + _parameter_constraints: dict = { + **EmpiricalCovariance._parameter_constraints, + "support_fraction": [Interval(Real, 0, 1, closed="right"), None], + "random_state": ["random_state"], + } _nonrobust_covariance = staticmethod(empirical_covariance) - @_deprecate_positional_args - def __init__(self, *, store_precision=True, assume_centered=False, - support_fraction=None, random_state=None): + def __init__( + self, + *, + store_precision=True, + assume_centered=False, + support_fraction=None, + random_state=None, + ): self.store_precision = store_precision self.assume_centered = assume_centered self.support_fraction = support_fraction self.random_state = random_state + @_fit_context(prefer_skip_nested_validation=True) def fit(self, X, y=None): - """Fits a Minimum Covariance Determinant with the FastMCD algorithm. + """Fit a Minimum Covariance Determinant with the FastMCD algorithm. Parameters ---------- @@ -638,23 +739,28 @@ def fit(self, X, y=None): Returns ------- self : object + Returns the instance itself. """ - X = self._validate_data(X, ensure_min_samples=2, estimator='MinCovDet') + X = validate_data(self, X, ensure_min_samples=2, estimator="MinCovDet") random_state = check_random_state(self.random_state) n_samples, n_features = X.shape # check that the empirical covariance is full rank if (linalg.svdvals(np.dot(X.T, X)) > 1e-8).sum() != n_features: - warnings.warn("The covariance matrix associated to your dataset " - "is not full rank") + warnings.warn( + "The covariance matrix associated to your dataset is not full rank" + ) # compute and store raw estimates raw_location, raw_covariance, raw_support, raw_dist = fast_mcd( - X, support_fraction=self.support_fraction, + X, + support_fraction=self.support_fraction, cov_computation_method=self._nonrobust_covariance, - random_state=random_state) + random_state=random_state, + ) if self.assume_centered: raw_location = np.zeros(n_features) - raw_covariance = self._nonrobust_covariance(X[raw_support], - assume_centered=True) + raw_covariance = self._nonrobust_covariance( + X[raw_support], assume_centered=True + ) # get precision matrix in an optimized way precision = linalg.pinvh(raw_covariance) raw_dist = np.sum(np.dot(X, precision) * X, 1) @@ -702,8 +808,10 @@ def correct_covariance(self, data): n_samples = len(self.dist_) n_support = np.sum(self.support_) if n_support < n_samples and np.allclose(self.raw_covariance_, 0): - raise ValueError('The covariance matrix of the support data ' - 'is equal to 0, try to increase support_fraction') + raise ValueError( + "The covariance matrix of the support data " + "is equal to 0, try to increase support_fraction" + ) correction = np.median(self.dist_) / chi2(data.shape[1]).isf(0.5) covariance_corrected = self.raw_covariance_ * correction self.dist_ /= correction @@ -750,13 +858,13 @@ def reweight_covariance(self, data): else: location_reweighted = data[mask].mean(0) covariance_reweighted = self._nonrobust_covariance( - data[mask], assume_centered=self.assume_centered) + data[mask], assume_centered=self.assume_centered + ) support_reweighted = np.zeros(n_samples, dtype=bool) support_reweighted[mask] = True self._set_covariance(covariance_reweighted) self.location_ = location_reweighted self.support_ = support_reweighted X_centered = data - self.location_ - self.dist_ = np.sum( - np.dot(X_centered, self.get_precision()) * X_centered, 1) + self.dist_ = np.sum(np.dot(X_centered, self.get_precision()) * X_centered, 1) return location_reweighted, covariance_reweighted, support_reweighted diff --git a/sklearn/covariance/_shrunk_covariance.py b/sklearn/covariance/_shrunk_covariance.py index 72b13681200ff..99d6f70f57d6e 100644 --- a/sklearn/covariance/_shrunk_covariance.py +++ b/sklearn/covariance/_shrunk_covariance.py @@ -6,32 +6,115 @@ """ -# Author: Alexandre Gramfort -# Gael Varoquaux -# Virgile Fritsch -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause # avoid division truncation import warnings +from numbers import Integral, Real + import numpy as np -from . import empirical_covariance, EmpiricalCovariance +from ..base import _fit_context from ..utils import check_array -from ..utils.validation import _deprecate_positional_args +from ..utils._param_validation import Interval, validate_params +from ..utils.validation import validate_data +from . import EmpiricalCovariance, empirical_covariance + + +def _ledoit_wolf(X, *, assume_centered, block_size): + """Estimate the shrunk Ledoit-Wolf covariance matrix.""" + # for only one feature, the result is the same whatever the shrinkage + if len(X.shape) == 2 and X.shape[1] == 1: + if not assume_centered: + X = X - X.mean() + return np.atleast_2d((X**2).mean()), 0.0 + n_features = X.shape[1] + + # get Ledoit-Wolf shrinkage + shrinkage = ledoit_wolf_shrinkage( + X, assume_centered=assume_centered, block_size=block_size + ) + emp_cov = empirical_covariance(X, assume_centered=assume_centered) + mu = np.sum(np.trace(emp_cov)) / n_features + shrunk_cov = (1.0 - shrinkage) * emp_cov + shrunk_cov.flat[:: n_features + 1] += shrinkage * mu + return shrunk_cov, shrinkage + + +def _oas(X, *, assume_centered=False): + """Estimate covariance with the Oracle Approximating Shrinkage algorithm. + + The formulation is based on [1]_. + [1] "Shrinkage algorithms for MMSE covariance estimation.", + Chen, Y., Wiesel, A., Eldar, Y. C., & Hero, A. O. + IEEE Transactions on Signal Processing, 58(10), 5016-5029, 2010. + https://arxiv.org/pdf/0907.4698.pdf + """ + if len(X.shape) == 2 and X.shape[1] == 1: + # for only one feature, the result is the same whatever the shrinkage + if not assume_centered: + X = X - X.mean() + return np.atleast_2d((X**2).mean()), 0.0 + + n_samples, n_features = X.shape + emp_cov = empirical_covariance(X, assume_centered=assume_centered) + + # The shrinkage is defined as: + # shrinkage = min( + # trace(S @ S.T) + trace(S)**2) / ((n + 1) (trace(S @ S.T) - trace(S)**2 / p), 1 + # ) + # where n and p are n_samples and n_features, respectively (cf. Eq. 23 in [1]). + # The factor 2 / p is omitted since it does not impact the value of the estimator + # for large p. + + # Instead of computing trace(S)**2, we can compute the average of the squared + # elements of S that is equal to trace(S)**2 / p**2. + # See the definition of the Frobenius norm: + # https://en.wikipedia.org/wiki/Matrix_norm#Frobenius_norm + alpha = np.mean(emp_cov**2) + mu = np.trace(emp_cov) / n_features + mu_squared = mu**2 + + # The factor 1 / p**2 will cancel out since it is in both the numerator and + # denominator + num = alpha + mu_squared + den = (n_samples + 1) * (alpha - mu_squared / n_features) + shrinkage = 1.0 if den == 0 else min(num / den, 1.0) + + # The shrunk covariance is defined as: + # (1 - shrinkage) * S + shrinkage * F (cf. Eq. 4 in [1]) + # where S is the empirical covariance and F is the shrinkage target defined as + # F = trace(S) / n_features * np.identity(n_features) (cf. Eq. 3 in [1]) + shrunk_cov = (1.0 - shrinkage) * emp_cov + shrunk_cov.flat[:: n_features + 1] += shrinkage * mu + + return shrunk_cov, shrinkage + + +############################################################################### +# Public API # ShrunkCovariance estimator + +@validate_params( + { + "emp_cov": ["array-like"], + "shrinkage": [Interval(Real, 0, 1, closed="both")], + }, + prefer_skip_nested_validation=True, +) def shrunk_covariance(emp_cov, shrinkage=0.1): - """Calculates a covariance matrix shrunk on the diagonal + """Calculate covariance matrices shrunk on the diagonal. Read more in the :ref:`User Guide `. Parameters ---------- - emp_cov : array-like of shape (n_features, n_features) - Covariance matrix to be shrunk + emp_cov : array-like of shape (..., n_features, n_features) + Covariance matrices to be shrunk, at least 2D ndarray. shrinkage : float, default=0.1 Coefficient in the convex combination used for the computation @@ -39,36 +122,49 @@ def shrunk_covariance(emp_cov, shrinkage=0.1): Returns ------- - shrunk_cov : ndarray of shape (n_features, n_features) - Shrunk covariance. + shrunk_cov : ndarray of shape (..., n_features, n_features) + Shrunk covariance matrices. Notes ----- - The regularized (shrunk) covariance is given by: + The regularized (shrunk) covariance is given by:: - (1 - shrinkage) * cov + shrinkage * mu * np.identity(n_features) + (1 - shrinkage) * cov + shrinkage * mu * np.identity(n_features) - where mu = trace(cov) / n_features + where `mu = trace(cov) / n_features`. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.datasets import make_gaussian_quantiles + >>> from sklearn.covariance import empirical_covariance, shrunk_covariance + >>> real_cov = np.array([[.8, .3], [.3, .4]]) + >>> rng = np.random.RandomState(0) + >>> X = rng.multivariate_normal(mean=[0, 0], cov=real_cov, size=500) + >>> shrunk_covariance(empirical_covariance(X)) + array([[0.739, 0.254], + [0.254, 0.411]]) """ - emp_cov = check_array(emp_cov) - n_features = emp_cov.shape[0] + emp_cov = check_array(emp_cov, allow_nd=True) + n_features = emp_cov.shape[-1] - mu = np.trace(emp_cov) / n_features - shrunk_cov = (1. - shrinkage) * emp_cov - shrunk_cov.flat[::n_features + 1] += shrinkage * mu + shrunk_cov = (1.0 - shrinkage) * emp_cov + mu = np.trace(emp_cov, axis1=-2, axis2=-1) / n_features + mu = np.expand_dims(mu, axis=tuple(range(mu.ndim, emp_cov.ndim))) + shrunk_cov += shrinkage * mu * np.eye(n_features) return shrunk_cov class ShrunkCovariance(EmpiricalCovariance): - """Covariance estimator with shrinkage + """Covariance estimator with shrinkage. Read more in the :ref:`User Guide `. Parameters ---------- store_precision : bool, default=True - Specify if the estimated precision is stored + Specify if the estimated precision is stored. assume_centered : bool, default=False If True, data will not be centered before computation. @@ -92,6 +188,39 @@ class ShrunkCovariance(EmpiricalCovariance): Estimated pseudo inverse matrix. (stored only if store_precision is True) + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + EllipticEnvelope : An object for detecting outliers in + a Gaussian distributed dataset. + EmpiricalCovariance : Maximum likelihood covariance estimator. + GraphicalLasso : Sparse inverse covariance estimation + with an l1-penalized estimator. + GraphicalLassoCV : Sparse inverse covariance with cross-validated + choice of the l1 penalty. + LedoitWolf : LedoitWolf Estimator. + MinCovDet : Minimum Covariance Determinant + (robust estimator of covariance). + OAS : Oracle Approximating Shrinkage Estimator. + + Notes + ----- + The regularized covariance is given by: + + (1 - shrinkage) * cov + shrinkage * mu * np.identity(n_features) + + where mu = trace(cov) / n_features + Examples -------- >>> import numpy as np @@ -105,35 +234,32 @@ class ShrunkCovariance(EmpiricalCovariance): ... size=500) >>> cov = ShrunkCovariance().fit(X) >>> cov.covariance_ - array([[0.7387..., 0.2536...], - [0.2536..., 0.4110...]]) + array([[0.7387, 0.2536], + [0.2536, 0.4110]]) >>> cov.location_ - array([0.0622..., 0.0193...]) - - Notes - ----- - The regularized covariance is given by: + array([0.0622, 0.0193]) + """ - (1 - shrinkage) * cov + shrinkage * mu * np.identity(n_features) + _parameter_constraints: dict = { + **EmpiricalCovariance._parameter_constraints, + "shrinkage": [Interval(Real, 0, 1, closed="both")], + } - where mu = trace(cov) / n_features - """ - @_deprecate_positional_args - def __init__(self, *, store_precision=True, assume_centered=False, - shrinkage=0.1): - super().__init__(store_precision=store_precision, - assume_centered=assume_centered) + def __init__(self, *, store_precision=True, assume_centered=False, shrinkage=0.1): + super().__init__( + store_precision=store_precision, assume_centered=assume_centered + ) self.shrinkage = shrinkage + @_fit_context(prefer_skip_nested_validation=True) def fit(self, X, y=None): - """Fit the shrunk covariance model according to the given training data - and parameters. + """Fit the shrunk covariance model to X. Parameters ---------- X : array-like of shape (n_samples, n_features) - Training data, where n_samples is the number of samples - and n_features is the number of features. + Training data, where `n_samples` is the number of samples + and `n_features` is the number of features. y : Ignored Not used, present for API consistency by convention. @@ -141,16 +267,16 @@ def fit(self, X, y=None): Returns ------- self : object + Returns the instance itself. """ - X = self._validate_data(X) + X = validate_data(self, X) # Not calling the parent object to fit, to avoid a potential # matrix inversion when setting the precision if self.assume_centered: self.location_ = np.zeros(X.shape[1]) else: self.location_ = X.mean(0) - covariance = empirical_covariance( - X, assume_centered=self.assume_centered) + covariance = empirical_covariance(X, assume_centered=self.assume_centered) covariance = shrunk_covariance(covariance, self.shrinkage) self._set_covariance(covariance) @@ -159,8 +285,17 @@ def fit(self, X, y=None): # Ledoit-Wolf estimator + +@validate_params( + { + "X": ["array-like"], + "assume_centered": ["boolean"], + "block_size": [Interval(Integral, 1, None, closed="left")], + }, + prefer_skip_nested_validation=True, +) def ledoit_wolf_shrinkage(X, assume_centered=False, block_size=1000): - """Estimates the shrunk Ledoit-Wolf covariance matrix. + """Estimate the shrunk Ledoit-Wolf covariance matrix. Read more in the :ref:`User Guide `. @@ -191,17 +326,29 @@ def ledoit_wolf_shrinkage(X, assume_centered=False, block_size=1000): (1 - shrinkage) * cov + shrinkage * mu * np.identity(n_features) where mu = trace(cov) / n_features + + Examples + -------- + >>> import numpy as np + >>> from sklearn.covariance import ledoit_wolf_shrinkage + >>> real_cov = np.array([[.4, .2], [.2, .8]]) + >>> rng = np.random.RandomState(0) + >>> X = rng.multivariate_normal(mean=[0, 0], cov=real_cov, size=50) + >>> shrinkage_coefficient = ledoit_wolf_shrinkage(X) + >>> shrinkage_coefficient + np.float64(0.23) """ - X = np.asarray(X) + X = check_array(X) # for only one feature, the result is the same whatever the shrinkage if len(X.shape) == 2 and X.shape[1] == 1: - return 0. + return 0.0 if X.ndim == 1: X = np.reshape(X, (1, -1)) if X.shape[0] == 1: - warnings.warn("Only one sample available. " - "You may want to reshape your data array") + warnings.warn( + "Only one sample available. You may want to reshape your data array" + ) n_samples, n_features = X.shape # optionally center data @@ -213,11 +360,11 @@ def ledoit_wolf_shrinkage(X, assume_centered=False, block_size=1000): # number of blocks to split the covariance matrix into n_splits = int(n_features / block_size) - X2 = X ** 2 + X2 = X**2 emp_cov_trace = np.sum(X2, axis=0) / n_samples mu = np.sum(emp_cov_trace) / n_features - beta_ = 0. # sum of the coefficients of - delta_ = 0. # sum of the *squared* coefficients of + beta_ = 0.0 # sum of the coefficients of + delta_ = 0.0 # sum of the *squared* coefficients of # starting block computation for i in range(n_splits): for j in range(n_splits): @@ -226,26 +373,26 @@ def ledoit_wolf_shrinkage(X, assume_centered=False, block_size=1000): beta_ += np.sum(np.dot(X2.T[rows], X2[:, cols])) delta_ += np.sum(np.dot(X.T[rows], X[:, cols]) ** 2) rows = slice(block_size * i, block_size * (i + 1)) - beta_ += np.sum(np.dot(X2.T[rows], X2[:, block_size * n_splits:])) - delta_ += np.sum( - np.dot(X.T[rows], X[:, block_size * n_splits:]) ** 2) + beta_ += np.sum(np.dot(X2.T[rows], X2[:, block_size * n_splits :])) + delta_ += np.sum(np.dot(X.T[rows], X[:, block_size * n_splits :]) ** 2) for j in range(n_splits): cols = slice(block_size * j, block_size * (j + 1)) - beta_ += np.sum(np.dot(X2.T[block_size * n_splits:], X2[:, cols])) - delta_ += np.sum( - np.dot(X.T[block_size * n_splits:], X[:, cols]) ** 2) - delta_ += np.sum(np.dot(X.T[block_size * n_splits:], - X[:, block_size * n_splits:]) ** 2) - delta_ /= n_samples ** 2 - beta_ += np.sum(np.dot(X2.T[block_size * n_splits:], - X2[:, block_size * n_splits:])) + beta_ += np.sum(np.dot(X2.T[block_size * n_splits :], X2[:, cols])) + delta_ += np.sum(np.dot(X.T[block_size * n_splits :], X[:, cols]) ** 2) + delta_ += np.sum( + np.dot(X.T[block_size * n_splits :], X[:, block_size * n_splits :]) ** 2 + ) + delta_ /= n_samples**2 + beta_ += np.sum( + np.dot(X2.T[block_size * n_splits :], X2[:, block_size * n_splits :]) + ) # use delta_ to compute beta - beta = 1. / (n_features * n_samples) * (beta_ / n_samples - delta_) + beta = 1.0 / (n_features * n_samples) * (beta_ / n_samples - delta_) # delta is the sum of the squared coefficients of ( - mu*Id) / p - delta = delta_ - 2. * mu * emp_cov_trace.sum() + n_features * mu ** 2 + delta = delta_ - 2.0 * mu * emp_cov_trace.sum() + n_features * mu**2 delta /= n_features # get final beta as the min between beta and delta - # We do this to prevent shrinking more than "1", which whould invert + # We do this to prevent shrinking more than "1", which would invert # the value of covariances beta = min(beta, delta) # finally get shrinkage @@ -253,16 +400,19 @@ def ledoit_wolf_shrinkage(X, assume_centered=False, block_size=1000): return shrinkage -@_deprecate_positional_args +@validate_params( + {"X": ["array-like"]}, + prefer_skip_nested_validation=False, +) def ledoit_wolf(X, *, assume_centered=False, block_size=1000): - """Estimates the shrunk Ledoit-Wolf covariance matrix. + """Estimate the shrunk Ledoit-Wolf covariance matrix. Read more in the :ref:`User Guide `. Parameters ---------- X : array-like of shape (n_samples, n_features) - Data from which to compute the covariance estimate + Data from which to compute the covariance estimate. assume_centered : bool, default=False If True, data will not be centered before computation. @@ -290,34 +440,32 @@ def ledoit_wolf(X, *, assume_centered=False, block_size=1000): (1 - shrinkage) * cov + shrinkage * mu * np.identity(n_features) where mu = trace(cov) / n_features - """ - X = np.asarray(X) - # for only one feature, the result is the same whatever the shrinkage - if len(X.shape) == 2 and X.shape[1] == 1: - if not assume_centered: - X = X - X.mean() - return np.atleast_2d((X ** 2).mean()), 0. - if X.ndim == 1: - X = np.reshape(X, (1, -1)) - warnings.warn("Only one sample available. " - "You may want to reshape your data array") - n_features = X.size - else: - _, n_features = X.shape - # get Ledoit-Wolf shrinkage - shrinkage = ledoit_wolf_shrinkage( - X, assume_centered=assume_centered, block_size=block_size) - emp_cov = empirical_covariance(X, assume_centered=assume_centered) - mu = np.sum(np.trace(emp_cov)) / n_features - shrunk_cov = (1. - shrinkage) * emp_cov - shrunk_cov.flat[::n_features + 1] += shrinkage * mu + Examples + -------- + >>> import numpy as np + >>> from sklearn.covariance import empirical_covariance, ledoit_wolf + >>> real_cov = np.array([[.4, .2], [.2, .8]]) + >>> rng = np.random.RandomState(0) + >>> X = rng.multivariate_normal(mean=[0, 0], cov=real_cov, size=50) + >>> covariance, shrinkage = ledoit_wolf(X) + >>> covariance + array([[0.44, 0.16], + [0.16, 0.80]]) + >>> shrinkage + np.float64(0.23) + """ + estimator = LedoitWolf( + assume_centered=assume_centered, + block_size=block_size, + store_precision=False, + ).fit(X) - return shrunk_cov, shrinkage + return estimator.covariance_, estimator.shrinkage_ class LedoitWolf(EmpiricalCovariance): - """LedoitWolf Estimator + """LedoitWolf Estimator. Ledoit-Wolf is a particular form of shrinkage, where the shrinkage coefficient is computed using O. Ledoit and M. Wolf's formula as @@ -359,22 +507,30 @@ class LedoitWolf(EmpiricalCovariance): Coefficient in the convex combination used for the computation of the shrunk estimate. Range is [0, 1]. - Examples + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also -------- - >>> import numpy as np - >>> from sklearn.covariance import LedoitWolf - >>> real_cov = np.array([[.4, .2], - ... [.2, .8]]) - >>> np.random.seed(0) - >>> X = np.random.multivariate_normal(mean=[0, 0], - ... cov=real_cov, - ... size=50) - >>> cov = LedoitWolf().fit(X) - >>> cov.covariance_ - array([[0.4406..., 0.1616...], - [0.1616..., 0.8022...]]) - >>> cov.location_ - array([ 0.0595... , -0.0075...]) + EllipticEnvelope : An object for detecting outliers in + a Gaussian distributed dataset. + EmpiricalCovariance : Maximum likelihood covariance estimator. + GraphicalLasso : Sparse inverse covariance estimation + with an l1-penalized estimator. + GraphicalLassoCV : Sparse inverse covariance with cross-validated + choice of the l1 penalty. + MinCovDet : Minimum Covariance Determinant + (robust estimator of covariance). + OAS : Oracle Approximating Shrinkage Estimator. + ShrunkCovariance : Covariance estimator with shrinkage. Notes ----- @@ -390,17 +546,43 @@ class LedoitWolf(EmpiricalCovariance): "A Well-Conditioned Estimator for Large-Dimensional Covariance Matrices", Ledoit and Wolf, Journal of Multivariate Analysis, Volume 88, Issue 2, February 2004, pages 365-411. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.covariance import LedoitWolf + >>> real_cov = np.array([[.4, .2], + ... [.2, .8]]) + >>> np.random.seed(0) + >>> X = np.random.multivariate_normal(mean=[0, 0], + ... cov=real_cov, + ... size=50) + >>> cov = LedoitWolf().fit(X) + >>> cov.covariance_ + array([[0.4406, 0.1616], + [0.1616, 0.8022]]) + >>> cov.location_ + array([ 0.0595 , -0.0075]) + + See also :ref:`sphx_glr_auto_examples_covariance_plot_covariance_estimation.py` + and :ref:`sphx_glr_auto_examples_covariance_plot_lw_vs_oas.py` + for more detailed examples. """ - @_deprecate_positional_args - def __init__(self, *, store_precision=True, assume_centered=False, - block_size=1000): - super().__init__(store_precision=store_precision, - assume_centered=assume_centered) + + _parameter_constraints: dict = { + **EmpiricalCovariance._parameter_constraints, + "block_size": [Interval(Integral, 1, None, closed="left")], + } + + def __init__(self, *, store_precision=True, assume_centered=False, block_size=1000): + super().__init__( + store_precision=store_precision, assume_centered=assume_centered + ) self.block_size = block_size + @_fit_context(prefer_skip_nested_validation=True) def fit(self, X, y=None): - """Fit the Ledoit-Wolf shrunk covariance model according to the given - training data and parameters. + """Fit the Ledoit-Wolf shrunk covariance model to X. Parameters ---------- @@ -413,17 +595,18 @@ def fit(self, X, y=None): Returns ------- self : object + Returns the instance itself. """ # Not calling the parent object to fit, to avoid computing the # covariance matrix (and potentially the precision) - X = self._validate_data(X) + X = validate_data(self, X) if self.assume_centered: self.location_ = np.zeros(X.shape[1]) else: self.location_ = X.mean(0) - covariance, shrinkage = ledoit_wolf(X - self.location_, - assume_centered=True, - block_size=self.block_size) + covariance, shrinkage = _ledoit_wolf( + X - self.location_, assume_centered=True, block_size=self.block_size + ) self.shrinkage_ = shrinkage self._set_covariance(covariance) @@ -431,9 +614,14 @@ def fit(self, X, y=None): # OAS estimator -@_deprecate_positional_args +@validate_params( + {"X": ["array-like"]}, + prefer_skip_nested_validation=False, +) def oas(X, *, assume_centered=False): - """Estimate covariance with the Oracle Approximating Shrinkage algorithm. + """Estimate covariance with the Oracle Approximating Shrinkage. + + Read more in the :ref:`User Guide `. Parameters ---------- @@ -457,60 +645,51 @@ def oas(X, *, assume_centered=False): Notes ----- - The regularised (shrunk) covariance is: + The regularised covariance is: - (1 - shrinkage) * cov + shrinkage * mu * np.identity(n_features) + (1 - shrinkage) * cov + shrinkage * mu * np.identity(n_features), - where mu = trace(cov) / n_features + where mu = trace(cov) / n_features and shrinkage is given by the OAS formula + (see [1]_). - The formula we used to implement the OAS is slightly modified compared - to the one given in the article. See :class:`OAS` for more details. - """ - X = np.asarray(X) - # for only one feature, the result is the same whatever the shrinkage - if len(X.shape) == 2 and X.shape[1] == 1: - if not assume_centered: - X = X - X.mean() - return np.atleast_2d((X ** 2).mean()), 0. - if X.ndim == 1: - X = np.reshape(X, (1, -1)) - warnings.warn("Only one sample available. " - "You may want to reshape your data array") - n_samples = 1 - n_features = X.size - else: - n_samples, n_features = X.shape + The shrinkage formulation implemented here differs from Eq. 23 in [1]_. In + the original article, formula (23) states that 2/p (p being the number of + features) is multiplied by Trace(cov*cov) in both the numerator and + denominator, but this operation is omitted because for a large p, the value + of 2/p is so small that it doesn't affect the value of the estimator. - emp_cov = empirical_covariance(X, assume_centered=assume_centered) - mu = np.trace(emp_cov) / n_features - - # formula from Chen et al.'s **implementation** - alpha = np.mean(emp_cov ** 2) - num = alpha + mu ** 2 - den = (n_samples + 1.) * (alpha - (mu ** 2) / n_features) - - shrinkage = 1. if den == 0 else min(num / den, 1.) - shrunk_cov = (1. - shrinkage) * emp_cov - shrunk_cov.flat[::n_features + 1] += shrinkage * mu + References + ---------- + .. [1] :arxiv:`"Shrinkage algorithms for MMSE covariance estimation.", + Chen, Y., Wiesel, A., Eldar, Y. C., & Hero, A. O. + IEEE Transactions on Signal Processing, 58(10), 5016-5029, 2010. + <0907.4698>` - return shrunk_cov, shrinkage + Examples + -------- + >>> import numpy as np + >>> from sklearn.covariance import oas + >>> rng = np.random.RandomState(0) + >>> real_cov = [[.8, .3], [.3, .4]] + >>> X = rng.multivariate_normal(mean=[0, 0], cov=real_cov, size=500) + >>> shrunk_cov, shrinkage = oas(X) + >>> shrunk_cov + array([[0.7533, 0.2763], + [0.2763, 0.3964]]) + >>> shrinkage + np.float64(0.0195) + """ + estimator = OAS( + assume_centered=assume_centered, + ).fit(X) + return estimator.covariance_, estimator.shrinkage_ class OAS(EmpiricalCovariance): - """Oracle Approximating Shrinkage Estimator + """Oracle Approximating Shrinkage Estimator. Read more in the :ref:`User Guide `. - OAS is a particular form of shrinkage described in - "Shrinkage Algorithms for MMSE Covariance Estimation" - Chen et al., IEEE Trans. on Sign. Proc., Volume 58, Issue 10, October 2010. - - The formula used here does not correspond to the one given in the - article. In the original article, formula (23) states that 2/p is - multiplied by Trace(cov*cov) in both the numerator and denominator, but - this operation is omitted because for a large p, the value of 2/p is - so small that it doesn't affect the value of the estimator. - Parameters ---------- store_precision : bool, default=True @@ -538,6 +717,53 @@ class OAS(EmpiricalCovariance): coefficient in the convex combination used for the computation of the shrunk estimate. Range is [0, 1]. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + EllipticEnvelope : An object for detecting outliers in + a Gaussian distributed dataset. + EmpiricalCovariance : Maximum likelihood covariance estimator. + GraphicalLasso : Sparse inverse covariance estimation + with an l1-penalized estimator. + GraphicalLassoCV : Sparse inverse covariance with cross-validated + choice of the l1 penalty. + LedoitWolf : LedoitWolf Estimator. + MinCovDet : Minimum Covariance Determinant + (robust estimator of covariance). + ShrunkCovariance : Covariance estimator with shrinkage. + + Notes + ----- + The regularised covariance is: + + (1 - shrinkage) * cov + shrinkage * mu * np.identity(n_features), + + where mu = trace(cov) / n_features and shrinkage is given by the OAS formula + (see [1]_). + + The shrinkage formulation implemented here differs from Eq. 23 in [1]_. In + the original article, formula (23) states that 2/p (p being the number of + features) is multiplied by Trace(cov*cov) in both the numerator and + denominator, but this operation is omitted because for a large p, the value + of 2/p is so small that it doesn't affect the value of the estimator. + + References + ---------- + .. [1] :arxiv:`"Shrinkage algorithms for MMSE covariance estimation.", + Chen, Y., Wiesel, A., Eldar, Y. C., & Hero, A. O. + IEEE Transactions on Signal Processing, 58(10), 5016-5029, 2010. + <0907.4698>` + Examples -------- >>> import numpy as np @@ -551,32 +777,22 @@ class OAS(EmpiricalCovariance): ... size=500) >>> oas = OAS().fit(X) >>> oas.covariance_ - array([[0.7533..., 0.2763...], - [0.2763..., 0.3964...]]) + array([[0.7533, 0.2763], + [0.2763, 0.3964]]) >>> oas.precision_ - array([[ 1.7833..., -1.2431... ], - [-1.2431..., 3.3889...]]) + array([[ 1.7833, -1.2431 ], + [-1.2431, 3.3889]]) >>> oas.shrinkage_ - 0.0195... - - Notes - ----- - The regularised covariance is: + np.float64(0.0195) - (1 - shrinkage) * cov + shrinkage * mu * np.identity(n_features) - - where mu = trace(cov) / n_features - and shrinkage is given by the OAS formula (see References) - - References - ---------- - "Shrinkage Algorithms for MMSE Covariance Estimation" - Chen et al., IEEE Trans. on Sign. Proc., Volume 58, Issue 10, October 2010. + See also :ref:`sphx_glr_auto_examples_covariance_plot_covariance_estimation.py` + and :ref:`sphx_glr_auto_examples_covariance_plot_lw_vs_oas.py` + for more detailed examples. """ + @_fit_context(prefer_skip_nested_validation=True) def fit(self, X, y=None): - """Fit the Oracle Approximating Shrinkage covariance model - according to the given training data and parameters. + """Fit the Oracle Approximating Shrinkage covariance model to X. Parameters ---------- @@ -589,8 +805,9 @@ def fit(self, X, y=None): Returns ------- self : object + Returns the instance itself. """ - X = self._validate_data(X) + X = validate_data(self, X) # Not calling the parent object to fit, to avoid computing the # covariance matrix (and potentially the precision) if self.assume_centered: @@ -598,7 +815,7 @@ def fit(self, X, y=None): else: self.location_ = X.mean(0) - covariance, shrinkage = oas(X - self.location_, assume_centered=True) + covariance, shrinkage = _oas(X - self.location_, assume_centered=True) self.shrinkage_ = shrinkage self._set_covariance(covariance) diff --git a/sklearn/covariance/tests/test_covariance.py b/sklearn/covariance/tests/test_covariance.py index bcf163e8182d8..9c55012c158e1 100644 --- a/sklearn/covariance/tests/test_covariance.py +++ b/sklearn/covariance/tests/test_covariance.py @@ -1,21 +1,30 @@ -# Author: Alexandre Gramfort -# Gael Varoquaux -# Virgile Fritsch -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import numpy as np import pytest -from sklearn.utils._testing import assert_almost_equal -from sklearn.utils._testing import assert_array_almost_equal -from sklearn.utils._testing import assert_array_equal -from sklearn.utils._testing import assert_warns - from sklearn import datasets -from sklearn.covariance import empirical_covariance, EmpiricalCovariance, \ - ShrunkCovariance, shrunk_covariance, \ - LedoitWolf, ledoit_wolf, ledoit_wolf_shrinkage, OAS, oas +from sklearn.covariance import ( + OAS, + EmpiricalCovariance, + LedoitWolf, + ShrunkCovariance, + empirical_covariance, + ledoit_wolf, + ledoit_wolf_shrinkage, + oas, + shrunk_covariance, +) +from sklearn.covariance._shrunk_covariance import _ledoit_wolf +from sklearn.utils._testing import ( + assert_allclose, + assert_almost_equal, + assert_array_almost_equal, + assert_array_equal, +) + +from .._shrunk_covariance import _oas X, _ = datasets.load_diabetes(return_X_y=True) X_1d = X[:, 0] @@ -30,16 +39,12 @@ def test_covariance(): emp_cov = empirical_covariance(X) assert_array_almost_equal(emp_cov, cov.covariance_, 4) assert_almost_equal(cov.error_norm(emp_cov), 0) - assert_almost_equal( - cov.error_norm(emp_cov, norm='spectral'), 0) - assert_almost_equal( - cov.error_norm(emp_cov, norm='frobenius'), 0) - assert_almost_equal( - cov.error_norm(emp_cov, scaling=False), 0) - assert_almost_equal( - cov.error_norm(emp_cov, squared=False), 0) + assert_almost_equal(cov.error_norm(emp_cov, norm="spectral"), 0) + assert_almost_equal(cov.error_norm(emp_cov, norm="frobenius"), 0) + assert_almost_equal(cov.error_norm(emp_cov, scaling=False), 0) + assert_almost_equal(cov.error_norm(emp_cov, squared=False), 0) with pytest.raises(NotImplementedError): - cov.error_norm(emp_cov, norm='foo') + cov.error_norm(emp_cov, norm="foo") # Mahalanobis distances computation test mahal_dist = cov.mahalanobis(X) assert np.amin(mahal_dist) > 0 @@ -50,16 +55,17 @@ def test_covariance(): cov.fit(X_1d) assert_array_almost_equal(empirical_covariance(X_1d), cov.covariance_, 4) assert_almost_equal(cov.error_norm(empirical_covariance(X_1d)), 0) - assert_almost_equal( - cov.error_norm(empirical_covariance(X_1d), norm='spectral'), 0) + assert_almost_equal(cov.error_norm(empirical_covariance(X_1d), norm="spectral"), 0) # test with one sample # Create X with 1 sample and 5 features X_1sample = np.arange(5).reshape(1, 5) cov = EmpiricalCovariance() - assert_warns(UserWarning, cov.fit, X_1sample) - assert_array_almost_equal(cov.covariance_, - np.zeros(shape=(5, 5), dtype=np.float64)) + warn_msg = "Only one sample available. You may want to reshape your data array" + with pytest.warns(UserWarning, match=warn_msg): + cov.fit(X_1sample) + + assert_array_almost_equal(cov.covariance_, np.zeros(shape=(5, 5), dtype=np.float64)) # test integer type X_integer = np.asarray([[0, 1], [1, 0]]) @@ -72,23 +78,42 @@ def test_covariance(): assert_array_equal(cov.location_, np.zeros(X.shape[1])) +@pytest.mark.parametrize("n_matrices", [1, 3]) +def test_shrunk_covariance_func(n_matrices): + """Check `shrunk_covariance` function.""" + + n_features = 2 + cov = np.ones((n_features, n_features)) + cov_target = np.array([[1, 0.5], [0.5, 1]]) + + if n_matrices > 1: + cov = np.repeat(cov[np.newaxis, ...], n_matrices, axis=0) + cov_target = np.repeat(cov_target[np.newaxis, ...], n_matrices, axis=0) + + cov_shrunk = shrunk_covariance(cov, 0.5) + assert_allclose(cov_shrunk, cov_target) + + def test_shrunk_covariance(): + """Check consistency between `ShrunkCovariance` and `shrunk_covariance`.""" + # Tests ShrunkCovariance module on a simple dataset. # compare shrunk covariance obtained from data and from MLE estimate cov = ShrunkCovariance(shrinkage=0.5) cov.fit(X) assert_array_almost_equal( - shrunk_covariance(empirical_covariance(X), shrinkage=0.5), - cov.covariance_, 4) + shrunk_covariance(empirical_covariance(X), shrinkage=0.5), cov.covariance_, 4 + ) # same test with shrinkage not provided cov = ShrunkCovariance() cov.fit(X) assert_array_almost_equal( - shrunk_covariance(empirical_covariance(X)), cov.covariance_, 4) + shrunk_covariance(empirical_covariance(X)), cov.covariance_, 4 + ) # same test with shrinkage = 0 (<==> empirical_covariance) - cov = ShrunkCovariance(shrinkage=0.) + cov = ShrunkCovariance(shrinkage=0.0) cov.fit(X) assert_array_almost_equal(empirical_covariance(X), cov.covariance_, 4) @@ -101,7 +126,7 @@ def test_shrunk_covariance(): # test shrinkage coeff on a simple data set (without saving precision) cov = ShrunkCovariance(shrinkage=0.5, store_precision=False) cov.fit(X) - assert(cov.precision_ is None) + assert cov.precision_ is None def test_ledoit_wolf(): @@ -113,15 +138,17 @@ def test_ledoit_wolf(): shrinkage_ = lw.shrinkage_ score_ = lw.score(X_centered) - assert_almost_equal(ledoit_wolf_shrinkage(X_centered, - assume_centered=True), - shrinkage_) - assert_almost_equal(ledoit_wolf_shrinkage(X_centered, assume_centered=True, - block_size=6), - shrinkage_) + assert_almost_equal( + ledoit_wolf_shrinkage(X_centered, assume_centered=True), shrinkage_ + ) + assert_almost_equal( + ledoit_wolf_shrinkage(X_centered, assume_centered=True, block_size=6), + shrinkage_, + ) # compare shrunk covariance obtained from data and from MLE estimate - lw_cov_from_mle, lw_shrinkage_from_mle = ledoit_wolf(X_centered, - assume_centered=True) + lw_cov_from_mle, lw_shrinkage_from_mle = ledoit_wolf( + X_centered, assume_centered=True + ) assert_array_almost_equal(lw_cov_from_mle, lw.covariance_, 4) assert_almost_equal(lw_shrinkage_from_mle, lw.shrinkage_) # compare estimates given by LW and ShrunkCovariance @@ -133,17 +160,16 @@ def test_ledoit_wolf(): X_1d = X[:, 0].reshape((-1, 1)) lw = LedoitWolf(assume_centered=True) lw.fit(X_1d) - lw_cov_from_mle, lw_shrinkage_from_mle = ledoit_wolf(X_1d, - assume_centered=True) + lw_cov_from_mle, lw_shrinkage_from_mle = ledoit_wolf(X_1d, assume_centered=True) assert_array_almost_equal(lw_cov_from_mle, lw.covariance_, 4) assert_almost_equal(lw_shrinkage_from_mle, lw.shrinkage_) - assert_array_almost_equal((X_1d ** 2).sum() / n_samples, lw.covariance_, 4) + assert_array_almost_equal((X_1d**2).sum() / n_samples, lw.covariance_, 4) # test shrinkage coeff on a simple data set (without saving precision) lw = LedoitWolf(store_precision=False, assume_centered=True) lw.fit(X_centered) assert_almost_equal(lw.score(X_centered), score_, 4) - assert(lw.precision_ is None) + assert lw.precision_ is None # Same tests without assuming centered data # test shrinkage coeff on a simple data set @@ -152,6 +178,9 @@ def test_ledoit_wolf(): assert_almost_equal(lw.shrinkage_, shrinkage_, 4) assert_almost_equal(lw.shrinkage_, ledoit_wolf_shrinkage(X)) assert_almost_equal(lw.shrinkage_, ledoit_wolf(X)[1]) + assert_almost_equal( + lw.shrinkage_, _ledoit_wolf(X=X, assume_centered=False, block_size=10000)[1] + ) assert_almost_equal(lw.score(X), score_, 4) # compare shrunk covariance obtained from data and from MLE estimate lw_cov_from_mle, lw_shrinkage_from_mle = ledoit_wolf(X) @@ -166,6 +195,10 @@ def test_ledoit_wolf(): X_1d = X[:, 0].reshape((-1, 1)) lw = LedoitWolf() lw.fit(X_1d) + assert_allclose( + X_1d.var(ddof=0), + _ledoit_wolf(X=X_1d, assume_centered=False, block_size=10000)[0], + ) lw_cov_from_mle, lw_shrinkage_from_mle = ledoit_wolf(X_1d) assert_array_almost_equal(lw_cov_from_mle, lw.covariance_, 4) assert_almost_equal(lw_shrinkage_from_mle, lw.shrinkage_) @@ -175,15 +208,18 @@ def test_ledoit_wolf(): # warning should be raised when using only 1 sample X_1sample = np.arange(5).reshape(1, 5) lw = LedoitWolf() - assert_warns(UserWarning, lw.fit, X_1sample) - assert_array_almost_equal(lw.covariance_, - np.zeros(shape=(5, 5), dtype=np.float64)) + + warn_msg = "Only one sample available. You may want to reshape your data array" + with pytest.warns(UserWarning, match=warn_msg): + lw.fit(X_1sample) + + assert_array_almost_equal(lw.covariance_, np.zeros(shape=(5, 5), dtype=np.float64)) # test shrinkage coeff on a simple data set (without saving precision) lw = LedoitWolf(store_precision=False) lw.fit(X) assert_almost_equal(lw.score(X), score_, 4) - assert(lw.precision_ is None) + assert lw.precision_ is None def _naive_ledoit_wolf_shrinkage(X): @@ -197,11 +233,14 @@ def _naive_ledoit_wolf_shrinkage(X): emp_cov = empirical_covariance(X, assume_centered=False) mu = np.trace(emp_cov) / n_features delta_ = emp_cov.copy() - delta_.flat[::n_features + 1] -= mu - delta = (delta_ ** 2).sum() / n_features - X2 = X ** 2 - beta_ = 1. / (n_features * n_samples) \ - * np.sum(np.dot(X2.T, X2) / n_samples - emp_cov ** 2) + delta_.flat[:: n_features + 1] -= mu + delta = (delta_**2).sum() / n_features + X2 = X**2 + beta_ = ( + 1.0 + / (n_features * n_samples) + * np.sum(np.dot(X2.T, X2) / n_samples - emp_cov**2) + ) beta = min(beta_, delta) shrinkage = beta / delta @@ -233,6 +272,16 @@ def test_ledoit_wolf_large(): assert_almost_equal(lw.covariance_, cov) +@pytest.mark.parametrize( + "ledoit_wolf_fitting_function", [LedoitWolf().fit, ledoit_wolf_shrinkage] +) +def test_ledoit_wolf_empty_array(ledoit_wolf_fitting_function): + """Check that we validate X and raise proper error with 0-sample array.""" + X_empty = np.zeros((0, 2)) + with pytest.raises(ValueError, match="Found array with 0 sample"): + ledoit_wolf_fitting_function(X_empty) + + def test_oas(): # Tests OAS module on a simple dataset. # test shrinkage coeff on a simple data set @@ -242,8 +291,7 @@ def test_oas(): shrinkage_ = oa.shrinkage_ score_ = oa.score(X_centered) # compare shrunk covariance obtained from data and from MLE estimate - oa_cov_from_mle, oa_shrinkage_from_mle = oas(X_centered, - assume_centered=True) + oa_cov_from_mle, oa_shrinkage_from_mle = oas(X_centered, assume_centered=True) assert_array_almost_equal(oa_cov_from_mle, oa.covariance_, 4) assert_almost_equal(oa_shrinkage_from_mle, oa.shrinkage_) # compare estimates given by OAS and ShrunkCovariance @@ -258,13 +306,13 @@ def test_oas(): oa_cov_from_mle, oa_shrinkage_from_mle = oas(X_1d, assume_centered=True) assert_array_almost_equal(oa_cov_from_mle, oa.covariance_, 4) assert_almost_equal(oa_shrinkage_from_mle, oa.shrinkage_) - assert_array_almost_equal((X_1d ** 2).sum() / n_samples, oa.covariance_, 4) + assert_array_almost_equal((X_1d**2).sum() / n_samples, oa.covariance_, 4) # test shrinkage coeff on a simple data set (without saving precision) oa = OAS(store_precision=False, assume_centered=True) oa.fit(X_centered) assert_almost_equal(oa.score(X_centered), score_, 4) - assert(oa.precision_ is None) + assert oa.precision_ is None # Same tests without assuming centered data-------------------------------- # test shrinkage coeff on a simple data set @@ -294,22 +342,33 @@ def test_oas(): # warning should be raised when using only 1 sample X_1sample = np.arange(5).reshape(1, 5) oa = OAS() - assert_warns(UserWarning, oa.fit, X_1sample) - assert_array_almost_equal(oa.covariance_, - np.zeros(shape=(5, 5), dtype=np.float64)) + warn_msg = "Only one sample available. You may want to reshape your data array" + with pytest.warns(UserWarning, match=warn_msg): + oa.fit(X_1sample) + + assert_array_almost_equal(oa.covariance_, np.zeros(shape=(5, 5), dtype=np.float64)) # test shrinkage coeff on a simple data set (without saving precision) oa = OAS(store_precision=False) oa.fit(X) assert_almost_equal(oa.score(X), score_, 4) - assert(oa.precision_ is None) + assert oa.precision_ is None + + # test function _oas without assuming centered data + X_1f = X[:, 0:1] + oa = OAS() + oa.fit(X_1f) + # compare shrunk covariance obtained from data and from MLE estimate + _oa_cov_from_mle, _oa_shrinkage_from_mle = _oas(X_1f) + assert_array_almost_equal(_oa_cov_from_mle, oa.covariance_, 4) + assert_almost_equal(_oa_shrinkage_from_mle, oa.shrinkage_) + assert_array_almost_equal((X_1f**2).sum() / n_samples, oa.covariance_, 4) def test_EmpiricalCovariance_validates_mahalanobis(): """Checks that EmpiricalCovariance validates data with mahalanobis.""" cov = EmpiricalCovariance().fit(X) - msg = (f"X has 2 features, but \\w+ is expecting {X.shape[1]} " - "features as input") + msg = f"X has 2 features, but \\w+ is expecting {X.shape[1]} features as input" with pytest.raises(ValueError, match=msg): cov.mahalanobis(X[:, :2]) diff --git a/sklearn/covariance/tests/test_elliptic_envelope.py b/sklearn/covariance/tests/test_elliptic_envelope.py index 676a6c2689bf2..ca85717fb3782 100644 --- a/sklearn/covariance/tests/test_elliptic_envelope.py +++ b/sklearn/covariance/tests/test_elliptic_envelope.py @@ -6,14 +6,16 @@ import pytest from sklearn.covariance import EllipticEnvelope -from sklearn.utils._testing import assert_almost_equal -from sklearn.utils._testing import assert_array_almost_equal -from sklearn.utils._testing import assert_array_equal from sklearn.exceptions import NotFittedError +from sklearn.utils._testing import ( + assert_almost_equal, + assert_array_almost_equal, + assert_array_equal, +) -def test_elliptic_envelope(): - rnd = np.random.RandomState(0) +def test_elliptic_envelope(global_random_seed): + rnd = np.random.RandomState(global_random_seed) X = rnd.randn(100, 10) clf = EllipticEnvelope(contamination=0.1) with pytest.raises(NotFittedError): @@ -25,21 +27,26 @@ def test_elliptic_envelope(): scores = clf.score_samples(X) decisions = clf.decision_function(X) - assert_array_almost_equal( - scores, -clf.mahalanobis(X)) + assert_array_almost_equal(scores, -clf.mahalanobis(X)) assert_array_almost_equal(clf.mahalanobis(X), clf.dist_) - assert_almost_equal(clf.score(X, np.ones(100)), - (100 - y_pred[y_pred == -1].size) / 100.) - assert(sum(y_pred == -1) == sum(decisions < 0)) + assert_almost_equal( + clf.score(X, np.ones(100)), (100 - y_pred[y_pred == -1].size) / 100.0 + ) + assert sum(y_pred == -1) == sum(decisions < 0) def test_score_samples(): X_train = [[1, 1], [1, 2], [2, 1]] clf1 = EllipticEnvelope(contamination=0.2).fit(X_train) clf2 = EllipticEnvelope().fit(X_train) - assert_array_equal(clf1.score_samples([[2., 2.]]), - clf1.decision_function([[2., 2.]]) + clf1.offset_) - assert_array_equal(clf2.score_samples([[2., 2.]]), - clf2.decision_function([[2., 2.]]) + clf2.offset_) - assert_array_equal(clf1.score_samples([[2., 2.]]), - clf2.score_samples([[2., 2.]])) + assert_array_equal( + clf1.score_samples([[2.0, 2.0]]), + clf1.decision_function([[2.0, 2.0]]) + clf1.offset_, + ) + assert_array_equal( + clf2.score_samples([[2.0, 2.0]]), + clf2.decision_function([[2.0, 2.0]]) + clf2.offset_, + ) + assert_array_equal( + clf1.score_samples([[2.0, 2.0]]), clf2.score_samples([[2.0, 2.0]]) + ) diff --git a/sklearn/covariance/tests/test_graphical_lasso.py b/sklearn/covariance/tests/test_graphical_lasso.py index 9bcce6673dd65..9698b64bf4407 100644 --- a/sklearn/covariance/tests/test_graphical_lasso.py +++ b/sklearn/covariance/tests/test_graphical_lasso.py @@ -1,87 +1,125 @@ -""" Test the graphical_lasso module. -""" +"""Test the graphical_lasso module.""" + import sys -import pytest +from io import StringIO import numpy as np -from scipy import linalg - +import pytest from numpy.testing import assert_allclose -from sklearn.utils._testing import assert_array_almost_equal -from sklearn.utils._testing import assert_array_less +from scipy import linalg -from sklearn.covariance import (graphical_lasso, GraphicalLasso, - GraphicalLassoCV, empirical_covariance) +from sklearn import config_context, datasets +from sklearn.covariance import ( + GraphicalLasso, + GraphicalLassoCV, + empirical_covariance, + graphical_lasso, +) from sklearn.datasets import make_sparse_spd_matrix -from io import StringIO +from sklearn.model_selection import GroupKFold from sklearn.utils import check_random_state -from sklearn import datasets +from sklearn.utils._testing import ( + _convert_container, + assert_array_almost_equal, + assert_array_less, +) -def test_graphical_lasso(random_state=0): +def test_graphical_lassos(random_state=1): + """Test the graphical lasso solvers. + + This checks is unstable for some random seeds where the covariance found with "cd" + and "lars" solvers are different (4 cases / 100 tries). + """ # Sample data from a sparse multivariate normal dim = 20 n_samples = 100 random_state = check_random_state(random_state) - prec = make_sparse_spd_matrix(dim, alpha=.95, - random_state=random_state) + prec = make_sparse_spd_matrix(dim, alpha=0.95, random_state=random_state) cov = linalg.inv(prec) X = random_state.multivariate_normal(np.zeros(dim), cov, size=n_samples) emp_cov = empirical_covariance(X) - for alpha in (0., .1, .25): + for alpha in (0.0, 0.1, 0.25): covs = dict() icovs = dict() - for method in ('cd', 'lars'): - cov_, icov_, costs = graphical_lasso(emp_cov, return_costs=True, - alpha=alpha, mode=method) + for method in ("cd", "lars"): + cov_, icov_, costs = graphical_lasso( + emp_cov, return_costs=True, alpha=alpha, mode=method + ) covs[method] = cov_ icovs[method] = icov_ costs, dual_gap = np.array(costs).T # Check that the costs always decrease (doesn't hold if alpha == 0) if not alpha == 0: - assert_array_less(np.diff(costs), 0) + # use 1e-12 since the cost can be exactly 0 + assert_array_less(np.diff(costs), 1e-12) # Check that the 2 approaches give similar results - assert_array_almost_equal(covs['cd'], covs['lars'], decimal=4) - assert_array_almost_equal(icovs['cd'], icovs['lars'], decimal=4) + assert_allclose(covs["cd"], covs["lars"], atol=5e-4) + assert_allclose(icovs["cd"], icovs["lars"], atol=5e-4) # Smoke test the estimator - model = GraphicalLasso(alpha=.25).fit(X) + model = GraphicalLasso(alpha=0.25).fit(X) model.score(X) - assert_array_almost_equal(model.covariance_, covs['cd'], decimal=4) - assert_array_almost_equal(model.covariance_, covs['lars'], decimal=4) + assert_array_almost_equal(model.covariance_, covs["cd"], decimal=4) + assert_array_almost_equal(model.covariance_, covs["lars"], decimal=4) # For a centered matrix, assume_centered could be chosen True or False # Check that this returns indeed the same result for centered data Z = X - X.mean(0) precs = list() for assume_centered in (False, True): - prec_ = GraphicalLasso( - assume_centered=assume_centered).fit(Z).precision_ + prec_ = GraphicalLasso(assume_centered=assume_centered).fit(Z).precision_ precs.append(prec_) assert_array_almost_equal(precs[0], precs[1]) +def test_graphical_lasso_when_alpha_equals_0(): + """Test graphical_lasso's early return condition when alpha=0.""" + X = np.random.randn(100, 10) + emp_cov = empirical_covariance(X, assume_centered=True) + + model = GraphicalLasso(alpha=0, covariance="precomputed").fit(emp_cov) + assert_allclose(model.precision_, np.linalg.inv(emp_cov)) + + _, precision = graphical_lasso(emp_cov, alpha=0) + assert_allclose(precision, np.linalg.inv(emp_cov)) + + +@pytest.mark.parametrize("mode", ["cd", "lars"]) +def test_graphical_lasso_n_iter(mode): + X, _ = datasets.make_classification(n_samples=5_000, n_features=20, random_state=0) + emp_cov = empirical_covariance(X) + + _, _, n_iter = graphical_lasso( + emp_cov, 0.2, mode=mode, max_iter=2, return_n_iter=True + ) + assert n_iter == 2 + + def test_graphical_lasso_iris(): # Hard-coded solution from R glasso package for alpha=1.0 # (need to set penalize.diagonal to FALSE) - cov_R = np.array([ - [0.68112222, 0.0000000, 0.265820, 0.02464314], - [0.00000000, 0.1887129, 0.000000, 0.00000000], - [0.26582000, 0.0000000, 3.095503, 0.28697200], - [0.02464314, 0.0000000, 0.286972, 0.57713289] - ]) - icov_R = np.array([ - [1.5190747, 0.000000, -0.1304475, 0.0000000], - [0.0000000, 5.299055, 0.0000000, 0.0000000], - [-0.1304475, 0.000000, 0.3498624, -0.1683946], - [0.0000000, 0.000000, -0.1683946, 1.8164353] - ]) + cov_R = np.array( + [ + [0.68112222, 0.0000000, 0.265820, 0.02464314], + [0.00000000, 0.1887129, 0.000000, 0.00000000], + [0.26582000, 0.0000000, 3.095503, 0.28697200], + [0.02464314, 0.0000000, 0.286972, 0.57713289], + ] + ) + icov_R = np.array( + [ + [1.5190747, 0.000000, -0.1304475, 0.0000000], + [0.0000000, 5.299055, 0.0000000, 0.0000000], + [-0.1304475, 0.000000, 0.3498624, -0.1683946], + [0.0000000, 0.000000, -0.1683946, 1.8164353], + ] + ) X = datasets.load_iris().data emp_cov = empirical_covariance(X) - for method in ('cd', 'lars'): - cov, icov = graphical_lasso(emp_cov, alpha=1.0, return_costs=False, - mode=method) + for method in ("cd", "lars"): + cov, icov = graphical_lasso(emp_cov, alpha=1.0, return_costs=False, mode=method) assert_array_almost_equal(cov, cov_R) assert_array_almost_equal(icov, icov_R) @@ -89,16 +127,13 @@ def test_graphical_lasso_iris(): def test_graph_lasso_2D(): # Hard-coded solution from Python skggm package # obtained by calling `quic(emp_cov, lam=.1, tol=1e-8)` - cov_skggm = np.array([[3.09550269, 1.186972], - [1.186972, 0.57713289]]) + cov_skggm = np.array([[3.09550269, 1.186972], [1.186972, 0.57713289]]) - icov_skggm = np.array([[1.52836773, -3.14334831], - [-3.14334831, 8.19753385]]) + icov_skggm = np.array([[1.52836773, -3.14334831], [-3.14334831, 8.19753385]]) X = datasets.load_iris().data[:, 2:] emp_cov = empirical_covariance(X) - for method in ('cd', 'lars'): - cov, icov = graphical_lasso(emp_cov, alpha=.1, return_costs=False, - mode=method) + for method in ("cd", "lars"): + cov, icov = graphical_lasso(emp_cov, alpha=0.1, return_costs=False, mode=method) assert_array_almost_equal(cov, cov_skggm) assert_array_almost_equal(icov, icov_skggm) @@ -109,23 +144,28 @@ def test_graphical_lasso_iris_singular(): indices = np.arange(10, 13) # Hard-coded solution from R glasso package for alpha=0.01 - cov_R = np.array([ - [0.08, 0.056666662595, 0.00229729713223, 0.00153153142149], - [0.056666662595, 0.082222222222, 0.00333333333333, 0.00222222222222], - [0.002297297132, 0.003333333333, 0.00666666666667, 0.00009009009009], - [0.001531531421, 0.002222222222, 0.00009009009009, 0.00222222222222] - ]) - icov_R = np.array([ - [24.42244057, -16.831679593, 0.0, 0.0], - [-16.83168201, 24.351841681, -6.206896552, -12.5], - [0.0, -6.206896171, 153.103448276, 0.0], - [0.0, -12.499999143, 0.0, 462.5] - ]) + cov_R = np.array( + [ + [0.08, 0.056666662595, 0.00229729713223, 0.00153153142149], + [0.056666662595, 0.082222222222, 0.00333333333333, 0.00222222222222], + [0.002297297132, 0.003333333333, 0.00666666666667, 0.00009009009009], + [0.001531531421, 0.002222222222, 0.00009009009009, 0.00222222222222], + ] + ) + icov_R = np.array( + [ + [24.42244057, -16.831679593, 0.0, 0.0], + [-16.83168201, 24.351841681, -6.206896552, -12.5], + [0.0, -6.206896171, 153.103448276, 0.0], + [0.0, -12.499999143, 0.0, 462.5], + ] + ) X = datasets.load_iris().data[indices, :] emp_cov = empirical_covariance(X) - for method in ('cd', 'lars'): - cov, icov = graphical_lasso(emp_cov, alpha=0.01, return_costs=False, - mode=method) + for method in ("cd", "lars"): + cov, icov = graphical_lasso( + emp_cov, alpha=0.01, return_costs=False, mode=method + ) assert_array_almost_equal(cov, cov_R, decimal=5) assert_array_almost_equal(icov, icov_R, decimal=5) @@ -135,8 +175,7 @@ def test_graphical_lasso_cv(random_state=1): dim = 5 n_samples = 6 random_state = check_random_state(random_state) - prec = make_sparse_spd_matrix(dim, alpha=.96, - random_state=random_state) + prec = make_sparse_spd_matrix(dim, alpha=0.96, random_state=random_state) cov = linalg.inv(prec) X = random_state.multivariate_normal(np.zeros(dim), cov, size=n_samples) # Capture stdout, to smoke test the verbose mode @@ -148,57 +187,125 @@ def test_graphical_lasso_cv(random_state=1): finally: sys.stdout = orig_stdout - # Smoke test with specified alphas - GraphicalLassoCV(alphas=[0.8, 0.5], tol=1e-1, n_jobs=1).fit(X) +@pytest.mark.parametrize("alphas_container_type", ["list", "tuple", "array"]) +def test_graphical_lasso_cv_alphas_iterable(alphas_container_type): + """Check that we can pass an array-like to `alphas`. -# TODO: Remove in 1.1 when grid_scores_ is deprecated -def test_graphical_lasso_cv_grid_scores_and_cv_alphas_deprecated(): - splits = 4 - n_alphas = 5 - n_refinements = 3 - true_cov = np.array([[0.8, 0.0, 0.2, 0.0], - [0.0, 0.4, 0.0, 0.0], - [0.2, 0.0, 0.3, 0.1], - [0.0, 0.0, 0.1, 0.7]]) + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/22489 + """ + true_cov = np.array( + [ + [0.8, 0.0, 0.2, 0.0], + [0.0, 0.4, 0.0, 0.0], + [0.2, 0.0, 0.3, 0.1], + [0.0, 0.0, 0.1, 0.7], + ] + ) rng = np.random.RandomState(0) X = rng.multivariate_normal(mean=[0, 0, 0, 0], cov=true_cov, size=200) - cov = GraphicalLassoCV(cv=splits, alphas=n_alphas, - n_refinements=n_refinements).fit(X) + alphas = _convert_container([0.02, 0.03], alphas_container_type) + GraphicalLassoCV(alphas=alphas, tol=1e-1, n_jobs=1).fit(X) - total_alphas = n_refinements * n_alphas + 1 - msg = (r"The grid_scores_ attribute is deprecated in version 0\.24 in " - r"favor of cv_results_ and will be removed in version 1\.1 " - r"\(renaming of 0\.26\).") - with pytest.warns(FutureWarning, match=msg): - assert cov.grid_scores_.shape == (total_alphas, splits) - msg = (r"The cv_alphas_ attribute is deprecated in version 0\.24 in " - r"favor of cv_results_\['alpha'\] and will be removed in version " - r"1\.1 \(renaming of 0\.26\)") - with pytest.warns(FutureWarning, match=msg): - assert len(cov.cv_alphas_) == total_alphas +@pytest.mark.parametrize( + "alphas,err_type,err_msg", + [ + ([-0.02, 0.03], ValueError, "must be > 0"), + ([0, 0.03], ValueError, "must be > 0"), + (["not_number", 0.03], TypeError, "must be an instance of float"), + ], +) +def test_graphical_lasso_cv_alphas_invalid_array(alphas, err_type, err_msg): + """Check that if an array-like containing a value + outside of (0, inf] is passed to `alphas`, a ValueError is raised. + Check if a string is passed, a TypeError is raised. + """ + true_cov = np.array( + [ + [0.8, 0.0, 0.2, 0.0], + [0.0, 0.4, 0.0, 0.0], + [0.2, 0.0, 0.3, 0.1], + [0.0, 0.0, 0.1, 0.7], + ] + ) + rng = np.random.RandomState(0) + X = rng.multivariate_normal(mean=[0, 0, 0, 0], cov=true_cov, size=200) + + with pytest.raises(err_type, match=err_msg): + GraphicalLassoCV(alphas=alphas, tol=1e-1, n_jobs=1).fit(X) def test_graphical_lasso_cv_scores(): splits = 4 n_alphas = 5 n_refinements = 3 - true_cov = np.array([[0.8, 0.0, 0.2, 0.0], - [0.0, 0.4, 0.0, 0.0], - [0.2, 0.0, 0.3, 0.1], - [0.0, 0.0, 0.1, 0.7]]) + true_cov = np.array( + [ + [0.8, 0.0, 0.2, 0.0], + [0.0, 0.4, 0.0, 0.0], + [0.2, 0.0, 0.3, 0.1], + [0.0, 0.0, 0.1, 0.7], + ] + ) rng = np.random.RandomState(0) X = rng.multivariate_normal(mean=[0, 0, 0, 0], cov=true_cov, size=200) - cov = GraphicalLassoCV(cv=splits, alphas=n_alphas, - n_refinements=n_refinements).fit(X) + cov = GraphicalLassoCV(cv=splits, alphas=n_alphas, n_refinements=n_refinements).fit( + X + ) + + _assert_graphical_lasso_cv_scores( + cov=cov, + n_splits=splits, + n_refinements=n_refinements, + n_alphas=n_alphas, + ) + + +@config_context(enable_metadata_routing=True) +def test_graphical_lasso_cv_scores_with_routing(global_random_seed): + """Check that `GraphicalLassoCV` internally dispatches metadata to + the splitter. + """ + splits = 5 + n_alphas = 5 + n_refinements = 3 + true_cov = np.array( + [ + [0.8, 0.0, 0.2, 0.0], + [0.0, 0.4, 0.0, 0.0], + [0.2, 0.0, 0.3, 0.1], + [0.0, 0.0, 0.1, 0.7], + ] + ) + rng = np.random.RandomState(global_random_seed) + X = rng.multivariate_normal(mean=[0, 0, 0, 0], cov=true_cov, size=300) + n_samples = X.shape[0] + groups = rng.randint(0, 5, n_samples) + params = {"groups": groups} + cv = GroupKFold(n_splits=splits) + cv.set_split_request(groups=True) + + cov = GraphicalLassoCV(cv=cv, alphas=n_alphas, n_refinements=n_refinements).fit( + X, **params + ) + + _assert_graphical_lasso_cv_scores( + cov=cov, + n_splits=splits, + n_refinements=n_refinements, + n_alphas=n_alphas, + ) + +def _assert_graphical_lasso_cv_scores(cov, n_splits, n_refinements, n_alphas): cv_results = cov.cv_results_ # alpha and one for each split total_alphas = n_refinements * n_alphas + 1 - keys = ['alphas'] - split_keys = ['split{}_score'.format(i) for i in range(splits)] + keys = ["alphas"] + split_keys = [f"split{i}_test_score" for i in range(n_splits)] for key in keys + split_keys: assert key in cv_results assert len(cv_results[key]) == total_alphas @@ -207,5 +314,5 @@ def test_graphical_lasso_cv_scores(): expected_mean = cv_scores.mean(axis=0) expected_std = cv_scores.std(axis=0) - assert_allclose(cov.cv_results_["mean_score"], expected_mean) - assert_allclose(cov.cv_results_["std_score"], expected_std) + assert_allclose(cov.cv_results_["mean_test_score"], expected_mean) + assert_allclose(cov.cv_results_["std_test_score"], expected_std) diff --git a/sklearn/covariance/tests/test_robust_covariance.py b/sklearn/covariance/tests/test_robust_covariance.py index 55100702bd365..a7bd3996b9e4b 100644 --- a/sklearn/covariance/tests/test_robust_covariance.py +++ b/sklearn/covariance/tests/test_robust_covariance.py @@ -1,84 +1,82 @@ -# Author: Alexandre Gramfort -# Gael Varoquaux -# Virgile Fritsch -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import itertools import numpy as np - -from sklearn.utils._testing import assert_array_almost_equal -from sklearn.utils._testing import assert_raise_message -from sklearn.utils._testing import assert_warns_message +import pytest from sklearn import datasets -from sklearn.covariance import empirical_covariance, MinCovDet -from sklearn.covariance import fast_mcd +from sklearn.covariance import MinCovDet, empirical_covariance, fast_mcd +from sklearn.utils._testing import assert_array_almost_equal X = datasets.load_iris().data X_1d = X[:, 0] n_samples, n_features = X.shape -def test_mcd(): +def test_mcd(global_random_seed): # Tests the FastMCD algorithm implementation # Small data set # test without outliers (random independent normal data) - launch_mcd_on_dataset(100, 5, 0, 0.01, 0.1, 80) + launch_mcd_on_dataset(100, 5, 0, 0.02, 0.1, 75, global_random_seed) # test with a contaminated data set (medium contamination) - launch_mcd_on_dataset(100, 5, 20, 0.01, 0.01, 70) + launch_mcd_on_dataset(100, 5, 20, 0.3, 0.3, 65, global_random_seed) # test with a contaminated data set (strong contamination) - launch_mcd_on_dataset(100, 5, 40, 0.1, 0.1, 50) + launch_mcd_on_dataset(100, 5, 40, 0.1, 0.1, 50, global_random_seed) # Medium data set - launch_mcd_on_dataset(1000, 5, 450, 0.1, 0.1, 540) + launch_mcd_on_dataset(1000, 5, 450, 0.1, 0.1, 540, global_random_seed) # Large data set - launch_mcd_on_dataset(1700, 5, 800, 0.1, 0.1, 870) + launch_mcd_on_dataset(1700, 5, 800, 0.1, 0.1, 870, global_random_seed) # 1D data set - launch_mcd_on_dataset(500, 1, 100, 0.001, 0.001, 350) + launch_mcd_on_dataset(500, 1, 100, 0.02, 0.02, 350, global_random_seed) + + # n_samples == n_features + launch_mcd_on_dataset(20, 20, 0, 0.1, 0.1, 15, global_random_seed) def test_fast_mcd_on_invalid_input(): X = np.arange(100) - assert_raise_message(ValueError, 'Expected 2D array, got 1D array instead', - fast_mcd, X) + msg = "Expected 2D array, got 1D array instead" + with pytest.raises(ValueError, match=msg): + fast_mcd(X) def test_mcd_class_on_invalid_input(): X = np.arange(100) mcd = MinCovDet() - assert_raise_message(ValueError, 'Expected 2D array, got 1D array instead', - mcd.fit, X) - + msg = "Expected 2D array, got 1D array instead" + with pytest.raises(ValueError, match=msg): + mcd.fit(X) -def launch_mcd_on_dataset(n_samples, n_features, n_outliers, tol_loc, tol_cov, - tol_support): - rand_gen = np.random.RandomState(0) +def launch_mcd_on_dataset( + n_samples, n_features, n_outliers, tol_loc, tol_cov, tol_support, seed +): + rand_gen = np.random.RandomState(seed) data = rand_gen.randn(n_samples, n_features) # add some outliers outliers_index = rand_gen.permutation(n_samples)[:n_outliers] - outliers_offset = 10. * \ - (rand_gen.randint(2, size=(n_outliers, n_features)) - 0.5) + outliers_offset = 10.0 * (rand_gen.randint(2, size=(n_outliers, n_features)) - 0.5) data[outliers_index] += outliers_offset inliers_mask = np.ones(n_samples).astype(bool) inliers_mask[outliers_index] = False pure_data = data[inliers_mask] # compute MCD by fitting an object - mcd_fit = MinCovDet(random_state=rand_gen).fit(data) + mcd_fit = MinCovDet(random_state=seed).fit(data) T = mcd_fit.location_ S = mcd_fit.covariance_ H = mcd_fit.support_ # compare with the estimates learnt from the inliers error_location = np.mean((pure_data.mean(0) - T) ** 2) - assert(error_location < tol_loc) + assert error_location < tol_loc error_cov = np.mean((empirical_covariance(pure_data) - S) ** 2) - assert(error_cov < tol_cov) - assert(np.sum(H) >= tol_support) + assert error_cov < tol_cov + assert np.sum(H) >= tol_support assert_array_almost_equal(mcd_fit.mahalanobis(data), mcd_fit.dist_) @@ -91,10 +89,10 @@ def test_mcd_issue1127(): mcd.fit(X) -def test_mcd_issue3367(): +def test_mcd_issue3367(global_random_seed): # Check that MCD completes when the covariance matrix is singular # i.e. one of the rows and columns are all zeros - rand_gen = np.random.RandomState(0) + rand_gen = np.random.RandomState(global_random_seed) # Think of these as the values for X and Y -> 10 values between -5 and 5 data_values = np.linspace(-5, 5, 10).tolist() @@ -130,39 +128,44 @@ def test_mcd_support_covariance_is_zero(): X_1 = X_1.reshape(-1, 1) X_2 = np.array([0.5, 0.3, 0.3, 0.3, 0.957, 0.3, 0.3, 0.3, 0.4285, 0.3]) X_2 = X_2.reshape(-1, 1) - msg = ('The covariance matrix of the support data is equal to 0, try to ' - 'increase support_fraction') + msg = ( + "The covariance matrix of the support data is equal to 0, try to " + "increase support_fraction" + ) for X in [X_1, X_2]: - assert_raise_message(ValueError, msg, MinCovDet().fit, X) + with pytest.raises(ValueError, match=msg): + MinCovDet().fit(X) -def test_mcd_increasing_det_warning(): +def test_mcd_increasing_det_warning(global_random_seed): # Check that a warning is raised if we observe increasing determinants # during the c_step. In theory the sequence of determinants should be # decreasing. Increasing determinants are likely due to ill-conditioned # covariance matrices that result in poor precision matrices. - X = [[5.1, 3.5, 1.4, 0.2], - [4.9, 3.0, 1.4, 0.2], - [4.7, 3.2, 1.3, 0.2], - [4.6, 3.1, 1.5, 0.2], - [5.0, 3.6, 1.4, 0.2], - [4.6, 3.4, 1.4, 0.3], - [5.0, 3.4, 1.5, 0.2], - [4.4, 2.9, 1.4, 0.2], - [4.9, 3.1, 1.5, 0.1], - [5.4, 3.7, 1.5, 0.2], - [4.8, 3.4, 1.6, 0.2], - [4.8, 3.0, 1.4, 0.1], - [4.3, 3.0, 1.1, 0.1], - [5.1, 3.5, 1.4, 0.3], - [5.7, 3.8, 1.7, 0.3], - [5.4, 3.4, 1.7, 0.2], - [4.6, 3.6, 1.0, 0.2], - [5.0, 3.0, 1.6, 0.2], - [5.2, 3.5, 1.5, 0.2]] - - mcd = MinCovDet(random_state=1) - assert_warns_message(RuntimeWarning, - "Determinant has increased", - mcd.fit, X) + X = [ + [5.1, 3.5, 1.4, 0.2], + [4.9, 3.0, 1.4, 0.2], + [4.7, 3.2, 1.3, 0.2], + [4.6, 3.1, 1.5, 0.2], + [5.0, 3.6, 1.4, 0.2], + [4.6, 3.4, 1.4, 0.3], + [5.0, 3.4, 1.5, 0.2], + [4.4, 2.9, 1.4, 0.2], + [4.9, 3.1, 1.5, 0.1], + [5.4, 3.7, 1.5, 0.2], + [4.8, 3.4, 1.6, 0.2], + [4.8, 3.0, 1.4, 0.1], + [4.3, 3.0, 1.1, 0.1], + [5.1, 3.5, 1.4, 0.3], + [5.7, 3.8, 1.7, 0.3], + [5.4, 3.4, 1.7, 0.2], + [4.6, 3.6, 1.0, 0.2], + [5.0, 3.0, 1.6, 0.2], + [5.2, 3.5, 1.5, 0.2], + ] + + mcd = MinCovDet(support_fraction=0.5, random_state=global_random_seed) + warn_msg = "Determinant has increased" + with pytest.warns(RuntimeWarning, match=warn_msg): + mcd.fit(X) diff --git a/sklearn/cross_decomposition/__init__.py b/sklearn/cross_decomposition/__init__.py index bf01b9840c902..f78f33811e5c7 100644 --- a/sklearn/cross_decomposition/__init__.py +++ b/sklearn/cross_decomposition/__init__.py @@ -1,3 +1,8 @@ -from ._pls import PLSCanonical, PLSRegression, PLSSVD, CCA +"""Algorithms for cross decomposition.""" -__all__ = ['PLSCanonical', 'PLSRegression', 'PLSSVD', 'CCA'] +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from ._pls import CCA, PLSSVD, PLSCanonical, PLSRegression + +__all__ = ["CCA", "PLSSVD", "PLSCanonical", "PLSRegression"] diff --git a/sklearn/cross_decomposition/_pls.py b/sklearn/cross_decomposition/_pls.py index 42d727b9ae2be..0bf6ec8f01d06 100644 --- a/sklearn/cross_decomposition/_pls.py +++ b/sklearn/cross_decomposition/_pls.py @@ -2,50 +2,78 @@ The :mod:`sklearn.pls` module implements Partial Least Squares (PLS). """ -# Author: Edouard Duchesnay -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import warnings from abc import ABCMeta, abstractmethod +from numbers import Integral, Real import numpy as np -from scipy.linalg import pinv2, svd - -from ..base import BaseEstimator, RegressorMixin, TransformerMixin -from ..base import MultiOutputMixin +from scipy.linalg import pinv, svd + +from ..base import ( + BaseEstimator, + ClassNamePrefixFeaturesOutMixin, + MultiOutputMixin, + RegressorMixin, + TransformerMixin, + _fit_context, +) +from ..exceptions import ConvergenceWarning from ..utils import check_array, check_consistent_length +from ..utils._param_validation import Interval, StrOptions from ..utils.extmath import svd_flip -from ..utils.validation import check_is_fitted, FLOAT_DTYPES -from ..utils.validation import _deprecate_positional_args -from ..exceptions import ConvergenceWarning -from ..utils.deprecation import deprecated +from ..utils.validation import FLOAT_DTYPES, check_is_fitted, validate_data + +__all__ = ["PLSSVD", "PLSCanonical", "PLSRegression"] + -__all__ = ['PLSCanonical', 'PLSRegression', 'PLSSVD'] +def _pinv2_old(a): + # Used previous scipy pinv2 that was updated in: + # https://github.com/scipy/scipy/pull/10067 + # We can not set `cond` or `rcond` for pinv2 in scipy >= 1.3 to keep the + # same behavior of pinv2 for scipy < 1.3, because the condition used to + # determine the rank is dependent on the output of svd. + u, s, vh = svd(a, full_matrices=False, check_finite=False) + t = u.dtype.char.lower() + factor = {"f": 1e3, "d": 1e6} + cond = np.max(s) * factor[t] * np.finfo(t).eps + rank = np.sum(s > cond) -def _get_first_singular_vectors_power_method(X, Y, mode="A", max_iter=500, - tol=1e-06, norm_y_weights=False): - """Return the first left and right singular vectors of X'Y. + u = u[:, :rank] + u /= s[:rank] + return np.transpose(np.conjugate(np.dot(u, vh[:rank]))) - Provides an alternative to the svd(X'Y) and uses the power method instead. + +def _get_first_singular_vectors_power_method( + X, y, mode="A", max_iter=500, tol=1e-06, norm_y_weights=False +): + """Return the first left and right singular vectors of X'y. + + Provides an alternative to the svd(X'y) and uses the power method instead. With norm_y_weights to True and in mode A, this corresponds to the algorithm section 11.3 of the Wegelin's review, except this starts at the "update saliences" part. """ eps = np.finfo(X.dtype).eps - y_score = next(col for col in Y.T if np.any(np.abs(col) > eps)) + try: + y_score = next(col for col in y.T if np.any(np.abs(col) > eps)) + except StopIteration as e: + raise StopIteration("y residual is constant") from e + x_weights_old = 100 # init to big value for first convergence check - if mode == 'B': + if mode == "B": # Precompute pseudo inverse matrices # Basically: X_pinv = (X.T X)^-1 X.T # Which requires inverting a (n_features, n_features) matrix. # As a result, and as detailed in the Wegelin's review, CCA (i.e. mode # B) will be unstable if n_features > n_samples or n_targets > # n_samples - X_pinv = pinv2(X, check_finite=False, cond=10*eps) - Y_pinv = pinv2(Y, check_finite=False, cond=10*eps) + X_pinv, y_pinv = _pinv2_old(X), _pinv2_old(y) for i in range(max_iter): if mode == "B": @@ -57,62 +85,61 @@ def _get_first_singular_vectors_power_method(X, Y, mode="A", max_iter=500, x_score = np.dot(X, x_weights) if mode == "B": - y_weights = np.dot(Y_pinv, x_score) + y_weights = np.dot(y_pinv, x_score) else: - y_weights = np.dot(Y.T, x_score) / np.dot(x_score.T, x_score) + y_weights = np.dot(y.T, x_score) / np.dot(x_score.T, x_score) if norm_y_weights: y_weights /= np.sqrt(np.dot(y_weights, y_weights)) + eps - y_score = np.dot(Y, y_weights) / (np.dot(y_weights, y_weights) + eps) + y_score = np.dot(y, y_weights) / (np.dot(y_weights, y_weights) + eps) x_weights_diff = x_weights - x_weights_old - if np.dot(x_weights_diff, x_weights_diff) < tol or Y.shape[1] == 1: + if np.dot(x_weights_diff, x_weights_diff) < tol or y.shape[1] == 1: break x_weights_old = x_weights n_iter = i + 1 if n_iter == max_iter: - warnings.warn('Maximum number of iterations reached', - ConvergenceWarning) + warnings.warn("Maximum number of iterations reached", ConvergenceWarning) return x_weights, y_weights, n_iter -def _get_first_singular_vectors_svd(X, Y): - """Return the first left and right singular vectors of X'Y. +def _get_first_singular_vectors_svd(X, y): + """Return the first left and right singular vectors of X'y. Here the whole SVD is computed. """ - C = np.dot(X.T, Y) + C = np.dot(X.T, y) U, _, Vt = svd(C, full_matrices=False) return U[:, 0], Vt[0, :] -def _center_scale_xy(X, Y, scale=True): - """ Center X, Y and scale if the scale parameter==True +def _center_scale_xy(X, y, scale=True): + """Center X, y and scale if the scale parameter==True Returns ------- - X, Y, x_mean, y_mean, x_std, y_std + X, y, x_mean, y_mean, x_std, y_std """ # center x_mean = X.mean(axis=0) X -= x_mean - y_mean = Y.mean(axis=0) - Y -= y_mean + y_mean = y.mean(axis=0) + y -= y_mean # scale if scale: x_std = X.std(axis=0, ddof=1) x_std[x_std == 0.0] = 1.0 X /= x_std - y_std = Y.std(axis=0, ddof=1) + y_std = y.std(axis=0, ddof=1) y_std[y_std == 0.0] = 1.0 - Y /= y_std + y /= y_std else: x_std = np.ones(X.shape[1]) - y_std = np.ones(Y.shape[1]) - return X, Y, x_mean, y_mean, x_std, y_std + y_std = np.ones(y.shape[1]) + return X, y, x_mean, y_mean, x_std, y_std def _svd_flip_1d(u, v): @@ -125,22 +152,47 @@ def _svd_flip_1d(u, v): v *= sign -class _PLS(TransformerMixin, RegressorMixin, MultiOutputMixin, BaseEstimator, - metaclass=ABCMeta): +class _PLS( + ClassNamePrefixFeaturesOutMixin, + TransformerMixin, + RegressorMixin, + MultiOutputMixin, + BaseEstimator, + metaclass=ABCMeta, +): """Partial Least Squares (PLS) This class implements the generic PLS algorithm. Main ref: Wegelin, a survey of Partial Least Squares (PLS) methods, with emphasis on the two-block case - https://www.stat.washington.edu/research/reports/2000/tr371.pdf + https://stat.uw.edu/sites/default/files/files/reports/2000/tr371.pdf """ + _parameter_constraints: dict = { + "n_components": [Interval(Integral, 1, None, closed="left")], + "scale": ["boolean"], + "deflation_mode": [StrOptions({"regression", "canonical"})], + "mode": [StrOptions({"A", "B"})], + "algorithm": [StrOptions({"svd", "nipals"})], + "max_iter": [Interval(Integral, 1, None, closed="left")], + "tol": [Interval(Real, 0, None, closed="left")], + "copy": ["boolean"], + } + @abstractmethod - def __init__(self, n_components=2, *, scale=True, - deflation_mode="regression", - mode="A", algorithm="nipals", max_iter=500, tol=1e-06, - copy=True): + def __init__( + self, + n_components=2, + *, + scale=True, + deflation_mode="regression", + mode="A", + algorithm="nipals", + max_iter=500, + tol=1e-06, + copy=True, + ): self.n_components = n_components self.deflation_mode = deflation_mode self.mode = mode @@ -150,7 +202,8 @@ def __init__(self, n_components=2, *, scale=True, self.tol = tol self.copy = copy - def fit(self, X, Y): + @_fit_context(prefer_skip_nested_validation=True) + def fit(self, X, y): """Fit model to data. Parameters @@ -159,65 +212,62 @@ def fit(self, X, Y): Training vectors, where `n_samples` is the number of samples and `n_features` is the number of predictors. - Y : array-like of shape (n_samples,) or (n_samples, n_targets) + y : array-like of shape (n_samples,) or (n_samples, n_targets) Target vectors, where `n_samples` is the number of samples and `n_targets` is the number of response variables. - """ - check_consistent_length(X, Y) - X = self._validate_data(X, dtype=np.float64, copy=self.copy, - ensure_min_samples=2) - Y = check_array(Y, dtype=np.float64, copy=self.copy, ensure_2d=False) - if Y.ndim == 1: - Y = Y.reshape(-1, 1) + Returns + ------- + self : object + Fitted model. + """ + check_consistent_length(X, y) + X = validate_data( + self, + X, + dtype=np.float64, + force_writeable=True, + copy=self.copy, + ensure_min_samples=2, + ) + y = check_array( + y, + input_name="y", + dtype=np.float64, + force_writeable=True, + copy=self.copy, + ensure_2d=False, + ) + if y.ndim == 1: + self._predict_1d = True + y = y.reshape(-1, 1) + else: + self._predict_1d = False n = X.shape[0] p = X.shape[1] - q = Y.shape[1] + q = y.shape[1] n_components = self.n_components - if self.deflation_mode == 'regression': - # With PLSRegression n_components is bounded by the rank of (X.T X) - # see Wegelin page 25 - rank_upper_bound = p - if not 1 <= n_components <= rank_upper_bound: - # TODO: raise an error in 1.1 - warnings.warn( - f"As of version 0.24, n_components({n_components}) should " - f"be in [1, n_features]." - f"n_components={rank_upper_bound} will be used instead. " - f"In version 1.1 (renaming of 0.26), an error will be " - f"raised.", - FutureWarning - ) - n_components = rank_upper_bound - else: - # With CCA and PLSCanonical, n_components is bounded by the rank of - # X and the rank of Y: see Wegelin page 12 - rank_upper_bound = min(n, p, q) - if not 1 <= self.n_components <= rank_upper_bound: - # TODO: raise an error in 1.1 - warnings.warn( - f"As of version 0.24, n_components({n_components}) should " - f"be in [1, min(n_features, n_samples, n_targets)] = " - f"[1, {rank_upper_bound}]. " - f"n_components={rank_upper_bound} will be used instead. " - f"In version 1.1 (renaming of 0.26), an error will be " - f"raised.", - FutureWarning - ) - n_components = rank_upper_bound - - if self.algorithm not in ("svd", "nipals"): - raise ValueError("algorithm should be 'svd' or 'nipals', got " - f"{self.algorithm}.") - - self._norm_y_weights = (self.deflation_mode == 'canonical') # 1.1 + # With PLSRegression n_components is bounded by the rank of (X.T X) see + # Wegelin page 25. With CCA and PLSCanonical, n_components is bounded + # by the rank of X and the rank of y: see Wegelin page 12 + rank_upper_bound = ( + min(n, p) if self.deflation_mode == "regression" else min(n, p, q) + ) + if n_components > rank_upper_bound: + raise ValueError( + f"`n_components` upper bound is {rank_upper_bound}. " + f"Got {n_components} instead. Reduce `n_components`." + ) + + self._norm_y_weights = self.deflation_mode == "canonical" # 1.1 norm_y_weights = self._norm_y_weights # Scale (in place) - Xk, Yk, self._x_mean, self._y_mean, self._x_std, self._y_std = ( - _center_scale_xy(X, Y, self.scale)) + Xk, yk, self._x_mean, self._y_mean, self._x_std, self._y_std = _center_scale_xy( + X, y, self.scale + ) self.x_weights_ = np.zeros((p, n_components)) # U self.y_weights_ = np.zeros((q, n_components)) # V @@ -230,47 +280,62 @@ def fit(self, X, Y): # This whole thing corresponds to the algorithm in section 4.1 of the # review from Wegelin. See above for a notation mapping from code to # paper. - Y_eps = np.finfo(Yk.dtype).eps + y_eps = np.finfo(yk.dtype).eps for k in range(n_components): - # Find first left and right singular vectors of the X.T.dot(Y) + # Find first left and right singular vectors of the X.T.dot(y) # cross-covariance matrix. if self.algorithm == "nipals": # Replace columns that are all close to zero with zeros - Yk_mask = np.all(np.abs(Yk) < 10 * Y_eps, axis=0) - Yk[:, Yk_mask] = 0.0 + yk_mask = np.all(np.abs(yk) < 10 * y_eps, axis=0) + yk[:, yk_mask] = 0.0 + + try: + ( + x_weights, + y_weights, + n_iter_, + ) = _get_first_singular_vectors_power_method( + Xk, + yk, + mode=self.mode, + max_iter=self.max_iter, + tol=self.tol, + norm_y_weights=norm_y_weights, + ) + except StopIteration as e: + if str(e) != "y residual is constant": + raise + warnings.warn(f"y residual is constant at iteration {k}") + break - x_weights, y_weights, n_iter_ = \ - _get_first_singular_vectors_power_method( - Xk, Yk, mode=self.mode, max_iter=self.max_iter, - tol=self.tol, norm_y_weights=norm_y_weights) self.n_iter_.append(n_iter_) elif self.algorithm == "svd": - x_weights, y_weights = _get_first_singular_vectors_svd(Xk, Yk) + x_weights, y_weights = _get_first_singular_vectors_svd(Xk, yk) # inplace sign flip for consistency across solvers and archs _svd_flip_1d(x_weights, y_weights) - # compute scores, i.e. the projections of X and Y + # compute scores, i.e. the projections of X and y x_scores = np.dot(Xk, x_weights) if norm_y_weights: y_ss = 1 else: y_ss = np.dot(y_weights, y_weights) - y_scores = np.dot(Yk, y_weights) / y_ss + y_scores = np.dot(yk, y_weights) / y_ss - # Deflation: subtract rank-one approx to obtain Xk+1 and Yk+1 + # Deflation: subtract rank-one approx to obtain Xk+1 and yk+1 x_loadings = np.dot(x_scores, Xk) / np.dot(x_scores, x_scores) Xk -= np.outer(x_scores, x_loadings) if self.deflation_mode == "canonical": - # regress Yk on y_score - y_loadings = np.dot(y_scores, Yk) / np.dot(y_scores, y_scores) - Yk -= np.outer(y_scores, y_loadings) + # regress yk on y_score + y_loadings = np.dot(y_scores, yk) / np.dot(y_scores, y_scores) + yk -= np.outer(y_scores, y_loadings) if self.deflation_mode == "regression": - # regress Yk on x_score - y_loadings = np.dot(x_scores, Yk) / np.dot(x_scores, x_scores) - Yk -= np.outer(x_scores, y_loadings) + # regress yk on x_score + y_loadings = np.dot(x_scores, yk) / np.dot(x_scores, x_scores) + yk -= np.outer(x_scores, y_loadings) self.x_weights_[:, k] = x_weights self.y_weights_[:, k] = y_weights @@ -283,22 +348,24 @@ def fit(self, X, Y): # Xi . Gamma.T is a sum of n_components rank-1 matrices. X_(R+1) is # whatever is left to fully reconstruct X, and can be 0 if X is of rank # n_components. - # Similiarly, Y was approximated as Omega . Delta.T + Y_(R+1) + # Similarly, y was approximated as Omega . Delta.T + y_(R+1) # Compute transformation matrices (rotations_). See User Guide. self.x_rotations_ = np.dot( self.x_weights_, - pinv2(np.dot(self.x_loadings_.T, self.x_weights_), - check_finite=False)) + pinv(np.dot(self.x_loadings_.T, self.x_weights_), check_finite=False), + ) self.y_rotations_ = np.dot( - self.y_weights_, pinv2(np.dot(self.y_loadings_.T, self.y_weights_), - check_finite=False)) - + self.y_weights_, + pinv(np.dot(self.y_loadings_.T, self.y_weights_), check_finite=False), + ) self.coef_ = np.dot(self.x_rotations_, self.y_loadings_.T) - self.coef_ = self.coef_ * self._y_std + self.coef_ = (self.coef_ * self._y_std).T / self._x_std + self.intercept_ = self._y_mean + self._n_features_out = self.x_rotations_.shape[1] return self - def transform(self, X, Y=None, copy=True): + def transform(self, X, y=None, copy=True): """Apply the dimension reduction. Parameters @@ -306,35 +373,38 @@ def transform(self, X, Y=None, copy=True): X : array-like of shape (n_samples, n_features) Samples to transform. - Y : array-like of shape (n_samples, n_targets), default=None + y : array-like of shape (n_samples, n_targets), default=None Target vectors. copy : bool, default=True - Whether to copy `X` and `Y`, or perform in-place normalization. + Whether to copy `X` and `y`, or perform in-place normalization. Returns ------- - `x_scores` if `Y` is not given, `(x_scores, y_scores)` otherwise. + x_scores, y_scores : array-like or tuple of array-like + Return `x_scores` if `y` is not given, `(x_scores, y_scores)` otherwise. """ check_is_fitted(self) - X = self._validate_data(X, copy=copy, dtype=FLOAT_DTYPES, reset=False) + X = validate_data(self, X, copy=copy, dtype=FLOAT_DTYPES, reset=False) # Normalize X -= self._x_mean X /= self._x_std # Apply rotation x_scores = np.dot(X, self.x_rotations_) - if Y is not None: - Y = check_array(Y, ensure_2d=False, copy=copy, dtype=FLOAT_DTYPES) - if Y.ndim == 1: - Y = Y.reshape(-1, 1) - Y -= self._y_mean - Y /= self._y_std - y_scores = np.dot(Y, self.y_rotations_) + if y is not None: + y = check_array( + y, input_name="y", ensure_2d=False, copy=copy, dtype=FLOAT_DTYPES + ) + if y.ndim == 1: + y = y.reshape(-1, 1) + y -= self._y_mean + y /= self._y_std + y_scores = np.dot(y, self.y_rotations_) return x_scores, y_scores return x_scores - def inverse_transform(self, X): + def inverse_transform(self, X, y=None): """Transform data back to its original space. Parameters @@ -343,22 +413,39 @@ def inverse_transform(self, X): New data, where `n_samples` is the number of samples and `n_components` is the number of pls components. + y : array-like of shape (n_samples,) or (n_samples, n_components) + New target, where `n_samples` is the number of samples + and `n_components` is the number of pls components. + Returns ------- - x_reconstructed : array-like of shape (n_samples, n_features) + X_original : ndarray of shape (n_samples, n_features) + Return the reconstructed `X` data. + + y_original : ndarray of shape (n_samples, n_targets) + Return the reconstructed `X` target. Only returned when `y` is given. Notes ----- This transformation will only be exact if `n_components=n_features`. """ check_is_fitted(self) - X = check_array(X, dtype=FLOAT_DTYPES) + X = check_array(X, input_name="X", dtype=FLOAT_DTYPES) # From pls space to original space X_reconstructed = np.matmul(X, self.x_loadings_.T) - # Denormalize X_reconstructed *= self._x_std X_reconstructed += self._x_mean + + if y is not None: + y = check_array(y, input_name="y", dtype=FLOAT_DTYPES) + # From pls space to original space + y_reconstructed = np.matmul(y, self.y_loadings_.T) + # Denormalize + y_reconstructed *= self._y_std + y_reconstructed += self._y_mean + return X_reconstructed, y_reconstructed + return X_reconstructed def predict(self, X, copy=True): @@ -370,7 +457,12 @@ def predict(self, X, copy=True): Samples. copy : bool, default=True - Whether to copy `X` and `Y`, or perform in-place normalization. + Whether to copy `X` or perform in-place normalization. + + Returns + ------- + y_pred : ndarray of shape (n_samples,) or (n_samples, n_targets) + Returns predicted values. Notes ----- @@ -379,12 +471,11 @@ def predict(self, X, copy=True): space. """ check_is_fitted(self) - X = self._validate_data(X, copy=copy, dtype=FLOAT_DTYPES, reset=False) - # Normalize + X = validate_data(self, X, copy=copy, dtype=FLOAT_DTYPES, reset=False) + # Only center X but do not scale it since the coefficients are already scaled X -= self._x_mean - X /= self._x_std - Ypred = np.dot(X, self.coef_) - return Ypred + self._y_mean + y_pred = X @ self.coef_.T + self.intercept_ + return y_pred.ravel() if self._predict_1d else y_pred def fit_transform(self, X, y=None): """Learn and apply the dimension reduction on the train data. @@ -392,91 +483,36 @@ def fit_transform(self, X, y=None): Parameters ---------- X : array-like of shape (n_samples, n_features) - Training vectors, where n_samples is the number of samples and - n_features is the number of predictors. + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of predictors. y : array-like of shape (n_samples, n_targets), default=None - Target vectors, where n_samples is the number of samples and - n_targets is the number of response variables. + Target vectors, where `n_samples` is the number of samples and + `n_targets` is the number of response variables. Returns ------- - x_scores if Y is not given, (x_scores, y_scores) otherwise. + self : ndarray of shape (n_samples, n_components) + Return `x_scores` if `y` is not given, `(x_scores, y_scores)` otherwise. """ return self.fit(X, y).transform(X, y) - # mypy error: Decorated property not supported - @deprecated( # type: ignore - "Attribute norm_y_weights was deprecated in version 0.24 and " - "will be removed in 1.1 (renaming of 0.26).") - @property - def norm_y_weights(self): - return self._norm_y_weights - - @deprecated( # type: ignore - "Attribute x_mean_ was deprecated in version 0.24 and " - "will be removed in 1.1 (renaming of 0.26).") - @property - def x_mean_(self): - return self._x_mean - - @deprecated( # type: ignore - "Attribute y_mean_ was deprecated in version 0.24 and " - "will be removed in 1.1 (renaming of 0.26).") - @property - def y_mean_(self): - return self._y_mean - - @deprecated( # type: ignore - "Attribute x_std_ was deprecated in version 0.24 and " - "will be removed in 1.1 (renaming of 0.26).") - @property - def x_std_(self): - return self._x_std - - @deprecated( # type: ignore - "Attribute y_std_ was deprecated in version 0.24 and " - "will be removed in 1.1 (renaming of 0.26).") - @property - def y_std_(self): - return self._y_std - - @property - def x_scores_(self): - # TODO: raise error in 1.1 instead - if not isinstance(self, PLSRegression): - pass - warnings.warn( - "Attribute x_scores_ was deprecated in version 0.24 and " - "will be removed in 1.1 (renaming of 0.26). Use " - "est.transform(X) on the training data instead.", - FutureWarning - ) - return self._x_scores - - @property - def y_scores_(self): - # TODO: raise error in 1.1 instead - if not isinstance(self, PLSRegression): - warnings.warn( - "Attribute y_scores_ was deprecated in version 0.24 and " - "will be removed in 1.1 (renaming of 0.26). Use " - "est.transform(X) on the training data instead.", - FutureWarning - ) - return self._y_scores - - def _more_tags(self): - return {'poor_score': True, - 'requires_y': False} + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.regressor_tags.poor_score = True + tags.target_tags.required = False + return tags class PLSRegression(_PLS): - """PLS regression + """PLS regression. PLSRegression is also known as PLS2 or PLS1, depending on the number of targets. + For a comparison between other cross decomposition algorithms, see + :ref:`sphx_glr_auto_examples_cross_decomposition_plot_compare_cross_decomposition.py`. + Read more in the :ref:`User Guide `. .. versionadded:: 0.8 @@ -484,11 +520,10 @@ class PLSRegression(_PLS): Parameters ---------- n_components : int, default=2 - Number of components to keep. Should be in `[1, min(n_samples, - n_features, n_targets)]`. + Number of components to keep. Should be in `[1, n_features]`. scale : bool, default=True - Whether to scale `X` and `Y`. + Whether to scale `X` and `y`. max_iter : int, default=500 The maximum number of iterations of the power method when @@ -500,9 +535,9 @@ class PLSRegression(_PLS): than `tol`, where `u` corresponds to the left singular vector. copy : bool, default=True - Whether to copy `X` and `Y` in fit before applying centering, and - potentially scaling. If False, these operations will be done inplace, - modifying both arrays. + Whether to copy `X` and `y` in :term:`fit` before applying centering, + and potentially scaling. If `False`, these operations will be done + inplace, modifying both arrays. Attributes ---------- @@ -518,7 +553,7 @@ class PLSRegression(_PLS): The loadings of `X`. y_loadings_ : ndarray of shape (n_targets, n_components) - The loadings of `Y`. + The loadings of `y`. x_scores_ : ndarray of shape (n_samples, n_components) The transformed training samples. @@ -529,12 +564,18 @@ class PLSRegression(_PLS): x_rotations_ : ndarray of shape (n_features, n_components) The projection matrix used to transform `X`. - y_rotations_ : ndarray of shape (n_features, n_components) - The projection matrix used to transform `Y`. + y_rotations_ : ndarray of shape (n_targets, n_components) + The projection matrix used to transform `y`. + + coef_ : ndarray of shape (n_target, n_features) + The coefficients of the linear model such that `y` is approximated as + `y = X @ coef_.T + intercept_`. - coef_ : ndarray of shape (n_features, n_targets) - The coefficients of the linear model such that `Y` is approximated as - `Y = X @ coef_`. + intercept_ : ndarray of shape (n_targets,) + The intercepts of the linear model such that `y` is approximated as + `y = X @ coef_.T + intercept_`. + + .. versionadded:: 1.1 n_iter_ : list of shape (n_components,) Number of iterations of the power method, for each @@ -543,36 +584,85 @@ class PLSRegression(_PLS): n_features_in_ : int Number of features seen during :term:`fit`. + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + PLSCanonical : Partial Least Squares transformer and regressor. + Examples -------- >>> from sklearn.cross_decomposition import PLSRegression >>> X = [[0., 0., 1.], [1.,0.,0.], [2.,2.,2.], [2.,5.,4.]] - >>> Y = [[0.1, -0.2], [0.9, 1.1], [6.2, 5.9], [11.9, 12.3]] + >>> y = [[0.1, -0.2], [0.9, 1.1], [6.2, 5.9], [11.9, 12.3]] >>> pls2 = PLSRegression(n_components=2) - >>> pls2.fit(X, Y) + >>> pls2.fit(X, y) PLSRegression() - >>> Y_pred = pls2.predict(X) + >>> y_pred = pls2.predict(X) + + For a comparison between PLS Regression and :class:`~sklearn.decomposition.PCA`, see + :ref:`sphx_glr_auto_examples_cross_decomposition_plot_pcr_vs_pls.py`. """ + _parameter_constraints: dict = {**_PLS._parameter_constraints} + for param in ("deflation_mode", "mode", "algorithm"): + _parameter_constraints.pop(param) + # This implementation provides the same results that 3 PLS packages # provided in the R language (R-project): - # - "mixOmics" with function pls(X, Y, mode = "regression") - # - "plspm " with function plsreg2(X, Y) - # - "pls" with function oscorespls.fit(X, Y) + # - "mixOmics" with function pls(X, y, mode = "regression") + # - "plspm " with function plsreg2(X, y) + # - "pls" with function oscorespls.fit(X, y) - @_deprecate_positional_args - def __init__(self, n_components=2, *, scale=True, - max_iter=500, tol=1e-06, copy=True): + def __init__( + self, n_components=2, *, scale=True, max_iter=500, tol=1e-06, copy=True + ): super().__init__( - n_components=n_components, scale=scale, - deflation_mode="regression", mode="A", - algorithm='nipals', max_iter=max_iter, - tol=tol, copy=copy) + n_components=n_components, + scale=scale, + deflation_mode="regression", + mode="A", + algorithm="nipals", + max_iter=max_iter, + tol=tol, + copy=copy, + ) + + def fit(self, X, y): + """Fit model to data. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of predictors. + + y : array-like of shape (n_samples,) or (n_samples, n_targets) + Target vectors, where `n_samples` is the number of samples and + `n_targets` is the number of response variables. + + Returns + ------- + self : object + Fitted model. + """ + super().fit(X, y) + # expose the fitted attributes `x_scores_` and `y_scores_` + self.x_scores_ = self._x_scores + self.y_scores_ = self._y_scores + return self class PLSCanonical(_PLS): """Partial Least Squares transformer and regressor. + For a comparison between other cross decomposition algorithms, see + :ref:`sphx_glr_auto_examples_cross_decomposition_plot_compare_cross_decomposition.py`. + Read more in the :ref:`User Guide `. .. versionadded:: 0.8 @@ -584,7 +674,7 @@ class PLSCanonical(_PLS): n_features, n_targets)]`. scale : bool, default=True - Whether to scale `X` and `Y`. + Whether to scale `X` and `y`. algorithm : {'nipals', 'svd'}, default='nipals' The algorithm used to estimate the first singular vectors of the @@ -592,7 +682,7 @@ class PLSCanonical(_PLS): will compute the whole SVD. max_iter : int, default=500 - the maximum number of iterations of the power method when + The maximum number of iterations of the power method when `algorithm='nipals'`. Ignored otherwise. tol : float, default=1e-06 @@ -601,7 +691,7 @@ class PLSCanonical(_PLS): than `tol`, where `u` corresponds to the left singular vector. copy : bool, default=True - Whether to copy `X` and `Y` in fit before applying centering, and + Whether to copy `X` and `y` in fit before applying centering, and potentially scaling. If False, these operations will be done inplace, modifying both arrays. @@ -619,33 +709,23 @@ class PLSCanonical(_PLS): The loadings of `X`. y_loadings_ : ndarray of shape (n_targets, n_components) - The loadings of `Y`. - - x_scores_ : ndarray of shape (n_samples, n_components) - The transformed training samples. - - .. deprecated:: 0.24 - `x_scores_` is deprecated in 0.24 and will be removed in 1.1 - (renaming of 0.26). You can just call `transform` on the training - data instead. - - y_scores_ : ndarray of shape (n_samples, n_components) - The transformed training targets. - - .. deprecated:: 0.24 - `y_scores_` is deprecated in 0.24 and will be removed in 1.1 - (renaming of 0.26). You can just call `transform` on the training - data instead. + The loadings of `y`. x_rotations_ : ndarray of shape (n_features, n_components) The projection matrix used to transform `X`. - y_rotations_ : ndarray of shape (n_features, n_components) - The projection matrix used to transform `Y`. + y_rotations_ : ndarray of shape (n_targets, n_components) + The projection matrix used to transform `y`. + + coef_ : ndarray of shape (n_targets, n_features) + The coefficients of the linear model such that `y` is approximated as + `y = X @ coef_.T + intercept_`. + + intercept_ : ndarray of shape (n_targets,) + The intercepts of the linear model such that `y` is approximated as + `y = X @ coef_.T + intercept_`. - coef_ : ndarray of shape (n_features, n_targets) - The coefficients of the linear model such that `Y` is approximated as - `Y = X @ coef_`. + .. versionadded:: 1.1 n_iter_ : list of shape (n_components,) Number of iterations of the power method, for each @@ -654,42 +734,68 @@ class PLSCanonical(_PLS): n_features_in_ : int Number of features seen during :term:`fit`. + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + CCA : Canonical Correlation Analysis. + PLSSVD : Partial Least Square SVD. + Examples -------- >>> from sklearn.cross_decomposition import PLSCanonical >>> X = [[0., 0., 1.], [1.,0.,0.], [2.,2.,2.], [2.,5.,4.]] - >>> Y = [[0.1, -0.2], [0.9, 1.1], [6.2, 5.9], [11.9, 12.3]] + >>> y = [[0.1, -0.2], [0.9, 1.1], [6.2, 5.9], [11.9, 12.3]] >>> plsca = PLSCanonical(n_components=2) - >>> plsca.fit(X, Y) + >>> plsca.fit(X, y) PLSCanonical() - >>> X_c, Y_c = plsca.transform(X, Y) - - See Also - -------- - CCA - PLSSVD + >>> X_c, y_c = plsca.transform(X, y) """ + + _parameter_constraints: dict = {**_PLS._parameter_constraints} + for param in ("deflation_mode", "mode"): + _parameter_constraints.pop(param) + # This implementation provides the same results that the "plspm" package - # provided in the R language (R-project), using the function plsca(X, Y). + # provided in the R language (R-project), using the function plsca(X, y). # Results are equal or collinear with the function # ``pls(..., mode = "canonical")`` of the "mixOmics" package. The # difference relies in the fact that mixOmics implementation does not # exactly implement the Wold algorithm since it does not normalize # y_weights to one. - @_deprecate_positional_args - def __init__(self, n_components=2, *, scale=True, algorithm="nipals", - max_iter=500, tol=1e-06, copy=True): + def __init__( + self, + n_components=2, + *, + scale=True, + algorithm="nipals", + max_iter=500, + tol=1e-06, + copy=True, + ): super().__init__( - n_components=n_components, scale=scale, - deflation_mode="canonical", mode="A", + n_components=n_components, + scale=scale, + deflation_mode="canonical", + mode="A", algorithm=algorithm, - max_iter=max_iter, tol=tol, copy=copy) + max_iter=max_iter, + tol=tol, + copy=copy, + ) class CCA(_PLS): """Canonical Correlation Analysis, also known as "Mode B" PLS. + For a comparison between other cross decomposition algorithms, see + :ref:`sphx_glr_auto_examples_cross_decomposition_plot_compare_cross_decomposition.py`. + Read more in the :ref:`User Guide `. Parameters @@ -699,10 +805,10 @@ class CCA(_PLS): n_features, n_targets)]`. scale : bool, default=True - Whether to scale `X` and `Y`. + Whether to scale `X` and `y`. max_iter : int, default=500 - the maximum number of iterations of the power method. + The maximum number of iterations of the power method. tol : float, default=1e-06 The tolerance used as convergence criteria in the power method: the @@ -710,7 +816,7 @@ class CCA(_PLS): than `tol`, where `u` corresponds to the left singular vector. copy : bool, default=True - Whether to copy `X` and `Y` in fit before applying centering, and + Whether to copy `X` and `y` in fit before applying centering, and potentially scaling. If False, these operations will be done inplace, modifying both arrays. @@ -728,33 +834,23 @@ class CCA(_PLS): The loadings of `X`. y_loadings_ : ndarray of shape (n_targets, n_components) - The loadings of `Y`. - - x_scores_ : ndarray of shape (n_samples, n_components) - The transformed training samples. - - .. deprecated:: 0.24 - `x_scores_` is deprecated in 0.24 and will be removed in 1.1 - (renaming of 0.26). You can just call `transform` on the training - data instead. - - y_scores_ : ndarray of shape (n_samples, n_components) - The transformed training targets. - - .. deprecated:: 0.24 - `y_scores_` is deprecated in 0.24 and will be removed in 1.1 - (renaming of 0.26). You can just call `transform` on the training - data instead. + The loadings of `y`. x_rotations_ : ndarray of shape (n_features, n_components) The projection matrix used to transform `X`. - y_rotations_ : ndarray of shape (n_features, n_components) - The projection matrix used to transform `Y`. + y_rotations_ : ndarray of shape (n_targets, n_components) + The projection matrix used to transform `y`. + + coef_ : ndarray of shape (n_targets, n_features) + The coefficients of the linear model such that `y` is approximated as + `y = X @ coef_.T + intercept_`. - coef_ : ndarray of shape (n_features, n_targets) - The coefficients of the linear model such that `Y` is approximated as - `Y = X @ coef_`. + intercept_ : ndarray of shape (n_targets,) + The intercepts of the linear model such that `y` is approximated as + `y = X @ coef_.T + intercept_`. + + .. versionadded:: 1.1 n_iter_ : list of shape (n_components,) Number of iterations of the power method, for each @@ -763,38 +859,54 @@ class CCA(_PLS): n_features_in_ : int Number of features seen during :term:`fit`. + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + PLSCanonical : Partial Least Squares transformer and regressor. + PLSSVD : Partial Least Square SVD. + Examples -------- >>> from sklearn.cross_decomposition import CCA >>> X = [[0., 0., 1.], [1.,0.,0.], [2.,2.,2.], [3.,5.,4.]] - >>> Y = [[0.1, -0.2], [0.9, 1.1], [6.2, 5.9], [11.9, 12.3]] + >>> y = [[0.1, -0.2], [0.9, 1.1], [6.2, 5.9], [11.9, 12.3]] >>> cca = CCA(n_components=1) - >>> cca.fit(X, Y) + >>> cca.fit(X, y) CCA(n_components=1) - >>> X_c, Y_c = cca.transform(X, Y) - - See Also - -------- - PLSCanonical - PLSSVD + >>> X_c, y_c = cca.transform(X, y) """ - @_deprecate_positional_args - def __init__(self, n_components=2, *, scale=True, - max_iter=500, tol=1e-06, copy=True): - super().__init__(n_components=n_components, scale=scale, - deflation_mode="canonical", mode="B", - algorithm="nipals", max_iter=max_iter, tol=tol, - copy=copy) + _parameter_constraints: dict = {**_PLS._parameter_constraints} + for param in ("deflation_mode", "mode", "algorithm"): + _parameter_constraints.pop(param) - -class PLSSVD(TransformerMixin, BaseEstimator): + def __init__( + self, n_components=2, *, scale=True, max_iter=500, tol=1e-06, copy=True + ): + super().__init__( + n_components=n_components, + scale=scale, + deflation_mode="canonical", + mode="B", + algorithm="nipals", + max_iter=max_iter, + tol=tol, + copy=copy, + ) + + +class PLSSVD(ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator): """Partial Least Square SVD. - This transformer simply performs a SVD on the crosscovariance matrix X'Y. - It is able to project both the training data `X` and the targets `Y`. The - training data X is projected on the left singular vectors, while the - targets are projected on the right singular vectors. + This transformer simply performs a SVD on the cross-covariance matrix + `X'y`. It is able to project both the training data `X` and the targets + `y`. The training data `X` is projected on the left singular vectors, while + the targets are projected on the right singular vectors. Read more in the :ref:`User Guide `. @@ -807,41 +919,36 @@ class PLSSVD(TransformerMixin, BaseEstimator): min(n_samples, n_features, n_targets)]`. scale : bool, default=True - Whether to scale `X` and `Y`. + Whether to scale `X` and `y`. copy : bool, default=True - Whether to copy `X` and `Y` in fit before applying centering, and - potentially scaling. If False, these operations will be done inplace, + Whether to copy `X` and `y` in fit before applying centering, and + potentially scaling. If `False`, these operations will be done inplace, modifying both arrays. Attributes ---------- x_weights_ : ndarray of shape (n_features, n_components) The left singular vectors of the SVD of the cross-covariance matrix. - Used to project `X` in `transform`. + Used to project `X` in :meth:`transform`. y_weights_ : ndarray of (n_targets, n_components) The right singular vectors of the SVD of the cross-covariance matrix. - Used to project `X` in `transform`. + Used to project `X` in :meth:`transform`. - x_scores_ : ndarray of shape (n_samples, n_components) - The transformed training samples. + n_features_in_ : int + Number of features seen during :term:`fit`. - .. deprecated:: 0.24 - `x_scores_` is deprecated in 0.24 and will be removed in 1.1 - (renaming of 0.26). You can just call `transform` on the training - data instead. + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. - y_scores_ : ndarray of shape (n_samples, n_components) - The transformed training targets. - - .. deprecated:: 0.24 - `y_scores_` is deprecated in 0.24 and will be removed in 1.1 - (renaming of 0.26). You can just call `transform` on the training - data instead. + .. versionadded:: 1.0 - n_features_in_ : int - Number of features seen during :term:`fit`. + See Also + -------- + PLSCanonical : Partial Least Squares transformer and regressor. + CCA : Canonical Correlation Analysis. Examples -------- @@ -851,27 +958,29 @@ class PLSSVD(TransformerMixin, BaseEstimator): ... [1., 0., 0.], ... [2., 2., 2.], ... [2., 5., 4.]]) - >>> Y = np.array([[0.1, -0.2], + >>> y = np.array([[0.1, -0.2], ... [0.9, 1.1], ... [6.2, 5.9], ... [11.9, 12.3]]) - >>> pls = PLSSVD(n_components=2).fit(X, Y) - >>> X_c, Y_c = pls.transform(X, Y) - >>> X_c.shape, Y_c.shape + >>> pls = PLSSVD(n_components=2).fit(X, y) + >>> X_c, y_c = pls.transform(X, y) + >>> X_c.shape, y_c.shape ((4, 2), (4, 2)) - - See Also - -------- - PLSCanonical - CCA """ - @_deprecate_positional_args + + _parameter_constraints: dict = { + "n_components": [Interval(Integral, 1, None, closed="left")], + "scale": ["boolean"], + "copy": ["boolean"], + } + def __init__(self, n_components=2, *, scale=True, copy=True): self.n_components = n_components self.scale = scale self.copy = copy - def fit(self, X, Y): + @_fit_context(prefer_skip_nested_validation=True) + def fit(self, X, y): """Fit model to data. Parameters @@ -879,99 +988,63 @@ def fit(self, X, Y): X : array-like of shape (n_samples, n_features) Training samples. - Y : array-like of shape (n_samples,) or (n_samples, n_targets) + y : array-like of shape (n_samples,) or (n_samples, n_targets) Targets. + + Returns + ------- + self : object + Fitted estimator. """ - check_consistent_length(X, Y) - X = self._validate_data(X, dtype=np.float64, copy=self.copy, - ensure_min_samples=2) - Y = check_array(Y, dtype=np.float64, copy=self.copy, ensure_2d=False) - if Y.ndim == 1: - Y = Y.reshape(-1, 1) - - # we'll compute the SVD of the cross-covariance matrix = X.T.dot(Y) + check_consistent_length(X, y) + X = validate_data( + self, + X, + dtype=np.float64, + force_writeable=True, + copy=self.copy, + ensure_min_samples=2, + ) + y = check_array( + y, + input_name="y", + dtype=np.float64, + force_writeable=True, + copy=self.copy, + ensure_2d=False, + ) + if y.ndim == 1: + y = y.reshape(-1, 1) + + # we'll compute the SVD of the cross-covariance matrix = X.T.dot(y) # This matrix rank is at most min(n_samples, n_features, n_targets) so # n_components cannot be bigger than that. n_components = self.n_components - rank_upper_bound = min(X.shape[0], X.shape[1], Y.shape[1]) - if not 1 <= n_components <= rank_upper_bound: - # TODO: raise an error in 1.1 - warnings.warn( - f"As of version 0.24, n_components({n_components}) should be " - f"in [1, min(n_features, n_samples, n_targets)] = " - f"[1, {rank_upper_bound}]. " - f"n_components={rank_upper_bound} will be used instead. " - f"In version 1.1 (renaming of 0.26), an error will be raised.", - FutureWarning + rank_upper_bound = min(X.shape[0], X.shape[1], y.shape[1]) + if n_components > rank_upper_bound: + raise ValueError( + f"`n_components` upper bound is {rank_upper_bound}. " + f"Got {n_components} instead. Reduce `n_components`." ) - n_components = rank_upper_bound - X, Y, self._x_mean, self._y_mean, self._x_std, self._y_std = ( - _center_scale_xy(X, Y, self.scale)) + X, y, self._x_mean, self._y_mean, self._x_std, self._y_std = _center_scale_xy( + X, y, self.scale + ) # Compute SVD of cross-covariance matrix - C = np.dot(X.T, Y) + C = np.dot(X.T, y) U, s, Vt = svd(C, full_matrices=False) U = U[:, :n_components] Vt = Vt[:n_components] U, Vt = svd_flip(U, Vt) V = Vt.T - self._x_scores = np.dot(X, U) # TODO: remove in 1.1 - self._y_scores = np.dot(Y, V) # TODO: remove in 1.1 self.x_weights_ = U self.y_weights_ = V + self._n_features_out = self.x_weights_.shape[1] return self - # mypy error: Decorated property not supported - @deprecated( # type: ignore - "Attribute x_scores_ was deprecated in version 0.24 and " - "will be removed in 1.1 (renaming of 0.26). Use est.transform(X) on " - "the training data instead." - ) - @property - def x_scores_(self): - return self._x_scores - - # mypy error: Decorated property not supported - @deprecated( # type: ignore - "Attribute y_scores_ was deprecated in version 0.24 and " - "will be removed in 1.1 (renaming of 0.26). Use est.transform(X, Y) " - "on the training data instead." - ) - @property - def y_scores_(self): - return self._y_scores - - @deprecated( # type: ignore - "Attribute x_mean_ was deprecated in version 0.24 and " - "will be removed in 1.1 (renaming of 0.26).") - @property - def x_mean_(self): - return self._x_mean - - @deprecated( # type: ignore - "Attribute y_mean_ was deprecated in version 0.24 and " - "will be removed in 1.1 (renaming of 0.26).") - @property - def y_mean_(self): - return self._y_mean - - @deprecated( # type: ignore - "Attribute x_std_ was deprecated in version 0.24 and " - "will be removed in 1.1 (renaming of 0.26).") - @property - def x_std_(self): - return self._x_std - - @deprecated( # type: ignore - "Attribute y_std_ was deprecated in version 0.24 and " - "will be removed in 1.1 (renaming of 0.26).") - @property - def y_std_(self): - return self._y_std - - def transform(self, X, Y=None): + def transform(self, X, y=None): """ Apply the dimensionality reduction. @@ -980,26 +1053,26 @@ def transform(self, X, Y=None): X : array-like of shape (n_samples, n_features) Samples to be transformed. - Y : array-like of shape (n_samples,) or (n_samples, n_targets), \ + y : array-like of shape (n_samples,) or (n_samples, n_targets), \ default=None Targets. Returns ------- - out : array-like or tuple of array-like - The transformed data `X_tranformed` if `Y` is not None, - `(X_transformed, Y_transformed)` otherwise. + x_scores : array-like or tuple of array-like + The transformed data `X_transformed` if `y is not None`, + `(X_transformed, y_transformed)` otherwise. """ check_is_fitted(self) - X = self._validate_data(X, dtype=np.float64, reset=False) + X = validate_data(self, X, dtype=np.float64, reset=False) Xr = (X - self._x_mean) / self._x_std x_scores = np.dot(Xr, self.x_weights_) - if Y is not None: - Y = check_array(Y, ensure_2d=False, dtype=np.float64) - if Y.ndim == 1: - Y = Y.reshape(-1, 1) - Yr = (Y - self._y_mean) / self._y_std - y_scores = np.dot(Yr, self.y_weights_) + if y is not None: + y = check_array(y, input_name="y", ensure_2d=False, dtype=np.float64) + if y.ndim == 1: + y = y.reshape(-1, 1) + yr = (y - self._y_mean) / self._y_std + y_scores = np.dot(yr, self.y_weights_) return x_scores, y_scores return x_scores @@ -1018,7 +1091,7 @@ def fit_transform(self, X, y=None): Returns ------- out : array-like or tuple of array-like - The transformed data `X_tranformed` if `Y` is not None, - `(X_transformed, Y_transformed)` otherwise. + The transformed data `X_transformed` if `y is not None`, + `(X_transformed, y_transformed)` otherwise. """ return self.fit(X, y).transform(X, y) diff --git a/sklearn/cross_decomposition/tests/test_pls.py b/sklearn/cross_decomposition/tests/test_pls.py index 04c791fd4154a..7e516d71b6f98 100644 --- a/sklearn/cross_decomposition/tests/test_pls.py +++ b/sklearn/cross_decomposition/tests/test_pls.py @@ -1,21 +1,22 @@ -import pytest +import warnings + import numpy as np -from numpy.testing import (assert_array_almost_equal, assert_array_equal, - assert_allclose) +import pytest +from numpy.testing import assert_allclose, assert_array_almost_equal, assert_array_equal -from sklearn.datasets import load_linnerud +from sklearn.cross_decomposition import CCA, PLSSVD, PLSCanonical, PLSRegression from sklearn.cross_decomposition._pls import ( _center_scale_xy, _get_first_singular_vectors_power_method, _get_first_singular_vectors_svd, - _svd_flip_1d + _svd_flip_1d, ) -from sklearn.cross_decomposition import CCA -from sklearn.cross_decomposition import PLSSVD, PLSRegression, PLSCanonical -from sklearn.datasets import make_regression +from sklearn.datasets import load_linnerud, make_regression +from sklearn.ensemble import VotingRegressor +from sklearn.exceptions import ConvergenceWarning +from sklearn.linear_model import LinearRegression from sklearn.utils import check_random_state from sklearn.utils.extmath import svd_flip -from sklearn.exceptions import ConvergenceWarning def assert_matrix_orthogonal(M): @@ -27,37 +28,40 @@ def test_pls_canonical_basics(): # Basic checks for PLSCanonical d = load_linnerud() X = d.data - Y = d.target + y = d.target pls = PLSCanonical(n_components=X.shape[1]) - pls.fit(X, Y) + pls.fit(X, y) assert_matrix_orthogonal(pls.x_weights_) assert_matrix_orthogonal(pls.y_weights_) assert_matrix_orthogonal(pls._x_scores) assert_matrix_orthogonal(pls._y_scores) - # Check X = TP' and Y = UQ' + # Check X = TP' and y = UQ' T = pls._x_scores P = pls.x_loadings_ U = pls._y_scores Q = pls.y_loadings_ # Need to scale first - Xc, Yc, x_mean, y_mean, x_std, y_std = _center_scale_xy( - X.copy(), Y.copy(), scale=True) + Xc, yc, x_mean, y_mean, x_std, y_std = _center_scale_xy( + X.copy(), y.copy(), scale=True + ) assert_array_almost_equal(Xc, np.dot(T, P.T)) - assert_array_almost_equal(Yc, np.dot(U, Q.T)) + assert_array_almost_equal(yc, np.dot(U, Q.T)) # Check that rotations on training data lead to scores Xt = pls.transform(X) assert_array_almost_equal(Xt, pls._x_scores) - Xt, Yt = pls.transform(X, Y) + Xt, yt = pls.transform(X, y) assert_array_almost_equal(Xt, pls._x_scores) - assert_array_almost_equal(Yt, pls._y_scores) + assert_array_almost_equal(yt, pls._y_scores) # Check that inverse_transform works X_back = pls.inverse_transform(Xt) assert_array_almost_equal(X_back, X) + _, y_back = pls.inverse_transform(Xt, yt) + assert_array_almost_equal(y_back, y) def test_sanity_check_pls_regression(): @@ -66,39 +70,52 @@ def test_sanity_check_pls_regression(): d = load_linnerud() X = d.data - Y = d.target + y = d.target pls = PLSRegression(n_components=X.shape[1]) - pls.fit(X, Y) + X_trans, _ = pls.fit_transform(X, y) + + # FIXME: one would expect y_trans == pls.y_scores_ but this is not + # the case. + # xref: https://github.com/scikit-learn/scikit-learn/issues/22420 + assert_allclose(X_trans, pls.x_scores_) expected_x_weights = np.array( - [[-0.61330704, -0.00443647, 0.78983213], - [-0.74697144, -0.32172099, -0.58183269], - [-0.25668686, 0.94682413, -0.19399983]]) + [ + [-0.61330704, -0.00443647, 0.78983213], + [-0.74697144, -0.32172099, -0.58183269], + [-0.25668686, 0.94682413, -0.19399983], + ] + ) expected_x_loadings = np.array( - [[-0.61470416, -0.24574278, 0.78983213], - [-0.65625755, -0.14396183, -0.58183269], - [-0.51733059, 1.00609417, -0.19399983]]) + [ + [-0.61470416, -0.24574278, 0.78983213], + [-0.65625755, -0.14396183, -0.58183269], + [-0.51733059, 1.00609417, -0.19399983], + ] + ) expected_y_weights = np.array( - [[+0.32456184, 0.29892183, 0.20316322], - [+0.42439636, 0.61970543, 0.19320542], - [-0.13143144, -0.26348971, -0.17092916]]) + [ + [+0.32456184, 0.29892183, 0.20316322], + [+0.42439636, 0.61970543, 0.19320542], + [-0.13143144, -0.26348971, -0.17092916], + ] + ) expected_y_loadings = np.array( - [[+0.32456184, 0.29892183, 0.20316322], - [+0.42439636, 0.61970543, 0.19320542], - [-0.13143144, -0.26348971, -0.17092916]]) - - assert_array_almost_equal(np.abs(pls.x_loadings_), - np.abs(expected_x_loadings)) - assert_array_almost_equal(np.abs(pls.x_weights_), - np.abs(expected_x_weights)) - assert_array_almost_equal(np.abs(pls.y_loadings_), - np.abs(expected_y_loadings)) - assert_array_almost_equal(np.abs(pls.y_weights_), - np.abs(expected_y_weights)) + [ + [+0.32456184, 0.29892183, 0.20316322], + [+0.42439636, 0.61970543, 0.19320542], + [-0.13143144, -0.26348971, -0.17092916], + ] + ) + + assert_array_almost_equal(np.abs(pls.x_loadings_), np.abs(expected_x_loadings)) + assert_array_almost_equal(np.abs(pls.x_weights_), np.abs(expected_x_weights)) + assert_array_almost_equal(np.abs(pls.y_loadings_), np.abs(expected_y_loadings)) + assert_array_almost_equal(np.abs(pls.y_weights_), np.abs(expected_y_weights)) # The R / Python difference in the signs should be consistent across # loadings, weights, etc. @@ -110,47 +127,51 @@ def test_sanity_check_pls_regression(): assert_array_almost_equal(y_loadings_sign_flip, y_weights_sign_flip) -def test_sanity_check_pls_regression_constant_column_Y(): - # Check behavior when the first column of Y is constant +def test_sanity_check_pls_regression_constant_column_y(): + # Check behavior when the first column of y is constant # The results are checked against a modified version of plsreg2 # from the R-package plsdepot d = load_linnerud() X = d.data - Y = d.target - Y[:, 0] = 1 + y = d.target + y[:, 0] = 1 pls = PLSRegression(n_components=X.shape[1]) - pls.fit(X, Y) + pls.fit(X, y) expected_x_weights = np.array( - [[-0.6273573, 0.007081799, 0.7786994], - [-0.7493417, -0.277612681, -0.6011807], - [-0.2119194, 0.960666981, -0.1794690]]) + [ + [-0.6273573, 0.007081799, 0.7786994], + [-0.7493417, -0.277612681, -0.6011807], + [-0.2119194, 0.960666981, -0.1794690], + ] + ) expected_x_loadings = np.array( - [[-0.6273512, -0.22464538, 0.7786994], - [-0.6643156, -0.09871193, -0.6011807], - [-0.5125877, 1.01407380, -0.1794690]]) + [ + [-0.6273512, -0.22464538, 0.7786994], + [-0.6643156, -0.09871193, -0.6011807], + [-0.5125877, 1.01407380, -0.1794690], + ] + ) expected_y_loadings = np.array( - [[0.0000000, 0.0000000, 0.0000000], - [0.4357300, 0.5828479, 0.2174802], - [-0.1353739, -0.2486423, -0.1810386]]) - - assert_array_almost_equal(np.abs(expected_x_weights), - np.abs(pls.x_weights_)) - assert_array_almost_equal(np.abs(expected_x_loadings), - np.abs(pls.x_loadings_)) + [ + [0.0000000, 0.0000000, 0.0000000], + [0.4357300, 0.5828479, 0.2174802], + [-0.1353739, -0.2486423, -0.1810386], + ] + ) + + assert_array_almost_equal(np.abs(expected_x_weights), np.abs(pls.x_weights_)) + assert_array_almost_equal(np.abs(expected_x_loadings), np.abs(pls.x_loadings_)) # For the PLSRegression with default parameters, y_loadings == y_weights - assert_array_almost_equal(np.abs(pls.y_loadings_), - np.abs(expected_y_loadings)) - assert_array_almost_equal(np.abs(pls.y_weights_), - np.abs(expected_y_loadings)) + assert_array_almost_equal(np.abs(pls.y_loadings_), np.abs(expected_y_loadings)) + assert_array_almost_equal(np.abs(pls.y_weights_), np.abs(expected_y_loadings)) x_loadings_sign_flip = np.sign(expected_x_loadings / pls.x_loadings_) x_weights_sign_flip = np.sign(expected_x_weights / pls.x_weights_) # we ignore the first full-zeros row for y - y_loadings_sign_flip = np.sign(expected_y_loadings[1:] / - pls.y_loadings_[1:]) + y_loadings_sign_flip = np.sign(expected_y_loadings[1:] / pls.y_loadings_[1:]) assert_array_equal(x_loadings_sign_flip, x_weights_sign_flip) assert_array_equal(x_loadings_sign_flip[1:], y_loadings_sign_flip) @@ -162,39 +183,47 @@ def test_sanity_check_pls_canonical(): d = load_linnerud() X = d.data - Y = d.target + y = d.target pls = PLSCanonical(n_components=X.shape[1]) - pls .fit(X, Y) + pls.fit(X, y) expected_x_weights = np.array( - [[-0.61330704, 0.25616119, -0.74715187], - [-0.74697144, 0.11930791, 0.65406368], - [-0.25668686, -0.95924297, -0.11817271]]) + [ + [-0.61330704, 0.25616119, -0.74715187], + [-0.74697144, 0.11930791, 0.65406368], + [-0.25668686, -0.95924297, -0.11817271], + ] + ) expected_x_rotations = np.array( - [[-0.61330704, 0.41591889, -0.62297525], - [-0.74697144, 0.31388326, 0.77368233], - [-0.25668686, -0.89237972, -0.24121788]]) + [ + [-0.61330704, 0.41591889, -0.62297525], + [-0.74697144, 0.31388326, 0.77368233], + [-0.25668686, -0.89237972, -0.24121788], + ] + ) expected_y_weights = np.array( - [[+0.58989127, 0.7890047, 0.1717553], - [+0.77134053, -0.61351791, 0.16920272], - [-0.23887670, -0.03267062, 0.97050016]]) + [ + [+0.58989127, 0.7890047, 0.1717553], + [+0.77134053, -0.61351791, 0.16920272], + [-0.23887670, -0.03267062, 0.97050016], + ] + ) expected_y_rotations = np.array( - [[+0.58989127, 0.7168115, 0.30665872], - [+0.77134053, -0.70791757, 0.19786539], - [-0.23887670, -0.00343595, 0.94162826]]) - - assert_array_almost_equal(np.abs(pls.x_rotations_), - np.abs(expected_x_rotations)) - assert_array_almost_equal(np.abs(pls.x_weights_), - np.abs(expected_x_weights)) - assert_array_almost_equal(np.abs(pls.y_rotations_), - np.abs(expected_y_rotations)) - assert_array_almost_equal(np.abs(pls.y_weights_), - np.abs(expected_y_weights)) + [ + [+0.58989127, 0.7168115, 0.30665872], + [+0.77134053, -0.70791757, 0.19786539], + [-0.23887670, -0.00343595, 0.94162826], + ] + ) + + assert_array_almost_equal(np.abs(pls.x_rotations_), np.abs(expected_x_rotations)) + assert_array_almost_equal(np.abs(pls.x_weights_), np.abs(expected_x_weights)) + assert_array_almost_equal(np.abs(pls.y_rotations_), np.abs(expected_y_rotations)) + assert_array_almost_equal(np.abs(pls.y_weights_), np.abs(expected_y_weights)) x_rotations_sign_flip = np.sign(pls.x_rotations_ / expected_x_rotations) x_weights_sign_flip = np.sign(pls.x_weights_ / expected_x_weights) @@ -222,77 +251,83 @@ def test_sanity_check_pls_canonical_random(): l2 = rng.normal(size=n) latents = np.array([l1, l1, l2, l2]).T X = latents + rng.normal(size=4 * n).reshape((n, 4)) - Y = latents + rng.normal(size=4 * n).reshape((n, 4)) - X = np.concatenate( - (X, rng.normal(size=p_noise * n).reshape(n, p_noise)), axis=1) - Y = np.concatenate( - (Y, rng.normal(size=q_noise * n).reshape(n, q_noise)), axis=1) + y = latents + rng.normal(size=4 * n).reshape((n, 4)) + X = np.concatenate((X, rng.normal(size=p_noise * n).reshape(n, p_noise)), axis=1) + y = np.concatenate((y, rng.normal(size=q_noise * n).reshape(n, q_noise)), axis=1) pls = PLSCanonical(n_components=3) - pls.fit(X, Y) + pls.fit(X, y) expected_x_weights = np.array( - [[0.65803719, 0.19197924, 0.21769083], - [0.7009113, 0.13303969, -0.15376699], - [0.13528197, -0.68636408, 0.13856546], - [0.16854574, -0.66788088, -0.12485304], - [-0.03232333, -0.04189855, 0.40690153], - [0.1148816, -0.09643158, 0.1613305], - [0.04792138, -0.02384992, 0.17175319], - [-0.06781, -0.01666137, -0.18556747], - [-0.00266945, -0.00160224, 0.11893098], - [-0.00849528, -0.07706095, 0.1570547], - [-0.00949471, -0.02964127, 0.34657036], - [-0.03572177, 0.0945091, 0.3414855], - [0.05584937, -0.02028961, -0.57682568], - [0.05744254, -0.01482333, -0.17431274]]) + [ + [0.65803719, 0.19197924, 0.21769083], + [0.7009113, 0.13303969, -0.15376699], + [0.13528197, -0.68636408, 0.13856546], + [0.16854574, -0.66788088, -0.12485304], + [-0.03232333, -0.04189855, 0.40690153], + [0.1148816, -0.09643158, 0.1613305], + [0.04792138, -0.02384992, 0.17175319], + [-0.06781, -0.01666137, -0.18556747], + [-0.00266945, -0.00160224, 0.11893098], + [-0.00849528, -0.07706095, 0.1570547], + [-0.00949471, -0.02964127, 0.34657036], + [-0.03572177, 0.0945091, 0.3414855], + [0.05584937, -0.02028961, -0.57682568], + [0.05744254, -0.01482333, -0.17431274], + ] + ) expected_x_loadings = np.array( - [[0.65649254, 0.1847647, 0.15270699], - [0.67554234, 0.15237508, -0.09182247], - [0.19219925, -0.67750975, 0.08673128], - [0.2133631, -0.67034809, -0.08835483], - [-0.03178912, -0.06668336, 0.43395268], - [0.15684588, -0.13350241, 0.20578984], - [0.03337736, -0.03807306, 0.09871553], - [-0.06199844, 0.01559854, -0.1881785], - [0.00406146, -0.00587025, 0.16413253], - [-0.00374239, -0.05848466, 0.19140336], - [0.00139214, -0.01033161, 0.32239136], - [-0.05292828, 0.0953533, 0.31916881], - [0.04031924, -0.01961045, -0.65174036], - [0.06172484, -0.06597366, -0.1244497]]) + [ + [0.65649254, 0.1847647, 0.15270699], + [0.67554234, 0.15237508, -0.09182247], + [0.19219925, -0.67750975, 0.08673128], + [0.2133631, -0.67034809, -0.08835483], + [-0.03178912, -0.06668336, 0.43395268], + [0.15684588, -0.13350241, 0.20578984], + [0.03337736, -0.03807306, 0.09871553], + [-0.06199844, 0.01559854, -0.1881785], + [0.00406146, -0.00587025, 0.16413253], + [-0.00374239, -0.05848466, 0.19140336], + [0.00139214, -0.01033161, 0.32239136], + [-0.05292828, 0.0953533, 0.31916881], + [0.04031924, -0.01961045, -0.65174036], + [0.06172484, -0.06597366, -0.1244497], + ] + ) expected_y_weights = np.array( - [[0.66101097, 0.18672553, 0.22826092], - [0.69347861, 0.18463471, -0.23995597], - [0.14462724, -0.66504085, 0.17082434], - [0.22247955, -0.6932605, -0.09832993], - [0.07035859, 0.00714283, 0.67810124], - [0.07765351, -0.0105204, -0.44108074], - [-0.00917056, 0.04322147, 0.10062478], - [-0.01909512, 0.06182718, 0.28830475], - [0.01756709, 0.04797666, 0.32225745]]) + [ + [0.66101097, 0.18672553, 0.22826092], + [0.69347861, 0.18463471, -0.23995597], + [0.14462724, -0.66504085, 0.17082434], + [0.22247955, -0.6932605, -0.09832993], + [0.07035859, 0.00714283, 0.67810124], + [0.07765351, -0.0105204, -0.44108074], + [-0.00917056, 0.04322147, 0.10062478], + [-0.01909512, 0.06182718, 0.28830475], + [0.01756709, 0.04797666, 0.32225745], + ] + ) expected_y_loadings = np.array( - [[0.68568625, 0.1674376, 0.0969508], - [0.68782064, 0.20375837, -0.1164448], - [0.11712173, -0.68046903, 0.12001505], - [0.17860457, -0.6798319, -0.05089681], - [0.06265739, -0.0277703, 0.74729584], - [0.0914178, 0.00403751, -0.5135078], - [-0.02196918, -0.01377169, 0.09564505], - [-0.03288952, 0.09039729, 0.31858973], - [0.04287624, 0.05254676, 0.27836841]]) - - assert_array_almost_equal(np.abs(pls.x_loadings_), - np.abs(expected_x_loadings)) - assert_array_almost_equal(np.abs(pls.x_weights_), - np.abs(expected_x_weights)) - assert_array_almost_equal(np.abs(pls.y_loadings_), - np.abs(expected_y_loadings)) - assert_array_almost_equal(np.abs(pls.y_weights_), - np.abs(expected_y_weights)) + [ + [0.68568625, 0.1674376, 0.0969508], + [0.68782064, 0.20375837, -0.1164448], + [0.11712173, -0.68046903, 0.12001505], + [0.17860457, -0.6798319, -0.05089681], + [0.06265739, -0.0277703, 0.74729584], + [0.0914178, 0.00403751, -0.5135078], + [-0.02196918, -0.01377169, 0.09564505], + [-0.03288952, 0.09039729, 0.31858973], + [0.04287624, 0.05254676, 0.27836841], + ] + ) + + assert_array_almost_equal(np.abs(pls.x_loadings_), np.abs(expected_x_loadings)) + assert_array_almost_equal(np.abs(pls.x_weights_), np.abs(expected_x_weights)) + assert_array_almost_equal(np.abs(pls.y_loadings_), np.abs(expected_y_loadings)) + assert_array_almost_equal(np.abs(pls.y_weights_), np.abs(expected_y_weights)) x_loadings_sign_flip = np.sign(pls.x_loadings_ / expected_x_loadings) x_weights_sign_flip = np.sign(pls.x_weights_ / expected_x_weights) @@ -312,57 +347,56 @@ def test_convergence_fail(): # Make sure ConvergenceWarning is raised if max_iter is too small d = load_linnerud() X = d.data - Y = d.target + y = d.target pls_nipals = PLSCanonical(n_components=X.shape[1], max_iter=2) with pytest.warns(ConvergenceWarning): - pls_nipals.fit(X, Y) + pls_nipals.fit(X, y) -@pytest.mark.filterwarnings('ignore:.*scores_ was deprecated') # 1.1 -@pytest.mark.parametrize('Est', (PLSSVD, PLSRegression, PLSCanonical)) +@pytest.mark.parametrize("Est", (PLSSVD, PLSRegression, PLSCanonical)) def test_attibutes_shapes(Est): # Make sure attributes are of the correct shape depending on n_components d = load_linnerud() X = d.data - Y = d.target + y = d.target n_components = 2 pls = Est(n_components=n_components) - pls.fit(X, Y) - assert all(attr.shape[1] == n_components - for attr in (pls.x_scores_, pls.y_scores_, pls.x_weights_, - pls.y_weights_)) + pls.fit(X, y) + assert all( + attr.shape[1] == n_components for attr in (pls.x_weights_, pls.y_weights_) + ) -@pytest.mark.parametrize('Est', (PLSRegression, PLSCanonical, CCA)) +@pytest.mark.parametrize("Est", (PLSRegression, PLSCanonical, CCA)) def test_univariate_equivalence(Est): - # Ensure 2D Y with 1 column is equivalent to 1D Y + # Ensure 2D y with 1 column is equivalent to 1D y d = load_linnerud() X = d.data - Y = d.target + y = d.target est = Est(n_components=1) - one_d_coeff = est.fit(X, Y[:, 0]).coef_ - two_d_coeff = est.fit(X, Y[:, :1]).coef_ + one_d_coeff = est.fit(X, y[:, 0]).coef_ + two_d_coeff = est.fit(X, y[:, :1]).coef_ assert one_d_coeff.shape == two_d_coeff.shape assert_array_almost_equal(one_d_coeff, two_d_coeff) -@pytest.mark.parametrize('Est', (PLSRegression, PLSCanonical, CCA, PLSSVD)) +@pytest.mark.parametrize("Est", (PLSRegression, PLSCanonical, CCA, PLSSVD)) def test_copy(Est): # check that the "copy" keyword works d = load_linnerud() X = d.data - Y = d.target + y = d.target X_orig = X.copy() # copy=True won't modify inplace - pls = Est(copy=True).fit(X, Y) + pls = Est(copy=True).fit(X, y) assert_array_equal(X, X_orig) # copy=False will modify inplace with pytest.raises(AssertionError): - Est(copy=False).fit(X, Y) + Est(copy=False).fit(X, y) assert_array_almost_equal(X, X_orig) if Est is PLSSVD: @@ -370,19 +404,21 @@ def test_copy(Est): X_orig = X.copy() with pytest.raises(AssertionError): - pls.transform(X, Y, copy=False), + pls.transform(X, y, copy=False) assert_array_almost_equal(X, X_orig) X_orig = X.copy() with pytest.raises(AssertionError): - pls.predict(X, copy=False), + pls.predict(X, copy=False) assert_array_almost_equal(X, X_orig) # Make sure copy=True gives same transform and predictions as predict=False - assert_array_almost_equal(pls.transform(X, Y, copy=True), - pls.transform(X.copy(), Y.copy(), copy=False)) - assert_array_almost_equal(pls.predict(X, copy=True), - pls.predict(X.copy(), copy=False)) + assert_array_almost_equal( + pls.transform(X, y, copy=True), pls.transform(X.copy(), y.copy(), copy=False) + ) + assert_array_almost_equal( + pls.predict(X, copy=True), pls.predict(X.copy(), copy=False) + ) def _generate_test_scale_and_stability_datasets(): @@ -393,150 +429,98 @@ def _generate_test_scale_and_stability_datasets(): n_targets = 5 n_features = 10 Q = rng.randn(n_targets, n_features) - Y = rng.randn(n_samples, n_targets) - X = np.dot(Y, Q) + 2 * rng.randn(n_samples, n_features) + 1 + y = rng.randn(n_samples, n_targets) + X = np.dot(y, Q) + 2 * rng.randn(n_samples, n_features) + 1 X *= 1000 - yield X, Y + yield X, y - # Data set where one of the features is constaint - X, Y = load_linnerud(return_X_y=True) + # Data set where one of the features is constraint + X, y = load_linnerud(return_X_y=True) # causes X[:, -1].std() to be zero X[:, -1] = 1.0 - yield X, Y - - X = np.array([[0., 0., 1.], - [1., 0., 0.], - [2., 2., 2.], - [3., 5., 4.]]) - Y = np.array([[0.1, -0.2], - [0.9, 1.1], - [6.2, 5.9], - [11.9, 12.3]]) - yield X, Y + yield X, y + + X = np.array([[0.0, 0.0, 1.0], [1.0, 0.0, 0.0], [2.0, 2.0, 2.0], [3.0, 5.0, 4.0]]) + y = np.array([[0.1, -0.2], [0.9, 1.1], [6.2, 5.9], [11.9, 12.3]]) + yield X, y # Seeds that provide a non-regression test for #18746, where CCA fails seeds = [530, 741] for seed in seeds: rng = np.random.RandomState(seed) X = rng.randn(4, 3) - Y = rng.randn(4, 2) - yield X, Y + y = rng.randn(4, 2) + yield X, y -@pytest.mark.parametrize('Est', (CCA, PLSCanonical, PLSRegression, PLSSVD)) -@pytest.mark.parametrize('X, Y', _generate_test_scale_and_stability_datasets()) -def test_scale_and_stability(Est, X, Y): +@pytest.mark.parametrize("Est", (CCA, PLSCanonical, PLSRegression, PLSSVD)) +@pytest.mark.parametrize("X, y", _generate_test_scale_and_stability_datasets()) +def test_scale_and_stability(Est, X, y): """scale=True is equivalent to scale=False on centered/scaled data This allows to check numerical stability over platforms as well""" - X_s, Y_s, *_ = _center_scale_xy(X, Y) + X_s, y_s, *_ = _center_scale_xy(X, y) - X_score, Y_score = Est(scale=True).fit_transform(X, Y) - X_s_score, Y_s_score = Est(scale=False).fit_transform(X_s, Y_s) + X_score, y_score = Est(scale=True).fit_transform(X, y) + X_s_score, y_s_score = Est(scale=False).fit_transform(X_s, y_s) assert_allclose(X_s_score, X_score, atol=1e-4) - assert_allclose(Y_s_score, Y_score, atol=1e-4) + assert_allclose(y_s_score, y_score, atol=1e-4) -@pytest.mark.parametrize('Est', (PLSSVD, PLSCanonical, CCA)) -@pytest.mark.parametrize('n_components', (0, 4)) -def test_n_components_bounds(Est, n_components): - # n_components should be in [1, min(n_samples, n_features, n_targets)] - # TODO: catch error instead of warning in 1.1 +@pytest.mark.parametrize("Estimator", (PLSSVD, PLSRegression, PLSCanonical, CCA)) +def test_n_components_upper_bounds(Estimator): + """Check the validation of `n_components` upper bounds for `PLS` regressors.""" rng = np.random.RandomState(0) X = rng.randn(10, 5) - Y = rng.randn(10, 3) - est = Est(n_components=n_components) - with pytest.warns(FutureWarning, - match="n_components=3 will be used instead"): - est.fit(X, Y) - # make sure upper bound of rank is used as a fallback - assert est.transform(X).shape[1] == 3 - - -@pytest.mark.parametrize('n_components', (0, 6)) -def test_n_components_bounds_pls_regression(n_components): - # For PLSRegression, the upper bound for n_components is n_features - # TODO: catch error instead of warning in 1.1 - rng = np.random.RandomState(0) - X = rng.randn(10, 5) - Y = rng.randn(10, 3) - est = PLSRegression(n_components=n_components) - with pytest.warns(FutureWarning, - match="n_components=5 will be used instead"): - est.fit(X, Y) - # make sure upper bound of rank is used as a fallback - assert est.transform(X).shape[1] == 5 - - -@pytest.mark.parametrize('Est', (PLSSVD, CCA, PLSCanonical)) -def test_scores_deprecations(Est): - # Make sure x_scores_ and y_scores_ are deprecated. - # It's not deprecated for PLSRegression because y_score_ is different from - # transform(Y_train) - # TODO: remove attributes and test in 1.1 - rng = np.random.RandomState(0) - X = rng.randn(10, 5) - Y = rng.randn(10, 3) - est = Est().fit(X, Y) - with pytest.warns(FutureWarning, match="x_scores_ was deprecated"): - assert_allclose(est.x_scores_, est.transform(X)) - with pytest.warns(FutureWarning, match="y_scores_ was deprecated"): - assert_allclose(est.y_scores_, est.transform(X, Y)[1]) + y = rng.randn(10, 3) + est = Estimator(n_components=10) + err_msg = "`n_components` upper bound is .*. Got 10 instead. Reduce `n_components`." + with pytest.raises(ValueError, match=err_msg): + est.fit(X, y) -@pytest.mark.parametrize('Est', (PLSRegression, PLSCanonical, CCA)) -def test_norm_y_weights_deprecation(Est): +def test_n_components_upper_PLSRegression(): + """Check the validation of `n_components` upper bounds for PLSRegression.""" rng = np.random.RandomState(0) - X = rng.randn(10, 5) - Y = rng.randn(10, 3) - est = Est().fit(X, Y) - with pytest.warns(FutureWarning, match="norm_y_weights was deprecated"): - est.norm_y_weights - - -# TODO: Remove test in 1.1 -@pytest.mark.parametrize('Estimator', - (PLSRegression, PLSCanonical, CCA, PLSSVD)) -@pytest.mark.parametrize('attribute', - ("x_mean_", "y_mean_", "x_std_", "y_std_")) -def test_mean_and_std_deprecation(Estimator, attribute): - rng = np.random.RandomState(0) - X = rng.randn(10, 5) - Y = rng.randn(10, 3) - estimator = Estimator().fit(X, Y) - with pytest.warns(FutureWarning, match=f"{attribute} was deprecated"): - getattr(estimator, attribute) + X = rng.randn(20, 64) + y = rng.randn(20, 3) + est = PLSRegression(n_components=30) + err_msg = "`n_components` upper bound is 20. Got 30 instead. Reduce `n_components`." + with pytest.raises(ValueError, match=err_msg): + est.fit(X, y) -@pytest.mark.parametrize('n_samples, n_features', [(100, 10), (100, 200)]) -@pytest.mark.parametrize('seed', range(10)) -def test_singular_value_helpers(n_samples, n_features, seed): +@pytest.mark.parametrize("n_samples, n_features", [(100, 10), (100, 200)]) +def test_singular_value_helpers(n_samples, n_features, global_random_seed): # Make sure SVD and power method give approximately the same results - X, Y = make_regression(n_samples, n_features, n_targets=5, - random_state=seed) - u1, v1, _ = _get_first_singular_vectors_power_method(X, Y, - norm_y_weights=True) - u2, v2 = _get_first_singular_vectors_svd(X, Y) + X, y = make_regression( + n_samples, n_features, n_targets=5, random_state=global_random_seed + ) + u1, v1, _ = _get_first_singular_vectors_power_method(X, y, norm_y_weights=True) + u2, v2 = _get_first_singular_vectors_svd(X, y) _svd_flip_1d(u1, v1) _svd_flip_1d(u2, v2) - rtol = 1e-1 - assert_allclose(u1, u2, rtol=rtol) - assert_allclose(v1, v2, rtol=rtol) + rtol = 1e-3 + # Setting atol because some coordinates are very close to zero + assert_allclose(u1, u2, atol=u2.max() * rtol) + assert_allclose(v1, v2, atol=v2.max() * rtol) -def test_one_component_equivalence(): +def test_one_component_equivalence(global_random_seed): # PLSSVD, PLSRegression and PLSCanonical should all be equivalent when # n_components is 1 - X, Y = make_regression(100, 10, n_targets=5, random_state=0) - svd = PLSSVD(n_components=1).fit(X, Y).transform(X) - reg = PLSRegression(n_components=1).fit(X, Y).transform(X) - canonical = PLSCanonical(n_components=1).fit(X, Y).transform(X) + X, y = make_regression(100, 10, n_targets=5, random_state=global_random_seed) + svd = PLSSVD(n_components=1).fit(X, y).transform(X) + reg = PLSRegression(n_components=1).fit(X, y).transform(X) + canonical = PLSCanonical(n_components=1).fit(X, y).transform(X) - assert_allclose(svd, reg, rtol=1e-2) - assert_allclose(svd, canonical, rtol=1e-2) + rtol = 1e-3 + # Setting atol because some entries are very close to zero + assert_allclose(svd, reg, atol=reg.max() * rtol) + assert_allclose(svd, canonical, atol=canonical.max() * rtol) def test_svd_flip_1d(): @@ -552,3 +536,142 @@ def test_svd_flip_1d(): assert_allclose(v, v_expected.ravel()) assert_allclose(v, [-1, -2, -3]) + + +def test_loadings_converges(global_random_seed): + """Test that CCA converges. Non-regression test for #19549.""" + X, y = make_regression( + n_samples=200, n_features=20, n_targets=20, random_state=global_random_seed + ) + + cca = CCA(n_components=10, max_iter=500) + + with warnings.catch_warnings(): + warnings.simplefilter("error", ConvergenceWarning) + + cca.fit(X, y) + + # Loadings converges to reasonable values + assert np.all(np.abs(cca.x_loadings_) < 1) + + +def test_pls_constant_y(): + """Checks warning when y is constant. Non-regression test for #19831""" + rng = np.random.RandomState(42) + x = rng.rand(100, 3) + y = np.zeros(100) + + pls = PLSRegression() + + msg = "y residual is constant at iteration" + with pytest.warns(UserWarning, match=msg): + pls.fit(x, y) + + assert_allclose(pls.x_rotations_, 0) + + +@pytest.mark.parametrize("PLSEstimator", [PLSRegression, PLSCanonical, CCA]) +def test_pls_coef_shape(PLSEstimator): + """Check the shape of `coef_` attribute. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/12410 + """ + d = load_linnerud() + X = d.data + y = d.target + + pls = PLSEstimator(copy=True).fit(X, y) + + n_targets, n_features = y.shape[1], X.shape[1] + assert pls.coef_.shape == (n_targets, n_features) + + +@pytest.mark.parametrize("scale", [True, False]) +@pytest.mark.parametrize("PLSEstimator", [PLSRegression, PLSCanonical, CCA]) +def test_pls_prediction(PLSEstimator, scale): + """Check the behaviour of the prediction function.""" + d = load_linnerud() + X = d.data + y = d.target + + pls = PLSEstimator(copy=True, scale=scale).fit(X, y) + y_pred = pls.predict(X, copy=True) + + y_mean = y.mean(axis=0) + X_trans = X - X.mean(axis=0) + + assert_allclose(pls.intercept_, y_mean) + assert_allclose(y_pred, X_trans @ pls.coef_.T + pls.intercept_) + + +@pytest.mark.parametrize("Klass", [CCA, PLSSVD, PLSRegression, PLSCanonical]) +def test_pls_feature_names_out(Klass): + """Check `get_feature_names_out` cross_decomposition module.""" + X, y = load_linnerud(return_X_y=True) + + est = Klass().fit(X, y) + names_out = est.get_feature_names_out() + + class_name_lower = Klass.__name__.lower() + expected_names_out = np.array( + [f"{class_name_lower}{i}" for i in range(est.x_weights_.shape[1])], + dtype=object, + ) + assert_array_equal(names_out, expected_names_out) + + +@pytest.mark.parametrize("Klass", [CCA, PLSSVD, PLSRegression, PLSCanonical]) +def test_pls_set_output(Klass): + """Check `set_output` in cross_decomposition module.""" + pd = pytest.importorskip("pandas") + X, y = load_linnerud(return_X_y=True, as_frame=True) + + est = Klass().set_output(transform="pandas").fit(X, y) + X_trans, y_trans = est.transform(X, y) + assert isinstance(y_trans, np.ndarray) + assert isinstance(X_trans, pd.DataFrame) + assert_array_equal(X_trans.columns, est.get_feature_names_out()) + + +def test_pls_regression_fit_1d_y(): + """Check that when fitting with 1d `y`, prediction should also be 1d. + + Non-regression test for Issue #26549. + """ + X = np.array([[1, 1], [2, 4], [3, 9], [4, 16], [5, 25], [6, 36]]) + y = np.array([2, 6, 12, 20, 30, 42]) + expected = y.copy() + + plsr = PLSRegression().fit(X, y) + y_pred = plsr.predict(X) + assert y_pred.shape == expected.shape + + # Check that it works in VotingRegressor + lr = LinearRegression().fit(X, y) + vr = VotingRegressor([("lr", lr), ("plsr", plsr)]) + y_pred = vr.fit(X, y).predict(X) + assert y_pred.shape == expected.shape + assert_allclose(y_pred, expected) + + +def test_pls_regression_scaling_coef(): + """Check that when using `scale=True`, the coefficients are using the std. dev. from + both `X` and `y`. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/27964 + """ + # handcrafted data where we can predict y from X with an additional scaling factor + rng = np.random.RandomState(0) + coef = rng.uniform(size=(3, 5)) + X = rng.normal(scale=10, size=(30, 5)) # add a std of 10 + y = X @ coef.T + + # we need to make sure that the dimension of the latent space is large enough to + # perfectly predict `y` from `X` (no information loss) + pls = PLSRegression(n_components=5, scale=True).fit(X, y) + assert_allclose(pls.coef_, coef) + + # we therefore should be able to predict `y` from `X` + assert_allclose(pls.predict(X), y) diff --git a/sklearn/datasets/__init__.py b/sklearn/datasets/__init__.py index e7c93bb180567..8863fe489f3b6 100644 --- a/sklearn/datasets/__init__.py +++ b/sklearn/datasets/__init__.py @@ -1,99 +1,166 @@ -""" -The :mod:`sklearn.datasets` module includes utilities to load datasets, -including methods to load and fetch popular reference datasets. It also -features some artificial data generators. -""" -from ._base import load_breast_cancer -from ._base import load_boston -from ._base import load_diabetes -from ._base import load_digits -from ._base import load_files -from ._base import load_iris -from ._base import load_linnerud -from ._base import load_sample_images -from ._base import load_sample_image -from ._base import load_wine -from ._base import get_data_home -from ._base import clear_data_home +"""Utilities to load popular datasets and artificial data generators.""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import textwrap + +from ._base import ( + clear_data_home, + fetch_file, + get_data_home, + load_breast_cancer, + load_diabetes, + load_digits, + load_files, + load_iris, + load_linnerud, + load_sample_image, + load_sample_images, + load_wine, +) +from ._california_housing import fetch_california_housing from ._covtype import fetch_covtype from ._kddcup99 import fetch_kddcup99 -from ._lfw import fetch_lfw_pairs -from ._lfw import fetch_lfw_people -from ._twenty_newsgroups import fetch_20newsgroups -from ._twenty_newsgroups import fetch_20newsgroups_vectorized -from ._openml import fetch_openml -from ._samples_generator import make_classification -from ._samples_generator import make_multilabel_classification -from ._samples_generator import make_hastie_10_2 -from ._samples_generator import make_regression -from ._samples_generator import make_blobs -from ._samples_generator import make_moons -from ._samples_generator import make_circles -from ._samples_generator import make_friedman1 -from ._samples_generator import make_friedman2 -from ._samples_generator import make_friedman3 -from ._samples_generator import make_low_rank_matrix -from ._samples_generator import make_sparse_coded_signal -from ._samples_generator import make_sparse_uncorrelated -from ._samples_generator import make_spd_matrix -from ._samples_generator import make_swiss_roll -from ._samples_generator import make_s_curve -from ._samples_generator import make_sparse_spd_matrix -from ._samples_generator import make_gaussian_quantiles -from ._samples_generator import make_biclusters -from ._samples_generator import make_checkerboard -from ._svmlight_format_io import load_svmlight_file -from ._svmlight_format_io import load_svmlight_files -from ._svmlight_format_io import dump_svmlight_file +from ._lfw import fetch_lfw_pairs, fetch_lfw_people from ._olivetti_faces import fetch_olivetti_faces -from ._species_distributions import fetch_species_distributions -from ._california_housing import fetch_california_housing +from ._openml import fetch_openml from ._rcv1 import fetch_rcv1 +from ._samples_generator import ( + make_biclusters, + make_blobs, + make_checkerboard, + make_circles, + make_classification, + make_friedman1, + make_friedman2, + make_friedman3, + make_gaussian_quantiles, + make_hastie_10_2, + make_low_rank_matrix, + make_moons, + make_multilabel_classification, + make_regression, + make_s_curve, + make_sparse_coded_signal, + make_sparse_spd_matrix, + make_sparse_uncorrelated, + make_spd_matrix, + make_swiss_roll, +) +from ._species_distributions import fetch_species_distributions +from ._svmlight_format_io import ( + dump_svmlight_file, + load_svmlight_file, + load_svmlight_files, +) +from ._twenty_newsgroups import fetch_20newsgroups, fetch_20newsgroups_vectorized + +__all__ = [ + "clear_data_home", + "dump_svmlight_file", + "fetch_20newsgroups", + "fetch_20newsgroups_vectorized", + "fetch_california_housing", + "fetch_covtype", + "fetch_file", + "fetch_kddcup99", + "fetch_lfw_pairs", + "fetch_lfw_people", + "fetch_olivetti_faces", + "fetch_openml", + "fetch_rcv1", + "fetch_species_distributions", + "get_data_home", + "load_breast_cancer", + "load_diabetes", + "load_digits", + "load_files", + "load_iris", + "load_linnerud", + "load_sample_image", + "load_sample_images", + "load_svmlight_file", + "load_svmlight_files", + "load_wine", + "make_biclusters", + "make_blobs", + "make_checkerboard", + "make_circles", + "make_classification", + "make_friedman1", + "make_friedman2", + "make_friedman3", + "make_gaussian_quantiles", + "make_hastie_10_2", + "make_low_rank_matrix", + "make_moons", + "make_multilabel_classification", + "make_regression", + "make_s_curve", + "make_sparse_coded_signal", + "make_sparse_spd_matrix", + "make_sparse_uncorrelated", + "make_spd_matrix", + "make_swiss_roll", +] + + +def __getattr__(name): + if name == "load_boston": + msg = textwrap.dedent( + """ + `load_boston` has been removed from scikit-learn since version 1.2. + + The Boston housing prices dataset has an ethical problem: as + investigated in [1], the authors of this dataset engineered a + non-invertible variable "B" assuming that racial self-segregation had a + positive impact on house prices [2]. Furthermore the goal of the + research that led to the creation of this dataset was to study the + impact of air quality but it did not give adequate demonstration of the + validity of this assumption. + + The scikit-learn maintainers therefore strongly discourage the use of + this dataset unless the purpose of the code is to study and educate + about ethical issues in data science and machine learning. + + In this special case, you can fetch the dataset from the original + source:: + + import pandas as pd + import numpy as np + + data_url = "http://lib.stat.cmu.edu/datasets/boston" + raw_df = pd.read_csv(data_url, sep="\\s+", skiprows=22, header=None) + data = np.hstack([raw_df.values[::2, :], raw_df.values[1::2, :2]]) + target = raw_df.values[1::2, 2] + + Alternative datasets include the California housing dataset and the + Ames housing dataset. You can load the datasets as follows:: + + from sklearn.datasets import fetch_california_housing + housing = fetch_california_housing() + + for the California housing dataset and:: + + from sklearn.datasets import fetch_openml + housing = fetch_openml(name="house_prices", as_frame=True) + + for the Ames housing dataset. + [1] M Carlisle. + "Racist data destruction?" + -__all__ = ['clear_data_home', - 'dump_svmlight_file', - 'fetch_20newsgroups', - 'fetch_20newsgroups_vectorized', - 'fetch_lfw_pairs', - 'fetch_lfw_people', - 'fetch_olivetti_faces', - 'fetch_species_distributions', - 'fetch_california_housing', - 'fetch_covtype', - 'fetch_rcv1', - 'fetch_kddcup99', - 'fetch_openml', - 'get_data_home', - 'load_boston', - 'load_diabetes', - 'load_digits', - 'load_files', - 'load_iris', - 'load_breast_cancer', - 'load_linnerud', - 'load_sample_image', - 'load_sample_images', - 'load_svmlight_file', - 'load_svmlight_files', - 'load_wine', - 'make_biclusters', - 'make_blobs', - 'make_circles', - 'make_classification', - 'make_checkerboard', - 'make_friedman1', - 'make_friedman2', - 'make_friedman3', - 'make_gaussian_quantiles', - 'make_hastie_10_2', - 'make_low_rank_matrix', - 'make_moons', - 'make_multilabel_classification', - 'make_regression', - 'make_s_curve', - 'make_sparse_coded_signal', - 'make_sparse_spd_matrix', - 'make_sparse_uncorrelated', - 'make_spd_matrix', - 'make_swiss_roll'] + [2] Harrison Jr, David, and Daniel L. Rubinfeld. + "Hedonic housing prices and the demand for clean air." + Journal of environmental economics and management 5.1 (1978): 81-102. + + """ + ) + raise ImportError(msg) + try: + return globals()[name] + except KeyError: + # This is turned into the appropriate ImportError + raise AttributeError diff --git a/sklearn/datasets/_arff_parser.py b/sklearn/datasets/_arff_parser.py new file mode 100644 index 0000000000000..fb6e629a73c8d --- /dev/null +++ b/sklearn/datasets/_arff_parser.py @@ -0,0 +1,543 @@ +"""Implementation of ARFF parsers: via LIAC-ARFF and pandas.""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import itertools +import re +from collections import OrderedDict +from collections.abc import Generator +from typing import List + +import numpy as np +import scipy as sp + +from ..externals import _arff +from ..externals._arff import ArffSparseDataType +from ..utils._chunking import chunk_generator, get_chunk_n_rows +from ..utils._optional_dependencies import check_pandas_support +from ..utils.fixes import pd_fillna + + +def _split_sparse_columns( + arff_data: ArffSparseDataType, include_columns: List +) -> ArffSparseDataType: + """Obtains several columns from sparse ARFF representation. Additionally, + the column indices are re-labelled, given the columns that are not + included. (e.g., when including [1, 2, 3], the columns will be relabelled + to [0, 1, 2]). + + Parameters + ---------- + arff_data : tuple + A tuple of three lists of equal size; first list indicating the value, + second the x coordinate and the third the y coordinate. + + include_columns : list + A list of columns to include. + + Returns + ------- + arff_data_new : tuple + Subset of arff data with only the include columns indicated by the + include_columns argument. + """ + arff_data_new: ArffSparseDataType = (list(), list(), list()) + reindexed_columns = { + column_idx: array_idx for array_idx, column_idx in enumerate(include_columns) + } + for val, row_idx, col_idx in zip(arff_data[0], arff_data[1], arff_data[2]): + if col_idx in include_columns: + arff_data_new[0].append(val) + arff_data_new[1].append(row_idx) + arff_data_new[2].append(reindexed_columns[col_idx]) + return arff_data_new + + +def _sparse_data_to_array( + arff_data: ArffSparseDataType, include_columns: List +) -> np.ndarray: + # turns the sparse data back into an array (can't use toarray() function, + # as this does only work on numeric data) + num_obs = max(arff_data[1]) + 1 + y_shape = (num_obs, len(include_columns)) + reindexed_columns = { + column_idx: array_idx for array_idx, column_idx in enumerate(include_columns) + } + # TODO: improve for efficiency + y = np.empty(y_shape, dtype=np.float64) + for val, row_idx, col_idx in zip(arff_data[0], arff_data[1], arff_data[2]): + if col_idx in include_columns: + y[row_idx, reindexed_columns[col_idx]] = val + return y + + +def _post_process_frame(frame, feature_names, target_names): + """Post process a dataframe to select the desired columns in `X` and `y`. + + Parameters + ---------- + frame : dataframe + The dataframe to split into `X` and `y`. + + feature_names : list of str + The list of feature names to populate `X`. + + target_names : list of str + The list of target names to populate `y`. + + Returns + ------- + X : dataframe + The dataframe containing the features. + + y : {series, dataframe} or None + The series or dataframe containing the target. + """ + X = frame[feature_names] + if len(target_names) >= 2: + y = frame[target_names] + elif len(target_names) == 1: + y = frame[target_names[0]] + else: + y = None + return X, y + + +def _liac_arff_parser( + gzip_file, + output_arrays_type, + openml_columns_info, + feature_names_to_select, + target_names_to_select, + shape=None, +): + """ARFF parser using the LIAC-ARFF library coded purely in Python. + + This parser is quite slow but consumes a generator. Currently it is needed + to parse sparse datasets. For dense datasets, it is recommended to instead + use the pandas-based parser, although it does not always handles the + dtypes exactly the same. + + Parameters + ---------- + gzip_file : GzipFile instance + The file compressed to be read. + + output_arrays_type : {"numpy", "sparse", "pandas"} + The type of the arrays that will be returned. The possibilities ara: + + - `"numpy"`: both `X` and `y` will be NumPy arrays; + - `"sparse"`: `X` will be sparse matrix and `y` will be a NumPy array; + - `"pandas"`: `X` will be a pandas DataFrame and `y` will be either a + pandas Series or DataFrame. + + columns_info : dict + The information provided by OpenML regarding the columns of the ARFF + file. + + feature_names_to_select : list of str + A list of the feature names to be selected. + + target_names_to_select : list of str + A list of the target names to be selected. + + Returns + ------- + X : {ndarray, sparse matrix, dataframe} + The data matrix. + + y : {ndarray, dataframe, series} + The target. + + frame : dataframe or None + A dataframe containing both `X` and `y`. `None` if + `output_array_type != "pandas"`. + + categories : list of str or None + The names of the features that are categorical. `None` if + `output_array_type == "pandas"`. + """ + + def _io_to_generator(gzip_file): + for line in gzip_file: + yield line.decode("utf-8") + + stream = _io_to_generator(gzip_file) + + # find which type (dense or sparse) ARFF type we will have to deal with + return_type = _arff.COO if output_arrays_type == "sparse" else _arff.DENSE_GEN + # we should not let LIAC-ARFF to encode the nominal attributes with NumPy + # arrays to have only numerical values. + encode_nominal = not (output_arrays_type == "pandas") + arff_container = _arff.load( + stream, return_type=return_type, encode_nominal=encode_nominal + ) + columns_to_select = feature_names_to_select + target_names_to_select + + categories = { + name: cat + for name, cat in arff_container["attributes"] + if isinstance(cat, list) and name in columns_to_select + } + if output_arrays_type == "pandas": + pd = check_pandas_support("fetch_openml with as_frame=True") + + columns_info = OrderedDict(arff_container["attributes"]) + columns_names = list(columns_info.keys()) + + # calculate chunksize + first_row = next(arff_container["data"]) + first_df = pd.DataFrame([first_row], columns=columns_names, copy=False) + + row_bytes = first_df.memory_usage(deep=True).sum() + chunksize = get_chunk_n_rows(row_bytes) + + # read arff data with chunks + columns_to_keep = [col for col in columns_names if col in columns_to_select] + dfs = [first_df[columns_to_keep]] + for data in chunk_generator(arff_container["data"], chunksize): + dfs.append( + pd.DataFrame(data, columns=columns_names, copy=False)[columns_to_keep] + ) + # dfs[0] contains only one row, which may not have enough data to infer to + # column's dtype. Here we use `dfs[1]` to configure the dtype in dfs[0] + if len(dfs) >= 2: + dfs[0] = dfs[0].astype(dfs[1].dtypes) + + # liac-arff parser does not depend on NumPy and uses None to represent + # missing values. To be consistent with the pandas parser, we replace + # None with np.nan. + frame = pd.concat(dfs, ignore_index=True) + frame = pd_fillna(pd, frame) + del dfs, first_df + + # cast the columns frame + dtypes = {} + for name in frame.columns: + column_dtype = openml_columns_info[name]["data_type"] + if column_dtype.lower() == "integer": + # Use a pandas extension array instead of np.int64 to be able + # to support missing values. + dtypes[name] = "Int64" + elif column_dtype.lower() == "nominal": + dtypes[name] = "category" + else: + dtypes[name] = frame.dtypes[name] + frame = frame.astype(dtypes) + + X, y = _post_process_frame( + frame, feature_names_to_select, target_names_to_select + ) + else: + arff_data = arff_container["data"] + + feature_indices_to_select = [ + int(openml_columns_info[col_name]["index"]) + for col_name in feature_names_to_select + ] + target_indices_to_select = [ + int(openml_columns_info[col_name]["index"]) + for col_name in target_names_to_select + ] + + if isinstance(arff_data, Generator): + if shape is None: + raise ValueError( + "shape must be provided when arr['data'] is a Generator" + ) + if shape[0] == -1: + count = -1 + else: + count = shape[0] * shape[1] + data = np.fromiter( + itertools.chain.from_iterable(arff_data), + dtype="float64", + count=count, + ) + data = data.reshape(*shape) + X = data[:, feature_indices_to_select] + y = data[:, target_indices_to_select] + elif isinstance(arff_data, tuple): + arff_data_X = _split_sparse_columns(arff_data, feature_indices_to_select) + num_obs = max(arff_data[1]) + 1 + X_shape = (num_obs, len(feature_indices_to_select)) + X = sp.sparse.coo_matrix( + (arff_data_X[0], (arff_data_X[1], arff_data_X[2])), + shape=X_shape, + dtype=np.float64, + ) + X = X.tocsr() + y = _sparse_data_to_array(arff_data, target_indices_to_select) + else: + # This should never happen + raise ValueError( + f"Unexpected type for data obtained from arff: {type(arff_data)}" + ) + + is_classification = { + col_name in categories for col_name in target_names_to_select + } + if not is_classification: + # No target + pass + elif all(is_classification): + y = np.hstack( + [ + np.take( + np.asarray(categories.pop(col_name), dtype="O"), + y[:, i : i + 1].astype(int, copy=False), + ) + for i, col_name in enumerate(target_names_to_select) + ] + ) + elif any(is_classification): + raise ValueError( + "Mix of nominal and non-nominal targets is not currently supported" + ) + + # reshape y back to 1-D array, if there is only 1 target column; + # back to None if there are not target columns + if y.shape[1] == 1: + y = y.reshape((-1,)) + elif y.shape[1] == 0: + y = None + + if output_arrays_type == "pandas": + return X, y, frame, None + return X, y, None, categories + + +def _pandas_arff_parser( + gzip_file, + output_arrays_type, + openml_columns_info, + feature_names_to_select, + target_names_to_select, + read_csv_kwargs=None, +): + """ARFF parser using `pandas.read_csv`. + + This parser uses the metadata fetched directly from OpenML and skips the metadata + headers of ARFF file itself. The data is loaded as a CSV file. + + Parameters + ---------- + gzip_file : GzipFile instance + The GZip compressed file with the ARFF formatted payload. + + output_arrays_type : {"numpy", "sparse", "pandas"} + The type of the arrays that will be returned. The possibilities are: + + - `"numpy"`: both `X` and `y` will be NumPy arrays; + - `"sparse"`: `X` will be sparse matrix and `y` will be a NumPy array; + - `"pandas"`: `X` will be a pandas DataFrame and `y` will be either a + pandas Series or DataFrame. + + openml_columns_info : dict + The information provided by OpenML regarding the columns of the ARFF + file. + + feature_names_to_select : list of str + A list of the feature names to be selected to build `X`. + + target_names_to_select : list of str + A list of the target names to be selected to build `y`. + + read_csv_kwargs : dict, default=None + Keyword arguments to pass to `pandas.read_csv`. It allows to overwrite + the default options. + + Returns + ------- + X : {ndarray, sparse matrix, dataframe} + The data matrix. + + y : {ndarray, dataframe, series} + The target. + + frame : dataframe or None + A dataframe containing both `X` and `y`. `None` if + `output_array_type != "pandas"`. + + categories : list of str or None + The names of the features that are categorical. `None` if + `output_array_type == "pandas"`. + """ + import pandas as pd + + # read the file until the data section to skip the ARFF metadata headers + for line in gzip_file: + if line.decode("utf-8").lower().startswith("@data"): + break + + dtypes = {} + for name in openml_columns_info: + column_dtype = openml_columns_info[name]["data_type"] + if column_dtype.lower() == "integer": + # Use Int64 to infer missing values from data + # XXX: this line is not covered by our tests. Is this really needed? + dtypes[name] = "Int64" + elif column_dtype.lower() == "nominal": + dtypes[name] = "category" + # since we will not pass `names` when reading the ARFF file, we need to translate + # `dtypes` from column names to column indices to pass to `pandas.read_csv` + dtypes_positional = { + col_idx: dtypes[name] + for col_idx, name in enumerate(openml_columns_info) + if name in dtypes + } + + default_read_csv_kwargs = { + "header": None, + "index_col": False, # always force pandas to not use the first column as index + "na_values": ["?"], # missing values are represented by `?` + "keep_default_na": False, # only `?` is a missing value given the ARFF specs + "comment": "%", # skip line starting by `%` since they are comments + "quotechar": '"', # delimiter to use for quoted strings + "skipinitialspace": True, # skip spaces after delimiter to follow ARFF specs + "escapechar": "\\", + "dtype": dtypes_positional, + } + read_csv_kwargs = {**default_read_csv_kwargs, **(read_csv_kwargs or {})} + frame = pd.read_csv(gzip_file, **read_csv_kwargs) + try: + # Setting the columns while reading the file will select the N first columns + # and not raise a ParserError. Instead, we set the columns after reading the + # file and raise a ParserError if the number of columns does not match the + # number of columns in the metadata given by OpenML. + frame.columns = [name for name in openml_columns_info] + except ValueError as exc: + raise pd.errors.ParserError( + "The number of columns provided by OpenML does not match the number of " + "columns inferred by pandas when reading the file." + ) from exc + + columns_to_select = feature_names_to_select + target_names_to_select + columns_to_keep = [col for col in frame.columns if col in columns_to_select] + frame = frame[columns_to_keep] + + # `pd.read_csv` automatically handles double quotes for quoting non-numeric + # CSV cell values. Contrary to LIAC-ARFF, `pd.read_csv` cannot be configured to + # consider either single quotes and double quotes as valid quoting chars at + # the same time since this case does not occur in regular (non-ARFF) CSV files. + # To mimic the behavior of LIAC-ARFF parser, we manually strip single quotes + # on categories as a post-processing steps if needed. + # + # Note however that we intentionally do not attempt to do this kind of manual + # post-processing of (non-categorical) string-typed columns because we cannot + # resolve the ambiguity of the case of CSV cell with nesting quoting such as + # `"'some string value'"` with pandas. + single_quote_pattern = re.compile(r"^'(?P.*)'$") + + def strip_single_quotes(input_string): + match = re.search(single_quote_pattern, input_string) + if match is None: + return input_string + + return match.group("contents") + + categorical_columns = [ + name + for name, dtype in frame.dtypes.items() + if isinstance(dtype, pd.CategoricalDtype) + ] + for col in categorical_columns: + frame[col] = frame[col].cat.rename_categories(strip_single_quotes) + + X, y = _post_process_frame(frame, feature_names_to_select, target_names_to_select) + + if output_arrays_type == "pandas": + return X, y, frame, None + else: + X, y = X.to_numpy(), y.to_numpy() + + categories = { + name: dtype.categories.tolist() + for name, dtype in frame.dtypes.items() + if isinstance(dtype, pd.CategoricalDtype) + } + return X, y, None, categories + + +def load_arff_from_gzip_file( + gzip_file, + parser, + output_type, + openml_columns_info, + feature_names_to_select, + target_names_to_select, + shape=None, + read_csv_kwargs=None, +): + """Load a compressed ARFF file using a given parser. + + Parameters + ---------- + gzip_file : GzipFile instance + The file compressed to be read. + + parser : {"pandas", "liac-arff"} + The parser used to parse the ARFF file. "pandas" is recommended + but only supports loading dense datasets. + + output_type : {"numpy", "sparse", "pandas"} + The type of the arrays that will be returned. The possibilities ara: + + - `"numpy"`: both `X` and `y` will be NumPy arrays; + - `"sparse"`: `X` will be sparse matrix and `y` will be a NumPy array; + - `"pandas"`: `X` will be a pandas DataFrame and `y` will be either a + pandas Series or DataFrame. + + openml_columns_info : dict + The information provided by OpenML regarding the columns of the ARFF + file. + + feature_names_to_select : list of str + A list of the feature names to be selected. + + target_names_to_select : list of str + A list of the target names to be selected. + + read_csv_kwargs : dict, default=None + Keyword arguments to pass to `pandas.read_csv`. It allows to overwrite + the default options. + + Returns + ------- + X : {ndarray, sparse matrix, dataframe} + The data matrix. + + y : {ndarray, dataframe, series} + The target. + + frame : dataframe or None + A dataframe containing both `X` and `y`. `None` if + `output_array_type != "pandas"`. + + categories : list of str or None + The names of the features that are categorical. `None` if + `output_array_type == "pandas"`. + """ + if parser == "liac-arff": + return _liac_arff_parser( + gzip_file, + output_type, + openml_columns_info, + feature_names_to_select, + target_names_to_select, + shape, + ) + elif parser == "pandas": + return _pandas_arff_parser( + gzip_file, + output_type, + openml_columns_info, + feature_names_to_select, + target_names_to_select, + read_csv_kwargs, + ) + else: + raise ValueError( + f"Unknown parser: '{parser}'. Should be 'liac-arff' or 'pandas'." + ) diff --git a/sklearn/datasets/_base.py b/sklearn/datasets/_base.py index 17d2db9f2075b..e6e6939ddbc19 100644 --- a/sklearn/datasets/_base.py +++ b/sklearn/datasets/_base.py @@ -2,38 +2,56 @@ Base IO code for all datasets """ -# Copyright (c) 2007 David Cournapeau -# 2010 Fabian Pedregosa -# 2010 Olivier Grisel -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import csv +import gzip import hashlib import os +import re import shutil +import time +import unicodedata +import warnings from collections import namedtuple +from importlib import resources +from numbers import Integral from os import environ, listdir, makedirs -from os.path import dirname, expanduser, isdir, join, splitext - -from ..utils import Bunch -from ..utils import check_random_state -from ..utils import check_pandas_support -from ..utils.validation import _deprecate_positional_args +from os.path import expanduser, isdir, join, splitext +from pathlib import Path +from tempfile import NamedTemporaryFile +from urllib.error import URLError +from urllib.parse import urlparse +from urllib.request import urlretrieve import numpy as np -from urllib.request import urlretrieve +from ..preprocessing import scale +from ..utils import Bunch, check_random_state +from ..utils._optional_dependencies import check_pandas_support +from ..utils._param_validation import Interval, StrOptions, validate_params -RemoteFileMetadata = namedtuple('RemoteFileMetadata', - ['filename', 'url', 'checksum']) +DATA_MODULE = "sklearn.datasets.data" +DESCR_MODULE = "sklearn.datasets.descr" +IMAGES_MODULE = "sklearn.datasets.images" +RemoteFileMetadata = namedtuple("RemoteFileMetadata", ["filename", "url", "checksum"]) + +@validate_params( + { + "data_home": [str, os.PathLike, None], + }, + prefer_skip_nested_validation=True, +) def get_data_home(data_home=None) -> str: - """Return the path of the scikit-learn data dir. + """Return the path of the scikit-learn data directory. This folder is used by some large dataset loaders to avoid downloading the data several times. - By default the data dir is set to a folder named 'scikit_learn_data' in the + By default the data directory is set to a folder named 'scikit_learn_data' in the user home folder. Alternatively, it can be set by the 'SCIKIT_LEARN_DATA' environment @@ -44,40 +62,62 @@ def get_data_home(data_home=None) -> str: Parameters ---------- - data_home : str, default=None + data_home : str or path-like, default=None The path to scikit-learn data directory. If `None`, the default path - is `~/sklearn_learn_data`. + is `~/scikit_learn_data`. + + Returns + ------- + data_home: str + The path to scikit-learn data directory. + + Examples + -------- + >>> import os + >>> from sklearn.datasets import get_data_home + >>> data_home_path = get_data_home() + >>> os.path.exists(data_home_path) + True """ if data_home is None: - data_home = environ.get('SCIKIT_LEARN_DATA', - join('~', 'scikit_learn_data')) + data_home = environ.get("SCIKIT_LEARN_DATA", join("~", "scikit_learn_data")) data_home = expanduser(data_home) makedirs(data_home, exist_ok=True) return data_home +@validate_params( + { + "data_home": [str, os.PathLike, None], + }, + prefer_skip_nested_validation=True, +) def clear_data_home(data_home=None): """Delete all the content of the data home cache. Parameters ---------- - data_home : str, default=None + data_home : str or path-like, default=None The path to scikit-learn data directory. If `None`, the default path - is `~/sklearn_learn_data`. + is `~/scikit_learn_data`. + + Examples + -------- + >>> from sklearn.datasets import clear_data_home + >>> clear_data_home() # doctest: +SKIP """ data_home = get_data_home(data_home) shutil.rmtree(data_home) -def _convert_data_dataframe(caller_name, data, target, - feature_names, target_names, sparse_data=False): - pd = check_pandas_support('{} with as_frame=True'.format(caller_name)) +def _convert_data_dataframe( + caller_name, data, target, feature_names, target_names, sparse_data=False +): + pd = check_pandas_support("{} with as_frame=True".format(caller_name)) if not sparse_data: - data_df = pd.DataFrame(data, columns=feature_names) + data_df = pd.DataFrame(data, columns=feature_names, copy=False) else: - data_df = pd.DataFrame.sparse.from_spmatrix( - data, columns=feature_names - ) + data_df = pd.DataFrame.sparse.from_spmatrix(data, columns=feature_names) target_df = pd.DataFrame(target, columns=target_names) combined_df = pd.concat([data_df, target_df], axis=1) @@ -88,15 +128,39 @@ def _convert_data_dataframe(caller_name, data, target, return combined_df, X, y -@_deprecate_positional_args -def load_files(container_path, *, description=None, categories=None, - load_content=True, shuffle=True, encoding=None, - decode_error='strict', random_state=0): +@validate_params( + { + "container_path": [str, os.PathLike], + "description": [str, None], + "categories": [list, None], + "load_content": ["boolean"], + "shuffle": ["boolean"], + "encoding": [str, None], + "decode_error": [StrOptions({"strict", "ignore", "replace"})], + "random_state": ["random_state"], + "allowed_extensions": [list, None], + }, + prefer_skip_nested_validation=True, +) +def load_files( + container_path, + *, + description=None, + categories=None, + load_content=True, + shuffle=True, + encoding=None, + decode_error="strict", + random_state=0, + allowed_extensions=None, +): """Load text files with categories as subfolder names. Individual samples are assumed to be files stored a two levels folder structure such as the following: + .. code-block:: text + container_folder/ category_1_folder/ file_1.txt @@ -116,7 +180,7 @@ def load_files(container_path, *, description=None, categories=None, load the files in memory. To use text files in a scikit-learn classification or clustering algorithm, - you will need to use the :mod`~sklearn.feature_extraction.text` module to + you will need to use the :mod:`~sklearn.feature_extraction.text` module to build a feature extraction transformer that suits your problem. If you set load_content=True, you should also specify the encoding of the @@ -128,14 +192,17 @@ def load_files(container_path, *, description=None, categories=None, Similar feature extractors should be built for other kind of unstructured data input such as images, audio, video, ... + If you want files with a specific file extension (e.g. `.txt`) then you + can pass a list of those file extensions to `allowed_extensions`. + Read more in the :ref:`User Guide `. Parameters ---------- - container_path : str or unicode - Path to the main folder holding one subfolder per category + container_path : str + Path to the main folder holding one subfolder per category. - description : str or unicode, default=None + description : str, default=None A paragraph describing the characteristic of the dataset: its source, reference, etc. @@ -169,6 +236,9 @@ def load_files(container_path, *, description=None, categories=None, for reproducible output across multiple function calls. See :term:`Glossary `. + allowed_extensions : list of str, default=None + List of desired file extensions to filter the files to be loaded. + Returns ------- data : :class:`~sklearn.utils.Bunch` @@ -185,22 +255,40 @@ def load_files(container_path, *, description=None, categories=None, The full description of the dataset. filenames: ndarray The filenames holding the dataset. + + Examples + -------- + >>> from sklearn.datasets import load_files + >>> container_path = "./" + >>> load_files(container_path) # doctest: +SKIP """ + target = [] target_names = [] filenames = [] - folders = [f for f in sorted(listdir(container_path)) - if isdir(join(container_path, f))] + folders = [ + f for f in sorted(listdir(container_path)) if isdir(join(container_path, f)) + ] if categories is not None: folders = [f for f in folders if f in categories] + if allowed_extensions is not None: + allowed_extensions = frozenset(allowed_extensions) + for label, folder in enumerate(folders): target_names.append(folder) folder_path = join(container_path, folder) - documents = [join(folder_path, d) - for d in sorted(listdir(folder_path))] + files = sorted(listdir(folder_path)) + if allowed_extensions is not None: + documents = [ + join(folder_path, file) + for file in files + if os.path.splitext(file)[1] in allowed_extensions + ] + else: + documents = [join(folder_path, file) for file in files] target.extend(len(documents) * [label]) filenames.extend(documents) @@ -218,49 +306,76 @@ def load_files(container_path, *, description=None, categories=None, if load_content: data = [] for filename in filenames: - with open(filename, 'rb') as f: - data.append(f.read()) + data.append(Path(filename).read_bytes()) if encoding is not None: data = [d.decode(encoding, decode_error) for d in data] - return Bunch(data=data, - filenames=filenames, - target_names=target_names, - target=target, - DESCR=description) + return Bunch( + data=data, + filenames=filenames, + target_names=target_names, + target=target, + DESCR=description, + ) - return Bunch(filenames=filenames, - target_names=target_names, - target=target, - DESCR=description) + return Bunch( + filenames=filenames, target_names=target_names, target=target, DESCR=description + ) -def load_data(module_path, data_file_name): - """Loads data from module_path/data/data_file_name. +def load_csv_data( + data_file_name, + *, + data_module=DATA_MODULE, + descr_file_name=None, + descr_module=DESCR_MODULE, + encoding="utf-8", +): + """Loads `data_file_name` from `data_module with `importlib.resources`. Parameters ---------- - module_path : string - The module path. + data_file_name : str + Name of csv file to be loaded from `data_module/data_file_name`. + For example `'wine_data.csv'`. - data_file_name : string - Name of csv file to be loaded from - module_path/data/data_file_name. For example 'wine_data.csv'. + data_module : str or module, default='sklearn.datasets.data' + Module where data lives. The default is `'sklearn.datasets.data'`. + + descr_file_name : str, default=None + Name of rst file to be loaded from `descr_module/descr_file_name`. + For example `'wine_data.rst'`. See also :func:`load_descr`. + If not None, also returns the corresponding description of + the dataset. + + descr_module : str or module, default='sklearn.datasets.descr' + Module where `descr_file_name` lives. See also :func:`load_descr`. + The default is `'sklearn.datasets.descr'`. Returns ------- - data : Numpy array + data : ndarray of shape (n_samples, n_features) A 2D array with each row representing one sample and each column representing the features of a given sample. - target : Numpy array - A 1D array holding target variables for all the samples in `data. - For example target[0] is the target varible for data[0]. + target : ndarry of shape (n_samples,) + A 1D array holding target variables for all the samples in `data`. + For example target[0] is the target variable for data[0]. - target_names : Numpy array + target_names : ndarry of shape (n_samples,) A 1D array containing the names of the classifications. For example target_names[0] is the name of the target[0] class. + + descr : str, optional + Description of the dataset (the content of `descr_file_name`). + Only returned if `descr_file_name` is not None. + + encoding : str, optional + Text encoding of the CSV file. + + .. versionadded:: 1.4 """ - with open(join(module_path, 'data', data_file_name)) as csv_file: + data_path = resources.files(data_module) / data_file_name + with data_path.open("r", encoding="utf-8") as csv_file: data_file = csv.reader(csv_file) temp = next(data_file) n_samples = int(temp[0]) @@ -273,10 +388,116 @@ def load_data(module_path, data_file_name): data[i] = np.asarray(ir[:-1], dtype=np.float64) target[i] = np.asarray(ir[-1], dtype=int) - return data, target, target_names + if descr_file_name is None: + return data, target, target_names + else: + assert descr_module is not None + descr = load_descr(descr_module=descr_module, descr_file_name=descr_file_name) + return data, target, target_names, descr + + +def load_gzip_compressed_csv_data( + data_file_name, + *, + data_module=DATA_MODULE, + descr_file_name=None, + descr_module=DESCR_MODULE, + encoding="utf-8", + **kwargs, +): + """Loads gzip-compressed with `importlib.resources`. + + 1) Open resource file with `importlib.resources.open_binary` + 2) Decompress file obj with `gzip.open` + 3) Load decompressed data with `np.loadtxt` + + Parameters + ---------- + data_file_name : str + Name of gzip-compressed csv file (`'*.csv.gz'`) to be loaded from + `data_module/data_file_name`. For example `'diabetes_data.csv.gz'`. + + data_module : str or module, default='sklearn.datasets.data' + Module where data lives. The default is `'sklearn.datasets.data'`. + + descr_file_name : str, default=None + Name of rst file to be loaded from `descr_module/descr_file_name`. + For example `'wine_data.rst'`. See also :func:`load_descr`. + If not None, also returns the corresponding description of + the dataset. + + descr_module : str or module, default='sklearn.datasets.descr' + Module where `descr_file_name` lives. See also :func:`load_descr`. + The default is `'sklearn.datasets.descr'`. + + encoding : str, default="utf-8" + Name of the encoding that the gzip-decompressed file will be + decoded with. The default is 'utf-8'. + + **kwargs : dict, optional + Keyword arguments to be passed to `np.loadtxt`; + e.g. delimiter=','. + + Returns + ------- + data : ndarray of shape (n_samples, n_features) + A 2D array with each row representing one sample and each column + representing the features and/or target of a given sample. + + descr : str, optional + Description of the dataset (the content of `descr_file_name`). + Only returned if `descr_file_name` is not None. + """ + data_path = resources.files(data_module) / data_file_name + with data_path.open("rb") as compressed_file: + compressed_file = gzip.open(compressed_file, mode="rt", encoding=encoding) + data = np.loadtxt(compressed_file, **kwargs) + + if descr_file_name is None: + return data + else: + assert descr_module is not None + descr = load_descr(descr_module=descr_module, descr_file_name=descr_file_name) + return data, descr + + +def load_descr(descr_file_name, *, descr_module=DESCR_MODULE, encoding="utf-8"): + """Load `descr_file_name` from `descr_module` with `importlib.resources`. + + Parameters + ---------- + descr_file_name : str, default=None + Name of rst file to be loaded from `descr_module/descr_file_name`. + For example `'wine_data.rst'`. See also :func:`load_descr`. + If not None, also returns the corresponding description of + the dataset. + + descr_module : str or module, default='sklearn.datasets.descr' + Module where `descr_file_name` lives. See also :func:`load_descr`. + The default is `'sklearn.datasets.descr'`. + + encoding : str, default="utf-8" + Name of the encoding that `descr_file_name` will be decoded with. + The default is 'utf-8'. + .. versionadded:: 1.4 -@_deprecate_positional_args + Returns + ------- + fdescr : str + Content of `descr_file_name`. + """ + path = resources.files(descr_module) / descr_file_name + return path.read_text(encoding=encoding) + + +@validate_params( + { + "return_X_y": ["boolean"], + "as_frame": ["boolean"], + }, + prefer_skip_nested_validation=True, +) def load_wine(*, return_X_y=False, as_frame=False): """Load and return the wine dataset (classification). @@ -293,6 +514,10 @@ def load_wine(*, return_X_y=False, as_frame=False): Features real, positive ================= ============== + The copy of UCI ML Wine Data Set dataset is downloaded and modified to fit + standard format from: + https://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data + Read more in the :ref:`User Guide `. Parameters @@ -334,10 +559,9 @@ def load_wine(*, return_X_y=False, as_frame=False): The full description of the dataset. (data, target) : tuple if ``return_X_y`` is True - - The copy of UCI ML Wine Data Set dataset is downloaded and modified to fit - standard format from: - https://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data + A tuple of two ndarrays by default. The first contains a 2D array of shape + (178, 13) with each row representing one sample and each column representing + the features. The second array of shape (178,) contains the target samples. Examples -------- @@ -349,49 +573,55 @@ def load_wine(*, return_X_y=False, as_frame=False): >>> data.target[[10, 80, 140]] array([0, 1, 2]) >>> list(data.target_names) - ['class_0', 'class_1', 'class_2'] + [np.str_('class_0'), np.str_('class_1'), np.str_('class_2')] """ - module_path = dirname(__file__) - data, target, target_names = load_data(module_path, 'wine_data.csv') - - with open(join(module_path, 'descr', 'wine_data.rst')) as rst_file: - fdescr = rst_file.read() - - feature_names = ['alcohol', - 'malic_acid', - 'ash', - 'alcalinity_of_ash', - 'magnesium', - 'total_phenols', - 'flavanoids', - 'nonflavanoid_phenols', - 'proanthocyanins', - 'color_intensity', - 'hue', - 'od280/od315_of_diluted_wines', - 'proline'] + + data, target, target_names, fdescr = load_csv_data( + data_file_name="wine_data.csv", descr_file_name="wine_data.rst" + ) + + feature_names = [ + "alcohol", + "malic_acid", + "ash", + "alcalinity_of_ash", + "magnesium", + "total_phenols", + "flavanoids", + "nonflavanoid_phenols", + "proanthocyanins", + "color_intensity", + "hue", + "od280/od315_of_diluted_wines", + "proline", + ] frame = None - target_columns = ['target', ] + target_columns = [ + "target", + ] if as_frame: - frame, data, target = _convert_data_dataframe("load_wine", - data, - target, - feature_names, - target_columns) + frame, data, target = _convert_data_dataframe( + "load_wine", data, target, feature_names, target_columns + ) if return_X_y: return data, target - return Bunch(data=data, - target=target, - frame=frame, - target_names=target_names, - DESCR=fdescr, - feature_names=feature_names) + return Bunch( + data=data, + target=target, + frame=frame, + target_names=target_names, + DESCR=fdescr, + feature_names=feature_names, + ) -@_deprecate_positional_args +@validate_params( + {"return_X_y": ["boolean"], "as_frame": ["boolean"]}, + prefer_skip_nested_validation=True, +) def load_iris(*, return_X_y=False, as_frame=False): """Load and return the iris dataset (classification). @@ -408,6 +638,11 @@ def load_iris(*, return_X_y=False, as_frame=False): Read more in the :ref:`User Guide `. + .. versionchanged:: 0.20 + Fixed two wrong data points according to Fisher's paper. + The new version is the same as in R, but not as in the UCI + Machine Learning Repository. + Parameters ---------- return_X_y : bool, default=False @@ -438,7 +673,7 @@ def load_iris(*, return_X_y=False, as_frame=False): a pandas Series. feature_names: list The names of the dataset columns. - target_names: list + target_names: ndarray of shape (3, ) The names of target classes. frame: DataFrame of shape (150, 5) Only present when `as_frame=True`. DataFrame with `data` and @@ -453,16 +688,13 @@ def load_iris(*, return_X_y=False, as_frame=False): .. versionadded:: 0.20 (data, target) : tuple if ``return_X_y`` is True + A tuple of two ndarray. The first containing a 2D array of shape + (n_samples, n_features) with each row representing one sample and + each column representing the features. The second ndarray of shape + (n_samples,) containing the target samples. .. versionadded:: 0.18 - Notes - ----- - .. versionchanged:: 0.20 - Fixed two wrong data points according to Fisher's paper. - The new version is the same as in R, but not as in the UCI - Machine Learning Repository. - Examples -------- Let's say you are interested in the samples 10, 25, and 50, and want to @@ -473,42 +705,53 @@ def load_iris(*, return_X_y=False, as_frame=False): >>> data.target[[10, 25, 50]] array([0, 0, 1]) >>> list(data.target_names) - ['setosa', 'versicolor', 'virginica'] - """ - module_path = dirname(__file__) - data, target, target_names = load_data(module_path, 'iris.csv') - iris_csv_filename = join(module_path, 'data', 'iris.csv') - - with open(join(module_path, 'descr', 'iris.rst')) as rst_file: - fdescr = rst_file.read() + [np.str_('setosa'), np.str_('versicolor'), np.str_('virginica')] - feature_names = ['sepal length (cm)', 'sepal width (cm)', - 'petal length (cm)', 'petal width (cm)'] + See :ref:`sphx_glr_auto_examples_decomposition_plot_pca_iris.py` for a more + detailed example of how to work with the iris dataset. + """ + data_file_name = "iris.csv" + data, target, target_names, fdescr = load_csv_data( + data_file_name=data_file_name, descr_file_name="iris.rst" + ) + + feature_names = [ + "sepal length (cm)", + "sepal width (cm)", + "petal length (cm)", + "petal width (cm)", + ] frame = None - target_columns = ['target', ] + target_columns = [ + "target", + ] if as_frame: - frame, data, target = _convert_data_dataframe("load_iris", - data, - target, - feature_names, - target_columns) + frame, data, target = _convert_data_dataframe( + "load_iris", data, target, feature_names, target_columns + ) if return_X_y: return data, target - return Bunch(data=data, - target=target, - frame=frame, - target_names=target_names, - DESCR=fdescr, - feature_names=feature_names, - filename=iris_csv_filename) - - -@_deprecate_positional_args + return Bunch( + data=data, + target=target, + frame=frame, + target_names=target_names, + DESCR=fdescr, + feature_names=feature_names, + filename=data_file_name, + data_module=DATA_MODULE, + ) + + +@validate_params( + {"return_X_y": ["boolean"], "as_frame": ["boolean"]}, + prefer_skip_nested_validation=True, +) def load_breast_cancer(*, return_X_y=False, as_frame=False): - """Load and return the breast cancer wisconsin dataset (classification). + """Load and return the breast cancer Wisconsin dataset (classification). The breast cancer dataset is a classic and very easy binary classification dataset. @@ -521,6 +764,10 @@ def load_breast_cancer(*, return_X_y=False, as_frame=False): Features real, positive ================= ============== + The copy of UCI ML Breast Cancer Wisconsin (Diagnostic) dataset is + downloaded from: + https://archive.ics.uci.edu/dataset/17/breast+cancer+wisconsin+diagnostic + Read more in the :ref:`User Guide `. Parameters @@ -548,33 +795,34 @@ def load_breast_cancer(*, return_X_y=False, as_frame=False): data : {ndarray, dataframe} of shape (569, 30) The data matrix. If `as_frame=True`, `data` will be a pandas DataFrame. - target: {ndarray, Series} of shape (569,) + target : {ndarray, Series} of shape (569,) The classification target. If `as_frame=True`, `target` will be a pandas Series. - feature_names: list + feature_names : ndarray of shape (30,) The names of the dataset columns. - target_names: list + target_names : ndarray of shape (2,) The names of target classes. - frame: DataFrame of shape (569, 31) + frame : DataFrame of shape (569, 31) Only present when `as_frame=True`. DataFrame with `data` and `target`. .. versionadded:: 0.23 - DESCR: str + DESCR : str The full description of the dataset. - filename: str + filename : str The path to the location of the data. .. versionadded:: 0.20 (data, target) : tuple if ``return_X_y`` is True + A tuple of two ndarrays by default. The first contains a 2D ndarray of + shape (569, 30) with each row representing one sample and each column + representing the features. The second ndarray of shape (569,) contains + the target samples. If `as_frame=True`, both arrays are pandas objects, + i.e. `X` a dataframe and `y` a series. .. versionadded:: 0.18 - The copy of UCI ML Breast Cancer Wisconsin (Diagnostic) dataset is - downloaded from: - https://goo.gl/U2Uwz2 - Examples -------- Let's say you are interested in the samples 10, 50, and 85, and want to @@ -585,53 +833,80 @@ def load_breast_cancer(*, return_X_y=False, as_frame=False): >>> data.target[[10, 50, 85]] array([0, 1, 0]) >>> list(data.target_names) - ['malignant', 'benign'] + [np.str_('malignant'), np.str_('benign')] """ - module_path = dirname(__file__) - data, target, target_names = load_data(module_path, 'breast_cancer.csv') - csv_filename = join(module_path, 'data', 'breast_cancer.csv') - - with open(join(module_path, 'descr', 'breast_cancer.rst')) as rst_file: - fdescr = rst_file.read() - - feature_names = np.array(['mean radius', 'mean texture', - 'mean perimeter', 'mean area', - 'mean smoothness', 'mean compactness', - 'mean concavity', 'mean concave points', - 'mean symmetry', 'mean fractal dimension', - 'radius error', 'texture error', - 'perimeter error', 'area error', - 'smoothness error', 'compactness error', - 'concavity error', 'concave points error', - 'symmetry error', 'fractal dimension error', - 'worst radius', 'worst texture', - 'worst perimeter', 'worst area', - 'worst smoothness', 'worst compactness', - 'worst concavity', 'worst concave points', - 'worst symmetry', 'worst fractal dimension']) + data_file_name = "breast_cancer.csv" + data, target, target_names, fdescr = load_csv_data( + data_file_name=data_file_name, descr_file_name="breast_cancer.rst" + ) + + feature_names = np.array( + [ + "mean radius", + "mean texture", + "mean perimeter", + "mean area", + "mean smoothness", + "mean compactness", + "mean concavity", + "mean concave points", + "mean symmetry", + "mean fractal dimension", + "radius error", + "texture error", + "perimeter error", + "area error", + "smoothness error", + "compactness error", + "concavity error", + "concave points error", + "symmetry error", + "fractal dimension error", + "worst radius", + "worst texture", + "worst perimeter", + "worst area", + "worst smoothness", + "worst compactness", + "worst concavity", + "worst concave points", + "worst symmetry", + "worst fractal dimension", + ] + ) frame = None - target_columns = ['target', ] + target_columns = [ + "target", + ] if as_frame: - frame, data, target = _convert_data_dataframe("load_breast_cancer", - data, - target, - feature_names, - target_columns) + frame, data, target = _convert_data_dataframe( + "load_breast_cancer", data, target, feature_names, target_columns + ) if return_X_y: return data, target - return Bunch(data=data, - target=target, - frame=frame, - target_names=target_names, - DESCR=fdescr, - feature_names=feature_names, - filename=csv_filename) - - -@_deprecate_positional_args + return Bunch( + data=data, + target=target, + frame=frame, + target_names=target_names, + DESCR=fdescr, + feature_names=feature_names, + filename=data_file_name, + data_module=DATA_MODULE, + ) + + +@validate_params( + { + "n_class": [Interval(Integral, 1, 10, closed="both")], + "return_X_y": ["boolean"], + "as_frame": ["boolean"], + }, + prefer_skip_nested_validation=True, +) def load_digits(*, n_class=10, return_X_y=False, as_frame=False): """Load and return the digits dataset (classification). @@ -645,6 +920,9 @@ def load_digits(*, n_class=10, return_X_y=False, as_frame=False): Features integers 0-16 ================= ============== + This is a copy of the test set of the UCI ML hand-written digits datasets + https://archive.ics.uci.edu/ml/datasets/Optical+Recognition+of+Handwritten+Digits + Read more in the :ref:`User Guide `. Parameters @@ -696,12 +974,14 @@ def load_digits(*, n_class=10, return_X_y=False, as_frame=False): The full description of the dataset. (data, target) : tuple if ``return_X_y`` is True + A tuple of two ndarrays by default. The first contains a 2D ndarray of + shape (1797, 64) with each row representing one sample and each column + representing the features. The second ndarray of shape (1797) contains + the target samples. If `as_frame=True`, both arrays are pandas objects, + i.e. `X` a dataframe and `y` a series. .. versionadded:: 0.18 - This is a copy of the test set of the UCI ML hand-written digits datasets - https://archive.ics.uci.edu/ml/datasets/Optical+Recognition+of+Handwritten+Digits - Examples -------- To load the data and visualize the images:: @@ -710,16 +990,16 @@ def load_digits(*, n_class=10, return_X_y=False, as_frame=False): >>> digits = load_digits() >>> print(digits.data.shape) (1797, 64) - >>> import matplotlib.pyplot as plt #doctest: +SKIP - >>> plt.gray() #doctest: +SKIP - >>> plt.matshow(digits.images[0]) #doctest: +SKIP - >>> plt.show() #doctest: +SKIP + >>> import matplotlib.pyplot as plt + >>> plt.matshow(digits.images[0], cmap="gray") + <...> + >>> plt.show() """ - module_path = dirname(__file__) - data = np.loadtxt(join(module_path, 'data', 'digits.csv.gz'), - delimiter=',') - with open(join(module_path, 'descr', 'digits.rst')) as f: - descr = f.read() + + data, fdescr = load_gzip_compressed_csv_data( + data_file_name="digits.csv.gz", descr_file_name="digits.rst", delimiter="," + ) + target = data[:, -1].astype(int, copy=False) flat_data = data[:, :-1] images = flat_data.view() @@ -730,33 +1010,40 @@ def load_digits(*, n_class=10, return_X_y=False, as_frame=False): flat_data, target = flat_data[idx], target[idx] images = images[idx] - feature_names = ['pixel_{}_{}'.format(row_idx, col_idx) - for row_idx in range(8) - for col_idx in range(8)] + feature_names = [ + "pixel_{}_{}".format(row_idx, col_idx) + for row_idx in range(8) + for col_idx in range(8) + ] frame = None - target_columns = ['target', ] + target_columns = [ + "target", + ] if as_frame: - frame, flat_data, target = _convert_data_dataframe("load_digits", - flat_data, - target, - feature_names, - target_columns) + frame, flat_data, target = _convert_data_dataframe( + "load_digits", flat_data, target, feature_names, target_columns + ) if return_X_y: return flat_data, target - return Bunch(data=flat_data, - target=target, - frame=frame, - feature_names=feature_names, - target_names=np.arange(10), - images=images, - DESCR=descr) - - -@_deprecate_positional_args -def load_diabetes(*, return_X_y=False, as_frame=False): + return Bunch( + data=flat_data, + target=target, + frame=frame, + feature_names=feature_names, + target_names=np.arange(10), + images=images, + DESCR=fdescr, + ) + + +@validate_params( + {"return_X_y": ["boolean"], "as_frame": ["boolean"], "scaled": ["boolean"]}, + prefer_skip_nested_validation=True, +) +def load_diabetes(*, return_X_y=False, as_frame=False, scaled=True): """Load and return the diabetes dataset (regression). ============== ================== @@ -766,11 +1053,17 @@ def load_diabetes(*, return_X_y=False, as_frame=False): Targets integer 25 - 346 ============== ================== + .. note:: + The meaning of each feature (i.e. `feature_names`) might be unclear + (especially for `ltg`) as the documentation of the original dataset is + not explicit. We provide information that seems correct in regard with + the scientific literature in this field of research. + Read more in the :ref:`User Guide `. Parameters ---------- - return_X_y : bool, default=False. + return_X_y : bool, default=False If True, returns ``(data, target)`` instead of a Bunch object. See below for more information about the `data` and `target` object. @@ -785,6 +1078,13 @@ def load_diabetes(*, return_X_y=False, as_frame=False): .. versionadded:: 0.23 + scaled : bool, default=True + If True, the feature variables are mean centered and scaled by the + standard deviation times the square root of `n_samples`. + If False, raw data is returned for the feature variables. + + .. versionadded:: 1.1 + Returns ------- data : :class:`~sklearn.utils.Bunch` @@ -811,48 +1111,69 @@ def load_diabetes(*, return_X_y=False, as_frame=False): The path to the location of the target. (data, target) : tuple if ``return_X_y`` is True + Returns a tuple of two ndarray of shape (n_samples, n_features) + A 2D array with each row representing one sample and each column + representing the features and/or target of a given sample. .. versionadded:: 0.18 + + Examples + -------- + >>> from sklearn.datasets import load_diabetes + >>> diabetes = load_diabetes() + >>> diabetes.target[:3] + array([151., 75., 141.]) + >>> diabetes.data.shape + (442, 10) """ - module_path = dirname(__file__) - base_dir = join(module_path, 'data') - data_filename = join(base_dir, 'diabetes_data.csv.gz') - data = np.loadtxt(data_filename) - target_filename = join(base_dir, 'diabetes_target.csv.gz') - target = np.loadtxt(target_filename) + data_filename = "diabetes_data_raw.csv.gz" + target_filename = "diabetes_target.csv.gz" + data = load_gzip_compressed_csv_data(data_filename) + target = load_gzip_compressed_csv_data(target_filename) + + if scaled: + data = scale(data, copy=False) + data /= data.shape[0] ** 0.5 - with open(join(module_path, 'descr', 'diabetes.rst')) as rst_file: - fdescr = rst_file.read() + fdescr = load_descr("diabetes.rst") - feature_names = ['age', 'sex', 'bmi', 'bp', - 's1', 's2', 's3', 's4', 's5', 's6'] + feature_names = ["age", "sex", "bmi", "bp", "s1", "s2", "s3", "s4", "s5", "s6"] frame = None - target_columns = ['target', ] + target_columns = [ + "target", + ] if as_frame: - frame, data, target = _convert_data_dataframe("load_diabetes", - data, - target, - feature_names, - target_columns) + frame, data, target = _convert_data_dataframe( + "load_diabetes", data, target, feature_names, target_columns + ) if return_X_y: return data, target - return Bunch(data=data, - target=target, - frame=frame, - DESCR=fdescr, - feature_names=feature_names, - data_filename=data_filename, - target_filename=target_filename) - - -@_deprecate_positional_args + return Bunch( + data=data, + target=target, + frame=frame, + DESCR=fdescr, + feature_names=feature_names, + data_filename=data_filename, + target_filename=target_filename, + data_module=DATA_MODULE, + ) + + +@validate_params( + { + "return_X_y": ["boolean"], + "as_frame": ["boolean"], + }, + prefer_skip_nested_validation=True, +) def load_linnerud(*, return_X_y=False, as_frame=False): - """Load and return the physical excercise linnerud dataset. + """Load and return the physical exercise Linnerud dataset. - This dataset is suitable for multi-ouput regression tasks. + This dataset is suitable for multi-output regression tasks. ============== ============================ Samples total 20 @@ -910,134 +1231,63 @@ def load_linnerud(*, return_X_y=False, as_frame=False): .. versionadded:: 0.20 (data, target) : tuple if ``return_X_y`` is True + Returns a tuple of two ndarrays or dataframe of shape + `(20, 3)`. Each row represents one sample and each column represents the + features in `X` and a target in `y` of a given sample. .. versionadded:: 0.18 - """ - base_dir = join(dirname(__file__), 'data/') - data_filename = join(base_dir, 'linnerud_exercise.csv') - target_filename = join(base_dir, 'linnerud_physiological.csv') - # Read data - data_exercise = np.loadtxt(data_filename, skiprows=1) - data_physiological = np.loadtxt(target_filename, skiprows=1) + Examples + -------- + >>> from sklearn.datasets import load_linnerud + >>> linnerud = load_linnerud() + >>> linnerud.data.shape + (20, 3) + >>> linnerud.target.shape + (20, 3) + """ + data_filename = "linnerud_exercise.csv" + target_filename = "linnerud_physiological.csv" - # Read header - with open(data_filename) as f: + data_module_path = resources.files(DATA_MODULE) + # Read header and data + data_path = data_module_path / data_filename + with data_path.open("r", encoding="utf-8") as f: header_exercise = f.readline().split() - with open(target_filename) as f: + f.seek(0) # reset file obj + data_exercise = np.loadtxt(f, skiprows=1) + + target_path = data_module_path / target_filename + with target_path.open("r", encoding="utf-8") as f: header_physiological = f.readline().split() + f.seek(0) # reset file obj + data_physiological = np.loadtxt(f, skiprows=1) - with open(dirname(__file__) + '/descr/linnerud.rst') as f: - descr = f.read() + fdescr = load_descr("linnerud.rst") frame = None if as_frame: - (frame, - data_exercise, - data_physiological) = _convert_data_dataframe("load_linnerud", - data_exercise, - data_physiological, - header_exercise, - header_physiological) + (frame, data_exercise, data_physiological) = _convert_data_dataframe( + "load_linnerud", + data_exercise, + data_physiological, + header_exercise, + header_physiological, + ) if return_X_y: return data_exercise, data_physiological - return Bunch(data=data_exercise, - feature_names=header_exercise, - target=data_physiological, - target_names=header_physiological, - frame=frame, - DESCR=descr, - data_filename=data_filename, - target_filename=target_filename) - - -@_deprecate_positional_args -def load_boston(*, return_X_y=False): - """Load and return the boston house-prices dataset (regression). - - ============== ============== - Samples total 506 - Dimensionality 13 - Features real, positive - Targets real 5. - 50. - ============== ============== - - Read more in the :ref:`User Guide `. - - Parameters - ---------- - return_X_y : bool, default=False - If True, returns ``(data, target)`` instead of a Bunch object. - See below for more information about the `data` and `target` object. - - .. versionadded:: 0.18 - - Returns - ------- - data : :class:`~sklearn.utils.Bunch` - Dictionary-like object, with the following attributes. - - data : ndarray of shape (506, 13) - The data matrix. - target : ndarray of shape (506, ) - The regression target. - filename : str - The physical location of boston csv dataset. - - .. versionadded:: 0.20 - - DESCR : str - The full description of the dataset. - feature_names : ndarray - The names of features - - (data, target) : tuple if ``return_X_y`` is True - - .. versionadded:: 0.18 - - Notes - ----- - .. versionchanged:: 0.20 - Fixed a wrong data point at [445, 0]. - - Examples - -------- - >>> from sklearn.datasets import load_boston - >>> X, y = load_boston(return_X_y=True) - >>> print(X.shape) - (506, 13) - """ - module_path = dirname(__file__) - - fdescr_name = join(module_path, 'descr', 'boston_house_prices.rst') - with open(fdescr_name) as f: - descr_text = f.read() - - data_file_name = join(module_path, 'data', 'boston_house_prices.csv') - with open(data_file_name) as f: - data_file = csv.reader(f) - temp = next(data_file) - n_samples = int(temp[0]) - n_features = int(temp[1]) - data = np.empty((n_samples, n_features)) - target = np.empty((n_samples,)) - temp = next(data_file) # names of features - feature_names = np.array(temp) - - for i, d in enumerate(data_file): - data[i] = np.asarray(d[:-1], dtype=np.float64) - target[i] = np.asarray(d[-1], dtype=np.float64) - - if return_X_y: - return data, target - - return Bunch(data=data, - target=target, - # last column is target value - feature_names=feature_names[:-1], - DESCR=descr_text, - filename=data_file_name) + return Bunch( + data=data_exercise, + feature_names=header_exercise, + target=data_physiological, + target_names=header_physiological, + frame=frame, + DESCR=fdescr, + data_filename=data_filename, + target_filename=target_filename, + data_module=DATA_MODULE, + ) def load_sample_images(): @@ -1073,37 +1323,56 @@ def load_sample_images(): >>> first_img_data.dtype #doctest: +SKIP dtype('uint8') """ - # import PIL only when needed - from ..externals._pilutil import imread + try: + from PIL import Image + except ImportError: + raise ImportError( + "The Python Imaging Library (PIL) is required to load data " + "from jpeg files. Please refer to " + "https://pillow.readthedocs.io/en/stable/installation.html " + "for installing PIL." + ) + + descr = load_descr("README.txt", descr_module=IMAGES_MODULE) + + filenames, images = [], [] - module_path = join(dirname(__file__), "images") - with open(join(module_path, 'README.txt')) as f: - descr = f.read() - filenames = [join(module_path, filename) - for filename in sorted(os.listdir(module_path)) - if filename.endswith(".jpg")] - # Load image data for each image in the source folder. - images = [imread(filename) for filename in filenames] + jpg_paths = sorted( + resource + for resource in resources.files(IMAGES_MODULE).iterdir() + if resource.is_file() and resource.match("*.jpg") + ) - return Bunch(images=images, - filenames=filenames, - DESCR=descr) + for path in jpg_paths: + filenames.append(str(path)) + with path.open("rb") as image_file: + pil_image = Image.open(image_file) + image = np.asarray(pil_image) + images.append(image) + return Bunch(images=images, filenames=filenames, DESCR=descr) + +@validate_params( + { + "image_name": [StrOptions({"china.jpg", "flower.jpg"})], + }, + prefer_skip_nested_validation=True, +) def load_sample_image(image_name): - """Load the numpy array of a single sample image + """Load the numpy array of a single sample image. Read more in the :ref:`User Guide `. Parameters ---------- image_name : {`china.jpg`, `flower.jpg`} - The name of the sample image loaded + The name of the sample image loaded. Returns ------- img : 3D array - The image as a numpy array: height x width x color + The image as a numpy array: height x width x color. Examples -------- @@ -1161,35 +1430,207 @@ def _sha256(path): return sha256hash.hexdigest() -def _fetch_remote(remote, dirname=None): - """Helper function to download a remote dataset into path +def _fetch_remote(remote, dirname=None, n_retries=3, delay=1): + """Helper function to download a remote dataset. Fetch a dataset pointed by remote's url, save into path using remote's - filename and ensure its integrity based on the SHA256 Checksum of the + filename and ensure its integrity based on the SHA256 checksum of the downloaded file. + .. versionchanged:: 1.6 + + If the file already exists locally and the SHA256 checksums match, the + path to the local file is returned without re-downloading. + Parameters ---------- remote : RemoteFileMetadata Named tuple containing remote dataset meta information: url, filename - and checksum + and checksum. - dirname : string - Directory to save the file to. + dirname : str or Path, default=None + Directory to save the file to. If None, the current working directory + is used. + + n_retries : int, default=3 + Number of retries when HTTP errors are encountered. + + .. versionadded:: 1.5 + + delay : int, default=1 + Number of seconds between retries. + + .. versionadded:: 1.5 Returns ------- - file_path: string + file_path: Path Full path of the created file. """ + if dirname is None: + folder_path = Path(".") + else: + folder_path = Path(dirname) + + file_path = folder_path / remote.filename + + if file_path.exists(): + if remote.checksum is None: + return file_path + + checksum = _sha256(file_path) + if checksum == remote.checksum: + return file_path + else: + warnings.warn( + f"SHA256 checksum of existing local file {file_path.name} " + f"({checksum}) differs from expected ({remote.checksum}): " + f"re-downloading from {remote.url} ." + ) + + # We create a temporary file dedicated to this particular download to avoid + # conflicts with parallel downloads. If the download is successful, the + # temporary file is atomically renamed to the final file path (with + # `shutil.move`). We therefore pass `delete=False` to `NamedTemporaryFile`. + # Otherwise, garbage collecting temp_file would raise an error when + # attempting to delete a file that was already renamed. If the download + # fails or the result does not match the expected SHA256 digest, the + # temporary file is removed manually in the except block. + temp_file = NamedTemporaryFile( + prefix=remote.filename + ".part_", dir=folder_path, delete=False + ) + # Note that Python 3.12's `delete_on_close=True` is ignored as we set + # `delete=False` explicitly. So after this line the empty temporary file still + # exists on disk to make sure that it's uniquely reserved for this specific call of + # `_fetch_remote` and therefore it protects against any corruption by parallel + # calls. + temp_file.close() + try: + temp_file_path = Path(temp_file.name) + while True: + try: + urlretrieve(remote.url, temp_file_path) + break + except (URLError, TimeoutError): + if n_retries == 0: + # If no more retries are left, re-raise the caught exception. + raise + warnings.warn(f"Retry downloading from url: {remote.url}") + n_retries -= 1 + time.sleep(delay) + + checksum = _sha256(temp_file_path) + if remote.checksum is not None and remote.checksum != checksum: + raise OSError( + f"The SHA256 checksum of {remote.filename} ({checksum}) " + f"differs from expected ({remote.checksum})." + ) + except (Exception, KeyboardInterrupt): + os.unlink(temp_file.name) + raise + + # The following renaming is atomic whenever temp_file_path and + # file_path are on the same filesystem. This should be the case most of + # the time, but we still use shutil.move instead of os.rename in case + # they are not. + shutil.move(temp_file_path, file_path) - file_path = (remote.filename if dirname is None - else join(dirname, remote.filename)) - urlretrieve(remote.url, file_path) - checksum = _sha256(file_path) - if remote.checksum != checksum: - raise IOError("{} has an SHA256 checksum ({}) " - "differing from expected ({}), " - "file may be corrupted.".format(file_path, checksum, - remote.checksum)) return file_path + + +def _filter_filename(value, filter_dots=True): + """Derive a name that is safe to use as filename from the given string. + + Adapted from the `slugify` function of django: + https://github.com/django/django/blob/master/django/utils/text.py + + Convert spaces or repeated dashes to single dashes. Replace characters that + aren't alphanumerics, underscores, hyphens or dots by underscores. Convert + to lowercase. Also strip leading and trailing whitespace, dashes, and + underscores. + """ + value = unicodedata.normalize("NFKD", value).lower() + if filter_dots: + value = re.sub(r"[^\w\s-]+", "_", value) + else: + value = re.sub(r"[^.\w\s-]+", "_", value) + value = re.sub(r"[\s-]+", "-", value) + return value.strip("-_.") + + +def _derive_folder_and_filename_from_url(url): + parsed_url = urlparse(url) + if not parsed_url.hostname: + raise ValueError(f"Invalid URL: {url}") + folder_components = [_filter_filename(parsed_url.hostname, filter_dots=False)] + path = parsed_url.path + + if "/" in path: + base_folder, raw_filename = path.rsplit("/", 1) + + base_folder = _filter_filename(base_folder) + if base_folder: + folder_components.append(base_folder) + else: + raw_filename = path + + filename = _filter_filename(raw_filename, filter_dots=False) + if not filename: + filename = "downloaded_file" + + return "/".join(folder_components), filename + + +def fetch_file( + url, folder=None, local_filename=None, sha256=None, n_retries=3, delay=1 +): + """Fetch a file from the web if not already present in the local folder. + + If the file already exists locally (and the SHA256 checksums match when + provided), the path to the local file is returned without re-downloading. + + .. versionadded:: 1.6 + + Parameters + ---------- + url : str + URL of the file to download. + + folder : str or Path, default=None + Directory to save the file to. If None, the file is downloaded in a + folder with a name derived from the URL host name and path under + scikit-learn data home folder. + + local_filename : str, default=None + Name of the file to save. If None, the filename is inferred from the + URL. + + sha256 : str, default=None + SHA256 checksum of the file. If None, no checksum is verified. + + n_retries : int, default=3 + Number of retries when HTTP errors are encountered. + + delay : int, default=1 + Number of seconds between retries. + + Returns + ------- + file_path : Path + Full path of the downloaded file. + """ + folder_from_url, filename_from_url = _derive_folder_and_filename_from_url(url) + + if local_filename is None: + local_filename = filename_from_url + + if folder is None: + folder = Path(get_data_home()) / folder_from_url + makedirs(folder, exist_ok=True) + + remote_metadata = RemoteFileMetadata( + filename=local_filename, url=url, checksum=sha256 + ) + return _fetch_remote( + remote_metadata, dirname=folder, n_retries=n_retries, delay=delay + ) diff --git a/sklearn/datasets/_california_housing.py b/sklearn/datasets/_california_housing.py index a25f8d63eceef..749f8528da338 100644 --- a/sklearn/datasets/_california_housing.py +++ b/sklearn/datasets/_california_housing.py @@ -15,44 +15,64 @@ ---------- Pace, R. Kelley and Ronald Barry, Sparse Spatial Autoregressions, -Statistics and Probability Letters, 33 (1997) 291-297. +Statistics and Probability Letters, 33:291-297, 1997. """ -# Authors: Peter Prettenhofer -# License: BSD 3 clause -from os.path import dirname, exists, join -from os import makedirs, remove -import tarfile +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numpy as np import logging +import tarfile +from numbers import Integral, Real +from os import PathLike, makedirs, remove +from os.path import exists import joblib +import numpy as np -from . import get_data_home -from ._base import _convert_data_dataframe -from ._base import _fetch_remote -from ._base import _pkl_filepath -from ._base import RemoteFileMetadata from ..utils import Bunch -from ..utils.validation import _deprecate_positional_args - +from ..utils._param_validation import Interval, validate_params +from . import get_data_home +from ._base import ( + RemoteFileMetadata, + _convert_data_dataframe, + _fetch_remote, + _pkl_filepath, + load_descr, +) # The original data can be found at: # https://www.dcc.fc.up.pt/~ltorgo/Regression/cal_housing.tgz ARCHIVE = RemoteFileMetadata( - filename='cal_housing.tgz', - url='https://ndownloader.figshare.com/files/5976036', - checksum=('aaa5c9a6afe2225cc2aed2723682ae40' - '3280c4a3695a2ddda4ffb5d8215ea681')) + filename="cal_housing.tgz", + url="https://ndownloader.figshare.com/files/5976036", + checksum="aaa5c9a6afe2225cc2aed2723682ae403280c4a3695a2ddda4ffb5d8215ea681", +) logger = logging.getLogger(__name__) -@_deprecate_positional_args -def fetch_california_housing(*, data_home=None, download_if_missing=True, - return_X_y=False, as_frame=False): +@validate_params( + { + "data_home": [str, PathLike, None], + "download_if_missing": ["boolean"], + "return_X_y": ["boolean"], + "as_frame": ["boolean"], + "n_retries": [Interval(Integral, 1, None, closed="left")], + "delay": [Interval(Real, 0.0, None, closed="neither")], + }, + prefer_skip_nested_validation=True, +) +def fetch_california_housing( + *, + data_home=None, + download_if_missing=True, + return_X_y=False, + as_frame=False, + n_retries=3, + delay=1.0, +): """Load the California housing dataset (regression). ============== ============== @@ -66,16 +86,15 @@ def fetch_california_housing(*, data_home=None, download_if_missing=True, Parameters ---------- - data_home : str, default=None + data_home : str or path-like, default=None Specify another download and cache folder for the datasets. By default all scikit-learn data is stored in '~/scikit_learn_data' subfolders. download_if_missing : bool, default=True - If False, raise a IOError if the data is not locally available + If False, raise an OSError if the data is not locally available instead of trying to download the data from the source site. - - return_X_y : bool, default=False. + return_X_y : bool, default=False If True, returns ``(data.data, data.target)`` instead of a Bunch object. @@ -88,6 +107,16 @@ def fetch_california_housing(*, data_home=None, download_if_missing=True, .. versionadded:: 0.23 + n_retries : int, default=3 + Number of retries when HTTP errors are encountered. + + .. versionadded:: 1.5 + + delay : float, default=1.0 + Number of seconds between retries. + + .. versionadded:: 1.5 + Returns ------- dataset : :class:`~sklearn.utils.Bunch` @@ -102,7 +131,7 @@ def fetch_california_housing(*, data_home=None, download_if_missing=True, If ``as_frame`` is True, ``target`` is a pandas object. feature_names : list of length 8 Array of ordered feature names used in the dataset. - DESCR : string + DESCR : str Description of the California housing dataset. frame : pandas DataFrame Only present when `as_frame=True`. DataFrame with ``data`` and @@ -111,6 +140,10 @@ def fetch_california_housing(*, data_home=None, download_if_missing=True, .. versionadded:: 0.23 (data, target) : tuple if ``return_X_y`` is True + A tuple of two ndarray. The first containing a 2D array of + shape (n_samples, n_features) with each row representing one + sample and each column representing the features. The second + ndarray of shape (n_samples,) containing the target samples. .. versionadded:: 0.20 @@ -118,25 +151,40 @@ def fetch_california_housing(*, data_home=None, download_if_missing=True, ----- This dataset consists of 20,640 samples and 9 features. + + Examples + -------- + >>> from sklearn.datasets import fetch_california_housing + >>> housing = fetch_california_housing() + >>> print(housing.data.shape, housing.target.shape) + (20640, 8) (20640,) + >>> print(housing.feature_names[0:6]) + ['MedInc', 'HouseAge', 'AveRooms', 'AveBedrms', 'Population', 'AveOccup'] """ data_home = get_data_home(data_home=data_home) if not exists(data_home): makedirs(data_home) - filepath = _pkl_filepath(data_home, 'cal_housing.pkz') + filepath = _pkl_filepath(data_home, "cal_housing.pkz") if not exists(filepath): if not download_if_missing: - raise IOError("Data not found and `download_if_missing` is False") + raise OSError("Data not found and `download_if_missing` is False") - logger.info('Downloading Cal. housing from {} to {}'.format( - ARCHIVE.url, data_home)) + logger.info( + "Downloading Cal. housing from {} to {}".format(ARCHIVE.url, data_home) + ) - archive_path = _fetch_remote(ARCHIVE, dirname=data_home) + archive_path = _fetch_remote( + ARCHIVE, + dirname=data_home, + n_retries=n_retries, + delay=delay, + ) with tarfile.open(mode="r:gz", name=archive_path) as f: cal_housing = np.loadtxt( - f.extractfile('CaliforniaHousing/cal_housing.data'), - delimiter=',') + f.extractfile("CaliforniaHousing/cal_housing.data"), delimiter="," + ) # Columns are not in the same order compared to the previous # URL resource on lib.stat.cmu.edu columns_index = [8, 7, 2, 3, 4, 5, 6, 1, 0] @@ -148,8 +196,16 @@ def fetch_california_housing(*, data_home=None, download_if_missing=True, else: cal_housing = joblib.load(filepath) - feature_names = ["MedInc", "HouseAge", "AveRooms", "AveBedrms", - "Population", "AveOccup", "Latitude", "Longitude"] + feature_names = [ + "MedInc", + "HouseAge", + "AveRooms", + "AveBedrms", + "Population", + "AveOccup", + "Latitude", + "Longitude", + ] target, data = cal_housing[:, 0], cal_housing[:, 1:] @@ -165,28 +221,28 @@ def fetch_california_housing(*, data_home=None, download_if_missing=True, # target in units of 100,000 target = target / 100000.0 - module_path = dirname(__file__) - with open(join(module_path, 'descr', 'california_housing.rst')) as dfile: - descr = dfile.read() + descr = load_descr("california_housing.rst") X = data y = target frame = None - target_names = ["MedHouseVal", ] + target_names = [ + "MedHouseVal", + ] if as_frame: - frame, X, y = _convert_data_dataframe("fetch_california_housing", - data, - target, - feature_names, - target_names) + frame, X, y = _convert_data_dataframe( + "fetch_california_housing", data, target, feature_names, target_names + ) if return_X_y: return X, y - return Bunch(data=X, - target=y, - frame=frame, - target_names=target_names, - feature_names=feature_names, - DESCR=descr) + return Bunch( + data=X, + target=y, + frame=frame, + target_names=target_names, + feature_names=feature_names, + DESCR=descr, + ) diff --git a/sklearn/datasets/_covtype.py b/sklearn/datasets/_covtype.py index 305f465369604..6a0138bafa9c5 100644 --- a/sklearn/datasets/_covtype.py +++ b/sklearn/datasets/_covtype.py @@ -10,59 +10,83 @@ Courtesy of Jock A. Blackard and Colorado State University. """ -# Author: Lars Buitinck -# Peter Prettenhofer -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -from gzip import GzipFile import logging -from os.path import dirname, exists, join -from os import remove, makedirs +import os +from gzip import GzipFile +from numbers import Integral, Real +from os.path import exists, join +from tempfile import TemporaryDirectory -import numpy as np import joblib +import numpy as np +from ..utils import Bunch, check_random_state +from ..utils._param_validation import Interval, validate_params from . import get_data_home -from ._base import _convert_data_dataframe -from ._base import _fetch_remote -from ._base import RemoteFileMetadata -from ..utils import Bunch -from ._base import _pkl_filepath -from ..utils import check_random_state -from ..utils.validation import _deprecate_positional_args - +from ._base import ( + RemoteFileMetadata, + _convert_data_dataframe, + _fetch_remote, + _pkl_filepath, + load_descr, +) # The original data can be found in: # https://archive.ics.uci.edu/ml/machine-learning-databases/covtype/covtype.data.gz ARCHIVE = RemoteFileMetadata( - filename='covtype.data.gz', - url='https://ndownloader.figshare.com/files/5976039', - checksum=('614360d0257557dd1792834a85a1cdeb' - 'fadc3c4f30b011d56afee7ffb5b15771')) + filename="covtype.data.gz", + url="https://ndownloader.figshare.com/files/5976039", + checksum="614360d0257557dd1792834a85a1cdebfadc3c4f30b011d56afee7ffb5b15771", +) logger = logging.getLogger(__name__) # Column names reference: # https://archive.ics.uci.edu/ml/machine-learning-databases/covtype/covtype.info -FEATURE_NAMES = ["Elevation", - "Aspect", - "Slope", - "Horizontal_Distance_To_Hydrology", - "Vertical_Distance_To_Hydrology", - "Horizontal_Distance_To_Roadways", - "Hillshade_9am", - "Hillshade_Noon", - "Hillshade_3pm", - "Horizontal_Distance_To_Fire_Points"] +FEATURE_NAMES = [ + "Elevation", + "Aspect", + "Slope", + "Horizontal_Distance_To_Hydrology", + "Vertical_Distance_To_Hydrology", + "Horizontal_Distance_To_Roadways", + "Hillshade_9am", + "Hillshade_Noon", + "Hillshade_3pm", + "Horizontal_Distance_To_Fire_Points", +] FEATURE_NAMES += [f"Wilderness_Area_{i}" for i in range(4)] FEATURE_NAMES += [f"Soil_Type_{i}" for i in range(40)] TARGET_NAMES = ["Cover_Type"] -@_deprecate_positional_args -def fetch_covtype(*, data_home=None, download_if_missing=True, - random_state=None, shuffle=False, return_X_y=False, - as_frame=False): +@validate_params( + { + "data_home": [str, os.PathLike, None], + "download_if_missing": ["boolean"], + "random_state": ["random_state"], + "shuffle": ["boolean"], + "return_X_y": ["boolean"], + "as_frame": ["boolean"], + "n_retries": [Interval(Integral, 1, None, closed="left")], + "delay": [Interval(Real, 0.0, None, closed="neither")], + }, + prefer_skip_nested_validation=True, +) +def fetch_covtype( + *, + data_home=None, + download_if_missing=True, + random_state=None, + shuffle=False, + return_X_y=False, + as_frame=False, + n_retries=3, + delay=1.0, +): """Load the covertype dataset (classification). Download it if necessary. @@ -78,12 +102,12 @@ def fetch_covtype(*, data_home=None, download_if_missing=True, Parameters ---------- - data_home : str, default=None + data_home : str or path-like, default=None Specify another download and cache folder for the datasets. By default all scikit-learn data is stored in '~/scikit_learn_data' subfolders. download_if_missing : bool, default=True - If False, raise a IOError if the data is not locally available + If False, raise an OSError if the data is not locally available instead of trying to download the data from the source site. random_state : int, RandomState instance or None, default=None @@ -109,6 +133,16 @@ def fetch_covtype(*, data_home=None, download_if_missing=True, .. versionadded:: 0.24 + n_retries : int, default=3 + Number of retries when HTTP errors are encountered. + + .. versionadded:: 1.5 + + delay : float, default=1.0 + Number of seconds between retries. + + .. versionadded:: 1.5 + Returns ------- dataset : :class:`~sklearn.utils.Bunch` @@ -130,35 +164,57 @@ def fetch_covtype(*, data_home=None, download_if_missing=True, The names of the target columns. (data, target) : tuple if ``return_X_y`` is True + A tuple of two ndarray. The first containing a 2D array of + shape (n_samples, n_features) with each row representing one + sample and each column representing the features. The second + ndarray of shape (n_samples,) containing the target samples. .. versionadded:: 0.20 + Examples + -------- + >>> from sklearn.datasets import fetch_covtype + >>> cov_type = fetch_covtype() + >>> cov_type.data.shape + (581012, 54) + >>> cov_type.target.shape + (581012,) + >>> # Let's check the 4 first feature names + >>> cov_type.feature_names[:4] + ['Elevation', 'Aspect', 'Slope', 'Horizontal_Distance_To_Hydrology'] """ - data_home = get_data_home(data_home=data_home) covtype_dir = join(data_home, "covertype") samples_path = _pkl_filepath(covtype_dir, "samples") targets_path = _pkl_filepath(covtype_dir, "targets") - available = exists(samples_path) + available = exists(samples_path) and exists(targets_path) if download_if_missing and not available: - if not exists(covtype_dir): - makedirs(covtype_dir) - logger.info("Downloading %s" % ARCHIVE.url) + os.makedirs(covtype_dir, exist_ok=True) + + # Creating temp_dir as a direct subdirectory of the target directory + # guarantees that both reside on the same filesystem, so that we can use + # os.rename to atomically move the data files to their target location. + with TemporaryDirectory(dir=covtype_dir) as temp_dir: + logger.info(f"Downloading {ARCHIVE.url}") + archive_path = _fetch_remote( + ARCHIVE, dirname=temp_dir, n_retries=n_retries, delay=delay + ) + Xy = np.genfromtxt(GzipFile(filename=archive_path), delimiter=",") - archive_path = _fetch_remote(ARCHIVE, dirname=covtype_dir) - Xy = np.genfromtxt(GzipFile(filename=archive_path), delimiter=',') - # delete archive - remove(archive_path) + X = Xy[:, :-1] + y = Xy[:, -1].astype(np.int32, copy=False) - X = Xy[:, :-1] - y = Xy[:, -1].astype(np.int32, copy=False) + samples_tmp_path = _pkl_filepath(temp_dir, "samples") + joblib.dump(X, samples_tmp_path, compress=9) + os.rename(samples_tmp_path, samples_path) - joblib.dump(X, samples_path, compress=9) - joblib.dump(y, targets_path, compress=9) + targets_tmp_path = _pkl_filepath(temp_dir, "targets") + joblib.dump(y, targets_tmp_path, compress=9) + os.rename(targets_tmp_path, targets_path) elif not available and not download_if_missing: - raise IOError("Data not found and `download_if_missing` is False") + raise OSError("Data not found and `download_if_missing` is False") try: X, y except NameError: @@ -172,23 +228,25 @@ def fetch_covtype(*, data_home=None, download_if_missing=True, X = X[ind] y = y[ind] - module_path = dirname(__file__) - with open(join(module_path, 'descr', 'covtype.rst')) as rst_file: - fdescr = rst_file.read() + fdescr = load_descr("covtype.rst") frame = None if as_frame: - frame, X, y = _convert_data_dataframe(caller_name="fetch_covtype", - data=X, - target=y, - feature_names=FEATURE_NAMES, - target_names=TARGET_NAMES) + frame, X, y = _convert_data_dataframe( + caller_name="fetch_covtype", + data=X, + target=y, + feature_names=FEATURE_NAMES, + target_names=TARGET_NAMES, + ) if return_X_y: return X, y - return Bunch(data=X, - target=y, - frame=frame, - target_names=TARGET_NAMES, - feature_names=FEATURE_NAMES, - DESCR=fdescr) + return Bunch( + data=X, + target=y, + frame=frame, + target_names=TARGET_NAMES, + feature_names=FEATURE_NAMES, + DESCR=fdescr, + ) diff --git a/sklearn/datasets/_kddcup99.py b/sklearn/datasets/_kddcup99.py index 539b7ffaf862e..f379da42eb9df 100644 --- a/sklearn/datasets/_kddcup99.py +++ b/sklearn/datasets/_kddcup99.py @@ -8,49 +8,77 @@ """ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import errno -from gzip import GzipFile import logging import os -from os.path import dirname, exists, join +from gzip import GzipFile +from numbers import Integral, Real +from os.path import exists, join -import numpy as np import joblib +import numpy as np -from ._base import _fetch_remote -from ._base import _convert_data_dataframe -from . import get_data_home -from ._base import RemoteFileMetadata -from ..utils import Bunch -from ..utils import check_random_state +from ..utils import Bunch, check_random_state from ..utils import shuffle as shuffle_method -from ..utils.validation import _deprecate_positional_args - +from ..utils._param_validation import Interval, StrOptions, validate_params +from . import get_data_home +from ._base import ( + RemoteFileMetadata, + _convert_data_dataframe, + _fetch_remote, + load_descr, +) # The original data can be found at: # https://archive.ics.uci.edu/ml/machine-learning-databases/kddcup99-mld/kddcup.data.gz ARCHIVE = RemoteFileMetadata( - filename='kddcup99_data', - url='https://ndownloader.figshare.com/files/5976045', - checksum=('3b6c942aa0356c0ca35b7b595a26c89d' - '343652c9db428893e7494f837b274292')) + filename="kddcup99_data", + url="https://ndownloader.figshare.com/files/5976045", + checksum="3b6c942aa0356c0ca35b7b595a26c89d343652c9db428893e7494f837b274292", +) # The original data can be found at: # https://archive.ics.uci.edu/ml/machine-learning-databases/kddcup99-mld/kddcup.data_10_percent.gz ARCHIVE_10_PERCENT = RemoteFileMetadata( - filename='kddcup99_10_data', - url='https://ndownloader.figshare.com/files/5976042', - checksum=('8045aca0d84e70e622d1148d7df78249' - '6f6333bf6eb979a1b0837c42a9fd9561')) + filename="kddcup99_10_data", + url="https://ndownloader.figshare.com/files/5976042", + checksum="8045aca0d84e70e622d1148d7df782496f6333bf6eb979a1b0837c42a9fd9561", +) logger = logging.getLogger(__name__) -@_deprecate_positional_args -def fetch_kddcup99(*, subset=None, data_home=None, shuffle=False, - random_state=None, - percent10=True, download_if_missing=True, return_X_y=False, - as_frame=False): +@validate_params( + { + "subset": [StrOptions({"SA", "SF", "http", "smtp"}), None], + "data_home": [str, os.PathLike, None], + "shuffle": ["boolean"], + "random_state": ["random_state"], + "percent10": ["boolean"], + "download_if_missing": ["boolean"], + "return_X_y": ["boolean"], + "as_frame": ["boolean"], + "n_retries": [Interval(Integral, 1, None, closed="left")], + "delay": [Interval(Real, 0.0, None, closed="neither")], + }, + prefer_skip_nested_validation=True, +) +def fetch_kddcup99( + *, + subset=None, + data_home=None, + shuffle=False, + random_state=None, + percent10=True, + download_if_missing=True, + return_X_y=False, + as_frame=False, + n_retries=3, + delay=1.0, +): """Load the kddcup99 dataset (classification). Download it if necessary. @@ -72,9 +100,10 @@ def fetch_kddcup99(*, subset=None, data_home=None, shuffle=False, To return the corresponding classical subsets of kddcup 99. If None, return the entire kddcup 99 dataset. - data_home : str, default=None + data_home : str or path-like, default=None Specify another download and cache folder for the datasets. By default all scikit-learn data is stored in '~/scikit_learn_data' subfolders. + .. versionadded:: 0.19 shuffle : bool, default=False @@ -90,7 +119,7 @@ def fetch_kddcup99(*, subset=None, data_home=None, shuffle=False, Whether to load only 10 percent of the data. download_if_missing : bool, default=True - If False, raise a IOError if the data is not locally available + If False, raise an OSError if the data is not locally available instead of trying to download the data from the source site. return_X_y : bool, default=False @@ -106,6 +135,16 @@ def fetch_kddcup99(*, subset=None, data_home=None, shuffle=False, .. versionadded:: 0.24 + n_retries : int, default=3 + Number of retries when HTTP errors are encountered. + + .. versionadded:: 1.5 + + delay : float, default=1.0 + Number of seconds between retries. + + .. versionadded:: 1.5 + Returns ------- data : :class:`~sklearn.utils.Bunch` @@ -127,6 +166,10 @@ def fetch_kddcup99(*, subset=None, data_home=None, shuffle=False, The names of the target columns (data, target) : tuple if ``return_X_y`` is True + A tuple of two ndarray. The first containing a 2D array of + shape (n_samples, n_features) with each row representing one + sample and each column representing the features. The second + ndarray of shape (n_samples,) containing the target samples. .. versionadded:: 0.20 """ @@ -134,7 +177,9 @@ def fetch_kddcup99(*, subset=None, data_home=None, shuffle=False, kddcup99 = _fetch_brute_kddcup99( data_home=data_home, percent10=percent10, - download_if_missing=download_if_missing + download_if_missing=download_if_missing, + n_retries=n_retries, + delay=delay, ) data = kddcup99.data @@ -142,8 +187,8 @@ def fetch_kddcup99(*, subset=None, data_home=None, shuffle=False, feature_names = kddcup99.feature_names target_names = kddcup99.target_names - if subset == 'SA': - s = target == b'normal.' + if subset == "SA": + s = target == b"normal." t = np.logical_not(s) normal_samples = data[s, :] normal_targets = target[s] @@ -160,7 +205,7 @@ def fetch_kddcup99(*, subset=None, data_home=None, shuffle=False, data = np.r_[normal_samples, abnormal_samples] target = np.r_[normal_targets, abnormal_targets] - if subset == 'SF' or subset == 'http' or subset == 'smtp': + if subset == "SF" or subset == "http" or subset == "smtp": # select all samples with positive logged_in attribute: s = data[:, 11] == 1 data = np.c_[data[s, :11], data[s, 12:]] @@ -171,33 +216,33 @@ def fetch_kddcup99(*, subset=None, data_home=None, shuffle=False, data[:, 4] = np.log((data[:, 4] + 0.1).astype(float, copy=False)) data[:, 5] = np.log((data[:, 5] + 0.1).astype(float, copy=False)) - if subset == 'http': - s = data[:, 2] == b'http' + if subset == "http": + s = data[:, 2] == b"http" data = data[s] target = target[s] data = np.c_[data[:, 0], data[:, 4], data[:, 5]] - feature_names = [feature_names[0], feature_names[4], - feature_names[5]] + feature_names = [feature_names[0], feature_names[4], feature_names[5]] - if subset == 'smtp': - s = data[:, 2] == b'smtp' + if subset == "smtp": + s = data[:, 2] == b"smtp" data = data[s] target = target[s] data = np.c_[data[:, 0], data[:, 4], data[:, 5]] - feature_names = [feature_names[0], feature_names[4], - feature_names[5]] + feature_names = [feature_names[0], feature_names[4], feature_names[5]] - if subset == 'SF': + if subset == "SF": data = np.c_[data[:, 0], data[:, 2], data[:, 4], data[:, 5]] - feature_names = [feature_names[0], feature_names[2], - feature_names[4], feature_names[5]] + feature_names = [ + feature_names[0], + feature_names[2], + feature_names[4], + feature_names[5], + ] if shuffle: data, target = shuffle_method(data, target, random_state=random_state) - module_path = dirname(__file__) - with open(join(module_path, 'descr', 'kddcup99.rst')) as rst_file: - fdescr = rst_file.read() + fdescr = load_descr("kddcup99.rst") frame = None if as_frame: @@ -218,9 +263,9 @@ def fetch_kddcup99(*, subset=None, data_home=None, shuffle=False, ) -def _fetch_brute_kddcup99(data_home=None, - download_if_missing=True, percent10=True): - +def _fetch_brute_kddcup99( + data_home=None, download_if_missing=True, percent10=True, n_retries=3, delay=1.0 +): """Load the kddcup99 dataset, downloading it if necessary. Parameters @@ -230,12 +275,18 @@ def _fetch_brute_kddcup99(data_home=None, all scikit-learn data is stored in '~/scikit_learn_data' subfolders. download_if_missing : bool, default=True - If False, raise a IOError if the data is not locally available + If False, raise an OSError if the data is not locally available instead of trying to download the data from the source site. percent10 : bool, default=True Whether to load only 10 percent of the data. + n_retries : int, default=3 + Number of retries when HTTP errors are encountered. + + delay : float, default=1.0 + Number of seconds between retries. + Returns ------- dataset : :class:`~sklearn.utils.Bunch` @@ -269,66 +320,79 @@ def _fetch_brute_kddcup99(data_home=None, targets_path = join(kddcup_dir, "targets") available = exists(samples_path) - dt = [('duration', int), - ('protocol_type', 'S4'), - ('service', 'S11'), - ('flag', 'S6'), - ('src_bytes', int), - ('dst_bytes', int), - ('land', int), - ('wrong_fragment', int), - ('urgent', int), - ('hot', int), - ('num_failed_logins', int), - ('logged_in', int), - ('num_compromised', int), - ('root_shell', int), - ('su_attempted', int), - ('num_root', int), - ('num_file_creations', int), - ('num_shells', int), - ('num_access_files', int), - ('num_outbound_cmds', int), - ('is_host_login', int), - ('is_guest_login', int), - ('count', int), - ('srv_count', int), - ('serror_rate', float), - ('srv_serror_rate', float), - ('rerror_rate', float), - ('srv_rerror_rate', float), - ('same_srv_rate', float), - ('diff_srv_rate', float), - ('srv_diff_host_rate', float), - ('dst_host_count', int), - ('dst_host_srv_count', int), - ('dst_host_same_srv_rate', float), - ('dst_host_diff_srv_rate', float), - ('dst_host_same_src_port_rate', float), - ('dst_host_srv_diff_host_rate', float), - ('dst_host_serror_rate', float), - ('dst_host_srv_serror_rate', float), - ('dst_host_rerror_rate', float), - ('dst_host_srv_rerror_rate', float), - ('labels', 'S16')] + dt = [ + ("duration", int), + ("protocol_type", "S4"), + ("service", "S11"), + ("flag", "S6"), + ("src_bytes", int), + ("dst_bytes", int), + ("land", int), + ("wrong_fragment", int), + ("urgent", int), + ("hot", int), + ("num_failed_logins", int), + ("logged_in", int), + ("num_compromised", int), + ("root_shell", int), + ("su_attempted", int), + ("num_root", int), + ("num_file_creations", int), + ("num_shells", int), + ("num_access_files", int), + ("num_outbound_cmds", int), + ("is_host_login", int), + ("is_guest_login", int), + ("count", int), + ("srv_count", int), + ("serror_rate", float), + ("srv_serror_rate", float), + ("rerror_rate", float), + ("srv_rerror_rate", float), + ("same_srv_rate", float), + ("diff_srv_rate", float), + ("srv_diff_host_rate", float), + ("dst_host_count", int), + ("dst_host_srv_count", int), + ("dst_host_same_srv_rate", float), + ("dst_host_diff_srv_rate", float), + ("dst_host_same_src_port_rate", float), + ("dst_host_srv_diff_host_rate", float), + ("dst_host_serror_rate", float), + ("dst_host_srv_serror_rate", float), + ("dst_host_rerror_rate", float), + ("dst_host_srv_rerror_rate", float), + ("labels", "S16"), + ] column_names = [c[0] for c in dt] target_names = column_names[-1] feature_names = column_names[:-1] - if download_if_missing and not available: + + if available: + try: + X = joblib.load(samples_path) + y = joblib.load(targets_path) + except Exception as e: + raise OSError( + "The cache for fetch_kddcup99 is invalid, please delete " + f"{kddcup_dir} and run the fetch_kddcup99 again" + ) from e + + elif download_if_missing: _mkdirp(kddcup_dir) logger.info("Downloading %s" % archive.url) - _fetch_remote(archive, dirname=kddcup_dir) + _fetch_remote(archive, dirname=kddcup_dir, n_retries=n_retries, delay=delay) DT = np.dtype(dt) logger.debug("extracting archive") archive_path = join(kddcup_dir, archive.filename) - file_ = GzipFile(filename=archive_path, mode='r') + file_ = GzipFile(filename=archive_path, mode="r") Xy = [] for line in file_.readlines(): line = line.decode() - Xy.append(line.replace('\n', '').split(',')) + Xy.append(line.replace("\n", "").split(",")) file_.close() - logger.debug('extraction done') + logger.debug("extraction done") os.remove(archive_path) Xy = np.asarray(Xy, dtype=object) @@ -343,15 +407,8 @@ def _fetch_brute_kddcup99(data_home=None, joblib.dump(X, samples_path, compress=0) joblib.dump(y, targets_path, compress=0) - elif not available: - if not download_if_missing: - raise IOError("Data not found and `download_if_missing` is False") - - try: - X, y - except NameError: - X = joblib.load(samples_path) - y = joblib.load(targets_path) + else: + raise OSError("Data not found and `download_if_missing` is False") return Bunch( data=X, diff --git a/sklearn/datasets/_lfw.py b/sklearn/datasets/_lfw.py index dd56e532afdc3..06420c41ed246 100644 --- a/sklearn/datasets/_lfw.py +++ b/sklearn/datasets/_lfw.py @@ -5,40 +5,44 @@ http://vis-www.cs.umass.edu/lfw/ """ -# Copyright (c) 2011 Olivier Grisel -# License: BSD 3 clause -from os import listdir, makedirs, remove -from os.path import dirname, join, exists, isdir +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import logging +from numbers import Integral, Real +from os import PathLike, listdir, makedirs, remove +from os.path import exists, isdir, join import numpy as np -import joblib from joblib import Memory -from ._base import get_data_home, _fetch_remote, RemoteFileMetadata from ..utils import Bunch -from ..utils.validation import _deprecate_positional_args -from ..utils.fixes import parse_version +from ..utils._param_validation import Hidden, Interval, StrOptions, validate_params +from ._base import ( + RemoteFileMetadata, + _fetch_remote, + get_data_home, + load_descr, +) logger = logging.getLogger(__name__) # The original data can be found in: # http://vis-www.cs.umass.edu/lfw/lfw.tgz ARCHIVE = RemoteFileMetadata( - filename='lfw.tgz', - url='https://ndownloader.figshare.com/files/5976018', - checksum=('055f7d9c632d7370e6fb4afc7468d40f' - '970c34a80d4c6f50ffec63f5a8d536c0')) + filename="lfw.tgz", + url="https://ndownloader.figshare.com/files/5976018", + checksum="055f7d9c632d7370e6fb4afc7468d40f970c34a80d4c6f50ffec63f5a8d536c0", +) # The original funneled data can be found in: # http://vis-www.cs.umass.edu/lfw/lfw-funneled.tgz FUNNELED_ARCHIVE = RemoteFileMetadata( - filename='lfw-funneled.tgz', - url='https://ndownloader.figshare.com/files/5976015', - checksum=('b47c8422c8cded889dc5a13418c4bc2a' - 'bbda121092b3533a83306f90d900100a')) + filename="lfw-funneled.tgz", + url="https://ndownloader.figshare.com/files/5976015", + checksum="b47c8422c8cded889dc5a13418c4bc2abbda121092b3533a83306f90d900100a", +) # The original target data can be found in: # http://vis-www.cs.umass.edu/lfw/pairsDevTrain.txt', @@ -46,22 +50,20 @@ # http://vis-www.cs.umass.edu/lfw/pairs.txt', TARGETS = ( RemoteFileMetadata( - filename='pairsDevTrain.txt', - url='https://ndownloader.figshare.com/files/5976012', - checksum=('1d454dada7dfeca0e7eab6f65dc4e97a' - '6312d44cf142207be28d688be92aabfa')), - + filename="pairsDevTrain.txt", + url="https://ndownloader.figshare.com/files/5976012", + checksum="1d454dada7dfeca0e7eab6f65dc4e97a6312d44cf142207be28d688be92aabfa", + ), RemoteFileMetadata( - filename='pairsDevTest.txt', - url='https://ndownloader.figshare.com/files/5976009', - checksum=('7cb06600ea8b2814ac26e946201cdb30' - '4296262aad67d046a16a7ec85d0ff87c')), - + filename="pairsDevTest.txt", + url="https://ndownloader.figshare.com/files/5976009", + checksum="7cb06600ea8b2814ac26e946201cdb304296262aad67d046a16a7ec85d0ff87c", + ), RemoteFileMetadata( - filename='pairs.txt', - url='https://ndownloader.figshare.com/files/5976006', - checksum=('ea42330c62c92989f9d7c03237ed5d59' - '1365e89b3e649747777b70e692dc1592')), + filename="pairs.txt", + url="https://ndownloader.figshare.com/files/5976006", + checksum="ea42330c62c92989f9d7c03237ed5d591365e89b3e649747777b70e692dc1592", + ), ) @@ -71,7 +73,9 @@ # -def _check_fetch_lfw(data_home=None, funneled=True, download_if_missing=True): +def _check_fetch_lfw( + data_home=None, funneled=True, download_if_missing=True, n_retries=3, delay=1.0 +): """Helper function to download any missing LFW data""" data_home = get_data_home(data_home=data_home) @@ -85,9 +89,11 @@ def _check_fetch_lfw(data_home=None, funneled=True, download_if_missing=True): if not exists(target_filepath): if download_if_missing: logger.info("Downloading LFW metadata: %s", target.url) - _fetch_remote(target, dirname=lfw_home) + _fetch_remote( + target, dirname=lfw_home, n_retries=n_retries, delay=delay + ) else: - raise IOError("%s is missing" % target_filepath) + raise OSError("%s is missing" % target_filepath) if funneled: data_folder_path = join(lfw_home, "lfw_funneled") @@ -100,15 +106,22 @@ def _check_fetch_lfw(data_home=None, funneled=True, download_if_missing=True): archive_path = join(lfw_home, archive.filename) if not exists(archive_path): if download_if_missing: - logger.info("Downloading LFW data (~200MB): %s", - archive.url) - _fetch_remote(archive, dirname=lfw_home) + logger.info("Downloading LFW data (~200MB): %s", archive.url) + _fetch_remote( + archive, dirname=lfw_home, n_retries=n_retries, delay=delay + ) else: - raise IOError("%s is missing" % archive_path) + raise OSError("%s is missing" % archive_path) import tarfile + logger.debug("Decompressing the data archive to %s", data_folder_path) - tarfile.open(archive_path, "r:gz").extractall(path=lfw_home) + with tarfile.open(archive_path, "r:gz") as fp: + # Use filter="data" to prevent the most dangerous security issues. + # For more details, see + # https://docs.python.org/3.9/library/tarfile.html#tarfile.TarFile.extractall + fp.extractall(path=lfw_home, filter="data") + remove(archive_path) return lfw_home, data_folder_path @@ -116,8 +129,15 @@ def _check_fetch_lfw(data_home=None, funneled=True, download_if_missing=True): def _load_imgs(file_paths, slice_, color, resize): """Internally used to load images""" - # import PIL only when needed - from ..externals._pilutil import imread, imresize + try: + from PIL import Image + except ImportError: + raise ImportError( + "The Python Imaging Library (PIL) is required to load data " + "from jpeg files. Please refer to " + "https://pillow.readthedocs.io/en/stable/installation.html " + "for installing PIL." + ) # compute the portion of the images to load to respect the slice_ parameter # given by the caller @@ -151,16 +171,21 @@ def _load_imgs(file_paths, slice_, color, resize): # Checks if jpeg reading worked. Refer to issue #3594 for more # details. - img = imread(file_path) - if img.ndim == 0: - raise RuntimeError("Failed to read the image file %s, " - "Please make sure that libjpeg is installed" - % file_path) + pil_img = Image.open(file_path) + pil_img = pil_img.crop( + (w_slice.start, h_slice.start, w_slice.stop, h_slice.stop) + ) + if resize is not None: + pil_img = pil_img.resize((w, h)) + face = np.asarray(pil_img, dtype=np.float32) + + if face.ndim == 0: + raise RuntimeError( + "Failed to read the image file %s, " + "Please make sure that libjpeg is installed" % file_path + ) - face = np.asarray(img[slice_], dtype=np.float32) face /= 255.0 # scale uint8 coded colors to the [0.0, 1.0] floats - if resize is not None: - face = imresize(face, resize) if not color: # average the color channels to compute a gray levels # representation @@ -175,8 +200,10 @@ def _load_imgs(file_paths, slice_, color, resize): # Task #1: Face Identification on picture with names # -def _fetch_lfw_people(data_folder_path, slice_=None, color=False, resize=None, - min_faces_per_person=0): + +def _fetch_lfw_people( + data_folder_path, slice_=None, color=False, resize=None, min_faces_per_person=0 +): """Perform the actual data loading for the lfw people dataset This operation is meant to be cached by a joblib wrapper. @@ -191,14 +218,15 @@ def _fetch_lfw_people(data_folder_path, slice_=None, color=False, resize=None, paths = [join(folder_path, f) for f in sorted(listdir(folder_path))] n_pictures = len(paths) if n_pictures >= min_faces_per_person: - person_name = person_name.replace('_', ' ') + person_name = person_name.replace("_", " ") person_names.extend([person_name] * n_pictures) file_paths.extend(paths) n_faces = len(file_paths) if n_faces == 0: - raise ValueError("min_faces_per_person=%d is too restrictive" % - min_faces_per_person) + raise ValueError( + "min_faces_per_person=%d is too restrictive" % min_faces_per_person + ) target_names = np.unique(person_names) target = np.searchsorted(target_names, person_names) @@ -216,11 +244,34 @@ def _fetch_lfw_people(data_folder_path, slice_=None, color=False, resize=None, return faces, target, target_names -@_deprecate_positional_args -def fetch_lfw_people(*, data_home=None, funneled=True, resize=0.5, - min_faces_per_person=0, color=False, - slice_=(slice(70, 195), slice(78, 172)), - download_if_missing=True, return_X_y=False): +@validate_params( + { + "data_home": [str, PathLike, None], + "funneled": ["boolean"], + "resize": [Interval(Real, 0, None, closed="neither"), None], + "min_faces_per_person": [Interval(Integral, 0, None, closed="left"), None], + "color": ["boolean"], + "slice_": [tuple, Hidden(None)], + "download_if_missing": ["boolean"], + "return_X_y": ["boolean"], + "n_retries": [Interval(Integral, 1, None, closed="left")], + "delay": [Interval(Real, 0.0, None, closed="neither")], + }, + prefer_skip_nested_validation=True, +) +def fetch_lfw_people( + *, + data_home=None, + funneled=True, + resize=0.5, + min_faces_per_person=0, + color=False, + slice_=(slice(70, 195), slice(78, 172)), + download_if_missing=True, + return_X_y=False, + n_retries=3, + delay=1.0, +): """Load the Labeled Faces in the Wild (LFW) people dataset \ (classification). @@ -233,19 +284,23 @@ def fetch_lfw_people(*, data_home=None, funneled=True, resize=0.5, Features real, between 0 and 255 ================= ======================= + For a usage example of this dataset, see + :ref:`sphx_glr_auto_examples_applications_plot_face_recognition.py`. + Read more in the :ref:`User Guide `. Parameters ---------- - data_home : str, default=None + data_home : str or path-like, default=None Specify another download and cache folder for the datasets. By default all scikit-learn data is stored in '~/scikit_learn_data' subfolders. funneled : bool, default=True Download and use the funneled variant of the dataset. - resize : float, default=0.5 - Ratio used to resize the each face picture. + resize : float or None, default=0.5 + Ratio used to resize the each face picture. If `None`, no resizing is + performed. min_faces_per_person : int, default=None The extracted dataset will only retain pictures of people that have at @@ -259,10 +314,10 @@ def fetch_lfw_people(*, data_home=None, funneled=True, resize=0.5, slice_ : tuple of slice, default=(slice(70, 195), slice(78, 172)) Provide a custom 2D slice (height, width) to extract the 'interesting' part of the jpeg files and avoid use statistical - correlation from the background + correlation from the background. download_if_missing : bool, default=True - If False, raise a IOError if the data is not locally available + If False, raise an OSError if the data is not locally available instead of trying to download the data from the source site. return_X_y : bool, default=False @@ -272,6 +327,16 @@ def fetch_lfw_people(*, data_home=None, funneled=True, resize=0.5, .. versionadded:: 0.20 + n_retries : int, default=3 + Number of retries when HTTP errors are encountered. + + .. versionadded:: 1.5 + + delay : float, default=1.0 + Number of seconds between retries. + + .. versionadded:: 1.5 + Returns ------- dataset : :class:`~sklearn.utils.Bunch` @@ -289,46 +354,70 @@ def fetch_lfw_people(*, data_home=None, funneled=True, resize=0.5, target : numpy array of shape (13233,) Labels associated to each face image. Those labels range from 0-5748 and correspond to the person IDs. - DESCR : string + target_names : numpy array of shape (5749,) + Names of all persons in the dataset. + Position in array corresponds to the person ID in the target array. + DESCR : str Description of the Labeled Faces in the Wild (LFW) dataset. (data, target) : tuple if ``return_X_y`` is True + A tuple of two ndarray. The first containing a 2D array of + shape (n_samples, n_features) with each row representing one + sample and each column representing the features. The second + ndarray of shape (n_samples,) containing the target samples. .. versionadded:: 0.20 + Examples + -------- + >>> from sklearn.datasets import fetch_lfw_people + >>> lfw_people = fetch_lfw_people() + >>> lfw_people.data.shape + (13233, 2914) + >>> lfw_people.target.shape + (13233,) + >>> for name in lfw_people.target_names[:5]: + ... print(name) + AJ Cook + AJ Lamas + Aaron Eckhart + Aaron Guiel + Aaron Patterson """ lfw_home, data_folder_path = _check_fetch_lfw( - data_home=data_home, funneled=funneled, - download_if_missing=download_if_missing) - logger.debug('Loading LFW people faces from %s', lfw_home) + data_home=data_home, + funneled=funneled, + download_if_missing=download_if_missing, + n_retries=n_retries, + delay=delay, + ) + logger.debug("Loading LFW people faces from %s", lfw_home) # wrap the loader in a memoizing function that will return memmaped data # arrays for optimal memory usage - if parse_version(joblib.__version__) < parse_version('0.12'): - # Deal with change of API in joblib - m = Memory(cachedir=lfw_home, compress=6, verbose=0) - else: - m = Memory(location=lfw_home, compress=6, verbose=0) + m = Memory(location=lfw_home, compress=6, verbose=0) load_func = m.cache(_fetch_lfw_people) # load and memoize the pairs as np arrays faces, target, target_names = load_func( - data_folder_path, resize=resize, - min_faces_per_person=min_faces_per_person, color=color, slice_=slice_) + data_folder_path, + resize=resize, + min_faces_per_person=min_faces_per_person, + color=color, + slice_=slice_, + ) X = faces.reshape(len(faces), -1) - module_path = dirname(__file__) - with open(join(module_path, 'descr', 'lfw.rst')) as rst_file: - fdescr = rst_file.read() + fdescr = load_descr("lfw.rst") if return_X_y: return X, target # pack the results as a Bunch instance - return Bunch(data=X, images=faces, - target=target, target_names=target_names, - DESCR=fdescr) + return Bunch( + data=X, images=faces, target=target, target_names=target_names, DESCR=fdescr + ) # @@ -336,16 +425,17 @@ def fetch_lfw_people(*, data_home=None, funneled=True, resize=0.5, # -def _fetch_lfw_pairs(index_file_path, data_folder_path, slice_=None, - color=False, resize=None): +def _fetch_lfw_pairs( + index_file_path, data_folder_path, slice_=None, color=False, resize=None +): """Perform the actual data loading for the LFW pairs dataset This operation is meant to be cached by a joblib wrapper. """ # parse the index file to find the number of pairs to be able to allocate # the right amount of memory before starting to decode the jpeg files - with open(index_file_path, 'rb') as index_file: - split_lines = [ln.decode().strip().split('\t') for ln in index_file] + with open(index_file_path, "rb") as index_file: + split_lines = [ln.decode().strip().split("\t") for ln in index_file] pair_specs = [sl for sl in split_lines if len(sl) > 2] n_pairs = len(pair_specs) @@ -372,7 +462,7 @@ def _fetch_lfw_pairs(index_file_path, data_folder_path, slice_=None, try: person_folder = join(data_folder_path, name) except TypeError: - person_folder = join(data_folder_path, str(name, 'UTF-8')) + person_folder = join(data_folder_path, str(name, "UTF-8")) filenames = list(sorted(listdir(person_folder))) file_path = join(person_folder, filenames[idx]) file_paths.append(file_path) @@ -384,14 +474,35 @@ def _fetch_lfw_pairs(index_file_path, data_folder_path, slice_=None, shape.insert(0, n_faces // 2) pairs.shape = shape - return pairs, target, np.array(['Different persons', 'Same person']) - - -@_deprecate_positional_args -def fetch_lfw_pairs(*, subset='train', data_home=None, funneled=True, - resize=0.5, - color=False, slice_=(slice(70, 195), slice(78, 172)), - download_if_missing=True): + return pairs, target, np.array(["Different persons", "Same person"]) + + +@validate_params( + { + "subset": [StrOptions({"train", "test", "10_folds"})], + "data_home": [str, PathLike, None], + "funneled": ["boolean"], + "resize": [Interval(Real, 0, None, closed="neither"), None], + "color": ["boolean"], + "slice_": [tuple, Hidden(None)], + "download_if_missing": ["boolean"], + "n_retries": [Interval(Integral, 1, None, closed="left")], + "delay": [Interval(Real, 0.0, None, closed="neither")], + }, + prefer_skip_nested_validation=True, +) +def fetch_lfw_pairs( + *, + subset="train", + data_home=None, + funneled=True, + resize=0.5, + color=False, + slice_=(slice(70, 195), slice(78, 172)), + download_if_missing=True, + n_retries=3, + delay=1.0, +): """Load the Labeled Faces in the Wild (LFW) pairs dataset (classification). Download it if necessary. @@ -403,11 +514,11 @@ def fetch_lfw_pairs(*, subset='train', data_home=None, funneled=True, Features real, between 0 and 255 ================= ======================= - In the official `README.txt`_ this task is described as the - "Restricted" task. As I am not sure as to implement the - "Unrestricted" variant correctly, I left it as unsupported for now. - - .. _`README.txt`: http://vis-www.cs.umass.edu/lfw/README.txt + In the `original paper `_ + the "pairs" version corresponds to the "restricted task", where + the experimenter should not use the name of a person to infer + the equivalence or non-equivalence of two face images that + are not explicitly given in the training set. The original images are 250 x 250 pixels, but the default slice and resize arguments reduce them to 62 x 47. @@ -422,7 +533,7 @@ def fetch_lfw_pairs(*, subset='train', data_home=None, funneled=True, official evaluation set that is meant to be used with a 10-folds cross validation. - data_home : str, default=None + data_home : str or path-like, default=None Specify another download and cache folder for the datasets. By default all scikit-learn data is stored in '~/scikit_learn_data' subfolders. @@ -441,12 +552,22 @@ def fetch_lfw_pairs(*, subset='train', data_home=None, funneled=True, slice_ : tuple of slice, default=(slice(70, 195), slice(78, 172)) Provide a custom 2D slice (height, width) to extract the 'interesting' part of the jpeg files and avoid use statistical - correlation from the background + correlation from the background. download_if_missing : bool, default=True - If False, raise a IOError if the data is not locally available + If False, raise an OSError if the data is not locally available instead of trying to download the data from the source site. + n_retries : int, default=3 + Number of retries when HTTP errors are encountered. + + .. versionadded:: 1.5 + + delay : float, default=1.0 + Number of seconds between retries. + + .. versionadded:: 1.5 + Returns ------- data : :class:`~sklearn.utils.Bunch` @@ -466,45 +587,64 @@ def fetch_lfw_pairs(*, subset='train', data_home=None, funneled=True, target : numpy array of shape (2200,). Shape depends on ``subset``. Labels associated to each pair of images. The two label values being different persons or the same person. - DESCR : string + target_names : numpy array of shape (2,) + Explains the target values of the target array. + 0 corresponds to "Different person", 1 corresponds to "same person". + DESCR : str Description of the Labeled Faces in the Wild (LFW) dataset. + Examples + -------- + >>> from sklearn.datasets import fetch_lfw_pairs + >>> lfw_pairs_train = fetch_lfw_pairs(subset='train') + >>> list(lfw_pairs_train.target_names) + [np.str_('Different persons'), np.str_('Same person')] + >>> lfw_pairs_train.pairs.shape + (2200, 2, 62, 47) + >>> lfw_pairs_train.data.shape + (2200, 5828) + >>> lfw_pairs_train.target.shape + (2200,) """ lfw_home, data_folder_path = _check_fetch_lfw( - data_home=data_home, funneled=funneled, - download_if_missing=download_if_missing) - logger.debug('Loading %s LFW pairs from %s', subset, lfw_home) + data_home=data_home, + funneled=funneled, + download_if_missing=download_if_missing, + n_retries=n_retries, + delay=delay, + ) + logger.debug("Loading %s LFW pairs from %s", subset, lfw_home) # wrap the loader in a memoizing function that will return memmaped data # arrays for optimal memory usage - if parse_version(joblib.__version__) < parse_version('0.12'): - # Deal with change of API in joblib - m = Memory(cachedir=lfw_home, compress=6, verbose=0) - else: - m = Memory(location=lfw_home, compress=6, verbose=0) + m = Memory(location=lfw_home, compress=6, verbose=0) load_func = m.cache(_fetch_lfw_pairs) # select the right metadata file according to the requested subset label_filenames = { - 'train': 'pairsDevTrain.txt', - 'test': 'pairsDevTest.txt', - '10_folds': 'pairs.txt', + "train": "pairsDevTrain.txt", + "test": "pairsDevTest.txt", + "10_folds": "pairs.txt", } if subset not in label_filenames: - raise ValueError("subset='%s' is invalid: should be one of %r" % ( - subset, list(sorted(label_filenames.keys())))) + raise ValueError( + "subset='%s' is invalid: should be one of %r" + % (subset, list(sorted(label_filenames.keys()))) + ) index_file_path = join(lfw_home, label_filenames[subset]) # load and memoize the pairs as np arrays pairs, target, target_names = load_func( - index_file_path, data_folder_path, resize=resize, color=color, - slice_=slice_) + index_file_path, data_folder_path, resize=resize, color=color, slice_=slice_ + ) - module_path = dirname(__file__) - with open(join(module_path, 'descr', 'lfw.rst')) as rst_file: - fdescr = rst_file.read() + fdescr = load_descr("lfw.rst") # pack the results as a Bunch instance - return Bunch(data=pairs.reshape(len(pairs), -1), pairs=pairs, - target=target, target_names=target_names, - DESCR=fdescr) + return Bunch( + data=pairs.reshape(len(pairs), -1), + pairs=pairs, + target=target, + target_names=target_names, + DESCR=fdescr, + ) diff --git a/sklearn/datasets/_olivetti_faces.py b/sklearn/datasets/_olivetti_faces.py index 76388a4a92a42..efb382b1dcdda 100644 --- a/sklearn/datasets/_olivetti_faces.py +++ b/sklearn/datasets/_olivetti_faces.py @@ -10,35 +10,53 @@ https://cs.nyu.edu/~roweis/ """ -# Copyright (c) 2011 David Warde-Farley -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -from os.path import dirname, exists, join -from os import makedirs, remove +from numbers import Integral, Real +from os import PathLike, makedirs, remove +from os.path import exists -import numpy as np -from scipy.io.matlab import loadmat import joblib +import numpy as np +from scipy.io import loadmat +from ..utils import Bunch, check_random_state +from ..utils._param_validation import Interval, validate_params from . import get_data_home -from ._base import _fetch_remote -from ._base import RemoteFileMetadata -from ._base import _pkl_filepath -from ..utils import check_random_state, Bunch -from ..utils.validation import _deprecate_positional_args +from ._base import RemoteFileMetadata, _fetch_remote, _pkl_filepath, load_descr # The original data can be found at: # https://cs.nyu.edu/~roweis/data/olivettifaces.mat FACES = RemoteFileMetadata( - filename='olivettifaces.mat', - url='https://ndownloader.figshare.com/files/5976027', - checksum=('b612fb967f2dc77c9c62d3e1266e0c73' - 'd5fca46a4b8906c18e454d41af987794')) - - -@_deprecate_positional_args -def fetch_olivetti_faces(*, data_home=None, shuffle=False, random_state=0, - download_if_missing=True, return_X_y=False): + filename="olivettifaces.mat", + url="https://ndownloader.figshare.com/files/5976027", + checksum="b612fb967f2dc77c9c62d3e1266e0c73d5fca46a4b8906c18e454d41af987794", +) + + +@validate_params( + { + "data_home": [str, PathLike, None], + "shuffle": ["boolean"], + "random_state": ["random_state"], + "download_if_missing": ["boolean"], + "return_X_y": ["boolean"], + "n_retries": [Interval(Integral, 1, None, closed="left")], + "delay": [Interval(Real, 0.0, None, closed="neither")], + }, + prefer_skip_nested_validation=True, +) +def fetch_olivetti_faces( + *, + data_home=None, + shuffle=False, + random_state=0, + download_if_missing=True, + return_X_y=False, + n_retries=3, + delay=1.0, +): """Load the Olivetti faces data-set from AT&T (classification). Download it if necessary. @@ -54,7 +72,7 @@ def fetch_olivetti_faces(*, data_home=None, shuffle=False, random_state=0, Parameters ---------- - data_home : str, default=None + data_home : str or path-like, default=None Specify another download and cache folder for the datasets. By default all scikit-learn data is stored in '~/scikit_learn_data' subfolders. @@ -68,7 +86,7 @@ def fetch_olivetti_faces(*, data_home=None, shuffle=False, random_state=0, See :term:`Glossary `. download_if_missing : bool, default=True - If False, raise a IOError if the data is not locally available + If False, raise an OSError if the data is not locally available instead of trying to download the data from the source site. return_X_y : bool, default=False @@ -77,6 +95,16 @@ def fetch_olivetti_faces(*, data_home=None, shuffle=False, random_state=0, .. versionadded:: 0.22 + n_retries : int, default=3 + Number of retries when HTTP errors are encountered. + + .. versionadded:: 1.5 + + delay : float, default=1.0 + Number of seconds between retries. + + .. versionadded:: 1.5 + Returns ------- data : :class:`~sklearn.utils.Bunch` @@ -96,24 +124,38 @@ def fetch_olivetti_faces(*, data_home=None, shuffle=False, random_state=0, Description of the modified Olivetti Faces Dataset. (data, target) : tuple if `return_X_y=True` + Tuple with the `data` and `target` objects described above. + .. versionadded:: 0.22 + + Examples + -------- + >>> from sklearn.datasets import fetch_olivetti_faces + >>> olivetti_faces = fetch_olivetti_faces() + >>> olivetti_faces.data.shape + (400, 4096) + >>> olivetti_faces.target.shape + (400,) + >>> olivetti_faces.images.shape + (400, 64, 64) """ data_home = get_data_home(data_home=data_home) if not exists(data_home): makedirs(data_home) - filepath = _pkl_filepath(data_home, 'olivetti.pkz') + filepath = _pkl_filepath(data_home, "olivetti.pkz") if not exists(filepath): if not download_if_missing: - raise IOError("Data not found and `download_if_missing` is False") + raise OSError("Data not found and `download_if_missing` is False") - print('downloading Olivetti faces from %s to %s' - % (FACES.url, data_home)) - mat_path = _fetch_remote(FACES, dirname=data_home) + print("downloading Olivetti faces from %s to %s" % (FACES.url, data_home)) + mat_path = _fetch_remote( + FACES, dirname=data_home, n_retries=n_retries, delay=delay + ) mfile = loadmat(file_name=mat_path) # delete raw .mat data remove(mat_path) - faces = mfile['faces'].T.copy() + faces = mfile["faces"].T.copy() joblib.dump(faces, filepath, compress=6) del mfile else: @@ -134,14 +176,9 @@ def fetch_olivetti_faces(*, data_home=None, shuffle=False, random_state=0, target = target[order] faces_vectorized = faces.reshape(len(faces), -1) - module_path = dirname(__file__) - with open(join(module_path, 'descr', 'olivetti_faces.rst')) as rst_file: - fdescr = rst_file.read() + fdescr = load_descr("olivetti_faces.rst") if return_X_y: return faces_vectorized, target - return Bunch(data=faces_vectorized, - images=faces, - target=target, - DESCR=fdescr) + return Bunch(data=faces_vectorized, images=faces, target=target, DESCR=fdescr) diff --git a/sklearn/datasets/_openml.py b/sklearn/datasets/_openml.py index b589c9faa5213..47ecdcd14de9d 100644 --- a/sklearn/datasets/_openml.py +++ b/sklearn/datasets/_openml.py @@ -1,58 +1,62 @@ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import gzip +import hashlib import json import os import shutil -import hashlib -from os.path import join -from warnings import warn +import time from contextlib import closing from functools import wraps -from typing import Callable, Optional, Dict, Tuple, List, Any, Union -import itertools -from collections.abc import Generator -from collections import OrderedDict -from functools import partial - -from urllib.request import urlopen, Request +from os.path import join +from tempfile import TemporaryDirectory +from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from urllib.error import HTTPError, URLError +from urllib.parse import urlparse +from urllib.request import Request, urlopen +from warnings import warn import numpy as np -import scipy.sparse -from ..externals import _arff -from ..externals._arff import ArffSparseDataType, ArffContainerType -from . import get_data_home -from urllib.error import HTTPError from ..utils import Bunch -from ..utils import is_scalar_nan -from ..utils import get_chunk_n_rows -from ..utils import _chunk_generator -from ..utils import check_pandas_support # noqa -from ..utils.validation import _deprecate_positional_args +from ..utils._optional_dependencies import check_pandas_support +from ..utils._param_validation import ( + Integral, + Interval, + Real, + StrOptions, + validate_params, +) +from . import get_data_home +from ._arff_parser import load_arff_from_gzip_file -__all__ = ['fetch_openml'] +__all__ = ["fetch_openml"] -_OPENML_PREFIX = "https://openml.org/" -_SEARCH_NAME = "api/v1/json/data/list/data_name/{}/limit/2" -_DATA_INFO = "api/v1/json/data/{}" -_DATA_FEATURES = "api/v1/json/data/features/{}" -_DATA_QUALITIES = "api/v1/json/data/qualities/{}" -_DATA_FILE = "data/v1/download/{}" +_SEARCH_NAME = "https://api.openml.org/api/v1/json/data/list/data_name/{}/limit/2" +_DATA_INFO = "https://api.openml.org/api/v1/json/data/{}" +_DATA_FEATURES = "https://api.openml.org/api/v1/json/data/features/{}" +_DATA_QUALITIES = "https://api.openml.org/api/v1/json/data/qualities/{}" OpenmlQualitiesType = List[Dict[str, str]] OpenmlFeaturesType = List[Dict[str, str]] def _get_local_path(openml_path: str, data_home: str) -> str: - return os.path.join(data_home, 'openml.org', openml_path + ".gz") + return os.path.join(data_home, "openml.org", openml_path + ".gz") def _retry_with_clean_cache( - openml_path: str, data_home: Optional[str] + openml_path: str, + data_home: Optional[str], + no_retry_exception: Optional[Exception] = None, ) -> Callable: """If the first call to the decorated function fails, the local cached file is removed, and the function is called again. If ``data_home`` is - ``None``, then the function is called once. + ``None``, then the function is called once. We can provide a specific + exception to not retry on using `no_retry_exception` parameter. """ + def decorator(f): @wraps(f) def wrapper(*args, **kw): @@ -60,66 +64,125 @@ def wrapper(*args, **kw): return f(*args, **kw) try: return f(*args, **kw) - except HTTPError: + except URLError: raise - except Exception: + except Exception as exc: + if no_retry_exception is not None and isinstance( + exc, no_retry_exception + ): + raise warn("Invalid cache, redownloading file", RuntimeWarning) local_path = _get_local_path(openml_path, data_home) if os.path.exists(local_path): os.unlink(local_path) return f(*args, **kw) + + return wrapper + + return decorator + + +def _retry_on_network_error( + n_retries: int = 3, delay: float = 1.0, url: str = "" +) -> Callable: + """If the function call results in a network error, call the function again + up to ``n_retries`` times with a ``delay`` between each call. If the error + has a 412 status code, don't call the function again as this is a specific + OpenML error. + The url parameter is used to give more information to the user about the + error. + """ + + def decorator(f): + @wraps(f) + def wrapper(*args, **kwargs): + retry_counter = n_retries + while True: + try: + return f(*args, **kwargs) + except (URLError, TimeoutError) as e: + # 412 is a specific OpenML error code. + if isinstance(e, HTTPError) and e.code == 412: + raise + if retry_counter == 0: + raise + warn( + f"A network error occurred while downloading {url}. Retrying..." + ) + retry_counter -= 1 + time.sleep(delay) + return wrapper + return decorator -def _open_openml_url(openml_path: str, data_home: Optional[str]): +def _open_openml_url( + url: str, data_home: Optional[str], n_retries: int = 3, delay: float = 1.0 +): """ Returns a resource from OpenML.org. Caches it to data_home if required. Parameters ---------- - openml_path : str - OpenML URL that will be accessed. This will be prefixes with - _OPENML_PREFIX + url : str + OpenML URL that will be downloaded and cached locally. The path component + of the URL is used to replicate the tree structure as sub-folders of the local + cache folder. data_home : str Directory to which the files will be cached. If None, no caching will be applied. + n_retries : int, default=3 + Number of retries when HTTP errors are encountered. Error with status + code 412 won't be retried as they represent OpenML generic errors. + + delay : float, default=1.0 + Number of seconds between retries. + Returns ------- result : stream - A stream to the OpenML resource + A stream to the OpenML resource. """ + def is_gzip_encoded(_fsrc): - return _fsrc.info().get('Content-Encoding', '') == 'gzip' + return _fsrc.info().get("Content-Encoding", "") == "gzip" - req = Request(_OPENML_PREFIX + openml_path) - req.add_header('Accept-encoding', 'gzip') + req = Request(url) + req.add_header("Accept-encoding", "gzip") if data_home is None: - fsrc = urlopen(req) + fsrc = _retry_on_network_error(n_retries, delay, req.full_url)(urlopen)(req) if is_gzip_encoded(fsrc): - return gzip.GzipFile(fileobj=fsrc, mode='rb') + return gzip.GzipFile(fileobj=fsrc, mode="rb") return fsrc + openml_path = urlparse(url).path.lstrip("/") local_path = _get_local_path(openml_path, data_home) + dir_name, file_name = os.path.split(local_path) if not os.path.exists(local_path): + os.makedirs(dir_name, exist_ok=True) try: - os.makedirs(os.path.dirname(local_path)) - except OSError: - # potentially, the directory has been created already - pass - - try: - with closing(urlopen(req)) as fsrc: - opener: Callable - if is_gzip_encoded(fsrc): - opener = open - else: - opener = gzip.GzipFile - with opener(local_path, 'wb') as fdst: - shutil.copyfileobj(fsrc, fdst) + # Create a tmpdir as a subfolder of dir_name where the final file will + # be moved to if the download is successful. This guarantees that the + # renaming operation to the final location is atomic to ensure the + # concurrence safety of the dataset caching mechanism. + with TemporaryDirectory(dir=dir_name) as tmpdir: + with closing( + _retry_on_network_error(n_retries, delay, req.full_url)(urlopen)( + req + ) + ) as fsrc: + opener: Callable + if is_gzip_encoded(fsrc): + opener = open + else: + opener = gzip.GzipFile + with opener(os.path.join(tmpdir, file_name), "wb") as fdst: + shutil.copyfileobj(fsrc, fdst) + shutil.move(fdst.name, local_path) except Exception: if os.path.exists(local_path): os.unlink(local_path) @@ -127,35 +190,45 @@ def is_gzip_encoded(_fsrc): # XXX: First time, decompression will not be necessary (by using fsrc), but # it will happen nonetheless - return gzip.GzipFile(local_path, 'rb') + return gzip.GzipFile(local_path, "rb") class OpenMLError(ValueError): """HTTP 412 is a specific OpenML error code, indicating a generic error""" + pass def _get_json_content_from_openml_api( url: str, error_message: Optional[str], - data_home: Optional[str] + data_home: Optional[str], + n_retries: int = 3, + delay: float = 1.0, ) -> Dict: """ - Loads json data from the openml api + Loads json data from the openml api. Parameters ---------- url : str - The URL to load from. Should be an official OpenML endpoint + The URL to load from. Should be an official OpenML endpoint. error_message : str or None The error message to raise if an acceptable OpenML error is thrown (acceptable error is, e.g., data id not found. Other errors, like 404's - will throw the native error message) + will throw the native error message). data_home : str or None Location to cache the response. None if no cache is required. + n_retries : int, default=3 + Number of retries when HTTP errors are encountered. Error with status + code 412 won't be retried as they represent OpenML generic errors. + + delay : float, default=1.0 + Number of seconds between retries. + Returns ------- json_data : json @@ -163,9 +236,11 @@ def _get_json_content_from_openml_api( An exception otherwise. """ - @_retry_with_clean_cache(url, data_home) + @_retry_with_clean_cache(url, data_home=data_home) def _load_json(): - with closing(_open_openml_url(url, data_home)) as response: + with closing( + _open_openml_url(url, data_home, n_retries=n_retries, delay=delay) + ) as response: return json.loads(response.read().decode("utf-8")) try: @@ -180,194 +255,12 @@ def _load_json(): raise OpenMLError(error_message) -def _split_sparse_columns( - arff_data: ArffSparseDataType, include_columns: List -) -> ArffSparseDataType: - """ - obtains several columns from sparse arff representation. Additionally, the - column indices are re-labelled, given the columns that are not included. - (e.g., when including [1, 2, 3], the columns will be relabelled to - [0, 1, 2]) - - Parameters - ---------- - arff_data : tuple - A tuple of three lists of equal size; first list indicating the value, - second the x coordinate and the third the y coordinate. - - include_columns : list - A list of columns to include. - - Returns - ------- - arff_data_new : tuple - Subset of arff data with only the include columns indicated by the - include_columns argument. - """ - arff_data_new: ArffSparseDataType = (list(), list(), list()) - reindexed_columns = {column_idx: array_idx for array_idx, column_idx - in enumerate(include_columns)} - for val, row_idx, col_idx in zip(arff_data[0], arff_data[1], arff_data[2]): - if col_idx in include_columns: - arff_data_new[0].append(val) - arff_data_new[1].append(row_idx) - arff_data_new[2].append(reindexed_columns[col_idx]) - return arff_data_new - - -def _sparse_data_to_array( - arff_data: ArffSparseDataType, include_columns: List -) -> np.ndarray: - # turns the sparse data back into an array (can't use toarray() function, - # as this does only work on numeric data) - num_obs = max(arff_data[1]) + 1 - y_shape = (num_obs, len(include_columns)) - reindexed_columns = {column_idx: array_idx for array_idx, column_idx - in enumerate(include_columns)} - # TODO: improve for efficiency - y = np.empty(y_shape, dtype=np.float64) - for val, row_idx, col_idx in zip(arff_data[0], arff_data[1], arff_data[2]): - if col_idx in include_columns: - y[row_idx, reindexed_columns[col_idx]] = val - return y - - -def _convert_arff_data( - arff: ArffContainerType, - col_slice_x: List[int], - col_slice_y: List[int], - shape: Optional[Tuple] = None -) -> Tuple: - """ - converts the arff object into the appropriate matrix type (np.array or - scipy.sparse.csr_matrix) based on the 'data part' (i.e., in the - liac-arff dict, the object from the 'data' key) - - Parameters - ---------- - arff : dict - As obtained from liac-arff object. - - col_slice_x : list - The column indices that are sliced from the original array to return - as X data - - col_slice_y : list - The column indices that are sliced from the original array to return - as y data - - Returns - ------- - X : np.array or scipy.sparse.csr_matrix - y : np.array - """ - arff_data = arff['data'] - if isinstance(arff_data, Generator): - if shape is None: - raise ValueError( - "shape must be provided when arr['data'] is a Generator" - ) - if shape[0] == -1: - count = -1 - else: - count = shape[0] * shape[1] - data = np.fromiter(itertools.chain.from_iterable(arff_data), - dtype='float64', count=count) - data = data.reshape(*shape) - X = data[:, col_slice_x] - y = data[:, col_slice_y] - return X, y - elif isinstance(arff_data, tuple): - arff_data_X = _split_sparse_columns(arff_data, col_slice_x) - num_obs = max(arff_data[1]) + 1 - X_shape = (num_obs, len(col_slice_x)) - X = scipy.sparse.coo_matrix( - (arff_data_X[0], (arff_data_X[1], arff_data_X[2])), - shape=X_shape, dtype=np.float64) - X = X.tocsr() - y = _sparse_data_to_array(arff_data, col_slice_y) - return X, y - else: - # This should never happen - raise ValueError('Unexpected Data Type obtained from arff.') - - -def _feature_to_dtype(feature: Dict[str, str]): - """Map feature to dtype for pandas DataFrame - """ - if feature['data_type'] == 'string': - return object - elif feature['data_type'] == 'nominal': - return 'category' - # only numeric, integer, real are left - elif (feature['number_of_missing_values'] != '0' or - feature['data_type'] in ['numeric', 'real']): - # cast to floats when there are any missing values - return np.float64 - elif feature['data_type'] == 'integer': - return np.int64 - raise ValueError('Unsupported feature: {}'.format(feature)) - - -def _convert_arff_data_dataframe( - arff: ArffContainerType, columns: List, features_dict: Dict[str, Any] -) -> Tuple: - """Convert the ARFF object into a pandas DataFrame. - - Parameters - ---------- - arff : dict - As obtained from liac-arff object. - - columns : list - Columns from dataframe to return. - - features_dict : dict - Maps feature name to feature info from openml. - - Returns - ------- - result : tuple - tuple with the resulting dataframe - """ - pd = check_pandas_support('fetch_openml with as_frame=True') - - attributes = OrderedDict(arff['attributes']) - arff_columns = list(attributes) - - if not isinstance(arff['data'], Generator): - raise ValueError( - "arff['data'] must be a generator when converting to pd.DataFrame." - ) - - # calculate chunksize - first_row = next(arff['data']) - first_df = pd.DataFrame([first_row], columns=arff_columns) - - row_bytes = first_df.memory_usage(deep=True).sum() - chunksize = get_chunk_n_rows(row_bytes) - - # read arff data with chunks - columns_to_keep = [col for col in arff_columns if col in columns] - dfs = [] - dfs.append(first_df[columns_to_keep]) - for data in _chunk_generator(arff['data'], chunksize): - dfs.append(pd.DataFrame(data, columns=arff_columns)[columns_to_keep]) - df = pd.concat(dfs, ignore_index=True) - - for column in columns_to_keep: - dtype = _feature_to_dtype(features_dict[column]) - if dtype == 'category': - cats_without_missing = [cat for cat in attributes[column] - if cat is not None and - not is_scalar_nan(cat)] - dtype = pd.api.types.CategoricalDtype(cats_without_missing) - df[column] = df[column].astype(dtype, copy=False) - return (df, ) - - def _get_data_info_by_name( - name: str, version: Union[int, str], data_home: Optional[str] + name: str, + version: Union[int, str], + data_home: Optional[str], + n_retries: int = 3, + delay: float = 1.0, ): """ Utilizes the openml dataset listing api to find a dataset by @@ -389,6 +282,13 @@ def _get_data_info_by_name( data_home : str or None Location to cache the response. None if no cache is required. + n_retries : int, default=3 + Number of retries when HTTP errors are encountered. Error with status + code 412 won't be retried as they represent OpenML generic errors. + + delay : float, default=1.0 + Number of seconds between retries. + Returns ------- first_dataset : json @@ -401,21 +301,38 @@ def _get_data_info_by_name( url = _SEARCH_NAME.format(name) + "/status/active/" error_msg = "No active dataset {} found.".format(name) json_data = _get_json_content_from_openml_api( - url, error_msg, data_home=data_home + url, + error_msg, + data_home=data_home, + n_retries=n_retries, + delay=delay, ) - res = json_data['data']['dataset'] + res = json_data["data"]["dataset"] if len(res) > 1: - warn("Multiple active versions of the dataset matching the name" - " {name} exist. Versions may be fundamentally different, " - "returning version" - " {version}.".format(name=name, version=res[0]['version'])) + first_version = version = res[0]["version"] + warning_msg = ( + "Multiple active versions of the dataset matching the name" + f" {name} exist. Versions may be fundamentally different, " + f"returning version {first_version}. " + "Available versions:\n" + ) + for r in res: + warning_msg += f"- version {r['version']}, status: {r['status']}\n" + warning_msg += ( + f" url: https://www.openml.org/search?type=data&id={r['did']}\n" + ) + warn(warning_msg) return res[0] # an integer version has been provided url = (_SEARCH_NAME + "/data_version/{}").format(name, version) try: json_data = _get_json_content_from_openml_api( - url, error_message=None, data_home=data_home + url, + error_message=None, + data_home=data_home, + n_retries=n_retries, + delay=delay, ) except OpenMLError: # we can do this in 1 function call if OpenML does not require the @@ -423,53 +340,77 @@ def _get_data_info_by_name( # given name / version regardless of active, deactivated, etc. ) # TODO: feature request OpenML. url += "/status/deactivated" - error_msg = "Dataset {} with version {} not found.".format(name, - version) + error_msg = "Dataset {} with version {} not found.".format(name, version) json_data = _get_json_content_from_openml_api( - url, error_msg, data_home=data_home + url, + error_msg, + data_home=data_home, + n_retries=n_retries, + delay=delay, ) - return json_data['data']['dataset'][0] + return json_data["data"]["dataset"][0] def _get_data_description_by_id( - data_id: int, data_home: Optional[str] + data_id: int, + data_home: Optional[str], + n_retries: int = 3, + delay: float = 1.0, ) -> Dict[str, Any]: # OpenML API function: https://www.openml.org/api_docs#!/data/get_data_id url = _DATA_INFO.format(data_id) error_message = "Dataset with data_id {} not found.".format(data_id) json_data = _get_json_content_from_openml_api( - url, error_message, data_home=data_home + url, + error_message, + data_home=data_home, + n_retries=n_retries, + delay=delay, ) - return json_data['data_set_description'] + return json_data["data_set_description"] def _get_data_features( - data_id: int, data_home: Optional[str] + data_id: int, + data_home: Optional[str], + n_retries: int = 3, + delay: float = 1.0, ) -> OpenmlFeaturesType: # OpenML function: # https://www.openml.org/api_docs#!/data/get_data_features_id url = _DATA_FEATURES.format(data_id) error_message = "Dataset with data_id {} not found.".format(data_id) json_data = _get_json_content_from_openml_api( - url, error_message, data_home=data_home + url, + error_message, + data_home=data_home, + n_retries=n_retries, + delay=delay, ) - return json_data['data_features']['feature'] + return json_data["data_features"]["feature"] def _get_data_qualities( - data_id: int, data_home: Optional[str] + data_id: int, + data_home: Optional[str], + n_retries: int = 3, + delay: float = 1.0, ) -> OpenmlQualitiesType: # OpenML API function: # https://www.openml.org/api_docs#!/data/get_data_qualities_id url = _DATA_QUALITIES.format(data_id) error_message = "Dataset with data_id {} not found.".format(data_id) json_data = _get_json_content_from_openml_api( - url, error_message, data_home=data_home + url, + error_message, + data_home=data_home, + n_retries=n_retries, + delay=delay, ) # the qualities might not be available, but we still try to process # the data - return json_data.get('data_qualities', {}).get('quality', []) + return json_data.get("data_qualities", {}).get("quality", []) def _get_num_samples(data_qualities: OpenmlQualitiesType) -> int: @@ -489,49 +430,144 @@ def _get_num_samples(data_qualities: OpenmlQualitiesType) -> int: # If the data qualities are unavailable, we return -1 default_n_samples = -1 - qualities = {d['name']: d['value'] for d in data_qualities} - return int(float(qualities.get('NumberOfInstances', default_n_samples))) + qualities = {d["name"]: d["value"] for d in data_qualities} + return int(float(qualities.get("NumberOfInstances", default_n_samples))) def _load_arff_response( url: str, data_home: Optional[str], - return_type, encode_nominal: bool, - parse_arff: Callable[[ArffContainerType], Tuple], - md5_checksum: str -) -> Tuple: - """Load arff data with url and parses arff response with parse_arff""" - response = _open_openml_url(url, data_home) + parser: str, + output_type: str, + openml_columns_info: dict, + feature_names_to_select: List[str], + target_names_to_select: List[str], + shape: Optional[Tuple[int, int]], + md5_checksum: str, + n_retries: int = 3, + delay: float = 1.0, + read_csv_kwargs: Optional[Dict] = None, +): + """Load the ARFF data associated with the OpenML URL. + + In addition of loading the data, this function will also check the + integrity of the downloaded file from OpenML using MD5 checksum. + + Parameters + ---------- + url : str + The URL of the ARFF file on OpenML. - with closing(response): - # Note that if the data is dense, no reading is done until the data - # generator is iterated. - actual_md5_checksum = hashlib.md5() + data_home : str + The location where to cache the data. + + parser : {"liac-arff", "pandas"} + The parser used to parse the ARFF file. + + output_type : {"numpy", "pandas", "sparse"} + The type of the arrays that will be returned. The possibilities are: + + - `"numpy"`: both `X` and `y` will be NumPy arrays; + - `"sparse"`: `X` will be sparse matrix and `y` will be a NumPy array; + - `"pandas"`: `X` will be a pandas DataFrame and `y` will be either a + pandas Series or DataFrame. + + openml_columns_info : dict + The information provided by OpenML regarding the columns of the ARFF + file. + + feature_names_to_select : list of str + The list of the features to be selected. + + target_names_to_select : list of str + The list of the target variables to be selected. + + shape : tuple or None + With `parser="liac-arff"`, when using a generator to load the data, + one needs to provide the shape of the data beforehand. + + md5_checksum : str + The MD5 checksum provided by OpenML to check the data integrity. + + n_retries : int, default=3 + The number of times to retry downloading the data if it fails. + + delay : float, default=1.0 + The delay between two consecutive downloads in seconds. + + read_csv_kwargs : dict, default=None + Keyword arguments to pass to `pandas.read_csv` when using the pandas parser. + It allows to overwrite the default options. - def _stream_checksum_generator(response): - for line in response: - actual_md5_checksum.update(line) - yield line.decode('utf-8') + .. versionadded:: 1.3 - stream = _stream_checksum_generator(response) + Returns + ------- + X : {ndarray, sparse matrix, dataframe} + The data matrix. + + y : {ndarray, dataframe, series} + The target. + + frame : dataframe or None + A dataframe containing both `X` and `y`. `None` if + `output_array_type != "pandas"`. + + categories : list of str or None + The names of the features that are categorical. `None` if + `output_array_type == "pandas"`. + """ + gzip_file = _open_openml_url(url, data_home, n_retries=n_retries, delay=delay) + with closing(gzip_file): + md5 = hashlib.md5() + for chunk in iter(lambda: gzip_file.read(4096), b""): + md5.update(chunk) + actual_md5_checksum = md5.hexdigest() + + if actual_md5_checksum != md5_checksum: + raise ValueError( + f"md5 checksum of local file for {url} does not match description: " + f"expected: {md5_checksum} but got {actual_md5_checksum}. " + "Downloaded file could have been modified / corrupted, clean cache " + "and retry..." + ) - arff = _arff.load(stream, - return_type=return_type, - encode_nominal=encode_nominal) + def _open_url_and_load_gzip_file(url, data_home, n_retries, delay, arff_params): + gzip_file = _open_openml_url(url, data_home, n_retries=n_retries, delay=delay) + with closing(gzip_file): + return load_arff_from_gzip_file(gzip_file, **arff_params) + + arff_params: Dict = dict( + parser=parser, + output_type=output_type, + openml_columns_info=openml_columns_info, + feature_names_to_select=feature_names_to_select, + target_names_to_select=target_names_to_select, + shape=shape, + read_csv_kwargs=read_csv_kwargs or {}, + ) + try: + X, y, frame, categories = _open_url_and_load_gzip_file( + url, data_home, n_retries, delay, arff_params + ) + except Exception as exc: + if parser != "pandas": + raise - parsed_arff = parse_arff(arff) + from pandas.errors import ParserError - # consume remaining stream, if early exited - for _ in stream: - pass + if not isinstance(exc, ParserError): + raise - if actual_md5_checksum.hexdigest() != md5_checksum: - raise ValueError("md5 checksum of local file for " + url + - " does not match description. " - "Downloaded file could have been modified / " - "corrupted, clean cache and retry...") + # A parsing error could come from providing the wrong quotechar + # to pandas. By default, we use a double quote. Thus, we retry + # with a single quote before to raise the error. + arff_params["read_csv_kwargs"].update(quotechar="'") + X, y, frame, categories = _open_url_and_load_gzip_file( + url, data_home, n_retries, delay, arff_params + ) - return parsed_arff + return X, y, frame, categories def _download_data_to_bunch( @@ -540,140 +576,164 @@ def _download_data_to_bunch( data_home: Optional[str], *, as_frame: bool, - features_list: List, - data_columns: List[int], - target_columns: List, + openml_columns_info: List[dict], + data_columns: List[str], + target_columns: List[str], shape: Optional[Tuple[int, int]], - md5_checksum: str + md5_checksum: str, + n_retries: int = 3, + delay: float = 1.0, + parser: str, + read_csv_kwargs: Optional[Dict] = None, ): - """Download OpenML ARFF and convert to Bunch of data - """ - # NB: this function is long in order to handle retry for any failure - # during the streaming parse of the ARFF. + """Download ARFF data, load it to a specific container and create to Bunch. - # Prepare which columns and data types should be returned for the X and y - features_dict = {feature['name']: feature for feature in features_list} + This function has a mechanism to retry/cache/clean the data. - # XXX: col_slice_y should be all nominal or all numeric - _verify_target_data_type(features_dict, target_columns) + Parameters + ---------- + url : str + The URL of the ARFF file on OpenML. + + sparse : bool + Whether the dataset is expected to use the sparse ARFF format. - col_slice_y = [int(features_dict[col_name]['index']) - for col_name in target_columns] + data_home : str + The location where to cache the data. - col_slice_x = [int(features_dict[col_name]['index']) - for col_name in data_columns] - for col_idx in col_slice_y: - feat = features_list[col_idx] - nr_missing = int(feat['number_of_missing_values']) - if nr_missing > 0: - raise ValueError('Target column {} has {} missing values. ' - 'Missing values are not supported for target ' - 'columns. '.format(feat['name'], nr_missing)) + as_frame : bool + Whether or not to return the data into a pandas DataFrame. - # Access an ARFF file on the OpenML server. Documentation: - # https://www.openml.org/api_data_docs#!/data/get_download_id + openml_columns_info : list of dict + The information regarding the columns provided by OpenML for the + ARFF dataset. The information is stored as a list of dictionaries. - if sparse is True: - return_type = _arff.COO - else: - return_type = _arff.DENSE_GEN - - frame = nominal_attributes = None - - parse_arff: Callable - postprocess: Callable - if as_frame: - columns = data_columns + target_columns - parse_arff = partial(_convert_arff_data_dataframe, columns=columns, - features_dict=features_dict) - - def postprocess(frame): - X = frame[data_columns] - if len(target_columns) >= 2: - y = frame[target_columns] - elif len(target_columns) == 1: - y = frame[target_columns[0]] - else: - y = None - return X, y, frame, nominal_attributes + data_columns : list of str + The list of the features to be selected. + + target_columns : list of str + The list of the target variables to be selected. + + shape : tuple or None + With `parser="liac-arff"`, when using a generator to load the data, + one needs to provide the shape of the data beforehand. + + md5_checksum : str + The MD5 checksum provided by OpenML to check the data integrity. + + n_retries : int, default=3 + Number of retries when HTTP errors are encountered. Error with status + code 412 won't be retried as they represent OpenML generic errors. + + delay : float, default=1.0 + Number of seconds between retries. + + parser : {"liac-arff", "pandas"} + The parser used to parse the ARFF file. + + read_csv_kwargs : dict, default=None + Keyword arguments to pass to `pandas.read_csv` when using the pandas parser. + It allows to overwrite the default options. + + .. versionadded:: 1.3 + + Returns + ------- + data : :class:`~sklearn.utils.Bunch` + Dictionary-like object, with the following attributes. + + X : {ndarray, sparse matrix, dataframe} + The data matrix. + y : {ndarray, dataframe, series} + The target. + frame : dataframe or None + A dataframe containing both `X` and `y`. `None` if + `output_array_type != "pandas"`. + categories : list of str or None + The names of the features that are categorical. `None` if + `output_array_type == "pandas"`. + """ + # Prepare which columns and data types should be returned for the X and y + features_dict = {feature["name"]: feature for feature in openml_columns_info} + + if sparse: + output_type = "sparse" + elif as_frame: + output_type = "pandas" else: - def parse_arff(arff): - X, y = _convert_arff_data(arff, col_slice_x, col_slice_y, shape) - # nominal attributes is a dict mapping from the attribute name to - # the possible values. Includes also the target column (which will - # be popped off below, before it will be packed in the Bunch - # object) - nominal_attributes = {k: v for k, v in arff['attributes'] - if isinstance(v, list) and - k in data_columns + target_columns} - return X, y, nominal_attributes - - def postprocess(X, y, nominal_attributes): - is_classification = {col_name in nominal_attributes - for col_name in target_columns} - if not is_classification: - # No target - pass - elif all(is_classification): - y = np.hstack([ - np.take( - np.asarray(nominal_attributes.pop(col_name), - dtype='O'), - y[:, i:i + 1].astype(int, copy=False)) - for i, col_name in enumerate(target_columns) - ]) - elif any(is_classification): - raise ValueError('Mix of nominal and non-nominal targets is ' - 'not currently supported') - - # reshape y back to 1-D array, if there is only 1 target column; - # back to None if there are not target columns - if y.shape[1] == 1: - y = y.reshape((-1,)) - elif y.shape[1] == 0: - y = None - return X, y, frame, nominal_attributes - - out = _retry_with_clean_cache(url, data_home)( - _load_arff_response)(url, data_home, - return_type=return_type, - encode_nominal=not as_frame, - parse_arff=parse_arff, - md5_checksum=md5_checksum) - X, y, frame, nominal_attributes = postprocess(*out) - - return Bunch(data=X, target=y, frame=frame, - categories=nominal_attributes, - feature_names=data_columns, - target_names=target_columns) + output_type = "numpy" + + # XXX: target columns should all be categorical or all numeric + _verify_target_data_type(features_dict, target_columns) + for name in target_columns: + column_info = features_dict[name] + n_missing_values = int(column_info["number_of_missing_values"]) + if n_missing_values > 0: + raise ValueError( + f"Target column '{column_info['name']}' has {n_missing_values} missing " + "values. Missing values are not supported for target columns." + ) + + no_retry_exception = None + if parser == "pandas": + # If we get a ParserError with pandas, then we don't want to retry and we raise + # early. + from pandas.errors import ParserError + + no_retry_exception = ParserError + + X, y, frame, categories = _retry_with_clean_cache( + url, data_home, no_retry_exception + )(_load_arff_response)( + url, + data_home, + parser=parser, + output_type=output_type, + openml_columns_info=features_dict, + feature_names_to_select=data_columns, + target_names_to_select=target_columns, + shape=shape, + md5_checksum=md5_checksum, + n_retries=n_retries, + delay=delay, + read_csv_kwargs=read_csv_kwargs, + ) + + return Bunch( + data=X, + target=y, + frame=frame, + categories=categories, + feature_names=data_columns, + target_names=target_columns, + ) def _verify_target_data_type(features_dict, target_columns): # verifies the data type of the y array in case there are multiple targets # (throws an error if these targets do not comply with sklearn support) if not isinstance(target_columns, list): - raise ValueError('target_column should be list, ' - 'got: %s' % type(target_columns)) + raise ValueError("target_column should be list, got: %s" % type(target_columns)) found_types = set() for target_column in target_columns: if target_column not in features_dict: - raise KeyError('Could not find target_column={}') - if features_dict[target_column]['data_type'] == "numeric": + raise KeyError(f"Could not find target_column='{target_column}'") + if features_dict[target_column]["data_type"] == "numeric": found_types.add(np.float64) else: found_types.add(object) # note: we compare to a string, not boolean - if features_dict[target_column]['is_ignore'] == 'true': - warn('target_column={} has flag is_ignore.'.format( - target_column)) - if features_dict[target_column]['is_row_identifier'] == 'true': - warn('target_column={} has flag is_row_identifier.'.format( - target_column)) + if features_dict[target_column]["is_ignore"] == "true": + warn(f"target_column='{target_column}' has flag is_ignore.") + if features_dict[target_column]["is_row_identifier"] == "true": + warn(f"target_column='{target_column}' has flag is_row_identifier.") if len(found_types) > 1: - raise ValueError('Can only handle homogeneous multi-target datasets, ' - 'i.e., all targets are either numeric or ' - 'categorical.') + raise ValueError( + "Can only handle homogeneous multi-target datasets, " + "i.e., all targets are either numeric or " + "categorical." + ) def _valid_data_column_names(features_list, target_columns): @@ -683,24 +743,48 @@ def _valid_data_column_names(features_list, target_columns): # excluded. valid_data_column_names = [] for feature in features_list: - if (feature['name'] not in target_columns - and feature['is_ignore'] != 'true' - and feature['is_row_identifier'] != 'true'): - valid_data_column_names.append(feature['name']) + if ( + feature["name"] not in target_columns + and feature["is_ignore"] != "true" + and feature["is_row_identifier"] != "true" + ): + valid_data_column_names.append(feature["name"]) return valid_data_column_names -@_deprecate_positional_args +@validate_params( + { + "name": [str, None], + "version": [Interval(Integral, 1, None, closed="left"), StrOptions({"active"})], + "data_id": [Interval(Integral, 1, None, closed="left"), None], + "data_home": [str, os.PathLike, None], + "target_column": [str, list, None], + "cache": [bool], + "return_X_y": [bool], + "as_frame": [bool, StrOptions({"auto"})], + "n_retries": [Interval(Integral, 1, None, closed="left")], + "delay": [Interval(Real, 0.0, None, closed="neither")], + "parser": [ + StrOptions({"auto", "pandas", "liac-arff"}), + ], + "read_csv_kwargs": [dict, None], + }, + prefer_skip_nested_validation=True, +) def fetch_openml( name: Optional[str] = None, *, - version: Union[str, int] = 'active', + version: Union[str, int] = "active", data_id: Optional[int] = None, - data_home: Optional[str] = None, - target_column: Optional[Union[str, List]] = 'default-target', + data_home: Optional[Union[str, os.PathLike]] = None, + target_column: Optional[Union[str, List]] = "default-target", cache: bool = True, return_X_y: bool = False, - as_frame: Union[str, bool] = 'auto' + as_frame: Union[str, bool] = "auto", + n_retries: int = 3, + delay: float = 1.0, + parser: str = "auto", + read_csv_kwargs: Optional[Dict] = None, ): """Fetch dataset from openml by name or dataset id. @@ -738,7 +822,7 @@ def fetch_openml( dataset. If data_id is not given, name (and potential version) are used to obtain a dataset. - data_home : str, default=None + data_home : str or path-like, default=None Specify another download and cache folder for the data sets. By default all scikit-learn data is stored in '~/scikit_learn_data' subfolders. @@ -748,10 +832,10 @@ def fetch_openml( is used. If ``None``, all columns are returned as data and the target is ``None``. If list (of strings), all columns with these names are returned as multi-target (Note: not all scikit-learn classifiers - can handle all types of multi-output combinations) + can handle all types of multi-output combinations). cache : bool, default=True - Whether to cache downloaded datasets using joblib. + Whether to cache the downloaded datasets into `data_home`. return_X_y : bool, default=False If True, returns ``(data, target)`` instead of a Bunch object. See @@ -765,17 +849,52 @@ def fetch_openml( data. If ``return_X_y`` is True, then ``(data, target)`` will be pandas DataFrames or Series as describe above. - If as_frame is 'auto', the data and target will be converted to - DataFrame or Series as if as_frame is set to True, unless the dataset + If `as_frame` is 'auto', the data and target will be converted to + DataFrame or Series as if `as_frame` is set to True, unless the dataset is stored in sparse format. + If `as_frame` is False, the data and target will be NumPy arrays and + the `data` will only contain numerical values when `parser="liac-arff"` + where the categories are provided in the attribute `categories` of the + `Bunch` instance. When `parser="pandas"`, no ordinal encoding is made. + .. versionchanged:: 0.24 The default value of `as_frame` changed from `False` to `'auto'` in 0.24. + n_retries : int, default=3 + Number of retries when HTTP errors or network timeouts are encountered. + Error with status code 412 won't be retried as they represent OpenML + generic errors. + + delay : float, default=1.0 + Number of seconds between retries. + + parser : {"auto", "pandas", "liac-arff"}, default="auto" + Parser used to load the ARFF file. Two parsers are implemented: + + - `"pandas"`: this is the most efficient parser. However, it requires + pandas to be installed and can only open dense datasets. + - `"liac-arff"`: this is a pure Python ARFF parser that is much less + memory- and CPU-efficient. It deals with sparse ARFF datasets. + + If `"auto"`, the parser is chosen automatically such that `"liac-arff"` + is selected for sparse ARFF datasets, otherwise `"pandas"` is selected. + + .. versionadded:: 1.2 + .. versionchanged:: 1.4 + The default value of `parser` changes from `"liac-arff"` to + `"auto"`. + + read_csv_kwargs : dict, default=None + Keyword arguments passed to :func:`pandas.read_csv` when loading the data + from a ARFF file and using the pandas parser. It can allow to + overwrite some default parameters. + + .. versionadded:: 1.3 + Returns ------- - data : :class:`~sklearn.utils.Bunch` Dictionary-like object, with the following attributes. @@ -786,11 +905,11 @@ def fetch_openml( Dtype is float if numeric, and object if categorical. If ``as_frame`` is True, ``target`` is a pandas object. DESCR : str - The full description of the dataset + The full description of the dataset. feature_names : list - The names of the dataset columns + The names of the dataset columns. target_names: list - The names of the target columns + The names of the target columns. .. versionadded:: 0.22 @@ -799,7 +918,7 @@ def fetch_openml( that the value encoded as i is ith in the list. If ``as_frame`` is True, this is None. details : dict - More metadata from OpenML + More metadata from OpenML. frame : pandas DataFrame Only present when `as_frame=True`. DataFrame with ``data`` and ``target``. @@ -814,14 +933,69 @@ def fetch_openml( Missing values in the 'data' are represented as NaN's. Missing values in 'target' are represented as NaN's (numerical target) or None - (categorical target) + (categorical target). + + Notes + ----- + The `"pandas"` and `"liac-arff"` parsers can lead to different data types + in the output. The notable differences are the following: + + - The `"liac-arff"` parser always encodes categorical features as `str` objects. + To the contrary, the `"pandas"` parser instead infers the type while + reading and numerical categories will be casted into integers whenever + possible. + - The `"liac-arff"` parser uses float64 to encode numerical features + tagged as 'REAL' and 'NUMERICAL' in the metadata. The `"pandas"` + parser instead infers if these numerical features corresponds + to integers and uses panda's Integer extension dtype. + - In particular, classification datasets with integer categories are + typically loaded as such `(0, 1, ...)` with the `"pandas"` parser while + `"liac-arff"` will force the use of string encoded class labels such as + `"0"`, `"1"` and so on. + - The `"pandas"` parser will not strip single quotes - i.e. `'` - from + string columns. For instance, a string `'my string'` will be kept as is + while the `"liac-arff"` parser will strip the single quotes. For + categorical columns, the single quotes are stripped from the values. + + In addition, when `as_frame=False` is used, the `"liac-arff"` parser + returns ordinally encoded data where the categories are provided in the + attribute `categories` of the `Bunch` instance. Instead, `"pandas"` returns + a NumPy array were the categories are not encoded. + + Examples + -------- + >>> from sklearn.datasets import fetch_openml + >>> adult = fetch_openml("adult", version=2) # doctest: +SKIP + >>> adult.frame.info() # doctest: +SKIP + + RangeIndex: 48842 entries, 0 to 48841 + Data columns (total 15 columns): + # Column Non-Null Count Dtype + --- ------ -------------- ----- + 0 age 48842 non-null int64 + 1 workclass 46043 non-null category + 2 fnlwgt 48842 non-null int64 + 3 education 48842 non-null category + 4 education-num 48842 non-null int64 + 5 marital-status 48842 non-null category + 6 occupation 46033 non-null category + 7 relationship 48842 non-null category + 8 race 48842 non-null category + 9 sex 48842 non-null category + 10 capital-gain 48842 non-null int64 + 11 capital-loss 48842 non-null int64 + 12 hours-per-week 48842 non-null int64 + 13 native-country 47985 non-null category + 14 class 48842 non-null category + dtypes: category(9), int64(6) + memory usage: 2.7 MB """ if cache is False: # no caching will be applied data_home = None else: data_home = get_data_home(data_home=data_home) - data_home = join(data_home, 'openml') + data_home = join(str(data_home), "openml") # check valid function arguments. data_id XOR (name, version) should be # provided @@ -833,76 +1007,114 @@ def fetch_openml( raise ValueError( "Dataset data_id={} and name={} passed, but you can only " "specify a numeric data_id or a name, not " - "both.".format(data_id, name)) - data_info = _get_data_info_by_name(name, version, data_home) - data_id = data_info['did'] + "both.".format(data_id, name) + ) + data_info = _get_data_info_by_name( + name, version, data_home, n_retries=n_retries, delay=delay + ) + data_id = data_info["did"] elif data_id is not None: # from the previous if statement, it is given that name is None if version != "active": raise ValueError( "Dataset data_id={} and version={} passed, but you can only " "specify a numeric data_id or a version, not " - "both.".format(data_id, name)) + "both.".format(data_id, version) + ) else: raise ValueError( - "Neither name nor data_id are provided. Please provide name or " - "data_id.") + "Neither name nor data_id are provided. Please provide name or data_id." + ) data_description = _get_data_description_by_id(data_id, data_home) - if data_description['status'] != "active": - warn("Version {} of dataset {} is inactive, meaning that issues have " - "been found in the dataset. Try using a newer version from " - "this URL: {}".format( - data_description['version'], - data_description['name'], - data_description['url'])) - if 'error' in data_description: - warn("OpenML registered a problem with the dataset. It might be " - "unusable. Error: {}".format(data_description['error'])) - if 'warning' in data_description: - warn("OpenML raised a warning on the dataset. It might be " - "unusable. Warning: {}".format(data_description['warning'])) - - return_sparse = False - if data_description['format'].lower() == 'sparse_arff': - return_sparse = True - - if as_frame == 'auto': - as_frame = not return_sparse - - if as_frame and return_sparse: - raise ValueError('Cannot return dataframe with sparse data') + if data_description["status"] != "active": + warn( + "Version {} of dataset {} is inactive, meaning that issues have " + "been found in the dataset. Try using a newer version from " + "this URL: {}".format( + data_description["version"], + data_description["name"], + data_description["url"], + ) + ) + if "error" in data_description: + warn( + "OpenML registered a problem with the dataset. It might be " + "unusable. Error: {}".format(data_description["error"]) + ) + if "warning" in data_description: + warn( + "OpenML raised a warning on the dataset. It might be " + "unusable. Warning: {}".format(data_description["warning"]) + ) + + return_sparse = data_description["format"].lower() == "sparse_arff" + as_frame = not return_sparse if as_frame == "auto" else as_frame + if parser == "auto": + parser_ = "liac-arff" if return_sparse else "pandas" + else: + parser_ = parser + + if parser_ == "pandas": + try: + check_pandas_support("`fetch_openml`") + except ImportError as exc: + if as_frame: + err_msg = ( + "Returning pandas objects requires pandas to be installed. " + "Alternatively, explicitly set `as_frame=False` and " + "`parser='liac-arff'`." + ) + else: + err_msg = ( + f"Using `parser={parser!r}` with dense data requires pandas to be " + "installed. Alternatively, explicitly set `parser='liac-arff'`." + ) + raise ImportError(err_msg) from exc + + if return_sparse: + if as_frame: + raise ValueError( + "Sparse ARFF datasets cannot be loaded with as_frame=True. " + "Use as_frame=False or as_frame='auto' instead." + ) + if parser_ == "pandas": + raise ValueError( + f"Sparse ARFF datasets cannot be loaded with parser={parser!r}. " + "Use parser='liac-arff' or parser='auto' instead." + ) # download data features, meta-info about column types features_list = _get_data_features(data_id, data_home) if not as_frame: for feature in features_list: - if 'true' in (feature['is_ignore'], feature['is_row_identifier']): + if "true" in (feature["is_ignore"], feature["is_row_identifier"]): continue - if feature['data_type'] == 'string': - raise ValueError('STRING attributes are not supported for ' - 'array representation. Try as_frame=True') + if feature["data_type"] == "string": + raise ValueError( + "STRING attributes are not supported for " + "array representation. Try as_frame=True" + ) if target_column == "default-target": # determines the default target based on the data feature results # (which is currently more reliable than the data description; # see issue: https://github.com/openml/OpenML/issues/768) - target_columns = [feature['name'] for feature in features_list - if feature['is_target'] == 'true'] + target_columns = [ + feature["name"] + for feature in features_list + if feature["is_target"] == "true" + ] elif isinstance(target_column, str): # for code-simplicity, make target_column by default a list target_columns = [target_column] elif target_column is None: target_columns = [] - elif isinstance(target_column, list): - target_columns = target_column else: - raise TypeError("Did not recognize type of target_column" - "Should be str, list or None. Got: " - "{}".format(type(target_column))) - data_columns = _valid_data_column_names(features_list, - target_columns) + # target_column already is of type list + target_columns = target_column + data_columns = _valid_data_column_names(features_list, target_columns) shape: Optional[Tuple[int, int]] # determine arff encoding to return @@ -915,23 +1127,34 @@ def fetch_openml( shape = None # obtain the data - url = _DATA_FILE.format(data_description['file_id']) - bunch = _download_data_to_bunch(url, return_sparse, data_home, - as_frame=bool(as_frame), - features_list=features_list, shape=shape, - target_columns=target_columns, - data_columns=data_columns, - md5_checksum=data_description[ - "md5_checksum"]) + url = data_description["url"] + bunch = _download_data_to_bunch( + url, + return_sparse, + data_home, + as_frame=bool(as_frame), + openml_columns_info=features_list, + shape=shape, + target_columns=target_columns, + data_columns=data_columns, + md5_checksum=data_description["md5_checksum"], + n_retries=n_retries, + delay=delay, + parser=parser_, + read_csv_kwargs=read_csv_kwargs, + ) if return_X_y: return bunch.data, bunch.target description = "{}\n\nDownloaded from openml.org.".format( - data_description.pop('description')) + data_description.pop("description") + ) bunch.update( - DESCR=description, details=data_description, - url="https://www.openml.org/d/{}".format(data_id)) + DESCR=description, + details=data_description, + url="https://www.openml.org/d/{}".format(data_id), + ) return bunch diff --git a/sklearn/datasets/_rcv1.py b/sklearn/datasets/_rcv1.py index 2d3a809848e83..b673f938f0e46 100644 --- a/sklearn/datasets/_rcv1.py +++ b/sklearn/datasets/_rcv1.py @@ -5,28 +5,25 @@ http://jmlr.csail.mit.edu/papers/volume5/lewis04a/ """ -# Author: Tom Dupre la Tour -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import logging - -from os import remove, makedirs -from os.path import dirname, exists, join from gzip import GzipFile +from numbers import Integral, Real +from os import PathLike, makedirs, remove +from os.path import exists, join +import joblib import numpy as np import scipy.sparse as sp -import joblib +from ..utils import Bunch +from ..utils import shuffle as shuffle_ +from ..utils._param_validation import Interval, StrOptions, validate_params from . import get_data_home -from ._base import _pkl_filepath -from ._base import _fetch_remote -from ._base import RemoteFileMetadata +from ._base import RemoteFileMetadata, _fetch_remote, _pkl_filepath, load_descr from ._svmlight_format_io import load_svmlight_files -from ..utils import shuffle as shuffle_ -from ..utils import Bunch -from ..utils.validation import _deprecate_positional_args - # The original vectorized data can be found at: # http://www.ai.mit.edu/projects/jmlr/papers/volume5/lewis04a/a13-vector-files/lyrl2004_vectors_test_pt0.dat.gz @@ -39,46 +36,67 @@ # http://www.ai.mit.edu/projects/jmlr/papers/volume5/lewis04a/lyrl2004_rcv1v2_README.htm XY_METADATA = ( RemoteFileMetadata( - url='https://ndownloader.figshare.com/files/5976069', - checksum=('ed40f7e418d10484091b059703eeb95a' - 'e3199fe042891dcec4be6696b9968374'), - filename='lyrl2004_vectors_test_pt0.dat.gz'), + url="https://ndownloader.figshare.com/files/5976069", + checksum="ed40f7e418d10484091b059703eeb95ae3199fe042891dcec4be6696b9968374", + filename="lyrl2004_vectors_test_pt0.dat.gz", + ), RemoteFileMetadata( - url='https://ndownloader.figshare.com/files/5976066', - checksum=('87700668ae45d45d5ca1ef6ae9bd81ab' - '0f5ec88cc95dcef9ae7838f727a13aa6'), - filename='lyrl2004_vectors_test_pt1.dat.gz'), + url="https://ndownloader.figshare.com/files/5976066", + checksum="87700668ae45d45d5ca1ef6ae9bd81ab0f5ec88cc95dcef9ae7838f727a13aa6", + filename="lyrl2004_vectors_test_pt1.dat.gz", + ), RemoteFileMetadata( - url='https://ndownloader.figshare.com/files/5976063', - checksum=('48143ac703cbe33299f7ae9f4995db4' - '9a258690f60e5debbff8995c34841c7f5'), - filename='lyrl2004_vectors_test_pt2.dat.gz'), + url="https://ndownloader.figshare.com/files/5976063", + checksum="48143ac703cbe33299f7ae9f4995db49a258690f60e5debbff8995c34841c7f5", + filename="lyrl2004_vectors_test_pt2.dat.gz", + ), RemoteFileMetadata( - url='https://ndownloader.figshare.com/files/5976060', - checksum=('dfcb0d658311481523c6e6ca0c3f5a3' - 'e1d3d12cde5d7a8ce629a9006ec7dbb39'), - filename='lyrl2004_vectors_test_pt3.dat.gz'), + url="https://ndownloader.figshare.com/files/5976060", + checksum="dfcb0d658311481523c6e6ca0c3f5a3e1d3d12cde5d7a8ce629a9006ec7dbb39", + filename="lyrl2004_vectors_test_pt3.dat.gz", + ), RemoteFileMetadata( - url='https://ndownloader.figshare.com/files/5976057', - checksum=('5468f656d0ba7a83afc7ad44841cf9a5' - '3048a5c083eedc005dcdb5cc768924ae'), - filename='lyrl2004_vectors_train.dat.gz') + url="https://ndownloader.figshare.com/files/5976057", + checksum="5468f656d0ba7a83afc7ad44841cf9a53048a5c083eedc005dcdb5cc768924ae", + filename="lyrl2004_vectors_train.dat.gz", + ), ) # The original data can be found at: # http://jmlr.csail.mit.edu/papers/volume5/lewis04a/a08-topic-qrels/rcv1-v2.topics.qrels.gz TOPICS_METADATA = RemoteFileMetadata( - url='https://ndownloader.figshare.com/files/5976048', - checksum=('2a98e5e5d8b770bded93afc8930d882' - '99474317fe14181aee1466cc754d0d1c1'), - filename='rcv1v2.topics.qrels.gz') + url="https://ndownloader.figshare.com/files/5976048", + checksum="2a98e5e5d8b770bded93afc8930d88299474317fe14181aee1466cc754d0d1c1", + filename="rcv1v2.topics.qrels.gz", +) logger = logging.getLogger(__name__) -@_deprecate_positional_args -def fetch_rcv1(*, data_home=None, subset='all', download_if_missing=True, - random_state=None, shuffle=False, return_X_y=False): +@validate_params( + { + "data_home": [str, PathLike, None], + "subset": [StrOptions({"train", "test", "all"})], + "download_if_missing": ["boolean"], + "random_state": ["random_state"], + "shuffle": ["boolean"], + "return_X_y": ["boolean"], + "n_retries": [Interval(Integral, 1, None, closed="left")], + "delay": [Interval(Real, 0.0, None, closed="neither")], + }, + prefer_skip_nested_validation=True, +) +def fetch_rcv1( + *, + data_home=None, + subset="all", + download_if_missing=True, + random_state=None, + shuffle=False, + return_X_y=False, + n_retries=3, + delay=1.0, +): """Load the RCV1 multilabel dataset (classification). Download it if necessary. @@ -98,7 +116,7 @@ def fetch_rcv1(*, data_home=None, subset='all', download_if_missing=True, Parameters ---------- - data_home : str, default=None + data_home : str or path-like, default=None Specify another download and cache folder for the datasets. By default all scikit-learn data is stored in '~/scikit_learn_data' subfolders. @@ -109,7 +127,7 @@ def fetch_rcv1(*, data_home=None, subset='all', download_if_missing=True, This follows the official LYRL2004 chronological split. download_if_missing : bool, default=True - If False, raise a IOError if the data is not locally available + If False, raise an OSError if the data is not locally available instead of trying to download the data from the source site. random_state : int, RandomState instance or None, default=None @@ -127,26 +145,48 @@ def fetch_rcv1(*, data_home=None, subset='all', download_if_missing=True, .. versionadded:: 0.20 + n_retries : int, default=3 + Number of retries when HTTP errors are encountered. + + .. versionadded:: 1.5 + + delay : float, default=1.0 + Number of seconds between retries. + + .. versionadded:: 1.5 + Returns ------- dataset : :class:`~sklearn.utils.Bunch` - Dictionary-like object, with the following attributes. + Dictionary-like object. Returned only if `return_X_y` is False. + `dataset` has the following attributes: - data : sparse matrix of shape (804414, 47236), dtype=np.float64 + - data : sparse matrix of shape (804414, 47236), dtype=np.float64 The array has 0.16% of non zero values. Will be of CSR format. - target : sparse matrix of shape (804414, 103), dtype=np.uint8 + - target : sparse matrix of shape (804414, 103), dtype=np.uint8 Each sample has a value of 1 in its categories, and 0 in others. The array has 3.15% of non zero values. Will be of CSR format. - sample_id : ndarray of shape (804414,), dtype=np.uint32, + - sample_id : ndarray of shape (804414,), dtype=np.uint32, Identification number of each sample, as ordered in dataset.data. - target_names : ndarray of shape (103,), dtype=object + - target_names : ndarray of shape (103,), dtype=object Names of each target (RCV1 topics), as ordered in dataset.target. - DESCR : str + - DESCR : str Description of the RCV1 dataset. - (data, target) : tuple if ``return_X_y`` is True + (data, target) : tuple + A tuple consisting of `dataset.data` and `dataset.target`, as + described above. Returned only if `return_X_y` is True. .. versionadded:: 0.20 + + Examples + -------- + >>> from sklearn.datasets import fetch_rcv1 + >>> rcv1 = fetch_rcv1() + >>> rcv1.data.shape + (804414, 47236) + >>> rcv1.target.shape + (804414, 103) """ N_SAMPLES = 804414 N_FEATURES = 47236 @@ -165,12 +205,13 @@ def fetch_rcv1(*, data_home=None, subset='all', download_if_missing=True, topics_path = _pkl_filepath(rcv1_dir, "topics_names.pkl") # load data (X) and sample_id - if download_if_missing and (not exists(samples_path) or - not exists(sample_id_path)): + if download_if_missing and (not exists(samples_path) or not exists(sample_id_path)): files = [] for each in XY_METADATA: logger.info("Downloading %s" % each.url) - file_path = _fetch_remote(each, dirname=rcv1_dir) + file_path = _fetch_remote( + each, dirname=rcv1_dir, n_retries=n_retries, delay=delay + ) files.append(GzipFile(filename=file_path)) Xy = load_svmlight_files(files, n_features=N_FEATURES) @@ -192,11 +233,13 @@ def fetch_rcv1(*, data_home=None, subset='all', download_if_missing=True, sample_id = joblib.load(sample_id_path) # load target (y), categories, and sample_id_bis - if download_if_missing and (not exists(sample_topics_path) or - not exists(topics_path)): + if download_if_missing and ( + not exists(sample_topics_path) or not exists(topics_path) + ): logger.info("Downloading %s" % TOPICS_METADATA.url) - topics_archive_path = _fetch_remote(TOPICS_METADATA, - dirname=rcv1_dir) + topics_archive_path = _fetch_remote( + TOPICS_METADATA, dirname=rcv1_dir, n_retries=n_retries, delay=delay + ) # parse the target file n_cat = -1 @@ -205,7 +248,7 @@ def fetch_rcv1(*, data_home=None, subset='all', download_if_missing=True, y = np.zeros((N_SAMPLES, N_CATEGORIES), dtype=np.uint8) sample_id_bis = np.zeros(N_SAMPLES, dtype=np.int32) category_names = {} - with GzipFile(filename=topics_archive_path, mode='rb') as f: + with GzipFile(filename=topics_archive_path, mode="rb") as f: for line in f: line_components = line.decode("ascii").split(" ") if len(line_components) == 3: @@ -245,32 +288,33 @@ def fetch_rcv1(*, data_home=None, subset='all', download_if_missing=True, y = joblib.load(sample_topics_path) categories = joblib.load(topics_path) - if subset == 'all': + if subset == "all": pass - elif subset == 'train': + elif subset == "train": X = X[:N_TRAIN, :] y = y[:N_TRAIN, :] sample_id = sample_id[:N_TRAIN] - elif subset == 'test': + elif subset == "test": X = X[N_TRAIN:, :] y = y[N_TRAIN:, :] sample_id = sample_id[N_TRAIN:] else: - raise ValueError("Unknown subset parameter. Got '%s' instead of one" - " of ('all', 'train', test')" % subset) + raise ValueError( + "Unknown subset parameter. Got '%s' instead of one" + " of ('all', 'train', test')" % subset + ) if shuffle: X, y, sample_id = shuffle_(X, y, sample_id, random_state=random_state) - module_path = dirname(__file__) - with open(join(module_path, 'descr', 'rcv1.rst')) as rst_file: - fdescr = rst_file.read() + fdescr = load_descr("rcv1.rst") if return_X_y: return X, y - return Bunch(data=X, target=y, sample_id=sample_id, - target_names=categories, DESCR=fdescr) + return Bunch( + data=X, target=y, sample_id=sample_id, target_names=categories, DESCR=fdescr + ) def _inverse_permutation(p): diff --git a/sklearn/datasets/_samples_generator.py b/sklearn/datasets/_samples_generator.py index 66d1baaaa9cb2..e2d80422e7df7 100644 --- a/sklearn/datasets/_samples_generator.py +++ b/sklearn/datasets/_samples_generator.py @@ -2,44 +2,83 @@ Generate samples of synthetic data sets. """ -# Authors: B. Thirion, G. Varoquaux, A. Gramfort, V. Michel, O. Grisel, -# G. Louppe, J. Nothman -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import numbers import array +import numbers from collections.abc import Iterable +from numbers import Integral, Real import numpy as np -from scipy import linalg import scipy.sparse as sp +from scipy import linalg + +from sklearn.utils import Bunch from ..preprocessing import MultiLabelBinarizer from ..utils import check_array, check_random_state from ..utils import shuffle as util_shuffle +from ..utils._param_validation import Interval, StrOptions, validate_params from ..utils.random import sample_without_replacement -from ..utils.validation import _deprecate_positional_args def _generate_hypercube(samples, dimensions, rng): - """Returns distinct binary samples of length dimensions. - """ + """Returns distinct binary samples of length dimensions.""" if dimensions > 30: - return np.hstack([rng.randint(2, size=(samples, dimensions - 30)), - _generate_hypercube(samples, 30, rng)]) - out = sample_without_replacement(2 ** dimensions, samples, - random_state=rng).astype(dtype='>u4', - copy=False) - out = np.unpackbits(out.view('>u1')).reshape((-1, 32))[:, -dimensions:] + return np.hstack( + [ + rng.randint(2, size=(samples, dimensions - 30)), + _generate_hypercube(samples, 30, rng), + ] + ) + out = sample_without_replacement(2**dimensions, samples, random_state=rng).astype( + dtype=">u4", copy=False + ) + out = np.unpackbits(out.view(">u1")).reshape((-1, 32))[:, -dimensions:] return out -@_deprecate_positional_args -def make_classification(n_samples=100, n_features=20, *, n_informative=2, - n_redundant=2, n_repeated=0, n_classes=2, - n_clusters_per_class=2, weights=None, flip_y=0.01, - class_sep=1.0, hypercube=True, shift=0.0, scale=1.0, - shuffle=True, random_state=None): +@validate_params( + { + "n_samples": [Interval(Integral, 1, None, closed="left")], + "n_features": [Interval(Integral, 1, None, closed="left")], + "n_informative": [Interval(Integral, 1, None, closed="left")], + "n_redundant": [Interval(Integral, 0, None, closed="left")], + "n_repeated": [Interval(Integral, 0, None, closed="left")], + "n_classes": [Interval(Integral, 1, None, closed="left")], + "n_clusters_per_class": [Interval(Integral, 1, None, closed="left")], + "weights": ["array-like", None], + "flip_y": [Interval(Real, 0, 1, closed="both")], + "class_sep": [Interval(Real, 0, None, closed="neither")], + "hypercube": ["boolean"], + "shift": [Interval(Real, None, None, closed="neither"), "array-like", None], + "scale": [Interval(Real, 0, None, closed="neither"), "array-like", None], + "shuffle": ["boolean"], + "random_state": ["random_state"], + "return_X_y": ["boolean"], + }, + prefer_skip_nested_validation=True, +) +def make_classification( + n_samples=100, + n_features=20, + *, + n_informative=2, + n_redundant=2, + n_repeated=0, + n_classes=2, + n_clusters_per_class=2, + weights=None, + flip_y=0.01, + class_sep=1.0, + hypercube=True, + shift=0.0, + scale=1.0, + shuffle=True, + random_state=None, + return_X_y=True, +): """Generate a random n-class classification problem. This initially creates clusters of points normally distributed (std=1) @@ -133,13 +172,37 @@ def make_classification(n_samples=100, n_features=20, *, n_informative=2, for reproducible output across multiple function calls. See :term:`Glossary `. + return_X_y : bool, default=True + If True, a tuple ``(X, y)`` instead of a Bunch object is returned. + + .. versionadded:: 1.7 + Returns ------- - X : ndarray of shape (n_samples, n_features) - The generated samples. + data : :class:`~sklearn.utils.Bunch` if `return_X_y` is `False`. + Dictionary-like object, with the following attributes. + + DESCR : str + A description of the function that generated the dataset. + parameter : dict + A dictionary that stores the values of the arguments passed to the + generator function. + feature_info : list of len(n_features) + A description for each generated feature. + X : ndarray of shape (n_samples, n_features) + The generated samples. + y : ndarray of shape (n_samples,) + An integer label for class membership of each sample. + + .. versionadded:: 1.7 + + (X, y) : tuple if ``return_X_y`` is True + A tuple of generated samples and labels. - y : ndarray of shape (n_samples,) - The integer labels for class membership of each sample. + See Also + -------- + make_blobs : Simplified variant. + make_multilabel_classification : Unrelated generator for multilabel tasks. Notes ----- @@ -151,45 +214,59 @@ def make_classification(n_samples=100, n_features=20, *, n_informative=2, .. [1] I. Guyon, "Design of experiments for the NIPS 2003 variable selection benchmark", 2003. - See Also + Examples -------- - make_blobs : Simplified variant. - make_multilabel_classification : Unrelated generator for multilabel tasks. + >>> from sklearn.datasets import make_classification + >>> X, y = make_classification(random_state=42) + >>> X.shape + (100, 20) + >>> y.shape + (100,) + >>> list(y[:5]) + [np.int64(0), np.int64(0), np.int64(1), np.int64(1), np.int64(0)] """ generator = check_random_state(random_state) # Count features, clusters and samples if n_informative + n_redundant + n_repeated > n_features: - raise ValueError("Number of informative, redundant and repeated " - "features must sum to less than the number of total" - " features") + raise ValueError( + "Number of informative, redundant and repeated " + "features must sum to less than the number of total" + " features" + ) # Use log2 to avoid overflow errors if n_informative < np.log2(n_classes * n_clusters_per_class): msg = "n_classes({}) * n_clusters_per_class({}) must be" msg += " smaller or equal 2**n_informative({})={}" - raise ValueError(msg.format(n_classes, n_clusters_per_class, - n_informative, 2**n_informative)) + raise ValueError( + msg.format(n_classes, n_clusters_per_class, n_informative, 2**n_informative) + ) if weights is not None: + # we define new variable, weight_, instead of modifying user defined parameter. if len(weights) not in [n_classes, n_classes - 1]: - raise ValueError("Weights specified but incompatible with number " - "of classes.") + raise ValueError( + "Weights specified but incompatible with number of classes." + ) if len(weights) == n_classes - 1: if isinstance(weights, list): - weights = weights + [1.0 - sum(weights)] + weights_ = weights + [1.0 - sum(weights)] else: - weights = np.resize(weights, n_classes) - weights[-1] = 1.0 - sum(weights[:-1]) + weights_ = np.resize(weights, n_classes) + weights_[-1] = 1.0 - sum(weights_[:-1]) + else: + weights_ = weights.copy() else: - weights = [1.0 / n_classes] * n_classes + weights_ = [1.0 / n_classes] * n_classes - n_useless = n_features - n_informative - n_redundant - n_repeated + n_random = n_features - n_informative - n_redundant - n_repeated n_clusters = n_classes * n_clusters_per_class # Distribute samples among clusters by weight n_samples_per_cluster = [ - int(n_samples * weights[k % n_classes] / n_clusters_per_class) - for k in range(n_clusters)] + int(n_samples * weights_[k % n_classes] / n_clusters_per_class) + for k in range(n_clusters) + ] for i in range(n_samples - sum(n_samples_per_cluster)): n_samples_per_cluster[i % n_clusters] += 1 @@ -199,16 +276,17 @@ def make_classification(n_samples=100, n_features=20, *, n_informative=2, y = np.zeros(n_samples, dtype=int) # Build the polytope whose vertices become cluster centroids - centroids = _generate_hypercube(n_clusters, n_informative, - generator).astype(float, copy=False) + centroids = _generate_hypercube(n_clusters, n_informative, generator).astype( + float, copy=False + ) centroids *= 2 * class_sep centroids -= class_sep if not hypercube: - centroids *= generator.rand(n_clusters, 1) - centroids *= generator.rand(1, n_informative) + centroids *= generator.uniform(size=(n_clusters, 1)) + centroids *= generator.uniform(size=(1, n_informative)) # Initially draw informative features from the standard normal - X[:, :n_informative] = generator.randn(n_samples, n_informative) + X[:, :n_informative] = generator.standard_normal(size=(n_samples, n_informative)) # Create each cluster; a variant of make_blobs stop = 0 @@ -217,60 +295,122 @@ def make_classification(n_samples=100, n_features=20, *, n_informative=2, y[start:stop] = k % n_classes # assign labels X_k = X[start:stop, :n_informative] # slice a view of the cluster - A = 2 * generator.rand(n_informative, n_informative) - 1 + A = 2 * generator.uniform(size=(n_informative, n_informative)) - 1 X_k[...] = np.dot(X_k, A) # introduce random covariance X_k += centroid # shift the cluster to a vertex # Create redundant features if n_redundant > 0: - B = 2 * generator.rand(n_informative, n_redundant) - 1 - X[:, n_informative:n_informative + n_redundant] = \ - np.dot(X[:, :n_informative], B) + B = 2 * generator.uniform(size=(n_informative, n_redundant)) - 1 + X[:, n_informative : n_informative + n_redundant] = np.dot( + X[:, :n_informative], B + ) # Repeat some features + n = n_informative + n_redundant if n_repeated > 0: - n = n_informative + n_redundant - indices = ((n - 1) * generator.rand(n_repeated) + 0.5).astype(np.intp) - X[:, n:n + n_repeated] = X[:, indices] + indices = ((n - 1) * generator.uniform(size=n_repeated) + 0.5).astype(np.intp) + X[:, n : n + n_repeated] = X[:, indices] # Fill useless features - if n_useless > 0: - X[:, -n_useless:] = generator.randn(n_samples, n_useless) + if n_random > 0: + X[:, -n_random:] = generator.standard_normal(size=(n_samples, n_random)) # Randomly replace labels if flip_y >= 0.0: - flip_mask = generator.rand(n_samples) < flip_y + flip_mask = generator.uniform(size=n_samples) < flip_y y[flip_mask] = generator.randint(n_classes, size=flip_mask.sum()) # Randomly shift and scale if shift is None: - shift = (2 * generator.rand(n_features) - 1) * class_sep + shift = (2 * generator.uniform(size=n_features) - 1) * class_sep X += shift if scale is None: - scale = 1 + 100 * generator.rand(n_features) + scale = 1 + 100 * generator.uniform(size=n_features) X *= scale + indices = np.arange(n_features) if shuffle: # Randomly permute samples X, y = util_shuffle(X, y, random_state=generator) # Randomly permute features - indices = np.arange(n_features) generator.shuffle(indices) X[:, :] = X[:, indices] - return X, y - + if return_X_y: + return X, y -@_deprecate_positional_args -def make_multilabel_classification(n_samples=100, n_features=20, *, - n_classes=5, - n_labels=2, length=50, allow_unlabeled=True, - sparse=False, return_indicator='dense', - return_distributions=False, - random_state=None): + # feat_desc describes features in X + feat_desc = ["random"] * n_features + for i, index in enumerate(indices): + if index < n_informative: + feat_desc[i] = "informative" + elif n_informative <= index < n_informative + n_redundant: + feat_desc[i] = "redundant" + elif n <= index < n + n_repeated: + feat_desc[i] = "repeated" + + parameters = { + "n_samples": n_samples, + "n_features": n_features, + "n_informative": n_informative, + "n_redundant": n_redundant, + "n_repeated": n_repeated, + "n_classes": n_classes, + "n_clusters_per_class": n_clusters_per_class, + "weights": weights, + "flip_y": flip_y, + "class_sep": class_sep, + "hypercube": hypercube, + "shift": shift, + "scale": scale, + "shuffle": shuffle, + "random_state": random_state, + "return_X_y": return_X_y, + } + + bunch = Bunch( + DESCR=make_classification.__doc__, + parameters=parameters, + feature_info=feat_desc, + X=X, + y=y, + ) + + return bunch + + +@validate_params( + { + "n_samples": [Interval(Integral, 1, None, closed="left")], + "n_features": [Interval(Integral, 1, None, closed="left")], + "n_classes": [Interval(Integral, 1, None, closed="left")], + "n_labels": [Interval(Integral, 0, None, closed="left")], + "length": [Interval(Integral, 1, None, closed="left")], + "allow_unlabeled": ["boolean"], + "sparse": ["boolean"], + "return_indicator": [StrOptions({"dense", "sparse"}), "boolean"], + "return_distributions": ["boolean"], + "random_state": ["random_state"], + }, + prefer_skip_nested_validation=True, +) +def make_multilabel_classification( + n_samples=100, + n_features=20, + *, + n_classes=5, + n_labels=2, + length=50, + allow_unlabeled=True, + sparse=False, + return_indicator="dense", + return_distributions=False, + random_state=None, +): """Generate a random multilabel classification problem. For each sample, the generative process is: @@ -283,6 +423,9 @@ def make_multilabel_classification(n_samples=100, n_features=20, *, n is never zero or more than `n_classes`, and that the document length is never zero. Likewise, we reject classes which have already been chosen. + For an example of usage, see + :ref:`sphx_glr_auto_examples_datasets_plot_random_multilabel_dataset.py`. + Read more in the :ref:`User Guide `. Parameters @@ -311,7 +454,7 @@ def make_multilabel_classification(n_samples=100, n_features=20, *, If ``True``, some instances might not belong to any class. sparse : bool, default=False - If ``True``, return a sparse feature matrix + If ``True``, return a sparse feature matrix. .. versionadded:: 0.17 parameter to allow *sparse* output. @@ -347,23 +490,23 @@ def make_multilabel_classification(n_samples=100, n_features=20, *, The probability of each feature being drawn given each class. Only returned if ``return_distributions=True``. + Examples + -------- + >>> from sklearn.datasets import make_multilabel_classification + >>> X, y = make_multilabel_classification(n_labels=3, random_state=42) + >>> X.shape + (100, 20) + >>> y.shape + (100, 5) + >>> list(y[:3]) + [array([1, 1, 0, 1, 0]), array([0, 1, 1, 1, 0]), array([0, 1, 0, 0, 0])] """ - if n_classes < 1: - raise ValueError( - "'n_classes' should be an integer greater than 0. Got {} instead." - .format(n_classes) - ) - if length < 1: - raise ValueError( - "'length' should be an integer greater than 0. Got {} instead." - .format(length) - ) generator = check_random_state(random_state) - p_c = generator.rand(n_classes) + p_c = generator.uniform(size=n_classes) p_c /= p_c.sum() cumulative_p_c = np.cumsum(p_c) - p_w_c = generator.rand(n_features, n_classes) + p_w_c = generator.uniform(size=(n_features, n_classes)) p_w_c /= np.sum(p_w_c, axis=0) def sample_example(): @@ -378,8 +521,7 @@ def sample_example(): y = set() while len(y) != y_size: # pick a class with probability P(c) - c = np.searchsorted(cumulative_p_c, - generator.rand(y_size - len(y))) + c = np.searchsorted(cumulative_p_c, generator.uniform(size=y_size - len(y))) y.update(c) y = list(y) @@ -397,11 +539,11 @@ def sample_example(): # sample words with replacement from selected classes cumulative_p_w_sample = p_w_c.take(y, axis=1).sum(axis=1).cumsum() cumulative_p_w_sample /= cumulative_p_w_sample[-1] - words = np.searchsorted(cumulative_p_w_sample, generator.rand(n_words)) + words = np.searchsorted(cumulative_p_w_sample, generator.uniform(size=n_words)) return words, y - X_indices = array.array('i') - X_indptr = array.array('i', [0]) + X_indices = array.array("i") + X_indptr = array.array("i", [0]) Y = [] for i in range(n_samples): words, y = sample_example() @@ -409,28 +551,29 @@ def sample_example(): X_indptr.append(len(X_indices)) Y.append(y) X_data = np.ones(len(X_indices), dtype=np.float64) - X = sp.csr_matrix((X_data, X_indices, X_indptr), - shape=(n_samples, n_features)) + X = sp.csr_matrix((X_data, X_indices, X_indptr), shape=(n_samples, n_features)) X.sum_duplicates() if not sparse: X = X.toarray() # return_indicator can be True due to backward compatibility - if return_indicator in (True, 'sparse', 'dense'): - lb = MultiLabelBinarizer(sparse_output=(return_indicator == 'sparse')) + if return_indicator in (True, "sparse", "dense"): + lb = MultiLabelBinarizer(sparse_output=(return_indicator == "sparse")) Y = lb.fit([range(n_classes)]).transform(Y) - elif return_indicator is not False: - raise ValueError("return_indicator must be either 'sparse', 'dense' " - 'or False.') if return_distributions: return X, Y, p_c, p_w_c return X, Y -@_deprecate_positional_args +@validate_params( + { + "n_samples": [Interval(Integral, 1, None, closed="left")], + "random_state": ["random_state"], + }, + prefer_skip_nested_validation=True, +) def make_hastie_10_2(n_samples=12000, *, random_state=None): - """Generates data for binary classification used in - Hastie et al. 2009, Example 10.2. + """Generate data for binary classification used in Hastie et al. 2009, Example 10.2. The ten features are standard independent Gaussian and the target ``y`` is defined by:: @@ -457,30 +600,67 @@ def make_hastie_10_2(n_samples=12000, *, random_state=None): y : ndarray of shape (n_samples,) The output values. + See Also + -------- + make_gaussian_quantiles : A generalization of this dataset approach. + References ---------- .. [1] T. Hastie, R. Tibshirani and J. Friedman, "Elements of Statistical Learning Ed. 2", Springer, 2009. - See Also + Examples -------- - make_gaussian_quantiles : A generalization of this dataset approach. + >>> from sklearn.datasets import make_hastie_10_2 + >>> X, y = make_hastie_10_2(n_samples=24000, random_state=42) + >>> X.shape + (24000, 10) + >>> y.shape + (24000,) + >>> list(y[:5]) + [np.float64(-1.0), np.float64(1.0), np.float64(-1.0), np.float64(1.0), + np.float64(-1.0)] """ rs = check_random_state(random_state) shape = (n_samples, 10) X = rs.normal(size=shape).reshape(shape) - y = ((X ** 2.0).sum(axis=1) > 9.34).astype(np.float64, copy=False) + y = ((X**2.0).sum(axis=1) > 9.34).astype(np.float64, copy=False) y[y == 0.0] = -1.0 return X, y -@_deprecate_positional_args -def make_regression(n_samples=100, n_features=100, *, n_informative=10, - n_targets=1, bias=0.0, effective_rank=None, - tail_strength=0.5, noise=0.0, shuffle=True, coef=False, - random_state=None): +@validate_params( + { + "n_samples": [Interval(Integral, 1, None, closed="left")], + "n_features": [Interval(Integral, 1, None, closed="left")], + "n_informative": [Interval(Integral, 0, None, closed="left")], + "n_targets": [Interval(Integral, 1, None, closed="left")], + "bias": [Interval(Real, None, None, closed="neither")], + "effective_rank": [Interval(Integral, 1, None, closed="left"), None], + "tail_strength": [Interval(Real, 0, 1, closed="both")], + "noise": [Interval(Real, 0, None, closed="left")], + "shuffle": ["boolean"], + "coef": ["boolean"], + "random_state": ["random_state"], + }, + prefer_skip_nested_validation=True, +) +def make_regression( + n_samples=100, + n_features=100, + *, + n_informative=10, + n_targets=1, + bias=0.0, + effective_rank=None, + tail_strength=0.5, + noise=0.0, + shuffle=True, + coef=False, + random_state=None, +): """Generate a random regression problem. The input set can either be well conditioned (by default) or have a low @@ -514,12 +694,12 @@ def make_regression(n_samples=100, n_features=100, *, n_informative=10, The bias term in the underlying linear model. effective_rank : int, default=None - if not None: + If not None: The approximate number of singular vectors required to explain most of the input data by linear combinations. Using this kind of singular spectrum in the input allows the generator to reproduce the correlations often observed in practice. - if None: + If None: The input set is well conditioned, centered and gaussian with unit variance. @@ -553,28 +733,44 @@ def make_regression(n_samples=100, n_features=100, *, n_informative=10, coef : ndarray of shape (n_features,) or (n_features, n_targets) The coefficient of the underlying linear model. It is returned only if coef is True. + + Examples + -------- + >>> from sklearn.datasets import make_regression + >>> X, y = make_regression(n_samples=5, n_features=2, noise=1, random_state=42) + >>> X + array([[ 0.4967, -0.1382 ], + [ 0.6476, 1.523], + [-0.2341, -0.2341], + [-0.4694, 0.5425], + [ 1.579, 0.7674]]) + >>> y + array([ 6.737, 37.79, -10.27, 0.4017, 42.22]) """ n_informative = min(n_features, n_informative) generator = check_random_state(random_state) if effective_rank is None: # Randomly generate a well conditioned input set - X = generator.randn(n_samples, n_features) + X = generator.standard_normal(size=(n_samples, n_features)) else: # Randomly generate a low rank, fat tail input set - X = make_low_rank_matrix(n_samples=n_samples, - n_features=n_features, - effective_rank=effective_rank, - tail_strength=tail_strength, - random_state=generator) + X = make_low_rank_matrix( + n_samples=n_samples, + n_features=n_features, + effective_rank=effective_rank, + tail_strength=tail_strength, + random_state=generator, + ) # Generate a ground truth model with only n_informative features being non # zeros (the other features are not correlated to y and should be ignored # by a sparsifying regularizers such as L1 or elastic net) ground_truth = np.zeros((n_features, n_targets)) - ground_truth[:n_informative, :] = 100 * generator.rand(n_informative, - n_targets) + ground_truth[:n_informative, :] = 100 * generator.uniform( + size=(n_informative, n_targets) + ) y = np.dot(X, ground_truth) + bias @@ -600,9 +796,19 @@ def make_regression(n_samples=100, n_features=100, *, n_informative=10, return X, y -@_deprecate_positional_args -def make_circles(n_samples=100, *, shuffle=True, noise=None, random_state=None, - factor=.8): +@validate_params( + { + "n_samples": [Interval(Integral, 0, None, closed="left"), tuple], + "shuffle": ["boolean"], + "noise": [Interval(Real, 0, None, closed="left"), None], + "random_state": ["random_state"], + "factor": [Interval(Real, 0, 1, closed="left")], + }, + prefer_skip_nested_validation=True, +) +def make_circles( + n_samples=100, *, shuffle=True, noise=None, random_state=None, factor=0.8 +): """Make a large circle containing a smaller circle in 2d. A simple toy dataset to visualize clustering and classification @@ -634,7 +840,7 @@ def make_circles(n_samples=100, *, shuffle=True, noise=None, random_state=None, See :term:`Glossary `. factor : float, default=.8 - Scale factor between inner and outer circle in the range `(0, 1)`. + Scale factor between inner and outer circle in the range `[0, 1)`. Returns ------- @@ -643,20 +849,25 @@ def make_circles(n_samples=100, *, shuffle=True, noise=None, random_state=None, y : ndarray of shape (n_samples,) The integer labels (0 or 1) for class membership of each sample. - """ - - if factor >= 1 or factor < 0: - raise ValueError("'factor' has to be between 0 and 1.") + Examples + -------- + >>> from sklearn.datasets import make_circles + >>> X, y = make_circles(random_state=42) + >>> X.shape + (100, 2) + >>> y.shape + (100,) + >>> list(y[:5]) + [np.int64(1), np.int64(1), np.int64(1), np.int64(0), np.int64(0)] + """ if isinstance(n_samples, numbers.Integral): n_samples_out = n_samples // 2 n_samples_in = n_samples - n_samples_out - else: - try: - n_samples_out, n_samples_in = n_samples - except ValueError as e: - raise ValueError('`n_samples` can be either an int or ' - 'a two-element tuple.') from e + else: # n_samples is a tuple + if len(n_samples) != 2: + raise ValueError("When a tuple, n_samples must have exactly two elements.") + n_samples_out, n_samples_in = n_samples generator = check_random_state(random_state) # so as not to have the first point = last point, we set endpoint=False @@ -667,10 +878,12 @@ def make_circles(n_samples=100, *, shuffle=True, noise=None, random_state=None, inner_circ_x = np.cos(linspace_in) * factor inner_circ_y = np.sin(linspace_in) * factor - X = np.vstack([np.append(outer_circ_x, inner_circ_x), - np.append(outer_circ_y, inner_circ_y)]).T - y = np.hstack([np.zeros(n_samples_out, dtype=np.intp), - np.ones(n_samples_in, dtype=np.intp)]) + X = np.vstack( + [np.append(outer_circ_x, inner_circ_x), np.append(outer_circ_y, inner_circ_y)] + ).T + y = np.hstack( + [np.zeros(n_samples_out, dtype=np.intp), np.ones(n_samples_in, dtype=np.intp)] + ) if shuffle: X, y = util_shuffle(X, y, random_state=generator) @@ -680,7 +893,15 @@ def make_circles(n_samples=100, *, shuffle=True, noise=None, random_state=None, return X, y -@_deprecate_positional_args +@validate_params( + { + "n_samples": [Interval(Integral, 1, None, closed="left"), tuple], + "shuffle": ["boolean"], + "noise": [Interval(Real, 0, None, closed="left"), None], + "random_state": ["random_state"], + }, + prefer_skip_nested_validation=True, +) def make_moons(n_samples=100, *, shuffle=True, noise=None, random_state=None): """Make two interleaving half circles. @@ -714,6 +935,15 @@ def make_moons(n_samples=100, *, shuffle=True, noise=None, random_state=None): y : ndarray of shape (n_samples,) The integer labels (0 or 1) for class membership of each sample. + + Examples + -------- + >>> from sklearn.datasets import make_moons + >>> X, y = make_moons(n_samples=200, noise=0.2, random_state=42) + >>> X.shape + (200, 2) + >>> y.shape + (200,) """ if isinstance(n_samples, numbers.Integral): @@ -723,20 +953,23 @@ def make_moons(n_samples=100, *, shuffle=True, noise=None, random_state=None): try: n_samples_out, n_samples_in = n_samples except ValueError as e: - raise ValueError('`n_samples` can be either an int or ' - 'a two-element tuple.') from e + raise ValueError( + "`n_samples` can be either an int or a two-element tuple." + ) from e generator = check_random_state(random_state) outer_circ_x = np.cos(np.linspace(0, np.pi, n_samples_out)) outer_circ_y = np.sin(np.linspace(0, np.pi, n_samples_out)) inner_circ_x = 1 - np.cos(np.linspace(0, np.pi, n_samples_in)) - inner_circ_y = 1 - np.sin(np.linspace(0, np.pi, n_samples_in)) - .5 + inner_circ_y = 1 - np.sin(np.linspace(0, np.pi, n_samples_in)) - 0.5 - X = np.vstack([np.append(outer_circ_x, inner_circ_x), - np.append(outer_circ_y, inner_circ_y)]).T - y = np.hstack([np.zeros(n_samples_out, dtype=np.intp), - np.ones(n_samples_in, dtype=np.intp)]) + X = np.vstack( + [np.append(outer_circ_x, inner_circ_x), np.append(outer_circ_y, inner_circ_y)] + ).T + y = np.hstack( + [np.zeros(n_samples_out, dtype=np.intp), np.ones(n_samples_in, dtype=np.intp)] + ) if shuffle: X, y = util_shuffle(X, y, random_state=generator) @@ -747,10 +980,30 @@ def make_moons(n_samples=100, *, shuffle=True, noise=None, random_state=None): return X, y -@_deprecate_positional_args -def make_blobs(n_samples=100, n_features=2, *, centers=None, cluster_std=1.0, - center_box=(-10.0, 10.0), shuffle=True, random_state=None, - return_centers=False): +@validate_params( + { + "n_samples": [Interval(Integral, 1, None, closed="left"), "array-like"], + "n_features": [Interval(Integral, 1, None, closed="left")], + "centers": [Interval(Integral, 1, None, closed="left"), "array-like", None], + "cluster_std": [Interval(Real, 0, None, closed="left"), "array-like"], + "center_box": [tuple], + "shuffle": ["boolean"], + "random_state": ["random_state"], + "return_centers": ["boolean"], + }, + prefer_skip_nested_validation=True, +) +def make_blobs( + n_samples=100, + n_features=2, + *, + centers=None, + cluster_std=1.0, + center_box=(-10.0, 10.0), + shuffle=True, + random_state=None, + return_centers=False, +): """Generate isotropic Gaussian blobs for clustering. Read more in the :ref:`User Guide `. @@ -769,7 +1022,7 @@ def make_blobs(n_samples=100, n_features=2, *, centers=None, cluster_std=1.0, n_features : int, default=2 The number of features for each sample. - centers : int or ndarray of shape (n_centers, n_features), default=None + centers : int or array-like of shape (n_centers, n_features), default=None The number of centers to generate, or the fixed center locations. If n_samples is an int and centers is None, 3 centers are generated. If n_samples is array-like, centers must be @@ -791,7 +1044,7 @@ def make_blobs(n_samples=100, n_features=2, *, centers=None, cluster_std=1.0, See :term:`Glossary `. return_centers : bool, default=False - If True, then return the centers of each cluster + If True, then return the centers of each cluster. .. versionadded:: 0.23 @@ -807,6 +1060,10 @@ def make_blobs(n_samples=100, n_features=2, *, centers=None, cluster_std=1.0, The centers of each cluster. Only returned if ``return_centers=True``. + See Also + -------- + make_classification : A more intricate variant. + Examples -------- >>> from sklearn.datasets import make_blobs @@ -822,10 +1079,6 @@ def make_blobs(n_samples=100, n_features=2, *, centers=None, cluster_std=1.0, (10, 2) >>> y array([0, 1, 2, 0, 2, 2, 2, 1, 1, 0]) - - See Also - -------- - make_classification : A more intricate variant. """ generator = check_random_state(random_state) @@ -836,8 +1089,9 @@ def make_blobs(n_samples=100, n_features=2, *, centers=None, cluster_std=1.0, if isinstance(centers, numbers.Integral): n_centers = centers - centers = generator.uniform(center_box[0], center_box[1], - size=(n_centers, n_features)) + centers = generator.uniform( + center_box[0], center_box[1], size=(n_centers, n_features) + ) else: centers = check_array(centers) @@ -848,35 +1102,35 @@ def make_blobs(n_samples=100, n_features=2, *, centers=None, cluster_std=1.0, # Set n_centers by looking at [n_samples] arg n_centers = len(n_samples) if centers is None: - centers = generator.uniform(center_box[0], center_box[1], - size=(n_centers, n_features)) - try: - assert len(centers) == n_centers - except TypeError as e: - raise ValueError("Parameter `centers` must be array-like. " - "Got {!r} instead".format(centers)) from e - except AssertionError as e: + centers = generator.uniform( + center_box[0], center_box[1], size=(n_centers, n_features) + ) + if not isinstance(centers, Iterable): + raise ValueError( + "Parameter `centers` must be array-like. Got {!r} instead".format( + centers + ) + ) + if len(centers) != n_centers: raise ValueError( - f"Length of `n_samples` not consistent with number of " + "Length of `n_samples` not consistent with number of " f"centers. Got n_samples = {n_samples} and centers = {centers}" - ) from e - else: - centers = check_array(centers) - n_features = centers.shape[1] + ) + centers = check_array(centers) + n_features = centers.shape[1] # stds: if cluster_std is given as list, it must be consistent # with the n_centers - if (hasattr(cluster_std, "__len__") and len(cluster_std) != n_centers): - raise ValueError("Length of `clusters_std` not consistent with " - "number of centers. Got centers = {} " - "and cluster_std = {}".format(centers, cluster_std)) + if hasattr(cluster_std, "__len__") and len(cluster_std) != n_centers: + raise ValueError( + "Length of `clusters_std` not consistent with " + "number of centers. Got centers = {} " + "and cluster_std = {}".format(centers, cluster_std) + ) if isinstance(cluster_std, numbers.Real): cluster_std = np.full(len(centers), cluster_std) - X = [] - y = [] - if isinstance(n_samples, Iterable): n_samples_per_center = n_samples else: @@ -885,20 +1139,20 @@ def make_blobs(n_samples=100, n_features=2, *, centers=None, cluster_std=1.0, for i in range(n_samples % n_centers): n_samples_per_center[i] += 1 - for i, (n, std) in enumerate(zip(n_samples_per_center, cluster_std)): - X.append(generator.normal(loc=centers[i], scale=std, - size=(n, n_features))) - y += [i] * n + cum_sum_n_samples = np.cumsum(n_samples_per_center) + X = np.empty(shape=(sum(n_samples_per_center), n_features), dtype=np.float64) + y = np.empty(shape=(sum(n_samples_per_center),), dtype=int) - X = np.concatenate(X) - y = np.array(y) + for i, (n, std) in enumerate(zip(n_samples_per_center, cluster_std)): + start_idx = cum_sum_n_samples[i - 1] if i > 0 else 0 + end_idx = cum_sum_n_samples[i] + X[start_idx:end_idx] = generator.normal( + loc=centers[i], scale=std, size=(n, n_features) + ) + y[start_idx:end_idx] = i if shuffle: - total_n_samples = np.sum(n_samples) - indices = np.arange(total_n_samples) - generator.shuffle(indices) - X = X[indices] - y = y[indices] + X, y = util_shuffle(X, y, random_state=generator) if return_centers: return X, y, centers @@ -906,9 +1160,16 @@ def make_blobs(n_samples=100, n_features=2, *, centers=None, cluster_std=1.0, return X, y -@_deprecate_positional_args -def make_friedman1(n_samples=100, n_features=10, *, noise=0.0, - random_state=None): +@validate_params( + { + "n_samples": [Interval(Integral, 1, None, closed="left")], + "n_features": [Interval(Integral, 5, None, closed="left")], + "noise": [Interval(Real, 0.0, None, closed="left")], + "random_state": ["random_state"], + }, + prefer_skip_nested_validation=True, +) +def make_friedman1(n_samples=100, n_features=10, *, noise=0.0, random_state=None): """Generate the "Friedman #1" regression problem. This dataset is described in Friedman [1] and Breiman [2]. @@ -957,20 +1218,40 @@ def make_friedman1(n_samples=100, n_features=10, *, noise=0.0, .. [2] L. Breiman, "Bagging predictors", Machine Learning 24, pages 123-140, 1996. - """ - if n_features < 5: - raise ValueError("n_features must be at least five.") + Examples + -------- + >>> from sklearn.datasets import make_friedman1 + >>> X, y = make_friedman1(random_state=42) + >>> X.shape + (100, 10) + >>> y.shape + (100,) + >>> list(y[:3]) + [np.float64(16.8), np.float64(5.87), np.float64(9.46)] + """ generator = check_random_state(random_state) - X = generator.rand(n_samples, n_features) - y = 10 * np.sin(np.pi * X[:, 0] * X[:, 1]) + 20 * (X[:, 2] - 0.5) ** 2 \ - + 10 * X[:, 3] + 5 * X[:, 4] + noise * generator.randn(n_samples) + X = generator.uniform(size=(n_samples, n_features)) + y = ( + 10 * np.sin(np.pi * X[:, 0] * X[:, 1]) + + 20 * (X[:, 2] - 0.5) ** 2 + + 10 * X[:, 3] + + 5 * X[:, 4] + + noise * generator.standard_normal(size=(n_samples)) + ) return X, y -@_deprecate_positional_args +@validate_params( + { + "n_samples": [Interval(Integral, 1, None, closed="left")], + "noise": [Interval(Real, 0, None, closed="left")], + "random_state": ["random_state"], + }, + prefer_skip_nested_validation=True, +) def make_friedman2(n_samples=100, *, noise=0.0, random_state=None): """Generate the "Friedman #2" regression problem. @@ -1019,24 +1300,42 @@ def make_friedman2(n_samples=100, *, noise=0.0, random_state=None): .. [2] L. Breiman, "Bagging predictors", Machine Learning 24, pages 123-140, 1996. + + Examples + -------- + >>> from sklearn.datasets import make_friedman2 + >>> X, y = make_friedman2(random_state=42) + >>> X.shape + (100, 4) + >>> y.shape + (100,) + >>> list(y[:3]) + [np.float64(1229.4), np.float64(27.0), np.float64(65.6)] """ generator = check_random_state(random_state) - X = generator.rand(n_samples, 4) + X = generator.uniform(size=(n_samples, 4)) X[:, 0] *= 100 X[:, 1] *= 520 * np.pi X[:, 1] += 40 * np.pi X[:, 3] *= 10 X[:, 3] += 1 - y = (X[:, 0] ** 2 - + (X[:, 1] * X[:, 2] - 1 / (X[:, 1] * X[:, 3])) ** 2) ** 0.5 \ - + noise * generator.randn(n_samples) + y = ( + X[:, 0] ** 2 + (X[:, 1] * X[:, 2] - 1 / (X[:, 1] * X[:, 3])) ** 2 + ) ** 0.5 + noise * generator.standard_normal(size=(n_samples)) return X, y -@_deprecate_positional_args +@validate_params( + { + "n_samples": [Interval(Integral, 1, None, closed="left")], + "noise": [Interval(Real, 0, None, closed="left")], + "random_state": ["random_state"], + }, + prefer_skip_nested_validation=True, +) def make_friedman3(n_samples=100, *, noise=0.0, random_state=None): """Generate the "Friedman #3" regression problem. @@ -1085,25 +1384,52 @@ def make_friedman3(n_samples=100, *, noise=0.0, random_state=None): .. [2] L. Breiman, "Bagging predictors", Machine Learning 24, pages 123-140, 1996. + + Examples + -------- + >>> from sklearn.datasets import make_friedman3 + >>> X, y = make_friedman3(random_state=42) + >>> X.shape + (100, 4) + >>> y.shape + (100,) + >>> list(y[:3]) + [np.float64(1.54), np.float64(0.956), np.float64(0.414)] """ generator = check_random_state(random_state) - X = generator.rand(n_samples, 4) + X = generator.uniform(size=(n_samples, 4)) X[:, 0] *= 100 X[:, 1] *= 520 * np.pi X[:, 1] += 40 * np.pi X[:, 3] *= 10 X[:, 3] += 1 - y = np.arctan((X[:, 1] * X[:, 2] - 1 / (X[:, 1] * X[:, 3])) / X[:, 0]) \ - + noise * generator.randn(n_samples) + y = np.arctan( + (X[:, 1] * X[:, 2] - 1 / (X[:, 1] * X[:, 3])) / X[:, 0] + ) + noise * generator.standard_normal(size=(n_samples)) return X, y -@_deprecate_positional_args -def make_low_rank_matrix(n_samples=100, n_features=100, *, effective_rank=10, - tail_strength=0.5, random_state=None): +@validate_params( + { + "n_samples": [Interval(Integral, 1, None, closed="left")], + "n_features": [Interval(Integral, 1, None, closed="left")], + "effective_rank": [Interval(Integral, 1, None, closed="left")], + "tail_strength": [Interval(Real, 0, 1, closed="both")], + "random_state": ["random_state"], + }, + prefer_skip_nested_validation=True, +) +def make_low_rank_matrix( + n_samples=100, + n_features=100, + *, + effective_rank=10, + tail_strength=0.5, + random_state=None, +): """Generate a mostly low rank matrix with bell-shaped singular values. Most of the variance can be explained by a bell-shaped curve of width @@ -1151,52 +1477,86 @@ def make_low_rank_matrix(n_samples=100, n_features=100, *, effective_rank=10, ------- X : ndarray of shape (n_samples, n_features) The matrix. + + Examples + -------- + >>> from numpy.linalg import svd + >>> from sklearn.datasets import make_low_rank_matrix + >>> X = make_low_rank_matrix( + ... n_samples=50, + ... n_features=25, + ... effective_rank=5, + ... tail_strength=0.01, + ... random_state=0, + ... ) + >>> X.shape + (50, 25) """ generator = check_random_state(random_state) n = min(n_samples, n_features) # Random (ortho normal) vectors - u, _ = linalg.qr(generator.randn(n_samples, n), mode='economic', - check_finite=False) - v, _ = linalg.qr(generator.randn(n_features, n), mode='economic', - check_finite=False) + u, _ = linalg.qr( + generator.standard_normal(size=(n_samples, n)), + mode="economic", + check_finite=False, + ) + v, _ = linalg.qr( + generator.standard_normal(size=(n_features, n)), + mode="economic", + check_finite=False, + ) # Index of the singular values singular_ind = np.arange(n, dtype=np.float64) # Build the singular profile by assembling signal and noise components - low_rank = ((1 - tail_strength) * - np.exp(-1.0 * (singular_ind / effective_rank) ** 2)) + low_rank = (1 - tail_strength) * np.exp(-1.0 * (singular_ind / effective_rank) ** 2) tail = tail_strength * np.exp(-0.1 * singular_ind / effective_rank) s = np.identity(n) * (low_rank + tail) return np.dot(np.dot(u, s), v.T) -@_deprecate_positional_args -def make_sparse_coded_signal(n_samples, *, n_components, n_features, - n_nonzero_coefs, random_state=None): +@validate_params( + { + "n_samples": [Interval(Integral, 1, None, closed="left")], + "n_components": [Interval(Integral, 1, None, closed="left")], + "n_features": [Interval(Integral, 1, None, closed="left")], + "n_nonzero_coefs": [Interval(Integral, 1, None, closed="left")], + "random_state": ["random_state"], + }, + prefer_skip_nested_validation=True, +) +def make_sparse_coded_signal( + n_samples, + *, + n_components, + n_features, + n_nonzero_coefs, + random_state=None, +): """Generate a signal as a sparse combination of dictionary elements. - Returns a matrix Y = DX, such as D is (n_features, n_components), - X is (n_components, n_samples) and each column of X has exactly - n_nonzero_coefs non-zero elements. + Returns matrices `Y`, `D` and `X` such that `Y = XD` where `X` is of shape + `(n_samples, n_components)`, `D` is of shape `(n_components, n_features)`, and + each row of `X` has exactly `n_nonzero_coefs` non-zero elements. Read more in the :ref:`User Guide `. Parameters ---------- n_samples : int - Number of samples to generate + Number of samples to generate. n_components : int - Number of components in the dictionary + Number of components in the dictionary. n_features : int - Number of features of the dataset to generate + Number of features of the dataset to generate. n_nonzero_coefs : int - Number of active (non-zero) coefficients in each sample + Number of active (non-zero) coefficients in each sample. random_state : int, RandomState instance or None, default=None Determines random number generation for dataset creation. Pass an int @@ -1205,22 +1565,38 @@ def make_sparse_coded_signal(n_samples, *, n_components, n_features, Returns ------- - data : ndarray of shape (n_features, n_samples) + data : ndarray of shape (n_samples, n_features) The encoded signal (Y). - dictionary : ndarray of shape (n_features, n_components) + dictionary : ndarray of shape (n_components, n_features) The dictionary with normalized components (D). - code : ndarray of shape (n_components, n_samples) + code : ndarray of shape (n_samples, n_components) The sparse code such that each column of this matrix has exactly n_nonzero_coefs non-zero items (X). + Examples + -------- + >>> from sklearn.datasets import make_sparse_coded_signal + >>> data, dictionary, code = make_sparse_coded_signal( + ... n_samples=50, + ... n_components=100, + ... n_features=10, + ... n_nonzero_coefs=4, + ... random_state=0 + ... ) + >>> data.shape + (50, 10) + >>> dictionary.shape + (100, 10) + >>> code.shape + (50, 100) """ generator = check_random_state(random_state) # generate dictionary - D = generator.randn(n_features, n_components) - D /= np.sqrt(np.sum((D ** 2), axis=0)) + D = generator.standard_normal(size=(n_features, n_components)) + D /= np.sqrt(np.sum((D**2), axis=0)) # generate code X = np.zeros((n_components, n_samples)) @@ -1228,17 +1604,26 @@ def make_sparse_coded_signal(n_samples, *, n_components, n_features, idx = np.arange(n_components) generator.shuffle(idx) idx = idx[:n_nonzero_coefs] - X[idx, i] = generator.randn(n_nonzero_coefs) + X[idx, i] = generator.standard_normal(size=n_nonzero_coefs) # encode signal Y = np.dot(D, X) + # Transpose to have shapes consistent with the rest of the API + Y, D, X = Y.T, D.T, X.T + return map(np.squeeze, (Y, D, X)) -@_deprecate_positional_args -def make_sparse_uncorrelated(n_samples=100, n_features=10, *, - random_state=None): +@validate_params( + { + "n_samples": [Interval(Integral, 1, None, closed="left")], + "n_features": [Interval(Integral, 1, None, closed="left")], + "random_state": ["random_state"], + }, + prefer_skip_nested_validation=True, +) +def make_sparse_uncorrelated(n_samples=100, n_features=10, *, random_state=None): """Generate a random regression problem with sparse uncorrelated design. This dataset is described in Celeux et al [1]. as:: @@ -1277,19 +1662,34 @@ def make_sparse_uncorrelated(n_samples=100, n_features=10, *, .. [1] G. Celeux, M. El Anbari, J.-M. Marin, C. P. Robert, "Regularization in regression: comparing Bayesian and frequentist methods in a poorly informative situation", 2009. + + Examples + -------- + >>> from sklearn.datasets import make_sparse_uncorrelated + >>> X, y = make_sparse_uncorrelated(random_state=0) + >>> X.shape + (100, 10) + >>> y.shape + (100,) """ generator = check_random_state(random_state) X = generator.normal(loc=0, scale=1, size=(n_samples, n_features)) - y = generator.normal(loc=(X[:, 0] + - 2 * X[:, 1] - - 2 * X[:, 2] - - 1.5 * X[:, 3]), scale=np.ones(n_samples)) + y = generator.normal( + loc=(X[:, 0] + 2 * X[:, 1] - 2 * X[:, 2] - 1.5 * X[:, 3]), + scale=np.ones(n_samples), + ) return X, y -@_deprecate_positional_args +@validate_params( + { + "n_dim": [Interval(Integral, 1, None, closed="left")], + "random_state": ["random_state"], + }, + prefer_skip_nested_validation=True, +) def make_spd_matrix(n_dim, *, random_state=None): """Generate a random symmetric, positive-definite matrix. @@ -1312,37 +1712,68 @@ def make_spd_matrix(n_dim, *, random_state=None): See Also -------- - make_sparse_spd_matrix + make_sparse_spd_matrix: Generate a sparse symmetric definite positive matrix. + + Examples + -------- + >>> from sklearn.datasets import make_spd_matrix + >>> make_spd_matrix(n_dim=2, random_state=42) + array([[2.093, 0.346], + [0.346, 0.218]]) """ generator = check_random_state(random_state) - A = generator.rand(n_dim, n_dim) + A = generator.uniform(size=(n_dim, n_dim)) U, _, Vt = linalg.svd(np.dot(A.T, A), check_finite=False) - X = np.dot(np.dot(U, 1.0 + np.diag(generator.rand(n_dim))), Vt) + X = np.dot(np.dot(U, 1.0 + np.diag(generator.uniform(size=n_dim))), Vt) return X -@_deprecate_positional_args -def make_sparse_spd_matrix(dim=1, *, alpha=0.95, norm_diag=False, - smallest_coef=.1, largest_coef=.9, - random_state=None): +@validate_params( + { + "n_dim": [Interval(Integral, 1, None, closed="left")], + "alpha": [Interval(Real, 0, 1, closed="both")], + "norm_diag": ["boolean"], + "smallest_coef": [Interval(Real, 0, 1, closed="both")], + "largest_coef": [Interval(Real, 0, 1, closed="both")], + "sparse_format": [ + StrOptions({"bsr", "coo", "csc", "csr", "dia", "dok", "lil"}), + None, + ], + "random_state": ["random_state"], + }, + prefer_skip_nested_validation=True, +) +def make_sparse_spd_matrix( + n_dim=1, + *, + alpha=0.95, + norm_diag=False, + smallest_coef=0.1, + largest_coef=0.9, + sparse_format=None, + random_state=None, +): """Generate a sparse symmetric definite positive matrix. Read more in the :ref:`User Guide `. Parameters ---------- - dim : int, default=1 + n_dim : int, default=1 The size of the random matrix to generate. + .. versionchanged:: 1.4 + Renamed from ``dim`` to ``n_dim``. + alpha : float, default=0.95 The probability that a coefficient is zero (see notes). Larger values enforce more sparsity. The value should be in the range 0 and 1. norm_diag : bool, default=False Whether to normalize the output matrix to make the leading diagonal - elements all 1 + elements all 1. smallest_coef : float, default=0.1 The value of the smallest coefficient between 0 and 1. @@ -1350,6 +1781,12 @@ def make_sparse_spd_matrix(dim=1, *, alpha=0.95, norm_diag=False, largest_coef : float, default=0.9 The value of the largest coefficient between 0 and 1. + sparse_format : str, default=None + String representing the output sparse format, such as 'csc', 'csr', etc. + If ``None``, return a dense numpy ndarray. + + .. versionadded:: 1.4 + random_state : int, RandomState instance or None, default=None Determines random number generation for dataset creation. Pass an int for reproducible output across multiple function calls. @@ -1357,8 +1794,13 @@ def make_sparse_spd_matrix(dim=1, *, alpha=0.95, norm_diag=False, Returns ------- - prec : sparse matrix of shape (dim, dim) - The generated matrix. + prec : ndarray or sparse matrix of shape (dim, dim) + The generated matrix. If ``sparse_format=None``, this would be an ndarray. + Otherwise, this will be a sparse matrix of the specified format. + + See Also + -------- + make_spd_matrix : Generate a random symmetric, positive-definite matrix. Notes ----- @@ -1366,40 +1808,58 @@ def make_sparse_spd_matrix(dim=1, *, alpha=0.95, norm_diag=False, Thus alpha does not translate directly into the filling fraction of the matrix itself. - See Also + Examples -------- - make_spd_matrix + >>> from sklearn.datasets import make_sparse_spd_matrix + >>> make_sparse_spd_matrix(n_dim=4, norm_diag=False, random_state=42) + array([[1., 0., 0., 0.], + [0., 1., 0., 0.], + [0., 0., 1., 0.], + [0., 0., 0., 1.]]) """ random_state = check_random_state(random_state) - chol = -np.eye(dim) - aux = random_state.rand(dim, dim) - aux[aux < alpha] = 0 - aux[aux > alpha] = (smallest_coef - + (largest_coef - smallest_coef) - * random_state.rand(np.sum(aux > alpha))) - aux = np.tril(aux, k=-1) + chol = -sp.eye(n_dim) + aux = sp.random( + m=n_dim, + n=n_dim, + density=1 - alpha, + data_rvs=lambda x: random_state.uniform( + low=smallest_coef, high=largest_coef, size=x + ), + random_state=random_state, + ) + # We need to avoid "coo" format because it does not support slicing + aux = sp.tril(aux, k=-1, format="csc") # Permute the lines: we don't want to have asymmetries in the final # SPD matrix - permutation = random_state.permutation(dim) + permutation = random_state.permutation(n_dim) aux = aux[permutation].T[permutation] chol += aux - prec = np.dot(chol.T, chol) + prec = chol.T @ chol if norm_diag: # Form the diagonal vector into a row matrix - d = np.diag(prec).reshape(1, prec.shape[0]) - d = 1. / np.sqrt(d) - - prec *= d - prec *= d.T + d = sp.diags(1.0 / np.sqrt(prec.diagonal())) + prec = d @ prec @ d - return prec - - -@_deprecate_positional_args -def make_swiss_roll(n_samples=100, *, noise=0.0, random_state=None): + if sparse_format is None: + return prec.toarray() + else: + return prec.asformat(sparse_format) + + +@validate_params( + { + "n_samples": [Interval(Integral, 1, None, closed="left")], + "noise": [Interval(Real, 0, None, closed="left")], + "random_state": ["random_state"], + "hole": ["boolean"], + }, + prefer_skip_nested_validation=True, +) +def make_swiss_roll(n_samples=100, *, noise=0.0, random_state=None, hole=False): """Generate a swiss roll dataset. Read more in the :ref:`User Guide `. @@ -1407,7 +1867,7 @@ def make_swiss_roll(n_samples=100, *, noise=0.0, random_state=None): Parameters ---------- n_samples : int, default=100 - The number of sample points on the S curve. + The number of sample points on the Swiss Roll. noise : float, default=0.0 The standard deviation of the gaussian noise. @@ -1417,6 +1877,9 @@ def make_swiss_roll(n_samples=100, *, noise=0.0, random_state=None): for reproducible output across multiple function calls. See :term:`Glossary `. + hole : bool, default=False + If True generates the swiss roll with hole dataset. + Returns ------- X : ndarray of shape (n_samples, 3) @@ -1432,26 +1895,52 @@ def make_swiss_roll(n_samples=100, *, noise=0.0, random_state=None): References ---------- - .. [1] S. Marsland, "Machine Learning: An Algorithmic Perspective", - Chapter 10, 2009. - http://seat.massey.ac.nz/personal/s.r.marsland/Code/10/lle.py + .. [1] S. Marsland, "Machine Learning: An Algorithmic Perspective", 2nd edition, + Chapter 6, 2014. + https://homepages.ecs.vuw.ac.nz/~marslast/Code/Ch6/lle.py + + Examples + -------- + >>> from sklearn.datasets import make_swiss_roll + >>> X, t = make_swiss_roll(noise=0.05, random_state=0) + >>> X.shape + (100, 3) + >>> t.shape + (100,) """ generator = check_random_state(random_state) - t = 1.5 * np.pi * (1 + 2 * generator.rand(1, n_samples)) + if not hole: + t = 1.5 * np.pi * (1 + 2 * generator.uniform(size=n_samples)) + y = 21 * generator.uniform(size=n_samples) + else: + corners = np.array( + [[np.pi * (1.5 + i), j * 7] for i in range(3) for j in range(3)] + ) + corners = np.delete(corners, 4, axis=0) + corner_index = generator.choice(8, n_samples) + parameters = generator.uniform(size=(2, n_samples)) * np.array([[np.pi], [7]]) + t, y = corners[corner_index].T + parameters + x = t * np.cos(t) - y = 21 * generator.rand(1, n_samples) z = t * np.sin(t) - X = np.concatenate((x, y, z)) - X += noise * generator.randn(3, n_samples) + X = np.vstack((x, y, z)) + X += noise * generator.standard_normal(size=(3, n_samples)) X = X.T t = np.squeeze(t) return X, t -@_deprecate_positional_args +@validate_params( + { + "n_samples": [Interval(Integral, 1, None, closed="left")], + "noise": [Interval(Real, 0, None, closed="left")], + "random_state": ["random_state"], + }, + prefer_skip_nested_validation=True, +) def make_s_curve(n_samples=100, *, noise=0.0, random_state=None): """Generate an S curve dataset. @@ -1476,28 +1965,53 @@ def make_s_curve(n_samples=100, *, noise=0.0, random_state=None): The points. t : ndarray of shape (n_samples,) - The univariate position of the sample according to the main dimension - of the points in the manifold. + The univariate position of the sample according + to the main dimension of the points in the manifold. + + Examples + -------- + >>> from sklearn.datasets import make_s_curve + >>> X, t = make_s_curve(noise=0.05, random_state=0) + >>> X.shape + (100, 3) + >>> t.shape + (100,) """ generator = check_random_state(random_state) - t = 3 * np.pi * (generator.rand(1, n_samples) - 0.5) - x = np.sin(t) - y = 2.0 * generator.rand(1, n_samples) - z = np.sign(t) * (np.cos(t) - 1) - - X = np.concatenate((x, y, z)) - X += noise * generator.randn(3, n_samples) - X = X.T + t = 3 * np.pi * (generator.uniform(size=(1, n_samples)) - 0.5) + X = np.empty(shape=(n_samples, 3), dtype=np.float64) + X[:, 0] = np.sin(t) + X[:, 1] = 2.0 * generator.uniform(size=n_samples) + X[:, 2] = np.sign(t) * (np.cos(t) - 1) + X += noise * generator.standard_normal(size=(3, n_samples)).T t = np.squeeze(t) return X, t -@_deprecate_positional_args -def make_gaussian_quantiles(*, mean=None, cov=1., n_samples=100, - n_features=2, n_classes=3, - shuffle=True, random_state=None): +@validate_params( + { + "mean": ["array-like", None], + "cov": [Interval(Real, 0, None, closed="left")], + "n_samples": [Interval(Integral, 1, None, closed="left")], + "n_features": [Interval(Integral, 1, None, closed="left")], + "n_classes": [Interval(Integral, 1, None, closed="left")], + "shuffle": ["boolean"], + "random_state": ["random_state"], + }, + prefer_skip_nested_validation=True, +) +def make_gaussian_quantiles( + *, + mean=None, + cov=1.0, + n_samples=100, + n_features=2, + n_classes=3, + shuffle=True, + random_state=None, +): r"""Generate isotropic Gaussian and label samples by quantile. This classification dataset is constructed by taking a multi-dimensional @@ -1509,7 +2023,7 @@ def make_gaussian_quantiles(*, mean=None, cov=1., n_samples=100, Parameters ---------- - mean : ndarray of shape (n_features,), default=None + mean : array-like of shape (n_features,), default=None The mean of the multi-dimensional normal distribution. If None then use the origin (0, 0, ...). @@ -1524,7 +2038,7 @@ def make_gaussian_quantiles(*, mean=None, cov=1., n_samples=100, The number of features for each sample. n_classes : int, default=3 - The number of classes + The number of classes. shuffle : bool, default=True Shuffle the samples. @@ -1550,6 +2064,16 @@ def make_gaussian_quantiles(*, mean=None, cov=1., n_samples=100, ---------- .. [1] J. Zhu, H. Zou, S. Rosset, T. Hastie, "Multi-class AdaBoost", 2009. + Examples + -------- + >>> from sklearn.datasets import make_gaussian_quantiles + >>> X, y = make_gaussian_quantiles(random_state=42) + >>> X.shape + (100, 2) + >>> y.shape + (100,) + >>> list(y[:5]) + [np.int64(2), np.int64(0), np.int64(1), np.int64(0), np.int64(2)] """ if n_samples < n_classes: raise ValueError("n_samples must be at least n_classes") @@ -1562,8 +2086,7 @@ def make_gaussian_quantiles(*, mean=None, cov=1., n_samples=100, mean = np.array(mean) # Build multivariate normal distribution - X = generator.multivariate_normal(mean, cov * np.identity(n_features), - (n_samples,)) + X = generator.multivariate_normal(mean, cov * np.identity(n_features), (n_samples,)) # Sort by distance from origin idx = np.argsort(np.sum((X - mean[np.newaxis, :]) ** 2, axis=1)) @@ -1572,8 +2095,12 @@ def make_gaussian_quantiles(*, mean=None, cov=1., n_samples=100, # Label by quantile step = n_samples // n_classes - y = np.hstack([np.repeat(np.arange(n_classes), step), - np.repeat(n_classes - 1, n_samples - step * n_classes)]) + y = np.hstack( + [ + np.repeat(np.arange(n_classes), step), + np.repeat(n_classes - 1, n_samples - step * n_classes), + ] + ) if shuffle: X, y = util_shuffle(X, y, random_state=generator) @@ -1590,17 +2117,35 @@ def _shuffle(data, random_state=None): return result, row_idx, col_idx -@_deprecate_positional_args -def make_biclusters(shape, n_clusters, *, noise=0.0, minval=10, - maxval=100, shuffle=True, random_state=None): - """Generate an array with constant block diagonal structure for - biclustering. +@validate_params( + { + "shape": [tuple], + "n_clusters": [Interval(Integral, 1, None, closed="left")], + "noise": [Interval(Real, 0, None, closed="left")], + "minval": [Interval(Real, None, None, closed="neither")], + "maxval": [Interval(Real, None, None, closed="neither")], + "shuffle": ["boolean"], + "random_state": ["random_state"], + }, + prefer_skip_nested_validation=True, +) +def make_biclusters( + shape, + n_clusters, + *, + noise=0.0, + minval=10, + maxval=100, + shuffle=True, + random_state=None, +): + """Generate a constant block diagonal structure array for biclustering. Read more in the :ref:`User Guide `. Parameters ---------- - shape : iterable of shape (n_rows, n_cols) + shape : tuple of shape (n_rows, n_cols) The shape of the result. n_clusters : int @@ -1609,10 +2154,10 @@ def make_biclusters(shape, n_clusters, *, noise=0.0, minval=10, noise : float, default=0.0 The standard deviation of the gaussian noise. - minval : int, default=10 + minval : float, default=10 Minimum value of a bicluster. - maxval : int, default=100 + maxval : float, default=100 Maximum value of a bicluster. shuffle : bool, default=True @@ -1634,6 +2179,11 @@ def make_biclusters(shape, n_clusters, *, noise=0.0, minval=10, cols : ndarray of shape (n_clusters, X.shape[1]) The indicators for cluster membership of each column. + See Also + -------- + make_checkerboard: Generate an array with block checkerboard structure for + biclustering. + References ---------- @@ -1642,26 +2192,33 @@ def make_biclusters(shape, n_clusters, *, noise=0.0, minval=10, of the seventh ACM SIGKDD international conference on Knowledge discovery and data mining (pp. 269-274). ACM. - See Also + Examples -------- - make_checkerboard + >>> from sklearn.datasets import make_biclusters + >>> data, rows, cols = make_biclusters( + ... shape=(10, 20), n_clusters=2, random_state=42 + ... ) + >>> data.shape + (10, 20) + >>> rows.shape + (2, 10) + >>> cols.shape + (2, 20) """ generator = check_random_state(random_state) n_rows, n_cols = shape consts = generator.uniform(minval, maxval, n_clusters) # row and column clusters of approximately equal sizes - row_sizes = generator.multinomial(n_rows, - np.repeat(1.0 / n_clusters, - n_clusters)) - col_sizes = generator.multinomial(n_cols, - np.repeat(1.0 / n_clusters, - n_clusters)) - - row_labels = np.hstack(list(np.repeat(val, rep) for val, rep in - zip(range(n_clusters), row_sizes))) - col_labels = np.hstack(list(np.repeat(val, rep) for val, rep in - zip(range(n_clusters), col_sizes))) + row_sizes = generator.multinomial(n_rows, np.repeat(1.0 / n_clusters, n_clusters)) + col_sizes = generator.multinomial(n_cols, np.repeat(1.0 / n_clusters, n_clusters)) + + row_labels = np.hstack( + [np.repeat(val, rep) for val, rep in zip(range(n_clusters), row_sizes)] + ) + col_labels = np.hstack( + [np.repeat(val, rep) for val, rep in zip(range(n_clusters), col_sizes)] + ) result = np.zeros(shape, dtype=np.float64) for i in range(n_clusters): @@ -1682,11 +2239,29 @@ def make_biclusters(shape, n_clusters, *, noise=0.0, minval=10, return result, rows, cols -@_deprecate_positional_args -def make_checkerboard(shape, n_clusters, *, noise=0.0, minval=10, - maxval=100, shuffle=True, random_state=None): - """Generate an array with block checkerboard structure for - biclustering. +@validate_params( + { + "shape": [tuple], + "n_clusters": [Interval(Integral, 1, None, closed="left"), "array-like"], + "noise": [Interval(Real, 0, None, closed="left")], + "minval": [Interval(Real, None, None, closed="neither")], + "maxval": [Interval(Real, None, None, closed="neither")], + "shuffle": ["boolean"], + "random_state": ["random_state"], + }, + prefer_skip_nested_validation=True, +) +def make_checkerboard( + shape, + n_clusters, + *, + noise=0.0, + minval=10, + maxval=100, + shuffle=True, + random_state=None, +): + """Generate an array with block checkerboard structure for biclustering. Read more in the :ref:`User Guide `. @@ -1701,10 +2276,10 @@ def make_checkerboard(shape, n_clusters, *, noise=0.0, minval=10, noise : float, default=0.0 The standard deviation of the gaussian noise. - minval : int, default=10 + minval : float, default=10 Minimum value of a bicluster. - maxval : int, default=100 + maxval : float, default=100 Maximum value of a bicluster. shuffle : bool, default=True @@ -1726,17 +2301,30 @@ def make_checkerboard(shape, n_clusters, *, noise=0.0, minval=10, cols : ndarray of shape (n_clusters, X.shape[1]) The indicators for cluster membership of each column. + See Also + -------- + make_biclusters : Generate an array with constant block diagonal structure + for biclustering. References ---------- - .. [1] Kluger, Y., Basri, R., Chang, J. T., & Gerstein, M. (2003). Spectral biclustering of microarray data: coclustering genes and conditions. Genome research, 13(4), 703-716. - See Also + Examples -------- - make_biclusters + >>> from sklearn.datasets import make_checkerboard + >>> data, rows, columns = make_checkerboard(shape=(300, 300), n_clusters=10, + ... random_state=42) + >>> data.shape + (300, 300) + >>> rows.shape + (100, 300) + >>> columns.shape + (100, 300) + >>> print(rows[0][:5], columns[0][:5]) + [False False False True False] [False False False False False] """ generator = check_random_state(random_state) @@ -1747,17 +2335,19 @@ def make_checkerboard(shape, n_clusters, *, noise=0.0, minval=10, # row and column clusters of approximately equal sizes n_rows, n_cols = shape - row_sizes = generator.multinomial(n_rows, - np.repeat(1.0 / n_row_clusters, - n_row_clusters)) - col_sizes = generator.multinomial(n_cols, - np.repeat(1.0 / n_col_clusters, - n_col_clusters)) - - row_labels = np.hstack(list(np.repeat(val, rep) for val, rep in - zip(range(n_row_clusters), row_sizes))) - col_labels = np.hstack(list(np.repeat(val, rep) for val, rep in - zip(range(n_col_clusters), col_sizes))) + row_sizes = generator.multinomial( + n_rows, np.repeat(1.0 / n_row_clusters, n_row_clusters) + ) + col_sizes = generator.multinomial( + n_cols, np.repeat(1.0 / n_col_clusters, n_col_clusters) + ) + + row_labels = np.hstack( + [np.repeat(val, rep) for val, rep in zip(range(n_row_clusters), row_sizes)] + ) + col_labels = np.hstack( + [np.repeat(val, rep) for val, rep in zip(range(n_col_clusters), col_sizes)] + ) result = np.zeros(shape, dtype=np.float64) for i in range(n_row_clusters): @@ -1773,11 +2363,19 @@ def make_checkerboard(shape, n_clusters, *, noise=0.0, minval=10, row_labels = row_labels[row_idx] col_labels = col_labels[col_idx] - rows = np.vstack([row_labels == label - for label in range(n_row_clusters) - for _ in range(n_col_clusters)]) - cols = np.vstack([col_labels == label - for _ in range(n_row_clusters) - for label in range(n_col_clusters)]) + rows = np.vstack( + [ + row_labels == label + for label in range(n_row_clusters) + for _ in range(n_col_clusters) + ] + ) + cols = np.vstack( + [ + col_labels == label + for _ in range(n_row_clusters) + for label in range(n_col_clusters) + ] + ) return result, rows, cols diff --git a/sklearn/datasets/_species_distributions.py b/sklearn/datasets/_species_distributions.py index bc3fa3bcc7a04..e871949e41312 100644 --- a/sklearn/datasets/_species_distributions.py +++ b/sklearn/datasets/_species_distributions.py @@ -23,51 +23,40 @@ `"Maximum entropy modeling of species geographic distributions" `_ S. J. Phillips, R. P. Anderson, R. E. Schapire - Ecological Modelling, 190:231-259, 2006. - -Notes ------ - -For an example of using this dataset, see -:ref:`examples/applications/plot_species_distribution_modeling.py -`. """ -# Authors: Peter Prettenhofer -# Jake Vanderplas -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +import logging from io import BytesIO -from os import makedirs, remove +from numbers import Integral, Real +from os import PathLike, makedirs, remove from os.path import exists -import logging -import numpy as np - import joblib +import numpy as np -from . import get_data_home -from ._base import _fetch_remote -from ._base import RemoteFileMetadata from ..utils import Bunch -from ..utils.validation import _deprecate_positional_args -from ._base import _pkl_filepath +from ..utils._param_validation import Interval, validate_params +from . import get_data_home +from ._base import RemoteFileMetadata, _fetch_remote, _pkl_filepath # The original data can be found at: # https://biodiversityinformatics.amnh.org/open_source/maxent/samples.zip SAMPLES = RemoteFileMetadata( - filename='samples.zip', - url='https://ndownloader.figshare.com/files/5976075', - checksum=('abb07ad284ac50d9e6d20f1c4211e0fd' - '3c098f7f85955e89d321ee8efe37ac28')) + filename="samples.zip", + url="https://ndownloader.figshare.com/files/5976075", + checksum="abb07ad284ac50d9e6d20f1c4211e0fd3c098f7f85955e89d321ee8efe37ac28", +) # The original data can be found at: # https://biodiversityinformatics.amnh.org/open_source/maxent/coverages.zip COVERAGES = RemoteFileMetadata( - filename='coverages.zip', - url='https://ndownloader.figshare.com/files/5976078', - checksum=('4d862674d72e79d6cee77e63b98651ec' - '7926043ba7d39dcb31329cf3f6073807')) + filename="coverages.zip", + url="https://ndownloader.figshare.com/files/5976078", + checksum="4d862674d72e79d6cee77e63b98651ec7926043ba7d39dcb31329cf3f6073807", +) DATA_ARCHIVE_NAME = "species_coverage.pkz" @@ -85,7 +74,7 @@ def _load_coverage(F, header_length=6, dtype=np.int16): header = dict([make_tuple(line) for line in header]) M = np.loadtxt(F, dtype=dtype) - nodata = int(header[b'NODATA_value']) + nodata = int(header[b"NODATA_value"]) if nodata != -9999: M[nodata] = -9999 return M @@ -104,9 +93,9 @@ def _load_csv(F): rec : np.ndarray record array representing the data """ - names = F.readline().decode('ascii').strip().split(',') + names = F.readline().decode("ascii").strip().split(",") - rec = np.loadtxt(F, skiprows=0, delimiter=',', dtype='a22,f4,f4') + rec = np.loadtxt(F, skiprows=0, delimiter=",", dtype="S22,f4,f4") rec.dtype.names = names return rec @@ -138,23 +127,46 @@ def construct_grids(batch): return (xgrid, ygrid) -@_deprecate_positional_args -def fetch_species_distributions(*, data_home=None, - download_if_missing=True): - """Loader for species distribution dataset from Phillips et. al. (2006) - - Read more in the :ref:`User Guide `. +@validate_params( + { + "data_home": [str, PathLike, None], + "download_if_missing": ["boolean"], + "n_retries": [Interval(Integral, 1, None, closed="left")], + "delay": [Interval(Real, 0.0, None, closed="neither")], + }, + prefer_skip_nested_validation=True, +) +def fetch_species_distributions( + *, + data_home=None, + download_if_missing=True, + n_retries=3, + delay=1.0, +): + """Loader for species distribution dataset from Phillips et. al. (2006). + + Read more in the :ref:`User Guide `. Parameters ---------- - data_home : str, default=None + data_home : str or path-like, default=None Specify another download and cache folder for the datasets. By default all scikit-learn data is stored in '~/scikit_learn_data' subfolders. download_if_missing : bool, default=True - If False, raise a IOError if the data is not locally available + If False, raise an OSError if the data is not locally available instead of trying to download the data from the source site. + n_retries : int, default=3 + Number of retries when HTTP errors are encountered. + + .. versionadded:: 1.5 + + delay : float, default=1.0 + Number of seconds between retries. + + .. versionadded:: 1.5 + Returns ------- data : :class:`~sklearn.utils.Bunch` @@ -180,14 +192,6 @@ def fetch_species_distributions(*, data_home=None, grid_size : float The spacing between points of the grid, in degrees - References - ---------- - - * `"Maximum entropy modeling of species geographic distributions" - `_ - S. J. Phillips, R. P. Anderson, R. E. Schapire - Ecological Modelling, - 190:231-259, 2006. - Notes ----- @@ -205,9 +209,28 @@ def fetch_species_distributions(*, data_home=None, also known as the Forest Small Rice Rat, a rodent that lives in Peru, Colombia, Ecuador, Peru, and Venezuela. - - For an example of using this dataset with scikit-learn, see - :ref:`examples/applications/plot_species_distribution_modeling.py - `. + References + ---------- + + * `"Maximum entropy modeling of species geographic distributions" + `_ + S. J. Phillips, R. P. Anderson, R. E. Schapire - Ecological Modelling, + 190:231-259, 2006. + + Examples + -------- + >>> from sklearn.datasets import fetch_species_distributions + >>> species = fetch_species_distributions() + >>> species.train[:5] + array([(b'microryzomys_minutus', -64.7 , -17.85 ), + (b'microryzomys_minutus', -67.8333, -16.3333), + (b'microryzomys_minutus', -67.8833, -16.3 ), + (b'microryzomys_minutus', -67.8 , -16.2667), + (b'microryzomys_minutus', -67.9833, -15.9 )], + dtype=[('species', 'S22'), ('dd long', ' -# Lars Buitinck -# Olivier Grisel -# License: BSD 3 clause -# -# cython: boundscheck=False, wraparound=False +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import array from cpython cimport array cimport cython from libc.string cimport strchr -cimport numpy as np import numpy as np -import scipy.sparse as sp - -np.import_array() cdef bytes COMMA = u','.encode('ascii') @@ -118,3 +110,143 @@ def _load_svmlight_file(f, dtype, bint multilabel, bint zero_based, break return (dtype, data, indices, indptr, labels, query) + + +# Two fused types are defined to be able to +# use all possible combinations of parameters. +ctypedef fused int_or_float: + cython.integral + cython.floating + signed long long + +ctypedef fused double_or_longlong: + double + signed long long + +ctypedef fused int_or_longlong: + cython.integral + signed long long + + +def get_dense_row_string( + const int_or_float[:, :] X, + Py_ssize_t[:] x_inds, + double_or_longlong[:] x_vals, + Py_ssize_t row, + str value_pattern, + bint one_based, +): + cdef: + Py_ssize_t row_length = X.shape[1] + Py_ssize_t x_nz_used = 0 + Py_ssize_t k + int_or_float val + + for k in range(row_length): + val = X[row, k] + if val == 0: + continue + x_inds[x_nz_used] = k + x_vals[x_nz_used] = val + x_nz_used += 1 + + reprs = [ + value_pattern % (x_inds[i] + one_based, x_vals[i]) + for i in range(x_nz_used) + ] + + return " ".join(reprs) + + +def get_sparse_row_string( + int_or_float[:] X_data, + int[:] X_indptr, + int[:] X_indices, + Py_ssize_t row, + str value_pattern, + bint one_based, +): + cdef: + Py_ssize_t row_start = X_indptr[row] + Py_ssize_t row_end = X_indptr[row+1] + + reprs = [ + value_pattern % (X_indices[i] + one_based, X_data[i]) + for i in range(row_start, row_end) + ] + + return " ".join(reprs) + + +def _dump_svmlight_file( + X, + y, + f, + bint multilabel, + bint one_based, + int_or_longlong[:] query_id, + bint X_is_sp, + bint y_is_sp, +): + cdef bint X_is_integral + cdef bint query_id_is_not_empty = query_id.size > 0 + X_is_integral = X.dtype.kind == "i" + if X_is_integral: + value_pattern = "%d:%d" + else: + value_pattern = "%d:%.16g" + if y.dtype.kind == "i": + label_pattern = "%d" + else: + label_pattern = "%.16g" + + line_pattern = "%s" + if query_id_is_not_empty: + line_pattern += " qid:%d" + line_pattern += " %s\n" + + cdef: + Py_ssize_t num_labels = y.shape[1] + Py_ssize_t x_len = X.shape[0] + Py_ssize_t row_length = X.shape[1] + Py_ssize_t i + Py_ssize_t j + Py_ssize_t col_start + Py_ssize_t col_end + Py_ssize_t[:] x_inds = np.empty(row_length, dtype=np.intp) + signed long long[:] x_vals_int + double[:] x_vals_float + + if not X_is_sp: + if X_is_integral: + x_vals_int = np.zeros(row_length, dtype=np.longlong) + else: + x_vals_float = np.zeros(row_length, dtype=np.float64) + + for i in range(x_len): + if not X_is_sp: + if X_is_integral: + s = get_dense_row_string(X, x_inds, x_vals_int, i, value_pattern, one_based) + else: + s = get_dense_row_string(X, x_inds, x_vals_float, i, value_pattern, one_based) + else: + s = get_sparse_row_string(X.data, X.indptr, X.indices, i, value_pattern, one_based) + if multilabel: + if y_is_sp: + col_start = y.indptr[i] + col_end = y.indptr[i+1] + labels_str = ','.join(tuple(label_pattern % y.indices[j] for j in range(col_start, col_end) if y.data[j] != 0)) + else: + labels_str = ','.join(label_pattern % j for j in range(num_labels) if y[i, j] != 0) + else: + if y_is_sp: + labels_str = label_pattern % y.data[i] + else: + labels_str = label_pattern % y[i, 0] + + if query_id_is_not_empty: + feat = (labels_str, query_id[i], s) + else: + feat = (labels_str, s) + + f.write((line_pattern % feat).encode("utf-8")) diff --git a/sklearn/datasets/_svmlight_format_io.py b/sklearn/datasets/_svmlight_format_io.py index 8997624da0755..e3a833efb86c0 100644 --- a/sklearn/datasets/_svmlight_format_io.py +++ b/sklearn/datasets/_svmlight_format_io.py @@ -10,39 +10,55 @@ libsvm command line programs. """ -# Authors: Mathieu Blondel -# Lars Buitinck -# Olivier Grisel -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -from contextlib import closing -import io import os.path +from contextlib import closing +from numbers import Integral import numpy as np import scipy.sparse as sp from .. import __version__ - -from ..utils import check_array, IS_PYPY -from ..utils.validation import _deprecate_positional_args - -if not IS_PYPY: - from ._svmlight_format_fast import _load_svmlight_file -else: - def _load_svmlight_file(*args, **kwargs): - raise NotImplementedError( - 'load_svmlight_file is currently not ' - 'compatible with PyPy (see ' - 'https://github.com/scikit-learn/scikit-learn/issues/11543 ' - 'for the status updates).') - - -@_deprecate_positional_args -def load_svmlight_file(f, *, n_features=None, dtype=np.float64, - multilabel=False, zero_based="auto", query_id=False, - offset=0, length=-1): - """Load datasets in the svmlight / libsvm format into sparse CSR matrix +from ..utils import check_array +from ..utils._param_validation import HasMethods, Interval, StrOptions, validate_params +from ._svmlight_format_fast import ( + _dump_svmlight_file, + _load_svmlight_file, +) + + +@validate_params( + { + "f": [ + str, + Interval(Integral, 0, None, closed="left"), + os.PathLike, + HasMethods("read"), + ], + "n_features": [Interval(Integral, 1, None, closed="left"), None], + "dtype": "no_validation", # delegate validation to numpy + "multilabel": ["boolean"], + "zero_based": ["boolean", StrOptions({"auto"})], + "query_id": ["boolean"], + "offset": [Interval(Integral, 0, None, closed="left")], + "length": [Integral], + }, + prefer_skip_nested_validation=True, +) +def load_svmlight_file( + f, + *, + n_features=None, + dtype=np.float64, + multilabel=False, + zero_based="auto", + query_id=False, + offset=0, + length=-1, +): + """Load datasets in the svmlight / libsvm format into sparse CSR matrix. This format is a text-based format, with one sample per line. It does not store zero valued features hence is suitable for sparse dataset. @@ -53,8 +69,8 @@ def load_svmlight_file(f, *, n_features=None, dtype=np.float64, This format is used as the default format for both svmlight and the libsvm command line programs. - Parsing a text based source can be expensive. When working on - repeatedly on the same dataset, it is recommended to wrap this + Parsing a text based source can be expensive. When repeatedly + working on the same dataset, it is recommended to wrap this loader with joblib.Memory.cache to store a memmapped backup of the CSR results of the first call and benefit from the near instantaneous loading of memmapped structures for the subsequent calls. @@ -69,17 +85,19 @@ def load_svmlight_file(f, *, n_features=None, dtype=np.float64, This implementation is written in Cython and is reasonably fast. However, a faster API-compatible loader is also available at: - - https://github.com/mblondel/svmlight-loader + https://github.com/mblondel/svmlight-loader Parameters ---------- - f : str, file-like or int + f : str, path-like, file-like or int (Path to) a file to load. If a path ends in ".gz" or ".bz2", it will be uncompressed on the fly. If an integer is passed, it is assumed to be a file descriptor. A file-like or file descriptor will not be closed by this function. A file-like object must be opened in binary mode. + .. versionchanged:: 1.2 + Path-like objects are now accepted. + n_features : int, default=None The number of features to use. If None, it will be inferred. This argument is useful to load several files that are subsets of a @@ -95,7 +113,7 @@ def load_svmlight_file(f, *, n_features=None, dtype=np.float64, multilabel : bool, default=False Samples may have several labels each (see - https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multilabel.html) + https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multilabel.html). zero_based : bool or "auto", default="auto" Whether column indices in f are zero-based (True) or one-based @@ -124,12 +142,14 @@ def load_svmlight_file(f, *, n_features=None, dtype=np.float64, Returns ------- X : scipy.sparse matrix of shape (n_samples, n_features) + The data matrix. - y : ndarray of shape (n_samples,), or, in the multilabel a list of - tuples of length n_samples. + y : ndarray of shape (n_samples,), or a list of tuples of length n_samples + The target. It is a list of tuples when ``multilabel=True``, else a + ndarray. query_id : array of shape (n_samples,) - query_id for each sample. Only returned when query_id is set to + The query_id for each sample. Only returned when query_id is set to True. See Also @@ -142,7 +162,7 @@ def load_svmlight_file(f, *, n_features=None, dtype=np.float64, To use joblib.Memory to cache the svmlight file:: from joblib import Memory - from .datasets import load_svmlight_file + from sklearn.datasets import load_svmlight_file mem = Memory("./mycache") @mem.cache @@ -152,61 +172,95 @@ def get_data(): X, y = get_data() """ - return tuple(load_svmlight_files([f], n_features=n_features, - dtype=dtype, - multilabel=multilabel, - zero_based=zero_based, - query_id=query_id, - offset=offset, - length=length)) + return tuple( + load_svmlight_files( + [f], + n_features=n_features, + dtype=dtype, + multilabel=multilabel, + zero_based=zero_based, + query_id=query_id, + offset=offset, + length=length, + ) + ) def _gen_open(f): if isinstance(f, int): # file descriptor - return io.open(f, "rb", closefd=False) + return open(f, "rb", closefd=False) + elif isinstance(f, os.PathLike): + f = os.fspath(f) elif not isinstance(f, str): - raise TypeError("expected {str, int, file-like}, got %s" % type(f)) + raise TypeError("expected {str, int, path-like, file-like}, got %s" % type(f)) _, ext = os.path.splitext(f) if ext == ".gz": import gzip + return gzip.open(f, "rb") elif ext == ".bz2": from bz2 import BZ2File + return BZ2File(f, "rb") else: return open(f, "rb") -def _open_and_load(f, dtype, multilabel, zero_based, query_id, - offset=0, length=-1): +def _open_and_load(f, dtype, multilabel, zero_based, query_id, offset=0, length=-1): if hasattr(f, "read"): - actual_dtype, data, ind, indptr, labels, query = \ - _load_svmlight_file(f, dtype, multilabel, zero_based, query_id, - offset, length) + actual_dtype, data, ind, indptr, labels, query = _load_svmlight_file( + f, dtype, multilabel, zero_based, query_id, offset, length + ) else: with closing(_gen_open(f)) as f: - actual_dtype, data, ind, indptr, labels, query = \ - _load_svmlight_file(f, dtype, multilabel, zero_based, query_id, - offset, length) + actual_dtype, data, ind, indptr, labels, query = _load_svmlight_file( + f, dtype, multilabel, zero_based, query_id, offset, length + ) # convert from array.array, give data the right dtype if not multilabel: labels = np.frombuffer(labels, np.float64) data = np.frombuffer(data, actual_dtype) indices = np.frombuffer(ind, np.longlong) - indptr = np.frombuffer(indptr, dtype=np.longlong) # never empty + indptr = np.frombuffer(indptr, dtype=np.longlong) # never empty query = np.frombuffer(query, np.int64) - data = np.asarray(data, dtype=dtype) # no-op for float{32,64} + data = np.asarray(data, dtype=dtype) # no-op for float{32,64} return data, indices, indptr, labels, query -@_deprecate_positional_args -def load_svmlight_files(files, *, n_features=None, dtype=np.float64, - multilabel=False, zero_based="auto", query_id=False, - offset=0, length=-1): - """Load dataset from multiple files in SVMlight format +@validate_params( + { + "files": [ + "array-like", + str, + os.PathLike, + HasMethods("read"), + Interval(Integral, 0, None, closed="left"), + ], + "n_features": [Interval(Integral, 1, None, closed="left"), None], + "dtype": "no_validation", # delegate validation to numpy + "multilabel": ["boolean"], + "zero_based": ["boolean", StrOptions({"auto"})], + "query_id": ["boolean"], + "offset": [Interval(Integral, 0, None, closed="left")], + "length": [Integral], + }, + prefer_skip_nested_validation=True, +) +def load_svmlight_files( + files, + *, + n_features=None, + dtype=np.float64, + multilabel=False, + zero_based="auto", + query_id=False, + offset=0, + length=-1, +): + """Load dataset from multiple files in SVMlight format. This function is equivalent to mapping load_svmlight_file over a list of files, except that the results are concatenated into a single, flat list @@ -223,13 +277,16 @@ def load_svmlight_files(files, *, n_features=None, dtype=np.float64, Parameters ---------- - files : array-like, dtype=str, file-like or int + files : array-like, dtype=str, path-like, file-like or int (Paths of) files to load. If a path ends in ".gz" or ".bz2", it will be uncompressed on the fly. If an integer is passed, it is assumed to be a file descriptor. File-likes and file descriptors will not be closed by this function. File-like objects must be opened in binary mode. + .. versionchanged:: 1.2 + Path-like objects are now accepted. + n_features : int, default=None The number of features to use. If None, it will be inferred from the maximum column index occurring in any of the files. @@ -244,7 +301,7 @@ def load_svmlight_files(files, *, n_features=None, dtype=np.float64, multilabel : bool, default=False Samples may have several labels each (see - https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multilabel.html) + https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multilabel.html). zero_based : bool or "auto", default="auto" Whether column indices in f are zero-based (True) or one-based @@ -272,12 +329,15 @@ def load_svmlight_files(files, *, n_features=None, dtype=np.float64, Returns ------- - [X1, y1, ..., Xn, yn] - where each (Xi, yi) pair is the result from load_svmlight_file(files[i]). + [X1, y1, ..., Xn, yn] or [X1, y1, q1, ..., Xn, yn, qn]: list of arrays + Each (Xi, yi) pair is the result from load_svmlight_file(files[i]). + If query_id is set to True, this will return instead (Xi, yi, qi) + triplets. - If query_id is set to True, this will return instead [X1, y1, q1, - ..., Xn, yn, qn] where (Xi, yi, qi) is the result from - load_svmlight_file(files[i]) + See Also + -------- + load_svmlight_file: Similar function for loading a single file in this + format. Notes ----- @@ -286,9 +346,22 @@ def load_svmlight_files(files, *, n_features=None, dtype=np.float64, number of features (X_train.shape[1] == X_test.shape[1]). This may not be the case if you load the files individually with load_svmlight_file. - See Also + Examples -------- - load_svmlight_file + To use joblib.Memory to cache the svmlight file:: + + from joblib import Memory + from sklearn.datasets import load_svmlight_file + mem = Memory("./mycache") + + @mem.cache + def get_data(): + data_train, target_train, data_test, target_test = load_svmlight_files( + ["svmlight_file_train", "svmlight_file_test"] + ) + return data_train, target_train, data_test, target_test + + X_train, y_train, X_test, y_test = get_data() """ if (offset != 0 or length > 0) and zero_based == "auto": # disable heuristic search to avoid getting inconsistent results on @@ -296,16 +369,24 @@ def load_svmlight_files(files, *, n_features=None, dtype=np.float64, zero_based = True if (offset != 0 or length > 0) and n_features is None: - raise ValueError( - "n_features is required when offset or length is specified.") - - r = [_open_and_load(f, dtype, multilabel, bool(zero_based), bool(query_id), - offset=offset, length=length) - for f in files] + raise ValueError("n_features is required when offset or length is specified.") + + r = [ + _open_and_load( + f, + dtype, + multilabel, + bool(zero_based), + bool(query_id), + offset=offset, + length=length, + ) + for f in files + ] - if (zero_based is False or - zero_based == "auto" and all(len(tmp[1]) and np.min(tmp[1]) > 0 - for tmp in r)): + if zero_based is False or ( + zero_based == "auto" and all(len(tmp[1]) and np.min(tmp[1]) > 0 for tmp in r) + ): for _, indices, _, _, _ in r: indices -= 1 @@ -314,9 +395,11 @@ def load_svmlight_files(files, *, n_features=None, dtype=np.float64, if n_features is None: n_features = n_f elif n_features < n_f: - raise ValueError("n_features was set to {}," - " but input file contains {} features" - .format(n_features, n_f)) + raise ValueError( + "n_features was set to {}, but input file contains {} features".format( + n_features, n_f + ) + ) result = [] for data, indices, indptr, y, query_values in r: @@ -331,66 +414,56 @@ def load_svmlight_files(files, *, n_features=None, dtype=np.float64, def _dump_svmlight(X, y, f, multilabel, one_based, comment, query_id): - X_is_sp = int(hasattr(X, "tocsr")) - y_is_sp = int(hasattr(y, "tocsr")) - if X.dtype.kind == 'i': - value_pattern = "%d:%d" - else: - value_pattern = "%d:%.16g" - - if y.dtype.kind == 'i': - label_pattern = "%d" - else: - label_pattern = "%.16g" - - line_pattern = "%s" - if query_id is not None: - line_pattern += " qid:%d" - line_pattern += " %s\n" - if comment: - f.write(("# Generated by dump_svmlight_file from scikit-learn %s\n" - % __version__).encode()) - f.write(("# Column indices are %s-based\n" - % ["zero", "one"][one_based]).encode()) + f.write( + ( + "# Generated by dump_svmlight_file from scikit-learn %s\n" % __version__ + ).encode() + ) + f.write( + ("# Column indices are %s-based\n" % ["zero", "one"][one_based]).encode() + ) f.write(b"#\n") f.writelines(b"# %s\n" % line for line in comment.splitlines()) - - for i in range(X.shape[0]): - if X_is_sp: - span = slice(X.indptr[i], X.indptr[i + 1]) - row = zip(X.indices[span], X.data[span]) - else: - nz = X[i] != 0 - row = zip(np.where(nz)[0], X[i, nz]) - - s = " ".join(value_pattern % (j + one_based, x) for j, x in row) - - if multilabel: - if y_is_sp: - nz_labels = y[i].nonzero()[1] - else: - nz_labels = np.where(y[i] != 0)[0] - labels_str = ",".join(label_pattern % j for j in nz_labels) - else: - if y_is_sp: - labels_str = label_pattern % y.data[i] - else: - labels_str = label_pattern % y[i] - - if query_id is not None: - feat = (labels_str, query_id[i], s) - else: - feat = (labels_str, s) - - f.write((line_pattern % feat).encode('ascii')) - - -@_deprecate_positional_args -def dump_svmlight_file(X, y, f, *, zero_based=True, comment=None, - query_id=None, - multilabel=False): + X_is_sp = sp.issparse(X) + y_is_sp = sp.issparse(y) + if not multilabel and not y_is_sp: + y = y[:, np.newaxis] + _dump_svmlight_file( + X, + y, + f, + multilabel, + one_based, + query_id, + X_is_sp, + y_is_sp, + ) + + +@validate_params( + { + "X": ["array-like", "sparse matrix"], + "y": ["array-like", "sparse matrix"], + "f": [str, HasMethods(["write"])], + "zero_based": ["boolean"], + "comment": [str, bytes, None], + "query_id": ["array-like", None], + "multilabel": ["boolean"], + }, + prefer_skip_nested_validation=True, +) +def dump_svmlight_file( + X, + y, + f, + *, + zero_based=True, + comment=None, + query_id=None, + multilabel=False, +): """Dump the dataset in svmlight / libsvm file format. This format is a text-based format, with one sample per line. It does @@ -402,24 +475,24 @@ def dump_svmlight_file(X, y, f, *, zero_based=True, comment=None, Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) - Training vectors, where n_samples is the number of samples and - n_features is the number of features. + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. - y : {array-like, sparse matrix}, shape = [n_samples (, n_labels)] + y : {array-like, sparse matrix}, shape = (n_samples,) or (n_samples, n_labels) Target values. Class labels must be an integer or float, or array-like objects of integer or float for multilabel classifications. - f : string or file-like in binary mode + f : str or file-like in binary mode If string, specifies the path that will contain the data. If file-like, data will be written to f. f should be opened in binary mode. - zero_based : boolean, default=True + zero_based : bool, default=True Whether column indices should be written zero-based (True) or one-based (False). - comment : string, default=None + comment : str or bytes, default=None Comment to insert at the top of the file. This should be either a Unicode string, which will be encoded as UTF-8, or an ASCII byte string. @@ -431,18 +504,24 @@ def dump_svmlight_file(X, y, f, *, zero_based=True, comment=None, Array containing pairwise preference constraints (qid in svmlight format). - multilabel : boolean, default=False + multilabel : bool, default=False Samples may have several labels each (see - https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multilabel.html) + https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multilabel.html). .. versionadded:: 0.17 - parameter *multilabel* to support multilabel datasets. + parameter `multilabel` to support multilabel datasets. + + Examples + -------- + >>> from sklearn.datasets import dump_svmlight_file, make_classification + >>> X, y = make_classification(random_state=0) + >>> output_file = "my_dataset.svmlight" + >>> dump_svmlight_file(X, y, output_file) # doctest: +SKIP """ if comment is not None: # Convert comment string to list of lines in UTF-8. # If a byte string is passed, then check whether it's ASCII; # if a user wants to get fancy, they'll have to decode themselves. - # Avoid mention of str and unicode types for Python 3.x compat. if isinstance(comment, bytes): comment.decode("ascii") # just for the exception else: @@ -450,21 +529,21 @@ def dump_svmlight_file(X, y, f, *, zero_based=True, comment=None, if b"\0" in comment: raise ValueError("comment string contains NUL byte") - yval = check_array(y, accept_sparse='csr', ensure_2d=False) + yval = check_array(y, accept_sparse="csr", ensure_2d=False) if sp.issparse(yval): if yval.shape[1] != 1 and not multilabel: - raise ValueError("expected y of shape (n_samples, 1)," - " got %r" % (yval.shape,)) + raise ValueError( + "expected y of shape (n_samples, 1), got %r" % (yval.shape,) + ) else: if yval.ndim != 1 and not multilabel: - raise ValueError("expected y of shape (n_samples,), got %r" - % (yval.shape,)) + raise ValueError("expected y of shape (n_samples,), got %r" % (yval.shape,)) - Xval = check_array(X, accept_sparse='csr') + Xval = check_array(X, accept_sparse="csr") if Xval.shape[0] != yval.shape[0]: raise ValueError( - "X.shape[0] and y.shape[0] should be the same, got" - " %r and %r instead." % (Xval.shape[0], yval.shape[0]) + "X.shape[0] and y.shape[0] should be the same, got %r and %r instead." + % (Xval.shape[0], yval.shape[0]) ) # We had some issues with CSR matrices with unsorted indices (e.g. #1501), @@ -484,11 +563,18 @@ def dump_svmlight_file(X, y, f, *, zero_based=True, comment=None, if hasattr(X, "sort_indices"): X.sort_indices() - if query_id is not None: + if query_id is None: + # NOTE: query_id is passed to Cython functions using a fused type on query_id. + # Yet as of Cython>=3.0, memory views can't be None otherwise the runtime + # would not known which concrete implementation to dispatch the Python call to. + # TODO: simplify interfaces and implementations in _svmlight_format_fast.pyx. + query_id = np.array([], dtype=np.int32) + else: query_id = np.asarray(query_id) if query_id.shape[0] != y.shape[0]: - raise ValueError("expected query_id of shape (n_samples,), got %r" - % (query_id.shape,)) + raise ValueError( + "expected query_id of shape (n_samples,), got %r" % (query_id.shape,) + ) one_based = not zero_based diff --git a/sklearn/datasets/_twenty_newsgroups.py b/sklearn/datasets/_twenty_newsgroups.py index cb181d2108403..62db8c5cbdc8e 100644 --- a/sklearn/datasets/_twenty_newsgroups.py +++ b/sklearn/datasets/_twenty_newsgroups.py @@ -21,68 +21,81 @@ test sets. The compressed dataset size is around 14 Mb compressed. Once uncompressed the train set is 52 MB and the test set is 34 MB. """ -# Copyright (c) 2011 Olivier Grisel -# License: BSD 3 clause -import os -from os.path import dirname, join +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import codecs import logging -import tarfile +import os import pickle -import shutil import re -import codecs +import shutil +import tarfile +from contextlib import suppress +from numbers import Integral, Real +import joblib import numpy as np import scipy.sparse as sp -import joblib -from . import get_data_home -from . import load_files -from ._base import _convert_data_dataframe -from ._base import _pkl_filepath -from ._base import _fetch_remote -from ._base import RemoteFileMetadata -from ..feature_extraction.text import CountVectorizer from .. import preprocessing -from ..utils import check_random_state, Bunch -from ..utils.validation import _deprecate_positional_args +from ..feature_extraction.text import CountVectorizer +from ..utils import Bunch, check_random_state +from ..utils._param_validation import Interval, StrOptions, validate_params +from . import get_data_home, load_files +from ._base import ( + RemoteFileMetadata, + _convert_data_dataframe, + _fetch_remote, + _pkl_filepath, + load_descr, +) logger = logging.getLogger(__name__) # The original data can be found at: # https://people.csail.mit.edu/jrennie/20Newsgroups/20news-bydate.tar.gz ARCHIVE = RemoteFileMetadata( - filename='20news-bydate.tar.gz', - url='https://ndownloader.figshare.com/files/5975967', - checksum=('8f1b2514ca22a5ade8fbb9cfa5727df9' - '5fa587f4c87b786e15c759fa66d95610')) + filename="20news-bydate.tar.gz", + url="https://ndownloader.figshare.com/files/5975967", + checksum="8f1b2514ca22a5ade8fbb9cfa5727df95fa587f4c87b786e15c759fa66d95610", +) CACHE_NAME = "20news-bydate.pkz" TRAIN_FOLDER = "20news-bydate-train" TEST_FOLDER = "20news-bydate-test" -def _download_20newsgroups(target_dir, cache_path): +def _download_20newsgroups(target_dir, cache_path, n_retries, delay): """Download the 20 newsgroups data and stored it as a zipped pickle.""" train_path = os.path.join(target_dir, TRAIN_FOLDER) test_path = os.path.join(target_dir, TEST_FOLDER) - if not os.path.exists(target_dir): - os.makedirs(target_dir) + os.makedirs(target_dir, exist_ok=True) logger.info("Downloading dataset from %s (14 MB)", ARCHIVE.url) - archive_path = _fetch_remote(ARCHIVE, dirname=target_dir) + archive_path = _fetch_remote( + ARCHIVE, dirname=target_dir, n_retries=n_retries, delay=delay + ) logger.debug("Decompressing %s", archive_path) - tarfile.open(archive_path, "r:gz").extractall(path=target_dir) - os.remove(archive_path) + with tarfile.open(archive_path, "r:gz") as fp: + # Use filter="data" to prevent the most dangerous security issues. + # For more details, see + # https://docs.python.org/3.9/library/tarfile.html#tarfile.TarFile.extractall + fp.extractall(path=target_dir, filter="data") + + with suppress(FileNotFoundError): + os.remove(archive_path) # Store a zipped pickle - cache = dict(train=load_files(train_path, encoding='latin1'), - test=load_files(test_path, encoding='latin1')) - compressed_content = codecs.encode(pickle.dumps(cache), 'zlib_codec') - with open(cache_path, 'wb') as f: + cache = dict( + train=load_files(train_path, encoding="latin1"), + test=load_files(test_path, encoding="latin1"), + ) + compressed_content = codecs.encode(pickle.dumps(cache), "zlib_codec") + with open(cache_path, "wb") as f: f.write(compressed_content) shutil.rmtree(target_dir) @@ -99,12 +112,13 @@ def strip_newsgroup_header(text): text : str The text from which to remove the signature block. """ - _before, _blankline, after = text.partition('\n\n') + _before, _blankline, after = text.partition("\n\n") return after -_QUOTE_RE = re.compile(r'(writes in|writes:|wrote:|says:|said:' - r'|^In article|^Quoted from|^\||^>)') +_QUOTE_RE = re.compile( + r"(writes in|writes:|wrote:|says:|said:|^In article|^Quoted from|^\||^>)" +) def strip_newsgroup_quoting(text): @@ -118,9 +132,8 @@ def strip_newsgroup_quoting(text): text : str The text from which to remove the signature block. """ - good_lines = [line for line in text.split('\n') - if not _QUOTE_RE.search(line)] - return '\n'.join(good_lines) + good_lines = [line for line in text.split("\n") if not _QUOTE_RE.search(line)] + return "\n".join(good_lines) def strip_newsgroup_footer(text): @@ -136,23 +149,46 @@ def strip_newsgroup_footer(text): text : str The text from which to remove the signature block. """ - lines = text.strip().split('\n') + lines = text.strip().split("\n") for line_num in range(len(lines) - 1, -1, -1): line = lines[line_num] - if line.strip().strip('-') == '': + if line.strip().strip("-") == "": break if line_num > 0: - return '\n'.join(lines[:line_num]) + return "\n".join(lines[:line_num]) else: return text -@_deprecate_positional_args -def fetch_20newsgroups(*, data_home=None, subset='train', categories=None, - shuffle=True, random_state=42, - remove=(), - download_if_missing=True, return_X_y=False): +@validate_params( + { + "data_home": [str, os.PathLike, None], + "subset": [StrOptions({"train", "test", "all"})], + "categories": ["array-like", None], + "shuffle": ["boolean"], + "random_state": ["random_state"], + "remove": [tuple], + "download_if_missing": ["boolean"], + "return_X_y": ["boolean"], + "n_retries": [Interval(Integral, 1, None, closed="left")], + "delay": [Interval(Real, 0.0, None, closed="neither")], + }, + prefer_skip_nested_validation=True, +) +def fetch_20newsgroups( + *, + data_home=None, + subset="train", + categories=None, + shuffle=True, + random_state=42, + remove=(), + download_if_missing=True, + return_X_y=False, + n_retries=3, + delay=1.0, +): """Load the filenames and data from the 20 newsgroups dataset \ (classification). @@ -169,7 +205,7 @@ def fetch_20newsgroups(*, data_home=None, subset='train', categories=None, Parameters ---------- - data_home : str, default=None + data_home : str or path-like, default=None Specify a download and cache folder for the datasets. If None, all scikit-learn data is stored in '~/scikit_learn_data' subfolders. @@ -177,7 +213,7 @@ def fetch_20newsgroups(*, data_home=None, subset='train', categories=None, Select the dataset to load: 'train' for the training set, 'test' for the test set, 'all' for both, with shuffled ordering. - categories : array-like, dtype=str or unicode, default=None + categories : array-like, dtype=str, default=None If None (default), load all the categories. If not None, list of category names to load (other categories ignored). @@ -187,7 +223,7 @@ def fetch_20newsgroups(*, data_home=None, subset='train', categories=None, make the assumption that the samples are independent and identically distributed (i.i.d.), such as stochastic gradient descent. - random_state : int, RandomState instance or None, default=None + random_state : int, RandomState instance or None, default=42 Determines random number generation for dataset shuffling. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. @@ -206,7 +242,7 @@ def fetch_20newsgroups(*, data_home=None, subset='train', categories=None, correct. download_if_missing : bool, default=True - If False, raise an IOError if the data is not locally available + If False, raise an OSError if the data is not locally available instead of trying to download the data from the source site. return_X_y : bool, default=False @@ -215,6 +251,16 @@ def fetch_20newsgroups(*, data_home=None, subset='train', categories=None, .. versionadded:: 0.22 + n_retries : int, default=3 + Number of retries when HTTP errors are encountered. + + .. versionadded:: 1.5 + + delay : float, default=1.0 + Number of seconds between retries. + + .. versionadded:: 1.5 + Returns ------- bunch : :class:`~sklearn.utils.Bunch` @@ -232,7 +278,26 @@ def fetch_20newsgroups(*, data_home=None, subset='train', categories=None, The names of target classes. (data, target) : tuple if `return_X_y=True` + A tuple of two ndarrays. The first contains a 2D array of shape + (n_samples, n_classes) with each row representing one sample and each + column representing the features. The second array of shape + (n_samples,) contains the target samples. + .. versionadded:: 0.22 + + Examples + -------- + >>> from sklearn.datasets import fetch_20newsgroups + >>> cats = ['alt.atheism', 'sci.space'] + >>> newsgroups_train = fetch_20newsgroups(subset='train', categories=cats) + >>> list(newsgroups_train.target_names) + ['alt.atheism', 'sci.space'] + >>> newsgroups_train.filenames.shape + (1073,) + >>> newsgroups_train.target.shape + (1073,) + >>> newsgroups_train.target[:10] + array([0, 1, 1, 1, 0, 1, 1, 0, 0, 0]) """ data_home = get_data_home(data_home=data_home) @@ -241,33 +306,35 @@ def fetch_20newsgroups(*, data_home=None, subset='train', categories=None, cache = None if os.path.exists(cache_path): try: - with open(cache_path, 'rb') as f: + with open(cache_path, "rb") as f: compressed_content = f.read() - uncompressed_content = codecs.decode( - compressed_content, 'zlib_codec') + uncompressed_content = codecs.decode(compressed_content, "zlib_codec") cache = pickle.loads(uncompressed_content) except Exception as e: - print(80 * '_') - print('Cache loading failed') - print(80 * '_') + print(80 * "_") + print("Cache loading failed") + print(80 * "_") print(e) if cache is None: if download_if_missing: - logger.info("Downloading 20news dataset. " - "This may take a few minutes.") - cache = _download_20newsgroups(target_dir=twenty_home, - cache_path=cache_path) + logger.info("Downloading 20news dataset. This may take a few minutes.") + cache = _download_20newsgroups( + target_dir=twenty_home, + cache_path=cache_path, + n_retries=n_retries, + delay=delay, + ) else: - raise IOError('20Newsgroups dataset not found') + raise OSError("20Newsgroups dataset not found") - if subset in ('train', 'test'): + if subset in ("train", "test"): data = cache[subset] - elif subset == 'all': + elif subset == "all": data_lst = list() target = list() filenames = list() - for subset in ('train', 'test'): + for subset in ("train", "test"): data = cache[subset] data_lst.extend(data.data) target.extend(data.target) @@ -276,21 +343,16 @@ def fetch_20newsgroups(*, data_home=None, subset='train', categories=None, data.data = data_lst data.target = np.array(target) data.filenames = np.array(filenames) - else: - raise ValueError( - "subset can only be 'train', 'test' or 'all', got '%s'" % subset) - module_path = dirname(__file__) - with open(join(module_path, 'descr', 'twenty_newsgroups.rst')) as rst_file: - fdescr = rst_file.read() + fdescr = load_descr("twenty_newsgroups.rst") data.DESCR = fdescr - if 'headers' in remove: + if "headers" in remove: data.data = [strip_newsgroup_header(text) for text in data.data] - if 'footers' in remove: + if "footers" in remove: data.data = [strip_newsgroup_footer(text) for text in data.data] - if 'quotes' in remove: + if "quotes" in remove: data.data = [strip_newsgroup_quoting(text) for text in data.data] if categories is not None: @@ -298,7 +360,7 @@ def fetch_20newsgroups(*, data_home=None, subset='train', categories=None, # Sort the categories to have the ordering of the labels labels.sort() labels, categories = zip(*labels) - mask = np.in1d(data.target, labels) + mask = np.isin(data.target, labels) data.filenames = data.filenames[mask] data.target = data.target[mask] # searchsorted to have continuous labels @@ -326,10 +388,32 @@ def fetch_20newsgroups(*, data_home=None, subset='train', categories=None, return data -@_deprecate_positional_args -def fetch_20newsgroups_vectorized(*, subset="train", remove=(), data_home=None, - download_if_missing=True, return_X_y=False, - normalize=True, as_frame=False): +@validate_params( + { + "subset": [StrOptions({"train", "test", "all"})], + "remove": [tuple], + "data_home": [str, os.PathLike, None], + "download_if_missing": ["boolean"], + "return_X_y": ["boolean"], + "normalize": ["boolean"], + "as_frame": ["boolean"], + "n_retries": [Interval(Integral, 1, None, closed="left")], + "delay": [Interval(Real, 0.0, None, closed="neither")], + }, + prefer_skip_nested_validation=True, +) +def fetch_20newsgroups_vectorized( + *, + subset="train", + remove=(), + data_home=None, + download_if_missing=True, + return_X_y=False, + normalize=True, + as_frame=False, + n_retries=3, + delay=1.0, +): """Load and vectorize the 20 newsgroups dataset (classification). Download it if necessary. @@ -372,12 +456,12 @@ def fetch_20newsgroups_vectorized(*, subset="train", remove=(), data_home=None, ends of posts that look like signatures, and 'quotes' removes lines that appear to be quoting another post. - data_home : str, default=None + data_home : str or path-like, default=None Specify an download and cache folder for the datasets. If None, all scikit-learn data is stored in '~/scikit_learn_data' subfolders. download_if_missing : bool, default=True - If False, raise an IOError if the data is not locally available + If False, raise an OSError if the data is not locally available instead of trying to download the data from the source site. return_X_y : bool, default=False @@ -400,6 +484,16 @@ def fetch_20newsgroups_vectorized(*, subset="train", remove=(), data_home=None, .. versionadded:: 0.24 + n_retries : int, default=3 + Number of retries when HTTP errors are encountered. + + .. versionadded:: 1.5 + + delay : float, default=1.0 + Number of seconds between retries. + + .. versionadded:: 1.5 + Returns ------- bunch : :class:`~sklearn.utils.Bunch` @@ -426,29 +520,46 @@ def fetch_20newsgroups_vectorized(*, subset="train", remove=(), data_home=None, description above. .. versionadded:: 0.20 + + Examples + -------- + >>> from sklearn.datasets import fetch_20newsgroups_vectorized + >>> newsgroups_vectorized = fetch_20newsgroups_vectorized(subset='test') + >>> newsgroups_vectorized.data.shape + (7532, 130107) + >>> newsgroups_vectorized.target.shape + (7532,) """ data_home = get_data_home(data_home=data_home) - filebase = '20newsgroup_vectorized' + filebase = "20newsgroup_vectorized" if remove: - filebase += 'remove-' + ('-'.join(remove)) + filebase += "remove-" + "-".join(remove) target_file = _pkl_filepath(data_home, filebase + ".pkl") # we shuffle but use a fixed seed for the memoization - data_train = fetch_20newsgroups(data_home=data_home, - subset='train', - categories=None, - shuffle=True, - random_state=12, - remove=remove, - download_if_missing=download_if_missing) - - data_test = fetch_20newsgroups(data_home=data_home, - subset='test', - categories=None, - shuffle=True, - random_state=12, - remove=remove, - download_if_missing=download_if_missing) + data_train = fetch_20newsgroups( + data_home=data_home, + subset="train", + categories=None, + shuffle=True, + random_state=12, + remove=remove, + download_if_missing=download_if_missing, + n_retries=n_retries, + delay=delay, + ) + + data_test = fetch_20newsgroups( + data_home=data_home, + subset="test", + categories=None, + shuffle=True, + random_state=12, + remove=remove, + download_if_missing=download_if_missing, + n_retries=n_retries, + delay=delay, + ) if os.path.exists(target_file): try: @@ -456,15 +567,15 @@ def fetch_20newsgroups_vectorized(*, subset="train", remove=(), data_home=None, except ValueError as e: raise ValueError( f"The cached dataset located in {target_file} was fetched " - f"with an older scikit-learn version and it is not compatible " - f"with the scikit-learn version imported. You need to " + "with an older scikit-learn version and it is not compatible " + "with the scikit-learn version imported. You need to " f"manually delete the file: {target_file}." ) from e else: vectorizer = CountVectorizer(dtype=np.int16) X_train = vectorizer.fit_transform(data_train.data).tocsr() X_test = vectorizer.transform(data_test.data).tocsr() - feature_names = vectorizer.get_feature_names() + feature_names = vectorizer.get_feature_names_out() joblib.dump((X_train, X_test, feature_names), target_file, compress=9) @@ -487,16 +598,11 @@ def fetch_20newsgroups_vectorized(*, subset="train", remove=(), data_home=None, elif subset == "all": data = sp.vstack((X_train, X_test)).tocsr() target = np.concatenate((data_train.target, data_test.target)) - else: - raise ValueError("%r is not a valid subset: should be one of " - "['train', 'test', 'all']" % subset) - module_path = dirname(__file__) - with open(join(module_path, 'descr', 'twenty_newsgroups.rst')) as rst_file: - fdescr = rst_file.read() + fdescr = load_descr("twenty_newsgroups.rst") frame = None - target_name = ['category_class'] + target_name = ["category_class"] if as_frame: frame, data, target = _convert_data_dataframe( @@ -505,15 +611,17 @@ def fetch_20newsgroups_vectorized(*, subset="train", remove=(), data_home=None, target, feature_names, target_names=target_name, - sparse_data=True + sparse_data=True, ) if return_X_y: return data, target - return Bunch(data=data, - target=target, - frame=frame, - target_names=target_names, - feature_names=feature_names, - DESCR=fdescr) + return Bunch( + data=data, + target=target, + frame=frame, + target_names=target_names, + feature_names=feature_names, + DESCR=fdescr, + ) diff --git a/sklearn/datasets/data/__init__.py b/sklearn/datasets/data/__init__.py new file mode 100644 index 0000000000000..67dd18fb94b59 --- /dev/null +++ b/sklearn/datasets/data/__init__.py @@ -0,0 +1,2 @@ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause diff --git a/sklearn/datasets/data/boston_house_prices.csv b/sklearn/datasets/data/boston_house_prices.csv deleted file mode 100644 index 61193a5d646cc..0000000000000 --- a/sklearn/datasets/data/boston_house_prices.csv +++ /dev/null @@ -1,508 +0,0 @@ -506,13,,,,,,,,,,,, -"CRIM","ZN","INDUS","CHAS","NOX","RM","AGE","DIS","RAD","TAX","PTRATIO","B","LSTAT","MEDV" -0.00632,18,2.31,0,0.538,6.575,65.2,4.09,1,296,15.3,396.9,4.98,24 -0.02731,0,7.07,0,0.469,6.421,78.9,4.9671,2,242,17.8,396.9,9.14,21.6 -0.02729,0,7.07,0,0.469,7.185,61.1,4.9671,2,242,17.8,392.83,4.03,34.7 -0.03237,0,2.18,0,0.458,6.998,45.8,6.0622,3,222,18.7,394.63,2.94,33.4 -0.06905,0,2.18,0,0.458,7.147,54.2,6.0622,3,222,18.7,396.9,5.33,36.2 -0.02985,0,2.18,0,0.458,6.43,58.7,6.0622,3,222,18.7,394.12,5.21,28.7 -0.08829,12.5,7.87,0,0.524,6.012,66.6,5.5605,5,311,15.2,395.6,12.43,22.9 -0.14455,12.5,7.87,0,0.524,6.172,96.1,5.9505,5,311,15.2,396.9,19.15,27.1 -0.21124,12.5,7.87,0,0.524,5.631,100,6.0821,5,311,15.2,386.63,29.93,16.5 -0.17004,12.5,7.87,0,0.524,6.004,85.9,6.5921,5,311,15.2,386.71,17.1,18.9 -0.22489,12.5,7.87,0,0.524,6.377,94.3,6.3467,5,311,15.2,392.52,20.45,15 -0.11747,12.5,7.87,0,0.524,6.009,82.9,6.2267,5,311,15.2,396.9,13.27,18.9 -0.09378,12.5,7.87,0,0.524,5.889,39,5.4509,5,311,15.2,390.5,15.71,21.7 -0.62976,0,8.14,0,0.538,5.949,61.8,4.7075,4,307,21,396.9,8.26,20.4 -0.63796,0,8.14,0,0.538,6.096,84.5,4.4619,4,307,21,380.02,10.26,18.2 -0.62739,0,8.14,0,0.538,5.834,56.5,4.4986,4,307,21,395.62,8.47,19.9 -1.05393,0,8.14,0,0.538,5.935,29.3,4.4986,4,307,21,386.85,6.58,23.1 -0.7842,0,8.14,0,0.538,5.99,81.7,4.2579,4,307,21,386.75,14.67,17.5 -0.80271,0,8.14,0,0.538,5.456,36.6,3.7965,4,307,21,288.99,11.69,20.2 -0.7258,0,8.14,0,0.538,5.727,69.5,3.7965,4,307,21,390.95,11.28,18.2 -1.25179,0,8.14,0,0.538,5.57,98.1,3.7979,4,307,21,376.57,21.02,13.6 -0.85204,0,8.14,0,0.538,5.965,89.2,4.0123,4,307,21,392.53,13.83,19.6 -1.23247,0,8.14,0,0.538,6.142,91.7,3.9769,4,307,21,396.9,18.72,15.2 -0.98843,0,8.14,0,0.538,5.813,100,4.0952,4,307,21,394.54,19.88,14.5 -0.75026,0,8.14,0,0.538,5.924,94.1,4.3996,4,307,21,394.33,16.3,15.6 -0.84054,0,8.14,0,0.538,5.599,85.7,4.4546,4,307,21,303.42,16.51,13.9 -0.67191,0,8.14,0,0.538,5.813,90.3,4.682,4,307,21,376.88,14.81,16.6 -0.95577,0,8.14,0,0.538,6.047,88.8,4.4534,4,307,21,306.38,17.28,14.8 -0.77299,0,8.14,0,0.538,6.495,94.4,4.4547,4,307,21,387.94,12.8,18.4 -1.00245,0,8.14,0,0.538,6.674,87.3,4.239,4,307,21,380.23,11.98,21 -1.13081,0,8.14,0,0.538,5.713,94.1,4.233,4,307,21,360.17,22.6,12.7 -1.35472,0,8.14,0,0.538,6.072,100,4.175,4,307,21,376.73,13.04,14.5 -1.38799,0,8.14,0,0.538,5.95,82,3.99,4,307,21,232.6,27.71,13.2 -1.15172,0,8.14,0,0.538,5.701,95,3.7872,4,307,21,358.77,18.35,13.1 -1.61282,0,8.14,0,0.538,6.096,96.9,3.7598,4,307,21,248.31,20.34,13.5 -0.06417,0,5.96,0,0.499,5.933,68.2,3.3603,5,279,19.2,396.9,9.68,18.9 -0.09744,0,5.96,0,0.499,5.841,61.4,3.3779,5,279,19.2,377.56,11.41,20 -0.08014,0,5.96,0,0.499,5.85,41.5,3.9342,5,279,19.2,396.9,8.77,21 -0.17505,0,5.96,0,0.499,5.966,30.2,3.8473,5,279,19.2,393.43,10.13,24.7 -0.02763,75,2.95,0,0.428,6.595,21.8,5.4011,3,252,18.3,395.63,4.32,30.8 -0.03359,75,2.95,0,0.428,7.024,15.8,5.4011,3,252,18.3,395.62,1.98,34.9 -0.12744,0,6.91,0,0.448,6.77,2.9,5.7209,3,233,17.9,385.41,4.84,26.6 -0.1415,0,6.91,0,0.448,6.169,6.6,5.7209,3,233,17.9,383.37,5.81,25.3 -0.15936,0,6.91,0,0.448,6.211,6.5,5.7209,3,233,17.9,394.46,7.44,24.7 -0.12269,0,6.91,0,0.448,6.069,40,5.7209,3,233,17.9,389.39,9.55,21.2 -0.17142,0,6.91,0,0.448,5.682,33.8,5.1004,3,233,17.9,396.9,10.21,19.3 -0.18836,0,6.91,0,0.448,5.786,33.3,5.1004,3,233,17.9,396.9,14.15,20 -0.22927,0,6.91,0,0.448,6.03,85.5,5.6894,3,233,17.9,392.74,18.8,16.6 -0.25387,0,6.91,0,0.448,5.399,95.3,5.87,3,233,17.9,396.9,30.81,14.4 -0.21977,0,6.91,0,0.448,5.602,62,6.0877,3,233,17.9,396.9,16.2,19.4 -0.08873,21,5.64,0,0.439,5.963,45.7,6.8147,4,243,16.8,395.56,13.45,19.7 -0.04337,21,5.64,0,0.439,6.115,63,6.8147,4,243,16.8,393.97,9.43,20.5 -0.0536,21,5.64,0,0.439,6.511,21.1,6.8147,4,243,16.8,396.9,5.28,25 -0.04981,21,5.64,0,0.439,5.998,21.4,6.8147,4,243,16.8,396.9,8.43,23.4 -0.0136,75,4,0,0.41,5.888,47.6,7.3197,3,469,21.1,396.9,14.8,18.9 -0.01311,90,1.22,0,0.403,7.249,21.9,8.6966,5,226,17.9,395.93,4.81,35.4 -0.02055,85,0.74,0,0.41,6.383,35.7,9.1876,2,313,17.3,396.9,5.77,24.7 -0.01432,100,1.32,0,0.411,6.816,40.5,8.3248,5,256,15.1,392.9,3.95,31.6 -0.15445,25,5.13,0,0.453,6.145,29.2,7.8148,8,284,19.7,390.68,6.86,23.3 -0.10328,25,5.13,0,0.453,5.927,47.2,6.932,8,284,19.7,396.9,9.22,19.6 -0.14932,25,5.13,0,0.453,5.741,66.2,7.2254,8,284,19.7,395.11,13.15,18.7 -0.17171,25,5.13,0,0.453,5.966,93.4,6.8185,8,284,19.7,378.08,14.44,16 -0.11027,25,5.13,0,0.453,6.456,67.8,7.2255,8,284,19.7,396.9,6.73,22.2 -0.1265,25,5.13,0,0.453,6.762,43.4,7.9809,8,284,19.7,395.58,9.5,25 -0.01951,17.5,1.38,0,0.4161,7.104,59.5,9.2229,3,216,18.6,393.24,8.05,33 -0.03584,80,3.37,0,0.398,6.29,17.8,6.6115,4,337,16.1,396.9,4.67,23.5 -0.04379,80,3.37,0,0.398,5.787,31.1,6.6115,4,337,16.1,396.9,10.24,19.4 -0.05789,12.5,6.07,0,0.409,5.878,21.4,6.498,4,345,18.9,396.21,8.1,22 -0.13554,12.5,6.07,0,0.409,5.594,36.8,6.498,4,345,18.9,396.9,13.09,17.4 -0.12816,12.5,6.07,0,0.409,5.885,33,6.498,4,345,18.9,396.9,8.79,20.9 -0.08826,0,10.81,0,0.413,6.417,6.6,5.2873,4,305,19.2,383.73,6.72,24.2 -0.15876,0,10.81,0,0.413,5.961,17.5,5.2873,4,305,19.2,376.94,9.88,21.7 -0.09164,0,10.81,0,0.413,6.065,7.8,5.2873,4,305,19.2,390.91,5.52,22.8 -0.19539,0,10.81,0,0.413,6.245,6.2,5.2873,4,305,19.2,377.17,7.54,23.4 -0.07896,0,12.83,0,0.437,6.273,6,4.2515,5,398,18.7,394.92,6.78,24.1 -0.09512,0,12.83,0,0.437,6.286,45,4.5026,5,398,18.7,383.23,8.94,21.4 -0.10153,0,12.83,0,0.437,6.279,74.5,4.0522,5,398,18.7,373.66,11.97,20 -0.08707,0,12.83,0,0.437,6.14,45.8,4.0905,5,398,18.7,386.96,10.27,20.8 -0.05646,0,12.83,0,0.437,6.232,53.7,5.0141,5,398,18.7,386.4,12.34,21.2 -0.08387,0,12.83,0,0.437,5.874,36.6,4.5026,5,398,18.7,396.06,9.1,20.3 -0.04113,25,4.86,0,0.426,6.727,33.5,5.4007,4,281,19,396.9,5.29,28 -0.04462,25,4.86,0,0.426,6.619,70.4,5.4007,4,281,19,395.63,7.22,23.9 -0.03659,25,4.86,0,0.426,6.302,32.2,5.4007,4,281,19,396.9,6.72,24.8 -0.03551,25,4.86,0,0.426,6.167,46.7,5.4007,4,281,19,390.64,7.51,22.9 -0.05059,0,4.49,0,0.449,6.389,48,4.7794,3,247,18.5,396.9,9.62,23.9 -0.05735,0,4.49,0,0.449,6.63,56.1,4.4377,3,247,18.5,392.3,6.53,26.6 -0.05188,0,4.49,0,0.449,6.015,45.1,4.4272,3,247,18.5,395.99,12.86,22.5 -0.07151,0,4.49,0,0.449,6.121,56.8,3.7476,3,247,18.5,395.15,8.44,22.2 -0.0566,0,3.41,0,0.489,7.007,86.3,3.4217,2,270,17.8,396.9,5.5,23.6 -0.05302,0,3.41,0,0.489,7.079,63.1,3.4145,2,270,17.8,396.06,5.7,28.7 -0.04684,0,3.41,0,0.489,6.417,66.1,3.0923,2,270,17.8,392.18,8.81,22.6 -0.03932,0,3.41,0,0.489,6.405,73.9,3.0921,2,270,17.8,393.55,8.2,22 -0.04203,28,15.04,0,0.464,6.442,53.6,3.6659,4,270,18.2,395.01,8.16,22.9 -0.02875,28,15.04,0,0.464,6.211,28.9,3.6659,4,270,18.2,396.33,6.21,25 -0.04294,28,15.04,0,0.464,6.249,77.3,3.615,4,270,18.2,396.9,10.59,20.6 -0.12204,0,2.89,0,0.445,6.625,57.8,3.4952,2,276,18,357.98,6.65,28.4 -0.11504,0,2.89,0,0.445,6.163,69.6,3.4952,2,276,18,391.83,11.34,21.4 -0.12083,0,2.89,0,0.445,8.069,76,3.4952,2,276,18,396.9,4.21,38.7 -0.08187,0,2.89,0,0.445,7.82,36.9,3.4952,2,276,18,393.53,3.57,43.8 -0.0686,0,2.89,0,0.445,7.416,62.5,3.4952,2,276,18,396.9,6.19,33.2 -0.14866,0,8.56,0,0.52,6.727,79.9,2.7778,5,384,20.9,394.76,9.42,27.5 -0.11432,0,8.56,0,0.52,6.781,71.3,2.8561,5,384,20.9,395.58,7.67,26.5 -0.22876,0,8.56,0,0.52,6.405,85.4,2.7147,5,384,20.9,70.8,10.63,18.6 -0.21161,0,8.56,0,0.52,6.137,87.4,2.7147,5,384,20.9,394.47,13.44,19.3 -0.1396,0,8.56,0,0.52,6.167,90,2.421,5,384,20.9,392.69,12.33,20.1 -0.13262,0,8.56,0,0.52,5.851,96.7,2.1069,5,384,20.9,394.05,16.47,19.5 -0.1712,0,8.56,0,0.52,5.836,91.9,2.211,5,384,20.9,395.67,18.66,19.5 -0.13117,0,8.56,0,0.52,6.127,85.2,2.1224,5,384,20.9,387.69,14.09,20.4 -0.12802,0,8.56,0,0.52,6.474,97.1,2.4329,5,384,20.9,395.24,12.27,19.8 -0.26363,0,8.56,0,0.52,6.229,91.2,2.5451,5,384,20.9,391.23,15.55,19.4 -0.10793,0,8.56,0,0.52,6.195,54.4,2.7778,5,384,20.9,393.49,13,21.7 -0.10084,0,10.01,0,0.547,6.715,81.6,2.6775,6,432,17.8,395.59,10.16,22.8 -0.12329,0,10.01,0,0.547,5.913,92.9,2.3534,6,432,17.8,394.95,16.21,18.8 -0.22212,0,10.01,0,0.547,6.092,95.4,2.548,6,432,17.8,396.9,17.09,18.7 -0.14231,0,10.01,0,0.547,6.254,84.2,2.2565,6,432,17.8,388.74,10.45,18.5 -0.17134,0,10.01,0,0.547,5.928,88.2,2.4631,6,432,17.8,344.91,15.76,18.3 -0.13158,0,10.01,0,0.547,6.176,72.5,2.7301,6,432,17.8,393.3,12.04,21.2 -0.15098,0,10.01,0,0.547,6.021,82.6,2.7474,6,432,17.8,394.51,10.3,19.2 -0.13058,0,10.01,0,0.547,5.872,73.1,2.4775,6,432,17.8,338.63,15.37,20.4 -0.14476,0,10.01,0,0.547,5.731,65.2,2.7592,6,432,17.8,391.5,13.61,19.3 -0.06899,0,25.65,0,0.581,5.87,69.7,2.2577,2,188,19.1,389.15,14.37,22 -0.07165,0,25.65,0,0.581,6.004,84.1,2.1974,2,188,19.1,377.67,14.27,20.3 -0.09299,0,25.65,0,0.581,5.961,92.9,2.0869,2,188,19.1,378.09,17.93,20.5 -0.15038,0,25.65,0,0.581,5.856,97,1.9444,2,188,19.1,370.31,25.41,17.3 -0.09849,0,25.65,0,0.581,5.879,95.8,2.0063,2,188,19.1,379.38,17.58,18.8 -0.16902,0,25.65,0,0.581,5.986,88.4,1.9929,2,188,19.1,385.02,14.81,21.4 -0.38735,0,25.65,0,0.581,5.613,95.6,1.7572,2,188,19.1,359.29,27.26,15.7 -0.25915,0,21.89,0,0.624,5.693,96,1.7883,4,437,21.2,392.11,17.19,16.2 -0.32543,0,21.89,0,0.624,6.431,98.8,1.8125,4,437,21.2,396.9,15.39,18 -0.88125,0,21.89,0,0.624,5.637,94.7,1.9799,4,437,21.2,396.9,18.34,14.3 -0.34006,0,21.89,0,0.624,6.458,98.9,2.1185,4,437,21.2,395.04,12.6,19.2 -1.19294,0,21.89,0,0.624,6.326,97.7,2.271,4,437,21.2,396.9,12.26,19.6 -0.59005,0,21.89,0,0.624,6.372,97.9,2.3274,4,437,21.2,385.76,11.12,23 -0.32982,0,21.89,0,0.624,5.822,95.4,2.4699,4,437,21.2,388.69,15.03,18.4 -0.97617,0,21.89,0,0.624,5.757,98.4,2.346,4,437,21.2,262.76,17.31,15.6 -0.55778,0,21.89,0,0.624,6.335,98.2,2.1107,4,437,21.2,394.67,16.96,18.1 -0.32264,0,21.89,0,0.624,5.942,93.5,1.9669,4,437,21.2,378.25,16.9,17.4 -0.35233,0,21.89,0,0.624,6.454,98.4,1.8498,4,437,21.2,394.08,14.59,17.1 -0.2498,0,21.89,0,0.624,5.857,98.2,1.6686,4,437,21.2,392.04,21.32,13.3 -0.54452,0,21.89,0,0.624,6.151,97.9,1.6687,4,437,21.2,396.9,18.46,17.8 -0.2909,0,21.89,0,0.624,6.174,93.6,1.6119,4,437,21.2,388.08,24.16,14 -1.62864,0,21.89,0,0.624,5.019,100,1.4394,4,437,21.2,396.9,34.41,14.4 -3.32105,0,19.58,1,0.871,5.403,100,1.3216,5,403,14.7,396.9,26.82,13.4 -4.0974,0,19.58,0,0.871,5.468,100,1.4118,5,403,14.7,396.9,26.42,15.6 -2.77974,0,19.58,0,0.871,4.903,97.8,1.3459,5,403,14.7,396.9,29.29,11.8 -2.37934,0,19.58,0,0.871,6.13,100,1.4191,5,403,14.7,172.91,27.8,13.8 -2.15505,0,19.58,0,0.871,5.628,100,1.5166,5,403,14.7,169.27,16.65,15.6 -2.36862,0,19.58,0,0.871,4.926,95.7,1.4608,5,403,14.7,391.71,29.53,14.6 -2.33099,0,19.58,0,0.871,5.186,93.8,1.5296,5,403,14.7,356.99,28.32,17.8 -2.73397,0,19.58,0,0.871,5.597,94.9,1.5257,5,403,14.7,351.85,21.45,15.4 -1.6566,0,19.58,0,0.871,6.122,97.3,1.618,5,403,14.7,372.8,14.1,21.5 -1.49632,0,19.58,0,0.871,5.404,100,1.5916,5,403,14.7,341.6,13.28,19.6 -1.12658,0,19.58,1,0.871,5.012,88,1.6102,5,403,14.7,343.28,12.12,15.3 -2.14918,0,19.58,0,0.871,5.709,98.5,1.6232,5,403,14.7,261.95,15.79,19.4 -1.41385,0,19.58,1,0.871,6.129,96,1.7494,5,403,14.7,321.02,15.12,17 -3.53501,0,19.58,1,0.871,6.152,82.6,1.7455,5,403,14.7,88.01,15.02,15.6 -2.44668,0,19.58,0,0.871,5.272,94,1.7364,5,403,14.7,88.63,16.14,13.1 -1.22358,0,19.58,0,0.605,6.943,97.4,1.8773,5,403,14.7,363.43,4.59,41.3 -1.34284,0,19.58,0,0.605,6.066,100,1.7573,5,403,14.7,353.89,6.43,24.3 -1.42502,0,19.58,0,0.871,6.51,100,1.7659,5,403,14.7,364.31,7.39,23.3 -1.27346,0,19.58,1,0.605,6.25,92.6,1.7984,5,403,14.7,338.92,5.5,27 -1.46336,0,19.58,0,0.605,7.489,90.8,1.9709,5,403,14.7,374.43,1.73,50 -1.83377,0,19.58,1,0.605,7.802,98.2,2.0407,5,403,14.7,389.61,1.92,50 -1.51902,0,19.58,1,0.605,8.375,93.9,2.162,5,403,14.7,388.45,3.32,50 -2.24236,0,19.58,0,0.605,5.854,91.8,2.422,5,403,14.7,395.11,11.64,22.7 -2.924,0,19.58,0,0.605,6.101,93,2.2834,5,403,14.7,240.16,9.81,25 -2.01019,0,19.58,0,0.605,7.929,96.2,2.0459,5,403,14.7,369.3,3.7,50 -1.80028,0,19.58,0,0.605,5.877,79.2,2.4259,5,403,14.7,227.61,12.14,23.8 -2.3004,0,19.58,0,0.605,6.319,96.1,2.1,5,403,14.7,297.09,11.1,23.8 -2.44953,0,19.58,0,0.605,6.402,95.2,2.2625,5,403,14.7,330.04,11.32,22.3 -1.20742,0,19.58,0,0.605,5.875,94.6,2.4259,5,403,14.7,292.29,14.43,17.4 -2.3139,0,19.58,0,0.605,5.88,97.3,2.3887,5,403,14.7,348.13,12.03,19.1 -0.13914,0,4.05,0,0.51,5.572,88.5,2.5961,5,296,16.6,396.9,14.69,23.1 -0.09178,0,4.05,0,0.51,6.416,84.1,2.6463,5,296,16.6,395.5,9.04,23.6 -0.08447,0,4.05,0,0.51,5.859,68.7,2.7019,5,296,16.6,393.23,9.64,22.6 -0.06664,0,4.05,0,0.51,6.546,33.1,3.1323,5,296,16.6,390.96,5.33,29.4 -0.07022,0,4.05,0,0.51,6.02,47.2,3.5549,5,296,16.6,393.23,10.11,23.2 -0.05425,0,4.05,0,0.51,6.315,73.4,3.3175,5,296,16.6,395.6,6.29,24.6 -0.06642,0,4.05,0,0.51,6.86,74.4,2.9153,5,296,16.6,391.27,6.92,29.9 -0.0578,0,2.46,0,0.488,6.98,58.4,2.829,3,193,17.8,396.9,5.04,37.2 -0.06588,0,2.46,0,0.488,7.765,83.3,2.741,3,193,17.8,395.56,7.56,39.8 -0.06888,0,2.46,0,0.488,6.144,62.2,2.5979,3,193,17.8,396.9,9.45,36.2 -0.09103,0,2.46,0,0.488,7.155,92.2,2.7006,3,193,17.8,394.12,4.82,37.9 -0.10008,0,2.46,0,0.488,6.563,95.6,2.847,3,193,17.8,396.9,5.68,32.5 -0.08308,0,2.46,0,0.488,5.604,89.8,2.9879,3,193,17.8,391,13.98,26.4 -0.06047,0,2.46,0,0.488,6.153,68.8,3.2797,3,193,17.8,387.11,13.15,29.6 -0.05602,0,2.46,0,0.488,7.831,53.6,3.1992,3,193,17.8,392.63,4.45,50 -0.07875,45,3.44,0,0.437,6.782,41.1,3.7886,5,398,15.2,393.87,6.68,32 -0.12579,45,3.44,0,0.437,6.556,29.1,4.5667,5,398,15.2,382.84,4.56,29.8 -0.0837,45,3.44,0,0.437,7.185,38.9,4.5667,5,398,15.2,396.9,5.39,34.9 -0.09068,45,3.44,0,0.437,6.951,21.5,6.4798,5,398,15.2,377.68,5.1,37 -0.06911,45,3.44,0,0.437,6.739,30.8,6.4798,5,398,15.2,389.71,4.69,30.5 -0.08664,45,3.44,0,0.437,7.178,26.3,6.4798,5,398,15.2,390.49,2.87,36.4 -0.02187,60,2.93,0,0.401,6.8,9.9,6.2196,1,265,15.6,393.37,5.03,31.1 -0.01439,60,2.93,0,0.401,6.604,18.8,6.2196,1,265,15.6,376.7,4.38,29.1 -0.01381,80,0.46,0,0.422,7.875,32,5.6484,4,255,14.4,394.23,2.97,50 -0.04011,80,1.52,0,0.404,7.287,34.1,7.309,2,329,12.6,396.9,4.08,33.3 -0.04666,80,1.52,0,0.404,7.107,36.6,7.309,2,329,12.6,354.31,8.61,30.3 -0.03768,80,1.52,0,0.404,7.274,38.3,7.309,2,329,12.6,392.2,6.62,34.6 -0.0315,95,1.47,0,0.403,6.975,15.3,7.6534,3,402,17,396.9,4.56,34.9 -0.01778,95,1.47,0,0.403,7.135,13.9,7.6534,3,402,17,384.3,4.45,32.9 -0.03445,82.5,2.03,0,0.415,6.162,38.4,6.27,2,348,14.7,393.77,7.43,24.1 -0.02177,82.5,2.03,0,0.415,7.61,15.7,6.27,2,348,14.7,395.38,3.11,42.3 -0.0351,95,2.68,0,0.4161,7.853,33.2,5.118,4,224,14.7,392.78,3.81,48.5 -0.02009,95,2.68,0,0.4161,8.034,31.9,5.118,4,224,14.7,390.55,2.88,50 -0.13642,0,10.59,0,0.489,5.891,22.3,3.9454,4,277,18.6,396.9,10.87,22.6 -0.22969,0,10.59,0,0.489,6.326,52.5,4.3549,4,277,18.6,394.87,10.97,24.4 -0.25199,0,10.59,0,0.489,5.783,72.7,4.3549,4,277,18.6,389.43,18.06,22.5 -0.13587,0,10.59,1,0.489,6.064,59.1,4.2392,4,277,18.6,381.32,14.66,24.4 -0.43571,0,10.59,1,0.489,5.344,100,3.875,4,277,18.6,396.9,23.09,20 -0.17446,0,10.59,1,0.489,5.96,92.1,3.8771,4,277,18.6,393.25,17.27,21.7 -0.37578,0,10.59,1,0.489,5.404,88.6,3.665,4,277,18.6,395.24,23.98,19.3 -0.21719,0,10.59,1,0.489,5.807,53.8,3.6526,4,277,18.6,390.94,16.03,22.4 -0.14052,0,10.59,0,0.489,6.375,32.3,3.9454,4,277,18.6,385.81,9.38,28.1 -0.28955,0,10.59,0,0.489,5.412,9.8,3.5875,4,277,18.6,348.93,29.55,23.7 -0.19802,0,10.59,0,0.489,6.182,42.4,3.9454,4,277,18.6,393.63,9.47,25 -0.0456,0,13.89,1,0.55,5.888,56,3.1121,5,276,16.4,392.8,13.51,23.3 -0.07013,0,13.89,0,0.55,6.642,85.1,3.4211,5,276,16.4,392.78,9.69,28.7 -0.11069,0,13.89,1,0.55,5.951,93.8,2.8893,5,276,16.4,396.9,17.92,21.5 -0.11425,0,13.89,1,0.55,6.373,92.4,3.3633,5,276,16.4,393.74,10.5,23 -0.35809,0,6.2,1,0.507,6.951,88.5,2.8617,8,307,17.4,391.7,9.71,26.7 -0.40771,0,6.2,1,0.507,6.164,91.3,3.048,8,307,17.4,395.24,21.46,21.7 -0.62356,0,6.2,1,0.507,6.879,77.7,3.2721,8,307,17.4,390.39,9.93,27.5 -0.6147,0,6.2,0,0.507,6.618,80.8,3.2721,8,307,17.4,396.9,7.6,30.1 -0.31533,0,6.2,0,0.504,8.266,78.3,2.8944,8,307,17.4,385.05,4.14,44.8 -0.52693,0,6.2,0,0.504,8.725,83,2.8944,8,307,17.4,382,4.63,50 -0.38214,0,6.2,0,0.504,8.04,86.5,3.2157,8,307,17.4,387.38,3.13,37.6 -0.41238,0,6.2,0,0.504,7.163,79.9,3.2157,8,307,17.4,372.08,6.36,31.6 -0.29819,0,6.2,0,0.504,7.686,17,3.3751,8,307,17.4,377.51,3.92,46.7 -0.44178,0,6.2,0,0.504,6.552,21.4,3.3751,8,307,17.4,380.34,3.76,31.5 -0.537,0,6.2,0,0.504,5.981,68.1,3.6715,8,307,17.4,378.35,11.65,24.3 -0.46296,0,6.2,0,0.504,7.412,76.9,3.6715,8,307,17.4,376.14,5.25,31.7 -0.57529,0,6.2,0,0.507,8.337,73.3,3.8384,8,307,17.4,385.91,2.47,41.7 -0.33147,0,6.2,0,0.507,8.247,70.4,3.6519,8,307,17.4,378.95,3.95,48.3 -0.44791,0,6.2,1,0.507,6.726,66.5,3.6519,8,307,17.4,360.2,8.05,29 -0.33045,0,6.2,0,0.507,6.086,61.5,3.6519,8,307,17.4,376.75,10.88,24 -0.52058,0,6.2,1,0.507,6.631,76.5,4.148,8,307,17.4,388.45,9.54,25.1 -0.51183,0,6.2,0,0.507,7.358,71.6,4.148,8,307,17.4,390.07,4.73,31.5 -0.08244,30,4.93,0,0.428,6.481,18.5,6.1899,6,300,16.6,379.41,6.36,23.7 -0.09252,30,4.93,0,0.428,6.606,42.2,6.1899,6,300,16.6,383.78,7.37,23.3 -0.11329,30,4.93,0,0.428,6.897,54.3,6.3361,6,300,16.6,391.25,11.38,22 -0.10612,30,4.93,0,0.428,6.095,65.1,6.3361,6,300,16.6,394.62,12.4,20.1 -0.1029,30,4.93,0,0.428,6.358,52.9,7.0355,6,300,16.6,372.75,11.22,22.2 -0.12757,30,4.93,0,0.428,6.393,7.8,7.0355,6,300,16.6,374.71,5.19,23.7 -0.20608,22,5.86,0,0.431,5.593,76.5,7.9549,7,330,19.1,372.49,12.5,17.6 -0.19133,22,5.86,0,0.431,5.605,70.2,7.9549,7,330,19.1,389.13,18.46,18.5 -0.33983,22,5.86,0,0.431,6.108,34.9,8.0555,7,330,19.1,390.18,9.16,24.3 -0.19657,22,5.86,0,0.431,6.226,79.2,8.0555,7,330,19.1,376.14,10.15,20.5 -0.16439,22,5.86,0,0.431,6.433,49.1,7.8265,7,330,19.1,374.71,9.52,24.5 -0.19073,22,5.86,0,0.431,6.718,17.5,7.8265,7,330,19.1,393.74,6.56,26.2 -0.1403,22,5.86,0,0.431,6.487,13,7.3967,7,330,19.1,396.28,5.9,24.4 -0.21409,22,5.86,0,0.431,6.438,8.9,7.3967,7,330,19.1,377.07,3.59,24.8 -0.08221,22,5.86,0,0.431,6.957,6.8,8.9067,7,330,19.1,386.09,3.53,29.6 -0.36894,22,5.86,0,0.431,8.259,8.4,8.9067,7,330,19.1,396.9,3.54,42.8 -0.04819,80,3.64,0,0.392,6.108,32,9.2203,1,315,16.4,392.89,6.57,21.9 -0.03548,80,3.64,0,0.392,5.876,19.1,9.2203,1,315,16.4,395.18,9.25,20.9 -0.01538,90,3.75,0,0.394,7.454,34.2,6.3361,3,244,15.9,386.34,3.11,44 -0.61154,20,3.97,0,0.647,8.704,86.9,1.801,5,264,13,389.7,5.12,50 -0.66351,20,3.97,0,0.647,7.333,100,1.8946,5,264,13,383.29,7.79,36 -0.65665,20,3.97,0,0.647,6.842,100,2.0107,5,264,13,391.93,6.9,30.1 -0.54011,20,3.97,0,0.647,7.203,81.8,2.1121,5,264,13,392.8,9.59,33.8 -0.53412,20,3.97,0,0.647,7.52,89.4,2.1398,5,264,13,388.37,7.26,43.1 -0.52014,20,3.97,0,0.647,8.398,91.5,2.2885,5,264,13,386.86,5.91,48.8 -0.82526,20,3.97,0,0.647,7.327,94.5,2.0788,5,264,13,393.42,11.25,31 -0.55007,20,3.97,0,0.647,7.206,91.6,1.9301,5,264,13,387.89,8.1,36.5 -0.76162,20,3.97,0,0.647,5.56,62.8,1.9865,5,264,13,392.4,10.45,22.8 -0.7857,20,3.97,0,0.647,7.014,84.6,2.1329,5,264,13,384.07,14.79,30.7 -0.57834,20,3.97,0,0.575,8.297,67,2.4216,5,264,13,384.54,7.44,50 -0.5405,20,3.97,0,0.575,7.47,52.6,2.872,5,264,13,390.3,3.16,43.5 -0.09065,20,6.96,1,0.464,5.92,61.5,3.9175,3,223,18.6,391.34,13.65,20.7 -0.29916,20,6.96,0,0.464,5.856,42.1,4.429,3,223,18.6,388.65,13,21.1 -0.16211,20,6.96,0,0.464,6.24,16.3,4.429,3,223,18.6,396.9,6.59,25.2 -0.1146,20,6.96,0,0.464,6.538,58.7,3.9175,3,223,18.6,394.96,7.73,24.4 -0.22188,20,6.96,1,0.464,7.691,51.8,4.3665,3,223,18.6,390.77,6.58,35.2 -0.05644,40,6.41,1,0.447,6.758,32.9,4.0776,4,254,17.6,396.9,3.53,32.4 -0.09604,40,6.41,0,0.447,6.854,42.8,4.2673,4,254,17.6,396.9,2.98,32 -0.10469,40,6.41,1,0.447,7.267,49,4.7872,4,254,17.6,389.25,6.05,33.2 -0.06127,40,6.41,1,0.447,6.826,27.6,4.8628,4,254,17.6,393.45,4.16,33.1 -0.07978,40,6.41,0,0.447,6.482,32.1,4.1403,4,254,17.6,396.9,7.19,29.1 -0.21038,20,3.33,0,0.4429,6.812,32.2,4.1007,5,216,14.9,396.9,4.85,35.1 -0.03578,20,3.33,0,0.4429,7.82,64.5,4.6947,5,216,14.9,387.31,3.76,45.4 -0.03705,20,3.33,0,0.4429,6.968,37.2,5.2447,5,216,14.9,392.23,4.59,35.4 -0.06129,20,3.33,1,0.4429,7.645,49.7,5.2119,5,216,14.9,377.07,3.01,46 -0.01501,90,1.21,1,0.401,7.923,24.8,5.885,1,198,13.6,395.52,3.16,50 -0.00906,90,2.97,0,0.4,7.088,20.8,7.3073,1,285,15.3,394.72,7.85,32.2 -0.01096,55,2.25,0,0.389,6.453,31.9,7.3073,1,300,15.3,394.72,8.23,22 -0.01965,80,1.76,0,0.385,6.23,31.5,9.0892,1,241,18.2,341.6,12.93,20.1 -0.03871,52.5,5.32,0,0.405,6.209,31.3,7.3172,6,293,16.6,396.9,7.14,23.2 -0.0459,52.5,5.32,0,0.405,6.315,45.6,7.3172,6,293,16.6,396.9,7.6,22.3 -0.04297,52.5,5.32,0,0.405,6.565,22.9,7.3172,6,293,16.6,371.72,9.51,24.8 -0.03502,80,4.95,0,0.411,6.861,27.9,5.1167,4,245,19.2,396.9,3.33,28.5 -0.07886,80,4.95,0,0.411,7.148,27.7,5.1167,4,245,19.2,396.9,3.56,37.3 -0.03615,80,4.95,0,0.411,6.63,23.4,5.1167,4,245,19.2,396.9,4.7,27.9 -0.08265,0,13.92,0,0.437,6.127,18.4,5.5027,4,289,16,396.9,8.58,23.9 -0.08199,0,13.92,0,0.437,6.009,42.3,5.5027,4,289,16,396.9,10.4,21.7 -0.12932,0,13.92,0,0.437,6.678,31.1,5.9604,4,289,16,396.9,6.27,28.6 -0.05372,0,13.92,0,0.437,6.549,51,5.9604,4,289,16,392.85,7.39,27.1 -0.14103,0,13.92,0,0.437,5.79,58,6.32,4,289,16,396.9,15.84,20.3 -0.06466,70,2.24,0,0.4,6.345,20.1,7.8278,5,358,14.8,368.24,4.97,22.5 -0.05561,70,2.24,0,0.4,7.041,10,7.8278,5,358,14.8,371.58,4.74,29 -0.04417,70,2.24,0,0.4,6.871,47.4,7.8278,5,358,14.8,390.86,6.07,24.8 -0.03537,34,6.09,0,0.433,6.59,40.4,5.4917,7,329,16.1,395.75,9.5,22 -0.09266,34,6.09,0,0.433,6.495,18.4,5.4917,7,329,16.1,383.61,8.67,26.4 -0.1,34,6.09,0,0.433,6.982,17.7,5.4917,7,329,16.1,390.43,4.86,33.1 -0.05515,33,2.18,0,0.472,7.236,41.1,4.022,7,222,18.4,393.68,6.93,36.1 -0.05479,33,2.18,0,0.472,6.616,58.1,3.37,7,222,18.4,393.36,8.93,28.4 -0.07503,33,2.18,0,0.472,7.42,71.9,3.0992,7,222,18.4,396.9,6.47,33.4 -0.04932,33,2.18,0,0.472,6.849,70.3,3.1827,7,222,18.4,396.9,7.53,28.2 -0.49298,0,9.9,0,0.544,6.635,82.5,3.3175,4,304,18.4,396.9,4.54,22.8 -0.3494,0,9.9,0,0.544,5.972,76.7,3.1025,4,304,18.4,396.24,9.97,20.3 -2.63548,0,9.9,0,0.544,4.973,37.8,2.5194,4,304,18.4,350.45,12.64,16.1 -0.79041,0,9.9,0,0.544,6.122,52.8,2.6403,4,304,18.4,396.9,5.98,22.1 -0.26169,0,9.9,0,0.544,6.023,90.4,2.834,4,304,18.4,396.3,11.72,19.4 -0.26938,0,9.9,0,0.544,6.266,82.8,3.2628,4,304,18.4,393.39,7.9,21.6 -0.3692,0,9.9,0,0.544,6.567,87.3,3.6023,4,304,18.4,395.69,9.28,23.8 -0.25356,0,9.9,0,0.544,5.705,77.7,3.945,4,304,18.4,396.42,11.5,16.2 -0.31827,0,9.9,0,0.544,5.914,83.2,3.9986,4,304,18.4,390.7,18.33,17.8 -0.24522,0,9.9,0,0.544,5.782,71.7,4.0317,4,304,18.4,396.9,15.94,19.8 -0.40202,0,9.9,0,0.544,6.382,67.2,3.5325,4,304,18.4,395.21,10.36,23.1 -0.47547,0,9.9,0,0.544,6.113,58.8,4.0019,4,304,18.4,396.23,12.73,21 -0.1676,0,7.38,0,0.493,6.426,52.3,4.5404,5,287,19.6,396.9,7.2,23.8 -0.18159,0,7.38,0,0.493,6.376,54.3,4.5404,5,287,19.6,396.9,6.87,23.1 -0.35114,0,7.38,0,0.493,6.041,49.9,4.7211,5,287,19.6,396.9,7.7,20.4 -0.28392,0,7.38,0,0.493,5.708,74.3,4.7211,5,287,19.6,391.13,11.74,18.5 -0.34109,0,7.38,0,0.493,6.415,40.1,4.7211,5,287,19.6,396.9,6.12,25 -0.19186,0,7.38,0,0.493,6.431,14.7,5.4159,5,287,19.6,393.68,5.08,24.6 -0.30347,0,7.38,0,0.493,6.312,28.9,5.4159,5,287,19.6,396.9,6.15,23 -0.24103,0,7.38,0,0.493,6.083,43.7,5.4159,5,287,19.6,396.9,12.79,22.2 -0.06617,0,3.24,0,0.46,5.868,25.8,5.2146,4,430,16.9,382.44,9.97,19.3 -0.06724,0,3.24,0,0.46,6.333,17.2,5.2146,4,430,16.9,375.21,7.34,22.6 -0.04544,0,3.24,0,0.46,6.144,32.2,5.8736,4,430,16.9,368.57,9.09,19.8 -0.05023,35,6.06,0,0.4379,5.706,28.4,6.6407,1,304,16.9,394.02,12.43,17.1 -0.03466,35,6.06,0,0.4379,6.031,23.3,6.6407,1,304,16.9,362.25,7.83,19.4 -0.05083,0,5.19,0,0.515,6.316,38.1,6.4584,5,224,20.2,389.71,5.68,22.2 -0.03738,0,5.19,0,0.515,6.31,38.5,6.4584,5,224,20.2,389.4,6.75,20.7 -0.03961,0,5.19,0,0.515,6.037,34.5,5.9853,5,224,20.2,396.9,8.01,21.1 -0.03427,0,5.19,0,0.515,5.869,46.3,5.2311,5,224,20.2,396.9,9.8,19.5 -0.03041,0,5.19,0,0.515,5.895,59.6,5.615,5,224,20.2,394.81,10.56,18.5 -0.03306,0,5.19,0,0.515,6.059,37.3,4.8122,5,224,20.2,396.14,8.51,20.6 -0.05497,0,5.19,0,0.515,5.985,45.4,4.8122,5,224,20.2,396.9,9.74,19 -0.06151,0,5.19,0,0.515,5.968,58.5,4.8122,5,224,20.2,396.9,9.29,18.7 -0.01301,35,1.52,0,0.442,7.241,49.3,7.0379,1,284,15.5,394.74,5.49,32.7 -0.02498,0,1.89,0,0.518,6.54,59.7,6.2669,1,422,15.9,389.96,8.65,16.5 -0.02543,55,3.78,0,0.484,6.696,56.4,5.7321,5,370,17.6,396.9,7.18,23.9 -0.03049,55,3.78,0,0.484,6.874,28.1,6.4654,5,370,17.6,387.97,4.61,31.2 -0.03113,0,4.39,0,0.442,6.014,48.5,8.0136,3,352,18.8,385.64,10.53,17.5 -0.06162,0,4.39,0,0.442,5.898,52.3,8.0136,3,352,18.8,364.61,12.67,17.2 -0.0187,85,4.15,0,0.429,6.516,27.7,8.5353,4,351,17.9,392.43,6.36,23.1 -0.01501,80,2.01,0,0.435,6.635,29.7,8.344,4,280,17,390.94,5.99,24.5 -0.02899,40,1.25,0,0.429,6.939,34.5,8.7921,1,335,19.7,389.85,5.89,26.6 -0.06211,40,1.25,0,0.429,6.49,44.4,8.7921,1,335,19.7,396.9,5.98,22.9 -0.0795,60,1.69,0,0.411,6.579,35.9,10.7103,4,411,18.3,370.78,5.49,24.1 -0.07244,60,1.69,0,0.411,5.884,18.5,10.7103,4,411,18.3,392.33,7.79,18.6 -0.01709,90,2.02,0,0.41,6.728,36.1,12.1265,5,187,17,384.46,4.5,30.1 -0.04301,80,1.91,0,0.413,5.663,21.9,10.5857,4,334,22,382.8,8.05,18.2 -0.10659,80,1.91,0,0.413,5.936,19.5,10.5857,4,334,22,376.04,5.57,20.6 -8.98296,0,18.1,1,0.77,6.212,97.4,2.1222,24,666,20.2,377.73,17.6,17.8 -3.8497,0,18.1,1,0.77,6.395,91,2.5052,24,666,20.2,391.34,13.27,21.7 -5.20177,0,18.1,1,0.77,6.127,83.4,2.7227,24,666,20.2,395.43,11.48,22.7 -4.26131,0,18.1,0,0.77,6.112,81.3,2.5091,24,666,20.2,390.74,12.67,22.6 -4.54192,0,18.1,0,0.77,6.398,88,2.5182,24,666,20.2,374.56,7.79,25 -3.83684,0,18.1,0,0.77,6.251,91.1,2.2955,24,666,20.2,350.65,14.19,19.9 -3.67822,0,18.1,0,0.77,5.362,96.2,2.1036,24,666,20.2,380.79,10.19,20.8 -4.22239,0,18.1,1,0.77,5.803,89,1.9047,24,666,20.2,353.04,14.64,16.8 -3.47428,0,18.1,1,0.718,8.78,82.9,1.9047,24,666,20.2,354.55,5.29,21.9 -4.55587,0,18.1,0,0.718,3.561,87.9,1.6132,24,666,20.2,354.7,7.12,27.5 -3.69695,0,18.1,0,0.718,4.963,91.4,1.7523,24,666,20.2,316.03,14,21.9 -13.5222,0,18.1,0,0.631,3.863,100,1.5106,24,666,20.2,131.42,13.33,23.1 -4.89822,0,18.1,0,0.631,4.97,100,1.3325,24,666,20.2,375.52,3.26,50 -5.66998,0,18.1,1,0.631,6.683,96.8,1.3567,24,666,20.2,375.33,3.73,50 -6.53876,0,18.1,1,0.631,7.016,97.5,1.2024,24,666,20.2,392.05,2.96,50 -9.2323,0,18.1,0,0.631,6.216,100,1.1691,24,666,20.2,366.15,9.53,50 -8.26725,0,18.1,1,0.668,5.875,89.6,1.1296,24,666,20.2,347.88,8.88,50 -11.1081,0,18.1,0,0.668,4.906,100,1.1742,24,666,20.2,396.9,34.77,13.8 -18.4982,0,18.1,0,0.668,4.138,100,1.137,24,666,20.2,396.9,37.97,13.8 -19.6091,0,18.1,0,0.671,7.313,97.9,1.3163,24,666,20.2,396.9,13.44,15 -15.288,0,18.1,0,0.671,6.649,93.3,1.3449,24,666,20.2,363.02,23.24,13.9 -9.82349,0,18.1,0,0.671,6.794,98.8,1.358,24,666,20.2,396.9,21.24,13.3 -23.6482,0,18.1,0,0.671,6.38,96.2,1.3861,24,666,20.2,396.9,23.69,13.1 -17.8667,0,18.1,0,0.671,6.223,100,1.3861,24,666,20.2,393.74,21.78,10.2 -88.9762,0,18.1,0,0.671,6.968,91.9,1.4165,24,666,20.2,396.9,17.21,10.4 -15.8744,0,18.1,0,0.671,6.545,99.1,1.5192,24,666,20.2,396.9,21.08,10.9 -9.18702,0,18.1,0,0.7,5.536,100,1.5804,24,666,20.2,396.9,23.6,11.3 -7.99248,0,18.1,0,0.7,5.52,100,1.5331,24,666,20.2,396.9,24.56,12.3 -20.0849,0,18.1,0,0.7,4.368,91.2,1.4395,24,666,20.2,285.83,30.63,8.8 -16.8118,0,18.1,0,0.7,5.277,98.1,1.4261,24,666,20.2,396.9,30.81,7.2 -24.3938,0,18.1,0,0.7,4.652,100,1.4672,24,666,20.2,396.9,28.28,10.5 -22.5971,0,18.1,0,0.7,5,89.5,1.5184,24,666,20.2,396.9,31.99,7.4 -14.3337,0,18.1,0,0.7,4.88,100,1.5895,24,666,20.2,372.92,30.62,10.2 -8.15174,0,18.1,0,0.7,5.39,98.9,1.7281,24,666,20.2,396.9,20.85,11.5 -6.96215,0,18.1,0,0.7,5.713,97,1.9265,24,666,20.2,394.43,17.11,15.1 -5.29305,0,18.1,0,0.7,6.051,82.5,2.1678,24,666,20.2,378.38,18.76,23.2 -11.5779,0,18.1,0,0.7,5.036,97,1.77,24,666,20.2,396.9,25.68,9.7 -8.64476,0,18.1,0,0.693,6.193,92.6,1.7912,24,666,20.2,396.9,15.17,13.8 -13.3598,0,18.1,0,0.693,5.887,94.7,1.7821,24,666,20.2,396.9,16.35,12.7 -8.71675,0,18.1,0,0.693,6.471,98.8,1.7257,24,666,20.2,391.98,17.12,13.1 -5.87205,0,18.1,0,0.693,6.405,96,1.6768,24,666,20.2,396.9,19.37,12.5 -7.67202,0,18.1,0,0.693,5.747,98.9,1.6334,24,666,20.2,393.1,19.92,8.5 -38.3518,0,18.1,0,0.693,5.453,100,1.4896,24,666,20.2,396.9,30.59,5 -9.91655,0,18.1,0,0.693,5.852,77.8,1.5004,24,666,20.2,338.16,29.97,6.3 -25.0461,0,18.1,0,0.693,5.987,100,1.5888,24,666,20.2,396.9,26.77,5.6 -14.2362,0,18.1,0,0.693,6.343,100,1.5741,24,666,20.2,396.9,20.32,7.2 -9.59571,0,18.1,0,0.693,6.404,100,1.639,24,666,20.2,376.11,20.31,12.1 -24.8017,0,18.1,0,0.693,5.349,96,1.7028,24,666,20.2,396.9,19.77,8.3 -41.5292,0,18.1,0,0.693,5.531,85.4,1.6074,24,666,20.2,329.46,27.38,8.5 -67.9208,0,18.1,0,0.693,5.683,100,1.4254,24,666,20.2,384.97,22.98,5 -20.7162,0,18.1,0,0.659,4.138,100,1.1781,24,666,20.2,370.22,23.34,11.9 -11.9511,0,18.1,0,0.659,5.608,100,1.2852,24,666,20.2,332.09,12.13,27.9 -7.40389,0,18.1,0,0.597,5.617,97.9,1.4547,24,666,20.2,314.64,26.4,17.2 -14.4383,0,18.1,0,0.597,6.852,100,1.4655,24,666,20.2,179.36,19.78,27.5 -51.1358,0,18.1,0,0.597,5.757,100,1.413,24,666,20.2,2.6,10.11,15 -14.0507,0,18.1,0,0.597,6.657,100,1.5275,24,666,20.2,35.05,21.22,17.2 -18.811,0,18.1,0,0.597,4.628,100,1.5539,24,666,20.2,28.79,34.37,17.9 -28.6558,0,18.1,0,0.597,5.155,100,1.5894,24,666,20.2,210.97,20.08,16.3 -45.7461,0,18.1,0,0.693,4.519,100,1.6582,24,666,20.2,88.27,36.98,7 -18.0846,0,18.1,0,0.679,6.434,100,1.8347,24,666,20.2,27.25,29.05,7.2 -10.8342,0,18.1,0,0.679,6.782,90.8,1.8195,24,666,20.2,21.57,25.79,7.5 -25.9406,0,18.1,0,0.679,5.304,89.1,1.6475,24,666,20.2,127.36,26.64,10.4 -73.5341,0,18.1,0,0.679,5.957,100,1.8026,24,666,20.2,16.45,20.62,8.8 -11.8123,0,18.1,0,0.718,6.824,76.5,1.794,24,666,20.2,48.45,22.74,8.4 -11.0874,0,18.1,0,0.718,6.411,100,1.8589,24,666,20.2,318.75,15.02,16.7 -7.02259,0,18.1,0,0.718,6.006,95.3,1.8746,24,666,20.2,319.98,15.7,14.2 -12.0482,0,18.1,0,0.614,5.648,87.6,1.9512,24,666,20.2,291.55,14.1,20.8 -7.05042,0,18.1,0,0.614,6.103,85.1,2.0218,24,666,20.2,2.52,23.29,13.4 -8.79212,0,18.1,0,0.584,5.565,70.6,2.0635,24,666,20.2,3.65,17.16,11.7 -15.8603,0,18.1,0,0.679,5.896,95.4,1.9096,24,666,20.2,7.68,24.39,8.3 -12.2472,0,18.1,0,0.584,5.837,59.7,1.9976,24,666,20.2,24.65,15.69,10.2 -37.6619,0,18.1,0,0.679,6.202,78.7,1.8629,24,666,20.2,18.82,14.52,10.9 -7.36711,0,18.1,0,0.679,6.193,78.1,1.9356,24,666,20.2,96.73,21.52,11 -9.33889,0,18.1,0,0.679,6.38,95.6,1.9682,24,666,20.2,60.72,24.08,9.5 -8.49213,0,18.1,0,0.584,6.348,86.1,2.0527,24,666,20.2,83.45,17.64,14.5 -10.0623,0,18.1,0,0.584,6.833,94.3,2.0882,24,666,20.2,81.33,19.69,14.1 -6.44405,0,18.1,0,0.584,6.425,74.8,2.2004,24,666,20.2,97.95,12.03,16.1 -5.58107,0,18.1,0,0.713,6.436,87.9,2.3158,24,666,20.2,100.19,16.22,14.3 -13.9134,0,18.1,0,0.713,6.208,95,2.2222,24,666,20.2,100.63,15.17,11.7 -11.1604,0,18.1,0,0.74,6.629,94.6,2.1247,24,666,20.2,109.85,23.27,13.4 -14.4208,0,18.1,0,0.74,6.461,93.3,2.0026,24,666,20.2,27.49,18.05,9.6 -15.1772,0,18.1,0,0.74,6.152,100,1.9142,24,666,20.2,9.32,26.45,8.7 -13.6781,0,18.1,0,0.74,5.935,87.9,1.8206,24,666,20.2,68.95,34.02,8.4 -9.39063,0,18.1,0,0.74,5.627,93.9,1.8172,24,666,20.2,396.9,22.88,12.8 -22.0511,0,18.1,0,0.74,5.818,92.4,1.8662,24,666,20.2,391.45,22.11,10.5 -9.72418,0,18.1,0,0.74,6.406,97.2,2.0651,24,666,20.2,385.96,19.52,17.1 -5.66637,0,18.1,0,0.74,6.219,100,2.0048,24,666,20.2,395.69,16.59,18.4 -9.96654,0,18.1,0,0.74,6.485,100,1.9784,24,666,20.2,386.73,18.85,15.4 -12.8023,0,18.1,0,0.74,5.854,96.6,1.8956,24,666,20.2,240.52,23.79,10.8 -10.6718,0,18.1,0,0.74,6.459,94.8,1.9879,24,666,20.2,43.06,23.98,11.8 -6.28807,0,18.1,0,0.74,6.341,96.4,2.072,24,666,20.2,318.01,17.79,14.9 -9.92485,0,18.1,0,0.74,6.251,96.6,2.198,24,666,20.2,388.52,16.44,12.6 -9.32909,0,18.1,0,0.713,6.185,98.7,2.2616,24,666,20.2,396.9,18.13,14.1 -7.52601,0,18.1,0,0.713,6.417,98.3,2.185,24,666,20.2,304.21,19.31,13 -6.71772,0,18.1,0,0.713,6.749,92.6,2.3236,24,666,20.2,0.32,17.44,13.4 -5.44114,0,18.1,0,0.713,6.655,98.2,2.3552,24,666,20.2,355.29,17.73,15.2 -5.09017,0,18.1,0,0.713,6.297,91.8,2.3682,24,666,20.2,385.09,17.27,16.1 -8.24809,0,18.1,0,0.713,7.393,99.3,2.4527,24,666,20.2,375.87,16.74,17.8 -9.51363,0,18.1,0,0.713,6.728,94.1,2.4961,24,666,20.2,6.68,18.71,14.9 -4.75237,0,18.1,0,0.713,6.525,86.5,2.4358,24,666,20.2,50.92,18.13,14.1 -4.66883,0,18.1,0,0.713,5.976,87.9,2.5806,24,666,20.2,10.48,19.01,12.7 -8.20058,0,18.1,0,0.713,5.936,80.3,2.7792,24,666,20.2,3.5,16.94,13.5 -7.75223,0,18.1,0,0.713,6.301,83.7,2.7831,24,666,20.2,272.21,16.23,14.9 -6.80117,0,18.1,0,0.713,6.081,84.4,2.7175,24,666,20.2,396.9,14.7,20 -4.81213,0,18.1,0,0.713,6.701,90,2.5975,24,666,20.2,255.23,16.42,16.4 -3.69311,0,18.1,0,0.713,6.376,88.4,2.5671,24,666,20.2,391.43,14.65,17.7 -6.65492,0,18.1,0,0.713,6.317,83,2.7344,24,666,20.2,396.9,13.99,19.5 -5.82115,0,18.1,0,0.713,6.513,89.9,2.8016,24,666,20.2,393.82,10.29,20.2 -7.83932,0,18.1,0,0.655,6.209,65.4,2.9634,24,666,20.2,396.9,13.22,21.4 -3.1636,0,18.1,0,0.655,5.759,48.2,3.0665,24,666,20.2,334.4,14.13,19.9 -3.77498,0,18.1,0,0.655,5.952,84.7,2.8715,24,666,20.2,22.01,17.15,19 -4.42228,0,18.1,0,0.584,6.003,94.5,2.5403,24,666,20.2,331.29,21.32,19.1 -15.5757,0,18.1,0,0.58,5.926,71,2.9084,24,666,20.2,368.74,18.13,19.1 -13.0751,0,18.1,0,0.58,5.713,56.7,2.8237,24,666,20.2,396.9,14.76,20.1 -4.34879,0,18.1,0,0.58,6.167,84,3.0334,24,666,20.2,396.9,16.29,19.9 -4.03841,0,18.1,0,0.532,6.229,90.7,3.0993,24,666,20.2,395.33,12.87,19.6 -3.56868,0,18.1,0,0.58,6.437,75,2.8965,24,666,20.2,393.37,14.36,23.2 -4.64689,0,18.1,0,0.614,6.98,67.6,2.5329,24,666,20.2,374.68,11.66,29.8 -8.05579,0,18.1,0,0.584,5.427,95.4,2.4298,24,666,20.2,352.58,18.14,13.8 -6.39312,0,18.1,0,0.584,6.162,97.4,2.206,24,666,20.2,302.76,24.1,13.3 -4.87141,0,18.1,0,0.614,6.484,93.6,2.3053,24,666,20.2,396.21,18.68,16.7 -15.0234,0,18.1,0,0.614,5.304,97.3,2.1007,24,666,20.2,349.48,24.91,12 -10.233,0,18.1,0,0.614,6.185,96.7,2.1705,24,666,20.2,379.7,18.03,14.6 -14.3337,0,18.1,0,0.614,6.229,88,1.9512,24,666,20.2,383.32,13.11,21.4 -5.82401,0,18.1,0,0.532,6.242,64.7,3.4242,24,666,20.2,396.9,10.74,23 -5.70818,0,18.1,0,0.532,6.75,74.9,3.3317,24,666,20.2,393.07,7.74,23.7 -5.73116,0,18.1,0,0.532,7.061,77,3.4106,24,666,20.2,395.28,7.01,25 -2.81838,0,18.1,0,0.532,5.762,40.3,4.0983,24,666,20.2,392.92,10.42,21.8 -2.37857,0,18.1,0,0.583,5.871,41.9,3.724,24,666,20.2,370.73,13.34,20.6 -3.67367,0,18.1,0,0.583,6.312,51.9,3.9917,24,666,20.2,388.62,10.58,21.2 -5.69175,0,18.1,0,0.583,6.114,79.8,3.5459,24,666,20.2,392.68,14.98,19.1 -4.83567,0,18.1,0,0.583,5.905,53.2,3.1523,24,666,20.2,388.22,11.45,20.6 -0.15086,0,27.74,0,0.609,5.454,92.7,1.8209,4,711,20.1,395.09,18.06,15.2 -0.18337,0,27.74,0,0.609,5.414,98.3,1.7554,4,711,20.1,344.05,23.97,7 -0.20746,0,27.74,0,0.609,5.093,98,1.8226,4,711,20.1,318.43,29.68,8.1 -0.10574,0,27.74,0,0.609,5.983,98.8,1.8681,4,711,20.1,390.11,18.07,13.6 -0.11132,0,27.74,0,0.609,5.983,83.5,2.1099,4,711,20.1,396.9,13.35,20.1 -0.17331,0,9.69,0,0.585,5.707,54,2.3817,6,391,19.2,396.9,12.01,21.8 -0.27957,0,9.69,0,0.585,5.926,42.6,2.3817,6,391,19.2,396.9,13.59,24.5 -0.17899,0,9.69,0,0.585,5.67,28.8,2.7986,6,391,19.2,393.29,17.6,23.1 -0.2896,0,9.69,0,0.585,5.39,72.9,2.7986,6,391,19.2,396.9,21.14,19.7 -0.26838,0,9.69,0,0.585,5.794,70.6,2.8927,6,391,19.2,396.9,14.1,18.3 -0.23912,0,9.69,0,0.585,6.019,65.3,2.4091,6,391,19.2,396.9,12.92,21.2 -0.17783,0,9.69,0,0.585,5.569,73.5,2.3999,6,391,19.2,395.77,15.1,17.5 -0.22438,0,9.69,0,0.585,6.027,79.7,2.4982,6,391,19.2,396.9,14.33,16.8 -0.06263,0,11.93,0,0.573,6.593,69.1,2.4786,1,273,21,391.99,9.67,22.4 -0.04527,0,11.93,0,0.573,6.12,76.7,2.2875,1,273,21,396.9,9.08,20.6 -0.06076,0,11.93,0,0.573,6.976,91,2.1675,1,273,21,396.9,5.64,23.9 -0.10959,0,11.93,0,0.573,6.794,89.3,2.3889,1,273,21,393.45,6.48,22 -0.04741,0,11.93,0,0.573,6.03,80.8,2.505,1,273,21,396.9,7.88,11.9 diff --git a/sklearn/datasets/data/diabetes_data.csv.gz b/sklearn/datasets/data/diabetes_data.csv.gz deleted file mode 100644 index 60217e8e40d64..0000000000000 Binary files a/sklearn/datasets/data/diabetes_data.csv.gz and /dev/null differ diff --git a/sklearn/datasets/data/diabetes_data_raw.csv.gz b/sklearn/datasets/data/diabetes_data_raw.csv.gz new file mode 100644 index 0000000000000..ac76c7d33bec2 Binary files /dev/null and b/sklearn/datasets/data/diabetes_data_raw.csv.gz differ diff --git a/sklearn/datasets/descr/__init__.py b/sklearn/datasets/descr/__init__.py new file mode 100644 index 0000000000000..67dd18fb94b59 --- /dev/null +++ b/sklearn/datasets/descr/__init__.py @@ -0,0 +1,2 @@ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause diff --git a/sklearn/datasets/descr/boston_house_prices.rst b/sklearn/datasets/descr/boston_house_prices.rst deleted file mode 100644 index 948bccf080c82..0000000000000 --- a/sklearn/datasets/descr/boston_house_prices.rst +++ /dev/null @@ -1,50 +0,0 @@ -.. _boston_dataset: - -Boston house prices dataset ---------------------------- - -**Data Set Characteristics:** - - :Number of Instances: 506 - - :Number of Attributes: 13 numeric/categorical predictive. Median Value (attribute 14) is usually the target. - - :Attribute Information (in order): - - CRIM per capita crime rate by town - - ZN proportion of residential land zoned for lots over 25,000 sq.ft. - - INDUS proportion of non-retail business acres per town - - CHAS Charles River dummy variable (= 1 if tract bounds river; 0 otherwise) - - NOX nitric oxides concentration (parts per 10 million) - - RM average number of rooms per dwelling - - AGE proportion of owner-occupied units built prior to 1940 - - DIS weighted distances to five Boston employment centres - - RAD index of accessibility to radial highways - - TAX full-value property-tax rate per $10,000 - - PTRATIO pupil-teacher ratio by town - - B 1000(Bk - 0.63)^2 where Bk is the proportion of black people by town - - LSTAT % lower status of the population - - MEDV Median value of owner-occupied homes in $1000's - - :Missing Attribute Values: None - - :Creator: Harrison, D. and Rubinfeld, D.L. - -This is a copy of UCI ML housing dataset. -https://archive.ics.uci.edu/ml/machine-learning-databases/housing/ - - -This dataset was taken from the StatLib library which is maintained at Carnegie Mellon University. - -The Boston house-price data of Harrison, D. and Rubinfeld, D.L. 'Hedonic -prices and the demand for clean air', J. Environ. Economics & Management, -vol.5, 81-102, 1978. Used in Belsley, Kuh & Welsch, 'Regression diagnostics -...', Wiley, 1980. N.B. Various transformations are used in the table on -pages 244-261 of the latter. - -The Boston house-price data has been used in many machine learning papers that address regression -problems. - -.. topic:: References - - - Belsley, Kuh & Welsch, 'Regression diagnostics: Identifying Influential Data and Sources of Collinearity', Wiley, 1980. 244-261. - - Quinlan,R. (1993). Combining Instance-Based and Model-Based Learning. In Proceedings on the Tenth International Conference of Machine Learning, 236-243, University of Massachusetts, Amherst. Morgan Kaufmann. diff --git a/sklearn/datasets/descr/breast_cancer.rst b/sklearn/datasets/descr/breast_cancer.rst index bc4d60b9a363d..10def5d56af30 100644 --- a/sklearn/datasets/descr/breast_cancer.rst +++ b/sklearn/datasets/descr/breast_cancer.rst @@ -1,81 +1,81 @@ .. _breast_cancer_dataset: -Breast cancer wisconsin (diagnostic) dataset +Breast cancer Wisconsin (diagnostic) dataset -------------------------------------------- **Data Set Characteristics:** - :Number of Instances: 569 - - :Number of Attributes: 30 numeric, predictive attributes and the class - - :Attribute Information: - - radius (mean of distances from center to points on the perimeter) - - texture (standard deviation of gray-scale values) - - perimeter - - area - - smoothness (local variation in radius lengths) - - compactness (perimeter^2 / area - 1.0) - - concavity (severity of concave portions of the contour) - - concave points (number of concave portions of the contour) - - symmetry - - fractal dimension ("coastline approximation" - 1) - - The mean, standard error, and "worst" or largest (mean of the three - worst/largest values) of these features were computed for each image, - resulting in 30 features. For instance, field 0 is Mean Radius, field - 10 is Radius SE, field 20 is Worst Radius. - - - class: - - WDBC-Malignant - - WDBC-Benign - - :Summary Statistics: - - ===================================== ====== ====== - Min Max - ===================================== ====== ====== - radius (mean): 6.981 28.11 - texture (mean): 9.71 39.28 - perimeter (mean): 43.79 188.5 - area (mean): 143.5 2501.0 - smoothness (mean): 0.053 0.163 - compactness (mean): 0.019 0.345 - concavity (mean): 0.0 0.427 - concave points (mean): 0.0 0.201 - symmetry (mean): 0.106 0.304 - fractal dimension (mean): 0.05 0.097 - radius (standard error): 0.112 2.873 - texture (standard error): 0.36 4.885 - perimeter (standard error): 0.757 21.98 - area (standard error): 6.802 542.2 - smoothness (standard error): 0.002 0.031 - compactness (standard error): 0.002 0.135 - concavity (standard error): 0.0 0.396 - concave points (standard error): 0.0 0.053 - symmetry (standard error): 0.008 0.079 - fractal dimension (standard error): 0.001 0.03 - radius (worst): 7.93 36.04 - texture (worst): 12.02 49.54 - perimeter (worst): 50.41 251.2 - area (worst): 185.2 4254.0 - smoothness (worst): 0.071 0.223 - compactness (worst): 0.027 1.058 - concavity (worst): 0.0 1.252 - concave points (worst): 0.0 0.291 - symmetry (worst): 0.156 0.664 - fractal dimension (worst): 0.055 0.208 - ===================================== ====== ====== - - :Missing Attribute Values: None - - :Class Distribution: 212 - Malignant, 357 - Benign - - :Creator: Dr. William H. Wolberg, W. Nick Street, Olvi L. Mangasarian - - :Donor: Nick Street - - :Date: November, 1995 +:Number of Instances: 569 + +:Number of Attributes: 30 numeric, predictive attributes and the class + +:Attribute Information: + - radius (mean of distances from center to points on the perimeter) + - texture (standard deviation of gray-scale values) + - perimeter + - area + - smoothness (local variation in radius lengths) + - compactness (perimeter^2 / area - 1.0) + - concavity (severity of concave portions of the contour) + - concave points (number of concave portions of the contour) + - symmetry + - fractal dimension ("coastline approximation" - 1) + + The mean, standard error, and "worst" or largest (mean of the three + worst/largest values) of these features were computed for each image, + resulting in 30 features. For instance, field 0 is Mean Radius, field + 10 is Radius SE, field 20 is Worst Radius. + + - class: + - WDBC-Malignant + - WDBC-Benign + +:Summary Statistics: + +===================================== ====== ====== + Min Max +===================================== ====== ====== +radius (mean): 6.981 28.11 +texture (mean): 9.71 39.28 +perimeter (mean): 43.79 188.5 +area (mean): 143.5 2501.0 +smoothness (mean): 0.053 0.163 +compactness (mean): 0.019 0.345 +concavity (mean): 0.0 0.427 +concave points (mean): 0.0 0.201 +symmetry (mean): 0.106 0.304 +fractal dimension (mean): 0.05 0.097 +radius (standard error): 0.112 2.873 +texture (standard error): 0.36 4.885 +perimeter (standard error): 0.757 21.98 +area (standard error): 6.802 542.2 +smoothness (standard error): 0.002 0.031 +compactness (standard error): 0.002 0.135 +concavity (standard error): 0.0 0.396 +concave points (standard error): 0.0 0.053 +symmetry (standard error): 0.008 0.079 +fractal dimension (standard error): 0.001 0.03 +radius (worst): 7.93 36.04 +texture (worst): 12.02 49.54 +perimeter (worst): 50.41 251.2 +area (worst): 185.2 4254.0 +smoothness (worst): 0.071 0.223 +compactness (worst): 0.027 1.058 +concavity (worst): 0.0 1.252 +concave points (worst): 0.0 0.291 +symmetry (worst): 0.156 0.664 +fractal dimension (worst): 0.055 0.208 +===================================== ====== ====== + +:Missing Attribute Values: None + +:Class Distribution: 212 - Malignant, 357 - Benign + +:Creator: Dr. William H. Wolberg, W. Nick Street, Olvi L. Mangasarian + +:Donor: Nick Street + +:Date: November, 1995 This is a copy of UCI ML Breast Cancer Wisconsin (Diagnostic) datasets. https://goo.gl/U2Uwz2 @@ -104,15 +104,15 @@ This database is also available through the UW CS ftp server: ftp ftp.cs.wisc.edu cd math-prog/cpo-dataset/machine-learn/WDBC/ -.. topic:: References - - - W.N. Street, W.H. Wolberg and O.L. Mangasarian. Nuclear feature extraction - for breast tumor diagnosis. IS&T/SPIE 1993 International Symposium on - Electronic Imaging: Science and Technology, volume 1905, pages 861-870, - San Jose, CA, 1993. - - O.L. Mangasarian, W.N. Street and W.H. Wolberg. Breast cancer diagnosis and - prognosis via linear programming. Operations Research, 43(4), pages 570-577, - July-August 1995. - - W.H. Wolberg, W.N. Street, and O.L. Mangasarian. Machine learning techniques - to diagnose breast cancer from fine-needle aspirates. Cancer Letters 77 (1994) - 163-171. \ No newline at end of file +.. dropdown:: References + + - W.N. Street, W.H. Wolberg and O.L. Mangasarian. Nuclear feature extraction + for breast tumor diagnosis. IS&T/SPIE 1993 International Symposium on + Electronic Imaging: Science and Technology, volume 1905, pages 861-870, + San Jose, CA, 1993. + - O.L. Mangasarian, W.N. Street and W.H. Wolberg. Breast cancer diagnosis and + prognosis via linear programming. Operations Research, 43(4), pages 570-577, + July-August 1995. + - W.H. Wolberg, W.N. Street, and O.L. Mangasarian. Machine learning techniques + to diagnose breast cancer from fine-needle aspirates. Cancer Letters 77 (1994) + 163-171. diff --git a/sklearn/datasets/descr/california_housing.rst b/sklearn/datasets/descr/california_housing.rst index 9ab3b679b68f5..47a25b9ba272a 100644 --- a/sklearn/datasets/descr/california_housing.rst +++ b/sklearn/datasets/descr/california_housing.rst @@ -5,36 +5,42 @@ California Housing dataset **Data Set Characteristics:** - :Number of Instances: 20640 +:Number of Instances: 20640 - :Number of Attributes: 8 numeric, predictive attributes and the target +:Number of Attributes: 8 numeric, predictive attributes and the target - :Attribute Information: - - MedInc median income in block - - HouseAge median house age in block - - AveRooms average number of rooms - - AveBedrms average number of bedrooms - - Population block population - - AveOccup average house occupancy - - Latitude house block latitude - - Longitude house block longitude +:Attribute Information: + - MedInc median income in block group + - HouseAge median house age in block group + - AveRooms average number of rooms per household + - AveBedrms average number of bedrooms per household + - Population block group population + - AveOccup average number of household members + - Latitude block group latitude + - Longitude block group longitude - :Missing Attribute Values: None +:Missing Attribute Values: None This dataset was obtained from the StatLib repository. -http://lib.stat.cmu.edu/datasets/ +https://www.dcc.fc.up.pt/~ltorgo/Regression/cal_housing.html -The target variable is the median house value for California districts. +The target variable is the median house value for California districts, +expressed in hundreds of thousands of dollars ($100,000). This dataset was derived from the 1990 U.S. census, using one row per census block group. A block group is the smallest geographical unit for which the U.S. Census Bureau publishes sample data (a block group typically has a population of 600 to 3,000 people). +A household is a group of people residing within a home. Since the average +number of rooms and bedrooms in this dataset are provided per household, these +columns may take surprisingly large values for block groups with few households +and many empty houses, such as vacation resorts. + It can be downloaded/loaded using the :func:`sklearn.datasets.fetch_california_housing` function. -.. topic:: References +.. rubric:: References - - Pace, R. Kelley and Ronald Barry, Sparse Spatial Autoregressions, - Statistics and Probability Letters, 33 (1997) 291-297 +- Pace, R. Kelley and Ronald Barry, Sparse Spatial Autoregressions, + Statistics and Probability Letters, 33:291-297, 1997. diff --git a/sklearn/datasets/descr/covtype.rst b/sklearn/datasets/descr/covtype.rst index 0090b8e4a6b7d..f4b752ade17a7 100644 --- a/sklearn/datasets/descr/covtype.rst +++ b/sklearn/datasets/descr/covtype.rst @@ -14,12 +14,12 @@ while others are discrete or continuous measurements. **Data Set Characteristics:** - ================= ============ - Classes 7 - Samples total 581012 - Dimensionality 54 - Features int - ================= ============ +================= ============ +Classes 7 +Samples total 581012 +Dimensionality 54 +Features int +================= ============ :func:`sklearn.datasets.fetch_covtype` will load the covertype dataset; it returns a dictionary-like 'Bunch' object diff --git a/sklearn/datasets/descr/diabetes.rst b/sklearn/datasets/descr/diabetes.rst index 771b3e5fe282a..b977c36cf29a0 100644 --- a/sklearn/datasets/descr/diabetes.rst +++ b/sklearn/datasets/descr/diabetes.rst @@ -10,29 +10,29 @@ quantitative measure of disease progression one year after baseline. **Data Set Characteristics:** - :Number of Instances: 442 +:Number of Instances: 442 - :Number of Attributes: First 10 columns are numeric predictive values +:Number of Attributes: First 10 columns are numeric predictive values - :Target: Column 11 is a quantitative measure of disease progression one year after baseline +:Target: Column 11 is a quantitative measure of disease progression one year after baseline - :Attribute Information: - - age age in years - - sex - - bmi body mass index - - bp average blood pressure - - s1 tc, T-Cells (a type of white blood cells) - - s2 ldl, low-density lipoproteins - - s3 hdl, high-density lipoproteins - - s4 tch, thyroid stimulating hormone - - s5 ltg, lamotrigine - - s6 glu, blood sugar level +:Attribute Information: + - age age in years + - sex + - bmi body mass index + - bp average blood pressure + - s1 tc, total serum cholesterol + - s2 ldl, low-density lipoproteins + - s3 hdl, high-density lipoproteins + - s4 tch, total cholesterol / HDL + - s5 ltg, possibly log of serum triglycerides level + - s6 glu, blood sugar level -Note: Each of these 10 feature variables have been mean centered and scaled by the standard deviation times `n_samples` (i.e. the sum of squares of each column totals 1). +Note: Each of these 10 feature variables have been mean centered and scaled by the standard deviation times the square root of `n_samples` (i.e. the sum of squares of each column totals 1). Source URL: https://www4.stat.ncsu.edu/~boos/var.select/diabetes.html For more information see: Bradley Efron, Trevor Hastie, Iain Johnstone and Robert Tibshirani (2004) "Least Angle Regression," Annals of Statistics (with discussion), 407-499. -(https://web.stanford.edu/~hastie/Papers/LARS/LeastAngle_2002.pdf) \ No newline at end of file +(https://web.stanford.edu/~hastie/Papers/LARS/LeastAngle_2002.pdf) diff --git a/sklearn/datasets/descr/digits.rst b/sklearn/datasets/descr/digits.rst index 244f34f316865..7297584a1b4ac 100644 --- a/sklearn/datasets/descr/digits.rst +++ b/sklearn/datasets/descr/digits.rst @@ -5,12 +5,12 @@ Optical recognition of handwritten digits dataset **Data Set Characteristics:** - :Number of Instances: 1797 - :Number of Attributes: 64 - :Attribute Information: 8x8 image of integer pixels in the range 0..16. - :Missing Attribute Values: None - :Creator: E. Alpaydin (alpaydin '@' boun.edu.tr) - :Date: July; 1998 +:Number of Instances: 1797 +:Number of Attributes: 64 +:Attribute Information: 8x8 image of integer pixels in the range 0..16. +:Missing Attribute Values: None +:Creator: E. Alpaydin (alpaydin '@' boun.edu.tr) +:Date: July; 1998 This is a copy of the test set of the UCI ML hand-written digits datasets https://archive.ics.uci.edu/ml/datasets/Optical+Recognition+of+Handwritten+Digits @@ -32,7 +32,7 @@ T. Candela, D. L. Dimmick, J. Geist, P. J. Grother, S. A. Janet, and C. L. Wilson, NIST Form-Based Handprint Recognition System, NISTIR 5469, 1994. -.. topic:: References +.. dropdown:: References - C. Kaynak (1995) Methods of Combining Multiple Classifiers and Their Applications to Handwritten Digit Recognition, MSc Thesis, Institute of diff --git a/sklearn/datasets/descr/iris.rst b/sklearn/datasets/descr/iris.rst index e05206454d218..98651543620e6 100644 --- a/sklearn/datasets/descr/iris.rst +++ b/sklearn/datasets/descr/iris.rst @@ -5,34 +5,34 @@ Iris plants dataset **Data Set Characteristics:** - :Number of Instances: 150 (50 in each of three classes) - :Number of Attributes: 4 numeric, predictive attributes and the class - :Attribute Information: - - sepal length in cm - - sepal width in cm - - petal length in cm - - petal width in cm - - class: - - Iris-Setosa - - Iris-Versicolour - - Iris-Virginica - - :Summary Statistics: - - ============== ==== ==== ======= ===== ==================== - Min Max Mean SD Class Correlation - ============== ==== ==== ======= ===== ==================== - sepal length: 4.3 7.9 5.84 0.83 0.7826 - sepal width: 2.0 4.4 3.05 0.43 -0.4194 - petal length: 1.0 6.9 3.76 1.76 0.9490 (high!) - petal width: 0.1 2.5 1.20 0.76 0.9565 (high!) - ============== ==== ==== ======= ===== ==================== - - :Missing Attribute Values: None - :Class Distribution: 33.3% for each of 3 classes. - :Creator: R.A. Fisher - :Donor: Michael Marshall (MARSHALL%PLU@io.arc.nasa.gov) - :Date: July, 1988 +:Number of Instances: 150 (50 in each of three classes) +:Number of Attributes: 4 numeric, predictive attributes and the class +:Attribute Information: + - sepal length in cm + - sepal width in cm + - petal length in cm + - petal width in cm + - class: + - Iris-Setosa + - Iris-Versicolour + - Iris-Virginica + +:Summary Statistics: + +============== ==== ==== ======= ===== ==================== + Min Max Mean SD Class Correlation +============== ==== ==== ======= ===== ==================== +sepal length: 4.3 7.9 5.84 0.83 0.7826 +sepal width: 2.0 4.4 3.05 0.43 -0.4194 +petal length: 1.0 6.9 3.76 1.76 0.9490 (high!) +petal width: 0.1 2.5 1.20 0.76 0.9565 (high!) +============== ==== ==== ======= ===== ==================== + +:Missing Attribute Values: None +:Class Distribution: 33.3% for each of 3 classes. +:Creator: R.A. Fisher +:Donor: Michael Marshall (MARSHALL%PLU@io.arc.nasa.gov) +:Date: July, 1988 The famous Iris database, first used by Sir R.A. Fisher. The dataset is taken from Fisher's paper. Note that it's the same as in R, but not as in the UCI @@ -45,19 +45,19 @@ data set contains 3 classes of 50 instances each, where each class refers to a type of iris plant. One class is linearly separable from the other 2; the latter are NOT linearly separable from each other. -.. topic:: References - - - Fisher, R.A. "The use of multiple measurements in taxonomic problems" - Annual Eugenics, 7, Part II, 179-188 (1936); also in "Contributions to - Mathematical Statistics" (John Wiley, NY, 1950). - - Duda, R.O., & Hart, P.E. (1973) Pattern Classification and Scene Analysis. - (Q327.D83) John Wiley & Sons. ISBN 0-471-22361-1. See page 218. - - Dasarathy, B.V. (1980) "Nosing Around the Neighborhood: A New System - Structure and Classification Rule for Recognition in Partially Exposed - Environments". IEEE Transactions on Pattern Analysis and Machine - Intelligence, Vol. PAMI-2, No. 1, 67-71. - - Gates, G.W. (1972) "The Reduced Nearest Neighbor Rule". IEEE Transactions - on Information Theory, May 1972, 431-433. - - See also: 1988 MLC Proceedings, 54-64. Cheeseman et al"s AUTOCLASS II - conceptual clustering system finds 3 classes in the data. - - Many, many more ... \ No newline at end of file +.. dropdown:: References + + - Fisher, R.A. "The use of multiple measurements in taxonomic problems" + Annual Eugenics, 7, Part II, 179-188 (1936); also in "Contributions to + Mathematical Statistics" (John Wiley, NY, 1950). + - Duda, R.O., & Hart, P.E. (1973) Pattern Classification and Scene Analysis. + (Q327.D83) John Wiley & Sons. ISBN 0-471-22361-1. See page 218. + - Dasarathy, B.V. (1980) "Nosing Around the Neighborhood: A New System + Structure and Classification Rule for Recognition in Partially Exposed + Environments". IEEE Transactions on Pattern Analysis and Machine + Intelligence, Vol. PAMI-2, No. 1, 67-71. + - Gates, G.W. (1972) "The Reduced Nearest Neighbor Rule". IEEE Transactions + on Information Theory, May 1972, 431-433. + - See also: 1988 MLC Proceedings, 54-64. Cheeseman et al"s AUTOCLASS II + conceptual clustering system finds 3 classes in the data. + - Many, many more ... diff --git a/sklearn/datasets/descr/kddcup99.rst b/sklearn/datasets/descr/kddcup99.rst index d53a7c878dd17..0eae813be27a9 100644 --- a/sklearn/datasets/descr/kddcup99.rst +++ b/sklearn/datasets/descr/kddcup99.rst @@ -30,50 +30,50 @@ We thus transform the KDD Data set into two different data sets: SA and SF. * http and smtp are two subsets of SF corresponding with third feature equal to 'http' (resp. to 'smtp'). -General KDD structure : - - ================ ========================================== - Samples total 4898431 - Dimensionality 41 - Features discrete (int) or continuous (float) - Targets str, 'normal.' or name of the anomaly type - ================ ========================================== - - SA structure : - - ================ ========================================== - Samples total 976158 - Dimensionality 41 - Features discrete (int) or continuous (float) - Targets str, 'normal.' or name of the anomaly type - ================ ========================================== - - SF structure : - - ================ ========================================== - Samples total 699691 - Dimensionality 4 - Features discrete (int) or continuous (float) - Targets str, 'normal.' or name of the anomaly type - ================ ========================================== - - http structure : - - ================ ========================================== - Samples total 619052 - Dimensionality 3 - Features discrete (int) or continuous (float) - Targets str, 'normal.' or name of the anomaly type - ================ ========================================== - - smtp structure : - - ================ ========================================== - Samples total 95373 - Dimensionality 3 - Features discrete (int) or continuous (float) - Targets str, 'normal.' or name of the anomaly type - ================ ========================================== +General KDD structure: + +================ ========================================== +Samples total 4898431 +Dimensionality 41 +Features discrete (int) or continuous (float) +Targets str, 'normal.' or name of the anomaly type +================ ========================================== + +SA structure: + +================ ========================================== +Samples total 976158 +Dimensionality 41 +Features discrete (int) or continuous (float) +Targets str, 'normal.' or name of the anomaly type +================ ========================================== + +SF structure: + +================ ========================================== +Samples total 699691 +Dimensionality 4 +Features discrete (int) or continuous (float) +Targets str, 'normal.' or name of the anomaly type +================ ========================================== + +http structure: + +================ ========================================== +Samples total 619052 +Dimensionality 3 +Features discrete (int) or continuous (float) +Targets str, 'normal.' or name of the anomaly type +================ ========================================== + +smtp structure: + +================ ========================================== +Samples total 95373 +Dimensionality 3 +Features discrete (int) or continuous (float) +Targets str, 'normal.' or name of the anomaly type +================ ========================================== :func:`sklearn.datasets.fetch_kddcup99` will load the kddcup99 dataset; it returns a dictionary-like object with the feature matrix in the ``data`` member @@ -81,14 +81,14 @@ and the target values in ``target``. The "as_frame" optional argument converts ``data`` into a pandas DataFrame and ``target`` into a pandas Series. The dataset will be downloaded from the web if necessary. -.. topic:: References +.. rubric:: References - .. [2] Analysis and Results of the 1999 DARPA Off-Line Intrusion - Detection Evaluation, Richard Lippmann, Joshua W. Haines, - David J. Fried, Jonathan Korba, Kumar Das. +.. [2] Analysis and Results of the 1999 DARPA Off-Line Intrusion + Detection Evaluation, Richard Lippmann, Joshua W. Haines, + David J. Fried, Jonathan Korba, Kumar Das. - .. [3] K. Yamanishi, J.-I. Takeuchi, G. Williams, and P. Milne. Online - unsupervised outlier detection using finite mixtures with - discounting learning algorithms. In Proceedings of the sixth - ACM SIGKDD international conference on Knowledge discovery - and data mining, pages 320-324. ACM Press, 2000. +.. [3] K. Yamanishi, J.-I. Takeuchi, G. Williams, and P. Milne. Online + unsupervised outlier detection using finite mixtures with + discounting learning algorithms. In Proceedings of the sixth + ACM SIGKDD international conference on Knowledge discovery + and data mining, pages 320-324. ACM Press, 2000. diff --git a/sklearn/datasets/descr/lfw.rst b/sklearn/datasets/descr/lfw.rst index e7fc35c3caabc..bf1da3f4432e6 100644 --- a/sklearn/datasets/descr/lfw.rst +++ b/sklearn/datasets/descr/lfw.rst @@ -4,9 +4,9 @@ The Labeled Faces in the Wild face recognition dataset ------------------------------------------------------ This dataset is a collection of JPEG pictures of famous people collected -over the internet, all details are available on the official website: +over the internet, and the details are available on the Kaggle website: - http://vis-www.cs.umass.edu/lfw/ +https://www.kaggle.com/datasets/jessicali9530/lfw-dataset Each picture is centered on a single face. The typical task is called Face Verification: given a pair of two pictures, a binary classifier @@ -25,102 +25,100 @@ face detector from various online websites. **Data Set Characteristics:** - ================= ======================= - Classes 5749 - Samples total 13233 - Dimensionality 5828 - Features real, between 0 and 255 - ================= ======================= +================= ======================= +Classes 5749 +Samples total 13233 +Dimensionality 5828 +Features real, between 0 and 255 +================= ======================= -Usage -~~~~~ +.. dropdown:: Usage -``scikit-learn`` provides two loaders that will automatically download, -cache, parse the metadata files, decode the jpeg and convert the -interesting slices into memmapped numpy arrays. This dataset size is more -than 200 MB. The first load typically takes more than a couple of minutes -to fully decode the relevant part of the JPEG files into numpy arrays. If -the dataset has been loaded once, the following times the loading times -less than 200ms by using a memmapped version memoized on the disk in the -``~/scikit_learn_data/lfw_home/`` folder using ``joblib``. + ``scikit-learn`` provides two loaders that will automatically download, + cache, parse the metadata files, decode the jpeg and convert the + interesting slices into memmapped numpy arrays. This dataset size is more + than 200 MB. The first load typically takes more than a couple of minutes + to fully decode the relevant part of the JPEG files into numpy arrays. If + the dataset has been loaded once, the following times the loading times + less than 200ms by using a memmapped version memoized on the disk in the + ``~/scikit_learn_data/lfw_home/`` folder using ``joblib``. -The first loader is used for the Face Identification task: a multi-class -classification task (hence supervised learning):: + The first loader is used for the Face Identification task: a multi-class + classification task (hence supervised learning):: - >>> from sklearn.datasets import fetch_lfw_people - >>> lfw_people = fetch_lfw_people(min_faces_per_person=70, resize=0.4) + >>> from sklearn.datasets import fetch_lfw_people + >>> lfw_people = fetch_lfw_people(min_faces_per_person=70, resize=0.4) - >>> for name in lfw_people.target_names: - ... print(name) - ... - Ariel Sharon - Colin Powell - Donald Rumsfeld - George W Bush - Gerhard Schroeder - Hugo Chavez - Tony Blair + >>> for name in lfw_people.target_names: + ... print(name) + ... + Ariel Sharon + Colin Powell + Donald Rumsfeld + George W Bush + Gerhard Schroeder + Hugo Chavez + Tony Blair -The default slice is a rectangular shape around the face, removing -most of the background:: + The default slice is a rectangular shape around the face, removing + most of the background:: - >>> lfw_people.data.dtype - dtype('float32') + >>> lfw_people.data.dtype + dtype('float32') - >>> lfw_people.data.shape - (1288, 1850) + >>> lfw_people.data.shape + (1288, 1850) - >>> lfw_people.images.shape - (1288, 50, 37) + >>> lfw_people.images.shape + (1288, 50, 37) -Each of the ``1140`` faces is assigned to a single person id in the ``target`` -array:: + Each of the ``1140`` faces is assigned to a single person id in the ``target`` + array:: - >>> lfw_people.target.shape - (1288,) + >>> lfw_people.target.shape + (1288,) - >>> list(lfw_people.target[:10]) - [5, 6, 3, 1, 0, 1, 3, 4, 3, 0] + >>> list(lfw_people.target[:10]) + [5, 6, 3, 1, 0, 1, 3, 4, 3, 0] -The second loader is typically used for the face verification task: each sample -is a pair of two picture belonging or not to the same person:: + The second loader is typically used for the face verification task: each sample + is a pair of two picture belonging or not to the same person:: - >>> from sklearn.datasets import fetch_lfw_pairs - >>> lfw_pairs_train = fetch_lfw_pairs(subset='train') + >>> from sklearn.datasets import fetch_lfw_pairs + >>> lfw_pairs_train = fetch_lfw_pairs(subset='train') - >>> list(lfw_pairs_train.target_names) - ['Different persons', 'Same person'] + >>> list(lfw_pairs_train.target_names) + ['Different persons', 'Same person'] - >>> lfw_pairs_train.pairs.shape - (2200, 2, 62, 47) + >>> lfw_pairs_train.pairs.shape + (2200, 2, 62, 47) - >>> lfw_pairs_train.data.shape - (2200, 5828) + >>> lfw_pairs_train.data.shape + (2200, 5828) - >>> lfw_pairs_train.target.shape - (2200,) + >>> lfw_pairs_train.target.shape + (2200,) -Both for the :func:`sklearn.datasets.fetch_lfw_people` and -:func:`sklearn.datasets.fetch_lfw_pairs` function it is -possible to get an additional dimension with the RGB color channels by -passing ``color=True``, in that case the shape will be -``(2200, 2, 62, 47, 3)``. + Both for the :func:`sklearn.datasets.fetch_lfw_people` and + :func:`sklearn.datasets.fetch_lfw_pairs` function it is + possible to get an additional dimension with the RGB color channels by + passing ``color=True``, in that case the shape will be + ``(2200, 2, 62, 47, 3)``. -The :func:`sklearn.datasets.fetch_lfw_pairs` datasets is subdivided into -3 subsets: the development ``train`` set, the development ``test`` set and -an evaluation ``10_folds`` set meant to compute performance metrics using a -10-folds cross validation scheme. + The :func:`sklearn.datasets.fetch_lfw_pairs` datasets is subdivided into + 3 subsets: the development ``train`` set, the development ``test`` set and + an evaluation ``10_folds`` set meant to compute performance metrics using a + 10-folds cross validation scheme. -.. topic:: References: +.. rubric:: References - * `Labeled Faces in the Wild: A Database for Studying Face Recognition - in Unconstrained Environments. - `_ - Gary B. Huang, Manu Ramesh, Tamara Berg, and Erik Learned-Miller. - University of Massachusetts, Amherst, Technical Report 07-49, October, 2007. +* `Labeled Faces in the Wild: A Database for Studying Face Recognition + in Unconstrained Environments. + `_ + Gary B. Huang, Manu Ramesh, Tamara Berg, and Erik Learned-Miller. + University of Massachusetts, Amherst, Technical Report 07-49, October, 2007. -Examples -~~~~~~~~ +.. rubric:: Examples -:ref:`sphx_glr_auto_examples_applications_plot_face_recognition.py` +* :ref:`sphx_glr_auto_examples_applications_plot_face_recognition.py` diff --git a/sklearn/datasets/descr/linnerud.rst b/sklearn/datasets/descr/linnerud.rst index 55eda902448d9..6deb231fe67a9 100644 --- a/sklearn/datasets/descr/linnerud.rst +++ b/sklearn/datasets/descr/linnerud.rst @@ -5,12 +5,12 @@ Linnerrud dataset **Data Set Characteristics:** - :Number of Instances: 20 - :Number of Attributes: 3 - :Missing Attribute Values: None +:Number of Instances: 20 +:Number of Attributes: 3 +:Missing Attribute Values: None The Linnerud dataset is a multi-output regression dataset. It consists of three -excercise (data) and three physiological (target) variables collected from +exercise (data) and three physiological (target) variables collected from twenty middle-aged men in a fitness club: - *physiological* - CSV containing 20 observations on 3 physiological variables: @@ -18,7 +18,7 @@ twenty middle-aged men in a fitness club: - *exercise* - CSV containing 20 observations on 3 exercise variables: Chins, Situps and Jumps. -.. topic:: References +.. dropdown:: References - * Tenenhaus, M. (1998). La regression PLS: theorie et pratique. Paris: - Editions Technic. + * Tenenhaus, M. (1998). La regression PLS: theorie et pratique. Paris: + Editions Technic. diff --git a/sklearn/datasets/descr/olivetti_faces.rst b/sklearn/datasets/descr/olivetti_faces.rst index c6193d5056538..060c866213e8e 100644 --- a/sklearn/datasets/descr/olivetti_faces.rst +++ b/sklearn/datasets/descr/olivetti_faces.rst @@ -3,13 +3,13 @@ The Olivetti faces dataset -------------------------- -`This dataset contains a set of face images`_ taken between April 1992 and +`This dataset contains a set of face images`_ taken between April 1992 and April 1994 at AT&T Laboratories Cambridge. The :func:`sklearn.datasets.fetch_olivetti_faces` function is the data fetching / caching function that downloads the data archive from AT&T. -.. _This dataset contains a set of face images: http://www.cl.cam.ac.uk/research/dtg/attarchive/facedatabase.html +.. _This dataset contains a set of face images: https://cam-orl.co.uk/facedatabase.html As described on the original website: @@ -17,20 +17,20 @@ As described on the original website: subjects, the images were taken at different times, varying the lighting, facial expressions (open / closed eyes, smiling / not smiling) and facial details (glasses / no glasses). All the images were taken against a dark - homogeneous background with the subjects in an upright, frontal position + homogeneous background with the subjects in an upright, frontal position (with tolerance for some side movement). **Data Set Characteristics:** - ================= ===================== - Classes 40 - Samples total 400 - Dimensionality 4096 - Features real, between 0 and 1 - ================= ===================== +================= ===================== +Classes 40 +Samples total 400 +Dimensionality 4096 +Features real, between 0 and 1 +================= ===================== -The image is quantized to 256 grey levels and stored as unsigned 8-bit -integers; the loader will convert these to floating point values on the +The image is quantized to 256 grey levels and stored as unsigned 8-bit +integers; the loader will convert these to floating point values on the interval [0, 1], which are easier to work with for many algorithms. The "target" for this database is an integer from 0 to 39 indicating the diff --git a/sklearn/datasets/descr/rcv1.rst b/sklearn/datasets/descr/rcv1.rst index afaadbfb45afc..3f14cf01934a0 100644 --- a/sklearn/datasets/descr/rcv1.rst +++ b/sklearn/datasets/descr/rcv1.rst @@ -3,20 +3,20 @@ RCV1 dataset ------------ -Reuters Corpus Volume I (RCV1) is an archive of over 800,000 manually -categorized newswire stories made available by Reuters, Ltd. for research +Reuters Corpus Volume I (RCV1) is an archive of over 800,000 manually +categorized newswire stories made available by Reuters, Ltd. for research purposes. The dataset is extensively described in [1]_. **Data Set Characteristics:** - ============== ===================== - Classes 103 - Samples total 804414 - Dimensionality 47236 - Features real, between 0 and 1 - ============== ===================== +============== ===================== +Classes 103 +Samples total 804414 +Dimensionality 47236 +Features real, between 0 and 1 +============== ===================== -:func:`sklearn.datasets.fetch_rcv1` will load the following +:func:`sklearn.datasets.fetch_rcv1` will load the following version: RCV1-v2, vectors, full sets, topics multilabels:: >>> from sklearn.datasets import fetch_rcv1 @@ -28,32 +28,32 @@ It returns a dictionary-like object, with the following attributes: The feature matrix is a scipy CSR sparse matrix, with 804414 samples and 47236 features. Non-zero values contains cosine-normalized, log TF-IDF vectors. A nearly chronological split is proposed in [1]_: The first 23149 samples are -the training set. The last 781265 samples are the testing set. This follows -the official LYRL2004 chronological split. The array has 0.16% of non zero +the training set. The last 781265 samples are the testing set. This follows +the official LYRL2004 chronological split. The array has 0.16% of non zero values:: >>> rcv1.data.shape (804414, 47236) ``target``: -The target values are stored in a scipy CSR sparse matrix, with 804414 samples -and 103 categories. Each sample has a value of 1 in its categories, and 0 in +The target values are stored in a scipy CSR sparse matrix, with 804414 samples +and 103 categories. Each sample has a value of 1 in its categories, and 0 in others. The array has 3.15% of non zero values:: >>> rcv1.target.shape (804414, 103) ``sample_id``: -Each sample can be identified by its ID, ranging (with gaps) from 2286 +Each sample can be identified by its ID, ranging (with gaps) from 2286 to 810596:: >>> rcv1.sample_id[:3] array([2286, 2287, 2288], dtype=uint32) ``target_names``: -The target values are the topics of each sample. Each sample belongs to at -least one topic, and to up to 17 topics. There are 103 topics, each -represented by a string. Their corpus frequencies span five orders of +The target values are the topics of each sample. Each sample belongs to at +least one topic, and to up to 17 topics. There are 103 topics, each +represented by a string. Their corpus frequencies span five orders of magnitude, from 5 occurrences for 'GMIL', to 381327 for 'CCAT':: >>> rcv1.target_names[:3].tolist() # doctest: +SKIP @@ -65,8 +65,8 @@ The compressed size is about 656 MB. .. _rcv1 homepage: http://jmlr.csail.mit.edu/papers/volume5/lewis04a/ -.. topic:: References +.. rubric:: References - .. [1] Lewis, D. D., Yang, Y., Rose, T. G., & Li, F. (2004). - RCV1: A new benchmark collection for text categorization research. - The Journal of Machine Learning Research, 5, 361-397. +.. [1] Lewis, D. D., Yang, Y., Rose, T. G., & Li, F. (2004). + RCV1: A new benchmark collection for text categorization research. + The Journal of Machine Learning Research, 5, 361-397. diff --git a/sklearn/datasets/descr/species_distributions.rst b/sklearn/datasets/descr/species_distributions.rst new file mode 100644 index 0000000000000..a74905681468d --- /dev/null +++ b/sklearn/datasets/descr/species_distributions.rst @@ -0,0 +1,40 @@ +.. _species_distribution_dataset: + +Species distribution dataset +---------------------------- + +This dataset represents the geographic distribution of two species in Central and +South America. The two species are: + +- `"Bradypus variegatus" `_ , + the Brown-throated Sloth. + +- `"Microryzomys minutus" `_ , + also known as the Forest Small Rice Rat, a rodent that lives in Peru, + Colombia, Ecuador, Peru, and Venezuela. + +The dataset is not a typical dataset since a :class:`~sklearn.datasets.base.Bunch` +containing the attributes `data` and `target` is not returned. Instead, we have +information allowing to create a "density" map of the different species. + +The grid for the map can be built using the attributes `x_left_lower_corner`, +`y_left_lower_corner`, `Nx`, `Ny` and `grid_size`, which respectively correspond +to the x and y coordinates of the lower left corner of the grid, the number of +points along the x- and y-axis and the size of the step on the grid. + +The density at each location of the grid is contained in the `coverage` attribute. + +Finally, the `train` and `test` attributes contain information regarding the location +of a species at a specific location. + +The dataset is provided by Phillips et. al. (2006). + +.. rubric:: References + +* `"Maximum entropy modeling of species geographic distributions" + `_ S. J. Phillips, + R. P. Anderson, R. E. Schapire - Ecological Modelling, 190:231-259, 2006. + +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_applications_plot_species_distribution_modeling.py` diff --git a/sklearn/datasets/descr/twenty_newsgroups.rst b/sklearn/datasets/descr/twenty_newsgroups.rst index 6318028b8afa3..e68257b50904e 100644 --- a/sklearn/datasets/descr/twenty_newsgroups.rst +++ b/sklearn/datasets/descr/twenty_newsgroups.rst @@ -20,214 +20,229 @@ extractor. **Data Set Characteristics:** - ================= ========== - Classes 20 - Samples total 18846 - Dimensionality 1 - Features text - ================= ========== - -Usage -~~~~~ - -The :func:`sklearn.datasets.fetch_20newsgroups` function is a data -fetching / caching functions that downloads the data archive from -the original `20 newsgroups website`_, extracts the archive contents -in the ``~/scikit_learn_data/20news_home`` folder and calls the -:func:`sklearn.datasets.load_files` on either the training or -testing set folder, or both of them:: - - >>> from sklearn.datasets import fetch_20newsgroups - >>> newsgroups_train = fetch_20newsgroups(subset='train') - - >>> from pprint import pprint - >>> pprint(list(newsgroups_train.target_names)) - ['alt.atheism', - 'comp.graphics', - 'comp.os.ms-windows.misc', - 'comp.sys.ibm.pc.hardware', - 'comp.sys.mac.hardware', - 'comp.windows.x', - 'misc.forsale', - 'rec.autos', - 'rec.motorcycles', - 'rec.sport.baseball', - 'rec.sport.hockey', - 'sci.crypt', - 'sci.electronics', - 'sci.med', - 'sci.space', - 'soc.religion.christian', - 'talk.politics.guns', - 'talk.politics.mideast', - 'talk.politics.misc', - 'talk.religion.misc'] - -The real data lies in the ``filenames`` and ``target`` attributes. The target -attribute is the integer index of the category:: - - >>> newsgroups_train.filenames.shape - (11314,) - >>> newsgroups_train.target.shape - (11314,) - >>> newsgroups_train.target[:10] - array([ 7, 4, 4, 1, 14, 16, 13, 3, 2, 4]) - -It is possible to load only a sub-selection of the categories by passing the -list of the categories to load to the -:func:`sklearn.datasets.fetch_20newsgroups` function:: - - >>> cats = ['alt.atheism', 'sci.space'] - >>> newsgroups_train = fetch_20newsgroups(subset='train', categories=cats) - - >>> list(newsgroups_train.target_names) - ['alt.atheism', 'sci.space'] - >>> newsgroups_train.filenames.shape - (1073,) - >>> newsgroups_train.target.shape - (1073,) - >>> newsgroups_train.target[:10] - array([0, 1, 1, 1, 0, 1, 1, 0, 0, 0]) - -Converting text to vectors -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In order to feed predictive or clustering models with the text data, -one first need to turn the text into vectors of numerical values suitable -for statistical analysis. This can be achieved with the utilities of the -``sklearn.feature_extraction.text`` as demonstrated in the following -example that extract `TF-IDF`_ vectors of unigram tokens -from a subset of 20news:: - - >>> from sklearn.feature_extraction.text import TfidfVectorizer - >>> categories = ['alt.atheism', 'talk.religion.misc', - ... 'comp.graphics', 'sci.space'] - >>> newsgroups_train = fetch_20newsgroups(subset='train', - ... categories=categories) - >>> vectorizer = TfidfVectorizer() - >>> vectors = vectorizer.fit_transform(newsgroups_train.data) - >>> vectors.shape - (2034, 34118) - -The extracted TF-IDF vectors are very sparse, with an average of 159 non-zero -components by sample in a more than 30000-dimensional space -(less than .5% non-zero features):: - - >>> vectors.nnz / float(vectors.shape[0]) - 159.01327... - -:func:`sklearn.datasets.fetch_20newsgroups_vectorized` is a function which -returns ready-to-use token counts features instead of file names. - -.. _`20 newsgroups website`: http://people.csail.mit.edu/jrennie/20Newsgroups/ -.. _`TF-IDF`: https://en.wikipedia.org/wiki/Tf-idf - - -Filtering text for more realistic training -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -It is easy for a classifier to overfit on particular things that appear in the -20 Newsgroups data, such as newsgroup headers. Many classifiers achieve very -high F-scores, but their results would not generalize to other documents that -aren't from this window of time. - -For example, let's look at the results of a multinomial Naive Bayes classifier, -which is fast to train and achieves a decent F-score:: - - >>> from sklearn.naive_bayes import MultinomialNB - >>> from sklearn import metrics - >>> newsgroups_test = fetch_20newsgroups(subset='test', - ... categories=categories) - >>> vectors_test = vectorizer.transform(newsgroups_test.data) - >>> clf = MultinomialNB(alpha=.01) - >>> clf.fit(vectors, newsgroups_train.target) - MultinomialNB(alpha=0.01, class_prior=None, fit_prior=True) - - >>> pred = clf.predict(vectors_test) - >>> metrics.f1_score(newsgroups_test.target, pred, average='macro') - 0.88213... - -(The example :ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py` shuffles -the training and test data, instead of segmenting by time, and in that case -multinomial Naive Bayes gets a much higher F-score of 0.88. Are you suspicious -yet of what's going on inside this classifier?) - -Let's take a look at what the most informative features are: - - >>> import numpy as np - >>> def show_top10(classifier, vectorizer, categories): - ... feature_names = np.asarray(vectorizer.get_feature_names()) - ... for i, category in enumerate(categories): - ... top10 = np.argsort(classifier.coef_[i])[-10:] - ... print("%s: %s" % (category, " ".join(feature_names[top10]))) - ... - >>> show_top10(clf, vectorizer, newsgroups_train.target_names) - alt.atheism: edu it and in you that is of to the - comp.graphics: edu in graphics it is for and of to the - sci.space: edu it that is in and space to of the - talk.religion.misc: not it you in is that and to of the - - -You can now see many things that these features have overfit to: - -- Almost every group is distinguished by whether headers such as - ``NNTP-Posting-Host:`` and ``Distribution:`` appear more or less often. -- Another significant feature involves whether the sender is affiliated with - a university, as indicated either by their headers or their signature. -- The word "article" is a significant feature, based on how often people quote - previous posts like this: "In article [article ID], [name] <[e-mail address]> - wrote:" -- Other features match the names and e-mail addresses of particular people who - were posting at the time. - -With such an abundance of clues that distinguish newsgroups, the classifiers -barely have to identify topics from text at all, and they all perform at the -same high level. - -For this reason, the functions that load 20 Newsgroups data provide a -parameter called **remove**, telling it what kinds of information to strip out -of each file. **remove** should be a tuple containing any subset of -``('headers', 'footers', 'quotes')``, telling it to remove headers, signature -blocks, and quotation blocks respectively. - - >>> newsgroups_test = fetch_20newsgroups(subset='test', - ... remove=('headers', 'footers', 'quotes'), - ... categories=categories) - >>> vectors_test = vectorizer.transform(newsgroups_test.data) - >>> pred = clf.predict(vectors_test) - >>> metrics.f1_score(pred, newsgroups_test.target, average='macro') - 0.77310... - -This classifier lost over a lot of its F-score, just because we removed -metadata that has little to do with topic classification. -It loses even more if we also strip this metadata from the training data: - - >>> newsgroups_train = fetch_20newsgroups(subset='train', - ... remove=('headers', 'footers', 'quotes'), - ... categories=categories) - >>> vectors = vectorizer.fit_transform(newsgroups_train.data) - >>> clf = MultinomialNB(alpha=.01) - >>> clf.fit(vectors, newsgroups_train.target) - MultinomialNB(alpha=0.01, class_prior=None, fit_prior=True) - - >>> vectors_test = vectorizer.transform(newsgroups_test.data) - >>> pred = clf.predict(vectors_test) - >>> metrics.f1_score(newsgroups_test.target, pred, average='macro') - 0.76995... - -Some other classifiers cope better with this harder version of the task. Try -running :ref:`sphx_glr_auto_examples_model_selection_grid_search_text_feature_extraction.py` with and without -the ``--filter`` option to compare the results. - -.. topic:: Recommendation - - When evaluating text classifiers on the 20 Newsgroups data, you - should strip newsgroup-related metadata. In scikit-learn, you can do this by - setting ``remove=('headers', 'footers', 'quotes')``. The F-score will be +================= ========== +Classes 20 +Samples total 18846 +Dimensionality 1 +Features text +================= ========== + +.. dropdown:: Usage + + The :func:`sklearn.datasets.fetch_20newsgroups` function is a data + fetching / caching functions that downloads the data archive from + the original `20 newsgroups website `__, + extracts the archive contents + in the ``~/scikit_learn_data/20news_home`` folder and calls the + :func:`sklearn.datasets.load_files` on either the training or + testing set folder, or both of them:: + + >>> from sklearn.datasets import fetch_20newsgroups + >>> newsgroups_train = fetch_20newsgroups(subset='train') + + >>> from pprint import pprint + >>> pprint(list(newsgroups_train.target_names)) + ['alt.atheism', + 'comp.graphics', + 'comp.os.ms-windows.misc', + 'comp.sys.ibm.pc.hardware', + 'comp.sys.mac.hardware', + 'comp.windows.x', + 'misc.forsale', + 'rec.autos', + 'rec.motorcycles', + 'rec.sport.baseball', + 'rec.sport.hockey', + 'sci.crypt', + 'sci.electronics', + 'sci.med', + 'sci.space', + 'soc.religion.christian', + 'talk.politics.guns', + 'talk.politics.mideast', + 'talk.politics.misc', + 'talk.religion.misc'] + + The real data lies in the ``filenames`` and ``target`` attributes. The target + attribute is the integer index of the category:: + + >>> newsgroups_train.filenames.shape + (11314,) + >>> newsgroups_train.target.shape + (11314,) + >>> newsgroups_train.target[:10] + array([ 7, 4, 4, 1, 14, 16, 13, 3, 2, 4]) + + It is possible to load only a sub-selection of the categories by passing the + list of the categories to load to the + :func:`sklearn.datasets.fetch_20newsgroups` function:: + + >>> cats = ['alt.atheism', 'sci.space'] + >>> newsgroups_train = fetch_20newsgroups(subset='train', categories=cats) + + >>> list(newsgroups_train.target_names) + ['alt.atheism', 'sci.space'] + >>> newsgroups_train.filenames.shape + (1073,) + >>> newsgroups_train.target.shape + (1073,) + >>> newsgroups_train.target[:10] + array([0, 1, 1, 1, 0, 1, 1, 0, 0, 0]) + +.. dropdown:: Converting text to vectors + + In order to feed predictive or clustering models with the text data, + one first need to turn the text into vectors of numerical values suitable + for statistical analysis. This can be achieved with the utilities of the + ``sklearn.feature_extraction.text`` as demonstrated in the following + example that extract `TF-IDF `__ vectors + of unigram tokens from a subset of 20news:: + + >>> from sklearn.feature_extraction.text import TfidfVectorizer + >>> categories = ['alt.atheism', 'talk.religion.misc', + ... 'comp.graphics', 'sci.space'] + >>> newsgroups_train = fetch_20newsgroups(subset='train', + ... categories=categories) + >>> vectorizer = TfidfVectorizer() + >>> vectors = vectorizer.fit_transform(newsgroups_train.data) + >>> vectors.shape + (2034, 34118) + + The extracted TF-IDF vectors are very sparse, with an average of 159 non-zero + components by sample in a more than 30000-dimensional space + (less than .5% non-zero features):: + + >>> vectors.nnz / float(vectors.shape[0]) + 159.01327... + + :func:`sklearn.datasets.fetch_20newsgroups_vectorized` is a function which + returns ready-to-use token counts features instead of file names. + +.. dropdown:: Filtering text for more realistic training + + It is easy for a classifier to overfit on particular things that appear in the + 20 Newsgroups data, such as newsgroup headers. Many classifiers achieve very + high F-scores, but their results would not generalize to other documents that + aren't from this window of time. + + For example, let's look at the results of a multinomial Naive Bayes classifier, + which is fast to train and achieves a decent F-score:: + + >>> from sklearn.naive_bayes import MultinomialNB + >>> from sklearn import metrics + >>> newsgroups_test = fetch_20newsgroups(subset='test', + ... categories=categories) + >>> vectors_test = vectorizer.transform(newsgroups_test.data) + >>> clf = MultinomialNB(alpha=.01) + >>> clf.fit(vectors, newsgroups_train.target) + MultinomialNB(alpha=0.01, class_prior=None, fit_prior=True) + + >>> pred = clf.predict(vectors_test) + >>> metrics.f1_score(newsgroups_test.target, pred, average='macro') + 0.88213... + + (The example :ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py` shuffles + the training and test data, instead of segmenting by time, and in that case + multinomial Naive Bayes gets a much higher F-score of 0.88. Are you suspicious + yet of what's going on inside this classifier?) + + Let's take a look at what the most informative features are: + + >>> import numpy as np + >>> def show_top10(classifier, vectorizer, categories): + ... feature_names = vectorizer.get_feature_names_out() + ... for i, category in enumerate(categories): + ... top10 = np.argsort(classifier.coef_[i])[-10:] + ... print("%s: %s" % (category, " ".join(feature_names[top10]))) + ... + >>> show_top10(clf, vectorizer, newsgroups_train.target_names) + alt.atheism: edu it and in you that is of to the + comp.graphics: edu in graphics it is for and of to the + sci.space: edu it that is in and space to of the + talk.religion.misc: not it you in is that and to of the + + + You can now see many things that these features have overfit to: + + - Almost every group is distinguished by whether headers such as + ``NNTP-Posting-Host:`` and ``Distribution:`` appear more or less often. + - Another significant feature involves whether the sender is affiliated with + a university, as indicated either by their headers or their signature. + - The word "article" is a significant feature, based on how often people quote + previous posts like this: "In article [article ID], [name] <[e-mail address]> + wrote:" + - Other features match the names and e-mail addresses of particular people who + were posting at the time. + + With such an abundance of clues that distinguish newsgroups, the classifiers + barely have to identify topics from text at all, and they all perform at the + same high level. + + For this reason, the functions that load 20 Newsgroups data provide a + parameter called **remove**, telling it what kinds of information to strip out + of each file. **remove** should be a tuple containing any subset of + ``('headers', 'footers', 'quotes')``, telling it to remove headers, signature + blocks, and quotation blocks respectively. + + >>> newsgroups_test = fetch_20newsgroups(subset='test', + ... remove=('headers', 'footers', 'quotes'), + ... categories=categories) + >>> vectors_test = vectorizer.transform(newsgroups_test.data) + >>> pred = clf.predict(vectors_test) + >>> metrics.f1_score(pred, newsgroups_test.target, average='macro') + 0.77310... + + This classifier lost over a lot of its F-score, just because we removed + metadata that has little to do with topic classification. + It loses even more if we also strip this metadata from the training data: + + >>> newsgroups_train = fetch_20newsgroups(subset='train', + ... remove=('headers', 'footers', 'quotes'), + ... categories=categories) + >>> vectors = vectorizer.fit_transform(newsgroups_train.data) + >>> clf = MultinomialNB(alpha=.01) + >>> clf.fit(vectors, newsgroups_train.target) + MultinomialNB(alpha=0.01, class_prior=None, fit_prior=True) + + >>> vectors_test = vectorizer.transform(newsgroups_test.data) + >>> pred = clf.predict(vectors_test) + >>> metrics.f1_score(newsgroups_test.target, pred, average='macro') + 0.76995... + + Some other classifiers cope better with this harder version of the task. Try the + :ref:`sphx_glr_auto_examples_model_selection_plot_grid_search_text_feature_extraction.py` + example with and without the `remove` option to compare the results. + +.. rubric:: Data Considerations + +The Cleveland Indians is a major league baseball team based in Cleveland, +Ohio, USA. In December 2020, it was reported that "After several months of +discussion sparked by the death of George Floyd and a national reckoning over +race and colonialism, the Cleveland Indians have decided to change their +name." Team owner Paul Dolan "did make it clear that the team will not make +its informal nickname -- the Tribe -- its new team name." "It's not going to +be a half-step away from the Indians," Dolan said."We will not have a Native +American-themed name." + +https://www.mlb.com/news/cleveland-indians-team-name-change + +.. rubric:: Recommendation + +- When evaluating text classifiers on the 20 Newsgroups data, you + should strip newsgroup-related metadata. In scikit-learn, you can do this + by setting ``remove=('headers', 'footers', 'quotes')``. The F-score will be lower because it is more realistic. - -.. topic:: Examples - - * :ref:`sphx_glr_auto_examples_model_selection_grid_search_text_feature_extraction.py` - - * :ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py` +- This text dataset contains data which may be inappropriate for certain NLP + applications. An example is listed in the "Data Considerations" section + above. The challenge with using current text datasets in NLP for tasks such + as sentence completion, clustering, and other applications is that text + that is culturally biased and inflammatory will propagate biases. This + should be taken into consideration when using the dataset, reviewing the + output, and the bias should be documented. + +.. rubric:: Examples + +* :ref:`sphx_glr_auto_examples_model_selection_plot_grid_search_text_feature_extraction.py` +* :ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py` +* :ref:`sphx_glr_auto_examples_text_plot_hashing_vs_dict_vectorizer.py` +* :ref:`sphx_glr_auto_examples_text_plot_document_clustering.py` diff --git a/sklearn/datasets/descr/wine_data.rst b/sklearn/datasets/descr/wine_data.rst index bfde9288fa4dd..64efe49900ebf 100644 --- a/sklearn/datasets/descr/wine_data.rst +++ b/sklearn/datasets/descr/wine_data.rst @@ -5,53 +5,52 @@ Wine recognition dataset **Data Set Characteristics:** - :Number of Instances: 178 (50 in each of three classes) - :Number of Attributes: 13 numeric, predictive attributes and the class - :Attribute Information: - - Alcohol - - Malic acid - - Ash - - Alcalinity of ash - - Magnesium - - Total phenols - - Flavanoids - - Nonflavanoid phenols - - Proanthocyanins - - Color intensity - - Hue - - OD280/OD315 of diluted wines - - Proline - +:Number of Instances: 178 +:Number of Attributes: 13 numeric, predictive attributes and the class +:Attribute Information: + - Alcohol + - Malic acid + - Ash + - Alcalinity of ash + - Magnesium + - Total phenols + - Flavanoids + - Nonflavanoid phenols + - Proanthocyanins + - Color intensity + - Hue + - OD280/OD315 of diluted wines + - Proline - class: - - class_0 - - class_1 - - class_2 - - :Summary Statistics: - - ============================= ==== ===== ======= ===== - Min Max Mean SD - ============================= ==== ===== ======= ===== - Alcohol: 11.0 14.8 13.0 0.8 - Malic Acid: 0.74 5.80 2.34 1.12 - Ash: 1.36 3.23 2.36 0.27 - Alcalinity of Ash: 10.6 30.0 19.5 3.3 - Magnesium: 70.0 162.0 99.7 14.3 - Total Phenols: 0.98 3.88 2.29 0.63 - Flavanoids: 0.34 5.08 2.03 1.00 - Nonflavanoid Phenols: 0.13 0.66 0.36 0.12 - Proanthocyanins: 0.41 3.58 1.59 0.57 - Colour Intensity: 1.3 13.0 5.1 2.3 - Hue: 0.48 1.71 0.96 0.23 - OD280/OD315 of diluted wines: 1.27 4.00 2.61 0.71 - Proline: 278 1680 746 315 - ============================= ==== ===== ======= ===== - - :Missing Attribute Values: None - :Class Distribution: class_0 (59), class_1 (71), class_2 (48) - :Creator: R.A. Fisher - :Donor: Michael Marshall (MARSHALL%PLU@io.arc.nasa.gov) - :Date: July, 1988 + - class_0 + - class_1 + - class_2 + +:Summary Statistics: + +============================= ==== ===== ======= ===== + Min Max Mean SD +============================= ==== ===== ======= ===== +Alcohol: 11.0 14.8 13.0 0.8 +Malic Acid: 0.74 5.80 2.34 1.12 +Ash: 1.36 3.23 2.36 0.27 +Alcalinity of Ash: 10.6 30.0 19.5 3.3 +Magnesium: 70.0 162.0 99.7 14.3 +Total Phenols: 0.98 3.88 2.29 0.63 +Flavanoids: 0.34 5.08 2.03 1.00 +Nonflavanoid Phenols: 0.13 0.66 0.36 0.12 +Proanthocyanins: 0.41 3.58 1.59 0.57 +Colour Intensity: 1.3 13.0 5.1 2.3 +Hue: 0.48 1.71 0.96 0.23 +OD280/OD315 of diluted wines: 1.27 4.00 2.61 0.71 +Proline: 278 1680 746 315 +============================= ==== ===== ======= ===== + +:Missing Attribute Values: None +:Class Distribution: class_0 (59), class_1 (71), class_2 (48) +:Creator: R.A. Fisher +:Donor: Michael Marshall (MARSHALL%PLU@io.arc.nasa.gov) +:Date: July, 1988 This is a copy of UCI ML Wine recognition datasets. https://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data @@ -61,10 +60,10 @@ region in Italy by three different cultivators. There are thirteen different measurements taken for different constituents found in the three types of wine. -Original Owners: +Original Owners: -Forina, M. et al, PARVUS - -An Extendible Package for Data Exploration, Classification and Correlation. +Forina, M. et al, PARVUS - +An Extendible Package for Data Exploration, Classification and Correlation. Institute of Pharmaceutical and Food Analysis and Technologies, Via Brigata Salerno, 16147 Genoa, Italy. @@ -72,24 +71,24 @@ Citation: Lichman, M. (2013). UCI Machine Learning Repository [https://archive.ics.uci.edu/ml]. Irvine, CA: University of California, -School of Information and Computer Science. - -.. topic:: References - - (1) S. Aeberhard, D. Coomans and O. de Vel, - Comparison of Classifiers in High Dimensional Settings, - Tech. Rep. no. 92-02, (1992), Dept. of Computer Science and Dept. of - Mathematics and Statistics, James Cook University of North Queensland. - (Also submitted to Technometrics). - - The data was used with many others for comparing various - classifiers. The classes are separable, though only RDA - has achieved 100% correct classification. - (RDA : 100%, QDA 99.4%, LDA 98.9%, 1NN 96.1% (z-transformed data)) - (All results using the leave-one-out technique) - - (2) S. Aeberhard, D. Coomans and O. de Vel, - "THE CLASSIFICATION PERFORMANCE OF RDA" - Tech. Rep. no. 92-01, (1992), Dept. of Computer Science and Dept. of - Mathematics and Statistics, James Cook University of North Queensland. - (Also submitted to Journal of Chemometrics). +School of Information and Computer Science. + +.. dropdown:: References + + (1) S. Aeberhard, D. Coomans and O. de Vel, + Comparison of Classifiers in High Dimensional Settings, + Tech. Rep. no. 92-02, (1992), Dept. of Computer Science and Dept. of + Mathematics and Statistics, James Cook University of North Queensland. + (Also submitted to Technometrics). + + The data was used with many others for comparing various + classifiers. The classes are separable, though only RDA + has achieved 100% correct classification. + (RDA : 100%, QDA 99.4%, LDA 98.9%, 1NN 96.1% (z-transformed data)) + (All results using the leave-one-out technique) + + (2) S. Aeberhard, D. Coomans and O. de Vel, + "THE CLASSIFICATION PERFORMANCE OF RDA" + Tech. Rep. no. 92-01, (1992), Dept. of Computer Science and Dept. of + Mathematics and Statistics, James Cook University of North Queensland. + (Also submitted to Journal of Chemometrics). diff --git a/sklearn/datasets/images/README.txt b/sklearn/datasets/images/README.txt index a95a5d42500d4..e699e7d6836e6 100644 --- a/sklearn/datasets/images/README.txt +++ b/sklearn/datasets/images/README.txt @@ -16,6 +16,3 @@ Retrieved 21st August, 2011 from [3] by Robert Layton [1] https://creativecommons.org/licenses/by/2.0/ [2] https://www.flickr.com/photos/vultilion/ [3] https://www.flickr.com/photos/vultilion/6056698931/sizes/z/in/photostream/ - - - diff --git a/sklearn/datasets/images/__init__.py b/sklearn/datasets/images/__init__.py new file mode 100644 index 0000000000000..67dd18fb94b59 --- /dev/null +++ b/sklearn/datasets/images/__init__.py @@ -0,0 +1,2 @@ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause diff --git a/sklearn/datasets/meson.build b/sklearn/datasets/meson.build new file mode 100644 index 0000000000000..4efcd279315de --- /dev/null +++ b/sklearn/datasets/meson.build @@ -0,0 +1,7 @@ +py.extension_module( + '_svmlight_format_fast', + cython_gen.process('_svmlight_format_fast.pyx'), + dependencies: [np_dep], + subdir: 'sklearn/datasets', + install: true +) diff --git a/sklearn/datasets/setup.py b/sklearn/datasets/setup.py deleted file mode 100644 index 1107505d42070..0000000000000 --- a/sklearn/datasets/setup.py +++ /dev/null @@ -1,24 +0,0 @@ - -import numpy -import os -import platform - - -def configuration(parent_package='', top_path=None): - from numpy.distutils.misc_util import Configuration - config = Configuration('datasets', parent_package, top_path) - config.add_data_dir('data') - config.add_data_dir('descr') - config.add_data_dir('images') - config.add_data_dir(os.path.join('tests', 'data')) - if platform.python_implementation() != 'PyPy': - config.add_extension('_svmlight_format_fast', - sources=['_svmlight_format_fast.pyx'], - include_dirs=[numpy.get_include()]) - config.add_subpackage('tests') - return config - - -if __name__ == '__main__': - from numpy.distutils.core import setup - setup(**configuration(top_path='').todict()) diff --git a/sklearn/datasets/tests/conftest.py b/sklearn/datasets/tests/conftest.py deleted file mode 100644 index cf356d6ca3b10..0000000000000 --- a/sklearn/datasets/tests/conftest.py +++ /dev/null @@ -1,17 +0,0 @@ -""" Network tests are only run, if data is already locally available, -or if download is specifically requested by environment variable.""" -import builtins -import pytest - - -@pytest.fixture -def hide_available_pandas(monkeypatch): - """ Pretend pandas was not installed. """ - import_orig = builtins.__import__ - - def mocked_import(name, *args, **kwargs): - if name == 'pandas': - raise ImportError() - return import_orig(name, *args, **kwargs) - - monkeypatch.setattr(builtins, '__import__', mocked_import) diff --git a/sklearn/datasets/tests/data/__init__.py b/sklearn/datasets/tests/data/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/sklearn/datasets/tests/data/openml/__init__.py b/sklearn/datasets/tests/data/openml/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/sklearn/datasets/tests/data/openml/id_1/__init__.py b/sklearn/datasets/tests/data/openml/id_1/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/sklearn/datasets/tests/data/openml/1/api-v1-json-data-1.json.gz b/sklearn/datasets/tests/data/openml/id_1/api-v1-jd-1.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/1/api-v1-json-data-1.json.gz rename to sklearn/datasets/tests/data/openml/id_1/api-v1-jd-1.json.gz diff --git a/sklearn/datasets/tests/data/openml/1/api-v1-json-data-features-1.json.gz b/sklearn/datasets/tests/data/openml/id_1/api-v1-jdf-1.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/1/api-v1-json-data-features-1.json.gz rename to sklearn/datasets/tests/data/openml/id_1/api-v1-jdf-1.json.gz diff --git a/sklearn/datasets/tests/data/openml/1/api-v1-json-data-qualities-1.json.gz b/sklearn/datasets/tests/data/openml/id_1/api-v1-jdq-1.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/1/api-v1-json-data-qualities-1.json.gz rename to sklearn/datasets/tests/data/openml/id_1/api-v1-jdq-1.json.gz diff --git a/sklearn/datasets/tests/data/openml/1/data-v1-download-1.arff.gz b/sklearn/datasets/tests/data/openml/id_1/data-v1-dl-1.arff.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/1/data-v1-download-1.arff.gz rename to sklearn/datasets/tests/data/openml/id_1/data-v1-dl-1.arff.gz diff --git a/sklearn/datasets/tests/data/openml/id_1119/__init__.py b/sklearn/datasets/tests/data/openml/id_1119/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/sklearn/datasets/tests/data/openml/1119/api-v1-json-data-1119.json.gz b/sklearn/datasets/tests/data/openml/id_1119/api-v1-jd-1119.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/1119/api-v1-json-data-1119.json.gz rename to sklearn/datasets/tests/data/openml/id_1119/api-v1-jd-1119.json.gz diff --git a/sklearn/datasets/tests/data/openml/1119/api-v1-json-data-features-1119.json.gz b/sklearn/datasets/tests/data/openml/id_1119/api-v1-jdf-1119.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/1119/api-v1-json-data-features-1119.json.gz rename to sklearn/datasets/tests/data/openml/id_1119/api-v1-jdf-1119.json.gz diff --git a/sklearn/datasets/tests/data/openml/1119/api-v1-json-data-list-data_name-adult-census-limit-2-data_version-1.json.gz b/sklearn/datasets/tests/data/openml/id_1119/api-v1-jdl-dn-adult-census-l-2-dv-1.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/1119/api-v1-json-data-list-data_name-adult-census-limit-2-data_version-1.json.gz rename to sklearn/datasets/tests/data/openml/id_1119/api-v1-jdl-dn-adult-census-l-2-dv-1.json.gz diff --git a/sklearn/datasets/tests/data/openml/1119/api-v1-json-data-list-data_name-adult-census-limit-2-status-active-.json.gz b/sklearn/datasets/tests/data/openml/id_1119/api-v1-jdl-dn-adult-census-l-2-s-act-.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/1119/api-v1-json-data-list-data_name-adult-census-limit-2-status-active-.json.gz rename to sklearn/datasets/tests/data/openml/id_1119/api-v1-jdl-dn-adult-census-l-2-s-act-.json.gz diff --git a/sklearn/datasets/tests/data/openml/1119/api-v1-json-data-qualities-1119.json.gz b/sklearn/datasets/tests/data/openml/id_1119/api-v1-jdq-1119.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/1119/api-v1-json-data-qualities-1119.json.gz rename to sklearn/datasets/tests/data/openml/id_1119/api-v1-jdq-1119.json.gz diff --git a/sklearn/datasets/tests/data/openml/1119/data-v1-download-54002.arff.gz b/sklearn/datasets/tests/data/openml/id_1119/data-v1-dl-54002.arff.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/1119/data-v1-download-54002.arff.gz rename to sklearn/datasets/tests/data/openml/id_1119/data-v1-dl-54002.arff.gz diff --git a/sklearn/datasets/tests/data/openml/id_1590/__init__.py b/sklearn/datasets/tests/data/openml/id_1590/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/sklearn/datasets/tests/data/openml/id_1590/api-v1-jd-1590.json.gz b/sklearn/datasets/tests/data/openml/id_1590/api-v1-jd-1590.json.gz new file mode 100644 index 0000000000000..8a1fa63fef0f6 Binary files /dev/null and b/sklearn/datasets/tests/data/openml/id_1590/api-v1-jd-1590.json.gz differ diff --git a/sklearn/datasets/tests/data/openml/id_1590/api-v1-jdf-1590.json.gz b/sklearn/datasets/tests/data/openml/id_1590/api-v1-jdf-1590.json.gz new file mode 100644 index 0000000000000..667376beec461 Binary files /dev/null and b/sklearn/datasets/tests/data/openml/id_1590/api-v1-jdf-1590.json.gz differ diff --git a/sklearn/datasets/tests/data/openml/id_1590/api-v1-jdq-1590.json.gz b/sklearn/datasets/tests/data/openml/id_1590/api-v1-jdq-1590.json.gz new file mode 100644 index 0000000000000..b88b58d21b8bc Binary files /dev/null and b/sklearn/datasets/tests/data/openml/id_1590/api-v1-jdq-1590.json.gz differ diff --git a/sklearn/datasets/tests/data/openml/id_1590/data-v1-dl-1595261.arff.gz b/sklearn/datasets/tests/data/openml/id_1590/data-v1-dl-1595261.arff.gz new file mode 100644 index 0000000000000..e1131f1ce42de Binary files /dev/null and b/sklearn/datasets/tests/data/openml/id_1590/data-v1-dl-1595261.arff.gz differ diff --git a/sklearn/datasets/tests/data/openml/id_2/__init__.py b/sklearn/datasets/tests/data/openml/id_2/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/sklearn/datasets/tests/data/openml/2/api-v1-json-data-2.json.gz b/sklearn/datasets/tests/data/openml/id_2/api-v1-jd-2.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/2/api-v1-json-data-2.json.gz rename to sklearn/datasets/tests/data/openml/id_2/api-v1-jd-2.json.gz diff --git a/sklearn/datasets/tests/data/openml/2/api-v1-json-data-features-2.json.gz b/sklearn/datasets/tests/data/openml/id_2/api-v1-jdf-2.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/2/api-v1-json-data-features-2.json.gz rename to sklearn/datasets/tests/data/openml/id_2/api-v1-jdf-2.json.gz diff --git a/sklearn/datasets/tests/data/openml/2/api-v1-json-data-list-data_name-anneal-limit-2-data_version-1.json.gz b/sklearn/datasets/tests/data/openml/id_2/api-v1-jdl-dn-anneal-l-2-dv-1.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/2/api-v1-json-data-list-data_name-anneal-limit-2-data_version-1.json.gz rename to sklearn/datasets/tests/data/openml/id_2/api-v1-jdl-dn-anneal-l-2-dv-1.json.gz diff --git a/sklearn/datasets/tests/data/openml/2/api-v1-json-data-list-data_name-anneal-limit-2-status-active-.json.gz b/sklearn/datasets/tests/data/openml/id_2/api-v1-jdl-dn-anneal-l-2-s-act-.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/2/api-v1-json-data-list-data_name-anneal-limit-2-status-active-.json.gz rename to sklearn/datasets/tests/data/openml/id_2/api-v1-jdl-dn-anneal-l-2-s-act-.json.gz diff --git a/sklearn/datasets/tests/data/openml/2/api-v1-json-data-qualities-2.json.gz b/sklearn/datasets/tests/data/openml/id_2/api-v1-jdq-2.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/2/api-v1-json-data-qualities-2.json.gz rename to sklearn/datasets/tests/data/openml/id_2/api-v1-jdq-2.json.gz diff --git a/sklearn/datasets/tests/data/openml/2/data-v1-download-1666876.arff.gz b/sklearn/datasets/tests/data/openml/id_2/data-v1-dl-1666876.arff.gz similarity index 96% rename from sklearn/datasets/tests/data/openml/2/data-v1-download-1666876.arff.gz rename to sklearn/datasets/tests/data/openml/id_2/data-v1-dl-1666876.arff.gz index cdf3254add760..2144153771bfa 100644 Binary files a/sklearn/datasets/tests/data/openml/2/data-v1-download-1666876.arff.gz and b/sklearn/datasets/tests/data/openml/id_2/data-v1-dl-1666876.arff.gz differ diff --git a/sklearn/datasets/tests/data/openml/id_292/__init__.py b/sklearn/datasets/tests/data/openml/id_292/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/sklearn/datasets/tests/data/openml/292/api-v1-json-data-292.json.gz b/sklearn/datasets/tests/data/openml/id_292/api-v1-jd-292.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/292/api-v1-json-data-292.json.gz rename to sklearn/datasets/tests/data/openml/id_292/api-v1-jd-292.json.gz diff --git a/sklearn/datasets/tests/data/openml/292/api-v1-json-data-40981.json.gz b/sklearn/datasets/tests/data/openml/id_292/api-v1-jd-40981.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/292/api-v1-json-data-40981.json.gz rename to sklearn/datasets/tests/data/openml/id_292/api-v1-jd-40981.json.gz diff --git a/sklearn/datasets/tests/data/openml/292/api-v1-json-data-features-292.json.gz b/sklearn/datasets/tests/data/openml/id_292/api-v1-jdf-292.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/292/api-v1-json-data-features-292.json.gz rename to sklearn/datasets/tests/data/openml/id_292/api-v1-jdf-292.json.gz diff --git a/sklearn/datasets/tests/data/openml/292/api-v1-json-data-features-40981.json.gz b/sklearn/datasets/tests/data/openml/id_292/api-v1-jdf-40981.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/292/api-v1-json-data-features-40981.json.gz rename to sklearn/datasets/tests/data/openml/id_292/api-v1-jdf-40981.json.gz diff --git a/sklearn/datasets/tests/data/openml/292/api-v1-json-data-list-data_name-australian-limit-2-data_version-1-status-deactivated.json.gz b/sklearn/datasets/tests/data/openml/id_292/api-v1-jdl-dn-australian-l-2-dv-1-s-dact.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/292/api-v1-json-data-list-data_name-australian-limit-2-data_version-1-status-deactivated.json.gz rename to sklearn/datasets/tests/data/openml/id_292/api-v1-jdl-dn-australian-l-2-dv-1-s-dact.json.gz diff --git a/sklearn/datasets/tests/data/openml/292/api-v1-json-data-list-data_name-australian-limit-2-data_version-1.json.gz b/sklearn/datasets/tests/data/openml/id_292/api-v1-jdl-dn-australian-l-2-dv-1.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/292/api-v1-json-data-list-data_name-australian-limit-2-data_version-1.json.gz rename to sklearn/datasets/tests/data/openml/id_292/api-v1-jdl-dn-australian-l-2-dv-1.json.gz diff --git a/sklearn/datasets/tests/data/openml/292/api-v1-json-data-list-data_name-australian-limit-2-status-active-.json.gz b/sklearn/datasets/tests/data/openml/id_292/api-v1-jdl-dn-australian-l-2-s-act-.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/292/api-v1-json-data-list-data_name-australian-limit-2-status-active-.json.gz rename to sklearn/datasets/tests/data/openml/id_292/api-v1-jdl-dn-australian-l-2-s-act-.json.gz diff --git a/sklearn/datasets/tests/data/openml/292/data-v1-download-49822.arff.gz b/sklearn/datasets/tests/data/openml/id_292/data-v1-dl-49822.arff.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/292/data-v1-download-49822.arff.gz rename to sklearn/datasets/tests/data/openml/id_292/data-v1-dl-49822.arff.gz diff --git a/sklearn/datasets/tests/data/openml/id_3/__init__.py b/sklearn/datasets/tests/data/openml/id_3/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/sklearn/datasets/tests/data/openml/3/api-v1-json-data-3.json.gz b/sklearn/datasets/tests/data/openml/id_3/api-v1-jd-3.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/3/api-v1-json-data-3.json.gz rename to sklearn/datasets/tests/data/openml/id_3/api-v1-jd-3.json.gz diff --git a/sklearn/datasets/tests/data/openml/3/api-v1-json-data-features-3.json.gz b/sklearn/datasets/tests/data/openml/id_3/api-v1-jdf-3.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/3/api-v1-json-data-features-3.json.gz rename to sklearn/datasets/tests/data/openml/id_3/api-v1-jdf-3.json.gz diff --git a/sklearn/datasets/tests/data/openml/3/api-v1-json-data-qualities-3.json.gz b/sklearn/datasets/tests/data/openml/id_3/api-v1-jdq-3.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/3/api-v1-json-data-qualities-3.json.gz rename to sklearn/datasets/tests/data/openml/id_3/api-v1-jdq-3.json.gz diff --git a/sklearn/datasets/tests/data/openml/3/data-v1-download-3.arff.gz b/sklearn/datasets/tests/data/openml/id_3/data-v1-dl-3.arff.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/3/data-v1-download-3.arff.gz rename to sklearn/datasets/tests/data/openml/id_3/data-v1-dl-3.arff.gz diff --git a/sklearn/datasets/tests/data/openml/id_40589/__init__.py b/sklearn/datasets/tests/data/openml/id_40589/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/sklearn/datasets/tests/data/openml/40589/api-v1-json-data-40589.json.gz b/sklearn/datasets/tests/data/openml/id_40589/api-v1-jd-40589.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/40589/api-v1-json-data-40589.json.gz rename to sklearn/datasets/tests/data/openml/id_40589/api-v1-jd-40589.json.gz diff --git a/sklearn/datasets/tests/data/openml/40589/api-v1-json-data-features-40589.json.gz b/sklearn/datasets/tests/data/openml/id_40589/api-v1-jdf-40589.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/40589/api-v1-json-data-features-40589.json.gz rename to sklearn/datasets/tests/data/openml/id_40589/api-v1-jdf-40589.json.gz diff --git a/sklearn/datasets/tests/data/openml/40589/api-v1-json-data-list-data_name-emotions-limit-2-data_version-3.json.gz b/sklearn/datasets/tests/data/openml/id_40589/api-v1-jdl-dn-emotions-l-2-dv-3.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/40589/api-v1-json-data-list-data_name-emotions-limit-2-data_version-3.json.gz rename to sklearn/datasets/tests/data/openml/id_40589/api-v1-jdl-dn-emotions-l-2-dv-3.json.gz diff --git a/sklearn/datasets/tests/data/openml/40589/api-v1-json-data-list-data_name-emotions-limit-2-status-active-.json.gz b/sklearn/datasets/tests/data/openml/id_40589/api-v1-jdl-dn-emotions-l-2-s-act-.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/40589/api-v1-json-data-list-data_name-emotions-limit-2-status-active-.json.gz rename to sklearn/datasets/tests/data/openml/id_40589/api-v1-jdl-dn-emotions-l-2-s-act-.json.gz diff --git a/sklearn/datasets/tests/data/openml/40589/api-v1-json-data-qualities-40589.json.gz b/sklearn/datasets/tests/data/openml/id_40589/api-v1-jdq-40589.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/40589/api-v1-json-data-qualities-40589.json.gz rename to sklearn/datasets/tests/data/openml/id_40589/api-v1-jdq-40589.json.gz diff --git a/sklearn/datasets/tests/data/openml/40589/data-v1-download-4644182.arff.gz b/sklearn/datasets/tests/data/openml/id_40589/data-v1-dl-4644182.arff.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/40589/data-v1-download-4644182.arff.gz rename to sklearn/datasets/tests/data/openml/id_40589/data-v1-dl-4644182.arff.gz diff --git a/sklearn/datasets/tests/data/openml/id_40675/__init__.py b/sklearn/datasets/tests/data/openml/id_40675/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/sklearn/datasets/tests/data/openml/40675/api-v1-json-data-40675.json.gz b/sklearn/datasets/tests/data/openml/id_40675/api-v1-jd-40675.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/40675/api-v1-json-data-40675.json.gz rename to sklearn/datasets/tests/data/openml/id_40675/api-v1-jd-40675.json.gz diff --git a/sklearn/datasets/tests/data/openml/40675/api-v1-json-data-features-40675.json.gz b/sklearn/datasets/tests/data/openml/id_40675/api-v1-jdf-40675.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/40675/api-v1-json-data-features-40675.json.gz rename to sklearn/datasets/tests/data/openml/id_40675/api-v1-jdf-40675.json.gz diff --git a/sklearn/datasets/tests/data/openml/40675/api-v1-json-data-list-data_name-glass2-limit-2-data_version-1-status-deactivated.json.gz b/sklearn/datasets/tests/data/openml/id_40675/api-v1-jdl-dn-glass2-l-2-dv-1-s-dact.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/40675/api-v1-json-data-list-data_name-glass2-limit-2-data_version-1-status-deactivated.json.gz rename to sklearn/datasets/tests/data/openml/id_40675/api-v1-jdl-dn-glass2-l-2-dv-1-s-dact.json.gz diff --git a/sklearn/datasets/tests/data/openml/40675/api-v1-json-data-list-data_name-glass2-limit-2-data_version-1.json.gz b/sklearn/datasets/tests/data/openml/id_40675/api-v1-jdl-dn-glass2-l-2-dv-1.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/40675/api-v1-json-data-list-data_name-glass2-limit-2-data_version-1.json.gz rename to sklearn/datasets/tests/data/openml/id_40675/api-v1-jdl-dn-glass2-l-2-dv-1.json.gz diff --git a/sklearn/datasets/tests/data/openml/40675/api-v1-json-data-list-data_name-glass2-limit-2-status-active-.json.gz b/sklearn/datasets/tests/data/openml/id_40675/api-v1-jdl-dn-glass2-l-2-s-act-.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/40675/api-v1-json-data-list-data_name-glass2-limit-2-status-active-.json.gz rename to sklearn/datasets/tests/data/openml/id_40675/api-v1-jdl-dn-glass2-l-2-s-act-.json.gz diff --git a/sklearn/datasets/tests/data/openml/40675/api-v1-json-data-qualities-40675.json.gz b/sklearn/datasets/tests/data/openml/id_40675/api-v1-jdq-40675.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/40675/api-v1-json-data-qualities-40675.json.gz rename to sklearn/datasets/tests/data/openml/id_40675/api-v1-jdq-40675.json.gz diff --git a/sklearn/datasets/tests/data/openml/40675/data-v1-download-4965250.arff.gz b/sklearn/datasets/tests/data/openml/id_40675/data-v1-dl-4965250.arff.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/40675/data-v1-download-4965250.arff.gz rename to sklearn/datasets/tests/data/openml/id_40675/data-v1-dl-4965250.arff.gz diff --git a/sklearn/datasets/tests/data/openml/id_40945/__init__.py b/sklearn/datasets/tests/data/openml/id_40945/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/sklearn/datasets/tests/data/openml/40945/api-v1-json-data-40945.json.gz b/sklearn/datasets/tests/data/openml/id_40945/api-v1-jd-40945.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/40945/api-v1-json-data-40945.json.gz rename to sklearn/datasets/tests/data/openml/id_40945/api-v1-jd-40945.json.gz diff --git a/sklearn/datasets/tests/data/openml/40945/api-v1-json-data-features-40945.json.gz b/sklearn/datasets/tests/data/openml/id_40945/api-v1-jdf-40945.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/40945/api-v1-json-data-features-40945.json.gz rename to sklearn/datasets/tests/data/openml/id_40945/api-v1-jdf-40945.json.gz diff --git a/sklearn/datasets/tests/data/openml/40945/api-v1-json-data-qualities-40945.json.gz b/sklearn/datasets/tests/data/openml/id_40945/api-v1-jdq-40945.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/40945/api-v1-json-data-qualities-40945.json.gz rename to sklearn/datasets/tests/data/openml/id_40945/api-v1-jdq-40945.json.gz diff --git a/sklearn/datasets/tests/data/openml/40945/data-v1-download-16826755.arff.gz b/sklearn/datasets/tests/data/openml/id_40945/data-v1-dl-16826755.arff.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/40945/data-v1-download-16826755.arff.gz rename to sklearn/datasets/tests/data/openml/id_40945/data-v1-dl-16826755.arff.gz diff --git a/sklearn/datasets/tests/data/openml/id_40966/__init__.py b/sklearn/datasets/tests/data/openml/id_40966/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/sklearn/datasets/tests/data/openml/40966/api-v1-json-data-40966.json.gz b/sklearn/datasets/tests/data/openml/id_40966/api-v1-jd-40966.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/40966/api-v1-json-data-40966.json.gz rename to sklearn/datasets/tests/data/openml/id_40966/api-v1-jd-40966.json.gz diff --git a/sklearn/datasets/tests/data/openml/40966/api-v1-json-data-features-40966.json.gz b/sklearn/datasets/tests/data/openml/id_40966/api-v1-jdf-40966.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/40966/api-v1-json-data-features-40966.json.gz rename to sklearn/datasets/tests/data/openml/id_40966/api-v1-jdf-40966.json.gz diff --git a/sklearn/datasets/tests/data/openml/40966/api-v1-json-data-list-data_name-miceprotein-limit-2-data_version-4.json.gz b/sklearn/datasets/tests/data/openml/id_40966/api-v1-jdl-dn-miceprotein-l-2-dv-4.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/40966/api-v1-json-data-list-data_name-miceprotein-limit-2-data_version-4.json.gz rename to sklearn/datasets/tests/data/openml/id_40966/api-v1-jdl-dn-miceprotein-l-2-dv-4.json.gz diff --git a/sklearn/datasets/tests/data/openml/40966/api-v1-json-data-list-data_name-miceprotein-limit-2-status-active-.json.gz b/sklearn/datasets/tests/data/openml/id_40966/api-v1-jdl-dn-miceprotein-l-2-s-act-.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/40966/api-v1-json-data-list-data_name-miceprotein-limit-2-status-active-.json.gz rename to sklearn/datasets/tests/data/openml/id_40966/api-v1-jdl-dn-miceprotein-l-2-s-act-.json.gz diff --git a/sklearn/datasets/tests/data/openml/40966/api-v1-json-data-qualities-40966.json.gz b/sklearn/datasets/tests/data/openml/id_40966/api-v1-jdq-40966.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/40966/api-v1-json-data-qualities-40966.json.gz rename to sklearn/datasets/tests/data/openml/id_40966/api-v1-jdq-40966.json.gz diff --git a/sklearn/datasets/tests/data/openml/40966/data-v1-download-17928620.arff.gz b/sklearn/datasets/tests/data/openml/id_40966/data-v1-dl-17928620.arff.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/40966/data-v1-download-17928620.arff.gz rename to sklearn/datasets/tests/data/openml/id_40966/data-v1-dl-17928620.arff.gz diff --git a/sklearn/datasets/tests/data/openml/id_42074/__init__.py b/sklearn/datasets/tests/data/openml/id_42074/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/sklearn/datasets/tests/data/openml/id_42074/api-v1-jd-42074.json.gz b/sklearn/datasets/tests/data/openml/id_42074/api-v1-jd-42074.json.gz new file mode 100644 index 0000000000000..21761d5ca69ba Binary files /dev/null and b/sklearn/datasets/tests/data/openml/id_42074/api-v1-jd-42074.json.gz differ diff --git a/sklearn/datasets/tests/data/openml/id_42074/api-v1-jdf-42074.json.gz b/sklearn/datasets/tests/data/openml/id_42074/api-v1-jdf-42074.json.gz new file mode 100644 index 0000000000000..622baf5bda2aa Binary files /dev/null and b/sklearn/datasets/tests/data/openml/id_42074/api-v1-jdf-42074.json.gz differ diff --git a/sklearn/datasets/tests/data/openml/id_42074/api-v1-jdq-42074.json.gz b/sklearn/datasets/tests/data/openml/id_42074/api-v1-jdq-42074.json.gz new file mode 100644 index 0000000000000..241c38b25714e Binary files /dev/null and b/sklearn/datasets/tests/data/openml/id_42074/api-v1-jdq-42074.json.gz differ diff --git a/sklearn/datasets/tests/data/openml/id_42074/data-v1-dl-21552912.arff.gz b/sklearn/datasets/tests/data/openml/id_42074/data-v1-dl-21552912.arff.gz new file mode 100644 index 0000000000000..72be9d36cad74 Binary files /dev/null and b/sklearn/datasets/tests/data/openml/id_42074/data-v1-dl-21552912.arff.gz differ diff --git a/sklearn/datasets/tests/data/openml/id_42585/__init__.py b/sklearn/datasets/tests/data/openml/id_42585/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/sklearn/datasets/tests/data/openml/42585/api-v1-json-data-42585.json.gz b/sklearn/datasets/tests/data/openml/id_42585/api-v1-jd-42585.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/42585/api-v1-json-data-42585.json.gz rename to sklearn/datasets/tests/data/openml/id_42585/api-v1-jd-42585.json.gz diff --git a/sklearn/datasets/tests/data/openml/42585/api-v1-json-data-features-42585.json.gz b/sklearn/datasets/tests/data/openml/id_42585/api-v1-jdf-42585.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/42585/api-v1-json-data-features-42585.json.gz rename to sklearn/datasets/tests/data/openml/id_42585/api-v1-jdf-42585.json.gz diff --git a/sklearn/datasets/tests/data/openml/42585/api-v1-json-data-qualities-42585.json.gz b/sklearn/datasets/tests/data/openml/id_42585/api-v1-jdq-42585.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/42585/api-v1-json-data-qualities-42585.json.gz rename to sklearn/datasets/tests/data/openml/id_42585/api-v1-jdq-42585.json.gz diff --git a/sklearn/datasets/tests/data/openml/42585/data-v1-download-21854866.arff.gz b/sklearn/datasets/tests/data/openml/id_42585/data-v1-dl-21854866.arff.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/42585/data-v1-download-21854866.arff.gz rename to sklearn/datasets/tests/data/openml/id_42585/data-v1-dl-21854866.arff.gz diff --git a/sklearn/datasets/tests/data/openml/id_561/__init__.py b/sklearn/datasets/tests/data/openml/id_561/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/sklearn/datasets/tests/data/openml/561/api-v1-json-data-561.json.gz b/sklearn/datasets/tests/data/openml/id_561/api-v1-jd-561.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/561/api-v1-json-data-561.json.gz rename to sklearn/datasets/tests/data/openml/id_561/api-v1-jd-561.json.gz diff --git a/sklearn/datasets/tests/data/openml/561/api-v1-json-data-features-561.json.gz b/sklearn/datasets/tests/data/openml/id_561/api-v1-jdf-561.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/561/api-v1-json-data-features-561.json.gz rename to sklearn/datasets/tests/data/openml/id_561/api-v1-jdf-561.json.gz diff --git a/sklearn/datasets/tests/data/openml/561/api-v1-json-data-list-data_name-cpu-limit-2-data_version-1.json.gz b/sklearn/datasets/tests/data/openml/id_561/api-v1-jdl-dn-cpu-l-2-dv-1.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/561/api-v1-json-data-list-data_name-cpu-limit-2-data_version-1.json.gz rename to sklearn/datasets/tests/data/openml/id_561/api-v1-jdl-dn-cpu-l-2-dv-1.json.gz diff --git a/sklearn/datasets/tests/data/openml/561/api-v1-json-data-list-data_name-cpu-limit-2-status-active-.json.gz b/sklearn/datasets/tests/data/openml/id_561/api-v1-jdl-dn-cpu-l-2-s-act-.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/561/api-v1-json-data-list-data_name-cpu-limit-2-status-active-.json.gz rename to sklearn/datasets/tests/data/openml/id_561/api-v1-jdl-dn-cpu-l-2-s-act-.json.gz diff --git a/sklearn/datasets/tests/data/openml/561/api-v1-json-data-qualities-561.json.gz b/sklearn/datasets/tests/data/openml/id_561/api-v1-jdq-561.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/561/api-v1-json-data-qualities-561.json.gz rename to sklearn/datasets/tests/data/openml/id_561/api-v1-jdq-561.json.gz diff --git a/sklearn/datasets/tests/data/openml/561/data-v1-download-52739.arff.gz b/sklearn/datasets/tests/data/openml/id_561/data-v1-dl-52739.arff.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/561/data-v1-download-52739.arff.gz rename to sklearn/datasets/tests/data/openml/id_561/data-v1-dl-52739.arff.gz diff --git a/sklearn/datasets/tests/data/openml/id_61/__init__.py b/sklearn/datasets/tests/data/openml/id_61/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/sklearn/datasets/tests/data/openml/61/api-v1-json-data-61.json.gz b/sklearn/datasets/tests/data/openml/id_61/api-v1-jd-61.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/61/api-v1-json-data-61.json.gz rename to sklearn/datasets/tests/data/openml/id_61/api-v1-jd-61.json.gz diff --git a/sklearn/datasets/tests/data/openml/61/api-v1-json-data-features-61.json.gz b/sklearn/datasets/tests/data/openml/id_61/api-v1-jdf-61.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/61/api-v1-json-data-features-61.json.gz rename to sklearn/datasets/tests/data/openml/id_61/api-v1-jdf-61.json.gz diff --git a/sklearn/datasets/tests/data/openml/61/api-v1-json-data-list-data_name-iris-limit-2-data_version-1.json.gz b/sklearn/datasets/tests/data/openml/id_61/api-v1-jdl-dn-iris-l-2-dv-1.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/61/api-v1-json-data-list-data_name-iris-limit-2-data_version-1.json.gz rename to sklearn/datasets/tests/data/openml/id_61/api-v1-jdl-dn-iris-l-2-dv-1.json.gz diff --git a/sklearn/datasets/tests/data/openml/61/api-v1-json-data-list-data_name-iris-limit-2-status-active-.json.gz b/sklearn/datasets/tests/data/openml/id_61/api-v1-jdl-dn-iris-l-2-s-act-.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/61/api-v1-json-data-list-data_name-iris-limit-2-status-active-.json.gz rename to sklearn/datasets/tests/data/openml/id_61/api-v1-jdl-dn-iris-l-2-s-act-.json.gz diff --git a/sklearn/datasets/tests/data/openml/61/api-v1-json-data-qualities-61.json.gz b/sklearn/datasets/tests/data/openml/id_61/api-v1-jdq-61.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/61/api-v1-json-data-qualities-61.json.gz rename to sklearn/datasets/tests/data/openml/id_61/api-v1-jdq-61.json.gz diff --git a/sklearn/datasets/tests/data/openml/61/data-v1-download-61.arff.gz b/sklearn/datasets/tests/data/openml/id_61/data-v1-dl-61.arff.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/61/data-v1-download-61.arff.gz rename to sklearn/datasets/tests/data/openml/id_61/data-v1-dl-61.arff.gz diff --git a/sklearn/datasets/tests/data/openml/id_62/__init__.py b/sklearn/datasets/tests/data/openml/id_62/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/sklearn/datasets/tests/data/openml/62/api-v1-json-data-62.json.gz b/sklearn/datasets/tests/data/openml/id_62/api-v1-jd-62.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/62/api-v1-json-data-62.json.gz rename to sklearn/datasets/tests/data/openml/id_62/api-v1-jd-62.json.gz diff --git a/sklearn/datasets/tests/data/openml/62/api-v1-json-data-features-62.json.gz b/sklearn/datasets/tests/data/openml/id_62/api-v1-jdf-62.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/62/api-v1-json-data-features-62.json.gz rename to sklearn/datasets/tests/data/openml/id_62/api-v1-jdf-62.json.gz diff --git a/sklearn/datasets/tests/data/openml/62/api-v1-json-data-qualities-62.json.gz b/sklearn/datasets/tests/data/openml/id_62/api-v1-jdq-62.json.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/62/api-v1-json-data-qualities-62.json.gz rename to sklearn/datasets/tests/data/openml/id_62/api-v1-jdq-62.json.gz diff --git a/sklearn/datasets/tests/data/openml/62/data-v1-download-52352.arff.gz b/sklearn/datasets/tests/data/openml/id_62/data-v1-dl-52352.arff.gz similarity index 100% rename from sklearn/datasets/tests/data/openml/62/data-v1-download-52352.arff.gz rename to sklearn/datasets/tests/data/openml/id_62/data-v1-dl-52352.arff.gz diff --git a/sklearn/datasets/tests/data/svmlight_classification.txt b/sklearn/datasets/tests/data/svmlight_classification.txt index a3c4a3364cac1..7826fb40d47d2 100644 --- a/sklearn/datasets/tests/data/svmlight_classification.txt +++ b/sklearn/datasets/tests/data/svmlight_classification.txt @@ -1,7 +1,7 @@ # comment # note: the next line contains a tab 1.0 3:2.5 11:-5.2 16:1.5 # and an inline comment -2.0 6:1.0 13:-3 +2.0 6:1.0 13:-3 # another comment 3.0 21:27 4.0 2:1.234567890123456e10 # double precision value diff --git a/sklearn/datasets/tests/data/svmlight_multilabel.txt b/sklearn/datasets/tests/data/svmlight_multilabel.txt index a8194e5fef163..047d5e0fd29af 100644 --- a/sklearn/datasets/tests/data/svmlight_multilabel.txt +++ b/sklearn/datasets/tests/data/svmlight_multilabel.txt @@ -1,5 +1,5 @@ # multilabel dataset in SVMlight format 1,0 2:2.5 10:-5.2 15:1.5 -2 5:1.0 12:-3 +2 5:1.0 12:-3 2:3.5 11:26 1,2 20:27 diff --git a/sklearn/datasets/tests/test_20news.py b/sklearn/datasets/tests/test_20news.py index 77f671994618f..84e7c91d3176f 100644 --- a/sklearn/datasets/tests/test_20news.py +++ b/sklearn/datasets/tests/test_20news.py @@ -1,27 +1,31 @@ """Test the 20news downloader, if the data is available, or if specifically requested via environment variable -(e.g. for travis cron job).""" +(e.g. for CI jobs).""" + from functools import partial from unittest.mock import patch -import pytest - import numpy as np +import pytest import scipy.sparse as sp -from sklearn.datasets.tests.test_common import check_as_frame -from sklearn.datasets.tests.test_common import check_pandas_dependency_message -from sklearn.datasets.tests.test_common import check_return_X_y -from sklearn.utils._testing import assert_allclose_dense_sparse +from sklearn.datasets.tests.test_common import ( + check_as_frame, + check_pandas_dependency_message, + check_return_X_y, +) from sklearn.preprocessing import normalize +from sklearn.utils._testing import assert_allclose_dense_sparse def test_20news(fetch_20newsgroups_fxt): - data = fetch_20newsgroups_fxt(subset='all', shuffle=False) + data = fetch_20newsgroups_fxt(subset="all", shuffle=False) + assert data.DESCR.startswith(".. _20newsgroups_dataset:") # Extract a reduced dataset data2cats = fetch_20newsgroups_fxt( - subset='all', categories=data.target_names[-1:-3:-1], shuffle=False) + subset="all", categories=data.target_names[-1:-3:-1], shuffle=False + ) # Check that the ordering of the target_names is the same # as the ordering in the full dataset assert data2cats.target_names == data.target_names[-2:] @@ -41,7 +45,7 @@ def test_20news(fetch_20newsgroups_fxt): assert entry1 == entry2 # check that return_X_y option - X, y = fetch_20newsgroups_fxt(subset='all', shuffle=False, return_X_y=True) + X, y = fetch_20newsgroups_fxt(subset="all", shuffle=False, return_X_y=True) assert len(X) == len(data.data) assert y.shape == data.target.shape @@ -52,51 +56,54 @@ def test_20news_length_consistency(fetch_20newsgroups_fxt): This is a non-regression test for a bug present in 0.16.1. """ # Extract the full dataset - data = fetch_20newsgroups_fxt(subset='all') - assert len(data['data']) == len(data.data) - assert len(data['target']) == len(data.target) - assert len(data['filenames']) == len(data.filenames) + data = fetch_20newsgroups_fxt(subset="all") + assert len(data["data"]) == len(data.data) + assert len(data["target"]) == len(data.target) + assert len(data["filenames"]) == len(data.filenames) def test_20news_vectorized(fetch_20newsgroups_vectorized_fxt): # test subset = train bunch = fetch_20newsgroups_vectorized_fxt(subset="train") - assert sp.isspmatrix_csr(bunch.data) + assert sp.issparse(bunch.data) and bunch.data.format == "csr" assert bunch.data.shape == (11314, 130107) assert bunch.target.shape[0] == 11314 assert bunch.data.dtype == np.float64 + assert bunch.DESCR.startswith(".. _20newsgroups_dataset:") # test subset = test bunch = fetch_20newsgroups_vectorized_fxt(subset="test") - assert sp.isspmatrix_csr(bunch.data) + assert sp.issparse(bunch.data) and bunch.data.format == "csr" assert bunch.data.shape == (7532, 130107) assert bunch.target.shape[0] == 7532 assert bunch.data.dtype == np.float64 + assert bunch.DESCR.startswith(".. _20newsgroups_dataset:") # test return_X_y option - fetch_func = partial(fetch_20newsgroups_vectorized_fxt, subset='test') + fetch_func = partial(fetch_20newsgroups_vectorized_fxt, subset="test") check_return_X_y(bunch, fetch_func) # test subset = all - bunch = fetch_20newsgroups_vectorized_fxt(subset='all') - assert sp.isspmatrix_csr(bunch.data) + bunch = fetch_20newsgroups_vectorized_fxt(subset="all") + assert sp.issparse(bunch.data) and bunch.data.format == "csr" assert bunch.data.shape == (11314 + 7532, 130107) assert bunch.target.shape[0] == 11314 + 7532 assert bunch.data.dtype == np.float64 + assert bunch.DESCR.startswith(".. _20newsgroups_dataset:") def test_20news_normalization(fetch_20newsgroups_vectorized_fxt): X = fetch_20newsgroups_vectorized_fxt(normalize=False) X_ = fetch_20newsgroups_vectorized_fxt(normalize=True) - X_norm = X_['data'][:100] - X = X['data'][:100] + X_norm = X_["data"][:100] + X = X["data"][:100] assert_allclose_dense_sparse(X_norm, normalize(X)) assert np.allclose(np.linalg.norm(X_norm.todense(), axis=1), 1) def test_20news_as_frame(fetch_20newsgroups_vectorized_fxt): - pd = pytest.importorskip('pandas') + pd = pytest.importorskip("pandas") bunch = fetch_20newsgroups_vectorized_fxt(as_frame=True) check_as_frame(bunch, fetch_20newsgroups_vectorized_fxt) @@ -120,9 +127,7 @@ def test_20news_as_frame(fetch_20newsgroups_vectorized_fxt): assert bunch.target.name == "category_class" -def test_as_frame_no_pandas( - fetch_20newsgroups_vectorized_fxt, hide_available_pandas -): +def test_as_frame_no_pandas(fetch_20newsgroups_vectorized_fxt, hide_available_pandas): check_pandas_dependency_message(fetch_20newsgroups_vectorized_fxt) diff --git a/sklearn/datasets/tests/test_arff_parser.py b/sklearn/datasets/tests/test_arff_parser.py new file mode 100644 index 0000000000000..c4f9e3eb00ffd --- /dev/null +++ b/sklearn/datasets/tests/test_arff_parser.py @@ -0,0 +1,284 @@ +import textwrap +from io import BytesIO + +import pytest + +from sklearn.datasets._arff_parser import ( + _liac_arff_parser, + _pandas_arff_parser, + _post_process_frame, + load_arff_from_gzip_file, +) + + +@pytest.mark.parametrize( + "feature_names, target_names", + [ + ( + [ + "col_int_as_integer", + "col_int_as_numeric", + "col_float_as_real", + "col_float_as_numeric", + ], + ["col_categorical", "col_string"], + ), + ( + [ + "col_int_as_integer", + "col_int_as_numeric", + "col_float_as_real", + "col_float_as_numeric", + ], + ["col_categorical"], + ), + ( + [ + "col_int_as_integer", + "col_int_as_numeric", + "col_float_as_real", + "col_float_as_numeric", + ], + [], + ), + ], +) +def test_post_process_frame(feature_names, target_names): + """Check the behaviour of the post-processing function for splitting a dataframe.""" + pd = pytest.importorskip("pandas") + + X_original = pd.DataFrame( + { + "col_int_as_integer": [1, 2, 3], + "col_int_as_numeric": [1, 2, 3], + "col_float_as_real": [1.0, 2.0, 3.0], + "col_float_as_numeric": [1.0, 2.0, 3.0], + "col_categorical": ["a", "b", "c"], + "col_string": ["a", "b", "c"], + } + ) + + X, y = _post_process_frame(X_original, feature_names, target_names) + assert isinstance(X, pd.DataFrame) + if len(target_names) >= 2: + assert isinstance(y, pd.DataFrame) + elif len(target_names) == 1: + assert isinstance(y, pd.Series) + else: + assert y is None + + +def test_load_arff_from_gzip_file_error_parser(): + """An error will be raised if the parser is not known.""" + # None of the input parameters are required to be accurate since the check + # of the parser will be carried out first. + + err_msg = "Unknown parser: 'xxx'. Should be 'liac-arff' or 'pandas'" + with pytest.raises(ValueError, match=err_msg): + load_arff_from_gzip_file("xxx", "xxx", "xxx", "xxx", "xxx", "xxx") + + +@pytest.mark.parametrize("parser_func", [_liac_arff_parser, _pandas_arff_parser]) +def test_pandas_arff_parser_strip_single_quotes(parser_func): + """Check that we properly strip single quotes from the data.""" + pd = pytest.importorskip("pandas") + + arff_file = BytesIO( + textwrap.dedent( + """ + @relation 'toy' + @attribute 'cat_single_quote' {'A', 'B', 'C'} + @attribute 'str_single_quote' string + @attribute 'str_nested_quote' string + @attribute 'class' numeric + @data + 'A','some text','\"expect double quotes\"',0 + """ + ).encode("utf-8") + ) + + columns_info = { + "cat_single_quote": { + "data_type": "nominal", + "name": "cat_single_quote", + }, + "str_single_quote": { + "data_type": "string", + "name": "str_single_quote", + }, + "str_nested_quote": { + "data_type": "string", + "name": "str_nested_quote", + }, + "class": { + "data_type": "numeric", + "name": "class", + }, + } + + feature_names = [ + "cat_single_quote", + "str_single_quote", + "str_nested_quote", + ] + target_names = ["class"] + + # We don't strip single quotes for string columns with the pandas parser. + expected_values = { + "cat_single_quote": "A", + "str_single_quote": ( + "some text" if parser_func is _liac_arff_parser else "'some text'" + ), + "str_nested_quote": ( + '"expect double quotes"' + if parser_func is _liac_arff_parser + else "'\"expect double quotes\"'" + ), + "class": 0, + } + + _, _, frame, _ = parser_func( + arff_file, + output_arrays_type="pandas", + openml_columns_info=columns_info, + feature_names_to_select=feature_names, + target_names_to_select=target_names, + ) + + assert frame.columns.tolist() == feature_names + target_names + pd.testing.assert_series_equal(frame.iloc[0], pd.Series(expected_values, name=0)) + + +@pytest.mark.parametrize("parser_func", [_liac_arff_parser, _pandas_arff_parser]) +def test_pandas_arff_parser_strip_double_quotes(parser_func): + """Check that we properly strip double quotes from the data.""" + pd = pytest.importorskip("pandas") + + arff_file = BytesIO( + textwrap.dedent( + """ + @relation 'toy' + @attribute 'cat_double_quote' {"A", "B", "C"} + @attribute 'str_double_quote' string + @attribute 'str_nested_quote' string + @attribute 'class' numeric + @data + "A","some text","\'expect double quotes\'",0 + """ + ).encode("utf-8") + ) + + columns_info = { + "cat_double_quote": { + "data_type": "nominal", + "name": "cat_double_quote", + }, + "str_double_quote": { + "data_type": "string", + "name": "str_double_quote", + }, + "str_nested_quote": { + "data_type": "string", + "name": "str_nested_quote", + }, + "class": { + "data_type": "numeric", + "name": "class", + }, + } + + feature_names = [ + "cat_double_quote", + "str_double_quote", + "str_nested_quote", + ] + target_names = ["class"] + + expected_values = { + "cat_double_quote": "A", + "str_double_quote": "some text", + "str_nested_quote": "'expect double quotes'", + "class": 0, + } + + _, _, frame, _ = parser_func( + arff_file, + output_arrays_type="pandas", + openml_columns_info=columns_info, + feature_names_to_select=feature_names, + target_names_to_select=target_names, + ) + + assert frame.columns.tolist() == feature_names + target_names + pd.testing.assert_series_equal(frame.iloc[0], pd.Series(expected_values, name=0)) + + +@pytest.mark.parametrize( + "parser_func", + [ + # internal quotes are not considered to follow the ARFF spec in LIAC ARFF + pytest.param(_liac_arff_parser, marks=pytest.mark.xfail), + _pandas_arff_parser, + ], +) +def test_pandas_arff_parser_strip_no_quotes(parser_func): + """Check that we properly parse with no quotes characters.""" + pd = pytest.importorskip("pandas") + + arff_file = BytesIO( + textwrap.dedent( + """ + @relation 'toy' + @attribute 'cat_without_quote' {A, B, C} + @attribute 'str_without_quote' string + @attribute 'str_internal_quote' string + @attribute 'class' numeric + @data + A,some text,'internal' quote,0 + """ + ).encode("utf-8") + ) + + columns_info = { + "cat_without_quote": { + "data_type": "nominal", + "name": "cat_without_quote", + }, + "str_without_quote": { + "data_type": "string", + "name": "str_without_quote", + }, + "str_internal_quote": { + "data_type": "string", + "name": "str_internal_quote", + }, + "class": { + "data_type": "numeric", + "name": "class", + }, + } + + feature_names = [ + "cat_without_quote", + "str_without_quote", + "str_internal_quote", + ] + target_names = ["class"] + + expected_values = { + "cat_without_quote": "A", + "str_without_quote": "some text", + "str_internal_quote": "'internal' quote", + "class": 0, + } + + _, _, frame, _ = parser_func( + arff_file, + output_arrays_type="pandas", + openml_columns_info=columns_info, + feature_names_to_select=feature_names, + target_names_to_select=target_names, + ) + + assert frame.columns.tolist() == feature_names + target_names + pd.testing.assert_series_equal(frame.iloc[0], pd.Series(expected_values, name=0)) diff --git a/sklearn/datasets/tests/test_base.py b/sklearn/datasets/tests/test_base.py index e698c6c43e238..4396b7921f3ee 100644 --- a/sklearn/datasets/tests/test_base.py +++ b/sklearn/datasets/tests/test_base.py @@ -1,32 +1,55 @@ +import hashlib +import io import os +import re import shutil import tempfile import warnings -from pickle import loads -from pickle import dumps from functools import partial +from importlib import resources +from pathlib import Path +from pickle import dumps, loads +from unittest.mock import Mock +from urllib.error import HTTPError +from urllib.parse import urlparse +import numpy as np import pytest -import numpy as np -from sklearn.datasets import get_data_home -from sklearn.datasets import clear_data_home -from sklearn.datasets import load_files -from sklearn.datasets import load_sample_images -from sklearn.datasets import load_sample_image -from sklearn.datasets import load_digits -from sklearn.datasets import load_diabetes -from sklearn.datasets import load_linnerud -from sklearn.datasets import load_iris -from sklearn.datasets import load_breast_cancer -from sklearn.datasets import load_boston -from sklearn.datasets import load_wine -from sklearn.utils import Bunch +from sklearn.datasets import ( + clear_data_home, + fetch_file, + get_data_home, + load_breast_cancer, + load_diabetes, + load_digits, + load_files, + load_iris, + load_linnerud, + load_sample_image, + load_sample_images, + load_wine, +) +from sklearn.datasets._base import ( + RemoteFileMetadata, + _derive_folder_and_filename_from_url, + _fetch_remote, + load_csv_data, + load_gzip_compressed_csv_data, +) from sklearn.datasets.tests.test_common import check_as_frame +from sklearn.preprocessing import scale +from sklearn.utils import Bunch + -from sklearn.externals._pilutil import pillow_installed +class _DummyPath: + """Minimal class that implements the os.PathLike interface.""" -from sklearn.utils import IS_PYPY + def __init__(self, path): + self.path = path + + def __fspath__(self): + return self.path def _remove_dir(path): @@ -51,8 +74,7 @@ def load_files_root(tmpdir_factory): @pytest.fixture def test_category_dir_1(load_files_root): test_category_dir1 = tempfile.mkdtemp(dir=load_files_root) - sample_file = tempfile.NamedTemporaryFile(dir=test_category_dir1, - delete=False) + sample_file = tempfile.NamedTemporaryFile(dir=test_category_dir1, delete=False) sample_file.write(b"Hello World!\n") sample_file.close() yield str(test_category_dir1) @@ -66,13 +88,18 @@ def test_category_dir_2(load_files_root): _remove_dir(test_category_dir2) -def test_data_home(data_home): +@pytest.mark.parametrize("path_container", [None, Path, _DummyPath]) +def test_data_home(path_container, data_home): # get_data_home will point to a pre-existing folder + if path_container is not None: + data_home = path_container(data_home) data_home = get_data_home(data_home=data_home) assert data_home == data_home assert os.path.exists(data_home) # clear_data_home will delete both the content and the folder it-self + if path_container is not None: + data_home = path_container(data_home) clear_data_home(data_home=data_home) assert not os.path.exists(data_home) @@ -88,10 +115,7 @@ def test_default_empty_load_files(load_files_root): assert res.DESCR is None -def test_default_load_files(test_category_dir_1, test_category_dir_2, - load_files_root): - if IS_PYPY: - pytest.xfail('[PyPy] fails due to string containing NUL characters') +def test_default_load_files(test_category_dir_1, test_category_dir_2, load_files_root): res = load_files(load_files_root) assert len(res.filenames) == 1 assert len(res.target_names) == 2 @@ -100,12 +124,13 @@ def test_default_load_files(test_category_dir_1, test_category_dir_2, def test_load_files_w_categories_desc_and_encoding( - test_category_dir_1, test_category_dir_2, load_files_root): - if IS_PYPY: - pytest.xfail('[PyPy] fails due to string containing NUL characters') - category = os.path.abspath(test_category_dir_1).split('/').pop() - res = load_files(load_files_root, description="test", - categories=category, encoding="utf-8") + test_category_dir_1, test_category_dir_2, load_files_root +): + category = os.path.abspath(test_category_dir_1).split(os.sep).pop() + res = load_files( + load_files_root, description="test", categories=[category], encoding="utf-8" + ) + assert len(res.filenames) == 1 assert len(res.target_names) == 1 assert res.DESCR == "test" @@ -113,12 +138,91 @@ def test_load_files_w_categories_desc_and_encoding( def test_load_files_wo_load_content( - test_category_dir_1, test_category_dir_2, load_files_root): + test_category_dir_1, test_category_dir_2, load_files_root +): res = load_files(load_files_root, load_content=False) assert len(res.filenames) == 1 assert len(res.target_names) == 2 assert res.DESCR is None - assert res.get('data') is None + assert res.get("data") is None + + +@pytest.mark.parametrize("allowed_extensions", ([".txt"], [".txt", ".json"])) +def test_load_files_allowed_extensions(tmp_path, allowed_extensions): + """Check the behaviour of `allowed_extension` in `load_files`.""" + d = tmp_path / "sub" + d.mkdir() + files = ("file1.txt", "file2.json", "file3.json", "file4.md") + paths = [d / f for f in files] + for p in paths: + p.write_bytes(b"hello") + res = load_files(tmp_path, allowed_extensions=allowed_extensions) + assert set([str(p) for p in paths if p.suffix in allowed_extensions]) == set( + res.filenames + ) + + +@pytest.mark.parametrize( + "filename, expected_n_samples, expected_n_features, expected_target_names", + [ + ("wine_data.csv", 178, 13, ["class_0", "class_1", "class_2"]), + ("iris.csv", 150, 4, ["setosa", "versicolor", "virginica"]), + ("breast_cancer.csv", 569, 30, ["malignant", "benign"]), + ], +) +def test_load_csv_data( + filename, expected_n_samples, expected_n_features, expected_target_names +): + actual_data, actual_target, actual_target_names = load_csv_data(filename) + assert actual_data.shape[0] == expected_n_samples + assert actual_data.shape[1] == expected_n_features + assert actual_target.shape[0] == expected_n_samples + np.testing.assert_array_equal(actual_target_names, expected_target_names) + + +def test_load_csv_data_with_descr(): + data_file_name = "iris.csv" + descr_file_name = "iris.rst" + + res_without_descr = load_csv_data(data_file_name=data_file_name) + res_with_descr = load_csv_data( + data_file_name=data_file_name, descr_file_name=descr_file_name + ) + assert len(res_with_descr) == 4 + assert len(res_without_descr) == 3 + + np.testing.assert_array_equal(res_with_descr[0], res_without_descr[0]) + np.testing.assert_array_equal(res_with_descr[1], res_without_descr[1]) + np.testing.assert_array_equal(res_with_descr[2], res_without_descr[2]) + + assert res_with_descr[-1].startswith(".. _iris_dataset:") + + +@pytest.mark.parametrize( + "filename, kwargs, expected_shape", + [ + ("diabetes_data_raw.csv.gz", {}, [442, 10]), + ("diabetes_target.csv.gz", {}, [442]), + ("digits.csv.gz", {"delimiter": ","}, [1797, 65]), + ], +) +def test_load_gzip_compressed_csv_data(filename, kwargs, expected_shape): + actual_data = load_gzip_compressed_csv_data(filename, **kwargs) + assert actual_data.shape == tuple(expected_shape) + + +def test_load_gzip_compressed_csv_data_with_descr(): + data_file_name = "diabetes_target.csv.gz" + descr_file_name = "diabetes.rst" + + expected_data = load_gzip_compressed_csv_data(data_file_name=data_file_name) + actual_data, descr = load_gzip_compressed_csv_data( + data_file_name=data_file_name, + descr_file_name=descr_file_name, + ) + + np.testing.assert_array_equal(actual_data, expected_data) + assert descr.startswith(".. _diabetes_dataset:") def test_load_sample_images(): @@ -129,11 +233,9 @@ def test_load_sample_images(): images = res.images # assert is china image - assert np.all(images[0][0, 0, :] == - np.array([174, 201, 231], dtype=np.uint8)) + assert np.all(images[0][0, 0, :] == np.array([174, 201, 231], dtype=np.uint8)) # assert is flower image - assert np.all(images[1][0, 0, :] == - np.array([2, 19, 13], dtype=np.uint8)) + assert np.all(images[1][0, 0, :] == np.array([2, 19, 13], dtype=np.uint8)) assert res.DESCR except ImportError: warnings.warn("Could not load sample images, PIL is not available.") @@ -141,35 +243,49 @@ def test_load_sample_images(): def test_load_sample_image(): try: - china = load_sample_image('china.jpg') - assert china.dtype == 'uint8' + china = load_sample_image("china.jpg") + assert china.dtype == "uint8" assert china.shape == (427, 640, 3) except ImportError: warnings.warn("Could not load sample images, PIL is not available.") -def test_load_missing_sample_image_error(): - if pillow_installed: - with pytest.raises(AttributeError): - load_sample_image('blop.jpg') - else: - warnings.warn("Could not load sample images, PIL is not available.") +def test_load_diabetes_raw(): + """Test to check that we load a scaled version by default but that we can + get an unscaled version when setting `scaled=False`.""" + diabetes_raw = load_diabetes(scaled=False) + assert diabetes_raw.data.shape == (442, 10) + assert diabetes_raw.target.size == 442 + assert len(diabetes_raw.feature_names) == 10 + assert diabetes_raw.DESCR + + diabetes_default = load_diabetes() + + np.testing.assert_allclose( + scale(diabetes_raw.data) / (442**0.5), diabetes_default.data, atol=1e-04 + ) @pytest.mark.parametrize( "loader_func, data_shape, target_shape, n_target, has_descr, filenames", - [(load_breast_cancer, (569, 30), (569,), 2, True, ["filename"]), - (load_wine, (178, 13), (178,), 3, True, []), - (load_iris, (150, 4), (150,), 3, True, ["filename"]), - (load_linnerud, (20, 3), (20, 3), 3, True, - ["data_filename", "target_filename"]), - (load_diabetes, (442, 10), (442,), None, True, []), - (load_digits, (1797, 64), (1797,), 10, True, []), - (partial(load_digits, n_class=9), (1617, 64), (1617,), 10, True, []), - (load_boston, (506, 13), (506,), None, True, ["filename"])] + [ + (load_breast_cancer, (569, 30), (569,), 2, True, ["filename"]), + (load_wine, (178, 13), (178,), 3, True, []), + (load_iris, (150, 4), (150,), 3, True, ["filename"]), + ( + load_linnerud, + (20, 3), + (20, 3), + 3, + True, + ["data_filename", "target_filename"], + ), + (load_diabetes, (442, 10), (442,), None, True, []), + (load_digits, (1797, 64), (1797,), 10, True, []), + (partial(load_digits, n_class=9), (1617, 64), (1617,), 10, True, []), + ], ) -def test_loader(loader_func, data_shape, target_shape, n_target, has_descr, - filenames): +def test_loader(loader_func, data_shape, target_shape, n_target, has_descr, filenames): bunch = loader_func() assert isinstance(bunch, Bunch) @@ -182,33 +298,46 @@ def test_loader(loader_func, data_shape, target_shape, n_target, has_descr, if has_descr: assert bunch.DESCR if filenames: - assert all([os.path.exists(bunch.get(f, False)) for f in filenames]) + assert "data_module" in bunch + assert all( + [ + f in bunch + and (resources.files(bunch["data_module"]) / bunch[f]).is_file() + for f in filenames + ] + ) -@pytest.mark.parametrize("loader_func, data_dtype, target_dtype", [ - (load_breast_cancer, np.float64, int), - (load_diabetes, np.float64, np.float64), - (load_digits, np.float64, int), - (load_iris, np.float64, int), - (load_linnerud, np.float64, np.float64), - (load_wine, np.float64, int), -]) +@pytest.mark.parametrize( + "loader_func, data_dtype, target_dtype", + [ + (load_breast_cancer, np.float64, int), + (load_diabetes, np.float64, np.float64), + (load_digits, np.float64, int), + (load_iris, np.float64, int), + (load_linnerud, np.float64, np.float64), + (load_wine, np.float64, int), + ], +) def test_toy_dataset_frame_dtype(loader_func, data_dtype, target_dtype): default_result = loader_func() - check_as_frame(default_result, loader_func, - expected_data_dtype=data_dtype, - expected_target_dtype=target_dtype) + check_as_frame( + default_result, + loader_func, + expected_data_dtype=data_dtype, + expected_target_dtype=target_dtype, + ) def test_loads_dumps_bunch(): bunch = Bunch(x="x") bunch_from_pkl = loads(dumps(bunch)) bunch_from_pkl.x = "y" - assert bunch_from_pkl['x'] == bunch_from_pkl.x + assert bunch_from_pkl["x"] == bunch_from_pkl.x def test_bunch_pickle_generated_with_0_16_and_read_with_0_17(): - bunch = Bunch(key='original') + bunch = Bunch(key="original") # This reproduces a problem when Bunch pickles have been created # with scikit-learn 0.16 and are read with 0.17. Basically there # is a surprising behaviour because reading bunch.key uses @@ -216,19 +345,314 @@ def test_bunch_pickle_generated_with_0_16_and_read_with_0_17(): # whereas assigning into bunch.key uses bunch.__setattr__. See # https://github.com/scikit-learn/scikit-learn/issues/6196 for # more details - bunch.__dict__['key'] = 'set from __dict__' + bunch.__dict__["key"] = "set from __dict__" bunch_from_pkl = loads(dumps(bunch)) # After loading from pickle the __dict__ should have been ignored - assert bunch_from_pkl.key == 'original' - assert bunch_from_pkl['key'] == 'original' + assert bunch_from_pkl.key == "original" + assert bunch_from_pkl["key"] == "original" # Making sure that changing the attr does change the value # associated with __getitem__ as well - bunch_from_pkl.key = 'changed' - assert bunch_from_pkl.key == 'changed' - assert bunch_from_pkl['key'] == 'changed' + bunch_from_pkl.key = "changed" + assert bunch_from_pkl.key == "changed" + assert bunch_from_pkl["key"] == "changed" def test_bunch_dir(): # check that dir (important for autocomplete) shows attributes data = load_iris() assert "data" in dir(data) + + +def test_load_boston_error(): + """Check that we raise the ethical warning when trying to import `load_boston`.""" + msg = "The Boston housing prices dataset has an ethical problem" + with pytest.raises(ImportError, match=msg): + from sklearn.datasets import load_boston # noqa: F401 + + # other non-existing function should raise the usual import error + msg = "cannot import name 'non_existing_function' from 'sklearn.datasets'" + with pytest.raises(ImportError, match=msg): + from sklearn.datasets import non_existing_function # noqa: F401 + + +def test_fetch_remote_raise_warnings_with_invalid_url(monkeypatch): + """Check retry mechanism in _fetch_remote.""" + + url = "https://scikit-learn.org/this_file_does_not_exist.tar.gz" + invalid_remote_file = RemoteFileMetadata("invalid_file", url, None) + urlretrieve_mock = Mock( + side_effect=HTTPError( + url=url, code=404, msg="Not Found", hdrs=None, fp=io.BytesIO() + ) + ) + monkeypatch.setattr("sklearn.datasets._base.urlretrieve", urlretrieve_mock) + + with pytest.warns(UserWarning, match="Retry downloading") as record: + with pytest.raises(HTTPError, match="HTTP Error 404"): + _fetch_remote(invalid_remote_file, n_retries=3, delay=0) + + assert urlretrieve_mock.call_count == 4 + + for r in record: + assert str(r.message) == f"Retry downloading from url: {url}" + assert len(record) == 3 + + +def test_derive_folder_and_filename_from_url(): + folder, filename = _derive_folder_and_filename_from_url( + "https://example.com/file.tar.gz" + ) + assert folder == "example.com" + assert filename == "file.tar.gz" + + folder, filename = _derive_folder_and_filename_from_url( + "https://example.com/Ų†Ų…ŲˆŲ†Ų‡ Ų†Ų…ایŲ†Ø¯Ų‡.data" + ) + assert folder == "example.com" + assert filename == "Ų†Ų…ŲˆŲ†Ų‡-Ų†Ų…ایŲ†Ø¯Ų‡.data" + + folder, filename = _derive_folder_and_filename_from_url( + "https://example.com/path/to-/.file.tar.gz" + ) + assert folder == "example.com/path_to" + assert filename == "file.tar.gz" + + folder, filename = _derive_folder_and_filename_from_url("https://example.com/") + assert folder == "example.com" + assert filename == "downloaded_file" + + folder, filename = _derive_folder_and_filename_from_url("https://example.com") + assert folder == "example.com" + assert filename == "downloaded_file" + + folder, filename = _derive_folder_and_filename_from_url( + "https://example.com/path/@to/data.json?param=value" + ) + assert folder == "example.com/path_to" + assert filename == "data.json" + + folder, filename = _derive_folder_and_filename_from_url( + "https://example.com/path/@@to._/-_.data.json.#anchor" + ) + assert folder == "example.com/path_to" + assert filename == "data.json" + + folder, filename = _derive_folder_and_filename_from_url( + "https://example.com//some_file.txt" + ) + assert folder == "example.com" + assert filename == "some_file.txt" + + folder, filename = _derive_folder_and_filename_from_url( + "http://example/../some_file.txt" + ) + assert folder == "example" + assert filename == "some_file.txt" + + folder, filename = _derive_folder_and_filename_from_url( + "https://example.com/!.'.,/some_file.txt" + ) + assert folder == "example.com" + assert filename == "some_file.txt" + + folder, filename = _derive_folder_and_filename_from_url( + "https://example.com/a/!.'.,/b/some_file.txt" + ) + assert folder == "example.com/a_b" + assert filename == "some_file.txt" + + folder, filename = _derive_folder_and_filename_from_url("https://example.com/!.'.,") + assert folder == "example.com" + assert filename == "downloaded_file" + + with pytest.raises(ValueError, match="Invalid URL"): + _derive_folder_and_filename_from_url("https:/../") + + +def _mock_urlretrieve(server_side): + def _urlretrieve_mock(url, local_path): + server_root = Path(server_side) + file_path = urlparse(url).path.strip("/") + if not (server_root / file_path).exists(): + raise HTTPError(url, 404, "Not Found", None, None) + shutil.copy(server_root / file_path, local_path) + + return Mock(side_effect=_urlretrieve_mock) + + +def test_fetch_file_using_data_home(monkeypatch, tmpdir): + tmpdir = Path(tmpdir) + server_side = tmpdir / "server_side" + server_side.mkdir() + data_file = server_side / "data.jsonl" + server_data = '{"a": 1, "b": 2}\n' + data_file.write_text(server_data, encoding="utf-8") + + server_subfolder = server_side / "subfolder" + server_subfolder.mkdir() + other_data_file = server_subfolder / "other_file.txt" + other_data_file.write_text("Some important text data.", encoding="utf-8") + + data_home = tmpdir / "data_home" + data_home.mkdir() + + urlretrieve_mock = _mock_urlretrieve(server_side) + monkeypatch.setattr("sklearn.datasets._base.urlretrieve", urlretrieve_mock) + + monkeypatch.setattr( + "sklearn.datasets._base.get_data_home", Mock(return_value=data_home) + ) + fetched_file_path = fetch_file( + "https://example.com/data.jsonl", + ) + assert fetched_file_path == data_home / "example.com" / "data.jsonl" + assert fetched_file_path.read_text(encoding="utf-8") == server_data + + fetched_file_path = fetch_file( + "https://example.com/subfolder/other_file.txt", + ) + assert ( + fetched_file_path == data_home / "example.com" / "subfolder" / "other_file.txt" + ) + assert fetched_file_path.read_text(encoding="utf-8") == other_data_file.read_text( + "utf-8" + ) + + expected_warning_msg = re.escape( + "Retry downloading from url: https://example.com/subfolder/invalid.txt" + ) + with pytest.raises(HTTPError): + with pytest.warns(match=expected_warning_msg): + fetch_file( + "https://example.com/subfolder/invalid.txt", + delay=0, + ) + + local_subfolder = data_home / "example.com" / "subfolder" + assert sorted(local_subfolder.iterdir()) == [local_subfolder / "other_file.txt"] + + +def test_fetch_file_without_sha256(monkeypatch, tmpdir): + server_side = tmpdir.mkdir("server_side") + data_file = Path(server_side / "data.jsonl") + server_data = '{"a": 1, "b": 2}\n' + data_file.write_text(server_data, encoding="utf-8") + + client_side = tmpdir.mkdir("client_side") + + urlretrieve_mock = _mock_urlretrieve(server_side) + monkeypatch.setattr("sklearn.datasets._base.urlretrieve", urlretrieve_mock) + + # The first call should trigger a download: + fetched_file_path = fetch_file( + "https://example.com/data.jsonl", + folder=client_side, + ) + assert fetched_file_path == client_side / "data.jsonl" + assert fetched_file_path.read_text(encoding="utf-8") == server_data + assert urlretrieve_mock.call_count == 1 + + # Fetching again the same file to the same folder should do nothing: + fetched_file_path = fetch_file( + "https://example.com/data.jsonl", + folder=client_side, + ) + assert fetched_file_path == client_side / "data.jsonl" + assert fetched_file_path.read_text(encoding="utf-8") == server_data + assert urlretrieve_mock.call_count == 1 + + # Deleting and calling again should re-download + fetched_file_path.unlink() + fetched_file_path = fetch_file( + "https://example.com/data.jsonl", + folder=client_side, + ) + assert fetched_file_path == client_side / "data.jsonl" + assert fetched_file_path.read_text(encoding="utf-8") == server_data + assert urlretrieve_mock.call_count == 2 + + +def test_fetch_file_with_sha256(monkeypatch, tmpdir): + server_side = tmpdir.mkdir("server_side") + data_file = Path(server_side / "data.jsonl") + server_data = '{"a": 1, "b": 2}\n' + data_file.write_text(server_data, encoding="utf-8") + expected_sha256 = hashlib.sha256(data_file.read_bytes()).hexdigest() + + client_side = tmpdir.mkdir("client_side") + + urlretrieve_mock = _mock_urlretrieve(server_side) + monkeypatch.setattr("sklearn.datasets._base.urlretrieve", urlretrieve_mock) + + # The first call should trigger a download. + fetched_file_path = fetch_file( + "https://example.com/data.jsonl", folder=client_side, sha256=expected_sha256 + ) + assert fetched_file_path == client_side / "data.jsonl" + assert fetched_file_path.read_text(encoding="utf-8") == server_data + assert urlretrieve_mock.call_count == 1 + + # Fetching again the same file to the same folder should do nothing when + # the sha256 match: + fetched_file_path = fetch_file( + "https://example.com/data.jsonl", folder=client_side, sha256=expected_sha256 + ) + assert fetched_file_path == client_side / "data.jsonl" + assert fetched_file_path.read_text(encoding="utf-8") == server_data + assert urlretrieve_mock.call_count == 1 + + # Corrupting the local data should yield a warning and trigger a new download: + fetched_file_path.write_text("corrupted contents", encoding="utf-8") + expected_msg = ( + r"SHA256 checksum of existing local file data.jsonl " + rf"\(.*\) differs from expected \({expected_sha256}\): " + r"re-downloading from https://example.com/data.jsonl \." + ) + with pytest.warns(match=expected_msg): + fetched_file_path = fetch_file( + "https://example.com/data.jsonl", folder=client_side, sha256=expected_sha256 + ) + assert fetched_file_path == client_side / "data.jsonl" + assert fetched_file_path.read_text(encoding="utf-8") == server_data + assert urlretrieve_mock.call_count == 2 + + # Calling again should do nothing: + fetched_file_path = fetch_file( + "https://example.com/data.jsonl", folder=client_side, sha256=expected_sha256 + ) + assert fetched_file_path == client_side / "data.jsonl" + assert fetched_file_path.read_text(encoding="utf-8") == server_data + assert urlretrieve_mock.call_count == 2 + + # Deleting the local file and calling again should redownload without warning: + fetched_file_path.unlink() + fetched_file_path = fetch_file( + "https://example.com/data.jsonl", folder=client_side, sha256=expected_sha256 + ) + assert fetched_file_path == client_side / "data.jsonl" + assert fetched_file_path.read_text(encoding="utf-8") == server_data + assert urlretrieve_mock.call_count == 3 + + # Calling without a sha256 should also work without redownloading: + fetched_file_path = fetch_file( + "https://example.com/data.jsonl", + folder=client_side, + ) + assert fetched_file_path == client_side / "data.jsonl" + assert fetched_file_path.read_text(encoding="utf-8") == server_data + assert urlretrieve_mock.call_count == 3 + + # Calling with a wrong sha256 should raise an informative exception: + non_matching_sha256 = "deadbabecafebeef" + expected_warning_msg = "differs from expected" + expected_error_msg = re.escape( + f"The SHA256 checksum of data.jsonl ({expected_sha256}) differs from " + f"expected ({non_matching_sha256})." + ) + with pytest.raises(OSError, match=expected_error_msg): + with pytest.warns(match=expected_warning_msg): + fetch_file( + "https://example.com/data.jsonl", + folder=client_side, + sha256=non_matching_sha256, + ) diff --git a/sklearn/datasets/tests/test_california_housing.py b/sklearn/datasets/tests/test_california_housing.py index a8c5514e2ec73..b24fb5bd66a56 100644 --- a/sklearn/datasets/tests/test_california_housing.py +++ b/sklearn/datasets/tests/test_california_housing.py @@ -1,16 +1,19 @@ """Test the california_housing loader, if the data is available, or if specifically requested via environment variable -(e.g. for travis cron job).""" +(e.g. for CI jobs).""" + +from functools import partial + import pytest from sklearn.datasets.tests.test_common import check_return_X_y -from functools import partial def test_fetch(fetch_california_housing_fxt): data = fetch_california_housing_fxt() - assert((20640, 8) == data.data.shape) - assert((20640, ) == data.target.shape) + assert (20640, 8) == data.data.shape + assert (20640,) == data.target.shape + assert data.DESCR.startswith(".. _california_housing_dataset:") # test return_X_y option fetch_func = partial(fetch_california_housing_fxt) @@ -18,20 +21,18 @@ def test_fetch(fetch_california_housing_fxt): def test_fetch_asframe(fetch_california_housing_fxt): - pd = pytest.importorskip('pandas') + pd = pytest.importorskip("pandas") bunch = fetch_california_housing_fxt(as_frame=True) frame = bunch.frame - assert hasattr(bunch, 'frame') is True + assert hasattr(bunch, "frame") is True assert frame.shape == (20640, 9) assert isinstance(bunch.data, pd.DataFrame) assert isinstance(bunch.target, pd.Series) -def test_pandas_dependency_message(fetch_california_housing_fxt, - hide_available_pandas): +def test_pandas_dependency_message(fetch_california_housing_fxt, hide_available_pandas): # Check that pandas is imported lazily and that an informative error # message is raised when pandas is missing: - expected_msg = ('fetch_california_housing with as_frame=True' - ' requires pandas') + expected_msg = "fetch_california_housing with as_frame=True requires pandas" with pytest.raises(ImportError, match=expected_msg): fetch_california_housing_fxt(as_frame=True) diff --git a/sklearn/datasets/tests/test_common.py b/sklearn/datasets/tests/test_common.py index 2a905b75e94eb..33219deab6915 100644 --- a/sklearn/datasets/tests/test_common.py +++ b/sklearn/datasets/tests/test_common.py @@ -1,16 +1,18 @@ """Test loaders for common functionality.""" + import inspect import os -import pytest import numpy as np +import pytest import sklearn.datasets def is_pillow_installed(): try: - import PIL # noqa + import PIL # noqa: F401 + return True except ImportError: return False @@ -25,27 +27,27 @@ def is_pillow_installed(): reason="fetch_opeml requires a dataset name or id" ), "fetch_lfw_people": pytest.mark.skipif( - not is_pillow_installed(), - reason="pillow is not installed" - ) + not is_pillow_installed(), reason="pillow is not installed" + ), }, "as_frame": { "fetch_openml": pytest.mark.xfail( reason="fetch_opeml requires a dataset name or id" ), - } + }, } def check_pandas_dependency_message(fetch_func): try: - import pandas # noqa + import pandas # noqa: F401 + pytest.skip("This test requires pandas to not be installed") except ImportError: # Check that pandas is imported lazily and that an informative error # message is raised when pandas is missing: name = fetch_func.__name__ - expected_msg = f'{name} with as_frame=True requires pandas' + expected_msg = f"{name} with as_frame=True requires pandas" with pytest.raises(ImportError, match=expected_msg): fetch_func(as_frame=True) @@ -57,11 +59,12 @@ def check_return_X_y(bunch, dataset_func): assert X_y_tuple[1].shape == bunch.target.shape -def check_as_frame(bunch, dataset_func, - expected_data_dtype=None, expected_target_dtype=None): - pd = pytest.importorskip('pandas') +def check_as_frame( + bunch, dataset_func, expected_data_dtype=None, expected_target_dtype=None +): + pd = pytest.importorskip("pandas") frame_bunch = dataset_func(as_frame=True) - assert hasattr(frame_bunch, 'frame') + assert hasattr(frame_bunch, "frame") assert isinstance(frame_bunch.frame, pd.DataFrame) assert isinstance(frame_bunch.data, pd.DataFrame) assert frame_bunch.data.shape == bunch.data.shape @@ -85,7 +88,7 @@ def check_as_frame(bunch, dataset_func, def _skip_network_tests(): - return os.environ.get('SKLEARN_SKIP_NETWORK_TESTS', '1') == '1' + return os.environ.get("SKLEARN_SKIP_NETWORK_TESTS", "1") == "1" def _generate_func_supporting_param(param, dataset_type=("load", "fetch")): @@ -98,10 +101,12 @@ def _generate_func_supporting_param(param, dataset_type=("load", "fetch")): is_support_param = param in inspect.signature(obj).parameters if is_dataset_type and is_support_param: # check if we should skip if we don't have network support - marks = [pytest.mark.skipif( - condition=name.startswith("fetch") and _skip_network_tests(), - reason="Skip because fetcher requires internet network", - )] + marks = [ + pytest.mark.skipif( + condition=name.startswith("fetch") and _skip_network_tests(), + reason="Skip because fetcher requires internet network", + ) + ] if name in markers_fetch: marks.append(markers_fetch[name]) diff --git a/sklearn/datasets/tests/test_covtype.py b/sklearn/datasets/tests/test_covtype.py index 1db2ab65bde11..018505bc4fa05 100644 --- a/sklearn/datasets/tests/test_covtype.py +++ b/sklearn/datasets/tests/test_covtype.py @@ -1,25 +1,32 @@ """Test the covtype loader, if the data is available, or if specifically requested via environment variable -(e.g. for travis cron job).""" +(e.g. for CI jobs).""" + from functools import partial + import pytest + from sklearn.datasets.tests.test_common import check_return_X_y -def test_fetch(fetch_covtype_fxt): - data1 = fetch_covtype_fxt(shuffle=True, random_state=42) - data2 = fetch_covtype_fxt(shuffle=True, random_state=37) +def test_fetch(fetch_covtype_fxt, global_random_seed): + data1 = fetch_covtype_fxt(shuffle=True, random_state=global_random_seed) + data2 = fetch_covtype_fxt(shuffle=True, random_state=global_random_seed + 1) - X1, X2 = data1['data'], data2['data'] + X1, X2 = data1["data"], data2["data"] assert (581012, 54) == X1.shape assert X1.shape == X2.shape assert X1.sum() == X2.sum() - y1, y2 = data1['target'], data2['target'] + y1, y2 = data1["target"], data2["target"] assert (X1.shape[0],) == y1.shape assert (X1.shape[0],) == y2.shape + descr_prefix = ".. _covtype_dataset:" + assert data1.DESCR.startswith(descr_prefix) + assert data2.DESCR.startswith(descr_prefix) + # test return_X_y option fetch_func = partial(fetch_covtype_fxt) check_return_X_y(data1, fetch_func) @@ -29,7 +36,7 @@ def test_fetch_asframe(fetch_covtype_fxt): pytest.importorskip("pandas") bunch = fetch_covtype_fxt(as_frame=True) - assert hasattr(bunch, 'frame') + assert hasattr(bunch, "frame") frame = bunch.frame assert frame.shape == (581012, 55) assert bunch.data.shape == (581012, 54) @@ -42,9 +49,7 @@ def test_fetch_asframe(fetch_covtype_fxt): assert set(f"Soil_Type_{i}" for i in range(40)) < column_names -def test_pandas_dependency_message(fetch_covtype_fxt, - hide_available_pandas): - expected_msg = ('fetch_covtype with as_frame=True' - ' requires pandas') +def test_pandas_dependency_message(fetch_covtype_fxt, hide_available_pandas): + expected_msg = "fetch_covtype with as_frame=True requires pandas" with pytest.raises(ImportError, match=expected_msg): fetch_covtype_fxt(as_frame=True) diff --git a/sklearn/datasets/tests/test_kddcup99.py b/sklearn/datasets/tests/test_kddcup99.py index 5119d0cda13a2..8fa5e397ead90 100644 --- a/sklearn/datasets/tests/test_kddcup99.py +++ b/sklearn/datasets/tests/test_kddcup99.py @@ -1,27 +1,32 @@ """Test kddcup99 loader, if the data is available, or if specifically requested via environment variable -(e.g. for travis cron job). +(e.g. for CI jobs). Only 'percent10' mode is tested, as the full data is too big to use in unit-testing. """ from functools import partial + import pytest -from sklearn.datasets.tests.test_common import check_as_frame -from sklearn.datasets.tests.test_common import check_pandas_dependency_message -from sklearn.datasets.tests.test_common import check_return_X_y +from sklearn.datasets.tests.test_common import ( + check_as_frame, + check_pandas_dependency_message, + check_return_X_y, +) @pytest.mark.parametrize("as_frame", [True, False]) @pytest.mark.parametrize( "subset, n_samples, n_features", - [(None, 494021, 41), - ("SA", 100655, 41), - ("SF", 73237, 4), - ("http", 58725, 3), - ("smtp", 9571, 3)] + [ + (None, 494021, 41), + ("SA", 100655, 41), + ("SF", 73237, 4), + ("http", 58725, 3), + ("smtp", 9571, 3), + ], ) def test_fetch_kddcup99_percent10( fetch_kddcup99_fxt, as_frame, subset, n_samples, n_features @@ -31,10 +36,11 @@ def test_fetch_kddcup99_percent10( assert data.target.shape == (n_samples,) if as_frame: assert data.frame.shape == (n_samples, n_features + 1) + assert data.DESCR.startswith(".. _kddcup99_dataset:") def test_fetch_kddcup99_return_X_y(fetch_kddcup99_fxt): - fetch_func = partial(fetch_kddcup99_fxt, subset='smtp') + fetch_func = partial(fetch_kddcup99_fxt, subset="smtp") data = fetch_func() check_return_X_y(data, fetch_func) @@ -46,15 +52,38 @@ def test_fetch_kddcup99_as_frame(fetch_kddcup99_fxt): def test_fetch_kddcup99_shuffle(fetch_kddcup99_fxt): dataset = fetch_kddcup99_fxt( - random_state=0, subset='SA', percent10=True, + random_state=0, + subset="SA", + percent10=True, ) dataset_shuffled = fetch_kddcup99_fxt( - random_state=0, subset='SA', shuffle=True, percent10=True, + random_state=0, + subset="SA", + shuffle=True, + percent10=True, ) - assert set(dataset['target']) == set(dataset_shuffled['target']) + assert set(dataset["target"]) == set(dataset_shuffled["target"]) assert dataset_shuffled.data.shape == dataset.data.shape assert dataset_shuffled.target.shape == dataset.target.shape def test_pandas_dependency_message(fetch_kddcup99_fxt, hide_available_pandas): check_pandas_dependency_message(fetch_kddcup99_fxt) + + +def test_corrupted_file_error_message(fetch_kddcup99_fxt, tmp_path): + """Check that a nice error message is raised when cache is corrupted.""" + kddcup99_dir = tmp_path / "kddcup99_10-py3" + kddcup99_dir.mkdir() + samples_path = kddcup99_dir / "samples" + + with samples_path.open("wb") as f: + f.write(b"THIS IS CORRUPTED") + + msg = ( + "The cache for fetch_kddcup99 is invalid, please " + f"delete {kddcup99_dir} and run the fetch_kddcup99 again" + ) + + with pytest.raises(OSError, match=msg): + fetch_kddcup99_fxt(data_home=str(tmp_path)) diff --git a/sklearn/datasets/tests/test_lfw.py b/sklearn/datasets/tests/test_lfw.py index 19cda818d8d55..cc86fe8637232 100644 --- a/sklearn/datasets/tests/test_lfw.py +++ b/sklearn/datasets/tests/test_lfw.py @@ -9,51 +9,41 @@ """ import random -import os -import shutil -import tempfile +from functools import partial + import numpy as np import pytest -from functools import partial -from sklearn.externals._pilutil import pillow_installed, imsave -from sklearn.datasets import fetch_lfw_pairs -from sklearn.datasets import fetch_lfw_people -from sklearn.utils._testing import assert_array_equal -from sklearn.utils._testing import SkipTest +from sklearn.datasets import fetch_lfw_pairs, fetch_lfw_people from sklearn.datasets.tests.test_common import check_return_X_y - - -SCIKIT_LEARN_DATA = None -SCIKIT_LEARN_EMPTY_DATA = None -LFW_HOME = None +from sklearn.utils._testing import assert_array_equal FAKE_NAMES = [ - 'Abdelatif_Smith', - 'Abhati_Kepler', - 'Camara_Alvaro', - 'Chen_Dupont', - 'John_Lee', - 'Lin_Bauman', - 'Onur_Lopez', + "Abdelatif_Smith", + "Abhati_Kepler", + "Camara_Alvaro", + "Chen_Dupont", + "John_Lee", + "Lin_Bauman", + "Onur_Lopez", ] -def setup_module(): - """Test fixture run once and common to all tests of this module""" - if not pillow_installed: - raise SkipTest("PIL not installed.") +@pytest.fixture(scope="module") +def mock_empty_data_home(tmp_path_factory): + data_dir = tmp_path_factory.mktemp("scikit_learn_empty_test") - global SCIKIT_LEARN_DATA, SCIKIT_LEARN_EMPTY_DATA, LFW_HOME + yield data_dir - SCIKIT_LEARN_DATA = tempfile.mkdtemp(prefix="scikit_learn_lfw_test_") - LFW_HOME = os.path.join(SCIKIT_LEARN_DATA, 'lfw_home') - SCIKIT_LEARN_EMPTY_DATA = tempfile.mkdtemp( - prefix="scikit_learn_empty_test_") +@pytest.fixture(scope="module") +def mock_data_home(tmp_path_factory): + """Test fixture run once and common to all tests of this module""" + Image = pytest.importorskip("PIL.Image") - if not os.path.exists(LFW_HOME): - os.makedirs(LFW_HOME) + data_dir = tmp_path_factory.mktemp("scikit_learn_lfw_test") + lfw_home = data_dir / "lfw_home" + lfw_home.mkdir(parents=True, exist_ok=True) random_state = random.Random(42) np_rng = np.random.RandomState(42) @@ -61,67 +51,59 @@ def setup_module(): # generate some random jpeg files for each person counts = {} for name in FAKE_NAMES: - folder_name = os.path.join(LFW_HOME, 'lfw_funneled', name) - if not os.path.exists(folder_name): - os.makedirs(folder_name) + folder_name = lfw_home / "lfw_funneled" / name + folder_name.mkdir(parents=True, exist_ok=True) n_faces = np_rng.randint(1, 5) counts[name] = n_faces for i in range(n_faces): - file_path = os.path.join(folder_name, name + '_%04d.jpg' % i) + file_path = folder_name / (name + "_%04d.jpg" % i) uniface = np_rng.randint(0, 255, size=(250, 250, 3)) - try: - imsave(file_path, uniface) - except ImportError: - raise SkipTest("PIL not installed") + img = Image.fromarray(uniface.astype(np.uint8)) + img.save(file_path) # add some random file pollution to test robustness - with open(os.path.join(LFW_HOME, 'lfw_funneled', '.test.swp'), 'wb') as f: - f.write(b'Text file to be ignored by the dataset loader.') + (lfw_home / "lfw_funneled" / ".test.swp").write_bytes( + b"Text file to be ignored by the dataset loader." + ) # generate some pairing metadata files using the same format as LFW - with open(os.path.join(LFW_HOME, 'pairsDevTrain.txt'), 'wb') as f: + with open(lfw_home / "pairsDevTrain.txt", "wb") as f: f.write(b"10\n") - more_than_two = [name for name, count in counts.items() - if count >= 2] + more_than_two = [name for name, count in counts.items() if count >= 2] for i in range(5): name = random_state.choice(more_than_two) first, second = random_state.sample(range(counts[name]), 2) - f.write(('%s\t%d\t%d\n' % (name, first, second)).encode()) + f.write(("%s\t%d\t%d\n" % (name, first, second)).encode()) for i in range(5): first_name, second_name = random_state.sample(FAKE_NAMES, 2) - first_index = random_state.choice(np.arange(counts[first_name])) - second_index = random_state.choice(np.arange(counts[second_name])) - f.write(('%s\t%d\t%s\t%d\n' % (first_name, first_index, - second_name, second_index) - ).encode()) + first_index = np_rng.choice(np.arange(counts[first_name])) + second_index = np_rng.choice(np.arange(counts[second_name])) + f.write( + ( + "%s\t%d\t%s\t%d\n" + % (first_name, first_index, second_name, second_index) + ).encode() + ) - with open(os.path.join(LFW_HOME, 'pairsDevTest.txt'), 'wb') as f: - f.write(b"Fake place holder that won't be tested") + (lfw_home / "pairsDevTest.txt").write_bytes( + b"Fake place holder that won't be tested" + ) + (lfw_home / "pairs.txt").write_bytes(b"Fake place holder that won't be tested") - with open(os.path.join(LFW_HOME, 'pairs.txt'), 'wb') as f: - f.write(b"Fake place holder that won't be tested") + yield data_dir -def teardown_module(): - """Test fixture (clean up) run once after all tests of this module""" - if os.path.isdir(SCIKIT_LEARN_DATA): - shutil.rmtree(SCIKIT_LEARN_DATA) - if os.path.isdir(SCIKIT_LEARN_EMPTY_DATA): - shutil.rmtree(SCIKIT_LEARN_EMPTY_DATA) +def test_load_empty_lfw_people(mock_empty_data_home): + with pytest.raises(OSError): + fetch_lfw_people(data_home=mock_empty_data_home, download_if_missing=False) -def test_load_empty_lfw_people(): - with pytest.raises(IOError): - fetch_lfw_people(data_home=SCIKIT_LEARN_EMPTY_DATA, - download_if_missing=False) - - -def test_load_fake_lfw_people(): - lfw_people = fetch_lfw_people(data_home=SCIKIT_LEARN_DATA, - min_faces_per_person=3, - download_if_missing=False) +def test_load_fake_lfw_people(mock_data_home): + lfw_people = fetch_lfw_people( + data_home=mock_data_home, min_faces_per_person=3, download_if_missing=False + ) # The data is croped around the center as a rectangular bounding box # around the face. Colors are converted to gray levels: @@ -132,46 +114,68 @@ def test_load_fake_lfw_people(): assert_array_equal(lfw_people.target, [2, 0, 1, 0, 2, 0, 2, 1, 1, 2]) # names of the persons can be found using the target_names array - expected_classes = ['Abdelatif Smith', 'Abhati Kepler', 'Onur Lopez'] + expected_classes = ["Abdelatif Smith", "Abhati Kepler", "Onur Lopez"] assert_array_equal(lfw_people.target_names, expected_classes) # It is possible to ask for the original data without any croping or color # conversion and not limit on the number of picture per person - lfw_people = fetch_lfw_people(data_home=SCIKIT_LEARN_DATA, resize=None, - slice_=None, color=True, - download_if_missing=False) + lfw_people = fetch_lfw_people( + data_home=mock_data_home, + resize=None, + slice_=None, + color=True, + download_if_missing=False, + ) assert lfw_people.images.shape == (17, 250, 250, 3) + assert lfw_people.DESCR.startswith(".. _labeled_faces_in_the_wild_dataset:") # the ids and class names are the same as previously - assert_array_equal(lfw_people.target, - [0, 0, 1, 6, 5, 6, 3, 6, 0, 3, 6, 1, 2, 4, 5, 1, 2]) - assert_array_equal(lfw_people.target_names, - ['Abdelatif Smith', 'Abhati Kepler', 'Camara Alvaro', - 'Chen Dupont', 'John Lee', 'Lin Bauman', 'Onur Lopez']) + assert_array_equal( + lfw_people.target, [0, 0, 1, 6, 5, 6, 3, 6, 0, 3, 6, 1, 2, 4, 5, 1, 2] + ) + assert_array_equal( + lfw_people.target_names, + [ + "Abdelatif Smith", + "Abhati Kepler", + "Camara Alvaro", + "Chen Dupont", + "John Lee", + "Lin Bauman", + "Onur Lopez", + ], + ) # test return_X_y option - fetch_func = partial(fetch_lfw_people, data_home=SCIKIT_LEARN_DATA, - resize=None, - slice_=None, color=True, - download_if_missing=False) + fetch_func = partial( + fetch_lfw_people, + data_home=mock_data_home, + resize=None, + slice_=None, + color=True, + download_if_missing=False, + ) check_return_X_y(lfw_people, fetch_func) -def test_load_fake_lfw_people_too_restrictive(): +def test_load_fake_lfw_people_too_restrictive(mock_data_home): with pytest.raises(ValueError): - fetch_lfw_people(data_home=SCIKIT_LEARN_DATA, min_faces_per_person=100, - download_if_missing=False) + fetch_lfw_people( + data_home=mock_data_home, + min_faces_per_person=100, + download_if_missing=False, + ) -def test_load_empty_lfw_pairs(): - with pytest.raises(IOError): - fetch_lfw_pairs(data_home=SCIKIT_LEARN_EMPTY_DATA, - download_if_missing=False) +def test_load_empty_lfw_pairs(mock_empty_data_home): + with pytest.raises(OSError): + fetch_lfw_pairs(data_home=mock_empty_data_home, download_if_missing=False) -def test_load_fake_lfw_pairs(): - lfw_pairs_train = fetch_lfw_pairs(data_home=SCIKIT_LEARN_DATA, - download_if_missing=False) +def test_load_fake_lfw_pairs(mock_data_home): + lfw_pairs_train = fetch_lfw_pairs( + data_home=mock_data_home, download_if_missing=False + ) # The data is croped around the center as a rectangular bounding box # around the face. Colors are converted to gray levels: @@ -181,16 +185,45 @@ def test_load_fake_lfw_pairs(): assert_array_equal(lfw_pairs_train.target, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0]) # names of the persons can be found using the target_names array - expected_classes = ['Different persons', 'Same person'] + expected_classes = ["Different persons", "Same person"] assert_array_equal(lfw_pairs_train.target_names, expected_classes) # It is possible to ask for the original data without any croping or color # conversion - lfw_pairs_train = fetch_lfw_pairs(data_home=SCIKIT_LEARN_DATA, resize=None, - slice_=None, color=True, - download_if_missing=False) + lfw_pairs_train = fetch_lfw_pairs( + data_home=mock_data_home, + resize=None, + slice_=None, + color=True, + download_if_missing=False, + ) assert lfw_pairs_train.pairs.shape == (10, 2, 250, 250, 3) # the ids and class names are the same as previously assert_array_equal(lfw_pairs_train.target, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0]) assert_array_equal(lfw_pairs_train.target_names, expected_classes) + + assert lfw_pairs_train.DESCR.startswith(".. _labeled_faces_in_the_wild_dataset:") + + +def test_fetch_lfw_people_internal_cropping(mock_data_home): + """Check that we properly crop the images. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/24942 + """ + # If cropping was not done properly and we don't resize the images, the images would + # have their original size (250x250) and the image would not fit in the NumPy array + # pre-allocated based on `slice_` parameter. + slice_ = (slice(70, 195), slice(78, 172)) + lfw = fetch_lfw_people( + data_home=mock_data_home, + min_faces_per_person=3, + download_if_missing=False, + resize=None, + slice_=slice_, + ) + assert lfw.images[0].shape == ( + slice_[0].stop - slice_[0].start, + slice_[1].stop - slice_[1].start, + ) diff --git a/sklearn/datasets/tests/test_olivetti_faces.py b/sklearn/datasets/tests/test_olivetti_faces.py index f0c7aa1216e76..e5d6c853aa454 100644 --- a/sklearn/datasets/tests/test_olivetti_faces.py +++ b/sklearn/datasets/tests/test_olivetti_faces.py @@ -1,12 +1,11 @@ """Test Olivetti faces fetcher, if the data is available, or if specifically requested via environment variable -(e.g. for travis cron job).""" +(e.g. for CI jobs).""" import numpy as np -from sklearn.utils import Bunch from sklearn.datasets.tests.test_common import check_return_X_y - +from sklearn.utils import Bunch from sklearn.utils._testing import assert_array_equal @@ -14,13 +13,14 @@ def test_olivetti_faces(fetch_olivetti_faces_fxt): data = fetch_olivetti_faces_fxt(shuffle=True, random_state=0) assert isinstance(data, Bunch) - for expected_keys in ('data', 'images', 'target', 'DESCR'): + for expected_keys in ("data", "images", "target", "DESCR"): assert expected_keys in data.keys() assert data.data.shape == (400, 4096) assert data.images.shape == (400, 64, 64) assert data.target.shape == (400,) assert_array_equal(np.unique(np.sort(data.target)), np.arange(40)) + assert data.DESCR.startswith(".. _olivetti_faces_dataset:") # test the return_X_y option check_return_X_y(data, fetch_olivetti_faces_fxt) diff --git a/sklearn/datasets/tests/test_openml.py b/sklearn/datasets/tests/test_openml.py index dac0762eb2160..40e086ec6f6d3 100644 --- a/sklearn/datasets/tests/test_openml.py +++ b/sklearn/datasets/tests/test_openml.py @@ -1,161 +1,38 @@ -"""Test the openml loader. -""" +"""Test the openml loader.""" + import gzip -import warnings import json import os import re +from functools import partial +from importlib import resources from io import BytesIO +from urllib.error import HTTPError import numpy as np +import pytest import scipy.sparse + import sklearn -import pytest from sklearn import config_context -from sklearn.datasets import fetch_openml -from sklearn.datasets._openml import (_open_openml_url, - _arff, - _DATA_FILE, - _convert_arff_data, - _convert_arff_data_dataframe, - _get_data_description_by_id, - _get_local_path, - _retry_with_clean_cache, - _feature_to_dtype) -from sklearn.utils._testing import (assert_warns_message, - assert_raise_message) -from sklearn.utils import is_scalar_nan -from sklearn.utils._testing import assert_allclose, assert_array_equal -from urllib.error import HTTPError -from sklearn.datasets.tests.test_common import check_return_X_y -from sklearn.externals._arff import ArffContainerType -from functools import partial -from sklearn.utils._testing import fails_if_pypy - - -currdir = os.path.dirname(os.path.abspath(__file__)) +from sklearn.datasets import fetch_openml as fetch_openml_orig +from sklearn.datasets._openml import ( + _get_local_path, + _open_openml_url, + _retry_with_clean_cache, +) +from sklearn.utils import Bunch +from sklearn.utils._optional_dependencies import check_pandas_support +from sklearn.utils._testing import ( + SkipTest, + assert_allclose, + assert_array_equal, +) + +OPENML_TEST_DATA_MODULE = "sklearn.datasets.tests.data.openml" # if True, urlopen will be monkey patched to only use local files test_offline = True - - -def _test_features_list(data_id): - # XXX Test is intended to verify/ensure correct decoding behavior - # Not usable with sparse data or datasets that have columns marked as - # {row_identifier, ignore} - def decode_column(data_bunch, col_idx): - col_name = data_bunch.feature_names[col_idx] - if col_name in data_bunch.categories: - # XXX: This would be faster with np.take, although it does not - # handle missing values fast (also not with mode='wrap') - cat = data_bunch.categories[col_name] - result = [None if is_scalar_nan(idx) else cat[int(idx)] - for idx in data_bunch.data[:, col_idx]] - return np.array(result, dtype='O') - else: - # non-nominal attribute - return data_bunch.data[:, col_idx] - - data_bunch = fetch_openml(data_id=data_id, cache=False, - target_column=None, as_frame=False) - - # also obtain decoded arff - data_description = _get_data_description_by_id(data_id, None) - sparse = data_description['format'].lower() == 'sparse_arff' - if sparse is True: - raise ValueError('This test is not intended for sparse data, to keep ' - 'code relatively simple') - url = _DATA_FILE.format(data_description['file_id']) - with _open_openml_url(url, data_home=None) as f: - data_arff = _arff.load((line.decode('utf-8') for line in f), - return_type=(_arff.COO if sparse - else _arff.DENSE_GEN), - encode_nominal=False) - - data_downloaded = np.array(list(data_arff['data']), dtype='O') - - for i in range(len(data_bunch.feature_names)): - # XXX: Test per column, as this makes it easier to avoid problems with - # missing values - - np.testing.assert_array_equal(data_downloaded[:, i], - decode_column(data_bunch, i)) - - -def _fetch_dataset_from_openml(data_id, data_name, data_version, - target_column, - expected_observations, expected_features, - expected_missing, - expected_data_dtype, expected_target_dtype, - expect_sparse, compare_default_target): - # fetches a dataset in three various ways from OpenML, using the - # fetch_openml function, and does various checks on the validity of the - # result. Note that this function can be mocked (by invoking - # _monkey_patch_webbased_functions before invoking this function) - data_by_name_id = fetch_openml(name=data_name, version=data_version, - cache=False, as_frame=False) - assert int(data_by_name_id.details['id']) == data_id - - # Please note that cache=False is crucial, as the monkey patched files are - # not consistent with reality - with warnings.catch_warnings(): - # See discussion in PR #19373 - # Catching UserWarnings about multiple versions of dataset - warnings.simplefilter("ignore", category=UserWarning) - fetch_openml(name=data_name, cache=False, as_frame=False) - # without specifying the version, there is no guarantee that the data id - # will be the same - - # fetch with dataset id - data_by_id = fetch_openml(data_id=data_id, cache=False, - target_column=target_column, as_frame=False) - assert data_by_id.details['name'] == data_name - assert data_by_id.data.shape == (expected_observations, expected_features) - if isinstance(target_column, str): - # single target, so target is vector - assert data_by_id.target.shape == (expected_observations, ) - assert data_by_id.target_names == [target_column] - elif isinstance(target_column, list): - # multi target, so target is array - assert data_by_id.target.shape == (expected_observations, - len(target_column)) - assert data_by_id.target_names == target_column - assert data_by_id.data.dtype == expected_data_dtype - assert data_by_id.target.dtype == expected_target_dtype - assert len(data_by_id.feature_names) == expected_features - for feature in data_by_id.feature_names: - assert isinstance(feature, str) - - # TODO: pass in a list of expected nominal features - for feature, categories in data_by_id.categories.items(): - feature_idx = data_by_id.feature_names.index(feature) - values = np.unique(data_by_id.data[:, feature_idx]) - values = values[np.isfinite(values)] - assert set(values) <= set(range(len(categories))) - - if compare_default_target: - # check whether the data by id and data by id target are equal - data_by_id_default = fetch_openml(data_id=data_id, cache=False, - as_frame=False) - np.testing.assert_allclose(data_by_id.data, data_by_id_default.data) - if data_by_id.target.dtype == np.float64: - np.testing.assert_allclose(data_by_id.target, - data_by_id_default.target) - else: - assert np.array_equal(data_by_id.target, data_by_id_default.target) - - if expect_sparse: - assert isinstance(data_by_id.data, scipy.sparse.csr_matrix) - else: - assert isinstance(data_by_id.data, np.ndarray) - # np.isnan doesn't work on CSR matrix - assert (np.count_nonzero(np.isnan(data_by_id.data)) == - expected_missing) - - # test return_X_y option - fetch_func = partial(fetch_openml, data_id=data_id, cache=False, - target_column=target_column, as_frame=False) - check_return_X_y(data_by_id, fetch_func) - return data_by_id +_MONKEY_PATCH_LOCAL_OPENML_PATH = "data/v1/download/{}" class _MockHTTPResponse: @@ -171,7 +48,7 @@ def close(self): def info(self): if self.is_gzip: - return {'Content-Encoding': 'gzip'} + return {"Content-Encoding": "gzip"} return {} def __iter__(self): @@ -184,93 +61,130 @@ def __exit__(self, exc_type, exc_val, exc_tb): return False -def _monkey_patch_webbased_functions(context, - data_id, - gzip_response): +# Disable the disk-based cache when testing `fetch_openml`: +# the mock data in sklearn/datasets/tests/data/openml/ is not always consistent +# with the version on openml.org. If one were to load the dataset outside of +# the tests, it may result in data that does not represent openml.org. +fetch_openml = partial(fetch_openml_orig, data_home=None) + + +def _monkey_patch_webbased_functions(context, data_id, gzip_response): # monkey patches the urlopen function. Important note: Do NOT use this # in combination with a regular cache directory, as the files that are # stored as cache should not be mixed up with real openml datasets - url_prefix_data_description = "https://openml.org/api/v1/json/data/" - url_prefix_data_features = "https://openml.org/api/v1/json/data/features/" - url_prefix_download_data = "https://openml.org/data/v1/" - url_prefix_data_list = "https://openml.org/api/v1/json/data/list/" + url_prefix_data_description = "https://api.openml.org/api/v1/json/data/" + url_prefix_data_features = "https://api.openml.org/api/v1/json/data/features/" + url_prefix_download_data = "https://www.openml.org/data/v1/download" + url_prefix_data_list = "https://api.openml.org/api/v1/json/data/list/" - path_suffix = '.gz' + path_suffix = ".gz" read_fn = gzip.open + data_module = OPENML_TEST_DATA_MODULE + "." + f"id_{data_id}" + def _file_name(url, suffix): - return (re.sub(r'\W', '-', url[len("https://openml.org/"):]) - + suffix + path_suffix) + output = ( + re.sub(r"\W", "-", url[len("https://api.openml.org/") :]) + + suffix + + path_suffix + ) + # Shorten the filenames to have better compatibility with windows 10 + # and filenames > 260 characters + return ( + output.replace("-json-data-list", "-jdl") + .replace("-json-data-features", "-jdf") + .replace("-json-data-qualities", "-jdq") + .replace("-json-data", "-jd") + .replace("-data_name", "-dn") + .replace("-download", "-dl") + .replace("-limit", "-l") + .replace("-data_version", "-dv") + .replace("-status", "-s") + .replace("-deactivated", "-dact") + .replace("-active", "-act") + ) + + def _mock_urlopen_shared(url, has_gzip_header, expected_prefix, suffix): + assert url.startswith(expected_prefix), ( + f"{expected_prefix!r} does not match {url!r}" + ) + + data_file_name = _file_name(url, suffix) + data_file_path = resources.files(data_module) / data_file_name + + with data_file_path.open("rb") as f: + if has_gzip_header and gzip_response: + fp = BytesIO(f.read()) + return _MockHTTPResponse(fp, True) + else: + decompressed_f = read_fn(f, "rb") + fp = BytesIO(decompressed_f.read()) + return _MockHTTPResponse(fp, False) def _mock_urlopen_data_description(url, has_gzip_header): - assert url.startswith(url_prefix_data_description) - - path = os.path.join(currdir, 'data', 'openml', str(data_id), - _file_name(url, '.json')) - - if has_gzip_header and gzip_response: - with open(path, 'rb') as f: - fp = BytesIO(f.read()) - return _MockHTTPResponse(fp, True) - else: - with read_fn(path, 'rb') as f: - fp = BytesIO(f.read()) - return _MockHTTPResponse(fp, False) + return _mock_urlopen_shared( + url=url, + has_gzip_header=has_gzip_header, + expected_prefix=url_prefix_data_description, + suffix=".json", + ) def _mock_urlopen_data_features(url, has_gzip_header): - assert url.startswith(url_prefix_data_features) - path = os.path.join(currdir, 'data', 'openml', str(data_id), - _file_name(url, '.json')) - - if has_gzip_header and gzip_response: - with open(path, 'rb') as f: - fp = BytesIO(f.read()) - return _MockHTTPResponse(fp, True) - else: - with read_fn(path, 'rb') as f: - fp = BytesIO(f.read()) - return _MockHTTPResponse(fp, False) + return _mock_urlopen_shared( + url=url, + has_gzip_header=has_gzip_header, + expected_prefix=url_prefix_data_features, + suffix=".json", + ) def _mock_urlopen_download_data(url, has_gzip_header): - assert (url.startswith(url_prefix_download_data)) - - path = os.path.join(currdir, 'data', 'openml', str(data_id), - _file_name(url, '.arff')) - - if has_gzip_header and gzip_response: - with open(path, 'rb') as f: - fp = BytesIO(f.read()) - return _MockHTTPResponse(fp, True) - else: - with read_fn(path, 'rb') as f: - fp = BytesIO(f.read()) - return _MockHTTPResponse(fp, False) + # For simplicity the mock filenames don't contain the filename, i.e. + # the last part of the data description url after the last /. + # For example for id_1, data description download url is: + # gunzip -c sklearn/datasets/tests/data/openml/id_1/api-v1-jd-1.json.gz | grep '"url" # noqa: E501 + # "https:\/\/www.openml.org\/data\/v1\/download\/1\/anneal.arff" + # but the mock filename does not contain anneal.arff and is: + # sklearn/datasets/tests/data/openml/id_1/data-v1-dl-1.arff.gz. + # We only keep the part of the url before the last / + url_without_filename = url.rsplit("/", 1)[0] + + return _mock_urlopen_shared( + url=url_without_filename, + has_gzip_header=has_gzip_header, + expected_prefix=url_prefix_download_data, + suffix=".arff", + ) def _mock_urlopen_data_list(url, has_gzip_header): - assert url.startswith(url_prefix_data_list) + assert url.startswith(url_prefix_data_list), ( + f"{url_prefix_data_list!r} does not match {url!r}" + ) + + data_file_name = _file_name(url, ".json") + data_file_path = resources.files(data_module) / data_file_name - json_file_path = os.path.join(currdir, 'data', 'openml', - str(data_id), _file_name(url, '.json')) # load the file itself, to simulate a http error - json_data = json.loads(read_fn(json_file_path, 'rb'). - read().decode('utf-8')) - if 'error' in json_data: - raise HTTPError(url=None, code=412, - msg='Simulated mock error', - hdrs=None, fp=None) - - if has_gzip_header: - with open(json_file_path, 'rb') as f: - fp = BytesIO(f.read()) - return _MockHTTPResponse(fp, True) - else: - with read_fn(json_file_path, 'rb') as f: + with data_file_path.open("rb") as f: + decompressed_f = read_fn(f, "rb") + decoded_s = decompressed_f.read().decode("utf-8") + json_data = json.loads(decoded_s) + if "error" in json_data: + raise HTTPError( + url=None, code=412, msg="Simulated mock error", hdrs=None, fp=BytesIO() + ) + + with data_file_path.open("rb") as f: + if has_gzip_header: fp = BytesIO(f.read()) - return _MockHTTPResponse(fp, False) + return _MockHTTPResponse(fp, True) + else: + decompressed_f = read_fn(f, "rb") + fp = BytesIO(decompressed_f.read()) + return _MockHTTPResponse(fp, False) - def _mock_urlopen(request): + def _mock_urlopen(request, *args, **kwargs): url = request.get_full_url() - has_gzip_header = request.get_header('Accept-encoding') == "gzip" + has_gzip_header = request.get_header("Accept-encoding") == "gzip" if url.startswith(url_prefix_data_list): return _mock_urlopen_data_list(url, has_gzip_header) elif url.startswith(url_prefix_data_features): @@ -280,747 +194,1212 @@ def _mock_urlopen(request): elif url.startswith(url_prefix_data_description): return _mock_urlopen_data_description(url, has_gzip_header) else: - raise ValueError('Unknown mocking URL pattern: %s' % url) + raise ValueError("Unknown mocking URL pattern: %s" % url) # XXX: Global variable if test_offline: - context.setattr(sklearn.datasets._openml, 'urlopen', _mock_urlopen) - - -@pytest.mark.parametrize('feature, expected_dtype', [ - ({'data_type': 'string', 'number_of_missing_values': '0'}, object), - ({'data_type': 'string', 'number_of_missing_values': '1'}, object), - ({'data_type': 'numeric', 'number_of_missing_values': '0'}, np.float64), - ({'data_type': 'numeric', 'number_of_missing_values': '1'}, np.float64), - ({'data_type': 'real', 'number_of_missing_values': '0'}, np.float64), - ({'data_type': 'real', 'number_of_missing_values': '1'}, np.float64), - ({'data_type': 'integer', 'number_of_missing_values': '0'}, np.int64), - ({'data_type': 'integer', 'number_of_missing_values': '1'}, np.float64), - ({'data_type': 'nominal', 'number_of_missing_values': '0'}, 'category'), - ({'data_type': 'nominal', 'number_of_missing_values': '1'}, 'category'), -]) -def test_feature_to_dtype(feature, expected_dtype): - assert _feature_to_dtype(feature) == expected_dtype - - -@pytest.mark.parametrize('feature', [ - {'data_type': 'datatime', 'number_of_missing_values': '0'} -]) -def test_feature_to_dtype_error(feature): - msg = 'Unsupported feature: {}'.format(feature) - with pytest.raises(ValueError, match=msg): - _feature_to_dtype(feature) - - -# Known failure of PyPy for OpenML. See the following issue: -# https://github.com/scikit-learn/scikit-learn/issues/18906 -@fails_if_pypy -def test_fetch_openml_iris_pandas(monkeypatch): - # classification dataset with numeric only columns - pd = pytest.importorskip('pandas') - CategoricalDtype = pd.api.types.CategoricalDtype - data_id = 61 - data_shape = (150, 4) - target_shape = (150, ) - frame_shape = (150, 5) - - target_dtype = CategoricalDtype(['Iris-setosa', 'Iris-versicolor', - 'Iris-virginica']) - data_dtypes = [np.float64] * 4 - data_names = ['sepallength', 'sepalwidth', 'petallength', 'petalwidth'] - target_name = 'class' + context.setattr(sklearn.datasets._openml, "urlopen", _mock_urlopen) + + +############################################################################### +# Test the behaviour of `fetch_openml` depending of the input parameters. + + +@pytest.mark.parametrize( + "data_id, dataset_params, n_samples, n_features, n_targets", + [ + # iris + (61, {"data_id": 61}, 150, 4, 1), + (61, {"name": "iris", "version": 1}, 150, 4, 1), + # anneal + (2, {"data_id": 2}, 11, 38, 1), + (2, {"name": "anneal", "version": 1}, 11, 38, 1), + # cpu + (561, {"data_id": 561}, 209, 7, 1), + (561, {"name": "cpu", "version": 1}, 209, 7, 1), + # emotions + (40589, {"data_id": 40589}, 13, 72, 6), + # adult-census + (1119, {"data_id": 1119}, 10, 14, 1), + (1119, {"name": "adult-census"}, 10, 14, 1), + # miceprotein + (40966, {"data_id": 40966}, 7, 77, 1), + (40966, {"name": "MiceProtein"}, 7, 77, 1), + # titanic + (40945, {"data_id": 40945}, 1309, 13, 1), + ], +) +@pytest.mark.parametrize("parser", ["liac-arff", "pandas"]) +@pytest.mark.parametrize("gzip_response", [True, False]) +def test_fetch_openml_as_frame_true( + monkeypatch, + data_id, + dataset_params, + n_samples, + n_features, + n_targets, + parser, + gzip_response, +): + """Check the behaviour of `fetch_openml` with `as_frame=True`. + + Fetch by ID and/or name (depending if the file was previously cached). + """ + pd = pytest.importorskip("pandas") + + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=gzip_response) + bunch = fetch_openml( + as_frame=True, + cache=False, + parser=parser, + **dataset_params, + ) - _monkey_patch_webbased_functions(monkeypatch, data_id, True) + assert int(bunch.details["id"]) == data_id + assert isinstance(bunch, Bunch) - bunch = fetch_openml(data_id=data_id, as_frame=True, cache=False) - data = bunch.data - target = bunch.target - frame = bunch.frame + assert isinstance(bunch.frame, pd.DataFrame) + assert bunch.frame.shape == (n_samples, n_features + n_targets) - assert isinstance(data, pd.DataFrame) - assert np.all(data.dtypes == data_dtypes) - assert data.shape == data_shape - assert np.all(data.columns == data_names) - assert np.all(bunch.feature_names == data_names) - assert bunch.target_names == [target_name] + assert isinstance(bunch.data, pd.DataFrame) + assert bunch.data.shape == (n_samples, n_features) - assert isinstance(target, pd.Series) - assert target.dtype == target_dtype - assert target.shape == target_shape - assert target.name == target_name - assert target.index.is_unique + if n_targets == 1: + assert isinstance(bunch.target, pd.Series) + assert bunch.target.shape == (n_samples,) + else: + assert isinstance(bunch.target, pd.DataFrame) + assert bunch.target.shape == (n_samples, n_targets) + + assert bunch.categories is None + + +@pytest.mark.parametrize( + "data_id, dataset_params, n_samples, n_features, n_targets", + [ + # iris + (61, {"data_id": 61}, 150, 4, 1), + (61, {"name": "iris", "version": 1}, 150, 4, 1), + # anneal + (2, {"data_id": 2}, 11, 38, 1), + (2, {"name": "anneal", "version": 1}, 11, 38, 1), + # cpu + (561, {"data_id": 561}, 209, 7, 1), + (561, {"name": "cpu", "version": 1}, 209, 7, 1), + # emotions + (40589, {"data_id": 40589}, 13, 72, 6), + # adult-census + (1119, {"data_id": 1119}, 10, 14, 1), + (1119, {"name": "adult-census"}, 10, 14, 1), + # miceprotein + (40966, {"data_id": 40966}, 7, 77, 1), + (40966, {"name": "MiceProtein"}, 7, 77, 1), + ], +) +@pytest.mark.parametrize("parser", ["liac-arff", "pandas"]) +def test_fetch_openml_as_frame_false( + monkeypatch, + data_id, + dataset_params, + n_samples, + n_features, + n_targets, + parser, +): + """Check the behaviour of `fetch_openml` with `as_frame=False`. + + Fetch both by ID and/or name + version. + """ + pytest.importorskip("pandas") + + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=True) + bunch = fetch_openml( + as_frame=False, + cache=False, + parser=parser, + **dataset_params, + ) + assert int(bunch.details["id"]) == data_id + assert isinstance(bunch, Bunch) - assert isinstance(frame, pd.DataFrame) - assert frame.shape == frame_shape - assert np.all(frame.dtypes == data_dtypes + [target_dtype]) - assert frame.index.is_unique + assert bunch.frame is None + assert isinstance(bunch.data, np.ndarray) + assert bunch.data.shape == (n_samples, n_features) -# Known failure of PyPy for OpenML. See the following issue: -# https://github.com/scikit-learn/scikit-learn/issues/18906 -@fails_if_pypy -def test_fetch_openml_iris_pandas_equal_to_no_frame(monkeypatch): - # as_frame = True returns the same underlying data as as_frame = False - pytest.importorskip('pandas') - data_id = 61 + assert isinstance(bunch.target, np.ndarray) + if n_targets == 1: + assert bunch.target.shape == (n_samples,) + else: + assert bunch.target.shape == (n_samples, n_targets) - _monkey_patch_webbased_functions(monkeypatch, data_id, True) + assert isinstance(bunch.categories, dict) - frame_bunch = fetch_openml(data_id=data_id, as_frame=True, cache=False) - frame_data = frame_bunch.data - frame_target = frame_bunch.target - norm_bunch = fetch_openml(data_id=data_id, as_frame=False, cache=False) - norm_data = norm_bunch.data - norm_target = norm_bunch.target +@pytest.mark.parametrize("data_id", [61, 1119, 40945]) +def test_fetch_openml_consistency_parser(monkeypatch, data_id): + """Check the consistency of the LIAC-ARFF and pandas parsers.""" + pd = pytest.importorskip("pandas") - assert_allclose(norm_data, frame_data) - assert_array_equal(norm_target, frame_target) + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=True) + bunch_liac = fetch_openml( + data_id=data_id, + as_frame=True, + cache=False, + parser="liac-arff", + ) + bunch_pandas = fetch_openml( + data_id=data_id, + as_frame=True, + cache=False, + parser="pandas", + ) + # The data frames for the input features should match up to some numerical + # dtype conversions (e.g. float64 <=> Int64) due to limitations of the + # LIAC-ARFF parser. + data_liac, data_pandas = bunch_liac.data, bunch_pandas.data -# Known failure of PyPy for OpenML. See the following issue: -# https://github.com/scikit-learn/scikit-learn/issues/18906 -@fails_if_pypy -def test_fetch_openml_iris_multitarget_pandas(monkeypatch): - # classification dataset with numeric only columns - pd = pytest.importorskip('pandas') - CategoricalDtype = pd.api.types.CategoricalDtype - data_id = 61 - data_shape = (150, 3) - target_shape = (150, 2) - frame_shape = (150, 5) - target_column = ['petalwidth', 'petallength'] + def convert_numerical_dtypes(series): + pandas_series = data_pandas[series.name] + if pd.api.types.is_numeric_dtype(pandas_series): + return series.astype(pandas_series.dtype) + else: + return series + + data_liac_with_fixed_dtypes = data_liac.apply(convert_numerical_dtypes) + pd.testing.assert_frame_equal(data_liac_with_fixed_dtypes, data_pandas) + + # Let's also check that the .frame attributes also match + frame_liac, frame_pandas = bunch_liac.frame, bunch_pandas.frame + + # Note that the .frame attribute is a superset of the .data attribute: + pd.testing.assert_frame_equal(frame_pandas[bunch_pandas.feature_names], data_pandas) + + # However the remaining columns, typically the target(s), are not necessarily + # dtyped similarly by both parsers due to limitations of the LIAC-ARFF parser. + # Therefore, extra dtype conversions are required for those columns: + + def convert_numerical_and_categorical_dtypes(series): + pandas_series = frame_pandas[series.name] + if pd.api.types.is_numeric_dtype(pandas_series): + return series.astype(pandas_series.dtype) + elif isinstance(pandas_series.dtype, pd.CategoricalDtype): + # Compare categorical features by converting categorical liac uses + # strings to denote the categories, we rename the categories to make + # them comparable to the pandas parser. Fixing this behavior in + # LIAC-ARFF would allow to check the consistency in the future but + # we do not plan to maintain the LIAC-ARFF on the long term. + return series.cat.rename_categories(pandas_series.cat.categories) + else: + return series - cat_dtype = CategoricalDtype(['Iris-setosa', 'Iris-versicolor', - 'Iris-virginica']) - data_dtypes = [np.float64, np.float64] + [cat_dtype] - data_names = ['sepallength', 'sepalwidth', 'class'] - target_dtypes = [np.float64, np.float64] - target_names = ['petalwidth', 'petallength'] + frame_liac_with_fixed_dtypes = frame_liac.apply( + convert_numerical_and_categorical_dtypes + ) + pd.testing.assert_frame_equal(frame_liac_with_fixed_dtypes, frame_pandas) - _monkey_patch_webbased_functions(monkeypatch, data_id, True) - bunch = fetch_openml(data_id=data_id, as_frame=True, cache=False, - target_column=target_column) - data = bunch.data - target = bunch.target - frame = bunch.frame +@pytest.mark.parametrize("parser", ["liac-arff", "pandas"]) +def test_fetch_openml_equivalence_array_dataframe(monkeypatch, parser): + """Check the equivalence of the dataset when using `as_frame=False` and + `as_frame=True`. + """ + pytest.importorskip("pandas") - assert isinstance(data, pd.DataFrame) - assert np.all(data.dtypes == data_dtypes) - assert data.shape == data_shape - assert np.all(data.columns == data_names) - assert np.all(bunch.feature_names == data_names) - assert bunch.target_names == target_names + data_id = 61 + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=True) + bunch_as_frame_true = fetch_openml( + data_id=data_id, + as_frame=True, + cache=False, + parser=parser, + ) - assert isinstance(target, pd.DataFrame) - assert np.all(target.dtypes == target_dtypes) - assert target.shape == target_shape - assert np.all(target.columns == target_names) + bunch_as_frame_false = fetch_openml( + data_id=data_id, + as_frame=False, + cache=False, + parser=parser, + ) - assert isinstance(frame, pd.DataFrame) - assert frame.shape == frame_shape - assert np.all(frame.dtypes == [np.float64] * 4 + [cat_dtype]) + assert_allclose(bunch_as_frame_false.data, bunch_as_frame_true.data) + assert_array_equal(bunch_as_frame_false.target, bunch_as_frame_true.target) -# Known failure of PyPy for OpenML. See the following issue: -# https://github.com/scikit-learn/scikit-learn/issues/18906 -@fails_if_pypy -def test_fetch_openml_anneal_pandas(monkeypatch): - # classification dataset with numeric and categorical columns - pd = pytest.importorskip('pandas') +@pytest.mark.parametrize("parser", ["liac-arff", "pandas"]) +def test_fetch_openml_iris_pandas(monkeypatch, parser): + """Check fetching on a numerical only dataset with string labels.""" + pd = pytest.importorskip("pandas") CategoricalDtype = pd.api.types.CategoricalDtype + data_id = 61 + data_shape = (150, 4) + target_shape = (150,) + frame_shape = (150, 5) - data_id = 2 - target_column = 'class' - data_shape = (11, 38) - target_shape = (11,) - frame_shape = (11, 39) - expected_data_categories = 32 - expected_data_floats = 6 + target_dtype = CategoricalDtype( + ["Iris-setosa", "Iris-versicolor", "Iris-virginica"] + ) + data_dtypes = [np.float64] * 4 + data_names = ["sepallength", "sepalwidth", "petallength", "petalwidth"] + target_name = "class" _monkey_patch_webbased_functions(monkeypatch, data_id, True) - bunch = fetch_openml(data_id=data_id, as_frame=True, - target_column=target_column, cache=False) - data = bunch.data - target = bunch.target - frame = bunch.frame - - assert isinstance(data, pd.DataFrame) - assert data.shape == data_shape - n_categories = len([dtype for dtype in data.dtypes - if isinstance(dtype, CategoricalDtype)]) - n_floats = len([dtype for dtype in data.dtypes if dtype.kind == 'f']) - assert expected_data_categories == n_categories - assert expected_data_floats == n_floats - - assert isinstance(target, pd.Series) - assert target.shape == target_shape - assert isinstance(target.dtype, CategoricalDtype) - - assert isinstance(frame, pd.DataFrame) - assert frame.shape == frame_shape - - -# Known failure of PyPy for OpenML. See the following issue: -# https://github.com/scikit-learn/scikit-learn/issues/18906 -@fails_if_pypy -def test_fetch_openml_cpu_pandas(monkeypatch): - # regression dataset with numeric and categorical columns - pd = pytest.importorskip('pandas') - CategoricalDtype = pd.api.types.CategoricalDtype - data_id = 561 - data_shape = (209, 7) - target_shape = (209, ) - frame_shape = (209, 8) - - cat_dtype = CategoricalDtype(['adviser', 'amdahl', 'apollo', 'basf', - 'bti', 'burroughs', 'c.r.d', 'cdc', - 'cambex', 'dec', 'dg', 'formation', - 'four-phase', 'gould', 'hp', 'harris', - 'honeywell', 'ibm', 'ipl', 'magnuson', - 'microdata', 'nas', 'ncr', 'nixdorf', - 'perkin-elmer', 'prime', 'siemens', - 'sperry', 'sratus', 'wang']) - data_dtypes = [cat_dtype] + [np.float64] * 6 - feature_names = ['vendor', 'MYCT', 'MMIN', 'MMAX', 'CACH', - 'CHMIN', 'CHMAX'] - target_name = 'class' - - _monkey_patch_webbased_functions(monkeypatch, data_id, True) - bunch = fetch_openml(data_id=data_id, as_frame=True, cache=False) + bunch = fetch_openml( + data_id=data_id, + as_frame=True, + cache=False, + parser=parser, + ) data = bunch.data target = bunch.target frame = bunch.frame assert isinstance(data, pd.DataFrame) - assert data.shape == data_shape assert np.all(data.dtypes == data_dtypes) - assert np.all(data.columns == feature_names) - assert np.all(bunch.feature_names == feature_names) + assert data.shape == data_shape + assert np.all(data.columns == data_names) + assert np.all(bunch.feature_names == data_names) assert bunch.target_names == [target_name] assert isinstance(target, pd.Series) + assert target.dtype == target_dtype assert target.shape == target_shape - assert target.dtype == np.float64 assert target.name == target_name + assert target.index.is_unique assert isinstance(frame, pd.DataFrame) assert frame.shape == frame_shape + assert np.all(frame.dtypes == data_dtypes + [target_dtype]) + assert frame.index.is_unique -def test_fetch_openml_australian_pandas_error_sparse(monkeypatch): - data_id = 292 +@pytest.mark.parametrize("parser", ["liac-arff", "pandas"]) +@pytest.mark.parametrize("target_column", ["petalwidth", ["petalwidth", "petallength"]]) +def test_fetch_openml_forcing_targets(monkeypatch, parser, target_column): + """Check that we can force the target to not be the default target.""" + pd = pytest.importorskip("pandas") + data_id = 61 _monkey_patch_webbased_functions(monkeypatch, data_id, True) + bunch_forcing_target = fetch_openml( + data_id=data_id, + as_frame=True, + cache=False, + target_column=target_column, + parser=parser, + ) + bunch_default = fetch_openml( + data_id=data_id, + as_frame=True, + cache=False, + parser=parser, + ) - msg = 'Cannot return dataframe with sparse data' - with pytest.raises(ValueError, match=msg): - fetch_openml(data_id=data_id, as_frame=True, cache=False) - - -# Known failure of PyPy for OpenML. See the following issue: -# https://github.com/scikit-learn/scikit-learn/issues/18906 -@fails_if_pypy -def test_fetch_openml_as_frame_auto(monkeypatch): - pd = pytest.importorskip('pandas') + pd.testing.assert_frame_equal(bunch_forcing_target.frame, bunch_default.frame) + if isinstance(target_column, list): + pd.testing.assert_index_equal( + bunch_forcing_target.target.columns, pd.Index(target_column) + ) + assert bunch_forcing_target.data.shape == (150, 3) + else: + assert bunch_forcing_target.target.name == target_column + assert bunch_forcing_target.data.shape == (150, 4) + + +@pytest.mark.parametrize("data_id", [61, 2, 561, 40589, 1119]) +@pytest.mark.parametrize("parser", ["liac-arff", "pandas"]) +def test_fetch_openml_equivalence_frame_return_X_y(monkeypatch, data_id, parser): + """Check the behaviour of `return_X_y=True` when `as_frame=True`.""" + pd = pytest.importorskip("pandas") + + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=True) + bunch = fetch_openml( + data_id=data_id, + as_frame=True, + cache=False, + return_X_y=False, + parser=parser, + ) + X, y = fetch_openml( + data_id=data_id, + as_frame=True, + cache=False, + return_X_y=True, + parser=parser, + ) - data_id = 61 # iris dataset version 1 - _monkey_patch_webbased_functions(monkeypatch, data_id, True) - data = fetch_openml(data_id=data_id, as_frame='auto', cache=False) - assert isinstance(data.data, pd.DataFrame) + pd.testing.assert_frame_equal(bunch.data, X) + if isinstance(y, pd.Series): + pd.testing.assert_series_equal(bunch.target, y) + else: + pd.testing.assert_frame_equal(bunch.target, y) + + +@pytest.mark.parametrize("data_id", [61, 561, 40589, 1119]) +@pytest.mark.parametrize("parser", ["liac-arff", "pandas"]) +def test_fetch_openml_equivalence_array_return_X_y(monkeypatch, data_id, parser): + """Check the behaviour of `return_X_y=True` when `as_frame=False`.""" + pytest.importorskip("pandas") + + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=True) + bunch = fetch_openml( + data_id=data_id, + as_frame=False, + cache=False, + return_X_y=False, + parser=parser, + ) + X, y = fetch_openml( + data_id=data_id, + as_frame=False, + cache=False, + return_X_y=True, + parser=parser, + ) - data_id = 292 # Australian dataset version 1 - _monkey_patch_webbased_functions(monkeypatch, data_id, True) - data = fetch_openml(data_id=data_id, as_frame='auto', cache=False) - assert isinstance(data.data, scipy.sparse.csr_matrix) + assert_array_equal(bunch.data, X) + assert_array_equal(bunch.target, y) -# Known failure of PyPy for OpenML. See the following issue: -# https://github.com/scikit-learn/scikit-learn/issues/18906 -@fails_if_pypy -def test_convert_arff_data_dataframe_warning_low_memory_pandas(monkeypatch): - pytest.importorskip('pandas') +def test_fetch_openml_difference_parsers(monkeypatch): + """Check the difference between liac-arff and pandas parser.""" + pytest.importorskip("pandas") data_id = 1119 - _monkey_patch_webbased_functions(monkeypatch, data_id, True) - - msg = 'Could not adhere to working_memory config.' - with pytest.warns(UserWarning, match=msg): - with config_context(working_memory=1e-6): - fetch_openml(data_id=data_id, as_frame=True, cache=False) + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=True) + # When `as_frame=False`, the categories will be ordinally encoded with + # liac-arff parser while this is not the case with pandas parser. + as_frame = False + bunch_liac_arff = fetch_openml( + data_id=data_id, + as_frame=as_frame, + cache=False, + parser="liac-arff", + ) + bunch_pandas = fetch_openml( + data_id=data_id, + as_frame=as_frame, + cache=False, + parser="pandas", + ) + assert bunch_liac_arff.data.dtype.kind == "f" + assert bunch_pandas.data.dtype == "O" + + +############################################################################### +# Test the ARFF parsing on several dataset to check if detect the correct +# types (categories, integers, floats). + + +@pytest.fixture(scope="module") +def datasets_column_names(): + """Returns the columns names for each dataset.""" + return { + 61: ["sepallength", "sepalwidth", "petallength", "petalwidth", "class"], + 2: [ + "family", + "product-type", + "steel", + "carbon", + "hardness", + "temper_rolling", + "condition", + "formability", + "strength", + "non-ageing", + "surface-finish", + "surface-quality", + "enamelability", + "bc", + "bf", + "bt", + "bw%2Fme", + "bl", + "m", + "chrom", + "phos", + "cbond", + "marvi", + "exptl", + "ferro", + "corr", + "blue%2Fbright%2Fvarn%2Fclean", + "lustre", + "jurofm", + "s", + "p", + "shape", + "thick", + "width", + "len", + "oil", + "bore", + "packing", + "class", + ], + 561: ["vendor", "MYCT", "MMIN", "MMAX", "CACH", "CHMIN", "CHMAX", "class"], + 40589: [ + "Mean_Acc1298_Mean_Mem40_Centroid", + "Mean_Acc1298_Mean_Mem40_Rolloff", + "Mean_Acc1298_Mean_Mem40_Flux", + "Mean_Acc1298_Mean_Mem40_MFCC_0", + "Mean_Acc1298_Mean_Mem40_MFCC_1", + "Mean_Acc1298_Mean_Mem40_MFCC_2", + "Mean_Acc1298_Mean_Mem40_MFCC_3", + "Mean_Acc1298_Mean_Mem40_MFCC_4", + "Mean_Acc1298_Mean_Mem40_MFCC_5", + "Mean_Acc1298_Mean_Mem40_MFCC_6", + "Mean_Acc1298_Mean_Mem40_MFCC_7", + "Mean_Acc1298_Mean_Mem40_MFCC_8", + "Mean_Acc1298_Mean_Mem40_MFCC_9", + "Mean_Acc1298_Mean_Mem40_MFCC_10", + "Mean_Acc1298_Mean_Mem40_MFCC_11", + "Mean_Acc1298_Mean_Mem40_MFCC_12", + "Mean_Acc1298_Std_Mem40_Centroid", + "Mean_Acc1298_Std_Mem40_Rolloff", + "Mean_Acc1298_Std_Mem40_Flux", + "Mean_Acc1298_Std_Mem40_MFCC_0", + "Mean_Acc1298_Std_Mem40_MFCC_1", + "Mean_Acc1298_Std_Mem40_MFCC_2", + "Mean_Acc1298_Std_Mem40_MFCC_3", + "Mean_Acc1298_Std_Mem40_MFCC_4", + "Mean_Acc1298_Std_Mem40_MFCC_5", + "Mean_Acc1298_Std_Mem40_MFCC_6", + "Mean_Acc1298_Std_Mem40_MFCC_7", + "Mean_Acc1298_Std_Mem40_MFCC_8", + "Mean_Acc1298_Std_Mem40_MFCC_9", + "Mean_Acc1298_Std_Mem40_MFCC_10", + "Mean_Acc1298_Std_Mem40_MFCC_11", + "Mean_Acc1298_Std_Mem40_MFCC_12", + "Std_Acc1298_Mean_Mem40_Centroid", + "Std_Acc1298_Mean_Mem40_Rolloff", + "Std_Acc1298_Mean_Mem40_Flux", + "Std_Acc1298_Mean_Mem40_MFCC_0", + "Std_Acc1298_Mean_Mem40_MFCC_1", + "Std_Acc1298_Mean_Mem40_MFCC_2", + "Std_Acc1298_Mean_Mem40_MFCC_3", + "Std_Acc1298_Mean_Mem40_MFCC_4", + "Std_Acc1298_Mean_Mem40_MFCC_5", + "Std_Acc1298_Mean_Mem40_MFCC_6", + "Std_Acc1298_Mean_Mem40_MFCC_7", + "Std_Acc1298_Mean_Mem40_MFCC_8", + "Std_Acc1298_Mean_Mem40_MFCC_9", + "Std_Acc1298_Mean_Mem40_MFCC_10", + "Std_Acc1298_Mean_Mem40_MFCC_11", + "Std_Acc1298_Mean_Mem40_MFCC_12", + "Std_Acc1298_Std_Mem40_Centroid", + "Std_Acc1298_Std_Mem40_Rolloff", + "Std_Acc1298_Std_Mem40_Flux", + "Std_Acc1298_Std_Mem40_MFCC_0", + "Std_Acc1298_Std_Mem40_MFCC_1", + "Std_Acc1298_Std_Mem40_MFCC_2", + "Std_Acc1298_Std_Mem40_MFCC_3", + "Std_Acc1298_Std_Mem40_MFCC_4", + "Std_Acc1298_Std_Mem40_MFCC_5", + "Std_Acc1298_Std_Mem40_MFCC_6", + "Std_Acc1298_Std_Mem40_MFCC_7", + "Std_Acc1298_Std_Mem40_MFCC_8", + "Std_Acc1298_Std_Mem40_MFCC_9", + "Std_Acc1298_Std_Mem40_MFCC_10", + "Std_Acc1298_Std_Mem40_MFCC_11", + "Std_Acc1298_Std_Mem40_MFCC_12", + "BH_LowPeakAmp", + "BH_LowPeakBPM", + "BH_HighPeakAmp", + "BH_HighPeakBPM", + "BH_HighLowRatio", + "BHSUM1", + "BHSUM2", + "BHSUM3", + "amazed.suprised", + "happy.pleased", + "relaxing.calm", + "quiet.still", + "sad.lonely", + "angry.aggresive", + ], + 1119: [ + "age", + "workclass", + "fnlwgt:", + "education:", + "education-num:", + "marital-status:", + "occupation:", + "relationship:", + "race:", + "sex:", + "capital-gain:", + "capital-loss:", + "hours-per-week:", + "native-country:", + "class", + ], + 40966: [ + "DYRK1A_N", + "ITSN1_N", + "BDNF_N", + "NR1_N", + "NR2A_N", + "pAKT_N", + "pBRAF_N", + "pCAMKII_N", + "pCREB_N", + "pELK_N", + "pERK_N", + "pJNK_N", + "PKCA_N", + "pMEK_N", + "pNR1_N", + "pNR2A_N", + "pNR2B_N", + "pPKCAB_N", + "pRSK_N", + "AKT_N", + "BRAF_N", + "CAMKII_N", + "CREB_N", + "ELK_N", + "ERK_N", + "GSK3B_N", + "JNK_N", + "MEK_N", + "TRKA_N", + "RSK_N", + "APP_N", + "Bcatenin_N", + "SOD1_N", + "MTOR_N", + "P38_N", + "pMTOR_N", + "DSCR1_N", + "AMPKA_N", + "NR2B_N", + "pNUMB_N", + "RAPTOR_N", + "TIAM1_N", + "pP70S6_N", + "NUMB_N", + "P70S6_N", + "pGSK3B_N", + "pPKCG_N", + "CDK5_N", + "S6_N", + "ADARB1_N", + "AcetylH3K9_N", + "RRP1_N", + "BAX_N", + "ARC_N", + "ERBB4_N", + "nNOS_N", + "Tau_N", + "GFAP_N", + "GluR3_N", + "GluR4_N", + "IL1B_N", + "P3525_N", + "pCASP9_N", + "PSD95_N", + "SNCA_N", + "Ubiquitin_N", + "pGSK3B_Tyr216_N", + "SHH_N", + "BAD_N", + "BCL2_N", + "pS6_N", + "pCFOS_N", + "SYP_N", + "H3AcK18_N", + "EGR1_N", + "H3MeK4_N", + "CaNA_N", + "class", + ], + 40945: [ + "pclass", + "survived", + "name", + "sex", + "age", + "sibsp", + "parch", + "ticket", + "fare", + "cabin", + "embarked", + "boat", + "body", + "home.dest", + ], + } -# Known failure of PyPy for OpenML. See the following issue: -# https://github.com/scikit-learn/scikit-learn/issues/18906 -@fails_if_pypy -def test_fetch_openml_adultcensus_pandas_return_X_y(monkeypatch): - pd = pytest.importorskip('pandas') - CategoricalDtype = pd.api.types.CategoricalDtype - data_id = 1119 - data_shape = (10, 14) - target_shape = (10, ) +@pytest.fixture(scope="module") +def datasets_missing_values(): + return { + 61: {}, + 2: { + "family": 11, + "temper_rolling": 9, + "condition": 2, + "formability": 4, + "non-ageing": 10, + "surface-finish": 11, + "enamelability": 11, + "bc": 11, + "bf": 10, + "bt": 11, + "bw%2Fme": 8, + "bl": 9, + "m": 11, + "chrom": 11, + "phos": 11, + "cbond": 10, + "marvi": 11, + "exptl": 11, + "ferro": 11, + "corr": 11, + "blue%2Fbright%2Fvarn%2Fclean": 11, + "lustre": 8, + "jurofm": 11, + "s": 11, + "p": 11, + "oil": 10, + "packing": 11, + }, + 561: {}, + 40589: {}, + 1119: {}, + 40966: {"BCL2_N": 7}, + 40945: { + "age": 263, + "fare": 1, + "cabin": 1014, + "embarked": 2, + "boat": 823, + "body": 1188, + "home.dest": 564, + }, + } - expected_data_categories = 8 - expected_data_floats = 6 - target_column = 'class' - _monkey_patch_webbased_functions(monkeypatch, data_id, True) - X, y = fetch_openml(data_id=data_id, as_frame=True, cache=False, - return_X_y=True) - assert isinstance(X, pd.DataFrame) - assert X.shape == data_shape - n_categories = len([dtype for dtype in X.dtypes - if isinstance(dtype, CategoricalDtype)]) - n_floats = len([dtype for dtype in X.dtypes if dtype.kind == 'f']) - assert expected_data_categories == n_categories - assert expected_data_floats == n_floats - - assert isinstance(y, pd.Series) - assert y.shape == target_shape - assert y.name == target_column - - -# Known failure of PyPy for OpenML. See the following issue: -# https://github.com/scikit-learn/scikit-learn/issues/18906 -@fails_if_pypy -def test_fetch_openml_adultcensus_pandas(monkeypatch): - pd = pytest.importorskip('pandas') +@pytest.mark.parametrize( + "data_id, parser, expected_n_categories, expected_n_floats, expected_n_ints", + [ + # iris dataset + (61, "liac-arff", 1, 4, 0), + (61, "pandas", 1, 4, 0), + # anneal dataset + (2, "liac-arff", 33, 6, 0), + (2, "pandas", 33, 2, 4), + # cpu dataset + (561, "liac-arff", 1, 7, 0), + (561, "pandas", 1, 0, 7), + # emotions dataset + (40589, "liac-arff", 6, 72, 0), + (40589, "pandas", 6, 69, 3), + # adult-census dataset + (1119, "liac-arff", 9, 6, 0), + (1119, "pandas", 9, 0, 6), + # miceprotein + (40966, "liac-arff", 1, 77, 0), + (40966, "pandas", 1, 77, 0), + # titanic + (40945, "liac-arff", 3, 6, 0), + (40945, "pandas", 3, 3, 3), + ], +) +@pytest.mark.parametrize("gzip_response", [True, False]) +def test_fetch_openml_types_inference( + monkeypatch, + data_id, + parser, + expected_n_categories, + expected_n_floats, + expected_n_ints, + gzip_response, + datasets_column_names, + datasets_missing_values, +): + """Check that `fetch_openml` infer the right number of categories, integers, and + floats.""" + pd = pytest.importorskip("pandas") CategoricalDtype = pd.api.types.CategoricalDtype - # Check because of the numeric row attribute (issue #12329) - data_id = 1119 - data_shape = (10, 14) - target_shape = (10, ) - frame_shape = (10, 15) - - expected_data_categories = 8 - expected_data_floats = 6 - target_column = 'class' + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=gzip_response) - _monkey_patch_webbased_functions(monkeypatch, data_id, True) - bunch = fetch_openml(data_id=data_id, as_frame=True, cache=False) - data = bunch.data - target = bunch.target + bunch = fetch_openml( + data_id=data_id, + as_frame=True, + cache=False, + parser=parser, + ) frame = bunch.frame - assert isinstance(data, pd.DataFrame) - assert data.shape == data_shape - n_categories = len([dtype for dtype in data.dtypes - if isinstance(dtype, CategoricalDtype)]) - n_floats = len([dtype for dtype in data.dtypes if dtype.kind == 'f']) - assert expected_data_categories == n_categories - assert expected_data_floats == n_floats - - assert isinstance(target, pd.Series) - assert target.shape == target_shape - assert target.name == target_column - - assert isinstance(frame, pd.DataFrame) - assert frame.shape == frame_shape - - -# Known failure of PyPy for OpenML. See the following issue: -# https://github.com/scikit-learn/scikit-learn/issues/18906 -@fails_if_pypy -def test_fetch_openml_miceprotein_pandas(monkeypatch): - # JvR: very important check, as this dataset defined several row ids - # and ignore attributes. Note that data_features json has 82 attributes, - # and row id (1), ignore attributes (3) have been removed. - pd = pytest.importorskip('pandas') - CategoricalDtype = pd.api.types.CategoricalDtype - - data_id = 40966 - data_shape = (7, 77) - target_shape = (7, ) - frame_shape = (7, 78) - - target_column = 'class' - frame_n_categories = 1 - frame_n_floats = 77 - + n_categories = len( + [dtype for dtype in frame.dtypes if isinstance(dtype, CategoricalDtype)] + ) + n_floats = len([dtype for dtype in frame.dtypes if dtype.kind == "f"]) + n_ints = len([dtype for dtype in frame.dtypes if dtype.kind == "i"]) + + assert n_categories == expected_n_categories + assert n_floats == expected_n_floats + assert n_ints == expected_n_ints + + assert frame.columns.tolist() == datasets_column_names[data_id] + + frame_feature_to_n_nan = frame.isna().sum().to_dict() + for name, n_missing in frame_feature_to_n_nan.items(): + expected_missing = datasets_missing_values[data_id].get(name, 0) + assert n_missing == expected_missing + + +############################################################################### +# Test some more specific behaviour + + +@pytest.mark.parametrize( + "params, err_msg", + [ + ( + {"parser": "unknown"}, + "The 'parser' parameter of fetch_openml must be a str among", + ), + ( + {"as_frame": "unknown"}, + "The 'as_frame' parameter of fetch_openml must be an instance", + ), + ], +) +def test_fetch_openml_validation_parameter(monkeypatch, params, err_msg): + data_id = 1119 _monkey_patch_webbased_functions(monkeypatch, data_id, True) - bunch = fetch_openml(data_id=data_id, as_frame=True, cache=False) - data = bunch.data - target = bunch.target - frame = bunch.frame - - assert isinstance(data, pd.DataFrame) - assert data.shape == data_shape - assert np.all(data.dtypes == np.float64) - - assert isinstance(target, pd.Series) - assert isinstance(target.dtype, CategoricalDtype) - assert target.shape == target_shape - assert target.name == target_column - - assert isinstance(frame, pd.DataFrame) - assert frame.shape == frame_shape - n_categories = len([dtype for dtype in frame.dtypes - if isinstance(dtype, CategoricalDtype)]) - n_floats = len([dtype for dtype in frame.dtypes if dtype.kind == 'f']) - assert frame_n_categories == n_categories - assert frame_n_floats == n_floats - - -# Known failure of PyPy for OpenML. See the following issue: -# https://github.com/scikit-learn/scikit-learn/issues/18906 -@fails_if_pypy -def test_fetch_openml_emotions_pandas(monkeypatch): - # classification dataset with multiple targets (natively) - pd = pytest.importorskip('pandas') - CategoricalDtype = pd.api.types.CategoricalDtype - - data_id = 40589 - target_column = ['amazed.suprised', 'happy.pleased', 'relaxing.calm', - 'quiet.still', 'sad.lonely', 'angry.aggresive'] - data_shape = (13, 72) - target_shape = (13, 6) - frame_shape = (13, 78) - - expected_frame_categories = 6 - expected_frame_floats = 72 + with pytest.raises(ValueError, match=err_msg): + fetch_openml(data_id=data_id, **params) + + +@pytest.mark.parametrize( + "params", + [ + {"as_frame": True, "parser": "auto"}, + {"as_frame": "auto", "parser": "auto"}, + {"as_frame": False, "parser": "pandas"}, + {"as_frame": False, "parser": "auto"}, + ], +) +def test_fetch_openml_requires_pandas_error(monkeypatch, params): + """Check that we raise the proper errors when we require pandas.""" + data_id = 1119 + try: + check_pandas_support("test_fetch_openml_requires_pandas") + except ImportError: + _monkey_patch_webbased_functions(monkeypatch, data_id, True) + err_msg = "requires pandas to be installed. Alternatively, explicitly" + with pytest.raises(ImportError, match=err_msg): + fetch_openml(data_id=data_id, **params) + else: + raise SkipTest("This test requires pandas to not be installed.") + + +@pytest.mark.filterwarnings("ignore:Version 1 of dataset Australian is inactive") +@pytest.mark.parametrize( + "params, err_msg", + [ + ( + {"parser": "pandas"}, + "Sparse ARFF datasets cannot be loaded with parser='pandas'", + ), + ( + {"as_frame": True}, + "Sparse ARFF datasets cannot be loaded with as_frame=True.", + ), + ( + {"parser": "pandas", "as_frame": True}, + "Sparse ARFF datasets cannot be loaded with as_frame=True.", + ), + ], +) +def test_fetch_openml_sparse_arff_error(monkeypatch, params, err_msg): + """Check that we raise the expected error for sparse ARFF datasets and + a wrong set of incompatible parameters. + """ + pytest.importorskip("pandas") + data_id = 292 _monkey_patch_webbased_functions(monkeypatch, data_id, True) - bunch = fetch_openml(data_id=data_id, as_frame=True, cache=False, - target_column=target_column) - data = bunch.data - target = bunch.target - frame = bunch.frame - - assert isinstance(data, pd.DataFrame) - assert data.shape == data_shape - - assert isinstance(target, pd.DataFrame) - assert target.shape == target_shape - assert np.all(target.columns == target_column) - - assert isinstance(frame, pd.DataFrame) - assert frame.shape == frame_shape - n_categories = len([dtype for dtype in frame.dtypes - if isinstance(dtype, CategoricalDtype)]) - n_floats = len([dtype for dtype in frame.dtypes if dtype.kind == 'f']) - assert expected_frame_categories == n_categories - assert expected_frame_floats == n_floats - - -# Known failure of PyPy for OpenML. See the following issue: -# https://github.com/scikit-learn/scikit-learn/issues/18906 -@fails_if_pypy -def test_fetch_openml_titanic_pandas(monkeypatch): - # dataset with strings - pd = pytest.importorskip('pandas') - CategoricalDtype = pd.api.types.CategoricalDtype - - data_id = 40945 - data_shape = (1309, 13) - target_shape = (1309, ) - frame_shape = (1309, 14) - name_to_dtype = { - 'pclass': np.float64, - 'name': object, - 'sex': CategoricalDtype(['female', 'male']), - 'age': np.float64, - 'sibsp': np.float64, - 'parch': np.float64, - 'ticket': object, - 'fare': np.float64, - 'cabin': object, - 'embarked': CategoricalDtype(['C', 'Q', 'S']), - 'boat': object, - 'body': np.float64, - 'home.dest': object, - 'survived': CategoricalDtype(['0', '1']) - } - - frame_columns = ['pclass', 'survived', 'name', 'sex', 'age', 'sibsp', - 'parch', 'ticket', 'fare', 'cabin', 'embarked', - 'boat', 'body', 'home.dest'] - frame_dtypes = [name_to_dtype[col] for col in frame_columns] - feature_names = ['pclass', 'name', 'sex', 'age', 'sibsp', - 'parch', 'ticket', 'fare', 'cabin', 'embarked', - 'boat', 'body', 'home.dest'] - target_name = 'survived' + with pytest.raises(ValueError, match=err_msg): + fetch_openml( + data_id=data_id, + cache=False, + **params, + ) + + +@pytest.mark.filterwarnings("ignore:Version 1 of dataset Australian is inactive") +@pytest.mark.parametrize( + "data_id, data_type", + [ + (61, "dataframe"), # iris dataset version 1 + (292, "sparse"), # Australian dataset version 1 + ], +) +def test_fetch_openml_auto_mode(monkeypatch, data_id, data_type): + """Check the auto mode of `fetch_openml`.""" + pd = pytest.importorskip("pandas") _monkey_patch_webbased_functions(monkeypatch, data_id, True) - bunch = fetch_openml(data_id=data_id, as_frame=True, cache=False) - data = bunch.data - target = bunch.target - frame = bunch.frame - - assert isinstance(data, pd.DataFrame) - assert data.shape == data_shape - assert np.all(data.columns == feature_names) - assert bunch.target_names == [target_name] + data = fetch_openml(data_id=data_id, as_frame="auto", cache=False) + klass = pd.DataFrame if data_type == "dataframe" else scipy.sparse.csr_matrix + assert isinstance(data.data, klass) - assert isinstance(target, pd.Series) - assert target.shape == target_shape - assert target.name == target_name - assert target.dtype == name_to_dtype[target_name] - assert isinstance(frame, pd.DataFrame) - assert frame.shape == frame_shape - assert np.all(frame.dtypes == frame_dtypes) +def test_convert_arff_data_dataframe_warning_low_memory_pandas(monkeypatch): + """Check that we raise a warning regarding the working memory when using + LIAC-ARFF parser.""" + pytest.importorskip("pandas") + data_id = 1119 + _monkey_patch_webbased_functions(monkeypatch, data_id, True) -@pytest.mark.parametrize('gzip_response', [True, False]) -def test_fetch_openml_iris(monkeypatch, gzip_response): - # classification dataset with numeric only columns + msg = "Could not adhere to working_memory config." + with pytest.warns(UserWarning, match=msg): + with config_context(working_memory=1e-6): + fetch_openml( + data_id=data_id, + as_frame=True, + cache=False, + parser="liac-arff", + ) + + +@pytest.mark.parametrize("gzip_response", [True, False]) +def test_fetch_openml_iris_warn_multiple_version(monkeypatch, gzip_response): + """Check that a warning is raised when multiple versions exist and no version is + requested.""" data_id = 61 - data_name = 'iris' + data_name = "iris" _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) - msg = ("Multiple active versions of the dataset matching the name" - " iris exist. Versions may be fundamentally different, " - "returning version 1.") + msg = re.escape( + "Multiple active versions of the dataset matching the name" + " iris exist. Versions may be fundamentally different, " + "returning version 1. Available versions:\n" + "- version 1, status: active\n" + " url: https://www.openml.org/search?type=data&id=61\n" + "- version 3, status: active\n" + " url: https://www.openml.org/search?type=data&id=969\n" + ) with pytest.warns(UserWarning, match=msg): - fetch_openml(name=data_name, as_frame=False, cache=False) + fetch_openml( + name=data_name, + as_frame=False, + cache=False, + parser="liac-arff", + ) -def test_decode_iris(monkeypatch): +@pytest.mark.parametrize("gzip_response", [True, False]) +def test_fetch_openml_no_target(monkeypatch, gzip_response): + """Check that we can get a dataset without target.""" data_id = 61 - _monkey_patch_webbased_functions(monkeypatch, data_id, False) - _test_features_list(data_id) - - -@pytest.mark.parametrize('gzip_response', [True, False]) -def test_fetch_openml_iris_multitarget(monkeypatch, gzip_response): - # classification dataset with numeric only columns - data_id = 61 - data_name = 'iris' - data_version = 1 - target_column = ['sepallength', 'sepalwidth'] + target_column = None expected_observations = 150 - expected_features = 3 - expected_missing = 0 - - _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) - _fetch_dataset_from_openml(data_id, data_name, data_version, target_column, - expected_observations, expected_features, - expected_missing, - np.float64, np.float64, expect_sparse=False, - compare_default_target=False) - + expected_features = 5 -@pytest.mark.parametrize('gzip_response', [True, False]) -def test_fetch_openml_anneal(monkeypatch, gzip_response): - # classification dataset with numeric and categorical columns - data_id = 2 - data_name = 'anneal' - data_version = 1 - target_column = 'class' - # Not all original instances included for space reasons - expected_observations = 11 - expected_features = 38 - expected_missing = 267 _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) - _fetch_dataset_from_openml(data_id, data_name, data_version, target_column, - expected_observations, expected_features, - expected_missing, - np.float64, object, expect_sparse=False, - compare_default_target=True) + data = fetch_openml( + data_id=data_id, + target_column=target_column, + cache=False, + as_frame=False, + parser="liac-arff", + ) + assert data.data.shape == (expected_observations, expected_features) + assert data.target is None -def test_decode_anneal(monkeypatch): - data_id = 2 - _monkey_patch_webbased_functions(monkeypatch, data_id, False) - _test_features_list(data_id) +@pytest.mark.parametrize("gzip_response", [True, False]) +@pytest.mark.parametrize("parser", ["liac-arff", "pandas"]) +def test_missing_values_pandas(monkeypatch, gzip_response, parser): + """check that missing values in categories are compatible with pandas + categorical""" + pytest.importorskip("pandas") + data_id = 42585 + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response=gzip_response) + penguins = fetch_openml( + data_id=data_id, + cache=False, + as_frame=True, + parser=parser, + ) -@pytest.mark.parametrize('gzip_response', [True, False]) -def test_fetch_openml_anneal_multitarget(monkeypatch, gzip_response): - # classification dataset with numeric and categorical columns - data_id = 2 - data_name = 'anneal' - data_version = 1 - target_column = ['class', 'product-type', 'shape'] - # Not all original instances included for space reasons - expected_observations = 11 - expected_features = 36 - expected_missing = 267 + cat_dtype = penguins.data.dtypes["sex"] + # there are nans in the categorical + assert penguins.data["sex"].isna().any() + assert_array_equal(cat_dtype.categories, ["FEMALE", "MALE", "_"]) + + +@pytest.mark.parametrize("gzip_response", [True, False]) +@pytest.mark.parametrize( + "dataset_params", + [ + {"data_id": 40675}, + {"data_id": None, "name": "glass2", "version": 1}, + ], +) +def test_fetch_openml_inactive(monkeypatch, gzip_response, dataset_params): + """Check that we raise a warning when the dataset is inactive.""" + data_id = 40675 + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) + msg = "Version 1 of dataset glass2 is inactive," + with pytest.warns(UserWarning, match=msg): + glass2 = fetch_openml( + cache=False, as_frame=False, parser="liac-arff", **dataset_params + ) + assert glass2.data.shape == (163, 9) + assert glass2.details["id"] == "40675" + + +@pytest.mark.parametrize("gzip_response", [True, False]) +@pytest.mark.parametrize( + "data_id, params, err_type, err_msg", + [ + (40675, {"name": "glass2"}, ValueError, "No active dataset glass2 found"), + ( + 61, + {"data_id": 61, "target_column": ["sepalwidth", "class"]}, + ValueError, + "Can only handle homogeneous multi-target datasets", + ), + ( + 40945, + {"data_id": 40945, "as_frame": False}, + ValueError, + ( + "STRING attributes are not supported for array representation. Try" + " as_frame=True" + ), + ), + ( + 2, + {"data_id": 2, "target_column": "family", "as_frame": True}, + ValueError, + "Target column 'family'", + ), + ( + 2, + {"data_id": 2, "target_column": "family", "as_frame": False}, + ValueError, + "Target column 'family'", + ), + ( + 61, + {"data_id": 61, "target_column": "undefined"}, + KeyError, + "Could not find target_column='undefined'", + ), + ( + 61, + {"data_id": 61, "target_column": ["undefined", "class"]}, + KeyError, + "Could not find target_column='undefined'", + ), + ], +) +@pytest.mark.parametrize("parser", ["liac-arff", "pandas"]) +def test_fetch_openml_error( + monkeypatch, gzip_response, data_id, params, err_type, err_msg, parser +): _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) - _fetch_dataset_from_openml(data_id, data_name, data_version, target_column, - expected_observations, expected_features, - expected_missing, - np.float64, object, expect_sparse=False, - compare_default_target=False) - - -@pytest.mark.parametrize('gzip_response', [True, False]) -def test_fetch_openml_cpu(monkeypatch, gzip_response): - # regression dataset with numeric and categorical columns - data_id = 561 - data_name = 'cpu' - data_version = 1 - target_column = 'class' - expected_observations = 209 - expected_features = 7 - expected_missing = 0 + if params.get("as_frame", True) or parser == "pandas": + pytest.importorskip("pandas") + with pytest.raises(err_type, match=err_msg): + fetch_openml(cache=False, parser=parser, **params) + + +@pytest.mark.parametrize( + "params, err_type, err_msg", + [ + ( + {"data_id": -1, "name": None, "version": "version"}, + ValueError, + "The 'version' parameter of fetch_openml must be an int in the range", + ), + ( + {"data_id": -1, "name": "nAmE"}, + ValueError, + "The 'data_id' parameter of fetch_openml must be an int in the range", + ), + ( + {"data_id": -1, "name": "nAmE", "version": "version"}, + ValueError, + "The 'version' parameter of fetch_openml must be an int", + ), + ( + {}, + ValueError, + "Neither name nor data_id are provided. Please provide name or data_id.", + ), + ], +) +def test_fetch_openml_raises_illegal_argument(params, err_type, err_msg): + with pytest.raises(err_type, match=err_msg): + fetch_openml(**params) + + +@pytest.mark.parametrize("gzip_response", [True, False]) +def test_warn_ignore_attribute(monkeypatch, gzip_response): + data_id = 40966 + expected_row_id_msg = "target_column='{}' has flag is_row_identifier." + expected_ignore_msg = "target_column='{}' has flag is_ignore." _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) - _fetch_dataset_from_openml(data_id, data_name, data_version, target_column, - expected_observations, expected_features, - expected_missing, - np.float64, np.float64, expect_sparse=False, - compare_default_target=True) + # single column test + target_col = "MouseID" + msg = expected_row_id_msg.format(target_col) + with pytest.warns(UserWarning, match=msg): + fetch_openml( + data_id=data_id, + target_column=target_col, + cache=False, + as_frame=False, + parser="liac-arff", + ) + target_col = "Genotype" + msg = expected_ignore_msg.format(target_col) + with pytest.warns(UserWarning, match=msg): + fetch_openml( + data_id=data_id, + target_column=target_col, + cache=False, + as_frame=False, + parser="liac-arff", + ) + # multi column test + target_col = "MouseID" + msg = expected_row_id_msg.format(target_col) + with pytest.warns(UserWarning, match=msg): + fetch_openml( + data_id=data_id, + target_column=[target_col, "class"], + cache=False, + as_frame=False, + parser="liac-arff", + ) + target_col = "Genotype" + msg = expected_ignore_msg.format(target_col) + with pytest.warns(UserWarning, match=msg): + fetch_openml( + data_id=data_id, + target_column=[target_col, "class"], + cache=False, + as_frame=False, + parser="liac-arff", + ) -def test_decode_cpu(monkeypatch): - data_id = 561 - _monkey_patch_webbased_functions(monkeypatch, data_id, False) - _test_features_list(data_id) +@pytest.mark.parametrize("gzip_response", [True, False]) +def test_dataset_with_openml_error(monkeypatch, gzip_response): + data_id = 1 + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) + msg = "OpenML registered a problem with the dataset. It might be unusable. Error:" + with pytest.warns(UserWarning, match=msg): + fetch_openml(data_id=data_id, cache=False, as_frame=False, parser="liac-arff") -@pytest.mark.parametrize('gzip_response', [True, False]) -def test_fetch_openml_australian(monkeypatch, gzip_response): - # sparse dataset - # Australian is the only sparse dataset that is reasonably small - # as it is inactive, we need to catch the warning. Due to mocking - # framework, it is not deactivated in our tests - data_id = 292 - data_name = 'Australian' - data_version = 1 - target_column = 'Y' - # Not all original instances included for space reasons - expected_observations = 85 - expected_features = 14 - expected_missing = 0 +@pytest.mark.parametrize("gzip_response", [True, False]) +def test_dataset_with_openml_warning(monkeypatch, gzip_response): + data_id = 3 _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) - assert_warns_message( - UserWarning, - "Version 1 of dataset Australian is inactive,", - _fetch_dataset_from_openml, - **{'data_id': data_id, 'data_name': data_name, - 'data_version': data_version, - 'target_column': target_column, - 'expected_observations': expected_observations, - 'expected_features': expected_features, - 'expected_missing': expected_missing, - 'expect_sparse': True, - 'expected_data_dtype': np.float64, - 'expected_target_dtype': object, - 'compare_default_target': False} # numpy specific check - ) + msg = "OpenML raised a warning on the dataset. It might be unusable. Warning:" + with pytest.warns(UserWarning, match=msg): + fetch_openml(data_id=data_id, cache=False, as_frame=False, parser="liac-arff") -@pytest.mark.parametrize('gzip_response', [True, False]) -def test_fetch_openml_adultcensus(monkeypatch, gzip_response): - # Check because of the numeric row attribute (issue #12329) - data_id = 1119 - data_name = 'adult-census' - data_version = 1 - target_column = 'class' - # Not all original instances included for space reasons - expected_observations = 10 - expected_features = 14 - expected_missing = 0 - _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) - _fetch_dataset_from_openml(data_id, data_name, data_version, target_column, - expected_observations, expected_features, - expected_missing, - np.float64, object, expect_sparse=False, - compare_default_target=True) - - -@pytest.mark.parametrize('gzip_response', [True, False]) -def test_fetch_openml_miceprotein(monkeypatch, gzip_response): - # JvR: very important check, as this dataset defined several row ids - # and ignore attributes. Note that data_features json has 82 attributes, - # and row id (1), ignore attributes (3) have been removed (and target is - # stored in data.target) - data_id = 40966 - data_name = 'MiceProtein' - data_version = 4 - target_column = 'class' - # Not all original instances included for space reasons - expected_observations = 7 - expected_features = 77 - expected_missing = 7 - _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) - _fetch_dataset_from_openml(data_id, data_name, data_version, target_column, - expected_observations, expected_features, - expected_missing, - np.float64, object, expect_sparse=False, - compare_default_target=True) - - -@pytest.mark.parametrize('gzip_response', [True, False]) -def test_fetch_openml_emotions(monkeypatch, gzip_response): - # classification dataset with multiple targets (natively) - data_id = 40589 - data_name = 'emotions' - data_version = 3 - target_column = ['amazed.suprised', 'happy.pleased', 'relaxing.calm', - 'quiet.still', 'sad.lonely', 'angry.aggresive'] - expected_observations = 13 - expected_features = 72 - expected_missing = 0 - _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) +def test_fetch_openml_overwrite_default_params_read_csv(monkeypatch): + """Check that we can overwrite the default parameters of `read_csv`.""" + pytest.importorskip("pandas") + data_id = 1590 + _monkey_patch_webbased_functions(monkeypatch, data_id=data_id, gzip_response=False) - _fetch_dataset_from_openml(data_id, data_name, data_version, target_column, - expected_observations, expected_features, - expected_missing, - np.float64, object, expect_sparse=False, - compare_default_target=True) + common_params = { + "data_id": data_id, + "as_frame": True, + "cache": False, + "parser": "pandas", + } + + # By default, the initial spaces are skipped. We checked that setting the parameter + # `skipinitialspace` to False will have an effect. + adult_without_spaces = fetch_openml(**common_params) + adult_with_spaces = fetch_openml( + **common_params, read_csv_kwargs={"skipinitialspace": False} + ) + assert all( + cat.startswith(" ") for cat in adult_with_spaces.frame["class"].cat.categories + ) + assert not any( + cat.startswith(" ") + for cat in adult_without_spaces.frame["class"].cat.categories + ) -def test_decode_emotions(monkeypatch): - data_id = 40589 - _monkey_patch_webbased_functions(monkeypatch, data_id, False) - _test_features_list(data_id) +############################################################################### +# Test cache, retry mechanisms, checksum, etc. -@pytest.mark.parametrize('gzip_response', [True, False]) +@pytest.mark.parametrize("gzip_response", [True, False]) def test_open_openml_url_cache(monkeypatch, gzip_response, tmpdir): data_id = 61 - _monkey_patch_webbased_functions( - monkeypatch, data_id, gzip_response) - openml_path = sklearn.datasets._openml._DATA_FILE.format(data_id) - cache_directory = str(tmpdir.mkdir('scikit_learn_data')) + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) + openml_path = _MONKEY_PATCH_LOCAL_OPENML_PATH.format(data_id) + "/filename.arff" + url = f"https://www.openml.org/{openml_path}" + cache_directory = str(tmpdir.mkdir("scikit_learn_data")) # first fill the cache - response1 = _open_openml_url(openml_path, cache_directory) + response1 = _open_openml_url(url, cache_directory) # assert file exists location = _get_local_path(openml_path, cache_directory) assert os.path.isfile(location) # redownload, to utilize cache - response2 = _open_openml_url(openml_path, cache_directory) + response2 = _open_openml_url(url, cache_directory) assert response1.read() == response2.read() -@pytest.mark.parametrize('gzip_response', [True, False]) -@pytest.mark.parametrize('write_to_disk', [True, False]) -def test_open_openml_url_unlinks_local_path( - monkeypatch, gzip_response, tmpdir, write_to_disk): +@pytest.mark.parametrize("write_to_disk", [True, False]) +def test_open_openml_url_unlinks_local_path(monkeypatch, tmpdir, write_to_disk): data_id = 61 - openml_path = sklearn.datasets._openml._DATA_FILE.format(data_id) - cache_directory = str(tmpdir.mkdir('scikit_learn_data')) + openml_path = _MONKEY_PATCH_LOCAL_OPENML_PATH.format(data_id) + "/filename.arff" + url = f"https://www.openml.org/{openml_path}" + cache_directory = str(tmpdir.mkdir("scikit_learn_data")) location = _get_local_path(openml_path, cache_directory) - def _mock_urlopen(request): + def _mock_urlopen(request, *args, **kwargs): if write_to_disk: with open(location, "w") as f: f.write("") raise ValueError("Invalid request") - monkeypatch.setattr(sklearn.datasets._openml, 'urlopen', _mock_urlopen) + monkeypatch.setattr(sklearn.datasets._openml, "urlopen", _mock_urlopen) with pytest.raises(ValueError, match="Invalid request"): - _open_openml_url(openml_path, cache_directory) + _open_openml_url(url, cache_directory) assert not os.path.exists(location) def test_retry_with_clean_cache(tmpdir): data_id = 61 - openml_path = sklearn.datasets._openml._DATA_FILE.format(data_id) - cache_directory = str(tmpdir.mkdir('scikit_learn_data')) + openml_path = _MONKEY_PATCH_LOCAL_OPENML_PATH.format(data_id) + cache_directory = str(tmpdir.mkdir("scikit_learn_data")) location = _get_local_path(openml_path, cache_directory) os.makedirs(os.path.dirname(location)) - with open(location, 'w') as f: + with open(location, "w") as f: f.write("") @_retry_with_clean_cache(openml_path, cache_directory) @@ -1038,290 +1417,218 @@ def _load_data(): def test_retry_with_clean_cache_http_error(tmpdir): data_id = 61 - openml_path = sklearn.datasets._openml._DATA_FILE.format(data_id) - cache_directory = str(tmpdir.mkdir('scikit_learn_data')) + openml_path = _MONKEY_PATCH_LOCAL_OPENML_PATH.format(data_id) + cache_directory = str(tmpdir.mkdir("scikit_learn_data")) @_retry_with_clean_cache(openml_path, cache_directory) def _load_data(): - raise HTTPError(url=None, code=412, - msg='Simulated mock error', - hdrs=None, fp=None) + raise HTTPError( + url=None, code=412, msg="Simulated mock error", hdrs=None, fp=BytesIO() + ) error_msg = "Simulated mock error" with pytest.raises(HTTPError, match=error_msg): _load_data() -@pytest.mark.parametrize('gzip_response', [True, False]) +@pytest.mark.parametrize("gzip_response", [True, False]) def test_fetch_openml_cache(monkeypatch, gzip_response, tmpdir): - def _mock_urlopen_raise(request): - raise ValueError('This mechanism intends to test correct cache' - 'handling. As such, urlopen should never be ' - 'accessed. URL: %s' % request.get_full_url()) - data_id = 2 - cache_directory = str(tmpdir.mkdir('scikit_learn_data')) - _monkey_patch_webbased_functions( - monkeypatch, data_id, gzip_response) - X_fetched, y_fetched = fetch_openml(data_id=data_id, cache=True, - data_home=cache_directory, - return_X_y=True, as_frame=False) - - monkeypatch.setattr(sklearn.datasets._openml, 'urlopen', - _mock_urlopen_raise) - - X_cached, y_cached = fetch_openml(data_id=data_id, cache=True, - data_home=cache_directory, - return_X_y=True, as_frame=False) - np.testing.assert_array_equal(X_fetched, X_cached) - np.testing.assert_array_equal(y_fetched, y_cached) - + def _mock_urlopen_raise(request, *args, **kwargs): + raise ValueError( + "This mechanism intends to test correct cache" + "handling. As such, urlopen should never be " + "accessed. URL: %s" % request.get_full_url() + ) -@pytest.mark.parametrize('gzip_response', [True, False]) -def test_fetch_openml_notarget(monkeypatch, gzip_response): data_id = 61 - target_column = None - expected_observations = 150 - expected_features = 5 - - _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) - data = fetch_openml(data_id=data_id, target_column=target_column, - cache=False, as_frame=False) - assert data.data.shape == (expected_observations, expected_features) - assert data.target is None - - -@pytest.mark.parametrize('gzip_response', [True, False]) -def test_fetch_openml_inactive(monkeypatch, gzip_response): - # fetch inactive dataset by id - data_id = 40675 - _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) - glas2 = assert_warns_message( - UserWarning, "Version 1 of dataset glass2 is inactive,", fetch_openml, - data_id=data_id, cache=False, as_frame=False) - # fetch inactive dataset by name and version - assert glas2.data.shape == (163, 9) - glas2_by_version = assert_warns_message( - UserWarning, "Version 1 of dataset glass2 is inactive,", fetch_openml, - data_id=None, name="glass2", version=1, cache=False, as_frame=False) - assert int(glas2_by_version.details['id']) == data_id - - -@pytest.mark.parametrize('gzip_response', [True, False]) -def test_fetch_nonexiting(monkeypatch, gzip_response): - # there is no active version of glass2 - data_id = 40675 - _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) - # Note that we only want to search by name (not data id) - assert_raise_message(ValueError, "No active dataset glass2 found", - fetch_openml, name='glass2', cache=False) - - -@pytest.mark.parametrize('gzip_response', [True, False]) -def test_raises_illegal_multitarget(monkeypatch, gzip_response): - data_id = 61 - targets = ['sepalwidth', 'class'] - _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) - # Note that we only want to search by name (not data id) - assert_raise_message(ValueError, - "Can only handle homogeneous multi-target datasets,", - fetch_openml, data_id=data_id, - target_column=targets, cache=False) - - -@pytest.mark.parametrize('gzip_response', [True, False]) -def test_warn_ignore_attribute(monkeypatch, gzip_response): - data_id = 40966 - expected_row_id_msg = "target_column={} has flag is_row_identifier." - expected_ignore_msg = "target_column={} has flag is_ignore." - _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) - # single column test - assert_warns_message(UserWarning, expected_row_id_msg.format('MouseID'), - fetch_openml, data_id=data_id, - target_column='MouseID', - cache=False, as_frame=False) - assert_warns_message(UserWarning, expected_ignore_msg.format('Genotype'), - fetch_openml, data_id=data_id, - target_column='Genotype', - cache=False, as_frame=False) - # multi column test - assert_warns_message(UserWarning, expected_row_id_msg.format('MouseID'), - fetch_openml, data_id=data_id, - target_column=['MouseID', 'class'], - cache=False, as_frame=False) - assert_warns_message(UserWarning, expected_ignore_msg.format('Genotype'), - fetch_openml, data_id=data_id, - target_column=['Genotype', 'class'], - cache=False, as_frame=False) - - -@pytest.mark.parametrize('gzip_response', [True, False]) -def test_string_attribute_without_dataframe(monkeypatch, gzip_response): - data_id = 40945 - _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) - # single column test - assert_raise_message(ValueError, - ('STRING attributes are not supported for ' - 'array representation. Try as_frame=True'), - fetch_openml, data_id=data_id, cache=False, - as_frame=False) - - -@pytest.mark.parametrize('gzip_response', [True, False]) -def test_dataset_with_openml_error(monkeypatch, gzip_response): - data_id = 1 + cache_directory = str(tmpdir.mkdir("scikit_learn_data")) _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) - assert_warns_message( - UserWarning, - "OpenML registered a problem with the dataset. It might be unusable. " - "Error:", - fetch_openml, data_id=data_id, cache=False, as_frame=False + X_fetched, y_fetched = fetch_openml( + data_id=data_id, + cache=True, + data_home=cache_directory, + return_X_y=True, + as_frame=False, + parser="liac-arff", ) + monkeypatch.setattr(sklearn.datasets._openml, "urlopen", _mock_urlopen_raise) -@pytest.mark.parametrize('gzip_response', [True, False]) -def test_dataset_with_openml_warning(monkeypatch, gzip_response): - data_id = 3 - _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) - assert_warns_message( - UserWarning, - "OpenML raised a warning on the dataset. It might be unusable. " - "Warning:", - fetch_openml, data_id=data_id, cache=False, as_frame=False + X_cached, y_cached = fetch_openml( + data_id=data_id, + cache=True, + data_home=cache_directory, + return_X_y=True, + as_frame=False, + parser="liac-arff", ) + np.testing.assert_array_equal(X_fetched, X_cached) + np.testing.assert_array_equal(y_fetched, y_cached) -@pytest.mark.parametrize('gzip_response', [True, False]) -def test_illegal_column(monkeypatch, gzip_response): - data_id = 61 - _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) - assert_raise_message(KeyError, "Could not find target_column=", - fetch_openml, data_id=data_id, - target_column='undefined', cache=False) - - assert_raise_message(KeyError, "Could not find target_column=", - fetch_openml, data_id=data_id, - target_column=['undefined', 'class'], - cache=False) - - -@pytest.mark.parametrize('gzip_response', [True, False]) -def test_fetch_openml_raises_missing_values_target(monkeypatch, gzip_response): - data_id = 2 - _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) - assert_raise_message(ValueError, "Target column ", - fetch_openml, data_id=data_id, target_column='family') - - -def test_fetch_openml_raises_illegal_argument(): - assert_raise_message(ValueError, "Dataset data_id=", - fetch_openml, data_id=-1, name="name") - - assert_raise_message(ValueError, "Dataset data_id=", - fetch_openml, data_id=-1, name=None, - version="version") - - assert_raise_message(ValueError, "Dataset data_id=", - fetch_openml, data_id=-1, name="name", - version="version") - - assert_raise_message(ValueError, "Neither name nor data_id are provided. " - "Please provide name or data_id.", fetch_openml) - - -@pytest.mark.parametrize('gzip_response', [True, False]) -def test_fetch_openml_with_ignored_feature(monkeypatch, gzip_response): - # Regression test for #14340 - # 62 is the ID of the ZOO dataset - data_id = 62 - _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) - - dataset = sklearn.datasets.fetch_openml(data_id=data_id, cache=False, - as_frame=False) - assert dataset is not None - # The dataset has 17 features, including 1 ignored (animal), - # so we assert that we don't have the ignored feature in the final Bunch - assert dataset['data'].shape == (101, 16) - assert 'animal' not in dataset['feature_names'] - - -# Known failure of PyPy for OpenML. See the following issue: -# https://github.com/scikit-learn/scikit-learn/issues/18906 -@fails_if_pypy -@pytest.mark.parametrize('as_frame', [True, False]) -def test_fetch_openml_verify_checksum(monkeypatch, as_frame, cache, tmpdir): - if as_frame: - pytest.importorskip('pandas') +@pytest.mark.parametrize( + "as_frame, parser", + [ + (True, "liac-arff"), + (False, "liac-arff"), + (True, "pandas"), + (False, "pandas"), + ], +) +def test_fetch_openml_verify_checksum(monkeypatch, as_frame, tmpdir, parser): + """Check that the checksum is working as expected.""" + if as_frame or parser == "pandas": + pytest.importorskip("pandas") data_id = 2 _monkey_patch_webbased_functions(monkeypatch, data_id, True) # create a temporary modified arff file - dataset_dir = os.path.join(currdir, 'data', 'openml', str(data_id)) - original_data_path = os.path.join(dataset_dir, - 'data-v1-download-1666876.arff.gz') - corrupt_copy = os.path.join(tmpdir, "test_invalid_checksum.arff") - with gzip.GzipFile(original_data_path, "rb") as orig_gzip, \ - gzip.GzipFile(corrupt_copy, "wb") as modified_gzip: + original_data_module = OPENML_TEST_DATA_MODULE + "." + f"id_{data_id}" + original_data_file_name = "data-v1-dl-1666876.arff.gz" + original_data_path = resources.files(original_data_module) / original_data_file_name + corrupt_copy_path = tmpdir / "test_invalid_checksum.arff" + with original_data_path.open("rb") as orig_file: + orig_gzip = gzip.open(orig_file, "rb") data = bytearray(orig_gzip.read()) - data[len(data)-1] = 37 + data[len(data) - 1] = 37 + + with gzip.GzipFile(corrupt_copy_path, "wb") as modified_gzip: modified_gzip.write(data) # Requests are already mocked by monkey_patch_webbased_functions. - # We want to re-use that mock for all requests except file download, + # We want to reuse that mock for all requests except file download, # hence creating a thin mock over the original mock mocked_openml_url = sklearn.datasets._openml.urlopen - def swap_file_mock(request): + def swap_file_mock(request, *args, **kwargs): url = request.get_full_url() - if url.endswith('data/v1/download/1666876'): - return _MockHTTPResponse(open(corrupt_copy, "rb"), is_gzip=True) + if url.endswith("data/v1/download/1666876/anneal.arff"): + with open(corrupt_copy_path, "rb") as f: + corrupted_data = f.read() + return _MockHTTPResponse(BytesIO(corrupted_data), is_gzip=True) else: return mocked_openml_url(request) - monkeypatch.setattr(sklearn.datasets._openml, 'urlopen', swap_file_mock) + monkeypatch.setattr(sklearn.datasets._openml, "urlopen", swap_file_mock) # validate failed checksum with pytest.raises(ValueError) as exc: - sklearn.datasets.fetch_openml(data_id=data_id, cache=False, - as_frame=as_frame) + sklearn.datasets.fetch_openml( + data_id=data_id, cache=False, as_frame=as_frame, parser=parser + ) # exception message should have file-path assert exc.match("1666876") -def test_convert_arff_data_type(): - pytest.importorskip('pandas') +def test_open_openml_url_retry_on_network_error(monkeypatch): + def _mock_urlopen_network_error(request, *args, **kwargs): + raise HTTPError( + url=None, code=404, msg="Simulated network error", hdrs=None, fp=BytesIO() + ) - arff: ArffContainerType = { - 'data': (el for el in range(2)), - 'description': '', - 'relation': '', - 'attributes': [] - } - msg = r"shape must be provided when arr\['data'\] is a Generator" - with pytest.raises(ValueError, match=msg): - _convert_arff_data(arff, [0], [0], shape=None) - - arff = { - 'data': list(range(2)), - 'description': '', - 'relation': '', - 'attributes': [] - } - msg = r"arff\['data'\] must be a generator when converting to pd.DataFrame" - with pytest.raises(ValueError, match=msg): - _convert_arff_data_dataframe(arff, ['a'], {}) + monkeypatch.setattr( + sklearn.datasets._openml, "urlopen", _mock_urlopen_network_error + ) + invalid_openml_url = "https://api.openml.org/invalid-url" -def test_missing_values_pandas(monkeypatch): - """check that missing values in categories are compatible with pandas - categorical""" - pytest.importorskip('pandas') + with pytest.warns( + UserWarning, + match=re.escape( + "A network error occurred while downloading" + f" {invalid_openml_url}. Retrying..." + ), + ) as record: + with pytest.raises(HTTPError, match="Simulated network error"): + _open_openml_url(invalid_openml_url, None, delay=0) + assert len(record) == 3 + + +############################################################################### +# Non-regressiont tests + + +@pytest.mark.parametrize("gzip_response", [True, False]) +@pytest.mark.parametrize("parser", ("liac-arff", "pandas")) +def test_fetch_openml_with_ignored_feature(monkeypatch, gzip_response, parser): + """Check that we can load the "zoo" dataset. + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/14340 + """ + if parser == "pandas": + pytest.importorskip("pandas") + data_id = 62 + _monkey_patch_webbased_functions(monkeypatch, data_id, gzip_response) - data_id = 42585 - _monkey_patch_webbased_functions(monkeypatch, data_id, True) - penguins = fetch_openml(data_id=data_id, cache=False, as_frame=True) + dataset = sklearn.datasets.fetch_openml( + data_id=data_id, cache=False, as_frame=False, parser=parser + ) + assert dataset is not None + # The dataset has 17 features, including 1 ignored (animal), + # so we assert that we don't have the ignored feature in the final Bunch + assert dataset["data"].shape == (101, 16) + assert "animal" not in dataset["feature_names"] - cat_dtype = penguins.data.dtypes['sex'] - # there are nans in the categorical - assert penguins.data['sex'].isna().any() - assert_array_equal(cat_dtype.categories, ['FEMALE', 'MALE', '_']) + +def test_fetch_openml_strip_quotes(monkeypatch): + """Check that we strip the single quotes when used as a string delimiter. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/23381 + """ + pd = pytest.importorskip("pandas") + data_id = 40966 + _monkey_patch_webbased_functions(monkeypatch, data_id=data_id, gzip_response=False) + + common_params = {"as_frame": True, "cache": False, "data_id": data_id} + mice_pandas = fetch_openml(parser="pandas", **common_params) + mice_liac_arff = fetch_openml(parser="liac-arff", **common_params) + pd.testing.assert_series_equal(mice_pandas.target, mice_liac_arff.target) + assert not mice_pandas.target.str.startswith("'").any() + assert not mice_pandas.target.str.endswith("'").any() + + # similar behaviour should be observed when the column is not the target + mice_pandas = fetch_openml(parser="pandas", target_column="NUMB_N", **common_params) + mice_liac_arff = fetch_openml( + parser="liac-arff", target_column="NUMB_N", **common_params + ) + pd.testing.assert_series_equal( + mice_pandas.frame["class"], mice_liac_arff.frame["class"] + ) + assert not mice_pandas.frame["class"].str.startswith("'").any() + assert not mice_pandas.frame["class"].str.endswith("'").any() + + +def test_fetch_openml_leading_whitespace(monkeypatch): + """Check that we can strip leading whitespace in pandas parser. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/25311 + """ + pd = pytest.importorskip("pandas") + data_id = 1590 + _monkey_patch_webbased_functions(monkeypatch, data_id=data_id, gzip_response=False) + + common_params = {"as_frame": True, "cache": False, "data_id": data_id} + adult_pandas = fetch_openml(parser="pandas", **common_params) + adult_liac_arff = fetch_openml(parser="liac-arff", **common_params) + pd.testing.assert_series_equal( + adult_pandas.frame["class"], adult_liac_arff.frame["class"] + ) + + +def test_fetch_openml_quotechar_escapechar(monkeypatch): + """Check that we can handle escapechar and single/double quotechar. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/25478 + """ + pd = pytest.importorskip("pandas") + data_id = 42074 + _monkey_patch_webbased_functions(monkeypatch, data_id=data_id, gzip_response=False) + + common_params = {"as_frame": True, "cache": False, "data_id": data_id} + adult_pandas = fetch_openml(parser="pandas", **common_params) + adult_liac_arff = fetch_openml(parser="liac-arff", **common_params) + pd.testing.assert_frame_equal(adult_pandas.frame, adult_liac_arff.frame) diff --git a/sklearn/datasets/tests/test_rcv1.py b/sklearn/datasets/tests/test_rcv1.py index 2c21201dce40e..fbb9d67015a30 100644 --- a/sklearn/datasets/tests/test_rcv1.py +++ b/sklearn/datasets/tests/test_rcv1.py @@ -1,16 +1,17 @@ """Test the rcv1 loader, if the data is available, or if specifically requested via environment variable -(e.g. for travis cron job).""" +(e.g. for CI jobs).""" -import scipy.sparse as sp -import numpy as np from functools import partial + +import numpy as np +import scipy.sparse as sp + from sklearn.datasets.tests.test_common import check_return_X_y -from sklearn.utils._testing import assert_almost_equal -from sklearn.utils._testing import assert_array_equal +from sklearn.utils._testing import assert_almost_equal, assert_array_equal -def test_fetch_rcv1(fetch_rcv1_fxt): +def test_fetch_rcv1(fetch_rcv1_fxt, global_random_seed): data1 = fetch_rcv1_fxt(shuffle=False) X1, Y1 = data1.data, data1.target cat_list, s1 = data1.target_names.tolist(), data1.sample_id @@ -27,24 +28,29 @@ def test_fetch_rcv1(fetch_rcv1_fxt): assert (804414,) == s1.shape assert 103 == len(cat_list) + # test descr + assert data1.DESCR.startswith(".. _rcv1_dataset:") + # test ordering of categories - first_categories = ['C11', 'C12', 'C13', 'C14', 'C15', 'C151'] + first_categories = ["C11", "C12", "C13", "C14", "C15", "C151"] assert_array_equal(first_categories, cat_list[:6]) # test number of sample for some categories - some_categories = ('GMIL', 'E143', 'CCAT') + some_categories = ("GMIL", "E143", "CCAT") number_non_zero_in_cat = (5, 1206, 381327) for num, cat in zip(number_non_zero_in_cat, some_categories): j = cat_list.index(cat) assert num == Y1[:, j].data.size # test shuffling and subset - data2 = fetch_rcv1_fxt(shuffle=True, subset='train', random_state=77) + data2 = fetch_rcv1_fxt( + shuffle=True, subset="train", random_state=global_random_seed + ) X2, Y2 = data2.data, data2.target s2 = data2.sample_id # test return_X_y option - fetch_func = partial(fetch_rcv1_fxt, shuffle=False, subset='train') + fetch_func = partial(fetch_rcv1_fxt, shuffle=False, subset="train") check_return_X_y(data2, fetch_func) # The first 23149 samples are the training samples diff --git a/sklearn/datasets/tests/test_samples_generator.py b/sklearn/datasets/tests/test_samples_generator.py index fcdb1222bd116..81e8183c6722e 100644 --- a/sklearn/datasets/tests/test_samples_generator.py +++ b/sklearn/datasets/tests/test_samples_generator.py @@ -1,4 +1,4 @@ - +import re from collections import defaultdict from functools import partial @@ -6,40 +6,53 @@ import pytest import scipy.sparse as sp -from sklearn.utils._testing import assert_array_equal -from sklearn.utils._testing import assert_almost_equal -from sklearn.utils._testing import assert_array_almost_equal -from sklearn.utils._testing import assert_raise_message - -from sklearn.datasets import make_classification -from sklearn.datasets import make_multilabel_classification -from sklearn.datasets import make_hastie_10_2 -from sklearn.datasets import make_regression -from sklearn.datasets import make_blobs -from sklearn.datasets import make_friedman1 -from sklearn.datasets import make_friedman2 -from sklearn.datasets import make_friedman3 -from sklearn.datasets import make_low_rank_matrix -from sklearn.datasets import make_moons -from sklearn.datasets import make_circles -from sklearn.datasets import make_sparse_coded_signal -from sklearn.datasets import make_sparse_uncorrelated -from sklearn.datasets import make_spd_matrix -from sklearn.datasets import make_swiss_roll -from sklearn.datasets import make_s_curve -from sklearn.datasets import make_biclusters -from sklearn.datasets import make_checkerboard - +from sklearn.datasets import ( + make_biclusters, + make_blobs, + make_checkerboard, + make_circles, + make_classification, + make_friedman1, + make_friedman2, + make_friedman3, + make_hastie_10_2, + make_low_rank_matrix, + make_moons, + make_multilabel_classification, + make_regression, + make_s_curve, + make_sparse_coded_signal, + make_sparse_spd_matrix, + make_sparse_uncorrelated, + make_spd_matrix, + make_swiss_roll, +) +from sklearn.utils._testing import ( + assert_allclose, + assert_allclose_dense_sparse, + assert_almost_equal, + assert_array_almost_equal, + assert_array_equal, +) from sklearn.utils.validation import assert_all_finite def test_make_classification(): weights = [0.1, 0.25] - X, y = make_classification(n_samples=100, n_features=20, n_informative=5, - n_redundant=1, n_repeated=1, n_classes=3, - n_clusters_per_class=1, hypercube=False, - shift=None, scale=None, weights=weights, - random_state=0) + X, y = make_classification( + n_samples=100, + n_features=20, + n_informative=5, + n_redundant=1, + n_repeated=1, + n_classes=3, + n_clusters_per_class=1, + hypercube=False, + shift=None, + scale=None, + weights=weights, + random_state=0, + ) assert weights == [0.1, 0.25] assert X.shape == (100, 20), "X shape mismatch" @@ -50,15 +63,26 @@ def test_make_classification(): assert sum(y == 2) == 65, "Unexpected number of samples in class #2" # Test for n_features > 30 - X, y = make_classification(n_samples=2000, n_features=31, n_informative=31, - n_redundant=0, n_repeated=0, hypercube=True, - scale=0.5, random_state=0) + X, y = make_classification( + n_samples=2000, + n_features=31, + n_informative=31, + n_redundant=0, + n_repeated=0, + hypercube=True, + scale=0.5, + random_state=0, + ) assert X.shape == (2000, 31), "X shape mismatch" assert y.shape == (2000,), "y shape mismatch" - assert (np.unique(X.view([('', X.dtype)]*X.shape[1])).view(X.dtype) - .reshape(-1, X.shape[1]).shape[0] == 2000), ( - "Unexpected number of unique rows") + assert ( + np.unique(X.view([("", X.dtype)] * X.shape[1])) + .view(X.dtype) + .reshape(-1, X.shape[1]) + .shape[0] + == 2000 + ), "Unexpected number of unique rows" def test_make_classification_informative_features(): @@ -70,96 +94,172 @@ def test_make_classification_informative_features(): # Create very separate clusters; check that vertices are unique and # correspond to classes class_sep = 1e6 - make = partial(make_classification, class_sep=class_sep, n_redundant=0, - n_repeated=0, flip_y=0, shift=0, scale=1, shuffle=False) - - for n_informative, weights, n_clusters_per_class in [(2, [1], 1), - (2, [1/3] * 3, 1), - (2, [1/4] * 4, 1), - (2, [1/2] * 2, 2), - (2, [3/4, 1/4], 2), - (10, [1/3] * 3, 10), - (int(64), [1], 1) - ]: + make = partial( + make_classification, + class_sep=class_sep, + n_redundant=0, + n_repeated=0, + flip_y=0, + shift=0, + scale=1, + shuffle=False, + ) + + for n_informative, weights, n_clusters_per_class in [ + (2, [1], 1), + (2, [1 / 3] * 3, 1), + (2, [1 / 4] * 4, 1), + (2, [1 / 2] * 2, 2), + (2, [3 / 4, 1 / 4], 2), + (10, [1 / 3] * 3, 10), + (64, [1], 1), + ]: n_classes = len(weights) n_clusters = n_classes * n_clusters_per_class n_samples = n_clusters * 50 for hypercube in (False, True): - X, y = make(n_samples=n_samples, n_classes=n_classes, - weights=weights, n_features=n_informative, - n_informative=n_informative, - n_clusters_per_class=n_clusters_per_class, - hypercube=hypercube, random_state=0) + X, y = make( + n_samples=n_samples, + n_classes=n_classes, + weights=weights, + n_features=n_informative, + n_informative=n_informative, + n_clusters_per_class=n_clusters_per_class, + hypercube=hypercube, + random_state=0, + ) assert X.shape == (n_samples, n_informative) assert y.shape == (n_samples,) # Cluster by sign, viewed as strings to allow uniquing signs = np.sign(X) - signs = signs.view(dtype='|S{0}'.format(signs.strides[0])) - unique_signs, cluster_index = np.unique(signs, - return_inverse=True) + signs = signs.view(dtype="|S{0}".format(signs.strides[0])).ravel() + unique_signs, cluster_index = np.unique(signs, return_inverse=True) assert len(unique_signs) == n_clusters, ( - "Wrong number of clusters, or not in distinct quadrants") + "Wrong number of clusters, or not in distinct quadrants" + ) clusters_by_class = defaultdict(set) for cluster, cls in zip(cluster_index, y): clusters_by_class[cls].add(cluster) for clusters in clusters_by_class.values(): assert len(clusters) == n_clusters_per_class, ( - "Wrong number of clusters per class") - assert (len(clusters_by_class) == n_classes), ( - "Wrong number of classes") + "Wrong number of clusters per class" + ) + assert len(clusters_by_class) == n_classes, "Wrong number of classes" - assert_array_almost_equal(np.bincount(y) / len(y) // weights, - [1] * n_classes, - err_msg="Wrong number of samples " - "per class") + assert_array_almost_equal( + np.bincount(y) / len(y) // weights, + [1] * n_classes, + err_msg="Wrong number of samples per class", + ) # Ensure on vertices of hypercube for cluster in range(len(unique_signs)): centroid = X[cluster_index == cluster].mean(axis=0) if hypercube: - assert_array_almost_equal(np.abs(centroid) / class_sep, - np.ones(n_informative), - decimal=5, - err_msg="Clusters are not " - "centered on hypercube " - "vertices") + assert_array_almost_equal( + np.abs(centroid) / class_sep, + np.ones(n_informative), + decimal=5, + err_msg="Clusters are not centered on hypercube vertices", + ) else: with pytest.raises(AssertionError): - assert_array_almost_equal(np.abs(centroid) / class_sep, - np.ones(n_informative), - decimal=5, - err_msg="Clusters should " - "not be centered " - "on hypercube " - "vertices") + assert_array_almost_equal( + np.abs(centroid) / class_sep, + np.ones(n_informative), + decimal=5, + err_msg=( + "Clusters should not be centered on hypercube vertices" + ), + ) with pytest.raises(ValueError): - make(n_features=2, n_informative=2, n_classes=5, - n_clusters_per_class=1) + make(n_features=2, n_informative=2, n_classes=5, n_clusters_per_class=1) with pytest.raises(ValueError): - make(n_features=2, n_informative=2, n_classes=3, - n_clusters_per_class=2) + make(n_features=2, n_informative=2, n_classes=3, n_clusters_per_class=2) + + +def test_make_classification_return_x_y(): + """ + Test that make_classification returns a Bunch when return_X_y is False. + + Also that bunch.X is the same as X + """ + + kwargs = { + "n_samples": 100, + "n_features": 20, + "n_informative": 5, + "n_redundant": 1, + "n_repeated": 1, + "n_classes": 3, + "n_clusters_per_class": 2, + "weights": None, + "flip_y": 0.01, + "class_sep": 1.0, + "hypercube": True, + "shift": 0.0, + "scale": 1.0, + "shuffle": True, + "random_state": 42, + "return_X_y": True, + } + + X, y = make_classification(**kwargs) + + kwargs["return_X_y"] = False + bunch = make_classification(**kwargs) + + assert ( + hasattr(bunch, "DESCR") + and hasattr(bunch, "parameters") + and hasattr(bunch, "feature_info") + and hasattr(bunch, "X") + and hasattr(bunch, "y") + ) + + def count(str_): + return bunch.feature_info.count(str_) + + assert np.array_equal(X, bunch.X) + assert np.array_equal(y, bunch.y) + assert bunch.DESCR == make_classification.__doc__ + assert bunch.parameters == kwargs + assert count("informative") == kwargs["n_informative"] + assert count("redundant") == kwargs["n_redundant"] + assert count("repeated") == kwargs["n_repeated"] @pytest.mark.parametrize( - 'weights, err_type, err_msg', + "weights, err_type, err_msg", [ - ([], ValueError, - "Weights specified but incompatible with number of classes."), - ([.25, .75, .1], ValueError, - "Weights specified but incompatible with number of classes."), - (np.array([]), ValueError, - "Weights specified but incompatible with number of classes."), - (np.array([.25, .75, .1]), ValueError, - "Weights specified but incompatible with number of classes."), - (np.random.random(3), ValueError, - "Weights specified but incompatible with number of classes.") - ] + ([], ValueError, "Weights specified but incompatible with number of classes."), + ( + [0.25, 0.75, 0.1], + ValueError, + "Weights specified but incompatible with number of classes.", + ), + ( + np.array([]), + ValueError, + "Weights specified but incompatible with number of classes.", + ), + ( + np.array([0.25, 0.75, 0.1]), + ValueError, + "Weights specified but incompatible with number of classes.", + ), + ( + np.random.random(3), + ValueError, + "Weights specified but incompatible with number of classes.", + ), + ], ) def test_make_classification_weights_type(weights, err_type, err_msg): with pytest.raises(err_type, match=err_msg): @@ -168,20 +268,22 @@ def test_make_classification_weights_type(weights, err_type, err_msg): @pytest.mark.parametrize("kwargs", [{}, {"n_classes": 3, "n_informative": 3}]) def test_make_classification_weights_array_or_list_ok(kwargs): - X1, y1 = make_classification(weights=[.1, .9], - random_state=0, **kwargs) - X2, y2 = make_classification(weights=np.array([.1, .9]), - random_state=0, **kwargs) + X1, y1 = make_classification(weights=[0.1, 0.9], random_state=0, **kwargs) + X2, y2 = make_classification(weights=np.array([0.1, 0.9]), random_state=0, **kwargs) assert_almost_equal(X1, X2) assert_almost_equal(y1, y2) def test_make_multilabel_classification_return_sequences(): for allow_unlabeled, min_length in zip((True, False), (0, 1)): - X, Y = make_multilabel_classification(n_samples=100, n_features=20, - n_classes=3, random_state=0, - return_indicator=False, - allow_unlabeled=allow_unlabeled) + X, Y = make_multilabel_classification( + n_samples=100, + n_features=20, + n_classes=3, + random_state=0, + return_indicator=False, + allow_unlabeled=allow_unlabeled, + ) assert X.shape == (100, 20), "X shape mismatch" if not allow_unlabeled: assert max([max(y) for y in Y]) == 2 @@ -191,17 +293,26 @@ def test_make_multilabel_classification_return_sequences(): def test_make_multilabel_classification_return_indicator(): for allow_unlabeled, min_length in zip((True, False), (0, 1)): - X, Y = make_multilabel_classification(n_samples=25, n_features=20, - n_classes=3, random_state=0, - allow_unlabeled=allow_unlabeled) + X, Y = make_multilabel_classification( + n_samples=25, + n_features=20, + n_classes=3, + random_state=0, + allow_unlabeled=allow_unlabeled, + ) assert X.shape == (25, 20), "X shape mismatch" assert Y.shape == (25, 3), "Y shape mismatch" assert np.all(np.sum(Y, axis=0) > min_length) # Also test return_distributions and return_indicator with True X2, Y2, p_c, p_w_c = make_multilabel_classification( - n_samples=25, n_features=20, n_classes=3, random_state=0, - allow_unlabeled=allow_unlabeled, return_distributions=True) + n_samples=25, + n_features=20, + n_classes=3, + random_state=0, + allow_unlabeled=allow_unlabeled, + return_distributions=True, + ) assert_array_almost_equal(X, X2) assert_array_equal(Y, Y2) @@ -213,27 +324,19 @@ def test_make_multilabel_classification_return_indicator(): def test_make_multilabel_classification_return_indicator_sparse(): for allow_unlabeled, min_length in zip((True, False), (0, 1)): - X, Y = make_multilabel_classification(n_samples=25, n_features=20, - n_classes=3, random_state=0, - return_indicator='sparse', - allow_unlabeled=allow_unlabeled) + X, Y = make_multilabel_classification( + n_samples=25, + n_features=20, + n_classes=3, + random_state=0, + return_indicator="sparse", + allow_unlabeled=allow_unlabeled, + ) assert X.shape == (25, 20), "X shape mismatch" assert Y.shape == (25, 3), "Y shape mismatch" assert sp.issparse(Y) -@pytest.mark.parametrize( - "params, err_msg", - [ - ({"n_classes": 0}, "'n_classes' should be an integer"), - ({"length": 0}, "'length' should be an integer") - ] -) -def test_make_multilabel_classification_valid_arguments(params, err_msg): - with pytest.raises(ValueError, match=err_msg): - make_multilabel_classification(**params) - - def test_make_hastie_10_2(): X, y = make_hastie_10_2(n_samples=100, random_state=0) assert X.shape == (100, 10), "X shape mismatch" @@ -241,13 +344,20 @@ def test_make_hastie_10_2(): assert np.unique(y).shape == (2,), "Unexpected number of classes" -def test_make_regression(): - X, y, c = make_regression(n_samples=100, n_features=10, n_informative=3, - effective_rank=5, coef=True, bias=0.0, - noise=1.0, random_state=0) - - assert X.shape == (100, 10), "X shape mismatch" - assert y.shape == (100,), "y shape mismatch" +def test_make_regression(global_random_seed): + X, y, c = make_regression( + n_samples=200, + n_features=10, + n_informative=3, + effective_rank=5, + coef=True, + bias=0.0, + noise=1.0, + random_state=global_random_seed, + ) + + assert X.shape == (200, 10), "X shape mismatch" + assert y.shape == (200,), "y shape mismatch" assert c.shape == (10,), "coef shape mismatch" assert sum(c != 0.0) == 3, "Unexpected number of informative features" @@ -259,25 +369,36 @@ def test_make_regression(): assert X.shape == (100, 1) -def test_make_regression_multitarget(): - X, y, c = make_regression(n_samples=100, n_features=10, n_informative=3, - n_targets=3, coef=True, noise=1., random_state=0) +def test_make_regression_multitarget(global_random_seed): + X, y, c = make_regression( + n_samples=100, + n_features=10, + n_informative=3, + n_targets=3, + coef=True, + noise=1.0, + random_state=global_random_seed, + ) assert X.shape == (100, 10), "X shape mismatch" assert y.shape == (100, 3), "y shape mismatch" assert c.shape == (10, 3), "coef shape mismatch" - assert_array_equal(sum(c != 0.0), 3, - "Unexpected number of informative features") + assert_array_equal(sum(c != 0.0), 3, "Unexpected number of informative features") # Test that y ~= np.dot(X, c) + bias + N(0, 1.0) assert_almost_equal(np.std(y - np.dot(X, c)), 1.0, decimal=1) -def test_make_blobs(): +def test_make_blobs(global_random_seed): cluster_stds = np.array([0.05, 0.2, 0.4]) cluster_centers = np.array([[0.0, 0.0], [1.0, 1.0], [0.0, 1.0]]) - X, y = make_blobs(random_state=0, n_samples=50, n_features=2, - centers=cluster_centers, cluster_std=cluster_stds) + X, y = make_blobs( + random_state=global_random_seed, + n_samples=50, + n_features=2, + centers=cluster_centers, + cluster_std=cluster_stds, + ) assert X.shape == (50, 2), "X shape mismatch" assert y.shape == (50,), "y shape mismatch" @@ -291,44 +412,49 @@ def test_make_blobs_n_samples_list(): X, y = make_blobs(n_samples=n_samples, n_features=2, random_state=0) assert X.shape == (sum(n_samples), 2), "X shape mismatch" - assert all(np.bincount(y, minlength=len(n_samples)) == n_samples), \ + assert all(np.bincount(y, minlength=len(n_samples)) == n_samples), ( "Incorrect number of samples per blob" + ) -def test_make_blobs_n_samples_list_with_centers(): +def test_make_blobs_n_samples_list_with_centers(global_random_seed): n_samples = [20, 20, 20] centers = np.array([[0.0, 0.0], [1.0, 1.0], [0.0, 1.0]]) cluster_stds = np.array([0.05, 0.2, 0.4]) - X, y = make_blobs(n_samples=n_samples, centers=centers, - cluster_std=cluster_stds, random_state=0) + X, y = make_blobs( + n_samples=n_samples, + centers=centers, + cluster_std=cluster_stds, + random_state=global_random_seed, + ) assert X.shape == (sum(n_samples), 2), "X shape mismatch" - assert all(np.bincount(y, minlength=len(n_samples)) == n_samples), \ + assert all(np.bincount(y, minlength=len(n_samples)) == n_samples), ( "Incorrect number of samples per blob" + ) for i, (ctr, std) in enumerate(zip(centers, cluster_stds)): assert_almost_equal((X[y == i] - ctr).std(), std, 1, "Unexpected std") @pytest.mark.parametrize( - "n_samples", - [[5, 3, 0], - np.array([5, 3, 0]), - tuple([5, 3, 0])] + "n_samples", [[5, 3, 0], np.array([5, 3, 0]), tuple([5, 3, 0])] ) def test_make_blobs_n_samples_centers_none(n_samples): centers = None X, y = make_blobs(n_samples=n_samples, centers=centers, random_state=0) assert X.shape == (sum(n_samples), 2), "X shape mismatch" - assert all(np.bincount(y, minlength=len(n_samples)) == n_samples), \ + assert all(np.bincount(y, minlength=len(n_samples)) == n_samples), ( "Incorrect number of samples per blob" + ) def test_make_blobs_return_centers(): n_samples = [10, 20] n_features = 3 - X, y, centers = make_blobs(n_samples=n_samples, n_features=n_features, - return_centers=True, random_state=0) + X, y, centers = make_blobs( + n_samples=n_samples, n_features=n_features, return_centers=True, random_state=0 + ) assert centers.shape == (len(n_samples), n_features) @@ -337,82 +463,96 @@ def test_make_blobs_error(): n_samples = [20, 20, 20] centers = np.array([[0.0, 0.0], [1.0, 1.0], [0.0, 1.0]]) cluster_stds = np.array([0.05, 0.2, 0.4]) - wrong_centers_msg = ("Length of `n_samples` not consistent " - "with number of centers. Got n_samples = {} " - "and centers = {}".format(n_samples, centers[:-1])) - assert_raise_message(ValueError, wrong_centers_msg, - make_blobs, n_samples, centers=centers[:-1]) - wrong_std_msg = ("Length of `clusters_std` not consistent with " - "number of centers. Got centers = {} " - "and cluster_std = {}".format(centers, cluster_stds[:-1])) - assert_raise_message(ValueError, wrong_std_msg, - make_blobs, n_samples, - centers=centers, cluster_std=cluster_stds[:-1]) - wrong_type_msg = ("Parameter `centers` must be array-like. " - "Got {!r} instead".format(3)) - assert_raise_message(ValueError, wrong_type_msg, - make_blobs, n_samples, centers=3) - - -def test_make_friedman1(): - X, y = make_friedman1(n_samples=5, n_features=10, noise=0.0, - random_state=0) + wrong_centers_msg = re.escape( + "Length of `n_samples` not consistent with number of centers. " + f"Got n_samples = {n_samples} and centers = {centers[:-1]}" + ) + with pytest.raises(ValueError, match=wrong_centers_msg): + make_blobs(n_samples, centers=centers[:-1]) + wrong_std_msg = re.escape( + "Length of `clusters_std` not consistent with number of centers. " + f"Got centers = {centers} and cluster_std = {cluster_stds[:-1]}" + ) + with pytest.raises(ValueError, match=wrong_std_msg): + make_blobs(n_samples, centers=centers, cluster_std=cluster_stds[:-1]) + wrong_type_msg = "Parameter `centers` must be array-like. Got {!r} instead".format( + 3 + ) + with pytest.raises(ValueError, match=wrong_type_msg): + make_blobs(n_samples, centers=3) + + +def test_make_friedman1(global_random_seed): + X, y = make_friedman1( + n_samples=5, n_features=10, noise=0.0, random_state=global_random_seed + ) assert X.shape == (5, 10), "X shape mismatch" assert y.shape == (5,), "y shape mismatch" - assert_array_almost_equal(y, - 10 * np.sin(np.pi * X[:, 0] * X[:, 1]) - + 20 * (X[:, 2] - 0.5) ** 2 - + 10 * X[:, 3] + 5 * X[:, 4]) + assert_array_almost_equal( + y, + 10 * np.sin(np.pi * X[:, 0] * X[:, 1]) + + 20 * (X[:, 2] - 0.5) ** 2 + + 10 * X[:, 3] + + 5 * X[:, 4], + ) -def test_make_friedman2(): - X, y = make_friedman2(n_samples=5, noise=0.0, random_state=0) +def test_make_friedman2(global_random_seed): + X, y = make_friedman2(n_samples=5, noise=0.0, random_state=global_random_seed) assert X.shape == (5, 4), "X shape mismatch" assert y.shape == (5,), "y shape mismatch" - assert_array_almost_equal(y, - (X[:, 0] ** 2 - + (X[:, 1] * X[:, 2] - 1 - / (X[:, 1] * X[:, 3])) ** 2) ** 0.5) + assert_array_almost_equal( + y, (X[:, 0] ** 2 + (X[:, 1] * X[:, 2] - 1 / (X[:, 1] * X[:, 3])) ** 2) ** 0.5 + ) -def test_make_friedman3(): - X, y = make_friedman3(n_samples=5, noise=0.0, random_state=0) +def test_make_friedman3(global_random_seed): + X, y = make_friedman3(n_samples=5, noise=0.0, random_state=global_random_seed) assert X.shape == (5, 4), "X shape mismatch" assert y.shape == (5,), "y shape mismatch" - assert_array_almost_equal(y, np.arctan((X[:, 1] * X[:, 2] - - 1 / (X[:, 1] * X[:, 3])) - / X[:, 0])) + assert_array_almost_equal( + y, np.arctan((X[:, 1] * X[:, 2] - 1 / (X[:, 1] * X[:, 3])) / X[:, 0]) + ) def test_make_low_rank_matrix(): - X = make_low_rank_matrix(n_samples=50, n_features=25, effective_rank=5, - tail_strength=0.01, random_state=0) + X = make_low_rank_matrix( + n_samples=50, + n_features=25, + effective_rank=5, + tail_strength=0.01, + random_state=0, + ) assert X.shape == (50, 25), "X shape mismatch" from numpy.linalg import svd + u, s, v = svd(X) assert sum(s) - 5 < 0.1, "X rank is not approximately 5" -def test_make_sparse_coded_signal(): - Y, D, X = make_sparse_coded_signal(n_samples=5, n_components=8, - n_features=10, n_nonzero_coefs=3, - random_state=0) - assert Y.shape == (10, 5), "Y shape mismatch" - assert D.shape == (10, 8), "D shape mismatch" - assert X.shape == (8, 5), "X shape mismatch" - for col in X.T: - assert len(np.flatnonzero(col)) == 3, 'Non-zero coefs mismatch' - assert_array_almost_equal(np.dot(D, X), Y) - assert_array_almost_equal(np.sqrt((D ** 2).sum(axis=0)), - np.ones(D.shape[1])) +def test_make_sparse_coded_signal(global_random_seed): + Y, D, X = make_sparse_coded_signal( + n_samples=5, + n_components=8, + n_features=10, + n_nonzero_coefs=3, + random_state=global_random_seed, + ) + assert Y.shape == (5, 10), "Y shape mismatch" + assert D.shape == (8, 10), "D shape mismatch" + assert X.shape == (5, 8), "X shape mismatch" + for row in X: + assert len(np.flatnonzero(row)) == 3, "Non-zero coefs mismatch" + assert_allclose(Y, X @ D) + assert_allclose(np.sqrt((D**2).sum(axis=1)), np.ones(D.shape[0])) def test_make_sparse_uncorrelated(): @@ -422,29 +562,66 @@ def test_make_sparse_uncorrelated(): assert y.shape == (5,), "y shape mismatch" -def test_make_spd_matrix(): - X = make_spd_matrix(n_dim=5, random_state=0) +def test_make_spd_matrix(global_random_seed): + X = make_spd_matrix(n_dim=5, random_state=global_random_seed) assert X.shape == (5, 5), "X shape mismatch" assert_array_almost_equal(X, X.T) from numpy.linalg import eig + eigenvalues, _ = eig(X) - assert_array_equal(eigenvalues > 0, np.array([True] * 5), - "X is not positive-definite") + assert np.all(eigenvalues > 0), "X is not positive-definite" -def test_make_swiss_roll(): - X, t = make_swiss_roll(n_samples=5, noise=0.0, random_state=0) +@pytest.mark.parametrize("norm_diag", [True, False]) +@pytest.mark.parametrize( + "sparse_format", [None, "bsr", "coo", "csc", "csr", "dia", "dok", "lil"] +) +def test_make_sparse_spd_matrix(norm_diag, sparse_format, global_random_seed): + n_dim = 5 + X = make_sparse_spd_matrix( + n_dim=n_dim, + norm_diag=norm_diag, + sparse_format=sparse_format, + random_state=global_random_seed, + ) + + assert X.shape == (n_dim, n_dim), "X shape mismatch" + if sparse_format is None: + assert not sp.issparse(X) + assert_allclose(X, X.T) + Xarr = X + else: + assert sp.issparse(X) and X.format == sparse_format + assert_allclose_dense_sparse(X, X.T) + Xarr = X.toarray() - assert X.shape == (5, 3), "X shape mismatch" - assert t.shape == (5,), "t shape mismatch" + from numpy.linalg import eig + + # Do not use scipy.sparse.linalg.eigs because it cannot find all eigenvalues + eigenvalues, _ = eig(Xarr) + assert np.all(eigenvalues > 0), "X is not positive-definite" + + if norm_diag: + # Check that leading diagonal elements are 1 + assert_array_almost_equal(Xarr.diagonal(), np.ones(n_dim)) + + +@pytest.mark.parametrize("hole", [False, True]) +def test_make_swiss_roll(global_random_seed, hole): + X, t = make_swiss_roll( + n_samples=5, noise=0.0, random_state=global_random_seed, hole=hole + ) + + assert X.shape == (5, 3) + assert t.shape == (5,) assert_array_almost_equal(X[:, 0], t * np.cos(t)) assert_array_almost_equal(X[:, 2], t * np.sin(t)) -def test_make_s_curve(): - X, t = make_s_curve(n_samples=5, noise=0.0, random_state=0) +def test_make_s_curve(global_random_seed): + X, t = make_s_curve(n_samples=5, noise=0.0, random_state=global_random_seed) assert X.shape == (5, 3), "X shape mismatch" assert t.shape == (5,), "t shape mismatch" @@ -454,106 +631,114 @@ def test_make_s_curve(): def test_make_biclusters(): X, rows, cols = make_biclusters( - shape=(100, 100), n_clusters=4, shuffle=True, random_state=0) + shape=(100, 100), n_clusters=4, shuffle=True, random_state=0 + ) assert X.shape == (100, 100), "X shape mismatch" assert rows.shape == (4, 100), "rows shape mismatch" - assert cols.shape == (4, 100,), "columns shape mismatch" + assert cols.shape == ( + 4, + 100, + ), "columns shape mismatch" assert_all_finite(X) assert_all_finite(rows) assert_all_finite(cols) - X2, _, _ = make_biclusters(shape=(100, 100), n_clusters=4, - shuffle=True, random_state=0) + X2, _, _ = make_biclusters( + shape=(100, 100), n_clusters=4, shuffle=True, random_state=0 + ) assert_array_almost_equal(X, X2) def test_make_checkerboard(): X, rows, cols = make_checkerboard( - shape=(100, 100), n_clusters=(20, 5), - shuffle=True, random_state=0) + shape=(100, 100), n_clusters=(20, 5), shuffle=True, random_state=0 + ) assert X.shape == (100, 100), "X shape mismatch" assert rows.shape == (100, 100), "rows shape mismatch" - assert cols.shape == (100, 100,), "columns shape mismatch" + assert cols.shape == ( + 100, + 100, + ), "columns shape mismatch" X, rows, cols = make_checkerboard( - shape=(100, 100), n_clusters=2, shuffle=True, random_state=0) + shape=(100, 100), n_clusters=2, shuffle=True, random_state=0 + ) assert_all_finite(X) assert_all_finite(rows) assert_all_finite(cols) - X1, _, _ = make_checkerboard(shape=(100, 100), n_clusters=2, - shuffle=True, random_state=0) - X2, _, _ = make_checkerboard(shape=(100, 100), n_clusters=2, - shuffle=True, random_state=0) + X1, _, _ = make_checkerboard( + shape=(100, 100), n_clusters=2, shuffle=True, random_state=0 + ) + X2, _, _ = make_checkerboard( + shape=(100, 100), n_clusters=2, shuffle=True, random_state=0 + ) assert_array_almost_equal(X1, X2) -def test_make_moons(): - X, y = make_moons(3, shuffle=False) +def test_make_moons(global_random_seed): + X, y = make_moons(3, shuffle=False, random_state=global_random_seed) for x, label in zip(X, y): center = [0.0, 0.0] if label == 0 else [1.0, 0.5] dist_sqr = ((x - center) ** 2).sum() - assert_almost_equal(dist_sqr, 1.0, - err_msg="Point is not on expected unit circle") + assert_almost_equal( + dist_sqr, 1.0, err_msg="Point is not on expected unit circle" + ) def test_make_moons_unbalanced(): X, y = make_moons(n_samples=(7, 5)) - assert np.sum(y == 0) == 7 and np.sum(y == 1) == 5, \ - 'Number of samples in a moon is wrong' + assert np.sum(y == 0) == 7 and np.sum(y == 1) == 5, ( + "Number of samples in a moon is wrong" + ) assert X.shape == (12, 2), "X shape mismatch" assert y.shape == (12,), "y shape mismatch" - with pytest.raises(ValueError, match=r'`n_samples` can be either an int ' - r'or a two-element tuple.'): - make_moons(n_samples=[1, 2, 3]) - - with pytest.raises(ValueError, match=r'`n_samples` can be either an int ' - r'or a two-element tuple.'): + with pytest.raises( + ValueError, + match=r"`n_samples` can be either an int or a two-element tuple.", + ): make_moons(n_samples=(10,)) def test_make_circles(): factor = 0.3 - for (n_samples, n_outer, n_inner) in [(7, 3, 4), (8, 4, 4)]: + for n_samples, n_outer, n_inner in [(7, 3, 4), (8, 4, 4)]: # Testing odd and even case, because in the past make_circles always # created an even number of samples. - X, y = make_circles(n_samples, shuffle=False, noise=None, - factor=factor) + X, y = make_circles(n_samples, shuffle=False, noise=None, factor=factor) assert X.shape == (n_samples, 2), "X shape mismatch" assert y.shape == (n_samples,), "y shape mismatch" center = [0.0, 0.0] for x, label in zip(X, y): dist_sqr = ((x - center) ** 2).sum() dist_exp = 1.0 if label == 0 else factor**2 - dist_exp = 1.0 if label == 0 else factor ** 2 - assert_almost_equal(dist_sqr, dist_exp, - err_msg="Point is not on expected circle") - - assert X[y == 0].shape == (n_outer, 2), ( - "Samples not correctly distributed across circles.") - assert X[y == 1].shape == (n_inner, 2), ( - "Samples not correctly distributed across circles.") + dist_exp = 1.0 if label == 0 else factor**2 + assert_almost_equal( + dist_sqr, dist_exp, err_msg="Point is not on expected circle" + ) - with pytest.raises(ValueError): - make_circles(factor=-0.01) - with pytest.raises(ValueError): - make_circles(factor=1.) + assert X[y == 0].shape == ( + n_outer, + 2, + ), "Samples not correctly distributed across circles." + assert X[y == 1].shape == ( + n_inner, + 2, + ), "Samples not correctly distributed across circles." def test_make_circles_unbalanced(): X, y = make_circles(n_samples=(2, 8)) - assert np.sum(y == 0) == 2, 'Number of samples in inner circle is wrong' - assert np.sum(y == 1) == 8, 'Number of samples in outer circle is wrong' + assert np.sum(y == 0) == 2, "Number of samples in inner circle is wrong" + assert np.sum(y == 1) == 8, "Number of samples in outer circle is wrong" assert X.shape == (10, 2), "X shape mismatch" assert y.shape == (10,), "y shape mismatch" - with pytest.raises(ValueError, match=r'`n_samples` can be either an int ' - r'or a two-element tuple.'): - make_circles(n_samples=[1, 2, 3]) - - with pytest.raises(ValueError, match=r'`n_samples` can be either an int ' - r'or a two-element tuple.'): + with pytest.raises( + ValueError, + match="When a tuple, n_samples must have exactly two elements.", + ): make_circles(n_samples=(10,)) diff --git a/sklearn/datasets/tests/test_svmlight_format.py b/sklearn/datasets/tests/test_svmlight_format.py index 336069c1c8251..ce19cc71da51c 100644 --- a/sklearn/datasets/tests/test_svmlight_format.py +++ b/sklearn/datasets/tests/test_svmlight_format.py @@ -1,33 +1,47 @@ -from bz2 import BZ2File import gzip -from io import BytesIO -import numpy as np -import scipy.sparse as sp import os import shutil +from bz2 import BZ2File +from importlib import resources +from io import BytesIO from tempfile import NamedTemporaryFile +import numpy as np import pytest - -from sklearn.utils._testing import assert_array_equal -from sklearn.utils._testing import assert_array_almost_equal -from sklearn.utils._testing import fails_if_pypy +import scipy.sparse as sp import sklearn -from sklearn.datasets import (load_svmlight_file, load_svmlight_files, - dump_svmlight_file) +from sklearn.datasets import dump_svmlight_file, load_svmlight_file, load_svmlight_files +from sklearn.utils._testing import ( + assert_allclose, + assert_array_almost_equal, + assert_array_equal, + create_memmap_backed_data, +) +from sklearn.utils.fixes import CSR_CONTAINERS + +TEST_DATA_MODULE = "sklearn.datasets.tests.data" +datafile = "svmlight_classification.txt" +multifile = "svmlight_multilabel.txt" +invalidfile = "svmlight_invalid.txt" +invalidfile2 = "svmlight_invalid_order.txt" + + +def _svmlight_local_test_file_path(filename): + return resources.files(TEST_DATA_MODULE) / filename -currdir = os.path.dirname(os.path.abspath(__file__)) -datafile = os.path.join(currdir, "data", "svmlight_classification.txt") -multifile = os.path.join(currdir, "data", "svmlight_multilabel.txt") -invalidfile = os.path.join(currdir, "data", "svmlight_invalid.txt") -invalidfile2 = os.path.join(currdir, "data", "svmlight_invalid_order.txt") -pytestmark = fails_if_pypy +def _load_svmlight_local_test_file(filename, **kwargs): + """ + Helper to load resource `filename` with `importlib.resources` + """ + data_path = _svmlight_local_test_file_path(filename) + with data_path.open("rb") as f: + return load_svmlight_file(f, **kwargs) def test_load_svmlight_file(): - X, y = load_svmlight_file(datafile) + X, y = _load_svmlight_local_test_file(datafile) # test X's shape assert X.indptr.shape[0] == 7 @@ -36,10 +50,14 @@ def test_load_svmlight_file(): assert y.shape[0] == 6 # test X's non-zero values - for i, j, val in ((0, 2, 2.5), (0, 10, -5.2), (0, 15, 1.5), - (1, 5, 1.0), (1, 12, -3), - (2, 20, 27)): - + for i, j, val in ( + (0, 2, 2.5), + (0, 10, -5.2), + (0, 15, 1.5), + (1, 5, 1.0), + (1, 12, -3), + (2, 20, 27), + ): assert X[i, j] == val # tests X's zero values @@ -59,9 +77,15 @@ def test_load_svmlight_file(): def test_load_svmlight_file_fd(): # test loading from file descriptor - X1, y1 = load_svmlight_file(datafile) - fd = os.open(datafile, os.O_RDONLY) + # GH20081: testing equality between path-based and + # fd-based load_svmlight_file + + data_path = resources.files(TEST_DATA_MODULE) / datafile + data_path = str(data_path) + X1, y1 = load_svmlight_file(data_path) + + fd = os.open(data_path, os.O_RDONLY) try: X2, y2 = load_svmlight_file(fd) assert_array_almost_equal(X1.data, X2.data) @@ -70,28 +94,39 @@ def test_load_svmlight_file_fd(): os.close(fd) +def test_load_svmlight_pathlib(): + # test loading from file descriptor + data_path = _svmlight_local_test_file_path(datafile) + X1, y1 = load_svmlight_file(str(data_path)) + X2, y2 = load_svmlight_file(data_path) + + assert_allclose(X1.data, X2.data) + assert_allclose(y1, y2) + + def test_load_svmlight_file_multilabel(): - X, y = load_svmlight_file(multifile, multilabel=True) + X, y = _load_svmlight_local_test_file(multifile, multilabel=True) assert y == [(0, 1), (2,), (), (1, 2)] def test_load_svmlight_files(): - X_train, y_train, X_test, y_test = load_svmlight_files([datafile] * 2, - dtype=np.float32) + data_path = _svmlight_local_test_file_path(datafile) + X_train, y_train, X_test, y_test = load_svmlight_files( + [str(data_path)] * 2, dtype=np.float32 + ) assert_array_equal(X_train.toarray(), X_test.toarray()) assert_array_almost_equal(y_train, y_test) assert X_train.dtype == np.float32 assert X_test.dtype == np.float32 - X1, y1, X2, y2, X3, y3 = load_svmlight_files([datafile] * 3, - dtype=np.float64) + X1, y1, X2, y2, X3, y3 = load_svmlight_files([str(data_path)] * 3, dtype=np.float64) assert X1.dtype == X2.dtype assert X2.dtype == X3.dtype assert X3.dtype == np.float64 def test_load_svmlight_file_n_features(): - X, y = load_svmlight_file(datafile, n_features=22) + X, y = _load_svmlight_local_test_file(datafile, n_features=22) # test X'shape assert X.indptr.shape[0] == 7 @@ -99,22 +134,20 @@ def test_load_svmlight_file_n_features(): assert X.shape[1] == 22 # test X's non-zero values - for i, j, val in ((0, 2, 2.5), (0, 10, -5.2), - (1, 5, 1.0), (1, 12, -3)): - + for i, j, val in ((0, 2, 2.5), (0, 10, -5.2), (1, 5, 1.0), (1, 12, -3)): assert X[i, j] == val # 21 features in file with pytest.raises(ValueError): - load_svmlight_file(datafile, n_features=20) + _load_svmlight_local_test_file(datafile, n_features=20) def test_load_compressed(): - X, y = load_svmlight_file(datafile) + X, y = _load_svmlight_local_test_file(datafile) with NamedTemporaryFile(prefix="sklearn-test", suffix=".gz") as tmp: tmp.close() # necessary under windows - with open(datafile, "rb") as f: + with _svmlight_local_test_file_path(datafile).open("rb") as f: with gzip.open(tmp.name, "wb") as fh_out: shutil.copyfileobj(f, fh_out) Xgz, ygz = load_svmlight_file(tmp.name) @@ -126,7 +159,7 @@ def test_load_compressed(): with NamedTemporaryFile(prefix="sklearn-test", suffix=".bz2") as tmp: tmp.close() # necessary under windows - with open(datafile, "rb") as f: + with _svmlight_local_test_file_path(datafile).open("rb") as f: with BZ2File(tmp.name, "wb") as fh_out: shutil.copyfileobj(f, fh_out) Xbz, ybz = load_svmlight_file(tmp.name) @@ -139,12 +172,12 @@ def test_load_compressed(): def test_load_invalid_file(): with pytest.raises(ValueError): - load_svmlight_file(invalidfile) + _load_svmlight_local_test_file(invalidfile) def test_load_invalid_order_file(): with pytest.raises(ValueError): - load_svmlight_file(invalidfile2) + _load_svmlight_local_test_file(invalidfile2) def test_load_zero_based(): @@ -176,49 +209,57 @@ def test_load_with_qid(): 7 qid:2 1:0.87 2:0.12""" X, y = load_svmlight_file(BytesIO(data), query_id=False) assert_array_equal(y, [3, 2, 7]) - assert_array_equal(X.toarray(), [[.53, .12], [.13, .1], [.87, .12]]) + assert_array_equal(X.toarray(), [[0.53, 0.12], [0.13, 0.1], [0.87, 0.12]]) res1 = load_svmlight_files([BytesIO(data)], query_id=True) res2 = load_svmlight_file(BytesIO(data), query_id=True) for X, y, qid in (res1, res2): assert_array_equal(y, [3, 2, 7]) assert_array_equal(qid, [1, 1, 2]) - assert_array_equal(X.toarray(), [[.53, .12], [.13, .1], [.87, .12]]) + assert_array_equal(X.toarray(), [[0.53, 0.12], [0.13, 0.1], [0.87, 0.12]]) -@pytest.mark.skip("testing the overflow of 32 bit sparse indexing requires a" - " large amount of memory") +@pytest.mark.skip( + "testing the overflow of 32 bit sparse indexing requires a large amount of memory" +) def test_load_large_qid(): """ load large libsvm / svmlight file with qid attribute. Tests 64-bit query ID """ - data = b"\n".join(("3 qid:{0} 1:0.53 2:0.12\n2 qid:{0} 1:0.13 2:0.1" - .format(i).encode() for i in range(1, 40*1000*1000))) + data = b"\n".join( + ( + "3 qid:{0} 1:0.53 2:0.12\n2 qid:{0} 1:0.13 2:0.1".format(i).encode() + for i in range(1, 40 * 1000 * 1000) + ) + ) X, y, qid = load_svmlight_file(BytesIO(data), query_id=True) assert_array_equal(y[-4:], [3, 2, 3, 2]) - assert_array_equal(np.unique(qid), np.arange(1, 40*1000*1000)) + assert_array_equal(np.unique(qid), np.arange(1, 40 * 1000 * 1000)) def test_load_invalid_file2(): with pytest.raises(ValueError): - load_svmlight_files([datafile, invalidfile, datafile]) + data_path = _svmlight_local_test_file_path(datafile) + invalid_path = _svmlight_local_test_file_path(invalidfile) + load_svmlight_files([str(data_path), str(invalid_path), str(data_path)]) def test_not_a_filename(): # in python 3 integers are valid file opening arguments (taken as unix # file descriptors) with pytest.raises(TypeError): - load_svmlight_file(.42) + load_svmlight_file(0.42) def test_invalid_filename(): - with pytest.raises(IOError): + with pytest.raises(OSError): load_svmlight_file("trou pic nic douille") -def test_dump(): - X_sparse, y_dense = load_svmlight_file(datafile) +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_dump(csr_container): + X_sparse, y_dense = _load_svmlight_local_test_file(datafile) X_dense = X_sparse.toarray() - y_sparse = sp.csr_matrix(y_dense) + y_sparse = csr_container(np.atleast_2d(y_dense)) # slicing a csr_matrix can unsort its .indices, so test that we sort # those correctly @@ -234,7 +275,7 @@ def test_dump(): # LibSVM doesn't grok comments so they're not put in by # default anymore. - if (sp.issparse(y) and y.shape[0] == 1): + if sp.issparse(y) and y.shape[0] == 1: # make sure y's shape is: (n_samples, n_labels) # when it is sparse y = y.T @@ -245,8 +286,9 @@ def test_dump(): # different from X_sparse.astype(dtype).asarray(). X_input = X.astype(dtype) - dump_svmlight_file(X_input, y, f, comment="test", - zero_based=zero_based) + dump_svmlight_file( + X_input, y, f, comment="test", zero_based=zero_based + ) f.seek(0) comment = f.readline() @@ -259,8 +301,7 @@ def test_dump(): assert ["one", "zero"][zero_based] + "-based" in comment - X2, y2 = load_svmlight_file(f, dtype=dtype, - zero_based=zero_based) + X2, y2 = load_svmlight_file(f, dtype=dtype, zero_based=zero_based) assert X2.dtype == dtype assert_array_equal(X2.sorted_indices().indices, X2.indices) @@ -272,24 +313,23 @@ def test_dump(): if dtype == np.float32: # allow a rounding error at the last decimal place + assert_array_almost_equal(X_input_dense, X2_dense, 4) assert_array_almost_equal( - X_input_dense, X2_dense, 4) - assert_array_almost_equal( - y_dense.astype(dtype, copy=False), y2, 4) + y_dense.astype(dtype, copy=False), y2, 4 + ) else: # allow a rounding error at the last decimal place + assert_array_almost_equal(X_input_dense, X2_dense, 15) assert_array_almost_equal( - X_input_dense, X2_dense, 15) - assert_array_almost_equal( - y_dense.astype(dtype, copy=False), y2, 15) + y_dense.astype(dtype, copy=False), y2, 15 + ) -def test_dump_multilabel(): - X = [[1, 0, 3, 0, 5], - [0, 0, 0, 0, 0], - [0, 5, 0, 1, 0]] +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_dump_multilabel(csr_container): + X = [[1, 0, 3, 0, 5], [0, 0, 0, 0, 0], [0, 5, 0, 1, 0]] y_dense = [[0, 1, 0], [1, 0, 1], [1, 1, 0]] - y_sparse = sp.csr_matrix(y_dense) + y_sparse = csr_container(y_dense) for y in [y_dense, y_sparse]: f = BytesIO() dump_svmlight_file(X, y, f, multilabel=True) @@ -307,18 +347,19 @@ def test_dump_concise(): exact = 1.000000000000001 # loses the last decimal place almost = 1.0000000000000001 - X = [[one, two, three, exact, almost], - [1e9, 2e18, 3e27, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 0]] + X = [ + [one, two, three, exact, almost], + [1e9, 2e18, 3e27, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + ] y = [one, two, three, exact, almost] f = BytesIO() dump_svmlight_file(X, y, f) f.seek(0) # make sure it's using the most concise format possible - assert (f.readline() == - b"1 0:1 1:2.1 2:3.01 3:1.000000000000001 4:1\n") + assert f.readline() == b"1 0:1 1:2.1 2:3.01 3:1.000000000000001 4:1\n" assert f.readline() == b"2.1 0:1000000000 1:2e+18 2:3e+27\n" assert f.readline() == b"3.01 \n" assert f.readline() == b"1.000000000000001 \n" @@ -331,7 +372,7 @@ def test_dump_concise(): def test_dump_comment(): - X, y = load_svmlight_file(datafile) + X, y = _load_svmlight_local_test_file(datafile) X = X.toarray() f = BytesIO() @@ -364,7 +405,7 @@ def test_dump_comment(): def test_dump_invalid(): - X, y = load_svmlight_file(datafile) + X, y = _load_svmlight_local_test_file(datafile) f = BytesIO() y2d = [y] @@ -378,7 +419,7 @@ def test_dump_invalid(): def test_dump_query_id(): # test dumping a file with query_id - X, y = load_svmlight_file(datafile) + X, y = _load_svmlight_local_test_file(datafile) X = X.toarray() query_id = np.arange(X.shape[0]) // 2 f = BytesIO() @@ -400,10 +441,12 @@ def test_load_with_long_qid(): 3 qid:9223372036854775807 0:1440446648 1:72048431380967004 2:236784985""" X, y, qid = load_svmlight_file(BytesIO(data), query_id=True) - true_X = [[1, 2, 3], - [1440446648, 72048431380967004, 236784985], - [1440446648, 72048431380967004, 236784985], - [1440446648, 72048431380967004, 236784985]] + true_X = [ + [1, 2, 3], + [1440446648, 72048431380967004, 236784985], + [1440446648, 72048431380967004, 236784985], + [1440446648, 72048431380967004, 236784985], + ] true_y = [1, 0, 0, 3] trueQID = [0, 72048431380967004, -9223372036854775807, 9223372036854775807] @@ -425,28 +468,30 @@ def test_load_with_long_qid(): assert_array_equal(X.toarray(), true_X) -def test_load_zeros(): +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_load_zeros(csr_container): f = BytesIO() - true_X = sp.csr_matrix(np.zeros(shape=(3, 4))) + true_X = csr_container(np.zeros(shape=(3, 4))) true_y = np.array([0, 1, 0]) dump_svmlight_file(true_X, true_y, f) - for zero_based in ['auto', True, False]: + for zero_based in ["auto", True, False]: f.seek(0) X, y = load_svmlight_file(f, n_features=4, zero_based=zero_based) assert_array_almost_equal(y, true_y) assert_array_almost_equal(X.toarray(), true_X.toarray()) -@pytest.mark.parametrize('sparsity', [0, 0.1, .5, 0.99, 1]) -@pytest.mark.parametrize('n_samples', [13, 101]) -@pytest.mark.parametrize('n_features', [2, 7, 41]) -def test_load_with_offsets(sparsity, n_samples, n_features): +@pytest.mark.parametrize("sparsity", [0, 0.1, 0.5, 0.99, 1]) +@pytest.mark.parametrize("n_samples", [13, 101]) +@pytest.mark.parametrize("n_features", [2, 7, 41]) +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_load_with_offsets(sparsity, n_samples, n_features, csr_container): rng = np.random.RandomState(0) X = rng.uniform(low=0.0, high=1.0, size=(n_samples, n_features)) if sparsity: X[X < sparsity] = 0.0 - X = sp.csr_matrix(X) + X = csr_container(X) y = rng.randint(low=0, high=2, size=n_samples) f = BytesIO() @@ -463,12 +508,13 @@ def test_load_with_offsets(sparsity, n_samples, n_features): length_1 = mark_2 - mark_1 # load the original sparse matrix into 3 independent CSR matrices - X_0, y_0 = load_svmlight_file(f, n_features=n_features, - offset=mark_0, length=length_0) - X_1, y_1 = load_svmlight_file(f, n_features=n_features, - offset=mark_1, length=length_1) - X_2, y_2 = load_svmlight_file(f, n_features=n_features, - offset=mark_2) + X_0, y_0 = load_svmlight_file( + f, n_features=n_features, offset=mark_0, length=length_0 + ) + X_1, y_1 = load_svmlight_file( + f, n_features=n_features, offset=mark_1, length=length_1 + ) + X_2, y_2 = load_svmlight_file(f, n_features=n_features, offset=mark_2) y_concat = np.concatenate([y_0, y_1, y_2]) X_concat = sp.vstack([X_0, X_1, X_2]) @@ -476,18 +522,21 @@ def test_load_with_offsets(sparsity, n_samples, n_features): assert_array_almost_equal(X.toarray(), X_concat.toarray()) -def test_load_offset_exhaustive_splits(): +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_load_offset_exhaustive_splits(csr_container): rng = np.random.RandomState(0) - X = np.array([ - [0, 0, 0, 0, 0, 0], - [1, 2, 3, 4, 0, 6], - [1, 2, 3, 4, 0, 6], - [0, 0, 0, 0, 0, 0], - [1, 0, 3, 0, 0, 0], - [0, 0, 0, 0, 0, 1], - [1, 0, 0, 0, 0, 0], - ]) - X = sp.csr_matrix(X) + X = np.array( + [ + [0, 0, 0, 0, 0, 0], + [1, 2, 3, 4, 0, 6], + [1, 2, 3, 4, 0, 6], + [0, 0, 0, 0, 0, 0], + [1, 0, 3, 0, 0, 0], + [0, 0, 0, 0, 0, 1], + [1, 0, 0, 0, 0, 0], + ] + ) + X = csr_container(X) n_samples, n_features = X.shape y = rng.randint(low=0, high=2, size=n_samples) query_id = np.arange(n_samples) // 2 @@ -502,12 +551,12 @@ def test_load_offset_exhaustive_splits(): # locate the split so has to test for particular boundary cases for mark in range(size): f.seek(0) - X_0, y_0, q_0 = load_svmlight_file(f, n_features=n_features, - query_id=True, offset=0, - length=mark) - X_1, y_1, q_1 = load_svmlight_file(f, n_features=n_features, - query_id=True, offset=mark, - length=-1) + X_0, y_0, q_0 = load_svmlight_file( + f, n_features=n_features, query_id=True, offset=0, length=mark + ) + X_1, y_1, q_1 = load_svmlight_file( + f, n_features=n_features, query_id=True, offset=mark, length=-1 + ) q_concat = np.concatenate([q_0, q_1]) y_concat = np.concatenate([y_0, y_1]) X_concat = sp.vstack([X_0, X_1]) @@ -518,4 +567,47 @@ def test_load_offset_exhaustive_splits(): def test_load_with_offsets_error(): with pytest.raises(ValueError, match="n_features is required"): - load_svmlight_file(datafile, offset=3, length=3) + _load_svmlight_local_test_file(datafile, offset=3, length=3) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_multilabel_y_explicit_zeros(tmp_path, csr_container): + """ + Ensure that if y contains explicit zeros (i.e. elements of y.data equal to + 0) then those explicit zeros are not encoded. + """ + save_path = str(tmp_path / "svm_explicit_zero") + rng = np.random.RandomState(42) + X = rng.randn(3, 5).astype(np.float64) + indptr = np.array([0, 2, 3, 6]) + indices = np.array([0, 2, 2, 0, 1, 2]) + # The first and last element are explicit zeros. + data = np.array([0, 1, 1, 1, 1, 0]) + y = csr_container((data, indices, indptr), shape=(3, 3)) + # y as a dense array would look like + # [[0, 0, 1], + # [0, 0, 1], + # [1, 1, 0]] + + dump_svmlight_file(X, y, save_path, multilabel=True) + + _, y_load = load_svmlight_file(save_path, multilabel=True) + y_true = [(2.0,), (2.0,), (0.0, 1.0)] + assert y_load == y_true + + +def test_dump_read_only(tmp_path): + """Ensure that there is no ValueError when dumping a read-only `X`. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/28026 + """ + rng = np.random.RandomState(42) + X = rng.randn(5, 2) + y = rng.randn(5) + + # Convert to memmap-backed which are read-only + X, y = create_memmap_backed_data([X, y]) + + save_path = str(tmp_path / "svm_read_only") + dump_svmlight_file(X, y, save_path) diff --git a/sklearn/decomposition/__init__.py b/sklearn/decomposition/__init__.py index bdda493a43623..6d3fa9b42895a 100644 --- a/sklearn/decomposition/__init__.py +++ b/sklearn/decomposition/__init__.py @@ -1,41 +1,54 @@ -""" -The :mod:`sklearn.decomposition` module includes matrix decomposition -algorithms, including among others PCA, NMF or ICA. Most of the algorithms of -this module can be regarded as dimensionality reduction techniques. +"""Matrix decomposition algorithms. + +These include PCA, NMF, ICA, and more. Most of the algorithms of this module can be +regarded as dimensionality reduction techniques. """ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -from ._nmf import NMF, non_negative_factorization -from ._pca import PCA +from ..utils.extmath import randomized_svd +from ._dict_learning import ( + DictionaryLearning, + MiniBatchDictionaryLearning, + SparseCoder, + dict_learning, + dict_learning_online, + sparse_encode, +) +from ._factor_analysis import FactorAnalysis +from ._fastica import FastICA, fastica from ._incremental_pca import IncrementalPCA from ._kernel_pca import KernelPCA -from ._sparse_pca import SparsePCA, MiniBatchSparsePCA -from ._truncated_svd import TruncatedSVD -from ._fastica import FastICA, fastica -from ._dict_learning import (dict_learning, dict_learning_online, - sparse_encode, DictionaryLearning, - MiniBatchDictionaryLearning, SparseCoder) -from ._factor_analysis import FactorAnalysis -from ..utils.extmath import randomized_svd from ._lda import LatentDirichletAllocation +from ._nmf import ( + NMF, + MiniBatchNMF, + non_negative_factorization, +) +from ._pca import PCA +from ._sparse_pca import MiniBatchSparsePCA, SparsePCA +from ._truncated_svd import TruncatedSVD - -__all__ = ['DictionaryLearning', - 'FastICA', - 'IncrementalPCA', - 'KernelPCA', - 'MiniBatchDictionaryLearning', - 'MiniBatchSparsePCA', - 'NMF', - 'PCA', - 'SparseCoder', - 'SparsePCA', - 'dict_learning', - 'dict_learning_online', - 'fastica', - 'non_negative_factorization', - 'randomized_svd', - 'sparse_encode', - 'FactorAnalysis', - 'TruncatedSVD', - 'LatentDirichletAllocation'] +__all__ = [ + "NMF", + "PCA", + "DictionaryLearning", + "FactorAnalysis", + "FastICA", + "IncrementalPCA", + "KernelPCA", + "LatentDirichletAllocation", + "MiniBatchDictionaryLearning", + "MiniBatchNMF", + "MiniBatchSparsePCA", + "SparseCoder", + "SparsePCA", + "TruncatedSVD", + "dict_learning", + "dict_learning_online", + "fastica", + "non_negative_factorization", + "randomized_svd", + "sparse_encode", +] diff --git a/sklearn/decomposition/_base.py b/sklearn/decomposition/_base.py index b944d23d3388d..783c316b50f27 100644 --- a/sklearn/decomposition/_base.py +++ b/sklearn/decomposition/_base.py @@ -1,27 +1,27 @@ """Principal Component Analysis Base Classes""" -# Author: Alexandre Gramfort -# Olivier Grisel -# Mathieu Blondel -# Denis A. Engemann -# Kyle Kastner -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from abc import ABCMeta, abstractmethod import numpy as np from scipy import linalg -from ..base import BaseEstimator, TransformerMixin -from ..utils.validation import check_is_fitted -from abc import ABCMeta, abstractmethod +from ..base import BaseEstimator, ClassNamePrefixFeaturesOutMixin, TransformerMixin +from ..utils._array_api import _fill_or_add_to_diagonal, device, get_namespace +from ..utils.validation import check_is_fitted, validate_data -class _BasePCA(TransformerMixin, BaseEstimator, metaclass=ABCMeta): +class _BasePCA( + ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator, metaclass=ABCMeta +): """Base class for PCA methods. Warning: This class should not be used directly. Use derived classes instead. """ + def get_covariance(self): """Compute data covariance with the generative model. @@ -31,16 +31,23 @@ def get_covariance(self): Returns ------- - cov : array, shape=(n_features, n_features) + cov : array of shape=(n_features, n_features) Estimated covariance of data. """ + xp, _ = get_namespace(self.components_) + components_ = self.components_ exp_var = self.explained_variance_ if self.whiten: - components_ = components_ * np.sqrt(exp_var[:, np.newaxis]) - exp_var_diff = np.maximum(exp_var - self.noise_variance_, 0.) - cov = np.dot(components_.T * exp_var_diff, components_) - cov.flat[::len(cov) + 1] += self.noise_variance_ # modify diag inplace + components_ = components_ * xp.sqrt(exp_var[:, np.newaxis]) + exp_var_diff = exp_var - self.noise_variance_ + exp_var_diff = xp.where( + exp_var > self.noise_variance_, + exp_var_diff, + xp.asarray(0.0, device=device(exp_var), dtype=exp_var.dtype), + ) + cov = (components_.T * exp_var_diff) @ components_ + _fill_or_add_to_diagonal(cov, self.noise_variance_, xp) return cov def get_precision(self): @@ -54,26 +61,38 @@ def get_precision(self): precision : array, shape=(n_features, n_features) Estimated precision of data. """ + xp, is_array_api_compliant = get_namespace(self.components_) + n_features = self.components_.shape[1] # handle corner cases first if self.n_components_ == 0: - return np.eye(n_features) / self.noise_variance_ - if self.n_components_ == n_features: - return linalg.inv(self.get_covariance()) + return xp.eye(n_features) / self.noise_variance_ + + if is_array_api_compliant: + linalg_inv = xp.linalg.inv + else: + linalg_inv = linalg.inv + + if self.noise_variance_ == 0.0: + return linalg_inv(self.get_covariance()) # Get precision using matrix inversion lemma components_ = self.components_ exp_var = self.explained_variance_ if self.whiten: - components_ = components_ * np.sqrt(exp_var[:, np.newaxis]) - exp_var_diff = np.maximum(exp_var - self.noise_variance_, 0.) - precision = np.dot(components_, components_.T) / self.noise_variance_ - precision.flat[::len(precision) + 1] += 1. / exp_var_diff - precision = np.dot(components_.T, - np.dot(linalg.inv(precision), components_)) - precision /= -(self.noise_variance_ ** 2) - precision.flat[::len(precision) + 1] += 1. / self.noise_variance_ + components_ = components_ * xp.sqrt(exp_var[:, np.newaxis]) + exp_var_diff = exp_var - self.noise_variance_ + exp_var_diff = xp.where( + exp_var > self.noise_variance_, + exp_var_diff, + xp.asarray(0.0, device=device(exp_var)), + ) + precision = components_ @ components_.T / self.noise_variance_ + _fill_or_add_to_diagonal(precision, 1.0 / exp_var_diff, xp) + precision = components_.T @ linalg_inv(precision) @ components_ + precision /= -(self.noise_variance_**2) + _fill_or_add_to_diagonal(precision, 1.0 / self.noise_variance_, xp) return precision @abstractmethod @@ -84,9 +103,9 @@ def fit(self, X, y=None): Parameters ---------- - X : array-like, shape (n_samples, n_features) - Training data, where n_samples is the number of samples and - n_features is the number of features. + X : array-like of shape (n_samples, n_features) + Training data, where `n_samples` is the number of samples and + `n_features` is the number of features. Returns ------- @@ -102,46 +121,82 @@ def transform(self, X): Parameters ---------- - X : array-like, shape (n_samples, n_features) - New data, where n_samples is the number of samples - and n_features is the number of features. + X : {array-like, sparse matrix} of shape (n_samples, n_features) + New data, where `n_samples` is the number of samples + and `n_features` is the number of features. Returns ------- - X_new : array-like, shape (n_samples, n_components) + X_new : array-like of shape (n_samples, n_components) + Projection of X in the first principal components, where `n_samples` + is the number of samples and `n_components` is the number of the components. """ + xp, _ = get_namespace(X, self.components_, self.explained_variance_) + check_is_fitted(self) - X = self._validate_data(X, dtype=[np.float64, np.float32], reset=False) - if self.mean_ is not None: - X = X - self.mean_ - X_transformed = np.dot(X, self.components_.T) + X = validate_data( + self, + X, + dtype=[xp.float64, xp.float32], + accept_sparse=("csr", "csc"), + reset=False, + ) + return self._transform(X, xp=xp, x_is_centered=False) + + def _transform(self, X, xp, x_is_centered=False): + X_transformed = X @ self.components_.T + if not x_is_centered: + # Apply the centering after the projection. + # For dense X this avoids copying or mutating the data passed by + # the caller. + # For sparse X it keeps sparsity and avoids having to wrap X into + # a linear operator. + X_transformed -= xp.reshape(self.mean_, (1, -1)) @ self.components_.T if self.whiten: - X_transformed /= np.sqrt(self.explained_variance_) + # For some solvers (such as "arpack" and "covariance_eigh"), on + # rank deficient data, some components can have a variance + # arbitrarily close to zero, leading to non-finite results when + # whitening. To avoid this problem we clip the variance below. + scale = xp.sqrt(self.explained_variance_) + min_scale = xp.finfo(scale.dtype).eps + scale[scale < min_scale] = min_scale + X_transformed /= scale return X_transformed def inverse_transform(self, X): """Transform data back to its original space. - In other words, return an input X_original whose transform would be X. + In other words, return an input `X_original` whose transform would be X. Parameters ---------- - X : array-like, shape (n_samples, n_components) - New data, where n_samples is the number of samples - and n_components is the number of components. + X : array-like of shape (n_samples, n_components) + New data, where `n_samples` is the number of samples + and `n_components` is the number of components. Returns ------- - X_original array-like, shape (n_samples, n_features) + X_original : array-like of shape (n_samples, n_features) + Original data, where `n_samples` is the number of samples + and `n_features` is the number of features. Notes ----- If whitening is enabled, inverse_transform will compute the exact inverse operation, which includes reversing whitening. """ + xp, _ = get_namespace(X) + if self.whiten: - return np.dot(X, np.sqrt(self.explained_variance_[:, np.newaxis]) * - self.components_) + self.mean_ + scaled_components = ( + xp.sqrt(self.explained_variance_[:, np.newaxis]) * self.components_ + ) + return X @ scaled_components + self.mean_ else: - return np.dot(X, self.components_) + self.mean_ + return X @ self.components_ + self.mean_ + + @property + def _n_features_out(self): + """Number of transformed output features.""" + return self.components_.shape[0] diff --git a/sklearn/decomposition/_cdnmf_fast.pyx b/sklearn/decomposition/_cdnmf_fast.pyx index 9c6b171096ced..b2a07fb275bde 100644 --- a/sklearn/decomposition/_cdnmf_fast.pyx +++ b/sklearn/decomposition/_cdnmf_fast.pyx @@ -1,9 +1,5 @@ -# cython: cdivision=True -# cython: boundscheck=False -# cython: wraparound=False - -# Author: Mathieu Blondel, Tom Dupre la Tour -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause from cython cimport floating from libc.math cimport fabs @@ -38,5 +34,5 @@ def _update_cdnmf_fast(floating[:, ::1] W, floating[:, :] HHt, if hess != 0: W[i, t] = max(W[i, t] - grad / hess, 0.) - + return violation diff --git a/sklearn/decomposition/_dict_learning.py b/sklearn/decomposition/_dict_learning.py index e2ae9f8355a54..ae40e28e9f013 100644 --- a/sklearn/decomposition/_dict_learning.py +++ b/sklearn/decomposition/_dict_learning.py @@ -1,44 +1,55 @@ -""" Dictionary learning. -""" -# Author: Vlad Niculae, Gael Varoquaux, Alexandre Gramfort -# License: BSD 3 clause +"""Dictionary learning.""" -import time -import sys -import itertools -import warnings +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -from math import ceil +import itertools +import sys +import time +from numbers import Integral, Real import numpy as np +from joblib import effective_n_jobs from scipy import linalg -from joblib import Parallel, effective_n_jobs -from ..base import BaseEstimator, TransformerMixin -from ..utils import deprecated -from ..utils import (check_array, check_random_state, gen_even_slices, - gen_batches) -from ..utils.extmath import randomized_svd, row_norms, svd_flip -from ..utils.validation import check_is_fitted, _deprecate_positional_args -from ..utils.fixes import delayed -from ..linear_model import Lasso, orthogonal_mp_gram, LassoLars, Lars +from ..base import ( + BaseEstimator, + ClassNamePrefixFeaturesOutMixin, + TransformerMixin, + _fit_context, +) +from ..linear_model import Lars, Lasso, LassoLars, orthogonal_mp_gram +from ..utils import check_array, check_random_state, gen_batches, gen_even_slices +from ..utils._param_validation import Interval, StrOptions, validate_params +from ..utils.extmath import _randomized_svd, row_norms, svd_flip +from ..utils.parallel import Parallel, delayed +from ..utils.validation import check_is_fitted, validate_data def _check_positive_coding(method, positive): if positive and method in ["omp", "lars"]: raise ValueError( - "Positive constraint not supported for '{}' " - "coding method.".format(method) - ) - + "Positive constraint not supported for '{}' coding method.".format(method) + ) -def _sparse_encode(X, dictionary, gram, cov=None, algorithm='lasso_lars', - regularization=None, copy_cov=True, - init=None, max_iter=1000, check_input=True, verbose=0, - positive=False): - """Generic sparse coding. - Each column of the result is the solution to a Lasso problem. +def _sparse_encode_precomputed( + X, + dictionary, + *, + gram=None, + cov=None, + algorithm="lasso_lars", + regularization=None, + copy_cov=True, + init=None, + max_iter=1000, + verbose=0, + positive=False, +): + """Generic sparse coding with precomputed Gram and/or covariance matrices. + + Each row of the result is the solution to a Lasso problem. Parameters ---------- @@ -49,7 +60,7 @@ def _sparse_encode(X, dictionary, gram, cov=None, algorithm='lasso_lars', The dictionary matrix against which to solve the sparse coding of the data. Some of the algorithms assume normalized rows. - gram : ndarray of shape (n_components, n_components) or None + gram : ndarray of shape (n_components, n_components), default=None Precomputed Gram matrix, `dictionary * dictionary'` gram can be `None` if method is 'threshold'. @@ -88,9 +99,6 @@ def _sparse_encode(X, dictionary, gram, cov=None, algorithm='lasso_lars', Whether to copy the precomputed covariance matrix; if `False`, it may be overwritten. - check_input : bool, default=True - If `False`, the input arrays `X` and dictionary will not be checked. - verbose : int, default=0 Controls the verbosity; the higher, the more messages. @@ -103,102 +111,135 @@ def _sparse_encode(X, dictionary, gram, cov=None, algorithm='lasso_lars', ------- code : ndarray of shape (n_components, n_features) The sparse codes. - - See Also - -------- - sklearn.linear_model.lars_path - sklearn.linear_model.orthogonal_mp - sklearn.linear_model.Lasso - SparseCoder """ - if X.ndim == 1: - X = X[:, np.newaxis] n_samples, n_features = X.shape n_components = dictionary.shape[0] - if dictionary.shape[1] != X.shape[1]: - raise ValueError("Dictionary and X have different numbers of features:" - "dictionary.shape: {} X.shape{}".format( - dictionary.shape, X.shape)) - if cov is None and algorithm != 'lasso_cd': - # overwriting cov is safe - copy_cov = False - cov = np.dot(dictionary, X.T) - _check_positive_coding(algorithm, positive) - - if algorithm == 'lasso_lars': + if algorithm == "lasso_lars": alpha = float(regularization) / n_features # account for scaling try: - err_mgt = np.seterr(all='ignore') + err_mgt = np.seterr(all="ignore") # Not passing in verbose=max(0, verbose-1) because Lars.fit already # corrects the verbosity level. - lasso_lars = LassoLars(alpha=alpha, fit_intercept=False, - verbose=verbose, normalize=False, - precompute=gram, fit_path=False, - positive=positive, max_iter=max_iter) + lasso_lars = LassoLars( + alpha=alpha, + fit_intercept=False, + verbose=verbose, + precompute=gram, + fit_path=False, + positive=positive, + max_iter=max_iter, + ) lasso_lars.fit(dictionary.T, X.T, Xy=cov) new_code = lasso_lars.coef_ finally: np.seterr(**err_mgt) - elif algorithm == 'lasso_cd': + elif algorithm == "lasso_cd": alpha = float(regularization) / n_features # account for scaling # TODO: Make verbosity argument for Lasso? # sklearn.linear_model.coordinate_descent.enet_path has a verbosity # argument that we could pass in from Lasso. - clf = Lasso(alpha=alpha, fit_intercept=False, normalize=False, - precompute=gram, max_iter=max_iter, warm_start=True, - positive=positive) + clf = Lasso( + alpha=alpha, + fit_intercept=False, + precompute=gram, + max_iter=max_iter, + warm_start=True, + positive=positive, + ) if init is not None: + # In some workflows using coordinate descent algorithms: + # - users might provide NumPy arrays with read-only buffers + # - `joblib` might memmap arrays making their buffer read-only + # TODO: move this handling (which is currently too broad) + # closer to the actual private function which need buffers to be writable. + if not init.flags["WRITEABLE"]: + init = np.array(init) clf.coef_ = init - clf.fit(dictionary.T, X.T, check_input=check_input) + clf.fit(dictionary.T, X.T, check_input=False) new_code = clf.coef_ - elif algorithm == 'lars': + elif algorithm == "lars": try: - err_mgt = np.seterr(all='ignore') + err_mgt = np.seterr(all="ignore") # Not passing in verbose=max(0, verbose-1) because Lars.fit already # corrects the verbosity level. - lars = Lars(fit_intercept=False, verbose=verbose, normalize=False, - precompute=gram, n_nonzero_coefs=int(regularization), - fit_path=False) + lars = Lars( + fit_intercept=False, + verbose=verbose, + precompute=gram, + n_nonzero_coefs=int(regularization), + fit_path=False, + ) lars.fit(dictionary.T, X.T, Xy=cov) new_code = lars.coef_ finally: np.seterr(**err_mgt) - elif algorithm == 'threshold': - new_code = ((np.sign(cov) * - np.maximum(np.abs(cov) - regularization, 0)).T) + elif algorithm == "threshold": + new_code = (np.sign(cov) * np.maximum(np.abs(cov) - regularization, 0)).T if positive: np.clip(new_code, 0, None, out=new_code) - elif algorithm == 'omp': + elif algorithm == "omp": new_code = orthogonal_mp_gram( - Gram=gram, Xy=cov, n_nonzero_coefs=int(regularization), - tol=None, norms_squared=row_norms(X, squared=True), - copy_Xy=copy_cov).T - else: - raise ValueError('Sparse coding method must be "lasso_lars" ' - '"lasso_cd", "lasso", "threshold" or "omp", got %s.' - % algorithm) - if new_code.ndim != 2: - return new_code.reshape(n_samples, n_components) - return new_code - - + Gram=gram, + Xy=cov, + n_nonzero_coefs=int(regularization), + tol=None, + norms_squared=row_norms(X, squared=True), + copy_Xy=copy_cov, + ).T + + return new_code.reshape(n_samples, n_components) + + +@validate_params( + { + "X": ["array-like"], + "dictionary": ["array-like"], + "gram": ["array-like", None], + "cov": ["array-like", None], + "algorithm": [ + StrOptions({"lasso_lars", "lasso_cd", "lars", "omp", "threshold"}) + ], + "n_nonzero_coefs": [Interval(Integral, 1, None, closed="left"), None], + "alpha": [Interval(Real, 0, None, closed="left"), None], + "copy_cov": ["boolean"], + "init": ["array-like", None], + "max_iter": [Interval(Integral, 0, None, closed="left")], + "n_jobs": [Integral, None], + "check_input": ["boolean"], + "verbose": ["verbose"], + "positive": ["boolean"], + }, + prefer_skip_nested_validation=True, +) # XXX : could be moved to the linear_model module -@_deprecate_positional_args -def sparse_encode(X, dictionary, *, gram=None, cov=None, - algorithm='lasso_lars', n_nonzero_coefs=None, alpha=None, - copy_cov=True, init=None, max_iter=1000, n_jobs=None, - check_input=True, verbose=0, positive=False): - """Sparse coding +def sparse_encode( + X, + dictionary, + *, + gram=None, + cov=None, + algorithm="lasso_lars", + n_nonzero_coefs=None, + alpha=None, + copy_cov=True, + init=None, + max_iter=1000, + n_jobs=None, + check_input=True, + verbose=0, + positive=False, +): + """Sparse coding. Each row of the result is the solution to a sparse coding problem. The goal is to find a sparse array `code` such that:: @@ -209,18 +250,18 @@ def sparse_encode(X, dictionary, *, gram=None, cov=None, Parameters ---------- - X : ndarray of shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Data matrix. - dictionary : ndarray of shape (n_components, n_features) + dictionary : array-like of shape (n_components, n_features) The dictionary matrix against which to solve the sparse coding of the data. Some of the algorithms assume normalized rows for meaningful output. - gram : ndarray of shape (n_components, n_components), default=None + gram : array-like of shape (n_components, n_components), default=None Precomputed Gram matrix, `dictionary * dictionary'`. - cov : ndarray of shape (n_components, n_samples), default=None + cov : array-like of shape (n_components, n_samples), default=None Precomputed covariance, `dictionary' * X`. algorithm : {'lasso_lars', 'lasso_cd', 'lars', 'omp', 'threshold'}, \ @@ -286,97 +327,183 @@ def sparse_encode(X, dictionary, *, gram=None, cov=None, Returns ------- code : ndarray of shape (n_samples, n_components) - The sparse codes + The sparse codes. See Also -------- - sklearn.linear_model.lars_path - sklearn.linear_model.orthogonal_mp - sklearn.linear_model.Lasso - SparseCoder + sklearn.linear_model.lars_path : Compute Least Angle Regression or Lasso + path using LARS algorithm. + sklearn.linear_model.orthogonal_mp : Solves Orthogonal Matching Pursuit problems. + sklearn.linear_model.Lasso : Train Linear Model with L1 prior as regularizer. + SparseCoder : Find a sparse representation of data from a fixed precomputed + dictionary. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.decomposition import sparse_encode + >>> X = np.array([[-1, -1, -1], [0, 0, 3]]) + >>> dictionary = np.array( + ... [[0, 1, 0], + ... [-1, -1, 2], + ... [1, 1, 1], + ... [0, 1, 1], + ... [0, 2, 1]], + ... dtype=np.float64 + ... ) + >>> sparse_encode(X, dictionary, alpha=1e-10) + array([[ 0., 0., -1., 0., 0.], + [ 0., 1., 1., 0., 0.]]) """ if check_input: - if algorithm == 'lasso_cd': - dictionary = check_array(dictionary, order='C', dtype='float64') - X = check_array(X, order='C', dtype='float64') + if algorithm == "lasso_cd": + dictionary = check_array( + dictionary, order="C", dtype=[np.float64, np.float32] + ) + X = check_array(X, order="C", dtype=[np.float64, np.float32]) else: dictionary = check_array(dictionary) X = check_array(X) - n_samples, n_features = X.shape - n_components = dictionary.shape[0] + if dictionary.shape[1] != X.shape[1]: + raise ValueError( + "Dictionary and X have different numbers of features:" + "dictionary.shape: {} X.shape{}".format(dictionary.shape, X.shape) + ) - if gram is None and algorithm != 'threshold': - gram = np.dot(dictionary, dictionary.T) + _check_positive_coding(algorithm, positive) - if cov is None and algorithm != 'lasso_cd': - copy_cov = False - cov = np.dot(dictionary, X.T) + return _sparse_encode( + X, + dictionary, + gram=gram, + cov=cov, + algorithm=algorithm, + n_nonzero_coefs=n_nonzero_coefs, + alpha=alpha, + copy_cov=copy_cov, + init=init, + max_iter=max_iter, + n_jobs=n_jobs, + verbose=verbose, + positive=positive, + ) + + +def _sparse_encode( + X, + dictionary, + *, + gram=None, + cov=None, + algorithm="lasso_lars", + n_nonzero_coefs=None, + alpha=None, + copy_cov=True, + init=None, + max_iter=1000, + n_jobs=None, + verbose=0, + positive=False, +): + """Sparse coding without input/parameter validation.""" - if algorithm in ('lars', 'omp'): + n_samples, n_features = X.shape + n_components = dictionary.shape[0] + + if algorithm in ("lars", "omp"): regularization = n_nonzero_coefs if regularization is None: regularization = min(max(n_features / 10, 1), n_components) else: regularization = alpha if regularization is None: - regularization = 1. - - if effective_n_jobs(n_jobs) == 1 or algorithm == 'threshold': - code = _sparse_encode(X, - dictionary, gram, cov=cov, - algorithm=algorithm, - regularization=regularization, copy_cov=copy_cov, - init=init, - max_iter=max_iter, - check_input=False, - verbose=verbose, - positive=positive) + regularization = 1.0 + + if gram is None and algorithm != "threshold": + gram = np.dot(dictionary, dictionary.T) + + if cov is None and algorithm != "lasso_cd": + copy_cov = False + cov = np.dot(dictionary, X.T) + + if effective_n_jobs(n_jobs) == 1 or algorithm == "threshold": + code = _sparse_encode_precomputed( + X, + dictionary, + gram=gram, + cov=cov, + algorithm=algorithm, + regularization=regularization, + copy_cov=copy_cov, + init=init, + max_iter=max_iter, + verbose=verbose, + positive=positive, + ) return code # Enter parallel code block + n_samples = X.shape[0] + n_components = dictionary.shape[0] code = np.empty((n_samples, n_components)) slices = list(gen_even_slices(n_samples, effective_n_jobs(n_jobs))) code_views = Parallel(n_jobs=n_jobs, verbose=verbose)( - delayed(_sparse_encode)( - X[this_slice], dictionary, gram, - cov[:, this_slice] if cov is not None else None, - algorithm, - regularization=regularization, copy_cov=copy_cov, + delayed(_sparse_encode_precomputed)( + X[this_slice], + dictionary, + gram=gram, + cov=cov[:, this_slice] if cov is not None else None, + algorithm=algorithm, + regularization=regularization, + copy_cov=copy_cov, init=init[this_slice] if init is not None else None, max_iter=max_iter, - check_input=False, verbose=verbose, - positive=positive) - for this_slice in slices) + positive=positive, + ) + for this_slice in slices + ) for this_slice, this_view in zip(slices, code_views): code[this_slice] = this_view return code -def _update_dict(dictionary, Y, code, verbose=False, return_r2=False, - random_state=None, positive=False): +def _update_dict( + dictionary, + Y, + code, + A=None, + B=None, + verbose=False, + random_state=None, + positive=False, +): """Update the dense dictionary factor in place. Parameters ---------- - dictionary : ndarray of shape (n_features, n_components) + dictionary : ndarray of shape (n_components, n_features) Value of the dictionary at the previous iteration. - Y : ndarray of shape (n_features, n_samples) + Y : ndarray of shape (n_samples, n_features) Data matrix. - code : ndarray of shape (n_components, n_samples) + code : ndarray of shape (n_samples, n_components) Sparse coding of the data against which to optimize the dictionary. + A : ndarray of shape (n_components, n_components), default=None + Together with `B`, sufficient stats of the online model to update the + dictionary. + + B : ndarray of shape (n_features, n_components), default=None + Together with `A`, sufficient stats of the online model to update the + dictionary. + verbose: bool, default=False Degree of output the procedure will print. - return_r2 : bool, default=False - Whether to compute and return the residual sum of squares corresponding - to the computed solution. - random_state : int, RandomState instance or None, default=None Used for randomly initializing the dictionary. Pass an int for reproducible results across multiple function calls. @@ -386,183 +513,67 @@ def _update_dict(dictionary, Y, code, verbose=False, return_r2=False, Whether to enforce positivity when finding the dictionary. .. versionadded:: 0.20 - - Returns - ------- - dictionary : ndarray of shape (n_features, n_components) - Updated dictionary. """ - n_components = len(code) - n_features = Y.shape[0] + n_samples, n_components = code.shape random_state = check_random_state(random_state) - # Get BLAS functions - gemm, = linalg.get_blas_funcs(('gemm',), (dictionary, code, Y)) - ger, = linalg.get_blas_funcs(('ger',), (dictionary, code)) - nrm2, = linalg.get_blas_funcs(('nrm2',), (dictionary,)) - # Residuals, computed with BLAS for speed and efficiency - # R <- -1.0 * U * V^T + 1.0 * Y - # Outputs R as Fortran array for efficiency - R = gemm(-1.0, dictionary, code, 1.0, Y) - for k in range(n_components): - # R <- 1.0 * U_k * V_k^T + R - R = ger(1.0, dictionary[:, k], code[k, :], a=R, overwrite_a=True) - dictionary[:, k] = np.dot(R, code[k, :]) - if positive: - np.clip(dictionary[:, k], 0, None, out=dictionary[:, k]) - # Scale k'th atom - # (U_k * U_k) ** 0.5 - atom_norm = nrm2(dictionary[:, k]) - if atom_norm < 1e-10: - if verbose == 1: - sys.stdout.write("+") - sys.stdout.flush() - elif verbose: - print("Adding new random atom") - dictionary[:, k] = random_state.randn(n_features) - if positive: - np.clip(dictionary[:, k], 0, None, out=dictionary[:, k]) - # Setting corresponding coefs to 0 - code[k, :] = 0.0 - # (U_k * U_k) ** 0.5 - atom_norm = nrm2(dictionary[:, k]) - dictionary[:, k] /= atom_norm - else: - dictionary[:, k] /= atom_norm - # R <- -1.0 * U_k * V_k^T + R - R = ger(-1.0, dictionary[:, k], code[k, :], a=R, overwrite_a=True) - if return_r2: - R = nrm2(R) ** 2.0 - return dictionary, R - return dictionary - - -@_deprecate_positional_args -def dict_learning(X, n_components, *, alpha, max_iter=100, tol=1e-8, - method='lars', n_jobs=None, dict_init=None, code_init=None, - callback=None, verbose=False, random_state=None, - return_n_iter=False, positive_dict=False, - positive_code=False, method_max_iter=1000): - """Solves a dictionary learning matrix factorization problem. - - Finds the best dictionary and the corresponding sparse code for - approximating the data matrix X by solving:: - - (U^*, V^*) = argmin 0.5 || X - U V ||_2^2 + alpha * || U ||_1 - (U,V) - with || V_k ||_2 = 1 for all 0 <= k < n_components - - where V is the dictionary and U is the sparse code. - - Read more in the :ref:`User Guide `. - - Parameters - ---------- - X : ndarray of shape (n_samples, n_features) - Data matrix. - - n_components : int - Number of dictionary atoms to extract. - - alpha : int - Sparsity controlling parameter. - - max_iter : int, default=100 - Maximum number of iterations to perform. - - tol : float, default=1e-8 - Tolerance for the stopping condition. - - method : {'lars', 'cd'}, default='lars' - The method used: - - * `'lars'`: uses the least angle regression method to solve the lasso - problem (`linear_model.lars_path`); - * `'cd'`: uses the coordinate descent method to compute the - Lasso solution (`linear_model.Lasso`). Lars will be faster if - the estimated components are sparse. - - n_jobs : int, default=None - Number of parallel jobs to run. - ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. - ``-1`` means using all processors. See :term:`Glossary ` - for more details. - - dict_init : ndarray of shape (n_components, n_features), default=None - Initial value for the dictionary for warm restart scenarios. Only used - if `code_init` and `dict_init` are not None. - - code_init : ndarray of shape (n_samples, n_components), default=None - Initial value for the sparse code for warm restart scenarios. Only used - if `code_init` and `dict_init` are not None. - - callback : callable, default=None - Callable that gets invoked every five iterations - - verbose : bool, default=False - To control the verbosity of the procedure. - - random_state : int, RandomState instance or None, default=None - Used for randomly initializing the dictionary. Pass an int for - reproducible results across multiple function calls. - See :term:`Glossary `. - - return_n_iter : bool, default=False - Whether or not to return the number of iterations. - - positive_dict : bool, default=False - Whether to enforce positivity when finding the dictionary. - - .. versionadded:: 0.20 - - positive_code : bool, default=False - Whether to enforce positivity when finding the code. - - .. versionadded:: 0.20 - method_max_iter : int, default=1000 - Maximum number of iterations to perform. + if A is None: + A = code.T @ code + if B is None: + B = Y.T @ code - .. versionadded:: 0.22 + n_unused = 0 - Returns - ------- - code : ndarray of shape (n_samples, n_components) - The sparse code factor in the matrix factorization. - - dictionary : ndarray of shape (n_components, n_features), - The dictionary factor in the matrix factorization. - - errors : array - Vector of errors at each iteration. - - n_iter : int - Number of iterations run. Returned only if `return_n_iter` is - set to True. - - See Also - -------- - dict_learning_online - DictionaryLearning - MiniBatchDictionaryLearning - SparsePCA - MiniBatchSparsePCA - """ - if method not in ('lars', 'cd'): - raise ValueError('Coding method %r not supported as a fit algorithm.' - % method) + for k in range(n_components): + if A[k, k] > 1e-6: + # 1e-6 is arbitrary but consistent with the spams implementation + dictionary[k] += (B[:, k] - A[k] @ dictionary) / A[k, k] + else: + # kth atom is almost never used -> sample a new one from the data + newd = Y[random_state.choice(n_samples)] - _check_positive_coding(method, positive_code) + # add small noise to avoid making the sparse coding ill conditioned + noise_level = 0.01 * (newd.std() or 1) # avoid 0 std + noise = random_state.normal(0, noise_level, size=len(newd)) - method = 'lasso_' + method + dictionary[k] = newd + noise + code[:, k] = 0 + n_unused += 1 + if positive: + np.clip(dictionary[k], 0, None, out=dictionary[k]) + + # Projection on the constraint set ||V_k|| <= 1 + dictionary[k] /= max(linalg.norm(dictionary[k]), 1) + + if verbose and n_unused > 0: + print(f"{n_unused} unused atoms resampled.") + + +def _dict_learning( + X, + n_components, + *, + alpha, + max_iter, + tol, + method, + n_jobs, + dict_init, + code_init, + callback, + verbose, + random_state, + return_n_iter, + positive_dict, + positive_code, + method_max_iter, +): + """Main dictionary learning algorithm""" t0 = time.time() - # Avoid integer division problems - alpha = float(alpha) - random_state = check_random_state(random_state) - # Init the code and the dictionary with SVD of Y if code_init is not None and dict_init is not None: - code = np.array(code_init, order='F') + code = np.array(code_init, order="F") # Don't copy V, it will happen below dictionary = dict_init else: @@ -576,46 +587,61 @@ def dict_learning(X, n_components, *, alpha, max_iter=100, tol=1e-8, dictionary = dictionary[:n_components, :] else: code = np.c_[code, np.zeros((len(code), n_components - r))] - dictionary = np.r_[dictionary, - np.zeros((n_components - r, dictionary.shape[1]))] - - # Fortran-order dict, as we are going to access its row vectors - dictionary = np.array(dictionary, order='F') + dictionary = np.r_[ + dictionary, np.zeros((n_components - r, dictionary.shape[1])) + ] - residuals = 0 + # Fortran-order dict better suited for the sparse coding which is the + # bottleneck of this algorithm. + dictionary = np.asfortranarray(dictionary) errors = [] current_cost = np.nan if verbose == 1: - print('[dict_learning]', end=' ') + print("[dict_learning]", end=" ") # If max_iter is 0, number of iterations returned should be zero ii = -1 for ii in range(max_iter): - dt = (time.time() - t0) + dt = time.time() - t0 if verbose == 1: sys.stdout.write(".") sys.stdout.flush() elif verbose: - print("Iteration % 3i " - "(elapsed time: % 3is, % 4.1fmn, current cost % 7.3f)" - % (ii, dt, dt / 60, current_cost)) + print( + "Iteration % 3i (elapsed time: % 3is, % 4.1fmn, current cost % 7.3f)" + % (ii, dt, dt / 60, current_cost) + ) # Update code - code = sparse_encode(X, dictionary, algorithm=method, alpha=alpha, - init=code, n_jobs=n_jobs, positive=positive_code, - max_iter=method_max_iter, verbose=verbose) - # Update dictionary - dictionary, residuals = _update_dict(dictionary.T, X.T, code.T, - verbose=verbose, return_r2=True, - random_state=random_state, - positive=positive_dict) - dictionary = dictionary.T + code = sparse_encode( + X, + dictionary, + algorithm=method, + alpha=alpha, + init=code, + n_jobs=n_jobs, + positive=positive_code, + max_iter=method_max_iter, + verbose=verbose, + ) + + # Update dictionary in place + _update_dict( + dictionary, + X, + code, + verbose=verbose, + random_state=random_state, + positive=positive_dict, + ) # Cost function - current_cost = 0.5 * residuals + alpha * np.sum(np.abs(code)) + current_cost = 0.5 * np.sum((X - code @ dictionary) ** 2) + alpha * np.sum( + np.abs(code) + ) errors.append(current_cost) if ii > 0: @@ -637,56 +663,89 @@ def dict_learning(X, n_components, *, alpha, max_iter=100, tol=1e-8, return code, dictionary, errors -@_deprecate_positional_args -def dict_learning_online(X, n_components=2, *, alpha=1, n_iter=100, - return_code=True, dict_init=None, callback=None, - batch_size=3, verbose=False, shuffle=True, - n_jobs=None, method='lars', iter_offset=0, - random_state=None, return_inner_stats=False, - inner_stats=None, return_n_iter=False, - positive_dict=False, positive_code=False, - method_max_iter=1000): - """Solves a dictionary learning matrix factorization problem online. +@validate_params( + { + "X": ["array-like"], + "return_code": ["boolean"], + "method": [StrOptions({"cd", "lars"})], + "method_max_iter": [Interval(Integral, 0, None, closed="left")], + }, + prefer_skip_nested_validation=False, +) +def dict_learning_online( + X, + n_components=2, + *, + alpha=1, + max_iter=100, + return_code=True, + dict_init=None, + callback=None, + batch_size=256, + verbose=False, + shuffle=True, + n_jobs=None, + method="lars", + random_state=None, + positive_dict=False, + positive_code=False, + method_max_iter=1000, + tol=1e-3, + max_no_improvement=10, +): + """Solve a dictionary learning matrix factorization problem online. Finds the best dictionary and the corresponding sparse code for approximating the data matrix X by solving:: - (U^*, V^*) = argmin 0.5 || X - U V ||_2^2 + alpha * || U ||_1 + (U^*, V^*) = argmin 0.5 || X - U V ||_Fro^2 + alpha * || U ||_1,1 (U,V) with || V_k ||_2 = 1 for all 0 <= k < n_components - where V is the dictionary and U is the sparse code. This is - accomplished by repeatedly iterating over mini-batches by slicing + where V is the dictionary and U is the sparse code. ||.||_Fro stands for + the Frobenius norm and ||.||_1,1 stands for the entry-wise matrix norm + which is the sum of the absolute values of all the entries in the matrix. + This is accomplished by repeatedly iterating over mini-batches by slicing the input data. Read more in the :ref:`User Guide `. Parameters ---------- - X : ndarray of shape (n_samples, n_features) + X : array-like of shape (n_samples, n_features) Data matrix. - n_components : int, default=2 - Number of dictionary atoms to extract. + n_components : int or None, default=2 + Number of dictionary atoms to extract. If None, then ``n_components`` + is set to ``n_features``. alpha : float, default=1 Sparsity controlling parameter. - n_iter : int, default=100 - Number of mini-batch iterations to perform. + max_iter : int, default=100 + Maximum number of iterations over the complete dataset before + stopping independently of any early stopping criterion heuristics. + + .. versionadded:: 1.1 return_code : bool, default=True Whether to also return the code U or just the dictionary `V`. dict_init : ndarray of shape (n_components, n_features), default=None - Initial value for the dictionary for warm restart scenarios. + Initial values for the dictionary for warm restart scenarios. + If `None`, the initial values for the dictionary are created + with an SVD decomposition of the data via + :func:`~sklearn.utils.extmath.randomized_svd`. callback : callable, default=None - callable that gets invoked every five iterations. + A callable that gets invoked at the end of each iteration. - batch_size : int, default=3 + batch_size : int, default=256 The number of samples to take in each batch. + .. versionchanged:: 1.3 + The default value of `batch_size` changed from 3 to 256 in version 1.3. + verbose : bool, default=False To control the verbosity of the procedure. @@ -706,10 +765,6 @@ def dict_learning_online(X, n_components=2, *, alpha=1, n_iter=100, Lasso solution (`linear_model.Lasso`). Lars will be faster if the estimated components are sparse. - iter_offset : int, default=0 - Number of previous iterations completed on the dictionary used for - initialization. - random_state : int, RandomState instance or None, default=None Used for initializing the dictionary when ``dict_init`` is not specified, randomly shuffling the data when ``shuffle`` is set to @@ -717,22 +772,6 @@ def dict_learning_online(X, n_components=2, *, alpha=1, n_iter=100, results across multiple function calls. See :term:`Glossary `. - return_inner_stats : bool, default=False - Return the inner statistics A (dictionary covariance) and B - (data approximation). Useful to restart the algorithm in an - online setting. If `return_inner_stats` is `True`, `return_code` is - ignored. - - inner_stats : tuple of (A, B) ndarrays, default=None - Inner sufficient statistics that are kept by the algorithm. - Passing them at initialization is useful in online settings, to - avoid losing the history of the evolution. - `A` `(n_components, n_components)` is the dictionary covariance matrix. - `B` `(n_features, n_components)` is the data approximation matrix. - - return_n_iter : bool, default=False - Whether or not to return the number of iterations. - positive_dict : bool, default=False Whether to enforce positivity when finding the dictionary. @@ -748,6 +787,24 @@ def dict_learning_online(X, n_components=2, *, alpha=1, n_iter=100, .. versionadded:: 0.22 + tol : float, default=1e-3 + Control early stopping based on the norm of the differences in the + dictionary between 2 steps. + + To disable early stopping based on changes in the dictionary, set + `tol` to 0.0. + + .. versionadded:: 1.1 + + max_no_improvement : int, default=10 + Control early stopping based on the consecutive number of mini batches + that does not yield an improvement on the smoothed cost function. + + To disable convergence detection based on cost function, set + `max_no_improvement` to None. + + .. versionadded:: 1.1 + Returns ------- code : ndarray of shape (n_samples, n_components), @@ -762,145 +819,270 @@ def dict_learning_online(X, n_components=2, *, alpha=1, n_iter=100, See Also -------- - dict_learning - DictionaryLearning - MiniBatchDictionaryLearning - SparsePCA - MiniBatchSparsePCA - """ - if n_components is None: - n_components = X.shape[1] + dict_learning : Solve a dictionary learning matrix factorization problem. + DictionaryLearning : Find a dictionary that sparsely encodes data. + MiniBatchDictionaryLearning : A faster, less accurate, version of the dictionary + learning algorithm. + SparsePCA : Sparse Principal Components Analysis. + MiniBatchSparsePCA : Mini-batch Sparse Principal Components Analysis. - if method not in ('lars', 'cd'): - raise ValueError('Coding method not supported as a fit algorithm.') + Examples + -------- + >>> import numpy as np + >>> from sklearn.datasets import make_sparse_coded_signal + >>> from sklearn.decomposition import dict_learning_online + >>> X, _, _ = make_sparse_coded_signal( + ... n_samples=30, n_components=15, n_features=20, n_nonzero_coefs=10, + ... random_state=42, + ... ) + >>> U, V = dict_learning_online( + ... X, n_components=15, alpha=0.2, max_iter=20, batch_size=3, random_state=42 + ... ) - _check_positive_coding(method, positive_code) + We can check the level of sparsity of `U`: - method = 'lasso_' + method + >>> np.mean(U == 0) + np.float64(0.53) - t0 = time.time() - n_samples, n_features = X.shape - # Avoid integer division problems - alpha = float(alpha) - random_state = check_random_state(random_state) + We can compare the average squared euclidean norm of the reconstruction + error of the sparse coded signal relative to the squared euclidean norm of + the original signal: - # Init V with SVD of X - if dict_init is not None: - dictionary = dict_init - else: - _, S, dictionary = randomized_svd(X, n_components, - random_state=random_state) - dictionary = S[:, np.newaxis] * dictionary - r = len(dictionary) - if n_components <= r: - dictionary = dictionary[:n_components, :] + >>> X_hat = U @ V + >>> np.mean(np.sum((X_hat - X) ** 2, axis=1) / np.sum(X ** 2, axis=1)) + np.float64(0.053) + """ + transform_algorithm = "lasso_" + method + + est = MiniBatchDictionaryLearning( + n_components=n_components, + alpha=alpha, + max_iter=max_iter, + n_jobs=n_jobs, + fit_algorithm=method, + batch_size=batch_size, + shuffle=shuffle, + dict_init=dict_init, + random_state=random_state, + transform_algorithm=transform_algorithm, + transform_alpha=alpha, + positive_code=positive_code, + positive_dict=positive_dict, + transform_max_iter=method_max_iter, + verbose=verbose, + callback=callback, + tol=tol, + max_no_improvement=max_no_improvement, + ).fit(X) + + if not return_code: + return est.components_ else: - dictionary = np.r_[dictionary, - np.zeros((n_components - r, dictionary.shape[1]))] + code = est.transform(X) + return code, est.components_ + + +@validate_params( + { + "X": ["array-like"], + "method": [StrOptions({"lars", "cd"})], + "return_n_iter": ["boolean"], + "method_max_iter": [Interval(Integral, 0, None, closed="left")], + }, + prefer_skip_nested_validation=False, +) +def dict_learning( + X, + n_components, + *, + alpha, + max_iter=100, + tol=1e-8, + method="lars", + n_jobs=None, + dict_init=None, + code_init=None, + callback=None, + verbose=False, + random_state=None, + return_n_iter=False, + positive_dict=False, + positive_code=False, + method_max_iter=1000, +): + """Solve a dictionary learning matrix factorization problem. + + Finds the best dictionary and the corresponding sparse code for + approximating the data matrix X by solving:: + + (U^*, V^*) = argmin 0.5 || X - U V ||_Fro^2 + alpha * || U ||_1,1 + (U,V) + with || V_k ||_2 = 1 for all 0 <= k < n_components + + where V is the dictionary and U is the sparse code. ||.||_Fro stands for + the Frobenius norm and ||.||_1,1 stands for the entry-wise matrix norm + which is the sum of the absolute values of all the entries in the matrix. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Data matrix. + + n_components : int + Number of dictionary atoms to extract. + + alpha : int or float + Sparsity controlling parameter. + + max_iter : int, default=100 + Maximum number of iterations to perform. + + tol : float, default=1e-8 + Tolerance for the stopping condition. + + method : {'lars', 'cd'}, default='lars' + The method used: + + * `'lars'`: uses the least angle regression method to solve the lasso + problem (`linear_model.lars_path`); + * `'cd'`: uses the coordinate descent method to compute the + Lasso solution (`linear_model.Lasso`). Lars will be faster if + the estimated components are sparse. + + n_jobs : int, default=None + Number of parallel jobs to run. + ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. + ``-1`` means using all processors. See :term:`Glossary ` + for more details. + + dict_init : ndarray of shape (n_components, n_features), default=None + Initial value for the dictionary for warm restart scenarios. Only used + if `code_init` and `dict_init` are not None. + + code_init : ndarray of shape (n_samples, n_components), default=None + Initial value for the sparse code for warm restart scenarios. Only used + if `code_init` and `dict_init` are not None. + + callback : callable, default=None + Callable that gets invoked every five iterations. + + verbose : bool, default=False + To control the verbosity of the procedure. + + random_state : int, RandomState instance or None, default=None + Used for randomly initializing the dictionary. Pass an int for + reproducible results across multiple function calls. + See :term:`Glossary `. + + return_n_iter : bool, default=False + Whether or not to return the number of iterations. + + positive_dict : bool, default=False + Whether to enforce positivity when finding the dictionary. + + .. versionadded:: 0.20 + + positive_code : bool, default=False + Whether to enforce positivity when finding the code. - if verbose == 1: - print('[dict_learning]', end=' ') + .. versionadded:: 0.20 - if shuffle: - X_train = X.copy() - random_state.shuffle(X_train) - else: - X_train = X + method_max_iter : int, default=1000 + Maximum number of iterations to perform. - dictionary = check_array(dictionary.T, order='F', dtype=np.float64, - copy=False) - dictionary = np.require(dictionary, requirements='W') + .. versionadded:: 0.22 - X_train = check_array(X_train, order='C', dtype=np.float64, copy=False) + Returns + ------- + code : ndarray of shape (n_samples, n_components) + The sparse code factor in the matrix factorization. - batches = gen_batches(n_samples, batch_size) - batches = itertools.cycle(batches) + dictionary : ndarray of shape (n_components, n_features), + The dictionary factor in the matrix factorization. - # The covariance of the dictionary - if inner_stats is None: - A = np.zeros((n_components, n_components)) - # The data approximation - B = np.zeros((n_features, n_components)) - else: - A = inner_stats[0].copy() - B = inner_stats[1].copy() + errors : array + Vector of errors at each iteration. - # If n_iter is zero, we need to return zero. - ii = iter_offset - 1 + n_iter : int + Number of iterations run. Returned only if `return_n_iter` is + set to True. - for ii, batch in zip(range(iter_offset, iter_offset + n_iter), batches): - this_X = X_train[batch] - dt = (time.time() - t0) - if verbose == 1: - sys.stdout.write(".") - sys.stdout.flush() - elif verbose: - if verbose > 10 or ii % ceil(100. / verbose) == 0: - print("Iteration % 3i (elapsed time: % 3is, % 4.1fmn)" - % (ii, dt, dt / 60)) - - this_code = sparse_encode(this_X, dictionary.T, algorithm=method, - alpha=alpha, n_jobs=n_jobs, - check_input=False, - positive=positive_code, - max_iter=method_max_iter, verbose=verbose).T - - # Update the auxiliary variables - if ii < batch_size - 1: - theta = float((ii + 1) * batch_size) - else: - theta = float(batch_size ** 2 + ii + 1 - batch_size) - beta = (theta + 1 - batch_size) / (theta + 1) + See Also + -------- + dict_learning_online : Solve a dictionary learning matrix factorization + problem online. + DictionaryLearning : Find a dictionary that sparsely encodes data. + MiniBatchDictionaryLearning : A faster, less accurate version + of the dictionary learning algorithm. + SparsePCA : Sparse Principal Components Analysis. + MiniBatchSparsePCA : Mini-batch Sparse Principal Components Analysis. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.datasets import make_sparse_coded_signal + >>> from sklearn.decomposition import dict_learning + >>> X, _, _ = make_sparse_coded_signal( + ... n_samples=30, n_components=15, n_features=20, n_nonzero_coefs=10, + ... random_state=42, + ... ) + >>> U, V, errors = dict_learning(X, n_components=15, alpha=0.1, random_state=42) - A *= beta - A += np.dot(this_code, this_code.T) - B *= beta - B += np.dot(this_X.T, this_code.T) + We can check the level of sparsity of `U`: - # Update dictionary - dictionary = _update_dict(dictionary, B, A, verbose=verbose, - random_state=random_state, - positive=positive_dict) - # XXX: Can the residuals be of any use? - - # Maybe we need a stopping criteria based on the amount of - # modification in the dictionary - if callback is not None: - callback(locals()) + >>> np.mean(U == 0) + np.float64(0.62) - if return_inner_stats: - if return_n_iter: - return dictionary.T, (A, B), ii - iter_offset + 1 - else: - return dictionary.T, (A, B) - if return_code: - if verbose > 1: - print('Learning code...', end=' ') - elif verbose == 1: - print('|', end=' ') - code = sparse_encode(X, dictionary.T, algorithm=method, alpha=alpha, - n_jobs=n_jobs, check_input=False, - positive=positive_code, max_iter=method_max_iter, - verbose=verbose) - if verbose > 1: - dt = (time.time() - t0) - print('done (total time: % 3is, % 4.1fmn)' % (dt, dt / 60)) - if return_n_iter: - return code, dictionary.T, ii - iter_offset + 1 - else: - return code, dictionary.T + We can compare the average squared euclidean norm of the reconstruction + error of the sparse coded signal relative to the squared euclidean norm of + the original signal: + >>> X_hat = U @ V + >>> np.mean(np.sum((X_hat - X) ** 2, axis=1) / np.sum(X ** 2, axis=1)) + np.float64(0.0192) + """ + estimator = DictionaryLearning( + n_components=n_components, + alpha=alpha, + max_iter=max_iter, + tol=tol, + fit_algorithm=method, + n_jobs=n_jobs, + dict_init=dict_init, + callback=callback, + code_init=code_init, + verbose=verbose, + random_state=random_state, + positive_code=positive_code, + positive_dict=positive_dict, + transform_max_iter=method_max_iter, + ).set_output(transform="default") + code = estimator.fit_transform(X) if return_n_iter: - return dictionary.T, ii - iter_offset + 1 - else: - return dictionary.T + return ( + code, + estimator.components_, + estimator.error_, + estimator.n_iter_, + ) + return code, estimator.components_, estimator.error_ -class _BaseSparseCoding(TransformerMixin): +class _BaseSparseCoding(ClassNamePrefixFeaturesOutMixin, TransformerMixin): """Base class from SparseCoder and DictionaryLearning algorithms.""" - def __init__(self, transform_algorithm, transform_n_nonzero_coefs, - transform_alpha, split_sign, n_jobs, positive_code, - transform_max_iter): + + def __init__( + self, + transform_algorithm, + transform_n_nonzero_coefs, + transform_alpha, + split_sign, + n_jobs, + positive_code, + transform_max_iter, + ): self.transform_algorithm = transform_algorithm self.transform_n_nonzero_coefs = transform_n_nonzero_coefs self.transform_alpha = transform_alpha @@ -910,26 +1092,25 @@ def __init__(self, transform_algorithm, transform_n_nonzero_coefs, self.positive_code = positive_code def _transform(self, X, dictionary): - """Private method allowing to accomodate both DictionaryLearning and + """Private method allowing to accommodate both DictionaryLearning and SparseCoder.""" - X = self._validate_data(X, reset=False) - - # transform_alpha has to be changed in _transform - # this is done for consistency with the value of alpha - if (hasattr(self, "alpha") and self.alpha != 1. and - self.transform_alpha is None): - warnings.warn("By default transform_alpha will be equal to" - "alpha instead of 1.0 starting from version 1.2", - FutureWarning) - transform_alpha = 1. # TODO change to self.alpha in 1.2 + X = validate_data(self, X, reset=False) + + if hasattr(self, "alpha") and self.transform_alpha is None: + transform_alpha = self.alpha else: transform_alpha = self.transform_alpha code = sparse_encode( - X, dictionary, algorithm=self.transform_algorithm, + X, + dictionary, + algorithm=self.transform_algorithm, n_nonzero_coefs=self.transform_n_nonzero_coefs, - alpha=transform_alpha, max_iter=self.transform_max_iter, - n_jobs=self.n_jobs, positive=self.positive_code) + alpha=transform_alpha, + max_iter=self.transform_max_iter, + n_jobs=self.n_jobs, + positive=self.positive_code, + ) if self.split_sign: # feature vector is split into a positive and negative side @@ -961,9 +1142,47 @@ def transform(self, X): check_is_fitted(self) return self._transform(X, self.components_) + def _inverse_transform(self, code, dictionary): + """Private method allowing to accommodate both DictionaryLearning and + SparseCoder.""" + code = check_array(code) + # compute number of expected features in code + expected_n_components = dictionary.shape[0] + if self.split_sign: + expected_n_components += expected_n_components + if not code.shape[1] == expected_n_components: + raise ValueError( + "The number of components in the code is different from the " + "number of components in the dictionary." + f"Expected {expected_n_components}, got {code.shape[1]}." + ) + if self.split_sign: + n_samples, n_features = code.shape + n_features //= 2 + code = code[:, :n_features] - code[:, n_features:] + + return code @ dictionary + + def inverse_transform(self, X): + """Transform data back to its original space. + + Parameters + ---------- + X : array-like of shape (n_samples, n_components) + Data to be transformed back. Must have the same number of + components as the data used to train the model. + + Returns + ------- + X_original : ndarray of shape (n_samples, n_features) + Transformed data. + """ + check_is_fitted(self) + return self._inverse_transform(X, self.components_) + class SparseCoder(_BaseSparseCoding, BaseEstimator): - """Sparse coding + """Sparse coding. Finds a sparse representation of data against a fixed, precomputed dictionary. @@ -1036,12 +1255,29 @@ class SparseCoder(_BaseSparseCoding, BaseEstimator): Attributes ---------- - components_ : ndarray of shape (n_components, n_features) - The unchanged dictionary atoms. + n_components_ : int + Number of atoms. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 - .. deprecated:: 0.24 - This attribute is deprecated in 0.24 and will be removed in - 1.1 (renaming of 0.26). Use `dictionary` instead. + See Also + -------- + DictionaryLearning : Find a dictionary that sparsely encodes data. + MiniBatchDictionaryLearning : A faster, less accurate, version of the + dictionary learning algorithm. + MiniBatchSparsePCA : Mini-batch Sparse Principal Components Analysis. + SparsePCA : Sparse Principal Components Analysis. + sparse_encode : Sparse coding where each row of the result is the solution + to a sparse coding problem. Examples -------- @@ -1063,26 +1299,28 @@ class SparseCoder(_BaseSparseCoding, BaseEstimator): >>> coder.transform(X) array([[ 0., 0., -1., 0., 0.], [ 0., 1., 1., 0., 0.]]) - - See Also - -------- - DictionaryLearning - MiniBatchDictionaryLearning - SparsePCA - MiniBatchSparsePCA - sparse_encode """ - _required_parameters = ["dictionary"] - @_deprecate_positional_args - def __init__(self, dictionary, *, transform_algorithm='omp', - transform_n_nonzero_coefs=None, transform_alpha=None, - split_sign=False, n_jobs=None, positive_code=False, - transform_max_iter=1000): + def __init__( + self, + dictionary, + *, + transform_algorithm="omp", + transform_n_nonzero_coefs=None, + transform_alpha=None, + split_sign=False, + n_jobs=None, + positive_code=False, + transform_max_iter=1000, + ): super().__init__( - transform_algorithm, transform_n_nonzero_coefs, - transform_alpha, split_sign, n_jobs, positive_code, - transform_max_iter + transform_algorithm, + transform_n_nonzero_coefs, + transform_alpha, + split_sign, + n_jobs, + positive_code, + transform_max_iter, ) self.dictionary = dictionary @@ -1095,22 +1333,18 @@ def fit(self, X, y=None): Parameters ---------- X : Ignored + Not used, present for API consistency by convention. y : Ignored + Not used, present for API consistency by convention. Returns ------- self : object + Returns the instance itself. """ return self - @deprecated("The attribute 'components_' is deprecated " # type: ignore - "in 0.24 and will be removed in 1.1 (renaming of 0.26). Use " - "the 'dictionary' instead.") - @property - def components_(self): - return self.dictionary - def transform(self, X, y=None): """Encode the data as a sparse combination of the dictionary atoms. @@ -1120,10 +1354,11 @@ def transform(self, X, y=None): Parameters ---------- X : ndarray of shape (n_samples, n_features) - Test data to be transformed, must have the same number of - features as the data used to train the model. + Training vector, where `n_samples` is the number of samples + and `n_features` is the number of features. y : Ignored + Not used, present for API consistency by convention. Returns ------- @@ -1132,36 +1367,67 @@ def transform(self, X, y=None): """ return super()._transform(X, self.dictionary) - def _more_tags(self): - return {"requires_fit": False} + def inverse_transform(self, X): + """Transform data back to its original space. + + Parameters + ---------- + X : array-like of shape (n_samples, n_components) + Data to be transformed back. Must have the same number of + components as the data used to train the model. + + Returns + ------- + X_original : ndarray of shape (n_samples, n_features) + Transformed data. + """ + return self._inverse_transform(X, self.dictionary) + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.requires_fit = False + tags.transformer_tags.preserves_dtype = ["float64", "float32"] + return tags @property def n_components_(self): + """Number of atoms.""" return self.dictionary.shape[0] @property def n_features_in_(self): + """Number of features seen during `fit`.""" return self.dictionary.shape[1] + @property + def _n_features_out(self): + """Number of transformed output features.""" + return self.n_components_ + class DictionaryLearning(_BaseSparseCoding, BaseEstimator): - """Dictionary learning + """Dictionary learning. - Finds a dictionary (a set of atoms) that can best be used to represent data - using a sparse code. + Finds a dictionary (a set of atoms) that performs well at sparsely + encoding the fitted data. Solves the optimization problem:: - (U^*,V^*) = argmin 0.5 || X - U V ||_2^2 + alpha * || U ||_1 + (U^*,V^*) = argmin 0.5 || X - U V ||_Fro^2 + alpha * || U ||_1,1 (U,V) - with || V_k ||_2 = 1 for all 0 <= k < n_components + with || V_k ||_2 <= 1 for all 0 <= k < n_components + + ||.||_Fro stands for the Frobenius norm and ||.||_1,1 stands for + the entry-wise matrix norm which is the sum of the absolute values + of all the entries in the matrix. Read more in the :ref:`User Guide `. Parameters ---------- - n_components : int, default=n_features - Number of dictionary elements to extract. + n_components : int, default=None + Number of dictionary elements to extract. If None, then ``n_components`` + is set to ``n_features``. alpha : float, default=1.0 Sparsity controlling parameter. @@ -1213,6 +1479,9 @@ class DictionaryLearning(_BaseSparseCoding, BaseEstimator): threshold below which coefficients will be squashed to zero. If `None`, defaults to `alpha`. + .. versionchanged:: 1.2 + When None, default value changed from 1.0 to `alpha`. + n_jobs : int or None, default=None Number of parallel jobs to run. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. @@ -1227,6 +1496,11 @@ class DictionaryLearning(_BaseSparseCoding, BaseEstimator): Initial values for the dictionary, for warm restart. Only used if `code_init` and `dict_init` are not None. + callback : callable, default=None + Callable that gets invoked every five iterations. + + .. versionadded:: 1.3 + verbose : bool, default=False To control the verbosity of the procedure. @@ -1248,7 +1522,7 @@ class DictionaryLearning(_BaseSparseCoding, BaseEstimator): .. versionadded:: 0.20 positive_dict : bool, default=False - Whether to enforce positivity when finding the dictionary + Whether to enforce positivity when finding the dictionary. .. versionadded:: 0.20 @@ -1266,27 +1540,54 @@ class DictionaryLearning(_BaseSparseCoding, BaseEstimator): error_ : array vector of errors at each iteration + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + n_iter_ : int Number of iterations run. + See Also + -------- + MiniBatchDictionaryLearning: A faster, less accurate, version of the + dictionary learning algorithm. + MiniBatchSparsePCA : Mini-batch Sparse Principal Components Analysis. + SparseCoder : Find a sparse representation of data from a fixed, + precomputed dictionary. + SparsePCA : Sparse Principal Components Analysis. + + References + ---------- + + J. Mairal, F. Bach, J. Ponce, G. Sapiro, 2009: Online dictionary learning + for sparse coding (https://www.di.ens.fr/~fbach/mairal_icml09.pdf) + Examples -------- >>> import numpy as np >>> from sklearn.datasets import make_sparse_coded_signal >>> from sklearn.decomposition import DictionaryLearning >>> X, dictionary, code = make_sparse_coded_signal( - ... n_samples=100, n_components=15, n_features=20, n_nonzero_coefs=10, + ... n_samples=30, n_components=15, n_features=20, n_nonzero_coefs=10, ... random_state=42, ... ) >>> dict_learner = DictionaryLearning( - ... n_components=15, transform_algorithm='lasso_lars', random_state=42, + ... n_components=15, transform_algorithm='lasso_lars', transform_alpha=0.1, + ... random_state=42, ... ) - >>> X_transformed = dict_learner.fit_transform(X) + >>> X_transformed = dict_learner.fit(X).transform(X) We can check the level of sparsity of `X_transformed`: >>> np.mean(X_transformed == 0) - 0.88... + np.float64(0.527) We can compare the average squared euclidean norm of the reconstruction error of the sparse coded signal relative to the squared euclidean norm of @@ -1294,34 +1595,62 @@ class DictionaryLearning(_BaseSparseCoding, BaseEstimator): >>> X_hat = X_transformed @ dict_learner.components_ >>> np.mean(np.sum((X_hat - X) ** 2, axis=1) / np.sum(X ** 2, axis=1)) - 0.07... - - Notes - ----- - **References:** - - J. Mairal, F. Bach, J. Ponce, G. Sapiro, 2009: Online dictionary learning - for sparse coding (https://www.di.ens.fr/sierra/pdfs/icml09.pdf) - - See Also - -------- - SparseCoder - MiniBatchDictionaryLearning - SparsePCA - MiniBatchSparsePCA + np.float64(0.056) """ - @_deprecate_positional_args - def __init__(self, n_components=None, *, alpha=1, max_iter=1000, tol=1e-8, - fit_algorithm='lars', transform_algorithm='omp', - transform_n_nonzero_coefs=None, transform_alpha=None, - n_jobs=None, code_init=None, dict_init=None, verbose=False, - split_sign=False, random_state=None, positive_code=False, - positive_dict=False, transform_max_iter=1000): + _parameter_constraints: dict = { + "n_components": [Interval(Integral, 1, None, closed="left"), None], + "alpha": [Interval(Real, 0, None, closed="left")], + "max_iter": [Interval(Integral, 0, None, closed="left")], + "tol": [Interval(Real, 0, None, closed="left")], + "fit_algorithm": [StrOptions({"lars", "cd"})], + "transform_algorithm": [ + StrOptions({"lasso_lars", "lasso_cd", "lars", "omp", "threshold"}) + ], + "transform_n_nonzero_coefs": [Interval(Integral, 1, None, closed="left"), None], + "transform_alpha": [Interval(Real, 0, None, closed="left"), None], + "n_jobs": [Integral, None], + "code_init": [np.ndarray, None], + "dict_init": [np.ndarray, None], + "callback": [callable, None], + "verbose": ["verbose"], + "split_sign": ["boolean"], + "random_state": ["random_state"], + "positive_code": ["boolean"], + "positive_dict": ["boolean"], + "transform_max_iter": [Interval(Integral, 0, None, closed="left")], + } + + def __init__( + self, + n_components=None, + *, + alpha=1, + max_iter=1000, + tol=1e-8, + fit_algorithm="lars", + transform_algorithm="omp", + transform_n_nonzero_coefs=None, + transform_alpha=None, + n_jobs=None, + code_init=None, + dict_init=None, + callback=None, + verbose=False, + split_sign=False, + random_state=None, + positive_code=False, + positive_dict=False, + transform_max_iter=1000, + ): super().__init__( - transform_algorithm, transform_n_nonzero_coefs, - transform_alpha, split_sign, n_jobs, positive_code, - transform_max_iter + transform_algorithm, + transform_n_nonzero_coefs, + transform_alpha, + split_sign, + n_jobs, + positive_code, + transform_max_iter, ) self.n_components = n_components self.alpha = alpha @@ -1330,6 +1659,7 @@ def __init__(self, n_components=None, *, alpha=1, max_iter=1000, tol=1e-8, self.fit_algorithm = fit_algorithm self.code_init = code_init self.dict_init = dict_init + self.callback = callback self.verbose = verbose self.random_state = random_state self.positive_dict = positive_dict @@ -1340,52 +1670,99 @@ def fit(self, X, y=None): Parameters ---------- X : array-like of shape (n_samples, n_features) - Training vector, where `n_samples` in the number of samples + Training vector, where `n_samples` is the number of samples and `n_features` is the number of features. y : Ignored + Not used, present for API consistency by convention. Returns ------- self : object - Returns the object itself. + Returns the instance itself. + """ + self.fit_transform(X) + return self + + @_fit_context(prefer_skip_nested_validation=True) + def fit_transform(self, X, y=None): + """Fit the model from data in X and return the transformed data. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training vector, where `n_samples` is the number of samples + and `n_features` is the number of features. + + y : Ignored + Not used, present for API consistency by convention. + + Returns + ------- + V : ndarray of shape (n_samples, n_components) + Transformed data. """ + _check_positive_coding(method=self.fit_algorithm, positive=self.positive_code) + + method = "lasso_" + self.fit_algorithm + random_state = check_random_state(self.random_state) - X = self._validate_data(X) + X = validate_data(self, X) + if self.n_components is None: n_components = X.shape[1] else: n_components = self.n_components - V, U, E, self.n_iter_ = dict_learning( - X, n_components, alpha=self.alpha, - tol=self.tol, max_iter=self.max_iter, - method=self.fit_algorithm, + V, U, E, self.n_iter_ = _dict_learning( + X, + n_components, + alpha=self.alpha, + tol=self.tol, + max_iter=self.max_iter, + method=method, method_max_iter=self.transform_max_iter, n_jobs=self.n_jobs, code_init=self.code_init, dict_init=self.dict_init, + callback=self.callback, verbose=self.verbose, random_state=random_state, return_n_iter=True, positive_dict=self.positive_dict, - positive_code=self.positive_code) + positive_code=self.positive_code, + ) self.components_ = U self.error_ = E - return self + + return V + + @property + def _n_features_out(self): + """Number of transformed output features.""" + return self.components_.shape[0] + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.transformer_tags.preserves_dtype = ["float64", "float32"] + return tags class MiniBatchDictionaryLearning(_BaseSparseCoding, BaseEstimator): - """Mini-batch dictionary learning + """Mini-batch dictionary learning. - Finds a dictionary (a set of atoms) that can best be used to represent data - using a sparse code. + Finds a dictionary (a set of atoms) that performs well at sparsely + encoding the fitted data. Solves the optimization problem:: - (U^*,V^*) = argmin 0.5 || X - U V ||_2^2 + alpha * || U ||_1 + (U^*,V^*) = argmin 0.5 || X - U V ||_Fro^2 + alpha * || U ||_1,1 (U,V) - with || V_k ||_2 = 1 for all 0 <= k < n_components + with || V_k ||_2 <= 1 for all 0 <= k < n_components + + ||.||_Fro stands for the Frobenius norm and ||.||_1,1 stands for + the entry-wise matrix norm which is the sum of the absolute values + of all the entries in the matrix. Read more in the :ref:`User Guide `. @@ -1397,8 +1774,11 @@ class MiniBatchDictionaryLearning(_BaseSparseCoding, BaseEstimator): alpha : float, default=1 Sparsity controlling parameter. - n_iter : int, default=1000 - Total number of iterations to perform. + max_iter : int, default=1_000 + Maximum number of iterations over the complete dataset before + stopping independently of any early stopping criterion heuristics. + + .. versionadded:: 1.1 fit_algorithm : {'lars', 'cd'}, default='lars' The algorithm used: @@ -1415,14 +1795,17 @@ class MiniBatchDictionaryLearning(_BaseSparseCoding, BaseEstimator): ``-1`` means using all processors. See :term:`Glossary ` for more details. - batch_size : int, default=3 + batch_size : int, default=256 Number of samples in each mini-batch. + .. versionchanged:: 1.3 + The default value of `batch_size` changed from 3 to 256 in version 1.3. + shuffle : bool, default=True Whether to shuffle the samples before forming batches. dict_init : ndarray of shape (n_components, n_features), default=None - initial value of the dictionary for warm restart scenarios + Initial value of the dictionary for warm restart scenarios. transform_algorithm : {'lasso_lars', 'lasso_cd', 'lars', 'omp', \ 'threshold'}, default='omp' @@ -1452,7 +1835,10 @@ class MiniBatchDictionaryLearning(_BaseSparseCoding, BaseEstimator): threshold below which coefficients will be squashed to zero. If `None`, defaults to `alpha`. - verbose : bool, default=False + .. versionchanged:: 1.2 + When None, default value changed from 1.0 to `alpha`. + + verbose : bool or int, default=False To control the verbosity of the procedure. split_sign : bool, default=False @@ -1483,29 +1869,66 @@ class MiniBatchDictionaryLearning(_BaseSparseCoding, BaseEstimator): .. versionadded:: 0.22 + callback : callable, default=None + A callable that gets invoked at the end of each iteration. + + .. versionadded:: 1.1 + + tol : float, default=1e-3 + Control early stopping based on the norm of the differences in the + dictionary between 2 steps. + + To disable early stopping based on changes in the dictionary, set + `tol` to 0.0. + + .. versionadded:: 1.1 + + max_no_improvement : int, default=10 + Control early stopping based on the consecutive number of mini batches + that does not yield an improvement on the smoothed cost function. + + To disable convergence detection based on cost function, set + `max_no_improvement` to None. + + .. versionadded:: 1.1 + Attributes ---------- components_ : ndarray of shape (n_components, n_features) Components extracted from the data. - inner_stats_ : tuple of (A, B) ndarrays - Internal sufficient statistics that are kept by the algorithm. - Keeping them is useful in online settings, to avoid losing the - history of the evolution, but they shouldn't have any use for the - end user. - `A` `(n_components, n_components)` is the dictionary covariance matrix. - `B` `(n_features, n_components)` is the data approximation matrix. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 n_iter_ : int - Number of iterations run. + Number of iterations over the full dataset. + + n_steps_ : int + Number of mini-batches processed. + + .. versionadded:: 1.1 + + See Also + -------- + DictionaryLearning : Find a dictionary that sparsely encodes data. + MiniBatchSparsePCA : Mini-batch Sparse Principal Components Analysis. + SparseCoder : Find a sparse representation of data from a fixed, + precomputed dictionary. + SparsePCA : Sparse Principal Components Analysis. - iter_offset_ : int - The number of iteration on data batches that has been - performed before. + References + ---------- - random_state_ : RandomState instance - RandomState instance that is generated either from a seed, the random - number generattor or by `np.random`. + J. Mairal, F. Bach, J. Ponce, G. Sapiro, 2009: Online dictionary learning + for sparse coding (https://www.di.ens.fr/~fbach/mairal_icml09.pdf) Examples -------- @@ -1513,17 +1936,17 @@ class MiniBatchDictionaryLearning(_BaseSparseCoding, BaseEstimator): >>> from sklearn.datasets import make_sparse_coded_signal >>> from sklearn.decomposition import MiniBatchDictionaryLearning >>> X, dictionary, code = make_sparse_coded_signal( - ... n_samples=100, n_components=15, n_features=20, n_nonzero_coefs=10, + ... n_samples=30, n_components=15, n_features=20, n_nonzero_coefs=10, ... random_state=42) >>> dict_learner = MiniBatchDictionaryLearning( - ... n_components=15, transform_algorithm='lasso_lars', random_state=42, - ... ) + ... n_components=15, batch_size=3, transform_algorithm='lasso_lars', + ... transform_alpha=0.1, max_iter=20, random_state=42) >>> X_transformed = dict_learner.fit_transform(X) We can check the level of sparsity of `X_transformed`: - >>> np.mean(X_transformed == 0) - 0.87... + >>> np.mean(X_transformed == 0) > 0.5 + np.True_ We can compare the average squared euclidean norm of the reconstruction error of the sparse coded signal relative to the squared euclidean norm of @@ -1531,39 +1954,70 @@ class MiniBatchDictionaryLearning(_BaseSparseCoding, BaseEstimator): >>> X_hat = X_transformed @ dict_learner.components_ >>> np.mean(np.sum((X_hat - X) ** 2, axis=1) / np.sum(X ** 2, axis=1)) - 0.10... - - Notes - ----- - **References:** - - J. Mairal, F. Bach, J. Ponce, G. Sapiro, 2009: Online dictionary learning - for sparse coding (https://www.di.ens.fr/sierra/pdfs/icml09.pdf) - - See Also - -------- - SparseCoder - DictionaryLearning - SparsePCA - MiniBatchSparsePCA - + np.float64(0.052) """ - @_deprecate_positional_args - def __init__(self, n_components=None, *, alpha=1, n_iter=1000, - fit_algorithm='lars', n_jobs=None, batch_size=3, shuffle=True, - dict_init=None, transform_algorithm='omp', - transform_n_nonzero_coefs=None, transform_alpha=None, - verbose=False, split_sign=False, random_state=None, - positive_code=False, positive_dict=False, - transform_max_iter=1000): + _parameter_constraints: dict = { + "n_components": [Interval(Integral, 1, None, closed="left"), None], + "alpha": [Interval(Real, 0, None, closed="left")], + "max_iter": [Interval(Integral, 0, None, closed="left")], + "fit_algorithm": [StrOptions({"cd", "lars"})], + "n_jobs": [None, Integral], + "batch_size": [Interval(Integral, 1, None, closed="left")], + "shuffle": ["boolean"], + "dict_init": [None, np.ndarray], + "transform_algorithm": [ + StrOptions({"lasso_lars", "lasso_cd", "lars", "omp", "threshold"}) + ], + "transform_n_nonzero_coefs": [Interval(Integral, 1, None, closed="left"), None], + "transform_alpha": [Interval(Real, 0, None, closed="left"), None], + "verbose": ["verbose"], + "split_sign": ["boolean"], + "random_state": ["random_state"], + "positive_code": ["boolean"], + "positive_dict": ["boolean"], + "transform_max_iter": [Interval(Integral, 0, None, closed="left")], + "callback": [None, callable], + "tol": [Interval(Real, 0, None, closed="left")], + "max_no_improvement": [Interval(Integral, 0, None, closed="left"), None], + } + + def __init__( + self, + n_components=None, + *, + alpha=1, + max_iter=1_000, + fit_algorithm="lars", + n_jobs=None, + batch_size=256, + shuffle=True, + dict_init=None, + transform_algorithm="omp", + transform_n_nonzero_coefs=None, + transform_alpha=None, + verbose=False, + split_sign=False, + random_state=None, + positive_code=False, + positive_dict=False, + transform_max_iter=1000, + callback=None, + tol=1e-3, + max_no_improvement=10, + ): super().__init__( - transform_algorithm, transform_n_nonzero_coefs, transform_alpha, - split_sign, n_jobs, positive_code, transform_max_iter + transform_algorithm, + transform_n_nonzero_coefs, + transform_alpha, + split_sign, + n_jobs, + positive_code, + transform_max_iter, ) self.n_components = n_components self.alpha = alpha - self.n_iter = n_iter + self.max_iter = max_iter self.fit_algorithm = fit_algorithm self.dict_init = dict_init self.verbose = verbose @@ -1572,93 +2026,304 @@ def __init__(self, n_components=None, *, alpha=1, n_iter=1000, self.split_sign = split_sign self.random_state = random_state self.positive_dict = positive_dict + self.callback = callback + self.max_no_improvement = max_no_improvement + self.tol = tol + + def _check_params(self, X): + # n_components + self._n_components = self.n_components + if self._n_components is None: + self._n_components = X.shape[1] + + # fit_algorithm + _check_positive_coding(self.fit_algorithm, self.positive_code) + self._fit_algorithm = "lasso_" + self.fit_algorithm + + # batch_size + self._batch_size = min(self.batch_size, X.shape[0]) + + def _initialize_dict(self, X, random_state): + """Initialization of the dictionary.""" + if self.dict_init is not None: + dictionary = self.dict_init + else: + # Init V with SVD of X + _, S, dictionary = _randomized_svd( + X, self._n_components, random_state=random_state + ) + dictionary = S[:, np.newaxis] * dictionary + + if self._n_components <= len(dictionary): + dictionary = dictionary[: self._n_components, :] + else: + dictionary = np.concatenate( + ( + dictionary, + np.zeros( + (self._n_components - len(dictionary), dictionary.shape[1]), + dtype=dictionary.dtype, + ), + ) + ) + + dictionary = check_array(dictionary, order="F", dtype=X.dtype, copy=False) + dictionary = np.require(dictionary, requirements="W") + return dictionary + + def _update_inner_stats(self, X, code, batch_size, step): + """Update the inner stats inplace.""" + if step < batch_size - 1: + theta = (step + 1) * batch_size + else: + theta = batch_size**2 + step + 1 - batch_size + beta = (theta + 1 - batch_size) / (theta + 1) + + self._A *= beta + self._A += code.T @ code / batch_size + self._B *= beta + self._B += X.T @ code / batch_size + + def _minibatch_step(self, X, dictionary, random_state, step): + """Perform the update on the dictionary for one minibatch.""" + batch_size = X.shape[0] + + # Compute code for this batch + code = _sparse_encode( + X, + dictionary, + algorithm=self._fit_algorithm, + alpha=self.alpha, + n_jobs=self.n_jobs, + positive=self.positive_code, + max_iter=self.transform_max_iter, + verbose=self.verbose, + ) + + batch_cost = ( + 0.5 * ((X - code @ dictionary) ** 2).sum() + + self.alpha * np.sum(np.abs(code)) + ) / batch_size + + # Update inner stats + self._update_inner_stats(X, code, batch_size, step) + + # Update dictionary + _update_dict( + dictionary, + X, + code, + self._A, + self._B, + verbose=self.verbose, + random_state=random_state, + positive=self.positive_dict, + ) + + return batch_cost + + def _check_convergence( + self, X, batch_cost, new_dict, old_dict, n_samples, step, n_steps + ): + """Helper function to encapsulate the early stopping logic. + + Early stopping is based on two factors: + - A small change of the dictionary between two minibatch updates. This is + controlled by the tol parameter. + - No more improvement on a smoothed estimate of the objective function for a + a certain number of consecutive minibatch updates. This is controlled by + the max_no_improvement parameter. + """ + batch_size = X.shape[0] + + # counts steps starting from 1 for user friendly verbose mode. + step = step + 1 + + # Ignore 100 first steps or 1 epoch to avoid initializing the ewa_cost with a + # too bad value + if step <= min(100, n_samples / batch_size): + if self.verbose: + print(f"Minibatch step {step}/{n_steps}: mean batch cost: {batch_cost}") + return False + + # Compute an Exponentially Weighted Average of the cost function to + # monitor the convergence while discarding minibatch-local stochastic + # variability: https://en.wikipedia.org/wiki/Moving_average + if self._ewa_cost is None: + self._ewa_cost = batch_cost + else: + alpha = batch_size / (n_samples + 1) + alpha = min(alpha, 1) + self._ewa_cost = self._ewa_cost * (1 - alpha) + batch_cost * alpha + + if self.verbose: + print( + f"Minibatch step {step}/{n_steps}: mean batch cost: " + f"{batch_cost}, ewa cost: {self._ewa_cost}" + ) + + # Early stopping based on change of dictionary + dict_diff = linalg.norm(new_dict - old_dict) / self._n_components + if self.tol > 0 and dict_diff <= self.tol: + if self.verbose: + print(f"Converged (small dictionary change) at step {step}/{n_steps}") + return True + + # Early stopping heuristic due to lack of improvement on smoothed + # cost function + if self._ewa_cost_min is None or self._ewa_cost < self._ewa_cost_min: + self._no_improvement = 0 + self._ewa_cost_min = self._ewa_cost + else: + self._no_improvement += 1 + + if ( + self.max_no_improvement is not None + and self._no_improvement >= self.max_no_improvement + ): + if self.verbose: + print( + "Converged (lack of improvement in objective function) " + f"at step {step}/{n_steps}" + ) + return True + + return False + + @_fit_context(prefer_skip_nested_validation=True) def fit(self, X, y=None): """Fit the model from data in X. Parameters ---------- X : array-like of shape (n_samples, n_features) - Training vector, where n_samples in the number of samples - and n_features is the number of features. + Training vector, where `n_samples` is the number of samples + and `n_features` is the number of features. y : Ignored + Not used, present for API consistency by convention. Returns ------- self : object Returns the instance itself. """ - random_state = check_random_state(self.random_state) - X = self._validate_data(X) + X = validate_data( + self, X, dtype=[np.float64, np.float32], order="C", copy=False + ) + + self._check_params(X) + self._random_state = check_random_state(self.random_state) + + dictionary = self._initialize_dict(X, self._random_state) + old_dict = dictionary.copy() + + if self.shuffle: + X_train = X.copy() + self._random_state.shuffle(X_train) + else: + X_train = X + + n_samples, n_features = X_train.shape + + if self.verbose: + print("[dict_learning]") + + # Inner stats + self._A = np.zeros( + (self._n_components, self._n_components), dtype=X_train.dtype + ) + self._B = np.zeros((n_features, self._n_components), dtype=X_train.dtype) + + # Attributes to monitor the convergence + self._ewa_cost = None + self._ewa_cost_min = None + self._no_improvement = 0 + + batches = gen_batches(n_samples, self._batch_size) + batches = itertools.cycle(batches) + n_steps_per_iter = int(np.ceil(n_samples / self._batch_size)) + n_steps = self.max_iter * n_steps_per_iter + + i = -1 # to allow max_iter = 0 + + for i, batch in zip(range(n_steps), batches): + X_batch = X_train[batch] + + batch_cost = self._minibatch_step( + X_batch, dictionary, self._random_state, i + ) + + if self._check_convergence( + X_batch, batch_cost, dictionary, old_dict, n_samples, i, n_steps + ): + break + + # XXX callback param added for backward compat in #18975 but a common + # unified callback API should be preferred + if self.callback is not None: + self.callback(locals()) + + old_dict[:] = dictionary + + self.n_steps_ = i + 1 + self.n_iter_ = np.ceil(self.n_steps_ / n_steps_per_iter) + self.components_ = dictionary - U, (A, B), self.n_iter_ = dict_learning_online( - X, self.n_components, alpha=self.alpha, - n_iter=self.n_iter, return_code=False, - method=self.fit_algorithm, - method_max_iter=self.transform_max_iter, - n_jobs=self.n_jobs, dict_init=self.dict_init, - batch_size=self.batch_size, shuffle=self.shuffle, - verbose=self.verbose, random_state=random_state, - return_inner_stats=True, - return_n_iter=True, - positive_dict=self.positive_dict, - positive_code=self.positive_code) - self.components_ = U - # Keep track of the state of the algorithm to be able to do - # some online fitting (partial_fit) - self.inner_stats_ = (A, B) - self.iter_offset_ = self.n_iter - self.random_state_ = random_state return self - def partial_fit(self, X, y=None, iter_offset=None): - """Updates the model using the data in X as a mini-batch. + @_fit_context(prefer_skip_nested_validation=True) + def partial_fit(self, X, y=None): + """Update the model using the data in X as a mini-batch. Parameters ---------- X : array-like of shape (n_samples, n_features) - Training vector, where n_samples in the number of samples - and n_features is the number of features. + Training vector, where `n_samples` is the number of samples + and `n_features` is the number of features. y : Ignored - - iter_offset : int, default=None - The number of iteration on data batches that has been - performed before this call to partial_fit. This is optional: - if no number is passed, the memory of the object is - used. + Not used, present for API consistency by convention. Returns ------- self : object - Returns the instance itself. + Return the instance itself. """ - if not hasattr(self, 'random_state_'): - self.random_state_ = check_random_state(self.random_state) - if hasattr(self, 'components_'): - dict_init = self.components_ + has_components = hasattr(self, "components_") + + X = validate_data( + self, X, dtype=[np.float64, np.float32], order="C", reset=not has_components + ) + + if not has_components: + # This instance has not been fitted yet (fit or partial_fit) + self._check_params(X) + self._random_state = check_random_state(self.random_state) + + dictionary = self._initialize_dict(X, self._random_state) + + self.n_steps_ = 0 + + self._A = np.zeros((self._n_components, self._n_components), dtype=X.dtype) + self._B = np.zeros((X.shape[1], self._n_components), dtype=X.dtype) else: - dict_init = self.dict_init - inner_stats = getattr(self, 'inner_stats_', None) - if iter_offset is None: - iter_offset = getattr(self, 'iter_offset_', 0) - X = self._validate_data(X, reset=(iter_offset == 0)) - U, (A, B) = dict_learning_online( - X, self.n_components, alpha=self.alpha, - n_iter=1, method=self.fit_algorithm, - method_max_iter=self.transform_max_iter, - n_jobs=self.n_jobs, dict_init=dict_init, - batch_size=len(X), shuffle=False, - verbose=self.verbose, return_code=False, - iter_offset=iter_offset, random_state=self.random_state_, - return_inner_stats=True, inner_stats=inner_stats, - positive_dict=self.positive_dict, - positive_code=self.positive_code) - self.components_ = U + dictionary = self.components_ + + self._minibatch_step(X, dictionary, self._random_state, self.n_steps_) + + self.components_ = dictionary + self.n_steps_ += 1 - # Keep track of the state of the algorithm to be able to do - # some online fitting (partial_fit) - self.inner_stats_ = (A, B) - self.iter_offset_ = iter_offset + 1 return self + + @property + def _n_features_out(self): + """Number of transformed output features.""" + return self.components_.shape[0] + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.transformer_tags.preserves_dtype = ["float64", "float32"] + return tags diff --git a/sklearn/decomposition/_factor_analysis.py b/sklearn/decomposition/_factor_analysis.py index 5dd9f13094a89..d6d5e72a5b7d3 100644 --- a/sklearn/decomposition/_factor_analysis.py +++ b/sklearn/decomposition/_factor_analysis.py @@ -13,26 +13,30 @@ Algorithm 21.1 """ -# Author: Christian Osendorfer -# Alexandre Gramfort -# Denis A. Engemann - -# License: BSD3 +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import warnings -from math import sqrt, log +from math import log, sqrt +from numbers import Integral, Real + import numpy as np from scipy import linalg - -from ..base import BaseEstimator, TransformerMixin -from ..utils import check_random_state -from ..utils.extmath import fast_logdet, randomized_svd, squared_norm -from ..utils.validation import check_is_fitted, _deprecate_positional_args +from ..base import ( + BaseEstimator, + ClassNamePrefixFeaturesOutMixin, + TransformerMixin, + _fit_context, +) from ..exceptions import ConvergenceWarning +from ..utils import check_random_state +from ..utils._param_validation import Interval, StrOptions +from ..utils.extmath import _randomized_svd, fast_logdet, squared_norm +from ..utils.validation import check_is_fitted, validate_data -class FactorAnalysis(TransformerMixin, BaseEstimator): +class FactorAnalysis(ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator): """Factor Analysis (FA). A simple linear generative model with Gaussian latent variables. @@ -45,7 +49,7 @@ class FactorAnalysis(TransformerMixin, BaseEstimator): If we would restrict the model further, by assuming that the Gaussian noise is even isotropic (all diagonal entries are the same) we would obtain - :class:`PPCA`. + :class:`PCA`. FactorAnalysis performs a maximum likelihood estimate of the so-called `loading` matrix, the transformation of the latent variables to the @@ -62,7 +66,7 @@ class FactorAnalysis(TransformerMixin, BaseEstimator): of ``X`` that are obtained after ``transform``. If None, n_components is set to the number of features. - tol : float, defaul=1e-2 + tol : float, default=1e-2 Stopping tolerance for log-likelihood increase. copy : bool, default=True @@ -72,7 +76,7 @@ class FactorAnalysis(TransformerMixin, BaseEstimator): max_iter : int, default=1000 Maximum number of iterations. - noise_variance_init : ndarray of shape (n_features,), default=None + noise_variance_init : array-like of shape (n_features,), default=None The initial guess of the noise variance for each feature. If None, it defaults to np.ones(n_features). @@ -120,23 +124,16 @@ class FactorAnalysis(TransformerMixin, BaseEstimator): mean_ : ndarray of shape (n_features,) Per-feature empirical mean, estimated from the training set. - Examples - -------- - >>> from sklearn.datasets import load_digits - >>> from sklearn.decomposition import FactorAnalysis - >>> X, _ = load_digits(return_X_y=True) - >>> transformer = FactorAnalysis(n_components=7, random_state=0) - >>> X_transformed = transformer.fit_transform(X) - >>> X_transformed.shape - (1797, 7) + n_features_in_ : int + Number of features seen during :term:`fit`. - References - ---------- - - David Barber, Bayesian Reasoning and Machine Learning, - Algorithm 21.1. + .. versionadded:: 0.24 - - Christopher M. Bishop: Pattern Recognition and Machine Learning, - Chapter 12.2.4. + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 See Also -------- @@ -146,19 +143,55 @@ class FactorAnalysis(TransformerMixin, BaseEstimator): computed in closed form. FastICA: Independent component analysis, a latent variable model with non-Gaussian latent variables. + + References + ---------- + - David Barber, Bayesian Reasoning and Machine Learning, + Algorithm 21.1. + + - Christopher M. Bishop: Pattern Recognition and Machine Learning, + Chapter 12.2.4. + + Examples + -------- + >>> from sklearn.datasets import load_digits + >>> from sklearn.decomposition import FactorAnalysis + >>> X, _ = load_digits(return_X_y=True) + >>> transformer = FactorAnalysis(n_components=7, random_state=0) + >>> X_transformed = transformer.fit_transform(X) + >>> X_transformed.shape + (1797, 7) """ - @_deprecate_positional_args - def __init__(self, n_components=None, *, tol=1e-2, copy=True, - max_iter=1000, - noise_variance_init=None, svd_method='randomized', - iterated_power=3, rotation=None, random_state=0): + + _parameter_constraints: dict = { + "n_components": [Interval(Integral, 0, None, closed="left"), None], + "tol": [Interval(Real, 0.0, None, closed="left")], + "copy": ["boolean"], + "max_iter": [Interval(Integral, 1, None, closed="left")], + "noise_variance_init": ["array-like", None], + "svd_method": [StrOptions({"randomized", "lapack"})], + "iterated_power": [Interval(Integral, 0, None, closed="left")], + "rotation": [StrOptions({"varimax", "quartimax"}), None], + "random_state": ["random_state"], + } + + def __init__( + self, + n_components=None, + *, + tol=1e-2, + copy=True, + max_iter=1000, + noise_variance_init=None, + svd_method="randomized", + iterated_power=3, + rotation=None, + random_state=0, + ): self.n_components = n_components self.copy = copy self.tol = tol self.max_iter = max_iter - if svd_method not in ['lapack', 'randomized']: - raise ValueError('SVD method %s is not supported. Please consider' - ' the documentation' % svd_method) self.svd_method = svd_method self.noise_variance_init = noise_variance_init @@ -166,8 +199,9 @@ def __init__(self, n_components=None, *, tol=1e-2, copy=True, self.random_state = random_state self.rotation = rotation + @_fit_context(prefer_skip_nested_validation=True) def fit(self, X, y=None): - """Fit the FactorAnalysis model to X using SVD based approach + """Fit the FactorAnalysis model to X using SVD based approach. Parameters ---------- @@ -175,12 +209,16 @@ def fit(self, X, y=None): Training data. y : Ignored + Ignored parameter. Returns ------- - self + self : object + FactorAnalysis class instance. """ - X = self._validate_data(X, copy=self.copy, dtype=np.float64) + X = validate_data( + self, X, copy=self.copy, dtype=np.float64, force_writeable=True + ) n_samples, n_features = X.shape n_components = self.n_components @@ -192,16 +230,18 @@ def fit(self, X, y=None): # some constant terms nsqrt = sqrt(n_samples) - llconst = n_features * log(2. * np.pi) + n_components + llconst = n_features * log(2.0 * np.pi) + n_components var = np.var(X, axis=0) if self.noise_variance_init is None: psi = np.ones(n_features, dtype=X.dtype) else: if len(self.noise_variance_init) != n_features: - raise ValueError("noise_variance_init dimension does not " - "with number of features : %d != %d" % - (len(self.noise_variance_init), n_features)) + raise ValueError( + "noise_variance_init dimension does not " + "with number of features : %d != %d" + % (len(self.noise_variance_init), n_features) + ) psi = np.array(self.noise_variance_init) loglike = [] @@ -210,24 +250,27 @@ def fit(self, X, y=None): # we'll modify svd outputs to return unexplained variance # to allow for unified computation of loglikelihood - if self.svd_method == 'lapack': + if self.svd_method == "lapack": + def my_svd(X): - _, s, Vt = linalg.svd(X, - full_matrices=False, - check_finite=False) - return (s[:n_components], Vt[:n_components], - squared_norm(s[n_components:])) - elif self.svd_method == 'randomized': + _, s, Vt = linalg.svd(X, full_matrices=False, check_finite=False) + return ( + s[:n_components], + Vt[:n_components], + squared_norm(s[n_components:]), + ) + + else: # svd_method == "randomized" random_state = check_random_state(self.random_state) def my_svd(X): - _, s, Vt = randomized_svd(X, n_components, - random_state=random_state, - n_iter=self.iterated_power) + _, s, Vt = _randomized_svd( + X, + n_components, + random_state=random_state, + n_iter=self.iterated_power, + ) return s, Vt, squared_norm(X) - squared_norm(s) - else: - raise ValueError('SVD method %s is not supported. Please consider' - ' the documentation' % self.svd_method) for i in range(self.max_iter): # SMALL helps numerics @@ -235,25 +278,27 @@ def my_svd(X): s, Vt, unexp_var = my_svd(X / (sqrt_psi * nsqrt)) s **= 2 # Use 'maximum' here to avoid sqrt problems. - W = np.sqrt(np.maximum(s - 1., 0.))[:, np.newaxis] * Vt + W = np.sqrt(np.maximum(s - 1.0, 0.0))[:, np.newaxis] * Vt del Vt W *= sqrt_psi # loglikelihood ll = llconst + np.sum(np.log(s)) ll += unexp_var + np.sum(np.log(psi)) - ll *= -n_samples / 2. + ll *= -n_samples / 2.0 loglike.append(ll) if (ll - old_ll) < self.tol: break old_ll = ll - psi = np.maximum(var - np.sum(W ** 2, axis=0), SMALL) + psi = np.maximum(var - np.sum(W**2, axis=0), SMALL) else: - warnings.warn('FactorAnalysis did not converge.' + - ' You might want' + - ' to increase the number of iterations.', - ConvergenceWarning) + warnings.warn( + "FactorAnalysis did not converge." + " You might want" + " to increase the number of iterations.", + ConvergenceWarning, + ) self.components_ = W if self.rotation is not None: @@ -281,7 +326,7 @@ def transform(self, X): """ check_is_fitted(self) - X = self._validate_data(X, reset=False) + X = validate_data(self, X, reset=False) Ih = np.eye(len(self.components_)) X_transformed = X - self.mean_ @@ -306,7 +351,7 @@ def get_covariance(self): check_is_fitted(self) cov = np.dot(self.components_.T, self.components_) - cov.flat[::len(cov) + 1] += self.noise_variance_ # modify diag inplace + cov.flat[:: len(cov) + 1] += self.noise_variance_ # modify diag inplace return cov def get_precision(self): @@ -323,75 +368,74 @@ def get_precision(self): # handle corner cases first if self.n_components == 0: - return np.diag(1. / self.noise_variance_) + return np.diag(1.0 / self.noise_variance_) if self.n_components == n_features: return linalg.inv(self.get_covariance()) # Get precision using matrix inversion lemma components_ = self.components_ precision = np.dot(components_ / self.noise_variance_, components_.T) - precision.flat[::len(precision) + 1] += 1. - precision = np.dot(components_.T, - np.dot(linalg.inv(precision), components_)) + precision.flat[:: len(precision) + 1] += 1.0 + precision = np.dot(components_.T, np.dot(linalg.inv(precision), components_)) precision /= self.noise_variance_[:, np.newaxis] precision /= -self.noise_variance_[np.newaxis, :] - precision.flat[::len(precision) + 1] += 1. / self.noise_variance_ + precision.flat[:: len(precision) + 1] += 1.0 / self.noise_variance_ return precision def score_samples(self, X): - """Compute the log-likelihood of each sample + """Compute the log-likelihood of each sample. Parameters ---------- X : ndarray of shape (n_samples, n_features) - The data + The data. Returns ------- ll : ndarray of shape (n_samples,) - Log-likelihood of each sample under the current model + Log-likelihood of each sample under the current model. """ check_is_fitted(self) - X = self._validate_data(X, reset=False) + X = validate_data(self, X, reset=False) Xr = X - self.mean_ precision = self.get_precision() n_features = X.shape[1] - log_like = -.5 * (Xr * (np.dot(Xr, precision))).sum(axis=1) - log_like -= .5 * (n_features * log(2. * np.pi) - - fast_logdet(precision)) + log_like = -0.5 * (Xr * (np.dot(Xr, precision))).sum(axis=1) + log_like -= 0.5 * (n_features * log(2.0 * np.pi) - fast_logdet(precision)) return log_like def score(self, X, y=None): - """Compute the average log-likelihood of the samples + """Compute the average log-likelihood of the samples. Parameters ---------- X : ndarray of shape (n_samples, n_features) - The data + The data. y : Ignored + Ignored parameter. Returns ------- ll : float - Average log-likelihood of the samples under the current model + Average log-likelihood of the samples under the current model. """ return np.mean(self.score_samples(X)) def _rotate(self, components, n_components=None, tol=1e-6): "Rotate the factor analysis solution." # note that tol is not exposed - implemented = ("varimax", "quartimax") - method = self.rotation - if method in implemented: - return _ortho_rotation(components.T, method=method, - tol=tol)[:self.n_components] - else: - raise ValueError("'method' must be in %s, not %s" - % (implemented, method)) + return _ortho_rotation(components.T, method=self.rotation, tol=tol)[ + : self.n_components + ] + + @property + def _n_features_out(self): + """Number of transformed output features.""" + return self.components_.shape[0] -def _ortho_rotation(components, method='varimax', tol=1e-6, max_iter=100): +def _ortho_rotation(components, method="varimax", tol=1e-6, max_iter=100): """Return rotated components.""" nrow, ncol = components.shape rotation_matrix = np.eye(ncol) @@ -400,11 +444,10 @@ def _ortho_rotation(components, method='varimax', tol=1e-6, max_iter=100): for _ in range(max_iter): comp_rot = np.dot(components, rotation_matrix) if method == "varimax": - tmp = comp_rot * np.transpose((comp_rot ** 2).sum(axis=0) / nrow) + tmp = comp_rot * np.transpose((comp_rot**2).sum(axis=0) / nrow) elif method == "quartimax": tmp = 0 - u, s, v = np.linalg.svd( - np.dot(components.T, comp_rot ** 3 - tmp)) + u, s, v = np.linalg.svd(np.dot(components.T, comp_rot**3 - tmp)) rotation_matrix = np.dot(u, v) var_new = np.sum(s) if var != 0 and var_new < var * (1 + tol): diff --git a/sklearn/decomposition/_fastica.py b/sklearn/decomposition/_fastica.py index 27f7f646ea579..efda7bfca56b6 100644 --- a/sklearn/decomposition/_fastica.py +++ b/sklearn/decomposition/_fastica.py @@ -5,24 +5,27 @@ Independent Component Analysis, by Hyvarinen et al. """ -# Authors: Pierre Lafaye de Micheaux, Stefan van der Walt, Gael Varoquaux, -# Bertrand Thirion, Alexandre Gramfort, Denis A. Engemann -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import warnings +from numbers import Integral, Real import numpy as np from scipy import linalg -from ..base import BaseEstimator, TransformerMixin +from ..base import ( + BaseEstimator, + ClassNamePrefixFeaturesOutMixin, + TransformerMixin, + _fit_context, +) from ..exceptions import ConvergenceWarning +from ..utils import as_float_array, check_array, check_random_state +from ..utils._param_validation import Interval, Options, StrOptions, validate_params +from ..utils.validation import check_is_fitted, validate_data -from ..utils import check_array, as_float_array, check_random_state -from ..utils.validation import check_is_fitted -from ..utils.validation import FLOAT_DTYPES -from ..utils.validation import _deprecate_positional_args - -__all__ = ['fastica', 'FastICA'] +__all__ = ["FastICA", "fastica"] def _gs_decorrelation(w, W, j): @@ -51,13 +54,18 @@ def _gs_decorrelation(w, W, j): def _sym_decorrelation(W): - """ Symmetric decorrelation + """Symmetric decorrelation i.e. W <- (W * W.T) ^{-1/2} * W """ s, u = linalg.eigh(np.dot(W, W.T)) + # Avoid sqrt of negative values because of rounding errors. Note that + # np.sqrt(tiny) is larger than tiny and therefore this clipping also + # prevents division by zero in the next step. + s = np.clip(s, a_min=np.finfo(W.dtype).tiny, a_max=None) + # u (resp. s) contains the eigenvectors (resp. square roots of # the eigenvalues) of W * W.T - return np.linalg.multi_dot([u * (1. / np.sqrt(s)), u.T, W]) + return np.linalg.multi_dot([u * (1.0 / np.sqrt(s)), u.T, W]) def _ica_def(X, tol, g, fun_args, max_iter, w_init): @@ -73,7 +81,7 @@ def _ica_def(X, tol, g, fun_args, max_iter, w_init): # j is the index of the extracted component for j in range(n_components): w = w_init[j, :].copy() - w /= np.sqrt((w ** 2).sum()) + w /= np.sqrt((w**2).sum()) for i in range(max_iter): gwtx, g_wtx = g(np.dot(w.T, X), fun_args) @@ -82,7 +90,7 @@ def _ica_def(X, tol, g, fun_args, max_iter, w_init): _gs_decorrelation(w1, W, j) - w1 /= np.sqrt((w1 ** 2).sum()) + w1 /= np.sqrt((w1**2).sum()) lim = np.abs(np.abs((w1 * w).sum()) - 1) w = w1 @@ -106,18 +114,23 @@ def _ica_par(X, tol, g, fun_args, max_iter, w_init): p_ = float(X.shape[1]) for ii in range(max_iter): gwtx, g_wtx = g(np.dot(W, X), fun_args) - W1 = _sym_decorrelation(np.dot(gwtx, X.T) / p_ - - g_wtx[:, np.newaxis] * W) + W1 = _sym_decorrelation(np.dot(gwtx, X.T) / p_ - g_wtx[:, np.newaxis] * W) del gwtx, g_wtx # builtin max, abs are faster than numpy counter parts. - lim = max(abs(abs(np.diag(np.dot(W1, W.T))) - 1)) + # np.einsum allows having the lowest memory footprint. + # It is faster than np.diag(np.dot(W1, W.T)). + lim = max(abs(abs(np.einsum("ij,ij->i", W1, W)) - 1)) W = W1 if lim < tol: break else: - warnings.warn('FastICA did not converge. Consider increasing ' - 'tolerance or the maximum number of iterations.', - ConvergenceWarning) + warnings.warn( + ( + "FastICA did not converge. Consider increasing " + "tolerance or the maximum number of iterations." + ), + ConvergenceWarning, + ) return W, ii + 1 @@ -125,56 +138,84 @@ def _ica_par(X, tol, g, fun_args, max_iter, w_init): # Some standard non-linear functions. # XXX: these should be optimized, as they can be a bottleneck. def _logcosh(x, fun_args=None): - alpha = fun_args.get('alpha', 1.0) # comment it out? + alpha = fun_args.get("alpha", 1.0) # comment it out? x *= alpha gx = np.tanh(x, x) # apply the tanh inplace - g_x = np.empty(x.shape[0]) + g_x = np.empty(x.shape[0], dtype=x.dtype) # XXX compute in chunks to avoid extra allocation for i, gx_i in enumerate(gx): # please don't vectorize. - g_x[i] = (alpha * (1 - gx_i ** 2)).mean() + g_x[i] = (alpha * (1 - gx_i**2)).mean() return gx, g_x def _exp(x, fun_args): - exp = np.exp(-(x ** 2) / 2) + exp = np.exp(-(x**2) / 2) gx = x * exp - g_x = (1 - x ** 2) * exp + g_x = (1 - x**2) * exp return gx, g_x.mean(axis=-1) def _cube(x, fun_args): - return x ** 3, (3 * x ** 2).mean(axis=-1) - - -@_deprecate_positional_args -def fastica(X, n_components=None, *, algorithm="parallel", whiten=True, - fun="logcosh", fun_args=None, max_iter=200, tol=1e-04, w_init=None, - random_state=None, return_X_mean=False, compute_sources=True, - return_n_iter=False): + return x**3, (3 * x**2).mean(axis=-1) + + +@validate_params( + { + "X": ["array-like"], + "return_X_mean": ["boolean"], + "compute_sources": ["boolean"], + "return_n_iter": ["boolean"], + }, + prefer_skip_nested_validation=False, +) +def fastica( + X, + n_components=None, + *, + algorithm="parallel", + whiten="unit-variance", + fun="logcosh", + fun_args=None, + max_iter=200, + tol=1e-04, + w_init=None, + whiten_solver="svd", + random_state=None, + return_X_mean=False, + compute_sources=True, + return_n_iter=False, +): """Perform Fast Independent Component Analysis. + The implementation is based on [1]_. + Read more in the :ref:`User Guide `. Parameters ---------- X : array-like of shape (n_samples, n_features) - Training vector, where n_samples is the number of samples and - n_features is the number of features. + Training vector, where `n_samples` is the number of samples and + `n_features` is the number of features. n_components : int, default=None - Number of components to extract. If None no dimension reduction - is performed. + Number of components to use. If None is passed, all are used. algorithm : {'parallel', 'deflation'}, default='parallel' - Apply a parallel or deflational FASTICA algorithm. + Specify which algorithm to use for FastICA. + + whiten : str or bool, default='unit-variance' + Specify the whitening strategy to use. - whiten : bool, default=True - If True perform an initial whitening of the data. - If False, the data is assumed to have already been - preprocessed: it should be centered, normed and white. - Otherwise you will get incorrect results. - In this case the parameter n_components will be ignored. + - If 'arbitrary-variance', a whitening with variance + arbitrary is used. + - If 'unit-variance', the whitening matrix is rescaled to ensure that + each recovered source has unit variance. + - If False, the data is already considered to be whitened, and no + whitening is performed. + + .. versionchanged:: 1.3 + The default value of `whiten` changed to 'unit-variance' in 1.3. fun : {'logcosh', 'exp', 'cube'} or callable, default='logcosh' The functional form of the G function used in the @@ -183,26 +224,38 @@ def fastica(X, n_components=None, *, algorithm="parallel", whiten=True, You can also provide your own function. It should return a tuple containing the value of the function, and of its derivative, in the point. The derivative should be averaged along its last dimension. - Example: + Example:: - def my_g(x): - return x ** 3, np.mean(3 * x ** 2, axis=-1) + def my_g(x): + return x ** 3, (3 * x ** 2).mean(axis=-1) fun_args : dict, default=None Arguments to send to the functional form. If empty or None and if fun='logcosh', fun_args will take value - {'alpha' : 1.0} + {'alpha' : 1.0}. max_iter : int, default=200 Maximum number of iterations to perform. - tol : float, default=1e-04 + tol : float, default=1e-4 A positive scalar giving the tolerance at which the un-mixing matrix is considered to have converged. w_init : ndarray of shape (n_components, n_components), default=None - Initial un-mixing array of dimension (n.comp,n.comp). - If None (default) then an array of normal r.v.'s is used. + Initial un-mixing array. If `w_init=None`, then an array of values + drawn from a normal distribution is used. + + whiten_solver : {"eigh", "svd"}, default="svd" + The solver to use for whitening. + + - "svd" is more stable numerically if the problem is degenerate, and + often faster when `n_samples <= n_features`. + + - "eigh" is generally more memory efficient when + `n_samples >= n_features`, and can be faster when + `n_samples >= 50 * n_features`. + + .. versionadded:: 1.2 random_state : int, RandomState instance or None, default=None Used to initialize ``w_init`` when not specified, with a @@ -233,7 +286,7 @@ def my_g(x): if K is not None, else it is the inverse of W. S : ndarray of shape (n_samples, n_components) or None - Estimated source matrix + Estimated source matrix. X_mean : ndarray of shape (n_features,) The mean over features. Returned only if return_X_mean is True. @@ -246,7 +299,6 @@ def my_g(x): Notes ----- - The data matrix X is considered to be a linear combination of non-Gaussian (independent) components i.e. X = AS where columns of S contain the independent components and A is a linear mixing @@ -262,48 +314,61 @@ def my_g(x): before the algorithm is applied. This makes it slightly faster for Fortran-ordered input. - Implemented using FastICA: - *A. Hyvarinen and E. Oja, Independent Component Analysis: - Algorithms and Applications, Neural Networks, 13(4-5), 2000, - pp. 411-430* + References + ---------- + .. [1] A. Hyvarinen and E. Oja, "Fast Independent Component Analysis", + Algorithms and Applications, Neural Networks, 13(4-5), 2000, + pp. 411-430. + Examples + -------- + >>> from sklearn.datasets import load_digits + >>> from sklearn.decomposition import fastica + >>> X, _ = load_digits(return_X_y=True) + >>> K, W, S = fastica(X, n_components=7, random_state=0, whiten='unit-variance') + >>> K.shape + (7, 64) + >>> W.shape + (7, 7) + >>> S.shape + (1797, 7) """ + est = FastICA( + n_components=n_components, + algorithm=algorithm, + whiten=whiten, + fun=fun, + fun_args=fun_args, + max_iter=max_iter, + tol=tol, + w_init=w_init, + whiten_solver=whiten_solver, + random_state=random_state, + ) + est._validate_params() + S = est._fit_transform(X, compute_sources=compute_sources) + + if est.whiten in ["unit-variance", "arbitrary-variance"]: + K = est.whitening_ + X_mean = est.mean_ + else: + K = None + X_mean = None - est = FastICA(n_components=n_components, algorithm=algorithm, - whiten=whiten, fun=fun, fun_args=fun_args, - max_iter=max_iter, tol=tol, w_init=w_init, - random_state=random_state) - sources = est._fit(X, compute_sources=compute_sources) - - if whiten: - if return_X_mean: - if return_n_iter: - return (est.whitening_, est._unmixing, sources, est.mean_, - est.n_iter_) - else: - return est.whitening_, est._unmixing, sources, est.mean_ - else: - if return_n_iter: - return est.whitening_, est._unmixing, sources, est.n_iter_ - else: - return est.whitening_, est._unmixing, sources + returned_values = [K, est._unmixing, S] + if return_X_mean: + returned_values.append(X_mean) + if return_n_iter: + returned_values.append(est.n_iter_) - else: - if return_X_mean: - if return_n_iter: - return None, est._unmixing, sources, None, est.n_iter_ - else: - return None, est._unmixing, sources, None - else: - if return_n_iter: - return None, est._unmixing, sources, est.n_iter_ - else: - return None, est._unmixing, sources + return returned_values -class FastICA(TransformerMixin, BaseEstimator): +class FastICA(ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator): """FastICA: a fast algorithm for Independent Component Analysis. + The implementation is based on [1]_. + Read more in the :ref:`User Guide `. Parameters @@ -312,11 +377,20 @@ class FastICA(TransformerMixin, BaseEstimator): Number of components to use. If None is passed, all are used. algorithm : {'parallel', 'deflation'}, default='parallel' - Apply parallel or deflational algorithm for FastICA. + Specify which algorithm to use for FastICA. - whiten : bool, default=True - If whiten is false, the data is already considered to be - whitened, and no whitening is performed. + whiten : str or bool, default='unit-variance' + Specify the whitening strategy to use. + + - If 'arbitrary-variance', a whitening with variance + arbitrary is used. + - If 'unit-variance', the whitening matrix is rescaled to ensure that + each recovered source has unit variance. + - If False, the data is already considered to be whitened, and no + whitening is performed. + + .. versionchanged:: 1.3 + The default value of `whiten` changed to 'unit-variance' in 1.3. fun : {'logcosh', 'exp', 'cube'} or callable, default='logcosh' The functional form of the G function used in the @@ -324,24 +398,39 @@ class FastICA(TransformerMixin, BaseEstimator): or 'cube'. You can also provide your own function. It should return a tuple containing the value of the function, and of its derivative, in the - point. Example:: + point. The derivative should be averaged along its last dimension. + Example:: def my_g(x): return x ** 3, (3 * x ** 2).mean(axis=-1) fun_args : dict, default=None Arguments to send to the functional form. - If empty and if fun='logcosh', fun_args will take value + If empty or None and if fun='logcosh', fun_args will take value {'alpha' : 1.0}. max_iter : int, default=200 Maximum number of iterations during fit. tol : float, default=1e-4 - Tolerance on update at each iteration. + A positive scalar giving the tolerance at which the + un-mixing matrix is considered to have converged. - w_init : ndarray of shape (n_components, n_components), default=None - The mixing matrix to be used to initialize the algorithm. + w_init : array-like of shape (n_components, n_components), default=None + Initial un-mixing array. If `w_init=None`, then an array of values + drawn from a normal distribution is used. + + whiten_solver : {"eigh", "svd"}, default="svd" + The solver to use for whitening. + + - "svd" is more stable numerically if the problem is degenerate, and + often faster when `n_samples <= n_features`. + + - "eigh" is generally more memory efficient when + `n_samples >= n_features`, and can be faster when + `n_samples >= 50 * n_features`. + + .. versionadded:: 1.2 random_state : int, RandomState instance or None, default=None Used to initialize ``w_init`` when not specified, with a @@ -364,6 +453,17 @@ def my_g(x): mean_ : ndarray of shape(n_features,) The mean over features. Only set if `self.whiten` is True. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + n_iter_ : int If the algorithm is "deflation", n_iter is the maximum number of iterations run across all components. Else @@ -373,33 +473,64 @@ def my_g(x): Only set if whiten is 'True'. This is the pre-whitening matrix that projects data onto the first `n_components` principal components. + See Also + -------- + PCA : Principal component analysis (PCA). + IncrementalPCA : Incremental principal components analysis (IPCA). + KernelPCA : Kernel Principal component analysis (KPCA). + MiniBatchSparsePCA : Mini-batch Sparse Principal Components Analysis. + SparsePCA : Sparse Principal Components Analysis (SparsePCA). + + References + ---------- + .. [1] A. Hyvarinen and E. Oja, Independent Component Analysis: + Algorithms and Applications, Neural Networks, 13(4-5), 2000, + pp. 411-430. + Examples -------- >>> from sklearn.datasets import load_digits >>> from sklearn.decomposition import FastICA >>> X, _ = load_digits(return_X_y=True) >>> transformer = FastICA(n_components=7, - ... random_state=0) + ... random_state=0, + ... whiten='unit-variance') >>> X_transformed = transformer.fit_transform(X) >>> X_transformed.shape (1797, 7) - - Notes - ----- - Implementation based on - *A. Hyvarinen and E. Oja, Independent Component Analysis: - Algorithms and Applications, Neural Networks, 13(4-5), 2000, - pp. 411-430* - """ - @_deprecate_positional_args - def __init__(self, n_components=None, *, algorithm='parallel', whiten=True, - fun='logcosh', fun_args=None, max_iter=200, tol=1e-4, - w_init=None, random_state=None): + + _parameter_constraints: dict = { + "n_components": [Interval(Integral, 1, None, closed="left"), None], + "algorithm": [StrOptions({"parallel", "deflation"})], + "whiten": [ + StrOptions({"arbitrary-variance", "unit-variance"}), + Options(bool, {False}), + ], + "fun": [StrOptions({"logcosh", "exp", "cube"}), callable], + "fun_args": [dict, None], + "max_iter": [Interval(Integral, 1, None, closed="left")], + "tol": [Interval(Real, 0.0, None, closed="left")], + "w_init": ["array-like", None], + "whiten_solver": [StrOptions({"eigh", "svd"})], + "random_state": ["random_state"], + } + + def __init__( + self, + n_components=None, + *, + algorithm="parallel", + whiten="unit-variance", + fun="logcosh", + fun_args=None, + max_iter=200, + tol=1e-4, + w_init=None, + whiten_solver="svd", + random_state=None, + ): super().__init__() - if max_iter < 1: - raise ValueError("max_iter should be greater than 1, got " - "(max_iter={})".format(max_iter)) self.n_components = n_components self.algorithm = algorithm self.whiten = whiten @@ -408,16 +539,17 @@ def __init__(self, n_components=None, *, algorithm='parallel', whiten=True, self.max_iter = max_iter self.tol = tol self.w_init = w_init + self.whiten_solver = whiten_solver self.random_state = random_state - def _fit(self, X, compute_sources=False): - """Fit the model + def _fit_transform(self, X, compute_sources=False): + """Fit the model. Parameters ---------- X : array-like of shape (n_samples, n_features) - Training data, where n_samples is the number of samples - and n_features is the number of features. + Training data, where `n_samples` is the number of samples + and `n_features` is the number of features. compute_sources : bool, default=False If False, sources are not computes but only the rotation matrix. @@ -425,109 +557,132 @@ def _fit(self, X, compute_sources=False): Returns ------- - X_new : ndarray of shape (n_samples, n_components) + S : ndarray of shape (n_samples, n_components) or None + Sources matrix. `None` if `compute_sources` is `False`. """ - - X = self._validate_data(X, copy=self.whiten, dtype=FLOAT_DTYPES, - ensure_min_samples=2).T + XT = validate_data( + self, + X, + copy=self.whiten, + dtype=[np.float64, np.float32], + ensure_min_samples=2, + ).T fun_args = {} if self.fun_args is None else self.fun_args random_state = check_random_state(self.random_state) - alpha = fun_args.get('alpha', 1.0) + alpha = fun_args.get("alpha", 1.0) if not 1 <= alpha <= 2: - raise ValueError('alpha must be in [1,2]') + raise ValueError("alpha must be in [1,2]") - if self.fun == 'logcosh': + if self.fun == "logcosh": g = _logcosh - elif self.fun == 'exp': + elif self.fun == "exp": g = _exp - elif self.fun == 'cube': + elif self.fun == "cube": g = _cube elif callable(self.fun): + def g(x, fun_args): return self.fun(x, **fun_args) - else: - exc = ValueError if isinstance(self.fun, str) else TypeError - raise exc( - "Unknown function %r;" - " should be one of 'logcosh', 'exp', 'cube' or callable" - % self.fun - ) - - n_samples, n_features = X.shape + n_features, n_samples = XT.shape n_components = self.n_components if not self.whiten and n_components is not None: n_components = None - warnings.warn('Ignoring n_components with whiten=False.') + warnings.warn("Ignoring n_components with whiten=False.") if n_components is None: n_components = min(n_samples, n_features) - if (n_components > min(n_samples, n_features)): + if n_components > min(n_samples, n_features): n_components = min(n_samples, n_features) warnings.warn( - 'n_components is too large: it will be set to %s' - % n_components + "n_components is too large: it will be set to %s" % n_components ) if self.whiten: - # Centering the columns (ie the variables) - X_mean = X.mean(axis=-1) - X -= X_mean[:, np.newaxis] + # Centering the features of X + X_mean = XT.mean(axis=-1) + XT -= X_mean[:, np.newaxis] # Whitening and preprocessing by PCA - u, d, _ = linalg.svd(X, full_matrices=False, check_finite=False) + if self.whiten_solver == "eigh": + # Faster when num_samples >> n_features + d, u = linalg.eigh(XT.dot(X)) + sort_indices = np.argsort(d)[::-1] + eps = np.finfo(d.dtype).eps * 10 + degenerate_idx = d < eps + if np.any(degenerate_idx): + warnings.warn( + "There are some small singular values, using " + "whiten_solver = 'svd' might lead to more " + "accurate results." + ) + d[degenerate_idx] = eps # For numerical issues + np.sqrt(d, out=d) + d, u = d[sort_indices], u[:, sort_indices] + elif self.whiten_solver == "svd": + u, d = linalg.svd(XT, full_matrices=False, check_finite=False)[:2] + + # Give consistent eigenvectors for both svd solvers + u *= np.sign(u[0]) - del _ K = (u / d).T[:n_components] # see (6.33) p.140 del u, d - X1 = np.dot(K, X) + X1 = np.dot(K, XT) # see (13.6) p.267 Here X1 is white and data # in X has been projected onto a subspace by PCA - X1 *= np.sqrt(n_features) + X1 *= np.sqrt(n_samples) else: # X must be casted to floats to avoid typing issues with numpy # 2.0 and the line below - X1 = as_float_array(X, copy=False) # copy has been taken care of + X1 = as_float_array(XT, copy=False) # copy has been taken care of w_init = self.w_init if w_init is None: - w_init = np.asarray(random_state.normal( - size=(n_components, n_components)), dtype=X1.dtype) + w_init = np.asarray( + random_state.normal(size=(n_components, n_components)), dtype=X1.dtype + ) else: w_init = np.asarray(w_init) if w_init.shape != (n_components, n_components): raise ValueError( - 'w_init has invalid shape -- should be %(shape)s' - % {'shape': (n_components, n_components)}) - - kwargs = {'tol': self.tol, - 'g': g, - 'fun_args': fun_args, - 'max_iter': self.max_iter, - 'w_init': w_init} - - if self.algorithm == 'parallel': + "w_init has invalid shape -- should be %(shape)s" + % {"shape": (n_components, n_components)} + ) + + kwargs = { + "tol": self.tol, + "g": g, + "fun_args": fun_args, + "max_iter": self.max_iter, + "w_init": w_init, + } + + if self.algorithm == "parallel": W, n_iter = _ica_par(X1, **kwargs) - elif self.algorithm == 'deflation': + elif self.algorithm == "deflation": W, n_iter = _ica_def(X1, **kwargs) - else: - raise ValueError('Invalid algorithm: must be either `parallel` or' - ' `deflation`.') del X1 + self.n_iter_ = n_iter + if compute_sources: if self.whiten: - S = np.linalg.multi_dot([W, K, X]).T + S = np.linalg.multi_dot([W, K, XT]).T else: - S = np.dot(W, X).T + S = np.dot(W, XT).T else: S = None - self.n_iter_ = n_iter - if self.whiten: + if self.whiten == "unit-variance": + if not compute_sources: + S = np.linalg.multi_dot([W, K, XT]).T + S_std = np.std(S, axis=0, keepdims=True) + S /= S_std + W /= S_std.T + self.components_ = np.dot(W, K) self.mean_ = X_mean self.whitening_ = K @@ -539,39 +694,46 @@ def g(x, fun_args): return S + @_fit_context(prefer_skip_nested_validation=True) def fit_transform(self, X, y=None): """Fit the model and recover the sources from X. Parameters ---------- X : array-like of shape (n_samples, n_features) - Training data, where n_samples is the number of samples - and n_features is the number of features. + Training data, where `n_samples` is the number of samples + and `n_features` is the number of features. y : Ignored + Not used, present for API consistency by convention. Returns ------- X_new : ndarray of shape (n_samples, n_components) + Estimated sources obtained by transforming the data with the + estimated unmixing matrix. """ - return self._fit(X, compute_sources=True) + return self._fit_transform(X, compute_sources=True) + @_fit_context(prefer_skip_nested_validation=True) def fit(self, X, y=None): """Fit the model to X. Parameters ---------- X : array-like of shape (n_samples, n_features) - Training data, where n_samples is the number of samples - and n_features is the number of features. + Training data, where `n_samples` is the number of samples + and `n_features` is the number of features. y : Ignored + Not used, present for API consistency by convention. Returns ------- - self + self : object + Returns the instance itself. """ - self._fit(X, compute_sources=False) + self._fit_transform(X, compute_sources=False) return self def transform(self, X, copy=True): @@ -580,8 +742,8 @@ def transform(self, X, copy=True): Parameters ---------- X : array-like of shape (n_samples, n_features) - Data to transform, where n_samples is the number of samples - and n_features is the number of features. + Data to transform, where `n_samples` is the number of samples + and `n_features` is the number of features. copy : bool, default=True If False, data passed to fit can be overwritten. Defaults to True. @@ -589,11 +751,18 @@ def transform(self, X, copy=True): Returns ------- X_new : ndarray of shape (n_samples, n_components) + Estimated sources obtained by transforming the data with the + estimated unmixing matrix. """ check_is_fitted(self) - X = self._validate_data(X, copy=(copy and self.whiten), - dtype=FLOAT_DTYPES, reset=False) + X = validate_data( + self, + X, + copy=(copy and self.whiten), + dtype=[np.float64, np.float32], + reset=False, + ) if self.whiten: X -= self.mean_ @@ -605,20 +774,31 @@ def inverse_transform(self, X, copy=True): Parameters ---------- X : array-like of shape (n_samples, n_components) - Sources, where n_samples is the number of samples - and n_components is the number of components. + Sources, where `n_samples` is the number of samples + and `n_components` is the number of components. copy : bool, default=True If False, data passed to fit are overwritten. Defaults to True. Returns ------- - X_new : ndarray of shape (n_samples, n_features) + X_original : ndarray of shape (n_samples, n_features) + Reconstructed data obtained with the mixing matrix. """ check_is_fitted(self) - X = check_array(X, copy=(copy and self.whiten), dtype=FLOAT_DTYPES) + X = check_array(X, copy=(copy and self.whiten), dtype=[np.float64, np.float32]) X = np.dot(X, self.mixing_.T) if self.whiten: X += self.mean_ return X + + @property + def _n_features_out(self): + """Number of transformed output features.""" + return self.components_.shape[0] + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.transformer_tags.preserves_dtype = ["float64", "float32"] + return tags diff --git a/sklearn/decomposition/_incremental_pca.py b/sklearn/decomposition/_incremental_pca.py index 10a1cceadd65e..da617ef8fa787 100644 --- a/sklearn/decomposition/_incremental_pca.py +++ b/sklearn/decomposition/_incremental_pca.py @@ -1,16 +1,21 @@ """Incremental Principal Components Analysis.""" -# Author: Kyle Kastner -# Giorgio Patrini -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from numbers import Integral import numpy as np from scipy import linalg, sparse -from ._base import _BasePCA +from sklearn.utils import metadata_routing + +from ..base import _fit_context from ..utils import gen_batches -from ..utils.extmath import svd_flip, _incremental_mean_and_var -from ..utils.validation import _deprecate_positional_args +from ..utils._param_validation import Interval +from ..utils.extmath import _incremental_mean_and_var, svd_flip +from ..utils.validation import validate_data +from ._base import _BasePCA class IncrementalPCA(_BasePCA): @@ -36,6 +41,9 @@ class IncrementalPCA(_BasePCA): computations to get the principal components, versus 1 large SVD of complexity ``O(n_samples * n_features ** 2)`` for PCA. + For a usage example, see + :ref:`sphx_glr_auto_examples_decomposition_plot_incremental_pca.py`. + Read more in the :ref:`User Guide `. .. versionadded:: 0.16 @@ -69,7 +77,10 @@ class IncrementalPCA(_BasePCA): Attributes ---------- components_ : ndarray of shape (n_components, n_features) - Components with maximum variance. + Principal axes in feature space, representing the directions of + maximum variance in the data. Equivalently, the right singular + vectors of the centered input data, parallel to its eigenvectors. + The components are sorted by decreasing ``explained_variance_``. explained_variance_ : ndarray of shape (n_components,) Variance explained by each of the selected components. @@ -108,21 +119,23 @@ class IncrementalPCA(_BasePCA): batch_size_ : int Inferred batch size from ``batch_size``. - Examples + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also -------- - >>> from sklearn.datasets import load_digits - >>> from sklearn.decomposition import IncrementalPCA - >>> from scipy import sparse - >>> X, _ = load_digits(return_X_y=True) - >>> transformer = IncrementalPCA(n_components=7, batch_size=200) - >>> # either partially fit on smaller batches of data - >>> transformer.partial_fit(X[:100, :]) - IncrementalPCA(batch_size=200, n_components=7) - >>> # or let the fit function itself divide the data into batches - >>> X_sparse = sparse.csr_matrix(X) - >>> X_transformed = transformer.fit_transform(X_sparse) - >>> X_transformed.shape - (1797, 7) + PCA : Principal component analysis (PCA). + KernelPCA : Kernel Principal component analysis (KPCA). + SparsePCA : Sparse Principal Components Analysis (SparsePCA). + TruncatedSVD : Dimensionality reduction using truncated SVD. Notes ----- @@ -133,10 +146,9 @@ class IncrementalPCA(_BasePCA): See https://www.cs.toronto.edu/~dross/ivt/RossLimLinYang_ijcv.pdf This model is an extension of the Sequential Karhunen-Loeve Transform from: - *A. Levy and M. Lindenbaum, Sequential Karhunen-Loeve Basis Extraction and + :doi:`A. Levy and M. Lindenbaum, Sequential Karhunen-Loeve Basis Extraction and its Application to Images, IEEE Transactions on Image Processing, Volume 9, - Number 8, pp. 1371-1374, August 2000.* - See https://www.cs.technion.ac.il/~mic/doc/skl-ip.pdf + Number 8, pp. 1371-1374, August 2000. <10.1109/83.855432>` We have specifically abstained from an optimization used by authors of both papers, a QR decomposition used in specific situations to reduce the @@ -157,31 +169,50 @@ class IncrementalPCA(_BasePCA): G. Golub and C. Van Loan. Matrix Computations, Third Edition, Chapter 5, Section 5.4.4, pp. 252-253. - See Also + Examples -------- - PCA - KernelPCA - SparsePCA - TruncatedSVD + >>> from sklearn.datasets import load_digits + >>> from sklearn.decomposition import IncrementalPCA + >>> from scipy import sparse + >>> X, _ = load_digits(return_X_y=True) + >>> transformer = IncrementalPCA(n_components=7, batch_size=200) + >>> # either partially fit on smaller batches of data + >>> transformer.partial_fit(X[:100, :]) + IncrementalPCA(batch_size=200, n_components=7) + >>> # or let the fit function itself divide the data into batches + >>> X_sparse = sparse.csr_matrix(X) + >>> X_transformed = transformer.fit_transform(X_sparse) + >>> X_transformed.shape + (1797, 7) """ - @_deprecate_positional_args - def __init__(self, n_components=None, *, whiten=False, copy=True, - batch_size=None): + + __metadata_request__partial_fit = {"check_input": metadata_routing.UNUSED} + + _parameter_constraints: dict = { + "n_components": [Interval(Integral, 1, None, closed="left"), None], + "whiten": ["boolean"], + "copy": ["boolean"], + "batch_size": [Interval(Integral, 1, None, closed="left"), None], + } + + def __init__(self, n_components=None, *, whiten=False, copy=True, batch_size=None): self.n_components = n_components self.whiten = whiten self.copy = copy self.batch_size = batch_size + @_fit_context(prefer_skip_nested_validation=True) def fit(self, X, y=None): """Fit the model with X, using minibatches of size batch_size. Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) - Training data, where n_samples is the number of samples and - n_features is the number of features. + Training data, where `n_samples` is the number of samples and + `n_features` is the number of features. y : Ignored + Not used, present for API consistency by convention. Returns ------- @@ -190,15 +221,21 @@ def fit(self, X, y=None): """ self.components_ = None self.n_samples_seen_ = 0 - self.mean_ = .0 - self.var_ = .0 + self.mean_ = 0.0 + self.var_ = 0.0 self.singular_values_ = None self.explained_variance_ = None self.explained_variance_ratio_ = None self.noise_variance_ = None - X = self._validate_data(X, accept_sparse=['csr', 'csc', 'lil'], - copy=self.copy, dtype=[np.float64, np.float32]) + X = validate_data( + self, + X, + accept_sparse=["csr", "csc", "lil"], + copy=self.copy, + dtype=[np.float64, np.float32], + force_writeable=True, + ) n_samples, n_features = X.shape if self.batch_size is None: @@ -206,8 +243,9 @@ def fit(self, X, y=None): else: self.batch_size_ = self.batch_size - for batch in gen_batches(n_samples, self.batch_size_, - min_batch_size=self.n_components or 0): + for batch in gen_batches( + n_samples, self.batch_size_, min_batch_size=self.n_components or 0 + ): X_batch = X[batch] if sparse.issparse(X_batch): X_batch = X_batch.toarray() @@ -215,35 +253,44 @@ def fit(self, X, y=None): return self + @_fit_context(prefer_skip_nested_validation=True) def partial_fit(self, X, y=None, check_input=True): """Incremental fit with X. All of X is processed as a single batch. Parameters ---------- X : array-like of shape (n_samples, n_features) - Training data, where n_samples is the number of samples and - n_features is the number of features. + Training data, where `n_samples` is the number of samples and + `n_features` is the number of features. + + y : Ignored + Not used, present for API consistency by convention. check_input : bool, default=True Run check_array on X. - y : Ignored - Returns ------- self : object Returns the instance itself. """ first_pass = not hasattr(self, "components_") + if check_input: if sparse.issparse(X): raise TypeError( "IncrementalPCA.partial_fit does not support " "sparse input. Either convert data to dense " - "or use IncrementalPCA.fit to do so in batches.") - X = self._validate_data( - X, copy=self.copy, dtype=[np.float64, np.float32], - reset=first_pass) + "or use IncrementalPCA.fit to do so in batches." + ) + X = validate_data( + self, + X, + copy=self.copy, + dtype=[np.float64, np.float32], + force_writeable=True, + reset=first_pass, + ) n_samples, n_features = X.shape if first_pass: self.components_ = None @@ -253,35 +300,44 @@ def partial_fit(self, X, y=None, check_input=True): self.n_components_ = min(n_samples, n_features) else: self.n_components_ = self.components_.shape[0] - elif not 1 <= self.n_components <= n_features: - raise ValueError("n_components=%r invalid for n_features=%d, need " - "more rows than columns for IncrementalPCA " - "processing" % (self.n_components, n_features)) - elif not self.n_components <= n_samples: - raise ValueError("n_components=%r must be less or equal to " - "the batch number of samples " - "%d." % (self.n_components, n_samples)) + elif not self.n_components <= n_features: + raise ValueError( + "n_components=%r invalid for n_features=%d, need " + "more rows than columns for IncrementalPCA " + "processing" % (self.n_components, n_features) + ) + elif self.n_components > n_samples and first_pass: + raise ValueError( + f"n_components={self.n_components} must be less or equal to " + f"the batch number of samples {n_samples} for the first " + "partial_fit call." + ) else: self.n_components_ = self.n_components - if (self.components_ is not None) and (self.components_.shape[0] != - self.n_components_): - raise ValueError("Number of input features has changed from %i " - "to %i between calls to partial_fit! Try " - "setting n_components to a fixed value." % - (self.components_.shape[0], self.n_components_)) + if (self.components_ is not None) and ( + self.components_.shape[0] != self.n_components_ + ): + raise ValueError( + "Number of input features has changed from %i " + "to %i between calls to partial_fit! Try " + "setting n_components to a fixed value." + % (self.components_.shape[0], self.n_components_) + ) # This is the first partial_fit - if not hasattr(self, 'n_samples_seen_'): + if not hasattr(self, "n_samples_seen_"): self.n_samples_seen_ = 0 - self.mean_ = .0 - self.var_ = .0 + self.mean_ = 0.0 + self.var_ = 0.0 # Update stats - they are 0 if this is the first step - col_mean, col_var, n_total_samples = \ - _incremental_mean_and_var( - X, last_mean=self.mean_, last_variance=self.var_, - last_sample_count=np.repeat(self.n_samples_seen_, X.shape[1])) + col_mean, col_var, n_total_samples = _incremental_mean_and_var( + X, + last_mean=self.mean_, + last_variance=self.var_, + last_sample_count=np.repeat(self.n_samples_seen_, X.shape[1]), + ) n_total_samples = n_total_samples[0] # Whitening @@ -292,30 +348,34 @@ def partial_fit(self, X, y=None, check_input=True): col_batch_mean = np.mean(X, axis=0) X -= col_batch_mean # Build matrix of combined previous basis and new data - mean_correction = \ - np.sqrt((self.n_samples_seen_ / n_total_samples) * - n_samples) * (self.mean_ - col_batch_mean) - X = np.vstack((self.singular_values_.reshape((-1, 1)) * - self.components_, X, mean_correction)) + mean_correction = np.sqrt( + (self.n_samples_seen_ / n_total_samples) * n_samples + ) * (self.mean_ - col_batch_mean) + X = np.vstack( + ( + self.singular_values_.reshape((-1, 1)) * self.components_, + X, + mean_correction, + ) + ) U, S, Vt = linalg.svd(X, full_matrices=False, check_finite=False) U, Vt = svd_flip(U, Vt, u_based_decision=False) - explained_variance = S ** 2 / (n_total_samples - 1) - explained_variance_ratio = S ** 2 / np.sum(col_var * n_total_samples) + explained_variance = S**2 / (n_total_samples - 1) + explained_variance_ratio = S**2 / np.sum(col_var * n_total_samples) self.n_samples_seen_ = n_total_samples - self.components_ = Vt[:self.n_components_] - self.singular_values_ = S[:self.n_components_] + self.components_ = Vt[: self.n_components_] + self.singular_values_ = S[: self.n_components_] self.mean_ = col_mean self.var_ = col_var - self.explained_variance_ = explained_variance[:self.n_components_] - self.explained_variance_ratio_ = \ - explained_variance_ratio[:self.n_components_] - if self.n_components_ < n_features: - self.noise_variance_ = \ - explained_variance[self.n_components_:].mean() + self.explained_variance_ = explained_variance[: self.n_components_] + self.explained_variance_ratio_ = explained_variance_ratio[: self.n_components_] + # we already checked `self.n_components <= n_samples` above + if self.n_components_ not in (n_samples, n_features): + self.noise_variance_ = explained_variance[self.n_components_ :].mean() else: - self.noise_variance_ = 0. + self.noise_variance_ = 0.0 return self def transform(self, X): @@ -328,12 +388,13 @@ def transform(self, X): Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) - New data, where n_samples is the number of samples - and n_features is the number of features. + New data, where `n_samples` is the number of samples + and `n_features` is the number of features. Returns ------- X_new : ndarray of shape (n_samples, n_components) + Projection of X in the first principal components. Examples -------- @@ -350,9 +411,16 @@ def transform(self, X): if sparse.issparse(X): n_samples = X.shape[0] output = [] - for batch in gen_batches(n_samples, self.batch_size_, - min_batch_size=self.n_components or 0): + for batch in gen_batches( + n_samples, self.batch_size_, min_batch_size=self.n_components or 0 + ): output.append(super().transform(X[batch].toarray())) return np.vstack(output) else: return super().transform(X) + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + # Beware that fit accepts sparse data but partial_fit doesn't + tags.input_tags.sparse = True + return tags diff --git a/sklearn/decomposition/_kernel_pca.py b/sklearn/decomposition/_kernel_pca.py index 5655eddb0bf31..79573651eeb84 100644 --- a/sklearn/decomposition/_kernel_pca.py +++ b/sklearn/decomposition/_kernel_pca.py @@ -1,28 +1,52 @@ """Kernel Principal Components Analysis.""" -# Author: Mathieu Blondel -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from numbers import Integral, Real import numpy as np from scipy import linalg +from scipy.linalg import eigh from scipy.sparse.linalg import eigsh -from ..utils._arpack import _init_arpack_v0 -from ..utils.extmath import svd_flip -from ..utils.validation import check_is_fitted, _check_psd_eigenvalues -from ..utils.deprecation import deprecated +from ..base import ( + BaseEstimator, + ClassNamePrefixFeaturesOutMixin, + TransformerMixin, + _fit_context, +) from ..exceptions import NotFittedError -from ..base import BaseEstimator, TransformerMixin -from ..preprocessing import KernelCenterer from ..metrics.pairwise import pairwise_kernels -from ..utils.validation import _deprecate_positional_args +from ..preprocessing import KernelCenterer +from ..utils._arpack import _init_arpack_v0 +from ..utils._param_validation import Interval, StrOptions +from ..utils.extmath import _randomized_eigsh, svd_flip +from ..utils.validation import ( + _check_psd_eigenvalues, + check_is_fitted, + validate_data, +) -class KernelPCA(TransformerMixin, BaseEstimator): +class KernelPCA(ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator): """Kernel Principal component analysis (KPCA). - Non-linear dimensionality reduction through the use of kernels (see - :ref:`metrics`). + Non-linear dimensionality reduction through the use of kernels [1]_, see also + :ref:`metrics`. + + It uses the :func:`scipy.linalg.eigh` LAPACK implementation of the full SVD + or the :func:`scipy.sparse.linalg.eigsh` ARPACK implementation of the + truncated SVD, depending on the shape of the input data and the number of + components to extract. It can also use a randomized truncated SVD by the + method proposed in [3]_, see `eigen_solver`. + + For a usage example and comparison between + Principal Components Analysis (PCA) and its kernelized version (KPCA), see + :ref:`sphx_glr_auto_examples_decomposition_plot_kernel_pca.py`. + + For a usage example in denoising images using KPCA, see + :ref:`sphx_glr_auto_examples_applications_plot_digits_denoising.py`. Read more in the :ref:`User Guide `. @@ -31,15 +55,15 @@ class KernelPCA(TransformerMixin, BaseEstimator): n_components : int, default=None Number of components. If None, all non-zero components are kept. - kernel : {'linear', 'poly', \ - 'rbf', 'sigmoid', 'cosine', 'precomputed'}, default='linear' + kernel : {'linear', 'poly', 'rbf', 'sigmoid', 'cosine', 'precomputed'} \ + or callable, default='linear' Kernel used for PCA. gamma : float, default=None Kernel coefficient for rbf, poly and sigmoid kernels. Ignored by other kernels. If ``gamma`` is ``None``, then it is set to ``1/n_features``. - degree : int, default=3 + degree : float, default=3 Degree for poly kernels. Ignored by other kernels. coef0 : float, default=1 @@ -56,13 +80,42 @@ class KernelPCA(TransformerMixin, BaseEstimator): inverse transform (when fit_inverse_transform=True). fit_inverse_transform : bool, default=False - Learn the inverse transform for non-precomputed kernels. - (i.e. learn to find the pre-image of a point) - - eigen_solver : {'auto', 'dense', 'arpack'}, default='auto' - Select eigensolver to use. If n_components is much less than - the number of training samples, arpack may be more efficient - than the dense eigensolver. + Learn the inverse transform for non-precomputed kernels + (i.e. learn to find the pre-image of a point). This method is based + on [2]_. + + eigen_solver : {'auto', 'dense', 'arpack', 'randomized'}, \ + default='auto' + Select eigensolver to use. If `n_components` is much + less than the number of training samples, randomized (or arpack to a + smaller extent) may be more efficient than the dense eigensolver. + Randomized SVD is performed according to the method of Halko et al + [3]_. + + auto : + the solver is selected by a default policy based on n_samples + (the number of training samples) and `n_components`: + if the number of components to extract is less than 10 (strict) and + the number of samples is more than 200 (strict), the 'arpack' + method is enabled. Otherwise the exact full eigenvalue + decomposition is computed and optionally truncated afterwards + ('dense' method). + dense : + run exact full eigenvalue decomposition calling the standard + LAPACK solver via `scipy.linalg.eigh`, and select the components + by postprocessing + arpack : + run SVD truncated to n_components calling ARPACK solver using + `scipy.sparse.linalg.eigsh`. It requires strictly + 0 < n_components < n_samples + randomized : + run randomized SVD by the method of Halko et al. [3]_. The current + implementation selects eigenvalues based on their module; therefore + using this method can lead to unexpected results if the kernel is + not positive semi-definite. See also [4]_. + + .. versionchanged:: 1.0 + `'randomized'` was added. tol : float, default=0 Convergence tolerance for arpack. @@ -72,6 +125,13 @@ class KernelPCA(TransformerMixin, BaseEstimator): Maximum number of iterations for arpack. If None, optimal value will be chosen by arpack. + iterated_power : int >= 0, or 'auto', default='auto' + Number of iterations for the power method computed by + svd_solver == 'randomized'. When 'auto', it is set to 7 when + `n_components < 0.1 * min(X.shape)`, other it is set to 4. + + .. versionadded:: 1.0 + remove_zero_eig : bool, default=False If True, then all components with zero eigenvalues are removed, so that the number of components in the output may be < n_components @@ -80,8 +140,8 @@ class KernelPCA(TransformerMixin, BaseEstimator): with zero eigenvalues are removed regardless. random_state : int, RandomState instance or None, default=None - Used when ``eigen_solver`` == 'arpack'. Pass an int for reproducible - results across multiple function calls. + Used when ``eigen_solver`` == 'arpack' or 'randomized'. Pass an int + for reproducible results across multiple function calls. See :term:`Glossary `. .. versionadded:: 0.18 @@ -103,12 +163,12 @@ class KernelPCA(TransformerMixin, BaseEstimator): Attributes ---------- - lambdas_ : ndarray of shape (n_components,) + eigenvalues_ : ndarray of shape (n_components,) Eigenvalues of the centered kernel matrix in decreasing order. If `n_components` and `remove_zero_eig` are not set, then all values are stored. - alphas_ : ndarray of shape (n_samples, n_components) + eigenvectors_ : ndarray of shape (n_samples, n_components) Eigenvectors of the centered kernel matrix. If `n_components` and `remove_zero_eig` are not set, then all components are stored. @@ -124,6 +184,56 @@ class KernelPCA(TransformerMixin, BaseEstimator): The data used to fit the model. If `copy_X=False`, then `X_fit_` is a reference. This attribute is used for the calls to transform. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + gamma_ : float + Kernel coefficient for rbf, poly and sigmoid kernels. When `gamma` + is explicitly provided, this is just the same as `gamma`. When `gamma` + is `None`, this is the actual value of kernel coefficient. + + .. versionadded:: 1.3 + + See Also + -------- + FastICA : A fast algorithm for Independent Component Analysis. + IncrementalPCA : Incremental Principal Component Analysis. + NMF : Non-Negative Matrix Factorization. + PCA : Principal Component Analysis. + SparsePCA : Sparse Principal Component Analysis. + TruncatedSVD : Dimensionality reduction using truncated SVD. + + References + ---------- + .. [1] `SchÃļlkopf, Bernhard, Alexander Smola, and Klaus-Robert MÃŧller. + "Kernel principal component analysis." + International conference on artificial neural networks. + Springer, Berlin, Heidelberg, 1997. + `_ + + .. [2] `BakÄąr, GÃļkhan H., Jason Weston, and Bernhard SchÃļlkopf. + "Learning to find pre-images." + Advances in neural information processing systems 16 (2004): 449-456. + `_ + + .. [3] :arxiv:`Halko, Nathan, Per-Gunnar Martinsson, and Joel A. Tropp. + "Finding structure with randomness: Probabilistic algorithms for + constructing approximate matrix decompositions." + SIAM review 53.2 (2011): 217-288. <0909.4061>` + + .. [4] `Martinsson, Per-Gunnar, Vladimir Rokhlin, and Mark Tygert. + "A randomized algorithm for the decomposition of matrices." + Applied and Computational Harmonic Analysis 30.1 (2011): 47-68. + `_ + Examples -------- >>> from sklearn.datasets import load_digits @@ -133,24 +243,62 @@ class KernelPCA(TransformerMixin, BaseEstimator): >>> X_transformed = transformer.fit_transform(X) >>> X_transformed.shape (1797, 7) - - References - ---------- - Kernel PCA was introduced in: - Bernhard Schoelkopf, Alexander J. Smola, - and Klaus-Robert Mueller. 1999. Kernel principal - component analysis. In Advances in kernel methods, - MIT Press, Cambridge, MA, USA 327-352. """ - @_deprecate_positional_args - def __init__(self, n_components=None, *, kernel="linear", - gamma=None, degree=3, coef0=1, kernel_params=None, - alpha=1.0, fit_inverse_transform=False, eigen_solver='auto', - tol=0, max_iter=None, remove_zero_eig=False, - random_state=None, copy_X=True, n_jobs=None): - if fit_inverse_transform and kernel == 'precomputed': - raise ValueError( - "Cannot fit_inverse_transform with a precomputed kernel.") + + _parameter_constraints: dict = { + "n_components": [ + Interval(Integral, 1, None, closed="left"), + None, + ], + "kernel": [ + StrOptions({"linear", "poly", "rbf", "sigmoid", "cosine", "precomputed"}), + callable, + ], + "gamma": [ + Interval(Real, 0, None, closed="left"), + None, + ], + "degree": [Interval(Real, 0, None, closed="left")], + "coef0": [Interval(Real, None, None, closed="neither")], + "kernel_params": [dict, None], + "alpha": [Interval(Real, 0, None, closed="left")], + "fit_inverse_transform": ["boolean"], + "eigen_solver": [StrOptions({"auto", "dense", "arpack", "randomized"})], + "tol": [Interval(Real, 0, None, closed="left")], + "max_iter": [ + Interval(Integral, 1, None, closed="left"), + None, + ], + "iterated_power": [ + Interval(Integral, 0, None, closed="left"), + StrOptions({"auto"}), + ], + "remove_zero_eig": ["boolean"], + "random_state": ["random_state"], + "copy_X": ["boolean"], + "n_jobs": [None, Integral], + } + + def __init__( + self, + n_components=None, + *, + kernel="linear", + gamma=None, + degree=3, + coef0=1, + kernel_params=None, + alpha=1.0, + fit_inverse_transform=False, + eigen_solver="auto", + tol=0, + max_iter=None, + iterated_power="auto", + remove_zero_eig=False, + random_state=None, + copy_X=True, + n_jobs=None, + ): self.n_components = n_components self.kernel = kernel self.kernel_params = kernel_params @@ -160,79 +308,79 @@ def __init__(self, n_components=None, *, kernel="linear", self.alpha = alpha self.fit_inverse_transform = fit_inverse_transform self.eigen_solver = eigen_solver - self.remove_zero_eig = remove_zero_eig self.tol = tol self.max_iter = max_iter + self.iterated_power = iterated_power + self.remove_zero_eig = remove_zero_eig self.random_state = random_state self.n_jobs = n_jobs self.copy_X = copy_X - # TODO: Remove in 1.1 - # mypy error: Decorated property not supported - @deprecated("Attribute _pairwise was deprecated in " # type: ignore - "version 0.24 and will be removed in 1.1 (renaming of 0.26).") - @property - def _pairwise(self): - return self.kernel == "precomputed" - def _get_kernel(self, X, Y=None): if callable(self.kernel): params = self.kernel_params or {} else: - params = {"gamma": self.gamma, - "degree": self.degree, - "coef0": self.coef0} - return pairwise_kernels(X, Y, metric=self.kernel, - filter_params=True, n_jobs=self.n_jobs, - **params) - - def _fit_transform(self, K): - """ Fit's using kernel K""" - # center kernel - K = self._centerer.fit_transform(K) + params = {"gamma": self.gamma_, "degree": self.degree, "coef0": self.coef0} + return pairwise_kernels( + X, Y, metric=self.kernel, filter_params=True, n_jobs=self.n_jobs, **params + ) + def _fit_transform_in_place(self, K): + """Fit's using kernel K""" + # center kernel in place + K = self._centerer.fit(K).transform(K, copy=False) + + # adjust n_components according to user inputs if self.n_components is None: - n_components = K.shape[0] + n_components = K.shape[0] # use all dimensions else: n_components = min(K.shape[0], self.n_components) # compute eigenvectors - if self.eigen_solver == 'auto': + if self.eigen_solver == "auto": if K.shape[0] > 200 and n_components < 10: - eigen_solver = 'arpack' + eigen_solver = "arpack" else: - eigen_solver = 'dense' + eigen_solver = "dense" else: eigen_solver = self.eigen_solver - if eigen_solver == 'dense': - self.lambdas_, self.alphas_ = linalg.eigh( - K, eigvals=(K.shape[0] - n_components, K.shape[0] - 1)) - elif eigen_solver == 'arpack': + if eigen_solver == "dense": + # Note: subset_by_index specifies the indices of smallest/largest to return + self.eigenvalues_, self.eigenvectors_ = eigh( + K, subset_by_index=(K.shape[0] - n_components, K.shape[0] - 1) + ) + elif eigen_solver == "arpack": v0 = _init_arpack_v0(K.shape[0], self.random_state) - self.lambdas_, self.alphas_ = eigsh(K, n_components, - which="LA", - tol=self.tol, - maxiter=self.max_iter, - v0=v0) + self.eigenvalues_, self.eigenvectors_ = eigsh( + K, n_components, which="LA", tol=self.tol, maxiter=self.max_iter, v0=v0 + ) + elif eigen_solver == "randomized": + self.eigenvalues_, self.eigenvectors_ = _randomized_eigsh( + K, + n_components=n_components, + n_iter=self.iterated_power, + random_state=self.random_state, + selection="module", + ) # make sure that the eigenvalues are ok and fix numerical issues - self.lambdas_ = _check_psd_eigenvalues(self.lambdas_, - enable_warnings=False) + self.eigenvalues_ = _check_psd_eigenvalues( + self.eigenvalues_, enable_warnings=False + ) # flip eigenvectors' sign to enforce deterministic output - self.alphas_, _ = svd_flip(self.alphas_, - np.zeros_like(self.alphas_).T) + self.eigenvectors_, _ = svd_flip(u=self.eigenvectors_, v=None) # sort eigenvectors in descending order - indices = self.lambdas_.argsort()[::-1] - self.lambdas_ = self.lambdas_[indices] - self.alphas_ = self.alphas_[:, indices] + indices = self.eigenvalues_.argsort()[::-1] + self.eigenvalues_ = self.eigenvalues_[indices] + self.eigenvectors_ = self.eigenvectors_[:, indices] # remove eigenvectors with a zero eigenvalue (null space) if required if self.remove_zero_eig or self.n_components is None: - self.alphas_ = self.alphas_[:, self.lambdas_ > 0] - self.lambdas_ = self.lambdas_[self.lambdas_ > 0] + self.eigenvectors_ = self.eigenvectors_[:, self.eigenvalues_ > 0] + self.eigenvalues_ = self.eigenvalues_[self.eigenvalues_ > 0] # Maintenance note on Eigenvectors normalization # ---------------------------------------------- @@ -243,12 +391,12 @@ def _fit_transform(self, K): # if u is an eigenvector of Phi(X)Phi(X)' # then Phi(X)'u is an eigenvector of Phi(X)'Phi(X) # - # At this stage our self.alphas_ (the v) have norm 1, we need to scale + # At this stage our self.eigenvectors_ (the v) have norm 1, we need to scale # them so that eigenvectors in kernel feature space (the u) have norm=1 # instead # # We COULD scale them here: - # self.alphas_ = self.alphas_ / np.sqrt(self.lambdas_) + # self.eigenvectors_ = self.eigenvectors_ / np.sqrt(self.eigenvalues_) # # But choose to perform that LATER when needed, in `fit()` and in # `transform()`. @@ -257,37 +405,47 @@ def _fit_transform(self, K): def _fit_inverse_transform(self, X_transformed, X): if hasattr(X, "tocsr"): - raise NotImplementedError("Inverse transform not implemented for " - "sparse matrices!") + raise NotImplementedError( + "Inverse transform not implemented for sparse matrices!" + ) n_samples = X_transformed.shape[0] K = self._get_kernel(X_transformed) - K.flat[::n_samples + 1] += self.alpha - self.dual_coef_ = linalg.solve(K, X, sym_pos=True, overwrite_a=True) + K.flat[:: n_samples + 1] += self.alpha + self.dual_coef_ = linalg.solve(K, X, assume_a="pos", overwrite_a=True) self.X_transformed_fit_ = X_transformed + @_fit_context(prefer_skip_nested_validation=True) def fit(self, X, y=None): """Fit the model from data in X. Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) - Training vector, where n_samples in the number of samples - and n_features is the number of features. + Training vector, where `n_samples` is the number of samples + and `n_features` is the number of features. + + y : Ignored + Not used, present for API consistency by convention. Returns ------- self : object Returns the instance itself. """ - X = self._validate_data(X, accept_sparse='csr', copy=self.copy_X) - self._centerer = KernelCenterer() + if self.fit_inverse_transform and self.kernel == "precomputed": + raise ValueError("Cannot fit_inverse_transform with a precomputed kernel.") + X = validate_data(self, X, accept_sparse="csr", copy=self.copy_X) + self.gamma_ = 1 / X.shape[1] if self.gamma is None else self.gamma + self._centerer = KernelCenterer().set_output(transform="default") K = self._get_kernel(X) - self._fit_transform(K) + # When kernel="precomputed", K is X but it's safe to perform in place operations + # on K because a copy was made before if requested by copy_X. + self._fit_transform_in_place(K) if self.fit_inverse_transform: # no need to use the kernel to transform X, use shortcut expression - X_transformed = self.alphas_ * np.sqrt(self.lambdas_) + X_transformed = self.eigenvectors_ * np.sqrt(self.eigenvalues_) self._fit_inverse_transform(X_transformed, X) @@ -300,17 +458,25 @@ def fit_transform(self, X, y=None, **params): Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) - Training vector, where n_samples in the number of samples - and n_features is the number of features. + Training vector, where `n_samples` is the number of samples + and `n_features` is the number of features. + + y : Ignored + Not used, present for API consistency by convention. + + **params : kwargs + Parameters (keyword arguments) and values passed to + the fit_transform instance. Returns ------- X_new : ndarray of shape (n_samples, n_components) + Returns the instance itself. """ self.fit(X, **params) # no need to use the kernel to transform X, use shortcut expression - X_transformed = self.alphas_ * np.sqrt(self.lambdas_) + X_transformed = self.eigenvectors_ * np.sqrt(self.eigenvalues_) if self.fit_inverse_transform: self._fit_inverse_transform(X_transformed, X) @@ -323,22 +489,26 @@ def transform(self, X): Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training vector, where `n_samples` is the number of samples + and `n_features` is the number of features. Returns ------- X_new : ndarray of shape (n_samples, n_components) + Returns the instance itself. """ check_is_fitted(self) - X = self._validate_data(X, accept_sparse='csr', reset=False) + X = validate_data(self, X, accept_sparse="csr", reset=False) # Compute centered gram matrix between X and training data X_fit_ K = self._centerer.transform(self._get_kernel(X, self.X_fit_)) # scale eigenvectors (properly account for null-space for dot product) - non_zeros = np.flatnonzero(self.lambdas_) - scaled_alphas = np.zeros_like(self.alphas_) - scaled_alphas[:, non_zeros] = (self.alphas_[:, non_zeros] - / np.sqrt(self.lambdas_[non_zeros])) + non_zeros = np.flatnonzero(self.eigenvalues_) + scaled_alphas = np.zeros_like(self.eigenvectors_) + scaled_alphas[:, non_zeros] = self.eigenvectors_[:, non_zeros] / np.sqrt( + self.eigenvalues_[non_zeros] + ) # Project with a scalar product between K and the scaled eigenvectors return np.dot(K, scaled_alphas) @@ -346,28 +516,62 @@ def transform(self, X): def inverse_transform(self, X): """Transform X back to original space. + ``inverse_transform`` approximates the inverse transformation using + a learned pre-image. The pre-image is learned by kernel ridge + regression of the original data on their low-dimensional representation + vectors. + + .. note: + :meth:`~sklearn.decomposition.fit` internally uses a centered + kernel. As the centered kernel no longer contains the information + of the mean of kernel features, such information is not taken into + account in reconstruction. + + .. note:: + When users want to compute inverse transformation for 'linear' + kernel, it is recommended that they use + :class:`~sklearn.decomposition.PCA` instead. Unlike + :class:`~sklearn.decomposition.PCA`, + :class:`~sklearn.decomposition.KernelPCA`'s ``inverse_transform`` + does not reconstruct the mean of data when 'linear' kernel is used + due to the use of centered kernel. + Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_components) + Training vector, where `n_samples` is the number of samples + and `n_features` is the number of features. Returns ------- - X_new : ndarray of shape (n_samples, n_features) + X_original : ndarray of shape (n_samples, n_features) + Returns the instance itself. References ---------- - "Learning to Find Pre-Images", G BakIr et al, 2004. + `BakÄąr, GÃļkhan H., Jason Weston, and Bernhard SchÃļlkopf. + "Learning to find pre-images." + Advances in neural information processing systems 16 (2004): 449-456. + `_ """ if not self.fit_inverse_transform: - raise NotFittedError("The fit_inverse_transform parameter was not" - " set to True when instantiating and hence " - "the inverse transform is not available.") + raise NotFittedError( + "The fit_inverse_transform parameter was not" + " set to True when instantiating and hence " + "the inverse transform is not available." + ) K = self._get_kernel(X, self.X_transformed_fit_) - n_samples = self.X_transformed_fit_.shape[0] - K.flat[::n_samples + 1] += self.alpha return np.dot(K, self.dual_coef_) - def _more_tags(self): - return {'preserves_dtype': [np.float64, np.float32], - 'pairwise': self.kernel == 'precomputed'} + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.sparse = True + tags.transformer_tags.preserves_dtype = ["float64", "float32"] + tags.input_tags.pairwise = self.kernel == "precomputed" + return tags + + @property + def _n_features_out(self): + """Number of transformed output features.""" + return self.eigenvalues_.shape[0] diff --git a/sklearn/decomposition/_lda.py b/sklearn/decomposition/_lda.py index e554d299fe478..94b1413745a22 100644 --- a/sklearn/decomposition/_lda.py +++ b/sklearn/decomposition/_lda.py @@ -8,30 +8,48 @@ Link: https://github.com/blei-lab/onlineldavb """ -# Author: Chyi-Kwei Yau -# Author: Matthew D. Hoffman (original onlineldavb implementation) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from numbers import Integral, Real import numpy as np import scipy.sparse as sp +from joblib import effective_n_jobs from scipy.special import gammaln, logsumexp -from joblib import Parallel, effective_n_jobs -from ..base import BaseEstimator, TransformerMixin +from ..base import ( + BaseEstimator, + ClassNamePrefixFeaturesOutMixin, + TransformerMixin, + _fit_context, +) from ..utils import check_random_state, gen_batches, gen_even_slices -from ..utils.validation import check_non_negative -from ..utils.validation import check_is_fitted -from ..utils.validation import _deprecate_positional_args -from ..utils.fixes import delayed - -from ._online_lda_fast import (mean_change, _dirichlet_expectation_1d, - _dirichlet_expectation_2d) +from ..utils._param_validation import Interval, StrOptions +from ..utils.parallel import Parallel, delayed +from ..utils.validation import check_is_fitted, check_non_negative, validate_data +from ._online_lda_fast import ( + _dirichlet_expectation_1d as cy_dirichlet_expectation_1d, +) +from ._online_lda_fast import ( + _dirichlet_expectation_2d, +) +from ._online_lda_fast import ( + mean_change as cy_mean_change, +) EPS = np.finfo(float).eps -def _update_doc_distribution(X, exp_topic_word_distr, doc_topic_prior, - max_iters, - mean_change_tol, cal_sstats, random_state): +def _update_doc_distribution( + X, + exp_topic_word_distr, + doc_topic_prior, + max_doc_update_iter, + mean_change_tol, + cal_sstats, + random_state, +): """E-step: update document-topic distribution. Parameters @@ -46,12 +64,12 @@ def _update_doc_distribution(X, exp_topic_word_distr, doc_topic_prior, doc_topic_prior : float Prior of document topic distribution `theta`. - max_iters : int + max_doc_update_iter : int Max number of iterations for updating document topic distribution in the E-step. mean_change_tol : float - Stopping tolerance for updating document topic distribution in E-setp. + Stopping tolerance for updating document topic distribution in E-step. cal_sstats : bool Parameter that indicate to calculate sufficient statistics or not. @@ -77,25 +95,38 @@ def _update_doc_distribution(X, exp_topic_word_distr, doc_topic_prior, n_topics = exp_topic_word_distr.shape[0] if random_state: - doc_topic_distr = random_state.gamma(100., 0.01, (n_samples, n_topics)) + doc_topic_distr = random_state.gamma(100.0, 0.01, (n_samples, n_topics)).astype( + X.dtype, copy=False + ) else: - doc_topic_distr = np.ones((n_samples, n_topics)) + doc_topic_distr = np.ones((n_samples, n_topics), dtype=X.dtype) # In the literature, this is `exp(E[log(theta)])` exp_doc_topic = np.exp(_dirichlet_expectation_2d(doc_topic_distr)) # diff on `component_` (only calculate it when `cal_diff` is True) - suff_stats = np.zeros(exp_topic_word_distr.shape) if cal_sstats else None + suff_stats = ( + np.zeros(exp_topic_word_distr.shape, dtype=X.dtype) if cal_sstats else None + ) if is_sparse_x: X_data = X.data X_indices = X.indices X_indptr = X.indptr + # These cython functions are called in a nested loop on usually very small arrays + # (length=n_topics). In that case, finding the appropriate signature of the + # fused-typed function can be more costly than its execution, hence the dispatch + # is done outside of the loop. + ctype = "float" if X.dtype == np.float32 else "double" + mean_change = cy_mean_change[ctype] + dirichlet_expectation_1d = cy_dirichlet_expectation_1d[ctype] + eps = np.finfo(X.dtype).eps + for idx_d in range(n_samples): if is_sparse_x: - ids = X_indices[X_indptr[idx_d]:X_indptr[idx_d + 1]] - cnts = X_data[X_indptr[idx_d]:X_indptr[idx_d + 1]] + ids = X_indices[X_indptr[idx_d] : X_indptr[idx_d + 1]] + cnts = X_data[X_indptr[idx_d] : X_indptr[idx_d + 1]] else: ids = np.nonzero(X[idx_d, :])[0] cnts = X[idx_d, ids] @@ -106,18 +137,16 @@ def _update_doc_distribution(X, exp_topic_word_distr, doc_topic_prior, exp_topic_word_d = exp_topic_word_distr[:, ids] # Iterate between `doc_topic_d` and `norm_phi` until convergence - for _ in range(0, max_iters): + for _ in range(0, max_doc_update_iter): last_d = doc_topic_d # The optimal phi_{dwk} is proportional to # exp(E[log(theta_{dk})]) * exp(E[log(beta_{dw})]). - norm_phi = np.dot(exp_doc_topic_d, exp_topic_word_d) + EPS + norm_phi = np.dot(exp_doc_topic_d, exp_topic_word_d) + eps - doc_topic_d = (exp_doc_topic_d * - np.dot(cnts / norm_phi, exp_topic_word_d.T)) + doc_topic_d = exp_doc_topic_d * np.dot(cnts / norm_phi, exp_topic_word_d.T) # Note: adds doc_topic_prior to doc_topic_d, in-place. - _dirichlet_expectation_1d(doc_topic_d, doc_topic_prior, - exp_doc_topic_d) + dirichlet_expectation_1d(doc_topic_d, doc_topic_prior, exp_doc_topic_d) if mean_change(last_d, doc_topic_d) < mean_change_tol: break @@ -126,14 +155,18 @@ def _update_doc_distribution(X, exp_topic_word_distr, doc_topic_prior, # Contribution of document d to the expected sufficient # statistics for the M step. if cal_sstats: - norm_phi = np.dot(exp_doc_topic_d, exp_topic_word_d) + EPS + norm_phi = np.dot(exp_doc_topic_d, exp_topic_word_d) + eps suff_stats[:, ids] += np.outer(exp_doc_topic_d, cnts / norm_phi) return (doc_topic_distr, suff_stats) -class LatentDirichletAllocation(TransformerMixin, BaseEstimator): - """Latent Dirichlet Allocation with online variational Bayes algorithm +class LatentDirichletAllocation( + ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator +): + """Latent Dirichlet Allocation with online variational Bayes algorithm. + + The implementation is based on [1]_ and [2]_. .. versionadded:: 0.17 @@ -162,15 +195,14 @@ class LatentDirichletAllocation(TransformerMixin, BaseEstimator): In general, if the data size is large, the online update will be much faster than the batch update. - Valid options:: + Valid options: - 'batch': Batch variational Bayes method. Use all training data in - each EM update. - Old `components_` will be overwritten in each iteration. - 'online': Online variational Bayes method. In each EM update, use - mini-batch of training data to update the ``components_`` - variable incrementally. The learning rate is controlled by the - ``learning_decay`` and the ``learning_offset`` parameters. + - 'batch': Batch variational Bayes method. Use all training data in each EM + update. Old `components_` will be overwritten in each iteration. + - 'online': Online variational Bayes method. In each EM update, use mini-batch + of training data to update the ``components_`` variable incrementally. The + learning rate is controlled by the ``learning_decay`` and the + ``learning_offset`` parameters. .. versionchanged:: 0.20 The default learning method is now ``"batch"``. @@ -182,13 +214,15 @@ class LatentDirichletAllocation(TransformerMixin, BaseEstimator): ``n_samples``, the update method is same as batch learning. In the literature, this is called kappa. - learning_offset : float, default=10. + learning_offset : float, default=10.0 A (positive) parameter that downweights early iterations in online learning. It should be greater than 1.0. In the literature, this is called tau_0. max_iter : int, default=10 - The maximum number of iterations. + The maximum number of passes over the training data (aka epochs). + It only impacts the behavior in the :meth:`fit` method, and not the + :meth:`partial_fit` method. batch_size : int, default=128 Number of documents to use in each EM iteration. Only used in online @@ -206,8 +240,7 @@ class LatentDirichletAllocation(TransformerMixin, BaseEstimator): Total number of documents. Only used in the :meth:`partial_fit` method. perp_tol : float, default=1e-1 - Perplexity tolerance in batch learning. Only used when - ``evaluate_every`` is greater than 0. + Perplexity tolerance. Only used when ``evaluate_every`` is greater than 0. mean_change_tol : float, default=1e-3 Stopping tolerance for updating document topic distribution in E-step. @@ -247,6 +280,17 @@ class LatentDirichletAllocation(TransformerMixin, BaseEstimator): n_batch_iter_ : int Number of iterations of the EM step. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + n_iter_ : int Number of passes over the dataset. @@ -265,6 +309,21 @@ class LatentDirichletAllocation(TransformerMixin, BaseEstimator): Prior of topic word distribution `beta`. If the value is None, it is `1 / n_components`. + See Also + -------- + sklearn.discriminant_analysis.LinearDiscriminantAnalysis: + A classifier with a linear decision boundary, generated by fitting + class conditional densities to the data and using Bayes' rule. + + References + ---------- + .. [1] "Online Learning for Latent Dirichlet Allocation", Matthew D. + Hoffman, David M. Blei, Francis Bach, 2010 + https://github.com/blei-lab/onlineldavb + + .. [2] "Stochastic Variational Inference", Matthew D. Hoffman, + David M. Blei, Chong Wang, John Paisley, 2013 + Examples -------- >>> from sklearn.decomposition import LatentDirichletAllocation @@ -280,26 +339,47 @@ class LatentDirichletAllocation(TransformerMixin, BaseEstimator): >>> lda.transform(X[-2:]) array([[0.00360392, 0.25499205, 0.0036211 , 0.64236448, 0.09541846], [0.15297572, 0.00362644, 0.44412786, 0.39568399, 0.003586 ]]) - - References - ---------- - .. [1] "Online Learning for Latent Dirichlet Allocation", Matthew D. - Hoffman, David M. Blei, Francis Bach, 2010 - - [2] "Stochastic Variational Inference", Matthew D. Hoffman, David M. Blei, - Chong Wang, John Paisley, 2013 - - [3] Matthew D. Hoffman's onlineldavb code. Link: - https://github.com/blei-lab/onlineldavb - """ - @_deprecate_positional_args - def __init__(self, n_components=10, *, doc_topic_prior=None, - topic_word_prior=None, learning_method='batch', - learning_decay=.7, learning_offset=10., max_iter=10, - batch_size=128, evaluate_every=-1, total_samples=1e6, - perp_tol=1e-1, mean_change_tol=1e-3, max_doc_update_iter=100, - n_jobs=None, verbose=0, random_state=None): + + _parameter_constraints: dict = { + "n_components": [Interval(Integral, 0, None, closed="neither")], + "doc_topic_prior": [None, Interval(Real, 0, 1, closed="both")], + "topic_word_prior": [None, Interval(Real, 0, 1, closed="both")], + "learning_method": [StrOptions({"batch", "online"})], + "learning_decay": [Interval(Real, 0, 1, closed="both")], + "learning_offset": [Interval(Real, 1.0, None, closed="left")], + "max_iter": [Interval(Integral, 0, None, closed="left")], + "batch_size": [Interval(Integral, 0, None, closed="neither")], + "evaluate_every": [Interval(Integral, None, None, closed="neither")], + "total_samples": [Interval(Real, 0, None, closed="neither")], + "perp_tol": [Interval(Real, 0, None, closed="left")], + "mean_change_tol": [Interval(Real, 0, None, closed="left")], + "max_doc_update_iter": [Interval(Integral, 0, None, closed="left")], + "n_jobs": [None, Integral], + "verbose": ["verbose"], + "random_state": ["random_state"], + } + + def __init__( + self, + n_components=10, + *, + doc_topic_prior=None, + topic_word_prior=None, + learning_method="batch", + learning_decay=0.7, + learning_offset=10.0, + max_iter=10, + batch_size=128, + evaluate_every=-1, + total_samples=1e6, + perp_tol=1e-1, + mean_change_tol=1e-3, + max_doc_update_iter=100, + n_jobs=None, + verbose=0, + random_state=None, + ): self.n_components = n_components self.doc_topic_prior = doc_topic_prior self.topic_word_prior = topic_word_prior @@ -317,25 +397,7 @@ def __init__(self, n_components=10, *, doc_topic_prior=None, self.verbose = verbose self.random_state = random_state - def _check_params(self): - """Check model parameters.""" - if self.n_components <= 0: - raise ValueError("Invalid 'n_components' parameter: %r" - % self.n_components) - - if self.total_samples <= 0: - raise ValueError("Invalid 'total_samples' parameter: %r" - % self.total_samples) - - if self.learning_offset < 0: - raise ValueError("Invalid 'learning_offset' parameter: %r" - % self.learning_offset) - - if self.learning_method not in ("batch", "online"): - raise ValueError("Invalid 'learning_method' parameter: %r" - % self.learning_method) - - def _init_latent_vars(self, n_features): + def _init_latent_vars(self, n_features, dtype=np.float64): """Initialize latent variables.""" self.random_state_ = check_random_state(self.random_state) @@ -343,24 +405,26 @@ def _init_latent_vars(self, n_features): self.n_iter_ = 0 if self.doc_topic_prior is None: - self.doc_topic_prior_ = 1. / self.n_components + self.doc_topic_prior_ = 1.0 / self.n_components else: self.doc_topic_prior_ = self.doc_topic_prior if self.topic_word_prior is None: - self.topic_word_prior_ = 1. / self.n_components + self.topic_word_prior_ = 1.0 / self.n_components else: self.topic_word_prior_ = self.topic_word_prior - init_gamma = 100. - init_var = 1. / init_gamma + init_gamma = 100.0 + init_var = 1.0 / init_gamma # In the literature, this is called `lambda` self.components_ = self.random_state_.gamma( - init_gamma, init_var, (self.n_components, n_features)) + init_gamma, init_var, (self.n_components, n_features) + ).astype(dtype, copy=False) # In the literature, this is `exp(E[log(beta)])` self.exp_dirichlet_component_ = np.exp( - _dirichlet_expectation_2d(self.components_)) + _dirichlet_expectation_2d(self.components_) + ) def _e_step(self, X, cal_sstats, random_init, parallel=None): """E-step in EM update. @@ -398,16 +462,19 @@ def _e_step(self, X, cal_sstats, random_init, parallel=None): # TODO: make Parallel._effective_n_jobs public instead? n_jobs = effective_n_jobs(self.n_jobs) if parallel is None: - parallel = Parallel(n_jobs=n_jobs, verbose=max(0, - self.verbose - 1)) + parallel = Parallel(n_jobs=n_jobs, verbose=max(0, self.verbose - 1)) results = parallel( - delayed(_update_doc_distribution)(X[idx_slice, :], - self.exp_dirichlet_component_, - self.doc_topic_prior_, - self.max_doc_update_iter, - self.mean_change_tol, cal_sstats, - random_state) - for idx_slice in gen_even_slices(X.shape[0], n_jobs)) + delayed(_update_doc_distribution)( + X[idx_slice, :], + self.exp_dirichlet_component_, + self.doc_topic_prior_, + self.max_doc_update_iter, + self.mean_change_tol, + cal_sstats, + random_state, + ) + for idx_slice in gen_even_slices(X.shape[0], n_jobs) + ) # merge result doc_topics, sstats_list = zip(*results) @@ -416,7 +483,7 @@ def _e_step(self, X, cal_sstats, random_init, parallel=None): if cal_sstats: # This step finishes computing the sufficient statistics for the # M-step. - suff_stats = np.zeros(self.components_.shape) + suff_stats = np.zeros(self.components_.shape, dtype=self.components_.dtype) for sstats in sstats_list: suff_stats += sstats suff_stats *= self.exp_dirichlet_component_ @@ -428,7 +495,7 @@ def _e_step(self, X, cal_sstats, random_init, parallel=None): def _em_step(self, X, total_samples, batch_update, parallel=None): """EM update for 1 iteration. - update `_component` by batch VB or online VB. + update `component_` by batch VB or online VB. Parameters ---------- @@ -453,8 +520,9 @@ def _em_step(self, X, total_samples, batch_update, parallel=None): """ # E-step - _, suff_stats = self._e_step(X, cal_sstats=True, random_init=True, - parallel=parallel) + _, suff_stats = self._e_step( + X, cal_sstats=True, random_init=True, parallel=parallel + ) # M-step if batch_update: @@ -462,21 +530,28 @@ def _em_step(self, X, total_samples, batch_update, parallel=None): else: # online update # In the literature, the weight is `rho` - weight = np.power(self.learning_offset + self.n_batch_iter_, - -self.learning_decay) + weight = np.power( + self.learning_offset + self.n_batch_iter_, -self.learning_decay + ) doc_ratio = float(total_samples) / X.shape[0] - self.components_ *= (1 - weight) - self.components_ += (weight * (self.topic_word_prior_ - + doc_ratio * suff_stats)) + self.components_ *= 1 - weight + self.components_ += weight * ( + self.topic_word_prior_ + doc_ratio * suff_stats + ) # update `component_` related variables self.exp_dirichlet_component_ = np.exp( - _dirichlet_expectation_2d(self.components_)) + _dirichlet_expectation_2d(self.components_) + ) self.n_batch_iter_ += 1 return - def _more_tags(self): - return {'requires_positive_X': True} + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.positive_only = True + tags.input_tags.sparse = True + tags.transformer_tags.preserves_dtype = ["float32", "float64"] + return tags def _check_non_neg_array(self, X, reset_n_features, whom): """check X format @@ -488,11 +563,20 @@ def _check_non_neg_array(self, X, reset_n_features, whom): X : array-like or sparse matrix """ - X = self._validate_data(X, reset=reset_n_features, - accept_sparse='csr') + dtype = [np.float64, np.float32] if reset_n_features else self.components_.dtype + + X = validate_data( + self, + X, + reset=reset_n_features, + accept_sparse="csr", + dtype=dtype, + ) check_non_negative(X, whom) + return X + @_fit_context(prefer_skip_nested_validation=True) def partial_fit(self, X, y=None): """Online VB with Mini-Batch update. @@ -502,40 +586,45 @@ def partial_fit(self, X, y=None): Document word matrix. y : Ignored + Not used, present here for API consistency by convention. Returns ------- self + Partially fitted estimator. """ - self._check_params() - first_time = not hasattr(self, 'components_') + first_time = not hasattr(self, "components_") + X = self._check_non_neg_array( - X, reset_n_features=first_time, - whom="LatentDirichletAllocation.partial_fit") + X, reset_n_features=first_time, whom="LatentDirichletAllocation.partial_fit" + ) n_samples, n_features = X.shape batch_size = self.batch_size # initialize parameters or check if first_time: - self._init_latent_vars(n_features) + self._init_latent_vars(n_features, dtype=X.dtype) if n_features != self.components_.shape[1]: raise ValueError( "The provided data has %d dimensions while " - "the model was trained with feature size %d." % - (n_features, self.components_.shape[1])) + "the model was trained with feature size %d." + % (n_features, self.components_.shape[1]) + ) n_jobs = effective_n_jobs(self.n_jobs) - with Parallel(n_jobs=n_jobs, - verbose=max(0, self.verbose - 1)) as parallel: + with Parallel(n_jobs=n_jobs, verbose=max(0, self.verbose - 1)) as parallel: for idx_slice in gen_batches(n_samples, batch_size): - self._em_step(X[idx_slice, :], - total_samples=self.total_samples, - batch_update=False, - parallel=parallel) + self._em_step( + X[idx_slice, :], + total_samples=self.total_samples, + batch_update=False, + parallel=parallel, + ) return self + @_fit_context(prefer_skip_nested_validation=True) def fit(self, X, y=None): """Learn model for the data X with variational Bayes method. @@ -548,14 +637,16 @@ def fit(self, X, y=None): Document word matrix. y : Ignored + Not used, present here for API consistency by convention. Returns ------- self + Fitted estimator. """ - self._check_params() - X = self._check_non_neg_array(X, reset_n_features=True, - whom="LatentDirichletAllocation.fit") + X = self._check_non_neg_array( + X, reset_n_features=True, whom="LatentDirichletAllocation.fit" + ) n_samples, n_features = X.shape max_iter = self.max_iter evaluate_every = self.evaluate_every @@ -564,47 +655,55 @@ def fit(self, X, y=None): batch_size = self.batch_size # initialize parameters - self._init_latent_vars(n_features) + self._init_latent_vars(n_features, dtype=X.dtype) # change to perplexity later last_bound = None n_jobs = effective_n_jobs(self.n_jobs) - with Parallel(n_jobs=n_jobs, - verbose=max(0, self.verbose - 1)) as parallel: + with Parallel(n_jobs=n_jobs, verbose=max(0, self.verbose - 1)) as parallel: for i in range(max_iter): - if learning_method == 'online': + if learning_method == "online": for idx_slice in gen_batches(n_samples, batch_size): - self._em_step(X[idx_slice, :], total_samples=n_samples, - batch_update=False, parallel=parallel) + self._em_step( + X[idx_slice, :], + total_samples=n_samples, + batch_update=False, + parallel=parallel, + ) else: # batch update - self._em_step(X, total_samples=n_samples, - batch_update=True, parallel=parallel) + self._em_step( + X, total_samples=n_samples, batch_update=True, parallel=parallel + ) # check perplexity if evaluate_every > 0 and (i + 1) % evaluate_every == 0: - doc_topics_distr, _ = self._e_step(X, cal_sstats=False, - random_init=False, - parallel=parallel) - bound = self._perplexity_precomp_distr(X, doc_topics_distr, - sub_sampling=False) + doc_topics_distr, _ = self._e_step( + X, cal_sstats=False, random_init=False, parallel=parallel + ) + bound = self._perplexity_precomp_distr( + X, doc_topics_distr, sub_sampling=False + ) if self.verbose: - print('iteration: %d of max_iter: %d, perplexity: %.4f' - % (i + 1, max_iter, bound)) + print( + "iteration: %d of max_iter: %d, perplexity: %.4f" + % (i + 1, max_iter, bound) + ) if last_bound and abs(last_bound - bound) < self.perp_tol: break last_bound = bound elif self.verbose: - print('iteration: %d of max_iter: %d' % (i + 1, max_iter)) + print("iteration: %d of max_iter: %d" % (i + 1, max_iter)) self.n_iter_ += 1 # calculate final perplexity value on train set - doc_topics_distr, _ = self._e_step(X, cal_sstats=False, - random_init=False, - parallel=parallel) - self.bound_ = self._perplexity_precomp_distr(X, doc_topics_distr, - sub_sampling=False) + doc_topics_distr, _ = self._e_step( + X, cal_sstats=False, random_init=False, parallel=parallel + ) + self.bound_ = self._perplexity_precomp_distr( + X, doc_topics_distr, sub_sampling=False + ) return self @@ -621,35 +720,24 @@ def _unnormalized_transform(self, X): doc_topic_distr : ndarray of shape (n_samples, n_components) Document topic distribution for X. """ - check_is_fitted(self) - - # make sure feature size is the same in fitted model and in X - X = self._check_non_neg_array( - X, reset_n_features=True, - whom="LatentDirichletAllocation.transform") - n_samples, n_features = X.shape - if n_features != self.components_.shape[1]: - raise ValueError( - "The provided data has %d dimensions while " - "the model was trained with feature size %d." % - (n_features, self.components_.shape[1])) - - doc_topic_distr, _ = self._e_step(X, cal_sstats=False, - random_init=False) + doc_topic_distr, _ = self._e_step(X, cal_sstats=False, random_init=False) return doc_topic_distr - def transform(self, X): + def transform(self, X, *, normalize=True): """Transform data X according to the fitted model. - .. versionchanged:: 0.18 - *doc_topic_distr* is now normalized + .. versionchanged:: 0.18 + `doc_topic_distr` is now normalized. Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) Document word matrix. + normalize : bool, default=True + Whether to normalize the document topic distribution. + Returns ------- doc_topic_distr : ndarray of shape (n_samples, n_components) @@ -657,12 +745,38 @@ def transform(self, X): """ check_is_fitted(self) X = self._check_non_neg_array( - X, reset_n_features=False, - whom="LatentDirichletAllocation.transform") + X, reset_n_features=False, whom="LatentDirichletAllocation.transform" + ) doc_topic_distr = self._unnormalized_transform(X) - doc_topic_distr /= doc_topic_distr.sum(axis=1)[:, np.newaxis] + if normalize: + doc_topic_distr /= doc_topic_distr.sum(axis=1)[:, np.newaxis] return doc_topic_distr + def fit_transform(self, X, y=None, *, normalize=True): + """ + Fit to data, then transform it. + + Fits transformer to `X` and `y` and returns a transformed version of `X`. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Input samples. + + y : array-like of shape (n_samples,) or (n_samples, n_outputs), \ + default=None + Target values (None for unsupervised transformations). + + normalize : bool, default=True + Whether to normalize the document topic distribution in `transform`. + + Returns + ------- + X_new : ndarray array of shape (n_samples, n_components) + Transformed array. + """ + return self.fit(X, y).transform(X, normalize=normalize) + def _approx_bound(self, X, doc_topic_distr, sub_sampling): """Estimate the variational bound. @@ -714,19 +828,21 @@ def _loglikelihood(prior, distr, dirichlet_distr, size): # E[log p(docs | theta, beta)] for idx_d in range(0, n_samples): if is_sparse_x: - ids = X_indices[X_indptr[idx_d]:X_indptr[idx_d + 1]] - cnts = X_data[X_indptr[idx_d]:X_indptr[idx_d + 1]] + ids = X_indices[X_indptr[idx_d] : X_indptr[idx_d + 1]] + cnts = X_data[X_indptr[idx_d] : X_indptr[idx_d + 1]] else: ids = np.nonzero(X[idx_d, :])[0] cnts = X[idx_d, ids] - temp = (dirichlet_doc_topic[idx_d, :, np.newaxis] - + dirichlet_component_[:, ids]) + temp = ( + dirichlet_doc_topic[idx_d, :, np.newaxis] + dirichlet_component_[:, ids] + ) norm_phi = logsumexp(temp, axis=0) score += np.dot(cnts, norm_phi) # compute E[log p(theta | alpha) - log q(theta | gamma)] - score += _loglikelihood(doc_topic_prior, doc_topic_distr, - dirichlet_doc_topic, self.n_components) + score += _loglikelihood( + doc_topic_prior, doc_topic_distr, dirichlet_doc_topic, self.n_components + ) # Compensate for the subsampling of the population of documents if sub_sampling: @@ -734,8 +850,9 @@ def _loglikelihood(prior, distr, dirichlet_distr, size): score *= doc_ratio # E[log p(beta | eta) - log q (beta | lambda)] - score += _loglikelihood(topic_word_prior, self.components_, - dirichlet_component_, n_features) + score += _loglikelihood( + topic_word_prior, self.components_, dirichlet_component_, n_features + ) return score @@ -748,6 +865,7 @@ def score(self, X, y=None): Document word matrix. y : Ignored + Not used, present here for API consistency by convention. Returns ------- @@ -755,15 +873,15 @@ def score(self, X, y=None): Use approximate bound as score. """ check_is_fitted(self) - X = self._check_non_neg_array(X, reset_n_features=False, - whom="LatentDirichletAllocation.score") + X = self._check_non_neg_array( + X, reset_n_features=False, whom="LatentDirichletAllocation.score" + ) doc_topic_distr = self._unnormalized_transform(X) score = self._approx_bound(X, doc_topic_distr, sub_sampling=False) return score - def _perplexity_precomp_distr(self, X, doc_topic_distr=None, - sub_sampling=False): + def _perplexity_precomp_distr(self, X, doc_topic_distr=None, sub_sampling=False): """Calculate approximate perplexity for data X with ability to accept precomputed doc_topic_distr @@ -784,19 +902,14 @@ def _perplexity_precomp_distr(self, X, doc_topic_distr=None, score : float Perplexity score. """ - check_is_fitted(self) - - X = self._check_non_neg_array( - X, reset_n_features=True, - whom="LatentDirichletAllocation.perplexity") - if doc_topic_distr is None: doc_topic_distr = self._unnormalized_transform(X) else: n_samples, n_components = doc_topic_distr.shape if n_samples != X.shape[0]: - raise ValueError("Number of samples in X and doc_topic_distr" - " do not match.") + raise ValueError( + "Number of samples in X and doc_topic_distr do not match." + ) if n_components != self.n_components: raise ValueError("Number of topics does not match.") @@ -834,4 +947,13 @@ def perplexity(self, X, sub_sampling=False): score : float Perplexity score. """ + check_is_fitted(self) + X = self._check_non_neg_array( + X, reset_n_features=True, whom="LatentDirichletAllocation.perplexity" + ) return self._perplexity_precomp_distr(X, sub_sampling=sub_sampling) + + @property + def _n_features_out(self): + """Number of transformed output features.""" + return self.components_.shape[0] diff --git a/sklearn/decomposition/_nmf.py b/sklearn/decomposition/_nmf.py index b978f1a33d3af..4c963538619a3 100644 --- a/sklearn/decomposition/_nmf.py +++ b/sklearn/decomposition/_nmf.py @@ -1,26 +1,40 @@ -""" Non-negative matrix factorization. -""" -# Author: Vlad Niculae -# Lars Buitinck -# Mathieu Blondel -# Tom Dupre la Tour -# License: BSD 3 clause - -import numbers -import numpy as np -import scipy.sparse as sp +"""Non-negative matrix factorization.""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import itertools import time import warnings +from abc import ABC from math import sqrt +from numbers import Integral, Real + +import numpy as np +import scipy.sparse as sp +from scipy import linalg -from ._cdnmf_fast import _update_cdnmf_fast from .._config import config_context -from ..base import BaseEstimator, TransformerMixin +from ..base import ( + BaseEstimator, + ClassNamePrefixFeaturesOutMixin, + TransformerMixin, + _fit_context, +) from ..exceptions import ConvergenceWarning -from ..utils import check_random_state, check_array -from ..utils.extmath import randomized_svd, safe_sparse_dot, squared_norm -from ..utils.validation import check_is_fitted, check_non_negative -from ..utils.validation import _deprecate_positional_args +from ..utils import check_array, check_random_state, gen_batches +from ..utils._param_validation import ( + Interval, + StrOptions, + validate_params, +) +from ..utils.extmath import _randomized_svd, safe_sparse_dot, squared_norm +from ..utils.validation import ( + check_is_fitted, + check_non_negative, + validate_data, +) +from ._cdnmf_fast import _update_cdnmf_fast EPSILON = np.finfo(np.float32).eps @@ -28,7 +42,7 @@ def norm(x): """Dot product-based Euclidean norm implementation. - See: http://fseoane.net/blog/2011/computing-the-vector-norm/ + See: http://fa.bianp.net/blog/2011/computing-the-vector-norm/ Parameters ---------- @@ -53,12 +67,19 @@ def trace_dot(X, Y): def _check_init(A, shape, whom): A = check_array(A) - if np.shape(A) != shape: - raise ValueError('Array with wrong shape passed to %s. Expected %s, ' - 'but got %s ' % (whom, shape, np.shape(A))) + if shape[0] != "auto" and A.shape[0] != shape[0]: + raise ValueError( + f"Array with wrong first dimension passed to {whom}. Expected {shape[0]}, " + f"but got {A.shape[0]}." + ) + if shape[1] != "auto" and A.shape[1] != shape[1]: + raise ValueError( + f"Array with wrong second dimension passed to {whom}. Expected {shape[1]}, " + f"but got {A.shape[1]}." + ) check_non_negative(A, whom) if np.max(A) == 0: - raise ValueError('Array passed to %s is full of zeros.' % whom) + raise ValueError(f"Array passed to {whom} is full of zeros.") def _beta_divergence(X, W, H, beta, square_root=False): @@ -102,10 +123,10 @@ def _beta_divergence(X, W, H, beta, square_root=False): if sp.issparse(X): norm_X = np.dot(X.data, X.data) norm_WH = trace_dot(np.linalg.multi_dot([W.T, W, H]), H) - cross_prod = trace_dot((X * H.T), W) - res = (norm_X + norm_WH - 2. * cross_prod) / 2. + cross_prod = trace_dot((X @ H.T), W) + res = (norm_X + norm_WH - 2.0 * cross_prod) / 2.0 else: - res = squared_norm(X - np.dot(W, H)) / 2. + res = squared_norm(X - np.dot(W, H)) / 2.0 if square_root: return np.sqrt(res * 2) @@ -127,7 +148,7 @@ def _beta_divergence(X, W, H, beta, square_root=False): X_data = X_data[indices] # used to avoid division by zero - WH_data[WH_data == 0] = EPSILON + WH_data[WH_data < EPSILON] = EPSILON # generalized Kullback-Leibler divergence if beta == 1: @@ -142,7 +163,7 @@ def _beta_divergence(X, W, H, beta, square_root=False): # Itakura-Saito divergence elif beta == 0: div = X_data / WH_data - res = np.sum(div) - np.product(X.shape) - np.sum(np.log(div)) + res = np.sum(div) - np.prod(X.shape) - np.sum(np.log(div)) # beta-divergence, beta not in (0, 1, 2) else: @@ -154,14 +175,15 @@ def _beta_divergence(X, W, H, beta, square_root=False): sum_WH_beta += np.sum(np.dot(W, H[:, i]) ** beta) else: - sum_WH_beta = np.sum(WH ** beta) + sum_WH_beta = np.sum(WH**beta) sum_X_WH = np.dot(X_data, WH_data ** (beta - 1)) - res = (X_data ** beta).sum() - beta * sum_X_WH + res = (X_data**beta).sum() - beta * sum_X_WH res += sum_WH_beta * (beta - 1) res /= beta * (beta - 1) if square_root: + res = max(res, 0) # avoid negative number due to rounding errors return np.sqrt(2 * res) else: return res @@ -178,8 +200,9 @@ def _special_sparse_dot(W, H, X): batch_size = max(n_components, n_vals // n_components) for start in range(0, n_vals, batch_size): batch = slice(start, start + batch_size) - dot_vals[batch] = np.multiply(W[ii[batch], :], - H.T[jj[batch], :]).sum(axis=1) + dot_vals[batch] = np.multiply(W[ii[batch], :], H.T[jj[batch], :]).sum( + axis=1 + ) WH = sp.coo_matrix((dot_vals, (ii, jj)), shape=X.shape) return WH.tocsr() @@ -187,69 +210,15 @@ def _special_sparse_dot(W, H, X): return np.dot(W, H) -def _compute_regularization(alpha, l1_ratio, regularization): - """Compute L1 and L2 regularization coefficients for W and H.""" - alpha_H = 0. - alpha_W = 0. - if regularization in ('both', 'components'): - alpha_H = float(alpha) - if regularization in ('both', 'transformation'): - alpha_W = float(alpha) - - l1_reg_W = alpha_W * l1_ratio - l1_reg_H = alpha_H * l1_ratio - l2_reg_W = alpha_W * (1. - l1_ratio) - l2_reg_H = alpha_H * (1. - l1_ratio) - return l1_reg_W, l1_reg_H, l2_reg_W, l2_reg_H - - -def _check_string_param(solver, regularization, beta_loss, init): - allowed_solver = ('cd', 'mu') - if solver not in allowed_solver: - raise ValueError( - 'Invalid solver parameter: got %r instead of one of %r' % - (solver, allowed_solver)) - - allowed_regularization = ('both', 'components', 'transformation', None) - if regularization not in allowed_regularization: - raise ValueError( - 'Invalid regularization parameter: got %r instead of one of %r' % - (regularization, allowed_regularization)) - - # 'mu' is the only solver that handles other beta losses than 'frobenius' - if solver != 'mu' and beta_loss not in (2, 'frobenius'): - raise ValueError( - 'Invalid beta_loss parameter: solver %r does not handle beta_loss' - ' = %r' % (solver, beta_loss)) - - if solver == 'mu' and init == 'nndsvd': - warnings.warn("The multiplicative update ('mu') solver cannot update " - "zeros present in the initialization, and so leads to " - "poorer results when used jointly with init='nndsvd'. " - "You may try init='nndsvda' or init='nndsvdar' instead.", - UserWarning) - - beta_loss = _beta_loss_to_float(beta_loss) - return beta_loss - - def _beta_loss_to_float(beta_loss): """Convert string beta_loss to float.""" - allowed_beta_loss = {'frobenius': 2, - 'kullback-leibler': 1, - 'itakura-saito': 0} - if isinstance(beta_loss, str) and beta_loss in allowed_beta_loss: - beta_loss = allowed_beta_loss[beta_loss] - - if not isinstance(beta_loss, numbers.Number): - raise ValueError('Invalid beta_loss parameter: got %r instead ' - 'of one of %r, or a float.' % - (beta_loss, allowed_beta_loss.keys())) + beta_loss_map = {"frobenius": 2, "kullback-leibler": 1, "itakura-saito": 0} + if isinstance(beta_loss, str): + beta_loss = beta_loss_map[beta_loss] return beta_loss -def _initialize_nmf(X, n_components, init='warn', eps=1e-6, - random_state=None): +def _initialize_nmf(X, n_components, init=None, eps=1e-6, random_state=None): """Algorithms for NMF initialization. Computes an initial guess for the non-negative @@ -265,10 +234,9 @@ def _initialize_nmf(X, n_components, init='warn', eps=1e-6, init : {'random', 'nndsvd', 'nndsvda', 'nndsvdar'}, default=None Method used to initialize the procedure. - Default: None. Valid options: - - None: 'nndsvd' if n_components <= min(n_samples, n_features), + - None: 'nndsvda' if n_components <= min(n_samples, n_features), otherwise 'random'. - 'random': non-negative random matrices, scaled with: @@ -286,6 +254,10 @@ def _initialize_nmf(X, n_components, init='warn', eps=1e-6, - 'custom': use custom matrices W and H + .. versionchanged:: 1.1 + When `init=None` and n_components is less than n_samples and n_features + defaults to `nndsvda` instead of `nndsvd`. + eps : float, default=1e-6 Truncate all values less then this in output to zero. @@ -308,42 +280,41 @@ def _initialize_nmf(X, n_components, init='warn', eps=1e-6, nonnegative matrix factorization - Pattern Recognition, 2008 http://tinyurl.com/nndsvd """ - if init == 'warn': - warnings.warn(("The 'init' value, when 'init=None' and " - "n_components is less than n_samples and " - "n_features, will be changed from 'nndsvd' to " - "'nndsvda' in 1.1 (renaming of 0.26)."), FutureWarning) - init = None - check_non_negative(X, "NMF initialization") n_samples, n_features = X.shape - if (init is not None and init != 'random' - and n_components > min(n_samples, n_features)): - raise ValueError("init = '{}' can only be used when " - "n_components <= min(n_samples, n_features)" - .format(init)) + if ( + init is not None + and init != "random" + and n_components > min(n_samples, n_features) + ): + raise ValueError( + "init = '{}' can only be used when " + "n_components <= min(n_samples, n_features)".format(init) + ) if init is None: if n_components <= min(n_samples, n_features): - init = 'nndsvd' + init = "nndsvda" else: - init = 'random' + init = "random" # Random initialization - if init == 'random': + if init == "random": avg = np.sqrt(X.mean() / n_components) rng = check_random_state(random_state) - H = avg * rng.randn(n_components, n_features).astype(X.dtype, - copy=False) - W = avg * rng.randn(n_samples, n_components).astype(X.dtype, - copy=False) + H = avg * rng.standard_normal(size=(n_components, n_features)).astype( + X.dtype, copy=False + ) + W = avg * rng.standard_normal(size=(n_samples, n_components)).astype( + X.dtype, copy=False + ) np.abs(H, out=H) np.abs(W, out=W) return W, H # NNDSVD initialization - U, S, V = randomized_svd(X, n_components, random_state=random_state) + U, S, V = _randomized_svd(X, n_components, random_state=random_state) W = np.zeros_like(U) H = np.zeros_like(V) @@ -391,18 +362,18 @@ def _initialize_nmf(X, n_components, init='warn', eps=1e-6, elif init == "nndsvdar": rng = check_random_state(random_state) avg = X.mean() - W[W == 0] = abs(avg * rng.randn(len(W[W == 0])) / 100) - H[H == 0] = abs(avg * rng.randn(len(H[H == 0])) / 100) + W[W == 0] = abs(avg * rng.standard_normal(size=len(W[W == 0])) / 100) + H[H == 0] = abs(avg * rng.standard_normal(size=len(H[H == 0])) / 100) else: raise ValueError( - 'Invalid init parameter: got %r instead of one of %r' % - (init, (None, 'random', 'nndsvd', 'nndsvda', 'nndsvdar'))) + "Invalid init parameter: got %r instead of one of %r" + % (init, (None, "random", "nndsvd", "nndsvda", "nndsvdar")) + ) return W, H -def _update_coordinate_descent(X, W, Ht, l1_reg, l2_reg, shuffle, - random_state): +def _update_coordinate_descent(X, W, Ht, l1_reg, l2_reg, shuffle, random_state): """Helper function for _fit_coordinate_descent. Update W to minimize the objective function, iterating once over all @@ -416,11 +387,11 @@ def _update_coordinate_descent(X, W, Ht, l1_reg, l2_reg, shuffle, XHt = safe_sparse_dot(X, Ht) # L2 regularization corresponds to increase of the diagonal of HHt - if l2_reg != 0.: + if l2_reg != 0.0: # adds l2_reg only on the diagonal - HHt.flat[::n_components + 1] += l2_reg + HHt.flat[:: n_components + 1] += l2_reg # L1 regularization corresponds to decrease of each element of XHt - if l1_reg != 0.: + if l1_reg != 0.0: XHt -= l1_reg if shuffle: @@ -432,9 +403,21 @@ def _update_coordinate_descent(X, W, Ht, l1_reg, l2_reg, shuffle, return _update_cdnmf_fast(W, HHt, XHt, permutation) -def _fit_coordinate_descent(X, W, H, tol=1e-4, max_iter=200, l1_reg_W=0, - l1_reg_H=0, l2_reg_W=0, l2_reg_H=0, update_H=True, - verbose=0, shuffle=False, random_state=None): +def _fit_coordinate_descent( + X, + W, + H, + tol=1e-4, + max_iter=200, + l1_reg_W=0, + l1_reg_H=0, + l2_reg_W=0, + l2_reg_H=0, + update_H=True, + verbose=0, + shuffle=False, + random_state=None, +): """Compute Non-negative Matrix Factorization (NMF) with Coordinate Descent The objective function is minimized with an alternating minimization of W @@ -499,27 +482,29 @@ def _fit_coordinate_descent(X, W, H, tol=1e-4, max_iter=200, l1_reg_W=0, References ---------- - Cichocki, Andrzej, and Phan, Anh-Huy. "Fast local algorithms for - large scale nonnegative matrix and tensor factorizations." - IEICE transactions on fundamentals of electronics, communications and - computer sciences 92.3: 708-721, 2009. + .. [1] :doi:`"Fast local algorithms for large scale nonnegative matrix and tensor + factorizations" <10.1587/transfun.E92.A.708>` + Cichocki, Andrzej, and P. H. A. N. Anh-Huy. IEICE transactions on fundamentals + of electronics, communications and computer sciences 92.3: 708-721, 2009. """ # so W and Ht are both in C order in memory - Ht = check_array(H.T, order='C') - X = check_array(X, accept_sparse='csr') + Ht = check_array(H.T, order="C") + X = check_array(X, accept_sparse="csr") rng = check_random_state(random_state) for n_iter in range(1, max_iter + 1): - violation = 0. + violation = 0.0 # Update W - violation += _update_coordinate_descent(X, W, Ht, l1_reg_W, - l2_reg_W, shuffle, rng) + violation += _update_coordinate_descent( + X, W, Ht, l1_reg_W, l2_reg_W, shuffle, rng + ) # Update H if update_H: - violation += _update_coordinate_descent(X.T, Ht, W, l1_reg_H, - l2_reg_H, shuffle, rng) + violation += _update_coordinate_descent( + X.T, Ht, W, l1_reg_H, l2_reg_H, shuffle, rng + ) if n_iter == 1: violation_init = violation @@ -538,8 +523,19 @@ def _fit_coordinate_descent(X, W, H, tol=1e-4, max_iter=200, l1_reg_W=0, return W, Ht.T, n_iter -def _multiplicative_update_w(X, W, H, beta_loss, l1_reg_W, l2_reg_W, gamma, - H_sum=None, HHt=None, XHt=None, update_H=True): +def _multiplicative_update_w( + X, + W, + H, + beta_loss, + l1_reg_W, + l2_reg_W, + gamma, + H_sum=None, + HHt=None, + XHt=None, + update_H=True, +): """Update W in Multiplicative Update NMF.""" if beta_loss == 2: # Numerator @@ -569,12 +565,12 @@ def _multiplicative_update_w(X, W, H, beta_loss, l1_reg_W, l2_reg_W, gamma, X_data = X # copy used in the Denominator WH = WH_safe_X.copy() - if beta_loss - 1. < 0: - WH[WH == 0] = EPSILON + if beta_loss - 1.0 < 0: + WH[WH < EPSILON] = EPSILON # to avoid taking a negative power of zero - if beta_loss - 2. < 0: - WH_safe_X_data[WH_safe_X_data == 0] = EPSILON + if beta_loss - 2.0 < 0: + WH_safe_X_data[WH_safe_X_data < EPSILON] = EPSILON if beta_loss == 1: np.divide(X_data, WH_safe_X_data, out=WH_safe_X_data) @@ -608,7 +604,7 @@ def _multiplicative_update_w(X, W, H, beta_loss, l1_reg_W, l2_reg_W, gamma, for i in range(X.shape[0]): WHi = np.dot(W[i, :], H) if beta_loss - 1 < 0: - WHi[WHi == 0] = EPSILON + WHi[WHi < EPSILON] = EPSILON WHi **= beta_loss - 1 WHHt[i, :] = np.dot(WHi, H.T) else: @@ -630,11 +626,15 @@ def _multiplicative_update_w(X, W, H, beta_loss, l1_reg_W, l2_reg_W, gamma, if gamma != 1: delta_W **= gamma - return delta_W, H_sum, HHt, XHt + W *= delta_W + + return W, H_sum, HHt, XHt -def _multiplicative_update_h(X, W, H, beta_loss, l1_reg_H, l2_reg_H, gamma): - """Update H in Multiplicative Update NMF.""" +def _multiplicative_update_h( + X, W, H, beta_loss, l1_reg_H, l2_reg_H, gamma, A=None, B=None, rho=None +): + """update H in Multiplicative Update NMF.""" if beta_loss == 2: numerator = safe_sparse_dot(W.T, X) denominator = np.linalg.multi_dot([W.T, W, H]) @@ -650,12 +650,12 @@ def _multiplicative_update_h(X, W, H, beta_loss, l1_reg_H, l2_reg_H, gamma): X_data = X # copy used in the Denominator WH = WH_safe_X.copy() - if beta_loss - 1. < 0: - WH[WH == 0] = EPSILON + if beta_loss - 1.0 < 0: + WH[WH < EPSILON] = EPSILON # to avoid division by zero - if beta_loss - 2. < 0: - WH_safe_X_data[WH_safe_X_data == 0] = EPSILON + if beta_loss - 2.0 < 0: + WH_safe_X_data[WH_safe_X_data < EPSILON] = EPSILON if beta_loss == 1: np.divide(X_data, WH_safe_X_data, out=WH_safe_X_data) @@ -677,7 +677,7 @@ def _multiplicative_update_h(X, W, H, beta_loss, l1_reg_H, l2_reg_H, gamma): # Denominator if beta_loss == 1: W_sum = np.sum(W, axis=0) # shape(n_components, ) - W_sum[W_sum == 0] = 1. + W_sum[W_sum == 0] = 1.0 denominator = W_sum[:, np.newaxis] # beta_loss not in (1, 2) @@ -690,7 +690,7 @@ def _multiplicative_update_h(X, W, H, beta_loss, l1_reg_H, l2_reg_H, gamma): for i in range(X.shape[1]): WHi = np.dot(W, H[:, i]) if beta_loss - 1 < 0: - WHi[WHi == 0] = EPSILON + WHi[WHi < EPSILON] = EPSILON WHi **= beta_loss - 1 WtWH[:, i] = np.dot(W.T, WHi) else: @@ -705,20 +705,43 @@ def _multiplicative_update_h(X, W, H, beta_loss, l1_reg_H, l2_reg_H, gamma): denominator = denominator + l2_reg_H * H denominator[denominator == 0] = EPSILON - numerator /= denominator - delta_H = numerator - - # gamma is in ]0, 1] - if gamma != 1: - delta_H **= gamma - - return delta_H - - -def _fit_multiplicative_update(X, W, H, beta_loss='frobenius', - max_iter=200, tol=1e-4, - l1_reg_W=0, l1_reg_H=0, l2_reg_W=0, l2_reg_H=0, - update_H=True, verbose=0): + if A is not None and B is not None: + # Updates for the online nmf + if gamma != 1: + H **= 1 / gamma + numerator *= H + A *= rho + B *= rho + A += numerator + B += denominator + H = A / B + + if gamma != 1: + H **= gamma + else: + delta_H = numerator + delta_H /= denominator + if gamma != 1: + delta_H **= gamma + H *= delta_H + + return H + + +def _fit_multiplicative_update( + X, + W, + H, + beta_loss="frobenius", + max_iter=200, + tol=1e-4, + l1_reg_W=0, + l1_reg_H=0, + l2_reg_W=0, + l2_reg_H=0, + update_H=True, + verbose=0, +): """Compute Non-negative Matrix Factorization with Multiplicative Update. The objective function is _beta_divergence(X, WH) and is minimized with an @@ -783,6 +806,8 @@ def _fit_multiplicative_update(X, W, H, beta_loss='frobenius', References ---------- + Lee, D. D., & Seung, H., S. (2001). Algorithms for Non-negative Matrix + Factorization. Adv. Neural Inform. Process. Syst.. 13. Fevotte, C., & Idier, J. (2011). Algorithms for nonnegative matrix factorization with the beta-divergence. Neural Computation, 23(9). """ @@ -792,11 +817,11 @@ def _fit_multiplicative_update(X, W, H, beta_loss='frobenius', # gamma for Maximization-Minimization (MM) algorithm [Fevotte 2011] if beta_loss < 1: - gamma = 1. / (2. - beta_loss) + gamma = 1.0 / (2.0 - beta_loss) elif beta_loss > 2: - gamma = 1. / (beta_loss - 1.) + gamma = 1.0 / (beta_loss - 1.0) else: - gamma = 1. + gamma = 1.0 # used for the convergence criterion error_at_init = _beta_divergence(X, W, H, beta_loss, square_root=True) @@ -806,27 +831,42 @@ def _fit_multiplicative_update(X, W, H, beta_loss='frobenius', for n_iter in range(1, max_iter + 1): # update W # H_sum, HHt and XHt are saved and reused if not update_H - delta_W, H_sum, HHt, XHt = _multiplicative_update_w( - X, W, H, beta_loss, l1_reg_W, l2_reg_W, gamma, - H_sum, HHt, XHt, update_H) - W *= delta_W + W, H_sum, HHt, XHt = _multiplicative_update_w( + X, + W, + H, + beta_loss=beta_loss, + l1_reg_W=l1_reg_W, + l2_reg_W=l2_reg_W, + gamma=gamma, + H_sum=H_sum, + HHt=HHt, + XHt=XHt, + update_H=update_H, + ) # necessary for stability with beta_loss < 1 if beta_loss < 1: - W[W < np.finfo(np.float64).eps] = 0. + W[W < np.finfo(np.float64).eps] = 0.0 - # update H + # update H (only at fit or fit_transform) if update_H: - delta_H = _multiplicative_update_h(X, W, H, beta_loss, l1_reg_H, - l2_reg_H, gamma) - H *= delta_H + H = _multiplicative_update_h( + X, + W, + H, + beta_loss=beta_loss, + l1_reg_H=l1_reg_H, + l2_reg_H=l2_reg_H, + gamma=gamma, + ) # These values will be recomputed since H changed H_sum, HHt, XHt = None, None, None # necessary for stability with beta_loss < 1 if beta_loss <= 1: - H[H < np.finfo(np.float64).eps] = 0. + H[H < np.finfo(np.float64).eps] = 0.0 # test convergence criterion every 10 iterations if tol > 0 and n_iter % 10 == 0: @@ -834,8 +874,10 @@ def _fit_multiplicative_update(X, W, H, beta_loss='frobenius', if verbose: iter_time = time.time() - print("Epoch %02d reached after %.3f seconds, error: %f" % - (n_iter, iter_time - start_time, error)) + print( + "Epoch %02d reached after %.3f seconds, error: %f" + % (n_iter, iter_time - start_time, error) + ) if (previous_error - error) / error_at_init < tol: break @@ -844,19 +886,41 @@ def _fit_multiplicative_update(X, W, H, beta_loss='frobenius', # do not print if we have already printed in the convergence test if verbose and (tol == 0 or n_iter % 10 != 0): end_time = time.time() - print("Epoch %02d reached after %.3f seconds." % - (n_iter, end_time - start_time)) + print( + "Epoch %02d reached after %.3f seconds." % (n_iter, end_time - start_time) + ) return W, H, n_iter -@_deprecate_positional_args -def non_negative_factorization(X, W=None, H=None, n_components=None, *, - init='warn', update_H=True, solver='cd', - beta_loss='frobenius', tol=1e-4, - max_iter=200, alpha=0., l1_ratio=0., - regularization=None, random_state=None, - verbose=0, shuffle=False): +@validate_params( + { + "X": ["array-like", "sparse matrix"], + "W": ["array-like", None], + "H": ["array-like", None], + "update_H": ["boolean"], + }, + prefer_skip_nested_validation=False, +) +def non_negative_factorization( + X, + W=None, + H=None, + n_components="auto", + *, + init=None, + update_H=True, + solver="cd", + beta_loss="frobenius", + tol=1e-4, + max_iter=200, + alpha_W=0.0, + alpha_H="same", + l1_ratio=0.0, + random_state=None, + verbose=0, + shuffle=False, +): """Compute Non-negative Matrix Factorization (NMF). Find two non-negative matrices (W, H) whose product approximates the non- @@ -865,71 +929,89 @@ def non_negative_factorization(X, W=None, H=None, n_components=None, *, The objective function is: - .. math:: - - 0.5 * ||X - WH||_{loss}^2 + alpha * l1_{ratio} * ||vec(W)||_1 + .. math:: - + alpha * l1_{ratio} * ||vec(H)||_1 + L(W, H) &= 0.5 * ||X - WH||_{loss}^2 - + 0.5 * alpha * (1 - l1_{ratio}) * ||W||_{Fro}^2 + &+ alpha\\_W * l1\\_ratio * n\\_features * ||vec(W)||_1 - + 0.5 * alpha * (1 - l1_{ratio}) * ||H||_{Fro}^2 + &+ alpha\\_H * l1\\_ratio * n\\_samples * ||vec(H)||_1 - Where: + &+ 0.5 * alpha\\_W * (1 - l1\\_ratio) * n\\_features * ||W||_{Fro}^2 - :math:`||A||_{Fro}^2 = \\sum_{i,j} A_{ij}^2` (Frobenius norm) + &+ 0.5 * alpha\\_H * (1 - l1\\_ratio) * n\\_samples * ||H||_{Fro}^2, + where :math:`||A||_{Fro}^2 = \\sum_{i,j} A_{ij}^2` (Frobenius norm) and :math:`||vec(A)||_1 = \\sum_{i,j} abs(A_{ij})` (Elementwise L1 norm) The generic norm :math:`||X - WH||_{loss}^2` may represent the Frobenius norm or another supported beta-divergence loss. The choice between options is controlled by the `beta_loss` parameter. + The regularization terms are scaled by `n_features` for `W` and by `n_samples` for + `H` to keep their impact balanced with respect to one another and to the data fit + term as independent as possible of the size `n_samples` of the training set. + The objective function is minimized with an alternating minimization of W and H. If H is given and update_H=False, it solves for W only. + Note that the transformed data is named W and the components matrix is named H. In + the NMF literature, the naming convention is usually the opposite since the data + matrix X is transposed. + Parameters ---------- - X : array-like of shape (n_samples, n_features) + X : {array-like, sparse matrix} of shape (n_samples, n_features) Constant matrix. W : array-like of shape (n_samples, n_components), default=None - If init='custom', it is used as initial guess for the solution. + If `init='custom'`, it is used as initial guess for the solution. + If `update_H=False`, it is initialised as an array of zeros, unless + `solver='mu'`, then it is filled with values calculated by + `np.sqrt(X.mean() / self._n_components)`. + If `None`, uses the initialisation method specified in `init`. H : array-like of shape (n_components, n_features), default=None - If init='custom', it is used as initial guess for the solution. - If update_H=False, it is used as a constant, to solve for W only. + If `init='custom'`, it is used as initial guess for the solution. + If `update_H=False`, it is used as a constant, to solve for W only. + If `None`, uses the initialisation method specified in `init`. + + n_components : int or {'auto'} or None, default='auto' + Number of components. If `None`, all features are kept. + If `n_components='auto'`, the number of components is automatically inferred + from `W` or `H` shapes. - n_components : int, default=None - Number of components, if n_components is not set all features - are kept. + .. versionchanged:: 1.4 + Added `'auto'` value. + + .. versionchanged:: 1.6 + Default value changed from `None` to `'auto'`. init : {'random', 'nndsvd', 'nndsvda', 'nndsvdar', 'custom'}, default=None Method used to initialize the procedure. Valid options: - - None: 'nndsvd' if n_components < n_features, otherwise 'random'. - + - None: 'nndsvda' if n_components < n_features, otherwise 'random'. - 'random': non-negative random matrices, scaled with: - sqrt(X.mean() / n_components) - + `sqrt(X.mean() / n_components)` - 'nndsvd': Nonnegative Double Singular Value Decomposition (NNDSVD) - initialization (better for sparseness) - + initialization (better for sparseness) - 'nndsvda': NNDSVD with zeros filled with the average of X - (better when sparsity is not desired) - + (better when sparsity is not desired) - 'nndsvdar': NNDSVD with zeros filled with small random values - (generally faster, less accurate alternative to NNDSVDa - for when sparsity is not desired) - - - 'custom': use custom matrices W and H if `update_H=True`. If - `update_H=False`, then only custom matrix H is used. + (generally faster, less accurate alternative to NNDSVDa + for when sparsity is not desired) + - 'custom': If `update_H=True`, use custom matrices W and H which must both + be provided. If `update_H=False`, then only custom matrix H is used. .. versionchanged:: 0.23 The default value of `init` changed from 'random' to None in 0.23. + .. versionchanged:: 1.1 + When `init=None` and n_components is less than n_samples and n_features + defaults to `nndsvda` instead of `nndsvd`. + update_H : bool, default=True Set to True, both W and H will be estimated from initial guesses. Set to False, only W will be estimated. @@ -938,8 +1020,7 @@ def non_negative_factorization(X, W=None, H=None, n_components=None, *, Numerical solver to use: - 'cd' is a Coordinate Descent solver that uses Fast Hierarchical - Alternating Least Squares (Fast HALS). - + Alternating Least Squares (Fast HALS). - 'mu' is a Multiplicative Update solver. .. versionadded:: 0.17 @@ -964,20 +1045,26 @@ def non_negative_factorization(X, W=None, H=None, n_components=None, *, max_iter : int, default=200 Maximum number of iterations before timing out. - alpha : float, default=0. - Constant that multiplies the regularization terms. + alpha_W : float, default=0.0 + Constant that multiplies the regularization terms of `W`. Set it to zero + (default) to have no regularization on `W`. - l1_ratio : float, default=0. + .. versionadded:: 1.0 + + alpha_H : float or "same", default="same" + Constant that multiplies the regularization terms of `H`. Set it to zero to + have no regularization on `H`. If "same" (default), it takes the same value as + `alpha_W`. + + .. versionadded:: 1.0 + + l1_ratio : float, default=0.0 The regularization mixing parameter, with 0 <= l1_ratio <= 1. For l1_ratio = 0 the penalty is an elementwise L2 penalty (aka Frobenius Norm). For l1_ratio = 1 it is an elementwise L1 penalty. For 0 < l1_ratio < 1, the penalty is a combination of L1 and L2. - regularization : {'both', 'components', 'transformation'}, default=None - Select whether the regularization affects the components (H), the - transformation (W), both or none of them. - random_state : int, RandomState instance or None, default=None Used for NMF initialisation (when ``init`` == 'nndsvdar' or 'random'), and in Coordinate Descent. Pass an int for reproducible @@ -1001,31 +1088,42 @@ def non_negative_factorization(X, W=None, H=None, n_components=None, *, n_iter : int Actual number of iterations. + References + ---------- + .. [1] :doi:`"Fast local algorithms for large scale nonnegative matrix and tensor + factorizations" <10.1587/transfun.E92.A.708>` + Cichocki, Andrzej, and P. H. A. N. Anh-Huy. IEICE transactions on fundamentals + of electronics, communications and computer sciences 92.3: 708-721, 2009. + + .. [2] :doi:`"Algorithms for nonnegative matrix factorization with the + beta-divergence" <10.1162/NECO_a_00168>` + Fevotte, C., & Idier, J. (2011). Neural Computation, 23(9). + Examples -------- >>> import numpy as np >>> X = np.array([[1,1], [2, 1], [3, 1.2], [4, 1], [5, 0.8], [6, 1]]) >>> from sklearn.decomposition import non_negative_factorization - >>> W, H, n_iter = non_negative_factorization(X, n_components=2, - ... init='random', random_state=0) - - References - ---------- - Cichocki, Andrzej, and P. H. A. N. Anh-Huy. "Fast local algorithms for - large scale nonnegative matrix and tensor factorizations." - IEICE transactions on fundamentals of electronics, communications and - computer sciences 92.3: 708-721, 2009. - - Fevotte, C., & Idier, J. (2011). Algorithms for nonnegative matrix - factorization with the beta-divergence. Neural Computation, 23(9). + >>> W, H, n_iter = non_negative_factorization( + ... X, n_components=2, init='random', random_state=0) """ - X = check_array(X, accept_sparse=('csr', 'csc'), - dtype=[np.float64, np.float32]) - - est = NMF(n_components=n_components, init=init, solver=solver, - beta_loss=beta_loss, tol=tol, max_iter=max_iter, - random_state=random_state, alpha=alpha, l1_ratio=l1_ratio, - verbose=verbose, shuffle=shuffle, regularization=regularization) + est = NMF( + n_components=n_components, + init=init, + solver=solver, + beta_loss=beta_loss, + tol=tol, + max_iter=max_iter, + random_state=random_state, + alpha_W=alpha_W, + alpha_H=alpha_H, + l1_ratio=l1_ratio, + verbose=verbose, + shuffle=shuffle, + ) + est._validate_params() + + X = check_array(X, accept_sparse=("csr", "csc"), dtype=[np.float64, np.float32]) with config_context(assume_finite=True): W, H, n_iter = est._fit_transform(X, W=W, H=H, update_H=update_H) @@ -1033,56 +1131,260 @@ def non_negative_factorization(X, W=None, H=None, n_components=None, *, return W, H, n_iter -class NMF(TransformerMixin, BaseEstimator): +class _BaseNMF(ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator, ABC): + """Base class for NMF and MiniBatchNMF.""" + + _parameter_constraints: dict = { + "n_components": [ + Interval(Integral, 1, None, closed="left"), + None, + StrOptions({"auto"}), + ], + "init": [ + StrOptions({"random", "nndsvd", "nndsvda", "nndsvdar", "custom"}), + None, + ], + "beta_loss": [ + StrOptions({"frobenius", "kullback-leibler", "itakura-saito"}), + Real, + ], + "tol": [Interval(Real, 0, None, closed="left")], + "max_iter": [Interval(Integral, 1, None, closed="left")], + "random_state": ["random_state"], + "alpha_W": [Interval(Real, 0, None, closed="left")], + "alpha_H": [Interval(Real, 0, None, closed="left"), StrOptions({"same"})], + "l1_ratio": [Interval(Real, 0, 1, closed="both")], + "verbose": ["verbose"], + } + + def __init__( + self, + n_components="auto", + *, + init=None, + beta_loss="frobenius", + tol=1e-4, + max_iter=200, + random_state=None, + alpha_W=0.0, + alpha_H="same", + l1_ratio=0.0, + verbose=0, + ): + self.n_components = n_components + self.init = init + self.beta_loss = beta_loss + self.tol = tol + self.max_iter = max_iter + self.random_state = random_state + self.alpha_W = alpha_W + self.alpha_H = alpha_H + self.l1_ratio = l1_ratio + self.verbose = verbose + + def _check_params(self, X): + # n_components + self._n_components = self.n_components + if self._n_components is None: + self._n_components = X.shape[1] + + # beta_loss + self._beta_loss = _beta_loss_to_float(self.beta_loss) + + def _check_w_h(self, X, W, H, update_H): + """Check W and H, or initialize them.""" + n_samples, n_features = X.shape + + if self.init == "custom" and update_H: + _check_init(H, (self._n_components, n_features), "NMF (input H)") + _check_init(W, (n_samples, self._n_components), "NMF (input W)") + if self._n_components == "auto": + self._n_components = H.shape[0] + + if H.dtype != X.dtype or W.dtype != X.dtype: + raise TypeError( + "H and W should have the same dtype as X. Got " + "H.dtype = {} and W.dtype = {}.".format(H.dtype, W.dtype) + ) + + elif not update_H: + if W is not None: + warnings.warn( + "When update_H=False, the provided initial W is not used.", + RuntimeWarning, + ) + + _check_init(H, (self._n_components, n_features), "NMF (input H)") + if self._n_components == "auto": + self._n_components = H.shape[0] + + if H.dtype != X.dtype: + raise TypeError( + "H should have the same dtype as X. Got H.dtype = {}.".format( + H.dtype + ) + ) + + # 'mu' solver should not be initialized by zeros + if self.solver == "mu": + avg = np.sqrt(X.mean() / self._n_components) + W = np.full((n_samples, self._n_components), avg, dtype=X.dtype) + else: + W = np.zeros((n_samples, self._n_components), dtype=X.dtype) + + else: + if W is not None or H is not None: + warnings.warn( + ( + "When init!='custom', provided W or H are ignored. Set " + " init='custom' to use them as initialization." + ), + RuntimeWarning, + ) + + if self._n_components == "auto": + self._n_components = X.shape[1] + + W, H = _initialize_nmf( + X, self._n_components, init=self.init, random_state=self.random_state + ) + + return W, H + + def _compute_regularization(self, X): + """Compute scaled regularization terms.""" + n_samples, n_features = X.shape + alpha_W = self.alpha_W + alpha_H = self.alpha_W if self.alpha_H == "same" else self.alpha_H + + l1_reg_W = n_features * alpha_W * self.l1_ratio + l1_reg_H = n_samples * alpha_H * self.l1_ratio + l2_reg_W = n_features * alpha_W * (1.0 - self.l1_ratio) + l2_reg_H = n_samples * alpha_H * (1.0 - self.l1_ratio) + + return l1_reg_W, l1_reg_H, l2_reg_W, l2_reg_H + + def fit(self, X, y=None, **params): + """Learn a NMF model for the data X. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training vector, where `n_samples` is the number of samples + and `n_features` is the number of features. + + y : Ignored + Not used, present for API consistency by convention. + + **params : kwargs + Parameters (keyword arguments) and values passed to + the fit_transform instance. + + Returns + ------- + self : object + Returns the instance itself. + """ + # param validation is done in fit_transform + + self.fit_transform(X, **params) + return self + + def inverse_transform(self, X): + """Transform data back to its original space. + + .. versionadded:: 0.18 + + Parameters + ---------- + X : {ndarray, sparse matrix} of shape (n_samples, n_components) + Transformed data matrix. + + Returns + ------- + X_original : ndarray of shape (n_samples, n_features) + Returns a data matrix of the original shape. + """ + + check_is_fitted(self) + return X @ self.components_ + + @property + def _n_features_out(self): + """Number of transformed output features.""" + return self.components_.shape[0] + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.positive_only = True + tags.input_tags.sparse = True + tags.transformer_tags.preserves_dtype = ["float64", "float32"] + return tags + + +class NMF(_BaseNMF): """Non-Negative Matrix Factorization (NMF). - Find two non-negative matrices (W, H) whose product approximates the non- - negative matrix X. This factorization can be used for example for - dimensionality reduction, source separation or topic extraction. + Find two non-negative matrices, i.e. matrices with all non-negative elements, (W, H) + whose product approximates the non-negative matrix X. This factorization can be used + for example for dimensionality reduction, source separation or topic extraction. The objective function is: - .. math:: + .. math:: - 0.5 * ||X - WH||_{loss}^2 + alpha * l1_{ratio} * ||vec(W)||_1 + L(W, H) &= 0.5 * ||X - WH||_{loss}^2 - + alpha * l1_{ratio} * ||vec(H)||_1 + &+ alpha\\_W * l1\\_ratio * n\\_features * ||vec(W)||_1 - + 0.5 * alpha * (1 - l1_{ratio}) * ||W||_{Fro}^2 + &+ alpha\\_H * l1\\_ratio * n\\_samples * ||vec(H)||_1 - + 0.5 * alpha * (1 - l1_{ratio}) * ||H||_{Fro}^2 + &+ 0.5 * alpha\\_W * (1 - l1\\_ratio) * n\\_features * ||W||_{Fro}^2 - Where: + &+ 0.5 * alpha\\_H * (1 - l1\\_ratio) * n\\_samples * ||H||_{Fro}^2, - :math:`||A||_{Fro}^2 = \\sum_{i,j} A_{ij}^2` (Frobenius norm) - - :math:`||vec(A)||_1 = \\sum_{i,j} abs(A_{ij})` (Elementwise L1 norm) + where :math:`||A||_{Fro}^2 = \\sum_{i,j} A_{ij}^2` (Frobenius norm) and + :math:`||vec(A)||_1 = \\sum_{i,j} abs(A_{ij})` (Elementwise L1 norm). The generic norm :math:`||X - WH||_{loss}` may represent the Frobenius norm or another supported beta-divergence loss. The choice between options is controlled by the `beta_loss` parameter. + The regularization terms are scaled by `n_features` for `W` and by `n_samples` for + `H` to keep their impact balanced with respect to one another and to the data fit + term as independent as possible of the size `n_samples` of the training set. + The objective function is minimized with an alternating minimization of W and H. + Note that the transformed data is named W and the components matrix is named H. In + the NMF literature, the naming convention is usually the opposite since the data + matrix X is transposed. + Read more in the :ref:`User Guide `. Parameters ---------- - n_components : int, default=None - Number of components, if n_components is not set all features - are kept. + n_components : int or {'auto'} or None, default='auto' + Number of components. If `None`, all features are kept. + If `n_components='auto'`, the number of components is automatically inferred + from W or H shapes. + + .. versionchanged:: 1.4 + Added `'auto'` value. + + .. versionchanged:: 1.6 + Default value changed from `None` to `'auto'`. init : {'random', 'nndsvd', 'nndsvda', 'nndsvdar', 'custom'}, default=None Method used to initialize the procedure. - Default: None. Valid options: - - `None`: 'nndsvd' if n_components <= min(n_samples, n_features), + - `None`: 'nndsvda' if n_components <= min(n_samples, n_features), otherwise random. - `'random'`: non-negative random matrices, scaled with: - sqrt(X.mean() / n_components) + `sqrt(X.mean() / n_components)` - `'nndsvd'`: Nonnegative Double Singular Value Decomposition (NNDSVD) initialization (better for sparseness) @@ -1094,12 +1396,17 @@ class NMF(TransformerMixin, BaseEstimator): (generally faster, less accurate alternative to NNDSVDa for when sparsity is not desired) - - `'custom'`: use custom matrices W and H + - `'custom'`: Use custom matrices `W` and `H` which must both be provided. + + .. versionchanged:: 1.1 + When `init=None` and n_components is less than n_samples and n_features + defaults to `nndsvda` instead of `nndsvd`. solver : {'cd', 'mu'}, default='cd' Numerical solver to use: - 'cd' is a Coordinate Descent solver. - 'mu' is a Multiplicative Update solver. + + - 'cd' is a Coordinate Descent solver. + - 'mu' is a Multiplicative Update solver. .. versionadded:: 0.17 Coordinate Descent solver. @@ -1129,14 +1436,20 @@ class NMF(TransformerMixin, BaseEstimator): results across multiple function calls. See :term:`Glossary `. - alpha : float, default=0. - Constant that multiplies the regularization terms. Set it to zero to - have no regularization. + alpha_W : float, default=0.0 + Constant that multiplies the regularization terms of `W`. Set it to zero + (default) to have no regularization on `W`. - .. versionadded:: 0.17 - *alpha* used in the Coordinate Descent solver. + .. versionadded:: 1.0 + + alpha_H : float or "same", default="same" + Constant that multiplies the regularization terms of `H`. Set it to zero to + have no regularization on `H`. If "same" (default), it takes the same value as + `alpha_W`. + + .. versionadded:: 1.0 - l1_ratio : float, default=0. + l1_ratio : float, default=0.0 The regularization mixing parameter, with 0 <= l1_ratio <= 1. For l1_ratio = 0 the penalty is an elementwise L2 penalty (aka Frobenius Norm). @@ -1156,13 +1469,6 @@ class NMF(TransformerMixin, BaseEstimator): .. versionadded:: 0.17 *shuffle* parameter used in the Coordinate Descent solver. - regularization : {'both', 'components', 'transformation', None}, \ - default='both' - Select whether the regularization affects the components (H), the - transformation (W), both or none of them. - - .. versionadded:: 0.24 - Attributes ---------- components_ : ndarray of shape (n_components, n_features) @@ -1181,6 +1487,38 @@ class NMF(TransformerMixin, BaseEstimator): n_iter_ : int Actual number of iterations. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + DictionaryLearning : Find a dictionary that sparsely encodes data. + MiniBatchSparsePCA : Mini-batch Sparse Principal Components Analysis. + PCA : Principal component analysis. + SparseCoder : Find a sparse representation of data from a fixed, + precomputed dictionary. + SparsePCA : Sparse Principal Components Analysis. + TruncatedSVD : Dimensionality reduction using truncated SVD. + + References + ---------- + .. [1] :doi:`"Fast local algorithms for large scale nonnegative matrix and tensor + factorizations" <10.1587/transfun.E92.A.708>` + Cichocki, Andrzej, and P. H. A. N. Anh-Huy. IEICE transactions on fundamentals + of electronics, communications and computer sciences 92.3: 708-721, 2009. + + .. [2] :doi:`"Algorithms for nonnegative matrix factorization with the + beta-divergence" <10.1162/NECO_a_00168>` + Fevotte, C., & Idier, J. (2011). Neural Computation, 23(9). + Examples -------- >>> import numpy as np @@ -1189,84 +1527,70 @@ class NMF(TransformerMixin, BaseEstimator): >>> model = NMF(n_components=2, init='random', random_state=0) >>> W = model.fit_transform(X) >>> H = model.components_ + """ - References - ---------- - Cichocki, Andrzej, and P. H. A. N. Anh-Huy. "Fast local algorithms for - large scale nonnegative matrix and tensor factorizations." - IEICE transactions on fundamentals of electronics, communications and - computer sciences 92.3: 708-721, 2009. + _parameter_constraints: dict = { + **_BaseNMF._parameter_constraints, + "solver": [StrOptions({"mu", "cd"})], + "shuffle": ["boolean"], + } + + def __init__( + self, + n_components="auto", + *, + init=None, + solver="cd", + beta_loss="frobenius", + tol=1e-4, + max_iter=200, + random_state=None, + alpha_W=0.0, + alpha_H="same", + l1_ratio=0.0, + verbose=0, + shuffle=False, + ): + super().__init__( + n_components=n_components, + init=init, + beta_loss=beta_loss, + tol=tol, + max_iter=max_iter, + random_state=random_state, + alpha_W=alpha_W, + alpha_H=alpha_H, + l1_ratio=l1_ratio, + verbose=verbose, + ) - Fevotte, C., & Idier, J. (2011). Algorithms for nonnegative matrix - factorization with the beta-divergence. Neural Computation, 23(9). - """ - @_deprecate_positional_args - def __init__(self, n_components=None, *, init='warn', solver='cd', - beta_loss='frobenius', tol=1e-4, max_iter=200, - random_state=None, alpha=0., l1_ratio=0., verbose=0, - shuffle=False, regularization='both'): - self.n_components = n_components - self.init = init self.solver = solver - self.beta_loss = beta_loss - self.tol = tol - self.max_iter = max_iter - self.random_state = random_state - self.alpha = alpha - self.l1_ratio = l1_ratio - self.verbose = verbose self.shuffle = shuffle - self.regularization = regularization - - def _more_tags(self): - return {'requires_positive_X': True} def _check_params(self, X): - self._n_components = self.n_components - if self._n_components is None: - self._n_components = X.shape[1] - if not isinstance( - self._n_components, numbers.Integral - ) or self._n_components <= 0: - raise ValueError("Number of components must be a positive integer;" - " got (n_components=%r)" % self._n_components) - if not isinstance( - self.max_iter, numbers.Integral - ) or self.max_iter < 0: - raise ValueError("Maximum number of iterations must be a positive " - "integer; got (max_iter=%r)" % self.max_iter) - if not isinstance(self.tol, numbers.Number) or self.tol < 0: - raise ValueError("Tolerance for stopping criteria must be " - "positive; got (tol=%r)" % self.tol) - return self + super()._check_params(X) + + # solver + if self.solver != "mu" and self.beta_loss not in (2, "frobenius"): + # 'mu' is the only solver that handles other beta losses than 'frobenius' + raise ValueError( + f"Invalid beta_loss parameter: solver {self.solver!r} does not handle " + f"beta_loss = {self.beta_loss!r}" + ) + if self.solver == "mu" and self.init == "nndsvd": + warnings.warn( + ( + "The multiplicative update ('mu') solver cannot update " + "zeros present in the initialization, and so leads to " + "poorer results when used jointly with init='nndsvd'. " + "You may try init='nndsvda' or init='nndsvdar' instead." + ), + UserWarning, + ) - def _check_w_h(self, X, W, H, update_H): - # check W and H, or initialize them - n_samples, n_features = X.shape - if self.init == 'custom' and update_H: - _check_init(H, (self._n_components, n_features), "NMF (input H)") - _check_init(W, (n_samples, self._n_components), "NMF (input W)") - if H.dtype != X.dtype or W.dtype != X.dtype: - raise TypeError("H and W should have the same dtype as X. Got " - "H.dtype = {} and W.dtype = {}." - .format(H.dtype, W.dtype)) - elif not update_H: - _check_init(H, (self._n_components, n_features), "NMF (input H)") - if H.dtype != X.dtype: - raise TypeError("H should have the same dtype as X. Got " - "H.dtype = {}.".format(H.dtype)) - # 'mu' solver should not be initialized by zeros - if self.solver == 'mu': - avg = np.sqrt(X.mean() / self._n_components) - W = np.full((n_samples, self._n_components), - avg, dtype=X.dtype) - else: - W = np.zeros((n_samples, self._n_components), dtype=X.dtype) - else: - W, H = _initialize_nmf(X, self._n_components, init=self.init, - random_state=self.random_state) - return W, H + return self + @_fit_context(prefer_skip_nested_validation=True) def fit_transform(self, X, y=None, W=None, H=None): """Learn a NMF model for the data X and returns the transformed data. @@ -1275,29 +1599,35 @@ def fit_transform(self, X, y=None, W=None, H=None): Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) - Data matrix to be decomposed + Training vector, where `n_samples` is the number of samples + and `n_features` is the number of features. y : Ignored + Not used, present for API consistency by convention. - W : array-like of shape (n_samples, n_components) - If init='custom', it is used as initial guess for the solution. + W : array-like of shape (n_samples, n_components), default=None + If `init='custom'`, it is used as initial guess for the solution. + If `None`, uses the initialisation method specified in `init`. - H : array-like of shape (n_components, n_features) - If init='custom', it is used as initial guess for the solution. + H : array-like of shape (n_components, n_features), default=None + If `init='custom'`, it is used as initial guess for the solution. + If `None`, uses the initialisation method specified in `init`. Returns ------- W : ndarray of shape (n_samples, n_components) Transformed data. """ - X = self._validate_data(X, accept_sparse=('csr', 'csc'), - dtype=[np.float64, np.float32]) + X = validate_data( + self, X, accept_sparse=("csr", "csc"), dtype=[np.float64, np.float32] + ) with config_context(assume_finite=True): W, H, n_iter = self._fit_transform(X, W=W, H=H) - self.reconstruction_err_ = _beta_divergence(X, W, H, self._beta_loss, - square_root=True) + self.reconstruction_err_ = _beta_divergence( + X, W, H, self._beta_loss, square_root=True + ) self.n_components_ = H.shape[0] self.components_ = H @@ -1315,12 +1645,17 @@ def _fit_transform(self, X, y=None, W=None, H=None, update_H=True): y : Ignored - W : array-like of shape (n_samples, n_components) - If init='custom', it is used as initial guess for the solution. + W : array-like of shape (n_samples, n_components), default=None + If `init='custom'`, it is used as initial guess for the solution. + If `update_H=False`, it is initialised as an array of zeros, unless + `solver='mu'`, then it is filled with values calculated by + `np.sqrt(X.mean() / self._n_components)`. + If `None`, uses the initialisation method specified in `init`. - H : array-like of shape (n_components, n_features) - If init='custom', it is used as initial guess for the solution. - If update_H=False, it is used as a constant, to solve for W only. + H : array-like of shape (n_components, n_features), default=None + If `init='custom'`, it is used as initial guess for the solution. + If `update_H=False`, it is used as a constant, to solve for W only. + If `None`, uses the initialisation method specified in `init`. update_H : bool, default=True If True, both W and H will be estimated from initial guesses, @@ -1339,66 +1674,655 @@ def _fit_transform(self, X, y=None, W=None, H=None, update_H=True): n_iter_ : int Actual number of iterations. """ - check_non_negative(X, "NMF (input X)") - self._beta_loss = _check_string_param(self.solver, self.regularization, - self.beta_loss, self.init) - - if X.min() == 0 and self._beta_loss <= 0: - raise ValueError("When beta_loss <= 0 and X contains zeros, " - "the solver may diverge. Please add small values " - "to X, or use a positive beta_loss.") - - n_samples, n_features = X.shape - # check parameters self._check_params(X) + if X.min() == 0 and self._beta_loss <= 0: + raise ValueError( + "When beta_loss <= 0 and X contains zeros, " + "the solver may diverge. Please add small values " + "to X, or use a positive beta_loss." + ) + # initialize or check W and H W, H = self._check_w_h(X, W, H, update_H) - l1_reg_W, l1_reg_H, l2_reg_W, l2_reg_H = _compute_regularization( - self.alpha, self.l1_ratio, self.regularization) + # scale the regularization terms + l1_reg_W, l1_reg_H, l2_reg_W, l2_reg_H = self._compute_regularization(X) - if self.solver == 'cd': + if self.solver == "cd": W, H, n_iter = _fit_coordinate_descent( - X, W, H, self.tol, self.max_iter, l1_reg_W, l1_reg_H, - l2_reg_W, l2_reg_H, update_H=update_H, - verbose=self.verbose, shuffle=self.shuffle, - random_state=self.random_state) - elif self.solver == 'mu': - W, H, n_iter = _fit_multiplicative_update( - X, W, H, self._beta_loss, self.max_iter, self.tol, - l1_reg_W, l1_reg_H, l2_reg_W, l2_reg_H, - update_H=update_H, verbose=self.verbose) + X, + W, + H, + self.tol, + self.max_iter, + l1_reg_W, + l1_reg_H, + l2_reg_W, + l2_reg_H, + update_H=update_H, + verbose=self.verbose, + shuffle=self.shuffle, + random_state=self.random_state, + ) + elif self.solver == "mu": + W, H, n_iter, *_ = _fit_multiplicative_update( + X, + W, + H, + self._beta_loss, + self.max_iter, + self.tol, + l1_reg_W, + l1_reg_H, + l2_reg_W, + l2_reg_H, + update_H, + self.verbose, + ) else: raise ValueError("Invalid solver parameter '%s'." % self.solver) if n_iter == self.max_iter and self.tol > 0: - warnings.warn("Maximum number of iterations %d reached. Increase " - "it to improve convergence." % self.max_iter, - ConvergenceWarning) + warnings.warn( + "Maximum number of iterations %d reached. Increase " + "it to improve convergence." % self.max_iter, + ConvergenceWarning, + ) return W, H, n_iter - def fit(self, X, y=None, **params): - """Learn a NMF model for the data X. + def transform(self, X): + """Transform the data X according to the fitted NMF model. Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) - Data matrix to be decomposed + Training vector, where `n_samples` is the number of samples + and `n_features` is the number of features. + + Returns + ------- + W : ndarray of shape (n_samples, n_components) + Transformed data. + """ + check_is_fitted(self) + X = validate_data( + self, + X, + accept_sparse=("csr", "csc"), + dtype=[np.float64, np.float32], + reset=False, + ensure_non_negative=True, + ) + + with config_context(assume_finite=True): + W, *_ = self._fit_transform(X, H=self.components_, update_H=False) + + return W + + +class MiniBatchNMF(_BaseNMF): + """Mini-Batch Non-Negative Matrix Factorization (NMF). + + .. versionadded:: 1.1 + + Find two non-negative matrices, i.e. matrices with all non-negative elements, + (`W`, `H`) whose product approximates the non-negative matrix `X`. This + factorization can be used for example for dimensionality reduction, source + separation or topic extraction. + + The objective function is: + + .. math:: + + L(W, H) &= 0.5 * ||X - WH||_{loss}^2 + + &+ alpha\\_W * l1\\_ratio * n\\_features * ||vec(W)||_1 + + &+ alpha\\_H * l1\\_ratio * n\\_samples * ||vec(H)||_1 + + &+ 0.5 * alpha\\_W * (1 - l1\\_ratio) * n\\_features * ||W||_{Fro}^2 + + &+ 0.5 * alpha\\_H * (1 - l1\\_ratio) * n\\_samples * ||H||_{Fro}^2, + + where :math:`||A||_{Fro}^2 = \\sum_{i,j} A_{ij}^2` (Frobenius norm) and + :math:`||vec(A)||_1 = \\sum_{i,j} abs(A_{ij})` (Elementwise L1 norm). + + The generic norm :math:`||X - WH||_{loss}^2` may represent + the Frobenius norm or another supported beta-divergence loss. + The choice between options is controlled by the `beta_loss` parameter. + + The objective function is minimized with an alternating minimization of `W` + and `H`. + + Note that the transformed data is named `W` and the components matrix is + named `H`. In the NMF literature, the naming convention is usually the opposite + since the data matrix `X` is transposed. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + n_components : int or {'auto'} or None, default='auto' + Number of components. If `None`, all features are kept. + If `n_components='auto'`, the number of components is automatically inferred + from W or H shapes. + + .. versionchanged:: 1.4 + Added `'auto'` value. + + .. versionchanged:: 1.6 + Default value changed from `None` to `'auto'`. + + init : {'random', 'nndsvd', 'nndsvda', 'nndsvdar', 'custom'}, default=None + Method used to initialize the procedure. + Valid options: + + - `None`: 'nndsvda' if `n_components <= min(n_samples, n_features)`, + otherwise random. + + - `'random'`: non-negative random matrices, scaled with: + `sqrt(X.mean() / n_components)` + + - `'nndsvd'`: Nonnegative Double Singular Value Decomposition (NNDSVD) + initialization (better for sparseness). + + - `'nndsvda'`: NNDSVD with zeros filled with the average of X + (better when sparsity is not desired). + + - `'nndsvdar'` NNDSVD with zeros filled with small random values + (generally faster, less accurate alternative to NNDSVDa + for when sparsity is not desired). + + - `'custom'`: Use custom matrices `W` and `H` which must both be provided. + + batch_size : int, default=1024 + Number of samples in each mini-batch. Large batch sizes + give better long-term convergence at the cost of a slower start. + + beta_loss : float or {'frobenius', 'kullback-leibler', \ + 'itakura-saito'}, default='frobenius' + Beta divergence to be minimized, measuring the distance between `X` + and the dot product `WH`. Note that values different from 'frobenius' + (or 2) and 'kullback-leibler' (or 1) lead to significantly slower + fits. Note that for `beta_loss <= 0` (or 'itakura-saito'), the input + matrix `X` cannot contain zeros. + + tol : float, default=1e-4 + Control early stopping based on the norm of the differences in `H` + between 2 steps. To disable early stopping based on changes in `H`, set + `tol` to 0.0. + + max_no_improvement : int, default=10 + Control early stopping based on the consecutive number of mini batches + that does not yield an improvement on the smoothed cost function. + To disable convergence detection based on cost function, set + `max_no_improvement` to None. + + max_iter : int, default=200 + Maximum number of iterations over the complete dataset before + timing out. + + alpha_W : float, default=0.0 + Constant that multiplies the regularization terms of `W`. Set it to zero + (default) to have no regularization on `W`. + + alpha_H : float or "same", default="same" + Constant that multiplies the regularization terms of `H`. Set it to zero to + have no regularization on `H`. If "same" (default), it takes the same value as + `alpha_W`. + + l1_ratio : float, default=0.0 + The regularization mixing parameter, with 0 <= l1_ratio <= 1. + For l1_ratio = 0 the penalty is an elementwise L2 penalty + (aka Frobenius Norm). + For l1_ratio = 1 it is an elementwise L1 penalty. + For 0 < l1_ratio < 1, the penalty is a combination of L1 and L2. + + forget_factor : float, default=0.7 + Amount of rescaling of past information. Its value could be 1 with + finite datasets. Choosing values < 1 is recommended with online + learning as more recent batches will weight more than past batches. + + fresh_restarts : bool, default=False + Whether to completely solve for W at each step. Doing fresh restarts will likely + lead to a better solution for a same number of iterations but it is much slower. + + fresh_restarts_max_iter : int, default=30 + Maximum number of iterations when solving for W at each step. Only used when + doing fresh restarts. These iterations may be stopped early based on a small + change of W controlled by `tol`. + + transform_max_iter : int, default=None + Maximum number of iterations when solving for W at transform time. + If None, it defaults to `max_iter`. + + random_state : int, RandomState instance or None, default=None + Used for initialisation (when ``init`` == 'nndsvdar' or + 'random'), and in Coordinate Descent. Pass an int for reproducible + results across multiple function calls. + See :term:`Glossary `. + + verbose : bool, default=False + Whether to be verbose. + + Attributes + ---------- + components_ : ndarray of shape (n_components, n_features) + Factorization matrix, sometimes called 'dictionary'. + + n_components_ : int + The number of components. It is same as the `n_components` parameter + if it was given. Otherwise, it will be same as the number of + features. + + reconstruction_err_ : float + Frobenius norm of the matrix difference, or beta-divergence, between + the training data `X` and the reconstructed data `WH` from + the fitted model. + + n_iter_ : int + Actual number of started iterations over the whole dataset. + + n_steps_ : int + Number of mini-batches processed. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + See Also + -------- + NMF : Non-negative matrix factorization. + MiniBatchDictionaryLearning : Finds a dictionary that can best be used to represent + data using a sparse code. + + References + ---------- + .. [1] :doi:`"Fast local algorithms for large scale nonnegative matrix and tensor + factorizations" <10.1587/transfun.E92.A.708>` + Cichocki, Andrzej, and P. H. A. N. Anh-Huy. IEICE transactions on fundamentals + of electronics, communications and computer sciences 92.3: 708-721, 2009. + + .. [2] :doi:`"Algorithms for nonnegative matrix factorization with the + beta-divergence" <10.1162/NECO_a_00168>` + Fevotte, C., & Idier, J. (2011). Neural Computation, 23(9). + + .. [3] :doi:`"Online algorithms for nonnegative matrix factorization with the + Itakura-Saito divergence" <10.1109/ASPAA.2011.6082314>` + Lefevre, A., Bach, F., Fevotte, C. (2011). WASPA. + + Examples + -------- + >>> import numpy as np + >>> X = np.array([[1, 1], [2, 1], [3, 1.2], [4, 1], [5, 0.8], [6, 1]]) + >>> from sklearn.decomposition import MiniBatchNMF + >>> model = MiniBatchNMF(n_components=2, init='random', random_state=0) + >>> W = model.fit_transform(X) + >>> H = model.components_ + """ + + _parameter_constraints: dict = { + **_BaseNMF._parameter_constraints, + "max_no_improvement": [Interval(Integral, 1, None, closed="left"), None], + "batch_size": [Interval(Integral, 1, None, closed="left")], + "forget_factor": [Interval(Real, 0, 1, closed="both")], + "fresh_restarts": ["boolean"], + "fresh_restarts_max_iter": [Interval(Integral, 1, None, closed="left")], + "transform_max_iter": [Interval(Integral, 1, None, closed="left"), None], + } + + def __init__( + self, + n_components="auto", + *, + init=None, + batch_size=1024, + beta_loss="frobenius", + tol=1e-4, + max_no_improvement=10, + max_iter=200, + alpha_W=0.0, + alpha_H="same", + l1_ratio=0.0, + forget_factor=0.7, + fresh_restarts=False, + fresh_restarts_max_iter=30, + transform_max_iter=None, + random_state=None, + verbose=0, + ): + super().__init__( + n_components=n_components, + init=init, + beta_loss=beta_loss, + tol=tol, + max_iter=max_iter, + random_state=random_state, + alpha_W=alpha_W, + alpha_H=alpha_H, + l1_ratio=l1_ratio, + verbose=verbose, + ) + + self.max_no_improvement = max_no_improvement + self.batch_size = batch_size + self.forget_factor = forget_factor + self.fresh_restarts = fresh_restarts + self.fresh_restarts_max_iter = fresh_restarts_max_iter + self.transform_max_iter = transform_max_iter + + def _check_params(self, X): + super()._check_params(X) + + # batch_size + self._batch_size = min(self.batch_size, X.shape[0]) + + # forget_factor + self._rho = self.forget_factor ** (self._batch_size / X.shape[0]) + + # gamma for Maximization-Minimization (MM) algorithm [Fevotte 2011] + if self._beta_loss < 1: + self._gamma = 1.0 / (2.0 - self._beta_loss) + elif self._beta_loss > 2: + self._gamma = 1.0 / (self._beta_loss - 1.0) + else: + self._gamma = 1.0 + + # transform_max_iter + self._transform_max_iter = ( + self.max_iter + if self.transform_max_iter is None + else self.transform_max_iter + ) + + return self + + def _solve_W(self, X, H, max_iter): + """Minimize the objective function w.r.t W. + + Update W with H being fixed, until convergence. This is the heart + of `transform` but it's also used during `fit` when doing fresh restarts. + """ + avg = np.sqrt(X.mean() / self._n_components) + W = np.full((X.shape[0], self._n_components), avg, dtype=X.dtype) + W_buffer = W.copy() + + # Get scaled regularization terms. Done for each minibatch to take into account + # variable sizes of minibatches. + l1_reg_W, _, l2_reg_W, _ = self._compute_regularization(X) + + for _ in range(max_iter): + W, *_ = _multiplicative_update_w( + X, W, H, self._beta_loss, l1_reg_W, l2_reg_W, self._gamma + ) + + W_diff = linalg.norm(W - W_buffer) / linalg.norm(W) + if self.tol > 0 and W_diff <= self.tol: + break + + W_buffer[:] = W + + return W + + def _minibatch_step(self, X, W, H, update_H): + """Perform the update of W and H for one minibatch.""" + batch_size = X.shape[0] + + # get scaled regularization terms. Done for each minibatch to take into account + # variable sizes of minibatches. + l1_reg_W, l1_reg_H, l2_reg_W, l2_reg_H = self._compute_regularization(X) + + # update W + if self.fresh_restarts or W is None: + W = self._solve_W(X, H, self.fresh_restarts_max_iter) + else: + W, *_ = _multiplicative_update_w( + X, W, H, self._beta_loss, l1_reg_W, l2_reg_W, self._gamma + ) + + # necessary for stability with beta_loss < 1 + if self._beta_loss < 1: + W[W < np.finfo(np.float64).eps] = 0.0 + + batch_cost = ( + _beta_divergence(X, W, H, self._beta_loss) + + l1_reg_W * W.sum() + + l1_reg_H * H.sum() + + l2_reg_W * (W**2).sum() + + l2_reg_H * (H**2).sum() + ) / batch_size + + # update H (only at fit or fit_transform) + if update_H: + H[:] = _multiplicative_update_h( + X, + W, + H, + beta_loss=self._beta_loss, + l1_reg_H=l1_reg_H, + l2_reg_H=l2_reg_H, + gamma=self._gamma, + A=self._components_numerator, + B=self._components_denominator, + rho=self._rho, + ) + + # necessary for stability with beta_loss < 1 + if self._beta_loss <= 1: + H[H < np.finfo(np.float64).eps] = 0.0 + + return batch_cost + + def _minibatch_convergence( + self, X, batch_cost, H, H_buffer, n_samples, step, n_steps + ): + """Helper function to encapsulate the early stopping logic""" + batch_size = X.shape[0] + + # counts steps starting from 1 for user friendly verbose mode. + step = step + 1 + + # Ignore first iteration because H is not updated yet. + if step == 1: + if self.verbose: + print(f"Minibatch step {step}/{n_steps}: mean batch cost: {batch_cost}") + return False + + # Compute an Exponentially Weighted Average of the cost function to + # monitor the convergence while discarding minibatch-local stochastic + # variability: https://en.wikipedia.org/wiki/Moving_average + if self._ewa_cost is None: + self._ewa_cost = batch_cost + else: + alpha = batch_size / (n_samples + 1) + alpha = min(alpha, 1) + self._ewa_cost = self._ewa_cost * (1 - alpha) + batch_cost * alpha + + # Log progress to be able to monitor convergence + if self.verbose: + print( + f"Minibatch step {step}/{n_steps}: mean batch cost: " + f"{batch_cost}, ewa cost: {self._ewa_cost}" + ) + + # Early stopping based on change of H + H_diff = linalg.norm(H - H_buffer) / linalg.norm(H) + if self.tol > 0 and H_diff <= self.tol: + if self.verbose: + print(f"Converged (small H change) at step {step}/{n_steps}") + return True + + # Early stopping heuristic due to lack of improvement on smoothed + # cost function + if self._ewa_cost_min is None or self._ewa_cost < self._ewa_cost_min: + self._no_improvement = 0 + self._ewa_cost_min = self._ewa_cost + else: + self._no_improvement += 1 + + if ( + self.max_no_improvement is not None + and self._no_improvement >= self.max_no_improvement + ): + if self.verbose: + print( + "Converged (lack of improvement in objective function) " + f"at step {step}/{n_steps}" + ) + return True + + return False + + @_fit_context(prefer_skip_nested_validation=True) + def fit_transform(self, X, y=None, W=None, H=None): + """Learn a NMF model for the data X and returns the transformed data. + + This is more efficient than calling fit followed by transform. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Data matrix to be decomposed. y : Ignored + Not used, present here for API consistency by convention. + + W : array-like of shape (n_samples, n_components), default=None + If `init='custom'`, it is used as initial guess for the solution. + If `None`, uses the initialisation method specified in `init`. + + H : array-like of shape (n_components, n_features), default=None + If `init='custom'`, it is used as initial guess for the solution. + If `None`, uses the initialisation method specified in `init`. Returns ------- - self + W : ndarray of shape (n_samples, n_components) + Transformed data. """ - self.fit_transform(X, **params) - return self + X = validate_data( + self, X, accept_sparse=("csr", "csc"), dtype=[np.float64, np.float32] + ) + + with config_context(assume_finite=True): + W, H, n_iter, n_steps = self._fit_transform(X, W=W, H=H) + + self.reconstruction_err_ = _beta_divergence( + X, W, H, self._beta_loss, square_root=True + ) + + self.n_components_ = H.shape[0] + self.components_ = H + self.n_iter_ = n_iter + self.n_steps_ = n_steps + + return W + + def _fit_transform(self, X, W=None, H=None, update_H=True): + """Learn a NMF model for the data X and returns the transformed data. + + Parameters + ---------- + X : {ndarray, sparse matrix} of shape (n_samples, n_features) + Data matrix to be decomposed. + + W : array-like of shape (n_samples, n_components), default=None + If `init='custom'`, it is used as initial guess for the solution. + If `update_H=False`, it is initialised as an array of zeros, unless + `solver='mu'`, then it is filled with values calculated by + `np.sqrt(X.mean() / self._n_components)`. + If `None`, uses the initialisation method specified in `init`. + + H : array-like of shape (n_components, n_features), default=None + If `init='custom'`, it is used as initial guess for the solution. + If `update_H=False`, it is used as a constant, to solve for W only. + If `None`, uses the initialisation method specified in `init`. + + update_H : bool, default=True + If True, both W and H will be estimated from initial guesses, + this corresponds to a call to the `fit_transform` method. + If False, only W will be estimated, this corresponds to a call + to the `transform` method. + + Returns + ------- + W : ndarray of shape (n_samples, n_components) + Transformed data. + + H : ndarray of shape (n_components, n_features) + Factorization matrix, sometimes called 'dictionary'. + + n_iter : int + Actual number of started iterations over the whole dataset. + + n_steps : int + Number of mini-batches processed. + """ + check_non_negative(X, "MiniBatchNMF (input X)") + self._check_params(X) + + if X.min() == 0 and self._beta_loss <= 0: + raise ValueError( + "When beta_loss <= 0 and X contains zeros, " + "the solver may diverge. Please add small values " + "to X, or use a positive beta_loss." + ) + + n_samples = X.shape[0] + + # initialize or check W and H + W, H = self._check_w_h(X, W, H, update_H) + H_buffer = H.copy() + + # Initialize auxiliary matrices + self._components_numerator = H.copy() + self._components_denominator = np.ones(H.shape, dtype=H.dtype) + + # Attributes to monitor the convergence + self._ewa_cost = None + self._ewa_cost_min = None + self._no_improvement = 0 + + batches = gen_batches(n_samples, self._batch_size) + batches = itertools.cycle(batches) + n_steps_per_iter = int(np.ceil(n_samples / self._batch_size)) + n_steps = self.max_iter * n_steps_per_iter + + for i, batch in zip(range(n_steps), batches): + batch_cost = self._minibatch_step(X[batch], W[batch], H, update_H) + + if update_H and self._minibatch_convergence( + X[batch], batch_cost, H, H_buffer, n_samples, i, n_steps + ): + break + + H_buffer[:] = H + + if self.fresh_restarts: + W = self._solve_W(X, H, self._transform_max_iter) + + n_steps = i + 1 + n_iter = int(np.ceil(n_steps / n_steps_per_iter)) + + if n_iter == self.max_iter and self.tol > 0: + warnings.warn( + ( + f"Maximum number of iterations {self.max_iter} reached. " + "Increase it to improve convergence." + ), + ConvergenceWarning, + ) + + return W, H, n_iter, n_steps def transform(self, X): - """Transform the data X according to the fitted NMF model. + """Transform the data X according to the fitted MiniBatchNMF model. Parameters ---------- @@ -1411,29 +2335,75 @@ def transform(self, X): Transformed data. """ check_is_fitted(self) - X = self._validate_data(X, accept_sparse=('csr', 'csc'), - dtype=[np.float64, np.float32], - reset=False) + X = validate_data( + self, + X, + accept_sparse=("csr", "csc"), + dtype=[np.float64, np.float32], + reset=False, + ) - with config_context(assume_finite=True): - W, *_ = self._fit_transform(X, H=self.components_, update_H=False) + W = self._solve_W(X, self.components_, self._transform_max_iter) return W - def inverse_transform(self, W): - """Transform data back to its original space. + @_fit_context(prefer_skip_nested_validation=True) + def partial_fit(self, X, y=None, W=None, H=None): + """Update the model using the data in `X` as a mini-batch. + + This method is expected to be called several times consecutively + on different chunks of a dataset so as to implement out-of-core + or online learning. + + This is especially useful when the whole dataset is too big to fit in + memory at once (see :ref:`scaling_strategies`). Parameters ---------- - W : {ndarray, sparse matrix} of shape (n_samples, n_components) - Transformed data matrix. + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Data matrix to be decomposed. + + y : Ignored + Not used, present here for API consistency by convention. + + W : array-like of shape (n_samples, n_components), default=None + If `init='custom'`, it is used as initial guess for the solution. + Only used for the first call to `partial_fit`. + + H : array-like of shape (n_components, n_features), default=None + If `init='custom'`, it is used as initial guess for the solution. + Only used for the first call to `partial_fit`. Returns ------- - X : {ndarray, sparse matrix} of shape (n_samples, n_features) - Data matrix of original shape. - - .. versionadded:: 0.18 + self + Returns the instance itself. """ - check_is_fitted(self) - return np.dot(W, self.components_) + has_components = hasattr(self, "components_") + + X = validate_data( + self, + X, + accept_sparse=("csr", "csc"), + dtype=[np.float64, np.float32], + reset=not has_components, + ) + + if not has_components: + # This instance has not been fitted yet (fit or partial_fit) + self._check_params(X) + _, H = self._check_w_h(X, W=W, H=H, update_H=True) + + self._components_numerator = H.copy() + self._components_denominator = np.ones(H.shape, dtype=H.dtype) + self.n_steps_ = 0 + else: + H = self.components_ + + self._minibatch_step(X, None, H, update_H=True) + + self.n_components_ = H.shape[0] + self.components_ = H + self.n_steps_ += 1 + + return self diff --git a/sklearn/decomposition/_online_lda_fast.pyx b/sklearn/decomposition/_online_lda_fast.pyx index 1c00af02d2375..14f45ba9675f5 100644 --- a/sklearn/decomposition/_online_lda_fast.pyx +++ b/sklearn/decomposition/_online_lda_fast.pyx @@ -1,25 +1,20 @@ -# -# cython: boundscheck=False, wraparound=False - -cimport cython -cimport numpy as np import numpy as np -np.import_array() +from cython cimport floating from libc.math cimport exp, fabs, log -from numpy.math cimport EULER + +from ..utils._typedefs cimport float64_t, intp_t -def mean_change(np.ndarray[ndim=1, dtype=np.float64_t] arr_1, - np.ndarray[ndim=1, dtype=np.float64_t] arr_2): +def mean_change(const floating[:] arr_1, const floating[:] arr_2): """Calculate the mean difference between two arrays. Equivalent to np.abs(arr_1 - arr2).mean(). """ - cdef np.float64_t total, diff - cdef np.npy_intp i, size + cdef float64_t total, diff + cdef intp_t i, size size = arr_1.shape[0] total = 0.0 @@ -30,9 +25,11 @@ def mean_change(np.ndarray[ndim=1, dtype=np.float64_t] arr_1, return total / size -def _dirichlet_expectation_1d(np.ndarray[ndim=1, dtype=np.float64_t] doc_topic, - double doc_topic_prior, - np.ndarray[ndim=1, dtype=np.float64_t] out): +def _dirichlet_expectation_1d( + floating[:] doc_topic, + floating doc_topic_prior, + floating[:] out +): """Dirichlet expectation for a single sample: exp(E[log(theta)]) for theta ~ Dir(doc_topic) after adding doc_topic_prior to doc_topic, in-place. @@ -42,8 +39,8 @@ def _dirichlet_expectation_1d(np.ndarray[ndim=1, dtype=np.float64_t] doc_topic, out[:] = np.exp(psi(doc_topic) - psi(np.sum(doc_topic))) """ - cdef np.float64_t dt, psi_total, total - cdef np.npy_intp i, size + cdef floating dt, psi_total, total + cdef intp_t i, size size = doc_topic.shape[0] @@ -58,7 +55,7 @@ def _dirichlet_expectation_1d(np.ndarray[ndim=1, dtype=np.float64_t] doc_topic, out[i] = exp(psi(doc_topic[i]) - psi_total) -def _dirichlet_expectation_2d(np.ndarray[ndim=2, dtype=np.float64_t] arr): +def _dirichlet_expectation_2d(const floating[:, :] arr): """Dirichlet expectation for multiple samples: E[log(theta)] for theta ~ Dir(arr). @@ -67,9 +64,9 @@ def _dirichlet_expectation_2d(np.ndarray[ndim=2, dtype=np.float64_t] arr): Note that unlike _dirichlet_expectation_1d, this function doesn't compute the exp and doesn't add in the prior. """ - cdef np.float64_t row_total, psi_row_total - cdef np.ndarray[ndim=2, dtype=np.float64_t] d_exp - cdef np.npy_intp i, j, n_rows, n_cols + cdef floating row_total, psi_row_total + cdef floating[:, :] d_exp + cdef intp_t i, j, n_rows, n_cols n_rows = arr.shape[0] n_cols = arr.shape[1] @@ -84,20 +81,20 @@ def _dirichlet_expectation_2d(np.ndarray[ndim=2, dtype=np.float64_t] arr): for j in range(n_cols): d_exp[i, j] = psi(arr[i, j]) - psi_row_total - return d_exp + return d_exp.base # Psi function for positive arguments. Optimized for speed, not accuracy. # # After: J. Bernardo (1976). Algorithm AS 103: Psi (Digamma) Function. # https://www.uv.es/~bernardo/1976AppStatist.pdf -@cython.cdivision(True) -cdef double psi(double x) nogil: +cdef floating psi(floating x) noexcept nogil: + cdef double EULER = 0.577215664901532860606512090082402431 if x <= 1e-6: # psi(x) = -EULER - 1/x + O(x) return -EULER - 1. / x - cdef double r, result = 0 + cdef floating r, result = 0 # psi(x + 1) = psi(x) + 1/x while x < 6: @@ -110,4 +107,4 @@ cdef double psi(double x) nogil: result += log(x) - .5 * r r = r * r result -= r * ((1./12.) - r * ((1./120.) - r * (1./252.))) - return result; + return result diff --git a/sklearn/decomposition/_pca.py b/sklearn/decomposition/_pca.py index eb0a73919021a..1b0d21d5d38be 100644 --- a/sklearn/decomposition/_pca.py +++ b/sklearn/decomposition/_pca.py @@ -1,31 +1,25 @@ -""" Principal Component Analysis. -""" +"""Principal Component Analysis.""" -# Author: Alexandre Gramfort -# Olivier Grisel -# Mathieu Blondel -# Denis A. Engemann -# Michael Eickenberg -# Giorgio Patrini -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -from math import log, sqrt -import numbers +from math import lgamma, log, sqrt +from numbers import Integral, Real import numpy as np from scipy import linalg -from scipy.special import gammaln from scipy.sparse import issparse from scipy.sparse.linalg import svds -from ._base import _BasePCA +from ..base import _fit_context from ..utils import check_random_state from ..utils._arpack import _init_arpack_v0 -from ..utils.extmath import fast_logdet, randomized_svd, svd_flip -from ..utils.extmath import stable_cumsum -from ..utils.validation import check_is_fitted -from ..utils.validation import _deprecate_positional_args +from ..utils._array_api import _convert_to_numpy, get_namespace +from ..utils._param_validation import Interval, RealNotInt, StrOptions +from ..utils.extmath import _randomized_svd, fast_logdet, stable_cumsum, svd_flip +from ..utils.sparsefuncs import _implicit_column_offset, mean_variance_axis +from ..utils.validation import check_is_fitted, validate_data +from ._base import _BasePCA def _assess_dimension(spectrum, rank, n_samples): @@ -57,6 +51,7 @@ def _assess_dimension(spectrum, rank, n_samples): Automatic Choice of Dimensionality for PCA. NIPS 2000: 598-604 `_ """ + xp, _ = get_namespace(spectrum) n_features = spectrum.shape[0] if not 1 <= rank < n_features: @@ -70,31 +65,33 @@ def _assess_dimension(spectrum, rank, n_samples): # small and won't be the max anyway. Also, it can lead to numerical # issues below when computing pa, in particular in log((spectrum[i] - # spectrum[j]) because this will take the log of something very small. - return -np.inf + return -xp.inf - pu = -rank * log(2.) + pu = -rank * log(2.0) for i in range(1, rank + 1): - pu += (gammaln((n_features - i + 1) / 2.) - - log(np.pi) * (n_features - i + 1) / 2.) + pu += ( + lgamma((n_features - i + 1) / 2.0) - log(xp.pi) * (n_features - i + 1) / 2.0 + ) - pl = np.sum(np.log(spectrum[:rank])) - pl = -pl * n_samples / 2. + pl = xp.sum(xp.log(spectrum[:rank])) + pl = -pl * n_samples / 2.0 - v = max(eps, np.sum(spectrum[rank:]) / (n_features - rank)) - pv = -np.log(v) * n_samples * (n_features - rank) / 2. + v = max(eps, xp.sum(spectrum[rank:]) / (n_features - rank)) + pv = -log(v) * n_samples * (n_features - rank) / 2.0 - m = n_features * rank - rank * (rank + 1.) / 2. - pp = log(2. * np.pi) * (m + rank) / 2. + m = n_features * rank - rank * (rank + 1.0) / 2.0 + pp = log(2.0 * xp.pi) * (m + rank) / 2.0 - pa = 0. - spectrum_ = spectrum.copy() + pa = 0.0 + spectrum_ = xp.asarray(spectrum, copy=True) spectrum_[rank:n_features] = v for i in range(rank): - for j in range(i + 1, len(spectrum)): - pa += log((spectrum[i] - spectrum[j]) * - (1. / spectrum_[j] - 1. / spectrum_[i])) + log(n_samples) + for j in range(i + 1, spectrum.shape[0]): + pa += log( + (spectrum[i] - spectrum[j]) * (1.0 / spectrum_[j] - 1.0 / spectrum_[i]) + ) + log(n_samples) - ll = pu + pl + pv + pp - pa / 2. - rank * log(n_samples) / 2. + ll = pu + pl + pv + pp - pa / 2.0 - rank * log(n_samples) / 2.0 return ll @@ -104,11 +101,13 @@ def _infer_dimension(spectrum, n_samples): The returned value will be in [1, n_features - 1]. """ - ll = np.empty_like(spectrum) - ll[0] = -np.inf # we don't want to return n_components = 0 + xp, _ = get_namespace(spectrum) + + ll = xp.empty_like(spectrum) + ll[0] = -xp.inf # we don't want to return n_components = 0 for rank in range(1, spectrum.shape[0]): ll[rank] = _assess_dimension(spectrum, rank, n_samples) - return ll.argmax() + return xp.argmax(ll) class PCA(_BasePCA): @@ -122,11 +121,16 @@ class PCA(_BasePCA): SVD by the method of Halko et al. 2009, depending on the shape of the input data and the number of components to extract. - It can also use the scipy.sparse.linalg ARPACK implementation of the - truncated SVD. + With sparse inputs, the ARPACK implementation of the truncated SVD can be + used (i.e. through :func:`scipy.sparse.linalg.svds`). Alternatively, one + may consider :class:`TruncatedSVD` where the data are not centered. - Notice that this class does not support sparse input. See - :class:`TruncatedSVD` for an alternative with sparse data. + Notice that this class only supports sparse inputs for some solvers such as + "arpack" and "covariance_eigh". See :class:`TruncatedSVD` for an + alternative with sparse data. + + For a usage example, see + :ref:`sphx_glr_auto_examples_decomposition_plot_pca_iris.py` Read more in the :ref:`User Guide `. @@ -168,26 +172,43 @@ class PCA(_BasePCA): improve the predictive accuracy of the downstream estimators by making their data respect some hard-wired assumptions. - svd_solver : {'auto', 'full', 'arpack', 'randomized'}, default='auto' - If auto : - The solver is selected by a default policy based on `X.shape` and - `n_components`: if the input data is larger than 500x500 and the - number of components to extract is lower than 80% of the smallest - dimension of the data, then the more efficient 'randomized' - method is enabled. Otherwise the exact full SVD is computed and - optionally truncated afterwards. - If full : - run exact full SVD calling the standard LAPACK solver via + svd_solver : {'auto', 'full', 'covariance_eigh', 'arpack', 'randomized'},\ + default='auto' + "auto" : + The solver is selected by a default 'auto' policy is based on `X.shape` and + `n_components`: if the input data has fewer than 1000 features and + more than 10 times as many samples, then the "covariance_eigh" + solver is used. Otherwise, if the input data is larger than 500x500 + and the number of components to extract is lower than 80% of the + smallest dimension of the data, then the more efficient + "randomized" method is selected. Otherwise the exact "full" SVD is + computed and optionally truncated afterwards. + "full" : + Run exact full SVD calling the standard LAPACK solver via `scipy.linalg.svd` and select the components by postprocessing - If arpack : - run SVD truncated to n_components calling ARPACK solver via + "covariance_eigh" : + Precompute the covariance matrix (on centered data), run a + classical eigenvalue decomposition on the covariance matrix + typically using LAPACK and select the components by postprocessing. + This solver is very efficient for n_samples >> n_features and small + n_features. It is, however, not tractable otherwise for large + n_features (large memory footprint required to materialize the + covariance matrix). Also note that compared to the "full" solver, + this solver effectively doubles the condition number and is + therefore less numerical stable (e.g. on input data with a large + range of singular values). + "arpack" : + Run SVD truncated to `n_components` calling ARPACK solver via `scipy.sparse.linalg.svds`. It requires strictly - 0 < n_components < min(X.shape) - If randomized : - run randomized SVD by the method of Halko et al. + `0 < n_components < min(X.shape)` + "randomized" : + Run randomized SVD by the method of Halko et al. .. versionadded:: 0.18.0 + .. versionchanged:: 1.5 + Added the 'covariance_eigh' solver. + tol : float, default=0.0 Tolerance for singular values computed by svd_solver == 'arpack'. Must be of range [0.0, infinity). @@ -201,6 +222,21 @@ class PCA(_BasePCA): .. versionadded:: 0.18.0 + n_oversamples : int, default=10 + This parameter is only relevant when `svd_solver="randomized"`. + It corresponds to the additional number of random vectors to sample the + range of `X` so as to ensure proper conditioning. See + :func:`~sklearn.utils.extmath.randomized_svd` for more details. + + .. versionadded:: 1.1 + + power_iteration_normalizer : {'auto', 'QR', 'LU', 'none'}, default='auto' + Power iteration normalizer for randomized SVD solver. + Not used by ARPACK. See :func:`~sklearn.utils.extmath.randomized_svd` + for more details. + + .. versionadded:: 1.1 + random_state : int, RandomState instance or None, default=None Used when the 'arpack' or 'randomized' solvers are used. Pass an int for reproducible results across multiple function calls. @@ -212,8 +248,9 @@ class PCA(_BasePCA): ---------- components_ : ndarray of shape (n_components, n_features) Principal axes in feature space, representing the directions of - maximum variance in the data. The components are sorted by - ``explained_variance_``. + maximum variance in the data. Equivalently, the right singular + vectors of the centered input data, parallel to its eigenvectors. + The components are sorted by decreasing ``explained_variance_``. explained_variance_ : ndarray of shape (n_components,) The amount of variance explained by each of the selected components. @@ -249,9 +286,6 @@ class PCA(_BasePCA): n_components, or the lesser value of n_features and n_samples if n_components is None. - n_features_ : int - Number of features in the training data. - n_samples_ : int Number of samples in the training data. @@ -265,6 +299,17 @@ class PCA(_BasePCA): Equal to the average of (min(n_features, n_samples) - n_components) smallest eigenvalues of the covariance matrix of X. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + See Also -------- KernelPCA : Kernel Principal Component Analysis. @@ -288,16 +333,16 @@ class PCA(_BasePCA): For svd_solver == 'arpack', refer to `scipy.sparse.linalg.svds`. For svd_solver == 'randomized', see: - `Halko, N., Martinsson, P. G., and Tropp, J. A. (2011). + :doi:`Halko, N., Martinsson, P. G., and Tropp, J. A. (2011). "Finding structure with randomness: Probabilistic algorithms for constructing approximate matrix decompositions". SIAM review, 53(2), 217-288. - `_ + <10.1137/090771806>` and also - `Martinsson, P. G., Rokhlin, V., and Tygert, M. (2011). + :doi:`Martinsson, P. G., Rokhlin, V., and Tygert, M. (2011). "A randomized algorithm for the decomposition of matrices". - Applied and Computational Harmonic Analysis, 30(1), 47-68 - `_. + Applied and Computational Harmonic Analysis, 30(1), 47-68. + <10.1016/j.acha.2010.02.003>` Examples -------- @@ -308,48 +353,84 @@ class PCA(_BasePCA): >>> pca.fit(X) PCA(n_components=2) >>> print(pca.explained_variance_ratio_) - [0.9924... 0.0075...] + [0.9924 0.0075] >>> print(pca.singular_values_) - [6.30061... 0.54980...] + [6.30061 0.54980] >>> pca = PCA(n_components=2, svd_solver='full') >>> pca.fit(X) PCA(n_components=2, svd_solver='full') >>> print(pca.explained_variance_ratio_) - [0.9924... 0.00755...] + [0.9924 0.00755] >>> print(pca.singular_values_) - [6.30061... 0.54980...] + [6.30061 0.54980] >>> pca = PCA(n_components=1, svd_solver='arpack') >>> pca.fit(X) PCA(n_components=1, svd_solver='arpack') >>> print(pca.explained_variance_ratio_) - [0.99244...] + [0.99244] >>> print(pca.singular_values_) - [6.30061...] + [6.30061] """ - @_deprecate_positional_args - def __init__(self, n_components=None, *, copy=True, whiten=False, - svd_solver='auto', tol=0.0, iterated_power='auto', - random_state=None): + + _parameter_constraints: dict = { + "n_components": [ + Interval(Integral, 0, None, closed="left"), + Interval(RealNotInt, 0, 1, closed="neither"), + StrOptions({"mle"}), + None, + ], + "copy": ["boolean"], + "whiten": ["boolean"], + "svd_solver": [ + StrOptions({"auto", "full", "covariance_eigh", "arpack", "randomized"}) + ], + "tol": [Interval(Real, 0, None, closed="left")], + "iterated_power": [ + StrOptions({"auto"}), + Interval(Integral, 0, None, closed="left"), + ], + "n_oversamples": [Interval(Integral, 1, None, closed="left")], + "power_iteration_normalizer": [StrOptions({"auto", "QR", "LU", "none"})], + "random_state": ["random_state"], + } + + def __init__( + self, + n_components=None, + *, + copy=True, + whiten=False, + svd_solver="auto", + tol=0.0, + iterated_power="auto", + n_oversamples=10, + power_iteration_normalizer="auto", + random_state=None, + ): self.n_components = n_components self.copy = copy self.whiten = whiten self.svd_solver = svd_solver self.tol = tol self.iterated_power = iterated_power + self.n_oversamples = n_oversamples + self.power_iteration_normalizer = power_iteration_normalizer self.random_state = random_state + @_fit_context(prefer_skip_nested_validation=True) def fit(self, X, y=None): """Fit the model with X. Parameters ---------- - X : array-like of shape (n_samples, n_features) - Training data, where n_samples is the number of samples - and n_features is the number of features. + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training data, where `n_samples` is the number of samples + and `n_features` is the number of features. y : Ignored + Ignored. Returns ------- @@ -359,16 +440,18 @@ def fit(self, X, y=None): self._fit(X) return self + @_fit_context(prefer_skip_nested_validation=True) def fit_transform(self, X, y=None): """Fit the model with X and apply the dimensionality reduction on X. Parameters ---------- - X : array-like of shape (n_samples, n_features) - Training data, where n_samples is the number of samples - and n_features is the number of features. + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training data, where `n_samples` is the number of samples + and `n_features` is the number of features. y : Ignored + Ignored. Returns ------- @@ -380,195 +463,337 @@ def fit_transform(self, X, y=None): This method returns a Fortran-ordered array. To convert it to a C-ordered array, use 'np.ascontiguousarray'. """ - U, S, Vt = self._fit(X) - U = U[:, :self.n_components_] + U, S, _, X, x_is_centered, xp = self._fit(X) + if U is not None: + U = U[:, : self.n_components_] - if self.whiten: - # X_new = X * V / S * sqrt(n_samples) = U * sqrt(n_samples) - U *= sqrt(X.shape[0] - 1) - else: - # X_new = X * V = U * S * Vt * V = U * S - U *= S[:self.n_components_] + if self.whiten: + # X_new = X * V / S * sqrt(n_samples) = U * sqrt(n_samples) + U *= sqrt(X.shape[0] - 1) + else: + # X_new = X * V = U * S * Vt * V = U * S + U *= S[: self.n_components_] - return U + return U + else: # solver="covariance_eigh" does not compute U at fit time. + return self._transform(X, xp, x_is_centered=x_is_centered) def _fit(self, X): """Dispatch to the right submethod depending on the chosen solver.""" + xp, is_array_api_compliant = get_namespace(X) + + # Raise an error for sparse input and unsupported svd_solver + if issparse(X) and self.svd_solver not in ["auto", "arpack", "covariance_eigh"]: + raise TypeError( + 'PCA only support sparse inputs with the "arpack" and' + f' "covariance_eigh" solvers, while "{self.svd_solver}" was passed. See' + " TruncatedSVD for a possible alternative." + ) + if self.svd_solver == "arpack" and is_array_api_compliant: + raise ValueError( + "PCA with svd_solver='arpack' is not supported for Array API inputs." + ) + + # Validate the data, without ever forcing a copy as any solver that + # supports sparse input data and the `covariance_eigh` solver are + # written in a way to avoid the need for any inplace modification of + # the input data contrary to the other solvers. + # The copy will happen + # later, only if needed, once the solver negotiation below is done. + X = validate_data( + self, + X, + dtype=[xp.float64, xp.float32], + force_writeable=True, + accept_sparse=("csr", "csc"), + ensure_2d=True, + copy=False, + ) + self._fit_svd_solver = self.svd_solver + if self._fit_svd_solver == "auto" and issparse(X): + self._fit_svd_solver = "arpack" - # Raise an error for sparse input. - # This is more informative than the generic one raised by check_array. - if issparse(X): - raise TypeError('PCA does not support sparse input. See ' - 'TruncatedSVD for a possible alternative.') - - X = self._validate_data(X, dtype=[np.float64, np.float32], - ensure_2d=True, copy=self.copy) - - # Handle n_components==None if self.n_components is None: - if self.svd_solver != 'arpack': + if self._fit_svd_solver != "arpack": n_components = min(X.shape) else: n_components = min(X.shape) - 1 else: n_components = self.n_components - # Handle svd_solver - self._fit_svd_solver = self.svd_solver - if self._fit_svd_solver == 'auto': + if self._fit_svd_solver == "auto": + # Tall and skinny problems are best handled by precomputing the + # covariance matrix. + if X.shape[1] <= 1_000 and X.shape[0] >= 10 * X.shape[1]: + self._fit_svd_solver = "covariance_eigh" # Small problem or n_components == 'mle', just call full PCA - if max(X.shape) <= 500 or n_components == 'mle': - self._fit_svd_solver = 'full' - elif n_components >= 1 and n_components < .8 * min(X.shape): - self._fit_svd_solver = 'randomized' - # This is also the case of n_components in (0,1) + elif max(X.shape) <= 500 or n_components == "mle": + self._fit_svd_solver = "full" + elif 1 <= n_components < 0.8 * min(X.shape): + self._fit_svd_solver = "randomized" + # This is also the case of n_components in (0, 1) else: - self._fit_svd_solver = 'full' + self._fit_svd_solver = "full" # Call different fits for either full or truncated SVD - if self._fit_svd_solver == 'full': - return self._fit_full(X, n_components) - elif self._fit_svd_solver in ['arpack', 'randomized']: - return self._fit_truncated(X, n_components, self._fit_svd_solver) - else: - raise ValueError("Unrecognized svd_solver='{0}'" - "".format(self._fit_svd_solver)) + if self._fit_svd_solver in ("full", "covariance_eigh"): + return self._fit_full(X, n_components, xp, is_array_api_compliant) + elif self._fit_svd_solver in ["arpack", "randomized"]: + return self._fit_truncated(X, n_components, xp) - def _fit_full(self, X, n_components): + def _fit_full(self, X, n_components, xp, is_array_api_compliant): """Fit the model by computing full SVD on X.""" n_samples, n_features = X.shape - if n_components == 'mle': + if n_components == "mle": if n_samples < n_features: - raise ValueError("n_components='mle' is only supported " - "if n_samples >= n_features") + raise ValueError( + "n_components='mle' is only supported if n_samples >= n_features" + ) elif not 0 <= n_components <= min(n_samples, n_features): - raise ValueError("n_components=%r must be between 0 and " - "min(n_samples, n_features)=%r with " - "svd_solver='full'" - % (n_components, min(n_samples, n_features))) - elif n_components >= 1: - if not isinstance(n_components, numbers.Integral): - raise ValueError("n_components=%r must be of type int " - "when greater than or equal to 1, " - "was of type=%r" - % (n_components, type(n_components))) + raise ValueError( + f"n_components={n_components} must be between 0 and " + f"min(n_samples, n_features)={min(n_samples, n_features)} with " + f"svd_solver={self._fit_svd_solver!r}" + ) + + self.mean_ = xp.mean(X, axis=0) + # When X is a scipy sparse matrix, self.mean_ is a numpy matrix, so we need + # to transform it to a 1D array. Note that this is not the case when X + # is a scipy sparse array. + # TODO: remove the following two lines when scikit-learn only depends + # on scipy versions that no longer support scipy.sparse matrices. + self.mean_ = xp.reshape(xp.asarray(self.mean_), (-1,)) + + if self._fit_svd_solver == "full": + X_centered = xp.asarray(X, copy=True) if self.copy else X + X_centered -= self.mean_ + x_is_centered = not self.copy + + if not is_array_api_compliant: + # Use scipy.linalg with NumPy/SciPy inputs for the sake of not + # introducing unanticipated behavior changes. In the long run we + # could instead decide to always use xp.linalg.svd for all inputs, + # but that would make this code rely on numpy's SVD instead of + # scipy's. It's not 100% clear whether they use the same LAPACK + # solver by default though (assuming both are built against the + # same BLAS). + U, S, Vt = linalg.svd(X_centered, full_matrices=False) + else: + U, S, Vt = xp.linalg.svd(X_centered, full_matrices=False) + explained_variance_ = (S**2) / (n_samples - 1) - # Center data - self.mean_ = np.mean(X, axis=0) - X -= self.mean_ + else: + assert self._fit_svd_solver == "covariance_eigh" + # In the following, we center the covariance matrix C afterwards + # (without centering the data X first) to avoid an unnecessary copy + # of X. Note that the mean_ attribute is still needed to center + # test data in the transform method. + # + # Note: at the time of writing, `xp.cov` does not exist in the + # Array API standard: + # https://github.com/data-apis/array-api/issues/43 + # + # Besides, using `numpy.cov`, as of numpy 1.26.0, would not be + # memory efficient for our use case when `n_samples >> n_features`: + # `numpy.cov` centers a copy of the data before computing the + # matrix product instead of subtracting a small `(n_features, + # n_features)` square matrix from the gram matrix X.T @ X, as we do + # below. + x_is_centered = False + C = X.T @ X + C -= ( + n_samples + * xp.reshape(self.mean_, (-1, 1)) + * xp.reshape(self.mean_, (1, -1)) + ) + C /= n_samples - 1 + eigenvals, eigenvecs = xp.linalg.eigh(C) + + # When X is a scipy sparse matrix, the following two datastructures + # are returned as instances of the soft-deprecated numpy.matrix + # class. Note that this problem does not occur when X is a scipy + # sparse array (or another other kind of supported array). + # TODO: remove the following two lines when scikit-learn only + # depends on scipy versions that no longer support scipy.sparse + # matrices. + eigenvals = xp.reshape(xp.asarray(eigenvals), (-1,)) + eigenvecs = xp.asarray(eigenvecs) + + eigenvals = xp.flip(eigenvals, axis=0) + eigenvecs = xp.flip(eigenvecs, axis=1) + + # The covariance matrix C is positive semi-definite by + # construction. However, the eigenvalues returned by xp.linalg.eigh + # can be slightly negative due to numerical errors. This would be + # an issue for the subsequent sqrt, hence the manual clipping. + eigenvals[eigenvals < 0.0] = 0.0 + explained_variance_ = eigenvals + + # Re-construct SVD of centered X indirectly and make it consistent + # with the other solvers. + S = xp.sqrt(eigenvals * (n_samples - 1)) + Vt = eigenvecs.T + U = None - U, S, Vt = linalg.svd(X, full_matrices=False) # flip eigenvectors' sign to enforce deterministic output - U, Vt = svd_flip(U, Vt) + U, Vt = svd_flip(U, Vt, u_based_decision=False) components_ = Vt # Get variance explained by singular values - explained_variance_ = (S ** 2) / (n_samples - 1) - total_var = explained_variance_.sum() + total_var = xp.sum(explained_variance_) explained_variance_ratio_ = explained_variance_ / total_var - singular_values_ = S.copy() # Store the singular values. + singular_values_ = xp.asarray(S, copy=True) # Store the singular values. # Postprocess the number of components required - if n_components == 'mle': - n_components = \ - _infer_dimension(explained_variance_, n_samples) + if n_components == "mle": + n_components = _infer_dimension(explained_variance_, n_samples) elif 0 < n_components < 1.0: # number of components for which the cumulated explained # variance percentage is superior to the desired threshold # side='right' ensures that number of features selected # their variance is always greater than n_components float # passed. More discussion in issue: #15669 - ratio_cumsum = stable_cumsum(explained_variance_ratio_) - n_components = np.searchsorted(ratio_cumsum, n_components, - side='right') + 1 + if is_array_api_compliant: + # Convert to numpy as xp.cumsum and xp.searchsorted are not + # part of the Array API standard yet: + # + # https://github.com/data-apis/array-api/issues/597 + # https://github.com/data-apis/array-api/issues/688 + # + # Furthermore, it's not always safe to call them for namespaces + # that already implement them: for instance as + # cupy.searchsorted does not accept a float as second argument. + explained_variance_ratio_np = _convert_to_numpy( + explained_variance_ratio_, xp=xp + ) + else: + explained_variance_ratio_np = explained_variance_ratio_ + ratio_cumsum = stable_cumsum(explained_variance_ratio_np) + n_components = np.searchsorted(ratio_cumsum, n_components, side="right") + 1 + # Compute noise covariance using Probabilistic PCA model # The sigma2 maximum likelihood (cf. eq. 12.46) if n_components < min(n_features, n_samples): - self.noise_variance_ = explained_variance_[n_components:].mean() + self.noise_variance_ = xp.mean(explained_variance_[n_components:]) else: - self.noise_variance_ = 0. + self.noise_variance_ = 0.0 - self.n_samples_, self.n_features_ = n_samples, n_features - self.components_ = components_[:n_components] + self.n_samples_ = n_samples self.n_components_ = n_components - self.explained_variance_ = explained_variance_[:n_components] - self.explained_variance_ratio_ = \ - explained_variance_ratio_[:n_components] - self.singular_values_ = singular_values_[:n_components] - - return U, S, Vt - - def _fit_truncated(self, X, n_components, svd_solver): + # Assign a copy of the result of the truncation of the components in + # order to: + # - release the memory used by the discarded components, + # - ensure that the kept components are allocated contiguously in + # memory to make the transform method faster by leveraging cache + # locality. + self.components_ = xp.asarray(components_[:n_components, :], copy=True) + + # We do the same for the other arrays for the sake of consistency. + self.explained_variance_ = xp.asarray( + explained_variance_[:n_components], copy=True + ) + self.explained_variance_ratio_ = xp.asarray( + explained_variance_ratio_[:n_components], copy=True + ) + self.singular_values_ = xp.asarray(singular_values_[:n_components], copy=True) + + return U, S, Vt, X, x_is_centered, xp + + def _fit_truncated(self, X, n_components, xp): """Fit the model by computing truncated SVD (by ARPACK or randomized) on X. """ n_samples, n_features = X.shape + svd_solver = self._fit_svd_solver if isinstance(n_components, str): - raise ValueError("n_components=%r cannot be a string " - "with svd_solver='%s'" - % (n_components, svd_solver)) + raise ValueError( + "n_components=%r cannot be a string with svd_solver='%s'" + % (n_components, svd_solver) + ) elif not 1 <= n_components <= min(n_samples, n_features): - raise ValueError("n_components=%r must be between 1 and " - "min(n_samples, n_features)=%r with " - "svd_solver='%s'" - % (n_components, min(n_samples, n_features), - svd_solver)) - elif not isinstance(n_components, numbers.Integral): - raise ValueError("n_components=%r must be of type int " - "when greater than or equal to 1, was of type=%r" - % (n_components, type(n_components))) - elif svd_solver == 'arpack' and n_components == min(n_samples, - n_features): - raise ValueError("n_components=%r must be strictly less than " - "min(n_samples, n_features)=%r with " - "svd_solver='%s'" - % (n_components, min(n_samples, n_features), - svd_solver)) + raise ValueError( + "n_components=%r must be between 1 and " + "min(n_samples, n_features)=%r with " + "svd_solver='%s'" + % (n_components, min(n_samples, n_features), svd_solver) + ) + elif svd_solver == "arpack" and n_components == min(n_samples, n_features): + raise ValueError( + "n_components=%r must be strictly less than " + "min(n_samples, n_features)=%r with " + "svd_solver='%s'" + % (n_components, min(n_samples, n_features), svd_solver) + ) random_state = check_random_state(self.random_state) # Center data - self.mean_ = np.mean(X, axis=0) - X -= self.mean_ + total_var = None + if issparse(X): + self.mean_, var = mean_variance_axis(X, axis=0) + total_var = var.sum() * n_samples / (n_samples - 1) # ddof=1 + X_centered = _implicit_column_offset(X, self.mean_) + x_is_centered = False + else: + self.mean_ = xp.mean(X, axis=0) + X_centered = xp.asarray(X, copy=True) if self.copy else X + X_centered -= self.mean_ + x_is_centered = not self.copy - if svd_solver == 'arpack': + if svd_solver == "arpack": v0 = _init_arpack_v0(min(X.shape), random_state) - U, S, Vt = svds(X, k=n_components, tol=self.tol, v0=v0) + U, S, Vt = svds(X_centered, k=n_components, tol=self.tol, v0=v0) # svds doesn't abide by scipy.linalg.svd/randomized_svd # conventions, so reverse its outputs. S = S[::-1] # flip eigenvectors' sign to enforce deterministic output - U, Vt = svd_flip(U[:, ::-1], Vt[::-1]) + U, Vt = svd_flip(U[:, ::-1], Vt[::-1], u_based_decision=False) - elif svd_solver == 'randomized': + elif svd_solver == "randomized": # sign flipping is done inside - U, S, Vt = randomized_svd(X, n_components=n_components, - n_iter=self.iterated_power, - flip_sign=True, - random_state=random_state) - - self.n_samples_, self.n_features_ = n_samples, n_features + U, S, Vt = _randomized_svd( + X_centered, + n_components=n_components, + n_oversamples=self.n_oversamples, + n_iter=self.iterated_power, + power_iteration_normalizer=self.power_iteration_normalizer, + flip_sign=False, + random_state=random_state, + ) + U, Vt = svd_flip(U, Vt, u_based_decision=False) + + self.n_samples_ = n_samples self.components_ = Vt self.n_components_ = n_components # Get variance explained by singular values - self.explained_variance_ = (S ** 2) / (n_samples - 1) - total_var = np.var(X, ddof=1, axis=0) - self.explained_variance_ratio_ = \ - self.explained_variance_ / total_var.sum() - self.singular_values_ = S.copy() # Store the singular values. + self.explained_variance_ = (S**2) / (n_samples - 1) + + # Workaround in-place variance calculation since at the time numpy + # did not have a way to calculate variance in-place. + # + # TODO: update this code to either: + # * Use the array-api variance calculation, unless memory usage suffers + # * Update sklearn.utils.extmath._incremental_mean_and_var to support array-api + # See: https://github.com/scikit-learn/scikit-learn/pull/18689#discussion_r1335540991 + if total_var is None: + N = X.shape[0] - 1 + X_centered **= 2 + total_var = xp.sum(X_centered) / N + + self.explained_variance_ratio_ = self.explained_variance_ / total_var + self.singular_values_ = xp.asarray(S, copy=True) # Store the singular values. if self.n_components_ < min(n_features, n_samples): - self.noise_variance_ = (total_var.sum() - - self.explained_variance_.sum()) + self.noise_variance_ = total_var - xp.sum(self.explained_variance_) self.noise_variance_ /= min(n_features, n_samples) - n_components else: - self.noise_variance_ = 0. + self.noise_variance_ = 0.0 - return U, S, Vt + return U, S, Vt, X, x_is_centered, xp def score_samples(self, X): """Return the log-likelihood of each sample. @@ -588,14 +813,13 @@ def score_samples(self, X): Log-likelihood of each sample under the current model. """ check_is_fitted(self) - - X = self._validate_data(X, dtype=[np.float64, np.float32], reset=False) + xp, _ = get_namespace(X) + X = validate_data(self, X, dtype=[xp.float64, xp.float32], reset=False) Xr = X - self.mean_ n_features = X.shape[1] precision = self.get_precision() - log_like = -.5 * (Xr * (np.dot(Xr, precision))).sum(axis=1) - log_like -= .5 * (n_features * log(2. * np.pi) - - fast_logdet(precision)) + log_like = -0.5 * xp.sum(Xr * (Xr @ precision), axis=1) + log_like -= 0.5 * (n_features * log(2.0 * np.pi) - fast_logdet(precision)) return log_like def score(self, X, y=None): @@ -611,13 +835,23 @@ def score(self, X, y=None): The data. y : Ignored + Ignored. Returns ------- ll : float Average log-likelihood of the samples under the current model. """ - return np.mean(self.score_samples(X)) - - def _more_tags(self): - return {'preserves_dtype': [np.float64, np.float32]} + xp, _ = get_namespace(X) + return float(xp.mean(self.score_samples(X))) + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.transformer_tags.preserves_dtype = ["float64", "float32"] + tags.array_api_support = True + tags.input_tags.sparse = self.svd_solver in ( + "auto", + "arpack", + "covariance_eigh", + ) + return tags diff --git a/sklearn/decomposition/_sparse_pca.py b/sklearn/decomposition/_sparse_pca.py index 2348ada255fd4..2717230c9df92 100644 --- a/sklearn/decomposition/_sparse_pca.py +++ b/sklearn/decomposition/_sparse_pca.py @@ -1,18 +1,162 @@ """Matrix factorization with Sparse PCA.""" -# Author: Vlad Niculae, Gael Varoquaux, Alexandre Gramfort -# License: BSD 3 clause + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from numbers import Integral, Real import numpy as np -from ..utils import check_random_state -from ..utils.validation import check_is_fitted -from ..utils.validation import _deprecate_positional_args +from ..base import ( + BaseEstimator, + ClassNamePrefixFeaturesOutMixin, + TransformerMixin, + _fit_context, +) from ..linear_model import ridge_regression -from ..base import BaseEstimator, TransformerMixin -from ._dict_learning import dict_learning, dict_learning_online +from ..utils import check_random_state +from ..utils._param_validation import Interval, StrOptions +from ..utils.extmath import svd_flip +from ..utils.validation import check_array, check_is_fitted, validate_data +from ._dict_learning import MiniBatchDictionaryLearning, dict_learning + + +class _BaseSparsePCA(ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator): + """Base class for SparsePCA and MiniBatchSparsePCA""" + + _parameter_constraints: dict = { + "n_components": [None, Interval(Integral, 1, None, closed="left")], + "alpha": [Interval(Real, 0.0, None, closed="left")], + "ridge_alpha": [Interval(Real, 0.0, None, closed="left")], + "max_iter": [Interval(Integral, 0, None, closed="left")], + "tol": [Interval(Real, 0.0, None, closed="left")], + "method": [StrOptions({"lars", "cd"})], + "n_jobs": [Integral, None], + "verbose": ["verbose"], + "random_state": ["random_state"], + } + + def __init__( + self, + n_components=None, + *, + alpha=1, + ridge_alpha=0.01, + max_iter=1000, + tol=1e-8, + method="lars", + n_jobs=None, + verbose=False, + random_state=None, + ): + self.n_components = n_components + self.alpha = alpha + self.ridge_alpha = ridge_alpha + self.max_iter = max_iter + self.tol = tol + self.method = method + self.n_jobs = n_jobs + self.verbose = verbose + self.random_state = random_state + + @_fit_context(prefer_skip_nested_validation=True) + def fit(self, X, y=None): + """Fit the model from data in X. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features) + Training vector, where `n_samples` is the number of samples + and `n_features` is the number of features. + + y : Ignored + Not used, present here for API consistency by convention. + + Returns + ------- + self : object + Returns the instance itself. + """ + random_state = check_random_state(self.random_state) + X = validate_data(self, X) + self.mean_ = X.mean(axis=0) + X = X - self.mean_ -class SparsePCA(TransformerMixin, BaseEstimator): + if self.n_components is None: + n_components = X.shape[1] + else: + n_components = self.n_components + + return self._fit(X, n_components, random_state) + + def transform(self, X): + """Least Squares projection of the data onto the sparse components. + + To avoid instability issues in case the system is under-determined, + regularization can be applied (Ridge regression) via the + `ridge_alpha` parameter. + + Note that Sparse PCA components orthogonality is not enforced as in PCA + hence one cannot use a simple linear projection. + + Parameters + ---------- + X : ndarray of shape (n_samples, n_features) + Test data to be transformed, must have the same number of + features as the data used to train the model. + + Returns + ------- + X_new : ndarray of shape (n_samples, n_components) + Transformed data. + """ + check_is_fitted(self) + + X = validate_data(self, X, reset=False) + X = X - self.mean_ + + U = ridge_regression( + self.components_.T, X.T, self.ridge_alpha, solver="cholesky" + ) + + return U + + def inverse_transform(self, X): + """Transform data from the latent space to the original space. + + This inversion is an approximation due to the loss of information + induced by the forward decomposition. + + .. versionadded:: 1.2 + + Parameters + ---------- + X : ndarray of shape (n_samples, n_components) + Data in the latent space. + + Returns + ------- + X_original : ndarray of shape (n_samples, n_features) + Reconstructed data in the original space. + """ + check_is_fitted(self) + X = check_array(X) + + return (X @ self.components_) + self.mean_ + + @property + def _n_features_out(self): + """Number of transformed output features.""" + return self.components_.shape[0] + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.transformer_tags.preserves_dtype = ["float64", "float32"] + return tags + + +class SparsePCA(_BaseSparsePCA): """Sparse Principal Components Analysis (SparsePCA). Finds the set of sparse components that can optimally reconstruct @@ -24,7 +168,8 @@ class SparsePCA(TransformerMixin, BaseEstimator): Parameters ---------- n_components : int, default=None - Number of sparse atoms to extract. + Number of sparse atoms to extract. If None, then ``n_components`` + is set to ``n_features``. alpha : float, default=1 Sparsity controlling parameter. Higher values lead to sparser @@ -41,6 +186,7 @@ class SparsePCA(TransformerMixin, BaseEstimator): Tolerance for the stopping condition. method : {'lars', 'cd'}, default='lars' + Method to be used for optimization. lars: uses the least angle regression method to solve the lasso problem (linear_model.lars_path) cd: uses the coordinate descent method to compute the @@ -89,6 +235,24 @@ class SparsePCA(TransformerMixin, BaseEstimator): Per-feature empirical mean, estimated from the training set. Equal to ``X.mean(axis=0)``. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + PCA : Principal Component Analysis implementation. + MiniBatchSparsePCA : Mini batch variant of `SparsePCA` that is faster but less + accurate. + DictionaryLearning : Generic dictionary learning problem using a sparse code. + Examples -------- >>> import numpy as np @@ -103,72 +267,67 @@ class SparsePCA(TransformerMixin, BaseEstimator): (200, 5) >>> # most values in the components_ are zero (sparsity) >>> np.mean(transformer.components_ == 0) - 0.9666... - - See Also - -------- - PCA - MiniBatchSparsePCA - DictionaryLearning + np.float64(0.9666) """ - @_deprecate_positional_args - def __init__(self, n_components=None, *, alpha=1, ridge_alpha=0.01, - max_iter=1000, tol=1e-8, method='lars', n_jobs=None, - U_init=None, V_init=None, verbose=False, random_state=None): - self.n_components = n_components - self.alpha = alpha - self.ridge_alpha = ridge_alpha - self.max_iter = max_iter - self.tol = tol - self.method = method - self.n_jobs = n_jobs + + _parameter_constraints: dict = { + **_BaseSparsePCA._parameter_constraints, + "U_init": [None, np.ndarray], + "V_init": [None, np.ndarray], + } + + def __init__( + self, + n_components=None, + *, + alpha=1, + ridge_alpha=0.01, + max_iter=1000, + tol=1e-8, + method="lars", + n_jobs=None, + U_init=None, + V_init=None, + verbose=False, + random_state=None, + ): + super().__init__( + n_components=n_components, + alpha=alpha, + ridge_alpha=ridge_alpha, + max_iter=max_iter, + tol=tol, + method=method, + n_jobs=n_jobs, + verbose=verbose, + random_state=random_state, + ) self.U_init = U_init self.V_init = V_init - self.verbose = verbose - self.random_state = random_state - def fit(self, X, y=None): - """Fit the model from data in X. + def _fit(self, X, n_components, random_state): + """Specialized `fit` for SparsePCA.""" - Parameters - ---------- - X : array-like of shape (n_samples, n_features) - Training vector, where n_samples in the number of samples - and n_features is the number of features. - - y : Ignored - - Returns - ------- - self : object - Returns the instance itself. - """ - random_state = check_random_state(self.random_state) - X = self._validate_data(X) - - self.mean_ = X.mean(axis=0) - X = X - self.mean_ - - if self.n_components is None: - n_components = X.shape[1] - else: - n_components = self.n_components code_init = self.V_init.T if self.V_init is not None else None dict_init = self.U_init.T if self.U_init is not None else None - Vt, _, E, self.n_iter_ = dict_learning(X.T, n_components, - alpha=self.alpha, - tol=self.tol, - max_iter=self.max_iter, - method=self.method, - n_jobs=self.n_jobs, - verbose=self.verbose, - random_state=random_state, - code_init=code_init, - dict_init=dict_init, - return_n_iter=True) - self.components_ = Vt.T - components_norm = np.linalg.norm( - self.components_, axis=1)[:, np.newaxis] + code, dictionary, E, self.n_iter_ = dict_learning( + X.T, + n_components, + alpha=self.alpha, + tol=self.tol, + max_iter=self.max_iter, + method=self.method, + n_jobs=self.n_jobs, + verbose=self.verbose, + random_state=random_state, + code_init=code_init, + dict_init=dict_init, + return_n_iter=True, + ) + # flip eigenvectors' sign to enforce deterministic output + code, dictionary = svd_flip(code, dictionary, u_based_decision=True) + self.components_ = code.T + components_norm = np.linalg.norm(self.components_, axis=1)[:, np.newaxis] components_norm[components_norm == 0] = 1 self.components_ /= components_norm self.n_components_ = len(self.components_) @@ -176,51 +335,24 @@ def fit(self, X, y=None): self.error_ = E return self - def transform(self, X): - """Least Squares projection of the data onto the sparse components. - - To avoid instability issues in case the system is under-determined, - regularization can be applied (Ridge regression) via the - `ridge_alpha` parameter. - - Note that Sparse PCA components orthogonality is not enforced as in PCA - hence one cannot use a simple linear projection. - - Parameters - ---------- - X : ndarray of shape (n_samples, n_features) - Test data to be transformed, must have the same number of - features as the data used to train the model. - - Returns - ------- - X_new : ndarray of shape (n_samples, n_components) - Transformed data. - """ - check_is_fitted(self) - - X = self._validate_data(X, reset=False) - X = X - self.mean_ - U = ridge_regression(self.components_.T, X.T, self.ridge_alpha, - solver='cholesky') - - return U - - -class MiniBatchSparsePCA(SparsePCA): - """Mini-batch Sparse Principal Components Analysis +class MiniBatchSparsePCA(_BaseSparsePCA): + """Mini-batch Sparse Principal Components Analysis. Finds the set of sparse components that can optimally reconstruct the data. The amount of sparseness is controllable by the coefficient of the L1 penalty, given by the parameter alpha. + For an example comparing sparse PCA to PCA, see + :ref:`sphx_glr_auto_examples_decomposition_plot_faces_decomposition.py` + Read more in the :ref:`User Guide `. Parameters ---------- n_components : int, default=None - number of sparse atoms to extract + Number of sparse atoms to extract. If None, then ``n_components`` + is set to ``n_features``. alpha : int, default=1 Sparsity controlling parameter. Higher values lead to sparser @@ -230,20 +362,23 @@ class MiniBatchSparsePCA(SparsePCA): Amount of ridge shrinkage to apply in order to improve conditioning when calling the transform method. - n_iter : int, default=100 - number of iterations to perform for each mini batch + max_iter : int, default=1_000 + Maximum number of iterations over the complete dataset before + stopping independently of any early stopping criterion heuristics. + + .. versionadded:: 1.2 callback : callable, default=None - callable that gets invoked every five iterations + Callable that gets invoked every five iterations. batch_size : int, default=3 - the number of features to take in each mini batch + The number of features to take in each mini batch. verbose : int or bool, default=False Controls the verbosity; the higher, the more messages. Defaults to 0. shuffle : bool, default=True - whether to shuffle the data before splitting it in batches + Whether to shuffle the data before splitting it in batches. n_jobs : int, default=None Number of parallel jobs to run. @@ -252,6 +387,7 @@ class MiniBatchSparsePCA(SparsePCA): for more details. method : {'lars', 'cd'}, default='lars' + Method to be used for optimization. lars: uses the least angle regression method to solve the lasso problem (linear_model.lars_path) cd: uses the coordinate descent method to compute the @@ -264,6 +400,24 @@ class MiniBatchSparsePCA(SparsePCA): across multiple function calls. See :term:`Glossary `. + tol : float, default=1e-3 + Control early stopping based on the norm of the differences in the + dictionary between 2 steps. + + To disable early stopping based on changes in the dictionary, set + `tol` to 0.0. + + .. versionadded:: 1.1 + + max_no_improvement : int or None, default=10 + Control early stopping based on the consecutive number of mini batches + that does not yield an improvement on the smoothed cost function. + + To disable convergence detection based on cost function, set + `max_no_improvement` to `None`. + + .. versionadded:: 1.1 + Attributes ---------- components_ : ndarray of shape (n_components, n_features) @@ -281,6 +435,25 @@ class MiniBatchSparsePCA(SparsePCA): Per-feature empirical mean, estimated from the training set. Equal to ``X.mean(axis=0)``. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + DictionaryLearning : Find a dictionary that sparsely encodes data. + IncrementalPCA : Incremental principal components analysis. + PCA : Principal component analysis. + SparsePCA : Sparse Principal Components Analysis. + TruncatedSVD : Dimensionality reduction using truncated SVD. + Examples -------- >>> import numpy as np @@ -288,7 +461,7 @@ class MiniBatchSparsePCA(SparsePCA): >>> from sklearn.decomposition import MiniBatchSparsePCA >>> X, _ = make_friedman1(n_samples=200, n_features=30, random_state=0) >>> transformer = MiniBatchSparsePCA(n_components=5, batch_size=50, - ... random_state=0) + ... max_iter=10, random_state=0) >>> transformer.fit(X) MiniBatchSparsePCA(...) >>> X_transformed = transformer.transform(X) @@ -296,67 +469,78 @@ class MiniBatchSparsePCA(SparsePCA): (200, 5) >>> # most values in the components_ are zero (sparsity) >>> np.mean(transformer.components_ == 0) - 0.94 - - See Also - -------- - PCA - SparsePCA - DictionaryLearning + np.float64(0.9) """ - @_deprecate_positional_args - def __init__(self, n_components=None, *, alpha=1, ridge_alpha=0.01, - n_iter=100, callback=None, batch_size=3, verbose=False, - shuffle=True, n_jobs=None, method='lars', random_state=None): + + _parameter_constraints: dict = { + **_BaseSparsePCA._parameter_constraints, + "max_iter": [Interval(Integral, 0, None, closed="left")], + "callback": [None, callable], + "batch_size": [Interval(Integral, 1, None, closed="left")], + "shuffle": ["boolean"], + "max_no_improvement": [Interval(Integral, 0, None, closed="left"), None], + } + + def __init__( + self, + n_components=None, + *, + alpha=1, + ridge_alpha=0.01, + max_iter=1_000, + callback=None, + batch_size=3, + verbose=False, + shuffle=True, + n_jobs=None, + method="lars", + random_state=None, + tol=1e-3, + max_no_improvement=10, + ): super().__init__( - n_components=n_components, alpha=alpha, verbose=verbose, - ridge_alpha=ridge_alpha, n_jobs=n_jobs, method=method, - random_state=random_state) - self.n_iter = n_iter + n_components=n_components, + alpha=alpha, + ridge_alpha=ridge_alpha, + max_iter=max_iter, + tol=tol, + method=method, + n_jobs=n_jobs, + verbose=verbose, + random_state=random_state, + ) self.callback = callback self.batch_size = batch_size self.shuffle = shuffle + self.max_no_improvement = max_no_improvement - def fit(self, X, y=None): - """Fit the model from data in X. - - Parameters - ---------- - X : array-like of shape (n_samples, n_features) - Training vector, where n_samples in the number of samples - and n_features is the number of features. - - y : Ignored - - Returns - ------- - self : object - Returns the instance itself. - """ - random_state = check_random_state(self.random_state) - X = self._validate_data(X) - - self.mean_ = X.mean(axis=0) - X = X - self.mean_ + def _fit(self, X, n_components, random_state): + """Specialized `fit` for MiniBatchSparsePCA.""" - if self.n_components is None: - n_components = X.shape[1] - else: - n_components = self.n_components - Vt, _, self.n_iter_ = dict_learning_online( - X.T, n_components, alpha=self.alpha, - n_iter=self.n_iter, return_code=True, - dict_init=None, verbose=self.verbose, - callback=self.callback, + transform_algorithm = "lasso_" + self.method + est = MiniBatchDictionaryLearning( + n_components=n_components, + alpha=self.alpha, + max_iter=self.max_iter, + dict_init=None, batch_size=self.batch_size, shuffle=self.shuffle, - n_jobs=self.n_jobs, method=self.method, + n_jobs=self.n_jobs, + fit_algorithm=self.method, random_state=random_state, - return_n_iter=True) - self.components_ = Vt.T + transform_algorithm=transform_algorithm, + transform_alpha=self.alpha, + verbose=self.verbose, + callback=self.callback, + tol=self.tol, + max_no_improvement=self.max_no_improvement, + ) + est.set_output(transform="default") + est.fit(X.T) + + self.components_, self.n_iter_ = est.transform(X.T).T, est.n_iter_ - components_norm = np.linalg.norm( - self.components_, axis=1)[:, np.newaxis] + components_norm = np.linalg.norm(self.components_, axis=1)[:, np.newaxis] components_norm[components_norm == 0] = 1 self.components_ /= components_norm self.n_components_ = len(self.components_) diff --git a/sklearn/decomposition/_truncated_svd.py b/sklearn/decomposition/_truncated_svd.py index 1ea6b15c3ebd7..6165aba4e8db6 100644 --- a/sklearn/decomposition/_truncated_svd.py +++ b/sklearn/decomposition/_truncated_svd.py @@ -1,28 +1,31 @@ -"""Truncated SVD for sparse matrices, aka latent semantic analysis (LSA). -""" +"""Truncated SVD for sparse matrices, aka latent semantic analysis (LSA).""" -# Author: Lars Buitinck -# Olivier Grisel -# Michael Becker -# License: 3-clause BSD. +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from numbers import Integral, Real import numpy as np import scipy.sparse as sp from scipy.sparse.linalg import svds -from ..base import BaseEstimator, TransformerMixin +from ..base import ( + BaseEstimator, + ClassNamePrefixFeaturesOutMixin, + TransformerMixin, + _fit_context, +) from ..utils import check_array, check_random_state from ..utils._arpack import _init_arpack_v0 -from ..utils.extmath import randomized_svd, safe_sparse_dot, svd_flip +from ..utils._param_validation import Interval, StrOptions +from ..utils.extmath import _randomized_svd, safe_sparse_dot, svd_flip from ..utils.sparsefuncs import mean_variance_axis -from ..utils.validation import _deprecate_positional_args -from ..utils.validation import check_is_fitted - +from ..utils.validation import check_is_fitted, validate_data __all__ = ["TruncatedSVD"] -class TruncatedSVD(TransformerMixin, BaseEstimator): +class TruncatedSVD(ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator): """Dimensionality reduction using truncated SVD (aka LSA). This transformer performs linear dimensionality reduction by means of @@ -45,7 +48,8 @@ class TruncatedSVD(TransformerMixin, BaseEstimator): ---------- n_components : int, default=2 Desired dimensionality of output data. - Must be strictly less than the number of features. + If algorithm='arpack', must be strictly less than the number of features. + If algorithm='randomized', must be less than or equal to the number of features. The default value is useful for visualisation. For LSA, a value of 100 is recommended. @@ -60,18 +64,33 @@ class TruncatedSVD(TransformerMixin, BaseEstimator): :func:`~sklearn.utils.extmath.randomized_svd` to handle sparse matrices that may have large slowly decaying spectrum. + n_oversamples : int, default=10 + Number of oversamples for randomized SVD solver. Not used by ARPACK. + See :func:`~sklearn.utils.extmath.randomized_svd` for a complete + description. + + .. versionadded:: 1.1 + + power_iteration_normalizer : {'auto', 'QR', 'LU', 'none'}, default='auto' + Power iteration normalizer for randomized SVD solver. + Not used by ARPACK. See :func:`~sklearn.utils.extmath.randomized_svd` + for more details. + + .. versionadded:: 1.1 + random_state : int, RandomState instance or None, default=None Used during randomized svd. Pass an int for reproducible results across multiple function calls. See :term:`Glossary `. - tol : float, default=0. + tol : float, default=0.0 Tolerance for ARPACK. 0 means machine precision. Ignored by randomized SVD solver. Attributes ---------- components_ : ndarray of shape (n_components, n_features) + The right singular vectors of the input data. explained_variance_ : ndarray of shape (n_components,) The variance of the training samples transformed by a projection to @@ -80,36 +99,31 @@ class TruncatedSVD(TransformerMixin, BaseEstimator): explained_variance_ratio_ : ndarray of shape (n_components,) Percentage of variance explained by each of the selected components. - singular_values_ : ndarray od shape (n_components,) + singular_values_ : ndarray of shape (n_components,) The singular values corresponding to each of the selected components. The singular values are equal to the 2-norms of the ``n_components`` variables in the lower-dimensional space. - Examples - -------- - >>> from sklearn.decomposition import TruncatedSVD - >>> from scipy.sparse import random as sparse_random - >>> X = sparse_random(100, 100, density=0.01, format='csr', - ... random_state=42) - >>> svd = TruncatedSVD(n_components=5, n_iter=7, random_state=42) - >>> svd.fit(X) - TruncatedSVD(n_components=5, n_iter=7, random_state=42) - >>> print(svd.explained_variance_ratio_) - [0.0646... 0.0633... 0.0639... 0.0535... 0.0406...] - >>> print(svd.explained_variance_ratio_.sum()) - 0.286... - >>> print(svd.singular_values_) - [1.553... 1.512... 1.510... 1.370... 1.199...] + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 See Also -------- - PCA - - References - ---------- - Finding structure with randomness: Stochastic algorithms for constructing - approximate matrix decompositions - Halko, et al., 2009 (arXiv:909) https://arxiv.org/pdf/0909.4061.pdf + DictionaryLearning : Find a dictionary that sparsely encodes data. + FactorAnalysis : A simple linear generative model with + Gaussian latent variables. + IncrementalPCA : Incremental principal components analysis. + KernelPCA : Kernel Principal component analysis. + NMF : Non-Negative Matrix Factorization. + PCA : Principal component analysis. Notes ----- @@ -118,13 +132,58 @@ class TruncatedSVD(TransformerMixin, BaseEstimator): algorithm and random state. To work around this, fit instances of this class to data once, then keep the instance around to do transformations. + References + ---------- + :arxiv:`Halko, et al. (2009). "Finding structure with randomness: + Stochastic algorithms for constructing approximate matrix decompositions" + <0909.4061>` + + Examples + -------- + >>> from sklearn.decomposition import TruncatedSVD + >>> from scipy.sparse import csr_matrix + >>> import numpy as np + >>> np.random.seed(0) + >>> X_dense = np.random.rand(100, 100) + >>> X_dense[:, 2 * np.arange(50)] = 0 + >>> X = csr_matrix(X_dense) + >>> svd = TruncatedSVD(n_components=5, n_iter=7, random_state=42) + >>> svd.fit(X) + TruncatedSVD(n_components=5, n_iter=7, random_state=42) + >>> print(svd.explained_variance_ratio_) + [0.0157 0.0512 0.0499 0.0479 0.0453] + >>> print(svd.explained_variance_ratio_.sum()) + 0.2102 + >>> print(svd.singular_values_) + [35.2410 4.5981 4.5420 4.4486 4.3288] """ - @_deprecate_positional_args - def __init__(self, n_components=2, *, algorithm="randomized", n_iter=5, - random_state=None, tol=0.): + + _parameter_constraints: dict = { + "n_components": [Interval(Integral, 1, None, closed="left")], + "algorithm": [StrOptions({"arpack", "randomized"})], + "n_iter": [Interval(Integral, 0, None, closed="left")], + "n_oversamples": [Interval(Integral, 1, None, closed="left")], + "power_iteration_normalizer": [StrOptions({"auto", "OR", "LU", "none"})], + "random_state": ["random_state"], + "tol": [Interval(Real, 0, None, closed="left")], + } + + def __init__( + self, + n_components=2, + *, + algorithm="randomized", + n_iter=5, + n_oversamples=10, + power_iteration_normalizer="auto", + random_state=None, + tol=0.0, + ): self.algorithm = algorithm self.n_components = n_components self.n_iter = n_iter + self.n_oversamples = n_oversamples + self.power_iteration_normalizer = power_iteration_normalizer self.random_state = random_state self.tol = tol @@ -137,6 +196,7 @@ def fit(self, X, y=None): Training data. y : Ignored + Not used, present here for API consistency by convention. Returns ------- @@ -146,6 +206,7 @@ def fit(self, X, y=None): self.fit_transform(X) return self + @_fit_context(prefer_skip_nested_validation=True) def fit_transform(self, X, y=None): """Fit model to X and perform dimensionality reduction on X. @@ -155,14 +216,14 @@ def fit_transform(self, X, y=None): Training data. y : Ignored + Not used, present here for API consistency by convention. Returns ------- X_new : ndarray of shape (n_samples, n_components) Reduced version of X. This will always be a dense array. """ - X = self._validate_data(X, accept_sparse=['csr', 'csc'], - ensure_min_features=2) + X = validate_data(self, X, accept_sparse=["csr", "csc"], ensure_min_features=2) random_state = check_random_state(self.random_state) if self.algorithm == "arpack": @@ -171,26 +232,33 @@ def fit_transform(self, X, y=None): # svds doesn't abide by scipy.linalg.svd/randomized_svd # conventions, so reverse its outputs. Sigma = Sigma[::-1] - U, VT = svd_flip(U[:, ::-1], VT[::-1]) + # u_based_decision=False is needed to be consistent with PCA. + U, VT = svd_flip(U[:, ::-1], VT[::-1], u_based_decision=False) elif self.algorithm == "randomized": - k = self.n_components - n_features = X.shape[1] - if k >= n_features: - raise ValueError("n_components must be < n_features;" - " got %d >= %d" % (k, n_features)) - U, Sigma, VT = randomized_svd(X, self.n_components, - n_iter=self.n_iter, - random_state=random_state) - else: - raise ValueError("unknown algorithm %r" % self.algorithm) + if self.n_components > X.shape[1]: + raise ValueError( + f"n_components({self.n_components}) must be <=" + f" n_features({X.shape[1]})." + ) + U, Sigma, VT = _randomized_svd( + X, + self.n_components, + n_iter=self.n_iter, + n_oversamples=self.n_oversamples, + power_iteration_normalizer=self.power_iteration_normalizer, + random_state=random_state, + flip_sign=False, + ) + U, VT = svd_flip(U, VT, u_based_decision=False) self.components_ = VT # As a result of the SVD approximation error on X ~ U @ Sigma @ V.T, # X @ V is not the same as U @ Sigma - if self.algorithm == "randomized" or \ - (self.algorithm == "arpack" and self.tol > 0): + if self.algorithm == "randomized" or ( + self.algorithm == "arpack" and self.tol > 0 + ): X_transformed = safe_sparse_dot(X, self.components_.T) else: X_transformed = U * Sigma @@ -221,7 +289,7 @@ def transform(self, X): Reduced version of X. This will always be a dense array. """ check_is_fitted(self) - X = self._validate_data(X, accept_sparse=['csr', 'csc'], reset=False) + X = validate_data(self, X, accept_sparse=["csr", "csc"], reset=False) return safe_sparse_dot(X, self.components_.T) def inverse_transform(self, X): @@ -242,5 +310,13 @@ def inverse_transform(self, X): X = check_array(X) return np.dot(X, self.components_) - def _more_tags(self): - return {'preserves_dtype': [np.float64, np.float32]} + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.sparse = True + tags.transformer_tags.preserves_dtype = ["float64", "float32"] + return tags + + @property + def _n_features_out(self): + """Number of transformed output features.""" + return self.components_.shape[0] diff --git a/sklearn/decomposition/meson.build b/sklearn/decomposition/meson.build new file mode 100644 index 0000000000000..75b67a46981f4 --- /dev/null +++ b/sklearn/decomposition/meson.build @@ -0,0 +1,14 @@ +py.extension_module( + '_online_lda_fast', + [cython_gen.process('_online_lda_fast.pyx'), utils_cython_tree], + subdir: 'sklearn/decomposition', + install: true +) + +py.extension_module( + '_cdnmf_fast', + cython_gen.process('_cdnmf_fast.pyx'), + dependencies: [np_dep], + subdir: 'sklearn/decomposition', + install: true +) diff --git a/sklearn/decomposition/setup.py b/sklearn/decomposition/setup.py deleted file mode 100644 index f915d6d78fda1..0000000000000 --- a/sklearn/decomposition/setup.py +++ /dev/null @@ -1,29 +0,0 @@ -import os -import numpy -from numpy.distutils.misc_util import Configuration - - -def configuration(parent_package="", top_path=None): - config = Configuration("decomposition", parent_package, top_path) - - libraries = [] - if os.name == 'posix': - libraries.append('m') - - config.add_extension("_online_lda_fast", - sources=["_online_lda_fast.pyx"], - include_dirs=[numpy.get_include()], - libraries=libraries) - - config.add_extension('_cdnmf_fast', - sources=['_cdnmf_fast.pyx'], - include_dirs=[numpy.get_include()], - libraries=libraries) - - config.add_subpackage("tests") - - return config - -if __name__ == "__main__": - from numpy.distutils.core import setup - setup(**configuration().todict()) diff --git a/sklearn/decomposition/tests/test_dict_learning.py b/sklearn/decomposition/tests/test_dict_learning.py index a13c07a6ac728..717c56d0abdbe 100644 --- a/sklearn/decomposition/tests/test_dict_learning.py +++ b/sklearn/decomposition/tests/test_dict_learning.py @@ -1,30 +1,36 @@ -import pytest +import itertools +import warnings +from functools import partial import numpy as np -from functools import partial -import itertools +import pytest +import sklearn from sklearn.base import clone - +from sklearn.decomposition import ( + DictionaryLearning, + MiniBatchDictionaryLearning, + SparseCoder, + dict_learning, + dict_learning_online, + sparse_encode, +) +from sklearn.decomposition._dict_learning import _update_dict from sklearn.exceptions import ConvergenceWarning - from sklearn.utils import check_array - -from sklearn.utils._testing import assert_array_almost_equal -from sklearn.utils._testing import assert_array_equal -from sklearn.utils._testing import ignore_warnings -from sklearn.utils._testing import TempMemmap - -from sklearn.decomposition import DictionaryLearning -from sklearn.decomposition import MiniBatchDictionaryLearning -from sklearn.decomposition import SparseCoder -from sklearn.decomposition import dict_learning -from sklearn.decomposition import dict_learning_online -from sklearn.decomposition import sparse_encode -from sklearn.utils.estimator_checks import check_transformer_data_not_an_array -from sklearn.utils.estimator_checks import check_transformer_general -from sklearn.utils.estimator_checks import check_transformers_unfitted - +from sklearn.utils._testing import ( + TempMemmap, + assert_allclose, + assert_array_almost_equal, + assert_array_equal, + ignore_warnings, +) +from sklearn.utils.estimator_checks import ( + check_transformer_data_not_an_array, + check_transformer_general, + check_transformers_unfitted, +) +from sklearn.utils.parallel import Parallel rng_global = np.random.RandomState(0) n_samples, n_features = 10, 8 @@ -33,13 +39,12 @@ def test_sparse_encode_shapes_omp(): rng = np.random.RandomState(0) - algorithms = ['omp', 'lasso_lars', 'lasso_cd', 'lars', 'threshold'] + algorithms = ["omp", "lasso_lars", "lasso_cd", "lars", "threshold"] for n_components, n_samples in itertools.product([1, 5], [1, 9]): X_ = rng.randn(n_samples, n_features) dictionary = rng.randn(n_components, n_features) - for algorithm, n_jobs in itertools.product(algorithms, [1, 3]): - code = sparse_encode(X_, dictionary, algorithm=algorithm, - n_jobs=n_jobs) + for algorithm, n_jobs in itertools.product(algorithms, [1, 2]): + code = sparse_encode(X_, dictionary, algorithm=algorithm, n_jobs=n_jobs) assert code.shape == (n_samples, n_components) @@ -64,9 +69,11 @@ def test_max_iter(): def ricker_function(resolution, center, width): """Discrete sub-sampled Ricker (Mexican hat) wavelet""" x = np.linspace(0, resolution - 1, resolution) - x = ((2 / (np.sqrt(3 * width) * np.pi ** .25)) - * (1 - (x - center) ** 2 / width ** 2) - * np.exp(-(x - center) ** 2 / (2 * width ** 2))) + x = ( + (2 / (np.sqrt(3 * width) * np.pi**0.25)) + * (1 - (x - center) ** 2 / width**2) + * np.exp(-((x - center) ** 2) / (2 * width**2)) + ) return x def ricker_matrix(width, resolution, n_components): @@ -75,37 +82,44 @@ def ricker_matrix(width, resolution, n_components): D = np.empty((n_components, resolution)) for i, center in enumerate(centers): D[i] = ricker_function(resolution, center, width) - D /= np.sqrt(np.sum(D ** 2, axis=1))[:, np.newaxis] + D /= np.sqrt(np.sum(D**2, axis=1))[:, np.newaxis] return D - transform_algorithm = 'lasso_cd' + transform_algorithm = "lasso_cd" resolution = 1024 subsampling = 3 # subsampling factor n_components = resolution // subsampling # Compute a wavelet dictionary - D_multi = np.r_[tuple(ricker_matrix(width=w, resolution=resolution, - n_components=n_components // 5) - for w in (10, 50, 100, 500, 1000))] + D_multi = np.r_[ + tuple( + ricker_matrix( + width=w, resolution=resolution, n_components=n_components // 5 + ) + for w in (10, 50, 100, 500, 1000) + ) + ] X = np.linspace(0, resolution - 1, resolution) first_quarter = X < resolution / 4 - X[first_quarter] = 3. - X[np.logical_not(first_quarter)] = -1. + X[first_quarter] = 3.0 + X[np.logical_not(first_quarter)] = -1.0 X = X.reshape(1, -1) # check that the underlying model fails to converge with pytest.warns(ConvergenceWarning): - model = SparseCoder(D_multi, transform_algorithm=transform_algorithm, - transform_max_iter=1) + model = SparseCoder( + D_multi, transform_algorithm=transform_algorithm, transform_max_iter=1 + ) model.fit_transform(X) # check that the underlying model converges w/o warnings - with pytest.warns(None) as record: - model = SparseCoder(D_multi, transform_algorithm=transform_algorithm, - transform_max_iter=2000) + with warnings.catch_warnings(): + warnings.simplefilter("error", ConvergenceWarning) + model = SparseCoder( + D_multi, transform_algorithm=transform_algorithm, transform_max_iter=2000 + ) model.fit_transform(X) - assert not record.list def test_dict_learning_lars_positive_parameter(): @@ -116,21 +130,26 @@ def test_dict_learning_lars_positive_parameter(): dict_learning(X, n_components, alpha=alpha, positive_code=True) -@pytest.mark.parametrize("transform_algorithm", [ - "lasso_lars", - "lasso_cd", - "threshold", -]) +@pytest.mark.parametrize( + "transform_algorithm", + [ + "lasso_lars", + "lasso_cd", + "threshold", + ], +) @pytest.mark.parametrize("positive_code", [False, True]) @pytest.mark.parametrize("positive_dict", [False, True]) -def test_dict_learning_positivity(transform_algorithm, - positive_code, - positive_dict): +def test_dict_learning_positivity(transform_algorithm, positive_code, positive_dict): n_components = 5 dico = DictionaryLearning( - n_components, transform_algorithm=transform_algorithm, random_state=0, - positive_code=positive_code, positive_dict=positive_dict, - fit_algorithm="cd").fit(X) + n_components, + transform_algorithm=transform_algorithm, + random_state=0, + positive_code=positive_code, + positive_dict=positive_dict, + fit_algorithm="cd", + ).fit(X) code = dico.transform(X) if positive_dict: @@ -147,8 +166,12 @@ def test_dict_learning_positivity(transform_algorithm, def test_dict_learning_lars_dict_positivity(positive_dict): n_components = 5 dico = DictionaryLearning( - n_components, transform_algorithm="lars", random_state=0, - positive_dict=positive_dict, fit_algorithm="cd").fit(X) + n_components, + transform_algorithm="lars", + random_state=0, + positive_dict=positive_dict, + fit_algorithm="cd", + ).fit(X) if positive_dict: assert (dico.components_ >= 0).all() @@ -159,8 +182,12 @@ def test_dict_learning_lars_dict_positivity(positive_dict): def test_dict_learning_lars_code_positivity(): n_components = 5 dico = DictionaryLearning( - n_components, transform_algorithm="lars", random_state=0, - positive_code=True, fit_algorithm="cd").fit(X) + n_components, + transform_algorithm="lars", + random_state=0, + positive_code=True, + fit_algorithm="cd", + ).fit(X) err_msg = "Positive constraint not supported for '{}' coding method." err_msg = err_msg.format("lars") @@ -170,14 +197,21 @@ def test_dict_learning_lars_code_positivity(): def test_dict_learning_reconstruction(): n_components = 12 - dico = DictionaryLearning(n_components, transform_algorithm='omp', - transform_alpha=0.001, random_state=0) + dico = DictionaryLearning( + n_components, transform_algorithm="omp", transform_alpha=0.001, random_state=0 + ) code = dico.fit(X).transform(X) assert_array_almost_equal(np.dot(code, dico.components_), X) + assert_array_almost_equal(dico.inverse_transform(code), X) - dico.set_params(transform_algorithm='lasso_lars') + dico.set_params(transform_algorithm="lasso_lars") code = dico.transform(X) assert_array_almost_equal(np.dot(code, dico.components_), X, decimal=2) + assert_array_almost_equal(dico.inverse_transform(code), X, decimal=2) + + # test error raised for wrong code size + with pytest.raises(ValueError, match="Expected 12, got 11."): + dico.inverse_transform(code[:, :-1]) # used to test lars here too, but there's no guarantee the number of # nonzero atoms is right. @@ -186,12 +220,17 @@ def test_dict_learning_reconstruction(): def test_dict_learning_reconstruction_parallel(): # regression test that parallel reconstruction works with n_jobs>1 n_components = 12 - dico = DictionaryLearning(n_components, transform_algorithm='omp', - transform_alpha=0.001, random_state=0, n_jobs=4) + dico = DictionaryLearning( + n_components, + transform_algorithm="omp", + transform_alpha=0.001, + random_state=0, + n_jobs=4, + ) code = dico.fit(X).transform(X) assert_array_almost_equal(np.dot(code, dico.components_), X) - dico.set_params(transform_algorithm='lasso_lars') + dico.set_params(transform_algorithm="lasso_lars") code = dico.transform(X) assert_array_almost_equal(np.dot(code, dico.components_), X, decimal=2) @@ -199,78 +238,114 @@ def test_dict_learning_reconstruction_parallel(): def test_dict_learning_lassocd_readonly_data(): n_components = 12 with TempMemmap(X) as X_read_only: - dico = DictionaryLearning(n_components, transform_algorithm='lasso_cd', - transform_alpha=0.001, random_state=0, - n_jobs=4) + dico = DictionaryLearning( + n_components, + transform_algorithm="lasso_cd", + transform_alpha=0.001, + random_state=0, + n_jobs=4, + ) with ignore_warnings(category=ConvergenceWarning): code = dico.fit(X_read_only).transform(X_read_only) - assert_array_almost_equal(np.dot(code, dico.components_), X_read_only, - decimal=2) + assert_array_almost_equal( + np.dot(code, dico.components_), X_read_only, decimal=2 + ) def test_dict_learning_nonzero_coefs(): n_components = 4 - dico = DictionaryLearning(n_components, transform_algorithm='lars', - transform_n_nonzero_coefs=3, random_state=0) + dico = DictionaryLearning( + n_components, + transform_algorithm="lars", + transform_n_nonzero_coefs=3, + random_state=0, + ) code = dico.fit(X).transform(X[np.newaxis, 1]) assert len(np.flatnonzero(code)) == 3 - dico.set_params(transform_algorithm='omp') + dico.set_params(transform_algorithm="omp") code = dico.transform(X[np.newaxis, 1]) assert len(np.flatnonzero(code)) == 3 -def test_dict_learning_unknown_fit_algorithm(): - n_components = 5 - dico = DictionaryLearning(n_components, fit_algorithm='') - with pytest.raises(ValueError): - dico.fit(X) - - def test_dict_learning_split(): n_components = 5 - dico = DictionaryLearning(n_components, transform_algorithm='threshold', - random_state=0) + dico = DictionaryLearning( + n_components, transform_algorithm="threshold", random_state=0 + ) code = dico.fit(X).transform(X) + Xr = dico.inverse_transform(code) + dico.split_sign = True split_code = dico.transform(X) - assert_array_almost_equal(split_code[:, :n_components] - - split_code[:, n_components:], code) + assert_array_almost_equal( + split_code[:, :n_components] - split_code[:, n_components:], code + ) + + Xr2 = dico.inverse_transform(split_code) + assert_array_almost_equal(Xr, Xr2) def test_dict_learning_online_shapes(): rng = np.random.RandomState(0) n_components = 8 - code, dictionary = dict_learning_online(X, n_components=n_components, - alpha=1, random_state=rng) + + code, dictionary = dict_learning_online( + X, + n_components=n_components, + batch_size=4, + max_iter=10, + method="cd", + random_state=rng, + return_code=True, + ) assert code.shape == (n_samples, n_components) assert dictionary.shape == (n_components, n_features) assert np.dot(code, dictionary).shape == X.shape + dictionary = dict_learning_online( + X, + n_components=n_components, + batch_size=4, + max_iter=10, + method="cd", + random_state=rng, + return_code=False, + ) + assert dictionary.shape == (n_components, n_features) + def test_dict_learning_online_lars_positive_parameter(): - alpha = 1 err_msg = "Positive constraint not supported for 'lars' coding method." with pytest.raises(ValueError, match=err_msg): - dict_learning_online(X, alpha=alpha, positive_code=True) + dict_learning_online(X, batch_size=4, max_iter=10, positive_code=True) -@pytest.mark.parametrize("transform_algorithm", [ - "lasso_lars", - "lasso_cd", - "threshold", -]) +@pytest.mark.parametrize( + "transform_algorithm", + [ + "lasso_lars", + "lasso_cd", + "threshold", + ], +) @pytest.mark.parametrize("positive_code", [False, True]) @pytest.mark.parametrize("positive_dict", [False, True]) -def test_minibatch_dictionary_learning_positivity(transform_algorithm, - positive_code, - positive_dict): +def test_minibatch_dictionary_learning_positivity( + transform_algorithm, positive_code, positive_dict +): n_components = 8 dico = MiniBatchDictionaryLearning( - n_components, transform_algorithm=transform_algorithm, random_state=0, - positive_code=positive_code, positive_dict=positive_dict, - fit_algorithm='cd').fit(X) + n_components, + batch_size=4, + max_iter=10, + transform_algorithm=transform_algorithm, + random_state=0, + positive_code=positive_code, + positive_dict=positive_dict, + fit_algorithm="cd", + ).fit(X) code = dico.transform(X) if positive_dict: @@ -288,8 +363,14 @@ def test_minibatch_dictionary_learning_lars(positive_dict): n_components = 8 dico = MiniBatchDictionaryLearning( - n_components, transform_algorithm="lars", random_state=0, - positive_dict=positive_dict, fit_algorithm='cd').fit(X) + n_components, + batch_size=4, + max_iter=10, + transform_algorithm="lars", + random_state=0, + positive_dict=positive_dict, + fit_algorithm="cd", + ).fit(X) if positive_dict: assert (dico.components_ >= 0).all() @@ -299,16 +380,20 @@ def test_minibatch_dictionary_learning_lars(positive_dict): @pytest.mark.parametrize("positive_code", [False, True]) @pytest.mark.parametrize("positive_dict", [False, True]) -def test_dict_learning_online_positivity(positive_code, - positive_dict): +def test_dict_learning_online_positivity(positive_code, positive_dict): rng = np.random.RandomState(0) n_components = 8 - code, dictionary = dict_learning_online(X, n_components=n_components, - method="cd", - alpha=1, random_state=rng, - positive_dict=positive_dict, - positive_code=positive_code) + code, dictionary = dict_learning_online( + X, + n_components=n_components, + batch_size=4, + method="cd", + alpha=1, + random_state=rng, + positive_dict=positive_dict, + positive_code=positive_code, + ) if positive_dict: assert (dictionary >= 0).all() else: @@ -320,24 +405,52 @@ def test_dict_learning_online_positivity(positive_code, def test_dict_learning_online_verbosity(): + # test verbosity for better coverage n_components = 5 - # test verbosity - from io import StringIO import sys + from io import StringIO old_stdout = sys.stdout try: sys.stdout = StringIO() - dico = MiniBatchDictionaryLearning(n_components, n_iter=20, verbose=1, - random_state=0) + + # convergence monitoring verbosity + dico = MiniBatchDictionaryLearning( + n_components, batch_size=4, max_iter=5, verbose=1, tol=0.1, random_state=0 + ) dico.fit(X) - dico = MiniBatchDictionaryLearning(n_components, n_iter=20, verbose=2, - random_state=0) + dico = MiniBatchDictionaryLearning( + n_components, + batch_size=4, + max_iter=5, + verbose=1, + max_no_improvement=2, + random_state=0, + ) dico.fit(X) - dict_learning_online(X, n_components=n_components, alpha=1, verbose=1, - random_state=0) - dict_learning_online(X, n_components=n_components, alpha=1, verbose=2, - random_state=0) + # higher verbosity level + dico = MiniBatchDictionaryLearning( + n_components, batch_size=4, max_iter=5, verbose=2, random_state=0 + ) + dico.fit(X) + + # function API verbosity + dict_learning_online( + X, + n_components=n_components, + batch_size=4, + alpha=1, + verbose=1, + random_state=0, + ) + dict_learning_online( + X, + n_components=n_components, + batch_size=4, + alpha=1, + verbose=2, + random_state=0, + ) finally: sys.stdout = old_stdout @@ -346,15 +459,18 @@ def test_dict_learning_online_verbosity(): def test_dict_learning_online_estimator_shapes(): n_components = 5 - dico = MiniBatchDictionaryLearning(n_components, n_iter=20, random_state=0) + dico = MiniBatchDictionaryLearning( + n_components, batch_size=4, max_iter=5, random_state=0 + ) dico.fit(X) assert dico.components_.shape == (n_components, n_features) def test_dict_learning_online_overcomplete(): n_components = 12 - dico = MiniBatchDictionaryLearning(n_components, n_iter=20, - random_state=0).fit(X) + dico = MiniBatchDictionaryLearning( + n_components, batch_size=4, max_iter=5, random_state=0 + ).fit(X) assert dico.components_.shape == (n_components, n_features) @@ -362,8 +478,9 @@ def test_dict_learning_online_initialization(): n_components = 12 rng = np.random.RandomState(0) V = rng.randn(n_components, n_features) - dico = MiniBatchDictionaryLearning(n_components, n_iter=0, - dict_init=V, random_state=0).fit(X) + dico = MiniBatchDictionaryLearning( + n_components, batch_size=4, max_iter=0, dict_init=V, random_state=0 + ).fit(X) assert_array_equal(dico.components_, V) @@ -372,69 +489,63 @@ def test_dict_learning_online_readonly_initialization(): rng = np.random.RandomState(0) V = rng.randn(n_components, n_features) V.setflags(write=False) - MiniBatchDictionaryLearning(n_components, n_iter=1, dict_init=V, - random_state=0, shuffle=False).fit(X) + MiniBatchDictionaryLearning( + n_components, + batch_size=4, + max_iter=1, + dict_init=V, + random_state=0, + shuffle=False, + ).fit(X) def test_dict_learning_online_partial_fit(): n_components = 12 rng = np.random.RandomState(0) V = rng.randn(n_components, n_features) # random init - V /= np.sum(V ** 2, axis=1)[:, np.newaxis] - dict1 = MiniBatchDictionaryLearning(n_components, n_iter=10 * len(X), - batch_size=1, - alpha=1, shuffle=False, dict_init=V, - random_state=0).fit(X) - dict2 = MiniBatchDictionaryLearning(n_components, alpha=1, - n_iter=1, dict_init=V, - random_state=0) + V /= np.sum(V**2, axis=1)[:, np.newaxis] + dict1 = MiniBatchDictionaryLearning( + n_components, + max_iter=10, + batch_size=1, + alpha=1, + shuffle=False, + dict_init=V, + max_no_improvement=None, + tol=0.0, + random_state=0, + ).fit(X) + dict2 = MiniBatchDictionaryLearning( + n_components, alpha=1, dict_init=V, random_state=0 + ) for i in range(10): for sample in X: dict2.partial_fit(sample[np.newaxis, :]) assert not np.all(sparse_encode(X, dict1.components_, alpha=1) == 0) - assert_array_almost_equal(dict1.components_, dict2.components_, - decimal=2) - - -def test_dict_learning_iter_offset(): - n_components = 12 - rng = np.random.RandomState(0) - V = rng.randn(n_components, n_features) - dict1 = MiniBatchDictionaryLearning(n_components, n_iter=10, - dict_init=V, random_state=0, - shuffle=False) - dict2 = MiniBatchDictionaryLearning(n_components, n_iter=10, - dict_init=V, random_state=0, - shuffle=False) - dict1.fit(X) - for sample in X: - dict2.partial_fit(sample[np.newaxis, :]) + assert_array_almost_equal(dict1.components_, dict2.components_, decimal=2) - assert dict1.iter_offset_ == dict2.iter_offset_ + # partial_fit should ignore max_iter (#17433) + assert dict1.n_steps_ == dict2.n_steps_ == 100 def test_sparse_encode_shapes(): n_components = 12 rng = np.random.RandomState(0) V = rng.randn(n_components, n_features) # random init - V /= np.sum(V ** 2, axis=1)[:, np.newaxis] - for algo in ('lasso_lars', 'lasso_cd', 'lars', 'omp', 'threshold'): + V /= np.sum(V**2, axis=1)[:, np.newaxis] + for algo in ("lasso_lars", "lasso_cd", "lars", "omp", "threshold"): code = sparse_encode(X, V, algorithm=algo) assert code.shape == (n_samples, n_components) -@pytest.mark.parametrize("algo", [ - 'lasso_lars', - 'lasso_cd', - 'threshold' -]) +@pytest.mark.parametrize("algo", ["lasso_lars", "lasso_cd", "threshold"]) @pytest.mark.parametrize("positive", [False, True]) def test_sparse_encode_positivity(algo, positive): n_components = 12 rng = np.random.RandomState(0) V = rng.randn(n_components, n_features) # random init - V /= np.sum(V ** 2, axis=1)[:, np.newaxis] + V /= np.sum(V**2, axis=1)[:, np.newaxis] code = sparse_encode(X, V, algorithm=algo, positive=positive) if positive: assert (code >= 0).all() @@ -442,12 +553,12 @@ def test_sparse_encode_positivity(algo, positive): assert (code < 0).any() -@pytest.mark.parametrize("algo", ['lars', 'omp']) +@pytest.mark.parametrize("algo", ["lars", "omp"]) def test_sparse_encode_unavailable_positivity(algo): n_components = 12 rng = np.random.RandomState(0) V = rng.randn(n_components, n_features) # random init - V /= np.sum(V ** 2, axis=1)[:, np.newaxis] + V /= np.sum(V**2, axis=1)[:, np.newaxis] err_msg = "Positive constraint not supported for '{}' coding method." err_msg = err_msg.format(algo) with pytest.raises(ValueError, match=err_msg): @@ -458,9 +569,9 @@ def test_sparse_encode_input(): n_components = 100 rng = np.random.RandomState(0) V = rng.randn(n_components, n_features) # random init - V /= np.sum(V ** 2, axis=1)[:, np.newaxis] - Xf = check_array(X, order='F') - for algo in ('lasso_lars', 'lasso_cd', 'lars', 'omp', 'threshold'): + V /= np.sum(V**2, axis=1)[:, np.newaxis] + Xf = check_array(X, order="F") + for algo in ("lasso_lars", "lasso_cd", "lars", "omp", "threshold"): a = sparse_encode(X, V, algorithm=algo) b = sparse_encode(Xf, V, algorithm=algo) assert_array_almost_equal(a, b) @@ -470,7 +581,7 @@ def test_sparse_encode_error(): n_components = 12 rng = np.random.RandomState(0) V = rng.randn(n_components, n_features) # random init - V /= np.sum(V ** 2, axis=1)[:, np.newaxis] + V /= np.sum(V**2, axis=1)[:, np.newaxis] code = sparse_encode(X, V, alpha=0.001) assert not np.all(code == 0) assert np.sqrt(np.sum((np.dot(code, V) - X) ** 2)) < 0.1 @@ -480,37 +591,33 @@ def test_sparse_encode_error_default_sparsity(): rng = np.random.RandomState(0) X = rng.randn(100, 64) D = rng.randn(2, 64) - code = ignore_warnings(sparse_encode)(X, D, algorithm='omp', - n_nonzero_coefs=None) + code = ignore_warnings(sparse_encode)(X, D, algorithm="omp", n_nonzero_coefs=None) assert code.shape == (100, 2) -def test_unknown_method(): - n_components = 12 - rng = np.random.RandomState(0) - V = rng.randn(n_components, n_features) # random init - with pytest.raises(ValueError): - sparse_encode(X, V, algorithm="") - - def test_sparse_coder_estimator(): n_components = 12 rng = np.random.RandomState(0) V = rng.randn(n_components, n_features) # random init - V /= np.sum(V ** 2, axis=1)[:, np.newaxis] - coder = SparseCoder(dictionary=V, transform_algorithm='lasso_lars', - transform_alpha=0.001).transform(X) - assert not np.all(coder == 0) - assert np.sqrt(np.sum((np.dot(coder, V) - X) ** 2)) < 0.1 + V /= np.sum(V**2, axis=1)[:, np.newaxis] + coder = SparseCoder( + dictionary=V, transform_algorithm="lasso_lars", transform_alpha=0.001 + ) + code = coder.fit_transform(X) + Xr = coder.inverse_transform(code) + assert not np.all(code == 0) + assert np.sqrt(np.sum((np.dot(code, V) - X) ** 2)) < 0.1 + np.testing.assert_allclose(Xr, np.dot(code, V)) def test_sparse_coder_estimator_clone(): n_components = 12 rng = np.random.RandomState(0) V = rng.randn(n_components, n_features) # random init - V /= np.sum(V ** 2, axis=1)[:, np.newaxis] - coder = SparseCoder(dictionary=V, transform_algorithm='lasso_lars', - transform_alpha=0.001) + V /= np.sum(V**2, axis=1)[:, np.newaxis] + coder = SparseCoder( + dictionary=V, transform_algorithm="lasso_lars", transform_alpha=0.001 + ) cloned = clone(coder) assert id(cloned) != id(coder) np.testing.assert_allclose(cloned.dictionary, coder.dictionary) @@ -518,8 +625,7 @@ def test_sparse_coder_estimator_clone(): assert cloned.n_components_ == coder.n_components_ assert cloned.n_features_in_ == coder.n_features_in_ data = np.random.rand(n_samples, n_features).astype(np.float32) - np.testing.assert_allclose(cloned.transform(data), - coder.transform(data)) + np.testing.assert_allclose(cloned.transform(data), coder.transform(data)) def test_sparse_coder_parallel_mmap(): @@ -537,7 +643,7 @@ def test_sparse_coder_parallel_mmap(): n_samples = int(2e6) // (4 * n_features) data = np.random.rand(n_samples, n_features).astype(np.float32) - sc = SparseCoder(init_dict, transform_algorithm='omp', n_jobs=2) + sc = SparseCoder(init_dict, transform_algorithm="omp", n_jobs=2) sc.fit_transform(data) @@ -557,27 +663,326 @@ def test_sparse_coder_common_transformer(): check_transformers_unfitted(sc.__class__.__name__, sc) -# TODO: remove in 1.1 -def test_sparse_coder_deprecation(): - # check that we raise a deprecation warning when accessing `components_` - rng = np.random.RandomState(777) - n_components, n_features = 40, 64 - init_dict = rng.rand(n_components, n_features) - sc = SparseCoder(init_dict) - - with pytest.warns(FutureWarning, match="'components_' is deprecated"): - sc.components_ - - def test_sparse_coder_n_features_in(): d = np.array([[1, 2, 3], [1, 2, 3]]) sc = SparseCoder(d) assert sc.n_features_in_ == d.shape[1] -@pytest.mark.parametrize("Estimator", [DictionaryLearning, - MiniBatchDictionaryLearning]) -def test_warning_default_transform_alpha(Estimator): - dl = Estimator(alpha=0.1) - with pytest.warns(FutureWarning, match="default transform_alpha"): - dl.fit_transform(X) +def test_update_dict(): + # Check the dict update in batch mode vs online mode + # Non-regression test for #4866 + rng = np.random.RandomState(0) + + code = np.array([[0.5, -0.5], [0.1, 0.9]]) + dictionary = np.array([[1.0, 0.0], [0.6, 0.8]]) + + X = np.dot(code, dictionary) + rng.randn(2, 2) + + # full batch update + newd_batch = dictionary.copy() + _update_dict(newd_batch, X, code) + + # online update + A = np.dot(code.T, code) + B = np.dot(X.T, code) + newd_online = dictionary.copy() + _update_dict(newd_online, X, code, A, B) + + assert_allclose(newd_batch, newd_online) + + +@pytest.mark.parametrize( + "algorithm", ("lasso_lars", "lasso_cd", "lars", "threshold", "omp") +) +@pytest.mark.parametrize("data_type", (np.float32, np.float64)) +# Note: do not check integer input because `lasso_lars` and `lars` fail with +# `ValueError` in `_lars_path_solver` +def test_sparse_encode_dtype_match(data_type, algorithm): + n_components = 6 + rng = np.random.RandomState(0) + dictionary = rng.randn(n_components, n_features) + code = sparse_encode( + X.astype(data_type), dictionary.astype(data_type), algorithm=algorithm + ) + assert code.dtype == data_type + + +@pytest.mark.parametrize( + "algorithm", ("lasso_lars", "lasso_cd", "lars", "threshold", "omp") +) +def test_sparse_encode_numerical_consistency(algorithm): + # verify numerical consistency among np.float32 and np.float64 + rtol = 1e-4 + n_components = 6 + rng = np.random.RandomState(0) + dictionary = rng.randn(n_components, n_features) + code_32 = sparse_encode( + X.astype(np.float32), dictionary.astype(np.float32), algorithm=algorithm + ) + code_64 = sparse_encode( + X.astype(np.float64), dictionary.astype(np.float64), algorithm=algorithm + ) + assert_allclose(code_32, code_64, rtol=rtol) + + +@pytest.mark.parametrize( + "transform_algorithm", ("lasso_lars", "lasso_cd", "lars", "threshold", "omp") +) +@pytest.mark.parametrize("data_type", (np.float32, np.float64)) +# Note: do not check integer input because `lasso_lars` and `lars` fail with +# `ValueError` in `_lars_path_solver` +def test_sparse_coder_dtype_match(data_type, transform_algorithm): + # Verify preserving dtype for transform in sparse coder + n_components = 6 + rng = np.random.RandomState(0) + dictionary = rng.randn(n_components, n_features) + coder = SparseCoder( + dictionary.astype(data_type), transform_algorithm=transform_algorithm + ) + code = coder.transform(X.astype(data_type)) + assert code.dtype == data_type + + +@pytest.mark.parametrize("fit_algorithm", ("lars", "cd")) +@pytest.mark.parametrize( + "transform_algorithm", ("lasso_lars", "lasso_cd", "lars", "threshold", "omp") +) +@pytest.mark.parametrize( + "data_type, expected_type", + ( + (np.float32, np.float32), + (np.float64, np.float64), + (np.int32, np.float64), + (np.int64, np.float64), + ), +) +def test_dictionary_learning_dtype_match( + data_type, + expected_type, + fit_algorithm, + transform_algorithm, +): + # Verify preserving dtype for fit and transform in dictionary learning class + dict_learner = DictionaryLearning( + n_components=8, + fit_algorithm=fit_algorithm, + transform_algorithm=transform_algorithm, + random_state=0, + ) + dict_learner.fit(X.astype(data_type)) + assert dict_learner.components_.dtype == expected_type + assert dict_learner.transform(X.astype(data_type)).dtype == expected_type + + +@pytest.mark.parametrize("fit_algorithm", ("lars", "cd")) +@pytest.mark.parametrize( + "transform_algorithm", ("lasso_lars", "lasso_cd", "lars", "threshold", "omp") +) +@pytest.mark.parametrize( + "data_type, expected_type", + ( + (np.float32, np.float32), + (np.float64, np.float64), + (np.int32, np.float64), + (np.int64, np.float64), + ), +) +def test_minibatch_dictionary_learning_dtype_match( + data_type, + expected_type, + fit_algorithm, + transform_algorithm, +): + # Verify preserving dtype for fit and transform in minibatch dictionary learning + dict_learner = MiniBatchDictionaryLearning( + n_components=8, + batch_size=10, + fit_algorithm=fit_algorithm, + transform_algorithm=transform_algorithm, + max_iter=100, + tol=1e-1, + random_state=0, + ) + dict_learner.fit(X.astype(data_type)) + + assert dict_learner.components_.dtype == expected_type + assert dict_learner.transform(X.astype(data_type)).dtype == expected_type + assert dict_learner._A.dtype == expected_type + assert dict_learner._B.dtype == expected_type + + +@pytest.mark.parametrize("method", ("lars", "cd")) +@pytest.mark.parametrize( + "data_type, expected_type", + ( + (np.float32, np.float32), + (np.float64, np.float64), + (np.int32, np.float64), + (np.int64, np.float64), + ), +) +def test_dict_learning_dtype_match(data_type, expected_type, method): + # Verify output matrix dtype + rng = np.random.RandomState(0) + n_components = 8 + code, dictionary, _ = dict_learning( + X.astype(data_type), + n_components=n_components, + alpha=1, + random_state=rng, + method=method, + ) + assert code.dtype == expected_type + assert dictionary.dtype == expected_type + + +@pytest.mark.parametrize("method", ("lars", "cd")) +def test_dict_learning_numerical_consistency(method): + # verify numerically consistent among np.float32 and np.float64 + rtol = 1e-6 + n_components = 4 + alpha = 2 + + U_64, V_64, _ = dict_learning( + X.astype(np.float64), + n_components=n_components, + alpha=alpha, + random_state=0, + method=method, + ) + U_32, V_32, _ = dict_learning( + X.astype(np.float32), + n_components=n_components, + alpha=alpha, + random_state=0, + method=method, + ) + + # Optimal solution (U*, V*) is not unique. + # If (U*, V*) is optimal solution, (-U*,-V*) is also optimal, + # and (column permutated U*, row permutated V*) are also optional + # as long as holding UV. + # So here UV, ||U||_1,1 and sum(||V_k||_2^2) are verified + # instead of comparing directly U and V. + assert_allclose(np.matmul(U_64, V_64), np.matmul(U_32, V_32), rtol=rtol) + assert_allclose(np.sum(np.abs(U_64)), np.sum(np.abs(U_32)), rtol=rtol) + assert_allclose(np.sum(V_64**2), np.sum(V_32**2), rtol=rtol) + # verify an obtained solution is not degenerate + assert np.mean(U_64 != 0.0) > 0.05 + assert np.count_nonzero(U_64 != 0.0) == np.count_nonzero(U_32 != 0.0) + + +@pytest.mark.parametrize("method", ("lars", "cd")) +@pytest.mark.parametrize( + "data_type, expected_type", + ( + (np.float32, np.float32), + (np.float64, np.float64), + (np.int32, np.float64), + (np.int64, np.float64), + ), +) +def test_dict_learning_online_dtype_match(data_type, expected_type, method): + # Verify output matrix dtype + rng = np.random.RandomState(0) + n_components = 8 + code, dictionary = dict_learning_online( + X.astype(data_type), + n_components=n_components, + alpha=1, + batch_size=10, + random_state=rng, + method=method, + ) + assert code.dtype == expected_type + assert dictionary.dtype == expected_type + + +@pytest.mark.parametrize("method", ("lars", "cd")) +def test_dict_learning_online_numerical_consistency(method): + # verify numerically consistent among np.float32 and np.float64 + rtol = 1e-4 + n_components = 4 + alpha = 1 + + U_64, V_64 = dict_learning_online( + X.astype(np.float64), + n_components=n_components, + max_iter=1_000, + alpha=alpha, + batch_size=10, + random_state=0, + method=method, + tol=0.0, + max_no_improvement=None, + ) + U_32, V_32 = dict_learning_online( + X.astype(np.float32), + n_components=n_components, + max_iter=1_000, + alpha=alpha, + batch_size=10, + random_state=0, + method=method, + tol=0.0, + max_no_improvement=None, + ) + + # Optimal solution (U*, V*) is not unique. + # If (U*, V*) is optimal solution, (-U*,-V*) is also optimal, + # and (column permutated U*, row permutated V*) are also optional + # as long as holding UV. + # So here UV, ||U||_1,1 and sum(||V_k||_2) are verified + # instead of comparing directly U and V. + assert_allclose(np.matmul(U_64, V_64), np.matmul(U_32, V_32), rtol=rtol) + assert_allclose(np.sum(np.abs(U_64)), np.sum(np.abs(U_32)), rtol=rtol) + assert_allclose(np.sum(V_64**2), np.sum(V_32**2), rtol=rtol) + # verify an obtained solution is not degenerate + assert np.mean(U_64 != 0.0) > 0.05 + assert np.count_nonzero(U_64 != 0.0) == np.count_nonzero(U_32 != 0.0) + + +@pytest.mark.parametrize( + "estimator", + [ + SparseCoder(X.T), + DictionaryLearning(), + MiniBatchDictionaryLearning(batch_size=4, max_iter=10), + ], + ids=lambda x: x.__class__.__name__, +) +def test_get_feature_names_out(estimator): + """Check feature names for dict learning estimators.""" + estimator.fit(X) + n_components = X.shape[1] + + feature_names_out = estimator.get_feature_names_out() + estimator_name = estimator.__class__.__name__.lower() + assert_array_equal( + feature_names_out, + [f"{estimator_name}{i}" for i in range(n_components)], + ) + + +def test_cd_work_on_joblib_memmapped_data(monkeypatch): + monkeypatch.setattr( + sklearn.decomposition._dict_learning, + "Parallel", + partial(Parallel, max_nbytes=100), + ) + + rng = np.random.RandomState(0) + X_train = rng.randn(10, 10) + + dict_learner = DictionaryLearning( + n_components=5, + random_state=0, + n_jobs=2, + fit_algorithm="cd", + max_iter=50, + verbose=True, + ) + + # This must run and complete without error. + dict_learner.fit(X_train) diff --git a/sklearn/decomposition/tests/test_factor_analysis.py b/sklearn/decomposition/tests/test_factor_analysis.py index f889e49ea4a3a..9175829695b0d 100644 --- a/sklearn/decomposition/tests/test_factor_analysis.py +++ b/sklearn/decomposition/tests/test_factor_analysis.py @@ -1,27 +1,20 @@ -# Author: Christian Osendorfer -# Alexandre Gramfort -# License: BSD3 +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause from itertools import combinations import numpy as np import pytest -from sklearn.utils._testing import assert_warns -from sklearn.utils._testing import assert_raises -from sklearn.utils._testing import assert_almost_equal -from sklearn.utils._testing import assert_array_almost_equal -from sklearn.exceptions import ConvergenceWarning from sklearn.decomposition import FactorAnalysis -from sklearn.utils._testing import ignore_warnings from sklearn.decomposition._factor_analysis import _ortho_rotation +from sklearn.exceptions import ConvergenceWarning +from sklearn.utils._testing import assert_almost_equal, assert_array_almost_equal -# Ignore warnings from switching to more power iterations in randomized_svd -@ignore_warnings -def test_factor_analysis(): +def test_factor_analysis(global_random_seed): # Test FactorAnalysis ability to recover the data covariance structure - rng = np.random.RandomState(0) + rng = np.random.RandomState(global_random_seed) n_samples, n_features, n_components = 20, 5, 3 # Some random settings for the generative model @@ -36,14 +29,8 @@ def test_factor_analysis(): # wlog, mean is 0 X = np.dot(h, W) + noise - with pytest.raises(ValueError): - FactorAnalysis(svd_method='foo') - fa_fail = FactorAnalysis() - fa_fail.svd_method = 'foo' - with pytest.raises(ValueError): - fa_fail.fit(X) fas = [] - for method in ['randomized', 'lapack']: + for method in ["randomized", "lapack"]: fa = FactorAnalysis(n_components=n_components, svd_method=method) fa.fit(X) fas.append(fa) @@ -55,28 +42,32 @@ def test_factor_analysis(): assert_almost_equal(fa.score_samples(X).mean(), fa.score(X)) diff = np.all(np.diff(fa.loglike_)) - assert diff > 0., 'Log likelihood dif not increase' + assert diff > 0.0, "Log likelihood dif not increase" # Sample Covariance - scov = np.cov(X, rowvar=0., bias=1.) + scov = np.cov(X, rowvar=0.0, bias=1.0) # Model Covariance mcov = fa.get_covariance() diff = np.sum(np.abs(scov - mcov)) / W.size - assert diff < 0.1, "Mean absolute difference is %f" % diff - fa = FactorAnalysis(n_components=n_components, - noise_variance_init=np.ones(n_features)) + assert diff < 0.2, "Mean absolute difference is %f" % diff + fa = FactorAnalysis( + n_components=n_components, noise_variance_init=np.ones(n_features) + ) with pytest.raises(ValueError): fa.fit(X[:, :2]) - f = lambda x, y: np.abs(getattr(x, y)) # sign will not be equal + def f(x, y): + return np.abs(getattr(x, y)) # sign will not be equal + fa1, fa2 = fas - for attr in ['loglike_', 'components_', 'noise_variance_']: + for attr in ["loglike_", "components_", "noise_variance_"]: assert_almost_equal(f(fa1, attr), f(fa2, attr)) fa1.max_iter = 1 fa1.verbose = True - assert_warns(ConvergenceWarning, fa1.fit, X) + with pytest.warns(ConvergenceWarning): + fa1.fit(X) # Test get_covariance and get_precision with n_components == n_features # with n_components < n_features and with n_components == 0 @@ -85,35 +76,34 @@ def test_factor_analysis(): fa.fit(X) cov = fa.get_covariance() precision = fa.get_precision() - assert_array_almost_equal(np.dot(cov, precision), - np.eye(X.shape[1]), 12) + assert_array_almost_equal(np.dot(cov, precision), np.eye(X.shape[1]), 12) # test rotation n_components = 2 results, projections = {}, {} - for method in (None, "varimax", 'quartimax'): - fa_var = FactorAnalysis(n_components=n_components, - rotation=method) + for method in (None, "varimax", "quartimax"): + fa_var = FactorAnalysis(n_components=n_components, rotation=method) results[method] = fa_var.fit_transform(X) projections[method] = fa_var.get_covariance() - for rot1, rot2 in combinations([None, 'varimax', 'quartimax'], 2): + for rot1, rot2 in combinations([None, "varimax", "quartimax"], 2): assert not np.allclose(results[rot1], results[rot2]) assert np.allclose(projections[rot1], projections[rot2], atol=3) - assert_raises(ValueError, - FactorAnalysis(rotation='not_implemented').fit_transform, X) - # test against R's psych::principal with rotate="varimax" # (i.e., the values below stem from rotating the components in R) # R's factor analysis returns quite different values; therefore, we only # test the rotation itself factors = np.array( - [[0.89421016, -0.35854928, -0.27770122, 0.03773647], - [-0.45081822, -0.89132754, 0.0932195, -0.01787973], - [0.99500666, -0.02031465, 0.05426497, -0.11539407], - [0.96822861, -0.06299656, 0.24411001, 0.07540887]]) - r_solution = np.array([[0.962, 0.052], [-0.141, 0.989], - [0.949, -0.300], [0.937, -0.251]]) - rotated = _ortho_rotation(factors[:, :n_components], method='varimax').T + [ + [0.89421016, -0.35854928, -0.27770122, 0.03773647], + [-0.45081822, -0.89132754, 0.0932195, -0.01787973], + [0.99500666, -0.02031465, 0.05426497, -0.11539407], + [0.96822861, -0.06299656, 0.24411001, 0.07540887], + ] + ) + r_solution = np.array( + [[0.962, 0.052], [-0.141, 0.989], [0.949, -0.300], [0.937, -0.251]] + ) + rotated = _ortho_rotation(factors[:, :n_components], method="varimax").T assert_array_almost_equal(np.abs(rotated), np.abs(r_solution), decimal=3) diff --git a/sklearn/decomposition/tests/test_fastica.py b/sklearn/decomposition/tests/test_fastica.py index 9f37ac25c2f76..6f8c9c55db621 100644 --- a/sklearn/decomposition/tests/test_fastica.py +++ b/sklearn/decomposition/tests/test_fastica.py @@ -1,70 +1,103 @@ """ Test the fastica algorithm. """ + import itertools +import os import warnings -import pytest import numpy as np +import pytest from scipy import stats -from sklearn.utils._testing import assert_almost_equal -from sklearn.utils._testing import assert_array_almost_equal -from sklearn.utils._testing import assert_warns - -from sklearn.decomposition import FastICA, fastica, PCA +from sklearn.decomposition import PCA, FastICA, fastica from sklearn.decomposition._fastica import _gs_decorrelation from sklearn.exceptions import ConvergenceWarning +from sklearn.utils._testing import assert_allclose, ignore_warnings def center_and_norm(x, axis=-1): - """ Centers and norms x **in place** - - Parameters - ----------- - x: ndarray - Array with an axis of observations (statistical units) measured on - random variables. - axis: int, optional - Axis along which the mean and variance are calculated. + """Centers and norms x **in place** + + Parameters + ----------- + x: ndarray + Array with an axis of observations (statistical units) measured on + random variables. + axis: int, optional + Axis along which the mean and variance are calculated. """ x = np.rollaxis(x, axis) x -= x.mean(axis=0) x /= x.std(axis=0) -def test_gs(): +def test_gs(global_random_seed): # Test gram schmidt orthonormalization # generate a random orthogonal matrix - rng = np.random.RandomState(0) + rng = np.random.RandomState(global_random_seed) W, _, _ = np.linalg.svd(rng.randn(10, 10)) w = rng.randn(10) _gs_decorrelation(w, W, 10) - assert (w ** 2).sum() < 1.e-10 + assert (w**2).sum() < 1.0e-10 w = rng.randn(10) u = _gs_decorrelation(w, W, 5) tmp = np.dot(u, W.T) - assert (tmp[:5] ** 2).sum() < 1.e-10 + assert (tmp[:5] ** 2).sum() < 1.0e-10 + + +def test_fastica_attributes_dtypes(global_dtype): + rng = np.random.RandomState(0) + X = rng.random_sample((100, 10)).astype(global_dtype, copy=False) + fica = FastICA( + n_components=5, max_iter=1000, whiten="unit-variance", random_state=0 + ).fit(X) + assert fica.components_.dtype == global_dtype + assert fica.mixing_.dtype == global_dtype + assert fica.mean_.dtype == global_dtype + assert fica.whitening_.dtype == global_dtype + + +def test_fastica_return_dtypes(global_dtype): + rng = np.random.RandomState(0) + X = rng.random_sample((100, 10)).astype(global_dtype, copy=False) + k_, mixing_, s_ = fastica( + X, max_iter=1000, whiten="unit-variance", random_state=rng + ) + assert k_.dtype == global_dtype + assert mixing_.dtype == global_dtype + assert s_.dtype == global_dtype @pytest.mark.parametrize("add_noise", [True, False]) -@pytest.mark.parametrize("seed", range(1)) -def test_fastica_simple(add_noise, seed): +def test_fastica_simple(add_noise, global_random_seed, global_dtype): + if ( + global_random_seed == 20 + and global_dtype == np.float32 + and not add_noise + and os.getenv("DISTRIB") == "ubuntu" + ): + pytest.xfail( + "FastICA instability with Ubuntu Atlas build with float32 " + "global_dtype. For more details, see " + "https://github.com/scikit-learn/scikit-learn/issues/24131#issuecomment-1208091119" + ) + # Test the FastICA algorithm on very simple data. - rng = np.random.RandomState(seed) - # scipy.stats uses the global RNG: + rng = np.random.RandomState(global_random_seed) n_samples = 1000 # Generate two sources: s1 = (2 * np.sin(np.linspace(0, 100, n_samples)) > 0) - 1 - s2 = stats.t.rvs(1, size=n_samples) + s2 = stats.t.rvs(1, size=n_samples, random_state=global_random_seed) s = np.c_[s1, s2].T center_and_norm(s) + s = s.astype(global_dtype) s1, s2 = s # Mixing angle phi = 0.6 - mixing = np.array([[np.cos(phi), np.sin(phi)], - [np.sin(phi), -np.cos(phi)]]) + mixing = np.array([[np.cos(phi), np.sin(phi)], [np.sin(phi), -np.cos(phi)]]) + mixing = mixing.astype(global_dtype) m = np.dot(mixing, s) if add_noise: @@ -74,28 +107,36 @@ def test_fastica_simple(add_noise, seed): # function as fun arg def g_test(x): - return x ** 3, (3 * x ** 2).mean(axis=-1) + return x**3, (3 * x**2).mean(axis=-1) - algos = ['parallel', 'deflation'] - nls = ['logcosh', 'exp', 'cube', g_test] - whitening = [True, False] + algos = ["parallel", "deflation"] + nls = ["logcosh", "exp", "cube", g_test] + whitening = ["arbitrary-variance", "unit-variance", False] for algo, nl, whiten in itertools.product(algos, nls, whitening): if whiten: - k_, mixing_, s_ = fastica(m.T, fun=nl, algorithm=algo, - random_state=rng) + k_, mixing_, s_ = fastica( + m.T, fun=nl, whiten=whiten, algorithm=algo, random_state=rng + ) with pytest.raises(ValueError): - fastica(m.T, fun=np.tanh, algorithm=algo) + fastica(m.T, fun=np.tanh, whiten=whiten, algorithm=algo) else: pca = PCA(n_components=2, whiten=True, random_state=rng) X = pca.fit_transform(m.T) - k_, mixing_, s_ = fastica(X, fun=nl, algorithm=algo, whiten=False, - random_state=rng) + k_, mixing_, s_ = fastica( + X, fun=nl, algorithm=algo, whiten=False, random_state=rng + ) with pytest.raises(ValueError): fastica(X, fun=np.tanh, algorithm=algo) s_ = s_.T # Check that the mixing model described in the docstring holds: if whiten: - assert_almost_equal(s_, np.dot(np.dot(mixing_, k_), m)) + # XXX: exact reconstruction to standard relative tolerance is not + # possible. This is probably expected when add_noise is True but we + # also need a non-trivial atol in float32 when add_noise is False. + # + # Note that the 2 sources are non-Gaussian in this test. + atol = 1e-5 if global_dtype == np.float32 else 0 + assert_allclose(np.dot(np.dot(mixing_, k_), m), s_, atol=atol) center_and_norm(s_) s1_, s2_ = s_ @@ -108,32 +149,32 @@ def g_test(x): # Check that we have estimated the original sources if not add_noise: - assert_almost_equal(np.dot(s1_, s1) / n_samples, 1, decimal=2) - assert_almost_equal(np.dot(s2_, s2) / n_samples, 1, decimal=2) + assert_allclose(np.dot(s1_, s1) / n_samples, 1, atol=1e-2) + assert_allclose(np.dot(s2_, s2) / n_samples, 1, atol=1e-2) else: - assert_almost_equal(np.dot(s1_, s1) / n_samples, 1, decimal=1) - assert_almost_equal(np.dot(s2_, s2) / n_samples, 1, decimal=1) + assert_allclose(np.dot(s1_, s1) / n_samples, 1, atol=1e-1) + assert_allclose(np.dot(s2_, s2) / n_samples, 1, atol=1e-1) # Test FastICA class - _, _, sources_fun = fastica(m.T, fun=nl, algorithm=algo, - random_state=seed) - ica = FastICA(fun=nl, algorithm=algo, random_state=seed) + _, _, sources_fun = fastica( + m.T, fun=nl, algorithm=algo, random_state=global_random_seed + ) + ica = FastICA(fun=nl, algorithm=algo, random_state=global_random_seed) sources = ica.fit_transform(m.T) assert ica.components_.shape == (2, 2) assert sources.shape == (1000, 2) - assert_array_almost_equal(sources_fun, sources) - assert_array_almost_equal(sources, ica.transform(m.T)) + assert_allclose(sources_fun, sources) + # Set atol to account for the different magnitudes of the elements in sources + # (from 1e-4 to 1e1). + atol = np.max(np.abs(sources)) * (1e-5 if global_dtype == np.float32 else 1e-7) + assert_allclose(sources, ica.transform(m.T), atol=atol) assert ica.mixing_.shape == (2, 2) - for fn in [np.tanh, "exp(-.5(x^2))"]: - ica = FastICA(fun=fn, algorithm=algo) - with pytest.raises(ValueError): - ica.fit(m.T) - - with pytest.raises(TypeError): - FastICA(fun=range(10)).fit(m.T) + ica = FastICA(fun=np.tanh, algorithm=algo) + with pytest.raises(ValueError): + ica.fit(m.T) def test_fastica_nowhiten(): @@ -141,15 +182,17 @@ def test_fastica_nowhiten(): # test for issue #697 ica = FastICA(n_components=1, whiten=False, random_state=0) - assert_warns(UserWarning, ica.fit, m) - assert hasattr(ica, 'mixing_') + warn_msg = "Ignoring n_components with whiten=False." + with pytest.warns(UserWarning, match=warn_msg): + ica.fit(m) + assert hasattr(ica, "mixing_") -def test_fastica_convergence_fail(): +def test_fastica_convergence_fail(global_random_seed): # Test the FastICA algorithm on very simple data # (see test_non_square_fastica). # Ensure a ConvergenceWarning raised if the tolerance is sufficiently low. - rng = np.random.RandomState(0) + rng = np.random.RandomState(global_random_seed) n_samples = 1000 # Generate two sources: @@ -164,15 +207,21 @@ def test_fastica_convergence_fail(): m = np.dot(mixing, s) # Do fastICA with tolerance 0. to ensure failing convergence - ica = FastICA(algorithm="parallel", n_components=2, random_state=rng, - max_iter=2, tol=0.) - assert_warns(ConvergenceWarning, ica.fit, m.T) + warn_msg = ( + "FastICA did not converge. Consider increasing tolerance " + "or the maximum number of iterations." + ) + with pytest.warns(ConvergenceWarning, match=warn_msg): + ica = FastICA( + algorithm="parallel", n_components=2, random_state=rng, max_iter=2, tol=0.0 + ) + ica.fit(m.T) -@pytest.mark.parametrize('add_noise', [True, False]) -def test_non_square_fastica(add_noise): +@pytest.mark.parametrize("add_noise", [True, False]) +def test_non_square_fastica(global_random_seed, add_noise): # Test the FastICA algorithm on very simple data. - rng = np.random.RandomState(0) + rng = np.random.RandomState(global_random_seed) n_samples = 1000 # Generate two sources: @@ -192,11 +241,13 @@ def test_non_square_fastica(add_noise): center_and_norm(m) - k_, mixing_, s_ = fastica(m.T, n_components=2, random_state=rng) + k_, mixing_, s_ = fastica( + m.T, n_components=2, whiten="unit-variance", random_state=rng + ) s_ = s_.T # Check that the mixing model described in the docstring holds: - assert_almost_equal(s_, np.dot(np.dot(mixing_, k_), m)) + assert_allclose(s_, np.dot(np.dot(mixing_, k_), m)) center_and_norm(s_) s1_, s2_ = s_ @@ -209,59 +260,102 @@ def test_non_square_fastica(add_noise): # Check that we have estimated the original sources if not add_noise: - assert_almost_equal(np.dot(s1_, s1) / n_samples, 1, decimal=3) - assert_almost_equal(np.dot(s2_, s2) / n_samples, 1, decimal=3) + assert_allclose(np.dot(s1_, s1) / n_samples, 1, atol=1e-3) + assert_allclose(np.dot(s2_, s2) / n_samples, 1, atol=1e-3) -def test_fit_transform(): - # Test FastICA.fit_transform - rng = np.random.RandomState(0) - X = rng.random_sample((100, 10)) - for whiten, n_components in [[True, 5], [False, None]]: - n_components_ = (n_components if n_components is not None else - X.shape[1]) +def test_fit_transform(global_random_seed, global_dtype): + """Test unit variance of transformed data using FastICA algorithm. - ica = FastICA(n_components=n_components, whiten=whiten, random_state=0) - Xt = ica.fit_transform(X) - assert ica.components_.shape == (n_components_, 10) - assert Xt.shape == (100, n_components_) + Check that `fit_transform` gives the same result as applying + `fit` and then `transform`. - ica = FastICA(n_components=n_components, whiten=whiten, random_state=0) - ica.fit(X) + Bug #13056 + """ + # multivariate uniform data in [0, 1] + rng = np.random.RandomState(global_random_seed) + X = rng.random_sample((100, 10)).astype(global_dtype) + max_iter = 300 + for whiten, n_components in [["unit-variance", 5], [False, None]]: + n_components_ = n_components if n_components is not None else X.shape[1] + + ica = FastICA( + n_components=n_components, max_iter=max_iter, whiten=whiten, random_state=0 + ) + with warnings.catch_warnings(): + # make sure that numerical errors do not cause sqrt of negative + # values + warnings.simplefilter("error", RuntimeWarning) + # XXX: for some seeds, the model does not converge. + # However this is not what we test here. + warnings.simplefilter("ignore", ConvergenceWarning) + Xt = ica.fit_transform(X) assert ica.components_.shape == (n_components_, 10) - Xt2 = ica.transform(X) - - assert_array_almost_equal(Xt, Xt2) - - -def test_inverse_transform(): + assert Xt.shape == (X.shape[0], n_components_) + + ica2 = FastICA( + n_components=n_components, max_iter=max_iter, whiten=whiten, random_state=0 + ) + with warnings.catch_warnings(): + # make sure that numerical errors do not cause sqrt of negative + # values + warnings.simplefilter("error", RuntimeWarning) + warnings.simplefilter("ignore", ConvergenceWarning) + ica2.fit(X) + assert ica2.components_.shape == (n_components_, 10) + Xt2 = ica2.transform(X) + + # XXX: we have to set atol for this test to pass for all seeds when + # fitting with float32 data. Is this revealing a bug? + if global_dtype: + atol = np.abs(Xt2).mean() / 1e6 + else: + atol = 0.0 # the default rtol is enough for float64 data + assert_allclose(Xt, Xt2, atol=atol) + + +@pytest.mark.filterwarnings("ignore:Ignoring n_components with whiten=False.") +@pytest.mark.parametrize( + "whiten, n_components, expected_mixing_shape", + [ + ("arbitrary-variance", 5, (10, 5)), + ("arbitrary-variance", 10, (10, 10)), + ("unit-variance", 5, (10, 5)), + ("unit-variance", 10, (10, 10)), + (False, 5, (10, 10)), + (False, 10, (10, 10)), + ], +) +def test_inverse_transform( + whiten, n_components, expected_mixing_shape, global_random_seed, global_dtype +): # Test FastICA.inverse_transform - n_features = 10 n_samples = 100 - n1, n2 = 5, 10 - rng = np.random.RandomState(0) - X = rng.random_sample((n_samples, n_features)) - expected = {(True, n1): (n_features, n1), - (True, n2): (n_features, n2), - (False, n1): (n_features, n2), - (False, n2): (n_features, n2)} - for whiten in [True, False]: - for n_components in [n1, n2]: - n_components_ = (n_components if n_components is not None else - X.shape[1]) - ica = FastICA(n_components=n_components, random_state=rng, - whiten=whiten) - with warnings.catch_warnings(record=True): - # catch "n_components ignored" warning - Xt = ica.fit_transform(X) - expected_shape = expected[(whiten, n_components_)] - assert ica.mixing_.shape == expected_shape - X2 = ica.inverse_transform(Xt) - assert X.shape == X2.shape - - # reversibility test in non-reduction case - if n_components == X.shape[1]: - assert_array_almost_equal(X, X2) + rng = np.random.RandomState(global_random_seed) + X = rng.random_sample((n_samples, 10)).astype(global_dtype) + + ica = FastICA(n_components=n_components, random_state=rng, whiten=whiten) + with warnings.catch_warnings(): + # For some dataset (depending on the value of global_dtype) the model + # can fail to converge but this should not impact the definition of + # a valid inverse transform. + warnings.simplefilter("ignore", ConvergenceWarning) + Xt = ica.fit_transform(X) + assert ica.mixing_.shape == expected_mixing_shape + X2 = ica.inverse_transform(Xt) + assert X.shape == X2.shape + + # reversibility test in non-reduction case + if n_components == X.shape[1]: + # XXX: we have to set atol for this test to pass for all seeds when + # fitting with float32 data. Is this revealing a bug? + if global_dtype: + # XXX: dividing by a smaller number makes + # tests fail for some seeds. + atol = np.abs(X2).mean() / 1e5 + else: + atol = 0.0 # the default rtol is enough for float64 data + assert_allclose(X, X2, atol=atol) def test_fastica_errors(): @@ -270,21 +364,31 @@ def test_fastica_errors(): rng = np.random.RandomState(0) X = rng.random_sample((n_samples, n_features)) w_init = rng.randn(n_features + 1, n_features + 1) - with pytest.raises(ValueError, match='max_iter should be greater than 1'): - FastICA(max_iter=0) - with pytest.raises(ValueError, match=r'alpha must be in \[1,2\]'): - fastica(X, fun_args={'alpha': 0}) - with pytest.raises(ValueError, match='w_init has invalid shape.+' - r'should be \(3L?, 3L?\)'): + with pytest.raises(ValueError, match=r"alpha must be in \[1,2\]"): + fastica(X, fun_args={"alpha": 0}) + with pytest.raises( + ValueError, match=r"w_init has invalid shape.+should be \(3L?, 3L?\)" + ): fastica(X, w_init=w_init) - with pytest.raises(ValueError, match='Invalid algorithm.+must ' - 'be.+parallel.+or.+deflation'): - fastica(X, algorithm='pizza') -@pytest.mark.parametrize('whiten', [True, False]) -@pytest.mark.parametrize('return_X_mean', [True, False]) -@pytest.mark.parametrize('return_n_iter', [True, False]) +def test_fastica_whiten_unit_variance(global_random_seed): + """Test unit variance of transformed data using FastICA algorithm. + + Bug #13056 + """ + rng = np.random.RandomState(global_random_seed) + X = rng.random_sample((100, 10)) + n_components = X.shape[1] + ica = FastICA(n_components=n_components, whiten="unit-variance", random_state=0) + Xt = ica.fit_transform(X) + + assert np.var(Xt) == pytest.approx(1.0) + + +@pytest.mark.parametrize("whiten", ["arbitrary-variance", "unit-variance", False]) +@pytest.mark.parametrize("return_X_mean", [True, False]) +@pytest.mark.parametrize("return_n_iter", [True, False]) def test_fastica_output_shape(whiten, return_X_mean, return_n_iter): n_features = 3 n_samples = 10 @@ -293,9 +397,61 @@ def test_fastica_output_shape(whiten, return_X_mean, return_n_iter): expected_len = 3 + return_X_mean + return_n_iter - out = fastica(X, whiten=whiten, return_n_iter=return_n_iter, - return_X_mean=return_X_mean) + out = fastica( + X, whiten=whiten, return_n_iter=return_n_iter, return_X_mean=return_X_mean + ) assert len(out) == expected_len if not whiten: assert out[0] is None + + +@pytest.mark.parametrize("add_noise", [True, False]) +def test_fastica_simple_different_solvers(add_noise, global_random_seed): + """Test FastICA is consistent between whiten_solvers.""" + rng = np.random.RandomState(global_random_seed) + n_samples = 1000 + # Generate two sources: + s1 = (2 * np.sin(np.linspace(0, 100, n_samples)) > 0) - 1 + s2 = stats.t.rvs(1, size=n_samples, random_state=rng) + s = np.c_[s1, s2].T + center_and_norm(s) + s1, s2 = s + + # Mixing angle + phi = rng.rand() * 2 * np.pi + mixing = np.array([[np.cos(phi), np.sin(phi)], [np.sin(phi), -np.cos(phi)]]) + m = np.dot(mixing, s) + + if add_noise: + m += 0.1 * rng.randn(2, 1000) + + center_and_norm(m) + + outs = {} + for solver in ("svd", "eigh"): + ica = FastICA(random_state=0, whiten="unit-variance", whiten_solver=solver) + sources = ica.fit_transform(m.T) + outs[solver] = sources + assert ica.components_.shape == (2, 2) + assert sources.shape == (1000, 2) + + # compared numbers are not all on the same magnitude. Using a small atol to + # make the test less brittle + assert_allclose(outs["eigh"], outs["svd"], atol=1e-12) + + +def test_fastica_eigh_low_rank_warning(global_random_seed): + """Test FastICA eigh solver raises warning for low-rank data.""" + rng = np.random.RandomState(global_random_seed) + A = rng.randn(10, 2) + X = A @ A.T + ica = FastICA(random_state=0, whiten="unit-variance", whiten_solver="eigh") + msg = "There are some small singular values" + + with pytest.warns(UserWarning, match=msg): + with ignore_warnings(category=ConvergenceWarning): + # The FastICA solver may not converge for some data with specific + # random seeds but this happens after the whiten step so this is + # not want we want to test here. + ica.fit(X) diff --git a/sklearn/decomposition/tests/test_incremental_pca.py b/sklearn/decomposition/tests/test_incremental_pca.py index d198b67c720c1..c4ea1c222901c 100644 --- a/sklearn/decomposition/tests/test_incremental_pca.py +++ b/sklearn/decomposition/tests/test_incremental_pca.py @@ -1,15 +1,20 @@ """Tests for Incremental PCA.""" + +import itertools +import warnings + import numpy as np import pytest - -from sklearn.utils._testing import assert_almost_equal -from sklearn.utils._testing import assert_array_almost_equal -from sklearn.utils._testing import assert_allclose_dense_sparse +from numpy.testing import assert_allclose, assert_array_equal from sklearn import datasets from sklearn.decomposition import PCA, IncrementalPCA - -from scipy import sparse +from sklearn.utils._testing import ( + assert_allclose_dense_sparse, + assert_almost_equal, + assert_array_almost_equal, +) +from sklearn.utils.fixes import CSC_CONTAINERS, CSR_CONTAINERS, LIL_CONTAINERS iris = datasets.load_iris() @@ -25,57 +30,68 @@ def test_incremental_pca(): X_transformed = ipca.fit_transform(X) assert X_transformed.shape == (X.shape[0], 2) - np.testing.assert_allclose(ipca.explained_variance_ratio_.sum(), - pca.explained_variance_ratio_.sum(), rtol=1e-3) + np.testing.assert_allclose( + ipca.explained_variance_ratio_.sum(), + pca.explained_variance_ratio_.sum(), + rtol=1e-3, + ) for n_components in [1, 2, X.shape[1]]: ipca = IncrementalPCA(n_components, batch_size=batch_size) ipca.fit(X) cov = ipca.get_covariance() precision = ipca.get_precision() - np.testing.assert_allclose(np.dot(cov, precision), - np.eye(X.shape[1]), atol=1e-13) + np.testing.assert_allclose( + np.dot(cov, precision), np.eye(X.shape[1]), atol=1e-13 + ) @pytest.mark.parametrize( - "matrix_class", - [sparse.csc_matrix, sparse.csr_matrix, sparse.lil_matrix]) -def test_incremental_pca_sparse(matrix_class): + "sparse_container", CSC_CONTAINERS + CSR_CONTAINERS + LIL_CONTAINERS +) +def test_incremental_pca_sparse(sparse_container): # Incremental PCA on sparse arrays. X = iris.data pca = PCA(n_components=2) pca.fit_transform(X) - X_sparse = matrix_class(X) + X_sparse = sparse_container(X) batch_size = X_sparse.shape[0] // 3 ipca = IncrementalPCA(n_components=2, batch_size=batch_size) X_transformed = ipca.fit_transform(X_sparse) assert X_transformed.shape == (X_sparse.shape[0], 2) - np.testing.assert_allclose(ipca.explained_variance_ratio_.sum(), - pca.explained_variance_ratio_.sum(), rtol=1e-3) + np.testing.assert_allclose( + ipca.explained_variance_ratio_.sum(), + pca.explained_variance_ratio_.sum(), + rtol=1e-3, + ) for n_components in [1, 2, X.shape[1]]: ipca = IncrementalPCA(n_components, batch_size=batch_size) ipca.fit(X_sparse) cov = ipca.get_covariance() precision = ipca.get_precision() - np.testing.assert_allclose(np.dot(cov, precision), - np.eye(X_sparse.shape[1]), atol=1e-13) + np.testing.assert_allclose( + np.dot(cov, precision), np.eye(X_sparse.shape[1]), atol=1e-13 + ) with pytest.raises( - TypeError, - match="IncrementalPCA.partial_fit does not support " + TypeError, + match=( + "IncrementalPCA.partial_fit does not support " "sparse input. Either convert data to dense " - "or use IncrementalPCA.fit to do so in batches."): + "or use IncrementalPCA.fit to do so in batches." + ), + ): ipca.partial_fit(X_sparse) -def test_incremental_pca_check_projection(): +def test_incremental_pca_check_projection(global_random_seed): # Test that the projection of data is correct. - rng = np.random.RandomState(1999) + rng = np.random.RandomState(global_random_seed) n, p = 100, 3 - X = rng.randn(n, p) * .1 + X = rng.randn(n, p) * 0.1 X[:10] += np.array([3, 4, 5]) Xt = 0.1 * rng.randn(1, p) + np.array([3, 4, 5]) @@ -85,19 +101,19 @@ def test_incremental_pca_check_projection(): Yt = IncrementalPCA(n_components=2).fit(X).transform(Xt) # Normalize - Yt /= np.sqrt((Yt ** 2).sum()) + Yt /= np.sqrt((Yt**2).sum()) # Make sure that the first element of Yt is ~1, this means # the reconstruction worked as expected - assert_almost_equal(np.abs(Yt[0][0]), 1., 1) + assert_almost_equal(np.abs(Yt[0][0]), 1.0, 1) -def test_incremental_pca_inverse(): +def test_incremental_pca_inverse(global_random_seed): # Test that the projection of data can be inverted. - rng = np.random.RandomState(1999) + rng = np.random.RandomState(global_random_seed) n, p = 50, 3 X = rng.randn(n, p) # spherical data - X[:, 1] *= .00001 # make middle component relatively small + X[:, 1] *= 0.00001 # make middle component relatively small X += [5, 4, 3] # make a large mean # same check that we can find the original data from the transformed @@ -109,25 +125,45 @@ def test_incremental_pca_inverse(): def test_incremental_pca_validation(): - # Test that n_components is >=1 and <= n_features. + # Test that n_components is <= n_features. X = np.array([[0, 1, 0], [1, 0, 0]]) n_samples, n_features = X.shape - for n_components in [-1, 0, .99, 4]: - with pytest.raises(ValueError, match="n_components={} invalid" - " for n_features={}, need more rows than" - " columns for IncrementalPCA" - " processing".format(n_components, - n_features)): - IncrementalPCA(n_components, batch_size=10).fit(X) - - # Tests that n_components is also <= n_samples. + n_components = 4 + with pytest.raises( + ValueError, + match=( + "n_components={} invalid" + " for n_features={}, need more rows than" + " columns for IncrementalPCA" + " processing".format(n_components, n_features) + ), + ): + IncrementalPCA(n_components, batch_size=10).fit(X) + + # Test that n_components is also <= n_samples in first call to partial fit. n_components = 3 - with pytest.raises(ValueError, match="n_components={} must be" - " less or equal to the batch number of" - " samples {}".format(n_components, n_samples)): + with pytest.raises( + ValueError, + match=( + f"n_components={n_components} must be less or equal to the batch " + f"number of samples {n_samples} for the first partial_fit call." + ), + ): IncrementalPCA(n_components=n_components).partial_fit(X) +def test_n_samples_equal_n_components(): + # Ensures no warning is raised when n_samples==n_components + # Non-regression test for gh-19050 + ipca = IncrementalPCA(n_components=5) + with warnings.catch_warnings(): + warnings.simplefilter("error", RuntimeWarning) + ipca.partial_fit(np.random.randn(5, 7)) + with warnings.catch_warnings(): + warnings.simplefilter("error", RuntimeWarning) + ipca.fit(np.random.randn(5, 7)) + + def test_n_components_none(): # Ensures that n_components == None is handled correctly rng = np.random.RandomState(1999) @@ -181,9 +217,9 @@ def test_incremental_pca_num_features_change(): ipca.partial_fit(X2) -def test_incremental_pca_batch_signs(): +def test_incremental_pca_batch_signs(global_random_seed): # Test that components_ sign is stable over batch sizes. - rng = np.random.RandomState(1999) + rng = np.random.RandomState(global_random_seed) n_samples = 100 n_features = 3 X = rng.randn(n_samples, n_features) @@ -193,13 +229,34 @@ def test_incremental_pca_batch_signs(): ipca = IncrementalPCA(n_components=None, batch_size=batch_size).fit(X) all_components.append(ipca.components_) - for i, j in zip(all_components[:-1], all_components[1:]): + for i, j in itertools.pairwise(all_components): assert_almost_equal(np.sign(i), np.sign(j), decimal=6) -def test_incremental_pca_batch_values(): - # Test that components_ values are stable over batch sizes. +def test_incremental_pca_partial_fit_small_batch(): + # Test that there is no minimum batch size after the first partial_fit + # Non-regression test rng = np.random.RandomState(1999) + n, p = 50, 3 + X = rng.randn(n, p) # spherical data + X[:, 1] *= 0.00001 # make middle component relatively small + X += [5, 4, 3] # make a large mean + + n_components = p + pipca = IncrementalPCA(n_components=n_components) + pipca.partial_fit(X[:n_components]) + for idx in range(n_components, n): + pipca.partial_fit(X[idx : idx + 1]) + + pca = PCA(n_components=n_components) + pca.fit(X) + + assert_allclose(pca.components_, pipca.components_, atol=1e-3) + + +def test_incremental_pca_batch_values(global_random_seed): + # Test that components_ values are stable over batch sizes. + rng = np.random.RandomState(global_random_seed) n_samples = 100 n_features = 3 X = rng.randn(n_samples, n_features) @@ -209,7 +266,7 @@ def test_incremental_pca_batch_values(): ipca = IncrementalPCA(n_components=None, batch_size=batch_size).fit(X) all_components.append(ipca.components_) - for i, j in zip(all_components[:-1], all_components[1:]): + for i, j in itertools.pairwise(all_components): assert_almost_equal(i, j, decimal=1) @@ -225,17 +282,16 @@ def test_incremental_pca_batch_rank(): ipca = IncrementalPCA(n_components=20, batch_size=batch_size).fit(X) all_components.append(ipca.components_) - for components_i, components_j in zip(all_components[:-1], - all_components[1:]): + for components_i, components_j in itertools.pairwise(all_components): assert_allclose_dense_sparse(components_i, components_j) -def test_incremental_pca_partial_fit(): +def test_incremental_pca_partial_fit(global_random_seed): # Test that fit and partial_fit get equivalent results. - rng = np.random.RandomState(1999) + rng = np.random.RandomState(global_random_seed) n, p = 50, 3 X = rng.randn(n, p) # spherical data - X[:, 1] *= .00001 # make middle component relatively small + X[:, 1] *= 0.00001 # make middle component relatively small X += [5, 4, 3] # make a large mean # same check that we can find the original data from the transformed @@ -245,7 +301,7 @@ def test_incremental_pca_partial_fit(): pipca = IncrementalPCA(n_components=2, batch_size=batch_size) # Add one to make sure endpoint is included batch_itr = np.arange(0, n + 1, batch_size) - for i, j in zip(batch_itr[:-1], batch_itr[1:]): + for i, j in itertools.pairwise(batch_itr): pipca.partial_fit(X[i:j, :]) assert_almost_equal(ipca.components_, pipca.components_, decimal=3) @@ -260,9 +316,9 @@ def test_incremental_pca_against_pca_iris(): assert_almost_equal(np.abs(Y_pca), np.abs(Y_ipca), 1) -def test_incremental_pca_against_pca_random_data(): +def test_incremental_pca_against_pca_random_data(global_random_seed): # Test that IncrementalPCA and PCA are approximate (to a sign flip). - rng = np.random.RandomState(1999) + rng = np.random.RandomState(global_random_seed) n_samples = 100 n_features = 3 X = rng.randn(n_samples, n_features) + 5 * rng.rand(1, n_features) @@ -275,58 +331,66 @@ def test_incremental_pca_against_pca_random_data(): def test_explained_variances(): # Test that PCA and IncrementalPCA calculations match - X = datasets.make_low_rank_matrix(1000, 100, tail_strength=0., - effective_rank=10, random_state=1999) + X = datasets.make_low_rank_matrix( + 1000, 100, tail_strength=0.0, effective_rank=10, random_state=1999 + ) prec = 3 n_samples, n_features = X.shape for nc in [None, 99]: pca = PCA(n_components=nc).fit(X) ipca = IncrementalPCA(n_components=nc, batch_size=100).fit(X) - assert_almost_equal(pca.explained_variance_, ipca.explained_variance_, - decimal=prec) - assert_almost_equal(pca.explained_variance_ratio_, - ipca.explained_variance_ratio_, decimal=prec) - assert_almost_equal(pca.noise_variance_, ipca.noise_variance_, - decimal=prec) + assert_almost_equal( + pca.explained_variance_, ipca.explained_variance_, decimal=prec + ) + assert_almost_equal( + pca.explained_variance_ratio_, ipca.explained_variance_ratio_, decimal=prec + ) + assert_almost_equal(pca.noise_variance_, ipca.noise_variance_, decimal=prec) -def test_singular_values(): +def test_singular_values(global_random_seed): # Check that the IncrementalPCA output has the correct singular values - rng = np.random.RandomState(0) + rng = np.random.RandomState(global_random_seed) n_samples = 1000 n_features = 100 - X = datasets.make_low_rank_matrix(n_samples, n_features, tail_strength=0.0, - effective_rank=10, random_state=rng) + X = datasets.make_low_rank_matrix( + n_samples, n_features, tail_strength=0.0, effective_rank=10, random_state=rng + ) - pca = PCA(n_components=10, svd_solver='full', random_state=rng).fit(X) - ipca = IncrementalPCA(n_components=10, batch_size=100).fit(X) + pca = PCA(n_components=10, svd_solver="full", random_state=rng).fit(X) + ipca = IncrementalPCA(n_components=10, batch_size=150).fit(X) assert_array_almost_equal(pca.singular_values_, ipca.singular_values_, 2) # Compare to the Frobenius norm X_pca = pca.transform(X) X_ipca = ipca.transform(X) - assert_array_almost_equal(np.sum(pca.singular_values_**2.0), - np.linalg.norm(X_pca, "fro")**2.0, 12) - assert_array_almost_equal(np.sum(ipca.singular_values_**2.0), - np.linalg.norm(X_ipca, "fro")**2.0, 2) + assert_array_almost_equal( + np.sum(pca.singular_values_**2.0), np.linalg.norm(X_pca, "fro") ** 2.0, 12 + ) + assert_array_almost_equal( + np.sum(ipca.singular_values_**2.0), np.linalg.norm(X_ipca, "fro") ** 2.0, 2 + ) # Compare to the 2-norms of the score vectors - assert_array_almost_equal(pca.singular_values_, - np.sqrt(np.sum(X_pca**2.0, axis=0)), 12) - assert_array_almost_equal(ipca.singular_values_, - np.sqrt(np.sum(X_ipca**2.0, axis=0)), 2) + assert_array_almost_equal( + pca.singular_values_, np.sqrt(np.sum(X_pca**2.0, axis=0)), 12 + ) + assert_array_almost_equal( + ipca.singular_values_, np.sqrt(np.sum(X_ipca**2.0, axis=0)), 2 + ) # Set the singular values and see what we get back - rng = np.random.RandomState(0) + rng = np.random.RandomState(global_random_seed) n_samples = 100 n_features = 110 - X = datasets.make_low_rank_matrix(n_samples, n_features, tail_strength=0.0, - effective_rank=3, random_state=rng) + X = datasets.make_low_rank_matrix( + n_samples, n_features, tail_strength=0.0, effective_rank=3, random_state=rng + ) - pca = PCA(n_components=3, svd_solver='full', random_state=rng) + pca = PCA(n_components=3, svd_solver="full", random_state=rng) ipca = IncrementalPCA(n_components=3, batch_size=100) X_pca = pca.fit_transform(X) @@ -341,25 +405,38 @@ def test_singular_values(): assert_array_almost_equal(ipca.singular_values_, [3.142, 2.718, 1.0], 14) -def test_whitening(): +def test_whitening(global_random_seed): # Test that PCA and IncrementalPCA transforms match to sign flip. - X = datasets.make_low_rank_matrix(1000, 10, tail_strength=0., - effective_rank=2, random_state=1999) - prec = 3 - n_samples, n_features = X.shape + X = datasets.make_low_rank_matrix( + 1000, 10, tail_strength=0.0, effective_rank=2, random_state=global_random_seed + ) + atol = 1e-3 for nc in [None, 9]: pca = PCA(whiten=True, n_components=nc).fit(X) - ipca = IncrementalPCA(whiten=True, n_components=nc, - batch_size=250).fit(X) + ipca = IncrementalPCA(whiten=True, n_components=nc, batch_size=250).fit(X) + + # Since the data is rank deficient, some components are pure noise. We + # should not expect those dimensions to carry any signal and their + # values might be arbitrarily changed by implementation details of the + # internal SVD solver. We therefore filter them out before comparison. + stable_mask = pca.explained_variance_ratio_ > 1e-12 Xt_pca = pca.transform(X) Xt_ipca = ipca.transform(X) - assert_almost_equal(np.abs(Xt_pca), np.abs(Xt_ipca), decimal=prec) + assert_allclose( + np.abs(Xt_pca)[:, stable_mask], + np.abs(Xt_ipca)[:, stable_mask], + atol=atol, + ) + + # The noisy dimensions are in the null space of the inverse transform, + # so they are not influencing the reconstruction. We therefore don't + # need to apply the mask here. Xinv_ipca = ipca.inverse_transform(Xt_ipca) Xinv_pca = pca.inverse_transform(Xt_pca) - assert_almost_equal(X, Xinv_ipca, decimal=prec) - assert_almost_equal(X, Xinv_pca, decimal=prec) - assert_almost_equal(Xinv_pca, Xinv_ipca, decimal=prec) + assert_allclose(X, Xinv_ipca, atol=atol) + assert_allclose(X, Xinv_pca, atol=atol) + assert_allclose(Xinv_pca, Xinv_ipca, atol=atol) def test_incremental_pca_partial_fit_float_division(): @@ -382,8 +459,9 @@ def test_incremental_pca_partial_fit_float_division(): pca2.partial_fit(B) singular_vals_int_samples_seen = pca2.singular_values_ - np.testing.assert_allclose(singular_vals_float_samples_seen, - singular_vals_int_samples_seen) + np.testing.assert_allclose( + singular_vals_float_samples_seen, singular_vals_int_samples_seen + ) def test_incremental_pca_fit_overflow_error(): @@ -399,3 +477,11 @@ def test_incremental_pca_fit_overflow_error(): pca.fit(A) np.testing.assert_allclose(ipca.singular_values_, pca.singular_values_) + + +def test_incremental_pca_feature_names_out(): + """Check feature names out for IncrementalPCA.""" + ipca = IncrementalPCA(n_components=2).fit(iris.data) + + names = ipca.get_feature_names_out() + assert_array_equal([f"incrementalpca{i}" for i in range(2)], names) diff --git a/sklearn/decomposition/tests/test_kernel_pca.py b/sklearn/decomposition/tests/test_kernel_pca.py index 2acccb0df6781..57ae75c184622 100644 --- a/sklearn/decomposition/tests/test_kernel_pca.py +++ b/sklearn/decomposition/tests/test_kernel_pca.py @@ -1,44 +1,57 @@ +import warnings + import numpy as np -import scipy.sparse as sp import pytest -from sklearn.utils._testing import (assert_array_almost_equal, - assert_allclose) - +import sklearn +from sklearn.datasets import load_iris, make_blobs, make_circles from sklearn.decomposition import PCA, KernelPCA -from sklearn.datasets import make_circles -from sklearn.datasets import make_blobs +from sklearn.exceptions import NotFittedError from sklearn.linear_model import Perceptron +from sklearn.metrics.pairwise import rbf_kernel +from sklearn.model_selection import GridSearchCV from sklearn.pipeline import Pipeline from sklearn.preprocessing import StandardScaler -from sklearn.model_selection import GridSearchCV -from sklearn.metrics.pairwise import rbf_kernel +from sklearn.utils._testing import ( + assert_allclose, + assert_array_almost_equal, + assert_array_equal, +) +from sklearn.utils.fixes import CSR_CONTAINERS from sklearn.utils.validation import _check_psd_eigenvalues -def test_kernel_pca(): - rng = np.random.RandomState(0) +def test_kernel_pca(global_random_seed): + """Nominal test for all solvers and all known kernels + a custom one + + It tests + - that fit_transform is equivalent to fit+transform + - that the shapes of transforms and inverse transforms are correct + """ + rng = np.random.RandomState(global_random_seed) X_fit = rng.random_sample((5, 4)) X_pred = rng.random_sample((2, 4)) def histogram(x, y, **kwargs): # Histogram kernel implemented as a callable. - assert kwargs == {} # no kernel_params that we didn't ask for + assert kwargs == {} # no kernel_params that we didn't ask for return np.minimum(x, y).sum() - for eigen_solver in ("auto", "dense", "arpack"): + for eigen_solver in ("auto", "dense", "arpack", "randomized"): for kernel in ("linear", "rbf", "poly", histogram): # histogram kernel produces singular matrix inside linalg.solve # XXX use a least-squares approximation? inv = not callable(kernel) # transform fit data - kpca = KernelPCA(4, kernel=kernel, eigen_solver=eigen_solver, - fit_inverse_transform=inv) + kpca = KernelPCA( + 4, kernel=kernel, eigen_solver=eigen_solver, fit_inverse_transform=inv + ) X_fit_transformed = kpca.fit_transform(X_fit) X_fit_transformed2 = kpca.fit(X_fit).transform(X_fit) - assert_array_almost_equal(np.abs(X_fit_transformed), - np.abs(X_fit_transformed2)) + assert_array_almost_equal( + np.abs(X_fit_transformed), np.abs(X_fit_transformed2) + ) # non-regression test: previously, gamma would be 0 by default, # forcing all eigenvalues to 0 under the poly kernel @@ -46,8 +59,7 @@ def histogram(x, y, **kwargs): # transform new data X_pred_transformed = kpca.transform(X_pred) - assert (X_pred_transformed.shape[1] == - X_fit_transformed.shape[1]) + assert X_pred_transformed.shape[1] == X_fit_transformed.shape[1] # inverse transform if inv: @@ -56,13 +68,28 @@ def histogram(x, y, **kwargs): def test_kernel_pca_invalid_parameters(): - with pytest.raises(ValueError): - KernelPCA(10, fit_inverse_transform=True, kernel='precomputed') + """Check that kPCA raises an error if the parameters are invalid + + Tests fitting inverse transform with a precomputed kernel raises a + ValueError. + """ + estimator = KernelPCA( + n_components=10, fit_inverse_transform=True, kernel="precomputed" + ) + err_ms = "Cannot fit_inverse_transform with a precomputed kernel" + with pytest.raises(ValueError, match=err_ms): + estimator.fit(np.random.randn(10, 10)) + +def test_kernel_pca_consistent_transform(global_random_seed): + """Check robustness to mutations in the original training array -def test_kernel_pca_consistent_transform(): + Test that after fitting a kPCA model, it stays independent of any + mutation of the values of the original data object by relying on an + internal copy. + """ # X_fit_ needs to retain the old, unmodified copy of X - state = np.random.RandomState(0) + state = np.random.RandomState(global_random_seed) X = state.rand(10, 10) kpca = KernelPCA(random_state=state).fit(X) transformed1 = kpca.transform(X) @@ -73,66 +100,98 @@ def test_kernel_pca_consistent_transform(): assert_array_almost_equal(transformed1, transformed2) -def test_kernel_pca_deterministic_output(): - rng = np.random.RandomState(0) +def test_kernel_pca_deterministic_output(global_random_seed): + """Test that Kernel PCA produces deterministic output + + Tests that the same inputs and random state produce the same output. + """ + rng = np.random.RandomState(global_random_seed) X = rng.rand(10, 10) - eigen_solver = ('arpack', 'dense') + eigen_solver = ("arpack", "dense") for solver in eigen_solver: transformed_X = np.zeros((20, 2)) for i in range(20): - kpca = KernelPCA(n_components=2, eigen_solver=solver, - random_state=rng) + kpca = KernelPCA(n_components=2, eigen_solver=solver, random_state=rng) transformed_X[i, :] = kpca.fit_transform(X)[0] - assert_allclose( - transformed_X, np.tile(transformed_X[0, :], 20).reshape(20, 2)) + assert_allclose(transformed_X, np.tile(transformed_X[0, :], 20).reshape(20, 2)) -def test_kernel_pca_sparse(): - rng = np.random.RandomState(0) - X_fit = sp.csr_matrix(rng.random_sample((5, 4))) - X_pred = sp.csr_matrix(rng.random_sample((2, 4))) +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_kernel_pca_sparse(csr_container, global_random_seed): + """Test that kPCA works on a sparse data input. - for eigen_solver in ("auto", "arpack"): + Same test as ``test_kernel_pca except inverse_transform`` since it's not + implemented for sparse matrices. + """ + rng = np.random.RandomState(global_random_seed) + X_fit = csr_container(rng.random_sample((5, 4))) + X_pred = csr_container(rng.random_sample((2, 4))) + + for eigen_solver in ("auto", "arpack", "randomized"): for kernel in ("linear", "rbf", "poly"): # transform fit data - kpca = KernelPCA(4, kernel=kernel, eigen_solver=eigen_solver, - fit_inverse_transform=False) + kpca = KernelPCA( + 4, + kernel=kernel, + eigen_solver=eigen_solver, + fit_inverse_transform=False, + random_state=0, + ) X_fit_transformed = kpca.fit_transform(X_fit) X_fit_transformed2 = kpca.fit(X_fit).transform(X_fit) - assert_array_almost_equal(np.abs(X_fit_transformed), - np.abs(X_fit_transformed2)) + assert_array_almost_equal( + np.abs(X_fit_transformed), np.abs(X_fit_transformed2) + ) # transform new data X_pred_transformed = kpca.transform(X_pred) - assert (X_pred_transformed.shape[1] == - X_fit_transformed.shape[1]) + assert X_pred_transformed.shape[1] == X_fit_transformed.shape[1] - # inverse transform - # X_pred2 = kpca.inverse_transform(X_pred_transformed) - # assert X_pred2.shape == X_pred.shape) + # inverse transform: not available for sparse matrices + # XXX: should we raise another exception type here? For instance: + # NotImplementedError. + with pytest.raises(NotFittedError): + kpca.inverse_transform(X_pred_transformed) -def test_kernel_pca_linear_kernel(): - rng = np.random.RandomState(0) - X_fit = rng.random_sample((5, 4)) - X_pred = rng.random_sample((2, 4)) +@pytest.mark.parametrize("solver", ["auto", "dense", "arpack", "randomized"]) +@pytest.mark.parametrize("n_features", [4, 10]) +def test_kernel_pca_linear_kernel(solver, n_features, global_random_seed): + """Test that kPCA with linear kernel is equivalent to PCA for all solvers. + + KernelPCA with linear kernel should produce the same output as PCA. + """ + rng = np.random.RandomState(global_random_seed) + X_fit = rng.random_sample((5, n_features)) + X_pred = rng.random_sample((2, n_features)) # for a linear kernel, kernel PCA should find the same projection as PCA # modulo the sign (direction) # fit only the first four components: fifth is near zero eigenvalue, so # can be trimmed due to roundoff error + n_comps = 3 if solver == "arpack" else 4 assert_array_almost_equal( - np.abs(KernelPCA(4).fit(X_fit).transform(X_pred)), - np.abs(PCA(4).fit(X_fit).transform(X_pred))) + np.abs(KernelPCA(n_comps, eigen_solver=solver).fit(X_fit).transform(X_pred)), + np.abs( + PCA(n_comps, svd_solver=solver if solver != "dense" else "full") + .fit(X_fit) + .transform(X_pred) + ), + ) def test_kernel_pca_n_components(): + """Test that `n_components` is correctly taken into account for projections + + For all solvers this tests that the output has the correct shape depending + on the selected number of components. + """ rng = np.random.RandomState(0) X_fit = rng.random_sample((5, 4)) X_pred = rng.random_sample((2, 4)) - for eigen_solver in ("dense", "arpack"): + for eigen_solver in ("dense", "arpack", "randomized"): for c in [1, 2, 4]: kpca = KernelPCA(n_components=c, eigen_solver=eigen_solver) shape = kpca.fit(X_fit).transform(X_pred).shape @@ -141,6 +200,11 @@ def test_kernel_pca_n_components(): def test_remove_zero_eig(): + """Check that the ``remove_zero_eig`` parameter works correctly. + + Tests that the null-space (Zero) eigenvalues are removed when + remove_zero_eig=True, whereas they are not by default. + """ X = np.array([[1 - 1e-30, 1], [1, 1], [1, 1 - 1e-20]]) # n_components=None (default) => remove_zero_eig is True @@ -158,16 +222,22 @@ def test_remove_zero_eig(): def test_leave_zero_eig(): - """This test checks that fit().transform() returns the same result as + """Non-regression test for issue #12141 (PR #12143) + + This test checks that fit().transform() returns the same result as fit_transform() in case of non-removed zero eigenvalue. - Non-regression test for issue #12141 (PR #12143)""" + """ X_fit = np.array([[1, 1], [0, 0]]) # Assert that even with all np warnings on, there is no div by zero warning - with pytest.warns(None) as record: - with np.errstate(all='warn'): - k = KernelPCA(n_components=2, remove_zero_eig=False, - eigen_solver="dense") + with warnings.catch_warnings(): + # There might be warnings about the kernel being badly conditioned, + # but there should not be warnings about division by zero. + # (Numpy division by zero warning can have many message variants, but + # at least we know that it is a RuntimeWarning so lets check only this) + warnings.simplefilter("error", RuntimeWarning) + with np.errstate(all="warn"): + k = KernelPCA(n_components=2, remove_zero_eig=False, eigen_solver="dense") # Fit, then transform A = k.fit(X_fit).transform(X_fit) # Do both at once @@ -175,81 +245,111 @@ def test_leave_zero_eig(): # Compare assert_array_almost_equal(np.abs(A), np.abs(B)) - for w in record: - # There might be warnings about the kernel being badly conditioned, - # but there should not be warnings about division by zero. - # (Numpy division by zero warning can have many message variants, but - # at least we know that it is a RuntimeWarning so lets check only this) - assert not issubclass(w.category, RuntimeWarning) - -def test_kernel_pca_precomputed(): - rng = np.random.RandomState(0) +def test_kernel_pca_precomputed(global_random_seed): + """Test that kPCA works with a precomputed kernel, for all solvers""" + rng = np.random.RandomState(global_random_seed) X_fit = rng.random_sample((5, 4)) X_pred = rng.random_sample((2, 4)) - for eigen_solver in ("dense", "arpack"): - X_kpca = KernelPCA(4, eigen_solver=eigen_solver).\ - fit(X_fit).transform(X_pred) - X_kpca2 = KernelPCA( - 4, eigen_solver=eigen_solver, kernel='precomputed').fit( - np.dot(X_fit, X_fit.T)).transform(np.dot(X_pred, X_fit.T)) + for eigen_solver in ("dense", "arpack", "randomized"): + X_kpca = ( + KernelPCA(4, eigen_solver=eigen_solver, random_state=0) + .fit(X_fit) + .transform(X_pred) + ) + + X_kpca2 = ( + KernelPCA( + 4, eigen_solver=eigen_solver, kernel="precomputed", random_state=0 + ) + .fit(np.dot(X_fit, X_fit.T)) + .transform(np.dot(X_pred, X_fit.T)) + ) X_kpca_train = KernelPCA( - 4, eigen_solver=eigen_solver, - kernel='precomputed').fit_transform(np.dot(X_fit, X_fit.T)) - X_kpca_train2 = KernelPCA( - 4, eigen_solver=eigen_solver, kernel='precomputed').fit( - np.dot(X_fit, X_fit.T)).transform(np.dot(X_fit, X_fit.T)) + 4, eigen_solver=eigen_solver, kernel="precomputed", random_state=0 + ).fit_transform(np.dot(X_fit, X_fit.T)) - assert_array_almost_equal(np.abs(X_kpca), - np.abs(X_kpca2)) + X_kpca_train2 = ( + KernelPCA( + 4, eigen_solver=eigen_solver, kernel="precomputed", random_state=0 + ) + .fit(np.dot(X_fit, X_fit.T)) + .transform(np.dot(X_fit, X_fit.T)) + ) - assert_array_almost_equal(np.abs(X_kpca_train), - np.abs(X_kpca_train2)) + assert_array_almost_equal(np.abs(X_kpca), np.abs(X_kpca2)) + assert_array_almost_equal(np.abs(X_kpca_train), np.abs(X_kpca_train2)) -def test_kernel_pca_invalid_kernel(): - rng = np.random.RandomState(0) - X_fit = rng.random_sample((2, 4)) - kpca = KernelPCA(kernel="tototiti") - with pytest.raises(ValueError): - kpca.fit(X_fit) + +@pytest.mark.parametrize("solver", ["auto", "dense", "arpack", "randomized"]) +def test_kernel_pca_precomputed_non_symmetric(solver): + """Check that the kernel centerer works. + + Tests that a non symmetric precomputed kernel is actually accepted + because the kernel centerer does its job correctly. + """ + + # a non symmetric gram matrix + K = [[1, 2], [3, 40]] + kpca = KernelPCA( + kernel="precomputed", eigen_solver=solver, n_components=1, random_state=0 + ) + kpca.fit(K) # no error + + # same test with centered kernel + Kc = [[9, -9], [-9, 9]] + kpca_c = KernelPCA( + kernel="precomputed", eigen_solver=solver, n_components=1, random_state=0 + ) + kpca_c.fit(Kc) + + # comparison between the non-centered and centered versions + assert_array_equal(kpca.eigenvectors_, kpca_c.eigenvectors_) + assert_array_equal(kpca.eigenvalues_, kpca_c.eigenvalues_) def test_gridsearch_pipeline(): - # Test if we can do a grid-search to find parameters to separate - # circles with a perceptron model. - X, y = make_circles(n_samples=400, factor=.3, noise=.05, - random_state=0) + """Check that kPCA works as expected in a grid search pipeline + + Test if we can do a grid-search to find parameters to separate + circles with a perceptron model. + """ + X, y = make_circles(n_samples=400, factor=0.3, noise=0.05, random_state=0) kpca = KernelPCA(kernel="rbf", n_components=2) - pipeline = Pipeline([("kernel_pca", kpca), - ("Perceptron", Perceptron(max_iter=5))]) - param_grid = dict(kernel_pca__gamma=2. ** np.arange(-2, 2)) + pipeline = Pipeline([("kernel_pca", kpca), ("Perceptron", Perceptron(max_iter=5))]) + param_grid = dict(kernel_pca__gamma=2.0 ** np.arange(-2, 2)) grid_search = GridSearchCV(pipeline, cv=3, param_grid=param_grid) grid_search.fit(X, y) assert grid_search.best_score_ == 1 def test_gridsearch_pipeline_precomputed(): - # Test if we can do a grid-search to find parameters to separate - # circles with a perceptron model using a precomputed kernel. - X, y = make_circles(n_samples=400, factor=.3, noise=.05, - random_state=0) + """Check that kPCA works as expected in a grid search pipeline (2) + + Test if we can do a grid-search to find parameters to separate + circles with a perceptron model. This test uses a precomputed kernel. + """ + X, y = make_circles(n_samples=400, factor=0.3, noise=0.05, random_state=0) kpca = KernelPCA(kernel="precomputed", n_components=2) - pipeline = Pipeline([("kernel_pca", kpca), - ("Perceptron", Perceptron(max_iter=5))]) + pipeline = Pipeline([("kernel_pca", kpca), ("Perceptron", Perceptron(max_iter=5))]) param_grid = dict(Perceptron__max_iter=np.arange(1, 5)) grid_search = GridSearchCV(pipeline, cv=3, param_grid=param_grid) - X_kernel = rbf_kernel(X, gamma=2.) + X_kernel = rbf_kernel(X, gamma=2.0) grid_search.fit(X_kernel, y) assert grid_search.best_score_ == 1 def test_nested_circles(): - # Test the linear separability of the first 2D KPCA transform - X, y = make_circles(n_samples=400, factor=.3, noise=.05, - random_state=0) + """Check that kPCA projects in a space where nested circles are separable + + Tests that 2D nested circles become separable with a perceptron when + projected in the first 2 kPCA using an RBF kernel, while raw samples + are not directly separable in the original space. + """ + X, y = make_circles(n_samples=400, factor=0.3, noise=0.05, random_state=0) # 2D nested circles are not linearly separable train_score = Perceptron(max_iter=5).fit(X, y).score(X, y) @@ -260,8 +360,9 @@ def test_nested_circles(): # Note that the gamma value is data dependent. If this test breaks # and the gamma value has to be updated, the Kernel PCA example will # have to be updated too. - kpca = KernelPCA(kernel="rbf", n_components=2, - fit_inverse_transform=True, gamma=2.) + kpca = KernelPCA( + kernel="rbf", n_components=2, fit_inverse_transform=True, gamma=2.0 + ) X_kpca = kpca.fit_transform(X) # The data is perfectly linearly separable in that space @@ -270,55 +371,196 @@ def test_nested_circles(): def test_kernel_conditioning(): - """ Test that ``_check_psd_eigenvalues`` is correctly called - Non-regression test for issue #12140 (PR #12145)""" + """Check that ``_check_psd_eigenvalues`` is correctly called in kPCA + + Non-regression test for issue #12140 (PR #12145). + """ # create a pathological X leading to small non-zero eigenvalue - X = [[5, 1], - [5+1e-8, 1e-8], - [5+1e-8, 0]] - kpca = KernelPCA(kernel="linear", n_components=2, - fit_inverse_transform=True) + X = [[5, 1], [5 + 1e-8, 1e-8], [5 + 1e-8, 0]] + kpca = KernelPCA(kernel="linear", n_components=2, fit_inverse_transform=True) kpca.fit(X) # check that the small non-zero eigenvalue was correctly set to zero - assert kpca.lambdas_.min() == 0 - assert np.all(kpca.lambdas_ == _check_psd_eigenvalues(kpca.lambdas_)) + assert kpca.eigenvalues_.min() == 0 + assert np.all(kpca.eigenvalues_ == _check_psd_eigenvalues(kpca.eigenvalues_)) + + +@pytest.mark.parametrize("solver", ["auto", "dense", "arpack", "randomized"]) +def test_precomputed_kernel_not_psd(solver): + """Check how KernelPCA works with non-PSD kernels depending on n_components + + Tests for all methods what happens with a non PSD gram matrix (this + can happen in an isomap scenario, or with custom kernel functions, or + maybe with ill-posed datasets). + + When ``n_component`` is large enough to capture a negative eigenvalue, an + error should be raised. Otherwise, KernelPCA should run without error + since the negative eigenvalues are not selected. + """ + + # a non PSD kernel with large eigenvalues, already centered + # it was captured from an isomap call and multiplied by 100 for compacity + K = [ + [4.48, -1.0, 8.07, 2.33, 2.33, 2.33, -5.76, -12.78], + [-1.0, -6.48, 4.5, -1.24, -1.24, -1.24, -0.81, 7.49], + [8.07, 4.5, 15.48, 2.09, 2.09, 2.09, -11.1, -23.23], + [2.33, -1.24, 2.09, 4.0, -3.65, -3.65, 1.02, -0.9], + [2.33, -1.24, 2.09, -3.65, 4.0, -3.65, 1.02, -0.9], + [2.33, -1.24, 2.09, -3.65, -3.65, 4.0, 1.02, -0.9], + [-5.76, -0.81, -11.1, 1.02, 1.02, 1.02, 4.86, 9.75], + [-12.78, 7.49, -23.23, -0.9, -0.9, -0.9, 9.75, 21.46], + ] + # this gram matrix has 5 positive eigenvalues and 3 negative ones + # [ 52.72, 7.65, 7.65, 5.02, 0. , -0. , -6.13, -15.11] + + # 1. ask for enough components to get a significant negative one + kpca = KernelPCA(kernel="precomputed", eigen_solver=solver, n_components=7) + # make sure that the appropriate error is raised + with pytest.raises(ValueError, match="There are significant negative eigenvalues"): + kpca.fit(K) + + # 2. ask for a small enough n_components to get only positive ones + kpca = KernelPCA(kernel="precomputed", eigen_solver=solver, n_components=2) + if solver == "randomized": + # the randomized method is still inconsistent with the others on this + # since it selects the eigenvalues based on the largest 2 modules, not + # on the largest 2 values. + # + # At least we can ensure that we return an error instead of returning + # the wrong eigenvalues + with pytest.raises( + ValueError, match="There are significant negative eigenvalues" + ): + kpca.fit(K) + else: + # general case: make sure that it works + kpca.fit(K) + + +@pytest.mark.parametrize("n_components", [4, 10, 20]) +def test_kernel_pca_solvers_equivalence(n_components): + """Check that 'dense' 'arpack' & 'randomized' solvers give similar results""" + + # Generate random data + n_train, n_test = 1_000, 100 + X, _ = make_circles( + n_samples=(n_train + n_test), factor=0.3, noise=0.05, random_state=0 + ) + X_fit, X_pred = X[:n_train, :], X[n_train:, :] + + # reference (full) + ref_pred = ( + KernelPCA(n_components, eigen_solver="dense", random_state=0) + .fit(X_fit) + .transform(X_pred) + ) + + # arpack + a_pred = ( + KernelPCA(n_components, eigen_solver="arpack", random_state=0) + .fit(X_fit) + .transform(X_pred) + ) + # check that the result is still correct despite the approx + assert_array_almost_equal(np.abs(a_pred), np.abs(ref_pred)) + + # randomized + r_pred = ( + KernelPCA(n_components, eigen_solver="randomized", random_state=0) + .fit(X_fit) + .transform(X_pred) + ) + # check that the result is still correct despite the approximation + assert_array_almost_equal(np.abs(r_pred), np.abs(ref_pred)) + +def test_kernel_pca_inverse_transform_reconstruction(): + """Test if the reconstruction is a good approximation. -@pytest.mark.parametrize("kernel", - ["linear", "poly", "rbf", "sigmoid", "cosine"]) -def test_kernel_pca_inverse_transform(kernel): - X, *_ = make_blobs(n_samples=100, n_features=4, centers=[[1, 1, 1, 1]], - random_state=0) + Note that in general it is not possible to get an arbitrarily good + reconstruction because of kernel centering that does not + preserve all the information of the original data. + """ + X, *_ = make_blobs(n_samples=100, n_features=4, random_state=0) - kp = KernelPCA(n_components=2, kernel=kernel, fit_inverse_transform=True) - X_trans = kp.fit_transform(X) - X_inv = kp.inverse_transform(X_trans) - assert_allclose(X, X_inv) + kpca = KernelPCA( + n_components=20, kernel="rbf", fit_inverse_transform=True, alpha=1e-3 + ) + X_trans = kpca.fit_transform(X) + X_reconst = kpca.inverse_transform(X_trans) + assert np.linalg.norm(X - X_reconst) / np.linalg.norm(X) < 1e-1 + + +def test_kernel_pca_raise_not_fitted_error(): + X = np.random.randn(15).reshape(5, 3) + kpca = KernelPCA() + kpca.fit(X) + with pytest.raises(NotFittedError): + kpca.inverse_transform(X) def test_32_64_decomposition_shape(): - """ Test that the decomposition is similar for 32 and 64 bits data """ - # see https://github.com/scikit-learn/scikit-learn/issues/18146 + """Test that the decomposition is similar for 32 and 64 bits data + + Non regression test for + https://github.com/scikit-learn/scikit-learn/issues/18146 + """ X, y = make_blobs( - n_samples=30, - centers=[[0, 0, 0], [1, 1, 1]], - random_state=0, - cluster_std=0.1 + n_samples=30, centers=[[0, 0, 0], [1, 1, 1]], random_state=0, cluster_std=0.1 ) X = StandardScaler().fit_transform(X) X -= X.min() # Compare the shapes (corresponds to the number of non-zero eigenvalues) kpca = KernelPCA() - assert (kpca.fit_transform(X).shape == - kpca.fit_transform(X.astype(np.float32)).shape) + assert kpca.fit_transform(X).shape == kpca.fit_transform(X.astype(np.float32)).shape + + +def test_kernel_pca_feature_names_out(): + """Check feature names out for KernelPCA.""" + X, *_ = make_blobs(n_samples=100, n_features=4, random_state=0) + kpca = KernelPCA(n_components=2).fit(X) + + names = kpca.get_feature_names_out() + assert_array_equal([f"kernelpca{i}" for i in range(2)], names) + + +def test_kernel_pca_inverse_correct_gamma(global_random_seed): + """Check that gamma is set correctly when not provided. + + Non-regression test for #26280 + """ + rng = np.random.RandomState(global_random_seed) + X = rng.random_sample((5, 4)) + + kwargs = { + "n_components": 2, + "random_state": rng, + "fit_inverse_transform": True, + "kernel": "rbf", + } + + expected_gamma = 1 / X.shape[1] + kpca1 = KernelPCA(gamma=None, **kwargs).fit(X) + kpca2 = KernelPCA(gamma=expected_gamma, **kwargs).fit(X) + + assert kpca1.gamma_ == expected_gamma + assert kpca2.gamma_ == expected_gamma + + X1_recon = kpca1.inverse_transform(kpca1.transform(X)) + X2_recon = kpca2.inverse_transform(kpca1.transform(X)) + + assert_allclose(X1_recon, X2_recon) + +def test_kernel_pca_pandas_output(): + """Check that KernelPCA works with pandas output when the solver is arpack. -# TODO: Remove in 1.1 -def test_kernel_pcc_pairwise_is_deprecated(): - kp = KernelPCA(kernel='precomputed') - msg = r"Attribute _pairwise was deprecated in version 0\.24" - with pytest.warns(FutureWarning, match=msg): - kp._pairwise + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/27579 + """ + pytest.importorskip("pandas") + X, _ = load_iris(as_frame=True, return_X_y=True) + with sklearn.config_context(transform_output="pandas"): + KernelPCA(n_components=2, eigen_solver="arpack").fit_transform(X) diff --git a/sklearn/decomposition/tests/test_nmf.py b/sklearn/decomposition/tests/test_nmf.py index 88c1ba406ad99..17be798b3f392 100644 --- a/sklearn/decomposition/tests/test_nmf.py +++ b/sklearn/decomposition/tests/test_nmf.py @@ -1,78 +1,80 @@ -import numpy as np -import scipy.sparse as sp - -from scipy import linalg -from sklearn.decomposition import NMF, non_negative_factorization -from sklearn.decomposition import _nmf as nmf # For testing internals -from scipy.sparse import csc_matrix +import re +import sys +from io import StringIO +import numpy as np import pytest +from scipy import linalg -from sklearn.utils._testing import assert_raise_message -from sklearn.utils._testing import assert_array_equal -from sklearn.utils._testing import assert_array_almost_equal -from sklearn.utils._testing import assert_almost_equal -from sklearn.utils._testing import assert_allclose -from sklearn.utils._testing import ignore_warnings -from sklearn.utils.extmath import squared_norm from sklearn.base import clone +from sklearn.decomposition import NMF, MiniBatchNMF, non_negative_factorization +from sklearn.decomposition import _nmf as nmf # For testing internals from sklearn.exceptions import ConvergenceWarning +from sklearn.utils._testing import ( + assert_allclose, + assert_almost_equal, + assert_array_almost_equal, + assert_array_equal, +) +from sklearn.utils.extmath import squared_norm +from sklearn.utils.fixes import CSC_CONTAINERS, CSR_CONTAINERS -@pytest.mark.parametrize('solver', ['cd', 'mu']) -@pytest.mark.parametrize('regularization', - [None, 'both', 'components', 'transformation']) -def test_convergence_warning(solver, regularization): - convergence_warning = ("Maximum number of iterations 1 reached. " - "Increase it to improve convergence.") +@pytest.mark.parametrize( + ["Estimator", "solver"], + [[NMF, {"solver": "cd"}], [NMF, {"solver": "mu"}], [MiniBatchNMF, {}]], +) +def test_convergence_warning(Estimator, solver): + convergence_warning = ( + "Maximum number of iterations 1 reached. Increase it to improve convergence." + ) A = np.ones((2, 2)) with pytest.warns(ConvergenceWarning, match=convergence_warning): - NMF(solver=solver, regularization=regularization, max_iter=1).fit(A) + Estimator(max_iter=1, n_components="auto", **solver).fit(A) def test_initialize_nn_output(): # Test that initialization does not return negative values rng = np.random.mtrand.RandomState(42) data = np.abs(rng.randn(10, 10)) - for init in ('random', 'nndsvd', 'nndsvda', 'nndsvdar'): + for init in ("random", "nndsvd", "nndsvda", "nndsvdar"): W, H = nmf._initialize_nmf(data, 10, init=init, random_state=0) assert not ((W < 0).any() or (H < 0).any()) +@pytest.mark.filterwarnings( + r"ignore:The multiplicative update \('mu'\) solver cannot update zeros present in" + r" the initialization", +) def test_parameter_checking(): + # Here we only check for invalid parameter values that are not already + # automatically tested in the common tests. + A = np.ones((2, 2)) - name = 'spam' - # FIXME : should be removed in 1.1 - init = 'nndsvda' - msg = "Invalid solver parameter: got 'spam' instead of one of" - assert_raise_message(ValueError, msg, NMF(solver=name, init=init).fit, A) - msg = "Invalid init parameter: got 'spam' instead of one of" - assert_raise_message(ValueError, msg, NMF(init=name).fit, A) - msg = "Invalid regularization parameter: got 'spam' instead of one of" - assert_raise_message(ValueError, msg, NMF(regularization=name, - init=init).fit, A) - msg = "Invalid beta_loss parameter: got 'spam' instead of one" - assert_raise_message(ValueError, msg, NMF(solver='mu', init=init, - beta_loss=name).fit, A) - msg = "Invalid beta_loss parameter: solver 'cd' does not handle " - msg += "beta_loss = 1.0" - assert_raise_message(ValueError, msg, NMF(solver='cd', init=init, - beta_loss=1.0).fit, A) + msg = "Invalid beta_loss parameter: solver 'cd' does not handle beta_loss = 1.0" + with pytest.raises(ValueError, match=msg): + NMF(solver="cd", beta_loss=1.0).fit(A) msg = "Negative values in data passed to" - assert_raise_message(ValueError, msg, NMF(init=init).fit, -A) - assert_raise_message(ValueError, msg, nmf._initialize_nmf, -A, - 2, 'nndsvd') - clf = NMF(2, tol=0.1, init=init).fit(A) - assert_raise_message(ValueError, msg, clf.transform, -A) - - for init in ['nndsvd', 'nndsvda', 'nndsvdar']: - msg = ("init = '{}' can only be used when " - "n_components <= min(n_samples, n_features)" - .format(init)) - assert_raise_message(ValueError, msg, NMF(3, init=init).fit, A) - assert_raise_message(ValueError, msg, nmf._initialize_nmf, A, - 3, init) + with pytest.raises(ValueError, match=msg): + NMF().fit(-A) + clf = NMF(2, tol=0.1).fit(A) + with pytest.raises(ValueError, match=msg): + clf.transform(-A) + with pytest.raises(ValueError, match=msg): + nmf._initialize_nmf(-A, 2, "nndsvd") + + for init in ["nndsvd", "nndsvda", "nndsvdar"]: + msg = re.escape( + "init = '{}' can only be used when " + "n_components <= min(n_samples, n_features)".format(init) + ) + with pytest.raises(ValueError, match=msg): + NMF(3, init=init).fit(A) + with pytest.raises(ValueError, match=msg): + MiniBatchNMF(3, init=init).fit(A) + with pytest.raises(ValueError, match=msg): + nmf._initialize_nmf(A, 3, init) def test_initialize_close(): @@ -81,7 +83,7 @@ def test_initialize_close(): # the entries in the matrix. rng = np.random.mtrand.RandomState(42) A = np.abs(rng.randn(10, 10)) - W, H = nmf._initialize_nmf(A, 10, init='nndsvd') + W, H = nmf._initialize_nmf(A, 10, init="nndsvd") error = linalg.norm(np.dot(W, H) - A) sdev = linalg.norm(A - A.mean()) assert error <= sdev @@ -93,60 +95,146 @@ def test_initialize_variants(): # 'nndsvd' only where the basic version has zeros. rng = np.random.mtrand.RandomState(42) data = np.abs(rng.randn(10, 10)) - W0, H0 = nmf._initialize_nmf(data, 10, init='nndsvd') - Wa, Ha = nmf._initialize_nmf(data, 10, init='nndsvda') - War, Har = nmf._initialize_nmf(data, 10, init='nndsvdar', - random_state=0) + W0, H0 = nmf._initialize_nmf(data, 10, init="nndsvd") + Wa, Ha = nmf._initialize_nmf(data, 10, init="nndsvda") + War, Har = nmf._initialize_nmf(data, 10, init="nndsvdar", random_state=0) for ref, evl in ((W0, Wa), (W0, War), (H0, Ha), (H0, Har)): assert_almost_equal(evl[ref != 0], ref[ref != 0]) # ignore UserWarning raised when both solver='mu' and init='nndsvd' -@ignore_warnings(category=UserWarning) -@pytest.mark.parametrize('solver', ('cd', 'mu')) -@pytest.mark.parametrize('init', - (None, 'nndsvd', 'nndsvda', 'nndsvdar', 'random')) -@pytest.mark.parametrize('regularization', - (None, 'both', 'components', 'transformation')) -def test_nmf_fit_nn_output(solver, init, regularization): +@pytest.mark.filterwarnings( + r"ignore:The multiplicative update \('mu'\) solver cannot update zeros present in" + r" the initialization" +) +@pytest.mark.parametrize( + ["Estimator", "solver"], + [[NMF, {"solver": "cd"}], [NMF, {"solver": "mu"}], [MiniBatchNMF, {}]], +) +@pytest.mark.parametrize("init", (None, "nndsvd", "nndsvda", "nndsvdar", "random")) +@pytest.mark.parametrize("alpha_W", (0.0, 1.0)) +@pytest.mark.parametrize("alpha_H", (0.0, 1.0, "same")) +def test_nmf_fit_nn_output(Estimator, solver, init, alpha_W, alpha_H): # Test that the decomposition does not contain negative values - A = np.c_[5. - np.arange(1, 6), - 5. + np.arange(1, 6)] - model = NMF(n_components=2, solver=solver, init=init, - regularization=regularization, random_state=0) + A = np.c_[5.0 - np.arange(1, 6), 5.0 + np.arange(1, 6)] + model = Estimator( + n_components=2, + init=init, + alpha_W=alpha_W, + alpha_H=alpha_H, + random_state=0, + **solver, + ) transf = model.fit_transform(A) - assert not((model.components_ < 0).any() or - (transf < 0).any()) + assert not ((model.components_ < 0).any() or (transf < 0).any()) -@pytest.mark.parametrize('solver', ('cd', 'mu')) -@pytest.mark.parametrize('regularization', - (None, 'both', 'components', 'transformation')) -def test_nmf_fit_close(solver, regularization): +@pytest.mark.parametrize( + ["Estimator", "solver"], + [[NMF, {"solver": "cd"}], [NMF, {"solver": "mu"}], [MiniBatchNMF, {}]], +) +def test_nmf_fit_close(Estimator, solver): rng = np.random.mtrand.RandomState(42) # Test that the fit is not too far away - pnmf = NMF(5, solver=solver, init='nndsvdar', random_state=0, - regularization=regularization, max_iter=600) + pnmf = Estimator( + 5, + init="nndsvdar", + random_state=0, + max_iter=600, + **solver, + ) X = np.abs(rng.randn(6, 5)) assert pnmf.fit(X).reconstruction_err_ < 0.1 -@pytest.mark.parametrize('solver', ('cd', 'mu')) -@pytest.mark.parametrize('regularization', - (None, 'both', 'components', 'transformation')) -def test_nmf_transform(solver, regularization): +def test_nmf_true_reconstruction(): + # Test that the fit is not too far away from an exact solution + # (by construction) + n_samples = 15 + n_features = 10 + n_components = 5 + beta_loss = 1 + batch_size = 3 + max_iter = 1000 + + rng = np.random.mtrand.RandomState(42) + W_true = np.zeros([n_samples, n_components]) + W_array = np.abs(rng.randn(n_samples)) + for j in range(n_components): + W_true[j % n_samples, j] = W_array[j % n_samples] + H_true = np.zeros([n_components, n_features]) + H_array = np.abs(rng.randn(n_components)) + for j in range(n_features): + H_true[j % n_components, j] = H_array[j % n_components] + X = np.dot(W_true, H_true) + + model = NMF( + n_components=n_components, + solver="mu", + beta_loss=beta_loss, + max_iter=max_iter, + random_state=0, + ) + transf = model.fit_transform(X) + X_calc = np.dot(transf, model.components_) + + assert model.reconstruction_err_ < 0.1 + assert_allclose(X, X_calc) + + mbmodel = MiniBatchNMF( + n_components=n_components, + beta_loss=beta_loss, + batch_size=batch_size, + random_state=0, + max_iter=max_iter, + ) + transf = mbmodel.fit_transform(X) + X_calc = np.dot(transf, mbmodel.components_) + + assert mbmodel.reconstruction_err_ < 0.1 + assert_allclose(X, X_calc, atol=1) + + +@pytest.mark.parametrize("solver", ["cd", "mu"]) +def test_nmf_transform(solver): + # Test that fit_transform is equivalent to fit.transform for NMF # Test that NMF.transform returns close values rng = np.random.mtrand.RandomState(42) A = np.abs(rng.randn(6, 5)) - m = NMF(solver=solver, n_components=3, init='random', - regularization=regularization, random_state=0, tol=1e-5) + m = NMF( + solver=solver, + n_components=3, + init="random", + random_state=0, + tol=1e-6, + ) ft = m.fit_transform(A) t = m.transform(A) - assert_array_almost_equal(ft, t, decimal=2) + assert_allclose(ft, t, atol=1e-1) + +def test_minibatch_nmf_transform(): + # Test that fit_transform is equivalent to fit.transform for MiniBatchNMF + # Only guaranteed with fresh restarts + rng = np.random.mtrand.RandomState(42) + A = np.abs(rng.randn(6, 5)) + m = MiniBatchNMF( + n_components=3, + random_state=0, + tol=1e-3, + fresh_restarts=True, + ) + ft = m.fit_transform(A) + t = m.transform(A) + assert_allclose(ft, t) -def test_nmf_transform_custom_init(): + +@pytest.mark.parametrize( + ["Estimator", "solver"], + [[NMF, {"solver": "cd"}], [NMF, {"solver": "mu"}], [MiniBatchNMF, {}]], +) +def test_nmf_transform_custom_init(Estimator, solver): # Smoke test that checks if NMF.transform works with custom initialization random_state = np.random.RandomState(0) A = np.abs(random_state.randn(6, 5)) @@ -155,50 +243,78 @@ def test_nmf_transform_custom_init(): H_init = np.abs(avg * random_state.randn(n_components, 5)) W_init = np.abs(avg * random_state.randn(6, n_components)) - m = NMF(solver='cd', n_components=n_components, init='custom', - random_state=0) + m = Estimator( + n_components=n_components, init="custom", random_state=0, tol=1e-3, **solver + ) m.fit_transform(A, W=W_init, H=H_init) m.transform(A) -@pytest.mark.parametrize('solver', ('cd', 'mu')) -@pytest.mark.parametrize('regularization', - (None, 'both', 'components', 'transformation')) -def test_nmf_inverse_transform(solver, regularization): +@pytest.mark.parametrize("solver", ("cd", "mu")) +def test_nmf_inverse_transform(solver): # Test that NMF.inverse_transform returns close values random_state = np.random.RandomState(0) A = np.abs(random_state.randn(6, 4)) - m = NMF(solver=solver, n_components=4, init='random', random_state=0, - regularization=regularization, max_iter=1000) + m = NMF( + solver=solver, + n_components=4, + init="random", + random_state=0, + max_iter=1000, + ) ft = m.fit_transform(A) A_new = m.inverse_transform(ft) assert_array_almost_equal(A, A_new, decimal=2) -def test_n_components_greater_n_features(): +def test_mbnmf_inverse_transform(): + # Test that MiniBatchNMF.transform followed by MiniBatchNMF.inverse_transform + # is close to the identity + rng = np.random.RandomState(0) + A = np.abs(rng.randn(6, 4)) + nmf = MiniBatchNMF( + random_state=rng, + max_iter=500, + init="nndsvdar", + fresh_restarts=True, + ) + ft = nmf.fit_transform(A) + A_new = nmf.inverse_transform(ft) + assert_allclose(A, A_new, rtol=1e-3, atol=1e-2) + + +@pytest.mark.parametrize("Estimator", [NMF, MiniBatchNMF]) +def test_n_components_greater_n_features(Estimator): # Smoke test for the case of more components than features. rng = np.random.mtrand.RandomState(42) A = np.abs(rng.randn(30, 10)) - # FIXME : should be removed in 1.1 - init = 'random' - NMF(n_components=15, random_state=0, tol=1e-2, init=init).fit(A) + Estimator(n_components=15, random_state=0, tol=1e-2).fit(A) -@pytest.mark.parametrize('solver', ['cd', 'mu']) -@pytest.mark.parametrize('regularization', - [None, 'both', 'components', 'transformation']) -def test_nmf_sparse_input(solver, regularization): +@pytest.mark.parametrize( + ["Estimator", "solver"], + [[NMF, {"solver": "cd"}], [NMF, {"solver": "mu"}], [MiniBatchNMF, {}]], +) +@pytest.mark.parametrize("sparse_container", CSC_CONTAINERS + CSR_CONTAINERS) +@pytest.mark.parametrize("alpha_W", (0.0, 1.0)) +@pytest.mark.parametrize("alpha_H", (0.0, 1.0, "same")) +def test_nmf_sparse_input(Estimator, solver, sparse_container, alpha_W, alpha_H): # Test that sparse matrices are accepted as input - from scipy.sparse import csc_matrix - rng = np.random.mtrand.RandomState(42) A = np.abs(rng.randn(10, 10)) A[:, 2 * np.arange(5)] = 0 - A_sparse = csc_matrix(A) - - est1 = NMF(solver=solver, n_components=5, init='random', - regularization=regularization, random_state=0, - tol=1e-2) + A_sparse = sparse_container(A) + + est1 = Estimator( + n_components=5, + init="random", + alpha_W=alpha_W, + alpha_H=alpha_H, + random_state=0, + tol=0, + max_iter=100, + **solver, + ) est2 = clone(est1) W1 = est1.fit_transform(A) @@ -206,72 +322,97 @@ def test_nmf_sparse_input(solver, regularization): H1 = est1.components_ H2 = est2.components_ - assert_array_almost_equal(W1, W2) - assert_array_almost_equal(H1, H2) + assert_allclose(W1, W2) + assert_allclose(H1, H2) -def test_nmf_sparse_transform(): +@pytest.mark.parametrize( + ["Estimator", "solver"], + [[NMF, {"solver": "cd"}], [NMF, {"solver": "mu"}], [MiniBatchNMF, {}]], +) +@pytest.mark.parametrize("csc_container", CSC_CONTAINERS) +def test_nmf_sparse_transform(Estimator, solver, csc_container): # Test that transform works on sparse data. Issue #2124 rng = np.random.mtrand.RandomState(42) A = np.abs(rng.randn(3, 2)) A[1, 1] = 0 - A = csc_matrix(A) + A = csc_container(A) - for solver in ('cd', 'mu'): - model = NMF(solver=solver, random_state=0, n_components=2, - max_iter=400, init='nndsvd') - A_fit_tr = model.fit_transform(A) - A_tr = model.transform(A) - assert_array_almost_equal(A_fit_tr, A_tr, decimal=1) + model = Estimator(random_state=0, n_components=2, max_iter=400, **solver) + A_fit_tr = model.fit_transform(A) + A_tr = model.transform(A) + assert_allclose(A_fit_tr, A_tr, atol=1e-1) -@pytest.mark.parametrize('init', ['random', 'nndsvd']) -@pytest.mark.parametrize('solver', ('cd', 'mu')) -@pytest.mark.parametrize('regularization', - (None, 'both', 'components', 'transformation')) -def test_non_negative_factorization_consistency(init, solver, regularization): +@pytest.mark.parametrize("init", ["random", "nndsvd"]) +@pytest.mark.parametrize("solver", ("cd", "mu")) +@pytest.mark.parametrize("alpha_W", (0.0, 1.0)) +@pytest.mark.parametrize("alpha_H", (0.0, 1.0, "same")) +def test_non_negative_factorization_consistency(init, solver, alpha_W, alpha_H): # Test that the function is called in the same way, either directly # or through the NMF class + max_iter = 500 rng = np.random.mtrand.RandomState(42) A = np.abs(rng.randn(10, 10)) A[:, 2 * np.arange(5)] = 0 W_nmf, H, _ = non_negative_factorization( - A, init=init, solver=solver, - regularization=regularization, random_state=1, tol=1e-2) - W_nmf_2, _, _ = non_negative_factorization( - A, H=H, update_H=False, init=init, solver=solver, - regularization=regularization, random_state=1, tol=1e-2) - - model_class = NMF(init=init, solver=solver, - regularization=regularization, - random_state=1, tol=1e-2) + A, + init=init, + solver=solver, + max_iter=max_iter, + alpha_W=alpha_W, + alpha_H=alpha_H, + random_state=1, + tol=1e-2, + ) + W_nmf_2, H, _ = non_negative_factorization( + A, + H=H, + update_H=False, + init=init, + solver=solver, + max_iter=max_iter, + alpha_W=alpha_W, + alpha_H=alpha_H, + random_state=1, + tol=1e-2, + ) + + model_class = NMF( + init=init, + solver=solver, + max_iter=max_iter, + alpha_W=alpha_W, + alpha_H=alpha_H, + random_state=1, + tol=1e-2, + ) W_cls = model_class.fit_transform(A) W_cls_2 = model_class.transform(A) - assert_array_almost_equal(W_nmf, W_cls, decimal=10) - assert_array_almost_equal(W_nmf_2, W_cls_2, decimal=10) + assert_allclose(W_nmf, W_cls) + assert_allclose(W_nmf_2, W_cls_2) def test_non_negative_factorization_checking(): + # Note that the validity of parameter types and range of possible values + # for scalar numerical or str parameters is already checked in the common + # tests. Here we only check for problems that cannot be captured by simple + # declarative constraints on the valid parameter values. + A = np.ones((2, 2)) - # Test parameters checking is public function + # Test parameters checking in public function nnmf = non_negative_factorization - msg = ("Number of components must be a positive integer; " - "got (n_components=1.5)") - assert_raise_message(ValueError, msg, nnmf, A, A, A, 1.5, init='random') - msg = ("Number of components must be a positive integer; " - "got (n_components='2')") - assert_raise_message(ValueError, msg, nnmf, A, A, A, '2', init='random') - msg = "Negative values in data passed to NMF (input H)" - assert_raise_message(ValueError, msg, nnmf, A, A, -A, 2, init='custom') - msg = "Negative values in data passed to NMF (input W)" - assert_raise_message(ValueError, msg, nnmf, A, -A, A, 2, init='custom') - msg = "Array passed to NMF (input H) is full of zeros" - assert_raise_message(ValueError, msg, nnmf, A, A, 0 * A, 2, init='custom') - msg = "Invalid regularization parameter: got 'spam' instead of one of" - assert_raise_message(ValueError, msg, nnmf, A, A, 0 * A, 2, init='custom', - regularization='spam') + msg = re.escape("Negative values in data passed to NMF (input H)") + with pytest.raises(ValueError, match=msg): + nnmf(A, A, -A, 2, init="custom") + msg = re.escape("Negative values in data passed to NMF (input W)") + with pytest.raises(ValueError, match=msg): + nnmf(A, -A, A, 2, init="custom") + msg = re.escape("Array passed to NMF (input H) is full of zeros") + with pytest.raises(ValueError, match=msg): + nnmf(A, A, 0 * A, 2, init="custom") def _beta_divergence_dense(X, W, H, beta): @@ -296,27 +437,28 @@ def _beta_divergence_dense(X, W, H, beta): div = X_nonzero / WH_Xnonzero res = np.sum(div) - X.size - np.sum(np.log(div)) else: - res = (X_nonzero ** beta).sum() - res += (beta - 1) * (WH ** beta).sum() + res = (X_nonzero**beta).sum() + res += (beta - 1) * (WH**beta).sum() res -= beta * (X_nonzero * (WH_Xnonzero ** (beta - 1))).sum() res /= beta * (beta - 1) return res -def test_beta_divergence(): +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_beta_divergence(csr_container): # Compare _beta_divergence with the reference _beta_divergence_dense n_samples = 20 n_features = 10 n_components = 5 - beta_losses = [0., 0.5, 1., 1.5, 2.] + beta_losses = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0] # initialization rng = np.random.mtrand.RandomState(42) X = rng.randn(n_samples, n_features) np.clip(X, 0, None, out=X) - X_csr = sp.csr_matrix(X) - W, H = nmf._initialize_nmf(X, n_components, init='random', random_state=42) + X_csr = csr_container(X) + W, H = nmf._initialize_nmf(X, n_components, init="random", random_state=42) for beta in beta_losses: ref = _beta_divergence_dense(X, W, H, beta) @@ -327,7 +469,8 @@ def test_beta_divergence(): assert_almost_equal(ref, loss_csr, decimal=7) -def test_special_sparse_dot(): +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_special_sparse_dot(csr_container): # Test the function that computes np.dot(W, H), only where X is non zero. n_samples = 10 n_features = 5 @@ -335,7 +478,7 @@ def test_special_sparse_dot(): rng = np.random.mtrand.RandomState(42) X = rng.randn(n_samples, n_features) np.clip(X, 0, None, out=X) - X_csr = sp.csr_matrix(X) + X_csr = csr_container(X) W = np.abs(rng.randn(n_samples, n_components)) H = np.abs(rng.randn(n_components, n_features)) @@ -354,8 +497,9 @@ def test_special_sparse_dot(): assert_array_equal(WH_safe.shape, X_csr.shape) -@ignore_warnings(category=ConvergenceWarning) -def test_nmf_multiplicative_update_sparse(): +@pytest.mark.filterwarnings("ignore::sklearn.exceptions.ConvergenceWarning") +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_nmf_multiplicative_update_sparse(csr_container): # Compare sparse and dense input in multiplicative update NMF # Also test continuity of the results with respect to beta_loss parameter n_samples = 20 @@ -369,42 +513,72 @@ def test_nmf_multiplicative_update_sparse(): rng = np.random.mtrand.RandomState(1337) X = rng.randn(n_samples, n_features) X = np.abs(X) - X_csr = sp.csr_matrix(X) - W0, H0 = nmf._initialize_nmf(X, n_components, init='random', - random_state=42) + X_csr = csr_container(X) + W0, H0 = nmf._initialize_nmf(X, n_components, init="random", random_state=42) - for beta_loss in (-1.2, 0, 0.2, 1., 2., 2.5): + for beta_loss in (-1.2, 0, 0.2, 1.0, 2.0, 2.5): # Reference with dense array X W, H = W0.copy(), H0.copy() W1, H1, _ = non_negative_factorization( - X, W, H, n_components, init='custom', update_H=True, - solver='mu', beta_loss=beta_loss, max_iter=n_iter, alpha=alpha, - l1_ratio=l1_ratio, regularization='both', random_state=42) + X, + W, + H, + n_components, + init="custom", + update_H=True, + solver="mu", + beta_loss=beta_loss, + max_iter=n_iter, + alpha_W=alpha, + l1_ratio=l1_ratio, + random_state=42, + ) # Compare with sparse X W, H = W0.copy(), H0.copy() W2, H2, _ = non_negative_factorization( - X_csr, W, H, n_components, init='custom', update_H=True, - solver='mu', beta_loss=beta_loss, max_iter=n_iter, alpha=alpha, - l1_ratio=l1_ratio, regularization='both', random_state=42) - - assert_array_almost_equal(W1, W2, decimal=7) - assert_array_almost_equal(H1, H2, decimal=7) + X_csr, + W, + H, + n_components, + init="custom", + update_H=True, + solver="mu", + beta_loss=beta_loss, + max_iter=n_iter, + alpha_W=alpha, + l1_ratio=l1_ratio, + random_state=42, + ) + + assert_allclose(W1, W2, atol=1e-7) + assert_allclose(H1, H2, atol=1e-7) # Compare with almost same beta_loss, since some values have a specific # behavior, but the results should be continuous w.r.t beta_loss - beta_loss -= 1.e-5 + beta_loss -= 1.0e-5 W, H = W0.copy(), H0.copy() W3, H3, _ = non_negative_factorization( - X_csr, W, H, n_components, init='custom', update_H=True, - solver='mu', beta_loss=beta_loss, max_iter=n_iter, alpha=alpha, - l1_ratio=l1_ratio, regularization='both', random_state=42) - - assert_array_almost_equal(W1, W3, decimal=4) - assert_array_almost_equal(H1, H3, decimal=4) - - -def test_nmf_negative_beta_loss(): + X_csr, + W, + H, + n_components, + init="custom", + update_H=True, + solver="mu", + beta_loss=beta_loss, + max_iter=n_iter, + alpha_W=alpha, + l1_ratio=l1_ratio, + random_state=42, + ) + + assert_allclose(W1, W3, atol=1e-4) + assert_allclose(H1, H3, atol=1e-4) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_nmf_negative_beta_loss(csr_container): # Test that an error is raised if beta_loss < 0 and X contains zeros. # Test that the output has not NaN values when the input contains zeros. n_samples = 6 @@ -414,26 +588,51 @@ def test_nmf_negative_beta_loss(): rng = np.random.mtrand.RandomState(42) X = rng.randn(n_samples, n_features) np.clip(X, 0, None, out=X) - X_csr = sp.csr_matrix(X) + X_csr = csr_container(X) def _assert_nmf_no_nan(X, beta_loss): W, H, _ = non_negative_factorization( - X, init='random', n_components=n_components, solver='mu', - beta_loss=beta_loss, random_state=0, max_iter=1000) + X, + init="random", + n_components=n_components, + solver="mu", + beta_loss=beta_loss, + random_state=0, + max_iter=1000, + ) assert not np.any(np.isnan(W)) assert not np.any(np.isnan(H)) msg = "When beta_loss <= 0 and X contains zeros, the solver may diverge." - for beta_loss in (-0.6, 0.): - assert_raise_message(ValueError, msg, _assert_nmf_no_nan, X, beta_loss) + for beta_loss in (-0.6, 0.0): + with pytest.raises(ValueError, match=msg): + _assert_nmf_no_nan(X, beta_loss) _assert_nmf_no_nan(X + 1e-9, beta_loss) - for beta_loss in (0.2, 1., 1.2, 2., 2.5): + for beta_loss in (0.2, 1.0, 1.2, 2.0, 2.5): _assert_nmf_no_nan(X, beta_loss) _assert_nmf_no_nan(X_csr, beta_loss) -def test_nmf_regularization(): +@pytest.mark.parametrize("beta_loss", [-0.5, 0.0]) +def test_minibatch_nmf_negative_beta_loss(beta_loss): + """Check that an error is raised if beta_loss < 0 and X contains zeros.""" + rng = np.random.RandomState(0) + X = rng.normal(size=(6, 5)) + X[X < 0] = 0 + + nmf = MiniBatchNMF(beta_loss=beta_loss, random_state=0) + + msg = "When beta_loss <= 0 and X contains zeros, the solver may diverge." + with pytest.raises(ValueError, match=msg): + nmf.fit(X) + + +@pytest.mark.parametrize( + ["Estimator", "solver"], + [[NMF, {"solver": "cd"}], [NMF, {"solver": "mu"}], [MiniBatchNMF, {}]], +) +def test_nmf_regularization(Estimator, solver): # Test the effect of L1 and L2 regularizations n_samples = 6 n_features = 5 @@ -441,88 +640,119 @@ def test_nmf_regularization(): rng = np.random.mtrand.RandomState(42) X = np.abs(rng.randn(n_samples, n_features)) - # FIXME : should be removed in 1.1 - init = 'nndsvda' # L1 regularization should increase the number of zeros - l1_ratio = 1. - for solver in ['cd', 'mu']: - regul = nmf.NMF(n_components=n_components, solver=solver, - alpha=0.5, l1_ratio=l1_ratio, random_state=42, - init=init) - model = nmf.NMF(n_components=n_components, solver=solver, - alpha=0., l1_ratio=l1_ratio, random_state=42, - init=init) - - W_regul = regul.fit_transform(X) - W_model = model.fit_transform(X) - - H_regul = regul.components_ - H_model = model.components_ - - W_regul_n_zeros = W_regul[W_regul == 0].size - W_model_n_zeros = W_model[W_model == 0].size - H_regul_n_zeros = H_regul[H_regul == 0].size - H_model_n_zeros = H_model[H_model == 0].size - - assert W_regul_n_zeros > W_model_n_zeros - assert H_regul_n_zeros > H_model_n_zeros - - # L2 regularization should decrease the mean of the coefficients - l1_ratio = 0. - for solver in ['cd', 'mu']: - regul = nmf.NMF(n_components=n_components, solver=solver, - alpha=0.5, l1_ratio=l1_ratio, random_state=42, - init=init) - model = nmf.NMF(n_components=n_components, solver=solver, - alpha=0., l1_ratio=l1_ratio, random_state=42, - init=init) - - W_regul = regul.fit_transform(X) - W_model = model.fit_transform(X) - - H_regul = regul.components_ - H_model = model.components_ - - assert (linalg.norm(W_model))**2. + (linalg.norm(H_model))**2. > \ - (linalg.norm(W_regul))**2. + (linalg.norm(H_regul))**2. - - -@ignore_warnings(category=ConvergenceWarning) -def test_nmf_decreasing(): + l1_ratio = 1.0 + regul = Estimator( + n_components=n_components, + alpha_W=0.5, + l1_ratio=l1_ratio, + random_state=42, + **solver, + ) + model = Estimator( + n_components=n_components, + alpha_W=0.0, + l1_ratio=l1_ratio, + random_state=42, + **solver, + ) + + W_regul = regul.fit_transform(X) + W_model = model.fit_transform(X) + + H_regul = regul.components_ + H_model = model.components_ + + eps = np.finfo(np.float64).eps + W_regul_n_zeros = W_regul[W_regul <= eps].size + W_model_n_zeros = W_model[W_model <= eps].size + H_regul_n_zeros = H_regul[H_regul <= eps].size + H_model_n_zeros = H_model[H_model <= eps].size + + assert W_regul_n_zeros > W_model_n_zeros + assert H_regul_n_zeros > H_model_n_zeros + + # L2 regularization should decrease the sum of the squared norm + # of the matrices W and H + l1_ratio = 0.0 + regul = Estimator( + n_components=n_components, + alpha_W=0.5, + l1_ratio=l1_ratio, + random_state=42, + **solver, + ) + model = Estimator( + n_components=n_components, + alpha_W=0.0, + l1_ratio=l1_ratio, + random_state=42, + **solver, + ) + + W_regul = regul.fit_transform(X) + W_model = model.fit_transform(X) + + H_regul = regul.components_ + H_model = model.components_ + + assert (linalg.norm(W_model)) ** 2.0 + (linalg.norm(H_model)) ** 2.0 > ( + linalg.norm(W_regul) + ) ** 2.0 + (linalg.norm(H_regul)) ** 2.0 + + +@pytest.mark.filterwarnings("ignore::sklearn.exceptions.ConvergenceWarning") +@pytest.mark.parametrize("solver", ("cd", "mu")) +def test_nmf_decreasing(solver): # test that the objective function is decreasing at each iteration n_samples = 20 n_features = 15 n_components = 10 alpha = 0.1 l1_ratio = 0.5 - tol = 0. + tol = 0.0 # initialization rng = np.random.mtrand.RandomState(42) X = rng.randn(n_samples, n_features) np.abs(X, X) - W0, H0 = nmf._initialize_nmf(X, n_components, init='random', - random_state=42) - - for beta_loss in (-1.2, 0, 0.2, 1., 2., 2.5): - for solver in ('cd', 'mu'): - if solver != 'mu' and beta_loss != 2: - # not implemented - continue - W, H = W0.copy(), H0.copy() - previous_loss = None - for _ in range(30): - # one more iteration starting from the previous results - W, H, _ = non_negative_factorization( - X, W, H, beta_loss=beta_loss, init='custom', - n_components=n_components, max_iter=1, alpha=alpha, - solver=solver, tol=tol, l1_ratio=l1_ratio, verbose=0, - regularization='both', random_state=0, update_H=True) - - loss = nmf._beta_divergence(X, W, H, beta_loss) - if previous_loss is not None: - assert previous_loss > loss - previous_loss = loss + W0, H0 = nmf._initialize_nmf(X, n_components, init="random", random_state=42) + + for beta_loss in (-1.2, 0, 0.2, 1.0, 2.0, 2.5): + if solver != "mu" and beta_loss != 2: + # not implemented + continue + W, H = W0.copy(), H0.copy() + previous_loss = None + for _ in range(30): + # one more iteration starting from the previous results + W, H, _ = non_negative_factorization( + X, + W, + H, + beta_loss=beta_loss, + init="custom", + n_components=n_components, + max_iter=1, + alpha_W=alpha, + solver=solver, + tol=tol, + l1_ratio=l1_ratio, + verbose=0, + random_state=0, + update_H=True, + ) + + loss = ( + nmf._beta_divergence(X, W, H, beta_loss) + + alpha * l1_ratio * n_features * W.sum() + + alpha * l1_ratio * n_samples * H.sum() + + alpha * (1 - l1_ratio) * n_features * (W**2).sum() + + alpha * (1 - l1_ratio) * n_samples * (H**2).sum() + ) + if previous_loss is not None: + assert previous_loss > loss + previous_loss = loss def test_nmf_underflow(): @@ -540,47 +770,55 @@ def test_nmf_underflow(): assert_almost_equal(res, ref) -@pytest.mark.parametrize("dtype_in, dtype_out", [ - (np.float32, np.float32), - (np.float64, np.float64), - (np.int32, np.float64), - (np.int64, np.float64)]) -@pytest.mark.parametrize("solver", ["cd", "mu"]) -@pytest.mark.parametrize("regularization", - (None, "both", "components", "transformation")) -def test_nmf_dtype_match(dtype_in, dtype_out, solver, regularization): +@pytest.mark.parametrize( + "dtype_in, dtype_out", + [ + (np.float32, np.float32), + (np.float64, np.float64), + (np.int32, np.float64), + (np.int64, np.float64), + ], +) +@pytest.mark.parametrize( + ["Estimator", "solver"], + [[NMF, {"solver": "cd"}], [NMF, {"solver": "mu"}], [MiniBatchNMF, {}]], +) +def test_nmf_dtype_match(Estimator, solver, dtype_in, dtype_out): # Check that NMF preserves dtype (float32 and float64) X = np.random.RandomState(0).randn(20, 15).astype(dtype_in, copy=False) np.abs(X, out=X) - # FIXME : should be removed in 1.1 - init = 'nndsvda' - nmf = NMF(solver=solver, regularization=regularization, init=init) + + nmf = Estimator( + alpha_W=1.0, + alpha_H=1.0, + tol=1e-2, + random_state=0, + **solver, + ) assert nmf.fit(X).transform(X).dtype == dtype_out assert nmf.fit_transform(X).dtype == dtype_out assert nmf.components_.dtype == dtype_out -@pytest.mark.parametrize("solver", ["cd", "mu"]) -@pytest.mark.parametrize("regularization", - (None, "both", "components", "transformation")) -def test_nmf_float32_float64_consistency(solver, regularization): +@pytest.mark.parametrize( + ["Estimator", "solver"], + [[NMF, {"solver": "cd"}], [NMF, {"solver": "mu"}], [MiniBatchNMF, {}]], +) +def test_nmf_float32_float64_consistency(Estimator, solver): # Check that the result of NMF is the same between float32 and float64 X = np.random.RandomState(0).randn(50, 7) np.abs(X, out=X) - # FIXME : should be removed in 1.1 - init = 'nndsvda' - nmf32 = NMF(solver=solver, regularization=regularization, random_state=0, - init=init) + nmf32 = Estimator(random_state=0, tol=1e-3, **solver) W32 = nmf32.fit_transform(X.astype(np.float32)) - nmf64 = NMF(solver=solver, regularization=regularization, random_state=0, - init=init) + nmf64 = Estimator(random_state=0, tol=1e-3, **solver) W64 = nmf64.fit_transform(X) - assert_allclose(W32, W64, rtol=1e-6, atol=1e-5) + assert_allclose(W32, W64, atol=1e-5) -def test_nmf_custom_init_dtype_error(): +@pytest.mark.parametrize("Estimator", [NMF, MiniBatchNMF]) +def test_nmf_custom_init_dtype_error(Estimator): # Check that an error is raise if custom H and/or W don't have the same # dtype as X. rng = np.random.RandomState(0) @@ -589,24 +827,184 @@ def test_nmf_custom_init_dtype_error(): W = rng.random_sample((20, 15)) with pytest.raises(TypeError, match="should have the same dtype as X"): - NMF(init='custom').fit(X, H=H, W=W) + Estimator(init="custom").fit(X, H=H, W=W) with pytest.raises(TypeError, match="should have the same dtype as X"): non_negative_factorization(X, H=H, update_H=False) -# FIXME : should be removed in 1.1 -def test_init_default_deprecation(): - # Test FutureWarning on init default - msg = (r"The 'init' value, when 'init=None' and " - r"n_components is less than n_samples and " - r"n_features, will be changed from 'nndsvd' to " - r"'nndsvda' in 1.1 \(renaming of 0.26\).") +@pytest.mark.parametrize("beta_loss", [-0.5, 0, 0.5, 1, 1.5, 2, 2.5]) +def test_nmf_minibatchnmf_equivalence(beta_loss): + # Test that MiniBatchNMF is equivalent to NMF when batch_size = n_samples and + # forget_factor 0.0 (stopping criterion put aside) rng = np.random.mtrand.RandomState(42) - A = np.abs(rng.randn(6, 5)) - with pytest.warns(FutureWarning, match=msg): - nmf._initialize_nmf(A, 3) - with pytest.warns(FutureWarning, match=msg): - NMF().fit(A) - with pytest.warns(FutureWarning, match=msg): - non_negative_factorization(A) + X = np.abs(rng.randn(48, 5)) + + nmf = NMF( + n_components=5, + beta_loss=beta_loss, + solver="mu", + random_state=0, + tol=0, + ) + mbnmf = MiniBatchNMF( + n_components=5, + beta_loss=beta_loss, + random_state=0, + tol=0, + max_no_improvement=None, + batch_size=X.shape[0], + forget_factor=0.0, + ) + W = nmf.fit_transform(X) + mbW = mbnmf.fit_transform(X) + assert_allclose(W, mbW) + + +def test_minibatch_nmf_partial_fit(): + # Check fit / partial_fit equivalence. Applicable only with fresh restarts. + rng = np.random.mtrand.RandomState(42) + X = np.abs(rng.randn(100, 5)) + + n_components = 5 + batch_size = 10 + max_iter = 2 + + mbnmf1 = MiniBatchNMF( + n_components=n_components, + init="custom", + random_state=0, + max_iter=max_iter, + batch_size=batch_size, + tol=0, + max_no_improvement=None, + fresh_restarts=False, + ) + mbnmf2 = MiniBatchNMF(n_components=n_components, init="custom", random_state=0) + + # Force the same init of H (W is recomputed anyway) to be able to compare results. + W, H = nmf._initialize_nmf( + X, n_components=n_components, init="random", random_state=0 + ) + + mbnmf1.fit(X, W=W, H=H) + for i in range(max_iter): + for j in range(batch_size): + mbnmf2.partial_fit(X[j : j + batch_size], W=W[:batch_size], H=H) + + assert mbnmf1.n_steps_ == mbnmf2.n_steps_ + assert_allclose(mbnmf1.components_, mbnmf2.components_) + + +def test_feature_names_out(): + """Check feature names out for NMF.""" + random_state = np.random.RandomState(0) + X = np.abs(random_state.randn(10, 4)) + nmf = NMF(n_components=3).fit(X) + + names = nmf.get_feature_names_out() + assert_array_equal([f"nmf{i}" for i in range(3)], names) + + +def test_minibatch_nmf_verbose(): + # Check verbose mode of MiniBatchNMF for better coverage. + A = np.random.RandomState(0).random_sample((100, 10)) + nmf = MiniBatchNMF(tol=1e-2, random_state=0, verbose=1) + old_stdout = sys.stdout + sys.stdout = StringIO() + try: + nmf.fit(A) + finally: + sys.stdout = old_stdout + + +@pytest.mark.parametrize("Estimator", [NMF, MiniBatchNMF]) +def test_nmf_n_components_auto(Estimator): + # Check that n_components is correctly inferred + # from the provided custom initialization. + rng = np.random.RandomState(0) + X = rng.random_sample((6, 5)) + W = rng.random_sample((6, 2)) + H = rng.random_sample((2, 5)) + est = Estimator( + n_components="auto", + init="custom", + random_state=0, + tol=1e-6, + ) + est.fit_transform(X, W=W, H=H) + assert est._n_components == H.shape[0] + + +def test_nmf_non_negative_factorization_n_components_auto(): + # Check that n_components is correctly inferred from the provided + # custom initialization. + rng = np.random.RandomState(0) + X = rng.random_sample((6, 5)) + W_init = rng.random_sample((6, 2)) + H_init = rng.random_sample((2, 5)) + W, H, _ = non_negative_factorization( + X, W=W_init, H=H_init, init="custom", n_components="auto" + ) + assert H.shape == H_init.shape + assert W.shape == W_init.shape + + +def test_nmf_n_components_auto_no_h_update(): + # Tests that non_negative_factorization does not fail when setting + # n_components="auto" also tests that the inferred n_component + # value is the right one. + rng = np.random.RandomState(0) + X = rng.random_sample((6, 5)) + H_true = rng.random_sample((2, 5)) + W, H, _ = non_negative_factorization( + X, H=H_true, n_components="auto", update_H=False + ) # should not fail + assert_allclose(H, H_true) + assert W.shape == (X.shape[0], H_true.shape[0]) + + +def test_nmf_w_h_not_used_warning(): + # Check that warnings are raised if user provided W and H are not used + # and initialization overrides value of W or H + rng = np.random.RandomState(0) + X = rng.random_sample((6, 5)) + W_init = rng.random_sample((6, 2)) + H_init = rng.random_sample((2, 5)) + with pytest.warns( + RuntimeWarning, + match="When init!='custom', provided W or H are ignored", + ): + non_negative_factorization(X, H=H_init, update_H=True, n_components="auto") + + with pytest.warns( + RuntimeWarning, + match="When init!='custom', provided W or H are ignored", + ): + non_negative_factorization( + X, W=W_init, H=H_init, update_H=True, n_components="auto" + ) + + with pytest.warns( + RuntimeWarning, match="When update_H=False, the provided initial W is not used." + ): + # When update_H is False, W is ignored regardless of init + # TODO: use the provided W when init="custom". + non_negative_factorization( + X, W=W_init, H=H_init, update_H=False, n_components="auto" + ) + + +def test_nmf_custom_init_shape_error(): + # Check that an informative error is raised when custom initialization does not + # have the right shape + rng = np.random.RandomState(0) + X = rng.random_sample((6, 5)) + H = rng.random_sample((2, 5)) + nmf = NMF(n_components=2, init="custom", random_state=0) + + with pytest.raises(ValueError, match="Array with wrong first dimension passed"): + nmf.fit(X, H=H, W=rng.random_sample((5, 2))) + + with pytest.raises(ValueError, match="Array with wrong second dimension passed"): + nmf.fit(X, H=H, W=rng.random_sample((6, 3))) diff --git a/sklearn/decomposition/tests/test_online_lda.py b/sklearn/decomposition/tests/test_online_lda.py index 3d64c9889a881..c3dafa1912eba 100644 --- a/sklearn/decomposition/tests/test_online_lda.py +++ b/sklearn/decomposition/tests/test_online_lda.py @@ -1,58 +1,67 @@ import sys +from io import StringIO import numpy as np +import pytest +from numpy.testing import assert_array_equal from scipy.linalg import block_diag -from scipy.sparse import csr_matrix from scipy.special import psi -import pytest - from sklearn.decomposition import LatentDirichletAllocation -from sklearn.decomposition._lda import (_dirichlet_expectation_1d, - _dirichlet_expectation_2d) - -from sklearn.utils._testing import assert_allclose -from sklearn.utils._testing import assert_array_almost_equal -from sklearn.utils._testing import assert_almost_equal -from sklearn.utils._testing import if_safe_multiprocessing_with_blas - +from sklearn.decomposition._online_lda_fast import ( + _dirichlet_expectation_1d, + _dirichlet_expectation_2d, +) from sklearn.exceptions import NotFittedError -from io import StringIO +from sklearn.utils._testing import ( + assert_allclose, + assert_almost_equal, + assert_array_almost_equal, + if_safe_multiprocessing_with_blas, +) +from sklearn.utils.fixes import CSR_CONTAINERS -def _build_sparse_mtx(): +def _build_sparse_array(csr_container): # Create 3 topics and each topic has 3 distinct words. # (Each word only belongs to a single topic.) n_components = 3 block = np.full((3, 3), n_components, dtype=int) blocks = [block] * n_components X = block_diag(*blocks) - X = csr_matrix(X) + X = csr_container(X) return (n_components, X) -def test_lda_default_prior_params(): +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_lda_default_prior_params(csr_container): # default prior parameter should be `1 / topics` # and verbose params should not affect result - n_components, X = _build_sparse_mtx() - prior = 1. / n_components - lda_1 = LatentDirichletAllocation(n_components=n_components, - doc_topic_prior=prior, - topic_word_prior=prior, random_state=0) - lda_2 = LatentDirichletAllocation(n_components=n_components, - random_state=0) + n_components, X = _build_sparse_array(csr_container) + prior = 1.0 / n_components + lda_1 = LatentDirichletAllocation( + n_components=n_components, + doc_topic_prior=prior, + topic_word_prior=prior, + random_state=0, + ) + lda_2 = LatentDirichletAllocation(n_components=n_components, random_state=0) topic_distr_1 = lda_1.fit_transform(X) topic_distr_2 = lda_2.fit_transform(X) assert_almost_equal(topic_distr_1, topic_distr_2) -def test_lda_fit_batch(): +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_lda_fit_batch(csr_container): # Test LDA batch learning_offset (`fit` method with 'batch' learning) rng = np.random.RandomState(0) - n_components, X = _build_sparse_mtx() - lda = LatentDirichletAllocation(n_components=n_components, - evaluate_every=1, learning_method='batch', - random_state=rng) + n_components, X = _build_sparse_array(csr_container) + lda = LatentDirichletAllocation( + n_components=n_components, + evaluate_every=1, + learning_method="batch", + random_state=rng, + ) lda.fit(X) correct_idx_grps = [(0, 1, 2), (3, 4, 5), (6, 7, 8)] @@ -62,13 +71,18 @@ def test_lda_fit_batch(): assert tuple(sorted(top_idx)) in correct_idx_grps -def test_lda_fit_online(): +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_lda_fit_online(csr_container): # Test LDA online learning (`fit` method with 'online' learning) rng = np.random.RandomState(0) - n_components, X = _build_sparse_mtx() - lda = LatentDirichletAllocation(n_components=n_components, - learning_offset=10., evaluate_every=1, - learning_method='online', random_state=rng) + n_components, X = _build_sparse_array(csr_container) + lda = LatentDirichletAllocation( + n_components=n_components, + learning_offset=10.0, + evaluate_every=1, + learning_method="online", + random_state=rng, + ) lda.fit(X) correct_idx_grps = [(0, 1, 2), (3, 4, 5), (6, 7, 8)] @@ -78,14 +92,18 @@ def test_lda_fit_online(): assert tuple(sorted(top_idx)) in correct_idx_grps -def test_lda_partial_fit(): +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_lda_partial_fit(csr_container): # Test LDA online learning (`partial_fit` method) # (same as test_lda_batch) rng = np.random.RandomState(0) - n_components, X = _build_sparse_mtx() - lda = LatentDirichletAllocation(n_components=n_components, - learning_offset=10., total_samples=100, - random_state=rng) + n_components, X = _build_sparse_array(csr_container) + lda = LatentDirichletAllocation( + n_components=n_components, + learning_offset=10.0, + total_samples=100, + random_state=rng, + ) for i in range(3): lda.partial_fit(X) @@ -95,12 +113,14 @@ def test_lda_partial_fit(): assert tuple(sorted(top_idx)) in correct_idx_grps -def test_lda_dense_input(): +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_lda_dense_input(csr_container): # Test LDA with dense input. rng = np.random.RandomState(0) - n_components, X = _build_sparse_mtx() - lda = LatentDirichletAllocation(n_components=n_components, - learning_method='batch', random_state=rng) + n_components, X = _build_sparse_array(csr_container) + lda = LatentDirichletAllocation( + n_components=n_components, learning_method="batch", random_state=rng + ) lda.fit(X.toarray()) correct_idx_grps = [(0, 1, 2), (3, 4, 5), (6, 7, 8)] @@ -112,51 +132,38 @@ def test_lda_dense_input(): def test_lda_transform(): # Test LDA transform. - # Transform result cannot be negative and should be normalized + # Transform result cannot be negative and should be normalized by default rng = np.random.RandomState(0) X = rng.randint(5, size=(20, 10)) n_components = 3 - lda = LatentDirichletAllocation(n_components=n_components, - random_state=rng) + lda = LatentDirichletAllocation(n_components=n_components, random_state=rng) X_trans = lda.fit_transform(X) assert (X_trans > 0.0).any() - assert_array_almost_equal(np.sum(X_trans, axis=1), - np.ones(X_trans.shape[0])) + assert_array_almost_equal(np.sum(X_trans, axis=1), np.ones(X_trans.shape[0])) + + X_trans_unnormalized = lda.transform(X, normalize=False) + assert_array_almost_equal( + X_trans, X_trans_unnormalized / X_trans_unnormalized.sum(axis=1)[:, np.newaxis] + ) -@pytest.mark.parametrize('method', ('online', 'batch')) +@pytest.mark.parametrize("method", ("online", "batch")) def test_lda_fit_transform(method): # Test LDA fit_transform & transform # fit_transform and transform result should be the same rng = np.random.RandomState(0) X = rng.randint(10, size=(50, 20)) - lda = LatentDirichletAllocation(n_components=5, learning_method=method, - random_state=rng) + lda = LatentDirichletAllocation( + n_components=5, learning_method=method, random_state=rng + ) X_fit = lda.fit_transform(X) X_trans = lda.transform(X) assert_array_almost_equal(X_fit, X_trans, 4) -def test_invalid_params(): - # test `_check_params` method - X = np.ones((5, 10)) - - invalid_models = ( - ('n_components', LatentDirichletAllocation(n_components=0)), - ('learning_method', - LatentDirichletAllocation(learning_method='unknown')), - ('total_samples', LatentDirichletAllocation(total_samples=0)), - ('learning_offset', LatentDirichletAllocation(learning_offset=-1)), - ) - for param, model in invalid_models: - regex = r"^Invalid %r parameter" % param - with pytest.raises(ValueError, match=regex): - model.fit(X) - - def test_lda_negative_input(): # test pass dense matrix with sparse negative input. - X = np.full((5, 10), -1.) + X = np.full((5, 10), -1.0) lda = LatentDirichletAllocation() regex = r"^Negative values in data passed" with pytest.raises(ValueError, match=regex): @@ -168,22 +175,29 @@ def test_lda_no_component_error(): rng = np.random.RandomState(0) X = rng.randint(4, size=(20, 10)) lda = LatentDirichletAllocation() - regex = ("This LatentDirichletAllocation instance is not fitted yet. " - "Call 'fit' with appropriate arguments before using this " - "estimator.") + regex = ( + "This LatentDirichletAllocation instance is not fitted yet. " + "Call 'fit' with appropriate arguments before using this " + "estimator." + ) with pytest.raises(NotFittedError, match=regex): lda.perplexity(X) @if_safe_multiprocessing_with_blas -@pytest.mark.parametrize('method', ('online', 'batch')) -def test_lda_multi_jobs(method): - n_components, X = _build_sparse_mtx() +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +@pytest.mark.parametrize("method", ("online", "batch")) +def test_lda_multi_jobs(method, csr_container): + n_components, X = _build_sparse_array(csr_container) # Test LDA batch training with multi CPU rng = np.random.RandomState(0) - lda = LatentDirichletAllocation(n_components=n_components, n_jobs=2, - learning_method=method, - evaluate_every=1, random_state=rng) + lda = LatentDirichletAllocation( + n_components=n_components, + n_jobs=2, + learning_method=method, + evaluate_every=1, + random_state=rng, + ) lda.fit(X) correct_idx_grps = [(0, 1, 2), (3, 4, 5), (6, 7, 8)] @@ -193,13 +207,18 @@ def test_lda_multi_jobs(method): @if_safe_multiprocessing_with_blas -def test_lda_partial_fit_multi_jobs(): +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_lda_partial_fit_multi_jobs(csr_container): # Test LDA online training with multi CPU rng = np.random.RandomState(0) - n_components, X = _build_sparse_mtx() - lda = LatentDirichletAllocation(n_components=n_components, n_jobs=2, - learning_offset=5., total_samples=30, - random_state=rng) + n_components, X = _build_sparse_array(csr_container) + lda = LatentDirichletAllocation( + n_components=n_components, + n_jobs=2, + learning_offset=5.0, + total_samples=30, + random_state=rng, + ) for i in range(2): lda.partial_fit(X) @@ -215,31 +234,43 @@ def test_lda_preplexity_mismatch(): n_components = rng.randint(3, 6) n_samples = rng.randint(6, 10) X = np.random.randint(4, size=(n_samples, 10)) - lda = LatentDirichletAllocation(n_components=n_components, - learning_offset=5., total_samples=20, - random_state=rng) + lda = LatentDirichletAllocation( + n_components=n_components, + learning_offset=5.0, + total_samples=20, + random_state=rng, + ) lda.fit(X) # invalid samples invalid_n_samples = rng.randint(4, size=(n_samples + 1, n_components)) - with pytest.raises(ValueError, match=r'Number of samples'): + with pytest.raises(ValueError, match=r"Number of samples"): lda._perplexity_precomp_distr(X, invalid_n_samples) # invalid topic number invalid_n_components = rng.randint(4, size=(n_samples, n_components + 1)) - with pytest.raises(ValueError, match=r'Number of topics'): + with pytest.raises(ValueError, match=r"Number of topics"): lda._perplexity_precomp_distr(X, invalid_n_components) -@pytest.mark.parametrize('method', ('online', 'batch')) -def test_lda_perplexity(method): +@pytest.mark.parametrize("method", ("online", "batch")) +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_lda_perplexity(method, csr_container): # Test LDA perplexity for batch training # perplexity should be lower after each iteration - n_components, X = _build_sparse_mtx() - lda_1 = LatentDirichletAllocation(n_components=n_components, - max_iter=1, learning_method=method, - total_samples=100, random_state=0) - lda_2 = LatentDirichletAllocation(n_components=n_components, - max_iter=10, learning_method=method, - total_samples=100, random_state=0) + n_components, X = _build_sparse_array(csr_container) + lda_1 = LatentDirichletAllocation( + n_components=n_components, + max_iter=1, + learning_method=method, + total_samples=100, + random_state=0, + ) + lda_2 = LatentDirichletAllocation( + n_components=n_components, + max_iter=10, + learning_method=method, + total_samples=100, + random_state=0, + ) lda_1.fit(X) perp_1 = lda_1.perplexity(X, sub_sampling=False) @@ -252,17 +283,26 @@ def test_lda_perplexity(method): assert perp_1_subsampling >= perp_2_subsampling -@pytest.mark.parametrize('method', ('online', 'batch')) -def test_lda_score(method): +@pytest.mark.parametrize("method", ("online", "batch")) +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_lda_score(method, csr_container): # Test LDA score for batch training # score should be higher after each iteration - n_components, X = _build_sparse_mtx() - lda_1 = LatentDirichletAllocation(n_components=n_components, - max_iter=1, learning_method=method, - total_samples=100, random_state=0) - lda_2 = LatentDirichletAllocation(n_components=n_components, - max_iter=10, learning_method=method, - total_samples=100, random_state=0) + n_components, X = _build_sparse_array(csr_container) + lda_1 = LatentDirichletAllocation( + n_components=n_components, + max_iter=1, + learning_method=method, + total_samples=100, + random_state=0, + ) + lda_2 = LatentDirichletAllocation( + n_components=n_components, + max_iter=10, + learning_method=method, + total_samples=100, + random_state=0, + ) lda_1.fit_transform(X) score_1 = lda_1.score(X) @@ -271,39 +311,51 @@ def test_lda_score(method): assert score_2 >= score_1 -def test_perplexity_input_format(): +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_perplexity_input_format(csr_container): # Test LDA perplexity for sparse and dense input # score should be the same for both dense and sparse input - n_components, X = _build_sparse_mtx() - lda = LatentDirichletAllocation(n_components=n_components, max_iter=1, - learning_method='batch', - total_samples=100, random_state=0) + n_components, X = _build_sparse_array(csr_container) + lda = LatentDirichletAllocation( + n_components=n_components, + max_iter=1, + learning_method="batch", + total_samples=100, + random_state=0, + ) lda.fit(X) perp_1 = lda.perplexity(X) perp_2 = lda.perplexity(X.toarray()) assert_almost_equal(perp_1, perp_2) -def test_lda_score_perplexity(): +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_lda_score_perplexity(csr_container): # Test the relationship between LDA score and perplexity - n_components, X = _build_sparse_mtx() - lda = LatentDirichletAllocation(n_components=n_components, max_iter=10, - random_state=0) + n_components, X = _build_sparse_array(csr_container) + lda = LatentDirichletAllocation( + n_components=n_components, max_iter=10, random_state=0 + ) lda.fit(X) perplexity_1 = lda.perplexity(X, sub_sampling=False) score = lda.score(X) - perplexity_2 = np.exp(-1. * (score / np.sum(X.data))) + perplexity_2 = np.exp(-1.0 * (score / np.sum(X.data))) assert_almost_equal(perplexity_1, perplexity_2) -def test_lda_fit_perplexity(): +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_lda_fit_perplexity(csr_container): # Test that the perplexity computed during fit is consistent with what is # returned by the perplexity method - n_components, X = _build_sparse_mtx() - lda = LatentDirichletAllocation(n_components=n_components, max_iter=1, - learning_method='batch', random_state=0, - evaluate_every=1) + n_components, X = _build_sparse_array(csr_container) + lda = LatentDirichletAllocation( + n_components=n_components, + max_iter=1, + learning_method="batch", + random_state=0, + evaluate_every=1, + ) lda.fit(X) # Perplexity computed at end of fit method @@ -315,13 +367,15 @@ def test_lda_fit_perplexity(): assert_almost_equal(perplexity1, perplexity2) -def test_lda_empty_docs(): +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_lda_empty_docs(csr_container): """Test LDA on empty document (all-zero rows).""" Z = np.zeros((5, 4)) - for X in [Z, csr_matrix(Z)]: + for X in [Z, csr_container(Z)]: lda = LatentDirichletAllocation(max_iter=750).fit(X) - assert_almost_equal(lda.components_.sum(axis=0), - np.ones(lda.components_.shape[1])) + assert_almost_equal( + lda.components_.sum(axis=0), np.ones(lda.components_.shape[1]) + ) def test_dirichlet_expectation(): @@ -329,23 +383,29 @@ def test_dirichlet_expectation(): x = np.logspace(-100, 10, 10000) expectation = np.empty_like(x) _dirichlet_expectation_1d(x, 0, expectation) - assert_allclose(expectation, np.exp(psi(x) - psi(np.sum(x))), - atol=1e-19) + assert_allclose(expectation, np.exp(psi(x) - psi(np.sum(x))), atol=1e-19) x = x.reshape(100, 100) - assert_allclose(_dirichlet_expectation_2d(x), - psi(x) - psi(np.sum(x, axis=1)[:, np.newaxis]), - rtol=1e-11, atol=3e-9) - - -def check_verbosity(verbose, evaluate_every, expected_lines, - expected_perplexities): - n_components, X = _build_sparse_mtx() - lda = LatentDirichletAllocation(n_components=n_components, max_iter=3, - learning_method='batch', - verbose=verbose, - evaluate_every=evaluate_every, - random_state=0) + assert_allclose( + _dirichlet_expectation_2d(x), + psi(x) - psi(np.sum(x, axis=1)[:, np.newaxis]), + rtol=1e-11, + atol=3e-9, + ) + + +def check_verbosity( + verbose, evaluate_every, expected_lines, expected_perplexities, csr_container +): + n_components, X = _build_sparse_array(csr_container) + lda = LatentDirichletAllocation( + n_components=n_components, + max_iter=3, + learning_method="batch", + verbose=verbose, + evaluate_every=evaluate_every, + random_state=0, + ) out = StringIO() old_out, sys.stdout = sys.stdout, out try: @@ -353,20 +413,70 @@ def check_verbosity(verbose, evaluate_every, expected_lines, finally: sys.stdout = old_out - n_lines = out.getvalue().count('\n') - n_perplexity = out.getvalue().count('perplexity') + n_lines = out.getvalue().count("\n") + n_perplexity = out.getvalue().count("perplexity") assert expected_lines == n_lines assert expected_perplexities == n_perplexity @pytest.mark.parametrize( - 'verbose,evaluate_every,expected_lines,expected_perplexities', - [(False, 1, 0, 0), - (False, 0, 0, 0), - (True, 0, 3, 0), - (True, 1, 3, 3), - (True, 2, 3, 1)]) -def test_verbosity(verbose, evaluate_every, expected_lines, - expected_perplexities): - check_verbosity(verbose, evaluate_every, expected_lines, - expected_perplexities) + "verbose,evaluate_every,expected_lines,expected_perplexities", + [ + (False, 1, 0, 0), + (False, 0, 0, 0), + (True, 0, 3, 0), + (True, 1, 3, 3), + (True, 2, 3, 1), + ], +) +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_verbosity( + verbose, evaluate_every, expected_lines, expected_perplexities, csr_container +): + check_verbosity( + verbose, evaluate_every, expected_lines, expected_perplexities, csr_container + ) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_lda_feature_names_out(csr_container): + """Check feature names out for LatentDirichletAllocation.""" + n_components, X = _build_sparse_array(csr_container) + lda = LatentDirichletAllocation(n_components=n_components).fit(X) + + names = lda.get_feature_names_out() + assert_array_equal( + [f"latentdirichletallocation{i}" for i in range(n_components)], names + ) + + +@pytest.mark.parametrize("learning_method", ("batch", "online")) +def test_lda_dtype_match(learning_method, global_dtype): + """Check data type preservation of fitted attributes.""" + rng = np.random.RandomState(0) + X = rng.uniform(size=(20, 10)).astype(global_dtype, copy=False) + + lda = LatentDirichletAllocation( + n_components=5, random_state=0, learning_method=learning_method + ) + lda.fit(X) + assert lda.components_.dtype == global_dtype + assert lda.exp_dirichlet_component_.dtype == global_dtype + + +@pytest.mark.parametrize("learning_method", ("batch", "online")) +def test_lda_numerical_consistency(learning_method, global_random_seed): + """Check numerical consistency between np.float32 and np.float64.""" + rng = np.random.RandomState(global_random_seed) + X64 = rng.uniform(size=(20, 10)) + X32 = X64.astype(np.float32) + + lda_64 = LatentDirichletAllocation( + n_components=5, random_state=global_random_seed, learning_method=learning_method + ).fit(X64) + lda_32 = LatentDirichletAllocation( + n_components=5, random_state=global_random_seed, learning_method=learning_method + ).fit(X32) + + assert_allclose(lda_32.components_, lda_64.components_) + assert_allclose(lda_32.transform(X32), lda_64.transform(X64)) diff --git a/sklearn/decomposition/tests/test_pca.py b/sklearn/decomposition/tests/test_pca.py index 3548c91286da1..2b97138c4dea3 100644 --- a/sklearn/decomposition/tests/test_pca.py +++ b/sklearn/decomposition/tests/test_pca.py @@ -1,22 +1,57 @@ -import numpy as np -import scipy as sp +import os +import re +import warnings +import numpy as np import pytest +import scipy as sp +from numpy.testing import assert_array_equal -from sklearn.utils._testing import assert_allclose - -from sklearn import datasets +from sklearn import config_context, datasets +from sklearn.base import clone +from sklearn.datasets import load_iris, make_classification, make_low_rank_matrix from sklearn.decomposition import PCA -from sklearn.datasets import load_iris -from sklearn.decomposition._pca import _assess_dimension -from sklearn.decomposition._pca import _infer_dimension +from sklearn.decomposition._pca import _assess_dimension, _infer_dimension +from sklearn.utils._array_api import ( + _atol_for_type, + _convert_to_numpy, + _get_namespace_device_dtype_ids, + yield_namespace_device_dtype_combinations, +) +from sklearn.utils._array_api import device as array_device +from sklearn.utils._test_common.instance_generator import _get_check_estimator_ids +from sklearn.utils._testing import _array_api_for_tests, assert_allclose +from sklearn.utils.estimator_checks import ( + check_array_api_input_and_values, +) +from sklearn.utils.fixes import CSC_CONTAINERS, CSR_CONTAINERS iris = datasets.load_iris() -PCA_SOLVERS = ['full', 'arpack', 'randomized', 'auto'] +PCA_SOLVERS = ["full", "covariance_eigh", "arpack", "randomized", "auto"] + +# `SPARSE_M` and `SPARSE_N` could be larger, but be aware: +# * SciPy's generation of random sparse matrix can be costly +# * A (SPARSE_M, SPARSE_N) dense array is allocated to compare against +SPARSE_M, SPARSE_N = 1000, 300 # arbitrary +SPARSE_MAX_COMPONENTS = min(SPARSE_M, SPARSE_N) -@pytest.mark.parametrize('svd_solver', PCA_SOLVERS) -@pytest.mark.parametrize('n_components', range(1, iris.data.shape[1])) +def _check_fitted_pca_close(pca1, pca2, rtol=1e-7, atol=1e-12): + assert_allclose(pca1.components_, pca2.components_, rtol=rtol, atol=atol) + assert_allclose( + pca1.explained_variance_, pca2.explained_variance_, rtol=rtol, atol=atol + ) + assert_allclose(pca1.singular_values_, pca2.singular_values_, rtol=rtol, atol=atol) + assert_allclose(pca1.mean_, pca2.mean_, rtol=rtol, atol=atol) + assert_allclose(pca1.noise_variance_, pca2.noise_variance_, rtol=rtol, atol=atol) + + assert pca1.n_components_ == pca2.n_components_ + assert pca1.n_samples_ == pca2.n_samples_ + assert pca1.n_features_in_ == pca2.n_features_in_ + + +@pytest.mark.parametrize("svd_solver", PCA_SOLVERS) +@pytest.mark.parametrize("n_components", range(1, iris.data.shape[1])) def test_pca(svd_solver, n_components): X = iris.data pca = PCA(n_components=n_components, svd_solver=svd_solver) @@ -37,19 +72,152 @@ def test_pca(svd_solver, n_components): assert_allclose(np.dot(cov, precision), np.eye(X.shape[1]), atol=1e-12) +@pytest.mark.parametrize("density", [0.01, 0.1, 0.30]) +@pytest.mark.parametrize("n_components", [1, 2, 10]) +@pytest.mark.parametrize("sparse_container", CSR_CONTAINERS + CSC_CONTAINERS) +@pytest.mark.parametrize("svd_solver", ["arpack", "covariance_eigh"]) +@pytest.mark.parametrize("scale", [1, 10, 100]) +def test_pca_sparse( + global_random_seed, svd_solver, sparse_container, n_components, density, scale +): + """Check that the results are the same for sparse and dense input.""" + + # Set atol in addition of the default rtol to account for the very wide range of + # result values (1e-8 to 1e0). + atol = 1e-12 + transform_atol = 1e-10 + + random_state = np.random.default_rng(global_random_seed) + X = sparse_container( + sp.sparse.random( + SPARSE_M, + SPARSE_N, + random_state=random_state, + density=density, + ) + ) + # Scale the data + vary the column means + scale_vector = random_state.random(X.shape[1]) * scale + X = X.multiply(scale_vector) + + pca = PCA( + n_components=n_components, + svd_solver=svd_solver, + random_state=global_random_seed, + ) + pca.fit(X) + + Xd = X.toarray() + pcad = PCA( + n_components=n_components, + svd_solver=svd_solver, + random_state=global_random_seed, + ) + pcad.fit(Xd) + + # Fitted attributes equality + _check_fitted_pca_close(pca, pcad, atol=atol) + + # Test transform + X2 = sparse_container( + sp.sparse.random( + SPARSE_M, + SPARSE_N, + random_state=random_state, + density=density, + ) + ) + X2d = X2.toarray() + + assert_allclose(pca.transform(X2), pca.transform(X2d), atol=transform_atol) + assert_allclose(pca.transform(X2), pcad.transform(X2d), atol=transform_atol) + + +@pytest.mark.parametrize("sparse_container", CSR_CONTAINERS + CSC_CONTAINERS) +def test_pca_sparse_fit_transform(global_random_seed, sparse_container): + random_state = np.random.default_rng(global_random_seed) + X = sparse_container( + sp.sparse.random( + SPARSE_M, + SPARSE_N, + random_state=random_state, + density=0.01, + ) + ) + X2 = sparse_container( + sp.sparse.random( + SPARSE_M, + SPARSE_N, + random_state=random_state, + density=0.01, + ) + ) + + pca_fit = PCA(n_components=10, svd_solver="arpack", random_state=global_random_seed) + pca_fit_transform = PCA( + n_components=10, svd_solver="arpack", random_state=global_random_seed + ) + + pca_fit.fit(X) + transformed_X = pca_fit_transform.fit_transform(X) + + _check_fitted_pca_close(pca_fit, pca_fit_transform) + assert_allclose(transformed_X, pca_fit_transform.transform(X)) + assert_allclose(transformed_X, pca_fit.transform(X)) + assert_allclose(pca_fit.transform(X2), pca_fit_transform.transform(X2)) + + +@pytest.mark.parametrize("svd_solver", ["randomized", "full"]) +@pytest.mark.parametrize("sparse_container", CSR_CONTAINERS + CSC_CONTAINERS) +def test_sparse_pca_solver_error(global_random_seed, svd_solver, sparse_container): + random_state = np.random.RandomState(global_random_seed) + X = sparse_container( + sp.sparse.random( + SPARSE_M, + SPARSE_N, + random_state=random_state, + ) + ) + pca = PCA(n_components=30, svd_solver=svd_solver) + error_msg_pattern = ( + 'PCA only support sparse inputs with the "arpack" and "covariance_eigh"' + f' solvers, while "{svd_solver}" was passed' + ) + with pytest.raises(TypeError, match=error_msg_pattern): + pca.fit(X) + + +@pytest.mark.parametrize("sparse_container", CSR_CONTAINERS + CSC_CONTAINERS) +def test_sparse_pca_auto_arpack_singluar_values_consistency( + global_random_seed, sparse_container +): + """Check that "auto" and "arpack" solvers are equivalent for sparse inputs.""" + random_state = np.random.RandomState(global_random_seed) + X = sparse_container( + sp.sparse.random( + SPARSE_M, + SPARSE_N, + random_state=random_state, + ) + ) + pca_arpack = PCA(n_components=10, svd_solver="arpack").fit(X) + pca_auto = PCA(n_components=10, svd_solver="auto").fit(X) + assert_allclose(pca_arpack.singular_values_, pca_auto.singular_values_, rtol=5e-3) + + def test_no_empty_slice_warning(): # test if we avoid numpy warnings for computing over empty arrays n_components = 10 n_features = n_components + 2 # anything > n_comps triggered it in 0.16 X = np.random.uniform(-1, 1, size=(n_components, n_features)) pca = PCA(n_components=n_components) - with pytest.warns(None) as record: + with warnings.catch_warnings(): + warnings.simplefilter("error", RuntimeWarning) pca.fit(X) - assert not record.list -@pytest.mark.parametrize('copy', [True, False]) -@pytest.mark.parametrize('solver', PCA_SOLVERS) +@pytest.mark.parametrize("copy", [True, False]) +@pytest.mark.parametrize("solver", PCA_SOLVERS) def test_whitening(solver, copy): # Check that PCA output has unit-variance rng = np.random.RandomState(0) @@ -59,9 +227,10 @@ def test_whitening(solver, copy): rank = 50 # some low rank data with correlated features - X = np.dot(rng.randn(n_samples, rank), - np.dot(np.diag(np.linspace(10.0, 1.0, rank)), - rng.randn(rank, n_features))) + X = np.dot( + rng.randn(n_samples, rank), + np.dot(np.diag(np.linspace(10.0, 1.0, rank)), rng.randn(rank, n_features)), + ) # the component-wise variance of the first 50 features is 3 times the # mean component-wise variance of the remaining 30 features X[:, :50] *= 3 @@ -73,8 +242,14 @@ def test_whitening(solver, copy): # whiten the data while projecting to the lower dim subspace X_ = X.copy() # make sure we keep an original across iterations. - pca = PCA(n_components=n_components, whiten=True, copy=copy, - svd_solver=solver, random_state=0, iterated_power=7) + pca = PCA( + n_components=n_components, + whiten=True, + copy=copy, + svd_solver=solver, + random_state=0, + iterated_power=7, + ) # test fit_transform X_whitened = pca.fit_transform(X_.copy()) assert X_whitened.shape == (n_samples, n_components) @@ -82,13 +257,12 @@ def test_whitening(solver, copy): assert_allclose(X_whitened, X_whitened2, rtol=5e-4) assert_allclose(X_whitened.std(ddof=1, axis=0), np.ones(n_components)) - assert_allclose( - X_whitened.mean(axis=0), np.zeros(n_components), atol=1e-12 - ) + assert_allclose(X_whitened.mean(axis=0), np.zeros(n_components), atol=1e-12) X_ = X.copy() - pca = PCA(n_components=n_components, whiten=False, copy=copy, - svd_solver=solver).fit(X_) + pca = PCA( + n_components=n_components, whiten=False, copy=copy, svd_solver=solver + ).fit(X_.copy()) X_unwhitened = pca.transform(X_) assert X_unwhitened.shape == (n_samples, n_components) @@ -97,38 +271,156 @@ def test_whitening(solver, copy): # we always center, so no test for non-centering. -@pytest.mark.parametrize('svd_solver', ['arpack', 'randomized']) -def test_pca_explained_variance_equivalence_solver(svd_solver): - rng = np.random.RandomState(0) - n_samples, n_features = 100, 80 - X = rng.randn(n_samples, n_features) - - pca_full = PCA(n_components=2, svd_solver='full') - pca_other = PCA(n_components=2, svd_solver=svd_solver, random_state=0) - - pca_full.fit(X) - pca_other.fit(X) - - assert_allclose( - pca_full.explained_variance_, - pca_other.explained_variance_, - rtol=5e-2 +@pytest.mark.parametrize( + "other_svd_solver", sorted(list(set(PCA_SOLVERS) - {"full", "auto"})) +) +@pytest.mark.parametrize("data_shape", ["tall", "wide"]) +@pytest.mark.parametrize("rank_deficient", [False, True]) +@pytest.mark.parametrize("whiten", [False, True]) +def test_pca_solver_equivalence( + other_svd_solver, + data_shape, + rank_deficient, + whiten, + global_random_seed, + global_dtype, +): + if data_shape == "tall": + n_samples, n_features = 100, 30 + else: + n_samples, n_features = 30, 100 + n_samples_test = 10 + + if rank_deficient: + rng = np.random.default_rng(global_random_seed) + rank = min(n_samples, n_features) // 2 + X = rng.standard_normal( + size=(n_samples + n_samples_test, rank) + ) @ rng.standard_normal(size=(rank, n_features)) + else: + X = make_low_rank_matrix( + n_samples=n_samples + n_samples_test, + n_features=n_features, + tail_strength=0.5, + random_state=global_random_seed, + ) + # With a non-zero tail strength, the data is actually full-rank. + rank = min(n_samples, n_features) + + X = X.astype(global_dtype, copy=False) + X_train, X_test = X[:n_samples], X[n_samples:] + + if global_dtype == np.float32: + tols = dict(atol=3e-2, rtol=1e-5) + variance_threshold = 1e-5 + else: + tols = dict(atol=1e-10, rtol=1e-12) + variance_threshold = 1e-12 + + extra_other_kwargs = {} + if other_svd_solver == "randomized": + # Only check for a truncated result with a large number of iterations + # to make sure that we can recover precise results. + n_components = 10 + extra_other_kwargs = {"iterated_power": 50} + elif other_svd_solver == "arpack": + # Test all components except the last one which cannot be estimated by + # arpack. + n_components = np.minimum(n_samples, n_features) - 1 + else: + # Test all components to high precision. + n_components = None + + pca_full = PCA(n_components=n_components, svd_solver="full", whiten=whiten) + pca_other = PCA( + n_components=n_components, + svd_solver=other_svd_solver, + whiten=whiten, + random_state=global_random_seed, + **extra_other_kwargs, ) + X_trans_full_train = pca_full.fit_transform(X_train) + assert np.isfinite(X_trans_full_train).all() + assert X_trans_full_train.dtype == global_dtype + X_trans_other_train = pca_other.fit_transform(X_train) + assert np.isfinite(X_trans_other_train).all() + assert X_trans_other_train.dtype == global_dtype + + assert (pca_full.explained_variance_ >= 0).all() + assert_allclose(pca_full.explained_variance_, pca_other.explained_variance_, **tols) assert_allclose( pca_full.explained_variance_ratio_, pca_other.explained_variance_ratio_, - rtol=5e-2 + **tols, + ) + reference_components = pca_full.components_ + assert np.isfinite(reference_components).all() + other_components = pca_other.components_ + assert np.isfinite(other_components).all() + + # For some choice of n_components and data distribution, some components + # might be pure noise, let's ignore them in the comparison: + stable = pca_full.explained_variance_ > variance_threshold + assert stable.sum() > 1 + assert_allclose(reference_components[stable], other_components[stable], **tols) + + # As a result the output of fit_transform should be the same: + assert_allclose( + X_trans_other_train[:, stable], X_trans_full_train[:, stable], **tols ) + # And similarly for the output of transform on new data (except for the + # last component that can be underdetermined): + X_trans_full_test = pca_full.transform(X_test) + assert np.isfinite(X_trans_full_test).all() + assert X_trans_full_test.dtype == global_dtype + X_trans_other_test = pca_other.transform(X_test) + assert np.isfinite(X_trans_other_test).all() + assert X_trans_other_test.dtype == global_dtype + assert_allclose(X_trans_other_test[:, stable], X_trans_full_test[:, stable], **tols) + + # Check that inverse transform reconstructions for both solvers are + # compatible. + X_recons_full_test = pca_full.inverse_transform(X_trans_full_test) + assert np.isfinite(X_recons_full_test).all() + assert X_recons_full_test.dtype == global_dtype + X_recons_other_test = pca_other.inverse_transform(X_trans_other_test) + assert np.isfinite(X_recons_other_test).all() + assert X_recons_other_test.dtype == global_dtype + + if pca_full.components_.shape[0] == pca_full.components_.shape[1]: + # In this case, the models should have learned the same invertible + # transform. They should therefore both be able to reconstruct the test + # data. + assert_allclose(X_recons_full_test, X_test, **tols) + assert_allclose(X_recons_other_test, X_test, **tols) + elif pca_full.components_.shape[0] < rank: + # In the absence of noisy components, both models should be able to + # reconstruct the same low-rank approximation of the original data. + assert pca_full.explained_variance_.min() > variance_threshold + assert_allclose(X_recons_full_test, X_recons_other_test, **tols) + else: + # When n_features > n_samples and n_components is larger than the rank + # of the training set, the output of the `inverse_transform` function + # is ill-defined. We can only check that we reach the same fixed point + # after another round of transform: + assert_allclose( + pca_full.transform(X_recons_full_test)[:, stable], + pca_other.transform(X_recons_other_test)[:, stable], + **tols, + ) + @pytest.mark.parametrize( - 'X', - [np.random.RandomState(0).randn(100, 80), - datasets.make_classification(100, 80, n_informative=78, - random_state=0)[0]], - ids=['random-data', 'correlated-data'] + "X", + [ + np.random.RandomState(0).randn(100, 80), + datasets.make_classification(100, 80, n_informative=78, random_state=0)[0], + np.random.RandomState(0).randn(10, 100), + ], + ids=["random-tall", "correlated-tall", "random-wide"], ) -@pytest.mark.parametrize('svd_solver', PCA_SOLVERS) +@pytest.mark.parametrize("svd_solver", PCA_SOLVERS) def test_pca_explained_variance_empirical(X, svd_solver): pca = PCA(n_components=2, svd_solver=svd_solver, random_state=0) X_pca = pca.fit_transform(X) @@ -139,21 +431,19 @@ def test_pca_explained_variance_empirical(X, svd_solver): assert_allclose(pca.explained_variance_, expected_result, rtol=5e-3) -@pytest.mark.parametrize("svd_solver", ['arpack', 'randomized']) +@pytest.mark.parametrize("svd_solver", ["arpack", "randomized"]) def test_pca_singular_values_consistency(svd_solver): rng = np.random.RandomState(0) n_samples, n_features = 100, 80 X = rng.randn(n_samples, n_features) - pca_full = PCA(n_components=2, svd_solver='full', random_state=rng) + pca_full = PCA(n_components=2, svd_solver="full", random_state=rng) pca_other = PCA(n_components=2, svd_solver=svd_solver, random_state=rng) pca_full.fit(X) pca_other.fit(X) - assert_allclose( - pca_full.singular_values_, pca_other.singular_values_, rtol=5e-3 - ) + assert_allclose(pca_full.singular_values_, pca_other.singular_values_, rtol=5e-3) @pytest.mark.parametrize("svd_solver", PCA_SOLVERS) @@ -167,12 +457,10 @@ def test_pca_singular_values(svd_solver): # compare to the Frobenius norm assert_allclose( - np.sum(pca.singular_values_ ** 2), np.linalg.norm(X_trans, "fro") ** 2 + np.sum(pca.singular_values_**2), np.linalg.norm(X_trans, "fro") ** 2 ) # Compare to the 2-norms of the score vectors - assert_allclose( - pca.singular_values_, np.sqrt(np.sum(X_trans ** 2, axis=0)) - ) + assert_allclose(pca.singular_values_, np.sqrt(np.sum(X_trans**2, axis=0))) # set the singular values and see what er get back n_samples, n_features = 100, 110 @@ -180,7 +468,7 @@ def test_pca_singular_values(svd_solver): pca = PCA(n_components=3, svd_solver=svd_solver, random_state=rng) X_trans = pca.fit_transform(X) - X_trans /= np.sqrt(np.sum(X_trans ** 2, axis=0)) + X_trans /= np.sqrt(np.sum(X_trans**2, axis=0)) X_trans[:, 0] *= 3.142 X_trans[:, 1] *= 2.718 X_hat = np.dot(X_trans, pca.components_) @@ -193,14 +481,14 @@ def test_pca_check_projection(svd_solver): # Test that the projection of data is correct rng = np.random.RandomState(0) n, p = 100, 3 - X = rng.randn(n, p) * .1 + X = rng.randn(n, p) * 0.1 X[:10] += np.array([3, 4, 5]) Xt = 0.1 * rng.randn(1, p) + np.array([3, 4, 5]) Yt = PCA(n_components=2, svd_solver=svd_solver).fit(X).transform(Xt) - Yt /= np.sqrt((Yt ** 2).sum()) + Yt /= np.sqrt((Yt**2).sum()) - assert_allclose(np.abs(Yt[0][0]), 1., rtol=5e-3) + assert_allclose(np.abs(Yt[0][0]), 1.0, rtol=5e-3) @pytest.mark.parametrize("svd_solver", PCA_SOLVERS) @@ -214,14 +502,14 @@ def test_pca_check_projection_list(svd_solver): assert_allclose(X_trans.std(), 0.71, rtol=5e-3) -@pytest.mark.parametrize("svd_solver", ['full', 'arpack', 'randomized']) +@pytest.mark.parametrize("svd_solver", ["full", "arpack", "randomized"]) @pytest.mark.parametrize("whiten", [False, True]) def test_pca_inverse(svd_solver, whiten): # Test that the projection of data can be inverted rng = np.random.RandomState(0) n, p = 50, 3 X = rng.randn(n, p) # spherical data - X[:, 1] *= .00001 # make middle component relatively small + X[:, 1] *= 0.00001 # make middle component relatively small X += [5, 4, 3] # make a large mean # same check that we can find the original data from the transformed @@ -233,52 +521,53 @@ def test_pca_inverse(svd_solver, whiten): @pytest.mark.parametrize( - 'data', - [np.array([[0, 1, 0], [1, 0, 0]]), np.array([[0, 1, 0], [1, 0, 0]]).T] + "data", [np.array([[0, 1, 0], [1, 0, 0]]), np.array([[0, 1, 0], [1, 0, 0]]).T] ) @pytest.mark.parametrize( "svd_solver, n_components, err_msg", - [('arpack', 0, r'must be between 1 and min\(n_samples, n_features\)'), - ('randomized', 0, r'must be between 1 and min\(n_samples, n_features\)'), - ('arpack', 2, r'must be strictly less than min'), - ('auto', -1, (r"n_components={}L? must be between {}L? and " - r"min\(n_samples, n_features\)={}L? with " - r"svd_solver=\'{}\'")), - ('auto', 3, (r"n_components={}L? must be between {}L? and " - r"min\(n_samples, n_features\)={}L? with " - r"svd_solver=\'{}\'")), - ('auto', 1.0, "must be of type int")] + [ + ("arpack", 0, r"must be between 1 and min\(n_samples, n_features\)"), + ("randomized", 0, r"must be between 1 and min\(n_samples, n_features\)"), + ("arpack", 2, r"must be strictly less than min"), + ( + "auto", + 3, + ( + r"n_components=3 must be between 0 and min\(n_samples, " + r"n_features\)=2 with svd_solver='full'" + ), + ), + ], ) def test_pca_validation(svd_solver, data, n_components, err_msg): # Ensures that solver-specific extreme inputs for the n_components # parameter raise errors smallest_d = 2 # The smallest dimension - lower_limit = {'randomized': 1, 'arpack': 1, 'full': 0, 'auto': 0} pca_fitted = PCA(n_components, svd_solver=svd_solver) - solver_reported = 'full' if svd_solver == 'auto' else svd_solver - err_msg = err_msg.format( - n_components, lower_limit[svd_solver], smallest_d, solver_reported - ) with pytest.raises(ValueError, match=err_msg): pca_fitted.fit(data) # Additional case for arpack - if svd_solver == 'arpack': + if svd_solver == "arpack": n_components = smallest_d - err_msg = ("n_components={}L? must be strictly less than " - r"min\(n_samples, n_features\)={}L? with " - "svd_solver=\'arpack\'".format(n_components, smallest_d)) + err_msg = ( + "n_components={}L? must be strictly less than " + r"min\(n_samples, n_features\)={}L? with " + "svd_solver='arpack'".format(n_components, smallest_d) + ) with pytest.raises(ValueError, match=err_msg): PCA(n_components, svd_solver=svd_solver).fit(data) @pytest.mark.parametrize( - 'solver, n_components_', - [('full', min(iris.data.shape)), - ('arpack', min(iris.data.shape) - 1), - ('randomized', min(iris.data.shape))] + "solver, n_components_", + [ + ("full", min(iris.data.shape)), + ("arpack", min(iris.data.shape) - 1), + ("randomized", min(iris.data.shape)), + ], ) @pytest.mark.parametrize("data", [iris.data, iris.data.T]) def test_n_components_none(data, solver, n_components_): @@ -287,13 +576,13 @@ def test_n_components_none(data, solver, n_components_): assert pca.n_components_ == n_components_ -@pytest.mark.parametrize("svd_solver", ['auto', 'full']) +@pytest.mark.parametrize("svd_solver", ["auto", "full"]) def test_n_components_mle(svd_solver): # Ensure that n_components == 'mle' doesn't raise error for auto/full rng = np.random.RandomState(0) n_samples, n_features = 600, 10 X = rng.randn(n_samples, n_features) - pca = PCA(n_components='mle', svd_solver=svd_solver) + pca = PCA(n_components="mle", svd_solver=svd_solver) pca.fit(X) assert pca.n_components_ == 1 @@ -305,9 +594,10 @@ def test_n_components_mle_error(svd_solver): rng = np.random.RandomState(0) n_samples, n_features = 600, 10 X = rng.randn(n_samples, n_features) - pca = PCA(n_components='mle', svd_solver=svd_solver) - err_msg = ("n_components='mle' cannot be a string with svd_solver='{}'" - .format(svd_solver)) + pca = PCA(n_components="mle", svd_solver=svd_solver) + err_msg = "n_components='mle' cannot be a string with svd_solver='{}'".format( + svd_solver + ) with pytest.raises(ValueError, match=err_msg): pca.fit(X) @@ -316,10 +606,10 @@ def test_pca_dim(): # Check automated dimensionality setting rng = np.random.RandomState(0) n, p = 100, 5 - X = rng.randn(n, p) * .1 + X = rng.randn(n, p) * 0.1 X[:10] += np.array([3, 4, 5, 1, 2]) - pca = PCA(n_components='mle', svd_solver='full').fit(X) - assert pca.n_components == 'mle' + pca = PCA(n_components="mle", svd_solver="full").fit(X) + assert pca.n_components == "mle" assert pca.n_components_ == 1 @@ -328,13 +618,16 @@ def test_infer_dim_1(): # Or at least use explicit variable names... n, p = 1000, 5 rng = np.random.RandomState(0) - X = (rng.randn(n, p) * .1 + rng.randn(n, 1) * np.array([3, 4, 5, 1, 2]) + - np.array([1, 0, 7, 4, 6])) - pca = PCA(n_components=p, svd_solver='full') + X = ( + rng.randn(n, p) * 0.1 + + rng.randn(n, 1) * np.array([3, 4, 5, 1, 2]) + + np.array([1, 0, 7, 4, 6]) + ) + pca = PCA(n_components=p, svd_solver="full") pca.fit(X) spect = pca.explained_variance_ ll = np.array([_assess_dimension(spect, k, n) for k in range(1, p)]) - assert ll[1] > ll.max() - .01 * n + assert ll[1] > ll.max() - 0.01 * n def test_infer_dim_2(): @@ -342,10 +635,10 @@ def test_infer_dim_2(): # Or at least use explicit variable names... n, p = 1000, 5 rng = np.random.RandomState(0) - X = rng.randn(n, p) * .1 + X = rng.randn(n, p) * 0.1 X[:10] += np.array([3, 4, 5, 1, 2]) X[10:20] += np.array([6, 0, 7, 2, -1]) - pca = PCA(n_components=p, svd_solver='full') + pca = PCA(n_components=p, svd_solver="full") pca.fit(X) spect = pca.explained_variance_ assert _infer_dimension(spect, n) > 1 @@ -354,11 +647,11 @@ def test_infer_dim_2(): def test_infer_dim_3(): n, p = 100, 5 rng = np.random.RandomState(0) - X = rng.randn(n, p) * .1 + X = rng.randn(n, p) * 0.1 X[:10] += np.array([3, 4, 5, 1, 2]) X[10:20] += np.array([6, 0, 7, 2, -1]) X[30:40] += 2 * np.array([-1, 1, -1, 1, -1]) - pca = PCA(n_components=p, svd_solver='full') + pca = PCA(n_components=p, svd_solver="full") pca.fit(X) spect = pca.explained_variance_ assert _infer_dimension(spect, n) > 2 @@ -366,13 +659,14 @@ def test_infer_dim_3(): @pytest.mark.parametrize( "X, n_components, n_components_validated", - [(iris.data, 0.95, 2), # row > col - (iris.data, 0.01, 1), # row > col - (np.random.RandomState(0).rand(5, 20), 0.5, 2)] # row < col + [ + (iris.data, 0.95, 2), # row > col + (iris.data, 0.01, 1), # row > col + (np.random.RandomState(0).rand(5, 20), 0.5, 2), + ], # row < col ) -def test_infer_dim_by_explained_variance(X, n_components, - n_components_validated): - pca = PCA(n_components=n_components, svd_solver='full') +def test_infer_dim_by_explained_variance(X, n_components, n_components_validated): + pca = PCA(n_components=n_components, svd_solver="full") pca.fit(X) assert pca.n_components == pytest.approx(n_components) assert pca.n_components_ == n_components_validated @@ -383,15 +677,15 @@ def test_pca_score(svd_solver): # Test that probabilistic PCA scoring yields a reasonable score n, p = 1000, 3 rng = np.random.RandomState(0) - X = rng.randn(n, p) * .1 + np.array([3, 4, 5]) + X = rng.randn(n, p) * 0.1 + np.array([3, 4, 5]) pca = PCA(n_components=2, svd_solver=svd_solver) pca.fit(X) ll1 = pca.score(X) - h = -0.5 * np.log(2 * np.pi * np.exp(1) * 0.1 ** 2) * p + h = -0.5 * np.log(2 * np.pi * np.exp(1) * 0.1**2) * p assert_allclose(ll1 / h, 1, rtol=5e-2) - ll2 = pca.score(rng.randn(n, p) * .2 + np.array([3, 4, 5])) + ll2 = pca.score(rng.randn(n, p) * 0.2 + np.array([3, 4, 5])) assert ll1 > ll2 pca = PCA(n_components=2, whiten=True, svd_solver=svd_solver) @@ -404,13 +698,11 @@ def test_pca_score3(): # Check that probabilistic PCA selects the right model n, p = 200, 3 rng = np.random.RandomState(0) - Xl = (rng.randn(n, p) + rng.randn(n, 1) * np.array([3, 4, 5]) + - np.array([1, 0, 7])) - Xt = (rng.randn(n, p) + rng.randn(n, 1) * np.array([3, 4, 5]) + - np.array([1, 0, 7])) + Xl = rng.randn(n, p) + rng.randn(n, 1) * np.array([3, 4, 5]) + np.array([1, 0, 7]) + Xt = rng.randn(n, p) + rng.randn(n, 1) * np.array([3, 4, 5]) + np.array([1, 0, 7]) ll = np.zeros(p) for k in range(p): - pca = PCA(n_components=k, svd_solver='full') + pca = PCA(n_components=k, svd_solver="full") pca.fit(Xl) ll[k] = pca.score(Xt) @@ -433,7 +725,7 @@ def test_pca_sanity_noise_variance(svd_solver): def test_pca_score_consistency_solvers(svd_solver): # Check the consistency of score between solvers X, _ = datasets.load_digits(return_X_y=True) - pca_full = PCA(n_components=30, svd_solver='full', random_state=0) + pca_full = PCA(n_components=30, svd_solver="full", random_state=0) pca_other = PCA(n_components=30, svd_solver=svd_solver, random_state=0) pca_full.fit(X) pca_other.fit(X) @@ -447,56 +739,49 @@ def test_pca_zero_noise_variance_edge_cases(svd_solver): # when n_components == min(n_samples, n_features) n, p = 100, 3 rng = np.random.RandomState(0) - X = rng.randn(n, p) * .1 + np.array([3, 4, 5]) + X = rng.randn(n, p) * 0.1 + np.array([3, 4, 5]) pca = PCA(n_components=p, svd_solver=svd_solver) pca.fit(X) assert pca.noise_variance_ == 0 + # Non-regression test for gh-12489 + # ensure no divide-by-zero error for n_components == n_features < n_samples + pca.score(X) pca.fit(X.T) assert pca.noise_variance_ == 0 + # Non-regression test for gh-12489 + # ensure no divide-by-zero error for n_components == n_samples < n_features + pca.score(X.T) @pytest.mark.parametrize( - 'data, n_components, expected_solver', - [ # case: n_components in (0,1) => 'full' - (np.random.RandomState(0).uniform(size=(1000, 50)), 0.5, 'full'), - # case: max(X.shape) <= 500 => 'full' - (np.random.RandomState(0).uniform(size=(10, 50)), 5, 'full'), + "n_samples, n_features, n_components, expected_solver", + [ + # case: n_samples < 10 * n_features and max(X.shape) <= 500 => 'full' + (10, 50, 5, "full"), + # case: n_samples > 10 * n_features and n_features < 500 => 'covariance_eigh' + (1000, 50, 50, "covariance_eigh"), # case: n_components >= .8 * min(X.shape) => 'full' - (np.random.RandomState(0).uniform(size=(1000, 50)), 50, 'full'), + (1000, 500, 400, "full"), # n_components >= 1 and n_components < .8*min(X.shape) => 'randomized' - (np.random.RandomState(0).uniform(size=(1000, 50)), 10, 'randomized') - ] + (1000, 500, 10, "randomized"), + # case: n_components in (0,1) => 'full' + (1000, 500, 0.5, "full"), + ], ) -def test_pca_svd_solver_auto(data, n_components, expected_solver): +def test_pca_svd_solver_auto(n_samples, n_features, n_components, expected_solver): + data = np.random.RandomState(0).uniform(size=(n_samples, n_features)) pca_auto = PCA(n_components=n_components, random_state=0) pca_test = PCA( n_components=n_components, svd_solver=expected_solver, random_state=0 ) pca_auto.fit(data) + assert pca_auto._fit_svd_solver == expected_solver pca_test.fit(data) assert_allclose(pca_auto.components_, pca_test.components_) -@pytest.mark.parametrize('svd_solver', PCA_SOLVERS) -def test_pca_sparse_input(svd_solver): - X = np.random.RandomState(0).rand(5, 4) - X = sp.sparse.csr_matrix(X) - assert sp.sparse.issparse(X) - - pca = PCA(n_components=3, svd_solver=svd_solver) - with pytest.raises(TypeError): - pca.fit(X) - - -def test_pca_bad_solver(): - X = np.random.RandomState(0).rand(5, 4) - pca = PCA(n_components=3, svd_solver='bad_argument') - with pytest.raises(ValueError): - pca.fit(X) - - @pytest.mark.parametrize("svd_solver", PCA_SOLVERS) def test_pca_deterministic_output(svd_solver): rng = np.random.RandomState(0) @@ -506,37 +791,37 @@ def test_pca_deterministic_output(svd_solver): for i in range(20): pca = PCA(n_components=2, svd_solver=svd_solver, random_state=rng) transformed_X[i, :] = pca.fit_transform(X)[0] - assert_allclose( - transformed_X, np.tile(transformed_X[0, :], 20).reshape(20, 2) - ) + assert_allclose(transformed_X, np.tile(transformed_X[0, :], 20).reshape(20, 2)) -@pytest.mark.parametrize('svd_solver', PCA_SOLVERS) -def test_pca_dtype_preservation(svd_solver): - check_pca_float_dtype_preservation(svd_solver) +@pytest.mark.parametrize("svd_solver", PCA_SOLVERS) +def test_pca_dtype_preservation(svd_solver, global_random_seed): + check_pca_float_dtype_preservation(svd_solver, global_random_seed) check_pca_int_dtype_upcast_to_double(svd_solver) -def check_pca_float_dtype_preservation(svd_solver): +def check_pca_float_dtype_preservation(svd_solver, seed): # Ensure that PCA does not upscale the dtype when input is float32 - X_64 = np.random.RandomState(0).rand(1000, 4).astype(np.float64, - copy=False) - X_32 = X_64.astype(np.float32) + X = np.random.RandomState(seed).rand(1000, 4) + X_float64 = X.astype(np.float64, copy=False) + X_float32 = X.astype(np.float32) - pca_64 = PCA(n_components=3, svd_solver=svd_solver, - random_state=0).fit(X_64) - pca_32 = PCA(n_components=3, svd_solver=svd_solver, - random_state=0).fit(X_32) + pca_64 = PCA(n_components=3, svd_solver=svd_solver, random_state=seed).fit( + X_float64 + ) + pca_32 = PCA(n_components=3, svd_solver=svd_solver, random_state=seed).fit( + X_float32 + ) assert pca_64.components_.dtype == np.float64 assert pca_32.components_.dtype == np.float32 - assert pca_64.transform(X_64).dtype == np.float64 - assert pca_32.transform(X_32).dtype == np.float32 + assert pca_64.transform(X_float64).dtype == np.float64 + assert pca_32.transform(X_float32).dtype == np.float32 - # the rtol is set such that the test passes on all platforms tested on - # conda-forge: PR#15775 - # see: https://github.com/conda-forge/scikit-learn-feedstock/pull/113 - assert_allclose(pca_64.components_, pca_32.components_, rtol=2e-4) + # The atol and rtol are set such that the test passes for all random seeds + # on all supported platforms on our CI and conda-forge with the default + # random seed. + assert_allclose(pca_64.components_, pca_32.components_, rtol=1e-3, atol=1e-3) def check_pca_int_dtype_upcast_to_double(svd_solver): @@ -545,10 +830,8 @@ def check_pca_int_dtype_upcast_to_double(svd_solver): X_i64 = X_i64.astype(np.int64, copy=False) X_i32 = X_i64.astype(np.int32, copy=False) - pca_64 = PCA(n_components=3, svd_solver=svd_solver, - random_state=0).fit(X_i64) - pca_32 = PCA(n_components=3, svd_solver=svd_solver, - random_state=0).fit(X_i32) + pca_64 = PCA(n_components=3, svd_solver=svd_solver, random_state=0).fit(X_i64) + pca_32 = PCA(n_components=3, svd_solver=svd_solver, random_state=0).fit(X_i32) assert pca_64.components_.dtype == np.float64 assert pca_32.components_.dtype == np.float64 @@ -575,8 +858,7 @@ def test_assess_dimension_bad_rank(): spectrum = np.array([1, 1e-30, 1e-30, 1e-30]) n_samples = 10 for rank in (0, 5): - with pytest.raises(ValueError, - match=r"should be in \[1, n_features - 1\]"): + with pytest.raises(ValueError, match=r"should be in \[1, n_features - 1\]"): _assess_dimension(spectrum, rank, n_samples) @@ -596,24 +878,28 @@ def test_small_eigenvalues_mle(): def test_mle_redundant_data(): # Test 'mle' with pathological X: only one relevant feature should give a # rank of 1 - X, _ = datasets.make_classification(n_features=20, - n_informative=1, n_repeated=18, - n_redundant=1, n_clusters_per_class=1, - random_state=42) - pca = PCA(n_components='mle').fit(X) + X, _ = datasets.make_classification( + n_features=20, + n_informative=1, + n_repeated=18, + n_redundant=1, + n_clusters_per_class=1, + random_state=42, + ) + pca = PCA(n_components="mle").fit(X) assert pca.n_components_ == 1 def test_fit_mle_too_few_samples(): # Tests that an error is raised when the number of samples is smaller # than the number of features during an mle fit - X, _ = datasets.make_classification(n_samples=20, n_features=21, - random_state=42) + X, _ = datasets.make_classification(n_samples=20, n_features=21, random_state=42) - pca = PCA(n_components='mle', svd_solver='full') - with pytest.raises(ValueError, match="n_components='mle' is only " - "supported if " - "n_samples >= n_features"): + pca = PCA(n_components="mle", svd_solver="full") + with pytest.raises( + ValueError, + match="n_components='mle' is only supported if n_samples >= n_features", + ): pca.fit(X) @@ -623,7 +909,7 @@ def test_mle_simple_case(): n_samples, n_dim = 1000, 10 X = np.random.RandomState(0).randn(n_samples, n_dim) X[:, -1] = np.mean(X[:, :-1], axis=-1) # true X dim is ndim - 1 - pca_skl = PCA('mle', svd_solver='full') + pca_skl = PCA("mle", svd_solver="full") pca_skl.fit(X) assert pca_skl.n_components_ == n_dim - 1 @@ -634,8 +920,235 @@ def test_assess_dimesion_rank_one(): X = np.ones((n_samples, n_features)) # rank 1 matrix _, s, _ = np.linalg.svd(X, full_matrices=True) # except for rank 1, all eigenvalues are 0 resp. close to 0 (FP) - assert_allclose(s[1:], np.zeros(n_features-1), atol=1e-12) + assert_allclose(s[1:], np.zeros(n_features - 1), atol=1e-12) assert np.isfinite(_assess_dimension(s, rank=1, n_samples=n_samples)) for rank in range(2, n_features): assert _assess_dimension(s, rank, n_samples) == -np.inf + + +def test_pca_randomized_svd_n_oversamples(): + """Check that exposing and setting `n_oversamples` will provide accurate results + even when `X` as a large number of features. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/20589 + """ + rng = np.random.RandomState(0) + n_features = 100 + X = rng.randn(1_000, n_features) + + # The default value of `n_oversamples` will lead to inaccurate results + # We force it to the number of features. + pca_randomized = PCA( + n_components=1, + svd_solver="randomized", + n_oversamples=n_features, + random_state=0, + ).fit(X) + pca_full = PCA(n_components=1, svd_solver="full").fit(X) + pca_arpack = PCA(n_components=1, svd_solver="arpack", random_state=0).fit(X) + + assert_allclose(np.abs(pca_full.components_), np.abs(pca_arpack.components_)) + assert_allclose(np.abs(pca_randomized.components_), np.abs(pca_arpack.components_)) + + +def test_feature_names_out(): + """Check feature names out for PCA.""" + pca = PCA(n_components=2).fit(iris.data) + + names = pca.get_feature_names_out() + assert_array_equal([f"pca{i}" for i in range(2)], names) + + +@pytest.mark.parametrize("copy", [True, False]) +def test_variance_correctness(copy): + """Check the accuracy of PCA's internal variance calculation""" + rng = np.random.RandomState(0) + X = rng.randn(1000, 200) + pca = PCA().fit(X) + pca_var = pca.explained_variance_ / pca.explained_variance_ratio_ + true_var = np.var(X, ddof=1, axis=0).sum() + np.testing.assert_allclose(pca_var, true_var) + + +def check_array_api_get_precision(name, estimator, array_namespace, device, dtype_name): + xp = _array_api_for_tests(array_namespace, device) + iris_np = iris.data.astype(dtype_name) + iris_xp = xp.asarray(iris_np, device=device) + + estimator.fit(iris_np) + precision_np = estimator.get_precision() + covariance_np = estimator.get_covariance() + + rtol = 2e-4 if iris_np.dtype == "float32" else 2e-7 + with config_context(array_api_dispatch=True): + estimator_xp = clone(estimator).fit(iris_xp) + precision_xp = estimator_xp.get_precision() + assert precision_xp.shape == (4, 4) + assert precision_xp.dtype == iris_xp.dtype + + assert_allclose( + _convert_to_numpy(precision_xp, xp=xp), + precision_np, + rtol=rtol, + atol=_atol_for_type(dtype_name), + ) + covariance_xp = estimator_xp.get_covariance() + assert covariance_xp.shape == (4, 4) + assert covariance_xp.dtype == iris_xp.dtype + + assert_allclose( + _convert_to_numpy(covariance_xp, xp=xp), + covariance_np, + rtol=rtol, + atol=_atol_for_type(dtype_name), + ) + + +@pytest.mark.parametrize( + "array_namespace, device, dtype_name", + yield_namespace_device_dtype_combinations(), + ids=_get_namespace_device_dtype_ids, +) +@pytest.mark.parametrize( + "check", + [check_array_api_input_and_values, check_array_api_get_precision], + ids=_get_check_estimator_ids, +) +@pytest.mark.parametrize( + "estimator", + [ + PCA(n_components=2, svd_solver="full"), + PCA(n_components=2, svd_solver="full", whiten=True), + PCA(n_components=0.1, svd_solver="full", whiten=True), + PCA(n_components=2, svd_solver="covariance_eigh"), + PCA(n_components=2, svd_solver="covariance_eigh", whiten=True), + PCA( + n_components=2, + svd_solver="randomized", + power_iteration_normalizer="QR", + random_state=0, # how to use global_random_seed here? + ), + ], + ids=_get_check_estimator_ids, +) +def test_pca_array_api_compliance( + estimator, check, array_namespace, device, dtype_name +): + name = estimator.__class__.__name__ + check(name, estimator, array_namespace, device=device, dtype_name=dtype_name) + + +@pytest.mark.parametrize( + "array_namespace, device, dtype_name", + yield_namespace_device_dtype_combinations(), + ids=_get_namespace_device_dtype_ids, +) +@pytest.mark.parametrize( + "check", + [check_array_api_get_precision], + ids=_get_check_estimator_ids, +) +@pytest.mark.parametrize( + "estimator", + [ + # PCA with mle cannot use check_array_api_input_and_values because of + # rounding errors in the noisy (low variance) components. Even checking + # the shape of the `components_` is problematic because the number of + # components depends on trimming threshold of the mle algorithm which + # can depend on device-specific rounding errors. + PCA(n_components="mle", svd_solver="full"), + ], + ids=_get_check_estimator_ids, +) +def test_pca_mle_array_api_compliance( + estimator, check, array_namespace, device, dtype_name +): + name = estimator.__class__.__name__ + check(name, estimator, array_namespace, device=device, dtype_name=dtype_name) + + # Simpler variant of the generic check_array_api_input checker tailored for + # the specific case of PCA with mle-trimmed components. + xp = _array_api_for_tests(array_namespace, device) + + X, y = make_classification(random_state=42) + X = X.astype(dtype_name, copy=False) + atol = _atol_for_type(X.dtype) + + est = clone(estimator) + + X_xp = xp.asarray(X, device=device) + y_xp = xp.asarray(y, device=device) + + est.fit(X, y) + + components_np = est.components_ + explained_variance_np = est.explained_variance_ + + est_xp = clone(est) + with config_context(array_api_dispatch=True): + est_xp.fit(X_xp, y_xp) + components_xp = est_xp.components_ + assert array_device(components_xp) == array_device(X_xp) + components_xp_np = _convert_to_numpy(components_xp, xp=xp) + + explained_variance_xp = est_xp.explained_variance_ + assert array_device(explained_variance_xp) == array_device(X_xp) + explained_variance_xp_np = _convert_to_numpy(explained_variance_xp, xp=xp) + + assert components_xp_np.dtype == components_np.dtype + assert components_xp_np.shape[1] == components_np.shape[1] + assert explained_variance_xp_np.dtype == explained_variance_np.dtype + + # Check that the explained variance values match for the + # common components: + min_components = min(components_xp_np.shape[0], components_np.shape[0]) + assert_allclose( + explained_variance_xp_np[:min_components], + explained_variance_np[:min_components], + atol=atol, + ) + + # If the number of components differ, check that the explained variance of + # the trimmed components is very small. + if components_xp_np.shape[0] != components_np.shape[0]: + reference_variance = explained_variance_np[-1] + extra_variance_np = explained_variance_np[min_components:] + extra_variance_xp_np = explained_variance_xp_np[min_components:] + assert all(np.abs(extra_variance_np - reference_variance) < atol) + assert all(np.abs(extra_variance_xp_np - reference_variance) < atol) + + +@pytest.mark.skipif( + os.environ.get("SCIPY_ARRAY_API") != "1", reason="SCIPY_ARRAY_API not set to 1." +) +def test_array_api_error_and_warnings_on_unsupported_params(): + xp = pytest.importorskip("array_api_strict") + iris_xp = xp.asarray(iris.data) + + pca = PCA(n_components=2, svd_solver="arpack", random_state=0) + expected_msg = re.escape( + "PCA with svd_solver='arpack' is not supported for Array API inputs." + ) + with pytest.raises(ValueError, match=expected_msg): + with config_context(array_api_dispatch=True): + pca.fit(iris_xp) + + pca.set_params(svd_solver="randomized", power_iteration_normalizer="LU") + expected_msg = re.escape( + "Array API does not support LU factorization. Set" + " `power_iteration_normalizer='QR'` instead." + ) + with pytest.raises(ValueError, match=expected_msg): + with config_context(array_api_dispatch=True): + pca.fit(iris_xp) + + pca.set_params(svd_solver="randomized", power_iteration_normalizer="auto") + expected_msg = re.escape( + "Array API does not support LU factorization, falling back to QR instead. Set" + " `power_iteration_normalizer='QR'` explicitly to silence this warning." + ) + with pytest.warns(UserWarning, match=expected_msg): + with config_context(array_api_dispatch=True): + pca.fit(iris_xp) diff --git a/sklearn/decomposition/tests/test_sparse_pca.py b/sklearn/decomposition/tests/test_sparse_pca.py index d6ddfa01a49d0..f8c71a5d0e752 100644 --- a/sklearn/decomposition/tests/test_sparse_pca.py +++ b/sklearn/decomposition/tests/test_sparse_pca.py @@ -1,17 +1,21 @@ -# Author: Vlad Niculae -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import sys -import pytest import numpy as np +import pytest +from numpy.testing import assert_array_equal -from sklearn.utils._testing import assert_array_almost_equal -from sklearn.utils._testing import assert_allclose -from sklearn.utils._testing import if_safe_multiprocessing_with_blas - -from sklearn.decomposition import SparsePCA, MiniBatchSparsePCA, PCA +from sklearn.datasets import make_low_rank_matrix +from sklearn.decomposition import PCA, MiniBatchSparsePCA, SparsePCA from sklearn.utils import check_random_state +from sklearn.utils._testing import ( + assert_allclose, + assert_array_almost_equal, + if_safe_multiprocessing_with_blas, +) +from sklearn.utils.extmath import svd_flip + def generate_toy_data(n_components, n_samples, image_size, random_state=None): n_features = image_size[0] * image_size[1] @@ -34,6 +38,7 @@ def generate_toy_data(n_components, n_samples, image_size, random_state=None): Y += 0.1 * rng.randn(Y.shape[0], Y.shape[1]) # Add noise return Y, U, V + # SparsePCA can be a bit slow. To avoid having test times go up, we # test different aspects of the code in the same test @@ -52,141 +57,120 @@ def test_correct_shapes(): assert U.shape == (12, 13) -def test_fit_transform(): +def test_fit_transform(global_random_seed): alpha = 1 - rng = np.random.RandomState(0) + rng = np.random.RandomState(global_random_seed) Y, _, _ = generate_toy_data(3, 10, (8, 8), random_state=rng) # wide array - spca_lars = SparsePCA(n_components=3, method='lars', alpha=alpha, - random_state=0) + spca_lars = SparsePCA( + n_components=3, method="lars", alpha=alpha, random_state=global_random_seed + ) spca_lars.fit(Y) # Test that CD gives similar results - spca_lasso = SparsePCA(n_components=3, method='cd', random_state=0, - alpha=alpha) + spca_lasso = SparsePCA( + n_components=3, method="cd", random_state=global_random_seed, alpha=alpha + ) spca_lasso.fit(Y) assert_array_almost_equal(spca_lasso.components_, spca_lars.components_) @if_safe_multiprocessing_with_blas -def test_fit_transform_parallel(): +def test_fit_transform_parallel(global_random_seed): alpha = 1 - rng = np.random.RandomState(0) + rng = np.random.RandomState(global_random_seed) Y, _, _ = generate_toy_data(3, 10, (8, 8), random_state=rng) # wide array - spca_lars = SparsePCA(n_components=3, method='lars', alpha=alpha, - random_state=0) + spca_lars = SparsePCA( + n_components=3, method="lars", alpha=alpha, random_state=global_random_seed + ) spca_lars.fit(Y) U1 = spca_lars.transform(Y) # Test multiple CPUs - spca = SparsePCA(n_components=3, n_jobs=2, method='lars', alpha=alpha, - random_state=0).fit(Y) + spca = SparsePCA( + n_components=3, + n_jobs=2, + method="lars", + alpha=alpha, + random_state=global_random_seed, + ).fit(Y) U2 = spca.transform(Y) assert not np.all(spca_lars.components_ == 0) assert_array_almost_equal(U1, U2) -def test_transform_nan(): +def test_transform_nan(global_random_seed): # Test that SparsePCA won't return NaN when there is 0 feature in all # samples. - rng = np.random.RandomState(0) + rng = np.random.RandomState(global_random_seed) Y, _, _ = generate_toy_data(3, 10, (8, 8), random_state=rng) # wide array Y[:, 0] = 0 - estimator = SparsePCA(n_components=8) + estimator = SparsePCA(n_components=8, random_state=global_random_seed) assert not np.any(np.isnan(estimator.fit_transform(Y))) -def test_fit_transform_tall(): - rng = np.random.RandomState(0) +def test_fit_transform_tall(global_random_seed): + rng = np.random.RandomState(global_random_seed) Y, _, _ = generate_toy_data(3, 65, (8, 8), random_state=rng) # tall array - spca_lars = SparsePCA(n_components=3, method='lars', random_state=rng) + spca_lars = SparsePCA(n_components=3, method="lars", random_state=rng) U1 = spca_lars.fit_transform(Y) - spca_lasso = SparsePCA(n_components=3, method='cd', random_state=rng) + spca_lasso = SparsePCA(n_components=3, method="cd", random_state=rng) U2 = spca_lasso.fit(Y).transform(Y) assert_array_almost_equal(U1, U2) -def test_initialization(): - rng = np.random.RandomState(0) +def test_initialization(global_random_seed): + rng = np.random.RandomState(global_random_seed) U_init = rng.randn(5, 3) V_init = rng.randn(3, 4) - model = SparsePCA(n_components=3, U_init=U_init, V_init=V_init, max_iter=0, - random_state=rng) + model = SparsePCA( + n_components=3, U_init=U_init, V_init=V_init, max_iter=0, random_state=rng + ) model.fit(rng.randn(5, 4)) - assert_allclose(model.components_, - V_init / np.linalg.norm(V_init, axis=1)[:, None]) + + expected_components = V_init / np.linalg.norm(V_init, axis=1, keepdims=True) + expected_components = svd_flip(u=expected_components.T, v=None)[0].T + assert_allclose(model.components_, expected_components) def test_mini_batch_correct_shapes(): rng = np.random.RandomState(0) X = rng.randn(12, 10) - pca = MiniBatchSparsePCA(n_components=8, random_state=rng) + pca = MiniBatchSparsePCA(n_components=8, max_iter=1, random_state=rng) U = pca.fit_transform(X) assert pca.components_.shape == (8, 10) assert U.shape == (12, 8) # test overcomplete decomposition - pca = MiniBatchSparsePCA(n_components=13, random_state=rng) + pca = MiniBatchSparsePCA(n_components=13, max_iter=1, random_state=rng) U = pca.fit_transform(X) assert pca.components_.shape == (13, 10) assert U.shape == (12, 13) -# XXX: test always skipped -@pytest.mark.skipif(True, reason="skipping mini_batch_fit_transform.") -def test_mini_batch_fit_transform(): +def test_scaling_fit_transform(global_random_seed): alpha = 1 - rng = np.random.RandomState(0) - Y, _, _ = generate_toy_data(3, 10, (8, 8), random_state=rng) # wide array - spca_lars = MiniBatchSparsePCA(n_components=3, random_state=0, - alpha=alpha).fit(Y) - U1 = spca_lars.transform(Y) - # Test multiple CPUs - if sys.platform == 'win32': # fake parallelism for win32 - import joblib - _mp = joblib.parallel.multiprocessing - joblib.parallel.multiprocessing = None - try: - spca = MiniBatchSparsePCA(n_components=3, n_jobs=2, alpha=alpha, - random_state=0) - U2 = spca.fit(Y).transform(Y) - finally: - joblib.parallel.multiprocessing = _mp - else: # we can efficiently use parallelism - spca = MiniBatchSparsePCA(n_components=3, n_jobs=2, alpha=alpha, - random_state=0) - U2 = spca.fit(Y).transform(Y) - assert not np.all(spca_lars.components_ == 0) - assert_array_almost_equal(U1, U2) - # Test that CD gives similar results - spca_lasso = MiniBatchSparsePCA(n_components=3, method='cd', alpha=alpha, - random_state=0).fit(Y) - assert_array_almost_equal(spca_lasso.components_, spca_lars.components_) - - -def test_scaling_fit_transform(): - alpha = 1 - rng = np.random.RandomState(0) + rng = np.random.RandomState(global_random_seed) Y, _, _ = generate_toy_data(3, 1000, (8, 8), random_state=rng) - spca_lars = SparsePCA(n_components=3, method='lars', alpha=alpha, - random_state=rng) + spca_lars = SparsePCA(n_components=3, method="lars", alpha=alpha, random_state=rng) results_train = spca_lars.fit_transform(Y) results_test = spca_lars.transform(Y[:10]) assert_allclose(results_train[0], results_test[0]) -def test_pca_vs_spca(): - rng = np.random.RandomState(0) +def test_pca_vs_spca(global_random_seed): + rng = np.random.RandomState(global_random_seed) Y, _, _ = generate_toy_data(3, 1000, (8, 8), random_state=rng) Z, _, _ = generate_toy_data(3, 10, (8, 8), random_state=rng) - spca = SparsePCA(alpha=0, ridge_alpha=0, n_components=2) - pca = PCA(n_components=2) + spca = SparsePCA(alpha=0, ridge_alpha=0, n_components=2, random_state=rng) + pca = PCA(n_components=2, random_state=rng) pca.fit(Y) spca.fit(Y) results_test_pca = pca.transform(Z) results_test_spca = spca.transform(Z) - assert_allclose(np.abs(spca.components_.dot(pca.components_.T)), - np.eye(2), atol=1e-5) + assert_allclose( + np.abs(spca.components_.dot(pca.components_.T)), np.eye(2), atol=1e-4 + ) results_test_pca *= np.sign(results_test_pca[0, :]) results_test_spca *= np.sign(results_test_spca[0, :]) - assert_allclose(results_test_pca, results_test_spca) + assert_allclose(results_test_pca, results_test_spca, atol=1e-4) @pytest.mark.parametrize("SPCA", [SparsePCA, MiniBatchSparsePCA]) @@ -202,3 +186,162 @@ def test_spca_n_components_(SPCA, n_components): assert model.n_components_ == n_components else: assert model.n_components_ == n_features + + +@pytest.mark.parametrize("SPCA", (SparsePCA, MiniBatchSparsePCA)) +@pytest.mark.parametrize("method", ("lars", "cd")) +@pytest.mark.parametrize( + "data_type, expected_type", + ( + (np.float32, np.float32), + (np.float64, np.float64), + (np.int32, np.float64), + (np.int64, np.float64), + ), +) +def test_sparse_pca_dtype_match(SPCA, method, data_type, expected_type): + # Verify output matrix dtype + n_samples, n_features, n_components = 12, 10, 3 + rng = np.random.RandomState(0) + input_array = rng.randn(n_samples, n_features).astype(data_type) + model = SPCA(n_components=n_components, method=method) + transformed = model.fit_transform(input_array) + + assert transformed.dtype == expected_type + assert model.components_.dtype == expected_type + + +@pytest.mark.parametrize("SPCA", (SparsePCA, MiniBatchSparsePCA)) +@pytest.mark.parametrize("method", ("lars", "cd")) +def test_sparse_pca_numerical_consistency(SPCA, method, global_random_seed): + # Verify numericall consistentency among np.float32 and np.float64 + n_samples, n_features, n_components = 20, 20, 5 + input_array = make_low_rank_matrix( + n_samples=n_samples, + n_features=n_features, + effective_rank=n_components, + random_state=global_random_seed, + ) + + model_32 = SPCA( + n_components=n_components, + method=method, + random_state=global_random_seed, + ) + transformed_32 = model_32.fit_transform(input_array.astype(np.float32)) + + model_64 = SPCA( + n_components=n_components, + method=method, + random_state=global_random_seed, + ) + transformed_64 = model_64.fit_transform(input_array.astype(np.float64)) + assert_allclose(transformed_64, transformed_32) + assert_allclose(model_64.components_, model_32.components_) + + +@pytest.mark.parametrize("SPCA", [SparsePCA, MiniBatchSparsePCA]) +def test_spca_feature_names_out(SPCA): + """Check feature names out for *SparsePCA.""" + rng = np.random.RandomState(0) + n_samples, n_features = 12, 10 + X = rng.randn(n_samples, n_features) + + model = SPCA(n_components=4).fit(X) + names = model.get_feature_names_out() + + estimator_name = SPCA.__name__.lower() + assert_array_equal([f"{estimator_name}{i}" for i in range(4)], names) + + +def test_spca_early_stopping(global_random_seed): + """Check that `tol` and `max_no_improvement` act as early stopping.""" + rng = np.random.RandomState(global_random_seed) + n_samples, n_features = 50, 10 + X = rng.randn(n_samples, n_features) + + # vary the tolerance to force the early stopping of one of the model + model_early_stopped = MiniBatchSparsePCA( + max_iter=100, tol=0.5, random_state=global_random_seed + ).fit(X) + model_not_early_stopped = MiniBatchSparsePCA( + max_iter=100, tol=1e-3, random_state=global_random_seed + ).fit(X) + assert model_early_stopped.n_iter_ < model_not_early_stopped.n_iter_ + + # force the max number of no improvement to a large value to check that + # it does help to early stop + model_early_stopped = MiniBatchSparsePCA( + max_iter=100, tol=1e-6, max_no_improvement=2, random_state=global_random_seed + ).fit(X) + model_not_early_stopped = MiniBatchSparsePCA( + max_iter=100, tol=1e-6, max_no_improvement=100, random_state=global_random_seed + ).fit(X) + assert model_early_stopped.n_iter_ < model_not_early_stopped.n_iter_ + + +def test_equivalence_components_pca_spca(global_random_seed): + """Check the equivalence of the components found by PCA and SparsePCA. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/23932 + """ + rng = np.random.RandomState(global_random_seed) + X = rng.randn(50, 4) + + n_components = 2 + pca = PCA( + n_components=n_components, + svd_solver="randomized", + random_state=0, + ).fit(X) + spca = SparsePCA( + n_components=n_components, + method="lars", + ridge_alpha=0, + alpha=0, + random_state=0, + ).fit(X) + + assert_allclose(pca.components_, spca.components_) + + +def test_sparse_pca_inverse_transform(global_random_seed): + """Check that `inverse_transform` in `SparsePCA` and `PCA` are similar.""" + rng = np.random.RandomState(global_random_seed) + n_samples, n_features = 10, 5 + X = rng.randn(n_samples, n_features) + + n_components = 2 + spca = SparsePCA( + n_components=n_components, + alpha=1e-12, + ridge_alpha=1e-12, + random_state=global_random_seed, + ) + pca = PCA(n_components=n_components, random_state=global_random_seed) + X_trans_spca = spca.fit_transform(X) + X_trans_pca = pca.fit_transform(X) + assert_allclose( + spca.inverse_transform(X_trans_spca), pca.inverse_transform(X_trans_pca) + ) + + +@pytest.mark.parametrize("SPCA", [SparsePCA, MiniBatchSparsePCA]) +def test_transform_inverse_transform_round_trip(SPCA, global_random_seed): + """Check the `transform` and `inverse_transform` round trip with no loss of + information. + """ + rng = np.random.RandomState(global_random_seed) + n_samples, n_features = 10, 5 + X = rng.randn(n_samples, n_features) + + n_components = n_features + spca = SPCA( + n_components=n_components, + alpha=1e-12, + ridge_alpha=1e-12, + random_state=global_random_seed, + ) + X_trans_spca = spca.fit_transform(X) + assert_allclose(spca.inverse_transform(X_trans_spca), X) diff --git a/sklearn/decomposition/tests/test_truncated_svd.py b/sklearn/decomposition/tests/test_truncated_svd.py index faf3ca39446c3..07b35c873ee3e 100644 --- a/sklearn/decomposition/tests/test_truncated_svd.py +++ b/sklearn/decomposition/tests/test_truncated_svd.py @@ -1,18 +1,17 @@ """Test truncated SVD transformer.""" import numpy as np -import scipy.sparse as sp - import pytest +import scipy.sparse as sp -from sklearn.decomposition import TruncatedSVD, PCA +from sklearn.decomposition import PCA, TruncatedSVD from sklearn.utils import check_random_state -from sklearn.utils._testing import assert_array_less, assert_allclose +from sklearn.utils._testing import assert_allclose, assert_array_less -SVD_SOLVERS = ['arpack', 'randomized'] +SVD_SOLVERS = ["arpack", "randomized"] -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def X_sparse(): # Make an X that looks somewhat like a small tf-idf matrix. rng = check_random_state(42) @@ -21,12 +20,12 @@ def X_sparse(): return X -@pytest.mark.parametrize("solver", ['randomized']) -@pytest.mark.parametrize('kind', ('dense', 'sparse')) +@pytest.mark.parametrize("solver", ["randomized"]) +@pytest.mark.parametrize("kind", ("dense", "sparse")) def test_solvers(X_sparse, solver, kind): - X = X_sparse if kind == 'sparse' else X_sparse.toarray() + X = X_sparse if kind == "sparse" else X_sparse.toarray() svd_a = TruncatedSVD(30, algorithm="arpack") - svd = TruncatedSVD(30, algorithm=solver, random_state=42) + svd = TruncatedSVD(30, algorithm=solver, random_state=42, n_oversamples=100) Xa = svd_a.fit_transform(X)[:, :6] Xr = svd.fit_transform(X)[:, :6] @@ -39,7 +38,7 @@ def test_solvers(X_sparse, solver, kind): assert_allclose(comp_a[9:], comp[9:], atol=1e-2) -@pytest.mark.parametrize("n_components", (10, 25, 41)) +@pytest.mark.parametrize("n_components", (10, 25, 41, 55)) def test_attributes(n_components, X_sparse): n_features = X_sparse.shape[1] tsvd = TruncatedSVD(n_components).fit(X_sparse) @@ -47,20 +46,24 @@ def test_attributes(n_components, X_sparse): assert tsvd.components_.shape == (n_components, n_features) -@pytest.mark.parametrize('algorithm', SVD_SOLVERS) -def test_too_many_components(algorithm, X_sparse): - n_features = X_sparse.shape[1] - for n_components in (n_features, n_features + 1): - tsvd = TruncatedSVD(n_components=n_components, algorithm=algorithm) - with pytest.raises(ValueError): - tsvd.fit(X_sparse) +@pytest.mark.parametrize( + "algorithm, n_components", + [ + ("arpack", 55), + ("arpack", 56), + ("randomized", 56), + ], +) +def test_too_many_components(X_sparse, algorithm, n_components): + tsvd = TruncatedSVD(n_components=n_components, algorithm=algorithm) + with pytest.raises(ValueError): + tsvd.fit(X_sparse) -@pytest.mark.parametrize('fmt', ("array", "csr", "csc", "coo", "lil")) +@pytest.mark.parametrize("fmt", ("array", "csr", "csc", "coo", "lil")) def test_sparse_formats(fmt, X_sparse): n_samples = X_sparse.shape[0] - Xfmt = (X_sparse.toarray() - if fmt == "dense" else getattr(X_sparse, "to" + fmt)()) + Xfmt = X_sparse.toarray() if fmt == "dense" else getattr(X_sparse, "to" + fmt)() tsvd = TruncatedSVD(n_components=11) Xtrans = tsvd.fit_transform(Xfmt) assert Xtrans.shape == (n_samples, 11) @@ -68,7 +71,7 @@ def test_sparse_formats(fmt, X_sparse): assert Xtrans.shape == (n_samples, 11) -@pytest.mark.parametrize('algo', SVD_SOLVERS) +@pytest.mark.parametrize("algo", SVD_SOLVERS) def test_inverse_transform(algo, X_sparse): # We need a lot of components for the reconstruction to be "almost # equal" in all positions. XXX Test means or sums instead? @@ -86,11 +89,11 @@ def test_integers(X_sparse): assert Xtrans.shape == (n_samples, tsvd.n_components) -@pytest.mark.parametrize('kind', ('dense', 'sparse')) -@pytest.mark.parametrize('n_components', [10, 20]) -@pytest.mark.parametrize('solver', SVD_SOLVERS) +@pytest.mark.parametrize("kind", ("dense", "sparse")) +@pytest.mark.parametrize("n_components", [10, 20]) +@pytest.mark.parametrize("solver", SVD_SOLVERS) def test_explained_variance(X_sparse, kind, n_components, solver): - X = X_sparse if kind == 'sparse' else X_sparse.toarray() + X = X_sparse if kind == "sparse" else X_sparse.toarray() svd = TruncatedSVD(n_components, algorithm=solver) X_tr = svd.fit_transform(X) # Assert that all the values are greater than 0 @@ -110,10 +113,10 @@ def test_explained_variance(X_sparse, kind, n_components, solver): ) -@pytest.mark.parametrize('kind', ('dense', 'sparse')) -@pytest.mark.parametrize('solver', SVD_SOLVERS) +@pytest.mark.parametrize("kind", ("dense", "sparse")) +@pytest.mark.parametrize("solver", SVD_SOLVERS) def test_explained_variance_components_10_20(X_sparse, kind, solver): - X = X_sparse if kind == 'sparse' else X_sparse.toarray() + X = X_sparse if kind == "sparse" else X_sparse.toarray() svd_10 = TruncatedSVD(10, algorithm=solver, n_iter=10).fit(X) svd_20 = TruncatedSVD(20, algorithm=solver, n_iter=10).fit(X) @@ -126,42 +129,43 @@ def test_explained_variance_components_10_20(X_sparse, kind, solver): # Assert that 20 components has higher explained variance than 10 assert ( - svd_20.explained_variance_ratio_.sum() > - svd_10.explained_variance_ratio_.sum() + svd_20.explained_variance_ratio_.sum() > svd_10.explained_variance_ratio_.sum() ) -@pytest.mark.parametrize('solver', SVD_SOLVERS) -def test_singular_values_consistency(solver): +@pytest.mark.parametrize("solver", SVD_SOLVERS) +def test_singular_values_consistency(solver, global_random_seed): # Check that the TruncatedSVD output has the correct singular values - rng = np.random.RandomState(0) + rng = np.random.RandomState(global_random_seed) n_samples, n_features = 100, 80 X = rng.randn(n_samples, n_features) - pca = TruncatedSVD(n_components=2, algorithm=solver, - random_state=rng).fit(X) + pca = TruncatedSVD(n_components=2, algorithm=solver, random_state=rng).fit(X) # Compare to the Frobenius norm X_pca = pca.transform(X) - assert_allclose(np.sum(pca.singular_values_**2.0), - np.linalg.norm(X_pca, "fro")**2.0, rtol=1e-2) + assert_allclose( + np.sum(pca.singular_values_**2.0), + np.linalg.norm(X_pca, "fro") ** 2.0, + rtol=1e-2, + ) # Compare to the 2-norms of the score vectors - assert_allclose(pca.singular_values_, - np.sqrt(np.sum(X_pca**2.0, axis=0)), rtol=1e-2) + assert_allclose( + pca.singular_values_, np.sqrt(np.sum(X_pca**2.0, axis=0)), rtol=1e-2 + ) -@pytest.mark.parametrize('solver', SVD_SOLVERS) -def test_singular_values_expected(solver): +@pytest.mark.parametrize("solver", SVD_SOLVERS) +def test_singular_values_expected(solver, global_random_seed): # Set the singular values and see what we get back - rng = np.random.RandomState(0) + rng = np.random.RandomState(global_random_seed) n_samples = 100 n_features = 110 X = rng.randn(n_samples, n_features) - pca = TruncatedSVD(n_components=3, algorithm=solver, - random_state=rng) + pca = TruncatedSVD(n_components=3, algorithm=solver, random_state=rng) X_pca = pca.fit_transform(X) X_pca /= np.sqrt(np.sum(X_pca**2.0, axis=0)) @@ -182,8 +186,8 @@ def test_truncated_svd_eq_pca(X_sparse): params = dict(n_components=10, random_state=42) - svd = TruncatedSVD(algorithm='arpack', **params) - pca = PCA(svd_solver='arpack', **params) + svd = TruncatedSVD(algorithm="arpack", **params) + pca = PCA(svd_solver="arpack", **params) Xt_svd = svd.fit_transform(X_c) Xt_pca = pca.fit_transform(X_c) @@ -193,14 +197,16 @@ def test_truncated_svd_eq_pca(X_sparse): assert_allclose(svd.components_, pca.components_) -@pytest.mark.parametrize("algorithm, tol", [ - ('randomized', 0.), ('arpack', 1e-6), ('arpack', 0.)]) -@pytest.mark.parametrize('kind', ('dense', 'sparse')) +@pytest.mark.parametrize( + "algorithm, tol", [("randomized", 0.0), ("arpack", 1e-6), ("arpack", 0.0)] +) +@pytest.mark.parametrize("kind", ("dense", "sparse")) def test_fit_transform(X_sparse, algorithm, tol, kind): # fit_transform(X) should equal fit(X).transform(X) - X = X_sparse if kind == 'sparse' else X_sparse.toarray() - svd = TruncatedSVD(n_components=5, n_iter=7, random_state=42, - algorithm=algorithm, tol=tol) + X = X_sparse if kind == "sparse" else X_sparse.toarray() + svd = TruncatedSVD( + n_components=5, n_iter=7, random_state=42, algorithm=algorithm, tol=tol + ) X_transformed_1 = svd.fit_transform(X) X_transformed_2 = svd.fit(X).transform(X) assert_allclose(X_transformed_1, X_transformed_2) diff --git a/sklearn/discriminant_analysis.py b/sklearn/discriminant_analysis.py index c5c18ac9136d2..6df26a05a8781 100644 --- a/sklearn/discriminant_analysis.py +++ b/sklearn/discriminant_analysis.py @@ -1,31 +1,32 @@ -""" -Linear Discriminant Analysis and Quadratic Discriminant Analysis -""" +"""Linear and quadratic discriminant analysis.""" -# Authors: Clemens Brunner -# Martin Billinger -# Matthieu Perrot -# Mathieu Blondel - -# License: BSD 3-Clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import warnings +from numbers import Integral, Real + import numpy as np +import scipy.linalg from scipy import linalg -from scipy.special import expit -from .base import BaseEstimator, TransformerMixin, ClassifierMixin +from .base import ( + BaseEstimator, + ClassifierMixin, + ClassNamePrefixFeaturesOutMixin, + TransformerMixin, + _fit_context, +) +from .covariance import empirical_covariance, ledoit_wolf, shrunk_covariance from .linear_model._base import LinearClassifierMixin -from .covariance import ledoit_wolf, empirical_covariance, shrunk_covariance -from .utils.multiclass import unique_labels -from .utils.validation import check_is_fitted -from .utils.multiclass import check_classification_targets -from .utils.extmath import softmax from .preprocessing import StandardScaler -from .utils.validation import _deprecate_positional_args - +from .utils._array_api import _expit, device, get_namespace, size +from .utils._param_validation import HasMethods, Interval, StrOptions +from .utils.extmath import softmax +from .utils.multiclass import check_classification_targets, unique_labels +from .utils.validation import check_is_fitted, validate_data -__all__ = ['LinearDiscriminantAnalysis', 'QuadraticDiscriminantAnalysis'] +__all__ = ["LinearDiscriminantAnalysis", "QuadraticDiscriminantAnalysis"] def _cov(X, shrinkage=None, covariance_estimator=None): @@ -62,30 +63,28 @@ def _cov(X, shrinkage=None, covariance_estimator=None): if covariance_estimator is None: shrinkage = "empirical" if shrinkage is None else shrinkage if isinstance(shrinkage, str): - if shrinkage == 'auto': + if shrinkage == "auto": sc = StandardScaler() # standardize features X = sc.fit_transform(X) s = ledoit_wolf(X)[0] # rescale s = sc.scale_[:, np.newaxis] * s * sc.scale_[np.newaxis, :] - elif shrinkage == 'empirical': + elif shrinkage == "empirical": s = empirical_covariance(X) - else: - raise ValueError('unknown shrinkage parameter') - elif isinstance(shrinkage, float) or isinstance(shrinkage, int): - if shrinkage < 0 or shrinkage > 1: - raise ValueError('shrinkage parameter must be between 0 and 1') + elif isinstance(shrinkage, Real): s = shrunk_covariance(empirical_covariance(X), shrinkage) - else: - raise TypeError('shrinkage must be a float or a string') else: if shrinkage is not None and shrinkage != 0: - raise ValueError("covariance_estimator and shrinkage parameters " - "are not None. Only one of the two can be set.") + raise ValueError( + "covariance_estimator and shrinkage parameters " + "are not None. Only one of the two can be set." + ) covariance_estimator.fit(X) - if not hasattr(covariance_estimator, 'covariance_'): - raise ValueError("%s does not have a covariance_ attribute" % - covariance_estimator.__class__.__name__) + if not hasattr(covariance_estimator, "covariance_"): + raise ValueError( + "%s does not have a covariance_ attribute" + % covariance_estimator.__class__.__name__ + ) s = covariance_estimator.covariance_ return s @@ -106,11 +105,19 @@ def _class_means(X, y): means : array-like of shape (n_classes, n_features) Class means. """ - classes, y = np.unique(y, return_inverse=True) - cnt = np.bincount(y) - means = np.zeros(shape=(len(classes), X.shape[1])) - np.add.at(means, y, X) - means /= cnt[:, None] + xp, is_array_api_compliant = get_namespace(X) + classes, y = xp.unique_inverse(y) + means = xp.zeros((classes.shape[0], X.shape[1]), device=device(X), dtype=X.dtype) + + if is_array_api_compliant: + for i in range(classes.shape[0]): + means[i, :] = xp.mean(X[y == i], axis=0) + else: + # TODO: Explore the choice of using bincount + add.at as it seems sub optimal + # from a performance-wise + cnt = np.bincount(y) + np.add.at(means, y, X) + means /= cnt[:, None] return means @@ -157,15 +164,95 @@ def _class_cov(X, y, priors, shrinkage=None, covariance_estimator=None): cov = np.zeros(shape=(X.shape[1], X.shape[1])) for idx, group in enumerate(classes): Xg = X[y == group, :] - cov += priors[idx] * np.atleast_2d( - _cov(Xg, shrinkage, covariance_estimator)) + cov += priors[idx] * np.atleast_2d(_cov(Xg, shrinkage, covariance_estimator)) return cov -class LinearDiscriminantAnalysis(LinearClassifierMixin, - TransformerMixin, - BaseEstimator): - """Linear Discriminant Analysis +class DiscriminantAnalysisPredictionMixin: + """Mixin class for QuadraticDiscriminantAnalysis and NearestCentroid.""" + + def decision_function(self, X): + """Apply decision function to an array of samples. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Array of samples (test vectors). + + Returns + ------- + y_scores : ndarray of shape (n_samples,) or (n_samples, n_classes) + Decision function values related to each class, per sample. + In the two-class case, the shape is `(n_samples,)`, giving the + log likelihood ratio of the positive class. + """ + y_scores = self._decision_function(X) + if len(self.classes_) == 2: + return y_scores[:, 1] - y_scores[:, 0] + return y_scores + + def predict(self, X): + """Perform classification on an array of vectors `X`. + + Returns the class label for each sample. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Input vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. + + Returns + ------- + y_pred : ndarray of shape (n_samples,) + Class label for each sample. + """ + scores = self._decision_function(X) + return self.classes_.take(scores.argmax(axis=1)) + + def predict_proba(self, X): + """Estimate class probabilities. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Input data. + + Returns + ------- + y_proba : ndarray of shape (n_samples, n_classes) + Probability estimate of the sample for each class in the + model, where classes are ordered as they are in `self.classes_`. + """ + return np.exp(self.predict_log_proba(X)) + + def predict_log_proba(self, X): + """Estimate log class probabilities. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Input data. + + Returns + ------- + y_log_proba : ndarray of shape (n_samples, n_classes) + Estimated log probabilities. + """ + scores = self._decision_function(X) + log_likelihood = scores - scores.max(axis=1)[:, np.newaxis] + return log_likelihood - np.log( + np.exp(log_likelihood).sum(axis=1)[:, np.newaxis] + ) + + +class LinearDiscriminantAnalysis( + ClassNamePrefixFeaturesOutMixin, + LinearClassifierMixin, + TransformerMixin, + BaseEstimator, +): + """Linear Discriminant Analysis. A classifier with a linear decision boundary, generated by fitting class conditional densities to the data and using Bayes' rule. @@ -178,7 +265,11 @@ class LinearDiscriminantAnalysis(LinearClassifierMixin, `transform` method. .. versionadded:: 0.17 - *LinearDiscriminantAnalysis*. + + For a comparison between + :class:`~sklearn.discriminant_analysis.LinearDiscriminantAnalysis` + and :class:`~sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis`, see + :ref:`sphx_glr_auto_examples_classification_plot_lda_qda.py`. Read more in the :ref:`User Guide `. @@ -194,6 +285,10 @@ class LinearDiscriminantAnalysis(LinearClassifierMixin, - 'eigen': Eigenvalue decomposition. Can be combined with shrinkage or custom covariance estimator. + .. versionchanged:: 1.2 + `solver="svd"` now has experimental Array API support. See the + :ref:`Array API User Guide ` for more details. + shrinkage : 'auto' or float, default=None Shrinkage parameter, possible values: - None: no shrinkage (default). @@ -203,6 +298,9 @@ class LinearDiscriminantAnalysis(LinearClassifierMixin, This should be left to None if `covariance_estimator` is used. Note that shrinkage works only with 'lsqr' and 'eigen' solvers. + For a usage example, see + :ref:`sphx_glr_auto_examples_classification_plot_lda.py`. + priors : array-like of shape (n_classes,), default=None The class prior probabilities. By default, the class proportions are inferred from the training data. @@ -213,8 +311,11 @@ class LinearDiscriminantAnalysis(LinearClassifierMixin, min(n_classes - 1, n_features). This parameter only affects the `transform` method. + For a usage example, see + :ref:`sphx_glr_auto_examples_decomposition_plot_pca_vs_lda.py`. + store_covariance : bool, default=False - If True, explicitely compute the weighted within-class covariance + If True, explicitly compute the weighted within-class covariance matrix when solver is 'svd'. The matrix is always computed and stored for the other solvers. @@ -279,6 +380,17 @@ class LinearDiscriminantAnalysis(LinearClassifierMixin, classes_ : array-like of shape (n_classes,) Unique class labels. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + See Also -------- QuadraticDiscriminantAnalysis : Quadratic Discriminant Analysis. @@ -296,9 +408,26 @@ class LinearDiscriminantAnalysis(LinearClassifierMixin, [1] """ - def __init__(self, solver='svd', shrinkage=None, priors=None, - n_components=None, store_covariance=False, tol=1e-4, - covariance_estimator=None): + _parameter_constraints: dict = { + "solver": [StrOptions({"svd", "lsqr", "eigen"})], + "shrinkage": [StrOptions({"auto"}), Interval(Real, 0, 1, closed="both"), None], + "n_components": [Interval(Integral, 1, None, closed="left"), None], + "priors": ["array-like", None], + "store_covariance": ["boolean"], + "tol": [Interval(Real, 0, None, closed="left")], + "covariance_estimator": [HasMethods("fit"), None], + } + + def __init__( + self, + solver="svd", + shrinkage=None, + priors=None, + n_components=None, + store_covariance=False, + tol=1e-4, + covariance_estimator=None, + ): self.solver = solver self.shrinkage = shrinkage self.priors = priors @@ -307,7 +436,7 @@ def __init__(self, solver='svd', shrinkage=None, priors=None, self.tol = tol # used only in svd solver self.covariance_estimator = covariance_estimator - def _solve_lsqr(self, X, y, shrinkage, covariance_estimator): + def _solve_lstsq(self, X, y, shrinkage, covariance_estimator): """Least squares solver. The least squares solver computes a straightforward solution of the @@ -355,14 +484,15 @@ def _solve_lsqr(self, X, y, shrinkage, covariance_estimator): 0-471-05669-3. """ self.means_ = _class_means(X, y) - self.covariance_ = _class_cov(X, y, self.priors_, shrinkage, - covariance_estimator) + self.covariance_ = _class_cov( + X, y, self.priors_, shrinkage, covariance_estimator + ) self.coef_ = linalg.lstsq(self.covariance_, self.means_.T)[0].T - self.intercept_ = (-0.5 * np.diag(np.dot(self.means_, self.coef_.T)) + - np.log(self.priors_)) + self.intercept_ = -0.5 * np.diag(np.dot(self.means_, self.coef_.T)) + np.log( + self.priors_ + ) - def _solve_eigen(self, X, y, shrinkage, - covariance_estimator): + def _solve_eigen(self, X, y, shrinkage, covariance_estimator): """Eigenvalue solver. The eigenvalue solver computes the optimal solution of the Rayleigh @@ -408,22 +538,25 @@ class scatter). This solver supports both classification and 0-471-05669-3. """ self.means_ = _class_means(X, y) - self.covariance_ = _class_cov(X, y, self.priors_, shrinkage, - covariance_estimator) + self.covariance_ = _class_cov( + X, y, self.priors_, shrinkage, covariance_estimator + ) Sw = self.covariance_ # within scatter St = _cov(X, shrinkage, covariance_estimator) # total scatter Sb = St - Sw # between scatter evals, evecs = linalg.eigh(Sb, Sw) - self.explained_variance_ratio_ = np.sort(evals / np.sum(evals) - )[::-1][:self._max_components] + self.explained_variance_ratio_ = np.sort(evals / np.sum(evals))[::-1][ + : self._max_components + ] evecs = evecs[:, np.argsort(evals)[::-1]] # sort eigenvectors self.scalings_ = evecs self.coef_ = np.dot(self.means_, evecs).dot(evecs.T) - self.intercept_ = (-0.5 * np.diag(np.dot(self.means_, self.coef_.T)) + - np.log(self.priors_)) + self.intercept_ = -0.5 * np.diag(np.dot(self.means_, self.coef_.T)) + np.log( + self.priors_ + ) def _solve_svd(self, X, y): """SVD solver. @@ -436,8 +569,15 @@ def _solve_svd(self, X, y): y : array-like of shape (n_samples,) or (n_samples, n_targets) Target values. """ + xp, is_array_api_compliant = get_namespace(X) + + if is_array_api_compliant: + svd = xp.linalg.svd + else: + svd = scipy.linalg.svd + n_samples, n_features = X.shape - n_classes = len(self.classes_) + n_classes = self.classes_.shape[0] self.means_ = _class_means(X, y) if self.store_covariance: @@ -445,56 +585,62 @@ def _solve_svd(self, X, y): Xc = [] for idx, group in enumerate(self.classes_): - Xg = X[y == group, :] - Xc.append(Xg - self.means_[idx]) + Xg = X[y == group] + Xc.append(Xg - self.means_[idx, :]) - self.xbar_ = np.dot(self.priors_, self.means_) + self.xbar_ = self.priors_ @ self.means_ - Xc = np.concatenate(Xc, axis=0) + Xc = xp.concat(Xc, axis=0) # 1) within (univariate) scaling by with classes std-dev - std = Xc.std(axis=0) + std = xp.std(Xc, axis=0) # avoid division by zero in normalization - std[std == 0] = 1. - fac = 1. / (n_samples - n_classes) + std[std == 0] = 1.0 + fac = xp.asarray(1.0 / (n_samples - n_classes), dtype=X.dtype, device=device(X)) # 2) Within variance scaling - X = np.sqrt(fac) * (Xc / std) + X = xp.sqrt(fac) * (Xc / std) # SVD of centered (within)scaled data - U, S, Vt = linalg.svd(X, full_matrices=False) + U, S, Vt = svd(X, full_matrices=False) - rank = np.sum(S > self.tol) + rank = xp.sum(xp.astype(S > self.tol, xp.int32)) # Scaling of within covariance is: V' 1/S - scalings = (Vt[:rank] / std).T / S[:rank] + scalings = (Vt[:rank, :] / std).T / S[:rank] + fac = 1.0 if n_classes == 1 else 1.0 / (n_classes - 1) # 3) Between variance scaling # Scale weighted centers - X = np.dot(((np.sqrt((n_samples * self.priors_) * fac)) * - (self.means_ - self.xbar_).T).T, scalings) + X = ( + (xp.sqrt((n_samples * self.priors_) * fac)) * (self.means_ - self.xbar_).T + ).T @ scalings # Centers are living in a space with n_classes-1 dim (maximum) # Use SVD to find projection in the space spanned by the # (n_classes) centers - _, S, Vt = linalg.svd(X, full_matrices=0) - - self.explained_variance_ratio_ = (S**2 / np.sum( - S**2))[:self._max_components] - rank = np.sum(S > self.tol * S[0]) - self.scalings_ = np.dot(scalings, Vt.T[:, :rank]) - coef = np.dot(self.means_ - self.xbar_, self.scalings_) - self.intercept_ = (-0.5 * np.sum(coef ** 2, axis=1) + - np.log(self.priors_)) - self.coef_ = np.dot(coef, self.scalings_.T) - self.intercept_ -= np.dot(self.xbar_, self.coef_.T) + _, S, Vt = svd(X, full_matrices=False) + if self._max_components == 0: + self.explained_variance_ratio_ = xp.empty((0,), dtype=S.dtype) + else: + self.explained_variance_ratio_ = (S**2 / xp.sum(S**2))[ + : self._max_components + ] + + rank = xp.sum(xp.astype(S > self.tol * S[0], xp.int32)) + self.scalings_ = scalings @ Vt.T[:, :rank] + coef = (self.means_ - self.xbar_) @ self.scalings_ + self.intercept_ = -0.5 * xp.sum(coef**2, axis=1) + xp.log(self.priors_) + self.coef_ = coef @ self.scalings_.T + self.intercept_ -= self.xbar_ @ self.coef_.T + + @_fit_context( + # LinearDiscriminantAnalysis.covariance_estimator is not validated yet + prefer_skip_nested_validation=False + ) def fit(self, X, y): - """Fit LinearDiscriminantAnalysis model according to the given - training data and parameters. - - .. versionchanged:: 0.19 - *store_covariance* has been moved to main constructor. + """Fit the Linear Discriminant Analysis model. - .. versionchanged:: 0.19 - *tol* has been moved to main constructor. + .. versionchanged:: 0.19 + `store_covariance` and `tol` has been moved to main constructor. Parameters ---------- @@ -503,68 +649,84 @@ def fit(self, X, y): y : array-like of shape (n_samples,) Target values. + + Returns + ------- + self : object + Fitted estimator. """ - X, y = self._validate_data(X, y, ensure_min_samples=2, estimator=self, - dtype=[np.float64, np.float32]) + xp, _ = get_namespace(X) + + X, y = validate_data( + self, X, y, ensure_min_samples=2, dtype=[xp.float64, xp.float32] + ) self.classes_ = unique_labels(y) n_samples, _ = X.shape - n_classes = len(self.classes_) + n_classes = self.classes_.shape[0] if n_samples == n_classes: - raise ValueError("The number of samples must be more " - "than the number of classes.") + raise ValueError( + "The number of samples must be more than the number of classes." + ) if self.priors is None: # estimate priors from sample - _, y_t = np.unique(y, return_inverse=True) # non-negative ints - self.priors_ = np.bincount(y_t) / float(len(y)) + _, cnts = xp.unique_counts(y) # non-negative ints + self.priors_ = xp.astype(cnts, X.dtype) / float(y.shape[0]) else: - self.priors_ = np.asarray(self.priors) + self.priors_ = xp.asarray(self.priors, dtype=X.dtype) - if (self.priors_ < 0).any(): + if xp.any(self.priors_ < 0): raise ValueError("priors must be non-negative") - if not np.isclose(self.priors_.sum(), 1.0): - warnings.warn("The priors do not sum to 1. Renormalizing", - UserWarning) + + if xp.abs(xp.sum(self.priors_) - 1.0) > 1e-5: + warnings.warn("The priors do not sum to 1. Renormalizing", UserWarning) self.priors_ = self.priors_ / self.priors_.sum() # Maximum number of components no matter what n_components is # specified: - max_components = min(len(self.classes_) - 1, X.shape[1]) + max_components = min(n_classes - 1, X.shape[1]) if self.n_components is None: self._max_components = max_components else: if self.n_components > max_components: raise ValueError( - "n_components cannot be larger than min(n_features, " - "n_classes - 1)." + "n_components cannot be larger than min(n_features, n_classes - 1)." ) self._max_components = self.n_components - if self.solver == 'svd': + if self.solver == "svd": if self.shrinkage is not None: - raise NotImplementedError('shrinkage not supported') + raise NotImplementedError("shrinkage not supported with 'svd' solver.") if self.covariance_estimator is not None: raise ValueError( - 'covariance estimator ' - 'is not supported ' - 'with svd solver. Try another solver') + "covariance estimator " + "is not supported " + "with svd solver. Try another solver" + ) self._solve_svd(X, y) - elif self.solver == 'lsqr': - self._solve_lsqr(X, y, shrinkage=self.shrinkage, - covariance_estimator=self.covariance_estimator) - elif self.solver == 'eigen': - self._solve_eigen(X, y, - shrinkage=self.shrinkage, - covariance_estimator=self.covariance_estimator) - else: - raise ValueError("unknown solver {} (valid solvers are 'svd', " - "'lsqr', and 'eigen').".format(self.solver)) - if self.classes_.size == 2: # treat binary case as a special case - self.coef_ = np.array(self.coef_[1, :] - self.coef_[0, :], ndmin=2, - dtype=X.dtype) - self.intercept_ = np.array(self.intercept_[1] - self.intercept_[0], - ndmin=1, dtype=X.dtype) + elif self.solver == "lsqr": + self._solve_lstsq( + X, + y, + shrinkage=self.shrinkage, + covariance_estimator=self.covariance_estimator, + ) + elif self.solver == "eigen": + self._solve_eigen( + X, + y, + shrinkage=self.shrinkage, + covariance_estimator=self.covariance_estimator, + ) + if size(self.classes_) == 2: # treat binary case as a special case + coef_ = xp.asarray(self.coef_[1, :] - self.coef_[0, :], dtype=X.dtype) + self.coef_ = xp.reshape(coef_, (1, -1)) + intercept_ = xp.asarray( + self.intercept_[1] - self.intercept_[0], dtype=X.dtype + ) + self.intercept_ = xp.reshape(intercept_, (1,)) + self._n_features_out = self._max_components return self def transform(self, X): @@ -577,21 +739,25 @@ def transform(self, X): Returns ------- - X_new : ndarray of shape (n_samples, n_components) - Transformed data. + X_new : ndarray of shape (n_samples, n_components) or \ + (n_samples, min(rank, n_components)) + Transformed data. In the case of the 'svd' solver, the shape + is (n_samples, min(rank, n_components)). """ - if self.solver == 'lsqr': - raise NotImplementedError("transform not implemented for 'lsqr' " - "solver (use 'svd' or 'eigen').") + if self.solver == "lsqr": + raise NotImplementedError( + "transform not implemented for 'lsqr' solver (use 'svd' or 'eigen')." + ) check_is_fitted(self) + xp, _ = get_namespace(X) + X = validate_data(self, X, reset=False) - X = self._validate_data(X, reset=False) - if self.solver == 'svd': - X_new = np.dot(X - self.xbar_, self.scalings_) - elif self.solver == 'eigen': - X_new = np.dot(X, self.scalings_) + if self.solver == "svd": + X_new = (X - self.xbar_) @ self.scalings_ + elif self.solver == "eigen": + X_new = X @ self.scalings_ - return X_new[:, :self._max_components] + return X_new[:, : self._max_components] def predict_proba(self, X): """Estimate probability. @@ -607,11 +773,11 @@ def predict_proba(self, X): Estimated probabilities. """ check_is_fitted(self) - + xp, is_array_api_compliant = get_namespace(X) decision = self.decision_function(X) - if self.classes_.size == 2: - proba = expit(decision) - return np.vstack([1-proba, proba]).T + if size(self.classes_) == 2: + proba = _expit(decision, xp) + return xp.stack([1 - proba, proba], axis=1) else: return softmax(decision) @@ -628,9 +794,18 @@ def predict_log_proba(self, X): C : ndarray of shape (n_samples, n_classes) Estimated log probabilities. """ + xp, _ = get_namespace(X) prediction = self.predict_proba(X) - prediction[prediction == 0.0] += np.finfo(prediction.dtype).tiny - return np.log(prediction) + + info = xp.finfo(prediction.dtype) + if hasattr(info, "smallest_normal"): + smallest_normal = info.smallest_normal + else: + # smallest_normal was introduced in NumPy 1.22 + smallest_normal = info.tiny + + prediction[prediction == 0.0] += smallest_normal + return xp.log(prediction) def decision_function(self, X): """Apply decision function to an array of samples. @@ -647,17 +822,24 @@ def decision_function(self, X): Returns ------- - C : ndarray of shape (n_samples,) or (n_samples, n_classes) + y_scores : ndarray of shape (n_samples,) or (n_samples, n_classes) Decision function values related to each class, per sample. - In the two-class case, the shape is (n_samples,), giving the + In the two-class case, the shape is `(n_samples,)`, giving the log likelihood ratio of the positive class. """ # Only override for the doc return super().decision_function(X) + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.array_api_support = True + return tags + -class QuadraticDiscriminantAnalysis(ClassifierMixin, BaseEstimator): - """Quadratic Discriminant Analysis +class QuadraticDiscriminantAnalysis( + DiscriminantAnalysisPredictionMixin, ClassifierMixin, BaseEstimator +): + """Quadratic Discriminant Analysis. A classifier with a quadratic decision boundary, generated by fitting class conditional densities to the data @@ -666,13 +848,17 @@ class QuadraticDiscriminantAnalysis(ClassifierMixin, BaseEstimator): The model fits a Gaussian density to each class. .. versionadded:: 0.17 - *QuadraticDiscriminantAnalysis* + + For a comparison between + :class:`~sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis` + and :class:`~sklearn.discriminant_analysis.LinearDiscriminantAnalysis`, see + :ref:`sphx_glr_auto_examples_classification_plot_lda_qda.py`. Read more in the :ref:`User Guide `. Parameters ---------- - priors : ndarray of shape (n_classes,), default=None + priors : array-like of shape (n_classes,), default=None Class priors. By default, the class proportions are inferred from the training data. @@ -682,17 +868,17 @@ class QuadraticDiscriminantAnalysis(ClassifierMixin, BaseEstimator): where S2 corresponds to the `scaling_` attribute of a given class. store_covariance : bool, default=False - If True, the class covariance matrices are explicitely computed and + If True, the class covariance matrices are explicitly computed and stored in the `self.covariance_` attribute. .. versionadded:: 0.17 tol : float, default=1.0e-4 - Absolute threshold for a singular value to be considered significant, - used to estimate the rank of `Xk` where `Xk` is the centered matrix - of samples in class k. This parameter does not affect the - predictions. It only controls a warning that is raised when features - are considered to be colinear. + Absolute threshold for the covariance matrix to be considered rank + deficient after applying some regularization (see `reg_param`) to each + `Sk` where `Sk` represents covariance matrix for k-th class. This + parameter does not affect the predictions. It controls when a warning + is raised if the covariance matrix is not full rank. .. versionadded:: 0.17 @@ -729,6 +915,21 @@ class QuadraticDiscriminantAnalysis(ClassifierMixin, BaseEstimator): classes_ : ndarray of shape (n_classes,) Unique class labels. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + LinearDiscriminantAnalysis : Linear Discriminant Analysis. + Examples -------- >>> from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis @@ -740,50 +941,62 @@ class QuadraticDiscriminantAnalysis(ClassifierMixin, BaseEstimator): QuadraticDiscriminantAnalysis() >>> print(clf.predict([[-0.8, -1]])) [1] - - See Also - -------- - LinearDiscriminantAnalysis : Linear Discriminant Analysis. """ - @_deprecate_positional_args - def __init__(self, *, priors=None, reg_param=0., store_covariance=False, - tol=1.0e-4): - self.priors = np.asarray(priors) if priors is not None else None + + _parameter_constraints: dict = { + "priors": ["array-like", None], + "reg_param": [Interval(Real, 0, 1, closed="both")], + "store_covariance": ["boolean"], + "tol": [Interval(Real, 0, None, closed="left")], + } + + def __init__( + self, *, priors=None, reg_param=0.0, store_covariance=False, tol=1.0e-4 + ): + self.priors = priors self.reg_param = reg_param self.store_covariance = store_covariance self.tol = tol + @_fit_context(prefer_skip_nested_validation=True) def fit(self, X, y): """Fit the model according to the given training data and parameters. - .. versionchanged:: 0.19 - ``store_covariances`` has been moved to main constructor as - ``store_covariance`` + .. versionchanged:: 0.19 + ``store_covariances`` has been moved to main constructor as + ``store_covariance``. - .. versionchanged:: 0.19 - ``tol`` has been moved to main constructor. + .. versionchanged:: 0.19 + ``tol`` has been moved to main constructor. Parameters ---------- X : array-like of shape (n_samples, n_features) - Training vector, where n_samples is the number of samples and - n_features is the number of features. + Training vector, where `n_samples` is the number of samples and + `n_features` is the number of features. y : array-like of shape (n_samples,) - Target values (integers) + Target values (integers). + + Returns + ------- + self : object + Fitted estimator. """ - X, y = self._validate_data(X, y) + X, y = validate_data(self, X, y) check_classification_targets(y) self.classes_, y = np.unique(y, return_inverse=True) n_samples, n_features = X.shape n_classes = len(self.classes_) if n_classes < 2: - raise ValueError('The number of classes has to be greater than' - ' one; got %d class' % (n_classes)) + raise ValueError( + "The number of classes has to be greater than one; got %d class" + % (n_classes) + ) if self.priors is None: self.priors_ = np.bincount(y) / float(n_samples) else: - self.priors_ = self.priors + self.priors_ = np.array(self.priors) cov = None store_covariance = self.store_covariance @@ -797,16 +1010,23 @@ def fit(self, X, y): meang = Xg.mean(0) means.append(meang) if len(Xg) == 1: - raise ValueError('y has only 1 sample in class %s, covariance ' - 'is ill defined.' % str(self.classes_[ind])) + raise ValueError( + "y has only 1 sample in class %s, covariance is ill defined." + % str(self.classes_[ind]) + ) Xgc = Xg - meang # Xgc = U * S * V.T _, S, Vt = np.linalg.svd(Xgc, full_matrices=False) - rank = np.sum(S > self.tol) - if rank < n_features: - warnings.warn("Variables are collinear") - S2 = (S ** 2) / (len(Xg) - 1) + S2 = (S**2) / (len(Xg) - 1) S2 = ((1 - self.reg_param) * S2) + self.reg_param + rank = np.sum(S2 > self.tol) + if rank < n_features: + warnings.warn( + f"The covariance matrix of class {ind} is not full rank. " + "Increasing the value of parameter `reg_param` might help" + " reducing the collinearity.", + linalg.LinAlgWarning, + ) if self.store_covariance or store_covariance: # cov = V * (S^2 / (n-1)) * V.T cov.append(np.dot(S2 * Vt.T, Vt)) @@ -823,17 +1043,17 @@ def _decision_function(self, X): # return log posterior, see eq (4.12) p. 110 of the ESL. check_is_fitted(self) - X = self._validate_data(X, reset=False) + X = validate_data(self, X, reset=False) norm2 = [] for i in range(len(self.classes_)): R = self.rotations_[i] S = self.scalings_[i] Xm = X - self.means_[i] X2 = np.dot(Xm, R * (S ** (-0.5))) - norm2.append(np.sum(X2 ** 2, axis=1)) + norm2.append(np.sum(X2**2, axis=1)) norm2 = np.array(norm2).T # shape = [len(X), n_classes] u = np.asarray([np.sum(np.log(s)) for s in self.scalings_]) - return (-0.5 * (norm2 + u) + np.log(self.priors_)) + return -0.5 * (norm2 + u) + np.log(self.priors_) def decision_function(self, X): """Apply decision function to an array of samples. @@ -852,14 +1072,10 @@ def decision_function(self, X): ------- C : ndarray of shape (n_samples,) or (n_samples, n_classes) Decision function values related to each class, per sample. - In the two-class case, the shape is (n_samples,), giving the + In the two-class case, the shape is `(n_samples,)`, giving the log likelihood ratio of the positive class. """ - dec_func = self._decision_function(X) - # handle special case of two classes - if len(self.classes_) == 2: - return dec_func[:, 1] - dec_func[:, 0] - return dec_func + return super().decision_function(X) def predict(self, X): """Perform classification on an array of test vectors X. @@ -869,14 +1085,15 @@ def predict(self, X): Parameters ---------- X : array-like of shape (n_samples, n_features) + Vector to be scored, where `n_samples` is the number of samples and + `n_features` is the number of features. Returns ------- C : ndarray of shape (n_samples,) + Estimated probabilities. """ - d = self._decision_function(X) - y_pred = self.classes_.take(d.argmax(1)) - return y_pred + return super().predict(X) def predict_proba(self, X): """Return posterior probabilities of classification. @@ -891,12 +1108,9 @@ def predict_proba(self, X): C : ndarray of shape (n_samples, n_classes) Posterior probabilities of classification per class. """ - values = self._decision_function(X) # compute the likelihood of the underlying gaussian models # up to a multiplicative constant. - likelihood = np.exp(values - values.max(axis=1)[:, np.newaxis]) - # compute posterior probabilities - return likelihood / likelihood.sum(axis=1)[:, np.newaxis] + return super().predict_proba(X) def predict_log_proba(self, X): """Return log of posterior probabilities of classification. @@ -912,5 +1126,4 @@ def predict_log_proba(self, X): Posterior log-probabilities of classification per class. """ # XXX : can do better to avoid precision overflows - probas_ = self.predict_proba(X) - return np.log(probas_) + return super().predict_log_proba(X) diff --git a/sklearn/dummy.py b/sklearn/dummy.py index ad5ab3f24731d..7d44fa2e473bb 100644 --- a/sklearn/dummy.py +++ b/sklearn/dummy.py @@ -1,31 +1,54 @@ -# Author: Mathieu Blondel -# Arnaud Joly -# Maheshakya Wijewardena -# License: BSD 3 clause +"""Dummy estimators that implement simple rules of thumb.""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import warnings +from numbers import Integral, Real + import numpy as np import scipy.sparse as sp -from .base import BaseEstimator, ClassifierMixin, RegressorMixin -from .base import MultiOutputMixin +from .base import ( + BaseEstimator, + ClassifierMixin, + MultiOutputMixin, + RegressorMixin, + _fit_context, +) from .utils import check_random_state -from .utils.validation import _num_samples -from .utils.validation import check_array -from .utils.validation import check_consistent_length -from .utils.validation import check_is_fitted, _check_sample_weight +from .utils._param_validation import Interval, StrOptions +from .utils.multiclass import class_distribution from .utils.random import _random_choice_csc from .utils.stats import _weighted_percentile -from .utils.multiclass import class_distribution -from .utils.validation import _deprecate_positional_args +from .utils.validation import ( + _check_sample_weight, + _num_samples, + check_array, + check_consistent_length, + check_is_fitted, + validate_data, +) class DummyClassifier(MultiOutputMixin, ClassifierMixin, BaseEstimator): - """ - DummyClassifier is a classifier that makes predictions using simple rules. + """DummyClassifier makes predictions that ignore the input features. - This classifier is useful as a simple baseline to compare with other - (real) classifiers. Do not use it for real problems. + This classifier serves as a simple baseline to compare against other more + complex classifiers. + + The specific behavior of the baseline is selected with the `strategy` + parameter. + + All strategies make predictions that ignore the input feature values passed + as the `X` argument to `fit` and `predict`. The predictions, however, + typically depend on values observed in the `y` parameter passed to `fit`. + + Note that the "stratified" and "uniform" strategies lead to + non-deterministic predictions that can be rendered deterministic by setting + the `random_state` parameter if needed. The other strategies are naturally + deterministic and, once fit, always return the same constant prediction + for any value of `X`. Read more in the :ref:`User Guide `. @@ -33,20 +56,32 @@ class DummyClassifier(MultiOutputMixin, ClassifierMixin, BaseEstimator): Parameters ---------- - strategy : {"stratified", "most_frequent", "prior", "uniform", \ + strategy : {"most_frequent", "prior", "stratified", "uniform", \ "constant"}, default="prior" Strategy to use to generate predictions. - * "stratified": generates predictions by respecting the training - set's class distribution. - * "most_frequent": always predicts the most frequent label in the - training set. - * "prior": always predicts the class that maximizes the class prior - (like "most_frequent") and ``predict_proba`` returns the class prior. - * "uniform": generates predictions uniformly at random. + * "most_frequent": the `predict` method always returns the most + frequent class label in the observed `y` argument passed to `fit`. + The `predict_proba` method returns the matching one-hot encoded + vector. + * "prior": the `predict` method always returns the most frequent + class label in the observed `y` argument passed to `fit` (like + "most_frequent"). ``predict_proba`` always returns the empirical + class distribution of `y` also known as the empirical class prior + distribution. + * "stratified": the `predict_proba` method randomly samples one-hot + vectors from a multinomial distribution parametrized by the empirical + class prior probabilities. + The `predict` method returns the class label which got probability + one in the one-hot vector of `predict_proba`. + Each sampled row of both methods is therefore independent and + identically distributed. + * "uniform": generates predictions uniformly at random from the list + of unique classes observed in `y`, i.e. each class has equal + probability. * "constant": always predicts a constant label that is provided by the user. This is useful for metrics that evaluate a non-majority - class + class. .. versionchanged:: 0.24 The default value of `strategy` has changed to "prior" in version @@ -58,27 +93,42 @@ class DummyClassifier(MultiOutputMixin, ClassifierMixin, BaseEstimator): Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. - constant : int or str or array-like of shape (n_outputs,) + constant : int or str or array-like of shape (n_outputs,), default=None The explicit constant as predicted by the "constant" strategy. This parameter is useful only for the "constant" strategy. Attributes ---------- classes_ : ndarray of shape (n_classes,) or list of such arrays - Class labels for each output. + Unique class labels observed in `y`. For multi-output classification + problems, this attribute is a list of arrays as each output has an + independent set of possible classes. n_classes_ : int or list of int Number of label for each output. class_prior_ : ndarray of shape (n_classes,) or list of such arrays - Probability of each class for each output. + Frequency of each class observed in `y`. For multioutput classification + problems, this is computed independently for each output. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` has + feature names that are all strings. n_outputs_ : int Number of outputs. sparse_output_ : bool True if the array returned from predict is to be in sparse CSC format. - Is automatically set to True if the input y is passed in sparse format. + Is automatically set to True if the input `y` is passed in sparse + format. + + See Also + -------- + DummyRegressor : Regressor that makes predictions using simple rules. Examples -------- @@ -94,15 +144,23 @@ class DummyClassifier(MultiOutputMixin, ClassifierMixin, BaseEstimator): >>> dummy_clf.score(X, y) 0.75 """ - @_deprecate_positional_args - def __init__(self, *, strategy="prior", random_state=None, - constant=None): + + _parameter_constraints: dict = { + "strategy": [ + StrOptions({"most_frequent", "prior", "stratified", "uniform", "constant"}) + ], + "random_state": ["random_state"], + "constant": [Integral, str, "array-like", None], + } + + def __init__(self, *, strategy="prior", random_state=None, constant=None): self.strategy = strategy self.random_state = random_state self.constant = constant + @_fit_context(prefer_skip_nested_validation=True) def fit(self, X, y, sample_weight=None): - """Fit the random classifier. + """Fit the baseline classifier. Parameters ---------- @@ -118,23 +176,23 @@ def fit(self, X, y, sample_weight=None): Returns ------- self : object + Returns the instance itself. """ - allowed_strategies = ("most_frequent", "stratified", "uniform", - "constant", "prior") - - if self.strategy not in allowed_strategies: - raise ValueError("Unknown strategy type: %s, expected one of %s." - % (self.strategy, allowed_strategies)) + validate_data(self, X, skip_check_array=True) self._strategy = self.strategy if self._strategy == "uniform" and sp.issparse(y): y = y.toarray() - warnings.warn('A local copy of the target data has been converted ' - 'to a numpy array. Predicting on sparse target data ' - 'with the uniform strategy would not save memory ' - 'and would be slower.', - UserWarning) + warnings.warn( + ( + "A local copy of the target data has been converted " + "to a numpy array. Predicting on sparse target data " + "with the uniform strategy would not save memory " + "and would be slower." + ), + UserWarning, + ) self.sparse_output_ = sp.issparse(y) @@ -147,8 +205,6 @@ def fit(self, X, y, sample_weight=None): self.n_outputs_ = y.shape[1] - self.n_features_in_ = None # No input validation is done for X - check_consistent_length(X, y) if sample_weight is not None: @@ -156,27 +212,34 @@ def fit(self, X, y, sample_weight=None): if self._strategy == "constant": if self.constant is None: - raise ValueError("Constant target value has to be specified " - "when the constant strategy is used.") + raise ValueError( + "Constant target value has to be specified " + "when the constant strategy is used." + ) else: constant = np.reshape(np.atleast_1d(self.constant), (-1, 1)) if constant.shape[0] != self.n_outputs_: - raise ValueError("Constant target value should have " - "shape (%d, 1)." % self.n_outputs_) + raise ValueError( + "Constant target value should have shape (%d, 1)." + % self.n_outputs_ + ) - (self.classes_, - self.n_classes_, - self.class_prior_) = class_distribution(y, sample_weight) + (self.classes_, self.n_classes_, self.class_prior_) = class_distribution( + y, sample_weight + ) if self._strategy == "constant": for k in range(self.n_outputs_): if not any(constant[k][0] == c for c in self.classes_[k]): # Checking in case of constant strategy if the constant # provided by the user is in y. - err_msg = ("The constant target value must be present in " - "the training data. You provided constant={}. " - "Possible values are: {}." - .format(self.constant, list(self.classes_[k]))) + err_msg = ( + "The constant target value must be present in " + "the training data. You provided constant={}. " + "Possible values are: {}.".format( + self.constant, self.classes_[k].tolist() + ) + ) raise ValueError(err_msg) if self.n_outputs_ == 1: @@ -231,26 +294,38 @@ def predict(self, X): class_prob = class_prior_ elif self._strategy == "uniform": - raise ValueError("Sparse target prediction is not " - "supported with the uniform strategy") + raise ValueError( + "Sparse target prediction is not " + "supported with the uniform strategy" + ) elif self._strategy == "constant": classes_ = [np.array([c]) for c in constant] - y = _random_choice_csc(n_samples, classes_, class_prob, - self.random_state) + y = _random_choice_csc(n_samples, classes_, class_prob, self.random_state) else: if self._strategy in ("most_frequent", "prior"): - y = np.tile([classes_[k][class_prior_[k].argmax()] for - k in range(self.n_outputs_)], [n_samples, 1]) + y = np.tile( + [ + classes_[k][class_prior_[k].argmax()] + for k in range(self.n_outputs_) + ], + [n_samples, 1], + ) elif self._strategy == "stratified": - y = np.vstack([classes_[k][proba[k].argmax(axis=1)] for - k in range(self.n_outputs_)]).T + y = np.vstack( + [ + classes_[k][proba[k].argmax(axis=1)] + for k in range(self.n_outputs_) + ] + ).T elif self._strategy == "uniform": - ret = [classes_[k][rs.randint(n_classes_[k], size=n_samples)] - for k in range(self.n_outputs_)] + ret = [ + classes_[k][rs.randint(n_classes_[k], size=n_samples)] + for k in range(self.n_outputs_) + ] y = np.vstack(ret).T elif self._strategy == "constant": @@ -331,7 +406,7 @@ def predict_log_proba(self, X): Parameters ---------- X : {array-like, object with finite length or shape} - Training data, requires length = n_samples + Training data. Returns ------- @@ -346,19 +421,15 @@ def predict_log_proba(self, X): else: return [np.log(p) for p in proba] - def _more_tags(self): - return { - 'poor_score': True, 'no_validation': True, - '_xfail_checks': { - 'check_methods_subset_invariance': - 'fails for the predict method', - 'check_methods_sample_order_invariance': - 'fails for the predict method' - } - } + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.sparse = True + tags.classifier_tags.poor_score = True + tags.no_validation = True + return tags def score(self, X, y, sample_weight=None): - """Returns the mean accuracy on the given test data and labels. + """Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that @@ -380,8 +451,7 @@ def score(self, X, y, sample_weight=None): Returns ------- score : float - Mean accuracy of self.predict(X) wrt. y. - + Mean accuracy of self.predict(X) w.r.t. y. """ if X is None: X = np.zeros(shape=(len(y), 1)) @@ -389,9 +459,7 @@ def score(self, X, y, sample_weight=None): class DummyRegressor(MultiOutputMixin, RegressorMixin, BaseEstimator): - """ - DummyRegressor is a regressor that makes predictions using - simple rules. + """Regressor that makes predictions using simple rules. This regressor is useful as a simple baseline to compare with other (real) regressors. Do not use it for real problems. @@ -427,9 +495,20 @@ class DummyRegressor(MultiOutputMixin, RegressorMixin, BaseEstimator): Mean or median or quantile of the training targets or constant value given by the user. + n_features_in_ : int + Number of features seen during :term:`fit`. + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` has + feature names that are all strings. + n_outputs_ : int Number of outputs. + See Also + -------- + DummyClassifier: Classifier that makes predictions using simple rules. + Examples -------- >>> import numpy as np @@ -444,14 +523,25 @@ class DummyRegressor(MultiOutputMixin, RegressorMixin, BaseEstimator): >>> dummy_regr.score(X, y) 0.0 """ - @_deprecate_positional_args + + _parameter_constraints: dict = { + "strategy": [StrOptions({"mean", "median", "quantile", "constant"})], + "quantile": [Interval(Real, 0.0, 1.0, closed="both"), None], + "constant": [ + Interval(Real, None, None, closed="neither"), + "array-like", + None, + ], + } + def __init__(self, *, strategy="mean", constant=None, quantile=None): self.strategy = strategy self.constant = constant self.quantile = quantile + @_fit_context(prefer_skip_nested_validation=True) def fit(self, X, y, sample_weight=None): - """Fit the random regressor. + """Fit the baseline regressor. Parameters ---------- @@ -467,14 +557,11 @@ def fit(self, X, y, sample_weight=None): Returns ------- self : object + Fitted estimator. """ - allowed_strategies = ("mean", "median", "quantile", "constant") - if self.strategy not in allowed_strategies: - raise ValueError("Unknown strategy type: %s, expected one of %s." - % (self.strategy, allowed_strategies)) + validate_data(self, X, skip_check_array=True) - y = check_array(y, ensure_2d=False) - self.n_features_in_ = None # No input validation is done for X + y = check_array(y, ensure_2d=False, input_name="y") if len(y) == 0: raise ValueError("y must not be empty.") @@ -494,45 +581,52 @@ def fit(self, X, y, sample_weight=None): if sample_weight is None: self.constant_ = np.median(y, axis=0) else: - self.constant_ = [_weighted_percentile(y[:, k], sample_weight, - percentile=50.) - for k in range(self.n_outputs_)] + self.constant_ = [ + _weighted_percentile(y[:, k], sample_weight, percentile_rank=50.0) + for k in range(self.n_outputs_) + ] elif self.strategy == "quantile": - if self.quantile is None or not np.isscalar(self.quantile): - raise ValueError("Quantile must be a scalar in the range " - "[0.0, 1.0], but got %s." % self.quantile) - - percentile = self.quantile * 100.0 + if self.quantile is None: + raise ValueError( + "When using `strategy='quantile', you have to specify the desired " + "quantile in the range [0, 1]." + ) + percentile_rank = self.quantile * 100.0 if sample_weight is None: - self.constant_ = np.percentile(y, axis=0, q=percentile) + self.constant_ = np.percentile(y, axis=0, q=percentile_rank) else: - self.constant_ = [_weighted_percentile(y[:, k], sample_weight, - percentile=percentile) - for k in range(self.n_outputs_)] + self.constant_ = [ + _weighted_percentile( + y[:, k], sample_weight, percentile_rank=percentile_rank + ) + for k in range(self.n_outputs_) + ] elif self.strategy == "constant": if self.constant is None: - raise TypeError("Constant target value has to be specified " - "when the constant strategy is used.") - - self.constant = check_array(self.constant, - accept_sparse=['csr', 'csc', 'coo'], - ensure_2d=False, ensure_min_samples=0) - - if self.n_outputs_ != 1 and self.constant.shape[0] != y.shape[1]: + raise TypeError( + "Constant target value has to be specified " + "when the constant strategy is used." + ) + + self.constant_ = check_array( + self.constant, + accept_sparse=["csr", "csc", "coo"], + ensure_2d=False, + ensure_min_samples=0, + ) + + if self.n_outputs_ != 1 and self.constant_.shape[0] != y.shape[1]: raise ValueError( - "Constant target value should have " - "shape (%d, 1)." % y.shape[1]) - - self.constant_ = self.constant + "Constant target value should have shape (%d, 1)." % y.shape[1] + ) self.constant_ = np.reshape(self.constant_, (1, -1)) return self def predict(self, X, return_std=False): - """ - Perform classification on test vectors X. + """Perform classification on test vectors X. Parameters ---------- @@ -556,8 +650,11 @@ def predict(self, X, return_std=False): check_is_fitted(self) n_samples = _num_samples(X) - y = np.full((n_samples, self.n_outputs_), self.constant_, - dtype=np.array(self.constant_).dtype) + y = np.full( + (n_samples, self.n_outputs_), + self.constant_, + dtype=np.array(self.constant_).dtype, + ) y_std = np.zeros((n_samples, self.n_outputs_)) if self.n_outputs_ == 1: @@ -566,25 +663,29 @@ def predict(self, X, return_std=False): return (y, y_std) if return_std else y - def _more_tags(self): - return {'poor_score': True, 'no_validation': True} + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.sparse = True + tags.regressor_tags.poor_score = True + tags.no_validation = True + return tags def score(self, X, y, sample_weight=None): - """Returns the coefficient of determination R^2 of the prediction. + """Return the coefficient of determination R^2 of the prediction. - The coefficient R^2 is defined as (1 - u/v), where u is the residual - sum of squares ((y_true - y_pred) ** 2).sum() and v is the total - sum of squares ((y_true - y_true.mean()) ** 2).sum(). - The best possible score is 1.0 and it can be negative (because the - model can be arbitrarily worse). A constant model that always - predicts the expected value of y, disregarding the input features, - would get a R^2 score of 0.0. + The coefficient R^2 is defined as `(1 - u/v)`, where `u` is the + residual sum of squares `((y_true - y_pred) ** 2).sum()` and `v` is the + total sum of squares `((y_true - y_true.mean()) ** 2).sum()`. The best + possible score is 1.0 and it can be negative (because the model can be + arbitrarily worse). A constant model that always predicts the expected + value of y, disregarding the input features, would get a R^2 score of + 0.0. Parameters ---------- X : None or array-like of shape (n_samples, n_features) Test samples. Passing None as test samples gives the same result - as passing real test samples, since DummyRegressor + as passing real test samples, since `DummyRegressor` operates independently of the sampled observations. y : array-like of shape (n_samples,) or (n_samples, n_outputs) @@ -596,7 +697,7 @@ def score(self, X, y, sample_weight=None): Returns ------- score : float - R^2 of self.predict(X) wrt. y. + R^2 of `self.predict(X)` w.r.t. y. """ if X is None: X = np.zeros(shape=(len(y), 1)) diff --git a/sklearn/ensemble/__init__.py b/sklearn/ensemble/__init__.py index ae86349ad9af0..62a538d340318 100644 --- a/sklearn/ensemble/__init__.py +++ b/sklearn/ensemble/__init__.py @@ -1,40 +1,45 @@ -""" -The :mod:`sklearn.ensemble` module includes ensemble-based methods for -classification, regression and anomaly detection. -""" -import typing +"""Ensemble-based methods for classification, regression and anomaly detection.""" +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from ._bagging import BaggingClassifier, BaggingRegressor from ._base import BaseEnsemble -from ._forest import RandomForestClassifier -from ._forest import RandomForestRegressor -from ._forest import RandomTreesEmbedding -from ._forest import ExtraTreesClassifier -from ._forest import ExtraTreesRegressor -from ._bagging import BaggingClassifier -from ._bagging import BaggingRegressor +from ._forest import ( + ExtraTreesClassifier, + ExtraTreesRegressor, + RandomForestClassifier, + RandomForestRegressor, + RandomTreesEmbedding, +) +from ._gb import GradientBoostingClassifier, GradientBoostingRegressor +from ._hist_gradient_boosting.gradient_boosting import ( + HistGradientBoostingClassifier, + HistGradientBoostingRegressor, +) from ._iforest import IsolationForest -from ._weight_boosting import AdaBoostClassifier -from ._weight_boosting import AdaBoostRegressor -from ._gb import GradientBoostingClassifier -from ._gb import GradientBoostingRegressor -from ._voting import VotingClassifier -from ._voting import VotingRegressor -from ._stacking import StackingClassifier -from ._stacking import StackingRegressor - -if typing.TYPE_CHECKING: - # Avoid errors in type checkers (e.g. mypy) for experimental estimators. - # TODO: remove this check once the estimator is no longer experimental. - from ._hist_gradient_boosting.gradient_boosting import ( # noqa - HistGradientBoostingRegressor, HistGradientBoostingClassifier - ) +from ._stacking import StackingClassifier, StackingRegressor +from ._voting import VotingClassifier, VotingRegressor +from ._weight_boosting import AdaBoostClassifier, AdaBoostRegressor -__all__ = ["BaseEnsemble", - "RandomForestClassifier", "RandomForestRegressor", - "RandomTreesEmbedding", "ExtraTreesClassifier", - "ExtraTreesRegressor", "BaggingClassifier", - "BaggingRegressor", "IsolationForest", "GradientBoostingClassifier", - "GradientBoostingRegressor", "AdaBoostClassifier", - "AdaBoostRegressor", "VotingClassifier", "VotingRegressor", - "StackingClassifier", "StackingRegressor", - ] +__all__ = [ + "AdaBoostClassifier", + "AdaBoostRegressor", + "BaggingClassifier", + "BaggingRegressor", + "BaseEnsemble", + "ExtraTreesClassifier", + "ExtraTreesRegressor", + "GradientBoostingClassifier", + "GradientBoostingRegressor", + "HistGradientBoostingClassifier", + "HistGradientBoostingRegressor", + "IsolationForest", + "RandomForestClassifier", + "RandomForestRegressor", + "RandomTreesEmbedding", + "StackingClassifier", + "StackingRegressor", + "VotingClassifier", + "VotingRegressor", +] diff --git a/sklearn/ensemble/_bagging.py b/sklearn/ensemble/_bagging.py index 1ac309f00ad69..34b613b15281a 100644 --- a/sklearn/ensemble/_bagging.py +++ b/sklearn/ensemble/_bagging.py @@ -1,33 +1,52 @@ """Bagging meta-estimator.""" -# Author: Gilles Louppe -# License: BSD 3 clause - +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import itertools import numbers -import numpy as np from abc import ABCMeta, abstractmethod +from functools import partial +from numbers import Integral from warnings import warn -from joblib import Parallel +import numpy as np -from ._base import BaseEnsemble, _partition_estimators -from ..base import ClassifierMixin, RegressorMixin -from ..metrics import r2_score, accuracy_score +from ..base import ClassifierMixin, RegressorMixin, _fit_context +from ..metrics import accuracy_score, r2_score from ..tree import DecisionTreeClassifier, DecisionTreeRegressor -from ..utils import check_random_state, column_or_1d, deprecated -from ..utils import indices_to_mask -from ..utils.metaestimators import if_delegate_has_method +from ..utils import ( + Bunch, + _safe_indexing, + check_random_state, + column_or_1d, +) +from ..utils._mask import indices_to_mask +from ..utils._param_validation import HasMethods, Interval, RealNotInt +from ..utils._tags import get_tags +from ..utils.metadata_routing import ( + MetadataRouter, + MethodMapping, + _raise_for_params, + _routing_enabled, + get_routing_for_object, + process_routing, +) +from ..utils.metaestimators import available_if from ..utils.multiclass import check_classification_targets +from ..utils.parallel import Parallel, delayed from ..utils.random import sample_without_replacement -from ..utils.validation import has_fit_parameter, check_is_fitted, \ - _check_sample_weight, _deprecate_positional_args -from ..utils.fixes import delayed - +from ..utils.validation import ( + _check_method_params, + _check_sample_weight, + _estimator_has, + check_is_fitted, + has_fit_parameter, + validate_data, +) +from ._base import BaseEnsemble, _partition_estimators -__all__ = ["BaggingClassifier", - "BaggingRegressor"] +__all__ = ["BaggingClassifier", "BaggingRegressor"] MAX_INT = np.iinfo(np.int32).max @@ -38,30 +57,48 @@ def _generate_indices(random_state, bootstrap, n_population, n_samples): if bootstrap: indices = random_state.randint(0, n_population, n_samples) else: - indices = sample_without_replacement(n_population, n_samples, - random_state=random_state) + indices = sample_without_replacement( + n_population, n_samples, random_state=random_state + ) return indices -def _generate_bagging_indices(random_state, bootstrap_features, - bootstrap_samples, n_features, n_samples, - max_features, max_samples): +def _generate_bagging_indices( + random_state, + bootstrap_features, + bootstrap_samples, + n_features, + n_samples, + max_features, + max_samples, +): """Randomly draw feature and sample indices.""" # Get valid random state random_state = check_random_state(random_state) # Draw indices - feature_indices = _generate_indices(random_state, bootstrap_features, - n_features, max_features) - sample_indices = _generate_indices(random_state, bootstrap_samples, - n_samples, max_samples) + feature_indices = _generate_indices( + random_state, bootstrap_features, n_features, max_features + ) + sample_indices = _generate_indices( + random_state, bootstrap_samples, n_samples, max_samples + ) return feature_indices, sample_indices -def _parallel_build_estimators(n_estimators, ensemble, X, y, sample_weight, - seeds, total_n_estimators, verbose): +def _parallel_build_estimators( + n_estimators, + ensemble, + X, + y, + seeds, + total_n_estimators, + verbose, + check_input, + fit_params, +): """Private function used to build a batch of estimators within a job.""" # Retrieve settings n_samples, n_features = X.shape @@ -69,37 +106,74 @@ def _parallel_build_estimators(n_estimators, ensemble, X, y, sample_weight, max_samples = ensemble._max_samples bootstrap = ensemble.bootstrap bootstrap_features = ensemble.bootstrap_features - support_sample_weight = has_fit_parameter(ensemble.base_estimator_, - "sample_weight") - if not support_sample_weight and sample_weight is not None: - raise ValueError("The base estimator doesn't support sample weight") + has_check_input = has_fit_parameter(ensemble.estimator_, "check_input") + requires_feature_indexing = bootstrap_features or max_features != n_features # Build estimators estimators = [] estimators_features = [] + # TODO: (slep6) remove if condition for unrouted sample_weight when metadata + # routing can't be disabled. + support_sample_weight = has_fit_parameter(ensemble.estimator_, "sample_weight") + if not _routing_enabled() and ( + not support_sample_weight and fit_params.get("sample_weight") is not None + ): + raise ValueError( + "The base estimator doesn't support sample weight, but sample_weight is " + "passed to the fit method." + ) + for i in range(n_estimators): if verbose > 1: - print("Building estimator %d of %d for this parallel run " - "(total %d)..." % (i + 1, n_estimators, total_n_estimators)) + print( + "Building estimator %d of %d for this parallel run (total %d)..." + % (i + 1, n_estimators, total_n_estimators) + ) random_state = seeds[i] - estimator = ensemble._make_estimator(append=False, - random_state=random_state) + estimator = ensemble._make_estimator(append=False, random_state=random_state) + + if has_check_input: + estimator_fit = partial(estimator.fit, check_input=check_input) + else: + estimator_fit = estimator.fit # Draw random feature, sample indices - features, indices = _generate_bagging_indices(random_state, - bootstrap_features, - bootstrap, n_features, - n_samples, max_features, - max_samples) - - # Draw samples, using sample weights, and then fit - if support_sample_weight: - if sample_weight is None: - curr_sample_weight = np.ones((n_samples,)) - else: - curr_sample_weight = sample_weight.copy() + features, indices = _generate_bagging_indices( + random_state, + bootstrap_features, + bootstrap, + n_features, + n_samples, + max_features, + max_samples, + ) + + fit_params_ = fit_params.copy() + + # TODO(SLEP6): remove if condition for unrouted sample_weight when metadata + # routing can't be disabled. + # 1. If routing is enabled, we will check if the routing supports sample + # weight and use it if it does. + # 2. If routing is not enabled, we will check if the base + # estimator supports sample_weight and use it if it does. + + # Note: Row sampling can be achieved either through setting sample_weight or + # by indexing. The former is more efficient. Therefore, use this method + # if possible, otherwise use indexing. + if _routing_enabled(): + request_or_router = get_routing_for_object(ensemble.estimator_) + consumes_sample_weight = request_or_router.consumes( + "fit", ("sample_weight",) + ) + else: + consumes_sample_weight = support_sample_weight + if consumes_sample_weight: + # Draw sub samples, using sample weights, and then fit + curr_sample_weight = _check_sample_weight( + fit_params_.pop("sample_weight", None), X + ).copy() if bootstrap: sample_counts = np.bincount(indices, minlength=n_samples) @@ -108,10 +182,17 @@ def _parallel_build_estimators(n_estimators, ensemble, X, y, sample_weight, not_indices_mask = ~indices_to_mask(indices, n_samples) curr_sample_weight[not_indices_mask] = 0 - estimator.fit(X[:, features], y, sample_weight=curr_sample_weight) - + fit_params_["sample_weight"] = curr_sample_weight + X_ = X[:, features] if requires_feature_indexing else X + estimator_fit(X_, y, **fit_params_) else: - estimator.fit((X[indices])[:, features], y[indices]) + # cannot use sample_weight, so use indexing + y_ = _safe_indexing(y, indices) + X_ = _safe_indexing(X, indices) + fit_params_ = _check_method_params(X, params=fit_params_, indices=indices) + if requires_feature_indexing: + X_ = X_[:, features] + estimator_fit(X_, y_, **fit_params_) estimators.append(estimator) estimators_features.append(features) @@ -119,25 +200,37 @@ def _parallel_build_estimators(n_estimators, ensemble, X, y, sample_weight, return estimators, estimators_features -def _parallel_predict_proba(estimators, estimators_features, X, n_classes): +def _parallel_predict_proba( + estimators, + estimators_features, + X, + n_classes, + predict_params=None, + predict_proba_params=None, +): """Private function used to compute (proba-)predictions within a job.""" n_samples = X.shape[0] proba = np.zeros((n_samples, n_classes)) for estimator, features in zip(estimators, estimators_features): if hasattr(estimator, "predict_proba"): - proba_estimator = estimator.predict_proba(X[:, features]) + proba_estimator = estimator.predict_proba( + X[:, features], **(predict_params or {}) + ) if n_classes == len(estimator.classes_): proba += proba_estimator else: - proba[:, estimator.classes_] += \ - proba_estimator[:, range(len(estimator.classes_))] + proba[:, estimator.classes_] += proba_estimator[ + :, range(len(estimator.classes_)) + ] else: # Resort to voting - predictions = estimator.predict(X[:, features]) + predictions = estimator.predict( + X[:, features], **(predict_proba_params or {}) + ) for i in range(n_samples): proba[i, predictions[i]] += 1 @@ -145,7 +238,7 @@ def _parallel_predict_proba(estimators, estimators_features, X, n_classes): return proba -def _parallel_predict_log_proba(estimators, estimators_features, X, n_classes): +def _parallel_predict_log_proba(estimators, estimators_features, X, n_classes, params): """Private function used to compute log probabilities within a job.""" n_samples = X.shape[0] log_proba = np.empty((n_samples, n_classes)) @@ -153,7 +246,7 @@ def _parallel_predict_log_proba(estimators, estimators_features, X, n_classes): all_classes = np.arange(n_classes, dtype=int) for estimator, features in zip(estimators, estimators_features): - log_proba_estimator = estimator.predict_log_proba(X[:, features]) + log_proba_estimator = estimator.predict_log_proba(X[:, features], **params) if n_classes == len(estimator.classes_): log_proba = np.logaddexp(log_proba, log_proba_estimator) @@ -161,27 +254,29 @@ def _parallel_predict_log_proba(estimators, estimators_features, X, n_classes): else: log_proba[:, estimator.classes_] = np.logaddexp( log_proba[:, estimator.classes_], - log_proba_estimator[:, range(len(estimator.classes_))]) + log_proba_estimator[:, range(len(estimator.classes_))], + ) missing = np.setdiff1d(all_classes, estimator.classes_) - log_proba[:, missing] = np.logaddexp(log_proba[:, missing], - -np.inf) + log_proba[:, missing] = np.logaddexp(log_proba[:, missing], -np.inf) return log_proba -def _parallel_decision_function(estimators, estimators_features, X): +def _parallel_decision_function(estimators, estimators_features, X, params): """Private function used to compute decisions within a job.""" - return sum(estimator.decision_function(X[:, features]) - for estimator, features in zip(estimators, - estimators_features)) + return sum( + estimator.decision_function(X[:, features], **params) + for estimator, features in zip(estimators, estimators_features) + ) -def _parallel_predict_regression(estimators, estimators_features, X): +def _parallel_predict_regression(estimators, estimators_features, X, params): """Private function used to compute predictions within a job.""" - return sum(estimator.predict(X[:, features]) - for estimator, features in zip(estimators, - estimators_features)) + return sum( + estimator.predict(X[:, features], **params) + for estimator, features in zip(estimators, estimators_features) + ) class BaseBagging(BaseEnsemble, metaclass=ABCMeta): @@ -191,23 +286,46 @@ class BaseBagging(BaseEnsemble, metaclass=ABCMeta): instead. """ + _parameter_constraints: dict = { + "estimator": [HasMethods(["fit", "predict"]), None], + "n_estimators": [Interval(Integral, 1, None, closed="left")], + "max_samples": [ + Interval(Integral, 1, None, closed="left"), + Interval(RealNotInt, 0, 1, closed="right"), + ], + "max_features": [ + Interval(Integral, 1, None, closed="left"), + Interval(RealNotInt, 0, 1, closed="right"), + ], + "bootstrap": ["boolean"], + "bootstrap_features": ["boolean"], + "oob_score": ["boolean"], + "warm_start": ["boolean"], + "n_jobs": [None, Integral], + "random_state": ["random_state"], + "verbose": ["verbose"], + } + @abstractmethod - def __init__(self, - base_estimator=None, - n_estimators=10, *, - max_samples=1.0, - max_features=1.0, - bootstrap=True, - bootstrap_features=False, - oob_score=False, - warm_start=False, - n_jobs=None, - random_state=None, - verbose=0): + def __init__( + self, + estimator=None, + n_estimators=10, + *, + max_samples=1.0, + max_features=1.0, + bootstrap=True, + bootstrap_features=False, + oob_score=False, + warm_start=False, + n_jobs=None, + random_state=None, + verbose=0, + ): super().__init__( - base_estimator=base_estimator, - n_estimators=n_estimators) - + estimator=estimator, + n_estimators=n_estimators, + ) self.max_samples = max_samples self.max_features = max_features self.bootstrap = bootstrap @@ -218,9 +336,12 @@ def __init__(self, self.random_state = random_state self.verbose = verbose - def fit(self, X, y, sample_weight=None): - """Build a Bagging ensemble of estimators from the training - set (X, y). + @_fit_context( + # BaseBagging.estimator is not validated yet + prefer_skip_nested_validation=False + ) + def fit(self, X, y, sample_weight=None, **fit_params): + """Build a Bagging ensemble of estimators from the training set (X, y). Parameters ---------- @@ -236,17 +357,56 @@ def fit(self, X, y, sample_weight=None): Sample weights. If None, then samples are equally weighted. Note that this is supported only if the base estimator supports sample weighting. + **fit_params : dict + Parameters to pass to the underlying estimators. + + .. versionadded:: 1.5 + + Only available if `enable_metadata_routing=True`, + which can be set by using + ``sklearn.set_config(enable_metadata_routing=True)``. + See :ref:`Metadata Routing User Guide ` for + more details. Returns ------- self : object + Fitted estimator. """ - return self._fit(X, y, self.max_samples, sample_weight=sample_weight) + _raise_for_params(fit_params, self, "fit") + + # Convert data (X is required to be 2d and indexable) + X, y = validate_data( + self, + X, + y, + accept_sparse=["csr", "csc"], + dtype=None, + ensure_all_finite=False, + multi_output=True, + ) + + return self._fit( + X, + y, + max_samples=self.max_samples, + sample_weight=sample_weight, + **fit_params, + ) def _parallel_args(self): return {} - def _fit(self, X, y, max_samples=None, max_depth=None, sample_weight=None): + def _fit( + self, + X, + y, + max_samples=None, + max_depth=None, + check_input=True, + sample_weight=None, + **fit_params, + ): """Build a Bagging ensemble of estimators from the training set (X, y). @@ -267,35 +427,51 @@ def _fit(self, X, y, max_samples=None, max_depth=None, sample_weight=None): Override value used when constructing base estimator. Only supported if the base estimator has a max_depth parameter. + check_input : bool, default=True + Override value used when fitting base estimator. Only supported + if the base estimator has a check_input parameter for fit function. + If the meta-estimator already checks the input, set this value to + False to prevent redundant input validation. + sample_weight : array-like of shape (n_samples,), default=None Sample weights. If None, then samples are equally weighted. Note that this is supported only if the base estimator supports sample weighting. + **fit_params : dict, default=None + Parameters to pass to the :term:`fit` method of the underlying + estimator. + Returns ------- self : object + Fitted estimator. """ random_state = check_random_state(self.random_state) - # Convert data (X is required to be 2d and indexable) - X, y = self._validate_data( - X, y, accept_sparse=['csr', 'csc'], dtype=None, - force_all_finite=False, multi_output=True - ) - if sample_weight is not None: - sample_weight = _check_sample_weight(sample_weight, X, dtype=None) - # Remap output n_samples = X.shape[0] self._n_samples = n_samples y = self._validate_y(y) # Check parameters - self._validate_estimator() + self._validate_estimator(self._get_estimator()) + + if sample_weight is not None: + fit_params["sample_weight"] = sample_weight + + if _routing_enabled(): + routed_params = process_routing(self, "fit", **fit_params) + else: + routed_params = Bunch() + routed_params.estimator = Bunch(fit=fit_params) + if "sample_weight" in fit_params: + routed_params.estimator.fit["sample_weight"] = fit_params[ + "sample_weight" + ] if max_depth is not None: - self.base_estimator_.max_depth = max_depth + self.estimator_.max_depth = max_depth # Validate max_samples if max_samples is None: @@ -303,8 +479,8 @@ def _fit(self, X, y, max_samples=None, max_depth=None, sample_weight=None): elif not isinstance(max_samples, numbers.Integral): max_samples = int(max_samples * X.shape[0]) - if not (0 < max_samples <= X.shape[0]): - raise ValueError("max_samples must be in (0, n_samples]") + if max_samples > X.shape[0]: + raise ValueError("max_samples must be <= n_samples") # Store validated integer row sampling value self._max_samples = max_samples @@ -313,12 +489,10 @@ def _fit(self, X, y, max_samples=None, max_depth=None, sample_weight=None): if isinstance(self.max_features, numbers.Integral): max_features = self.max_features elif isinstance(self.max_features, float): - max_features = self.max_features * self.n_features_in_ - else: - raise ValueError("max_features must be int or float") + max_features = int(self.max_features * self.n_features_in_) - if not (0 < max_features <= self.n_features_in_): - raise ValueError("max_features must be in (0, n_features]") + if max_features > self.n_features_in_: + raise ValueError("max_features must be <= n_features") max_features = max(1, int(max_features)) @@ -327,17 +501,15 @@ def _fit(self, X, y, max_samples=None, max_depth=None, sample_weight=None): # Other checks if not self.bootstrap and self.oob_score: - raise ValueError("Out of bag estimation only available" - " if bootstrap=True") + raise ValueError("Out of bag estimation only available if bootstrap=True") if self.warm_start and self.oob_score: - raise ValueError("Out of bag estimate only available" - " if warm_start=False") + raise ValueError("Out of bag estimate only available if warm_start=False") if hasattr(self, "oob_score_") and self.warm_start: del self.oob_score_ - if not self.warm_start or not hasattr(self, 'estimators_'): + if not self.warm_start or not hasattr(self, "estimators_"): # Free allocated memory, if any self.estimators_ = [] self.estimators_features_ = [] @@ -345,18 +517,23 @@ def _fit(self, X, y, max_samples=None, max_depth=None, sample_weight=None): n_more_estimators = self.n_estimators - len(self.estimators_) if n_more_estimators < 0: - raise ValueError('n_estimators=%d must be larger or equal to ' - 'len(estimators_)=%d when warm_start==True' - % (self.n_estimators, len(self.estimators_))) + raise ValueError( + "n_estimators=%d must be larger or equal to " + "len(estimators_)=%d when warm_start==True" + % (self.n_estimators, len(self.estimators_)) + ) elif n_more_estimators == 0: - warn("Warm-start fitting without increasing n_estimators does not " - "fit new trees.") + warn( + "Warm-start fitting without increasing n_estimators does not " + "fit new trees." + ) return self # Parallel loop - n_jobs, n_estimators, starts = _partition_estimators(n_more_estimators, - self.n_jobs) + n_jobs, n_estimators, starts = _partition_estimators( + n_more_estimators, self.n_jobs + ) total_n_estimators = sum(n_estimators) # Advance random state to state after training @@ -367,24 +544,30 @@ def _fit(self, X, y, max_samples=None, max_depth=None, sample_weight=None): seeds = random_state.randint(MAX_INT, size=n_more_estimators) self._seeds = seeds - all_results = Parallel(n_jobs=n_jobs, verbose=self.verbose, - **self._parallel_args())( + all_results = Parallel( + n_jobs=n_jobs, verbose=self.verbose, **self._parallel_args() + )( delayed(_parallel_build_estimators)( n_estimators[i], self, X, y, - sample_weight, - seeds[starts[i]:starts[i + 1]], + seeds[starts[i] : starts[i + 1]], total_n_estimators, - verbose=self.verbose) - for i in range(n_jobs)) + verbose=self.verbose, + check_input=check_input, + fit_params=routed_params.estimator.fit, + ) + for i in range(n_jobs) + ) # Reduce - self.estimators_ += list(itertools.chain.from_iterable( - t[0] for t in all_results)) - self.estimators_features_ += list(itertools.chain.from_iterable( - t[1] for t in all_results)) + self.estimators_ += list( + itertools.chain.from_iterable(t[0] for t in all_results) + ) + self.estimators_features_ += list( + itertools.chain.from_iterable(t[1] for t in all_results) + ) if self.oob_score: self._set_oob_score(X, y) @@ -398,8 +581,7 @@ def _set_oob_score(self, X, y): def _validate_y(self, y): if len(y.shape) == 1 or y.shape[1] == 1: return column_or_1d(y, warn=True) - else: - return y + return y def _get_estimators_indices(self): # Get drawn indices along both sample and feature axes @@ -407,9 +589,14 @@ def _get_estimators_indices(self): # Operations accessing random_state must be performed identically # to those in `_parallel_build_estimators()` feature_indices, sample_indices = _generate_bagging_indices( - seed, self.bootstrap_features, self.bootstrap, - self.n_features_in_, self._n_samples, self._max_features, - self._max_samples) + seed, + self.bootstrap_features, + self.bootstrap, + self.n_features_in_, + self._n_samples, + self._max_features, + self._max_samples, + ) yield feature_indices, sample_indices @@ -426,18 +613,75 @@ def estimators_samples_(self): to reduce the object memory footprint by not storing the sampling data. Thus fetching the property may be slower than expected. """ - return [sample_indices - for _, sample_indices in self._get_estimators_indices()] - - # TODO: Remove in 1.2 - # mypy error: Decorated property not supported - @deprecated( # type: ignore - "Attribute n_features_ was deprecated in version 1.0 and will be " - "removed in 1.2. Use 'n_features_in_' instead." - ) - @property - def n_features_(self): - return self.n_features_in_ + return [sample_indices for _, sample_indices in self._get_estimators_indices()] + + def get_metadata_routing(self): + """Get metadata routing of this object. + + Please check :ref:`User Guide ` on how the routing + mechanism works. + + .. versionadded:: 1.5 + + Returns + ------- + routing : MetadataRouter + A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating + routing information. + """ + router = MetadataRouter(owner=self.__class__.__name__) + + method_mapping = MethodMapping() + method_mapping.add(caller="fit", callee="fit").add( + caller="decision_function", callee="decision_function" + ) + + # the router needs to be built depending on whether the sub-estimator has a + # `predict_proba` method (as BaggingClassifier decides dynamically at runtime): + if hasattr(self._get_estimator(), "predict_proba"): + ( + method_mapping.add(caller="predict", callee="predict_proba").add( + caller="predict_proba", callee="predict_proba" + ) + ) + + else: + ( + method_mapping.add(caller="predict", callee="predict").add( + caller="predict_proba", callee="predict" + ) + ) + + # the router needs to be built depending on whether the sub-estimator has a + # `predict_log_proba` method (as BaggingClassifier decides dynamically at + # runtime): + if hasattr(self._get_estimator(), "predict_log_proba"): + method_mapping.add(caller="predict_log_proba", callee="predict_log_proba") + + else: + # if `predict_log_proba` is not available in BaggingClassifier's + # sub-estimator, the routing should go to its `predict_proba` if it is + # available or else to its `predict` method; according to how + # `sample_weight` is passed to the respective methods dynamically at + # runtime: + if hasattr(self._get_estimator(), "predict_proba"): + method_mapping.add(caller="predict_log_proba", callee="predict_proba") + + else: + method_mapping.add(caller="predict_log_proba", callee="predict") + + router.add(estimator=self._get_estimator(), method_mapping=method_mapping) + return router + + @abstractmethod + def _get_estimator(self): + """Resolve which estimator to return.""" + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.sparse = get_tags(self._get_estimator()).input_tags.sparse + tags.input_tags.allow_nan = get_tags(self._get_estimator()).input_tags.allow_nan + return tags class BaggingClassifier(ClassifierMixin, BaseBagging): @@ -466,11 +710,14 @@ class BaggingClassifier(ClassifierMixin, BaseBagging): Parameters ---------- - base_estimator : object, default=None + estimator : object, default=None The base estimator to fit on random subsets of the dataset. If None, then the base estimator is a :class:`~sklearn.tree.DecisionTreeClassifier`. + .. versionadded:: 1.2 + `base_estimator` was renamed to `estimator`. + n_estimators : int, default=10 The number of base estimators in the ensemble. @@ -487,7 +734,7 @@ class BaggingClassifier(ClassifierMixin, BaseBagging): details). - If int, then draw `max_features` features. - - If float, then draw `max_features * X.shape[1]` features. + - If float, then draw `max(1, int(max_features * n_features_in_))` features. bootstrap : bool, default=True Whether samples are drawn with replacement. If False, sampling @@ -527,15 +774,22 @@ class BaggingClassifier(ClassifierMixin, BaseBagging): Attributes ---------- - base_estimator_ : estimator + estimator_ : estimator The base estimator from which the ensemble is grown. - n_features_ : int - The number of features when :meth:`fit` is performed. + .. versionadded:: 1.2 + `base_estimator_` was renamed to `estimator_`. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 - .. deprecated:: 1.0 - Attribute `n_features_` was deprecated in version 1.0 and will be - removed in 1.2. Use `n_features_in_` instead. + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 estimators_ : list of estimators The collection of fitted base estimators. @@ -564,18 +818,9 @@ class BaggingClassifier(ClassifierMixin, BaseBagging): `oob_decision_function_` might contain NaN. This attribute exists only when ``oob_score`` is True. - Examples + See Also -------- - >>> from sklearn.svm import SVC - >>> from sklearn.ensemble import BaggingClassifier - >>> from sklearn.datasets import make_classification - >>> X, y = make_classification(n_samples=100, n_features=4, - ... n_informative=2, n_redundant=0, - ... random_state=0, shuffle=False) - >>> clf = BaggingClassifier(base_estimator=SVC(), - ... n_estimators=10, random_state=0).fit(X, y) - >>> clf.predict([[0, 0, 0, 0]]) - array([1]) + BaggingRegressor : A Bagging regressor. References ---------- @@ -592,23 +837,38 @@ class BaggingClassifier(ClassifierMixin, BaseBagging): .. [4] G. Louppe and P. Geurts, "Ensembles on Random Patches", Machine Learning and Knowledge Discovery in Databases, 346-361, 2012. + + Examples + -------- + >>> from sklearn.svm import SVC + >>> from sklearn.ensemble import BaggingClassifier + >>> from sklearn.datasets import make_classification + >>> X, y = make_classification(n_samples=100, n_features=4, + ... n_informative=2, n_redundant=0, + ... random_state=0, shuffle=False) + >>> clf = BaggingClassifier(estimator=SVC(), + ... n_estimators=10, random_state=0).fit(X, y) + >>> clf.predict([[0, 0, 0, 0]]) + array([1]) """ - @_deprecate_positional_args - def __init__(self, - base_estimator=None, - n_estimators=10, *, - max_samples=1.0, - max_features=1.0, - bootstrap=True, - bootstrap_features=False, - oob_score=False, - warm_start=False, - n_jobs=None, - random_state=None, - verbose=0): + def __init__( + self, + estimator=None, + n_estimators=10, + *, + max_samples=1.0, + max_features=1.0, + bootstrap=True, + bootstrap_features=False, + oob_score=False, + warm_start=False, + n_jobs=None, + random_state=None, + verbose=0, + ): super().__init__( - base_estimator, + estimator=estimator, n_estimators=n_estimators, max_samples=max_samples, max_features=max_features, @@ -618,12 +878,14 @@ def __init__(self, warm_start=warm_start, n_jobs=n_jobs, random_state=random_state, - verbose=verbose) + verbose=verbose, + ) - def _validate_estimator(self): - """Check the estimator and set the base_estimator_ attribute.""" - super()._validate_estimator( - default=DecisionTreeClassifier()) + def _get_estimator(self): + """Resolve which estimator to return (default is DecisionTreeClassifier)""" + if self.estimator is None: + return DecisionTreeClassifier() + return self.estimator def _set_oob_score(self, X, y): n_samples = y.shape[0] @@ -631,15 +893,16 @@ def _set_oob_score(self, X, y): predictions = np.zeros((n_samples, n_classes_)) - for estimator, samples, features in zip(self.estimators_, - self.estimators_samples_, - self.estimators_features_): + for estimator, samples, features in zip( + self.estimators_, self.estimators_samples_, self.estimators_features_ + ): # Create mask for OOB samples mask = ~indices_to_mask(samples, n_samples) if hasattr(estimator, "predict_proba"): predictions[mask, :] += estimator.predict_proba( - (X[mask, :])[:, features]) + (X[mask, :])[:, features] + ) else: p = estimator.predict((X[mask, :])[:, features]) @@ -651,12 +914,13 @@ def _set_oob_score(self, X, y): j += 1 if (predictions.sum(axis=1) == 0).any(): - warn("Some inputs do not have OOB scores. " - "This probably means too few estimators were used " - "to compute any reliable oob estimates.") + warn( + "Some inputs do not have OOB scores. " + "This probably means too few estimators were used " + "to compute any reliable oob estimates." + ) - oob_decision_function = (predictions / - predictions.sum(axis=1)[:, np.newaxis]) + oob_decision_function = predictions / predictions.sum(axis=1)[:, np.newaxis] oob_score = accuracy_score(y, np.argmax(predictions, axis=1)) self.oob_decision_function_ = oob_decision_function @@ -670,7 +934,7 @@ def _validate_y(self, y): return y - def predict(self, X): + def predict(self, X, **params): """Predict class for X. The predicted class of an input sample is computed as the class with @@ -683,16 +947,28 @@ def predict(self, X): The training input samples. Sparse matrices are accepted only if they are supported by the base estimator. + **params : dict + Parameters routed to the `predict_proba` (if available) or the `predict` + method (otherwise) of the sub-estimators via the metadata routing API. + + .. versionadded:: 1.7 + + Only available if + `sklearn.set_config(enable_metadata_routing=True)` is set. See + :ref:`Metadata Routing User Guide ` for more + details. + Returns ------- y : ndarray of shape (n_samples,) The predicted classes. """ - predicted_probabilitiy = self.predict_proba(X) - return self.classes_.take((np.argmax(predicted_probabilitiy, axis=1)), - axis=0) + _raise_for_params(params, self, "predict") + + predicted_probabilitiy = self.predict_proba(X, **params) + return self.classes_.take((np.argmax(predicted_probabilitiy, axis=1)), axis=0) - def predict_proba(self, X): + def predict_proba(self, X, **params): """Predict class probabilities for X. The predicted class probabilities of an input sample is computed as @@ -708,38 +984,65 @@ def predict_proba(self, X): The training input samples. Sparse matrices are accepted only if they are supported by the base estimator. + **params : dict + Parameters routed to the `predict_proba` (if available) or the `predict` + method (otherwise) of the sub-estimators via the metadata routing API. + + .. versionadded:: 1.7 + + Only available if + `sklearn.set_config(enable_metadata_routing=True)` is set. See + :ref:`Metadata Routing User Guide ` for more + details. + Returns ------- p : ndarray of shape (n_samples, n_classes) The class probabilities of the input samples. The order of the classes corresponds to that in the attribute :term:`classes_`. """ + _raise_for_params(params, self, "predict_proba") + check_is_fitted(self) # Check data - X = self._validate_data( - X, accept_sparse=['csr', 'csc'], dtype=None, - force_all_finite=False, reset=False + X = validate_data( + self, + X, + accept_sparse=["csr", "csc"], + dtype=None, + ensure_all_finite=False, + reset=False, ) + if _routing_enabled(): + routed_params = process_routing(self, "predict_proba", **params) + else: + routed_params = Bunch() + routed_params.estimator = Bunch(predict_proba=Bunch()) + # Parallel loop - n_jobs, n_estimators, starts = _partition_estimators(self.n_estimators, - self.n_jobs) + n_jobs, _, starts = _partition_estimators(self.n_estimators, self.n_jobs) - all_proba = Parallel(n_jobs=n_jobs, verbose=self.verbose, - **self._parallel_args())( + all_proba = Parallel( + n_jobs=n_jobs, verbose=self.verbose, **self._parallel_args() + )( delayed(_parallel_predict_proba)( - self.estimators_[starts[i]:starts[i + 1]], - self.estimators_features_[starts[i]:starts[i + 1]], + self.estimators_[starts[i] : starts[i + 1]], + self.estimators_features_[starts[i] : starts[i + 1]], X, - self.n_classes_) - for i in range(n_jobs)) + self.n_classes_, + predict_params=routed_params.estimator.get("predict", None), + predict_proba_params=routed_params.estimator.get("predict_proba", None), + ) + for i in range(n_jobs) + ) # Reduce proba = sum(all_proba) / self.n_estimators return proba - def predict_log_proba(self, X): + def predict_log_proba(self, X, **params): """Predict class log-probabilities for X. The predicted class log-probabilities of an input sample is computed as @@ -752,31 +1055,59 @@ def predict_log_proba(self, X): The training input samples. Sparse matrices are accepted only if they are supported by the base estimator. + **params : dict + Parameters routed to the `predict_log_proba`, the `predict_proba` or the + `proba` method of the sub-estimators via the metadata routing API. The + routing is tried in the mentioned order depending on whether this method is + available on the sub-estimator. + + .. versionadded:: 1.7 + + Only available if + `sklearn.set_config(enable_metadata_routing=True)` is set. See + :ref:`Metadata Routing User Guide ` for more + details. + Returns ------- p : ndarray of shape (n_samples, n_classes) The class log-probabilities of the input samples. The order of the classes corresponds to that in the attribute :term:`classes_`. """ + _raise_for_params(params, self, "predict_log_proba") + check_is_fitted(self) - if hasattr(self.base_estimator_, "predict_log_proba"): + + if hasattr(self.estimator_, "predict_log_proba"): # Check data - X = self._validate_data( - X, accept_sparse=['csr', 'csc'], dtype=None, - force_all_finite=False, reset=False + X = validate_data( + self, + X, + accept_sparse=["csr", "csc"], + dtype=None, + ensure_all_finite=False, + reset=False, ) + if _routing_enabled(): + routed_params = process_routing(self, "predict_log_proba", **params) + else: + routed_params = Bunch() + routed_params.estimator = Bunch(predict_log_proba=Bunch()) + # Parallel loop - n_jobs, n_estimators, starts = _partition_estimators( - self.n_estimators, self.n_jobs) + n_jobs, _, starts = _partition_estimators(self.n_estimators, self.n_jobs) all_log_proba = Parallel(n_jobs=n_jobs, verbose=self.verbose)( delayed(_parallel_predict_log_proba)( - self.estimators_[starts[i]:starts[i + 1]], - self.estimators_features_[starts[i]:starts[i + 1]], + self.estimators_[starts[i] : starts[i + 1]], + self.estimators_features_[starts[i] : starts[i + 1]], X, - self.n_classes_) - for i in range(n_jobs)) + self.n_classes_, + params=routed_params.estimator.predict_log_proba, + ) + for i in range(n_jobs) + ) # Reduce log_proba = all_log_proba[0] @@ -786,13 +1117,15 @@ def predict_log_proba(self, X): log_proba -= np.log(self.n_estimators) - return log_proba - else: - return np.log(self.predict_proba(X)) + log_proba = np.log(self.predict_proba(X, **params)) + + return log_proba - @if_delegate_has_method(delegate='base_estimator') - def decision_function(self, X): + @available_if( + _estimator_has("decision_function", delegates=("estimators_", "estimator")) + ) + def decision_function(self, X, **params): """Average of the decision functions of the base classifiers. Parameters @@ -801,6 +1134,17 @@ def decision_function(self, X): The training input samples. Sparse matrices are accepted only if they are supported by the base estimator. + **params : dict + Parameters routed to the `decision_function` method of the sub-estimators + via the metadata routing API. + + .. versionadded:: 1.7 + + Only available if + `sklearn.set_config(enable_metadata_routing=True)` is set. See + :ref:`Metadata Routing User Guide ` for more + details. + Returns ------- score : ndarray of shape (n_samples, k) @@ -808,26 +1152,39 @@ def decision_function(self, X): to the classes in sorted order, as they appear in the attribute ``classes_``. Regression and binary classification are special cases with ``k == 1``, otherwise ``k==n_classes``. - """ + _raise_for_params(params, self, "decision_function") + check_is_fitted(self) # Check data - X = self._validate_data( - X, accept_sparse=['csr', 'csc'], dtype=None, - force_all_finite=False, reset=False + X = validate_data( + self, + X, + accept_sparse=["csr", "csc"], + dtype=None, + ensure_all_finite=False, + reset=False, ) + if _routing_enabled(): + routed_params = process_routing(self, "decision_function", **params) + else: + routed_params = Bunch() + routed_params.estimator = Bunch(decision_function=Bunch()) + # Parallel loop - n_jobs, n_estimators, starts = _partition_estimators(self.n_estimators, - self.n_jobs) + n_jobs, _, starts = _partition_estimators(self.n_estimators, self.n_jobs) all_decisions = Parallel(n_jobs=n_jobs, verbose=self.verbose)( delayed(_parallel_decision_function)( - self.estimators_[starts[i]:starts[i + 1]], - self.estimators_features_[starts[i]:starts[i + 1]], - X) - for i in range(n_jobs)) + self.estimators_[starts[i] : starts[i + 1]], + self.estimators_features_[starts[i] : starts[i + 1]], + X, + params=routed_params.estimator.decision_function, + ) + for i in range(n_jobs) + ) # Reduce decisions = sum(all_decisions) / self.n_estimators @@ -861,11 +1218,14 @@ class BaggingRegressor(RegressorMixin, BaseBagging): Parameters ---------- - base_estimator : object, default=None + estimator : object, default=None The base estimator to fit on random subsets of the dataset. If None, then the base estimator is a :class:`~sklearn.tree.DecisionTreeRegressor`. + .. versionadded:: 1.2 + `base_estimator` was renamed to `estimator`. + n_estimators : int, default=10 The number of base estimators in the ensemble. @@ -882,7 +1242,7 @@ class BaggingRegressor(RegressorMixin, BaseBagging): details). - If int, then draw `max_features` features. - - If float, then draw `max_features * X.shape[1]` features. + - If float, then draw `max(1, int(max_features * n_features_in_))` features. bootstrap : bool, default=True Whether samples are drawn with replacement. If False, sampling @@ -919,15 +1279,22 @@ class BaggingRegressor(RegressorMixin, BaseBagging): Attributes ---------- - base_estimator_ : estimator + estimator_ : estimator The base estimator from which the ensemble is grown. - n_features_ : int - The number of features when :meth:`fit` is performed. + .. versionadded:: 1.2 + `base_estimator_` was renamed to `estimator_`. + + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. - .. deprecated:: 1.0 - Attribute `n_features_` was deprecated in version 1.0 and will be - removed in 1.2. Use `n_features_in_` instead. + .. versionadded:: 1.0 estimators_ : list of estimators The collection of fitted sub-estimators. @@ -950,18 +1317,9 @@ class BaggingRegressor(RegressorMixin, BaseBagging): `oob_prediction_` might contain NaN. This attribute exists only when ``oob_score`` is True. - Examples + See Also -------- - >>> from sklearn.svm import SVR - >>> from sklearn.ensemble import BaggingRegressor - >>> from sklearn.datasets import make_regression - >>> X, y = make_regression(n_samples=100, n_features=4, - ... n_informative=2, n_targets=1, - ... random_state=0, shuffle=False) - >>> regr = BaggingRegressor(base_estimator=SVR(), - ... n_estimators=10, random_state=0).fit(X, y) - >>> regr.predict([[0, 0, 0, 0]]) - array([-2.8720...]) + BaggingClassifier : A Bagging classifier. References ---------- @@ -978,22 +1336,38 @@ class BaggingRegressor(RegressorMixin, BaseBagging): .. [4] G. Louppe and P. Geurts, "Ensembles on Random Patches", Machine Learning and Knowledge Discovery in Databases, 346-361, 2012. + + Examples + -------- + >>> from sklearn.svm import SVR + >>> from sklearn.ensemble import BaggingRegressor + >>> from sklearn.datasets import make_regression + >>> X, y = make_regression(n_samples=100, n_features=4, + ... n_informative=2, n_targets=1, + ... random_state=0, shuffle=False) + >>> regr = BaggingRegressor(estimator=SVR(), + ... n_estimators=10, random_state=0).fit(X, y) + >>> regr.predict([[0, 0, 0, 0]]) + array([-2.8720]) """ - @_deprecate_positional_args - def __init__(self, - base_estimator=None, - n_estimators=10, *, - max_samples=1.0, - max_features=1.0, - bootstrap=True, - bootstrap_features=False, - oob_score=False, - warm_start=False, - n_jobs=None, - random_state=None, - verbose=0): + + def __init__( + self, + estimator=None, + n_estimators=10, + *, + max_samples=1.0, + max_features=1.0, + bootstrap=True, + bootstrap_features=False, + oob_score=False, + warm_start=False, + n_jobs=None, + random_state=None, + verbose=0, + ): super().__init__( - base_estimator, + estimator=estimator, n_estimators=n_estimators, max_samples=max_samples, max_features=max_features, @@ -1003,9 +1377,10 @@ def __init__(self, warm_start=warm_start, n_jobs=n_jobs, random_state=random_state, - verbose=verbose) + verbose=verbose, + ) - def predict(self, X): + def predict(self, X, **params): """Predict regression target for X. The predicted regression target of an input sample is computed as the @@ -1017,48 +1392,68 @@ def predict(self, X): The training input samples. Sparse matrices are accepted only if they are supported by the base estimator. + **params : dict + Parameters routed to the `predict` method of the sub-estimators via the + metadata routing API. + + .. versionadded:: 1.7 + + Only available if + `sklearn.set_config(enable_metadata_routing=True)` is set. See + :ref:`Metadata Routing User Guide ` for more + details. + Returns ------- y : ndarray of shape (n_samples,) The predicted values. """ + _raise_for_params(params, self, "predict") + check_is_fitted(self) # Check data - X = self._validate_data( - X, accept_sparse=['csr', 'csc'], dtype=None, - force_all_finite=False, reset=False + X = validate_data( + self, + X, + accept_sparse=["csr", "csc"], + dtype=None, + ensure_all_finite=False, + reset=False, ) + if _routing_enabled(): + routed_params = process_routing(self, "predict", **params) + else: + routed_params = Bunch() + routed_params.estimator = Bunch(predict=Bunch()) + # Parallel loop - n_jobs, n_estimators, starts = _partition_estimators(self.n_estimators, - self.n_jobs) + n_jobs, _, starts = _partition_estimators(self.n_estimators, self.n_jobs) all_y_hat = Parallel(n_jobs=n_jobs, verbose=self.verbose)( delayed(_parallel_predict_regression)( - self.estimators_[starts[i]:starts[i + 1]], - self.estimators_features_[starts[i]:starts[i + 1]], - X) - for i in range(n_jobs)) + self.estimators_[starts[i] : starts[i + 1]], + self.estimators_features_[starts[i] : starts[i + 1]], + X, + params=routed_params.estimator.predict, + ) + for i in range(n_jobs) + ) # Reduce y_hat = sum(all_y_hat) / self.n_estimators return y_hat - def _validate_estimator(self): - """Check the estimator and set the base_estimator_ attribute.""" - super()._validate_estimator( - default=DecisionTreeRegressor()) - def _set_oob_score(self, X, y): n_samples = y.shape[0] predictions = np.zeros((n_samples,)) n_predictions = np.zeros((n_samples,)) - for estimator, samples, features in zip(self.estimators_, - self.estimators_samples_, - self.estimators_features_): + for estimator, samples, features in zip( + self.estimators_, self.estimators_samples_, self.estimators_features_ + ): # Create mask for OOB samples mask = ~indices_to_mask(samples, n_samples) @@ -1066,12 +1461,20 @@ def _set_oob_score(self, X, y): n_predictions[mask] += 1 if (n_predictions == 0).any(): - warn("Some inputs do not have OOB scores. " - "This probably means too few estimators were used " - "to compute any reliable oob estimates.") + warn( + "Some inputs do not have OOB scores. " + "This probably means too few estimators were used " + "to compute any reliable oob estimates." + ) n_predictions[n_predictions == 0] = 1 predictions /= n_predictions self.oob_prediction_ = predictions self.oob_score_ = r2_score(y, predictions) + + def _get_estimator(self): + """Resolve which estimator to return (default is DecisionTreeClassifier)""" + if self.estimator is None: + return DecisionTreeRegressor() + return self.estimator diff --git a/sklearn/ensemble/_base.py b/sklearn/ensemble/_base.py index 93891a2b719ab..e04645eec174f 100644 --- a/sklearn/ensemble/_base.py +++ b/sklearn/ensemble/_base.py @@ -1,42 +1,42 @@ """Base class for ensemble-based estimators.""" -# Authors: Gilles Louppe -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause from abc import ABCMeta, abstractmethod -import numbers -from typing import List import numpy as np - from joblib import effective_n_jobs -from ..base import clone -from ..base import is_classifier, is_regressor -from ..base import BaseEstimator -from ..base import MetaEstimatorMixin -from ..utils import Bunch, _print_elapsed_time -from ..utils import check_random_state +from ..base import BaseEstimator, MetaEstimatorMixin, clone, is_classifier, is_regressor +from ..utils import Bunch, check_random_state +from ..utils._tags import get_tags +from ..utils._user_interface import _print_elapsed_time +from ..utils.metadata_routing import _routing_enabled from ..utils.metaestimators import _BaseComposition -def _fit_single_estimator(estimator, X, y, sample_weight=None, - message_clsname=None, message=None): +def _fit_single_estimator( + estimator, X, y, fit_params, message_clsname=None, message=None +): """Private function used to fit an estimator within a job.""" - if sample_weight is not None: + # TODO(SLEP6): remove if-condition for unrouted sample_weight when metadata + # routing can't be disabled. + if not _routing_enabled() and "sample_weight" in fit_params: try: with _print_elapsed_time(message_clsname, message): - estimator.fit(X, y, sample_weight=sample_weight) + estimator.fit(X, y, sample_weight=fit_params["sample_weight"]) except TypeError as exc: if "unexpected keyword argument 'sample_weight'" in str(exc): raise TypeError( - "Underlying estimator {} does not support sample weights." - .format(estimator.__class__.__name__) + "Underlying estimator {} does not support sample weights.".format( + estimator.__class__.__name__ + ) ) from exc raise else: with _print_elapsed_time(message_clsname, message): - estimator.fit(X, y) + estimator.fit(X, y, **fit_params) return estimator @@ -71,7 +71,7 @@ def _set_random_states(estimator, random_state=None): random_state = check_random_state(random_state) to_set = {} for key in sorted(estimator.get_params(deep=True)): - if key == 'random_state' or key.endswith('__random_state'): + if key == "random_state" or key.endswith("__random_state"): to_set[key] = random_state.randint(np.iinfo(np.int32).max) if to_set: @@ -86,7 +86,7 @@ class BaseEnsemble(MetaEstimatorMixin, BaseEstimator, metaclass=ABCMeta): Parameters ---------- - base_estimator : object + estimator : object The base estimator from which the ensemble is built. n_estimators : int, default=10 @@ -98,58 +98,48 @@ class BaseEnsemble(MetaEstimatorMixin, BaseEstimator, metaclass=ABCMeta): Attributes ---------- - base_estimator_ : estimator + estimator_ : estimator The base estimator from which the ensemble is grown. estimators_ : list of estimators The collection of fitted base estimators. """ - # overwrite _required_parameters from MetaEstimatorMixin - _required_parameters: List[str] = [] - @abstractmethod - def __init__(self, base_estimator, *, n_estimators=10, - estimator_params=tuple()): + def __init__( + self, + estimator=None, + *, + n_estimators=10, + estimator_params=tuple(), + ): # Set parameters - self.base_estimator = base_estimator + self.estimator = estimator self.n_estimators = n_estimators self.estimator_params = estimator_params - # Don't instantiate estimators now! Parameters of base_estimator might + # Don't instantiate estimators now! Parameters of estimator might # still change. Eg., when grid-searching with the nested object syntax. # self.estimators_ needs to be filled by the derived classes in fit. def _validate_estimator(self, default=None): - """Check the estimator and the n_estimator attribute. + """Check the base estimator. - Sets the base_estimator_` attributes. + Sets the `estimator_` attributes. """ - if not isinstance(self.n_estimators, numbers.Integral): - raise ValueError("n_estimators must be an integer, " - "got {0}.".format(type(self.n_estimators))) - - if self.n_estimators <= 0: - raise ValueError("n_estimators must be greater than zero, " - "got {0}.".format(self.n_estimators)) - - if self.base_estimator is not None: - self.base_estimator_ = self.base_estimator + if self.estimator is not None: + self.estimator_ = self.estimator else: - self.base_estimator_ = default - - if self.base_estimator_ is None: - raise ValueError("base_estimator cannot be None") + self.estimator_ = default def _make_estimator(self, append=True, random_state=None): - """Make and configure a copy of the `base_estimator_` attribute. + """Make and configure a copy of the `estimator_` attribute. Warning: This method should be used to properly instantiate new sub-estimators. """ - estimator = clone(self.base_estimator_) - estimator.set_params(**{p: getattr(self, p) - for p in self.estimator_params}) + estimator = clone(self.estimator_) + estimator.set_params(**{p: getattr(self, p) for p in self.estimator_params}) if random_state is not None: _set_random_states(estimator, random_state) @@ -178,16 +168,16 @@ def _partition_estimators(n_estimators, n_jobs): n_jobs = min(effective_n_jobs(n_jobs), n_estimators) # Partition estimators between jobs - n_estimators_per_job = np.full(n_jobs, n_estimators // n_jobs, - dtype=int) - n_estimators_per_job[:n_estimators % n_jobs] += 1 + n_estimators_per_job = np.full(n_jobs, n_estimators // n_jobs, dtype=int) + n_estimators_per_job[: n_estimators % n_jobs] += 1 starts = np.cumsum(n_estimators_per_job) return n_jobs, n_estimators_per_job.tolist(), [0] + starts.tolist() -class _BaseHeterogeneousEnsemble(MetaEstimatorMixin, _BaseComposition, - metaclass=ABCMeta): +class _BaseHeterogeneousEnsemble( + MetaEstimatorMixin, _BaseComposition, metaclass=ABCMeta +): """Base class for heterogeneous ensemble of learners. Parameters @@ -206,10 +196,14 @@ class _BaseHeterogeneousEnsemble(MetaEstimatorMixin, _BaseComposition, appear in `estimators_`. """ - _required_parameters = ['estimators'] - @property def named_estimators(self): + """Dictionary to access any fitted sub-estimators by name. + + Returns + ------- + :class:`~sklearn.utils.Bunch` + """ return Bunch(**dict(self.estimators)) @abstractmethod @@ -217,27 +211,29 @@ def __init__(self, estimators): self.estimators = estimators def _validate_estimators(self): - if self.estimators is None or len(self.estimators) == 0: + if len(self.estimators) == 0 or not all( + isinstance(item, (tuple, list)) and isinstance(item[0], str) + for item in self.estimators + ): raise ValueError( - "Invalid 'estimators' attribute, 'estimators' should be a list" - " of (string, estimator) tuples." + "Invalid 'estimators' attribute, 'estimators' should be a " + "non-empty list of (string, estimator) tuples." ) names, estimators = zip(*self.estimators) # defined by MetaEstimatorMixin self._validate_names(names) - has_estimator = any(est != 'drop' for est in estimators) + has_estimator = any(est != "drop" for est in estimators) if not has_estimator: raise ValueError( "All estimators are dropped. At least one is required " "to be an estimator." ) - is_estimator_type = (is_classifier if is_classifier(self) - else is_regressor) + is_estimator_type = is_classifier if is_classifier(self) else is_regressor for est in estimators: - if est != 'drop' and not is_estimator_type(est): + if est != "drop" and not is_estimator_type(est): raise ValueError( "The estimator {} should be a {}.".format( est.__class__.__name__, is_estimator_type.__name__[3:] @@ -262,8 +258,13 @@ def set_params(self, **params): parameters of the estimator, the individual estimator of the estimators can also be set, or can be removed by setting them to 'drop'. + + Returns + ------- + self : object + Estimator instance. """ - super()._set_params('estimators', **params) + super()._set_params("estimators", **params) return self def get_params(self, deep=True): @@ -278,5 +279,29 @@ def get_params(self, deep=True): deep : bool, default=True Setting it to True gets the various estimators and the parameters of the estimators as well. + + Returns + ------- + params : dict + Parameter and estimator names mapped to their values or parameter + names mapped to their values. """ - return super()._get_params('estimators', deep=deep) + return super()._get_params("estimators", deep=deep) + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + try: + tags.input_tags.allow_nan = all( + get_tags(est[1]).input_tags.allow_nan if est[1] != "drop" else True + for est in self.estimators + ) + tags.input_tags.sparse = all( + get_tags(est[1]).input_tags.sparse if est[1] != "drop" else True + for est in self.estimators + ) + except Exception: + # If `estimators` does not comply with our API (list of tuples) then it will + # fail. In this case, we assume that `allow_nan` and `sparse` are False but + # the parameter validation will raise an error during `fit`. + pass # pragma: no cover + return tags diff --git a/sklearn/ensemble/_forest.py b/sklearn/ensemble/_forest.py index a93e9b7ee877e..5def6ac60816b 100644 --- a/sklearn/ensemble/_forest.py +++ b/sklearn/ensemble/_forest.py @@ -32,46 +32,58 @@ class calls the ``fit`` method of each sub-estimator on random samples Single and multi-output problems are both handled. """ -# Authors: Gilles Louppe -# Brian Holt -# Joly Arnaud -# Fares Hedayati -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause - -import numbers -from warnings import catch_warnings, simplefilter, warn import threading - from abc import ABCMeta, abstractmethod +from numbers import Integral, Real +from warnings import catch_warnings, simplefilter, warn + import numpy as np -from scipy.sparse import issparse from scipy.sparse import hstack as sparse_hstack -from joblib import Parallel +from scipy.sparse import issparse -from ..base import is_classifier -from ..base import ClassifierMixin, RegressorMixin, MultiOutputMixin +from ..base import ( + ClassifierMixin, + MultiOutputMixin, + RegressorMixin, + TransformerMixin, + _fit_context, + is_classifier, +) +from ..exceptions import DataConversionWarning from ..metrics import accuracy_score, r2_score from ..preprocessing import OneHotEncoder -from ..tree import (DecisionTreeClassifier, DecisionTreeRegressor, - ExtraTreeClassifier, ExtraTreeRegressor) -from ..tree._tree import DTYPE, DOUBLE -from ..utils import check_random_state, compute_sample_weight, deprecated -from ..exceptions import DataConversionWarning -from ._base import BaseEnsemble, _partition_estimators -from ..utils.fixes import delayed -from ..utils.fixes import _joblib_parallel_args +from ..tree import ( + BaseDecisionTree, + DecisionTreeClassifier, + DecisionTreeRegressor, + ExtraTreeClassifier, + ExtraTreeRegressor, +) +from ..tree._tree import DOUBLE, DTYPE +from ..utils import check_random_state, compute_sample_weight +from ..utils._param_validation import Interval, RealNotInt, StrOptions +from ..utils._tags import get_tags from ..utils.multiclass import check_classification_targets, type_of_target -from ..utils.validation import check_is_fitted, _check_sample_weight -from ..utils.validation import _deprecate_positional_args - +from ..utils.parallel import Parallel, delayed +from ..utils.validation import ( + _check_feature_names_in, + _check_sample_weight, + _num_samples, + check_is_fitted, + validate_data, +) +from ._base import BaseEnsemble, _partition_estimators -__all__ = ["RandomForestClassifier", - "RandomForestRegressor", - "ExtraTreesClassifier", - "ExtraTreesRegressor", - "RandomTreesEmbedding"] +__all__ = [ + "ExtraTreesClassifier", + "ExtraTreesRegressor", + "RandomForestClassifier", + "RandomForestRegressor", + "RandomTreesEmbedding", +] MAX_INT = np.iinfo(np.int32).max @@ -87,7 +99,7 @@ def _get_n_samples_bootstrap(n_samples, max_samples): max_samples : int or float The maximum number of samples to draw from the total available: - if float, this indicates a fraction of the total and should be - the interval `(0, 1)`; + the interval `(0.0, 1.0]`; - if int, this indicates the exact number of samples; - if None, this indicates the total number of samples. @@ -99,20 +111,14 @@ def _get_n_samples_bootstrap(n_samples, max_samples): if max_samples is None: return n_samples - if isinstance(max_samples, numbers.Integral): - if not (1 <= max_samples <= n_samples): - msg = "`max_samples` must be in range 1 to {} but got value {}" + if isinstance(max_samples, Integral): + if max_samples > n_samples: + msg = "`max_samples` must be <= n_samples={} but got value {}" raise ValueError(msg.format(n_samples, max_samples)) return max_samples - if isinstance(max_samples, numbers.Real): - if not (0 < max_samples < 1): - msg = "`max_samples` must be in range (0, 1) but got value {}" - raise ValueError(msg.format(max_samples)) - return round(n_samples * max_samples) - - msg = "`max_samples` should be int or float, but got type '{}'" - raise TypeError(msg.format(type(max_samples))) + if isinstance(max_samples, Real): + return max(round(n_samples * max_samples), 1) def _generate_sample_indices(random_state, n_samples, n_samples_bootstrap): @@ -120,7 +126,9 @@ def _generate_sample_indices(random_state, n_samples, n_samples_bootstrap): Private function used to _parallel_build_trees function.""" random_instance = check_random_state(random_state) - sample_indices = random_instance.randint(0, n_samples, n_samples_bootstrap) + sample_indices = random_instance.randint( + 0, n_samples, n_samples_bootstrap, dtype=np.int32 + ) return sample_indices @@ -128,8 +136,9 @@ def _generate_sample_indices(random_state, n_samples, n_samples_bootstrap): def _generate_unsampled_indices(random_state, n_samples, n_samples_bootstrap): """ Private function used to forest._set_oob_score function.""" - sample_indices = _generate_sample_indices(random_state, n_samples, - n_samples_bootstrap) + sample_indices = _generate_sample_indices( + random_state, n_samples, n_samples_bootstrap + ) sample_counts = np.bincount(sample_indices, minlength=n_samples) unsampled_mask = sample_counts == 0 indices_range = np.arange(n_samples) @@ -138,38 +147,59 @@ def _generate_unsampled_indices(random_state, n_samples, n_samples_bootstrap): return unsampled_indices -def _parallel_build_trees(tree, forest, X, y, sample_weight, tree_idx, n_trees, - verbose=0, class_weight=None, - n_samples_bootstrap=None): +def _parallel_build_trees( + tree, + bootstrap, + X, + y, + sample_weight, + tree_idx, + n_trees, + verbose=0, + class_weight=None, + n_samples_bootstrap=None, + missing_values_in_feature_mask=None, +): """ Private function used to fit a single tree in parallel.""" if verbose > 1: print("building tree %d of %d" % (tree_idx + 1, n_trees)) - if forest.bootstrap: + if bootstrap: n_samples = X.shape[0] if sample_weight is None: curr_sample_weight = np.ones((n_samples,), dtype=np.float64) else: curr_sample_weight = sample_weight.copy() - indices = _generate_sample_indices(tree.random_state, n_samples, - n_samples_bootstrap) + indices = _generate_sample_indices( + tree.random_state, n_samples, n_samples_bootstrap + ) sample_counts = np.bincount(indices, minlength=n_samples) curr_sample_weight *= sample_counts - if class_weight == 'subsample': + if class_weight == "subsample": with catch_warnings(): - simplefilter('ignore', DeprecationWarning) - curr_sample_weight *= compute_sample_weight('auto', y, - indices=indices) - elif class_weight == 'balanced_subsample': - curr_sample_weight *= compute_sample_weight('balanced', y, - indices=indices) - - tree.fit(X, y, sample_weight=curr_sample_weight, check_input=False) + simplefilter("ignore", DeprecationWarning) + curr_sample_weight *= compute_sample_weight("auto", y, indices=indices) + elif class_weight == "balanced_subsample": + curr_sample_weight *= compute_sample_weight("balanced", y, indices=indices) + + tree._fit( + X, + y, + sample_weight=curr_sample_weight, + check_input=False, + missing_values_in_feature_mask=missing_values_in_feature_mask, + ) else: - tree.fit(X, y, sample_weight=sample_weight, check_input=False) + tree._fit( + X, + y, + sample_weight=sample_weight, + check_input=False, + missing_values_in_feature_mask=missing_values_in_feature_mask, + ) return tree @@ -182,23 +212,42 @@ class BaseForest(MultiOutputMixin, BaseEnsemble, metaclass=ABCMeta): instead. """ + _parameter_constraints: dict = { + "n_estimators": [Interval(Integral, 1, None, closed="left")], + "bootstrap": ["boolean"], + "oob_score": ["boolean", callable], + "n_jobs": [Integral, None], + "random_state": ["random_state"], + "verbose": ["verbose"], + "warm_start": ["boolean"], + "max_samples": [ + None, + Interval(RealNotInt, 0.0, 1.0, closed="right"), + Interval(Integral, 1, None, closed="left"), + ], + } + @abstractmethod - def __init__(self, - base_estimator, - n_estimators=100, *, - estimator_params=tuple(), - bootstrap=False, - oob_score=False, - n_jobs=None, - random_state=None, - verbose=0, - warm_start=False, - class_weight=None, - max_samples=None): + def __init__( + self, + estimator, + n_estimators=100, + *, + estimator_params=tuple(), + bootstrap=False, + oob_score=False, + n_jobs=None, + random_state=None, + verbose=0, + warm_start=False, + class_weight=None, + max_samples=None, + ): super().__init__( - base_estimator=base_estimator, + estimator=estimator, n_estimators=n_estimators, - estimator_params=estimator_params) + estimator_params=estimator_params, + ) self.bootstrap = bootstrap self.oob_score = oob_score @@ -227,10 +276,11 @@ def apply(self, X): return the index of the leaf x ends up in. """ X = self._validate_X_predict(X) - results = Parallel(n_jobs=self.n_jobs, verbose=self.verbose, - **_joblib_parallel_args(prefer="threads"))( - delayed(tree.apply)(X, check_input=False) - for tree in self.estimators_) + results = Parallel( + n_jobs=self.n_jobs, + verbose=self.verbose, + prefer="threads", + )(delayed(tree.apply)(X, check_input=False) for tree in self.estimators_) return np.array(results).T @@ -257,13 +307,16 @@ def decision_path(self, X): n_nodes_ptr : ndarray of shape (n_estimators + 1,) The columns from indicator[n_nodes_ptr[i]:n_nodes_ptr[i+1]] gives the indicator value for the i-th estimator. - """ X = self._validate_X_predict(X) - indicators = Parallel(n_jobs=self.n_jobs, verbose=self.verbose, - **_joblib_parallel_args(prefer='threads'))( + indicators = Parallel( + n_jobs=self.n_jobs, + verbose=self.verbose, + prefer="threads", + )( delayed(tree.decision_path)(X, check_input=False) - for tree in self.estimators_) + for tree in self.estimators_ + ) n_nodes = [0] n_nodes.extend([i.shape[1] for i in indicators]) @@ -271,6 +324,7 @@ def decision_path(self, X): return sparse_hstack(indicators).tocsr(), n_nodes_ptr + @_fit_context(prefer_skip_nested_validation=True) def fit(self, X, y, sample_weight=None): """ Build a forest of trees from the training set (X, y). @@ -296,14 +350,32 @@ def fit(self, X, y, sample_weight=None): Returns ------- self : object + Fitted estimator. """ # Validate or convert input data if issparse(y): - raise ValueError( - "sparse multilabel-indicator for y is not supported." + raise ValueError("sparse multilabel-indicator for y is not supported.") + + X, y = validate_data( + self, + X, + y, + multi_output=True, + accept_sparse="csc", + dtype=DTYPE, + ensure_all_finite=False, + ) + # _compute_missing_values_in_feature_mask checks if X has missing values and + # will raise an error if the underlying tree base estimator can't handle missing + # values. Only the criterion is required to determine if the tree supports + # missing values. + estimator = type(self.estimator)(criterion=self.criterion) + missing_values_in_feature_mask = ( + estimator._compute_missing_values_in_feature_mask( + X, estimator_name=self.__class__.__name__ ) - X, y = self._validate_data(X, y, multi_output=True, - accept_sparse="csc", dtype=DTYPE) + ) + if sample_weight is not None: sample_weight = _check_sample_weight(sample_weight, X) @@ -314,17 +386,34 @@ def fit(self, X, y, sample_weight=None): y = np.atleast_1d(y) if y.ndim == 2 and y.shape[1] == 1: - warn("A column-vector y was passed when a 1d array was" - " expected. Please change the shape of y to " - "(n_samples,), for example using ravel().", - DataConversionWarning, stacklevel=2) + warn( + ( + "A column-vector y was passed when a 1d array was" + " expected. Please change the shape of y to " + "(n_samples,), for example using ravel()." + ), + DataConversionWarning, + stacklevel=2, + ) if y.ndim == 1: # reshape is necessary to preserve the data contiguity against vs # [:, np.newaxis] that does not. y = np.reshape(y, (-1, 1)) - self.n_outputs_ = y.shape[1] + if self.criterion == "poisson": + if np.any(y < 0): + raise ValueError( + "Some value(s) of y are negative which is " + "not allowed for Poisson regression." + ) + if np.sum(y) <= 0: + raise ValueError( + "Sum of y is not strictly positive which " + "is necessary for Poisson regression." + ) + + self._n_samples, self.n_outputs_ = y.shape y, expanded_class_weight = self._validate_y_class_weight(y) @@ -337,18 +426,25 @@ def fit(self, X, y, sample_weight=None): else: sample_weight = expanded_class_weight - # Get bootstrap sample size - n_samples_bootstrap = _get_n_samples_bootstrap( - n_samples=X.shape[0], - max_samples=self.max_samples - ) + if not self.bootstrap and self.max_samples is not None: + raise ValueError( + "`max_sample` cannot be set if `bootstrap=False`. " + "Either switch to `bootstrap=True` or set " + "`max_sample=None`." + ) + elif self.bootstrap: + n_samples_bootstrap = _get_n_samples_bootstrap( + n_samples=X.shape[0], max_samples=self.max_samples + ) + else: + n_samples_bootstrap = None + + self._n_samples_bootstrap = n_samples_bootstrap - # Check parameters self._validate_estimator() if not self.bootstrap and self.oob_score: - raise ValueError("Out of bag estimation only available" - " if bootstrap=True") + raise ValueError("Out of bag estimation only available if bootstrap=True") random_state = check_random_state(self.random_state) @@ -359,22 +455,27 @@ def fit(self, X, y, sample_weight=None): n_more_estimators = self.n_estimators - len(self.estimators_) if n_more_estimators < 0: - raise ValueError('n_estimators=%d must be larger or equal to ' - 'len(estimators_)=%d when warm_start==True' - % (self.n_estimators, len(self.estimators_))) + raise ValueError( + "n_estimators=%d must be larger or equal to " + "len(estimators_)=%d when warm_start==True" + % (self.n_estimators, len(self.estimators_)) + ) elif n_more_estimators == 0: - warn("Warm-start fitting without increasing n_estimators does not " - "fit new trees.") + warn( + "Warm-start fitting without increasing n_estimators does not " + "fit new trees." + ) else: if self.warm_start and len(self.estimators_) > 0: # We draw from the random state to get the random state we # would have got if we hadn't used a warm_start. random_state.randint(MAX_INT, size=len(self.estimators_)) - trees = [self._make_estimator(append=False, - random_state=random_state) - for i in range(n_more_estimators)] + trees = [ + self._make_estimator(append=False, random_state=random_state) + for i in range(n_more_estimators) + ] # Parallel loop: we prefer the threading backend as the Cython code # for fitting the trees is internally releasing the Python GIL @@ -382,31 +483,54 @@ def fit(self, X, y, sample_weight=None): # that case. However, for joblib 0.12+ we respect any # parallel_backend contexts set at a higher level, # since correctness does not rely on using threads. - trees = Parallel(n_jobs=self.n_jobs, verbose=self.verbose, - **_joblib_parallel_args(prefer='threads'))( + trees = Parallel( + n_jobs=self.n_jobs, + verbose=self.verbose, + prefer="threads", + )( delayed(_parallel_build_trees)( - t, self, X, y, sample_weight, i, len(trees), - verbose=self.verbose, class_weight=self.class_weight, - n_samples_bootstrap=n_samples_bootstrap) - for i, t in enumerate(trees)) + t, + self.bootstrap, + X, + y, + sample_weight, + i, + len(trees), + verbose=self.verbose, + class_weight=self.class_weight, + n_samples_bootstrap=n_samples_bootstrap, + missing_values_in_feature_mask=missing_values_in_feature_mask, + ) + for i, t in enumerate(trees) + ) # Collect newly grown trees self.estimators_.extend(trees) - if self.oob_score: + if self.oob_score and ( + n_more_estimators > 0 or not hasattr(self, "oob_score_") + ): y_type = type_of_target(y) - if y_type in ("multiclass-multioutput", "unknown"): + if y_type == "unknown" or ( + is_classifier(self) and y_type == "multiclass-multioutput" + ): # FIXME: we could consider to support multiclass-multioutput if # we introduce or reuse a constructor parameter (e.g. # oob_score) allowing our user to pass a callable defining the # scoring strategy on OOB sample. raise ValueError( - f"The type of target cannot be used to compute OOB " + "The type of target cannot be used to compute OOB " f"estimates. Got {y_type} while only the following are " - f"supported: continuous, continuous-multioutput, binary, " - f"multiclass, multilabel-indicator." + "supported: continuous, continuous-multioutput, binary, " + "multiclass, multilabel-indicator." + ) + + if callable(self.oob_score): + self._set_oob_score_and_attributes( + X, y, scoring_function=self.oob_score ) - self._set_oob_score_and_attributes(X, y) + else: + self._set_oob_score_and_attributes(X, y) # Decapsulate classes_ attributes if hasattr(self, "classes_") and self.n_outputs_ == 1: @@ -416,7 +540,7 @@ def fit(self, X, y, sample_weight=None): return self @abstractmethod - def _set_oob_score_and_attributes(self, X, y): + def _set_oob_score_and_attributes(self, X, y, scoring_function=None): """Compute and set the OOB score and attributes. Parameters @@ -425,6 +549,10 @@ def _set_oob_score_and_attributes(self, X, y): The data matrix. y : ndarray of shape (n_samples, n_outputs) The target matrix. + scoring_function : callable, default=None + Scoring function for OOB score. Default depends on whether + this is a regression (R2 score) or classification problem + (accuracy score). """ def _compute_oob_predictions(self, X, y): @@ -442,9 +570,10 @@ def _compute_oob_predictions(self, X, y): oob_pred : ndarray of shape (n_samples, n_classes, n_outputs) or \ (n_samples, 1, n_outputs) The OOB predictions. - """ - X = self._validate_data(X, dtype=DTYPE, accept_sparse='csr', - reset=False) + """ + # Prediction requires X to be in CSR format + if issparse(X): + X = X.tocsr() n_samples = y.shape[0] n_outputs = self.n_outputs_ @@ -463,25 +592,29 @@ def _compute_oob_predictions(self, X, y): n_oob_pred = np.zeros((n_samples, n_outputs), dtype=np.int64) n_samples_bootstrap = _get_n_samples_bootstrap( - n_samples, self.max_samples, + n_samples, + self.max_samples, ) for estimator in self.estimators_: unsampled_indices = _generate_unsampled_indices( - estimator.random_state, n_samples, n_samples_bootstrap, + estimator.random_state, + n_samples, + n_samples_bootstrap, ) - y_pred = self._get_oob_predictions( - estimator, X[unsampled_indices, :] - ) + y_pred = self._get_oob_predictions(estimator, X[unsampled_indices, :]) oob_pred[unsampled_indices, ...] += y_pred n_oob_pred[unsampled_indices, :] += 1 for k in range(n_outputs): if (n_oob_pred == 0).any(): warn( - "Some inputs do not have OOB scores. This probably means " - "too few trees were used to compute any reliable OOB " - "estimates.", UserWarning + ( + "Some inputs do not have OOB scores. This probably means " + "too few trees were used to compute any reliable OOB " + "estimates." + ), + UserWarning, ) n_oob_pred[n_oob_pred == 0] = 1 oob_pred[..., k] /= n_oob_pred[..., [k]] @@ -496,8 +629,22 @@ def _validate_X_predict(self, X): """ Validate X whenever one tries to predict, apply, predict_proba.""" check_is_fitted(self) - - return self.estimators_[0]._validate_X_predict(X, check_input=True) + if self.estimators_[0]._support_missing_values(X): + ensure_all_finite = "allow-nan" + else: + ensure_all_finite = True + + X = validate_data( + self, + X, + dtype=DTYPE, + accept_sparse="csr", + reset=False, + ensure_all_finite=ensure_all_finite, + ) + if issparse(X) and (X.indices.dtype != np.intc or X.indptr.dtype != np.intc): + raise ValueError("No support for np.int64 index based sparse matrices") + return X @property def feature_importances_(self): @@ -522,27 +669,55 @@ def feature_importances_(self): """ check_is_fitted(self) - all_importances = Parallel(n_jobs=self.n_jobs, - **_joblib_parallel_args(prefer='threads'))( - delayed(getattr)(tree, 'feature_importances_') - for tree in self.estimators_ if tree.tree_.node_count > 1) + all_importances = Parallel(n_jobs=self.n_jobs, prefer="threads")( + delayed(getattr)(tree, "feature_importances_") + for tree in self.estimators_ + if tree.tree_.node_count > 1 + ) if not all_importances: return np.zeros(self.n_features_in_, dtype=np.float64) - all_importances = np.mean(all_importances, - axis=0, dtype=np.float64) + all_importances = np.mean(all_importances, axis=0, dtype=np.float64) return all_importances / np.sum(all_importances) - # TODO: Remove in 1.2 - # mypy error: Decorated property not supported - @deprecated( # type: ignore - "Attribute n_features_ was deprecated in version 1.0 and will be " - "removed in 1.2. Use 'n_features_in_' instead." - ) + def _get_estimators_indices(self): + # Get drawn indices along both sample and feature axes + for tree in self.estimators_: + if not self.bootstrap: + yield np.arange(self._n_samples, dtype=np.int32) + else: + # tree.random_state is actually an immutable integer seed rather + # than a mutable RandomState instance, so it's safe to use it + # repeatedly when calling this property. + seed = tree.random_state + # Operations accessing random_state must be performed identically + # to those in `_parallel_build_trees()` + yield _generate_sample_indices( + seed, self._n_samples, self._n_samples_bootstrap + ) + @property - def n_features_(self): - return self.n_features_in_ + def estimators_samples_(self): + """The subset of drawn samples for each base estimator. + + Returns a dynamically generated list of indices identifying + the samples used for fitting each member of the ensemble, i.e., + the in-bag samples. + + Note: the list is re-created at each call to the property in order + to reduce the object memory footprint by not storing the sampling + data. Thus fetching the property may be slower than expected. + """ + return [sample_indices for sample_indices in self._get_estimators_indices()] + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + # Only the criterion is required to determine if the tree supports + # missing values + estimator = type(self.estimator)(criterion=self.criterion) + tags.input_tags.allow_nan = get_tags(estimator).input_tags.allow_nan + return tags def _accumulate_prediction(predict, X, out, lock): @@ -570,20 +745,23 @@ class ForestClassifier(ClassifierMixin, BaseForest, metaclass=ABCMeta): """ @abstractmethod - def __init__(self, - base_estimator, - n_estimators=100, *, - estimator_params=tuple(), - bootstrap=False, - oob_score=False, - n_jobs=None, - random_state=None, - verbose=0, - warm_start=False, - class_weight=None, - max_samples=None): + def __init__( + self, + estimator, + n_estimators=100, + *, + estimator_params=tuple(), + bootstrap=False, + oob_score=False, + n_jobs=None, + random_state=None, + verbose=0, + warm_start=False, + class_weight=None, + max_samples=None, + ): super().__init__( - base_estimator, + estimator=estimator, n_estimators=n_estimators, estimator_params=estimator_params, bootstrap=bootstrap, @@ -593,7 +771,8 @@ def __init__(self, verbose=verbose, warm_start=warm_start, class_weight=class_weight, - max_samples=max_samples) + max_samples=max_samples, + ) @staticmethod def _get_oob_predictions(tree, X): @@ -612,7 +791,7 @@ def _get_oob_predictions(tree, X): The OOB associated predictions. """ y_pred = tree.predict_proba(X, check_input=False) - y_pred = np.array(y_pred, copy=False) + y_pred = np.asarray(y_pred) if y_pred.ndim == 2: # binary and multiclass y_pred = y_pred[..., np.newaxis] @@ -623,7 +802,7 @@ def _get_oob_predictions(tree, X): y_pred = np.rollaxis(y_pred, axis=0, start=3) return y_pred - def _set_oob_score_and_attributes(self, X, y): + def _set_oob_score_and_attributes(self, X, y, scoring_function=None): """Compute and set the OOB score and attributes. Parameters @@ -632,14 +811,18 @@ def _set_oob_score_and_attributes(self, X, y): The data matrix. y : ndarray of shape (n_samples, n_outputs) The target matrix. + scoring_function : callable, default=None + Scoring function for OOB score. Defaults to `accuracy_score`. """ self.oob_decision_function_ = super()._compute_oob_predictions(X, y) if self.oob_decision_function_.shape[-1] == 1: # drop the n_outputs axis if there is a single output - self.oob_decision_function_ = self.oob_decision_function_.squeeze( - axis=-1 - ) - self.oob_score_ = accuracy_score( + self.oob_decision_function_ = self.oob_decision_function_.squeeze(axis=-1) + + if scoring_function is None: + scoring_function = accuracy_score + + self.oob_score_ = scoring_function( y, np.argmax(self.oob_decision_function_, axis=1) ) @@ -657,40 +840,42 @@ def _validate_y_class_weight(self, y): y_store_unique_indices = np.zeros(y.shape, dtype=int) for k in range(self.n_outputs_): - classes_k, y_store_unique_indices[:, k] = \ - np.unique(y[:, k], return_inverse=True) + classes_k, y_store_unique_indices[:, k] = np.unique( + y[:, k], return_inverse=True + ) self.classes_.append(classes_k) self.n_classes_.append(classes_k.shape[0]) y = y_store_unique_indices if self.class_weight is not None: - valid_presets = ('balanced', 'balanced_subsample') + valid_presets = ("balanced", "balanced_subsample") if isinstance(self.class_weight, str): if self.class_weight not in valid_presets: - raise ValueError('Valid presets for class_weight include ' - '"balanced" and "balanced_subsample".' - 'Given "%s".' - % self.class_weight) + raise ValueError( + "Valid presets for class_weight include " + '"balanced" and "balanced_subsample".' + 'Given "%s".' % self.class_weight + ) if self.warm_start: - warn('class_weight presets "balanced" or ' - '"balanced_subsample" are ' - 'not recommended for warm_start if the fitted data ' - 'differs from the full dataset. In order to use ' - '"balanced" weights, use compute_class_weight ' - '("balanced", classes, y). In place of y you can use ' - 'a large enough sample of the full training set ' - 'target to properly estimate the class frequency ' - 'distributions. Pass the resulting weights as the ' - 'class_weight parameter.') - - if (self.class_weight != 'balanced_subsample' or - not self.bootstrap): + warn( + 'class_weight presets "balanced" or ' + '"balanced_subsample" are ' + "not recommended for warm_start if the fitted data " + "differs from the full dataset. In order to use " + '"balanced" weights, use compute_class_weight ' + '("balanced", classes, y). In place of y you can use ' + "a large enough sample of the full training set " + "target to properly estimate the class frequency " + "distributions. Pass the resulting weights as the " + "class_weight parameter." + ) + + if self.class_weight != "balanced_subsample" or not self.bootstrap: if self.class_weight == "balanced_subsample": class_weight = "balanced" else: class_weight = self.class_weight - expanded_class_weight = compute_sample_weight(class_weight, - y_original) + expanded_class_weight = compute_sample_weight(class_weight, y_original) return y, expanded_class_weight @@ -724,13 +909,12 @@ def predict(self, X): n_samples = proba[0].shape[0] # all dtypes should be the same, so just take the first class_type = self.classes_[0].dtype - predictions = np.empty((n_samples, self.n_outputs_), - dtype=class_type) + predictions = np.empty((n_samples, self.n_outputs_), dtype=class_type) for k in range(self.n_outputs_): - predictions[:, k] = self.classes_[k].take(np.argmax(proba[k], - axis=1), - axis=0) + predictions[:, k] = self.classes_[k].take( + np.argmax(proba[k], axis=1), axis=0 + ) return predictions @@ -764,14 +948,15 @@ def predict_proba(self, X): n_jobs, _, _ = _partition_estimators(self.n_estimators, self.n_jobs) # avoid storing the output of every estimator by summing them here - all_proba = [np.zeros((X.shape[0], j), dtype=np.float64) - for j in np.atleast_1d(self.n_classes_)] + all_proba = [ + np.zeros((X.shape[0], j), dtype=np.float64) + for j in np.atleast_1d(self.n_classes_) + ] lock = threading.Lock() - Parallel(n_jobs=n_jobs, verbose=self.verbose, - **_joblib_parallel_args(require="sharedmem"))( - delayed(_accumulate_prediction)(e.predict_proba, X, all_proba, - lock) - for e in self.estimators_) + Parallel(n_jobs=n_jobs, verbose=self.verbose, require="sharedmem")( + delayed(_accumulate_prediction)(e.predict_proba, X, all_proba, lock) + for e in self.estimators_ + ) for proba in all_proba: proba /= len(self.estimators_) @@ -813,6 +998,12 @@ def predict_log_proba(self, X): return proba + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.classifier_tags.multi_label = True + tags.input_tags.sparse = True + return tags + class ForestRegressor(RegressorMixin, BaseForest, metaclass=ABCMeta): """ @@ -823,19 +1014,22 @@ class ForestRegressor(RegressorMixin, BaseForest, metaclass=ABCMeta): """ @abstractmethod - def __init__(self, - base_estimator, - n_estimators=100, *, - estimator_params=tuple(), - bootstrap=False, - oob_score=False, - n_jobs=None, - random_state=None, - verbose=0, - warm_start=False, - max_samples=None): + def __init__( + self, + estimator, + n_estimators=100, + *, + estimator_params=tuple(), + bootstrap=False, + oob_score=False, + n_jobs=None, + random_state=None, + verbose=0, + warm_start=False, + max_samples=None, + ): super().__init__( - base_estimator, + estimator, n_estimators=n_estimators, estimator_params=estimator_params, bootstrap=bootstrap, @@ -844,7 +1038,8 @@ def __init__(self, random_state=random_state, verbose=verbose, warm_start=warm_start, - max_samples=max_samples) + max_samples=max_samples, + ) def predict(self, X): """ @@ -880,10 +1075,10 @@ def predict(self, X): # Parallel loop lock = threading.Lock() - Parallel(n_jobs=n_jobs, verbose=self.verbose, - **_joblib_parallel_args(require="sharedmem"))( + Parallel(n_jobs=n_jobs, verbose=self.verbose, require="sharedmem")( delayed(_accumulate_prediction)(e.predict, X, [y_hat], lock) - for e in self.estimators_) + for e in self.estimators_ + ) y_hat /= len(self.estimators_) @@ -914,7 +1109,7 @@ def _get_oob_predictions(tree, X): y_pred = y_pred[:, np.newaxis, :] return y_pred - def _set_oob_score_and_attributes(self, X, y): + def _set_oob_score_and_attributes(self, X, y, scoring_function=None): """Compute and set the OOB score and attributes. Parameters @@ -923,24 +1118,28 @@ def _set_oob_score_and_attributes(self, X, y): The data matrix. y : ndarray of shape (n_samples, n_outputs) The target matrix. + scoring_function : callable, default=None + Scoring function for OOB score. Defaults to `r2_score`. """ - self.oob_prediction_ = super()._compute_oob_predictions(X, y).squeeze( - axis=1 - ) + self.oob_prediction_ = super()._compute_oob_predictions(X, y).squeeze(axis=1) if self.oob_prediction_.shape[-1] == 1: # drop the n_outputs axis if there is a single output self.oob_prediction_ = self.oob_prediction_.squeeze(axis=-1) - self.oob_score_ = r2_score(y, self.oob_prediction_) + + if scoring_function is None: + scoring_function = r2_score + + self.oob_score_ = scoring_function(y, self.oob_prediction_) def _compute_partial_dependence_recursion(self, grid, target_features): """Fast partial dependence computation. Parameters ---------- - grid : ndarray of shape (n_samples, n_target_features) + grid : ndarray of shape (n_samples, n_target_features), dtype=DTYPE The grid points on which the partial dependence should be evaluated. - target_features : ndarray of shape (n_target_features) + target_features : ndarray of shape (n_target_features), dtype=np.intp The set of target features for which the partial dependence should be evaluated. @@ -949,20 +1148,28 @@ def _compute_partial_dependence_recursion(self, grid, target_features): averaged_predictions : ndarray of shape (n_samples,) The value of the partial dependence function on each grid point. """ - grid = np.asarray(grid, dtype=DTYPE, order='C') - averaged_predictions = np.zeros(shape=grid.shape[0], - dtype=np.float64, order='C') + grid = np.asarray(grid, dtype=DTYPE, order="C") + target_features = np.asarray(target_features, dtype=np.intp, order="C") + averaged_predictions = np.zeros( + shape=grid.shape[0], dtype=np.float64, order="C" + ) for tree in self.estimators_: # Note: we don't sum in parallel because the GIL isn't released in # the fast method. tree.tree_.compute_partial_dependence( - grid, target_features, averaged_predictions) + grid, target_features, averaged_predictions + ) # Average over the forest averaged_predictions /= len(self.estimators_) return averaged_predictions + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.sparse = True + return tags + class RandomForestClassifier(ForestClassifier): """ @@ -971,10 +1178,22 @@ class RandomForestClassifier(ForestClassifier): A random forest is a meta estimator that fits a number of decision tree classifiers on various sub-samples of the dataset and uses averaging to improve the predictive accuracy and control over-fitting. + Trees in the forest use the best split strategy, i.e. equivalent to passing + `splitter="best"` to the underlying :class:`~sklearn.tree.DecisionTreeClassifier`. The sub-sample size is controlled with the `max_samples` parameter if `bootstrap=True` (default), otherwise the whole dataset is used to build each tree. + For a comparison between tree-based ensemble models see the example + :ref:`sphx_glr_auto_examples_ensemble_plot_forest_hist_grad_boosting_comparison.py`. + + This estimator has native support for missing values (NaNs). During training, + the tree grower learns at each split point whether samples with missing values + should go to the left or right child, based on the potential gain. When predicting, + samples with missing values are assigned to the left or right child consequently. + If no missing values were encountered for a given feature during training, then + samples with missing values are mapped to whichever child has the most samples. + Read more in the :ref:`User Guide `. Parameters @@ -986,10 +1205,11 @@ class RandomForestClassifier(ForestClassifier): The default value of ``n_estimators`` changed from 10 to 100 in 0.22. - criterion : {"gini", "entropy"}, default="gini" + criterion : {"gini", "entropy", "log_loss"}, default="gini" The function to measure the quality of a split. Supported criteria are - "gini" for the Gini impurity and "entropy" for the information gain. - Note: this parameter is tree-specific. + "gini" for the Gini impurity and "log_loss" and "entropy" both for the + Shannon information gain, see :ref:`tree_mathematical_formulation`. + Note: This parameter is tree-specific. max_depth : int, default=None The maximum depth of the tree. If None, then nodes are expanded until @@ -1027,18 +1247,20 @@ class RandomForestClassifier(ForestClassifier): the input samples) required to be at a leaf node. Samples have equal weight when sample_weight is not provided. - max_features : {"auto", "sqrt", "log2"}, int or float, default="auto" + max_features : {"sqrt", "log2", None}, int or float, default="sqrt" The number of features to consider when looking for the best split: - If int, then consider `max_features` features at each split. - If float, then `max_features` is a fraction and - `round(max_features * n_features)` features are considered at each + `max(1, int(max_features * n_features_in_))` features are considered at each split. - - If "auto", then `max_features=sqrt(n_features)`. - - If "sqrt", then `max_features=sqrt(n_features)` (same as "auto"). + - If "sqrt", then `max_features=sqrt(n_features)`. - If "log2", then `max_features=log2(n_features)`. - If None, then `max_features=n_features`. + .. versionchanged:: 1.1 + The default of `max_features` changed from `"auto"` to `"sqrt"`. + Note: the search for a split does not stop until at least one valid partition of the node samples is found, even if it requires to effectively inspect more than ``max_features`` features. @@ -1066,24 +1288,15 @@ class RandomForestClassifier(ForestClassifier): .. versionadded:: 0.19 - min_impurity_split : float, default=None - Threshold for early stopping in tree growth. A node will split - if its impurity is above the threshold, otherwise it is a leaf. - - .. deprecated:: 0.19 - ``min_impurity_split`` has been deprecated in favor of - ``min_impurity_decrease`` in 0.19. The default value of - ``min_impurity_split`` has changed from 1e-7 to 0 in 0.23 and it - will be removed in 1.0 (renaming of 0.25). - Use ``min_impurity_decrease`` instead. - bootstrap : bool, default=True Whether bootstrap samples are used when building trees. If False, the whole dataset is used to build each tree. - oob_score : bool, default=False + oob_score : bool or callable, default=False Whether to use out-of-bag samples to estimate the generalization score. - Only available if bootstrap=True. + By default, :func:`~sklearn.metrics.accuracy_score` is used. + Provide a callable with signature `metric(y_true, y_pred)` to use a + custom metric. Only available if `bootstrap=True`. n_jobs : int, default=None The number of jobs to run in parallel. :meth:`fit`, :meth:`predict`, @@ -1105,7 +1318,8 @@ class RandomForestClassifier(ForestClassifier): warm_start : bool, default=False When set to ``True``, reuse the solution of the previous call to fit and add more estimators to the ensemble, otherwise, just fit a whole - new forest. See :term:`the Glossary `. + new forest. See :term:`Glossary ` and + :ref:`tree_ensemble_warm_start` for details. class_weight : {"balanced", "balanced_subsample"}, dict or list of dicts, \ default=None @@ -1137,7 +1351,9 @@ class RandomForestClassifier(ForestClassifier): Complexity parameter used for Minimal Cost-Complexity Pruning. The subtree with the largest cost complexity that is smaller than ``ccp_alpha`` will be chosen. By default, no pruning is performed. See - :ref:`minimal_cost_complexity_pruning` for details. + :ref:`minimal_cost_complexity_pruning` for details. See + :ref:`sphx_glr_auto_examples_tree_plot_cost_complexity_pruning.py` + for an example of such pruning. .. versionadded:: 0.22 @@ -1147,17 +1363,39 @@ class RandomForestClassifier(ForestClassifier): - If None (default), then draw `X.shape[0]` samples. - If int, then draw `max_samples` samples. - - If float, then draw `max_samples * X.shape[0]` samples. Thus, - `max_samples` should be in the interval `(0, 1)`. + - If float, then draw `max(round(n_samples * max_samples), 1)` samples. Thus, + `max_samples` should be in the interval `(0.0, 1.0]`. .. versionadded:: 0.22 + monotonic_cst : array-like of int of shape (n_features), default=None + Indicates the monotonicity constraint to enforce on each feature. + - 1: monotonic increase + - 0: no constraint + - -1: monotonic decrease + + If monotonic_cst is None, no constraints are applied. + + Monotonicity constraints are not supported for: + - multiclass classifications (i.e. when `n_classes > 2`), + - multioutput classifications (i.e. when `n_outputs_ > 1`), + - classifications trained on data with missing values. + + The constraints hold over the probability of the positive class. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 1.4 + Attributes ---------- - base_estimator_ : DecisionTreeClassifier + estimator_ : :class:`~sklearn.tree.DecisionTreeClassifier` The child estimator template used to create the collection of fitted sub-estimators. + .. versionadded:: 1.2 + `base_estimator_` was renamed to `estimator_`. + estimators_ : list of DecisionTreeClassifier The collection of fitted sub-estimators. @@ -1169,12 +1407,16 @@ class labels (multi-output problem). The number of classes (single output problem), or a list containing the number of classes for each output (multi-output problem). - n_features_ : int - The number of features when ``fit`` is performed. + n_features_in_ : int + Number of features seen during :term:`fit`. - .. deprecated:: 1.0 - Attribute `n_features_` was deprecated in version 1.0 and will be - removed in 1.2. Use `n_features_in_` instead. + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 n_outputs_ : int The number of outputs when ``fit`` is performed. @@ -1202,9 +1444,20 @@ class labels (multi-output problem). `oob_decision_function_` might contain NaN. This attribute exists only when ``oob_score`` is True. + estimators_samples_ : list of arrays + The subset of drawn samples (i.e., the in-bag samples) for each base + estimator. Each subset is defined by an array of the indices selected. + + .. versionadded:: 1.4 + See Also -------- - DecisionTreeClassifier, ExtraTreesClassifier + sklearn.tree.DecisionTreeClassifier : A decision tree classifier. + sklearn.ensemble.ExtraTreesClassifier : Ensemble of extremely randomized + tree classifiers. + sklearn.ensemble.HistGradientBoostingClassifier : A Histogram-based Gradient + Boosting Classification Tree, very fast for big datasets (n_samples >= + 10_000). Notes ----- @@ -1238,35 +1491,58 @@ class labels (multi-output problem). >>> print(clf.predict([[0, 0, 0, 0]])) [1] """ - @_deprecate_positional_args - def __init__(self, - n_estimators=100, *, - criterion="gini", - max_depth=None, - min_samples_split=2, - min_samples_leaf=1, - min_weight_fraction_leaf=0., - max_features="auto", - max_leaf_nodes=None, - min_impurity_decrease=0., - min_impurity_split=None, - bootstrap=True, - oob_score=False, - n_jobs=None, - random_state=None, - verbose=0, - warm_start=False, - class_weight=None, - ccp_alpha=0.0, - max_samples=None): + + _parameter_constraints: dict = { + **ForestClassifier._parameter_constraints, + **DecisionTreeClassifier._parameter_constraints, + "class_weight": [ + StrOptions({"balanced_subsample", "balanced"}), + dict, + list, + None, + ], + } + _parameter_constraints.pop("splitter") + + def __init__( + self, + n_estimators=100, + *, + criterion="gini", + max_depth=None, + min_samples_split=2, + min_samples_leaf=1, + min_weight_fraction_leaf=0.0, + max_features="sqrt", + max_leaf_nodes=None, + min_impurity_decrease=0.0, + bootstrap=True, + oob_score=False, + n_jobs=None, + random_state=None, + verbose=0, + warm_start=False, + class_weight=None, + ccp_alpha=0.0, + max_samples=None, + monotonic_cst=None, + ): super().__init__( - base_estimator=DecisionTreeClassifier(), + estimator=DecisionTreeClassifier(), n_estimators=n_estimators, - estimator_params=("criterion", "max_depth", "min_samples_split", - "min_samples_leaf", "min_weight_fraction_leaf", - "max_features", "max_leaf_nodes", - "min_impurity_decrease", "min_impurity_split", - "random_state", "ccp_alpha"), + estimator_params=( + "criterion", + "max_depth", + "min_samples_split", + "min_samples_leaf", + "min_weight_fraction_leaf", + "max_features", + "max_leaf_nodes", + "min_impurity_decrease", + "random_state", + "ccp_alpha", + "monotonic_cst", + ), bootstrap=bootstrap, oob_score=oob_score, n_jobs=n_jobs, @@ -1274,7 +1550,8 @@ def __init__(self, verbose=verbose, warm_start=warm_start, class_weight=class_weight, - max_samples=max_samples) + max_samples=max_samples, + ) self.criterion = criterion self.max_depth = max_depth @@ -1284,7 +1561,7 @@ def __init__(self, self.max_features = max_features self.max_leaf_nodes = max_leaf_nodes self.min_impurity_decrease = min_impurity_decrease - self.min_impurity_split = min_impurity_split + self.monotonic_cst = monotonic_cst self.ccp_alpha = ccp_alpha @@ -1292,13 +1569,25 @@ class RandomForestRegressor(ForestRegressor): """ A random forest regressor. - A random forest is a meta estimator that fits a number of classifying - decision trees on various sub-samples of the dataset and uses averaging - to improve the predictive accuracy and control over-fitting. + A random forest is a meta estimator that fits a number of decision tree + regressors on various sub-samples of the dataset and uses averaging to + improve the predictive accuracy and control over-fitting. + Trees in the forest use the best split strategy, i.e. equivalent to passing + `splitter="best"` to the underlying :class:`~sklearn.tree.DecisionTreeRegressor`. The sub-sample size is controlled with the `max_samples` parameter if `bootstrap=True` (default), otherwise the whole dataset is used to build each tree. + This estimator has native support for missing values (NaNs). During training, + the tree grower learns at each split point whether samples with missing values + should go to the left or right child, based on the potential gain. When predicting, + samples with missing values are assigned to the left or right child consequently. + If no missing values were encountered for a given feature during training, then + samples with missing values are mapped to whichever child has the most samples. + + For a comparison between tree-based ensemble models see the example + :ref:`sphx_glr_auto_examples_ensemble_plot_forest_hist_grad_boosting_comparison.py`. + Read more in the :ref:`User Guide `. Parameters @@ -1310,15 +1599,25 @@ class RandomForestRegressor(ForestRegressor): The default value of ``n_estimators`` changed from 10 to 100 in 0.22. - criterion : {"mse", "mae"}, default="mse" + criterion : {"squared_error", "absolute_error", "friedman_mse", "poisson"}, \ + default="squared_error" The function to measure the quality of a split. Supported criteria - are "mse" for the mean squared error, which is equal to variance - reduction as feature selection criterion, and "mae" for the mean - absolute error. + are "squared_error" for the mean squared error, which is equal to + variance reduction as feature selection criterion and minimizes the L2 + loss using the mean of each terminal node, "friedman_mse", which uses + mean squared error with Friedman's improvement score for potential + splits, "absolute_error" for the mean absolute error, which minimizes + the L1 loss using the median of each terminal node, and "poisson" which + uses reduction in Poisson deviance to find splits. + Training using "absolute_error" is significantly slower + than when using "squared_error". .. versionadded:: 0.18 Mean Absolute Error (MAE) criterion. + .. versionadded:: 1.0 + Poisson criterion. + max_depth : int, default=None The maximum depth of the tree. If None, then nodes are expanded until all leaves are pure or until all leaves contain less than @@ -1355,17 +1654,23 @@ class RandomForestRegressor(ForestRegressor): the input samples) required to be at a leaf node. Samples have equal weight when sample_weight is not provided. - max_features : {"auto", "sqrt", "log2"}, int or float, default="auto" + max_features : {"sqrt", "log2", None}, int or float, default=1.0 The number of features to consider when looking for the best split: - If int, then consider `max_features` features at each split. - If float, then `max_features` is a fraction and - `round(max_features * n_features)` features are considered at each + `max(1, int(max_features * n_features_in_))` features are considered at each split. - - If "auto", then `max_features=n_features`. - If "sqrt", then `max_features=sqrt(n_features)`. - If "log2", then `max_features=log2(n_features)`. - - If None, then `max_features=n_features`. + - If None or 1.0, then `max_features=n_features`. + + .. note:: + The default of 1.0 is equivalent to bagged trees and more + randomness can be achieved by setting smaller values, e.g. 0.3. + + .. versionchanged:: 1.1 + The default of `max_features` changed from `"auto"` to 1.0. Note: the search for a split does not stop until at least one valid partition of the node samples is found, even if it requires to @@ -1394,24 +1699,15 @@ class RandomForestRegressor(ForestRegressor): .. versionadded:: 0.19 - min_impurity_split : float, default=None - Threshold for early stopping in tree growth. A node will split - if its impurity is above the threshold, otherwise it is a leaf. - - .. deprecated:: 0.19 - ``min_impurity_split`` has been deprecated in favor of - ``min_impurity_decrease`` in 0.19. The default value of - ``min_impurity_split`` has changed from 1e-7 to 0 in 0.23 and it - will be removed in 1.0 (renaming of 0.25). - Use ``min_impurity_decrease`` instead. - bootstrap : bool, default=True Whether bootstrap samples are used when building trees. If False, the whole dataset is used to build each tree. - oob_score : bool, default=False + oob_score : bool or callable, default=False Whether to use out-of-bag samples to estimate the generalization score. - Only available if bootstrap=True. + By default, :func:`~sklearn.metrics.r2_score` is used. + Provide a callable with signature `metric(y_true, y_pred)` to use a + custom metric. Only available if `bootstrap=True`. n_jobs : int, default=None The number of jobs to run in parallel. :meth:`fit`, :meth:`predict`, @@ -1433,13 +1729,16 @@ class RandomForestRegressor(ForestRegressor): warm_start : bool, default=False When set to ``True``, reuse the solution of the previous call to fit and add more estimators to the ensemble, otherwise, just fit a whole - new forest. See :term:`the Glossary `. + new forest. See :term:`Glossary ` and + :ref:`tree_ensemble_warm_start` for details. ccp_alpha : non-negative float, default=0.0 Complexity parameter used for Minimal Cost-Complexity Pruning. The subtree with the largest cost complexity that is smaller than ``ccp_alpha`` will be chosen. By default, no pruning is performed. See - :ref:`minimal_cost_complexity_pruning` for details. + :ref:`minimal_cost_complexity_pruning` for details. See + :ref:`sphx_glr_auto_examples_tree_plot_cost_complexity_pruning.py` + for an example of such pruning. .. versionadded:: 0.22 @@ -1449,17 +1748,36 @@ class RandomForestRegressor(ForestRegressor): - If None (default), then draw `X.shape[0]` samples. - If int, then draw `max_samples` samples. - - If float, then draw `max_samples * X.shape[0]` samples. Thus, - `max_samples` should be in the interval `(0, 1)`. + - If float, then draw `max(round(n_samples * max_samples), 1)` samples. Thus, + `max_samples` should be in the interval `(0.0, 1.0]`. .. versionadded:: 0.22 + monotonic_cst : array-like of int of shape (n_features), default=None + Indicates the monotonicity constraint to enforce on each feature. + - 1: monotonically increasing + - 0: no constraint + - -1: monotonically decreasing + + If monotonic_cst is None, no constraints are applied. + + Monotonicity constraints are not supported for: + - multioutput regressions (i.e. when `n_outputs_ > 1`), + - regressions trained on data with missing values. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 1.4 + Attributes ---------- - base_estimator_ : DecisionTreeRegressor + estimator_ : :class:`~sklearn.tree.DecisionTreeRegressor` The child estimator template used to create the collection of fitted sub-estimators. + .. versionadded:: 1.2 + `base_estimator_` was renamed to `estimator_`. + estimators_ : list of DecisionTreeRegressor The collection of fitted sub-estimators. @@ -1474,12 +1792,16 @@ class RandomForestRegressor(ForestRegressor): high cardinality features (many unique values). See :func:`sklearn.inspection.permutation_importance` as an alternative. - n_features_ : int - The number of features when ``fit`` is performed. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 - .. deprecated:: 1.0 - Attribute `n_features_` was deprecated in version 1.0 and will be - removed in 1.2. Use `n_features_in_` instead. + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 n_outputs_ : int The number of outputs when ``fit`` is performed. @@ -1492,9 +1814,20 @@ class RandomForestRegressor(ForestRegressor): Prediction computed with out-of-bag estimate on the training set. This attribute exists only when ``oob_score`` is True. + estimators_samples_ : list of arrays + The subset of drawn samples (i.e., the in-bag samples) for each base + estimator. Each subset is defined by an array of the indices selected. + + .. versionadded:: 1.4 + See Also -------- - DecisionTreeRegressor, ExtraTreesRegressor + sklearn.tree.DecisionTreeRegressor : A decision tree regressor. + sklearn.ensemble.ExtraTreesRegressor : Ensemble of extremely randomized + tree regressors. + sklearn.ensemble.HistGradientBoostingRegressor : A Histogram-based Gradient + Boosting Regression Tree, very fast for big datasets (n_samples >= + 10_000). Notes ----- @@ -1511,7 +1844,7 @@ class RandomForestRegressor(ForestRegressor): search of the best split. To obtain a deterministic behaviour during fitting, ``random_state`` has to be fixed. - The default value ``max_features="auto"`` uses ``n_features`` + The default value ``max_features=1.0`` uses ``n_features`` rather than ``n_features / 3``. The latter was originally suggested in [1], whereas the former was more recently justified empirically in [2]. @@ -1534,41 +1867,59 @@ class RandomForestRegressor(ForestRegressor): >>> print(regr.predict([[0, 0, 0, 0]])) [-8.32987858] """ - @_deprecate_positional_args - def __init__(self, - n_estimators=100, *, - criterion="mse", - max_depth=None, - min_samples_split=2, - min_samples_leaf=1, - min_weight_fraction_leaf=0., - max_features="auto", - max_leaf_nodes=None, - min_impurity_decrease=0., - min_impurity_split=None, - bootstrap=True, - oob_score=False, - n_jobs=None, - random_state=None, - verbose=0, - warm_start=False, - ccp_alpha=0.0, - max_samples=None): + + _parameter_constraints: dict = { + **ForestRegressor._parameter_constraints, + **DecisionTreeRegressor._parameter_constraints, + } + _parameter_constraints.pop("splitter") + + def __init__( + self, + n_estimators=100, + *, + criterion="squared_error", + max_depth=None, + min_samples_split=2, + min_samples_leaf=1, + min_weight_fraction_leaf=0.0, + max_features=1.0, + max_leaf_nodes=None, + min_impurity_decrease=0.0, + bootstrap=True, + oob_score=False, + n_jobs=None, + random_state=None, + verbose=0, + warm_start=False, + ccp_alpha=0.0, + max_samples=None, + monotonic_cst=None, + ): super().__init__( - base_estimator=DecisionTreeRegressor(), + estimator=DecisionTreeRegressor(), n_estimators=n_estimators, - estimator_params=("criterion", "max_depth", "min_samples_split", - "min_samples_leaf", "min_weight_fraction_leaf", - "max_features", "max_leaf_nodes", - "min_impurity_decrease", "min_impurity_split", - "random_state", "ccp_alpha"), + estimator_params=( + "criterion", + "max_depth", + "min_samples_split", + "min_samples_leaf", + "min_weight_fraction_leaf", + "max_features", + "max_leaf_nodes", + "min_impurity_decrease", + "random_state", + "ccp_alpha", + "monotonic_cst", + ), bootstrap=bootstrap, oob_score=oob_score, n_jobs=n_jobs, random_state=random_state, verbose=verbose, warm_start=warm_start, - max_samples=max_samples) + max_samples=max_samples, + ) self.criterion = criterion self.max_depth = max_depth @@ -1578,8 +1929,8 @@ def __init__(self, self.max_features = max_features self.max_leaf_nodes = max_leaf_nodes self.min_impurity_decrease = min_impurity_decrease - self.min_impurity_split = min_impurity_split self.ccp_alpha = ccp_alpha + self.monotonic_cst = monotonic_cst class ExtraTreesClassifier(ForestClassifier): @@ -1591,6 +1942,14 @@ class ExtraTreesClassifier(ForestClassifier): of the dataset and uses averaging to improve the predictive accuracy and control over-fitting. + This estimator has native support for missing values (NaNs) for + random splits. During training, a random threshold will be chosen + to split the non-missing values on. Then the non-missing values will be sent + to the left and right child based on the randomly selected threshold, while + the missing values will also be randomly sent to the left or right child. + This is repeated for every feature considered at each split. The best split + among these is chosen. + Read more in the :ref:`User Guide `. Parameters @@ -1602,9 +1961,11 @@ class ExtraTreesClassifier(ForestClassifier): The default value of ``n_estimators`` changed from 10 to 100 in 0.22. - criterion : {"gini", "entropy"}, default="gini" + criterion : {"gini", "entropy", "log_loss"}, default="gini" The function to measure the quality of a split. Supported criteria are - "gini" for the Gini impurity and "entropy" for the information gain. + "gini" for the Gini impurity and "log_loss" and "entropy" both for the + Shannon information gain, see :ref:`tree_mathematical_formulation`. + Note: This parameter is tree-specific. max_depth : int, default=None The maximum depth of the tree. If None, then nodes are expanded until @@ -1642,18 +2003,20 @@ class ExtraTreesClassifier(ForestClassifier): the input samples) required to be at a leaf node. Samples have equal weight when sample_weight is not provided. - max_features : {"auto", "sqrt", "log2"}, int or float, default="auto" + max_features : {"sqrt", "log2", None}, int or float, default="sqrt" The number of features to consider when looking for the best split: - If int, then consider `max_features` features at each split. - If float, then `max_features` is a fraction and - `round(max_features * n_features)` features are considered at each + `max(1, int(max_features * n_features_in_))` features are considered at each split. - - If "auto", then `max_features=sqrt(n_features)`. - If "sqrt", then `max_features=sqrt(n_features)`. - If "log2", then `max_features=log2(n_features)`. - If None, then `max_features=n_features`. + .. versionchanged:: 1.1 + The default of `max_features` changed from `"auto"` to `"sqrt"`. + Note: the search for a split does not stop until at least one valid partition of the node samples is found, even if it requires to effectively inspect more than ``max_features`` features. @@ -1681,24 +2044,15 @@ class ExtraTreesClassifier(ForestClassifier): .. versionadded:: 0.19 - min_impurity_split : float, default=None - Threshold for early stopping in tree growth. A node will split - if its impurity is above the threshold, otherwise it is a leaf. - - .. deprecated:: 0.19 - ``min_impurity_split`` has been deprecated in favor of - ``min_impurity_decrease`` in 0.19. The default value of - ``min_impurity_split`` has changed from 1e-7 to 0 in 0.23 and it - will be removed in 1.0 (renaming of 0.25). - Use ``min_impurity_decrease`` instead. - bootstrap : bool, default=False Whether bootstrap samples are used when building trees. If False, the whole dataset is used to build each tree. - oob_score : bool, default=False + oob_score : bool or callable, default=False Whether to use out-of-bag samples to estimate the generalization score. - Only available if bootstrap=True. + By default, :func:`~sklearn.metrics.accuracy_score` is used. + Provide a callable with signature `metric(y_true, y_pred)` to use a + custom metric. Only available if `bootstrap=True`. n_jobs : int, default=None The number of jobs to run in parallel. :meth:`fit`, :meth:`predict`, @@ -1724,7 +2078,8 @@ class ExtraTreesClassifier(ForestClassifier): warm_start : bool, default=False When set to ``True``, reuse the solution of the previous call to fit and add more estimators to the ensemble, otherwise, just fit a whole - new forest. See :term:`the Glossary `. + new forest. See :term:`Glossary ` and + :ref:`tree_ensemble_warm_start` for details. class_weight : {"balanced", "balanced_subsample"}, dict or list of dicts, \ default=None @@ -1756,7 +2111,9 @@ class ExtraTreesClassifier(ForestClassifier): Complexity parameter used for Minimal Cost-Complexity Pruning. The subtree with the largest cost complexity that is smaller than ``ccp_alpha`` will be chosen. By default, no pruning is performed. See - :ref:`minimal_cost_complexity_pruning` for details. + :ref:`minimal_cost_complexity_pruning` for details. See + :ref:`sphx_glr_auto_examples_tree_plot_cost_complexity_pruning.py` + for an example of such pruning. .. versionadded:: 0.22 @@ -1767,16 +2124,38 @@ class ExtraTreesClassifier(ForestClassifier): - If None (default), then draw `X.shape[0]` samples. - If int, then draw `max_samples` samples. - If float, then draw `max_samples * X.shape[0]` samples. Thus, - `max_samples` should be in the interval `(0, 1)`. + `max_samples` should be in the interval `(0.0, 1.0]`. .. versionadded:: 0.22 + monotonic_cst : array-like of int of shape (n_features), default=None + Indicates the monotonicity constraint to enforce on each feature. + - 1: monotonically increasing + - 0: no constraint + - -1: monotonically decreasing + + If monotonic_cst is None, no constraints are applied. + + Monotonicity constraints are not supported for: + - multiclass classifications (i.e. when `n_classes > 2`), + - multioutput classifications (i.e. when `n_outputs_ > 1`), + - classifications trained on data with missing values. + + The constraints hold over the probability of the positive class. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 1.4 + Attributes ---------- - base_estimator_ : ExtraTreesClassifier + estimator_ : :class:`~sklearn.tree.ExtraTreeClassifier` The child estimator template used to create the collection of fitted sub-estimators. + .. versionadded:: 1.2 + `base_estimator_` was renamed to `estimator_`. + estimators_ : list of DecisionTreeClassifier The collection of fitted sub-estimators. @@ -1799,12 +2178,16 @@ class labels (multi-output problem). high cardinality features (many unique values). See :func:`sklearn.inspection.permutation_importance` as an alternative. - n_features_ : int - The number of features when ``fit`` is performed. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. - .. deprecated:: 1.0 - Attribute `n_features_` was deprecated in version 1.0 and will be - removed in 1.2. Use `n_features_in_` instead. + .. versionadded:: 1.0 n_outputs_ : int The number of outputs when ``fit`` is performed. @@ -1821,11 +2204,17 @@ class labels (multi-output problem). `oob_decision_function_` might contain NaN. This attribute exists only when ``oob_score`` is True. + estimators_samples_ : list of arrays + The subset of drawn samples (i.e., the in-bag samples) for each base + estimator. Each subset is defined by an array of the indices selected. + + .. versionadded:: 1.4 + See Also -------- - sklearn.tree.ExtraTreeClassifier : Base classifier for this ensemble. - RandomForestClassifier : Ensemble Classifier based on trees with optimal - splits. + ExtraTreesRegressor : An extra-trees regressor with random splits. + RandomForestClassifier : A random forest classifier with optimal splits. + RandomForestRegressor : Ensemble regressor using trees with optimal splits. Notes ----- @@ -1851,35 +2240,58 @@ class labels (multi-output problem). >>> clf.predict([[0, 0, 0, 0]]) array([1]) """ - @_deprecate_positional_args - def __init__(self, - n_estimators=100, *, - criterion="gini", - max_depth=None, - min_samples_split=2, - min_samples_leaf=1, - min_weight_fraction_leaf=0., - max_features="auto", - max_leaf_nodes=None, - min_impurity_decrease=0., - min_impurity_split=None, - bootstrap=False, - oob_score=False, - n_jobs=None, - random_state=None, - verbose=0, - warm_start=False, - class_weight=None, - ccp_alpha=0.0, - max_samples=None): + + _parameter_constraints: dict = { + **ForestClassifier._parameter_constraints, + **DecisionTreeClassifier._parameter_constraints, + "class_weight": [ + StrOptions({"balanced_subsample", "balanced"}), + dict, + list, + None, + ], + } + _parameter_constraints.pop("splitter") + + def __init__( + self, + n_estimators=100, + *, + criterion="gini", + max_depth=None, + min_samples_split=2, + min_samples_leaf=1, + min_weight_fraction_leaf=0.0, + max_features="sqrt", + max_leaf_nodes=None, + min_impurity_decrease=0.0, + bootstrap=False, + oob_score=False, + n_jobs=None, + random_state=None, + verbose=0, + warm_start=False, + class_weight=None, + ccp_alpha=0.0, + max_samples=None, + monotonic_cst=None, + ): super().__init__( - base_estimator=ExtraTreeClassifier(), + estimator=ExtraTreeClassifier(), n_estimators=n_estimators, - estimator_params=("criterion", "max_depth", "min_samples_split", - "min_samples_leaf", "min_weight_fraction_leaf", - "max_features", "max_leaf_nodes", - "min_impurity_decrease", "min_impurity_split", - "random_state", "ccp_alpha"), + estimator_params=( + "criterion", + "max_depth", + "min_samples_split", + "min_samples_leaf", + "min_weight_fraction_leaf", + "max_features", + "max_leaf_nodes", + "min_impurity_decrease", + "random_state", + "ccp_alpha", + "monotonic_cst", + ), bootstrap=bootstrap, oob_score=oob_score, n_jobs=n_jobs, @@ -1887,7 +2299,8 @@ def __init__(self, verbose=verbose, warm_start=warm_start, class_weight=class_weight, - max_samples=max_samples) + max_samples=max_samples, + ) self.criterion = criterion self.max_depth = max_depth @@ -1897,8 +2310,8 @@ def __init__(self, self.max_features = max_features self.max_leaf_nodes = max_leaf_nodes self.min_impurity_decrease = min_impurity_decrease - self.min_impurity_split = min_impurity_split self.ccp_alpha = ccp_alpha + self.monotonic_cst = monotonic_cst class ExtraTreesRegressor(ForestRegressor): @@ -1910,6 +2323,14 @@ class ExtraTreesRegressor(ForestRegressor): of the dataset and uses averaging to improve the predictive accuracy and control over-fitting. + This estimator has native support for missing values (NaNs) for + random splits. During training, a random threshold will be chosen + to split the non-missing values on. Then the non-missing values will be sent + to the left and right child based on the randomly selected threshold, while + the missing values will also be randomly sent to the left or right child. + This is repeated for every feature considered at each split. The best split + among these is chosen. + Read more in the :ref:`User Guide `. Parameters @@ -1921,11 +2342,18 @@ class ExtraTreesRegressor(ForestRegressor): The default value of ``n_estimators`` changed from 10 to 100 in 0.22. - criterion : {"mse", "mae"}, default="mse" + criterion : {"squared_error", "absolute_error", "friedman_mse", "poisson"}, \ + default="squared_error" The function to measure the quality of a split. Supported criteria - are "mse" for the mean squared error, which is equal to variance - reduction as feature selection criterion, and "mae" for the mean - absolute error. + are "squared_error" for the mean squared error, which is equal to + variance reduction as feature selection criterion and minimizes the L2 + loss using the mean of each terminal node, "friedman_mse", which uses + mean squared error with Friedman's improvement score for potential + splits, "absolute_error" for the mean absolute error, which minimizes + the L1 loss using the median of each terminal node, and "poisson" which + uses reduction in Poisson deviance to find splits. + Training using "absolute_error" is significantly slower + than when using "squared_error". .. versionadded:: 0.18 Mean Absolute Error (MAE) criterion. @@ -1966,17 +2394,23 @@ class ExtraTreesRegressor(ForestRegressor): the input samples) required to be at a leaf node. Samples have equal weight when sample_weight is not provided. - max_features : {"auto", "sqrt", "log2"}, int or float, default="auto" + max_features : {"sqrt", "log2", None}, int or float, default=1.0 The number of features to consider when looking for the best split: - If int, then consider `max_features` features at each split. - If float, then `max_features` is a fraction and - `round(max_features * n_features)` features are considered at each + `max(1, int(max_features * n_features_in_))` features are considered at each split. - - If "auto", then `max_features=n_features`. - If "sqrt", then `max_features=sqrt(n_features)`. - If "log2", then `max_features=log2(n_features)`. - - If None, then `max_features=n_features`. + - If None or 1.0, then `max_features=n_features`. + + .. note:: + The default of 1.0 is equivalent to bagged trees and more + randomness can be achieved by setting smaller values, e.g. 0.3. + + .. versionchanged:: 1.1 + The default of `max_features` changed from `"auto"` to 1.0. Note: the search for a split does not stop until at least one valid partition of the node samples is found, even if it requires to @@ -2005,24 +2439,15 @@ class ExtraTreesRegressor(ForestRegressor): .. versionadded:: 0.19 - min_impurity_split : float, default=None - Threshold for early stopping in tree growth. A node will split - if its impurity is above the threshold, otherwise it is a leaf. - - .. deprecated:: 0.19 - ``min_impurity_split`` has been deprecated in favor of - ``min_impurity_decrease`` in 0.19. The default value of - ``min_impurity_split`` has changed from 1e-7 to 0 in 0.23 and it - will be removed in 1.0 (renaming of 0.25). - Use ``min_impurity_decrease`` instead. - bootstrap : bool, default=False Whether bootstrap samples are used when building trees. If False, the whole dataset is used to build each tree. - oob_score : bool, default=False + oob_score : bool or callable, default=False Whether to use out-of-bag samples to estimate the generalization score. - Only available if bootstrap=True. + By default, :func:`~sklearn.metrics.r2_score` is used. + Provide a callable with signature `metric(y_true, y_pred)` to use a + custom metric. Only available if `bootstrap=True`. n_jobs : int, default=None The number of jobs to run in parallel. :meth:`fit`, :meth:`predict`, @@ -2048,13 +2473,16 @@ class ExtraTreesRegressor(ForestRegressor): warm_start : bool, default=False When set to ``True``, reuse the solution of the previous call to fit and add more estimators to the ensemble, otherwise, just fit a whole - new forest. See :term:`the Glossary `. + new forest. See :term:`Glossary ` and + :ref:`tree_ensemble_warm_start` for details. ccp_alpha : non-negative float, default=0.0 Complexity parameter used for Minimal Cost-Complexity Pruning. The subtree with the largest cost complexity that is smaller than ``ccp_alpha`` will be chosen. By default, no pruning is performed. See - :ref:`minimal_cost_complexity_pruning` for details. + :ref:`minimal_cost_complexity_pruning` for details. See + :ref:`sphx_glr_auto_examples_tree_plot_cost_complexity_pruning.py` + for an example of such pruning. .. versionadded:: 0.22 @@ -2065,16 +2493,35 @@ class ExtraTreesRegressor(ForestRegressor): - If None (default), then draw `X.shape[0]` samples. - If int, then draw `max_samples` samples. - If float, then draw `max_samples * X.shape[0]` samples. Thus, - `max_samples` should be in the interval `(0, 1)`. + `max_samples` should be in the interval `(0.0, 1.0]`. .. versionadded:: 0.22 + monotonic_cst : array-like of int of shape (n_features), default=None + Indicates the monotonicity constraint to enforce on each feature. + - 1: monotonically increasing + - 0: no constraint + - -1: monotonically decreasing + + If monotonic_cst is None, no constraints are applied. + + Monotonicity constraints are not supported for: + - multioutput regressions (i.e. when `n_outputs_ > 1`), + - regressions trained on data with missing values. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 1.4 + Attributes ---------- - base_estimator_ : ExtraTreeRegressor + estimator_ : :class:`~sklearn.tree.ExtraTreeRegressor` The child estimator template used to create the collection of fitted sub-estimators. + .. versionadded:: 1.2 + `base_estimator_` was renamed to `estimator_`. + estimators_ : list of DecisionTreeRegressor The collection of fitted sub-estimators. @@ -2089,12 +2536,16 @@ class ExtraTreesRegressor(ForestRegressor): high cardinality features (many unique values). See :func:`sklearn.inspection.permutation_importance` as an alternative. - n_features_ : int - The number of features. + n_features_in_ : int + Number of features seen during :term:`fit`. - .. deprecated:: 1.0 - Attribute `n_features_` was deprecated in version 1.0 and will be - removed in 1.2. Use `n_features_in_` instead. + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 n_outputs_ : int The number of outputs. @@ -2107,9 +2558,16 @@ class ExtraTreesRegressor(ForestRegressor): Prediction computed with out-of-bag estimate on the training set. This attribute exists only when ``oob_score`` is True. + estimators_samples_ : list of arrays + The subset of drawn samples (i.e., the in-bag samples) for each base + estimator. Each subset is defined by an array of the indices selected. + + .. versionadded:: 1.4 + See Also -------- - sklearn.tree.ExtraTreeRegressor : Base estimator for this ensemble. + ExtraTreesClassifier : An extra-trees classifier with random splits. + RandomForestClassifier : A random forest classifier with optimal splits. RandomForestRegressor : Ensemble regressor using trees with optimal splits. Notes @@ -2136,43 +2594,61 @@ class ExtraTreesRegressor(ForestRegressor): >>> reg = ExtraTreesRegressor(n_estimators=100, random_state=0).fit( ... X_train, y_train) >>> reg.score(X_test, y_test) - 0.2708... + 0.2727... """ - @_deprecate_positional_args - def __init__(self, - n_estimators=100, *, - criterion="mse", - max_depth=None, - min_samples_split=2, - min_samples_leaf=1, - min_weight_fraction_leaf=0., - max_features="auto", - max_leaf_nodes=None, - min_impurity_decrease=0., - min_impurity_split=None, - bootstrap=False, - oob_score=False, - n_jobs=None, - random_state=None, - verbose=0, - warm_start=False, - ccp_alpha=0.0, - max_samples=None): + + _parameter_constraints: dict = { + **ForestRegressor._parameter_constraints, + **DecisionTreeRegressor._parameter_constraints, + } + _parameter_constraints.pop("splitter") + + def __init__( + self, + n_estimators=100, + *, + criterion="squared_error", + max_depth=None, + min_samples_split=2, + min_samples_leaf=1, + min_weight_fraction_leaf=0.0, + max_features=1.0, + max_leaf_nodes=None, + min_impurity_decrease=0.0, + bootstrap=False, + oob_score=False, + n_jobs=None, + random_state=None, + verbose=0, + warm_start=False, + ccp_alpha=0.0, + max_samples=None, + monotonic_cst=None, + ): super().__init__( - base_estimator=ExtraTreeRegressor(), + estimator=ExtraTreeRegressor(), n_estimators=n_estimators, - estimator_params=("criterion", "max_depth", "min_samples_split", - "min_samples_leaf", "min_weight_fraction_leaf", - "max_features", "max_leaf_nodes", - "min_impurity_decrease", "min_impurity_split", - "random_state", "ccp_alpha"), + estimator_params=( + "criterion", + "max_depth", + "min_samples_split", + "min_samples_leaf", + "min_weight_fraction_leaf", + "max_features", + "max_leaf_nodes", + "min_impurity_decrease", + "random_state", + "ccp_alpha", + "monotonic_cst", + ), bootstrap=bootstrap, oob_score=oob_score, n_jobs=n_jobs, random_state=random_state, verbose=verbose, warm_start=warm_start, - max_samples=max_samples) + max_samples=max_samples, + ) self.criterion = criterion self.max_depth = max_depth @@ -2182,11 +2658,11 @@ def __init__(self, self.max_features = max_features self.max_leaf_nodes = max_leaf_nodes self.min_impurity_decrease = min_impurity_decrease - self.min_impurity_split = min_impurity_split self.ccp_alpha = ccp_alpha + self.monotonic_cst = monotonic_cst -class RandomTreesEmbedding(BaseForest): +class RandomTreesEmbedding(TransformerMixin, BaseForest): """ An ensemble of totally random trees. @@ -2200,6 +2676,10 @@ class RandomTreesEmbedding(BaseForest): ``n_out <= n_estimators * max_leaf_nodes``. If ``max_leaf_nodes == None``, the number of leaf nodes is at most ``n_estimators * 2 ** max_depth``. + For an example of applying Random Trees Embedding to non-linear + classification, see + :ref:`sphx_glr_auto_examples_ensemble_plot_random_forest_embedding.py`. + Read more in the :ref:`User Guide `. Parameters @@ -2270,17 +2750,6 @@ class RandomTreesEmbedding(BaseForest): .. versionadded:: 0.19 - min_impurity_split : float, default=None - Threshold for early stopping in tree growth. A node will split - if its impurity is above the threshold, otherwise it is a leaf. - - .. deprecated:: 0.19 - ``min_impurity_split`` has been deprecated in favor of - ``min_impurity_decrease`` in 0.19. The default value of - ``min_impurity_split`` has changed from 1e-7 to 0 in 0.23 and it - will be removed in 1.0 (renaming of 0.25). - Use ``min_impurity_decrease`` instead. - sparse_output : bool, default=True Whether or not to return a sparse CSR matrix, as default behavior, or to return a dense array compatible with dense pipeline operators. @@ -2303,26 +2772,34 @@ class RandomTreesEmbedding(BaseForest): warm_start : bool, default=False When set to ``True``, reuse the solution of the previous call to fit and add more estimators to the ensemble, otherwise, just fit a whole - new forest. See :term:`the Glossary `. + new forest. See :term:`Glossary ` and + :ref:`tree_ensemble_warm_start` for details. Attributes ---------- - base_estimator_ : DecisionTreeClassifier instance + estimator_ : :class:`~sklearn.tree.ExtraTreeRegressor` instance The child estimator template used to create the collection of fitted sub-estimators. - estimators_ : list of DecisionTreeClassifier instances + .. versionadded:: 1.2 + `base_estimator_` was renamed to `estimator_`. + + estimators_ : list of :class:`~sklearn.tree.ExtraTreeRegressor` instances The collection of fitted sub-estimators. feature_importances_ : ndarray of shape (n_features,) The feature importances (the higher, the more important the feature). - n_features_ : int - The number of features when ``fit`` is performed. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. - .. deprecated:: 1.0 - Attribute `n_features_` was deprecated in version 1.0 and will be - removed in 1.2. Use `n_features_in_` instead. + .. versionadded:: 1.0 n_outputs_ : int The number of outputs when ``fit`` is performed. @@ -2330,6 +2807,23 @@ class RandomTreesEmbedding(BaseForest): one_hot_encoder_ : OneHotEncoder instance One-hot encoder used to create the sparse embedding. + estimators_samples_ : list of arrays + The subset of drawn samples (i.e., the in-bag samples) for each base + estimator. Each subset is defined by an array of the indices selected. + + .. versionadded:: 1.4 + + See Also + -------- + ExtraTreesClassifier : An extra-trees classifier. + ExtraTreesRegressor : An extra-trees regressor. + RandomForestClassifier : A random forest classifier. + RandomForestRegressor : A random forest regressor. + sklearn.tree.ExtraTreeClassifier: An extremely randomized + tree classifier. + sklearn.tree.ExtraTreeRegressor : An extremely randomized + tree regressor. + References ---------- .. [1] P. Geurts, D. Ernst., and L. Wehenkel, "Extremely randomized trees", @@ -2353,39 +2847,58 @@ class RandomTreesEmbedding(BaseForest): [0., 1., 1., 0., 1., 0., 0., 1., 1., 0.]]) """ - criterion = 'mse' + _parameter_constraints: dict = { + "n_estimators": [Interval(Integral, 1, None, closed="left")], + "n_jobs": [Integral, None], + "verbose": ["verbose"], + "warm_start": ["boolean"], + **BaseDecisionTree._parameter_constraints, + "sparse_output": ["boolean"], + } + for param in ("max_features", "ccp_alpha", "splitter", "monotonic_cst"): + _parameter_constraints.pop(param) + + criterion = "squared_error" max_features = 1 - @_deprecate_positional_args - def __init__(self, - n_estimators=100, *, - max_depth=5, - min_samples_split=2, - min_samples_leaf=1, - min_weight_fraction_leaf=0., - max_leaf_nodes=None, - min_impurity_decrease=0., - min_impurity_split=None, - sparse_output=True, - n_jobs=None, - random_state=None, - verbose=0, - warm_start=False): + def __init__( + self, + n_estimators=100, + *, + max_depth=5, + min_samples_split=2, + min_samples_leaf=1, + min_weight_fraction_leaf=0.0, + max_leaf_nodes=None, + min_impurity_decrease=0.0, + sparse_output=True, + n_jobs=None, + random_state=None, + verbose=0, + warm_start=False, + ): super().__init__( - base_estimator=ExtraTreeRegressor(), + estimator=ExtraTreeRegressor(), n_estimators=n_estimators, - estimator_params=("criterion", "max_depth", "min_samples_split", - "min_samples_leaf", "min_weight_fraction_leaf", - "max_features", "max_leaf_nodes", - "min_impurity_decrease", "min_impurity_split", - "random_state"), + estimator_params=( + "criterion", + "max_depth", + "min_samples_split", + "min_samples_leaf", + "min_weight_fraction_leaf", + "max_features", + "max_leaf_nodes", + "min_impurity_decrease", + "random_state", + ), bootstrap=False, oob_score=False, n_jobs=n_jobs, random_state=random_state, verbose=verbose, warm_start=warm_start, - max_samples=None) + max_samples=None, + ) self.max_depth = max_depth self.min_samples_split = min_samples_split @@ -2393,10 +2906,9 @@ def __init__(self, self.min_weight_fraction_leaf = min_weight_fraction_leaf self.max_leaf_nodes = max_leaf_nodes self.min_impurity_decrease = min_impurity_decrease - self.min_impurity_split = min_impurity_split self.sparse_output = sparse_output - def _set_oob_score_and_attributes(self, X, y): + def _set_oob_score_and_attributes(self, X, y, scoring_function=None): raise NotImplementedError("OOB score not supported by tree embedding") def fit(self, X, y=None, sample_weight=None): @@ -2423,11 +2935,13 @@ def fit(self, X, y=None, sample_weight=None): Returns ------- self : object - + Returns the instance itself. """ + # Parameters are validated in fit_transform self.fit_transform(X, y, sample_weight=sample_weight) return self + @_fit_context(prefer_skip_nested_validation=True) def fit_transform(self, X, y=None, sample_weight=None): """ Fit estimator and transform dataset. @@ -2453,18 +2967,46 @@ def fit_transform(self, X, y=None, sample_weight=None): X_transformed : sparse matrix of shape (n_samples, n_out) Transformed dataset. """ - X = self._validate_data(X, accept_sparse=['csc']) - if issparse(X): - # Pre-sort indices to avoid that each individual tree of the - # ensemble sorts the indices. - X.sort_indices() - rnd = check_random_state(self.random_state) - y = rnd.uniform(size=X.shape[0]) + y = rnd.uniform(size=_num_samples(X)) super().fit(X, y, sample_weight=sample_weight) - self.one_hot_encoder_ = OneHotEncoder(sparse=self.sparse_output) - return self.one_hot_encoder_.fit_transform(self.apply(X)) + self.one_hot_encoder_ = OneHotEncoder(sparse_output=self.sparse_output) + output = self.one_hot_encoder_.fit_transform(self.apply(X)) + self._n_features_out = output.shape[1] + return output + + def get_feature_names_out(self, input_features=None): + """Get output feature names for transformation. + + Parameters + ---------- + input_features : array-like of str or None, default=None + Only used to validate feature names with the names seen in :meth:`fit`. + + Returns + ------- + feature_names_out : ndarray of str objects + Transformed feature names, in the format of + `randomtreesembedding_{tree}_{leaf}`, where `tree` is the tree used + to generate the leaf and `leaf` is the index of a leaf node + in that tree. Note that the node indexing scheme is used to + index both nodes with children (split nodes) and leaf nodes. + Only the latter can be present as output features. + As a consequence, there are missing indices in the output + feature names. + """ + check_is_fitted(self, "_n_features_out") + _check_feature_names_in( + self, input_features=input_features, generate_names=False + ) + + feature_names = [ + f"randomtreesembedding_{tree}_{leaf}" + for tree in range(self.n_estimators) + for leaf in self.one_hot_encoder_.categories_[tree] + ] + return np.asarray(feature_names, dtype=object) def transform(self, X): """ @@ -2484,3 +3026,8 @@ def transform(self, X): """ check_is_fitted(self) return self.one_hot_encoder_.transform(self.apply(X)) + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.sparse = True + return tags diff --git a/sklearn/ensemble/_gb.py b/sklearn/ensemble/_gb.py index e9f7402188860..55c8e79e062df 100644 --- a/sklearn/ensemble/_gb.py +++ b/sklearn/ensemble/_gb.py @@ -16,45 +16,259 @@ regression problems. """ -# Authors: Peter Prettenhofer, Scott White, Gilles Louppe, Emanuele Olivetti, -# Arnaud Joly, Jacob Schreiber -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -from abc import ABCMeta -from abc import abstractmethod +import math import warnings +from abc import ABCMeta, abstractmethod +from numbers import Integral, Real +from time import time +import numpy as np +from scipy.sparse import csc_matrix, csr_matrix, issparse + +from .._loss.loss import ( + _LOSSES, + AbsoluteError, + ExponentialLoss, + HalfBinomialLoss, + HalfMultinomialLoss, + HalfSquaredError, + HuberLoss, + PinballLoss, +) +from ..base import ClassifierMixin, RegressorMixin, _fit_context, is_classifier +from ..dummy import DummyClassifier, DummyRegressor +from ..exceptions import NotFittedError +from ..model_selection import train_test_split +from ..preprocessing import LabelEncoder +from ..tree import DecisionTreeRegressor +from ..tree._tree import DOUBLE, DTYPE, TREE_LEAF +from ..utils import check_array, check_random_state, column_or_1d +from ..utils._param_validation import HasMethods, Interval, StrOptions +from ..utils.multiclass import check_classification_targets +from ..utils.stats import _weighted_percentile +from ..utils.validation import _check_sample_weight, check_is_fitted, validate_data from ._base import BaseEnsemble -from ..base import ClassifierMixin -from ..base import RegressorMixin -from ..base import BaseEstimator -from ..base import is_classifier -from ..utils import deprecated +from ._gradient_boosting import _random_sample_mask, predict_stage, predict_stages + +_LOSSES = _LOSSES.copy() +_LOSSES.update( + { + "quantile": PinballLoss, + "huber": HuberLoss, + } +) + + +def _safe_divide(numerator, denominator): + """Prevents overflow and division by zero.""" + # This is used for classifiers where the denominator might become zero exactly. + # For instance for log loss, HalfBinomialLoss, if proba=0 or proba=1 exactly, then + # denominator = hessian = 0, and we should set the node value in the line search to + # zero as there is no improvement of the loss possible. + # For numerical safety, we do this already for extremely tiny values. + if abs(denominator) < 1e-150: + return 0.0 + else: + # Cast to Python float to trigger Python errors, e.g. ZeroDivisionError, + # without relying on `np.errstate` that is not supported by Pyodide. + result = float(numerator) / float(denominator) + # Cast to Python float to trigger a ZeroDivisionError without relying + # on `np.errstate` that is not supported by Pyodide. + result = float(numerator) / float(denominator) + if math.isinf(result): + warnings.warn("overflow encountered in _safe_divide", RuntimeWarning) + return result + + +def _init_raw_predictions(X, estimator, loss, use_predict_proba): + """Return the initial raw predictions. -from ._gradient_boosting import predict_stages -from ._gradient_boosting import predict_stage -from ._gradient_boosting import _random_sample_mask + Parameters + ---------- + X : ndarray of shape (n_samples, n_features) + The data array. + estimator : object + The estimator to use to compute the predictions. + loss : BaseLoss + An instance of a loss function class. + use_predict_proba : bool + Whether estimator.predict_proba is used instead of estimator.predict. + + Returns + ------- + raw_predictions : ndarray of shape (n_samples, K) + The initial raw predictions. K is equal to 1 for binary + classification and regression, and equal to the number of classes + for multiclass classification. ``raw_predictions`` is casted + into float64. + """ + # TODO: Use loss.fit_intercept_only where appropriate instead of + # DummyRegressor which is the default given by the `init` parameter, + # see also _init_state. + if use_predict_proba: + # Our parameter validation, set via _fit_context and _parameter_constraints + # already guarantees that estimator has a predict_proba method. + predictions = estimator.predict_proba(X) + if not loss.is_multiclass: + predictions = predictions[:, 1] # probability of positive class + eps = np.finfo(np.float32).eps # FIXME: This is quite large! + predictions = np.clip(predictions, eps, 1 - eps, dtype=np.float64) + else: + predictions = estimator.predict(X).astype(np.float64) + + if predictions.ndim == 1: + return loss.link.link(predictions).reshape(-1, 1) + else: + return loss.link.link(predictions) + + +def _update_terminal_regions( + loss, + tree, + X, + y, + neg_gradient, + raw_prediction, + sample_weight, + sample_mask, + learning_rate=0.1, + k=0, +): + """Update the leaf values to be predicted by the tree and raw_prediction. + + The current raw predictions of the model (of this stage) are updated. + + Additionally, the terminal regions (=leaves) of the given tree are updated as well. + This corresponds to the line search step in "Greedy Function Approximation" by + Friedman, Algorithm 1 step 5. + + Update equals: + argmin_{x} loss(y_true, raw_prediction_old + x * tree.value) + + For non-trivial cases like the Binomial loss, the update has no closed formula and + is an approximation, again, see the Friedman paper. + + Also note that the update formula for the SquaredError is the identity. Therefore, + in this case, the leaf values don't need an update and only the raw_predictions are + updated (with the learning rate included). -import numbers -import numpy as np + Parameters + ---------- + loss : BaseLoss + tree : tree.Tree + The tree object. + X : ndarray of shape (n_samples, n_features) + The data array. + y : ndarray of shape (n_samples,) + The target labels. + neg_gradient : ndarray of shape (n_samples,) + The negative gradient. + raw_prediction : ndarray of shape (n_samples, n_trees_per_iteration) + The raw predictions (i.e. values from the tree leaves) of the + tree ensemble at iteration ``i - 1``. + sample_weight : ndarray of shape (n_samples,) + The weight of each sample. + sample_mask : ndarray of shape (n_samples,) + The sample mask to be used. + learning_rate : float, default=0.1 + Learning rate shrinks the contribution of each tree by + ``learning_rate``. + k : int, default=0 + The index of the estimator being updated. + """ + # compute leaf for each sample in ``X``. + terminal_regions = tree.apply(X) + + if not isinstance(loss, HalfSquaredError): + # mask all which are not in sample mask. + masked_terminal_regions = terminal_regions.copy() + masked_terminal_regions[~sample_mask] = -1 + + if isinstance(loss, HalfBinomialLoss): + + def compute_update(y_, indices, neg_gradient, raw_prediction, k): + # Make a single Newton-Raphson step, see "Additive Logistic Regression: + # A Statistical View of Boosting" FHT00 and note that we use a slightly + # different version (factor 2) of "F" with proba=expit(raw_prediction). + # Our node estimate is given by: + # sum(w * (y - prob)) / sum(w * prob * (1 - prob)) + # we take advantage that: y - prob = neg_gradient + neg_g = neg_gradient.take(indices, axis=0) + prob = y_ - neg_g + # numerator = negative gradient = y - prob + numerator = np.average(neg_g, weights=sw) + # denominator = hessian = prob * (1 - prob) + denominator = np.average(prob * (1 - prob), weights=sw) + return _safe_divide(numerator, denominator) + + elif isinstance(loss, HalfMultinomialLoss): + + def compute_update(y_, indices, neg_gradient, raw_prediction, k): + # we take advantage that: y - prob = neg_gradient + neg_g = neg_gradient.take(indices, axis=0) + prob = y_ - neg_g + K = loss.n_classes + # numerator = negative gradient * (k - 1) / k + # Note: The factor (k - 1)/k appears in the original papers "Greedy + # Function Approximation" by Friedman and "Additive Logistic + # Regression" by Friedman, Hastie, Tibshirani. This factor is, however, + # wrong or at least arbitrary as it directly multiplies the + # learning_rate. We keep it for backward compatibility. + numerator = np.average(neg_g, weights=sw) + numerator *= (K - 1) / K + # denominator = (diagonal) hessian = prob * (1 - prob) + denominator = np.average(prob * (1 - prob), weights=sw) + return _safe_divide(numerator, denominator) + + elif isinstance(loss, ExponentialLoss): + + def compute_update(y_, indices, neg_gradient, raw_prediction, k): + neg_g = neg_gradient.take(indices, axis=0) + # numerator = negative gradient = y * exp(-raw) - (1-y) * exp(raw) + numerator = np.average(neg_g, weights=sw) + # denominator = hessian = y * exp(-raw) + (1-y) * exp(raw) + # if y=0: hessian = exp(raw) = -neg_g + # y=1: hessian = exp(-raw) = neg_g + hessian = neg_g.copy() + hessian[y_ == 0] *= -1 + denominator = np.average(hessian, weights=sw) + return _safe_divide(numerator, denominator) -from scipy.sparse import csc_matrix -from scipy.sparse import csr_matrix -from scipy.sparse import issparse + else: -from time import time -from ..model_selection import train_test_split -from ..tree import DecisionTreeRegressor -from ..tree._tree import DTYPE, DOUBLE -from . import _gb_losses + def compute_update(y_, indices, neg_gradient, raw_prediction, k): + return loss.fit_intercept_only( + y_true=y_ - raw_prediction[indices, k], + sample_weight=sw, + ) + + # update each leaf (= perform line search) + for leaf in np.nonzero(tree.children_left == TREE_LEAF)[0]: + indices = np.nonzero(masked_terminal_regions == leaf)[ + 0 + ] # of terminal regions + y_ = y.take(indices, axis=0) + sw = None if sample_weight is None else sample_weight[indices] + update = compute_update(y_, indices, neg_gradient, raw_prediction, k) + + # TODO: Multiply here by learning rate instead of everywhere else. + tree.value[leaf, 0, 0] = update + + # update predictions (both in-bag and out-of-bag) + raw_prediction[:, k] += learning_rate * tree.value[:, 0, 0].take( + terminal_regions, axis=0 + ) -from ..utils import check_random_state -from ..utils import check_array -from ..utils import column_or_1d -from ..utils.validation import check_is_fitted, _check_sample_weight -from ..utils.multiclass import check_classification_targets -from ..exceptions import NotFittedError -from ..utils.validation import _deprecate_positional_args + +def set_huber_delta(loss, y_true, raw_prediction, sample_weight=None): + """Calculate and set self.closs.delta based on self.quantile.""" + abserr = np.abs(y_true - raw_prediction.squeeze()) + # sample_weight is always a ndarray, never None. + delta = _weighted_percentile(abserr, sample_weight, 100 * loss.quantile) + loss.closs.delta = float(delta) class VerboseReporter: @@ -67,6 +281,7 @@ class VerboseReporter: (when iteration mod verbose_mod is zero).; if larger than 1 then output is printed for each update. """ + def __init__(self, verbose): self.verbose = verbose @@ -82,20 +297,19 @@ def init(self, est, begin_at_stage=0): stage at which to begin reporting """ # header fields and line format str - header_fields = ['Iter', 'Train Loss'] - verbose_fmt = ['{iter:>10d}', '{train_score:>16.4f}'] + header_fields = ["Iter", "Train Loss"] + verbose_fmt = ["{iter:>10d}", "{train_score:>16.4f}"] # do oob? if est.subsample < 1: - header_fields.append('OOB Improve') - verbose_fmt.append('{oob_impr:>16.4f}') - header_fields.append('Remaining Time') - verbose_fmt.append('{remaining_time:>16s}') + header_fields.append("OOB Improve") + verbose_fmt.append("{oob_impr:>16.4f}") + header_fields.append("Remaining Time") + verbose_fmt.append("{remaining_time:>16s}") # print the header line - print(('%10s ' + '%16s ' * - (len(header_fields) - 1)) % tuple(header_fields)) + print(("%10s " + "%16s " * (len(header_fields) - 1)) % tuple(header_fields)) - self.verbose_fmt = ' '.join(verbose_fmt) + self.verbose_fmt = " ".join(verbose_fmt) # plot verbose info each time i % verbose_mod == 0 self.verbose_mod = 1 self.start_time = time() @@ -116,16 +330,21 @@ def update(self, j, est): i = j - self.begin_at_stage # iteration relative to the start iter if (i + 1) % self.verbose_mod == 0: oob_impr = est.oob_improvement_[j] if do_oob else 0 - remaining_time = ((est.n_estimators - (j + 1)) * - (time() - self.start_time) / float(i + 1)) + remaining_time = ( + (est.n_estimators - (j + 1)) * (time() - self.start_time) / float(i + 1) + ) if remaining_time > 60: - remaining_time = '{0:.2f}m'.format(remaining_time / 60.0) + remaining_time = "{0:.2f}m".format(remaining_time / 60.0) else: - remaining_time = '{0:.2f}s'.format(remaining_time) - print(self.verbose_fmt.format(iter=j + 1, - train_score=est.train_score_[j], - oob_impr=oob_impr, - remaining_time=remaining_time)) + remaining_time = "{0:.2f}s".format(remaining_time) + print( + self.verbose_fmt.format( + iter=j + 1, + train_score=est.train_score_[j], + oob_impr=oob_impr, + remaining_time=remaining_time, + ) + ) if self.verbose == 1 and ((i + 1) // (self.verbose_mod * 10) > 0): # adjust verbose frequency (powers of 10) self.verbose_mod *= 10 @@ -134,15 +353,47 @@ def update(self, j, est): class BaseGradientBoosting(BaseEnsemble, metaclass=ABCMeta): """Abstract base class for Gradient Boosting.""" - @abstractmethod - def __init__(self, *, loss, learning_rate, n_estimators, criterion, - min_samples_split, min_samples_leaf, min_weight_fraction_leaf, - max_depth, min_impurity_decrease, min_impurity_split, - init, subsample, max_features, ccp_alpha, - random_state, alpha=0.9, verbose=0, max_leaf_nodes=None, - warm_start=False, validation_fraction=0.1, - n_iter_no_change=None, tol=1e-4): + _parameter_constraints: dict = { + **DecisionTreeRegressor._parameter_constraints, + "learning_rate": [Interval(Real, 0.0, None, closed="left")], + "n_estimators": [Interval(Integral, 1, None, closed="left")], + "criterion": [StrOptions({"friedman_mse", "squared_error"})], + "subsample": [Interval(Real, 0.0, 1.0, closed="right")], + "verbose": ["verbose"], + "warm_start": ["boolean"], + "validation_fraction": [Interval(Real, 0.0, 1.0, closed="neither")], + "n_iter_no_change": [Interval(Integral, 1, None, closed="left"), None], + "tol": [Interval(Real, 0.0, None, closed="left")], + } + _parameter_constraints.pop("splitter") + _parameter_constraints.pop("monotonic_cst") + @abstractmethod + def __init__( + self, + *, + loss, + learning_rate, + n_estimators, + criterion, + min_samples_split, + min_samples_leaf, + min_weight_fraction_leaf, + max_depth, + min_impurity_decrease, + init, + subsample, + max_features, + ccp_alpha, + random_state, + alpha=0.9, + verbose=0, + max_leaf_nodes=None, + warm_start=False, + validation_fraction=0.1, + n_iter_no_change=None, + tol=1e-4, + ): self.n_estimators = n_estimators self.learning_rate = learning_rate self.loss = loss @@ -154,7 +405,6 @@ def __init__(self, *, loss, learning_rate, n_estimators, criterion, self.max_features = max_features self.max_depth = max_depth self.min_impurity_decrease = min_impurity_decrease - self.min_impurity_split = min_impurity_split self.ccp_alpha = ccp_alpha self.init = init self.random_state = random_state @@ -167,109 +417,101 @@ def __init__(self, *, loss, learning_rate, n_estimators, criterion, self.tol = tol @abstractmethod - def _validate_y(self, y, sample_weight=None): - """Called by fit to validate y.""" + def _encode_y(self, y=None, sample_weight=None): + """Called by fit to validate and encode y.""" - def _fit_stage(self, i, X, y, raw_predictions, sample_weight, sample_mask, - random_state, X_csc=None, X_csr=None): - """Fit another stage of ``_n_classes`` trees to the boosting model.""" - - assert sample_mask.dtype == bool - loss = self.loss_ + @abstractmethod + def _get_loss(self, sample_weight): + """Get loss object from sklearn._loss.loss.""" + + def _fit_stage( + self, + i, + X, + y, + raw_predictions, + sample_weight, + sample_mask, + random_state, + X_csc=None, + X_csr=None, + ): + """Fit another stage of ``n_trees_per_iteration_`` trees.""" original_y = y - # Need to pass a copy of raw_predictions to negative_gradient() - # because raw_predictions is partially updated at the end of the loop - # in update_terminal_regions(), and gradients need to be evaluated at - # iteration i - 1. - raw_predictions_copy = raw_predictions.copy() + if isinstance(self._loss, HuberLoss): + set_huber_delta( + loss=self._loss, + y_true=y, + raw_prediction=raw_predictions, + sample_weight=sample_weight, + ) + # TODO: Without oob, i.e. with self.subsample = 1.0, we could call + # self._loss.loss_gradient and use it to set train_score_. + # But note that train_score_[i] is the score AFTER fitting the i-th tree. + # Note: We need the negative gradient! + neg_gradient = -self._loss.gradient( + y_true=y, + raw_prediction=raw_predictions, + sample_weight=None, # We pass sample_weights to the tree directly. + ) + # 2-d views of shape (n_samples, n_trees_per_iteration_) or (n_samples, 1) + # on neg_gradient to simplify the loop over n_trees_per_iteration_. + if neg_gradient.ndim == 1: + neg_g_view = neg_gradient.reshape((-1, 1)) + else: + neg_g_view = neg_gradient - for k in range(loss.K): - if loss.is_multi_class: + for k in range(self.n_trees_per_iteration_): + if self._loss.is_multiclass: y = np.array(original_y == k, dtype=np.float64) - residual = loss.negative_gradient(y, raw_predictions_copy, k=k, - sample_weight=sample_weight) - - # induce regression tree on residuals + # induce regression tree on the negative gradient tree = DecisionTreeRegressor( criterion=self.criterion, - splitter='best', + splitter="best", max_depth=self.max_depth, min_samples_split=self.min_samples_split, min_samples_leaf=self.min_samples_leaf, min_weight_fraction_leaf=self.min_weight_fraction_leaf, min_impurity_decrease=self.min_impurity_decrease, - min_impurity_split=self.min_impurity_split, max_features=self.max_features, max_leaf_nodes=self.max_leaf_nodes, random_state=random_state, - ccp_alpha=self.ccp_alpha) + ccp_alpha=self.ccp_alpha, + ) if self.subsample < 1.0: # no inplace multiplication! sample_weight = sample_weight * sample_mask.astype(np.float64) - X = X_csr if X_csr is not None else X - tree.fit(X, residual, sample_weight=sample_weight, - check_input=False) + X = X_csc if X_csc is not None else X + tree.fit( + X, neg_g_view[:, k], sample_weight=sample_weight, check_input=False + ) # update tree leaves - loss.update_terminal_regions( - tree.tree_, X, y, residual, raw_predictions, sample_weight, - sample_mask, learning_rate=self.learning_rate, k=k) + X_for_tree_update = X_csr if X_csr is not None else X + _update_terminal_regions( + self._loss, + tree.tree_, + X_for_tree_update, + y, + neg_g_view[:, k], + raw_predictions, + sample_weight, + sample_mask, + learning_rate=self.learning_rate, + k=k, + ) # add tree to ensemble self.estimators_[i, k] = tree return raw_predictions - def _check_params(self): - """Check validity of parameters and raise ValueError if not valid.""" - if self.n_estimators <= 0: - raise ValueError("n_estimators must be greater than 0 but " - "was %r" % self.n_estimators) - - if self.learning_rate <= 0.0: - raise ValueError("learning_rate must be greater than 0 but " - "was %r" % self.learning_rate) - - if (self.loss not in self._SUPPORTED_LOSS - or self.loss not in _gb_losses.LOSS_FUNCTIONS): - raise ValueError("Loss '{0:s}' not supported. ".format(self.loss)) - - if self.loss == 'deviance': - loss_class = (_gb_losses.MultinomialDeviance - if len(self.classes_) > 2 - else _gb_losses.BinomialDeviance) - else: - loss_class = _gb_losses.LOSS_FUNCTIONS[self.loss] - - if is_classifier(self): - self.loss_ = loss_class(self.n_classes_) - elif self.loss in ("huber", "quantile"): - self.loss_ = loss_class(self.alpha) - else: - self.loss_ = loss_class() - - if not (0.0 < self.subsample <= 1.0): - raise ValueError("subsample must be in (0,1] but " - "was %r" % self.subsample) - - if self.init is not None: - # init must be an estimator or 'zero' - if isinstance(self.init, BaseEstimator): - self.loss_.check_init_estimator(self.init) - elif not (isinstance(self.init, str) and self.init == 'zero'): - raise ValueError( - "The init parameter must be an estimator or 'zero'. " - "Got init={}".format(self.init) - ) - - if not (0.0 < self.alpha < 1.0): - raise ValueError("alpha must be in (0.0, 1.0) but " - "was %r" % self.alpha) - + def _set_max_features(self): + """Set self.max_features_.""" if isinstance(self.max_features, str): if self.max_features == "auto": if is_classifier(self): @@ -278,57 +520,56 @@ def _check_params(self): max_features = self.n_features_in_ elif self.max_features == "sqrt": max_features = max(1, int(np.sqrt(self.n_features_in_))) - elif self.max_features == "log2": + else: # self.max_features == "log2" max_features = max(1, int(np.log2(self.n_features_in_))) - else: - raise ValueError("Invalid value for max_features: %r. " - "Allowed string values are 'auto', 'sqrt' " - "or 'log2'." % self.max_features) elif self.max_features is None: max_features = self.n_features_in_ - elif isinstance(self.max_features, numbers.Integral): + elif isinstance(self.max_features, Integral): max_features = self.max_features else: # float - if 0. < self.max_features <= 1.: - max_features = max(int(self.max_features * - self.n_features_in_), 1) - else: - raise ValueError("max_features must be in (0, n_features]") + max_features = max(1, int(self.max_features * self.n_features_in_)) self.max_features_ = max_features - if not isinstance(self.n_iter_no_change, - (numbers.Integral, type(None))): - raise ValueError("n_iter_no_change should either be None or an " - "integer. %r was passed" - % self.n_iter_no_change) - def _init_state(self): - """Initialize model state and allocate model state data structures. """ + """Initialize model state and allocate model state data structures.""" self.init_ = self.init if self.init_ is None: - self.init_ = self.loss_.init_estimator() + if is_classifier(self): + self.init_ = DummyClassifier(strategy="prior") + elif isinstance(self._loss, (AbsoluteError, HuberLoss)): + self.init_ = DummyRegressor(strategy="quantile", quantile=0.5) + elif isinstance(self._loss, PinballLoss): + self.init_ = DummyRegressor(strategy="quantile", quantile=self.alpha) + else: + self.init_ = DummyRegressor(strategy="mean") - self.estimators_ = np.empty((self.n_estimators, self.loss_.K), - dtype=object) + self.estimators_ = np.empty( + (self.n_estimators, self.n_trees_per_iteration_), dtype=object + ) self.train_score_ = np.zeros((self.n_estimators,), dtype=np.float64) # do oob? if self.subsample < 1.0: - self.oob_improvement_ = np.zeros((self.n_estimators), - dtype=np.float64) + self.oob_improvement_ = np.zeros((self.n_estimators), dtype=np.float64) + self.oob_scores_ = np.zeros((self.n_estimators), dtype=np.float64) + self.oob_score_ = np.nan def _clear_state(self): - """Clear the state of the gradient boosting model. """ - if hasattr(self, 'estimators_'): + """Clear the state of the gradient boosting model.""" + if hasattr(self, "estimators_"): self.estimators_ = np.empty((0, 0), dtype=object) - if hasattr(self, 'train_score_'): + if hasattr(self, "train_score_"): del self.train_score_ - if hasattr(self, 'oob_improvement_'): + if hasattr(self, "oob_improvement_"): del self.oob_improvement_ - if hasattr(self, 'init_'): + if hasattr(self, "oob_scores_"): + del self.oob_scores_ + if hasattr(self, "oob_score_"): + del self.oob_score_ + if hasattr(self, "init_"): del self.init_ - if hasattr(self, '_rng'): + if hasattr(self, "_rng"): del self._rng def _resize_state(self): @@ -336,32 +577,41 @@ def _resize_state(self): # self.n_estimators is the number of additional est to fit total_n_estimators = self.n_estimators if total_n_estimators < self.estimators_.shape[0]: - raise ValueError('resize with smaller n_estimators %d < %d' % - (total_n_estimators, self.estimators_[0])) + raise ValueError( + "resize with smaller n_estimators %d < %d" + % (total_n_estimators, self.estimators_[0]) + ) - self.estimators_ = np.resize(self.estimators_, - (total_n_estimators, self.loss_.K)) + self.estimators_ = np.resize( + self.estimators_, (total_n_estimators, self.n_trees_per_iteration_) + ) self.train_score_ = np.resize(self.train_score_, total_n_estimators) - if (self.subsample < 1 or hasattr(self, 'oob_improvement_')): + if self.subsample < 1 or hasattr(self, "oob_improvement_"): # if do oob resize arrays or create new if not available - if hasattr(self, 'oob_improvement_'): - self.oob_improvement_ = np.resize(self.oob_improvement_, - total_n_estimators) + if hasattr(self, "oob_improvement_"): + self.oob_improvement_ = np.resize( + self.oob_improvement_, total_n_estimators + ) + self.oob_scores_ = np.resize(self.oob_scores_, total_n_estimators) + self.oob_score_ = np.nan else: - self.oob_improvement_ = np.zeros((total_n_estimators,), - dtype=np.float64) + self.oob_improvement_ = np.zeros( + (total_n_estimators,), dtype=np.float64 + ) + self.oob_scores_ = np.zeros((total_n_estimators,), dtype=np.float64) + self.oob_score_ = np.nan - def _is_initialized(self): - return len(getattr(self, 'estimators_', [])) > 0 + def _is_fitted(self): + return len(getattr(self, "estimators_", [])) > 0 def _check_initialized(self): """Check that the estimator is initialized, raising an error if not.""" check_is_fitted(self) - @abstractmethod - def _warn_mae_for_criterion(self): - pass - + @_fit_context( + # GradientBoosting*.init is not validated yet + prefer_skip_nested_validation=False + ) def fit(self, X, y, sample_weight=None, monitor=None): """Fit the gradient boosting model. @@ -391,17 +641,13 @@ def fit(self, X, y, sample_weight=None, monitor=None): locals())``. If the callable returns ``True`` the fitting procedure is stopped. The monitor can be used for various things such as computing held-out estimates, early stopping, model introspect, and - snapshoting. + snapshotting. Returns ------- self : object + Fitted estimator. """ - if self.criterion == 'mae': - # TODO: This should raise an error from 1.1 - self._warn_mae_for_criterion() - - # if not warmstart - clear the estimator state if not self.warm_start: self._clear_state() @@ -409,114 +655,173 @@ def fit(self, X, y, sample_weight=None, monitor=None): # Since check_array converts both X and y to the same dtype, but the # trees use different types for X and y, checking them separately. - X, y = self._validate_data(X, y, accept_sparse=['csr', 'csc', 'coo'], - dtype=DTYPE, multi_output=True) - + X, y = validate_data( + self, + X, + y, + accept_sparse=["csr", "csc", "coo"], + dtype=DTYPE, + multi_output=True, + ) sample_weight_is_none = sample_weight is None - sample_weight = _check_sample_weight(sample_weight, X) + if sample_weight_is_none: + y = self._encode_y(y=y, sample_weight=None) + else: + y = self._encode_y(y=y, sample_weight=sample_weight) + y = column_or_1d(y, warn=True) # TODO: Is this still required? - y = column_or_1d(y, warn=True) + self._set_max_features() - if is_classifier(self): - y = self._validate_y(y, sample_weight) - else: - y = self._validate_y(y) + # self.loss is guaranteed to be a string + self._loss = self._get_loss(sample_weight=sample_weight) if self.n_iter_no_change is not None: stratify = y if is_classifier(self) else None - X, X_val, y, y_val, sample_weight, sample_weight_val = ( - train_test_split(X, y, sample_weight, - random_state=self.random_state, - test_size=self.validation_fraction, - stratify=stratify)) + ( + X_train, + X_val, + y_train, + y_val, + sample_weight_train, + sample_weight_val, + ) = train_test_split( + X, + y, + sample_weight, + random_state=self.random_state, + test_size=self.validation_fraction, + stratify=stratify, + ) if is_classifier(self): - if self._n_classes != np.unique(y).shape[0]: + if self.n_classes_ != np.unique(y_train).shape[0]: # We choose to error here. The problem is that the init # estimator would be trained on y, which has some missing # classes now, so its predictions would not have the # correct shape. raise ValueError( - 'The training data after the early stopping split ' - 'is missing some classes. Try using another random ' - 'seed.' + "The training data after the early stopping split " + "is missing some classes. Try using another random " + "seed." ) else: + X_train, y_train, sample_weight_train = X, y, sample_weight X_val = y_val = sample_weight_val = None - self._check_params() + n_samples = X_train.shape[0] - if not self._is_initialized(): + # First time calling fit. + if not self._is_fitted(): # init state self._init_state() # fit initial model and initialize raw predictions - if self.init_ == 'zero': - raw_predictions = np.zeros(shape=(X.shape[0], self.loss_.K), - dtype=np.float64) + if self.init_ == "zero": + raw_predictions = np.zeros( + shape=(n_samples, self.n_trees_per_iteration_), + dtype=np.float64, + ) else: # XXX clean this once we have a support_sample_weight tag if sample_weight_is_none: - self.init_.fit(X, y) + self.init_.fit(X_train, y_train) else: - msg = ("The initial estimator {} does not support sample " - "weights.".format(self.init_.__class__.__name__)) + msg = ( + "The initial estimator {} does not support sample " + "weights.".format(self.init_.__class__.__name__) + ) try: - self.init_.fit(X, y, sample_weight=sample_weight) + self.init_.fit( + X_train, y_train, sample_weight=sample_weight_train + ) except TypeError as e: - # regular estimator without SW support - raise ValueError(msg) from e + if "unexpected keyword argument 'sample_weight'" in str(e): + # regular estimator without SW support + raise ValueError(msg) from e + else: # regular estimator whose input checking failed + raise except ValueError as e: - if "pass parameters to specific steps of "\ - "your pipeline using the "\ - "stepname__parameter" in str(e): # pipeline + if ( + "pass parameters to specific steps of " + "your pipeline using the " + "stepname__parameter" in str(e) + ): # pipeline raise ValueError(msg) from e else: # regular estimator whose input checking failed raise - raw_predictions = \ - self.loss_.get_init_raw_predictions(X, self.init_) + raw_predictions = _init_raw_predictions( + X_train, self.init_, self._loss, is_classifier(self) + ) begin_at_stage = 0 # The rng state must be preserved if warm_start is True self._rng = check_random_state(self.random_state) + # warm start: this is not the first time fit was called else: # add more estimators to fitted model # invariant: warm_start = True if self.n_estimators < self.estimators_.shape[0]: - raise ValueError('n_estimators=%d must be larger or equal to ' - 'estimators_.shape[0]=%d when ' - 'warm_start==True' - % (self.n_estimators, - self.estimators_.shape[0])) + raise ValueError( + "n_estimators=%d must be larger or equal to " + "estimators_.shape[0]=%d when " + "warm_start==True" % (self.n_estimators, self.estimators_.shape[0]) + ) begin_at_stage = self.estimators_.shape[0] - # The requirements of _decision_function (called in two lines - # below) are more constrained than fit. It accepts only CSR - # matrices. - X = check_array(X, dtype=DTYPE, order="C", accept_sparse='csr') - raw_predictions = self._raw_predict(X) + # The requirements of _raw_predict + # are more constrained than fit. It accepts only CSR + # matrices. Finite values have already been checked in _validate_data. + X_train = check_array( + X_train, + dtype=DTYPE, + order="C", + accept_sparse="csr", + ensure_all_finite=False, + ) + raw_predictions = self._raw_predict(X_train) self._resize_state() # fit the boosting stages n_stages = self._fit_stages( - X, y, raw_predictions, sample_weight, self._rng, X_val, y_val, - sample_weight_val, begin_at_stage, monitor) + X_train, + y_train, + raw_predictions, + sample_weight_train, + self._rng, + X_val, + y_val, + sample_weight_val, + begin_at_stage, + monitor, + ) # change shape of arrays after fit (early-stopping or additional ests) if n_stages != self.estimators_.shape[0]: self.estimators_ = self.estimators_[:n_stages] self.train_score_ = self.train_score_[:n_stages] - if hasattr(self, 'oob_improvement_'): + if hasattr(self, "oob_improvement_"): + # OOB scores were computed self.oob_improvement_ = self.oob_improvement_[:n_stages] - + self.oob_scores_ = self.oob_scores_[:n_stages] + self.oob_score_ = self.oob_scores_[-1] self.n_estimators_ = n_stages return self - def _fit_stages(self, X, y, raw_predictions, sample_weight, random_state, - X_val, y_val, sample_weight_val, - begin_at_stage=0, monitor=None): + def _fit_stages( + self, + X, + y, + raw_predictions, + sample_weight, + random_state, + X_val, + y_val, + sample_weight_val, + begin_at_stage=0, + monitor=None, + ): """Iteratively fits the stages. For each stage it computes the progress (OOB, train score) @@ -526,9 +831,8 @@ def _fit_stages(self, X, y, raw_predictions, sample_weight, random_state, """ n_samples = X.shape[0] do_oob = self.subsample < 1.0 - sample_mask = np.ones((n_samples, ), dtype=bool) + sample_mask = np.ones((n_samples,), dtype=bool) n_inbag = max(1, int(self.subsample * n_samples)) - loss_ = self.loss_ if self.verbose: verbose_reporter = VerboseReporter(verbose=self.verbose) @@ -541,38 +845,75 @@ def _fit_stages(self, X, y, raw_predictions, sample_weight, random_state, loss_history = np.full(self.n_iter_no_change, np.inf) # We create a generator to get the predictions for X_val after # the addition of each successive stage - y_val_pred_iter = self._staged_raw_predict(X_val) + y_val_pred_iter = self._staged_raw_predict(X_val, check_input=False) + + # Older versions of GBT had its own loss functions. With the new common + # private loss function submodule _loss, we often are a factor of 2 + # away from the old version. Here we keep backward compatibility for + # oob_scores_ and oob_improvement_, even if the old way is quite + # inconsistent (sometimes the gradient is half the gradient, sometimes + # not). + if isinstance( + self._loss, + ( + HalfSquaredError, + HalfBinomialLoss, + ), + ): + factor = 2 + else: + factor = 1 # perform boosting iterations i = begin_at_stage for i in range(begin_at_stage, self.n_estimators): - # subsampling if do_oob: - sample_mask = _random_sample_mask(n_samples, n_inbag, - random_state) - # OOB score before adding this stage - old_oob_score = loss_(y[~sample_mask], - raw_predictions[~sample_mask], - sample_weight[~sample_mask]) + sample_mask = _random_sample_mask(n_samples, n_inbag, random_state) + y_oob_masked = y[~sample_mask] + sample_weight_oob_masked = sample_weight[~sample_mask] + if i == 0: # store the initial loss to compute the OOB score + initial_loss = factor * self._loss( + y_true=y_oob_masked, + raw_prediction=raw_predictions[~sample_mask], + sample_weight=sample_weight_oob_masked, + ) # fit next stage of trees raw_predictions = self._fit_stage( - i, X, y, raw_predictions, sample_weight, sample_mask, - random_state, X_csc, X_csr) + i, + X, + y, + raw_predictions, + sample_weight, + sample_mask, + random_state, + X_csc=X_csc, + X_csr=X_csr, + ) - # track deviance (= loss) + # track loss if do_oob: - self.train_score_[i] = loss_(y[sample_mask], - raw_predictions[sample_mask], - sample_weight[sample_mask]) - self.oob_improvement_[i] = ( - old_oob_score - loss_(y[~sample_mask], - raw_predictions[~sample_mask], - sample_weight[~sample_mask])) + self.train_score_[i] = factor * self._loss( + y_true=y[sample_mask], + raw_prediction=raw_predictions[sample_mask], + sample_weight=sample_weight[sample_mask], + ) + self.oob_scores_[i] = factor * self._loss( + y_true=y_oob_masked, + raw_prediction=raw_predictions[~sample_mask], + sample_weight=sample_weight_oob_masked, + ) + previous_loss = initial_loss if i == 0 else self.oob_scores_[i - 1] + self.oob_improvement_[i] = previous_loss - self.oob_scores_[i] + self.oob_score_ = self.oob_scores_[-1] else: # no need to fancy index w/ no subsampling - self.train_score_[i] = loss_(y, raw_predictions, sample_weight) + self.train_score_[i] = factor * self._loss( + y_true=y, + raw_prediction=raw_predictions, + sample_weight=sample_weight, + ) if self.verbose > 0: verbose_reporter.update(i, self) @@ -587,8 +928,9 @@ def _fit_stages(self, X, y, raw_predictions, sample_weight, random_state, if self.n_iter_no_change is not None: # By calling next(y_val_pred_iter), we get the predictions # for X_val after the addition of the current stage - validation_loss = loss_(y_val, next(y_val_pred_iter), - sample_weight_val) + validation_loss = factor * self._loss( + y_val, next(y_val_pred_iter), sample_weight_val + ) # Require validation_score to be better (less) than at least # one of the last n_iter_no_change evaluations @@ -607,22 +949,24 @@ def _raw_predict_init(self, X): """Check input and compute raw predictions of the init estimator.""" self._check_initialized() X = self.estimators_[0, 0]._validate_X_predict(X, check_input=True) - if self.init_ == 'zero': - raw_predictions = np.zeros(shape=(X.shape[0], self.loss_.K), - dtype=np.float64) + if self.init_ == "zero": + raw_predictions = np.zeros( + shape=(X.shape[0], self.n_trees_per_iteration_), dtype=np.float64 + ) else: - raw_predictions = self.loss_.get_init_raw_predictions( - X, self.init_).astype(np.float64) + raw_predictions = _init_raw_predictions( + X, self.init_, self._loss, is_classifier(self) + ) return raw_predictions def _raw_predict(self, X): """Return the sum of the trees raw predictions (+ init estimator).""" + check_is_fitted(self) raw_predictions = self._raw_predict_init(X) - predict_stages(self.estimators_, X, self.learning_rate, - raw_predictions) + predict_stages(self.estimators_, X, self.learning_rate, raw_predictions) return raw_predictions - def _staged_raw_predict(self, X): + def _staged_raw_predict(self, X, check_input=True): """Compute raw predictions of ``X`` for each iteration. This method allows monitoring (i.e. determine error on testing set) @@ -635,6 +979,9 @@ def _staged_raw_predict(self, X): ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csr_matrix``. + check_input : bool, default=True + If False, the input arrays X will not be checked. + Returns ------- raw_predictions : generator of ndarray of shape (n_samples, k) @@ -643,12 +990,13 @@ def _staged_raw_predict(self, X): Regression and binary classification are special cases with ``k == 1``, otherwise ``k==n_classes``. """ - X = self._validate_data(X, dtype=DTYPE, order="C", accept_sparse='csr', - reset=False) + if check_input: + X = validate_data( + self, X, dtype=DTYPE, order="C", accept_sparse="csr", reset=False + ) raw_predictions = self._raw_predict_init(X) for i in range(self.estimators_.shape[0]): - predict_stage(self.estimators_, i, X, self.learning_rate, - raw_predictions) + predict_stage(self.estimators_, i, X, self.learning_rate, raw_predictions) yield raw_predictions.copy() @property @@ -673,9 +1021,12 @@ def feature_importances_(self): """ self._check_initialized() - relevant_trees = [tree - for stage in self.estimators_ for tree in stage - if tree.tree_.node_count > 1] + relevant_trees = [ + tree + for stage in self.estimators_ + for tree in stage + if tree.tree_.node_count > 1 + ] if not relevant_trees: # degenerate case where all trees have only one node return np.zeros(shape=self.n_features_in_, dtype=np.float64) @@ -684,8 +1035,9 @@ def feature_importances_(self): tree.tree_.compute_feature_importances(normalize=False) for tree in relevant_trees ] - avg_feature_importances = np.mean(relevant_feature_importances, - axis=0, dtype=np.float64) + avg_feature_importances = np.mean( + relevant_feature_importances, axis=0, dtype=np.float64 + ) return avg_feature_importances / np.sum(avg_feature_importances) def _compute_partial_dependence_recursion(self, grid, target_features): @@ -693,35 +1045,39 @@ def _compute_partial_dependence_recursion(self, grid, target_features): Parameters ---------- - grid : ndarray of shape (n_samples, n_target_features) + grid : ndarray of shape (n_samples, n_target_features), dtype=np.float32 The grid points on which the partial dependence should be evaluated. - target_features : ndarray of shape (n_target_features,) + target_features : ndarray of shape (n_target_features,), dtype=np.intp The set of target features for which the partial dependence should be evaluated. Returns ------- averaged_predictions : ndarray of shape \ - (n_trees_per_iteration, n_samples) + (n_trees_per_iteration_, n_samples) The value of the partial dependence function on each grid point. """ if self.init is not None: warnings.warn( - 'Using recursion method with a non-constant init predictor ' - 'will lead to incorrect partial dependence values. ' - 'Got init=%s.' % self.init, - UserWarning + "Using recursion method with a non-constant init predictor " + "will lead to incorrect partial dependence values. " + "Got init=%s." % self.init, + UserWarning, ) - grid = np.asarray(grid, dtype=DTYPE, order='C') + grid = np.asarray(grid, dtype=DTYPE, order="C") n_estimators, n_trees_per_stage = self.estimators_.shape - averaged_predictions = np.zeros((n_trees_per_stage, grid.shape[0]), - dtype=np.float64, order='C') + averaged_predictions = np.zeros( + (n_trees_per_stage, grid.shape[0]), dtype=np.float64, order="C" + ) + target_features = np.asarray(target_features, dtype=np.intp, order="C") + for stage in range(n_estimators): for k in range(n_trees_per_stage): tree = self.estimators_[stage, k].tree_ - tree.compute_partial_dependence(grid, target_features, - averaged_predictions[k]) + tree.compute_partial_dependence( + grid, target_features, averaged_predictions[k] + ) averaged_predictions *= self.learning_rate return averaged_predictions @@ -761,45 +1117,50 @@ def apply(self, X): return leaves - # TODO: Remove in 1.2 - # mypy error: Decorated property not supported - @deprecated( # type: ignore - "Attribute n_features_ was deprecated in version 1.0 and will be " - "removed in 1.2. Use 'n_features_in_' instead." - ) - @property - def n_features_(self): - return self.n_features_in_ + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.sparse = True + return tags class GradientBoostingClassifier(ClassifierMixin, BaseGradientBoosting): """Gradient Boosting for classification. - GB builds an additive model in a - forward stage-wise fashion; it allows for the optimization of - arbitrary differentiable loss functions. In each stage ``n_classes_`` - regression trees are fit on the negative gradient of the - binomial or multinomial deviance loss function. Binary classification - is a special case where only a single regression tree is induced. + This algorithm builds an additive model in a forward stage-wise fashion; it + allows for the optimization of arbitrary differentiable loss functions. In + each stage ``n_classes_`` regression trees are fit on the negative gradient + of the loss function, e.g. binary or multiclass log loss. Binary + classification is a special case where only a single regression tree is + induced. + + :class:`~sklearn.ensemble.HistGradientBoostingClassifier` is a much faster variant + of this algorithm for intermediate and large datasets (`n_samples >= 10_000`) and + supports monotonic constraints. Read more in the :ref:`User Guide `. Parameters ---------- - loss : {'deviance', 'exponential'}, default='deviance' - The loss function to be optimized. 'deviance' refers to - deviance (= logistic regression) for classification - with probabilistic outputs. For loss 'exponential' gradient - boosting recovers the AdaBoost algorithm. + loss : {'log_loss', 'exponential'}, default='log_loss' + The loss function to be optimized. 'log_loss' refers to binomial and + multinomial deviance, the same as used in logistic regression. + It is a good choice for classification with probabilistic outputs. + For loss 'exponential', gradient boosting recovers the AdaBoost algorithm. learning_rate : float, default=0.1 Learning rate shrinks the contribution of each tree by `learning_rate`. There is a trade-off between learning_rate and n_estimators. + Values must be in the range `[0.0, inf)`. + + For an example of the effects of this parameter and its interaction with + ``subsample``, see + :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_regularization.py`. n_estimators : int, default=100 The number of boosting stages to perform. Gradient boosting is fairly robust to over-fitting so a large number usually results in better performance. + Values must be in the range `[1, inf)`. subsample : float, default=1.0 The fraction of samples to be used for fitting the individual base @@ -807,29 +1168,23 @@ class GradientBoostingClassifier(ClassifierMixin, BaseGradientBoosting): Boosting. `subsample` interacts with the parameter `n_estimators`. Choosing `subsample < 1.0` leads to a reduction of variance and an increase in bias. + Values must be in the range `(0.0, 1.0]`. - criterion : {'friedman_mse', 'mse', 'mae'}, default='friedman_mse' - The function to measure the quality of a split. Supported criteria - are 'friedman_mse' for the mean squared error with improvement - score by Friedman, 'mse' for mean squared error, and 'mae' for - the mean absolute error. The default value of 'friedman_mse' is - generally the best as it can provide a better approximation in - some cases. + criterion : {'friedman_mse', 'squared_error'}, default='friedman_mse' + The function to measure the quality of a split. Supported criteria are + 'friedman_mse' for the mean squared error with improvement score by + Friedman, 'squared_error' for mean squared error. The default value of + 'friedman_mse' is generally the best as it can provide a better + approximation in some cases. .. versionadded:: 0.18 - .. deprecated:: 0.24 - `criterion='mae'` is deprecated and will be removed in version - 1.1 (renaming of 0.26). Use `criterion='friedman_mse'` or `'mse'` - instead, as trees should use a least-square criterion in - Gradient Boosting. min_samples_split : int or float, default=2 The minimum number of samples required to split an internal node: - - If int, then consider `min_samples_split` as the minimum number. - - If float, then `min_samples_split` is a fraction and - `ceil(min_samples_split * n_samples)` are the minimum - number of samples for each split. + - If int, values must be in the range `[2, inf)`. + - If float, values must be in the range `(0.0, 1.0]` and `min_samples_split` + will be `ceil(min_samples_split * n_samples)`. .. versionchanged:: 0.18 Added float values for fractions. @@ -841,10 +1196,9 @@ class GradientBoostingClassifier(ClassifierMixin, BaseGradientBoosting): right branches. This may have the effect of smoothing the model, especially in regression. - - If int, then consider `min_samples_leaf` as the minimum number. - - If float, then `min_samples_leaf` is a fraction and - `ceil(min_samples_leaf * n_samples)` are the minimum - number of samples for each node. + - If int, values must be in the range `[1, inf)`. + - If float, values must be in the range `(0.0, 1.0)` and `min_samples_leaf` + will be `ceil(min_samples_leaf * n_samples)`. .. versionchanged:: 0.18 Added float values for fractions. @@ -853,16 +1207,21 @@ class GradientBoostingClassifier(ClassifierMixin, BaseGradientBoosting): The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node. Samples have equal weight when sample_weight is not provided. + Values must be in the range `[0.0, 0.5]`. - max_depth : int, default=3 - The maximum depth of the individual regression estimators. The maximum + max_depth : int or None, default=3 + Maximum depth of the individual regression estimators. The maximum depth limits the number of nodes in the tree. Tune this parameter for best performance; the best value depends on the interaction - of the input variables. + of the input variables. If None, then nodes are expanded until + all leaves are pure or until all leaves contain less than + min_samples_split samples. + If int, values must be in the range `[1, inf)`. min_impurity_decrease : float, default=0.0 A node will be split if this split induces a decrease of the impurity greater than or equal to this value. + Values must be in the range `[0.0, inf)`. The weighted impurity decrease equation is the following:: @@ -878,20 +1237,9 @@ class GradientBoostingClassifier(ClassifierMixin, BaseGradientBoosting): .. versionadded:: 0.19 - min_impurity_split : float, default=None - Threshold for early stopping in tree growth. A node will split - if its impurity is above the threshold, otherwise it is a leaf. - - .. deprecated:: 0.19 - ``min_impurity_split`` has been deprecated in favor of - ``min_impurity_decrease`` in 0.19. The default value of - ``min_impurity_split`` has changed from 1e-7 to 0 in 0.23 and it - will be removed in 1.0 (renaming of 0.25). - Use ``min_impurity_decrease`` instead. - init : estimator or 'zero', default=None An estimator object that is used to compute the initial predictions. - ``init`` has to provide :meth:`fit` and :meth:`predict_proba`. If + ``init`` has to provide :term:`fit` and :term:`predict_proba`. If 'zero', the initial raw predictions are set to zero. By default, a ``DummyEstimator`` predicting the classes priors is used. @@ -900,19 +1248,17 @@ class GradientBoostingClassifier(ClassifierMixin, BaseGradientBoosting): boosting iteration. In addition, it controls the random permutation of the features at each split (see Notes for more details). - It also controls the random spliting of the training data to obtain a + It also controls the random splitting of the training data to obtain a validation set if `n_iter_no_change` is not None. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. - max_features : {'auto', 'sqrt', 'log2'}, int or float, default=None + max_features : {'sqrt', 'log2'}, int or float, default=None The number of features to consider when looking for the best split: - - If int, then consider `max_features` features at each split. - - If float, then `max_features` is a fraction and - `int(max_features * n_features)` features are considered at each - split. - - If 'auto', then `max_features=sqrt(n_features)`. + - If int, values must be in the range `[1, inf)`. + - If float, values must be in the range `(0.0, 1.0]` and the features + considered at each split will be `max(1, int(max_features * n_features_in_))`. - If 'sqrt', then `max_features=sqrt(n_features)`. - If 'log2', then `max_features=log2(n_features)`. - If None, then `max_features=n_features`. @@ -928,11 +1274,13 @@ class GradientBoostingClassifier(ClassifierMixin, BaseGradientBoosting): Enable verbose output. If 1 then it prints progress and performance once in a while (the more trees the lower the frequency). If greater than 1 then it prints progress and performance for every tree. + Values must be in the range `[0, inf)`. max_leaf_nodes : int, default=None Grow trees with ``max_leaf_nodes`` in best-first fashion. Best nodes are defined as relative reduction in impurity. - If None then unlimited number of leaf nodes. + Values must be in the range `[2, inf)`. + If `None`, then unlimited number of leaf nodes. warm_start : bool, default=False When set to ``True``, reuse the solution of the previous call to fit @@ -941,7 +1289,7 @@ class GradientBoostingClassifier(ClassifierMixin, BaseGradientBoosting): validation_fraction : float, default=0.1 The proportion of training data to set aside as validation set for - early stopping. Must be between 0 and 1. + early stopping. Values must be in the range `(0.0, 1.0)`. Only used if ``n_iter_no_change`` is set to an integer. .. versionadded:: 0.20 @@ -954,6 +1302,9 @@ class GradientBoostingClassifier(ClassifierMixin, BaseGradientBoosting): data as validation and terminate training when validation score is not improving in all of the previous ``n_iter_no_change`` numbers of iterations. The split is stratified. + Values must be in the range `[1, inf)`. + See + :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_early_stopping.py`. .. versionadded:: 0.20 @@ -961,14 +1312,18 @@ class GradientBoostingClassifier(ClassifierMixin, BaseGradientBoosting): Tolerance for the early stopping. When the loss is not improving by at least tol for ``n_iter_no_change`` iterations (if set to a number), the training stops. + Values must be in the range `[0.0, inf)`. .. versionadded:: 0.20 ccp_alpha : non-negative float, default=0.0 Complexity parameter used for Minimal Cost-Complexity Pruning. The subtree with the largest cost complexity that is smaller than - ``ccp_alpha`` will be chosen. By default, no pruning is performed. See - :ref:`minimal_cost_complexity_pruning` for details. + ``ccp_alpha`` will be chosen. By default, no pruning is performed. + Values must be in the range `[0.0, inf)`. + See :ref:`minimal_cost_complexity_pruning` for details. See + :ref:`sphx_glr_auto_examples_tree_plot_cost_complexity_pruning.py` + for an example of such pruning. .. versionadded:: 0.22 @@ -981,6 +1336,12 @@ class GradientBoostingClassifier(ClassifierMixin, BaseGradientBoosting): .. versionadded:: 0.20 + n_trees_per_iteration_ : int + The number of trees that are built at each iteration. For binary classifiers, + this is always 1. + + .. versionadded:: 1.4.0 + feature_importances_ : ndarray of shape (n_features,) The impurity-based feature importances. The higher, the more important the feature. @@ -993,38 +1354,51 @@ class GradientBoostingClassifier(ClassifierMixin, BaseGradientBoosting): :func:`sklearn.inspection.permutation_importance` as an alternative. oob_improvement_ : ndarray of shape (n_estimators,) - The improvement in loss (= deviance) on the out-of-bag samples + The improvement in loss on the out-of-bag samples relative to the previous iteration. ``oob_improvement_[0]`` is the improvement in loss of the first stage over the ``init`` estimator. - Only available if ``subsample < 1.0`` + Only available if ``subsample < 1.0``. + + oob_scores_ : ndarray of shape (n_estimators,) + The full history of the loss values on the out-of-bag + samples. Only available if `subsample < 1.0`. + + .. versionadded:: 1.3 + + oob_score_ : float + The last value of the loss on the out-of-bag samples. It is + the same as `oob_scores_[-1]`. Only available if `subsample < 1.0`. + + .. versionadded:: 1.3 train_score_ : ndarray of shape (n_estimators,) - The i-th score ``train_score_[i]`` is the deviance (= loss) of the + The i-th score ``train_score_[i]`` is the loss of the model at iteration ``i`` on the in-bag sample. - If ``subsample == 1`` this is the deviance on the training data. - - loss_ : LossFunction - The concrete ``LossFunction`` object. + If ``subsample == 1`` this is the loss on the training data. init_ : estimator - The estimator that provides the initial predictions. - Set via the ``init`` argument or ``loss.init_estimator``. + The estimator that provides the initial predictions. Set via the ``init`` + argument. estimators_ : ndarray of DecisionTreeRegressor of \ - shape (n_estimators, ``loss_.K``) - The collection of fitted sub-estimators. ``loss_.K`` is 1 for binary - classification, otherwise n_classes. + shape (n_estimators, ``n_trees_per_iteration_``) + The collection of fitted sub-estimators. ``n_trees_per_iteration_`` is 1 for + binary classification, otherwise ``n_classes``. classes_ : ndarray of shape (n_classes,) The classes labels. - n_features_ : int - The number of data features. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 - .. deprecated:: 1.0 - Attribute `n_features_` was deprecated in version 1.0 and will be - removed in 1.2. Use `n_features_in_` instead. + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 n_classes_ : int The number of classes. @@ -1080,57 +1454,109 @@ class GradientBoostingClassifier(ClassifierMixin, BaseGradientBoosting): >>> clf = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0, ... max_depth=1, random_state=0).fit(X_train, y_train) >>> clf.score(X_test, y_test) - 0.913... + 0.913 """ - _SUPPORTED_LOSS = ('deviance', 'exponential') - - @_deprecate_positional_args - def __init__(self, *, loss='deviance', learning_rate=0.1, n_estimators=100, - subsample=1.0, criterion='friedman_mse', min_samples_split=2, - min_samples_leaf=1, min_weight_fraction_leaf=0., - max_depth=3, min_impurity_decrease=0., - min_impurity_split=None, init=None, - random_state=None, max_features=None, verbose=0, - max_leaf_nodes=None, warm_start=False, - validation_fraction=0.1, n_iter_no_change=None, tol=1e-4, - ccp_alpha=0.0): - + _parameter_constraints: dict = { + **BaseGradientBoosting._parameter_constraints, + "loss": [StrOptions({"log_loss", "exponential"})], + "init": [StrOptions({"zero"}), None, HasMethods(["fit", "predict_proba"])], + } + + def __init__( + self, + *, + loss="log_loss", + learning_rate=0.1, + n_estimators=100, + subsample=1.0, + criterion="friedman_mse", + min_samples_split=2, + min_samples_leaf=1, + min_weight_fraction_leaf=0.0, + max_depth=3, + min_impurity_decrease=0.0, + init=None, + random_state=None, + max_features=None, + verbose=0, + max_leaf_nodes=None, + warm_start=False, + validation_fraction=0.1, + n_iter_no_change=None, + tol=1e-4, + ccp_alpha=0.0, + ): super().__init__( - loss=loss, learning_rate=learning_rate, n_estimators=n_estimators, - criterion=criterion, min_samples_split=min_samples_split, + loss=loss, + learning_rate=learning_rate, + n_estimators=n_estimators, + criterion=criterion, + min_samples_split=min_samples_split, min_samples_leaf=min_samples_leaf, min_weight_fraction_leaf=min_weight_fraction_leaf, - max_depth=max_depth, init=init, subsample=subsample, + max_depth=max_depth, + init=init, + subsample=subsample, max_features=max_features, - random_state=random_state, verbose=verbose, + random_state=random_state, + verbose=verbose, max_leaf_nodes=max_leaf_nodes, min_impurity_decrease=min_impurity_decrease, - min_impurity_split=min_impurity_split, - warm_start=warm_start, validation_fraction=validation_fraction, - n_iter_no_change=n_iter_no_change, tol=tol, ccp_alpha=ccp_alpha) - - def _validate_y(self, y, sample_weight): + warm_start=warm_start, + validation_fraction=validation_fraction, + n_iter_no_change=n_iter_no_change, + tol=tol, + ccp_alpha=ccp_alpha, + ) + + def _encode_y(self, y, sample_weight): + # encode classes into 0 ... n_classes - 1 and sets attributes classes_ + # and n_trees_per_iteration_ check_classification_targets(y) - self.classes_, y = np.unique(y, return_inverse=True) - n_trim_classes = np.count_nonzero(np.bincount(y, sample_weight)) - if n_trim_classes < 2: - raise ValueError("y contains %d class after sample_weight " - "trimmed classes with zero weights, while a " - "minimum of 2 classes are required." - % n_trim_classes) - self._n_classes = len(self.classes_) + + label_encoder = LabelEncoder() + encoded_y_int = label_encoder.fit_transform(y) + self.classes_ = label_encoder.classes_ + n_classes = self.classes_.shape[0] + # only 1 tree for binary classification. For multiclass classification, + # we build 1 tree per class. + self.n_trees_per_iteration_ = 1 if n_classes <= 2 else n_classes + encoded_y = encoded_y_int.astype(float, copy=False) + + # From here on, it is additional to the HGBT case. # expose n_classes_ attribute - self.n_classes_ = self._n_classes - return y + self.n_classes_ = n_classes + if sample_weight is None: + n_trim_classes = n_classes + else: + n_trim_classes = np.count_nonzero(np.bincount(encoded_y_int, sample_weight)) - def _warn_mae_for_criterion(self): - # TODO: This should raise an error from 1.1 - warnings.warn("criterion='mae' was deprecated in version 0.24 and " - "will be removed in version 1.1 (renaming of 0.26). Use " - "criterion='friedman_mse' or 'mse' instead, as trees " - "should use a least-square criterion in Gradient " - "Boosting.", FutureWarning) + if n_trim_classes < 2: + raise ValueError( + "y contains %d class after sample_weight " + "trimmed classes with zero weights, while a " + "minimum of 2 classes are required." % n_trim_classes + ) + return encoded_y + + def _get_loss(self, sample_weight): + if self.loss == "log_loss": + if self.n_classes_ == 2: + return HalfBinomialLoss(sample_weight=sample_weight) + else: + return HalfMultinomialLoss( + sample_weight=sample_weight, n_classes=self.n_classes_ + ) + elif self.loss == "exponential": + if self.n_classes_ > 2: + raise ValueError( + f"loss='{self.loss}' is only suitable for a binary classification " + f"problem, you have n_classes={self.n_classes_}. " + "Please use loss='log_loss' instead." + ) + else: + return ExponentialLoss(sample_weight=sample_weight) def decision_function(self, X): """Compute the decision function of ``X``. @@ -1151,8 +1577,9 @@ def decision_function(self, X): :term:`classes_`. Regression and binary classification produce an array of shape (n_samples,). """ - X = self._validate_data(X, dtype=DTYPE, order="C", accept_sparse='csr', - reset=False) + X = validate_data( + self, X, dtype=DTYPE, order="C", accept_sparse="csr", reset=False + ) raw_predictions = self._raw_predict(X) if raw_predictions.shape[1] == 1: return raw_predictions.ravel() @@ -1171,8 +1598,8 @@ def staged_decision_function(self, X): ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csr_matrix``. - Returns - ------- + Yields + ------ score : generator of ndarray of shape (n_samples, k) The decision function of the input samples, which corresponds to the raw values predicted from the trees of the ensemble . The @@ -1198,9 +1625,11 @@ def predict(self, X): The predicted values. """ raw_predictions = self.decision_function(X) - encoded_labels = \ - self.loss_._raw_prediction_to_decision(raw_predictions) - return self.classes_.take(encoded_labels, axis=0) + if raw_predictions.ndim == 1: # decision_function already squeezed it + encoded_classes = (raw_predictions >= 0).astype(int) + else: + encoded_classes = np.argmax(raw_predictions, axis=1) + return self.classes_[encoded_classes] def staged_predict(self, X): """Predict class at each stage for X. @@ -1215,15 +1644,19 @@ def staged_predict(self, X): ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csr_matrix``. - Returns - ------- + Yields + ------ y : generator of ndarray of shape (n_samples,) The predicted value of the input samples. """ - for raw_predictions in self._staged_raw_predict(X): - encoded_labels = \ - self.loss_._raw_prediction_to_decision(raw_predictions) - yield self.classes_.take(encoded_labels, axis=0) + if self.n_classes_ == 2: # n_trees_per_iteration_ = 1 + for raw_predictions in self._staged_raw_predict(X): + encoded_classes = (raw_predictions.squeeze() >= 0).astype(int) + yield self.classes_.take(encoded_classes, axis=0) + else: + for raw_predictions in self._staged_raw_predict(X): + encoded_classes = np.argmax(raw_predictions, axis=1) + yield self.classes_.take(encoded_classes, axis=0) def predict_proba(self, X): """Predict class probabilities for X. @@ -1235,25 +1668,19 @@ def predict_proba(self, X): ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csr_matrix``. - Raises - ------ - AttributeError - If the ``loss`` does not support probabilities. - Returns ------- p : ndarray of shape (n_samples, n_classes) The class probabilities of the input samples. The order of the classes corresponds to that in the attribute :term:`classes_`. + + Raises + ------ + AttributeError + If the ``loss`` does not support probabilities. """ raw_predictions = self.decision_function(X) - try: - return self.loss_._raw_prediction_to_proba(raw_predictions) - except NotFittedError: - raise - except AttributeError as e: - raise AttributeError('loss=%r does not support predict_proba' % - self.loss) from e + return self._loss.predict_proba(raw_predictions) def predict_log_proba(self, X): """Predict class log-probabilities for X. @@ -1265,16 +1692,16 @@ def predict_log_proba(self, X): ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csr_matrix``. - Raises - ------ - AttributeError - If the ``loss`` does not support probabilities. - Returns ------- p : ndarray of shape (n_samples, n_classes) The class log-probabilities of the input samples. The order of the classes corresponds to that in the attribute :term:`classes_`. + + Raises + ------ + AttributeError + If the ``loss`` does not support probabilities. """ proba = self.predict_proba(X) return np.log(proba) @@ -1292,48 +1719,60 @@ def staged_predict_proba(self, X): ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csr_matrix``. - Returns - ------- + Yields + ------ y : generator of ndarray of shape (n_samples,) The predicted value of the input samples. """ try: for raw_predictions in self._staged_raw_predict(X): - yield self.loss_._raw_prediction_to_proba(raw_predictions) + yield self._loss.predict_proba(raw_predictions) except NotFittedError: raise except AttributeError as e: - raise AttributeError('loss=%r does not support predict_proba' % - self.loss) from e + raise AttributeError( + "loss=%r does not support predict_proba" % self.loss + ) from e class GradientBoostingRegressor(RegressorMixin, BaseGradientBoosting): """Gradient Boosting for regression. - GB builds an additive model in a forward stage-wise fashion; - it allows for the optimization of arbitrary differentiable loss functions. - In each stage a regression tree is fit on the negative gradient of the - given loss function. + This estimator builds an additive model in a forward stage-wise fashion; it + allows for the optimization of arbitrary differentiable loss functions. In + each stage a regression tree is fit on the negative gradient of the given + loss function. + + :class:`~sklearn.ensemble.HistGradientBoostingRegressor` is a much faster variant + of this algorithm for intermediate and large datasets (`n_samples >= 10_000`) and + supports monotonic constraints. Read more in the :ref:`User Guide `. Parameters ---------- - loss : {'ls', 'lad', 'huber', 'quantile'}, default='ls' - Loss function to be optimized. 'ls' refers to least squares - regression. 'lad' (least absolute deviation) is a highly robust - loss function solely based on order information of the input - variables. 'huber' is a combination of the two. 'quantile' - allows quantile regression (use `alpha` to specify the quantile). + loss : {'squared_error', 'absolute_error', 'huber', 'quantile'}, \ + default='squared_error' + Loss function to be optimized. 'squared_error' refers to the squared + error for regression. 'absolute_error' refers to the absolute error of + regression and is a robust loss function. 'huber' is a + combination of the two. 'quantile' allows quantile regression (use + `alpha` to specify the quantile). + See + :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_quantile.py` + for an example that demonstrates quantile regression for creating + prediction intervals with `loss='quantile'`. learning_rate : float, default=0.1 Learning rate shrinks the contribution of each tree by `learning_rate`. There is a trade-off between learning_rate and n_estimators. + Values must be in the range `[0.0, inf)`. n_estimators : int, default=100 The number of boosting stages to perform. Gradient boosting is fairly robust to over-fitting so a large number usually results in better performance. + Values must be in the range `[1, inf)`. subsample : float, default=1.0 The fraction of samples to be used for fitting the individual base @@ -1341,28 +1780,23 @@ class GradientBoostingRegressor(RegressorMixin, BaseGradientBoosting): Boosting. `subsample` interacts with the parameter `n_estimators`. Choosing `subsample < 1.0` leads to a reduction of variance and an increase in bias. + Values must be in the range `(0.0, 1.0]`. - criterion : {'friedman_mse', 'mse', 'mae'}, default='friedman_mse' - The function to measure the quality of a split. Supported criteria - are "friedman_mse" for the mean squared error with improvement - score by Friedman, "mse" for mean squared error, and "mae" for - the mean absolute error. The default value of "friedman_mse" is - generally the best as it can provide a better approximation in - some cases. + criterion : {'friedman_mse', 'squared_error'}, default='friedman_mse' + The function to measure the quality of a split. Supported criteria are + "friedman_mse" for the mean squared error with improvement score by + Friedman, "squared_error" for mean squared error. The default value of + "friedman_mse" is generally the best as it can provide a better + approximation in some cases. .. versionadded:: 0.18 - .. deprecated:: 0.24 - `criterion='mae'` is deprecated and will be removed in version - 1.1 (renaming of 0.26). The correct way of minimizing the absolute - error is to use `loss='lad'` instead. min_samples_split : int or float, default=2 The minimum number of samples required to split an internal node: - - If int, then consider `min_samples_split` as the minimum number. - - If float, then `min_samples_split` is a fraction and - `ceil(min_samples_split * n_samples)` are the minimum - number of samples for each split. + - If int, values must be in the range `[2, inf)`. + - If float, values must be in the range `(0.0, 1.0]` and `min_samples_split` + will be `ceil(min_samples_split * n_samples)`. .. versionchanged:: 0.18 Added float values for fractions. @@ -1374,10 +1808,9 @@ class GradientBoostingRegressor(RegressorMixin, BaseGradientBoosting): right branches. This may have the effect of smoothing the model, especially in regression. - - If int, then consider `min_samples_leaf` as the minimum number. - - If float, then `min_samples_leaf` is a fraction and - `ceil(min_samples_leaf * n_samples)` are the minimum - number of samples for each node. + - If int, values must be in the range `[1, inf)`. + - If float, values must be in the range `(0.0, 1.0)` and `min_samples_leaf` + will be `ceil(min_samples_leaf * n_samples)`. .. versionchanged:: 0.18 Added float values for fractions. @@ -1386,16 +1819,21 @@ class GradientBoostingRegressor(RegressorMixin, BaseGradientBoosting): The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node. Samples have equal weight when sample_weight is not provided. + Values must be in the range `[0.0, 0.5]`. - max_depth : int, default=3 + max_depth : int or None, default=3 Maximum depth of the individual regression estimators. The maximum depth limits the number of nodes in the tree. Tune this parameter for best performance; the best value depends on the interaction - of the input variables. + of the input variables. If None, then nodes are expanded until + all leaves are pure or until all leaves contain less than + min_samples_split samples. + If int, values must be in the range `[1, inf)`. min_impurity_decrease : float, default=0.0 A node will be split if this split induces a decrease of the impurity greater than or equal to this value. + Values must be in the range `[0.0, inf)`. The weighted impurity decrease equation is the following:: @@ -1411,42 +1849,29 @@ class GradientBoostingRegressor(RegressorMixin, BaseGradientBoosting): .. versionadded:: 0.19 - min_impurity_split : float, default=None - Threshold for early stopping in tree growth. A node will split - if its impurity is above the threshold, otherwise it is a leaf. - - .. deprecated:: 0.19 - ``min_impurity_split`` has been deprecated in favor of - ``min_impurity_decrease`` in 0.19. The default value of - ``min_impurity_split`` has changed from 1e-7 to 0 in 0.23 and it - will be removed in 1.0 (renaming of 0.25). - Use ``min_impurity_decrease`` instead. - init : estimator or 'zero', default=None An estimator object that is used to compute the initial predictions. ``init`` has to provide :term:`fit` and :term:`predict`. If 'zero', the initial raw predictions are set to zero. By default a ``DummyEstimator`` is used, predicting either the average target value - (for loss='ls'), or a quantile for the other losses. + (for loss='squared_error'), or a quantile for the other losses. random_state : int, RandomState instance or None, default=None Controls the random seed given to each Tree estimator at each boosting iteration. In addition, it controls the random permutation of the features at each split (see Notes for more details). - It also controls the random spliting of the training data to obtain a + It also controls the random splitting of the training data to obtain a validation set if `n_iter_no_change` is not None. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. - max_features : {'auto', 'sqrt', 'log2'}, int or float, default=None + max_features : {'sqrt', 'log2'}, int or float, default=None The number of features to consider when looking for the best split: - - If int, then consider `max_features` features at each split. - - If float, then `max_features` is a fraction and - `int(max_features * n_features)` features are considered at each - split. - - If "auto", then `max_features=n_features`. + - If int, values must be in the range `[1, inf)`. + - If float, values must be in the range `(0.0, 1.0]` and the features + considered at each split will be `max(1, int(max_features * n_features_in_))`. - If "sqrt", then `max_features=sqrt(n_features)`. - If "log2", then `max_features=log2(n_features)`. - If None, then `max_features=n_features`. @@ -1461,16 +1886,19 @@ class GradientBoostingRegressor(RegressorMixin, BaseGradientBoosting): alpha : float, default=0.9 The alpha-quantile of the huber loss function and the quantile loss function. Only if ``loss='huber'`` or ``loss='quantile'``. + Values must be in the range `(0.0, 1.0)`. verbose : int, default=0 Enable verbose output. If 1 then it prints progress and performance once in a while (the more trees the lower the frequency). If greater than 1 then it prints progress and performance for every tree. + Values must be in the range `[0, inf)`. max_leaf_nodes : int, default=None Grow trees with ``max_leaf_nodes`` in best-first fashion. Best nodes are defined as relative reduction in impurity. - If None then unlimited number of leaf nodes. + Values must be in the range `[2, inf)`. + If None, then unlimited number of leaf nodes. warm_start : bool, default=False When set to ``True``, reuse the solution of the previous call to fit @@ -1479,7 +1907,7 @@ class GradientBoostingRegressor(RegressorMixin, BaseGradientBoosting): validation_fraction : float, default=0.1 The proportion of training data to set aside as validation set for - early stopping. Must be between 0 and 1. + early stopping. Values must be in the range `(0.0, 1.0)`. Only used if ``n_iter_no_change`` is set to an integer. .. versionadded:: 0.20 @@ -1492,6 +1920,9 @@ class GradientBoostingRegressor(RegressorMixin, BaseGradientBoosting): data as validation and terminate training when validation score is not improving in all of the previous ``n_iter_no_change`` numbers of iterations. + Values must be in the range `[1, inf)`. + See + :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_early_stopping.py`. .. versionadded:: 0.20 @@ -1499,19 +1930,34 @@ class GradientBoostingRegressor(RegressorMixin, BaseGradientBoosting): Tolerance for the early stopping. When the loss is not improving by at least tol for ``n_iter_no_change`` iterations (if set to a number), the training stops. + Values must be in the range `[0.0, inf)`. .. versionadded:: 0.20 ccp_alpha : non-negative float, default=0.0 Complexity parameter used for Minimal Cost-Complexity Pruning. The subtree with the largest cost complexity that is smaller than - ``ccp_alpha`` will be chosen. By default, no pruning is performed. See - :ref:`minimal_cost_complexity_pruning` for details. + ``ccp_alpha`` will be chosen. By default, no pruning is performed. + Values must be in the range `[0.0, inf)`. + See :ref:`minimal_cost_complexity_pruning` for details. See + :ref:`sphx_glr_auto_examples_tree_plot_cost_complexity_pruning.py` + for an example of such pruning. .. versionadded:: 0.22 Attributes ---------- + n_estimators_ : int + The number of estimators as selected by early stopping (if + ``n_iter_no_change`` is specified). Otherwise it is set to + ``n_estimators``. + + n_trees_per_iteration_ : int + The number of trees that are built at each iteration. For regressors, this is + always 1. + + .. versionadded:: 1.4.0 + feature_importances_ : ndarray of shape (n_features,) The impurity-based feature importances. The higher, the more important the feature. @@ -1524,45 +1970,46 @@ class GradientBoostingRegressor(RegressorMixin, BaseGradientBoosting): :func:`sklearn.inspection.permutation_importance` as an alternative. oob_improvement_ : ndarray of shape (n_estimators,) - The improvement in loss (= deviance) on the out-of-bag samples + The improvement in loss on the out-of-bag samples relative to the previous iteration. ``oob_improvement_[0]`` is the improvement in loss of the first stage over the ``init`` estimator. - Only available if ``subsample < 1.0`` + Only available if ``subsample < 1.0``. + + oob_scores_ : ndarray of shape (n_estimators,) + The full history of the loss values on the out-of-bag + samples. Only available if `subsample < 1.0`. + + .. versionadded:: 1.3 + + oob_score_ : float + The last value of the loss on the out-of-bag samples. It is + the same as `oob_scores_[-1]`. Only available if `subsample < 1.0`. + + .. versionadded:: 1.3 train_score_ : ndarray of shape (n_estimators,) - The i-th score ``train_score_[i]`` is the deviance (= loss) of the + The i-th score ``train_score_[i]`` is the loss of the model at iteration ``i`` on the in-bag sample. - If ``subsample == 1`` this is the deviance on the training data. - - loss_ : LossFunction - The concrete ``LossFunction`` object. + If ``subsample == 1`` this is the loss on the training data. init_ : estimator - The estimator that provides the initial predictions. - Set via the ``init`` argument or ``loss.init_estimator``. + The estimator that provides the initial predictions. Set via the ``init`` + argument. estimators_ : ndarray of DecisionTreeRegressor of shape (n_estimators, 1) The collection of fitted sub-estimators. - n_classes_ : int - The number of classes, set to 1 for regressors. + n_features_in_ : int + Number of features seen during :term:`fit`. - .. deprecated:: 0.24 - Attribute ``n_classes_`` was deprecated in version 0.24 and - will be removed in 1.1 (renaming of 0.26). + .. versionadded:: 0.24 - n_estimators_ : int - The number of estimators as selected by early stopping (if - ``n_iter_no_change`` is specified). Otherwise it is set to - ``n_estimators``. - - n_features_ : int - The number of data features. + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. - .. deprecated:: 1.0 - Attribute `n_features_` was deprecated in version 1.0 and will be - removed in 1.2. Use `n_features_in_` instead. + .. versionadded:: 1.0 max_features_ : int The inferred value of max_features. @@ -1572,7 +2019,7 @@ class GradientBoostingRegressor(RegressorMixin, BaseGradientBoosting): HistGradientBoostingRegressor : Histogram-based Gradient Boosting Classification Tree. sklearn.tree.DecisionTreeRegressor : A decision tree regressor. - sklearn.tree.RandomForestRegressor : A random forest regressor. + sklearn.ensemble.RandomForestRegressor : A random forest regressor. Notes ----- @@ -1583,6 +2030,16 @@ class GradientBoostingRegressor(RegressorMixin, BaseGradientBoosting): split. To obtain a deterministic behaviour during fitting, ``random_state`` has to be fixed. + References + ---------- + J. Friedman, Greedy Function Approximation: A Gradient Boosting + Machine, The Annals of Statistics, Vol. 29, No. 5, 2001. + + J. Friedman, Stochastic Gradient Boosting, 1999 + + T. Hastie, R. Tibshirani and J. Friedman. + Elements of Statistical Learning Ed. 2, Springer, 2009. + Examples -------- >>> from sklearn.datasets import make_regression @@ -1595,58 +2052,83 @@ class GradientBoostingRegressor(RegressorMixin, BaseGradientBoosting): >>> reg.fit(X_train, y_train) GradientBoostingRegressor(random_state=0) >>> reg.predict(X_test[1:2]) - array([-61...]) + array([-61.1]) >>> reg.score(X_test, y_test) 0.4... - References - ---------- - J. Friedman, Greedy Function Approximation: A Gradient Boosting - Machine, The Annals of Statistics, Vol. 29, No. 5, 2001. - - J. Friedman, Stochastic Gradient Boosting, 1999 - - T. Hastie, R. Tibshirani and J. Friedman. - Elements of Statistical Learning Ed. 2, Springer, 2009. + For a detailed example of utilizing + :class:`~sklearn.ensemble.GradientBoostingRegressor` + to fit an ensemble of weak predictive models, please refer to + :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_regression.py`. """ - _SUPPORTED_LOSS = ('ls', 'lad', 'huber', 'quantile') - - @_deprecate_positional_args - def __init__(self, *, loss='ls', learning_rate=0.1, n_estimators=100, - subsample=1.0, criterion='friedman_mse', min_samples_split=2, - min_samples_leaf=1, min_weight_fraction_leaf=0., - max_depth=3, min_impurity_decrease=0., - min_impurity_split=None, init=None, random_state=None, - max_features=None, alpha=0.9, verbose=0, max_leaf_nodes=None, - warm_start=False, validation_fraction=0.1, - n_iter_no_change=None, tol=1e-4, ccp_alpha=0.0): - + _parameter_constraints: dict = { + **BaseGradientBoosting._parameter_constraints, + "loss": [StrOptions({"squared_error", "absolute_error", "huber", "quantile"})], + "init": [StrOptions({"zero"}), None, HasMethods(["fit", "predict"])], + "alpha": [Interval(Real, 0.0, 1.0, closed="neither")], + } + + def __init__( + self, + *, + loss="squared_error", + learning_rate=0.1, + n_estimators=100, + subsample=1.0, + criterion="friedman_mse", + min_samples_split=2, + min_samples_leaf=1, + min_weight_fraction_leaf=0.0, + max_depth=3, + min_impurity_decrease=0.0, + init=None, + random_state=None, + max_features=None, + alpha=0.9, + verbose=0, + max_leaf_nodes=None, + warm_start=False, + validation_fraction=0.1, + n_iter_no_change=None, + tol=1e-4, + ccp_alpha=0.0, + ): super().__init__( - loss=loss, learning_rate=learning_rate, n_estimators=n_estimators, - criterion=criterion, min_samples_split=min_samples_split, + loss=loss, + learning_rate=learning_rate, + n_estimators=n_estimators, + criterion=criterion, + min_samples_split=min_samples_split, min_samples_leaf=min_samples_leaf, min_weight_fraction_leaf=min_weight_fraction_leaf, - max_depth=max_depth, init=init, subsample=subsample, + max_depth=max_depth, + init=init, + subsample=subsample, max_features=max_features, min_impurity_decrease=min_impurity_decrease, - min_impurity_split=min_impurity_split, - random_state=random_state, alpha=alpha, verbose=verbose, - max_leaf_nodes=max_leaf_nodes, warm_start=warm_start, + random_state=random_state, + alpha=alpha, + verbose=verbose, + max_leaf_nodes=max_leaf_nodes, + warm_start=warm_start, validation_fraction=validation_fraction, - n_iter_no_change=n_iter_no_change, tol=tol, ccp_alpha=ccp_alpha) - - def _validate_y(self, y, sample_weight=None): - if y.dtype.kind == 'O': - y = y.astype(DOUBLE) + n_iter_no_change=n_iter_no_change, + tol=tol, + ccp_alpha=ccp_alpha, + ) + + def _encode_y(self, y=None, sample_weight=None): + # Just convert y to the expected dtype + self.n_trees_per_iteration_ = 1 + y = y.astype(DOUBLE, copy=False) return y - def _warn_mae_for_criterion(self): - # TODO: This should raise an error from 1.1 - warnings.warn("criterion='mae' was deprecated in version 0.24 and " - "will be removed in version 1.1 (renaming of 0.26). The " - "correct way of minimizing the absolute error is to use " - " loss='lad' instead.", FutureWarning) + def _get_loss(self, sample_weight): + if self.loss in ("quantile", "huber"): + return _LOSSES[self.loss](sample_weight=sample_weight, quantile=self.alpha) + else: + return _LOSSES[self.loss](sample_weight=sample_weight) def predict(self, X): """Predict regression target for X. @@ -1663,8 +2145,9 @@ def predict(self, X): y : ndarray of shape (n_samples,) The predicted values. """ - X = self._validate_data(X, dtype=DTYPE, order="C", accept_sparse='csr', - reset=False) + X = validate_data( + self, X, dtype=DTYPE, order="C", accept_sparse="csr", reset=False + ) # In regression we can directly return the raw value from the trees. return self._raw_predict(X).ravel() @@ -1681,8 +2164,8 @@ def staged_predict(self, X): ``dtype=np.float32`` and if a sparse matrix is provided to a sparse ``csr_matrix``. - Returns - ------- + Yields + ------ y : generator of ndarray of shape (n_samples,) The predicted value of the input samples. """ @@ -1711,19 +2194,3 @@ def apply(self, X): leaves = super().apply(X) leaves = leaves.reshape(X.shape[0], self.estimators_.shape[0]) return leaves - - # FIXME: to be removed in 1.1 - # mypy error: Decorated property not supported - @deprecated("Attribute n_classes_ was deprecated " # type: ignore - "in version 0.24 and will be removed in 1.1 " - "(renaming of 0.26).") - @property - def n_classes_(self): - try: - check_is_fitted(self) - except NotFittedError as nfe: - raise AttributeError( - "{} object has no n_classes_ attribute." - .format(self.__class__.__name__) - ) from nfe - return 1 diff --git a/sklearn/ensemble/_gb_losses.py b/sklearn/ensemble/_gb_losses.py deleted file mode 100644 index 82b496ae8109d..0000000000000 --- a/sklearn/ensemble/_gb_losses.py +++ /dev/null @@ -1,866 +0,0 @@ -"""Losses and corresponding default initial estimators for gradient boosting -decision trees. -""" - -from abc import ABCMeta -from abc import abstractmethod - -import numpy as np -from scipy.special import expit, logsumexp - -from ..tree._tree import TREE_LEAF -from ..utils.stats import _weighted_percentile -from ..dummy import DummyClassifier -from ..dummy import DummyRegressor - - -class LossFunction(metaclass=ABCMeta): - """Abstract base class for various loss functions. - - Parameters - ---------- - n_classes : int - Number of classes. - - Attributes - ---------- - K : int - The number of regression trees to be induced; - 1 for regression and binary classification; - ``n_classes`` for multi-class classification. - """ - - is_multi_class = False - - def __init__(self, n_classes): - self.K = n_classes - - def init_estimator(self): - """Default ``init`` estimator for loss function. """ - raise NotImplementedError() - - @abstractmethod - def __call__(self, y, raw_predictions, sample_weight=None): - """Compute the loss. - - Parameters - ---------- - y : ndarray of shape (n_samples,) - True labels. - - raw_predictions : ndarray of shape (n_samples, K) - The raw predictions (i.e. values from the tree leaves). - - sample_weight : ndarray of shape (n_samples,), default=None - Sample weights. - """ - - @abstractmethod - def negative_gradient(self, y, raw_predictions, **kargs): - """Compute the negative gradient. - - Parameters - ---------- - y : ndarray of shape (n_samples,) - The target labels. - - raw_predictions : ndarray of shape (n_samples, K) - The raw predictions (i.e. values from the tree leaves) of the - tree ensemble at iteration ``i - 1``. - """ - - def update_terminal_regions(self, tree, X, y, residual, raw_predictions, - sample_weight, sample_mask, - learning_rate=0.1, k=0): - """Update the terminal regions (=leaves) of the given tree and - updates the current predictions of the model. Traverses tree - and invokes template method `_update_terminal_region`. - - Parameters - ---------- - tree : tree.Tree - The tree object. - X : ndarray of shape (n_samples, n_features) - The data array. - y : ndarray of shape (n_samples,) - The target labels. - residual : ndarray of shape (n_samples,) - The residuals (usually the negative gradient). - raw_predictions : ndarray of shape (n_samples, K) - The raw predictions (i.e. values from the tree leaves) of the - tree ensemble at iteration ``i - 1``. - sample_weight : ndarray of shape (n_samples,) - The weight of each sample. - sample_mask : ndarray of shape (n_samples,) - The sample mask to be used. - learning_rate : float, default=0.1 - Learning rate shrinks the contribution of each tree by - ``learning_rate``. - k : int, default=0 - The index of the estimator being updated. - - """ - # compute leaf for each sample in ``X``. - terminal_regions = tree.apply(X) - - # mask all which are not in sample mask. - masked_terminal_regions = terminal_regions.copy() - masked_terminal_regions[~sample_mask] = -1 - - # update each leaf (= perform line search) - for leaf in np.where(tree.children_left == TREE_LEAF)[0]: - self._update_terminal_region(tree, masked_terminal_regions, - leaf, X, y, residual, - raw_predictions[:, k], sample_weight) - - # update predictions (both in-bag and out-of-bag) - raw_predictions[:, k] += \ - learning_rate * tree.value[:, 0, 0].take(terminal_regions, axis=0) - - @abstractmethod - def _update_terminal_region(self, tree, terminal_regions, leaf, X, y, - residual, raw_predictions, sample_weight): - """Template method for updating terminal regions (i.e., leaves).""" - - @abstractmethod - def get_init_raw_predictions(self, X, estimator): - """Return the initial raw predictions. - - Parameters - ---------- - X : ndarray of shape (n_samples, n_features) - The data array. - estimator : object - The estimator to use to compute the predictions. - - Returns - ------- - raw_predictions : ndarray of shape (n_samples, K) - The initial raw predictions. K is equal to 1 for binary - classification and regression, and equal to the number of classes - for multiclass classification. ``raw_predictions`` is casted - into float64. - """ - pass - - -class RegressionLossFunction(LossFunction, metaclass=ABCMeta): - """Base class for regression loss functions.""" - def __init__(self): - super().__init__(n_classes=1) - - def check_init_estimator(self, estimator): - """Make sure estimator has the required fit and predict methods. - - Parameters - ---------- - estimator : object - The init estimator to check. - """ - if not (hasattr(estimator, 'fit') and hasattr(estimator, 'predict')): - raise ValueError( - "The init parameter must be a valid estimator and " - "support both fit and predict." - ) - - def get_init_raw_predictions(self, X, estimator): - predictions = estimator.predict(X) - return predictions.reshape(-1, 1).astype(np.float64) - - -class LeastSquaresError(RegressionLossFunction): - """Loss function for least squares (LS) estimation. - Terminal regions do not need to be updated for least squares. - - Parameters - ---------- - n_classes : int - Number of classes. - """ - - def init_estimator(self): - return DummyRegressor(strategy='mean') - - def __call__(self, y, raw_predictions, sample_weight=None): - """Compute the least squares loss. - - Parameters - ---------- - y : ndarray of shape (n_samples,) - True labels. - - raw_predictions : ndarray of shape (n_samples, K) - The raw predictions (i.e. values from the tree leaves). - - sample_weight : ndarray of shape (n_samples,), default=None - Sample weights. - """ - if sample_weight is None: - return np.mean((y - raw_predictions.ravel()) ** 2) - else: - return (1 / sample_weight.sum() * np.sum( - sample_weight * ((y - raw_predictions.ravel()) ** 2))) - - def negative_gradient(self, y, raw_predictions, **kargs): - """Compute half of the negative gradient. - - Parameters - ---------- - y : ndarray of shape (n_samples,) - The target labels. - - raw_predictions : ndarray of shape (n_samples,) - The raw predictions (i.e. values from the tree leaves) of the - tree ensemble at iteration ``i - 1``. - """ - return y - raw_predictions.ravel() - - def update_terminal_regions(self, tree, X, y, residual, raw_predictions, - sample_weight, sample_mask, - learning_rate=0.1, k=0): - """Least squares does not need to update terminal regions. - - But it has to update the predictions. - - Parameters - ---------- - tree : tree.Tree - The tree object. - X : ndarray of shape (n_samples, n_features) - The data array. - y : ndarray of shape (n_samples,) - The target labels. - residual : ndarray of shape (n_samples,) - The residuals (usually the negative gradient). - raw_predictions : ndarray of shape (n_samples, K) - The raw predictions (i.e. values from the tree leaves) of the - tree ensemble at iteration ``i - 1``. - sample_weight : ndarray of shape (n,) - The weight of each sample. - sample_mask : ndarray of shape (n,) - The sample mask to be used. - learning_rate : float, default=0.1 - Learning rate shrinks the contribution of each tree by - ``learning_rate``. - k : int, default=0 - The index of the estimator being updated. - """ - # update predictions - raw_predictions[:, k] += learning_rate * tree.predict(X).ravel() - - def _update_terminal_region(self, tree, terminal_regions, leaf, X, y, - residual, raw_predictions, sample_weight): - pass - - -class LeastAbsoluteError(RegressionLossFunction): - """Loss function for least absolute deviation (LAD) regression. - - Parameters - ---------- - n_classes : int - Number of classes - """ - def init_estimator(self): - return DummyRegressor(strategy='quantile', quantile=.5) - - def __call__(self, y, raw_predictions, sample_weight=None): - """Compute the least absolute error. - - Parameters - ---------- - y : ndarray of shape (n_samples,) - True labels. - - raw_predictions : ndarray of shape (n_samples, K) - The raw predictions (i.e. values from the tree leaves). - - sample_weight : ndarray of shape (n_samples,), default=None - Sample weights. - """ - if sample_weight is None: - return np.abs(y - raw_predictions.ravel()).mean() - else: - return (1 / sample_weight.sum() * np.sum( - sample_weight * np.abs(y - raw_predictions.ravel()))) - - def negative_gradient(self, y, raw_predictions, **kargs): - """Compute the negative gradient. - - 1.0 if y - raw_predictions > 0.0 else -1.0 - - Parameters - ---------- - y : ndarray of shape (n_samples,) - The target labels. - - raw_predictions : ndarray of shape (n_samples, K) - The raw predictions (i.e. values from the tree leaves) of the - tree ensemble at iteration ``i - 1``. - """ - raw_predictions = raw_predictions.ravel() - return 2 * (y - raw_predictions > 0) - 1 - - def _update_terminal_region(self, tree, terminal_regions, leaf, X, y, - residual, raw_predictions, sample_weight): - """LAD updates terminal regions to median estimates.""" - terminal_region = np.where(terminal_regions == leaf)[0] - sample_weight = sample_weight.take(terminal_region, axis=0) - diff = (y.take(terminal_region, axis=0) - - raw_predictions.take(terminal_region, axis=0)) - tree.value[leaf, 0, 0] = _weighted_percentile(diff, sample_weight, - percentile=50) - - -class HuberLossFunction(RegressionLossFunction): - """Huber loss function for robust regression. - - M-Regression proposed in Friedman 2001. - - Parameters - ---------- - alpha : float, default=0.9 - Percentile at which to extract score. - - References - ---------- - J. Friedman, Greedy Function Approximation: A Gradient Boosting - Machine, The Annals of Statistics, Vol. 29, No. 5, 2001. - """ - - def __init__(self, alpha=0.9): - super().__init__() - self.alpha = alpha - self.gamma = None - - def init_estimator(self): - return DummyRegressor(strategy='quantile', quantile=.5) - - def __call__(self, y, raw_predictions, sample_weight=None): - """Compute the Huber loss. - - Parameters - ---------- - y : ndarray of shape (n_samples,) - True labels. - - raw_predictions : ndarray of shape (n_samples, K) - The raw predictions (i.e. values from the tree leaves) of the - tree ensemble. - - sample_weight : ndarray of shape (n_samples,), default=None - Sample weights. - """ - raw_predictions = raw_predictions.ravel() - diff = y - raw_predictions - gamma = self.gamma - if gamma is None: - if sample_weight is None: - gamma = np.percentile(np.abs(diff), self.alpha * 100) - else: - gamma = _weighted_percentile(np.abs(diff), sample_weight, - self.alpha * 100) - - gamma_mask = np.abs(diff) <= gamma - if sample_weight is None: - sq_loss = np.sum(0.5 * diff[gamma_mask] ** 2) - lin_loss = np.sum(gamma * (np.abs(diff[~gamma_mask]) - - gamma / 2)) - loss = (sq_loss + lin_loss) / y.shape[0] - else: - sq_loss = np.sum(0.5 * sample_weight[gamma_mask] * - diff[gamma_mask] ** 2) - lin_loss = np.sum(gamma * sample_weight[~gamma_mask] * - (np.abs(diff[~gamma_mask]) - gamma / 2)) - loss = (sq_loss + lin_loss) / sample_weight.sum() - return loss - - def negative_gradient(self, y, raw_predictions, sample_weight=None, - **kargs): - """Compute the negative gradient. - - Parameters - ---------- - y : ndarray of shape (n_samples,) - The target labels. - - raw_predictions : ndarray of shape (n_samples, K) - The raw predictions (i.e. values from the tree leaves) of the - tree ensemble at iteration ``i - 1``. - - sample_weight : ndarray of shape (n_samples,), default=None - Sample weights. - """ - raw_predictions = raw_predictions.ravel() - diff = y - raw_predictions - if sample_weight is None: - gamma = np.percentile(np.abs(diff), self.alpha * 100) - else: - gamma = _weighted_percentile(np.abs(diff), sample_weight, - self.alpha * 100) - gamma_mask = np.abs(diff) <= gamma - residual = np.zeros((y.shape[0],), dtype=np.float64) - residual[gamma_mask] = diff[gamma_mask] - residual[~gamma_mask] = gamma * np.sign(diff[~gamma_mask]) - self.gamma = gamma - return residual - - def _update_terminal_region(self, tree, terminal_regions, leaf, X, y, - residual, raw_predictions, sample_weight): - terminal_region = np.where(terminal_regions == leaf)[0] - sample_weight = sample_weight.take(terminal_region, axis=0) - gamma = self.gamma - diff = (y.take(terminal_region, axis=0) - - raw_predictions.take(terminal_region, axis=0)) - median = _weighted_percentile(diff, sample_weight, percentile=50) - diff_minus_median = diff - median - tree.value[leaf, 0] = median + np.mean( - np.sign(diff_minus_median) * - np.minimum(np.abs(diff_minus_median), gamma)) - - -class QuantileLossFunction(RegressionLossFunction): - """Loss function for quantile regression. - - Quantile regression allows to estimate the percentiles - of the conditional distribution of the target. - - Parameters - ---------- - alpha : float, default=0.9 - The percentile. - """ - def __init__(self, alpha=0.9): - super().__init__() - self.alpha = alpha - self.percentile = alpha * 100 - - def init_estimator(self): - return DummyRegressor(strategy='quantile', quantile=self.alpha) - - def __call__(self, y, raw_predictions, sample_weight=None): - """Compute the Quantile loss. - - Parameters - ---------- - y : ndarray of shape (n_samples,) - True labels. - - raw_predictions : ndarray of shape (n_samples, K) - The raw predictions (i.e. values from the tree leaves) of the - tree ensemble. - - sample_weight : ndarray of shape (n_samples,), default=None - Sample weights. - """ - raw_predictions = raw_predictions.ravel() - diff = y - raw_predictions - alpha = self.alpha - - mask = y > raw_predictions - if sample_weight is None: - loss = (alpha * diff[mask].sum() - - (1 - alpha) * diff[~mask].sum()) / y.shape[0] - else: - loss = ((alpha * np.sum(sample_weight[mask] * diff[mask]) - - (1 - alpha) * np.sum(sample_weight[~mask] * - diff[~mask])) / sample_weight.sum()) - return loss - - def negative_gradient(self, y, raw_predictions, **kargs): - """Compute the negative gradient. - - Parameters - ---------- - y : ndarray of shape (n_samples,) - The target labels. - - raw_predictions : ndarray of shape (n_samples, K) - The raw predictions (i.e. values from the tree leaves) of the - tree ensemble at iteration ``i - 1``. - """ - alpha = self.alpha - raw_predictions = raw_predictions.ravel() - mask = y > raw_predictions - return (alpha * mask) - ((1 - alpha) * ~mask) - - def _update_terminal_region(self, tree, terminal_regions, leaf, X, y, - residual, raw_predictions, sample_weight): - terminal_region = np.where(terminal_regions == leaf)[0] - diff = (y.take(terminal_region, axis=0) - - raw_predictions.take(terminal_region, axis=0)) - sample_weight = sample_weight.take(terminal_region, axis=0) - - val = _weighted_percentile(diff, sample_weight, self.percentile) - tree.value[leaf, 0] = val - - -class ClassificationLossFunction(LossFunction, metaclass=ABCMeta): - """Base class for classification loss functions. """ - - def _raw_prediction_to_proba(self, raw_predictions): - """Template method to convert raw predictions into probabilities. - - Parameters - ---------- - raw_predictions : ndarray of shape (n_samples, K) - The raw predictions (i.e. values from the tree leaves) of the - tree ensemble. - - Returns - ------- - probas : ndarray of shape (n_samples, K) - The predicted probabilities. - """ - - @abstractmethod - def _raw_prediction_to_decision(self, raw_predictions): - """Template method to convert raw predictions to decisions. - - Parameters - ---------- - raw_predictions : ndarray of shape (n_samples, K) - The raw predictions (i.e. values from the tree leaves) of the - tree ensemble. - - Returns - ------- - encoded_predictions : ndarray of shape (n_samples, K) - The predicted encoded labels. - """ - - def check_init_estimator(self, estimator): - """Make sure estimator has fit and predict_proba methods. - - Parameters - ---------- - estimator : object - The init estimator to check. - """ - if not (hasattr(estimator, 'fit') and - hasattr(estimator, 'predict_proba')): - raise ValueError( - "The init parameter must be a valid estimator " - "and support both fit and predict_proba." - ) - - -class BinomialDeviance(ClassificationLossFunction): - """Binomial deviance loss function for binary classification. - - Binary classification is a special case; here, we only need to - fit one tree instead of ``n_classes`` trees. - - Parameters - ---------- - n_classes : int - Number of classes. - """ - def __init__(self, n_classes): - if n_classes != 2: - raise ValueError("{0:s} requires 2 classes; got {1:d} class(es)" - .format(self.__class__.__name__, n_classes)) - # we only need to fit one tree for binary clf. - super().__init__(n_classes=1) - - def init_estimator(self): - # return the most common class, taking into account the samples - # weights - return DummyClassifier(strategy='prior') - - def __call__(self, y, raw_predictions, sample_weight=None): - """Compute the deviance (= 2 * negative log-likelihood). - - Parameters - ---------- - y : ndarray of shape (n_samples,) - True labels. - - raw_predictions : ndarray of shape (n_samples, K) - The raw predictions (i.e. values from the tree leaves) of the - tree ensemble. - - sample_weight : ndarray of shape (n_samples,), default=None - Sample weights. - """ - # logaddexp(0, v) == log(1.0 + exp(v)) - raw_predictions = raw_predictions.ravel() - if sample_weight is None: - return -2 * np.mean((y * raw_predictions) - - np.logaddexp(0, raw_predictions)) - else: - return (-2 / sample_weight.sum() * np.sum( - sample_weight * ((y * raw_predictions) - - np.logaddexp(0, raw_predictions)))) - - def negative_gradient(self, y, raw_predictions, **kargs): - """Compute half of the negative gradient. - - Parameters - ---------- - y : ndarray of shape (n_samples,) - True labels. - - raw_predictions : ndarray of shape (n_samples, K) - The raw predictions (i.e. values from the tree leaves) of the - tree ensemble at iteration ``i - 1``. - """ - return y - expit(raw_predictions.ravel()) - - def _update_terminal_region(self, tree, terminal_regions, leaf, X, y, - residual, raw_predictions, sample_weight): - """Make a single Newton-Raphson step. - - our node estimate is given by: - - sum(w * (y - prob)) / sum(w * prob * (1 - prob)) - - we take advantage that: y - prob = residual - """ - terminal_region = np.where(terminal_regions == leaf)[0] - residual = residual.take(terminal_region, axis=0) - y = y.take(terminal_region, axis=0) - sample_weight = sample_weight.take(terminal_region, axis=0) - - numerator = np.sum(sample_weight * residual) - denominator = np.sum(sample_weight * - (y - residual) * (1 - y + residual)) - - # prevents overflow and division by zero - if abs(denominator) < 1e-150: - tree.value[leaf, 0, 0] = 0.0 - else: - tree.value[leaf, 0, 0] = numerator / denominator - - def _raw_prediction_to_proba(self, raw_predictions): - proba = np.ones((raw_predictions.shape[0], 2), dtype=np.float64) - proba[:, 1] = expit(raw_predictions.ravel()) - proba[:, 0] -= proba[:, 1] - return proba - - def _raw_prediction_to_decision(self, raw_predictions): - proba = self._raw_prediction_to_proba(raw_predictions) - return np.argmax(proba, axis=1) - - def get_init_raw_predictions(self, X, estimator): - probas = estimator.predict_proba(X) - proba_pos_class = probas[:, 1] - eps = np.finfo(np.float32).eps - proba_pos_class = np.clip(proba_pos_class, eps, 1 - eps) - # log(x / (1 - x)) is the inverse of the sigmoid (expit) function - raw_predictions = np.log(proba_pos_class / (1 - proba_pos_class)) - return raw_predictions.reshape(-1, 1).astype(np.float64) - - -class MultinomialDeviance(ClassificationLossFunction): - """Multinomial deviance loss function for multi-class classification. - - For multi-class classification we need to fit ``n_classes`` trees at - each stage. - - Parameters - ---------- - n_classes : int - Number of classes. - """ - - is_multi_class = True - - def __init__(self, n_classes): - if n_classes < 3: - raise ValueError("{0:s} requires more than 2 classes.".format( - self.__class__.__name__)) - super().__init__(n_classes) - - def init_estimator(self): - return DummyClassifier(strategy='prior') - - def __call__(self, y, raw_predictions, sample_weight=None): - """Compute the Multinomial deviance. - - Parameters - ---------- - y : ndarray of shape (n_samples,) - True labels. - - raw_predictions : ndarray of shape (n_samples, K) - The raw predictions (i.e. values from the tree leaves) of the - tree ensemble. - - sample_weight : ndarray of shape (n_samples,), default=None - Sample weights. - """ - # create one-hot label encoding - Y = np.zeros((y.shape[0], self.K), dtype=np.float64) - for k in range(self.K): - Y[:, k] = y == k - - return np.average( - -1 * (Y * raw_predictions).sum(axis=1) + - logsumexp(raw_predictions, axis=1), - weights=sample_weight - ) - - def negative_gradient(self, y, raw_predictions, k=0, **kwargs): - """Compute negative gradient for the ``k``-th class. - - Parameters - ---------- - y : ndarray of shape (n_samples,) - The target labels. - - raw_predictions : ndarray of shape (n_samples, K) - The raw predictions (i.e. values from the tree leaves) of the - tree ensemble at iteration ``i - 1``. - - k : int, default=0 - The index of the class. - """ - return y - np.nan_to_num(np.exp(raw_predictions[:, k] - - logsumexp(raw_predictions, axis=1))) - - def _update_terminal_region(self, tree, terminal_regions, leaf, X, y, - residual, raw_predictions, sample_weight): - """Make a single Newton-Raphson step. """ - terminal_region = np.where(terminal_regions == leaf)[0] - residual = residual.take(terminal_region, axis=0) - y = y.take(terminal_region, axis=0) - sample_weight = sample_weight.take(terminal_region, axis=0) - - numerator = np.sum(sample_weight * residual) - numerator *= (self.K - 1) / self.K - - denominator = np.sum(sample_weight * (y - residual) * - (1 - y + residual)) - - # prevents overflow and division by zero - if abs(denominator) < 1e-150: - tree.value[leaf, 0, 0] = 0.0 - else: - tree.value[leaf, 0, 0] = numerator / denominator - - def _raw_prediction_to_proba(self, raw_predictions): - return np.nan_to_num( - np.exp(raw_predictions - - (logsumexp(raw_predictions, axis=1)[:, np.newaxis]))) - - def _raw_prediction_to_decision(self, raw_predictions): - proba = self._raw_prediction_to_proba(raw_predictions) - return np.argmax(proba, axis=1) - - def get_init_raw_predictions(self, X, estimator): - probas = estimator.predict_proba(X) - eps = np.finfo(np.float32).eps - probas = np.clip(probas, eps, 1 - eps) - raw_predictions = np.log(probas).astype(np.float64) - return raw_predictions - - -class ExponentialLoss(ClassificationLossFunction): - """Exponential loss function for binary classification. - - Same loss as AdaBoost. - - Parameters - ---------- - n_classes : int - Number of classes. - - References - ---------- - Greg Ridgeway, Generalized Boosted Models: A guide to the gbm package, 2007 - """ - def __init__(self, n_classes): - if n_classes != 2: - raise ValueError("{0:s} requires 2 classes; got {1:d} class(es)" - .format(self.__class__.__name__, n_classes)) - # we only need to fit one tree for binary clf. - super().__init__(n_classes=1) - - def init_estimator(self): - return DummyClassifier(strategy='prior') - - def __call__(self, y, raw_predictions, sample_weight=None): - """Compute the exponential loss - - Parameters - ---------- - y : ndarray of shape (n_samples,) - True labels. - - raw_predictions : ndarray of shape (n_samples, K) - The raw predictions (i.e. values from the tree leaves) of the - tree ensemble. - - sample_weight : ndarray of shape (n_samples,), default=None - Sample weights. - """ - raw_predictions = raw_predictions.ravel() - if sample_weight is None: - return np.mean(np.exp(-(2. * y - 1.) * raw_predictions)) - else: - return (1.0 / sample_weight.sum() * np.sum( - sample_weight * np.exp(-(2 * y - 1) * raw_predictions))) - - def negative_gradient(self, y, raw_predictions, **kargs): - """Compute the residual (= negative gradient). - - Parameters - ---------- - y : ndarray of shape (n_samples,) - True labels. - - raw_predictions : ndarray of shape (n_samples, K) - The raw predictions (i.e. values from the tree leaves) of the - tree ensemble at iteration ``i - 1``. - """ - y_ = -(2. * y - 1.) - return y_ * np.exp(y_ * raw_predictions.ravel()) - - def _update_terminal_region(self, tree, terminal_regions, leaf, X, y, - residual, raw_predictions, sample_weight): - terminal_region = np.where(terminal_regions == leaf)[0] - raw_predictions = raw_predictions.take(terminal_region, axis=0) - y = y.take(terminal_region, axis=0) - sample_weight = sample_weight.take(terminal_region, axis=0) - - y_ = 2. * y - 1. - - numerator = np.sum(y_ * sample_weight * np.exp(-y_ * raw_predictions)) - denominator = np.sum(sample_weight * np.exp(-y_ * raw_predictions)) - - # prevents overflow and division by zero - if abs(denominator) < 1e-150: - tree.value[leaf, 0, 0] = 0.0 - else: - tree.value[leaf, 0, 0] = numerator / denominator - - def _raw_prediction_to_proba(self, raw_predictions): - proba = np.ones((raw_predictions.shape[0], 2), dtype=np.float64) - proba[:, 1] = expit(2.0 * raw_predictions.ravel()) - proba[:, 0] -= proba[:, 1] - return proba - - def _raw_prediction_to_decision(self, raw_predictions): - return (raw_predictions.ravel() >= 0).astype(int) - - def get_init_raw_predictions(self, X, estimator): - probas = estimator.predict_proba(X) - proba_pos_class = probas[:, 1] - eps = np.finfo(np.float32).eps - proba_pos_class = np.clip(proba_pos_class, eps, 1 - eps) - # according to The Elements of Statistical Learning sec. 10.5, the - # minimizer of the exponential loss is .5 * log odds ratio. So this is - # the equivalent to .5 * binomial_deviance.get_init_raw_predictions() - raw_predictions = .5 * np.log(proba_pos_class / (1 - proba_pos_class)) - return raw_predictions.reshape(-1, 1).astype(np.float64) - - -LOSS_FUNCTIONS = { - 'ls': LeastSquaresError, - 'lad': LeastAbsoluteError, - 'huber': HuberLossFunction, - 'quantile': QuantileLossFunction, - 'deviance': None, # for both, multinomial and binomial - 'exponential': ExponentialLoss, -} diff --git a/sklearn/ensemble/_gradient_boosting.pyx b/sklearn/ensemble/_gradient_boosting.pyx index 2e335fec62705..cd9845a217c7d 100644 --- a/sklearn/ensemble/_gradient_boosting.pyx +++ b/sklearn/ensemble/_gradient_boosting.pyx @@ -1,53 +1,35 @@ -# cython: cdivision=True -# cython: boundscheck=False -# cython: wraparound=False -# -# Author: Peter Prettenhofer -# -# License: BSD 3 clause - -cimport cython +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause from libc.stdlib cimport free from libc.string cimport memset import numpy as np -cimport numpy as np -np.import_array() - from scipy.sparse import issparse -from scipy.sparse import csr_matrix +from ..utils._typedefs cimport float32_t, float64_t, intp_t, int32_t, uint8_t +# Note: _tree uses cimport numpy, cnp.import_array, so we need to include +# numpy headers in the build configuration of this extension from ..tree._tree cimport Node from ..tree._tree cimport Tree -from ..tree._tree cimport DTYPE_t -from ..tree._tree cimport SIZE_t -from ..tree._tree cimport INT32_t from ..tree._utils cimport safe_realloc -ctypedef np.int32_t int32 -ctypedef np.float64_t float64 -ctypedef np.uint8_t uint8 # no namespace lookup for numpy dtype and array creation from numpy import zeros as np_zeros -from numpy import ones as np_ones -from numpy import float32 as np_float32 -from numpy import float64 as np_float64 # constant to mark tree leafs -cdef SIZE_t TREE_LEAF = -1 - -cdef void _predict_regression_tree_inplace_fast_dense(DTYPE_t *X, - Node* root_node, - double *value, - double scale, - Py_ssize_t k, - Py_ssize_t K, - Py_ssize_t n_samples, - Py_ssize_t n_features, - float64 *out): +cdef intp_t TREE_LEAF = -1 + +cdef void _predict_regression_tree_inplace_fast_dense( + const float32_t[:, ::1] X, + Node* root_node, + double *value, + double scale, + Py_ssize_t k, + float64_t[:, :] out +) noexcept nogil: """Predicts output for regression tree and stores it in ``out[i, k]``. This function operates directly on the data arrays of the tree @@ -59,8 +41,8 @@ cdef void _predict_regression_tree_inplace_fast_dense(DTYPE_t *X, Parameters ---------- - X : DTYPE_t pointer - The pointer to the data array of the input ``X``. + X : float32_t 2d memory view + The memory view on the data ndarray of the input ``X``. Assumes that the array is c-continuous. root_node : tree Node pointer Pointer to the main node array of the :class:``sklearn.tree.Tree``. @@ -72,56 +54,49 @@ cdef void _predict_regression_tree_inplace_fast_dense(DTYPE_t *X, k : int The index of the tree output to be predicted. Must satisfy 0 <= ``k`` < ``K``. - K : int - The number of regression tree outputs. For regression and - binary classification ``K == 1``, for multi-class - classification ``K == n_classes``. - n_samples : int - The number of samples in the input array ``X``; - ``n_samples == X.shape[0]``. - n_features : int - The number of features; ``n_samples == X.shape[1]``. - out : np.float64_t pointer - The pointer to the data array where the predictions are stored. + out : memory view on array of type np.float64_t + The data array where the predictions are stored. ``out`` is assumed to be a two-dimensional array of shape ``(n_samples, K)``. """ + cdef intp_t n_samples = X.shape[0] cdef Py_ssize_t i cdef Node *node for i in range(n_samples): node = root_node # While node not a leaf while node.left_child != TREE_LEAF: - if X[i * n_features + node.feature] <= node.threshold: + if X[i, node.feature] <= node.threshold: node = root_node + node.left_child else: node = root_node + node.right_child - out[i * K + k] += scale * value[node - root_node] + out[i, k] += scale * value[node - root_node] + -def _predict_regression_tree_stages_sparse(np.ndarray[object, ndim=2] estimators, - object X, double scale, - np.ndarray[float64, ndim=2] out): +def _predict_regression_tree_stages_sparse( + object[:, :] estimators, + object X, + double scale, + float64_t[:, :] out +): """Predicts output for regression tree inplace and adds scaled value to ``out[i, k]``. The function assumes that the ndarray that wraps ``X`` is csr_matrix. """ - cdef DTYPE_t* X_data = ( X.data).data - cdef INT32_t* X_indices = ( X.indices).data - cdef INT32_t* X_indptr = ( X.indptr).data + cdef const float32_t[::1] X_data = X.data + cdef const int32_t[::1] X_indices = X.indices + cdef const int32_t[::1] X_indptr = X.indptr - cdef SIZE_t n_samples = X.shape[0] - cdef SIZE_t n_features = X.shape[1] - cdef SIZE_t n_stages = estimators.shape[0] - cdef SIZE_t n_outputs = estimators.shape[1] - - # Initialize output - cdef float64* out_ptr = out.data + cdef intp_t n_samples = X.shape[0] + cdef intp_t n_features = X.shape[1] + cdef intp_t n_stages = estimators.shape[0] + cdef intp_t n_outputs = estimators.shape[1] # Indices and temporary variables - cdef SIZE_t sample_i - cdef SIZE_t feature_i - cdef SIZE_t stage_i - cdef SIZE_t output_i + cdef intp_t sample_i + cdef intp_t feature_i + cdef intp_t stage_i + cdef intp_t output_i cdef Node *root_node = NULL cdef Node *node = NULL cdef double *value = NULL @@ -138,18 +113,18 @@ def _predict_regression_tree_stages_sparse(np.ndarray[object, ndim=2] estimators values[stage_i * n_outputs + output_i] = tree.value # Initialize auxiliary data-structure - cdef DTYPE_t feature_value = 0. - cdef DTYPE_t* X_sample = NULL + cdef float32_t feature_value = 0. + cdef float32_t* X_sample = NULL # feature_to_sample as a data structure records the last seen sample # for each feature; functionally, it is an efficient way to identify # which features are nonzero in the present sample. - cdef SIZE_t* feature_to_sample = NULL + cdef intp_t* feature_to_sample = NULL safe_realloc(&X_sample, n_features) safe_realloc(&feature_to_sample, n_features) - memset(feature_to_sample, -1, n_features * sizeof(SIZE_t)) + memset(feature_to_sample, -1, n_features * sizeof(intp_t)) # Cycle through all samples for sample_i in range(n_samples): @@ -177,8 +152,7 @@ def _predict_regression_tree_stages_sparse(np.ndarray[object, ndim=2] estimators node = root_node + node.left_child else: node = root_node + node.right_child - out_ptr[sample_i * n_outputs + output_i] += (scale - * value[node - root_node]) + out[sample_i, output_i] += scale * value[node - root_node] # Free auxiliary arrays free(X_sample) @@ -187,9 +161,12 @@ def _predict_regression_tree_stages_sparse(np.ndarray[object, ndim=2] estimators free(values) -def predict_stages(np.ndarray[object, ndim=2] estimators, - object X, double scale, - np.ndarray[float64, ndim=2] out): +def predict_stages( + object[:, :] estimators, + object X, + double scale, + float64_t[:, :] out +): """Add predictions of ``estimators`` to ``out``. Each estimator is scaled by ``scale`` before its prediction @@ -205,11 +182,12 @@ def predict_stages(np.ndarray[object, ndim=2] estimators, if X.format != 'csr': raise ValueError("When X is a sparse matrix, a CSR format is" " expected, got {!r}".format(type(X))) - _predict_regression_tree_stages_sparse(estimators, X, scale, out) + _predict_regression_tree_stages_sparse( + estimators=estimators, X=X, scale=scale, out=out + ) else: if not isinstance(X, np.ndarray) or np.isfortran(X): - raise ValueError("X should be C-ordered np.ndarray," - " got {}".format(type(X))) + raise ValueError(f"X should be C-ordered np.ndarray, got {type(X)}") for i in range(n_estimators): for k in range(K): @@ -219,57 +197,66 @@ def predict_stages(np.ndarray[object, ndim=2] estimators, # and get data pointer # need brackets because of casting operator priority _predict_regression_tree_inplace_fast_dense( - ( X).data, - tree.nodes, tree.value, - scale, k, K, X.shape[0], X.shape[1], - ( out).data) - ## out += scale * tree.predict(X).reshape((X.shape[0], 1)) - - -def predict_stage(np.ndarray[object, ndim=2] estimators, - int stage, - object X, double scale, - np.ndarray[float64, ndim=2] out): + X=X, + root_node=tree.nodes, + value=tree.value, + scale=scale, + k=k, + out=out + ) + # out[:, k] += scale * tree.predict(X).ravel() + + +def predict_stage( + object[:, :] estimators, + int stage, + object X, + double scale, + float64_t[:, :] out +): """Add predictions of ``estimators[stage]`` to ``out``. Each estimator in the stage is scaled by ``scale`` before its prediction is added to ``out``. """ - return predict_stages(estimators[stage:stage + 1], X, scale, out) + return predict_stages( + estimators=estimators[stage:stage + 1], X=X, scale=scale, out=out + ) -def _random_sample_mask(np.npy_intp n_total_samples, - np.npy_intp n_total_in_bag, random_state): - """Create a random sample mask where ``n_total_in_bag`` elements are set. +def _random_sample_mask( + intp_t n_total_samples, + intp_t n_total_in_bag, + random_state +): + """Create a random sample mask where ``n_total_in_bag`` elements are set. - Parameters - ---------- - n_total_samples : int - The length of the resulting mask. + Parameters + ---------- + n_total_samples : int + The length of the resulting mask. - n_total_in_bag : int - The number of elements in the sample mask which are set to 1. + n_total_in_bag : int + The number of elements in the sample mask which are set to 1. - random_state : RandomState - A numpy ``RandomState`` object. + random_state : RandomState + A numpy ``RandomState`` object. - Returns - ------- - sample_mask : np.ndarray, shape=[n_total_samples] - An ndarray where ``n_total_in_bag`` elements are set to ``True`` - the others are ``False``. - """ - cdef np.ndarray[float64, ndim=1, mode="c"] rand = \ - random_state.rand(n_total_samples) - cdef np.ndarray[uint8, ndim=1, mode="c", cast=True] sample_mask = \ - np_zeros((n_total_samples,), dtype=bool) + Returns + ------- + sample_mask : np.ndarray, shape=[n_total_samples] + An ndarray where ``n_total_in_bag`` elements are set to ``True`` + the others are ``False``. + """ + cdef float64_t[::1] rand = random_state.uniform(size=n_total_samples) + cdef uint8_t[::1] sample_mask = np_zeros((n_total_samples,), dtype=bool) - cdef np.npy_intp n_bagged = 0 - cdef np.npy_intp i = 0 + cdef intp_t n_bagged = 0 + cdef intp_t i = 0 - for i in range(n_total_samples): - if rand[i] * (n_total_samples - i) < (n_total_in_bag - n_bagged): - sample_mask[i] = 1 - n_bagged += 1 + for i in range(n_total_samples): + if rand[i] * (n_total_samples - i) < (n_total_in_bag - n_bagged): + sample_mask[i] = 1 + n_bagged += 1 - return sample_mask + return sample_mask.base diff --git a/sklearn/ensemble/_hist_gradient_boosting/__init__.py b/sklearn/ensemble/_hist_gradient_boosting/__init__.py index 879fae1189f87..5939d83c84838 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/__init__.py +++ b/sklearn/ensemble/_hist_gradient_boosting/__init__.py @@ -3,3 +3,6 @@ The implementation is a port from pygbm which is itself strongly inspired from LightGBM. """ + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause diff --git a/sklearn/ensemble/_hist_gradient_boosting/_binning.pyx b/sklearn/ensemble/_hist_gradient_boosting/_binning.pyx index fc564c9a126ab..f343ada64cdd0 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/_binning.pyx +++ b/sklearn/ensemble/_hist_gradient_boosting/_binning.pyx @@ -1,27 +1,18 @@ -# cython: cdivision=True -# cython: boundscheck=False -# cython: wraparound=False -# cython: nonecheck=False -# cython: language_level=3 +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -# Author: Nicolas Hug - -cimport cython - -import numpy as np -cimport numpy as np -from numpy.math cimport INFINITY from cython.parallel import prange from libc.math cimport isnan from .common cimport X_DTYPE_C, X_BINNED_DTYPE_C - -np.import_array() +from ...utils._typedefs cimport uint8_t def _map_to_bins(const X_DTYPE_C [:, :] data, list binning_thresholds, - const unsigned char missing_values_bin_idx, + const uint8_t[::1] is_categorical, + const uint8_t missing_values_bin_idx, + int n_threads, X_BINNED_DTYPE_C [::1, :] binned): """Bin continuous and categorical values to discrete integer-coded levels. @@ -35,6 +26,10 @@ def _map_to_bins(const X_DTYPE_C [:, :] data, binning_thresholds : list of arrays For each feature, stores the increasing numeric values that are used to separate the bins. + is_categorical : ndarray of uint8_t of shape (n_features,) + Indicates categorical features. + n_threads : int + Number of OpenMP threads to use. binned : ndarray, shape (n_samples, n_features) Output array, must be fortran aligned. """ @@ -42,16 +37,24 @@ def _map_to_bins(const X_DTYPE_C [:, :] data, int feature_idx for feature_idx in range(data.shape[1]): - _map_col_to_bins(data[:, feature_idx], - binning_thresholds[feature_idx], - missing_values_bin_idx, - binned[:, feature_idx]) + _map_col_to_bins( + data[:, feature_idx], + binning_thresholds[feature_idx], + is_categorical[feature_idx], + missing_values_bin_idx, + n_threads, + binned[:, feature_idx] + ) -cdef void _map_col_to_bins(const X_DTYPE_C [:] data, - const X_DTYPE_C [:] binning_thresholds, - const unsigned char missing_values_bin_idx, - X_BINNED_DTYPE_C [:] binned): +cdef void _map_col_to_bins( + const X_DTYPE_C [:] data, + const X_DTYPE_C [:] binning_thresholds, + const uint8_t is_categorical, + const uint8_t missing_values_bin_idx, + int n_threads, + X_BINNED_DTYPE_C [:] binned +): """Binary search to find the bin index for each value in the data.""" cdef: int i @@ -59,9 +62,14 @@ cdef void _map_col_to_bins(const X_DTYPE_C [:] data, int right int middle - for i in prange(data.shape[0], schedule='static', nogil=True): - - if isnan(data[i]): + for i in prange(data.shape[0], schedule='static', nogil=True, + num_threads=n_threads): + if ( + isnan(data[i]) or + # To follow LightGBM's conventions, negative values for + # categorical features are considered as missing values. + (is_categorical and data[i] < 0) + ): binned[i] = missing_values_bin_idx else: # for known values, use binary search diff --git a/sklearn/ensemble/_hist_gradient_boosting/_bitset.pxd b/sklearn/ensemble/_hist_gradient_boosting/_bitset.pxd index 0ea642df3ddcf..c44477cfa2300 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/_bitset.pxd +++ b/sklearn/ensemble/_hist_gradient_boosting/_bitset.pxd @@ -1,19 +1,20 @@ -# cython: language_level=3 from .common cimport X_BINNED_DTYPE_C from .common cimport BITSET_DTYPE_C from .common cimport BITSET_INNER_DTYPE_C from .common cimport X_DTYPE_C +from ...utils._typedefs cimport uint8_t -cdef void init_bitset(BITSET_DTYPE_C bitset) nogil -cdef void set_bitset(BITSET_DTYPE_C bitset, X_BINNED_DTYPE_C val) nogil +cdef void init_bitset(BITSET_DTYPE_C bitset) noexcept nogil -cdef unsigned char in_bitset(BITSET_DTYPE_C bitset, X_BINNED_DTYPE_C val) nogil +cdef void set_bitset(BITSET_DTYPE_C bitset, X_BINNED_DTYPE_C val) noexcept nogil -cpdef unsigned char in_bitset_memoryview(const BITSET_INNER_DTYPE_C[:] bitset, - X_BINNED_DTYPE_C val) nogil +cdef uint8_t in_bitset(BITSET_DTYPE_C bitset, X_BINNED_DTYPE_C val) noexcept nogil -cdef unsigned char in_bitset_2d_memoryview( - const BITSET_INNER_DTYPE_C [:, :] bitset, +cpdef uint8_t in_bitset_memoryview(const BITSET_INNER_DTYPE_C[:] bitset, + X_BINNED_DTYPE_C val) noexcept nogil + +cdef uint8_t in_bitset_2d_memoryview( + const BITSET_INNER_DTYPE_C[:, :] bitset, X_BINNED_DTYPE_C val, - unsigned int row) nogil + unsigned int row) noexcept nogil diff --git a/sklearn/ensemble/_hist_gradient_boosting/_bitset.pyx b/sklearn/ensemble/_hist_gradient_boosting/_bitset.pyx index 2df03b047aad1..cab20f7d5af05 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/_bitset.pyx +++ b/sklearn/ensemble/_hist_gradient_boosting/_bitset.pyx @@ -1,11 +1,8 @@ -# cython: cdivision=True -# cython: boundscheck=False -# cython: wraparound=False -# cython: language_level=3 from .common cimport BITSET_INNER_DTYPE_C from .common cimport BITSET_DTYPE_C from .common cimport X_DTYPE_C from .common cimport X_BINNED_DTYPE_C +from ...utils._typedefs cimport uint8_t # A bitset is a data structure used to represent sets of integers in [0, n]. We @@ -16,7 +13,7 @@ from .common cimport X_BINNED_DTYPE_C # https://en.wikipedia.org/wiki/Bitwise_operation -cdef inline void init_bitset(BITSET_DTYPE_C bitset) nogil: # OUT +cdef inline void init_bitset(BITSET_DTYPE_C bitset) noexcept nogil: # OUT cdef: unsigned int i @@ -25,24 +22,23 @@ cdef inline void init_bitset(BITSET_DTYPE_C bitset) nogil: # OUT cdef inline void set_bitset(BITSET_DTYPE_C bitset, # OUT - X_BINNED_DTYPE_C val) nogil: + X_BINNED_DTYPE_C val) noexcept nogil: bitset[val // 32] |= (1 << (val % 32)) -cdef inline unsigned char in_bitset(BITSET_DTYPE_C bitset, - X_BINNED_DTYPE_C val) nogil: - +cdef inline uint8_t in_bitset(BITSET_DTYPE_C bitset, + X_BINNED_DTYPE_C val) noexcept nogil: return (bitset[val // 32] >> (val % 32)) & 1 -cpdef inline unsigned char in_bitset_memoryview(const BITSET_INNER_DTYPE_C[:] bitset, - X_BINNED_DTYPE_C val) nogil: +cpdef inline uint8_t in_bitset_memoryview(const BITSET_INNER_DTYPE_C[:] bitset, + X_BINNED_DTYPE_C val) noexcept nogil: return (bitset[val // 32] >> (val % 32)) & 1 -cdef inline unsigned char in_bitset_2d_memoryview(const BITSET_INNER_DTYPE_C [:, :] bitset, - X_BINNED_DTYPE_C val, - unsigned int row) nogil: +cdef inline uint8_t in_bitset_2d_memoryview(const BITSET_INNER_DTYPE_C[:, :] bitset, + X_BINNED_DTYPE_C val, + unsigned int row) noexcept nogil: # Same as above but works on 2d memory views to avoid the creation of 1d # memory views. See https://github.com/scikit-learn/scikit-learn/issues/17299 return (bitset[row, val // 32] >> (val % 32)) & 1 diff --git a/sklearn/ensemble/_hist_gradient_boosting/_gradient_boosting.pyx b/sklearn/ensemble/_hist_gradient_boosting/_gradient_boosting.pyx index 18f1b6a365421..dcbbf733ebb51 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/_gradient_boosting.pyx +++ b/sklearn/ensemble/_hist_gradient_boosting/_gradient_boosting.pyx @@ -1,24 +1,18 @@ -# cython: cdivision=True -# cython: boundscheck=False -# cython: wraparound=False -# cython: language_level=3 +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -# Author: Nicolas Hug - -cimport cython from cython.parallel import prange import numpy as np -cimport numpy as np from .common import Y_DTYPE from .common cimport Y_DTYPE_C -np.import_array() - def _update_raw_predictions( Y_DTYPE_C [::1] raw_predictions, # OUT - grower): + grower, + n_threads, +): """Update raw_predictions with the predictions of the newest tree. This is equivalent to (and much faster than): @@ -42,7 +36,7 @@ def _update_raw_predictions( values = np.array([leaf.value for leaf in leaves], dtype=Y_DTYPE) _update_raw_predictions_helper(raw_predictions, starts, stops, partition, - values) + values, n_threads) cdef inline void _update_raw_predictions_helper( @@ -50,13 +44,16 @@ cdef inline void _update_raw_predictions_helper( const unsigned int [::1] starts, const unsigned int [::1] stops, const unsigned int [::1] partition, - const Y_DTYPE_C [::1] values): + const Y_DTYPE_C [::1] values, + int n_threads, +): cdef: unsigned int position int leaf_idx int n_leaves = starts.shape[0] - for leaf_idx in prange(n_leaves, schedule='static', nogil=True): + for leaf_idx in prange(n_leaves, schedule='static', nogil=True, + num_threads=n_threads): for position in range(starts[leaf_idx], stops[leaf_idx]): raw_predictions[partition[position]] += values[leaf_idx] diff --git a/sklearn/ensemble/_hist_gradient_boosting/_loss.pyx b/sklearn/ensemble/_hist_gradient_boosting/_loss.pyx deleted file mode 100644 index 4114cd24aa8df..0000000000000 --- a/sklearn/ensemble/_hist_gradient_boosting/_loss.pyx +++ /dev/null @@ -1,213 +0,0 @@ -# cython: cdivision=True -# cython: boundscheck=False -# cython: wraparound=False -# cython: language_level=3 - -# Author: Nicolas Hug - -cimport cython -from cython.parallel import prange -import numpy as np -cimport numpy as np - -from libc.math cimport exp, log - -from .common cimport Y_DTYPE_C -from .common cimport G_H_DTYPE_C - -np.import_array() - - -def _update_gradients_least_squares( - G_H_DTYPE_C [::1] gradients, # OUT - const Y_DTYPE_C [::1] y_true, # IN - const Y_DTYPE_C [::1] raw_predictions): # IN - - cdef: - int n_samples - int i - - n_samples = raw_predictions.shape[0] - for i in prange(n_samples, schedule='static', nogil=True): - # Note: a more correct expression is 2 * (raw_predictions - y_true) - # but since we use 1 for the constant hessian value (and not 2) this - # is strictly equivalent for the leaves values. - gradients[i] = raw_predictions[i] - y_true[i] - - -def _update_gradients_hessians_least_squares( - G_H_DTYPE_C [::1] gradients, # OUT - G_H_DTYPE_C [::1] hessians, # OUT - const Y_DTYPE_C [::1] y_true, # IN - const Y_DTYPE_C [::1] raw_predictions, # IN - const Y_DTYPE_C [::1] sample_weight): # IN - - cdef: - int n_samples - int i - - n_samples = raw_predictions.shape[0] - for i in prange(n_samples, schedule='static', nogil=True): - # Note: a more correct exp is 2 * (raw_predictions - y_true) * sample_weight - # but since we use 1 for the constant hessian value (and not 2) this - # is strictly equivalent for the leaves values. - gradients[i] = (raw_predictions[i] - y_true[i]) * sample_weight[i] - hessians[i] = sample_weight[i] - - -def _update_gradients_hessians_least_absolute_deviation( - G_H_DTYPE_C [::1] gradients, # OUT - G_H_DTYPE_C [::1] hessians, # OUT - const Y_DTYPE_C [::1] y_true, # IN - const Y_DTYPE_C [::1] raw_predictions, # IN - const Y_DTYPE_C [::1] sample_weight): # IN - - cdef: - int n_samples - int i - - n_samples = raw_predictions.shape[0] - for i in prange(n_samples, schedule='static', nogil=True): - # gradient = sign(raw_predicition - y_pred) * sample_weight - gradients[i] = sample_weight[i] * (2 * - (y_true[i] - raw_predictions[i] < 0) - 1) - hessians[i] = sample_weight[i] - - -def _update_gradients_least_absolute_deviation( - G_H_DTYPE_C [::1] gradients, # OUT - const Y_DTYPE_C [::1] y_true, # IN - const Y_DTYPE_C [::1] raw_predictions): # IN - - cdef: - int n_samples - int i - - n_samples = raw_predictions.shape[0] - for i in prange(n_samples, schedule='static', nogil=True): - # gradient = sign(raw_predicition - y_pred) - gradients[i] = 2 * (y_true[i] - raw_predictions[i] < 0) - 1 - - -def _update_gradients_hessians_poisson( - G_H_DTYPE_C [::1] gradients, # OUT - G_H_DTYPE_C [::1] hessians, # OUT - const Y_DTYPE_C [::1] y_true, # IN - const Y_DTYPE_C [::1] raw_predictions, # IN - const Y_DTYPE_C [::1] sample_weight): # IN - - cdef: - int n_samples - int i - Y_DTYPE_C y_pred - - n_samples = raw_predictions.shape[0] - if sample_weight is None: - for i in prange(n_samples, schedule='static', nogil=True): - # Note: We use only half of the deviance loss. Therefore, there is - # no factor of 2. - y_pred = exp(raw_predictions[i]) - gradients[i] = (y_pred - y_true[i]) - hessians[i] = y_pred - else: - for i in prange(n_samples, schedule='static', nogil=True): - # Note: We use only half of the deviance loss. Therefore, there is - # no factor of 2. - y_pred = exp(raw_predictions[i]) - gradients[i] = (y_pred - y_true[i]) * sample_weight[i] - hessians[i] = y_pred * sample_weight[i] - - -def _update_gradients_hessians_binary_crossentropy( - G_H_DTYPE_C [::1] gradients, # OUT - G_H_DTYPE_C [::1] hessians, # OUT - const Y_DTYPE_C [::1] y_true, # IN - const Y_DTYPE_C [::1] raw_predictions, # IN - const Y_DTYPE_C [::1] sample_weight): # IN - cdef: - int n_samples - Y_DTYPE_C p_i # proba that ith sample belongs to positive class - int i - - n_samples = raw_predictions.shape[0] - if sample_weight is None: - for i in prange(n_samples, schedule='static', nogil=True): - p_i = _cexpit(raw_predictions[i]) - gradients[i] = p_i - y_true[i] - hessians[i] = p_i * (1. - p_i) - else: - for i in prange(n_samples, schedule='static', nogil=True): - p_i = _cexpit(raw_predictions[i]) - gradients[i] = (p_i - y_true[i]) * sample_weight[i] - hessians[i] = p_i * (1. - p_i) * sample_weight[i] - - -def _update_gradients_hessians_categorical_crossentropy( - G_H_DTYPE_C [:, ::1] gradients, # OUT - G_H_DTYPE_C [:, ::1] hessians, # OUT - const Y_DTYPE_C [::1] y_true, # IN - const Y_DTYPE_C [:, ::1] raw_predictions, # IN - const Y_DTYPE_C [::1] sample_weight): # IN - cdef: - int prediction_dim = raw_predictions.shape[0] - int n_samples = raw_predictions.shape[1] - int k # class index - int i # sample index - Y_DTYPE_C sw - # p[i, k] is the probability that class(ith sample) == k. - # It's the softmax of the raw predictions - Y_DTYPE_C [:, ::1] p = np.empty(shape=(n_samples, prediction_dim)) - Y_DTYPE_C p_i_k - - if sample_weight is None: - for i in prange(n_samples, schedule='static', nogil=True): - # first compute softmaxes of sample i for each class - for k in range(prediction_dim): - p[i, k] = raw_predictions[k, i] # prepare softmax - _compute_softmax(p, i) - # then update gradients and hessians - for k in range(prediction_dim): - p_i_k = p[i, k] - gradients[k, i] = p_i_k - (y_true[i] == k) - hessians[k, i] = p_i_k * (1. - p_i_k) - else: - for i in prange(n_samples, schedule='static', nogil=True): - # first compute softmaxes of sample i for each class - for k in range(prediction_dim): - p[i, k] = raw_predictions[k, i] # prepare softmax - _compute_softmax(p, i) - # then update gradients and hessians - sw = sample_weight[i] - for k in range(prediction_dim): - p_i_k = p[i, k] - gradients[k, i] = (p_i_k - (y_true[i] == k)) * sw - hessians[k, i] = (p_i_k * (1. - p_i_k)) * sw - - -cdef inline void _compute_softmax(Y_DTYPE_C [:, ::1] p, const int i) nogil: - """Compute softmaxes of values in p[i, :].""" - # i needs to be passed (and stays constant) because otherwise Cython does - # not generate optimal code - - cdef: - Y_DTYPE_C max_value = p[i, 0] - Y_DTYPE_C sum_exps = 0. - unsigned int k - unsigned prediction_dim = p.shape[1] - - # Compute max value of array for numerical stability - for k in range(1, prediction_dim): - if max_value < p[i, k]: - max_value = p[i, k] - - for k in range(prediction_dim): - p[i, k] = exp(p[i, k] - max_value) - sum_exps += p[i, k] - - for k in range(prediction_dim): - p[i, k] /= sum_exps - - -cdef inline Y_DTYPE_C _cexpit(const Y_DTYPE_C x) nogil: - """Custom expit (logistic sigmoid function)""" - return 1. / (1. + exp(-x)) diff --git a/sklearn/ensemble/_hist_gradient_boosting/_predictor.pyx b/sklearn/ensemble/_hist_gradient_boosting/_predictor.pyx index d1bb98d82c936..8257fa974c4a0 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/_predictor.pyx +++ b/sklearn/ensemble/_hist_gradient_boosting/_predictor.pyx @@ -1,41 +1,34 @@ -# cython: cdivision=True -# cython: boundscheck=False -# cython: wraparound=False -# cython: language_level=3 +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -# Author: Nicolas Hug - -cimport cython from cython.parallel import prange from libc.math cimport isnan import numpy as np -cimport numpy as np -from numpy.math cimport INFINITY +from ...utils._typedefs cimport intp_t, uint8_t from .common cimport X_DTYPE_C from .common cimport Y_DTYPE_C from .common import Y_DTYPE from .common cimport X_BINNED_DTYPE_C from .common cimport BITSET_INNER_DTYPE_C -from .common cimport BITSET_DTYPE_C from .common cimport node_struct from ._bitset cimport in_bitset_2d_memoryview -np.import_array() - def _predict_from_raw_data( # raw data = non-binned data - node_struct [:] nodes, + const node_struct [:] nodes, const X_DTYPE_C [:, :] numeric_data, const BITSET_INNER_DTYPE_C [:, ::1] raw_left_cat_bitsets, const BITSET_INNER_DTYPE_C [:, ::1] known_cat_bitsets, const unsigned int [::1] f_idx_map, + int n_threads, Y_DTYPE_C [:] out): cdef: int i - for i in prange(numeric_data.shape[0], schedule='static', nogil=True): + for i in prange(numeric_data.shape[0], schedule='static', nogil=True, + num_threads=n_threads): out[i] = _predict_one_from_raw_data( nodes, numeric_data, raw_left_cat_bitsets, known_cat_bitsets, @@ -43,12 +36,12 @@ def _predict_from_raw_data( # raw data = non-binned data cdef inline Y_DTYPE_C _predict_one_from_raw_data( - node_struct [:] nodes, + const node_struct [:] nodes, const X_DTYPE_C [:, :] numeric_data, const BITSET_INNER_DTYPE_C [:, ::1] raw_left_cat_bitsets, const BITSET_INNER_DTYPE_C [:, ::1] known_cat_bitsets, const unsigned int [::1] f_idx_map, - const int row) nogil: + const int row) noexcept nogil: # Need to pass the whole array and the row index, else prange won't work. # See issue Cython #2798 @@ -69,7 +62,10 @@ cdef inline Y_DTYPE_C _predict_one_from_raw_data( else: node_idx = node.right elif node.is_categorical: - if in_bitset_2d_memoryview( + if data_val < 0: + # data_val is not in the accepted range, so it is treated as missing value + node_idx = node.left if node.missing_go_to_left else node.right + elif in_bitset_2d_memoryview( raw_left_cat_bitsets, data_val, node.bitset_idx): @@ -94,13 +90,15 @@ def _predict_from_binned_data( node_struct [:] nodes, const X_BINNED_DTYPE_C [:, :] binned_data, BITSET_INNER_DTYPE_C [:, :] binned_left_cat_bitsets, - const unsigned char missing_values_bin_idx, + const uint8_t missing_values_bin_idx, + int n_threads, Y_DTYPE_C [:] out): cdef: int i - for i in prange(binned_data.shape[0], schedule='static', nogil=True): + for i in prange(binned_data.shape[0], schedule='static', nogil=True, + num_threads=n_threads): out[i] = _predict_one_from_binned_data(nodes, binned_data, binned_left_cat_bitsets, i, @@ -112,7 +110,7 @@ cdef inline Y_DTYPE_C _predict_one_from_binned_data( const X_BINNED_DTYPE_C [:, :] binned_data, const BITSET_INNER_DTYPE_C [:, :] binned_left_cat_bitsets, const int row, - const unsigned char missing_values_bin_idx) nogil: + const uint8_t missing_values_bin_idx) noexcept nogil: # Need to pass the whole array and the row index, else prange won't work. # See issue Cython #2798 @@ -151,8 +149,9 @@ cdef inline Y_DTYPE_C _predict_one_from_binned_data( def _compute_partial_dependence( node_struct [:] nodes, const X_DTYPE_C [:, ::1] X, - int [:] target_features, - Y_DTYPE_C [:] out): + const intp_t [:] target_features, + Y_DTYPE_C [:] out +): """Partial dependence of the response on the ``target_features`` set. For each sample in ``X`` a tree traversal is performed. @@ -175,7 +174,7 @@ def _compute_partial_dependence( X : view on 2d ndarray, shape (n_samples, n_target_features) The grid points on which the partial dependence should be evaluated. - target_features : view on 1d ndarray, shape (n_target_features) + target_features : view on 1d ndarray of intp_t, shape (n_target_features) The set of target features for which the partial dependence should be evaluated. out : view on 1d ndarray, shape (n_samples) @@ -192,7 +191,7 @@ def _compute_partial_dependence( node_struct * current_node # pointer to avoid copying attributes unsigned int sample_idx - unsigned feature_idx + intp_t feature_idx unsigned stack_size Y_DTYPE_C left_sample_frac Y_DTYPE_C current_weight @@ -254,5 +253,4 @@ def _compute_partial_dependence( # Sanity check. Should never happen. if not (0.999 < total_weight < 1.001): - raise ValueError("Total weight should be 1.0 but was %.9f" % - total_weight) + raise ValueError("Total weight should be 1.0 but was %.9f" %total_weight) diff --git a/sklearn/ensemble/_hist_gradient_boosting/binning.py b/sklearn/ensemble/_hist_gradient_boosting/binning.py index fff215d410459..eee26e68842b7 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/binning.py +++ b/sklearn/ensemble/_hist_gradient_boosting/binning.py @@ -5,16 +5,20 @@ Bin thresholds are computed with the quantiles so that each bin contains approximately the same number of samples. """ -# Author: Nicolas Hug + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import numpy as np -from ...utils import check_random_state, check_array from ...base import BaseEstimator, TransformerMixin +from ...utils import check_array, check_random_state +from ...utils._openmp_helpers import _openmp_effective_n_threads +from ...utils.parallel import Parallel, delayed from ...utils.validation import check_is_fitted from ._binning import _map_to_bins -from .common import X_DTYPE, X_BINNED_DTYPE, ALMOST_INF, X_BITSET_INNER_DTYPE from ._bitset import set_bitset_memoryview +from .common import ALMOST_INF, X_BINNED_DTYPE, X_BITSET_INNER_DTYPE, X_DTYPE def _find_binning_thresholds(col_data, max_bins): @@ -43,21 +47,23 @@ def _find_binning_thresholds(col_data, max_bins): missing_mask = np.isnan(col_data) if missing_mask.any(): col_data = col_data[~missing_mask] - col_data = np.ascontiguousarray(col_data, dtype=X_DTYPE) - distinct_values = np.unique(col_data) + # The data will be sorted anyway in np.unique and again in percentile, so we do it + # here. Sorting also returns a contiguous array. + col_data = np.sort(col_data) + distinct_values = np.unique(col_data).astype(X_DTYPE) if len(distinct_values) <= max_bins: midpoints = distinct_values[:-1] + distinct_values[1:] - midpoints *= .5 + midpoints *= 0.5 else: - # We sort again the data in this case. We could compute - # approximate midpoint percentiles using the output of + # We could compute approximate midpoint percentiles using the output of # np.unique(col_data, return_counts) instead but this is more # work and the performance benefit will be limited because we # work on a fixed-size subsample of the full data. percentiles = np.linspace(0, 100, num=max_bins + 1) percentiles = percentiles[1:-1] - midpoints = np.percentile(col_data, percentiles, - interpolation='midpoint').astype(X_DTYPE) + midpoints = np.percentile(col_data, percentiles, method="midpoint").astype( + X_DTYPE + ) assert midpoints.shape[0] == max_bins - 1 # We avoid having +inf thresholds: +inf thresholds are only allowed in @@ -113,7 +119,12 @@ class _BinMapper(TransformerMixin, BaseEstimator): Pseudo-random number generator to control the random sub-sampling. Pass an int for reproducible output across multiple function calls. - See :term: `Glossary `. + See :term:`Glossary `. + n_threads : int, default=None + Number of OpenMP threads to use. `_openmp_effective_n_threads` is called + to determine the effective number of threads use, which takes cgroups CPU + quotes into account. See the docstring of `_openmp_effective_n_threads` + for details. Attributes ---------- @@ -138,17 +149,26 @@ class _BinMapper(TransformerMixin, BaseEstimator): missing_values_bin_idx_ : np.uint8 The index of the bin where missing values are mapped. This is a constant across all features. This corresponds to the last bin, and - it is always equal to ``n_bins - 1``. Note that if ``n_bins_missing_`` + it is always equal to ``n_bins - 1``. Note that if ``n_bins_non_missing_`` is less than ``n_bins - 1`` for a given feature, then there are empty (and unused) bins. """ - def __init__(self, n_bins=256, subsample=int(2e5), is_categorical=None, - known_categories=None, random_state=None): + + def __init__( + self, + n_bins=256, + subsample=int(2e5), + is_categorical=None, + known_categories=None, + random_state=None, + n_threads=None, + ): self.n_bins = n_bins self.subsample = subsample self.is_categorical = is_categorical self.known_categories = known_categories self.random_state = random_state + self.n_threads = n_threads def fit(self, X, y=None): """Fit data X by computing the binning thresholds. @@ -169,10 +189,13 @@ def fit(self, X, y=None): """ if not (3 <= self.n_bins <= 256): # min is 3: at least 2 distinct bins and a missing values bin - raise ValueError('n_bins={} should be no smaller than 3 ' - 'and no larger than 256.'.format(self.n_bins)) + raise ValueError( + "n_bins={} should be no smaller than 3 and no larger than 256.".format( + self.n_bins + ) + ) - X = check_array(X, dtype=[X_DTYPE], force_all_finite=False) + X = check_array(X, dtype=[X_DTYPE], ensure_all_finite=False) max_bins = self.n_bins - 1 rng = check_random_state(self.random_state) @@ -183,8 +206,7 @@ def fit(self, X, y=None): if self.is_categorical is None: self.is_categorical_ = np.zeros(X.shape[1], dtype=np.uint8) else: - self.is_categorical_ = np.asarray(self.is_categorical, - dtype=np.uint8) + self.is_categorical_ = np.asarray(self.is_categorical, dtype=np.uint8) n_features = X.shape[1] known_categories = self.known_categories @@ -202,30 +224,36 @@ def fit(self, X, y=None): if not is_categorical and known_cats is not None: raise ValueError( f"Feature {f_idx} isn't marked as a categorical feature, " - f"but categories were passed." + "but categories were passed." ) self.missing_values_bin_idx_ = self.n_bins - 1 - self.bin_thresholds_ = [] - n_bins_non_missing = [] + self.bin_thresholds_ = [None] * n_features + n_bins_non_missing = [None] * n_features + non_cat_thresholds = Parallel(n_jobs=self.n_threads, backend="threading")( + delayed(_find_binning_thresholds)(X[:, f_idx], max_bins) + for f_idx in range(n_features) + if not self.is_categorical_[f_idx] + ) + + non_cat_idx = 0 for f_idx in range(n_features): - if not self.is_categorical_[f_idx]: - thresholds = _find_binning_thresholds(X[:, f_idx], max_bins) - n_bins_non_missing.append(thresholds.shape[0] + 1) - else: + if self.is_categorical_[f_idx]: # Since categories are assumed to be encoded in # [0, n_cats] and since n_cats <= max_bins, # the thresholds *are* the unique categorical values. This will # lead to the correct mapping in transform() thresholds = known_categories[f_idx] - n_bins_non_missing.append(thresholds.shape[0]) - - self.bin_thresholds_.append(thresholds) + n_bins_non_missing[f_idx] = thresholds.shape[0] + self.bin_thresholds_[f_idx] = thresholds + else: + self.bin_thresholds_[f_idx] = non_cat_thresholds[non_cat_idx] + n_bins_non_missing[f_idx] = self.bin_thresholds_[f_idx].shape[0] + 1 + non_cat_idx += 1 - self.n_bins_non_missing_ = np.array(n_bins_non_missing, - dtype=np.uint32) + self.n_bins_non_missing_ = np.array(n_bins_non_missing, dtype=np.uint32) return self def transform(self, X): @@ -248,17 +276,24 @@ def transform(self, X): X_binned : array-like of shape (n_samples, n_features) The binned data (fortran-aligned). """ - X = check_array(X, dtype=[X_DTYPE], force_all_finite=False) + X = check_array(X, dtype=[X_DTYPE], ensure_all_finite=False) check_is_fitted(self) if X.shape[1] != self.n_bins_non_missing_.shape[0]: raise ValueError( - 'This estimator was fitted with {} features but {} got passed ' - 'to transform()'.format(self.n_bins_non_missing_.shape[0], - X.shape[1]) + "This estimator was fitted with {} features but {} got passed " + "to transform()".format(self.n_bins_non_missing_.shape[0], X.shape[1]) ) - binned = np.zeros_like(X, dtype=X_BINNED_DTYPE, order='F') - _map_to_bins(X, self.bin_thresholds_, self.missing_values_bin_idx_, - binned) + + n_threads = _openmp_effective_n_threads(self.n_threads) + binned = np.zeros_like(X, dtype=X_BINNED_DTYPE, order="F") + _map_to_bins( + X, + self.bin_thresholds_, + self.is_categorical_, + self.missing_values_bin_idx_, + n_threads, + binned, + ) return binned def make_known_categories_bitsets(self): @@ -280,18 +315,19 @@ def make_known_categories_bitsets(self): f_idx_map = np.zeros(n_features, dtype=np.uint32) f_idx_map[categorical_features_indices] = np.arange( - n_categorical_features, dtype=np.uint32) + n_categorical_features, dtype=np.uint32 + ) known_categories = self.bin_thresholds_ - known_cat_bitsets = np.zeros((n_categorical_features, 8), - dtype=X_BITSET_INNER_DTYPE) + known_cat_bitsets = np.zeros( + (n_categorical_features, 8), dtype=X_BITSET_INNER_DTYPE + ) # TODO: complexity is O(n_categorical_features * 255). Maybe this is # worth cythonizing for mapped_f_idx, f_idx in enumerate(categorical_features_indices): for raw_cat_val in known_categories[f_idx]: - set_bitset_memoryview(known_cat_bitsets[mapped_f_idx], - raw_cat_val) + set_bitset_memoryview(known_cat_bitsets[mapped_f_idx], raw_cat_val) return known_cat_bitsets, f_idx_map diff --git a/sklearn/ensemble/_hist_gradient_boosting/common.pxd b/sklearn/ensemble/_hist_gradient_boosting/common.pxd index 6122b961fb91c..9ff9fc89800d7 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/common.pxd +++ b/sklearn/ensemble/_hist_gradient_boosting/common.pxd @@ -1,17 +1,14 @@ -# cython: language_level=3 -import numpy as np -cimport numpy as np +from ...utils._typedefs cimport float32_t, float64_t, intp_t, uint8_t, uint32_t -np.import_array() - -ctypedef np.npy_float64 X_DTYPE_C -ctypedef np.npy_uint8 X_BINNED_DTYPE_C -ctypedef np.npy_float64 Y_DTYPE_C -ctypedef np.npy_float32 G_H_DTYPE_C -ctypedef np.npy_uint32 BITSET_INNER_DTYPE_C +ctypedef float64_t X_DTYPE_C +ctypedef uint8_t X_BINNED_DTYPE_C +ctypedef float64_t Y_DTYPE_C +ctypedef float32_t G_H_DTYPE_C +ctypedef uint32_t BITSET_INNER_DTYPE_C ctypedef BITSET_INNER_DTYPE_C[8] BITSET_DTYPE_C + cdef packed struct hist_struct: # Same as histogram dtype but we need a struct to declare views. It needs # to be packed since by default numpy dtypes aren't aligned @@ -25,20 +22,21 @@ cdef packed struct node_struct: # needs to be packed since by default numpy dtypes aren't aligned Y_DTYPE_C value unsigned int count - unsigned int feature_idx + intp_t feature_idx X_DTYPE_C num_threshold - unsigned char missing_go_to_left + uint8_t missing_go_to_left unsigned int left unsigned int right Y_DTYPE_C gain unsigned int depth - unsigned char is_leaf + uint8_t is_leaf X_BINNED_DTYPE_C bin_threshold - unsigned char is_categorical + uint8_t is_categorical # The index of the corresponding bitsets in the Predictor's bitset arrays. # Only used if is_categorical is True unsigned int bitset_idx + cpdef enum MonotonicConstraint: NO_CST = 0 POS = 1 diff --git a/sklearn/ensemble/_hist_gradient_boosting/common.pyx b/sklearn/ensemble/_hist_gradient_boosting/common.pyx index f7b36f5796508..6b20e32813d5b 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/common.pyx +++ b/sklearn/ensemble/_hist_gradient_boosting/common.pyx @@ -10,6 +10,13 @@ X_BINNED_DTYPE = np.uint8 # hence max_bins == 256 G_H_DTYPE = np.float32 X_BITSET_INNER_DTYPE = np.uint32 +# Note that we use Y_DTYPE=float64 to avoid issues with floating point precision when +# summing gradients and hessians (both float32). Those are difficult to protect via +# tools like (Kahan-) Neumaier summation as in CPython, see +# https://github.com/python/cpython/issues/100425, or pairwise summation as numpy, see +# https://github.com/numpy/numpy/pull/3685, due to the way histograms are summed +# (number of additions per bin is not known in advance). See also comment in +# _subtract_histograms. HISTOGRAM_DTYPE = np.dtype([ ('sum_gradients', Y_DTYPE), # sum of sample gradients in bin ('sum_hessians', Y_DTYPE), # sum of sample hessians in bin @@ -19,7 +26,7 @@ HISTOGRAM_DTYPE = np.dtype([ PREDICTOR_RECORD_DTYPE = np.dtype([ ('value', Y_DTYPE), ('count', np.uint32), - ('feature_idx', np.uint32), + ('feature_idx', np.intp), ('num_threshold', X_DTYPE), ('missing_go_to_left', np.uint8), ('left', np.uint32), diff --git a/sklearn/ensemble/_hist_gradient_boosting/gradient_boosting.py b/sklearn/ensemble/_hist_gradient_boosting/gradient_boosting.py index 4fff6030b0d5a..064391abab24d 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/gradient_boosting.py +++ b/sklearn/ensemble/_hist_gradient_boosting/gradient_boosting.py @@ -1,41 +1,203 @@ """Fast Gradient Boosting decision trees for classification and regression.""" -# Author: Nicolas Hug +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import itertools from abc import ABC, abstractmethod +from contextlib import contextmanager, nullcontext, suppress from functools import partial +from numbers import Integral, Real +from time import time import numpy as np -from timeit import default_timer as time -from ...base import (BaseEstimator, RegressorMixin, ClassifierMixin, - is_classifier) -from ...utils import check_random_state, resample -from ...utils.validation import (check_is_fitted, - check_consistent_length, - _check_sample_weight, - _deprecate_positional_args) -from ...utils.multiclass import check_classification_targets + +from ..._loss.loss import ( + _LOSSES, + BaseLoss, + HalfBinomialLoss, + HalfGammaLoss, + HalfMultinomialLoss, + HalfPoissonLoss, + PinballLoss, +) +from ...base import ( + BaseEstimator, + ClassifierMixin, + RegressorMixin, + _fit_context, + is_classifier, +) +from ...compose import ColumnTransformer from ...metrics import check_scoring +from ...metrics._scorer import _SCORERS from ...model_selection import train_test_split -from ...preprocessing import LabelEncoder +from ...preprocessing import FunctionTransformer, LabelEncoder, OrdinalEncoder +from ...utils import check_random_state, compute_sample_weight, resample +from ...utils._missing import is_scalar_nan +from ...utils._openmp_helpers import _openmp_effective_n_threads +from ...utils._param_validation import Interval, RealNotInt, StrOptions +from ...utils.multiclass import check_classification_targets +from ...utils.validation import ( + _check_monotonic_cst, + _check_sample_weight, + _check_y, + _is_pandas_df, + check_array, + check_consistent_length, + check_is_fitted, + validate_data, +) from ._gradient_boosting import _update_raw_predictions -from .common import Y_DTYPE, X_DTYPE, X_BINNED_DTYPE - from .binning import _BinMapper +from .common import G_H_DTYPE, X_DTYPE, Y_DTYPE from .grower import TreeGrower -from .loss import _LOSSES -from .loss import BaseLoss + +_LOSSES = _LOSSES.copy() +_LOSSES.update( + { + "poisson": HalfPoissonLoss, + "gamma": HalfGammaLoss, + "quantile": PinballLoss, + } +) + + +def _update_leaves_values(loss, grower, y_true, raw_prediction, sample_weight): + """Update the leaf values to be predicted by the tree. + + Update equals: + loss.fit_intercept_only(y_true - raw_prediction) + + This is only applied if loss.differentiable is False. + Note: It only works, if the loss is a function of the residual, as is the + case for AbsoluteError and PinballLoss. Otherwise, one would need to get + the minimum of loss(y_true, raw_prediction + x) in x. A few examples: + - AbsoluteError: median(y_true - raw_prediction). + - PinballLoss: quantile(y_true - raw_prediction). + + More background: + For the standard gradient descent method according to "Greedy Function + Approximation: A Gradient Boosting Machine" by Friedman, all loss functions but the + squared loss need a line search step. BaseHistGradientBoosting, however, implements + a so called Newton boosting where the trees are fitted to a 2nd order + approximations of the loss in terms of gradients and hessians. In this case, the + line search step is only necessary if the loss is not smooth, i.e. not + differentiable, which renders the 2nd order approximation invalid. In fact, + non-smooth losses arbitrarily set hessians to 1 and effectively use the standard + gradient descent method with line search. + """ + # TODO: Ideally this should be computed in parallel over the leaves using something + # similar to _update_raw_predictions(), but this requires a cython version of + # median(). + for leaf in grower.finalized_leaves: + indices = leaf.sample_indices + if sample_weight is None: + sw = None + else: + sw = sample_weight[indices] + update = loss.fit_intercept_only( + y_true=y_true[indices] - raw_prediction[indices], + sample_weight=sw, + ) + leaf.value = grower.shrinkage * update + # Note that the regularization is ignored here + + +@contextmanager +def _patch_raw_predict(estimator, raw_predictions): + """Context manager that patches _raw_predict to return raw_predictions. + + `raw_predictions` is typically a precomputed array to avoid redundant + state-wise computations fitting with early stopping enabled: in this case + `raw_predictions` is incrementally updated whenever we add a tree to the + boosted ensemble. + + Note: this makes fitting HistGradientBoosting* models inherently non thread + safe at fit time. However thread-safety at fit time was never guaranteed nor + enforced for scikit-learn estimators in general. + + Thread-safety at prediction/transform time is another matter as those + operations are typically side-effect free and therefore often thread-safe by + default for most scikit-learn models and would like to keep it that way. + Therefore this context manager should only be used at fit time. + + TODO: in the future, we could explore the possibility to extend the scorer + public API to expose a way to compute vales from raw predictions. That would + probably require also making the scorer aware of the inverse link function + used by the estimator which is typically private API for now, hence the need + for this patching mechanism. + """ + orig_raw_predict = estimator._raw_predict + + def _patched_raw_predicts(*args, **kwargs): + return raw_predictions + + estimator._raw_predict = _patched_raw_predicts + yield estimator + estimator._raw_predict = orig_raw_predict class BaseHistGradientBoosting(BaseEstimator, ABC): """Base class for histogram-based gradient boosting estimators.""" + _parameter_constraints: dict = { + "loss": [BaseLoss], + "learning_rate": [Interval(Real, 0, None, closed="neither")], + "max_iter": [Interval(Integral, 1, None, closed="left")], + "max_leaf_nodes": [Interval(Integral, 2, None, closed="left"), None], + "max_depth": [Interval(Integral, 1, None, closed="left"), None], + "min_samples_leaf": [Interval(Integral, 1, None, closed="left")], + "l2_regularization": [Interval(Real, 0, None, closed="left")], + "max_features": [Interval(RealNotInt, 0, 1, closed="right")], + "monotonic_cst": ["array-like", dict, None], + "interaction_cst": [ + list, + tuple, + StrOptions({"pairwise", "no_interactions"}), + None, + ], + "n_iter_no_change": [Interval(Integral, 1, None, closed="left")], + "validation_fraction": [ + Interval(RealNotInt, 0, 1, closed="neither"), + Interval(Integral, 1, None, closed="left"), + None, + ], + "tol": [Interval(Real, 0, None, closed="left")], + "max_bins": [Interval(Integral, 2, 255, closed="both")], + "categorical_features": ["array-like", StrOptions({"from_dtype"}), None], + "warm_start": ["boolean"], + "early_stopping": [StrOptions({"auto"}), "boolean"], + "scoring": [str, callable, None], + "verbose": ["verbose"], + "random_state": ["random_state"], + } + @abstractmethod - def __init__(self, loss, *, learning_rate, max_iter, max_leaf_nodes, - max_depth, min_samples_leaf, l2_regularization, max_bins, - categorical_features, monotonic_cst, - warm_start, early_stopping, scoring, - validation_fraction, n_iter_no_change, tol, verbose, - random_state): + def __init__( + self, + loss, + *, + learning_rate, + max_iter, + max_leaf_nodes, + max_depth, + min_samples_leaf, + l2_regularization, + max_features, + max_bins, + categorical_features, + monotonic_cst, + interaction_cst, + warm_start, + early_stopping, + scoring, + validation_fraction, + n_iter_no_change, + tol, + verbose, + random_state, + ): self.loss = loss self.learning_rate = learning_rate self.max_iter = max_iter @@ -43,8 +205,10 @@ def __init__(self, loss, *, learning_rate, max_iter, max_leaf_nodes, self.max_depth = max_depth self.min_samples_leaf = min_samples_leaf self.l2_regularization = l2_regularization + self.max_features = max_features self.max_bins = max_bins self.monotonic_cst = monotonic_cst + self.interaction_cst = interaction_cst self.categorical_features = categorical_features self.warm_start = warm_start self.early_stopping = early_stopping @@ -60,120 +224,300 @@ def _validate_parameters(self): The parameters that are directly passed to the grower are checked in TreeGrower.""" - - if (self.loss not in self._VALID_LOSSES and - not isinstance(self.loss, BaseLoss)): - raise ValueError( - "Loss {} is not supported for {}. Accepted losses: " - "{}.".format(self.loss, self.__class__.__name__, - ', '.join(self._VALID_LOSSES))) - - if self.learning_rate <= 0: - raise ValueError('learning_rate={} must ' - 'be strictly positive'.format(self.learning_rate)) - if self.max_iter < 1: - raise ValueError('max_iter={} must not be smaller ' - 'than 1.'.format(self.max_iter)) - if self.n_iter_no_change < 0: - raise ValueError('n_iter_no_change={} must be ' - 'positive.'.format(self.n_iter_no_change)) - if (self.validation_fraction is not None and - self.validation_fraction <= 0): + if self.monotonic_cst is not None and self.n_trees_per_iteration_ != 1: raise ValueError( - 'validation_fraction={} must be strictly ' - 'positive, or None.'.format(self.validation_fraction)) - if self.tol < 0: - raise ValueError('tol={} ' - 'must not be smaller than 0.'.format(self.tol)) + "monotonic constraints are not supported for multiclass classification." + ) - if not (2 <= self.max_bins <= 255): - raise ValueError('max_bins={} should be no smaller than 2 ' - 'and no larger than 255.'.format(self.max_bins)) + def _finalize_sample_weight(self, sample_weight, y): + """Finalize sample weight. - if self.monotonic_cst is not None and self.n_trees_per_iteration_ != 1: - raise ValueError( - 'monotonic constraints are not supported for ' - 'multiclass classification.' + Used by subclasses to adjust sample_weights. This is useful for implementing + class weights. + """ + return sample_weight + + def _preprocess_X(self, X, *, reset): + """Preprocess and validate X. + + Parameters + ---------- + X : {array-like, pandas DataFrame} of shape (n_samples, n_features) + Input data. + + reset : bool + Whether to reset the `n_features_in_` and `feature_names_in_ attributes. + + Returns + ------- + X : ndarray of shape (n_samples, n_features) + Validated input data. + + known_categories : list of ndarray of shape (n_categories,) + List of known categories for each categorical feature. + """ + # If there is a preprocessor, we let the preprocessor handle the validation. + # Otherwise, we validate the data ourselves. + check_X_kwargs = dict(dtype=[X_DTYPE], ensure_all_finite=False) + if not reset: + if self._preprocessor is None: + return validate_data(self, X, reset=False, **check_X_kwargs) + return self._preprocessor.transform(X) + + # At this point, reset is False, which runs during `fit`. + self.is_categorical_ = self._check_categorical_features(X) + + if self.is_categorical_ is None: + self._preprocessor = None + self._is_categorical_remapped = None + + X = validate_data(self, X, **check_X_kwargs) + return X, None + + n_features = X.shape[1] + ordinal_encoder = OrdinalEncoder( + categories="auto", + handle_unknown="use_encoded_value", + unknown_value=np.nan, + encoded_missing_value=np.nan, + dtype=X_DTYPE, + ) + + check_X = partial(check_array, **check_X_kwargs) + numerical_preprocessor = FunctionTransformer(check_X) + self._preprocessor = ColumnTransformer( + [ + ("encoder", ordinal_encoder, self.is_categorical_), + ("numerical", numerical_preprocessor, ~self.is_categorical_), + ] + ) + self._preprocessor.set_output(transform="default") + X = self._preprocessor.fit_transform(X) + # check categories found by the OrdinalEncoder and get their encoded values + known_categories = self._check_categories() + self.n_features_in_ = self._preprocessor.n_features_in_ + with suppress(AttributeError): + self.feature_names_in_ = self._preprocessor.feature_names_in_ + + # The ColumnTransformer's output places the categorical features at the + # beginning + categorical_remapped = np.zeros(n_features, dtype=bool) + categorical_remapped[self._preprocessor.output_indices_["encoder"]] = True + self._is_categorical_remapped = categorical_remapped + + return X, known_categories + + def _check_categories(self): + """Check categories found by the preprocessor and return their encoded values. + + Returns a list of length ``self.n_features_in_``, with one entry per + input feature. + + For non-categorical features, the corresponding entry is ``None``. + + For categorical features, the corresponding entry is an array + containing the categories as encoded by the preprocessor (an + ``OrdinalEncoder``), excluding missing values. The entry is therefore + ``np.arange(n_categories)`` where ``n_categories`` is the number of + unique values in the considered feature column, after removing missing + values. + + If ``n_categories > self.max_bins`` for any feature, a ``ValueError`` + is raised. + """ + encoder = self._preprocessor.named_transformers_["encoder"] + known_categories = [None] * self._preprocessor.n_features_in_ + categorical_column_indices = np.arange(self._preprocessor.n_features_in_)[ + self._preprocessor.output_indices_["encoder"] + ] + for feature_idx, categories in zip( + categorical_column_indices, encoder.categories_ + ): + # OrdinalEncoder always puts np.nan as the last category if the + # training data has missing values. Here we remove it because it is + # already added by the _BinMapper. + if len(categories) and is_scalar_nan(categories[-1]): + categories = categories[:-1] + if categories.size > self.max_bins: + try: + feature_name = repr(encoder.feature_names_in_[feature_idx]) + except AttributeError: + feature_name = f"at index {feature_idx}" + raise ValueError( + f"Categorical feature {feature_name} is expected to " + f"have a cardinality <= {self.max_bins} but actually " + f"has a cardinality of {categories.size}." ) + known_categories[feature_idx] = np.arange(len(categories), dtype=X_DTYPE) + return known_categories - def _check_categories(self, X): + def _check_categorical_features(self, X): """Check and validate categorical features in X + Parameters + ---------- + X : {array-like, pandas DataFrame} of shape (n_samples, n_features) + Input data. + Return ------ is_categorical : ndarray of shape (n_features,) or None, dtype=bool Indicates whether a feature is categorical. If no feature is categorical, this is None. - known_categories : list of size n_features or None - The list contains, for each feature: - - an array of shape (n_categories,) with the unique cat values - - None if the feature is not categorical - None if no feature is categorical. """ - if self.categorical_features is None: - return None, None + # Special code for pandas because of a bug in recent pandas, which is + # fixed in main and maybe included in 2.2.1, see + # https://github.com/pandas-dev/pandas/pull/57173. + # Also pandas versions < 1.5.1 do not support the dataframe interchange + if _is_pandas_df(X): + X_is_dataframe = True + categorical_columns_mask = np.asarray(X.dtypes == "category") + elif hasattr(X, "__dataframe__"): + X_is_dataframe = True + categorical_columns_mask = np.asarray( + [ + c.dtype[0].name == "CATEGORICAL" + for c in X.__dataframe__().get_columns() + ] + ) + else: + X_is_dataframe = False + categorical_columns_mask = None + + categorical_features = self.categorical_features + + categorical_by_dtype = ( + isinstance(categorical_features, str) + and categorical_features == "from_dtype" + ) + no_categorical_dtype = categorical_features is None or ( + categorical_by_dtype and not X_is_dataframe + ) - categorical_features = np.asarray(self.categorical_features) + if no_categorical_dtype: + return None + + use_pandas_categorical = categorical_by_dtype and X_is_dataframe + if use_pandas_categorical: + categorical_features = categorical_columns_mask + else: + categorical_features = np.asarray(categorical_features) if categorical_features.size == 0: - return None, None + return None - if categorical_features.dtype.kind not in ('i', 'b'): - raise ValueError("categorical_features must be an array-like of " - "bools or array-like of ints.") + if categorical_features.dtype.kind not in ("i", "b", "U", "O"): + raise ValueError( + "categorical_features must be an array-like of bool, int or " + f"str, got: {categorical_features.dtype.name}." + ) - n_features = X.shape[1] + if categorical_features.dtype.kind == "O": + types = set(type(f) for f in categorical_features) + if types != {str}: + raise ValueError( + "categorical_features must be an array-like of bool, int or " + f"str, got: {', '.join(sorted(t.__name__ for t in types))}." + ) - # check for categorical features as indices - if categorical_features.dtype.kind == 'i': - if (np.max(categorical_features) >= n_features - or np.min(categorical_features) < 0): - raise ValueError("categorical_features set as integer " - "indices must be in [0, n_features - 1]") + n_features = X.shape[1] + # At this point `validate_data` was not called yet because we use the original + # dtypes to discover the categorical features. Thus `feature_names_in_` + # is not defined yet. + feature_names_in_ = getattr(X, "columns", None) + + if categorical_features.dtype.kind in ("U", "O"): + # check for feature names + if feature_names_in_ is None: + raise ValueError( + "categorical_features should be passed as an array of " + "integers or as a boolean mask when the model is fitted " + "on data without feature names." + ) + is_categorical = np.zeros(n_features, dtype=bool) + feature_names = list(feature_names_in_) + for feature_name in categorical_features: + try: + is_categorical[feature_names.index(feature_name)] = True + except ValueError as e: + raise ValueError( + f"categorical_features has a item value '{feature_name}' " + "which is not a valid feature name of the training " + f"data. Observed feature names: {feature_names}" + ) from e + elif categorical_features.dtype.kind == "i": + # check for categorical features as indices + if ( + np.max(categorical_features) >= n_features + or np.min(categorical_features) < 0 + ): + raise ValueError( + "categorical_features set as integer " + "indices must be in [0, n_features - 1]" + ) is_categorical = np.zeros(n_features, dtype=bool) is_categorical[categorical_features] = True else: if categorical_features.shape[0] != n_features: - raise ValueError("categorical_features set as a boolean mask " - "must have shape (n_features,), got: " - f"{categorical_features.shape}") + raise ValueError( + "categorical_features set as a boolean mask " + "must have shape (n_features,), got: " + f"{categorical_features.shape}" + ) is_categorical = categorical_features if not np.any(is_categorical): - return None, None - - # compute the known categories in the training data. We need to do - # that here instead of in the BinMapper because in case of early - # stopping, the mapper only gets a fraction of the training data. - known_categories = [] - - for f_idx in range(n_features): - if is_categorical[f_idx]: - categories = np.unique(X[:, f_idx]) - missing = np.isnan(categories) - if missing.any(): - categories = categories[~missing] + return None + return is_categorical + + def _check_interaction_cst(self, n_features): + """Check and validation for interaction constraints.""" + if self.interaction_cst is None: + return None + + if self.interaction_cst == "no_interactions": + interaction_cst = [[i] for i in range(n_features)] + elif self.interaction_cst == "pairwise": + interaction_cst = itertools.combinations(range(n_features), 2) + else: + interaction_cst = self.interaction_cst - if categories.size > self.max_bins: - raise ValueError( - f"Categorical feature at index {f_idx} is " - f"expected to have a " - f"cardinality <= {self.max_bins}" - ) + try: + constraints = [set(group) for group in interaction_cst] + except TypeError: + raise ValueError( + "Interaction constraints must be a sequence of tuples or lists, got:" + f" {self.interaction_cst!r}." + ) - if (categories >= self.max_bins).any(): + for group in constraints: + for x in group: + if not (isinstance(x, Integral) and 0 <= x < n_features): raise ValueError( - f"Categorical feature at index {f_idx} is " - f"expected to be encoded with " - f"values < {self.max_bins}" + "Interaction constraints must consist of integer indices in" + f" [0, n_features - 1] = [0, {n_features - 1}], specifying the" + " position of features, got invalid indices:" + f" {group!r}" ) - else: - categories = None - known_categories.append(categories) - - return is_categorical, known_categories - def fit(self, X, y, sample_weight=None): + # Add all not listed features as own group by default. + rest = set(range(n_features)) - set().union(*constraints) + if len(rest) > 0: + constraints.append(rest) + + return constraints + + @_fit_context(prefer_skip_nested_validation=True) + def fit( + self, + X, + y, + sample_weight=None, + *, + X_val=None, + y_val=None, + sample_weight_val=None, + ): """Fit the gradient boosting model. Parameters @@ -189,43 +533,100 @@ def fit(self, X, y, sample_weight=None): .. versionadded:: 0.23 + X_val : array-like of shape (n_val, n_features) + Additional sample of features for validation used in early stopping. + In a `Pipeline`, `X_val` can be transformed the same way as `X` with + `Pipeline(..., transform_input=["X_val"])`. + + .. versionadded:: 1.7 + + y_val : array-like of shape (n_samples,) + Additional sample of target values for validation used in early stopping. + + .. versionadded:: 1.7 + + sample_weight_val : array-like of shape (n_samples,) default=None + Additional weights for validation used in early stopping. + + .. versionadded:: 1.7 + Returns ------- self : object + Fitted estimator. """ fit_start_time = time() - acc_find_split_time = 0. # time spent finding the best splits - acc_apply_split_time = 0. # time spent splitting nodes - acc_compute_hist_time = 0. # time spent computing histograms + acc_find_split_time = 0.0 # time spent finding the best splits + acc_apply_split_time = 0.0 # time spent splitting nodes + acc_compute_hist_time = 0.0 # time spent computing histograms # time spent predicting X for gradient and hessians update - acc_prediction_time = 0. - X, y = self._validate_data(X, y, dtype=[X_DTYPE], - force_all_finite=False) + acc_prediction_time = 0.0 + X, known_categories = self._preprocess_X(X, reset=True) + y = _check_y(y, estimator=self) y = self._encode_y(y) check_consistent_length(X, y) # Do not create unit sample weights by default to later skip some # computation if sample_weight is not None: - sample_weight = _check_sample_weight(sample_weight, X, - dtype=np.float64) - # TODO: remove when PDP suports sample weights + sample_weight = _check_sample_weight(sample_weight, X, dtype=np.float64) + # TODO: remove when PDP supports sample weights self._fitted_with_sw = True + sample_weight = self._finalize_sample_weight(sample_weight, y) + + validation_data_provided = X_val is not None or y_val is not None + if validation_data_provided: + if y_val is None: + raise ValueError("X_val is provided, but y_val was not provided.") + if X_val is None: + raise ValueError("y_val is provided, but X_val was not provided.") + X_val = self._preprocess_X(X_val, reset=False) + y_val = _check_y(y_val, estimator=self) + y_val = self._encode_y_val(y_val) + check_consistent_length(X_val, y_val) + if sample_weight_val is not None: + sample_weight_val = _check_sample_weight( + sample_weight_val, X_val, dtype=np.float64 + ) + if self.early_stopping is False: + raise ValueError( + "X_val and y_val are passed to fit while at the same time " + "early_stopping is False. When passing X_val and y_val to fit," + "early_stopping should be set to either 'auto' or True." + ) + + # Note: At this point, we could delete self._label_encoder if it exists. + # But we don't to keep the code even simpler. + rng = check_random_state(self.random_state) - # When warm starting, we want to re-use the same seed that was used - # the first time fit was called (e.g. for subsampling or for the - # train/val split). - if not (self.warm_start and self._is_fitted()): - self._random_seed = rng.randint(np.iinfo(np.uint32).max, - dtype='u8') + # When warm starting, we want to reuse the same seed that was used + # the first time fit was called (e.g. train/val split). + # For feature subsampling, we want to continue with the rng we started with. + if not self.warm_start or not self._is_fitted(): + self._random_seed = rng.randint(np.iinfo(np.uint32).max, dtype="u8") + feature_subsample_seed = rng.randint(np.iinfo(np.uint32).max, dtype="u8") + self._feature_subsample_rng = np.random.default_rng(feature_subsample_seed) self._validate_parameters() + monotonic_cst = _check_monotonic_cst(self, self.monotonic_cst) + # _preprocess_X places the categorical features at the beginning, + # change the order of monotonic_cst accordingly + if self.is_categorical_ is not None: + monotonic_cst_remapped = np.concatenate( + ( + monotonic_cst[self.is_categorical_], + monotonic_cst[~self.is_categorical_], + ) + ) + else: + monotonic_cst_remapped = monotonic_cst # used for validation in predict n_samples, self._n_features = X.shape - self.is_categorical_, known_categories = self._check_categories(X) + # Encode constraints into a list of sets of features indices (integers). + interaction_cst = self._check_interaction_cst(self._n_features) # we need this stateful variable to tell raw_predict() that it was # called from fit() (this current method), and that the data it has @@ -237,21 +638,32 @@ def fit(self, X, y, sample_weight=None): # data. self._in_fit = True + # `_openmp_effective_n_threads` is used to take cgroups CPU quotes + # into account when determine the maximum number of threads to use. + n_threads = _openmp_effective_n_threads() + if isinstance(self.loss, str): self._loss = self._get_loss(sample_weight=sample_weight) elif isinstance(self.loss, BaseLoss): self._loss = self.loss - if self.early_stopping == 'auto': - self.do_early_stopping_ = n_samples > 10000 + if self.early_stopping == "auto": + self.do_early_stopping_ = n_samples > 10_000 else: self.do_early_stopping_ = self.early_stopping # create validation data if needed - self._use_validation_data = self.validation_fraction is not None - if self.do_early_stopping_ and self._use_validation_data: + self._use_validation_data = ( + self.validation_fraction is not None or validation_data_provided + ) + if ( + self.do_early_stopping_ + and self._use_validation_data + and not validation_data_provided + ): # stratify for classification - stratify = y if hasattr(self._loss, 'predict_proba') else None + # instead of checking predict_proba, loss.n_classes >= 2 would also work + stratify = y if hasattr(self._loss, "predict_proba") else None # Save the state of the RNG for the training and validation split. # This is needed in order to have the same split when using @@ -259,21 +671,35 @@ def fit(self, X, y, sample_weight=None): if sample_weight is None: X_train, X_val, y_train, y_val = train_test_split( - X, y, test_size=self.validation_fraction, + X, + y, + test_size=self.validation_fraction, stratify=stratify, - random_state=self._random_seed) + random_state=self._random_seed, + ) sample_weight_train = sample_weight_val = None else: # TODO: incorporate sample_weight in sampling here, as well as # stratify - (X_train, X_val, y_train, y_val, sample_weight_train, - sample_weight_val) = train_test_split( - X, y, sample_weight, test_size=self.validation_fraction, + ( + X_train, + X_val, + y_train, + y_val, + sample_weight_train, + sample_weight_val, + ) = train_test_split( + X, + y, + sample_weight, + test_size=self.validation_fraction, stratify=stratify, - random_state=self._random_seed) + random_state=self._random_seed, + ) else: X_train, y_train, sample_weight_train = X, y, sample_weight - X_val = y_val = sample_weight_val = None + if not validation_data_provided: + X_val = y_val = sample_weight_val = None # Bin the data # For ease of use of the API, the user-facing GBDT classes accept the @@ -286,9 +712,11 @@ def fit(self, X, y, sample_weight=None): n_bins = self.max_bins + 1 # + 1 for missing values self._bin_mapper = _BinMapper( n_bins=n_bins, - is_categorical=self.is_categorical_, + is_categorical=self._is_categorical_remapped, known_categories=known_categories, - random_state=self._random_seed) + random_state=self._random_seed, + n_threads=n_threads, + ) X_binned_train = self._bin_data(X_train, is_training_data=True) if X_val is not None: X_binned_val = self._bin_data(X_val, is_training_data=False) @@ -297,14 +725,19 @@ def fit(self, X, y, sample_weight=None): # Uses binned data to check for missing values has_missing_values = ( - X_binned_train == self._bin_mapper.missing_values_bin_idx_).any( - axis=0).astype(np.uint8) + (X_binned_train == self._bin_mapper.missing_values_bin_idx_) + .any(axis=0) + .astype(np.uint8) + ) if self.verbose: print("Fitting gradient boosted rounds:") n_samples = X_binned_train.shape[0] - + scoring_is_predefined_string = self.scoring in _SCORERS + need_raw_predictions_val = X_binned_val is not None and ( + scoring_is_predefined_string or self.scoring == "loss" + ) # First time calling fit, or no warm start if not (self._is_fitted() and self.warm_start): # Clear random state and score attributes @@ -312,15 +745,17 @@ def fit(self, X, y, sample_weight=None): # initialize raw_predictions: those are the accumulated values # predicted by the trees for the training data. raw_predictions has - # shape (n_trees_per_iteration, n_samples) where + # shape (n_samples, n_trees_per_iteration) where # n_trees_per_iterations is n_classes in multiclass classification, # else 1. - self._baseline_prediction = self._loss.get_baseline_prediction( - y_train, sample_weight_train, self.n_trees_per_iteration_ - ) + # self._baseline_prediction has shape (1, n_trees_per_iteration) + self._baseline_prediction = self._loss.fit_intercept_only( + y_true=y_train, sample_weight=sample_weight_train + ).reshape((1, -1)) raw_predictions = np.zeros( - shape=(self.n_trees_per_iteration_, n_samples), - dtype=self._baseline_prediction.dtype + shape=(n_samples, self.n_trees_per_iteration_), + dtype=self._baseline_prediction.dtype, + order="F", ) raw_predictions += self._baseline_prediction @@ -330,7 +765,7 @@ def fit(self, X, y, sample_weight=None): # Initialize structures and attributes related to early stopping self._scorer = None # set if scoring != loss - raw_predictions_val = None # set if scoring == loss and use val + raw_predictions_val = None # set if use val and scoring is a string self.train_score_ = [] self.validation_score_ = [] @@ -338,29 +773,34 @@ def fit(self, X, y, sample_weight=None): # populate train_score and validation_score with the # predictions of the initial model (before the first tree) - if self.scoring == 'loss': + # Create raw_predictions_val for storing the raw predictions of + # the validation data. + if need_raw_predictions_val: + raw_predictions_val = np.zeros( + shape=(X_binned_val.shape[0], self.n_trees_per_iteration_), + dtype=self._baseline_prediction.dtype, + order="F", + ) + + raw_predictions_val += self._baseline_prediction + + if self.scoring == "loss": # we're going to compute scoring w.r.t the loss. As losses # take raw predictions as input (unlike the scorers), we # can optimize a bit and avoid repeating computing the - # predictions of the previous trees. We'll re-use + # predictions of the previous trees. We'll reuse # raw_predictions (as it's needed for training anyway) for - # evaluating the training loss, and create - # raw_predictions_val for storing the raw predictions of - # the validation data. - - if self._use_validation_data: - raw_predictions_val = np.zeros( - shape=(self.n_trees_per_iteration_, - X_binned_val.shape[0]), - dtype=self._baseline_prediction.dtype - ) - - raw_predictions_val += self._baseline_prediction - - self._check_early_stopping_loss(raw_predictions, y_train, - sample_weight_train, - raw_predictions_val, y_val, - sample_weight_val) + # evaluating the training loss. + + self._check_early_stopping_loss( + raw_predictions=raw_predictions, + y_train=y_train, + sample_weight_train=sample_weight_train, + raw_predictions_val=raw_predictions_val, + y_val=y_val, + sample_weight_val=sample_weight_val, + n_threads=n_threads, + ) else: self._scorer = check_scoring(self, self.scoring) # _scorer is a callable with signature (est, X, y) and @@ -371,16 +811,37 @@ def fit(self, X, y, sample_weight=None): # the training set to compute train scores. # Compute the subsample set - (X_binned_small_train, - y_small_train, - sample_weight_small_train) = self._get_small_trainset( - X_binned_train, y_train, sample_weight_train, - self._random_seed) + ( + X_binned_small_train, + y_small_train, + sample_weight_small_train, + indices_small_train, + ) = self._get_small_trainset( + X_binned_train, + y_train, + sample_weight_train, + self._random_seed, + ) + + # If the scorer is a predefined string, then we optimize + # the evaluation by reusing the incrementally updated raw + # predictions. + if scoring_is_predefined_string: + raw_predictions_small_train = raw_predictions[ + indices_small_train + ] + else: + raw_predictions_small_train = None self._check_early_stopping_scorer( - X_binned_small_train, y_small_train, + X_binned_small_train, + y_small_train, sample_weight_small_train, - X_binned_val, y_val, sample_weight_val, + X_binned_val, + y_val, + sample_weight_val, + raw_predictions_small_train=raw_predictions_small_train, + raw_predictions_val=raw_predictions_val, ) begin_at_stage = 0 @@ -390,9 +851,8 @@ def fit(self, X, y, sample_weight=None): # than the number of iterations from the previous fit if self.max_iter < self.n_iter_: raise ValueError( - 'max_iter=%d must be larger than or equal to ' - 'n_iter_=%d when warm_start==True' - % (self.max_iter, self.n_iter_) + "max_iter=%d must be larger than or equal to " + "n_iter_=%d when warm_start==True" % (self.max_iter, self.n_iter_) ) # Convert array attributes to lists @@ -400,19 +860,24 @@ def fit(self, X, y, sample_weight=None): self.validation_score_ = self.validation_score_.tolist() # Compute raw predictions - raw_predictions = self._raw_predict(X_binned_train) - if self.do_early_stopping_ and self._use_validation_data: - raw_predictions_val = self._raw_predict(X_binned_val) + raw_predictions = self._raw_predict(X_binned_train, n_threads=n_threads) + if self.do_early_stopping_ and need_raw_predictions_val: + raw_predictions_val = self._raw_predict( + X_binned_val, n_threads=n_threads + ) else: raw_predictions_val = None - if self.do_early_stopping_ and self.scoring != 'loss': + if self.do_early_stopping_ and self.scoring != "loss": # Compute the subsample set - (X_binned_small_train, - y_small_train, - sample_weight_small_train) = self._get_small_trainset( - X_binned_train, y_train, sample_weight_train, - self._random_seed) + ( + X_binned_small_train, + y_small_train, + sample_weight_small_train, + indices_small_train, + ) = self._get_small_trainset( + X_binned_train, y_train, sample_weight_train, self._random_seed + ) # Get the predictors from the previous fit predictors = self._predictors @@ -420,52 +885,86 @@ def fit(self, X, y, sample_weight=None): begin_at_stage = self.n_iter_ # initialize gradients and hessians (empty arrays). - # shape = (n_trees_per_iteration, n_samples). - gradients, hessians = self._loss.init_gradients_and_hessians( - n_samples=n_samples, - prediction_dim=self.n_trees_per_iteration_, - sample_weight=sample_weight_train + # shape = (n_samples, n_trees_per_iteration). + gradient, hessian = self._loss.init_gradient_and_hessian( + n_samples=n_samples, dtype=G_H_DTYPE, order="F" ) for iteration in range(begin_at_stage, self.max_iter): - - if self.verbose: + if self.verbose >= 2: iteration_start_time = time() - print("[{}/{}] ".format(iteration + 1, self.max_iter), - end='', flush=True) + print( + "[{}/{}] ".format(iteration + 1, self.max_iter), end="", flush=True + ) # Update gradients and hessians, inplace - self._loss.update_gradients_and_hessians(gradients, hessians, - y_train, raw_predictions, - sample_weight_train) + # Note that self._loss expects shape (n_samples,) for + # n_trees_per_iteration = 1 else shape (n_samples, n_trees_per_iteration). + if self._loss.constant_hessian: + self._loss.gradient( + y_true=y_train, + raw_prediction=raw_predictions, + sample_weight=sample_weight_train, + gradient_out=gradient, + n_threads=n_threads, + ) + else: + self._loss.gradient_hessian( + y_true=y_train, + raw_prediction=raw_predictions, + sample_weight=sample_weight_train, + gradient_out=gradient, + hessian_out=hessian, + n_threads=n_threads, + ) # Append a list since there may be more than 1 predictor per iter predictors.append([]) + # 2-d views of shape (n_samples, n_trees_per_iteration_) or (n_samples, 1) + # on gradient and hessian to simplify the loop over n_trees_per_iteration_. + if gradient.ndim == 1: + g_view = gradient.reshape((-1, 1)) + h_view = hessian.reshape((-1, 1)) + else: + g_view = gradient + h_view = hessian + # Build `n_trees_per_iteration` trees. for k in range(self.n_trees_per_iteration_): grower = TreeGrower( - X_binned_train, gradients[k, :], hessians[k, :], + X_binned=X_binned_train, + gradients=g_view[:, k], + hessians=h_view[:, k], n_bins=n_bins, n_bins_non_missing=self._bin_mapper.n_bins_non_missing_, has_missing_values=has_missing_values, - is_categorical=self.is_categorical_, - monotonic_cst=self.monotonic_cst, + is_categorical=self._is_categorical_remapped, + monotonic_cst=monotonic_cst_remapped, + interaction_cst=interaction_cst, max_leaf_nodes=self.max_leaf_nodes, max_depth=self.max_depth, min_samples_leaf=self.min_samples_leaf, l2_regularization=self.l2_regularization, - shrinkage=self.learning_rate) + feature_fraction_per_split=self.max_features, + rng=self._feature_subsample_rng, + shrinkage=self.learning_rate, + n_threads=n_threads, + ) grower.grow() acc_apply_split_time += grower.total_apply_split_time acc_find_split_time += grower.total_find_split_time acc_compute_hist_time += grower.total_compute_hist_time - if self._loss.need_update_leaves_values: - self._loss.update_leaves_values(grower, y_train, - raw_predictions[k, :], - sample_weight_train) + if not self._loss.differentiable: + _update_leaves_values( + loss=self._loss, + grower=grower, + y_true=y_train, + raw_prediction=raw_predictions[:, k], + sample_weight=sample_weight_train, + ) predictor = grower.make_predictor( binning_thresholds=self._bin_mapper.bin_thresholds_ @@ -475,36 +974,54 @@ def fit(self, X, y, sample_weight=None): # Update raw_predictions with the predictions of the newly # created tree. tic_pred = time() - _update_raw_predictions(raw_predictions[k, :], grower) + _update_raw_predictions(raw_predictions[:, k], grower, n_threads) toc_pred = time() acc_prediction_time += toc_pred - tic_pred should_early_stop = False if self.do_early_stopping_: - if self.scoring == 'loss': - # Update raw_predictions_val with the newest tree(s) - if self._use_validation_data: - for k, pred in enumerate(self._predictors[-1]): - raw_predictions_val[k, :] += ( - pred.predict_binned( - X_binned_val, - self._bin_mapper.missing_values_bin_idx_ - ) - ) + # Update raw_predictions_val with the newest tree(s) + if need_raw_predictions_val: + for k, pred in enumerate(self._predictors[-1]): + raw_predictions_val[:, k] += pred.predict_binned( + X_binned_val, + self._bin_mapper.missing_values_bin_idx_, + n_threads, + ) + if self.scoring == "loss": should_early_stop = self._check_early_stopping_loss( - raw_predictions, y_train, sample_weight_train, - raw_predictions_val, y_val, sample_weight_val + raw_predictions=raw_predictions, + y_train=y_train, + sample_weight_train=sample_weight_train, + raw_predictions_val=raw_predictions_val, + y_val=y_val, + sample_weight_val=sample_weight_val, + n_threads=n_threads, ) else: + # If the scorer is a predefined string, then we optimize the + # evaluation by reusing the incrementally computed raw predictions. + if scoring_is_predefined_string: + raw_predictions_small_train = raw_predictions[ + indices_small_train + ] + else: + raw_predictions_small_train = None + should_early_stop = self._check_early_stopping_scorer( - X_binned_small_train, y_small_train, + X_binned_small_train, + y_small_train, sample_weight_small_train, - X_binned_val, y_val, sample_weight_val + X_binned_val, + y_val, + sample_weight_val, + raw_predictions_small_train=raw_predictions_small_train, + raw_predictions_val=raw_predictions_val, ) - if self.verbose: + if self.verbose >= 2: self._print_iteration_stats(iteration_start_time) # maybe we could also early stop if all the trees are stumps? @@ -520,17 +1037,31 @@ def fit(self, X, y, sample_weight=None): ) n_predictors = sum( len(predictors_at_ith_iteration) - for predictors_at_ith_iteration in self._predictors) - print("Fit {} trees in {:.3f} s, ({} total leaves)".format( - n_predictors, duration, n_total_leaves)) - print("{:<32} {:.3f}s".format('Time spent computing histograms:', - acc_compute_hist_time)) - print("{:<32} {:.3f}s".format('Time spent finding best splits:', - acc_find_split_time)) - print("{:<32} {:.3f}s".format('Time spent applying splits:', - acc_apply_split_time)) - print("{:<32} {:.3f}s".format('Time spent predicting:', - acc_prediction_time)) + for predictors_at_ith_iteration in self._predictors + ) + print( + "Fit {} trees in {:.3f} s, ({} total leaves)".format( + n_predictors, duration, n_total_leaves + ) + ) + print( + "{:<32} {:.3f}s".format( + "Time spent computing histograms:", acc_compute_hist_time + ) + ) + print( + "{:<32} {:.3f}s".format( + "Time spent finding best splits:", acc_find_split_time + ) + ) + print( + "{:<32} {:.3f}s".format( + "Time spent applying splits:", acc_apply_split_time + ) + ) + print( + "{:<32} {:.3f}s".format("Time spent predicting:", acc_prediction_time) + ) self.train_score_ = np.asarray(self.train_score_) self.validation_score_ = np.asarray(self.validation_score_) @@ -538,16 +1069,15 @@ def fit(self, X, y, sample_weight=None): return self def _is_fitted(self): - return len(getattr(self, '_predictors', [])) > 0 + return len(getattr(self, "_predictors", [])) > 0 def _clear_state(self): """Clear the state of the gradient boosting model.""" - for var in ('train_score_', 'validation_score_'): + for var in ("train_score_", "validation_score_"): if hasattr(self, var): delattr(self, var) - def _get_small_trainset(self, X_binned_train, y_train, sample_weight_train, - seed): + def _get_small_trainset(self, X_binned_train, y_train, sample_weight_train, seed): """Compute the indices of the subsample set and return this set. For efficiency, we need to subsample the training set to compute scores @@ -558,9 +1088,13 @@ def _get_small_trainset(self, X_binned_train, y_train, sample_weight_train, if X_binned_train.shape[0] > subsample_size: indices = np.arange(X_binned_train.shape[0]) stratify = y_train if is_classifier(self) else None - indices = resample(indices, n_samples=subsample_size, - replace=False, random_state=seed, - stratify=stratify) + indices = resample( + indices, + n_samples=subsample_size, + replace=False, + random_state=seed, + stratify=stratify, + ) X_binned_small_train = X_binned_train[indices] y_small_train = y_train[indices] if sample_weight_train is not None: @@ -568,14 +1102,26 @@ def _get_small_trainset(self, X_binned_train, y_train, sample_weight_train, else: sample_weight_small_train = None X_binned_small_train = np.ascontiguousarray(X_binned_small_train) - return (X_binned_small_train, y_small_train, - sample_weight_small_train) + return ( + X_binned_small_train, + y_small_train, + sample_weight_small_train, + indices, + ) else: - return X_binned_train, y_train, sample_weight_train - - def _check_early_stopping_scorer(self, X_binned_small_train, y_small_train, - sample_weight_small_train, - X_binned_val, y_val, sample_weight_val): + return X_binned_train, y_train, sample_weight_train, slice(None) + + def _check_early_stopping_scorer( + self, + X_binned_small_train, + y_small_train, + sample_weight_small_train, + X_binned_val, + y_val, + sample_weight_val, + raw_predictions_small_train=None, + raw_predictions_val=None, + ): """Check if fitting should be early-stopped based on scorer. Scores are computed on validation data or on training data. @@ -583,51 +1129,70 @@ def _check_early_stopping_scorer(self, X_binned_small_train, y_small_train, if is_classifier(self): y_small_train = self.classes_[y_small_train.astype(int)] - if sample_weight_small_train is None: - self.train_score_.append( - self._scorer(self, X_binned_small_train, y_small_train) - ) - else: - self.train_score_.append( - self._scorer(self, X_binned_small_train, y_small_train, - sample_weight=sample_weight_small_train) + self.train_score_.append( + self._score_with_raw_predictions( + X_binned_small_train, + y_small_train, + sample_weight_small_train, + raw_predictions_small_train, ) + ) if self._use_validation_data: if is_classifier(self): y_val = self.classes_[y_val.astype(int)] - if sample_weight_val is None: - self.validation_score_.append( - self._scorer(self, X_binned_val, y_val) - ) - else: - self.validation_score_.append( - self._scorer(self, X_binned_val, y_val, - sample_weight=sample_weight_val) + self.validation_score_.append( + self._score_with_raw_predictions( + X_binned_val, y_val, sample_weight_val, raw_predictions_val ) + ) return self._should_stop(self.validation_score_) else: return self._should_stop(self.train_score_) - def _check_early_stopping_loss(self, - raw_predictions, - y_train, - sample_weight_train, - raw_predictions_val, - y_val, - sample_weight_val): + def _score_with_raw_predictions(self, X, y, sample_weight, raw_predictions=None): + if raw_predictions is None: + patcher_raw_predict = nullcontext() + else: + patcher_raw_predict = _patch_raw_predict(self, raw_predictions) + + with patcher_raw_predict: + if sample_weight is None: + return self._scorer(self, X, y) + else: + return self._scorer(self, X, y, sample_weight=sample_weight) + + def _check_early_stopping_loss( + self, + raw_predictions, + y_train, + sample_weight_train, + raw_predictions_val, + y_val, + sample_weight_val, + n_threads=1, + ): """Check if fitting should be early-stopped based on loss. Scores are computed on validation data or on training data. """ - self.train_score_.append( - -self._loss(y_train, raw_predictions, sample_weight_train) + -self._loss( + y_true=y_train, + raw_prediction=raw_predictions, + sample_weight=sample_weight_train, + n_threads=n_threads, + ) ) if self._use_validation_data: self.validation_score_.append( - -self._loss(y_val, raw_predictions_val, sample_weight_val) + -self._loss( + y_true=y_val, + raw_prediction=raw_predictions_val, + sample_weight=sample_weight_val, + n_threads=n_threads, + ) ) return self._should_stop(self.validation_score_) else: @@ -647,9 +1212,8 @@ def _should_stop(self, scores): # the reference score, and therefore it is more likely to early stop # because of the lack of significant improvement. reference_score = scores[-reference_position] + self.tol - recent_scores = scores[-reference_position + 1:] - recent_improvements = [score > reference_score - for score in recent_scores] + recent_scores = scores[-reference_position + 1 :] + recent_improvements = [score > reference_score for score in recent_scores] return not any(recent_improvements) def _bin_data(self, X, is_training_data): @@ -659,10 +1223,13 @@ def _bin_data(self, X, is_training_data): Else, the binned data is converted to a C-contiguous array. """ - description = 'training' if is_training_data else 'validation' + description = "training" if is_training_data else "validation" if self.verbose: - print("Binning {:.3f} GB of {} data: ".format( - X.nbytes / 1e9, description), end="", flush=True) + print( + "Binning {:.3f} GB of {} data: ".format(X.nbytes / 1e9, description), + end="", + flush=True, + ) tic = time() if is_training_data: X_binned = self._bin_mapper.fit_transform(X) # F-aligned array @@ -680,97 +1247,111 @@ def _bin_data(self, X, is_training_data): def _print_iteration_stats(self, iteration_start_time): """Print info about the current fitting iteration.""" - log_msg = '' + log_msg = "" predictors_of_ith_iteration = [ - predictors_list for predictors_list in self._predictors[-1] + predictors_list + for predictors_list in self._predictors[-1] if predictors_list ] n_trees = len(predictors_of_ith_iteration) - max_depth = max(predictor.get_max_depth() - for predictor in predictors_of_ith_iteration) - n_leaves = sum(predictor.get_n_leaf_nodes() - for predictor in predictors_of_ith_iteration) + max_depth = max( + predictor.get_max_depth() for predictor in predictors_of_ith_iteration + ) + n_leaves = sum( + predictor.get_n_leaf_nodes() for predictor in predictors_of_ith_iteration + ) if n_trees == 1: - log_msg += ("{} tree, {} leaves, ".format(n_trees, n_leaves)) + log_msg += "{} tree, {} leaves, ".format(n_trees, n_leaves) else: - log_msg += ("{} trees, {} leaves ".format(n_trees, n_leaves)) - log_msg += ("({} on avg), ".format(int(n_leaves / n_trees))) + log_msg += "{} trees, {} leaves ".format(n_trees, n_leaves) + log_msg += "({} on avg), ".format(int(n_leaves / n_trees)) log_msg += "max depth = {}, ".format(max_depth) if self.do_early_stopping_: - if self.scoring == 'loss': + if self.scoring == "loss": factor = -1 # score_ arrays contain the negative loss - name = 'loss' + name = "loss" else: factor = 1 - name = 'score' - log_msg += "train {}: {:.5f}, ".format(name, factor * - self.train_score_[-1]) + name = "score" + log_msg += "train {}: {:.5f}, ".format(name, factor * self.train_score_[-1]) if self._use_validation_data: log_msg += "val {}: {:.5f}, ".format( - name, factor * self.validation_score_[-1]) + name, factor * self.validation_score_[-1] + ) iteration_time = time() - iteration_start_time log_msg += "in {:0.3f}s".format(iteration_time) print(log_msg) - def _raw_predict(self, X): + def _raw_predict(self, X, n_threads=None): """Return the sum of the leaves values over all predictors. Parameters ---------- X : array-like of shape (n_samples, n_features) The input samples. + n_threads : int, default=None + Number of OpenMP threads to use. `_openmp_effective_n_threads` is called + to determine the effective number of threads use, which takes cgroups CPU + quotes into account. See the docstring of `_openmp_effective_n_threads` + for details. Returns ------- - raw_predictions : array, shape (n_trees_per_iteration, n_samples) + raw_predictions : array, shape (n_samples, n_trees_per_iteration) The raw predicted values. """ - is_binned = getattr(self, '_in_fit', False) - dtype = X_BINNED_DTYPE if is_binned else X_DTYPE - X = self._validate_data(X, dtype=dtype, force_all_finite=False, - reset=False) check_is_fitted(self) - if X.shape[1] != self._n_features: - raise ValueError( - 'X has {} features but this estimator was trained with ' - '{} features.'.format(X.shape[1], self._n_features) - ) + is_binned = getattr(self, "_in_fit", False) + if not is_binned: + X = self._preprocess_X(X, reset=False) + n_samples = X.shape[0] raw_predictions = np.zeros( - shape=(self.n_trees_per_iteration_, n_samples), - dtype=self._baseline_prediction.dtype + shape=(n_samples, self.n_trees_per_iteration_), + dtype=self._baseline_prediction.dtype, + order="F", ) raw_predictions += self._baseline_prediction + + # We intentionally decouple the number of threads used at prediction + # time from the number of threads used at fit time because the model + # can be deployed on a different machine for prediction purposes. + n_threads = _openmp_effective_n_threads(n_threads) self._predict_iterations( - X, self._predictors, raw_predictions, is_binned + X, self._predictors, raw_predictions, is_binned, n_threads ) return raw_predictions - def _predict_iterations(self, X, predictors, raw_predictions, is_binned): + def _predict_iterations(self, X, predictors, raw_predictions, is_binned, n_threads): """Add the predictions of the predictors to raw_predictions.""" if not is_binned: - known_cat_bitsets, f_idx_map = ( - self._bin_mapper.make_known_categories_bitsets()) + ( + known_cat_bitsets, + f_idx_map, + ) = self._bin_mapper.make_known_categories_bitsets() for predictors_of_ith_iteration in predictors: for k, predictor in enumerate(predictors_of_ith_iteration): if is_binned: predict = partial( predictor.predict_binned, - missing_values_bin_idx=self._bin_mapper.missing_values_bin_idx_ # noqa + missing_values_bin_idx=self._bin_mapper.missing_values_bin_idx_, + n_threads=n_threads, ) else: predict = partial( predictor.predict, known_cat_bitsets=known_cat_bitsets, - f_idx_map=f_idx_map) - raw_predictions[k, :] += predict(X) + f_idx_map=f_idx_map, + n_threads=n_threads, + ) + raw_predictions[:, k] += predict(X) def _staged_raw_predict(self, X): """Compute raw predictions of ``X`` for each iteration. @@ -784,32 +1365,38 @@ def _staged_raw_predict(self, X): The input samples. Yields - ------- + ------ raw_predictions : generator of ndarray of shape \ - (n_trees_per_iteration, n_samples) + (n_samples, n_trees_per_iteration) The raw predictions of the input samples. The order of the classes corresponds to that in the attribute :term:`classes_`. """ - X = self._validate_data(X, dtype=X_DTYPE, force_all_finite=False, - reset=False) check_is_fitted(self) + X = self._preprocess_X(X, reset=False) if X.shape[1] != self._n_features: raise ValueError( - 'X has {} features but this estimator was trained with ' - '{} features.'.format(X.shape[1], self._n_features) + "X has {} features but this estimator was trained with " + "{} features.".format(X.shape[1], self._n_features) ) n_samples = X.shape[0] raw_predictions = np.zeros( - shape=(self.n_trees_per_iteration_, n_samples), - dtype=self._baseline_prediction.dtype + shape=(n_samples, self.n_trees_per_iteration_), + dtype=self._baseline_prediction.dtype, + order="F", ) raw_predictions += self._baseline_prediction + + # We intentionally decouple the number of threads used at prediction + # time from the number of threads used at fit time because the model + # can be deployed on a different machine for prediction purposes. + n_threads = _openmp_effective_n_threads() for iteration in range(len(self._predictors)): self._predict_iterations( X, - self._predictors[iteration:iteration + 1], + self._predictors[iteration : iteration + 1], raw_predictions, - is_binned=False + is_binned=False, + n_threads=n_threads, ) yield raw_predictions.copy() @@ -818,10 +1405,10 @@ def _compute_partial_dependence_recursion(self, grid, target_features): Parameters ---------- - grid : ndarray, shape (n_samples, n_target_features) + grid : ndarray, shape (n_samples, n_target_features), dtype=np.float32 The grid points on which the partial dependence should be evaluated. - target_features : ndarray, shape (n_target_features) + target_features : ndarray, shape (n_target_features), dtype=np.intp The set of target features for which the partial dependence should be evaluated. @@ -832,27 +1419,34 @@ def _compute_partial_dependence_recursion(self, grid, target_features): The value of the partial dependence function on each grid point. """ - if getattr(self, '_fitted_with_sw', False): - raise NotImplementedError("{} does not support partial dependence " - "plots with the 'recursion' method when " - "sample weights were given during fit " - "time.".format(self.__class__.__name__)) + if getattr(self, "_fitted_with_sw", False): + raise NotImplementedError( + "{} does not support partial dependence " + "plots with the 'recursion' method when " + "sample weights were given during fit " + "time.".format(self.__class__.__name__) + ) - grid = np.asarray(grid, dtype=X_DTYPE, order='C') + grid = np.asarray(grid, dtype=X_DTYPE, order="C") averaged_predictions = np.zeros( - (self.n_trees_per_iteration_, grid.shape[0]), dtype=Y_DTYPE) + (self.n_trees_per_iteration_, grid.shape[0]), dtype=Y_DTYPE + ) + target_features = np.asarray(target_features, dtype=np.intp, order="C") for predictors_of_ith_iteration in self._predictors: for k, predictor in enumerate(predictors_of_ith_iteration): - predictor.compute_partial_dependence(grid, target_features, - averaged_predictions[k]) + predictor.compute_partial_dependence( + grid, target_features, averaged_predictions[k] + ) # Note that the learning rate is already accounted for in the leaves # values. return averaged_predictions - def _more_tags(self): - return {'allow_nan': True} + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.allow_nan = True + return tags @abstractmethod def _get_loss(self, sample_weight): @@ -860,10 +1454,15 @@ def _get_loss(self, sample_weight): @abstractmethod def _encode_y(self, y=None): - pass + pass # pragma: no cover + + @abstractmethod + def _encode_y_val(self, y=None): + pass # pragma: no cover @property def n_iter_(self): + """Number of iterations of the boosting process.""" check_is_fitted(self) return len(self._predictors) @@ -882,38 +1481,40 @@ class HistGradientBoostingRegressor(RegressorMixin, BaseHistGradientBoosting): assigned to the left or right child consequently. If no missing values were encountered for a given feature during training, then samples with missing values are mapped to whichever child has the most samples. + See :ref:`sphx_glr_auto_examples_ensemble_plot_hgbt_regression.py` for a + usecase example of this feature. This implementation is inspired by `LightGBM `_. - .. note:: - - This estimator is still **experimental** for now: the predictions - and the API might change without any deprecation cycle. To use it, - you need to explicitly import ``enable_hist_gradient_boosting``:: - - >>> # explicitly require this experimental feature - >>> from sklearn.experimental import enable_hist_gradient_boosting # noqa - >>> # now you can import normally from ensemble - >>> from sklearn.ensemble import HistGradientBoostingRegressor - Read more in the :ref:`User Guide `. .. versionadded:: 0.21 Parameters ---------- - loss : {'least_squares', 'least_absolute_deviation', 'poisson'}, \ - default='least_squares' + loss : {'squared_error', 'absolute_error', 'gamma', 'poisson', 'quantile'}, \ + default='squared_error' The loss function to use in the boosting process. Note that the - "least squares" and "poisson" losses actually implement - "half least squares loss" and "half poisson deviance" to simplify the - computation of the gradient. Furthermore, "poisson" loss internally - uses a log-link and requires ``y >= 0`` + "squared error", "gamma" and "poisson" losses actually implement + "half least squares loss", "half gamma deviance" and "half poisson + deviance" to simplify the computation of the gradient. Furthermore, + "gamma" and "poisson" losses internally use a log-link, "gamma" + requires ``y > 0`` and "poisson" requires ``y >= 0``. + "quantile" uses the pinball loss. .. versionchanged:: 0.23 Added option 'poisson'. + .. versionchanged:: 1.1 + Added option 'quantile'. + + .. versionchanged:: 1.3 + Added option 'gamma'. + + quantile : float, default=None + If loss is "quantile", this parameter specifies which quantile to be estimated + and must be between 0 and 1. learning_rate : float, default=0.1 The learning rate, also known as *shrinkage*. This is used as a multiplicative factor for the leaves values. Use ``1`` for no @@ -933,8 +1534,17 @@ class HistGradientBoostingRegressor(RegressorMixin, BaseHistGradientBoosting): than a few hundred samples, it is recommended to lower this value since only very shallow trees would be built. l2_regularization : float, default=0 - The L2 regularization parameter. Use ``0`` for no regularization - (default). + The L2 regularization parameter penalizing leaves with small hessians. + Use ``0`` for no regularization (default). + max_features : float, default=1.0 + Proportion of randomly chosen features in each and every node split. + This is a form of regularization, smaller values make the trees weaker + learners and might prevent overfitting. + If interaction constraints from `interaction_cst` are present, only allowed + features are taken into account for the subsampling. + + .. versionadded:: 1.4 + max_bins : int, default=255 The maximum number of bins to use for non-missing values. Before training, each feature of the input array `X` is binned into @@ -942,30 +1552,82 @@ class HistGradientBoostingRegressor(RegressorMixin, BaseHistGradientBoosting): Features with a small number of unique values may use less than ``max_bins`` bins. In addition to the ``max_bins`` bins, one more bin is always reserved for missing values. Must be no larger than 255. - categorical_features : array-like of {bool, int} of shape (n_features) \ - or shape (n_categorical_features,), default=None. + categorical_features : array-like of {bool, int, str} of shape (n_features) \ + or shape (n_categorical_features,), default='from_dtype' Indicates the categorical features. - None : no feature will be considered categorical. - boolean array-like : boolean mask indicating categorical features. - integer array-like : integer indices indicating categorical features. + - str array-like: names of categorical features (assuming the training + data has feature names). + - `"from_dtype"`: dataframe columns with dtype "category" are + considered to be categorical features. The input must be an object + exposing a ``__dataframe__`` method such as pandas or polars + DataFrames to use this feature. For each categorical feature, there must be at most `max_bins` unique - categories, and each categorical value must be in [0, max_bins -1]. + categories. Negative values for categorical features encoded as numeric + dtypes are treated as missing values. All categorical values are + converted to floating point numbers. This means that categorical values + of 1.0 and 1 are treated as the same category. - Read more in the :ref:`User Guide `. + Read more in the :ref:`User Guide ` and + :ref:`sphx_glr_auto_examples_ensemble_plot_gradient_boosting_categorical.py`. .. versionadded:: 0.24 - monotonic_cst : array-like of int of shape (n_features), default=None - Indicates the monotonic constraint to enforce on each feature. -1, 1 - and 0 respectively correspond to a negative constraint, positive - constraint and no constraint. Read more in the :ref:`User Guide - `. + .. versionchanged:: 1.2 + Added support for feature names. + + .. versionchanged:: 1.4 + Added `"from_dtype"` option. + + .. versionchanged:: 1.6 + The default value changed from `None` to `"from_dtype"`. + + monotonic_cst : array-like of int of shape (n_features) or dict, default=None + Monotonic constraint to enforce on each feature are specified using the + following integer values: + + - 1: monotonic increase + - 0: no constraint + - -1: monotonic decrease + + If a dict with str keys, map feature to monotonic constraints by name. + If an array, the features are mapped to constraints by position. See + :ref:`monotonic_cst_features_names` for a usage example. + + Read more in the :ref:`User Guide `. .. versionadded:: 0.23 + .. versionchanged:: 1.2 + Accept dict of constraints with feature names as keys. + + interaction_cst : {"pairwise", "no_interactions"} or sequence of lists/tuples/sets \ + of int, default=None + Specify interaction constraints, the sets of features which can + interact with each other in child node splits. + + Each item specifies the set of feature indices that are allowed + to interact with each other. If there are more features than + specified in these constraints, they are treated as if they were + specified as an additional set. + + The strings "pairwise" and "no_interactions" are shorthands for + allowing only pairwise or no interactions, respectively. + + For instance, with 5 features in total, `interaction_cst=[{0, 1}]` + is equivalent to `interaction_cst=[{0, 1}, {2, 3, 4}]`, + and specifies that each branch of a tree will either only split + on features 0 and 1 or only split on features 2, 3 and 4. + + See :ref:`this example` on how to use `interaction_cst`. + + .. versionadded:: 1.2 + warm_start : bool, default=False When set to ``True``, reuse the solution of the previous call to fit and add more estimators to the ensemble. For results to be valid, the @@ -973,21 +1635,28 @@ class HistGradientBoostingRegressor(RegressorMixin, BaseHistGradientBoosting): See :term:`the Glossary `. early_stopping : 'auto' or bool, default='auto' If 'auto', early stopping is enabled if the sample size is larger than - 10000. If True, early stopping is enabled, otherwise early stopping is - disabled. + 10000 or if `X_val` and `y_val` are passed to `fit`. If True, early stopping + is enabled, otherwise early stopping is disabled. .. versionadded:: 0.23 scoring : str or callable or None, default='loss' - Scoring parameter to use for early stopping. It can be a single - string (see :ref:`scoring_parameter`) or a callable (see - :ref:`scoring`). If None, the estimator's default scorer is used. If - ``scoring='loss'``, early stopping is checked w.r.t the loss value. - Only used if early stopping is performed. + Scoring method to use for early stopping. Only used if `early_stopping` + is enabled. Options: + + - str: see :ref:`scoring_string_names` for options. + - callable: a scorer callable object (e.g., function) with signature + ``scorer(estimator, X, y)``. See :ref:`scoring_callable` for details. + - `None`: the :ref:`coefficient of determination ` + (:math:`R^2`) is used. + - 'loss': early stopping is checked w.r.t the loss value. + validation_fraction : int or float or None, default=0.1 Proportion (or absolute size) of training data to set aside as validation data for early stopping. If None, early stopping is done on - the training data. Only used if early stopping is performed. + the training data. + The value is ignored if either early stopping is not performed, e.g. + `early_stopping=False`, or if `X_val` and `y_val` are passed to fit. n_iter_no_change : int, default=10 Used to determine when to "early stop". The fitting process is stopped when none of the last ``n_iter_no_change`` scores are better @@ -1000,7 +1669,8 @@ class HistGradientBoostingRegressor(RegressorMixin, BaseHistGradientBoosting): iterations to be considered an improvement upon the reference score. verbose : int, default=0 The verbosity level. If not zero, print some information about the - fitting process. + fitting process. ``1`` prints only summary info, ``2`` prints info per + iteration. random_state : int, RandomState instance or None, default=None Pseudo-random number generator to control the subsampling in the binning process, and the train/validation data split if early stopping @@ -1032,11 +1702,33 @@ class HistGradientBoostingRegressor(RegressorMixin, BaseHistGradientBoosting): is_categorical_ : ndarray, shape (n_features, ) or None Boolean mask for the categorical features. ``None`` if there are no categorical features. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + GradientBoostingRegressor : Exact gradient boosting method that does not + scale as good on datasets with a large number of samples. + sklearn.tree.DecisionTreeRegressor : A decision tree regressor. + RandomForestRegressor : A meta-estimator that fits a number of decision + tree regressors on various sub-samples of the dataset and uses + averaging to improve the statistical performance and control + over-fitting. + AdaBoostRegressor : A meta-estimator that begins by fitting a regressor + on the original dataset and then fits additional copies of the + regressor on the same dataset but where the weights of instances are + adjusted according to the error of the current prediction. As such, + subsequent regressors focus more on difficult cases. Examples -------- - >>> # To use this experimental feature, we need to explicitly ask for it: - >>> from sklearn.experimental import enable_hist_gradient_boosting # noqa >>> from sklearn.ensemble import HistGradientBoostingRegressor >>> from sklearn.datasets import load_diabetes >>> X, y = load_diabetes(return_X_y=True) @@ -1045,30 +1737,71 @@ class HistGradientBoostingRegressor(RegressorMixin, BaseHistGradientBoosting): 0.92... """ - _VALID_LOSSES = ('least_squares', 'least_absolute_deviation', - 'poisson') - - @_deprecate_positional_args - def __init__(self, loss='least_squares', *, learning_rate=0.1, - max_iter=100, max_leaf_nodes=31, max_depth=None, - min_samples_leaf=20, l2_regularization=0., max_bins=255, - categorical_features=None, monotonic_cst=None, - warm_start=False, early_stopping='auto', - scoring='loss', validation_fraction=0.1, - n_iter_no_change=10, tol=1e-7, - verbose=0, random_state=None): - super(HistGradientBoostingRegressor, self).__init__( - loss=loss, learning_rate=learning_rate, max_iter=max_iter, - max_leaf_nodes=max_leaf_nodes, max_depth=max_depth, + _parameter_constraints: dict = { + **BaseHistGradientBoosting._parameter_constraints, + "loss": [ + StrOptions( + { + "squared_error", + "absolute_error", + "poisson", + "gamma", + "quantile", + } + ), + BaseLoss, + ], + "quantile": [Interval(Real, 0, 1, closed="both"), None], + } + + def __init__( + self, + loss="squared_error", + *, + quantile=None, + learning_rate=0.1, + max_iter=100, + max_leaf_nodes=31, + max_depth=None, + min_samples_leaf=20, + l2_regularization=0.0, + max_features=1.0, + max_bins=255, + categorical_features="from_dtype", + monotonic_cst=None, + interaction_cst=None, + warm_start=False, + early_stopping="auto", + scoring="loss", + validation_fraction=0.1, + n_iter_no_change=10, + tol=1e-7, + verbose=0, + random_state=None, + ): + super().__init__( + loss=loss, + learning_rate=learning_rate, + max_iter=max_iter, + max_leaf_nodes=max_leaf_nodes, + max_depth=max_depth, min_samples_leaf=min_samples_leaf, - l2_regularization=l2_regularization, max_bins=max_bins, + l2_regularization=l2_regularization, + max_features=max_features, + max_bins=max_bins, monotonic_cst=monotonic_cst, + interaction_cst=interaction_cst, categorical_features=categorical_features, early_stopping=early_stopping, - warm_start=warm_start, scoring=scoring, + warm_start=warm_start, + scoring=scoring, validation_fraction=validation_fraction, - n_iter_no_change=n_iter_no_change, tol=tol, verbose=verbose, - random_state=random_state) + n_iter_no_change=n_iter_no_change, + tol=tol, + verbose=verbose, + random_state=random_state, + ) + self.quantile = quantile def predict(self, X): """Predict values for X. @@ -1086,10 +1819,10 @@ def predict(self, X): check_is_fitted(self) # Return inverse link of raw predictions after converting # shape (n_samples, 1) to (n_samples,) - return self._loss.inverse_link_function(self._raw_predict(X).ravel()) + return self._loss.link.inverse(self._raw_predict(X).ravel()) def staged_predict(self, X): - """Predict regression target for each iteration + """Predict regression target for each iteration. This method allows monitoring (i.e. determine error on testing set) after each stage. @@ -1102,30 +1835,42 @@ def staged_predict(self, X): The input samples. Yields - ------- + ------ y : generator of ndarray of shape (n_samples,) The predicted values of the input samples, for each iteration. """ for raw_predictions in self._staged_raw_predict(X): - yield self._loss.inverse_link_function(raw_predictions.ravel()) + yield self._loss.link.inverse(raw_predictions.ravel()) def _encode_y(self, y): # Just convert y to the expected dtype self.n_trees_per_iteration_ = 1 y = y.astype(Y_DTYPE, copy=False) - if self.loss == 'poisson': + if self.loss == "gamma": + # Ensure y > 0 + if not np.all(y > 0): + raise ValueError("loss='gamma' requires strictly positive y.") + elif self.loss == "poisson": # Ensure y >= 0 and sum(y) > 0 if not (np.all(y >= 0) and np.sum(y) > 0): - raise ValueError("loss='poisson' requires non-negative y and " - "sum(y) > 0.") + raise ValueError( + "loss='poisson' requires non-negative y and sum(y) > 0." + ) return y + def _encode_y_val(self, y=None): + return self._encode_y(y) + def _get_loss(self, sample_weight): - return _LOSSES[self.loss](sample_weight=sample_weight) + if self.loss == "quantile": + return _LOSSES[self.loss]( + sample_weight=sample_weight, quantile=self.quantile + ) + else: + return _LOSSES[self.loss](sample_weight=sample_weight) -class HistGradientBoostingClassifier(ClassifierMixin, - BaseHistGradientBoosting): +class HistGradientBoostingClassifier(ClassifierMixin, BaseHistGradientBoosting): """Histogram-based Gradient Boosting Classification Tree. This estimator is much faster than @@ -1143,30 +1888,25 @@ class HistGradientBoostingClassifier(ClassifierMixin, This implementation is inspired by `LightGBM `_. - .. note:: - - This estimator is still **experimental** for now: the predictions - and the API might change without any deprecation cycle. To use it, - you need to explicitly import ``enable_hist_gradient_boosting``:: - - >>> # explicitly require this experimental feature - >>> from sklearn.experimental import enable_hist_gradient_boosting # noqa - >>> # now you can import normally from ensemble - >>> from sklearn.ensemble import HistGradientBoostingClassifier - Read more in the :ref:`User Guide `. .. versionadded:: 0.21 Parameters ---------- - loss : {'auto', 'binary_crossentropy', 'categorical_crossentropy'}, \ - default='auto' - The loss function to use in the boosting process. 'binary_crossentropy' - (also known as logistic loss) is used for binary classification and - generalizes to 'categorical_crossentropy' for multiclass - classification. 'auto' will automatically choose either loss depending - on the nature of the problem. + loss : {'log_loss'}, default='log_loss' + The loss function to use in the boosting process. + + For binary classification problems, 'log_loss' is also known as logistic loss, + binomial deviance or binary crossentropy. Internally, the model fits one tree + per boosting iteration and uses the logistic sigmoid function (expit) as + inverse link function to compute the predicted positive class probability. + + For multiclass classification problems, 'log_loss' is also known as multinomial + deviance or categorical crossentropy. Internally, the model fits one tree per + boosting iteration and per class and uses the softmax function as inverse link + function to compute the predicted probabilities of the classes. + learning_rate : float, default=0.1 The learning rate, also known as *shrinkage*. This is used as a multiplicative factor for the leaves values. Use ``1`` for no @@ -1187,7 +1927,17 @@ class HistGradientBoostingClassifier(ClassifierMixin, than a few hundred samples, it is recommended to lower this value since only very shallow trees would be built. l2_regularization : float, default=0 - The L2 regularization parameter. Use 0 for no regularization. + The L2 regularization parameter penalizing leaves with small hessians. + Use ``0`` for no regularization (default). + max_features : float, default=1.0 + Proportion of randomly chosen features in each and every node split. + This is a form of regularization, smaller values make the trees weaker + learners and might prevent overfitting. + If interaction constraints from `interaction_cst` are present, only allowed + features are taken into account for the subsampling. + + .. versionadded:: 1.4 + max_bins : int, default=255 The maximum number of bins to use for non-missing values. Before training, each feature of the input array `X` is binned into @@ -1195,30 +1945,83 @@ class HistGradientBoostingClassifier(ClassifierMixin, Features with a small number of unique values may use less than ``max_bins`` bins. In addition to the ``max_bins`` bins, one more bin is always reserved for missing values. Must be no larger than 255. - categorical_features : array-like of {bool, int} of shape (n_features) \ - or shape (n_categorical_features,), default=None. + categorical_features : array-like of {bool, int, str} of shape (n_features) \ + or shape (n_categorical_features,), default='from_dtype' Indicates the categorical features. - None : no feature will be considered categorical. - boolean array-like : boolean mask indicating categorical features. - integer array-like : integer indices indicating categorical features. + - str array-like: names of categorical features (assuming the training + data has feature names). + - `"from_dtype"`: dataframe columns with dtype "category" are + considered to be categorical features. The input must be an object + exposing a ``__dataframe__`` method such as pandas or polars + DataFrames to use this feature. For each categorical feature, there must be at most `max_bins` unique - categories, and each categorical value must be in [0, max_bins -1]. + categories. Negative values for categorical features encoded as numeric + dtypes are treated as missing values. All categorical values are + converted to floating point numbers. This means that categorical values + of 1.0 and 1 are treated as the same category. Read more in the :ref:`User Guide `. .. versionadded:: 0.24 - monotonic_cst : array-like of int of shape (n_features), default=None - Indicates the monotonic constraint to enforce on each feature. -1, 1 - and 0 respectively correspond to a negative constraint, positive - constraint and no constraint. Read more in the :ref:`User Guide - `. + .. versionchanged:: 1.2 + Added support for feature names. + + .. versionchanged:: 1.4 + Added `"from_dtype"` option. + + .. versionchanged:: 1.6 + The default value changed from `None` to `"from_dtype"`. + + monotonic_cst : array-like of int of shape (n_features) or dict, default=None + Monotonic constraint to enforce on each feature are specified using the + following integer values: + + - 1: monotonic increase + - 0: no constraint + - -1: monotonic decrease + + If a dict with str keys, map feature to monotonic constraints by name. + If an array, the features are mapped to constraints by position. See + :ref:`monotonic_cst_features_names` for a usage example. + + The constraints are only valid for binary classifications and hold + over the probability of the positive class. + Read more in the :ref:`User Guide `. .. versionadded:: 0.23 + .. versionchanged:: 1.2 + Accept dict of constraints with feature names as keys. + + interaction_cst : {"pairwise", "no_interactions"} or sequence of lists/tuples/sets \ + of int, default=None + Specify interaction constraints, the sets of features which can + interact with each other in child node splits. + + Each item specifies the set of feature indices that are allowed + to interact with each other. If there are more features than + specified in these constraints, they are treated as if they were + specified as an additional set. + + The strings "pairwise" and "no_interactions" are shorthands for + allowing only pairwise or no interactions, respectively. + + For instance, with 5 features in total, `interaction_cst=[{0, 1}]` + is equivalent to `interaction_cst=[{0, 1}, {2, 3, 4}]`, + and specifies that each branch of a tree will either only split + on features 0 and 1 or only split on features 2, 3 and 4. + + See :ref:`this example` on how to use `interaction_cst`. + + .. versionadded:: 1.2 + warm_start : bool, default=False When set to ``True``, reuse the solution of the previous call to fit and add more estimators to the ensemble. For results to be valid, the @@ -1226,21 +2029,27 @@ class HistGradientBoostingClassifier(ClassifierMixin, See :term:`the Glossary `. early_stopping : 'auto' or bool, default='auto' If 'auto', early stopping is enabled if the sample size is larger than - 10000. If True, early stopping is enabled, otherwise early stopping is - disabled. + 10000 or if `X_val` and `y_val` are passed to `fit`. If True, early stopping + is enabled, otherwise early stopping is disabled. .. versionadded:: 0.23 scoring : str or callable or None, default='loss' - Scoring parameter to use for early stopping. It can be a single - string (see :ref:`scoring_parameter`) or a callable (see - :ref:`scoring`). If None, the estimator's default scorer - is used. If ``scoring='loss'``, early stopping is checked - w.r.t the loss value. Only used if early stopping is performed. + Scoring method to use for early stopping. Only used if `early_stopping` + is enabled. Options: + + - str: see :ref:`scoring_string_names` for options. + - callable: a scorer callable object (e.g., function) with signature + ``scorer(estimator, X, y)``. See :ref:`scoring_callable` for details. + - `None`: :ref:`accuracy ` is used. + - 'loss': early stopping is checked w.r.t the loss value. + validation_fraction : int or float or None, default=0.1 Proportion (or absolute size) of training data to set aside as validation data for early stopping. If None, early stopping is done on - the training data. Only used if early stopping is performed. + the training data. + The value is ignored if either early stopping is not performed, e.g. + `early_stopping=False`, or if `X_val` and `y_val` are passed to fit. n_iter_no_change : int, default=10 Used to determine when to "early stop". The fitting process is stopped when none of the last ``n_iter_no_change`` scores are better @@ -1253,13 +2062,24 @@ class HistGradientBoostingClassifier(ClassifierMixin, considered an improvement upon the reference score. verbose : int, default=0 The verbosity level. If not zero, print some information about the - fitting process. + fitting process. ``1`` prints only summary info, ``2`` prints info per + iteration. random_state : int, RandomState instance or None, default=None Pseudo-random number generator to control the subsampling in the binning process, and the train/validation data split if early stopping is enabled. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. + class_weight : dict or 'balanced', default=None + Weights associated with classes in the form `{class_label: weight}`. + If not given, all classes are supposed to have weight one. + The "balanced" mode uses the values of y to automatically adjust + weights inversely proportional to class frequencies in the input data + as `n_samples / (n_classes * np.bincount(y))`. + Note that these weights will be multiplied with sample_weight (passed + through the fit method) if `sample_weight` is specified. + + .. versionadded:: 1.2 Attributes ---------- @@ -1288,11 +2108,32 @@ class HistGradientBoostingClassifier(ClassifierMixin, is_categorical_ : ndarray, shape (n_features, ) or None Boolean mask for the categorical features. ``None`` if there are no categorical features. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + GradientBoostingClassifier : Exact gradient boosting method that does not + scale as good on datasets with a large number of samples. + sklearn.tree.DecisionTreeClassifier : A decision tree classifier. + RandomForestClassifier : A meta-estimator that fits a number of decision + tree classifiers on various sub-samples of the dataset and uses + averaging to improve the predictive accuracy and control over-fitting. + AdaBoostClassifier : A meta-estimator that begins by fitting a classifier + on the original dataset and then fits additional copies of the + classifier on the same dataset where the weights of incorrectly + classified instances are adjusted such that subsequent classifiers + focus more on difficult cases. Examples -------- - >>> # To use this experimental feature, we need to explicitly ask for it: - >>> from sklearn.experimental import enable_hist_gradient_boosting # noqa >>> from sklearn.ensemble import HistGradientBoostingClassifier >>> from sklearn.datasets import load_iris >>> X, y = load_iris(return_X_y=True) @@ -1301,29 +2142,72 @@ class HistGradientBoostingClassifier(ClassifierMixin, 1.0 """ - _VALID_LOSSES = ('binary_crossentropy', 'categorical_crossentropy', - 'auto') - - @_deprecate_positional_args - def __init__(self, loss='auto', *, learning_rate=0.1, max_iter=100, - max_leaf_nodes=31, max_depth=None, min_samples_leaf=20, - l2_regularization=0., max_bins=255, - categorical_features=None, monotonic_cst=None, - warm_start=False, early_stopping='auto', scoring='loss', - validation_fraction=0.1, n_iter_no_change=10, tol=1e-7, - verbose=0, random_state=None): - super(HistGradientBoostingClassifier, self).__init__( - loss=loss, learning_rate=learning_rate, max_iter=max_iter, - max_leaf_nodes=max_leaf_nodes, max_depth=max_depth, + _parameter_constraints: dict = { + **BaseHistGradientBoosting._parameter_constraints, + "loss": [StrOptions({"log_loss"}), BaseLoss], + "class_weight": [dict, StrOptions({"balanced"}), None], + } + + def __init__( + self, + loss="log_loss", + *, + learning_rate=0.1, + max_iter=100, + max_leaf_nodes=31, + max_depth=None, + min_samples_leaf=20, + l2_regularization=0.0, + max_features=1.0, + max_bins=255, + categorical_features="from_dtype", + monotonic_cst=None, + interaction_cst=None, + warm_start=False, + early_stopping="auto", + scoring="loss", + validation_fraction=0.1, + n_iter_no_change=10, + tol=1e-7, + verbose=0, + random_state=None, + class_weight=None, + ): + super().__init__( + loss=loss, + learning_rate=learning_rate, + max_iter=max_iter, + max_leaf_nodes=max_leaf_nodes, + max_depth=max_depth, min_samples_leaf=min_samples_leaf, - l2_regularization=l2_regularization, max_bins=max_bins, + l2_regularization=l2_regularization, + max_features=max_features, + max_bins=max_bins, categorical_features=categorical_features, monotonic_cst=monotonic_cst, + interaction_cst=interaction_cst, warm_start=warm_start, - early_stopping=early_stopping, scoring=scoring, + early_stopping=early_stopping, + scoring=scoring, validation_fraction=validation_fraction, - n_iter_no_change=n_iter_no_change, tol=tol, verbose=verbose, - random_state=random_state) + n_iter_no_change=n_iter_no_change, + tol=tol, + verbose=verbose, + random_state=random_state, + ) + self.class_weight = class_weight + + def _finalize_sample_weight(self, sample_weight, y): + """Adjust sample_weights with class_weights.""" + if self.class_weight is None: + return sample_weight + + expanded_class_weight = compute_sample_weight(self.class_weight, y) + + if sample_weight is not None: + return sample_weight * expanded_class_weight + else: + return expanded_class_weight def predict(self, X): """Predict classes for X. @@ -1339,7 +2223,13 @@ def predict(self, X): The predicted classes. """ # TODO: This could be done in parallel - encoded_classes = np.argmax(self.predict_proba(X), axis=1) + raw_predictions = self._raw_predict(X) + if raw_predictions.shape[1] == 1: + # np.argmax([0.5, 0.5]) is 0, not 1. Therefore "> 0" not ">= 0" to be + # consistent with the multiclass case. + encoded_classes = (raw_predictions.ravel() > 0).astype(int) + else: + encoded_classes = np.argmax(raw_predictions, axis=1) return self.classes_[encoded_classes] def staged_predict(self, X): @@ -1356,12 +2246,16 @@ def staged_predict(self, X): The input samples. Yields - ------- + ------ y : generator of ndarray of shape (n_samples,) The predicted classes of the input samples, for each iteration. """ - for proba in self.staged_predict_proba(X): - encoded_classes = np.argmax(proba, axis=1) + for raw_predictions in self._staged_raw_predict(X): + if raw_predictions.shape[1] == 1: + # np.argmax([0, 0]) is 0, not 1, therefore "> 0" not ">= 0" + encoded_classes = (raw_predictions.ravel() > 0).astype(int) + else: + encoded_classes = np.argmax(raw_predictions, axis=1) yield self.classes_.take(encoded_classes, axis=0) def predict_proba(self, X): @@ -1392,7 +2286,7 @@ def staged_predict_proba(self, X): The input samples. Yields - ------- + ------ y : generator of ndarray of shape (n_samples,) The predicted class probabilities of the input samples, for each iteration. @@ -1417,9 +2311,9 @@ def decision_function(self, X): classes in multiclass classification. """ decision = self._raw_predict(X) - if decision.shape[0] == 1: + if decision.shape[1] == 1: decision = decision.ravel() - return decision.T + return decision def staged_decision_function(self, X): """Compute decision function of ``X`` for each iteration. @@ -1433,7 +2327,7 @@ def staged_decision_function(self, X): The input samples. Yields - ------- + ------ decision : generator of ndarray of shape (n_samples,) or \ (n_samples, n_trees_per_iteration) The decision function of the input samples, which corresponds to @@ -1441,18 +2335,21 @@ def staged_decision_function(self, X): classes corresponds to that in the attribute :term:`classes_`. """ for staged_decision in self._staged_raw_predict(X): - if staged_decision.shape[0] == 1: + if staged_decision.shape[1] == 1: staged_decision = staged_decision.ravel() - yield staged_decision.T + yield staged_decision def _encode_y(self, y): + """Create self._label_encoder and encode y correspondingly.""" # encode classes into 0 ... n_classes - 1 and sets attributes classes_ # and n_trees_per_iteration_ check_classification_targets(y) - label_encoder = LabelEncoder() - encoded_y = label_encoder.fit_transform(y) - self.classes_ = label_encoder.classes_ + # We need to store the label encoder in case y_val needs to be label encoded, + # too. + self._label_encoder = LabelEncoder() + encoded_y = self._label_encoder.fit_transform(y) + self.classes_ = self._label_encoder.classes_ n_classes = self.classes_.shape[0] # only 1 tree for binary classification. For multiclass classification, # we build 1 tree per class. @@ -1460,19 +2357,15 @@ def _encode_y(self, y): encoded_y = encoded_y.astype(Y_DTYPE, copy=False) return encoded_y - def _get_loss(self, sample_weight): - if (self.loss == 'categorical_crossentropy' and - self.n_trees_per_iteration_ == 1): - raise ValueError("'categorical_crossentropy' is not suitable for " - "a binary classification problem. Please use " - "'auto' or 'binary_crossentropy' instead.") - - if self.loss == 'auto': - if self.n_trees_per_iteration_ == 1: - return _LOSSES['binary_crossentropy']( - sample_weight=sample_weight) - else: - return _LOSSES['categorical_crossentropy']( - sample_weight=sample_weight) + def _encode_y_val(self, y): + encoded_y = self._label_encoder.transform(y) + return encoded_y.astype(Y_DTYPE, copy=False) - return _LOSSES[self.loss](sample_weight=sample_weight) + def _get_loss(self, sample_weight): + # At this point self.loss == "log_loss" + if self.n_trees_per_iteration_ == 1: + return HalfBinomialLoss(sample_weight=sample_weight) + else: + return HalfMultinomialLoss( + sample_weight=sample_weight, n_classes=self.n_trees_per_iteration_ + ) diff --git a/sklearn/ensemble/_hist_gradient_boosting/grower.py b/sklearn/ensemble/_hist_gradient_boosting/grower.py index cdf3020be9541..c3dbbe7d82948 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/grower.py +++ b/sklearn/ensemble/_hist_gradient_boosting/grower.py @@ -4,24 +4,27 @@ TreeGrower builds a regression tree fitting a Newton-Raphson step, based on the gradients and hessians of the training data. """ -# Author: Nicolas Hug -from heapq import heappush, heappop -import numpy as np -from timeit import default_timer as time +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + import numbers +from heapq import heappop, heappush +from timeit import default_timer as time -from .splitting import Splitter +import numpy as np + +from sklearn.utils._openmp_helpers import _openmp_effective_n_threads + +from ._bitset import set_raw_bitset_from_binned_bitset +from .common import ( + PREDICTOR_RECORD_DTYPE, + X_BITSET_INNER_DTYPE, + MonotonicConstraint, +) from .histogram import HistogramBuilder from .predictor import TreePredictor -from .utils import sum_parallel -from .common import PREDICTOR_RECORD_DTYPE -from .common import X_BITSET_INNER_DTYPE -from .common import Y_DTYPE -from .common import MonotonicConstraint -from ._bitset import set_raw_bitset_from_binned_bitset - -EPS = np.finfo(Y_DTYPE).eps # to avoid zero division errors +from .splitting import Splitter class TreeNode: @@ -34,8 +37,12 @@ class TreeNode: ---------- depth : int The depth of the node, i.e. its distance from the root. - sample_indices : ndarray of shape (n_samples_at_node,), dtype=np.uint + sample_indices : ndarray of shape (n_samples_at_node,), dtype=np.uint32 The indices of the samples at the node. + partition_start : int + start position of the node's sample_indices in splitter.partition. + partition_stop : int + stop position of the node's sample_indices in splitter.partition. sum_gradients : float The sum of the gradients of the samples at the node. sum_hessians : float @@ -45,7 +52,7 @@ class TreeNode: ---------- depth : int The depth of the node, i.e. its distance from the root. - sample_indices : ndarray of shape (n_samples_at_node,), dtype=np.uint + sample_indices : ndarray of shape (n_samples_at_node,), dtype=np.uint32 The indices of the samples at the node. sum_gradients : float The sum of the gradients of the samples at the node. @@ -53,6 +60,8 @@ class TreeNode: The sum of the hessians of the samples at the node. split_info : SplitInfo or None The result of the split evaluation. + is_leaf : bool + True if node is a leaf left_child : TreeNode or None The left child of the node. None for leaves. right_child : TreeNode or None @@ -64,26 +73,27 @@ class TreeNode: start position of the node's sample_indices in splitter.partition. partition_stop : int stop position of the node's sample_indices in splitter.partition. + allowed_features : None or ndarray, dtype=int + Indices of features allowed to split for children. + interaction_cst_indices : None or list of ints + Indices of the interaction sets that have to be applied on splits of + child nodes. The fewer sets the stronger the constraint as fewer sets + contain fewer features. + children_lower_bound : float + children_upper_bound : float """ - split_info = None - left_child = None - right_child = None - histograms = None - - # start and stop indices of the node in the splitter.partition - # array. Concretely, - # self.sample_indices = view(self.splitter.partition[start:stop]) - # Please see the comments about splitter.partition and - # splitter.split_indices for more info about this design. - # These 2 attributes are only used in _update_raw_prediction, because we - # need to iterate over the leaves and I don't know how to efficiently - # store the sample_indices views because they're all of different sizes. - partition_start = 0 - partition_stop = 0 - - def __init__(self, depth, sample_indices, sum_gradients, - sum_hessians, value=None): + def __init__( + self, + *, + depth, + sample_indices, + partition_start, + partition_stop, + sum_gradients, + sum_hessians, + value=None, + ): self.depth = depth self.sample_indices = sample_indices self.n_samples = sample_indices.shape[0] @@ -91,7 +101,23 @@ def __init__(self, depth, sample_indices, sum_gradients, self.sum_hessians = sum_hessians self.value = value self.is_leaf = False - self.set_children_bounds(float('-inf'), float('+inf')) + self.allowed_features = None + self.interaction_cst_indices = None + self.set_children_bounds(float("-inf"), float("+inf")) + self.split_info = None + self.left_child = None + self.right_child = None + self.histograms = None + # start and stop indices of the node in the splitter.partition + # array. Concretely, + # self.sample_indices = view(self.splitter.partition[start:stop]) + # Please see the comments about splitter.partition and + # splitter.split_indices for more info about this design. + # These 2 attributes are only used in _update_raw_prediction, because we + # need to iterate over the leaves and I don't know how to efficiently + # store the sample_indices views because they're all of different sizes. + self.partition_start = partition_start + self.partition_stop = partition_stop def set_children_bounds(self, lower, upper): """Set children values bounds to respect monotonic constraints.""" @@ -148,6 +174,10 @@ class TreeGrower: min_gain_to_split : float, default=0. The minimum gain needed to split a node. Splits with lower gain will be ignored. + min_hessian_to_split : float, default=1e-3 + The minimum sum of hessians needed in each node. Splits that result in + at least one child having a sum of hessians less than + ``min_hessian_to_split`` are discarded. n_bins : int, default=256 The total number of bins, including the bin for missing values. Used to define the shape of the histograms. @@ -162,143 +192,194 @@ class TreeGrower: If it's a bool, the same value is used for all features. is_categorical : ndarray of bool of shape (n_features,), default=None Indicates categorical features. - monotonic_cst : array-like of shape (n_features,), dtype=int, default=None - Indicates the monotonic constraint to enforce on each feature. -1, 1 - and 0 respectively correspond to a positive constraint, negative - constraint and no constraint. Read more in the :ref:`User Guide - `. + monotonic_cst : array-like of int of shape (n_features,), dtype=int, default=None + Indicates the monotonic constraint to enforce on each feature. + - 1: monotonic increase + - 0: no constraint + - -1: monotonic decrease + + Read more in the :ref:`User Guide `. + interaction_cst : list of sets of integers, default=None + List of interaction constraints. l2_regularization : float, default=0. - The L2 regularization parameter. - min_hessian_to_split : float, default=1e-3 - The minimum sum of hessians needed in each node. Splits that result in - at least one child having a sum of hessians less than - ``min_hessian_to_split`` are discarded. + The L2 regularization parameter penalizing leaves with small hessians. + Use ``0`` for no regularization (default). + feature_fraction_per_split : float, default=1 + Proportion of randomly chosen features in each and every node split. + This is a form of regularization, smaller values make the trees weaker + learners and might prevent overfitting. + rng : Generator + Numpy random Generator used for feature subsampling. shrinkage : float, default=1. The shrinkage parameter to apply to the leaves values, also known as learning rate. - """ + n_threads : int, default=None + Number of OpenMP threads to use. `_openmp_effective_n_threads` is called + to determine the effective number of threads use, which takes cgroups CPU + quotes into account. See the docstring of `_openmp_effective_n_threads` + for details. - def __init__(self, X_binned, gradients, hessians, max_leaf_nodes=None, - max_depth=None, min_samples_leaf=20, min_gain_to_split=0., - n_bins=256, n_bins_non_missing=None, has_missing_values=False, - is_categorical=None, monotonic_cst=None, - l2_regularization=0., min_hessian_to_split=1e-3, - shrinkage=1.): + Attributes + ---------- + histogram_builder : HistogramBuilder + splitter : Splitter + root : TreeNode + finalized_leaves : list of TreeNode + splittable_nodes : list of TreeNode + missing_values_bin_idx : int + Equals n_bins - 1 + n_categorical_splits : int + n_features : int + n_nodes : int + total_find_split_time : float + Time spent finding the best splits + total_compute_hist_time : float + Time spent computing histograms + total_apply_split_time : float + Time spent splitting nodes + with_monotonic_cst : bool + Whether there are monotonic constraints that apply. False iff monotonic_cst is + None. + """ - self._validate_parameters(X_binned, max_leaf_nodes, max_depth, - min_samples_leaf, min_gain_to_split, - l2_regularization, min_hessian_to_split) + def __init__( + self, + X_binned, + gradients, + hessians, + max_leaf_nodes=None, + max_depth=None, + min_samples_leaf=20, + min_gain_to_split=0.0, + min_hessian_to_split=1e-3, + n_bins=256, + n_bins_non_missing=None, + has_missing_values=False, + is_categorical=None, + monotonic_cst=None, + interaction_cst=None, + l2_regularization=0.0, + feature_fraction_per_split=1.0, + rng=np.random.default_rng(), + shrinkage=1.0, + n_threads=None, + ): + self._validate_parameters( + X_binned, + min_gain_to_split, + min_hessian_to_split, + ) + n_threads = _openmp_effective_n_threads(n_threads) if n_bins_non_missing is None: n_bins_non_missing = n_bins - 1 if isinstance(n_bins_non_missing, numbers.Integral): n_bins_non_missing = np.array( - [n_bins_non_missing] * X_binned.shape[1], - dtype=np.uint32) + [n_bins_non_missing] * X_binned.shape[1], dtype=np.uint32 + ) else: - n_bins_non_missing = np.asarray(n_bins_non_missing, - dtype=np.uint32) + n_bins_non_missing = np.asarray(n_bins_non_missing, dtype=np.uint32) if isinstance(has_missing_values, bool): has_missing_values = [has_missing_values] * X_binned.shape[1] has_missing_values = np.asarray(has_missing_values, dtype=np.uint8) + # `monotonic_cst` validation is done in _validate_monotonic_cst + # at the estimator level and therefore the following should not be + # needed when using the public API. if monotonic_cst is None: - self.with_monotonic_cst = False - monotonic_cst = np.full(shape=X_binned.shape[1], - fill_value=MonotonicConstraint.NO_CST, - dtype=np.int8) + monotonic_cst = np.full( + shape=X_binned.shape[1], + fill_value=MonotonicConstraint.NO_CST, + dtype=np.int8, + ) else: - self.with_monotonic_cst = True monotonic_cst = np.asarray(monotonic_cst, dtype=np.int8) - - if monotonic_cst.shape[0] != X_binned.shape[1]: - raise ValueError( - "monotonic_cst has shape {} but the input data " - "X has {} features.".format( - monotonic_cst.shape[0], X_binned.shape[1] - ) - ) - if np.any(monotonic_cst < -1) or np.any(monotonic_cst > 1): - raise ValueError( - "monotonic_cst must be None or an array-like of " - "-1, 0 or 1." - ) + self.with_monotonic_cst = np.any(monotonic_cst != MonotonicConstraint.NO_CST) if is_categorical is None: is_categorical = np.zeros(shape=X_binned.shape[1], dtype=np.uint8) else: is_categorical = np.asarray(is_categorical, dtype=np.uint8) - if np.any(np.logical_and(is_categorical == 1, - monotonic_cst != MonotonicConstraint.NO_CST)): - raise ValueError("Categorical features cannot have monotonic " - "constraints.") + if np.any( + np.logical_and( + is_categorical == 1, monotonic_cst != MonotonicConstraint.NO_CST + ) + ): + raise ValueError("Categorical features cannot have monotonic constraints.") hessians_are_constant = hessians.shape[0] == 1 self.histogram_builder = HistogramBuilder( - X_binned, n_bins, gradients, hessians, hessians_are_constant) + X_binned, n_bins, gradients, hessians, hessians_are_constant, n_threads + ) missing_values_bin_idx = n_bins - 1 self.splitter = Splitter( - X_binned, n_bins_non_missing, missing_values_bin_idx, - has_missing_values, is_categorical, monotonic_cst, - l2_regularization, min_hessian_to_split, - min_samples_leaf, min_gain_to_split, hessians_are_constant) + X_binned=X_binned, + n_bins_non_missing=n_bins_non_missing, + missing_values_bin_idx=missing_values_bin_idx, + has_missing_values=has_missing_values, + is_categorical=is_categorical, + monotonic_cst=monotonic_cst, + l2_regularization=l2_regularization, + min_hessian_to_split=min_hessian_to_split, + min_samples_leaf=min_samples_leaf, + min_gain_to_split=min_gain_to_split, + hessians_are_constant=hessians_are_constant, + feature_fraction_per_split=feature_fraction_per_split, + rng=rng, + n_threads=n_threads, + ) + self.X_binned = X_binned + self.max_leaf_nodes = max_leaf_nodes + self.max_depth = max_depth + self.min_samples_leaf = min_samples_leaf + self.min_gain_to_split = min_gain_to_split self.n_bins_non_missing = n_bins_non_missing self.missing_values_bin_idx = missing_values_bin_idx - self.max_leaf_nodes = max_leaf_nodes self.has_missing_values = has_missing_values - self.monotonic_cst = monotonic_cst self.is_categorical = is_categorical + self.monotonic_cst = monotonic_cst + self.interaction_cst = interaction_cst self.l2_regularization = l2_regularization - self.n_features = X_binned.shape[1] - self.max_depth = max_depth - self.min_samples_leaf = min_samples_leaf - self.X_binned = X_binned - self.min_gain_to_split = min_gain_to_split self.shrinkage = shrinkage + self.n_features = X_binned.shape[1] + self.n_threads = n_threads self.splittable_nodes = [] self.finalized_leaves = [] - self.total_find_split_time = 0. # time spent finding the best splits - self.total_compute_hist_time = 0. # time spent computing histograms - self.total_apply_split_time = 0. # time spent splitting nodes + self.total_find_split_time = 0.0 # time spent finding the best splits + self.total_compute_hist_time = 0.0 # time spent computing histograms + self.total_apply_split_time = 0.0 # time spent splitting nodes self.n_categorical_splits = 0 - self._intilialize_root(gradients, hessians, hessians_are_constant) + self._initialize_root() self.n_nodes = 1 - def _validate_parameters(self, X_binned, max_leaf_nodes, max_depth, - min_samples_leaf, min_gain_to_split, - l2_regularization, min_hessian_to_split): + def _validate_parameters( + self, + X_binned, + min_gain_to_split, + min_hessian_to_split, + ): """Validate parameters passed to __init__. Also validate parameters passed to splitter. """ if X_binned.dtype != np.uint8: - raise NotImplementedError( - "X_binned must be of type uint8.") + raise NotImplementedError("X_binned must be of type uint8.") if not X_binned.flags.f_contiguous: raise ValueError( "X_binned should be passed as Fortran contiguous " - "array for maximum efficiency.") - if max_leaf_nodes is not None and max_leaf_nodes <= 1: - raise ValueError('max_leaf_nodes={} should not be' - ' smaller than 2'.format(max_leaf_nodes)) - if max_depth is not None and max_depth < 1: - raise ValueError('max_depth={} should not be' - ' smaller than 1'.format(max_depth)) - if min_samples_leaf < 1: - raise ValueError('min_samples_leaf={} should ' - 'not be smaller than 1'.format(min_samples_leaf)) + "array for maximum efficiency." + ) if min_gain_to_split < 0: - raise ValueError('min_gain_to_split={} ' - 'must be positive.'.format(min_gain_to_split)) - if l2_regularization < 0: - raise ValueError('l2_regularization={} must be ' - 'positive.'.format(l2_regularization)) + raise ValueError( + "min_gain_to_split={} must be positive.".format(min_gain_to_split) + ) if min_hessian_to_split < 0: - raise ValueError('min_hessian_to_split={} ' - 'must be positive.'.format(min_hessian_to_split)) + raise ValueError( + "min_hessian_to_split={} must be positive.".format(min_hessian_to_split) + ) def grow(self): """Grow the tree, from root to leaves.""" @@ -319,26 +400,48 @@ def _apply_shrinkage(self): for leaf in self.finalized_leaves: leaf.value *= self.shrinkage - def _intilialize_root(self, gradients, hessians, hessians_are_constant): + def _initialize_root(self): """Initialize root node and finalize it if needed.""" + tic = time() + if self.interaction_cst is not None: + allowed_features = set().union(*self.interaction_cst) + allowed_features = np.fromiter( + allowed_features, dtype=np.uint32, count=len(allowed_features) + ) + arbitrary_feature = allowed_features[0] + else: + allowed_features = None + arbitrary_feature = 0 + + # TreeNode init needs the total sum of gradients and hessians. Therefore, we + # first compute the histograms and then compute the total grad/hess on an + # arbitrary feature histogram. This way we replace a loop over n_samples by a + # loop over n_bins. + histograms = self.histogram_builder.compute_histograms_brute( + self.splitter.partition, # =self.root.sample_indices + allowed_features, + ) + self.total_compute_hist_time += time() - tic + + tic = time() n_samples = self.X_binned.shape[0] depth = 0 - sum_gradients = sum_parallel(gradients) + histogram_array = np.asarray(histograms[arbitrary_feature]) + sum_gradients = histogram_array["sum_gradients"].sum() if self.histogram_builder.hessians_are_constant: - sum_hessians = hessians[0] * n_samples + sum_hessians = self.histogram_builder.hessians[0] * n_samples else: - sum_hessians = sum_parallel(hessians) + sum_hessians = histogram_array["sum_hessians"].sum() self.root = TreeNode( depth=depth, sample_indices=self.splitter.partition, + partition_start=0, + partition_stop=n_samples, sum_gradients=sum_gradients, sum_hessians=sum_hessians, - value=0 + value=0, ) - self.root.partition_start = 0 - self.root.partition_stop = n_samples - if self.root.n_samples < 2 * self.min_samples_leaf: # Do not even bother computing any splitting statistics. self._finalize_leaf(self.root) @@ -347,9 +450,14 @@ def _intilialize_root(self, gradients, hessians, hessians_are_constant): self._finalize_leaf(self.root) return - self.root.histograms = self.histogram_builder.compute_histograms_brute( - self.root.sample_indices) + if self.interaction_cst is not None: + self.root.interaction_cst_indices = range(len(self.interaction_cst)) + self.root.allowed_features = allowed_features + + self.root.histograms = histograms + self._compute_best_split_and_push(self.root) + self.total_find_split_time += time() - tic def _compute_best_split_and_push(self, node): """Compute the best possible split (SplitInfo) of a given node. @@ -361,9 +469,15 @@ def _compute_best_split_and_push(self, node): """ node.split_info = self.splitter.find_node_split( - node.n_samples, node.histograms, node.sum_gradients, - node.sum_hessians, node.value, node.children_lower_bound, - node.children_upper_bound) + n_samples=node.n_samples, + histograms=node.histograms, + sum_gradients=node.sum_gradients, + sum_hessians=node.sum_hessians, + value=node.value, + lower_bound=node.children_lower_bound, + upper_bound=node.children_upper_bound, + allowed_features=node.allowed_features, + ) if node.split_info.gain <= 0: # no valid split self._finalize_leaf(node) @@ -384,50 +498,64 @@ def split_next(self): node = heappop(self.splittable_nodes) tic = time() - (sample_indices_left, - sample_indices_right, - right_child_pos) = self.splitter.split_indices(node.split_info, - node.sample_indices) + ( + sample_indices_left, + sample_indices_right, + right_child_pos, + ) = self.splitter.split_indices(node.split_info, node.sample_indices) self.total_apply_split_time += time() - tic depth = node.depth + 1 n_leaf_nodes = len(self.finalized_leaves) + len(self.splittable_nodes) n_leaf_nodes += 2 - left_child_node = TreeNode(depth, - sample_indices_left, - node.split_info.sum_gradient_left, - node.split_info.sum_hessian_left, - value=node.split_info.value_left, - ) - right_child_node = TreeNode(depth, - sample_indices_right, - node.split_info.sum_gradient_right, - node.split_info.sum_hessian_right, - value=node.split_info.value_right, - ) + left_child_node = TreeNode( + depth=depth, + sample_indices=sample_indices_left, + partition_start=node.partition_start, + partition_stop=node.partition_start + right_child_pos, + sum_gradients=node.split_info.sum_gradient_left, + sum_hessians=node.split_info.sum_hessian_left, + value=node.split_info.value_left, + ) + right_child_node = TreeNode( + depth=depth, + sample_indices=sample_indices_right, + partition_start=left_child_node.partition_stop, + partition_stop=node.partition_stop, + sum_gradients=node.split_info.sum_gradient_right, + sum_hessians=node.split_info.sum_hessian_right, + value=node.split_info.value_right, + ) node.right_child = right_child_node node.left_child = left_child_node - # set start and stop indices - left_child_node.partition_start = node.partition_start - left_child_node.partition_stop = node.partition_start + right_child_pos - right_child_node.partition_start = left_child_node.partition_stop - right_child_node.partition_stop = node.partition_stop + # set interaction constraints (the indices of the constraints sets) + if self.interaction_cst is not None: + # Calculate allowed_features and interaction_cst_indices only once. Child + # nodes inherit them before they get split. + ( + left_child_node.allowed_features, + left_child_node.interaction_cst_indices, + ) = self._compute_interactions(node) + right_child_node.interaction_cst_indices = ( + left_child_node.interaction_cst_indices + ) + right_child_node.allowed_features = left_child_node.allowed_features if not self.has_missing_values[node.split_info.feature_idx]: # If no missing values are encountered at fit time, then samples # with missing values during predict() will go to whichever child # has the most samples. node.split_info.missing_go_to_left = ( - left_child_node.n_samples > right_child_node.n_samples) + left_child_node.n_samples > right_child_node.n_samples + ) self.n_nodes += 2 self.n_categorical_splits += node.split_info.is_categorical - if (self.max_leaf_nodes is not None - and n_leaf_nodes == self.max_leaf_nodes): + if self.max_leaf_nodes is not None and n_leaf_nodes == self.max_leaf_nodes: self._finalize_leaf(left_child_node) self._finalize_leaf(right_child_node) self._finalize_splittable_nodes() @@ -446,14 +574,18 @@ def split_next(self): if self.with_monotonic_cst: # Set value bounds for respecting monotonic constraints # See test_nodes_values() for details - if (self.monotonic_cst[node.split_info.feature_idx] == - MonotonicConstraint.NO_CST): + if ( + self.monotonic_cst[node.split_info.feature_idx] + == MonotonicConstraint.NO_CST + ): lower_left = lower_right = node.children_lower_bound upper_left = upper_right = node.children_upper_bound else: mid = (left_child_node.value + right_child_node.value) / 2 - if (self.monotonic_cst[node.split_info.feature_idx] == - MonotonicConstraint.POS): + if ( + self.monotonic_cst[node.split_info.feature_idx] + == MonotonicConstraint.POS + ): lower_left, upper_left = node.children_lower_bound, mid lower_right, upper_right = mid, node.children_upper_bound else: # NEG @@ -467,7 +599,6 @@ def split_next(self): should_split_left = not left_child_node.is_leaf should_split_right = not right_child_node.is_leaf if should_split_left or should_split_right: - # We will compute the histograms of both nodes even if one of them # is a leaf, since computing the second histogram is very cheap # (using histogram subtraction). @@ -483,13 +614,21 @@ def split_next(self): # We use the brute O(n_samples) method on the child that has the # smallest number of samples, and the subtraction trick O(n_bins) # on the other one. + # Note that both left and right child have the same allowed_features. tic = time() - smallest_child.histograms = \ - self.histogram_builder.compute_histograms_brute( - smallest_child.sample_indices) - largest_child.histograms = \ + smallest_child.histograms = self.histogram_builder.compute_histograms_brute( + smallest_child.sample_indices, smallest_child.allowed_features + ) + largest_child.histograms = ( self.histogram_builder.compute_histograms_subtraction( - node.histograms, smallest_child.histograms) + node.histograms, + smallest_child.histograms, + smallest_child.allowed_features, + ) + ) + # node.histograms is reused in largest_child.histograms. To break cyclic + # memory references and help garbage collection, we set it to None. + node.histograms = None self.total_compute_hist_time += time() - tic tic = time() @@ -511,6 +650,48 @@ def split_next(self): return left_child_node, right_child_node + def _compute_interactions(self, node): + r"""Compute features allowed by interactions to be inherited by child nodes. + + Example: Assume constraints [{0, 1}, {1, 2}]. + 1 <- Both constraint groups could be applied from now on + / \ + 1 2 <- Left split still fulfills both constraint groups. + / \ / \ Right split at feature 2 has only group {1, 2} from now on. + + LightGBM uses the same logic for overlapping groups. See + https://github.com/microsoft/LightGBM/issues/4481 for details. + + Parameters: + ---------- + node : TreeNode + A node that might have children. Based on its feature_idx, the interaction + constraints for possible child nodes are computed. + + Returns + ------- + allowed_features : ndarray, dtype=uint32 + Indices of features allowed to split for children. + interaction_cst_indices : list of ints + Indices of the interaction sets that have to be applied on splits of + child nodes. The fewer sets the stronger the constraint as fewer sets + contain fewer features. + """ + # Note: + # - Case of no interactions is already captured before function call. + # - This is for nodes that are already split and have a + # node.split_info.feature_idx. + allowed_features = set() + interaction_cst_indices = [] + for i in node.interaction_cst_indices: + if node.split_info.feature_idx in self.interaction_cst[i]: + interaction_cst_indices.append(i) + allowed_features.update(self.interaction_cst[i]) + return ( + np.fromiter(allowed_features, dtype=np.uint32, count=len(allowed_features)), + interaction_cst_indices, + ) + def _finalize_leaf(self, node): """Make node a leaf of the tree being grown.""" @@ -543,77 +724,98 @@ def make_predictor(self, binning_thresholds): A TreePredictor object. """ predictor_nodes = np.zeros(self.n_nodes, dtype=PREDICTOR_RECORD_DTYPE) - binned_left_cat_bitsets = np.zeros((self.n_categorical_splits, 8), - dtype=X_BITSET_INNER_DTYPE) - raw_left_cat_bitsets = np.zeros((self.n_categorical_splits, 8), - dtype=X_BITSET_INNER_DTYPE) - _fill_predictor_arrays(predictor_nodes, binned_left_cat_bitsets, - raw_left_cat_bitsets, - self.root, binning_thresholds, - self.n_bins_non_missing) - return TreePredictor(predictor_nodes, binned_left_cat_bitsets, - raw_left_cat_bitsets) - - -def _fill_predictor_arrays(predictor_nodes, binned_left_cat_bitsets, - raw_left_cat_bitsets, grower_node, - binning_thresholds, n_bins_non_missing, - next_free_node_idx=0, next_free_bitset_idx=0): + binned_left_cat_bitsets = np.zeros( + (self.n_categorical_splits, 8), dtype=X_BITSET_INNER_DTYPE + ) + raw_left_cat_bitsets = np.zeros( + (self.n_categorical_splits, 8), dtype=X_BITSET_INNER_DTYPE + ) + _fill_predictor_arrays( + predictor_nodes, + binned_left_cat_bitsets, + raw_left_cat_bitsets, + self.root, + binning_thresholds, + self.n_bins_non_missing, + ) + return TreePredictor( + predictor_nodes, binned_left_cat_bitsets, raw_left_cat_bitsets + ) + + +def _fill_predictor_arrays( + predictor_nodes, + binned_left_cat_bitsets, + raw_left_cat_bitsets, + grower_node, + binning_thresholds, + n_bins_non_missing, + next_free_node_idx=0, + next_free_bitset_idx=0, +): """Helper used in make_predictor to set the TreePredictor fields.""" node = predictor_nodes[next_free_node_idx] - node['count'] = grower_node.n_samples - node['depth'] = grower_node.depth + node["count"] = grower_node.n_samples + node["depth"] = grower_node.depth if grower_node.split_info is not None: - node['gain'] = grower_node.split_info.gain + node["gain"] = grower_node.split_info.gain else: - node['gain'] = -1 + node["gain"] = -1 - node['value'] = grower_node.value + node["value"] = grower_node.value if grower_node.is_leaf: # Leaf node - node['is_leaf'] = True + node["is_leaf"] = True return next_free_node_idx + 1, next_free_bitset_idx split_info = grower_node.split_info feature_idx, bin_idx = split_info.feature_idx, split_info.bin_idx - node['feature_idx'] = feature_idx - node['bin_threshold'] = bin_idx - node['missing_go_to_left'] = split_info.missing_go_to_left - node['is_categorical'] = split_info.is_categorical + node["feature_idx"] = feature_idx + node["bin_threshold"] = bin_idx + node["missing_go_to_left"] = split_info.missing_go_to_left + node["is_categorical"] = split_info.is_categorical if split_info.bin_idx == n_bins_non_missing[feature_idx] - 1: # Split is on the last non-missing bin: it's a "split on nans". # All nans go to the right, the rest go to the left. # Note: for categorical splits, bin_idx is 0 and we rely on the bitset - node['num_threshold'] = np.inf + node["num_threshold"] = np.inf elif split_info.is_categorical: categories = binning_thresholds[feature_idx] - node['bitset_idx'] = next_free_bitset_idx - binned_left_cat_bitsets[next_free_bitset_idx] = ( - split_info.left_cat_bitset) + node["bitset_idx"] = next_free_bitset_idx + binned_left_cat_bitsets[next_free_bitset_idx] = split_info.left_cat_bitset set_raw_bitset_from_binned_bitset( raw_left_cat_bitsets[next_free_bitset_idx], - split_info.left_cat_bitset, categories + split_info.left_cat_bitset, + categories, ) next_free_bitset_idx += 1 else: - node['num_threshold'] = binning_thresholds[feature_idx][bin_idx] + node["num_threshold"] = binning_thresholds[feature_idx][bin_idx] next_free_node_idx += 1 - node['left'] = next_free_node_idx + node["left"] = next_free_node_idx next_free_node_idx, next_free_bitset_idx = _fill_predictor_arrays( - predictor_nodes, binned_left_cat_bitsets, raw_left_cat_bitsets, - grower_node.left_child, binning_thresholds=binning_thresholds, + predictor_nodes, + binned_left_cat_bitsets, + raw_left_cat_bitsets, + grower_node.left_child, + binning_thresholds=binning_thresholds, n_bins_non_missing=n_bins_non_missing, next_free_node_idx=next_free_node_idx, - next_free_bitset_idx=next_free_bitset_idx) + next_free_bitset_idx=next_free_bitset_idx, + ) - node['right'] = next_free_node_idx + node["right"] = next_free_node_idx return _fill_predictor_arrays( - predictor_nodes, binned_left_cat_bitsets, raw_left_cat_bitsets, - grower_node.right_child, binning_thresholds=binning_thresholds, + predictor_nodes, + binned_left_cat_bitsets, + raw_left_cat_bitsets, + grower_node.right_child, + binning_thresholds=binning_thresholds, n_bins_non_missing=n_bins_non_missing, next_free_node_idx=next_free_node_idx, - next_free_bitset_idx=next_free_bitset_idx) + next_free_bitset_idx=next_free_bitset_idx, + ) diff --git a/sklearn/ensemble/_hist_gradient_boosting/histogram.pyx b/sklearn/ensemble/_hist_gradient_boosting/histogram.pyx index 625aecc4f09f5..e204eec6b9785 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/histogram.pyx +++ b/sklearn/ensemble/_hist_gradient_boosting/histogram.pyx @@ -1,23 +1,20 @@ -# cython: cdivision=True -# cython: boundscheck=False -# cython: wraparound=False -# cython: language_level=3 """This module contains routines for building histograms.""" -# Author: Nicolas Hug +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause cimport cython from cython.parallel import prange +from libc.string cimport memset import numpy as np -cimport numpy as np from .common import HISTOGRAM_DTYPE from .common cimport hist_struct from .common cimport X_BINNED_DTYPE_C from .common cimport G_H_DTYPE_C +from ...utils._typedefs cimport uint8_t -np.import_array() # Notes: # - IN views are read-only, OUT views are write-only @@ -53,7 +50,7 @@ cdef class HistogramBuilder: There are different ways to build a histogram: - by subtraction: hist(child) = hist(parent) - hist(sibling) - - from scratch. In this case we have rountines that update the hessians + - from scratch. In this case we have routines that update the hessians or not (not useful when hessians are constant for some losses e.g. least squares). Also, there's a special case for the root which contains all the samples, leading to some possible optimizations. @@ -84,12 +81,14 @@ cdef class HistogramBuilder: G_H_DTYPE_C [::1] hessians G_H_DTYPE_C [::1] ordered_gradients G_H_DTYPE_C [::1] ordered_hessians - unsigned char hessians_are_constant + uint8_t hessians_are_constant + int n_threads def __init__(self, const X_BINNED_DTYPE_C [::1, :] X_binned, unsigned int n_bins, G_H_DTYPE_C [::1] gradients, G_H_DTYPE_C [::1] hessians, - unsigned char hessians_are_constant): + uint8_t hessians_are_constant, + int n_threads): self.X_binned = X_binned self.n_features = X_binned.shape[1] @@ -102,10 +101,13 @@ cdef class HistogramBuilder: self.ordered_gradients = gradients.copy() self.ordered_hessians = hessians.copy() self.hessians_are_constant = hessians_are_constant + self.n_threads = n_threads def compute_histograms_brute( - HistogramBuilder self, - const unsigned int [::1] sample_indices): # IN + HistogramBuilder self, + const unsigned int [::1] sample_indices, # IN + const unsigned int [:] allowed_features=None, # IN + ): """Compute the histograms of the node by scanning through all the data. For a given feature, the complexity is O(n_samples) @@ -115,6 +117,10 @@ cdef class HistogramBuilder: sample_indices : array of int, shape (n_samples_at_node,) The indices of the samples at the node to split. + allowed_features : None or ndarray, dtype=np.uint32 + Indices of the features that are allowed by interaction constraints to be + split. + Returns ------- histograms : ndarray of HISTOGRAM_DTYPE, shape (n_features, n_bins) @@ -123,11 +129,11 @@ cdef class HistogramBuilder: cdef: int n_samples int feature_idx + int f_idx int i # need local views to avoid python interactions - unsigned char hessians_are_constant = \ - self.hessians_are_constant - int n_features = self.n_features + uint8_t hessians_are_constant = self.hessians_are_constant + int n_allowed_features = self.n_features G_H_DTYPE_C [::1] ordered_gradients = self.ordered_gradients G_H_DTYPE_C [::1] gradients = self.gradients G_H_DTYPE_C [::1] ordered_hessians = self.ordered_hessians @@ -137,6 +143,11 @@ cdef class HistogramBuilder: shape=(self.n_features, self.n_bins), dtype=HISTOGRAM_DTYPE ) + bint has_interaction_cst = allowed_features is not None + int n_threads = self.n_threads + + if has_interaction_cst: + n_allowed_features = allowed_features.shape[0] with nogil: n_samples = sample_indices.shape[0] @@ -146,17 +157,27 @@ cdef class HistogramBuilder: # cache hit. if sample_indices.shape[0] != gradients.shape[0]: if hessians_are_constant: - for i in prange(n_samples, schedule='static'): + for i in prange(n_samples, schedule='static', + num_threads=n_threads): ordered_gradients[i] = gradients[sample_indices[i]] else: - for i in prange(n_samples, schedule='static'): + for i in prange(n_samples, schedule='static', + num_threads=n_threads): ordered_gradients[i] = gradients[sample_indices[i]] ordered_hessians[i] = hessians[sample_indices[i]] - for feature_idx in prange(n_features, schedule='static'): - # Compute histogram of each feature + # Compute histogram of each feature + for f_idx in prange( + n_allowed_features, schedule='static', num_threads=n_threads + ): + if has_interaction_cst: + feature_idx = allowed_features[f_idx] + else: + feature_idx = f_idx + self._compute_histogram_brute_single_feature( - feature_idx, sample_indices, histograms) + feature_idx, sample_indices, histograms + ) return histograms @@ -164,7 +185,7 @@ cdef class HistogramBuilder: HistogramBuilder self, const int feature_idx, const unsigned int [::1] sample_indices, # IN - hist_struct [:, ::1] histograms) nogil: # OUT + hist_struct [:, ::1] histograms) noexcept nogil: # OUT """Compute the histogram for a given feature.""" cdef: @@ -176,14 +197,11 @@ cdef class HistogramBuilder: self.ordered_gradients[:n_samples] G_H_DTYPE_C [::1] ordered_hessians = \ self.ordered_hessians[:n_samples] - unsigned char hessians_are_constant = \ + uint8_t hessians_are_constant = \ self.hessians_are_constant - unsigned int bin_idx = 0 - - for bin_idx in range(self.n_bins): - histograms[feature_idx, bin_idx].sum_gradients = 0. - histograms[feature_idx, bin_idx].sum_hessians = 0. - histograms[feature_idx, bin_idx].count = 0 + + # Set histograms to zero. + memset(&histograms[feature_idx, 0], 0, self.n_bins * sizeof(hist_struct)) if root_node: if hessians_are_constant: @@ -205,9 +223,11 @@ cdef class HistogramBuilder: ordered_hessians, histograms) def compute_histograms_subtraction( - HistogramBuilder self, - hist_struct [:, ::1] parent_histograms, # IN - hist_struct [:, ::1] sibling_histograms): # IN + HistogramBuilder self, + hist_struct [:, ::1] parent_histograms, # IN and OUT + hist_struct [:, ::1] sibling_histograms, # IN + const unsigned int [:] allowed_features=None, # IN + ): """Compute the histograms of the node using the subtraction trick. hist(parent) = hist(left_child) + hist(right_child) @@ -224,29 +244,43 @@ cdef class HistogramBuilder: sibling_histograms : ndarray of HISTOGRAM_DTYPE, \ shape (n_features, n_bins) The histograms of the sibling. + allowed_features : None or ndarray, dtype=np.uint32 + Indices of the features that are allowed by interaction constraints to be + split. Returns ------- histograms : ndarray of HISTOGRAM_DTYPE, shape(n_features, n_bins) The computed histograms of the current node. + We repurpose parent_histograms for this and don't need to allocate new + memory. """ cdef: int feature_idx - int n_features = self.n_features - hist_struct [:, ::1] histograms = np.empty( - shape=(self.n_features, self.n_bins), - dtype=HISTOGRAM_DTYPE - ) + int f_idx + int n_allowed_features = self.n_features + bint has_interaction_cst = allowed_features is not None + int n_threads = self.n_threads + + if has_interaction_cst: + n_allowed_features = allowed_features.shape[0] + + # Compute histogram of each feature + for f_idx in prange(n_allowed_features, schedule='static', nogil=True, + num_threads=n_threads): + if has_interaction_cst: + feature_idx = allowed_features[f_idx] + else: + feature_idx = f_idx - for feature_idx in prange(n_features, schedule='static', nogil=True): - # Compute histogram of each feature - _subtract_histograms(feature_idx, - self.n_bins, - parent_histograms, - sibling_histograms, - histograms) - return histograms + _subtract_histograms( + feature_idx, + self.n_bins, + parent_histograms, + sibling_histograms, + ) + return parent_histograms cpdef void _build_histogram_naive( @@ -255,7 +289,7 @@ cpdef void _build_histogram_naive( X_BINNED_DTYPE_C [:] binned_feature, # IN G_H_DTYPE_C [:] ordered_gradients, # IN G_H_DTYPE_C [:] ordered_hessians, # IN - hist_struct [:, :] out) nogil: # OUT + hist_struct [:, :] out) noexcept nogil: # OUT """Build histogram in a naive way, without optimizing for cache hit. Used in tests to compare with the optimized version.""" @@ -276,25 +310,27 @@ cpdef void _build_histogram_naive( cpdef void _subtract_histograms( const int feature_idx, unsigned int n_bins, - hist_struct [:, ::1] hist_a, # IN + hist_struct [:, ::1] hist_a, # IN and OUT hist_struct [:, ::1] hist_b, # IN - hist_struct [:, ::1] out) nogil: # OUT - """compute (hist_a - hist_b) in out""" +) noexcept nogil: # OUT + """compute hist_a = hist_a - hist_b""" + # Note that subtraction of large sums of floating point numbers, as we have here, + # can exhibit catastrophic cancallation. This is in particular true for gradients + # as they can be positive and negative, while hessians are non-negative. + # Remember that gradients and hessians are originally computed in + # G_H_DTYPE_C = float32 precision. Therefore, if sum_gradients and sum_hessians are + # float64, we don't loose precision. But if we also used float32 for summation, we + # would need to take care of floating point errors. + # + # Note that we could protect for negative hessians by setting: + # sum_hessians = max(0, sum_hessians) + # But as we use float64 for summing float32, that's veeeery unlikely. cdef: unsigned int i = 0 for i in range(n_bins): - out[feature_idx, i].sum_gradients = ( - hist_a[feature_idx, i].sum_gradients - - hist_b[feature_idx, i].sum_gradients - ) - out[feature_idx, i].sum_hessians = ( - hist_a[feature_idx, i].sum_hessians - - hist_b[feature_idx, i].sum_hessians - ) - out[feature_idx, i].count = ( - hist_a[feature_idx, i].count - - hist_b[feature_idx, i].count - ) + hist_a[feature_idx, i].sum_gradients -= hist_b[feature_idx, i].sum_gradients + hist_a[feature_idx, i].sum_hessians -= hist_b[feature_idx, i].sum_hessians + hist_a[feature_idx, i].count -= hist_b[feature_idx, i].count cpdef void _build_histogram( @@ -303,7 +339,7 @@ cpdef void _build_histogram( const X_BINNED_DTYPE_C [::1] binned_feature, # IN const G_H_DTYPE_C [::1] ordered_gradients, # IN const G_H_DTYPE_C [::1] ordered_hessians, # IN - hist_struct [:, ::1] out) nogil: # OUT + hist_struct [:, ::1] out) noexcept nogil: # OUT """Return histogram for a given feature.""" cdef: unsigned int i = 0 @@ -349,7 +385,7 @@ cpdef void _build_histogram_no_hessian( const unsigned int [::1] sample_indices, # IN const X_BINNED_DTYPE_C [::1] binned_feature, # IN const G_H_DTYPE_C [::1] ordered_gradients, # IN - hist_struct [:, ::1] out) nogil: # OUT + hist_struct [:, ::1] out) noexcept nogil: # OUT """Return histogram for a given feature, not updating hessians. Used when the hessians of the loss are constant (typically LS loss). @@ -393,7 +429,7 @@ cpdef void _build_histogram_root( const X_BINNED_DTYPE_C [::1] binned_feature, # IN const G_H_DTYPE_C [::1] all_gradients, # IN const G_H_DTYPE_C [::1] all_hessians, # IN - hist_struct [:, ::1] out) nogil: # OUT + hist_struct [:, ::1] out) noexcept nogil: # OUT """Compute histogram of the root node. Unlike other nodes, the root node has to find the split among *all* the @@ -445,7 +481,7 @@ cpdef void _build_histogram_root_no_hessian( const int feature_idx, const X_BINNED_DTYPE_C [::1] binned_feature, # IN const G_H_DTYPE_C [::1] all_gradients, # IN - hist_struct [:, ::1] out) nogil: # OUT + hist_struct [:, ::1] out) noexcept nogil: # OUT """Compute histogram of the root node, not updating hessians. Used when the hessians of the loss are constant (typically LS loss). diff --git a/sklearn/ensemble/_hist_gradient_boosting/loss.py b/sklearn/ensemble/_hist_gradient_boosting/loss.py deleted file mode 100644 index 4bbf59dc01088..0000000000000 --- a/sklearn/ensemble/_hist_gradient_boosting/loss.py +++ /dev/null @@ -1,427 +0,0 @@ -""" -This module contains the loss classes. - -Specific losses are used for regression, binary classification or multiclass -classification. -""" -# Author: Nicolas Hug - -from abc import ABC, abstractmethod - -import numpy as np -from scipy.special import expit, logsumexp, xlogy - -from .common import Y_DTYPE -from .common import G_H_DTYPE -from ._loss import _update_gradients_least_squares -from ._loss import _update_gradients_hessians_least_squares -from ._loss import _update_gradients_least_absolute_deviation -from ._loss import _update_gradients_hessians_least_absolute_deviation -from ._loss import _update_gradients_hessians_binary_crossentropy -from ._loss import _update_gradients_hessians_categorical_crossentropy -from ._loss import _update_gradients_hessians_poisson -from ...utils.stats import _weighted_percentile - - -class BaseLoss(ABC): - """Base class for a loss.""" - - def __init__(self, hessians_are_constant): - self.hessians_are_constant = hessians_are_constant - - def __call__(self, y_true, raw_predictions, sample_weight): - """Return the weighted average loss""" - return np.average(self.pointwise_loss(y_true, raw_predictions), - weights=sample_weight) - - @abstractmethod - def pointwise_loss(self, y_true, raw_predictions): - """Return loss value for each input""" - - # This variable indicates whether the loss requires the leaves values to - # be updated once the tree has been trained. The trees are trained to - # predict a Newton-Raphson step (see grower._finalize_leaf()). But for - # some losses (e.g. least absolute deviation) we need to adjust the tree - # values to account for the "line search" of the gradient descent - # procedure. See the original paper Greedy Function Approximation: A - # Gradient Boosting Machine by Friedman - # (https://statweb.stanford.edu/~jhf/ftp/trebst.pdf) for the theory. - need_update_leaves_values = False - - def init_gradients_and_hessians(self, n_samples, prediction_dim, - sample_weight): - """Return initial gradients and hessians. - - Unless hessians are constant, arrays are initialized with undefined - values. - - Parameters - ---------- - n_samples : int - The number of samples passed to `fit()`. - - prediction_dim : int - The dimension of a raw prediction, i.e. the number of trees - built at each iteration. Equals 1 for regression and binary - classification, or K where K is the number of classes for - multiclass classification. - - sample_weight : array-like of shape(n_samples,) default=None - Weights of training data. - - Returns - ------- - gradients : ndarray, shape (prediction_dim, n_samples) - The initial gradients. The array is not initialized. - hessians : ndarray, shape (prediction_dim, n_samples) - If hessians are constant (e.g. for `LeastSquares` loss, the - array is initialized to ``1``. Otherwise, the array is allocated - without being initialized. - """ - shape = (prediction_dim, n_samples) - gradients = np.empty(shape=shape, dtype=G_H_DTYPE) - - if self.hessians_are_constant: - # If the hessians are constant, we consider they are equal to 1. - # - This is correct for the half LS loss - # - For LAD loss, hessians are actually 0, but they are always - # ignored anyway. - hessians = np.ones(shape=(1, 1), dtype=G_H_DTYPE) - else: - hessians = np.empty(shape=shape, dtype=G_H_DTYPE) - - return gradients, hessians - - @abstractmethod - def get_baseline_prediction(self, y_train, sample_weight, prediction_dim): - """Return initial predictions (before the first iteration). - - Parameters - ---------- - y_train : ndarray, shape (n_samples,) - The target training values. - - sample_weight : array-like of shape(n_samples,) default=None - Weights of training data. - - prediction_dim : int - The dimension of one prediction: 1 for binary classification and - regression, n_classes for multiclass classification. - - Returns - ------- - baseline_prediction : float or ndarray, shape (1, prediction_dim) - The baseline prediction. - """ - - @abstractmethod - def update_gradients_and_hessians(self, gradients, hessians, y_true, - raw_predictions, sample_weight): - """Update gradients and hessians arrays, inplace. - - The gradients (resp. hessians) are the first (resp. second) order - derivatives of the loss for each sample with respect to the - predictions of model, evaluated at iteration ``i - 1``. - - Parameters - ---------- - gradients : ndarray, shape (prediction_dim, n_samples) - The gradients (treated as OUT array). - - hessians : ndarray, shape (prediction_dim, n_samples) or \ - (1,) - The hessians (treated as OUT array). - - y_true : ndarray, shape (n_samples,) - The true target values or each training sample. - - raw_predictions : ndarray, shape (prediction_dim, n_samples) - The raw_predictions (i.e. values from the trees) of the tree - ensemble at iteration ``i - 1``. - - sample_weight : array-like of shape(n_samples,) default=None - Weights of training data. - """ - - -class LeastSquares(BaseLoss): - """Least squares loss, for regression. - - For a given sample x_i, least squares loss is defined as:: - - loss(x_i) = 0.5 * (y_true_i - raw_pred_i)**2 - - This actually computes the half least squares loss to simplify - the computation of the gradients and get a unit hessian (and be consistent - with what is done in LightGBM). - """ - - def __init__(self, sample_weight): - # If sample weights are provided, the hessians and gradients - # are multiplied by sample_weight, which means the hessians are - # equal to sample weights. - super().__init__(hessians_are_constant=sample_weight is None) - - def pointwise_loss(self, y_true, raw_predictions): - # shape (1, n_samples) --> (n_samples,). reshape(-1) is more likely to - # return a view. - raw_predictions = raw_predictions.reshape(-1) - loss = 0.5 * np.power(y_true - raw_predictions, 2) - return loss - - def get_baseline_prediction(self, y_train, sample_weight, prediction_dim): - return np.average(y_train, weights=sample_weight) - - @staticmethod - def inverse_link_function(raw_predictions): - return raw_predictions - - def update_gradients_and_hessians(self, gradients, hessians, y_true, - raw_predictions, sample_weight): - # shape (1, n_samples) --> (n_samples,). reshape(-1) is more likely to - # return a view. - raw_predictions = raw_predictions.reshape(-1) - gradients = gradients.reshape(-1) - if sample_weight is None: - _update_gradients_least_squares(gradients, y_true, raw_predictions) - else: - hessians = hessians.reshape(-1) - _update_gradients_hessians_least_squares(gradients, hessians, - y_true, raw_predictions, - sample_weight) - - -class LeastAbsoluteDeviation(BaseLoss): - """Least absolute deviation, for regression. - - For a given sample x_i, the loss is defined as:: - - loss(x_i) = |y_true_i - raw_pred_i| - """ - - def __init__(self, sample_weight): - # If sample weights are provided, the hessians and gradients - # are multiplied by sample_weight, which means the hessians are - # equal to sample weights. - super().__init__(hessians_are_constant=sample_weight is None) - - # This variable indicates whether the loss requires the leaves values to - # be updated once the tree has been trained. The trees are trained to - # predict a Newton-Raphson step (see grower._finalize_leaf()). But for - # some losses (e.g. least absolute deviation) we need to adjust the tree - # values to account for the "line search" of the gradient descent - # procedure. See the original paper Greedy Function Approximation: A - # Gradient Boosting Machine by Friedman - # (https://statweb.stanford.edu/~jhf/ftp/trebst.pdf) for the theory. - need_update_leaves_values = True - - def pointwise_loss(self, y_true, raw_predictions): - # shape (1, n_samples) --> (n_samples,). reshape(-1) is more likely to - # return a view. - raw_predictions = raw_predictions.reshape(-1) - loss = np.abs(y_true - raw_predictions) - return loss - - def get_baseline_prediction(self, y_train, sample_weight, prediction_dim): - if sample_weight is None: - return np.median(y_train) - else: - return _weighted_percentile(y_train, sample_weight, 50) - - @staticmethod - def inverse_link_function(raw_predictions): - return raw_predictions - - def update_gradients_and_hessians(self, gradients, hessians, y_true, - raw_predictions, sample_weight): - # shape (1, n_samples) --> (n_samples,). reshape(-1) is more likely to - # return a view. - raw_predictions = raw_predictions.reshape(-1) - gradients = gradients.reshape(-1) - if sample_weight is None: - _update_gradients_least_absolute_deviation(gradients, y_true, - raw_predictions) - else: - hessians = hessians.reshape(-1) - _update_gradients_hessians_least_absolute_deviation( - gradients, hessians, y_true, raw_predictions, sample_weight) - - def update_leaves_values(self, grower, y_true, raw_predictions, - sample_weight): - # Update the values predicted by the tree with - # median(y_true - raw_predictions). - # See note about need_update_leaves_values in BaseLoss. - - # TODO: ideally this should be computed in parallel over the leaves - # using something similar to _update_raw_predictions(), but this - # requires a cython version of median() - for leaf in grower.finalized_leaves: - indices = leaf.sample_indices - if sample_weight is None: - median_res = np.median(y_true[indices] - - raw_predictions[indices]) - else: - median_res = _weighted_percentile( - y_true[indices] - raw_predictions[indices], - sample_weight=sample_weight[indices], - percentile=50 - ) - leaf.value = grower.shrinkage * median_res - # Note that the regularization is ignored here - - -class Poisson(BaseLoss): - """Poisson deviance loss with log-link, for regression. - - For a given sample x_i, Poisson deviance loss is defined as:: - - loss(x_i) = y_true_i * log(y_true_i/exp(raw_pred_i)) - - y_true_i + exp(raw_pred_i)) - - This actually computes half the Poisson deviance to simplify - the computation of the gradients. - """ - - def __init__(self, sample_weight): - super().__init__(hessians_are_constant=False) - - inverse_link_function = staticmethod(np.exp) - - def pointwise_loss(self, y_true, raw_predictions): - # shape (1, n_samples) --> (n_samples,). reshape(-1) is more likely to - # return a view. - raw_predictions = raw_predictions.reshape(-1) - # TODO: For speed, we could remove the constant xlogy(y_true, y_true) - # Advantage of this form: minimum of zero at raw_predictions = y_true. - loss = (xlogy(y_true, y_true) - y_true * (raw_predictions + 1) - + np.exp(raw_predictions)) - return loss - - def get_baseline_prediction(self, y_train, sample_weight, prediction_dim): - y_pred = np.average(y_train, weights=sample_weight) - eps = np.finfo(y_train.dtype).eps - y_pred = np.clip(y_pred, eps, None) - return np.log(y_pred) - - def update_gradients_and_hessians(self, gradients, hessians, y_true, - raw_predictions, sample_weight): - # shape (1, n_samples) --> (n_samples,). reshape(-1) is more likely to - # return a view. - raw_predictions = raw_predictions.reshape(-1) - gradients = gradients.reshape(-1) - hessians = hessians.reshape(-1) - _update_gradients_hessians_poisson(gradients, hessians, - y_true, raw_predictions, - sample_weight) - - -class BinaryCrossEntropy(BaseLoss): - """Binary cross-entropy loss, for binary classification. - - For a given sample x_i, the binary cross-entropy loss is defined as the - negative log-likelihood of the model which can be expressed as:: - - loss(x_i) = log(1 + exp(raw_pred_i)) - y_true_i * raw_pred_i - - See The Elements of Statistical Learning, by Hastie, Tibshirani, Friedman, - section 4.4.1 (about logistic regression). - """ - - def __init__(self, sample_weight): - super().__init__(hessians_are_constant=False) - - inverse_link_function = staticmethod(expit) - - def pointwise_loss(self, y_true, raw_predictions): - # shape (1, n_samples) --> (n_samples,). reshape(-1) is more likely to - # return a view. - raw_predictions = raw_predictions.reshape(-1) - # logaddexp(0, x) = log(1 + exp(x)) - loss = np.logaddexp(0, raw_predictions) - y_true * raw_predictions - return loss - - def get_baseline_prediction(self, y_train, sample_weight, prediction_dim): - if prediction_dim > 2: - raise ValueError( - "loss='binary_crossentropy' is not defined for multiclass" - " classification with n_classes=%d, use" - " loss='categorical_crossentropy' instead" % prediction_dim) - proba_positive_class = np.average(y_train, weights=sample_weight) - eps = np.finfo(y_train.dtype).eps - proba_positive_class = np.clip(proba_positive_class, eps, 1 - eps) - # log(x / 1 - x) is the anti function of sigmoid, or the link function - # of the Binomial model. - return np.log(proba_positive_class / (1 - proba_positive_class)) - - def update_gradients_and_hessians(self, gradients, hessians, y_true, - raw_predictions, sample_weight): - # shape (1, n_samples) --> (n_samples,). reshape(-1) is more likely to - # return a view. - raw_predictions = raw_predictions.reshape(-1) - gradients = gradients.reshape(-1) - hessians = hessians.reshape(-1) - _update_gradients_hessians_binary_crossentropy( - gradients, hessians, y_true, raw_predictions, sample_weight) - - def predict_proba(self, raw_predictions): - # shape (1, n_samples) --> (n_samples,). reshape(-1) is more likely to - # return a view. - raw_predictions = raw_predictions.reshape(-1) - proba = np.empty((raw_predictions.shape[0], 2), dtype=Y_DTYPE) - proba[:, 1] = expit(raw_predictions) - proba[:, 0] = 1 - proba[:, 1] - return proba - - -class CategoricalCrossEntropy(BaseLoss): - """Categorical cross-entropy loss, for multiclass classification. - - For a given sample x_i, the categorical cross-entropy loss is defined as - the negative log-likelihood of the model and generalizes the binary - cross-entropy to more than 2 classes. - """ - - def __init__(self, sample_weight): - super().__init__(hessians_are_constant=False) - - def pointwise_loss(self, y_true, raw_predictions): - one_hot_true = np.zeros_like(raw_predictions) - prediction_dim = raw_predictions.shape[0] - for k in range(prediction_dim): - one_hot_true[k, :] = (y_true == k) - - loss = (logsumexp(raw_predictions, axis=0) - - (one_hot_true * raw_predictions).sum(axis=0)) - return loss - - def get_baseline_prediction(self, y_train, sample_weight, prediction_dim): - init_value = np.zeros(shape=(prediction_dim, 1), dtype=Y_DTYPE) - eps = np.finfo(y_train.dtype).eps - for k in range(prediction_dim): - proba_kth_class = np.average(y_train == k, - weights=sample_weight) - proba_kth_class = np.clip(proba_kth_class, eps, 1 - eps) - init_value[k, :] += np.log(proba_kth_class) - - return init_value - - def update_gradients_and_hessians(self, gradients, hessians, y_true, - raw_predictions, sample_weight): - _update_gradients_hessians_categorical_crossentropy( - gradients, hessians, y_true, raw_predictions, sample_weight) - - def predict_proba(self, raw_predictions): - # TODO: This could be done in parallel - # compute softmax (using exp(log(softmax))) - proba = np.exp(raw_predictions - - logsumexp(raw_predictions, axis=0)[np.newaxis, :]) - return proba.T - - -_LOSSES = { - 'least_squares': LeastSquares, - 'least_absolute_deviation': LeastAbsoluteDeviation, - 'binary_crossentropy': BinaryCrossEntropy, - 'categorical_crossentropy': CategoricalCrossEntropy, - 'poisson': Poisson, -} diff --git a/sklearn/ensemble/_hist_gradient_boosting/meson.build b/sklearn/ensemble/_hist_gradient_boosting/meson.build new file mode 100644 index 0000000000000..122a2102800f3 --- /dev/null +++ b/sklearn/ensemble/_hist_gradient_boosting/meson.build @@ -0,0 +1,20 @@ +hist_gradient_boosting_extension_metadata = { + '_gradient_boosting': {'sources': [cython_gen.process('_gradient_boosting.pyx')], + 'dependencies': [openmp_dep]}, + 'histogram': {'sources': [cython_gen.process('histogram.pyx')], 'dependencies': [openmp_dep]}, + 'splitting': {'sources': [cython_gen.process('splitting.pyx')], 'dependencies': [openmp_dep]}, + '_binning': {'sources': [cython_gen.process('_binning.pyx')], 'dependencies': [openmp_dep]}, + '_predictor': {'sources': [cython_gen.process('_predictor.pyx')], 'dependencies': [openmp_dep]}, + '_bitset': {'sources': [cython_gen.process('_bitset.pyx')]}, + 'common': {'sources': [cython_gen.process('common.pyx')]}, +} + +foreach ext_name, ext_dict : hist_gradient_boosting_extension_metadata + py.extension_module( + ext_name, + ext_dict.get('sources'), + dependencies: ext_dict.get('dependencies', []), + subdir: 'sklearn/ensemble/_hist_gradient_boosting', + install: true + ) +endforeach diff --git a/sklearn/ensemble/_hist_gradient_boosting/predictor.py b/sklearn/ensemble/_hist_gradient_boosting/predictor.py index cee247c5616ea..59bb6499c4501 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/predictor.py +++ b/sklearn/ensemble/_hist_gradient_boosting/predictor.py @@ -1,14 +1,18 @@ """ This module contains the TreePredictor class which is used for prediction. """ -# Author: Nicolas Hug + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import numpy as np -from .common import Y_DTYPE -from ._predictor import _predict_from_raw_data -from ._predictor import _predict_from_binned_data -from ._predictor import _compute_partial_dependence +from ._predictor import ( + _compute_partial_dependence, + _predict_from_binned_data, + _predict_from_raw_data, +) +from .common import PREDICTOR_RECORD_DTYPE, Y_DTYPE class TreePredictor: @@ -18,31 +22,28 @@ class TreePredictor: ---------- nodes : ndarray of PREDICTOR_RECORD_DTYPE The nodes of the tree. - binned_left_cat_bitsets : ndarray of shape (n_categorical_splits, 8), \ - dtype=uint32 + binned_left_cat_bitsets : ndarray of shape (n_categorical_splits, 8), dtype=uint32 Array of bitsets for binned categories used in predict_binned when a split is categorical. - raw_left_cat_bitsets : ndarray of shape (n_categorical_splits, 8), \ - dtype=uint32 + raw_left_cat_bitsets : ndarray of shape (n_categorical_splits, 8), dtype=uint32 Array of bitsets for raw categories used in predict when a split is categorical. - """ - def __init__(self, nodes, binned_left_cat_bitsets, - raw_left_cat_bitsets): + + def __init__(self, nodes, binned_left_cat_bitsets, raw_left_cat_bitsets): self.nodes = nodes self.binned_left_cat_bitsets = binned_left_cat_bitsets self.raw_left_cat_bitsets = raw_left_cat_bitsets def get_n_leaf_nodes(self): """Return number of leaves.""" - return int(self.nodes['is_leaf'].sum()) + return int(self.nodes["is_leaf"].sum()) def get_max_depth(self): """Return maximum depth among all leaves.""" - return int(self.nodes['depth'].max()) + return int(self.nodes["depth"].max()) - def predict(self, X, known_cat_bitsets, f_idx_map): + def predict(self, X, known_cat_bitsets, f_idx_map, n_threads): """Predict raw values for non-binned data. Parameters @@ -57,17 +58,28 @@ def predict(self, X, known_cat_bitsets, f_idx_map): Map from original feature index to the corresponding index in the known_cat_bitsets array. + n_threads : int + Number of OpenMP threads to use. + Returns ------- y : ndarray, shape (n_samples,) The raw predicted values. """ out = np.empty(X.shape[0], dtype=Y_DTYPE) - _predict_from_raw_data(self.nodes, X, self.raw_left_cat_bitsets, - known_cat_bitsets, f_idx_map, out) + + _predict_from_raw_data( + self.nodes, + X, + self.raw_left_cat_bitsets, + known_cat_bitsets, + f_idx_map, + n_threads, + out, + ) return out - def predict_binned(self, X, missing_values_bin_idx): + def predict_binned(self, X, missing_values_bin_idx, n_threads): """Predict raw values for binned data. Parameters @@ -78,6 +90,8 @@ def predict_binned(self, X, missing_values_bin_idx): Index of the bin that is used for missing values. This is the index of the last bin and is always equal to max_bins (as passed to the GBDT classes), or equivalently to n_bins - 1. + n_threads : int + Number of OpenMP threads to use. Returns ------- @@ -85,9 +99,14 @@ def predict_binned(self, X, missing_values_bin_idx): The raw predicted values. """ out = np.empty(X.shape[0], dtype=Y_DTYPE) - _predict_from_binned_data(self.nodes, X, - self.binned_left_cat_bitsets, - missing_values_bin_idx, out) + _predict_from_binned_data( + self.nodes, + X, + self.binned_left_cat_bitsets, + missing_values_bin_idx, + n_threads, + out, + ) return out def compute_partial_dependence(self, grid, target_features, out): @@ -106,3 +125,22 @@ def compute_partial_dependence(self, grid, target_features, out): point. """ _compute_partial_dependence(self.nodes, grid, target_features, out) + + def __setstate__(self, state): + try: + super().__setstate__(state) + except AttributeError: + self.__dict__.update(state) + + # The dtype of feature_idx is np.intp which is platform dependent. Here, we + # make sure that saving and loading on different bitness systems works without + # errors. For instance, on a 64 bit Python runtime, np.intp = np.int64, + # while on 32 bit np.intp = np.int32. + # + # TODO: consider always using platform agnostic dtypes for fitted + # estimator attributes. For this particular estimator, this would + # mean replacing the intp field of PREDICTOR_RECORD_DTYPE by an int32 + # field. Ideally this should be done consistently throughout + # scikit-learn along with a common test. + if self.nodes.dtype != PREDICTOR_RECORD_DTYPE: + self.nodes = self.nodes.astype(PREDICTOR_RECORD_DTYPE, casting="same_kind") diff --git a/sklearn/ensemble/_hist_gradient_boosting/splitting.pyx b/sklearn/ensemble/_hist_gradient_boosting/splitting.pyx index 39884e5e6ee63..c4cb22067cf37 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/splitting.pyx +++ b/sklearn/ensemble/_hist_gradient_boosting/splitting.pyx @@ -1,31 +1,25 @@ -# cython: cdivision=True -# cython: boundscheck=False -# cython: wraparound=False -# cython: language_level=3 - """This module contains routines and data structures to: - Find the best possible split of a node. For a given node, a split is characterized by a feature and a bin. - Apply a split to a node, i.e. split the indices of the samples at the node - into the newly created left and right childs. + into the newly created left and right children. """ -# Author: Nicolas Hug + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause cimport cython from cython.parallel import prange import numpy as np -cimport numpy as np -IF SKLEARN_OPENMP_PARALLELISM_ENABLED: - from openmp cimport omp_get_max_threads +from libc.math cimport INFINITY, ceil from libc.stdlib cimport malloc, free, qsort from libc.string cimport memcpy -from numpy.math cimport INFINITY +from ...utils._typedefs cimport uint8_t from .common cimport X_BINNED_DTYPE_C from .common cimport Y_DTYPE_C from .common cimport hist_struct -from .common import HISTOGRAM_DTYPE from .common cimport BITSET_INNER_DTYPE_C from .common cimport BITSET_DTYPE_C from .common cimport MonotonicConstraint @@ -33,8 +27,6 @@ from ._bitset cimport init_bitset from ._bitset cimport set_bitset from ._bitset cimport in_bitset -np.import_array() - cdef struct split_info_struct: # Same as the SplitInfo class, but we need a C struct to use it in the @@ -42,7 +34,7 @@ cdef struct split_info_struct: Y_DTYPE_C gain int feature_idx unsigned int bin_idx - unsigned char missing_go_to_left + uint8_t missing_go_to_left Y_DTYPE_C sum_gradient_left Y_DTYPE_C sum_gradient_right Y_DTYPE_C sum_hessian_left @@ -51,7 +43,7 @@ cdef struct split_info_struct: unsigned int n_samples_right Y_DTYPE_C value_left Y_DTYPE_C value_right - unsigned char is_categorical + uint8_t is_categorical BITSET_DTYPE_C left_cat_bitset @@ -146,6 +138,13 @@ cdef class Splitter: feature. is_categorical : ndarray of bool of shape (n_features,) Indicates categorical features. + monotonic_cst : ndarray of int of shape (n_features,), dtype=int + Indicates the monotonic constraint to enforce on each feature. + - 1: monotonic increase + - 0: no constraint + - -1: monotonic decrease + + Read more in the :ref:`User Guide `. l2_regularization : float The L2 regularization parameter. min_hessian_to_split : float, default=1e-3 @@ -159,53 +158,69 @@ cdef class Splitter: be ignored. hessians_are_constant: bool, default is False Whether hessians are constant. + feature_fraction_per_split : float, default=1 + Proportion of randomly chosen features in each and every node split. + This is a form of regularization, smaller values make the trees weaker + learners and might prevent overfitting. + rng : Generator + n_threads : int, default=1 + Number of OpenMP threads to use. """ cdef public: const X_BINNED_DTYPE_C [::1, :] X_binned unsigned int n_features const unsigned int [::1] n_bins_non_missing - unsigned char missing_values_bin_idx - const unsigned char [::1] has_missing_values - const unsigned char [::1] is_categorical + uint8_t missing_values_bin_idx + const uint8_t [::1] has_missing_values + const uint8_t [::1] is_categorical const signed char [::1] monotonic_cst - unsigned char hessians_are_constant + uint8_t hessians_are_constant Y_DTYPE_C l2_regularization Y_DTYPE_C min_hessian_to_split unsigned int min_samples_leaf Y_DTYPE_C min_gain_to_split + Y_DTYPE_C feature_fraction_per_split + rng unsigned int [::1] partition unsigned int [::1] left_indices_buffer unsigned int [::1] right_indices_buffer + int n_threads def __init__(self, const X_BINNED_DTYPE_C [::1, :] X_binned, const unsigned int [::1] n_bins_non_missing, - const unsigned char missing_values_bin_idx, - const unsigned char [::1] has_missing_values, - const unsigned char [::1] is_categorical, + const uint8_t missing_values_bin_idx, + const uint8_t [::1] has_missing_values, + const uint8_t [::1] is_categorical, const signed char [::1] monotonic_cst, Y_DTYPE_C l2_regularization, Y_DTYPE_C min_hessian_to_split=1e-3, unsigned int min_samples_leaf=20, Y_DTYPE_C min_gain_to_split=0., - unsigned char hessians_are_constant=False): + uint8_t hessians_are_constant=False, + Y_DTYPE_C feature_fraction_per_split=1.0, + rng=np.random.RandomState(), + unsigned int n_threads=1): self.X_binned = X_binned self.n_features = X_binned.shape[1] self.n_bins_non_missing = n_bins_non_missing self.missing_values_bin_idx = missing_values_bin_idx self.has_missing_values = has_missing_values - self.monotonic_cst = monotonic_cst self.is_categorical = is_categorical + self.monotonic_cst = monotonic_cst self.l2_regularization = l2_regularization self.min_hessian_to_split = min_hessian_to_split self.min_samples_leaf = min_samples_leaf self.min_gain_to_split = min_gain_to_split self.hessians_are_constant = hessians_are_constant + self.feature_fraction_per_split = feature_fraction_per_split + self.rng = rng + self.n_threads = n_threads # The partition array maps each sample index into the leaves of the - # tree (a leaf in this context is a node that isn't splitted yet, not + # tree (a leaf in this context is a node that isn't split yet, not # necessarily a 'finalized' leaf). Initially, the root contains all # the indices, e.g.: # partition = [abcdefghijkl] @@ -294,22 +309,19 @@ cdef class Splitter: cdef: int n_samples = sample_indices.shape[0] X_BINNED_DTYPE_C bin_idx = split_info.bin_idx - unsigned char missing_go_to_left = split_info.missing_go_to_left - unsigned char missing_values_bin_idx = self.missing_values_bin_idx + uint8_t missing_go_to_left = split_info.missing_go_to_left + uint8_t missing_values_bin_idx = self.missing_values_bin_idx int feature_idx = split_info.feature_idx const X_BINNED_DTYPE_C [::1] X_binned = \ self.X_binned[:, feature_idx] unsigned int [::1] left_indices_buffer = self.left_indices_buffer unsigned int [::1] right_indices_buffer = self.right_indices_buffer - unsigned char is_categorical = split_info.is_categorical + uint8_t is_categorical = split_info.is_categorical # Cython is unhappy if we set left_cat_bitset to # split_info.left_cat_bitset directly, so we need a tmp var BITSET_INNER_DTYPE_C [:] cat_bitset_tmp = split_info.left_cat_bitset BITSET_DTYPE_C left_cat_bitset - IF SKLEARN_OPENMP_PARALLELISM_ENABLED: - int n_threads = omp_get_max_threads() - ELSE: - int n_threads = 1 + int n_threads = self.n_threads int [:] sizes = np.full(n_threads, n_samples // n_threads, dtype=np.int32) @@ -324,7 +336,7 @@ cdef class Splitter: int thread_idx int sample_idx int right_child_position - unsigned char turn_left + uint8_t turn_left int [:] left_offset = np.zeros(n_threads, dtype=np.int32) int [:] right_offset = np.zeros(n_threads, dtype=np.int32) @@ -342,7 +354,7 @@ cdef class Splitter: # map indices from sample_indices to left/right_indices_buffer for thread_idx in prange(n_threads, schedule='static', - chunksize=1): + chunksize=1, num_threads=n_threads): left_count = 0 right_count = 0 @@ -384,17 +396,31 @@ cdef class Splitter: # sample_indices. This also updates self.partition since # sample_indices is a view. for thread_idx in prange(n_threads, schedule='static', - chunksize=1): + chunksize=1, num_threads=n_threads): memcpy( &sample_indices[left_offset[thread_idx]], &left_indices_buffer[offset_in_buffers[thread_idx]], sizeof(unsigned int) * left_counts[thread_idx] ) - memcpy( - &sample_indices[right_offset[thread_idx]], - &right_indices_buffer[offset_in_buffers[thread_idx]], - sizeof(unsigned int) * right_counts[thread_idx] - ) + if right_counts[thread_idx] > 0: + # If we're splitting the rightmost node of the tree, i.e. the + # rightmost node in the partition array, and if n_threads >= 2, one + # might have right_counts[-1] = 0 and right_offset[-1] = len(sample_indices) + # leading to evaluating + # + # &sample_indices[right_offset[-1]] = &samples_indices[n_samples_at_node] + # = &partition[n_samples_in_tree] + # + # which is an out-of-bounds read access that can cause a segmentation fault. + # When boundscheck=True, removing this check produces this exception: + # + # IndexError: Out of bounds on buffer access + # + memcpy( + &sample_indices[right_offset[thread_idx]], + &right_indices_buffer[offset_in_buffers[thread_idx]], + sizeof(unsigned int) * right_counts[thread_idx] + ) return (sample_indices[:right_child_position], sample_indices[right_child_position:], @@ -409,6 +435,7 @@ cdef class Splitter: const Y_DTYPE_C value, const Y_DTYPE_C lower_bound=-INFINITY, const Y_DTYPE_C upper_bound=INFINITY, + const unsigned int [:] allowed_features=None, ): """For each feature, find the best bin to split on at a given node. @@ -441,6 +468,9 @@ cdef class Splitter: upper_bound : float Upper bound for the children values for respecting the monotonic constraints. + allowed_features : None or ndarray, dtype=np.uint32 + Indices of the features that are allowed by interaction constraints to be + split. Returns ------- @@ -449,36 +479,76 @@ cdef class Splitter: """ cdef: int feature_idx - int best_feature_idx - int n_features = self.n_features + int split_info_idx + int best_split_info_idx + int n_allowed_features split_info_struct split_info split_info_struct * split_infos - const unsigned char [::1] has_missing_values = self.has_missing_values - const unsigned char [::1] is_categorical = self.is_categorical + const uint8_t [::1] has_missing_values = self.has_missing_values + const uint8_t [::1] is_categorical = self.is_categorical const signed char [::1] monotonic_cst = self.monotonic_cst + int n_threads = self.n_threads + bint has_interaction_cst = False + Y_DTYPE_C feature_fraction_per_split = self.feature_fraction_per_split + uint8_t [:] subsample_mask # same as npy_bool + int n_subsampled_features + + has_interaction_cst = allowed_features is not None + if has_interaction_cst: + n_allowed_features = allowed_features.shape[0] + else: + n_allowed_features = self.n_features + + if feature_fraction_per_split < 1.0: + # We do all random sampling before the nogil and make sure that we sample + # exactly n_subsampled_features >= 1 features. + n_subsampled_features = max( + 1, + int(ceil(feature_fraction_per_split * n_allowed_features)), + ) + subsample_mask_arr = np.full(n_allowed_features, False) + subsample_mask_arr[:n_subsampled_features] = True + self.rng.shuffle(subsample_mask_arr) + # https://github.com/numpy/numpy/issues/18273 + subsample_mask = subsample_mask_arr with nogil: split_infos = malloc( - self.n_features * sizeof(split_info_struct)) + n_allowed_features * sizeof(split_info_struct)) + + # split_info_idx is index of split_infos of size n_allowed_features. + # features_idx is the index of the feature column in X. + for split_info_idx in prange(n_allowed_features, schedule='static', + num_threads=n_threads): + if has_interaction_cst: + feature_idx = allowed_features[split_info_idx] + else: + feature_idx = split_info_idx - for feature_idx in prange(n_features, schedule='static'): - split_infos[feature_idx].feature_idx = feature_idx + split_infos[split_info_idx].feature_idx = feature_idx # For each feature, find best bin to split on - # Start with a gain of -1 (if no better split is found, that + # Start with a gain of -1 if no better split is found, that # means one of the constraints isn't respected - # (min_samples_leaf, etc) and the grower will later turn the + # (min_samples_leaf, etc.) and the grower will later turn the # node into a leaf. - split_infos[feature_idx].gain = -1 - split_infos[feature_idx].is_categorical = is_categorical[feature_idx] + split_infos[split_info_idx].gain = -1 + split_infos[split_info_idx].is_categorical = is_categorical[feature_idx] + + # Note that subsample_mask is indexed by split_info_idx and not by + # feature_idx because we only need to exclude the same features again + # and again. We do NOT need to access the features directly by using + # allowed_features. + if feature_fraction_per_split < 1.0 and not subsample_mask[split_info_idx]: + continue if is_categorical[feature_idx]: self._find_best_bin_to_split_category( feature_idx, has_missing_values[feature_idx], histograms, n_samples, sum_gradients, sum_hessians, value, monotonic_cst[feature_idx], lower_bound, - upper_bound, &split_infos[feature_idx]) + upper_bound, &split_infos[split_info_idx]) else: # We will scan bins from left to right (in all cases), and # if there are any missing values, we will also scan bins @@ -494,7 +564,7 @@ cdef class Splitter: feature_idx, has_missing_values[feature_idx], histograms, n_samples, sum_gradients, sum_hessians, value, monotonic_cst[feature_idx], - lower_bound, upper_bound, &split_infos[feature_idx]) + lower_bound, upper_bound, &split_infos[split_info_idx]) if has_missing_values[feature_idx]: # We need to explore both directions to check whether @@ -504,12 +574,14 @@ cdef class Splitter: feature_idx, histograms, n_samples, sum_gradients, sum_hessians, value, monotonic_cst[feature_idx], - lower_bound, upper_bound, &split_infos[feature_idx]) + lower_bound, upper_bound, &split_infos[split_info_idx]) # then compute best possible split among all features - best_feature_idx = self._find_best_feature_to_split_helper( - split_infos) - split_info = split_infos[best_feature_idx] + # split_info is set to the best of split_infos + best_split_info_idx = self._find_best_feature_to_split_helper( + split_infos, n_allowed_features + ) + split_info = split_infos[best_split_info_idx] out = SplitInfo( split_info.gain, @@ -534,24 +606,25 @@ cdef class Splitter: free(split_infos) return out - cdef unsigned int _find_best_feature_to_split_helper( - self, - split_info_struct * split_infos) nogil: # IN - """Returns the best feature among those in splits_infos.""" + cdef int _find_best_feature_to_split_helper( + self, + split_info_struct * split_infos, # IN + int n_allowed_features, + ) noexcept nogil: + """Return the index of split_infos with the best feature split.""" cdef: - unsigned int feature_idx - unsigned int best_feature_idx = 0 + int split_info_idx + int best_split_info_idx = 0 - for feature_idx in range(1, self.n_features): - if (split_infos[feature_idx].gain > - split_infos[best_feature_idx].gain): - best_feature_idx = feature_idx - return best_feature_idx + for split_info_idx in range(1, n_allowed_features): + if (split_infos[split_info_idx].gain > split_infos[best_split_info_idx].gain): + best_split_info_idx = split_info_idx + return best_split_info_idx cdef void _find_best_bin_to_split_left_to_right( Splitter self, unsigned int feature_idx, - unsigned char has_missing_values, + uint8_t has_missing_values, const hist_struct [:, ::1] histograms, # IN unsigned int n_samples, Y_DTYPE_C sum_gradients, @@ -560,7 +633,7 @@ cdef class Splitter: signed char monotonic_cst, Y_DTYPE_C lower_bound, Y_DTYPE_C upper_bound, - split_info_struct * split_info) nogil: # OUT + split_info_struct * split_info) noexcept nogil: # OUT """Find best bin to split on for a given feature. Splits that do not satisfy the splitting constraints @@ -587,13 +660,14 @@ cdef class Splitter: Y_DTYPE_C sum_gradient_right Y_DTYPE_C loss_current_node Y_DTYPE_C gain - unsigned char found_better_split = False + uint8_t found_better_split = False Y_DTYPE_C best_sum_hessian_left Y_DTYPE_C best_sum_gradient_left unsigned int best_bin_idx unsigned int best_n_samples_left Y_DTYPE_C best_gain = -1 + hist_struct hist sum_gradient_left, sum_hessian_left = 0., 0. n_samples_left = 0 @@ -601,17 +675,18 @@ cdef class Splitter: loss_current_node = _loss_from_value(value, sum_gradients) for bin_idx in range(end): - n_samples_left += histograms[feature_idx, bin_idx].count + hist = histograms[feature_idx, bin_idx] + n_samples_left += hist.count n_samples_right = n_samples_ - n_samples_left if self.hessians_are_constant: - sum_hessian_left += histograms[feature_idx, bin_idx].count + sum_hessian_left += hist.count else: sum_hessian_left += \ - histograms[feature_idx, bin_idx].sum_hessians + hist.sum_hessians sum_hessian_right = sum_hessians - sum_hessian_left - sum_gradient_left += histograms[feature_idx, bin_idx].sum_gradients + sum_gradient_left += hist.sum_gradients sum_gradient_right = sum_gradients - sum_gradient_left if n_samples_left < self.min_samples_leaf: @@ -674,7 +749,7 @@ cdef class Splitter: signed char monotonic_cst, Y_DTYPE_C lower_bound, Y_DTYPE_C upper_bound, - split_info_struct * split_info) nogil: # OUT + split_info_struct * split_info) noexcept nogil: # OUT """Find best bin to split on for a given feature. Splits that do not satisfy the splitting constraints @@ -700,13 +775,14 @@ cdef class Splitter: Y_DTYPE_C loss_current_node Y_DTYPE_C gain unsigned int start = self.n_bins_non_missing[feature_idx] - 2 - unsigned char found_better_split = False + uint8_t found_better_split = False Y_DTYPE_C best_sum_hessian_left Y_DTYPE_C best_sum_gradient_left unsigned int best_bin_idx unsigned int best_n_samples_left Y_DTYPE_C best_gain = split_info.gain # computed during previous scan + hist_struct hist sum_gradient_right, sum_hessian_right = 0., 0. n_samples_right = 0 @@ -714,18 +790,19 @@ cdef class Splitter: loss_current_node = _loss_from_value(value, sum_gradients) for bin_idx in range(start, -1, -1): - n_samples_right += histograms[feature_idx, bin_idx + 1].count + hist = histograms[feature_idx, bin_idx + 1] + n_samples_right += hist.count n_samples_left = n_samples_ - n_samples_right if self.hessians_are_constant: - sum_hessian_right += histograms[feature_idx, bin_idx + 1].count + sum_hessian_right += hist.count else: sum_hessian_right += \ - histograms[feature_idx, bin_idx + 1].sum_hessians + hist.sum_hessians sum_hessian_left = sum_hessians - sum_hessian_right sum_gradient_right += \ - histograms[feature_idx, bin_idx + 1].sum_gradients + hist.sum_gradients sum_gradient_left = sum_gradients - sum_gradient_right if n_samples_right < self.min_samples_leaf: @@ -777,11 +854,10 @@ cdef class Splitter: split_info.sum_gradient_right, split_info.sum_hessian_right, lower_bound, upper_bound, self.l2_regularization) - @cython.initializedcheck(False) cdef void _find_best_bin_to_split_category( self, unsigned int feature_idx, - unsigned char has_missing_values, + uint8_t has_missing_values, const hist_struct [:, ::1] histograms, # IN unsigned int n_samples, Y_DTYPE_C sum_gradients, @@ -790,7 +866,7 @@ cdef class Splitter: char monotonic_cst, Y_DTYPE_C lower_bound, Y_DTYPE_C upper_bound, - split_info_struct * split_info) nogil: # OUT + split_info_struct * split_info) noexcept nogil: # OUT """Find best split for categorical features. Categories are first sorted according to their variance, and then @@ -812,6 +888,7 @@ cdef class Splitter: unsigned int middle unsigned int i const hist_struct[::1] feature_hist = histograms[feature_idx, :] + hist_struct hist Y_DTYPE_C sum_gradients_bin Y_DTYPE_C sum_hessians_bin Y_DTYPE_C loss_current_node @@ -820,7 +897,7 @@ cdef class Splitter: unsigned int n_samples_left, n_samples_right Y_DTYPE_C gain Y_DTYPE_C best_gain = -1.0 - unsigned char found_better_split = False + uint8_t found_better_split = False Y_DTYPE_C best_sum_hessian_left Y_DTYPE_C best_sum_gradient_left unsigned int best_n_samples_left @@ -839,7 +916,7 @@ cdef class Splitter: # other category. The low-support categories will always be mapped to # the right child. We scan the sorted categories array from left to # right and from right to left, and we stop at the middle. - + # Considering ordered categories A B C D, with E being a low-support # category: A B C D # ^ @@ -873,13 +950,14 @@ cdef class Splitter: # fill cat_infos while filtering out categories based on MIN_CAT_SUPPORT for bin_idx in range(n_bins_non_missing): + hist = feature_hist[bin_idx] if self.hessians_are_constant: - sum_hessians_bin = feature_hist[bin_idx].count + sum_hessians_bin = hist.count else: - sum_hessians_bin = feature_hist[bin_idx].sum_hessians + sum_hessians_bin = hist.sum_hessians if sum_hessians_bin * support_factor >= MIN_CAT_SUPPORT: cat_infos[n_used_bins].bin_idx = bin_idx - sum_gradients_bin = feature_hist[bin_idx].sum_gradients + sum_gradients_bin = hist.sum_gradients cat_infos[n_used_bins].value = ( sum_gradients_bin / (sum_hessians_bin + MIN_CAT_SUPPORT) @@ -888,14 +966,15 @@ cdef class Splitter: # Also add missing values bin so that nans are considered as a category if has_missing_values: + hist = feature_hist[missing_values_bin_idx] if self.hessians_are_constant: - sum_hessians_bin = feature_hist[missing_values_bin_idx].count + sum_hessians_bin = hist.count else: - sum_hessians_bin = feature_hist[missing_values_bin_idx].sum_hessians + sum_hessians_bin = hist.sum_hessians if sum_hessians_bin * support_factor >= MIN_CAT_SUPPORT: cat_infos[n_used_bins].bin_idx = missing_values_bin_idx sum_gradients_bin = ( - feature_hist[missing_values_bin_idx].sum_gradients + hist.sum_gradients ) cat_infos[n_used_bins].value = ( @@ -926,32 +1005,37 @@ cdef class Splitter: for i in range(middle): sorted_cat_idx = i if direction == 1 else n_used_bins - 1 - i - bin_idx = cat_infos[sorted_cat_idx].bin_idx; + bin_idx = cat_infos[sorted_cat_idx].bin_idx + hist = feature_hist[bin_idx] - n_samples_left += feature_hist[bin_idx].count + n_samples_left += hist.count n_samples_right = n_samples - n_samples_left if self.hessians_are_constant: - sum_hessian_left += feature_hist[bin_idx].count + sum_hessian_left += hist.count else: - sum_hessian_left += feature_hist[bin_idx].sum_hessians + sum_hessian_left += hist.sum_hessians sum_hessian_right = sum_hessians - sum_hessian_left - sum_gradient_left += feature_hist[bin_idx].sum_gradients + sum_gradient_left += hist.sum_gradients sum_gradient_right = sum_gradients - sum_gradient_left - if (n_samples_left < self.min_samples_leaf or - sum_hessian_left < self.min_hessian_to_split): + if ( + n_samples_left < self.min_samples_leaf or + sum_hessian_left < self.min_hessian_to_split + ): continue - if (n_samples_right < self.min_samples_leaf or - sum_hessian_right < self.min_hessian_to_split): + if ( + n_samples_right < self.min_samples_leaf or + sum_hessian_right < self.min_hessian_to_split + ): break gain = _split_gain(sum_gradient_left, sum_hessian_left, - sum_gradient_right, sum_hessian_right, - loss_current_node, monotonic_cst, - lower_bound, upper_bound, - self.l2_regularization) + sum_gradient_right, sum_hessian_right, + loss_current_node, monotonic_cst, + lower_bound, upper_bound, + self.l2_regularization) if gain > best_gain and gain > self.min_gain_to_split: found_better_split = True best_gain = gain @@ -961,7 +1045,6 @@ cdef class Splitter: best_n_samples_left = n_samples_left best_direction = direction - if found_better_split: split_info.gain = best_gain @@ -1003,7 +1086,7 @@ cdef class Splitter: free(cat_infos) -cdef int compare_cat_infos(const void * a, const void * b) nogil: +cdef int compare_cat_infos(const void * a, const void * b) noexcept nogil: return -1 if (a).value < (b).value else 1 cdef inline Y_DTYPE_C _split_gain( @@ -1015,15 +1098,15 @@ cdef inline Y_DTYPE_C _split_gain( signed char monotonic_cst, Y_DTYPE_C lower_bound, Y_DTYPE_C upper_bound, - Y_DTYPE_C l2_regularization) nogil: + Y_DTYPE_C l2_regularization) noexcept nogil: """Loss reduction Compute the reduction in loss after taking a split, compared to keeping the node a leaf of the tree. See Equation 7 of: - XGBoost: A Scalable Tree Boosting System, T. Chen, C. Guestrin, 2016 - https://arxiv.org/abs/1603.02754 + :arxiv:`T. Chen, C. Guestrin, (2016) XGBoost: A Scalable Tree Boosting System, + <1603.02754>.` """ cdef: Y_DTYPE_C gain @@ -1035,8 +1118,8 @@ cdef inline Y_DTYPE_C _split_gain( lower_bound, upper_bound, l2_regularization) value_right = compute_node_value(sum_gradient_right, sum_hessian_right, - lower_bound, upper_bound, - l2_regularization) + lower_bound, upper_bound, + l2_regularization) if ((monotonic_cst == MonotonicConstraint.POS and value_left > value_right) or (monotonic_cst == MonotonicConstraint.NEG and value_left < value_right)): @@ -1057,22 +1140,22 @@ cdef inline Y_DTYPE_C _split_gain( cdef inline Y_DTYPE_C _loss_from_value( Y_DTYPE_C value, - Y_DTYPE_C sum_gradient) nogil: + Y_DTYPE_C sum_gradient) noexcept nogil: """Return loss of a node from its (bounded) value See Equation 6 of: - XGBoost: A Scalable Tree Boosting System, T. Chen, C. Guestrin, 2016 - https://arxiv.org/abs/1603.02754 + :arxiv:`T. Chen, C. Guestrin, (2016) XGBoost: A Scalable Tree Boosting System, + <1603.02754>.` """ return sum_gradient * value -cdef inline unsigned char sample_goes_left( - unsigned char missing_go_to_left, - unsigned char missing_values_bin_idx, +cdef inline uint8_t sample_goes_left( + uint8_t missing_go_to_left, + uint8_t missing_values_bin_idx, X_BINNED_DTYPE_C split_bin_idx, X_BINNED_DTYPE_C bin_value, - unsigned char is_categorical, - BITSET_DTYPE_C left_cat_bitset) nogil: + uint8_t is_categorical, + BITSET_DTYPE_C left_cat_bitset) noexcept nogil: """Helper to decide whether sample should go to left or right child.""" if is_categorical: @@ -1094,15 +1177,15 @@ cpdef inline Y_DTYPE_C compute_node_value( Y_DTYPE_C sum_hessian, Y_DTYPE_C lower_bound, Y_DTYPE_C upper_bound, - Y_DTYPE_C l2_regularization) nogil: + Y_DTYPE_C l2_regularization) noexcept nogil: """Compute a node's value. The value is capped in the [lower_bound, upper_bound] interval to respect monotonic constraints. Shrinkage is ignored. See Equation 5 of: - XGBoost: A Scalable Tree Boosting System, T. Chen, C. Guestrin, 2016 - https://arxiv.org/abs/1603.02754 + :arxiv:`T. Chen, C. Guestrin, (2016) XGBoost: A Scalable Tree Boosting System, + <1603.02754>.` """ cdef: diff --git a/sklearn/ensemble/_hist_gradient_boosting/tests/test_binning.py b/sklearn/ensemble/_hist_gradient_boosting/tests/test_binning.py index 5f31d9b898df5..6f9fcd0057141 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/tests/test_binning.py +++ b/sklearn/ensemble/_hist_gradient_boosting/tests/test_binning.py @@ -1,20 +1,27 @@ import numpy as np -from numpy.testing import assert_array_equal, assert_allclose import pytest +from numpy.testing import assert_allclose, assert_array_equal from sklearn.ensemble._hist_gradient_boosting.binning import ( _BinMapper, _find_binning_thresholds, - _map_to_bins + _map_to_bins, ) -from sklearn.ensemble._hist_gradient_boosting.common import X_DTYPE -from sklearn.ensemble._hist_gradient_boosting.common import X_BINNED_DTYPE -from sklearn.ensemble._hist_gradient_boosting.common import ALMOST_INF +from sklearn.ensemble._hist_gradient_boosting.common import ( + ALMOST_INF, + X_BINNED_DTYPE, + X_DTYPE, +) +from sklearn.utils._openmp_helpers import _openmp_effective_n_threads + +n_threads = _openmp_effective_n_threads() -DATA = np.random.RandomState(42).normal( - loc=[0, 10], scale=[1, 0.01], size=(int(1e6), 2) -).astype(X_DTYPE) +DATA = ( + np.random.RandomState(42) + .normal(loc=[0, 10], scale=[1, 0.01], size=(int(1e6), 2)) + .astype(X_DTYPE) +) def test_find_binning_thresholds_regular_data(): @@ -36,57 +43,63 @@ def test_find_binning_thresholds_small_regular_data(): assert_allclose(bin_thresholds, [1, 2, 3, 4, 5, 6, 7, 8, 9]) bin_thresholds = _find_binning_thresholds(data, max_bins=11) - assert_allclose(bin_thresholds, np.arange(10) + .5) + assert_allclose(bin_thresholds, np.arange(10) + 0.5) bin_thresholds = _find_binning_thresholds(data, max_bins=255) - assert_allclose(bin_thresholds, np.arange(10) + .5) + assert_allclose(bin_thresholds, np.arange(10) + 0.5) def test_find_binning_thresholds_random_data(): - bin_thresholds = [_find_binning_thresholds(DATA[:, i], max_bins=255) - for i in range(2)] + bin_thresholds = [ + _find_binning_thresholds(DATA[:, i], max_bins=255) for i in range(2) + ] for i in range(len(bin_thresholds)): assert bin_thresholds[i].shape == (254,) # 255 - 1 assert bin_thresholds[i].dtype == DATA.dtype - assert_allclose(bin_thresholds[0][[64, 128, 192]], - np.array([-0.7, 0.0, 0.7]), atol=1e-1) + assert_allclose( + bin_thresholds[0][[64, 128, 192]], np.array([-0.7, 0.0, 0.7]), atol=1e-1 + ) - assert_allclose(bin_thresholds[1][[64, 128, 192]], - np.array([9.99, 10.00, 10.01]), atol=1e-2) + assert_allclose( + bin_thresholds[1][[64, 128, 192]], np.array([9.99, 10.00, 10.01]), atol=1e-2 + ) def test_find_binning_thresholds_low_n_bins(): - bin_thresholds = [_find_binning_thresholds(DATA[:, i], max_bins=128) - for i in range(2)] + bin_thresholds = [ + _find_binning_thresholds(DATA[:, i], max_bins=128) for i in range(2) + ] for i in range(len(bin_thresholds)): assert bin_thresholds[i].shape == (127,) # 128 - 1 assert bin_thresholds[i].dtype == DATA.dtype -@pytest.mark.parametrize('n_bins', (2, 257)) +@pytest.mark.parametrize("n_bins", (2, 257)) def test_invalid_n_bins(n_bins): - err_msg = ( - 'n_bins={} should be no smaller than 3 and no larger than 256' - .format(n_bins)) + err_msg = "n_bins={} should be no smaller than 3 and no larger than 256".format( + n_bins + ) with pytest.raises(ValueError, match=err_msg): _BinMapper(n_bins=n_bins).fit(DATA) def test_bin_mapper_n_features_transform(): mapper = _BinMapper(n_bins=42, random_state=42).fit(DATA) - err_msg = 'This estimator was fitted with 2 features but 4 got passed' + err_msg = "This estimator was fitted with 2 features but 4 got passed" with pytest.raises(ValueError, match=err_msg): mapper.transform(np.repeat(DATA, 2, axis=1)) -@pytest.mark.parametrize('max_bins', [16, 128, 255]) +@pytest.mark.parametrize("max_bins", [16, 128, 255]) def test_map_to_bins(max_bins): - bin_thresholds = [_find_binning_thresholds(DATA[:, i], max_bins=max_bins) - for i in range(2)] - binned = np.zeros_like(DATA, dtype=X_BINNED_DTYPE, order='F') + bin_thresholds = [ + _find_binning_thresholds(DATA[:, i], max_bins=max_bins) for i in range(2) + ] + binned = np.zeros_like(DATA, dtype=X_BINNED_DTYPE, order="F") + is_categorical = np.zeros(2, dtype=np.uint8) last_bin_idx = max_bins - _map_to_bins(DATA, bin_thresholds, last_bin_idx, binned) + _map_to_bins(DATA, bin_thresholds, is_categorical, last_bin_idx, n_threads, binned) assert binned.shape == DATA.shape assert binned.dtype == np.uint8 assert binned.flags.f_contiguous @@ -115,8 +128,7 @@ def test_bin_mapper_random_data(max_bins): assert binned.shape == (n_samples, n_features) assert binned.dtype == np.uint8 assert_array_equal(binned.min(axis=0), np.array([0, 0])) - assert_array_equal(binned.max(axis=0), - np.array([max_bins - 1, max_bins - 1])) + assert_array_equal(binned.max(axis=0), np.array([max_bins - 1, max_bins - 1])) assert len(mapper.bin_thresholds_) == n_features for bin_thresholds_feature in mapper.bin_thresholds_: assert bin_thresholds_feature.shape == (max_bins - 1,) @@ -130,12 +142,7 @@ def test_bin_mapper_random_data(max_bins): assert abs(count - expected_count_per_bin) < tol -@pytest.mark.parametrize("n_samples, max_bins", [ - (5, 5), - (5, 10), - (5, 11), - (42, 255) -]) +@pytest.mark.parametrize("n_samples, max_bins", [(5, 5), (5, 10), (5, 11), (42, 255)]) def test_bin_mapper_small_random_data(n_samples, max_bins): data = np.random.RandomState(42).normal(size=n_samples).reshape(-1, 1) assert len(np.unique(data)) == n_samples @@ -147,15 +154,17 @@ def test_bin_mapper_small_random_data(n_samples, max_bins): assert binned.shape == data.shape assert binned.dtype == np.uint8 - assert_array_equal(binned.ravel()[np.argsort(data.ravel())], - np.arange(n_samples)) + assert_array_equal(binned.ravel()[np.argsort(data.ravel())], np.arange(n_samples)) -@pytest.mark.parametrize("max_bins, n_distinct, multiplier", [ - (5, 5, 1), - (5, 5, 3), - (255, 12, 42), -]) +@pytest.mark.parametrize( + "max_bins, n_distinct, multiplier", + [ + (5, 5, 1), + (5, 5, 3), + (255, 12, 42), + ], +) def test_bin_mapper_identity_repeated_values(max_bins, n_distinct, multiplier): data = np.array(list(range(n_distinct)) * multiplier).reshape(-1, 1) # max_bins is the number of bins for non-missing values @@ -164,7 +173,7 @@ def test_bin_mapper_identity_repeated_values(max_bins, n_distinct, multiplier): assert_array_equal(data, binned) -@pytest.mark.parametrize('n_distinct', [2, 7, 42]) +@pytest.mark.parametrize("n_distinct", [2, 7, 42]) def test_bin_mapper_repeated_values_invariance(n_distinct): rng = np.random.RandomState(42) distinct_values = rng.normal(size=n_distinct) @@ -189,11 +198,14 @@ def test_bin_mapper_repeated_values_invariance(n_distinct): assert_array_equal(binned_1, binned_2) -@pytest.mark.parametrize("max_bins, scale, offset", [ - (3, 2, -1), - (42, 1, 0), - (255, 0.3, 42), -]) +@pytest.mark.parametrize( + "max_bins, scale, offset", + [ + (3, 2, -1), + (42, 1, 0), + (255, 0.3, 42), + ], +) def test_bin_mapper_identity_small(max_bins, scale, offset): data = np.arange(max_bins).reshape(-1, 1) * scale + offset # max_bins is the number of bins for non-missing values @@ -202,15 +214,18 @@ def test_bin_mapper_identity_small(max_bins, scale, offset): assert_array_equal(binned, np.arange(max_bins).reshape(-1, 1)) -@pytest.mark.parametrize('max_bins_small, max_bins_large', [ - (2, 2), - (3, 3), - (4, 4), - (42, 42), - (255, 255), - (5, 17), - (42, 255), -]) +@pytest.mark.parametrize( + "max_bins_small, max_bins_large", + [ + (2, 2), + (3, 3), + (4, 4), + (42, 42), + (255, 255), + (5, 17), + (42, 255), + ], +) def test_bin_mapper_idempotence(max_bins_small, max_bins_large): assert max_bins_large >= max_bins_small data = np.random.RandomState(42).normal(size=30000).reshape(-1, 1) @@ -221,8 +236,8 @@ def test_bin_mapper_idempotence(max_bins_small, max_bins_large): assert_array_equal(binned_small, binned_large) -@pytest.mark.parametrize('n_bins', [10, 100, 256]) -@pytest.mark.parametrize('diff', [-5, 0, 5]) +@pytest.mark.parametrize("n_bins", [10, 100, 256]) +@pytest.mark.parametrize("diff", [-5, 0, 5]) def test_n_bins_non_missing(n_bins, diff): # Check that n_bins_non_missing is n_unique_values when # there are not a lot of unique values, else n_bins - 1. @@ -231,8 +246,7 @@ def test_n_bins_non_missing(n_bins, diff): X = list(range(n_unique_values)) * 2 X = np.array(X).reshape(-1, 1) mapper = _BinMapper(n_bins=n_bins).fit(X) - assert np.all(mapper.n_bins_non_missing_ == min( - n_bins - 1, n_unique_values)) + assert np.all(mapper.n_bins_non_missing_ == min(n_bins - 1, n_unique_values)) def test_subsample(): @@ -241,35 +255,54 @@ def test_subsample(): mapper_subsample = _BinMapper(subsample=256, random_state=0).fit(DATA) for feature in range(DATA.shape[1]): - assert not np.allclose(mapper_no_subsample.bin_thresholds_[feature], - mapper_subsample.bin_thresholds_[feature], - rtol=1e-4) + assert not np.allclose( + mapper_no_subsample.bin_thresholds_[feature], + mapper_subsample.bin_thresholds_[feature], + rtol=1e-4, + ) @pytest.mark.parametrize( - 'n_bins, n_bins_non_missing, X_trans_expected', [ - (256, [4, 2, 2], [[0, 0, 0], # 255 <=> missing value - [255, 255, 0], - [1, 0, 0], - [255, 1, 1], - [2, 1, 1], - [3, 0, 0]]), - (3, [2, 2, 2], [[0, 0, 0], # 2 <=> missing value - [2, 2, 0], - [0, 0, 0], - [2, 1, 1], - [1, 1, 1], - [1, 0, 0]])]) + "n_bins, n_bins_non_missing, X_trans_expected", + [ + ( + 256, + [4, 2, 2], + [ + [0, 0, 0], # 255 <=> missing value + [255, 255, 0], + [1, 0, 0], + [255, 1, 1], + [2, 1, 1], + [3, 0, 0], + ], + ), + ( + 3, + [2, 2, 2], + [ + [0, 0, 0], # 2 <=> missing value + [2, 2, 0], + [0, 0, 0], + [2, 1, 1], + [1, 1, 1], + [1, 0, 0], + ], + ), + ], +) def test_missing_values_support(n_bins, n_bins_non_missing, X_trans_expected): # check for missing values: make sure nans are mapped to the last bin # and that the _BinMapper attributes are correct - X = [[1, 1, 0], - [np.NaN, np.NaN, 0], - [2, 1, 0], - [np.NaN, 2, 1], - [3, 2, 1], - [4, 1, 0]] + X = [ + [1, 1, 0], + [np.nan, np.nan, 0], + [2, 1, 0], + [np.nan, 2, 1], + [3, 2, 1], + [4, 1, 0], + ] X = np.array(X) @@ -279,8 +312,10 @@ def test_missing_values_support(n_bins, n_bins_non_missing, X_trans_expected): assert_array_equal(mapper.n_bins_non_missing_, n_bins_non_missing) for feature_idx in range(X.shape[1]): - assert len(mapper.bin_thresholds_[feature_idx]) == \ - n_bins_non_missing[feature_idx] - 1 + assert ( + len(mapper.bin_thresholds_[feature_idx]) + == n_bins_non_missing[feature_idx] - 1 + ) assert mapper.missing_values_bin_idx_ == n_bins - 1 @@ -292,10 +327,10 @@ def test_infinite_values(): # Make sure infinite values are properly handled. bin_mapper = _BinMapper() - X = np.array([-np.inf, 0, 1, np.inf]).reshape(-1, 1) + X = np.array([-np.inf, 0, 1, np.inf]).reshape(-1, 1) bin_mapper.fit(X) - assert_allclose(bin_mapper.bin_thresholds_[0], [-np.inf, .5, ALMOST_INF]) + assert_allclose(bin_mapper.bin_thresholds_[0], [-np.inf, 0.5, ALMOST_INF]) assert bin_mapper.n_bins_non_missing_ == [4] expected_binned_X = np.array([0, 1, 2, 3]).reshape(-1, 1) @@ -307,18 +342,17 @@ def test_categorical_feature(n_bins): # Basic test for categorical features # we make sure that categories are mapped into [0, n_categories - 1] and # that nans are mapped to the last bin - X = np.array([[4] * 500 + - [1] * 3 + - [10] * 4 + - [0] * 4 + - [13] + - [7] * 5 + - [np.nan] * 2], dtype=X_DTYPE).T + X = np.array( + [[4] * 500 + [1] * 3 + [10] * 4 + [0] * 4 + [13] + [7] * 5 + [np.nan] * 2], + dtype=X_DTYPE, + ).T known_categories = [np.unique(X[~np.isnan(X)])] - bin_mapper = _BinMapper(n_bins=n_bins, - is_categorical=np.array([True]), - known_categories=known_categories).fit(X) + bin_mapper = _BinMapper( + n_bins=n_bins, + is_categorical=np.array([True]), + known_categories=known_categories, + ).fit(X) assert bin_mapper.n_bins_non_missing_ == [6] assert_array_equal(bin_mapper.bin_thresholds_[0], [0, 1, 4, 7, 10, 13]) @@ -326,10 +360,35 @@ def test_categorical_feature(n_bins): expected_trans = np.array([[0, 1, 2, n_bins - 1, 3, 4, 5]]).T assert_array_equal(bin_mapper.transform(X), expected_trans) - # For unknown categories, the mapping is incorrect / undefined. This never - # happens in practice. This check is only for illustration purpose. - X = np.array([[-1, 100]], dtype=X_DTYPE).T - expected_trans = np.array([[0, 6]]).T + # Negative categories are mapped to the missing values' bin + # (i.e. the bin of index `missing_values_bin_idx_ == n_bins - 1). + # Unknown positive categories does not happen in practice and tested + # for illustration purpose. + X = np.array([[-4, -1, 100]], dtype=X_DTYPE).T + expected_trans = np.array([[n_bins - 1, n_bins - 1, 6]]).T + assert_array_equal(bin_mapper.transform(X), expected_trans) + + +def test_categorical_feature_negative_missing(): + """Make sure bin mapper treats negative categories as missing values.""" + X = np.array( + [[4] * 500 + [1] * 3 + [5] * 10 + [-1] * 3 + [np.nan] * 4], dtype=X_DTYPE + ).T + bin_mapper = _BinMapper( + n_bins=4, + is_categorical=np.array([True]), + known_categories=[np.array([1, 4, 5], dtype=X_DTYPE)], + ).fit(X) + + assert bin_mapper.n_bins_non_missing_ == [3] + + X = np.array([[-1, 1, 3, 5, np.nan]], dtype=X_DTYPE).T + + # Negative values for categorical features are considered as missing values. + # They are mapped to the bin of index `bin_mapper.missing_values_bin_idx_`, + # which is 3 here. + assert bin_mapper.missing_values_bin_idx_ == 3 + expected_trans = np.array([[3, 0, 1, 2, 3]]).T assert_array_equal(bin_mapper.transform(X), expected_trans) @@ -342,9 +401,11 @@ def test_categorical_with_numerical_features(n_bins): X = np.c_[X1, X2] known_categories = [None, np.unique(X2).astype(X_DTYPE)] - bin_mapper = _BinMapper(n_bins=n_bins, - is_categorical=np.array([False, True]), - known_categories=known_categories).fit(X) + bin_mapper = _BinMapper( + n_bins=n_bins, + is_categorical=np.array([False, True]), + known_categories=known_categories, + ).fit(X) assert_array_equal(bin_mapper.n_bins_non_missing_, [10, 5]) @@ -352,29 +413,32 @@ def test_categorical_with_numerical_features(n_bins): assert len(bin_thresholds) == 2 assert_array_equal(bin_thresholds[1], np.arange(10, 15)) - expected_X_trans = [[0, 0], - [1, 1], - [2, 2], - [3, 3], - [4, 4], - [5, 0], - [6, 1], - [7, 2], - [8, 3], - [9, 4]] + expected_X_trans = [ + [0, 0], + [1, 1], + [2, 2], + [3, 3], + [4, 4], + [5, 0], + [6, 1], + [7, 2], + [8, 3], + [9, 4], + ] assert_array_equal(bin_mapper.transform(X), expected_X_trans) def test_make_known_categories_bitsets(): # Check the output of make_known_categories_bitsets - X = np.array([[14, 2, 30], - [30, 4, 70], - [40, 10, 180], - [40, 240, 180]], dtype=X_DTYPE) - - bin_mapper = _BinMapper(n_bins=256, - is_categorical=np.array([False, True, True]), - known_categories=[None, X[:, 1], X[:, 2]]) + X = np.array( + [[14, 2, 30], [30, 4, 70], [40, 10, 180], [40, 240, 180]], dtype=X_DTYPE + ) + + bin_mapper = _BinMapper( + n_bins=256, + is_categorical=np.array([False, True, True]), + known_categories=[None, X[:, 1], X[:, 2]], + ) bin_mapper.fit(X) known_cat_bitsets, f_idx_map = bin_mapper.make_known_categories_bitsets() @@ -402,19 +466,24 @@ def test_make_known_categories_bitsets(): assert_allclose(expected_cat_bitset, known_cat_bitsets) -@pytest.mark.parametrize('is_categorical, known_categories, match', [ - (np.array([True]), [None], - 'Known categories for feature 0 must be provided'), - - (np.array([False]), np.array([1, 2, 3]), - "isn't marked as a categorical feature, but categories were passed") -]) +@pytest.mark.parametrize( + "is_categorical, known_categories, match", + [ + (np.array([True]), [None], "Known categories for feature 0 must be provided"), + ( + np.array([False]), + np.array([1, 2, 3]), + "isn't marked as a categorical feature, but categories were passed", + ), + ], +) def test_categorical_parameters(is_categorical, known_categories, match): # test the validation of the is_categorical and known_categories parameters X = np.array([[1, 2, 3]], dtype=X_DTYPE) - bin_mapper = _BinMapper(is_categorical=is_categorical, - known_categories=known_categories) + bin_mapper = _BinMapper( + is_categorical=is_categorical, known_categories=known_categories + ) with pytest.raises(ValueError, match=match): bin_mapper.fit(X) diff --git a/sklearn/ensemble/_hist_gradient_boosting/tests/test_bitset.py b/sklearn/ensemble/_hist_gradient_boosting/tests/test_bitset.py index 09e2df40c7226..c02d66b666f80 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/tests/test_bitset.py +++ b/sklearn/ensemble/_hist_gradient_boosting/tests/test_bitset.py @@ -1,19 +1,25 @@ -import pytest import numpy as np +import pytest from numpy.testing import assert_allclose from sklearn.ensemble._hist_gradient_boosting._bitset import ( - set_bitset_memoryview, in_bitset_memoryview, - set_raw_bitset_from_binned_bitset + set_bitset_memoryview, + set_raw_bitset_from_binned_bitset, ) from sklearn.ensemble._hist_gradient_boosting.common import X_DTYPE -@pytest.mark.parametrize("values_to_insert, expected_bitset", [ - ([0, 4, 33], np.array([2**0 + 2**4, 2**1, 0], dtype=np.uint32)), - ([31, 32, 33, 79], np.array([2**31, 2**0 + 2**1, 2**15], dtype=np.uint32)) -]) +@pytest.mark.parametrize( + "values_to_insert, expected_bitset", + [ + ([0, 4, 33], np.array([2**0 + 2**4, 2**1, 0], dtype=np.uint32)), + ( + [31, 32, 33, 79], + np.array([2**31, 2**0 + 2**1, 2**15], dtype=np.uint32), + ), + ], +) def test_set_get_bitset(values_to_insert, expected_bitset): n_32bits_ints = 3 bitset = np.zeros(n_32bits_ints, dtype=np.uint32) @@ -28,18 +34,19 @@ def test_set_get_bitset(values_to_insert, expected_bitset): @pytest.mark.parametrize( - "raw_categories, binned_cat_to_insert, expected_raw_bitset", [ - ([3, 4, 5, 10, 31, 32, 43], - [0, 2, 4, 5, 6], - [2**3 + 2**5 + 2**31, 2**0 + 2**11]), - - ([3, 33, 50, 52], - [1, 3], - [0, 2**1 + 2**20]), - ] + "raw_categories, binned_cat_to_insert, expected_raw_bitset", + [ + ( + [3, 4, 5, 10, 31, 32, 43], + [0, 2, 4, 5, 6], + [2**3 + 2**5 + 2**31, 2**0 + 2**11], + ), + ([3, 33, 50, 52], [1, 3], [0, 2**1 + 2**20]), + ], ) -def test_raw_bitset_from_binned_bitset(raw_categories, binned_cat_to_insert, - expected_raw_bitset): +def test_raw_bitset_from_binned_bitset( + raw_categories, binned_cat_to_insert, expected_raw_bitset +): binned_bitset = np.zeros(2, dtype=np.uint32) raw_bitset = np.zeros(2, dtype=np.uint32) raw_categories = np.asarray(raw_categories, dtype=X_DTYPE) @@ -47,8 +54,7 @@ def test_raw_bitset_from_binned_bitset(raw_categories, binned_cat_to_insert, for val in binned_cat_to_insert: set_bitset_memoryview(binned_bitset, val) - set_raw_bitset_from_binned_bitset(raw_bitset, binned_bitset, - raw_categories) + set_raw_bitset_from_binned_bitset(raw_bitset, binned_bitset, raw_categories) assert_allclose(expected_raw_bitset, raw_bitset) for binned_cat_val, raw_cat_val in enumerate(raw_categories): diff --git a/sklearn/ensemble/_hist_gradient_boosting/tests/test_compare_lightgbm.py b/sklearn/ensemble/_hist_gradient_boosting/tests/test_compare_lightgbm.py index 4a6c4dbbb32c7..24b5b02aa0696 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/tests/test_compare_lightgbm.py +++ b/sklearn/ensemble/_hist_gradient_boosting/tests/test_compare_lightgbm.py @@ -1,26 +1,44 @@ -from sklearn.model_selection import train_test_split -from sklearn.metrics import accuracy_score -from sklearn.datasets import make_classification, make_regression import numpy as np import pytest -# To use this experimental feature, we need to explicitly ask for it: -from sklearn.experimental import enable_hist_gradient_boosting # noqa -from sklearn.ensemble import HistGradientBoostingRegressor -from sklearn.ensemble import HistGradientBoostingClassifier +from sklearn.datasets import make_classification, make_regression +from sklearn.ensemble import ( + HistGradientBoostingClassifier, + HistGradientBoostingRegressor, +) from sklearn.ensemble._hist_gradient_boosting.binning import _BinMapper -from sklearn.ensemble._hist_gradient_boosting.utils import ( - get_equivalent_estimator) - - -@pytest.mark.parametrize('seed', range(5)) -@pytest.mark.parametrize('min_samples_leaf', (1, 20)) -@pytest.mark.parametrize('n_samples, max_leaf_nodes', [ - (255, 4096), - (1000, 8), -]) -def test_same_predictions_regression(seed, min_samples_leaf, n_samples, - max_leaf_nodes): +from sklearn.ensemble._hist_gradient_boosting.utils import get_equivalent_estimator +from sklearn.metrics import accuracy_score +from sklearn.model_selection import train_test_split + + +# TODO(1.8) remove the filterwarnings decorator +@pytest.mark.filterwarnings( + "ignore:'force_all_finite' was renamed to 'ensure_all_finite':FutureWarning" +) +@pytest.mark.parametrize("seed", range(5)) +@pytest.mark.parametrize( + "loss", + [ + "squared_error", + "poisson", + pytest.param( + "gamma", + marks=pytest.mark.skip("LightGBM with gamma loss has larger deviation."), + ), + ], +) +@pytest.mark.parametrize("min_samples_leaf", (1, 20)) +@pytest.mark.parametrize( + "n_samples, max_leaf_nodes", + [ + (255, 4096), + (1000, 8), + ], +) +def test_same_predictions_regression( + seed, loss, min_samples_leaf, n_samples, max_leaf_nodes +): # Make sure sklearn has the same predictions as lightgbm for easy targets. # # In particular when the size of the trees are bound and the number of @@ -34,9 +52,9 @@ def test_same_predictions_regression(seed, min_samples_leaf, n_samples, # is not exactly the same. To avoid this issue we only compare the # predictions on the test set when the number of samples is large enough # and max_leaf_nodes is low enough. - # - To ignore discrepancies caused by small differences the binning + # - To ignore discrepancies caused by small differences in the binning # strategy, data is pre-binned if n_samples > 255. - # - We don't check the least_absolute_deviation loss here. This is because + # - We don't check the absolute_error loss here. This is because # LightGBM's computation of the median (used for the initial value of # raw_prediction) is a bit off (they'll e.g. return midpoints when there # is no need to.). Since these tests only run 1 iteration, the @@ -49,8 +67,13 @@ def test_same_predictions_regression(seed, min_samples_leaf, n_samples, max_iter = 1 max_bins = 255 - X, y = make_regression(n_samples=n_samples, n_features=5, - n_informative=5, random_state=0) + X, y = make_regression( + n_samples=n_samples, n_features=5, n_informative=5, random_state=0 + ) + + if loss in ("gamma", "poisson"): + # make the target positive + y = np.abs(y) + np.mean(np.abs(y)) if n_samples > 255: # bin data and convert it to float32 so that the estimator doesn't @@ -60,13 +83,16 @@ def test_same_predictions_regression(seed, min_samples_leaf, n_samples, X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=rng) est_sklearn = HistGradientBoostingRegressor( + loss=loss, max_iter=max_iter, max_bins=max_bins, learning_rate=1, early_stopping=False, min_samples_leaf=min_samples_leaf, - max_leaf_nodes=max_leaf_nodes) - est_lightgbm = get_equivalent_estimator(est_sklearn, lib='lightgbm') + max_leaf_nodes=max_leaf_nodes, + ) + est_lightgbm = get_equivalent_estimator(est_sklearn, lib="lightgbm") + est_lightgbm.set_params(min_sum_hessian_in_leaf=0) est_lightgbm.fit(X_train, y_train) est_sklearn.fit(X_train, y_train) @@ -76,33 +102,58 @@ def test_same_predictions_regression(seed, min_samples_leaf, n_samples, pred_lightgbm = est_lightgbm.predict(X_train) pred_sklearn = est_sklearn.predict(X_train) - # less than 1% of the predictions are different up to the 3rd decimal - assert np.mean(abs(pred_lightgbm - pred_sklearn) > 1e-3) < .011 - - if max_leaf_nodes < 10 and n_samples >= 1000: + if loss in ("gamma", "poisson"): + # More than 65% of the predictions must be close up to the 2nd decimal. + # TODO: We are not entirely satisfied with this lax comparison, but the root + # cause is not clear, maybe algorithmic differences. One such example is the + # poisson_max_delta_step parameter of LightGBM which does not exist in HGBT. + assert ( + np.mean(np.isclose(pred_lightgbm, pred_sklearn, rtol=1e-2, atol=1e-2)) + > 0.65 + ) + else: + # Less than 1% of the predictions may deviate more than 1e-3 in relative terms. + assert np.mean(np.isclose(pred_lightgbm, pred_sklearn, rtol=1e-3)) > 1 - 0.01 + + if max_leaf_nodes < 10 and n_samples >= 1000 and loss in ("squared_error",): pred_lightgbm = est_lightgbm.predict(X_test) pred_sklearn = est_sklearn.predict(X_test) - # less than 1% of the predictions are different up to the 4th decimal - assert np.mean(abs(pred_lightgbm - pred_sklearn) > 1e-4) < .01 - - -@pytest.mark.parametrize('seed', range(5)) -@pytest.mark.parametrize('min_samples_leaf', (1, 20)) -@pytest.mark.parametrize('n_samples, max_leaf_nodes', [ - (255, 4096), - (1000, 8), -]) -def test_same_predictions_classification(seed, min_samples_leaf, n_samples, - max_leaf_nodes): + # Less than 1% of the predictions may deviate more than 1e-4 in relative terms. + assert np.mean(np.isclose(pred_lightgbm, pred_sklearn, rtol=1e-4)) > 1 - 0.01 + + +# TODO(1.8) remove the filterwarnings decorator +@pytest.mark.filterwarnings( + "ignore:'force_all_finite' was renamed to 'ensure_all_finite':FutureWarning" +) +@pytest.mark.parametrize("seed", range(5)) +@pytest.mark.parametrize("min_samples_leaf", (1, 20)) +@pytest.mark.parametrize( + "n_samples, max_leaf_nodes", + [ + (255, 4096), + (1000, 8), + ], +) +def test_same_predictions_classification( + seed, min_samples_leaf, n_samples, max_leaf_nodes +): # Same as test_same_predictions_regression but for classification pytest.importorskip("lightgbm") rng = np.random.RandomState(seed=seed) max_iter = 1 + n_classes = 2 max_bins = 255 - X, y = make_classification(n_samples=n_samples, n_classes=2, n_features=5, - n_informative=5, n_redundant=0, random_state=0) + X, y = make_classification( + n_samples=n_samples, + n_classes=n_classes, + n_features=5, + n_informative=5, + n_redundant=0, + random_state=0, + ) if n_samples > 255: # bin data and convert it to float32 so that the estimator doesn't @@ -112,14 +163,17 @@ def test_same_predictions_classification(seed, min_samples_leaf, n_samples, X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=rng) est_sklearn = HistGradientBoostingClassifier( - loss='binary_crossentropy', + loss="log_loss", max_iter=max_iter, max_bins=max_bins, learning_rate=1, early_stopping=False, min_samples_leaf=min_samples_leaf, - max_leaf_nodes=max_leaf_nodes) - est_lightgbm = get_equivalent_estimator(est_sklearn, lib='lightgbm') + max_leaf_nodes=max_leaf_nodes, + ) + est_lightgbm = get_equivalent_estimator( + est_sklearn, lib="lightgbm", n_classes=n_classes + ) est_lightgbm.fit(X_train, y_train) est_sklearn.fit(X_train, y_train) @@ -129,42 +183,56 @@ def test_same_predictions_classification(seed, min_samples_leaf, n_samples, pred_lightgbm = est_lightgbm.predict(X_train) pred_sklearn = est_sklearn.predict(X_train) - assert np.mean(pred_sklearn == pred_lightgbm) > .89 + assert np.mean(pred_sklearn == pred_lightgbm) > 0.89 acc_lightgbm = accuracy_score(y_train, pred_lightgbm) acc_sklearn = accuracy_score(y_train, pred_sklearn) np.testing.assert_almost_equal(acc_lightgbm, acc_sklearn) if max_leaf_nodes < 10 and n_samples >= 1000: - pred_lightgbm = est_lightgbm.predict(X_test) pred_sklearn = est_sklearn.predict(X_test) - assert np.mean(pred_sklearn == pred_lightgbm) > .89 + assert np.mean(pred_sklearn == pred_lightgbm) > 0.89 acc_lightgbm = accuracy_score(y_test, pred_lightgbm) acc_sklearn = accuracy_score(y_test, pred_sklearn) np.testing.assert_almost_equal(acc_lightgbm, acc_sklearn, decimal=2) -@pytest.mark.parametrize('seed', range(5)) -@pytest.mark.parametrize('min_samples_leaf', (1, 20)) -@pytest.mark.parametrize('n_samples, max_leaf_nodes', [ - (255, 4096), - (10000, 8), -]) +# TODO(1.8) remove the filterwarnings decorator +@pytest.mark.filterwarnings( + "ignore:'force_all_finite' was renamed to 'ensure_all_finite':FutureWarning" +) +@pytest.mark.parametrize("seed", range(5)) +@pytest.mark.parametrize("min_samples_leaf", (1, 20)) +@pytest.mark.parametrize( + "n_samples, max_leaf_nodes", + [ + (255, 4096), + (10000, 8), + ], +) def test_same_predictions_multiclass_classification( - seed, min_samples_leaf, n_samples, max_leaf_nodes): + seed, min_samples_leaf, n_samples, max_leaf_nodes +): # Same as test_same_predictions_regression but for classification pytest.importorskip("lightgbm") rng = np.random.RandomState(seed=seed) + n_classes = 3 max_iter = 1 max_bins = 255 lr = 1 - X, y = make_classification(n_samples=n_samples, n_classes=3, n_features=5, - n_informative=5, n_redundant=0, - n_clusters_per_class=1, random_state=0) + X, y = make_classification( + n_samples=n_samples, + n_classes=n_classes, + n_features=5, + n_informative=5, + n_redundant=0, + n_clusters_per_class=1, + random_state=0, + ) if n_samples > 255: # bin data and convert it to float32 so that the estimator doesn't @@ -174,14 +242,17 @@ def test_same_predictions_multiclass_classification( X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=rng) est_sklearn = HistGradientBoostingClassifier( - loss='categorical_crossentropy', + loss="log_loss", max_iter=max_iter, max_bins=max_bins, learning_rate=lr, early_stopping=False, min_samples_leaf=min_samples_leaf, - max_leaf_nodes=max_leaf_nodes) - est_lightgbm = get_equivalent_estimator(est_sklearn, lib='lightgbm') + max_leaf_nodes=max_leaf_nodes, + ) + est_lightgbm = get_equivalent_estimator( + est_sklearn, lib="lightgbm", n_classes=n_classes + ) est_lightgbm.fit(X_train, y_train) est_sklearn.fit(X_train, y_train) @@ -191,29 +262,29 @@ def test_same_predictions_multiclass_classification( pred_lightgbm = est_lightgbm.predict(X_train) pred_sklearn = est_sklearn.predict(X_train) - assert np.mean(pred_sklearn == pred_lightgbm) > .89 + assert np.mean(pred_sklearn == pred_lightgbm) > 0.89 proba_lightgbm = est_lightgbm.predict_proba(X_train) proba_sklearn = est_sklearn.predict_proba(X_train) # assert more than 75% of the predicted probabilities are the same up to # the second decimal - assert np.mean(np.abs(proba_lightgbm - proba_sklearn) < 1e-2) > .75 + assert np.mean(np.abs(proba_lightgbm - proba_sklearn) < 1e-2) > 0.75 acc_lightgbm = accuracy_score(y_train, pred_lightgbm) acc_sklearn = accuracy_score(y_train, pred_sklearn) - np.testing.assert_almost_equal(acc_lightgbm, acc_sklearn, decimal=2) - if max_leaf_nodes < 10 and n_samples >= 1000: + np.testing.assert_allclose(acc_lightgbm, acc_sklearn, rtol=0, atol=5e-2) + if max_leaf_nodes < 10 and n_samples >= 1000: pred_lightgbm = est_lightgbm.predict(X_test) pred_sklearn = est_sklearn.predict(X_test) - assert np.mean(pred_sklearn == pred_lightgbm) > .89 + assert np.mean(pred_sklearn == pred_lightgbm) > 0.89 proba_lightgbm = est_lightgbm.predict_proba(X_train) proba_sklearn = est_sklearn.predict_proba(X_train) # assert more than 75% of the predicted probabilities are the same up # to the second decimal - assert np.mean(np.abs(proba_lightgbm - proba_sklearn) < 1e-2) > .75 + assert np.mean(np.abs(proba_lightgbm - proba_sklearn) < 1e-2) > 0.75 acc_lightgbm = accuracy_score(y_test, pred_lightgbm) acc_sklearn = accuracy_score(y_test, pred_sklearn) diff --git a/sklearn/ensemble/_hist_gradient_boosting/tests/test_gradient_boosting.py b/sklearn/ensemble/_hist_gradient_boosting/tests/test_gradient_boosting.py index 0e5d1e91c3dd0..7dde25f3d22df 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/tests/test_gradient_boosting.py +++ b/sklearn/ensemble/_hist_gradient_boosting/tests/test_gradient_boosting.py @@ -1,29 +1,46 @@ +import copyreg +import io +import pickle +import re +import warnings +from unittest.mock import Mock + +import joblib import numpy as np import pytest +from joblib.numpy_pickle import NumpyPickler from numpy.testing import assert_allclose, assert_array_equal -from sklearn.datasets import make_classification, make_regression -from sklearn.datasets import make_low_rank_matrix -from sklearn.preprocessing import KBinsDiscretizer, MinMaxScaler, OneHotEncoder -from sklearn.model_selection import train_test_split, cross_val_score -from sklearn.base import clone, BaseEstimator, TransformerMixin -from sklearn.base import is_regressor -from sklearn.pipeline import make_pipeline -from sklearn.metrics import mean_poisson_deviance -from sklearn.dummy import DummyRegressor -from sklearn.exceptions import NotFittedError -from sklearn.compose import make_column_transformer -# To use this experimental feature, we need to explicitly ask for it: -from sklearn.experimental import enable_hist_gradient_boosting # noqa -from sklearn.ensemble import HistGradientBoostingRegressor -from sklearn.ensemble import HistGradientBoostingClassifier -from sklearn.ensemble._hist_gradient_boosting.loss import _LOSSES -from sklearn.ensemble._hist_gradient_boosting.loss import LeastSquares -from sklearn.ensemble._hist_gradient_boosting.loss import BinaryCrossEntropy -from sklearn.ensemble._hist_gradient_boosting.grower import TreeGrower +import sklearn +from sklearn._loss.loss import ( + AbsoluteError, + HalfBinomialLoss, + HalfSquaredError, + PinballLoss, +) +from sklearn.base import BaseEstimator, TransformerMixin, clone, is_regressor +from sklearn.compose import make_column_transformer +from sklearn.datasets import make_classification, make_low_rank_matrix, make_regression +from sklearn.dummy import DummyRegressor +from sklearn.ensemble import ( + HistGradientBoostingClassifier, + HistGradientBoostingRegressor, +) from sklearn.ensemble._hist_gradient_boosting.binning import _BinMapper -from sklearn.utils import shuffle +from sklearn.ensemble._hist_gradient_boosting.common import G_H_DTYPE +from sklearn.ensemble._hist_gradient_boosting.grower import TreeGrower +from sklearn.ensemble._hist_gradient_boosting.predictor import TreePredictor +from sklearn.exceptions import NotFittedError +from sklearn.metrics import get_scorer, mean_gamma_deviance, mean_poisson_deviance +from sklearn.model_selection import cross_val_score, train_test_split +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import KBinsDiscretizer, MinMaxScaler, OneHotEncoder +from sklearn.utils import check_random_state, shuffle +from sklearn.utils._openmp_helpers import _openmp_effective_n_threads +from sklearn.utils._testing import _convert_container +from sklearn.utils.fixes import _IS_32BIT +n_threads = _openmp_effective_n_threads() X_classification, y_classification = make_classification(random_state=0) X_regression, y_regression = make_regression(random_state=0) @@ -36,60 +53,61 @@ def _make_dumb_dataset(n_samples): """Make a dumb dataset to test early stopping.""" rng = np.random.RandomState(42) X_dumb = rng.randn(n_samples, 1) - y_dumb = (X_dumb[:, 0] > 0).astype('int64') + y_dumb = (X_dumb[:, 0] > 0).astype("int64") return X_dumb, y_dumb -@pytest.mark.parametrize('GradientBoosting, X, y', [ - (HistGradientBoostingClassifier, X_classification, y_classification), - (HistGradientBoostingRegressor, X_regression, y_regression) -]) @pytest.mark.parametrize( - 'params, err_msg', - [({'loss': 'blah'}, 'Loss blah is not supported for'), - ({'learning_rate': 0}, 'learning_rate=0 must be strictly positive'), - ({'learning_rate': -1}, 'learning_rate=-1 must be strictly positive'), - ({'max_iter': 0}, 'max_iter=0 must not be smaller than 1'), - ({'max_leaf_nodes': 0}, 'max_leaf_nodes=0 should not be smaller than 2'), - ({'max_leaf_nodes': 1}, 'max_leaf_nodes=1 should not be smaller than 2'), - ({'max_depth': 0}, 'max_depth=0 should not be smaller than 1'), - ({'min_samples_leaf': 0}, 'min_samples_leaf=0 should not be smaller'), - ({'l2_regularization': -1}, 'l2_regularization=-1 must be positive'), - ({'max_bins': 1}, 'max_bins=1 should be no smaller than 2 and no larger'), - ({'max_bins': 256}, 'max_bins=256 should be no smaller than 2 and no'), - ({'n_iter_no_change': -1}, 'n_iter_no_change=-1 must be positive'), - ({'validation_fraction': -1}, 'validation_fraction=-1 must be strictly'), - ({'validation_fraction': 0}, 'validation_fraction=0 must be strictly'), - ({'tol': -1}, 'tol=-1 must not be smaller than 0')] + "GradientBoosting, X, y", + [ + (HistGradientBoostingClassifier, X_classification, y_classification), + (HistGradientBoostingRegressor, X_regression, y_regression), + ], +) +@pytest.mark.parametrize( + "params, err_msg", + [ + ( + {"interaction_cst": [0, 1]}, + "Interaction constraints must be a sequence of tuples or lists", + ), + ( + {"interaction_cst": [{0, 9999}]}, + r"Interaction constraints must consist of integer indices in \[0," + r" n_features - 1\] = \[.*\], specifying the position of features,", + ), + ( + {"interaction_cst": [{-1, 0}]}, + r"Interaction constraints must consist of integer indices in \[0," + r" n_features - 1\] = \[.*\], specifying the position of features,", + ), + ( + {"interaction_cst": [{0.5}]}, + r"Interaction constraints must consist of integer indices in \[0," + r" n_features - 1\] = \[.*\], specifying the position of features,", + ), + ], ) def test_init_parameters_validation(GradientBoosting, X, y, params, err_msg): - with pytest.raises(ValueError, match=err_msg): GradientBoosting(**params).fit(X, y) -def test_invalid_classification_loss(): - binary_clf = HistGradientBoostingClassifier(loss="binary_crossentropy") - err_msg = ("loss='binary_crossentropy' is not defined for multiclass " - "classification with n_classes=3, use " - "loss='categorical_crossentropy' instead") - with pytest.raises(ValueError, match=err_msg): - binary_clf.fit(np.zeros(shape=(3, 2)), np.arange(3)) - - @pytest.mark.parametrize( - 'scoring, validation_fraction, early_stopping, n_iter_no_change, tol', [ - ('neg_mean_squared_error', .1, True, 5, 1e-7), # use scorer - ('neg_mean_squared_error', None, True, 5, 1e-1), # use scorer on train - (None, .1, True, 5, 1e-7), # same with default scorer + "scoring, validation_fraction, early_stopping, n_iter_no_change, tol", + [ + ("neg_mean_squared_error", 0.1, True, 5, 1e-7), # use scorer + ("neg_mean_squared_error", None, True, 5, 1e-1), # use scorer on train + (None, 0.1, True, 5, 1e-7), # same with default scorer (None, None, True, 5, 1e-1), - ('loss', .1, True, 5, 1e-7), # use loss - ('loss', None, True, 5, 1e-1), # use loss on training data + ("loss", 0.1, True, 5, 1e-7), # use loss + ("loss", None, True, 5, 1e-1), # use loss on training data (None, None, False, 5, 0.0), # no early stopping - ]) -def test_early_stopping_regression(scoring, validation_fraction, - early_stopping, n_iter_no_change, tol): - + ], +) +def test_early_stopping_regression( + scoring, validation_fraction, early_stopping, n_iter_no_change, tol +): max_iter = 200 X, y = make_regression(n_samples=50, random_state=0) @@ -103,7 +121,7 @@ def test_early_stopping_regression(scoring, validation_fraction, validation_fraction=validation_fraction, max_iter=max_iter, n_iter_no_change=n_iter_no_change, - random_state=0 + random_state=0, ) gb.fit(X, y) @@ -113,30 +131,36 @@ def test_early_stopping_regression(scoring, validation_fraction, assert gb.n_iter_ == max_iter -@pytest.mark.parametrize('data', ( - make_classification(n_samples=30, random_state=0), - make_classification(n_samples=30, n_classes=3, n_clusters_per_class=1, - random_state=0) -)) @pytest.mark.parametrize( - 'scoring, validation_fraction, early_stopping, n_iter_no_change, tol', [ - ('accuracy', .1, True, 5, 1e-7), # use scorer - ('accuracy', None, True, 5, 1e-1), # use scorer on training data - (None, .1, True, 5, 1e-7), # same with default scorer + "data", + ( + make_classification(n_samples=30, random_state=0), + make_classification( + n_samples=30, n_classes=3, n_clusters_per_class=1, random_state=0 + ), + ), +) +@pytest.mark.parametrize( + "scoring, validation_fraction, early_stopping, n_iter_no_change, tol", + [ + ("accuracy", 0.1, True, 5, 1e-7), # use scorer + ("accuracy", None, True, 5, 1e-1), # use scorer on training data + (None, 0.1, True, 5, 1e-7), # same with default scorer (None, None, True, 5, 1e-1), - ('loss', .1, True, 5, 1e-7), # use loss - ('loss', None, True, 5, 1e-1), # use loss on training data + ("loss", 0.1, True, 5, 1e-7), # use loss + ("loss", None, True, 5, 1e-1), # use loss on training data (None, None, False, 5, 0.0), # no early stopping - ]) -def test_early_stopping_classification(data, scoring, validation_fraction, - early_stopping, n_iter_no_change, tol): - + ], +) +def test_early_stopping_classification( + data, scoring, validation_fraction, early_stopping, n_iter_no_change, tol +): max_iter = 50 X, y = data gb = HistGradientBoostingClassifier( - verbose=1, # just for coverage + verbose=2, # just for coverage min_samples_leaf=5, # easier to overfit fast scoring=scoring, tol=tol, @@ -144,7 +168,7 @@ def test_early_stopping_classification(data, scoring, validation_fraction, validation_fraction=validation_fraction, max_iter=max_iter, n_iter_no_change=n_iter_no_change, - random_state=0 + random_state=0, ) gb.fit(X, y) @@ -154,12 +178,15 @@ def test_early_stopping_classification(data, scoring, validation_fraction, assert gb.n_iter_ == max_iter -@pytest.mark.parametrize('GradientBoosting, X, y', [ - (HistGradientBoostingClassifier, *_make_dumb_dataset(10000)), - (HistGradientBoostingClassifier, *_make_dumb_dataset(10001)), - (HistGradientBoostingRegressor, *_make_dumb_dataset(10000)), - (HistGradientBoostingRegressor, *_make_dumb_dataset(10001)) -]) +@pytest.mark.parametrize( + "GradientBoosting, X, y", + [ + (HistGradientBoostingClassifier, *_make_dumb_dataset(10000)), + (HistGradientBoostingClassifier, *_make_dumb_dataset(10001)), + (HistGradientBoostingRegressor, *_make_dumb_dataset(10000)), + (HistGradientBoostingRegressor, *_make_dumb_dataset(10001)), + ], +) def test_early_stopping_default(GradientBoosting, X, y): # Test that early stopping is enabled by default if and only if there # are more than 10000 samples @@ -172,56 +199,146 @@ def test_early_stopping_default(GradientBoosting, X, y): @pytest.mark.parametrize( - 'scores, n_iter_no_change, tol, stopping', + "scores, n_iter_no_change, tol, stopping", [ ([], 1, 0.001, False), # not enough iterations ([1, 1, 1], 5, 0.001, False), # not enough iterations ([1, 1, 1, 1, 1], 5, 0.001, False), # not enough iterations ([1, 2, 3, 4, 5, 6], 5, 0.001, False), # significant improvement - ([1, 2, 3, 4, 5, 6], 5, 0., False), # significant improvement + ([1, 2, 3, 4, 5, 6], 5, 0.0, False), # significant improvement ([1, 2, 3, 4, 5, 6], 5, 0.999, False), # significant improvement ([1, 2, 3, 4, 5, 6], 5, 5 - 1e-5, False), # significant improvement - ([1] * 6, 5, 0., True), # no significant improvement + ([1] * 6, 5, 0.0, True), # no significant improvement ([1] * 6, 5, 0.001, True), # no significant improvement ([1] * 6, 5, 5, True), # no significant improvement - ] + ], ) def test_should_stop(scores, n_iter_no_change, tol, stopping): - - gbdt = HistGradientBoostingClassifier( - n_iter_no_change=n_iter_no_change, tol=tol - ) + gbdt = HistGradientBoostingClassifier(n_iter_no_change=n_iter_no_change, tol=tol) assert gbdt._should_stop(scores) == stopping -def test_least_absolute_deviation(): +def test_absolute_error(): # For coverage only. X, y = make_regression(n_samples=500, random_state=0) - gbdt = HistGradientBoostingRegressor(loss='least_absolute_deviation', - random_state=0) + gbdt = HistGradientBoostingRegressor(loss="absolute_error", random_state=0) gbdt.fit(X, y) - assert gbdt.score(X, y) > .9 + assert gbdt.score(X, y) > 0.9 -def test_least_absolute_deviation_sample_weight(): +def test_absolute_error_sample_weight(): # non regression test for issue #19400 # make sure no error is thrown during fit of - # HistGradientBoostingRegressor with least_absolute_deviation loss function + # HistGradientBoostingRegressor with absolute_error loss function # and passing sample_weight rng = np.random.RandomState(0) n_samples = 100 X = rng.uniform(-1, 1, size=(n_samples, 2)) y = rng.uniform(-1, 1, size=n_samples) sample_weight = rng.uniform(0, 1, size=n_samples) - gbdt = HistGradientBoostingRegressor(loss='least_absolute_deviation') + gbdt = HistGradientBoostingRegressor(loss="absolute_error") gbdt.fit(X, y, sample_weight=sample_weight) -@pytest.mark.parametrize('y', [([1., -2., 0.]), ([0., 0., 0.])]) +@pytest.mark.parametrize("y", [([1.0, -2.0, 0.0]), ([0.0, 1.0, 2.0])]) +def test_gamma_y_positive(y): + # Test that ValueError is raised if any y_i <= 0. + err_msg = r"loss='gamma' requires strictly positive y." + gbdt = HistGradientBoostingRegressor(loss="gamma", random_state=0) + with pytest.raises(ValueError, match=err_msg): + gbdt.fit(np.zeros(shape=(len(y), 1)), y) + + +def test_gamma(): + # For a Gamma distributed target, we expect an HGBT trained with the Gamma deviance + # (loss) to give better results than an HGBT with any other loss function, measured + # in out-of-sample Gamma deviance as metric/score. + # Note that squared error could potentially predict negative values which is + # invalid (np.inf) for the Gamma deviance. A Poisson HGBT (having a log link) + # does not have that defect. + # Important note: It seems that a Poisson HGBT almost always has better + # out-of-sample performance than the Gamma HGBT, measured in Gamma deviance. + # LightGBM shows the same behaviour. Hence, we only compare to a squared error + # HGBT, but not to a Poisson deviance HGBT. + rng = np.random.RandomState(42) + n_train, n_test, n_features = 500, 100, 20 + X = make_low_rank_matrix( + n_samples=n_train + n_test, + n_features=n_features, + random_state=rng, + ) + # We create a log-linear Gamma model. This gives y.min ~ 1e-2, y.max ~ 1e2 + coef = rng.uniform(low=-10, high=20, size=n_features) + # Numpy parametrizes gamma(shape=k, scale=theta) with mean = k * theta and + # variance = k * theta^2. We parametrize it instead with mean = exp(X @ coef) + # and variance = dispersion * mean^2 by setting k = 1 / dispersion, + # theta = dispersion * mean. + dispersion = 0.5 + y = rng.gamma(shape=1 / dispersion, scale=dispersion * np.exp(X @ coef)) + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=n_test, random_state=rng + ) + gbdt_gamma = HistGradientBoostingRegressor(loss="gamma", random_state=123) + gbdt_mse = HistGradientBoostingRegressor(loss="squared_error", random_state=123) + dummy = DummyRegressor(strategy="mean") + for model in (gbdt_gamma, gbdt_mse, dummy): + model.fit(X_train, y_train) + + for X, y in [(X_train, y_train), (X_test, y_test)]: + loss_gbdt_gamma = mean_gamma_deviance(y, gbdt_gamma.predict(X)) + # We restrict the squared error HGBT to predict at least the minimum seen y at + # train time to make it strictly positive. + loss_gbdt_mse = mean_gamma_deviance( + y, np.maximum(np.min(y_train), gbdt_mse.predict(X)) + ) + loss_dummy = mean_gamma_deviance(y, dummy.predict(X)) + assert loss_gbdt_gamma < loss_dummy + assert loss_gbdt_gamma < loss_gbdt_mse + + +@pytest.mark.parametrize("quantile", [0.2, 0.5, 0.8]) +def test_quantile_asymmetric_error(quantile): + """Test quantile regression for asymmetric distributed targets.""" + n_samples = 10_000 + rng = np.random.RandomState(42) + # take care that X @ coef + intercept > 0 + X = np.concatenate( + ( + np.abs(rng.randn(n_samples)[:, None]), + -rng.randint(2, size=(n_samples, 1)), + ), + axis=1, + ) + intercept = 1.23 + coef = np.array([0.5, -2]) + # For an exponential distribution with rate lambda, e.g. exp(-lambda * x), + # the quantile at level q is: + # quantile(q) = - log(1 - q) / lambda + # scale = 1/lambda = -quantile(q) / log(1-q) + y = rng.exponential( + scale=-(X @ coef + intercept) / np.log(1 - quantile), size=n_samples + ) + model = HistGradientBoostingRegressor( + loss="quantile", + quantile=quantile, + max_iter=25, + random_state=0, + max_leaf_nodes=10, + ).fit(X, y) + assert_allclose(np.mean(model.predict(X) > y), quantile, rtol=1e-2) + + pinball_loss = PinballLoss(quantile=quantile) + loss_true_quantile = pinball_loss(y, X @ coef + intercept) + loss_pred_quantile = pinball_loss(y, model.predict(X)) + # we are overfitting + assert loss_pred_quantile <= loss_true_quantile + + +@pytest.mark.parametrize("y", [([1.0, -2.0, 0.0]), ([0.0, 0.0, 0.0])]) def test_poisson_y_positive(y): # Test that ValueError is raised if either one y_i < 0 or sum(y_i) <= 0. err_msg = r"loss='poisson' requires non-negative y and sum\(y\) > 0." - gbdt = HistGradientBoostingRegressor(loss='poisson', random_state=0) + gbdt = HistGradientBoostingRegressor(loss="poisson", random_state=0) with pytest.raises(ValueError, match=err_msg): gbdt.fit(np.zeros(shape=(len(y), 1)), y) @@ -231,26 +348,26 @@ def test_poisson(): # than least squares measured in Poisson deviance as metric. rng = np.random.RandomState(42) n_train, n_test, n_features = 500, 100, 100 - X = make_low_rank_matrix(n_samples=n_train+n_test, n_features=n_features, - random_state=rng) + X = make_low_rank_matrix( + n_samples=n_train + n_test, n_features=n_features, random_state=rng + ) # We create a log-linear Poisson model and downscale coef as it will get # exponentiated. coef = rng.uniform(low=-2, high=2, size=n_features) / np.max(X, axis=0) y = rng.poisson(lam=np.exp(X @ coef)) - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=n_test, - random_state=rng) - gbdt_pois = HistGradientBoostingRegressor(loss='poisson', random_state=rng) - gbdt_ls = HistGradientBoostingRegressor(loss='least_squares', - random_state=rng) + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=n_test, random_state=rng + ) + gbdt_pois = HistGradientBoostingRegressor(loss="poisson", random_state=rng) + gbdt_ls = HistGradientBoostingRegressor(loss="squared_error", random_state=rng) gbdt_pois.fit(X_train, y_train) gbdt_ls.fit(X_train, y_train) dummy = DummyRegressor(strategy="mean").fit(X_train, y_train) for X, y in [(X_train, y_train), (X_test, y_test)]: metric_pois = mean_poisson_deviance(y, gbdt_pois.predict(X)) - # least_squares might produce non-positive predictions => clip - metric_ls = mean_poisson_deviance(y, np.clip(gbdt_ls.predict(X), 1e-15, - None)) + # squared_error might produce non-positive predictions => clip + metric_ls = mean_poisson_deviance(y, np.clip(gbdt_ls.predict(X), 1e-15, None)) metric_dummy = mean_poisson_deviance(y, dummy.predict(X)) assert metric_pois < metric_ls assert metric_pois < metric_dummy @@ -261,11 +378,9 @@ def test_binning_train_validation_are_separated(): # See issue 13926 rng = np.random.RandomState(0) - validation_fraction = .2 + validation_fraction = 0.2 gb = HistGradientBoostingClassifier( - early_stopping=True, - validation_fraction=validation_fraction, - random_state=rng + early_stopping=True, validation_fraction=validation_fraction, random_state=rng ) gb.fit(X_classification, y_classification) mapper_training_data = gb._bin_mapper @@ -276,10 +391,14 @@ def test_binning_train_validation_are_separated(): mapper_whole_data.fit(X_classification) n_samples = X_classification.shape[0] - assert np.all(mapper_training_data.n_bins_non_missing_ == - int((1 - validation_fraction) * n_samples)) - assert np.all(mapper_training_data.n_bins_non_missing_ != - mapper_whole_data.n_bins_non_missing_) + assert np.all( + mapper_training_data.n_bins_non_missing_ + == int((1 - validation_fraction) * n_samples) + ) + assert np.all( + mapper_training_data.n_bins_non_missing_ + != mapper_whole_data.n_bins_non_missing_ + ) def test_missing_values_trivial(): @@ -292,7 +411,7 @@ def test_missing_values_trivial(): rng = np.random.RandomState(0) X = rng.normal(size=(n_samples, n_features)) - mask = rng.binomial(1, .5, size=X.shape).astype(bool) + mask = rng.binomial(1, 0.5, size=X.shape).astype(bool) X[mask] = np.nan y = mask.ravel() gb = HistGradientBoostingClassifier() @@ -301,31 +420,44 @@ def test_missing_values_trivial(): assert gb.score(X, y) == pytest.approx(1) -@pytest.mark.parametrize('problem', ('classification', 'regression')) +@pytest.mark.parametrize("problem", ("classification", "regression")) @pytest.mark.parametrize( - 'missing_proportion, expected_min_score_classification, ' - 'expected_min_score_regression', [ - (.1, .97, .89), - (.2, .93, .81), - (.5, .79, .52)]) -def test_missing_values_resilience(problem, missing_proportion, - expected_min_score_classification, - expected_min_score_regression): + ( + "missing_proportion, expected_min_score_classification, " + "expected_min_score_regression" + ), + [(0.1, 0.97, 0.89), (0.2, 0.93, 0.81), (0.5, 0.79, 0.52)], +) +def test_missing_values_resilience( + problem, + missing_proportion, + expected_min_score_classification, + expected_min_score_regression, +): # Make sure the estimators can deal with missing values and still yield # decent predictions rng = np.random.RandomState(0) n_samples = 1000 n_features = 2 - if problem == 'regression': - X, y = make_regression(n_samples=n_samples, n_features=n_features, - n_informative=n_features, random_state=rng) + if problem == "regression": + X, y = make_regression( + n_samples=n_samples, + n_features=n_features, + n_informative=n_features, + random_state=rng, + ) gb = HistGradientBoostingRegressor() expected_min_score = expected_min_score_regression else: - X, y = make_classification(n_samples=n_samples, n_features=n_features, - n_informative=n_features, n_redundant=0, - n_repeated=0, random_state=rng) + X, y = make_classification( + n_samples=n_samples, + n_features=n_features, + n_informative=n_features, + n_redundant=0, + n_repeated=0, + random_state=rng, + ) gb = HistGradientBoostingClassifier() expected_min_score = expected_min_score_classification @@ -337,10 +469,14 @@ def test_missing_values_resilience(problem, missing_proportion, assert gb.score(X, y) > expected_min_score -@pytest.mark.parametrize('data', [ - make_classification(random_state=0, n_classes=2), - make_classification(random_state=0, n_classes=3, n_informative=3) -], ids=['binary_crossentropy', 'categorical_crossentropy']) +@pytest.mark.parametrize( + "data", + [ + make_classification(random_state=0, n_classes=2), + make_classification(random_state=0, n_classes=3, n_informative=3), + ], + ids=["binary_log_loss", "multiclass_log_loss"], +) def test_zero_division_hessians(data): # non regression test for issue #14018 # make sure we avoid zero division errors when computing the leaves values. @@ -361,19 +497,20 @@ def test_small_trainset(): original_distrib = {0: 0.1, 1: 0.2, 2: 0.3, 3: 0.4} rng = np.random.RandomState(42) X = rng.randn(n_samples).reshape(n_samples, 1) - y = [[class_] * int(prop * n_samples) for (class_, prop) - in original_distrib.items()] + y = [ + [class_] * int(prop * n_samples) for (class_, prop) in original_distrib.items() + ] y = shuffle(np.concatenate(y)) gb = HistGradientBoostingClassifier() # Compute the small training set - X_small, y_small, _ = gb._get_small_trainset(X, y, seed=42, - sample_weight_train=None) + X_small, y_small, *_ = gb._get_small_trainset( + X, y, seed=42, sample_weight_train=None + ) # Compute the class distribution in the small training set unique, counts = np.unique(y_small, return_counts=True) - small_distrib = {class_: count / 10000 for (class_, count) - in zip(unique, counts)} + small_distrib = {class_: count / 10000 for (class_, count) in zip(unique, counts)} # Test that the small training set has the expected length assert X_small.shape[0] == 10000 @@ -405,10 +542,9 @@ def test_missing_values_minmax_imputation(): # "Missing In Attributes" (MIA) missing value handling for decision trees # https://www.sciencedirect.com/science/article/abs/pii/S0167865508000305 # The implementation of MIA as an imputation transformer was suggested by - # "Remark 3" in https://arxiv.org/abs/1902.06931 + # "Remark 3" in :arxiv:'<1902.06931>` class MinMaxImputer(TransformerMixin, BaseEstimator): - def fit(self, X, y=None): mm = MinMaxScaler().fit(X) self.data_min_ = mm.data_min_ @@ -427,13 +563,14 @@ def transform(self, X): def make_missing_value_data(n_samples=int(1e4), seed=0): rng = np.random.RandomState(seed) - X, y = make_regression(n_samples=n_samples, n_features=4, - random_state=rng) + X, y = make_regression(n_samples=n_samples, n_features=4, random_state=rng) # Pre-bin the data to ensure a deterministic handling by the 2 # strategies and also make it easier to insert np.nan in a structured # way: - X = KBinsDiscretizer(n_bins=42, encode="ordinal").fit_transform(X) + X = KBinsDiscretizer( + n_bins=42, encode="ordinal", quantile_method="averaged_inverted_cdf" + ).fit_transform(X) # First feature has missing values completely at random: rnd_mask = rng.rand(X.shape[0]) > 0.9 @@ -465,25 +602,22 @@ def make_missing_value_data(n_samples=int(1e4), seed=0): # n_samples need to be large enough to minimize the likelihood of having # several candidate splits with the same gain value in a given tree. X_train, X_test, y_train, y_test = make_missing_value_data( - n_samples=int(1e4), seed=0) + n_samples=int(1e4), seed=0 + ) # Use a small number of leaf nodes and iterations so as to keep # under-fitting models to minimize the likelihood of ties when training the # model. - gbm1 = HistGradientBoostingRegressor(max_iter=100, - max_leaf_nodes=5, - random_state=0) + gbm1 = HistGradientBoostingRegressor(max_iter=100, max_leaf_nodes=5, random_state=0) gbm1.fit(X_train, y_train) gbm2 = make_pipeline(MinMaxImputer(), clone(gbm1)) gbm2.fit(X_train, y_train) # Check that the model reach the same score: - assert gbm1.score(X_train, y_train) == \ - pytest.approx(gbm2.score(X_train, y_train)) + assert gbm1.score(X_train, y_train) == pytest.approx(gbm2.score(X_train, y_train)) - assert gbm1.score(X_test, y_test) == \ - pytest.approx(gbm2.score(X_test, y_test)) + assert gbm1.score(X_test, y_test) == pytest.approx(gbm2.score(X_test, y_test)) # Check the individual prediction match as a finer grained # decision function check. @@ -505,14 +639,14 @@ def test_infinite_values(): def test_consistent_lengths(): X = np.array([-np.inf, 0, 1, np.inf]).reshape(-1, 1) y = np.array([0, 0, 1, 1]) - sample_weight = np.array([.1, .3, .1]) + sample_weight = np.array([0.1, 0.3, 0.1]) gbdt = HistGradientBoostingRegressor() - with pytest.raises(ValueError, - match=r"sample_weight.shape == \(3,\), expected"): + with pytest.raises(ValueError, match=r"sample_weight.shape == \(3,\), expected"): gbdt.fit(X, y, sample_weight) - with pytest.raises(ValueError, - match="Found input variables with inconsistent number"): + with pytest.raises( + ValueError, match="Found input variables with inconsistent number" + ): gbdt.fit(X, y[1:]) @@ -526,31 +660,21 @@ def test_infinite_values_missing_values(): y_isnan = np.isnan(X.ravel()) y_isinf = X.ravel() == np.inf - stump_clf = HistGradientBoostingClassifier(min_samples_leaf=1, max_iter=1, - learning_rate=1, max_depth=2) + stump_clf = HistGradientBoostingClassifier( + min_samples_leaf=1, max_iter=1, learning_rate=1, max_depth=2 + ) assert stump_clf.fit(X, y_isinf).score(X, y_isinf) == 1 assert stump_clf.fit(X, y_isnan).score(X, y_isnan) == 1 -def test_crossentropy_binary_problem(): - # categorical_crossentropy should only be used if there are more than two - # classes present. PR #14869 - X = [[1], [0]] - y = [0, 1] - gbrt = HistGradientBoostingClassifier(loss='categorical_crossentropy') - with pytest.raises(ValueError, - match="'categorical_crossentropy' is not suitable for"): - gbrt.fit(X, y) - - -@pytest.mark.parametrize("scoring", [None, 'loss']) +@pytest.mark.parametrize("scoring", [None, "loss"]) def test_string_target_early_stopping(scoring): # Regression tests for #14709 where the targets need to be encoded before # to compute the score rng = np.random.RandomState(42) X = rng.randn(100, 10) - y = np.array(['x'] * 50 + ['y'] * 50, dtype=object) + y = np.array(["x"] * 50 + ["y"] * 50, dtype=object) gbrt = HistGradientBoostingClassifier(n_iter_no_change=10, scoring=scoring) gbrt.fit(X, y) @@ -559,10 +683,7 @@ def test_zero_sample_weights_regression(): # Make sure setting a SW to zero amounts to ignoring the corresponding # sample - X = [[1, 0], - [1, 0], - [1, 0], - [0, 1]] + X = [[1, 0], [1, 0], [1, 0], [0, 1]] y = [0, 0, 1, 0] # ignore the first 2 training samples by setting their weight to 0 sample_weight = [0, 0, 1, 1] @@ -575,38 +696,27 @@ def test_zero_sample_weights_classification(): # Make sure setting a SW to zero amounts to ignoring the corresponding # sample - X = [[1, 0], - [1, 0], - [1, 0], - [0, 1]] + X = [[1, 0], [1, 0], [1, 0], [0, 1]] y = [0, 0, 1, 0] # ignore the first 2 training samples by setting their weight to 0 sample_weight = [0, 0, 1, 1] - gb = HistGradientBoostingClassifier(loss='binary_crossentropy', - min_samples_leaf=1) + gb = HistGradientBoostingClassifier(loss="log_loss", min_samples_leaf=1) gb.fit(X, y, sample_weight=sample_weight) assert_array_equal(gb.predict([[1, 0]]), [1]) - X = [[1, 0], - [1, 0], - [1, 0], - [0, 1], - [1, 1]] + X = [[1, 0], [1, 0], [1, 0], [0, 1], [1, 1]] y = [0, 0, 1, 0, 2] # ignore the first 2 training samples by setting their weight to 0 sample_weight = [0, 0, 1, 1, 1] - gb = HistGradientBoostingClassifier(loss='categorical_crossentropy', - min_samples_leaf=1) + gb = HistGradientBoostingClassifier(loss="log_loss", min_samples_leaf=1) gb.fit(X, y, sample_weight=sample_weight) assert_array_equal(gb.predict([[1, 0]]), [1]) -@pytest.mark.parametrize('problem', ( - 'regression', - 'binary_classification', - 'multiclass_classification' -)) -@pytest.mark.parametrize('duplication', ('half', 'all')) +@pytest.mark.parametrize( + "problem", ("regression", "binary_classification", "multiclass_classification") +) +@pytest.mark.parametrize("duplication", ("half", "all")) def test_sample_weight_effect(problem, duplication): # High level test to make sure that duplicating a sample is equivalent to # giving it weight of 2. @@ -616,16 +726,25 @@ def test_sample_weight_effect(problem, duplication): # sure only unique values are used so SW have no effect on binning. n_samples = 255 n_features = 2 - if problem == 'regression': - X, y = make_regression(n_samples=n_samples, n_features=n_features, - n_informative=n_features, random_state=0) + if problem == "regression": + X, y = make_regression( + n_samples=n_samples, + n_features=n_features, + n_informative=n_features, + random_state=0, + ) Klass = HistGradientBoostingRegressor else: - n_classes = 2 if problem == 'binary_classification' else 3 - X, y = make_classification(n_samples=n_samples, n_features=n_features, - n_informative=n_features, n_redundant=0, - n_clusters_per_class=1, - n_classes=n_classes, random_state=0) + n_classes = 2 if problem == "binary_classification" else 3 + X, y = make_classification( + n_samples=n_samples, + n_features=n_features, + n_informative=n_features, + n_redundant=0, + n_clusters_per_class=1, + n_classes=n_classes, + random_state=0, + ) Klass = HistGradientBoostingClassifier # This test can't pass if min_samples_leaf > 1 because that would force 2 @@ -635,7 +754,7 @@ def test_sample_weight_effect(problem, duplication): est = Klass(min_samples_leaf=1) # Create dataset with duplicate and corresponding sample weights - if duplication == 'half': + if duplication == "half": lim = n_samples // 2 else: lim = n_samples @@ -648,13 +767,11 @@ def test_sample_weight_effect(problem, duplication): est_dup = clone(est).fit(X_dup, y_dup) # checking raw_predict is stricter than just predict for classification - assert np.allclose(est_sw._raw_predict(X_dup), - est_dup._raw_predict(X_dup)) + assert np.allclose(est_sw._raw_predict(X_dup), est_dup._raw_predict(X_dup)) -@pytest.mark.parametrize('loss_name', ('least_squares', - 'least_absolute_deviation')) -def test_sum_hessians_are_sample_weight(loss_name): +@pytest.mark.parametrize("Loss", (HalfSquaredError, AbsoluteError)) +def test_sum_hessians_are_sample_weight(Loss): # For losses with constant hessians, the sum_hessians field of the # histograms must be equal to the sum of the sample weight of samples at # the corresponding bin. @@ -662,19 +779,27 @@ def test_sum_hessians_are_sample_weight(loss_name): rng = np.random.RandomState(0) n_samples = 1000 n_features = 2 - X, y = make_regression(n_samples=n_samples, n_features=n_features, - random_state=rng) + X, y = make_regression(n_samples=n_samples, n_features=n_features, random_state=rng) bin_mapper = _BinMapper() X_binned = bin_mapper.fit_transform(X) + # While sample weights are supposed to be positive, this still works. sample_weight = rng.normal(size=n_samples) - loss = _LOSSES[loss_name](sample_weight=sample_weight) - gradients, hessians = loss.init_gradients_and_hessians( - n_samples=n_samples, prediction_dim=1, sample_weight=sample_weight) - raw_predictions = rng.normal(size=(1, n_samples)) - loss.update_gradients_and_hessians(gradients, hessians, y, - raw_predictions, sample_weight) + loss = Loss(sample_weight=sample_weight) + gradients, hessians = loss.init_gradient_and_hessian( + n_samples=n_samples, dtype=G_H_DTYPE + ) + gradients, hessians = gradients.reshape((-1, 1)), hessians.reshape((-1, 1)) + raw_predictions = rng.normal(size=(n_samples, 1)) + loss.gradient_hessian( + y_true=y, + raw_prediction=raw_predictions, + sample_weight=sample_weight, + gradient_out=gradients, + hessian_out=hessians, + n_threads=n_threads, + ) # build sum_sample_weight which contains the sum of the sample weights at # each bin (for each feature). This must be equal to the sum_hessians @@ -682,19 +807,23 @@ def test_sum_hessians_are_sample_weight(loss_name): sum_sw = np.zeros(shape=(n_features, bin_mapper.n_bins)) for feature_idx in range(n_features): for sample_idx in range(n_samples): - sum_sw[feature_idx, X_binned[sample_idx, feature_idx]] += ( - sample_weight[sample_idx]) + sum_sw[feature_idx, X_binned[sample_idx, feature_idx]] += sample_weight[ + sample_idx + ] # Build histogram - grower = TreeGrower(X_binned, gradients[0], hessians[0], - n_bins=bin_mapper.n_bins) + grower = TreeGrower( + X_binned, gradients[:, 0], hessians[:, 0], n_bins=bin_mapper.n_bins + ) histograms = grower.histogram_builder.compute_histograms_brute( - grower.root.sample_indices) + grower.root.sample_indices + ) for feature_idx in range(n_features): for bin_idx in range(bin_mapper.n_bins): - assert histograms[feature_idx, bin_idx]['sum_hessians'] == ( - pytest.approx(sum_sw[feature_idx, bin_idx], rel=1e-5)) + assert histograms[feature_idx, bin_idx]["sum_hessians"] == ( + pytest.approx(sum_sw[feature_idx, bin_idx], rel=1e-5) + ) def test_max_depth_max_leaf_nodes(): @@ -704,8 +833,9 @@ def test_max_depth_max_leaf_nodes(): # met at the same time, which would lead to max_leaf_nodes not being # respected. X, y = make_classification(random_state=0) - est = HistGradientBoostingClassifier(max_depth=2, max_leaf_nodes=3, - max_iter=1).fit(X, y) + est = HistGradientBoostingClassifier(max_depth=2, max_leaf_nodes=3, max_iter=1).fit( + X, y + ) tree = est._predictors[0][0] assert tree.get_max_depth() == 2 assert tree.get_n_leaf_nodes() == 3 # would be 4 prior to bug fix @@ -716,8 +846,13 @@ def test_early_stopping_on_test_set_with_warm_start(): # warm_start=True, early_stopping is on, and no validation set X, y = make_classification(random_state=0) gb = HistGradientBoostingClassifier( - max_iter=1, scoring='loss', warm_start=True, early_stopping=True, - n_iter_no_change=1, validation_fraction=None) + max_iter=1, + scoring="loss", + warm_start=True, + early_stopping=True, + n_iter_no_change=1, + validation_fraction=None, + ) gb.fit(X, y) # does not raise on second call @@ -725,8 +860,70 @@ def test_early_stopping_on_test_set_with_warm_start(): gb.fit(X, y) -@pytest.mark.parametrize('Est', (HistGradientBoostingClassifier, - HistGradientBoostingRegressor)) +def test_early_stopping_with_sample_weights(monkeypatch): + """Check that sample weights is passed in to the scorer and _raw_predict is not + called.""" + + mock_scorer = Mock(side_effect=get_scorer("neg_median_absolute_error")) + + def mock_check_scoring(estimator, scoring): + assert scoring == "neg_median_absolute_error" + return mock_scorer + + monkeypatch.setattr( + sklearn.ensemble._hist_gradient_boosting.gradient_boosting, + "check_scoring", + mock_check_scoring, + ) + + X, y = make_regression(random_state=0) + sample_weight = np.ones_like(y) + hist = HistGradientBoostingRegressor( + max_iter=2, + early_stopping=True, + random_state=0, + scoring="neg_median_absolute_error", + ) + mock_raw_predict = Mock(side_effect=hist._raw_predict) + hist._raw_predict = mock_raw_predict + hist.fit(X, y, sample_weight=sample_weight) + + # _raw_predict should never be called with scoring as a string + assert mock_raw_predict.call_count == 0 + + # For scorer is called twice (train and val) for the baseline score, and twice + # per iteration (train and val) after that. So 6 times in total for `max_iter=2`. + assert mock_scorer.call_count == 6 + for arg_list in mock_scorer.call_args_list: + assert "sample_weight" in arg_list[1] + + +def test_raw_predict_is_called_with_custom_scorer(): + """Custom scorer will still call _raw_predict.""" + + mock_scorer = Mock(side_effect=get_scorer("neg_median_absolute_error")) + + X, y = make_regression(random_state=0) + hist = HistGradientBoostingRegressor( + max_iter=2, + early_stopping=True, + random_state=0, + scoring=mock_scorer, + ) + mock_raw_predict = Mock(side_effect=hist._raw_predict) + hist._raw_predict = mock_raw_predict + hist.fit(X, y) + + # `_raw_predict` and scorer is called twice (train and val) for the baseline score, + # and twice per iteration (train and val) after that. So 6 times in total for + # `max_iter=2`. + assert mock_raw_predict.call_count == 6 + assert mock_scorer.call_count == 6 + + +@pytest.mark.parametrize( + "Est", (HistGradientBoostingClassifier, HistGradientBoostingRegressor) +) def test_single_node_trees(Est): # Make sure it's still possible to build single-node trees. In that case # the value of the root is set to 0. That's a correct value: if the tree is @@ -741,45 +938,50 @@ def test_single_node_trees(Est): est.fit(X, y) assert all(len(predictor[0].nodes) == 1 for predictor in est._predictors) - assert all(predictor[0].nodes[0]['value'] == 0 - for predictor in est._predictors) + assert all(predictor[0].nodes[0]["value"] == 0 for predictor in est._predictors) # Still gives correct predictions thanks to the baseline prediction assert_allclose(est.predict(X), y) -@pytest.mark.parametrize('Est, loss, X, y', [ - ( - HistGradientBoostingClassifier, - BinaryCrossEntropy(sample_weight=None), - X_classification, - y_classification - ), - ( - HistGradientBoostingRegressor, - LeastSquares(sample_weight=None), - X_regression, - y_regression - ) -]) +@pytest.mark.parametrize( + "Est, loss, X, y", + [ + ( + HistGradientBoostingClassifier, + HalfBinomialLoss(sample_weight=None), + X_classification, + y_classification, + ), + ( + HistGradientBoostingRegressor, + HalfSquaredError(sample_weight=None), + X_regression, + y_regression, + ), + ], +) def test_custom_loss(Est, loss, X, y): est = Est(loss=loss, max_iter=20) est.fit(X, y) -@pytest.mark.parametrize('HistGradientBoosting, X, y', [ - (HistGradientBoostingClassifier, X_classification, y_classification), - (HistGradientBoostingRegressor, X_regression, y_regression), - (HistGradientBoostingClassifier, - X_multi_classification, y_multi_classification), -]) +@pytest.mark.parametrize( + "HistGradientBoosting, X, y", + [ + (HistGradientBoostingClassifier, X_classification, y_classification), + (HistGradientBoostingRegressor, X_regression, y_regression), + ( + HistGradientBoostingClassifier, + X_multi_classification, + y_multi_classification, + ), + ], +) def test_staged_predict(HistGradientBoosting, X, y): - # Test whether staged predictor eventually gives # the same prediction. X_train, X_test, y_train, y_test = train_test_split( - X, y, - test_size=0.5, - random_state=0 + X, y, test_size=0.5, random_state=0 ) gb = HistGradientBoosting(max_iter=10) @@ -794,12 +996,12 @@ def test_staged_predict(HistGradientBoosting, X, y): # trained from scratch. # this also test limit case when max_iter = 1 method_names = ( - ['predict'] if is_regressor(gb) - else ['predict', 'predict_proba', 'decision_function'] + ["predict"] + if is_regressor(gb) + else ["predict", "predict_proba", "decision_function"] ) for method_name in method_names: - - staged_method = getattr(gb, 'staged_' + method_name) + staged_method = getattr(gb, "staged_" + method_name) staged_predictions = list(staged_method(X_test)) assert len(staged_predictions) == gb.n_iter_ for n_iter, staged_predictions in enumerate(staged_method(X_test), 1): @@ -812,11 +1014,14 @@ def test_staged_predict(HistGradientBoosting, X, y): @pytest.mark.parametrize("insert_missing", [False, True]) -@pytest.mark.parametrize("Est", (HistGradientBoostingRegressor, - HistGradientBoostingClassifier)) +@pytest.mark.parametrize( + "Est", (HistGradientBoostingRegressor, HistGradientBoostingClassifier) +) @pytest.mark.parametrize("bool_categorical_parameter", [True, False]) -def test_unknown_categories_nan(insert_missing, Est, - bool_categorical_parameter): +@pytest.mark.parametrize("missing_value", [np.nan, -1]) +def test_unknown_categories_nan( + insert_missing, Est, bool_categorical_parameter, missing_value +): # Make sure no error is raised at predict if a category wasn't seen during # fit. We also make sure they're treated as nans. @@ -836,7 +1041,7 @@ def test_unknown_categories_nan(insert_missing, Est, if insert_missing: mask = rng.binomial(1, 0.01, size=X.shape).astype(bool) assert mask.sum() > 0 - X[mask] = np.nan + X[mask] = missing_value est = Est(max_iter=20, categorical_features=categorical_features).fit(X, y) assert_array_equal(est.is_categorical_, [False, True]) @@ -845,7 +1050,7 @@ def test_unknown_categories_nan(insert_missing, Est, # unknown categories will be treated as nans X_test = np.zeros((10, X.shape[1]), dtype=float) X_test[:5, 1] = 30 - X_test[5:, 1] = np.nan + X_test[5:, 1] = missing_value assert len(np.unique(est.predict(X_test))) == 1 @@ -871,12 +1076,27 @@ def test_categorical_encoding_strategies(): # influence predictions too much with max_iter = 1 assert 0.49 < y.mean() < 0.51 - clf_cat = HistGradientBoostingClassifier( - max_iter=1, max_depth=1, categorical_features=[False, True]) + native_cat_specs = [ + [False, True], + [1], + ] + try: + import pandas as pd + + X = pd.DataFrame(X, columns=["f_0", "f_1"]) + native_cat_specs.append(["f_1"]) + except ImportError: + pass + + for native_cat_spec in native_cat_specs: + clf_cat = HistGradientBoostingClassifier( + max_iter=1, max_depth=1, categorical_features=native_cat_spec + ) + clf_cat.fit(X, y) - # Using native categorical encoding, we get perfect predictions with just - # one split - assert cross_val_score(clf_cat, X, y).mean() == 1 + # Using native categorical encoding, we get perfect predictions with just + # one split + assert cross_val_score(clf_cat, X, y).mean() == 1 # quick sanity check for the bitset: 0, 2, 4 = 2**0 + 2**2 + 2**4 = 21 expected_left_bitset = [21, 0, 0, 0, 0, 0, 0, 0] @@ -885,60 +1105,125 @@ def test_categorical_encoding_strategies(): # Treating categories as ordered, we need more depth / more splits to get # the same predictions - clf_no_cat = HistGradientBoostingClassifier(max_iter=1, max_depth=4, - categorical_features=None) - assert cross_val_score(clf_no_cat, X, y).mean() < .9 + clf_no_cat = HistGradientBoostingClassifier( + max_iter=1, max_depth=4, categorical_features=None + ) + assert cross_val_score(clf_no_cat, X, y).mean() < 0.9 clf_no_cat.set_params(max_depth=5) assert cross_val_score(clf_no_cat, X, y).mean() == 1 # Using OHEd data, we need less splits than with pure OEd data, but we # still need more splits than with the native categorical splits - ct = make_column_transformer((OneHotEncoder(sparse=False), [1]), - remainder='passthrough') + ct = make_column_transformer( + (OneHotEncoder(sparse_output=False), [1]), remainder="passthrough" + ) X_ohe = ct.fit_transform(X) clf_no_cat.set_params(max_depth=2) - assert cross_val_score(clf_no_cat, X_ohe, y).mean() < .9 + assert cross_val_score(clf_no_cat, X_ohe, y).mean() < 0.9 clf_no_cat.set_params(max_depth=3) assert cross_val_score(clf_no_cat, X_ohe, y).mean() == 1 -@pytest.mark.parametrize('Est', (HistGradientBoostingClassifier, - HistGradientBoostingRegressor)) -@pytest.mark.parametrize("categorical_features, monotonic_cst, expected_msg", [ - (["hello", "world"], None, - ("categorical_features must be an array-like of bools or array-like of " - "ints.")), - ([0, -1], None, - (r"categorical_features set as integer indices must be in " - r"\[0, n_features - 1\]")), - ([True, True, False, False, True], None, - r"categorical_features set as a boolean mask must have shape " - r"\(n_features,\)"), - ([True, True, False, False], [0, -1, 0, 1], - "Categorical features cannot have monotonic constraints"), -]) -def test_categorical_spec_errors(Est, categorical_features, monotonic_cst, - expected_msg): +@pytest.mark.parametrize( + "Est", (HistGradientBoostingClassifier, HistGradientBoostingRegressor) +) +@pytest.mark.parametrize( + "categorical_features, monotonic_cst, expected_msg", + [ + ( + [b"hello", b"world"], + None, + re.escape( + "categorical_features must be an array-like of bool, int or str, " + "got: bytes40." + ), + ), + ( + np.array([b"hello", 1.3], dtype=object), + None, + re.escape( + "categorical_features must be an array-like of bool, int or str, " + "got: bytes, float." + ), + ), + ( + [0, -1], + None, + re.escape( + "categorical_features set as integer indices must be in " + "[0, n_features - 1]" + ), + ), + ( + [True, True, False, False, True], + None, + re.escape( + "categorical_features set as a boolean mask must have shape " + "(n_features,)" + ), + ), + ( + [True, True, False, False], + [0, -1, 0, 1], + "Categorical features cannot have monotonic constraints", + ), + ], +) +def test_categorical_spec_errors( + Est, categorical_features, monotonic_cst, expected_msg +): # Test errors when categories are specified incorrectly n_samples = 100 - X, y = make_classification(random_state=0, n_features=4, - n_samples=n_samples) + X, y = make_classification(random_state=0, n_features=4, n_samples=n_samples) rng = np.random.RandomState(0) X[:, 0] = rng.randint(0, 10, size=n_samples) X[:, 1] = rng.randint(0, 10, size=n_samples) - est = Est(categorical_features=categorical_features, - monotonic_cst=monotonic_cst) + est = Est(categorical_features=categorical_features, monotonic_cst=monotonic_cst) with pytest.raises(ValueError, match=expected_msg): est.fit(X, y) -@pytest.mark.parametrize('Est', (HistGradientBoostingClassifier, - HistGradientBoostingRegressor)) -@pytest.mark.parametrize('categorical_features', ([False, False], [])) -@pytest.mark.parametrize('as_array', (True, False)) +@pytest.mark.parametrize( + "Est", (HistGradientBoostingClassifier, HistGradientBoostingRegressor) +) +def test_categorical_spec_errors_with_feature_names(Est): + pd = pytest.importorskip("pandas") + n_samples = 10 + X = pd.DataFrame( + { + "f0": range(n_samples), + "f1": range(n_samples), + "f2": [1.0] * n_samples, + } + ) + y = [0, 1] * (n_samples // 2) + + est = Est(categorical_features=["f0", "f1", "f3"]) + expected_msg = re.escape( + "categorical_features has a item value 'f3' which is not a valid " + "feature name of the training data." + ) + with pytest.raises(ValueError, match=expected_msg): + est.fit(X, y) + + est = Est(categorical_features=["f0", "f1"]) + expected_msg = re.escape( + "categorical_features should be passed as an array of integers or " + "as a boolean mask when the model is fitted on data without feature " + "names." + ) + with pytest.raises(ValueError, match=expected_msg): + est.fit(X.to_numpy(), y) + + +@pytest.mark.parametrize( + "Est", (HistGradientBoostingClassifier, HistGradientBoostingRegressor) +) +@pytest.mark.parametrize("categorical_features", ([False, False], [])) +@pytest.mark.parametrize("as_array", (True, False)) def test_categorical_spec_no_categories(Est, categorical_features, as_array): # Make sure we can properly detect that no categorical features are present # even if the categorical_features parameter is not None @@ -950,24 +1235,27 @@ def test_categorical_spec_no_categories(Est, categorical_features, as_array): assert est.is_categorical_ is None -@pytest.mark.parametrize('Est', (HistGradientBoostingClassifier, - HistGradientBoostingRegressor)) -def test_categorical_bad_encoding_errors(Est): +@pytest.mark.parametrize( + "Est", (HistGradientBoostingClassifier, HistGradientBoostingRegressor) +) +@pytest.mark.parametrize( + "use_pandas, feature_name", [(False, "at index 0"), (True, "'f0'")] +) +def test_categorical_bad_encoding_errors(Est, use_pandas, feature_name): # Test errors when categories are encoded incorrectly gb = Est(categorical_features=[True], max_bins=2) - X = np.array([[0, 1, 2]]).T + if use_pandas: + pd = pytest.importorskip("pandas") + X = pd.DataFrame({"f0": [0, 1, 2]}) + else: + X = np.array([[0, 1, 2]]).T y = np.arange(3) - msg = ("Categorical feature at index 0 is expected to have a " - "cardinality <= 2") - with pytest.raises(ValueError, match=msg): - gb.fit(X, y) - - X = np.array([[0, 2]]).T - y = np.arange(2) - msg = ("Categorical feature at index 0 is expected to be encoded with " - "values < 2") + msg = ( + f"Categorical feature {feature_name} is expected to have a " + "cardinality <= 2 but actually has a cardinality of 3." + ) with pytest.raises(ValueError, match=msg): gb.fit(X, y) @@ -977,8 +1265,9 @@ def test_categorical_bad_encoding_errors(Est): gb.fit(X, y) -@pytest.mark.parametrize('Est', (HistGradientBoostingClassifier, - HistGradientBoostingRegressor)) +@pytest.mark.parametrize( + "Est", (HistGradientBoostingClassifier, HistGradientBoostingRegressor) +) def test_uint8_predict(Est): # Non regression test for # https://github.com/scikit-learn/scikit-learn/issues/18408 @@ -992,3 +1281,480 @@ def test_uint8_predict(Est): est = Est() est.fit(X, y) est.predict(X) + + +@pytest.mark.parametrize( + "interaction_cst, n_features, result", + [ + (None, 931, None), + ([{0, 1}], 2, [{0, 1}]), + ("pairwise", 2, [{0, 1}]), + ("pairwise", 4, [{0, 1}, {0, 2}, {0, 3}, {1, 2}, {1, 3}, {2, 3}]), + ("no_interactions", 2, [{0}, {1}]), + ("no_interactions", 4, [{0}, {1}, {2}, {3}]), + ([(1, 0), [5, 1]], 6, [{0, 1}, {1, 5}, {2, 3, 4}]), + ], +) +def test_check_interaction_cst(interaction_cst, n_features, result): + """Check that _check_interaction_cst returns the expected list of sets""" + est = HistGradientBoostingRegressor() + est.set_params(interaction_cst=interaction_cst) + assert est._check_interaction_cst(n_features) == result + + +def test_interaction_cst_numerically(): + """Check that interaction constraints have no forbidden interactions.""" + rng = np.random.RandomState(42) + n_samples = 1000 + X = rng.uniform(size=(n_samples, 2)) + # Construct y with a strong interaction term + # y = x0 + x1 + 5 * x0 * x1 + y = np.hstack((X, 5 * X[:, [0]] * X[:, [1]])).sum(axis=1) + + est = HistGradientBoostingRegressor(random_state=42) + est.fit(X, y) + est_no_interactions = HistGradientBoostingRegressor( + interaction_cst=[{0}, {1}], random_state=42 + ) + est_no_interactions.fit(X, y) + + delta = 0.25 + # Make sure we do not extrapolate out of the training set as tree-based estimators + # are very bad in doing so. + X_test = X[(X[:, 0] < 1 - delta) & (X[:, 1] < 1 - delta)] + X_delta_d_0 = X_test + [delta, 0] + X_delta_0_d = X_test + [0, delta] + X_delta_d_d = X_test + [delta, delta] + + # Note: For the y from above as a function of x0 and x1, we have + # y(x0+d, x1+d) = y(x0, x1) + 5 * d * (2/5 + x0 + x1) + 5 * d**2 + # y(x0+d, x1) = y(x0, x1) + 5 * d * (1/5 + x1) + # y(x0, x1+d) = y(x0, x1) + 5 * d * (1/5 + x0) + # Without interaction constraints, we would expect a result of 5 * d**2 for the + # following expression, but zero with constraints in place. + assert_allclose( + est_no_interactions.predict(X_delta_d_d) + + est_no_interactions.predict(X_test) + - est_no_interactions.predict(X_delta_d_0) + - est_no_interactions.predict(X_delta_0_d), + 0, + atol=1e-12, + ) + + # Correct result of the expressions is 5 * delta**2. But this is hard to achieve by + # a fitted tree-based model. However, with 100 iterations the expression should + # at least be positive! + assert np.all( + est.predict(X_delta_d_d) + + est.predict(X_test) + - est.predict(X_delta_d_0) + - est.predict(X_delta_0_d) + > 0.01 + ) + + +def test_no_user_warning_with_scoring(): + """Check that no UserWarning is raised when scoring is set. + + Non-regression test for #22907. + """ + pd = pytest.importorskip("pandas") + X, y = make_regression(n_samples=50, random_state=0) + X_df = pd.DataFrame(X, columns=[f"col{i}" for i in range(X.shape[1])]) + + est = HistGradientBoostingRegressor( + random_state=0, scoring="neg_mean_absolute_error", early_stopping=True + ) + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + est.fit(X_df, y) + + +def test_class_weights(): + """High level test to check class_weights.""" + n_samples = 255 + n_features = 2 + + X, y = make_classification( + n_samples=n_samples, + n_features=n_features, + n_informative=n_features, + n_redundant=0, + n_clusters_per_class=1, + n_classes=2, + random_state=0, + ) + y_is_1 = y == 1 + + # class_weight is the same as sample weights with the corresponding class + clf = HistGradientBoostingClassifier( + min_samples_leaf=2, random_state=0, max_depth=2 + ) + sample_weight = np.ones(shape=(n_samples)) + sample_weight[y_is_1] = 3.0 + clf.fit(X, y, sample_weight=sample_weight) + + class_weight = {0: 1.0, 1: 3.0} + clf_class_weighted = clone(clf).set_params(class_weight=class_weight) + clf_class_weighted.fit(X, y) + + assert_allclose(clf.decision_function(X), clf_class_weighted.decision_function(X)) + + # Check that sample_weight and class_weight are multiplicative + clf.fit(X, y, sample_weight=sample_weight**2) + clf_class_weighted.fit(X, y, sample_weight=sample_weight) + assert_allclose(clf.decision_function(X), clf_class_weighted.decision_function(X)) + + # Make imbalanced dataset + X_imb = np.concatenate((X[~y_is_1], X[y_is_1][:10])) + y_imb = np.concatenate((y[~y_is_1], y[y_is_1][:10])) + + # class_weight="balanced" is the same as sample_weights to be + # inversely proportional to n_samples / (n_classes * np.bincount(y)) + clf_balanced = clone(clf).set_params(class_weight="balanced") + clf_balanced.fit(X_imb, y_imb) + + class_weight = y_imb.shape[0] / (2 * np.bincount(y_imb)) + sample_weight = class_weight[y_imb] + clf_sample_weight = clone(clf).set_params(class_weight=None) + clf_sample_weight.fit(X_imb, y_imb, sample_weight=sample_weight) + + assert_allclose( + clf_balanced.decision_function(X_imb), + clf_sample_weight.decision_function(X_imb), + ) + + +def test_unknown_category_that_are_negative(): + """Check that unknown categories that are negative does not error. + + Non-regression test for #24274. + """ + rng = np.random.RandomState(42) + n_samples = 1000 + X = np.c_[rng.rand(n_samples), rng.randint(4, size=n_samples)] + y = np.zeros(shape=n_samples) + y[X[:, 1] % 2 == 0] = 1 + + hist = HistGradientBoostingRegressor( + random_state=0, + categorical_features=[False, True], + max_iter=10, + ).fit(X, y) + + # Check that negative values from the second column are treated like a + # missing category + X_test_neg = np.asarray([[1, -2], [3, -4]]) + X_test_nan = np.asarray([[1, np.nan], [3, np.nan]]) + + assert_allclose(hist.predict(X_test_neg), hist.predict(X_test_nan)) + + +@pytest.mark.parametrize( + ("GradientBoosting", "make_X_y"), + [ + (HistGradientBoostingClassifier, make_classification), + (HistGradientBoostingRegressor, make_regression), + ], +) +@pytest.mark.parametrize("sample_weight", [False, True]) +def test_X_val_in_fit(GradientBoosting, make_X_y, sample_weight, global_random_seed): + """Test that passing X_val, y_val in fit is same as validation fraction.""" + rng = np.random.RandomState(42) + n_samples = 100 + X, y = make_X_y(n_samples=n_samples, random_state=rng) + if sample_weight: + sample_weight = np.abs(rng.normal(size=n_samples)) + data = (X, y, sample_weight) + else: + sample_weight = None + data = (X, y) + rng_seed = global_random_seed + + # Fit with validation fraction and early stopping. + m1 = GradientBoosting( + early_stopping=True, + validation_fraction=0.5, + random_state=rng_seed, + ) + m1.fit(X, y, sample_weight) + + # Do train-test split ourselves. + rng = check_random_state(rng_seed) + # We do the same as in the fit method. + stratify = y if isinstance(m1, HistGradientBoostingClassifier) else None + random_seed = rng.randint(np.iinfo(np.uint32).max, dtype="u8") + X_train, X_val, y_train, y_val, *sw = train_test_split( + *data, + test_size=0.5, + stratify=stratify, + random_state=random_seed, + ) + if sample_weight is not None: + sample_weight_train = sw[0] + sample_weight_val = sw[1] + else: + sample_weight_train = None + sample_weight_val = None + m2 = GradientBoosting( + early_stopping=True, + random_state=rng_seed, + ) + m2.fit( + X_train, + y_train, + sample_weight=sample_weight_train, + X_val=X_val, + y_val=y_val, + sample_weight_val=sample_weight_val, + ) + + assert_allclose(m2.n_iter_, m1.n_iter_) + assert_allclose(m2.predict(X), m1.predict(X)) + + +def test_X_val_raises_missing_y_val(): + """Test that an error is raised if X_val given but y_val None.""" + X, y = make_classification(n_samples=4) + X, X_val = X[:2], X[2:] + y, y_val = y[:2], y[2:] + with pytest.raises( + ValueError, + match="X_val is provided, but y_val was not provided", + ): + HistGradientBoostingClassifier().fit(X, y, X_val=X_val) + with pytest.raises( + ValueError, + match="y_val is provided, but X_val was not provided", + ): + HistGradientBoostingClassifier().fit(X, y, y_val=y_val) + + +def test_X_val_raises_with_early_stopping_false(): + """Test that an error is raised if X_val given but early_stopping is False.""" + X, y = make_regression(n_samples=4) + X, X_val = X[:2], X[2:] + y, y_val = y[:2], y[2:] + with pytest.raises( + ValueError, + match="X_val and y_val are passed to fit while at the same time", + ): + HistGradientBoostingRegressor(early_stopping=False).fit( + X, y, X_val=X_val, y_val=y_val + ) + + +@pytest.mark.parametrize("dataframe_lib", ["pandas", "polars"]) +@pytest.mark.parametrize( + "HistGradientBoosting", + [HistGradientBoostingClassifier, HistGradientBoostingRegressor], +) +def test_dataframe_categorical_results_same_as_ndarray( + dataframe_lib, HistGradientBoosting +): + """Check that pandas categorical give the same results as ndarray.""" + pytest.importorskip(dataframe_lib) + + rng = np.random.RandomState(42) + n_samples = 5_000 + n_cardinality = 50 + max_bins = 100 + f_num = rng.rand(n_samples) + f_cat = rng.randint(n_cardinality, size=n_samples) + + # Make f_cat an informative feature + y = (f_cat % 3 == 0) & (f_num > 0.2) + + X = np.c_[f_num, f_cat] + f_cat = [f"cat{c:0>3}" for c in f_cat] + X_df = _convert_container( + np.asarray([f_num, f_cat]).T, + dataframe_lib, + ["f_num", "f_cat"], + categorical_feature_names=["f_cat"], + ) + + X_train, X_test, X_train_df, X_test_df, y_train, y_test = train_test_split( + X, X_df, y, random_state=0 + ) + + hist_kwargs = dict(max_iter=10, max_bins=max_bins, random_state=0) + hist_np = HistGradientBoosting(categorical_features=[False, True], **hist_kwargs) + hist_np.fit(X_train, y_train) + + hist_pd = HistGradientBoosting(categorical_features="from_dtype", **hist_kwargs) + hist_pd.fit(X_train_df, y_train) + + # Check categories are correct and sorted + categories = hist_pd._preprocessor.named_transformers_["encoder"].categories_[0] + assert_array_equal(categories, np.unique(f_cat)) + + assert len(hist_np._predictors) == len(hist_pd._predictors) + for predictor_1, predictor_2 in zip(hist_np._predictors, hist_pd._predictors): + assert len(predictor_1[0].nodes) == len(predictor_2[0].nodes) + + score_np = hist_np.score(X_test, y_test) + score_pd = hist_pd.score(X_test_df, y_test) + assert score_np == pytest.approx(score_pd) + assert_allclose(hist_np.predict(X_test), hist_pd.predict(X_test_df)) + + +@pytest.mark.parametrize("dataframe_lib", ["pandas", "polars"]) +@pytest.mark.parametrize( + "HistGradientBoosting", + [HistGradientBoostingClassifier, HistGradientBoostingRegressor], +) +def test_dataframe_categorical_errors(dataframe_lib, HistGradientBoosting): + """Check error cases for pandas categorical feature.""" + pytest.importorskip(dataframe_lib) + msg = "Categorical feature 'f_cat' is expected to have a cardinality <= 16" + hist = HistGradientBoosting(categorical_features="from_dtype", max_bins=16) + + rng = np.random.RandomState(42) + f_cat = rng.randint(0, high=100, size=100).astype(str) + X_df = _convert_container( + f_cat[:, None], dataframe_lib, ["f_cat"], categorical_feature_names=["f_cat"] + ) + y = rng.randint(0, high=2, size=100) + + with pytest.raises(ValueError, match=msg): + hist.fit(X_df, y) + + +@pytest.mark.parametrize("dataframe_lib", ["pandas", "polars"]) +def test_categorical_different_order_same_model(dataframe_lib): + """Check that the order of the categorical gives same model.""" + pytest.importorskip(dataframe_lib) + rng = np.random.RandomState(42) + n_samples = 1_000 + f_ints = rng.randint(low=0, high=2, size=n_samples) + + # Construct a target with some noise + y = f_ints.copy() + flipped = rng.choice([True, False], size=n_samples, p=[0.1, 0.9]) + y[flipped] = 1 - y[flipped] + + # Construct categorical where 0 -> A and 1 -> B and 1 -> A and 0 -> B + f_cat_a_b = np.asarray(["A", "B"])[f_ints] + f_cat_b_a = np.asarray(["B", "A"])[f_ints] + df_a_b = _convert_container( + f_cat_a_b[:, None], + dataframe_lib, + ["f_cat"], + categorical_feature_names=["f_cat"], + ) + df_b_a = _convert_container( + f_cat_b_a[:, None], + dataframe_lib, + ["f_cat"], + categorical_feature_names=["f_cat"], + ) + + hist_a_b = HistGradientBoostingClassifier( + categorical_features="from_dtype", random_state=0 + ) + hist_b_a = HistGradientBoostingClassifier( + categorical_features="from_dtype", random_state=0 + ) + + hist_a_b.fit(df_a_b, y) + hist_b_a.fit(df_b_a, y) + + assert len(hist_a_b._predictors) == len(hist_b_a._predictors) + for predictor_1, predictor_2 in zip(hist_a_b._predictors, hist_b_a._predictors): + assert len(predictor_1[0].nodes) == len(predictor_2[0].nodes) + + +def get_different_bitness_node_ndarray(node_ndarray): + new_dtype_for_indexing_fields = np.int64 if _IS_32BIT else np.int32 + + # field names in Node struct with np.intp types (see + # sklearn/ensemble/_hist_gradient_boosting/common.pyx) + indexing_field_names = ["feature_idx"] + + new_dtype_dict = { + name: dtype for name, (dtype, _) in node_ndarray.dtype.fields.items() + } + for name in indexing_field_names: + new_dtype_dict[name] = new_dtype_for_indexing_fields + + new_dtype = np.dtype( + {"names": list(new_dtype_dict.keys()), "formats": list(new_dtype_dict.values())} + ) + return node_ndarray.astype(new_dtype, casting="same_kind") + + +def reduce_predictor_with_different_bitness(predictor): + cls, args, state = predictor.__reduce__() + + new_state = state.copy() + new_state["nodes"] = get_different_bitness_node_ndarray(new_state["nodes"]) + + return (cls, args, new_state) + + +def test_different_bitness_pickle(): + X, y = make_classification(random_state=0) + + clf = HistGradientBoostingClassifier(random_state=0, max_depth=3) + clf.fit(X, y) + score = clf.score(X, y) + + def pickle_dump_with_different_bitness(): + f = io.BytesIO() + p = pickle.Pickler(f) + p.dispatch_table = copyreg.dispatch_table.copy() + p.dispatch_table[TreePredictor] = reduce_predictor_with_different_bitness + + p.dump(clf) + f.seek(0) + return f + + # Simulate loading a pickle of the same model trained on a platform with different + # bitness that than the platform it will be used to make predictions on: + new_clf = pickle.load(pickle_dump_with_different_bitness()) + new_score = new_clf.score(X, y) + assert score == pytest.approx(new_score) + + +def test_different_bitness_joblib_pickle(): + # Make sure that a platform specific pickle generated on a 64 bit + # platform can be converted at pickle load time into an estimator + # with Cython code that works with the host's native integer precision + # to index nodes in the tree data structure when the host is a 32 bit + # platform (and vice versa). + # + # This is in particular useful to be able to train a model on a 64 bit Linux + # server and deploy the model as part of a (32 bit) WASM in-browser + # application using pyodide. + X, y = make_classification(random_state=0) + + clf = HistGradientBoostingClassifier(random_state=0, max_depth=3) + clf.fit(X, y) + score = clf.score(X, y) + + def joblib_dump_with_different_bitness(): + f = io.BytesIO() + p = NumpyPickler(f) + p.dispatch_table = copyreg.dispatch_table.copy() + p.dispatch_table[TreePredictor] = reduce_predictor_with_different_bitness + + p.dump(clf) + f.seek(0) + return f + + new_clf = joblib.load(joblib_dump_with_different_bitness()) + new_score = new_clf.score(X, y) + assert score == pytest.approx(new_score) + + +def test_pandas_nullable_dtype(): + # Non regression test for https://github.com/scikit-learn/scikit-learn/issues/28317 + pd = pytest.importorskip("pandas") + + rng = np.random.default_rng(0) + X = pd.DataFrame({"a": rng.integers(10, size=100)}).astype(pd.Int64Dtype()) + y = rng.integers(2, size=100) + + clf = HistGradientBoostingClassifier() + clf.fit(X, y) diff --git a/sklearn/ensemble/_hist_gradient_boosting/tests/test_grower.py b/sklearn/ensemble/_hist_gradient_boosting/tests/test_grower.py index 4e76422cbbef8..a55cb871e3c72 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/tests/test_grower.py +++ b/sklearn/ensemble/_hist_gradient_boosting/tests/test_grower.py @@ -1,18 +1,21 @@ import numpy as np import pytest +from numpy.testing import assert_allclose, assert_array_equal from pytest import approx -from numpy.testing import assert_array_equal -from numpy.testing import assert_allclose -from sklearn.preprocessing import OneHotEncoder -from sklearn.ensemble._hist_gradient_boosting.grower import TreeGrower from sklearn.ensemble._hist_gradient_boosting.binning import _BinMapper -from sklearn.ensemble._hist_gradient_boosting.common import X_BINNED_DTYPE -from sklearn.ensemble._hist_gradient_boosting.common import X_DTYPE -from sklearn.ensemble._hist_gradient_boosting.common import Y_DTYPE -from sklearn.ensemble._hist_gradient_boosting.common import G_H_DTYPE from sklearn.ensemble._hist_gradient_boosting.common import ( - X_BITSET_INNER_DTYPE) + G_H_DTYPE, + X_BINNED_DTYPE, + X_BITSET_INNER_DTYPE, + X_DTYPE, + Y_DTYPE, +) +from sklearn.ensemble._hist_gradient_boosting.grower import TreeGrower +from sklearn.preprocessing import OneHotEncoder +from sklearn.utils._openmp_helpers import _openmp_effective_n_threads + +n_threads = _openmp_effective_n_threads() def _make_training_data(n_bins=256, constant_hessian=True): @@ -21,8 +24,7 @@ def _make_training_data(n_bins=256, constant_hessian=True): # Generate some test data directly binned so as to test the grower code # independently of the binning logic. - X_binned = rng.randint(0, n_bins - 1, size=(n_samples, 2), - dtype=X_BINNED_DTYPE) + X_binned = rng.randint(0, n_bins - 1, size=(n_samples, 2), dtype=X_BINNED_DTYPE) X_binned = np.asfortranarray(X_binned) def true_decision_function(input_features): @@ -37,8 +39,7 @@ def true_decision_function(input_features): else: return -1 if input_features[1] <= n_bins // 3 else 1 - target = np.array([true_decision_function(x) for x in X_binned], - dtype=Y_DTYPE) + target = np.array([true_decision_function(x) for x in X_binned], dtype=Y_DTYPE) # Assume a square loss applied to an initial model that always predicts 0 # (hardcoded for this test): @@ -56,33 +57,35 @@ def _check_children_consistency(parent, left, right): assert parent.right_child is right # each sample from the parent is propagated to one of the two children - assert (len(left.sample_indices) + len(right.sample_indices) - == len(parent.sample_indices)) + assert len(left.sample_indices) + len(right.sample_indices) == len( + parent.sample_indices + ) - assert (set(left.sample_indices).union(set(right.sample_indices)) - == set(parent.sample_indices)) + assert set(left.sample_indices).union(set(right.sample_indices)) == set( + parent.sample_indices + ) # samples are sent either to the left or the right node, never to both - assert (set(left.sample_indices).intersection(set(right.sample_indices)) - == set()) + assert set(left.sample_indices).intersection(set(right.sample_indices)) == set() @pytest.mark.parametrize( - 'n_bins, constant_hessian, stopping_param, shrinkage', + "n_bins, constant_hessian, stopping_param, shrinkage", [ (11, True, "min_gain_to_split", 0.5), - (11, False, "min_gain_to_split", 1.), - (11, True, "max_leaf_nodes", 1.), + (11, False, "min_gain_to_split", 1.0), + (11, True, "max_leaf_nodes", 1.0), (11, False, "max_leaf_nodes", 0.1), (42, True, "max_leaf_nodes", 0.01), - (42, False, "max_leaf_nodes", 1.), - (256, True, "min_gain_to_split", 1.), + (42, False, "max_leaf_nodes", 1.0), + (256, True, "min_gain_to_split", 1.0), (256, True, "max_leaf_nodes", 0.1), - ] + ], ) def test_grow_tree(n_bins, constant_hessian, stopping_param, shrinkage): X_binned, all_gradients, all_hessians = _make_training_data( - n_bins=n_bins, constant_hessian=constant_hessian) + n_bins=n_bins, constant_hessian=constant_hessian + ) n_samples = X_binned.shape[0] if stopping_param == "max_leaf_nodes": @@ -90,11 +93,17 @@ def test_grow_tree(n_bins, constant_hessian, stopping_param, shrinkage): else: stopping_param = {"min_gain_to_split": 0.01} - grower = TreeGrower(X_binned, all_gradients, all_hessians, - n_bins=n_bins, shrinkage=shrinkage, - min_samples_leaf=1, **stopping_param) + grower = TreeGrower( + X_binned, + all_gradients, + all_hessians, + n_bins=n_bins, + shrinkage=shrinkage, + min_samples_leaf=1, + **stopping_param, + ) - # The root node is not yet splitted, but the best possible split has + # The root node is not yet split, but the best possible split has # already been evaluated: assert grower.root.left_child is None assert grower.root.right_child is None @@ -108,7 +117,7 @@ def test_grow_tree(n_bins, constant_hessian, stopping_param, shrinkage): # for each of the two newly introduced children nodes. left_node, right_node = grower.split_next() - # All training samples have ben splitted in the two nodes, approximately + # All training samples have ben split in the two nodes, approximately # 50%/50% _check_children_consistency(grower.root, left_node, right_node) assert len(left_node.sample_indices) > 0.4 * n_samples @@ -119,9 +128,9 @@ def test_grow_tree(n_bins, constant_hessian, stopping_param, shrinkage): assert left_node.split_info.gain < grower.min_gain_to_split assert left_node in grower.finalized_leaves - # The right node can still be splitted further, this time on feature #1 + # The right node can still be split further, this time on feature #1 split_info = right_node.split_info - assert split_info.gain > 1. + assert split_info.gain > 1.0 assert split_info.feature_idx == 1 assert split_info.bin_idx == n_bins // 3 assert right_node.left_child is None @@ -145,18 +154,22 @@ def test_grow_tree(n_bins, constant_hessian, stopping_param, shrinkage): # Check the values of the leaves: assert grower.root.left_child.value == approx(shrinkage) assert grower.root.right_child.left_child.value == approx(shrinkage) - assert grower.root.right_child.right_child.value == approx(-shrinkage, - rel=1e-3) + assert grower.root.right_child.right_child.value == approx(-shrinkage, rel=1e-3) def test_predictor_from_grower(): # Build a tree on the toy 3-leaf dataset to extract the predictor. n_bins = 256 - X_binned, all_gradients, all_hessians = _make_training_data( - n_bins=n_bins) - grower = TreeGrower(X_binned, all_gradients, all_hessians, - n_bins=n_bins, shrinkage=1., - max_leaf_nodes=3, min_samples_leaf=5) + X_binned, all_gradients, all_hessians = _make_training_data(n_bins=n_bins) + grower = TreeGrower( + X_binned, + all_gradients, + all_hessians, + n_bins=n_bins, + shrinkage=1.0, + max_leaf_nodes=3, + min_samples_leaf=5, + ) grower.grow() assert grower.n_nodes == 5 # (2 decision nodes + 3 leaves) @@ -167,35 +180,38 @@ def test_predictor_from_grower(): binning_thresholds=np.zeros((X_binned.shape[1], n_bins)) ) assert predictor.nodes.shape[0] == 5 - assert predictor.nodes['is_leaf'].sum() == 3 + assert predictor.nodes["is_leaf"].sum() == 3 # Probe some predictions for each leaf of the tree # each group of 3 samples corresponds to a condition in _make_training_data - input_data = np.array([ - [0, 0], - [42, 99], - [128, 254], - - [129, 0], - [129, 85], - [254, 85], - - [129, 86], - [129, 254], - [242, 100], - ], dtype=np.uint8) + input_data = np.array( + [ + [0, 0], + [42, 99], + [128, 254], + [129, 0], + [129, 85], + [254, 85], + [129, 86], + [129, 254], + [242, 100], + ], + dtype=np.uint8, + ) missing_values_bin_idx = n_bins - 1 - predictions = predictor.predict_binned(input_data, missing_values_bin_idx) + predictions = predictor.predict_binned( + input_data, missing_values_bin_idx, n_threads + ) expected_targets = [1, 1, 1, 1, 1, 1, -1, -1, -1] assert np.allclose(predictions, expected_targets) # Check that training set can be recovered exactly: - predictions = predictor.predict_binned(X_binned, missing_values_bin_idx) + predictions = predictor.predict_binned(X_binned, missing_values_bin_idx, n_threads) assert np.allclose(predictions, -all_gradients) @pytest.mark.parametrize( - 'n_samples, min_samples_leaf, n_bins, constant_hessian, noise', + "n_samples, min_samples_leaf, n_bins, constant_hessian, noise", [ (11, 10, 7, True, 0), (13, 10, 42, False, 0), @@ -204,10 +220,9 @@ def test_predictor_from_grower(): (200, 42, 42, False, 0), (300, 55, 255, True, 0.1), (300, 301, 255, True, 0.1), - ] + ], ) -def test_min_samples_leaf(n_samples, min_samples_leaf, n_bins, - constant_hessian, noise): +def test_min_samples_leaf(n_samples, min_samples_leaf, n_bins, constant_hessian, noise): rng = np.random.RandomState(seed=0) # data = linear target, 3 features, 1 irrelevant. X = rng.normal(size=(n_samples, 3)) @@ -221,27 +236,29 @@ def test_min_samples_leaf(n_samples, min_samples_leaf, n_bins, all_gradients = y.astype(G_H_DTYPE) shape_hessian = 1 if constant_hessian else all_gradients.shape all_hessians = np.ones(shape=shape_hessian, dtype=G_H_DTYPE) - grower = TreeGrower(X, all_gradients, all_hessians, - n_bins=n_bins, shrinkage=1., - min_samples_leaf=min_samples_leaf, - max_leaf_nodes=n_samples) + grower = TreeGrower( + X, + all_gradients, + all_hessians, + n_bins=n_bins, + shrinkage=1.0, + min_samples_leaf=min_samples_leaf, + max_leaf_nodes=n_samples, + ) grower.grow() - predictor = grower.make_predictor( - binning_thresholds=mapper.bin_thresholds_) + predictor = grower.make_predictor(binning_thresholds=mapper.bin_thresholds_) if n_samples >= min_samples_leaf: for node in predictor.nodes: - if node['is_leaf']: - assert node['count'] >= min_samples_leaf + if node["is_leaf"]: + assert node["count"] >= min_samples_leaf else: assert predictor.nodes.shape[0] == 1 - assert predictor.nodes[0]['is_leaf'] - assert predictor.nodes[0]['count'] == n_samples + assert predictor.nodes[0]["is_leaf"] + assert predictor.nodes[0]["count"] == n_samples -@pytest.mark.parametrize('n_samples, min_samples_leaf', [ - (99, 50), - (100, 50)]) +@pytest.mark.parametrize("n_samples, min_samples_leaf", [(99, 50), (100, 50)]) def test_min_samples_leaf_root(n_samples, min_samples_leaf): # Make sure root node isn't split if n_samples is not at least twice # min_samples_leaf @@ -257,10 +274,15 @@ def test_min_samples_leaf_root(n_samples, min_samples_leaf): all_gradients = y.astype(G_H_DTYPE) all_hessians = np.ones(shape=1, dtype=G_H_DTYPE) - grower = TreeGrower(X, all_gradients, all_hessians, - n_bins=n_bins, shrinkage=1., - min_samples_leaf=min_samples_leaf, - max_leaf_nodes=n_samples) + grower = TreeGrower( + X, + all_gradients, + all_hessians, + n_bins=n_bins, + shrinkage=1.0, + min_samples_leaf=min_samples_leaf, + max_leaf_nodes=n_samples, + ) grower.grow() if n_samples >= min_samples_leaf * 2: assert len(grower.finalized_leaves) >= 2 @@ -275,7 +297,7 @@ def assert_is_stump(grower): assert leaf.right_child is None -@pytest.mark.parametrize('max_depth', [1, 2, 3]) +@pytest.mark.parametrize("max_depth", [1, 2, 3]) def test_max_depth(max_depth): # Make sure max_depth parameter works as expected rng = np.random.RandomState(seed=0) @@ -302,33 +324,26 @@ def test_max_depth(max_depth): def test_input_validation(): - X_binned, all_gradients, all_hessians = _make_training_data() X_binned_float = X_binned.astype(np.float32) - with pytest.raises(NotImplementedError, - match="X_binned must be of type uint8"): + with pytest.raises(NotImplementedError, match="X_binned must be of type uint8"): TreeGrower(X_binned_float, all_gradients, all_hessians) X_binned_C_array = np.ascontiguousarray(X_binned) with pytest.raises( - ValueError, - match="X_binned should be passed as Fortran contiguous array"): + ValueError, match="X_binned should be passed as Fortran contiguous array" + ): TreeGrower(X_binned_C_array, all_gradients, all_hessians) def test_init_parameters_validation(): X_binned, all_gradients, all_hessians = _make_training_data() - with pytest.raises(ValueError, - match="min_gain_to_split=-1 must be positive"): - - TreeGrower(X_binned, all_gradients, all_hessians, - min_gain_to_split=-1) + with pytest.raises(ValueError, match="min_gain_to_split=-1 must be positive"): + TreeGrower(X_binned, all_gradients, all_hessians, min_gain_to_split=-1) - with pytest.raises(ValueError, - match="min_hessian_to_split=-1 must be positive"): - TreeGrower(X_binned, all_gradients, all_hessians, - min_hessian_to_split=-1) + with pytest.raises(ValueError, match="min_hessian_to_split=-1 must be positive"): + TreeGrower(X_binned, all_gradients, all_hessians, min_hessian_to_split=-1) def test_missing_value_predict_only(): @@ -344,8 +359,9 @@ def test_missing_value_predict_only(): gradients = rng.normal(size=n_samples).astype(G_H_DTYPE) hessians = np.ones(shape=1, dtype=G_H_DTYPE) - grower = TreeGrower(X_binned, gradients, hessians, min_samples_leaf=5, - has_missing_values=False) + grower = TreeGrower( + X_binned, gradients, hessians, min_samples_leaf=5, has_missing_values=False + ) grower.grow() # We pass undefined binning_thresholds because we won't use predict anyway @@ -356,12 +372,12 @@ def test_missing_value_predict_only(): # go from root to a leaf, always following node with the most samples. # That's the path nans are supposed to take node = predictor.nodes[0] - while not node['is_leaf']: - left = predictor.nodes[node['left']] - right = predictor.nodes[node['right']] - node = left if left['count'] > right['count'] else right + while not node["is_leaf"]: + left = predictor.nodes[node["left"]] + right = predictor.nodes[node["right"]] + node = left if left["count"] > right["count"] else right - prediction_main_path = node['value'] + prediction_main_path = node["value"] # now build X_test with only nans, and make sure all predictions are equal # to prediction_main_path @@ -369,7 +385,7 @@ def test_missing_value_predict_only(): known_cat_bitsets = np.zeros((0, 8), dtype=X_BITSET_INNER_DTYPE) f_idx_map = np.zeros(0, dtype=np.uint32) - y_pred = predictor.predict(all_nans, known_cat_bitsets, f_idx_map) + y_pred = predictor.predict(all_nans, known_cat_bitsets, f_idx_map, n_threads) assert np.all(y_pred == prediction_main_path) @@ -390,29 +406,35 @@ def test_split_on_nan_with_infinite_values(): n_bins_non_missing = 3 has_missing_values = True - grower = TreeGrower(X_binned, gradients, hessians, - n_bins_non_missing=n_bins_non_missing, - has_missing_values=has_missing_values, - min_samples_leaf=1) + grower = TreeGrower( + X_binned, + gradients, + hessians, + n_bins_non_missing=n_bins_non_missing, + has_missing_values=has_missing_values, + min_samples_leaf=1, + n_threads=n_threads, + ) grower.grow() - predictor = grower.make_predictor( - binning_thresholds=bin_mapper.bin_thresholds_ - ) + predictor = grower.make_predictor(binning_thresholds=bin_mapper.bin_thresholds_) # sanity check: this was a split on nan - assert predictor.nodes[0]['num_threshold'] == np.inf - assert predictor.nodes[0]['bin_threshold'] == n_bins_non_missing - 1 + assert predictor.nodes[0]["num_threshold"] == np.inf + assert predictor.nodes[0]["bin_threshold"] == n_bins_non_missing - 1 known_cat_bitsets, f_idx_map = bin_mapper.make_known_categories_bitsets() # Make sure in particular that the +inf sample is mapped to the left child # Note that lightgbm "fails" here and will assign the inf sample to the # right child, even though it's a "split on nan" situation. - predictions = predictor.predict(X, known_cat_bitsets, f_idx_map) + predictions = predictor.predict(X, known_cat_bitsets, f_idx_map, n_threads) predictions_binned = predictor.predict_binned( - X_binned, missing_values_bin_idx=bin_mapper.missing_values_bin_idx_) + X_binned, + missing_values_bin_idx=bin_mapper.missing_values_bin_idx_, + n_threads=n_threads, + ) np.testing.assert_allclose(predictions, -gradients) np.testing.assert_allclose(predictions_binned, -gradients) @@ -427,23 +449,30 @@ def test_grow_tree_categories(): all_hessians = np.ones(1, dtype=G_H_DTYPE) is_categorical = np.ones(1, dtype=np.uint8) - grower = TreeGrower(X_binned, all_gradients, all_hessians, - n_bins=4, shrinkage=1.0, min_samples_leaf=1, - is_categorical=is_categorical) + grower = TreeGrower( + X_binned, + all_gradients, + all_hessians, + n_bins=4, + shrinkage=1.0, + min_samples_leaf=1, + is_categorical=is_categorical, + n_threads=n_threads, + ) grower.grow() assert grower.n_nodes == 3 categories = [np.array([4, 9], dtype=X_DTYPE)] predictor = grower.make_predictor(binning_thresholds=categories) root = predictor.nodes[0] - assert root['count'] == 23 - assert root['depth'] == 0 - assert root['is_categorical'] + assert root["count"] == 23 + assert root["depth"] == 0 + assert root["is_categorical"] - left, right = predictor.nodes[root['left']], predictor.nodes[root['right']] + left, right = predictor.nodes[root["left"]], predictor.nodes[root["right"]] # arbitrary validation, but this means ones go to the left. - assert left['count'] >= right['count'] + assert left["count"] >= right["count"] # check binned category value (1) expected_binned_cat_bitset = [2**1] + [0] * 7 @@ -459,41 +488,44 @@ def test_grow_tree_categories(): # values aren't part of the bitsets. However, we expect the missing values # to go to the biggest child (i.e. the left one). # The left child has a value of -1 = negative gradient. - assert root['missing_go_to_left'] + assert root["missing_go_to_left"] # make sure binned missing values are mapped to the left child during # prediction prediction_binned = predictor.predict_binned( - np.asarray([[6]]).astype(X_BINNED_DTYPE), missing_values_bin_idx=6) + np.asarray([[6]]).astype(X_BINNED_DTYPE), + missing_values_bin_idx=6, + n_threads=n_threads, + ) assert_allclose(prediction_binned, [-1]) # negative gradient # make sure raw missing values are mapped to the left child during # prediction known_cat_bitsets = np.zeros((1, 8), dtype=np.uint32) # ignored anyway f_idx_map = np.array([0], dtype=np.uint32) - prediction = predictor.predict(np.array([[np.nan]]), known_cat_bitsets, - f_idx_map) + prediction = predictor.predict( + np.array([[np.nan]]), known_cat_bitsets, f_idx_map, n_threads + ) assert_allclose(prediction, [-1]) -@pytest.mark.parametrize('min_samples_leaf', (1, 20)) -@pytest.mark.parametrize('n_unique_categories', (2, 10, 100)) -@pytest.mark.parametrize('target', ('binary', 'random', 'equal')) +@pytest.mark.parametrize("min_samples_leaf", (1, 20)) +@pytest.mark.parametrize("n_unique_categories", (2, 10, 100)) +@pytest.mark.parametrize("target", ("binary", "random", "equal")) def test_ohe_equivalence(min_samples_leaf, n_unique_categories, target): # Make sure that native categorical splits are equivalent to using a OHE, # when given enough depth rng = np.random.RandomState(0) n_samples = 10_000 - X_binned = rng.randint(0, n_unique_categories, - size=(n_samples, 1), dtype=np.uint8) + X_binned = rng.randint(0, n_unique_categories, size=(n_samples, 1), dtype=np.uint8) - X_ohe = OneHotEncoder(sparse=False).fit_transform(X_binned) + X_ohe = OneHotEncoder(sparse_output=False).fit_transform(X_binned) X_ohe = np.asfortranarray(X_ohe).astype(np.uint8) - if target == 'equal': + if target == "equal": gradients = X_binned.reshape(-1) - elif target == 'binary': + elif target == "binary": gradients = (X_binned % 2).reshape(-1) else: gradients = rng.randn(n_samples) @@ -502,30 +534,117 @@ def test_ohe_equivalence(min_samples_leaf, n_unique_categories, target): hessians = np.ones(shape=1, dtype=G_H_DTYPE) grower_params = { - 'min_samples_leaf': min_samples_leaf, - 'max_depth': None, - 'max_leaf_nodes': None, + "min_samples_leaf": min_samples_leaf, + "max_depth": None, + "max_leaf_nodes": None, } - grower = TreeGrower(X_binned, gradients, hessians, is_categorical=[True], - **grower_params) + grower = TreeGrower( + X_binned, gradients, hessians, is_categorical=[True], **grower_params + ) grower.grow() # we pass undefined bin_thresholds because we won't use predict() predictor = grower.make_predictor( binning_thresholds=np.zeros((1, n_unique_categories)) ) - preds = predictor.predict_binned(X_binned, missing_values_bin_idx=255) + preds = predictor.predict_binned( + X_binned, missing_values_bin_idx=255, n_threads=n_threads + ) grower_ohe = TreeGrower(X_ohe, gradients, hessians, **grower_params) grower_ohe.grow() predictor_ohe = grower_ohe.make_predictor( binning_thresholds=np.zeros((X_ohe.shape[1], n_unique_categories)) ) - preds_ohe = predictor_ohe.predict_binned(X_ohe, missing_values_bin_idx=255) + preds_ohe = predictor_ohe.predict_binned( + X_ohe, missing_values_bin_idx=255, n_threads=n_threads + ) assert predictor.get_max_depth() <= predictor_ohe.get_max_depth() - if target == 'binary' and n_unique_categories > 2: + if target == "binary" and n_unique_categories > 2: # OHE needs more splits to achieve the same predictions assert predictor.get_max_depth() < predictor_ohe.get_max_depth() np.testing.assert_allclose(preds, preds_ohe) + + +def test_grower_interaction_constraints(): + """Check that grower respects interaction constraints.""" + n_features = 6 + interaction_cst = [{0, 1}, {1, 2}, {3, 4, 5}] + n_samples = 10 + n_bins = 6 + root_feature_splits = [] + + def get_all_children(node): + res = [] + if node.is_leaf: + return res + for n in [node.left_child, node.right_child]: + res.append(n) + res.extend(get_all_children(n)) + return res + + for seed in range(20): + rng = np.random.RandomState(seed) + + X_binned = rng.randint( + 0, n_bins - 1, size=(n_samples, n_features), dtype=X_BINNED_DTYPE + ) + X_binned = np.asfortranarray(X_binned) + gradients = rng.normal(size=n_samples).astype(G_H_DTYPE) + hessians = np.ones(shape=1, dtype=G_H_DTYPE) + + grower = TreeGrower( + X_binned, + gradients, + hessians, + n_bins=n_bins, + min_samples_leaf=1, + interaction_cst=interaction_cst, + n_threads=n_threads, + ) + grower.grow() + + root_feature_idx = grower.root.split_info.feature_idx + root_feature_splits.append(root_feature_idx) + + feature_idx_to_constraint_set = { + 0: {0, 1}, + 1: {0, 1, 2}, + 2: {1, 2}, + 3: {3, 4, 5}, + 4: {3, 4, 5}, + 5: {3, 4, 5}, + } + + root_constraint_set = feature_idx_to_constraint_set[root_feature_idx] + for node in (grower.root.left_child, grower.root.right_child): + # Root's children's allowed_features must be the root's constraints set. + assert_array_equal(node.allowed_features, list(root_constraint_set)) + for node in get_all_children(grower.root): + if node.is_leaf: + continue + # Ensure that each node uses a subset of features of its parent node. + parent_interaction_cst_indices = set(node.interaction_cst_indices) + right_interactions_cst_indices = set( + node.right_child.interaction_cst_indices + ) + left_interactions_cst_indices = set(node.left_child.interaction_cst_indices) + + assert right_interactions_cst_indices.issubset( + parent_interaction_cst_indices + ) + assert left_interactions_cst_indices.issubset( + parent_interaction_cst_indices + ) + # The features used for split must have been present in the root's + # constraint set. + assert node.split_info.feature_idx in root_constraint_set + + # Make sure that every feature is used at least once as split for the root node. + assert ( + len(set(root_feature_splits)) + == len(set().union(*interaction_cst)) + == n_features + ) diff --git a/sklearn/ensemble/_hist_gradient_boosting/tests/test_histogram.py b/sklearn/ensemble/_hist_gradient_boosting/tests/test_histogram.py index c5f10bcf238f6..22375c7d4ea2c 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/tests/test_histogram.py +++ b/sklearn/ensemble/_hist_gradient_boosting/tests/test_histogram.py @@ -1,24 +1,23 @@ import numpy as np import pytest +from numpy.testing import assert_allclose, assert_array_equal -from numpy.testing import assert_allclose -from numpy.testing import assert_array_equal - +from sklearn.ensemble._hist_gradient_boosting.common import ( + G_H_DTYPE, + HISTOGRAM_DTYPE, + X_BINNED_DTYPE, +) from sklearn.ensemble._hist_gradient_boosting.histogram import ( - _build_histogram_naive, _build_histogram, + _build_histogram_naive, _build_histogram_no_hessian, - _build_histogram_root_no_hessian, _build_histogram_root, - _subtract_histograms + _build_histogram_root_no_hessian, + _subtract_histograms, ) -from sklearn.ensemble._hist_gradient_boosting.common import HISTOGRAM_DTYPE -from sklearn.ensemble._hist_gradient_boosting.common import G_H_DTYPE -from sklearn.ensemble._hist_gradient_boosting.common import X_BINNED_DTYPE -@pytest.mark.parametrize( - 'build_func', [_build_histogram_naive, _build_histogram]) +@pytest.mark.parametrize("build_func", [_build_histogram_naive, _build_histogram]) def test_build_histogram(build_func): binned_feature = np.array([0, 2, 0, 1, 2, 0, 2, 1], dtype=X_BINNED_DTYPE) @@ -28,12 +27,13 @@ def test_build_histogram(build_func): sample_indices = np.array([0, 2, 3], dtype=np.uint32) hist = np.zeros((1, 3), dtype=HISTOGRAM_DTYPE) - build_func(0, sample_indices, binned_feature, ordered_gradients, - ordered_hessians, hist) + build_func( + 0, sample_indices, binned_feature, ordered_gradients, ordered_hessians, hist + ) hist = hist[0] - assert_array_equal(hist['count'], [2, 1, 0]) - assert_allclose(hist['sum_gradients'], [1, 3, 0]) - assert_allclose(hist['sum_hessians'], [2, 2, 0]) + assert_array_equal(hist["count"], [2, 1, 0]) + assert_allclose(hist["sum_gradients"], [1, 3, 0]) + assert_allclose(hist["sum_hessians"], [2, 2, 0]) # Larger sample_indices (above unrolling threshold) sample_indices = np.array([0, 2, 3, 6, 7], dtype=np.uint32) @@ -41,12 +41,13 @@ def test_build_histogram(build_func): ordered_hessians = np.array([1, 1, 2, 1, 0], dtype=G_H_DTYPE) hist = np.zeros((1, 3), dtype=HISTOGRAM_DTYPE) - build_func(0, sample_indices, binned_feature, ordered_gradients, - ordered_hessians, hist) + build_func( + 0, sample_indices, binned_feature, ordered_gradients, ordered_hessians, hist + ) hist = hist[0] - assert_array_equal(hist['count'], [2, 2, 1]) - assert_allclose(hist['sum_gradients'], [1, 4, 0]) - assert_allclose(hist['sum_hessians'], [2, 2, 1]) + assert_array_equal(hist["count"], [2, 2, 1]) + assert_allclose(hist["sum_gradients"], [1, 4, 0]) + assert_allclose(hist["sum_hessians"], [2, 2, 1]) def test_histogram_sample_order_independence(): @@ -57,42 +58,53 @@ def test_histogram_sample_order_independence(): n_samples = 1000 n_bins = 256 - binned_feature = rng.randint(0, n_bins - 1, size=n_samples, - dtype=X_BINNED_DTYPE) - sample_indices = rng.choice(np.arange(n_samples, dtype=np.uint32), - n_sub_samples, replace=False) + binned_feature = rng.randint(0, n_bins - 1, size=n_samples, dtype=X_BINNED_DTYPE) + sample_indices = rng.choice( + np.arange(n_samples, dtype=np.uint32), n_sub_samples, replace=False + ) ordered_gradients = rng.randn(n_sub_samples).astype(G_H_DTYPE) hist_gc = np.zeros((1, n_bins), dtype=HISTOGRAM_DTYPE) - _build_histogram_no_hessian(0, sample_indices, binned_feature, - ordered_gradients, hist_gc) + _build_histogram_no_hessian( + 0, sample_indices, binned_feature, ordered_gradients, hist_gc + ) ordered_hessians = rng.exponential(size=n_sub_samples).astype(G_H_DTYPE) hist_ghc = np.zeros((1, n_bins), dtype=HISTOGRAM_DTYPE) - _build_histogram(0, sample_indices, binned_feature, - ordered_gradients, ordered_hessians, hist_ghc) + _build_histogram( + 0, sample_indices, binned_feature, ordered_gradients, ordered_hessians, hist_ghc + ) permutation = rng.permutation(n_sub_samples) hist_gc_perm = np.zeros((1, n_bins), dtype=HISTOGRAM_DTYPE) - _build_histogram_no_hessian(0, sample_indices[permutation], - binned_feature, ordered_gradients[permutation], - hist_gc_perm) + _build_histogram_no_hessian( + 0, + sample_indices[permutation], + binned_feature, + ordered_gradients[permutation], + hist_gc_perm, + ) hist_ghc_perm = np.zeros((1, n_bins), dtype=HISTOGRAM_DTYPE) - _build_histogram(0, sample_indices[permutation], binned_feature, - ordered_gradients[permutation], - ordered_hessians[permutation], hist_ghc_perm) + _build_histogram( + 0, + sample_indices[permutation], + binned_feature, + ordered_gradients[permutation], + ordered_hessians[permutation], + hist_ghc_perm, + ) hist_gc = hist_gc[0] hist_ghc = hist_ghc[0] hist_gc_perm = hist_gc_perm[0] hist_ghc_perm = hist_ghc_perm[0] - assert_allclose(hist_gc['sum_gradients'], hist_gc_perm['sum_gradients']) - assert_array_equal(hist_gc['count'], hist_gc_perm['count']) + assert_allclose(hist_gc["sum_gradients"], hist_gc_perm["sum_gradients"]) + assert_array_equal(hist_gc["count"], hist_gc_perm["count"]) - assert_allclose(hist_ghc['sum_gradients'], hist_ghc_perm['sum_gradients']) - assert_allclose(hist_ghc['sum_hessians'], hist_ghc_perm['sum_hessians']) - assert_array_equal(hist_ghc['count'], hist_ghc_perm['count']) + assert_allclose(hist_ghc["sum_gradients"], hist_ghc_perm["sum_gradients"]) + assert_allclose(hist_ghc["sum_hessians"], hist_ghc_perm["sum_hessians"]) + assert_array_equal(hist_ghc["count"], hist_ghc_perm["count"]) @pytest.mark.parametrize("constant_hessian", [True, False]) @@ -116,16 +128,24 @@ def test_unrolled_equivalent_to_naive(constant_hessian): hist_ghc = np.zeros((1, n_bins), dtype=HISTOGRAM_DTYPE) hist_naive = np.zeros((1, n_bins), dtype=HISTOGRAM_DTYPE) - _build_histogram_root_no_hessian(0, binned_feature, - ordered_gradients, hist_gc_root) - _build_histogram_root(0, binned_feature, ordered_gradients, - ordered_hessians, hist_ghc_root) - _build_histogram_no_hessian(0, sample_indices, binned_feature, - ordered_gradients, hist_gc) - _build_histogram(0, sample_indices, binned_feature, - ordered_gradients, ordered_hessians, hist_ghc) - _build_histogram_naive(0, sample_indices, binned_feature, - ordered_gradients, ordered_hessians, hist_naive) + _build_histogram_root_no_hessian(0, binned_feature, ordered_gradients, hist_gc_root) + _build_histogram_root( + 0, binned_feature, ordered_gradients, ordered_hessians, hist_ghc_root + ) + _build_histogram_no_hessian( + 0, sample_indices, binned_feature, ordered_gradients, hist_gc + ) + _build_histogram( + 0, sample_indices, binned_feature, ordered_gradients, ordered_hessians, hist_ghc + ) + _build_histogram_naive( + 0, + sample_indices, + binned_feature, + ordered_gradients, + ordered_hessians, + hist_naive, + ) hist_naive = hist_naive[0] hist_gc_root = hist_gc_root[0] @@ -133,12 +153,12 @@ def test_unrolled_equivalent_to_naive(constant_hessian): hist_gc = hist_gc[0] hist_ghc = hist_ghc[0] for hist in (hist_gc_root, hist_ghc_root, hist_gc, hist_ghc): - assert_array_equal(hist['count'], hist_naive['count']) - assert_allclose(hist['sum_gradients'], hist_naive['sum_gradients']) + assert_array_equal(hist["count"], hist_naive["count"]) + assert_allclose(hist["sum_gradients"], hist_naive["sum_gradients"]) for hist in (hist_ghc_root, hist_ghc): - assert_allclose(hist['sum_hessians'], hist_naive['sum_hessians']) + assert_allclose(hist["sum_hessians"], hist_naive["sum_hessians"]) for hist in (hist_gc_root, hist_gc): - assert_array_equal(hist['sum_hessians'], np.zeros(n_bins)) + assert_array_equal(hist["sum_hessians"], np.zeros(n_bins)) @pytest.mark.parametrize("constant_hessian", [True, False]) @@ -158,11 +178,18 @@ def test_hist_subtraction(constant_hessian): hist_parent = np.zeros((1, n_bins), dtype=HISTOGRAM_DTYPE) if constant_hessian: - _build_histogram_no_hessian(0, sample_indices, binned_feature, - ordered_gradients, hist_parent) + _build_histogram_no_hessian( + 0, sample_indices, binned_feature, ordered_gradients, hist_parent + ) else: - _build_histogram(0, sample_indices, binned_feature, - ordered_gradients, ordered_hessians, hist_parent) + _build_histogram( + 0, + sample_indices, + binned_feature, + ordered_gradients, + ordered_hessians, + hist_parent, + ) mask = rng.randint(0, 2, n_samples).astype(bool) @@ -171,32 +198,42 @@ def test_hist_subtraction(constant_hessian): ordered_hessians_left = ordered_hessians[mask] hist_left = np.zeros((1, n_bins), dtype=HISTOGRAM_DTYPE) if constant_hessian: - _build_histogram_no_hessian(0, sample_indices_left, - binned_feature, ordered_gradients_left, - hist_left) + _build_histogram_no_hessian( + 0, sample_indices_left, binned_feature, ordered_gradients_left, hist_left + ) else: - _build_histogram(0, sample_indices_left, binned_feature, - ordered_gradients_left, ordered_hessians_left, - hist_left) + _build_histogram( + 0, + sample_indices_left, + binned_feature, + ordered_gradients_left, + ordered_hessians_left, + hist_left, + ) sample_indices_right = sample_indices[~mask] ordered_gradients_right = ordered_gradients[~mask] ordered_hessians_right = ordered_hessians[~mask] hist_right = np.zeros((1, n_bins), dtype=HISTOGRAM_DTYPE) if constant_hessian: - _build_histogram_no_hessian(0, sample_indices_right, - binned_feature, ordered_gradients_right, - hist_right) + _build_histogram_no_hessian( + 0, sample_indices_right, binned_feature, ordered_gradients_right, hist_right + ) else: - _build_histogram(0, sample_indices_right, binned_feature, - ordered_gradients_right, ordered_hessians_right, - hist_right) - - hist_left_sub = np.zeros((1, n_bins), dtype=HISTOGRAM_DTYPE) - hist_right_sub = np.zeros((1, n_bins), dtype=HISTOGRAM_DTYPE) - _subtract_histograms(0, n_bins, hist_parent, hist_right, hist_left_sub) - _subtract_histograms(0, n_bins, hist_parent, hist_left, hist_right_sub) - - for key in ('count', 'sum_hessians', 'sum_gradients'): + _build_histogram( + 0, + sample_indices_right, + binned_feature, + ordered_gradients_right, + ordered_hessians_right, + hist_right, + ) + + hist_left_sub = np.copy(hist_parent) + hist_right_sub = np.copy(hist_parent) + _subtract_histograms(0, n_bins, hist_left_sub, hist_right) + _subtract_histograms(0, n_bins, hist_right_sub, hist_left) + + for key in ("count", "sum_hessians", "sum_gradients"): assert_allclose(hist_left[key], hist_left_sub[key], rtol=1e-6) assert_allclose(hist_right[key], hist_right_sub[key], rtol=1e-6) diff --git a/sklearn/ensemble/_hist_gradient_boosting/tests/test_loss.py b/sklearn/ensemble/_hist_gradient_boosting/tests/test_loss.py deleted file mode 100644 index 221b94183a7ff..0000000000000 --- a/sklearn/ensemble/_hist_gradient_boosting/tests/test_loss.py +++ /dev/null @@ -1,333 +0,0 @@ -import numpy as np -from numpy.testing import assert_almost_equal -from numpy.testing import assert_allclose -from scipy.optimize import newton -from scipy.special import logit -from sklearn.utils import assert_all_finite -from sklearn.utils.fixes import sp_version, parse_version -import pytest - -from sklearn.ensemble._hist_gradient_boosting.loss import _LOSSES -from sklearn.ensemble._hist_gradient_boosting.common import Y_DTYPE -from sklearn.ensemble._hist_gradient_boosting.common import G_H_DTYPE -from sklearn.utils._testing import skip_if_32bit - - -def get_derivatives_helper(loss): - """Return get_gradients() and get_hessians() functions for a given loss. - """ - - def get_gradients(y_true, raw_predictions): - # create gradients and hessians array, update inplace, and return - gradients = np.empty_like(raw_predictions, dtype=G_H_DTYPE) - hessians = np.empty_like(raw_predictions, dtype=G_H_DTYPE) - loss.update_gradients_and_hessians(gradients, hessians, y_true, - raw_predictions, None) - return gradients - - def get_hessians(y_true, raw_predictions): - # create gradients and hessians array, update inplace, and return - gradients = np.empty_like(raw_predictions, dtype=G_H_DTYPE) - hessians = np.empty_like(raw_predictions, dtype=G_H_DTYPE) - loss.update_gradients_and_hessians(gradients, hessians, y_true, - raw_predictions, None) - - if loss.__class__.__name__ == 'LeastSquares': - # hessians aren't updated because they're constant: - # the value is 1 (and not 2) because the loss is actually an half - # least squares loss. - hessians = np.full_like(raw_predictions, fill_value=1) - elif loss.__class__.__name__ == 'LeastAbsoluteDeviation': - # hessians aren't updated because they're constant - hessians = np.full_like(raw_predictions, fill_value=0) - - return hessians - - return get_gradients, get_hessians - - -@pytest.mark.parametrize('loss, x0, y_true', [ - ('least_squares', -2., 42), - ('least_squares', 117., 1.05), - ('least_squares', 0., 0.), - # The argmin of binary_crossentropy for y_true=0 and y_true=1 is resp. -inf - # and +inf due to logit, cf. "complete separation". Therefore, we use - # 0 < y_true < 1. - ('binary_crossentropy', 0.3, 0.1), - ('binary_crossentropy', -12, 0.2), - ('binary_crossentropy', 30, 0.9), - ('poisson', 12., 1.), - ('poisson', 0., 2.), - ('poisson', -22., 10.), -]) -@pytest.mark.skipif(sp_version == parse_version('1.2.0'), - reason='bug in scipy 1.2.0, see scipy issue #9608') -@skip_if_32bit -def test_derivatives(loss, x0, y_true): - # Check that gradients are zero when the loss is minimized on a single - # value/sample using Halley's method with the first and second order - # derivatives computed by the Loss instance. - # Note that methods of Loss instances operate on arrays while the newton - # root finder expects a scalar or a one-element array for this purpose. - - loss = _LOSSES[loss](sample_weight=None) - y_true = np.array([y_true], dtype=Y_DTYPE) - x0 = np.array([x0], dtype=Y_DTYPE).reshape(1, 1) - get_gradients, get_hessians = get_derivatives_helper(loss) - - def func(x: np.ndarray) -> np.ndarray: - if isinstance(loss, _LOSSES['binary_crossentropy']): - # Subtract a constant term such that the binary cross entropy - # has its minimum at zero, which is needed for the newton method. - actual_min = loss.pointwise_loss(y_true, logit(y_true)) - return loss.pointwise_loss(y_true, x) - actual_min - else: - return loss.pointwise_loss(y_true, x) - - def fprime(x: np.ndarray) -> np.ndarray: - return get_gradients(y_true, x) - - def fprime2(x: np.ndarray) -> np.ndarray: - return get_hessians(y_true, x) - - optimum = newton(func, x0=x0, fprime=fprime, fprime2=fprime2, - maxiter=70, tol=2e-8) - - # Need to ravel arrays because assert_allclose requires matching dimensions - y_true = y_true.ravel() - optimum = optimum.ravel() - assert_allclose(loss.inverse_link_function(optimum), y_true) - assert_allclose(func(optimum), 0, atol=1e-14) - assert_allclose(get_gradients(y_true, optimum), 0, atol=1e-7) - - -@pytest.mark.parametrize('loss, n_classes, prediction_dim', [ - ('least_squares', 0, 1), - ('least_absolute_deviation', 0, 1), - ('binary_crossentropy', 2, 1), - ('categorical_crossentropy', 3, 3), - ('poisson', 0, 1), -]) -@pytest.mark.skipif(Y_DTYPE != np.float64, - reason='Need 64 bits float precision for numerical checks') -def test_numerical_gradients(loss, n_classes, prediction_dim, seed=0): - # Make sure gradients and hessians computed in the loss are correct, by - # comparing with their approximations computed with finite central - # differences. - # See https://en.wikipedia.org/wiki/Finite_difference. - - rng = np.random.RandomState(seed) - n_samples = 100 - if loss in ('least_squares', 'least_absolute_deviation'): - y_true = rng.normal(size=n_samples).astype(Y_DTYPE) - elif loss in ('poisson'): - y_true = rng.poisson(size=n_samples).astype(Y_DTYPE) - else: - y_true = rng.randint(0, n_classes, size=n_samples).astype(Y_DTYPE) - raw_predictions = rng.normal( - size=(prediction_dim, n_samples) - ).astype(Y_DTYPE) - loss = _LOSSES[loss](sample_weight=None) - get_gradients, get_hessians = get_derivatives_helper(loss) - - # only take gradients and hessians of first tree / class. - gradients = get_gradients(y_true, raw_predictions)[0, :].ravel() - hessians = get_hessians(y_true, raw_predictions)[0, :].ravel() - - # Approximate gradients - # For multiclass loss, we should only change the predictions of one tree - # (here the first), hence the use of offset[0, :] += eps - # As a softmax is computed, offsetting the whole array by a constant would - # have no effect on the probabilities, and thus on the loss - eps = 1e-9 - offset = np.zeros_like(raw_predictions) - offset[0, :] = eps - f_plus_eps = loss.pointwise_loss(y_true, raw_predictions + offset / 2) - f_minus_eps = loss.pointwise_loss(y_true, raw_predictions - offset / 2) - numerical_gradients = (f_plus_eps - f_minus_eps) / eps - - # Approximate hessians - eps = 1e-4 # need big enough eps as we divide by its square - offset[0, :] = eps - f_plus_eps = loss.pointwise_loss(y_true, raw_predictions + offset) - f_minus_eps = loss.pointwise_loss(y_true, raw_predictions - offset) - f = loss.pointwise_loss(y_true, raw_predictions) - numerical_hessians = (f_plus_eps + f_minus_eps - 2 * f) / eps**2 - - assert_allclose(numerical_gradients, gradients, rtol=1e-4, atol=1e-7) - assert_allclose(numerical_hessians, hessians, rtol=1e-4, atol=1e-7) - - -def test_baseline_least_squares(): - rng = np.random.RandomState(0) - - loss = _LOSSES['least_squares'](sample_weight=None) - y_train = rng.normal(size=100) - baseline_prediction = loss.get_baseline_prediction(y_train, None, 1) - assert baseline_prediction.shape == tuple() # scalar - assert baseline_prediction.dtype == y_train.dtype - # Make sure baseline prediction is the mean of all targets - assert_almost_equal(baseline_prediction, y_train.mean()) - assert np.allclose(loss.inverse_link_function(baseline_prediction), - baseline_prediction) - - -def test_baseline_least_absolute_deviation(): - rng = np.random.RandomState(0) - - loss = _LOSSES['least_absolute_deviation'](sample_weight=None) - y_train = rng.normal(size=100) - baseline_prediction = loss.get_baseline_prediction(y_train, None, 1) - assert baseline_prediction.shape == tuple() # scalar - assert baseline_prediction.dtype == y_train.dtype - # Make sure baseline prediction is the median of all targets - assert np.allclose(loss.inverse_link_function(baseline_prediction), - baseline_prediction) - assert baseline_prediction == pytest.approx(np.median(y_train)) - - -def test_baseline_poisson(): - rng = np.random.RandomState(0) - - loss = _LOSSES['poisson'](sample_weight=None) - y_train = rng.poisson(size=100).astype(np.float64) - # Sanity check, make sure at least one sample is non-zero so we don't take - # log(0) - assert y_train.sum() > 0 - baseline_prediction = loss.get_baseline_prediction(y_train, None, 1) - assert np.isscalar(baseline_prediction) - assert baseline_prediction.dtype == y_train.dtype - assert_all_finite(baseline_prediction) - # Make sure baseline prediction produces the log of the mean of all targets - assert_almost_equal(np.log(y_train.mean()), baseline_prediction) - - # Test baseline for y_true = 0 - y_train.fill(0.) - baseline_prediction = loss.get_baseline_prediction(y_train, None, 1) - assert_all_finite(baseline_prediction) - - -def test_baseline_binary_crossentropy(): - rng = np.random.RandomState(0) - - loss = _LOSSES['binary_crossentropy'](sample_weight=None) - for y_train in (np.zeros(shape=100), np.ones(shape=100)): - y_train = y_train.astype(np.float64) - baseline_prediction = loss.get_baseline_prediction(y_train, None, 1) - assert_all_finite(baseline_prediction) - assert np.allclose(loss.inverse_link_function(baseline_prediction), - y_train[0]) - - # Make sure baseline prediction is equal to link_function(p), where p - # is the proba of the positive class. We want predict_proba() to return p, - # and by definition - # p = inverse_link_function(raw_prediction) = sigmoid(raw_prediction) - # So we want raw_prediction = link_function(p) = log(p / (1 - p)) - y_train = rng.randint(0, 2, size=100).astype(np.float64) - baseline_prediction = loss.get_baseline_prediction(y_train, None, 1) - assert baseline_prediction.shape == tuple() # scalar - assert baseline_prediction.dtype == y_train.dtype - p = y_train.mean() - assert np.allclose(baseline_prediction, np.log(p / (1 - p))) - - -def test_baseline_categorical_crossentropy(): - rng = np.random.RandomState(0) - - prediction_dim = 4 - loss = _LOSSES['categorical_crossentropy'](sample_weight=None) - for y_train in (np.zeros(shape=100), np.ones(shape=100)): - y_train = y_train.astype(np.float64) - baseline_prediction = loss.get_baseline_prediction(y_train, None, - prediction_dim) - assert baseline_prediction.dtype == y_train.dtype - assert_all_finite(baseline_prediction) - - # Same logic as for above test. Here inverse_link_function = softmax and - # link_function = log - y_train = rng.randint(0, prediction_dim + 1, size=100).astype(np.float32) - baseline_prediction = loss.get_baseline_prediction(y_train, None, - prediction_dim) - assert baseline_prediction.shape == (prediction_dim, 1) - for k in range(prediction_dim): - p = (y_train == k).mean() - assert np.allclose(baseline_prediction[k, :], np.log(p)) - - -@pytest.mark.parametrize('loss, problem', [ - ('least_squares', 'regression'), - ('least_absolute_deviation', 'regression'), - ('binary_crossentropy', 'classification'), - ('categorical_crossentropy', 'classification'), - ('poisson', 'poisson_regression'), - ]) -@pytest.mark.parametrize('sample_weight', ['ones', 'random']) -def test_sample_weight_multiplies_gradients(loss, problem, sample_weight): - # Make sure that passing sample weights to the gradient and hessians - # computation methods is equivalent to multiplying by the weights. - - rng = np.random.RandomState(42) - n_samples = 1000 - - if loss == 'categorical_crossentropy': - n_classes = prediction_dim = 3 - else: - n_classes = prediction_dim = 1 - - if problem == 'regression': - y_true = rng.normal(size=n_samples).astype(Y_DTYPE) - elif problem == 'poisson_regression': - y_true = rng.poisson(size=n_samples).astype(Y_DTYPE) - else: - y_true = rng.randint(0, n_classes, size=n_samples).astype(Y_DTYPE) - - if sample_weight == 'ones': - sample_weight = np.ones(shape=n_samples, dtype=Y_DTYPE) - else: - sample_weight = rng.normal(size=n_samples).astype(Y_DTYPE) - - loss_ = _LOSSES[loss](sample_weight=sample_weight) - - baseline_prediction = loss_.get_baseline_prediction( - y_true, None, prediction_dim - ) - raw_predictions = np.zeros(shape=(prediction_dim, n_samples), - dtype=baseline_prediction.dtype) - raw_predictions += baseline_prediction - - gradients = np.empty(shape=(prediction_dim, n_samples), dtype=G_H_DTYPE) - hessians = np.ones(shape=(prediction_dim, n_samples), dtype=G_H_DTYPE) - loss_.update_gradients_and_hessians(gradients, hessians, y_true, - raw_predictions, None) - - gradients_sw = np.empty(shape=(prediction_dim, n_samples), dtype=G_H_DTYPE) - hessians_sw = np.ones(shape=(prediction_dim, n_samples), dtype=G_H_DTYPE) - loss_.update_gradients_and_hessians(gradients_sw, hessians_sw, y_true, - raw_predictions, sample_weight) - - assert np.allclose(gradients * sample_weight, gradients_sw) - assert np.allclose(hessians * sample_weight, hessians_sw) - - -def test_init_gradient_and_hessians_sample_weight(): - # Make sure that passing sample_weight to a loss correctly influences the - # hessians_are_constant attribute, and consequently the shape of the - # hessians array. - - prediction_dim = 2 - n_samples = 5 - sample_weight = None - loss = _LOSSES['least_squares'](sample_weight=sample_weight) - _, hessians = loss.init_gradients_and_hessians( - n_samples=n_samples, prediction_dim=prediction_dim, - sample_weight=None) - assert loss.hessians_are_constant - assert hessians.shape == (1, 1) - - sample_weight = np.ones(n_samples) - loss = _LOSSES['least_squares'](sample_weight=sample_weight) - _, hessians = loss.init_gradients_and_hessians( - n_samples=n_samples, prediction_dim=prediction_dim, - sample_weight=sample_weight) - assert not loss.hessians_are_constant - assert hessians.shape == (prediction_dim, n_samples) diff --git a/sklearn/ensemble/_hist_gradient_boosting/tests/test_monotonic_contraints.py b/sklearn/ensemble/_hist_gradient_boosting/tests/test_monotonic_constraints.py similarity index 57% rename from sklearn/ensemble/_hist_gradient_boosting/tests/test_monotonic_contraints.py rename to sklearn/ensemble/_hist_gradient_boosting/tests/test_monotonic_constraints.py index 29fc95d4bb070..56b6068d794e8 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/tests/test_monotonic_contraints.py +++ b/sklearn/ensemble/_hist_gradient_boosting/tests/test_monotonic_constraints.py @@ -1,18 +1,27 @@ +import re + import numpy as np import pytest +from sklearn.ensemble import ( + HistGradientBoostingClassifier, + HistGradientBoostingRegressor, +) +from sklearn.ensemble._hist_gradient_boosting.common import ( + G_H_DTYPE, + X_BINNED_DTYPE, + MonotonicConstraint, +) from sklearn.ensemble._hist_gradient_boosting.grower import TreeGrower -from sklearn.ensemble._hist_gradient_boosting.common import G_H_DTYPE -from sklearn.ensemble._hist_gradient_boosting.common import X_BINNED_DTYPE -from sklearn.ensemble._hist_gradient_boosting.common import MonotonicConstraint +from sklearn.ensemble._hist_gradient_boosting.histogram import HistogramBuilder from sklearn.ensemble._hist_gradient_boosting.splitting import ( Splitter, - compute_node_value + compute_node_value, ) -from sklearn.ensemble._hist_gradient_boosting.histogram import HistogramBuilder -from sklearn.experimental import enable_hist_gradient_boosting # noqa -from sklearn.ensemble import HistGradientBoostingRegressor -from sklearn.ensemble import HistGradientBoostingClassifier +from sklearn.utils._openmp_helpers import _openmp_effective_n_threads +from sklearn.utils._testing import _convert_container + +n_threads = _openmp_effective_n_threads() def is_increasing(a): @@ -34,11 +43,11 @@ def get_leaves_values(): def depth_first_collect_leaf_values(node_idx): node = nodes[node_idx] - if node['is_leaf']: - values.append(node['value']) + if node["is_leaf"]: + values.append(node["value"]) return - depth_first_collect_leaf_values(node['left']) - depth_first_collect_leaf_values(node['right']) + depth_first_collect_leaf_values(node["left"]) + depth_first_collect_leaf_values(node["right"]) depth_first_collect_leaf_values(0) # start at root (0) return values @@ -69,15 +78,15 @@ def assert_children_values_monotonic(predictor, monotonic_cst): left_lower = [] left_greater = [] for node in nodes: - if node['is_leaf']: + if node["is_leaf"]: continue - left_idx = node['left'] - right_idx = node['right'] + left_idx = node["left"] + right_idx = node["right"] - if nodes[left_idx]['value'] < nodes[right_idx]['value']: + if nodes[left_idx]["value"] < nodes[right_idx]["value"]: left_lower.append(node) - elif nodes[left_idx]['value'] > nodes[right_idx]['value']: + elif nodes[left_idx]["value"] > nodes[right_idx]["value"]: left_greater.append(node) if monotonic_cst == MonotonicConstraint.NO_CST: @@ -106,35 +115,39 @@ def recursively_check_children_node_values(node, right_sibling=None): if right_sibling is not None: middle = (node.value + right_sibling.value) / 2 if monotonic_cst == MonotonicConstraint.POS: - assert (node.left_child.value <= - node.right_child.value <= - middle) + assert node.left_child.value <= node.right_child.value <= middle if not right_sibling.is_leaf: - assert (middle <= - right_sibling.left_child.value <= - right_sibling.right_child.value) + assert ( + middle + <= right_sibling.left_child.value + <= right_sibling.right_child.value + ) else: # NEG - assert (node.left_child.value >= - node.right_child.value >= - middle) + assert node.left_child.value >= node.right_child.value >= middle if not right_sibling.is_leaf: - assert (middle >= - right_sibling.left_child.value >= - right_sibling.right_child.value) - - recursively_check_children_node_values(node.left_child, - right_sibling=node.right_child) + assert ( + middle + >= right_sibling.left_child.value + >= right_sibling.right_child.value + ) + + recursively_check_children_node_values( + node.left_child, right_sibling=node.right_child + ) recursively_check_children_node_values(node.right_child) recursively_check_children_node_values(grower.root) -@pytest.mark.parametrize('seed', range(3)) -@pytest.mark.parametrize('monotonic_cst', ( - MonotonicConstraint.NO_CST, - MonotonicConstraint.POS, - MonotonicConstraint.NEG, -)) +@pytest.mark.parametrize("seed", range(3)) +@pytest.mark.parametrize( + "monotonic_cst", + ( + MonotonicConstraint.NO_CST, + MonotonicConstraint.POS, + MonotonicConstraint.NEG, + ), +) def test_nodes_values(monotonic_cst, seed): # Build a single tree with only one feature, and make sure the nodes # values respect the monotonic constraints. @@ -157,16 +170,15 @@ def test_nodes_values(monotonic_cst, seed): rng = np.random.RandomState(seed) n_samples = 1000 n_features = 1 - X_binned = rng.randint(0, 255, size=(n_samples, n_features), - dtype=np.uint8) + X_binned = rng.randint(0, 255, size=(n_samples, n_features), dtype=np.uint8) X_binned = np.asfortranarray(X_binned) gradients = rng.normal(size=n_samples).astype(G_H_DTYPE) hessians = np.ones(shape=1, dtype=G_H_DTYPE) - grower = TreeGrower(X_binned, gradients, hessians, - monotonic_cst=[monotonic_cst], - shrinkage=.1) + grower = TreeGrower( + X_binned, gradients, hessians, monotonic_cst=[monotonic_cst], shrinkage=0.1 + ) grower.grow() # grow() will shrink the leaves values at the very end. For our comparison @@ -192,31 +204,49 @@ def test_nodes_values(monotonic_cst, seed): assert_leaves_values_monotonic(predictor, monotonic_cst) -@pytest.mark.parametrize('seed', range(3)) -def test_predictions(seed): - # Train a model with a POS constraint on the first feature and a NEG - # constraint on the second feature, and make sure the constraints are - # respected by checking the predictions. +@pytest.mark.parametrize("use_feature_names", (True, False)) +def test_predictions(global_random_seed, use_feature_names): + # Train a model with a POS constraint on the first non-categorical feature + # and a NEG constraint on the second non-categorical feature, and make sure + # the constraints are respected by checking the predictions. # test adapted from lightgbm's test_monotone_constraint(), itself inspired # by https://xgboost.readthedocs.io/en/latest/tutorials/monotonic.html - rng = np.random.RandomState(seed) + rng = np.random.RandomState(global_random_seed) n_samples = 1000 f_0 = rng.rand(n_samples) # positive correlation with y - f_1 = rng.rand(n_samples) # negative correslation with y - X = np.c_[f_0, f_1] + f_1 = rng.rand(n_samples) # negative correlation with y + + # extra categorical features, no correlation with y, + # to check the correctness of monotonicity constraint remapping, see issue #28898 + f_a = rng.randint(low=0, high=9, size=n_samples) + f_b = rng.randint(low=0, high=9, size=n_samples) + f_c = rng.randint(low=0, high=9, size=n_samples) + + X = np.c_[f_a, f_0, f_b, f_1, f_c] + columns_name = ["f_a", "f_0", "f_b", "f_1", "f_c"] + constructor_name = "dataframe" if use_feature_names else "array" + X = _convert_container(X, constructor_name, columns_name=columns_name) + noise = rng.normal(loc=0.0, scale=0.01, size=n_samples) - y = (5 * f_0 + np.sin(10 * np.pi * f_0) - - 5 * f_1 - np.cos(10 * np.pi * f_1) + - noise) + y = 5 * f_0 + np.sin(10 * np.pi * f_0) - 5 * f_1 - np.cos(10 * np.pi * f_1) + noise + + if use_feature_names: + monotonic_cst = {"f_0": +1, "f_1": -1} + categorical_features = ["f_a", "f_b", "f_c"] + else: + monotonic_cst = [0, +1, 0, -1, 0] + categorical_features = [0, 2, 4] - gbdt = HistGradientBoostingRegressor(monotonic_cst=[1, -1]) + gbdt = HistGradientBoostingRegressor( + monotonic_cst=monotonic_cst, categorical_features=categorical_features + ) gbdt.fit(X, y) linspace = np.linspace(0, 1, 100) sin = np.sin(linspace) - constant = np.full_like(linspace, fill_value=.5) + constant = np.full_like(linspace, fill_value=0.5) # We now assert the predictions properly respect the constraints, on each # feature. When testing for a feature we need to set the other one to a @@ -228,23 +258,27 @@ def test_predictions(seed): # The constraint does not guanrantee that # x0 < x0' => f(x0, x1) < f(x0', x1') - # First feature (POS) + # First non-categorical feature (POS) # assert pred is all increasing when f_0 is all increasing - X = np.c_[linspace, constant] + X = np.c_[constant, linspace, constant, constant, constant] + X = _convert_container(X, constructor_name, columns_name=columns_name) pred = gbdt.predict(X) assert is_increasing(pred) # assert pred actually follows the variations of f_0 - X = np.c_[sin, constant] + X = np.c_[constant, sin, constant, constant, constant] + X = _convert_container(X, constructor_name, columns_name=columns_name) pred = gbdt.predict(X) assert np.all((np.diff(pred) >= 0) == (np.diff(sin) >= 0)) - # Second feature (NEG) + # Second non-categorical feature (NEG) # assert pred is all decreasing when f_1 is all increasing - X = np.c_[constant, linspace] + X = np.c_[constant, constant, constant, linspace, constant] + X = _convert_container(X, constructor_name, columns_name=columns_name) pred = gbdt.predict(X) assert is_decreasing(pred) # assert pred actually follows the inverse variations of f_1 - X = np.c_[constant, sin] + X = np.c_[constant, constant, constant, sin, constant] + X = _convert_container(X, constructor_name, columns_name=columns_name) pred = gbdt.predict(X) assert ((np.diff(pred) <= 0) == (np.diff(sin) >= 0)).all() @@ -254,23 +288,62 @@ def test_input_error(): y = [0, 1, 2] gbdt = HistGradientBoostingRegressor(monotonic_cst=[1, 0, -1]) - with pytest.raises(ValueError, - match='monotonic_cst has shape 3 but the input data'): + with pytest.raises( + ValueError, match=re.escape("monotonic_cst has shape (3,) but the input data") + ): gbdt.fit(X, y) - for monotonic_cst in ([1, 3], [1, -3]): + for monotonic_cst in ([1, 3], [1, -3], [0.3, -0.7]): gbdt = HistGradientBoostingRegressor(monotonic_cst=monotonic_cst) - with pytest.raises(ValueError, - match='must be None or an array-like of ' - '-1, 0 or 1'): + expected_msg = re.escape( + "must be an array-like of -1, 0 or 1. Observed values:" + ) + with pytest.raises(ValueError, match=expected_msg): gbdt.fit(X, y) gbdt = HistGradientBoostingClassifier(monotonic_cst=[0, 1]) with pytest.raises( - ValueError, - match='monotonic constraints are not supported ' - 'for multiclass classification' - ): + ValueError, + match="monotonic constraints are not supported for multiclass classification", + ): + gbdt.fit(X, y) + + +def test_input_error_related_to_feature_names(): + pd = pytest.importorskip("pandas") + X = pd.DataFrame({"a": [0, 1, 2], "b": [0, 1, 2]}) + y = np.array([0, 1, 0]) + + monotonic_cst = {"d": 1, "a": 1, "c": -1} + gbdt = HistGradientBoostingRegressor(monotonic_cst=monotonic_cst) + expected_msg = re.escape( + "monotonic_cst contains 2 unexpected feature names: ['c', 'd']." + ) + with pytest.raises(ValueError, match=expected_msg): + gbdt.fit(X, y) + + monotonic_cst = {k: 1 for k in "abcdefghijklmnopqrstuvwxyz"} + gbdt = HistGradientBoostingRegressor(monotonic_cst=monotonic_cst) + expected_msg = re.escape( + "monotonic_cst contains 24 unexpected feature names: " + "['c', 'd', 'e', 'f', 'g', '...']." + ) + with pytest.raises(ValueError, match=expected_msg): + gbdt.fit(X, y) + + monotonic_cst = {"a": 1} + gbdt = HistGradientBoostingRegressor(monotonic_cst=monotonic_cst) + expected_msg = re.escape( + "HistGradientBoostingRegressor was not fitted on data with feature " + "names. Pass monotonic_cst as an integer array instead." + ) + with pytest.raises(ValueError, match=expected_msg): + gbdt.fit(X.values, y) + + monotonic_cst = {"b": -1, "a": "+"} + gbdt = HistGradientBoostingRegressor(monotonic_cst=monotonic_cst) + expected_msg = re.escape("monotonic_cst['a'] must be either -1, 0 or 1. Got '+'.") + with pytest.raises(ValueError, match=expected_msg): gbdt.fit(X, y) @@ -294,24 +367,32 @@ def test_bounded_value_min_gain_to_split(): sum_hessians = all_hessians.sum() hessians_are_constant = False - builder = HistogramBuilder(X_binned, n_bins, all_gradients, - all_hessians, hessians_are_constant) - n_bins_non_missing = np.array([n_bins - 1] * X_binned.shape[1], - dtype=np.uint32) + builder = HistogramBuilder( + X_binned, n_bins, all_gradients, all_hessians, hessians_are_constant, n_threads + ) + n_bins_non_missing = np.array([n_bins - 1] * X_binned.shape[1], dtype=np.uint32) has_missing_values = np.array([False] * X_binned.shape[1], dtype=np.uint8) monotonic_cst = np.array( - [MonotonicConstraint.NO_CST] * X_binned.shape[1], - dtype=np.int8) + [MonotonicConstraint.NO_CST] * X_binned.shape[1], dtype=np.int8 + ) is_categorical = np.zeros_like(monotonic_cst, dtype=np.uint8) missing_values_bin_idx = n_bins - 1 children_lower_bound, children_upper_bound = -np.inf, np.inf min_gain_to_split = 2000 - splitter = Splitter(X_binned, n_bins_non_missing, missing_values_bin_idx, - has_missing_values, is_categorical, monotonic_cst, - l2_regularization, min_hessian_to_split, - min_samples_leaf, min_gain_to_split, - hessians_are_constant) + splitter = Splitter( + X_binned, + n_bins_non_missing, + missing_values_bin_idx, + has_missing_values, + is_categorical, + monotonic_cst, + l2_regularization, + min_hessian_to_split, + min_samples_leaf, + min_gain_to_split, + hessians_are_constant, + ) histograms = builder.compute_histograms_brute(sample_indices) @@ -320,15 +401,24 @@ def test_bounded_value_min_gain_to_split(): # and is equal to about 1307, which less than min_gain_to_split = 2000, so # the node is considered unsplittable (gain = -1) current_lower_bound, current_upper_bound = -np.inf, np.inf - value = compute_node_value(sum_gradients, sum_hessians, - current_lower_bound, current_upper_bound, - l2_regularization) + value = compute_node_value( + sum_gradients, + sum_hessians, + current_lower_bound, + current_upper_bound, + l2_regularization, + ) # the unbounded value is equal to -sum_gradients / sum_hessians assert value == pytest.approx(-104 / 5) - split_info = splitter.find_node_split(n_samples, histograms, - sum_gradients, sum_hessians, value, - lower_bound=children_lower_bound, - upper_bound=children_upper_bound) + split_info = splitter.find_node_split( + n_samples, + histograms, + sum_gradients, + sum_hessians, + value, + lower_bound=children_lower_bound, + upper_bound=children_upper_bound, + ) assert split_info.gain == -1 # min_gain_to_split not respected # here again the max possible gain is on the 3rd bin but we now cap the @@ -336,12 +426,21 @@ def test_bounded_value_min_gain_to_split(): # This means the gain is now about 2430 which is more than the # min_gain_to_split constraint. current_lower_bound, current_upper_bound = -10, np.inf - value = compute_node_value(sum_gradients, sum_hessians, - current_lower_bound, current_upper_bound, - l2_regularization) + value = compute_node_value( + sum_gradients, + sum_hessians, + current_lower_bound, + current_upper_bound, + l2_regularization, + ) assert value == -10 - split_info = splitter.find_node_split(n_samples, histograms, - sum_gradients, sum_hessians, value, - lower_bound=children_lower_bound, - upper_bound=children_upper_bound) + split_info = splitter.find_node_split( + n_samples, + histograms, + sum_gradients, + sum_hessians, + value, + lower_bound=children_lower_bound, + upper_bound=children_upper_bound, + ) assert split_info.gain > min_gain_to_split diff --git a/sklearn/ensemble/_hist_gradient_boosting/tests/test_predictor.py b/sklearn/ensemble/_hist_gradient_boosting/tests/test_predictor.py index f0c1348957aa2..3c3c9ae81bac2 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/tests/test_predictor.py +++ b/sklearn/ensemble/_hist_gradient_boosting/tests/test_predictor.py @@ -1,26 +1,36 @@ import numpy as np -from numpy.testing import assert_allclose -from sklearn.datasets import make_regression -from sklearn.model_selection import train_test_split -from sklearn.metrics import r2_score import pytest +from numpy.testing import assert_allclose +from sklearn.datasets import make_regression +from sklearn.ensemble._hist_gradient_boosting._bitset import ( + set_bitset_memoryview, + set_raw_bitset_from_binned_bitset, +) from sklearn.ensemble._hist_gradient_boosting.binning import _BinMapper +from sklearn.ensemble._hist_gradient_boosting.common import ( + ALMOST_INF, + G_H_DTYPE, + PREDICTOR_RECORD_DTYPE, + X_BINNED_DTYPE, + X_BITSET_INNER_DTYPE, + X_DTYPE, +) from sklearn.ensemble._hist_gradient_boosting.grower import TreeGrower from sklearn.ensemble._hist_gradient_boosting.predictor import TreePredictor -from sklearn.ensemble._hist_gradient_boosting.common import ( - G_H_DTYPE, PREDICTOR_RECORD_DTYPE, ALMOST_INF, X_BINNED_DTYPE, - X_BITSET_INNER_DTYPE, X_DTYPE) -from sklearn.ensemble._hist_gradient_boosting._bitset import ( - set_bitset_memoryview, set_raw_bitset_from_binned_bitset) +from sklearn.metrics import r2_score +from sklearn.model_selection import train_test_split +from sklearn.utils._openmp_helpers import _openmp_effective_n_threads + +n_threads = _openmp_effective_n_threads() -@pytest.mark.parametrize('n_bins', [200, 256]) +@pytest.mark.parametrize("n_bins", [200, 256]) def test_regression_dataset(n_bins): - X, y = make_regression(n_samples=500, n_features=10, n_informative=5, - random_state=42) - X_train, X_test, y_train, y_test = train_test_split( - X, y, random_state=42) + X, y = make_regression( + n_samples=500, n_features=10, n_informative=5, random_state=42 + ) + X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42) mapper = _BinMapper(n_bins=n_bins, random_state=42) X_train_binned = mapper.fit_transform(X_train) @@ -31,74 +41,82 @@ def test_regression_dataset(n_bins): min_samples_leaf = 10 max_leaf_nodes = 30 - grower = TreeGrower(X_train_binned, gradients, hessians, - min_samples_leaf=min_samples_leaf, - max_leaf_nodes=max_leaf_nodes, n_bins=n_bins, - n_bins_non_missing=mapper.n_bins_non_missing_) + grower = TreeGrower( + X_train_binned, + gradients, + hessians, + min_samples_leaf=min_samples_leaf, + max_leaf_nodes=max_leaf_nodes, + n_bins=n_bins, + n_bins_non_missing=mapper.n_bins_non_missing_, + ) grower.grow() - predictor = grower.make_predictor( - binning_thresholds=mapper.bin_thresholds_) + predictor = grower.make_predictor(binning_thresholds=mapper.bin_thresholds_) known_cat_bitsets = np.zeros((0, 8), dtype=X_BITSET_INNER_DTYPE) f_idx_map = np.zeros(0, dtype=np.uint32) - y_pred_train = predictor.predict(X_train, known_cat_bitsets, f_idx_map) + y_pred_train = predictor.predict(X_train, known_cat_bitsets, f_idx_map, n_threads) assert r2_score(y_train, y_pred_train) > 0.82 - y_pred_test = predictor.predict(X_test, known_cat_bitsets, f_idx_map) + y_pred_test = predictor.predict(X_test, known_cat_bitsets, f_idx_map, n_threads) assert r2_score(y_test, y_pred_test) > 0.67 -@pytest.mark.parametrize('num_threshold, expected_predictions', [ - (-np.inf, [0, 1, 1, 1]), - (10, [0, 0, 1, 1]), - (20, [0, 0, 0, 1]), - (ALMOST_INF, [0, 0, 0, 1]), - (np.inf, [0, 0, 0, 0]), -]) +@pytest.mark.parametrize( + "num_threshold, expected_predictions", + [ + (-np.inf, [0, 1, 1, 1]), + (10, [0, 0, 1, 1]), + (20, [0, 0, 0, 1]), + (ALMOST_INF, [0, 0, 0, 1]), + (np.inf, [0, 0, 0, 0]), + ], +) def test_infinite_values_and_thresholds(num_threshold, expected_predictions): # Make sure infinite values and infinite thresholds are handled properly. # In particular, if a value is +inf and the threshold is ALMOST_INF the # sample should go to the right child. If the threshold is inf (split on # nan), the +inf sample will go to the left child. - X = np.array([-np.inf, 10, 20, np.inf]).reshape(-1, 1) + X = np.array([-np.inf, 10, 20, np.inf]).reshape(-1, 1) nodes = np.zeros(3, dtype=PREDICTOR_RECORD_DTYPE) # We just construct a simple tree with 1 root and 2 children # parent node - nodes[0]['left'] = 1 - nodes[0]['right'] = 2 - nodes[0]['feature_idx'] = 0 - nodes[0]['num_threshold'] = num_threshold + nodes[0]["left"] = 1 + nodes[0]["right"] = 2 + nodes[0]["feature_idx"] = 0 + nodes[0]["num_threshold"] = num_threshold # left child - nodes[1]['is_leaf'] = True - nodes[1]['value'] = 0 + nodes[1]["is_leaf"] = True + nodes[1]["value"] = 0 # right child - nodes[2]['is_leaf'] = True - nodes[2]['value'] = 1 + nodes[2]["is_leaf"] = True + nodes[2]["value"] = 1 binned_cat_bitsets = np.zeros((0, 8), dtype=X_BITSET_INNER_DTYPE) raw_categorical_bitsets = np.zeros((0, 8), dtype=X_BITSET_INNER_DTYPE) known_cat_bitset = np.zeros((0, 8), dtype=X_BITSET_INNER_DTYPE) f_idx_map = np.zeros(0, dtype=np.uint32) - predictor = TreePredictor( - nodes, binned_cat_bitsets, raw_categorical_bitsets) - predictions = predictor.predict(X, known_cat_bitset, f_idx_map) + predictor = TreePredictor(nodes, binned_cat_bitsets, raw_categorical_bitsets) + predictions = predictor.predict(X, known_cat_bitset, f_idx_map, n_threads) assert np.all(predictions == expected_predictions) @pytest.mark.parametrize( - 'bins_go_left, expected_predictions', [ + "bins_go_left, expected_predictions", + [ ([0, 3, 4, 6], [1, 0, 0, 1, 1, 0]), ([0, 1, 2, 6], [1, 1, 1, 0, 0, 0]), - ([3, 5, 6], [0, 0, 0, 1, 0, 1]) - ]) + ([3, 5, 6], [0, 0, 0, 1, 0, 1]), + ], +) def test_categorical_predictor(bins_go_left, expected_predictions): # Test predictor outputs are correct with categorical features @@ -110,34 +128,35 @@ def test_categorical_predictor(bins_go_left, expected_predictions): # We just construct a simple tree with 1 root and 2 children # parent node nodes = np.zeros(3, dtype=PREDICTOR_RECORD_DTYPE) - nodes[0]['left'] = 1 - nodes[0]['right'] = 2 - nodes[0]['feature_idx'] = 0 - nodes[0]['is_categorical'] = True - nodes[0]['missing_go_to_left'] = True + nodes[0]["left"] = 1 + nodes[0]["right"] = 2 + nodes[0]["feature_idx"] = 0 + nodes[0]["is_categorical"] = True + nodes[0]["missing_go_to_left"] = True # left child - nodes[1]['is_leaf'] = True - nodes[1]['value'] = 1 + nodes[1]["is_leaf"] = True + nodes[1]["value"] = 1 # right child - nodes[2]['is_leaf'] = True - nodes[2]['value'] = 0 + nodes[2]["is_leaf"] = True + nodes[2]["value"] = 0 binned_cat_bitsets = np.zeros((1, 8), dtype=X_BITSET_INNER_DTYPE) raw_categorical_bitsets = np.zeros((1, 8), dtype=X_BITSET_INNER_DTYPE) for go_left in bins_go_left: set_bitset_memoryview(binned_cat_bitsets[0], go_left) - set_raw_bitset_from_binned_bitset(raw_categorical_bitsets[0], - binned_cat_bitsets[0], categories) + set_raw_bitset_from_binned_bitset( + raw_categorical_bitsets[0], binned_cat_bitsets[0], categories + ) - predictor = TreePredictor(nodes, binned_cat_bitsets, - raw_categorical_bitsets) + predictor = TreePredictor(nodes, binned_cat_bitsets, raw_categorical_bitsets) # Check binned data gives correct predictions - prediction_binned = predictor.predict_binned(X_binned, - missing_values_bin_idx=6) + prediction_binned = predictor.predict_binned( + X_binned, missing_values_bin_idx=6, n_threads=n_threads + ) assert_allclose(prediction_binned, expected_predictions) # manually construct bitset @@ -146,17 +165,23 @@ def test_categorical_predictor(bins_go_left, expected_predictions): f_idx_map = np.array([0], dtype=np.uint32) # Check with un-binned data - predictions = predictor.predict(categories.reshape(-1, 1), - known_cat_bitsets, f_idx_map) + predictions = predictor.predict( + categories.reshape(-1, 1), known_cat_bitsets, f_idx_map, n_threads + ) assert_allclose(predictions, expected_predictions) # Check missing goes left because missing_values_bin_idx=6 X_binned_missing = np.array([[6]], dtype=X_BINNED_DTYPE).T - predictions = predictor.predict_binned(X_binned_missing, - missing_values_bin_idx=6) + predictions = predictor.predict_binned( + X_binned_missing, missing_values_bin_idx=6, n_threads=n_threads + ) assert_allclose(predictions, [1]) # missing and unknown go left - predictions = predictor.predict(np.array([[np.nan, 17]], dtype=X_DTYPE).T, - known_cat_bitsets, f_idx_map) + predictions = predictor.predict( + np.array([[np.nan, 17]], dtype=X_DTYPE).T, + known_cat_bitsets, + f_idx_map, + n_threads, + ) assert_allclose(predictions, [1, 1]) diff --git a/sklearn/ensemble/_hist_gradient_boosting/tests/test_splitting.py b/sklearn/ensemble/_hist_gradient_boosting/tests/test_splitting.py index dd0f8bd2c0eda..388697340e08b 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/tests/test_splitting.py +++ b/sklearn/ensemble/_hist_gradient_boosting/tests/test_splitting.py @@ -2,28 +2,34 @@ import pytest from numpy.testing import assert_array_equal -from sklearn.ensemble._hist_gradient_boosting.common import HISTOGRAM_DTYPE -from sklearn.ensemble._hist_gradient_boosting.common import G_H_DTYPE -from sklearn.ensemble._hist_gradient_boosting.common import X_BINNED_DTYPE -from sklearn.ensemble._hist_gradient_boosting.common import MonotonicConstraint +from sklearn.ensemble._hist_gradient_boosting.common import ( + G_H_DTYPE, + HISTOGRAM_DTYPE, + X_BINNED_DTYPE, + MonotonicConstraint, +) +from sklearn.ensemble._hist_gradient_boosting.histogram import HistogramBuilder from sklearn.ensemble._hist_gradient_boosting.splitting import ( Splitter, - compute_node_value + compute_node_value, ) -from sklearn.ensemble._hist_gradient_boosting.histogram import HistogramBuilder +from sklearn.utils._openmp_helpers import _openmp_effective_n_threads from sklearn.utils._testing import skip_if_32bit +n_threads = _openmp_effective_n_threads() -@pytest.mark.parametrize('n_bins', [3, 32, 256]) + +@pytest.mark.parametrize("n_bins", [3, 32, 256]) def test_histogram_split(n_bins): rng = np.random.RandomState(42) feature_idx = 0 l2_regularization = 0 min_hessian_to_split = 1e-3 min_samples_leaf = 1 - min_gain_to_split = 0. + min_gain_to_split = 0.0 X_binned = np.asfortranarray( - rng.randint(0, n_bins - 1, size=(int(1e4), 1)), dtype=X_BINNED_DTYPE) + rng.randint(0, n_bins - 1, size=(int(1e4), 1)), dtype=X_BINNED_DTYPE + ) binned_feature = X_binned.T[feature_idx] sample_indices = np.arange(binned_feature.shape[0], dtype=np.uint32) ordered_hessians = np.ones_like(binned_feature, dtype=G_H_DTYPE) @@ -33,55 +39,63 @@ def test_histogram_split(n_bins): for true_bin in range(1, n_bins - 2): for sign in [-1, 1]: - ordered_gradients = np.full_like(binned_feature, sign, - dtype=G_H_DTYPE) + ordered_gradients = np.full_like(binned_feature, sign, dtype=G_H_DTYPE) ordered_gradients[binned_feature <= true_bin] *= -1 all_gradients = ordered_gradients sum_gradients = all_gradients.sum() - builder = HistogramBuilder(X_binned, - n_bins, - all_gradients, - all_hessians, - hessians_are_constant) - n_bins_non_missing = np.array([n_bins - 1] * X_binned.shape[1], - dtype=np.uint32) - has_missing_values = np.array([False] * X_binned.shape[1], - dtype=np.uint8) + builder = HistogramBuilder( + X_binned, + n_bins, + all_gradients, + all_hessians, + hessians_are_constant, + n_threads, + ) + n_bins_non_missing = np.array( + [n_bins - 1] * X_binned.shape[1], dtype=np.uint32 + ) + has_missing_values = np.array([False] * X_binned.shape[1], dtype=np.uint8) monotonic_cst = np.array( - [MonotonicConstraint.NO_CST] * X_binned.shape[1], - dtype=np.int8) + [MonotonicConstraint.NO_CST] * X_binned.shape[1], dtype=np.int8 + ) is_categorical = np.zeros_like(monotonic_cst, dtype=np.uint8) missing_values_bin_idx = n_bins - 1 - splitter = Splitter(X_binned, - n_bins_non_missing, - missing_values_bin_idx, - has_missing_values, - is_categorical, - monotonic_cst, - l2_regularization, - min_hessian_to_split, - min_samples_leaf, min_gain_to_split, - hessians_are_constant) + splitter = Splitter( + X_binned, + n_bins_non_missing, + missing_values_bin_idx, + has_missing_values, + is_categorical, + monotonic_cst, + l2_regularization, + min_hessian_to_split, + min_samples_leaf, + min_gain_to_split, + hessians_are_constant, + ) histograms = builder.compute_histograms_brute(sample_indices) - value = compute_node_value(sum_gradients, sum_hessians, - -np.inf, np.inf, l2_regularization) + value = compute_node_value( + sum_gradients, sum_hessians, -np.inf, np.inf, l2_regularization + ) split_info = splitter.find_node_split( - sample_indices.shape[0], histograms, sum_gradients, - sum_hessians, value) + sample_indices.shape[0], histograms, sum_gradients, sum_hessians, value + ) assert split_info.bin_idx == true_bin assert split_info.gain >= 0 assert split_info.feature_idx == feature_idx - assert (split_info.n_samples_left + split_info.n_samples_right - == sample_indices.shape[0]) + assert ( + split_info.n_samples_left + split_info.n_samples_right + == sample_indices.shape[0] + ) # Constant hessian: 1. per sample. assert split_info.n_samples_left == split_info.sum_hessian_left @skip_if_32bit -@pytest.mark.parametrize('constant_hessian', [True, False]) +@pytest.mark.parametrize("constant_hessian", [True, False]) def test_gradient_and_hessian_sanity(constant_hessian): # This test checks that the values of gradients and hessians are # consistent in different places: @@ -96,13 +110,14 @@ def test_gradient_and_hessian_sanity(constant_hessian): n_bins = 10 n_features = 20 n_samples = 500 - l2_regularization = 0. + l2_regularization = 0.0 min_hessian_to_split = 1e-3 min_samples_leaf = 1 - min_gain_to_split = 0. + min_gain_to_split = 0.0 - X_binned = rng.randint(0, n_bins, size=(n_samples, n_features), - dtype=X_BINNED_DTYPE) + X_binned = rng.randint( + 0, n_bins, size=(n_samples, n_features), dtype=X_BINNED_DTYPE + ) X_binned = np.asfortranarray(X_binned) sample_indices = np.arange(n_samples, dtype=np.uint32) all_gradients = rng.randn(n_samples).astype(G_H_DTYPE) @@ -114,53 +129,79 @@ def test_gradient_and_hessian_sanity(constant_hessian): all_hessians = rng.lognormal(size=n_samples).astype(G_H_DTYPE) sum_hessians = all_hessians.sum() - builder = HistogramBuilder(X_binned, n_bins, all_gradients, - all_hessians, constant_hessian) - n_bins_non_missing = np.array([n_bins - 1] * X_binned.shape[1], - dtype=np.uint32) + builder = HistogramBuilder( + X_binned, n_bins, all_gradients, all_hessians, constant_hessian, n_threads + ) + n_bins_non_missing = np.array([n_bins - 1] * X_binned.shape[1], dtype=np.uint32) has_missing_values = np.array([False] * X_binned.shape[1], dtype=np.uint8) monotonic_cst = np.array( - [MonotonicConstraint.NO_CST] * X_binned.shape[1], - dtype=np.int8) + [MonotonicConstraint.NO_CST] * X_binned.shape[1], dtype=np.int8 + ) is_categorical = np.zeros_like(monotonic_cst, dtype=np.uint8) missing_values_bin_idx = n_bins - 1 - splitter = Splitter(X_binned, n_bins_non_missing, missing_values_bin_idx, - has_missing_values, is_categorical, monotonic_cst, - l2_regularization, min_hessian_to_split, - min_samples_leaf, min_gain_to_split, constant_hessian) + splitter = Splitter( + X_binned, + n_bins_non_missing, + missing_values_bin_idx, + has_missing_values, + is_categorical, + monotonic_cst, + l2_regularization, + min_hessian_to_split, + min_samples_leaf, + min_gain_to_split, + constant_hessian, + ) hists_parent = builder.compute_histograms_brute(sample_indices) - value_parent = compute_node_value(sum_gradients, sum_hessians, - -np.inf, np.inf, l2_regularization) - si_parent = splitter.find_node_split(n_samples, hists_parent, - sum_gradients, sum_hessians, - value_parent) + value_parent = compute_node_value( + sum_gradients, sum_hessians, -np.inf, np.inf, l2_regularization + ) + si_parent = splitter.find_node_split( + n_samples, hists_parent, sum_gradients, sum_hessians, value_parent + ) sample_indices_left, sample_indices_right, _ = splitter.split_indices( - si_parent, sample_indices) + si_parent, sample_indices + ) hists_left = builder.compute_histograms_brute(sample_indices_left) - value_left = compute_node_value(si_parent.sum_gradient_left, - si_parent.sum_hessian_left, - -np.inf, np.inf, l2_regularization) + value_left = compute_node_value( + si_parent.sum_gradient_left, + si_parent.sum_hessian_left, + -np.inf, + np.inf, + l2_regularization, + ) hists_right = builder.compute_histograms_brute(sample_indices_right) - value_right = compute_node_value(si_parent.sum_gradient_right, - si_parent.sum_hessian_right, - -np.inf, np.inf, l2_regularization) - si_left = splitter.find_node_split(n_samples, hists_left, - si_parent.sum_gradient_left, - si_parent.sum_hessian_left, - value_left) - si_right = splitter.find_node_split(n_samples, hists_right, - si_parent.sum_gradient_right, - si_parent.sum_hessian_right, - value_right) + value_right = compute_node_value( + si_parent.sum_gradient_right, + si_parent.sum_hessian_right, + -np.inf, + np.inf, + l2_regularization, + ) + si_left = splitter.find_node_split( + n_samples, + hists_left, + si_parent.sum_gradient_left, + si_parent.sum_hessian_left, + value_left, + ) + si_right = splitter.find_node_split( + n_samples, + hists_right, + si_parent.sum_gradient_right, + si_parent.sum_hessian_right, + value_right, + ) # make sure that si.sum_gradient_left + si.sum_gradient_right have their # expected value, same for hessians for si, indices in ( - (si_parent, sample_indices), - (si_left, sample_indices_left), - (si_right, sample_indices_right)): + (si_parent, sample_indices), + (si_left, sample_indices_left), + (si_right, sample_indices_right), + ): gradient = si.sum_gradient_right + si.sum_gradient_left expected_gradient = all_gradients[indices].sum() hessian = si.sum_hessian_right + si.sum_hessian_left @@ -178,18 +219,19 @@ def test_gradient_and_hessian_sanity(constant_hessian): hists_left = np.asarray(hists_left, dtype=HISTOGRAM_DTYPE) hists_right = np.asarray(hists_right, dtype=HISTOGRAM_DTYPE) for hists, indices in ( - (hists_parent, sample_indices), - (hists_left, sample_indices_left), - (hists_right, sample_indices_right)): + (hists_parent, sample_indices), + (hists_left, sample_indices_left), + (hists_right, sample_indices_right), + ): # note: gradients and hessians have shape (n_features,), # we're comparing them to *scalars*. This has the benefit of also # making sure that all the entries are equal across features. - gradients = hists['sum_gradients'].sum(axis=1) # shape = (n_features,) + gradients = hists["sum_gradients"].sum(axis=1) # shape = (n_features,) expected_gradient = all_gradients[indices].sum() # scalar - hessians = hists['sum_hessians'].sum(axis=1) + hessians = hists["sum_hessians"].sum(axis=1) if constant_hessian: # 0 is not the actual hessian, but it's not computed in this case - expected_hessian = 0. + expected_hessian = 0.0 else: expected_hessian = all_hessians[indices].sum() @@ -204,22 +246,24 @@ def test_split_indices(): n_bins = 5 n_samples = 10 - l2_regularization = 0. + l2_regularization = 0.0 min_hessian_to_split = 1e-3 min_samples_leaf = 1 - min_gain_to_split = 0. + min_gain_to_split = 0.0 # split will happen on feature 1 and on bin 3 - X_binned = [[0, 0], - [0, 3], - [0, 4], - [0, 0], - [0, 0], - [0, 0], - [0, 0], - [0, 4], - [0, 0], - [0, 4]] + X_binned = [ + [0, 0], + [0, 3], + [0, 4], + [0, 0], + [0, 0], + [0, 0], + [0, 0], + [0, 4], + [0, 0], + [0, 4], + ] X_binned = np.asfortranarray(X_binned, dtype=X_BINNED_DTYPE) sample_indices = np.arange(n_samples, dtype=np.uint32) all_gradients = rng.randn(n_samples).astype(G_H_DTYPE) @@ -228,37 +272,47 @@ def test_split_indices(): sum_hessians = 1 * n_samples hessians_are_constant = True - builder = HistogramBuilder(X_binned, n_bins, - all_gradients, all_hessians, - hessians_are_constant) - n_bins_non_missing = np.array([n_bins] * X_binned.shape[1], - dtype=np.uint32) + builder = HistogramBuilder( + X_binned, n_bins, all_gradients, all_hessians, hessians_are_constant, n_threads + ) + n_bins_non_missing = np.array([n_bins] * X_binned.shape[1], dtype=np.uint32) has_missing_values = np.array([False] * X_binned.shape[1], dtype=np.uint8) monotonic_cst = np.array( - [MonotonicConstraint.NO_CST] * X_binned.shape[1], - dtype=np.int8) + [MonotonicConstraint.NO_CST] * X_binned.shape[1], dtype=np.int8 + ) is_categorical = np.zeros_like(monotonic_cst, dtype=np.uint8) missing_values_bin_idx = n_bins - 1 - splitter = Splitter(X_binned, n_bins_non_missing, missing_values_bin_idx, - has_missing_values, is_categorical, monotonic_cst, - l2_regularization, min_hessian_to_split, - min_samples_leaf, min_gain_to_split, - hessians_are_constant) + splitter = Splitter( + X_binned, + n_bins_non_missing, + missing_values_bin_idx, + has_missing_values, + is_categorical, + monotonic_cst, + l2_regularization, + min_hessian_to_split, + min_samples_leaf, + min_gain_to_split, + hessians_are_constant, + ) assert np.all(sample_indices == splitter.partition) histograms = builder.compute_histograms_brute(sample_indices) - value = compute_node_value(sum_gradients, sum_hessians, - -np.inf, np.inf, l2_regularization) - si_root = splitter.find_node_split(n_samples, histograms, - sum_gradients, sum_hessians, value) + value = compute_node_value( + sum_gradients, sum_hessians, -np.inf, np.inf, l2_regularization + ) + si_root = splitter.find_node_split( + n_samples, histograms, sum_gradients, sum_hessians, value + ) # sanity checks for best split assert si_root.feature_idx == 1 assert si_root.bin_idx == 3 samples_left, samples_right, position_right = splitter.split_indices( - si_root, splitter.partition) + si_root, splitter.partition + ) assert set(samples_left) == set([0, 1, 3, 4, 5, 6, 8]) assert set(samples_right) == set([2, 7, 9]) @@ -280,11 +334,12 @@ def test_min_gain_to_split(): l2_regularization = 0 min_hessian_to_split = 0 min_samples_leaf = 1 - min_gain_to_split = 0. + min_gain_to_split = 0.0 n_bins = 255 n_samples = 100 X_binned = np.asfortranarray( - rng.randint(0, n_bins, size=(n_samples, 1)), dtype=X_BINNED_DTYPE) + rng.randint(0, n_bins, size=(n_samples, 1)), dtype=X_BINNED_DTYPE + ) binned_feature = X_binned[:, 0] sample_indices = np.arange(n_samples, dtype=np.uint32) all_hessians = np.ones_like(binned_feature, dtype=G_H_DTYPE) @@ -293,124 +348,151 @@ def test_min_gain_to_split(): sum_hessians = all_hessians.sum() hessians_are_constant = False - builder = HistogramBuilder(X_binned, n_bins, all_gradients, - all_hessians, hessians_are_constant) - n_bins_non_missing = np.array([n_bins - 1] * X_binned.shape[1], - dtype=np.uint32) + builder = HistogramBuilder( + X_binned, n_bins, all_gradients, all_hessians, hessians_are_constant, n_threads + ) + n_bins_non_missing = np.array([n_bins - 1] * X_binned.shape[1], dtype=np.uint32) has_missing_values = np.array([False] * X_binned.shape[1], dtype=np.uint8) monotonic_cst = np.array( - [MonotonicConstraint.NO_CST] * X_binned.shape[1], - dtype=np.int8) + [MonotonicConstraint.NO_CST] * X_binned.shape[1], dtype=np.int8 + ) is_categorical = np.zeros_like(monotonic_cst, dtype=np.uint8) missing_values_bin_idx = n_bins - 1 - splitter = Splitter(X_binned, n_bins_non_missing, missing_values_bin_idx, - has_missing_values, is_categorical, monotonic_cst, - l2_regularization, - min_hessian_to_split, min_samples_leaf, - min_gain_to_split, hessians_are_constant) + splitter = Splitter( + X_binned, + n_bins_non_missing, + missing_values_bin_idx, + has_missing_values, + is_categorical, + monotonic_cst, + l2_regularization, + min_hessian_to_split, + min_samples_leaf, + min_gain_to_split, + hessians_are_constant, + ) histograms = builder.compute_histograms_brute(sample_indices) - value = compute_node_value(sum_gradients, sum_hessians, - -np.inf, np.inf, l2_regularization) - split_info = splitter.find_node_split(n_samples, histograms, - sum_gradients, sum_hessians, value) + value = compute_node_value( + sum_gradients, sum_hessians, -np.inf, np.inf, l2_regularization + ) + split_info = splitter.find_node_split( + n_samples, histograms, sum_gradients, sum_hessians, value + ) assert split_info.gain == -1 @pytest.mark.parametrize( - 'X_binned, all_gradients, has_missing_values, n_bins_non_missing, ' - ' expected_split_on_nan, expected_bin_idx, expected_go_to_left', [ - + ( + "X_binned, all_gradients, has_missing_values, n_bins_non_missing, " + " expected_split_on_nan, expected_bin_idx, expected_go_to_left" + ), + [ # basic sanity check with no missing values: given the gradient # values, the split must occur on bin_idx=3 - ([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], # X_binned - [1, 1, 1, 1, 5, 5, 5, 5, 5, 5], # gradients - False, # no missing values - 10, # n_bins_non_missing - False, # don't split on nans - 3, # expected_bin_idx - 'not_applicable'), - + ( + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], # X_binned + [1, 1, 1, 1, 5, 5, 5, 5, 5, 5], # gradients + False, # no missing values + 10, # n_bins_non_missing + False, # don't split on nans + 3, # expected_bin_idx + "not_applicable", + ), # We replace 2 samples by NaNs (bin_idx=8) # These 2 samples were mapped to the left node before, so they should # be mapped to left node again # Notice how the bin_idx threshold changes from 3 to 1. - ([8, 0, 1, 8, 2, 3, 4, 5, 6, 7], # 8 <=> missing - [1, 1, 1, 1, 5, 5, 5, 5, 5, 5], - True, # missing values - 8, # n_bins_non_missing - False, # don't split on nans - 1, # cut on bin_idx=1 - True), # missing values go to left - + ( + [8, 0, 1, 8, 2, 3, 4, 5, 6, 7], # 8 <=> missing + [1, 1, 1, 1, 5, 5, 5, 5, 5, 5], + True, # missing values + 8, # n_bins_non_missing + False, # don't split on nans + 1, # cut on bin_idx=1 + True, + ), # missing values go to left # same as above, but with non-consecutive missing_values_bin - ([9, 0, 1, 9, 2, 3, 4, 5, 6, 7], # 9 <=> missing - [1, 1, 1, 1, 5, 5, 5, 5, 5, 5], - True, # missing values - 8, # n_bins_non_missing - False, # don't split on nans - 1, # cut on bin_idx=1 - True), # missing values go to left - + ( + [9, 0, 1, 9, 2, 3, 4, 5, 6, 7], # 9 <=> missing + [1, 1, 1, 1, 5, 5, 5, 5, 5, 5], + True, # missing values + 8, # n_bins_non_missing + False, # don't split on nans + 1, # cut on bin_idx=1 + True, + ), # missing values go to left # this time replacing 2 samples that were on the right. - ([0, 1, 2, 3, 8, 4, 8, 5, 6, 7], # 8 <=> missing - [1, 1, 1, 1, 5, 5, 5, 5, 5, 5], - True, # missing values - 8, # n_bins_non_missing - False, # don't split on nans - 3, # cut on bin_idx=3 (like in first case) - False), # missing values go to right - + ( + [0, 1, 2, 3, 8, 4, 8, 5, 6, 7], # 8 <=> missing + [1, 1, 1, 1, 5, 5, 5, 5, 5, 5], + True, # missing values + 8, # n_bins_non_missing + False, # don't split on nans + 3, # cut on bin_idx=3 (like in first case) + False, + ), # missing values go to right # same as above, but with non-consecutive missing_values_bin - ([0, 1, 2, 3, 9, 4, 9, 5, 6, 7], # 9 <=> missing - [1, 1, 1, 1, 5, 5, 5, 5, 5, 5], - True, # missing values - 8, # n_bins_non_missing - False, # don't split on nans - 3, # cut on bin_idx=3 (like in first case) - False), # missing values go to right - + ( + [0, 1, 2, 3, 9, 4, 9, 5, 6, 7], # 9 <=> missing + [1, 1, 1, 1, 5, 5, 5, 5, 5, 5], + True, # missing values + 8, # n_bins_non_missing + False, # don't split on nans + 3, # cut on bin_idx=3 (like in first case) + False, + ), # missing values go to right # For the following cases, split_on_nans is True (we replace all of # the samples with nans, instead of just 2). - ([0, 1, 2, 3, 4, 4, 4, 4, 4, 4], # 4 <=> missing - [1, 1, 1, 1, 5, 5, 5, 5, 5, 5], - True, # missing values - 4, # n_bins_non_missing - True, # split on nans - 3, # cut on bin_idx=3 - False), # missing values go to right - + ( + [0, 1, 2, 3, 4, 4, 4, 4, 4, 4], # 4 <=> missing + [1, 1, 1, 1, 5, 5, 5, 5, 5, 5], + True, # missing values + 4, # n_bins_non_missing + True, # split on nans + 3, # cut on bin_idx=3 + False, + ), # missing values go to right # same as above, but with non-consecutive missing_values_bin - ([0, 1, 2, 3, 9, 9, 9, 9, 9, 9], # 9 <=> missing - [1, 1, 1, 1, 1, 1, 5, 5, 5, 5], - True, # missing values - 4, # n_bins_non_missing - True, # split on nans - 3, # cut on bin_idx=3 - False), # missing values go to right - - ([6, 6, 6, 6, 0, 1, 2, 3, 4, 5], # 6 <=> missing - [1, 1, 1, 1, 5, 5, 5, 5, 5, 5], - True, # missing values - 6, # n_bins_non_missing - True, # split on nans - 5, # cut on bin_idx=5 - False), # missing values go to right - + ( + [0, 1, 2, 3, 9, 9, 9, 9, 9, 9], # 9 <=> missing + [1, 1, 1, 1, 1, 1, 5, 5, 5, 5], + True, # missing values + 4, # n_bins_non_missing + True, # split on nans + 3, # cut on bin_idx=3 + False, + ), # missing values go to right + ( + [6, 6, 6, 6, 0, 1, 2, 3, 4, 5], # 6 <=> missing + [1, 1, 1, 1, 5, 5, 5, 5, 5, 5], + True, # missing values + 6, # n_bins_non_missing + True, # split on nans + 5, # cut on bin_idx=5 + False, + ), # missing values go to right # same as above, but with non-consecutive missing_values_bin - ([9, 9, 9, 9, 0, 1, 2, 3, 4, 5], # 9 <=> missing - [1, 1, 1, 1, 5, 5, 5, 5, 5, 5], - True, # missing values - 6, # n_bins_non_missing - True, # split on nans - 5, # cut on bin_idx=5 - False), # missing values go to right - ] + ( + [9, 9, 9, 9, 0, 1, 2, 3, 4, 5], # 9 <=> missing + [1, 1, 1, 1, 5, 5, 5, 5, 5, 5], + True, # missing values + 6, # n_bins_non_missing + True, # split on nans + 5, # cut on bin_idx=5 + False, + ), # missing values go to right + ], ) -def test_splitting_missing_values(X_binned, all_gradients, - has_missing_values, n_bins_non_missing, - expected_split_on_nan, expected_bin_idx, - expected_go_to_left): +def test_splitting_missing_values( + X_binned, + all_gradients, + has_missing_values, + n_bins_non_missing, + expected_split_on_nan, + expected_bin_idx, + expected_go_to_left, +): # Make sure missing values are properly supported. # we build an artificial example with gradients such that the best split # is on bin_idx=3, when there are no missing values. @@ -422,10 +504,10 @@ def test_splitting_missing_values(X_binned, all_gradients, n_bins = max(X_binned) + 1 n_samples = len(X_binned) - l2_regularization = 0. + l2_regularization = 0.0 min_hessian_to_split = 1e-3 min_samples_leaf = 1 - min_gain_to_split = 0. + min_gain_to_split = 0.0 sample_indices = np.arange(n_samples, dtype=np.uint32) X_binned = np.array(X_binned, dtype=X_BINNED_DTYPE).reshape(-1, 1) @@ -437,28 +519,37 @@ def test_splitting_missing_values(X_binned, all_gradients, sum_hessians = 1 * n_samples hessians_are_constant = True - builder = HistogramBuilder(X_binned, n_bins, - all_gradients, all_hessians, - hessians_are_constant) + builder = HistogramBuilder( + X_binned, n_bins, all_gradients, all_hessians, hessians_are_constant, n_threads + ) n_bins_non_missing = np.array([n_bins_non_missing], dtype=np.uint32) monotonic_cst = np.array( - [MonotonicConstraint.NO_CST] * X_binned.shape[1], - dtype=np.int8) + [MonotonicConstraint.NO_CST] * X_binned.shape[1], dtype=np.int8 + ) is_categorical = np.zeros_like(monotonic_cst, dtype=np.uint8) missing_values_bin_idx = n_bins - 1 - splitter = Splitter(X_binned, n_bins_non_missing, - missing_values_bin_idx, has_missing_values, - is_categorical, monotonic_cst, - l2_regularization, min_hessian_to_split, - min_samples_leaf, min_gain_to_split, - hessians_are_constant) + splitter = Splitter( + X_binned, + n_bins_non_missing, + missing_values_bin_idx, + has_missing_values, + is_categorical, + monotonic_cst, + l2_regularization, + min_hessian_to_split, + min_samples_leaf, + min_gain_to_split, + hessians_are_constant, + ) histograms = builder.compute_histograms_brute(sample_indices) - value = compute_node_value(sum_gradients, sum_hessians, - -np.inf, np.inf, l2_regularization) - split_info = splitter.find_node_split(n_samples, histograms, - sum_gradients, sum_hessians, value) + value = compute_node_value( + sum_gradients, sum_hessians, -np.inf, np.inf, l2_regularization + ) + split_info = splitter.find_node_split( + n_samples, histograms, sum_gradients, sum_hessians, value + ) assert split_info.bin_idx == expected_bin_idx if has_missing_values: @@ -471,7 +562,8 @@ def test_splitting_missing_values(X_binned, all_gradients, # This also make sure missing values are properly assigned to the correct # child in split_indices() samples_left, samples_right, _ = splitter.split_indices( - split_info, splitter.partition) + split_info, splitter.partition + ) if not expected_split_on_nan: # When we don't split on nans, the split should always be the same. @@ -481,34 +573,35 @@ def test_splitting_missing_values(X_binned, all_gradients, # When we split on nans, samples with missing values are always mapped # to the right child. missing_samples_indices = np.flatnonzero( - np.array(X_binned) == missing_values_bin_idx) + np.array(X_binned) == missing_values_bin_idx + ) non_missing_samples_indices = np.flatnonzero( - np.array(X_binned) != missing_values_bin_idx) + np.array(X_binned) != missing_values_bin_idx + ) assert set(samples_right) == set(missing_samples_indices) assert set(samples_left) == set(non_missing_samples_indices) @pytest.mark.parametrize( - 'X_binned, has_missing_values, n_bins_non_missing, ', [ + "X_binned, has_missing_values, n_bins_non_missing, ", + [ # one category ([0] * 20, False, 1), - # all categories appear less than MIN_CAT_SUPPORT (hardcoded to 10) ([0] * 9 + [1] * 8, False, 2), - # only one category appears more than MIN_CAT_SUPPORT ([0] * 12 + [1] * 8, False, 2), - # missing values + category appear less than MIN_CAT_SUPPORT # 9 is missing ([0] * 9 + [1] * 8 + [9] * 4, True, 2), - # no non-missing category ([9] * 11, True, 0), - ]) -def test_splitting_categorical_cat_smooth(X_binned, has_missing_values, - n_bins_non_missing): + ], +) +def test_splitting_categorical_cat_smooth( + X_binned, has_missing_values, n_bins_non_missing +): # Checks categorical splits are correct when the MIN_CAT_SUPPORT constraint # isn't respected: there are no splits @@ -530,27 +623,38 @@ def test_splitting_categorical_cat_smooth(X_binned, has_missing_values, sum_hessians = n_samples hessians_are_constant = True - builder = HistogramBuilder(X_binned, n_bins, all_gradients, - all_hessians, hessians_are_constant) + builder = HistogramBuilder( + X_binned, n_bins, all_gradients, all_hessians, hessians_are_constant, n_threads + ) n_bins_non_missing = np.array([n_bins_non_missing], dtype=np.uint32) - monotonic_cst = np.array([MonotonicConstraint.NO_CST] * X_binned.shape[1], - dtype=np.int8) + monotonic_cst = np.array( + [MonotonicConstraint.NO_CST] * X_binned.shape[1], dtype=np.int8 + ) is_categorical = np.ones_like(monotonic_cst, dtype=np.uint8) missing_values_bin_idx = n_bins - 1 - splitter = Splitter(X_binned, n_bins_non_missing, - missing_values_bin_idx, has_missing_values, - is_categorical, monotonic_cst, - l2_regularization, min_hessian_to_split, - min_samples_leaf, min_gain_to_split, - hessians_are_constant) + splitter = Splitter( + X_binned, + n_bins_non_missing, + missing_values_bin_idx, + has_missing_values, + is_categorical, + monotonic_cst, + l2_regularization, + min_hessian_to_split, + min_samples_leaf, + min_gain_to_split, + hessians_are_constant, + ) histograms = builder.compute_histograms_brute(sample_indices) - value = compute_node_value(sum_gradients, sum_hessians, - -np.inf, np.inf, l2_regularization) - split_info = splitter.find_node_split(n_samples, histograms, - sum_gradients, sum_hessians, value) + value = compute_node_value( + sum_gradients, sum_hessians, -np.inf, np.inf, l2_regularization + ) + split_info = splitter.find_node_split( + n_samples, histograms, sum_gradients, sum_hessians, value + ) # no split found assert split_info.gain == -1 @@ -572,104 +676,120 @@ def _assert_categories_equals_bitset(categories, bitset): @pytest.mark.parametrize( - "X_binned, all_gradients, expected_categories_left, n_bins_non_missing," - "missing_values_bin_idx, has_missing_values, expected_missing_go_to_left", + ( + "X_binned, all_gradients, expected_categories_left, n_bins_non_missing," + "missing_values_bin_idx, has_missing_values, expected_missing_go_to_left" + ), [ # 4 categories - ([0, 1, 2, 3] * 11, # X_binned - [10, 1, 10, 10] * 11, # all_gradients - [1], # expected_categories_left - 4, # n_bins_non_missing - 4, # missing_values_bin_idx - False, # has_missing_values - None), # expected_missing_go_to_left, unchecked - + ( + [0, 1, 2, 3] * 11, # X_binned + [10, 1, 10, 10] * 11, # all_gradients + [1], # expected_categories_left + 4, # n_bins_non_missing + 4, # missing_values_bin_idx + False, # has_missing_values + None, + ), # expected_missing_go_to_left, unchecked # Make sure that the categories that are on the right (second half) of # the sorted categories array can still go in the left child. In this # case, the best split was found when scanning from right to left. - ([0, 1, 2, 3] * 11, # X_binned - [10, 10, 10, 1] * 11, # all_gradients - [3], # expected_categories_left - 4, # n_bins_non_missing - 4, # missing_values_bin_idx - False, # has_missing_values - None), # expected_missing_go_to_left, unchecked - + ( + [0, 1, 2, 3] * 11, # X_binned + [10, 10, 10, 1] * 11, # all_gradients + [3], # expected_categories_left + 4, # n_bins_non_missing + 4, # missing_values_bin_idx + False, # has_missing_values + None, + ), # expected_missing_go_to_left, unchecked # categories that don't respect MIN_CAT_SUPPORT (cat 4) are always # mapped to the right child - ([0, 1, 2, 3] * 11 + [4] * 5, # X_binned - [10, 10, 10, 1] * 11 + [10] * 5, # all_gradients - [3], # expected_categories_left - 4, # n_bins_non_missing - 4, # missing_values_bin_idx - False, # has_missing_values - None), # expected_missing_go_to_left, unchecked - + ( + [0, 1, 2, 3] * 11 + [4] * 5, # X_binned + [10, 10, 10, 1] * 11 + [10] * 5, # all_gradients + [3], # expected_categories_left + 4, # n_bins_non_missing + 4, # missing_values_bin_idx + False, # has_missing_values + None, + ), # expected_missing_go_to_left, unchecked # categories that don't respect MIN_CAT_SUPPORT are always mapped to # the right child: in this case a more sensible split could have been # 3, 4 - 0, 1, 2 # But the split is still 3 - 0, 1, 2, 4. this is because we only scan # up to the middle of the sorted category array (0, 1, 2, 3), and # because we exclude cat 4 in this array. - ([0, 1, 2, 3] * 11 + [4] * 5, # X_binned - [10, 10, 10, 1] * 11 + [1] * 5, # all_gradients - [3], # expected_categories_left - 4, # n_bins_non_missing - 4, # missing_values_bin_idx - False, # has_missing_values - None), # expected_missing_go_to_left, unchecked - + ( + [0, 1, 2, 3] * 11 + [4] * 5, # X_binned + [10, 10, 10, 1] * 11 + [1] * 5, # all_gradients + [3], # expected_categories_left + 4, # n_bins_non_missing + 4, # missing_values_bin_idx + False, # has_missing_values + None, + ), # expected_missing_go_to_left, unchecked # 4 categories with missing values that go to the right - ([0, 1, 2] * 11 + [9] * 11, # X_binned - [10, 1, 10] * 11 + [10] * 11, # all_gradients - [1], # expected_categories_left - 3, # n_bins_non_missing - 9, # missing_values_bin_idx - True, # has_missing_values - False), # expected_missing_go_to_left - + ( + [0, 1, 2] * 11 + [9] * 11, # X_binned + [10, 1, 10] * 11 + [10] * 11, # all_gradients + [1], # expected_categories_left + 3, # n_bins_non_missing + 9, # missing_values_bin_idx + True, # has_missing_values + False, + ), # expected_missing_go_to_left # 4 categories with missing values that go to the left - ([0, 1, 2] * 11 + [9] * 11, # X_binned - [10, 1, 10] * 11 + [1] * 11, # all_gradients - [1, 9], # expected_categories_left - 3, # n_bins_non_missing - 9, # missing_values_bin_idx - True, # has_missing_values - True), # expected_missing_go_to_left - + ( + [0, 1, 2] * 11 + [9] * 11, # X_binned + [10, 1, 10] * 11 + [1] * 11, # all_gradients + [1, 9], # expected_categories_left + 3, # n_bins_non_missing + 9, # missing_values_bin_idx + True, # has_missing_values + True, + ), # expected_missing_go_to_left # split is on the missing value - ([0, 1, 2, 3, 4] * 11 + [255] * 12, # X_binned - [10, 10, 10, 10, 10] * 11 + [1] * 12, # all_gradients - [255], # expected_categories_left - 5, # n_bins_non_missing - 255, # missing_values_bin_idx - True, # has_missing_values - True), # expected_missing_go_to_left - + ( + [0, 1, 2, 3, 4] * 11 + [255] * 12, # X_binned + [10, 10, 10, 10, 10] * 11 + [1] * 12, # all_gradients + [255], # expected_categories_left + 5, # n_bins_non_missing + 255, # missing_values_bin_idx + True, # has_missing_values + True, + ), # expected_missing_go_to_left # split on even categories - (list(range(60)) * 12, # X_binned - [10, 1] * 360, # all_gradients - list(range(1, 60, 2)), # expected_categories_left - 59, # n_bins_non_missing - 59, # missing_values_bin_idx - True, # has_missing_values - True), # expected_missing_go_to_left - + ( + list(range(60)) * 12, # X_binned + [10, 1] * 360, # all_gradients + list(range(1, 60, 2)), # expected_categories_left + 59, # n_bins_non_missing + 59, # missing_values_bin_idx + True, # has_missing_values + True, + ), # expected_missing_go_to_left # split on every 8 categories - (list(range(256)) * 12, # X_binned - [10, 10, 10, 10, 10, 10, 10, 1] * 384, # all_gradients - list(range(7, 256, 8)), # expected_categories_left - 255, # n_bins_non_missing - 255, # missing_values_bin_idx - True, # has_missing_values - True), # expected_missing_go_to_left - ]) -def test_splitting_categorical_sanity(X_binned, all_gradients, - expected_categories_left, - n_bins_non_missing, - missing_values_bin_idx, - has_missing_values, - expected_missing_go_to_left): + ( + list(range(256)) * 12, # X_binned + [10, 10, 10, 10, 10, 10, 10, 1] * 384, # all_gradients + list(range(7, 256, 8)), # expected_categories_left + 255, # n_bins_non_missing + 255, # missing_values_bin_idx + True, # has_missing_values + True, + ), # expected_missing_go_to_left + ], +) +def test_splitting_categorical_sanity( + X_binned, + all_gradients, + expected_categories_left, + n_bins_non_missing, + missing_values_bin_idx, + has_missing_values, + expected_missing_go_to_left, +): # Tests various combinations of categorical splits n_samples = len(X_binned) @@ -681,7 +801,7 @@ def test_splitting_categorical_sanity(X_binned, all_gradients, l2_regularization = 0.0 min_hessian_to_split = 1e-3 min_samples_leaf = 1 - min_gain_to_split = 0. + min_gain_to_split = 0.0 sample_indices = np.arange(n_samples, dtype=np.uint32) all_gradients = np.array(all_gradients, dtype=G_H_DTYPE) @@ -691,32 +811,44 @@ def test_splitting_categorical_sanity(X_binned, all_gradients, sum_hessians = n_samples hessians_are_constant = True - builder = HistogramBuilder(X_binned, n_bins, all_gradients, - all_hessians, hessians_are_constant) + builder = HistogramBuilder( + X_binned, n_bins, all_gradients, all_hessians, hessians_are_constant, n_threads + ) n_bins_non_missing = np.array([n_bins_non_missing], dtype=np.uint32) - monotonic_cst = np.array([MonotonicConstraint.NO_CST] * X_binned.shape[1], - dtype=np.int8) + monotonic_cst = np.array( + [MonotonicConstraint.NO_CST] * X_binned.shape[1], dtype=np.int8 + ) is_categorical = np.ones_like(monotonic_cst, dtype=np.uint8) - splitter = Splitter(X_binned, n_bins_non_missing, - missing_values_bin_idx, has_missing_values, - is_categorical, monotonic_cst, - l2_regularization, min_hessian_to_split, - min_samples_leaf, min_gain_to_split, - hessians_are_constant) + splitter = Splitter( + X_binned, + n_bins_non_missing, + missing_values_bin_idx, + has_missing_values, + is_categorical, + monotonic_cst, + l2_regularization, + min_hessian_to_split, + min_samples_leaf, + min_gain_to_split, + hessians_are_constant, + ) histograms = builder.compute_histograms_brute(sample_indices) - value = compute_node_value(sum_gradients, sum_hessians, - -np.inf, np.inf, l2_regularization) - split_info = splitter.find_node_split(n_samples, histograms, - sum_gradients, sum_hessians, value) + value = compute_node_value( + sum_gradients, sum_hessians, -np.inf, np.inf, l2_regularization + ) + split_info = splitter.find_node_split( + n_samples, histograms, sum_gradients, sum_hessians, value + ) assert split_info.is_categorical assert split_info.gain > 0 - _assert_categories_equals_bitset(expected_categories_left, - split_info.left_cat_bitset) + _assert_categories_equals_bitset( + expected_categories_left, split_info.left_cat_bitset + ) if has_missing_values: assert split_info.missing_go_to_left == expected_missing_go_to_left # If there is no missing value during training, the flag missing_go_to_left @@ -724,8 +856,215 @@ def test_splitting_categorical_sanity(X_binned, all_gradients, # make sure samples are split correctly samples_left, samples_right, _ = splitter.split_indices( - split_info, splitter.partition) + split_info, splitter.partition + ) left_mask = np.isin(X_binned.ravel(), expected_categories_left) assert_array_equal(sample_indices[left_mask], samples_left) assert_array_equal(sample_indices[~left_mask], samples_right) + + +def test_split_interaction_constraints(): + """Check that allowed_features are respected.""" + n_features = 4 + # features 1 and 2 are not allowed to be split on + allowed_features = np.array([0, 3], dtype=np.uint32) + n_bins = 5 + n_samples = 10 + l2_regularization = 0.0 + min_hessian_to_split = 1e-3 + min_samples_leaf = 1 + min_gain_to_split = 0.0 + + sample_indices = np.arange(n_samples, dtype=np.uint32) + all_hessians = np.ones(1, dtype=G_H_DTYPE) + sum_hessians = n_samples + hessians_are_constant = True + + split_features = [] + + # The loop is to ensure that we split at least once on each allowed feature (0, 3). + # This is tracked by split_features and checked at the end. + for i in range(10): + rng = np.random.RandomState(919 + i) + X_binned = np.asfortranarray( + rng.randint(0, n_bins - 1, size=(n_samples, n_features)), + dtype=X_BINNED_DTYPE, + ) + X_binned = np.asfortranarray(X_binned, dtype=X_BINNED_DTYPE) + + # Make feature 1 very important + all_gradients = (10 * X_binned[:, 1] + rng.randn(n_samples)).astype(G_H_DTYPE) + sum_gradients = all_gradients.sum() + + builder = HistogramBuilder( + X_binned, + n_bins, + all_gradients, + all_hessians, + hessians_are_constant, + n_threads, + ) + n_bins_non_missing = np.array([n_bins] * X_binned.shape[1], dtype=np.uint32) + has_missing_values = np.array([False] * X_binned.shape[1], dtype=np.uint8) + monotonic_cst = np.array( + [MonotonicConstraint.NO_CST] * X_binned.shape[1], dtype=np.int8 + ) + is_categorical = np.zeros_like(monotonic_cst, dtype=np.uint8) + missing_values_bin_idx = n_bins - 1 + splitter = Splitter( + X_binned, + n_bins_non_missing, + missing_values_bin_idx, + has_missing_values, + is_categorical, + monotonic_cst, + l2_regularization, + min_hessian_to_split, + min_samples_leaf, + min_gain_to_split, + hessians_are_constant, + ) + + assert np.all(sample_indices == splitter.partition) + + histograms = builder.compute_histograms_brute(sample_indices) + value = compute_node_value( + sum_gradients, sum_hessians, -np.inf, np.inf, l2_regularization + ) + + # with all features allowed, feature 1 should be split on as it is the most + # important one by construction of the gradients + si_root = splitter.find_node_split( + n_samples, + histograms, + sum_gradients, + sum_hessians, + value, + allowed_features=None, + ) + assert si_root.feature_idx == 1 + + # only features 0 and 3 are allowed to be split on + si_root = splitter.find_node_split( + n_samples, + histograms, + sum_gradients, + sum_hessians, + value, + allowed_features=allowed_features, + ) + split_features.append(si_root.feature_idx) + assert si_root.feature_idx in allowed_features + + # make sure feature 0 and feature 3 are split on in the constraint setting + assert set(allowed_features) == set(split_features) + + +@pytest.mark.parametrize("forbidden_features", [set(), {1, 3}]) +def test_split_feature_fraction_per_split(forbidden_features): + """Check that feature_fraction_per_split is respected. + + Because we set `n_features = 4` and `feature_fraction_per_split = 0.25`, it means + that calling `splitter.find_node_split` will be allowed to select a split for a + single completely random feature at each call. So if we iterate enough, we should + cover all the allowed features, irrespective of the values of the gradients and + Hessians of the objective. + """ + n_features = 4 + allowed_features = np.array( + list(set(range(n_features)) - forbidden_features), dtype=np.uint32 + ) + n_bins = 5 + n_samples = 40 + l2_regularization = 0.0 + min_hessian_to_split = 1e-3 + min_samples_leaf = 1 + min_gain_to_split = 0.0 + rng = np.random.default_rng(42) + + sample_indices = np.arange(n_samples, dtype=np.uint32) + all_gradients = rng.uniform(low=0.5, high=1, size=n_samples).astype(G_H_DTYPE) + sum_gradients = all_gradients.sum() + all_hessians = np.ones(1, dtype=G_H_DTYPE) + sum_hessians = n_samples + hessians_are_constant = True + + X_binned = np.asfortranarray( + rng.integers(low=0, high=n_bins - 1, size=(n_samples, n_features)), + dtype=X_BINNED_DTYPE, + ) + X_binned = np.asfortranarray(X_binned, dtype=X_BINNED_DTYPE) + builder = HistogramBuilder( + X_binned, + n_bins, + all_gradients, + all_hessians, + hessians_are_constant, + n_threads, + ) + histograms = builder.compute_histograms_brute(sample_indices) + value = compute_node_value( + sum_gradients, sum_hessians, -np.inf, np.inf, l2_regularization + ) + n_bins_non_missing = np.array([n_bins] * X_binned.shape[1], dtype=np.uint32) + has_missing_values = np.array([False] * X_binned.shape[1], dtype=np.uint8) + monotonic_cst = np.array( + [MonotonicConstraint.NO_CST] * X_binned.shape[1], dtype=np.int8 + ) + is_categorical = np.zeros_like(monotonic_cst, dtype=np.uint8) + missing_values_bin_idx = n_bins - 1 + + params = dict( + X_binned=X_binned, + n_bins_non_missing=n_bins_non_missing, + missing_values_bin_idx=missing_values_bin_idx, + has_missing_values=has_missing_values, + is_categorical=is_categorical, + monotonic_cst=monotonic_cst, + l2_regularization=l2_regularization, + min_hessian_to_split=min_hessian_to_split, + min_samples_leaf=min_samples_leaf, + min_gain_to_split=min_gain_to_split, + hessians_are_constant=hessians_are_constant, + rng=rng, + ) + splitter_subsample = Splitter( + feature_fraction_per_split=0.25, # THIS is the important setting here. + **params, + ) + splitter_all_features = Splitter(feature_fraction_per_split=1.0, **params) + + assert np.all(sample_indices == splitter_subsample.partition) + + split_features_subsample = [] + split_features_all = [] + # The loop is to ensure that we split at least once on each feature. + # This is tracked by split_features and checked at the end. + for i in range(20): + si_root = splitter_subsample.find_node_split( + n_samples, + histograms, + sum_gradients, + sum_hessians, + value, + allowed_features=allowed_features, + ) + split_features_subsample.append(si_root.feature_idx) + + # This second splitter is our "counterfactual". + si_root = splitter_all_features.find_node_split( + n_samples, + histograms, + sum_gradients, + sum_hessians, + value, + allowed_features=allowed_features, + ) + split_features_all.append(si_root.feature_idx) + + # Make sure all features are split on. + assert set(split_features_subsample) == set(allowed_features) + + # Make sure, our counterfactual always splits on same feature. + assert len(set(split_features_all)) == 1 diff --git a/sklearn/ensemble/_hist_gradient_boosting/tests/test_warm_start.py b/sklearn/ensemble/_hist_gradient_boosting/tests/test_warm_start.py index 2417de4f6cc63..03a2720b36127 100644 --- a/sklearn/ensemble/_hist_gradient_boosting/tests/test_warm_start.py +++ b/sklearn/ensemble/_hist_gradient_boosting/tests/test_warm_start.py @@ -1,19 +1,15 @@ import numpy as np -from numpy.testing import assert_array_equal -from numpy.testing import assert_allclose - import pytest +from numpy.testing import assert_allclose, assert_array_equal from sklearn.base import clone from sklearn.datasets import make_classification, make_regression - -# To use this experimental feature, we need to explicitly ask for it: -from sklearn.experimental import enable_hist_gradient_boosting # noqa -from sklearn.ensemble import HistGradientBoostingRegressor -from sklearn.ensemble import HistGradientBoostingClassifier +from sklearn.ensemble import ( + HistGradientBoostingClassifier, + HistGradientBoostingRegressor, +) from sklearn.metrics import check_scoring - X_classification, y_classification = make_classification(random_state=0) X_regression, y_regression = make_regression(random_state=0) @@ -21,37 +17,43 @@ def _assert_predictor_equal(gb_1, gb_2, X): """Assert that two HistGBM instances are identical.""" # Check identical nodes for each tree - for (pred_ith_1, pred_ith_2) in zip(gb_1._predictors, gb_2._predictors): - for (predictor_1, predictor_2) in zip(pred_ith_1, pred_ith_2): + for pred_ith_1, pred_ith_2 in zip(gb_1._predictors, gb_2._predictors): + for predictor_1, predictor_2 in zip(pred_ith_1, pred_ith_2): assert_array_equal(predictor_1.nodes, predictor_2.nodes) # Check identical predictions assert_allclose(gb_1.predict(X), gb_2.predict(X)) -@pytest.mark.parametrize('GradientBoosting, X, y', [ - (HistGradientBoostingClassifier, X_classification, y_classification), - (HistGradientBoostingRegressor, X_regression, y_regression) -]) +@pytest.mark.parametrize( + "GradientBoosting, X, y", + [ + (HistGradientBoostingClassifier, X_classification, y_classification), + (HistGradientBoostingRegressor, X_regression, y_regression), + ], +) def test_max_iter_with_warm_start_validation(GradientBoosting, X, y): # Check that a ValueError is raised when the maximum number of iterations # is smaller than the number of iterations from the previous fit when warm # start is True. - estimator = GradientBoosting(max_iter=10, early_stopping=False, - warm_start=True) + estimator = GradientBoosting(max_iter=10, early_stopping=False, warm_start=True) estimator.fit(X, y) estimator.set_params(max_iter=5) - err_msg = ('max_iter=5 must be larger than or equal to n_iter_=10 ' - 'when warm_start==True') + err_msg = ( + "max_iter=5 must be larger than or equal to n_iter_=10 when warm_start==True" + ) with pytest.raises(ValueError, match=err_msg): estimator.fit(X, y) -@pytest.mark.parametrize('GradientBoosting, X, y', [ - (HistGradientBoostingClassifier, X_classification, y_classification), - (HistGradientBoostingRegressor, X_regression, y_regression) -]) +@pytest.mark.parametrize( + "GradientBoosting, X, y", + [ + (HistGradientBoostingClassifier, X_classification, y_classification), + (HistGradientBoostingRegressor, X_regression, y_regression), + ], +) def test_warm_start_yields_identical_results(GradientBoosting, X, y): # Make sure that fitting 50 iterations and then 25 with warm start is # equivalent to fitting 75 iterations. @@ -71,14 +73,22 @@ def test_warm_start_yields_identical_results(GradientBoosting, X, y): _assert_predictor_equal(gb_warm_start, gb_no_warm_start, X) -@pytest.mark.parametrize('GradientBoosting, X, y', [ - (HistGradientBoostingClassifier, X_classification, y_classification), - (HistGradientBoostingRegressor, X_regression, y_regression) -]) +@pytest.mark.parametrize( + "GradientBoosting, X, y", + [ + (HistGradientBoostingClassifier, X_classification, y_classification), + (HistGradientBoostingRegressor, X_regression, y_regression), + ], +) def test_warm_start_max_depth(GradientBoosting, X, y): # Test if possible to fit trees of different depth in ensemble. - gb = GradientBoosting(max_iter=20, min_samples_leaf=1, - warm_start=True, max_depth=2, early_stopping=False) + gb = GradientBoosting( + max_iter=20, + min_samples_leaf=1, + warm_start=True, + max_depth=2, + early_stopping=False, + ) gb.fit(X, y) gb.set_params(max_iter=30, max_depth=3, n_iter_no_change=110) gb.fit(X, y) @@ -91,19 +101,27 @@ def test_warm_start_max_depth(GradientBoosting, X, y): assert gb._predictors[-i][0].get_max_depth() == 3 -@pytest.mark.parametrize('GradientBoosting, X, y', [ - (HistGradientBoostingClassifier, X_classification, y_classification), - (HistGradientBoostingRegressor, X_regression, y_regression) -]) -@pytest.mark.parametrize('scoring', (None, 'loss')) +@pytest.mark.parametrize( + "GradientBoosting, X, y", + [ + (HistGradientBoostingClassifier, X_classification, y_classification), + (HistGradientBoostingRegressor, X_regression, y_regression), + ], +) +@pytest.mark.parametrize("scoring", (None, "loss")) def test_warm_start_early_stopping(GradientBoosting, X, y, scoring): # Make sure that early stopping occurs after a small number of iterations # when fitting a second time with warm starting. n_iter_no_change = 5 gb = GradientBoosting( - n_iter_no_change=n_iter_no_change, max_iter=10000, early_stopping=True, - random_state=42, warm_start=True, tol=1e-3, scoring=scoring, + n_iter_no_change=n_iter_no_change, + max_iter=10000, + early_stopping=True, + random_state=42, + warm_start=True, + tol=1e-3, + scoring=scoring, ) gb.fit(X, y) n_iter_first_fit = gb.n_iter_ @@ -112,35 +130,39 @@ def test_warm_start_early_stopping(GradientBoosting, X, y, scoring): assert 0 < n_iter_second_fit - n_iter_first_fit < n_iter_no_change -@pytest.mark.parametrize('GradientBoosting, X, y', [ - (HistGradientBoostingClassifier, X_classification, y_classification), - (HistGradientBoostingRegressor, X_regression, y_regression) -]) +@pytest.mark.parametrize( + "GradientBoosting, X, y", + [ + (HistGradientBoostingClassifier, X_classification, y_classification), + (HistGradientBoostingRegressor, X_regression, y_regression), + ], +) def test_warm_start_equal_n_estimators(GradientBoosting, X, y): # Test if warm start with equal n_estimators does nothing gb_1 = GradientBoosting(max_depth=2, early_stopping=False) gb_1.fit(X, y) gb_2 = clone(gb_1) - gb_2.set_params(max_iter=gb_1.max_iter, warm_start=True, - n_iter_no_change=5) + gb_2.set_params(max_iter=gb_1.max_iter, warm_start=True, n_iter_no_change=5) gb_2.fit(X, y) # Check that both predictors are equal _assert_predictor_equal(gb_1, gb_2, X) -@pytest.mark.parametrize('GradientBoosting, X, y', [ - (HistGradientBoostingClassifier, X_classification, y_classification), - (HistGradientBoostingRegressor, X_regression, y_regression) -]) +@pytest.mark.parametrize( + "GradientBoosting, X, y", + [ + (HistGradientBoostingClassifier, X_classification, y_classification), + (HistGradientBoostingRegressor, X_regression, y_regression), + ], +) def test_warm_start_clear(GradientBoosting, X, y): # Test if fit clears state. gb_1 = GradientBoosting(n_iter_no_change=5, random_state=42) gb_1.fit(X, y) - gb_2 = GradientBoosting(n_iter_no_change=5, random_state=42, - warm_start=True) + gb_2 = GradientBoosting(n_iter_no_change=5, random_state=42, warm_start=True) gb_2.fit(X, y) # inits state gb_2.set_params(warm_start=False) gb_2.fit(X, y) # clears old state and equals est @@ -154,26 +176,28 @@ def test_warm_start_clear(GradientBoosting, X, y): _assert_predictor_equal(gb_1, gb_2, X) -@pytest.mark.parametrize('GradientBoosting, X, y', [ - (HistGradientBoostingClassifier, X_classification, y_classification), - (HistGradientBoostingRegressor, X_regression, y_regression) -]) -@pytest.mark.parametrize('rng_type', ('none', 'int', 'instance')) +@pytest.mark.parametrize( + "GradientBoosting, X, y", + [ + (HistGradientBoostingClassifier, X_classification, y_classification), + (HistGradientBoostingRegressor, X_regression, y_regression), + ], +) +@pytest.mark.parametrize("rng_type", ("none", "int", "instance")) def test_random_seeds_warm_start(GradientBoosting, X, y, rng_type): # Make sure the seeds for train/val split and small trainset subsampling # are correctly set in a warm start context. def _get_rng(rng_type): # Helper to avoid consuming rngs - if rng_type == 'none': + if rng_type == "none": return None - elif rng_type == 'int': + elif rng_type == "int": return 42 else: return np.random.RandomState(0) random_state = _get_rng(rng_type) - gb_1 = GradientBoosting(early_stopping=True, max_iter=2, - random_state=random_state) + gb_1 = GradientBoosting(early_stopping=True, max_iter=2, random_state=random_state) gb_1.set_params(scoring=check_scoring(gb_1)) gb_1.fit(X, y) random_seed_1_1 = gb_1._random_seed @@ -182,8 +206,9 @@ def _get_rng(rng_type): random_seed_1_2 = gb_1._random_seed # clear the old state, different seed random_state = _get_rng(rng_type) - gb_2 = GradientBoosting(early_stopping=True, max_iter=2, - random_state=random_state, warm_start=True) + gb_2 = GradientBoosting( + early_stopping=True, max_iter=2, random_state=random_state, warm_start=True + ) gb_2.set_params(scoring=check_scoring(gb_2)) gb_2.fit(X, y) # inits state random_seed_2_1 = gb_2._random_seed @@ -195,9 +220,9 @@ def _get_rng(rng_type): # * all equal if random state is an integer # * different when refitting and equal with a new estimator (because # the random state is mutated) - if rng_type == 'none': + if rng_type == "none": assert random_seed_1_1 != random_seed_1_2 != random_seed_2_1 - elif rng_type == 'int': + elif rng_type == "int": assert random_seed_1_1 == random_seed_1_2 == random_seed_2_1 else: assert random_seed_1_1 == random_seed_2_1 != random_seed_1_2 diff --git a/sklearn/ensemble/_hist_gradient_boosting/utils.py b/sklearn/ensemble/_hist_gradient_boosting/utils.py new file mode 100644 index 0000000000000..429fbed611c22 --- /dev/null +++ b/sklearn/ensemble/_hist_gradient_boosting/utils.py @@ -0,0 +1,149 @@ +"""This module contains utility routines.""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from ...base import is_classifier +from .binning import _BinMapper + + +def get_equivalent_estimator(estimator, lib="lightgbm", n_classes=None): + """Return an unfitted estimator from another lib with matching hyperparams. + + This utility function takes care of renaming the sklearn parameters into + their LightGBM, XGBoost or CatBoost equivalent parameters. + + # unmapped XGB parameters: + # - min_samples_leaf + # - min_data_in_bin + # - min_split_gain (there is min_split_loss though?) + + # unmapped Catboost parameters: + # max_leaves + # min_* + """ + + if lib not in ("lightgbm", "xgboost", "catboost"): + raise ValueError( + "accepted libs are lightgbm, xgboost, and catboost. got {}".format(lib) + ) + + sklearn_params = estimator.get_params() + + if sklearn_params["loss"] == "auto": + raise ValueError( + "auto loss is not accepted. We need to know if " + "the problem is binary or multiclass classification." + ) + if sklearn_params["early_stopping"]: + raise NotImplementedError("Early stopping should be deactivated.") + + lightgbm_loss_mapping = { + "squared_error": "regression_l2", + "absolute_error": "regression_l1", + "log_loss": "binary" if n_classes == 2 else "multiclass", + "gamma": "gamma", + "poisson": "poisson", + } + + lightgbm_params = { + "objective": lightgbm_loss_mapping[sklearn_params["loss"]], + "learning_rate": sklearn_params["learning_rate"], + "n_estimators": sklearn_params["max_iter"], + "num_leaves": sklearn_params["max_leaf_nodes"], + "max_depth": sklearn_params["max_depth"], + "min_data_in_leaf": sklearn_params["min_samples_leaf"], + "reg_lambda": sklearn_params["l2_regularization"], + "max_bin": sklearn_params["max_bins"], + "min_data_in_bin": 1, + "min_sum_hessian_in_leaf": 1e-3, + "min_split_gain": 0, + "verbosity": 10 if sklearn_params["verbose"] else -10, + "boost_from_average": True, + "enable_bundle": False, # also makes feature order consistent + "subsample_for_bin": _BinMapper().subsample, + "poisson_max_delta_step": 1e-12, + "feature_fraction_bynode": sklearn_params["max_features"], + } + + if sklearn_params["loss"] == "log_loss" and n_classes > 2: + # LightGBM multiplies hessians by 2 in multiclass loss. + lightgbm_params["min_sum_hessian_in_leaf"] *= 2 + # LightGBM 3.0 introduced a different scaling of the hessian for the multiclass + # case. + # It is equivalent of scaling the learning rate. + # See https://github.com/microsoft/LightGBM/pull/3256. + if n_classes is not None: + lightgbm_params["learning_rate"] *= n_classes / (n_classes - 1) + + # XGB + xgboost_loss_mapping = { + "squared_error": "reg:linear", + "absolute_error": "LEAST_ABSOLUTE_DEV_NOT_SUPPORTED", + "log_loss": "reg:logistic" if n_classes == 2 else "multi:softmax", + "gamma": "reg:gamma", + "poisson": "count:poisson", + } + + xgboost_params = { + "tree_method": "hist", + "grow_policy": "lossguide", # so that we can set max_leaves + "objective": xgboost_loss_mapping[sklearn_params["loss"]], + "learning_rate": sklearn_params["learning_rate"], + "n_estimators": sklearn_params["max_iter"], + "max_leaves": sklearn_params["max_leaf_nodes"], + "max_depth": sklearn_params["max_depth"] or 0, + "lambda": sklearn_params["l2_regularization"], + "max_bin": sklearn_params["max_bins"], + "min_child_weight": 1e-3, + "verbosity": 2 if sklearn_params["verbose"] else 0, + "silent": sklearn_params["verbose"] == 0, + "n_jobs": -1, + "colsample_bynode": sklearn_params["max_features"], + } + + # Catboost + catboost_loss_mapping = { + "squared_error": "RMSE", + # catboost does not support MAE when leaf_estimation_method is Newton + "absolute_error": "LEAST_ASBOLUTE_DEV_NOT_SUPPORTED", + "log_loss": "Logloss" if n_classes == 2 else "MultiClass", + "gamma": None, + "poisson": "Poisson", + } + + catboost_params = { + "loss_function": catboost_loss_mapping[sklearn_params["loss"]], + "learning_rate": sklearn_params["learning_rate"], + "iterations": sklearn_params["max_iter"], + "depth": sklearn_params["max_depth"], + "reg_lambda": sklearn_params["l2_regularization"], + "max_bin": sklearn_params["max_bins"], + "feature_border_type": "Median", + "leaf_estimation_method": "Newton", + "verbose": bool(sklearn_params["verbose"]), + } + + if lib == "lightgbm": + from lightgbm import LGBMClassifier, LGBMRegressor + + if is_classifier(estimator): + return LGBMClassifier(**lightgbm_params) + else: + return LGBMRegressor(**lightgbm_params) + + elif lib == "xgboost": + from xgboost import XGBClassifier, XGBRegressor + + if is_classifier(estimator): + return XGBClassifier(**xgboost_params) + else: + return XGBRegressor(**xgboost_params) + + else: + from catboost import CatBoostClassifier, CatBoostRegressor + + if is_classifier(estimator): + return CatBoostClassifier(**catboost_params) + else: + return CatBoostRegressor(**catboost_params) diff --git a/sklearn/ensemble/_hist_gradient_boosting/utils.pyx b/sklearn/ensemble/_hist_gradient_boosting/utils.pyx deleted file mode 100644 index cf2c5a51c90dd..0000000000000 --- a/sklearn/ensemble/_hist_gradient_boosting/utils.pyx +++ /dev/null @@ -1,155 +0,0 @@ -# cython: cdivision=True -# cython: boundscheck=False -# cython: wraparound=False -# cython: language_level=3 -"""This module contains utility routines.""" -# Author: Nicolas Hug - -from cython.parallel import prange - -from ...base import is_classifier -from .binning import _BinMapper -from .common cimport G_H_DTYPE_C -from .common cimport Y_DTYPE_C - - -def get_equivalent_estimator(estimator, lib='lightgbm'): - """Return an unfitted estimator from another lib with matching hyperparams. - - This utility function takes care of renaming the sklearn parameters into - their LightGBM, XGBoost or CatBoost equivalent parameters. - - # unmapped XGB parameters: - # - min_samples_leaf - # - min_data_in_bin - # - min_split_gain (there is min_split_loss though?) - - # unmapped Catboost parameters: - # max_leaves - # min_* - """ - - if lib not in ('lightgbm', 'xgboost', 'catboost'): - raise ValueError('accepted libs are lightgbm, xgboost, and catboost. ' - ' got {}'.format(lib)) - - sklearn_params = estimator.get_params() - - if sklearn_params['loss'] == 'auto': - raise ValueError('auto loss is not accepted. We need to know if ' - 'the problem is binary or multiclass classification.') - if sklearn_params['early_stopping']: - raise NotImplementedError('Early stopping should be deactivated.') - - lightgbm_loss_mapping = { - 'least_squares': 'regression_l2', - 'least_absolute_deviation': 'regression_l1', - 'binary_crossentropy': 'binary', - 'categorical_crossentropy': 'multiclass' - } - - lightgbm_params = { - 'objective': lightgbm_loss_mapping[sklearn_params['loss']], - 'learning_rate': sklearn_params['learning_rate'], - 'n_estimators': sklearn_params['max_iter'], - 'num_leaves': sklearn_params['max_leaf_nodes'], - 'max_depth': sklearn_params['max_depth'], - 'min_child_samples': sklearn_params['min_samples_leaf'], - 'reg_lambda': sklearn_params['l2_regularization'], - 'max_bin': sklearn_params['max_bins'], - 'min_data_in_bin': 1, - 'min_child_weight': 1e-3, - 'min_sum_hessian_in_leaf': 1e-3, - 'min_split_gain': 0, - 'verbosity': 10 if sklearn_params['verbose'] else -10, - 'boost_from_average': True, - 'enable_bundle': False, # also makes feature order consistent - 'min_data_in_bin': 1, - 'subsample_for_bin': _BinMapper().subsample, - } - - if sklearn_params['loss'] == 'categorical_crossentropy': - # LightGBM multiplies hessians by 2 in multiclass loss. - lightgbm_params['min_sum_hessian_in_leaf'] *= 2 - lightgbm_params['learning_rate'] *= 2 - - # XGB - xgboost_loss_mapping = { - 'least_squares': 'reg:linear', - 'least_absolute_deviation': 'LEAST_ABSOLUTE_DEV_NOT_SUPPORTED', - 'binary_crossentropy': 'reg:logistic', - 'categorical_crossentropy': 'multi:softmax' - } - - xgboost_params = { - 'tree_method': 'hist', - 'grow_policy': 'lossguide', # so that we can set max_leaves - 'objective': xgboost_loss_mapping[sklearn_params['loss']], - 'learning_rate': sklearn_params['learning_rate'], - 'n_estimators': sklearn_params['max_iter'], - 'max_leaves': sklearn_params['max_leaf_nodes'], - 'max_depth': sklearn_params['max_depth'] or 0, - 'lambda': sklearn_params['l2_regularization'], - 'max_bin': sklearn_params['max_bins'], - 'min_child_weight': 1e-3, - 'verbosity': 2 if sklearn_params['verbose'] else 0, - 'silent': sklearn_params['verbose'] == 0, - 'n_jobs': -1, - } - - # Catboost - catboost_loss_mapping = { - 'least_squares': 'RMSE', - # catboost does not support MAE when leaf_estimation_method is Newton - 'least_absolute_deviation': 'LEAST_ASBOLUTE_DEV_NOT_SUPPORTED', - 'binary_crossentropy': 'Logloss', - 'categorical_crossentropy': 'MultiClass' - } - - catboost_params = { - 'loss_function': catboost_loss_mapping[sklearn_params['loss']], - 'learning_rate': sklearn_params['learning_rate'], - 'iterations': sklearn_params['max_iter'], - 'depth': sklearn_params['max_depth'], - 'reg_lambda': sklearn_params['l2_regularization'], - 'max_bin': sklearn_params['max_bins'], - 'feature_border_type': 'Median', - 'leaf_estimation_method': 'Newton', - 'verbose': bool(sklearn_params['verbose']), - } - - if lib == 'lightgbm': - from lightgbm import LGBMRegressor - from lightgbm import LGBMClassifier - if is_classifier(estimator): - return LGBMClassifier(**lightgbm_params) - else: - return LGBMRegressor(**lightgbm_params) - - elif lib == 'xgboost': - from xgboost import XGBRegressor - from xgboost import XGBClassifier - if is_classifier(estimator): - return XGBClassifier(**xgboost_params) - else: - return XGBRegressor(**xgboost_params) - - else: - from catboost import CatBoostRegressor - from catboost import CatBoostClassifier - if is_classifier(estimator): - return CatBoostClassifier(**catboost_params) - else: - return CatBoostRegressor(**catboost_params) - - -def sum_parallel(G_H_DTYPE_C [:] array): - - cdef: - Y_DTYPE_C out = 0. - int i = 0 - - for i in prange(array.shape[0], schedule='static', nogil=True): - out += array[i] - - return out diff --git a/sklearn/ensemble/_iforest.py b/sklearn/ensemble/_iforest.py index 588b1bbef299c..4e5287af7f699 100644 --- a/sklearn/ensemble/_iforest.py +++ b/sklearn/ensemble/_iforest.py @@ -1,29 +1,56 @@ -# Authors: Nicolas Goix -# Alexandre Gramfort -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause import numbers +import threading +from numbers import Integral, Real +from warnings import warn + import numpy as np from scipy.sparse import issparse -from warnings import warn +from ..base import OutlierMixin, _fit_context from ..tree import ExtraTreeRegressor +from ..tree._tree import DTYPE as tree_dtype from ..utils import ( - check_random_state, check_array, + check_random_state, gen_batches, - get_chunk_n_rows, ) -from ..utils.fixes import _joblib_parallel_args -from ..utils.validation import check_is_fitted, _num_samples -from ..utils.validation import _deprecate_positional_args -from ..base import OutlierMixin - +from ..utils._chunking import get_chunk_n_rows +from ..utils._param_validation import Interval, RealNotInt, StrOptions +from ..utils.parallel import Parallel, delayed +from ..utils.validation import _num_samples, check_is_fitted, validate_data from ._bagging import BaseBagging __all__ = ["IsolationForest"] +def _parallel_compute_tree_depths( + tree, + X, + features, + tree_decision_path_lengths, + tree_avg_path_lengths, + depths, + lock, +): + """Parallel computation of isolation tree depth.""" + if features is None: + X_subset = X + else: + X_subset = X[:, features] + + leaves_index = tree.apply(X_subset, check_input=False) + + with lock: + depths += ( + tree_decision_path_lengths[leaves_index] + + tree_avg_path_lengths[leaves_index] + - 1.0 + ) + + class IsolationForest(OutlierMixin, BaseBagging): """ Isolation Forest Algorithm. @@ -56,9 +83,10 @@ class IsolationForest(OutlierMixin, BaseBagging): max_samples : "auto", int or float, default="auto" The number of samples to draw from X to train each base estimator. - - If int, then draw `max_samples` samples. - - If float, then draw `max_samples * X.shape[0]` samples. - - If "auto", then `max_samples=min(256, n_samples)`. + + - If int, then draw `max_samples` samples. + - If float, then draw `max_samples * X.shape[0]` samples. + - If "auto", then `max_samples=min(256, n_samples)`. If max_samples is larger than the number of samples provided, all samples will be used for all trees (no sampling). @@ -68,9 +96,9 @@ class IsolationForest(OutlierMixin, BaseBagging): of outliers in the data set. Used when fitting to define the threshold on the scores of the samples. - - If 'auto', the threshold is determined as in the - original paper. - - If float, the contamination should be in the range (0, 0.5]. + - If 'auto', the threshold is determined as in the + original paper. + - If float, the contamination should be in the range (0, 0.5]. .. versionchanged:: 0.22 The default value of ``contamination`` changed from 0.1 @@ -79,8 +107,11 @@ class IsolationForest(OutlierMixin, BaseBagging): max_features : int or float, default=1.0 The number of features to draw from X to train each base estimator. - - If int, then draw `max_features` features. - - If float, then draw `max_features * X.shape[1]` features. + - If int, then draw `max_features` features. + - If float, then draw `max(1, int(max_features * n_features_in_))` features. + + Note: using a float number less than 1.0 or integer less than number of + features will enable feature subsampling and leads to a longer runtime. bootstrap : bool, default=False If True, individual trees are fit on random subsets of the training @@ -88,10 +119,9 @@ class IsolationForest(OutlierMixin, BaseBagging): is performed. n_jobs : int, default=None - The number of jobs to run in parallel for both :meth:`fit` and - :meth:`predict`. ``None`` means 1 unless in a - :obj:`joblib.parallel_backend` context. ``-1`` means using all - processors. See :term:`Glossary ` for more details. + The number of jobs to run in parallel for :meth:`fit`. ``None`` means 1 + unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using + all processors. See :term:`Glossary ` for more details. random_state : int, RandomState instance or None, default=None Controls the pseudo-randomness of the selection of the feature @@ -112,10 +142,13 @@ class IsolationForest(OutlierMixin, BaseBagging): Attributes ---------- - base_estimator_ : ExtraTreeRegressor instance + estimator_ : :class:`~sklearn.tree.ExtraTreeRegressor` instance The child estimator template used to create the collection of fitted sub-estimators. + .. versionadded:: 1.2 + `base_estimator_` was renamed to `estimator_`. + estimators_ : list of ExtraTreeRegressor instances The collection of fitted sub-estimators. @@ -141,12 +174,26 @@ class IsolationForest(OutlierMixin, BaseBagging): .. versionadded:: 0.20 - n_features_ : int - The number of features when ``fit`` is performed. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 - .. deprecated:: 1.0 - Attribute `n_features_` was deprecated in version 1.0 and will be - removed in 1.2. Use `n_features_in_` instead. + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + sklearn.covariance.EllipticEnvelope : An object for detecting outliers in a + Gaussian distributed dataset. + sklearn.svm.OneClassSVM : Unsupervised Outlier Detection. + Estimate the support of a high-dimensional distribution. + The implementation is based on libsvm. + sklearn.neighbors.LocalOutlierFactor : Unsupervised Outlier Detection + using Local Outlier Factor (LOF). Notes ----- @@ -163,16 +210,6 @@ class IsolationForest(OutlierMixin, BaseBagging): anomaly detection." ACM Transactions on Knowledge Discovery from Data (TKDD) 6.1 (2012): 3. - See Also - ---------- - sklearn.covariance.EllipticEnvelope : An object for detecting outliers in a - Gaussian distributed dataset. - sklearn.svm.OneClassSVM : Unsupervised Outlier Detection. - Estimate the support of a high-dimensional distribution. - The implementation is based on libsvm. - sklearn.neighbors.LocalOutlierFactor : Unsupervised Outlier Detection - using Local Outlier Factor (LOF). - Examples -------- >>> from sklearn.ensemble import IsolationForest @@ -180,23 +217,48 @@ class IsolationForest(OutlierMixin, BaseBagging): >>> clf = IsolationForest(random_state=0).fit(X) >>> clf.predict([[0.1], [0], [90]]) array([ 1, 1, -1]) + + For an example of using isolation forest for anomaly detection see + :ref:`sphx_glr_auto_examples_ensemble_plot_isolation_forest.py`. """ - @_deprecate_positional_args - def __init__(self, *, - n_estimators=100, - max_samples="auto", - contamination="auto", - max_features=1., - bootstrap=False, - n_jobs=None, - random_state=None, - verbose=0, - warm_start=False): + + _parameter_constraints: dict = { + "n_estimators": [Interval(Integral, 1, None, closed="left")], + "max_samples": [ + StrOptions({"auto"}), + Interval(Integral, 1, None, closed="left"), + Interval(RealNotInt, 0, 1, closed="right"), + ], + "contamination": [ + StrOptions({"auto"}), + Interval(Real, 0, 0.5, closed="right"), + ], + "max_features": [ + Integral, + Interval(Real, 0, 1, closed="right"), + ], + "bootstrap": ["boolean"], + "n_jobs": [Integral, None], + "random_state": ["random_state"], + "verbose": ["verbose"], + "warm_start": ["boolean"], + } + + def __init__( + self, + *, + n_estimators=100, + max_samples="auto", + contamination="auto", + max_features=1.0, + bootstrap=False, + n_jobs=None, + random_state=None, + verbose=0, + warm_start=False, + ): super().__init__( - base_estimator=ExtraTreeRegressor( - max_features=1, - splitter='random', - random_state=random_state), + estimator=None, # here above max_features has no links with self.max_features bootstrap=bootstrap, bootstrap_features=False, @@ -206,10 +268,19 @@ def __init__(self, *, warm_start=warm_start, n_jobs=n_jobs, random_state=random_state, - verbose=verbose) + verbose=verbose, + ) self.contamination = contamination + def _get_estimator(self): + return ExtraTreeRegressor( + # here max_features has no links with self.max_features + max_features=1, + splitter="random", + random_state=self.random_state, + ) + def _set_oob_score(self, X, y): raise NotImplementedError("OOB score not supported by iforest") @@ -217,9 +288,10 @@ def _parallel_args(self): # ExtraTreeRegressor releases the GIL, so it's more efficient to use # a thread-based backend rather than a process-based backend so as # to avoid suffering from communication overhead and extra memory - # copies. - return _joblib_parallel_args(prefer='threads') + # copies. This is only used in the fit method. + return {"prefer": "threads"} + @_fit_context(prefer_skip_nested_validation=True) def fit(self, X, y=None, sample_weight=None): """ Fit estimator. @@ -242,7 +314,9 @@ def fit(self, X, y=None, sample_weight=None): self : object Fitted estimator. """ - X = self._validate_data(X, accept_sparse=['csc']) + X = validate_data( + self, X, accept_sparse=["csc"], dtype=tree_dtype, ensure_all_finite=False + ) if issparse(X): # Pre-sort indices to avoid that each individual tree of the # ensemble sorts the indices. @@ -254,39 +328,43 @@ def fit(self, X, y=None, sample_weight=None): # ensure that max_sample is in [1, n_samples]: n_samples = X.shape[0] - if self.contamination != 'auto': - if not(0. < self.contamination <= .5): - raise ValueError("contamination must be in (0, 0.5], " - "got: %f" % self.contamination) - - if isinstance(self.max_samples, str): - if self.max_samples == 'auto': - max_samples = min(256, n_samples) - else: - raise ValueError('max_samples (%s) is not supported.' - 'Valid choices are: "auto", int or' - 'float' % self.max_samples) + if isinstance(self.max_samples, str) and self.max_samples == "auto": + max_samples = min(256, n_samples) elif isinstance(self.max_samples, numbers.Integral): if self.max_samples > n_samples: - warn("max_samples (%s) is greater than the " - "total number of samples (%s). max_samples " - "will be set to n_samples for estimation." - % (self.max_samples, n_samples)) + warn( + "max_samples (%s) is greater than the " + "total number of samples (%s). max_samples " + "will be set to n_samples for estimation." + % (self.max_samples, n_samples) + ) max_samples = n_samples else: max_samples = self.max_samples - else: # float - if not 0. < self.max_samples <= 1.: - raise ValueError("max_samples must be in (0, 1], got %r" - % self.max_samples) + else: # max_samples is float max_samples = int(self.max_samples * X.shape[0]) self.max_samples_ = max_samples max_depth = int(np.ceil(np.log2(max(max_samples, 2)))) - super()._fit(X, y, max_samples, - max_depth=max_depth, - sample_weight=sample_weight) + super()._fit( + X, + y, + max_samples, + max_depth=max_depth, + sample_weight=sample_weight, + check_input=False, + ) + + self._average_path_length_per_tree, self._decision_path_lengths = zip( + *[ + ( + _average_path_length(tree.tree_.n_node_samples), + tree.tree_.compute_node_depths(), + ) + for tree in self.estimators_ + ] + ) if self.contamination == "auto": # 0.5 plays a special role as described in the original paper. @@ -294,9 +372,13 @@ def fit(self, X, y=None, sample_weight=None): self.offset_ = -0.5 return self - # else, define offset_ wrt contamination parameter - self.offset_ = np.percentile(self.score_samples(X), - 100. * self.contamination) + # Else, define offset_ wrt contamination parameter + # To avoid performing input validation a second time we call + # _score_samples rather than score_samples. + # _score_samples expects a CSR matrix, so we convert if necessary. + if issparse(X): + X = X.tocsr() + self.offset_ = np.percentile(self._score_samples(X), 100.0 * self.contamination) return self @@ -316,11 +398,28 @@ def predict(self, X): is_inlier : ndarray of shape (n_samples,) For each observation, tells whether or not (+1 or -1) it should be considered as an inlier according to the fitted model. + + Notes + ----- + The predict method can be parallelized by setting a joblib context. This + inherently does NOT use the ``n_jobs`` parameter initialized in the class, + which is used during ``fit``. This is because, predict may actually be faster + without parallelization for a small number of samples, + such as for 1000 samples or less. The user can set the + number of jobs in the joblib context to control the number of parallel jobs. + + .. code-block:: python + + from joblib import parallel_backend + + # Note, we use threading here as the predict method is not CPU bound. + with parallel_backend("threading", n_jobs=4): + model.predict(X) """ check_is_fitted(self) - X = self._validate_data(X, accept_sparse='csr', reset=False) - is_inlier = np.ones(X.shape[0], dtype=int) - is_inlier[self.decision_function(X) < 0] = -1 + decision_func = self.decision_function(X) + is_inlier = np.ones_like(decision_func, dtype=int) + is_inlier[decision_func < 0] = -1 return is_inlier def decision_function(self, X): @@ -349,6 +448,25 @@ def decision_function(self, X): The anomaly score of the input samples. The lower, the more abnormal. Negative scores represent outliers, positive scores represent inliers. + + Notes + ----- + The decision_function method can be parallelized by setting a joblib context. + This inherently does NOT use the ``n_jobs`` parameter initialized in the class, + which is used during ``fit``. This is because, calculating the score may + actually be faster without parallelization for a small number of samples, + such as for 1000 samples or less. + The user can set the number of jobs in the joblib context to control the + number of parallel jobs. + + .. code-block:: python + + from joblib import parallel_backend + + # Note, we use threading here as the decision_function method is + # not CPU bound. + with parallel_backend("threading", n_jobs=4): + model.decision_function(X) """ # We subtract self.offset_ to make 0 be the threshold value for being # an outlier: @@ -378,20 +496,50 @@ def score_samples(self, X): scores : ndarray of shape (n_samples,) The anomaly score of the input samples. The lower, the more abnormal. - """ - # code structure from ForestClassifier/predict_proba - check_is_fitted(self) + Notes + ----- + The score function method can be parallelized by setting a joblib context. This + inherently does NOT use the ``n_jobs`` parameter initialized in the class, + which is used during ``fit``. This is because, calculating the score may + actually be faster without parallelization for a small number of samples, + such as for 1000 samples or less. + The user can set the number of jobs in the joblib context to control the + number of parallel jobs. + + .. code-block:: python + + from joblib import parallel_backend + # Note, we use threading here as the score_samples method is not CPU bound. + with parallel_backend("threading", n_jobs=4): + model.score(X) + """ # Check data - X = self._validate_data(X, accept_sparse='csr', reset=False) + X = validate_data( + self, + X, + accept_sparse="csr", + dtype=tree_dtype, + reset=False, + ensure_all_finite=False, + ) + + return self._score_samples(X) + + def _score_samples(self, X): + """Private version of score_samples without input validation. + + Input validation would remove feature names, so we disable it. + """ + # Code structure from ForestClassifier/predict_proba + + check_is_fitted(self) - # Take the opposite of the scores as bigger is better (here less - # abnormal) + # Take the opposite of the scores as bigger is better (here less abnormal) return -self._compute_chunked_score_samples(X) def _compute_chunked_score_samples(self, X): - n_samples = _num_samples(X) if self._max_features == X.shape[1]: @@ -410,8 +558,9 @@ def _compute_chunked_score_samples(self, X): # the data needed to compute the scores -- the returned scores # themselves are 1D. - chunk_n_rows = get_chunk_n_rows(row_bytes=16 * self._max_features, - max_n_rows=n_samples) + chunk_n_rows = get_chunk_n_rows( + row_bytes=16 * self._max_features, max_n_rows=n_samples + ) slices = gen_batches(n_samples, chunk_n_rows) scores = np.zeros(n_samples, order="f") @@ -433,38 +582,59 @@ def _compute_score_samples(self, X, subsample_features): subsample_features : bool Whether features should be subsampled. + + Returns + ------- + scores : ndarray of shape (n_samples,) + The score of each sample in X. """ n_samples = X.shape[0] depths = np.zeros(n_samples, order="f") - for tree, features in zip(self.estimators_, self.estimators_features_): - X_subset = X[:, features] if subsample_features else X - - leaves_index = tree.apply(X_subset) - node_indicator = tree.decision_path(X_subset) - n_samples_leaf = tree.tree_.n_node_samples[leaves_index] - - depths += ( - np.ravel(node_indicator.sum(axis=1)) - + _average_path_length(n_samples_leaf) - - 1.0 + average_path_length_max_samples = _average_path_length([self._max_samples]) + + # Note: we use default n_jobs value, i.e. sequential computation, which + # we expect to be more performant that parallelizing for small number + # of samples, e.g. < 1k samples. Default n_jobs value can be overridden + # by using joblib.parallel_backend context manager around + # ._compute_score_samples. Using a higher n_jobs may speed up the + # computation of the scores, e.g. for > 1k samples. See + # https://github.com/scikit-learn/scikit-learn/pull/28622 for more + # details. + lock = threading.Lock() + Parallel( + verbose=self.verbose, + require="sharedmem", + )( + delayed(_parallel_compute_tree_depths)( + tree, + X, + features if subsample_features else None, + self._decision_path_lengths[tree_idx], + self._average_path_length_per_tree[tree_idx], + depths, + lock, + ) + for tree_idx, (tree, features) in enumerate( + zip(self.estimators_, self.estimators_features_) ) + ) + denominator = len(self.estimators_) * average_path_length_max_samples scores = 2 ** ( - -depths - / (len(self.estimators_) - * _average_path_length([self.max_samples_])) + # For a single training sample, denominator and depth are 0. + # Therefore, we set the score manually to 1. + -np.divide( + depths, denominator, out=np.ones_like(depths), where=denominator != 0 + ) ) return scores - def _more_tags(self): - return { - '_xfail_checks': { - 'check_sample_weights_invariance': - 'zero sample_weight is not equivalent to removing samples', - } - } + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.allow_nan = True + return tags def _average_path_length(n_samples_leaf): @@ -493,8 +663,8 @@ def _average_path_length(n_samples_leaf): mask_2 = n_samples_leaf == 2 not_mask = ~np.logical_or(mask_1, mask_2) - average_path_length[mask_1] = 0. - average_path_length[mask_2] = 1. + average_path_length[mask_1] = 0.0 + average_path_length[mask_2] = 1.0 average_path_length[not_mask] = ( 2.0 * (np.log(n_samples_leaf[not_mask] - 1.0) + np.euler_gamma) - 2.0 * (n_samples_leaf[not_mask] - 1.0) / n_samples_leaf[not_mask] diff --git a/sklearn/ensemble/_stacking.py b/sklearn/ensemble/_stacking.py index 303015cc9f751..2894d8f174c13 100644 --- a/sklearn/ensemble/_stacking.py +++ b/sklearn/ensemble/_stacking.py @@ -1,49 +1,75 @@ """Stacking classifier and regressor.""" -# Authors: Guillaume Lemaitre -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause from abc import ABCMeta, abstractmethod from copy import deepcopy +from numbers import Integral import numpy as np -from joblib import Parallel import scipy.sparse as sparse -from ..base import clone -from ..base import ClassifierMixin, RegressorMixin, TransformerMixin -from ..base import is_classifier, is_regressor +from ..base import ( + ClassifierMixin, + RegressorMixin, + TransformerMixin, + _fit_context, + clone, + is_classifier, + is_regressor, +) from ..exceptions import NotFittedError -from ..utils._estimator_html_repr import _VisualBlock - -from ._base import _fit_single_estimator -from ._base import _BaseHeterogeneousEnsemble - -from ..linear_model import LogisticRegression -from ..linear_model import RidgeCV - -from ..model_selection import cross_val_predict -from ..model_selection import check_cv - +from ..linear_model import LogisticRegression, RidgeCV +from ..model_selection import check_cv, cross_val_predict from ..preprocessing import LabelEncoder - from ..utils import Bunch -from ..utils.metaestimators import if_delegate_has_method -from ..utils.multiclass import check_classification_targets -from ..utils.validation import check_is_fitted -from ..utils.validation import column_or_1d -from ..utils.validation import _deprecate_positional_args -from ..utils.fixes import delayed - - -class _BaseStacking(TransformerMixin, _BaseHeterogeneousEnsemble, - metaclass=ABCMeta): +from ..utils._param_validation import HasMethods, StrOptions +from ..utils._repr_html.estimator import _VisualBlock +from ..utils.metadata_routing import ( + MetadataRouter, + MethodMapping, + _raise_for_params, + _routing_enabled, + process_routing, +) +from ..utils.metaestimators import available_if +from ..utils.multiclass import check_classification_targets, type_of_target +from ..utils.parallel import Parallel, delayed +from ..utils.validation import ( + _check_feature_names_in, + _check_response_method, + _estimator_has, + check_is_fitted, + column_or_1d, +) +from ._base import _BaseHeterogeneousEnsemble, _fit_single_estimator + + +class _BaseStacking(TransformerMixin, _BaseHeterogeneousEnsemble, metaclass=ABCMeta): """Base class for stacking method.""" + _parameter_constraints: dict = { + "estimators": [list], + "final_estimator": [None, HasMethods("fit")], + "cv": ["cv_object", StrOptions({"prefit"})], + "n_jobs": [None, Integral], + "passthrough": ["boolean"], + "verbose": ["verbose"], + } + @abstractmethod - def __init__(self, estimators, final_estimator=None, *, cv=None, - stack_method='auto', n_jobs=None, verbose=0, - passthrough=False): + def __init__( + self, + estimators, + final_estimator=None, + *, + cv=None, + stack_method="auto", + n_jobs=None, + verbose=0, + passthrough=False, + ): super().__init__(estimators=estimators) self.final_estimator = final_estimator self.cv = cv @@ -70,21 +96,40 @@ def _concatenate_predictions(self, X, predictions): This helper is in charge of ensuring the predictions are 2D arrays and it will drop one of the probability column when using probabilities in the binary case. Indeed, the p(y|c=0) = 1 - p(y|c=1) + + When `y` type is `"multilabel-indicator"`` and the method used is + `predict_proba`, `preds` can be either a `ndarray` of shape + `(n_samples, n_class)` or for some estimators a list of `ndarray`. + This function will drop one of the probability column in this situation as well. """ X_meta = [] for est_idx, preds in enumerate(predictions): - # case where the the estimator returned a 1D array - if preds.ndim == 1: + if isinstance(preds, list): + # `preds` is here a list of `n_targets` 2D ndarrays of + # `n_classes` columns. The k-th column contains the + # probabilities of the samples belonging the k-th class. + # + # Since those probabilities must sum to one for each sample, + # we can work with probabilities of `n_classes - 1` classes. + # Hence we drop the first column. + for pred in preds: + X_meta.append(pred[:, 1:]) + elif preds.ndim == 1: + # Some estimator return a 1D array for predictions + # which must be 2-dimensional arrays. X_meta.append(preds.reshape(-1, 1)) + elif ( + self.stack_method_[est_idx] == "predict_proba" + and len(self.classes_) == 2 + ): + # Remove the first column when using probabilities in + # binary classification because both features `preds` are perfectly + # collinear. + X_meta.append(preds[:, 1:]) else: - if (self.stack_method_[est_idx] == 'predict_proba' and - len(self.classes_) == 2): - # Remove the first column when using probabilities in - # binary classification because both features are perfectly - # collinear. - X_meta.append(preds[:, 1:]) - else: - X_meta.append(preds) + X_meta.append(preds) + + self._n_feature_outs = [pred.shape[1] for pred in X_meta] if self.passthrough: X_meta.append(X) if sparse.issparse(X): @@ -94,22 +139,24 @@ def _concatenate_predictions(self, X, predictions): @staticmethod def _method_name(name, estimator, method): - if estimator == 'drop': + if estimator == "drop": return None - if method == 'auto': - if getattr(estimator, 'predict_proba', None): - return 'predict_proba' - elif getattr(estimator, 'decision_function', None): - return 'decision_function' - else: - return 'predict' - else: - if not hasattr(estimator, method): - raise ValueError('Underlying estimator {} does not implement ' - 'the method {}.'.format(name, method)) - return method + if method == "auto": + method = ["predict_proba", "decision_function", "predict"] + try: + method_name = _check_response_method(estimator, method).__name__ + except AttributeError as e: + raise ValueError( + f"Underlying estimator {name} does not implement the method {method}." + ) from e - def fit(self, X, y, sample_weight=None): + return method_name + + @_fit_context( + # estimators in Stacking*.estimators are not validated yet + prefer_skip_nested_validation=False + ) + def fit(self, X, y, **fit_params): """Fit the estimators. Parameters @@ -121,14 +168,13 @@ def fit(self, X, y, sample_weight=None): y : array-like of shape (n_samples,) Target values. - sample_weight : array-like of shape (n_samples,) or default=None - Sample weights. If None, then samples are equally weighted. - Note that this is supported only if all underlying estimators - support sample weights. + **fit_params : dict + Dict of metadata, potentially containing sample_weight as a + key-value pair. If sample_weight is not present, then samples are + equally weighted. Note that sample_weight is supported only if all + underlying estimators support sample weights. - .. versionchanged:: 0.23 - when not None, `sample_weight` is passed to all underlying - estimators + .. versionadded:: 1.6 Returns ------- @@ -141,60 +187,94 @@ def fit(self, X, y, sample_weight=None): stack_method = [self.stack_method] * len(all_estimators) - # Fit the base estimators on the whole training data. Those - # base estimators will be used in transform, predict, and - # predict_proba. They are exposed publicly. - self.estimators_ = Parallel(n_jobs=self.n_jobs)( - delayed(_fit_single_estimator)(clone(est), X, y, sample_weight) - for est in all_estimators if est != 'drop' - ) + if _routing_enabled(): + routed_params = process_routing(self, "fit", **fit_params) + else: + routed_params = Bunch() + for name in names: + routed_params[name] = Bunch(fit={}) + if "sample_weight" in fit_params: + routed_params[name].fit["sample_weight"] = fit_params[ + "sample_weight" + ] + + if self.cv == "prefit": + self.estimators_ = [] + for estimator in all_estimators: + if estimator != "drop": + check_is_fitted(estimator) + self.estimators_.append(estimator) + else: + # Fit the base estimators on the whole training data. Those + # base estimators will be used in transform, predict, and + # predict_proba. They are exposed publicly. + self.estimators_ = Parallel(n_jobs=self.n_jobs)( + delayed(_fit_single_estimator)( + clone(est), X, y, routed_params[name]["fit"] + ) + for name, est in zip(names, all_estimators) + if est != "drop" + ) self.named_estimators_ = Bunch() est_fitted_idx = 0 for name_est, org_est in zip(names, all_estimators): - if org_est != 'drop': - self.named_estimators_[name_est] = self.estimators_[ - est_fitted_idx] + if org_est != "drop": + current_estimator = self.estimators_[est_fitted_idx] + self.named_estimators_[name_est] = current_estimator est_fitted_idx += 1 + if hasattr(current_estimator, "feature_names_in_"): + self.feature_names_in_ = current_estimator.feature_names_in_ else: - self.named_estimators_[name_est] = 'drop' - - # To train the meta-classifier using the most data as possible, we use - # a cross-validation to obtain the output of the stacked estimators. - - # To ensure that the data provided to each estimator are the same, we - # need to set the random state of the cv if there is one and we need to - # take a copy. - cv = check_cv(self.cv, y=y, classifier=is_classifier(self)) - if hasattr(cv, 'random_state') and cv.random_state is None: - cv.random_state = np.random.RandomState() + self.named_estimators_[name_est] = "drop" self.stack_method_ = [ self._method_name(name, est, meth) for name, est, meth in zip(names, all_estimators, stack_method) ] - fit_params = ({"sample_weight": sample_weight} - if sample_weight is not None - else None) - predictions = Parallel(n_jobs=self.n_jobs)( - delayed(cross_val_predict)(clone(est), X, y, cv=deepcopy(cv), - method=meth, n_jobs=self.n_jobs, - fit_params=fit_params, - verbose=self.verbose) - for est, meth in zip(all_estimators, self.stack_method_) - if est != 'drop' - ) + + if self.cv == "prefit": + # Generate predictions from prefit models + predictions = [ + getattr(estimator, predict_method)(X) + for estimator, predict_method in zip(all_estimators, self.stack_method_) + if estimator != "drop" + ] + else: + # To train the meta-classifier using the most data as possible, we use + # a cross-validation to obtain the output of the stacked estimators. + # To ensure that the data provided to each estimator are the same, + # we need to set the random state of the cv if there is one and we + # need to take a copy. + cv = check_cv(self.cv, y=y, classifier=is_classifier(self)) + if hasattr(cv, "random_state") and cv.random_state is None: + cv.random_state = np.random.RandomState() + + predictions = Parallel(n_jobs=self.n_jobs)( + delayed(cross_val_predict)( + clone(est), + X, + y, + cv=deepcopy(cv), + method=meth, + n_jobs=self.n_jobs, + params=routed_params[name]["fit"], + verbose=self.verbose, + ) + for name, est, meth in zip(names, all_estimators, self.stack_method_) + if est != "drop" + ) # Only not None or not 'drop' estimators will be used in transform. # Remove the None from the method as well. self.stack_method_ = [ - meth for (meth, est) in zip(self.stack_method_, all_estimators) - if est != 'drop' + meth + for (meth, est) in zip(self.stack_method_, all_estimators) + if est != "drop" ] X_meta = self._concatenate_predictions(X, predictions) - _fit_single_estimator(self.final_estimator_, X_meta, y, - sample_weight=sample_weight) + _fit_single_estimator(self.final_estimator_, X_meta, y, fit_params=fit_params) return self @@ -205,8 +285,8 @@ def n_features_in_(self): check_is_fitted(self) except NotFittedError as nfe: raise AttributeError( - f"{self.__class__.__name__} object has no attribute " - f"n_features_in_") from nfe + f"{self.__class__.__name__} object has no attribute n_features_in_" + ) from nfe return self.estimators_[0].n_features_in_ def _transform(self, X): @@ -215,25 +295,73 @@ def _transform(self, X): predictions = [ getattr(est, meth)(X) for est, meth in zip(self.estimators_, self.stack_method_) - if est != 'drop' + if est != "drop" ] return self._concatenate_predictions(X, predictions) - @if_delegate_has_method(delegate='final_estimator_') + def get_feature_names_out(self, input_features=None): + """Get output feature names for transformation. + + Parameters + ---------- + input_features : array-like of str or None, default=None + Input features. The input feature names are only used when `passthrough` is + `True`. + + - If `input_features` is `None`, then `feature_names_in_` is + used as feature names in. If `feature_names_in_` is not defined, + then names are generated: `[x0, x1, ..., x(n_features_in_ - 1)]`. + - If `input_features` is an array-like, then `input_features` must + match `feature_names_in_` if `feature_names_in_` is defined. + + If `passthrough` is `False`, then only the names of `estimators` are used + to generate the output feature names. + + Returns + ------- + feature_names_out : ndarray of str objects + Transformed feature names. + """ + check_is_fitted(self, "n_features_in_") + input_features = _check_feature_names_in( + self, input_features, generate_names=self.passthrough + ) + + class_name = self.__class__.__name__.lower() + non_dropped_estimators = ( + name for name, est in self.estimators if est != "drop" + ) + meta_names = [] + for est, n_features_out in zip(non_dropped_estimators, self._n_feature_outs): + if n_features_out == 1: + meta_names.append(f"{class_name}_{est}") + else: + meta_names.extend( + f"{class_name}_{est}{i}" for i in range(n_features_out) + ) + + if self.passthrough: + return np.concatenate((meta_names, input_features)) + + return np.asarray(meta_names, dtype=object) + + @available_if( + _estimator_has("predict", delegates=("final_estimator_", "final_estimator")) + ) def predict(self, X, **predict_params): """Predict target for X. Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) - Training vectors, where n_samples is the number of samples and - n_features is the number of features. + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. **predict_params : dict of str -> obj Parameters to the `predict` called by the `final_estimator`. Note that this may be used to return uncertainties from some estimators with `return_std` or `return_cov`. Be aware that it will only - accounts for uncertainty in the final estimator. + account for uncertainty in the final estimator. Returns ------- @@ -242,17 +370,53 @@ def predict(self, X, **predict_params): """ check_is_fitted(self) - return self.final_estimator_.predict( - self.transform(X), **predict_params - ) + return self.final_estimator_.predict(self.transform(X), **predict_params) - def _sk_visual_block_(self, final_estimator): + def _sk_visual_block_with_final_estimator(self, final_estimator): names, estimators = zip(*self.estimators) - parallel = _VisualBlock('parallel', estimators, names=names, - dash_wrapped=False) - serial = _VisualBlock('serial', (parallel, final_estimator), - dash_wrapped=False) - return _VisualBlock('serial', [serial]) + parallel = _VisualBlock("parallel", estimators, names=names, dash_wrapped=False) + + # final estimator is wrapped in a parallel block to show the label: + # 'final_estimator' in the html repr + final_block = _VisualBlock( + "parallel", [final_estimator], names=["final_estimator"], dash_wrapped=False + ) + return _VisualBlock("serial", (parallel, final_block), dash_wrapped=False) + + def get_metadata_routing(self): + """Get metadata routing of this object. + + Please check :ref:`User Guide ` on how the routing + mechanism works. + + .. versionadded:: 1.6 + + Returns + ------- + routing : MetadataRouter + A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating + routing information. + """ + router = MetadataRouter(owner=self.__class__.__name__) + + # `self.estimators` is a list of (name, est) tuples + for name, estimator in self.estimators: + router.add( + **{name: estimator}, + method_mapping=MethodMapping().add(callee="fit", caller="fit"), + ) + + try: + final_estimator_ = self.final_estimator_ + except AttributeError: + final_estimator_ = self.final_estimator + + router.add( + final_estimator_=final_estimator_, + method_mapping=MethodMapping().add(caller="predict", callee="predict"), + ) + + return router class StackingClassifier(ClassifierMixin, _BaseStacking): @@ -278,12 +442,16 @@ class StackingClassifier(ClassifierMixin, _BaseStacking): list is defined as a tuple of string (i.e. name) and an estimator instance. An estimator can be set to 'drop' using `set_params`. + The type of estimator is generally expected to be a classifier. + However, one can pass a regressor for some use case (e.g. ordinal + regression). + final_estimator : estimator, default=None A classifier which will be used to combine the base estimators. The default classifier is a :class:`~sklearn.linear_model.LogisticRegression`. - cv : int, cross-validation generator or an iterable, default=None + cv : int, cross-validation generator, iterable, or "prefit", default=None Determines the cross-validation splitting strategy used in `cross_val_predict` to train `final_estimator`. Possible inputs for cv are: @@ -291,16 +459,29 @@ class StackingClassifier(ClassifierMixin, _BaseStacking): * None, to use the default 5-fold cross validation, * integer, to specify the number of folds in a (Stratified) KFold, * An object to be used as a cross-validation generator, - * An iterable yielding train, test splits. + * An iterable yielding train, test splits, + * `"prefit"`, to assume the `estimators` are prefit. In this case, the + estimators will not be refitted. For integer/None inputs, if the estimator is a classifier and y is either binary or multiclass, :class:`~sklearn.model_selection.StratifiedKFold` is used. In all other cases, :class:`~sklearn.model_selection.KFold` is used. + These splitters are instantiated with `shuffle=False` so the splits + will be the same across calls. Refer :ref:`User Guide ` for the various cross-validation strategies that can be used here. + If "prefit" is passed, it is assumed that all `estimators` have + been fitted already. The `final_estimator_` is trained on the `estimators` + predictions on the full training set and are **not** cross validated + predictions. Please note that if the models have been trained on the same + data to train the stacking model, there is a very high risk of overfitting. + + .. versionadded:: 1.1 + The 'prefit' option was added in 1.1 + .. note:: A larger number of split will provide no benefits if the number of training samples is large enough. Indeed, the training time @@ -319,9 +500,9 @@ class StackingClassifier(ClassifierMixin, _BaseStacking): will raise an error. n_jobs : int, default=None - The number of jobs to run in parallel all `estimators` `fit`. + The number of jobs to run in parallel for `fit` of all `estimators`. `None` means 1 unless in a `joblib.parallel_backend` context. -1 means - using all processors. See Glossary for more details. + using all processors. See :term:`Glossary ` for more details. passthrough : bool, default=False When False, only the predictions of estimators will be used as @@ -334,31 +515,56 @@ class StackingClassifier(ClassifierMixin, _BaseStacking): Attributes ---------- - classes_ : ndarray of shape (n_classes,) + classes_ : ndarray of shape (n_classes,) or list of ndarray if `y` \ + is of type `"multilabel-indicator"`. Class labels. estimators_ : list of estimators - The elements of the estimators parameter, having been fitted on the + The elements of the `estimators` parameter, having been fitted on the training data. If an estimator has been set to `'drop'`, it - will not appear in `estimators_`. + will not appear in `estimators_`. When `cv="prefit"`, `estimators_` + is set to `estimators` and is not fitted again. named_estimators_ : :class:`~sklearn.utils.Bunch` Attribute to access any fitted sub-estimators by name. + n_features_in_ : int + Number of features seen during :term:`fit`. Only defined if the + underlying estimator exposes such an attribute when fit. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Only defined if the + underlying estimators expose such an attribute when fit. + + .. versionadded:: 1.0 + final_estimator_ : estimator - The classifier which predicts given the output of `estimators_`. + The classifier fit on the output of `estimators_` and responsible for + final predictions. stack_method_ : list of str The method used by each base estimator. + See Also + -------- + StackingRegressor : Stack of estimators with a final regressor. + Notes ----- When `predict_proba` is used by each estimator (i.e. most of the time for `stack_method='auto'` or specifically for `stack_method='predict_proba'`), - The first column predicted by each estimator will be dropped in the case + the first column predicted by each estimator will be dropped in the case of a binary classification problem. Indeed, both feature will be perfectly collinear. + In some cases (e.g. ordinal regression), one can pass regressors as the + first layer of the :class:`StackingClassifier`. However, note that `y` will + be internally encoded in a numerically increasing order or lexicographic + order. If this ordering is not adequate, one should manually numerically + encode the classes in the desired order. + References ---------- .. [1] Wolpert, David H. "Stacked generalization." Neural networks 5.2 @@ -388,12 +594,26 @@ class StackingClassifier(ClassifierMixin, _BaseStacking): ... ) >>> clf.fit(X_train, y_train).score(X_test, y_test) 0.9... - """ - @_deprecate_positional_args - def __init__(self, estimators, final_estimator=None, *, cv=None, - stack_method='auto', n_jobs=None, passthrough=False, - verbose=0): + + _parameter_constraints: dict = { + **_BaseStacking._parameter_constraints, + "stack_method": [ + StrOptions({"auto", "predict_proba", "decision_function", "predict"}) + ], + } + + def __init__( + self, + estimators, + final_estimator=None, + *, + cv=None, + stack_method="auto", + n_jobs=None, + passthrough=False, + verbose=0, + ): super().__init__( estimators=estimators, final_estimator=final_estimator, @@ -401,18 +621,42 @@ def __init__(self, estimators, final_estimator=None, *, cv=None, stack_method=stack_method, n_jobs=n_jobs, passthrough=passthrough, - verbose=verbose + verbose=verbose, ) def _validate_final_estimator(self): self._clone_final_estimator(default=LogisticRegression()) if not is_classifier(self.final_estimator_): raise ValueError( - "'final_estimator' parameter should be a classifier. Got {}" - .format(self.final_estimator_) + "'final_estimator' parameter should be a classifier. Got {}".format( + self.final_estimator_ + ) ) - def fit(self, X, y, sample_weight=None): + def _validate_estimators(self): + """Overload the method of `_BaseHeterogeneousEnsemble` to be more + lenient towards the type of `estimators`. + + Regressors can be accepted for some cases such as ordinal regression. + """ + if len(self.estimators) == 0: + raise ValueError( + "Invalid 'estimators' attribute, 'estimators' should be a " + "non-empty list of (string, estimator) tuples." + ) + names, estimators = zip(*self.estimators) + self._validate_names(names) + + has_estimator = any(est != "drop" for est in estimators) + if not has_estimator: + raise ValueError( + "All estimators are dropped. At least one is required " + "to be an estimator." + ) + + return names, estimators + + def fit(self, X, y, **fit_params): """Fit the estimators. Parameters @@ -422,56 +666,113 @@ def fit(self, X, y, sample_weight=None): `n_features` is the number of features. y : array-like of shape (n_samples,) - Target values. + Target values. Note that `y` will be internally encoded in + numerically increasing order or lexicographic order. If the order + matter (e.g. for ordinal regression), one should numerically encode + the target `y` before calling :term:`fit`. - sample_weight : array-like of shape (n_samples,), default=None - Sample weights. If None, then samples are equally weighted. - Note that this is supported only if all underlying estimators - support sample weights. + **fit_params : dict + Parameters to pass to the underlying estimators. + + .. versionadded:: 1.6 + + Only available if `enable_metadata_routing=True`, which can be + set by using ``sklearn.set_config(enable_metadata_routing=True)``. + See :ref:`Metadata Routing User Guide ` for + more details. Returns ------- self : object + Returns a fitted instance of estimator. """ + _raise_for_params(fit_params, self, "fit", allow=["sample_weight"]) + check_classification_targets(y) - self._le = LabelEncoder().fit(y) - self.classes_ = self._le.classes_ - return super().fit(X, self._le.transform(y), sample_weight) + if type_of_target(y) == "multilabel-indicator": + self._label_encoder = [LabelEncoder().fit(yk) for yk in y.T] + self.classes_ = [le.classes_ for le in self._label_encoder] + y_encoded = np.array( + [ + self._label_encoder[target_idx].transform(target) + for target_idx, target in enumerate(y.T) + ] + ).T + else: + self._label_encoder = LabelEncoder().fit(y) + self.classes_ = self._label_encoder.classes_ + y_encoded = self._label_encoder.transform(y) + + return super().fit(X, y_encoded, **fit_params) - @if_delegate_has_method(delegate='final_estimator_') + @available_if( + _estimator_has("predict", delegates=("final_estimator_", "final_estimator")) + ) def predict(self, X, **predict_params): """Predict target for X. Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) - Training vectors, where n_samples is the number of samples and - n_features is the number of features. + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. **predict_params : dict of str -> obj Parameters to the `predict` called by the `final_estimator`. Note that this may be used to return uncertainties from some estimators with `return_std` or `return_cov`. Be aware that it will only - accounts for uncertainty in the final estimator. + account for uncertainty in the final estimator. + + - If `enable_metadata_routing=False` (default): + Parameters directly passed to the `predict` method of the + `final_estimator`. + + - If `enable_metadata_routing=True`: Parameters safely routed to + the `predict` method of the `final_estimator`. See :ref:`Metadata + Routing User Guide ` for more details. + + .. versionchanged:: 1.6 + `**predict_params` can be routed via metadata routing API. Returns ------- y_pred : ndarray of shape (n_samples,) or (n_samples, n_output) Predicted targets. """ - y_pred = super().predict(X, **predict_params) - return self._le.inverse_transform(y_pred) + if _routing_enabled(): + routed_params = process_routing(self, "predict", **predict_params) + else: + # TODO(SLEP6): remove when metadata routing cannot be disabled. + routed_params = Bunch() + routed_params.final_estimator_ = Bunch(predict={}) + routed_params.final_estimator_.predict = predict_params + + y_pred = super().predict(X, **routed_params.final_estimator_["predict"]) + if isinstance(self._label_encoder, list): + # Handle the multilabel-indicator case + y_pred = np.array( + [ + self._label_encoder[target_idx].inverse_transform(target) + for target_idx, target in enumerate(y_pred.T) + ] + ).T + else: + y_pred = self._label_encoder.inverse_transform(y_pred) + return y_pred - @if_delegate_has_method(delegate='final_estimator_') + @available_if( + _estimator_has( + "predict_proba", delegates=("final_estimator_", "final_estimator") + ) + ) def predict_proba(self, X): - """Predict class probabilities for X using - `final_estimator_.predict_proba`. + """Predict class probabilities for `X` using the final estimator. Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) - Training vectors, where n_samples is the number of samples and - n_features is the number of features. + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. Returns ------- @@ -480,18 +781,26 @@ def predict_proba(self, X): The class probabilities of the input samples. """ check_is_fitted(self) - return self.final_estimator_.predict_proba(self.transform(X)) + y_pred = self.final_estimator_.predict_proba(self.transform(X)) + + if isinstance(self._label_encoder, list): + # Handle the multilabel-indicator cases + y_pred = np.array([preds[:, 0] for preds in y_pred]).T + return y_pred - @if_delegate_has_method(delegate='final_estimator_') + @available_if( + _estimator_has( + "decision_function", delegates=("final_estimator_", "final_estimator") + ) + ) def decision_function(self, X): - """Predict decision function for samples in X using - `final_estimator_.decision_function`. + """Decision function for samples in `X` using the final estimator. Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) - Training vectors, where n_samples is the number of samples and - n_features is the number of features. + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. Returns ------- @@ -526,7 +835,7 @@ def _sk_visual_block_(self): final_estimator = LogisticRegression() else: final_estimator = self.final_estimator - return super()._sk_visual_block_(final_estimator) + return super()._sk_visual_block_with_final_estimator(final_estimator) class StackingRegressor(RegressorMixin, _BaseStacking): @@ -556,7 +865,7 @@ class StackingRegressor(RegressorMixin, _BaseStacking): A regressor which will be used to combine the base estimators. The default regressor is a :class:`~sklearn.linear_model.RidgeCV`. - cv : int, cross-validation generator or an iterable, default=None + cv : int, cross-validation generator, iterable, or "prefit", default=None Determines the cross-validation splitting strategy used in `cross_val_predict` to train `final_estimator`. Possible inputs for cv are: @@ -564,16 +873,29 @@ class StackingRegressor(RegressorMixin, _BaseStacking): * None, to use the default 5-fold cross validation, * integer, to specify the number of folds in a (Stratified) KFold, * An object to be used as a cross-validation generator, - * An iterable yielding train, test splits. + * An iterable yielding train, test splits, + * `"prefit"`, to assume the `estimators` are prefit. In this case, the + estimators will not be refitted. For integer/None inputs, if the estimator is a classifier and y is either binary or multiclass, :class:`~sklearn.model_selection.StratifiedKFold` is used. In all other cases, :class:`~sklearn.model_selection.KFold` is used. + These splitters are instantiated with `shuffle=False` so the splits + will be the same across calls. Refer :ref:`User Guide ` for the various cross-validation strategies that can be used here. + If "prefit" is passed, it is assumed that all `estimators` have + been fitted already. The `final_estimator_` is trained on the `estimators` + predictions on the full training set and are **not** cross validated + predictions. Please note that if the models have been trained on the same + data to train the stacking model, there is a very high risk of overfitting. + + .. versionadded:: 1.1 + The 'prefit' option was added in 1.1 + .. note:: A larger number of split will provide no benefits if the number of training samples is large enough. Indeed, the training time @@ -583,7 +905,7 @@ class StackingRegressor(RegressorMixin, _BaseStacking): n_jobs : int, default=None The number of jobs to run in parallel for `fit` of all `estimators`. `None` means 1 unless in a `joblib.parallel_backend` context. -1 means - using all processors. See Glossary for more details. + using all processors. See :term:`Glossary ` for more details. passthrough : bool, default=False When False, only the predictions of estimators will be used as @@ -596,17 +918,37 @@ class StackingRegressor(RegressorMixin, _BaseStacking): Attributes ---------- - estimators_ : list of estimator - The elements of the estimators parameter, having been fitted on the + estimators_ : list of estimators + The elements of the `estimators` parameter, having been fitted on the training data. If an estimator has been set to `'drop'`, it - will not appear in `estimators_`. + will not appear in `estimators_`. When `cv="prefit"`, `estimators_` + is set to `estimators` and is not fitted again. named_estimators_ : :class:`~sklearn.utils.Bunch` Attribute to access any fitted sub-estimators by name. + n_features_in_ : int + Number of features seen during :term:`fit`. Only defined if the + underlying estimator exposes such an attribute when fit. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Only defined if the + underlying estimators expose such an attribute when fit. + + .. versionadded:: 1.0 final_estimator_ : estimator - The regressor to stacked the base estimators fitted. + The regressor fit on the output of `estimators_` and responsible for + final predictions. + + stack_method_ : list of str + The method used by each base estimator. + + See Also + -------- + StackingClassifier : Stack of estimators with a final classifier. References ---------- @@ -636,11 +978,18 @@ class StackingRegressor(RegressorMixin, _BaseStacking): ... ) >>> reg.fit(X_train, y_train).score(X_test, y_test) 0.3... - """ - @_deprecate_positional_args - def __init__(self, estimators, final_estimator=None, *, cv=None, - n_jobs=None, passthrough=False, verbose=0): + + def __init__( + self, + estimators, + final_estimator=None, + *, + cv=None, + n_jobs=None, + passthrough=False, + verbose=0, + ): super().__init__( estimators=estimators, final_estimator=final_estimator, @@ -648,40 +997,50 @@ def __init__(self, estimators, final_estimator=None, *, cv=None, stack_method="predict", n_jobs=n_jobs, passthrough=passthrough, - verbose=verbose + verbose=verbose, ) def _validate_final_estimator(self): self._clone_final_estimator(default=RidgeCV()) if not is_regressor(self.final_estimator_): raise ValueError( - "'final_estimator' parameter should be a regressor. Got {}" - .format(self.final_estimator_) + "'final_estimator' parameter should be a regressor. Got {}".format( + self.final_estimator_ + ) ) - def fit(self, X, y, sample_weight=None): + def fit(self, X, y, **fit_params): """Fit the estimators. Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) - Training vectors, where n_samples is the number of samples and - n_features is the number of features. + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. y : array-like of shape (n_samples,) Target values. - sample_weight : array-like of shape (n_samples,), default=None - Sample weights. If None, then samples are equally weighted. - Note that this is supported only if all underlying estimators - support sample weights. + **fit_params : dict + Parameters to pass to the underlying estimators. + + .. versionadded:: 1.6 + + Only available if `enable_metadata_routing=True`, which can be + set by using ``sklearn.set_config(enable_metadata_routing=True)``. + See :ref:`Metadata Routing User Guide ` for + more details. Returns ------- self : object + Returns a fitted instance. """ + _raise_for_params(fit_params, self, "fit", allow=["sample_weight"]) + y = column_or_1d(y, warn=True) - return super().fit(X, y, sample_weight) + + return super().fit(X, y, **fit_params) def transform(self, X): """Return the predictions for X for each estimator. @@ -699,6 +1058,83 @@ def transform(self, X): """ return self._transform(X) + def fit_transform(self, X, y, **fit_params): + """Fit the estimators and return the predictions for X for each estimator. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. + + y : array-like of shape (n_samples,) + Target values. + + **fit_params : dict + Parameters to pass to the underlying estimators. + + .. versionadded:: 1.6 + + Only available if `enable_metadata_routing=True`, which can be + set by using ``sklearn.set_config(enable_metadata_routing=True)``. + See :ref:`Metadata Routing User Guide ` for + more details. + + Returns + ------- + y_preds : ndarray of shape (n_samples, n_estimators) + Prediction outputs for each estimator. + """ + _raise_for_params(fit_params, self, "fit", allow=["sample_weight"]) + + return super().fit_transform(X, y, **fit_params) + + @available_if( + _estimator_has("predict", delegates=("final_estimator_", "final_estimator")) + ) + def predict(self, X, **predict_params): + """Predict target for X. + + Parameters + ---------- + X : {array-like, sparse matrix} of shape (n_samples, n_features) + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. + + **predict_params : dict of str -> obj + Parameters to the `predict` called by the `final_estimator`. Note + that this may be used to return uncertainties from some estimators + with `return_std` or `return_cov`. Be aware that it will only + account for uncertainty in the final estimator. + + - If `enable_metadata_routing=False` (default): + Parameters directly passed to the `predict` method of the + `final_estimator`. + + - If `enable_metadata_routing=True`: Parameters safely routed to + the `predict` method of the `final_estimator`. See :ref:`Metadata + Routing User Guide ` for more details. + + .. versionchanged:: 1.6 + `**predict_params` can be routed via metadata routing API. + + Returns + ------- + y_pred : ndarray of shape (n_samples,) or (n_samples, n_output) + Predicted targets. + """ + if _routing_enabled(): + routed_params = process_routing(self, "predict", **predict_params) + else: + # TODO(SLEP6): remove when metadata routing cannot be disabled. + routed_params = Bunch() + routed_params.final_estimator_ = Bunch(predict={}) + routed_params.final_estimator_.predict = predict_params + + y_pred = super().predict(X, **routed_params.final_estimator_["predict"]) + + return y_pred + def _sk_visual_block_(self): # If final_estimator's default changes then this should be # updated. @@ -706,4 +1142,4 @@ def _sk_visual_block_(self): final_estimator = RidgeCV() else: final_estimator = self.final_estimator - return super()._sk_visual_block_(final_estimator) + return super()._sk_visual_block_with_final_estimator(final_estimator) diff --git a/sklearn/ensemble/_voting.py b/sklearn/ensemble/_voting.py index 2072d5c7c5501..369d3f0f5553e 100644 --- a/sklearn/ensemble/_voting.py +++ b/sklearn/ensemble/_voting.py @@ -6,34 +6,42 @@ - A Voting regressor for regression estimators. """ -# Authors: Sebastian Raschka , -# Gilles Louppe , -# Ramil Nugmanov -# Mohamed Ali Jamaoui -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause from abc import abstractmethod +from numbers import Integral import numpy as np -from joblib import Parallel - -from ..base import ClassifierMixin -from ..base import RegressorMixin -from ..base import TransformerMixin -from ..base import clone -from ._base import _fit_single_estimator -from ._base import _BaseHeterogeneousEnsemble +from ..base import ( + ClassifierMixin, + RegressorMixin, + TransformerMixin, + _fit_context, + clone, +) +from ..exceptions import NotFittedError from ..preprocessing import LabelEncoder from ..utils import Bunch -from ..utils.validation import check_is_fitted -from ..utils.multiclass import check_classification_targets -from ..utils.validation import column_or_1d -from ..utils.validation import _deprecate_positional_args -from ..exceptions import NotFittedError -from ..utils._estimator_html_repr import _VisualBlock -from ..utils.fixes import delayed +from ..utils._param_validation import StrOptions +from ..utils._repr_html.estimator import _VisualBlock +from ..utils.metadata_routing import ( + MetadataRouter, + MethodMapping, + _raise_for_params, + _routing_enabled, + process_routing, +) +from ..utils.metaestimators import available_if +from ..utils.multiclass import type_of_target +from ..utils.parallel import Parallel, delayed +from ..utils.validation import ( + _check_feature_names_in, + check_is_fitted, + column_or_1d, +) +from ._base import _BaseHeterogeneousEnsemble, _fit_single_estimator class _BaseVoting(TransformerMixin, _BaseHeterogeneousEnsemble): @@ -43,53 +51,75 @@ class _BaseVoting(TransformerMixin, _BaseHeterogeneousEnsemble): instead. """ + _parameter_constraints: dict = { + "estimators": [list], + "weights": ["array-like", None], + "n_jobs": [None, Integral], + "verbose": ["verbose"], + } + def _log_message(self, name, idx, total): if not self.verbose: return None - return '(%d of %d) Processing %s' % (idx, total, name) + return f"({idx} of {total}) Processing {name}" @property def _weights_not_none(self): """Get the weights of not `None` estimators.""" if self.weights is None: return None - return [w for est, w in zip(self.estimators, self.weights) - if est[1] != 'drop'] + return [w for est, w in zip(self.estimators, self.weights) if est[1] != "drop"] def _predict(self, X): """Collect results from clf.predict calls.""" return np.asarray([est.predict(X) for est in self.estimators_]).T @abstractmethod - def fit(self, X, y, sample_weight=None): + def fit(self, X, y, **fit_params): """Get common fit operations.""" names, clfs = self._validate_estimators() - if (self.weights is not None and - len(self.weights) != len(self.estimators)): - raise ValueError('Number of `estimators` and weights must be equal' - '; got %d weights, %d estimators' - % (len(self.weights), len(self.estimators))) + if self.weights is not None and len(self.weights) != len(self.estimators): + raise ValueError( + "Number of `estimators` and weights must be equal; got" + f" {len(self.weights)} weights, {len(self.estimators)} estimators" + ) + + if _routing_enabled(): + routed_params = process_routing(self, "fit", **fit_params) + else: + routed_params = Bunch() + for name in names: + routed_params[name] = Bunch(fit={}) + if "sample_weight" in fit_params: + routed_params[name].fit["sample_weight"] = fit_params[ + "sample_weight" + ] self.estimators_ = Parallel(n_jobs=self.n_jobs)( - delayed(_fit_single_estimator)( - clone(clf), X, y, - sample_weight=sample_weight, - message_clsname='Voting', - message=self._log_message(names[idx], - idx + 1, len(clfs)) - ) - for idx, clf in enumerate(clfs) if clf != 'drop' + delayed(_fit_single_estimator)( + clone(clf), + X, + y, + fit_params=routed_params[name]["fit"], + message_clsname="Voting", + message=self._log_message(name, idx + 1, len(clfs)), ) + for idx, (name, clf) in enumerate(zip(names, clfs)) + if clf != "drop" + ) self.named_estimators_ = Bunch() # Uses 'drop' as placeholder for dropped estimators est_iter = iter(self.estimators_) for name, est in self.estimators: - current_est = est if est == 'drop' else next(est_iter) + current_est = est if est == "drop" else next(est_iter) self.named_estimators_[name] = current_est + if hasattr(current_est, "feature_names_in_"): + self.feature_names_in_ = current_est.feature_names_in_ + return self def fit_transform(self, X, y=None, **fit_params): @@ -101,7 +131,7 @@ def fit_transform(self, X, y=None, **fit_params): ---------- X : {array-like, sparse matrix, dataframe} of shape \ (n_samples, n_features) - Input samples + Input samples. y : ndarray of shape (n_samples,), default=None Target values (None for unsupervised transformations). @@ -118,24 +148,47 @@ def fit_transform(self, X, y=None, **fit_params): @property def n_features_in_(self): + """Number of features seen during :term:`fit`.""" # For consistency with other estimators we raise a AttributeError so # that hasattr() fails if the estimator isn't fitted. try: check_is_fitted(self) except NotFittedError as nfe: raise AttributeError( - "{} object has no n_features_in_ attribute." - .format(self.__class__.__name__) + "{} object has no n_features_in_ attribute.".format( + self.__class__.__name__ + ) ) from nfe return self.estimators_[0].n_features_in_ def _sk_visual_block_(self): names, estimators = zip(*self.estimators) - return _VisualBlock('parallel', estimators, names=names) + return _VisualBlock("parallel", estimators, names=names) + + def get_metadata_routing(self): + """Get metadata routing of this object. - def _more_tags(self): - return {"preserves_dtype": []} + Please check :ref:`User Guide ` on how the routing + mechanism works. + + .. versionadded:: 1.5 + + Returns + ------- + routing : MetadataRouter + A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating + routing information. + """ + router = MetadataRouter(owner=self.__class__.__name__) + + # `self.estimators` is a list of (name, est) tuples + for name, estimator in self.estimators: + router.add( + **{name: estimator}, + method_mapping=MethodMapping().add(callee="fit", caller="fit"), + ) + return router class VotingClassifier(ClassifierMixin, _BaseVoting): @@ -150,8 +203,8 @@ class VotingClassifier(ClassifierMixin, _BaseVoting): estimators : list of (str, estimator) tuples Invoking the ``fit`` method on the ``VotingClassifier`` will fit clones of those original estimators that will be stored in the class attribute - ``self.estimators_``. An estimator can be set to ``'drop'`` - using ``set_params``. + ``self.estimators_``. An estimator can be set to ``'drop'`` using + :meth:`set_params`. .. versionchanged:: 0.21 ``'drop'`` is accepted. Using None was deprecated in 0.22 and @@ -200,9 +253,25 @@ class VotingClassifier(ClassifierMixin, _BaseVoting): .. versionadded:: 0.20 - classes_ : array-like of shape (n_predictions,) + le_ : :class:`~sklearn.preprocessing.LabelEncoder` + Transformer used to encode the labels during fit and decode during + prediction. + + classes_ : ndarray of shape (n_classes,) The classes labels. + n_features_in_ : int + Number of features seen during :term:`fit`. Only defined if the + underlying classifier exposes such an attribute when fit. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Only defined if the + underlying estimators expose such an attribute when fit. + + .. versionadded:: 1.0 + See Also -------- VotingRegressor : Prediction voting regressor. @@ -213,7 +282,7 @@ class VotingClassifier(ClassifierMixin, _BaseVoting): >>> from sklearn.linear_model import LogisticRegression >>> from sklearn.naive_bayes import GaussianNB >>> from sklearn.ensemble import RandomForestClassifier, VotingClassifier - >>> clf1 = LogisticRegression(multi_class='multinomial', random_state=1) + >>> clf1 = LogisticRegression(random_state=1) >>> clf2 = RandomForestClassifier(n_estimators=50, random_state=1) >>> clf3 = GaussianNB() >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]]) @@ -232,6 +301,18 @@ class VotingClassifier(ClassifierMixin, _BaseVoting): >>> eclf2 = eclf2.fit(X, y) >>> print(eclf2.predict(X)) [1 1 1 2 2 2] + + To drop an estimator, :meth:`set_params` can be used to remove it. Here we + dropped one of the estimators, resulting in 2 fitted estimators: + + >>> eclf2 = eclf2.set_params(lr='drop') + >>> eclf2 = eclf2.fit(X, y) + >>> len(eclf2.estimators_) + 2 + + Setting `flatten_transform=True` with `voting='soft'` flattens output shape of + `transform`: + >>> eclf3 = VotingClassifier(estimators=[ ... ('lr', clf1), ('rf', clf2), ('gnb', clf3)], ... voting='soft', weights=[2,1,1], @@ -242,9 +323,23 @@ class VotingClassifier(ClassifierMixin, _BaseVoting): >>> print(eclf3.transform(X).shape) (6, 6) """ - @_deprecate_positional_args - def __init__(self, estimators, *, voting='hard', weights=None, - n_jobs=None, flatten_transform=True, verbose=False): + + _parameter_constraints: dict = { + **_BaseVoting._parameter_constraints, + "voting": [StrOptions({"hard", "soft"})], + "flatten_transform": ["boolean"], + } + + def __init__( + self, + estimators, + *, + voting="hard", + weights=None, + n_jobs=None, + flatten_transform=True, + verbose=False, + ): super().__init__(estimators=estimators) self.voting = voting self.weights = weights @@ -252,44 +347,62 @@ def __init__(self, estimators, *, voting='hard', weights=None, self.flatten_transform = flatten_transform self.verbose = verbose - def fit(self, X, y, sample_weight=None): + @_fit_context( + # estimators in VotingClassifier.estimators are not validated yet + prefer_skip_nested_validation=False + ) + def fit(self, X, y, **fit_params): """Fit the estimators. Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) - Training vectors, where n_samples is the number of samples and - n_features is the number of features. + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. y : array-like of shape (n_samples,) Target values. - sample_weight : array-like of shape (n_samples,), default=None - Sample weights. If None, then samples are equally weighted. - Note that this is supported only if all underlying estimators - support sample weights. + **fit_params : dict + Parameters to pass to the underlying estimators. + + .. versionadded:: 1.5 - .. versionadded:: 0.18 + Only available if `enable_metadata_routing=True`, + which can be set by using + ``sklearn.set_config(enable_metadata_routing=True)``. + See :ref:`Metadata Routing User Guide ` for + more details. Returns ------- self : object - + Returns the instance itself. """ - check_classification_targets(y) - if isinstance(y, np.ndarray) and len(y.shape) > 1 and y.shape[1] > 1: - raise NotImplementedError('Multilabel and multi-output' - ' classification is not supported.') - - if self.voting not in ('soft', 'hard'): - raise ValueError("Voting must be 'soft' or 'hard'; got (voting=%r)" - % self.voting) + _raise_for_params(fit_params, self, "fit", allow=["sample_weight"]) + + y_type = type_of_target(y, input_name="y") + if y_type in ("unknown", "continuous"): + # raise a specific ValueError for non-classification tasks + raise ValueError( + f"Unknown label type: {y_type}. Maybe you are trying to fit a " + "classifier, which expects discrete classes on a " + "regression target with continuous values." + ) + elif y_type not in ("binary", "multiclass"): + # raise a NotImplementedError for backward compatibility for non-supported + # classification tasks + raise NotImplementedError( + f"{self.__class__.__name__} only supports binary or multiclass " + "classification. Multilabel and multi-output classification are not " + "supported." + ) self.le_ = LabelEncoder().fit(y) self.classes_ = self.le_.classes_ transformed_y = self.le_.transform(y) - return super().fit(X, transformed_y, sample_weight) + return super().fit(X, transformed_y, **fit_params) def predict(self, X): """Predict class labels for X. @@ -305,15 +418,16 @@ def predict(self, X): Predicted class labels. """ check_is_fitted(self) - if self.voting == 'soft': + if self.voting == "soft": maj = np.argmax(self.predict_proba(X), axis=1) else: # 'hard' voting predictions = self._predict(X) maj = np.apply_along_axis( - lambda x: np.argmax( - np.bincount(x, weights=self._weights_not_none)), - axis=1, arr=predictions) + lambda x: np.argmax(np.bincount(x, weights=self._weights_not_none)), + axis=1, + arr=predictions, + ) maj = self.le_.inverse_transform(maj) @@ -323,15 +437,15 @@ def _collect_probas(self, X): """Collect results from clf.predict calls.""" return np.asarray([clf.predict_proba(X) for clf in self.estimators_]) - def _predict_proba(self, X): - """Predict class probabilities for X in 'soft' voting.""" - check_is_fitted(self) - avg = np.average(self._collect_probas(X), axis=0, - weights=self._weights_not_none) - return avg + def _check_voting(self): + if self.voting == "hard": + raise AttributeError( + f"predict_proba is not available when voting={self.voting!r}" + ) + return True - @property - def predict_proba(self): + @available_if(_check_voting) + def predict_proba(self, X): """Compute probabilities of possible outcomes for samples in X. Parameters @@ -344,10 +458,11 @@ def predict_proba(self): avg : array-like of shape (n_samples, n_classes) Weighted average probability for each class per sample. """ - if self.voting == 'hard': - raise AttributeError("predict_proba is not available when" - " voting=%r" % self.voting) - return self._predict_proba + check_is_fitted(self) + avg = np.average( + self._collect_probas(X), axis=0, weights=self._weights_not_none + ) + return avg def transform(self, X): """Return class labels or probabilities for X for each estimator. @@ -355,16 +470,15 @@ def transform(self, X): Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) - Training vectors, where n_samples is the number of samples and - n_features is the number of features. + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. Returns ------- probabilities_or_labels If `voting='soft'` and `flatten_transform=True`: - returns ndarray of shape (n_classifiers, n_samples * - n_classes), being class probabilities calculated by each - classifier. + returns ndarray of shape (n_samples, n_classifiers * n_classes), + being class probabilities calculated by each classifier. If `voting='soft' and `flatten_transform=False`: ndarray of shape (n_classifiers, n_samples, n_classes) If `voting='hard'`: @@ -373,7 +487,7 @@ class labels predicted by each classifier. """ check_is_fitted(self) - if self.voting == 'soft': + if self.voting == "soft": probas = self._collect_probas(X) if not self.flatten_transform: return probas @@ -382,6 +496,48 @@ class labels predicted by each classifier. else: return self._predict(X) + def get_feature_names_out(self, input_features=None): + """Get output feature names for transformation. + + Parameters + ---------- + input_features : array-like of str or None, default=None + Not used, present here for API consistency by convention. + + Returns + ------- + feature_names_out : ndarray of str objects + Transformed feature names. + """ + check_is_fitted(self, "n_features_in_") + if self.voting == "soft" and not self.flatten_transform: + raise ValueError( + "get_feature_names_out is not supported when `voting='soft'` and " + "`flatten_transform=False`" + ) + + _check_feature_names_in(self, input_features, generate_names=False) + class_name = self.__class__.__name__.lower() + + active_names = [name for name, est in self.estimators if est != "drop"] + + if self.voting == "hard": + return np.asarray( + [f"{class_name}_{name}" for name in active_names], dtype=object + ) + + # voting == "soft" + n_classes = len(self.classes_) + names_out = [ + f"{class_name}_{name}{i}" for name in active_names for i in range(n_classes) + ] + return np.asarray(names_out, dtype=object) + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.transformer_tags.preserves_dtype = [] + return tags + class VotingRegressor(RegressorMixin, _BaseVoting): """Prediction voting regressor for unfitted estimators. @@ -390,6 +546,9 @@ class VotingRegressor(RegressorMixin, _BaseVoting): regressors, each on the whole dataset. Then it averages the individual predictions to form a final prediction. + For a detailed example, refer to + :ref:`sphx_glr_auto_examples_ensemble_plot_voting_regressor.py`. + Read more in the :ref:`User Guide `. .. versionadded:: 0.21 @@ -400,7 +559,7 @@ class VotingRegressor(RegressorMixin, _BaseVoting): Invoking the ``fit`` method on the ``VotingRegressor`` will fit clones of those original estimators that will be stored in the class attribute ``self.estimators_``. An estimator can be set to ``'drop'`` using - ``set_params``. + :meth:`set_params`. .. versionchanged:: 0.21 ``'drop'`` is accepted. Using None was deprecated in 0.22 and @@ -428,11 +587,23 @@ class VotingRegressor(RegressorMixin, _BaseVoting): The collection of fitted sub-estimators as defined in ``estimators`` that are not 'drop'. - named_estimators_ : Bunch + named_estimators_ : :class:`~sklearn.utils.Bunch` Attribute to access any fitted sub-estimators by name. .. versionadded:: 0.20 + n_features_in_ : int + Number of features seen during :term:`fit`. Only defined if the + underlying regressor exposes such an attribute when fit. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Only defined if the + underlying estimators expose such an attribute when fit. + + .. versionadded:: 1.0 + See Also -------- VotingClassifier : Soft Voting/Majority Rule classifier. @@ -443,46 +614,68 @@ class VotingRegressor(RegressorMixin, _BaseVoting): >>> from sklearn.linear_model import LinearRegression >>> from sklearn.ensemble import RandomForestRegressor >>> from sklearn.ensemble import VotingRegressor + >>> from sklearn.neighbors import KNeighborsRegressor >>> r1 = LinearRegression() >>> r2 = RandomForestRegressor(n_estimators=10, random_state=1) + >>> r3 = KNeighborsRegressor() >>> X = np.array([[1, 1], [2, 4], [3, 9], [4, 16], [5, 25], [6, 36]]) >>> y = np.array([2, 6, 12, 20, 30, 42]) - >>> er = VotingRegressor([('lr', r1), ('rf', r2)]) + >>> er = VotingRegressor([('lr', r1), ('rf', r2), ('r3', r3)]) >>> print(er.fit(X, y).predict(X)) - [ 3.3 5.7 11.8 19.7 28. 40.3] + [ 6.8 8.4 12.5 17.8 26 34] + + In the following example, we drop the `'lr'` estimator with + :meth:`~VotingRegressor.set_params` and fit the remaining two estimators: + + >>> er = er.set_params(lr='drop') + >>> er = er.fit(X, y) + >>> len(er.estimators_) + 2 """ - @_deprecate_positional_args - def __init__(self, estimators, *, weights=None, n_jobs=None, - verbose=False): + + def __init__(self, estimators, *, weights=None, n_jobs=None, verbose=False): super().__init__(estimators=estimators) self.weights = weights self.n_jobs = n_jobs self.verbose = verbose - def fit(self, X, y, sample_weight=None): + @_fit_context( + # estimators in VotingRegressor.estimators are not validated yet + prefer_skip_nested_validation=False + ) + def fit(self, X, y, **fit_params): """Fit the estimators. Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) - Training vectors, where n_samples is the number of samples and - n_features is the number of features. + Training vectors, where `n_samples` is the number of samples and + `n_features` is the number of features. y : array-like of shape (n_samples,) Target values. - sample_weight : array-like of shape (n_samples,), default=None - Sample weights. If None, then samples are equally weighted. - Note that this is supported only if all underlying estimators - support sample weights. + **fit_params : dict + Parameters to pass to the underlying estimators. + + .. versionadded:: 1.5 + + Only available if `enable_metadata_routing=True`, + which can be set by using + ``sklearn.set_config(enable_metadata_routing=True)``. + See :ref:`Metadata Routing User Guide ` for + more details. Returns ------- self : object Fitted estimator. """ + _raise_for_params(fit_params, self, "fit", allow=["sample_weight"]) + y = column_or_1d(y, warn=True) - return super().fit(X, y, sample_weight) + + return super().fit(X, y, **fit_params) def predict(self, X): """Predict regression target for X. @@ -501,8 +694,7 @@ def predict(self, X): The predicted values. """ check_is_fitted(self) - return np.average(self._predict(X), axis=1, - weights=self._weights_not_none) + return np.average(self._predict(X), axis=1, weights=self._weights_not_none) def transform(self, X): """Return predictions for X for each estimator. @@ -514,8 +706,29 @@ def transform(self, X): Returns ------- - predictions: ndarray of shape (n_samples, n_classifiers) + predictions : ndarray of shape (n_samples, n_classifiers) Values predicted by each regressor. """ check_is_fitted(self) return self._predict(X) + + def get_feature_names_out(self, input_features=None): + """Get output feature names for transformation. + + Parameters + ---------- + input_features : array-like of str or None, default=None + Not used, present here for API consistency by convention. + + Returns + ------- + feature_names_out : ndarray of str objects + Transformed feature names. + """ + check_is_fitted(self, "n_features_in_") + _check_feature_names_in(self, input_features, generate_names=False) + class_name = self.__class__.__name__.lower() + return np.asarray( + [f"{class_name}_{name}" for name, est in self.estimators if est != "drop"], + dtype=object, + ) diff --git a/sklearn/ensemble/_weight_boosting.py b/sklearn/ensemble/_weight_boosting.py index d5354232a4385..37c6468a5ebf6 100644 --- a/sklearn/ensemble/_weight_boosting.py +++ b/sklearn/ensemble/_weight_boosting.py @@ -16,36 +16,43 @@ (AdaBoost.R2) for regression problems. """ -# Authors: Noel Dawe -# Gilles Louppe -# Hamzeh Alsalhi -# Arnaud Joly -# -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause +import warnings from abc import ABCMeta, abstractmethod +from numbers import Integral, Real import numpy as np -from scipy.special import xlogy - -from ._base import BaseEnsemble -from ..base import ClassifierMixin, RegressorMixin, is_classifier, is_regressor - -from ..tree import DecisionTreeClassifier, DecisionTreeRegressor -from ..utils import check_random_state, _safe_indexing -from ..utils.extmath import softmax -from ..utils.extmath import stable_cumsum +from ..base import ( + ClassifierMixin, + RegressorMixin, + _fit_context, + is_classifier, + is_regressor, +) from ..metrics import accuracy_score, r2_score -from ..utils.validation import check_is_fitted -from ..utils.validation import _check_sample_weight -from ..utils.validation import has_fit_parameter -from ..utils.validation import _num_samples -from ..utils.validation import _deprecate_positional_args +from ..tree import DecisionTreeClassifier, DecisionTreeRegressor +from ..utils import _safe_indexing, check_random_state +from ..utils._param_validation import HasMethods, Hidden, Interval, StrOptions +from ..utils.extmath import softmax, stable_cumsum +from ..utils.metadata_routing import ( + _raise_for_unsupported_routing, + _RoutingNotSupportedMixin, +) +from ..utils.validation import ( + _check_sample_weight, + _num_samples, + check_is_fitted, + has_fit_parameter, + validate_data, +) +from ._base import BaseEnsemble __all__ = [ - 'AdaBoostClassifier', - 'AdaBoostRegressor', + "AdaBoostClassifier", + "AdaBoostRegressor", ] @@ -56,28 +63,48 @@ class BaseWeightBoosting(BaseEnsemble, metaclass=ABCMeta): instead. """ - @abstractmethod - def __init__(self, - base_estimator=None, *, - n_estimators=50, - estimator_params=tuple(), - learning_rate=1., - random_state=None): + _parameter_constraints: dict = { + "estimator": [HasMethods(["fit", "predict"]), None], + "n_estimators": [Interval(Integral, 1, None, closed="left")], + "learning_rate": [Interval(Real, 0, None, closed="neither")], + "random_state": ["random_state"], + } + @abstractmethod + def __init__( + self, + estimator=None, + *, + n_estimators=50, + estimator_params=tuple(), + learning_rate=1.0, + random_state=None, + ): super().__init__( - base_estimator=base_estimator, + estimator=estimator, n_estimators=n_estimators, - estimator_params=estimator_params) + estimator_params=estimator_params, + ) self.learning_rate = learning_rate self.random_state = random_state def _check_X(self, X): # Only called to validate X in non-fit methods, therefore reset=False - return self._validate_data( - X, accept_sparse=['csr', 'csc'], ensure_2d=True, allow_nd=True, - dtype=None, reset=False) + return validate_data( + self, + X, + accept_sparse=["csr", "csc"], + ensure_2d=True, + allow_nd=True, + dtype=None, + reset=False, + ) + @_fit_context( + # AdaBoost*.estimator is not validated yet + prefer_skip_nested_validation=False + ) def fit(self, X, y, sample_weight=None): """Build a boosted classifier/regressor from the training set (X, y). @@ -88,8 +115,7 @@ def fit(self, X, y, sample_weight=None): DOK, or LIL. COO, DOK, and LIL are converted to CSR. y : array-like of shape (n_samples,) - The target values (class labels in classification, real numbers in - regression). + The target values. sample_weight : array-like of shape (n_samples,), default=None Sample weights. If None, the sample weights are initialized to @@ -98,22 +124,24 @@ def fit(self, X, y, sample_weight=None): Returns ------- self : object + Fitted estimator. """ - # Check parameters - if self.learning_rate <= 0: - raise ValueError("learning_rate must be greater than zero") - - X, y = self._validate_data(X, y, - accept_sparse=['csr', 'csc'], - ensure_2d=True, - allow_nd=True, - dtype=None, - y_numeric=is_regressor(self)) + _raise_for_unsupported_routing(self, "fit", sample_weight=sample_weight) + X, y = validate_data( + self, + X, + y, + accept_sparse=["csr", "csc"], + ensure_2d=True, + allow_nd=True, + dtype=None, + y_numeric=is_regressor(self), + ) - sample_weight = _check_sample_weight(sample_weight, X, np.float64) + sample_weight = _check_sample_weight( + sample_weight, X, dtype=np.float64, copy=True, ensure_non_negative=True + ) sample_weight /= sample_weight.sum() - if np.any(sample_weight < 0): - raise ValueError("sample_weight cannot contain negative weights") # Check parameters self._validate_estimator() @@ -123,22 +151,26 @@ def fit(self, X, y, sample_weight=None): self.estimator_weights_ = np.zeros(self.n_estimators, dtype=np.float64) self.estimator_errors_ = np.ones(self.n_estimators, dtype=np.float64) - # Initializion of the random number instance that will be used to + # Initialization of the random number instance that will be used to # generate a seed at each iteration random_state = check_random_state(self.random_state) + epsilon = np.finfo(sample_weight.dtype).eps + zero_weight_mask = sample_weight == 0.0 for iboost in range(self.n_estimators): + # avoid extremely small sample weight, for details see issue #20320 + sample_weight = np.clip(sample_weight, a_min=epsilon, a_max=None) + # do not clip sample weights that were exactly zero originally + sample_weight[zero_weight_mask] = 0.0 + # Boosting step sample_weight, estimator_weight, estimator_error = self._boost( - iboost, - X, y, - sample_weight, - random_state) + iboost, X, y, sample_weight, random_state + ) # Early termination if sample_weight is None: break - self.estimator_weights_[iboost] = estimator_weight self.estimator_errors_[iboost] = estimator_error @@ -148,6 +180,17 @@ def fit(self, X, y, sample_weight=None): sample_weight_sum = np.sum(sample_weight) + if not np.isfinite(sample_weight_sum): + warnings.warn( + ( + "Sample weights have reached infinite values," + f" at iteration {iboost}, causing overflow. " + "Iterations stopped. Try lowering the learning rate." + ), + stacklevel=2, + ) + break + # Stop if the sum of sample weights has become non-positive if sample_weight_sum <= 0: break @@ -248,20 +291,31 @@ def feature_importances_(self): The feature importances. """ if self.estimators_ is None or len(self.estimators_) == 0: - raise ValueError("Estimator not fitted, " - "call `fit` before `feature_importances_`.") + raise ValueError( + "Estimator not fitted, call `fit` before `feature_importances_`." + ) try: norm = self.estimator_weights_.sum() - return (sum(weight * clf.feature_importances_ for weight, clf - in zip(self.estimator_weights_, self.estimators_)) - / norm) + return ( + sum( + weight * clf.feature_importances_ + for weight, clf in zip(self.estimator_weights_, self.estimators_) + ) + / norm + ) except AttributeError as e: raise AttributeError( "Unable to compute feature importances " - "since base_estimator does not have a " - "feature_importances_ attribute") from e + "since estimator does not have a " + "feature_importances_ attribute" + ) from e + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.sparse = True + return tags def _samme_proba(estimator, n_classes, X): @@ -280,20 +334,23 @@ def _samme_proba(estimator, n_classes, X): np.clip(proba, np.finfo(proba.dtype).eps, None, out=proba) log_proba = np.log(proba) - return (n_classes - 1) * (log_proba - (1. / n_classes) - * log_proba.sum(axis=1)[:, np.newaxis]) + return (n_classes - 1) * ( + log_proba - (1.0 / n_classes) * log_proba.sum(axis=1)[:, np.newaxis] + ) -class AdaBoostClassifier(ClassifierMixin, BaseWeightBoosting): +class AdaBoostClassifier( + _RoutingNotSupportedMixin, ClassifierMixin, BaseWeightBoosting +): """An AdaBoost classifier. - An AdaBoost [1] classifier is a meta-estimator that begins by fitting a + An AdaBoost [1]_ classifier is a meta-estimator that begins by fitting a classifier on the original dataset and then fits additional copies of the classifier on the same dataset but where the weights of incorrectly classified instances are adjusted such that subsequent classifiers focus more on difficult cases. - This class implements the algorithm known as AdaBoost-SAMME [2]. + This class implements the algorithm based on [2]_. Read more in the :ref:`User Guide `. @@ -301,41 +358,49 @@ class AdaBoostClassifier(ClassifierMixin, BaseWeightBoosting): Parameters ---------- - base_estimator : object, default=None + estimator : object, default=None The base estimator from which the boosted ensemble is built. Support for sample weighting is required, as well as proper ``classes_`` and ``n_classes_`` attributes. If ``None``, then the base estimator is :class:`~sklearn.tree.DecisionTreeClassifier` initialized with `max_depth=1`. + .. versionadded:: 1.2 + `base_estimator` was renamed to `estimator`. + n_estimators : int, default=50 The maximum number of estimators at which boosting is terminated. In case of perfect fit, the learning procedure is stopped early. + Values must be in the range `[1, inf)`. + + learning_rate : float, default=1.0 + Weight applied to each classifier at each boosting iteration. A higher + learning rate increases the contribution of each classifier. There is + a trade-off between the `learning_rate` and `n_estimators` parameters. + Values must be in the range `(0.0, inf)`. - learning_rate : float, default=1. - Learning rate shrinks the contribution of each classifier by - ``learning_rate``. There is a trade-off between ``learning_rate`` and - ``n_estimators``. + algorithm : {'SAMME'}, default='SAMME' + Use the SAMME discrete boosting algorithm. - algorithm : {'SAMME', 'SAMME.R'}, default='SAMME.R' - If 'SAMME.R' then use the SAMME.R real boosting algorithm. - ``base_estimator`` must support calculation of class probabilities. - If 'SAMME' then use the SAMME discrete boosting algorithm. - The SAMME.R algorithm typically converges faster than SAMME, - achieving a lower test error with fewer boosting iterations. + .. deprecated:: 1.6 + `algorithm` is deprecated and will be removed in version 1.8. This + estimator only implements the 'SAMME' algorithm. random_state : int, RandomState instance or None, default=None - Controls the random seed given at each `base_estimator` at each + Controls the random seed given at each `estimator` at each boosting iteration. - Thus, it is only used when `base_estimator` exposes a `random_state`. + Thus, it is only used when `estimator` exposes a `random_state`. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. Attributes ---------- - base_estimator_ : estimator + estimator_ : estimator The base estimator from which the ensemble is grown. + .. versionadded:: 1.2 + `base_estimator_` was renamed to `estimator_`. + estimators_ : list of classifiers The collection of fitted sub-estimators. @@ -354,12 +419,23 @@ class AdaBoostClassifier(ClassifierMixin, BaseWeightBoosting): feature_importances_ : ndarray of shape (n_features,) The impurity-based feature importances if supported by the - ``base_estimator`` (when based on decision trees). + ``estimator`` (when based on decision trees). Warning: impurity-based feature importances can be misleading for high cardinality features (many unique values). See :func:`sklearn.inspection.permutation_importance` as an alternative. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + See Also -------- AdaBoostRegressor : An AdaBoost regressor that begins by fitting a @@ -383,7 +459,9 @@ class AdaBoostClassifier(ClassifierMixin, BaseWeightBoosting): .. [1] Y. Freund, R. Schapire, "A Decision-Theoretic Generalization of on-Line Learning and an Application to Boosting", 1995. - .. [2] J. Zhu, H. Zou, S. Rosset, T. Hastie, "Multi-class AdaBoost", 2009. + .. [2] :doi:`J. Zhu, H. Zou, S. Rosset, T. Hastie, "Multi-class adaboost." + Statistics and its Interface 2.3 (2009): 349-360. + <10.4310/SII.2009.v2.n3.a8>` Examples -------- @@ -398,76 +476,62 @@ class AdaBoostClassifier(ClassifierMixin, BaseWeightBoosting): >>> clf.predict([[0, 0, 0, 0]]) array([1]) >>> clf.score(X, y) - 0.983... + 0.96 + + For a detailed example of using AdaBoost to fit a sequence of DecisionTrees + as weaklearners, please refer to + :ref:`sphx_glr_auto_examples_ensemble_plot_adaboost_multiclass.py`. + + For a detailed example of using AdaBoost to fit a non-linearly separable + classification dataset composed of two Gaussian quantiles clusters, please + refer to :ref:`sphx_glr_auto_examples_ensemble_plot_adaboost_twoclass.py`. """ - @_deprecate_positional_args - def __init__(self, - base_estimator=None, *, - n_estimators=50, - learning_rate=1., - algorithm='SAMME.R', - random_state=None): + # TODO(1.8): remove "algorithm" entry + _parameter_constraints: dict = { + **BaseWeightBoosting._parameter_constraints, + "algorithm": [StrOptions({"SAMME"}), Hidden(StrOptions({"deprecated"}))], + } + + def __init__( + self, + estimator=None, + *, + n_estimators=50, + learning_rate=1.0, + algorithm="deprecated", + random_state=None, + ): super().__init__( - base_estimator=base_estimator, + estimator=estimator, n_estimators=n_estimators, learning_rate=learning_rate, - random_state=random_state) + random_state=random_state, + ) self.algorithm = algorithm - def fit(self, X, y, sample_weight=None): - """Build a boosted classifier from the training set (X, y). - - Parameters - ---------- - X : {array-like, sparse matrix} of shape (n_samples, n_features) - The training input samples. Sparse matrix can be CSC, CSR, COO, - DOK, or LIL. COO, DOK, and LIL are converted to CSR. - - y : array-like of shape (n_samples,) - The target values (class labels). - - sample_weight : array-like of shape (n_samples,), default=None - Sample weights. If None, the sample weights are initialized to - ``1 / n_samples``. - - Returns - ------- - self : object - Fitted estimator. - """ - # Check that algorithm is supported - if self.algorithm not in ('SAMME', 'SAMME.R'): - raise ValueError("algorithm %s is not supported" % self.algorithm) - - # Fit - return super().fit(X, y, sample_weight) - def _validate_estimator(self): - """Check the estimator and set the base_estimator_ attribute.""" - super()._validate_estimator( - default=DecisionTreeClassifier(max_depth=1)) - - # SAMME-R requires predict_proba-enabled base estimators - if self.algorithm == 'SAMME.R': - if not hasattr(self.base_estimator_, 'predict_proba'): - raise TypeError( - "AdaBoostClassifier with algorithm='SAMME.R' requires " - "that the weak learner supports the calculation of class " - "probabilities with a predict_proba method.\n" - "Please change the base estimator or set " - "algorithm='SAMME' instead.") - if not has_fit_parameter(self.base_estimator_, "sample_weight"): - raise ValueError("%s doesn't support sample_weight." - % self.base_estimator_.__class__.__name__) + """Check the estimator and set the estimator_ attribute.""" + super()._validate_estimator(default=DecisionTreeClassifier(max_depth=1)) + + if self.algorithm != "deprecated": + warnings.warn( + "The parameter 'algorithm' is deprecated in 1.6 and has no effect. " + "It will be removed in version 1.8.", + FutureWarning, + ) + + if not has_fit_parameter(self.estimator_, "sample_weight"): + raise ValueError( + f"{self.estimator.__class__.__name__} doesn't support sample_weight." + ) def _boost(self, iboost, X, y, sample_weight, random_state): """Implement a single boost. - Perform a single boost according to the real multi-class SAMME.R - algorithm or to the discrete SAMME algorithm and return the updated - sample weights. + Perform a single boost according to the discrete SAMME algorithm and return the + updated sample weights. Parameters ---------- @@ -501,73 +565,6 @@ def _boost(self, iboost, X, y, sample_weight, random_state): The classification error for the current boost. If None then boosting has terminated early. """ - if self.algorithm == 'SAMME.R': - return self._boost_real(iboost, X, y, sample_weight, random_state) - - else: # elif self.algorithm == "SAMME": - return self._boost_discrete(iboost, X, y, sample_weight, - random_state) - - def _boost_real(self, iboost, X, y, sample_weight, random_state): - """Implement a single boost using the SAMME.R real algorithm.""" - estimator = self._make_estimator(random_state=random_state) - - estimator.fit(X, y, sample_weight=sample_weight) - - y_predict_proba = estimator.predict_proba(X) - - if iboost == 0: - self.classes_ = getattr(estimator, 'classes_', None) - self.n_classes_ = len(self.classes_) - - y_predict = self.classes_.take(np.argmax(y_predict_proba, axis=1), - axis=0) - - # Instances incorrectly classified - incorrect = y_predict != y - - # Error fraction - estimator_error = np.mean( - np.average(incorrect, weights=sample_weight, axis=0)) - - # Stop if classification is perfect - if estimator_error <= 0: - return sample_weight, 1., 0. - - # Construct y coding as described in Zhu et al [2]: - # - # y_k = 1 if c == k else -1 / (K - 1) - # - # where K == n_classes_ and c, k in [0, K) are indices along the second - # axis of the y coding with c being the index corresponding to the true - # class label. - n_classes = self.n_classes_ - classes = self.classes_ - y_codes = np.array([-1. / (n_classes - 1), 1.]) - y_coding = y_codes.take(classes == y[:, np.newaxis]) - - # Displace zero probabilities so the log is defined. - # Also fix negative elements which may occur with - # negative sample weights. - proba = y_predict_proba # alias for readability - np.clip(proba, np.finfo(proba.dtype).eps, None, out=proba) - - # Boost weight using multi-class AdaBoost SAMME.R alg - estimator_weight = (-1. * self.learning_rate - * ((n_classes - 1.) / n_classes) - * xlogy(y_coding, y_predict_proba).sum(axis=1)) - - # Only boost the weights if it will fit again - if not iboost == self.n_estimators - 1: - # Only boost positive weights - sample_weight *= np.exp(estimator_weight * - ((sample_weight > 0) | - (estimator_weight < 0))) - - return sample_weight, 1., estimator_error - - def _boost_discrete(self, iboost, X, y, sample_weight, random_state): - """Implement a single boost using the SAMME discrete algorithm.""" estimator = self._make_estimator(random_state=random_state) estimator.fit(X, y, sample_weight=sample_weight) @@ -575,41 +572,44 @@ def _boost_discrete(self, iboost, X, y, sample_weight, random_state): y_predict = estimator.predict(X) if iboost == 0: - self.classes_ = getattr(estimator, 'classes_', None) + self.classes_ = getattr(estimator, "classes_", None) self.n_classes_ = len(self.classes_) # Instances incorrectly classified incorrect = y_predict != y # Error fraction - estimator_error = np.mean( - np.average(incorrect, weights=sample_weight, axis=0)) + estimator_error = np.mean(np.average(incorrect, weights=sample_weight, axis=0)) # Stop if classification is perfect if estimator_error <= 0: - return sample_weight, 1., 0. + return sample_weight, 1.0, 0.0 n_classes = self.n_classes_ # Stop if the error is at least as bad as random guessing - if estimator_error >= 1. - (1. / n_classes): + if estimator_error >= 1.0 - (1.0 / n_classes): self.estimators_.pop(-1) if len(self.estimators_) == 0: - raise ValueError('BaseClassifier in AdaBoostClassifier ' - 'ensemble is worse than random, ensemble ' - 'can not be fit.') + raise ValueError( + "BaseClassifier in AdaBoostClassifier " + "ensemble is worse than random, ensemble " + "can not be fit." + ) return None, None, None # Boost weight using multi-class AdaBoost SAMME alg estimator_weight = self.learning_rate * ( - np.log((1. - estimator_error) / estimator_error) + - np.log(n_classes - 1.)) + np.log((1.0 - estimator_error) / estimator_error) + np.log(n_classes - 1.0) + ) - # Only boost the weights if I will fit again + # Only boost the weights if it will fit again if not iboost == self.n_estimators - 1: # Only boost positive weights - sample_weight *= np.exp(estimator_weight * incorrect * - (sample_weight > 0)) + sample_weight = np.exp( + np.log(sample_weight) + + estimator_weight * incorrect * (sample_weight > 0) + ) return sample_weight, estimator_weight, estimator_error @@ -630,8 +630,6 @@ def predict(self, X): y : ndarray of shape (n_samples,) The predicted classes. """ - X = self._check_X(X) - pred = self.decision_function(X) if self.n_classes_ == 2: @@ -671,8 +669,7 @@ def staged_predict(self, X): else: for pred in self.staged_decision_function(X): - yield np.array(classes.take( - np.argmax(pred, axis=1), axis=0)) + yield np.array(classes.take(np.argmax(pred, axis=1), axis=0)) def decision_function(self, X): """Compute the decision function of ``X``. @@ -687,7 +684,7 @@ def decision_function(self, X): ------- score : ndarray of shape of (n_samples, k) The decision function of the input samples. The order of - outputs is the same of that of the :term:`classes_` attribute. + outputs is the same as that of the :term:`classes_` attribute. Binary classification is a special cases with ``k == 1``, otherwise ``k==n_classes``. For binary classification, values closer to -1 or 1 mean more like the first or second @@ -699,14 +696,17 @@ class in ``classes_``, respectively. n_classes = self.n_classes_ classes = self.classes_[:, np.newaxis] - if self.algorithm == 'SAMME.R': - # The weights are all 1. for SAMME.R - pred = sum(_samme_proba(estimator, n_classes, X) - for estimator in self.estimators_) - else: # self.algorithm == "SAMME" - pred = sum((estimator.predict(X) == classes).T * w - for estimator, w in zip(self.estimators_, - self.estimator_weights_)) + if n_classes == 1: + return np.zeros_like(X, shape=(X.shape[0], 1)) + + pred = sum( + np.where( + (estimator.predict(X) == classes).T, + w, + -1 / (n_classes - 1) * w, + ) + for estimator, w in zip(self.estimators_, self.estimator_weights_) + ) pred /= self.estimator_weights_.sum() if n_classes == 2: @@ -742,18 +742,16 @@ class in ``classes_``, respectively. n_classes = self.n_classes_ classes = self.classes_[:, np.newaxis] pred = None - norm = 0. + norm = 0.0 - for weight, estimator in zip(self.estimator_weights_, - self.estimators_): + for weight, estimator in zip(self.estimator_weights_, self.estimators_): norm += weight - if self.algorithm == 'SAMME.R': - # The weights are all 1. for SAMME.R - current_pred = _samme_proba(estimator, n_classes, X) - else: # elif self.algorithm == "SAMME": - current_pred = estimator.predict(X) - current_pred = (current_pred == classes).T * weight + current_pred = np.where( + (estimator.predict(X) == classes).T, + weight, + -1 / (n_classes - 1) * weight, + ) if pred is None: pred = current_pred @@ -771,7 +769,7 @@ class in ``classes_``, respectively. def _compute_proba_from_decision(decision, n_classes): """Compute probabilities from the decision function. - This is based eq. (4) of [1] where: + This is based eq. (15) of [1] where: p(y=c|X) = exp((1 / K-1) f_c(X)) / sum_k(exp((1 / K-1) f_k(X))) = softmax((1 / K-1) * f(X)) @@ -783,7 +781,7 @@ def _compute_proba_from_decision(decision, n_classes): if n_classes == 2: decision = np.vstack([-decision, decision]).T / 2 else: - decision /= (n_classes - 1) + decision /= n_classes - 1 return softmax(decision, copy=False) def predict_proba(self, X): @@ -806,8 +804,6 @@ def predict_proba(self, X): outputs is the same of that of the :term:`classes_` attribute. """ check_is_fitted(self) - X = self._check_X(X) - n_classes = self.n_classes_ if n_classes == 1: @@ -835,12 +831,11 @@ def staged_predict_proba(self, X): DOK, or LIL. COO, DOK, and LIL are converted to CSR. Yields - ------- + ------ p : generator of ndarray of shape (n_samples,) The class probabilities of the input samples. The order of outputs is the same of that of the :term:`classes_` attribute. """ - X = self._check_X(X) n_classes = self.n_classes_ @@ -866,11 +861,10 @@ def predict_log_proba(self, X): The class probabilities of the input samples. The order of outputs is the same of that of the :term:`classes_` attribute. """ - X = self._check_X(X) return np.log(self.predict_proba(X)) -class AdaBoostRegressor(RegressorMixin, BaseWeightBoosting): +class AdaBoostRegressor(_RoutingNotSupportedMixin, RegressorMixin, BaseWeightBoosting): """An AdaBoost regressor. An AdaBoost [1] regressor is a meta-estimator that begins by fitting a @@ -887,40 +881,48 @@ class AdaBoostRegressor(RegressorMixin, BaseWeightBoosting): Parameters ---------- - base_estimator : object, default=None + estimator : object, default=None The base estimator from which the boosted ensemble is built. If ``None``, then the base estimator is :class:`~sklearn.tree.DecisionTreeRegressor` initialized with `max_depth=3`. + .. versionadded:: 1.2 + `base_estimator` was renamed to `estimator`. + n_estimators : int, default=50 The maximum number of estimators at which boosting is terminated. In case of perfect fit, the learning procedure is stopped early. + Values must be in the range `[1, inf)`. - learning_rate : float, default=1. - Learning rate shrinks the contribution of each regressor by - ``learning_rate``. There is a trade-off between ``learning_rate`` and - ``n_estimators``. + learning_rate : float, default=1.0 + Weight applied to each regressor at each boosting iteration. A higher + learning rate increases the contribution of each regressor. There is + a trade-off between the `learning_rate` and `n_estimators` parameters. + Values must be in the range `(0.0, inf)`. loss : {'linear', 'square', 'exponential'}, default='linear' The loss function to use when updating the weights after each boosting iteration. random_state : int, RandomState instance or None, default=None - Controls the random seed given at each `base_estimator` at each + Controls the random seed given at each `estimator` at each boosting iteration. - Thus, it is only used when `base_estimator` exposes a `random_state`. + Thus, it is only used when `estimator` exposes a `random_state`. In addition, it controls the bootstrap of the weights used to train the - `base_estimator` at each boosting iteration. + `estimator` at each boosting iteration. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. Attributes ---------- - base_estimator_ : estimator + estimator_ : estimator The base estimator from which the ensemble is grown. - estimators_ : list of classifiers + .. versionadded:: 1.2 + `base_estimator_` was renamed to `estimator_`. + + estimators_ : list of regressors The collection of fitted sub-estimators. estimator_weights_ : ndarray of floats @@ -931,12 +933,36 @@ class AdaBoostRegressor(RegressorMixin, BaseWeightBoosting): feature_importances_ : ndarray of shape (n_features,) The impurity-based feature importances if supported by the - ``base_estimator`` (when based on decision trees). + ``estimator`` (when based on decision trees). Warning: impurity-based feature importances can be misleading for high cardinality features (many unique values). See :func:`sklearn.inspection.permutation_importance` as an alternative. + n_features_in_ : int + Number of features seen during :term:`fit`. + + .. versionadded:: 0.24 + + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of features seen during :term:`fit`. Defined only when `X` + has feature names that are all strings. + + .. versionadded:: 1.0 + + See Also + -------- + AdaBoostClassifier : An AdaBoost classifier. + GradientBoostingRegressor : Gradient Boosting Classification Tree. + sklearn.tree.DecisionTreeRegressor : A decision tree regressor. + + References + ---------- + .. [1] Y. Freund, R. Schapire, "A Decision-Theoretic Generalization of + on-Line Learning and an Application to Boosting", 1995. + + .. [2] H. Drucker, "Improving Regressors using Boosting Techniques", 1997. + Examples -------- >>> from sklearn.ensemble import AdaBoostRegressor @@ -947,72 +973,42 @@ class AdaBoostRegressor(RegressorMixin, BaseWeightBoosting): >>> regr.fit(X, y) AdaBoostRegressor(n_estimators=100, random_state=0) >>> regr.predict([[0, 0, 0, 0]]) - array([4.7972...]) + array([4.7972]) >>> regr.score(X, y) - 0.9771... - - See Also - -------- - AdaBoostClassifier, GradientBoostingRegressor, - sklearn.tree.DecisionTreeRegressor - - References - ---------- - .. [1] Y. Freund, R. Schapire, "A Decision-Theoretic Generalization of - on-Line Learning and an Application to Boosting", 1995. - - .. [2] H. Drucker, "Improving Regressors using Boosting Techniques", 1997. + 0.9771 + For a detailed example of utilizing :class:`~sklearn.ensemble.AdaBoostRegressor` + to fit a sequence of decision trees as weak learners, please refer to + :ref:`sphx_glr_auto_examples_ensemble_plot_adaboost_regression.py`. """ - @_deprecate_positional_args - def __init__(self, - base_estimator=None, *, - n_estimators=50, - learning_rate=1., - loss='linear', - random_state=None): + _parameter_constraints: dict = { + **BaseWeightBoosting._parameter_constraints, + "loss": [StrOptions({"linear", "square", "exponential"})], + } + + def __init__( + self, + estimator=None, + *, + n_estimators=50, + learning_rate=1.0, + loss="linear", + random_state=None, + ): super().__init__( - base_estimator=base_estimator, + estimator=estimator, n_estimators=n_estimators, learning_rate=learning_rate, - random_state=random_state) + random_state=random_state, + ) self.loss = loss self.random_state = random_state - def fit(self, X, y, sample_weight=None): - """Build a boosted regressor from the training set (X, y). - - Parameters - ---------- - X : {array-like, sparse matrix} of shape (n_samples, n_features) - The training input samples. Sparse matrix can be CSC, CSR, COO, - DOK, or LIL. COO, DOK, and LIL are converted to CSR. - - y : array-like of shape (n_samples,) - The target values (real numbers). - - sample_weight : array-like of shape (n_samples,), default=None - Sample weights. If None, the sample weights are initialized to - 1 / n_samples. - - Returns - ------- - self : object - """ - # Check loss - if self.loss not in ('linear', 'square', 'exponential'): - raise ValueError( - "loss must be 'linear', 'square', or 'exponential'") - - # Fit - return super().fit(X, y, sample_weight) - def _validate_estimator(self): - """Check the estimator and set the base_estimator_ attribute.""" - super()._validate_estimator( - default=DecisionTreeRegressor(max_depth=3)) + """Check the estimator and set the estimator_ attribute.""" + super()._validate_estimator(default=DecisionTreeRegressor(max_depth=3)) def _boost(self, iboost, X, y, sample_weight, random_state): """Implement a single boost for regression @@ -1040,7 +1036,6 @@ def _boost(self, iboost, X, y, sample_weight, random_state): `random_state` attribute. Controls also the bootstrap of the weights used to train the weak learner. - replacement. Returns ------- @@ -1060,8 +1055,10 @@ def _boost(self, iboost, X, y, sample_weight, random_state): # Weighted sampling of the training set with replacement bootstrap_idx = random_state.choice( - np.arange(_num_samples(X)), size=_num_samples(X), replace=True, - p=sample_weight + np.arange(_num_samples(X)), + size=_num_samples(X), + replace=True, + p=sample_weight, ) # Fit on the bootstrapped sample and obtain a prediction @@ -1080,17 +1077,17 @@ def _boost(self, iboost, X, y, sample_weight, random_state): if error_max != 0: masked_error_vector /= error_max - if self.loss == 'square': + if self.loss == "square": masked_error_vector **= 2 - elif self.loss == 'exponential': - masked_error_vector = 1. - np.exp(-masked_error_vector) + elif self.loss == "exponential": + masked_error_vector = 1.0 - np.exp(-masked_error_vector) # Calculate the average loss estimator_error = (masked_sample_weight * masked_error_vector).sum() if estimator_error <= 0: # Stop if fit is perfect - return sample_weight, 1., 0. + return sample_weight, 1.0, 0.0 elif estimator_error >= 0.5: # Discard current estimator only if it isn't the only one @@ -1098,22 +1095,21 @@ def _boost(self, iboost, X, y, sample_weight, random_state): self.estimators_.pop(-1) return None, None, None - beta = estimator_error / (1. - estimator_error) + beta = estimator_error / (1.0 - estimator_error) # Boost weight using AdaBoost.R2 alg - estimator_weight = self.learning_rate * np.log(1. / beta) + estimator_weight = self.learning_rate * np.log(1.0 / beta) if not iboost == self.n_estimators - 1: sample_weight[sample_mask] *= np.power( - beta, (1. - masked_error_vector) * self.learning_rate + beta, (1.0 - masked_error_vector) * self.learning_rate ) return sample_weight, estimator_weight, estimator_error def _get_median_predict(self, X, limit): # Evaluate predictions of all estimators - predictions = np.array([ - est.predict(X) for est in self.estimators_[:limit]]).T + predictions = np.array([est.predict(X) for est in self.estimators_[:limit]]).T # Sort the predictions sorted_idx = np.argsort(predictions, axis=1) @@ -1132,7 +1128,7 @@ def predict(self, X): """Predict regression value for X. The predicted regression value of an input sample is computed - as the weighted median prediction of the classifiers in the ensemble. + as the weighted median prediction of the regressors in the ensemble. Parameters ---------- @@ -1154,7 +1150,7 @@ def staged_predict(self, X): """Return staged predictions for X. The predicted regression value of an input sample is computed - as the weighted median prediction of the classifiers in the ensemble. + as the weighted median prediction of the regressors in the ensemble. This generator method yields the ensemble prediction after each iteration of boosting and therefore allows monitoring, such as to @@ -1166,7 +1162,7 @@ def staged_predict(self, X): The training input samples. Yields - ------- + ------ y : generator of ndarray of shape (n_samples,) The predicted regression values. """ diff --git a/sklearn/ensemble/meson.build b/sklearn/ensemble/meson.build new file mode 100644 index 0000000000000..893a4eb1a510a --- /dev/null +++ b/sklearn/ensemble/meson.build @@ -0,0 +1,9 @@ +py.extension_module( + '_gradient_boosting', + [cython_gen.process('_gradient_boosting.pyx')] + utils_cython_tree, + dependencies: [np_dep], + subdir: 'sklearn/ensemble', + install: true +) + +subdir('_hist_gradient_boosting') diff --git a/sklearn/ensemble/setup.py b/sklearn/ensemble/setup.py deleted file mode 100644 index 05d71cf314461..0000000000000 --- a/sklearn/ensemble/setup.py +++ /dev/null @@ -1,58 +0,0 @@ -import numpy -from numpy.distutils.misc_util import Configuration - - -def configuration(parent_package="", top_path=None): - config = Configuration("ensemble", parent_package, top_path) - - config.add_extension("_gradient_boosting", - sources=["_gradient_boosting.pyx"], - include_dirs=[numpy.get_include()]) - - config.add_subpackage("tests") - - # Histogram-based gradient boosting files - config.add_extension( - "_hist_gradient_boosting._gradient_boosting", - sources=["_hist_gradient_boosting/_gradient_boosting.pyx"], - include_dirs=[numpy.get_include()]) - - config.add_extension("_hist_gradient_boosting.histogram", - sources=["_hist_gradient_boosting/histogram.pyx"], - include_dirs=[numpy.get_include()]) - - config.add_extension("_hist_gradient_boosting.splitting", - sources=["_hist_gradient_boosting/splitting.pyx"], - include_dirs=[numpy.get_include()]) - - config.add_extension("_hist_gradient_boosting._binning", - sources=["_hist_gradient_boosting/_binning.pyx"], - include_dirs=[numpy.get_include()]) - - config.add_extension("_hist_gradient_boosting._predictor", - sources=["_hist_gradient_boosting/_predictor.pyx"], - include_dirs=[numpy.get_include()]) - - config.add_extension("_hist_gradient_boosting._loss", - sources=["_hist_gradient_boosting/_loss.pyx"], - include_dirs=[numpy.get_include()]) - - config.add_extension("_hist_gradient_boosting._bitset", - sources=["_hist_gradient_boosting/_bitset.pyx"], - include_dirs=[numpy.get_include()]) - - config.add_extension("_hist_gradient_boosting.common", - sources=["_hist_gradient_boosting/common.pyx"], - include_dirs=[numpy.get_include()]) - - config.add_extension("_hist_gradient_boosting.utils", - sources=["_hist_gradient_boosting/utils.pyx"], - include_dirs=[numpy.get_include()]) - - config.add_subpackage("_hist_gradient_boosting.tests") - - return config - -if __name__ == "__main__": - from numpy.distutils.core import setup - setup(**configuration().todict()) diff --git a/sklearn/ensemble/tests/test_bagging.py b/sklearn/ensemble/tests/test_bagging.py index b17cbf7c147ac..2cb9336bfd759 100644 --- a/sklearn/ensemble/tests/test_bagging.py +++ b/sklearn/ensemble/tests/test_bagging.py @@ -2,34 +2,48 @@ Testing for the bagging ensemble module (sklearn.ensemble.bagging). """ -# Author: Gilles Louppe -# License: BSD 3 clause -from itertools import product +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from itertools import cycle, product -import numpy as np import joblib +import numpy as np import pytest +from sklearn import config_context from sklearn.base import BaseEstimator - -from sklearn.utils._testing import assert_array_equal -from sklearn.utils._testing import assert_array_almost_equal +from sklearn.datasets import load_diabetes, load_iris, make_hastie_10_2 from sklearn.dummy import DummyClassifier, DummyRegressor -from sklearn.model_selection import GridSearchCV, ParameterGrid -from sklearn.ensemble import BaggingClassifier, BaggingRegressor -from sklearn.linear_model import Perceptron, LogisticRegression +from sklearn.ensemble import ( + AdaBoostClassifier, + AdaBoostRegressor, + BaggingClassifier, + BaggingRegressor, + HistGradientBoostingClassifier, + HistGradientBoostingRegressor, + RandomForestClassifier, + RandomForestRegressor, +) +from sklearn.feature_selection import SelectKBest +from sklearn.linear_model import LogisticRegression, Perceptron +from sklearn.model_selection import GridSearchCV, ParameterGrid, train_test_split from sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor -from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor -from sklearn.svm import SVC, SVR -from sklearn.random_projection import SparseRandomProjection from sklearn.pipeline import make_pipeline -from sklearn.feature_selection import SelectKBest -from sklearn.model_selection import train_test_split -from sklearn.datasets import load_diabetes, load_iris, make_hastie_10_2 -from sklearn.utils import check_random_state from sklearn.preprocessing import FunctionTransformer, scale - -from scipy.sparse import csc_matrix, csr_matrix +from sklearn.random_projection import SparseRandomProjection +from sklearn.svm import SVC, SVR +from sklearn.tests.metadata_routing_common import ( + ConsumingClassifierWithOnlyPredict, + ConsumingClassifierWithoutPredictLogProba, + ConsumingClassifierWithoutPredictProba, + _Registry, + check_recorded_metadata, +) +from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor +from sklearn.utils import check_random_state +from sklearn.utils._testing import assert_array_almost_equal, assert_array_equal +from sklearn.utils.fixes import CSC_CONTAINERS, CSR_CONTAINERS rng = check_random_state(0) @@ -51,52 +65,60 @@ def test_classification(): # Check classification for various parameter settings. rng = check_random_state(0) - X_train, X_test, y_train, y_test = train_test_split(iris.data, - iris.target, - random_state=rng) - grid = ParameterGrid({"max_samples": [0.5, 1.0], - "max_features": [1, 2, 4], - "bootstrap": [True, False], - "bootstrap_features": [True, False]}) - - for base_estimator in [None, - DummyClassifier(), - Perceptron(), - DecisionTreeClassifier(), - KNeighborsClassifier(), - SVC()]: - for params in grid: - BaggingClassifier(base_estimator=base_estimator, - random_state=rng, - **params).fit(X_train, y_train).predict(X_test) + X_train, X_test, y_train, y_test = train_test_split( + iris.data, iris.target, random_state=rng + ) + grid = ParameterGrid( + { + "max_samples": [0.5, 1.0], + "max_features": [1, 4], + "bootstrap": [True, False], + "bootstrap_features": [True, False], + } + ) + estimators = [ + None, + DummyClassifier(), + Perceptron(max_iter=20), + DecisionTreeClassifier(max_depth=2), + KNeighborsClassifier(), + SVC(), + ] + # Try different parameter settings with different base classifiers without + # doing the full cartesian product to keep the test durations low. + for params, estimator in zip(grid, cycle(estimators)): + BaggingClassifier( + estimator=estimator, + random_state=rng, + n_estimators=2, + **params, + ).fit(X_train, y_train).predict(X_test) @pytest.mark.parametrize( - 'sparse_format, params, method', + "sparse_container, params, method", product( - [csc_matrix, csr_matrix], - [{ - "max_samples": 0.5, - "max_features": 2, - "bootstrap": True, - "bootstrap_features": True - }, { - "max_samples": 1.0, - "max_features": 4, - "bootstrap": True, - "bootstrap_features": True - }, { - "max_features": 2, - "bootstrap": False, - "bootstrap_features": True - }, { - "max_samples": 0.5, - "bootstrap": True, - "bootstrap_features": False - }], - ['predict', 'predict_proba', - 'predict_log_proba', 'decision_function'])) -def test_sparse_classification(sparse_format, params, method): + CSR_CONTAINERS + CSC_CONTAINERS, + [ + { + "max_samples": 0.5, + "max_features": 2, + "bootstrap": True, + "bootstrap_features": True, + }, + { + "max_samples": 1.0, + "max_features": 4, + "bootstrap": True, + "bootstrap_features": True, + }, + {"max_features": 2, "bootstrap": False, "bootstrap_features": True}, + {"max_samples": 0.5, "bootstrap": True, "bootstrap_features": False}, + ], + ["predict", "predict_proba", "predict_log_proba", "decision_function"], + ), +) +def test_sparse_classification(sparse_container, params, method): # Check classification for various parameter settings on sparse input. class CustomSVC(SVC): @@ -108,27 +130,25 @@ def fit(self, X, y): return self rng = check_random_state(0) - X_train, X_test, y_train, y_test = train_test_split(scale(iris.data), - iris.target, - random_state=rng) + X_train, X_test, y_train, y_test = train_test_split( + scale(iris.data), iris.target, random_state=rng + ) - X_train_sparse = sparse_format(X_train) - X_test_sparse = sparse_format(X_test) + X_train_sparse = sparse_container(X_train) + X_test_sparse = sparse_container(X_test) # Trained on sparse format sparse_classifier = BaggingClassifier( - base_estimator=CustomSVC(kernel="linear", - decision_function_shape='ovr'), + estimator=CustomSVC(kernel="linear", decision_function_shape="ovr"), random_state=1, - **params + **params, ).fit(X_train_sparse, y_train) sparse_results = getattr(sparse_classifier, method)(X_test_sparse) # Trained on dense format dense_classifier = BaggingClassifier( - base_estimator=CustomSVC(kernel="linear", - decision_function_shape='ovr'), + estimator=CustomSVC(kernel="linear", decision_function_shape="ovr"), random_state=1, - **params + **params, ).fit(X_train, y_train) dense_results = getattr(dense_classifier, method)(X_test) assert_array_almost_equal(sparse_results, dense_results) @@ -142,31 +162,38 @@ def fit(self, X, y): def test_regression(): # Check regression for various parameter settings. rng = check_random_state(0) - X_train, X_test, y_train, y_test = train_test_split(diabetes.data[:50], - diabetes.target[:50], - random_state=rng) - grid = ParameterGrid({"max_samples": [0.5, 1.0], - "max_features": [0.5, 1.0], - "bootstrap": [True, False], - "bootstrap_features": [True, False]}) - - for base_estimator in [None, - DummyRegressor(), - DecisionTreeRegressor(), - KNeighborsRegressor(), - SVR()]: + X_train, X_test, y_train, y_test = train_test_split( + diabetes.data[:50], diabetes.target[:50], random_state=rng + ) + grid = ParameterGrid( + { + "max_samples": [0.5, 1.0], + "max_features": [0.5, 1.0], + "bootstrap": [True, False], + "bootstrap_features": [True, False], + } + ) + + for estimator in [ + None, + DummyRegressor(), + DecisionTreeRegressor(), + KNeighborsRegressor(), + SVR(), + ]: for params in grid: - BaggingRegressor(base_estimator=base_estimator, - random_state=rng, - **params).fit(X_train, y_train).predict(X_test) + BaggingRegressor(estimator=estimator, random_state=rng, **params).fit( + X_train, y_train + ).predict(X_test) -def test_sparse_regression(): +@pytest.mark.parametrize("sparse_container", CSR_CONTAINERS + CSC_CONTAINERS) +def test_sparse_regression(sparse_container): # Check regression for various parameter settings on sparse input. rng = check_random_state(0) - X_train, X_test, y_train, y_test = train_test_split(diabetes.data[:50], - diabetes.target[:50], - random_state=rng) + X_train, X_test, y_train, y_test = train_test_split( + diabetes.data[:50], diabetes.target[:50], random_state=rng + ) class CustomSVR(SVR): """SVC variant that records the nature of the training set""" @@ -177,89 +204,90 @@ def fit(self, X, y): return self parameter_sets = [ - {"max_samples": 0.5, - "max_features": 2, - "bootstrap": True, - "bootstrap_features": True}, - {"max_samples": 1.0, - "max_features": 4, - "bootstrap": True, - "bootstrap_features": True}, - {"max_features": 2, - "bootstrap": False, - "bootstrap_features": True}, - {"max_samples": 0.5, - "bootstrap": True, - "bootstrap_features": False}, + { + "max_samples": 0.5, + "max_features": 2, + "bootstrap": True, + "bootstrap_features": True, + }, + { + "max_samples": 1.0, + "max_features": 4, + "bootstrap": True, + "bootstrap_features": True, + }, + {"max_features": 2, "bootstrap": False, "bootstrap_features": True}, + {"max_samples": 0.5, "bootstrap": True, "bootstrap_features": False}, ] - for sparse_format in [csc_matrix, csr_matrix]: - X_train_sparse = sparse_format(X_train) - X_test_sparse = sparse_format(X_test) - for params in parameter_sets: - - # Trained on sparse format - sparse_classifier = BaggingRegressor( - base_estimator=CustomSVR(), - random_state=1, - **params - ).fit(X_train_sparse, y_train) - sparse_results = sparse_classifier.predict(X_test_sparse) - - # Trained on dense format - dense_results = BaggingRegressor( - base_estimator=CustomSVR(), - random_state=1, - **params - ).fit(X_train, y_train).predict(X_test) + X_train_sparse = sparse_container(X_train) + X_test_sparse = sparse_container(X_test) + for params in parameter_sets: + # Trained on sparse format + sparse_classifier = BaggingRegressor( + estimator=CustomSVR(), random_state=1, **params + ).fit(X_train_sparse, y_train) + sparse_results = sparse_classifier.predict(X_test_sparse) + + # Trained on dense format + dense_results = ( + BaggingRegressor(estimator=CustomSVR(), random_state=1, **params) + .fit(X_train, y_train) + .predict(X_test) + ) - sparse_type = type(X_train_sparse) - types = [i.data_type_ for i in sparse_classifier.estimators_] + sparse_type = type(X_train_sparse) + types = [i.data_type_ for i in sparse_classifier.estimators_] - assert_array_almost_equal(sparse_results, dense_results) - assert all([t == sparse_type for t in types]) - assert_array_almost_equal(sparse_results, dense_results) + assert_array_almost_equal(sparse_results, dense_results) + assert all([t == sparse_type for t in types]) + assert_array_almost_equal(sparse_results, dense_results) class DummySizeEstimator(BaseEstimator): - def fit(self, X, y): self.training_size_ = X.shape[0] self.training_hash_ = joblib.hash(X) + def predict(self, X): + return np.ones(X.shape[0]) + def test_bootstrap_samples(): # Test that bootstrapping samples generate non-perfect base estimators. rng = check_random_state(0) - X_train, X_test, y_train, y_test = train_test_split(diabetes.data, - diabetes.target, - random_state=rng) + X_train, X_test, y_train, y_test = train_test_split( + diabetes.data, diabetes.target, random_state=rng + ) - base_estimator = DecisionTreeRegressor().fit(X_train, y_train) + estimator = DecisionTreeRegressor().fit(X_train, y_train) # without bootstrap, all trees are perfect on the training set - ensemble = BaggingRegressor(base_estimator=DecisionTreeRegressor(), - max_samples=1.0, - bootstrap=False, - random_state=rng).fit(X_train, y_train) + ensemble = BaggingRegressor( + estimator=DecisionTreeRegressor(), + max_samples=1.0, + bootstrap=False, + random_state=rng, + ).fit(X_train, y_train) - assert (base_estimator.score(X_train, y_train) == - ensemble.score(X_train, y_train)) + assert estimator.score(X_train, y_train) == ensemble.score(X_train, y_train) # with bootstrap, trees are no longer perfect on the training set - ensemble = BaggingRegressor(base_estimator=DecisionTreeRegressor(), - max_samples=1.0, - bootstrap=True, - random_state=rng).fit(X_train, y_train) + ensemble = BaggingRegressor( + estimator=DecisionTreeRegressor(), + max_samples=1.0, + bootstrap=True, + random_state=rng, + ).fit(X_train, y_train) - assert (base_estimator.score(X_train, y_train) > - ensemble.score(X_train, y_train)) + assert estimator.score(X_train, y_train) > ensemble.score(X_train, y_train) # check that each sampling correspond to a complete bootstrap resample. # the size of each bootstrap should be the same as the input data but # the data should be different (checked using the hash of the data). - ensemble = BaggingRegressor(base_estimator=DummySizeEstimator(), - bootstrap=True).fit(X_train, y_train) + ensemble = BaggingRegressor(estimator=DummySizeEstimator(), bootstrap=True).fit( + X_train, y_train + ) training_hash = [] for estimator in ensemble.estimators_: assert estimator.training_size_ == X_train.shape[0] @@ -270,22 +298,26 @@ def test_bootstrap_samples(): def test_bootstrap_features(): # Test that bootstrapping features may generate duplicate features. rng = check_random_state(0) - X_train, X_test, y_train, y_test = train_test_split(diabetes.data, - diabetes.target, - random_state=rng) + X_train, X_test, y_train, y_test = train_test_split( + diabetes.data, diabetes.target, random_state=rng + ) - ensemble = BaggingRegressor(base_estimator=DecisionTreeRegressor(), - max_features=1.0, - bootstrap_features=False, - random_state=rng).fit(X_train, y_train) + ensemble = BaggingRegressor( + estimator=DecisionTreeRegressor(), + max_features=1.0, + bootstrap_features=False, + random_state=rng, + ).fit(X_train, y_train) for features in ensemble.estimators_features_: assert diabetes.data.shape[1] == np.unique(features).shape[0] - ensemble = BaggingRegressor(base_estimator=DecisionTreeRegressor(), - max_features=1.0, - bootstrap_features=True, - random_state=rng).fit(X_train, y_train) + ensemble = BaggingRegressor( + estimator=DecisionTreeRegressor(), + max_features=1.0, + bootstrap_features=True, + random_state=rng, + ).fit(X_train, y_train) for features in ensemble.estimators_features_: assert diabetes.data.shape[1] > np.unique(features).shape[0] @@ -294,49 +326,54 @@ def test_bootstrap_features(): def test_probability(): # Predict probabilities. rng = check_random_state(0) - X_train, X_test, y_train, y_test = train_test_split(iris.data, - iris.target, - random_state=rng) + X_train, X_test, y_train, y_test = train_test_split( + iris.data, iris.target, random_state=rng + ) with np.errstate(divide="ignore", invalid="ignore"): # Normal case - ensemble = BaggingClassifier(base_estimator=DecisionTreeClassifier(), - random_state=rng).fit(X_train, y_train) + ensemble = BaggingClassifier( + estimator=DecisionTreeClassifier(), random_state=rng + ).fit(X_train, y_train) - assert_array_almost_equal(np.sum(ensemble.predict_proba(X_test), - axis=1), - np.ones(len(X_test))) + assert_array_almost_equal( + np.sum(ensemble.predict_proba(X_test), axis=1), np.ones(len(X_test)) + ) - assert_array_almost_equal(ensemble.predict_proba(X_test), - np.exp(ensemble.predict_log_proba(X_test))) + assert_array_almost_equal( + ensemble.predict_proba(X_test), np.exp(ensemble.predict_log_proba(X_test)) + ) # Degenerate case, where some classes are missing - ensemble = BaggingClassifier(base_estimator=LogisticRegression(), - random_state=rng, - max_samples=5).fit(X_train, y_train) + ensemble = BaggingClassifier( + estimator=LogisticRegression(), random_state=rng, max_samples=5 + ).fit(X_train, y_train) - assert_array_almost_equal(np.sum(ensemble.predict_proba(X_test), - axis=1), - np.ones(len(X_test))) + assert_array_almost_equal( + np.sum(ensemble.predict_proba(X_test), axis=1), np.ones(len(X_test)) + ) - assert_array_almost_equal(ensemble.predict_proba(X_test), - np.exp(ensemble.predict_log_proba(X_test))) + assert_array_almost_equal( + ensemble.predict_proba(X_test), np.exp(ensemble.predict_log_proba(X_test)) + ) def test_oob_score_classification(): # Check that oob prediction is a good estimation of the generalization # error. rng = check_random_state(0) - X_train, X_test, y_train, y_test = train_test_split(iris.data, - iris.target, - random_state=rng) + X_train, X_test, y_train, y_test = train_test_split( + iris.data, iris.target, random_state=rng + ) - for base_estimator in [DecisionTreeClassifier(), SVC()]: - clf = BaggingClassifier(base_estimator=base_estimator, - n_estimators=100, - bootstrap=True, - oob_score=True, - random_state=rng).fit(X_train, y_train) + for estimator in [DecisionTreeClassifier(), SVC()]: + clf = BaggingClassifier( + estimator=estimator, + n_estimators=100, + bootstrap=True, + oob_score=True, + random_state=rng, + ).fit(X_train, y_train) test_score = clf.score(X_test, y_test) @@ -349,7 +386,7 @@ def test_oob_score_classification(): ) with pytest.warns(UserWarning, match=warn_msg): clf = BaggingClassifier( - base_estimator=base_estimator, + estimator=estimator, n_estimators=1, bootstrap=True, oob_score=True, @@ -362,15 +399,17 @@ def test_oob_score_regression(): # Check that oob prediction is a good estimation of the generalization # error. rng = check_random_state(0) - X_train, X_test, y_train, y_test = train_test_split(diabetes.data, - diabetes.target, - random_state=rng) + X_train, X_test, y_train, y_test = train_test_split( + diabetes.data, diabetes.target, random_state=rng + ) - clf = BaggingRegressor(base_estimator=DecisionTreeRegressor(), - n_estimators=50, - bootstrap=True, - oob_score=True, - random_state=rng).fit(X_train, y_train) + clf = BaggingRegressor( + estimator=DecisionTreeRegressor(), + n_estimators=50, + bootstrap=True, + oob_score=True, + random_state=rng, + ).fit(X_train, y_train) test_score = clf.score(X_test, y_test) @@ -383,26 +422,29 @@ def test_oob_score_regression(): ) with pytest.warns(UserWarning, match=warn_msg): regr = BaggingRegressor( - base_estimator=DecisionTreeRegressor(), + estimator=DecisionTreeRegressor(), n_estimators=1, bootstrap=True, oob_score=True, - random_state=rng) + random_state=rng, + ) regr.fit(X_train, y_train) def test_single_estimator(): # Check singleton ensembles. rng = check_random_state(0) - X_train, X_test, y_train, y_test = train_test_split(diabetes.data, - diabetes.target, - random_state=rng) + X_train, X_test, y_train, y_test = train_test_split( + diabetes.data, diabetes.target, random_state=rng + ) - clf1 = BaggingRegressor(base_estimator=KNeighborsRegressor(), - n_estimators=1, - bootstrap=False, - bootstrap_features=False, - random_state=rng).fit(X_train, y_train) + clf1 = BaggingRegressor( + estimator=KNeighborsRegressor(), + n_estimators=1, + bootstrap=False, + bootstrap_features=False, + random_state=rng, + ).fit(X_train, y_train) clf2 = KNeighborsRegressor().fit(X_train, y_train) @@ -410,79 +452,48 @@ def test_single_estimator(): def test_error(): - # Test that it gives proper exception on deficient input. + # Test support of decision_function X, y = iris.data, iris.target base = DecisionTreeClassifier() - - # Test max_samples - with pytest.raises(ValueError): - BaggingClassifier(base, max_samples=-1).fit(X, y) - with pytest.raises(ValueError): - BaggingClassifier(base, max_samples=0.0).fit(X, y) - with pytest.raises(ValueError): - BaggingClassifier(base, max_samples=2.0).fit(X, y) - with pytest.raises(ValueError): - BaggingClassifier(base, max_samples=1000).fit(X, y) - with pytest.raises(ValueError): - BaggingClassifier(base, max_samples="foobar").fit(X, y) - - # Test max_features - with pytest.raises(ValueError): - BaggingClassifier(base, max_features=-1).fit(X, y) - with pytest.raises(ValueError): - BaggingClassifier(base, max_features=0.0).fit(X, y) - with pytest.raises(ValueError): - BaggingClassifier(base, max_features=2.0).fit(X, y) - with pytest.raises(ValueError): - BaggingClassifier(base, max_features=5).fit(X, y) - with pytest.raises(ValueError): - BaggingClassifier(base, max_features="foobar").fit(X, y) - - # Test support of decision_function - assert not hasattr(BaggingClassifier(base).fit(X, y), 'decision_function') + assert not hasattr(BaggingClassifier(base).fit(X, y), "decision_function") def test_parallel_classification(): # Check parallel classification. - rng = check_random_state(0) - - # Classification - X_train, X_test, y_train, y_test = train_test_split(iris.data, - iris.target, - random_state=rng) + X_train, X_test, y_train, y_test = train_test_split( + iris.data, iris.target, random_state=0 + ) - ensemble = BaggingClassifier(DecisionTreeClassifier(), - n_jobs=3, - random_state=0).fit(X_train, y_train) + ensemble = BaggingClassifier( + DecisionTreeClassifier(), n_jobs=3, random_state=0 + ).fit(X_train, y_train) # predict_proba - ensemble.set_params(n_jobs=1) y1 = ensemble.predict_proba(X_test) - ensemble.set_params(n_jobs=2) + ensemble.set_params(n_jobs=1) y2 = ensemble.predict_proba(X_test) assert_array_almost_equal(y1, y2) - ensemble = BaggingClassifier(DecisionTreeClassifier(), - n_jobs=1, - random_state=0).fit(X_train, y_train) + ensemble = BaggingClassifier( + DecisionTreeClassifier(), n_jobs=1, random_state=0 + ).fit(X_train, y_train) y3 = ensemble.predict_proba(X_test) assert_array_almost_equal(y1, y3) # decision_function - ensemble = BaggingClassifier(SVC(decision_function_shape='ovr'), - n_jobs=3, - random_state=0).fit(X_train, y_train) + ensemble = BaggingClassifier( + SVC(decision_function_shape="ovr"), n_jobs=3, random_state=0 + ).fit(X_train, y_train) - ensemble.set_params(n_jobs=1) decisions1 = ensemble.decision_function(X_test) - ensemble.set_params(n_jobs=2) + ensemble.set_params(n_jobs=1) decisions2 = ensemble.decision_function(X_test) assert_array_almost_equal(decisions1, decisions2) - ensemble = BaggingClassifier(SVC(decision_function_shape='ovr'), - n_jobs=1, - random_state=0).fit(X_train, y_train) + ensemble = BaggingClassifier( + SVC(decision_function_shape="ovr"), n_jobs=1, random_state=0 + ).fit(X_train, y_train) decisions3 = ensemble.decision_function(X_test) assert_array_almost_equal(decisions1, decisions3) @@ -492,13 +503,13 @@ def test_parallel_regression(): # Check parallel regression. rng = check_random_state(0) - X_train, X_test, y_train, y_test = train_test_split(diabetes.data, - diabetes.target, - random_state=rng) + X_train, X_test, y_train, y_test = train_test_split( + diabetes.data, diabetes.target, random_state=rng + ) - ensemble = BaggingRegressor(DecisionTreeRegressor(), - n_jobs=3, - random_state=0).fit(X_train, y_train) + ensemble = BaggingRegressor(DecisionTreeRegressor(), n_jobs=3, random_state=0).fit( + X_train, y_train + ) ensemble.set_params(n_jobs=1) y1 = ensemble.predict(X_test) @@ -506,9 +517,9 @@ def test_parallel_regression(): y2 = ensemble.predict(X_test) assert_array_almost_equal(y1, y2) - ensemble = BaggingRegressor(DecisionTreeRegressor(), - n_jobs=1, - random_state=0).fit(X_train, y_train) + ensemble = BaggingRegressor(DecisionTreeRegressor(), n_jobs=1, random_state=0).fit( + X_train, y_train + ) y3 = ensemble.predict(X_test) assert_array_almost_equal(y1, y3) @@ -521,74 +532,64 @@ def test_gridsearch(): y[y == 2] = 1 # Grid search with scoring based on decision_function - parameters = {'n_estimators': (1, 2), - 'base_estimator__C': (1, 2)} + parameters = {"n_estimators": (1, 2), "estimator__C": (1, 2)} - GridSearchCV(BaggingClassifier(SVC()), - parameters, - scoring="roc_auc").fit(X, y) + GridSearchCV(BaggingClassifier(SVC()), parameters, scoring="roc_auc").fit(X, y) -def test_base_estimator(): - # Check base_estimator and its default values. +def test_estimator(): + # Check estimator and its default values. rng = check_random_state(0) # Classification - X_train, X_test, y_train, y_test = train_test_split(iris.data, - iris.target, - random_state=rng) + X_train, X_test, y_train, y_test = train_test_split( + iris.data, iris.target, random_state=rng + ) - ensemble = BaggingClassifier(None, - n_jobs=3, - random_state=0).fit(X_train, y_train) + ensemble = BaggingClassifier(None, n_jobs=3, random_state=0).fit(X_train, y_train) - assert isinstance(ensemble.base_estimator_, DecisionTreeClassifier) + assert isinstance(ensemble.estimator_, DecisionTreeClassifier) - ensemble = BaggingClassifier(DecisionTreeClassifier(), - n_jobs=3, - random_state=0).fit(X_train, y_train) + ensemble = BaggingClassifier( + DecisionTreeClassifier(), n_jobs=3, random_state=0 + ).fit(X_train, y_train) - assert isinstance(ensemble.base_estimator_, DecisionTreeClassifier) + assert isinstance(ensemble.estimator_, DecisionTreeClassifier) - ensemble = BaggingClassifier(Perceptron(), - n_jobs=3, - random_state=0).fit(X_train, y_train) + ensemble = BaggingClassifier(Perceptron(), n_jobs=3, random_state=0).fit( + X_train, y_train + ) - assert isinstance(ensemble.base_estimator_, Perceptron) + assert isinstance(ensemble.estimator_, Perceptron) # Regression - X_train, X_test, y_train, y_test = train_test_split(diabetes.data, - diabetes.target, - random_state=rng) + X_train, X_test, y_train, y_test = train_test_split( + diabetes.data, diabetes.target, random_state=rng + ) - ensemble = BaggingRegressor(None, - n_jobs=3, - random_state=0).fit(X_train, y_train) + ensemble = BaggingRegressor(None, n_jobs=3, random_state=0).fit(X_train, y_train) - assert isinstance(ensemble.base_estimator_, DecisionTreeRegressor) + assert isinstance(ensemble.estimator_, DecisionTreeRegressor) - ensemble = BaggingRegressor(DecisionTreeRegressor(), - n_jobs=3, - random_state=0).fit(X_train, y_train) + ensemble = BaggingRegressor(DecisionTreeRegressor(), n_jobs=3, random_state=0).fit( + X_train, y_train + ) - assert isinstance(ensemble.base_estimator_, DecisionTreeRegressor) + assert isinstance(ensemble.estimator_, DecisionTreeRegressor) - ensemble = BaggingRegressor(SVR(), - n_jobs=3, - random_state=0).fit(X_train, y_train) - assert isinstance(ensemble.base_estimator_, SVR) + ensemble = BaggingRegressor(SVR(), n_jobs=3, random_state=0).fit(X_train, y_train) + assert isinstance(ensemble.estimator_, SVR) def test_bagging_with_pipeline(): - estimator = BaggingClassifier(make_pipeline(SelectKBest(k=1), - DecisionTreeClassifier()), - max_features=2) + estimator = BaggingClassifier( + make_pipeline(SelectKBest(k=1), DecisionTreeClassifier()), max_features=2 + ) estimator.fit(iris.data, iris.target) assert isinstance(estimator[0].steps[-1][1].random_state, int) class DummyZeroEstimator(BaseEstimator): - def fit(self, X, y): self.classes_ = np.unique(y) return self @@ -603,8 +604,11 @@ def test_bagging_sample_weight_unsupported_but_passed(): estimator.fit(iris.data, iris.target).predict(iris.data) with pytest.raises(ValueError): - estimator.fit(iris.data, iris.target, - sample_weight=rng.randint(10, size=(iris.data.shape[0]))) + estimator.fit( + iris.data, + iris.target, + sample_weight=rng.randint(10, size=(iris.data.shape[0])), + ) def test_warm_start(random_state=42): @@ -615,20 +619,22 @@ def test_warm_start(random_state=42): clf_ws = None for n_estimators in [5, 10]: if clf_ws is None: - clf_ws = BaggingClassifier(n_estimators=n_estimators, - random_state=random_state, - warm_start=True) + clf_ws = BaggingClassifier( + n_estimators=n_estimators, random_state=random_state, warm_start=True + ) else: clf_ws.set_params(n_estimators=n_estimators) clf_ws.fit(X, y) assert len(clf_ws) == n_estimators - clf_no_ws = BaggingClassifier(n_estimators=10, random_state=random_state, - warm_start=False) + clf_no_ws = BaggingClassifier( + n_estimators=10, random_state=random_state, warm_start=False + ) clf_no_ws.fit(X, y) - assert (set([tree.random_state for tree in clf_ws]) == - set([tree.random_state for tree in clf_no_ws])) + assert set([tree.random_state for tree in clf_ws]) == set( + [tree.random_state for tree in clf_no_ws] + ) def test_warm_start_smaller_n_estimators(): @@ -651,7 +657,7 @@ def test_warm_start_equal_n_estimators(): y_pred = clf.predict(X_test) # modify X to nonsense values, this should not change anything - X_train += 1. + X_train += 1.0 warn_msg = "Warm-start fitting without increasing n_estimators does not" with pytest.warns(UserWarning, match=warn_msg): @@ -665,15 +671,13 @@ def test_warm_start_equivalence(): X, y = make_hastie_10_2(n_samples=20, random_state=1) X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=43) - clf_ws = BaggingClassifier(n_estimators=5, warm_start=True, - random_state=3141) + clf_ws = BaggingClassifier(n_estimators=5, warm_start=True, random_state=3141) clf_ws.fit(X_train, y_train) clf_ws.set_params(n_estimators=10) clf_ws.fit(X_train, y_train) y1 = clf_ws.predict(X_test) - clf = BaggingClassifier(n_estimators=10, warm_start=False, - random_state=3141) + clf = BaggingClassifier(n_estimators=10, warm_start=False, random_state=3141) clf.fit(X_train, y_train) y2 = clf.predict(X_test) @@ -689,12 +693,12 @@ def test_warm_start_with_oob_score_fails(): def test_oob_score_removed_on_warm_start(): - X, y = make_hastie_10_2(n_samples=2000, random_state=1) + X, y = make_hastie_10_2(n_samples=100, random_state=1) - clf = BaggingClassifier(n_estimators=50, oob_score=True) + clf = BaggingClassifier(n_estimators=5, oob_score=True) clf.fit(X, y) - clf.set_params(warm_start=True, oob_score=False, n_estimators=100) + clf.set_params(warm_start=True, oob_score=False, n_estimators=10) clf.fit(X, y) with pytest.raises(AttributeError): @@ -705,9 +709,13 @@ def test_oob_score_consistency(): # Make sure OOB scores are identical when random_state, estimator, and # training data are fixed and fitting is done twice X, y = make_hastie_10_2(n_samples=200, random_state=1) - bagging = BaggingClassifier(KNeighborsClassifier(), max_samples=0.5, - max_features=0.5, oob_score=True, - random_state=1) + bagging = BaggingClassifier( + KNeighborsClassifier(), + max_samples=0.5, + max_features=0.5, + oob_score=True, + random_state=1, + ) assert bagging.fit(X, y).oob_score_ == bagging.fit(X, y).oob_score_ @@ -716,9 +724,13 @@ def test_estimators_samples(): # generated at fit time can be identically reproduced at a later time # using data saved in object attributes. X, y = make_hastie_10_2(n_samples=200, random_state=1) - bagging = BaggingClassifier(LogisticRegression(), max_samples=0.5, - max_features=0.5, random_state=1, - bootstrap=False) + bagging = BaggingClassifier( + LogisticRegression(), + max_samples=0.5, + max_features=0.5, + random_state=1, + bootstrap=False, + ) bagging.fit(X, y) # Get relevant attributes @@ -729,7 +741,7 @@ def test_estimators_samples(): # Test for correct formatting assert len(estimators_samples) == len(estimators) assert len(estimators_samples[0]) == len(X) // 2 - assert estimators_samples[0].dtype.kind == 'i' + assert estimators_samples[0].dtype.kind == "i" # Re-fit single estimator to test for consistent sampling estimator_index = 0 @@ -756,11 +768,10 @@ def test_estimators_samples_deterministic(): iris = load_iris() X, y = iris.data, iris.target - base_pipeline = make_pipeline(SparseRandomProjection(n_components=2), - LogisticRegression()) - clf = BaggingClassifier(base_estimator=base_pipeline, - max_samples=0.5, - random_state=0) + base_pipeline = make_pipeline( + SparseRandomProjection(n_components=2), LogisticRegression() + ) + clf = BaggingClassifier(estimator=base_pipeline, max_samples=0.5, random_state=0) clf.fit(X, y) pipeline_estimator_coef = clf.estimators_[0].steps[-1][1].coef_.copy() @@ -779,10 +790,13 @@ def test_max_samples_consistency(): # Make sure validated max_samples and original max_samples are identical # when valid integer max_samples supplied by user max_samples = 100 - X, y = make_hastie_10_2(n_samples=2*max_samples, random_state=1) - bagging = BaggingClassifier(KNeighborsClassifier(), - max_samples=max_samples, - max_features=0.5, random_state=1) + X, y = make_hastie_10_2(n_samples=2 * max_samples, random_state=1) + bagging = BaggingClassifier( + KNeighborsClassifier(), + max_samples=max_samples, + max_features=0.5, + random_state=1, + ) bagging.fit(X, y) assert bagging._max_samples == max_samples @@ -792,48 +806,59 @@ def test_set_oob_score_label_encoding(): # See: https://github.com/scikit-learn/scikit-learn/issues/8933 random_state = 5 X = [[-1], [0], [1]] * 5 - Y1 = ['A', 'B', 'C'] * 5 + Y1 = ["A", "B", "C"] * 5 Y2 = [-1, 0, 1] * 5 Y3 = [0, 1, 2] * 5 - x1 = BaggingClassifier(oob_score=True, - random_state=random_state).fit(X, Y1).oob_score_ - x2 = BaggingClassifier(oob_score=True, - random_state=random_state).fit(X, Y2).oob_score_ - x3 = BaggingClassifier(oob_score=True, - random_state=random_state).fit(X, Y3).oob_score_ + x1 = ( + BaggingClassifier(oob_score=True, random_state=random_state) + .fit(X, Y1) + .oob_score_ + ) + x2 = ( + BaggingClassifier(oob_score=True, random_state=random_state) + .fit(X, Y2) + .oob_score_ + ) + x3 = ( + BaggingClassifier(oob_score=True, random_state=random_state) + .fit(X, Y3) + .oob_score_ + ) assert [x1, x2] == [x3, x3] def replace(X): - X = X.astype('float', copy=True) + X = X.astype("float", copy=True) X[~np.isfinite(X)] = 0 return X def test_bagging_regressor_with_missing_inputs(): # Check that BaggingRegressor can accept X with missing/infinite data - X = np.array([ - [1, 3, 5], - [2, None, 6], - [2, np.nan, 6], - [2, np.inf, 6], - [2, np.NINF, 6], - ]) + X = np.array( + [ + [1, 3, 5], + [2, None, 6], + [2, np.nan, 6], + [2, np.inf, 6], + [2, -np.inf, 6], + ] + ) y_values = [ np.array([2, 3, 3, 3, 3]), - np.array([ - [2, 1, 9], - [3, 6, 8], - [3, 6, 8], - [3, 6, 8], - [3, 6, 8], - ]) + np.array( + [ + [2, 1, 9], + [3, 6, 8], + [3, 6, 8], + [3, 6, 8], + [3, 6, 8], + ] + ), ] for y in y_values: regressor = DecisionTreeRegressor() - pipeline = make_pipeline( - FunctionTransformer(replace), regressor - ) + pipeline = make_pipeline(FunctionTransformer(replace), regressor) pipeline.fit(X, y).predict(X) bagging_regressor = BaggingRegressor(pipeline) y_hat = bagging_regressor.fit(X, y).predict(X) @@ -851,18 +876,18 @@ def test_bagging_regressor_with_missing_inputs(): def test_bagging_classifier_with_missing_inputs(): # Check that BaggingClassifier can accept X with missing/infinite data - X = np.array([ - [1, 3, 5], - [2, None, 6], - [2, np.nan, 6], - [2, np.inf, 6], - [2, np.NINF, 6], - ]) + X = np.array( + [ + [1, 3, 5], + [2, None, 6], + [2, np.nan, 6], + [2, np.inf, 6], + [2, -np.inf, 6], + ] + ) y = np.array([3, 6, 6, 6, 6]) classifier = DecisionTreeClassifier() - pipeline = make_pipeline( - FunctionTransformer(replace), classifier - ) + pipeline = make_pipeline(FunctionTransformer(replace), classifier) pipeline.fit(X, y).predict(X) bagging_classifier = BaggingClassifier(pipeline) bagging_classifier.fit(X, y) @@ -887,41 +912,132 @@ def test_bagging_small_max_features(): X = np.array([[1, 2], [3, 4]]) y = np.array([1, 0]) - bagging = BaggingClassifier(LogisticRegression(), - max_features=0.3, random_state=1) + bagging = BaggingClassifier(LogisticRegression(), max_features=0.3, random_state=1) bagging.fit(X, y) -def test_bagging_get_estimators_indices(): +def test_bagging_get_estimators_indices(global_random_seed): # Check that Bagging estimator can generate sample indices properly # Non-regression test for: # https://github.com/scikit-learn/scikit-learn/issues/16436 - rng = np.random.RandomState(0) + rng = np.random.RandomState(global_random_seed) X = rng.randn(13, 4) y = np.arange(13) class MyEstimator(DecisionTreeRegressor): """An estimator which stores y indices information at fit.""" + def fit(self, X, y): self._sample_indices = y - clf = BaggingRegressor(base_estimator=MyEstimator(), - n_estimators=1, random_state=0) + clf = BaggingRegressor(estimator=MyEstimator(), n_estimators=1, random_state=0) clf.fit(X, y) - assert_array_equal(clf.estimators_[0]._sample_indices, - clf.estimators_samples_[0]) + assert_array_equal(clf.estimators_[0]._sample_indices, clf.estimators_samples_[0]) -# FIXME: remove in 1.2 -@pytest.mark.parametrize("Estimator", [BaggingClassifier, BaggingRegressor]) -def test_n_features_deprecation(Estimator): - # Check that we raise the proper deprecation warning if accessing - # `n_features_`. - X = np.array([[1, 2], [3, 4]]) - y = np.array([1, 0]) - est = Estimator().fit(X, y) +@pytest.mark.parametrize( + "bagging, expected_allow_nan", + [ + (BaggingClassifier(HistGradientBoostingClassifier(max_iter=1)), True), + (BaggingRegressor(HistGradientBoostingRegressor(max_iter=1)), True), + (BaggingClassifier(LogisticRegression()), False), + (BaggingRegressor(SVR()), False), + ], +) +def test_bagging_allow_nan_tag(bagging, expected_allow_nan): + """Check that bagging inherits allow_nan tag.""" + assert bagging.__sklearn_tags__().input_tags.allow_nan == expected_allow_nan + + +# Metadata Routing Tests +# ====================== + + +@config_context(enable_metadata_routing=True) +@pytest.mark.parametrize( + "model", + [ + BaggingClassifier( + estimator=RandomForestClassifier(n_estimators=1), n_estimators=1 + ), + BaggingRegressor( + estimator=RandomForestRegressor(n_estimators=1), n_estimators=1 + ), + ], +) +def test_bagging_with_metadata_routing(model): + """Make sure that metadata routing works with non-default estimator.""" + model.fit(iris.data, iris.target) + + +@pytest.mark.parametrize( + "sub_estimator, caller, callee", + [ + (ConsumingClassifierWithoutPredictProba, "predict", "predict"), + ( + ConsumingClassifierWithoutPredictLogProba, + "predict_log_proba", + "predict_proba", + ), + (ConsumingClassifierWithOnlyPredict, "predict_log_proba", "predict"), + ], +) +@config_context(enable_metadata_routing=True) +def test_metadata_routing_with_dynamic_method_selection(sub_estimator, caller, callee): + """Test that metadata routing works in `BaggingClassifier` with dynamic selection of + the sub-estimator's methods. Here we test only specific test cases, where + sub-estimator methods are not present and are not tested with `ConsumingClassifier` + (which possesses all the methods) in + sklearn/tests/test_metaestimators_metadata_routing.py: `BaggingClassifier.predict()` + dynamically routes to `predict` if the sub-estimator doesn't have `predict_proba` + and `BaggingClassifier.predict_log_proba()` dynamically routes to `predict_proba` if + the sub-estimator doesn't have `predict_log_proba`, or to `predict`, if it doesn't + have it. + """ + X = np.array([[0, 2], [1, 4], [2, 6]]) + y = [1, 2, 3] + sample_weight, metadata = [1], "a" + registry = _Registry() + estimator = sub_estimator(registry=registry) + set_callee_request = "set_" + callee + "_request" + getattr(estimator, set_callee_request)(sample_weight=True, metadata=True) + + bagging = BaggingClassifier(estimator=estimator) + bagging.fit(X, y) + getattr(bagging, caller)( + X=np.array([[1, 1], [1, 3], [0, 2]]), + sample_weight=sample_weight, + metadata=metadata, + ) + + assert len(registry) + for estimator in registry: + check_recorded_metadata( + obj=estimator, + method=callee, + parent=caller, + sample_weight=sample_weight, + metadata=metadata, + ) + - with pytest.warns(FutureWarning, match="n_features_ was deprecated"): - est.n_features_ +# End of Metadata Routing Tests +# ============================= + + +@pytest.mark.parametrize( + "model", + [ + BaggingClassifier( + estimator=AdaBoostClassifier(n_estimators=1), + n_estimators=1, + ), + BaggingRegressor(estimator=AdaBoostRegressor(n_estimators=1), n_estimators=1), + ], +) +def test_bagging_without_support_metadata_routing(model): + """Make sure that we still can use an estimator that does not implement the + metadata routing.""" + model.fit(iris.data, iris.target) diff --git a/sklearn/ensemble/tests/test_base.py b/sklearn/ensemble/tests/test_base.py index 3c5b7564380c6..95a852b8a7cc5 100644 --- a/sklearn/ensemble/tests/test_base.py +++ b/sklearn/ensemble/tests/test_base.py @@ -2,26 +2,27 @@ Testing for the base module (sklearn.ensemble.base). """ -# Authors: Gilles Louppe -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from collections import OrderedDict import numpy as np -import pytest from sklearn.datasets import load_iris +from sklearn.discriminant_analysis import LinearDiscriminantAnalysis from sklearn.ensemble import BaggingClassifier from sklearn.ensemble._base import _set_random_states +from sklearn.feature_selection import SelectFromModel from sklearn.linear_model import Perceptron -from collections import OrderedDict -from sklearn.discriminant_analysis import LinearDiscriminantAnalysis from sklearn.pipeline import Pipeline -from sklearn.feature_selection import SelectFromModel def test_base(): # Check BaseEnsemble methods. ensemble = BaggingClassifier( - base_estimator=Perceptron(random_state=None), n_estimators=3) + estimator=Perceptron(random_state=None), n_estimators=3 + ) iris = load_iris() ensemble.fit(iris.data, iris.target) @@ -42,36 +43,12 @@ def test_base(): assert isinstance(ensemble[2].random_state, int) assert ensemble[1].random_state != ensemble[2].random_state - np_int_ensemble = BaggingClassifier(base_estimator=Perceptron(), - n_estimators=np.int32(3)) + np_int_ensemble = BaggingClassifier( + estimator=Perceptron(), n_estimators=np.int32(3) + ) np_int_ensemble.fit(iris.data, iris.target) -def test_base_zero_n_estimators(): - # Check that instantiating a BaseEnsemble with n_estimators<=0 raises - # a ValueError. - ensemble = BaggingClassifier(base_estimator=Perceptron(), - n_estimators=0) - iris = load_iris() - err_msg = "n_estimators must be greater than zero, got 0." - with pytest.raises(ValueError, match=err_msg): - ensemble.fit(iris.data, iris.target) - - -def test_base_not_int_n_estimators(): - # Check that instantiating a BaseEnsemble with a string as n_estimators - # raises a ValueError demanding n_estimators to be supplied as an integer. - string_ensemble = BaggingClassifier(base_estimator=Perceptron(), - n_estimators='3') - iris = load_iris() - with pytest.raises(ValueError, match="n_estimators must be an integer"): - string_ensemble.fit(iris.data, iris.target) - float_ensemble = BaggingClassifier(base_estimator=Perceptron(), - n_estimators=3.0) - with pytest.raises(ValueError, match="n_estimators must be an integer"): - float_ensemble.fit(iris.data, iris.target) - - def test_set_random_states(): # Linear Discriminant Analysis doesn't have random state: smoke test _set_random_states(LinearDiscriminantAnalysis(), random_state=17) @@ -92,15 +69,19 @@ def test_set_random_states(): # nested random_state def make_steps(): - return [('sel', SelectFromModel(Perceptron(random_state=None))), - ('clf', Perceptron(random_state=None))] + return [ + ("sel", SelectFromModel(Perceptron(random_state=None))), + ("clf", Perceptron(random_state=None)), + ] est1 = Pipeline(make_steps()) _set_random_states(est1, 3) assert isinstance(est1.steps[0][1].estimator.random_state, int) assert isinstance(est1.steps[1][1].random_state, int) - assert (est1.get_params()['sel__estimator__random_state'] != - est1.get_params()['clf__random_state']) + assert ( + est1.get_params()["sel__estimator__random_state"] + != est1.get_params()["clf__random_state"] + ) # ensure multiple random_state parameters are invariant to get_params() # iteration order @@ -118,7 +99,11 @@ def get_params(self, *args, **kwargs): for cls in [AlphaParamPipeline, RevParamPipeline]: est2 = cls(make_steps()) _set_random_states(est2, 3) - assert (est1.get_params()['sel__estimator__random_state'] == - est2.get_params()['sel__estimator__random_state']) - assert (est1.get_params()['clf__random_state'] == - est2.get_params()['clf__random_state']) + assert ( + est1.get_params()["sel__estimator__random_state"] + == est2.get_params()["sel__estimator__random_state"] + ) + assert ( + est1.get_params()["clf__random_state"] + == est2.get_params()["clf__random_state"] + ) diff --git a/sklearn/ensemble/tests/test_common.py b/sklearn/ensemble/tests/test_common.py index b8a34b4188802..6e83512ccd1d6 100644 --- a/sklearn/ensemble/tests/test_common.py +++ b/sklearn/ensemble/tests/test_common.py @@ -1,21 +1,25 @@ import numpy as np import pytest -from sklearn.base import clone -from sklearn.base import ClassifierMixin -from sklearn.base import is_classifier - -from sklearn.datasets import make_classification -from sklearn.datasets import make_regression -from sklearn.datasets import load_iris, load_diabetes +from sklearn.base import ClassifierMixin, clone, is_classifier +from sklearn.datasets import ( + load_diabetes, + load_iris, + make_classification, + make_regression, +) +from sklearn.ensemble import ( + RandomForestClassifier, + RandomForestRegressor, + StackingClassifier, + StackingRegressor, + VotingClassifier, + VotingRegressor, +) from sklearn.impute import SimpleImputer -from sklearn.linear_model import LogisticRegression, LinearRegression -from sklearn.svm import LinearSVC, LinearSVR, SVC, SVR +from sklearn.linear_model import LinearRegression, LogisticRegression from sklearn.pipeline import make_pipeline -from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor - -from sklearn.ensemble import StackingClassifier, StackingRegressor -from sklearn.ensemble import VotingClassifier, VotingRegressor +from sklearn.svm import SVC, SVR, LinearSVC, LinearSVR X, y = load_iris(return_X_y=True) @@ -24,24 +28,56 @@ @pytest.mark.parametrize( "X, y, estimator", - [(*make_classification(n_samples=10), - StackingClassifier(estimators=[('lr', LogisticRegression()), - ('svm', LinearSVC()), - ('rf', RandomForestClassifier())])), - (*make_classification(n_samples=10), - VotingClassifier(estimators=[('lr', LogisticRegression()), - ('svm', LinearSVC()), - ('rf', RandomForestClassifier())])), - (*make_regression(n_samples=10), - StackingRegressor(estimators=[('lr', LinearRegression()), - ('svm', LinearSVR()), - ('rf', RandomForestRegressor())])), - (*make_regression(n_samples=10), - VotingRegressor(estimators=[('lr', LinearRegression()), - ('svm', LinearSVR()), - ('rf', RandomForestRegressor())]))], - ids=['stacking-classifier', 'voting-classifier', - 'stacking-regressor', 'voting-regressor'] + [ + ( + *make_classification(n_samples=10), + StackingClassifier( + estimators=[ + ("lr", LogisticRegression()), + ("svm", LinearSVC()), + ("rf", RandomForestClassifier(n_estimators=5, max_depth=3)), + ], + cv=2, + ), + ), + ( + *make_classification(n_samples=10), + VotingClassifier( + estimators=[ + ("lr", LogisticRegression()), + ("svm", LinearSVC()), + ("rf", RandomForestClassifier(n_estimators=5, max_depth=3)), + ] + ), + ), + ( + *make_regression(n_samples=10), + StackingRegressor( + estimators=[ + ("lr", LinearRegression()), + ("svm", LinearSVR()), + ("rf", RandomForestRegressor(n_estimators=5, max_depth=3)), + ], + cv=2, + ), + ), + ( + *make_regression(n_samples=10), + VotingRegressor( + estimators=[ + ("lr", LinearRegression()), + ("svm", LinearSVR()), + ("rf", RandomForestRegressor(n_estimators=5, max_depth=3)), + ] + ), + ), + ], + ids=[ + "stacking-classifier", + "voting-classifier", + "stacking-regressor", + "voting-regressor", + ], ) def test_ensemble_heterogeneous_estimators_behavior(X, y, estimator): # check that the behavior of `estimators`, `estimators_`, @@ -49,36 +85,42 @@ def test_ensemble_heterogeneous_estimators_behavior(X, y, estimator): # ensemble classes and when using `set_params()`. # before fit - assert 'svm' in estimator.named_estimators + assert "svm" in estimator.named_estimators assert estimator.named_estimators.svm is estimator.estimators[1][1] - assert estimator.named_estimators.svm is estimator.named_estimators['svm'] + assert estimator.named_estimators.svm is estimator.named_estimators["svm"] # check fitted attributes estimator.fit(X, y) assert len(estimator.named_estimators) == 3 assert len(estimator.named_estimators_) == 3 - assert (sorted(list(estimator.named_estimators_.keys())) == - sorted(['lr', 'svm', 'rf'])) + assert sorted(list(estimator.named_estimators_.keys())) == sorted( + ["lr", "svm", "rf"] + ) # check that set_params() does not add a new attribute estimator_new_params = clone(estimator) svm_estimator = SVC() if is_classifier(estimator) else SVR() estimator_new_params.set_params(svm=svm_estimator).fit(X, y) - assert not hasattr(estimator_new_params, 'svm') - assert (estimator_new_params.named_estimators.lr.get_params() == - estimator.named_estimators.lr.get_params()) - assert (estimator_new_params.named_estimators.rf.get_params() == - estimator.named_estimators.rf.get_params()) + assert not hasattr(estimator_new_params, "svm") + assert ( + estimator_new_params.named_estimators.lr.get_params() + == estimator.named_estimators.lr.get_params() + ) + assert ( + estimator_new_params.named_estimators.rf.get_params() + == estimator.named_estimators.rf.get_params() + ) # check the behavior when setting an dropping an estimator estimator_dropped = clone(estimator) - estimator_dropped.set_params(svm='drop') + estimator_dropped.set_params(svm="drop") estimator_dropped.fit(X, y) assert len(estimator_dropped.named_estimators) == 3 - assert estimator_dropped.named_estimators.svm == 'drop' + assert estimator_dropped.named_estimators.svm == "drop" assert len(estimator_dropped.named_estimators_) == 3 - assert (sorted(list(estimator_dropped.named_estimators_.keys())) == - sorted(['lr', 'svm', 'rf'])) + assert sorted(list(estimator_dropped.named_estimators_.keys())) == sorted( + ["lr", "svm", "rf"] + ) for sub_est in estimator_dropped.named_estimators_: # check that the correspondence is correct assert not isinstance(sub_est, type(estimator.named_estimators.svm)) @@ -86,27 +128,32 @@ def test_ensemble_heterogeneous_estimators_behavior(X, y, estimator): # check that we can set the parameters of the underlying classifier estimator.set_params(svm__C=10.0) estimator.set_params(rf__max_depth=5) - assert (estimator.get_params()['svm__C'] == - estimator.get_params()['svm'].get_params()['C']) - assert (estimator.get_params()['rf__max_depth'] == - estimator.get_params()['rf'].get_params()['max_depth']) + assert ( + estimator.get_params()["svm__C"] + == estimator.get_params()["svm"].get_params()["C"] + ) + assert ( + estimator.get_params()["rf__max_depth"] + == estimator.get_params()["rf"].get_params()["max_depth"] + ) @pytest.mark.parametrize( "Ensemble", - [StackingClassifier, VotingClassifier, StackingRegressor, VotingRegressor] + [VotingClassifier, StackingRegressor, VotingRegressor], ) def test_ensemble_heterogeneous_estimators_type(Ensemble): # check that ensemble will fail during validation if the underlying # estimators are not of the same type (i.e. classifier or regressor) + # StackingClassifier can have an underlying regresor so it's not checked if issubclass(Ensemble, ClassifierMixin): X, y = make_classification(n_samples=10) - estimators = [('lr', LinearRegression())] - ensemble_type = 'classifier' + estimators = [("lr", LinearRegression())] + ensemble_type = "classifier" else: X, y = make_regression(n_samples=10) - estimators = [('lr', LogisticRegression())] - ensemble_type = 'regressor' + estimators = [("lr", LogisticRegression())] + ensemble_type = "regressor" ensemble = Ensemble(estimators=estimators) err_msg = "should be a {}".format(ensemble_type) @@ -116,17 +163,19 @@ def test_ensemble_heterogeneous_estimators_type(Ensemble): @pytest.mark.parametrize( "X, y, Ensemble", - [(*make_classification(n_samples=10), StackingClassifier), - (*make_classification(n_samples=10), VotingClassifier), - (*make_regression(n_samples=10), StackingRegressor), - (*make_regression(n_samples=10), VotingRegressor)] + [ + (*make_classification(n_samples=10), StackingClassifier), + (*make_classification(n_samples=10), VotingClassifier), + (*make_regression(n_samples=10), StackingRegressor), + (*make_regression(n_samples=10), VotingRegressor), + ], ) def test_ensemble_heterogeneous_estimators_name_validation(X, y, Ensemble): # raise an error when the name contains dunder if issubclass(Ensemble, ClassifierMixin): - estimators = [('lr__', LogisticRegression())] + estimators = [("lr__", LogisticRegression())] else: - estimators = [('lr__', LinearRegression())] + estimators = [("lr__", LinearRegression())] ensemble = Ensemble(estimators=estimators) err_msg = r"Estimator names must not contain __: got \['lr__'\]" @@ -135,11 +184,9 @@ def test_ensemble_heterogeneous_estimators_name_validation(X, y, Ensemble): # raise an error when the name is not unique if issubclass(Ensemble, ClassifierMixin): - estimators = [('lr', LogisticRegression()), - ('lr', LogisticRegression())] + estimators = [("lr", LogisticRegression()), ("lr", LogisticRegression())] else: - estimators = [('lr', LinearRegression()), - ('lr', LinearRegression())] + estimators = [("lr", LinearRegression()), ("lr", LinearRegression())] ensemble = Ensemble(estimators=estimators) err_msg = r"Names provided are not unique: \['lr', 'lr'\]" @@ -148,9 +195,9 @@ def test_ensemble_heterogeneous_estimators_name_validation(X, y, Ensemble): # raise an error when the name conflicts with the parameters if issubclass(Ensemble, ClassifierMixin): - estimators = [('estimators', LogisticRegression())] + estimators = [("estimators", LogisticRegression())] else: - estimators = [('estimators', LinearRegression())] + estimators = [("estimators", LinearRegression())] ensemble = Ensemble(estimators=estimators) err_msg = "Estimator names conflict with constructor arguments" @@ -160,45 +207,56 @@ def test_ensemble_heterogeneous_estimators_name_validation(X, y, Ensemble): @pytest.mark.parametrize( "X, y, estimator", - [(*make_classification(n_samples=10), - StackingClassifier(estimators=[('lr', LogisticRegression())])), - (*make_classification(n_samples=10), - VotingClassifier(estimators=[('lr', LogisticRegression())])), - (*make_regression(n_samples=10), - StackingRegressor(estimators=[('lr', LinearRegression())])), - (*make_regression(n_samples=10), - VotingRegressor(estimators=[('lr', LinearRegression())]))], - ids=['stacking-classifier', 'voting-classifier', - 'stacking-regressor', 'voting-regressor'] + [ + ( + *make_classification(n_samples=10), + StackingClassifier(estimators=[("lr", LogisticRegression())]), + ), + ( + *make_classification(n_samples=10), + VotingClassifier(estimators=[("lr", LogisticRegression())]), + ), + ( + *make_regression(n_samples=10), + StackingRegressor(estimators=[("lr", LinearRegression())]), + ), + ( + *make_regression(n_samples=10), + VotingRegressor(estimators=[("lr", LinearRegression())]), + ), + ], + ids=[ + "stacking-classifier", + "voting-classifier", + "stacking-regressor", + "voting-regressor", + ], ) def test_ensemble_heterogeneous_estimators_all_dropped(X, y, estimator): # check that we raise a consistent error when all estimators are # dropped - estimator.set_params(lr='drop') + estimator.set_params(lr="drop") with pytest.raises(ValueError, match="All estimators are dropped."): estimator.fit(X, y) @pytest.mark.parametrize( - "Ensemble, Estimator, X, y", - [(StackingClassifier, LogisticRegression, - X, y), - (StackingRegressor, LinearRegression, - X_r, y_r), - (VotingClassifier, LogisticRegression, - X, y), - (VotingRegressor, LinearRegression, - X_r, y_r)] - ) + "Ensemble, Estimator, X, y", + [ + (StackingClassifier, LogisticRegression, X, y), + (StackingRegressor, LinearRegression, X_r, y_r), + (VotingClassifier, LogisticRegression, X, y), + (VotingRegressor, LinearRegression, X_r, y_r), + ], +) # FIXME: we should move this test in `estimator_checks` once we are able # to construct meta-estimator instances -def test_heterogeneous_ensemble_support_missing_values(Ensemble, - Estimator, X, y): +def test_heterogeneous_ensemble_support_missing_values(Ensemble, Estimator, X, y): # check that Voting and Stacking predictor delegate the missing values # validation to the underlying estimator. X = X.copy() - mask = np.random.choice([1, 0], X.shape, p=[.1, .9]).astype(bool) + mask = np.random.choice([1, 0], X.shape, p=[0.1, 0.9]).astype(bool) X[mask] = np.nan pipe = make_pipeline(SimpleImputer(), Estimator()) - ensemble = Ensemble(estimators=[('pipe1', pipe), ('pipe2', pipe)]) + ensemble = Ensemble(estimators=[("pipe1", pipe), ("pipe2", pipe)]) ensemble.fit(X, y).score(X, y) diff --git a/sklearn/ensemble/tests/test_forest.py b/sklearn/ensemble/tests/test_forest.py index efb1a645842bc..5dec5c7ab90b2 100644 --- a/sklearn/ensemble/tests/test_forest.py +++ b/sklearn/ensemble/tests/test_forest.py @@ -2,55 +2,62 @@ Testing for the forest module (sklearn.ensemble.forest). """ -# Authors: Gilles Louppe, -# Brian Holt, -# Andreas Mueller, -# Arnaud Joly -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import pickle +import itertools import math +import pickle from collections import defaultdict -import itertools -from itertools import combinations -from itertools import product -from typing import Dict, Any +from functools import partial +from itertools import combinations, product +from typing import Any, Dict +from unittest.mock import patch +import joblib import numpy as np -from scipy.sparse import csr_matrix -from scipy.sparse import csc_matrix -from scipy.sparse import coo_matrix -from scipy.special import comb - import pytest +from scipy.special import comb -import joblib - -from sklearn.utils._testing import assert_almost_equal -from sklearn.utils._testing import assert_array_almost_equal -from sklearn.utils._testing import assert_array_equal -from sklearn.utils._testing import _convert_container -from sklearn.utils._testing import ignore_warnings -from sklearn.utils._testing import skip_if_no_parallel -from sklearn.utils.fixes import parse_version - -from sklearn.exceptions import NotFittedError - -from sklearn import datasets +import sklearn +from sklearn import clone, datasets +from sklearn.datasets import make_classification, make_hastie_10_2 from sklearn.decomposition import TruncatedSVD -from sklearn.datasets import make_classification -from sklearn.ensemble import ExtraTreesClassifier -from sklearn.ensemble import ExtraTreesRegressor -from sklearn.ensemble import RandomForestClassifier -from sklearn.ensemble import RandomForestRegressor -from sklearn.ensemble import RandomTreesEmbedding -from sklearn.model_selection import train_test_split -from sklearn.model_selection import GridSearchCV +from sklearn.dummy import DummyRegressor +from sklearn.ensemble import ( + ExtraTreesClassifier, + ExtraTreesRegressor, + RandomForestClassifier, + RandomForestRegressor, + RandomTreesEmbedding, +) +from sklearn.ensemble._forest import ( + _generate_unsampled_indices, + _get_n_samples_bootstrap, +) +from sklearn.exceptions import NotFittedError +from sklearn.metrics import ( + explained_variance_score, + f1_score, + mean_poisson_deviance, + mean_squared_error, +) +from sklearn.model_selection import GridSearchCV, cross_val_score, train_test_split from sklearn.svm import LinearSVC -from sklearn.utils.validation import check_random_state - from sklearn.tree._classes import SPARSE_SPLITTERS - +from sklearn.utils._testing import ( + _convert_container, + assert_allclose, + assert_almost_equal, + assert_array_almost_equal, + assert_array_equal, + ignore_warnings, + skip_if_no_parallel, +) +from sklearn.utils.fixes import COO_CONTAINERS, CSC_CONTAINERS, CSR_CONTAINERS +from sklearn.utils.multiclass import type_of_target +from sklearn.utils.parallel import Parallel +from sklearn.utils.validation import check_random_state # toy sample X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]] @@ -60,8 +67,14 @@ # Larger classification sample used for testing feature importances X_large, y_large = datasets.make_classification( - n_samples=500, n_features=10, n_informative=3, n_redundant=0, - n_repeated=0, shuffle=False, random_state=0) + n_samples=500, + n_features=10, + n_informative=3, + n_redundant=0, + n_repeated=0, + shuffle=False, + random_state=0, +) # also load the iris dataset # and randomly permute it @@ -72,8 +85,7 @@ iris.target = iris.target[perm] # Make regression dataset -X_reg, y_reg = datasets.make_regression(n_samples=500, n_features=10, - random_state=1) +X_reg, y_reg = datasets.make_regression(n_samples=500, n_features=10, random_state=1) # also make a hastie_10_2 dataset hastie_X, hastie_y = datasets.make_hastie_10_2(n_samples=20, random_state=1) @@ -106,7 +118,8 @@ FOREST_CLASSIFIERS_REGRESSORS.update(FOREST_REGRESSORS) -def check_classification_toy(name): +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS) +def test_classification_toy(name): """Check classification on a toy dataset.""" ForestClassifier = FOREST_CLASSIFIERS[name] @@ -125,62 +138,130 @@ def check_classification_toy(name): assert leaf_indices.shape == (len(X), clf.n_estimators) -@pytest.mark.parametrize('name', FOREST_CLASSIFIERS) -def test_classification_toy(name): - check_classification_toy(name) - - -def check_iris_criterion(name, criterion): +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS) +@pytest.mark.parametrize("criterion", ("gini", "log_loss")) +def test_iris_criterion(name, criterion): # Check consistency on dataset iris. ForestClassifier = FOREST_CLASSIFIERS[name] - clf = ForestClassifier(n_estimators=10, criterion=criterion, - random_state=1) + clf = ForestClassifier(n_estimators=10, criterion=criterion, random_state=1) clf.fit(iris.data, iris.target) score = clf.score(iris.data, iris.target) - assert score > 0.9, ("Failed with criterion %s and score = %f" - % (criterion, score)) + assert score > 0.9, "Failed with criterion %s and score = %f" % (criterion, score) - clf = ForestClassifier(n_estimators=10, criterion=criterion, - max_features=2, random_state=1) + clf = ForestClassifier( + n_estimators=10, criterion=criterion, max_features=2, random_state=1 + ) clf.fit(iris.data, iris.target) score = clf.score(iris.data, iris.target) - assert score > 0.5, ("Failed with criterion %s and score = %f" - % (criterion, score)) - - -@pytest.mark.parametrize('name', FOREST_CLASSIFIERS) -@pytest.mark.parametrize('criterion', ("gini", "entropy")) -def test_iris(name, criterion): - check_iris_criterion(name, criterion) + assert score > 0.5, "Failed with criterion %s and score = %f" % (criterion, score) -def check_regression_criterion(name, criterion): +@pytest.mark.parametrize("name", FOREST_REGRESSORS) +@pytest.mark.parametrize( + "criterion", ("squared_error", "absolute_error", "friedman_mse") +) +def test_regression_criterion(name, criterion): # Check consistency on regression dataset. ForestRegressor = FOREST_REGRESSORS[name] - reg = ForestRegressor(n_estimators=5, criterion=criterion, - random_state=1) + reg = ForestRegressor(n_estimators=5, criterion=criterion, random_state=1) reg.fit(X_reg, y_reg) score = reg.score(X_reg, y_reg) - assert score > 0.93, ("Failed with max_features=None, criterion %s " - "and score = %f" % (criterion, score)) + assert score > 0.93, ( + "Failed with max_features=None, criterion %s and score = %f" + % ( + criterion, + score, + ) + ) - reg = ForestRegressor(n_estimators=5, criterion=criterion, - max_features=6, random_state=1) + reg = ForestRegressor( + n_estimators=5, criterion=criterion, max_features=6, random_state=1 + ) reg.fit(X_reg, y_reg) score = reg.score(X_reg, y_reg) - assert score > 0.92, ("Failed with max_features=6, criterion %s " - "and score = %f" % (criterion, score)) + assert score > 0.92, "Failed with max_features=6, criterion %s and score = %f" % ( + criterion, + score, + ) + + +def test_poisson_vs_mse(): + """Test that random forest with poisson criterion performs better than + mse for a poisson target. + + There is a similar test for DecisionTreeRegressor. + """ + rng = np.random.RandomState(42) + n_train, n_test, n_features = 500, 500, 10 + X = datasets.make_low_rank_matrix( + n_samples=n_train + n_test, n_features=n_features, random_state=rng + ) + # We create a log-linear Poisson model and downscale coef as it will get + # exponentiated. + coef = rng.uniform(low=-2, high=2, size=n_features) / np.max(X, axis=0) + y = rng.poisson(lam=np.exp(X @ coef)) + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=n_test, random_state=rng + ) + # We prevent some overfitting by setting min_samples_split=10. + forest_poi = RandomForestRegressor( + criterion="poisson", min_samples_leaf=10, max_features="sqrt", random_state=rng + ) + forest_mse = RandomForestRegressor( + criterion="squared_error", + min_samples_leaf=10, + max_features="sqrt", + random_state=rng, + ) + + forest_poi.fit(X_train, y_train) + forest_mse.fit(X_train, y_train) + dummy = DummyRegressor(strategy="mean").fit(X_train, y_train) + + for X, y, data_name in [(X_train, y_train, "train"), (X_test, y_test, "test")]: + metric_poi = mean_poisson_deviance(y, forest_poi.predict(X)) + # squared_error forest might produce non-positive predictions => clip + # If y = 0 for those, the poisson deviance gets too good. + # If we drew more samples, we would eventually get y > 0 and the + # poisson deviance would explode, i.e. be undefined. Therefore, we do + # not clip to a tiny value like 1e-15, but to 1e-6. This acts like a + # small penalty to the non-positive predictions. + metric_mse = mean_poisson_deviance( + y, np.clip(forest_mse.predict(X), 1e-6, None) + ) + metric_dummy = mean_poisson_deviance(y, dummy.predict(X)) + # As squared_error might correctly predict 0 in train set, its train + # score can be better than Poisson. This is no longer the case for the + # test set. But keep the above comment for clipping in mind. + if data_name == "test": + assert metric_poi < metric_mse + assert metric_poi < 0.8 * metric_dummy + + +@pytest.mark.parametrize("criterion", ("poisson", "squared_error")) +def test_balance_property_random_forest(criterion): + """ "Test that sum(y_pred)==sum(y_true) on the training set.""" + rng = np.random.RandomState(42) + n_train, n_test, n_features = 500, 500, 10 + X = datasets.make_low_rank_matrix( + n_samples=n_train + n_test, n_features=n_features, random_state=rng + ) + + coef = rng.uniform(low=-2, high=2, size=n_features) / np.max(X, axis=0) + y = rng.poisson(lam=np.exp(X @ coef)) + reg = RandomForestRegressor( + criterion=criterion, n_estimators=10, bootstrap=False, random_state=rng + ) + reg.fit(X, y) -@pytest.mark.parametrize('name', FOREST_REGRESSORS) -@pytest.mark.parametrize('criterion', ("mse", "mae", "friedman_mse")) -def test_regression(name, criterion): - check_regression_criterion(name, criterion) + assert np.sum(reg.predict(X)) == pytest.approx(np.sum(y)) -def check_regressor_attributes(name): +@pytest.mark.parametrize("name", FOREST_REGRESSORS) +def test_regressor_attributes(name): # Regression models should not have a classes_ attribute. r = FOREST_REGRESSORS[name](random_state=0) assert not hasattr(r, "classes_") @@ -191,38 +272,43 @@ def check_regressor_attributes(name): assert not hasattr(r, "n_classes_") -@pytest.mark.parametrize('name', FOREST_REGRESSORS) -def test_regressor_attributes(name): - check_regressor_attributes(name) - - -def check_probability(name): +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS) +def test_probability(name): # Predict probabilities. ForestClassifier = FOREST_CLASSIFIERS[name] with np.errstate(divide="ignore"): - clf = ForestClassifier(n_estimators=10, random_state=1, max_features=1, - max_depth=1) + clf = ForestClassifier( + n_estimators=10, random_state=1, max_features=1, max_depth=1 + ) clf.fit(iris.data, iris.target) - assert_array_almost_equal(np.sum(clf.predict_proba(iris.data), axis=1), - np.ones(iris.data.shape[0])) - assert_array_almost_equal(clf.predict_proba(iris.data), - np.exp(clf.predict_log_proba(iris.data))) + assert_array_almost_equal( + np.sum(clf.predict_proba(iris.data), axis=1), np.ones(iris.data.shape[0]) + ) + assert_array_almost_equal( + clf.predict_proba(iris.data), np.exp(clf.predict_log_proba(iris.data)) + ) -@pytest.mark.parametrize('name', FOREST_CLASSIFIERS) -def test_probability(name): - check_probability(name) - +@pytest.mark.parametrize("dtype", (np.float64, np.float32)) +@pytest.mark.parametrize( + "name, criterion", + itertools.chain( + product(FOREST_CLASSIFIERS, ["gini", "log_loss"]), + product(FOREST_REGRESSORS, ["squared_error", "friedman_mse", "absolute_error"]), + ), +) +def test_importances(dtype, name, criterion): + tolerance = 0.01 + if name in FOREST_REGRESSORS and criterion == "absolute_error": + tolerance = 0.05 -def check_importances(name, criterion, dtype, tolerance): - # cast as dype + # cast as dtype X = X_large.astype(dtype, copy=False) y = y_large.astype(dtype, copy=False) ForestEstimator = FOREST_ESTIMATORS[name] - est = ForestEstimator(n_estimators=10, criterion=criterion, - random_state=0) + est = ForestEstimator(n_estimators=10, criterion=criterion, random_state=0) est.fit(X, y) importances = est.feature_importances_ @@ -247,27 +333,12 @@ def check_importances(name, criterion, dtype, tolerance): assert np.all(importances >= 0.0) for scale in [0.5, 100]: - est = ForestEstimator(n_estimators=10, random_state=0, - criterion=criterion) + est = ForestEstimator(n_estimators=10, random_state=0, criterion=criterion) est.fit(X, y, sample_weight=scale * sample_weight) importances_bis = est.feature_importances_ assert np.abs(importances - importances_bis).mean() < tolerance -@pytest.mark.parametrize('dtype', (np.float64, np.float32)) -@pytest.mark.parametrize( - 'name, criterion', - itertools.chain(product(FOREST_CLASSIFIERS, - ["gini", "entropy"]), - product(FOREST_REGRESSORS, - ["mse", "friedman_mse", "mae"]))) -def test_importances(dtype, name, criterion): - tolerance = 0.01 - if name in FOREST_REGRESSORS and criterion == "mae": - tolerance = 0.05 - check_importances(name, criterion, dtype, tolerance) - - def test_importances_asymptotic(): # Check whether variable importances of totally randomized trees # converge towards their theoretical values (See Louppe et al, @@ -278,10 +349,10 @@ def binomial(k, n): def entropy(samples): n_samples = len(samples) - entropy = 0. + entropy = 0.0 for count in np.bincount(samples): - p = 1. * count / n_samples + p = 1.0 * count / n_samples if p > 0: entropy -= p * np.log2(p) @@ -294,11 +365,11 @@ def mdi_importance(X_m, X, y): features.pop(X_m) values = [np.unique(X[:, i]) for i in range(n_features)] - imp = 0. + imp = 0.0 for k in range(n_features): # Weight of each B of size k - coef = 1. / (binomial(k, n_features) * (n_features - k)) + coef = 1.0 / (binomial(k, n_features) * (n_features - k)) # For all B of size k for B in combinations(features, k): @@ -319,24 +390,36 @@ def mdi_importance(X_m, X, y): mask_xi = X_[:, X_m] == xi children.append(y_[mask_xi]) - imp += (coef - * (1. * n_samples_b / n_samples) # P(B=b) - * (entropy(y_) - - sum([entropy(c) * len(c) / n_samples_b - for c in children]))) + imp += ( + coef + * (1.0 * n_samples_b / n_samples) # P(B=b) + * ( + entropy(y_) + - sum( + [ + entropy(c) * len(c) / n_samples_b + for c in children + ] + ) + ) + ) return imp - data = np.array([[0, 0, 1, 0, 0, 1, 0, 1], - [1, 0, 1, 1, 1, 0, 1, 2], - [1, 0, 1, 1, 0, 1, 1, 3], - [0, 1, 1, 1, 0, 1, 0, 4], - [1, 1, 0, 1, 0, 1, 1, 5], - [1, 1, 0, 1, 1, 1, 1, 6], - [1, 0, 1, 0, 0, 1, 0, 7], - [1, 1, 1, 1, 1, 1, 1, 8], - [1, 1, 1, 1, 0, 1, 1, 9], - [1, 1, 1, 0, 1, 1, 1, 0]]) + data = np.array( + [ + [0, 0, 1, 0, 0, 1, 0, 1], + [1, 0, 1, 1, 1, 0, 1, 2], + [1, 0, 1, 1, 0, 1, 1, 3], + [0, 1, 1, 1, 0, 1, 0, 4], + [1, 1, 0, 1, 0, 1, 1, 5], + [1, 1, 0, 1, 1, 1, 1, 6], + [1, 0, 1, 0, 0, 1, 0, 7], + [1, 1, 1, 1, 1, 1, 1, 8], + [1, 1, 1, 1, 0, 1, 1, 9], + [1, 1, 1, 0, 1, 1, 1, 0], + ] + ) X, y = np.array(data[:, :7], dtype=bool), data[:, 7] n_features = X.shape[1] @@ -348,26 +431,31 @@ def mdi_importance(X_m, X, y): true_importances[i] = mdi_importance(i, X, y) # Estimate importances with totally randomized trees - clf = ExtraTreesClassifier(n_estimators=500, - max_features=1, - criterion="entropy", - random_state=0).fit(X, y) - - importances = sum(tree.tree_.compute_feature_importances(normalize=False) - for tree in clf.estimators_) / clf.n_estimators + clf = ExtraTreesClassifier( + n_estimators=500, max_features=1, criterion="log_loss", random_state=0 + ).fit(X, y) + + importances = ( + sum( + tree.tree_.compute_feature_importances(normalize=False) + for tree in clf.estimators_ + ) + / clf.n_estimators + ) # Check correctness assert_almost_equal(entropy(y), sum(importances)) assert np.abs(true_importances - importances).mean() < 0.01 -@pytest.mark.parametrize('name', FOREST_ESTIMATORS) +@pytest.mark.parametrize("name", FOREST_ESTIMATORS) def test_unfitted_feature_importances(name): - err_msg = ("This {} instance is not fitted yet. Call 'fit' with " - "appropriate arguments before using this estimator." - .format(name)) + err_msg = ( + "This {} instance is not fitted yet. Call 'fit' with " + "appropriate arguments before using this estimator.".format(name) + ) with pytest.raises(NotFittedError, match=err_msg): - getattr(FOREST_ESTIMATORS[name](), 'feature_importances_') + getattr(FOREST_ESTIMATORS[name](), "feature_importances_") @pytest.mark.parametrize("ForestClassifier", FOREST_CLASSIFIERS.values()) @@ -376,9 +464,7 @@ def test_unfitted_feature_importances(name): "X, y, lower_bound_accuracy", [ ( - *datasets.make_classification( - n_samples=300, n_classes=2, random_state=0 - ), + *datasets.make_classification(n_samples=300, n_classes=2, random_state=0), 0.9, ), ( @@ -388,36 +474,47 @@ def test_unfitted_feature_importances(name): 0.65, ), ( - iris.data, iris.target * 2 + 1, 0.65, + iris.data, + iris.target * 2 + 1, + 0.65, ), ( - *datasets.make_multilabel_classification( - n_samples=300, random_state=0 - ), + *datasets.make_multilabel_classification(n_samples=300, random_state=0), 0.18, ), ], ) +@pytest.mark.parametrize("oob_score", [True, partial(f1_score, average="micro")]) def test_forest_classifier_oob( - ForestClassifier, X, y, X_type, lower_bound_accuracy + ForestClassifier, X, y, X_type, lower_bound_accuracy, oob_score ): """Check that OOB score is close to score on a test set.""" X = _convert_container(X, constructor_name=X_type) X_train, X_test, y_train, y_test = train_test_split( - X, y, test_size=0.5, random_state=0, + X, + y, + test_size=0.5, + random_state=0, ) classifier = ForestClassifier( - n_estimators=40, bootstrap=True, oob_score=True, random_state=0, + n_estimators=40, + bootstrap=True, + oob_score=oob_score, + random_state=0, ) assert not hasattr(classifier, "oob_score_") assert not hasattr(classifier, "oob_decision_function_") classifier.fit(X_train, y_train) - test_score = classifier.score(X_test, y_test) + if callable(oob_score): + test_score = oob_score(y_test, classifier.predict(X_test)) + else: + test_score = classifier.score(X_test, y_test) + assert classifier.oob_score_ >= lower_bound_accuracy - assert abs(test_score - classifier.oob_score_) <= 0.1 - assert classifier.oob_score_ >= lower_bound_accuracy + abs_diff = abs(test_score - classifier.oob_score_) + assert abs_diff <= 0.11, f"{abs_diff=} is greater than 0.11" assert hasattr(classifier, "oob_score_") assert not hasattr(classifier, "oob_prediction_") @@ -449,27 +546,35 @@ def test_forest_classifier_oob( ), ], ) -def test_forest_regressor_oob( - ForestRegressor, X, y, X_type, lower_bound_r2 -): +@pytest.mark.parametrize("oob_score", [True, explained_variance_score]) +def test_forest_regressor_oob(ForestRegressor, X, y, X_type, lower_bound_r2, oob_score): """Check that forest-based regressor provide an OOB score close to the score on a test set.""" X = _convert_container(X, constructor_name=X_type) X_train, X_test, y_train, y_test = train_test_split( - X, y, test_size=0.5, random_state=0, + X, + y, + test_size=0.5, + random_state=0, ) regressor = ForestRegressor( - n_estimators=50, bootstrap=True, oob_score=True, random_state=0, + n_estimators=50, + bootstrap=True, + oob_score=oob_score, + random_state=0, ) assert not hasattr(regressor, "oob_score_") assert not hasattr(regressor, "oob_prediction_") regressor.fit(X_train, y_train) - test_score = regressor.score(X_test, y_test) + if callable(oob_score): + test_score = oob_score(y_test, regressor.predict(X_test)) + else: + test_score = regressor.score(X_test, y_test) + assert regressor.oob_score_ >= lower_bound_r2 assert abs(test_score - regressor.oob_score_) <= 0.1 - assert regressor.oob_score_ >= lower_bound_r2 assert hasattr(regressor, "oob_score_") assert hasattr(regressor, "oob_prediction_") @@ -482,38 +587,79 @@ def test_forest_regressor_oob( assert regressor.oob_prediction_.shape == expected_shape -@pytest.mark.parametrize( - "ForestEstimator", FOREST_CLASSIFIERS_REGRESSORS.values() -) +@pytest.mark.parametrize("ForestEstimator", FOREST_CLASSIFIERS_REGRESSORS.values()) def test_forest_oob_warning(ForestEstimator): """Check that a warning is raised when not enough estimator and the OOB - estimates will be inacurrate.""" + estimates will be inaccurate.""" estimator = ForestEstimator( - n_estimators=1, oob_score=True, bootstrap=True, random_state=0, + n_estimators=1, + oob_score=True, + bootstrap=True, + random_state=0, ) with pytest.warns(UserWarning, match="Some inputs do not have OOB scores"): estimator.fit(iris.data, iris.target) -@pytest.mark.parametrize( - "ForestEstimator", FOREST_CLASSIFIERS_REGRESSORS.values() -) -@pytest.mark.parametrize( - "X, y, params, err_msg", - [ - (iris.data, iris.target, {"oob_score": True, "bootstrap": False}, - "Out of bag estimation only available if bootstrap=True"), - (iris.data, rng.randint(low=0, high=5, size=(iris.data.shape[0], 2)), - {"oob_score": True, "bootstrap": True}, - "The type of target cannot be used to compute OOB estimates") - ] -) -def test_forest_oob_error(ForestEstimator, X, y, params, err_msg): - estimator = ForestEstimator(**params) +@pytest.mark.parametrize("ForestEstimator", FOREST_CLASSIFIERS_REGRESSORS.values()) +def test_forest_oob_score_requires_bootstrap(ForestEstimator): + """Check that we raise an error if OOB score is requested without + activating bootstrapping. + """ + X = iris.data + y = iris.target + err_msg = "Out of bag estimation only available if bootstrap=True" + estimator = ForestEstimator(oob_score=True, bootstrap=False) with pytest.raises(ValueError, match=err_msg): estimator.fit(X, y) +@pytest.mark.parametrize("ForestClassifier", FOREST_CLASSIFIERS.values()) +def test_classifier_error_oob_score_multiclass_multioutput(ForestClassifier): + """Check that we raise an error with when requesting OOB score with + multiclass-multioutput classification target. + """ + rng = np.random.RandomState(42) + X = iris.data + y = rng.randint(low=0, high=5, size=(iris.data.shape[0], 2)) + y_type = type_of_target(y) + assert y_type == "multiclass-multioutput" + estimator = ForestClassifier(oob_score=True, bootstrap=True) + err_msg = "The type of target cannot be used to compute OOB estimates" + with pytest.raises(ValueError, match=err_msg): + estimator.fit(X, y) + + +@pytest.mark.parametrize("ForestRegressor", FOREST_REGRESSORS.values()) +def test_forest_multioutput_integral_regression_target(ForestRegressor): + """Check that multioutput regression with integral values is not interpreted + as a multiclass-multioutput target and OOB score can be computed. + """ + rng = np.random.RandomState(42) + X = iris.data + y = rng.randint(low=0, high=10, size=(iris.data.shape[0], 2)) + estimator = ForestRegressor( + n_estimators=30, oob_score=True, bootstrap=True, random_state=0 + ) + estimator.fit(X, y) + + n_samples_bootstrap = _get_n_samples_bootstrap(len(X), estimator.max_samples) + n_samples_test = X.shape[0] // 4 + oob_pred = np.zeros([n_samples_test, 2]) + for sample_idx, sample in enumerate(X[:n_samples_test]): + n_samples_oob = 0 + oob_pred_sample = np.zeros(2) + for tree in estimator.estimators_: + oob_unsampled_indices = _generate_unsampled_indices( + tree.random_state, len(X), n_samples_bootstrap + ) + if sample_idx in oob_unsampled_indices: + n_samples_oob += 1 + oob_pred_sample += tree.predict(sample.reshape(1, -1)).squeeze() + oob_pred[sample_idx] = oob_pred_sample / n_samples_oob + assert_allclose(oob_pred, estimator.oob_prediction_[:n_samples_test]) + + @pytest.mark.parametrize("oob_score", [True, False]) def test_random_trees_embedding_raise_error_oob(oob_score): with pytest.raises(TypeError, match="got an unexpected keyword argument"): @@ -522,20 +668,24 @@ def test_random_trees_embedding_raise_error_oob(oob_score): RandomTreesEmbedding()._set_oob_score_and_attributes(X, y) -def check_gridsearch(name): - forest = FOREST_CLASSIFIERS[name]() - clf = GridSearchCV(forest, {'n_estimators': (1, 2), 'max_depth': (1, 2)}) - clf.fit(iris.data, iris.target) - - -@pytest.mark.parametrize('name', FOREST_CLASSIFIERS) +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS) def test_gridsearch(name): # Check that base trees can be grid-searched. - check_gridsearch(name) + forest = FOREST_CLASSIFIERS[name]() + clf = GridSearchCV(forest, {"n_estimators": (1, 2), "max_depth": (1, 2)}) + clf.fit(iris.data, iris.target) -def check_parallel(name, X, y): +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS_REGRESSORS) +def test_parallel(name): """Check parallel computations in classification""" + if name in FOREST_CLASSIFIERS: + X = iris.data + y = iris.target + elif name in FOREST_REGRESSORS: + X = X_reg + y = y_reg + ForestEstimator = FOREST_ESTIMATORS[name] forest = ForestEstimator(n_estimators=10, n_jobs=3, random_state=0) @@ -549,20 +699,15 @@ def check_parallel(name, X, y): assert_array_almost_equal(y1, y2, 3) -@pytest.mark.parametrize('name', FOREST_CLASSIFIERS_REGRESSORS) -def test_parallel(name): +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS_REGRESSORS) +def test_pickle(name): + # Check pickability. if name in FOREST_CLASSIFIERS: - X = iris.data - y = iris.target + X = iris.data[::2] + y = iris.target[::2] elif name in FOREST_REGRESSORS: - X = X_reg - y = y_reg - - check_parallel(name, X, y) - - -def check_pickle(name, X, y): - # Check pickability. + X = X_reg[::2] + y = y_reg[::2] ForestEstimator = FOREST_ESTIMATORS[name] obj = ForestEstimator(random_state=0) @@ -576,25 +721,38 @@ def check_pickle(name, X, y): assert score == score2 -@pytest.mark.parametrize('name', FOREST_CLASSIFIERS_REGRESSORS) -def test_pickle(name): - if name in FOREST_CLASSIFIERS: - X = iris.data - y = iris.target - elif name in FOREST_REGRESSORS: - X = X_reg - y = y_reg - - check_pickle(name, X[::2], y[::2]) - - -def check_multioutput(name): +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS_REGRESSORS) +def test_multioutput(name): # Check estimators on multi-output problems. - X_train = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1], [-2, 1], - [-1, 1], [-1, 2], [2, -1], [1, -1], [1, -2]] - y_train = [[-1, 0], [-1, 0], [-1, 0], [1, 1], [1, 1], [1, 1], [-1, 2], - [-1, 2], [-1, 2], [1, 3], [1, 3], [1, 3]] + X_train = [ + [-2, -1], + [-1, -1], + [-1, -2], + [1, 1], + [1, 2], + [2, 1], + [-2, 1], + [-1, 1], + [-1, 2], + [2, -1], + [1, -1], + [1, -2], + ] + y_train = [ + [-1, 0], + [-1, 0], + [-1, 0], + [1, 1], + [1, 1], + [1, 1], + [-1, 2], + [-1, 2], + [-1, 2], + [1, 3], + [1, 3], + [1, 3], + ] X_test = [[-1, -1], [1, 1], [-1, 1], [1, -1]] y_test = [[-1, 0], [1, 1], [-1, 2], [1, 3]] @@ -615,24 +773,45 @@ def check_multioutput(name): assert log_proba[1].shape == (4, 4) -@pytest.mark.parametrize('name', FOREST_CLASSIFIERS_REGRESSORS) -def test_multioutput(name): - check_multioutput(name) - - -@pytest.mark.parametrize('name', FOREST_CLASSIFIERS) +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS) def test_multioutput_string(name): # Check estimators on multi-output problems with string outputs. - X_train = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1], [-2, 1], - [-1, 1], [-1, 2], [2, -1], [1, -1], [1, -2]] - y_train = [["red", "blue"], ["red", "blue"], ["red", "blue"], - ["green", "green"], ["green", "green"], ["green", "green"], - ["red", "purple"], ["red", "purple"], ["red", "purple"], - ["green", "yellow"], ["green", "yellow"], ["green", "yellow"]] + X_train = [ + [-2, -1], + [-1, -1], + [-1, -2], + [1, 1], + [1, 2], + [2, 1], + [-2, 1], + [-1, 1], + [-1, 2], + [2, -1], + [1, -1], + [1, -2], + ] + y_train = [ + ["red", "blue"], + ["red", "blue"], + ["red", "blue"], + ["green", "green"], + ["green", "green"], + ["green", "green"], + ["red", "purple"], + ["red", "purple"], + ["red", "purple"], + ["green", "yellow"], + ["green", "yellow"], + ["green", "yellow"], + ] X_test = [[-1, -1], [1, 1], [-1, 1], [1, -1]] - y_test = [["red", "blue"], ["green", "green"], - ["red", "purple"], ["green", "yellow"]] + y_test = [ + ["red", "blue"], + ["green", "green"], + ["red", "purple"], + ["green", "yellow"], + ] est = FOREST_ESTIMATORS[name](random_state=0, bootstrap=False) y_pred = est.fit(X_train, y_train).predict(X_test) @@ -650,7 +829,8 @@ def test_multioutput_string(name): assert log_proba[1].shape == (4, 4) -def check_classes_shape(name): +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS) +def test_classes_shape(name): # Test that n_classes_ and classes_ have proper shape. ForestClassifier = FOREST_CLASSIFIERS[name] @@ -668,11 +848,6 @@ def check_classes_shape(name): assert_array_equal(clf.classes_, [[-1, 1], [-2, 2]]) -@pytest.mark.parametrize('name', FOREST_CLASSIFIERS) -def test_classes_shape(name): - check_classes_shape(name) - - def test_random_trees_dense_type(): # Test that the `sparse_output` parameter of RandomTreesEmbedding # works by returning a dense array. @@ -682,8 +857,8 @@ def test_random_trees_dense_type(): X, y = datasets.make_circles(factor=0.5) X_transformed = hasher.fit_transform(X) - # Assert that type is ndarray, not scipy.sparse.csr.csr_matrix - assert type(X_transformed) == np.ndarray + # Assert that type is ndarray, not scipy.sparse.csr_matrix + assert isinstance(X_transformed, np.ndarray) def test_random_trees_dense_equal(): @@ -691,10 +866,12 @@ def test_random_trees_dense_equal(): # works by returning the same array for both argument values. # Create the RTEs - hasher_dense = RandomTreesEmbedding(n_estimators=10, sparse_output=False, - random_state=0) - hasher_sparse = RandomTreesEmbedding(n_estimators=10, sparse_output=True, - random_state=0) + hasher_dense = RandomTreesEmbedding( + n_estimators=10, sparse_output=False, random_state=0 + ) + hasher_sparse = RandomTreesEmbedding( + n_estimators=10, sparse_output=True, random_state=0 + ) X, y = datasets.make_circles(factor=0.5) X_transformed_dense = hasher_dense.fit_transform(X) X_transformed_sparse = hasher_sparse.fit_transform(X) @@ -703,8 +880,6 @@ def test_random_trees_dense_equal(): assert_array_equal(X_transformed_sparse.toarray(), X_transformed_dense) -# Ignore warnings from switching to more power iterations in randomized_svd -@ignore_warnings def test_random_hasher(): # test random forest hashing on circles dataset # make sure that it is linearly separable. @@ -716,8 +891,7 @@ def test_random_hasher(): # test fit and transform: hasher = RandomTreesEmbedding(n_estimators=30, random_state=1) - assert_array_equal(hasher.fit(X).transform(X).toarray(), - X_transformed.toarray()) + assert_array_equal(hasher.fit(X).transform(X).toarray(), X_transformed.toarray()) # one leaf active per data point per forest assert X_transformed.shape[0] == X.shape[0] @@ -726,14 +900,15 @@ def test_random_hasher(): X_reduced = svd.fit_transform(X_transformed) linear_clf = LinearSVC() linear_clf.fit(X_reduced, y) - assert linear_clf.score(X_reduced, y) == 1. + assert linear_clf.score(X_reduced, y) == 1.0 -def test_random_hasher_sparse_data(): +@pytest.mark.parametrize("csc_container", CSC_CONTAINERS) +def test_random_hasher_sparse_data(csc_container): X, y = datasets.make_multilabel_classification(random_state=0) hasher = RandomTreesEmbedding(n_estimators=30, random_state=1) X_transformed = hasher.fit_transform(X) - X_transformed_sparse = hasher.fit_transform(csc_matrix(X)) + X_transformed_sparse = hasher.fit_transform(csc_container(X)) assert_array_equal(X_transformed_sparse.toarray(), X_transformed.toarray()) @@ -744,14 +919,15 @@ def test_parallel_train(): y_train = rng.randint(0, 2, n_samples) clfs = [ - RandomForestClassifier(n_estimators=20, n_jobs=n_jobs, - random_state=12345).fit(X_train, y_train) + RandomForestClassifier(n_estimators=20, n_jobs=n_jobs, random_state=12345).fit( + X_train, y_train + ) for n_jobs in [1, 2, 3, 8, 16, 32] ] X_test = rng.randn(n_samples, n_features) probas = [clf.predict_proba(X_test) for clf in clfs] - for proba1, proba2 in zip(probas, probas[1:]): + for proba1, proba2 in itertools.pairwise(probas): assert_array_almost_equal(proba1, proba2) @@ -767,14 +943,14 @@ def test_distribution(): uniques = defaultdict(int) for tree in reg.estimators_: - tree = "".join(("%d,%d/" % (f, int(t)) if f >= 0 else "-") - for f, t in zip(tree.tree_.feature, - tree.tree_.threshold)) + tree = "".join( + ("%d,%d/" % (f, int(t)) if f >= 0 else "-") + for f, t in zip(tree.tree_.feature, tree.tree_.threshold) + ) uniques[tree] += 1 - uniques = sorted([(1. * count / n_trees, tree) - for tree, count in uniques.items()]) + uniques = sorted([(1.0 * count / n_trees, tree) for tree, count in uniques.items()]) # On a single variable problem where X_0 has 4 equiprobable values, there # are 5 ways to build a random tree. The more compact (0,1/0,0/--0,2/--) of @@ -798,9 +974,10 @@ def test_distribution(): uniques = defaultdict(int) for tree in reg.estimators_: - tree = "".join(("%d,%d/" % (f, int(t)) if f >= 0 else "-") - for f, t in zip(tree.tree_.feature, - tree.tree_.threshold)) + tree = "".join( + ("%d,%d/" % (f, int(t)) if f >= 0 else "-") + for f, t in zip(tree.tree_.feature, tree.tree_.threshold) + ) uniques[tree] += 1 @@ -808,72 +985,48 @@ def test_distribution(): assert len(uniques) == 8 -def check_max_leaf_nodes_max_depth(name): +@pytest.mark.parametrize("name", FOREST_ESTIMATORS) +def test_max_leaf_nodes_max_depth(name): X, y = hastie_X, hastie_y # Test precedence of max_leaf_nodes over max_depth. ForestEstimator = FOREST_ESTIMATORS[name] - est = ForestEstimator(max_depth=1, max_leaf_nodes=4, - n_estimators=1, random_state=0).fit(X, y) + est = ForestEstimator( + max_depth=1, max_leaf_nodes=4, n_estimators=1, random_state=0 + ).fit(X, y) assert est.estimators_[0].get_depth() == 1 - est = ForestEstimator(max_depth=1, n_estimators=1, - random_state=0).fit(X, y) + est = ForestEstimator(max_depth=1, n_estimators=1, random_state=0).fit(X, y) assert est.estimators_[0].get_depth() == 1 -@pytest.mark.parametrize('name', FOREST_ESTIMATORS) -def test_max_leaf_nodes_max_depth(name): - check_max_leaf_nodes_max_depth(name) - - -def check_min_samples_split(name): +@pytest.mark.parametrize("name", FOREST_ESTIMATORS) +def test_min_samples_split(name): X, y = hastie_X, hastie_y ForestEstimator = FOREST_ESTIMATORS[name] - # test boundary value - with pytest.raises(ValueError): - ForestEstimator(min_samples_split=-1).fit(X, y) - with pytest.raises(ValueError): - ForestEstimator(min_samples_split=0).fit(X, y) - with pytest.raises(ValueError): - ForestEstimator(min_samples_split=1.1).fit(X, y) - est = ForestEstimator(min_samples_split=10, n_estimators=1, random_state=0) est.fit(X, y) node_idx = est.estimators_[0].tree_.children_left != -1 node_samples = est.estimators_[0].tree_.n_node_samples[node_idx] - assert np.min(node_samples) > len(X) * 0.5 - 1, ( - "Failed with {0}".format(name)) + assert np.min(node_samples) > len(X) * 0.5 - 1, "Failed with {0}".format(name) - est = ForestEstimator(min_samples_split=0.5, n_estimators=1, - random_state=0) + est = ForestEstimator(min_samples_split=0.5, n_estimators=1, random_state=0) est.fit(X, y) node_idx = est.estimators_[0].tree_.children_left != -1 node_samples = est.estimators_[0].tree_.n_node_samples[node_idx] - assert np.min(node_samples) > len(X) * 0.5 - 1, ( - "Failed with {0}".format(name)) - - -@pytest.mark.parametrize('name', FOREST_ESTIMATORS) -def test_min_samples_split(name): - check_min_samples_split(name) + assert np.min(node_samples) > len(X) * 0.5 - 1, "Failed with {0}".format(name) -def check_min_samples_leaf(name): +@pytest.mark.parametrize("name", FOREST_ESTIMATORS) +def test_min_samples_leaf(name): X, y = hastie_X, hastie_y # Test if leaves contain more than leaf_count training examples ForestEstimator = FOREST_ESTIMATORS[name] - # test boundary value - with pytest.raises(ValueError): - ForestEstimator(min_samples_leaf=-1).fit(X, y) - with pytest.raises(ValueError): - ForestEstimator(min_samples_leaf=0).fit(X, y) - est = ForestEstimator(min_samples_leaf=5, n_estimators=1, random_state=0) est.fit(X, y) out = est.estimators_[0].tree_.apply(X) @@ -882,23 +1035,17 @@ def check_min_samples_leaf(name): leaf_count = node_counts[node_counts != 0] assert np.min(leaf_count) > 4, "Failed with {0}".format(name) - est = ForestEstimator(min_samples_leaf=0.25, n_estimators=1, - random_state=0) + est = ForestEstimator(min_samples_leaf=0.25, n_estimators=1, random_state=0) est.fit(X, y) out = est.estimators_[0].tree_.apply(X) node_counts = np.bincount(out) # drop inner nodes leaf_count = node_counts[node_counts != 0] - assert np.min(leaf_count) > len(X) * 0.25 - 1, ( - "Failed with {0}".format(name)) - - -@pytest.mark.parametrize('name', FOREST_ESTIMATORS) -def test_min_samples_leaf(name): - check_min_samples_leaf(name) + assert np.min(leaf_count) > len(X) * 0.25 - 1, "Failed with {0}".format(name) -def check_min_weight_fraction_leaf(name): +@pytest.mark.parametrize("name", FOREST_ESTIMATORS) +def test_min_weight_fraction_leaf(name): X, y = hastie_X, hastie_y # Test if leaves contain at least min_weight_fraction_leaf of the @@ -911,8 +1058,9 @@ def check_min_weight_fraction_leaf(name): # test both DepthFirstTreeBuilder and BestFirstTreeBuilder # by setting max_leaf_nodes for frac in np.linspace(0, 0.5, 6): - est = ForestEstimator(min_weight_fraction_leaf=frac, n_estimators=1, - random_state=0) + est = ForestEstimator( + min_weight_fraction_leaf=frac, n_estimators=1, random_state=0 + ) if "RandomForest" in name: est.bootstrap = False @@ -921,94 +1069,71 @@ def check_min_weight_fraction_leaf(name): node_weights = np.bincount(out, weights=weights) # drop inner nodes leaf_weights = node_weights[node_weights != 0] - assert ( - np.min(leaf_weights) >= - total_weight * est.min_weight_fraction_leaf), ( - "Failed with {0} min_weight_fraction_leaf={1}".format( - name, est.min_weight_fraction_leaf)) - + assert np.min(leaf_weights) >= total_weight * est.min_weight_fraction_leaf, ( + "Failed with {0} min_weight_fraction_leaf={1}".format( + name, est.min_weight_fraction_leaf + ) + ) -@pytest.mark.parametrize('name', FOREST_ESTIMATORS) -def test_min_weight_fraction_leaf(name): - check_min_weight_fraction_leaf(name) +@pytest.mark.parametrize("name", FOREST_ESTIMATORS) +@pytest.mark.parametrize( + "sparse_container", COO_CONTAINERS + CSC_CONTAINERS + CSR_CONTAINERS +) +def test_sparse_input(name, sparse_container): + X, y = datasets.make_multilabel_classification(random_state=0, n_samples=50) -def check_sparse_input(name, X, X_sparse, y): ForestEstimator = FOREST_ESTIMATORS[name] dense = ForestEstimator(random_state=0, max_depth=2).fit(X, y) - sparse = ForestEstimator(random_state=0, max_depth=2).fit(X_sparse, y) + sparse = ForestEstimator(random_state=0, max_depth=2).fit(sparse_container(X), y) assert_array_almost_equal(sparse.apply(X), dense.apply(X)) if name in FOREST_CLASSIFIERS or name in FOREST_REGRESSORS: assert_array_almost_equal(sparse.predict(X), dense.predict(X)) - assert_array_almost_equal(sparse.feature_importances_, - dense.feature_importances_) + assert_array_almost_equal( + sparse.feature_importances_, dense.feature_importances_ + ) if name in FOREST_CLASSIFIERS: - assert_array_almost_equal(sparse.predict_proba(X), - dense.predict_proba(X)) - assert_array_almost_equal(sparse.predict_log_proba(X), - dense.predict_log_proba(X)) + assert_array_almost_equal(sparse.predict_proba(X), dense.predict_proba(X)) + assert_array_almost_equal( + sparse.predict_log_proba(X), dense.predict_log_proba(X) + ) if name in FOREST_TRANSFORMERS: - assert_array_almost_equal(sparse.transform(X).toarray(), - dense.transform(X).toarray()) - assert_array_almost_equal(sparse.fit_transform(X).toarray(), - dense.fit_transform(X).toarray()) + assert_array_almost_equal( + sparse.transform(X).toarray(), dense.transform(X).toarray() + ) + assert_array_almost_equal( + sparse.fit_transform(X).toarray(), dense.fit_transform(X).toarray() + ) -@pytest.mark.parametrize('name', FOREST_ESTIMATORS) -@pytest.mark.parametrize('sparse_matrix', - (csr_matrix, csc_matrix, coo_matrix)) -def test_sparse_input(name, sparse_matrix): - X, y = datasets.make_multilabel_classification(random_state=0, - n_samples=50) - - check_sparse_input(name, X, sparse_matrix(X), y) - - -def check_memory_layout(name, dtype): - # Check that it works no matter the memory layout - +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS_REGRESSORS) +@pytest.mark.parametrize("dtype", (np.float64, np.float32)) +def test_memory_layout(name, dtype): + # Test that it works no matter the memory layout est = FOREST_ESTIMATORS[name](random_state=0, bootstrap=False) - # Nothing - X = np.asarray(iris.data, dtype=dtype) - y = iris.target - assert_array_almost_equal(est.fit(X, y).predict(X), y) - - # C-order - X = np.asarray(iris.data, order="C", dtype=dtype) - y = iris.target - assert_array_almost_equal(est.fit(X, y).predict(X), y) - - # F-order - X = np.asarray(iris.data, order="F", dtype=dtype) - y = iris.target - assert_array_almost_equal(est.fit(X, y).predict(X), y) - - # Contiguous - X = np.ascontiguousarray(iris.data, dtype=dtype) - y = iris.target - assert_array_almost_equal(est.fit(X, y).predict(X), y) - - if est.base_estimator.splitter in SPARSE_SPLITTERS: - # csr matrix - X = csr_matrix(iris.data, dtype=dtype) + # Dense + for container, kwargs in ( + (np.asarray, {}), # Nothing + (np.asarray, {"order": "C"}), # C-order + (np.asarray, {"order": "F"}), # F-order + (np.ascontiguousarray, {}), # Contiguous + ): + X = container(iris.data, dtype=dtype, **kwargs) y = iris.target assert_array_almost_equal(est.fit(X, y).predict(X), y) - # csc_matrix - X = csc_matrix(iris.data, dtype=dtype) - y = iris.target - assert_array_almost_equal(est.fit(X, y).predict(X), y) - - # coo_matrix - X = coo_matrix(iris.data, dtype=dtype) - y = iris.target - assert_array_almost_equal(est.fit(X, y).predict(X), y) + # Sparse (if applicable) + if est.estimator.splitter in SPARSE_SPLITTERS: + for sparse_container in COO_CONTAINERS + CSC_CONTAINERS + CSR_CONTAINERS: + X = sparse_container(iris.data, dtype=dtype) + y = iris.target + assert_array_almost_equal(est.fit(X, y).predict(X), y) # Strided X = np.asarray(iris.data[::3], dtype=dtype) @@ -1016,65 +1141,59 @@ def check_memory_layout(name, dtype): assert_array_almost_equal(est.fit(X, y).predict(X), y) -@pytest.mark.parametrize('name', FOREST_CLASSIFIERS_REGRESSORS) -@pytest.mark.parametrize('dtype', (np.float64, np.float32)) -def test_memory_layout(name, dtype): - check_memory_layout(name, dtype) - - -@ignore_warnings -def check_1d_input(name, X, X_2d, y): - ForestEstimator = FOREST_ESTIMATORS[name] - with pytest.raises(ValueError): - ForestEstimator(n_estimators=1, random_state=0).fit(X, y) - - est = ForestEstimator(random_state=0) - est.fit(X_2d, y) - - if name in FOREST_CLASSIFIERS or name in FOREST_REGRESSORS: - with pytest.raises(ValueError): - est.predict(X) - - -@pytest.mark.parametrize('name', FOREST_ESTIMATORS) +@pytest.mark.parametrize("name", FOREST_ESTIMATORS) def test_1d_input(name): X = iris.data[:, 0] X_2d = iris.data[:, 0].reshape((-1, 1)) y = iris.target with ignore_warnings(): - check_1d_input(name, X, X_2d, y) + ForestEstimator = FOREST_ESTIMATORS[name] + with pytest.raises(ValueError): + ForestEstimator(n_estimators=1, random_state=0).fit(X, y) + + est = ForestEstimator(random_state=0) + est.fit(X_2d, y) + if name in FOREST_CLASSIFIERS or name in FOREST_REGRESSORS: + with pytest.raises(ValueError): + est.predict(X) -def check_class_weights(name): + +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS) +def test_class_weights(name): # Check class_weights resemble sample_weights behavior. ForestClassifier = FOREST_CLASSIFIERS[name] # Iris is balanced, so no effect expected for using 'balanced' weights clf1 = ForestClassifier(random_state=0) clf1.fit(iris.data, iris.target) - clf2 = ForestClassifier(class_weight='balanced', random_state=0) + clf2 = ForestClassifier(class_weight="balanced", random_state=0) clf2.fit(iris.data, iris.target) assert_almost_equal(clf1.feature_importances_, clf2.feature_importances_) # Make a multi-output problem with three copies of Iris iris_multi = np.vstack((iris.target, iris.target, iris.target)).T # Create user-defined weights that should balance over the outputs - clf3 = ForestClassifier(class_weight=[{0: 2., 1: 2., 2: 1.}, - {0: 2., 1: 1., 2: 2.}, - {0: 1., 1: 2., 2: 2.}], - random_state=0) + clf3 = ForestClassifier( + class_weight=[ + {0: 2.0, 1: 2.0, 2: 1.0}, + {0: 2.0, 1: 1.0, 2: 2.0}, + {0: 1.0, 1: 2.0, 2: 2.0}, + ], + random_state=0, + ) clf3.fit(iris.data, iris_multi) assert_almost_equal(clf2.feature_importances_, clf3.feature_importances_) # Check against multi-output "balanced" which should also have no effect - clf4 = ForestClassifier(class_weight='balanced', random_state=0) + clf4 = ForestClassifier(class_weight="balanced", random_state=0) clf4.fit(iris.data, iris_multi) assert_almost_equal(clf3.feature_importances_, clf4.feature_importances_) # Inflate importance of class 1, check against user-defined weights sample_weight = np.ones(iris.target.shape) sample_weight[iris.target == 1] *= 100 - class_weight = {0: 1., 1: 100., 2: 1.} + class_weight = {0: 1.0, 1: 100.0, 2: 1.0} clf1 = ForestClassifier(random_state=0) clf1.fit(iris.data, iris.target, sample_weight) clf2 = ForestClassifier(class_weight=class_weight, random_state=0) @@ -1083,77 +1202,52 @@ def check_class_weights(name): # Check that sample_weight and class_weight are multiplicative clf1 = ForestClassifier(random_state=0) - clf1.fit(iris.data, iris.target, sample_weight ** 2) + clf1.fit(iris.data, iris.target, sample_weight**2) clf2 = ForestClassifier(class_weight=class_weight, random_state=0) clf2.fit(iris.data, iris.target, sample_weight) assert_almost_equal(clf1.feature_importances_, clf2.feature_importances_) -@pytest.mark.parametrize('name', FOREST_CLASSIFIERS) -def test_class_weights(name): - check_class_weights(name) - - -def check_class_weight_balanced_and_bootstrap_multi_output(name): +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS) +def test_class_weight_balanced_and_bootstrap_multi_output(name): # Test class_weight works for multi-output""" ForestClassifier = FOREST_CLASSIFIERS[name] _y = np.vstack((y, np.array(y) * 2)).T - clf = ForestClassifier(class_weight='balanced', random_state=0) + clf = ForestClassifier(class_weight="balanced", random_state=0) clf.fit(X, _y) - clf = ForestClassifier(class_weight=[{-1: 0.5, 1: 1.}, {-2: 1., 2: 1.}], - random_state=0) + clf = ForestClassifier( + class_weight=[{-1: 0.5, 1: 1.0}, {-2: 1.0, 2: 1.0}], random_state=0 + ) clf.fit(X, _y) # smoke test for balanced subsample - clf = ForestClassifier(class_weight='balanced_subsample', random_state=0) + clf = ForestClassifier(class_weight="balanced_subsample", random_state=0) clf.fit(X, _y) -@pytest.mark.parametrize('name', FOREST_CLASSIFIERS) -def test_class_weight_balanced_and_bootstrap_multi_output(name): - check_class_weight_balanced_and_bootstrap_multi_output(name) - - -def check_class_weight_errors(name): +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS) +def test_class_weight_errors(name): # Test if class_weight raises errors and warnings when expected. ForestClassifier = FOREST_CLASSIFIERS[name] _y = np.vstack((y, np.array(y) * 2)).T - # Invalid preset string - clf = ForestClassifier(class_weight='the larch', random_state=0) - with pytest.raises(ValueError): - clf.fit(X, y) - with pytest.raises(ValueError): - clf.fit(X, _y) - # Warning warm_start with preset - clf = ForestClassifier(class_weight='balanced', warm_start=True, - random_state=0) + clf = ForestClassifier(class_weight="balanced", warm_start=True, random_state=0) clf.fit(X, y) warn_msg = ( - "Warm-start fitting without increasing n_estimators does not fit new " - "trees." + "Warm-start fitting without increasing n_estimators does not fit new trees." ) with pytest.warns(UserWarning, match=warn_msg): clf.fit(X, _y) - # Not a list or preset for multi-output - clf = ForestClassifier(class_weight=1, random_state=0) - with pytest.raises(ValueError): - clf.fit(X, _y) - # Incorrect length list for multi-output - clf = ForestClassifier(class_weight=[{-1: 0.5, 1: 1.}], random_state=0) + clf = ForestClassifier(class_weight=[{-1: 0.5, 1: 1.0}], random_state=0) with pytest.raises(ValueError): clf.fit(X, _y) -@pytest.mark.parametrize('name', FOREST_CLASSIFIERS) -def test_class_weight_errors(name): - check_class_weight_errors(name) - - -def check_warm_start(name, random_state=42): +@pytest.mark.parametrize("name", FOREST_ESTIMATORS) +def test_warm_start(name): # Test if fitting incrementally with warm start gives a forest of the # right size and the same results as a normal fit. X, y = hastie_X, hastie_y @@ -1161,40 +1255,37 @@ def check_warm_start(name, random_state=42): est_ws = None for n_estimators in [5, 10]: if est_ws is None: - est_ws = ForestEstimator(n_estimators=n_estimators, - random_state=random_state, - warm_start=True) + est_ws = ForestEstimator( + n_estimators=n_estimators, random_state=42, warm_start=True + ) else: est_ws.set_params(n_estimators=n_estimators) est_ws.fit(X, y) assert len(est_ws) == n_estimators - est_no_ws = ForestEstimator(n_estimators=10, random_state=random_state, - warm_start=False) + est_no_ws = ForestEstimator(n_estimators=10, random_state=42, warm_start=False) est_no_ws.fit(X, y) - assert (set([tree.random_state for tree in est_ws]) == - set([tree.random_state for tree in est_no_ws])) - - assert_array_equal(est_ws.apply(X), est_no_ws.apply(X), - err_msg="Failed with {0}".format(name)) - + assert set([tree.random_state for tree in est_ws]) == set( + [tree.random_state for tree in est_no_ws] + ) -@pytest.mark.parametrize('name', FOREST_ESTIMATORS) -def test_warm_start(name): - check_warm_start(name) + assert_array_equal( + est_ws.apply(X), est_no_ws.apply(X), err_msg="Failed with {0}".format(name) + ) -def check_warm_start_clear(name): +@pytest.mark.parametrize("name", FOREST_ESTIMATORS) +def test_warm_start_clear(name): # Test if fit clears state and grows a new forest when warm_start==False. X, y = hastie_X, hastie_y ForestEstimator = FOREST_ESTIMATORS[name] - est = ForestEstimator(n_estimators=5, max_depth=1, warm_start=False, - random_state=1) + est = ForestEstimator(n_estimators=5, max_depth=1, warm_start=False, random_state=1) est.fit(X, y) - est_2 = ForestEstimator(n_estimators=5, max_depth=1, warm_start=True, - random_state=2) + est_2 = ForestEstimator( + n_estimators=5, max_depth=1, warm_start=True, random_state=2 + ) est_2.fit(X, y) # inits state est_2.set_params(warm_start=False, random_state=1) est_2.fit(X, y) # clears old state and equals est @@ -1202,12 +1293,8 @@ def check_warm_start_clear(name): assert_array_almost_equal(est_2.apply(X), est.apply(X)) -@pytest.mark.parametrize('name', FOREST_ESTIMATORS) -def test_warm_start_clear(name): - check_warm_start_clear(name) - - -def check_warm_start_smaller_n_estimators(name): +@pytest.mark.parametrize("name", FOREST_ESTIMATORS) +def test_warm_start_smaller_n_estimators(name): # Test if warm start second fit with smaller n_estimators raises error. X, y = hastie_X, hastie_y ForestEstimator = FOREST_ESTIMATORS[name] @@ -1218,29 +1305,24 @@ def check_warm_start_smaller_n_estimators(name): est.fit(X, y) -@pytest.mark.parametrize('name', FOREST_ESTIMATORS) -def test_warm_start_smaller_n_estimators(name): - check_warm_start_smaller_n_estimators(name) - - -def check_warm_start_equal_n_estimators(name): +@pytest.mark.parametrize("name", FOREST_ESTIMATORS) +def test_warm_start_equal_n_estimators(name): # Test if warm start with equal n_estimators does nothing and returns the # same forest and raises a warning. X, y = hastie_X, hastie_y ForestEstimator = FOREST_ESTIMATORS[name] - est = ForestEstimator(n_estimators=5, max_depth=3, warm_start=True, - random_state=1) + est = ForestEstimator(n_estimators=5, max_depth=3, warm_start=True, random_state=1) est.fit(X, y) - est_2 = ForestEstimator(n_estimators=5, max_depth=3, warm_start=True, - random_state=1) + est_2 = ForestEstimator( + n_estimators=5, max_depth=3, warm_start=True, random_state=1 + ) est_2.fit(X, y) # Now est_2 equals est. est_2.set_params(random_state=2) warn_msg = ( - "Warm-start fitting without increasing n_estimators does not fit " - "new trees." + "Warm-start fitting without increasing n_estimators does not fit new trees." ) with pytest.warns(UserWarning, match=warn_msg): est_2.fit(X, y) @@ -1249,36 +1331,50 @@ def check_warm_start_equal_n_estimators(name): assert_array_equal(est.apply(X), est_2.apply(X)) -@pytest.mark.parametrize('name', FOREST_ESTIMATORS) -def test_warm_start_equal_n_estimators(name): - check_warm_start_equal_n_estimators(name) - - -def check_warm_start_oob(name): +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS_REGRESSORS) +def test_warm_start_oob(name): # Test that the warm start computes oob score when asked. X, y = hastie_X, hastie_y ForestEstimator = FOREST_ESTIMATORS[name] # Use 15 estimators to avoid 'some inputs do not have OOB scores' warning. - est = ForestEstimator(n_estimators=15, max_depth=3, warm_start=False, - random_state=1, bootstrap=True, oob_score=True) + est = ForestEstimator( + n_estimators=15, + max_depth=3, + warm_start=False, + random_state=1, + bootstrap=True, + oob_score=True, + ) est.fit(X, y) - est_2 = ForestEstimator(n_estimators=5, max_depth=3, warm_start=False, - random_state=1, bootstrap=True, oob_score=False) + est_2 = ForestEstimator( + n_estimators=5, + max_depth=3, + warm_start=False, + random_state=1, + bootstrap=True, + oob_score=False, + ) est_2.fit(X, y) est_2.set_params(warm_start=True, oob_score=True, n_estimators=15) est_2.fit(X, y) - assert hasattr(est_2, 'oob_score_') + assert hasattr(est_2, "oob_score_") assert est.oob_score_ == est_2.oob_score_ # Test that oob_score is computed even if we don't need to train # additional trees. - est_3 = ForestEstimator(n_estimators=15, max_depth=3, warm_start=True, - random_state=1, bootstrap=True, oob_score=False) + est_3 = ForestEstimator( + n_estimators=15, + max_depth=3, + warm_start=True, + random_state=1, + bootstrap=True, + oob_score=False, + ) est_3.fit(X, y) - assert not hasattr(est_3, 'oob_score_') + assert not hasattr(est_3, "oob_score_") est_3.set_params(oob_score=True) ignore_warnings(est_3.fit)(X, y) @@ -1286,68 +1382,71 @@ def check_warm_start_oob(name): assert est.oob_score_ == est_3.oob_score_ -@pytest.mark.parametrize('name', FOREST_CLASSIFIERS_REGRESSORS) -def test_warm_start_oob(name): - check_warm_start_oob(name) +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS_REGRESSORS) +def test_oob_not_computed_twice(name): + # Check that oob_score is not computed twice when warm_start=True. + X, y = hastie_X, hastie_y + ForestEstimator = FOREST_ESTIMATORS[name] + + est = ForestEstimator( + n_estimators=10, warm_start=True, bootstrap=True, oob_score=True + ) + + with patch.object( + est, "_set_oob_score_and_attributes", wraps=est._set_oob_score_and_attributes + ) as mock_set_oob_score_and_attributes: + est.fit(X, y) + + with pytest.warns(UserWarning, match="Warm-start fitting without increasing"): + est.fit(X, y) + + mock_set_oob_score_and_attributes.assert_called_once() def test_dtype_convert(n_classes=15): classifier = RandomForestClassifier(random_state=0, bootstrap=False) X = np.eye(n_classes) - y = [ch for ch in 'ABCDEFGHIJKLMNOPQRSTU'[:n_classes]] + y = [ch for ch in "ABCDEFGHIJKLMNOPQRSTU"[:n_classes]] result = classifier.fit(X, y).predict(X) assert_array_equal(classifier.classes_, y) assert_array_equal(result, y) -def check_decision_path(name): +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS_REGRESSORS) +def test_decision_path(name): X, y = hastie_X, hastie_y n_samples = X.shape[0] ForestEstimator = FOREST_ESTIMATORS[name] - est = ForestEstimator(n_estimators=5, max_depth=1, warm_start=False, - random_state=1) + est = ForestEstimator(n_estimators=5, max_depth=1, warm_start=False, random_state=1) est.fit(X, y) indicator, n_nodes_ptr = est.decision_path(X) assert indicator.shape[1] == n_nodes_ptr[-1] assert indicator.shape[0] == n_samples - assert_array_equal(np.diff(n_nodes_ptr), - [e.tree_.node_count for e in est.estimators_]) + assert_array_equal( + np.diff(n_nodes_ptr), [e.tree_.node_count for e in est.estimators_] + ) # Assert that leaves index are correct leaves = est.apply(X) for est_id in range(leaves.shape[1]): - leave_indicator = [indicator[i, n_nodes_ptr[est_id] + j] - for i, j in enumerate(leaves[:, est_id])] + leave_indicator = [ + indicator[i, n_nodes_ptr[est_id] + j] + for i, j in enumerate(leaves[:, est_id]) + ] assert_array_almost_equal(leave_indicator, np.ones(shape=n_samples)) -@pytest.mark.parametrize('name', FOREST_CLASSIFIERS_REGRESSORS) -def test_decision_path(name): - check_decision_path(name) - - -def test_min_impurity_split(): - # Test if min_impurity_split of base estimators is set - # Regression test for #8006 - X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) - all_estimators = [RandomForestClassifier, RandomForestRegressor, - ExtraTreesClassifier, ExtraTreesRegressor] - - for Estimator in all_estimators: - est = Estimator(min_impurity_split=0.1) - with pytest.warns(FutureWarning, match="min_impurity_decrease"): - est = est.fit(X, y) - for tree in est.estimators_: - assert tree.min_impurity_split == 0.1 - - def test_min_impurity_decrease(): X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) - all_estimators = [RandomForestClassifier, RandomForestRegressor, - ExtraTreesClassifier, ExtraTreesRegressor] + all_estimators = [ + RandomForestClassifier, + RandomForestRegressor, + ExtraTreesClassifier, + ExtraTreesRegressor, + ] for Estimator in all_estimators: est = Estimator(min_impurity_decrease=0.1) @@ -1358,8 +1457,29 @@ def test_min_impurity_decrease(): assert tree.min_impurity_decrease == 0.1 +def test_poisson_y_positive_check(): + est = RandomForestRegressor(criterion="poisson") + X = np.zeros((3, 3)) + + y = [-1, 1, 3] + err_msg = ( + r"Some value\(s\) of y are negative which is " + r"not allowed for Poisson regression." + ) + with pytest.raises(ValueError, match=err_msg): + est.fit(X, y) + + y = [0, 0, 0] + err_msg = ( + r"Sum of y is not strictly positive which " + r"is necessary for Poisson regression." + ) + with pytest.raises(ValueError, match=err_msg): + est.fit(X, y) + + # mypy error: Variable "DEFAULT_JOBLIB_BACKEND" is not valid type -class MyBackend(DEFAULT_JOBLIB_BACKEND): # type: ignore +class MyBackend(DEFAULT_JOBLIB_BACKEND): # type: ignore[valid-type,misc] def __init__(self, *args, **kwargs): self.count = 0 super().__init__(*args, **kwargs) @@ -1369,11 +1489,9 @@ def start_call(self): return super().start_call() -joblib.register_parallel_backend('testing', MyBackend) +joblib.register_parallel_backend("testing", MyBackend) -@pytest.mark.skipif(parse_version(joblib.__version__) < parse_version('0.12'), - reason='tests not yet supported in joblib <0.12') @skip_if_no_parallel def test_backend_respected(): clf = RandomForestClassifier(n_estimators=10, n_jobs=2) @@ -1391,10 +1509,12 @@ def test_backend_respected(): def test_forest_feature_importances_sum(): - X, y = make_classification(n_samples=15, n_informative=3, random_state=1, - n_classes=3) - clf = RandomForestClassifier(min_samples_leaf=5, random_state=42, - n_estimators=200).fit(X, y) + X, y = make_classification( + n_samples=15, n_informative=3, random_state=1, n_classes=3 + ) + clf = RandomForestClassifier( + min_samples_leaf=5, random_state=42, n_estimators=200 + ).fit(X, y) assert math.isclose(1, clf.feature_importances_.sum(), abs_tol=1e-7) @@ -1403,51 +1523,83 @@ def test_forest_degenerate_feature_importances(): X = np.zeros((10, 10)) y = np.ones((10,)) gbr = RandomForestRegressor(n_estimators=10).fit(X, y) - assert_array_equal(gbr.feature_importances_, - np.zeros(10, dtype=np.float64)) + assert_array_equal(gbr.feature_importances_, np.zeros(10, dtype=np.float64)) -@pytest.mark.parametrize('name', FOREST_CLASSIFIERS_REGRESSORS) -@pytest.mark.parametrize( - 'max_samples, exc_type, exc_msg', - [(int(1e9), ValueError, - "`max_samples` must be in range 1 to 6 but got value 1000000000"), - (1.0, ValueError, - r"`max_samples` must be in range \(0, 1\) but got value 1.0"), - (2.0, ValueError, - r"`max_samples` must be in range \(0, 1\) but got value 2.0"), - (0.0, ValueError, - r"`max_samples` must be in range \(0, 1\) but got value 0.0"), - (np.nan, ValueError, - r"`max_samples` must be in range \(0, 1\) but got value nan"), - (np.inf, ValueError, - r"`max_samples` must be in range \(0, 1\) but got value inf"), - ('str max_samples?!', TypeError, - r"`max_samples` should be int or float, but got " - r"type '\'"), - (np.ones(2), TypeError, - r"`max_samples` should be int or float, but got type " - r"'\'")] -) -def test_max_samples_exceptions(name, max_samples, exc_type, exc_msg): +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS_REGRESSORS) +def test_max_samples_bootstrap(name): # Check invalid `max_samples` values - est = FOREST_CLASSIFIERS_REGRESSORS[name](max_samples=max_samples) - with pytest.raises(exc_type, match=exc_msg): + est = FOREST_CLASSIFIERS_REGRESSORS[name](bootstrap=False, max_samples=0.5) + err_msg = ( + r"`max_sample` cannot be set if `bootstrap=False`. " + r"Either switch to `bootstrap=True` or set " + r"`max_sample=None`." + ) + with pytest.raises(ValueError, match=err_msg): + est.fit(X, y) + + +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS_REGRESSORS) +def test_large_max_samples_exception(name): + # Check invalid `max_samples` + est = FOREST_CLASSIFIERS_REGRESSORS[name](bootstrap=True, max_samples=int(1e9)) + match = "`max_samples` must be <= n_samples=6 but got value 1000000000" + with pytest.raises(ValueError, match=match): est.fit(X, y) -def test_forest_y_sparse(): +@pytest.mark.parametrize("name", FOREST_REGRESSORS) +def test_max_samples_boundary_regressors(name): + X_train, X_test, y_train, y_test = train_test_split( + X_reg, y_reg, train_size=0.7, test_size=0.3, random_state=0 + ) + + ms_1_model = FOREST_REGRESSORS[name]( + bootstrap=True, max_samples=1.0, random_state=0 + ) + ms_1_predict = ms_1_model.fit(X_train, y_train).predict(X_test) + + ms_None_model = FOREST_REGRESSORS[name]( + bootstrap=True, max_samples=None, random_state=0 + ) + ms_None_predict = ms_None_model.fit(X_train, y_train).predict(X_test) + + ms_1_ms = mean_squared_error(ms_1_predict, y_test) + ms_None_ms = mean_squared_error(ms_None_predict, y_test) + + assert ms_1_ms == pytest.approx(ms_None_ms) + + +@pytest.mark.parametrize("name", FOREST_CLASSIFIERS) +def test_max_samples_boundary_classifiers(name): + X_train, X_test, y_train, _ = train_test_split( + X_large, y_large, random_state=0, stratify=y_large + ) + + ms_1_model = FOREST_CLASSIFIERS[name]( + bootstrap=True, max_samples=1.0, random_state=0 + ) + ms_1_proba = ms_1_model.fit(X_train, y_train).predict_proba(X_test) + + ms_None_model = FOREST_CLASSIFIERS[name]( + bootstrap=True, max_samples=None, random_state=0 + ) + ms_None_proba = ms_None_model.fit(X_train, y_train).predict_proba(X_test) + + np.testing.assert_allclose(ms_1_proba, ms_None_proba) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_forest_y_sparse(csr_container): X = [[1, 2, 3]] - y = csr_matrix([4, 5, 6]) + y = csr_container([[4, 5, 6]]) est = RandomForestClassifier() msg = "sparse multilabel-indicator for y is not supported." with pytest.raises(ValueError, match=msg): est.fit(X, y) -@pytest.mark.parametrize( - 'ForestClass', [RandomForestClassifier, RandomForestRegressor] -) +@pytest.mark.parametrize("ForestClass", [RandomForestClassifier, RandomForestRegressor]) def test_little_tree_with_small_max_samples(ForestClass): rng = np.random.RandomState(1) @@ -1478,25 +1630,7 @@ def test_little_tree_with_small_max_samples(ForestClass): assert tree1.node_count > tree2.node_count, msg -# FIXME: remove in 1.2 -@pytest.mark.parametrize( - "Estimator", - [ExtraTreesClassifier, ExtraTreesRegressor, - RandomForestClassifier, RandomForestRegressor, - RandomTreesEmbedding] -) -def test_n_features_deprecation(Estimator): - # Check that we raise the proper deprecation warning if accessing - # `n_features_`. - X = np.array([[1, 2], [3, 4]]) - y = np.array([1, 0]) - est = Estimator().fit(X, y) - - with pytest.warns(FutureWarning, match="n_features_ was deprecated"): - est.n_features_ - - -@pytest.mark.parametrize('Forest', FOREST_REGRESSORS) +@pytest.mark.parametrize("Forest", FOREST_REGRESSORS) def test_mse_criterion_object_segfault_smoke_test(Forest): # This is a smoke test to ensure that passing a mutable criterion # does not cause a segfault when fitting with concurrent threads. @@ -1507,8 +1641,225 @@ def test_mse_criterion_object_segfault_smoke_test(Forest): y = y_reg.reshape(-1, 1) n_samples, n_outputs = y.shape mse_criterion = MSE(n_outputs, n_samples) - est = FOREST_REGRESSORS[Forest]( - n_estimators=2, n_jobs=2, criterion=mse_criterion - ) + est = FOREST_REGRESSORS[Forest](n_estimators=2, n_jobs=2, criterion=mse_criterion) est.fit(X_reg, y) + + +def test_random_trees_embedding_feature_names_out(): + """Check feature names out for Random Trees Embedding.""" + random_state = np.random.RandomState(0) + X = np.abs(random_state.randn(100, 4)) + hasher = RandomTreesEmbedding( + n_estimators=2, max_depth=2, sparse_output=False, random_state=0 + ).fit(X) + names = hasher.get_feature_names_out() + expected_names = [ + f"randomtreesembedding_{tree}_{leaf}" + # Note: nodes with indices 0, 1 and 4 are internal split nodes and + # therefore do not appear in the expected output feature names. + for tree, leaf in [ + (0, 2), + (0, 3), + (0, 5), + (0, 6), + (1, 2), + (1, 3), + (1, 5), + (1, 6), + ] + ] + assert_array_equal(expected_names, names) + + +@pytest.mark.parametrize("csr_container", CSR_CONTAINERS) +def test_read_only_buffer(csr_container, monkeypatch): + """RandomForestClassifier must work on readonly sparse data. + + Non-regression test for: https://github.com/scikit-learn/scikit-learn/issues/25333 + """ + monkeypatch.setattr( + sklearn.ensemble._forest, + "Parallel", + partial(Parallel, max_nbytes=100), + ) + rng = np.random.RandomState(seed=0) + + X, y = make_classification(n_samples=100, n_features=200, random_state=rng) + X = csr_container(X, copy=True) + + clf = RandomForestClassifier(n_jobs=2, random_state=rng) + cross_val_score(clf, X, y, cv=2) + + +@pytest.mark.parametrize("class_weight", ["balanced_subsample", None]) +def test_round_samples_to_one_when_samples_too_low(class_weight): + """Check low max_samples works and is rounded to one. + + Non-regression test for gh-24037. + """ + X, y = datasets.load_wine(return_X_y=True) + forest = RandomForestClassifier( + n_estimators=10, max_samples=1e-4, class_weight=class_weight, random_state=0 + ) + forest.fit(X, y) + + +@pytest.mark.parametrize("seed", [None, 1]) +@pytest.mark.parametrize("bootstrap", [True, False]) +@pytest.mark.parametrize("ForestClass", FOREST_CLASSIFIERS_REGRESSORS.values()) +def test_estimators_samples(ForestClass, bootstrap, seed): + """Estimators_samples_ property should be consistent. + + Tests consistency across fits and whether or not the seed for the random generator + is set. + """ + X, y = make_hastie_10_2(n_samples=200, random_state=1) + + if bootstrap: + max_samples = 0.5 + else: + max_samples = None + est = ForestClass( + n_estimators=10, + max_samples=max_samples, + max_features=0.5, + random_state=seed, + bootstrap=bootstrap, + ) + est.fit(X, y) + + estimators_samples = est.estimators_samples_.copy() + + # Test repeated calls result in same set of indices + assert_array_equal(estimators_samples, est.estimators_samples_) + estimators = est.estimators_ + + assert isinstance(estimators_samples, list) + assert len(estimators_samples) == len(estimators) + assert estimators_samples[0].dtype == np.int32 + + for i in range(len(estimators)): + if bootstrap: + assert len(estimators_samples[i]) == len(X) // 2 + + # the bootstrap should be a resampling with replacement + assert len(np.unique(estimators_samples[i])) < len(estimators_samples[i]) + else: + assert len(set(estimators_samples[i])) == len(X) + + estimator_index = 0 + estimator_samples = estimators_samples[estimator_index] + estimator = estimators[estimator_index] + + X_train = X[estimator_samples] + y_train = y[estimator_samples] + + orig_tree_values = estimator.tree_.value + estimator = clone(estimator) + estimator.fit(X_train, y_train) + new_tree_values = estimator.tree_.value + assert_allclose(orig_tree_values, new_tree_values) + + +@pytest.mark.parametrize( + "make_data, Forest", + [ + (datasets.make_regression, RandomForestRegressor), + (datasets.make_classification, RandomForestClassifier), + (datasets.make_regression, ExtraTreesRegressor), + (datasets.make_classification, ExtraTreesClassifier), + ], +) +def test_missing_values_is_resilient(make_data, Forest): + """Check that forest can deal with missing values and has decent performance.""" + + rng = np.random.RandomState(0) + n_samples, n_features = 1000, 10 + X, y = make_data(n_samples=n_samples, n_features=n_features, random_state=rng) + + # Create dataset with missing values + X_missing = X.copy() + X_missing[rng.choice([False, True], size=X.shape, p=[0.95, 0.05])] = np.nan + assert np.isnan(X_missing).any() + + X_missing_train, X_missing_test, y_train, y_test = train_test_split( + X_missing, y, random_state=0 + ) + + # Train forest with missing values + forest_with_missing = Forest(random_state=rng, n_estimators=50) + forest_with_missing.fit(X_missing_train, y_train) + score_with_missing = forest_with_missing.score(X_missing_test, y_test) + + # Train forest without missing values + X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0) + forest = Forest(random_state=rng, n_estimators=50) + forest.fit(X_train, y_train) + score_without_missing = forest.score(X_test, y_test) + + # Score is still 80 percent of the forest's score that had no missing values + assert score_with_missing >= 0.80 * score_without_missing + + +@pytest.mark.parametrize( + "Forest", + [ + RandomForestClassifier, + RandomForestRegressor, + ExtraTreesRegressor, + ExtraTreesClassifier, + ], +) +def test_missing_value_is_predictive(Forest): + """Check that the forest learns when missing values are only present for + a predictive feature.""" + rng = np.random.RandomState(0) + n_samples = 300 + expected_score = 0.75 + + X_non_predictive = rng.standard_normal(size=(n_samples, 10)) + y = rng.randint(0, high=2, size=n_samples) + + # Create a predictive feature using `y` and with some noise + X_random_mask = rng.choice([False, True], size=n_samples, p=[0.95, 0.05]) + y_mask = y.astype(bool) + y_mask[X_random_mask] = ~y_mask[X_random_mask] + + predictive_feature = rng.standard_normal(size=n_samples) + predictive_feature[y_mask] = np.nan + assert np.isnan(predictive_feature).any() + + X_predictive = X_non_predictive.copy() + X_predictive[:, 5] = predictive_feature + + ( + X_predictive_train, + X_predictive_test, + X_non_predictive_train, + X_non_predictive_test, + y_train, + y_test, + ) = train_test_split(X_predictive, X_non_predictive, y, random_state=0) + forest_predictive = Forest(random_state=0).fit(X_predictive_train, y_train) + forest_non_predictive = Forest(random_state=0).fit(X_non_predictive_train, y_train) + + predictive_test_score = forest_predictive.score(X_predictive_test, y_test) + + assert predictive_test_score >= expected_score + assert predictive_test_score >= forest_non_predictive.score( + X_non_predictive_test, y_test + ) + + +@pytest.mark.parametrize("Forest", FOREST_REGRESSORS.values()) +def test_non_supported_criterion_raises_error_with_missing_values(Forest): + """Raise error for unsupported criterion when there are missing values.""" + X = np.array([[0, 1, 2], [np.nan, 0, 2.0]]) + y = [0.5, 1.0] + + forest = Forest(criterion="absolute_error") + + msg = ".*does not accept missing values" + with pytest.raises(ValueError, match=msg): + forest.fit(X, y) diff --git a/sklearn/ensemble/tests/test_gradient_boosting.py b/sklearn/ensemble/tests/test_gradient_boosting.py index 63d4e668e674f..f799d51eec25c 100644 --- a/sklearn/ensemble/tests/test_gradient_boosting.py +++ b/sklearn/ensemble/tests/test_gradient_boosting.py @@ -1,43 +1,39 @@ """ Testing for the gradient boosting module (sklearn.ensemble.gradient_boosting). """ -import warnings -import numpy as np -from scipy.sparse import csr_matrix -from scipy.sparse import csc_matrix -from scipy.sparse import coo_matrix -from scipy.special import expit +import re +import warnings +import numpy as np import pytest +from numpy.testing import assert_allclose from sklearn import datasets from sklearn.base import clone -from sklearn.datasets import (make_classification, - make_regression) -from sklearn.ensemble import GradientBoostingClassifier -from sklearn.ensemble import GradientBoostingRegressor +from sklearn.datasets import make_classification, make_regression +from sklearn.dummy import DummyClassifier, DummyRegressor +from sklearn.ensemble import GradientBoostingClassifier, GradientBoostingRegressor +from sklearn.ensemble._gb import _safe_divide from sklearn.ensemble._gradient_boosting import predict_stages -from sklearn.preprocessing import OneHotEncoder, scale -from sklearn.svm import LinearSVC +from sklearn.exceptions import DataConversionWarning, NotFittedError +from sklearn.linear_model import LinearRegression from sklearn.metrics import mean_squared_error from sklearn.model_selection import train_test_split -from sklearn.utils import check_random_state, tosequence -from sklearn.utils._mocking import NoSampleWeightWrapper -from sklearn.utils._testing import assert_almost_equal -from sklearn.utils._testing import assert_array_almost_equal -from sklearn.utils._testing import assert_array_equal -from sklearn.utils._testing import skip_if_32bit -from sklearn.exceptions import DataConversionWarning -from sklearn.exceptions import NotFittedError -from sklearn.dummy import DummyClassifier, DummyRegressor from sklearn.pipeline import make_pipeline -from sklearn.linear_model import LinearRegression +from sklearn.preprocessing import scale from sklearn.svm import NuSVR +from sklearn.utils import check_random_state +from sklearn.utils._mocking import NoSampleWeightWrapper +from sklearn.utils._param_validation import InvalidParameterError +from sklearn.utils._testing import ( + assert_array_almost_equal, + assert_array_equal, + skip_if_32bit, +) +from sklearn.utils.fixes import COO_CONTAINERS, CSC_CONTAINERS, CSR_CONTAINERS - -GRADIENT_BOOSTING_ESTIMATORS = [GradientBoostingClassifier, - GradientBoostingRegressor] +GRADIENT_BOOSTING_ESTIMATORS = [GradientBoostingClassifier, GradientBoostingRegressor] # toy sample X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]] @@ -47,7 +43,7 @@ # also make regression dataset X_reg, y_reg = make_regression( - n_samples=500, n_features=10, n_informative=8, noise=10, random_state=7 + n_samples=100, n_features=4, n_informative=8, noise=10, random_state=7 ) y_reg = scale(y_reg) @@ -60,11 +56,31 @@ iris.target = iris.target[perm] -@pytest.mark.parametrize('loss', ('deviance', 'exponential')) -def test_classification_toy(loss): +def test_exponential_n_classes_gt_2(): + """Test exponential loss raises for n_classes > 2.""" + clf = GradientBoostingClassifier(loss="exponential") + msg = "loss='exponential' is only suitable for a binary classification" + with pytest.raises(ValueError, match=msg): + clf.fit(iris.data, iris.target) + + +def test_raise_if_init_has_no_predict_proba(): + """Test raise if init_ has no predict_proba method.""" + clf = GradientBoostingClassifier(init=GradientBoostingRegressor) + msg = ( + "The 'init' parameter of GradientBoostingClassifier must be a str among " + "{'zero'}, None or an object implementing 'fit' and 'predict_proba'." + ) + with pytest.raises(ValueError, match=msg): + clf.fit(X, y) + + +@pytest.mark.parametrize("loss", ("log_loss", "exponential")) +def test_classification_toy(loss, global_random_seed): # Check classification on a toy dataset. - clf = GradientBoostingClassifier(loss=loss, n_estimators=10, - random_state=1) + clf = GradientBoostingClassifier( + loss=loss, n_estimators=10, random_state=global_random_seed + ) with pytest.raises(ValueError): clf.predict(T) @@ -73,125 +89,87 @@ def test_classification_toy(loss): assert_array_equal(clf.predict(T), true_result) assert 10 == len(clf.estimators_) - deviance_decrease = (clf.train_score_[:-1] - clf.train_score_[1:]) - assert np.any(deviance_decrease >= 0.0) + log_loss_decrease = clf.train_score_[:-1] - clf.train_score_[1:] + assert np.any(log_loss_decrease >= 0.0) leaves = clf.apply(X) assert leaves.shape == (6, 10, 1) -@pytest.mark.parametrize( - "params, err_msg", - [({"n_estimators": 0}, "n_estimators must be greater than 0"), - ({"n_estimators": -1}, "n_estimators must be greater than 0"), - ({"learning_rate": 0}, "learning_rate must be greater than 0"), - ({"learning_rate": -1.0}, "learning_rate must be greater than 0"), - ({"loss": "foobar"}, "Loss 'foobar' not supported"), - ({"min_samples_split": 0.0}, "min_samples_split must be an integer"), - ({"min_samples_split": -1.0}, "min_samples_split must be an integer"), - ({"min_samples_split": 1.1}, "min_samples_split must be an integer"), - ({"min_samples_leaf": 0}, "min_samples_leaf must be at least 1 or"), - ({"min_samples_leaf": -1.0}, "min_samples_leaf must be at least 1 or"), - ({"min_weight_fraction_leaf": -1.0}, "min_weight_fraction_leaf must in"), - ({"min_weight_fraction_leaf": 0.6}, "min_weight_fraction_leaf must in"), - ({"subsample": 0.0}, r"subsample must be in \(0,1\]"), - ({"subsample": 1.1}, r"subsample must be in \(0,1\]"), - ({"subsample": -0.1}, r"subsample must be in \(0,1\]"), - ({"max_depth": -0.1}, "max_depth must be greater than zero"), - ({"max_depth": 0}, "max_depth must be greater than zero"), - ({"init": {}}, "The init parameter must be an estimator or 'zero'"), - ({"max_features": "invalid"}, "Invalid value for max_features:"), - ({"max_features": 0}, r"max_features must be in \(0, n_features\]"), - ({"max_features": 100}, r"max_features must be in \(0, n_features\]"), - ({"max_features": -0.1}, r"max_features must be in \(0, n_features\]"), - ({"n_iter_no_change": "invalid"}, "n_iter_no_change should either be")] -) -@pytest.mark.parametrize( - "GradientBoosting, X, y", - [(GradientBoostingRegressor, X_reg, y_reg), - (GradientBoostingClassifier, iris.data, iris.target)] -) -def test_gbdt_parameter_checks(GradientBoosting, X, y, params, err_msg): - # Check input parameter validation for GradientBoosting - with pytest.raises(ValueError, match=err_msg): - GradientBoosting(**params).fit(X, y) - - -@pytest.mark.parametrize( - "params, err_msg", - [({"loss": "huber", "alpha": 1.2}, r"alpha must be in \(0.0, 1.0\)"), - ({"loss": "quantile", "alpha": 1.2}, r"alpha must be in \(0.0, 1.0\)")] -) -def test_gbdt_loss_alpha_error(params, err_msg): - # check that an error is raised when alpha is not proper for quantile and - # huber loss - with pytest.raises(ValueError, match=err_msg): - GradientBoostingRegressor(**params).fit(X_reg, y_reg) - - -@pytest.mark.parametrize( - "GradientBoosting, loss", - [(GradientBoostingClassifier, "ls"), - (GradientBoostingClassifier, "lad"), - (GradientBoostingClassifier, "quantile"), - (GradientBoostingClassifier, "huber"), - (GradientBoostingRegressor, "deviance"), - (GradientBoostingRegressor, "exponential")] -) -def test_wrong_type_loss_function(GradientBoosting, loss): - # check that we raise an error when not using the right type of loss - # function - with pytest.raises(ValueError): - GradientBoosting(loss=loss).fit(X, y) - - -@pytest.mark.parametrize('loss', ('deviance', 'exponential')) -def test_classification_synthetic(loss): +@pytest.mark.parametrize("loss", ("log_loss", "exponential")) +def test_classification_synthetic(loss, global_random_seed): # Test GradientBoostingClassifier on synthetic dataset used by - # Hastie et al. in ESLII Example 12.7. - X, y = datasets.make_hastie_10_2(n_samples=12000, random_state=1) - - X_train, X_test = X[:2000], X[2000:] - y_train, y_test = y[:2000], y[2000:] - - gbrt = GradientBoostingClassifier(n_estimators=100, min_samples_split=2, - max_depth=1, loss=loss, - learning_rate=1.0, random_state=0) - gbrt.fit(X_train, y_train) - error_rate = (1.0 - gbrt.score(X_test, y_test)) - assert error_rate < 0.09 - - gbrt = GradientBoostingClassifier(n_estimators=200, min_samples_split=2, - max_depth=1, loss=loss, - learning_rate=1.0, subsample=0.5, - random_state=0) - gbrt.fit(X_train, y_train) - error_rate = (1.0 - gbrt.score(X_test, y_test)) - assert error_rate < 0.08 - - -@pytest.mark.parametrize('loss', ('ls', 'lad', 'huber')) -@pytest.mark.parametrize('subsample', (1.0, 0.5)) -def test_regression_dataset(loss, subsample): + # Hastie et al. in ESLII - Figure 10.9 + # Note that Figure 10.9 reuses the dataset generated for figure 10.2 + # and should have 2_000 train data points and 10_000 test data points. + # Here we intentionally use a smaller variant to make the test run faster, + # but the conclusions are still the same, despite the smaller datasets. + X, y = datasets.make_hastie_10_2(n_samples=2000, random_state=global_random_seed) + + split_idx = 500 + X_train, X_test = X[:split_idx], X[split_idx:] + y_train, y_test = y[:split_idx], y[split_idx:] + + # Increasing the number of trees should decrease the test error + common_params = { + "max_depth": 1, + "learning_rate": 1.0, + "loss": loss, + "random_state": global_random_seed, + } + gbrt_10_stumps = GradientBoostingClassifier(n_estimators=10, **common_params) + gbrt_10_stumps.fit(X_train, y_train) + + gbrt_50_stumps = GradientBoostingClassifier(n_estimators=50, **common_params) + gbrt_50_stumps.fit(X_train, y_train) + + assert gbrt_10_stumps.score(X_test, y_test) < gbrt_50_stumps.score(X_test, y_test) + + # Decision stumps are better suited for this dataset with a large number of + # estimators. + common_params = { + "n_estimators": 200, + "learning_rate": 1.0, + "loss": loss, + "random_state": global_random_seed, + } + gbrt_stumps = GradientBoostingClassifier(max_depth=1, **common_params) + gbrt_stumps.fit(X_train, y_train) + + gbrt_10_nodes = GradientBoostingClassifier(max_leaf_nodes=10, **common_params) + gbrt_10_nodes.fit(X_train, y_train) + + assert gbrt_stumps.score(X_test, y_test) > gbrt_10_nodes.score(X_test, y_test) + + +@pytest.mark.parametrize("loss", ("squared_error", "absolute_error", "huber")) +@pytest.mark.parametrize("subsample", (1.0, 0.5)) +def test_regression_dataset(loss, subsample, global_random_seed): # Check consistency on regression dataset with least squares # and least absolute deviation. ones = np.ones(len(y_reg)) last_y_pred = None for sample_weight in [None, ones, 2 * ones]: - reg = GradientBoostingRegressor(n_estimators=100, - loss=loss, - max_depth=4, - subsample=subsample, - min_samples_split=2, - random_state=1) + # learning_rate, max_depth and n_estimators were adjusted to get a mode + # that is accurate enough to reach a low MSE on the training set while + # keeping the resource used to execute this test low enough. + reg = GradientBoostingRegressor( + n_estimators=30, + loss=loss, + max_depth=4, + subsample=subsample, + min_samples_split=2, + random_state=global_random_seed, + learning_rate=0.5, + ) reg.fit(X_reg, y_reg, sample_weight=sample_weight) leaves = reg.apply(X_reg) - assert leaves.shape == (500, 100) + assert leaves.shape == (100, 30) y_pred = reg.predict(X_reg) mse = mean_squared_error(y_reg, y_pred) - assert mse < 0.04 + assert mse < 0.05 if last_y_pred is not None: # FIXME: We temporarily bypass this test. This is due to the fact @@ -205,16 +183,18 @@ def test_regression_dataset(loss, subsample): last_y_pred = y_pred -@pytest.mark.parametrize('subsample', (1.0, 0.5)) -@pytest.mark.parametrize('sample_weight', (None, 1)) -def test_iris(subsample, sample_weight): +@pytest.mark.parametrize("subsample", (1.0, 0.5)) +@pytest.mark.parametrize("sample_weight", (None, 1)) +def test_iris(subsample, sample_weight, global_random_seed): if sample_weight == 1: sample_weight = np.ones(len(iris.target)) # Check consistency on dataset iris. - clf = GradientBoostingClassifier(n_estimators=100, - loss="deviance", - random_state=1, - subsample=subsample) + clf = GradientBoostingClassifier( + n_estimators=100, + loss="log_loss", + random_state=global_random_seed, + subsample=subsample, + ) clf.fit(iris.data, iris.target, sample_weight=sample_weight) score = clf.score(iris.data, iris.target) assert score > 0.9 @@ -223,25 +203,28 @@ def test_iris(subsample, sample_weight): assert leaves.shape == (150, 100, 3) -def test_regression_synthetic(): +def test_regression_synthetic(global_random_seed): # Test on synthetic regression datasets used in Leo Breiman, # `Bagging Predictors?. Machine Learning 24(2): 123-140 (1996). - random_state = check_random_state(1) - regression_params = {'n_estimators': 100, 'max_depth': 4, - 'min_samples_split': 2, 'learning_rate': 0.1, - 'loss': 'ls'} + random_state = check_random_state(global_random_seed) + regression_params = { + "n_estimators": 100, + "max_depth": 4, + "min_samples_split": 2, + "learning_rate": 0.1, + "loss": "squared_error", + "random_state": global_random_seed, + } # Friedman1 - X, y = datasets.make_friedman1(n_samples=1200, - random_state=random_state, - noise=1.0) + X, y = datasets.make_friedman1(n_samples=1200, random_state=random_state, noise=1.0) X_train, y_train = X[:200], y[:200] X_test, y_test = X[200:], y[200:] - clf = GradientBoostingRegressor() + clf = GradientBoostingRegressor(**regression_params) clf.fit(X_train, y_train) mse = mean_squared_error(y_test, clf.predict(X_test)) - assert mse < 5.0 + assert mse < 6.5 # Friedman2 X, y = datasets.make_friedman2(n_samples=1200, random_state=random_state) @@ -251,7 +234,7 @@ def test_regression_synthetic(): clf = GradientBoostingRegressor(**regression_params) clf.fit(X_train, y_train) mse = mean_squared_error(y_test, clf.predict(X_test)) - assert mse < 1700.0 + assert mse < 2500.0 # Friedman3 X, y = datasets.make_friedman3(n_samples=1200, random_state=random_state) @@ -261,13 +244,15 @@ def test_regression_synthetic(): clf = GradientBoostingRegressor(**regression_params) clf.fit(X_train, y_train) mse = mean_squared_error(y_test, clf.predict(X_test)) - assert mse < 0.015 + assert mse < 0.025 @pytest.mark.parametrize( "GradientBoosting, X, y", - [(GradientBoostingRegressor, X_reg, y_reg), - (GradientBoostingClassifier, iris.data, iris.target)] + [ + (GradientBoostingRegressor, X_reg, y_reg), + (GradientBoostingClassifier, iris.data, iris.target), + ], ) def test_feature_importances(GradientBoosting, X, y): # smoke test to check that the gradient boosting expose an attribute @@ -275,12 +260,12 @@ def test_feature_importances(GradientBoosting, X, y): gbdt = GradientBoosting() assert not hasattr(gbdt, "feature_importances_") gbdt.fit(X, y) - assert hasattr(gbdt, 'feature_importances_') + assert hasattr(gbdt, "feature_importances_") -def test_probability_log(): +def test_probability_log(global_random_seed): # Predict probabilities. - clf = GradientBoostingClassifier(n_estimators=100, random_state=1) + clf = GradientBoostingClassifier(n_estimators=100, random_state=global_random_seed) with pytest.raises(ValueError): clf.predict_proba(T) @@ -309,11 +294,12 @@ def test_single_class_with_sample_weight(): clf.fit(X, y, sample_weight=sample_weight) -def test_check_inputs_predict_stages(): +@pytest.mark.parametrize("csc_container", CSC_CONTAINERS) +def test_check_inputs_predict_stages(csc_container): # check that predict_stages through an error if the type of X is not # supported x, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) - x_sparse_csc = csc_matrix(x) + x_sparse_csc = csc_container(x) clf = GradientBoostingClassifier(n_estimators=100, random_state=1) clf.fit(x, y) score = np.zeros((y.shape)).reshape(-1, 1) @@ -325,22 +311,29 @@ def test_check_inputs_predict_stages(): predict_stages(clf.estimators_, x_fortran, clf.learning_rate, score) -def test_max_feature_regression(): +def test_max_feature_regression(global_random_seed): # Test to make sure random state is set properly. - X, y = datasets.make_hastie_10_2(n_samples=12000, random_state=1) + X, y = datasets.make_hastie_10_2(n_samples=12000, random_state=global_random_seed) X_train, X_test = X[:2000], X[2000:] y_train, y_test = y[:2000], y[2000:] - gbrt = GradientBoostingClassifier(n_estimators=100, min_samples_split=5, - max_depth=2, learning_rate=.1, - max_features=2, random_state=1) + gbrt = GradientBoostingClassifier( + n_estimators=100, + min_samples_split=5, + max_depth=2, + learning_rate=0.1, + max_features=2, + random_state=global_random_seed, + ) gbrt.fit(X_train, y_train) - deviance = gbrt.loss_(y_test, gbrt.decision_function(X_test)) - assert deviance < 0.5, "GB failed with deviance %.4f" % deviance + log_loss = gbrt._loss(y_test, gbrt.decision_function(X_test)) + assert log_loss < 0.5, "GB failed with deviance %.4f" % log_loss -def test_feature_importance_regression(fetch_california_housing_fxt): +def test_feature_importance_regression( + fetch_california_housing_fxt, global_random_seed +): """Test that Gini importance is calculated correctly. This test follows the example from [1]_ (pg. 373). @@ -350,25 +343,31 @@ def test_feature_importance_regression(fetch_california_housing_fxt): """ california = fetch_california_housing_fxt() X, y = california.data, california.target - X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0) + X_train, X_test, y_train, y_test = train_test_split( + X, y, random_state=global_random_seed + ) - reg = GradientBoostingRegressor(loss='huber', learning_rate=0.1, - max_leaf_nodes=6, n_estimators=100, - random_state=0) + reg = GradientBoostingRegressor( + loss="huber", + learning_rate=0.1, + max_leaf_nodes=6, + n_estimators=100, + random_state=global_random_seed, + ) reg.fit(X_train, y_train) sorted_idx = np.argsort(reg.feature_importances_)[::-1] sorted_features = [california.feature_names[s] for s in sorted_idx] # The most important feature is the median income by far. - assert sorted_features[0] == 'MedInc' + assert sorted_features[0] == "MedInc" # The three subsequent features are the following. Their relative ordering # might change a bit depending on the randomness of the trees and the # train / test split. - assert set(sorted_features[1:4]) == {'Longitude', 'AveOccup', 'Latitude'} + assert set(sorted_features[1:4]) == {"Longitude", "AveOccup", "Latitude"} -def test_max_feature_auto(): +def test_max_features(): # Test if max features is set properly for floats and str. X, y = datasets.make_hastie_10_2(n_samples=12000, random_state=1) _, n_features = X.shape @@ -376,11 +375,11 @@ def test_max_feature_auto(): X_train = X[:2000] y_train = y[:2000] - gbrt = GradientBoostingClassifier(n_estimators=1, max_features='auto') + gbrt = GradientBoostingClassifier(n_estimators=1, max_features=None) gbrt.fit(X_train, y_train) - assert gbrt.max_features_ == int(np.sqrt(n_features)) + assert gbrt.max_features_ == n_features - gbrt = GradientBoostingRegressor(n_estimators=1, max_features='auto') + gbrt = GradientBoostingRegressor(n_estimators=1, max_features=None) gbrt.fit(X_train, y_train) assert gbrt.max_features_ == n_features @@ -388,16 +387,15 @@ def test_max_feature_auto(): gbrt.fit(X_train, y_train) assert gbrt.max_features_ == int(n_features * 0.3) - gbrt = GradientBoostingRegressor(n_estimators=1, max_features='sqrt') + gbrt = GradientBoostingRegressor(n_estimators=1, max_features="sqrt") gbrt.fit(X_train, y_train) assert gbrt.max_features_ == int(np.sqrt(n_features)) - gbrt = GradientBoostingRegressor(n_estimators=1, max_features='log2') + gbrt = GradientBoostingRegressor(n_estimators=1, max_features="log2") gbrt.fit(X_train, y_train) assert gbrt.max_features_ == int(np.log2(n_features)) - gbrt = GradientBoostingRegressor(n_estimators=1, - max_features=0.01 / X.shape[1]) + gbrt = GradientBoostingRegressor(n_estimators=1, max_features=0.01 / X.shape[1]) gbrt.fit(X_train, y_train) assert gbrt.max_features_ == 1 @@ -405,8 +403,7 @@ def test_max_feature_auto(): def test_staged_predict(): # Test whether staged decision function eventually gives # the same prediction. - X, y = datasets.make_friedman1(n_samples=1200, - random_state=1, noise=1.0) + X, y = datasets.make_friedman1(n_samples=1200, random_state=1, noise=1.0) X_train, y_train = X[:200], y[:200] X_test = X[200:] clf = GradientBoostingRegressor() @@ -427,8 +424,7 @@ def test_staged_predict(): def test_staged_predict_proba(): # Test whether staged predict proba eventually gives # the same prediction. - X, y = datasets.make_hastie_10_2(n_samples=1200, - random_state=1) + X, y = datasets.make_hastie_10_2(n_samples=1200, random_state=1) X_train, y_train = X[:200], y[:200] X_test, y_test = X[200:], y[200:] clf = GradientBoostingClassifier(n_estimators=20) @@ -452,15 +448,15 @@ def test_staged_predict_proba(): assert_array_almost_equal(clf.predict_proba(X_test), staged_proba) -@pytest.mark.parametrize('Estimator', GRADIENT_BOOSTING_ESTIMATORS) -def test_staged_functions_defensive(Estimator): +@pytest.mark.parametrize("Estimator", GRADIENT_BOOSTING_ESTIMATORS) +def test_staged_functions_defensive(Estimator, global_random_seed): # test that staged_functions make defensive copies - rng = np.random.RandomState(0) + rng = np.random.RandomState(global_random_seed) X = rng.uniform(size=(10, 3)) y = (4 * X[:, 0]).astype(int) + 1 # don't predict zeros estimator = Estimator() estimator.fit(X, y) - for func in ['predict', 'decision_function', 'predict_proba']: + for func in ["predict", "decision_function", "predict_proba"]: staged_func = getattr(estimator, "staged_" + func, None) if staged_func is None: # regressor has no staged_predict_proba @@ -502,35 +498,42 @@ def test_degenerate_targets(): clf = GradientBoostingRegressor(n_estimators=100, random_state=1) clf.fit(X, np.ones(len(X))) clf.predict([rng.rand(2)]) - assert_array_equal(np.ones((1,), dtype=np.float64), - clf.predict([rng.rand(2)])) + assert_array_equal(np.ones((1,), dtype=np.float64), clf.predict([rng.rand(2)])) -def test_quantile_loss(): - # Check if quantile loss with alpha=0.5 equals lad. - clf_quantile = GradientBoostingRegressor(n_estimators=100, loss='quantile', - max_depth=4, alpha=0.5, - random_state=7) +def test_quantile_loss(global_random_seed): + # Check if quantile loss with alpha=0.5 equals absolute_error. + clf_quantile = GradientBoostingRegressor( + n_estimators=100, + loss="quantile", + max_depth=4, + alpha=0.5, + random_state=global_random_seed, + ) clf_quantile.fit(X_reg, y_reg) y_quantile = clf_quantile.predict(X_reg) - clf_lad = GradientBoostingRegressor(n_estimators=100, loss='lad', - max_depth=4, random_state=7) + clf_ae = GradientBoostingRegressor( + n_estimators=100, + loss="absolute_error", + max_depth=4, + random_state=global_random_seed, + ) - clf_lad.fit(X_reg, y_reg) - y_lad = clf_lad.predict(X_reg) - assert_array_almost_equal(y_quantile, y_lad, decimal=4) + clf_ae.fit(X_reg, y_reg) + y_ae = clf_ae.predict(X_reg) + assert_allclose(y_quantile, y_ae) def test_symbol_labels(): # Test with non-integer class labels. clf = GradientBoostingClassifier(n_estimators=100, random_state=1) - symbol_y = tosequence(map(str, y)) + symbol_y = list(map(str, y)) clf.fit(X, symbol_y) - assert_array_equal(clf.predict(T), tosequence(map(str, true_result))) + assert_array_equal(clf.predict(T), list(map(str, true_result))) assert 100 == len(clf.estimators_) @@ -541,8 +544,7 @@ def test_float_class_labels(): float_y = np.asarray(y, dtype=np.float32) clf.fit(X, float_y) - assert_array_equal(clf.predict(T), - np.asarray(true_result, dtype=np.float32)) + assert_array_equal(clf.predict(T), np.asarray(true_result, dtype=np.float32)) assert 100 == len(clf.estimators_) @@ -596,51 +598,112 @@ def test_mem_layout(): assert 100 == len(clf.estimators_) -def test_oob_improvement(): +@pytest.mark.parametrize("GradientBoostingEstimator", GRADIENT_BOOSTING_ESTIMATORS) +def test_oob_improvement(GradientBoostingEstimator): # Test if oob improvement has correct shape and regression test. - clf = GradientBoostingClassifier(n_estimators=100, random_state=1, - subsample=0.5) - clf.fit(X, y) - assert clf.oob_improvement_.shape[0] == 100 + estimator = GradientBoostingEstimator( + n_estimators=100, random_state=1, subsample=0.5 + ) + estimator.fit(X, y) + assert estimator.oob_improvement_.shape[0] == 100 # hard-coded regression test - change if modification in OOB computation - assert_array_almost_equal(clf.oob_improvement_[:5], - np.array([0.19, 0.15, 0.12, -0.12, -0.11]), - decimal=2) + assert_array_almost_equal( + estimator.oob_improvement_[:5], + np.array([0.19, 0.15, 0.12, -0.11, 0.11]), + decimal=2, + ) -def test_oob_improvement_raise(): - # Test if oob improvement has correct shape. - clf = GradientBoostingClassifier(n_estimators=100, random_state=1, - subsample=1.0) - clf.fit(X, y) +@pytest.mark.parametrize("GradientBoostingEstimator", GRADIENT_BOOSTING_ESTIMATORS) +def test_oob_scores(GradientBoostingEstimator): + # Test if oob scores has correct shape and regression test. + X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) + estimator = GradientBoostingEstimator( + n_estimators=100, random_state=1, subsample=0.5 + ) + estimator.fit(X, y) + assert estimator.oob_scores_.shape[0] == 100 + assert estimator.oob_scores_[-1] == pytest.approx(estimator.oob_score_) + + estimator = GradientBoostingEstimator( + n_estimators=100, + random_state=1, + subsample=0.5, + n_iter_no_change=5, + ) + estimator.fit(X, y) + assert estimator.oob_scores_.shape[0] < 100 + assert estimator.oob_scores_[-1] == pytest.approx(estimator.oob_score_) + + +@pytest.mark.parametrize( + "GradientBoostingEstimator, oob_attribute", + [ + (GradientBoostingClassifier, "oob_improvement_"), + (GradientBoostingClassifier, "oob_scores_"), + (GradientBoostingClassifier, "oob_score_"), + (GradientBoostingRegressor, "oob_improvement_"), + (GradientBoostingRegressor, "oob_scores_"), + (GradientBoostingRegressor, "oob_score_"), + ], +) +def test_oob_attributes_error(GradientBoostingEstimator, oob_attribute): + """ + Check that we raise an AttributeError when the OOB statistics were not computed. + """ + X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) + estimator = GradientBoostingEstimator( + n_estimators=100, + random_state=1, + subsample=1.0, + ) + estimator.fit(X, y) with pytest.raises(AttributeError): - clf.oob_improvement_ + estimator.oob_attribute def test_oob_multilcass_iris(): # Check OOB improvement on multi-class dataset. - clf = GradientBoostingClassifier(n_estimators=100, loss='deviance', - random_state=1, subsample=0.5) - clf.fit(iris.data, iris.target) - score = clf.score(iris.data, iris.target) + estimator = GradientBoostingClassifier( + n_estimators=100, loss="log_loss", random_state=1, subsample=0.5 + ) + estimator.fit(iris.data, iris.target) + score = estimator.score(iris.data, iris.target) assert score > 0.9 - assert clf.oob_improvement_.shape[0] == clf.n_estimators + assert estimator.oob_improvement_.shape[0] == estimator.n_estimators + assert estimator.oob_scores_.shape[0] == estimator.n_estimators + assert estimator.oob_scores_[-1] == pytest.approx(estimator.oob_score_) + + estimator = GradientBoostingClassifier( + n_estimators=100, + loss="log_loss", + random_state=1, + subsample=0.5, + n_iter_no_change=5, + ) + estimator.fit(iris.data, iris.target) + score = estimator.score(iris.data, iris.target) + assert estimator.oob_improvement_.shape[0] < estimator.n_estimators + assert estimator.oob_scores_.shape[0] < estimator.n_estimators + assert estimator.oob_scores_[-1] == pytest.approx(estimator.oob_score_) + # hard-coded regression test - change if modification in OOB computation # FIXME: the following snippet does not yield the same results on 32 bits - # assert_array_almost_equal(clf.oob_improvement_[:5], + # assert_array_almost_equal(estimator.oob_improvement_[:5], # np.array([12.68, 10.45, 8.18, 6.43, 5.13]), # decimal=2) def test_verbose_output(): # Check verbose=1 does not cause error. + import sys from io import StringIO - import sys old_stdout = sys.stdout sys.stdout = StringIO() - clf = GradientBoostingClassifier(n_estimators=100, random_state=1, - verbose=1, subsample=0.8) + clf = GradientBoostingClassifier( + n_estimators=100, random_state=1, verbose=1, subsample=0.8 + ) clf.fit(X, y) verbose_output = sys.stdout sys.stdout = old_stdout @@ -649,8 +712,12 @@ def test_verbose_output(): verbose_output.seek(0) header = verbose_output.readline().rstrip() # with OOB - true_header = ' '.join(['%10s'] + ['%16s'] * 3) % ( - 'Iter', 'Train Loss', 'OOB Improve', 'Remaining Time') + true_header = " ".join(["%10s"] + ["%16s"] * 3) % ( + "Iter", + "Train Loss", + "OOB Improve", + "Remaining Time", + ) assert true_header == header n_lines = sum(1 for l in verbose_output.readlines()) @@ -660,12 +727,12 @@ def test_verbose_output(): def test_more_verbose_output(): # Check verbose=2 does not cause error. - from io import StringIO import sys + from io import StringIO + old_stdout = sys.stdout sys.stdout = StringIO() - clf = GradientBoostingClassifier(n_estimators=100, random_state=1, - verbose=2) + clf = GradientBoostingClassifier(n_estimators=100, random_state=1, verbose=2) clf.fit(X, y) verbose_output = sys.stdout sys.stdout = old_stdout @@ -674,8 +741,11 @@ def test_more_verbose_output(): verbose_output.seek(0) header = verbose_output.readline().rstrip() # no OOB - true_header = ' '.join(['%10s'] + ['%16s'] * 2) % ( - 'Iter', 'Train Loss', 'Remaining Time') + true_header = " ".join(["%10s"] + ["%16s"] * 2) % ( + "Iter", + "Train Loss", + "Remaining Time", + ) assert true_header == header n_lines = sum(1 for l in verbose_output.readlines()) @@ -683,44 +753,47 @@ def test_more_verbose_output(): assert 100 == n_lines -@pytest.mark.parametrize('Cls', GRADIENT_BOOSTING_ESTIMATORS) -def test_warm_start(Cls): +@pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) +def test_warm_start(Cls, global_random_seed): # Test if warm start equals fit. - X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) - est = Cls(n_estimators=200, max_depth=1) + X, y = datasets.make_hastie_10_2(n_samples=100, random_state=global_random_seed) + est = Cls(n_estimators=200, max_depth=1, random_state=global_random_seed) est.fit(X, y) - est_ws = Cls(n_estimators=100, max_depth=1, warm_start=True) + est_ws = Cls( + n_estimators=100, max_depth=1, warm_start=True, random_state=global_random_seed + ) est_ws.fit(X, y) est_ws.set_params(n_estimators=200) est_ws.fit(X, y) if Cls is GradientBoostingRegressor: - assert_array_almost_equal(est_ws.predict(X), est.predict(X)) + assert_allclose(est_ws.predict(X), est.predict(X)) else: # Random state is preserved and hence predict_proba must also be # same assert_array_equal(est_ws.predict(X), est.predict(X)) - assert_array_almost_equal(est_ws.predict_proba(X), - est.predict_proba(X)) + assert_allclose(est_ws.predict_proba(X), est.predict_proba(X)) -@pytest.mark.parametrize('Cls', GRADIENT_BOOSTING_ESTIMATORS) -def test_warm_start_n_estimators(Cls): +@pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) +def test_warm_start_n_estimators(Cls, global_random_seed): # Test if warm start equals fit - set n_estimators. - X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) - est = Cls(n_estimators=300, max_depth=1) + X, y = datasets.make_hastie_10_2(n_samples=100, random_state=global_random_seed) + est = Cls(n_estimators=300, max_depth=1, random_state=global_random_seed) est.fit(X, y) - est_ws = Cls(n_estimators=100, max_depth=1, warm_start=True) + est_ws = Cls( + n_estimators=100, max_depth=1, warm_start=True, random_state=global_random_seed + ) est_ws.fit(X, y) est_ws.set_params(n_estimators=300) est_ws.fit(X, y) - assert_array_almost_equal(est_ws.predict(X), est.predict(X)) + assert_allclose(est_ws.predict(X), est.predict(X)) -@pytest.mark.parametrize('Cls', GRADIENT_BOOSTING_ESTIMATORS) +@pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) def test_warm_start_max_depth(Cls): # Test if possible to fit trees of different depth in ensemble. X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) @@ -735,7 +808,7 @@ def test_warm_start_max_depth(Cls): assert est.estimators_[-i, 0].max_depth == 2 -@pytest.mark.parametrize('Cls', GRADIENT_BOOSTING_ESTIMATORS) +@pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) def test_warm_start_clear(Cls): # Test if fit clears state. X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) @@ -750,18 +823,39 @@ def test_warm_start_clear(Cls): assert_array_almost_equal(est_2.predict(X), est.predict(X)) -@pytest.mark.parametrize('Cls', GRADIENT_BOOSTING_ESTIMATORS) -def test_warm_start_zero_n_estimators(Cls): - # Test if warm start with zero n_estimators raises error +@pytest.mark.parametrize("GradientBoosting", GRADIENT_BOOSTING_ESTIMATORS) +def test_warm_start_state_oob_scores(GradientBoosting): + """ + Check that the states of the OOB scores are cleared when used with `warm_start`. + """ X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) - est = Cls(n_estimators=100, max_depth=1, warm_start=True) - est.fit(X, y) - est.set_params(n_estimators=0) - with pytest.raises(ValueError): - est.fit(X, y) + n_estimators = 100 + estimator = GradientBoosting( + n_estimators=n_estimators, + max_depth=1, + subsample=0.5, + warm_start=True, + random_state=1, + ) + estimator.fit(X, y) + oob_scores, oob_score = estimator.oob_scores_, estimator.oob_score_ + assert len(oob_scores) == n_estimators + assert oob_scores[-1] == pytest.approx(oob_score) + + n_more_estimators = 200 + estimator.set_params(n_estimators=n_more_estimators).fit(X, y) + assert len(estimator.oob_scores_) == n_more_estimators + assert_allclose(estimator.oob_scores_[:n_estimators], oob_scores) + estimator.set_params(n_estimators=n_estimators, warm_start=False).fit(X, y) + assert estimator.oob_scores_ is not oob_scores + assert estimator.oob_score_ is not oob_score + assert_allclose(estimator.oob_scores_, oob_scores) + assert estimator.oob_score_ == pytest.approx(oob_score) + assert oob_scores[-1] == pytest.approx(oob_score) -@pytest.mark.parametrize('Cls', GRADIENT_BOOSTING_ESTIMATORS) + +@pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) def test_warm_start_smaller_n_estimators(Cls): # Test if warm start with smaller n_estimators raises error X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) @@ -772,7 +866,7 @@ def test_warm_start_smaller_n_estimators(Cls): est.fit(X, y) -@pytest.mark.parametrize('Cls', GRADIENT_BOOSTING_ESTIMATORS) +@pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) def test_warm_start_equal_n_estimators(Cls): # Test if warm start with equal n_estimators does nothing X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) @@ -786,7 +880,7 @@ def test_warm_start_equal_n_estimators(Cls): assert_array_almost_equal(est2.predict(X), est.predict(X)) -@pytest.mark.parametrize('Cls', GRADIENT_BOOSTING_ESTIMATORS) +@pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) def test_warm_start_oob_switch(Cls): # Test if oob can be turned on during warm start. X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) @@ -796,65 +890,82 @@ def test_warm_start_oob_switch(Cls): est.fit(X, y) assert_array_equal(est.oob_improvement_[:100], np.zeros(100)) + assert_array_equal(est.oob_scores_[:100], np.zeros(100)) + # the last 10 are not zeros - assert_array_equal(est.oob_improvement_[-10:] == 0.0, - np.zeros(10, dtype=bool)) + assert (est.oob_improvement_[-10:] != 0.0).all() + assert (est.oob_scores_[-10:] != 0.0).all() + assert est.oob_scores_[-1] == pytest.approx(est.oob_score_) -@pytest.mark.parametrize('Cls', GRADIENT_BOOSTING_ESTIMATORS) + +@pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) def test_warm_start_oob(Cls): # Test if warm start OOB equals fit. X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) - est = Cls(n_estimators=200, max_depth=1, subsample=0.5, - random_state=1) + est = Cls(n_estimators=200, max_depth=1, subsample=0.5, random_state=1) est.fit(X, y) - est_ws = Cls(n_estimators=100, max_depth=1, subsample=0.5, - random_state=1, warm_start=True) + est_ws = Cls( + n_estimators=100, max_depth=1, subsample=0.5, random_state=1, warm_start=True + ) est_ws.fit(X, y) est_ws.set_params(n_estimators=200) est_ws.fit(X, y) - assert_array_almost_equal(est_ws.oob_improvement_[:100], - est.oob_improvement_[:100]) + assert_array_almost_equal(est_ws.oob_improvement_[:100], est.oob_improvement_[:100]) + assert_array_almost_equal(est_ws.oob_scores_[:100], est.oob_scores_[:100]) + assert est.oob_scores_[-1] == pytest.approx(est.oob_score_) + assert est_ws.oob_scores_[-1] == pytest.approx(est_ws.oob_score_) -@pytest.mark.parametrize('Cls', GRADIENT_BOOSTING_ESTIMATORS) -def test_warm_start_sparse(Cls): +@pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) +@pytest.mark.parametrize( + "sparse_container", COO_CONTAINERS + CSC_CONTAINERS + CSR_CONTAINERS +) +def test_warm_start_sparse(Cls, sparse_container): # Test that all sparse matrix types are supported X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) - sparse_matrix_type = [csr_matrix, csc_matrix, coo_matrix] - est_dense = Cls(n_estimators=100, max_depth=1, subsample=0.5, - random_state=1, warm_start=True) + est_dense = Cls( + n_estimators=100, max_depth=1, subsample=0.5, random_state=1, warm_start=True + ) est_dense.fit(X, y) est_dense.predict(X) est_dense.set_params(n_estimators=200) est_dense.fit(X, y) y_pred_dense = est_dense.predict(X) - for sparse_constructor in sparse_matrix_type: - X_sparse = sparse_constructor(X) + X_sparse = sparse_container(X) - est_sparse = Cls(n_estimators=100, max_depth=1, subsample=0.5, - random_state=1, warm_start=True) - est_sparse.fit(X_sparse, y) - est_sparse.predict(X) - est_sparse.set_params(n_estimators=200) - est_sparse.fit(X_sparse, y) - y_pred_sparse = est_sparse.predict(X) - - assert_array_almost_equal(est_dense.oob_improvement_[:100], - est_sparse.oob_improvement_[:100]) - assert_array_almost_equal(y_pred_dense, y_pred_sparse) + est_sparse = Cls( + n_estimators=100, + max_depth=1, + subsample=0.5, + random_state=1, + warm_start=True, + ) + est_sparse.fit(X_sparse, y) + est_sparse.predict(X) + est_sparse.set_params(n_estimators=200) + est_sparse.fit(X_sparse, y) + y_pred_sparse = est_sparse.predict(X) + + assert_array_almost_equal( + est_dense.oob_improvement_[:100], est_sparse.oob_improvement_[:100] + ) + assert est_dense.oob_scores_[-1] == pytest.approx(est_dense.oob_score_) + assert_array_almost_equal(est_dense.oob_scores_[:100], est_sparse.oob_scores_[:100]) + assert est_sparse.oob_scores_[-1] == pytest.approx(est_sparse.oob_score_) + assert_array_almost_equal(y_pred_dense, y_pred_sparse) -@pytest.mark.parametrize('Cls', GRADIENT_BOOSTING_ESTIMATORS) -def test_warm_start_fortran(Cls): +@pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) +def test_warm_start_fortran(Cls, global_random_seed): # Test that feeding a X in Fortran-ordered is giving the same results as # in C-ordered - X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) - est_c = Cls(n_estimators=1, random_state=1, warm_start=True) - est_fortran = Cls(n_estimators=1, random_state=1, warm_start=True) + X, y = datasets.make_hastie_10_2(n_samples=100, random_state=global_random_seed) + est_c = Cls(n_estimators=1, random_state=global_random_seed, warm_start=True) + est_fortran = Cls(n_estimators=1, random_state=global_random_seed, warm_start=True) est_c.fit(X, y) est_c.set_params(n_estimators=11) @@ -865,18 +976,18 @@ def test_warm_start_fortran(Cls): est_fortran.set_params(n_estimators=11) est_fortran.fit(X_fortran, y) - assert_array_almost_equal(est_c.predict(X), est_fortran.predict(X)) + assert_allclose(est_c.predict(X), est_fortran.predict(X)) def early_stopping_monitor(i, est, locals): - """Returns True on the 10th iteration. """ + """Returns True on the 10th iteration.""" if i == 9: return True else: return False -@pytest.mark.parametrize('Cls', GRADIENT_BOOSTING_ESTIMATORS) +@pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) def test_monitor_early_stopping(Cls): # Test if monitor return value works. X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) @@ -887,6 +998,8 @@ def test_monitor_early_stopping(Cls): assert est.estimators_.shape[0] == 10 assert est.train_score_.shape[0] == 10 assert est.oob_improvement_.shape[0] == 10 + assert est.oob_scores_.shape[0] == 10 + assert est.oob_scores_[-1] == pytest.approx(est.oob_score_) # try refit est.set_params(n_estimators=30) @@ -894,14 +1007,20 @@ def test_monitor_early_stopping(Cls): assert est.n_estimators == 30 assert est.estimators_.shape[0] == 30 assert est.train_score_.shape[0] == 30 + assert est.oob_improvement_.shape[0] == 30 + assert est.oob_scores_.shape[0] == 30 + assert est.oob_scores_[-1] == pytest.approx(est.oob_score_) - est = Cls(n_estimators=20, max_depth=1, random_state=1, subsample=0.5, - warm_start=True) + est = Cls( + n_estimators=20, max_depth=1, random_state=1, subsample=0.5, warm_start=True + ) est.fit(X, y, monitor=early_stopping_monitor) assert est.n_estimators == 20 assert est.estimators_.shape[0] == 10 assert est.train_score_.shape[0] == 10 assert est.oob_improvement_.shape[0] == 10 + assert est.oob_scores_.shape[0] == 10 + assert est.oob_scores_[-1] == pytest.approx(est.oob_score_) # try refit est.set_params(n_estimators=30, warm_start=False) @@ -910,61 +1029,69 @@ def test_monitor_early_stopping(Cls): assert est.train_score_.shape[0] == 30 assert est.estimators_.shape[0] == 30 assert est.oob_improvement_.shape[0] == 30 + assert est.oob_scores_.shape[0] == 30 + assert est.oob_scores_[-1] == pytest.approx(est.oob_score_) def test_complete_classification(): # Test greedy trees with max_depth + 1 leafs. from sklearn.tree._tree import TREE_LEAF + X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) k = 4 - est = GradientBoostingClassifier(n_estimators=20, max_depth=None, - random_state=1, max_leaf_nodes=k + 1) + est = GradientBoostingClassifier( + n_estimators=20, max_depth=None, random_state=1, max_leaf_nodes=k + 1 + ) est.fit(X, y) tree = est.estimators_[0, 0].tree_ assert tree.max_depth == k - assert (tree.children_left[tree.children_left == TREE_LEAF].shape[0] == - k + 1) + assert tree.children_left[tree.children_left == TREE_LEAF].shape[0] == k + 1 def test_complete_regression(): # Test greedy trees with max_depth + 1 leafs. from sklearn.tree._tree import TREE_LEAF + k = 4 - est = GradientBoostingRegressor(n_estimators=20, max_depth=None, - random_state=1, max_leaf_nodes=k + 1) + est = GradientBoostingRegressor( + n_estimators=20, max_depth=None, random_state=1, max_leaf_nodes=k + 1 + ) est.fit(X_reg, y_reg) tree = est.estimators_[-1, 0].tree_ - assert (tree.children_left[tree.children_left == TREE_LEAF].shape[0] == - k + 1) + assert tree.children_left[tree.children_left == TREE_LEAF].shape[0] == k + 1 -def test_zero_estimator_reg(): - # Test if init='zero' works for regression. +def test_zero_estimator_reg(global_random_seed): + # Test if init='zero' works for regression by checking that it is better + # than a simple baseline. - est = GradientBoostingRegressor(n_estimators=20, max_depth=1, - random_state=1, init='zero') + baseline = DummyRegressor(strategy="mean").fit(X_reg, y_reg) + mse_baseline = mean_squared_error(baseline.predict(X_reg), y_reg) + est = GradientBoostingRegressor( + n_estimators=5, + max_depth=1, + random_state=global_random_seed, + init="zero", + learning_rate=0.5, + ) est.fit(X_reg, y_reg) y_pred = est.predict(X_reg) - mse = mean_squared_error(y_reg, y_pred) - assert_almost_equal(mse, 0.52, decimal=2) - - est = GradientBoostingRegressor(n_estimators=20, max_depth=1, - random_state=1, init='foobar') - with pytest.raises(ValueError): - est.fit(X_reg, y_reg) + mse_gbdt = mean_squared_error(y_reg, y_pred) + assert mse_gbdt < mse_baseline -def test_zero_estimator_clf(): +def test_zero_estimator_clf(global_random_seed): # Test if init='zero' works for classification. X = iris.data y = np.array(iris.target) - est = GradientBoostingClassifier(n_estimators=20, max_depth=1, - random_state=1, init='zero') + est = GradientBoostingClassifier( + n_estimators=20, max_depth=1, random_state=global_random_seed, init="zero" + ) est.fit(X, y) assert est.score(X, y) > 0.96 @@ -973,18 +1100,14 @@ def test_zero_estimator_clf(): mask = y != 0 y[mask] = 1 y[~mask] = 0 - est = GradientBoostingClassifier(n_estimators=20, max_depth=1, - random_state=1, init='zero') + est = GradientBoostingClassifier( + n_estimators=20, max_depth=1, random_state=global_random_seed, init="zero" + ) est.fit(X, y) assert est.score(X, y) > 0.96 - est = GradientBoostingClassifier(n_estimators=20, max_depth=1, - random_state=1, init='foobar') - with pytest.raises(ValueError): - est.fit(X, y) - -@pytest.mark.parametrize('GBEstimator', GRADIENT_BOOSTING_ESTIMATORS) +@pytest.mark.parametrize("GBEstimator", GRADIENT_BOOSTING_ESTIMATORS) def test_max_leaf_nodes_max_depth(GBEstimator): # Test precedence of max_leaf_nodes over max_depth. X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) @@ -1000,20 +1123,7 @@ def test_max_leaf_nodes_max_depth(GBEstimator): assert tree.max_depth == 1 -@pytest.mark.parametrize('GBEstimator', GRADIENT_BOOSTING_ESTIMATORS) -def test_min_impurity_split(GBEstimator): - # Test if min_impurity_split of base estimators is set - # Regression test for #8006 - X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) - - est = GBEstimator(min_impurity_split=0.1) - with pytest.warns(FutureWarning, match="min_impurity_decrease"): - est = est.fit(X, y) - for tree in est.estimators_.flat: - assert tree.min_impurity_split == 0.1 - - -@pytest.mark.parametrize('GBEstimator', GRADIENT_BOOSTING_ESTIMATORS) +@pytest.mark.parametrize("GBEstimator", GRADIENT_BOOSTING_ESTIMATORS) def test_min_impurity_decrease(GBEstimator): X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) @@ -1035,53 +1145,31 @@ def test_warm_start_wo_nestimators_change(): assert clf.estimators_.shape[0] == 10 -def test_probability_exponential(): - # Predict probabilities. - clf = GradientBoostingClassifier(loss='exponential', - n_estimators=100, random_state=1) - - with pytest.raises(ValueError): - clf.predict_proba(T) - - clf.fit(X, y) - assert_array_equal(clf.predict(T), true_result) - - # check if probabilities are in [0, 1]. - y_proba = clf.predict_proba(T) - assert np.all(y_proba >= 0.0) - assert np.all(y_proba <= 1.0) - score = clf.decision_function(T).ravel() - assert_array_almost_equal(y_proba[:, 1], expit(2 * score)) - - # derive predictions from probabilities - y_pred = clf.classes_.take(y_proba.argmax(axis=1), axis=0) - assert_array_equal(y_pred, true_result) - - -def test_non_uniform_weights_toy_edge_case_reg(): - X = [[1, 0], - [1, 0], - [1, 0], - [0, 1]] +@pytest.mark.parametrize( + ("loss", "value"), + [ + ("squared_error", 0.5), + ("absolute_error", 0.0), + ("huber", 0.5), + ("quantile", 0.5), + ], +) +def test_non_uniform_weights_toy_edge_case_reg(loss, value): + X = [[1, 0], [1, 0], [1, 0], [0, 1]] y = [0, 0, 1, 0] # ignore the first 2 training samples by setting their weight to 0 sample_weight = [0, 0, 1, 1] - for loss in ('huber', 'ls', 'lad', 'quantile'): - gb = GradientBoostingRegressor(learning_rate=1.0, n_estimators=2, - loss=loss) - gb.fit(X, y, sample_weight=sample_weight) - assert gb.predict([[1, 0]])[0] > 0.5 + gb = GradientBoostingRegressor(learning_rate=1.0, n_estimators=2, loss=loss) + gb.fit(X, y, sample_weight=sample_weight) + assert gb.predict([[1, 0]])[0] >= value def test_non_uniform_weights_toy_edge_case_clf(): - X = [[1, 0], - [1, 0], - [1, 0], - [0, 1]] + X = [[1, 0], [1, 0], [1, 0], [0, 1]] y = [0, 0, 1, 0] # ignore the first 2 training samples by setting their weight to 0 sample_weight = [0, 0, 1, 1] - for loss in ('deviance', 'exponential'): + for loss in ("log_loss", "exponential"): gb = GradientBoostingClassifier(n_estimators=5, loss=loss) gb.fit(X, y, sample_weight=sample_weight) assert_array_equal(gb.predict([[1, 0]]), [1]) @@ -1089,97 +1177,130 @@ def test_non_uniform_weights_toy_edge_case_clf(): @skip_if_32bit @pytest.mark.parametrize( - 'EstimatorClass', - (GradientBoostingClassifier, GradientBoostingRegressor) + "EstimatorClass", (GradientBoostingClassifier, GradientBoostingRegressor) +) +@pytest.mark.parametrize( + "sparse_container", COO_CONTAINERS + CSC_CONTAINERS + CSR_CONTAINERS ) -@pytest.mark.parametrize('sparse_matrix', (csr_matrix, csc_matrix, coo_matrix)) -def test_sparse_input(EstimatorClass, sparse_matrix): - y, X = datasets.make_multilabel_classification(random_state=0, - n_samples=50, - n_features=1, - n_classes=20) +def test_sparse_input(EstimatorClass, sparse_container): + y, X = datasets.make_multilabel_classification( + random_state=0, n_samples=50, n_features=1, n_classes=20 + ) y = y[:, 0] - X_sparse = sparse_matrix(X) + X_sparse = sparse_container(X) - dense = EstimatorClass(n_estimators=10, random_state=0, - max_depth=2, min_impurity_decrease=1e-7).fit(X, y) - sparse = EstimatorClass(n_estimators=10, random_state=0, - max_depth=2, - min_impurity_decrease=1e-7).fit(X_sparse, y) + dense = EstimatorClass( + n_estimators=10, random_state=0, max_depth=2, min_impurity_decrease=1e-7 + ).fit(X, y) + sparse = EstimatorClass( + n_estimators=10, random_state=0, max_depth=2, min_impurity_decrease=1e-7 + ).fit(X_sparse, y) assert_array_almost_equal(sparse.apply(X), dense.apply(X)) assert_array_almost_equal(sparse.predict(X), dense.predict(X)) - assert_array_almost_equal(sparse.feature_importances_, - dense.feature_importances_) + assert_array_almost_equal(sparse.feature_importances_, dense.feature_importances_) assert_array_almost_equal(sparse.predict(X_sparse), dense.predict(X)) assert_array_almost_equal(dense.predict(X_sparse), sparse.predict(X)) if issubclass(EstimatorClass, GradientBoostingClassifier): - assert_array_almost_equal(sparse.predict_proba(X), - dense.predict_proba(X)) - assert_array_almost_equal(sparse.predict_log_proba(X), - dense.predict_log_proba(X)) - - assert_array_almost_equal(sparse.decision_function(X_sparse), - sparse.decision_function(X)) - assert_array_almost_equal(dense.decision_function(X_sparse), - sparse.decision_function(X)) - for res_sparse, res in zip(sparse.staged_decision_function(X_sparse), - sparse.staged_decision_function(X)): + assert_array_almost_equal(sparse.predict_proba(X), dense.predict_proba(X)) + assert_array_almost_equal( + sparse.predict_log_proba(X), dense.predict_log_proba(X) + ) + + assert_array_almost_equal( + sparse.decision_function(X_sparse), sparse.decision_function(X) + ) + assert_array_almost_equal( + dense.decision_function(X_sparse), sparse.decision_function(X) + ) + for res_sparse, res in zip( + sparse.staged_decision_function(X_sparse), + sparse.staged_decision_function(X), + ): assert_array_almost_equal(res_sparse, res) -def test_gradient_boosting_early_stopping(): +@pytest.mark.parametrize( + "GradientBoostingEstimator", [GradientBoostingClassifier, GradientBoostingRegressor] +) +def test_gradient_boosting_early_stopping(GradientBoostingEstimator): + # Check if early stopping works as expected, that is empirically check that the + # number of trained estimators is increasing when the tolerance decreases. + + X, y = make_classification(n_samples=1000, random_state=0) + n_estimators = 1000 + + gb_large_tol = GradientBoostingEstimator( + n_estimators=n_estimators, + n_iter_no_change=10, + learning_rate=0.1, + max_depth=3, + random_state=42, + tol=1e-1, + ) + + gb_small_tol = GradientBoostingEstimator( + n_estimators=n_estimators, + n_iter_no_change=10, + learning_rate=0.1, + max_depth=3, + random_state=42, + tol=1e-3, + ) + + X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42) + gb_large_tol.fit(X_train, y_train) + gb_small_tol.fit(X_train, y_train) + + assert gb_large_tol.n_estimators_ < gb_small_tol.n_estimators_ < n_estimators + + assert gb_large_tol.score(X_test, y_test) > 0.7 + assert gb_small_tol.score(X_test, y_test) > 0.7 + + +def test_gradient_boosting_without_early_stopping(): + # When early stopping is not used, the number of trained estimators + # must be the one specified. X, y = make_classification(n_samples=1000, random_state=0) - gbc = GradientBoostingClassifier(n_estimators=1000, - n_iter_no_change=10, - learning_rate=0.1, max_depth=3, - random_state=42) - - gbr = GradientBoostingRegressor(n_estimators=1000, n_iter_no_change=10, - learning_rate=0.1, max_depth=3, - random_state=42) - - X_train, X_test, y_train, y_test = train_test_split(X, y, - random_state=42) - # Check if early_stopping works as expected - for est, tol, early_stop_n_estimators in ((gbc, 1e-1, 28), (gbr, 1e-1, 13), - (gbc, 1e-3, 70), - (gbr, 1e-3, 28)): - est.set_params(tol=tol) - est.fit(X_train, y_train) - assert est.n_estimators_ == early_stop_n_estimators - assert est.score(X_test, y_test) > 0.7 - - # Without early stopping - gbc = GradientBoostingClassifier(n_estimators=100, learning_rate=0.1, - max_depth=3, random_state=42) + gbc = GradientBoostingClassifier( + n_estimators=50, learning_rate=0.1, max_depth=3, random_state=42 + ) gbc.fit(X, y) - gbr = GradientBoostingRegressor(n_estimators=200, learning_rate=0.1, - max_depth=3, random_state=42) + gbr = GradientBoostingRegressor( + n_estimators=30, learning_rate=0.1, max_depth=3, random_state=42 + ) gbr.fit(X, y) - assert gbc.n_estimators_ == 100 - assert gbr.n_estimators_ == 200 + # The number of trained estimators must be the one specified. + assert gbc.n_estimators_ == 50 + assert gbr.n_estimators_ == 30 def test_gradient_boosting_validation_fraction(): X, y = make_classification(n_samples=1000, random_state=0) - gbc = GradientBoostingClassifier(n_estimators=100, - n_iter_no_change=10, - validation_fraction=0.1, - learning_rate=0.1, max_depth=3, - random_state=42) + gbc = GradientBoostingClassifier( + n_estimators=100, + n_iter_no_change=10, + validation_fraction=0.1, + learning_rate=0.1, + max_depth=3, + random_state=42, + ) gbc2 = clone(gbc).set_params(validation_fraction=0.3) gbc3 = clone(gbc).set_params(n_iter_no_change=20) - gbr = GradientBoostingRegressor(n_estimators=100, n_iter_no_change=10, - learning_rate=0.1, max_depth=3, - validation_fraction=0.1, - random_state=42) + gbr = GradientBoostingRegressor( + n_estimators=100, + n_iter_no_change=10, + learning_rate=0.1, + max_depth=3, + validation_fraction=0.1, + random_state=42, + ) gbr2 = clone(gbr).set_params(validation_fraction=0.3) gbr3 = clone(gbr).set_params(n_iter_no_change=20) @@ -1208,8 +1329,8 @@ def test_early_stopping_stratified(): gbc = GradientBoostingClassifier(n_iter_no_change=5) with pytest.raises( - ValueError, - match='The least populated class in y has only 1 member'): + ValueError, match="The least populated class in y has only 1 member" + ): gbc.fit(X, y) @@ -1219,18 +1340,23 @@ def _make_multiclass(): @pytest.mark.parametrize( "gb, dataset_maker, init_estimator", - [(GradientBoostingClassifier, make_classification, DummyClassifier), - (GradientBoostingClassifier, _make_multiclass, DummyClassifier), - (GradientBoostingRegressor, make_regression, DummyRegressor)], - ids=["binary classification", "multiclass classification", "regression"]) -def test_gradient_boosting_with_init(gb, dataset_maker, init_estimator): + [ + (GradientBoostingClassifier, make_classification, DummyClassifier), + (GradientBoostingClassifier, _make_multiclass, DummyClassifier), + (GradientBoostingRegressor, make_regression, DummyRegressor), + ], + ids=["binary classification", "multiclass classification", "regression"], +) +def test_gradient_boosting_with_init( + gb, dataset_maker, init_estimator, global_random_seed +): # Check that GradientBoostingRegressor works when init is a sklearn # estimator. # Check that an error is raised if trying to fit with sample weight but # initial estimator does not support sample weight X, y = dataset_maker() - sample_weight = np.random.RandomState(42).rand(100) + sample_weight = np.random.RandomState(global_random_seed).rand(100) # init supports sample weights init_est = init_estimator() @@ -1239,8 +1365,7 @@ def test_gradient_boosting_with_init(gb, dataset_maker, init_estimator): # init does not support sample weights init_est = NoSampleWeightWrapper(init_estimator()) gb(init=init_est).fit(X, y) # ok no sample weights - with pytest.raises(ValueError, - match="estimator.*does not support sample weights"): + with pytest.raises(ValueError, match="estimator.*does not support sample weights"): gb(init=init_est).fit(X, y, sample_weight=sample_weight) @@ -1253,38 +1378,27 @@ def test_gradient_boosting_with_init_pipeline(): gb.fit(X, y) # pipeline without sample_weight works fine with pytest.raises( - ValueError, - match='The initial estimator Pipeline does not support sample ' - 'weights'): + ValueError, + match="The initial estimator Pipeline does not support sample weights", + ): gb.fit(X, y, sample_weight=np.ones(X.shape[0])) # Passing sample_weight to a pipeline raises a ValueError. This test makes # sure we make the distinction between ValueError raised by a pipeline that - # was passed sample_weight, and a ValueError raised by a regular estimator - # whose input checking failed. - with pytest.raises( - ValueError, - match='nu <= 0 or nu > 1'): + # was passed sample_weight, and a InvalidParameterError raised by a regular + # estimator whose input checking failed. + invalid_nu = 1.5 + err_msg = ( + "The 'nu' parameter of NuSVR must be a float in the" + f" range (0.0, 1.0]. Got {invalid_nu} instead." + ) + with pytest.raises(InvalidParameterError, match=re.escape(err_msg)): # Note that NuSVR properly supports sample_weight - init = NuSVR(gamma='auto', nu=1.5) + init = NuSVR(gamma="auto", nu=invalid_nu) gb = GradientBoostingRegressor(init=init) gb.fit(X, y, sample_weight=np.ones(X.shape[0])) -@pytest.mark.parametrize('estimator, missing_method', [ - (GradientBoostingClassifier(init=LinearSVC()), 'predict_proba'), - (GradientBoostingRegressor(init=OneHotEncoder()), 'predict') -]) -def test_gradient_boosting_init_wrong_methods(estimator, missing_method): - # Make sure error is raised if init estimators don't have the required - # methods (fit, predict, predict_proba) - - message = ("The init parameter must be a valid estimator and support " - "both fit and " + missing_method) - with pytest.raises(ValueError, match=message): - estimator.fit(X, y) - - def test_early_stopping_n_classes(): # when doing early stopping (_, , y_train, _ = train_test_split(X, y)) # there might be classes in y that are missing in y_train. As the init @@ -1293,16 +1407,18 @@ def test_early_stopping_n_classes(): X = [[1]] * 10 y = [0, 0] + [1] * 8 # only 2 negative class over 10 samples - gb = GradientBoostingClassifier(n_iter_no_change=5, random_state=0, - validation_fraction=8) + gb = GradientBoostingClassifier( + n_iter_no_change=5, random_state=0, validation_fraction=0.8 + ) with pytest.raises( - ValueError, - match='The training data after the early stopping split'): + ValueError, match="The training data after the early stopping split" + ): gb.fit(X, y) # No error if we let training data be big enough - gb = GradientBoostingClassifier(n_iter_no_change=5, random_state=0, - validation_fraction=4) + gb = GradientBoostingClassifier( + n_iter_no_change=5, random_state=0, validation_fraction=0.4 + ) def test_gbr_degenerate_feature_importances(): @@ -1310,62 +1426,286 @@ def test_gbr_degenerate_feature_importances(): X = np.zeros((10, 10)) y = np.ones((10,)) gbr = GradientBoostingRegressor().fit(X, y) - assert_array_equal(gbr.feature_importances_, - np.zeros(10, dtype=np.float64)) + assert_array_equal(gbr.feature_importances_, np.zeros(10, dtype=np.float64)) -# TODO: Remove in 1.1 when `n_classes_` is deprecated -def test_gbr_deprecated_attr(): - # check that accessing n_classes_ in GradientBoostingRegressor raises - # a deprecation warning - X = np.zeros((10, 10)) - y = np.ones((10,)) - gbr = GradientBoostingRegressor().fit(X, y) - msg = "Attribute n_classes_ was deprecated" - with pytest.warns(FutureWarning, match=msg): - gbr.n_classes_ - - -# TODO: Remove in 1.1 when `n_classes_` is deprecated -@pytest.mark.filterwarnings("ignore:Attribute n_classes_ was deprecated") -def test_attr_error_raised_if_not_fitted(): - # check that accessing n_classes_ in not fitted GradientBoostingRegressor - # raises an AttributeError - gbr = GradientBoostingRegressor() - # test raise AttributeError if not fitted - msg = ( - f"{GradientBoostingRegressor.__name__} object has no n_classes_ " - f"attribute." +def test_huber_vs_mean_and_median(): + """Check that huber lies between absolute and squared error.""" + n_rep = 100 + n_samples = 10 + y = np.tile(np.arange(n_samples), n_rep) + x1 = np.minimum(y, n_samples / 2) + x2 = np.minimum(-y, -n_samples / 2) + X = np.c_[x1, x2] + + rng = np.random.RandomState(42) + # We want an asymmetric distribution. + y = y + rng.exponential(scale=1, size=y.shape) + + gbt_absolute_error = GradientBoostingRegressor(loss="absolute_error").fit(X, y) + gbt_huber = GradientBoostingRegressor(loss="huber").fit(X, y) + gbt_squared_error = GradientBoostingRegressor().fit(X, y) + + gbt_huber_predictions = gbt_huber.predict(X) + assert np.all(gbt_absolute_error.predict(X) <= gbt_huber_predictions) + assert np.all(gbt_huber_predictions <= gbt_squared_error.predict(X)) + + +def test_safe_divide(): + """Test that _safe_divide handles division by zero.""" + with warnings.catch_warnings(): + warnings.simplefilter("error") + assert _safe_divide(np.float64(1e300), 0) == 0 + assert _safe_divide(np.float64(0.0), np.float64(0.0)) == 0 + with pytest.warns(RuntimeWarning, match="overflow"): + # np.finfo(float).max = 1.7976931348623157e+308 + _safe_divide(np.float64(1e300), 1e-10) + + +def test_squared_error_exact_backward_compat(): + """Test squared error GBT backward compat on a simple dataset. + + The results to compare against are taken from scikit-learn v1.2.0. + """ + n_samples = 10 + y = np.arange(n_samples) + x1 = np.minimum(y, n_samples / 2) + x2 = np.minimum(-y, -n_samples / 2) + X = np.c_[x1, x2] + gbt = GradientBoostingRegressor(loss="squared_error", n_estimators=100).fit(X, y) + + pred_result = np.array( + [ + 1.39245726e-04, + 1.00010468e00, + 2.00007043e00, + 3.00004051e00, + 4.00000802e00, + 4.99998972e00, + 5.99996312e00, + 6.99993395e00, + 7.99989372e00, + 8.99985660e00, + ] + ) + assert_allclose(gbt.predict(X), pred_result, rtol=1e-8) + + train_score = np.array( + [ + 4.87246390e-08, + 3.95590036e-08, + 3.21267865e-08, + 2.60970300e-08, + 2.11820178e-08, + 1.71995782e-08, + 1.39695549e-08, + 1.13391770e-08, + 9.19931587e-09, + 7.47000575e-09, + ] ) - with pytest.raises(AttributeError, match=msg): - gbr.n_classes_ + assert_allclose(gbt.train_score_[-10:], train_score, rtol=1e-8) + # Same but with sample_weights + sample_weights = np.tile([1, 10], n_samples // 2) + gbt = GradientBoostingRegressor(loss="squared_error", n_estimators=100).fit( + X, y, sample_weight=sample_weights + ) -# TODO: Update in 1.1 to check for the error raised -@pytest.mark.parametrize('estimator', [ - GradientBoostingClassifier(criterion='mae'), - GradientBoostingRegressor(criterion='mae') -]) -def test_criterion_mae_deprecation(estimator): - # checks whether a deprecation warning is issues when criterion='mae' - # is used. - msg = ("criterion='mae' was deprecated in version 0.24 and " - "will be removed in version 1.1") - with pytest.warns(FutureWarning, match=msg): - estimator.fit(X, y) + pred_result = np.array( + [ + 1.52391462e-04, + 1.00011168e00, + 2.00007724e00, + 3.00004638e00, + 4.00001302e00, + 4.99999873e00, + 5.99997093e00, + 6.99994329e00, + 7.99991290e00, + 8.99988727e00, + ] + ) + assert_allclose(gbt.predict(X), pred_result, rtol=1e-6, atol=1e-5) + + train_score = np.array( + [ + 4.12445296e-08, + 3.34418322e-08, + 2.71151383e-08, + 2.19782469e-08, + 1.78173649e-08, + 1.44461976e-08, + 1.17120123e-08, + 9.49485678e-09, + 7.69772505e-09, + 6.24155316e-09, + ] + ) + assert_allclose(gbt.train_score_[-10:], train_score, rtol=1e-3, atol=1e-11) -# FIXME: remove in 1.2 -@pytest.mark.parametrize( - "Estimator", - [GradientBoostingClassifier, GradientBoostingRegressor] -) -def test_n_features_deprecation(Estimator): - # Check that we raise the proper deprecation warning if accessing - # `n_features_`. - X = np.array([[1, 2], [3, 4]]) - y = np.array([1, 0]) - est = Estimator().fit(X, y) - - with pytest.warns(FutureWarning, match="n_features_ was deprecated"): - est.n_features_ +@skip_if_32bit +def test_huber_exact_backward_compat(): + """Test huber GBT backward compat on a simple dataset. + + The results to compare against are taken from scikit-learn v1.2.0. + """ + n_samples = 10 + y = np.arange(n_samples) + x1 = np.minimum(y, n_samples / 2) + x2 = np.minimum(-y, -n_samples / 2) + X = np.c_[x1, x2] + gbt = GradientBoostingRegressor(loss="huber", n_estimators=100, alpha=0.8).fit(X, y) + + assert_allclose(gbt._loss.closs.delta, 0.0001655688041282133) + + pred_result = np.array( + [ + 1.48120765e-04, + 9.99949174e-01, + 2.00116957e00, + 2.99986716e00, + 4.00012064e00, + 5.00002462e00, + 5.99998898e00, + 6.99692549e00, + 8.00006356e00, + 8.99985099e00, + ] + ) + assert_allclose(gbt.predict(X), pred_result, rtol=1e-8) + + train_score = np.array( + [ + 2.59484709e-07, + 2.19165900e-07, + 1.89644782e-07, + 1.64556454e-07, + 1.38705110e-07, + 1.20373736e-07, + 1.04746082e-07, + 9.13835687e-08, + 8.20245756e-08, + 7.17122188e-08, + ] + ) + assert_allclose(gbt.train_score_[-10:], train_score, rtol=1e-8) + + +def test_binomial_error_exact_backward_compat(): + """Test binary log_loss GBT backward compat on a simple dataset. + + The results to compare against are taken from scikit-learn v1.2.0. + """ + n_samples = 10 + y = np.arange(n_samples) % 2 + x1 = np.minimum(y, n_samples / 2) + x2 = np.minimum(-y, -n_samples / 2) + X = np.c_[x1, x2] + gbt = GradientBoostingClassifier(loss="log_loss", n_estimators=100).fit(X, y) + + pred_result = np.array( + [ + [9.99978098e-01, 2.19017313e-05], + [2.19017313e-05, 9.99978098e-01], + [9.99978098e-01, 2.19017313e-05], + [2.19017313e-05, 9.99978098e-01], + [9.99978098e-01, 2.19017313e-05], + [2.19017313e-05, 9.99978098e-01], + [9.99978098e-01, 2.19017313e-05], + [2.19017313e-05, 9.99978098e-01], + [9.99978098e-01, 2.19017313e-05], + [2.19017313e-05, 9.99978098e-01], + ] + ) + assert_allclose(gbt.predict_proba(X), pred_result, rtol=1e-8) + + train_score = np.array( + [ + 1.07742210e-04, + 9.74889078e-05, + 8.82113863e-05, + 7.98167784e-05, + 7.22210566e-05, + 6.53481907e-05, + 5.91293869e-05, + 5.35023988e-05, + 4.84109045e-05, + 4.38039423e-05, + ] + ) + assert_allclose(gbt.train_score_[-10:], train_score, rtol=1e-8) + + +def test_multinomial_error_exact_backward_compat(): + """Test multiclass log_loss GBT backward compat on a simple dataset. + + The results to compare against are taken from scikit-learn v1.2.0. + """ + n_samples = 10 + y = np.arange(n_samples) % 4 + x1 = np.minimum(y, n_samples / 2) + x2 = np.minimum(-y, -n_samples / 2) + X = np.c_[x1, x2] + gbt = GradientBoostingClassifier(loss="log_loss", n_estimators=100).fit(X, y) + + pred_result = np.array( + [ + [9.99999727e-01, 1.11956255e-07, 8.04921671e-08, 8.04921668e-08], + [1.11956254e-07, 9.99999727e-01, 8.04921671e-08, 8.04921668e-08], + [1.19417637e-07, 1.19417637e-07, 9.99999675e-01, 8.60526098e-08], + [1.19417637e-07, 1.19417637e-07, 8.60526088e-08, 9.99999675e-01], + [9.99999727e-01, 1.11956255e-07, 8.04921671e-08, 8.04921668e-08], + [1.11956254e-07, 9.99999727e-01, 8.04921671e-08, 8.04921668e-08], + [1.19417637e-07, 1.19417637e-07, 9.99999675e-01, 8.60526098e-08], + [1.19417637e-07, 1.19417637e-07, 8.60526088e-08, 9.99999675e-01], + [9.99999727e-01, 1.11956255e-07, 8.04921671e-08, 8.04921668e-08], + [1.11956254e-07, 9.99999727e-01, 8.04921671e-08, 8.04921668e-08], + ] + ) + assert_allclose(gbt.predict_proba(X), pred_result, rtol=1e-8) + + train_score = np.array( + [ + 1.13300150e-06, + 9.75183397e-07, + 8.39348103e-07, + 7.22433588e-07, + 6.21804338e-07, + 5.35191943e-07, + 4.60643966e-07, + 3.96479930e-07, + 3.41253434e-07, + 2.93719550e-07, + ] + ) + assert_allclose(gbt.train_score_[-10:], train_score, rtol=1e-8) + + +def test_gb_denominator_zero(global_random_seed): + """Test _update_terminal_regions denominator is not zero. + + For instance for log loss based binary classification, the line search step might + become nan/inf as denominator = hessian = prob * (1 - prob) and prob = 0 or 1 can + happen. + Here, we create a situation were this happens (at least with roughly 80%) based + on the random seed. + """ + X, y = datasets.make_hastie_10_2(n_samples=100, random_state=20) + + params = { + "learning_rate": 1.0, + "subsample": 0.5, + "n_estimators": 100, + "max_leaf_nodes": 4, + "max_depth": None, + "random_state": global_random_seed, + "min_samples_leaf": 2, + } + + clf = GradientBoostingClassifier(**params) + # _safe_devide would raise a RuntimeWarning + with warnings.catch_warnings(): + warnings.simplefilter("error") + clf.fit(X, y) diff --git a/sklearn/ensemble/tests/test_gradient_boosting_loss_functions.py b/sklearn/ensemble/tests/test_gradient_boosting_loss_functions.py deleted file mode 100644 index 4d7ea9bfe9bb3..0000000000000 --- a/sklearn/ensemble/tests/test_gradient_boosting_loss_functions.py +++ /dev/null @@ -1,323 +0,0 @@ -""" -Testing for the gradient boosting loss functions and initial estimators. -""" -from itertools import product -import numpy as np -from numpy.testing import assert_allclose -import pytest -from pytest import approx - -from sklearn.utils import check_random_state -from sklearn.metrics import mean_pinball_loss -from sklearn.ensemble._gb_losses import RegressionLossFunction -from sklearn.ensemble._gb_losses import LeastSquaresError -from sklearn.ensemble._gb_losses import LeastAbsoluteError -from sklearn.ensemble._gb_losses import HuberLossFunction -from sklearn.ensemble._gb_losses import QuantileLossFunction -from sklearn.ensemble._gb_losses import BinomialDeviance -from sklearn.ensemble._gb_losses import MultinomialDeviance -from sklearn.ensemble._gb_losses import ExponentialLoss -from sklearn.ensemble._gb_losses import LOSS_FUNCTIONS - - -def test_binomial_deviance(): - # Check binomial deviance loss. - # Check against alternative definitions in ESLII. - bd = BinomialDeviance(2) - - # pred has the same BD for y in {0, 1} - assert (bd(np.array([0.]), np.array([0.])) == - bd(np.array([1.]), np.array([0.]))) - - assert bd(np.array([1., 1, 1]), np.array([100., 100, 100])) == approx(0) - assert bd(np.array([1., 0, 0]), np.array([100., -100, -100])) == approx(0) - - # check if same results as alternative definition of deviance, from ESLII - # Eq. (10.18): -loglike = log(1 + exp(-2*z*f)) - # Note: - # - We use y = {0, 1}, ESL (10.18) uses z in {-1, 1}, hence y=2*y-1 - # - ESL 2*f = pred_raw, hence the factor 2 of ESL disappears. - # - Deviance = -2*loglike + .., hence a factor of 2 in front. - def alt_dev(y, raw_pred): - z = 2 * y - 1 - return 2 * np.mean(np.log(1 + np.exp(-z * raw_pred))) - - test_data = product( - (np.array([0., 0, 0]), np.array([1., 1, 1])), - (np.array([-5., -5, -5]), np.array([3., 3, 3]))) - - for datum in test_data: - assert bd(*datum) == approx(alt_dev(*datum)) - - # check the negative gradient against altenative formula from ESLII - # Note: negative_gradient is half the negative gradient. - def alt_ng(y, raw_pred): - z = 2 * y - 1 - return z / (1 + np.exp(z * raw_pred)) - - for datum in test_data: - assert bd.negative_gradient(*datum) == approx(alt_ng(*datum)) - - -def test_sample_weight_smoke(): - rng = check_random_state(13) - y = rng.rand(100) - pred = rng.rand(100) - - # least squares - loss = LeastSquaresError() - loss_wo_sw = loss(y, pred) - loss_w_sw = loss(y, pred, np.ones(pred.shape[0], dtype=np.float32)) - assert loss_wo_sw == approx(loss_w_sw) - - -def test_sample_weight_init_estimators(): - # Smoke test for init estimators with sample weights. - rng = check_random_state(13) - X = rng.rand(100, 2) - sample_weight = np.ones(100) - reg_y = rng.rand(100) - - clf_y = rng.randint(0, 2, size=100) - - for Loss in LOSS_FUNCTIONS.values(): - if Loss is None: - continue - if issubclass(Loss, RegressionLossFunction): - y = reg_y - loss = Loss() - else: - k = 2 - y = clf_y - if Loss.is_multi_class: - # skip multiclass - continue - loss = Loss(k) - - init_est = loss.init_estimator() - init_est.fit(X, y) - out = loss.get_init_raw_predictions(X, init_est) - assert out.shape == (y.shape[0], 1) - - sw_init_est = loss.init_estimator() - sw_init_est.fit(X, y, sample_weight=sample_weight) - sw_out = loss.get_init_raw_predictions(X, sw_init_est) - assert sw_out.shape == (y.shape[0], 1) - - # check if predictions match - assert_allclose(out, sw_out, rtol=1e-2) - - -def test_quantile_loss_function(): - # Non regression test for the QuantileLossFunction object - # There was a sign problem when evaluating the function - # for negative values of 'ytrue - ypred' - x = np.asarray([-1.0, 0.0, 1.0]) - y_found = QuantileLossFunction(0.9)(x, np.zeros_like(x)) - y_expected = np.asarray([0.1, 0.0, 0.9]).mean() - np.testing.assert_allclose(y_found, y_expected) - y_found_p = mean_pinball_loss(x, np.zeros_like(x), alpha=0.9) - np.testing.assert_allclose(y_found, y_found_p) - - -def test_sample_weight_deviance(): - # Test if deviance supports sample weights. - rng = check_random_state(13) - sample_weight = np.ones(100) - reg_y = rng.rand(100) - clf_y = rng.randint(0, 2, size=100) - mclf_y = rng.randint(0, 3, size=100) - - for Loss in LOSS_FUNCTIONS.values(): - if Loss is None: - continue - if issubclass(Loss, RegressionLossFunction): - y = reg_y - p = reg_y - loss = Loss() - else: - k = 2 - y = clf_y - p = clf_y - if Loss.is_multi_class: - k = 3 - y = mclf_y - # one-hot encoding - p = np.zeros((y.shape[0], k), dtype=np.float64) - for i in range(k): - p[:, i] = y == i - loss = Loss(k) - - deviance_w_w = loss(y, p, sample_weight) - deviance_wo_w = loss(y, p) - assert deviance_wo_w == deviance_w_w - - -@pytest.mark.parametrize( - 'n_classes, n_samples', [(3, 100), (5, 57), (7, 13)] -) -def test_multinomial_deviance(n_classes, n_samples): - # Check multinomial deviance with and without sample weights. - rng = np.random.RandomState(13) - sample_weight = np.ones(n_samples) - y_true = rng.randint(0, n_classes, size=n_samples) - y_pred = np.zeros((n_samples, n_classes), dtype=np.float64) - for klass in range(y_pred.shape[1]): - y_pred[:, klass] = y_true == klass - - loss = MultinomialDeviance(n_classes) - loss_wo_sw = loss(y_true, y_pred) - assert loss_wo_sw > 0 - loss_w_sw = loss(y_true, y_pred, sample_weight=sample_weight) - assert loss_wo_sw == approx(loss_w_sw) - - # Multinomial deviance uses weighted average loss rather than - # weighted sum loss, so we make sure that the value remains the same - # when we device the weight by 2. - loss_w_sw = loss(y_true, y_pred, sample_weight=0.5 * sample_weight) - assert loss_wo_sw == approx(loss_w_sw) - - -def test_mdl_computation_weighted(): - raw_predictions = np.array([[1., -1., -.1], [-2., 1., 2.]]) - y_true = np.array([0, 1]) - weights = np.array([1, 3]) - expected_loss = 1.0909323 - # MultinomialDeviance loss computation with weights. - loss = MultinomialDeviance(3) - assert loss(y_true, raw_predictions, weights) == approx(expected_loss) - - -@pytest.mark.parametrize('n', [0, 1, 2]) -def test_mdl_exception(n): - # Check that MultinomialDeviance throws an exception when n_classes <= 2 - err_msg = 'MultinomialDeviance requires more than 2 classes.' - with pytest.raises(ValueError, match=err_msg): - MultinomialDeviance(n) - - -def test_init_raw_predictions_shapes(): - # Make sure get_init_raw_predictions returns float64 arrays with shape - # (n_samples, K) where K is 1 for binary classification and regression, and - # K = n_classes for multiclass classification - rng = np.random.RandomState(0) - - n_samples = 100 - X = rng.normal(size=(n_samples, 5)) - y = rng.normal(size=n_samples) - for loss in (LeastSquaresError(), - LeastAbsoluteError(), - QuantileLossFunction(), - HuberLossFunction()): - init_estimator = loss.init_estimator().fit(X, y) - raw_predictions = loss.get_init_raw_predictions(y, init_estimator) - assert raw_predictions.shape == (n_samples, 1) - assert raw_predictions.dtype == np.float64 - - y = rng.randint(0, 2, size=n_samples) - for loss in (BinomialDeviance(n_classes=2), - ExponentialLoss(n_classes=2)): - init_estimator = loss.init_estimator().fit(X, y) - raw_predictions = loss.get_init_raw_predictions(y, init_estimator) - assert raw_predictions.shape == (n_samples, 1) - assert raw_predictions.dtype == np.float64 - - for n_classes in range(3, 5): - y = rng.randint(0, n_classes, size=n_samples) - loss = MultinomialDeviance(n_classes=n_classes) - init_estimator = loss.init_estimator().fit(X, y) - raw_predictions = loss.get_init_raw_predictions(y, init_estimator) - assert raw_predictions.shape == (n_samples, n_classes) - assert raw_predictions.dtype == np.float64 - - -def test_init_raw_predictions_values(): - # Make sure the get_init_raw_predictions() returns the expected values for - # each loss. - rng = np.random.RandomState(0) - - n_samples = 100 - X = rng.normal(size=(n_samples, 5)) - y = rng.normal(size=n_samples) - - # Least squares loss - loss = LeastSquaresError() - init_estimator = loss.init_estimator().fit(X, y) - raw_predictions = loss.get_init_raw_predictions(y, init_estimator) - # Make sure baseline prediction is the mean of all targets - assert_allclose(raw_predictions, y.mean()) - - # Least absolute and huber loss - for Loss in (LeastAbsoluteError, HuberLossFunction): - loss = Loss() - init_estimator = loss.init_estimator().fit(X, y) - raw_predictions = loss.get_init_raw_predictions(y, init_estimator) - # Make sure baseline prediction is the median of all targets - assert_allclose(raw_predictions, np.median(y)) - - # Quantile loss - for alpha in (.1, .5, .9): - loss = QuantileLossFunction(alpha=alpha) - init_estimator = loss.init_estimator().fit(X, y) - raw_predictions = loss.get_init_raw_predictions(y, init_estimator) - # Make sure baseline prediction is the alpha-quantile of all targets - assert_allclose(raw_predictions, np.percentile(y, alpha * 100)) - - y = rng.randint(0, 2, size=n_samples) - - # Binomial deviance - loss = BinomialDeviance(n_classes=2) - init_estimator = loss.init_estimator().fit(X, y) - # Make sure baseline prediction is equal to link_function(p), where p - # is the proba of the positive class. We want predict_proba() to return p, - # and by definition - # p = inverse_link_function(raw_prediction) = sigmoid(raw_prediction) - # So we want raw_prediction = link_function(p) = log(p / (1 - p)) - raw_predictions = loss.get_init_raw_predictions(y, init_estimator) - p = y.mean() - assert_allclose(raw_predictions, np.log(p / (1 - p))) - - # Exponential loss - loss = ExponentialLoss(n_classes=2) - init_estimator = loss.init_estimator().fit(X, y) - raw_predictions = loss.get_init_raw_predictions(y, init_estimator) - p = y.mean() - assert_allclose(raw_predictions, .5 * np.log(p / (1 - p))) - - # Multinomial deviance loss - for n_classes in range(3, 5): - y = rng.randint(0, n_classes, size=n_samples) - loss = MultinomialDeviance(n_classes=n_classes) - init_estimator = loss.init_estimator().fit(X, y) - raw_predictions = loss.get_init_raw_predictions(y, init_estimator) - for k in range(n_classes): - p = (y == k).mean() - assert_allclose(raw_predictions[:, k], np.log(p)) - - -@pytest.mark.parametrize('seed', range(5)) -@pytest.mark.parametrize('alpha', [0.4, 0.5, 0.6]) -def test_lad_equals_quantiles(seed, alpha): - # Make sure quantile loss with alpha = .5 is equivalent to LAD - lad = LeastAbsoluteError() - ql = QuantileLossFunction(alpha=alpha) - - n_samples = 50 - rng = np.random.RandomState(seed) - raw_predictions = rng.normal(size=(n_samples)) - y_true = rng.normal(size=(n_samples)) - - lad_loss = lad(y_true, raw_predictions) - ql_loss = ql(y_true, raw_predictions) - if alpha == 0.5: - assert lad_loss == approx(2 * ql_loss) - - weights = np.linspace(0, 1, n_samples) ** 2 - lad_weighted_loss = lad(y_true, raw_predictions, sample_weight=weights) - ql_weighted_loss = ql(y_true, raw_predictions, sample_weight=weights) - if alpha == 0.5: - assert lad_weighted_loss == approx(2 * ql_weighted_loss) - pbl_weighted_loss = mean_pinball_loss(y_true, raw_predictions, - sample_weight=weights, - alpha=alpha) - assert pbl_weighted_loss == approx(ql_weighted_loss) diff --git a/sklearn/ensemble/tests/test_iforest.py b/sklearn/ensemble/tests/test_iforest.py index 0b3a521346b30..19e34bbf51808 100644 --- a/sklearn/ensemble/tests/test_iforest.py +++ b/sklearn/ensemble/tests/test_iforest.py @@ -2,124 +2,93 @@ Testing for Isolation Forest algorithm (sklearn.ensemble.iforest). """ -# Authors: Nicolas Goix -# Alexandre Gramfort -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -import pytest +import warnings +from unittest.mock import Mock, patch import numpy as np +import pytest +from joblib import parallel_backend -from sklearn.utils._testing import assert_array_equal -from sklearn.utils._testing import assert_array_almost_equal -from sklearn.utils._testing import ignore_warnings -from sklearn.utils._testing import assert_allclose - -from sklearn.model_selection import ParameterGrid +from sklearn.datasets import load_diabetes, load_iris, make_classification from sklearn.ensemble import IsolationForest from sklearn.ensemble._iforest import _average_path_length -from sklearn.model_selection import train_test_split -from sklearn.datasets import load_diabetes, load_iris -from sklearn.utils import check_random_state from sklearn.metrics import roc_auc_score +from sklearn.model_selection import ParameterGrid, train_test_split +from sklearn.utils import check_random_state +from sklearn.utils._testing import ( + assert_allclose, + assert_array_almost_equal, + assert_array_equal, + ignore_warnings, +) +from sklearn.utils.fixes import CSC_CONTAINERS, CSR_CONTAINERS -from scipy.sparse import csc_matrix, csr_matrix -from unittest.mock import Mock, patch - -rng = check_random_state(0) - -# load the iris dataset -# and randomly permute it +# load iris & diabetes dataset iris = load_iris() -perm = rng.permutation(iris.target.size) -iris.data = iris.data[perm] -iris.target = iris.target[perm] - -# also load the diabetes dataset -# and randomly permute it diabetes = load_diabetes() -perm = rng.permutation(diabetes.target.size) -diabetes.data = diabetes.data[perm] -diabetes.target = diabetes.target[perm] -def test_iforest(): +def test_iforest(global_random_seed): """Check Isolation Forest for various parameter settings.""" X_train = np.array([[0, 1], [1, 2]]) X_test = np.array([[2, 1], [1, 1]]) - grid = ParameterGrid({"n_estimators": [3], - "max_samples": [0.5, 1.0, 3], - "bootstrap": [True, False]}) + grid = ParameterGrid( + {"n_estimators": [3], "max_samples": [0.5, 1.0, 3], "bootstrap": [True, False]} + ) with ignore_warnings(): for params in grid: - IsolationForest(random_state=rng, - **params).fit(X_train).predict(X_test) + IsolationForest(random_state=global_random_seed, **params).fit( + X_train + ).predict(X_test) -def test_iforest_sparse(): +@pytest.mark.parametrize("sparse_container", CSC_CONTAINERS + CSR_CONTAINERS) +def test_iforest_sparse(global_random_seed, sparse_container): """Check IForest for various parameter settings on sparse input.""" - rng = check_random_state(0) - X_train, X_test, y_train, y_test = train_test_split(diabetes.data[:50], - diabetes.target[:50], - random_state=rng) - grid = ParameterGrid({"max_samples": [0.5, 1.0], - "bootstrap": [True, False]}) + rng = check_random_state(global_random_seed) + X_train, X_test = train_test_split(diabetes.data[:50], random_state=rng) + grid = ParameterGrid({"max_samples": [0.5, 1.0], "bootstrap": [True, False]}) - for sparse_format in [csc_matrix, csr_matrix]: - X_train_sparse = sparse_format(X_train) - X_test_sparse = sparse_format(X_test) + X_train_sparse = sparse_container(X_train) + X_test_sparse = sparse_container(X_test) - for params in grid: - # Trained on sparse format - sparse_classifier = IsolationForest( - n_estimators=10, random_state=1, **params).fit(X_train_sparse) - sparse_results = sparse_classifier.predict(X_test_sparse) + for params in grid: + # Trained on sparse format + sparse_classifier = IsolationForest( + n_estimators=10, random_state=global_random_seed, **params + ).fit(X_train_sparse) + sparse_results = sparse_classifier.predict(X_test_sparse) - # Trained on dense format - dense_classifier = IsolationForest( - n_estimators=10, random_state=1, **params).fit(X_train) - dense_results = dense_classifier.predict(X_test) + # Trained on dense format + dense_classifier = IsolationForest( + n_estimators=10, random_state=global_random_seed, **params + ).fit(X_train) + dense_results = dense_classifier.predict(X_test) - assert_array_equal(sparse_results, dense_results) + assert_array_equal(sparse_results, dense_results) def test_iforest_error(): """Test that it gives proper exception on deficient input.""" X = iris.data - # Test max_samples - with pytest.raises(ValueError): - IsolationForest(max_samples=-1).fit(X) - with pytest.raises(ValueError): - IsolationForest(max_samples=0.0).fit(X) - with pytest.raises(ValueError): - IsolationForest(max_samples=2.0).fit(X) # The dataset has less than 256 samples, explicitly setting # max_samples > n_samples should result in a warning. If not set # explicitly there should be no warning warn_msg = "max_samples will be set to n_samples for estimation" with pytest.warns(UserWarning, match=warn_msg): IsolationForest(max_samples=1000).fit(X) - # note that assert_no_warnings does not apply since it enables a - # PendingDeprecationWarning triggered by scipy.sparse's use of - # np.matrix. See issue #11251. - with pytest.warns(None) as record: - IsolationForest(max_samples='auto').fit(X) - user_warnings = [each for each in record - if issubclass(each.category, UserWarning)] - assert len(user_warnings) == 0 - with pytest.warns(None) as record: + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + IsolationForest(max_samples="auto").fit(X) + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) IsolationForest(max_samples=np.int64(2)).fit(X) - user_warnings = [each for each in record - if issubclass(each.category, UserWarning)] - assert len(user_warnings) == 0 - - with pytest.raises(ValueError): - IsolationForest(max_samples='foobar').fit(X) - with pytest.raises(ValueError): - IsolationForest(max_samples=1.5).fit(X) # test X_test n_features match X_train one: with pytest.raises(ValueError): @@ -146,19 +115,16 @@ def test_max_samples_attribute(): assert clf.max_samples_ == X.shape[0] clf = IsolationForest(max_samples=0.4).fit(X) - assert clf.max_samples_ == 0.4*X.shape[0] + assert clf.max_samples_ == 0.4 * X.shape[0] -def test_iforest_parallel_regression(): +def test_iforest_parallel_regression(global_random_seed): """Check parallel regression.""" - rng = check_random_state(0) + rng = check_random_state(global_random_seed) - X_train, X_test, y_train, y_test = train_test_split(diabetes.data, - diabetes.target, - random_state=rng) + X_train, X_test = train_test_split(diabetes.data, random_state=rng) - ensemble = IsolationForest(n_jobs=3, - random_state=0).fit(X_train) + ensemble = IsolationForest(n_jobs=3, random_state=global_random_seed).fit(X_train) ensemble.set_params(n_jobs=1) y1 = ensemble.predict(X_test) @@ -166,44 +132,43 @@ def test_iforest_parallel_regression(): y2 = ensemble.predict(X_test) assert_array_almost_equal(y1, y2) - ensemble = IsolationForest(n_jobs=1, - random_state=0).fit(X_train) + ensemble = IsolationForest(n_jobs=1, random_state=global_random_seed).fit(X_train) y3 = ensemble.predict(X_test) assert_array_almost_equal(y1, y3) -def test_iforest_performance(): +def test_iforest_performance(global_random_seed): """Test Isolation Forest performs well""" # Generate train/test data - rng = check_random_state(2) - X = 0.3 * rng.randn(120, 2) - X_train = np.r_[X + 2, X - 2] - X_train = X[:100] + rng = check_random_state(global_random_seed) + X = 0.3 * rng.randn(600, 2) + X = rng.permutation(np.vstack((X + 2, X - 2))) + X_train = X[:1000] # Generate some abnormal novel observations - X_outliers = rng.uniform(low=-4, high=4, size=(20, 2)) - X_test = np.r_[X[100:], X_outliers] - y_test = np.array([0] * 20 + [1] * 20) + X_outliers = rng.uniform(low=-1, high=1, size=(200, 2)) + X_test = np.vstack((X[1000:], X_outliers)) + y_test = np.array([0] * 200 + [1] * 200) # fit the model clf = IsolationForest(max_samples=100, random_state=rng).fit(X_train) # predict scores (the lower, the more normal) - y_pred = - clf.decision_function(X_test) + y_pred = -clf.decision_function(X_test) # check that there is at most 6 errors (false positive or false negative) assert roc_auc_score(y_test, y_pred) > 0.98 @pytest.mark.parametrize("contamination", [0.25, "auto"]) -def test_iforest_works(contamination): +def test_iforest_works(contamination, global_random_seed): # toy sample (the last two samples are outliers) - X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1], [6, 3], [-4, 7]] + X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1], [7, 4], [-5, 9]] # Test IsolationForest - clf = IsolationForest(random_state=rng, contamination=contamination) + clf = IsolationForest(random_state=global_random_seed, contamination=contamination) clf.fit(X) decision_func = -clf.decision_function(X) pred = clf.predict(X) @@ -222,9 +187,9 @@ def test_max_samples_consistency(): def test_iforest_subsampled_features(): # It tests non-regression for #5732 which failed at predict. rng = check_random_state(0) - X_train, X_test, y_train, y_test = train_test_split(diabetes.data[:50], - diabetes.target[:50], - random_state=rng) + X_train, X_test, y_train, y_test = train_test_split( + diabetes.data[:50], diabetes.target[:50], random_state=rng + ) clf = IsolationForest(max_features=0.8) clf.fit(X_train, y_train) clf.predict(X_test) @@ -254,23 +219,29 @@ def test_score_samples(): X_train = [[1, 1], [1, 2], [2, 1]] clf1 = IsolationForest(contamination=0.1).fit(X_train) clf2 = IsolationForest().fit(X_train) - assert_array_equal(clf1.score_samples([[2., 2.]]), - clf1.decision_function([[2., 2.]]) + clf1.offset_) - assert_array_equal(clf2.score_samples([[2., 2.]]), - clf2.decision_function([[2., 2.]]) + clf2.offset_) - assert_array_equal(clf1.score_samples([[2., 2.]]), - clf2.score_samples([[2., 2.]])) + assert_array_equal( + clf1.score_samples([[2.0, 2.0]]), + clf1.decision_function([[2.0, 2.0]]) + clf1.offset_, + ) + assert_array_equal( + clf2.score_samples([[2.0, 2.0]]), + clf2.decision_function([[2.0, 2.0]]) + clf2.offset_, + ) + assert_array_equal( + clf1.score_samples([[2.0, 2.0]]), clf2.score_samples([[2.0, 2.0]]) + ) def test_iforest_warm_start(): - """Test iterative addition of iTrees to an iForest """ + """Test iterative addition of iTrees to an iForest""" rng = check_random_state(0) X = rng.randn(20, 2) # fit first 10 trees - clf = IsolationForest(n_estimators=10, max_samples=20, - random_state=rng, warm_start=True) + clf = IsolationForest( + n_estimators=10, max_samples=20, random_state=rng, warm_start=True + ) clf.fit(X) # remember the 1st tree tree_1 = clf.estimators_[0] @@ -283,33 +254,29 @@ def test_iforest_warm_start(): # mock get_chunk_n_rows to actually test more than one chunk (here one -# chunk = 3 rows: +# chunk has 3 rows): @patch( "sklearn.ensemble._iforest.get_chunk_n_rows", side_effect=Mock(**{"return_value": 3}), ) -@pytest.mark.parametrize( - "contamination, n_predict_calls", [(0.25, 3), ("auto", 2)] -) +@pytest.mark.parametrize("contamination, n_predict_calls", [(0.25, 3), ("auto", 2)]) def test_iforest_chunks_works1( - mocked_get_chunk, contamination, n_predict_calls + mocked_get_chunk, contamination, n_predict_calls, global_random_seed ): - test_iforest_works(contamination) + test_iforest_works(contamination, global_random_seed) assert mocked_get_chunk.call_count == n_predict_calls -# idem with chunk_size = 5 rows +# idem with chunk_size = 10 rows @patch( "sklearn.ensemble._iforest.get_chunk_n_rows", side_effect=Mock(**{"return_value": 10}), ) -@pytest.mark.parametrize( - "contamination, n_predict_calls", [(0.25, 3), ("auto", 2)] -) +@pytest.mark.parametrize("contamination, n_predict_calls", [(0.25, 3), ("auto", 2)]) def test_iforest_chunks_works2( - mocked_get_chunk, contamination, n_predict_calls + mocked_get_chunk, contamination, n_predict_calls, global_random_seed ): - test_iforest_works(contamination) + test_iforest_works(contamination, global_random_seed) assert mocked_get_chunk.call_count == n_predict_calls @@ -347,13 +314,80 @@ def test_iforest_with_uniform_data(): assert all(iforest.predict(np.ones((100, 10))) == 1) -# FIXME: remove in 1.2 -def test_n_features_deprecation(): - # Check that we raise the proper deprecation warning if accessing - # `n_features_`. - X = np.array([[1, 2], [3, 4]]) - y = np.array([1, 0]) - est = IsolationForest().fit(X, y) +@pytest.mark.parametrize("csc_container", CSC_CONTAINERS) +def test_iforest_with_n_jobs_does_not_segfault(csc_container): + """Check that Isolation Forest does not segfault with n_jobs=2 + + Non-regression test for #23252 + """ + X, _ = make_classification(n_samples=85_000, n_features=100, random_state=0) + X = csc_container(X) + IsolationForest(n_estimators=10, max_samples=256, n_jobs=2).fit(X) + + +def test_iforest_preserve_feature_names(): + """Check that feature names are preserved when contamination is not "auto". + + Feature names are required for consistency checks during scoring. + + Non-regression test for Issue #25844 + """ + pd = pytest.importorskip("pandas") + rng = np.random.RandomState(0) + + X = pd.DataFrame(data=rng.randn(4), columns=["a"]) + model = IsolationForest(random_state=0, contamination=0.05) + + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + model.fit(X) + + +@pytest.mark.parametrize("sparse_container", CSC_CONTAINERS + CSR_CONTAINERS) +def test_iforest_sparse_input_float_contamination(sparse_container): + """Check that `IsolationForest` accepts sparse matrix input and float value for + contamination. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/27626 + """ + X, _ = make_classification(n_samples=50, n_features=4, random_state=0) + X = sparse_container(X) + X.sort_indices() + contamination = 0.1 + iforest = IsolationForest( + n_estimators=5, contamination=contamination, random_state=0 + ).fit(X) + + X_decision = iforest.decision_function(X) + assert (X_decision < 0).sum() / X.shape[0] == pytest.approx(contamination) + + +@pytest.mark.parametrize("n_jobs", [1, 2]) +@pytest.mark.parametrize("contamination", [0.25, "auto"]) +def test_iforest_predict_parallel(global_random_seed, contamination, n_jobs): + """Check that `IsolationForest.predict` is parallelized.""" + # toy sample (the last two samples are outliers) + X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1], [7, 4], [-5, 9]] + + # Test IsolationForest + clf = IsolationForest( + random_state=global_random_seed, contamination=contamination, n_jobs=None + ) + clf.fit(X) + decision_func = -clf.decision_function(X) + pred = clf.predict(X) + + # assert detect outliers: + assert np.min(decision_func[-2:]) > np.max(decision_func[:-2]) + assert_array_equal(pred, 6 * [1] + 2 * [-1]) + + clf_parallel = IsolationForest( + random_state=global_random_seed, contamination=contamination, n_jobs=-1 + ) + clf_parallel.fit(X) + with parallel_backend("threading", n_jobs=n_jobs): + pred_paralell = clf_parallel.predict(X) - with pytest.warns(FutureWarning, match="n_features_ was deprecated"): - est.n_features_ + # assert the same results as non-parallel + assert_array_equal(pred, pred_paralell) diff --git a/sklearn/ensemble/tests/test_stacking.py b/sklearn/ensemble/tests/test_stacking.py index d6b4c385b9073..e944ecc4abb52 100644 --- a/sklearn/ensemble/tests/test_stacking.py +++ b/sklearn/ensemble/tests/test_stacking.py @@ -1,50 +1,67 @@ """Test the stacking classifier and regressor.""" -# Authors: Guillaume Lemaitre -# License: BSD 3 clause +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import re +from unittest.mock import Mock -import pytest import numpy as np -import scipy.sparse as sparse - -from sklearn.base import BaseEstimator -from sklearn.base import ClassifierMixin -from sklearn.base import RegressorMixin -from sklearn.base import clone - -from sklearn.exceptions import ConvergenceWarning - -from sklearn.datasets import load_iris -from sklearn.datasets import load_diabetes -from sklearn.datasets import load_breast_cancer -from sklearn.datasets import make_regression -from sklearn.datasets import make_classification - -from sklearn.dummy import DummyClassifier -from sklearn.dummy import DummyRegressor -from sklearn.linear_model import LogisticRegression -from sklearn.linear_model import LinearRegression -from sklearn.svm import LinearSVC -from sklearn.svm import LinearSVR -from sklearn.svm import SVC -from sklearn.ensemble import RandomForestClassifier -from sklearn.ensemble import RandomForestRegressor +import pytest +from numpy.testing import assert_array_equal +from scipy import sparse + +from sklearn import config_context +from sklearn.base import BaseEstimator, ClassifierMixin, RegressorMixin, clone +from sklearn.datasets import ( + load_breast_cancer, + load_diabetes, + load_iris, + make_classification, + make_multilabel_classification, + make_regression, +) +from sklearn.dummy import DummyClassifier, DummyRegressor +from sklearn.ensemble import ( + RandomForestClassifier, + RandomForestRegressor, + StackingClassifier, + StackingRegressor, +) +from sklearn.exceptions import ConvergenceWarning, NotFittedError +from sklearn.linear_model import ( + LinearRegression, + LogisticRegression, + Ridge, + RidgeClassifier, +) +from sklearn.model_selection import KFold, StratifiedKFold, train_test_split +from sklearn.neighbors import KNeighborsClassifier +from sklearn.neural_network import MLPClassifier from sklearn.preprocessing import scale - -from sklearn.ensemble import StackingClassifier -from sklearn.ensemble import StackingRegressor - -from sklearn.model_selection import train_test_split -from sklearn.model_selection import StratifiedKFold -from sklearn.model_selection import KFold - +from sklearn.svm import SVC, LinearSVC, LinearSVR +from sklearn.tests.metadata_routing_common import ( + ConsumingClassifier, + ConsumingRegressor, + _Registry, + check_recorded_metadata, +) from sklearn.utils._mocking import CheckingClassifier -from sklearn.utils._testing import assert_allclose -from sklearn.utils._testing import assert_allclose_dense_sparse -from sklearn.utils._testing import ignore_warnings - -X_diabetes, y_diabetes = load_diabetes(return_X_y=True) -X_iris, y_iris = load_iris(return_X_y=True) +from sklearn.utils._testing import ( + assert_allclose, + assert_allclose_dense_sparse, + ignore_warnings, +) +from sklearn.utils.fixes import COO_CONTAINERS, CSC_CONTAINERS, CSR_CONTAINERS + +diabetes = load_diabetes() +X_diabetes, y_diabetes = diabetes.data, diabetes.target +iris = load_iris() +X_iris, y_iris = iris.data, iris.target +X_multilabel, y_multilabel = make_multilabel_classification( + n_classes=3, random_state=42 +) +X_binary, y_binary = make_classification(n_classes=2, random_state=42) @pytest.mark.parametrize( @@ -60,10 +77,12 @@ def test_stacking_classifier_iris(cv, final_estimator, passthrough): X_train, X_test, y_train, y_test = train_test_split( scale(X_iris), y_iris, stratify=y_iris, random_state=42 ) - estimators = [('lr', LogisticRegression()), ('svc', LinearSVC())] + estimators = [("lr", LogisticRegression()), ("svc", LinearSVC())] clf = StackingClassifier( - estimators=estimators, final_estimator=final_estimator, cv=cv, - passthrough=passthrough + estimators=estimators, + final_estimator=final_estimator, + cv=cv, + passthrough=passthrough, ) clf.fit(X_train, y_train) clf.predict(X_test) @@ -76,7 +95,7 @@ def test_stacking_classifier_iris(cv, final_estimator, passthrough): if passthrough: assert_allclose(X_test, X_trans[:, -4:]) - clf.set_params(lr='drop') + clf.set_params(lr="drop") clf.fit(X_train, y_train) clf.predict(X_test) clf.predict_proba(X_test) @@ -99,8 +118,10 @@ def test_stacking_classifier_drop_column_binary_classification(): ) # both classifiers implement 'predict_proba' and will both drop one column - estimators = [('lr', LogisticRegression()), - ('rf', RandomForestClassifier(random_state=42))] + estimators = [ + ("lr", LogisticRegression()), + ("rf", RandomForestClassifier(random_state=42)), + ] clf = StackingClassifier(estimators=estimators, cv=3) clf.fit(X_train, y_train) @@ -108,7 +129,7 @@ def test_stacking_classifier_drop_column_binary_classification(): assert X_trans.shape[1] == 2 # LinearSVC does not implement 'predict_proba' and will not drop one column - estimators = [('lr', LogisticRegression()), ('svc', LinearSVC())] + estimators = [("lr", LogisticRegression()), ("svc", LinearSVC())] clf.set_params(estimators=estimators) clf.fit(X_train, y_train) @@ -122,15 +143,14 @@ def test_stacking_classifier_drop_estimator(): X_train, X_test, y_train, _ = train_test_split( scale(X_iris), y_iris, stratify=y_iris, random_state=42 ) - estimators = [('lr', 'drop'), ('svc', LinearSVC(random_state=0))] + estimators = [("lr", "drop"), ("svc", LinearSVC(random_state=0))] rf = RandomForestClassifier(n_estimators=10, random_state=42) clf = StackingClassifier( - estimators=[('svc', LinearSVC(random_state=0))], - final_estimator=rf, cv=5 - ) - clf_drop = StackingClassifier( - estimators=estimators, final_estimator=rf, cv=5 + estimators=[("svc", LinearSVC(random_state=0))], + final_estimator=rf, + cv=5, ) + clf_drop = StackingClassifier(estimators=estimators, final_estimator=rf, cv=5) clf.fit(X_train, y_train) clf_drop.fit(X_train, y_train) @@ -145,15 +165,14 @@ def test_stacking_regressor_drop_estimator(): X_train, X_test, y_train, _ = train_test_split( scale(X_diabetes), y_diabetes, random_state=42 ) - estimators = [('lr', 'drop'), ('svr', LinearSVR(random_state=0))] + estimators = [("lr", "drop"), ("svr", LinearSVR(random_state=0))] rf = RandomForestRegressor(n_estimators=10, random_state=42) reg = StackingRegressor( - estimators=[('svr', LinearSVR(random_state=0))], - final_estimator=rf, cv=5 - ) - reg_drop = StackingRegressor( - estimators=estimators, final_estimator=rf, cv=5 + estimators=[("svr", LinearSVR(random_state=0))], + final_estimator=rf, + cv=5, ) + reg_drop = StackingRegressor(estimators=estimators, final_estimator=rf, cv=5) reg.fit(X_train, y_train) reg_drop.fit(X_train, y_train) @@ -161,27 +180,28 @@ def test_stacking_regressor_drop_estimator(): assert_allclose(reg.transform(X_test), reg_drop.transform(X_test)) -@pytest.mark.parametrize( - "cv", [3, KFold(n_splits=3, shuffle=True, random_state=42)] -) +@pytest.mark.parametrize("cv", [3, KFold(n_splits=3, shuffle=True, random_state=42)]) @pytest.mark.parametrize( "final_estimator, predict_params", - [(None, {}), - (RandomForestRegressor(random_state=42), {}), - (DummyRegressor(), {'return_std': True})] + [ + (None, {}), + (RandomForestRegressor(random_state=42), {}), + (DummyRegressor(), {"return_std": True}), + ], ) @pytest.mark.parametrize("passthrough", [False, True]) -def test_stacking_regressor_diabetes(cv, final_estimator, predict_params, - passthrough): +def test_stacking_regressor_diabetes(cv, final_estimator, predict_params, passthrough): # prescale the data to avoid convergence warning without using a pipeline # for later assert X_train, X_test, y_train, _ = train_test_split( scale(X_diabetes), y_diabetes, random_state=42 ) - estimators = [('lr', LinearRegression()), ('svr', LinearSVR())] + estimators = [("lr", LinearRegression()), ("svr", LinearSVR())] reg = StackingRegressor( - estimators=estimators, final_estimator=final_estimator, cv=cv, - passthrough=passthrough + estimators=estimators, + final_estimator=final_estimator, + cv=cv, + passthrough=passthrough, ) reg.fit(X_train, y_train) result = reg.predict(X_test, **predict_params) @@ -195,7 +215,7 @@ def test_stacking_regressor_diabetes(cv, final_estimator, predict_params, if passthrough: assert_allclose(X_test, X_trans[:, -10:]) - reg.set_params(lr='drop') + reg.set_params(lr="drop") reg.fit(X_train, y_train) reg.predict(X_test) @@ -206,14 +226,15 @@ def test_stacking_regressor_diabetes(cv, final_estimator, predict_params, assert_allclose(X_test, X_trans[:, -10:]) -@pytest.mark.parametrize('fmt', ['csc', 'csr', 'coo']) -def test_stacking_regressor_sparse_passthrough(fmt): +@pytest.mark.parametrize( + "sparse_container", COO_CONTAINERS + CSC_CONTAINERS + CSR_CONTAINERS +) +def test_stacking_regressor_sparse_passthrough(sparse_container): # Check passthrough behavior on a sparse X matrix X_train, X_test, y_train, _ = train_test_split( - sparse.coo_matrix(scale(X_diabetes)).asformat(fmt), - y_diabetes, random_state=42 + sparse_container(scale(X_diabetes)), y_diabetes, random_state=42 ) - estimators = [('lr', LinearRegression()), ('svr', LinearSVR())] + estimators = [("lr", LinearRegression()), ("svr", LinearSVR())] rf = RandomForestRegressor(n_estimators=10, random_state=42) clf = StackingRegressor( estimators=estimators, final_estimator=rf, cv=5, passthrough=True @@ -225,14 +246,15 @@ def test_stacking_regressor_sparse_passthrough(fmt): assert X_test.format == X_trans.format -@pytest.mark.parametrize('fmt', ['csc', 'csr', 'coo']) -def test_stacking_classifier_sparse_passthrough(fmt): +@pytest.mark.parametrize( + "sparse_container", COO_CONTAINERS + CSC_CONTAINERS + CSR_CONTAINERS +) +def test_stacking_classifier_sparse_passthrough(sparse_container): # Check passthrough behavior on a sparse X matrix X_train, X_test, y_train, _ = train_test_split( - sparse.coo_matrix(scale(X_iris)).asformat(fmt), - y_iris, random_state=42 + sparse_container(scale(X_iris)), y_iris, random_state=42 ) - estimators = [('lr', LogisticRegression()), ('svc', LinearSVC())] + estimators = [("lr", LogisticRegression()), ("svc", LinearSVC())] rf = RandomForestClassifier(n_estimators=10, random_state=42) clf = StackingClassifier( estimators=estimators, final_estimator=rf, cv=5, passthrough=True @@ -251,9 +273,7 @@ def test_stacking_classifier_drop_binary_prob(): # Select only the 2 first classes X_, y_ = scale(X_iris[:100]), y_iris[:100] - estimators = [ - ('lr', LogisticRegression()), ('rf', RandomForestClassifier()) - ] + estimators = [("lr", LogisticRegression()), ("rf", RandomForestClassifier())] clf = StackingClassifier(estimators=estimators) clf.fit(X_, y_) X_meta = clf.transform(X_) @@ -271,78 +291,112 @@ def predict(self, X): class NoWeightClassifier(ClassifierMixin, BaseEstimator): def fit(self, X, y): - self.clf = DummyClassifier(strategy='stratified') + self.clf = DummyClassifier(strategy="stratified") return self.clf.fit(X, y) @pytest.mark.parametrize( "y, params, type_err, msg_err", - [(y_iris, - {'estimators': None}, - ValueError, "Invalid 'estimators' attribute,"), - (y_iris, - {'estimators': []}, - ValueError, "Invalid 'estimators' attribute,"), - (y_iris, - {'estimators': [('lr', LogisticRegression()), - ('svm', SVC(max_iter=5e4))], - 'stack_method': 'predict_proba'}, - ValueError, 'does not implement the method predict_proba'), - (y_iris, - {'estimators': [('lr', LogisticRegression()), - ('cor', NoWeightClassifier())]}, - TypeError, 'does not support sample weight'), - (y_iris, - {'estimators': [('lr', LogisticRegression()), - ('cor', LinearSVC(max_iter=5e4))], - 'final_estimator': NoWeightClassifier()}, - TypeError, 'does not support sample weight')] + [ + (y_iris, {"estimators": []}, ValueError, "Invalid 'estimators' attribute,"), + ( + y_iris, + { + "estimators": [ + ("lr", LogisticRegression()), + ("svm", SVC(max_iter=50_000)), + ], + "stack_method": "predict_proba", + }, + ValueError, + "does not implement the method predict_proba", + ), + ( + y_iris, + { + "estimators": [ + ("lr", LogisticRegression()), + ("cor", NoWeightClassifier()), + ] + }, + TypeError, + "does not support sample weight", + ), + ( + y_iris, + { + "estimators": [ + ("lr", LogisticRegression()), + ("cor", LinearSVC(max_iter=50_000)), + ], + "final_estimator": NoWeightClassifier(), + }, + TypeError, + "does not support sample weight", + ), + ], ) def test_stacking_classifier_error(y, params, type_err, msg_err): with pytest.raises(type_err, match=msg_err): clf = StackingClassifier(**params, cv=3) - clf.fit( - scale(X_iris), y, sample_weight=np.ones(X_iris.shape[0]) - ) + clf.fit(scale(X_iris), y, sample_weight=np.ones(X_iris.shape[0])) @pytest.mark.parametrize( "y, params, type_err, msg_err", - [(y_diabetes, - {'estimators': None}, - ValueError, "Invalid 'estimators' attribute,"), - (y_diabetes, - {'estimators': []}, - ValueError, "Invalid 'estimators' attribute,"), - (y_diabetes, - {'estimators': [('lr', LinearRegression()), - ('cor', NoWeightRegressor())]}, - TypeError, 'does not support sample weight'), - (y_diabetes, - {'estimators': [('lr', LinearRegression()), - ('cor', LinearSVR())], - 'final_estimator': NoWeightRegressor()}, - TypeError, 'does not support sample weight')] + [ + (y_diabetes, {"estimators": []}, ValueError, "Invalid 'estimators' attribute,"), + ( + y_diabetes, + {"estimators": [("lr", LinearRegression()), ("cor", NoWeightRegressor())]}, + TypeError, + "does not support sample weight", + ), + ( + y_diabetes, + { + "estimators": [ + ("lr", LinearRegression()), + ("cor", LinearSVR()), + ], + "final_estimator": NoWeightRegressor(), + }, + TypeError, + "does not support sample weight", + ), + ], ) def test_stacking_regressor_error(y, params, type_err, msg_err): with pytest.raises(type_err, match=msg_err): reg = StackingRegressor(**params, cv=3) - reg.fit( - scale(X_diabetes), y, sample_weight=np.ones(X_diabetes.shape[0]) - ) + reg.fit(scale(X_diabetes), y, sample_weight=np.ones(X_diabetes.shape[0])) @pytest.mark.parametrize( "estimator, X, y", - [(StackingClassifier( - estimators=[('lr', LogisticRegression(random_state=0)), - ('svm', LinearSVC(random_state=0))]), - X_iris[:100], y_iris[:100]), # keep only classes 0 and 1 - (StackingRegressor( - estimators=[('lr', LinearRegression()), - ('svm', LinearSVR(random_state=0))]), - X_diabetes, y_diabetes)], - ids=['StackingClassifier', 'StackingRegressor'] + [ + ( + StackingClassifier( + estimators=[ + ("lr", LogisticRegression(random_state=0)), + ("svm", LinearSVC(random_state=0)), + ] + ), + X_iris[:100], + y_iris[:100], + ), # keep only classes 0 and 1 + ( + StackingRegressor( + estimators=[ + ("lr", LinearRegression()), + ("svm", LinearSVR(random_state=0)), + ] + ), + X_diabetes, + y_diabetes, + ), + ], + ids=["StackingClassifier", "StackingRegressor"], ) def test_stacking_randomness(estimator, X, y): # checking that fixing the random state of the CV will lead to the same @@ -353,22 +407,24 @@ def test_stacking_randomness(estimator, X, y): ) estimator_drop = clone(estimator) - estimator_drop.set_params(lr='drop') + estimator_drop.set_params(lr="drop") estimator_drop.set_params( cv=KFold(shuffle=True, random_state=np.random.RandomState(0)) ) assert_allclose( estimator_full.fit(X, y).transform(X)[:, 1:], - estimator_drop.fit(X, y).transform(X) + estimator_drop.fit(X, y).transform(X), ) def test_stacking_classifier_stratify_default(): # check that we stratify the classes for the default CV clf = StackingClassifier( - estimators=[('lr', LogisticRegression(max_iter=1e4)), - ('svm', LinearSVC(max_iter=1e4))] + estimators=[ + ("lr", LogisticRegression(max_iter=10_000)), + ("svm", LinearSVC(max_iter=10_000)), + ] ) # since iris is not shuffled, a simple k-fold would not contain the # 3 classes during training @@ -377,19 +433,32 @@ def test_stacking_classifier_stratify_default(): @pytest.mark.parametrize( "stacker, X, y", - [(StackingClassifier( - estimators=[('lr', LogisticRegression()), - ('svm', LinearSVC(random_state=42))], - final_estimator=LogisticRegression(), - cv=KFold(shuffle=True, random_state=42)), - *load_breast_cancer(return_X_y=True)), - (StackingRegressor( - estimators=[('lr', LinearRegression()), - ('svm', LinearSVR(random_state=42))], - final_estimator=LinearRegression(), - cv=KFold(shuffle=True, random_state=42)), - X_diabetes, y_diabetes)], - ids=['StackingClassifier', 'StackingRegressor'] + [ + ( + StackingClassifier( + estimators=[ + ("lr", LogisticRegression()), + ("svm", LinearSVC(random_state=42)), + ], + final_estimator=LogisticRegression(), + cv=KFold(shuffle=True, random_state=42), + ), + *load_breast_cancer(return_X_y=True), + ), + ( + StackingRegressor( + estimators=[ + ("lr", LinearRegression()), + ("svm", LinearSVR(random_state=42)), + ], + final_estimator=LinearRegression(), + cv=KFold(shuffle=True, random_state=42), + ), + X_diabetes, + y_diabetes, + ), + ], + ids=["StackingClassifier", "StackingRegressor"], ) def test_stacking_with_sample_weight(stacker, X, y): # check that sample weights has an influence on the fitting @@ -423,12 +492,8 @@ def test_stacking_with_sample_weight(stacker, X, y): def test_stacking_classifier_sample_weight_fit_param(): # check sample_weight is passed to all invocations of fit stacker = StackingClassifier( - estimators=[ - ('lr', CheckingClassifier(expected_fit_params=['sample_weight'])) - ], - final_estimator=CheckingClassifier( - expected_fit_params=['sample_weight'] - ) + estimators=[("lr", CheckingClassifier(expected_sample_weight=True))], + final_estimator=CheckingClassifier(expected_sample_weight=True), ) stacker.fit(X_iris, y_iris, sample_weight=np.ones(X_iris.shape[0])) @@ -436,17 +501,30 @@ def test_stacking_classifier_sample_weight_fit_param(): @pytest.mark.filterwarnings("ignore::sklearn.exceptions.ConvergenceWarning") @pytest.mark.parametrize( "stacker, X, y", - [(StackingClassifier( - estimators=[('lr', LogisticRegression()), - ('svm', LinearSVC(random_state=42))], - final_estimator=LogisticRegression()), - *load_breast_cancer(return_X_y=True)), - (StackingRegressor( - estimators=[('lr', LinearRegression()), - ('svm', LinearSVR(random_state=42))], - final_estimator=LinearRegression()), - X_diabetes, y_diabetes)], - ids=['StackingClassifier', 'StackingRegressor'] + [ + ( + StackingClassifier( + estimators=[ + ("lr", LogisticRegression()), + ("svm", LinearSVC(random_state=42)), + ], + final_estimator=LogisticRegression(), + ), + *load_breast_cancer(return_X_y=True), + ), + ( + StackingRegressor( + estimators=[ + ("lr", LinearRegression()), + ("svm", LinearSVR(random_state=42)), + ], + final_estimator=LinearRegression(), + ), + X_diabetes, + y_diabetes, + ), + ], + ids=["StackingClassifier", "StackingRegressor"], ) def test_stacking_cv_influence(stacker, X, y): # check that the stacking affects the fit of the final estimator but not @@ -463,32 +541,123 @@ def test_stacking_cv_influence(stacker, X, y): stacker_cv_5.fit(X, y) # the base estimators should be identical - for est_cv_3, est_cv_5 in zip(stacker_cv_3.estimators_, - stacker_cv_5.estimators_): + for est_cv_3, est_cv_5 in zip(stacker_cv_3.estimators_, stacker_cv_5.estimators_): assert_allclose(est_cv_3.coef_, est_cv_5.coef_) # the final estimator should be different - with pytest.raises(AssertionError, match='Not equal'): - assert_allclose(stacker_cv_3.final_estimator_.coef_, - stacker_cv_5.final_estimator_.coef_) + with pytest.raises(AssertionError, match="Not equal"): + assert_allclose( + stacker_cv_3.final_estimator_.coef_, stacker_cv_5.final_estimator_.coef_ + ) + + +@pytest.mark.parametrize( + "Stacker, Estimator, stack_method, final_estimator, X, y", + [ + ( + StackingClassifier, + DummyClassifier, + "predict_proba", + LogisticRegression(random_state=42), + X_iris, + y_iris, + ), + ( + StackingRegressor, + DummyRegressor, + "predict", + LinearRegression(), + X_diabetes, + y_diabetes, + ), + ], +) +def test_stacking_prefit(Stacker, Estimator, stack_method, final_estimator, X, y): + """Check the behaviour of stacking when `cv='prefit'`""" + X_train1, X_train2, y_train1, y_train2 = train_test_split( + X, y, random_state=42, test_size=0.5 + ) + estimators = [ + ("d0", Estimator().fit(X_train1, y_train1)), + ("d1", Estimator().fit(X_train1, y_train1)), + ] + + # mock out fit and stack_method to be asserted later + for _, estimator in estimators: + estimator.fit = Mock(name="fit") + stack_func = getattr(estimator, stack_method) + predict_method_mocked = Mock(side_effect=stack_func) + # Mocking a method will not provide a `__name__` while Python methods + # do and we are using it in `_get_response_method`. + predict_method_mocked.__name__ = stack_method + setattr(estimator, stack_method, predict_method_mocked) + + stacker = Stacker( + estimators=estimators, cv="prefit", final_estimator=final_estimator + ) + stacker.fit(X_train2, y_train2) + + assert stacker.estimators_ == [estimator for _, estimator in estimators] + # fit was not called again + assert all(estimator.fit.call_count == 0 for estimator in stacker.estimators_) + + # stack method is called with the proper inputs + for estimator in stacker.estimators_: + stack_func_mock = getattr(estimator, stack_method) + stack_func_mock.assert_called_with(X_train2) + + +@pytest.mark.parametrize( + "stacker, X, y", + [ + ( + StackingClassifier( + estimators=[("lr", LogisticRegression()), ("svm", SVC())], + cv="prefit", + ), + X_iris, + y_iris, + ), + ( + StackingRegressor( + estimators=[ + ("lr", LinearRegression()), + ("svm", LinearSVR()), + ], + cv="prefit", + ), + X_diabetes, + y_diabetes, + ), + ], +) +def test_stacking_prefit_error(stacker, X, y): + # check that NotFittedError is raised + # if base estimators are not fitted when cv="prefit" + with pytest.raises(NotFittedError): + stacker.fit(X, y) -@pytest.mark.parametrize("make_dataset, Stacking, Estimator", [ - (make_classification, StackingClassifier, LogisticRegression), - (make_regression, StackingRegressor, LinearRegression) -]) +@pytest.mark.parametrize( + "make_dataset, Stacking, Estimator", + [ + (make_classification, StackingClassifier, LogisticRegression), + (make_regression, StackingRegressor, LinearRegression), + ], +) def test_stacking_without_n_features_in(make_dataset, Stacking, Estimator): # Stacking supports estimators without `n_features_in_`. Regression test # for #17353 class MyEstimator(Estimator): """Estimator without n_features_in_""" + def fit(self, X, y): super().fit(X, y) del self.n_features_in_ X, y = make_dataset(random_state=0, n_samples=100) - stacker = Stacking(estimators=[('lr', MyEstimator())]) + stacker = Stacking(estimators=[("lr", MyEstimator())]) msg = f"{Stacking.__name__} object has no attribute n_features_in_" with pytest.raises(AttributeError, match=msg): @@ -500,3 +669,351 @@ def fit(self, X, y): msg = "'MyEstimator' object has no attribute 'n_features_in_'" with pytest.raises(AttributeError, match=msg): stacker.n_features_in_ + + +@pytest.mark.parametrize( + "estimator", + [ + # output a 2D array of the probability of the positive class for each output + MLPClassifier(random_state=42), + # output a list of 2D array containing the probability of each class + # for each output + RandomForestClassifier(random_state=42), + ], + ids=["MLPClassifier", "RandomForestClassifier"], +) +def test_stacking_classifier_multilabel_predict_proba(estimator): + """Check the behaviour for the multilabel classification case and the + `predict_proba` stacking method. + + Estimators are not consistent with the output arrays and we need to ensure that + we handle all cases. + """ + X_train, X_test, y_train, y_test = train_test_split( + X_multilabel, y_multilabel, stratify=y_multilabel, random_state=42 + ) + n_outputs = 3 + + estimators = [("est", estimator)] + stacker = StackingClassifier( + estimators=estimators, + final_estimator=KNeighborsClassifier(), + stack_method="predict_proba", + ).fit(X_train, y_train) + + X_trans = stacker.transform(X_test) + assert X_trans.shape == (X_test.shape[0], n_outputs) + # we should not have any collinear classes and thus nothing should sum to 1 + assert not any(np.isclose(X_trans.sum(axis=1), 1.0)) + + y_pred = stacker.predict(X_test) + assert y_pred.shape == y_test.shape + + +def test_stacking_classifier_multilabel_decision_function(): + """Check the behaviour for the multilabel classification case and the + `decision_function` stacking method. Only `RidgeClassifier` supports this + case. + """ + X_train, X_test, y_train, y_test = train_test_split( + X_multilabel, y_multilabel, stratify=y_multilabel, random_state=42 + ) + n_outputs = 3 + + estimators = [("est", RidgeClassifier())] + stacker = StackingClassifier( + estimators=estimators, + final_estimator=KNeighborsClassifier(), + stack_method="decision_function", + ).fit(X_train, y_train) + + X_trans = stacker.transform(X_test) + assert X_trans.shape == (X_test.shape[0], n_outputs) + + y_pred = stacker.predict(X_test) + assert y_pred.shape == y_test.shape + + +@pytest.mark.parametrize("stack_method", ["auto", "predict"]) +@pytest.mark.parametrize("passthrough", [False, True]) +def test_stacking_classifier_multilabel_auto_predict(stack_method, passthrough): + """Check the behaviour for the multilabel classification case for stack methods + supported for all estimators or automatically picked up. + """ + X_train, X_test, y_train, y_test = train_test_split( + X_multilabel, y_multilabel, stratify=y_multilabel, random_state=42 + ) + y_train_before_fit = y_train.copy() + n_outputs = 3 + + estimators = [ + ("mlp", MLPClassifier(random_state=42)), + ("rf", RandomForestClassifier(random_state=42)), + ("ridge", RidgeClassifier()), + ] + final_estimator = KNeighborsClassifier() + + clf = StackingClassifier( + estimators=estimators, + final_estimator=final_estimator, + passthrough=passthrough, + stack_method=stack_method, + ).fit(X_train, y_train) + + # make sure we don't change `y_train` inplace + assert_array_equal(y_train_before_fit, y_train) + + y_pred = clf.predict(X_test) + assert y_pred.shape == y_test.shape + + if stack_method == "auto": + expected_stack_methods = ["predict_proba", "predict_proba", "decision_function"] + else: + expected_stack_methods = ["predict"] * len(estimators) + assert clf.stack_method_ == expected_stack_methods + + n_features_X_trans = n_outputs * len(estimators) + if passthrough: + n_features_X_trans += X_train.shape[1] + X_trans = clf.transform(X_test) + assert X_trans.shape == (X_test.shape[0], n_features_X_trans) + + assert_array_equal(clf.classes_, [np.array([0, 1])] * n_outputs) + + +@pytest.mark.parametrize( + "stacker, feature_names, X, y, expected_names", + [ + ( + StackingClassifier( + estimators=[ + ("lr", LogisticRegression(random_state=0)), + ("svm", LinearSVC(random_state=0)), + ] + ), + iris.feature_names, + X_iris, + y_iris, + [ + "stackingclassifier_lr0", + "stackingclassifier_lr1", + "stackingclassifier_lr2", + "stackingclassifier_svm0", + "stackingclassifier_svm1", + "stackingclassifier_svm2", + ], + ), + ( + StackingClassifier( + estimators=[ + ("lr", LogisticRegression(random_state=0)), + ("other", "drop"), + ("svm", LinearSVC(random_state=0)), + ] + ), + iris.feature_names, + X_iris[:100], + y_iris[:100], # keep only classes 0 and 1 + [ + "stackingclassifier_lr", + "stackingclassifier_svm", + ], + ), + ( + StackingRegressor( + estimators=[ + ("lr", LinearRegression()), + ("svm", LinearSVR(random_state=0)), + ] + ), + diabetes.feature_names, + X_diabetes, + y_diabetes, + [ + "stackingregressor_lr", + "stackingregressor_svm", + ], + ), + ], + ids=[ + "StackingClassifier_multiclass", + "StackingClassifier_binary", + "StackingRegressor", + ], +) +@pytest.mark.parametrize("passthrough", [True, False]) +def test_get_feature_names_out( + stacker, feature_names, X, y, expected_names, passthrough +): + """Check get_feature_names_out works for stacking.""" + + stacker.set_params(passthrough=passthrough) + stacker.fit(scale(X), y) + + if passthrough: + expected_names = np.concatenate((expected_names, feature_names)) + + names_out = stacker.get_feature_names_out(feature_names) + assert_array_equal(names_out, expected_names) + + +def test_stacking_classifier_base_regressor(): + """Check that a regressor can be used as the first layer in `StackingClassifier`.""" + X_train, X_test, y_train, y_test = train_test_split( + scale(X_iris), y_iris, stratify=y_iris, random_state=42 + ) + clf = StackingClassifier(estimators=[("ridge", Ridge())]) + clf.fit(X_train, y_train) + clf.predict(X_test) + clf.predict_proba(X_test) + assert clf.score(X_test, y_test) > 0.8 + + +def test_stacking_final_estimator_attribute_error(): + """Check that we raise the proper AttributeError when the final estimator + does not implement the `decision_function` method, which is decorated with + `available_if`. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/28108 + """ + X, y = make_classification(random_state=42) + + estimators = [ + ("lr", LogisticRegression()), + ("rf", RandomForestClassifier(n_estimators=2, random_state=42)), + ] + # RandomForestClassifier does not implement 'decision_function' and should raise + # an AttributeError + final_estimator = RandomForestClassifier(n_estimators=2, random_state=42) + clf = StackingClassifier( + estimators=estimators, final_estimator=final_estimator, cv=3 + ) + + outer_msg = "This 'StackingClassifier' has no attribute 'decision_function'" + inner_msg = "'RandomForestClassifier' object has no attribute 'decision_function'" + with pytest.raises(AttributeError, match=outer_msg) as exec_info: + clf.fit(X, y).decision_function(X) + assert isinstance(exec_info.value.__cause__, AttributeError) + assert inner_msg in str(exec_info.value.__cause__) + + +# Metadata Routing Tests +# ====================== + + +@pytest.mark.parametrize( + "Estimator, Child", + [ + (StackingClassifier, ConsumingClassifier), + (StackingRegressor, ConsumingRegressor), + ], +) +def test_routing_passed_metadata_not_supported(Estimator, Child): + """Test that the right error message is raised when metadata is passed while + not supported when `enable_metadata_routing=False`.""" + + with pytest.raises( + ValueError, match="is only supported if enable_metadata_routing=True" + ): + Estimator(["clf", Child()]).fit( + X_iris, y_iris, sample_weight=[1, 1, 1, 1, 1], metadata="a" + ) + + +@pytest.mark.parametrize( + "Estimator, Child", + [ + (StackingClassifier, ConsumingClassifier), + (StackingRegressor, ConsumingRegressor), + ], +) +@config_context(enable_metadata_routing=True) +def test_get_metadata_routing_without_fit(Estimator, Child): + # Test that metadata_routing() doesn't raise when called before fit. + est = Estimator([("sub_est", Child())]) + est.get_metadata_routing() + + +@pytest.mark.parametrize( + "Estimator, Child", + [ + (StackingClassifier, ConsumingClassifier), + (StackingRegressor, ConsumingRegressor), + ], +) +@pytest.mark.parametrize( + "prop, prop_value", [("sample_weight", np.ones(X_iris.shape[0])), ("metadata", "a")] +) +@config_context(enable_metadata_routing=True) +def test_metadata_routing_for_stacking_estimators(Estimator, Child, prop, prop_value): + """Test that metadata is routed correctly for Stacking*.""" + + est = Estimator( + [ + ( + "sub_est1", + Child(registry=_Registry()).set_fit_request(**{prop: True}), + ), + ( + "sub_est2", + Child(registry=_Registry()).set_fit_request(**{prop: True}), + ), + ], + final_estimator=Child(registry=_Registry()).set_predict_request(**{prop: True}), + ) + + est.fit(X_iris, y_iris, **{prop: prop_value}) + est.fit_transform(X_iris, y_iris, **{prop: prop_value}) + + est.predict(X_iris, **{prop: prop_value}) + + for estimator in est.estimators: + # access sub-estimator in (name, est) with estimator[1]: + registry = estimator[1].registry + assert len(registry) + for sub_est in registry: + check_recorded_metadata( + obj=sub_est, + method="fit", + parent="fit", + split_params=(prop), + **{prop: prop_value}, + ) + # access final_estimator: + registry = est.final_estimator_.registry + assert len(registry) + check_recorded_metadata( + obj=registry[-1], + method="predict", + parent="predict", + split_params=(prop), + **{prop: prop_value}, + ) + + +@pytest.mark.parametrize( + "Estimator, Child", + [ + (StackingClassifier, ConsumingClassifier), + (StackingRegressor, ConsumingRegressor), + ], +) +@config_context(enable_metadata_routing=True) +def test_metadata_routing_error_for_stacking_estimators(Estimator, Child): + """Test that the right error is raised when metadata is not requested.""" + sample_weight, metadata = np.ones(X_iris.shape[0]), "a" + + est = Estimator([("sub_est", Child())]) + + error_message = ( + "[sample_weight, metadata] are passed but are not explicitly set as requested" + f" or not requested for {Child.__name__}.fit" + ) + + with pytest.raises(ValueError, match=re.escape(error_message)): + est.fit(X_iris, y_iris, sample_weight=sample_weight, metadata=metadata) + + +# End of Metadata Routing Tests +# ============================= diff --git a/sklearn/ensemble/tests/test_voting.py b/sklearn/ensemble/tests/test_voting.py index d36e71a3c6ff3..fc3fc82c2bee8 100644 --- a/sklearn/ensemble/tests/test_voting.py +++ b/sklearn/ensemble/tests/test_voting.py @@ -1,47 +1,70 @@ """Testing for the VotingClassifier and VotingRegressor""" -import warnings -import pytest import re + import numpy as np +import pytest -from sklearn.utils._testing import assert_almost_equal, assert_array_equal -from sklearn.utils._testing import assert_array_almost_equal -from sklearn.exceptions import NotFittedError -from sklearn.linear_model import LinearRegression -from sklearn.linear_model import LogisticRegression -from sklearn.naive_bayes import GaussianNB -from sklearn.ensemble import RandomForestClassifier -from sklearn.ensemble import RandomForestRegressor -from sklearn.ensemble import VotingClassifier, VotingRegressor -from sklearn.tree import DecisionTreeClassifier -from sklearn.tree import DecisionTreeRegressor -from sklearn.model_selection import GridSearchCV -from sklearn import datasets -from sklearn.model_selection import cross_val_score, train_test_split +from sklearn import config_context, datasets +from sklearn.base import BaseEstimator, ClassifierMixin, clone from sklearn.datasets import make_multilabel_classification -from sklearn.svm import SVC +from sklearn.dummy import DummyRegressor +from sklearn.ensemble import ( + RandomForestClassifier, + RandomForestRegressor, + VotingClassifier, + VotingRegressor, +) +from sklearn.exceptions import NotFittedError +from sklearn.linear_model import LinearRegression, LogisticRegression +from sklearn.model_selection import GridSearchCV, cross_val_score, train_test_split from sklearn.multiclass import OneVsRestClassifier +from sklearn.naive_bayes import GaussianNB from sklearn.neighbors import KNeighborsClassifier -from sklearn.base import BaseEstimator, ClassifierMixin, clone -from sklearn.dummy import DummyRegressor - +from sklearn.preprocessing import StandardScaler +from sklearn.svm import SVC +from sklearn.tests.metadata_routing_common import ( + ConsumingClassifier, + ConsumingRegressor, + _Registry, + check_recorded_metadata, +) +from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor +from sklearn.utils._testing import ( + assert_almost_equal, + assert_array_almost_equal, + assert_array_equal, +) # Load datasets iris = datasets.load_iris() X, y = iris.data[:, 1:3], iris.target +# Scaled to solve ConvergenceWarning throw by Logistic Regression +X_scaled = StandardScaler().fit_transform(X) X_r, y_r = datasets.load_diabetes(return_X_y=True) @pytest.mark.parametrize( "params, err_msg", - [({'estimators': []}, - "Invalid 'estimators' attribute, 'estimators' should be a list of"), - ({'estimators': [('lr', LogisticRegression())], 'voting': 'error'}, - r"Voting must be 'soft' or 'hard'; got \(voting='error'\)"), - ({'estimators': [('lr', LogisticRegression())], 'weights': [1, 2]}, - "Number of `estimators` and weights must be equal")] + [ + ( + {"estimators": []}, + "Invalid 'estimators' attribute, 'estimators' should be a non-empty list", + ), + ( + {"estimators": [LogisticRegression()]}, + "Invalid 'estimators' attribute, 'estimators' should be a non-empty list", + ), + ( + {"estimators": [(213, LogisticRegression())]}, + "Invalid 'estimators' attribute, 'estimators' should be a non-empty list", + ), + ( + {"estimators": [("lr", LogisticRegression())], "weights": [1, 2]}, + "Number of `estimators` and weights must be equal", + ), + ], ) def test_voting_classifier_estimator_init(params, err_msg): ensemble = VotingClassifier(**params) @@ -50,98 +73,111 @@ def test_voting_classifier_estimator_init(params, err_msg): def test_predictproba_hardvoting(): - eclf = VotingClassifier(estimators=[('lr1', LogisticRegression()), - ('lr2', LogisticRegression())], - voting='hard') - msg = "predict_proba is not available when voting='hard'" - with pytest.raises(AttributeError, match=msg): + eclf = VotingClassifier( + estimators=[("lr1", LogisticRegression()), ("lr2", LogisticRegression())], + voting="hard", + ) + + inner_msg = "predict_proba is not available when voting='hard'" + outer_msg = "'VotingClassifier' has no attribute 'predict_proba'" + with pytest.raises(AttributeError, match=outer_msg) as exec_info: eclf.predict_proba + assert isinstance(exec_info.value.__cause__, AttributeError) + assert inner_msg in str(exec_info.value.__cause__) assert not hasattr(eclf, "predict_proba") - eclf.fit(X, y) + eclf.fit(X_scaled, y) assert not hasattr(eclf, "predict_proba") def test_notfitted(): - eclf = VotingClassifier(estimators=[('lr1', LogisticRegression()), - ('lr2', LogisticRegression())], - voting='soft') - ereg = VotingRegressor([('dr', DummyRegressor())]) - msg = ("This %s instance is not fitted yet. Call \'fit\'" - " with appropriate arguments before using this estimator.") - with pytest.raises(NotFittedError, match=msg % 'VotingClassifier'): + eclf = VotingClassifier( + estimators=[("lr1", LogisticRegression()), ("lr2", LogisticRegression())], + voting="soft", + ) + ereg = VotingRegressor([("dr", DummyRegressor())]) + msg = ( + "This %s instance is not fitted yet. Call 'fit'" + " with appropriate arguments before using this estimator." + ) + with pytest.raises(NotFittedError, match=msg % "VotingClassifier"): eclf.predict(X) - with pytest.raises(NotFittedError, match=msg % 'VotingClassifier'): + with pytest.raises(NotFittedError, match=msg % "VotingClassifier"): eclf.predict_proba(X) - with pytest.raises(NotFittedError, match=msg % 'VotingClassifier'): + with pytest.raises(NotFittedError, match=msg % "VotingClassifier"): eclf.transform(X) - with pytest.raises(NotFittedError, match=msg % 'VotingRegressor'): + with pytest.raises(NotFittedError, match=msg % "VotingRegressor"): ereg.predict(X_r) - with pytest.raises(NotFittedError, match=msg % 'VotingRegressor'): + with pytest.raises(NotFittedError, match=msg % "VotingRegressor"): ereg.transform(X_r) -def test_majority_label_iris(): +def test_majority_label_iris(global_random_seed): """Check classification by majority label on dataset iris.""" - clf1 = LogisticRegression(solver='liblinear', random_state=123) - clf2 = RandomForestClassifier(n_estimators=10, random_state=123) + clf1 = LogisticRegression(random_state=global_random_seed) + clf2 = RandomForestClassifier(n_estimators=10, random_state=global_random_seed) clf3 = GaussianNB() - eclf = VotingClassifier(estimators=[ - ('lr', clf1), ('rf', clf2), ('gnb', clf3)], - voting='hard') - scores = cross_val_score(eclf, X, y, scoring='accuracy') - assert_almost_equal(scores.mean(), 0.95, decimal=2) + eclf = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("gnb", clf3)], voting="hard" + ) + scores = cross_val_score(eclf, X, y, scoring="accuracy") + + assert scores.mean() >= 0.9 def test_tie_situation(): """Check voting classifier selects smaller class label in tie situation.""" - clf1 = LogisticRegression(random_state=123, solver='liblinear') + clf1 = LogisticRegression(random_state=123) clf2 = RandomForestClassifier(random_state=123) - eclf = VotingClassifier(estimators=[('lr', clf1), ('rf', clf2)], - voting='hard') - assert clf1.fit(X, y).predict(X)[73] == 2 - assert clf2.fit(X, y).predict(X)[73] == 1 - assert eclf.fit(X, y).predict(X)[73] == 1 + eclf = VotingClassifier(estimators=[("lr", clf1), ("rf", clf2)], voting="hard") + assert clf1.fit(X, y).predict(X)[52] == 2 + assert clf2.fit(X, y).predict(X)[52] == 1 + assert eclf.fit(X, y).predict(X)[52] == 1 -def test_weights_iris(): +def test_weights_iris(global_random_seed): """Check classification by average probabilities on dataset iris.""" - clf1 = LogisticRegression(random_state=123) - clf2 = RandomForestClassifier(random_state=123) + clf1 = LogisticRegression(random_state=global_random_seed) + clf2 = RandomForestClassifier(n_estimators=10, random_state=global_random_seed) clf3 = GaussianNB() - eclf = VotingClassifier(estimators=[ - ('lr', clf1), ('rf', clf2), ('gnb', clf3)], - voting='soft', - weights=[1, 2, 10]) - scores = cross_val_score(eclf, X, y, scoring='accuracy') - assert_almost_equal(scores.mean(), 0.93, decimal=2) + eclf = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("gnb", clf3)], + voting="soft", + weights=[1, 2, 10], + ) + scores = cross_val_score(eclf, X_scaled, y, scoring="accuracy") + assert scores.mean() >= 0.9 def test_weights_regressor(): """Check weighted average regression prediction on diabetes dataset.""" - reg1 = DummyRegressor(strategy='mean') - reg2 = DummyRegressor(strategy='median') - reg3 = DummyRegressor(strategy='quantile', quantile=.2) - ereg = VotingRegressor([('mean', reg1), ('median', reg2), - ('quantile', reg3)], weights=[1, 2, 10]) + reg1 = DummyRegressor(strategy="mean") + reg2 = DummyRegressor(strategy="median") + reg3 = DummyRegressor(strategy="quantile", quantile=0.2) + ereg = VotingRegressor( + [("mean", reg1), ("median", reg2), ("quantile", reg3)], weights=[1, 2, 10] + ) - X_r_train, X_r_test, y_r_train, y_r_test = \ - train_test_split(X_r, y_r, test_size=.25) + X_r_train, X_r_test, y_r_train, y_r_test = train_test_split( + X_r, y_r, test_size=0.25 + ) reg1_pred = reg1.fit(X_r_train, y_r_train).predict(X_r_test) reg2_pred = reg2.fit(X_r_train, y_r_train).predict(X_r_test) reg3_pred = reg3.fit(X_r_train, y_r_train).predict(X_r_test) ereg_pred = ereg.fit(X_r_train, y_r_train).predict(X_r_test) - avg = np.average(np.asarray([reg1_pred, reg2_pred, reg3_pred]), axis=0, - weights=[1, 2, 10]) + avg = np.average( + np.asarray([reg1_pred, reg2_pred, reg3_pred]), axis=0, weights=[1, 2, 10] + ) assert_almost_equal(ereg_pred, avg, decimal=2) - ereg_weights_none = VotingRegressor([('mean', reg1), ('median', reg2), - ('quantile', reg3)], weights=None) - ereg_weights_equal = VotingRegressor([('mean', reg1), ('median', reg2), - ('quantile', reg3)], - weights=[1, 1, 1]) + ereg_weights_none = VotingRegressor( + [("mean", reg1), ("median", reg2), ("quantile", reg3)], weights=None + ) + ereg_weights_equal = VotingRegressor( + [("mean", reg1), ("median", reg2), ("quantile", reg3)], weights=[1, 1, 1] + ) ereg_weights_none.fit(X_r_train, y_r_train) ereg_weights_equal.fit(X_r_train, y_r_train) ereg_none_pred = ereg_weights_none.predict(X_r_test) @@ -149,18 +185,15 @@ def test_weights_regressor(): assert_almost_equal(ereg_none_pred, ereg_equal_pred, decimal=2) -def test_predict_on_toy_problem(): +def test_predict_on_toy_problem(global_random_seed): """Manually check predicted class labels for toy dataset.""" - clf1 = LogisticRegression(random_state=123) - clf2 = RandomForestClassifier(random_state=123) + clf1 = LogisticRegression(random_state=global_random_seed) + clf2 = RandomForestClassifier(n_estimators=10, random_state=global_random_seed) clf3 = GaussianNB() - X = np.array([[-1.1, -1.5], - [-1.2, -1.4], - [-3.4, -2.2], - [1.1, 1.2], - [2.1, 1.4], - [3.1, 2.3]]) + X = np.array( + [[-1.1, -1.5], [-1.2, -1.4], [-3.4, -2.2], [1.1, 1.2], [2.1, 1.4], [3.1, 2.3]] + ) y = np.array([1, 1, 1, 2, 2, 2]) @@ -168,16 +201,18 @@ def test_predict_on_toy_problem(): assert_array_equal(clf2.fit(X, y).predict(X), [1, 1, 1, 2, 2, 2]) assert_array_equal(clf3.fit(X, y).predict(X), [1, 1, 1, 2, 2, 2]) - eclf = VotingClassifier(estimators=[ - ('lr', clf1), ('rf', clf2), ('gnb', clf3)], - voting='hard', - weights=[1, 1, 1]) + eclf = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("gnb", clf3)], + voting="hard", + weights=[1, 1, 1], + ) assert_array_equal(eclf.fit(X, y).predict(X), [1, 1, 1, 2, 2, 2]) - eclf = VotingClassifier(estimators=[ - ('lr', clf1), ('rf', clf2), ('gnb', clf3)], - voting='soft', - weights=[1, 1, 1]) + eclf = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("gnb", clf3)], + voting="soft", + weights=[1, 1, 1], + ) assert_array_equal(eclf.fit(X, y).predict(X), [1, 1, 1, 2, 2, 2]) @@ -189,30 +224,31 @@ def test_predict_proba_on_toy_problem(): X = np.array([[-1.1, -1.5], [-1.2, -1.4], [-3.4, -2.2], [1.1, 1.2]]) y = np.array([1, 1, 2, 2]) - clf1_res = np.array([[0.59790391, 0.40209609], - [0.57622162, 0.42377838], - [0.50728456, 0.49271544], - [0.40241774, 0.59758226]]) - - clf2_res = np.array([[0.8, 0.2], - [0.8, 0.2], - [0.2, 0.8], - [0.3, 0.7]]) - - clf3_res = np.array([[0.9985082, 0.0014918], - [0.99845843, 0.00154157], - [0., 1.], - [0., 1.]]) - - t00 = (2*clf1_res[0][0] + clf2_res[0][0] + clf3_res[0][0]) / 4 - t11 = (2*clf1_res[1][1] + clf2_res[1][1] + clf3_res[1][1]) / 4 - t21 = (2*clf1_res[2][1] + clf2_res[2][1] + clf3_res[2][1]) / 4 - t31 = (2*clf1_res[3][1] + clf2_res[3][1] + clf3_res[3][1]) / 4 - - eclf = VotingClassifier(estimators=[ - ('lr', clf1), ('rf', clf2), ('gnb', clf3)], - voting='soft', - weights=[2, 1, 1]) + clf1_res = np.array( + [ + [0.59790391, 0.40209609], + [0.57622162, 0.42377838], + [0.50728456, 0.49271544], + [0.40241774, 0.59758226], + ] + ) + + clf2_res = np.array([[0.8, 0.2], [0.8, 0.2], [0.2, 0.8], [0.3, 0.7]]) + + clf3_res = np.array( + [[0.9985082, 0.0014918], [0.99845843, 0.00154157], [0.0, 1.0], [0.0, 1.0]] + ) + + t00 = (2 * clf1_res[0][0] + clf2_res[0][0] + clf3_res[0][0]) / 4 + t11 = (2 * clf1_res[1][1] + clf2_res[1][1] + clf3_res[1][1]) / 4 + t21 = (2 * clf1_res[2][1] + clf2_res[2][1] + clf3_res[2][1]) / 4 + t31 = (2 * clf1_res[3][1] + clf2_res[3][1] + clf3_res[3][1]) / 4 + + eclf = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("gnb", clf3)], + voting="soft", + weights=[2, 1, 1], + ) eclf_res = eclf.fit(X, y).predict_proba(X) assert_almost_equal(t00, eclf_res[0][0], decimal=1) @@ -220,23 +256,26 @@ def test_predict_proba_on_toy_problem(): assert_almost_equal(t21, eclf_res[2][1], decimal=1) assert_almost_equal(t31, eclf_res[3][1], decimal=1) - with pytest.raises( - AttributeError, - match="predict_proba is not available when voting='hard'"): - eclf = VotingClassifier(estimators=[ - ('lr', clf1), ('rf', clf2), ('gnb', clf3)], - voting='hard') + inner_msg = "predict_proba is not available when voting='hard'" + outer_msg = "'VotingClassifier' has no attribute 'predict_proba'" + with pytest.raises(AttributeError, match=outer_msg) as exec_info: + eclf = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("gnb", clf3)], voting="hard" + ) eclf.fit(X, y).predict_proba(X) + assert isinstance(exec_info.value.__cause__, AttributeError) + assert inner_msg in str(exec_info.value.__cause__) + def test_multilabel(): """Check if error is raised for multilabel classification.""" - X, y = make_multilabel_classification(n_classes=2, n_labels=1, - allow_unlabeled=False, - random_state=123) - clf = OneVsRestClassifier(SVC(kernel='linear')) + X, y = make_multilabel_classification( + n_classes=2, n_labels=1, allow_unlabeled=False, random_state=123 + ) + clf = OneVsRestClassifier(SVC(kernel="linear")) - eclf = VotingClassifier(estimators=[('ovr', clf)], voting='hard') + eclf = VotingClassifier(estimators=[("ovr", clf)], voting="hard") try: eclf.fit(X, y) @@ -247,111 +286,122 @@ def test_multilabel(): def test_gridsearch(): """Check GridSearch support.""" clf1 = LogisticRegression(random_state=1) - clf2 = RandomForestClassifier(random_state=1) + clf2 = RandomForestClassifier(random_state=1, n_estimators=3) clf3 = GaussianNB() - eclf = VotingClassifier(estimators=[ - ('lr', clf1), ('rf', clf2), ('gnb', clf3)], - voting='soft') + eclf = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("gnb", clf3)], voting="soft" + ) - params = {'lr__C': [1.0, 100.0], - 'voting': ['soft', 'hard'], - 'weights': [[0.5, 0.5, 0.5], [1.0, 0.5, 0.5]]} + params = { + "lr__C": [1.0, 100.0], + "voting": ["soft", "hard"], + "weights": [[0.5, 0.5, 0.5], [1.0, 0.5, 0.5]], + } - grid = GridSearchCV(estimator=eclf, param_grid=params) - grid.fit(iris.data, iris.target) + grid = GridSearchCV(estimator=eclf, param_grid=params, cv=2) + grid.fit(X_scaled, y) -def test_parallel_fit(): +def test_parallel_fit(global_random_seed): """Check parallel backend of VotingClassifier on toy dataset.""" - clf1 = LogisticRegression(random_state=123) - clf2 = RandomForestClassifier(random_state=123) + clf1 = LogisticRegression(random_state=global_random_seed) + clf2 = RandomForestClassifier(n_estimators=10, random_state=global_random_seed) clf3 = GaussianNB() X = np.array([[-1.1, -1.5], [-1.2, -1.4], [-3.4, -2.2], [1.1, 1.2]]) y = np.array([1, 1, 2, 2]) - eclf1 = VotingClassifier(estimators=[ - ('lr', clf1), ('rf', clf2), ('gnb', clf3)], - voting='soft', - n_jobs=1).fit(X, y) - eclf2 = VotingClassifier(estimators=[ - ('lr', clf1), ('rf', clf2), ('gnb', clf3)], - voting='soft', - n_jobs=2).fit(X, y) + eclf1 = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("gnb", clf3)], voting="soft", n_jobs=1 + ).fit(X, y) + eclf2 = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("gnb", clf3)], voting="soft", n_jobs=2 + ).fit(X, y) assert_array_equal(eclf1.predict(X), eclf2.predict(X)) assert_array_almost_equal(eclf1.predict_proba(X), eclf2.predict_proba(X)) -def test_sample_weight(): +def test_sample_weight(global_random_seed): """Tests sample_weight parameter of VotingClassifier""" - clf1 = LogisticRegression(random_state=123) - clf2 = RandomForestClassifier(random_state=123) - clf3 = SVC(probability=True, random_state=123) - eclf1 = VotingClassifier(estimators=[ - ('lr', clf1), ('rf', clf2), ('svc', clf3)], - voting='soft').fit(X, y, sample_weight=np.ones((len(y),))) - eclf2 = VotingClassifier(estimators=[ - ('lr', clf1), ('rf', clf2), ('svc', clf3)], - voting='soft').fit(X, y) - assert_array_equal(eclf1.predict(X), eclf2.predict(X)) - assert_array_almost_equal(eclf1.predict_proba(X), eclf2.predict_proba(X)) - - sample_weight = np.random.RandomState(123).uniform(size=(len(y),)) - eclf3 = VotingClassifier(estimators=[('lr', clf1)], voting='soft') - eclf3.fit(X, y, sample_weight) - clf1.fit(X, y, sample_weight) - assert_array_equal(eclf3.predict(X), clf1.predict(X)) - assert_array_almost_equal(eclf3.predict_proba(X), clf1.predict_proba(X)) + clf1 = LogisticRegression(random_state=global_random_seed) + clf2 = RandomForestClassifier(n_estimators=10, random_state=global_random_seed) + clf3 = SVC(probability=True, random_state=global_random_seed) + eclf1 = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("svc", clf3)], voting="soft" + ).fit(X_scaled, y, sample_weight=np.ones((len(y),))) + eclf2 = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("svc", clf3)], voting="soft" + ).fit(X_scaled, y) + assert_array_equal(eclf1.predict(X_scaled), eclf2.predict(X_scaled)) + assert_array_almost_equal( + eclf1.predict_proba(X_scaled), eclf2.predict_proba(X_scaled) + ) + sample_weight = np.random.RandomState(global_random_seed).uniform(size=(len(y),)) + eclf3 = VotingClassifier(estimators=[("lr", clf1)], voting="soft") + eclf3.fit(X_scaled, y, sample_weight=sample_weight) + clf1.fit(X_scaled, y, sample_weight) + assert_array_equal(eclf3.predict(X_scaled), clf1.predict(X_scaled)) + assert_array_almost_equal( + eclf3.predict_proba(X_scaled), clf1.predict_proba(X_scaled) + ) # check that an error is raised and indicative if sample_weight is not # supported. clf4 = KNeighborsClassifier() - eclf3 = VotingClassifier(estimators=[ - ('lr', clf1), ('svc', clf3), ('knn', clf4)], - voting='soft') - msg = ('Underlying estimator KNeighborsClassifier does not support ' - 'sample weights.') + eclf3 = VotingClassifier( + estimators=[("lr", clf1), ("svc", clf3), ("knn", clf4)], voting="soft" + ) + msg = "Underlying estimator KNeighborsClassifier does not support sample weights." with pytest.raises(TypeError, match=msg): - eclf3.fit(X, y, sample_weight) + eclf3.fit(X_scaled, y, sample_weight=sample_weight) # check that _fit_single_estimator will raise the right error # it should raise the original error if this is not linked to sample_weight class ClassifierErrorFit(ClassifierMixin, BaseEstimator): - def fit(self, X, y, sample_weight): - raise TypeError('Error unrelated to sample_weight.') + def fit(self, X_scaled, y, sample_weight): + raise TypeError("Error unrelated to sample_weight.") + clf = ClassifierErrorFit() - with pytest.raises(TypeError, match='Error unrelated to sample_weight'): - clf.fit(X, y, sample_weight=sample_weight) + with pytest.raises(TypeError, match="Error unrelated to sample_weight"): + clf.fit(X_scaled, y, sample_weight=sample_weight) def test_sample_weight_kwargs(): """Check that VotingClassifier passes sample_weight as kwargs""" + class MockClassifier(ClassifierMixin, BaseEstimator): """Mock Classifier to check that sample_weight is received as kwargs""" + def fit(self, X, y, *args, **sample_weight): - assert 'sample_weight' in sample_weight + assert "sample_weight" in sample_weight clf = MockClassifier() - eclf = VotingClassifier(estimators=[('mock', clf)], voting='soft') + eclf = VotingClassifier(estimators=[("mock", clf)], voting="soft") # Should not raise an error. eclf.fit(X, y, sample_weight=np.ones((len(y),))) -def test_voting_classifier_set_params(): +def test_voting_classifier_set_params(global_random_seed): # check equivalence in the output when setting underlying estimators - clf1 = LogisticRegression(random_state=123, C=1.0) - clf2 = RandomForestClassifier(random_state=123, max_depth=None) + clf1 = LogisticRegression(random_state=global_random_seed) + clf2 = RandomForestClassifier( + n_estimators=10, random_state=global_random_seed, max_depth=None + ) clf3 = GaussianNB() - eclf1 = VotingClassifier([('lr', clf1), ('rf', clf2)], voting='soft', - weights=[1, 2]).fit(X, y) - eclf2 = VotingClassifier([('lr', clf1), ('nb', clf3)], voting='soft', - weights=[1, 2]) - eclf2.set_params(nb=clf2).fit(X, y) + eclf1 = VotingClassifier( + [("lr", clf1), ("rf", clf2)], voting="soft", weights=[1, 2] + ).fit(X_scaled, y) + eclf2 = VotingClassifier( + [("lr", clf1), ("nb", clf3)], voting="soft", weights=[1, 2] + ) + eclf2.set_params(nb=clf2).fit(X_scaled, y) - assert_array_equal(eclf1.predict(X), eclf2.predict(X)) - assert_array_almost_equal(eclf1.predict_proba(X), eclf2.predict_proba(X)) + assert_array_equal(eclf1.predict(X_scaled), eclf2.predict(X_scaled)) + assert_array_almost_equal( + eclf1.predict_proba(X_scaled), eclf2.predict_proba(X_scaled) + ) assert eclf2.estimators[0][1].get_params() == clf1.get_params() assert eclf2.estimators[1][1].get_params() == clf2.get_params() @@ -362,181 +412,382 @@ def test_set_estimator_drop(): clf1 = LogisticRegression(random_state=123) clf2 = RandomForestClassifier(n_estimators=10, random_state=123) clf3 = GaussianNB() - eclf1 = VotingClassifier(estimators=[('lr', clf1), ('rf', clf2), - ('nb', clf3)], - voting='hard', weights=[1, 0, 0.5]).fit(X, y) - - eclf2 = VotingClassifier(estimators=[('lr', clf1), ('rf', clf2), - ('nb', clf3)], - voting='hard', weights=[1, 1, 0.5]) - with pytest.warns(None) as record: - with warnings.catch_warnings(): - # scipy 1.3.0 uses tostring which is deprecated in numpy - warnings.filterwarnings("ignore", "tostring", DeprecationWarning) - eclf2.set_params(rf='drop').fit(X, y) - - assert not record + eclf1 = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("nb", clf3)], + voting="hard", + weights=[1, 0, 0.5], + ).fit(X, y) + + eclf2 = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("nb", clf3)], + voting="hard", + weights=[1, 1, 0.5], + ) + eclf2.set_params(rf="drop").fit(X, y) + assert_array_equal(eclf1.predict(X), eclf2.predict(X)) - assert dict(eclf2.estimators)["rf"] == 'drop' + assert dict(eclf2.estimators)["rf"] == "drop" assert len(eclf2.estimators_) == 2 - assert all(isinstance(est, (LogisticRegression, GaussianNB)) - for est in eclf2.estimators_) - assert eclf2.get_params()["rf"] == 'drop' - - eclf1.set_params(voting='soft').fit(X, y) - with pytest.warns(None) as record: - with warnings.catch_warnings(): - # scipy 1.3.0 uses tostring which is deprecated in numpy - warnings.filterwarnings("ignore", "tostring", DeprecationWarning) - eclf2.set_params(voting='soft').fit(X, y) - - assert not record + assert all( + isinstance(est, (LogisticRegression, GaussianNB)) for est in eclf2.estimators_ + ) + assert eclf2.get_params()["rf"] == "drop" + + eclf1.set_params(voting="soft").fit(X, y) + eclf2.set_params(voting="soft").fit(X, y) + assert_array_equal(eclf1.predict(X), eclf2.predict(X)) assert_array_almost_equal(eclf1.predict_proba(X), eclf2.predict_proba(X)) - msg = 'All estimators are dropped. At least one is required' - with pytest.warns(None) as record: - with pytest.raises(ValueError, match=msg): - eclf2.set_params(lr='drop', rf='drop', nb='drop').fit(X, y) - assert not record + msg = "All estimators are dropped. At least one is required" + with pytest.raises(ValueError, match=msg): + eclf2.set_params(lr="drop", rf="drop", nb="drop").fit(X, y) # Test soft voting transform X1 = np.array([[1], [2]]) y1 = np.array([1, 2]) - eclf1 = VotingClassifier(estimators=[('rf', clf2), ('nb', clf3)], - voting='soft', weights=[0, 0.5], - flatten_transform=False).fit(X1, y1) - - eclf2 = VotingClassifier(estimators=[('rf', clf2), ('nb', clf3)], - voting='soft', weights=[1, 0.5], - flatten_transform=False) - with pytest.warns(None) as record: - with warnings.catch_warnings(): - # scipy 1.3.0 uses tostring which is deprecated in numpy - warnings.filterwarnings("ignore", "tostring", DeprecationWarning) - eclf2.set_params(rf='drop').fit(X1, y1) - assert not record - assert_array_almost_equal(eclf1.transform(X1), - np.array([[[0.7, 0.3], [0.3, 0.7]], - [[1., 0.], [0., 1.]]])) - assert_array_almost_equal(eclf2.transform(X1), - np.array([[[1., 0.], - [0., 1.]]])) - eclf1.set_params(voting='hard') - eclf2.set_params(voting='hard') + eclf1 = VotingClassifier( + estimators=[("rf", clf2), ("nb", clf3)], + voting="soft", + weights=[0, 0.5], + flatten_transform=False, + ).fit(X1, y1) + + eclf2 = VotingClassifier( + estimators=[("rf", clf2), ("nb", clf3)], + voting="soft", + weights=[1, 0.5], + flatten_transform=False, + ) + eclf2.set_params(rf="drop").fit(X1, y1) + assert_array_almost_equal( + eclf1.transform(X1), + np.array([[[0.7, 0.3], [0.3, 0.7]], [[1.0, 0.0], [0.0, 1.0]]]), + ) + assert_array_almost_equal(eclf2.transform(X1), np.array([[[1.0, 0.0], [0.0, 1.0]]])) + eclf1.set_params(voting="hard") + eclf2.set_params(voting="hard") assert_array_equal(eclf1.transform(X1), np.array([[0, 0], [1, 1]])) assert_array_equal(eclf2.transform(X1), np.array([[0], [1]])) -def test_estimator_weights_format(): +def test_estimator_weights_format(global_random_seed): # Test estimator weights inputs as list and array - clf1 = LogisticRegression(random_state=123) - clf2 = RandomForestClassifier(random_state=123) - eclf1 = VotingClassifier(estimators=[ - ('lr', clf1), ('rf', clf2)], - weights=[1, 2], - voting='soft') - eclf2 = VotingClassifier(estimators=[ - ('lr', clf1), ('rf', clf2)], - weights=np.array((1, 2)), - voting='soft') - eclf1.fit(X, y) - eclf2.fit(X, y) - assert_array_almost_equal(eclf1.predict_proba(X), eclf2.predict_proba(X)) + clf1 = LogisticRegression(random_state=global_random_seed) + clf2 = RandomForestClassifier(n_estimators=10, random_state=global_random_seed) + eclf1 = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2)], weights=[1, 2], voting="soft" + ) + eclf2 = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2)], weights=np.array((1, 2)), voting="soft" + ) + eclf1.fit(X_scaled, y) + eclf2.fit(X_scaled, y) + assert_array_almost_equal( + eclf1.predict_proba(X_scaled), eclf2.predict_proba(X_scaled) + ) -def test_transform(): +def test_transform(global_random_seed): """Check transform method of VotingClassifier on toy dataset.""" - clf1 = LogisticRegression(random_state=123) - clf2 = RandomForestClassifier(random_state=123) + clf1 = LogisticRegression(random_state=global_random_seed) + clf2 = RandomForestClassifier(n_estimators=10, random_state=global_random_seed) clf3 = GaussianNB() X = np.array([[-1.1, -1.5], [-1.2, -1.4], [-3.4, -2.2], [1.1, 1.2]]) y = np.array([1, 1, 2, 2]) - eclf1 = VotingClassifier(estimators=[ - ('lr', clf1), ('rf', clf2), ('gnb', clf3)], - voting='soft').fit(X, y) - eclf2 = VotingClassifier(estimators=[ - ('lr', clf1), ('rf', clf2), ('gnb', clf3)], - voting='soft', - flatten_transform=True).fit(X, y) - eclf3 = VotingClassifier(estimators=[ - ('lr', clf1), ('rf', clf2), ('gnb', clf3)], - voting='soft', - flatten_transform=False).fit(X, y) + eclf1 = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("gnb", clf3)], voting="soft" + ).fit(X, y) + eclf2 = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("gnb", clf3)], + voting="soft", + flatten_transform=True, + ).fit(X, y) + eclf3 = VotingClassifier( + estimators=[("lr", clf1), ("rf", clf2), ("gnb", clf3)], + voting="soft", + flatten_transform=False, + ).fit(X, y) assert_array_equal(eclf1.transform(X).shape, (4, 6)) assert_array_equal(eclf2.transform(X).shape, (4, 6)) assert_array_equal(eclf3.transform(X).shape, (3, 4, 2)) - assert_array_almost_equal(eclf1.transform(X), - eclf2.transform(X)) + assert_array_almost_equal(eclf1.transform(X), eclf2.transform(X)) assert_array_almost_equal( - eclf3.transform(X).swapaxes(0, 1).reshape((4, 6)), - eclf2.transform(X) + eclf3.transform(X).swapaxes(0, 1).reshape((4, 6)), eclf2.transform(X) ) @pytest.mark.parametrize( "X, y, voter", - [(X, y, VotingClassifier( - [('lr', LogisticRegression()), - ('rf', RandomForestClassifier(n_estimators=5))])), - (X_r, y_r, VotingRegressor( - [('lr', LinearRegression()), - ('rf', RandomForestRegressor(n_estimators=5))]))] + [ + ( + X, + y, + VotingClassifier( + [ + ("lr", LogisticRegression()), + ("rf", RandomForestClassifier(n_estimators=5)), + ] + ), + ), + ( + X_r, + y_r, + VotingRegressor( + [ + ("lr", LinearRegression()), + ("rf", RandomForestRegressor(n_estimators=5)), + ] + ), + ), + ], ) def test_none_estimator_with_weights(X, y, voter): # check that an estimator can be set to 'drop' and passing some weight # regression test for # https://github.com/scikit-learn/scikit-learn/issues/13777 voter = clone(voter) - voter.fit(X, y, sample_weight=np.ones(y.shape)) - voter.set_params(lr='drop') - with pytest.warns(None) as record: - voter.fit(X, y, sample_weight=np.ones(y.shape)) - assert not record - y_pred = voter.predict(X) + # Scaled to solve ConvergenceWarning throw by Logistic Regression + X_scaled = StandardScaler().fit_transform(X) + voter.fit(X_scaled, y, sample_weight=np.ones(y.shape)) + voter.set_params(lr="drop") + voter.fit(X_scaled, y, sample_weight=np.ones(y.shape)) + y_pred = voter.predict(X_scaled) assert y_pred.shape == y.shape @pytest.mark.parametrize( "est", - [VotingRegressor( - estimators=[('lr', LinearRegression()), - ('tree', DecisionTreeRegressor(random_state=0))]), - VotingClassifier( - estimators=[('lr', LogisticRegression(random_state=0)), - ('tree', DecisionTreeClassifier(random_state=0))])], - ids=['VotingRegressor', 'VotingClassifier'] + [ + VotingRegressor( + estimators=[ + ("lr", LinearRegression()), + ("tree", DecisionTreeRegressor(random_state=0)), + ] + ), + VotingClassifier( + estimators=[ + ("lr", LogisticRegression(random_state=0)), + ("tree", DecisionTreeClassifier(random_state=0)), + ] + ), + ], + ids=["VotingRegressor", "VotingClassifier"], ) def test_n_features_in(est): - X = [[1, 2], [3, 4], [5, 6]] y = [0, 1, 2] - assert not hasattr(est, 'n_features_in_') + assert not hasattr(est, "n_features_in_") est.fit(X, y) assert est.n_features_in_ == 2 @pytest.mark.parametrize( "estimator", - [VotingRegressor( - estimators=[('lr', LinearRegression()), - ('rf', RandomForestRegressor(random_state=123))], - verbose=True), - VotingClassifier( - estimators=[('lr', LogisticRegression(random_state=123)), - ('rf', RandomForestClassifier(random_state=123))], - verbose=True)] + [ + VotingRegressor( + estimators=[ + ("lr", LinearRegression()), + ("rf", RandomForestRegressor(random_state=123)), + ], + verbose=True, + ), + VotingClassifier( + estimators=[ + ("lr", LogisticRegression(random_state=123)), + ("rf", RandomForestClassifier(random_state=123)), + ], + verbose=True, + ), + ], ) def test_voting_verbose(estimator, capsys): - X = np.array([[-1.1, -1.5], [-1.2, -1.4], [-3.4, -2.2], [1.1, 1.2]]) y = np.array([1, 1, 2, 2]) - pattern = (r'\[Voting\].*\(1 of 2\) Processing lr, total=.*\n' - r'\[Voting\].*\(2 of 2\) Processing rf, total=.*\n$') - - estimator.fit(X, y) + pattern = ( + r"\[Voting\].*\(1 of 2\) Processing lr, total=.*\n" + r"\[Voting\].*\(2 of 2\) Processing rf, total=.*\n$" + ) + clone(estimator).fit(X, y) assert re.match(pattern, capsys.readouterr()[0]) + + +def test_get_features_names_out_regressor(): + """Check get_feature_names_out output for regressor.""" + + X = [[1, 2], [3, 4], [5, 6]] + y = [0, 1, 2] + + voting = VotingRegressor( + estimators=[ + ("lr", LinearRegression()), + ("tree", DecisionTreeRegressor(random_state=0)), + ("ignore", "drop"), + ] + ) + voting.fit(X, y) + + names_out = voting.get_feature_names_out() + expected_names = ["votingregressor_lr", "votingregressor_tree"] + assert_array_equal(names_out, expected_names) + + +@pytest.mark.parametrize( + "kwargs, expected_names", + [ + ( + {"voting": "soft", "flatten_transform": True}, + [ + "votingclassifier_lr0", + "votingclassifier_lr1", + "votingclassifier_lr2", + "votingclassifier_tree0", + "votingclassifier_tree1", + "votingclassifier_tree2", + ], + ), + ({"voting": "hard"}, ["votingclassifier_lr", "votingclassifier_tree"]), + ], +) +def test_get_features_names_out_classifier(kwargs, expected_names): + """Check get_feature_names_out for classifier for different settings.""" + X = [[1, 2], [3, 4], [5, 6], [1, 1.2]] + y = [0, 1, 2, 0] + + voting = VotingClassifier( + estimators=[ + ("lr", LogisticRegression(random_state=0)), + ("tree", DecisionTreeClassifier(random_state=0)), + ], + **kwargs, + ) + voting.fit(X, y) + X_trans = voting.transform(X) + names_out = voting.get_feature_names_out() + + assert X_trans.shape[1] == len(expected_names) + assert_array_equal(names_out, expected_names) + + +def test_get_features_names_out_classifier_error(): + """Check that error is raised when voting="soft" and flatten_transform=False.""" + X = [[1, 2], [3, 4], [5, 6]] + y = [0, 1, 2] + + voting = VotingClassifier( + estimators=[ + ("lr", LogisticRegression(random_state=0)), + ("tree", DecisionTreeClassifier(random_state=0)), + ], + voting="soft", + flatten_transform=False, + ) + voting.fit(X, y) + + msg = ( + "get_feature_names_out is not supported when `voting='soft'` and " + "`flatten_transform=False`" + ) + with pytest.raises(ValueError, match=msg): + voting.get_feature_names_out() + + +# Metadata Routing Tests +# ====================== + + +@pytest.mark.parametrize( + "Estimator, Child", + [(VotingClassifier, ConsumingClassifier), (VotingRegressor, ConsumingRegressor)], +) +def test_routing_passed_metadata_not_supported(Estimator, Child): + """Test that the right error message is raised when metadata is passed while + not supported when `enable_metadata_routing=False`.""" + + X = np.array([[0, 1], [2, 2], [4, 6]]) + y = [1, 2, 3] + + with pytest.raises( + ValueError, match="is only supported if enable_metadata_routing=True" + ): + Estimator(["clf", Child()]).fit(X, y, sample_weight=[1, 1, 1], metadata="a") + + +@pytest.mark.parametrize( + "Estimator, Child", + [(VotingClassifier, ConsumingClassifier), (VotingRegressor, ConsumingRegressor)], +) +@config_context(enable_metadata_routing=True) +def test_get_metadata_routing_without_fit(Estimator, Child): + # Test that metadata_routing() doesn't raise when called before fit. + est = Estimator([("sub_est", Child())]) + est.get_metadata_routing() + + +@pytest.mark.parametrize( + "Estimator, Child", + [(VotingClassifier, ConsumingClassifier), (VotingRegressor, ConsumingRegressor)], +) +@pytest.mark.parametrize("prop", ["sample_weight", "metadata"]) +@config_context(enable_metadata_routing=True) +def test_metadata_routing_for_voting_estimators(Estimator, Child, prop): + """Test that metadata is routed correctly for Voting*.""" + X = np.array([[0, 1], [2, 2], [4, 6]]) + y = [1, 2, 3] + sample_weight, metadata = [1, 1, 1], "a" + + est = Estimator( + [ + ( + "sub_est1", + Child(registry=_Registry()).set_fit_request(**{prop: True}), + ), + ( + "sub_est2", + Child(registry=_Registry()).set_fit_request(**{prop: True}), + ), + ] + ) + + est.fit(X, y, **{prop: sample_weight if prop == "sample_weight" else metadata}) + + for estimator in est.estimators: + if prop == "sample_weight": + kwargs = {prop: sample_weight} + else: + kwargs = {prop: metadata} + # access sub-estimator in (name, est) with estimator[1] + registry = estimator[1].registry + assert len(registry) + for sub_est in registry: + check_recorded_metadata(obj=sub_est, method="fit", parent="fit", **kwargs) + + +@pytest.mark.parametrize( + "Estimator, Child", + [(VotingClassifier, ConsumingClassifier), (VotingRegressor, ConsumingRegressor)], +) +@config_context(enable_metadata_routing=True) +def test_metadata_routing_error_for_voting_estimators(Estimator, Child): + """Test that the right error is raised when metadata is not requested.""" + X = np.array([[0, 1], [2, 2], [4, 6]]) + y = [1, 2, 3] + sample_weight, metadata = [1, 1, 1], "a" + + est = Estimator([("sub_est", Child())]) + + error_message = ( + "[sample_weight, metadata] are passed but are not explicitly set as requested" + f" or not requested for {Child.__name__}.fit" + ) + + with pytest.raises(ValueError, match=re.escape(error_message)): + est.fit(X, y, sample_weight=sample_weight, metadata=metadata) + + +# End of Metadata Routing Tests +# ============================= diff --git a/sklearn/ensemble/tests/test_weight_boosting.py b/sklearn/ensemble/tests/test_weight_boosting.py old mode 100755 new mode 100644 index 587e3f538359c..55825c438d76b --- a/sklearn/ensemble/tests/test_weight_boosting.py +++ b/sklearn/ensemble/tests/test_weight_boosting.py @@ -1,39 +1,40 @@ """Testing for the boost module (sklearn.ensemble.boost).""" +import re + import numpy as np import pytest -from scipy.sparse import csc_matrix -from scipy.sparse import csr_matrix -from scipy.sparse import coo_matrix -from scipy.sparse import dok_matrix -from scipy.sparse import lil_matrix - -from sklearn.utils._testing import assert_array_equal, assert_array_less -from sklearn.utils._testing import assert_array_almost_equal - -from sklearn.base import BaseEstimator -from sklearn.base import clone +from sklearn import datasets +from sklearn.base import BaseEstimator, clone from sklearn.dummy import DummyClassifier, DummyRegressor -from sklearn.linear_model import LinearRegression -from sklearn.model_selection import train_test_split -from sklearn.model_selection import GridSearchCV -from sklearn.ensemble import AdaBoostClassifier -from sklearn.ensemble import AdaBoostRegressor +from sklearn.ensemble import AdaBoostClassifier, AdaBoostRegressor from sklearn.ensemble._weight_boosting import _samme_proba +from sklearn.linear_model import LinearRegression +from sklearn.model_selection import GridSearchCV, train_test_split from sklearn.svm import SVC, SVR from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor from sklearn.utils import shuffle from sklearn.utils._mocking import NoSampleWeightWrapper -from sklearn import datasets - +from sklearn.utils._testing import ( + assert_allclose, + assert_array_almost_equal, + assert_array_equal, +) +from sklearn.utils.fixes import ( + COO_CONTAINERS, + CSC_CONTAINERS, + CSR_CONTAINERS, + DOK_CONTAINERS, + LIL_CONTAINERS, +) # Common random state rng = np.random.RandomState(0) # Toy sample X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]] -y_class = ["foo", "foo", "foo", 1, 1, 1] # test string class labels +y_class = ["foo", "foo", "foo", 1, 1, 1] # test string class labels y_regr = [-1, -1, -1, 1, 1, 1] T = [[-1, -1], [2, 2], [3, 2]] y_t_class = ["foo", 1, 1] @@ -46,18 +47,18 @@ # Load the diabetes dataset and randomly permute it diabetes = datasets.load_diabetes() -diabetes.data, diabetes.target = shuffle(diabetes.data, diabetes.target, - random_state=rng) +diabetes.data, diabetes.target = shuffle( + diabetes.data, diabetes.target, random_state=rng +) def test_samme_proba(): # Test the `_samme_proba` helper function. # Define some example (bad) `predict_proba` output. - probs = np.array([[1, 1e-6, 0], - [0.19, 0.6, 0.2], - [-999, 0.51, 0.5], - [1e-6, 1, 1e-9]]) + probs = np.array( + [[1, 1e-6, 0], [0.19, 0.6, 0.2], [-999, 0.51, 0.5], [1e-6, 1, 1e-9]] + ) probs /= np.abs(probs.sum(axis=1))[:, np.newaxis] # _samme_proba calls estimator.predict_proba. @@ -66,6 +67,7 @@ class MockEstimator: def predict_proba(self, X): assert_array_equal(X.shape, probs.shape) return probs + mock = MockEstimator() samme_proba = _samme_proba(mock, 3, np.ones_like(probs)) @@ -88,10 +90,9 @@ def test_oneclass_adaboost_proba(): assert_array_almost_equal(clf.predict_proba(X), np.ones((len(X), 1))) -@pytest.mark.parametrize("algorithm", ["SAMME", "SAMME.R"]) -def test_classification_toy(algorithm): +def test_classification_toy(): # Check classification on a toy dataset. - clf = AdaBoostClassifier(algorithm=algorithm, random_state=0) + clf = AdaBoostClassifier(random_state=0) clf.fit(X, y_class) assert_array_equal(clf.predict(T), y_t_class) assert_array_equal(np.unique(np.asarray(y_t_class)), clf.classes_) @@ -109,61 +110,46 @@ def test_regression_toy(): def test_iris(): # Check consistency on dataset iris. classes = np.unique(iris.target) - clf_samme = prob_samme = None - for alg in ['SAMME', 'SAMME.R']: - clf = AdaBoostClassifier(algorithm=alg) - clf.fit(iris.data, iris.target) + clf = AdaBoostClassifier() + clf.fit(iris.data, iris.target) + + assert_array_equal(classes, clf.classes_) + proba = clf.predict_proba(iris.data) - assert_array_equal(classes, clf.classes_) - proba = clf.predict_proba(iris.data) - if alg == "SAMME": - clf_samme = clf - prob_samme = proba - assert proba.shape[1] == len(classes) - assert clf.decision_function(iris.data).shape[1] == len(classes) - - score = clf.score(iris.data, iris.target) - assert score > 0.9, "Failed with algorithm %s and score = %f" % \ - (alg, score) - - # Check we used multiple estimators - assert len(clf.estimators_) > 1 - # Check for distinct random states (see issue #7408) - assert (len(set(est.random_state for est in clf.estimators_)) == - len(clf.estimators_)) - - # Somewhat hacky regression test: prior to - # ae7adc880d624615a34bafdb1d75ef67051b8200, - # predict_proba returned SAMME.R values for SAMME. - clf_samme.algorithm = "SAMME.R" - assert_array_less(0, - np.abs(clf_samme.predict_proba(iris.data) - prob_samme)) - - -@pytest.mark.parametrize('loss', ['linear', 'square', 'exponential']) + assert proba.shape[1] == len(classes) + assert clf.decision_function(iris.data).shape[1] == len(classes) + + score = clf.score(iris.data, iris.target) + assert score > 0.9, f"Failed with {score = }" + + # Check we used multiple estimators + assert len(clf.estimators_) > 1 + # Check for distinct random states (see issue #7408) + assert len(set(est.random_state for est in clf.estimators_)) == len(clf.estimators_) + + +@pytest.mark.parametrize("loss", ["linear", "square", "exponential"]) def test_diabetes(loss): # Check consistency on dataset diabetes. reg = AdaBoostRegressor(loss=loss, random_state=0) reg.fit(diabetes.data, diabetes.target) score = reg.score(diabetes.data, diabetes.target) - assert score > 0.6 + assert score > 0.55 # Check we used multiple estimators assert len(reg.estimators_) > 1 # Check for distinct random states (see issue #7408) - assert (len(set(est.random_state for est in reg.estimators_)) == - len(reg.estimators_)) + assert len(set(est.random_state for est in reg.estimators_)) == len(reg.estimators_) -@pytest.mark.parametrize("algorithm", ["SAMME", "SAMME.R"]) -def test_staged_predict(algorithm): +def test_staged_predict(): # Check staged predictions. rng = np.random.RandomState(0) iris_weights = rng.randint(10, size=iris.target.shape) diabetes_weights = rng.randint(10, size=diabetes.target.shape) - clf = AdaBoostClassifier(algorithm=algorithm, n_estimators=10) + clf = AdaBoostClassifier(n_estimators=10) clf.fit(iris.data, iris.target, sample_weight=iris_weights) predictions = clf.predict(iris.data) @@ -172,8 +158,8 @@ def test_staged_predict(algorithm): staged_probas = [p for p in clf.staged_predict_proba(iris.data)] score = clf.score(iris.data, iris.target, sample_weight=iris_weights) staged_scores = [ - s for s in clf.staged_score( - iris.data, iris.target, sample_weight=iris_weights)] + s for s in clf.staged_score(iris.data, iris.target, sample_weight=iris_weights) + ] assert len(staged_predictions) == 10 assert_array_almost_equal(predictions, staged_predictions[-1]) @@ -188,11 +174,13 @@ def test_staged_predict(algorithm): predictions = clf.predict(diabetes.data) staged_predictions = [p for p in clf.staged_predict(diabetes.data)] - score = clf.score(diabetes.data, diabetes.target, - sample_weight=diabetes_weights) + score = clf.score(diabetes.data, diabetes.target, sample_weight=diabetes_weights) staged_scores = [ - s for s in clf.staged_score( - diabetes.data, diabetes.target, sample_weight=diabetes_weights)] + s + for s in clf.staged_score( + diabetes.data, diabetes.target, sample_weight=diabetes_weights + ) + ] assert len(staged_predictions) == 10 assert_array_almost_equal(predictions, staged_predictions[-1]) @@ -203,18 +191,17 @@ def test_staged_predict(algorithm): def test_gridsearch(): # Check that base trees can be grid-searched. # AdaBoost classification - boost = AdaBoostClassifier(base_estimator=DecisionTreeClassifier()) - parameters = {'n_estimators': (1, 2), - 'base_estimator__max_depth': (1, 2), - 'algorithm': ('SAMME', 'SAMME.R')} + boost = AdaBoostClassifier(estimator=DecisionTreeClassifier()) + parameters = { + "n_estimators": (1, 2), + "estimator__max_depth": (1, 2), + } clf = GridSearchCV(boost, parameters) clf.fit(iris.data, iris.target) # AdaBoost regression - boost = AdaBoostRegressor(base_estimator=DecisionTreeRegressor(), - random_state=0) - parameters = {'n_estimators': (1, 2), - 'base_estimator__max_depth': (1, 2)} + boost = AdaBoostRegressor(estimator=DecisionTreeRegressor(), random_state=0) + parameters = {"n_estimators": (1, 2), "estimator__max_depth": (1, 2)} clf = GridSearchCV(boost, parameters) clf.fit(diabetes.data, diabetes.target) @@ -224,16 +211,15 @@ def test_pickle(): import pickle # Adaboost classifier - for alg in ['SAMME', 'SAMME.R']: - obj = AdaBoostClassifier(algorithm=alg) - obj.fit(iris.data, iris.target) - score = obj.score(iris.data, iris.target) - s = pickle.dumps(obj) + obj = AdaBoostClassifier() + obj.fit(iris.data, iris.target) + score = obj.score(iris.data, iris.target) + s = pickle.dumps(obj) - obj2 = pickle.loads(s) - assert type(obj2) == obj.__class__ - score2 = obj2.score(iris.data, iris.target) - assert score == score2 + obj2 = pickle.loads(s) + assert type(obj2) == obj.__class__ + score2 = obj2.score(iris.data, iris.target) + assert score == score2 # Adaboost regressor obj = AdaBoostRegressor(random_state=0) @@ -249,39 +235,35 @@ def test_pickle(): def test_importances(): # Check variable importances. - X, y = datasets.make_classification(n_samples=2000, - n_features=10, - n_informative=3, - n_redundant=0, - n_repeated=0, - shuffle=False, - random_state=1) - - for alg in ['SAMME', 'SAMME.R']: - clf = AdaBoostClassifier(algorithm=alg) - - clf.fit(X, y) - importances = clf.feature_importances_ - - assert importances.shape[0] == 10 - assert (importances[:3, np.newaxis] >= importances[3:]).all() + X, y = datasets.make_classification( + n_samples=2000, + n_features=10, + n_informative=3, + n_redundant=0, + n_repeated=0, + shuffle=False, + random_state=1, + ) + clf = AdaBoostClassifier() -def test_error(): - # Test that it gives proper exception on deficient input. + clf.fit(X, y) + importances = clf.feature_importances_ - with pytest.raises(ValueError): - AdaBoostClassifier(learning_rate=-1).fit(X, y_class) + assert importances.shape[0] == 10 + assert (importances[:3, np.newaxis] >= importances[3:]).all() - with pytest.raises(ValueError): - AdaBoostClassifier(algorithm="foo").fit(X, y_class) - with pytest.raises(ValueError): - AdaBoostClassifier().fit(X, y_class, sample_weight=np.asarray([-1])) +def test_adaboost_classifier_sample_weight_error(): + # Test that it gives proper exception on incorrect sample weight. + clf = AdaBoostClassifier() + msg = re.escape("sample_weight.shape == (1,), expected (6,)") + with pytest.raises(ValueError, match=msg): + clf.fit(X, y_class, sample_weight=np.asarray([-1])) -def test_base_estimator(): - # Test different base estimators. +def test_estimator(): + # Test different estimators. from sklearn.ensemble import RandomForestClassifier # XXX doesn't work with y_class because RF doesn't support classes_ @@ -289,7 +271,7 @@ def test_base_estimator(): clf = AdaBoostClassifier(RandomForestClassifier()) clf.fit(X, y_regr) - clf = AdaBoostClassifier(SVC(), algorithm="SAMME") + clf = AdaBoostClassifier(SVC()) clf.fit(X, y_class) from sklearn.ensemble import RandomForestRegressor @@ -303,12 +285,32 @@ def test_base_estimator(): # Check that an empty discrete ensemble fails in fit, not predict. X_fail = [[1, 1], [1, 1], [1, 1], [1, 1]] y_fail = ["foo", "bar", 1, 2] - clf = AdaBoostClassifier(SVC(), algorithm="SAMME") + clf = AdaBoostClassifier(SVC()) with pytest.raises(ValueError, match="worse than random"): clf.fit(X_fail, y_fail) -def test_sparse_classification(): +def test_sample_weights_infinite(): + msg = "Sample weights have reached infinite values" + clf = AdaBoostClassifier(n_estimators=30, learning_rate=23.0) + with pytest.warns(UserWarning, match=msg): + clf.fit(iris.data, iris.target) + + +@pytest.mark.parametrize( + "sparse_container, expected_internal_type", + zip( + [ + *CSC_CONTAINERS, + *CSR_CONTAINERS, + *LIL_CONTAINERS, + *COO_CONTAINERS, + *DOK_CONTAINERS, + ], + CSC_CONTAINERS + 4 * CSR_CONTAINERS, + ), +) +def test_sparse_classification(sparse_container, expected_internal_type): # Check classification with sparse input. class CustomSVC(SVC): @@ -320,92 +322,98 @@ def fit(self, X, y, sample_weight=None): self.data_type_ = type(X) return self - X, y = datasets.make_multilabel_classification(n_classes=1, n_samples=15, - n_features=5, - random_state=42) + X, y = datasets.make_multilabel_classification( + n_classes=1, n_samples=15, n_features=5, random_state=42 + ) # Flatten y to a 1d array y = np.ravel(y) X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0) - for sparse_format in [csc_matrix, csr_matrix, lil_matrix, coo_matrix, - dok_matrix]: - X_train_sparse = sparse_format(X_train) - X_test_sparse = sparse_format(X_test) - - # Trained on sparse format - sparse_classifier = AdaBoostClassifier( - base_estimator=CustomSVC(probability=True), - random_state=1, - algorithm="SAMME" - ).fit(X_train_sparse, y_train) - - # Trained on dense format - dense_classifier = AdaBoostClassifier( - base_estimator=CustomSVC(probability=True), - random_state=1, - algorithm="SAMME" - ).fit(X_train, y_train) - - # predict - sparse_results = sparse_classifier.predict(X_test_sparse) - dense_results = dense_classifier.predict(X_test) - assert_array_equal(sparse_results, dense_results) - - # decision_function - sparse_results = sparse_classifier.decision_function(X_test_sparse) - dense_results = dense_classifier.decision_function(X_test) - assert_array_almost_equal(sparse_results, dense_results) - - # predict_log_proba - sparse_results = sparse_classifier.predict_log_proba(X_test_sparse) - dense_results = dense_classifier.predict_log_proba(X_test) - assert_array_almost_equal(sparse_results, dense_results) - - # predict_proba - sparse_results = sparse_classifier.predict_proba(X_test_sparse) - dense_results = dense_classifier.predict_proba(X_test) - assert_array_almost_equal(sparse_results, dense_results) - - # score - sparse_results = sparse_classifier.score(X_test_sparse, y_test) - dense_results = dense_classifier.score(X_test, y_test) - assert_array_almost_equal(sparse_results, dense_results) - - # staged_decision_function - sparse_results = sparse_classifier.staged_decision_function( - X_test_sparse) - dense_results = dense_classifier.staged_decision_function(X_test) - for sprase_res, dense_res in zip(sparse_results, dense_results): - assert_array_almost_equal(sprase_res, dense_res) - - # staged_predict - sparse_results = sparse_classifier.staged_predict(X_test_sparse) - dense_results = dense_classifier.staged_predict(X_test) - for sprase_res, dense_res in zip(sparse_results, dense_results): - assert_array_equal(sprase_res, dense_res) - - # staged_predict_proba - sparse_results = sparse_classifier.staged_predict_proba(X_test_sparse) - dense_results = dense_classifier.staged_predict_proba(X_test) - for sprase_res, dense_res in zip(sparse_results, dense_results): - assert_array_almost_equal(sprase_res, dense_res) - - # staged_score - sparse_results = sparse_classifier.staged_score(X_test_sparse, - y_test) - dense_results = dense_classifier.staged_score(X_test, y_test) - for sprase_res, dense_res in zip(sparse_results, dense_results): - assert_array_equal(sprase_res, dense_res) - - # Verify sparsity of data is maintained during training - types = [i.data_type_ for i in sparse_classifier.estimators_] - - assert all([(t == csc_matrix or t == csr_matrix) - for t in types]) - - -def test_sparse_regression(): + X_train_sparse = sparse_container(X_train) + X_test_sparse = sparse_container(X_test) + + # Trained on sparse format + sparse_classifier = AdaBoostClassifier( + estimator=CustomSVC(probability=True), + random_state=1, + ).fit(X_train_sparse, y_train) + + # Trained on dense format + dense_classifier = AdaBoostClassifier( + estimator=CustomSVC(probability=True), + random_state=1, + ).fit(X_train, y_train) + + # predict + sparse_clf_results = sparse_classifier.predict(X_test_sparse) + dense_clf_results = dense_classifier.predict(X_test) + assert_array_equal(sparse_clf_results, dense_clf_results) + + # decision_function + sparse_clf_results = sparse_classifier.decision_function(X_test_sparse) + dense_clf_results = dense_classifier.decision_function(X_test) + assert_array_almost_equal(sparse_clf_results, dense_clf_results) + + # predict_log_proba + sparse_clf_results = sparse_classifier.predict_log_proba(X_test_sparse) + dense_clf_results = dense_classifier.predict_log_proba(X_test) + assert_array_almost_equal(sparse_clf_results, dense_clf_results) + + # predict_proba + sparse_clf_results = sparse_classifier.predict_proba(X_test_sparse) + dense_clf_results = dense_classifier.predict_proba(X_test) + assert_array_almost_equal(sparse_clf_results, dense_clf_results) + + # score + sparse_clf_results = sparse_classifier.score(X_test_sparse, y_test) + dense_clf_results = dense_classifier.score(X_test, y_test) + assert_array_almost_equal(sparse_clf_results, dense_clf_results) + + # staged_decision_function + sparse_clf_results = sparse_classifier.staged_decision_function(X_test_sparse) + dense_clf_results = dense_classifier.staged_decision_function(X_test) + for sparse_clf_res, dense_clf_res in zip(sparse_clf_results, dense_clf_results): + assert_array_almost_equal(sparse_clf_res, dense_clf_res) + + # staged_predict + sparse_clf_results = sparse_classifier.staged_predict(X_test_sparse) + dense_clf_results = dense_classifier.staged_predict(X_test) + for sparse_clf_res, dense_clf_res in zip(sparse_clf_results, dense_clf_results): + assert_array_equal(sparse_clf_res, dense_clf_res) + + # staged_predict_proba + sparse_clf_results = sparse_classifier.staged_predict_proba(X_test_sparse) + dense_clf_results = dense_classifier.staged_predict_proba(X_test) + for sparse_clf_res, dense_clf_res in zip(sparse_clf_results, dense_clf_results): + assert_array_almost_equal(sparse_clf_res, dense_clf_res) + + # staged_score + sparse_clf_results = sparse_classifier.staged_score(X_test_sparse, y_test) + dense_clf_results = dense_classifier.staged_score(X_test, y_test) + for sparse_clf_res, dense_clf_res in zip(sparse_clf_results, dense_clf_results): + assert_array_equal(sparse_clf_res, dense_clf_res) + + # Verify sparsity of data is maintained during training + types = [i.data_type_ for i in sparse_classifier.estimators_] + + assert all([t == expected_internal_type for t in types]) + + +@pytest.mark.parametrize( + "sparse_container, expected_internal_type", + zip( + [ + *CSC_CONTAINERS, + *CSR_CONTAINERS, + *LIL_CONTAINERS, + *COO_CONTAINERS, + *DOK_CONTAINERS, + ], + CSC_CONTAINERS + 4 * CSR_CONTAINERS, + ), +) +def test_sparse_regression(sparse_container, expected_internal_type): # Check regression with sparse input. class CustomSVR(SVR): @@ -417,43 +425,39 @@ def fit(self, X, y, sample_weight=None): self.data_type_ = type(X) return self - X, y = datasets.make_regression(n_samples=15, n_features=50, n_targets=1, - random_state=42) + X, y = datasets.make_regression( + n_samples=15, n_features=50, n_targets=1, random_state=42 + ) X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0) - for sparse_format in [csc_matrix, csr_matrix, lil_matrix, coo_matrix, - dok_matrix]: - X_train_sparse = sparse_format(X_train) - X_test_sparse = sparse_format(X_test) + X_train_sparse = sparse_container(X_train) + X_test_sparse = sparse_container(X_test) - # Trained on sparse format - sparse_classifier = AdaBoostRegressor( - base_estimator=CustomSVR(), - random_state=1 - ).fit(X_train_sparse, y_train) + # Trained on sparse format + sparse_regressor = AdaBoostRegressor(estimator=CustomSVR(), random_state=1).fit( + X_train_sparse, y_train + ) - # Trained on dense format - dense_classifier = dense_results = AdaBoostRegressor( - base_estimator=CustomSVR(), - random_state=1 - ).fit(X_train, y_train) + # Trained on dense format + dense_regressor = AdaBoostRegressor(estimator=CustomSVR(), random_state=1).fit( + X_train, y_train + ) - # predict - sparse_results = sparse_classifier.predict(X_test_sparse) - dense_results = dense_classifier.predict(X_test) - assert_array_almost_equal(sparse_results, dense_results) + # predict + sparse_regr_results = sparse_regressor.predict(X_test_sparse) + dense_regr_results = dense_regressor.predict(X_test) + assert_array_almost_equal(sparse_regr_results, dense_regr_results) - # staged_predict - sparse_results = sparse_classifier.staged_predict(X_test_sparse) - dense_results = dense_classifier.staged_predict(X_test) - for sprase_res, dense_res in zip(sparse_results, dense_results): - assert_array_almost_equal(sprase_res, dense_res) + # staged_predict + sparse_regr_results = sparse_regressor.staged_predict(X_test_sparse) + dense_regr_results = dense_regressor.staged_predict(X_test) + for sparse_regr_res, dense_regr_res in zip(sparse_regr_results, dense_regr_results): + assert_array_almost_equal(sparse_regr_res, dense_regr_res) - types = [i.data_type_ for i in sparse_classifier.estimators_] + types = [i.data_type_ for i in sparse_regressor.estimators_] - assert all([(t == csc_matrix or t == csr_matrix) - for t in types]) + assert all([t == expected_internal_type for t in types]) def test_sample_weight_adaboost_regressor(): @@ -462,8 +466,8 @@ def test_sample_weight_adaboost_regressor(): The random weighted sampling is done internally in the _boost method in AdaBoostRegressor. """ - class DummyEstimator(BaseEstimator): + class DummyEstimator(BaseEstimator): def fit(self, X, y): pass @@ -482,11 +486,11 @@ def test_multidimensional_X(): """ rng = np.random.RandomState(0) - X = rng.randn(50, 3, 3) - yc = rng.choice([0, 1], 50) - yr = rng.randn(50) + X = rng.randn(51, 3, 3) + yc = rng.choice([0, 1], 51) + yr = rng.randn(51) - boost = AdaBoostClassifier(DummyClassifier(strategy='most_frequent')) + boost = AdaBoostClassifier(DummyClassifier(strategy="most_frequent")) boost.fit(X, yc) boost.predict(X) boost.predict_proba(X) @@ -496,15 +500,11 @@ def test_multidimensional_X(): boost.predict(X) -@pytest.mark.parametrize("algorithm", ['SAMME', 'SAMME.R']) -def test_adaboostclassifier_without_sample_weight(algorithm): +def test_adaboostclassifier_without_sample_weight(): X, y = iris.data, iris.target - base_estimator = NoSampleWeightWrapper(DummyClassifier()) - clf = AdaBoostClassifier( - base_estimator=base_estimator, algorithm=algorithm - ) - err_msg = ("{} doesn't support sample_weight" - .format(base_estimator.__class__.__name__)) + estimator = NoSampleWeightWrapper(DummyClassifier()) + clf = AdaBoostClassifier(estimator=estimator) + err_msg = "{} doesn't support sample_weight".format(estimator.__class__.__name__) with pytest.raises(ValueError, match=err_msg): clf.fit(X, y) @@ -514,7 +514,7 @@ def test_adaboostregressor_sample_weight(): # for a weak learner rng = np.random.RandomState(42) X = np.linspace(0, 100, num=1000) - y = (.8 * X + 0.2) + (rng.rand(X.shape[0]) * 0.0001) + y = (0.8 * X + 0.2) + (rng.rand(X.shape[0]) * 0.0001) X = X.reshape(-1, 1) # add an arbitrary outlier @@ -523,7 +523,7 @@ def test_adaboostregressor_sample_weight(): # random_state=0 ensure that the underlying bootstrap will use the outlier regr_no_outlier = AdaBoostRegressor( - base_estimator=LinearRegression(), n_estimators=1, random_state=0 + estimator=LinearRegression(), n_estimators=1, random_state=0 ) regr_with_weight = clone(regr_no_outlier) regr_with_outlier = clone(regr_no_outlier) @@ -546,32 +546,94 @@ def test_adaboostregressor_sample_weight(): assert score_with_outlier < score_with_weight assert score_no_outlier == pytest.approx(score_with_weight) -@pytest.mark.parametrize("algorithm", ["SAMME", "SAMME.R"]) -def test_adaboost_consistent_predict(algorithm): + +def test_adaboost_consistent_predict(): # check that predict_proba and predict give consistent results # regression test for: # https://github.com/scikit-learn/scikit-learn/issues/14084 X_train, X_test, y_train, y_test = train_test_split( *datasets.load_digits(return_X_y=True), random_state=42 ) - model = AdaBoostClassifier(algorithm=algorithm, random_state=42) + model = AdaBoostClassifier(random_state=42) model.fit(X_train, y_train) assert_array_equal( - np.argmax(model.predict_proba(X_test), axis=1), - model.predict(X_test) + np.argmax(model.predict_proba(X_test), axis=1), model.predict(X_test) ) @pytest.mark.parametrize( - 'model, X, y', - [(AdaBoostClassifier(), iris.data, iris.target), - (AdaBoostRegressor(), diabetes.data, diabetes.target)] + "model, X, y", + [ + (AdaBoostClassifier(), iris.data, iris.target), + (AdaBoostRegressor(), diabetes.data, diabetes.target), + ], ) def test_adaboost_negative_weight_error(model, X, y): sample_weight = np.ones_like(y) sample_weight[-1] = -10 - err_msg = "sample_weight cannot contain negative weight" + err_msg = "Negative values in data passed to `sample_weight`" with pytest.raises(ValueError, match=err_msg): model.fit(X, y, sample_weight=sample_weight) + + +def test_adaboost_numerically_stable_feature_importance_with_small_weights(): + """Check that we don't create NaN feature importance with numerically + instable inputs. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/20320 + """ + rng = np.random.RandomState(42) + X = rng.normal(size=(1000, 10)) + y = rng.choice([0, 1], size=1000) + sample_weight = np.ones_like(y) * 1e-263 + tree = DecisionTreeClassifier(max_depth=10, random_state=12) + ada_model = AdaBoostClassifier(estimator=tree, n_estimators=20, random_state=12) + ada_model.fit(X, y, sample_weight=sample_weight) + assert np.isnan(ada_model.feature_importances_).sum() == 0 + + +def test_adaboost_decision_function(global_random_seed): + """Check that the decision function respects the symmetric constraint for weak + learners. + + Non-regression test for: + https://github.com/scikit-learn/scikit-learn/issues/26520 + """ + n_classes = 3 + X, y = datasets.make_classification( + n_classes=n_classes, n_clusters_per_class=1, random_state=global_random_seed + ) + clf = AdaBoostClassifier(n_estimators=1, random_state=global_random_seed).fit(X, y) + + y_score = clf.decision_function(X) + assert_allclose(y_score.sum(axis=1), 0, atol=1e-8) + + # With a single learner, we expect to have a decision function in + # {1, - 1 / (n_classes - 1)}. + assert set(np.unique(y_score)) == {1, -1 / (n_classes - 1)} + + # We can assert the same for staged_decision_function since we have a single learner + for y_score in clf.staged_decision_function(X): + assert_allclose(y_score.sum(axis=1), 0, atol=1e-8) + + # With a single learner, we expect to have a decision function in + # {1, - 1 / (n_classes - 1)}. + assert set(np.unique(y_score)) == {1, -1 / (n_classes - 1)} + + clf.set_params(n_estimators=5).fit(X, y) + + y_score = clf.decision_function(X) + assert_allclose(y_score.sum(axis=1), 0, atol=1e-8) + + for y_score in clf.staged_decision_function(X): + assert_allclose(y_score.sum(axis=1), 0, atol=1e-8) + + +# TODO(1.8): remove +def test_deprecated_algorithm(): + adaboost_clf = AdaBoostClassifier(n_estimators=1, algorithm="SAMME") + with pytest.warns(FutureWarning, match="The parameter 'algorithm' is deprecated"): + adaboost_clf.fit(X, y_class) diff --git a/sklearn/exceptions.py b/sklearn/exceptions.py index 2ab7545705115..7db5a2ff0435f 100644 --- a/sklearn/exceptions.py +++ b/sklearn/exceptions.py @@ -1,21 +1,46 @@ -""" -The :mod:`sklearn.exceptions` module includes all custom warnings and error -classes used across scikit-learn. -""" - -from .utils.deprecation import deprecated - -__all__ = ['NotFittedError', - 'ChangedBehaviorWarning', - 'ConvergenceWarning', - 'DataConversionWarning', - 'DataDimensionalityWarning', - 'EfficiencyWarning', - 'FitFailedWarning', - 'NonBLASDotWarning', - 'SkipTestWarning', - 'UndefinedMetricWarning', - 'PositiveSpectrumWarning'] +"""Custom warnings and errors used across scikit-learn.""" + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +__all__ = [ + "ConvergenceWarning", + "DataConversionWarning", + "DataDimensionalityWarning", + "EfficiencyWarning", + "EstimatorCheckFailedWarning", + "FitFailedWarning", + "NotFittedError", + "PositiveSpectrumWarning", + "SkipTestWarning", + "UndefinedMetricWarning", + "UnsetMetadataPassedError", +] + + +class UnsetMetadataPassedError(ValueError): + """Exception class to raise if a metadata is passed which is not explicitly \ + requested (metadata=True) or not requested (metadata=False). + + .. versionadded:: 1.3 + + Parameters + ---------- + message : str + The message + + unrequested_params : dict + A dictionary of parameters and their values which are provided but not + requested. + + routed_params : dict + A dictionary of routed parameters. + """ + + def __init__(self, *, message, unrequested_params, routed_params): + super().__init__(message) + self.unrequested_params = unrequested_params + self.routed_params = routed_params class NotFittedError(ValueError, AttributeError): @@ -40,16 +65,6 @@ class NotFittedError(ValueError, AttributeError): """ -@deprecated("ChangedBehaviorWarning is deprecated in 0.24 and will be removed " - "in 1.1") -class ChangedBehaviorWarning(UserWarning): - """Warning class used to notify the user of any change in the behavior. - - .. versionchanged:: 0.18 - Moved from sklearn.base. - """ - - class ConvergenceWarning(UserWarning): """Custom warning to capture convergence problems @@ -113,19 +128,6 @@ class FitFailedWarning(RuntimeWarning): """ -@deprecated("NonBLASDotWarning is deprecated in 0.24 and will be removed in " - "1.1") -class NonBLASDotWarning(EfficiencyWarning): - """Warning used when the dot operation does not use BLAS. - - This warning is used to notify the user that BLAS was not used for dot - operation and hence the efficiency may be affected. - - .. versionchanged:: 0.18 - Moved from sklearn.utils.validation, extends EfficiencyWarning. - """ - - class SkipTestWarning(UserWarning): """Warning class used to notify the user of a test that was skipped. @@ -153,3 +155,95 @@ class PositiveSpectrumWarning(UserWarning): .. versionadded:: 0.22 """ + + +class InconsistentVersionWarning(UserWarning): + """Warning raised when an estimator is unpickled with an inconsistent version. + + Parameters + ---------- + estimator_name : str + Estimator name. + + current_sklearn_version : str + Current scikit-learn version. + + original_sklearn_version : str + Original scikit-learn version. + """ + + def __init__( + self, *, estimator_name, current_sklearn_version, original_sklearn_version + ): + self.estimator_name = estimator_name + self.current_sklearn_version = current_sklearn_version + self.original_sklearn_version = original_sklearn_version + + def __str__(self): + return ( + f"Trying to unpickle estimator {self.estimator_name} from version" + f" {self.original_sklearn_version} when " + f"using version {self.current_sklearn_version}. This might lead to breaking" + " code or " + "invalid results. Use at your own risk. " + "For more info please refer to:\n" + "https://scikit-learn.org/stable/model_persistence.html" + "#security-maintainability-limitations" + ) + + +class EstimatorCheckFailedWarning(UserWarning): + """Warning raised when an estimator check from the common tests fails. + + Parameters + ---------- + estimator : estimator object + Estimator instance for which the test failed. + + check_name : str + Name of the check that failed. + + exception : Exception + Exception raised by the failed check. + + status : str + Status of the check. + + expected_to_fail : bool + Whether the check was expected to fail. + + expected_to_fail_reason : str + Reason for the expected failure. + """ + + def __init__( + self, + *, + estimator, + check_name: str, + exception: Exception, + status: str, + expected_to_fail: bool, + expected_to_fail_reason: str, + ): + self.estimator = estimator + self.check_name = check_name + self.exception = exception + self.status = status + self.expected_to_fail = expected_to_fail + self.expected_to_fail_reason = expected_to_fail_reason + + def __repr__(self): + expected_to_fail_str = ( + f"Expected to fail: {self.expected_to_fail_reason}" + if self.expected_to_fail + else "Not expected to fail" + ) + return ( + f"Test {self.check_name} failed for estimator {self.estimator!r}.\n" + f"Expected to fail reason: {expected_to_fail_str}\n" + f"Exception: {self.exception}" + ) + + def __str__(self): + return self.__repr__() diff --git a/sklearn/experimental/__init__.py b/sklearn/experimental/__init__.py index 0effaf5b05fa0..593d247e5bc40 100644 --- a/sklearn/experimental/__init__.py +++ b/sklearn/experimental/__init__.py @@ -1,7 +1,10 @@ -""" -The :mod:`sklearn.experimental` module provides importable modules that enable -the use of experimental features or estimators. +"""Importable modules that enable the use of experimental features or estimators. + +.. warning:: -The features and estimators that are experimental aren't subject to -deprecation cycles. Use them at your own risks! + The features and estimators that are experimental aren't subject to + deprecation cycles. Use them at your own risks! """ + +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause diff --git a/sklearn/experimental/enable_halving_search_cv.py b/sklearn/experimental/enable_halving_search_cv.py index 91ec9585a6028..85f93b26459d0 100644 --- a/sklearn/experimental/enable_halving_search_cv.py +++ b/sklearn/experimental/enable_halving_search_cv.py @@ -19,17 +19,17 @@ flake8 to ignore the import, which appears as unused. """ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +from .. import model_selection from ..model_selection._search_successive_halving import ( + HalvingGridSearchCV, HalvingRandomSearchCV, - HalvingGridSearchCV ) -from .. import model_selection - # use settattr to avoid mypy errors when monkeypatching -setattr(model_selection, "HalvingRandomSearchCV", - HalvingRandomSearchCV) -setattr(model_selection, "HalvingGridSearchCV", - HalvingGridSearchCV) +setattr(model_selection, "HalvingRandomSearchCV", HalvingRandomSearchCV) +setattr(model_selection, "HalvingGridSearchCV", HalvingGridSearchCV) -model_selection.__all__ += ['HalvingRandomSearchCV', 'HalvingGridSearchCV'] +model_selection.__all__ += ["HalvingRandomSearchCV", "HalvingGridSearchCV"] diff --git a/sklearn/experimental/enable_hist_gradient_boosting.py b/sklearn/experimental/enable_hist_gradient_boosting.py index d7ceefbd58a2f..589348fe9bc21 100644 --- a/sklearn/experimental/enable_hist_gradient_boosting.py +++ b/sklearn/experimental/enable_hist_gradient_boosting.py @@ -1,36 +1,23 @@ -"""Enables histogram-based gradient boosting estimators. +"""This is now a no-op and can be safely removed from your code. -The API and results of these estimators might change without any deprecation -cycle. - -Importing this file dynamically sets the +It used to enable the use of :class:`~sklearn.ensemble.HistGradientBoostingClassifier` and -:class:`~sklearn.ensemble.HistGradientBoostingRegressor` as attributes of the -ensemble module:: - - >>> # explicitly require this experimental feature - >>> from sklearn.experimental import enable_hist_gradient_boosting # noqa - >>> # now you can import normally from ensemble - >>> from sklearn.ensemble import HistGradientBoostingClassifier - >>> from sklearn.ensemble import HistGradientBoostingRegressor - - -The ``# noqa`` comment comment can be removed: it just tells linters like -flake8 to ignore the import, which appears as unused. +:class:`~sklearn.ensemble.HistGradientBoostingRegressor` when they were still +:term:`experimental`, but these estimators are now stable and can be imported +normally from `sklearn.ensemble`. """ -from ..ensemble._hist_gradient_boosting.gradient_boosting import ( - HistGradientBoostingClassifier, - HistGradientBoostingRegressor -) +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause -from .. import ensemble +# Don't remove this file, we don't want to break users code just because the +# feature isn't experimental anymore. -# use settattr to avoid mypy errors when monkeypatching -setattr(ensemble, "HistGradientBoostingClassifier", - HistGradientBoostingClassifier) -setattr(ensemble, "HistGradientBoostingRegressor", - HistGradientBoostingRegressor) +import warnings -ensemble.__all__ += ['HistGradientBoostingClassifier', - 'HistGradientBoostingRegressor'] +warnings.warn( + "Since version 1.0, " + "it is not needed to import enable_hist_gradient_boosting anymore. " + "HistGradientBoostingClassifier and HistGradientBoostingRegressor are now " + "stable and can be normally imported from sklearn.ensemble." +) diff --git a/sklearn/experimental/enable_iterative_imputer.py b/sklearn/experimental/enable_iterative_imputer.py index d139bb86ce6aa..544e0d60eea28 100644 --- a/sklearn/experimental/enable_iterative_imputer.py +++ b/sklearn/experimental/enable_iterative_imputer.py @@ -12,9 +12,12 @@ >>> from sklearn.impute import IterativeImputer """ -from ..impute._iterative import IterativeImputer +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + from .. import impute +from ..impute._iterative import IterativeImputer # use settattr to avoid mypy errors when monkeypatching -setattr(impute, 'IterativeImputer', IterativeImputer) -impute.__all__ += ['IterativeImputer'] +setattr(impute, "IterativeImputer", IterativeImputer) +impute.__all__ += ["IterativeImputer"] diff --git a/sklearn/experimental/tests/test_enable_hist_gradient_boosting.py b/sklearn/experimental/tests/test_enable_hist_gradient_boosting.py index 06c0976d95a1f..a247bfd3f6428 100644 --- a/sklearn/experimental/tests/test_enable_hist_gradient_boosting.py +++ b/sklearn/experimental/tests/test_enable_hist_gradient_boosting.py @@ -2,44 +2,18 @@ import textwrap -from sklearn.utils._testing import assert_run_python_script +import pytest +from sklearn.utils._testing import assert_run_python_script_without_output +from sklearn.utils.fixes import _IS_WASM -def test_imports_strategies(): - # Make sure different import strategies work or fail as expected. - # Since Python caches the imported modules, we need to run a child process - # for every test case. Else, the tests would not be independent - # (manually removing the imports from the cache (sys.modules) is not - # recommended and can lead to many complications). - - good_import = """ - from sklearn.experimental import enable_hist_gradient_boosting - from sklearn.ensemble import GradientBoostingClassifier - from sklearn.ensemble import GradientBoostingRegressor - """ - assert_run_python_script(textwrap.dedent(good_import)) - - good_import_with_ensemble_first = """ - import sklearn.ensemble - from sklearn.experimental import enable_hist_gradient_boosting - from sklearn.ensemble import GradientBoostingClassifier - from sklearn.ensemble import GradientBoostingRegressor - """ - assert_run_python_script(textwrap.dedent(good_import_with_ensemble_first)) - - bad_imports = """ +@pytest.mark.xfail(_IS_WASM, reason="cannot start subprocess") +def test_import_raises_warning(): + code = """ import pytest - - with pytest.raises(ImportError): - from sklearn.ensemble import HistGradientBoostingClassifier - - with pytest.raises(ImportError): - from sklearn.ensemble._hist_gradient_boosting import ( - HistGradientBoostingClassifier) - - import sklearn.experimental - with pytest.raises(ImportError): - from sklearn.ensemble import HistGradientBoostingClassifier + with pytest.warns(UserWarning, match="it is not needed to import"): + from sklearn.experimental import enable_hist_gradient_boosting # noqa """ - assert_run_python_script(textwrap.dedent(bad_imports)) + pattern = "it is not needed to import enable_hist_gradient_boosting anymore" + assert_run_python_script_without_output(textwrap.dedent(code), pattern=pattern) diff --git a/sklearn/experimental/tests/test_enable_iterative_imputer.py b/sklearn/experimental/tests/test_enable_iterative_imputer.py index 21d57b37c519c..17e9dfa0d0376 100644 --- a/sklearn/experimental/tests/test_enable_iterative_imputer.py +++ b/sklearn/experimental/tests/test_enable_iterative_imputer.py @@ -2,9 +2,13 @@ import textwrap -from sklearn.utils._testing import assert_run_python_script +import pytest +from sklearn.utils._testing import assert_run_python_script_without_output +from sklearn.utils.fixes import _IS_WASM + +@pytest.mark.xfail(_IS_WASM, reason="cannot start subprocess") def test_imports_strategies(): # Make sure different import strategies work or fail as expected. @@ -12,28 +16,36 @@ def test_imports_strategies(): # for every test case. Else, the tests would not be independent # (manually removing the imports from the cache (sys.modules) is not # recommended and can lead to many complications). - + pattern = "IterativeImputer is experimental" good_import = """ from sklearn.experimental import enable_iterative_imputer from sklearn.impute import IterativeImputer """ - assert_run_python_script(textwrap.dedent(good_import)) + assert_run_python_script_without_output( + textwrap.dedent(good_import), pattern=pattern + ) good_import_with_ensemble_first = """ import sklearn.ensemble from sklearn.experimental import enable_iterative_imputer from sklearn.impute import IterativeImputer """ - assert_run_python_script(textwrap.dedent(good_import_with_ensemble_first)) + assert_run_python_script_without_output( + textwrap.dedent(good_import_with_ensemble_first), + pattern=pattern, + ) - bad_imports = """ + bad_imports = f""" import pytest - with pytest.raises(ImportError): + with pytest.raises(ImportError, match={pattern!r}): from sklearn.impute import IterativeImputer import sklearn.experimental - with pytest.raises(ImportError): + with pytest.raises(ImportError, match={pattern!r}): from sklearn.impute import IterativeImputer """ - assert_run_python_script(textwrap.dedent(bad_imports)) + assert_run_python_script_without_output( + textwrap.dedent(bad_imports), + pattern=pattern, + ) diff --git a/sklearn/experimental/tests/test_enable_successive_halving.py b/sklearn/experimental/tests/test_enable_successive_halving.py index b79670bb4141c..0ba273f94cc49 100644 --- a/sklearn/experimental/tests/test_enable_successive_halving.py +++ b/sklearn/experimental/tests/test_enable_successive_halving.py @@ -2,9 +2,13 @@ import textwrap -from sklearn.utils._testing import assert_run_python_script +import pytest +from sklearn.utils._testing import assert_run_python_script_without_output +from sklearn.utils.fixes import _IS_WASM + +@pytest.mark.xfail(_IS_WASM, reason="cannot start subprocess") def test_imports_strategies(): # Make sure different import strategies work or fail as expected. @@ -12,13 +16,15 @@ def test_imports_strategies(): # for every test case. Else, the tests would not be independent # (manually removing the imports from the cache (sys.modules) is not # recommended and can lead to many complications). - + pattern = "Halving(Grid|Random)SearchCV is experimental" good_import = """ from sklearn.experimental import enable_halving_search_cv from sklearn.model_selection import HalvingGridSearchCV from sklearn.model_selection import HalvingRandomSearchCV """ - assert_run_python_script(textwrap.dedent(good_import)) + assert_run_python_script_without_output( + textwrap.dedent(good_import), pattern=pattern + ) good_import_with_model_selection_first = """ import sklearn.model_selection @@ -26,18 +32,22 @@ def test_imports_strategies(): from sklearn.model_selection import HalvingGridSearchCV from sklearn.model_selection import HalvingRandomSearchCV """ - assert_run_python_script( - textwrap.dedent(good_import_with_model_selection_first) + assert_run_python_script_without_output( + textwrap.dedent(good_import_with_model_selection_first), + pattern=pattern, ) - bad_imports = """ + bad_imports = f""" import pytest - with pytest.raises(ImportError): + with pytest.raises(ImportError, match={pattern!r}): from sklearn.model_selection import HalvingGridSearchCV import sklearn.experimental - with pytest.raises(ImportError): - from sklearn.model_selection import HalvingGridSearchCV + with pytest.raises(ImportError, match={pattern!r}): + from sklearn.model_selection import HalvingRandomSearchCV """ - assert_run_python_script(textwrap.dedent(bad_imports)) + assert_run_python_script_without_output( + textwrap.dedent(bad_imports), + pattern=pattern, + ) diff --git a/sklearn/externals/_arff.py b/sklearn/externals/_arff.py index ccfbbc5e5e971..7c9d51d0702ff 100644 --- a/sklearn/externals/_arff.py +++ b/sklearn/externals/_arff.py @@ -32,7 +32,7 @@ ARFF (Attribute-Relation File Format) is an file format specially created for describe datasets which are commonly used for machine learning experiments and -softwares. This file format was created to be used in Weka, the best +software. This file format was created to be used in Weka, the best representative software for machine learning automated experiments. An ARFF file can be divided into two sections: header and data. The Header @@ -148,9 +148,8 @@ __version__ = '2.4.0' import re -import sys import csv -import typing +from typing import TYPE_CHECKING from typing import Optional, List, Dict, Any, Iterator, Union, Tuple # CONSTANTS =================================================================== @@ -173,7 +172,7 @@ ArffSparseDataType = Tuple[List, ...] -if typing.TYPE_CHECKING: +if TYPE_CHECKING: # typing_extensions is available when mypy is installed from typing_extensions import TypedDict @@ -297,7 +296,7 @@ def _parse_values(s): try: return {int(k): _unquote(v) for k, v in _RE_SPARSE_KEY_VALUES.findall(s)} - except ValueError as exc: + except ValueError: # an ARFF syntax error in sparse data for match in _RE_SPARSE_KEY_VALUES.finditer(s): if not match.group(1): diff --git a/sklearn/externals/_array_api_compat_vendor.py b/sklearn/externals/_array_api_compat_vendor.py new file mode 100644 index 0000000000000..38cefd2fe6f3f --- /dev/null +++ b/sklearn/externals/_array_api_compat_vendor.py @@ -0,0 +1,5 @@ +# DO NOT RENAME THIS FILE +# This is a hook for array_api_extra/_lib/_compat.py +# to co-vendor array_api_compat and potentially override its functions. + +from .array_api_compat import * # noqa: F403 diff --git a/sklearn/externals/_lobpcg.py b/sklearn/externals/_lobpcg.py deleted file mode 100644 index fe9f2e1c4aa93..0000000000000 --- a/sklearn/externals/_lobpcg.py +++ /dev/null @@ -1,727 +0,0 @@ -""" -scikit-learn copy of scipy/sparse/linalg/eigen/lobpcg/lobpcg.py v1.3.0 -to be deleted after scipy 1.3.0 becomes a dependency in scikit-lean -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -Locally Optimal Block Preconditioned Conjugate Gradient Method (LOBPCG). - -References ----------- -.. [1] A. V. Knyazev (2001), - Toward the Optimal Preconditioned Eigensolver: Locally Optimal - Block Preconditioned Conjugate Gradient Method. - SIAM Journal on Scientific Computing 23, no. 2, - pp. 517-541. http://dx.doi.org/10.1137/S1064827500366124 - -.. [2] A. V. Knyazev, I. Lashuk, M. E. Argentati, and E. Ovchinnikov (2007), - Block Locally Optimal Preconditioned Eigenvalue Xolvers (BLOPEX) - in hypre and PETSc. https://arxiv.org/abs/0705.2626 - -.. [3] A. V. Knyazev's C and MATLAB implementations: - https://bitbucket.org/joseroman/blopex -""" - -from __future__ import division, print_function, absolute_import -import numpy as np -from scipy.linalg import (inv, eigh, cho_factor, cho_solve, cholesky, orth, - LinAlgError) -from scipy.sparse.linalg import aslinearoperator - -__all__ = ['lobpcg'] - - -def bmat(*args, **kwargs): - import warnings - with warnings.catch_warnings(record=True): - warnings.filterwarnings( - 'ignore', '.*the matrix subclass is not the recommended way.*') - return np.bmat(*args, **kwargs) - - -def _save(ar, fileName): - # Used only when verbosity level > 10. - np.savetxt(fileName, ar) - - -def _report_nonhermitian(M, name): - """ - Report if `M` is not a hermitian matrix given its type. - """ - from scipy.linalg import norm - - md = M - M.T.conj() - - nmd = norm(md, 1) - tol = 10 * np.finfo(M.dtype).eps - tol = max(tol, tol * norm(M, 1)) - if nmd > tol: - print('matrix %s of the type %s is not sufficiently Hermitian:' - % (name, M.dtype)) - print('condition: %.e < %e' % (nmd, tol)) - - -def _as2d(ar): - """ - If the input array is 2D return it, if it is 1D, append a dimension, - making it a column vector. - """ - if ar.ndim == 2: - return ar - else: # Assume 1! - aux = np.array(ar, copy=False) - aux.shape = (ar.shape[0], 1) - return aux - - -def _makeOperator(operatorInput, expectedShape): - """Takes a dense numpy array or a sparse matrix or - a function and makes an operator performing matrix * blockvector - products.""" - if operatorInput is None: - return None - else: - operator = aslinearoperator(operatorInput) - - if operator.shape != expectedShape: - raise ValueError('operator has invalid shape') - - return operator - - -def _applyConstraints(blockVectorV, factYBY, blockVectorBY, blockVectorY): - """Changes blockVectorV in place.""" - YBV = np.dot(blockVectorBY.T.conj(), blockVectorV) - tmp = cho_solve(factYBY, YBV) - blockVectorV -= np.dot(blockVectorY, tmp) - - -def _b_orthonormalize(B, blockVectorV, blockVectorBV=None, retInvR=False): - """B-orthonormalize the given block vector using Cholesky.""" - normalization = blockVectorV.max(axis=0)+np.finfo(blockVectorV.dtype).eps - blockVectorV = blockVectorV / normalization - if blockVectorBV is None: - if B is not None: - blockVectorBV = B(blockVectorV) - else: - blockVectorBV = blockVectorV # Shared data!!! - else: - blockVectorBV = blockVectorBV / normalization - VBV = np.matmul(blockVectorV.T.conj(), blockVectorBV) - try: - # VBV is a Cholesky factor from now on... - VBV = cholesky(VBV, overwrite_a=True) - VBV = inv(VBV, overwrite_a=True) - blockVectorV = np.matmul(blockVectorV, VBV) - # blockVectorV = (cho_solve((VBV.T, True), blockVectorV.T)).T - if B is not None: - blockVectorBV = np.matmul(blockVectorBV, VBV) - # blockVectorBV = (cho_solve((VBV.T, True), blockVectorBV.T)).T - else: - blockVectorBV = None - except LinAlgError: - # raise ValueError('Cholesky has failed') - blockVectorV = None - blockVectorBV = None - VBV = None - - if retInvR: - return blockVectorV, blockVectorBV, VBV, normalization - else: - return blockVectorV, blockVectorBV - - -def _get_indx(_lambda, num, largest): - """Get `num` indices into `_lambda` depending on `largest` option.""" - ii = np.argsort(_lambda) - if largest: - ii = ii[:-num-1:-1] - else: - ii = ii[:num] - - return ii - - -def lobpcg(A, X, - B=None, M=None, Y=None, - tol=None, maxiter=20, - largest=True, verbosityLevel=0, - retLambdaHistory=False, retResidualNormsHistory=False): - """Locally Optimal Block Preconditioned Conjugate Gradient Method (LOBPCG) - - LOBPCG is a preconditioned eigensolver for large symmetric positive - definite (SPD) generalized eigenproblems. - - Parameters - ---------- - A : {sparse matrix, dense matrix, LinearOperator} - The symmetric linear operator of the problem, usually a - sparse matrix. Often called the "stiffness matrix". - X : ndarray, float32 or float64 - Initial approximation to the ``k`` eigenvectors (non-sparse). If `A` - has ``shape=(n,n)`` then `X` should have shape ``shape=(n,k)``. - B : {dense matrix, sparse matrix, LinearOperator}, optional - The right hand side operator in a generalized eigenproblem. - By default, ``B = Identity``. Often called the "mass matrix". - M : {dense matrix, sparse matrix, LinearOperator}, optional - Preconditioner to `A`; by default ``M = Identity``. - `M` should approximate the inverse of `A`. - Y : ndarray, float32 or float64, optional - n-by-sizeY matrix of constraints (non-sparse), sizeY < n - The iterations will be performed in the B-orthogonal complement - of the column-space of Y. Y must be full rank. - tol : scalar, optional - Solver tolerance (stopping criterion). - The default is ``tol=n*sqrt(eps)``. - maxiter : int, optional - Maximum number of iterations. The default is ``maxiter=min(n, 20)``. - largest : bool, optional - When True, solve for the largest eigenvalues, otherwise the smallest. - verbosityLevel : int, optional - Controls solver output. The default is ``verbosityLevel=0``. - retLambdaHistory : bool, optional - Whether to return eigenvalue history. Default is False. - retResidualNormsHistory : bool, optional - Whether to return history of residual norms. Default is False. - - Returns - ------- - w : ndarray - Array of ``k`` eigenvalues - v : ndarray - An array of ``k`` eigenvectors. `v` has the same shape as `X`. - lambdas : list of ndarray, optional - The eigenvalue history, if `retLambdaHistory` is True. - rnorms : list of ndarray, optional - The history of residual norms, if `retResidualNormsHistory` is True. - - Notes - ----- - If both ``retLambdaHistory`` and ``retResidualNormsHistory`` are True, - the return tuple has the following format - ``(lambda, V, lambda history, residual norms history)``. - - In the following ``n`` denotes the matrix size and ``m`` the number - of required eigenvalues (smallest or largest). - - The LOBPCG code internally solves eigenproblems of the size ``3m`` on every - iteration by calling the "standard" dense eigensolver, so if ``m`` is not - small enough compared to ``n``, it does not make sense to call the LOBPCG - code, but rather one should use the "standard" eigensolver, e.g. numpy or - scipy function in this case. - If one calls the LOBPCG algorithm for ``5m > n``, it will most likely break - internally, so the code tries to call the standard function instead. - - It is not that ``n`` should be large for the LOBPCG to work, but rather the - ratio ``n / m`` should be large. It you call LOBPCG with ``m=1`` - and ``n=10``, it works though ``n`` is small. The method is intended - for extremely large ``n / m``, see e.g., reference [28] in - https://arxiv.org/abs/0705.2626 - - The convergence speed depends basically on two factors: - - 1. How well relatively separated the seeking eigenvalues are from the rest - of the eigenvalues. One can try to vary ``m`` to make this better. - - 2. How well conditioned the problem is. This can be changed by using proper - preconditioning. For example, a rod vibration test problem (under tests - directory) is ill-conditioned for large ``n``, so convergence will be - slow, unless efficient preconditioning is used. For this specific - problem, a good simple preconditioner function would be a linear solve - for `A`, which is easy to code since A is tridiagonal. - - References - ---------- - .. [1] A. V. Knyazev (2001), - Toward the Optimal Preconditioned Eigensolver: Locally Optimal - Block Preconditioned Conjugate Gradient Method. - SIAM Journal on Scientific Computing 23, no. 2, - pp. 517-541. http://dx.doi.org/10.1137/S1064827500366124 - - .. [2] A. V. Knyazev, I. Lashuk, M. E. Argentati, and E. Ovchinnikov - (2007), Block Locally Optimal Preconditioned Eigenvalue Xolvers - (BLOPEX) in hypre and PETSc. https://arxiv.org/abs/0705.2626 - - .. [3] A. V. Knyazev's C and MATLAB implementations: - https://bitbucket.org/joseroman/blopex - - Examples - -------- - - Solve ``A x = lambda x`` with constraints and preconditioning. - - >>> import numpy as np - >>> from scipy.sparse import spdiags, issparse - >>> from scipy.sparse.linalg import lobpcg, LinearOperator - >>> n = 100 - >>> vals = np.arange(1, n + 1) - >>> A = spdiags(vals, 0, n, n) - >>> A.toarray() - array([[ 1., 0., 0., ..., 0., 0., 0.], - [ 0., 2., 0., ..., 0., 0., 0.], - [ 0., 0., 3., ..., 0., 0., 0.], - ..., - [ 0., 0., 0., ..., 98., 0., 0.], - [ 0., 0., 0., ..., 0., 99., 0.], - [ 0., 0., 0., ..., 0., 0., 100.]]) - - Constraints: - - >>> Y = np.eye(n, 3) - - Initial guess for eigenvectors, should have linearly independent - columns. Column dimension = number of requested eigenvalues. - - >>> X = np.random.rand(n, 3) - - Preconditioner in the inverse of A in this example: - - >>> invA = spdiags([1./vals], 0, n, n) - - The preconditiner must be defined by a function: - - >>> def precond( x ): - ... return invA @ x - - The argument x of the preconditioner function is a matrix inside `lobpcg`, - thus the use of matrix-matrix product ``@``. - - The preconditioner function is passed to lobpcg as a `LinearOperator`: - - >>> M = LinearOperator(matvec=precond, matmat=precond, - ... shape=(n, n), dtype=float) - - Let us now solve the eigenvalue problem for the matrix A: - - >>> eigenvalues, _ = lobpcg(A, X, Y=Y, M=M, largest=False) - >>> eigenvalues - array([4., 5., 6.]) - - Note that the vectors passed in Y are the eigenvectors of the 3 smallest - eigenvalues. The results returned are orthogonal to those. - - """ - blockVectorX = X - blockVectorY = Y - residualTolerance = tol - maxIterations = maxiter - - if blockVectorY is not None: - sizeY = blockVectorY.shape[1] - else: - sizeY = 0 - - # Block size. - if len(blockVectorX.shape) != 2: - raise ValueError('expected rank-2 array for argument X') - - n, sizeX = blockVectorX.shape - - if verbosityLevel: - aux = "Solving " - if B is None: - aux += "standard" - else: - aux += "generalized" - aux += " eigenvalue problem with" - if M is None: - aux += "out" - aux += " preconditioning\n\n" - aux += "matrix size %d\n" % n - aux += "block size %d\n\n" % sizeX - if blockVectorY is None: - aux += "No constraints\n\n" - else: - if sizeY > 1: - aux += "%d constraints\n\n" % sizeY - else: - aux += "%d constraint\n\n" % sizeY - print(aux) - - A = _makeOperator(A, (n, n)) - B = _makeOperator(B, (n, n)) - M = _makeOperator(M, (n, n)) - - if (n - sizeY) < (5 * sizeX): - # warn('The problem size is small compared to the block size.' \ - # ' Using dense eigensolver instead of LOBPCG.') - - sizeX = min(sizeX, n) - - if blockVectorY is not None: - raise NotImplementedError('The dense eigensolver ' - 'does not support constraints.') - - # Define the closed range of indices of eigenvalues to return. - if largest: - eigvals = (n - sizeX, n-1) - else: - eigvals = (0, sizeX-1) - - A_dense = A(np.eye(n, dtype=A.dtype)) - B_dense = None if B is None else B(np.eye(n, dtype=B.dtype)) - - vals, vecs = eigh(A_dense, B_dense, eigvals=eigvals, - check_finite=False) - if largest: - # Reverse order to be compatible with eigs() in 'LM' mode. - vals = vals[::-1] - vecs = vecs[:, ::-1] - - return vals, vecs - - if (residualTolerance is None) or (residualTolerance <= 0.0): - residualTolerance = np.sqrt(1e-15) * n - - # Apply constraints to X. - if blockVectorY is not None: - - if B is not None: - blockVectorBY = B(blockVectorY) - else: - blockVectorBY = blockVectorY - - # gramYBY is a dense array. - gramYBY = np.dot(blockVectorY.T.conj(), blockVectorBY) - try: - # gramYBY is a Cholesky factor from now on... - gramYBY = cho_factor(gramYBY) - except LinAlgError as e: - raise ValueError('cannot handle linearly dependent constraints') from e - - _applyConstraints(blockVectorX, gramYBY, blockVectorBY, blockVectorY) - - ## - # B-orthonormalize X. - blockVectorX, blockVectorBX = _b_orthonormalize(B, blockVectorX) - - ## - # Compute the initial Ritz vectors: solve the eigenproblem. - blockVectorAX = A(blockVectorX) - gramXAX = np.dot(blockVectorX.T.conj(), blockVectorAX) - - _lambda, eigBlockVector = eigh(gramXAX, check_finite=False) - ii = _get_indx(_lambda, sizeX, largest) - _lambda = _lambda[ii] - - eigBlockVector = np.asarray(eigBlockVector[:, ii]) - blockVectorX = np.dot(blockVectorX, eigBlockVector) - blockVectorAX = np.dot(blockVectorAX, eigBlockVector) - if B is not None: - blockVectorBX = np.dot(blockVectorBX, eigBlockVector) - - ## - # Active index set. - activeMask = np.ones((sizeX,), dtype=bool) - - lambdaHistory = [_lambda] - residualNormsHistory = [] - - previousBlockSize = sizeX - ident = np.eye(sizeX, dtype=A.dtype) - ident0 = np.eye(sizeX, dtype=A.dtype) - - ## - # Main iteration loop. - - blockVectorP = None # set during iteration - blockVectorAP = None - blockVectorBP = None - - iterationNumber = -1 - restart = True - explicitGramFlag = False - while iterationNumber < maxIterations: - iterationNumber += 1 - if verbosityLevel > 0: - print('iteration %d' % iterationNumber) - - if B is not None: - aux = blockVectorBX * _lambda[np.newaxis, :] - else: - aux = blockVectorX * _lambda[np.newaxis, :] - - blockVectorR = blockVectorAX - aux - - aux = np.sum(blockVectorR.conj() * blockVectorR, 0) - residualNorms = np.sqrt(aux) - - residualNormsHistory.append(residualNorms) - - ii = np.where(residualNorms > residualTolerance, True, False) - activeMask = activeMask & ii - if verbosityLevel > 2: - print(activeMask) - - currentBlockSize = activeMask.sum() - if currentBlockSize != previousBlockSize: - previousBlockSize = currentBlockSize - ident = np.eye(currentBlockSize, dtype=A.dtype) - - if currentBlockSize == 0: - break - - if verbosityLevel > 0: - print('current block size:', currentBlockSize) - print('eigenvalue:', _lambda) - print('residual norms:', residualNorms) - if verbosityLevel > 10: - print(eigBlockVector) - - activeBlockVectorR = _as2d(blockVectorR[:, activeMask]) - - if iterationNumber > 0: - activeBlockVectorP = _as2d(blockVectorP[:, activeMask]) - activeBlockVectorAP = _as2d(blockVectorAP[:, activeMask]) - if B is not None: - activeBlockVectorBP = _as2d(blockVectorBP[:, activeMask]) - - if M is not None: - # Apply preconditioner T to the active residuals. - activeBlockVectorR = M(activeBlockVectorR) - - ## - # Apply constraints to the preconditioned residuals. - if blockVectorY is not None: - _applyConstraints(activeBlockVectorR, - gramYBY, blockVectorBY, blockVectorY) - - ## - # B-orthogonalize the preconditioned residuals to X. - if B is not None: - activeBlockVectorR = activeBlockVectorR - \ - np.matmul(blockVectorX, - np.matmul(blockVectorBX.T.conj(), - activeBlockVectorR)) - else: - activeBlockVectorR = activeBlockVectorR - \ - np.matmul(blockVectorX, - np.matmul(blockVectorX.T.conj(), - activeBlockVectorR)) - - ## - # B-orthonormalize the preconditioned residuals. - aux = _b_orthonormalize(B, activeBlockVectorR) - activeBlockVectorR, activeBlockVectorBR = aux - - activeBlockVectorAR = A(activeBlockVectorR) - - if iterationNumber > 0: - if B is not None: - aux = _b_orthonormalize(B, activeBlockVectorP, - activeBlockVectorBP, retInvR=True) - activeBlockVectorP, activeBlockVectorBP, invR, normal = aux - else: - aux = _b_orthonormalize(B, activeBlockVectorP, retInvR=True) - activeBlockVectorP, _, invR, normal = aux - # Function _b_orthonormalize returns None if Cholesky fails - if activeBlockVectorP is not None: - activeBlockVectorAP = activeBlockVectorAP / normal - activeBlockVectorAP = np.dot(activeBlockVectorAP, invR) - restart = False - else: - restart = True - - ## - # Perform the Rayleigh Ritz Procedure: - # Compute symmetric Gram matrices: - - if activeBlockVectorAR.dtype == 'float32': - myeps = 1 - elif activeBlockVectorR.dtype == 'float32': - myeps = 1e-4 - else: - myeps = 1e-8 - - if residualNorms.max() > myeps and not explicitGramFlag: - explicitGramFlag = False - else: - # Once explicitGramFlag, forever explicitGramFlag. - explicitGramFlag = True - - # Shared memory assingments to simplify the code - if B is None: - blockVectorBX = blockVectorX - activeBlockVectorBR = activeBlockVectorR - if not restart: - activeBlockVectorBP = activeBlockVectorP - - # Common submatrices: - gramXAR = np.dot(blockVectorX.T.conj(), activeBlockVectorAR) - gramRAR = np.dot(activeBlockVectorR.T.conj(), activeBlockVectorAR) - - if explicitGramFlag: - gramRAR = (gramRAR + gramRAR.T.conj())/2 - gramXAX = np.dot(blockVectorX.T.conj(), blockVectorAX) - gramXAX = (gramXAX + gramXAX.T.conj())/2 - gramXBX = np.dot(blockVectorX.T.conj(), blockVectorBX) - gramRBR = np.dot(activeBlockVectorR.T.conj(), activeBlockVectorBR) - gramXBR = np.dot(blockVectorX.T.conj(), activeBlockVectorBR) - else: - gramXAX = np.diag(_lambda) - gramXBX = ident0 - gramRBR = ident - gramXBR = np.zeros((sizeX, currentBlockSize), dtype=A.dtype) - - def _handle_gramA_gramB_verbosity(gramA, gramB): - if verbosityLevel > 0: - _report_nonhermitian(gramA, 'gramA') - _report_nonhermitian(gramB, 'gramB') - if verbosityLevel > 10: - # Note: not documented, but leave it in here for now - np.savetxt('gramA.txt', gramA) - np.savetxt('gramB.txt', gramB) - - if not restart: - gramXAP = np.dot(blockVectorX.T.conj(), activeBlockVectorAP) - gramRAP = np.dot(activeBlockVectorR.T.conj(), activeBlockVectorAP) - gramPAP = np.dot(activeBlockVectorP.T.conj(), activeBlockVectorAP) - gramXBP = np.dot(blockVectorX.T.conj(), activeBlockVectorBP) - gramRBP = np.dot(activeBlockVectorR.T.conj(), activeBlockVectorBP) - if explicitGramFlag: - gramPAP = (gramPAP + gramPAP.T.conj())/2 - gramPBP = np.dot(activeBlockVectorP.T.conj(), - activeBlockVectorBP) - else: - gramPBP = ident - - gramA = bmat([[gramXAX, gramXAR, gramXAP], - [gramXAR.T.conj(), gramRAR, gramRAP], - [gramXAP.T.conj(), gramRAP.T.conj(), gramPAP]]) - gramB = bmat([[gramXBX, gramXBR, gramXBP], - [gramXBR.T.conj(), gramRBR, gramRBP], - [gramXBP.T.conj(), gramRBP.T.conj(), gramPBP]]) - - _handle_gramA_gramB_verbosity(gramA, gramB) - - try: - _lambda, eigBlockVector = eigh(gramA, gramB, - check_finite=False) - except LinAlgError: - # try again after dropping the direction vectors P from RR - restart = True - - if restart: - gramA = bmat([[gramXAX, gramXAR], - [gramXAR.T.conj(), gramRAR]]) - gramB = bmat([[gramXBX, gramXBR], - [gramXBR.T.conj(), gramRBR]]) - - _handle_gramA_gramB_verbosity(gramA, gramB) - - try: - _lambda, eigBlockVector = eigh(gramA, gramB, - check_finite=False) - except LinAlgError as e: - raise ValueError('eigh has failed in lobpcg iterations') from e - - ii = _get_indx(_lambda, sizeX, largest) - if verbosityLevel > 10: - print(ii) - print(_lambda) - - _lambda = _lambda[ii] - eigBlockVector = eigBlockVector[:, ii] - - lambdaHistory.append(_lambda) - - if verbosityLevel > 10: - print('lambda:', _lambda) -# # Normalize eigenvectors! -# aux = np.sum( eigBlockVector.conj() * eigBlockVector, 0 ) -# eigVecNorms = np.sqrt( aux ) -# eigBlockVector = eigBlockVector / eigVecNorms[np.newaxis, :] -# eigBlockVector, aux = _b_orthonormalize( B, eigBlockVector ) - - if verbosityLevel > 10: - print(eigBlockVector) - - # Compute Ritz vectors. - if B is not None: - if not restart: - eigBlockVectorX = eigBlockVector[:sizeX] - eigBlockVectorR = eigBlockVector[sizeX:sizeX+currentBlockSize] - eigBlockVectorP = eigBlockVector[sizeX+currentBlockSize:] - - pp = np.dot(activeBlockVectorR, eigBlockVectorR) - pp += np.dot(activeBlockVectorP, eigBlockVectorP) - - app = np.dot(activeBlockVectorAR, eigBlockVectorR) - app += np.dot(activeBlockVectorAP, eigBlockVectorP) - - bpp = np.dot(activeBlockVectorBR, eigBlockVectorR) - bpp += np.dot(activeBlockVectorBP, eigBlockVectorP) - else: - eigBlockVectorX = eigBlockVector[:sizeX] - eigBlockVectorR = eigBlockVector[sizeX:] - - pp = np.dot(activeBlockVectorR, eigBlockVectorR) - app = np.dot(activeBlockVectorAR, eigBlockVectorR) - bpp = np.dot(activeBlockVectorBR, eigBlockVectorR) - - if verbosityLevel > 10: - print(pp) - print(app) - print(bpp) - - blockVectorX = np.dot(blockVectorX, eigBlockVectorX) + pp - blockVectorAX = np.dot(blockVectorAX, eigBlockVectorX) + app - blockVectorBX = np.dot(blockVectorBX, eigBlockVectorX) + bpp - - blockVectorP, blockVectorAP, blockVectorBP = pp, app, bpp - - else: - if not restart: - eigBlockVectorX = eigBlockVector[:sizeX] - eigBlockVectorR = eigBlockVector[sizeX:sizeX+currentBlockSize] - eigBlockVectorP = eigBlockVector[sizeX+currentBlockSize:] - - pp = np.dot(activeBlockVectorR, eigBlockVectorR) - pp += np.dot(activeBlockVectorP, eigBlockVectorP) - - app = np.dot(activeBlockVectorAR, eigBlockVectorR) - app += np.dot(activeBlockVectorAP, eigBlockVectorP) - else: - eigBlockVectorX = eigBlockVector[:sizeX] - eigBlockVectorR = eigBlockVector[sizeX:] - - pp = np.dot(activeBlockVectorR, eigBlockVectorR) - app = np.dot(activeBlockVectorAR, eigBlockVectorR) - - if verbosityLevel > 10: - print(pp) - print(app) - - blockVectorX = np.dot(blockVectorX, eigBlockVectorX) + pp - blockVectorAX = np.dot(blockVectorAX, eigBlockVectorX) + app - - blockVectorP, blockVectorAP = pp, app - - if B is not None: - aux = blockVectorBX * _lambda[np.newaxis, :] - - else: - aux = blockVectorX * _lambda[np.newaxis, :] - - blockVectorR = blockVectorAX - aux - - aux = np.sum(blockVectorR.conj() * blockVectorR, 0) - residualNorms = np.sqrt(aux) - - # Future work: Need to add Postprocessing here: - # Making sure eigenvectors "exactly" satisfy the blockVectorY constrains? - # Making sure eigenvecotrs are "exactly" othonormalized by final "exact" RR - # Computing the actual true residuals - - if verbosityLevel > 0: - print('final eigenvalue:', _lambda) - print('final residual norms:', residualNorms) - - if retLambdaHistory: - if retResidualNormsHistory: - return _lambda, blockVectorX, lambdaHistory, residualNormsHistory - else: - return _lambda, blockVectorX, lambdaHistory - else: - if retResidualNormsHistory: - return _lambda, blockVectorX, residualNormsHistory - else: - return _lambda, blockVectorX diff --git a/sklearn/externals/_packaging/__init__.py b/sklearn/externals/_packaging/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/sklearn/externals/_packaging/_structures.py b/sklearn/externals/_packaging/_structures.py new file mode 100644 index 0000000000000..837e3a7946d70 --- /dev/null +++ b/sklearn/externals/_packaging/_structures.py @@ -0,0 +1,90 @@ +"""Vendoered from +https://github.com/pypa/packaging/blob/main/packaging/_structures.py +""" +# Copyright (c) Donald Stufft and individual contributors. +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +class InfinityType: + def __repr__(self) -> str: + return "Infinity" + + def __hash__(self) -> int: + return hash(repr(self)) + + def __lt__(self, other: object) -> bool: + return False + + def __le__(self, other: object) -> bool: + return False + + def __eq__(self, other: object) -> bool: + return isinstance(other, self.__class__) + + def __ne__(self, other: object) -> bool: + return not isinstance(other, self.__class__) + + def __gt__(self, other: object) -> bool: + return True + + def __ge__(self, other: object) -> bool: + return True + + def __neg__(self: object) -> "NegativeInfinityType": + return NegativeInfinity + + +Infinity = InfinityType() + + +class NegativeInfinityType: + def __repr__(self) -> str: + return "-Infinity" + + def __hash__(self) -> int: + return hash(repr(self)) + + def __lt__(self, other: object) -> bool: + return True + + def __le__(self, other: object) -> bool: + return True + + def __eq__(self, other: object) -> bool: + return isinstance(other, self.__class__) + + def __ne__(self, other: object) -> bool: + return not isinstance(other, self.__class__) + + def __gt__(self, other: object) -> bool: + return False + + def __ge__(self, other: object) -> bool: + return False + + def __neg__(self: object) -> InfinityType: + return Infinity + + +NegativeInfinity = NegativeInfinityType() diff --git a/sklearn/externals/_packaging/version.py b/sklearn/externals/_packaging/version.py new file mode 100644 index 0000000000000..0f1e5b833699c --- /dev/null +++ b/sklearn/externals/_packaging/version.py @@ -0,0 +1,535 @@ +"""Vendoered from +https://github.com/pypa/packaging/blob/main/packaging/version.py +""" +# Copyright (c) Donald Stufft and individual contributors. +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import collections +import itertools +import re +import warnings +from typing import Callable, Iterator, List, Optional, SupportsInt, Tuple, Union + +from ._structures import Infinity, InfinityType, NegativeInfinity, NegativeInfinityType + +__all__ = ["parse", "Version", "LegacyVersion", "InvalidVersion", "VERSION_PATTERN"] + +InfiniteTypes = Union[InfinityType, NegativeInfinityType] +PrePostDevType = Union[InfiniteTypes, Tuple[str, int]] +SubLocalType = Union[InfiniteTypes, int, str] +LocalType = Union[ + NegativeInfinityType, + Tuple[ + Union[ + SubLocalType, + Tuple[SubLocalType, str], + Tuple[NegativeInfinityType, SubLocalType], + ], + ..., + ], +] +CmpKey = Tuple[ + int, Tuple[int, ...], PrePostDevType, PrePostDevType, PrePostDevType, LocalType +] +LegacyCmpKey = Tuple[int, Tuple[str, ...]] +VersionComparisonMethod = Callable[ + [Union[CmpKey, LegacyCmpKey], Union[CmpKey, LegacyCmpKey]], bool +] + +_Version = collections.namedtuple( + "_Version", ["epoch", "release", "dev", "pre", "post", "local"] +) + + +def parse(version: str) -> Union["LegacyVersion", "Version"]: + """Parse the given version from a string to an appropriate class. + + Parameters + ---------- + version : str + Version in a string format, eg. "0.9.1" or "1.2.dev0". + + Returns + ------- + version : :class:`Version` object or a :class:`LegacyVersion` object + Returned class depends on the given version: if is a valid + PEP 440 version or a legacy version. + """ + try: + return Version(version) + except InvalidVersion: + return LegacyVersion(version) + + +class InvalidVersion(ValueError): + """ + An invalid version was found, users should refer to PEP 440. + """ + + +class _BaseVersion: + _key: Union[CmpKey, LegacyCmpKey] + + def __hash__(self) -> int: + return hash(self._key) + + # Please keep the duplicated `isinstance` check + # in the six comparisons hereunder + # unless you find a way to avoid adding overhead function calls. + def __lt__(self, other: "_BaseVersion") -> bool: + if not isinstance(other, _BaseVersion): + return NotImplemented + + return self._key < other._key + + def __le__(self, other: "_BaseVersion") -> bool: + if not isinstance(other, _BaseVersion): + return NotImplemented + + return self._key <= other._key + + def __eq__(self, other: object) -> bool: + if not isinstance(other, _BaseVersion): + return NotImplemented + + return self._key == other._key + + def __ge__(self, other: "_BaseVersion") -> bool: + if not isinstance(other, _BaseVersion): + return NotImplemented + + return self._key >= other._key + + def __gt__(self, other: "_BaseVersion") -> bool: + if not isinstance(other, _BaseVersion): + return NotImplemented + + return self._key > other._key + + def __ne__(self, other: object) -> bool: + if not isinstance(other, _BaseVersion): + return NotImplemented + + return self._key != other._key + + +class LegacyVersion(_BaseVersion): + def __init__(self, version: str) -> None: + self._version = str(version) + self._key = _legacy_cmpkey(self._version) + + warnings.warn( + "Creating a LegacyVersion has been deprecated and will be " + "removed in the next major release", + DeprecationWarning, + ) + + def __str__(self) -> str: + return self._version + + def __repr__(self) -> str: + return f"" + + @property + def public(self) -> str: + return self._version + + @property + def base_version(self) -> str: + return self._version + + @property + def epoch(self) -> int: + return -1 + + @property + def release(self) -> None: + return None + + @property + def pre(self) -> None: + return None + + @property + def post(self) -> None: + return None + + @property + def dev(self) -> None: + return None + + @property + def local(self) -> None: + return None + + @property + def is_prerelease(self) -> bool: + return False + + @property + def is_postrelease(self) -> bool: + return False + + @property + def is_devrelease(self) -> bool: + return False + + +_legacy_version_component_re = re.compile(r"(\d+ | [a-z]+ | \.| -)", re.VERBOSE) + +_legacy_version_replacement_map = { + "pre": "c", + "preview": "c", + "-": "final-", + "rc": "c", + "dev": "@", +} + + +def _parse_version_parts(s: str) -> Iterator[str]: + for part in _legacy_version_component_re.split(s): + part = _legacy_version_replacement_map.get(part, part) + + if not part or part == ".": + continue + + if part[:1] in "0123456789": + # pad for numeric comparison + yield part.zfill(8) + else: + yield "*" + part + + # ensure that alpha/beta/candidate are before final + yield "*final" + + +def _legacy_cmpkey(version: str) -> LegacyCmpKey: + + # We hardcode an epoch of -1 here. A PEP 440 version can only have a epoch + # greater than or equal to 0. This will effectively put the LegacyVersion, + # which uses the defacto standard originally implemented by setuptools, + # as before all PEP 440 versions. + epoch = -1 + + # This scheme is taken from pkg_resources.parse_version setuptools prior to + # it's adoption of the packaging library. + parts: List[str] = [] + for part in _parse_version_parts(version.lower()): + if part.startswith("*"): + # remove "-" before a prerelease tag + if part < "*final": + while parts and parts[-1] == "*final-": + parts.pop() + + # remove trailing zeros from each series of numeric parts + while parts and parts[-1] == "00000000": + parts.pop() + + parts.append(part) + + return epoch, tuple(parts) + + +# Deliberately not anchored to the start and end of the string, to make it +# easier for 3rd party code to reuse +VERSION_PATTERN = r""" + v? + (?: + (?:(?P[0-9]+)!)? # epoch + (?P[0-9]+(?:\.[0-9]+)*) # release segment + (?P
                                          # pre-release
+            [-_\.]?
+            (?P(a|b|c|rc|alpha|beta|pre|preview))
+            [-_\.]?
+            (?P[0-9]+)?
+        )?
+        (?P                                         # post release
+            (?:-(?P[0-9]+))
+            |
+            (?:
+                [-_\.]?
+                (?Ppost|rev|r)
+                [-_\.]?
+                (?P[0-9]+)?
+            )
+        )?
+        (?P                                          # dev release
+            [-_\.]?
+            (?Pdev)
+            [-_\.]?
+            (?P[0-9]+)?
+        )?
+    )
+    (?:\+(?P[a-z0-9]+(?:[-_\.][a-z0-9]+)*))?       # local version
+"""
+
+
+class Version(_BaseVersion):
+
+    _regex = re.compile(r"^\s*" + VERSION_PATTERN + r"\s*$", re.VERBOSE | re.IGNORECASE)
+
+    def __init__(self, version: str) -> None:
+
+        # Validate the version and parse it into pieces
+        match = self._regex.search(version)
+        if not match:
+            raise InvalidVersion(f"Invalid version: '{version}'")
+
+        # Store the parsed out pieces of the version
+        self._version = _Version(
+            epoch=int(match.group("epoch")) if match.group("epoch") else 0,
+            release=tuple(int(i) for i in match.group("release").split(".")),
+            pre=_parse_letter_version(match.group("pre_l"), match.group("pre_n")),
+            post=_parse_letter_version(
+                match.group("post_l"), match.group("post_n1") or match.group("post_n2")
+            ),
+            dev=_parse_letter_version(match.group("dev_l"), match.group("dev_n")),
+            local=_parse_local_version(match.group("local")),
+        )
+
+        # Generate a key which will be used for sorting
+        self._key = _cmpkey(
+            self._version.epoch,
+            self._version.release,
+            self._version.pre,
+            self._version.post,
+            self._version.dev,
+            self._version.local,
+        )
+
+    def __repr__(self) -> str:
+        return f""
+
+    def __str__(self) -> str:
+        parts = []
+
+        # Epoch
+        if self.epoch != 0:
+            parts.append(f"{self.epoch}!")
+
+        # Release segment
+        parts.append(".".join(str(x) for x in self.release))
+
+        # Pre-release
+        if self.pre is not None:
+            parts.append("".join(str(x) for x in self.pre))
+
+        # Post-release
+        if self.post is not None:
+            parts.append(f".post{self.post}")
+
+        # Development release
+        if self.dev is not None:
+            parts.append(f".dev{self.dev}")
+
+        # Local version segment
+        if self.local is not None:
+            parts.append(f"+{self.local}")
+
+        return "".join(parts)
+
+    @property
+    def epoch(self) -> int:
+        _epoch: int = self._version.epoch
+        return _epoch
+
+    @property
+    def release(self) -> Tuple[int, ...]:
+        _release: Tuple[int, ...] = self._version.release
+        return _release
+
+    @property
+    def pre(self) -> Optional[Tuple[str, int]]:
+        _pre: Optional[Tuple[str, int]] = self._version.pre
+        return _pre
+
+    @property
+    def post(self) -> Optional[int]:
+        return self._version.post[1] if self._version.post else None
+
+    @property
+    def dev(self) -> Optional[int]:
+        return self._version.dev[1] if self._version.dev else None
+
+    @property
+    def local(self) -> Optional[str]:
+        if self._version.local:
+            return ".".join(str(x) for x in self._version.local)
+        else:
+            return None
+
+    @property
+    def public(self) -> str:
+        return str(self).split("+", 1)[0]
+
+    @property
+    def base_version(self) -> str:
+        parts = []
+
+        # Epoch
+        if self.epoch != 0:
+            parts.append(f"{self.epoch}!")
+
+        # Release segment
+        parts.append(".".join(str(x) for x in self.release))
+
+        return "".join(parts)
+
+    @property
+    def is_prerelease(self) -> bool:
+        return self.dev is not None or self.pre is not None
+
+    @property
+    def is_postrelease(self) -> bool:
+        return self.post is not None
+
+    @property
+    def is_devrelease(self) -> bool:
+        return self.dev is not None
+
+    @property
+    def major(self) -> int:
+        return self.release[0] if len(self.release) >= 1 else 0
+
+    @property
+    def minor(self) -> int:
+        return self.release[1] if len(self.release) >= 2 else 0
+
+    @property
+    def micro(self) -> int:
+        return self.release[2] if len(self.release) >= 3 else 0
+
+
+def _parse_letter_version(
+    letter: str, number: Union[str, bytes, SupportsInt]
+) -> Optional[Tuple[str, int]]:
+
+    if letter:
+        # We consider there to be an implicit 0 in a pre-release if there is
+        # not a numeral associated with it.
+        if number is None:
+            number = 0
+
+        # We normalize any letters to their lower case form
+        letter = letter.lower()
+
+        # We consider some words to be alternate spellings of other words and
+        # in those cases we want to normalize the spellings to our preferred
+        # spelling.
+        if letter == "alpha":
+            letter = "a"
+        elif letter == "beta":
+            letter = "b"
+        elif letter in ["c", "pre", "preview"]:
+            letter = "rc"
+        elif letter in ["rev", "r"]:
+            letter = "post"
+
+        return letter, int(number)
+    if not letter and number:
+        # We assume if we are given a number, but we are not given a letter
+        # then this is using the implicit post release syntax (e.g. 1.0-1)
+        letter = "post"
+
+        return letter, int(number)
+
+    return None
+
+
+_local_version_separators = re.compile(r"[\._-]")
+
+
+def _parse_local_version(local: str) -> Optional[LocalType]:
+    """
+    Takes a string like abc.1.twelve and turns it into ("abc", 1, "twelve").
+    """
+    if local is not None:
+        return tuple(
+            part.lower() if not part.isdigit() else int(part)
+            for part in _local_version_separators.split(local)
+        )
+    return None
+
+
+def _cmpkey(
+    epoch: int,
+    release: Tuple[int, ...],
+    pre: Optional[Tuple[str, int]],
+    post: Optional[Tuple[str, int]],
+    dev: Optional[Tuple[str, int]],
+    local: Optional[Tuple[SubLocalType]],
+) -> CmpKey:
+
+    # When we compare a release version, we want to compare it with all of the
+    # trailing zeros removed. So we'll use a reverse the list, drop all the now
+    # leading zeros until we come to something non zero, then take the rest
+    # re-reverse it back into the correct order and make it a tuple and use
+    # that for our sorting key.
+    _release = tuple(
+        reversed(list(itertools.dropwhile(lambda x: x == 0, reversed(release))))
+    )
+
+    # We need to "trick" the sorting algorithm to put 1.0.dev0 before 1.0a0.
+    # We'll do this by abusing the pre segment, but we _only_ want to do this
+    # if there is not a pre or a post segment. If we have one of those then
+    # the normal sorting rules will handle this case correctly.
+    if pre is None and post is None and dev is not None:
+        _pre: PrePostDevType = NegativeInfinity
+    # Versions without a pre-release (except as noted above) should sort after
+    # those with one.
+    elif pre is None:
+        _pre = Infinity
+    else:
+        _pre = pre
+
+    # Versions without a post segment should sort before those with one.
+    if post is None:
+        _post: PrePostDevType = NegativeInfinity
+
+    else:
+        _post = post
+
+    # Versions without a development segment should sort after those with one.
+    if dev is None:
+        _dev: PrePostDevType = Infinity
+
+    else:
+        _dev = dev
+
+    if local is None:
+        # Versions without a local segment should sort before those with one.
+        _local: LocalType = NegativeInfinity
+    else:
+        # Versions with a local segment need that segment parsed to implement
+        # the sorting rules in PEP440.
+        # - Alpha numeric segments sort before numeric segments
+        # - Alpha numeric segments sort lexicographically
+        # - Numeric segments sort numerically
+        # - Shorter versions sort before longer versions when the prefixes
+        #   match exactly
+        _local = tuple(
+            (i, "") if isinstance(i, int) else (NegativeInfinity, i) for i in local
+        )
+
+    return epoch, _release, _pre, _post, _dev, _local
diff --git a/sklearn/externals/_pep562.py b/sklearn/externals/_pep562.py
deleted file mode 100644
index 86d374960b49f..0000000000000
--- a/sklearn/externals/_pep562.py
+++ /dev/null
@@ -1,58 +0,0 @@
-"""
-Backport of PEP 562.
-
-https://pypi.org/search/?q=pep562
-
-Licensed under MIT
-Copyright (c) 2018 Isaac Muse 
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
-documentation files (the "Software"), to deal in the Software without restriction, including without limitation
-the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
-and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all copies or substantial portions
-of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
-TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
-CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-IN THE SOFTWARE.
-"""
-from __future__ import unicode_literals
-import sys
-
-__all__ = ('Pep562',)
-
-
-class Pep562(object):
-    """
-    Backport of PEP 562 .
-
-    Wraps the module in a class that exposes the mechanics to override `__dir__` and `__getattr__`.
-    The given module will be searched for overrides of `__dir__` and `__getattr__` and use them when needed.
-    """
-
-    def __init__(self, name):
-        """Acquire `__getattr__` and `__dir__`, but only replace module for versions less than Python 3.7."""
-
-        self._module = sys.modules[name]
-        self._get_attr = getattr(self._module, '__getattr__', None)
-        self._get_dir = getattr(self._module, '__dir__', None)
-        sys.modules[name] = self
-
-    def __dir__(self):
-        """Return the overridden `dir` if one was provided, else apply `dir` to the module."""
-
-        return self._get_dir() if self._get_dir else dir(self._module)
-
-    def __getattr__(self, name):
-        """Attempt to retrieve the attribute from the module, and if missing, use the overridden function if present."""
-
-        try:
-            return getattr(self._module, name)
-        except AttributeError:
-            if self._get_attr:
-                return self._get_attr(name)
-            raise
diff --git a/sklearn/externals/_pilutil.py b/sklearn/externals/_pilutil.py
deleted file mode 100644
index 38de59a5dfe51..0000000000000
--- a/sklearn/externals/_pilutil.py
+++ /dev/null
@@ -1,504 +0,0 @@
-"""
-A collection of image utilities using the Python Imaging Library (PIL).
-
-This is a local version of utility functions from scipy that are wrapping PIL
-functionality. These functions are deprecated in scipy 1.0.0 and will be
-removed in scipy 1.2.0. Therefore, the functionality used in sklearn is copied
-here. This file is taken from scipy/misc/pilutil.py in scipy
-1.0.0. Modifications include: making this module importable if pillow is not
-installed, removal of DeprecationWarning, removal of functions scikit-learn
-does not need.
-
-Copyright (c) 2001, 2002 Enthought, Inc.
-All rights reserved.
-
-Copyright (c) 2003-2017 SciPy Developers.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-  a. Redistributions of source code must retain the above copyright notice,
-     this list of conditions and the following disclaimer.
-  b. Redistributions in binary form must reproduce the above copyright
-     notice, this list of conditions and the following disclaimer in the
-     documentation and/or other materials provided with the distribution.
-  c. Neither the name of Enthought nor the names of the SciPy Developers
-     may be used to endorse or promote products derived from this software
-     without specific prior written permission.
-
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS
-BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
-OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
-THE POSSIBILITY OF SUCH DAMAGE.
-"""
-from __future__ import division, print_function, absolute_import
-
-
-import numpy
-
-from numpy import (amin, amax, ravel, asarray, arange, ones, newaxis,
-                   transpose, iscomplexobj, uint8, issubdtype, array)
-
-# Modification of original scipy pilutil.py to make this module importable if
-# pillow is not installed. If pillow is not installed, functions will raise
-# ImportError when called.
-try:
-    try:
-        from PIL import Image
-    except ImportError:
-        import Image
-    pillow_installed = True
-    if not hasattr(Image, 'frombytes'):
-        Image.frombytes = Image.fromstring
-except ImportError:
-    pillow_installed = False
-
-__all__ = ['bytescale', 'imread', 'imsave', 'fromimage', 'toimage', 'imresize']
-
-
-PILLOW_ERROR_MESSAGE = (
-    "The Python Imaging Library (PIL) is required to load data "
-    "from jpeg files. Please refer to "
-    "https://pillow.readthedocs.io/en/stable/installation.html "
-    "for installing PIL."
-)
-
-
-def bytescale(data, cmin=None, cmax=None, high=255, low=0):
-    """
-    Byte scales an array (image).
-
-    Byte scaling means converting the input image to uint8 dtype and scaling
-    the range to ``(low, high)`` (default 0-255).
-    If the input image already has dtype uint8, no scaling is done.
-
-    This function is only available if Python Imaging Library (PIL) is installed.
-
-    Parameters
-    ----------
-    data : ndarray
-        PIL image data array.
-    cmin : scalar, default=None
-        Bias scaling of small values. Default is ``data.min()``.
-    cmax : scalar, default=None
-        Bias scaling of large values. Default is ``data.max()``.
-    high : scalar, default=None
-        Scale max value to `high`.  Default is 255.
-    low : scalar, default=None
-        Scale min value to `low`.  Default is 0.
-
-    Returns
-    -------
-    img_array : uint8 ndarray
-        The byte-scaled array.
-
-    Examples
-    --------
-    >>> import numpy as np
-    >>> from scipy.misc import bytescale
-    >>> img = np.array([[ 91.06794177,   3.39058326,  84.4221549 ],
-    ...                 [ 73.88003259,  80.91433048,   4.88878881],
-    ...                 [ 51.53875334,  34.45808177,  27.5873488 ]])
-    >>> bytescale(img)
-    array([[255,   0, 236],
-           [205, 225,   4],
-           [140,  90,  70]], dtype=uint8)
-    >>> bytescale(img, high=200, low=100)
-    array([[200, 100, 192],
-           [180, 188, 102],
-           [155, 135, 128]], dtype=uint8)
-    >>> bytescale(img, cmin=0, cmax=255)
-    array([[91,  3, 84],
-           [74, 81,  5],
-           [52, 34, 28]], dtype=uint8)
-
-    """
-    if data.dtype == uint8:
-        return data
-
-    if high > 255:
-        raise ValueError("`high` should be less than or equal to 255.")
-    if low < 0:
-        raise ValueError("`low` should be greater than or equal to 0.")
-    if high < low:
-        raise ValueError("`high` should be greater than or equal to `low`.")
-
-    if cmin is None:
-        cmin = data.min()
-    if cmax is None:
-        cmax = data.max()
-
-    cscale = cmax - cmin
-    if cscale < 0:
-        raise ValueError("`cmax` should be larger than `cmin`.")
-    elif cscale == 0:
-        cscale = 1
-
-    scale = float(high - low) / cscale
-    bytedata = (data - cmin) * scale + low
-    return (bytedata.clip(low, high) + 0.5).astype(uint8)
-
-
-def imread(name, flatten=False, mode=None):
-    """
-    Read an image from a file as an array.
-
-    This function is only available if Python Imaging Library (PIL) is installed.
-
-    Parameters
-    ----------
-    name : str or file object
-        The file name or file object to be read.
-    flatten : bool, default=False
-        If True, flattens the color layers into a single gray-scale layer.
-    mode : str, default=None
-        Mode to convert image to, e.g. ``'RGB'``.  See the Notes for more
-        details.
-
-    Returns
-    -------
-    imread : ndarray
-        The array obtained by reading the image.
-
-    Notes
-    -----
-    `imread` uses the Python Imaging Library (PIL) to read an image.
-    The following notes are from the PIL documentation.
-
-    `mode` can be one of the following strings:
-
-    * 'L' (8-bit pixels, black and white)
-    * 'P' (8-bit pixels, mapped to any other mode using a color palette)
-    * 'RGB' (3x8-bit pixels, true color)
-    * 'RGBA' (4x8-bit pixels, true color with transparency mask)
-    * 'CMYK' (4x8-bit pixels, color separation)
-    * 'YCbCr' (3x8-bit pixels, color video format)
-    * 'I' (32-bit signed integer pixels)
-    * 'F' (32-bit floating point pixels)
-
-    PIL also provides limited support for a few special modes, including
-    'LA' ('L' with alpha), 'RGBX' (true color with padding) and 'RGBa'
-    (true color with premultiplied alpha).
-
-    When translating a color image to black and white (mode 'L', 'I' or
-    'F'), the library uses the ITU-R 601-2 luma transform::
-
-        L = R * 299/1000 + G * 587/1000 + B * 114/1000
-
-    When `flatten` is True, the image is converted using mode 'F'.
-    When `mode` is not None and `flatten` is True, the image is first
-    converted according to `mode`, and the result is then flattened using
-    mode 'F'.
-
-    """
-    if not pillow_installed:
-        raise ImportError(PILLOW_ERROR_MESSAGE)
-
-    im = Image.open(name)
-    return fromimage(im, flatten=flatten, mode=mode)
-
-
-def imsave(name, arr, format=None):
-    """
-    Save an array as an image.
-
-    This function is only available if Python Imaging Library (PIL) is installed.
-
-    .. warning::
-
-        This function uses `bytescale` under the hood to rescale images to use
-        the full (0, 255) range if ``mode`` is one of ``None, 'L', 'P', 'l'``.
-        It will also cast data for 2-D images to ``uint32`` for ``mode=None``
-        (which is the default).
-
-    Parameters
-    ----------
-    name : str or file object
-        Output file name or file object.
-    arr : ndarray, MxN or MxNx3 or MxNx4
-        Array containing image values.  If the shape is ``MxN``, the array
-        represents a grey-level image.  Shape ``MxNx3`` stores the red, green
-        and blue bands along the last dimension.  An alpha layer may be
-        included, specified as the last colour band of an ``MxNx4`` array.
-    format : str, default=None
-        Image format. If omitted, the format to use is determined from the
-        file name extension. If a file object was used instead of a file name,
-        this parameter should always be used.
-
-    Examples
-    --------
-    Construct an array of gradient intensity values and save to file:
-
-    >>> import numpy as np
-    >>> from scipy.misc import imsave
-    >>> x = np.zeros((255, 255))
-    >>> x = np.zeros((255, 255), dtype=np.uint8)
-    >>> x[:] = np.arange(255)
-    >>> imsave('gradient.png', x)
-
-    Construct an array with three colour bands (R, G, B) and store to file:
-
-    >>> rgb = np.zeros((255, 255, 3), dtype=np.uint8)
-    >>> rgb[..., 0] = np.arange(255)
-    >>> rgb[..., 1] = 55
-    >>> rgb[..., 2] = 1 - np.arange(255)
-    >>> imsave('rgb_gradient.png', rgb)
-
-    """
-    im = toimage(arr, channel_axis=2)
-    if format is None:
-        im.save(name)
-    else:
-        im.save(name, format)
-    return
-
-
-def fromimage(im, flatten=False, mode=None):
-    """
-    Return a copy of a PIL image as a numpy array.
-
-    This function is only available if Python Imaging Library (PIL) is installed.
-
-    Parameters
-    ----------
-    im : PIL image
-        Input image.
-    flatten : bool, default=False
-        If true, convert the output to grey-scale.
-    mode : str, default=None
-        Mode to convert image to, e.g. ``'RGB'``.  See the Notes of the
-        `imread` docstring for more details.
-
-    Returns
-    -------
-    fromimage : ndarray
-        The different colour bands/channels are stored in the
-        third dimension, such that a grey-image is MxN, an
-        RGB-image MxNx3 and an RGBA-image MxNx4.
-
-    """
-    if not pillow_installed:
-        raise ImportError(PILLOW_ERROR_MESSAGE)
-
-    if not Image.isImageType(im):
-        raise TypeError("Input is not a PIL image.")
-
-    if mode is not None:
-        if mode != im.mode:
-            im = im.convert(mode)
-    elif im.mode == 'P':
-        # Mode 'P' means there is an indexed "palette".  If we leave the mode
-        # as 'P', then when we do `a = array(im)` below, `a` will be a 2-D
-        # containing the indices into the palette, and not a 3-D array
-        # containing the RGB or RGBA values.
-        if 'transparency' in im.info:
-            im = im.convert('RGBA')
-        else:
-            im = im.convert('RGB')
-
-    if flatten:
-        im = im.convert('F')
-    elif im.mode == '1':
-        # Workaround for crash in PIL. When im is 1-bit, the call array(im)
-        # can cause a seg. fault, or generate garbage. See
-        # https://github.com/scipy/scipy/issues/2138 and
-        # https://github.com/python-pillow/Pillow/issues/350.
-        #
-        # This converts im from a 1-bit image to an 8-bit image.
-        im = im.convert('L')
-
-    a = array(im)
-    return a
-
-_errstr = "Mode is unknown or incompatible with input array shape."
-
-
-def toimage(arr, high=255, low=0, cmin=None, cmax=None, pal=None,
-            mode=None, channel_axis=None):
-    """Takes a numpy array and returns a PIL image.
-
-    This function is only available if Python Imaging Library (PIL) is installed.
-
-    The mode of the PIL image depends on the array shape and the `pal` and
-    `mode` keywords.
-
-    For 2-D arrays, if `pal` is a valid (N,3) byte-array giving the RGB values
-    (from 0 to 255) then ``mode='P'``, otherwise ``mode='L'``, unless mode
-    is given as 'F' or 'I' in which case a float and/or integer array is made.
-
-    .. warning::
-
-        This function uses `bytescale` under the hood to rescale images to use
-        the full (0, 255) range if ``mode`` is one of ``None, 'L', 'P', 'l'``.
-        It will also cast data for 2-D images to ``uint32`` for ``mode=None``
-        (which is the default).
-
-    Notes
-    -----
-    For 3-D arrays, the `channel_axis` argument tells which dimension of the
-    array holds the channel data.
-
-    For 3-D arrays if one of the dimensions is 3, the mode is 'RGB'
-    by default or 'YCbCr' if selected.
-
-    The numpy array must be either 2 dimensional or 3 dimensional.
-
-    """
-    if not pillow_installed:
-        raise ImportError(PILLOW_ERROR_MESSAGE)
-
-    data = asarray(arr)
-    if iscomplexobj(data):
-        raise ValueError("Cannot convert a complex-valued array.")
-    shape = list(data.shape)
-    valid = len(shape) == 2 or ((len(shape) == 3) and
-                                ((3 in shape) or (4 in shape)))
-    if not valid:
-        raise ValueError("'arr' does not have a suitable array shape for "
-                         "any mode.")
-    if len(shape) == 2:
-        shape = (shape[1], shape[0])  # columns show up first
-        if mode == 'F':
-            data32 = data.astype(numpy.float32)
-            image = Image.frombytes(mode, shape, data32.tobytes())
-            return image
-        if mode in [None, 'L', 'P']:
-            bytedata = bytescale(data, high=high, low=low,
-                                 cmin=cmin, cmax=cmax)
-            image = Image.frombytes('L', shape, bytedata.tobytes())
-            if pal is not None:
-                image.putpalette(asarray(pal, dtype=uint8).tobytes())
-                # Becomes a mode='P' automagically.
-            elif mode == 'P':  # default gray-scale
-                pal = (arange(0, 256, 1, dtype=uint8)[:, newaxis] *
-                       ones((3,), dtype=uint8)[newaxis, :])
-                image.putpalette(asarray(pal, dtype=uint8).tobytes())
-            return image
-        if mode == '1':  # high input gives threshold for 1
-            bytedata = (data > high)
-            image = Image.frombytes('1', shape, bytedata.tobytes())
-            return image
-        if cmin is None:
-            cmin = amin(ravel(data))
-        if cmax is None:
-            cmax = amax(ravel(data))
-        data = (data*1.0 - cmin)*(high - low)/(cmax - cmin) + low
-        if mode == 'I':
-            data32 = data.astype(numpy.uint32)
-            image = Image.frombytes(mode, shape, data32.tobytes())
-        else:
-            raise ValueError(_errstr)
-        return image
-
-    # if here then 3-d array with a 3 or a 4 in the shape length.
-    # Check for 3 in datacube shape --- 'RGB' or 'YCbCr'
-    if channel_axis is None:
-        if (3 in shape):
-            ca = numpy.flatnonzero(asarray(shape) == 3)[0]
-        else:
-            ca = numpy.flatnonzero(asarray(shape) == 4)
-            if len(ca):
-                ca = ca[0]
-            else:
-                raise ValueError("Could not find channel dimension.")
-    else:
-        ca = channel_axis
-
-    numch = shape[ca]
-    if numch not in [3, 4]:
-        raise ValueError("Channel axis dimension is not valid.")
-
-    bytedata = bytescale(data, high=high, low=low, cmin=cmin, cmax=cmax)
-    if ca == 2:
-        strdata = bytedata.tobytes()
-        shape = (shape[1], shape[0])
-    elif ca == 1:
-        strdata = transpose(bytedata, (0, 2, 1)).tobytes()
-        shape = (shape[2], shape[0])
-    elif ca == 0:
-        strdata = transpose(bytedata, (1, 2, 0)).tobytes()
-        shape = (shape[2], shape[1])
-    if mode is None:
-        if numch == 3:
-            mode = 'RGB'
-        else:
-            mode = 'RGBA'
-
-    if mode not in ['RGB', 'RGBA', 'YCbCr', 'CMYK']:
-        raise ValueError(_errstr)
-
-    if mode in ['RGB', 'YCbCr']:
-        if numch != 3:
-            raise ValueError("Invalid array shape for mode.")
-    if mode in ['RGBA', 'CMYK']:
-        if numch != 4:
-            raise ValueError("Invalid array shape for mode.")
-
-    # Here we know data and mode is correct
-    image = Image.frombytes(mode, shape, strdata)
-    return image
-
-
-def imresize(arr, size, interp='bilinear', mode=None):
-    """
-    Resize an image.
-
-    This function is only available if Python Imaging Library (PIL) is installed.
-
-    .. warning::
-
-        This function uses `bytescale` under the hood to rescale images to use
-        the full (0, 255) range if ``mode`` is one of ``None, 'L', 'P', 'l'``.
-        It will also cast data for 2-D images to ``uint32`` for ``mode=None``
-        (which is the default).
-
-    Parameters
-    ----------
-    arr : ndarray
-        The array of image to be resized.
-    size : int, float or tuple
-        * int   - Percentage of current size.
-        * float - Fraction of current size.
-        * tuple - Size of the output image (height, width).
-
-    interp : str, default='bilinear'
-        Interpolation to use for re-sizing ('nearest', 'lanczos', 'bilinear',
-        'bicubic' or 'cubic').
-    mode : str, default=None
-        The PIL image mode ('P', 'L', etc.) to convert `arr` before resizing.
-        If ``mode=None`` (the default), 2-D images will be treated like
-        ``mode='L'``, i.e. casting to long integer.  For 3-D and 4-D arrays,
-        `mode` will be set to ``'RGB'`` and ``'RGBA'`` respectively.
-
-    Returns
-    -------
-    imresize : ndarray
-        The resized array of image.
-
-    See Also
-    --------
-    toimage : Implicitly used to convert `arr` according to `mode`.
-    scipy.ndimage.zoom : More generic implementation that does not use PIL.
-
-    """
-    im = toimage(arr, mode=mode)
-    ts = type(size)
-    if issubdtype(ts, numpy.signedinteger):
-        percent = size / 100.0
-        size = tuple((array(im.size)*percent).astype(int))
-    elif issubdtype(type(size), numpy.floating):
-        size = tuple((array(im.size)*size).astype(int))
-    else:
-        size = (size[1], size[0])
-    func = {'nearest': 0, 'lanczos': 1, 'bilinear': 2, 'bicubic': 3, 'cubic': 3}
-    imnew = im.resize(size, resample=func[interp])
-    return fromimage(imnew)
diff --git a/sklearn/externals/_scipy/__init__.py b/sklearn/externals/_scipy/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/sklearn/externals/_scipy/sparse/__init__.py b/sklearn/externals/_scipy/sparse/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/sklearn/externals/_scipy/sparse/csgraph/__init__.py b/sklearn/externals/_scipy/sparse/csgraph/__init__.py
new file mode 100644
index 0000000000000..15fc11fc81f20
--- /dev/null
+++ b/sklearn/externals/_scipy/sparse/csgraph/__init__.py
@@ -0,0 +1 @@
+from ._laplacian import laplacian
diff --git a/sklearn/externals/_scipy/sparse/csgraph/_laplacian.py b/sklearn/externals/_scipy/sparse/csgraph/_laplacian.py
new file mode 100644
index 0000000000000..34c816628ee73
--- /dev/null
+++ b/sklearn/externals/_scipy/sparse/csgraph/_laplacian.py
@@ -0,0 +1,557 @@
+"""
+This file is a copy of the scipy.sparse.csgraph._laplacian module from SciPy 1.12
+
+scipy.sparse.csgraph.laplacian supports sparse arrays only starting from Scipy 1.12,
+see https://github.com/scipy/scipy/pull/19156. This vendored file can be removed as
+soon as Scipy 1.12 becomes the minimum supported version.
+
+Laplacian of a compressed-sparse graph
+"""
+
+# SPDX-License-Identifier: BSD-3-Clause
+
+import numpy as np
+from scipy.sparse import issparse
+from scipy.sparse.linalg import LinearOperator
+
+
+###############################################################################
+# Graph laplacian
+def laplacian(
+    csgraph,
+    normed=False,
+    return_diag=False,
+    use_out_degree=False,
+    *,
+    copy=True,
+    form="array",
+    dtype=None,
+    symmetrized=False,
+):
+    """
+    Return the Laplacian of a directed graph.
+
+    Parameters
+    ----------
+    csgraph : array_like or sparse matrix, 2 dimensions
+        Compressed-sparse graph, with shape (N, N).
+    normed : bool, optional
+        If True, then compute symmetrically normalized Laplacian.
+        Default: False.
+    return_diag : bool, optional
+        If True, then also return an array related to vertex degrees.
+        Default: False.
+    use_out_degree : bool, optional
+        If True, then use out-degree instead of in-degree.
+        This distinction matters only if the graph is asymmetric.
+        Default: False.
+    copy : bool, optional
+        If False, then change `csgraph` in place if possible,
+        avoiding doubling the memory use.
+        Default: True, for backward compatibility.
+    form : 'array', or 'function', or 'lo'
+        Determines the format of the output Laplacian:
+
+        * 'array' is a numpy array;
+        * 'function' is a pointer to evaluating the Laplacian-vector
+          or Laplacian-matrix product;
+        * 'lo' results in the format of the `LinearOperator`.
+
+        Choosing 'function' or 'lo' always avoids doubling
+        the memory use, ignoring `copy` value.
+        Default: 'array', for backward compatibility.
+    dtype : None or one of numeric numpy dtypes, optional
+        The dtype of the output. If ``dtype=None``, the dtype of the
+        output matches the dtype of the input csgraph, except for
+        the case ``normed=True`` and integer-like csgraph, where
+        the output dtype is 'float' allowing accurate normalization,
+        but dramatically increasing the memory use.
+        Default: None, for backward compatibility.
+    symmetrized : bool, optional
+        If True, then the output Laplacian is symmetric/Hermitian.
+        The symmetrization is done by ``csgraph + csgraph.T.conj``
+        without dividing by 2 to preserve integer dtypes if possible
+        prior to the construction of the Laplacian.
+        The symmetrization will increase the memory footprint of
+        sparse matrices unless the sparsity pattern is symmetric or
+        `form` is 'function' or 'lo'.
+        Default: False, for backward compatibility.
+
+    Returns
+    -------
+    lap : ndarray, or sparse matrix, or `LinearOperator`
+        The N x N Laplacian of csgraph. It will be a NumPy array (dense)
+        if the input was dense, or a sparse matrix otherwise, or
+        the format of a function or `LinearOperator` if
+        `form` equals 'function' or 'lo', respectively.
+    diag : ndarray, optional
+        The length-N main diagonal of the Laplacian matrix.
+        For the normalized Laplacian, this is the array of square roots
+        of vertex degrees or 1 if the degree is zero.
+
+    Notes
+    -----
+    The Laplacian matrix of a graph is sometimes referred to as the
+    "Kirchhoff matrix" or just the "Laplacian", and is useful in many
+    parts of spectral graph theory.
+    In particular, the eigen-decomposition of the Laplacian can give
+    insight into many properties of the graph, e.g.,
+    is commonly used for spectral data embedding and clustering.
+
+    The constructed Laplacian doubles the memory use if ``copy=True`` and
+    ``form="array"`` which is the default.
+    Choosing ``copy=False`` has no effect unless ``form="array"``
+    or the matrix is sparse in the ``coo`` format, or dense array, except
+    for the integer input with ``normed=True`` that forces the float output.
+
+    Sparse input is reformatted into ``coo`` if ``form="array"``,
+    which is the default.
+
+    If the input adjacency matrix is not symmetric, the Laplacian is
+    also non-symmetric unless ``symmetrized=True`` is used.
+
+    Diagonal entries of the input adjacency matrix are ignored and
+    replaced with zeros for the purpose of normalization where ``normed=True``.
+    The normalization uses the inverse square roots of row-sums of the input
+    adjacency matrix, and thus may fail if the row-sums contain
+    negative or complex with a non-zero imaginary part values.
+
+    The normalization is symmetric, making the normalized Laplacian also
+    symmetric if the input csgraph was symmetric.
+
+    References
+    ----------
+    .. [1] Laplacian matrix. https://en.wikipedia.org/wiki/Laplacian_matrix
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from scipy.sparse import csgraph
+
+    Our first illustration is the symmetric graph
+
+    >>> G = np.arange(4) * np.arange(4)[:, np.newaxis]
+    >>> G
+    array([[0, 0, 0, 0],
+           [0, 1, 2, 3],
+           [0, 2, 4, 6],
+           [0, 3, 6, 9]])
+
+    and its symmetric Laplacian matrix
+
+    >>> csgraph.laplacian(G)
+    array([[ 0,  0,  0,  0],
+           [ 0,  5, -2, -3],
+           [ 0, -2,  8, -6],
+           [ 0, -3, -6,  9]])
+
+    The non-symmetric graph
+
+    >>> G = np.arange(9).reshape(3, 3)
+    >>> G
+    array([[0, 1, 2],
+           [3, 4, 5],
+           [6, 7, 8]])
+
+    has different row- and column sums, resulting in two varieties
+    of the Laplacian matrix, using an in-degree, which is the default
+
+    >>> L_in_degree = csgraph.laplacian(G)
+    >>> L_in_degree
+    array([[ 9, -1, -2],
+           [-3,  8, -5],
+           [-6, -7,  7]])
+
+    or alternatively an out-degree
+
+    >>> L_out_degree = csgraph.laplacian(G, use_out_degree=True)
+    >>> L_out_degree
+    array([[ 3, -1, -2],
+           [-3,  8, -5],
+           [-6, -7, 13]])
+
+    Constructing a symmetric Laplacian matrix, one can add the two as
+
+    >>> L_in_degree + L_out_degree.T
+    array([[ 12,  -4,  -8],
+            [ -4,  16, -12],
+            [ -8, -12,  20]])
+
+    or use the ``symmetrized=True`` option
+
+    >>> csgraph.laplacian(G, symmetrized=True)
+    array([[ 12,  -4,  -8],
+           [ -4,  16, -12],
+           [ -8, -12,  20]])
+
+    that is equivalent to symmetrizing the original graph
+
+    >>> csgraph.laplacian(G + G.T)
+    array([[ 12,  -4,  -8],
+           [ -4,  16, -12],
+           [ -8, -12,  20]])
+
+    The goal of normalization is to make the non-zero diagonal entries
+    of the Laplacian matrix to be all unit, also scaling off-diagonal
+    entries correspondingly. The normalization can be done manually, e.g.,
+
+    >>> G = np.array([[0, 1, 1], [1, 0, 1], [1, 1, 0]])
+    >>> L, d = csgraph.laplacian(G, return_diag=True)
+    >>> L
+    array([[ 2, -1, -1],
+           [-1,  2, -1],
+           [-1, -1,  2]])
+    >>> d
+    array([2, 2, 2])
+    >>> scaling = np.sqrt(d)
+    >>> scaling
+    array([1.41421356, 1.41421356, 1.41421356])
+    >>> (1/scaling)*L*(1/scaling)
+    array([[ 1. , -0.5, -0.5],
+           [-0.5,  1. , -0.5],
+           [-0.5, -0.5,  1. ]])
+
+    Or using ``normed=True`` option
+
+    >>> L, d = csgraph.laplacian(G, return_diag=True, normed=True)
+    >>> L
+    array([[ 1. , -0.5, -0.5],
+           [-0.5,  1. , -0.5],
+           [-0.5, -0.5,  1. ]])
+
+    which now instead of the diagonal returns the scaling coefficients
+
+    >>> d
+    array([1.41421356, 1.41421356, 1.41421356])
+
+    Zero scaling coefficients are substituted with 1s, where scaling
+    has thus no effect, e.g.,
+
+    >>> G = np.array([[0, 0, 0], [0, 0, 1], [0, 1, 0]])
+    >>> G
+    array([[0, 0, 0],
+           [0, 0, 1],
+           [0, 1, 0]])
+    >>> L, d = csgraph.laplacian(G, return_diag=True, normed=True)
+    >>> L
+    array([[ 0., -0., -0.],
+           [-0.,  1., -1.],
+           [-0., -1.,  1.]])
+    >>> d
+    array([1., 1., 1.])
+
+    Only the symmetric normalization is implemented, resulting
+    in a symmetric Laplacian matrix if and only if its graph is symmetric
+    and has all non-negative degrees, like in the examples above.
+
+    The output Laplacian matrix is by default a dense array or a sparse matrix
+    inferring its shape, format, and dtype from the input graph matrix:
+
+    >>> G = np.array([[0, 1, 1], [1, 0, 1], [1, 1, 0]]).astype(np.float32)
+    >>> G
+    array([[0., 1., 1.],
+           [1., 0., 1.],
+           [1., 1., 0.]], dtype=float32)
+    >>> csgraph.laplacian(G)
+    array([[ 2., -1., -1.],
+           [-1.,  2., -1.],
+           [-1., -1.,  2.]], dtype=float32)
+
+    but can alternatively be generated matrix-free as a LinearOperator:
+
+    >>> L = csgraph.laplacian(G, form="lo")
+    >>> L
+    <3x3 _CustomLinearOperator with dtype=float32>
+    >>> L(np.eye(3))
+    array([[ 2., -1., -1.],
+           [-1.,  2., -1.],
+           [-1., -1.,  2.]])
+
+    or as a lambda-function:
+
+    >>> L = csgraph.laplacian(G, form="function")
+    >>> L
+    . at 0x0000012AE6F5A598>
+    >>> L(np.eye(3))
+    array([[ 2., -1., -1.],
+           [-1.,  2., -1.],
+           [-1., -1.,  2.]])
+
+    The Laplacian matrix is used for
+    spectral data clustering and embedding
+    as well as for spectral graph partitioning.
+    Our final example illustrates the latter
+    for a noisy directed linear graph.
+
+    >>> from scipy.sparse import diags, random
+    >>> from scipy.sparse.linalg import lobpcg
+
+    Create a directed linear graph with ``N=35`` vertices
+    using a sparse adjacency matrix ``G``:
+
+    >>> N = 35
+    >>> G = diags(np.ones(N-1), 1, format="csr")
+
+    Fix a random seed ``rng`` and add a random sparse noise to the graph ``G``:
+
+    >>> rng = np.random.default_rng()
+    >>> G += 1e-2 * random(N, N, density=0.1, random_state=rng)
+
+    Set initial approximations for eigenvectors:
+
+    >>> X = rng.random((N, 2))
+
+    The constant vector of ones is always a trivial eigenvector
+    of the non-normalized Laplacian to be filtered out:
+
+    >>> Y = np.ones((N, 1))
+
+    Alternating (1) the sign of the graph weights allows determining
+    labels for spectral max- and min- cuts in a single loop.
+    Since the graph is undirected, the option ``symmetrized=True``
+    must be used in the construction of the Laplacian.
+    The option ``normed=True`` cannot be used in (2) for the negative weights
+    here as the symmetric normalization evaluates square roots.
+    The option ``form="lo"`` in (2) is matrix-free, i.e., guarantees
+    a fixed memory footprint and read-only access to the graph.
+    Calling the eigenvalue solver ``lobpcg`` (3) computes the Fiedler vector
+    that determines the labels as the signs of its components in (5).
+    Since the sign in an eigenvector is not deterministic and can flip,
+    we fix the sign of the first component to be always +1 in (4).
+
+    >>> for cut in ["max", "min"]:
+    ...     G = -G  # 1.
+    ...     L = csgraph.laplacian(G, symmetrized=True, form="lo")  # 2.
+    ...     _, eves = lobpcg(L, X, Y=Y, largest=False, tol=1e-3)  # 3.
+    ...     eves *= np.sign(eves[0, 0])  # 4.
+    ...     print(cut + "-cut labels:\\n", 1 * (eves[:, 0]>0))  # 5.
+    max-cut labels:
+    [1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1]
+    min-cut labels:
+    [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
+
+    As anticipated for a (slightly noisy) linear graph,
+    the max-cut strips all the edges of the graph coloring all
+    odd vertices into one color and all even vertices into another one,
+    while the balanced min-cut partitions the graph
+    in the middle by deleting a single edge.
+    Both determined partitions are optimal.
+    """
+    if csgraph.ndim != 2 or csgraph.shape[0] != csgraph.shape[1]:
+        raise ValueError("csgraph must be a square matrix or array")
+
+    if normed and (
+        np.issubdtype(csgraph.dtype, np.signedinteger)
+        or np.issubdtype(csgraph.dtype, np.uint)
+    ):
+        csgraph = csgraph.astype(np.float64)
+
+    if form == "array":
+        create_lap = _laplacian_sparse if issparse(csgraph) else _laplacian_dense
+    else:
+        create_lap = (
+            _laplacian_sparse_flo if issparse(csgraph) else _laplacian_dense_flo
+        )
+
+    degree_axis = 1 if use_out_degree else 0
+
+    lap, d = create_lap(
+        csgraph,
+        normed=normed,
+        axis=degree_axis,
+        copy=copy,
+        form=form,
+        dtype=dtype,
+        symmetrized=symmetrized,
+    )
+    if return_diag:
+        return lap, d
+    return lap
+
+
+def _setdiag_dense(m, d):
+    step = len(d) + 1
+    m.flat[::step] = d
+
+
+def _laplace(m, d):
+    return lambda v: v * d[:, np.newaxis] - m @ v
+
+
+def _laplace_normed(m, d, nd):
+    laplace = _laplace(m, d)
+    return lambda v: nd[:, np.newaxis] * laplace(v * nd[:, np.newaxis])
+
+
+def _laplace_sym(m, d):
+    return (
+        lambda v: v * d[:, np.newaxis]
+        - m @ v
+        - np.transpose(np.conjugate(np.transpose(np.conjugate(v)) @ m))
+    )
+
+
+def _laplace_normed_sym(m, d, nd):
+    laplace_sym = _laplace_sym(m, d)
+    return lambda v: nd[:, np.newaxis] * laplace_sym(v * nd[:, np.newaxis])
+
+
+def _linearoperator(mv, shape, dtype):
+    return LinearOperator(matvec=mv, matmat=mv, shape=shape, dtype=dtype)
+
+
+def _laplacian_sparse_flo(graph, normed, axis, copy, form, dtype, symmetrized):
+    # The keyword argument `copy` is unused and has no effect here.
+    del copy
+
+    if dtype is None:
+        dtype = graph.dtype
+
+    graph_sum = np.asarray(graph.sum(axis=axis)).ravel()
+    graph_diagonal = graph.diagonal()
+    diag = graph_sum - graph_diagonal
+    if symmetrized:
+        graph_sum += np.asarray(graph.sum(axis=1 - axis)).ravel()
+        diag = graph_sum - graph_diagonal - graph_diagonal
+
+    if normed:
+        isolated_node_mask = diag == 0
+        w = np.where(isolated_node_mask, 1, np.sqrt(diag))
+        if symmetrized:
+            md = _laplace_normed_sym(graph, graph_sum, 1.0 / w)
+        else:
+            md = _laplace_normed(graph, graph_sum, 1.0 / w)
+        if form == "function":
+            return md, w.astype(dtype, copy=False)
+        elif form == "lo":
+            m = _linearoperator(md, shape=graph.shape, dtype=dtype)
+            return m, w.astype(dtype, copy=False)
+        else:
+            raise ValueError(f"Invalid form: {form!r}")
+    else:
+        if symmetrized:
+            md = _laplace_sym(graph, graph_sum)
+        else:
+            md = _laplace(graph, graph_sum)
+        if form == "function":
+            return md, diag.astype(dtype, copy=False)
+        elif form == "lo":
+            m = _linearoperator(md, shape=graph.shape, dtype=dtype)
+            return m, diag.astype(dtype, copy=False)
+        else:
+            raise ValueError(f"Invalid form: {form!r}")
+
+
+def _laplacian_sparse(graph, normed, axis, copy, form, dtype, symmetrized):
+    # The keyword argument `form` is unused and has no effect here.
+    del form
+
+    if dtype is None:
+        dtype = graph.dtype
+
+    needs_copy = False
+    if graph.format in ("lil", "dok"):
+        m = graph.tocoo()
+    else:
+        m = graph
+        if copy:
+            needs_copy = True
+
+    if symmetrized:
+        m += m.T.conj()
+
+    w = np.asarray(m.sum(axis=axis)).ravel() - m.diagonal()
+    if normed:
+        m = m.tocoo(copy=needs_copy)
+        isolated_node_mask = w == 0
+        w = np.where(isolated_node_mask, 1, np.sqrt(w))
+        m.data /= w[m.row]
+        m.data /= w[m.col]
+        m.data *= -1
+        m.setdiag(1 - isolated_node_mask)
+    else:
+        if m.format == "dia":
+            m = m.copy()
+        else:
+            m = m.tocoo(copy=needs_copy)
+        m.data *= -1
+        m.setdiag(w)
+
+    return m.astype(dtype, copy=False), w.astype(dtype)
+
+
+def _laplacian_dense_flo(graph, normed, axis, copy, form, dtype, symmetrized):
+    if copy:
+        m = np.array(graph)
+    else:
+        m = np.asarray(graph)
+
+    if dtype is None:
+        dtype = m.dtype
+
+    graph_sum = m.sum(axis=axis)
+    graph_diagonal = m.diagonal()
+    diag = graph_sum - graph_diagonal
+    if symmetrized:
+        graph_sum += m.sum(axis=1 - axis)
+        diag = graph_sum - graph_diagonal - graph_diagonal
+
+    if normed:
+        isolated_node_mask = diag == 0
+        w = np.where(isolated_node_mask, 1, np.sqrt(diag))
+        if symmetrized:
+            md = _laplace_normed_sym(m, graph_sum, 1.0 / w)
+        else:
+            md = _laplace_normed(m, graph_sum, 1.0 / w)
+        if form == "function":
+            return md, w.astype(dtype, copy=False)
+        elif form == "lo":
+            m = _linearoperator(md, shape=graph.shape, dtype=dtype)
+            return m, w.astype(dtype, copy=False)
+        else:
+            raise ValueError(f"Invalid form: {form!r}")
+    else:
+        if symmetrized:
+            md = _laplace_sym(m, graph_sum)
+        else:
+            md = _laplace(m, graph_sum)
+        if form == "function":
+            return md, diag.astype(dtype, copy=False)
+        elif form == "lo":
+            m = _linearoperator(md, shape=graph.shape, dtype=dtype)
+            return m, diag.astype(dtype, copy=False)
+        else:
+            raise ValueError(f"Invalid form: {form!r}")
+
+
+def _laplacian_dense(graph, normed, axis, copy, form, dtype, symmetrized):
+    if form != "array":
+        raise ValueError(f'{form!r} must be "array"')
+
+    if dtype is None:
+        dtype = graph.dtype
+
+    if copy:
+        m = np.array(graph)
+    else:
+        m = np.asarray(graph)
+
+    if dtype is None:
+        dtype = m.dtype
+
+    if symmetrized:
+        m += m.T.conj()
+    np.fill_diagonal(m, 0)
+    w = m.sum(axis=axis)
+    if normed:
+        isolated_node_mask = w == 0
+        w = np.where(isolated_node_mask, 1, np.sqrt(w))
+        m /= w
+        m /= w[:, np.newaxis]
+        m *= -1
+        _setdiag_dense(m, 1 - isolated_node_mask)
+    else:
+        m *= -1
+        _setdiag_dense(m, w)
+
+    return m.astype(dtype, copy=False), w.astype(dtype, copy=False)
diff --git a/sklearn/externals/array_api_compat/LICENSE b/sklearn/externals/array_api_compat/LICENSE
new file mode 100644
index 0000000000000..ca9f2fee821ca
--- /dev/null
+++ b/sklearn/externals/array_api_compat/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Consortium for Python Data API Standards
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/sklearn/externals/array_api_compat/README.md b/sklearn/externals/array_api_compat/README.md
new file mode 100644
index 0000000000000..a3360988cbc1c
--- /dev/null
+++ b/sklearn/externals/array_api_compat/README.md
@@ -0,0 +1 @@
+Update this directory using maint_tools/vendor_array_api_compat.sh
diff --git a/sklearn/externals/array_api_compat/__init__.py b/sklearn/externals/array_api_compat/__init__.py
new file mode 100644
index 0000000000000..653cb40a37607
--- /dev/null
+++ b/sklearn/externals/array_api_compat/__init__.py
@@ -0,0 +1,22 @@
+"""
+NumPy Array API compatibility library
+
+This is a small wrapper around NumPy, CuPy, JAX, sparse and others that are
+compatible with the Array API standard https://data-apis.org/array-api/latest/.
+See also NEP 47 https://numpy.org/neps/nep-0047-array-api-standard.html.
+
+Unlike array_api_strict, this is not a strict minimal implementation of the
+Array API, but rather just an extension of the main NumPy namespace with
+changes needed to be compliant with the Array API. See
+https://numpy.org/doc/stable/reference/array_api.html for a full list of
+changes. In particular, unlike array_api_strict, this package does not use a
+separate Array object, but rather just uses numpy.ndarray directly.
+
+Library authors using the Array API may wish to test against array_api_strict
+to ensure they are not using functionality outside of the standard, but prefer
+this implementation for the default when working with NumPy arrays.
+
+"""
+__version__ = '1.12.0'
+
+from .common import *  # noqa: F401, F403
diff --git a/sklearn/externals/array_api_compat/_internal.py b/sklearn/externals/array_api_compat/_internal.py
new file mode 100644
index 0000000000000..cd8d939f36de2
--- /dev/null
+++ b/sklearn/externals/array_api_compat/_internal.py
@@ -0,0 +1,59 @@
+"""
+Internal helpers
+"""
+
+from collections.abc import Callable
+from functools import wraps
+from inspect import signature
+from types import ModuleType
+from typing import TypeVar
+
+_T = TypeVar("_T")
+
+
+def get_xp(xp: ModuleType) -> Callable[[Callable[..., _T]], Callable[..., _T]]:
+    """
+    Decorator to automatically replace xp with the corresponding array module.
+
+    Use like
+
+    import numpy as np
+
+    @get_xp(np)
+    def func(x, /, xp, kwarg=None):
+        return xp.func(x, kwarg=kwarg)
+
+    Note that xp must be a keyword argument and come after all non-keyword
+    arguments.
+
+    """
+
+    def inner(f: Callable[..., _T], /) -> Callable[..., _T]:
+        @wraps(f)
+        def wrapped_f(*args: object, **kwargs: object) -> object:
+            return f(*args, xp=xp, **kwargs)
+
+        sig = signature(f)
+        new_sig = sig.replace(
+            parameters=[par for i, par in sig.parameters.items() if i != "xp"]
+        )
+
+        if wrapped_f.__doc__ is None:
+            wrapped_f.__doc__ = f"""\
+Array API compatibility wrapper for {f.__name__}.
+
+See the corresponding documentation in NumPy/CuPy and/or the array API
+specification for more details.
+
+"""
+        wrapped_f.__signature__ = new_sig  # pyright: ignore[reportAttributeAccessIssue]
+        return wrapped_f  # pyright: ignore[reportReturnType]
+
+    return inner
+
+
+__all__ = ["get_xp"]
+
+
+def __dir__() -> list[str]:
+    return __all__
diff --git a/sklearn/externals/array_api_compat/common/__init__.py b/sklearn/externals/array_api_compat/common/__init__.py
new file mode 100644
index 0000000000000..8236080738175
--- /dev/null
+++ b/sklearn/externals/array_api_compat/common/__init__.py
@@ -0,0 +1 @@
+from ._helpers import *  # noqa: F403
diff --git a/sklearn/externals/array_api_compat/common/_aliases.py b/sklearn/externals/array_api_compat/common/_aliases.py
new file mode 100644
index 0000000000000..8ea9162a9edc8
--- /dev/null
+++ b/sklearn/externals/array_api_compat/common/_aliases.py
@@ -0,0 +1,727 @@
+"""
+These are functions that are just aliases of existing functions in NumPy.
+"""
+
+from __future__ import annotations
+
+import inspect
+from typing import TYPE_CHECKING, Any, NamedTuple, Optional, Sequence, cast
+
+from ._helpers import _check_device, array_namespace
+from ._helpers import device as _get_device
+from ._helpers import is_cupy_namespace as _is_cupy_namespace
+from ._typing import Array, Device, DType, Namespace
+
+if TYPE_CHECKING:
+    # TODO: import from typing (requires Python >=3.13)
+    from typing_extensions import TypeIs
+
+# These functions are modified from the NumPy versions.
+
+# Creation functions add the device keyword (which does nothing for NumPy and Dask)
+
+
+def arange(
+    start: float,
+    /,
+    stop: float | None = None,
+    step: float = 1,
+    *,
+    xp: Namespace,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    **kwargs: object,
+) -> Array:
+    _check_device(xp, device)
+    return xp.arange(start, stop=stop, step=step, dtype=dtype, **kwargs)
+
+
+def empty(
+    shape: int | tuple[int, ...],
+    xp: Namespace,
+    *,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    **kwargs: object,
+) -> Array:
+    _check_device(xp, device)
+    return xp.empty(shape, dtype=dtype, **kwargs)
+
+
+def empty_like(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    **kwargs: object,
+) -> Array:
+    _check_device(xp, device)
+    return xp.empty_like(x, dtype=dtype, **kwargs)
+
+
+def eye(
+    n_rows: int,
+    n_cols: int | None = None,
+    /,
+    *,
+    xp: Namespace,
+    k: int = 0,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    **kwargs: object,
+) -> Array:
+    _check_device(xp, device)
+    return xp.eye(n_rows, M=n_cols, k=k, dtype=dtype, **kwargs)
+
+
+def full(
+    shape: int | tuple[int, ...],
+    fill_value: complex,
+    xp: Namespace,
+    *,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    **kwargs: object,
+) -> Array:
+    _check_device(xp, device)
+    return xp.full(shape, fill_value, dtype=dtype, **kwargs)
+
+
+def full_like(
+    x: Array,
+    /,
+    fill_value: complex,
+    *,
+    xp: Namespace,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    **kwargs: object,
+) -> Array:
+    _check_device(xp, device)
+    return xp.full_like(x, fill_value, dtype=dtype, **kwargs)
+
+
+def linspace(
+    start: float,
+    stop: float,
+    /,
+    num: int,
+    *,
+    xp: Namespace,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    endpoint: bool = True,
+    **kwargs: object,
+) -> Array:
+    _check_device(xp, device)
+    return xp.linspace(start, stop, num, dtype=dtype, endpoint=endpoint, **kwargs)
+
+
+def ones(
+    shape: int | tuple[int, ...],
+    xp: Namespace,
+    *,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    **kwargs: object,
+) -> Array:
+    _check_device(xp, device)
+    return xp.ones(shape, dtype=dtype, **kwargs)
+
+
+def ones_like(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    **kwargs: object,
+) -> Array:
+    _check_device(xp, device)
+    return xp.ones_like(x, dtype=dtype, **kwargs)
+
+
+def zeros(
+    shape: int | tuple[int, ...],
+    xp: Namespace,
+    *,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    **kwargs: object,
+) -> Array:
+    _check_device(xp, device)
+    return xp.zeros(shape, dtype=dtype, **kwargs)
+
+
+def zeros_like(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    **kwargs: object,
+) -> Array:
+    _check_device(xp, device)
+    return xp.zeros_like(x, dtype=dtype, **kwargs)
+
+
+# np.unique() is split into four functions in the array API:
+# unique_all, unique_counts, unique_inverse, and unique_values (this is done
+# to remove polymorphic return types).
+
+# The functions here return namedtuples (np.unique() returns a normal
+# tuple).
+
+
+# Note that these named tuples aren't actually part of the standard namespace,
+# but I don't see any issue with exporting the names here regardless.
+class UniqueAllResult(NamedTuple):
+    values: Array
+    indices: Array
+    inverse_indices: Array
+    counts: Array
+
+
+class UniqueCountsResult(NamedTuple):
+    values: Array
+    counts: Array
+
+
+class UniqueInverseResult(NamedTuple):
+    values: Array
+    inverse_indices: Array
+
+
+def _unique_kwargs(xp: Namespace) -> dict[str, bool]:
+    # Older versions of NumPy and CuPy do not have equal_nan. Rather than
+    # trying to parse version numbers, just check if equal_nan is in the
+    # signature.
+    s = inspect.signature(xp.unique)
+    if "equal_nan" in s.parameters:
+        return {"equal_nan": False}
+    return {}
+
+
+def unique_all(x: Array, /, xp: Namespace) -> UniqueAllResult:
+    kwargs = _unique_kwargs(xp)
+    values, indices, inverse_indices, counts = xp.unique(
+        x,
+        return_counts=True,
+        return_index=True,
+        return_inverse=True,
+        **kwargs,
+    )
+    # np.unique() flattens inverse indices, but they need to share x's shape
+    # See https://github.com/numpy/numpy/issues/20638
+    inverse_indices = inverse_indices.reshape(x.shape)
+    return UniqueAllResult(
+        values,
+        indices,
+        inverse_indices,
+        counts,
+    )
+
+
+def unique_counts(x: Array, /, xp: Namespace) -> UniqueCountsResult:
+    kwargs = _unique_kwargs(xp)
+    res = xp.unique(
+        x, return_counts=True, return_index=False, return_inverse=False, **kwargs
+    )
+
+    return UniqueCountsResult(*res)
+
+
+def unique_inverse(x: Array, /, xp: Namespace) -> UniqueInverseResult:
+    kwargs = _unique_kwargs(xp)
+    values, inverse_indices = xp.unique(
+        x,
+        return_counts=False,
+        return_index=False,
+        return_inverse=True,
+        **kwargs,
+    )
+    # xp.unique() flattens inverse indices, but they need to share x's shape
+    # See https://github.com/numpy/numpy/issues/20638
+    inverse_indices = inverse_indices.reshape(x.shape)
+    return UniqueInverseResult(values, inverse_indices)
+
+
+def unique_values(x: Array, /, xp: Namespace) -> Array:
+    kwargs = _unique_kwargs(xp)
+    return xp.unique(
+        x,
+        return_counts=False,
+        return_index=False,
+        return_inverse=False,
+        **kwargs,
+    )
+
+
+# These functions have different keyword argument names
+
+
+def std(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    axis: int | tuple[int, ...] | None = None,
+    correction: float = 0.0,  # correction instead of ddof
+    keepdims: bool = False,
+    **kwargs: object,
+) -> Array:
+    return xp.std(x, axis=axis, ddof=correction, keepdims=keepdims, **kwargs)
+
+
+def var(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    axis: int | tuple[int, ...] | None = None,
+    correction: float = 0.0,  # correction instead of ddof
+    keepdims: bool = False,
+    **kwargs: object,
+) -> Array:
+    return xp.var(x, axis=axis, ddof=correction, keepdims=keepdims, **kwargs)
+
+
+# cumulative_sum is renamed from cumsum, and adds the include_initial keyword
+# argument
+
+
+def cumulative_sum(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    axis: int | None = None,
+    dtype: DType | None = None,
+    include_initial: bool = False,
+    **kwargs: object,
+) -> Array:
+    wrapped_xp = array_namespace(x)
+
+    # TODO: The standard is not clear about what should happen when x.ndim == 0.
+    if axis is None:
+        if x.ndim > 1:
+            raise ValueError(
+                "axis must be specified in cumulative_sum for more than one dimension"
+            )
+        axis = 0
+
+    res = xp.cumsum(x, axis=axis, dtype=dtype, **kwargs)
+
+    # np.cumsum does not support include_initial
+    if include_initial:
+        initial_shape = list(x.shape)
+        initial_shape[axis] = 1
+        res = xp.concatenate(
+            [
+                wrapped_xp.zeros(
+                    shape=initial_shape, dtype=res.dtype, device=_get_device(res)
+                ),
+                res,
+            ],
+            axis=axis,
+        )
+    return res
+
+
+def cumulative_prod(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    axis: int | None = None,
+    dtype: DType | None = None,
+    include_initial: bool = False,
+    **kwargs: object,
+) -> Array:
+    wrapped_xp = array_namespace(x)
+
+    if axis is None:
+        if x.ndim > 1:
+            raise ValueError(
+                "axis must be specified in cumulative_prod for more than one dimension"
+            )
+        axis = 0
+
+    res = xp.cumprod(x, axis=axis, dtype=dtype, **kwargs)
+
+    # np.cumprod does not support include_initial
+    if include_initial:
+        initial_shape = list(x.shape)
+        initial_shape[axis] = 1
+        res = xp.concatenate(
+            [
+                wrapped_xp.ones(
+                    shape=initial_shape, dtype=res.dtype, device=_get_device(res)
+                ),
+                res,
+            ],
+            axis=axis,
+        )
+    return res
+
+
+# The min and max argument names in clip are different and not optional in numpy, and type
+# promotion behavior is different.
+def clip(
+    x: Array,
+    /,
+    min: float | Array | None = None,
+    max: float | Array | None = None,
+    *,
+    xp: Namespace,
+    # TODO: np.clip has other ufunc kwargs
+    out: Array | None = None,
+) -> Array:
+    def _isscalar(a: object) -> TypeIs[int | float | None]:
+        return isinstance(a, (int, float, type(None)))
+
+    min_shape = () if _isscalar(min) else min.shape
+    max_shape = () if _isscalar(max) else max.shape
+
+    wrapped_xp = array_namespace(x)
+
+    result_shape = xp.broadcast_shapes(x.shape, min_shape, max_shape)
+
+    # np.clip does type promotion but the array API clip requires that the
+    # output have the same dtype as x. We do this instead of just downcasting
+    # the result of xp.clip() to handle some corner cases better (e.g.,
+    # avoiding uint64 -> float64 promotion).
+
+    # Note: cases where min or max overflow (integer) or round (float) in the
+    # wrong direction when downcasting to x.dtype are unspecified. This code
+    # just does whatever NumPy does when it downcasts in the assignment, but
+    # other behavior could be preferred, especially for integers. For example,
+    # this code produces:
+
+    # >>> clip(asarray(0, dtype=int8), asarray(128, dtype=int16), None)
+    # -128
+
+    # but an answer of 0 might be preferred. See
+    # https://github.com/numpy/numpy/issues/24976 for more discussion on this issue.
+
+    # At least handle the case of Python integers correctly (see
+    # https://github.com/numpy/numpy/pull/26892).
+    if wrapped_xp.isdtype(x.dtype, "integral"):
+        if type(min) is int and min <= wrapped_xp.iinfo(x.dtype).min:
+            min = None
+        if type(max) is int and max >= wrapped_xp.iinfo(x.dtype).max:
+            max = None
+
+    dev = _get_device(x)
+    if out is None:
+        out = wrapped_xp.empty(result_shape, dtype=x.dtype, device=dev)
+    assert out is not None  # workaround for a type-narrowing issue in pyright
+    out[()] = x
+
+    if min is not None:
+        a = wrapped_xp.asarray(min, dtype=x.dtype, device=dev)
+        a = xp.broadcast_to(a, result_shape)
+        ia = (out < a) | xp.isnan(a)
+        out[ia] = a[ia]
+
+    if max is not None:
+        b = wrapped_xp.asarray(max, dtype=x.dtype, device=dev)
+        b = xp.broadcast_to(b, result_shape)
+        ib = (out > b) | xp.isnan(b)
+        out[ib] = b[ib]
+
+    # Return a scalar for 0-D
+    return out[()]
+
+
+# Unlike transpose(), the axes argument to permute_dims() is required.
+def permute_dims(x: Array, /, axes: tuple[int, ...], xp: Namespace) -> Array:
+    return xp.transpose(x, axes)
+
+
+# np.reshape calls the keyword argument 'newshape' instead of 'shape'
+def reshape(
+    x: Array,
+    /,
+    shape: tuple[int, ...],
+    xp: Namespace,
+    *,
+    copy: Optional[bool] = None,
+    **kwargs: object,
+) -> Array:
+    if copy is True:
+        x = x.copy()
+    elif copy is False:
+        y = x.view()
+        y.shape = shape
+        return y
+    return xp.reshape(x, shape, **kwargs)
+
+
+# The descending keyword is new in sort and argsort, and 'kind' replaced with
+# 'stable'
+def argsort(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    axis: int = -1,
+    descending: bool = False,
+    stable: bool = True,
+    **kwargs: object,
+) -> Array:
+    # Note: this keyword argument is different, and the default is different.
+    # We set it in kwargs like this because numpy.sort uses kind='quicksort'
+    # as the default whereas cupy.sort uses kind=None.
+    if stable:
+        kwargs["kind"] = "stable"
+    if not descending:
+        res = xp.argsort(x, axis=axis, **kwargs)
+    else:
+        # As NumPy has no native descending sort, we imitate it here. Note that
+        # simply flipping the results of xp.argsort(x, ...) would not
+        # respect the relative order like it would in native descending sorts.
+        res = xp.flip(
+            xp.argsort(xp.flip(x, axis=axis), axis=axis, **kwargs),
+            axis=axis,
+        )
+        # Rely on flip()/argsort() to validate axis
+        normalised_axis = axis if axis >= 0 else x.ndim + axis
+        max_i = x.shape[normalised_axis] - 1
+        res = max_i - res
+    return res
+
+
+def sort(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    axis: int = -1,
+    descending: bool = False,
+    stable: bool = True,
+    **kwargs: object,
+) -> Array:
+    # Note: this keyword argument is different, and the default is different.
+    # We set it in kwargs like this because numpy.sort uses kind='quicksort'
+    # as the default whereas cupy.sort uses kind=None.
+    if stable:
+        kwargs["kind"] = "stable"
+    res = xp.sort(x, axis=axis, **kwargs)
+    if descending:
+        res = xp.flip(res, axis=axis)
+    return res
+
+
+# nonzero should error for zero-dimensional arrays
+def nonzero(x: Array, /, xp: Namespace, **kwargs: object) -> tuple[Array, ...]:
+    if x.ndim == 0:
+        raise ValueError("nonzero() does not support zero-dimensional arrays")
+    return xp.nonzero(x, **kwargs)
+
+
+# ceil, floor, and trunc return integers for integer inputs
+
+
+def ceil(x: Array, /, xp: Namespace, **kwargs: object) -> Array:
+    if xp.issubdtype(x.dtype, xp.integer):
+        return x
+    return xp.ceil(x, **kwargs)
+
+
+def floor(x: Array, /, xp: Namespace, **kwargs: object) -> Array:
+    if xp.issubdtype(x.dtype, xp.integer):
+        return x
+    return xp.floor(x, **kwargs)
+
+
+def trunc(x: Array, /, xp: Namespace, **kwargs: object) -> Array:
+    if xp.issubdtype(x.dtype, xp.integer):
+        return x
+    return xp.trunc(x, **kwargs)
+
+
+# linear algebra functions
+
+
+def matmul(x1: Array, x2: Array, /, xp: Namespace, **kwargs: object) -> Array:
+    return xp.matmul(x1, x2, **kwargs)
+
+
+# Unlike transpose, matrix_transpose only transposes the last two axes.
+def matrix_transpose(x: Array, /, xp: Namespace) -> Array:
+    if x.ndim < 2:
+        raise ValueError("x must be at least 2-dimensional for matrix_transpose")
+    return xp.swapaxes(x, -1, -2)
+
+
+def tensordot(
+    x1: Array,
+    x2: Array,
+    /,
+    xp: Namespace,
+    *,
+    axes: int | tuple[Sequence[int], Sequence[int]] = 2,
+    **kwargs: object,
+) -> Array:
+    return xp.tensordot(x1, x2, axes=axes, **kwargs)
+
+
+def vecdot(x1: Array, x2: Array, /, xp: Namespace, *, axis: int = -1) -> Array:
+    if x1.shape[axis] != x2.shape[axis]:
+        raise ValueError("x1 and x2 must have the same size along the given axis")
+
+    if hasattr(xp, "broadcast_tensors"):
+        _broadcast = xp.broadcast_tensors
+    else:
+        _broadcast = xp.broadcast_arrays
+
+    x1_ = xp.moveaxis(x1, axis, -1)
+    x2_ = xp.moveaxis(x2, axis, -1)
+    x1_, x2_ = _broadcast(x1_, x2_)
+
+    res = xp.conj(x1_[..., None, :]) @ x2_[..., None]
+    return res[..., 0, 0]
+
+
+# isdtype is a new function in the 2022.12 array API specification.
+
+
+def isdtype(
+    dtype: DType,
+    kind: DType | str | tuple[DType | str, ...],
+    xp: Namespace,
+    *,
+    _tuple: bool = True,  # Disallow nested tuples
+) -> bool:
+    """
+    Returns a boolean indicating whether a provided dtype is of a specified data type ``kind``.
+
+    Note that outside of this function, this compat library does not yet fully
+    support complex numbers.
+
+    See
+    https://data-apis.org/array-api/latest/API_specification/generated/array_api.isdtype.html
+    for more details
+    """
+    if isinstance(kind, tuple) and _tuple:
+        return any(
+            isdtype(dtype, k, xp, _tuple=False)
+            for k in cast("tuple[DType | str, ...]", kind)
+        )
+    elif isinstance(kind, str):
+        if kind == "bool":
+            return dtype == xp.bool_
+        elif kind == "signed integer":
+            return xp.issubdtype(dtype, xp.signedinteger)
+        elif kind == "unsigned integer":
+            return xp.issubdtype(dtype, xp.unsignedinteger)
+        elif kind == "integral":
+            return xp.issubdtype(dtype, xp.integer)
+        elif kind == "real floating":
+            return xp.issubdtype(dtype, xp.floating)
+        elif kind == "complex floating":
+            return xp.issubdtype(dtype, xp.complexfloating)
+        elif kind == "numeric":
+            return xp.issubdtype(dtype, xp.number)
+        else:
+            raise ValueError(f"Unrecognized data type kind: {kind!r}")
+    else:
+        # This will allow things that aren't required by the spec, like
+        # isdtype(np.float64, float) or isdtype(np.int64, 'l'). Should we be
+        # more strict here to match the type annotation? Note that the
+        # array_api_strict implementation will be very strict.
+        return dtype == kind
+
+
+# unstack is a new function in the 2023.12 array API standard
+def unstack(x: Array, /, xp: Namespace, *, axis: int = 0) -> tuple[Array, ...]:
+    if x.ndim == 0:
+        raise ValueError("Input array must be at least 1-d.")
+    return tuple(xp.moveaxis(x, axis, 0))
+
+
+# numpy 1.26 does not use the standard definition for sign on complex numbers
+
+
+def sign(x: Array, /, xp: Namespace, **kwargs: object) -> Array:
+    if isdtype(x.dtype, "complex floating", xp=xp):
+        out = (x / xp.abs(x, **kwargs))[...]
+        # sign(0) = 0 but the above formula would give nan
+        out[x == 0j] = 0j
+    else:
+        out = xp.sign(x, **kwargs)
+    # CuPy sign() does not propagate nans. See
+    # https://github.com/data-apis/array-api-compat/issues/136
+    if _is_cupy_namespace(xp) and isdtype(x.dtype, "real floating", xp=xp):
+        out[xp.isnan(x)] = xp.nan
+    return out[()]
+
+
+def finfo(type_: DType | Array, /, xp: Namespace) -> Any:
+    # It is surprisingly difficult to recognize a dtype apart from an array.
+    # np.int64 is not the same as np.asarray(1).dtype!
+    try:
+        return xp.finfo(type_)
+    except (ValueError, TypeError):
+        return xp.finfo(type_.dtype)
+
+
+def iinfo(type_: DType | Array, /, xp: Namespace) -> Any:
+    try:
+        return xp.iinfo(type_)
+    except (ValueError, TypeError):
+        return xp.iinfo(type_.dtype)
+
+
+__all__ = [
+    "arange",
+    "empty",
+    "empty_like",
+    "eye",
+    "full",
+    "full_like",
+    "linspace",
+    "ones",
+    "ones_like",
+    "zeros",
+    "zeros_like",
+    "UniqueAllResult",
+    "UniqueCountsResult",
+    "UniqueInverseResult",
+    "unique_all",
+    "unique_counts",
+    "unique_inverse",
+    "unique_values",
+    "std",
+    "var",
+    "cumulative_sum",
+    "cumulative_prod",
+    "clip",
+    "permute_dims",
+    "reshape",
+    "argsort",
+    "sort",
+    "nonzero",
+    "ceil",
+    "floor",
+    "trunc",
+    "matmul",
+    "matrix_transpose",
+    "tensordot",
+    "vecdot",
+    "isdtype",
+    "unstack",
+    "sign",
+    "finfo",
+    "iinfo",
+]
+_all_ignore = ["inspect", "array_namespace", "NamedTuple"]
+
+
+def __dir__() -> list[str]:
+    return __all__
diff --git a/sklearn/externals/array_api_compat/common/_fft.py b/sklearn/externals/array_api_compat/common/_fft.py
new file mode 100644
index 0000000000000..18839d37f8494
--- /dev/null
+++ b/sklearn/externals/array_api_compat/common/_fft.py
@@ -0,0 +1,213 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Literal, TypeAlias
+
+from ._typing import Array, Device, DType, Namespace
+
+_Norm: TypeAlias = Literal["backward", "ortho", "forward"]
+
+# Note: NumPy fft functions improperly upcast float32 and complex64 to
+# complex128, which is why we require wrapping them all here.
+
+def fft(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    n: int | None = None,
+    axis: int = -1,
+    norm: _Norm = "backward",
+) -> Array:
+    res = xp.fft.fft(x, n=n, axis=axis, norm=norm)
+    if x.dtype in [xp.float32, xp.complex64]:
+        return res.astype(xp.complex64)
+    return res
+
+def ifft(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    n: int | None = None,
+    axis: int = -1,
+    norm: _Norm = "backward",
+) -> Array:
+    res = xp.fft.ifft(x, n=n, axis=axis, norm=norm)
+    if x.dtype in [xp.float32, xp.complex64]:
+        return res.astype(xp.complex64)
+    return res
+
+def fftn(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    s: Sequence[int] | None = None,
+    axes: Sequence[int] | None = None,
+    norm: _Norm = "backward",
+) -> Array:
+    res = xp.fft.fftn(x, s=s, axes=axes, norm=norm)
+    if x.dtype in [xp.float32, xp.complex64]:
+        return res.astype(xp.complex64)
+    return res
+
+def ifftn(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    s: Sequence[int] | None = None,
+    axes: Sequence[int] | None = None,
+    norm: _Norm = "backward",
+) -> Array:
+    res = xp.fft.ifftn(x, s=s, axes=axes, norm=norm)
+    if x.dtype in [xp.float32, xp.complex64]:
+        return res.astype(xp.complex64)
+    return res
+
+def rfft(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    n: int | None = None,
+    axis: int = -1,
+    norm: _Norm = "backward",
+) -> Array:
+    res = xp.fft.rfft(x, n=n, axis=axis, norm=norm)
+    if x.dtype == xp.float32:
+        return res.astype(xp.complex64)
+    return res
+
+def irfft(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    n: int | None = None,
+    axis: int = -1,
+    norm: _Norm = "backward",
+) -> Array:
+    res = xp.fft.irfft(x, n=n, axis=axis, norm=norm)
+    if x.dtype == xp.complex64:
+        return res.astype(xp.float32)
+    return res
+
+def rfftn(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    s: Sequence[int] | None = None,
+    axes: Sequence[int] | None = None,
+    norm: _Norm = "backward",
+) -> Array:
+    res = xp.fft.rfftn(x, s=s, axes=axes, norm=norm)
+    if x.dtype == xp.float32:
+        return res.astype(xp.complex64)
+    return res
+
+def irfftn(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    s: Sequence[int] | None = None,
+    axes: Sequence[int] | None = None,
+    norm: _Norm = "backward",
+) -> Array:
+    res = xp.fft.irfftn(x, s=s, axes=axes, norm=norm)
+    if x.dtype == xp.complex64:
+        return res.astype(xp.float32)
+    return res
+
+def hfft(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    n: int | None = None,
+    axis: int = -1,
+    norm: _Norm = "backward",
+) -> Array:
+    res = xp.fft.hfft(x, n=n, axis=axis, norm=norm)
+    if x.dtype in [xp.float32, xp.complex64]:
+        return res.astype(xp.float32)
+    return res
+
+def ihfft(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    n: int | None = None,
+    axis: int = -1,
+    norm: _Norm = "backward",
+) -> Array:
+    res = xp.fft.ihfft(x, n=n, axis=axis, norm=norm)
+    if x.dtype in [xp.float32, xp.complex64]:
+        return res.astype(xp.complex64)
+    return res
+
+def fftfreq(
+    n: int,
+    /,
+    xp: Namespace,
+    *,
+    d: float = 1.0,
+    dtype: DType | None = None,
+    device: Device | None = None,
+) -> Array:
+    if device not in ["cpu", None]:
+        raise ValueError(f"Unsupported device {device!r}")
+    res = xp.fft.fftfreq(n, d=d)
+    if dtype is not None:
+        return res.astype(dtype)
+    return res
+
+def rfftfreq(
+    n: int,
+    /,
+    xp: Namespace,
+    *,
+    d: float = 1.0,
+    dtype: DType | None = None,
+    device: Device | None = None,
+) -> Array:
+    if device not in ["cpu", None]:
+        raise ValueError(f"Unsupported device {device!r}")
+    res = xp.fft.rfftfreq(n, d=d)
+    if dtype is not None:
+        return res.astype(dtype)
+    return res
+
+def fftshift(
+    x: Array, /, xp: Namespace, *, axes: int | Sequence[int] | None = None
+) -> Array:
+    return xp.fft.fftshift(x, axes=axes)
+
+def ifftshift(
+    x: Array, /, xp: Namespace, *, axes: int | Sequence[int] | None = None
+) -> Array:
+    return xp.fft.ifftshift(x, axes=axes)
+
+__all__ = [
+    "fft",
+    "ifft",
+    "fftn",
+    "ifftn",
+    "rfft",
+    "irfft",
+    "rfftn",
+    "irfftn",
+    "hfft",
+    "ihfft",
+    "fftfreq",
+    "rfftfreq",
+    "fftshift",
+    "ifftshift",
+]
+
+def __dir__() -> list[str]:
+    return __all__
diff --git a/sklearn/externals/array_api_compat/common/_helpers.py b/sklearn/externals/array_api_compat/common/_helpers.py
new file mode 100644
index 0000000000000..77175d0d1e974
--- /dev/null
+++ b/sklearn/externals/array_api_compat/common/_helpers.py
@@ -0,0 +1,1058 @@
+"""
+Various helper functions which are not part of the spec.
+
+Functions which start with an underscore are for internal use only but helpers
+that are in __all__ are intended as additional helper functions for use by end
+users of the compat library.
+"""
+
+from __future__ import annotations
+
+import inspect
+import math
+import sys
+import warnings
+from collections.abc import Collection, Hashable
+from functools import lru_cache
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Final,
+    Literal,
+    SupportsIndex,
+    TypeAlias,
+    TypeGuard,
+    TypeVar,
+    cast,
+    overload,
+)
+
+from ._typing import Array, Device, HasShape, Namespace, SupportsArrayNamespace
+
+if TYPE_CHECKING:
+
+    import dask.array as da
+    import jax
+    import ndonnx as ndx
+    import numpy as np
+    import numpy.typing as npt
+    import sparse  # pyright: ignore[reportMissingTypeStubs]
+    import torch
+
+    # TODO: import from typing (requires Python >=3.13)
+    from typing_extensions import TypeIs, TypeVar
+
+    _SizeT = TypeVar("_SizeT", bound = int | None)
+
+    _ZeroGradientArray: TypeAlias = npt.NDArray[np.void]
+    _CupyArray: TypeAlias = Any  # cupy has no py.typed
+
+    _ArrayApiObj: TypeAlias = (
+        npt.NDArray[Any]
+        | da.Array
+        | jax.Array
+        | ndx.Array
+        | sparse.SparseArray
+        | torch.Tensor
+        | SupportsArrayNamespace[Any]
+        | _CupyArray
+    )
+
+_API_VERSIONS_OLD: Final = frozenset({"2021.12", "2022.12", "2023.12"})
+_API_VERSIONS: Final = _API_VERSIONS_OLD | frozenset({"2024.12"})
+
+
+@lru_cache(100)
+def _issubclass_fast(cls: type, modname: str, clsname: str) -> bool:
+    try:
+        mod = sys.modules[modname]
+    except KeyError:
+        return False
+    parent_cls = getattr(mod, clsname)
+    return issubclass(cls, parent_cls)
+
+
+def _is_jax_zero_gradient_array(x: object) -> TypeGuard[_ZeroGradientArray]:
+    """Return True if `x` is a zero-gradient array.
+
+    These arrays are a design quirk of Jax that may one day be removed.
+    See https://github.com/google/jax/issues/20620.
+    """
+    # Fast exit
+    try:
+        dtype = x.dtype  # type: ignore[attr-defined]
+    except AttributeError:
+        return False
+    cls = cast(Hashable, type(dtype))
+    if not _issubclass_fast(cls, "numpy.dtypes", "VoidDType"):
+        return False
+
+    if "jax" not in sys.modules:
+        return False
+
+    import jax
+    # jax.float0 is a np.dtype([('float0', 'V')])
+    return dtype == jax.float0
+
+
+def is_numpy_array(x: object) -> TypeGuard[npt.NDArray[Any]]:
+    """
+    Return True if `x` is a NumPy array.
+
+    This function does not import NumPy if it has not already been imported
+    and is therefore cheap to use.
+
+    This also returns True for `ndarray` subclasses and NumPy scalar objects.
+
+    See Also
+    --------
+
+    array_namespace
+    is_array_api_obj
+    is_cupy_array
+    is_torch_array
+    is_ndonnx_array
+    is_dask_array
+    is_jax_array
+    is_pydata_sparse_array
+    """
+    # TODO: Should we reject ndarray subclasses?
+    cls = cast(Hashable, type(x))
+    return (
+        _issubclass_fast(cls, "numpy", "ndarray") 
+        or _issubclass_fast(cls, "numpy", "generic")
+    ) and not _is_jax_zero_gradient_array(x)
+
+
+def is_cupy_array(x: object) -> bool:
+    """
+    Return True if `x` is a CuPy array.
+
+    This function does not import CuPy if it has not already been imported
+    and is therefore cheap to use.
+
+    This also returns True for `cupy.ndarray` subclasses and CuPy scalar objects.
+
+    See Also
+    --------
+
+    array_namespace
+    is_array_api_obj
+    is_numpy_array
+    is_torch_array
+    is_ndonnx_array
+    is_dask_array
+    is_jax_array
+    is_pydata_sparse_array
+    """
+    cls = cast(Hashable, type(x))
+    return _issubclass_fast(cls, "cupy", "ndarray")
+
+
+def is_torch_array(x: object) -> TypeIs[torch.Tensor]:
+    """
+    Return True if `x` is a PyTorch tensor.
+
+    This function does not import PyTorch if it has not already been imported
+    and is therefore cheap to use.
+
+    See Also
+    --------
+
+    array_namespace
+    is_array_api_obj
+    is_numpy_array
+    is_cupy_array
+    is_dask_array
+    is_jax_array
+    is_pydata_sparse_array
+    """
+    cls = cast(Hashable, type(x))
+    return _issubclass_fast(cls, "torch", "Tensor")
+
+
+def is_ndonnx_array(x: object) -> TypeIs[ndx.Array]:
+    """
+    Return True if `x` is a ndonnx Array.
+
+    This function does not import ndonnx if it has not already been imported
+    and is therefore cheap to use.
+
+    See Also
+    --------
+
+    array_namespace
+    is_array_api_obj
+    is_numpy_array
+    is_cupy_array
+    is_ndonnx_array
+    is_dask_array
+    is_jax_array
+    is_pydata_sparse_array
+    """
+    cls = cast(Hashable, type(x))
+    return _issubclass_fast(cls, "ndonnx", "Array")
+
+
+def is_dask_array(x: object) -> TypeIs[da.Array]:
+    """
+    Return True if `x` is a dask.array Array.
+
+    This function does not import dask if it has not already been imported
+    and is therefore cheap to use.
+
+    See Also
+    --------
+
+    array_namespace
+    is_array_api_obj
+    is_numpy_array
+    is_cupy_array
+    is_torch_array
+    is_ndonnx_array
+    is_jax_array
+    is_pydata_sparse_array
+    """
+    cls = cast(Hashable, type(x))
+    return _issubclass_fast(cls, "dask.array", "Array")
+
+
+def is_jax_array(x: object) -> TypeIs[jax.Array]:
+    """
+    Return True if `x` is a JAX array.
+
+    This function does not import JAX if it has not already been imported
+    and is therefore cheap to use.
+
+
+    See Also
+    --------
+
+    array_namespace
+    is_array_api_obj
+    is_numpy_array
+    is_cupy_array
+    is_torch_array
+    is_ndonnx_array
+    is_dask_array
+    is_pydata_sparse_array
+    """
+    cls = cast(Hashable, type(x))
+    return _issubclass_fast(cls, "jax", "Array") or _is_jax_zero_gradient_array(x)
+
+
+def is_pydata_sparse_array(x: object) -> TypeIs[sparse.SparseArray]:
+    """
+    Return True if `x` is an array from the `sparse` package.
+
+    This function does not import `sparse` if it has not already been imported
+    and is therefore cheap to use.
+
+
+    See Also
+    --------
+
+    array_namespace
+    is_array_api_obj
+    is_numpy_array
+    is_cupy_array
+    is_torch_array
+    is_ndonnx_array
+    is_dask_array
+    is_jax_array
+    """
+    # TODO: Account for other backends.
+    cls = cast(Hashable, type(x))
+    return _issubclass_fast(cls, "sparse", "SparseArray")
+
+
+def is_array_api_obj(x: object) -> TypeIs[_ArrayApiObj]:  # pyright: ignore[reportUnknownParameterType]
+    """
+    Return True if `x` is an array API compatible array object.
+
+    See Also
+    --------
+
+    array_namespace
+    is_numpy_array
+    is_cupy_array
+    is_torch_array
+    is_ndonnx_array
+    is_dask_array
+    is_jax_array
+    """
+    return (
+        hasattr(x, '__array_namespace__') 
+        or _is_array_api_cls(cast(Hashable, type(x)))
+    )
+
+
+@lru_cache(100)
+def _is_array_api_cls(cls: type) -> bool:
+    return (
+        # TODO: drop support for numpy<2 which didn't have __array_namespace__
+        _issubclass_fast(cls, "numpy", "ndarray")
+        or _issubclass_fast(cls, "numpy", "generic")
+        or _issubclass_fast(cls, "cupy", "ndarray")
+        or _issubclass_fast(cls, "torch", "Tensor")
+        or _issubclass_fast(cls, "dask.array", "Array")
+        or _issubclass_fast(cls, "sparse", "SparseArray")
+        # TODO: drop support for jax<0.4.32 which didn't have __array_namespace__
+        or _issubclass_fast(cls, "jax", "Array")
+    )
+
+
+def _compat_module_name() -> str:
+    assert __name__.endswith(".common._helpers")
+    return __name__.removesuffix(".common._helpers")
+
+
+@lru_cache(100)
+def is_numpy_namespace(xp: Namespace) -> bool:
+    """
+    Returns True if `xp` is a NumPy namespace.
+
+    This includes both NumPy itself and the version wrapped by array-api-compat.
+
+    See Also
+    --------
+
+    array_namespace
+    is_cupy_namespace
+    is_torch_namespace
+    is_ndonnx_namespace
+    is_dask_namespace
+    is_jax_namespace
+    is_pydata_sparse_namespace
+    is_array_api_strict_namespace
+    """
+    return xp.__name__ in {"numpy", _compat_module_name() + ".numpy"}
+
+
+@lru_cache(100)
+def is_cupy_namespace(xp: Namespace) -> bool:
+    """
+    Returns True if `xp` is a CuPy namespace.
+
+    This includes both CuPy itself and the version wrapped by array-api-compat.
+
+    See Also
+    --------
+
+    array_namespace
+    is_numpy_namespace
+    is_torch_namespace
+    is_ndonnx_namespace
+    is_dask_namespace
+    is_jax_namespace
+    is_pydata_sparse_namespace
+    is_array_api_strict_namespace
+    """
+    return xp.__name__ in {"cupy", _compat_module_name() + ".cupy"}
+
+
+@lru_cache(100)
+def is_torch_namespace(xp: Namespace) -> bool:
+    """
+    Returns True if `xp` is a PyTorch namespace.
+
+    This includes both PyTorch itself and the version wrapped by array-api-compat.
+
+    See Also
+    --------
+
+    array_namespace
+    is_numpy_namespace
+    is_cupy_namespace
+    is_ndonnx_namespace
+    is_dask_namespace
+    is_jax_namespace
+    is_pydata_sparse_namespace
+    is_array_api_strict_namespace
+    """
+    return xp.__name__ in {"torch", _compat_module_name() + ".torch"}
+
+
+def is_ndonnx_namespace(xp: Namespace) -> bool:
+    """
+    Returns True if `xp` is an NDONNX namespace.
+
+    See Also
+    --------
+
+    array_namespace
+    is_numpy_namespace
+    is_cupy_namespace
+    is_torch_namespace
+    is_dask_namespace
+    is_jax_namespace
+    is_pydata_sparse_namespace
+    is_array_api_strict_namespace
+    """
+    return xp.__name__ == "ndonnx"
+
+
+@lru_cache(100)
+def is_dask_namespace(xp: Namespace) -> bool:
+    """
+    Returns True if `xp` is a Dask namespace.
+
+    This includes both ``dask.array`` itself and the version wrapped by array-api-compat.
+
+    See Also
+    --------
+
+    array_namespace
+    is_numpy_namespace
+    is_cupy_namespace
+    is_torch_namespace
+    is_ndonnx_namespace
+    is_jax_namespace
+    is_pydata_sparse_namespace
+    is_array_api_strict_namespace
+    """
+    return xp.__name__ in {"dask.array", _compat_module_name() + ".dask.array"}
+
+
+def is_jax_namespace(xp: Namespace) -> bool:
+    """
+    Returns True if `xp` is a JAX namespace.
+
+    This includes ``jax.numpy`` and ``jax.experimental.array_api`` which existed in
+    older versions of JAX.
+
+    See Also
+    --------
+
+    array_namespace
+    is_numpy_namespace
+    is_cupy_namespace
+    is_torch_namespace
+    is_ndonnx_namespace
+    is_dask_namespace
+    is_pydata_sparse_namespace
+    is_array_api_strict_namespace
+    """
+    return xp.__name__ in {"jax.numpy", "jax.experimental.array_api"}
+
+
+def is_pydata_sparse_namespace(xp: Namespace) -> bool:
+    """
+    Returns True if `xp` is a pydata/sparse namespace.
+
+    See Also
+    --------
+
+    array_namespace
+    is_numpy_namespace
+    is_cupy_namespace
+    is_torch_namespace
+    is_ndonnx_namespace
+    is_dask_namespace
+    is_jax_namespace
+    is_array_api_strict_namespace
+    """
+    return xp.__name__ == "sparse"
+
+
+def is_array_api_strict_namespace(xp: Namespace) -> bool:
+    """
+    Returns True if `xp` is an array-api-strict namespace.
+
+    See Also
+    --------
+
+    array_namespace
+    is_numpy_namespace
+    is_cupy_namespace
+    is_torch_namespace
+    is_ndonnx_namespace
+    is_dask_namespace
+    is_jax_namespace
+    is_pydata_sparse_namespace
+    """
+    return xp.__name__ == "array_api_strict"
+
+
+def _check_api_version(api_version: str | None) -> None:
+    if api_version in _API_VERSIONS_OLD:
+        warnings.warn(
+            f"The {api_version} version of the array API specification was requested but the returned namespace is actually version 2024.12"
+        )
+    elif api_version is not None and api_version not in _API_VERSIONS:
+        raise ValueError(
+            "Only the 2024.12 version of the array API specification is currently supported"
+        )
+
+
+def array_namespace(
+    *xs: Array | complex | None,
+    api_version: str | None = None,
+    use_compat: bool | None = None,
+) -> Namespace:
+    """
+    Get the array API compatible namespace for the arrays `xs`.
+
+    Parameters
+    ----------
+    xs: arrays
+        one or more arrays. xs can also be Python scalars (bool, int, float,
+        complex, or None), which are ignored.
+
+    api_version: str
+        The newest version of the spec that you need support for (currently
+        the compat library wrapped APIs support v2024.12).
+
+    use_compat: bool or None
+        If None (the default), the native namespace will be returned if it is
+        already array API compatible, otherwise a compat wrapper is used. If
+        True, the compat library wrapped library will be returned. If False,
+        the native library namespace is returned.
+
+    Returns
+    -------
+
+    out: namespace
+        The array API compatible namespace corresponding to the arrays in `xs`.
+
+    Raises
+    ------
+    TypeError
+        If `xs` contains arrays from different array libraries or contains a
+        non-array.
+
+
+    Typical usage is to pass the arguments of a function to
+    `array_namespace()` at the top of a function to get the corresponding
+    array API namespace:
+
+    .. code:: python
+
+       def your_function(x, y):
+           xp = array_api_compat.array_namespace(x, y)
+           # Now use xp as the array library namespace
+           return xp.mean(x, axis=0) + 2*xp.std(y, axis=0)
+
+
+    Wrapped array namespaces can also be imported directly. For example,
+    `array_namespace(np.array(...))` will return `array_api_compat.numpy`.
+    This function will also work for any array library not wrapped by
+    array-api-compat if it explicitly defines `__array_namespace__
+    `__
+    (the wrapped namespace is always preferred if it exists).
+
+    See Also
+    --------
+
+    is_array_api_obj
+    is_numpy_array
+    is_cupy_array
+    is_torch_array
+    is_dask_array
+    is_jax_array
+    is_pydata_sparse_array
+
+    """
+    if use_compat not in [None, True, False]:
+        raise ValueError("use_compat must be None, True, or False")
+
+    _use_compat = use_compat in [None, True]
+
+    namespaces: set[Namespace] = set()
+    for x in xs:
+        if is_numpy_array(x):
+            import numpy as np
+
+            from .. import numpy as numpy_namespace
+
+            if use_compat is True:
+                _check_api_version(api_version)
+                namespaces.add(numpy_namespace)
+            elif use_compat is False:
+                namespaces.add(np)
+            else:
+                # numpy 2.0+ have __array_namespace__, however, they are not yet fully array API
+                # compatible.
+                namespaces.add(numpy_namespace)
+        elif is_cupy_array(x):
+            if _use_compat:
+                _check_api_version(api_version)
+                from .. import cupy as cupy_namespace
+
+                namespaces.add(cupy_namespace)
+            else:
+                import cupy as cp  # pyright: ignore[reportMissingTypeStubs]
+
+                namespaces.add(cp)
+        elif is_torch_array(x):
+            if _use_compat:
+                _check_api_version(api_version)
+                from .. import torch as torch_namespace
+
+                namespaces.add(torch_namespace)
+            else:
+                import torch
+
+                namespaces.add(torch)
+        elif is_dask_array(x):
+            if _use_compat:
+                _check_api_version(api_version)
+                from ..dask import array as dask_namespace
+
+                namespaces.add(dask_namespace)
+            else:
+                import dask.array as da
+
+                namespaces.add(da)
+        elif is_jax_array(x):
+            if use_compat is True:
+                _check_api_version(api_version)
+                raise ValueError("JAX does not have an array-api-compat wrapper")
+            elif use_compat is False:
+                import jax.numpy as jnp
+            else:
+                # JAX v0.4.32 and newer implements the array API directly in jax.numpy.
+                # For older JAX versions, it is available via jax.experimental.array_api.
+                import jax.numpy
+
+                if hasattr(jax.numpy, "__array_api_version__"):
+                    jnp = jax.numpy
+                else:
+                    import jax.experimental.array_api as jnp  # pyright: ignore[reportMissingImports]
+            namespaces.add(jnp)
+        elif is_pydata_sparse_array(x):
+            if use_compat is True:
+                _check_api_version(api_version)
+                raise ValueError("`sparse` does not have an array-api-compat wrapper")
+            else:
+                import sparse  # pyright: ignore[reportMissingTypeStubs]
+            # `sparse` is already an array namespace. We do not have a wrapper
+            # submodule for it.
+            namespaces.add(sparse)
+        elif hasattr(x, "__array_namespace__"):
+            if use_compat is True:
+                raise ValueError(
+                    "The given array does not have an array-api-compat wrapper"
+                )
+            x = cast("SupportsArrayNamespace[Any]", x)
+            namespaces.add(x.__array_namespace__(api_version=api_version))
+        elif isinstance(x, (bool, int, float, complex, type(None))):
+            continue
+        else:
+            # TODO: Support Python scalars?
+            raise TypeError(f"{type(x).__name__} is not a supported array type")
+
+    if not namespaces:
+        raise TypeError("Unrecognized array input")
+
+    if len(namespaces) != 1:
+        raise TypeError(f"Multiple namespaces for array inputs: {namespaces}")
+
+    (xp,) = namespaces
+
+    return xp
+
+
+# backwards compatibility alias
+get_namespace = array_namespace
+
+
+def _check_device(bare_xp: Namespace, device: Device) -> None:  # pyright: ignore[reportUnusedFunction]
+    """
+    Validate dummy device on device-less array backends.
+
+    Notes
+    -----
+    This function is also invoked by CuPy, which does have multiple devices
+    if there are multiple GPUs available.
+    However, CuPy multi-device support is currently impossible
+    without using the global device or a context manager:
+
+    https://github.com/data-apis/array-api-compat/pull/293
+    """
+    if bare_xp is sys.modules.get("numpy"):
+        if device not in ("cpu", None):
+            raise ValueError(f"Unsupported device for NumPy: {device!r}")
+
+    elif bare_xp is sys.modules.get("dask.array"):
+        if device not in ("cpu", _DASK_DEVICE, None):
+            raise ValueError(f"Unsupported device for Dask: {device!r}")
+
+
+# Placeholder object to represent the dask device
+# when the array backend is not the CPU.
+# (since it is not easy to tell which device a dask array is on)
+class _dask_device:
+    def __repr__(self) -> Literal["DASK_DEVICE"]:
+        return "DASK_DEVICE"
+
+
+_DASK_DEVICE = _dask_device()
+
+
+# device() is not on numpy.ndarray or dask.array and to_device() is not on numpy.ndarray
+# or cupy.ndarray. They are not included in array objects of this library
+# because this library just reuses the respective ndarray classes without
+# wrapping or subclassing them. These helper functions can be used instead of
+# the wrapper functions for libraries that need to support both NumPy/CuPy and
+# other libraries that use devices.
+def device(x: _ArrayApiObj, /) -> Device:
+    """
+    Hardware device the array data resides on.
+
+    This is equivalent to `x.device` according to the `standard
+    `__.
+    This helper is included because some array libraries either do not have
+    the `device` attribute or include it with an incompatible API.
+
+    Parameters
+    ----------
+    x: array
+        array instance from an array API compatible library.
+
+    Returns
+    -------
+    out: device
+        a ``device`` object (see the `Device Support `__
+        section of the array API specification).
+
+    Notes
+    -----
+
+    For NumPy the device is always `"cpu"`. For Dask, the device is always a
+    special `DASK_DEVICE` object.
+
+    See Also
+    --------
+
+    to_device : Move array data to a different device.
+
+    """
+    if is_numpy_array(x):
+        return "cpu"
+    elif is_dask_array(x):
+        # Peek at the metadata of the Dask array to determine type
+        if is_numpy_array(x._meta):  # pyright: ignore
+            # Must be on CPU since backed by numpy
+            return "cpu"
+        return _DASK_DEVICE
+    elif is_jax_array(x):
+        # FIXME Jitted JAX arrays do not have a device attribute
+        #       https://github.com/jax-ml/jax/issues/26000
+        #       Return None in this case. Note that this workaround breaks
+        #       the standard and will result in new arrays being created on the
+        #       default device instead of the same device as the input array(s).
+        x_device = getattr(x, "device", None)
+        # Older JAX releases had .device() as a method, which has been replaced
+        # with a property in accordance with the standard.
+        if inspect.ismethod(x_device):
+            return x_device()
+        else:
+            return x_device
+    elif is_pydata_sparse_array(x):
+        # `sparse` will gain `.device`, so check for this first.
+        x_device = getattr(x, "device", None)
+        if x_device is not None:
+            return x_device
+        # Everything but DOK has this attr.
+        try:
+            inner = x.data  # pyright: ignore
+        except AttributeError:
+            return "cpu"
+        # Return the device of the constituent array
+        return device(inner)  # pyright: ignore
+    return x.device  # pyright: ignore
+
+
+# Prevent shadowing, used below
+_device = device
+
+
+# Based on cupy.array_api.Array.to_device
+def _cupy_to_device(
+    x: _CupyArray,
+    device: Device,
+    /,
+    stream: int | Any | None = None,
+) -> _CupyArray:
+    import cupy as cp
+
+    if device == "cpu":
+        # allowing us to use `to_device(x, "cpu")`
+        # is useful for portable test swapping between
+        # host and device backends
+        return x.get()
+    if not isinstance(device, cp.cuda.Device):
+        raise TypeError(f"Unsupported device type {device!r}")
+
+    if stream is None:
+        with device:
+            return cp.asarray(x)
+
+    # stream can be an int as specified in __dlpack__, or a CuPy stream
+    if isinstance(stream, int):
+        stream = cp.cuda.ExternalStream(stream)
+    elif not isinstance(stream, cp.cuda.Stream):
+        raise TypeError(f"Unsupported stream type {stream!r}")
+
+    with device, stream:
+        return cp.asarray(x)
+
+
+def _torch_to_device(
+    x: torch.Tensor,
+    device: torch.device | str | int,
+    /,
+    stream: None = None,
+) -> torch.Tensor:
+    if stream is not None:
+        raise NotImplementedError
+    return x.to(device)
+
+
+def to_device(x: Array, device: Device, /, *, stream: int | Any | None = None) -> Array:
+    """
+    Copy the array from the device on which it currently resides to the specified ``device``.
+
+    This is equivalent to `x.to_device(device, stream=stream)` according to
+    the `standard
+    `__.
+    This helper is included because some array libraries do not have the
+    `to_device` method.
+
+    Parameters
+    ----------
+
+    x: array
+        array instance from an array API compatible library.
+
+    device: device
+        a ``device`` object (see the `Device Support `__
+        section of the array API specification).
+
+    stream: int | Any | None
+        stream object to use during copy. In addition to the types supported
+        in ``array.__dlpack__``, implementations may choose to support any
+        library-specific stream object with the caveat that any code using
+        such an object would not be portable.
+
+    Returns
+    -------
+
+    out: array
+        an array with the same data and data type as ``x`` and located on the
+        specified ``device``.
+
+    Notes
+    -----
+
+    For NumPy, this function effectively does nothing since the only supported
+    device is the CPU. For CuPy, this method supports CuPy CUDA
+    :external+cupy:class:`Device ` and
+    :external+cupy:class:`Stream ` objects. For PyTorch,
+    this is the same as :external+torch:meth:`x.to(device) `
+    (the ``stream`` argument is not supported in PyTorch).
+
+    See Also
+    --------
+
+    device : Hardware device the array data resides on.
+
+    """
+    if is_numpy_array(x):
+        if stream is not None:
+            raise ValueError("The stream argument to to_device() is not supported")
+        if device == "cpu":
+            return x
+        raise ValueError(f"Unsupported device {device!r}")
+    elif is_cupy_array(x):
+        # cupy does not yet have to_device
+        return _cupy_to_device(x, device, stream=stream)
+    elif is_torch_array(x):
+        return _torch_to_device(x, device, stream=stream)  # pyright: ignore[reportArgumentType]
+    elif is_dask_array(x):
+        if stream is not None:
+            raise ValueError("The stream argument to to_device() is not supported")
+        # TODO: What if our array is on the GPU already?
+        if device == "cpu":
+            return x
+        raise ValueError(f"Unsupported device {device!r}")
+    elif is_jax_array(x):
+        if not hasattr(x, "__array_namespace__"):
+            # In JAX v0.4.31 and older, this import adds to_device method to x...
+            import jax.experimental.array_api  # noqa: F401  # pyright: ignore
+
+            # ... but only on eager JAX. It won't work inside jax.jit.
+            if not hasattr(x, "to_device"):
+                return x
+        return x.to_device(device, stream=stream)
+    elif is_pydata_sparse_array(x) and device == _device(x):
+        # Perform trivial check to return the same array if
+        # device is same instead of err-ing.
+        return x
+    return x.to_device(device, stream=stream)  # pyright: ignore
+
+
+@overload
+def size(x: HasShape[Collection[SupportsIndex]]) -> int: ...
+@overload
+def size(x: HasShape[Collection[None]]) -> None: ...
+@overload
+def size(x: HasShape[Collection[SupportsIndex | None]]) -> int | None: ...
+def size(x: HasShape[Collection[SupportsIndex | None]]) -> int | None:
+    """
+    Return the total number of elements of x.
+
+    This is equivalent to `x.size` according to the `standard
+    `__.
+
+    This helper is included because PyTorch defines `size` in an
+    :external+torch:meth:`incompatible way `.
+    It also fixes dask.array's behaviour which returns nan for unknown sizes, whereas
+    the standard requires None.
+    """
+    # Lazy API compliant arrays, such as ndonnx, can contain None in their shape
+    if None in x.shape:
+        return None
+    out = math.prod(cast("Collection[SupportsIndex]", x.shape))
+    # dask.array.Array.shape can contain NaN
+    return None if math.isnan(out) else out
+
+
+@lru_cache(100)
+def _is_writeable_cls(cls: type) -> bool | None:
+    if (
+        _issubclass_fast(cls, "numpy", "generic")
+        or _issubclass_fast(cls, "jax", "Array")
+        or _issubclass_fast(cls, "sparse", "SparseArray")
+    ):
+        return False
+    if _is_array_api_cls(cls):
+        return True
+    return None
+
+
+def is_writeable_array(x: object) -> bool:
+    """
+    Return False if ``x.__setitem__`` is expected to raise; True otherwise.
+    Return False if `x` is not an array API compatible object.
+
+    Warning
+    -------
+    As there is no standard way to check if an array is writeable without actually
+    writing to it, this function blindly returns True for all unknown array types.
+    """
+    cls = cast(Hashable, type(x))
+    if _issubclass_fast(cls, "numpy", "ndarray"):
+        return cast("npt.NDArray", x).flags.writeable
+    res = _is_writeable_cls(cls)
+    if res is not None:
+        return res
+    return hasattr(x, '__array_namespace__')
+
+
+@lru_cache(100)
+def _is_lazy_cls(cls: type) -> bool | None:
+    if (
+        _issubclass_fast(cls, "numpy", "ndarray")
+        or _issubclass_fast(cls, "numpy", "generic")
+        or _issubclass_fast(cls, "cupy", "ndarray")
+        or _issubclass_fast(cls, "torch", "Tensor")
+        or _issubclass_fast(cls, "sparse", "SparseArray")
+    ):
+        return False
+    if (
+        _issubclass_fast(cls, "jax", "Array")
+        or _issubclass_fast(cls, "dask.array", "Array")
+        or _issubclass_fast(cls, "ndonnx", "Array")
+    ):
+        return True
+    return  None
+
+
+def is_lazy_array(x: object) -> bool:
+    """Return True if x is potentially a future or it may be otherwise impossible or
+    expensive to eagerly read its contents, regardless of their size, e.g. by
+    calling ``bool(x)`` or ``float(x)``.
+
+    Return False otherwise; e.g. ``bool(x)`` etc. is guaranteed to succeed and to be
+    cheap as long as the array has the right dtype and size.
+
+    Note
+    ----
+    This function errs on the side of caution for array types that may or may not be
+    lazy, e.g. JAX arrays, by always returning True for them.
+    """
+    # **JAX note:** while it is possible to determine if you're inside or outside
+    # jax.jit by testing the subclass of a jax.Array object, as well as testing bool()
+    # as we do below for unknown arrays, this is not recommended by JAX best practices.
+
+    # **Dask note:** Dask eagerly computes the graph on __bool__, __float__, and so on.
+    # This behaviour, while impossible to change without breaking backwards
+    # compatibility, is highly detrimental to performance as the whole graph will end
+    # up being computed multiple times.
+
+    # Note: skipping reclassification of JAX zero gradient arrays, as one will
+    # exclusively get them once they leave a jax.grad JIT context.
+    cls = cast(Hashable, type(x))
+    res = _is_lazy_cls(cls)
+    if res is not None:
+        return res
+
+    if not hasattr(x, "__array_namespace__"):
+        return False
+
+    # Unknown Array API compatible object. Note that this test may have dire consequences
+    # in terms of performance, e.g. for a lazy object that eagerly computes the graph
+    # on __bool__ (dask is one such example, which however is special-cased above).
+
+    # Select a single point of the array
+    s = size(cast("HasShape[Collection[SupportsIndex | None]]", x))
+    if s is None:
+        return True
+    xp = array_namespace(x)
+    if s > 1:
+        x = xp.reshape(x, (-1,))[0]
+    # Cast to dtype=bool and deal with size 0 arrays
+    x = xp.any(x)
+
+    try:
+        bool(x)
+        return False
+    # The Array API standard dictactes that __bool__ should raise TypeError if the
+    # output cannot be defined.
+    # Here we allow for it to raise arbitrary exceptions, e.g. like Dask does.
+    except Exception:
+        return True
+
+
+__all__ = [
+    "array_namespace",
+    "device",
+    "get_namespace",
+    "is_array_api_obj",
+    "is_array_api_strict_namespace",
+    "is_cupy_array",
+    "is_cupy_namespace",
+    "is_dask_array",
+    "is_dask_namespace",
+    "is_jax_array",
+    "is_jax_namespace",
+    "is_numpy_array",
+    "is_numpy_namespace",
+    "is_torch_array",
+    "is_torch_namespace",
+    "is_ndonnx_array",
+    "is_ndonnx_namespace",
+    "is_pydata_sparse_array",
+    "is_pydata_sparse_namespace",
+    "is_writeable_array",
+    "is_lazy_array",
+    "size",
+    "to_device",
+]
+
+_all_ignore = ['lru_cache', 'sys', 'math', 'inspect', 'warnings']
+
+def __dir__() -> list[str]:
+    return __all__
diff --git a/sklearn/externals/array_api_compat/common/_linalg.py b/sklearn/externals/array_api_compat/common/_linalg.py
new file mode 100644
index 0000000000000..7ad87a1be9105
--- /dev/null
+++ b/sklearn/externals/array_api_compat/common/_linalg.py
@@ -0,0 +1,232 @@
+from __future__ import annotations
+
+import math
+from typing import Literal, NamedTuple, cast
+
+import numpy as np
+
+if np.__version__[0] == "2":
+    from numpy.lib.array_utils import normalize_axis_tuple
+else:
+    from numpy.core.numeric import normalize_axis_tuple
+
+from .._internal import get_xp
+from ._aliases import isdtype, matmul, matrix_transpose, tensordot, vecdot
+from ._typing import Array, DType, JustFloat, JustInt, Namespace
+
+
+# These are in the main NumPy namespace but not in numpy.linalg
+def cross(
+    x1: Array,
+    x2: Array,
+    /,
+    xp: Namespace,
+    *,
+    axis: int = -1,
+    **kwargs: object,
+) -> Array:
+    return xp.cross(x1, x2, axis=axis, **kwargs)
+
+def outer(x1: Array, x2: Array, /, xp: Namespace, **kwargs: object) -> Array:
+    return xp.outer(x1, x2, **kwargs)
+
+class EighResult(NamedTuple):
+    eigenvalues: Array
+    eigenvectors: Array
+
+class QRResult(NamedTuple):
+    Q: Array
+    R: Array
+
+class SlogdetResult(NamedTuple):
+    sign: Array
+    logabsdet: Array
+
+class SVDResult(NamedTuple):
+    U: Array
+    S: Array
+    Vh: Array
+
+# These functions are the same as their NumPy counterparts except they return
+# a namedtuple.
+def eigh(x: Array, /, xp: Namespace, **kwargs: object) -> EighResult:
+    return EighResult(*xp.linalg.eigh(x, **kwargs))
+
+def qr(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    mode: Literal["reduced", "complete"] = "reduced",
+    **kwargs: object,
+) -> QRResult:
+    return QRResult(*xp.linalg.qr(x, mode=mode, **kwargs))
+
+def slogdet(x: Array, /, xp: Namespace, **kwargs: object) -> SlogdetResult:
+    return SlogdetResult(*xp.linalg.slogdet(x, **kwargs))
+
+def svd(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    full_matrices: bool = True,
+    **kwargs: object,
+) -> SVDResult:
+    return SVDResult(*xp.linalg.svd(x, full_matrices=full_matrices, **kwargs))
+
+# These functions have additional keyword arguments
+
+# The upper keyword argument is new from NumPy
+def cholesky(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    upper: bool = False,
+    **kwargs: object,
+) -> Array:
+    L = xp.linalg.cholesky(x, **kwargs)
+    if upper:
+        U = get_xp(xp)(matrix_transpose)(L)
+        if get_xp(xp)(isdtype)(U.dtype, 'complex floating'):
+            U = xp.conj(U)  # pyright: ignore[reportConstantRedefinition]
+        return U
+    return L
+
+# The rtol keyword argument of matrix_rank() and pinv() is new from NumPy.
+# Note that it has a different semantic meaning from tol and rcond.
+def matrix_rank(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    rtol: float | Array | None = None,
+    **kwargs: object,
+) -> Array:
+    # this is different from xp.linalg.matrix_rank, which supports 1
+    # dimensional arrays.
+    if x.ndim < 2:
+        raise xp.linalg.LinAlgError("1-dimensional array given. Array must be at least two-dimensional")
+    S: Array = get_xp(xp)(svdvals)(x, **kwargs)
+    if rtol is None:
+        tol = S.max(axis=-1, keepdims=True) * max(x.shape[-2:]) * xp.finfo(S.dtype).eps
+    else:
+        # this is different from xp.linalg.matrix_rank, which does not
+        # multiply the tolerance by the largest singular value.
+        tol = S.max(axis=-1, keepdims=True)*xp.asarray(rtol)[..., xp.newaxis]
+    return xp.count_nonzero(S > tol, axis=-1)
+
+def pinv(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    rtol: float | Array | None = None,
+    **kwargs: object,
+) -> Array:
+    # this is different from xp.linalg.pinv, which does not multiply the
+    # default tolerance by max(M, N).
+    if rtol is None:
+        rtol = max(x.shape[-2:]) * xp.finfo(x.dtype).eps
+    return xp.linalg.pinv(x, rcond=rtol, **kwargs)
+
+# These functions are new in the array API spec
+
+def matrix_norm(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    keepdims: bool = False,
+    ord: Literal[1, 2, -1, -2] | JustFloat | Literal["fro", "nuc"] | None = "fro",
+) -> Array:
+    return xp.linalg.norm(x, axis=(-2, -1), keepdims=keepdims, ord=ord)
+
+# svdvals is not in NumPy (but it is in SciPy). It is equivalent to
+# xp.linalg.svd(compute_uv=False).
+def svdvals(x: Array, /, xp: Namespace) -> Array | tuple[Array, ...]:
+    return xp.linalg.svd(x, compute_uv=False)
+
+def vector_norm(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    axis: int | tuple[int, ...] | None = None,
+    keepdims: bool = False,
+    ord: JustInt | JustFloat = 2,
+) -> Array:
+    # xp.linalg.norm tries to do a matrix norm whenever axis is a 2-tuple or
+    # when axis=None and the input is 2-D, so to force a vector norm, we make
+    # it so the input is 1-D (for axis=None), or reshape so that norm is done
+    # on a single dimension.
+    if axis is None:
+        # Note: xp.linalg.norm() doesn't handle 0-D arrays
+        _x = x.ravel()
+        _axis = 0
+    elif isinstance(axis, tuple):
+        # Note: The axis argument supports any number of axes, whereas
+        # xp.linalg.norm() only supports a single axis for vector norm.
+        normalized_axis = cast(
+            "tuple[int, ...]",
+            normalize_axis_tuple(axis, x.ndim),  # pyright: ignore[reportCallIssue]
+        )
+        rest = tuple(i for i in range(x.ndim) if i not in normalized_axis)
+        newshape = axis + rest
+        _x = xp.transpose(x, newshape).reshape(
+            (math.prod([x.shape[i] for i in axis]), *[x.shape[i] for i in rest]))
+        _axis = 0
+    else:
+        _x = x
+        _axis = axis
+
+    res = xp.linalg.norm(_x, axis=_axis, ord=ord)
+
+    if keepdims:
+        # We can't reuse xp.linalg.norm(keepdims) because of the reshape hacks
+        # above to avoid matrix norm logic.
+        shape = list(x.shape)
+        _axis = cast(
+            "tuple[int, ...]",
+            normalize_axis_tuple(  # pyright: ignore[reportCallIssue]
+                range(x.ndim) if axis is None else axis,
+                x.ndim,
+            ),
+        )
+        for i in _axis:
+            shape[i] = 1
+        res = xp.reshape(res, tuple(shape))
+
+    return res
+
+# xp.diagonal and xp.trace operate on the first two axes whereas these
+# operates on the last two
+
+def diagonal(x: Array, /, xp: Namespace, *, offset: int = 0, **kwargs: object) -> Array:
+    return xp.diagonal(x, offset=offset, axis1=-2, axis2=-1, **kwargs)
+
+def trace(
+    x: Array,
+    /,
+    xp: Namespace,
+    *,
+    offset: int = 0,
+    dtype: DType | None = None,
+    **kwargs: object,
+) -> Array:
+    return xp.asarray(
+        xp.trace(x, offset=offset, dtype=dtype, axis1=-2, axis2=-1, **kwargs)
+    )
+
+__all__ = ['cross', 'matmul', 'outer', 'tensordot', 'EighResult',
+           'QRResult', 'SlogdetResult', 'SVDResult', 'eigh', 'qr', 'slogdet',
+           'svd', 'cholesky', 'matrix_rank', 'pinv', 'matrix_norm',
+           'matrix_transpose', 'svdvals', 'vecdot', 'vector_norm', 'diagonal',
+           'trace']
+
+_all_ignore = ['math', 'normalize_axis_tuple', 'get_xp', 'np', 'isdtype']
+
+
+def __dir__() -> list[str]:
+    return __all__
diff --git a/sklearn/externals/array_api_compat/common/_typing.py b/sklearn/externals/array_api_compat/common/_typing.py
new file mode 100644
index 0000000000000..cd26feeba4dff
--- /dev/null
+++ b/sklearn/externals/array_api_compat/common/_typing.py
@@ -0,0 +1,192 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from types import ModuleType as Namespace
+from typing import (
+    TYPE_CHECKING,
+    Literal,
+    Protocol,
+    TypeAlias,
+    TypedDict,
+    TypeVar,
+    final,
+)
+
+if TYPE_CHECKING:
+    from _typeshed import Incomplete
+
+    SupportsBufferProtocol: TypeAlias = Incomplete
+    Array: TypeAlias = Incomplete
+    Device: TypeAlias = Incomplete
+    DType: TypeAlias = Incomplete
+else:
+    SupportsBufferProtocol = object
+    Array = object
+    Device = object
+    DType = object
+
+
+_T_co = TypeVar("_T_co", covariant=True)
+
+
+# These "Just" types are equivalent to the `Just` type from the `optype` library,
+# apart from them not being `@runtime_checkable`.
+# - docs: https://github.com/jorenham/optype/blob/master/README.md#just
+# - code: https://github.com/jorenham/optype/blob/master/optype/_core/_just.py
+@final
+class JustInt(Protocol):
+    @property
+    def __class__(self, /) -> type[int]: ...
+    @__class__.setter
+    def __class__(self, value: type[int], /) -> None: ...  # pyright: ignore[reportIncompatibleMethodOverride]
+
+
+@final
+class JustFloat(Protocol):
+    @property
+    def __class__(self, /) -> type[float]: ...
+    @__class__.setter
+    def __class__(self, value: type[float], /) -> None: ...  # pyright: ignore[reportIncompatibleMethodOverride]
+
+
+@final
+class JustComplex(Protocol):
+    @property
+    def __class__(self, /) -> type[complex]: ...
+    @__class__.setter
+    def __class__(self, value: type[complex], /) -> None: ...  # pyright: ignore[reportIncompatibleMethodOverride]
+
+
+#
+
+
+class NestedSequence(Protocol[_T_co]):
+    def __getitem__(self, key: int, /) -> _T_co | NestedSequence[_T_co]: ...
+    def __len__(self, /) -> int: ...
+
+
+class SupportsArrayNamespace(Protocol[_T_co]):
+    def __array_namespace__(self, /, *, api_version: str | None) -> _T_co: ...
+
+
+class HasShape(Protocol[_T_co]):
+    @property
+    def shape(self, /) -> _T_co: ...
+
+
+# Return type of `__array_namespace_info__.default_dtypes`
+Capabilities = TypedDict(
+    "Capabilities",
+    {
+        "boolean indexing": bool,
+        "data-dependent shapes": bool,
+        "max dimensions": int,
+    },
+)
+
+# Return type of `__array_namespace_info__.default_dtypes`
+DefaultDTypes = TypedDict(
+    "DefaultDTypes",
+    {
+        "real floating": DType,
+        "complex floating": DType,
+        "integral": DType,
+        "indexing": DType,
+    },
+)
+
+
+_DTypeKind: TypeAlias = Literal[
+    "bool",
+    "signed integer",
+    "unsigned integer",
+    "integral",
+    "real floating",
+    "complex floating",
+    "numeric",
+]
+# Type of the `kind` parameter in `__array_namespace_info__.dtypes`
+DTypeKind: TypeAlias = _DTypeKind | tuple[_DTypeKind, ...]
+
+
+# `__array_namespace_info__.dtypes(kind="bool")`
+class DTypesBool(TypedDict):
+    bool: DType
+
+
+# `__array_namespace_info__.dtypes(kind="signed integer")`
+class DTypesSigned(TypedDict):
+    int8: DType
+    int16: DType
+    int32: DType
+    int64: DType
+
+
+# `__array_namespace_info__.dtypes(kind="unsigned integer")`
+class DTypesUnsigned(TypedDict):
+    uint8: DType
+    uint16: DType
+    uint32: DType
+    uint64: DType
+
+
+# `__array_namespace_info__.dtypes(kind="integral")`
+class DTypesIntegral(DTypesSigned, DTypesUnsigned):
+    pass
+
+
+# `__array_namespace_info__.dtypes(kind="real floating")`
+class DTypesReal(TypedDict):
+    float32: DType
+    float64: DType
+
+
+# `__array_namespace_info__.dtypes(kind="complex floating")`
+class DTypesComplex(TypedDict):
+    complex64: DType
+    complex128: DType
+
+
+# `__array_namespace_info__.dtypes(kind="numeric")`
+class DTypesNumeric(DTypesIntegral, DTypesReal, DTypesComplex):
+    pass
+
+
+# `__array_namespace_info__.dtypes(kind=None)` (default)
+class DTypesAll(DTypesBool, DTypesNumeric):
+    pass
+
+
+# `__array_namespace_info__.dtypes(kind=?)` (fallback)
+DTypesAny: TypeAlias = Mapping[str, DType]
+
+
+__all__ = [
+    "Array",
+    "Capabilities",
+    "DType",
+    "DTypeKind",
+    "DTypesAny",
+    "DTypesAll",
+    "DTypesBool",
+    "DTypesNumeric",
+    "DTypesIntegral",
+    "DTypesSigned",
+    "DTypesUnsigned",
+    "DTypesReal",
+    "DTypesComplex",
+    "DefaultDTypes",
+    "Device",
+    "HasShape",
+    "Namespace",
+    "JustInt",
+    "JustFloat",
+    "JustComplex",
+    "NestedSequence",
+    "SupportsArrayNamespace",
+    "SupportsBufferProtocol",
+]
+
+
+def __dir__() -> list[str]:
+    return __all__
diff --git a/sklearn/externals/array_api_compat/cupy/__init__.py b/sklearn/externals/array_api_compat/cupy/__init__.py
new file mode 100644
index 0000000000000..9a30f95ddf12c
--- /dev/null
+++ b/sklearn/externals/array_api_compat/cupy/__init__.py
@@ -0,0 +1,13 @@
+from cupy import * # noqa: F403
+
+# from cupy import * doesn't overwrite these builtin names
+from cupy import abs, max, min, round # noqa: F401
+
+# These imports may overwrite names from the import * above.
+from ._aliases import * # noqa: F403
+
+# See the comment in the numpy __init__.py
+__import__(__package__ + '.linalg')
+__import__(__package__ + '.fft')
+
+__array_api_version__ = '2024.12'
diff --git a/sklearn/externals/array_api_compat/cupy/_aliases.py b/sklearn/externals/array_api_compat/cupy/_aliases.py
new file mode 100644
index 0000000000000..90b48f059bafa
--- /dev/null
+++ b/sklearn/externals/array_api_compat/cupy/_aliases.py
@@ -0,0 +1,156 @@
+from __future__ import annotations
+
+from typing import Optional
+
+import cupy as cp
+
+from ..common import _aliases, _helpers
+from ..common._typing import NestedSequence, SupportsBufferProtocol
+from .._internal import get_xp
+from ._info import __array_namespace_info__
+from ._typing import Array, Device, DType
+
+bool = cp.bool_
+
+# Basic renames
+acos = cp.arccos
+acosh = cp.arccosh
+asin = cp.arcsin
+asinh = cp.arcsinh
+atan = cp.arctan
+atan2 = cp.arctan2
+atanh = cp.arctanh
+bitwise_left_shift = cp.left_shift
+bitwise_invert = cp.invert
+bitwise_right_shift = cp.right_shift
+concat = cp.concatenate
+pow = cp.power
+
+arange = get_xp(cp)(_aliases.arange)
+empty = get_xp(cp)(_aliases.empty)
+empty_like = get_xp(cp)(_aliases.empty_like)
+eye = get_xp(cp)(_aliases.eye)
+full = get_xp(cp)(_aliases.full)
+full_like = get_xp(cp)(_aliases.full_like)
+linspace = get_xp(cp)(_aliases.linspace)
+ones = get_xp(cp)(_aliases.ones)
+ones_like = get_xp(cp)(_aliases.ones_like)
+zeros = get_xp(cp)(_aliases.zeros)
+zeros_like = get_xp(cp)(_aliases.zeros_like)
+UniqueAllResult = get_xp(cp)(_aliases.UniqueAllResult)
+UniqueCountsResult = get_xp(cp)(_aliases.UniqueCountsResult)
+UniqueInverseResult = get_xp(cp)(_aliases.UniqueInverseResult)
+unique_all = get_xp(cp)(_aliases.unique_all)
+unique_counts = get_xp(cp)(_aliases.unique_counts)
+unique_inverse = get_xp(cp)(_aliases.unique_inverse)
+unique_values = get_xp(cp)(_aliases.unique_values)
+std = get_xp(cp)(_aliases.std)
+var = get_xp(cp)(_aliases.var)
+cumulative_sum = get_xp(cp)(_aliases.cumulative_sum)
+cumulative_prod = get_xp(cp)(_aliases.cumulative_prod)
+clip = get_xp(cp)(_aliases.clip)
+permute_dims = get_xp(cp)(_aliases.permute_dims)
+reshape = get_xp(cp)(_aliases.reshape)
+argsort = get_xp(cp)(_aliases.argsort)
+sort = get_xp(cp)(_aliases.sort)
+nonzero = get_xp(cp)(_aliases.nonzero)
+ceil = get_xp(cp)(_aliases.ceil)
+floor = get_xp(cp)(_aliases.floor)
+trunc = get_xp(cp)(_aliases.trunc)
+matmul = get_xp(cp)(_aliases.matmul)
+matrix_transpose = get_xp(cp)(_aliases.matrix_transpose)
+tensordot = get_xp(cp)(_aliases.tensordot)
+sign = get_xp(cp)(_aliases.sign)
+finfo = get_xp(cp)(_aliases.finfo)
+iinfo = get_xp(cp)(_aliases.iinfo)
+
+
+# asarray also adds the copy keyword, which is not present in numpy 1.0.
+def asarray(
+    obj: (
+        Array 
+        | bool | int | float | complex 
+        | NestedSequence[bool | int | float | complex] 
+        | SupportsBufferProtocol
+    ),
+    /,
+    *,
+    dtype: Optional[DType] = None,
+    device: Optional[Device] = None,
+    copy: Optional[bool] = None,
+    **kwargs,
+) -> Array:
+    """
+    Array API compatibility wrapper for asarray().
+
+    See the corresponding documentation in the array library and/or the array API
+    specification for more details.
+    """
+    with cp.cuda.Device(device):
+        if copy is None:
+            return cp.asarray(obj, dtype=dtype, **kwargs)
+        else:
+            res = cp.array(obj, dtype=dtype, copy=copy, **kwargs)
+            if not copy and res is not obj:
+                raise ValueError("Unable to avoid copy while creating an array as requested")
+            return res
+
+
+def astype(
+    x: Array,
+    dtype: DType,
+    /,
+    *,
+    copy: bool = True,
+    device: Optional[Device] = None,
+) -> Array:
+    if device is None:
+        return x.astype(dtype=dtype, copy=copy)
+    out = _helpers.to_device(x.astype(dtype=dtype, copy=False), device)
+    return out.copy() if copy and out is x else out
+
+
+# cupy.count_nonzero does not have keepdims
+def count_nonzero(
+    x: Array,
+    axis=None,
+    keepdims=False
+) -> Array:
+   result = cp.count_nonzero(x, axis)
+   if keepdims:
+       if axis is None:
+            return cp.reshape(result, [1]*x.ndim)
+       return cp.expand_dims(result, axis)
+   return result
+
+
+# take_along_axis: axis defaults to -1 but in cupy (and numpy) axis is a required arg
+def take_along_axis(x: Array, indices: Array, /, *, axis: int = -1):
+    return cp.take_along_axis(x, indices, axis=axis)
+
+
+# These functions are completely new here. If the library already has them
+# (i.e., numpy 2.0), use the library version instead of our wrapper.
+if hasattr(cp, 'vecdot'):
+    vecdot = cp.vecdot
+else:
+    vecdot = get_xp(cp)(_aliases.vecdot)
+
+if hasattr(cp, 'isdtype'):
+    isdtype = cp.isdtype
+else:
+    isdtype = get_xp(cp)(_aliases.isdtype)
+
+if hasattr(cp, 'unstack'):
+    unstack = cp.unstack
+else:
+    unstack = get_xp(cp)(_aliases.unstack)
+
+__all__ = _aliases.__all__ + ['__array_namespace_info__', 'asarray', 'astype',
+                              'acos', 'acosh', 'asin', 'asinh', 'atan',
+                              'atan2', 'atanh', 'bitwise_left_shift',
+                              'bitwise_invert', 'bitwise_right_shift',
+                              'bool', 'concat', 'count_nonzero', 'pow', 'sign',
+                              'take_along_axis']
+
+_all_ignore = ['cp', 'get_xp']
diff --git a/sklearn/externals/array_api_compat/cupy/_info.py b/sklearn/externals/array_api_compat/cupy/_info.py
new file mode 100644
index 0000000000000..78e48a3358cf5
--- /dev/null
+++ b/sklearn/externals/array_api_compat/cupy/_info.py
@@ -0,0 +1,336 @@
+"""
+Array API Inspection namespace
+
+This is the namespace for inspection functions as defined by the array API
+standard. See
+https://data-apis.org/array-api/latest/API_specification/inspection.html for
+more details.
+
+"""
+from cupy import (
+    dtype,
+    cuda,
+    bool_ as bool,
+    intp,
+    int8,
+    int16,
+    int32,
+    int64,
+    uint8,
+    uint16,
+    uint32,
+    uint64,
+    float32,
+    float64,
+    complex64,
+    complex128,
+)
+
+
+class __array_namespace_info__:
+    """
+    Get the array API inspection namespace for CuPy.
+
+    The array API inspection namespace defines the following functions:
+
+    - capabilities()
+    - default_device()
+    - default_dtypes()
+    - dtypes()
+    - devices()
+
+    See
+    https://data-apis.org/array-api/latest/API_specification/inspection.html
+    for more details.
+
+    Returns
+    -------
+    info : ModuleType
+        The array API inspection namespace for CuPy.
+
+    Examples
+    --------
+    >>> info = xp.__array_namespace_info__()
+    >>> info.default_dtypes()
+    {'real floating': cupy.float64,
+     'complex floating': cupy.complex128,
+     'integral': cupy.int64,
+     'indexing': cupy.int64}
+
+    """
+
+    __module__ = 'cupy'
+
+    def capabilities(self):
+        """
+        Return a dictionary of array API library capabilities.
+
+        The resulting dictionary has the following keys:
+
+        - **"boolean indexing"**: boolean indicating whether an array library
+          supports boolean indexing. Always ``True`` for CuPy.
+
+        - **"data-dependent shapes"**: boolean indicating whether an array
+          library supports data-dependent output shapes. Always ``True`` for
+          CuPy.
+
+        See
+        https://data-apis.org/array-api/latest/API_specification/generated/array_api.info.capabilities.html
+        for more details.
+
+        See Also
+        --------
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.dtypes,
+        __array_namespace_info__.devices
+
+        Returns
+        -------
+        capabilities : dict
+            A dictionary of array API library capabilities.
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.capabilities()
+        {'boolean indexing': True,
+         'data-dependent shapes': True,
+         'max dimensions': 64}
+
+        """
+        return {
+            "boolean indexing": True,
+            "data-dependent shapes": True,
+            "max dimensions": 64,
+        }
+
+    def default_device(self):
+        """
+        The default device used for new CuPy arrays.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.dtypes,
+        __array_namespace_info__.devices
+
+        Returns
+        -------
+        device : Device
+            The default device used for new CuPy arrays.
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.default_device()
+        Device(0)
+
+        Notes
+        -----
+        This method returns the static default device when CuPy is initialized.
+        However, the *current* device used by creation functions (``empty`` etc.)
+        can be changed globally or with a context manager.
+
+        See Also
+        --------
+        https://github.com/data-apis/array-api/issues/835
+        """
+        return cuda.Device(0)
+
+    def default_dtypes(self, *, device=None):
+        """
+        The default data types used for new CuPy arrays.
+
+        For CuPy, this always returns the following dictionary:
+
+        - **"real floating"**: ``cupy.float64``
+        - **"complex floating"**: ``cupy.complex128``
+        - **"integral"**: ``cupy.intp``
+        - **"indexing"**: ``cupy.intp``
+
+        Parameters
+        ----------
+        device : str, optional
+            The device to get the default data types for.
+
+        Returns
+        -------
+        dtypes : dict
+            A dictionary describing the default data types used for new CuPy
+            arrays.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.dtypes,
+        __array_namespace_info__.devices
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.default_dtypes()
+        {'real floating': cupy.float64,
+         'complex floating': cupy.complex128,
+         'integral': cupy.int64,
+         'indexing': cupy.int64}
+
+        """
+        # TODO: Does this depend on device?
+        return {
+            "real floating": dtype(float64),
+            "complex floating": dtype(complex128),
+            "integral": dtype(intp),
+            "indexing": dtype(intp),
+        }
+
+    def dtypes(self, *, device=None, kind=None):
+        """
+        The array API data types supported by CuPy.
+
+        Note that this function only returns data types that are defined by
+        the array API.
+
+        Parameters
+        ----------
+        device : str, optional
+            The device to get the data types for.
+        kind : str or tuple of str, optional
+            The kind of data types to return. If ``None``, all data types are
+            returned. If a string, only data types of that kind are returned.
+            If a tuple, a dictionary containing the union of the given kinds
+            is returned. The following kinds are supported:
+
+            - ``'bool'``: boolean data types (i.e., ``bool``).
+            - ``'signed integer'``: signed integer data types (i.e., ``int8``,
+              ``int16``, ``int32``, ``int64``).
+            - ``'unsigned integer'``: unsigned integer data types (i.e.,
+              ``uint8``, ``uint16``, ``uint32``, ``uint64``).
+            - ``'integral'``: integer data types. Shorthand for ``('signed
+              integer', 'unsigned integer')``.
+            - ``'real floating'``: real-valued floating-point data types
+              (i.e., ``float32``, ``float64``).
+            - ``'complex floating'``: complex floating-point data types (i.e.,
+              ``complex64``, ``complex128``).
+            - ``'numeric'``: numeric data types. Shorthand for ``('integral',
+              'real floating', 'complex floating')``.
+
+        Returns
+        -------
+        dtypes : dict
+            A dictionary mapping the names of data types to the corresponding
+            CuPy data types.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.devices
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.dtypes(kind='signed integer')
+        {'int8': cupy.int8,
+         'int16': cupy.int16,
+         'int32': cupy.int32,
+         'int64': cupy.int64}
+
+        """
+        # TODO: Does this depend on device?
+        if kind is None:
+            return {
+                "bool": dtype(bool),
+                "int8": dtype(int8),
+                "int16": dtype(int16),
+                "int32": dtype(int32),
+                "int64": dtype(int64),
+                "uint8": dtype(uint8),
+                "uint16": dtype(uint16),
+                "uint32": dtype(uint32),
+                "uint64": dtype(uint64),
+                "float32": dtype(float32),
+                "float64": dtype(float64),
+                "complex64": dtype(complex64),
+                "complex128": dtype(complex128),
+            }
+        if kind == "bool":
+            return {"bool": bool}
+        if kind == "signed integer":
+            return {
+                "int8": dtype(int8),
+                "int16": dtype(int16),
+                "int32": dtype(int32),
+                "int64": dtype(int64),
+            }
+        if kind == "unsigned integer":
+            return {
+                "uint8": dtype(uint8),
+                "uint16": dtype(uint16),
+                "uint32": dtype(uint32),
+                "uint64": dtype(uint64),
+            }
+        if kind == "integral":
+            return {
+                "int8": dtype(int8),
+                "int16": dtype(int16),
+                "int32": dtype(int32),
+                "int64": dtype(int64),
+                "uint8": dtype(uint8),
+                "uint16": dtype(uint16),
+                "uint32": dtype(uint32),
+                "uint64": dtype(uint64),
+            }
+        if kind == "real floating":
+            return {
+                "float32": dtype(float32),
+                "float64": dtype(float64),
+            }
+        if kind == "complex floating":
+            return {
+                "complex64": dtype(complex64),
+                "complex128": dtype(complex128),
+            }
+        if kind == "numeric":
+            return {
+                "int8": dtype(int8),
+                "int16": dtype(int16),
+                "int32": dtype(int32),
+                "int64": dtype(int64),
+                "uint8": dtype(uint8),
+                "uint16": dtype(uint16),
+                "uint32": dtype(uint32),
+                "uint64": dtype(uint64),
+                "float32": dtype(float32),
+                "float64": dtype(float64),
+                "complex64": dtype(complex64),
+                "complex128": dtype(complex128),
+            }
+        if isinstance(kind, tuple):
+            res = {}
+            for k in kind:
+                res.update(self.dtypes(kind=k))
+            return res
+        raise ValueError(f"unsupported kind: {kind!r}")
+
+    def devices(self):
+        """
+        The devices supported by CuPy.
+
+        Returns
+        -------
+        devices : list[Device]
+            The devices supported by CuPy.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.dtypes
+
+        """
+        return [cuda.Device(i) for i in range(cuda.runtime.getDeviceCount())]
diff --git a/sklearn/externals/array_api_compat/cupy/_typing.py b/sklearn/externals/array_api_compat/cupy/_typing.py
new file mode 100644
index 0000000000000..d8e49ca773dc5
--- /dev/null
+++ b/sklearn/externals/array_api_compat/cupy/_typing.py
@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+__all__ = ["Array", "DType", "Device"]
+_all_ignore = ["cp"]
+
+from typing import TYPE_CHECKING
+
+import cupy as cp
+from cupy import ndarray as Array
+from cupy.cuda.device import Device
+
+if TYPE_CHECKING:
+    # NumPy 1.x on Python 3.10 fails to parse np.dtype[]
+    DType = cp.dtype[
+        cp.intp
+        | cp.int8
+        | cp.int16
+        | cp.int32
+        | cp.int64
+        | cp.uint8
+        | cp.uint16
+        | cp.uint32
+        | cp.uint64
+        | cp.float32
+        | cp.float64
+        | cp.complex64
+        | cp.complex128
+        | cp.bool_
+    ]
+else:
+    DType = cp.dtype
diff --git a/sklearn/externals/array_api_compat/cupy/fft.py b/sklearn/externals/array_api_compat/cupy/fft.py
new file mode 100644
index 0000000000000..307e0f7277710
--- /dev/null
+++ b/sklearn/externals/array_api_compat/cupy/fft.py
@@ -0,0 +1,36 @@
+from cupy.fft import * # noqa: F403
+# cupy.fft doesn't have __all__. If it is added, replace this with
+#
+# from cupy.fft import __all__ as linalg_all
+_n = {}
+exec('from cupy.fft import *', _n)
+del _n['__builtins__']
+fft_all = list(_n)
+del _n
+
+from ..common import _fft
+from .._internal import get_xp
+
+import cupy as cp
+
+fft = get_xp(cp)(_fft.fft)
+ifft = get_xp(cp)(_fft.ifft)
+fftn = get_xp(cp)(_fft.fftn)
+ifftn = get_xp(cp)(_fft.ifftn)
+rfft = get_xp(cp)(_fft.rfft)
+irfft = get_xp(cp)(_fft.irfft)
+rfftn = get_xp(cp)(_fft.rfftn)
+irfftn = get_xp(cp)(_fft.irfftn)
+hfft = get_xp(cp)(_fft.hfft)
+ihfft = get_xp(cp)(_fft.ihfft)
+fftfreq = get_xp(cp)(_fft.fftfreq)
+rfftfreq = get_xp(cp)(_fft.rfftfreq)
+fftshift = get_xp(cp)(_fft.fftshift)
+ifftshift = get_xp(cp)(_fft.ifftshift)
+
+__all__ = fft_all + _fft.__all__
+
+del get_xp
+del cp
+del fft_all
+del _fft
diff --git a/sklearn/externals/array_api_compat/cupy/linalg.py b/sklearn/externals/array_api_compat/cupy/linalg.py
new file mode 100644
index 0000000000000..7fcdd498e0073
--- /dev/null
+++ b/sklearn/externals/array_api_compat/cupy/linalg.py
@@ -0,0 +1,49 @@
+from cupy.linalg import * # noqa: F403
+# cupy.linalg doesn't have __all__. If it is added, replace this with
+#
+# from cupy.linalg import __all__ as linalg_all
+_n = {}
+exec('from cupy.linalg import *', _n)
+del _n['__builtins__']
+linalg_all = list(_n)
+del _n
+
+from ..common import _linalg
+from .._internal import get_xp
+
+import cupy as cp
+
+# These functions are in both the main and linalg namespaces
+from ._aliases import matmul, matrix_transpose, tensordot, vecdot # noqa: F401
+
+cross = get_xp(cp)(_linalg.cross)
+outer = get_xp(cp)(_linalg.outer)
+EighResult = _linalg.EighResult
+QRResult = _linalg.QRResult
+SlogdetResult = _linalg.SlogdetResult
+SVDResult = _linalg.SVDResult
+eigh = get_xp(cp)(_linalg.eigh)
+qr = get_xp(cp)(_linalg.qr)
+slogdet = get_xp(cp)(_linalg.slogdet)
+svd = get_xp(cp)(_linalg.svd)
+cholesky = get_xp(cp)(_linalg.cholesky)
+matrix_rank = get_xp(cp)(_linalg.matrix_rank)
+pinv = get_xp(cp)(_linalg.pinv)
+matrix_norm = get_xp(cp)(_linalg.matrix_norm)
+svdvals = get_xp(cp)(_linalg.svdvals)
+diagonal = get_xp(cp)(_linalg.diagonal)
+trace = get_xp(cp)(_linalg.trace)
+
+# These functions are completely new here. If the library already has them
+# (i.e., numpy 2.0), use the library version instead of our wrapper.
+if hasattr(cp.linalg, 'vector_norm'):
+    vector_norm = cp.linalg.vector_norm
+else:
+    vector_norm = get_xp(cp)(_linalg.vector_norm)
+
+__all__ = linalg_all + _linalg.__all__
+
+del get_xp
+del cp
+del linalg_all
+del _linalg
diff --git a/sklearn/externals/array_api_compat/dask/__init__.py b/sklearn/externals/array_api_compat/dask/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/sklearn/externals/array_api_compat/dask/array/__init__.py b/sklearn/externals/array_api_compat/dask/array/__init__.py
new file mode 100644
index 0000000000000..1e47b9606b774
--- /dev/null
+++ b/sklearn/externals/array_api_compat/dask/array/__init__.py
@@ -0,0 +1,12 @@
+from typing import Final
+
+from dask.array import *  # noqa: F403
+
+# These imports may overwrite names from the import * above.
+from ._aliases import *  # noqa: F403
+
+__array_api_version__: Final = "2024.12"
+
+# See the comment in the numpy __init__.py
+__import__(__package__ + '.linalg')
+__import__(__package__ + '.fft')
diff --git a/sklearn/externals/array_api_compat/dask/array/_aliases.py b/sklearn/externals/array_api_compat/dask/array/_aliases.py
new file mode 100644
index 0000000000000..d43881ab18f1c
--- /dev/null
+++ b/sklearn/externals/array_api_compat/dask/array/_aliases.py
@@ -0,0 +1,376 @@
+# pyright: reportPrivateUsage=false
+# pyright: reportUnknownArgumentType=false
+# pyright: reportUnknownMemberType=false
+# pyright: reportUnknownVariableType=false
+
+from __future__ import annotations
+
+from builtins import bool as py_bool
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from typing_extensions import TypeIs
+
+import dask.array as da
+import numpy as np
+from numpy import bool_ as bool
+from numpy import (
+    can_cast,
+    complex64,
+    complex128,
+    float32,
+    float64,
+    int8,
+    int16,
+    int32,
+    int64,
+    result_type,
+    uint8,
+    uint16,
+    uint32,
+    uint64,
+)
+
+from ..._internal import get_xp
+from ...common import _aliases, _helpers, array_namespace
+from ...common._typing import (
+    Array,
+    Device,
+    DType,
+    NestedSequence,
+    SupportsBufferProtocol,
+)
+from ._info import __array_namespace_info__
+
+isdtype = get_xp(np)(_aliases.isdtype)
+unstack = get_xp(da)(_aliases.unstack)
+
+
+# da.astype doesn't respect copy=True
+def astype(
+    x: Array,
+    dtype: DType,
+    /,
+    *,
+    copy: py_bool = True,
+    device: Device | None = None,
+) -> Array:
+    """
+    Array API compatibility wrapper for astype().
+
+    See the corresponding documentation in the array library and/or the array API
+    specification for more details.
+    """
+    # TODO: respect device keyword?
+    _helpers._check_device(da, device)
+
+    if not copy and dtype == x.dtype:
+        return x
+    x = x.astype(dtype)
+    return x.copy() if copy else x
+
+
+# Common aliases
+
+
+# This arange func is modified from the common one to
+# not pass stop/step as keyword arguments, which will cause
+# an error with dask
+def arange(
+    start: float,
+    /,
+    stop: float | None = None,
+    step: float = 1,
+    *,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    **kwargs: object,
+) -> Array:
+    """
+    Array API compatibility wrapper for arange().
+
+    See the corresponding documentation in the array library and/or the array API
+    specification for more details.
+    """
+    # TODO: respect device keyword?
+    _helpers._check_device(da, device)
+
+    args: list[Any] = [start]
+    if stop is not None:
+        args.append(stop)
+    else:
+        # stop is None, so start is actually stop
+        # prepend the default value for start which is 0
+        args.insert(0, 0)
+    args.append(step)
+
+    return da.arange(*args, dtype=dtype, **kwargs)
+
+
+eye = get_xp(da)(_aliases.eye)
+linspace = get_xp(da)(_aliases.linspace)
+UniqueAllResult = get_xp(da)(_aliases.UniqueAllResult)
+UniqueCountsResult = get_xp(da)(_aliases.UniqueCountsResult)
+UniqueInverseResult = get_xp(da)(_aliases.UniqueInverseResult)
+unique_all = get_xp(da)(_aliases.unique_all)
+unique_counts = get_xp(da)(_aliases.unique_counts)
+unique_inverse = get_xp(da)(_aliases.unique_inverse)
+unique_values = get_xp(da)(_aliases.unique_values)
+permute_dims = get_xp(da)(_aliases.permute_dims)
+std = get_xp(da)(_aliases.std)
+var = get_xp(da)(_aliases.var)
+cumulative_sum = get_xp(da)(_aliases.cumulative_sum)
+cumulative_prod = get_xp(da)(_aliases.cumulative_prod)
+empty = get_xp(da)(_aliases.empty)
+empty_like = get_xp(da)(_aliases.empty_like)
+full = get_xp(da)(_aliases.full)
+full_like = get_xp(da)(_aliases.full_like)
+ones = get_xp(da)(_aliases.ones)
+ones_like = get_xp(da)(_aliases.ones_like)
+zeros = get_xp(da)(_aliases.zeros)
+zeros_like = get_xp(da)(_aliases.zeros_like)
+reshape = get_xp(da)(_aliases.reshape)
+matrix_transpose = get_xp(da)(_aliases.matrix_transpose)
+vecdot = get_xp(da)(_aliases.vecdot)
+nonzero = get_xp(da)(_aliases.nonzero)
+ceil = get_xp(np)(_aliases.ceil)
+floor = get_xp(np)(_aliases.floor)
+trunc = get_xp(np)(_aliases.trunc)
+matmul = get_xp(np)(_aliases.matmul)
+tensordot = get_xp(np)(_aliases.tensordot)
+sign = get_xp(np)(_aliases.sign)
+finfo = get_xp(np)(_aliases.finfo)
+iinfo = get_xp(np)(_aliases.iinfo)
+
+
+# asarray also adds the copy keyword, which is not present in numpy 1.0.
+def asarray(
+    obj: complex | NestedSequence[complex] | Array | SupportsBufferProtocol,
+    /,
+    *,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    copy: py_bool | None = None,
+    **kwargs: object,
+) -> Array:
+    """
+    Array API compatibility wrapper for asarray().
+
+    See the corresponding documentation in the array library and/or the array API
+    specification for more details.
+    """
+    # TODO: respect device keyword?
+    _helpers._check_device(da, device)
+
+    if isinstance(obj, da.Array):
+        if dtype is not None and dtype != obj.dtype:
+            if copy is False:
+                raise ValueError("Unable to avoid copy when changing dtype")
+            obj = obj.astype(dtype)
+        return obj.copy() if copy else obj  # pyright: ignore[reportAttributeAccessIssue]
+
+    if copy is False:
+        raise ValueError(
+            "Unable to avoid copy when converting a non-dask object to dask"
+        )
+
+    # copy=None to be uniform across dask < 2024.12 and >= 2024.12
+    # see https://github.com/dask/dask/pull/11524/
+    obj = np.array(obj, dtype=dtype, copy=True)
+    return da.from_array(obj)
+
+
+# Element wise aliases
+from dask.array import arccos as acos
+from dask.array import arccosh as acosh
+from dask.array import arcsin as asin
+from dask.array import arcsinh as asinh
+from dask.array import arctan as atan
+from dask.array import arctan2 as atan2
+from dask.array import arctanh as atanh
+
+# Other
+from dask.array import concatenate as concat
+from dask.array import invert as bitwise_invert
+from dask.array import left_shift as bitwise_left_shift
+from dask.array import power as pow
+from dask.array import right_shift as bitwise_right_shift
+
+
+# dask.array.clip does not work unless all three arguments are provided.
+# Furthermore, the masking workaround in common._aliases.clip cannot work with
+# dask (meaning uint64 promoting to float64 is going to just be unfixed for
+# now).
+def clip(
+    x: Array,
+    /,
+    min: float | Array | None = None,
+    max: float | Array | None = None,
+) -> Array:
+    """
+    Array API compatibility wrapper for clip().
+
+    See the corresponding documentation in the array library and/or the array API
+    specification for more details.
+    """
+
+    def _isscalar(a: float | Array | None, /) -> TypeIs[float | None]:
+        return a is None or isinstance(a, (int, float))
+
+    min_shape = () if _isscalar(min) else min.shape
+    max_shape = () if _isscalar(max) else max.shape
+
+    # TODO: This won't handle dask unknown shapes
+    result_shape = np.broadcast_shapes(x.shape, min_shape, max_shape)
+
+    if min is not None:
+        min = da.broadcast_to(da.asarray(min), result_shape)
+    if max is not None:
+        max = da.broadcast_to(da.asarray(max), result_shape)
+
+    if min is None and max is None:
+        return da.positive(x)
+
+    if min is None:
+        return astype(da.minimum(x, max), x.dtype)
+    if max is None:
+        return astype(da.maximum(x, min), x.dtype)
+
+    return astype(da.minimum(da.maximum(x, min), max), x.dtype)
+
+
+def _ensure_single_chunk(x: Array, axis: int) -> tuple[Array, Callable[[Array], Array]]:
+    """
+    Make sure that Array is not broken into multiple chunks along axis.
+
+    Returns
+    -------
+    x : Array
+        The input Array with a single chunk along axis.
+    restore : Callable[Array, Array]
+        function to apply to the output to rechunk it back into reasonable chunks
+    """
+    if axis < 0:
+        axis += x.ndim
+    if x.numblocks[axis] < 2:
+        return x, lambda x: x
+
+    # Break chunks on other axes in an attempt to keep chunk size low
+    x = x.rechunk({i: -1 if i == axis else "auto" for i in range(x.ndim)})
+
+    # Rather than reconstructing the original chunks, which can be a
+    # very expensive affair, just break down oversized chunks without
+    # incurring in any transfers over the network.
+    # This has the downside of a risk of overchunking if the array is
+    # then used in operations against other arrays that match the
+    # original chunking pattern.
+    return x, lambda x: x.rechunk()
+
+
+def sort(
+    x: Array,
+    /,
+    *,
+    axis: int = -1,
+    descending: py_bool = False,
+    stable: py_bool = True,
+) -> Array:
+    """
+    Array API compatibility layer around the lack of sort() in Dask.
+
+    Warnings
+    --------
+    This function temporarily rechunks the array along `axis` to a single chunk.
+    This can be extremely inefficient and can lead to out-of-memory errors.
+
+    See the corresponding documentation in the array library and/or the array API
+    specification for more details.
+    """
+    x, restore = _ensure_single_chunk(x, axis)
+
+    meta_xp = array_namespace(x._meta)
+    x = da.map_blocks(
+        meta_xp.sort,
+        x,
+        axis=axis,
+        meta=x._meta,
+        dtype=x.dtype,
+        descending=descending,
+        stable=stable,
+    )
+
+    return restore(x)
+
+
+def argsort(
+    x: Array,
+    /,
+    *,
+    axis: int = -1,
+    descending: py_bool = False,
+    stable: py_bool = True,
+) -> Array:
+    """
+    Array API compatibility layer around the lack of argsort() in Dask.
+
+    See the corresponding documentation in the array library and/or the array API
+    specification for more details.
+
+    Warnings
+    --------
+    This function temporarily rechunks the array along `axis` into a single chunk.
+    This can be extremely inefficient and can lead to out-of-memory errors.
+    """
+    x, restore = _ensure_single_chunk(x, axis)
+
+    meta_xp = array_namespace(x._meta)
+    dtype = meta_xp.argsort(x._meta).dtype
+    meta = meta_xp.astype(x._meta, dtype)
+    x = da.map_blocks(
+        meta_xp.argsort,
+        x,
+        axis=axis,
+        meta=meta,
+        dtype=dtype,
+        descending=descending,
+        stable=stable,
+    )
+
+    return restore(x)
+
+
+# dask.array.count_nonzero does not have keepdims
+def count_nonzero(
+    x: Array,
+    axis: int | None = None,
+    keepdims: py_bool = False,
+) -> Array:
+    result = da.count_nonzero(x, axis)
+    if keepdims:
+        if axis is None:
+            return da.reshape(result, [1] * x.ndim)
+        return da.expand_dims(result, axis)
+    return result
+
+
+__all__ = [
+    "__array_namespace_info__",
+    "count_nonzero",
+    "bool",
+    "int8", "int16", "int32", "int64",
+    "uint8", "uint16", "uint32", "uint64",
+    "float32", "float64",
+    "complex64", "complex128",
+    "asarray", "astype", "can_cast", "result_type",
+    "pow",
+    "concat",
+    "acos", "acosh", "asin", "asinh", "atan", "atan2", "atanh",
+    "bitwise_left_shift", "bitwise_right_shift", "bitwise_invert",
+]  # fmt: skip
+__all__ += _aliases.__all__
+_all_ignore = ["array_namespace", "get_xp", "da", "np"]
+
+
+def __dir__() -> list[str]:
+    return __all__
diff --git a/sklearn/externals/array_api_compat/dask/array/_info.py b/sklearn/externals/array_api_compat/dask/array/_info.py
new file mode 100644
index 0000000000000..9e4d736f99657
--- /dev/null
+++ b/sklearn/externals/array_api_compat/dask/array/_info.py
@@ -0,0 +1,416 @@
+"""
+Array API Inspection namespace
+
+This is the namespace for inspection functions as defined by the array API
+standard. See
+https://data-apis.org/array-api/latest/API_specification/inspection.html for
+more details.
+
+"""
+
+# pyright: reportPrivateUsage=false
+
+from __future__ import annotations
+
+from typing import Literal as L
+from typing import TypeAlias, overload
+
+from numpy import bool_ as bool
+from numpy import (
+    complex64,
+    complex128,
+    dtype,
+    float32,
+    float64,
+    int8,
+    int16,
+    int32,
+    int64,
+    intp,
+    uint8,
+    uint16,
+    uint32,
+    uint64,
+)
+
+from ...common._helpers import _DASK_DEVICE, _dask_device
+from ...common._typing import (
+    Capabilities,
+    DefaultDTypes,
+    DType,
+    DTypeKind,
+    DTypesAll,
+    DTypesAny,
+    DTypesBool,
+    DTypesComplex,
+    DTypesIntegral,
+    DTypesNumeric,
+    DTypesReal,
+    DTypesSigned,
+    DTypesUnsigned,
+)
+
+_Device: TypeAlias = L["cpu"] | _dask_device
+
+
+class __array_namespace_info__:
+    """
+    Get the array API inspection namespace for Dask.
+
+    The array API inspection namespace defines the following functions:
+
+    - capabilities()
+    - default_device()
+    - default_dtypes()
+    - dtypes()
+    - devices()
+
+    See
+    https://data-apis.org/array-api/latest/API_specification/inspection.html
+    for more details.
+
+    Returns
+    -------
+    info : ModuleType
+        The array API inspection namespace for Dask.
+
+    Examples
+    --------
+    >>> info = xp.__array_namespace_info__()
+    >>> info.default_dtypes()
+    {'real floating': dask.float64,
+     'complex floating': dask.complex128,
+     'integral': dask.int64,
+     'indexing': dask.int64}
+
+    """
+
+    __module__ = "dask.array"
+
+    def capabilities(self) -> Capabilities:
+        """
+        Return a dictionary of array API library capabilities.
+
+        The resulting dictionary has the following keys:
+
+        - **"boolean indexing"**: boolean indicating whether an array library
+          supports boolean indexing.
+
+          Dask support boolean indexing as long as both the index
+          and the indexed arrays have known shapes.
+          Note however that the output .shape and .size properties
+          will contain a non-compliant math.nan instead of None.
+
+        - **"data-dependent shapes"**: boolean indicating whether an array
+          library supports data-dependent output shapes.
+
+          Dask implements unique_values et.al.
+          Note however that the output .shape and .size properties
+          will contain a non-compliant math.nan instead of None.
+
+        - **"max dimensions"**: integer indicating the maximum number of
+          dimensions supported by the array library.
+
+        See
+        https://data-apis.org/array-api/latest/API_specification/generated/array_api.info.capabilities.html
+        for more details.
+
+        See Also
+        --------
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.dtypes,
+        __array_namespace_info__.devices
+
+        Returns
+        -------
+        capabilities : dict
+            A dictionary of array API library capabilities.
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.capabilities()
+        {'boolean indexing': True,
+         'data-dependent shapes': True,
+         'max dimensions': 64}
+
+        """
+        return {
+            "boolean indexing": True,
+            "data-dependent shapes": True,
+            "max dimensions": 64,
+        }
+
+    def default_device(self) -> L["cpu"]:
+        """
+        The default device used for new Dask arrays.
+
+        For Dask, this always returns ``'cpu'``.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.dtypes,
+        __array_namespace_info__.devices
+
+        Returns
+        -------
+        device : Device
+            The default device used for new Dask arrays.
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.default_device()
+        'cpu'
+
+        """
+        return "cpu"
+
+    def default_dtypes(self, /, *, device: _Device | None = None) -> DefaultDTypes:
+        """
+        The default data types used for new Dask arrays.
+
+        For Dask, this always returns the following dictionary:
+
+        - **"real floating"**: ``numpy.float64``
+        - **"complex floating"**: ``numpy.complex128``
+        - **"integral"**: ``numpy.intp``
+        - **"indexing"**: ``numpy.intp``
+
+        Parameters
+        ----------
+        device : str, optional
+            The device to get the default data types for.
+
+        Returns
+        -------
+        dtypes : dict
+            A dictionary describing the default data types used for new Dask
+            arrays.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.dtypes,
+        __array_namespace_info__.devices
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.default_dtypes()
+        {'real floating': dask.float64,
+         'complex floating': dask.complex128,
+         'integral': dask.int64,
+         'indexing': dask.int64}
+
+        """
+        if device not in ["cpu", _DASK_DEVICE, None]:
+            raise ValueError(
+                f'Device not understood. Only "cpu" or _DASK_DEVICE is allowed, '
+                f"but received: {device!r}"
+            )
+        return {
+            "real floating": dtype(float64),
+            "complex floating": dtype(complex128),
+            "integral": dtype(intp),
+            "indexing": dtype(intp),
+        }
+
+    @overload
+    def dtypes(
+        self, /, *, device: _Device | None = None, kind: None = None
+    ) -> DTypesAll: ...
+    @overload
+    def dtypes(
+        self, /, *, device: _Device | None = None, kind: L["bool"]
+    ) -> DTypesBool: ...
+    @overload
+    def dtypes(
+        self, /, *, device: _Device | None = None, kind: L["signed integer"]
+    ) -> DTypesSigned: ...
+    @overload
+    def dtypes(
+        self, /, *, device: _Device | None = None, kind: L["unsigned integer"]
+    ) -> DTypesUnsigned: ...
+    @overload
+    def dtypes(
+        self, /, *, device: _Device | None = None, kind: L["integral"]
+    ) -> DTypesIntegral: ...
+    @overload
+    def dtypes(
+        self, /, *, device: _Device | None = None, kind: L["real floating"]
+    ) -> DTypesReal: ...
+    @overload
+    def dtypes(
+        self, /, *, device: _Device | None = None, kind: L["complex floating"]
+    ) -> DTypesComplex: ...
+    @overload
+    def dtypes(
+        self, /, *, device: _Device | None = None, kind: L["numeric"]
+    ) -> DTypesNumeric: ...
+    def dtypes(
+        self, /, *, device: _Device | None = None, kind: DTypeKind | None = None
+    ) -> DTypesAny:
+        """
+        The array API data types supported by Dask.
+
+        Note that this function only returns data types that are defined by
+        the array API.
+
+        Parameters
+        ----------
+        device : str, optional
+            The device to get the data types for.
+        kind : str or tuple of str, optional
+            The kind of data types to return. If ``None``, all data types are
+            returned. If a string, only data types of that kind are returned.
+            If a tuple, a dictionary containing the union of the given kinds
+            is returned. The following kinds are supported:
+
+            - ``'bool'``: boolean data types (i.e., ``bool``).
+            - ``'signed integer'``: signed integer data types (i.e., ``int8``,
+              ``int16``, ``int32``, ``int64``).
+            - ``'unsigned integer'``: unsigned integer data types (i.e.,
+              ``uint8``, ``uint16``, ``uint32``, ``uint64``).
+            - ``'integral'``: integer data types. Shorthand for ``('signed
+              integer', 'unsigned integer')``.
+            - ``'real floating'``: real-valued floating-point data types
+              (i.e., ``float32``, ``float64``).
+            - ``'complex floating'``: complex floating-point data types (i.e.,
+              ``complex64``, ``complex128``).
+            - ``'numeric'``: numeric data types. Shorthand for ``('integral',
+              'real floating', 'complex floating')``.
+
+        Returns
+        -------
+        dtypes : dict
+            A dictionary mapping the names of data types to the corresponding
+            Dask data types.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.devices
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.dtypes(kind='signed integer')
+        {'int8': dask.int8,
+         'int16': dask.int16,
+         'int32': dask.int32,
+         'int64': dask.int64}
+
+        """
+        if device not in ["cpu", _DASK_DEVICE, None]:
+            raise ValueError(
+                'Device not understood. Only "cpu" or _DASK_DEVICE is allowed, but received:'
+                f" {device}"
+            )
+        if kind is None:
+            return {
+                "bool": dtype(bool),
+                "int8": dtype(int8),
+                "int16": dtype(int16),
+                "int32": dtype(int32),
+                "int64": dtype(int64),
+                "uint8": dtype(uint8),
+                "uint16": dtype(uint16),
+                "uint32": dtype(uint32),
+                "uint64": dtype(uint64),
+                "float32": dtype(float32),
+                "float64": dtype(float64),
+                "complex64": dtype(complex64),
+                "complex128": dtype(complex128),
+            }
+        if kind == "bool":
+            return {"bool": bool}
+        if kind == "signed integer":
+            return {
+                "int8": dtype(int8),
+                "int16": dtype(int16),
+                "int32": dtype(int32),
+                "int64": dtype(int64),
+            }
+        if kind == "unsigned integer":
+            return {
+                "uint8": dtype(uint8),
+                "uint16": dtype(uint16),
+                "uint32": dtype(uint32),
+                "uint64": dtype(uint64),
+            }
+        if kind == "integral":
+            return {
+                "int8": dtype(int8),
+                "int16": dtype(int16),
+                "int32": dtype(int32),
+                "int64": dtype(int64),
+                "uint8": dtype(uint8),
+                "uint16": dtype(uint16),
+                "uint32": dtype(uint32),
+                "uint64": dtype(uint64),
+            }
+        if kind == "real floating":
+            return {
+                "float32": dtype(float32),
+                "float64": dtype(float64),
+            }
+        if kind == "complex floating":
+            return {
+                "complex64": dtype(complex64),
+                "complex128": dtype(complex128),
+            }
+        if kind == "numeric":
+            return {
+                "int8": dtype(int8),
+                "int16": dtype(int16),
+                "int32": dtype(int32),
+                "int64": dtype(int64),
+                "uint8": dtype(uint8),
+                "uint16": dtype(uint16),
+                "uint32": dtype(uint32),
+                "uint64": dtype(uint64),
+                "float32": dtype(float32),
+                "float64": dtype(float64),
+                "complex64": dtype(complex64),
+                "complex128": dtype(complex128),
+            }
+        if isinstance(kind, tuple):  # type: ignore[reportUnnecessaryIsinstanceCall]
+            res: dict[str, DType] = {}
+            for k in kind:
+                res.update(self.dtypes(kind=k))
+            return res
+        raise ValueError(f"unsupported kind: {kind!r}")
+
+    def devices(self) -> list[_Device]:
+        """
+        The devices supported by Dask.
+
+        For Dask, this always returns ``['cpu', DASK_DEVICE]``.
+
+        Returns
+        -------
+        devices : list[Device]
+            The devices supported by Dask.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.dtypes
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.devices()
+        ['cpu', DASK_DEVICE]
+
+        """
+        return ["cpu", _DASK_DEVICE]
diff --git a/sklearn/externals/array_api_compat/dask/array/fft.py b/sklearn/externals/array_api_compat/dask/array/fft.py
new file mode 100644
index 0000000000000..3f40dffe7abd5
--- /dev/null
+++ b/sklearn/externals/array_api_compat/dask/array/fft.py
@@ -0,0 +1,21 @@
+from dask.array.fft import * # noqa: F403
+# dask.array.fft doesn't have __all__. If it is added, replace this with
+#
+# from dask.array.fft import __all__ as linalg_all
+_n = {}
+exec('from dask.array.fft import *', _n)
+for k in ("__builtins__", "Sequence", "annotations", "warnings"):
+    _n.pop(k, None)
+fft_all = list(_n)
+del _n, k
+
+from ...common import _fft
+from ..._internal import get_xp
+
+import dask.array as da
+
+fftfreq = get_xp(da)(_fft.fftfreq)
+rfftfreq = get_xp(da)(_fft.rfftfreq)
+
+__all__ = fft_all + ["fftfreq", "rfftfreq"]
+_all_ignore = ["da", "fft_all", "get_xp", "warnings"]
diff --git a/sklearn/externals/array_api_compat/dask/array/linalg.py b/sklearn/externals/array_api_compat/dask/array/linalg.py
new file mode 100644
index 0000000000000..0825386ed5dc3
--- /dev/null
+++ b/sklearn/externals/array_api_compat/dask/array/linalg.py
@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from typing import Literal
+
+import dask.array as da
+
+# The `matmul` and `tensordot` functions are in both the main and linalg namespaces
+from dask.array import matmul, outer, tensordot
+
+# Exports
+from dask.array.linalg import *  # noqa: F403
+
+from ..._internal import get_xp
+from ...common import _linalg
+from ...common._typing import Array as _Array
+from ._aliases import matrix_transpose, vecdot
+
+# dask.array.linalg doesn't have __all__. If it is added, replace this with
+#
+# from dask.array.linalg import __all__ as linalg_all
+_n = {}
+exec('from dask.array.linalg import *', _n)
+for k in ('__builtins__', 'annotations', 'operator', 'warnings', 'Array'):
+    _n.pop(k, None)
+linalg_all = list(_n)
+del _n, k
+
+EighResult = _linalg.EighResult
+QRResult = _linalg.QRResult
+SlogdetResult = _linalg.SlogdetResult
+SVDResult = _linalg.SVDResult
+# TODO: use the QR wrapper once dask
+# supports the mode keyword on QR
+# https://github.com/dask/dask/issues/10388
+#qr = get_xp(da)(_linalg.qr)
+def qr(
+    x: _Array,
+    mode: Literal["reduced", "complete"] = "reduced",
+    **kwargs: object,
+) -> QRResult:
+    if mode != "reduced":
+        raise ValueError("dask arrays only support using mode='reduced'")
+    return QRResult(*da.linalg.qr(x, **kwargs))
+trace = get_xp(da)(_linalg.trace)
+cholesky = get_xp(da)(_linalg.cholesky)
+matrix_rank = get_xp(da)(_linalg.matrix_rank)
+matrix_norm = get_xp(da)(_linalg.matrix_norm)
+
+
+# Wrap the svd functions to not pass full_matrices to dask
+# when full_matrices=False (as that is the default behavior for dask),
+# and dask doesn't have the full_matrices keyword
+def svd(x: _Array, full_matrices: bool = True, **kwargs) -> SVDResult:
+    if full_matrices:
+        raise ValueError("full_matrics=True is not supported by dask.")
+    return da.linalg.svd(x, coerce_signs=False, **kwargs)
+
+def svdvals(x: _Array) -> _Array:
+    # TODO: can't avoid computing U or V for dask
+    _, s, _ =  svd(x)
+    return s
+
+vector_norm = get_xp(da)(_linalg.vector_norm)
+diagonal = get_xp(da)(_linalg.diagonal)
+
+__all__ = linalg_all + ["trace", "outer", "matmul", "tensordot",
+                        "matrix_transpose", "vecdot", "EighResult",
+                        "QRResult", "SlogdetResult", "SVDResult", "qr",
+                        "cholesky", "matrix_rank", "matrix_norm", "svdvals",
+                        "vector_norm", "diagonal"]
+
+_all_ignore = ['get_xp', 'da', 'linalg_all', 'warnings']
diff --git a/sklearn/externals/array_api_compat/numpy/__init__.py b/sklearn/externals/array_api_compat/numpy/__init__.py
new file mode 100644
index 0000000000000..3e138f53db006
--- /dev/null
+++ b/sklearn/externals/array_api_compat/numpy/__init__.py
@@ -0,0 +1,28 @@
+# ruff: noqa: PLC0414
+from typing import Final
+
+from numpy import *  # noqa: F403  # pyright: ignore[reportWildcardImportFromLibrary]
+
+# from numpy import * doesn't overwrite these builtin names
+from numpy import abs as abs
+from numpy import max as max
+from numpy import min as min
+from numpy import round as round
+
+# These imports may overwrite names from the import * above.
+from ._aliases import *  # noqa: F403
+
+# Don't know why, but we have to do an absolute import to import linalg. If we
+# instead do
+#
+# from . import linalg
+#
+# It doesn't overwrite np.linalg from above. The import is generated
+# dynamically so that the library can be vendored.
+__import__(__package__ + ".linalg")
+
+__import__(__package__ + ".fft")
+
+from .linalg import matrix_transpose, vecdot  # type: ignore[no-redef]  # noqa: F401
+
+__array_api_version__: Final = "2024.12"
diff --git a/sklearn/externals/array_api_compat/numpy/_aliases.py b/sklearn/externals/array_api_compat/numpy/_aliases.py
new file mode 100644
index 0000000000000..a1aee5c0df796
--- /dev/null
+++ b/sklearn/externals/array_api_compat/numpy/_aliases.py
@@ -0,0 +1,190 @@
+# pyright: reportPrivateUsage=false
+from __future__ import annotations
+
+from builtins import bool as py_bool
+from typing import TYPE_CHECKING, Any, Literal, TypeAlias, cast
+
+import numpy as np
+
+from .._internal import get_xp
+from ..common import _aliases, _helpers
+from ..common._typing import NestedSequence, SupportsBufferProtocol
+from ._info import __array_namespace_info__
+from ._typing import Array, Device, DType
+
+if TYPE_CHECKING:
+    from typing_extensions import Buffer, TypeIs
+
+# The values of the `_CopyMode` enum can be either `False`, `True`, or `2`:
+# https://github.com/numpy/numpy/blob/5a8a6a79d9c2fff8f07dcab5d41e14f8508d673f/numpy/_globals.pyi#L7-L10
+_Copy: TypeAlias = py_bool | Literal[2] | np._CopyMode
+
+bool = np.bool_
+
+# Basic renames
+acos = np.arccos
+acosh = np.arccosh
+asin = np.arcsin
+asinh = np.arcsinh
+atan = np.arctan
+atan2 = np.arctan2
+atanh = np.arctanh
+bitwise_left_shift = np.left_shift
+bitwise_invert = np.invert
+bitwise_right_shift = np.right_shift
+concat = np.concatenate
+pow = np.power
+
+arange = get_xp(np)(_aliases.arange)
+empty = get_xp(np)(_aliases.empty)
+empty_like = get_xp(np)(_aliases.empty_like)
+eye = get_xp(np)(_aliases.eye)
+full = get_xp(np)(_aliases.full)
+full_like = get_xp(np)(_aliases.full_like)
+linspace = get_xp(np)(_aliases.linspace)
+ones = get_xp(np)(_aliases.ones)
+ones_like = get_xp(np)(_aliases.ones_like)
+zeros = get_xp(np)(_aliases.zeros)
+zeros_like = get_xp(np)(_aliases.zeros_like)
+UniqueAllResult = get_xp(np)(_aliases.UniqueAllResult)
+UniqueCountsResult = get_xp(np)(_aliases.UniqueCountsResult)
+UniqueInverseResult = get_xp(np)(_aliases.UniqueInverseResult)
+unique_all = get_xp(np)(_aliases.unique_all)
+unique_counts = get_xp(np)(_aliases.unique_counts)
+unique_inverse = get_xp(np)(_aliases.unique_inverse)
+unique_values = get_xp(np)(_aliases.unique_values)
+std = get_xp(np)(_aliases.std)
+var = get_xp(np)(_aliases.var)
+cumulative_sum = get_xp(np)(_aliases.cumulative_sum)
+cumulative_prod = get_xp(np)(_aliases.cumulative_prod)
+clip = get_xp(np)(_aliases.clip)
+permute_dims = get_xp(np)(_aliases.permute_dims)
+reshape = get_xp(np)(_aliases.reshape)
+argsort = get_xp(np)(_aliases.argsort)
+sort = get_xp(np)(_aliases.sort)
+nonzero = get_xp(np)(_aliases.nonzero)
+ceil = get_xp(np)(_aliases.ceil)
+floor = get_xp(np)(_aliases.floor)
+trunc = get_xp(np)(_aliases.trunc)
+matmul = get_xp(np)(_aliases.matmul)
+matrix_transpose = get_xp(np)(_aliases.matrix_transpose)
+tensordot = get_xp(np)(_aliases.tensordot)
+sign = get_xp(np)(_aliases.sign)
+finfo = get_xp(np)(_aliases.finfo)
+iinfo = get_xp(np)(_aliases.iinfo)
+
+
+def _supports_buffer_protocol(obj: object) -> TypeIs[Buffer]:  # pyright: ignore[reportUnusedFunction]
+    try:
+        memoryview(obj)  # pyright: ignore[reportArgumentType]
+    except TypeError:
+        return False
+    return True
+
+
+# asarray also adds the copy keyword, which is not present in numpy 1.0.
+# asarray() is different enough between numpy, cupy, and dask, the logic
+# complicated enough that it's easier to define it separately for each module
+# rather than trying to combine everything into one function in common/
+def asarray(
+    obj: Array | complex | NestedSequence[complex] | SupportsBufferProtocol,
+    /,
+    *,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    copy: _Copy | None = None,
+    **kwargs: Any,
+) -> Array:
+    """
+    Array API compatibility wrapper for asarray().
+
+    See the corresponding documentation in the array library and/or the array API
+    specification for more details.
+    """
+    _helpers._check_device(np, device)
+
+    if copy is None:
+        copy = np._CopyMode.IF_NEEDED
+    elif copy is False:
+        copy = np._CopyMode.NEVER
+    elif copy is True:
+        copy = np._CopyMode.ALWAYS
+
+    return np.array(obj, copy=copy, dtype=dtype, **kwargs)  # pyright: ignore
+
+
+def astype(
+    x: Array,
+    dtype: DType,
+    /,
+    *,
+    copy: py_bool = True,
+    device: Device | None = None,
+) -> Array:
+    _helpers._check_device(np, device)
+    return x.astype(dtype=dtype, copy=copy)
+
+
+# count_nonzero returns a python int for axis=None and keepdims=False
+# https://github.com/numpy/numpy/issues/17562
+def count_nonzero(
+    x: Array,
+    axis: int | tuple[int, ...] | None = None,
+    keepdims: py_bool = False,
+) -> Array:
+    # NOTE: this is currently incorrectly typed in numpy, but will be fixed in
+    # numpy 2.2.5 and 2.3.0: https://github.com/numpy/numpy/pull/28750
+    result = cast("Any", np.count_nonzero(x, axis=axis, keepdims=keepdims))  # pyright: ignore[reportArgumentType, reportCallIssue]
+    if axis is None and not keepdims:
+        return np.asarray(result)
+    return result
+
+
+# take_along_axis: axis defaults to -1 but in numpy axis is a required arg
+def take_along_axis(x: Array, indices: Array, /, *, axis: int = -1):
+    return np.take_along_axis(x, indices, axis=axis)
+
+
+# These functions are completely new here. If the library already has them
+# (i.e., numpy 2.0), use the library version instead of our wrapper.
+if hasattr(np, "vecdot"):
+    vecdot = np.vecdot
+else:
+    vecdot = get_xp(np)(_aliases.vecdot)
+
+if hasattr(np, "isdtype"):
+    isdtype = np.isdtype
+else:
+    isdtype = get_xp(np)(_aliases.isdtype)
+
+if hasattr(np, "unstack"):
+    unstack = np.unstack
+else:
+    unstack = get_xp(np)(_aliases.unstack)
+
+__all__ = [
+    "__array_namespace_info__",
+    "asarray",
+    "astype",
+    "acos",
+    "acosh",
+    "asin",
+    "asinh",
+    "atan",
+    "atan2",
+    "atanh",
+    "bitwise_left_shift",
+    "bitwise_invert",
+    "bitwise_right_shift",
+    "bool",
+    "concat",
+    "count_nonzero",
+    "pow",
+    "take_along_axis"
+]
+__all__ += _aliases.__all__
+_all_ignore = ["np", "get_xp"]
+
+
+def __dir__() -> list[str]:
+    return __all__
diff --git a/sklearn/externals/array_api_compat/numpy/_info.py b/sklearn/externals/array_api_compat/numpy/_info.py
new file mode 100644
index 0000000000000..f307f62c5d5d5
--- /dev/null
+++ b/sklearn/externals/array_api_compat/numpy/_info.py
@@ -0,0 +1,366 @@
+"""
+Array API Inspection namespace
+
+This is the namespace for inspection functions as defined by the array API
+standard. See
+https://data-apis.org/array-api/latest/API_specification/inspection.html for
+more details.
+
+"""
+from __future__ import annotations
+
+from numpy import bool_ as bool
+from numpy import (
+    complex64,
+    complex128,
+    dtype,
+    float32,
+    float64,
+    int8,
+    int16,
+    int32,
+    int64,
+    intp,
+    uint8,
+    uint16,
+    uint32,
+    uint64,
+)
+
+from ._typing import Device, DType
+
+
+class __array_namespace_info__:
+    """
+    Get the array API inspection namespace for NumPy.
+
+    The array API inspection namespace defines the following functions:
+
+    - capabilities()
+    - default_device()
+    - default_dtypes()
+    - dtypes()
+    - devices()
+
+    See
+    https://data-apis.org/array-api/latest/API_specification/inspection.html
+    for more details.
+
+    Returns
+    -------
+    info : ModuleType
+        The array API inspection namespace for NumPy.
+
+    Examples
+    --------
+    >>> info = np.__array_namespace_info__()
+    >>> info.default_dtypes()
+    {'real floating': numpy.float64,
+     'complex floating': numpy.complex128,
+     'integral': numpy.int64,
+     'indexing': numpy.int64}
+
+    """
+
+    __module__ = 'numpy'
+
+    def capabilities(self):
+        """
+        Return a dictionary of array API library capabilities.
+
+        The resulting dictionary has the following keys:
+
+        - **"boolean indexing"**: boolean indicating whether an array library
+          supports boolean indexing. Always ``True`` for NumPy.
+
+        - **"data-dependent shapes"**: boolean indicating whether an array
+          library supports data-dependent output shapes. Always ``True`` for
+          NumPy.
+
+        See
+        https://data-apis.org/array-api/latest/API_specification/generated/array_api.info.capabilities.html
+        for more details.
+
+        See Also
+        --------
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.dtypes,
+        __array_namespace_info__.devices
+
+        Returns
+        -------
+        capabilities : dict
+            A dictionary of array API library capabilities.
+
+        Examples
+        --------
+        >>> info = np.__array_namespace_info__()
+        >>> info.capabilities()
+        {'boolean indexing': True,
+         'data-dependent shapes': True,
+         'max dimensions': 64}
+
+        """
+        return {
+            "boolean indexing": True,
+            "data-dependent shapes": True,
+            "max dimensions": 64,
+        }
+
+    def default_device(self):
+        """
+        The default device used for new NumPy arrays.
+
+        For NumPy, this always returns ``'cpu'``.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.dtypes,
+        __array_namespace_info__.devices
+
+        Returns
+        -------
+        device : Device
+            The default device used for new NumPy arrays.
+
+        Examples
+        --------
+        >>> info = np.__array_namespace_info__()
+        >>> info.default_device()
+        'cpu'
+
+        """
+        return "cpu"
+
+    def default_dtypes(
+        self,
+        *,
+        device: Device | None = None,
+    ) -> dict[str, dtype[intp | float64 | complex128]]:
+        """
+        The default data types used for new NumPy arrays.
+
+        For NumPy, this always returns the following dictionary:
+
+        - **"real floating"**: ``numpy.float64``
+        - **"complex floating"**: ``numpy.complex128``
+        - **"integral"**: ``numpy.intp``
+        - **"indexing"**: ``numpy.intp``
+
+        Parameters
+        ----------
+        device : str, optional
+            The device to get the default data types for. For NumPy, only
+            ``'cpu'`` is allowed.
+
+        Returns
+        -------
+        dtypes : dict
+            A dictionary describing the default data types used for new NumPy
+            arrays.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.dtypes,
+        __array_namespace_info__.devices
+
+        Examples
+        --------
+        >>> info = np.__array_namespace_info__()
+        >>> info.default_dtypes()
+        {'real floating': numpy.float64,
+         'complex floating': numpy.complex128,
+         'integral': numpy.int64,
+         'indexing': numpy.int64}
+
+        """
+        if device not in ["cpu", None]:
+            raise ValueError(
+                'Device not understood. Only "cpu" is allowed, but received:'
+                f' {device}'
+            )
+        return {
+            "real floating": dtype(float64),
+            "complex floating": dtype(complex128),
+            "integral": dtype(intp),
+            "indexing": dtype(intp),
+        }
+
+    def dtypes(
+        self,
+        *,
+        device: Device | None = None,
+        kind: str | tuple[str, ...] | None = None,
+    ) -> dict[str, DType]:
+        """
+        The array API data types supported by NumPy.
+
+        Note that this function only returns data types that are defined by
+        the array API.
+
+        Parameters
+        ----------
+        device : str, optional
+            The device to get the data types for. For NumPy, only ``'cpu'`` is
+            allowed.
+        kind : str or tuple of str, optional
+            The kind of data types to return. If ``None``, all data types are
+            returned. If a string, only data types of that kind are returned.
+            If a tuple, a dictionary containing the union of the given kinds
+            is returned. The following kinds are supported:
+
+            - ``'bool'``: boolean data types (i.e., ``bool``).
+            - ``'signed integer'``: signed integer data types (i.e., ``int8``,
+              ``int16``, ``int32``, ``int64``).
+            - ``'unsigned integer'``: unsigned integer data types (i.e.,
+              ``uint8``, ``uint16``, ``uint32``, ``uint64``).
+            - ``'integral'``: integer data types. Shorthand for ``('signed
+              integer', 'unsigned integer')``.
+            - ``'real floating'``: real-valued floating-point data types
+              (i.e., ``float32``, ``float64``).
+            - ``'complex floating'``: complex floating-point data types (i.e.,
+              ``complex64``, ``complex128``).
+            - ``'numeric'``: numeric data types. Shorthand for ``('integral',
+              'real floating', 'complex floating')``.
+
+        Returns
+        -------
+        dtypes : dict
+            A dictionary mapping the names of data types to the corresponding
+            NumPy data types.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.devices
+
+        Examples
+        --------
+        >>> info = np.__array_namespace_info__()
+        >>> info.dtypes(kind='signed integer')
+        {'int8': numpy.int8,
+         'int16': numpy.int16,
+         'int32': numpy.int32,
+         'int64': numpy.int64}
+
+        """
+        if device not in ["cpu", None]:
+            raise ValueError(
+                'Device not understood. Only "cpu" is allowed, but received:'
+                f' {device}'
+            )
+        if kind is None:
+            return {
+                "bool": dtype(bool),
+                "int8": dtype(int8),
+                "int16": dtype(int16),
+                "int32": dtype(int32),
+                "int64": dtype(int64),
+                "uint8": dtype(uint8),
+                "uint16": dtype(uint16),
+                "uint32": dtype(uint32),
+                "uint64": dtype(uint64),
+                "float32": dtype(float32),
+                "float64": dtype(float64),
+                "complex64": dtype(complex64),
+                "complex128": dtype(complex128),
+            }
+        if kind == "bool":
+            return {"bool": dtype(bool)}
+        if kind == "signed integer":
+            return {
+                "int8": dtype(int8),
+                "int16": dtype(int16),
+                "int32": dtype(int32),
+                "int64": dtype(int64),
+            }
+        if kind == "unsigned integer":
+            return {
+                "uint8": dtype(uint8),
+                "uint16": dtype(uint16),
+                "uint32": dtype(uint32),
+                "uint64": dtype(uint64),
+            }
+        if kind == "integral":
+            return {
+                "int8": dtype(int8),
+                "int16": dtype(int16),
+                "int32": dtype(int32),
+                "int64": dtype(int64),
+                "uint8": dtype(uint8),
+                "uint16": dtype(uint16),
+                "uint32": dtype(uint32),
+                "uint64": dtype(uint64),
+            }
+        if kind == "real floating":
+            return {
+                "float32": dtype(float32),
+                "float64": dtype(float64),
+            }
+        if kind == "complex floating":
+            return {
+                "complex64": dtype(complex64),
+                "complex128": dtype(complex128),
+            }
+        if kind == "numeric":
+            return {
+                "int8": dtype(int8),
+                "int16": dtype(int16),
+                "int32": dtype(int32),
+                "int64": dtype(int64),
+                "uint8": dtype(uint8),
+                "uint16": dtype(uint16),
+                "uint32": dtype(uint32),
+                "uint64": dtype(uint64),
+                "float32": dtype(float32),
+                "float64": dtype(float64),
+                "complex64": dtype(complex64),
+                "complex128": dtype(complex128),
+            }
+        if isinstance(kind, tuple):
+            res: dict[str, DType] = {}
+            for k in kind:
+                res.update(self.dtypes(kind=k))
+            return res
+        raise ValueError(f"unsupported kind: {kind!r}")
+
+    def devices(self) -> list[Device]:
+        """
+        The devices supported by NumPy.
+
+        For NumPy, this always returns ``['cpu']``.
+
+        Returns
+        -------
+        devices : list[Device]
+            The devices supported by NumPy.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.dtypes
+
+        Examples
+        --------
+        >>> info = np.__array_namespace_info__()
+        >>> info.devices()
+        ['cpu']
+
+        """
+        return ["cpu"]
+
+
+__all__ = ["__array_namespace_info__"]
+
+
+def __dir__() -> list[str]:
+    return __all__
diff --git a/sklearn/externals/array_api_compat/numpy/_typing.py b/sklearn/externals/array_api_compat/numpy/_typing.py
new file mode 100644
index 0000000000000..e771c788bbcab
--- /dev/null
+++ b/sklearn/externals/array_api_compat/numpy/_typing.py
@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Literal, TypeAlias
+
+import numpy as np
+
+Device: TypeAlias = Literal["cpu"]
+
+if TYPE_CHECKING:
+
+    # NumPy 1.x on Python 3.10 fails to parse np.dtype[]
+    DType: TypeAlias = np.dtype[
+        np.bool_
+        | np.integer[Any]
+        | np.float32
+        | np.float64
+        | np.complex64
+        | np.complex128
+    ]
+    Array: TypeAlias = np.ndarray[Any, DType]
+else:
+    DType: TypeAlias = np.dtype
+    Array: TypeAlias = np.ndarray
+
+__all__ = ["Array", "DType", "Device"]
+_all_ignore = ["np"]
+
+
+def __dir__() -> list[str]:
+    return __all__
diff --git a/sklearn/externals/array_api_compat/numpy/fft.py b/sklearn/externals/array_api_compat/numpy/fft.py
new file mode 100644
index 0000000000000..06875f00b4312
--- /dev/null
+++ b/sklearn/externals/array_api_compat/numpy/fft.py
@@ -0,0 +1,35 @@
+import numpy as np
+from numpy.fft import __all__ as fft_all
+from numpy.fft import fft2, ifft2, irfft2, rfft2
+
+from .._internal import get_xp
+from ..common import _fft
+
+fft = get_xp(np)(_fft.fft)
+ifft = get_xp(np)(_fft.ifft)
+fftn = get_xp(np)(_fft.fftn)
+ifftn = get_xp(np)(_fft.ifftn)
+rfft = get_xp(np)(_fft.rfft)
+irfft = get_xp(np)(_fft.irfft)
+rfftn = get_xp(np)(_fft.rfftn)
+irfftn = get_xp(np)(_fft.irfftn)
+hfft = get_xp(np)(_fft.hfft)
+ihfft = get_xp(np)(_fft.ihfft)
+fftfreq = get_xp(np)(_fft.fftfreq)
+rfftfreq = get_xp(np)(_fft.rfftfreq)
+fftshift = get_xp(np)(_fft.fftshift)
+ifftshift = get_xp(np)(_fft.ifftshift)
+
+
+__all__ = ["rfft2", "irfft2", "fft2", "ifft2"]
+__all__ += _fft.__all__
+
+
+def __dir__() -> list[str]:
+    return __all__
+
+
+del get_xp
+del np
+del fft_all
+del _fft
diff --git a/sklearn/externals/array_api_compat/numpy/linalg.py b/sklearn/externals/array_api_compat/numpy/linalg.py
new file mode 100644
index 0000000000000..2d3e731da3fc0
--- /dev/null
+++ b/sklearn/externals/array_api_compat/numpy/linalg.py
@@ -0,0 +1,143 @@
+# pyright: reportAttributeAccessIssue=false
+# pyright: reportUnknownArgumentType=false
+# pyright: reportUnknownMemberType=false
+# pyright: reportUnknownVariableType=false
+
+from __future__ import annotations
+
+import numpy as np
+
+# intersection of `np.linalg.__all__` on numpy 1.22 and 2.2, minus `_linalg.__all__`
+from numpy.linalg import (
+    LinAlgError,
+    cond,
+    det,
+    eig,
+    eigvals,
+    eigvalsh,
+    inv,
+    lstsq,
+    matrix_power,
+    multi_dot,
+    norm,
+    tensorinv,
+    tensorsolve,
+)
+
+from .._internal import get_xp
+from ..common import _linalg
+
+# These functions are in both the main and linalg namespaces
+from ._aliases import matmul, matrix_transpose, tensordot, vecdot  # noqa: F401
+from ._typing import Array
+
+cross = get_xp(np)(_linalg.cross)
+outer = get_xp(np)(_linalg.outer)
+EighResult = _linalg.EighResult
+QRResult = _linalg.QRResult
+SlogdetResult = _linalg.SlogdetResult
+SVDResult = _linalg.SVDResult
+eigh = get_xp(np)(_linalg.eigh)
+qr = get_xp(np)(_linalg.qr)
+slogdet = get_xp(np)(_linalg.slogdet)
+svd = get_xp(np)(_linalg.svd)
+cholesky = get_xp(np)(_linalg.cholesky)
+matrix_rank = get_xp(np)(_linalg.matrix_rank)
+pinv = get_xp(np)(_linalg.pinv)
+matrix_norm = get_xp(np)(_linalg.matrix_norm)
+svdvals = get_xp(np)(_linalg.svdvals)
+diagonal = get_xp(np)(_linalg.diagonal)
+trace = get_xp(np)(_linalg.trace)
+
+# Note: unlike np.linalg.solve, the array API solve() only accepts x2 as a
+# vector when it is exactly 1-dimensional. All other cases treat x2 as a stack
+# of matrices. The np.linalg.solve behavior of allowing stacks of both
+# matrices and vectors is ambiguous c.f.
+# https://github.com/numpy/numpy/issues/15349 and
+# https://github.com/data-apis/array-api/issues/285.
+
+# To workaround this, the below is the code from np.linalg.solve except
+# only calling solve1 in the exactly 1D case.
+
+
+# This code is here instead of in common because it is numpy specific. Also
+# note that CuPy's solve() does not currently support broadcasting (see
+# https://github.com/cupy/cupy/blob/main/cupy/cublas.py#L43).
+def solve(x1: Array, x2: Array, /) -> Array:
+    try:
+        from numpy.linalg._linalg import (
+            _assert_stacked_2d,
+            _assert_stacked_square,
+            _commonType,
+            _makearray,
+            _raise_linalgerror_singular,
+            isComplexType,
+        )
+    except ImportError:
+        from numpy.linalg.linalg import (
+            _assert_stacked_2d,
+            _assert_stacked_square,
+            _commonType,
+            _makearray,
+            _raise_linalgerror_singular,
+            isComplexType,
+        )
+    from numpy.linalg import _umath_linalg
+
+    x1, _ = _makearray(x1)
+    _assert_stacked_2d(x1)
+    _assert_stacked_square(x1)
+    x2, wrap = _makearray(x2)
+    t, result_t = _commonType(x1, x2)
+
+    # This part is different from np.linalg.solve
+    gufunc: np.ufunc
+    if x2.ndim == 1:
+        gufunc = _umath_linalg.solve1
+    else:
+        gufunc = _umath_linalg.solve
+
+    # This does nothing currently but is left in because it will be relevant
+    # when complex dtype support is added to the spec in 2022.
+    signature = "DD->D" if isComplexType(t) else "dd->d"
+    with np.errstate(
+        call=_raise_linalgerror_singular,
+        invalid="call",
+        over="ignore",
+        divide="ignore",
+        under="ignore",
+    ):
+        r: Array = gufunc(x1, x2, signature=signature)
+
+    return wrap(r.astype(result_t, copy=False))
+
+
+# These functions are completely new here. If the library already has them
+# (i.e., numpy 2.0), use the library version instead of our wrapper.
+if hasattr(np.linalg, "vector_norm"):
+    vector_norm = np.linalg.vector_norm
+else:
+    vector_norm = get_xp(np)(_linalg.vector_norm)
+
+
+__all__ = [
+    "LinAlgError",
+    "cond",
+    "det",
+    "eig",
+    "eigvals",
+    "eigvalsh",
+    "inv",
+    "lstsq",
+    "matrix_power",
+    "multi_dot",
+    "norm",
+    "tensorinv",
+    "tensorsolve",
+]
+__all__ += _linalg.__all__
+__all__ += ["solve", "vector_norm"]
+
+
+def __dir__() -> list[str]:
+    return __all__
diff --git a/sklearn/externals/array_api_compat/py.typed b/sklearn/externals/array_api_compat/py.typed
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/sklearn/externals/array_api_compat/torch/__init__.py b/sklearn/externals/array_api_compat/torch/__init__.py
new file mode 100644
index 0000000000000..69fd19ce83a56
--- /dev/null
+++ b/sklearn/externals/array_api_compat/torch/__init__.py
@@ -0,0 +1,22 @@
+from torch import * # noqa: F403
+
+# Several names are not included in the above import *
+import torch
+for n in dir(torch):
+    if (n.startswith('_')
+        or n.endswith('_')
+        or 'cuda' in n
+        or 'cpu' in n
+        or 'backward' in n):
+        continue
+    exec(f"{n} = torch.{n}")
+del n
+
+# These imports may overwrite names from the import * above.
+from ._aliases import * # noqa: F403
+
+# See the comment in the numpy __init__.py
+__import__(__package__ + '.linalg')
+__import__(__package__ + '.fft')
+
+__array_api_version__ = '2024.12'
diff --git a/sklearn/externals/array_api_compat/torch/_aliases.py b/sklearn/externals/array_api_compat/torch/_aliases.py
new file mode 100644
index 0000000000000..de5d1a5d40eb5
--- /dev/null
+++ b/sklearn/externals/array_api_compat/torch/_aliases.py
@@ -0,0 +1,855 @@
+from __future__ import annotations
+
+from functools import reduce as _reduce, wraps as _wraps
+from builtins import all as _builtin_all, any as _builtin_any
+from typing import Any, List, Optional, Sequence, Tuple, Union, Literal
+
+import torch
+
+from .._internal import get_xp
+from ..common import _aliases
+from ..common._typing import NestedSequence, SupportsBufferProtocol
+from ._info import __array_namespace_info__
+from ._typing import Array, Device, DType
+
+_int_dtypes = {
+    torch.uint8,
+    torch.int8,
+    torch.int16,
+    torch.int32,
+    torch.int64,
+}
+try:
+    # torch >=2.3
+    _int_dtypes |= {torch.uint16, torch.uint32, torch.uint64}
+except AttributeError:
+    pass
+
+
+_array_api_dtypes = {
+    torch.bool,
+    *_int_dtypes,
+    torch.float32,
+    torch.float64,
+    torch.complex64,
+    torch.complex128,
+}
+
+_promotion_table = {
+    # ints
+    (torch.int8, torch.int16): torch.int16,
+    (torch.int8, torch.int32): torch.int32,
+    (torch.int8, torch.int64): torch.int64,
+    (torch.int16, torch.int32): torch.int32,
+    (torch.int16, torch.int64): torch.int64,
+    (torch.int32, torch.int64): torch.int64,
+    # ints and uints (mixed sign)
+    (torch.uint8, torch.int8): torch.int16,
+    (torch.uint8, torch.int16): torch.int16,
+    (torch.uint8, torch.int32): torch.int32,
+    (torch.uint8, torch.int64): torch.int64,
+    # floats
+    (torch.float32, torch.float64): torch.float64,
+    # complexes
+    (torch.complex64, torch.complex128): torch.complex128,
+    # Mixed float and complex
+    (torch.float32, torch.complex64): torch.complex64,
+    (torch.float32, torch.complex128): torch.complex128,
+    (torch.float64, torch.complex64): torch.complex128,
+    (torch.float64, torch.complex128): torch.complex128,
+}
+
+_promotion_table.update({(b, a): c for (a, b), c in _promotion_table.items()})
+_promotion_table.update({(a, a): a for a in _array_api_dtypes})
+
+
+def _two_arg(f):
+    @_wraps(f)
+    def _f(x1, x2, /, **kwargs):
+        x1, x2 = _fix_promotion(x1, x2)
+        return f(x1, x2, **kwargs)
+    if _f.__doc__ is None:
+        _f.__doc__ = f"""\
+Array API compatibility wrapper for torch.{f.__name__}.
+
+See the corresponding PyTorch documentation and/or the array API specification
+for more details.
+
+"""
+    return _f
+
+def _fix_promotion(x1, x2, only_scalar=True):
+    if not isinstance(x1, torch.Tensor) or not isinstance(x2, torch.Tensor):
+        return x1, x2
+    if x1.dtype not in _array_api_dtypes or x2.dtype not in _array_api_dtypes:
+        return x1, x2
+    # If an argument is 0-D pytorch downcasts the other argument
+    if not only_scalar or x1.shape == ():
+        dtype = result_type(x1, x2)
+        x2 = x2.to(dtype)
+    if not only_scalar or x2.shape == ():
+        dtype = result_type(x1, x2)
+        x1 = x1.to(dtype)
+    return x1, x2
+
+
+_py_scalars = (bool, int, float, complex)
+
+
+def result_type(
+    *arrays_and_dtypes: Array | DType | bool | int | float | complex
+) -> DType:
+    num = len(arrays_and_dtypes)
+
+    if num == 0:
+        raise ValueError("At least one array or dtype must be provided")
+
+    elif num == 1:
+        x = arrays_and_dtypes[0]
+        if isinstance(x, torch.dtype):
+            return x
+        return x.dtype
+
+    if num == 2:
+        x, y = arrays_and_dtypes
+        return _result_type(x, y)
+
+    else:
+        # sort scalars so that they are treated last
+        scalars, others = [], []
+        for x in arrays_and_dtypes:
+            if isinstance(x, _py_scalars):
+                scalars.append(x)
+            else:
+                others.append(x)
+        if not others:
+            raise ValueError("At least one array or dtype must be provided")
+
+        # combine left-to-right
+        return _reduce(_result_type, others + scalars)
+
+
+def _result_type(
+    x: Array | DType | bool | int | float | complex,
+    y: Array | DType | bool | int | float | complex,
+) -> DType:
+    if not (isinstance(x, _py_scalars) or isinstance(y, _py_scalars)):
+        xdt = x if isinstance(x, torch.dtype) else x.dtype
+        ydt = y if isinstance(y, torch.dtype) else y.dtype
+
+        try:
+            return _promotion_table[xdt, ydt]
+        except KeyError:
+            pass
+
+    # This doesn't result_type(dtype, dtype) for non-array API dtypes
+    # because torch.result_type only accepts tensors. This does however, allow
+    # cross-kind promotion.
+    x = torch.tensor([], dtype=x) if isinstance(x, torch.dtype) else x
+    y = torch.tensor([], dtype=y) if isinstance(y, torch.dtype) else y
+    return torch.result_type(x, y)
+
+
+def can_cast(from_: Union[DType, Array], to: DType, /) -> bool:
+    if not isinstance(from_, torch.dtype):
+        from_ = from_.dtype
+    return torch.can_cast(from_, to)
+
+# Basic renames
+bitwise_invert = torch.bitwise_not
+newaxis = None
+# torch.conj sets the conjugation bit, which breaks conversion to other
+# libraries. See https://github.com/data-apis/array-api-compat/issues/173
+conj = torch.conj_physical
+
+# Two-arg elementwise functions
+# These require a wrapper to do the correct type promotion on 0-D tensors
+add = _two_arg(torch.add)
+atan2 = _two_arg(torch.atan2)
+bitwise_and = _two_arg(torch.bitwise_and)
+bitwise_left_shift = _two_arg(torch.bitwise_left_shift)
+bitwise_or = _two_arg(torch.bitwise_or)
+bitwise_right_shift = _two_arg(torch.bitwise_right_shift)
+bitwise_xor = _two_arg(torch.bitwise_xor)
+copysign = _two_arg(torch.copysign)
+divide = _two_arg(torch.divide)
+# Also a rename. torch.equal does not broadcast
+equal = _two_arg(torch.eq)
+floor_divide = _two_arg(torch.floor_divide)
+greater = _two_arg(torch.greater)
+greater_equal = _two_arg(torch.greater_equal)
+hypot = _two_arg(torch.hypot)
+less = _two_arg(torch.less)
+less_equal = _two_arg(torch.less_equal)
+logaddexp = _two_arg(torch.logaddexp)
+# logical functions are not included here because they only accept bool in the
+# spec, so type promotion is irrelevant.
+maximum = _two_arg(torch.maximum)
+minimum = _two_arg(torch.minimum)
+multiply = _two_arg(torch.multiply)
+not_equal = _two_arg(torch.not_equal)
+pow = _two_arg(torch.pow)
+remainder = _two_arg(torch.remainder)
+subtract = _two_arg(torch.subtract)
+
+
+def asarray(
+    obj: (
+    Array 
+        | bool | int | float | complex 
+        | NestedSequence[bool | int | float | complex] 
+        | SupportsBufferProtocol
+    ),
+    /,
+    *,
+    dtype: DType | None = None,
+    device: Device | None = None,
+    copy: bool | None = None,
+    **kwargs: Any,
+) -> Array:
+    # torch.asarray does not respect input->output device propagation
+    # https://github.com/pytorch/pytorch/issues/150199
+    if device is None and isinstance(obj, torch.Tensor):
+        device = obj.device
+    return torch.asarray(obj, dtype=dtype, device=device, copy=copy, **kwargs)
+
+
+# These wrappers are mostly based on the fact that pytorch uses 'dim' instead
+# of 'axis'.
+
+# torch.min and torch.max return a tuple and don't support multiple axes https://github.com/pytorch/pytorch/issues/58745
+def max(x: Array, /, *, axis: Optional[Union[int, Tuple[int, ...]]] = None, keepdims: bool = False) -> Array:
+    # https://github.com/pytorch/pytorch/issues/29137
+    if axis == ():
+        return torch.clone(x)
+    return torch.amax(x, axis, keepdims=keepdims)
+
+def min(x: Array, /, *, axis: Optional[Union[int, Tuple[int, ...]]] = None, keepdims: bool = False) -> Array:
+    # https://github.com/pytorch/pytorch/issues/29137
+    if axis == ():
+        return torch.clone(x)
+    return torch.amin(x, axis, keepdims=keepdims)
+
+clip = get_xp(torch)(_aliases.clip)
+unstack = get_xp(torch)(_aliases.unstack)
+cumulative_sum = get_xp(torch)(_aliases.cumulative_sum)
+cumulative_prod = get_xp(torch)(_aliases.cumulative_prod)
+finfo = get_xp(torch)(_aliases.finfo)
+iinfo = get_xp(torch)(_aliases.iinfo)
+
+
+# torch.sort also returns a tuple
+# https://github.com/pytorch/pytorch/issues/70921
+def sort(x: Array, /, *, axis: int = -1, descending: bool = False, stable: bool = True, **kwargs) -> Array:
+    return torch.sort(x, dim=axis, descending=descending, stable=stable, **kwargs).values
+
+def _normalize_axes(axis, ndim):
+    axes = []
+    if ndim == 0 and axis:
+        # Better error message in this case
+        raise IndexError(f"Dimension out of range: {axis[0]}")
+    lower, upper = -ndim, ndim - 1
+    for a in axis:
+        if a < lower or a > upper:
+            # Match torch error message (e.g., from sum())
+            raise IndexError(f"Dimension out of range (expected to be in range of [{lower}, {upper}], but got {a}")
+        if a < 0:
+            a = a + ndim
+        if a in axes:
+            # Use IndexError instead of RuntimeError, and "axis" instead of "dim"
+            raise IndexError(f"Axis {a} appears multiple times in the list of axes")
+        axes.append(a)
+    return sorted(axes)
+
+def _axis_none_keepdims(x, ndim, keepdims):
+    # Apply keepdims when axis=None
+    # (https://github.com/pytorch/pytorch/issues/71209)
+    # Note that this is only valid for the axis=None case.
+    if keepdims:
+        for i in range(ndim):
+            x = torch.unsqueeze(x, 0)
+    return x
+
+def _reduce_multiple_axes(f, x, axis, keepdims=False, **kwargs):
+    # Some reductions don't support multiple axes
+    # (https://github.com/pytorch/pytorch/issues/56586).
+    axes = _normalize_axes(axis, x.ndim)
+    for a in reversed(axes):
+        x = torch.movedim(x, a, -1)
+    x = torch.flatten(x, -len(axes))
+
+    out = f(x, -1, **kwargs)
+
+    if keepdims:
+        for a in axes:
+            out = torch.unsqueeze(out, a)
+    return out
+
+
+def _sum_prod_no_axis(x: Array, dtype: DType | None) -> Array:
+    """
+    Implements `sum(..., axis=())` and `prod(..., axis=())`.
+    
+    Works around https://github.com/pytorch/pytorch/issues/29137
+    """
+    if dtype is not None:
+        return x.clone() if dtype == x.dtype else x.to(dtype)
+
+    # We can't upcast uint8 according to the spec because there is no
+    # torch.uint64, so at least upcast to int64 which is what prod does
+    # when axis=None.
+    if x.dtype in (torch.uint8, torch.int8, torch.int16, torch.int32):
+        return x.to(torch.int64)
+
+    return x.clone()
+
+
+def prod(x: Array,
+         /,
+         *,
+         axis: Optional[Union[int, Tuple[int, ...]]] = None,
+         dtype: Optional[DType] = None,
+         keepdims: bool = False,
+         **kwargs) -> Array:
+
+    if axis == ():
+        return _sum_prod_no_axis(x, dtype)
+    # torch.prod doesn't support multiple axes
+    # (https://github.com/pytorch/pytorch/issues/56586).
+    if isinstance(axis, tuple):
+        return _reduce_multiple_axes(torch.prod, x, axis, keepdims=keepdims, dtype=dtype, **kwargs)
+    if axis is None:
+        # torch doesn't support keepdims with axis=None
+        # (https://github.com/pytorch/pytorch/issues/71209)
+        res = torch.prod(x, dtype=dtype, **kwargs)
+        res = _axis_none_keepdims(res, x.ndim, keepdims)
+        return res
+
+    return torch.prod(x, axis, dtype=dtype, keepdims=keepdims, **kwargs)
+
+
+def sum(x: Array,
+         /,
+         *,
+         axis: Optional[Union[int, Tuple[int, ...]]] = None,
+         dtype: Optional[DType] = None,
+         keepdims: bool = False,
+         **kwargs) -> Array:
+
+    if axis == ():
+        return _sum_prod_no_axis(x, dtype)
+    if axis is None:
+        # torch doesn't support keepdims with axis=None
+        # (https://github.com/pytorch/pytorch/issues/71209)
+        res = torch.sum(x, dtype=dtype, **kwargs)
+        res = _axis_none_keepdims(res, x.ndim, keepdims)
+        return res
+
+    return torch.sum(x, axis, dtype=dtype, keepdims=keepdims, **kwargs)
+
+def any(x: Array,
+        /,
+        *,
+        axis: Optional[Union[int, Tuple[int, ...]]] = None,
+        keepdims: bool = False,
+        **kwargs) -> Array:
+
+    if axis == ():
+        return x.to(torch.bool)
+    # torch.any doesn't support multiple axes
+    # (https://github.com/pytorch/pytorch/issues/56586).
+    if isinstance(axis, tuple):
+        res = _reduce_multiple_axes(torch.any, x, axis, keepdims=keepdims, **kwargs)
+        return res.to(torch.bool)
+    if axis is None:
+        # torch doesn't support keepdims with axis=None
+        # (https://github.com/pytorch/pytorch/issues/71209)
+        res = torch.any(x, **kwargs)
+        res = _axis_none_keepdims(res, x.ndim, keepdims)
+        return res.to(torch.bool)
+
+    # torch.any doesn't return bool for uint8
+    return torch.any(x, axis, keepdims=keepdims).to(torch.bool)
+
+def all(x: Array,
+        /,
+        *,
+        axis: Optional[Union[int, Tuple[int, ...]]] = None,
+        keepdims: bool = False,
+        **kwargs) -> Array:
+
+    if axis == ():
+        return x.to(torch.bool)
+    # torch.all doesn't support multiple axes
+    # (https://github.com/pytorch/pytorch/issues/56586).
+    if isinstance(axis, tuple):
+        res = _reduce_multiple_axes(torch.all, x, axis, keepdims=keepdims, **kwargs)
+        return res.to(torch.bool)
+    if axis is None:
+        # torch doesn't support keepdims with axis=None
+        # (https://github.com/pytorch/pytorch/issues/71209)
+        res = torch.all(x, **kwargs)
+        res = _axis_none_keepdims(res, x.ndim, keepdims)
+        return res.to(torch.bool)
+
+    # torch.all doesn't return bool for uint8
+    return torch.all(x, axis, keepdims=keepdims).to(torch.bool)
+
+def mean(x: Array,
+         /,
+         *,
+         axis: Optional[Union[int, Tuple[int, ...]]] = None,
+         keepdims: bool = False,
+         **kwargs) -> Array:
+    # https://github.com/pytorch/pytorch/issues/29137
+    if axis == ():
+        return torch.clone(x)
+    if axis is None:
+        # torch doesn't support keepdims with axis=None
+        # (https://github.com/pytorch/pytorch/issues/71209)
+        res = torch.mean(x, **kwargs)
+        res = _axis_none_keepdims(res, x.ndim, keepdims)
+        return res
+    return torch.mean(x, axis, keepdims=keepdims, **kwargs)
+
+def std(x: Array,
+        /,
+        *,
+        axis: Optional[Union[int, Tuple[int, ...]]] = None,
+        correction: Union[int, float] = 0.0,
+        keepdims: bool = False,
+        **kwargs) -> Array:
+    # Note, float correction is not supported
+    # https://github.com/pytorch/pytorch/issues/61492. We don't try to
+    # implement it here for now.
+
+    if isinstance(correction, float):
+        _correction = int(correction)
+        if correction != _correction:
+            raise NotImplementedError("float correction in torch std() is not yet supported")
+    else:
+        _correction = correction
+
+    # https://github.com/pytorch/pytorch/issues/29137
+    if axis == ():
+        return torch.zeros_like(x)
+    if isinstance(axis, int):
+        axis = (axis,)
+    if axis is None:
+        # torch doesn't support keepdims with axis=None
+        # (https://github.com/pytorch/pytorch/issues/71209)
+        res = torch.std(x, tuple(range(x.ndim)), correction=_correction, **kwargs)
+        res = _axis_none_keepdims(res, x.ndim, keepdims)
+        return res
+    return torch.std(x, axis, correction=_correction, keepdims=keepdims, **kwargs)
+
+def var(x: Array,
+        /,
+        *,
+        axis: Optional[Union[int, Tuple[int, ...]]] = None,
+        correction: Union[int, float] = 0.0,
+        keepdims: bool = False,
+        **kwargs) -> Array:
+    # Note, float correction is not supported
+    # https://github.com/pytorch/pytorch/issues/61492. We don't try to
+    # implement it here for now.
+
+    # if isinstance(correction, float):
+    #     correction = int(correction)
+
+    # https://github.com/pytorch/pytorch/issues/29137
+    if axis == ():
+        return torch.zeros_like(x)
+    if isinstance(axis, int):
+        axis = (axis,)
+    if axis is None:
+        # torch doesn't support keepdims with axis=None
+        # (https://github.com/pytorch/pytorch/issues/71209)
+        res = torch.var(x, tuple(range(x.ndim)), correction=correction, **kwargs)
+        res = _axis_none_keepdims(res, x.ndim, keepdims)
+        return res
+    return torch.var(x, axis, correction=correction, keepdims=keepdims, **kwargs)
+
+# torch.concat doesn't support dim=None
+# https://github.com/pytorch/pytorch/issues/70925
+def concat(arrays: Union[Tuple[Array, ...], List[Array]],
+           /,
+           *,
+           axis: Optional[int] = 0,
+           **kwargs) -> Array:
+    if axis is None:
+        arrays = tuple(ar.flatten() for ar in arrays)
+        axis = 0
+    return torch.concat(arrays, axis, **kwargs)
+
+# torch.squeeze only accepts int dim and doesn't require it
+# https://github.com/pytorch/pytorch/issues/70924. Support for tuple dim was
+# added at https://github.com/pytorch/pytorch/pull/89017.
+def squeeze(x: Array, /, axis: Union[int, Tuple[int, ...]]) -> Array:
+    if isinstance(axis, int):
+        axis = (axis,)
+    for a in axis:
+        if x.shape[a] != 1:
+            raise ValueError("squeezed dimensions must be equal to 1")
+    axes = _normalize_axes(axis, x.ndim)
+    # Remove this once pytorch 1.14 is released with the above PR #89017.
+    sequence = [a - i for i, a in enumerate(axes)]
+    for a in sequence:
+        x = torch.squeeze(x, a)
+    return x
+
+# torch.broadcast_to uses size instead of shape
+def broadcast_to(x: Array, /, shape: Tuple[int, ...], **kwargs) -> Array:
+    return torch.broadcast_to(x, shape, **kwargs)
+
+# torch.permute uses dims instead of axes
+def permute_dims(x: Array, /, axes: Tuple[int, ...]) -> Array:
+    return torch.permute(x, axes)
+
+# The axis parameter doesn't work for flip() and roll()
+# https://github.com/pytorch/pytorch/issues/71210. Also torch.flip() doesn't
+# accept axis=None
+def flip(x: Array, /, *, axis: Optional[Union[int, Tuple[int, ...]]] = None, **kwargs) -> Array:
+    if axis is None:
+        axis = tuple(range(x.ndim))
+    # torch.flip doesn't accept dim as an int but the method does
+    # https://github.com/pytorch/pytorch/issues/18095
+    return x.flip(axis, **kwargs)
+
+def roll(x: Array, /, shift: Union[int, Tuple[int, ...]], *, axis: Optional[Union[int, Tuple[int, ...]]] = None, **kwargs) -> Array:
+    return torch.roll(x, shift, axis, **kwargs)
+
+def nonzero(x: Array, /, **kwargs) -> Tuple[Array, ...]:
+    if x.ndim == 0:
+        raise ValueError("nonzero() does not support zero-dimensional arrays")
+    return torch.nonzero(x, as_tuple=True, **kwargs)
+
+
+# torch uses `dim` instead of `axis`
+def diff(
+    x: Array,
+    /,
+    *,
+    axis: int = -1,
+    n: int = 1,
+    prepend: Optional[Array] = None,
+    append: Optional[Array] = None,
+) -> Array:
+    return torch.diff(x, dim=axis, n=n, prepend=prepend, append=append)
+
+
+# torch uses `dim` instead of `axis`, does not have keepdims
+def count_nonzero(
+    x: Array,
+    /,
+    *,
+    axis: Optional[Union[int, Tuple[int, ...]]] = None,
+    keepdims: bool = False,
+) -> Array:
+    result = torch.count_nonzero(x, dim=axis)
+    if keepdims:
+        if isinstance(axis, int):
+            return result.unsqueeze(axis)
+        elif isinstance(axis, tuple):
+            n_axis = [x.ndim + ax if ax < 0 else ax for ax in axis]
+            sh = [1 if i in n_axis else x.shape[i] for i in range(x.ndim)]
+            return torch.reshape(result, sh)
+        return _axis_none_keepdims(result, x.ndim, keepdims)
+    else:
+        return result
+
+
+# "repeat" is torch.repeat_interleave;  also the dim argument
+def repeat(x: Array, repeats: int | Array, /, *, axis: int | None = None) -> Array:
+    return torch.repeat_interleave(x, repeats, axis)
+
+
+def where(
+    condition: Array, 
+    x1: Array | bool | int | float | complex, 
+    x2: Array | bool | int | float | complex,
+    /,
+) -> Array:
+    x1, x2 = _fix_promotion(x1, x2)
+    return torch.where(condition, x1, x2)
+
+
+# torch.reshape doesn't have the copy keyword
+def reshape(x: Array,
+            /,
+            shape: Tuple[int, ...],
+            *,
+            copy: Optional[bool] = None,
+            **kwargs) -> Array:
+    if copy is not None:
+        raise NotImplementedError("torch.reshape doesn't yet support the copy keyword")
+    return torch.reshape(x, shape, **kwargs)
+
+# torch.arange doesn't support returning empty arrays
+# (https://github.com/pytorch/pytorch/issues/70915), and doesn't support some
+# keyword argument combinations
+# (https://github.com/pytorch/pytorch/issues/70914)
+def arange(start: Union[int, float],
+           /,
+           stop: Optional[Union[int, float]] = None,
+           step: Union[int, float] = 1,
+           *,
+           dtype: Optional[DType] = None,
+           device: Optional[Device] = None,
+           **kwargs) -> Array:
+    if stop is None:
+        start, stop = 0, start
+    if step > 0 and stop <= start or step < 0 and stop >= start:
+        if dtype is None:
+            if _builtin_all(isinstance(i, int) for i in [start, stop, step]):
+                dtype = torch.int64
+            else:
+                dtype = torch.float32
+        return torch.empty(0, dtype=dtype, device=device, **kwargs)
+    return torch.arange(start, stop, step, dtype=dtype, device=device, **kwargs)
+
+# torch.eye does not accept None as a default for the second argument and
+# doesn't support off-diagonals (https://github.com/pytorch/pytorch/issues/70910)
+def eye(n_rows: int,
+        n_cols: Optional[int] = None,
+        /,
+        *,
+        k: int = 0,
+        dtype: Optional[DType] = None,
+        device: Optional[Device] = None,
+        **kwargs) -> Array:
+    if n_cols is None:
+        n_cols = n_rows
+    z = torch.zeros(n_rows, n_cols, dtype=dtype, device=device, **kwargs)
+    if abs(k) <= n_rows + n_cols:
+        z.diagonal(k).fill_(1)
+    return z
+
+# torch.linspace doesn't have the endpoint parameter
+def linspace(start: Union[int, float],
+             stop: Union[int, float],
+             /,
+             num: int,
+             *,
+             dtype: Optional[DType] = None,
+             device: Optional[Device] = None,
+             endpoint: bool = True,
+             **kwargs) -> Array:
+    if not endpoint:
+        return torch.linspace(start, stop, num+1, dtype=dtype, device=device, **kwargs)[:-1]
+    return torch.linspace(start, stop, num, dtype=dtype, device=device, **kwargs)
+
+# torch.full does not accept an int size
+# https://github.com/pytorch/pytorch/issues/70906
+def full(shape: Union[int, Tuple[int, ...]],
+         fill_value: bool | int | float | complex,
+         *,
+         dtype: Optional[DType] = None,
+         device: Optional[Device] = None,
+         **kwargs) -> Array:
+    if isinstance(shape, int):
+        shape = (shape,)
+
+    return torch.full(shape, fill_value, dtype=dtype, device=device, **kwargs)
+
+# ones, zeros, and empty do not accept shape as a keyword argument
+def ones(shape: Union[int, Tuple[int, ...]],
+         *,
+         dtype: Optional[DType] = None,
+         device: Optional[Device] = None,
+         **kwargs) -> Array:
+    return torch.ones(shape, dtype=dtype, device=device, **kwargs)
+
+def zeros(shape: Union[int, Tuple[int, ...]],
+         *,
+         dtype: Optional[DType] = None,
+         device: Optional[Device] = None,
+         **kwargs) -> Array:
+    return torch.zeros(shape, dtype=dtype, device=device, **kwargs)
+
+def empty(shape: Union[int, Tuple[int, ...]],
+         *,
+         dtype: Optional[DType] = None,
+         device: Optional[Device] = None,
+         **kwargs) -> Array:
+    return torch.empty(shape, dtype=dtype, device=device, **kwargs)
+
+# tril and triu do not call the keyword argument k
+
+def tril(x: Array, /, *, k: int = 0) -> Array:
+    return torch.tril(x, k)
+
+def triu(x: Array, /, *, k: int = 0) -> Array:
+    return torch.triu(x, k)
+
+# Functions that aren't in torch https://github.com/pytorch/pytorch/issues/58742
+def expand_dims(x: Array, /, *, axis: int = 0) -> Array:
+    return torch.unsqueeze(x, axis)
+
+
+def astype(
+    x: Array,
+    dtype: DType,
+    /,
+    *,
+    copy: bool = True,
+    device: Optional[Device] = None,
+) -> Array:
+    if device is not None:
+        return x.to(device, dtype=dtype, copy=copy)
+    return x.to(dtype=dtype, copy=copy)
+
+
+def broadcast_arrays(*arrays: Array) -> List[Array]:
+    shape = torch.broadcast_shapes(*[a.shape for a in arrays])
+    return [torch.broadcast_to(a, shape) for a in arrays]
+
+# Note that these named tuples aren't actually part of the standard namespace,
+# but I don't see any issue with exporting the names here regardless.
+from ..common._aliases import (UniqueAllResult, UniqueCountsResult,
+                               UniqueInverseResult)
+
+# https://github.com/pytorch/pytorch/issues/70920
+def unique_all(x: Array) -> UniqueAllResult:
+    # torch.unique doesn't support returning indices.
+    # https://github.com/pytorch/pytorch/issues/36748. The workaround
+    # suggested in that issue doesn't actually function correctly (it relies
+    # on non-deterministic behavior of scatter()).
+    raise NotImplementedError("unique_all() not yet implemented for pytorch (see https://github.com/pytorch/pytorch/issues/36748)")
+
+    # values, inverse_indices, counts = torch.unique(x, return_counts=True, return_inverse=True)
+    # # torch.unique incorrectly gives a 0 count for nan values.
+    # # https://github.com/pytorch/pytorch/issues/94106
+    # counts[torch.isnan(values)] = 1
+    # return UniqueAllResult(values, indices, inverse_indices, counts)
+
+def unique_counts(x: Array) -> UniqueCountsResult:
+    values, counts = torch.unique(x, return_counts=True)
+
+    # torch.unique incorrectly gives a 0 count for nan values.
+    # https://github.com/pytorch/pytorch/issues/94106
+    counts[torch.isnan(values)] = 1
+    return UniqueCountsResult(values, counts)
+
+def unique_inverse(x: Array) -> UniqueInverseResult:
+    values, inverse = torch.unique(x, return_inverse=True)
+    return UniqueInverseResult(values, inverse)
+
+def unique_values(x: Array) -> Array:
+    return torch.unique(x)
+
+def matmul(x1: Array, x2: Array, /, **kwargs) -> Array:
+    # torch.matmul doesn't type promote (but differently from _fix_promotion)
+    x1, x2 = _fix_promotion(x1, x2, only_scalar=False)
+    return torch.matmul(x1, x2, **kwargs)
+
+matrix_transpose = get_xp(torch)(_aliases.matrix_transpose)
+_vecdot = get_xp(torch)(_aliases.vecdot)
+
+def vecdot(x1: Array, x2: Array, /, *, axis: int = -1) -> Array:
+    x1, x2 = _fix_promotion(x1, x2, only_scalar=False)
+    return _vecdot(x1, x2, axis=axis)
+
+# torch.tensordot uses dims instead of axes
+def tensordot(
+    x1: Array,
+    x2: Array,
+    /,
+    *, 
+    axes: Union[int, Tuple[Sequence[int], Sequence[int]]] = 2, 
+    **kwargs,
+) -> Array:
+    # Note: torch.tensordot fails with integer dtypes when there is only 1
+    # element in the axis (https://github.com/pytorch/pytorch/issues/84530).
+    x1, x2 = _fix_promotion(x1, x2, only_scalar=False)
+    return torch.tensordot(x1, x2, dims=axes, **kwargs)
+
+
+def isdtype(
+    dtype: DType, kind: Union[DType, str, Tuple[Union[DType, str], ...]],
+    *, _tuple=True, # Disallow nested tuples
+) -> bool:
+    """
+    Returns a boolean indicating whether a provided dtype is of a specified data type ``kind``.
+
+    Note that outside of this function, this compat library does not yet fully
+    support complex numbers.
+
+    See
+    https://data-apis.org/array-api/latest/API_specification/generated/array_api.isdtype.html
+    for more details
+    """
+    if isinstance(kind, tuple) and _tuple:
+        return _builtin_any(isdtype(dtype, k, _tuple=False) for k in kind)
+    elif isinstance(kind, str):
+        if kind == 'bool':
+            return dtype == torch.bool
+        elif kind == 'signed integer':
+            return dtype in _int_dtypes and dtype.is_signed
+        elif kind == 'unsigned integer':
+            return dtype in _int_dtypes and not dtype.is_signed
+        elif kind == 'integral':
+            return dtype in _int_dtypes
+        elif kind == 'real floating':
+            return dtype.is_floating_point
+        elif kind == 'complex floating':
+            return dtype.is_complex
+        elif kind == 'numeric':
+            return isdtype(dtype, ('integral', 'real floating', 'complex floating'))
+        else:
+            raise ValueError(f"Unrecognized data type kind: {kind!r}")
+    else:
+        return dtype == kind
+
+def take(x: Array, indices: Array, /, *, axis: Optional[int] = None, **kwargs) -> Array:
+    if axis is None:
+        if x.ndim != 1:
+            raise ValueError("axis must be specified when ndim > 1")
+        axis = 0
+    return torch.index_select(x, axis, indices, **kwargs)
+
+
+def take_along_axis(x: Array, indices: Array, /, *, axis: int = -1) -> Array:
+    return torch.take_along_dim(x, indices, dim=axis)
+
+
+def sign(x: Array, /) -> Array:
+    # torch sign() does not support complex numbers and does not propagate
+    # nans. See https://github.com/data-apis/array-api-compat/issues/136
+    if x.dtype.is_complex:
+        out = x/torch.abs(x)
+        # sign(0) = 0 but the above formula would give nan
+        out[x == 0+0j] = 0+0j
+        return out
+    else:
+        out = torch.sign(x)
+        if x.dtype.is_floating_point:
+            out[torch.isnan(x)] = torch.nan
+        return out
+
+
+def meshgrid(*arrays: Array, indexing: Literal['xy', 'ij'] = 'xy') -> List[Array]:
+    # enforce the default of 'xy'
+    # TODO: is the return type a list or a tuple
+    return list(torch.meshgrid(*arrays, indexing='xy'))
+
+
+__all__ = ['__array_namespace_info__', 'asarray', 'result_type', 'can_cast',
+           'permute_dims', 'bitwise_invert', 'newaxis', 'conj', 'add',
+           'atan2', 'bitwise_and', 'bitwise_left_shift', 'bitwise_or',
+           'bitwise_right_shift', 'bitwise_xor', 'copysign', 'count_nonzero',
+           'diff', 'divide',
+           'equal', 'floor_divide', 'greater', 'greater_equal', 'hypot',
+           'less', 'less_equal', 'logaddexp', 'maximum', 'minimum',
+           'multiply', 'not_equal', 'pow', 'remainder', 'subtract', 'max',
+           'min', 'clip', 'unstack', 'cumulative_sum', 'cumulative_prod', 'sort', 'prod', 'sum',
+           'any', 'all', 'mean', 'std', 'var', 'concat', 'squeeze',
+           'broadcast_to', 'flip', 'roll', 'nonzero', 'where', 'reshape',
+           'arange', 'eye', 'linspace', 'full', 'ones', 'zeros', 'empty',
+           'tril', 'triu', 'expand_dims', 'astype', 'broadcast_arrays',
+           'UniqueAllResult', 'UniqueCountsResult', 'UniqueInverseResult',
+           'unique_all', 'unique_counts', 'unique_inverse', 'unique_values',
+           'matmul', 'matrix_transpose', 'vecdot', 'tensordot', 'isdtype',
+           'take', 'take_along_axis', 'sign', 'finfo', 'iinfo', 'repeat', 'meshgrid']
+
+_all_ignore = ['torch', 'get_xp']
diff --git a/sklearn/externals/array_api_compat/torch/_info.py b/sklearn/externals/array_api_compat/torch/_info.py
new file mode 100644
index 0000000000000..818e5d3702e38
--- /dev/null
+++ b/sklearn/externals/array_api_compat/torch/_info.py
@@ -0,0 +1,369 @@
+"""
+Array API Inspection namespace
+
+This is the namespace for inspection functions as defined by the array API
+standard. See
+https://data-apis.org/array-api/latest/API_specification/inspection.html for
+more details.
+
+"""
+import torch
+
+from functools import cache
+
+class __array_namespace_info__:
+    """
+    Get the array API inspection namespace for PyTorch.
+
+    The array API inspection namespace defines the following functions:
+
+    - capabilities()
+    - default_device()
+    - default_dtypes()
+    - dtypes()
+    - devices()
+
+    See
+    https://data-apis.org/array-api/latest/API_specification/inspection.html
+    for more details.
+
+    Returns
+    -------
+    info : ModuleType
+        The array API inspection namespace for PyTorch.
+
+    Examples
+    --------
+    >>> info = xp.__array_namespace_info__()
+    >>> info.default_dtypes()
+    {'real floating': numpy.float64,
+     'complex floating': numpy.complex128,
+     'integral': numpy.int64,
+     'indexing': numpy.int64}
+
+    """
+
+    __module__ = 'torch'
+
+    def capabilities(self):
+        """
+        Return a dictionary of array API library capabilities.
+
+        The resulting dictionary has the following keys:
+
+        - **"boolean indexing"**: boolean indicating whether an array library
+          supports boolean indexing. Always ``True`` for PyTorch.
+
+        - **"data-dependent shapes"**: boolean indicating whether an array
+          library supports data-dependent output shapes. Always ``True`` for
+          PyTorch.
+
+        See
+        https://data-apis.org/array-api/latest/API_specification/generated/array_api.info.capabilities.html
+        for more details.
+
+        See Also
+        --------
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.dtypes,
+        __array_namespace_info__.devices
+
+        Returns
+        -------
+        capabilities : dict
+            A dictionary of array API library capabilities.
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.capabilities()
+        {'boolean indexing': True,
+         'data-dependent shapes': True,
+         'max dimensions': 64}
+
+        """
+        return {
+            "boolean indexing": True,
+            "data-dependent shapes": True,
+            "max dimensions": 64,
+        }
+
+    def default_device(self):
+        """
+        The default device used for new PyTorch arrays.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.dtypes,
+        __array_namespace_info__.devices
+
+        Returns
+        -------
+        device : Device
+            The default device used for new PyTorch arrays.
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.default_device()
+        device(type='cpu')
+
+        Notes
+        -----
+        This method returns the static default device when PyTorch is initialized.
+        However, the *current* device used by creation functions (``empty`` etc.)
+        can be changed at runtime.
+
+        See Also
+        --------
+        https://github.com/data-apis/array-api/issues/835
+        """
+        return torch.device("cpu")
+
+    def default_dtypes(self, *, device=None):
+        """
+        The default data types used for new PyTorch arrays.
+
+        Parameters
+        ----------
+        device : Device, optional
+            The device to get the default data types for.
+            Unused for PyTorch, as all devices use the same default dtypes.
+
+        Returns
+        -------
+        dtypes : dict
+            A dictionary describing the default data types used for new PyTorch
+            arrays.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.dtypes,
+        __array_namespace_info__.devices
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.default_dtypes()
+        {'real floating': torch.float32,
+         'complex floating': torch.complex64,
+         'integral': torch.int64,
+         'indexing': torch.int64}
+
+        """
+        # Note: if the default is set to float64, the devices like MPS that
+        # don't support float64 will error. We still return the default_dtype
+        # value here because this error doesn't represent a different default
+        # per-device.
+        default_floating = torch.get_default_dtype()
+        default_complex = torch.complex64 if default_floating == torch.float32 else torch.complex128
+        default_integral = torch.int64
+        return {
+            "real floating": default_floating,
+            "complex floating": default_complex,
+            "integral": default_integral,
+            "indexing": default_integral,
+        }
+
+
+    def _dtypes(self, kind):
+        bool = torch.bool
+        int8 = torch.int8
+        int16 = torch.int16
+        int32 = torch.int32
+        int64 = torch.int64
+        uint8 = torch.uint8
+        # uint16, uint32, and uint64 are present in newer versions of pytorch,
+        # but they aren't generally supported by the array API functions, so
+        # we omit them from this function.
+        float32 = torch.float32
+        float64 = torch.float64
+        complex64 = torch.complex64
+        complex128 = torch.complex128
+
+        if kind is None:
+            return {
+                "bool": bool,
+                "int8": int8,
+                "int16": int16,
+                "int32": int32,
+                "int64": int64,
+                "uint8": uint8,
+                "float32": float32,
+                "float64": float64,
+                "complex64": complex64,
+                "complex128": complex128,
+            }
+        if kind == "bool":
+            return {"bool": bool}
+        if kind == "signed integer":
+            return {
+                "int8": int8,
+                "int16": int16,
+                "int32": int32,
+                "int64": int64,
+            }
+        if kind == "unsigned integer":
+            return {
+                "uint8": uint8,
+            }
+        if kind == "integral":
+            return {
+                "int8": int8,
+                "int16": int16,
+                "int32": int32,
+                "int64": int64,
+                "uint8": uint8,
+            }
+        if kind == "real floating":
+            return {
+                "float32": float32,
+                "float64": float64,
+            }
+        if kind == "complex floating":
+            return {
+                "complex64": complex64,
+                "complex128": complex128,
+            }
+        if kind == "numeric":
+            return {
+                "int8": int8,
+                "int16": int16,
+                "int32": int32,
+                "int64": int64,
+                "uint8": uint8,
+                "float32": float32,
+                "float64": float64,
+                "complex64": complex64,
+                "complex128": complex128,
+            }
+        if isinstance(kind, tuple):
+            res = {}
+            for k in kind:
+                res.update(self.dtypes(kind=k))
+            return res
+        raise ValueError(f"unsupported kind: {kind!r}")
+
+    @cache
+    def dtypes(self, *, device=None, kind=None):
+        """
+        The array API data types supported by PyTorch.
+
+        Note that this function only returns data types that are defined by
+        the array API.
+
+        Parameters
+        ----------
+        device : Device, optional
+            The device to get the data types for.
+            Unused for PyTorch, as all devices use the same dtypes.
+        kind : str or tuple of str, optional
+            The kind of data types to return. If ``None``, all data types are
+            returned. If a string, only data types of that kind are returned.
+            If a tuple, a dictionary containing the union of the given kinds
+            is returned. The following kinds are supported:
+
+            - ``'bool'``: boolean data types (i.e., ``bool``).
+            - ``'signed integer'``: signed integer data types (i.e., ``int8``,
+              ``int16``, ``int32``, ``int64``).
+            - ``'unsigned integer'``: unsigned integer data types (i.e.,
+              ``uint8``, ``uint16``, ``uint32``, ``uint64``).
+            - ``'integral'``: integer data types. Shorthand for ``('signed
+              integer', 'unsigned integer')``.
+            - ``'real floating'``: real-valued floating-point data types
+              (i.e., ``float32``, ``float64``).
+            - ``'complex floating'``: complex floating-point data types (i.e.,
+              ``complex64``, ``complex128``).
+            - ``'numeric'``: numeric data types. Shorthand for ``('integral',
+              'real floating', 'complex floating')``.
+
+        Returns
+        -------
+        dtypes : dict
+            A dictionary mapping the names of data types to the corresponding
+            PyTorch data types.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.devices
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.dtypes(kind='signed integer')
+        {'int8': numpy.int8,
+         'int16': numpy.int16,
+         'int32': numpy.int32,
+         'int64': numpy.int64}
+
+        """
+        res = self._dtypes(kind)
+        for k, v in res.copy().items():
+            try:
+                torch.empty((0,), dtype=v, device=device)
+            except:
+                del res[k]
+        return res
+
+    @cache
+    def devices(self):
+        """
+        The devices supported by PyTorch.
+
+        Returns
+        -------
+        devices : list[Device]
+            The devices supported by PyTorch.
+
+        See Also
+        --------
+        __array_namespace_info__.capabilities,
+        __array_namespace_info__.default_device,
+        __array_namespace_info__.default_dtypes,
+        __array_namespace_info__.dtypes
+
+        Examples
+        --------
+        >>> info = xp.__array_namespace_info__()
+        >>> info.devices()
+        [device(type='cpu'), device(type='mps', index=0), device(type='meta')]
+
+        """
+        # Torch doesn't have a straightforward way to get the list of all
+        # currently supported devices. To do this, we first parse the error
+        # message of torch.device to get the list of all possible types of
+        # device:
+        try:
+            torch.device('notadevice')
+            raise AssertionError("unreachable")  # pragma: nocover
+        except RuntimeError as e:
+            # The error message is something like:
+            # "Expected one of cpu, cuda, ipu, xpu, mkldnn, opengl, opencl, ideep, hip, ve, fpga, ort, xla, lazy, vulkan, mps, meta, hpu, mtia, privateuseone device type at start of device string: notadevice"
+            devices_names = e.args[0].split('Expected one of ')[1].split(' device type')[0].split(', ')
+
+        # Next we need to check for different indices for different devices.
+        # device(device_name, index=index) doesn't actually check if the
+        # device name or index is valid. We have to try to create a tensor
+        # with it (which is why this function is cached).
+        devices = []
+        for device_name in devices_names:
+            i = 0
+            while True:
+                try:
+                    a = torch.empty((0,), device=torch.device(device_name, index=i))
+                    if a.device in devices:
+                        break
+                    devices.append(a.device)
+                except:
+                    break
+                i += 1
+
+        return devices
diff --git a/sklearn/externals/array_api_compat/torch/_typing.py b/sklearn/externals/array_api_compat/torch/_typing.py
new file mode 100644
index 0000000000000..5267087156371
--- /dev/null
+++ b/sklearn/externals/array_api_compat/torch/_typing.py
@@ -0,0 +1,3 @@
+__all__ = ["Array", "Device", "DType"]
+
+from torch import device as Device, dtype as DType, Tensor as Array
diff --git a/sklearn/externals/array_api_compat/torch/fft.py b/sklearn/externals/array_api_compat/torch/fft.py
new file mode 100644
index 0000000000000..50e6a0d0a3968
--- /dev/null
+++ b/sklearn/externals/array_api_compat/torch/fft.py
@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from typing import Union, Sequence, Literal
+
+import torch
+import torch.fft
+from torch.fft import * # noqa: F403
+
+from ._typing import Array
+
+# Several torch fft functions do not map axes to dim
+
+def fftn(
+    x: Array,
+    /,
+    *,
+    s: Sequence[int] = None,
+    axes: Sequence[int] = None,
+    norm: Literal["backward", "ortho", "forward"] = "backward",
+    **kwargs,
+) -> Array:
+    return torch.fft.fftn(x, s=s, dim=axes, norm=norm, **kwargs)
+
+def ifftn(
+    x: Array,
+    /,
+    *,
+    s: Sequence[int] = None,
+    axes: Sequence[int] = None,
+    norm: Literal["backward", "ortho", "forward"] = "backward",
+    **kwargs,
+) -> Array:
+    return torch.fft.ifftn(x, s=s, dim=axes, norm=norm, **kwargs)
+
+def rfftn(
+    x: Array,
+    /,
+    *,
+    s: Sequence[int] = None,
+    axes: Sequence[int] = None,
+    norm: Literal["backward", "ortho", "forward"] = "backward",
+    **kwargs,
+) -> Array:
+    return torch.fft.rfftn(x, s=s, dim=axes, norm=norm, **kwargs)
+
+def irfftn(
+    x: Array,
+    /,
+    *,
+    s: Sequence[int] = None,
+    axes: Sequence[int] = None,
+    norm: Literal["backward", "ortho", "forward"] = "backward",
+    **kwargs,
+) -> Array:
+    return torch.fft.irfftn(x, s=s, dim=axes, norm=norm, **kwargs)
+
+def fftshift(
+    x: Array,
+    /,
+    *,
+    axes: Union[int, Sequence[int]] = None,
+    **kwargs,
+) -> Array:
+    return torch.fft.fftshift(x, dim=axes, **kwargs)
+
+def ifftshift(
+    x: Array,
+    /,
+    *,
+    axes: Union[int, Sequence[int]] = None,
+    **kwargs,
+) -> Array:
+    return torch.fft.ifftshift(x, dim=axes, **kwargs)
+
+
+__all__ = torch.fft.__all__ + [
+    "fftn",
+    "ifftn",
+    "rfftn",
+    "irfftn",
+    "fftshift",
+    "ifftshift",
+]
+
+_all_ignore = ['torch']
diff --git a/sklearn/externals/array_api_compat/torch/linalg.py b/sklearn/externals/array_api_compat/torch/linalg.py
new file mode 100644
index 0000000000000..70d7240500ce4
--- /dev/null
+++ b/sklearn/externals/array_api_compat/torch/linalg.py
@@ -0,0 +1,121 @@
+from __future__ import annotations
+
+import torch
+from typing import Optional, Union, Tuple
+
+from torch.linalg import * # noqa: F403
+
+# torch.linalg doesn't define __all__
+# from torch.linalg import __all__ as linalg_all
+from torch import linalg as torch_linalg
+linalg_all = [i for i in dir(torch_linalg) if not i.startswith('_')]
+
+# outer is implemented in torch but aren't in the linalg namespace
+from torch import outer
+from ._aliases import _fix_promotion, sum
+# These functions are in both the main and linalg namespaces
+from ._aliases import matmul, matrix_transpose, tensordot
+from ._typing import Array, DType
+from ..common._typing import JustInt, JustFloat
+
+# Note: torch.linalg.cross does not default to axis=-1 (it defaults to the
+# first axis with size 3), see https://github.com/pytorch/pytorch/issues/58743
+
+# torch.cross also does not support broadcasting when it would add new
+# dimensions https://github.com/pytorch/pytorch/issues/39656
+def cross(x1: Array, x2: Array, /, *, axis: int = -1) -> Array:
+    x1, x2 = _fix_promotion(x1, x2, only_scalar=False)
+    if not (-min(x1.ndim, x2.ndim) <= axis < max(x1.ndim, x2.ndim)):
+        raise ValueError(f"axis {axis} out of bounds for cross product of arrays with shapes {x1.shape} and {x2.shape}")
+    if not (x1.shape[axis] == x2.shape[axis] == 3):
+        raise ValueError(f"cross product axis must have size 3, got {x1.shape[axis]} and {x2.shape[axis]}")
+    x1, x2 = torch.broadcast_tensors(x1, x2)
+    return torch_linalg.cross(x1, x2, dim=axis)
+
+def vecdot(x1: Array, x2: Array, /, *, axis: int = -1, **kwargs) -> Array:
+    from ._aliases import isdtype
+
+    x1, x2 = _fix_promotion(x1, x2, only_scalar=False)
+
+    # torch.linalg.vecdot incorrectly allows broadcasting along the contracted dimension
+    if x1.shape[axis] != x2.shape[axis]:
+        raise ValueError("x1 and x2 must have the same size along the given axis")
+
+    # torch.linalg.vecdot doesn't support integer dtypes
+    if isdtype(x1.dtype, 'integral') or isdtype(x2.dtype, 'integral'):
+        if kwargs:
+            raise RuntimeError("vecdot kwargs not supported for integral dtypes")
+
+        x1_ = torch.moveaxis(x1, axis, -1)
+        x2_ = torch.moveaxis(x2, axis, -1)
+        x1_, x2_ = torch.broadcast_tensors(x1_, x2_)
+
+        res = x1_[..., None, :] @ x2_[..., None]
+        return res[..., 0, 0]
+    return torch.linalg.vecdot(x1, x2, dim=axis, **kwargs)
+
+def solve(x1: Array, x2: Array, /, **kwargs) -> Array:
+    x1, x2 = _fix_promotion(x1, x2, only_scalar=False)
+    # Torch tries to emulate NumPy 1 solve behavior by using batched 1-D solve
+    # whenever
+    # 1. x1.ndim - 1 == x2.ndim
+    # 2. x1.shape[:-1] == x2.shape
+    #
+    # See linalg_solve_is_vector_rhs in
+    # aten/src/ATen/native/LinearAlgebraUtils.h and
+    # TORCH_META_FUNC(_linalg_solve_ex) in
+    # aten/src/ATen/native/BatchLinearAlgebra.cpp in the PyTorch source code.
+    #
+    # The easiest way to work around this is to prepend a size 1 dimension to
+    # x2, since x2 is already one dimension less than x1.
+    #
+    # See https://github.com/pytorch/pytorch/issues/52915
+    if x2.ndim != 1 and x1.ndim - 1 == x2.ndim and x1.shape[:-1] == x2.shape:
+        x2 = x2[None]
+    return torch.linalg.solve(x1, x2, **kwargs)
+
+# torch.trace doesn't support the offset argument and doesn't support stacking
+def trace(x: Array, /, *, offset: int = 0, dtype: Optional[DType] = None) -> Array:
+    # Use our wrapped sum to make sure it does upcasting correctly
+    return sum(torch.diagonal(x, offset=offset, dim1=-2, dim2=-1), axis=-1, dtype=dtype)
+
+def vector_norm(
+    x: Array,
+    /,
+    *,
+    axis: Optional[Union[int, Tuple[int, ...]]] = None,
+    keepdims: bool = False,
+    # JustFloat stands for inf | -inf, which are not valid for Literal
+    ord: JustInt | JustFloat = 2,
+    **kwargs,
+) -> Array:
+    # torch.vector_norm incorrectly treats axis=() the same as axis=None
+    if axis == ():
+        out = kwargs.get('out')
+        if out is None:
+            dtype = None
+            if x.dtype == torch.complex64:
+                dtype = torch.float32
+            elif x.dtype == torch.complex128:
+                dtype = torch.float64
+
+            out = torch.zeros_like(x, dtype=dtype)
+
+        # The norm of a single scalar works out to abs(x) in every case except
+        # for ord=0, which is x != 0.
+        if ord == 0:
+            out[:] = (x != 0)
+        else:
+            out[:] = torch.abs(x)
+        return out
+    return torch.linalg.vector_norm(x, ord=ord, axis=axis, keepdim=keepdims, **kwargs)
+
+__all__ = linalg_all + ['outer', 'matmul', 'matrix_transpose', 'tensordot',
+                        'cross', 'vecdot', 'solve', 'trace', 'vector_norm']
+
+_all_ignore = ['torch_linalg', 'sum']
+
+del linalg_all
+
+def __dir__() -> list[str]:
+    return __all__
diff --git a/sklearn/externals/array_api_extra/LICENSE b/sklearn/externals/array_api_extra/LICENSE
new file mode 100644
index 0000000000000..45bbb94508771
--- /dev/null
+++ b/sklearn/externals/array_api_extra/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Consortium for Python Data API Standards
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/sklearn/externals/array_api_extra/README.md b/sklearn/externals/array_api_extra/README.md
new file mode 100644
index 0000000000000..fd9953b00ad7f
--- /dev/null
+++ b/sklearn/externals/array_api_extra/README.md
@@ -0,0 +1 @@
+Update this directory using maint_tools/vendor_array_api_extra.sh
diff --git a/sklearn/externals/array_api_extra/__init__.py b/sklearn/externals/array_api_extra/__init__.py
new file mode 100644
index 0000000000000..924c23b9351a3
--- /dev/null
+++ b/sklearn/externals/array_api_extra/__init__.py
@@ -0,0 +1,38 @@
+"""Extra array functions built on top of the array API standard."""
+
+from ._delegation import isclose, pad
+from ._lib._at import at
+from ._lib._funcs import (
+    apply_where,
+    atleast_nd,
+    broadcast_shapes,
+    cov,
+    create_diagonal,
+    expand_dims,
+    kron,
+    nunique,
+    setdiff1d,
+    sinc,
+)
+from ._lib._lazy import lazy_apply
+
+__version__ = "0.7.1"
+
+# pylint: disable=duplicate-code
+__all__ = [
+    "__version__",
+    "apply_where",
+    "at",
+    "atleast_nd",
+    "broadcast_shapes",
+    "cov",
+    "create_diagonal",
+    "expand_dims",
+    "isclose",
+    "kron",
+    "lazy_apply",
+    "nunique",
+    "pad",
+    "setdiff1d",
+    "sinc",
+]
diff --git a/sklearn/externals/array_api_extra/_delegation.py b/sklearn/externals/array_api_extra/_delegation.py
new file mode 100644
index 0000000000000..bb11b7ee24773
--- /dev/null
+++ b/sklearn/externals/array_api_extra/_delegation.py
@@ -0,0 +1,172 @@
+"""Delegation to existing implementations for Public API Functions."""
+
+from collections.abc import Sequence
+from types import ModuleType
+from typing import Literal
+
+from ._lib import Backend, _funcs
+from ._lib._utils._compat import array_namespace
+from ._lib._utils._helpers import asarrays
+from ._lib._utils._typing import Array
+
+__all__ = ["isclose", "pad"]
+
+
+def _delegate(xp: ModuleType, *backends: Backend) -> bool:
+    """
+    Check whether `xp` is one of the `backends` to delegate to.
+
+    Parameters
+    ----------
+    xp : array_namespace
+        Array namespace to check.
+    *backends : IsNamespace
+        Arbitrarily many backends (from the ``IsNamespace`` enum) to check.
+
+    Returns
+    -------
+    bool
+        ``True`` if `xp` matches one of the `backends`, ``False`` otherwise.
+    """
+    return any(backend.is_namespace(xp) for backend in backends)
+
+
+def isclose(
+    a: Array | complex,
+    b: Array | complex,
+    *,
+    rtol: float = 1e-05,
+    atol: float = 1e-08,
+    equal_nan: bool = False,
+    xp: ModuleType | None = None,
+) -> Array:
+    """
+    Return a boolean array where two arrays are element-wise equal within a tolerance.
+
+    The tolerance values are positive, typically very small numbers. The relative
+    difference ``(rtol * abs(b))`` and the absolute difference `atol` are added together
+    to compare against the absolute difference between `a` and `b`.
+
+    NaNs are treated as equal if they are in the same place and if ``equal_nan=True``.
+    Infs are treated as equal if they are in the same place and of the same sign in both
+    arrays.
+
+    Parameters
+    ----------
+    a, b : Array | int | float | complex | bool
+        Input objects to compare. At least one must be an array.
+    rtol : array_like, optional
+        The relative tolerance parameter (see Notes).
+    atol : array_like, optional
+        The absolute tolerance parameter (see Notes).
+    equal_nan : bool, optional
+        Whether to compare NaN's as equal. If True, NaN's in `a` will be considered
+        equal to NaN's in `b` in the output array.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `a` and `b`. Default: infer.
+
+    Returns
+    -------
+    Array
+        A boolean array of shape broadcasted from `a` and `b`, containing ``True`` where
+        `a` is close to `b`, and ``False`` otherwise.
+
+    Warnings
+    --------
+    The default `atol` is not appropriate for comparing numbers with magnitudes much
+    smaller than one (see notes).
+
+    See Also
+    --------
+    math.isclose : Similar function in stdlib for Python scalars.
+
+    Notes
+    -----
+    For finite values, `isclose` uses the following equation to test whether two
+    floating point values are equivalent::
+
+        absolute(a - b) <= (atol + rtol * absolute(b))
+
+    Unlike the built-in `math.isclose`,
+    the above equation is not symmetric in `a` and `b`,
+    so that ``isclose(a, b)`` might be different from ``isclose(b, a)`` in some rare
+    cases.
+
+    The default value of `atol` is not appropriate when the reference value `b` has
+    magnitude smaller than one. For example, it is unlikely that ``a = 1e-9`` and
+    ``b = 2e-9`` should be considered "close", yet ``isclose(1e-9, 2e-9)`` is ``True``
+    with default settings. Be sure to select `atol` for the use case at hand, especially
+    for defining the threshold below which a non-zero value in `a` will be considered
+    "close" to a very small or zero value in `b`.
+
+    The comparison of `a` and `b` uses standard broadcasting, which means that `a` and
+    `b` need not have the same shape in order for ``isclose(a, b)`` to evaluate to
+    ``True``.
+
+    `isclose` is not defined for non-numeric data types.
+    ``bool`` is considered a numeric data-type for this purpose.
+    """
+    xp = array_namespace(a, b) if xp is None else xp
+
+    if _delegate(xp, Backend.NUMPY, Backend.CUPY, Backend.DASK, Backend.JAX):
+        return xp.isclose(a, b, rtol=rtol, atol=atol, equal_nan=equal_nan)
+
+    if _delegate(xp, Backend.TORCH):
+        a, b = asarrays(a, b, xp=xp)  # Array API 2024.12 support
+        return xp.isclose(a, b, rtol=rtol, atol=atol, equal_nan=equal_nan)
+
+    return _funcs.isclose(a, b, rtol=rtol, atol=atol, equal_nan=equal_nan, xp=xp)
+
+
+def pad(
+    x: Array,
+    pad_width: int | tuple[int, int] | Sequence[tuple[int, int]],
+    mode: Literal["constant"] = "constant",
+    *,
+    constant_values: complex = 0,
+    xp: ModuleType | None = None,
+) -> Array:
+    """
+    Pad the input array.
+
+    Parameters
+    ----------
+    x : array
+        Input array.
+    pad_width : int or tuple of ints or sequence of pairs of ints
+        Pad the input array with this many elements from each side.
+        If a sequence of tuples, ``[(before_0, after_0), ... (before_N, after_N)]``,
+        each pair applies to the corresponding axis of ``x``.
+        A single tuple, ``(before, after)``, is equivalent to a list of ``x.ndim``
+        copies of this tuple.
+    mode : str, optional
+        Only "constant" mode is currently supported, which pads with
+        the value passed to `constant_values`.
+    constant_values : python scalar, optional
+        Use this value to pad the input. Default is zero.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `x`. Default: infer.
+
+    Returns
+    -------
+    array
+        The input array,
+        padded with ``pad_width`` elements equal to ``constant_values``.
+    """
+    xp = array_namespace(x) if xp is None else xp
+
+    if mode != "constant":
+        msg = "Only `'constant'` mode is currently supported"
+        raise NotImplementedError(msg)
+
+    # https://github.com/pytorch/pytorch/blob/cf76c05b4dc629ac989d1fb8e789d4fac04a095a/torch/_numpy/_funcs_impl.py#L2045-L2056
+    if _delegate(xp, Backend.TORCH):
+        pad_width = xp.asarray(pad_width)
+        pad_width = xp.broadcast_to(pad_width, (x.ndim, 2))
+        pad_width = xp.flip(pad_width, axis=(0,)).flatten()
+        return xp.nn.functional.pad(x, tuple(pad_width), value=constant_values)  # type: ignore[arg-type]  # pyright: ignore[reportArgumentType]
+
+    if _delegate(xp, Backend.NUMPY, Backend.JAX, Backend.CUPY, Backend.SPARSE):
+        return xp.pad(x, pad_width, mode, constant_values=constant_values)
+
+    return _funcs.pad(x, pad_width, constant_values=constant_values, xp=xp)
diff --git a/sklearn/externals/array_api_extra/_lib/__init__.py b/sklearn/externals/array_api_extra/_lib/__init__.py
new file mode 100644
index 0000000000000..b83d7e8c5c2b7
--- /dev/null
+++ b/sklearn/externals/array_api_extra/_lib/__init__.py
@@ -0,0 +1,5 @@
+"""Internals of array-api-extra."""
+
+from ._backends import Backend
+
+__all__ = ["Backend"]
diff --git a/sklearn/externals/array_api_extra/_lib/_at.py b/sklearn/externals/array_api_extra/_lib/_at.py
new file mode 100644
index 0000000000000..22e18d2c0c30c
--- /dev/null
+++ b/sklearn/externals/array_api_extra/_lib/_at.py
@@ -0,0 +1,454 @@
+"""Update operations for read-only arrays."""
+
+from __future__ import annotations
+
+import operator
+from collections.abc import Callable
+from enum import Enum
+from types import ModuleType
+from typing import TYPE_CHECKING, ClassVar, cast
+
+from ._utils._compat import (
+    array_namespace,
+    is_dask_array,
+    is_jax_array,
+    is_writeable_array,
+)
+from ._utils._helpers import meta_namespace
+from ._utils._typing import Array, SetIndex
+
+if TYPE_CHECKING:  # pragma: no cover
+    # TODO import from typing (requires Python >=3.11)
+    from typing_extensions import Self
+
+
+class _AtOp(Enum):
+    """Operations for use in `xpx.at`."""
+
+    SET = "set"
+    ADD = "add"
+    SUBTRACT = "subtract"
+    MULTIPLY = "multiply"
+    DIVIDE = "divide"
+    POWER = "power"
+    MIN = "min"
+    MAX = "max"
+
+    # @override from Python 3.12
+    def __str__(self) -> str:  # type: ignore[explicit-override]  # pyright: ignore[reportImplicitOverride]
+        """
+        Return string representation (useful for pytest logs).
+
+        Returns
+        -------
+        str
+            The operation's name.
+        """
+        return self.value
+
+
+class Undef(Enum):
+    """Sentinel for undefined values."""
+
+    UNDEF = 0
+
+
+_undef = Undef.UNDEF
+
+
+class at:  # pylint: disable=invalid-name  # numpydoc ignore=PR02
+    """
+    Update operations for read-only arrays.
+
+    This implements ``jax.numpy.ndarray.at`` for all writeable
+    backends (those that support ``__setitem__``) and routes
+    to the ``.at[]`` method for JAX arrays.
+
+    Parameters
+    ----------
+    x : array
+        Input array.
+    idx : index, optional
+        Only `array API standard compliant indices
+        `_
+        are supported.
+
+        You may use two alternate syntaxes::
+
+          >>> import array_api_extra as xpx
+          >>> xpx.at(x, idx).set(value)  # or add(value), etc.
+          >>> xpx.at(x)[idx].set(value)
+
+    copy : bool, optional
+        None (default)
+            The array parameter *may* be modified in place if it is
+            possible and beneficial for performance.
+            You should not reuse it after calling this function.
+        True
+            Ensure that the inputs are not modified.
+        False
+            Ensure that the update operation writes back to the input.
+            Raise ``ValueError`` if a copy cannot be avoided.
+
+    xp : array_namespace, optional
+        The standard-compatible namespace for `x`. Default: infer.
+
+    Returns
+    -------
+    Updated input array.
+
+    Warnings
+    --------
+    (a) When you omit the ``copy`` parameter, you should never reuse the parameter
+    array later on; ideally, you should reassign it immediately::
+
+        >>> import array_api_extra as xpx
+        >>> x = xpx.at(x, 0).set(2)
+
+    The above best practice pattern ensures that the behaviour won't change depending
+    on whether ``x`` is writeable or not, as the original ``x`` object is dereferenced
+    as soon as ``xpx.at`` returns; this way there is no risk to accidentally update it
+    twice.
+
+    On the reverse, the anti-pattern below must be avoided, as it will result in
+    different behaviour on read-only versus writeable arrays::
+
+        >>> x = xp.asarray([0, 0, 0])
+        >>> y = xpx.at(x, 0).set(2)
+        >>> z = xpx.at(x, 1).set(3)
+
+    In the above example, both calls to ``xpx.at`` update ``x`` in place *if possible*.
+    This causes the behaviour to diverge depending on whether ``x`` is writeable or not:
+
+    - If ``x`` is writeable, then after the snippet above you'll have
+      ``x == y == z == [2, 3, 0]``
+    - If ``x`` is read-only, then you'll end up with
+      ``x == [0, 0, 0]``, ``y == [2, 0, 0]`` and ``z == [0, 3, 0]``.
+
+    The correct pattern to use if you want diverging outputs from the same input is
+    to enforce copies::
+
+        >>> x = xp.asarray([0, 0, 0])
+        >>> y = xpx.at(x, 0).set(2, copy=True)  # Never updates x
+        >>> z = xpx.at(x, 1).set(3)  # May or may not update x in place
+        >>> del x  # avoid accidental reuse of x as we don't know its state anymore
+
+    (b) The array API standard does not support integer array indices.
+    The behaviour of update methods when the index is an array of integers is
+    undefined and will vary between backends; this is particularly true when the
+    index contains multiple occurrences of the same index, e.g.::
+
+        >>> import numpy as np
+        >>> import jax.numpy as jnp
+        >>> import array_api_extra as xpx
+        >>> xpx.at(np.asarray([123]), np.asarray([0, 0])).add(1)
+        array([124])
+        >>> xpx.at(jnp.asarray([123]), jnp.asarray([0, 0])).add(1)
+        Array([125], dtype=int32)
+
+    See Also
+    --------
+    jax.numpy.ndarray.at : Equivalent array method in JAX.
+
+    Notes
+    -----
+    `sparse `_, as well as read-only arrays from libraries
+    not explicitly covered by ``array-api-compat``, are not supported by update
+    methods.
+
+    Boolean masks are supported on Dask and jitted JAX arrays exclusively
+    when `idx` has the same shape as `x` and `y` is 0-dimensional.
+    Note that this support is not available in JAX's native
+    ``x.at[mask].set(y)``.
+
+    This pattern::
+
+        >>> mask = m(x)
+        >>> x[mask] = f(x[mask])
+
+    Can't be replaced by `at`, as it won't work on Dask and JAX inside jax.jit::
+
+        >>> mask = m(x)
+        >>> x = xpx.at(x, mask).set(f(x[mask])  # Crash on Dask and jax.jit
+
+    You should instead use::
+
+        >>> x = xp.where(m(x), f(x), x)
+
+    Examples
+    --------
+    Given either of these equivalent expressions::
+
+      >>> import array_api_extra as xpx
+      >>> x = xpx.at(x)[1].add(2)
+      >>> x = xpx.at(x, 1).add(2)
+
+    If x is a JAX array, they are the same as::
+
+      >>> x = x.at[1].add(2)
+
+    If x is a read-only NumPy array, they are the same as::
+
+      >>> x = x.copy()
+      >>> x[1] += 2
+
+    For other known backends, they are the same as::
+
+      >>> x[1] += 2
+    """
+
+    _x: Array
+    _idx: SetIndex | Undef
+    __slots__: ClassVar[tuple[str, ...]] = ("_idx", "_x")
+
+    def __init__(
+        self, x: Array, idx: SetIndex | Undef = _undef, /
+    ) -> None:  # numpydoc ignore=GL08
+        self._x = x
+        self._idx = idx
+
+    def __getitem__(self, idx: SetIndex, /) -> Self:  # numpydoc ignore=PR01,RT01
+        """
+        Allow for the alternate syntax ``at(x)[start:stop:step]``.
+
+        It looks prettier than ``at(x, slice(start, stop, step))``
+        and feels more intuitive coming from the JAX documentation.
+        """
+        if self._idx is not _undef:
+            msg = "Index has already been set"
+            raise ValueError(msg)
+        return type(self)(self._x, idx)
+
+    def _op(
+        self,
+        at_op: _AtOp,
+        in_place_op: Callable[[Array, Array | complex], Array] | None,
+        out_of_place_op: Callable[[Array, Array], Array] | None,
+        y: Array | complex,
+        /,
+        copy: bool | None,
+        xp: ModuleType | None,
+    ) -> Array:
+        """
+        Implement all update operations.
+
+        Parameters
+        ----------
+        at_op : _AtOp
+            Method of JAX's Array.at[].
+        in_place_op : Callable[[Array, Array | complex], Array] | None
+            In-place operation to apply on mutable backends::
+
+                x[idx] = in_place_op(x[idx], y)
+
+            If None::
+
+                x[idx] = y
+
+        out_of_place_op : Callable[[Array, Array], Array] | None
+            Out-of-place operation to apply when idx is a boolean mask and the backend
+            doesn't support in-place updates::
+
+                x = xp.where(idx, out_of_place_op(x, y), x)
+
+            If None::
+
+                x = xp.where(idx, y, x)
+
+        y : array or complex
+            Right-hand side of the operation.
+        copy : bool or None
+            Whether to copy the input array. See the class docstring for details.
+        xp : array_namespace, optional
+            The array namespace for the input array. Default: infer.
+
+        Returns
+        -------
+        Array
+            Updated `x`.
+        """
+        from ._funcs import apply_where  # pylint: disable=cyclic-import
+
+        x, idx = self._x, self._idx
+        xp = array_namespace(x, y) if xp is None else xp
+
+        if isinstance(idx, Undef):
+            msg = (
+                "Index has not been set.\n"
+                "Usage: either\n"
+                "    at(x, idx).set(value)\n"
+                "or\n"
+                "    at(x)[idx].set(value)\n"
+                "(same for all other methods)."
+            )
+            raise ValueError(msg)
+
+        if copy not in (True, False, None):
+            msg = f"copy must be True, False, or None; got {copy!r}"
+            raise ValueError(msg)
+
+        writeable = None if copy else is_writeable_array(x)
+
+        # JAX inside jax.jit doesn't support in-place updates with boolean
+        # masks; Dask exclusively supports __setitem__ but not iops.
+        # We can handle the common special case of 0-dimensional y
+        # with where(idx, y, x) instead.
+        if (
+            (is_dask_array(idx) or is_jax_array(idx))
+            and idx.dtype == xp.bool
+            and idx.shape == x.shape
+        ):
+            y_xp = xp.asarray(y, dtype=x.dtype)
+            if y_xp.ndim == 0:
+                if out_of_place_op:  # add(), subtract(), ...
+                    # suppress inf warnings on Dask
+                    out = apply_where(
+                        idx, (x, y_xp), out_of_place_op, fill_value=x, xp=xp
+                    )
+                    # Undo int->float promotion on JAX after _AtOp.DIVIDE
+                    out = xp.astype(out, x.dtype, copy=False)
+                else:  # set()
+                    out = xp.where(idx, y_xp, x)
+
+                if copy is False:
+                    x[()] = out
+                    return x
+                return out
+
+            # else: this will work on eager JAX and crash on jax.jit and Dask
+
+        if copy or (copy is None and not writeable):
+            if is_jax_array(x):
+                # Use JAX's at[]
+                func = cast(
+                    Callable[[Array | complex], Array],
+                    getattr(x.at[idx], at_op.value),  # type: ignore[attr-defined]  # pyright: ignore[reportAttributeAccessIssue,reportUnknownArgumentType]
+                )
+                out = func(y)
+                # Undo int->float promotion on JAX after _AtOp.DIVIDE
+                return xp.astype(out, x.dtype, copy=False)
+
+            # Emulate at[] behaviour for non-JAX arrays
+            # with a copy followed by an update
+
+            x = xp.asarray(x, copy=True)
+            # A copy of a read-only numpy array is writeable
+            # Note: this assumes that a copy of a writeable array is writeable
+            assert not writeable
+            writeable = None
+
+        if writeable is None:
+            writeable = is_writeable_array(x)
+        if not writeable:
+            # sparse crashes here
+            msg = f"Can't update read-only array {x}"
+            raise ValueError(msg)
+
+        if in_place_op:  # add(), subtract(), ...
+            x[idx] = in_place_op(x[idx], y)
+        else:  # set()
+            x[idx] = y
+        return x
+
+    def set(
+        self,
+        y: Array | complex,
+        /,
+        copy: bool | None = None,
+        xp: ModuleType | None = None,
+    ) -> Array:  # numpydoc ignore=PR01,RT01
+        """Apply ``x[idx] = y`` and return the update array."""
+        return self._op(_AtOp.SET, None, None, y, copy=copy, xp=xp)
+
+    def add(
+        self,
+        y: Array | complex,
+        /,
+        copy: bool | None = None,
+        xp: ModuleType | None = None,
+    ) -> Array:  # numpydoc ignore=PR01,RT01
+        """Apply ``x[idx] += y`` and return the updated array."""
+
+        # Note for this and all other methods based on _iop:
+        # operator.iadd and operator.add subtly differ in behaviour, as
+        # only iadd will trigger exceptions when y has an incompatible dtype.
+        return self._op(_AtOp.ADD, operator.iadd, operator.add, y, copy=copy, xp=xp)
+
+    def subtract(
+        self,
+        y: Array | complex,
+        /,
+        copy: bool | None = None,
+        xp: ModuleType | None = None,
+    ) -> Array:  # numpydoc ignore=PR01,RT01
+        """Apply ``x[idx] -= y`` and return the updated array."""
+        return self._op(
+            _AtOp.SUBTRACT, operator.isub, operator.sub, y, copy=copy, xp=xp
+        )
+
+    def multiply(
+        self,
+        y: Array | complex,
+        /,
+        copy: bool | None = None,
+        xp: ModuleType | None = None,
+    ) -> Array:  # numpydoc ignore=PR01,RT01
+        """Apply ``x[idx] *= y`` and return the updated array."""
+        return self._op(
+            _AtOp.MULTIPLY, operator.imul, operator.mul, y, copy=copy, xp=xp
+        )
+
+    def divide(
+        self,
+        y: Array | complex,
+        /,
+        copy: bool | None = None,
+        xp: ModuleType | None = None,
+    ) -> Array:  # numpydoc ignore=PR01,RT01
+        """Apply ``x[idx] /= y`` and return the updated array."""
+        return self._op(
+            _AtOp.DIVIDE, operator.itruediv, operator.truediv, y, copy=copy, xp=xp
+        )
+
+    def power(
+        self,
+        y: Array | complex,
+        /,
+        copy: bool | None = None,
+        xp: ModuleType | None = None,
+    ) -> Array:  # numpydoc ignore=PR01,RT01
+        """Apply ``x[idx] **= y`` and return the updated array."""
+        return self._op(_AtOp.POWER, operator.ipow, operator.pow, y, copy=copy, xp=xp)
+
+    def min(
+        self,
+        y: Array | complex,
+        /,
+        copy: bool | None = None,
+        xp: ModuleType | None = None,
+    ) -> Array:  # numpydoc ignore=PR01,RT01
+        """Apply ``x[idx] = minimum(x[idx], y)`` and return the updated array."""
+        # On Dask, this function runs on the chunks, so we need to determine the
+        # namespace that Dask is wrapping.
+        # Note that da.minimum _incidentally_ works on NumPy, CuPy, and sparse
+        # thanks to all these meta-namespaces implementing the __array_ufunc__
+        # interface, but there's no guarantee that it will work for other
+        # wrapped libraries in the future.
+        xp = array_namespace(self._x) if xp is None else xp
+        mxp = meta_namespace(self._x, xp=xp)
+        y = xp.asarray(y)
+        return self._op(_AtOp.MIN, mxp.minimum, mxp.minimum, y, copy=copy, xp=xp)
+
+    def max(
+        self,
+        y: Array | complex,
+        /,
+        copy: bool | None = None,
+        xp: ModuleType | None = None,
+    ) -> Array:  # numpydoc ignore=PR01,RT01
+        """Apply ``x[idx] = maximum(x[idx], y)`` and return the updated array."""
+        # See note on min()
+        xp = array_namespace(self._x) if xp is None else xp
+        mxp = meta_namespace(self._x, xp=xp)
+        y = xp.asarray(y)
+        return self._op(_AtOp.MAX, mxp.maximum, mxp.maximum, y, copy=copy, xp=xp)
diff --git a/sklearn/externals/array_api_extra/_lib/_backends.py b/sklearn/externals/array_api_extra/_lib/_backends.py
new file mode 100644
index 0000000000000..f044281ac17c9
--- /dev/null
+++ b/sklearn/externals/array_api_extra/_lib/_backends.py
@@ -0,0 +1,51 @@
+"""Backends with which array-api-extra interacts in delegation and testing."""
+
+from collections.abc import Callable
+from enum import Enum
+from types import ModuleType
+from typing import cast
+
+from ._utils import _compat
+
+__all__ = ["Backend"]
+
+
+class Backend(Enum):  # numpydoc ignore=PR01,PR02  # type: ignore[no-subclass-any]
+    """
+    All array library backends explicitly tested by array-api-extra.
+
+    Parameters
+    ----------
+    value : str
+        Name of the backend's module.
+    is_namespace : Callable[[ModuleType], bool]
+        Function to check whether an input module is the array namespace
+        corresponding to the backend.
+    """
+
+    ARRAY_API_STRICT = "array_api_strict", _compat.is_array_api_strict_namespace
+    NUMPY = "numpy", _compat.is_numpy_namespace
+    NUMPY_READONLY = "numpy_readonly", _compat.is_numpy_namespace
+    CUPY = "cupy", _compat.is_cupy_namespace
+    TORCH = "torch", _compat.is_torch_namespace
+    DASK = "dask.array", _compat.is_dask_namespace
+    SPARSE = "sparse", _compat.is_pydata_sparse_namespace
+    JAX = "jax.numpy", _compat.is_jax_namespace
+
+    def __new__(
+        cls, value: str, _is_namespace: Callable[[ModuleType], bool]
+    ):  # numpydoc ignore=GL08
+        obj = object.__new__(cls)
+        obj._value_ = value
+        return obj
+
+    def __init__(
+        self,
+        value: str,  # noqa: ARG002  # pylint: disable=unused-argument
+        is_namespace: Callable[[ModuleType], bool],
+    ):  # numpydoc ignore=GL08
+        self.is_namespace = is_namespace
+
+    def __str__(self) -> str:  # type: ignore[explicit-override]  # pyright: ignore[reportImplicitOverride]  # numpydoc ignore=RT01
+        """Pretty-print parameterized test names."""
+        return cast(str, self.value)
diff --git a/sklearn/externals/array_api_extra/_lib/_funcs.py b/sklearn/externals/array_api_extra/_lib/_funcs.py
new file mode 100644
index 0000000000000..efe2f377968ec
--- /dev/null
+++ b/sklearn/externals/array_api_extra/_lib/_funcs.py
@@ -0,0 +1,915 @@
+"""Array-agnostic implementations for the public API."""
+
+import math
+import warnings
+from collections.abc import Callable, Sequence
+from types import ModuleType, NoneType
+from typing import cast, overload
+
+from ._at import at
+from ._utils import _compat, _helpers
+from ._utils._compat import (
+    array_namespace,
+    is_dask_namespace,
+    is_jax_array,
+    is_jax_namespace,
+)
+from ._utils._helpers import asarrays, eager_shape, meta_namespace, ndindex
+from ._utils._typing import Array
+
+__all__ = [
+    "apply_where",
+    "atleast_nd",
+    "broadcast_shapes",
+    "cov",
+    "create_diagonal",
+    "expand_dims",
+    "kron",
+    "nunique",
+    "pad",
+    "setdiff1d",
+    "sinc",
+]
+
+
+@overload
+def apply_where(  # type: ignore[explicit-any,decorated-any] # numpydoc ignore=GL08
+    cond: Array,
+    args: Array | tuple[Array, ...],
+    f1: Callable[..., Array],
+    f2: Callable[..., Array],
+    /,
+    *,
+    xp: ModuleType | None = None,
+) -> Array: ...
+
+
+@overload
+def apply_where(  # type: ignore[explicit-any,decorated-any] # numpydoc ignore=GL08
+    cond: Array,
+    args: Array | tuple[Array, ...],
+    f1: Callable[..., Array],
+    /,
+    *,
+    fill_value: Array | complex,
+    xp: ModuleType | None = None,
+) -> Array: ...
+
+
+def apply_where(  # type: ignore[explicit-any] # numpydoc ignore=PR01,PR02
+    cond: Array,
+    args: Array | tuple[Array, ...],
+    f1: Callable[..., Array],
+    f2: Callable[..., Array] | None = None,
+    /,
+    *,
+    fill_value: Array | complex | None = None,
+    xp: ModuleType | None = None,
+) -> Array:
+    """
+    Run one of two elementwise functions depending on a condition.
+
+    Equivalent to ``f1(*args) if cond else fill_value`` performed elementwise
+    when `fill_value` is defined, otherwise to ``f1(*args) if cond else f2(*args)``.
+
+    Parameters
+    ----------
+    cond : array
+        The condition, expressed as a boolean array.
+    args : Array or tuple of Arrays
+        Argument(s) to `f1` (and `f2`). Must be broadcastable with `cond`.
+    f1 : callable
+        Elementwise function of `args`, returning a single array.
+        Where `cond` is True, output will be ``f1(arg0[cond], arg1[cond], ...)``.
+    f2 : callable, optional
+        Elementwise function of `args`, returning a single array.
+        Where `cond` is False, output will be ``f2(arg0[cond], arg1[cond], ...)``.
+        Mutually exclusive with `fill_value`.
+    fill_value : Array or scalar, optional
+        If provided, value with which to fill output array where `cond` is False.
+        It does not need to be scalar; it needs however to be broadcastable with
+        `cond` and `args`.
+        Mutually exclusive with `f2`. You must provide one or the other.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `cond` and `args`. Default: infer.
+
+    Returns
+    -------
+    Array
+        An array with elements from the output of `f1` where `cond` is True and either
+        the output of `f2` or `fill_value` where `cond` is False. The returned array has
+        data type determined by type promotion rules between the output of `f1` and
+        either `fill_value` or the output of `f2`.
+
+    Notes
+    -----
+    ``xp.where(cond, f1(*args), f2(*args))`` requires explicitly evaluating `f1` even
+    when `cond` is False, and `f2` when cond is True. This function evaluates each
+    function only for their matching condition, if the backend allows for it.
+
+    On Dask, `f1` and `f2` are applied to the individual chunks and should use functions
+    from the namespace of the chunks.
+
+    Examples
+    --------
+    >>> import array_api_strict as xp
+    >>> import array_api_extra as xpx
+    >>> a = xp.asarray([5, 4, 3])
+    >>> b = xp.asarray([0, 2, 2])
+    >>> def f(a, b):
+    ...     return a // b
+    >>> xpx.apply_where(b != 0, (a, b), f, fill_value=xp.nan)
+    array([ nan,  2., 1.])
+    """
+    # Parse and normalize arguments
+    if (f2 is None) == (fill_value is None):
+        msg = "Exactly one of `fill_value` or `f2` must be given."
+        raise TypeError(msg)
+    args_ = list(args) if isinstance(args, tuple) else [args]
+    del args
+
+    xp = array_namespace(cond, fill_value, *args_) if xp is None else xp
+
+    if isinstance(fill_value, int | float | complex | NoneType):
+        cond, *args_ = xp.broadcast_arrays(cond, *args_)
+    else:
+        cond, fill_value, *args_ = xp.broadcast_arrays(cond, fill_value, *args_)
+
+    if is_dask_namespace(xp):
+        meta_xp = meta_namespace(cond, fill_value, *args_, xp=xp)
+        # map_blocks doesn't descend into tuples of Arrays
+        return xp.map_blocks(_apply_where, cond, f1, f2, fill_value, *args_, xp=meta_xp)
+    return _apply_where(cond, f1, f2, fill_value, *args_, xp=xp)
+
+
+def _apply_where(  # type: ignore[explicit-any]  # numpydoc ignore=PR01,RT01
+    cond: Array,
+    f1: Callable[..., Array],
+    f2: Callable[..., Array] | None,
+    fill_value: Array | int | float | complex | bool | None,
+    *args: Array,
+    xp: ModuleType,
+) -> Array:
+    """Helper of `apply_where`. On Dask, this runs on a single chunk."""
+
+    if is_jax_namespace(xp):
+        # jax.jit does not support assignment by boolean mask
+        return xp.where(cond, f1(*args), f2(*args) if f2 is not None else fill_value)
+
+    temp1 = f1(*(arr[cond] for arr in args))
+
+    if f2 is None:
+        dtype = xp.result_type(temp1, fill_value)
+        if isinstance(fill_value, int | float | complex):
+            out = xp.full_like(cond, dtype=dtype, fill_value=fill_value)
+        else:
+            out = xp.astype(fill_value, dtype, copy=True)
+    else:
+        ncond = ~cond
+        temp2 = f2(*(arr[ncond] for arr in args))
+        dtype = xp.result_type(temp1, temp2)
+        out = xp.empty_like(cond, dtype=dtype)
+        out = at(out, ncond).set(temp2)
+
+    return at(out, cond).set(temp1)
+
+
+def atleast_nd(x: Array, /, *, ndim: int, xp: ModuleType | None = None) -> Array:
+    """
+    Recursively expand the dimension of an array to at least `ndim`.
+
+    Parameters
+    ----------
+    x : array
+        Input array.
+    ndim : int
+        The minimum number of dimensions for the result.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `x`. Default: infer.
+
+    Returns
+    -------
+    array
+        An array with ``res.ndim`` >= `ndim`.
+        If ``x.ndim`` >= `ndim`, `x` is returned.
+        If ``x.ndim`` < `ndim`, `x` is expanded by prepending new axes
+        until ``res.ndim`` equals `ndim`.
+
+    Examples
+    --------
+    >>> import array_api_strict as xp
+    >>> import array_api_extra as xpx
+    >>> x = xp.asarray([1])
+    >>> xpx.atleast_nd(x, ndim=3, xp=xp)
+    Array([[[1]]], dtype=array_api_strict.int64)
+
+    >>> x = xp.asarray([[[1, 2],
+    ...                  [3, 4]]])
+    >>> xpx.atleast_nd(x, ndim=1, xp=xp) is x
+    True
+    """
+    if xp is None:
+        xp = array_namespace(x)
+
+    if x.ndim < ndim:
+        x = xp.expand_dims(x, axis=0)
+        x = atleast_nd(x, ndim=ndim, xp=xp)
+    return x
+
+
+# `float` in signature to accept `math.nan` for Dask.
+# `int`s are still accepted as `float` is a superclass of `int` in typing
+def broadcast_shapes(*shapes: tuple[float | None, ...]) -> tuple[int | None, ...]:
+    """
+    Compute the shape of the broadcasted arrays.
+
+    Duplicates :func:`numpy.broadcast_shapes`, with additional support for
+    None and NaN sizes.
+
+    This is equivalent to ``xp.broadcast_arrays(arr1, arr2, ...)[0].shape``
+    without needing to worry about the backend potentially deep copying
+    the arrays.
+
+    Parameters
+    ----------
+    *shapes : tuple[int | None, ...]
+        Shapes of the arrays to broadcast.
+
+    Returns
+    -------
+    tuple[int | None, ...]
+        The shape of the broadcasted arrays.
+
+    See Also
+    --------
+    numpy.broadcast_shapes : Equivalent NumPy function.
+    array_api.broadcast_arrays : Function to broadcast actual arrays.
+
+    Notes
+    -----
+    This function accepts the Array API's ``None`` for unknown sizes,
+    as well as Dask's non-standard ``math.nan``.
+    Regardless of input, the output always contains ``None`` for unknown sizes.
+
+    Examples
+    --------
+    >>> import array_api_extra as xpx
+    >>> xpx.broadcast_shapes((2, 3), (2, 1))
+    (2, 3)
+    >>> xpx.broadcast_shapes((4, 2, 3), (2, 1), (1, 3))
+    (4, 2, 3)
+    """
+    if not shapes:
+        return ()  # Match NumPy output
+
+    ndim = max(len(shape) for shape in shapes)
+    out: list[int | None] = []
+    for axis in range(-ndim, 0):
+        sizes = {shape[axis] for shape in shapes if axis >= -len(shape)}
+        # Dask uses NaN for unknown shape, which predates the Array API spec for None
+        none_size = None in sizes or math.nan in sizes
+        sizes -= {1, None, math.nan}
+        if len(sizes) > 1:
+            msg = (
+                "shape mismatch: objects cannot be broadcast to a single shape: "
+                f"{shapes}."
+            )
+            raise ValueError(msg)
+        out.append(None if none_size else cast(int, sizes.pop()) if sizes else 1)
+
+    return tuple(out)
+
+
+def cov(m: Array, /, *, xp: ModuleType | None = None) -> Array:
+    """
+    Estimate a covariance matrix.
+
+    Covariance indicates the level to which two variables vary together.
+    If we examine N-dimensional samples, :math:`X = [x_1, x_2, ... x_N]^T`,
+    then the covariance matrix element :math:`C_{ij}` is the covariance of
+    :math:`x_i` and :math:`x_j`. The element :math:`C_{ii}` is the variance
+    of :math:`x_i`.
+
+    This provides a subset of the functionality of ``numpy.cov``.
+
+    Parameters
+    ----------
+    m : array
+        A 1-D or 2-D array containing multiple variables and observations.
+        Each row of `m` represents a variable, and each column a single
+        observation of all those variables.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `m`. Default: infer.
+
+    Returns
+    -------
+    array
+        The covariance matrix of the variables.
+
+    Examples
+    --------
+    >>> import array_api_strict as xp
+    >>> import array_api_extra as xpx
+
+    Consider two variables, :math:`x_0` and :math:`x_1`, which
+    correlate perfectly, but in opposite directions:
+
+    >>> x = xp.asarray([[0, 2], [1, 1], [2, 0]]).T
+    >>> x
+    Array([[0, 1, 2],
+           [2, 1, 0]], dtype=array_api_strict.int64)
+
+    Note how :math:`x_0` increases while :math:`x_1` decreases. The covariance
+    matrix shows this clearly:
+
+    >>> xpx.cov(x, xp=xp)
+    Array([[ 1., -1.],
+           [-1.,  1.]], dtype=array_api_strict.float64)
+
+    Note that element :math:`C_{0,1}`, which shows the correlation between
+    :math:`x_0` and :math:`x_1`, is negative.
+
+    Further, note how `x` and `y` are combined:
+
+    >>> x = xp.asarray([-2.1, -1,  4.3])
+    >>> y = xp.asarray([3,  1.1,  0.12])
+    >>> X = xp.stack((x, y), axis=0)
+    >>> xpx.cov(X, xp=xp)
+    Array([[11.71      , -4.286     ],
+           [-4.286     ,  2.14413333]], dtype=array_api_strict.float64)
+
+    >>> xpx.cov(x, xp=xp)
+    Array(11.71, dtype=array_api_strict.float64)
+
+    >>> xpx.cov(y, xp=xp)
+    Array(2.14413333, dtype=array_api_strict.float64)
+    """
+    if xp is None:
+        xp = array_namespace(m)
+
+    m = xp.asarray(m, copy=True)
+    dtype = (
+        xp.float64 if xp.isdtype(m.dtype, "integral") else xp.result_type(m, xp.float64)
+    )
+
+    m = atleast_nd(m, ndim=2, xp=xp)
+    m = xp.astype(m, dtype)
+
+    avg = _helpers.mean(m, axis=1, xp=xp)
+
+    m_shape = eager_shape(m)
+    fact = m_shape[1] - 1
+
+    if fact <= 0:
+        warnings.warn("Degrees of freedom <= 0 for slice", RuntimeWarning, stacklevel=2)
+        fact = 0
+
+    m -= avg[:, None]
+    m_transpose = m.T
+    if xp.isdtype(m_transpose.dtype, "complex floating"):
+        m_transpose = xp.conj(m_transpose)
+    c = m @ m_transpose
+    c /= fact
+    axes = tuple(axis for axis, length in enumerate(c.shape) if length == 1)
+    return xp.squeeze(c, axis=axes)
+
+
+def create_diagonal(
+    x: Array, /, *, offset: int = 0, xp: ModuleType | None = None
+) -> Array:
+    """
+    Construct a diagonal array.
+
+    Parameters
+    ----------
+    x : array
+        An array having shape ``(*batch_dims, k)``.
+    offset : int, optional
+        Offset from the leading diagonal (default is ``0``).
+        Use positive ints for diagonals above the leading diagonal,
+        and negative ints for diagonals below the leading diagonal.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `x`. Default: infer.
+
+    Returns
+    -------
+    array
+        An array having shape ``(*batch_dims, k+abs(offset), k+abs(offset))`` with `x`
+        on the diagonal (offset by `offset`).
+
+    Examples
+    --------
+    >>> import array_api_strict as xp
+    >>> import array_api_extra as xpx
+    >>> x = xp.asarray([2, 4, 8])
+
+    >>> xpx.create_diagonal(x, xp=xp)
+    Array([[2, 0, 0],
+           [0, 4, 0],
+           [0, 0, 8]], dtype=array_api_strict.int64)
+
+    >>> xpx.create_diagonal(x, offset=-2, xp=xp)
+    Array([[0, 0, 0, 0, 0],
+           [0, 0, 0, 0, 0],
+           [2, 0, 0, 0, 0],
+           [0, 4, 0, 0, 0],
+           [0, 0, 8, 0, 0]], dtype=array_api_strict.int64)
+    """
+    if xp is None:
+        xp = array_namespace(x)
+
+    if x.ndim == 0:
+        err_msg = "`x` must be at least 1-dimensional."
+        raise ValueError(err_msg)
+
+    x_shape = eager_shape(x)
+    batch_dims = x_shape[:-1]
+    n = x_shape[-1] + abs(offset)
+    diag = xp.zeros((*batch_dims, n**2), dtype=x.dtype, device=_compat.device(x))
+
+    target_slice = slice(
+        offset if offset >= 0 else abs(offset) * n,
+        min(n * (n - offset), diag.shape[-1]),
+        n + 1,
+    )
+    for index in ndindex(*batch_dims):
+        diag = at(diag)[(*index, target_slice)].set(x[(*index, slice(None))])
+    return xp.reshape(diag, (*batch_dims, n, n))
+
+
+def expand_dims(
+    a: Array, /, *, axis: int | tuple[int, ...] = (0,), xp: ModuleType | None = None
+) -> Array:
+    """
+    Expand the shape of an array.
+
+    Insert (a) new axis/axes that will appear at the position(s) specified by
+    `axis` in the expanded array shape.
+
+    This is ``xp.expand_dims`` for `axis` an int *or a tuple of ints*.
+    Roughly equivalent to ``numpy.expand_dims`` for NumPy arrays.
+
+    Parameters
+    ----------
+    a : array
+        Array to have its shape expanded.
+    axis : int or tuple of ints, optional
+        Position(s) in the expanded axes where the new axis (or axes) is/are placed.
+        If multiple positions are provided, they should be unique (note that a position
+        given by a positive index could also be referred to by a negative index -
+        that will also result in an error).
+        Default: ``(0,)``.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `a`. Default: infer.
+
+    Returns
+    -------
+    array
+        `a` with an expanded shape.
+
+    Examples
+    --------
+    >>> import array_api_strict as xp
+    >>> import array_api_extra as xpx
+    >>> x = xp.asarray([1, 2])
+    >>> x.shape
+    (2,)
+
+    The following is equivalent to ``x[xp.newaxis, :]`` or ``x[xp.newaxis]``:
+
+    >>> y = xpx.expand_dims(x, axis=0, xp=xp)
+    >>> y
+    Array([[1, 2]], dtype=array_api_strict.int64)
+    >>> y.shape
+    (1, 2)
+
+    The following is equivalent to ``x[:, xp.newaxis]``:
+
+    >>> y = xpx.expand_dims(x, axis=1, xp=xp)
+    >>> y
+    Array([[1],
+           [2]], dtype=array_api_strict.int64)
+    >>> y.shape
+    (2, 1)
+
+    ``axis`` may also be a tuple:
+
+    >>> y = xpx.expand_dims(x, axis=(0, 1), xp=xp)
+    >>> y
+    Array([[[1, 2]]], dtype=array_api_strict.int64)
+
+    >>> y = xpx.expand_dims(x, axis=(2, 0), xp=xp)
+    >>> y
+    Array([[[1],
+            [2]]], dtype=array_api_strict.int64)
+    """
+    if xp is None:
+        xp = array_namespace(a)
+
+    if not isinstance(axis, tuple):
+        axis = (axis,)
+    ndim = a.ndim + len(axis)
+    if axis != () and (min(axis) < -ndim or max(axis) >= ndim):
+        err_msg = (
+            f"a provided axis position is out of bounds for array of dimension {a.ndim}"
+        )
+        raise IndexError(err_msg)
+    axis = tuple(dim % ndim for dim in axis)
+    if len(set(axis)) != len(axis):
+        err_msg = "Duplicate dimensions specified in `axis`."
+        raise ValueError(err_msg)
+    for i in sorted(axis):
+        a = xp.expand_dims(a, axis=i)
+    return a
+
+
+def isclose(
+    a: Array | complex,
+    b: Array | complex,
+    *,
+    rtol: float = 1e-05,
+    atol: float = 1e-08,
+    equal_nan: bool = False,
+    xp: ModuleType,
+) -> Array:  # numpydoc ignore=PR01,RT01
+    """See docstring in array_api_extra._delegation."""
+    a, b = asarrays(a, b, xp=xp)
+
+    a_inexact = xp.isdtype(a.dtype, ("real floating", "complex floating"))
+    b_inexact = xp.isdtype(b.dtype, ("real floating", "complex floating"))
+    if a_inexact or b_inexact:
+        # prevent warnings on NumPy and Dask on inf - inf
+        mxp = meta_namespace(a, b, xp=xp)
+        out = apply_where(
+            xp.isinf(a) | xp.isinf(b),
+            (a, b),
+            lambda a, b: mxp.isinf(a) & mxp.isinf(b) & (mxp.sign(a) == mxp.sign(b)),  # pyright: ignore[reportUnknownArgumentType]
+            # Note: inf <= inf is True!
+            lambda a, b: mxp.abs(a - b) <= (atol + rtol * mxp.abs(b)),  # pyright: ignore[reportUnknownArgumentType]
+            xp=xp,
+        )
+        if equal_nan:
+            out = xp.where(xp.isnan(a) & xp.isnan(b), True, out)
+        return out
+
+    if xp.isdtype(a.dtype, "bool") or xp.isdtype(b.dtype, "bool"):
+        if atol >= 1 or rtol >= 1:
+            return xp.ones_like(a == b)
+        return a == b
+
+    # integer types
+    atol = int(atol)
+    if rtol == 0:
+        return xp.abs(a - b) <= atol
+
+    # Don't rely on OverflowError, as it is not guaranteed by the Array API.
+    nrtol = int(1.0 / rtol)
+    if nrtol > xp.iinfo(b.dtype).max:
+        # rtol * max_int < 1, so it's inconsequential
+        return xp.abs(a - b) <= atol
+    return xp.abs(a - b) <= (atol + xp.abs(b) // nrtol)
+
+
+def kron(
+    a: Array | complex,
+    b: Array | complex,
+    /,
+    *,
+    xp: ModuleType | None = None,
+) -> Array:
+    """
+    Kronecker product of two arrays.
+
+    Computes the Kronecker product, a composite array made of blocks of the
+    second array scaled by the first.
+
+    Equivalent to ``numpy.kron`` for NumPy arrays.
+
+    Parameters
+    ----------
+    a, b : Array | int | float | complex
+        Input arrays or scalars. At least one must be an array.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `a` and `b`. Default: infer.
+
+    Returns
+    -------
+    array
+        The Kronecker product of `a` and `b`.
+
+    Notes
+    -----
+    The function assumes that the number of dimensions of `a` and `b`
+    are the same, if necessary prepending the smallest with ones.
+    If ``a.shape = (r0,r1,..,rN)`` and ``b.shape = (s0,s1,...,sN)``,
+    the Kronecker product has shape ``(r0*s0, r1*s1, ..., rN*SN)``.
+    The elements are products of elements from `a` and `b`, organized
+    explicitly by::
+
+        kron(a,b)[k0,k1,...,kN] = a[i0,i1,...,iN] * b[j0,j1,...,jN]
+
+    where::
+
+        kt = it * st + jt,  t = 0,...,N
+
+    In the common 2-D case (N=1), the block structure can be visualized::
+
+        [[ a[0,0]*b,   a[0,1]*b,  ... , a[0,-1]*b  ],
+         [  ...                              ...   ],
+         [ a[-1,0]*b,  a[-1,1]*b, ... , a[-1,-1]*b ]]
+
+    Examples
+    --------
+    >>> import array_api_strict as xp
+    >>> import array_api_extra as xpx
+    >>> xpx.kron(xp.asarray([1, 10, 100]), xp.asarray([5, 6, 7]), xp=xp)
+    Array([  5,   6,   7,  50,  60,  70, 500,
+           600, 700], dtype=array_api_strict.int64)
+
+    >>> xpx.kron(xp.asarray([5, 6, 7]), xp.asarray([1, 10, 100]), xp=xp)
+    Array([  5,  50, 500,   6,  60, 600,   7,
+            70, 700], dtype=array_api_strict.int64)
+
+    >>> xpx.kron(xp.eye(2), xp.ones((2, 2)), xp=xp)
+    Array([[1., 1., 0., 0.],
+           [1., 1., 0., 0.],
+           [0., 0., 1., 1.],
+           [0., 0., 1., 1.]], dtype=array_api_strict.float64)
+
+    >>> a = xp.reshape(xp.arange(100), (2, 5, 2, 5))
+    >>> b = xp.reshape(xp.arange(24), (2, 3, 4))
+    >>> c = xpx.kron(a, b, xp=xp)
+    >>> c.shape
+    (2, 10, 6, 20)
+    >>> I = (1, 3, 0, 2)
+    >>> J = (0, 2, 1)
+    >>> J1 = (0,) + J             # extend to ndim=4
+    >>> S1 = (1,) + b.shape
+    >>> K = tuple(xp.asarray(I) * xp.asarray(S1) + xp.asarray(J1))
+    >>> c[K] == a[I]*b[J]
+    Array(True, dtype=array_api_strict.bool)
+    """
+    if xp is None:
+        xp = array_namespace(a, b)
+    a, b = asarrays(a, b, xp=xp)
+
+    singletons = (1,) * (b.ndim - a.ndim)
+    a = cast(Array, xp.broadcast_to(a, singletons + a.shape))
+
+    nd_b, nd_a = b.ndim, a.ndim
+    nd_max = max(nd_b, nd_a)
+    if nd_a == 0 or nd_b == 0:
+        return xp.multiply(a, b)
+
+    a_shape = eager_shape(a)
+    b_shape = eager_shape(b)
+
+    # Equalise the shapes by prepending smaller one with 1s
+    a_shape = (1,) * max(0, nd_b - nd_a) + a_shape
+    b_shape = (1,) * max(0, nd_a - nd_b) + b_shape
+
+    # Insert empty dimensions
+    a_arr = expand_dims(a, axis=tuple(range(nd_b - nd_a)), xp=xp)
+    b_arr = expand_dims(b, axis=tuple(range(nd_a - nd_b)), xp=xp)
+
+    # Compute the product
+    a_arr = expand_dims(a_arr, axis=tuple(range(1, nd_max * 2, 2)), xp=xp)
+    b_arr = expand_dims(b_arr, axis=tuple(range(0, nd_max * 2, 2)), xp=xp)
+    result = xp.multiply(a_arr, b_arr)
+
+    # Reshape back and return
+    res_shape = tuple(a_s * b_s for a_s, b_s in zip(a_shape, b_shape, strict=True))
+    return xp.reshape(result, res_shape)
+
+
+def nunique(x: Array, /, *, xp: ModuleType | None = None) -> Array:
+    """
+    Count the number of unique elements in an array.
+
+    Compatible with JAX and Dask, whose laziness would be otherwise
+    problematic.
+
+    Parameters
+    ----------
+    x : Array
+        Input array.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `x`. Default: infer.
+
+    Returns
+    -------
+    array: 0-dimensional integer array
+        The number of unique elements in `x`. It can be lazy.
+    """
+    if xp is None:
+        xp = array_namespace(x)
+
+    if is_jax_array(x):
+        # size= is JAX-specific
+        # https://github.com/data-apis/array-api/issues/883
+        _, counts = xp.unique_counts(x, size=_compat.size(x))
+        return xp.astype(counts, xp.bool).sum()
+
+    _, counts = xp.unique_counts(x)
+    n = _compat.size(counts)
+    # FIXME https://github.com/data-apis/array-api-compat/pull/231
+    if n is None:  # e.g. Dask, ndonnx
+        return xp.astype(counts, xp.bool).sum()
+    return xp.asarray(n, device=_compat.device(x))
+
+
+def pad(
+    x: Array,
+    pad_width: int | tuple[int, int] | Sequence[tuple[int, int]],
+    *,
+    constant_values: complex = 0,
+    xp: ModuleType,
+) -> Array:  # numpydoc ignore=PR01,RT01
+    """See docstring in `array_api_extra._delegation.py`."""
+    # make pad_width a list of length-2 tuples of ints
+    if isinstance(pad_width, int):
+        pad_width_seq = [(pad_width, pad_width)] * x.ndim
+    elif (
+        isinstance(pad_width, tuple)
+        and len(pad_width) == 2
+        and all(isinstance(i, int) for i in pad_width)
+    ):
+        pad_width_seq = [cast(tuple[int, int], pad_width)] * x.ndim
+    else:
+        pad_width_seq = cast(list[tuple[int, int]], list(pad_width))
+
+    # https://github.com/python/typeshed/issues/13376
+    slices: list[slice] = []  # type: ignore[explicit-any]
+    newshape: list[int] = []
+    for ax, w_tpl in enumerate(pad_width_seq):
+        if len(w_tpl) != 2:
+            msg = f"expect a 2-tuple (before, after), got {w_tpl}."
+            raise ValueError(msg)
+
+        sh = eager_shape(x)[ax]
+
+        if w_tpl[0] == 0 and w_tpl[1] == 0:
+            sl = slice(None, None, None)
+        else:
+            start, stop = w_tpl
+            stop = None if stop == 0 else -stop
+
+            sl = slice(start, stop, None)
+            sh += w_tpl[0] + w_tpl[1]
+
+        newshape.append(sh)
+        slices.append(sl)
+
+    padded = xp.full(
+        tuple(newshape),
+        fill_value=constant_values,
+        dtype=x.dtype,
+        device=_compat.device(x),
+    )
+    return at(padded, tuple(slices)).set(x)
+
+
+def setdiff1d(
+    x1: Array | complex,
+    x2: Array | complex,
+    /,
+    *,
+    assume_unique: bool = False,
+    xp: ModuleType | None = None,
+) -> Array:
+    """
+    Find the set difference of two arrays.
+
+    Return the unique values in `x1` that are not in `x2`.
+
+    Parameters
+    ----------
+    x1 : array | int | float | complex | bool
+        Input array.
+    x2 : array
+        Input comparison array.
+    assume_unique : bool
+        If ``True``, the input arrays are both assumed to be unique, which
+        can speed up the calculation. Default is ``False``.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `x1` and `x2`. Default: infer.
+
+    Returns
+    -------
+    array
+        1D array of values in `x1` that are not in `x2`. The result
+        is sorted when `assume_unique` is ``False``, but otherwise only sorted
+        if the input is sorted.
+
+    Examples
+    --------
+    >>> import array_api_strict as xp
+    >>> import array_api_extra as xpx
+
+    >>> x1 = xp.asarray([1, 2, 3, 2, 4, 1])
+    >>> x2 = xp.asarray([3, 4, 5, 6])
+    >>> xpx.setdiff1d(x1, x2, xp=xp)
+    Array([1, 2], dtype=array_api_strict.int64)
+    """
+    if xp is None:
+        xp = array_namespace(x1, x2)
+    # https://github.com/microsoft/pyright/issues/10103
+    x1_, x2_ = asarrays(x1, x2, xp=xp)
+
+    if assume_unique:
+        x1_ = xp.reshape(x1_, (-1,))
+        x2_ = xp.reshape(x2_, (-1,))
+    else:
+        x1_ = xp.unique_values(x1_)
+        x2_ = xp.unique_values(x2_)
+
+    return x1_[_helpers.in1d(x1_, x2_, assume_unique=True, invert=True, xp=xp)]
+
+
+def sinc(x: Array, /, *, xp: ModuleType | None = None) -> Array:
+    r"""
+    Return the normalized sinc function.
+
+    The sinc function is equal to :math:`\sin(\pi x)/(\pi x)` for any argument
+    :math:`x\ne 0`. ``sinc(0)`` takes the limit value 1, making ``sinc`` not
+    only everywhere continuous but also infinitely differentiable.
+
+    .. note::
+
+        Note the normalization factor of ``pi`` used in the definition.
+        This is the most commonly used definition in signal processing.
+        Use ``sinc(x / xp.pi)`` to obtain the unnormalized sinc function
+        :math:`\sin(x)/x` that is more common in mathematics.
+
+    Parameters
+    ----------
+    x : array
+        Array (possibly multi-dimensional) of values for which to calculate
+        ``sinc(x)``. Must have a real floating point dtype.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `x`. Default: infer.
+
+    Returns
+    -------
+    array
+        ``sinc(x)`` calculated elementwise, which has the same shape as the input.
+
+    Notes
+    -----
+    The name sinc is short for "sine cardinal" or "sinus cardinalis".
+
+    The sinc function is used in various signal processing applications,
+    including in anti-aliasing, in the construction of a Lanczos resampling
+    filter, and in interpolation.
+
+    For bandlimited interpolation of discrete-time signals, the ideal
+    interpolation kernel is proportional to the sinc function.
+
+    References
+    ----------
+    #. Weisstein, Eric W. "Sinc Function." From MathWorld--A Wolfram Web
+       Resource. https://mathworld.wolfram.com/SincFunction.html
+    #. Wikipedia, "Sinc function",
+       https://en.wikipedia.org/wiki/Sinc_function
+
+    Examples
+    --------
+    >>> import array_api_strict as xp
+    >>> import array_api_extra as xpx
+    >>> x = xp.linspace(-4, 4, 41)
+    >>> xpx.sinc(x, xp=xp)
+    Array([-3.89817183e-17, -4.92362781e-02,
+           -8.40918587e-02, -8.90384387e-02,
+           -5.84680802e-02,  3.89817183e-17,
+            6.68206631e-02,  1.16434881e-01,
+            1.26137788e-01,  8.50444803e-02,
+           -3.89817183e-17, -1.03943254e-01,
+           -1.89206682e-01, -2.16236208e-01,
+           -1.55914881e-01,  3.89817183e-17,
+            2.33872321e-01,  5.04551152e-01,
+            7.56826729e-01,  9.35489284e-01,
+            1.00000000e+00,  9.35489284e-01,
+            7.56826729e-01,  5.04551152e-01,
+            2.33872321e-01,  3.89817183e-17,
+           -1.55914881e-01, -2.16236208e-01,
+           -1.89206682e-01, -1.03943254e-01,
+           -3.89817183e-17,  8.50444803e-02,
+            1.26137788e-01,  1.16434881e-01,
+            6.68206631e-02,  3.89817183e-17,
+           -5.84680802e-02, -8.90384387e-02,
+           -8.40918587e-02, -4.92362781e-02,
+           -3.89817183e-17], dtype=array_api_strict.float64)
+    """
+    if xp is None:
+        xp = array_namespace(x)
+
+    if not xp.isdtype(x.dtype, "real floating"):
+        err_msg = "`x` must have a real floating data type."
+        raise ValueError(err_msg)
+    # no scalars in `where` - array-api#807
+    y = xp.pi * xp.where(
+        xp.astype(x, xp.bool),
+        x,
+        xp.asarray(xp.finfo(x.dtype).eps, dtype=x.dtype, device=_compat.device(x)),
+    )
+    return xp.sin(y) / y
diff --git a/sklearn/externals/array_api_extra/_lib/_lazy.py b/sklearn/externals/array_api_extra/_lib/_lazy.py
new file mode 100644
index 0000000000000..7b45eff91cda4
--- /dev/null
+++ b/sklearn/externals/array_api_extra/_lib/_lazy.py
@@ -0,0 +1,352 @@
+"""Public API Functions."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Callable, Sequence
+from functools import partial, wraps
+from types import ModuleType
+from typing import TYPE_CHECKING, Any, ParamSpec, TypeAlias, cast, overload
+
+from ._funcs import broadcast_shapes
+from ._utils import _compat
+from ._utils._compat import (
+    array_namespace,
+    is_dask_namespace,
+    is_jax_namespace,
+)
+from ._utils._helpers import is_python_scalar
+from ._utils._typing import Array, DType
+
+if TYPE_CHECKING:  # pragma: no cover
+    import numpy as np
+    from numpy.typing import ArrayLike
+
+    NumPyObject: TypeAlias = np.ndarray[Any, Any] | np.generic  # type: ignore[explicit-any]
+else:
+    # Sphinx hack
+    NumPyObject = Any
+
+P = ParamSpec("P")
+
+
+@overload
+def lazy_apply(  # type: ignore[decorated-any, valid-type]
+    func: Callable[P, Array | ArrayLike],
+    *args: Array | complex | None,
+    shape: tuple[int | None, ...] | None = None,
+    dtype: DType | None = None,
+    as_numpy: bool = False,
+    xp: ModuleType | None = None,
+    **kwargs: P.kwargs,  # pyright: ignore[reportGeneralTypeIssues]
+) -> Array: ...  # numpydoc ignore=GL08
+
+
+@overload
+def lazy_apply(  # type: ignore[decorated-any, valid-type]
+    func: Callable[P, Sequence[Array | ArrayLike]],
+    *args: Array | complex | None,
+    shape: Sequence[tuple[int | None, ...]],
+    dtype: Sequence[DType] | None = None,
+    as_numpy: bool = False,
+    xp: ModuleType | None = None,
+    **kwargs: P.kwargs,  # pyright: ignore[reportGeneralTypeIssues]
+) -> tuple[Array, ...]: ...  # numpydoc ignore=GL08
+
+
+def lazy_apply(  # type: ignore[valid-type]  # numpydoc ignore=GL07,SA04
+    func: Callable[P, Array | ArrayLike | Sequence[Array | ArrayLike]],
+    *args: Array | complex | None,
+    shape: tuple[int | None, ...] | Sequence[tuple[int | None, ...]] | None = None,
+    dtype: DType | Sequence[DType] | None = None,
+    as_numpy: bool = False,
+    xp: ModuleType | None = None,
+    **kwargs: P.kwargs,  # pyright: ignore[reportGeneralTypeIssues]
+) -> Array | tuple[Array, ...]:
+    """
+    Lazily apply an eager function.
+
+    If the backend of the input arrays is lazy, e.g. Dask or jitted JAX, the execution
+    of the function is delayed until the graph is materialized; if it's eager, the
+    function is executed immediately.
+
+    Parameters
+    ----------
+    func : callable
+        The function to apply.
+
+        It must accept one or more array API compliant arrays as positional arguments.
+        If `as_numpy=True`, inputs are converted to NumPy before they are passed to
+        `func`.
+        It must return either a single array-like or a sequence of array-likes.
+
+        `func` must be a pure function, i.e. without side effects, as depending on the
+        backend it may be executed more than once or never.
+    *args : Array | int | float | complex | bool | None
+        One or more Array API compliant arrays, Python scalars, or None's.
+
+        If `as_numpy=True`, you need to be able to apply :func:`numpy.asarray` to
+        non-None args to convert them to NumPy; read notes below about specific
+        backends.
+    shape : tuple[int | None, ...] | Sequence[tuple[int | None, ...]], optional
+        Output shape or sequence of output shapes, one for each output of `func`.
+        Default: assume single output and broadcast shapes of the input arrays.
+    dtype : DType | Sequence[DType], optional
+        Output dtype or sequence of output dtypes, one for each output of `func`.
+        dtype(s) must belong to the same array namespace as the input arrays.
+        Default: infer the result type(s) from the input arrays.
+    as_numpy : bool, optional
+        If True, convert the input arrays to NumPy before passing them to `func`.
+        This is particularly useful to make NumPy-only functions, e.g. written in Cython
+       or Numba, work transparently with array API-compliant arrays.
+        Default: False.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `args`. Default: infer.
+    **kwargs : Any, optional
+        Additional keyword arguments to pass verbatim to `func`.
+        They cannot contain Array objects.
+
+    Returns
+    -------
+    Array | tuple[Array, ...]
+        The result(s) of `func` applied to the input arrays, wrapped in the same
+        array namespace as the inputs.
+        If shape is omitted or a single `tuple[int | None, ...]`, return a single array.
+        Otherwise, return a tuple of arrays.
+
+    Notes
+    -----
+    JAX
+        This allows applying eager functions to jitted JAX arrays, which are lazy.
+        The function won't be applied until the JAX array is materialized.
+        When running inside ``jax.jit``, `shape` must be fully known, i.e. it cannot
+        contain any `None` elements.
+
+        .. warning::
+
+            `func` must never raise inside ``jax.jit``, as the resulting behavior is
+            undefined.
+
+        Using this with `as_numpy=False` is particularly useful to apply non-jittable
+        JAX functions to arrays on GPU devices.
+        If ``as_numpy=True``, the :doc:`jax:transfer_guard` may prevent arrays on a GPU
+        device from being transferred back to CPU. This is treated as an implicit
+        transfer.
+
+    PyTorch, CuPy
+        If ``as_numpy=True``, these backends raise by default if you attempt to convert
+        arrays on a GPU device to NumPy.
+
+    Sparse
+        If ``as_numpy=True``, by default sparse prevents implicit densification through
+        :func:`numpy.asarray`. `This safety mechanism can be disabled
+        `_.
+
+    Dask
+        This allows applying eager functions to Dask arrays.
+        The Dask graph won't be computed.
+
+        `lazy_apply` doesn't know if `func` reduces along any axes; also, shape
+        changes are non-trivial in chunked Dask arrays. For these reasons, all inputs
+        will be rechunked into a single chunk.
+
+        .. warning::
+
+           The whole operation needs to fit in memory all at once on a single worker.
+
+        The outputs will also be returned as a single chunk and you should consider
+        rechunking them into smaller chunks afterwards.
+
+        If you want to distribute the calculation across multiple workers, you
+        should use :func:`dask.array.map_blocks`, :func:`dask.array.map_overlap`,
+        :func:`dask.array.blockwise`, or a native Dask wrapper instead of
+        `lazy_apply`.
+
+    Dask wrapping around other backends
+        If ``as_numpy=False``, `func` will receive in input eager arrays of the meta
+        namespace, as defined by the ``._meta`` attribute of the input Dask arrays.
+        The outputs of `func` will be wrapped by the meta namespace, and then wrapped
+        again by Dask.
+
+    Raises
+    ------
+    ValueError
+        When ``xp=jax.numpy``, the output `shape` is unknown (it contains ``None`` on
+        one or more axes) and this function was called inside ``jax.jit``.
+    RuntimeError
+        When ``xp=sparse`` and auto-densification is disabled.
+    Exception (backend-specific)
+        When the backend disallows implicit device to host transfers and the input
+        arrays are on a non-CPU device, e.g. on GPU.
+
+    See Also
+    --------
+    jax.transfer_guard
+    jax.pure_callback
+    dask.array.map_blocks
+    dask.array.map_overlap
+    dask.array.blockwise
+    """
+    args_not_none = [arg for arg in args if arg is not None]
+    array_args = [arg for arg in args_not_none if not is_python_scalar(arg)]
+    if not array_args:
+        msg = "Must have at least one argument array"
+        raise ValueError(msg)
+    if xp is None:
+        xp = array_namespace(*args)
+
+    # Normalize and validate shape and dtype
+    shapes: list[tuple[int | None, ...]]
+    dtypes: list[DType]
+    multi_output = False
+
+    if shape is None:
+        shapes = [broadcast_shapes(*(arg.shape for arg in array_args))]
+    elif all(isinstance(s, int | None) for s in shape):
+        # Do not test for shape to be a tuple
+        # https://github.com/data-apis/array-api/issues/891#issuecomment-2637430522
+        shapes = [cast(tuple[int | None, ...], shape)]
+    else:
+        shapes = list(shape)  # type: ignore[arg-type]  # pyright: ignore[reportAssignmentType]
+        multi_output = True
+
+    if dtype is None:
+        dtypes = [xp.result_type(*args_not_none)] * len(shapes)
+    elif multi_output:
+        if not isinstance(dtype, Sequence):
+            msg = "Got multiple shapes but only one dtype"
+            raise ValueError(msg)
+        dtypes = list(dtype)  # pyright: ignore[reportUnknownArgumentType]
+    else:
+        if isinstance(dtype, Sequence):
+            msg = "Got single shape but multiple dtypes"
+            raise ValueError(msg)
+
+        dtypes = [dtype]
+
+    if len(shapes) != len(dtypes):
+        msg = f"Got {len(shapes)} shapes and {len(dtypes)} dtypes"
+        raise ValueError(msg)
+    del shape
+    del dtype
+    # End of shape and dtype parsing
+
+    # Backend-specific branches
+    if is_dask_namespace(xp):
+        import dask
+
+        metas: list[Array] = [arg._meta for arg in array_args]  # pylint: disable=protected-access    # pyright: ignore[reportAttributeAccessIssue]
+        meta_xp = array_namespace(*metas)
+
+        wrapped = dask.delayed(  # type: ignore[attr-defined]  # pyright: ignore[reportPrivateImportUsage]
+            _lazy_apply_wrapper(func, as_numpy, multi_output, meta_xp),
+            pure=True,
+        )
+        # This finalizes each arg, which is the same as arg.rechunk(-1).
+        # Please read docstring above for why we're not using
+        # dask.array.map_blocks or dask.array.blockwise!
+        delayed_out = wrapped(*args, **kwargs)
+
+        out = tuple(
+            xp.from_delayed(
+                delayed_out[i],  # pyright: ignore[reportIndexIssue]
+                # Dask's unknown shapes diverge from the Array API specification
+                shape=tuple(math.nan if s is None else s for s in shape),
+                dtype=dtype,
+                meta=metas[0],
+            )
+            for i, (shape, dtype) in enumerate(zip(shapes, dtypes, strict=True))
+        )
+
+    elif is_jax_namespace(xp) and _is_jax_jit_enabled(xp):
+        # Delay calling func with jax.pure_callback, which will forward to func eager
+        # JAX arrays. Do not use jax.pure_callback when running outside of the JIT,
+        # as it does not support raising exceptions:
+        # https://github.com/jax-ml/jax/issues/26102
+        import jax
+
+        if any(None in shape for shape in shapes):
+            msg = "Output shape must be fully known when running inside jax.jit"
+            raise ValueError(msg)
+
+        # Shield kwargs from being coerced into JAX arrays.
+        # jax.pure_callback calls jax.jit under the hood, but without the chance of
+        # passing static_argnames / static_argnums.
+        wrapped = _lazy_apply_wrapper(
+            partial(func, **kwargs), as_numpy, multi_output, xp
+        )
+
+        # suppress unused-ignore to run mypy in -e lint as well as -e dev
+        out = cast(  # type: ignore[bad-cast,unused-ignore]
+            tuple[Array, ...],
+            jax.pure_callback(
+                wrapped,
+                tuple(
+                    jax.ShapeDtypeStruct(shape, dtype)  # pyright: ignore[reportUnknownArgumentType]
+                    for shape, dtype in zip(shapes, dtypes, strict=True)
+                ),
+                *args,
+            ),
+        )
+
+    else:
+        # Eager backends, including non-jitted JAX
+        wrapped = _lazy_apply_wrapper(func, as_numpy, multi_output, xp)
+        out = wrapped(*args, **kwargs)
+
+    return out if multi_output else out[0]
+
+
+def _is_jax_jit_enabled(xp: ModuleType) -> bool:  # numpydoc ignore=PR01,RT01
+    """Return True if this function is being called inside ``jax.jit``."""
+    import jax  # pylint: disable=import-outside-toplevel
+
+    x = xp.asarray(False)
+    try:
+        return bool(x)
+    except jax.errors.TracerBoolConversionError:
+        return True
+
+
+def _lazy_apply_wrapper(  # type: ignore[explicit-any]  # numpydoc ignore=PR01,RT01
+    func: Callable[..., Array | ArrayLike | Sequence[Array | ArrayLike]],
+    as_numpy: bool,
+    multi_output: bool,
+    xp: ModuleType,
+) -> Callable[..., tuple[Array, ...]]:
+    """
+    Helper of `lazy_apply`.
+
+    Given a function that accepts one or more arrays as positional arguments and returns
+    a single array-like or a sequence of array-likes, return a function that accepts the
+    same number of Array API arrays and always returns a tuple of Array API array.
+
+    Any keyword arguments are passed through verbatim to the wrapped function.
+    """
+
+    # On Dask, @wraps causes the graph key to contain the wrapped function's name
+    @wraps(func)
+    def wrapper(  # type: ignore[decorated-any,explicit-any]
+        *args: Array | complex | None, **kwargs: Any
+    ) -> tuple[Array, ...]:  # numpydoc ignore=GL08
+        args_list = []
+        device = None
+        for arg in args:
+            if arg is not None and not is_python_scalar(arg):
+                if device is None:
+                    device = _compat.device(arg)
+                if as_numpy:
+                    import numpy as np
+
+                    arg = cast(Array, np.asarray(arg))  # type: ignore[bad-cast]  # noqa: PLW2901
+            args_list.append(arg)
+        assert device is not None
+
+        out = func(*args_list, **kwargs)
+
+        if multi_output:
+            assert isinstance(out, Sequence)
+            return tuple(xp.asarray(o, device=device) for o in out)
+        return (xp.asarray(out, device=device),)
+
+    return wrapper
diff --git a/sklearn/externals/array_api_extra/_lib/_testing.py b/sklearn/externals/array_api_extra/_lib/_testing.py
new file mode 100644
index 0000000000000..e5ec16a64c73e
--- /dev/null
+++ b/sklearn/externals/array_api_extra/_lib/_testing.py
@@ -0,0 +1,220 @@
+"""
+Testing utilities.
+
+Note that this is private API; don't expect it to be stable.
+See also ..testing for public testing utilities.
+"""
+
+import math
+from types import ModuleType
+from typing import cast
+
+import pytest
+
+from ._utils._compat import (
+    array_namespace,
+    is_array_api_strict_namespace,
+    is_cupy_namespace,
+    is_dask_namespace,
+    is_pydata_sparse_namespace,
+    is_torch_namespace,
+)
+from ._utils._typing import Array
+
+__all__ = ["xp_assert_close", "xp_assert_equal"]
+
+
+def _check_ns_shape_dtype(
+    actual: Array, desired: Array
+) -> ModuleType:  # numpydoc ignore=RT03
+    """
+    Assert that namespace, shape and dtype of the two arrays match.
+
+    Parameters
+    ----------
+    actual : Array
+        The array produced by the tested function.
+    desired : Array
+        The expected array (typically hardcoded).
+
+    Returns
+    -------
+    Arrays namespace.
+    """
+    actual_xp = array_namespace(actual)  # Raises on scalars and lists
+    desired_xp = array_namespace(desired)
+
+    msg = f"namespaces do not match: {actual_xp} != f{desired_xp}"
+    assert actual_xp == desired_xp, msg
+
+    actual_shape = actual.shape
+    desired_shape = desired.shape
+    if is_dask_namespace(desired_xp):
+        # Dask uses nan instead of None for unknown shapes
+        if any(math.isnan(i) for i in cast(tuple[float, ...], actual_shape)):
+            actual_shape = actual.compute().shape  # type: ignore[attr-defined]  # pyright: ignore[reportAttributeAccessIssue]
+        if any(math.isnan(i) for i in cast(tuple[float, ...], desired_shape)):
+            desired_shape = desired.compute().shape  # type: ignore[attr-defined]  # pyright: ignore[reportAttributeAccessIssue]
+
+    msg = f"shapes do not match: {actual_shape} != f{desired_shape}"
+    assert actual_shape == desired_shape, msg
+
+    msg = f"dtypes do not match: {actual.dtype} != {desired.dtype}"
+    assert actual.dtype == desired.dtype, msg
+
+    return desired_xp
+
+
+def xp_assert_equal(actual: Array, desired: Array, err_msg: str = "") -> None:
+    """
+    Array-API compatible version of `np.testing.assert_array_equal`.
+
+    Parameters
+    ----------
+    actual : Array
+        The array produced by the tested function.
+    desired : Array
+        The expected array (typically hardcoded).
+    err_msg : str, optional
+        Error message to display on failure.
+
+    See Also
+    --------
+    xp_assert_close : Similar function for inexact equality checks.
+    numpy.testing.assert_array_equal : Similar function for NumPy arrays.
+    """
+    xp = _check_ns_shape_dtype(actual, desired)
+
+    if is_cupy_namespace(xp):
+        xp.testing.assert_array_equal(actual, desired, err_msg=err_msg)
+    elif is_torch_namespace(xp):
+        # PyTorch recommends using `rtol=0, atol=0` like this
+        # to test for exact equality
+        xp.testing.assert_close(
+            actual,
+            desired,
+            rtol=0,
+            atol=0,
+            equal_nan=True,
+            check_dtype=False,
+            msg=err_msg or None,
+        )
+    else:
+        import numpy as np  # pylint: disable=import-outside-toplevel
+
+        if is_pydata_sparse_namespace(xp):
+            actual = actual.todense()  # type: ignore[attr-defined]  # pyright: ignore[reportAttributeAccessIssue]
+            desired = desired.todense()  # type: ignore[attr-defined]  # pyright: ignore[reportAttributeAccessIssue]
+
+        actual_np = None
+        desired_np = None
+        if is_array_api_strict_namespace(xp):
+            # __array__ doesn't work on array-api-strict device arrays
+            # We need to convert to the CPU device first
+            actual_np = np.asarray(xp.asarray(actual, device=xp.Device("CPU_DEVICE")))
+            desired_np = np.asarray(xp.asarray(desired, device=xp.Device("CPU_DEVICE")))
+
+        # JAX/Dask arrays work with `np.testing`
+        actual_np = actual if actual_np is None else actual_np
+        desired_np = desired if desired_np is None else desired_np
+        np.testing.assert_array_equal(actual_np, desired_np, err_msg=err_msg)  # pyright: ignore[reportUnknownArgumentType]
+
+
+def xp_assert_close(
+    actual: Array,
+    desired: Array,
+    *,
+    rtol: float | None = None,
+    atol: float = 0,
+    err_msg: str = "",
+) -> None:
+    """
+    Array-API compatible version of `np.testing.assert_allclose`.
+
+    Parameters
+    ----------
+    actual : Array
+        The array produced by the tested function.
+    desired : Array
+        The expected array (typically hardcoded).
+    rtol : float, optional
+        Relative tolerance. Default: dtype-dependent.
+    atol : float, optional
+        Absolute tolerance. Default: 0.
+    err_msg : str, optional
+        Error message to display on failure.
+
+    See Also
+    --------
+    xp_assert_equal : Similar function for exact equality checks.
+    isclose : Public function for checking closeness.
+    numpy.testing.assert_allclose : Similar function for NumPy arrays.
+
+    Notes
+    -----
+    The default `atol` and `rtol` differ from `xp.all(xpx.isclose(a, b))`.
+    """
+    xp = _check_ns_shape_dtype(actual, desired)
+
+    floating = xp.isdtype(actual.dtype, ("real floating", "complex floating"))
+    if rtol is None and floating:
+        # multiplier of 4 is used as for `np.float64` this puts the default `rtol`
+        # roughly half way between sqrt(eps) and the default for
+        # `numpy.testing.assert_allclose`, 1e-7
+        rtol = xp.finfo(actual.dtype).eps ** 0.5 * 4
+    elif rtol is None:
+        rtol = 1e-7
+
+    if is_cupy_namespace(xp):
+        xp.testing.assert_allclose(
+            actual, desired, rtol=rtol, atol=atol, err_msg=err_msg
+        )
+    elif is_torch_namespace(xp):
+        xp.testing.assert_close(
+            actual, desired, rtol=rtol, atol=atol, equal_nan=True, msg=err_msg or None
+        )
+    else:
+        import numpy as np  # pylint: disable=import-outside-toplevel
+
+        if is_pydata_sparse_namespace(xp):
+            actual = actual.todense()  # type: ignore[attr-defined]  # pyright: ignore[reportAttributeAccessIssue]
+            desired = desired.todense()  # type: ignore[attr-defined]  # pyright: ignore[reportAttributeAccessIssue]
+
+        actual_np = None
+        desired_np = None
+        if is_array_api_strict_namespace(xp):
+            # __array__ doesn't work on array-api-strict device arrays
+            # We need to convert to the CPU device first
+            actual_np = np.asarray(xp.asarray(actual, device=xp.Device("CPU_DEVICE")))
+            desired_np = np.asarray(xp.asarray(desired, device=xp.Device("CPU_DEVICE")))
+
+        # JAX/Dask arrays work with `np.testing`
+        actual_np = actual if actual_np is None else actual_np
+        desired_np = desired if desired_np is None else desired_np
+
+        assert isinstance(rtol, float)
+        np.testing.assert_allclose(  # pyright: ignore[reportCallIssue]
+            actual_np,  # type: ignore[arg-type]  # pyright: ignore[reportArgumentType]
+            desired_np,  # type: ignore[arg-type]  # pyright: ignore[reportArgumentType]
+            rtol=rtol,
+            atol=atol,
+            err_msg=err_msg,
+        )
+
+
+def xfail(request: pytest.FixtureRequest, reason: str) -> None:
+    """
+    XFAIL the currently running test.
+
+    Unlike ``pytest.xfail``, allow rest of test to execute instead of immediately
+    halting it, so that it may result in a XPASS.
+    xref https://github.com/pandas-dev/pandas/issues/38902
+
+    Parameters
+    ----------
+    request : pytest.FixtureRequest
+        ``request`` argument of the test function.
+    reason : str
+        Reason for the expected failure.
+    """
+    request.node.add_marker(pytest.mark.xfail(reason=reason))
diff --git a/sklearn/externals/array_api_extra/_lib/_utils/__init__.py b/sklearn/externals/array_api_extra/_lib/_utils/__init__.py
new file mode 100644
index 0000000000000..3628c45f0e0a4
--- /dev/null
+++ b/sklearn/externals/array_api_extra/_lib/_utils/__init__.py
@@ -0,0 +1 @@
+"""Modules housing private utility functions."""
diff --git a/sklearn/externals/array_api_extra/_lib/_utils/_compat.py b/sklearn/externals/array_api_extra/_lib/_utils/_compat.py
new file mode 100644
index 0000000000000..b9997450d23b5
--- /dev/null
+++ b/sklearn/externals/array_api_extra/_lib/_utils/_compat.py
@@ -0,0 +1,70 @@
+"""Acquire helpers from array-api-compat."""
+# Allow packages that vendor both `array-api-extra` and
+# `array-api-compat` to override the import location
+
+try:
+    from ...._array_api_compat_vendor import (
+        array_namespace,
+        device,
+        is_array_api_obj,
+        is_array_api_strict_namespace,
+        is_cupy_array,
+        is_cupy_namespace,
+        is_dask_array,
+        is_dask_namespace,
+        is_jax_array,
+        is_jax_namespace,
+        is_lazy_array,
+        is_numpy_array,
+        is_numpy_namespace,
+        is_pydata_sparse_array,
+        is_pydata_sparse_namespace,
+        is_torch_array,
+        is_torch_namespace,
+        is_writeable_array,
+        size,
+    )
+except ImportError:
+    from array_api_compat import (
+        array_namespace,
+        device,
+        is_array_api_obj,
+        is_array_api_strict_namespace,
+        is_cupy_array,
+        is_cupy_namespace,
+        is_dask_array,
+        is_dask_namespace,
+        is_jax_array,
+        is_jax_namespace,
+        is_lazy_array,
+        is_numpy_array,
+        is_numpy_namespace,
+        is_pydata_sparse_array,
+        is_pydata_sparse_namespace,
+        is_torch_array,
+        is_torch_namespace,
+        is_writeable_array,
+        size,
+    )
+
+__all__ = [
+    "array_namespace",
+    "device",
+    "is_array_api_obj",
+    "is_array_api_strict_namespace",
+    "is_cupy_array",
+    "is_cupy_namespace",
+    "is_dask_array",
+    "is_dask_namespace",
+    "is_jax_array",
+    "is_jax_namespace",
+    "is_lazy_array",
+    "is_numpy_array",
+    "is_numpy_namespace",
+    "is_pydata_sparse_array",
+    "is_pydata_sparse_namespace",
+    "is_torch_array",
+    "is_torch_namespace",
+    "is_writeable_array",
+    "size",
+]
diff --git a/sklearn/externals/array_api_extra/_lib/_utils/_compat.pyi b/sklearn/externals/array_api_extra/_lib/_utils/_compat.pyi
new file mode 100644
index 0000000000000..f40d7556dee87
--- /dev/null
+++ b/sklearn/externals/array_api_extra/_lib/_utils/_compat.pyi
@@ -0,0 +1,40 @@
+"""Static type stubs for `_compat.py`."""
+
+# https://github.com/scikit-learn/scikit-learn/pull/27910#issuecomment-2568023972
+from __future__ import annotations
+
+from types import ModuleType
+
+# TODO import from typing (requires Python >=3.13)
+from typing_extensions import TypeIs
+
+from ._typing import Array, Device
+
+# pylint: disable=missing-class-docstring,unused-argument
+
+class Namespace(ModuleType):
+    def device(self, x: Array, /) -> Device: ...
+
+def array_namespace(
+    *xs: Array | complex | None,
+    api_version: str | None = None,
+    use_compat: bool | None = None,
+) -> Namespace: ...
+def device(x: Array, /) -> Device: ...
+def is_array_api_obj(x: object, /) -> TypeIs[Array]: ...
+def is_array_api_strict_namespace(xp: ModuleType, /) -> TypeIs[Namespace]: ...
+def is_cupy_namespace(xp: ModuleType, /) -> TypeIs[Namespace]: ...
+def is_dask_namespace(xp: ModuleType, /) -> TypeIs[Namespace]: ...
+def is_jax_namespace(xp: ModuleType, /) -> TypeIs[Namespace]: ...
+def is_numpy_namespace(xp: ModuleType, /) -> TypeIs[Namespace]: ...
+def is_pydata_sparse_namespace(xp: ModuleType, /) -> TypeIs[Namespace]: ...
+def is_torch_namespace(xp: ModuleType, /) -> TypeIs[Namespace]: ...
+def is_cupy_array(x: object, /) -> TypeIs[Array]: ...
+def is_dask_array(x: object, /) -> TypeIs[Array]: ...
+def is_jax_array(x: object, /) -> TypeIs[Array]: ...
+def is_numpy_array(x: object, /) -> TypeIs[Array]: ...
+def is_pydata_sparse_array(x: object, /) -> TypeIs[Array]: ...
+def is_torch_array(x: object, /) -> TypeIs[Array]: ...
+def is_lazy_array(x: object, /) -> TypeIs[Array]: ...
+def is_writeable_array(x: object, /) -> TypeIs[Array]: ...
+def size(x: Array, /) -> int | None: ...
diff --git a/sklearn/externals/array_api_extra/_lib/_utils/_helpers.py b/sklearn/externals/array_api_extra/_lib/_utils/_helpers.py
new file mode 100644
index 0000000000000..9882d72e6c0ac
--- /dev/null
+++ b/sklearn/externals/array_api_extra/_lib/_utils/_helpers.py
@@ -0,0 +1,272 @@
+"""Helper functions used by `array_api_extra/_funcs.py`."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Generator, Iterable
+from types import ModuleType
+from typing import TYPE_CHECKING, cast
+
+from . import _compat
+from ._compat import (
+    array_namespace,
+    is_array_api_obj,
+    is_dask_namespace,
+    is_numpy_array,
+)
+from ._typing import Array
+
+if TYPE_CHECKING:  # pragma: no cover
+    # TODO import from typing (requires Python >=3.13)
+    from typing_extensions import TypeIs
+
+
+__all__ = [
+    "asarrays",
+    "eager_shape",
+    "in1d",
+    "is_python_scalar",
+    "mean",
+    "meta_namespace",
+]
+
+
+def in1d(
+    x1: Array,
+    x2: Array,
+    /,
+    *,
+    assume_unique: bool = False,
+    invert: bool = False,
+    xp: ModuleType | None = None,
+) -> Array:  # numpydoc ignore=PR01,RT01
+    """
+    Check whether each element of an array is also present in a second array.
+
+    Returns a boolean array the same length as `x1` that is True
+    where an element of `x1` is in `x2` and False otherwise.
+
+    This function has been adapted using the original implementation
+    present in numpy:
+    https://github.com/numpy/numpy/blob/v1.26.0/numpy/lib/arraysetops.py#L524-L758
+    """
+    if xp is None:
+        xp = array_namespace(x1, x2)
+
+    x1_shape = eager_shape(x1)
+    x2_shape = eager_shape(x2)
+
+    # This code is run to make the code significantly faster
+    if x2_shape[0] < 10 * x1_shape[0] ** 0.145 and isinstance(x2, Iterable):
+        if invert:
+            mask = xp.ones(x1_shape[0], dtype=xp.bool, device=_compat.device(x1))
+            for a in x2:
+                mask &= x1 != a
+        else:
+            mask = xp.zeros(x1_shape[0], dtype=xp.bool, device=_compat.device(x1))
+            for a in x2:
+                mask |= x1 == a
+        return mask
+
+    rev_idx = xp.empty(0)  # placeholder
+    if not assume_unique:
+        x1, rev_idx = xp.unique_inverse(x1)
+        x2 = xp.unique_values(x2)
+
+    ar = xp.concat((x1, x2))
+    device_ = _compat.device(ar)
+    # We need this to be a stable sort.
+    order = xp.argsort(ar, stable=True)
+    reverse_order = xp.argsort(order, stable=True)
+    sar = xp.take(ar, order, axis=0)
+    ar_size = _compat.size(sar)
+    assert ar_size is not None, "xp.unique*() on lazy backends raises"
+    if ar_size >= 1:
+        bool_ar = sar[1:] != sar[:-1] if invert else sar[1:] == sar[:-1]
+    else:
+        bool_ar = xp.asarray([False]) if invert else xp.asarray([True])
+    flag = xp.concat((bool_ar, xp.asarray([invert], device=device_)))
+    ret = xp.take(flag, reverse_order, axis=0)
+
+    if assume_unique:
+        return ret[: x1.shape[0]]
+    return xp.take(ret, rev_idx, axis=0)
+
+
+def mean(
+    x: Array,
+    /,
+    *,
+    axis: int | tuple[int, ...] | None = None,
+    keepdims: bool = False,
+    xp: ModuleType | None = None,
+) -> Array:  # numpydoc ignore=PR01,RT01
+    """
+    Complex mean, https://github.com/data-apis/array-api/issues/846.
+    """
+    if xp is None:
+        xp = array_namespace(x)
+
+    if xp.isdtype(x.dtype, "complex floating"):
+        x_real = xp.real(x)
+        x_imag = xp.imag(x)
+        mean_real = xp.mean(x_real, axis=axis, keepdims=keepdims)
+        mean_imag = xp.mean(x_imag, axis=axis, keepdims=keepdims)
+        return mean_real + (mean_imag * xp.asarray(1j))
+    return xp.mean(x, axis=axis, keepdims=keepdims)
+
+
+def is_python_scalar(x: object) -> TypeIs[complex]:  # numpydoc ignore=PR01,RT01
+    """Return True if `x` is a Python scalar, False otherwise."""
+    # isinstance(x, float) returns True for np.float64
+    # isinstance(x, complex) returns True for np.complex128
+    # bool is a subclass of int
+    return isinstance(x, int | float | complex) and not is_numpy_array(x)
+
+
+def asarrays(
+    a: Array | complex,
+    b: Array | complex,
+    xp: ModuleType,
+) -> tuple[Array, Array]:
+    """
+    Ensure both `a` and `b` are arrays.
+
+    If `b` is a python scalar, it is converted to the same dtype as `a`, and vice versa.
+
+    Behavior is not specified when mixing a Python ``float`` and an array with an
+    integer data type; this may give ``float32``, ``float64``, or raise an exception.
+    Behavior is implementation-specific.
+
+    Similarly, behavior is not specified when mixing a Python ``complex`` and an array
+    with a real-valued data type; this may give ``complex64``, ``complex128``, or raise
+    an exception. Behavior is implementation-specific.
+
+    Parameters
+    ----------
+    a, b : Array | int | float | complex | bool
+        Input arrays or scalars. At least one must be an array.
+    xp : array_namespace, optional
+        The standard-compatible namespace for `x`. Default: infer.
+
+    Returns
+    -------
+    Array, Array
+        The input arrays, possibly converted to arrays if they were scalars.
+
+    See Also
+    --------
+    mixing-arrays-with-python-scalars : Array API specification for the behavior.
+    """
+    a_scalar = is_python_scalar(a)
+    b_scalar = is_python_scalar(b)
+    if not a_scalar and not b_scalar:
+        # This includes misc. malformed input e.g. str
+        return a, b  # type: ignore[return-value]
+
+    swap = False
+    if a_scalar:
+        swap = True
+        b, a = a, b
+
+    if is_array_api_obj(a):
+        # a is an Array API object
+        # b is a int | float | complex | bool
+        xa = a
+
+        # https://data-apis.org/array-api/draft/API_specification/type_promotion.html#mixing-arrays-with-python-scalars
+        same_dtype = {
+            bool: "bool",
+            int: ("integral", "real floating", "complex floating"),
+            float: ("real floating", "complex floating"),
+            complex: "complex floating",
+        }
+        kind = same_dtype[type(cast(complex, b))]  # type: ignore[index]
+        if xp.isdtype(a.dtype, kind):
+            xb = xp.asarray(b, dtype=a.dtype)
+        else:
+            # Undefined behaviour. Let the function deal with it, if it can.
+            xb = xp.asarray(b)
+
+    else:
+        # Neither a nor b are Array API objects.
+        # Note: we can only reach this point when one explicitly passes
+        # xp=xp to the calling function; otherwise we fail earlier on
+        # array_namespace(a, b).
+        xa, xb = xp.asarray(a), xp.asarray(b)
+
+    return (xb, xa) if swap else (xa, xb)
+
+
+def ndindex(*x: int) -> Generator[tuple[int, ...]]:
+    """
+    Generate all N-dimensional indices for a given array shape.
+
+    Given the shape of an array, an ndindex instance iterates over the N-dimensional
+    index of the array. At each iteration a tuple of indices is returned, the last
+    dimension is iterated over first.
+
+    This has an identical API to numpy.ndindex.
+
+    Parameters
+    ----------
+    *x : int
+        The shape of the array.
+    """
+    if not x:
+        yield ()
+        return
+    for i in ndindex(*x[:-1]):
+        for j in range(x[-1]):
+            yield *i, j
+
+
+def eager_shape(x: Array, /) -> tuple[int, ...]:
+    """
+    Return shape of an array. Raise if shape is not fully defined.
+
+    Parameters
+    ----------
+    x : Array
+        Input array.
+
+    Returns
+    -------
+    tuple[int, ...]
+        Shape of the array.
+    """
+    shape = x.shape
+    # Dask arrays uses non-standard NaN instead of None
+    if any(s is None or math.isnan(s) for s in shape):
+        msg = "Unsupported lazy shape"
+        raise TypeError(msg)
+    return cast(tuple[int, ...], shape)
+
+
+def meta_namespace(
+    *arrays: Array | complex | None, xp: ModuleType | None = None
+) -> ModuleType:
+    """
+    Get the namespace of Dask chunks.
+
+    On all other backends, just return the namespace of the arrays.
+
+    Parameters
+    ----------
+    *arrays : Array | int | float | complex | bool | None
+        Input arrays.
+    xp : array_namespace, optional
+        The standard-compatible namespace for the input arrays. Default: infer.
+
+    Returns
+    -------
+    array_namespace
+        If xp is Dask, the namespace of the Dask chunks;
+        otherwise, the namespace of the arrays.
+    """
+    xp = array_namespace(*arrays) if xp is None else xp
+    if not is_dask_namespace(xp):
+        return xp
+    # Quietly skip scalars and None's
+    metas = [cast(Array | None, getattr(a, "_meta", None)) for a in arrays]
+    return array_namespace(*metas)
diff --git a/sklearn/externals/array_api_extra/_lib/_utils/_typing.py b/sklearn/externals/array_api_extra/_lib/_utils/_typing.py
new file mode 100644
index 0000000000000..d32a3a07c1ee9
--- /dev/null
+++ b/sklearn/externals/array_api_extra/_lib/_utils/_typing.py
@@ -0,0 +1,10 @@
+# numpydoc ignore=GL08
+# pylint: disable=missing-module-docstring
+
+Array = object
+DType = object
+Device = object
+GetIndex = object
+SetIndex = object
+
+__all__ = ["Array", "DType", "Device", "GetIndex", "SetIndex"]
diff --git a/sklearn/externals/array_api_extra/_lib/_utils/_typing.pyi b/sklearn/externals/array_api_extra/_lib/_utils/_typing.pyi
new file mode 100644
index 0000000000000..e32a59bd0cb9e
--- /dev/null
+++ b/sklearn/externals/array_api_extra/_lib/_utils/_typing.pyi
@@ -0,0 +1,105 @@
+"""Static typing helpers."""
+
+from __future__ import annotations
+
+from types import EllipsisType
+from typing import Protocol, TypeAlias
+
+# TODO import from typing (requires Python >=3.12)
+from typing_extensions import override
+
+# TODO: use array-api-typing once it is available
+
+class Array(Protocol):  # pylint: disable=missing-class-docstring
+    # Unary operations
+    def __abs__(self) -> Array: ...
+    def __pos__(self) -> Array: ...
+    def __neg__(self) -> Array: ...
+    def __invert__(self) -> Array: ...
+    # Binary operations
+    def __add__(self, other: Array | complex, /) -> Array: ...
+    def __sub__(self, other: Array | complex, /) -> Array: ...
+    def __mul__(self, other: Array | complex, /) -> Array: ...
+    def __truediv__(self, other: Array | complex, /) -> Array: ...
+    def __floordiv__(self, other: Array | complex, /) -> Array: ...
+    def __mod__(self, other: Array | complex, /) -> Array: ...
+    def __pow__(self, other: Array | complex, /) -> Array: ...
+    def __matmul__(self, other: Array, /) -> Array: ...
+    def __and__(self, other: Array | int, /) -> Array: ...
+    def __or__(self, other: Array | int, /) -> Array: ...
+    def __xor__(self, other: Array | int, /) -> Array: ...
+    def __lshift__(self, other: Array | int, /) -> Array: ...
+    def __rshift__(self, other: Array | int, /) -> Array: ...
+    def __lt__(self, other: Array | complex, /) -> Array: ...
+    def __le__(self, other: Array | complex, /) -> Array: ...
+    def __gt__(self, other: Array | complex, /) -> Array: ...
+    def __ge__(self, other: Array | complex, /) -> Array: ...
+    @override
+    def __eq__(self, other: Array | complex, /) -> Array: ...  # type: ignore[override]  # pyright: ignore[reportIncompatibleMethodOverride]
+    @override
+    def __ne__(self, other: Array | complex, /) -> Array: ...  # type: ignore[override]  # pyright: ignore[reportIncompatibleMethodOverride]
+    # Reflected operations
+    def __radd__(self, other: Array | complex, /) -> Array: ...
+    def __rsub__(self, other: Array | complex, /) -> Array: ...
+    def __rmul__(self, other: Array | complex, /) -> Array: ...
+    def __rtruediv__(self, other: Array | complex, /) -> Array: ...
+    def __rfloordiv__(self, other: Array | complex, /) -> Array: ...
+    def __rmod__(self, other: Array | complex, /) -> Array: ...
+    def __rpow__(self, other: Array | complex, /) -> Array: ...
+    def __rmatmul__(self, other: Array, /) -> Array: ...
+    def __rand__(self, other: Array | int, /) -> Array: ...
+    def __ror__(self, other: Array | int, /) -> Array: ...
+    def __rxor__(self, other: Array | int, /) -> Array: ...
+    def __rlshift__(self, other: Array | int, /) -> Array: ...
+    def __rrshift__(self, other: Array | int, /) -> Array: ...
+    # Attributes
+    @property
+    def dtype(self) -> DType: ...
+    @property
+    def device(self) -> Device: ...
+    @property
+    def mT(self) -> Array: ...  # pylint: disable=invalid-name
+    @property
+    def ndim(self) -> int: ...
+    @property
+    def shape(self) -> tuple[int | None, ...]: ...
+    @property
+    def size(self) -> int | None: ...
+    @property
+    def T(self) -> Array: ...  # pylint: disable=invalid-name
+    # Collection operations (note: an Array does not have to be Sized or Iterable)
+    def __getitem__(self, key: GetIndex, /) -> Array: ...
+    def __setitem__(self, key: SetIndex, value: Array | complex, /) -> None: ...
+    # Materialization methods (may raise on lazy arrays)
+    def __bool__(self) -> bool: ...
+    def __complex__(self) -> complex: ...
+    def __float__(self) -> float: ...
+    def __index__(self) -> int: ...
+    def __int__(self) -> int: ...
+
+    # Misc methods (frequently not implemented in Arrays wrapped by array-api-compat)
+    # def __array_namespace__(*, api_version: str | None) -> ModuleType: ...
+    # def __dlpack__(
+    #     *,
+    #     stream: int | Any | None = None,
+    #     max_version: tuple[int, int] | None = None,
+    #     dl_device: tuple[int, int] | None = None,  # tuple[Enum, int]
+    #     copy: bool | None = None,
+    # ) -> Any: ...
+    # def __dlpack_device__() -> tuple[int, int]: ...  # tuple[Enum, int]
+    # def to_device(device: Device, /, *, stream: int | Any | None = None) -> Array: ...
+
+class DType(Protocol):  # pylint: disable=missing-class-docstring
+    pass
+
+class Device(Protocol):  # pylint: disable=missing-class-docstring
+    pass
+
+SetIndex: TypeAlias = (  # type: ignore[explicit-any]
+    int | slice | EllipsisType | Array | tuple[int | slice | EllipsisType | Array, ...]
+)
+GetIndex: TypeAlias = (  # type: ignore[explicit-any]
+    SetIndex | None | tuple[int | slice | EllipsisType | None | Array, ...]
+)
+
+__all__ = ["Array", "DType", "Device", "GetIndex", "SetIndex"]
diff --git a/sklearn/externals/array_api_extra/py.typed b/sklearn/externals/array_api_extra/py.typed
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/sklearn/externals/array_api_extra/testing.py b/sklearn/externals/array_api_extra/testing.py
new file mode 100644
index 0000000000000..4f8288cf582ec
--- /dev/null
+++ b/sklearn/externals/array_api_extra/testing.py
@@ -0,0 +1,324 @@
+"""
+Public testing utilities.
+
+See also _lib._testing for additional private testing utilities.
+"""
+
+from __future__ import annotations
+
+import contextlib
+from collections.abc import Callable, Iterable, Iterator, Sequence
+from functools import wraps
+from types import ModuleType
+from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar, cast
+
+from ._lib._utils._compat import is_dask_namespace, is_jax_namespace
+
+__all__ = ["lazy_xp_function", "patch_lazy_xp_functions"]
+
+if TYPE_CHECKING:  # pragma: no cover
+    # TODO import override from typing (requires Python >=3.12)
+    import pytest
+    from dask.typing import Graph, Key, SchedulerGetCallable
+    from typing_extensions import override
+
+else:
+    # Sphinx hacks
+    SchedulerGetCallable = object
+
+    def override(func: object) -> object:
+        return func
+
+
+P = ParamSpec("P")
+T = TypeVar("T")
+
+_ufuncs_tags: dict[object, dict[str, Any]] = {}  # type: ignore[explicit-any]
+
+
+def lazy_xp_function(  # type: ignore[explicit-any]
+    func: Callable[..., Any],
+    *,
+    allow_dask_compute: int = 0,
+    jax_jit: bool = True,
+    static_argnums: int | Sequence[int] | None = None,
+    static_argnames: str | Iterable[str] | None = None,
+) -> None:  # numpydoc ignore=GL07
+    """
+    Tag a function to be tested on lazy backends.
+
+    Tag a function so that when any tests are executed with ``xp=jax.numpy`` the
+    function is replaced with a jitted version of itself, and when it is executed with
+    ``xp=dask.array`` the function will raise if it attempts to materialize the graph.
+    This will be later expanded to provide test coverage for other lazy backends.
+
+    In order for the tag to be effective, the test or a fixture must call
+    :func:`patch_lazy_xp_functions`.
+
+    Parameters
+    ----------
+    func : callable
+        Function to be tested.
+    allow_dask_compute : int, optional
+        Number of times `func` is allowed to internally materialize the Dask graph. This
+        is typically triggered by ``bool()``, ``float()``, or ``np.asarray()``.
+
+        Set to 1 if you are aware that `func` converts the input parameters to NumPy and
+        want to let it do so at least for the time being, knowing that it is going to be
+        extremely detrimental for performance.
+
+        If a test needs values higher than 1 to pass, it is a canary that the conversion
+        to NumPy/bool/float is happening multiple times, which translates to multiple
+        computations of the whole graph. Short of making the function fully lazy, you
+        should at least add explicit calls to ``np.asarray()`` early in the function.
+        *Note:* the counter of `allow_dask_compute` resets after each call to `func`, so
+        a test function that invokes `func` multiple times should still work with this
+        parameter set to 1.
+
+        Default: 0, meaning that `func` must be fully lazy and never materialize the
+        graph.
+    jax_jit : bool, optional
+        Set to True to replace `func` with ``jax.jit(func)`` after calling the
+        :func:`patch_lazy_xp_functions` test helper with ``xp=jax.numpy``. Set to False
+        if `func` is only compatible with eager (non-jitted) JAX. Default: True.
+    static_argnums : int | Sequence[int], optional
+        Passed to jax.jit. Positional arguments to treat as static (compile-time
+        constant). Default: infer from `static_argnames` using
+        `inspect.signature(func)`.
+    static_argnames : str | Iterable[str], optional
+        Passed to jax.jit. Named arguments to treat as static (compile-time constant).
+        Default: infer from `static_argnums` using `inspect.signature(func)`.
+
+    See Also
+    --------
+    patch_lazy_xp_functions : Companion function to call from the test or fixture.
+    jax.jit : JAX function to compile a function for performance.
+
+    Examples
+    --------
+    In ``test_mymodule.py``::
+
+      from array_api_extra.testing import lazy_xp_function from mymodule import myfunc
+
+      lazy_xp_function(myfunc)
+
+      def test_myfunc(xp):
+          a = xp.asarray([1, 2])
+          # When xp=jax.numpy, this is the same as `b = jax.jit(myfunc)(a)`
+          # When xp=dask.array, crash on compute() or persist()
+          b = myfunc(a)
+
+    Notes
+    -----
+    In order for this tag to be effective, the test function must be imported into the
+    test module globals without its namespace; alternatively its namespace must be
+    declared in a ``lazy_xp_modules`` list in the test module globals.
+
+    Example 1::
+
+      from mymodule import myfunc
+
+      lazy_xp_function(myfunc)
+
+      def test_myfunc(xp):
+          x = myfunc(xp.asarray([1, 2]))
+
+    Example 2::
+
+      import mymodule
+
+      lazy_xp_modules = [mymodule]
+      lazy_xp_function(mymodule.myfunc)
+
+      def test_myfunc(xp):
+          x = mymodule.myfunc(xp.asarray([1, 2]))
+
+    A test function can circumvent this monkey-patching system by using a namespace
+    outside of the two above patterns. You need to sanitize your code to make sure this
+    only happens intentionally.
+
+    Example 1::
+
+      import mymodule
+      from mymodule import myfunc
+
+      lazy_xp_function(myfunc)
+
+      def test_myfunc(xp):
+          a = xp.asarray([1, 2])
+          b = myfunc(a)  # This is wrapped when xp=jax.numpy or xp=dask.array
+          c = mymodule.myfunc(a)  # This is not
+
+    Example 2::
+
+      import mymodule
+
+      class naked:
+          myfunc = mymodule.myfunc
+
+      lazy_xp_modules = [mymodule]
+      lazy_xp_function(mymodule.myfunc)
+
+      def test_myfunc(xp):
+          a = xp.asarray([1, 2])
+          b = mymodule.myfunc(a)  # This is wrapped when xp=jax.numpy or xp=dask.array
+          c = naked.myfunc(a)  # This is not
+    """
+    tags = {
+        "allow_dask_compute": allow_dask_compute,
+        "jax_jit": jax_jit,
+        "static_argnums": static_argnums,
+        "static_argnames": static_argnames,
+    }
+    try:
+        func._lazy_xp_function = tags  # type: ignore[attr-defined]  # pylint: disable=protected-access  # pyright: ignore[reportFunctionMemberAccess]
+    except AttributeError:  # @cython.vectorize
+        _ufuncs_tags[func] = tags
+
+
+def patch_lazy_xp_functions(
+    request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch, *, xp: ModuleType
+) -> None:
+    """
+    Test lazy execution of functions tagged with :func:`lazy_xp_function`.
+
+    If ``xp==jax.numpy``, search for all functions which have been tagged with
+    :func:`lazy_xp_function` in the globals of the module that defines the current test,
+    as well as in the ``lazy_xp_modules`` list in the globals of the same module,
+    and wrap them with :func:`jax.jit`. Unwrap them at the end of the test.
+
+    If ``xp==dask.array``, wrap the functions with a decorator that disables
+    ``compute()`` and ``persist()`` and ensures that exceptions and warnings are raised
+    eagerly.
+
+    This function should be typically called by your library's `xp` fixture that runs
+    tests on multiple backends::
+
+        @pytest.fixture(params=[numpy, array_api_strict, jax.numpy, dask.array])
+        def xp(request, monkeypatch):
+            patch_lazy_xp_functions(request, monkeypatch, xp=request.param)
+            return request.param
+
+    but it can be otherwise be called by the test itself too.
+
+    Parameters
+    ----------
+    request : pytest.FixtureRequest
+        Pytest fixture, as acquired by the test itself or by one of its fixtures.
+    monkeypatch : pytest.MonkeyPatch
+        Pytest fixture, as acquired by the test itself or by one of its fixtures.
+    xp : array_namespace
+        Array namespace to be tested.
+
+    See Also
+    --------
+    lazy_xp_function : Tag a function to be tested on lazy backends.
+    pytest.FixtureRequest : `request` test function parameter.
+    """
+    mod = cast(ModuleType, request.module)
+    mods = [mod, *cast(list[ModuleType], getattr(mod, "lazy_xp_modules", []))]
+
+    def iter_tagged() -> (  # type: ignore[explicit-any]
+        Iterator[tuple[ModuleType, str, Callable[..., Any], dict[str, Any]]]
+    ):
+        for mod in mods:
+            for name, func in mod.__dict__.items():
+                tags: dict[str, Any] | None = None  # type: ignore[explicit-any]
+                with contextlib.suppress(AttributeError):
+                    tags = func._lazy_xp_function  # pylint: disable=protected-access
+                if tags is None:
+                    with contextlib.suppress(KeyError, TypeError):
+                        tags = _ufuncs_tags[func]
+                if tags is not None:
+                    yield mod, name, func, tags
+
+    if is_dask_namespace(xp):
+        for mod, name, func, tags in iter_tagged():
+            n = tags["allow_dask_compute"]
+            wrapped = _dask_wrap(func, n)
+            monkeypatch.setattr(mod, name, wrapped)
+
+    elif is_jax_namespace(xp):
+        import jax
+
+        for mod, name, func, tags in iter_tagged():
+            if tags["jax_jit"]:
+                # suppress unused-ignore to run mypy in -e lint as well as -e dev
+                wrapped = cast(  # type: ignore[explicit-any]
+                    Callable[..., Any],
+                    jax.jit(
+                        func,
+                        static_argnums=tags["static_argnums"],
+                        static_argnames=tags["static_argnames"],
+                    ),
+                )
+                monkeypatch.setattr(mod, name, wrapped)
+
+
+class CountingDaskScheduler(SchedulerGetCallable):
+    """
+    Dask scheduler that counts how many times `dask.compute` is called.
+
+    If the number of times exceeds 'max_count', it raises an error.
+    This is a wrapper around Dask's own 'synchronous' scheduler.
+
+    Parameters
+    ----------
+    max_count : int
+        Maximum number of allowed calls to `dask.compute`.
+    msg : str
+        Assertion to raise when the count exceeds `max_count`.
+    """
+
+    count: int
+    max_count: int
+    msg: str
+
+    def __init__(self, max_count: int, msg: str):  # numpydoc ignore=GL08
+        self.count = 0
+        self.max_count = max_count
+        self.msg = msg
+
+    @override
+    def __call__(self, dsk: Graph, keys: Sequence[Key] | Key, **kwargs: Any) -> Any:  # type: ignore[decorated-any,explicit-any] # numpydoc ignore=GL08
+        import dask
+
+        self.count += 1
+        # This should yield a nice traceback to the
+        # offending line in the user's code
+        assert self.count <= self.max_count, self.msg
+
+        return dask.get(dsk, keys, **kwargs)  # type: ignore[attr-defined,no-untyped-call] # pyright: ignore[reportPrivateImportUsage]
+
+
+def _dask_wrap(
+    func: Callable[P, T], n: int
+) -> Callable[P, T]:  # numpydoc ignore=PR01,RT01
+    """
+    Wrap `func` to raise if it attempts to call `dask.compute` more than `n` times.
+
+    After the function returns, materialize the graph in order to re-raise exceptions.
+    """
+    import dask
+
+    func_name = getattr(func, "__name__", str(func))
+    n_str = f"only up to {n}" if n else "no"
+    msg = (
+        f"Called `dask.compute()` or `dask.persist()` {n + 1} times, "
+        f"but {n_str} calls are allowed. Set "
+        f"`lazy_xp_function({func_name}, allow_dask_compute={n + 1})` "
+        "to allow for more (but note that this will harm performance). "
+    )
+
+    @wraps(func)
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:  # numpydoc ignore=GL08
+        scheduler = CountingDaskScheduler(n, msg)
+        with dask.config.set({"scheduler": scheduler}):  # pyright: ignore[reportPrivateImportUsage]
+            out = func(*args, **kwargs)
+
+        # Block until the graph materializes and reraise exceptions. This allows
+        # `pytest.raises` and `pytest.warns` to work as expected. Note that this would
+        # not work on scheduler='distributed', as it would not block.
+        return dask.persist(out, scheduler="threads")[0]  # type: ignore[attr-defined,no-untyped-call,func-returns-value,index]  # pyright: ignore[reportPrivateImportUsage]
+
+    return wrapper
diff --git a/sklearn/externals/conftest.py b/sklearn/externals/conftest.py
index c617107866b92..c763d9761a438 100644
--- a/sklearn/externals/conftest.py
+++ b/sklearn/externals/conftest.py
@@ -2,6 +2,5 @@
 # --ignore because --ignore needs a path and it is not convenient to pass in
 # the externals path (very long install-dependent path in site-packages) when
 # using --pyargs
-def pytest_ignore_collect(path, config):
+def pytest_ignore_collect(collection_path, config):
     return True
-
diff --git a/sklearn/feature_extraction/__init__.py b/sklearn/feature_extraction/__init__.py
index 4591bfc6980c8..0f8c53b4ffb6b 100644
--- a/sklearn/feature_extraction/__init__.py
+++ b/sklearn/feature_extraction/__init__.py
@@ -1,13 +1,18 @@
-"""
-The :mod:`sklearn.feature_extraction` module deals with feature extraction
-from raw data. It currently includes methods to extract features from text and
-images.
-"""
+"""Feature extraction from raw data."""
 
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+from . import image, text
 from ._dict_vectorizer import DictVectorizer
 from ._hash import FeatureHasher
-from .image import img_to_graph, grid_to_graph
-from . import text
+from .image import grid_to_graph, img_to_graph
 
-__all__ = ['DictVectorizer', 'image', 'img_to_graph', 'grid_to_graph', 'text',
-           'FeatureHasher']
+__all__ = [
+    "DictVectorizer",
+    "FeatureHasher",
+    "grid_to_graph",
+    "image",
+    "img_to_graph",
+    "text",
+]
diff --git a/sklearn/feature_extraction/_dict_vectorizer.py b/sklearn/feature_extraction/_dict_vectorizer.py
index e0516407c205a..689146bd229d8 100644
--- a/sklearn/feature_extraction/_dict_vectorizer.py
+++ b/sklearn/feature_extraction/_dict_vectorizer.py
@@ -1,26 +1,19 @@
-# Authors: Lars Buitinck
-#          Dan Blanchard 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 from array import array
-from collections.abc import Mapping, Iterable
-from operator import itemgetter
+from collections.abc import Iterable, Mapping
 from numbers import Number
+from operator import itemgetter
 
 import numpy as np
 import scipy.sparse as sp
 
-from ..base import BaseEstimator, TransformerMixin
-from ..utils import check_array, tosequence
-from ..utils.validation import _deprecate_positional_args
-
+from sklearn.utils import metadata_routing
 
-def _tosequence(X):
-    """Turn X into a sequence or ndarray, avoiding a copy if possible."""
-    if isinstance(X, Mapping):  # single sample
-        return [X]
-    else:
-        return tosequence(X)
+from ..base import BaseEstimator, TransformerMixin, _fit_context
+from ..utils import check_array
+from ..utils.validation import check_is_fitted
 
 
 class DictVectorizer(TransformerMixin, BaseEstimator):
@@ -50,6 +43,9 @@ class DictVectorizer(TransformerMixin, BaseEstimator):
     Features that do not occur in a sample (mapping) will have a zero value
     in the resulting array/matrix.
 
+    For an efficiency comparison of the different feature extractors, see
+    :ref:`sphx_glr_auto_examples_text_plot_hashing_vs_dict_vectorizer.py`.
+
     Read more in the :ref:`User Guide `.
 
     Parameters
@@ -75,6 +71,12 @@ class DictVectorizer(TransformerMixin, BaseEstimator):
         A list of length n_features containing the feature names (e.g., "f=ham"
         and "f=spam").
 
+    See Also
+    --------
+    FeatureHasher : Performs vectorization using only a hash function.
+    sklearn.preprocessing.OrdinalEncoder : Handles nominal/categorical
+        features encoded as columns of arbitrary data types.
+
     Examples
     --------
     >>> from sklearn.feature_extraction import DictVectorizer
@@ -89,33 +91,47 @@ class DictVectorizer(TransformerMixin, BaseEstimator):
     True
     >>> v.transform({'foo': 4, 'unseen_feature': 3})
     array([[0., 0., 4.]])
-
-    See Also
-    --------
-    FeatureHasher : Performs vectorization using only a hash function.
-    sklearn.preprocessing.OrdinalEncoder : Handles nominal/categorical
-        features encoded as columns of arbitrary data types.
     """
-    @_deprecate_positional_args
-    def __init__(self, *, dtype=np.float64, separator="=", sparse=True,
-                 sort=True):
+
+    # This isn't something that people should be routing / using in a pipeline.
+    __metadata_request__inverse_transform = {"dict_type": metadata_routing.UNUSED}
+
+    _parameter_constraints: dict = {
+        "dtype": "no_validation",  # validation delegated to numpy,
+        "separator": [str],
+        "sparse": ["boolean"],
+        "sort": ["boolean"],
+    }
+
+    def __init__(self, *, dtype=np.float64, separator="=", sparse=True, sort=True):
         self.dtype = dtype
         self.separator = separator
         self.sparse = sparse
         self.sort = sort
 
-    def _add_iterable_element(self, f, v, feature_names, vocab, *,
-                              fitting=True, transforming=False,
-                              indices=None, values=None):
+    def _add_iterable_element(
+        self,
+        f,
+        v,
+        feature_names,
+        vocab,
+        *,
+        fitting=True,
+        transforming=False,
+        indices=None,
+        values=None,
+    ):
         """Add feature names for iterable of strings"""
         for vv in v:
             if isinstance(vv, str):
                 feature_name = "%s%s%s" % (f, self.separator, vv)
                 vv = 1
             else:
-                raise TypeError(f'Unsupported type {type(vv)} in iterable '
-                                'value. Only iterables of string are '
-                                'supported.')
+                raise TypeError(
+                    f"Unsupported type {type(vv)} in iterable "
+                    "value. Only iterables of string are "
+                    "supported."
+                )
             if fitting and feature_name not in vocab:
                 vocab[feature_name] = len(feature_names)
                 feature_names.append(feature_name)
@@ -124,8 +140,7 @@ def _add_iterable_element(self, f, v, feature_names, vocab, *,
                 indices.append(vocab[feature_name])
                 values.append(self.dtype(vv))
 
-        return
-
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y=None):
         """Learn a list of feature name -> indices mappings.
 
@@ -139,10 +154,12 @@ def fit(self, X, y=None):
                Accepts multiple string values for one categorical feature.
 
         y : (ignored)
+            Ignored parameter.
 
         Returns
         -------
-        self
+        self : object
+            DictVectorizer class instance.
         """
         feature_names = []
         vocab = {}
@@ -151,13 +168,14 @@ def fit(self, X, y=None):
             for f, v in x.items():
                 if isinstance(v, str):
                     feature_name = "%s%s%s" % (f, self.separator, v)
-                    v = 1
                 elif isinstance(v, Number) or (v is None):
                     feature_name = f
                 elif isinstance(v, Mapping):
-                    raise TypeError(f'Unsupported value type {type(v)} '
-                                    f'for {f}: {v}.\n'
-                                    'Mapping objects are not supported.')
+                    raise TypeError(
+                        f"Unsupported value type {type(v)} "
+                        f"for {f}: {v}.\n"
+                        "Mapping objects are not supported."
+                    )
                 elif isinstance(v, Iterable):
                     feature_name = None
                     self._add_iterable_element(f, v, feature_names, vocab)
@@ -184,7 +202,8 @@ def _transform(self, X, fitting):
         assert array("i").itemsize == 4, (
             "sizeof(int) != 4 on your platform; please report this at"
             " https://github.com/scikit-learn/scikit-learn/issues and"
-            " include the output from platform.platform() in your bug report")
+            " include the output from platform.platform() in your bug report"
+        )
 
         dtype = self.dtype
         if fitting:
@@ -214,16 +233,24 @@ def _transform(self, X, fitting):
                     v = 1
                 elif isinstance(v, Number) or (v is None):
                     feature_name = f
-                elif isinstance(v, Mapping):
-                    raise TypeError(f'Unsupported value Type {type(v)} '
-                                    f'for {f}: {v}.\n'
-                                    'Mapping objects are not supported.')
-                elif isinstance(v, Iterable):
+                elif not isinstance(v, Mapping) and isinstance(v, Iterable):
                     feature_name = None
-                    self._add_iterable_element(f, v, feature_names, vocab,
-                                               fitting=fitting,
-                                               transforming=transforming,
-                                               indices=indices, values=values)
+                    self._add_iterable_element(
+                        f,
+                        v,
+                        feature_names,
+                        vocab,
+                        fitting=fitting,
+                        transforming=transforming,
+                        indices=indices,
+                        values=values,
+                    )
+                else:
+                    raise TypeError(
+                        f"Unsupported value Type {type(v)} "
+                        f"for {f}: {v}.\n"
+                        f"{type(v)} objects are not supported."
+                    )
 
                 if feature_name is not None:
                     if fitting and feature_name not in vocab:
@@ -242,8 +269,9 @@ def _transform(self, X, fitting):
         indices = np.frombuffer(indices, dtype=np.intc)
         shape = (len(indptr) - 1, len(vocab))
 
-        result_matrix = sp.csr_matrix((values, indices, indptr),
-                                      shape=shape, dtype=dtype)
+        result_matrix = sp.csr_matrix(
+            (values, indices, indptr), shape=shape, dtype=dtype
+        )
 
         # Sort everything if asked
         if fitting and self.sort:
@@ -265,6 +293,7 @@ def _transform(self, X, fitting):
 
         return result_matrix
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit_transform(self, X, y=None):
         """Learn a list of feature name -> indices mappings and transform X.
 
@@ -281,6 +310,7 @@ def fit_transform(self, X, y=None):
                Accepts multiple string values for one categorical feature.
 
         y : (ignored)
+            Ignored parameter.
 
         Returns
         -------
@@ -309,11 +339,13 @@ def inverse_transform(self, X, dict_type=dict):
 
         Returns
         -------
-        D : list of dict_type objects of shape (n_samples,)
+        X_original : list of dict_type objects of shape (n_samples,)
             Feature mappings for the samples in X.
         """
+        check_is_fitted(self, "feature_names_")
+
         # COO matrix is not subscriptable
-        X = check_array(X, accept_sparse=['csr', 'csc'])
+        X = check_array(X, accept_sparse=["csr", "csc"])
         n_samples = X.shape[0]
 
         names = self.feature_names_
@@ -347,34 +379,28 @@ def transform(self, X):
         Xa : {array, sparse matrix}
             Feature vectors; always 2-d.
         """
-        if self.sparse:
-            return self._transform(X, fitting=False)
+        check_is_fitted(self, ["feature_names_", "vocabulary_"])
+        return self._transform(X, fitting=False)
 
-        else:
-            dtype = self.dtype
-            vocab = self.vocabulary_
-            X = _tosequence(X)
-            Xa = np.zeros((len(X), len(vocab)), dtype=dtype)
-
-            for i, x in enumerate(X):
-                for f, v in x.items():
-                    if isinstance(v, str):
-                        f = "%s%s%s" % (f, self.separator, v)
-                        v = 1
-                    try:
-                        Xa[i, vocab[f]] = dtype(v)
-                    except KeyError:
-                        pass
-
-            return Xa
-
-    def get_feature_names(self):
-        """Returns a list of feature names, ordered by their indices.
-
-        If one-of-K coding is applied to categorical features, this will
-        include the constructed feature names but not the original ones.
+    def get_feature_names_out(self, input_features=None):
+        """Get output feature names for transformation.
+
+        Parameters
+        ----------
+        input_features : array-like of str or None, default=None
+            Not used, present here for API consistency by convention.
+
+        Returns
+        -------
+        feature_names_out : ndarray of str objects
+            Transformed feature names.
         """
-        return self.feature_names_
+        check_is_fitted(self, "feature_names_")
+        if any(not isinstance(name, str) for name in self.feature_names_):
+            feature_names = [str(name) for name in self.feature_names_]
+        else:
+            feature_names = self.feature_names_
+        return np.asarray(feature_names, dtype=object)
 
     def restrict(self, support, indices=False):
         """Restrict the features to those in support using feature selection.
@@ -391,7 +417,8 @@ def restrict(self, support, indices=False):
 
         Returns
         -------
-        self
+        self : object
+            DictVectorizer class instance.
 
         Examples
         --------
@@ -401,13 +428,15 @@ def restrict(self, support, indices=False):
         >>> D = [{'foo': 1, 'bar': 2}, {'foo': 3, 'baz': 1}]
         >>> X = v.fit_transform(D)
         >>> support = SelectKBest(chi2, k=2).fit(X, [0, 1])
-        >>> v.get_feature_names()
-        ['bar', 'baz', 'foo']
+        >>> v.get_feature_names_out()
+        array(['bar', 'baz', 'foo'], ...)
         >>> v.restrict(support.get_support())
         DictVectorizer()
-        >>> v.get_feature_names()
-        ['bar', 'foo']
+        >>> v.get_feature_names_out()
+        array(['bar', 'foo'], ...)
         """
+        check_is_fitted(self, "feature_names_")
+
         if not indices:
             support = np.where(support)[0]
 
@@ -417,10 +446,14 @@ def restrict(self, support, indices=False):
             new_vocab[names[i]] = len(new_vocab)
 
         self.vocabulary_ = new_vocab
-        self.feature_names_ = [f for f, i in sorted(new_vocab.items(),
-                                                    key=itemgetter(1))]
+        self.feature_names_ = [
+            f for f, i in sorted(new_vocab.items(), key=itemgetter(1))
+        ]
 
         return self
 
-    def _more_tags(self):
-        return {'X_types': ["dict"]}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.dict = True
+        tags.input_tags.two_d_array = False
+        return tags
diff --git a/sklearn/feature_extraction/_hash.py b/sklearn/feature_extraction/_hash.py
index 57f927649bd6f..ac0bed3110c4e 100644
--- a/sklearn/feature_extraction/_hash.py
+++ b/sklearn/feature_extraction/_hash.py
@@ -1,23 +1,17 @@
-# Author: Lars Buitinck
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-import numbers
+from itertools import chain
+from numbers import Integral
 
 import numpy as np
 import scipy.sparse as sp
 
-from ..utils import IS_PYPY
-from ..utils.validation import _deprecate_positional_args
-from ..base import BaseEstimator, TransformerMixin
+from sklearn.utils import metadata_routing
 
-if not IS_PYPY:
-    from ._hashing_fast import transform as _hashing_transform
-else:
-    def _hashing_transform(*args, **kwargs):
-        raise NotImplementedError(
-                'FeatureHasher is not compatible with PyPy (see '
-                'https://github.com/scikit-learn/scikit-learn/issues/11540 '
-                'for the status updates).')
+from ..base import BaseEstimator, TransformerMixin, _fit_context
+from ..utils._param_validation import Interval, StrOptions
+from ._hashing_fast import transform as _hashing_transform
 
 
 def _iteritems(d):
@@ -42,6 +36,9 @@ class FeatureHasher(TransformerMixin, BaseEstimator):
     where memory is tight, e.g. when running prediction code on embedded
     devices.
 
+    For an efficiency comparison of the different feature extractors, see
+    :ref:`sphx_glr_auto_examples_text_plot_hashing_vs_dict_vectorizer.py`.
+
     Read more in the :ref:`User Guide `.
 
     .. versionadded:: 0.13
@@ -52,7 +49,8 @@ class FeatureHasher(TransformerMixin, BaseEstimator):
         The number of features (columns) in the output matrices. Small numbers
         of features are likely to cause hash collisions, but large numbers
         will cause larger coefficient dimensions in linear learners.
-    input_type : {"dict", "pair", "string"}, default="dict"
+    input_type : str, default='dict'
+        Choose a string from {'dict', 'pair', 'string'}.
         Either "dict" (the default) to accept dictionaries over
         (feature_name, value); "pair" to accept pairs of (feature_name, value);
         or "string" to accept single strings.
@@ -74,6 +72,18 @@ class FeatureHasher(TransformerMixin, BaseEstimator):
             ``alternate_sign`` replaces the now deprecated ``non_negative``
             parameter.
 
+    See Also
+    --------
+    DictVectorizer : Vectorizes string-valued features using a hash table.
+    sklearn.preprocessing.OneHotEncoder : Handles nominal/categorical features.
+
+    Notes
+    -----
+    This estimator is :term:`stateless` and does not need to be fitted.
+    However, we recommend to call :meth:`fit_transform` instead of
+    :meth:`transform`, as parameter validation is only performed in
+    :meth:`fit`.
+
     Examples
     --------
     >>> from sklearn.feature_extraction import FeatureHasher
@@ -84,52 +94,61 @@ class FeatureHasher(TransformerMixin, BaseEstimator):
     array([[ 0.,  0., -4., -1.,  0.,  0.,  0.,  0.,  0.,  2.],
            [ 0.,  0.,  0., -2., -5.,  0.,  0.,  0.,  0.,  0.]])
 
-    See Also
-    --------
-    DictVectorizer : Vectorizes string-valued features using a hash table.
-    sklearn.preprocessing.OneHotEncoder : Handles nominal/categorical features.
+    With `input_type="string"`, the input must be an iterable over iterables of
+    strings:
+
+    >>> h = FeatureHasher(n_features=8, input_type="string")
+    >>> raw_X = [["dog", "cat", "snake"], ["snake", "dog"], ["cat", "bird"]]
+    >>> f = h.transform(raw_X)
+    >>> f.toarray()
+    array([[ 0.,  0.,  0., -1.,  0., -1.,  0.,  1.],
+           [ 0.,  0.,  0., -1.,  0., -1.,  0.,  0.],
+           [ 0., -1.,  0.,  0.,  0.,  0.,  0.,  1.]])
     """
-    @_deprecate_positional_args
-    def __init__(self, n_features=(2 ** 20), *, input_type="dict",
-                 dtype=np.float64, alternate_sign=True):
-        self._validate_params(n_features, input_type)
 
+    # raw_X should have been called X
+    __metadata_request__transform = {"raw_X": metadata_routing.UNUSED}
+
+    _parameter_constraints: dict = {
+        "n_features": [Interval(Integral, 1, np.iinfo(np.int32).max, closed="both")],
+        "input_type": [StrOptions({"dict", "pair", "string"})],
+        "dtype": "no_validation",  # delegate to numpy
+        "alternate_sign": ["boolean"],
+    }
+
+    def __init__(
+        self,
+        n_features=(2**20),
+        *,
+        input_type="dict",
+        dtype=np.float64,
+        alternate_sign=True,
+    ):
         self.dtype = dtype
         self.input_type = input_type
         self.n_features = n_features
         self.alternate_sign = alternate_sign
 
-    @staticmethod
-    def _validate_params(n_features, input_type):
-        # strangely, np.int16 instances are not instances of Integral,
-        # while np.int64 instances are...
-        if not isinstance(n_features, numbers.Integral):
-            raise TypeError("n_features must be integral, got %r (%s)."
-                            % (n_features, type(n_features)))
-        elif n_features < 1 or n_features >= np.iinfo(np.int32).max + 1:
-            raise ValueError("Invalid number of features (%d)." % n_features)
-
-        if input_type not in ("dict", "pair", "string"):
-            raise ValueError("input_type must be 'dict', 'pair' or 'string',"
-                             " got %r." % input_type)
-
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X=None, y=None):
-        """No-op.
+        """Only validates estimator's parameters.
 
-        This method doesn't do anything. It exists purely for compatibility
-        with the scikit-learn transformer API.
+        This method allows to: (i) validate the estimator's parameters and
+        (ii) be consistent with the scikit-learn transformer API.
 
         Parameters
         ----------
-        X : ndarray
+        X : Ignored
+            Not used, present here for API consistency by convention.
+
+        y : Ignored
+            Not used, present here for API consistency by convention.
 
         Returns
         -------
-        self : FeatureHasher
-
+        self : object
+            FeatureHasher class instance.
         """
-        # repeat input validation for grid search (which calls set_params)
-        self._validate_params(self.n_features, self.input_type)
         return self
 
     def transform(self, raw_X):
@@ -148,26 +167,42 @@ def transform(self, raw_X):
         -------
         X : sparse matrix of shape (n_samples, n_features)
             Feature matrix, for use with estimators or further transformers.
-
         """
         raw_X = iter(raw_X)
         if self.input_type == "dict":
             raw_X = (_iteritems(d) for d in raw_X)
         elif self.input_type == "string":
-            raw_X = (((f, 1) for f in x) for x in raw_X)
-        indices, indptr, values = \
-            _hashing_transform(raw_X, self.n_features, self.dtype,
-                               self.alternate_sign, seed=0)
+            first_raw_X = next(raw_X)
+            if isinstance(first_raw_X, str):
+                raise ValueError(
+                    "Samples can not be a single string. The input must be an iterable"
+                    " over iterables of strings."
+                )
+            raw_X_ = chain([first_raw_X], raw_X)
+            raw_X = (((f, 1) for f in x) for x in raw_X_)
+
+        indices, indptr, values = _hashing_transform(
+            raw_X, self.n_features, self.dtype, self.alternate_sign, seed=0
+        )
         n_samples = indptr.shape[0] - 1
 
         if n_samples == 0:
             raise ValueError("Cannot vectorize empty sequence.")
 
-        X = sp.csr_matrix((values, indices, indptr), dtype=self.dtype,
-                          shape=(n_samples, self.n_features))
+        X = sp.csr_matrix(
+            (values, indices, indptr),
+            dtype=self.dtype,
+            shape=(n_samples, self.n_features),
+        )
         X.sum_duplicates()  # also sorts the indices
 
         return X
 
-    def _more_tags(self):
-        return {'X_types': [self.input_type]}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.two_d_array = False
+        if self.input_type == "string":
+            tags.input_tags.string = True
+        elif self.input_type == "dict":
+            tags.input_tags.dict = True
+        return tags
diff --git a/sklearn/feature_extraction/_hashing_fast.pyx b/sklearn/feature_extraction/_hashing_fast.pyx
index 3a3102444af98..5069d555d60ea 100644
--- a/sklearn/feature_extraction/_hashing_fast.pyx
+++ b/sklearn/feature_extraction/_hashing_fast.pyx
@@ -1,19 +1,16 @@
-# Author: Lars Buitinck
-# License: BSD 3 clause
-#
-# cython: boundscheck=False, cdivision=True
-
-import sys
-import array
-from cpython cimport array
-cimport cython
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
 from libc.stdlib cimport abs
-cimport numpy as np
-import numpy as np
+from libcpp.vector cimport vector
 
+cimport numpy as cnp
+import numpy as np
+from ..utils._typedefs cimport int32_t, int64_t
 from ..utils.murmurhash cimport murmurhash3_bytes_s32
+from ..utils._vector_sentinel cimport vector_to_nd_array
 
-np.import_array()
+cnp.import_array()
 
 
 def transform(raw_X, Py_ssize_t n_features, dtype,
@@ -27,25 +24,18 @@ def transform(raw_X, Py_ssize_t n_features, dtype,
         For constructing a scipy.sparse.csr_matrix.
 
     """
-    assert n_features > 0
-
-    cdef np.int32_t h
+    cdef int32_t h
     cdef double value
 
-    cdef array.array indices
-    cdef array.array indptr
-    indices = array.array("i")
-    indices_array_dtype = "q"
-    indices_np_dtype = np.longlong
-
-
-    indptr = array.array(indices_array_dtype, [0])
+    cdef vector[int32_t] indices
+    cdef vector[int64_t] indptr
+    indptr.push_back(0)
 
     # Since Python array does not understand Numpy dtypes, we grow the indices
     # and values arrays ourselves. Use a Py_ssize_t capacity for safety.
     cdef Py_ssize_t capacity = 8192     # arbitrary
-    cdef np.int64_t size = 0
-    cdef np.ndarray values = np.empty(capacity, dtype=dtype)
+    cdef int64_t size = 0
+    cdef cnp.ndarray values = np.empty(capacity, dtype=dtype)
 
     for x in raw_X:
         for f, v in x:
@@ -67,13 +57,12 @@ def transform(raw_X, Py_ssize_t n_features, dtype,
 
             h = murmurhash3_bytes_s32(f, seed)
 
-            array.resize_smart(indices, len(indices) + 1)
             if h == - 2147483648:
                 # abs(-2**31) is undefined behavior because h is a `np.int32`
                 # The following is defined such that it is equal to: abs(-2**31) % n_features
-                indices[len(indices) - 1] = (2147483647 - (n_features - 1)) % n_features
+                indices.push_back((2147483647 - (n_features - 1)) % n_features)
             else:
-                indices[len(indices) - 1] = abs(h) % n_features
+                indices.push_back(abs(h) % n_features)
             # improve inner product preservation in the hashed space
             if alternate_sign:
                 value *= (h >= 0) * 2 - 1
@@ -86,16 +75,15 @@ def transform(raw_X, Py_ssize_t n_features, dtype,
                 # references to the arrays due to Cython's error checking
                 values = np.resize(values, capacity)
 
-        array.resize_smart(indptr, len(indptr) + 1)
-        indptr[len(indptr) - 1] = size
+        indptr.push_back(size)
 
-    indices_a = np.frombuffer(indices, dtype=np.int32)
-    indptr_a = np.frombuffer(indptr, dtype=indices_np_dtype)
+    indices_array = vector_to_nd_array(&indices)
+    indptr_array = vector_to_nd_array(&indptr)
 
-    if indptr[-1] > np.iinfo(np.int32).max:  # = 2**31 - 1
+    if indptr_array[indptr_array.shape[0]-1] > np.iinfo(np.int32).max:  # = 2**31 - 1
         # both indices and indptr have the same dtype in CSR arrays
-        indices_a = indices_a.astype(np.int64, copy=False)
+        indices_array = indices_array.astype(np.int64, copy=False)
     else:
-        indptr_a = indptr_a.astype(np.int32, copy=False)
+        indptr_array = indptr_array.astype(np.int32, copy=False)
 
-    return (indices_a, indptr_a, values[:size])
+    return (indices_array, indptr_array, values[:size])
diff --git a/sklearn/feature_extraction/_stop_words.py b/sklearn/feature_extraction/_stop_words.py
index 880f144c4e467..6bc8e6d2f37dc 100644
--- a/sklearn/feature_extraction/_stop_words.py
+++ b/sklearn/feature_extraction/_stop_words.py
@@ -1,45 +1,328 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
 # This list of English stop words is taken from the "Glasgow Information
 # Retrieval Group". The original list can be found at
 # http://ir.dcs.gla.ac.uk/resources/linguistic_utils/stop_words
-ENGLISH_STOP_WORDS = frozenset([
-    "a", "about", "above", "across", "after", "afterwards", "again", "against",
-    "all", "almost", "alone", "along", "already", "also", "although", "always",
-    "am", "among", "amongst", "amoungst", "amount", "an", "and", "another",
-    "any", "anyhow", "anyone", "anything", "anyway", "anywhere", "are",
-    "around", "as", "at", "back", "be", "became", "because", "become",
-    "becomes", "becoming", "been", "before", "beforehand", "behind", "being",
-    "below", "beside", "besides", "between", "beyond", "bill", "both",
-    "bottom", "but", "by", "call", "can", "cannot", "cant", "co", "con",
-    "could", "couldnt", "cry", "de", "describe", "detail", "do", "done",
-    "down", "due", "during", "each", "eg", "eight", "either", "eleven", "else",
-    "elsewhere", "empty", "enough", "etc", "even", "ever", "every", "everyone",
-    "everything", "everywhere", "except", "few", "fifteen", "fifty", "fill",
-    "find", "fire", "first", "five", "for", "former", "formerly", "forty",
-    "found", "four", "from", "front", "full", "further", "get", "give", "go",
-    "had", "has", "hasnt", "have", "he", "hence", "her", "here", "hereafter",
-    "hereby", "herein", "hereupon", "hers", "herself", "him", "himself", "his",
-    "how", "however", "hundred", "i", "ie", "if", "in", "inc", "indeed",
-    "interest", "into", "is", "it", "its", "itself", "keep", "last", "latter",
-    "latterly", "least", "less", "ltd", "made", "many", "may", "me",
-    "meanwhile", "might", "mill", "mine", "more", "moreover", "most", "mostly",
-    "move", "much", "must", "my", "myself", "name", "namely", "neither",
-    "never", "nevertheless", "next", "nine", "no", "nobody", "none", "noone",
-    "nor", "not", "nothing", "now", "nowhere", "of", "off", "often", "on",
-    "once", "one", "only", "onto", "or", "other", "others", "otherwise", "our",
-    "ours", "ourselves", "out", "over", "own", "part", "per", "perhaps",
-    "please", "put", "rather", "re", "same", "see", "seem", "seemed",
-    "seeming", "seems", "serious", "several", "she", "should", "show", "side",
-    "since", "sincere", "six", "sixty", "so", "some", "somehow", "someone",
-    "something", "sometime", "sometimes", "somewhere", "still", "such",
-    "system", "take", "ten", "than", "that", "the", "their", "them",
-    "themselves", "then", "thence", "there", "thereafter", "thereby",
-    "therefore", "therein", "thereupon", "these", "they", "thick", "thin",
-    "third", "this", "those", "though", "three", "through", "throughout",
-    "thru", "thus", "to", "together", "too", "top", "toward", "towards",
-    "twelve", "twenty", "two", "un", "under", "until", "up", "upon", "us",
-    "very", "via", "was", "we", "well", "were", "what", "whatever", "when",
-    "whence", "whenever", "where", "whereafter", "whereas", "whereby",
-    "wherein", "whereupon", "wherever", "whether", "which", "while", "whither",
-    "who", "whoever", "whole", "whom", "whose", "why", "will", "with",
-    "within", "without", "would", "yet", "you", "your", "yours", "yourself",
-    "yourselves"])
+ENGLISH_STOP_WORDS = frozenset(
+    [
+        "a",
+        "about",
+        "above",
+        "across",
+        "after",
+        "afterwards",
+        "again",
+        "against",
+        "all",
+        "almost",
+        "alone",
+        "along",
+        "already",
+        "also",
+        "although",
+        "always",
+        "am",
+        "among",
+        "amongst",
+        "amoungst",
+        "amount",
+        "an",
+        "and",
+        "another",
+        "any",
+        "anyhow",
+        "anyone",
+        "anything",
+        "anyway",
+        "anywhere",
+        "are",
+        "around",
+        "as",
+        "at",
+        "back",
+        "be",
+        "became",
+        "because",
+        "become",
+        "becomes",
+        "becoming",
+        "been",
+        "before",
+        "beforehand",
+        "behind",
+        "being",
+        "below",
+        "beside",
+        "besides",
+        "between",
+        "beyond",
+        "bill",
+        "both",
+        "bottom",
+        "but",
+        "by",
+        "call",
+        "can",
+        "cannot",
+        "cant",
+        "co",
+        "con",
+        "could",
+        "couldnt",
+        "cry",
+        "de",
+        "describe",
+        "detail",
+        "do",
+        "done",
+        "down",
+        "due",
+        "during",
+        "each",
+        "eg",
+        "eight",
+        "either",
+        "eleven",
+        "else",
+        "elsewhere",
+        "empty",
+        "enough",
+        "etc",
+        "even",
+        "ever",
+        "every",
+        "everyone",
+        "everything",
+        "everywhere",
+        "except",
+        "few",
+        "fifteen",
+        "fifty",
+        "fill",
+        "find",
+        "fire",
+        "first",
+        "five",
+        "for",
+        "former",
+        "formerly",
+        "forty",
+        "found",
+        "four",
+        "from",
+        "front",
+        "full",
+        "further",
+        "get",
+        "give",
+        "go",
+        "had",
+        "has",
+        "hasnt",
+        "have",
+        "he",
+        "hence",
+        "her",
+        "here",
+        "hereafter",
+        "hereby",
+        "herein",
+        "hereupon",
+        "hers",
+        "herself",
+        "him",
+        "himself",
+        "his",
+        "how",
+        "however",
+        "hundred",
+        "i",
+        "ie",
+        "if",
+        "in",
+        "inc",
+        "indeed",
+        "interest",
+        "into",
+        "is",
+        "it",
+        "its",
+        "itself",
+        "keep",
+        "last",
+        "latter",
+        "latterly",
+        "least",
+        "less",
+        "ltd",
+        "made",
+        "many",
+        "may",
+        "me",
+        "meanwhile",
+        "might",
+        "mill",
+        "mine",
+        "more",
+        "moreover",
+        "most",
+        "mostly",
+        "move",
+        "much",
+        "must",
+        "my",
+        "myself",
+        "name",
+        "namely",
+        "neither",
+        "never",
+        "nevertheless",
+        "next",
+        "nine",
+        "no",
+        "nobody",
+        "none",
+        "noone",
+        "nor",
+        "not",
+        "nothing",
+        "now",
+        "nowhere",
+        "of",
+        "off",
+        "often",
+        "on",
+        "once",
+        "one",
+        "only",
+        "onto",
+        "or",
+        "other",
+        "others",
+        "otherwise",
+        "our",
+        "ours",
+        "ourselves",
+        "out",
+        "over",
+        "own",
+        "part",
+        "per",
+        "perhaps",
+        "please",
+        "put",
+        "rather",
+        "re",
+        "same",
+        "see",
+        "seem",
+        "seemed",
+        "seeming",
+        "seems",
+        "serious",
+        "several",
+        "she",
+        "should",
+        "show",
+        "side",
+        "since",
+        "sincere",
+        "six",
+        "sixty",
+        "so",
+        "some",
+        "somehow",
+        "someone",
+        "something",
+        "sometime",
+        "sometimes",
+        "somewhere",
+        "still",
+        "such",
+        "system",
+        "take",
+        "ten",
+        "than",
+        "that",
+        "the",
+        "their",
+        "them",
+        "themselves",
+        "then",
+        "thence",
+        "there",
+        "thereafter",
+        "thereby",
+        "therefore",
+        "therein",
+        "thereupon",
+        "these",
+        "they",
+        "thick",
+        "thin",
+        "third",
+        "this",
+        "those",
+        "though",
+        "three",
+        "through",
+        "throughout",
+        "thru",
+        "thus",
+        "to",
+        "together",
+        "too",
+        "top",
+        "toward",
+        "towards",
+        "twelve",
+        "twenty",
+        "two",
+        "un",
+        "under",
+        "until",
+        "up",
+        "upon",
+        "us",
+        "very",
+        "via",
+        "was",
+        "we",
+        "well",
+        "were",
+        "what",
+        "whatever",
+        "when",
+        "whence",
+        "whenever",
+        "where",
+        "whereafter",
+        "whereas",
+        "whereby",
+        "wherein",
+        "whereupon",
+        "wherever",
+        "whether",
+        "which",
+        "while",
+        "whither",
+        "who",
+        "whoever",
+        "whole",
+        "whom",
+        "whose",
+        "why",
+        "will",
+        "with",
+        "within",
+        "without",
+        "would",
+        "yet",
+        "you",
+        "your",
+        "yours",
+        "yourself",
+        "yourselves",
+    ]
+)
diff --git a/sklearn/feature_extraction/image.py b/sklearn/feature_extraction/image.py
index 5cd692fd6aa4f..b571215de47be 100644
--- a/sklearn/feature_extraction/image.py
+++ b/sklearn/feature_extraction/image.py
@@ -1,29 +1,28 @@
-"""
-The :mod:`sklearn.feature_extraction.image` submodule gathers utilities to
-extract features from images.
-"""
+"""Utilities to extract features from images."""
 
-# Authors: Emmanuelle Gouillart 
-#          Gael Varoquaux 
-#          Olivier Grisel
-#          Vlad Niculae
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 from itertools import product
-import numbers
+from numbers import Integral, Number, Real
+
 import numpy as np
-from scipy import sparse
 from numpy.lib.stride_tricks import as_strided
+from scipy import sparse
 
+from ..base import BaseEstimator, TransformerMixin, _fit_context
 from ..utils import check_array, check_random_state
-from ..utils.validation import _deprecate_positional_args
-from ..base import BaseEstimator
+from ..utils._param_validation import Hidden, Interval, RealNotInt, validate_params
+
+__all__ = [
+    "PatchExtractor",
+    "extract_patches_2d",
+    "grid_to_graph",
+    "img_to_graph",
+    "reconstruct_from_patches_2d",
+]
 
-__all__ = ['PatchExtractor',
-           'extract_patches_2d',
-           'grid_to_graph',
-           'img_to_graph',
-           'reconstruct_from_patches_2d']
+from ..utils.validation import validate_data
 
 ###############################################################################
 # From an image to a graph
@@ -42,10 +41,8 @@ def _make_edges_3d(n_x, n_y, n_z=1):
         The size of the grid in the z direction, defaults to 1
     """
     vertices = np.arange(n_x * n_y * n_z).reshape((n_x, n_y, n_z))
-    edges_deep = np.vstack((vertices[:, :, :-1].ravel(),
-                            vertices[:, :, 1:].ravel()))
-    edges_right = np.vstack((vertices[:, :-1].ravel(),
-                             vertices[:, 1:].ravel()))
+    edges_deep = np.vstack((vertices[:, :, :-1].ravel(), vertices[:, :, 1:].ravel()))
+    edges_right = np.vstack((vertices[:, :-1].ravel(), vertices[:, 1:].ravel()))
     edges_down = np.vstack((vertices[:-1].ravel(), vertices[1:].ravel()))
     edges = np.hstack((edges_deep, edges_right, edges_down))
     return edges
@@ -53,23 +50,29 @@ def _make_edges_3d(n_x, n_y, n_z=1):
 
 def _compute_gradient_3d(edges, img):
     _, n_y, n_z = img.shape
-    gradient = np.abs(img[edges[0] // (n_y * n_z),
-                      (edges[0] % (n_y * n_z)) // n_z,
-                      (edges[0] % (n_y * n_z)) % n_z] -
-                      img[edges[1] // (n_y * n_z),
-                      (edges[1] % (n_y * n_z)) // n_z,
-                      (edges[1] % (n_y * n_z)) % n_z])
+    gradient = np.abs(
+        img[
+            edges[0] // (n_y * n_z),
+            (edges[0] % (n_y * n_z)) // n_z,
+            (edges[0] % (n_y * n_z)) % n_z,
+        ]
+        - img[
+            edges[1] // (n_y * n_z),
+            (edges[1] % (n_y * n_z)) // n_z,
+            (edges[1] % (n_y * n_z)) % n_z,
+        ]
+    )
     return gradient
 
 
 # XXX: Why mask the image after computing the weights?
 
+
 def _mask_edges_weights(mask, edges, weights=None):
     """Apply a mask to edges (weighted or not)"""
     inds = np.arange(mask.size)
     inds = inds[mask.ravel()]
-    ind_mask = np.logical_and(np.in1d(edges[0], inds),
-                              np.in1d(edges[1], inds))
+    ind_mask = np.logical_and(np.isin(edges[0], inds), np.isin(edges[1], inds))
     edges = edges[:, ind_mask]
     if weights is not None:
         weights = weights[ind_mask]
@@ -77,7 +80,7 @@ def _mask_edges_weights(mask, edges, weights=None):
         maxval = edges.max()
     else:
         maxval = 0
-    order = np.searchsorted(np.unique(edges.ravel()), np.arange(maxval + 1))
+    order = np.searchsorted(np.flatnonzero(mask), np.arange(maxval + 1))
     edges = order[edges]
     if weights is None:
         return edges
@@ -85,13 +88,13 @@ def _mask_edges_weights(mask, edges, weights=None):
         return edges, weights
 
 
-def _to_graph(n_x, n_y, n_z, mask=None, img=None,
-              return_as=sparse.coo_matrix, dtype=None):
-    """Auxiliary function for img_to_graph and grid_to_graph
-    """
+def _to_graph(
+    n_x, n_y, n_z, mask=None, img=None, return_as=sparse.coo_matrix, dtype=None
+):
+    """Auxiliary function for img_to_graph and grid_to_graph"""
     edges = _make_edges_3d(n_x, n_y, n_z)
 
-    if dtype is None:
+    if dtype is None:  # To not overwrite input dtype
         if img is None:
             dtype = int
         else:
@@ -109,7 +112,6 @@ def _to_graph(n_x, n_y, n_z, mask=None, img=None,
     else:
         if mask is not None:
             mask = mask.astype(dtype=bool, copy=False)
-            mask = np.asarray(mask, dtype=bool)
             edges = _mask_edges_weights(mask, edges)
             n_voxels = np.sum(mask)
         else:
@@ -120,19 +122,30 @@ def _to_graph(n_x, n_y, n_z, mask=None, img=None,
     diag_idx = np.arange(n_voxels)
     i_idx = np.hstack((edges[0], edges[1]))
     j_idx = np.hstack((edges[1], edges[0]))
-    graph = sparse.coo_matrix((np.hstack((weights, weights, diag)),
-                              (np.hstack((i_idx, diag_idx)),
-                               np.hstack((j_idx, diag_idx)))),
-                              (n_voxels, n_voxels),
-                              dtype=dtype)
+    graph = sparse.coo_matrix(
+        (
+            np.hstack((weights, weights, diag)),
+            (np.hstack((i_idx, diag_idx)), np.hstack((j_idx, diag_idx))),
+        ),
+        (n_voxels, n_voxels),
+        dtype=dtype,
+    )
     if return_as is np.ndarray:
         return graph.toarray()
     return return_as(graph)
 
 
-@_deprecate_positional_args
+@validate_params(
+    {
+        "img": ["array-like"],
+        "mask": [None, np.ndarray],
+        "return_as": [type],
+        "dtype": "no_validation",  # validation delegated to numpy
+    },
+    prefer_skip_nested_validation=True,
+)
 def img_to_graph(img, *, mask=None, return_as=sparse.coo_matrix, dtype=None):
-    """Graph of the pixel-to-pixel gradient connections
+    """Graph of the pixel-to-pixel gradient connections.
 
     Edges are weighted with the gradient values.
 
@@ -140,7 +153,7 @@ def img_to_graph(img, *, mask=None, return_as=sparse.coo_matrix, dtype=None):
 
     Parameters
     ----------
-    img : ndarray of shape (height, width) or (height, width, channel)
+    img : array-like of shape (height, width) or (height, width, channel)
         2D or 3D image.
     mask : ndarray of shape (height, width) or \
             (height, width, channel), dtype=bool, default=None
@@ -151,37 +164,57 @@ def img_to_graph(img, *, mask=None, return_as=sparse.coo_matrix, dtype=None):
         The class to use to build the returned adjacency matrix.
     dtype : dtype, default=None
         The data of the returned sparse matrix. By default it is the
-        dtype of img
+        dtype of img.
 
-    Notes
-    -----
-    For scikit-learn versions 0.14.1 and prior, return_as=np.ndarray was
-    handled by returning a dense np.matrix instance.  Going forward, np.ndarray
-    returns an np.ndarray, as expected.
+    Returns
+    -------
+    graph : ndarray or a sparse matrix class
+        The computed adjacency matrix.
 
-    For compatibility, user code relying on this method should wrap its
-    calls in ``np.asarray`` to avoid type issues.
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from sklearn.feature_extraction.image import img_to_graph
+    >>> img = np.array([[0, 0], [0, 1]])
+    >>> img_to_graph(img, return_as=np.ndarray)
+    array([[0, 0, 0, 0],
+           [0, 0, 0, 1],
+           [0, 0, 0, 1],
+           [0, 1, 1, 1]])
     """
     img = np.atleast_3d(img)
     n_x, n_y, n_z = img.shape
     return _to_graph(n_x, n_y, n_z, mask, img, return_as, dtype)
 
 
-@_deprecate_positional_args
-def grid_to_graph(n_x, n_y, n_z=1, *, mask=None, return_as=sparse.coo_matrix,
-                  dtype=int):
-    """Graph of the pixel-to-pixel connections
+@validate_params(
+    {
+        "n_x": [Interval(Integral, left=1, right=None, closed="left")],
+        "n_y": [Interval(Integral, left=1, right=None, closed="left")],
+        "n_z": [Interval(Integral, left=1, right=None, closed="left")],
+        "mask": [None, np.ndarray],
+        "return_as": [type],
+        "dtype": "no_validation",  # validation delegated to numpy
+    },
+    prefer_skip_nested_validation=True,
+)
+def grid_to_graph(
+    n_x, n_y, n_z=1, *, mask=None, return_as=sparse.coo_matrix, dtype=int
+):
+    """Graph of the pixel-to-pixel connections.
 
     Edges exist if 2 voxels are connected.
 
+    Read more in the :ref:`User Guide `.
+
     Parameters
     ----------
     n_x : int
-        Dimension in x axis
+        Dimension in x axis.
     n_y : int
-        Dimension in y axis
+        Dimension in y axis.
     n_z : int, default=1
-        Dimension in z axis
+        Dimension in z axis.
     mask : ndarray of shape (n_x, n_y, n_z), dtype=bool, default=None
         An optional mask of the image, to consider only part of the
         pixels.
@@ -189,24 +222,35 @@ def grid_to_graph(n_x, n_y, n_z=1, *, mask=None, return_as=sparse.coo_matrix,
             default=sparse.coo_matrix
         The class to use to build the returned adjacency matrix.
     dtype : dtype, default=int
-        The data of the returned sparse matrix. By default it is int
+        The data of the returned sparse matrix. By default it is int.
 
-    Notes
-    -----
-    For scikit-learn versions 0.14.1 and prior, return_as=np.ndarray was
-    handled by returning a dense np.matrix instance.  Going forward, np.ndarray
-    returns an np.ndarray, as expected.
+    Returns
+    -------
+    graph : np.ndarray or a sparse matrix class
+        The computed adjacency matrix.
 
-    For compatibility, user code relying on this method should wrap its
-    calls in ``np.asarray`` to avoid type issues.
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from sklearn.feature_extraction.image import grid_to_graph
+    >>> shape_img = (4, 4, 1)
+    >>> mask = np.zeros(shape=shape_img, dtype=bool)
+    >>> mask[[1, 2], [1, 2], :] = True
+    >>> graph = grid_to_graph(*shape_img, mask=mask)
+    >>> print(graph)
+    
+      Coords	Values
+      (0, 0)    1
+      (1, 1)    1
     """
-    return _to_graph(n_x, n_y, n_z, mask=mask, return_as=return_as,
-                     dtype=dtype)
+    return _to_graph(n_x, n_y, n_z, mask=mask, return_as=return_as, dtype=dtype)
 
 
 ###############################################################################
 # From an image to a set of small image patches
 
+
 def _compute_n_patches(i_h, i_w, p_h, p_w, max_patches=None):
     """Compute the number of patches that will be extracted in an image.
 
@@ -223,23 +267,20 @@ def _compute_n_patches(i_h, i_w, p_h, p_w, max_patches=None):
     p_w : int
         The width of a patch
     max_patches : int or float, default=None
-        The maximum number of patches to extract. If max_patches is a float
+        The maximum number of patches to extract. If `max_patches` is a float
         between 0 and 1, it is taken to be a proportion of the total number
-        of patches.
+        of patches. If `max_patches` is None, all possible patches are extracted.
     """
     n_h = i_h - p_h + 1
     n_w = i_w - p_w + 1
     all_patches = n_h * n_w
 
     if max_patches:
-        if (isinstance(max_patches, (numbers.Integral))
-                and max_patches < all_patches):
+        if isinstance(max_patches, (Integral)) and max_patches < all_patches:
             return max_patches
-        elif (isinstance(max_patches, (numbers.Integral))
-              and max_patches >= all_patches):
+        elif isinstance(max_patches, (Integral)) and max_patches >= all_patches:
             return all_patches
-        elif (isinstance(max_patches, (numbers.Real))
-                and 0 < max_patches < 1):
+        elif isinstance(max_patches, (Real)) and 0 < max_patches < 1:
             return int(max_patches * all_patches)
         else:
             raise ValueError("Invalid value for max_patches: %r" % max_patches)
@@ -285,9 +326,9 @@ def _extract_patches(arr, patch_shape=8, extraction_step=1):
 
     arr_ndim = arr.ndim
 
-    if isinstance(patch_shape, numbers.Number):
+    if isinstance(patch_shape, Number):
         patch_shape = tuple([patch_shape] * arr_ndim)
-    if isinstance(extraction_step, numbers.Number):
+    if isinstance(extraction_step, Number):
         extraction_step = tuple([extraction_step] * arr_ndim)
 
     patch_strides = arr.strides
@@ -295,8 +336,9 @@ def _extract_patches(arr, patch_shape=8, extraction_step=1):
     slices = tuple(slice(None, None, st) for st in extraction_step)
     indexing_strides = arr[slices].strides
 
-    patch_indices_shape = ((np.array(arr.shape) - np.array(patch_shape)) //
-                           np.array(extraction_step)) + 1
+    patch_indices_shape = (
+        (np.array(arr.shape) - np.array(patch_shape)) // np.array(extraction_step)
+    ) + 1
 
     shape = tuple(list(patch_indices_shape) + list(patch_shape))
     strides = tuple(list(indexing_strides) + list(patch_strides))
@@ -305,10 +347,21 @@ def _extract_patches(arr, patch_shape=8, extraction_step=1):
     return patches
 
 
-@_deprecate_positional_args
-def extract_patches_2d(image, patch_size, *, max_patches=None,
-                       random_state=None):
-    """Reshape a 2D image into a collection of patches
+@validate_params(
+    {
+        "image": [np.ndarray],
+        "patch_size": [tuple, list],
+        "max_patches": [
+            Interval(RealNotInt, 0, 1, closed="neither"),
+            Interval(Integral, 1, None, closed="left"),
+            None,
+        ],
+        "random_state": ["random_state"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def extract_patches_2d(image, patch_size, *, max_patches=None, random_state=None):
+    """Reshape a 2D image into a collection of patches.
 
     The resulting patches are allocated in a dedicated array.
 
@@ -327,7 +380,8 @@ def extract_patches_2d(image, patch_size, *, max_patches=None,
     max_patches : int or float, default=None
         The maximum number of patches to extract. If `max_patches` is a float
         between 0 and 1, it is taken to be a proportion of the total number
-        of patches.
+        of patches. If `max_patches` is None it corresponds to the total number
+        of patches that can be extracted.
 
     random_state : int, RandomState instance, default=None
         Determines the random number generator used for random sampling when
@@ -370,20 +424,22 @@ def extract_patches_2d(image, patch_size, *, max_patches=None,
     p_h, p_w = patch_size
 
     if p_h > i_h:
-        raise ValueError("Height of the patch should be less than the height"
-                         " of the image.")
+        raise ValueError(
+            "Height of the patch should be less than the height of the image."
+        )
 
     if p_w > i_w:
-        raise ValueError("Width of the patch should be less than the width"
-                         " of the image.")
+        raise ValueError(
+            "Width of the patch should be less than the width of the image."
+        )
 
     image = check_array(image, allow_nd=True)
     image = image.reshape((i_h, i_w, -1))
     n_colors = image.shape[-1]
 
-    extracted_patches = _extract_patches(image,
-                                         patch_shape=(p_h, p_w, n_colors),
-                                         extraction_step=1)
+    extracted_patches = _extract_patches(
+        image, patch_shape=(p_h, p_w, n_colors), extraction_step=1
+    )
 
     n_patches = _compute_n_patches(i_h, i_w, p_h, p_w, max_patches)
     if max_patches:
@@ -402,6 +458,10 @@ def extract_patches_2d(image, patch_size, *, max_patches=None,
         return patches
 
 
+@validate_params(
+    {"patches": [np.ndarray], "image_size": [tuple, Hidden(list)]},
+    prefer_skip_nested_validation=True,
+)
 def reconstruct_from_patches_2d(patches, image_size):
     """Reconstruct the image from all of its patches.
 
@@ -427,6 +487,23 @@ def reconstruct_from_patches_2d(patches, image_size):
     -------
     image : ndarray of shape image_size
         The reconstructed image.
+
+    Examples
+    --------
+    >>> from sklearn.datasets import load_sample_image
+    >>> from sklearn.feature_extraction import image
+    >>> one_image = load_sample_image("china.jpg")
+    >>> print('Image shape: {}'.format(one_image.shape))
+    Image shape: (427, 640, 3)
+    >>> image_patches = image.extract_patches_2d(image=one_image, patch_size=(10, 10))
+    >>> print('Patches shape: {}'.format(image_patches.shape))
+    Patches shape: (263758, 10, 10, 3)
+    >>> image_reconstructed = image.reconstruct_from_patches_2d(
+    ...     patches=image_patches,
+    ...     image_size=one_image.shape
+    ... )
+    >>> print(f"Reconstructed shape: {image_reconstructed.shape}")
+    Reconstructed shape: (427, 640, 3)
     """
     i_h, i_w = image_size[:2]
     p_h, p_w = patches.shape[1:3]
@@ -435,19 +512,18 @@ def reconstruct_from_patches_2d(patches, image_size):
     n_h = i_h - p_h + 1
     n_w = i_w - p_w + 1
     for p, (i, j) in zip(patches, product(range(n_h), range(n_w))):
-        img[i:i + p_h, j:j + p_w] += p
+        img[i : i + p_h, j : j + p_w] += p
 
     for i in range(i_h):
         for j in range(i_w):
             # divide by the amount of overlap
             # XXX: is this the most efficient way? memory-wise yes, cpu wise?
-            img[i, j] /= float(min(i + 1, p_h, i_h - i) *
-                               min(j + 1, p_w, i_w - j))
+            img[i, j] /= float(min(i + 1, p_h, i_h - i) * min(j + 1, p_w, i_w - j))
     return img
 
 
-class PatchExtractor(BaseEstimator):
-    """Extracts patches from a collection of images
+class PatchExtractor(TransformerMixin, BaseEstimator):
+    """Extracts patches from a collection of images.
 
     Read more in the :ref:`User Guide `.
 
@@ -456,60 +532,96 @@ class PatchExtractor(BaseEstimator):
     Parameters
     ----------
     patch_size : tuple of int (patch_height, patch_width), default=None
-        The dimensions of one patch.
+        The dimensions of one patch. If set to None, the patch size will be
+        automatically set to `(img_height // 10, img_width // 10)`, where
+        `img_height` and `img_width` are the dimensions of the input images.
 
     max_patches : int or float, default=None
-        The maximum number of patches per image to extract. If max_patches is a
-        float in (0, 1), it is taken to mean a proportion of the total number
-        of patches.
+        The maximum number of patches per image to extract. If `max_patches` is
+        a float in (0, 1), it is taken to mean a proportion of the total number
+        of patches. If set to None, extract all possible patches.
 
     random_state : int, RandomState instance, default=None
         Determines the random number generator used for random sampling when
-        `max_patches` is not None. Use an int to make the randomness
+        `max_patches is not None`. Use an int to make the randomness
         deterministic.
         See :term:`Glossary `.
 
+    See Also
+    --------
+    reconstruct_from_patches_2d : Reconstruct image from all of its patches.
+
+    Notes
+    -----
+    This estimator is stateless and does not need to be fitted. However, we
+    recommend to call :meth:`fit_transform` instead of :meth:`transform`, as
+    parameter validation is only performed in :meth:`fit`.
+
     Examples
     --------
     >>> from sklearn.datasets import load_sample_images
     >>> from sklearn.feature_extraction import image
     >>> # Use the array data from the second image in this dataset:
     >>> X = load_sample_images().images[1]
-    >>> print('Image shape: {}'.format(X.shape))
-    Image shape: (427, 640, 3)
-    >>> pe = image.PatchExtractor(patch_size=(2, 2))
-    >>> pe_fit = pe.fit(X)
+    >>> X = X[None, ...]
+    >>> print(f"Image shape: {X.shape}")
+    Image shape: (1, 427, 640, 3)
+    >>> pe = image.PatchExtractor(patch_size=(10, 10))
     >>> pe_trans = pe.transform(X)
-    >>> print('Patches shape: {}'.format(pe_trans.shape))
-    Patches shape: (545706, 2, 2)
+    >>> print(f"Patches shape: {pe_trans.shape}")
+    Patches shape: (263758, 10, 10, 3)
+    >>> X_reconstructed = image.reconstruct_from_patches_2d(pe_trans, X.shape[1:])
+    >>> print(f"Reconstructed shape: {X_reconstructed.shape}")
+    Reconstructed shape: (427, 640, 3)
     """
-    @_deprecate_positional_args
-    def __init__(self, *, patch_size=None, max_patches=None,
-                 random_state=None):
+
+    _parameter_constraints: dict = {
+        "patch_size": [tuple, None],
+        "max_patches": [
+            None,
+            Interval(RealNotInt, 0, 1, closed="neither"),
+            Interval(Integral, 1, None, closed="left"),
+        ],
+        "random_state": ["random_state"],
+    }
+
+    def __init__(self, *, patch_size=None, max_patches=None, random_state=None):
         self.patch_size = patch_size
         self.max_patches = max_patches
         self.random_state = random_state
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y=None):
-        """Do nothing and return the estimator unchanged.
+        """Only validate the parameters of the estimator.
 
-        This method is just there to implement the usual API and hence
-        work in pipelines.
+        This method allows to: (i) validate the parameters of the estimator  and
+        (ii) be consistent with the scikit-learn transformer API.
 
         Parameters
         ----------
-        X : array-like of shape (n_samples, n_features)
-            Training data.
+        X : ndarray of shape (n_samples, image_height, image_width) or \
+                (n_samples, image_height, image_width, n_channels)
+            Array of images from which to extract patches. For color images,
+            the last dimension specifies the channel: a RGB image would have
+            `n_channels=3`.
+
+        y : Ignored
+            Not used, present for API consistency by convention.
+
+        Returns
+        -------
+        self : object
+            Returns the instance itself.
         """
         return self
 
     def transform(self, X):
-        """Transforms the image samples in X into a matrix of patch data.
+        """Transform the image samples in `X` into a matrix of patch data.
 
         Parameters
         ----------
         X : ndarray of shape (n_samples, image_height, image_width) or \
-            (n_samples, image_height, image_width, n_channels)
+                (n_samples, image_height, image_width, n_channels)
             Array of images from which to extract patches. For color images,
             the last dimension specifies the channel: a RGB image would have
             `n_channels=3`.
@@ -517,34 +629,59 @@ def transform(self, X):
         Returns
         -------
         patches : array of shape (n_patches, patch_height, patch_width) or \
-             (n_patches, patch_height, patch_width, n_channels)
-             The collection of patches extracted from the images, where
-             `n_patches` is either `n_samples * max_patches` or the total
-             number of patches that can be extracted.
+                (n_patches, patch_height, patch_width, n_channels)
+            The collection of patches extracted from the images, where
+            `n_patches` is either `n_samples * max_patches` or the total
+            number of patches that can be extracted.
         """
-        self.random_state = check_random_state(self.random_state)
-        n_images, i_h, i_w = X.shape[:3]
-        X = np.reshape(X, (n_images, i_h, i_w, -1))
-        n_channels = X.shape[-1]
+        X = validate_data(
+            self,
+            X=X,
+            ensure_2d=False,
+            allow_nd=True,
+            ensure_min_samples=1,
+            ensure_min_features=1,
+            reset=False,
+        )
+        random_state = check_random_state(self.random_state)
+        n_imgs, img_height, img_width = X.shape[:3]
         if self.patch_size is None:
-            patch_size = i_h // 10, i_w // 10
+            patch_size = img_height // 10, img_width // 10
         else:
+            if len(self.patch_size) != 2:
+                raise ValueError(
+                    "patch_size must be a tuple of two integers. Got"
+                    f" {self.patch_size} instead."
+                )
             patch_size = self.patch_size
 
+        n_imgs, img_height, img_width = X.shape[:3]
+        X = np.reshape(X, (n_imgs, img_height, img_width, -1))
+        n_channels = X.shape[-1]
+
         # compute the dimensions of the patches array
-        p_h, p_w = patch_size
-        n_patches = _compute_n_patches(i_h, i_w, p_h, p_w, self.max_patches)
-        patches_shape = (n_images * n_patches,) + patch_size
+        patch_height, patch_width = patch_size
+        n_patches = _compute_n_patches(
+            img_height, img_width, patch_height, patch_width, self.max_patches
+        )
+        patches_shape = (n_imgs * n_patches,) + patch_size
         if n_channels > 1:
             patches_shape += (n_channels,)
 
         # extract the patches
         patches = np.empty(patches_shape)
         for ii, image in enumerate(X):
-            patches[ii * n_patches:(ii + 1) * n_patches] = extract_patches_2d(
-                image, patch_size, max_patches=self.max_patches,
-                random_state=self.random_state)
+            patches[ii * n_patches : (ii + 1) * n_patches] = extract_patches_2d(
+                image,
+                patch_size,
+                max_patches=self.max_patches,
+                random_state=random_state,
+            )
         return patches
 
-    def _more_tags(self):
-        return {'X_types': ['3darray']}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.two_d_array = False
+        tags.input_tags.three_d_array = True
+        tags.requires_fit = False
+        return tags
diff --git a/sklearn/feature_extraction/meson.build b/sklearn/feature_extraction/meson.build
new file mode 100644
index 0000000000000..f810d7b28576c
--- /dev/null
+++ b/sklearn/feature_extraction/meson.build
@@ -0,0 +1,7 @@
+py.extension_module(
+  '_hashing_fast',
+  [cython_gen_cpp.process('_hashing_fast.pyx'), utils_cython_tree],
+  dependencies: [np_dep],
+  subdir: 'sklearn/feature_extraction',
+  install: true
+)
diff --git a/sklearn/feature_extraction/setup.py b/sklearn/feature_extraction/setup.py
deleted file mode 100644
index 8c3bbb100f9d2..0000000000000
--- a/sklearn/feature_extraction/setup.py
+++ /dev/null
@@ -1,21 +0,0 @@
-import os
-import platform
-
-
-def configuration(parent_package='', top_path=None):
-    import numpy
-    from numpy.distutils.misc_util import Configuration
-
-    config = Configuration('feature_extraction', parent_package, top_path)
-    libraries = []
-    if os.name == 'posix':
-        libraries.append('m')
-
-    if platform.python_implementation() != 'PyPy':
-        config.add_extension('_hashing_fast',
-                             sources=['_hashing_fast.pyx'],
-                             include_dirs=[numpy.get_include()],
-                             libraries=libraries)
-    config.add_subpackage("tests")
-
-    return config
diff --git a/sklearn/feature_extraction/tests/test_dict_vectorizer.py b/sklearn/feature_extraction/tests/test_dict_vectorizer.py
index 519201b580598..7a539942d1e46 100644
--- a/sklearn/feature_extraction/tests/test_dict_vectorizer.py
+++ b/sklearn/feature_extraction/tests/test_dict_vectorizer.py
@@ -1,26 +1,24 @@
-# Authors: Lars Buitinck
-#          Dan Blanchard 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 from random import Random
-import numpy as np
-import scipy.sparse as sp
-from numpy.testing import assert_array_equal
 
+import numpy as np
 import pytest
+import scipy.sparse as sp
+from numpy.testing import assert_allclose, assert_array_equal
 
+from sklearn.exceptions import NotFittedError
 from sklearn.feature_extraction import DictVectorizer
 from sklearn.feature_selection import SelectKBest, chi2
 
 
-@pytest.mark.parametrize('sparse', (True, False))
-@pytest.mark.parametrize('dtype', (int, np.float32, np.int16))
-@pytest.mark.parametrize('sort', (True, False))
-@pytest.mark.parametrize('iterable', (True, False))
+@pytest.mark.parametrize("sparse", (True, False))
+@pytest.mark.parametrize("dtype", (int, np.float32, np.int16))
+@pytest.mark.parametrize("sort", (True, False))
+@pytest.mark.parametrize("iterable", (True, False))
 def test_dictvectorizer(sparse, dtype, sort, iterable):
-    D = [{"foo": 1, "bar": 3},
-         {"bar": 4, "baz": 2},
-         {"bar": 1, "quux": 1, "quuux": 2}]
+    D = [{"foo": 1, "bar": 3}, {"bar": 4, "baz": 2}, {"bar": 1, "quux": 1, "quuux": 2}]
 
     v = DictVectorizer(sparse=sparse, dtype=dtype, sort=sort)
     X = v.fit_transform(iter(D) if iterable else D)
@@ -32,24 +30,21 @@ def test_dictvectorizer(sparse, dtype, sort, iterable):
 
     if sparse:
         # CSR matrices can't be compared for equality
-        assert_array_equal(X.A, v.transform(iter(D) if iterable
-                                            else D).A)
+        assert_array_equal(
+            X.toarray(), v.transform(iter(D) if iterable else D).toarray()
+        )
     else:
-        assert_array_equal(X, v.transform(iter(D) if iterable
-                                          else D))
+        assert_array_equal(X, v.transform(iter(D) if iterable else D))
 
     if sort:
-        assert (v.feature_names_ ==
-                     sorted(v.feature_names_))
+        assert v.feature_names_ == sorted(v.feature_names_)
 
 
 def test_feature_selection():
     # make two feature dicts with two useful features and a bunch of useless
     # ones, in terms of chi2
-    d1 = dict([("useless%d" % i, 10) for i in range(20)],
-              useful1=1, useful2=20)
-    d2 = dict([("useless%d" % i, 10) for i in range(20)],
-              useful1=20, useful2=1)
+    d1 = dict([("useless%d" % i, 10) for i in range(20)], useful1=1, useful2=20)
+    d2 = dict([("useless%d" % i, 10) for i in range(20)], useful1=20, useful2=1)
 
     for indices in (True, False):
         v = DictVectorizer().fit([d1, d2])
@@ -57,13 +52,15 @@ def test_feature_selection():
         sel = SelectKBest(chi2, k=2).fit(X, [0, 1])
 
         v.restrict(sel.get_support(indices=indices), indices=indices)
-        assert v.get_feature_names() == ["useful1", "useful2"]
+        assert_array_equal(v.get_feature_names_out(), ["useful1", "useful2"])
 
 
 def test_one_of_k():
-    D_in = [{"version": "1", "ham": 2},
-            {"version": "2", "spam": .3},
-            {"version=3": True, "spam": -1}]
+    D_in = [
+        {"version": "1", "ham": 2},
+        {"version": "2", "spam": 0.3},
+        {"version=3": True, "spam": -1},
+    ]
     v = DictVectorizer()
     X = v.fit_transform(D_in)
     assert X.shape == (3, 5)
@@ -71,19 +68,23 @@ def test_one_of_k():
     D_out = v.inverse_transform(X)
     assert D_out[0] == {"version=1": 1, "ham": 2}
 
-    names = v.get_feature_names()
+    names = v.get_feature_names_out()
     assert "version=2" in names
     assert "version" not in names
 
 
 def test_iterable_value():
-    D_names = ['ham', 'spam', 'version=1', 'version=2', 'version=3']
-    X_expected = [[2.0, 0.0, 2.0, 1.0, 0.0],
-                  [0.0, 0.3, 0.0, 1.0, 0.0],
-                  [0.0, -1.0, 0.0, 0.0, 1.0]]
-    D_in = [{"version": ["1", "2", "1"], "ham": 2},
-            {"version": "2", "spam": .3},
-            {"version=3": True, "spam": -1}]
+    D_names = ["ham", "spam", "version=1", "version=2", "version=3"]
+    X_expected = [
+        [2.0, 0.0, 2.0, 1.0, 0.0],
+        [0.0, 0.3, 0.0, 1.0, 0.0],
+        [0.0, -1.0, 0.0, 0.0, 1.0],
+    ]
+    D_in = [
+        {"version": ["1", "2", "1"], "ham": 2},
+        {"version": "2", "spam": 0.3},
+        {"version=3": True, "spam": -1},
+    ]
     v = DictVectorizer()
     X = v.fit_transform(D_in)
     X = X.toarray()
@@ -92,17 +93,17 @@ def test_iterable_value():
     D_out = v.inverse_transform(X)
     assert D_out[0] == {"version=1": 2, "version=2": 1, "ham": 2}
 
-    names = v.get_feature_names()
+    names = v.get_feature_names_out()
 
-    assert names == D_names
+    assert_array_equal(names, D_names)
 
 
 def test_iterable_not_string_error():
-    error_value = ("Unsupported type  in iterable value. "
-                   "Only iterables of string are supported.")
-    D2 = [{'foo': '1', 'bar': '2'},
-          {'foo': '3', 'baz': '1'},
-          {'foo': [1, 'three']}]
+    error_value = (
+        "Unsupported type  in iterable value. "
+        "Only iterables of string are supported."
+    )
+    D2 = [{"foo": "1", "bar": "2"}, {"foo": "3", "baz": "1"}, {"foo": [1, "three"]}]
     v = DictVectorizer(sparse=False)
     with pytest.raises(TypeError) as error:
         v.fit(D2)
@@ -110,12 +111,16 @@ def test_iterable_not_string_error():
 
 
 def test_mapping_error():
-    error_value = ("Unsupported value type  "
-                   "for foo: {'one': 1, 'three': 3}.\n"
-                   "Mapping objects are not supported.")
-    D2 = [{'foo': '1', 'bar': '2'},
-          {'foo': '3', 'baz': '1'},
-          {'foo': {'one': 1, 'three': 3}}]
+    error_value = (
+        "Unsupported value type  "
+        "for foo: {'one': 1, 'three': 3}.\n"
+        "Mapping objects are not supported."
+    )
+    D2 = [
+        {"foo": "1", "bar": "2"},
+        {"foo": "3", "baz": "1"},
+        {"foo": {"one": 1, "three": 3}},
+    ]
     v = DictVectorizer(sparse=False)
     with pytest.raises(TypeError) as error:
         v.fit(D2)
@@ -137,16 +142,14 @@ def test_unseen_or_no_features():
             X = X.toarray()
         assert_array_equal(X, np.zeros((1, 2)))
 
-        try:
+        with pytest.raises(ValueError, match="empty"):
             v.transform([])
-        except ValueError as e:
-            assert "empty" in str(e)
 
 
-def test_deterministic_vocabulary():
+def test_deterministic_vocabulary(global_random_seed):
     # Generate equal dictionaries with different memory layouts
     items = [("%03d" % i, i) for i in range(1000)]
-    rng = Random(42)
+    rng = Random(global_random_seed)
     d_sorted = dict(items)
     rng.shuffle(items)
     d_shuffled = dict(items)
@@ -161,7 +164,98 @@ def test_deterministic_vocabulary():
 def test_n_features_in():
     # For vectorizers, n_features_in_ does not make sense and does not exist.
     dv = DictVectorizer()
-    assert not hasattr(dv, 'n_features_in_')
-    d = [{'foo': 1, 'bar': 2}, {'foo': 3, 'baz': 1}]
+    assert not hasattr(dv, "n_features_in_")
+    d = [{"foo": 1, "bar": 2}, {"foo": 3, "baz": 1}]
     dv.fit(d)
-    assert not hasattr(dv, 'n_features_in_')
+    assert not hasattr(dv, "n_features_in_")
+
+
+def test_dictvectorizer_dense_sparse_equivalence():
+    """Check the equivalence between between sparse and dense DictVectorizer.
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/19978
+    """
+    movie_entry_fit = [
+        {"category": ["thriller", "drama"], "year": 2003},
+        {"category": ["animation", "family"], "year": 2011},
+        {"year": 1974},
+    ]
+    movie_entry_transform = [{"category": ["thriller"], "unseen_feature": "3"}]
+    dense_vectorizer = DictVectorizer(sparse=False)
+    sparse_vectorizer = DictVectorizer(sparse=True)
+
+    dense_vector_fit = dense_vectorizer.fit_transform(movie_entry_fit)
+    sparse_vector_fit = sparse_vectorizer.fit_transform(movie_entry_fit)
+
+    assert not sp.issparse(dense_vector_fit)
+    assert sp.issparse(sparse_vector_fit)
+
+    assert_allclose(dense_vector_fit, sparse_vector_fit.toarray())
+
+    dense_vector_transform = dense_vectorizer.transform(movie_entry_transform)
+    sparse_vector_transform = sparse_vectorizer.transform(movie_entry_transform)
+
+    assert not sp.issparse(dense_vector_transform)
+    assert sp.issparse(sparse_vector_transform)
+
+    assert_allclose(dense_vector_transform, sparse_vector_transform.toarray())
+
+    dense_inverse_transform = dense_vectorizer.inverse_transform(dense_vector_transform)
+    sparse_inverse_transform = sparse_vectorizer.inverse_transform(
+        sparse_vector_transform
+    )
+
+    expected_inverse = [{"category=thriller": 1.0}]
+    assert dense_inverse_transform == expected_inverse
+    assert sparse_inverse_transform == expected_inverse
+
+
+def test_dict_vectorizer_unsupported_value_type():
+    """Check that we raise an error when the value associated to a feature
+    is not supported.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/19489
+    """
+
+    class A:
+        pass
+
+    vectorizer = DictVectorizer(sparse=True)
+    X = [{"foo": A()}]
+    err_msg = "Unsupported value Type"
+    with pytest.raises(TypeError, match=err_msg):
+        vectorizer.fit_transform(X)
+
+
+def test_dict_vectorizer_get_feature_names_out():
+    """Check that integer feature names are converted to strings in
+    feature_names_out."""
+
+    X = [{1: 2, 3: 4}, {2: 4}]
+    dv = DictVectorizer(sparse=False).fit(X)
+
+    feature_names = dv.get_feature_names_out()
+    assert isinstance(feature_names, np.ndarray)
+    assert feature_names.dtype == object
+    assert_array_equal(feature_names, ["1", "2", "3"])
+
+
+@pytest.mark.parametrize(
+    "method, input",
+    [
+        ("transform", [{1: 2, 3: 4}, {2: 4}]),
+        ("inverse_transform", [{1: 2, 3: 4}, {2: 4}]),
+        ("restrict", [True, False, True]),
+    ],
+)
+def test_dict_vectorizer_not_fitted_error(method, input):
+    """Check that unfitted DictVectorizer instance raises NotFittedError.
+
+    This should be part of the common test but currently they test estimator accepting
+    text input.
+    """
+    dv = DictVectorizer(sparse=False)
+
+    with pytest.raises(NotFittedError):
+        getattr(dv, method)(input)
diff --git a/sklearn/feature_extraction/tests/test_feature_hasher.py b/sklearn/feature_extraction/tests/test_feature_hasher.py
index c0cd50cef6e09..276d0d48b0770 100644
--- a/sklearn/feature_extraction/tests/test_feature_hasher.py
+++ b/sklearn/feature_extraction/tests/test_feature_hasher.py
@@ -1,21 +1,16 @@
-
 import numpy as np
-from numpy.testing import assert_array_equal
 import pytest
+from numpy.testing import assert_array_equal
 
 from sklearn.feature_extraction import FeatureHasher
-from sklearn.utils._testing import (ignore_warnings,
-                                   fails_if_pypy)
-
-pytestmark = fails_if_pypy
+from sklearn.feature_extraction._hashing_fast import transform as _hashing_transform
 
 
 def test_feature_hasher_dicts():
-    h = FeatureHasher(n_features=16)
-    assert "dict" == h.input_type
+    feature_hasher = FeatureHasher(n_features=16)
+    assert "dict" == feature_hasher.input_type
 
-    raw_X = [{"foo": "bar", "dada": 42, "tzara": 37},
-             {"foo": "baz", "gaga": "string1"}]
+    raw_X = [{"foo": "bar", "dada": 42, "tzara": 37}, {"foo": "baz", "gaga": "string1"}]
     X1 = FeatureHasher(n_features=16).transform(raw_X)
     gen = (iter(d.items()) for d in raw_X)
     X2 = FeatureHasher(n_features=16, input_type="pair").transform(gen)
@@ -24,17 +19,20 @@ def test_feature_hasher_dicts():
 
 def test_feature_hasher_strings():
     # mix byte and Unicode strings; note that "foo" is a duplicate in row 0
-    raw_X = [["foo", "bar", "baz", "foo".encode("ascii")],
-             ["bar".encode("ascii"), "baz", "quux"]]
+    raw_X = [
+        ["foo", "bar", "baz", "foo".encode("ascii")],
+        ["bar".encode("ascii"), "baz", "quux"],
+    ]
 
     for lg_n_features in (7, 9, 11, 16, 22):
-        n_features = 2 ** lg_n_features
+        n_features = 2**lg_n_features
 
-        it = (x for x in raw_X)                 # iterable
+        it = (x for x in raw_X)  # iterable
 
-        h = FeatureHasher(n_features=n_features, input_type="string",
-                          alternate_sign=False)
-        X = h.transform(it)
+        feature_hasher = FeatureHasher(
+            n_features=n_features, input_type="string", alternate_sign=False
+        )
+        X = feature_hasher.transform(it)
 
         assert X.shape[0] == len(raw_X)
         assert X.shape[1] == n_features
@@ -45,36 +43,54 @@ def test_feature_hasher_strings():
         assert X.nnz == 6
 
 
+@pytest.mark.parametrize(
+    "raw_X",
+    [
+        ["my_string", "another_string"],
+        (x for x in ["my_string", "another_string"]),
+    ],
+    ids=["list", "generator"],
+)
+def test_feature_hasher_single_string(raw_X):
+    """FeatureHasher raises error when a sample is a single string.
+
+    Non-regression test for gh-13199.
+    """
+    msg = "Samples can not be a single string"
+
+    feature_hasher = FeatureHasher(n_features=10, input_type="string")
+    with pytest.raises(ValueError, match=msg):
+        feature_hasher.transform(raw_X)
+
+
 def test_hashing_transform_seed():
     # check the influence of the seed when computing the hashes
-    # import is here to avoid importing on pypy
-    from sklearn.feature_extraction._hashing_fast import (
-            transform as _hashing_transform)
-    raw_X = [["foo", "bar", "baz", "foo".encode("ascii")],
-             ["bar".encode("ascii"), "baz", "quux"]]
+    raw_X = [
+        ["foo", "bar", "baz", "foo".encode("ascii")],
+        ["bar".encode("ascii"), "baz", "quux"],
+    ]
 
     raw_X_ = (((f, 1) for f in x) for x in raw_X)
-    indices, indptr, _ = _hashing_transform(raw_X_, 2 ** 7, str,
-                                            False)
+    indices, indptr, _ = _hashing_transform(raw_X_, 2**7, str, False)
 
     raw_X_ = (((f, 1) for f in x) for x in raw_X)
-    indices_0, indptr_0, _ = _hashing_transform(raw_X_, 2 ** 7, str,
-                                                False, seed=0)
+    indices_0, indptr_0, _ = _hashing_transform(raw_X_, 2**7, str, False, seed=0)
     assert_array_equal(indices, indices_0)
     assert_array_equal(indptr, indptr_0)
 
     raw_X_ = (((f, 1) for f in x) for x in raw_X)
-    indices_1, _, _ = _hashing_transform(raw_X_, 2 ** 7, str,
-                                         False, seed=1)
+    indices_1, _, _ = _hashing_transform(raw_X_, 2**7, str, False, seed=1)
     with pytest.raises(AssertionError):
         assert_array_equal(indices, indices_1)
 
 
 def test_feature_hasher_pairs():
-    raw_X = (iter(d.items()) for d in [{"foo": 1, "bar": 2},
-                                       {"baz": 3, "quux": 4, "foo": -1}])
-    h = FeatureHasher(n_features=16, input_type="pair")
-    x1, x2 = h.transform(raw_X).toarray()
+    raw_X = (
+        iter(d.items())
+        for d in [{"foo": 1, "bar": 2}, {"baz": 3, "quux": 4, "foo": -1}]
+    )
+    feature_hasher = FeatureHasher(n_features=16, input_type="pair")
+    x1, x2 = feature_hasher.transform(raw_X).toarray()
     x1_nz = sorted(np.abs(x1[x1 != 0]))
     x2_nz = sorted(np.abs(x2[x2 != 0]))
     assert [1, 2] == x1_nz
@@ -82,18 +98,19 @@ def test_feature_hasher_pairs():
 
 
 def test_feature_hasher_pairs_with_string_values():
-    raw_X = (iter(d.items()) for d in [{"foo": 1, "bar": "a"},
-                                       {"baz": "abc", "quux": 4, "foo": -1}])
-    h = FeatureHasher(n_features=16, input_type="pair")
-    x1, x2 = h.transform(raw_X).toarray()
+    raw_X = (
+        iter(d.items())
+        for d in [{"foo": 1, "bar": "a"}, {"baz": "abc", "quux": 4, "foo": -1}]
+    )
+    feature_hasher = FeatureHasher(n_features=16, input_type="pair")
+    x1, x2 = feature_hasher.transform(raw_X).toarray()
     x1_nz = sorted(np.abs(x1[x1 != 0]))
     x2_nz = sorted(np.abs(x2[x2 != 0]))
     assert [1, 1] == x1_nz
     assert [1, 1, 4] == x2_nz
 
-    raw_X = (iter(d.items()) for d in [{"bax": "abc"},
-                                       {"bax": "abc"}])
-    x1, x2 = h.transform(raw_X).toarray()
+    raw_X = (iter(d.items()) for d in [{"bax": "abc"}, {"bax": "abc"}])
+    x1, x2 = feature_hasher.transform(raw_X).toarray()
     x1_nz = np.abs(x1[x1 != 0])
     x2_nz = np.abs(x2[x2 != 0])
     assert [1] == x1_nz
@@ -105,67 +122,39 @@ def test_hash_empty_input():
     n_features = 16
     raw_X = [[], (), iter(range(0))]
 
-    h = FeatureHasher(n_features=n_features, input_type="string")
-    X = h.transform(raw_X)
-
-    assert_array_equal(X.A, np.zeros((len(raw_X), n_features)))
-
-
-def test_hasher_invalid_input():
-    with pytest.raises(ValueError):
-        FeatureHasher(input_type="gobbledygook")
-    with pytest.raises(ValueError):
-        FeatureHasher(n_features=-1)
-    with pytest.raises(ValueError):
-        FeatureHasher(n_features=0)
-    with pytest.raises(TypeError):
-        FeatureHasher(n_features='ham')
-
-    h = FeatureHasher(n_features=np.uint16(2 ** 6))
-    with pytest.raises(ValueError):
-        h.transform([])
-    with pytest.raises(Exception):
-        h.transform([[5.5]])
-    with pytest.raises(Exception):
-        h.transform([[None]])
-
+    feature_hasher = FeatureHasher(n_features=n_features, input_type="string")
+    X = feature_hasher.transform(raw_X)
 
-def test_hasher_set_params():
-    # Test delayed input validation in fit (useful for grid search).
-    hasher = FeatureHasher()
-    hasher.set_params(n_features=np.inf)
-    with pytest.raises(TypeError):
-        hasher.fit()
+    assert_array_equal(X.toarray(), np.zeros((len(raw_X), n_features)))
 
 
 def test_hasher_zeros():
     # Assert that no zeros are materialized in the output.
-    X = FeatureHasher().transform([{'foo': 0}])
+    X = FeatureHasher().transform([{"foo": 0}])
     assert X.data.shape == (0,)
 
 
-@ignore_warnings(category=FutureWarning)
 def test_hasher_alternate_sign():
     X = [list("Thequickbrownfoxjumped")]
 
-    Xt = FeatureHasher(alternate_sign=True,
-                       input_type='string').fit_transform(X)
+    Xt = FeatureHasher(alternate_sign=True, input_type="string").fit_transform(X)
     assert Xt.data.min() < 0 and Xt.data.max() > 0
 
-    Xt = FeatureHasher(alternate_sign=False,
-                       input_type='string').fit_transform(X)
+    Xt = FeatureHasher(alternate_sign=False, input_type="string").fit_transform(X)
     assert Xt.data.min() > 0
 
 
 def test_hash_collisions():
     X = [list("Thequickbrownfoxjumped")]
 
-    Xt = FeatureHasher(alternate_sign=True, n_features=1,
-                       input_type='string').fit_transform(X)
+    Xt = FeatureHasher(
+        alternate_sign=True, n_features=1, input_type="string"
+    ).fit_transform(X)
     # check that some of the hashed tokens are added
     # with an opposite sign and cancel out
     assert abs(Xt.data[0]) < len(X[0])
 
-    Xt = FeatureHasher(alternate_sign=False, n_features=1,
-                       input_type='string').fit_transform(X)
+    Xt = FeatureHasher(
+        alternate_sign=False, n_features=1, input_type="string"
+    ).fit_transform(X)
     assert Xt.data[0] == len(X[0])
diff --git a/sklearn/feature_extraction/tests/test_image.py b/sklearn/feature_extraction/tests/test_image.py
index 712eb840c63e2..2edf1a22d676a 100644
--- a/sklearn/feature_extraction/tests/test_image.py
+++ b/sklearn/feature_extraction/tests/test_image.py
@@ -1,17 +1,19 @@
-# Authors: Emmanuelle Gouillart 
-#          Gael Varoquaux 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import numpy as np
-import scipy as sp
+import pytest
 from scipy import ndimage
 from scipy.sparse.csgraph import connected_components
-import pytest
 
 from sklearn.feature_extraction.image import (
-    img_to_graph, grid_to_graph, extract_patches_2d,
-    reconstruct_from_patches_2d, PatchExtractor, _extract_patches)
-from sklearn.utils._testing import ignore_warnings
+    PatchExtractor,
+    _extract_patches,
+    extract_patches_2d,
+    grid_to_graph,
+    img_to_graph,
+    reconstruct_from_patches_2d,
+)
 
 
 def test_img_to_graph():
@@ -22,8 +24,24 @@ def test_img_to_graph():
     # Negative elements are the diagonal: the elements of the original
     # image. Positive elements are the values of the gradient, they
     # should all be equal on grad_x and grad_y
-    np.testing.assert_array_equal(grad_x.data[grad_x.data > 0],
-                                  grad_y.data[grad_y.data > 0])
+    np.testing.assert_array_equal(
+        grad_x.data[grad_x.data > 0], grad_y.data[grad_y.data > 0]
+    )
+
+
+def test_img_to_graph_sparse():
+    # Check that the edges are in the right position
+    #  when using a sparse image with a singleton component
+    mask = np.zeros((2, 3), dtype=bool)
+    mask[0, 0] = 1
+    mask[:, 2] = 1
+    x = np.zeros((2, 3))
+    x[0, 0] = 1
+    x[0, 2] = -1
+    x[1, 2] = -2
+    grad_x = img_to_graph(x, mask=mask).todense()
+    desired = np.array([[1, 0, 0], [0, -1, 1], [0, 1, -2]])
+    np.testing.assert_array_equal(grad_x, desired)
 
 
 def test_grid_to_graph():
@@ -35,10 +53,18 @@ def test_grid_to_graph():
     mask = np.zeros((size, size), dtype=bool)
     mask[0:roi_size, 0:roi_size] = True
     mask[-roi_size:, -roi_size:] = True
-    mask = mask.reshape(size ** 2)
+    mask = mask.reshape(size**2)
     A = grid_to_graph(n_x=size, n_y=size, mask=mask, return_as=np.ndarray)
     assert connected_components(A)[0] == 2
 
+    # check ordering
+    mask = np.zeros((2, 3), dtype=bool)
+    mask[0, 0] = 1
+    mask[:, 2] = 1
+    graph = grid_to_graph(2, 3, 1, mask=mask.ravel()).todense()
+    desired = np.array([[1, 0, 0], [0, 1, 1], [0, 1, 1]])
+    np.testing.assert_array_equal(graph, desired)
+
     # Checking that the function works whatever the type of mask is
     mask = np.ones((size, size), dtype=np.int16)
     A = grid_to_graph(n_x=size, n_y=size, n_z=size, mask=mask)
@@ -50,19 +76,12 @@ def test_grid_to_graph():
     assert A.dtype == bool
     A = grid_to_graph(n_x=size, n_y=size, n_z=size, mask=mask, dtype=int)
     assert A.dtype == int
-    A = grid_to_graph(n_x=size, n_y=size, n_z=size, mask=mask,
-                      dtype=np.float64)
+    A = grid_to_graph(n_x=size, n_y=size, n_z=size, mask=mask, dtype=np.float64)
     assert A.dtype == np.float64
 
 
-@ignore_warnings(category=DeprecationWarning)  # scipy deprecation inside face
-def test_connect_regions():
-    try:
-        face = sp.face(gray=True)
-    except AttributeError:
-        # Newer versions of scipy have face in misc
-        from scipy import misc
-        face = misc.face(gray=True)
+def test_connect_regions(raccoon_face_fxt):
+    face = raccoon_face_fxt
     # subsample by 4 to reduce run time
     face = face[::4, ::4]
     for thr in (50, 150):
@@ -71,14 +90,8 @@ def test_connect_regions():
         assert ndimage.label(mask)[1] == connected_components(graph)[0]
 
 
-@ignore_warnings(category=DeprecationWarning)  # scipy deprecation inside face
-def test_connect_regions_with_grid():
-    try:
-        face = sp.face(gray=True)
-    except AttributeError:
-        # Newer versions of scipy have face in misc
-        from scipy import misc
-        face = misc.face(gray=True)
+def test_connect_regions_with_grid(raccoon_face_fxt):
+    face = raccoon_face_fxt
 
     # subsample by 4 to reduce run time
     face = face[::4, ::4]
@@ -92,25 +105,19 @@ def test_connect_regions_with_grid():
     assert ndimage.label(mask)[1] == connected_components(graph)[0]
 
 
-def _downsampled_face():
-    try:
-        face = sp.face(gray=True)
-    except AttributeError:
-        # Newer versions of scipy have face in misc
-        from scipy import misc
-        face = misc.face(gray=True)
-    face = face.astype(np.float32)
-    face = (face[::2, ::2] + face[1::2, ::2] + face[::2, 1::2]
-            + face[1::2, 1::2])
-    face = (face[::2, ::2] + face[1::2, ::2] + face[::2, 1::2]
-            + face[1::2, 1::2])
+@pytest.fixture
+def downsampled_face(raccoon_face_fxt):
+    face = raccoon_face_fxt
+    face = face[::2, ::2] + face[1::2, ::2] + face[::2, 1::2] + face[1::2, 1::2]
+    face = face[::2, ::2] + face[1::2, ::2] + face[::2, 1::2] + face[1::2, 1::2]
     face = face.astype(np.float32)
     face /= 16.0
     return face
 
 
-def _orange_face(face=None):
-    face = _downsampled_face() if face is None else face
+@pytest.fixture
+def orange_face(downsampled_face):
+    face = downsampled_face
     face_color = np.zeros(face.shape + (3,))
     face_color[:, :, 0] = 256 - face
     face_color[:, :, 1] = 256 - face / 2
@@ -118,8 +125,7 @@ def _orange_face(face=None):
     return face_color
 
 
-def _make_images(face=None):
-    face = _downsampled_face() if face is None else face
+def _make_images(face):
     # make a collection of faces
     images = np.zeros((3,) + face.shape)
     images[0] = face
@@ -127,12 +133,13 @@ def _make_images(face=None):
     images[2] = face + 2
     return images
 
-downsampled_face = _downsampled_face()
-orange_face = _orange_face(downsampled_face)
-face_collection = _make_images(downsampled_face)
 
+@pytest.fixture
+def downsampled_face_collection(downsampled_face):
+    return _make_images(downsampled_face)
 
-def test_extract_patches_all():
+
+def test_extract_patches_all(downsampled_face):
     face = downsampled_face
     i_h, i_w = face.shape
     p_h, p_w = 16, 16
@@ -141,7 +148,7 @@ def test_extract_patches_all():
     assert patches.shape == (expected_n_patches, p_h, p_w)
 
 
-def test_extract_patches_all_color():
+def test_extract_patches_all_color(orange_face):
     face = orange_face
     i_h, i_w = face.shape[:2]
     p_h, p_w = 16, 16
@@ -150,7 +157,7 @@ def test_extract_patches_all_color():
     assert patches.shape == (expected_n_patches, p_h, p_w, 3)
 
 
-def test_extract_patches_all_rect():
+def test_extract_patches_all_rect(downsampled_face):
     face = downsampled_face
     face = face[:, 32:97]
     i_h, i_w = face.shape
@@ -161,7 +168,7 @@ def test_extract_patches_all_rect():
     assert patches.shape == (expected_n_patches, p_h, p_w)
 
 
-def test_extract_patches_max_patches():
+def test_extract_patches_max_patches(downsampled_face):
     face = downsampled_face
     i_h, i_w = face.shape
     p_h, p_w = 16, 16
@@ -179,7 +186,7 @@ def test_extract_patches_max_patches():
         extract_patches_2d(face, (p_h, p_w), max_patches=-1.0)
 
 
-def test_extract_patch_same_size_image():
+def test_extract_patch_same_size_image(downsampled_face):
     face = downsampled_face
     # Request patches of the same size as image
     # Should return just the single patch a.k.a. the image
@@ -187,7 +194,7 @@ def test_extract_patch_same_size_image():
     assert patches.shape[0] == 1
 
 
-def test_extract_patches_less_than_max_patches():
+def test_extract_patches_less_than_max_patches(downsampled_face):
     face = downsampled_face
     i_h, i_w = face.shape
     p_h, p_w = 3 * i_h // 4, 3 * i_w // 4
@@ -198,7 +205,7 @@ def test_extract_patches_less_than_max_patches():
     assert patches.shape == (expected_n_patches, p_h, p_w)
 
 
-def test_reconstruct_patches_perfect():
+def test_reconstruct_patches_perfect(downsampled_face):
     face = downsampled_face
     p_h, p_w = 16, 16
 
@@ -207,7 +214,7 @@ def test_reconstruct_patches_perfect():
     np.testing.assert_array_almost_equal(face, face_reconstructed)
 
 
-def test_reconstruct_patches_perfect_color():
+def test_reconstruct_patches_perfect_color(orange_face):
     face = orange_face
     p_h, p_w = 16, 16
 
@@ -216,42 +223,45 @@ def test_reconstruct_patches_perfect_color():
     np.testing.assert_array_almost_equal(face, face_reconstructed)
 
 
-def test_patch_extractor_fit():
-    faces = face_collection
+def test_patch_extractor_fit(downsampled_face_collection):
+    faces = downsampled_face_collection
     extr = PatchExtractor(patch_size=(8, 8), max_patches=100, random_state=0)
     assert extr == extr.fit(faces)
 
 
-def test_patch_extractor_max_patches():
-    faces = face_collection
+def test_patch_extractor_max_patches(downsampled_face_collection):
+    faces = downsampled_face_collection
     i_h, i_w = faces.shape[1:3]
     p_h, p_w = 8, 8
 
     max_patches = 100
     expected_n_patches = len(faces) * max_patches
-    extr = PatchExtractor(patch_size=(p_h, p_w), max_patches=max_patches,
-                          random_state=0)
+    extr = PatchExtractor(
+        patch_size=(p_h, p_w), max_patches=max_patches, random_state=0
+    )
     patches = extr.transform(faces)
     assert patches.shape == (expected_n_patches, p_h, p_w)
 
     max_patches = 0.5
-    expected_n_patches = len(faces) * int((i_h - p_h + 1) * (i_w - p_w + 1)
-                                          * max_patches)
-    extr = PatchExtractor(patch_size=(p_h, p_w), max_patches=max_patches,
-                          random_state=0)
+    expected_n_patches = len(faces) * int(
+        (i_h - p_h + 1) * (i_w - p_w + 1) * max_patches
+    )
+    extr = PatchExtractor(
+        patch_size=(p_h, p_w), max_patches=max_patches, random_state=0
+    )
     patches = extr.transform(faces)
     assert patches.shape == (expected_n_patches, p_h, p_w)
 
 
-def test_patch_extractor_max_patches_default():
-    faces = face_collection
+def test_patch_extractor_max_patches_default(downsampled_face_collection):
+    faces = downsampled_face_collection
     extr = PatchExtractor(max_patches=100, random_state=0)
     patches = extr.transform(faces)
     assert patches.shape == (len(faces) * 100, 19, 25)
 
 
-def test_patch_extractor_all_patches():
-    faces = face_collection
+def test_patch_extractor_all_patches(downsampled_face_collection):
+    faces = downsampled_face_collection
     i_h, i_w = faces.shape[1:3]
     p_h, p_w = 8, 8
     expected_n_patches = len(faces) * (i_h - p_h + 1) * (i_w - p_w + 1)
@@ -260,7 +270,7 @@ def test_patch_extractor_all_patches():
     assert patches.shape == (expected_n_patches, p_h, p_w)
 
 
-def test_patch_extractor_color():
+def test_patch_extractor_color(orange_face):
     faces = _make_images(orange_face)
     i_h, i_w = faces.shape[1:3]
     p_h, p_w = 8, 8
@@ -271,7 +281,6 @@ def test_patch_extractor_color():
 
 
 def test_extract_patches_strided():
-
     image_shapes_1D = [(10,), (10,), (11,), (10,)]
     patch_sizes_1D = [(1,), (2,), (3,), (8,)]
     patch_steps_1D = [(1,), (1,), (4,), (2,)]
@@ -299,31 +308,33 @@ def test_extract_patches_strided():
     expected_views = expected_views_1D + expected_views_2D + expected_views_3D
     last_patches = last_patch_1D + last_patch_2D + last_patch_3D
 
-    for (image_shape, patch_size, patch_step, expected_view,
-         last_patch) in zip(image_shapes, patch_sizes, patch_steps,
-                            expected_views, last_patches):
+    for image_shape, patch_size, patch_step, expected_view, last_patch in zip(
+        image_shapes, patch_sizes, patch_steps, expected_views, last_patches
+    ):
         image = np.arange(np.prod(image_shape)).reshape(image_shape)
-        patches = _extract_patches(image, patch_shape=patch_size,
-                                   extraction_step=patch_step)
+        patches = _extract_patches(
+            image, patch_shape=patch_size, extraction_step=patch_step
+        )
 
         ndim = len(image_shape)
 
         assert patches.shape[:ndim] == expected_view
-        last_patch_slices = tuple(slice(i, i + j, None) for i, j in
-                                  zip(last_patch, patch_size))
-        assert (patches[(-1, None, None) * ndim] ==
-                image[last_patch_slices].squeeze()).all()
+        last_patch_slices = tuple(
+            slice(i, i + j, None) for i, j in zip(last_patch, patch_size)
+        )
+        assert (
+            patches[(-1, None, None) * ndim] == image[last_patch_slices].squeeze()
+        ).all()
 
 
-def test_extract_patches_square():
+def test_extract_patches_square(downsampled_face):
     # test same patch size for all dimensions
     face = downsampled_face
     i_h, i_w = face.shape
     p = 8
     expected_n_patches = ((i_h - p + 1), (i_w - p + 1))
     patches = _extract_patches(face, patch_shape=p)
-    assert patches.shape == (expected_n_patches[0],
-                             expected_n_patches[1], p, p)
+    assert patches.shape == (expected_n_patches[0], expected_n_patches[1], p, p)
 
 
 def test_width_patch():
@@ -333,3 +344,12 @@ def test_width_patch():
         extract_patches_2d(x, (4, 1))
     with pytest.raises(ValueError):
         extract_patches_2d(x, (1, 4))
+
+
+def test_patch_extractor_wrong_input(orange_face):
+    """Check that an informative error is raised if the patch_size is not valid."""
+    faces = _make_images(orange_face)
+    err_msg = "patch_size must be a tuple of two integers"
+    extractor = PatchExtractor(patch_size=(8, 8, 8))
+    with pytest.raises(ValueError, match=err_msg):
+        extractor.transform(faces)
diff --git a/sklearn/feature_extraction/tests/test_text.py b/sklearn/feature_extraction/tests/test_text.py
index 767b04ddb5d95..ab3f84668fd2d 100644
--- a/sklearn/feature_extraction/tests/test_text.py
+++ b/sklearn/feature_extraction/tests/test_text.py
@@ -1,42 +1,38 @@
-# -*- coding: utf-8 -*-
-from collections.abc import Mapping
+import pickle
 import re
+import uuid
+import warnings
+from collections import defaultdict
+from collections.abc import Mapping
+from functools import partial
+from io import StringIO
+from itertools import product
 
+import numpy as np
 import pytest
+from numpy.testing import assert_array_almost_equal, assert_array_equal
 from scipy import sparse
 
-from sklearn.feature_extraction.text import strip_tags
-from sklearn.feature_extraction.text import strip_accents_unicode
-from sklearn.feature_extraction.text import strip_accents_ascii
-
-from sklearn.feature_extraction.text import HashingVectorizer
-from sklearn.feature_extraction.text import CountVectorizer
-from sklearn.feature_extraction.text import TfidfTransformer
-from sklearn.feature_extraction.text import TfidfVectorizer
-
-from sklearn.feature_extraction.text import ENGLISH_STOP_WORDS
-
-from sklearn.model_selection import train_test_split
-from sklearn.model_selection import cross_val_score
-from sklearn.model_selection import GridSearchCV
+from sklearn.base import clone
+from sklearn.feature_extraction.text import (
+    ENGLISH_STOP_WORDS,
+    CountVectorizer,
+    HashingVectorizer,
+    TfidfTransformer,
+    TfidfVectorizer,
+    strip_accents_ascii,
+    strip_accents_unicode,
+    strip_tags,
+)
+from sklearn.model_selection import GridSearchCV, cross_val_score, train_test_split
 from sklearn.pipeline import Pipeline
 from sklearn.svm import LinearSVC
-
-from sklearn.base import clone
-
-import numpy as np
-from numpy.testing import assert_array_almost_equal
-from numpy.testing import assert_array_equal
-from sklearn.utils import IS_PYPY
-from sklearn.utils._testing import (assert_almost_equal,
-                                    assert_raise_message,
-                                    fails_if_pypy,
-                                    assert_allclose_dense_sparse,
-                                    skip_if_32bit)
-from collections import defaultdict
-from functools import partial
-import pickle
-from io import StringIO
+from sklearn.utils._testing import (
+    assert_allclose_dense_sparse,
+    assert_almost_equal,
+    skip_if_32bit,
+)
+from sklearn.utils.fixes import _IS_WASM, CSC_CONTAINERS, CSR_CONTAINERS
 
 JUNK_FOOD_DOCS = (
     "the pizza pizza beer copyright",
@@ -63,7 +59,7 @@ def uppercase(s):
 
 
 def strip_eacute(s):
-    return s.replace('Ê', 'e')
+    return s.replace("Ê", "e")
 
 
 def split_tokenize(s):
@@ -71,31 +67,31 @@ def split_tokenize(s):
 
 
 def lazy_analyze(s):
-    return ['the_ultimate_feature']
+    return ["the_ultimate_feature"]
 
 
 def test_strip_accents():
     # check some classical latin accentuated symbols
-    a = 'Ã ÃĄÃĸÃŖäÃĨçèÊÃĒÃĢ'
-    expected = 'aaaaaaceeee'
+    a = "Ã ÃĄÃĸÃŖäÃĨçèÊÃĒÃĢ"
+    expected = "aaaaaaceeee"
     assert strip_accents_unicode(a) == expected
 
-    a = 'ÃŦÃ­ÃŽÃ¯ÃąÃ˛ÃŗôÃĩÃļÚÃēÃģÃŧÃŊ'
-    expected = 'iiiinooooouuuuy'
+    a = "ÃŦÃ­ÃŽÃ¯ÃąÃ˛ÃŗôÃĩÃļÚÃēÃģÃŧÃŊ"
+    expected = "iiiinooooouuuuy"
     assert strip_accents_unicode(a) == expected
 
     # check some arabic
-    a = '\u0625'  # alef with a hamza below: ØĨ
-    expected = '\u0627'  # simple alef: ا
+    a = "\u0625"  # alef with a hamza below: ØĨ
+    expected = "\u0627"  # simple alef: ا
     assert strip_accents_unicode(a) == expected
 
     # mix letters accentuated and not
     a = "this is à test"
-    expected = 'this is a test'
+    expected = "this is a test"
     assert strip_accents_unicode(a) == expected
 
     # strings that are already decomposed
-    a = "o\u0308"  # o with diaresis
+    a = "o\u0308"  # o with diaeresis
     expected = "o"
     assert strip_accents_unicode(a) == expected
 
@@ -112,72 +108,114 @@ def test_strip_accents():
 
 def test_to_ascii():
     # check some classical latin accentuated symbols
-    a = 'Ã ÃĄÃĸÃŖäÃĨçèÊÃĒÃĢ'
-    expected = 'aaaaaaceeee'
+    a = "Ã ÃĄÃĸÃŖäÃĨçèÊÃĒÃĢ"
+    expected = "aaaaaaceeee"
     assert strip_accents_ascii(a) == expected
 
     a = "ÃŦÃ­ÃŽÃ¯ÃąÃ˛ÃŗôÃĩÃļÚÃēÃģÃŧÃŊ"
-    expected = 'iiiinooooouuuuy'
+    expected = "iiiinooooouuuuy"
     assert strip_accents_ascii(a) == expected
 
     # check some arabic
-    a = '\u0625'  # halef with a hamza below
-    expected = ''  # halef has no direct ascii match
+    a = "\u0625"  # halef with a hamza below
+    expected = ""  # halef has no direct ascii match
     assert strip_accents_ascii(a) == expected
 
     # mix letters accentuated and not
     a = "this is à test"
-    expected = 'this is a test'
+    expected = "this is a test"
     assert strip_accents_ascii(a) == expected
 
 
-@pytest.mark.parametrize('Vectorizer', (CountVectorizer, HashingVectorizer))
+@pytest.mark.parametrize("Vectorizer", (CountVectorizer, HashingVectorizer))
 def test_word_analyzer_unigrams(Vectorizer):
-    wa = Vectorizer(strip_accents='ascii').build_analyzer()
-    text = ("J'ai mangÊ du kangourou  ce midi, "
-            "c'Êtait pas très bon.")
-    expected = ['ai', 'mange', 'du', 'kangourou', 'ce', 'midi',
-                'etait', 'pas', 'tres', 'bon']
+    wa = Vectorizer(strip_accents="ascii").build_analyzer()
+    text = "J'ai mangÊ du kangourou  ce midi, c'Êtait pas très bon."
+    expected = [
+        "ai",
+        "mange",
+        "du",
+        "kangourou",
+        "ce",
+        "midi",
+        "etait",
+        "pas",
+        "tres",
+        "bon",
+    ]
     assert wa(text) == expected
 
     text = "This is a test, really.\n\n I met Harry yesterday."
-    expected = ['this', 'is', 'test', 'really', 'met', 'harry',
-                'yesterday']
+    expected = ["this", "is", "test", "really", "met", "harry", "yesterday"]
     assert wa(text) == expected
 
-    wa = Vectorizer(input='file').build_analyzer()
+    wa = Vectorizer(input="file").build_analyzer()
     text = StringIO("This is a test with a file-like object!")
-    expected = ['this', 'is', 'test', 'with', 'file', 'like',
-                'object']
+    expected = ["this", "is", "test", "with", "file", "like", "object"]
     assert wa(text) == expected
 
     # with custom preprocessor
     wa = Vectorizer(preprocessor=uppercase).build_analyzer()
-    text = ("J'ai mangÊ du kangourou  ce midi, "
-            " c'Êtait pas très bon.")
-    expected = ['AI', 'MANGE', 'DU', 'KANGOUROU', 'CE', 'MIDI',
-                'ETAIT', 'PAS', 'TRES', 'BON']
+    text = "J'ai mangÊ du kangourou  ce midi,  c'Êtait pas très bon."
+    expected = [
+        "AI",
+        "MANGE",
+        "DU",
+        "KANGOUROU",
+        "CE",
+        "MIDI",
+        "ETAIT",
+        "PAS",
+        "TRES",
+        "BON",
+    ]
     assert wa(text) == expected
 
     # with custom tokenizer
-    wa = Vectorizer(tokenizer=split_tokenize,
-                    strip_accents='ascii').build_analyzer()
-    text = ("J'ai mangÊ du kangourou  ce midi, "
-            "c'Êtait pas très bon.")
-    expected = ["j'ai", 'mange', 'du', 'kangourou', 'ce', 'midi,',
-                "c'etait", 'pas', 'tres', 'bon.']
+    wa = Vectorizer(tokenizer=split_tokenize, strip_accents="ascii").build_analyzer()
+    text = "J'ai mangÊ du kangourou  ce midi, c'Êtait pas très bon."
+    expected = [
+        "j'ai",
+        "mange",
+        "du",
+        "kangourou",
+        "ce",
+        "midi,",
+        "c'etait",
+        "pas",
+        "tres",
+        "bon.",
+    ]
     assert wa(text) == expected
 
 
 def test_word_analyzer_unigrams_and_bigrams():
-    wa = CountVectorizer(analyzer="word", strip_accents='unicode',
-                         ngram_range=(1, 2)).build_analyzer()
+    wa = CountVectorizer(
+        analyzer="word", strip_accents="unicode", ngram_range=(1, 2)
+    ).build_analyzer()
 
     text = "J'ai mangÊ du kangourou  ce midi, c'Êtait pas très bon."
-    expected = ['ai', 'mange', 'du', 'kangourou', 'ce', 'midi',
-                'etait', 'pas', 'tres', 'bon', 'ai mange', 'mange du',
-                'du kangourou', 'kangourou ce', 'ce midi', 'midi etait',
-                'etait pas', 'pas tres', 'tres bon']
+    expected = [
+        "ai",
+        "mange",
+        "du",
+        "kangourou",
+        "ce",
+        "midi",
+        "etait",
+        "pas",
+        "tres",
+        "bon",
+        "ai mange",
+        "mange du",
+        "du kangourou",
+        "kangourou ce",
+        "ce midi",
+        "midi etait",
+        "etait pas",
+        "pas tres",
+        "tres bon",
+    ]
     assert wa(text) == expected
 
 
@@ -185,77 +223,86 @@ def test_unicode_decode_error():
     # decode_error default to strict, so this should fail
     # First, encode (as bytes) a unicode string.
     text = "J'ai mangÊ du kangourou  ce midi, c'Êtait pas très bon."
-    text_bytes = text.encode('utf-8')
+    text_bytes = text.encode("utf-8")
 
     # Then let the Analyzer try to decode it as ascii. It should fail,
     # because we have given it an incorrect encoding.
-    wa = CountVectorizer(ngram_range=(1, 2), encoding='ascii').build_analyzer()
+    wa = CountVectorizer(ngram_range=(1, 2), encoding="ascii").build_analyzer()
     with pytest.raises(UnicodeDecodeError):
         wa(text_bytes)
 
-    ca = CountVectorizer(analyzer='char', ngram_range=(3, 6),
-                         encoding='ascii').build_analyzer()
+    ca = CountVectorizer(
+        analyzer="char", ngram_range=(3, 6), encoding="ascii"
+    ).build_analyzer()
     with pytest.raises(UnicodeDecodeError):
         ca(text_bytes)
 
 
 def test_char_ngram_analyzer():
-    cnga = CountVectorizer(analyzer='char', strip_accents='unicode',
-                           ngram_range=(3, 6)).build_analyzer()
+    cnga = CountVectorizer(
+        analyzer="char", strip_accents="unicode", ngram_range=(3, 6)
+    ).build_analyzer()
 
     text = "J'ai mangÊ du kangourou  ce midi, c'Êtait pas très bon"
-    expected = ["j'a", "'ai", 'ai ', 'i m', ' ma']
+    expected = ["j'a", "'ai", "ai ", "i m", " ma"]
     assert cnga(text)[:5] == expected
-    expected = ['s tres', ' tres ', 'tres b', 'res bo', 'es bon']
+    expected = ["s tres", " tres ", "tres b", "res bo", "es bon"]
     assert cnga(text)[-5:] == expected
 
     text = "This \n\tis a test, really.\n\n I met Harry yesterday"
-    expected = ['thi', 'his', 'is ', 's i', ' is']
+    expected = ["thi", "his", "is ", "s i", " is"]
     assert cnga(text)[:5] == expected
 
-    expected = [' yeste', 'yester', 'esterd', 'sterda', 'terday']
+    expected = [" yeste", "yester", "esterd", "sterda", "terday"]
     assert cnga(text)[-5:] == expected
 
-    cnga = CountVectorizer(input='file', analyzer='char',
-                           ngram_range=(3, 6)).build_analyzer()
+    cnga = CountVectorizer(
+        input="file", analyzer="char", ngram_range=(3, 6)
+    ).build_analyzer()
     text = StringIO("This is a test with a file-like object!")
-    expected = ['thi', 'his', 'is ', 's i', ' is']
+    expected = ["thi", "his", "is ", "s i", " is"]
     assert cnga(text)[:5] == expected
 
 
 def test_char_wb_ngram_analyzer():
-    cnga = CountVectorizer(analyzer='char_wb', strip_accents='unicode',
-                           ngram_range=(3, 6)).build_analyzer()
+    cnga = CountVectorizer(
+        analyzer="char_wb", strip_accents="unicode", ngram_range=(3, 6)
+    ).build_analyzer()
 
     text = "This \n\tis a test, really.\n\n I met Harry yesterday"
-    expected = [' th', 'thi', 'his', 'is ', ' thi']
+    expected = [" th", "thi", "his", "is ", " thi"]
     assert cnga(text)[:5] == expected
 
-    expected = ['yester', 'esterd', 'sterda', 'terday', 'erday ']
+    expected = ["yester", "esterd", "sterda", "terday", "erday "]
     assert cnga(text)[-5:] == expected
 
-    cnga = CountVectorizer(input='file', analyzer='char_wb',
-                           ngram_range=(3, 6)).build_analyzer()
+    cnga = CountVectorizer(
+        input="file", analyzer="char_wb", ngram_range=(3, 6)
+    ).build_analyzer()
     text = StringIO("A test with a file-like object!")
-    expected = [' a ', ' te', 'tes', 'est', 'st ', ' tes']
+    expected = [" a ", " te", "tes", "est", "st ", " tes"]
     assert cnga(text)[:6] == expected
 
 
 def test_word_ngram_analyzer():
-    cnga = CountVectorizer(analyzer='word', strip_accents='unicode',
-                           ngram_range=(3, 6)).build_analyzer()
+    cnga = CountVectorizer(
+        analyzer="word", strip_accents="unicode", ngram_range=(3, 6)
+    ).build_analyzer()
 
     text = "This \n\tis a test, really.\n\n I met Harry yesterday"
-    expected = ['this is test', 'is test really', 'test really met']
+    expected = ["this is test", "is test really", "test really met"]
     assert cnga(text)[:3] == expected
 
-    expected = ['test really met harry yesterday',
-                'this is test really met harry',
-                'is test really met harry yesterday']
+    expected = [
+        "test really met harry yesterday",
+        "this is test really met harry",
+        "is test really met harry yesterday",
+    ]
     assert cnga(text)[-3:] == expected
 
-    cnga_file = CountVectorizer(input='file', analyzer='word',
-                                ngram_range=(3, 6)).build_analyzer()
+    cnga_file = CountVectorizer(
+        input="file", analyzer="word", ngram_range=(3, 6)
+    ).build_analyzer()
     file = StringIO(text)
     assert cnga_file(file) == cnga(text)
 
@@ -283,12 +330,14 @@ def test_countvectorizer_custom_vocabulary():
 
 def test_countvectorizer_custom_vocabulary_pipeline():
     what_we_like = ["pizza", "beer"]
-    pipe = Pipeline([
-        ('count', CountVectorizer(vocabulary=what_we_like)),
-        ('tfidf', TfidfTransformer())])
+    pipe = Pipeline(
+        [
+            ("count", CountVectorizer(vocabulary=what_we_like)),
+            ("tfidf", TfidfTransformer()),
+        ]
+    )
     X = pipe.fit_transform(ALL_FOOD_DOCS)
-    assert (set(pipe.named_steps['count'].vocabulary_) ==
-            set(what_we_like))
+    assert set(pipe.named_steps["count"].vocabulary_) == set(what_we_like)
     assert X.shape[1] == len(what_we_like)
 
 
@@ -304,20 +353,20 @@ def test_countvectorizer_custom_vocabulary_gap_index():
     vocab = {"pizza": 1, "beer": 2}
     with pytest.raises(ValueError, match="doesn't contain index"):
         vect = CountVectorizer(vocabulary=vocab)
-        vect.fit(['pasta_verdura'])
+        vect.fit(["pasta_verdura"])
 
 
 def test_countvectorizer_stop_words():
     cv = CountVectorizer()
-    cv.set_params(stop_words='english')
+    cv.set_params(stop_words="english")
     assert cv.get_stop_words() == ENGLISH_STOP_WORDS
-    cv.set_params(stop_words='_bad_str_stop_')
+    cv.set_params(stop_words="_bad_str_stop_")
     with pytest.raises(ValueError):
         cv.get_stop_words()
-    cv.set_params(stop_words='_bad_unicode_stop_')
+    cv.set_params(stop_words="_bad_unicode_stop_")
     with pytest.raises(ValueError):
         cv.get_stop_words()
-    stoplist = ['some', 'other', 'words']
+    stoplist = ["some", "other", "words"]
     cv.set_params(stop_words=stoplist)
     assert cv.get_stop_words() == set(stoplist)
 
@@ -341,21 +390,22 @@ def test_fit_countvectorizer_twice():
 
 
 def test_countvectorizer_custom_token_pattern():
-    """Check `get_feature_names()` when a custom token pattern is passed.
+    """Check `get_feature_names_out()` when a custom token pattern is passed.
     Non-regression test for:
     https://github.com/scikit-learn/scikit-learn/issues/12971
     """
     corpus = [
-        'This is the 1st document in my corpus.',
-        'This document is the 2nd sample.',
-        'And this is the 3rd one.',
-        'Is this the 4th document?',
+        "This is the 1st document in my corpus.",
+        "This document is the 2nd sample.",
+        "And this is the 3rd one.",
+        "Is this the 4th document?",
     ]
     token_pattern = r"[0-9]{1,3}(?:st|nd|rd|th)\s\b(\w{2,})\b"
     vectorizer = CountVectorizer(token_pattern=token_pattern)
     vectorizer.fit_transform(corpus)
-    expected = ['document', 'one', 'sample']
-    assert vectorizer.get_feature_names() == expected
+    expected = ["document", "one", "sample"]
+    feature_names_out = vectorizer.get_feature_names_out()
+    assert_array_equal(feature_names_out, expected)
 
 
 def test_countvectorizer_custom_token_pattern_with_several_group():
@@ -364,10 +414,10 @@ def test_countvectorizer_custom_token_pattern_with_several_group():
     https://github.com/scikit-learn/scikit-learn/issues/12971
     """
     corpus = [
-        'This is the 1st document in my corpus.',
-        'This document is the 2nd sample.',
-        'And this is the 3rd one.',
-        'Is this the 4th document?',
+        "This is the 1st document in my corpus.",
+        "This document is the 2nd sample.",
+        "And this is the 3rd one.",
+        "Is this the 4th document?",
     ]
 
     token_pattern = r"([0-9]{1,3}(?:st|nd|rd|th))\s\b(\w{2,})\b"
@@ -378,56 +428,74 @@ def test_countvectorizer_custom_token_pattern_with_several_group():
 
 
 def test_countvectorizer_uppercase_in_vocab():
-    vocabulary = ['Sample', 'Upper', 'Case' 'Vocabulary']
-    message = ("Upper case characters found in"
-               " vocabulary while 'lowercase'"
-               " is True. These entries will not"
-               " be matched with any documents")
+    # Check that the check for uppercase in the provided vocabulary is only done at fit
+    # time and not at transform time (#21251)
+    vocabulary = ["Sample", "Upper", "Case", "Vocabulary"]
+    message = (
+        "Upper case characters found in"
+        " vocabulary while 'lowercase'"
+        " is True. These entries will not"
+        " be matched with any documents"
+    )
 
     vectorizer = CountVectorizer(lowercase=True, vocabulary=vocabulary)
+
     with pytest.warns(UserWarning, match=message):
-        vectorizer.fit_transform(vocabulary)
+        vectorizer.fit(vocabulary)
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", UserWarning)
+        vectorizer.transform(vocabulary)
+
+
+def test_tf_transformer_feature_names_out():
+    """Check get_feature_names_out for TfidfTransformer"""
+    X = [[1, 1, 1], [1, 1, 0], [1, 0, 0]]
+    tr = TfidfTransformer(smooth_idf=True, norm="l2").fit(X)
+
+    feature_names_in = ["a", "c", "b"]
+    feature_names_out = tr.get_feature_names_out(feature_names_in)
+    assert_array_equal(feature_names_in, feature_names_out)
 
 
 def test_tf_idf_smoothing():
-    X = [[1, 1, 1],
-         [1, 1, 0],
-         [1, 0, 0]]
-    tr = TfidfTransformer(smooth_idf=True, norm='l2')
+    X = [[1, 1, 1], [1, 1, 0], [1, 0, 0]]
+    tr = TfidfTransformer(smooth_idf=True, norm="l2")
     tfidf = tr.fit_transform(X).toarray()
     assert (tfidf >= 0).all()
 
     # check normalization
-    assert_array_almost_equal((tfidf ** 2).sum(axis=1), [1., 1., 1.])
+    assert_array_almost_equal((tfidf**2).sum(axis=1), [1.0, 1.0, 1.0])
 
     # this is robust to features with only zeros
-    X = [[1, 1, 0],
-         [1, 1, 0],
-         [1, 0, 0]]
-    tr = TfidfTransformer(smooth_idf=True, norm='l2')
+    X = [[1, 1, 0], [1, 1, 0], [1, 0, 0]]
+    tr = TfidfTransformer(smooth_idf=True, norm="l2")
     tfidf = tr.fit_transform(X).toarray()
     assert (tfidf >= 0).all()
 
 
+@pytest.mark.xfail(
+    _IS_WASM,
+    reason=(
+        "no floating point exceptions, see"
+        " https://github.com/numpy/numpy/pull/21895#issuecomment-1311525881"
+    ),
+)
 def test_tfidf_no_smoothing():
-    X = [[1, 1, 1],
-         [1, 1, 0],
-         [1, 0, 0]]
-    tr = TfidfTransformer(smooth_idf=False, norm='l2')
+    X = [[1, 1, 1], [1, 1, 0], [1, 0, 0]]
+    tr = TfidfTransformer(smooth_idf=False, norm="l2")
     tfidf = tr.fit_transform(X).toarray()
     assert (tfidf >= 0).all()
 
     # check normalization
-    assert_array_almost_equal((tfidf ** 2).sum(axis=1), [1., 1., 1.])
+    assert_array_almost_equal((tfidf**2).sum(axis=1), [1.0, 1.0, 1.0])
 
     # the lack of smoothing make IDF fragile in the presence of feature with
     # only zeros
-    X = [[1, 1, 0],
-         [1, 1, 0],
-         [1, 0, 0]]
-    tr = TfidfTransformer(smooth_idf=False, norm='l2')
+    X = [[1, 1, 0], [1, 1, 0], [1, 0, 0]]
+    tr = TfidfTransformer(smooth_idf=False, norm="l2")
 
-    in_warning_message = 'divide by zero'
+    in_warning_message = "divide by zero"
     with pytest.warns(RuntimeWarning, match=in_warning_message):
         tr.fit_transform(X).toarray()
 
@@ -452,7 +520,7 @@ def test_vectorizer():
     # test without vocabulary
     v1 = CountVectorizer(max_df=0.5)
     counts_train = v1.fit_transform(train_data)
-    if hasattr(counts_train, 'tocsr'):
+    if hasattr(counts_train, "tocsr"):
         counts_train = counts_train.tocsr()
     assert counts_train[0, v1.vocabulary_["pizza"]] == 2
 
@@ -462,7 +530,7 @@ def test_vectorizer():
     # compare that the two vectorizer give the same output on the test sample
     for v in (v1, v2):
         counts_test = v.transform(test_data)
-        if hasattr(counts_test, 'tocsr'):
+        if hasattr(counts_test, "tocsr"):
             counts_test = counts_test.tocsr()
 
         vocabulary = v.vocabulary_
@@ -486,7 +554,7 @@ def test_vectorizer():
         assert counts_test[0, vocabulary["pizza"]] == 0
 
     # test tf-idf
-    t1 = TfidfTransformer(norm='l1')
+    t1 = TfidfTransformer(norm="l1")
     tfidf = t1.fit(counts_train).transform(counts_train).toarray()
     assert len(t1.idf_) == len(v1.vocabulary_)
     assert tfidf.shape == (n_train, len(v1.vocabulary_))
@@ -496,7 +564,7 @@ def test_vectorizer():
     assert tfidf_test.shape == (len(test_data), len(v1.vocabulary_))
 
     # test tf alone
-    t2 = TfidfTransformer(norm='l1', use_idf=False)
+    t2 = TfidfTransformer(norm="l1", use_idf=False)
     tf = t2.fit(counts_train).transform(counts_train).toarray()
     assert not hasattr(t2, "idf_")
 
@@ -505,22 +573,13 @@ def test_vectorizer():
     with pytest.raises(ValueError):
         t3.transform(counts_train)
 
-    # test idf transform with incompatible n_features
-    X = [[1, 1, 5],
-         [1, 1, 0]]
-    t3.fit(X)
-    X_incompt = [[1, 3],
-                 [1, 3]]
-    with pytest.raises(ValueError):
-        t3.transform(X_incompt)
-
     # L1-normalized term frequencies sum to one
     assert_array_almost_equal(np.sum(tf, axis=1), [1.0] * n_train)
 
     # test the direct tfidf vectorizer
     # (equivalent to term count vectorizer + tfidf transformer)
     train_data = iter(ALL_FOOD_DOCS[:-1])
-    tv = TfidfVectorizer(norm='l1')
+    tv = TfidfVectorizer(norm="l1")
 
     tv.max_df = v1.max_df
     tfidf2 = tv.fit_transform(train_data).toarray()
@@ -537,39 +596,53 @@ def test_vectorizer():
         v3.transform(train_data)
 
     # ascii preprocessor?
-    v3.set_params(strip_accents='ascii', lowercase=False)
+    v3.set_params(strip_accents="ascii", lowercase=False)
     processor = v3.build_preprocessor()
-    text = ("J'ai mangÊ du kangourou  ce midi, "
-            "c'Êtait pas très bon.")
+    text = "J'ai mangÊ du kangourou  ce midi, c'Êtait pas très bon."
     expected = strip_accents_ascii(text)
     result = processor(text)
     assert expected == result
 
     # error on bad strip_accents param
-    v3.set_params(strip_accents='_gabbledegook_', preprocessor=None)
+    v3.set_params(strip_accents="_gabbledegook_", preprocessor=None)
     with pytest.raises(ValueError):
         v3.build_preprocessor()
 
     # error with bad analyzer type
-    v3.set_params = '_invalid_analyzer_type_'
+    v3.set_params = "_invalid_analyzer_type_"
     with pytest.raises(ValueError):
         v3.build_analyzer()
 
 
 def test_tfidf_vectorizer_setters():
-    tv = TfidfVectorizer(norm='l2', use_idf=False, smooth_idf=False,
-                         sublinear_tf=False)
-    tv.norm = 'l1'
-    assert tv._tfidf.norm == 'l1'
+    norm, use_idf, smooth_idf, sublinear_tf = "l2", False, False, False
+    tv = TfidfVectorizer(
+        norm=norm, use_idf=use_idf, smooth_idf=smooth_idf, sublinear_tf=sublinear_tf
+    )
+    tv.fit(JUNK_FOOD_DOCS)
+    assert tv._tfidf.norm == norm
+    assert tv._tfidf.use_idf == use_idf
+    assert tv._tfidf.smooth_idf == smooth_idf
+    assert tv._tfidf.sublinear_tf == sublinear_tf
+
+    # assigning value to `TfidfTransformer` should not have any effect until
+    # fitting
+    tv.norm = "l1"
     tv.use_idf = True
-    assert tv._tfidf.use_idf
     tv.smooth_idf = True
-    assert tv._tfidf.smooth_idf
     tv.sublinear_tf = True
-    assert tv._tfidf.sublinear_tf
+    assert tv._tfidf.norm == norm
+    assert tv._tfidf.use_idf == use_idf
+    assert tv._tfidf.smooth_idf == smooth_idf
+    assert tv._tfidf.sublinear_tf == sublinear_tf
+
+    tv.fit(JUNK_FOOD_DOCS)
+    assert tv._tfidf.norm == tv.norm
+    assert tv._tfidf.use_idf == tv.use_idf
+    assert tv._tfidf.smooth_idf == tv.smooth_idf
+    assert tv._tfidf.sublinear_tf == tv.sublinear_tf
 
 
-@fails_if_pypy
 def test_hashing_vectorizer():
     v = HashingVectorizer()
     X = v.transform(ALL_FOOD_DOCS)
@@ -589,7 +662,7 @@ def test_hashing_vectorizer():
         assert_almost_equal(np.linalg.norm(X[0].data, 2), 1.0)
 
     # Check vectorization with some non-default parameters
-    v = HashingVectorizer(ngram_range=(1, 2), norm='l1')
+    v = HashingVectorizer(ngram_range=(1, 2), norm="l1")
     X = v.transform(ALL_FOOD_DOCS)
     assert X.shape == (len(ALL_FOOD_DOCS), v.n_features)
     assert X.dtype == v.dtype
@@ -613,7 +686,7 @@ def test_feature_names():
 
     # test for Value error on unfitted/empty vocabulary
     with pytest.raises(ValueError):
-        cv.get_feature_names()
+        cv.get_feature_names_out()
     assert not cv.fixed_vocabulary_
 
     # test for vocabulary learned from data
@@ -621,40 +694,72 @@ def test_feature_names():
     n_samples, n_features = X.shape
     assert len(cv.vocabulary_) == n_features
 
-    feature_names = cv.get_feature_names()
+    feature_names = cv.get_feature_names_out()
+    assert isinstance(feature_names, np.ndarray)
+    assert feature_names.dtype == object
+
     assert len(feature_names) == n_features
-    assert_array_equal(['beer', 'burger', 'celeri', 'coke', 'pizza',
-                        'salad', 'sparkling', 'tomato', 'water'],
-                       feature_names)
+    assert_array_equal(
+        [
+            "beer",
+            "burger",
+            "celeri",
+            "coke",
+            "pizza",
+            "salad",
+            "sparkling",
+            "tomato",
+            "water",
+        ],
+        feature_names,
+    )
 
     for idx, name in enumerate(feature_names):
         assert idx == cv.vocabulary_.get(name)
 
     # test for custom vocabulary
-    vocab = ['beer', 'burger', 'celeri', 'coke', 'pizza',
-             'salad', 'sparkling', 'tomato', 'water']
+    vocab = [
+        "beer",
+        "burger",
+        "celeri",
+        "coke",
+        "pizza",
+        "salad",
+        "sparkling",
+        "tomato",
+        "water",
+    ]
 
     cv = CountVectorizer(vocabulary=vocab)
-    feature_names = cv.get_feature_names()
-    assert_array_equal(['beer', 'burger', 'celeri', 'coke', 'pizza', 'salad',
-                        'sparkling', 'tomato', 'water'], feature_names)
+    feature_names = cv.get_feature_names_out()
+    assert_array_equal(
+        [
+            "beer",
+            "burger",
+            "celeri",
+            "coke",
+            "pizza",
+            "salad",
+            "sparkling",
+            "tomato",
+            "water",
+        ],
+        feature_names,
+    )
     assert cv.fixed_vocabulary_
 
     for idx, name in enumerate(feature_names):
         assert idx == cv.vocabulary_.get(name)
 
 
-@pytest.mark.parametrize('Vectorizer', (CountVectorizer, TfidfVectorizer))
+@pytest.mark.parametrize("Vectorizer", (CountVectorizer, TfidfVectorizer))
 def test_vectorizer_max_features(Vectorizer):
-    expected_vocabulary = {'burger', 'beer', 'salad', 'pizza'}
-    expected_stop_words = {'celeri', 'tomato', 'copyright', 'coke',
-                           'sparkling', 'water', 'the'}
+    expected_vocabulary = {"burger", "beer", "salad", "pizza"}
 
     # test bounded number of extracted features
     vectorizer = Vectorizer(max_df=0.6, max_features=4)
     vectorizer.fit(ALL_FOOD_DOCS)
     assert set(vectorizer.vocabulary_) == expected_vocabulary
-    assert vectorizer.stop_words_ == expected_stop_words
 
 
 def test_count_vectorizer_max_features():
@@ -668,9 +773,9 @@ def test_count_vectorizer_max_features():
     counts_3 = cv_3.fit_transform(JUNK_FOOD_DOCS).sum(axis=0)
     counts_None = cv_None.fit_transform(JUNK_FOOD_DOCS).sum(axis=0)
 
-    features_1 = cv_1.get_feature_names()
-    features_3 = cv_3.get_feature_names()
-    features_None = cv_None.get_feature_names()
+    features_1 = cv_1.get_feature_names_out()
+    features_3 = cv_3.get_feature_names_out()
+    features_None = cv_None.get_feature_names_out()
 
     # The most common feature is "the", with frequency 7.
     assert 7 == counts_1.max()
@@ -684,79 +789,65 @@ def test_count_vectorizer_max_features():
 
 
 def test_vectorizer_max_df():
-    test_data = ['abc', 'dea', 'eat']
-    vect = CountVectorizer(analyzer='char', max_df=1.0)
+    test_data = ["abc", "dea", "eat"]
+    vect = CountVectorizer(analyzer="char", max_df=1.0)
     vect.fit(test_data)
-    assert 'a' in vect.vocabulary_.keys()
+    assert "a" in vect.vocabulary_.keys()
     assert len(vect.vocabulary_.keys()) == 6
-    assert len(vect.stop_words_) == 0
 
     vect.max_df = 0.5  # 0.5 * 3 documents -> max_doc_count == 1.5
     vect.fit(test_data)
-    assert 'a' not in vect.vocabulary_.keys()  # {ae} ignored
-    assert len(vect.vocabulary_.keys()) == 4    # {bcdt} remain
-    assert 'a' in vect.stop_words_
-    assert len(vect.stop_words_) == 2
+    assert "a" not in vect.vocabulary_.keys()  # {ae} ignored
+    assert len(vect.vocabulary_.keys()) == 4  # {bcdt} remain
 
     vect.max_df = 1
     vect.fit(test_data)
-    assert 'a' not in vect.vocabulary_.keys()  # {ae} ignored
-    assert len(vect.vocabulary_.keys()) == 4    # {bcdt} remain
-    assert 'a' in vect.stop_words_
-    assert len(vect.stop_words_) == 2
+    assert "a" not in vect.vocabulary_.keys()  # {ae} ignored
+    assert len(vect.vocabulary_.keys()) == 4  # {bcdt} remain
 
 
 def test_vectorizer_min_df():
-    test_data = ['abc', 'dea', 'eat']
-    vect = CountVectorizer(analyzer='char', min_df=1)
+    test_data = ["abc", "dea", "eat"]
+    vect = CountVectorizer(analyzer="char", min_df=1)
     vect.fit(test_data)
-    assert 'a' in vect.vocabulary_.keys()
+    assert "a" in vect.vocabulary_.keys()
     assert len(vect.vocabulary_.keys()) == 6
-    assert len(vect.stop_words_) == 0
 
     vect.min_df = 2
     vect.fit(test_data)
-    assert 'c' not in vect.vocabulary_.keys()  # {bcdt} ignored
-    assert len(vect.vocabulary_.keys()) == 2    # {ae} remain
-    assert 'c' in vect.stop_words_
-    assert len(vect.stop_words_) == 4
+    assert "c" not in vect.vocabulary_.keys()  # {bcdt} ignored
+    assert len(vect.vocabulary_.keys()) == 2  # {ae} remain
 
     vect.min_df = 0.8  # 0.8 * 3 documents -> min_doc_count == 2.4
     vect.fit(test_data)
-    assert 'c' not in vect.vocabulary_.keys()  # {bcdet} ignored
-    assert len(vect.vocabulary_.keys()) == 1    # {a} remains
-    assert 'c' in vect.stop_words_
-    assert len(vect.stop_words_) == 5
+    assert "c" not in vect.vocabulary_.keys()  # {bcdet} ignored
+    assert len(vect.vocabulary_.keys()) == 1  # {a} remains
 
 
 def test_count_binary_occurrences():
     # by default multiple occurrences are counted as longs
-    test_data = ['aaabc', 'abbde']
-    vect = CountVectorizer(analyzer='char', max_df=1.0)
+    test_data = ["aaabc", "abbde"]
+    vect = CountVectorizer(analyzer="char", max_df=1.0)
     X = vect.fit_transform(test_data).toarray()
-    assert_array_equal(['a', 'b', 'c', 'd', 'e'], vect.get_feature_names())
-    assert_array_equal([[3, 1, 1, 0, 0],
-                        [1, 2, 0, 1, 1]], X)
+    assert_array_equal(["a", "b", "c", "d", "e"], vect.get_feature_names_out())
+    assert_array_equal([[3, 1, 1, 0, 0], [1, 2, 0, 1, 1]], X)
 
     # using boolean features, we can fetch the binary occurrence info
     # instead.
-    vect = CountVectorizer(analyzer='char', max_df=1.0, binary=True)
+    vect = CountVectorizer(analyzer="char", max_df=1.0, binary=True)
     X = vect.fit_transform(test_data).toarray()
-    assert_array_equal([[1, 1, 1, 0, 0],
-                        [1, 1, 0, 1, 1]], X)
+    assert_array_equal([[1, 1, 1, 0, 0], [1, 1, 0, 1, 1]], X)
 
     # check the ability to change the dtype
-    vect = CountVectorizer(analyzer='char', max_df=1.0,
-                           binary=True, dtype=np.float32)
+    vect = CountVectorizer(analyzer="char", max_df=1.0, binary=True, dtype=np.float32)
     X_sparse = vect.fit_transform(test_data)
     assert X_sparse.dtype == np.float32
 
 
-@fails_if_pypy
 def test_hashed_binary_occurrences():
     # by default multiple occurrences are counted as longs
-    test_data = ['aaabc', 'abbde']
-    vect = HashingVectorizer(alternate_sign=False, analyzer='char', norm=None)
+    test_data = ["aaabc", "abbde"]
+    vect = HashingVectorizer(alternate_sign=False, analyzer="char", norm=None)
     X = vect.transform(test_data)
     assert np.max(X[0:1].data) == 3
     assert np.max(X[1:2].data) == 2
@@ -764,20 +855,22 @@ def test_hashed_binary_occurrences():
 
     # using boolean features, we can fetch the binary occurrence info
     # instead.
-    vect = HashingVectorizer(analyzer='char', alternate_sign=False,
-                             binary=True, norm=None)
+    vect = HashingVectorizer(
+        analyzer="char", alternate_sign=False, binary=True, norm=None
+    )
     X = vect.transform(test_data)
     assert np.max(X.data) == 1
     assert X.dtype == np.float64
 
     # check the ability to change the dtype
-    vect = HashingVectorizer(analyzer='char', alternate_sign=False,
-                             binary=True, norm=None, dtype=np.float64)
+    vect = HashingVectorizer(
+        analyzer="char", alternate_sign=False, binary=True, norm=None, dtype=np.float64
+    )
     X = vect.transform(test_data)
     assert X.dtype == np.float64
 
 
-@pytest.mark.parametrize('Vectorizer', (CountVectorizer, TfidfVectorizer))
+@pytest.mark.parametrize("Vectorizer", (CountVectorizer, TfidfVectorizer))
 def test_vectorizer_inverse_transform(Vectorizer):
     # raw documents
     data = ALL_FOOD_DOCS
@@ -818,14 +911,14 @@ def test_count_vectorizer_pipeline_grid_selection():
 
     # split the dataset for model development and final evaluation
     train_data, test_data, target_train, target_test = train_test_split(
-        data, target, test_size=.2, random_state=0)
+        data, target, test_size=0.2, random_state=0
+    )
 
-    pipeline = Pipeline([('vect', CountVectorizer()),
-                         ('svc', LinearSVC())])
+    pipeline = Pipeline([("vect", CountVectorizer()), ("svc", LinearSVC())])
 
     parameters = {
-        'vect__ngram_range': [(1, 1), (1, 2)],
-        'svc__loss': ('hinge', 'squared_hinge')
+        "vect__ngram_range": [(1, 1), (1, 2)],
+        "svc__loss": ("hinge", "squared_hinge"),
     }
 
     # find the best parameters for both the feature extraction and the
@@ -841,7 +934,7 @@ def test_count_vectorizer_pipeline_grid_selection():
     # the grid_search is considered the best estimator since they all converge
     # to 100% accuracy models
     assert grid_search.best_score_ == 1.0
-    best_vectorizer = grid_search.best_estimator_.named_steps['vect']
+    best_vectorizer = grid_search.best_estimator_.named_steps["vect"]
     assert best_vectorizer.ngram_range == (1, 1)
 
 
@@ -854,15 +947,15 @@ def test_vectorizer_pipeline_grid_selection():
 
     # split the dataset for model development and final evaluation
     train_data, test_data, target_train, target_test = train_test_split(
-        data, target, test_size=.1, random_state=0)
+        data, target, test_size=0.1, random_state=0
+    )
 
-    pipeline = Pipeline([('vect', TfidfVectorizer()),
-                         ('svc', LinearSVC())])
+    pipeline = Pipeline([("vect", TfidfVectorizer()), ("svc", LinearSVC())])
 
     parameters = {
-        'vect__ngram_range': [(1, 1), (1, 2)],
-        'vect__norm': ('l1', 'l2'),
-        'svc__loss': ('hinge', 'squared_hinge'),
+        "vect__ngram_range": [(1, 1), (1, 2)],
+        "vect__norm": ("l1", "l2"),
+        "svc__loss": ("hinge", "squared_hinge"),
     }
 
     # find the best parameters for both the feature extraction and the
@@ -878,9 +971,9 @@ def test_vectorizer_pipeline_grid_selection():
     # the grid_search is considered the best estimator since they all converge
     # to 100% accuracy models
     assert grid_search.best_score_ == 1.0
-    best_vectorizer = grid_search.best_estimator_.named_steps['vect']
+    best_vectorizer = grid_search.best_estimator_.named_steps["vect"]
     assert best_vectorizer.ngram_range == (1, 1)
-    assert best_vectorizer.norm == 'l2'
+    assert best_vectorizer.norm == "l2"
     assert not best_vectorizer.fixed_vocabulary_
 
 
@@ -891,21 +984,19 @@ def test_vectorizer_pipeline_cross_validation():
     # label junk food as -1, the others as +1
     target = [-1] * len(JUNK_FOOD_DOCS) + [1] * len(NOTJUNK_FOOD_DOCS)
 
-    pipeline = Pipeline([('vect', TfidfVectorizer()),
-                         ('svc', LinearSVC())])
+    pipeline = Pipeline([("vect", TfidfVectorizer()), ("svc", LinearSVC())])
 
     cv_scores = cross_val_score(pipeline, data, target, cv=3)
-    assert_array_equal(cv_scores, [1., 1., 1.])
+    assert_array_equal(cv_scores, [1.0, 1.0, 1.0])
 
 
-@fails_if_pypy
 def test_vectorizer_unicode():
     # tests that the count vectorizer works with cyrillic.
     document = (
         "МаŅˆĐ¸ĐŊĐŊĐžĐĩ ОйŅƒŅ‡ĐĩĐŊиĐĩ — ОйŅˆĐ¸Ņ€ĐŊŅ‹Đš ĐŋОдŅ€Đ°ĐˇĐ´ĐĩĐģ иŅĐēŅƒŅŅŅ‚вĐĩĐŊĐŊĐžĐŗĐž "
         "иĐŊŅ‚ĐĩĐģĐģĐĩĐēŅ‚Đ°, иСŅƒŅ‡Đ°ŅŽŅ‰Đ¸Đš ĐŧĐĩŅ‚ОдŅ‹ ĐŋĐžŅŅ‚Ņ€ĐžĐĩĐŊиŅ Đ°ĐģĐŗĐžŅ€Đ¸Ņ‚ĐŧОв, "
         "ŅĐŋĐžŅĐžĐąĐŊŅ‹Ņ… ОйŅƒŅ‡Đ°Ņ‚ŅŒŅŅ."
-        )
+    )
 
     vect = CountVectorizer()
     X_counted = vect.fit_transform([document])
@@ -913,7 +1004,7 @@ def test_vectorizer_unicode():
 
     vect = HashingVectorizer(norm=None, alternate_sign=False)
     X_hashed = vect.transform([document])
-    assert X_hashed.shape == (1, 2 ** 20)
+    assert X_hashed.shape == (1, 2**20)
 
     # No collisions on such a small dataset
     assert X_counted.nnz == X_hashed.nnz
@@ -925,7 +1016,7 @@ def test_vectorizer_unicode():
 
 def test_tfidf_vectorizer_with_fixed_vocabulary():
     # non regression smoke test for inheritance issues
-    vocabulary = ['pizza', 'celeri']
+    vocabulary = ["pizza", "celeri"]
     vect = TfidfVectorizer(vocabulary=vocabulary)
     X_1 = vect.fit_transform(ALL_FOOD_DOCS)
     X_2 = vect.transform(ALL_FOOD_DOCS)
@@ -936,7 +1027,7 @@ def test_tfidf_vectorizer_with_fixed_vocabulary():
 def test_pickling_vectorizer():
     instances = [
         HashingVectorizer(),
-        HashingVectorizer(norm='l1'),
+        HashingVectorizer(norm="l1"),
         HashingVectorizer(binary=True),
         HashingVectorizer(ngram_range=(1, 2)),
         CountVectorizer(),
@@ -954,27 +1045,27 @@ def test_pickling_vectorizer():
         copy = pickle.loads(s)
         assert type(copy) == orig.__class__
         assert copy.get_params() == orig.get_params()
-        if IS_PYPY and isinstance(orig, HashingVectorizer):
-            continue
-        else:
-            assert_array_equal(
-                copy.fit_transform(JUNK_FOOD_DOCS).toarray(),
-                orig.fit_transform(JUNK_FOOD_DOCS).toarray())
+        assert_allclose_dense_sparse(
+            copy.fit_transform(JUNK_FOOD_DOCS),
+            orig.fit_transform(JUNK_FOOD_DOCS),
+        )
 
 
-@pytest.mark.parametrize('factory', [
-    CountVectorizer.build_analyzer,
-    CountVectorizer.build_preprocessor,
-    CountVectorizer.build_tokenizer,
-])
+@pytest.mark.parametrize(
+    "factory",
+    [
+        CountVectorizer.build_analyzer,
+        CountVectorizer.build_preprocessor,
+        CountVectorizer.build_tokenizer,
+    ],
+)
 def test_pickling_built_processors(factory):
     """Tokenizers cannot be pickled
     https://github.com/scikit-learn/scikit-learn/issues/12833
     """
     vec = CountVectorizer()
     function = factory(vec)
-    text = ("J'ai mangÊ du kangourou  ce midi, "
-            "c'Êtait pas très bon.")
+    text = "J'ai mangÊ du kangourou  ce midi, c'Êtait pas très bon."
     roundtripped_function = pickle.loads(pickle.dumps(function))
     expected = function(text)
     result = roundtripped_function(text)
@@ -985,21 +1076,45 @@ def test_countvectorizer_vocab_sets_when_pickling():
     # ensure that vocabulary of type set is coerced to a list to
     # preserve iteration ordering after deserialization
     rng = np.random.RandomState(0)
-    vocab_words = np.array(['beer', 'burger', 'celeri', 'coke', 'pizza',
-                            'salad', 'sparkling', 'tomato', 'water'])
+    vocab_words = np.array(
+        [
+            "beer",
+            "burger",
+            "celeri",
+            "coke",
+            "pizza",
+            "salad",
+            "sparkling",
+            "tomato",
+            "water",
+        ]
+    )
     for x in range(0, 100):
         vocab_set = set(rng.choice(vocab_words, size=5, replace=False))
         cv = CountVectorizer(vocabulary=vocab_set)
         unpickled_cv = pickle.loads(pickle.dumps(cv))
         cv.fit(ALL_FOOD_DOCS)
         unpickled_cv.fit(ALL_FOOD_DOCS)
-        assert cv.get_feature_names() == unpickled_cv.get_feature_names()
+        assert_array_equal(
+            cv.get_feature_names_out(), unpickled_cv.get_feature_names_out()
+        )
 
 
 def test_countvectorizer_vocab_dicts_when_pickling():
     rng = np.random.RandomState(0)
-    vocab_words = np.array(['beer', 'burger', 'celeri', 'coke', 'pizza',
-                            'salad', 'sparkling', 'tomato', 'water'])
+    vocab_words = np.array(
+        [
+            "beer",
+            "burger",
+            "celeri",
+            "coke",
+            "pizza",
+            "salad",
+            "sparkling",
+            "tomato",
+            "water",
+        ]
+    )
     for x in range(0, 100):
         vocab_dict = dict()
         words = rng.choice(vocab_words, size=5, replace=False)
@@ -1009,29 +1124,9 @@ def test_countvectorizer_vocab_dicts_when_pickling():
         unpickled_cv = pickle.loads(pickle.dumps(cv))
         cv.fit(ALL_FOOD_DOCS)
         unpickled_cv.fit(ALL_FOOD_DOCS)
-        assert cv.get_feature_names() == unpickled_cv.get_feature_names()
-
-
-def test_stop_words_removal():
-    # Ensure that deleting the stop_words_ attribute doesn't affect transform
-
-    fitted_vectorizers = (
-        TfidfVectorizer().fit(JUNK_FOOD_DOCS),
-        CountVectorizer(preprocessor=strip_tags).fit(JUNK_FOOD_DOCS),
-        CountVectorizer(strip_accents=strip_eacute).fit(JUNK_FOOD_DOCS)
-    )
-
-    for vect in fitted_vectorizers:
-        vect_transform = vect.transform(JUNK_FOOD_DOCS).toarray()
-
-        vect.stop_words_ = None
-        stop_None_transform = vect.transform(JUNK_FOOD_DOCS).toarray()
-
-        delattr(vect, 'stop_words_')
-        stop_del_transform = vect.transform(JUNK_FOOD_DOCS).toarray()
-
-        assert_array_equal(stop_None_transform, vect_transform)
-        assert_array_equal(stop_del_transform, vect_transform)
+        assert_array_equal(
+            cv.get_feature_names_out(), unpickled_cv.get_feature_names_out()
+        )
 
 
 def test_pickling_transformer():
@@ -1040,9 +1135,7 @@ def test_pickling_transformer():
     s = pickle.dumps(orig)
     copy = pickle.loads(s)
     assert type(copy) == orig.__class__
-    assert_array_equal(
-        copy.fit_transform(X).toarray(),
-        orig.fit_transform(X).toarray())
+    assert_array_equal(copy.fit_transform(X).toarray(), orig.fit_transform(X).toarray())
 
 
 def test_transformer_idf_setter():
@@ -1050,9 +1143,7 @@ def test_transformer_idf_setter():
     orig = TfidfTransformer().fit(X)
     copy = TfidfTransformer()
     copy.idf_ = orig.idf_
-    assert_array_equal(
-        copy.transform(X).toarray(),
-        orig.transform(X).toarray())
+    assert_array_equal(copy.transform(X).toarray(), orig.transform(X).toarray())
 
 
 def test_tfidf_vectorizer_setter():
@@ -1062,7 +1153,13 @@ def test_tfidf_vectorizer_setter():
     copy.idf_ = orig.idf_
     assert_array_equal(
         copy.transform(JUNK_FOOD_DOCS).toarray(),
-        orig.transform(JUNK_FOOD_DOCS).toarray())
+        orig.transform(JUNK_FOOD_DOCS).toarray(),
+    )
+    # `idf_` cannot be set with `use_idf=False`
+    copy = TfidfVectorizer(vocabulary=orig.vocabulary_, use_idf=False)
+    err_msg = "`idf_` cannot be set when `user_idf=False`."
+    with pytest.raises(ValueError, match=err_msg):
+        copy.idf_ = orig.idf_
 
 
 def test_tfidfvectorizer_invalid_idf_attr():
@@ -1072,17 +1169,16 @@ def test_tfidfvectorizer_invalid_idf_attr():
     expected_idf_len = len(vect.idf_)
     invalid_idf = [1.0] * (expected_idf_len + 1)
     with pytest.raises(ValueError):
-        setattr(copy, 'idf_', invalid_idf)
+        setattr(copy, "idf_", invalid_idf)
 
 
 def test_non_unique_vocab():
-    vocab = ['a', 'b', 'c', 'a', 'a']
+    vocab = ["a", "b", "c", "a", "a"]
     vect = CountVectorizer(vocabulary=vocab)
     with pytest.raises(ValueError):
         vect.fit([])
 
 
-@fails_if_pypy
 def test_hashingvectorizer_nan_in_docs():
     # np.nan can appear when using pandas to load text fields from a csv file
     # with missing values.
@@ -1091,9 +1187,10 @@ def test_hashingvectorizer_nan_in_docs():
 
     def func():
         hv = HashingVectorizer()
-        hv.fit_transform(['hello world', np.nan, 'hello hello'])
+        hv.fit_transform(["hello world", np.nan, "hello hello"])
 
-    assert_raise_message(exception, message, func)
+    with pytest.raises(exception, match=message):
+        func()
 
 
 def test_tfidfvectorizer_binary():
@@ -1101,9 +1198,9 @@ def test_tfidfvectorizer_binary():
     v = TfidfVectorizer(binary=True, use_idf=False, norm=None)
     assert v.binary
 
-    X = v.fit_transform(['hello world', 'hello hello']).toarray()
+    X = v.fit_transform(["hello world", "hello hello"]).toarray()
     assert_array_equal(X.ravel(), [1, 1, 1, 0])
-    X2 = v.transform(['hello world', 'hello hello']).toarray()
+    X2 = v.transform(["hello world", "hello hello"]).toarray()
     assert_array_equal(X2.ravel(), [1, 1, 1, 0])
 
 
@@ -1121,17 +1218,22 @@ def test_vectorizer_vocab_clone():
     assert vect_vocab_clone.vocabulary_ == vect_vocab.vocabulary_
 
 
-@pytest.mark.parametrize('Vectorizer',
-                         (CountVectorizer, TfidfVectorizer, HashingVectorizer))
+@pytest.mark.parametrize(
+    "Vectorizer", (CountVectorizer, TfidfVectorizer, HashingVectorizer)
+)
 def test_vectorizer_string_object_as_input(Vectorizer):
-    message = ("Iterable over raw text documents expected, "
-               "string object received.")
+    message = "Iterable over raw text documents expected, string object received."
     vec = Vectorizer()
-    assert_raise_message(
-            ValueError, message, vec.fit_transform, "hello world!")
-    assert_raise_message(ValueError, message, vec.fit, "hello world!")
+
+    with pytest.raises(ValueError, match=message):
+        vec.fit_transform("hello world!")
+
+    with pytest.raises(ValueError, match=message):
+        vec.fit("hello world!")
     vec.fit(["some text", "some other text"])
-    assert_raise_message(ValueError, message, vec.transform, "hello world!")
+
+    with pytest.raises(ValueError, match=message):
+        vec.transform("hello world!")
 
 
 @pytest.mark.parametrize("X_dtype", [np.float32, np.float64])
@@ -1141,10 +1243,13 @@ def test_tfidf_transformer_type(X_dtype):
     assert X_trans.dtype == X.dtype
 
 
-def test_tfidf_transformer_sparse():
+@pytest.mark.parametrize(
+    "csc_container, csr_container", product(CSC_CONTAINERS, CSR_CONTAINERS)
+)
+def test_tfidf_transformer_sparse(csc_container, csr_container):
     X = sparse.rand(10, 20000, dtype=np.float64, random_state=42)
-    X_csc = sparse.csc_matrix(X)
-    X_csr = sparse.csr_matrix(X)
+    X_csc = csc_container(X)
+    X_csr = csr_container(X)
 
     X_trans_csc = TfidfTransformer().fit_transform(X_csc)
     X_trans_csr = TfidfTransformer().fit_transform(X_csr)
@@ -1154,91 +1259,93 @@ def test_tfidf_transformer_sparse():
 
 @pytest.mark.parametrize(
     "vectorizer_dtype, output_dtype, warning_expected",
-    [(np.int32, np.float64, True),
-     (np.int64, np.float64, True),
-     (np.float32, np.float32, False),
-     (np.float64, np.float64, False)]
+    [
+        (np.int32, np.float64, True),
+        (np.int64, np.float64, True),
+        (np.float32, np.float32, False),
+        (np.float64, np.float64, False),
+    ],
 )
-def test_tfidf_vectorizer_type(vectorizer_dtype, output_dtype,
-                               warning_expected):
+def test_tfidf_vectorizer_type(vectorizer_dtype, output_dtype, warning_expected):
     X = np.array(["numpy", "scipy", "sklearn"])
     vectorizer = TfidfVectorizer(dtype=vectorizer_dtype)
 
     warning_msg_match = "'dtype' should be used."
-    warning_cls = UserWarning
-    expected_warning_cls = warning_cls if warning_expected else None
-    with pytest.warns(expected_warning_cls,
-                      match=warning_msg_match) as record:
-        X_idf = vectorizer.fit_transform(X)
-    if expected_warning_cls is None:
-        relevant_warnings = [w for w in record
-                             if isinstance(w, warning_cls)]
-        assert len(relevant_warnings) == 0
+    if warning_expected:
+        with pytest.warns(UserWarning, match=warning_msg_match):
+            X_idf = vectorizer.fit_transform(X)
+    else:
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", UserWarning)
+            X_idf = vectorizer.fit_transform(X)
     assert X_idf.dtype == output_dtype
 
 
-@pytest.mark.parametrize("vec", [
+@pytest.mark.parametrize(
+    "vec",
+    [
         HashingVectorizer(ngram_range=(2, 1)),
         CountVectorizer(ngram_range=(2, 1)),
-        TfidfVectorizer(ngram_range=(2, 1))
-    ])
+        TfidfVectorizer(ngram_range=(2, 1)),
+    ],
+)
 def test_vectorizers_invalid_ngram_range(vec):
     # vectorizers could be initialized with invalid ngram range
     # test for raising error message
     invalid_range = vec.ngram_range
-    message = ("Invalid value for ngram_range=%s "
-               "lower boundary larger than the upper boundary."
-               % str(invalid_range))
-    if isinstance(vec, HashingVectorizer) and IS_PYPY:
-        pytest.xfail(reason='HashingVectorizer is not supported on PyPy')
+    message = re.escape(
+        f"Invalid value for ngram_range={invalid_range} "
+        "lower boundary larger than the upper boundary."
+    )
 
-    assert_raise_message(
-        ValueError, message, vec.fit, ["good news everyone"])
-    assert_raise_message(
-        ValueError, message, vec.fit_transform, ["good news everyone"])
+    with pytest.raises(ValueError, match=message):
+        vec.fit(["good news everyone"])
+
+    with pytest.raises(ValueError, match=message):
+        vec.fit_transform(["good news everyone"])
 
     if isinstance(vec, HashingVectorizer):
-        assert_raise_message(
-            ValueError, message, vec.transform, ["good news everyone"])
+        with pytest.raises(ValueError, match=message):
+            vec.transform(["good news everyone"])
 
 
 def _check_stop_words_consistency(estimator):
     stop_words = estimator.get_stop_words()
     tokenize = estimator.build_tokenizer()
     preprocess = estimator.build_preprocessor()
-    return estimator._check_stop_words_consistency(stop_words, preprocess,
-                                                   tokenize)
+    return estimator._check_stop_words_consistency(stop_words, preprocess, tokenize)
 
 
-@fails_if_pypy
 def test_vectorizer_stop_words_inconsistent():
     lstr = r"\['and', 'll', 've'\]"
-    message = ('Your stop_words may be inconsistent with your '
-               'preprocessing. Tokenizing the stop words generated '
-               'tokens %s not in stop_words.' % lstr)
-    for vec in [CountVectorizer(),
-                TfidfVectorizer(), HashingVectorizer()]:
-        vec.set_params(stop_words=["you've", "you", "you'll", 'AND'])
+    message = (
+        "Your stop_words may be inconsistent with your "
+        "preprocessing. Tokenizing the stop words generated "
+        "tokens %s not in stop_words." % lstr
+    )
+    for vec in [CountVectorizer(), TfidfVectorizer(), HashingVectorizer()]:
+        vec.set_params(stop_words=["you've", "you", "you'll", "AND"])
         with pytest.warns(UserWarning, match=message):
-            vec.fit_transform(['hello world'])
+            vec.fit_transform(["hello world"])
         # reset stop word validation
         del vec._stop_words_id
         assert _check_stop_words_consistency(vec) is False
 
     # Only one warning per stop list
-    with pytest.warns(None) as record:
-        vec.fit_transform(['hello world'])
-    assert not len(record)
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", UserWarning)
+        vec.fit_transform(["hello world"])
     assert _check_stop_words_consistency(vec) is None
 
     # Test caching of inconsistency assessment
-    vec.set_params(stop_words=["you've", "you", "you'll", 'blah', 'AND'])
+    vec.set_params(stop_words=["you've", "you", "you'll", "blah", "AND"])
     with pytest.warns(UserWarning, match=message):
-        vec.fit_transform(['hello world'])
+        vec.fit_transform(["hello world"])
 
 
 @skip_if_32bit
-def test_countvectorizer_sort_features_64bit_sparse_indices():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_countvectorizer_sort_features_64bit_sparse_indices(csr_container):
     """
     Check that CountVectorizer._sort_features preserves the dtype of its sparse
     feature matrix.
@@ -1248,173 +1355,274 @@ def test_countvectorizer_sort_features_64bit_sparse_indices():
     for more details.
     """
 
-    X = sparse.csr_matrix((5, 5), dtype=np.int64)
+    X = csr_container((5, 5), dtype=np.int64)
 
     # force indices and indptr to int64.
     INDICES_DTYPE = np.int64
     X.indices = X.indices.astype(INDICES_DTYPE)
     X.indptr = X.indptr.astype(INDICES_DTYPE)
 
-    vocabulary = {
-            "scikit-learn": 0,
-            "is": 1,
-            "great!": 2
-            }
+    vocabulary = {"scikit-learn": 0, "is": 1, "great!": 2}
 
     Xs = CountVectorizer()._sort_features(X, vocabulary)
 
     assert INDICES_DTYPE == Xs.indices.dtype
 
 
-@fails_if_pypy
-@pytest.mark.parametrize('Estimator',
-                         [CountVectorizer, TfidfVectorizer, HashingVectorizer])
+@pytest.mark.parametrize(
+    "Estimator", [CountVectorizer, TfidfVectorizer, HashingVectorizer]
+)
 def test_stop_word_validation_custom_preprocessor(Estimator):
-    data = [{'text': 'some text'}]
+    data = [{"text": "some text"}]
 
     vec = Estimator()
     assert _check_stop_words_consistency(vec) is True
 
-    vec = Estimator(preprocessor=lambda x: x['text'],
-                    stop_words=['and'])
-    assert _check_stop_words_consistency(vec) == 'error'
+    vec = Estimator(preprocessor=lambda x: x["text"], stop_words=["and"])
+    assert _check_stop_words_consistency(vec) == "error"
     # checks are cached
     assert _check_stop_words_consistency(vec) is None
     vec.fit_transform(data)
 
     class CustomEstimator(Estimator):
         def build_preprocessor(self):
-            return lambda x: x['text']
+            return lambda x: x["text"]
 
-    vec = CustomEstimator(stop_words=['and'])
-    assert _check_stop_words_consistency(vec) == 'error'
+    vec = CustomEstimator(stop_words=["and"])
+    assert _check_stop_words_consistency(vec) == "error"
 
-    vec = Estimator(tokenizer=lambda doc: re.compile(r'\w{1,}')
-                                            .findall(doc),
-                    stop_words=['and'])
+    vec = Estimator(
+        tokenizer=lambda doc: re.compile(r"\w{1,}").findall(doc), stop_words=["and"]
+    )
     assert _check_stop_words_consistency(vec) is True
 
 
 @pytest.mark.parametrize(
-    'Estimator',
-    [CountVectorizer,
-     TfidfVectorizer,
-     HashingVectorizer]
+    "Estimator", [CountVectorizer, TfidfVectorizer, HashingVectorizer]
 )
 @pytest.mark.parametrize(
-    'input_type, err_type, err_msg',
-    [('filename', FileNotFoundError, ''),
-     ('file', AttributeError, "'str' object has no attribute 'read'")]
+    "input_type, err_type, err_msg",
+    [
+        ("filename", FileNotFoundError, ""),
+        ("file", AttributeError, "'str' object has no attribute 'read'"),
+    ],
 )
 def test_callable_analyzer_error(Estimator, input_type, err_type, err_msg):
-    if issubclass(Estimator, HashingVectorizer):
-        pytest.xfail('HashingVectorizer is not supported on PyPy')
-    data = ['this is text, not file or filename']
+    data = ["this is text, not file or filename"]
     with pytest.raises(err_type, match=err_msg):
-        Estimator(analyzer=lambda x: x.split(),
-                  input=input_type).fit_transform(data)
+        Estimator(analyzer=lambda x: x.split(), input=input_type).fit_transform(data)
 
 
 @pytest.mark.parametrize(
-    'Estimator',
-    [CountVectorizer,
-     TfidfVectorizer,
-     pytest.param(HashingVectorizer, marks=fails_if_pypy)]
+    "Estimator",
+    [
+        CountVectorizer,
+        TfidfVectorizer,
+        pytest.param(HashingVectorizer),
+    ],
 )
 @pytest.mark.parametrize(
-    'analyzer', [lambda doc: open(doc, 'r'), lambda doc: doc.read()]
+    "analyzer", [lambda doc: open(doc, "r"), lambda doc: doc.read()]
 )
-@pytest.mark.parametrize('input_type', ['file', 'filename'])
+@pytest.mark.parametrize("input_type", ["file", "filename"])
 def test_callable_analyzer_change_behavior(Estimator, analyzer, input_type):
-    data = ['this is text, not file or filename']
+    data = ["this is text, not file or filename"]
     with pytest.raises((FileNotFoundError, AttributeError)):
         Estimator(analyzer=analyzer, input=input_type).fit_transform(data)
 
 
 @pytest.mark.parametrize(
-    'Estimator',
-    [CountVectorizer,
-     TfidfVectorizer,
-     HashingVectorizer]
+    "Estimator", [CountVectorizer, TfidfVectorizer, HashingVectorizer]
 )
 def test_callable_analyzer_reraise_error(tmpdir, Estimator):
     # check if a custom exception from the analyzer is shown to the user
     def analyzer(doc):
         raise Exception("testing")
 
-    if issubclass(Estimator, HashingVectorizer):
-        pytest.xfail('HashingVectorizer is not supported on PyPy')
-
     f = tmpdir.join("file.txt")
     f.write("sample content\n")
 
     with pytest.raises(Exception, match="testing"):
-        Estimator(analyzer=analyzer, input='file').fit_transform([f])
+        Estimator(analyzer=analyzer, input="file").fit_transform([f])
 
 
 @pytest.mark.parametrize(
-    'Vectorizer',
-    [CountVectorizer, HashingVectorizer, TfidfVectorizer]
+    "Vectorizer", [CountVectorizer, HashingVectorizer, TfidfVectorizer]
 )
 @pytest.mark.parametrize(
-    'stop_words, tokenizer, preprocessor, ngram_range, token_pattern,'
-    'analyzer, unused_name, ovrd_name, ovrd_msg',
-    [(["you've", "you'll"], None, None, (1, 1), None, 'char',
-     "'stop_words'", "'analyzer'", "!= 'word'"),
-     (None, lambda s: s.split(), None, (1, 1), None, 'char',
-     "'tokenizer'", "'analyzer'", "!= 'word'"),
-     (None, lambda s: s.split(), None, (1, 1), r'\w+', 'word',
-      "'token_pattern'", "'tokenizer'", "is not None"),
-     (None, None, lambda s:s.upper(), (1, 1), r'\w+', lambda s:s.upper(),
-      "'preprocessor'", "'analyzer'", "is callable"),
-     (None, None, None, (1, 2), None, lambda s:s.upper(),
-      "'ngram_range'", "'analyzer'", "is callable"),
-     (None, None, None, (1, 1), r'\w+', 'char',
-      "'token_pattern'", "'analyzer'", "!= 'word'")]
+    (
+        "stop_words, tokenizer, preprocessor, ngram_range, token_pattern,"
+        "analyzer, unused_name, ovrd_name, ovrd_msg"
+    ),
+    [
+        (
+            ["you've", "you'll"],
+            None,
+            None,
+            (1, 1),
+            None,
+            "char",
+            "'stop_words'",
+            "'analyzer'",
+            "!= 'word'",
+        ),
+        (
+            None,
+            lambda s: s.split(),
+            None,
+            (1, 1),
+            None,
+            "char",
+            "'tokenizer'",
+            "'analyzer'",
+            "!= 'word'",
+        ),
+        (
+            None,
+            lambda s: s.split(),
+            None,
+            (1, 1),
+            r"\w+",
+            "word",
+            "'token_pattern'",
+            "'tokenizer'",
+            "is not None",
+        ),
+        (
+            None,
+            None,
+            lambda s: s.upper(),
+            (1, 1),
+            r"\w+",
+            lambda s: s.upper(),
+            "'preprocessor'",
+            "'analyzer'",
+            "is callable",
+        ),
+        (
+            None,
+            None,
+            None,
+            (1, 2),
+            None,
+            lambda s: s.upper(),
+            "'ngram_range'",
+            "'analyzer'",
+            "is callable",
+        ),
+        (
+            None,
+            None,
+            None,
+            (1, 1),
+            r"\w+",
+            "char",
+            "'token_pattern'",
+            "'analyzer'",
+            "!= 'word'",
+        ),
+    ],
 )
-def test_unused_parameters_warn(Vectorizer, stop_words,
-                                tokenizer, preprocessor,
-                                ngram_range, token_pattern,
-                                analyzer, unused_name, ovrd_name,
-                                ovrd_msg):
-
+def test_unused_parameters_warn(
+    Vectorizer,
+    stop_words,
+    tokenizer,
+    preprocessor,
+    ngram_range,
+    token_pattern,
+    analyzer,
+    unused_name,
+    ovrd_name,
+    ovrd_msg,
+):
     train_data = JUNK_FOOD_DOCS
     # setting parameter and checking for corresponding warning messages
     vect = Vectorizer()
-    vect.set_params(stop_words=stop_words, tokenizer=tokenizer,
-                    preprocessor=preprocessor, ngram_range=ngram_range,
-                    token_pattern=token_pattern, analyzer=analyzer)
-    msg = ("The parameter %s will not be used"
-           " since %s %s" % (unused_name, ovrd_name, ovrd_msg)
-           )
+    vect.set_params(
+        stop_words=stop_words,
+        tokenizer=tokenizer,
+        preprocessor=preprocessor,
+        ngram_range=ngram_range,
+        token_pattern=token_pattern,
+        analyzer=analyzer,
+    )
+    msg = "The parameter %s will not be used since %s %s" % (
+        unused_name,
+        ovrd_name,
+        ovrd_msg,
+    )
     with pytest.warns(UserWarning, match=msg):
         vect.fit(train_data)
 
 
-@pytest.mark.parametrize('Vectorizer, X', (
-    (HashingVectorizer, [{'foo': 1, 'bar': 2}, {'foo': 3, 'baz': 1}]),
-    (CountVectorizer, JUNK_FOOD_DOCS))
+@pytest.mark.parametrize(
+    "Vectorizer, X",
+    (
+        (HashingVectorizer, [{"foo": 1, "bar": 2}, {"foo": 3, "baz": 1}]),
+        (CountVectorizer, JUNK_FOOD_DOCS),
+    ),
 )
 def test_n_features_in(Vectorizer, X):
     # For vectorizers, n_features_in_ does not make sense
     vectorizer = Vectorizer()
-    assert not hasattr(vectorizer, 'n_features_in_')
+    assert not hasattr(vectorizer, "n_features_in_")
     vectorizer.fit(X)
-    assert not hasattr(vectorizer, 'n_features_in_')
+    assert not hasattr(vectorizer, "n_features_in_")
 
 
 def test_tie_breaking_sample_order_invariance():
     # Checks the sample order invariance when setting max_features
     # non-regression test for #17939
     vec = CountVectorizer(max_features=1)
-    vocab1 = vec.fit(['hello', 'world']).vocabulary_
-    vocab2 = vec.fit(['world', 'hello']).vocabulary_
+    vocab1 = vec.fit(["hello", "world"]).vocabulary_
+    vocab2 = vec.fit(["world", "hello"]).vocabulary_
     assert vocab1 == vocab2
 
 
-@fails_if_pypy
 def test_nonnegative_hashing_vectorizer_result_indices():
     # add test for pr 19035
     hashing = HashingVectorizer(n_features=1000000, ngram_range=(2, 3))
-    indices = hashing.transform(['22pcs efuture']).indices
+    indices = hashing.transform(["22pcs efuture"]).indices
     assert indices[0] >= 0
+
+
+@pytest.mark.parametrize(
+    "Estimator", [CountVectorizer, TfidfVectorizer, TfidfTransformer, HashingVectorizer]
+)
+def test_vectorizers_do_not_have_set_output(Estimator):
+    """Check that vectorizers do not define set_output."""
+    est = Estimator()
+    assert not hasattr(est, "set_output")
+
+
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_tfidf_transformer_copy(csr_container):
+    """Check the behaviour of TfidfTransformer.transform with the copy parameter."""
+    X = sparse.rand(10, 20000, dtype=np.float64, random_state=42)
+    X_csr = csr_container(X)
+
+    # keep a copy of the original matrix for later comparison
+    X_csr_original = X_csr.copy()
+
+    transformer = TfidfTransformer().fit(X_csr)
+
+    X_transform = transformer.transform(X_csr, copy=True)
+    assert_allclose_dense_sparse(X_csr, X_csr_original)
+    assert X_transform is not X_csr
+
+    X_transform = transformer.transform(X_csr, copy=False)
+    assert X_transform is X_csr
+    with pytest.raises(AssertionError):
+        assert_allclose_dense_sparse(X_csr, X_csr_original)
+
+
+@pytest.mark.parametrize("dtype", [np.float32, np.float64])
+def test_tfidf_vectorizer_perserve_dtype_idf(dtype):
+    """Check that `idf_` has the same dtype as the input data.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/30016
+    """
+    X = [str(uuid.uuid4()) for i in range(100_000)]
+    vectorizer = TfidfVectorizer(dtype=dtype).fit(X)
+    assert vectorizer.idf_.dtype == dtype
diff --git a/sklearn/feature_extraction/text.py b/sklearn/feature_extraction/text.py
index fad0e53ed31ca..eb3226b01c79e 100644
--- a/sklearn/feature_extraction/text.py
+++ b/sklearn/feature_extraction/text.py
@@ -1,49 +1,42 @@
-# -*- coding: utf-8 -*-
-# Authors: Olivier Grisel 
-#          Mathieu Blondel 
-#          Lars Buitinck
-#          Robert Layton 
-#          Jochen WersdÃļrfer 
-#          Roman Sinayev 
-#
-# License: BSD 3 clause
-"""
-The :mod:`sklearn.feature_extraction.text` submodule gathers utilities to
-build feature vectors from text documents.
-"""
+"""Utilities to build feature vectors from text documents."""
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import array
+import re
+import unicodedata
+import warnings
 from collections import defaultdict
 from collections.abc import Mapping
 from functools import partial
-import numbers
+from numbers import Integral
 from operator import itemgetter
-import re
-import unicodedata
-import warnings
 
 import numpy as np
 import scipy.sparse as sp
 
-from ..base import BaseEstimator, TransformerMixin
+from sklearn.utils import metadata_routing
+
+from ..base import BaseEstimator, OneToOneFeatureMixin, TransformerMixin, _fit_context
+from ..exceptions import NotFittedError
 from ..preprocessing import normalize
+from ..utils._param_validation import HasMethods, Interval, RealNotInt, StrOptions
+from ..utils.fixes import _IS_32BIT
+from ..utils.validation import FLOAT_DTYPES, check_array, check_is_fitted, validate_data
 from ._hash import FeatureHasher
 from ._stop_words import ENGLISH_STOP_WORDS
-from ..utils.validation import check_is_fitted, check_array, FLOAT_DTYPES
-from ..utils import _IS_32BIT
-from ..utils.fixes import _astype_copy_false
-from ..exceptions import NotFittedError
-from ..utils.validation import _deprecate_positional_args
-
 
-__all__ = ['HashingVectorizer',
-           'CountVectorizer',
-           'ENGLISH_STOP_WORDS',
-           'TfidfTransformer',
-           'TfidfVectorizer',
-           'strip_accents_ascii',
-           'strip_accents_unicode',
-           'strip_tags']
+__all__ = [
+    "ENGLISH_STOP_WORDS",
+    "CountVectorizer",
+    "HashingVectorizer",
+    "TfidfTransformer",
+    "TfidfVectorizer",
+    "strip_accents_ascii",
+    "strip_accents_unicode",
+    "strip_tags",
+]
 
 
 def _preprocess(doc, accent_function=None, lower=False):
@@ -58,7 +51,7 @@ def _preprocess(doc, accent_function=None, lower=False):
         Function for handling accented characters. Common strategies include
         normalizing and removing.
     lower: bool, default=False
-        Whether to use str.lower to lowercase all fo the text
+        Whether to use str.lower to lowercase all of the text
 
     Returns
     -------
@@ -72,8 +65,15 @@ def _preprocess(doc, accent_function=None, lower=False):
     return doc
 
 
-def _analyze(doc, analyzer=None, tokenizer=None, ngrams=None,
-             preprocessor=None, decoder=None, stop_words=None):
+def _analyze(
+    doc,
+    analyzer=None,
+    tokenizer=None,
+    ngrams=None,
+    preprocessor=None,
+    decoder=None,
+    stop_words=None,
+):
     """Chain together an optional series of text processing steps to go from
     a single document to ngrams, with or without tokenizing or preprocessing.
 
@@ -113,7 +113,7 @@ def _analyze(doc, analyzer=None, tokenizer=None, ngrams=None,
 
 
 def strip_accents_unicode(s):
-    """Transform accentuated unicode symbols into their simple counterpart
+    """Transform accentuated unicode symbols into their simple counterpart.
 
     Warning: the python-level loop and join operations make this
     implementation 20 times slower than the strip_accents_ascii basic
@@ -121,8 +121,13 @@ def strip_accents_unicode(s):
 
     Parameters
     ----------
-    s : string
-        The string to strip
+    s : str
+        The string to strip.
+
+    Returns
+    -------
+    s : str
+        The stripped string.
 
     See Also
     --------
@@ -135,39 +140,49 @@ def strip_accents_unicode(s):
         s.encode("ASCII", errors="strict")
         return s
     except UnicodeEncodeError:
-        normalized = unicodedata.normalize('NFKD', s)
-        return ''.join([c for c in normalized if not unicodedata.combining(c)])
+        normalized = unicodedata.normalize("NFKD", s)
+        return "".join([c for c in normalized if not unicodedata.combining(c)])
 
 
 def strip_accents_ascii(s):
-    """Transform accentuated unicode symbols into ascii or nothing
+    """Transform accentuated unicode symbols into ascii or nothing.
 
     Warning: this solution is only suited for languages that have a direct
     transliteration to ASCII symbols.
 
     Parameters
     ----------
-    s : string
-        The string to strip
+    s : str
+        The string to strip.
+
+    Returns
+    -------
+    s : str
+        The stripped string.
 
     See Also
     --------
     strip_accents_unicode : Remove accentuated char for any unicode symbol.
     """
-    nkfd_form = unicodedata.normalize('NFKD', s)
-    return nkfd_form.encode('ASCII', 'ignore').decode('ASCII')
+    nkfd_form = unicodedata.normalize("NFKD", s)
+    return nkfd_form.encode("ASCII", "ignore").decode("ASCII")
 
 
 def strip_tags(s):
-    """Basic regexp based HTML / XML tag stripper function
+    """Basic regexp based HTML / XML tag stripper function.
 
     For serious HTML/XML preprocessing you should rather use an external
     library such as lxml or BeautifulSoup.
 
     Parameters
     ----------
-    s : string
-        The string to strip
+    s : str
+        The string to strip.
+
+    Returns
+    -------
+    s : str
+        The stripped string.
     """
     return re.compile(r"<([^>]+)>", flags=re.UNICODE).sub(" ", s)
 
@@ -195,7 +210,7 @@ def decode(self, doc):
 
         Parameters
         ----------
-        doc : str
+        doc : bytes or str
             The string to decode.
 
         Returns
@@ -203,19 +218,20 @@ def decode(self, doc):
         doc: str
             A string of unicode symbols.
         """
-        if self.input == 'filename':
-            with open(doc, 'rb') as fh:
+        if self.input == "filename":
+            with open(doc, "rb") as fh:
                 doc = fh.read()
 
-        elif self.input == 'file':
+        elif self.input == "file":
             doc = doc.read()
 
         if isinstance(doc, bytes):
             doc = doc.decode(self.encoding, self.decode_error)
 
         if doc is np.nan:
-            raise ValueError("np.nan is an invalid document, expected byte or "
-                             "unicode string.")
+            raise ValueError(
+                "np.nan is an invalid document, expected byte or unicode string."
+            )
 
         return doc
 
@@ -243,10 +259,9 @@ def _word_ngrams(self, tokens, stop_words=None):
             tokens_append = tokens.append
             space_join = " ".join
 
-            for n in range(min_n,
-                           min(max_n + 1, n_original_tokens + 1)):
+            for n in range(min_n, min(max_n + 1, n_original_tokens + 1)):
                 for i in range(n_original_tokens - n + 1):
-                    tokens_append(space_join(original_tokens[i: i + n]))
+                    tokens_append(space_join(original_tokens[i : i + n]))
 
         return tokens
 
@@ -270,7 +285,7 @@ def _char_ngrams(self, text_document):
 
         for n in range(min_n, min(max_n + 1, text_len + 1)):
             for i in range(text_len - n + 1):
-                ngrams_append(text_document[i: i + n])
+                ngrams_append(text_document[i : i + n])
         return ngrams
 
     def _char_wb_ngrams(self, text_document):
@@ -289,15 +304,15 @@ def _char_wb_ngrams(self, text_document):
         ngrams_append = ngrams.append
 
         for w in text_document.split():
-            w = ' ' + w + ' '
+            w = " " + w + " "
             w_len = len(w)
             for n in range(min_n, max_n + 1):
                 offset = 0
-                ngrams_append(w[offset:offset + n])
+                ngrams_append(w[offset : offset + n])
                 while offset + n < w_len:
                     offset += 1
-                    ngrams_append(w[offset:offset + n])
-                if offset == 0:   # count a short word (w_len < n) only once
+                    ngrams_append(w[offset : offset + n])
+                if offset == 0:  # count a short word (w_len < n) only once
                     break
         return ngrams
 
@@ -317,17 +332,16 @@ def build_preprocessor(self):
             strip_accents = None
         elif callable(self.strip_accents):
             strip_accents = self.strip_accents
-        elif self.strip_accents == 'ascii':
+        elif self.strip_accents == "ascii":
             strip_accents = strip_accents_ascii
-        elif self.strip_accents == 'unicode':
+        elif self.strip_accents == "unicode":
             strip_accents = strip_accents_unicode
         else:
-            raise ValueError('Invalid value for "strip_accents": %s' %
-                             self.strip_accents)
+            raise ValueError(
+                'Invalid value for "strip_accents": %s' % self.strip_accents
+            )
 
-        return partial(
-            _preprocess, accent_function=strip_accents, lower=self.lowercase
-        )
+        return partial(_preprocess, accent_function=strip_accents, lower=self.lowercase)
 
     def build_tokenizer(self):
         """Return a function that splits a string into a sequence of tokens.
@@ -370,7 +384,7 @@ def _check_stop_words_consistency(self, stop_words, preprocess, tokenize):
                         performed (e.g. because of the use of a custom
                         preprocessor / tokenizer)
         """
-        if id(self.stop_words) == getattr(self, '_stop_words_id', None):
+        if id(self.stop_words) == getattr(self, "_stop_words_id", None):
             # Stop words are were previously validated
             return None
 
@@ -385,20 +399,23 @@ def _check_stop_words_consistency(self, stop_words, preprocess, tokenize):
             self._stop_words_id = id(self.stop_words)
 
             if inconsistent:
-                warnings.warn('Your stop_words may be inconsistent with '
-                              'your preprocessing. Tokenizing the stop '
-                              'words generated tokens %r not in '
-                              'stop_words.' % sorted(inconsistent))
+                warnings.warn(
+                    "Your stop_words may be inconsistent with "
+                    "your preprocessing. Tokenizing the stop "
+                    "words generated tokens %r not in "
+                    "stop_words." % sorted(inconsistent)
+                )
             return not inconsistent
         except Exception:
             # Failed to check stop words consistency (e.g. because a custom
             # preprocessor or tokenizer was used)
             self._stop_words_id = id(self.stop_words)
-            return 'error'
+            return "error"
 
     def build_analyzer(self):
-        """Return a callable that handles preprocessing, tokenization
-        and n-grams generation.
+        """Return a callable to process input data.
+
+        The callable handles preprocessing, tokenization, and n-grams generation.
 
         Returns
         -------
@@ -408,33 +425,43 @@ def build_analyzer(self):
         """
 
         if callable(self.analyzer):
-            return partial(
-                _analyze, analyzer=self.analyzer, decoder=self.decode
-            )
+            return partial(_analyze, analyzer=self.analyzer, decoder=self.decode)
 
         preprocess = self.build_preprocessor()
 
-        if self.analyzer == 'char':
-            return partial(_analyze, ngrams=self._char_ngrams,
-                           preprocessor=preprocess, decoder=self.decode)
-
-        elif self.analyzer == 'char_wb':
+        if self.analyzer == "char":
+            return partial(
+                _analyze,
+                ngrams=self._char_ngrams,
+                preprocessor=preprocess,
+                decoder=self.decode,
+            )
 
-            return partial(_analyze, ngrams=self._char_wb_ngrams,
-                           preprocessor=preprocess, decoder=self.decode)
+        elif self.analyzer == "char_wb":
+            return partial(
+                _analyze,
+                ngrams=self._char_wb_ngrams,
+                preprocessor=preprocess,
+                decoder=self.decode,
+            )
 
-        elif self.analyzer == 'word':
+        elif self.analyzer == "word":
             stop_words = self.get_stop_words()
             tokenize = self.build_tokenizer()
-            self._check_stop_words_consistency(stop_words, preprocess,
-                                               tokenize)
-            return partial(_analyze, ngrams=self._word_ngrams,
-                           tokenizer=tokenize, preprocessor=preprocess,
-                           decoder=self.decode, stop_words=stop_words)
+            self._check_stop_words_consistency(stop_words, preprocess, tokenize)
+            return partial(
+                _analyze,
+                ngrams=self._word_ngrams,
+                tokenizer=tokenize,
+                preprocessor=preprocess,
+                decoder=self.decode,
+                stop_words=stop_words,
+            )
 
         else:
-            raise ValueError('%s is not a valid tokenization scheme/analyzer' %
-                             self.analyzer)
+            raise ValueError(
+                "%s is not a valid tokenization scheme/analyzer" % self.analyzer
+            )
 
     def _validate_vocabulary(self):
         vocabulary = self.vocabulary
@@ -454,8 +481,10 @@ def _validate_vocabulary(self):
                     raise ValueError("Vocabulary contains repeated indices.")
                 for i in range(len(vocabulary)):
                     if i not in indices:
-                        msg = ("Vocabulary of size %d doesn't contain index "
-                               "%d." % (len(vocabulary), i))
+                        msg = "Vocabulary of size %d doesn't contain index %d." % (
+                            len(vocabulary),
+                            i,
+                        )
                         raise ValueError(msg)
             if not vocabulary:
                 raise ValueError("empty vocabulary passed to fit")
@@ -466,7 +495,7 @@ def _validate_vocabulary(self):
 
     def _check_vocabulary(self):
         """Check if vocabulary is empty or missing (not fitted)"""
-        if not hasattr(self, 'vocabulary_'):
+        if not hasattr(self, "vocabulary_"):
             self._validate_vocabulary()
             if not self.fixed_vocabulary_:
                 raise NotFittedError("Vocabulary not fitted or provided")
@@ -474,44 +503,62 @@ def _check_vocabulary(self):
         if len(self.vocabulary_) == 0:
             raise ValueError("Vocabulary is empty")
 
-    def _validate_params(self):
+    def _validate_ngram_range(self):
         """Check validity of ngram_range parameter"""
         min_n, max_m = self.ngram_range
         if min_n > max_m:
             raise ValueError(
                 "Invalid value for ngram_range=%s "
-                "lower boundary larger than the upper boundary."
-                % str(self.ngram_range))
+                "lower boundary larger than the upper boundary." % str(self.ngram_range)
+            )
 
     def _warn_for_unused_params(self):
-
         if self.tokenizer is not None and self.token_pattern is not None:
-            warnings.warn("The parameter 'token_pattern' will not be used"
-                          " since 'tokenizer' is not None'")
+            warnings.warn(
+                "The parameter 'token_pattern' will not be used"
+                " since 'tokenizer' is not None'"
+            )
 
         if self.preprocessor is not None and callable(self.analyzer):
-            warnings.warn("The parameter 'preprocessor' will not be used"
-                          " since 'analyzer' is callable'")
-
-        if (self.ngram_range != (1, 1) and self.ngram_range is not None
-                and callable(self.analyzer)):
-            warnings.warn("The parameter 'ngram_range' will not be used"
-                          " since 'analyzer' is callable'")
-        if self.analyzer != 'word' or callable(self.analyzer):
+            warnings.warn(
+                "The parameter 'preprocessor' will not be used"
+                " since 'analyzer' is callable'"
+            )
+
+        if (
+            self.ngram_range != (1, 1)
+            and self.ngram_range is not None
+            and callable(self.analyzer)
+        ):
+            warnings.warn(
+                "The parameter 'ngram_range' will not be used"
+                " since 'analyzer' is callable'"
+            )
+        if self.analyzer != "word" or callable(self.analyzer):
             if self.stop_words is not None:
-                warnings.warn("The parameter 'stop_words' will not be used"
-                              " since 'analyzer' != 'word'")
-            if self.token_pattern is not None and \
-               self.token_pattern != r"(?u)\b\w\w+\b":
-                warnings.warn("The parameter 'token_pattern' will not be used"
-                              " since 'analyzer' != 'word'")
+                warnings.warn(
+                    "The parameter 'stop_words' will not be used"
+                    " since 'analyzer' != 'word'"
+                )
+            if (
+                self.token_pattern is not None
+                and self.token_pattern != r"(?u)\b\w\w+\b"
+            ):
+                warnings.warn(
+                    "The parameter 'token_pattern' will not be used"
+                    " since 'analyzer' != 'word'"
+                )
             if self.tokenizer is not None:
-                warnings.warn("The parameter 'tokenizer' will not be used"
-                              " since 'analyzer' != 'word'")
+                warnings.warn(
+                    "The parameter 'tokenizer' will not be used"
+                    " since 'analyzer' != 'word'"
+                )
 
 
-class HashingVectorizer(TransformerMixin, _VectorizerMixin, BaseEstimator):
-    r"""Convert a collection of text documents to a matrix of token occurrences
+class HashingVectorizer(
+    TransformerMixin, _VectorizerMixin, BaseEstimator, auto_wrap_output_keys=None
+):
+    r"""Convert a collection of text documents to a matrix of token occurrences.
 
     It turns a collection of text documents into a scipy.sparse matrix holding
     token occurrence counts (or binary occurrence information), possibly
@@ -524,10 +571,10 @@ class HashingVectorizer(TransformerMixin, _VectorizerMixin, BaseEstimator):
     This strategy has several advantages:
 
     - it is very low memory scalable to large datasets as there is no need to
-      store a vocabulary dictionary in memory
+      store a vocabulary dictionary in memory.
 
     - it is fast to pickle and un-pickle as it holds no state besides the
-      constructor parameters
+      constructor parameters.
 
     - it can be used in a streaming (partial fit) or parallel pipeline as there
       is no state computed during fit.
@@ -547,11 +594,17 @@ class HashingVectorizer(TransformerMixin, _VectorizerMixin, BaseEstimator):
 
     The hash function employed is the signed 32-bit version of Murmurhash3.
 
+    For an efficiency comparison of the different feature extractors, see
+    :ref:`sphx_glr_auto_examples_text_plot_hashing_vs_dict_vectorizer.py`.
+
+    For an example of document clustering and comparison with
+    :class:`~sklearn.feature_extraction.text.TfidfVectorizer`, see
+    :ref:`sphx_glr_auto_examples_text_plot_document_clustering.py`.
+
     Read more in the :ref:`User Guide `.
 
     Parameters
     ----------
-
     input : {'filename', 'file', 'content'}, default='content'
         - If `'filename'`, the sequence passed as an argument to fit is
           expected to be a list of filenames that need reading to fetch
@@ -563,7 +616,7 @@ class HashingVectorizer(TransformerMixin, _VectorizerMixin, BaseEstimator):
         - If `'content'`, the input is expected to be a sequence of items that
           can be of type string or byte.
 
-    encoding : string, default='utf-8'
+    encoding : str, default='utf-8'
         If bytes or files are given to analyze, this encoding is used to
         decode.
 
@@ -573,13 +626,13 @@ class HashingVectorizer(TransformerMixin, _VectorizerMixin, BaseEstimator):
         'strict', meaning that a UnicodeDecodeError will be raised. Other
         values are 'ignore' and 'replace'.
 
-    strip_accents : {'ascii', 'unicode'}, default=None
+    strip_accents : {'ascii', 'unicode'} or callable, default=None
         Remove accents and perform other character normalization
         during the preprocessing step.
         'ascii' is a fast method that only works on characters that have
-        an direct ASCII mapping.
-        'unicode' is a slightly slower method that works on any characters.
-        None (default) does nothing.
+        a direct ASCII mapping.
+        'unicode' is a slightly slower method that works on any character.
+        None (default) means no character normalization is performed.
 
         Both 'ascii' and 'unicode' use NFKD normalization from
         :func:`unicodedata.normalize`.
@@ -590,7 +643,7 @@ class HashingVectorizer(TransformerMixin, _VectorizerMixin, BaseEstimator):
     preprocessor : callable, default=None
         Override the preprocessing (string transformation) stage while
         preserving the tokenizing and n-grams generation steps.
-        Only applies if ``analyzer is not callable``.
+        Only applies if ``analyzer`` is not callable.
 
     tokenizer : callable, default=None
         Override the string tokenization step while preserving the
@@ -606,7 +659,7 @@ class HashingVectorizer(TransformerMixin, _VectorizerMixin, BaseEstimator):
         will be removed from the resulting tokens.
         Only applies if ``analyzer == 'word'``.
 
-    token_pattern : str, default=r"(?u)\\b\\w\\w+\\b"
+    token_pattern : str or None, default=r"(?u)\\b\\w\\w+\\b"
         Regular expression denoting what constitutes a "token", only used
         if ``analyzer == 'word'``. The default regexp selects tokens of 2
         or more alphanumeric characters (punctuation is completely ignored
@@ -622,7 +675,7 @@ class HashingVectorizer(TransformerMixin, _VectorizerMixin, BaseEstimator):
         will be used. For example an ``ngram_range`` of ``(1, 1)`` means only
         unigrams, ``(1, 2)`` means unigrams and bigrams, and ``(2, 2)`` means
         only bigrams.
-        Only applies if ``analyzer is not callable``.
+        Only applies if ``analyzer`` is not callable.
 
     analyzer : {'word', 'char', 'char_wb'} or callable, default='word'
         Whether the feature should be made of word or character n-grams.
@@ -642,7 +695,7 @@ class HashingVectorizer(TransformerMixin, _VectorizerMixin, BaseEstimator):
         of features are likely to cause hash collisions, but large numbers
         will cause larger coefficient dimensions in linear learners.
 
-    binary : bool, default=False.
+    binary : bool, default=False
         If True, all non zero counts are set to 1. This is useful for discrete
         probabilistic models that model binary events rather than integer
         counts.
@@ -660,6 +713,20 @@ class HashingVectorizer(TransformerMixin, _VectorizerMixin, BaseEstimator):
     dtype : type, default=np.float64
         Type of the matrix returned by fit_transform() or transform().
 
+    See Also
+    --------
+    CountVectorizer : Convert a collection of text documents to a matrix of
+        token counts.
+    TfidfVectorizer : Convert a collection of raw documents to a matrix of
+        TF-IDF features.
+
+    Notes
+    -----
+    This estimator is :term:`stateless` and does not need to be fitted.
+    However, we recommend to call :meth:`fit_transform` instead of
+    :meth:`transform`, as parameter validation is only performed in
+    :meth:`fit`.
+
     Examples
     --------
     >>> from sklearn.feature_extraction.text import HashingVectorizer
@@ -673,20 +740,47 @@ class HashingVectorizer(TransformerMixin, _VectorizerMixin, BaseEstimator):
     >>> X = vectorizer.fit_transform(corpus)
     >>> print(X.shape)
     (4, 16)
-
-    See Also
-    --------
-    CountVectorizer, TfidfVectorizer
-
     """
-    @_deprecate_positional_args
-    def __init__(self, *, input='content', encoding='utf-8',
-                 decode_error='strict', strip_accents=None,
-                 lowercase=True, preprocessor=None, tokenizer=None,
-                 stop_words=None, token_pattern=r"(?u)\b\w\w+\b",
-                 ngram_range=(1, 1), analyzer='word', n_features=(2 ** 20),
-                 binary=False, norm='l2', alternate_sign=True,
-                 dtype=np.float64):
+
+    _parameter_constraints: dict = {
+        "input": [StrOptions({"filename", "file", "content"})],
+        "encoding": [str],
+        "decode_error": [StrOptions({"strict", "ignore", "replace"})],
+        "strip_accents": [StrOptions({"ascii", "unicode"}), None, callable],
+        "lowercase": ["boolean"],
+        "preprocessor": [callable, None],
+        "tokenizer": [callable, None],
+        "stop_words": [StrOptions({"english"}), list, None],
+        "token_pattern": [str, None],
+        "ngram_range": [tuple],
+        "analyzer": [StrOptions({"word", "char", "char_wb"}), callable],
+        "n_features": [Interval(Integral, 1, np.iinfo(np.int32).max, closed="left")],
+        "binary": ["boolean"],
+        "norm": [StrOptions({"l1", "l2"}), None],
+        "alternate_sign": ["boolean"],
+        "dtype": "no_validation",  # delegate to numpy
+    }
+
+    def __init__(
+        self,
+        *,
+        input="content",
+        encoding="utf-8",
+        decode_error="strict",
+        strip_accents=None,
+        lowercase=True,
+        preprocessor=None,
+        tokenizer=None,
+        stop_words=None,
+        token_pattern=r"(?u)\b\w\w+\b",
+        ngram_range=(1, 1),
+        analyzer="word",
+        n_features=(2**20),
+        binary=False,
+        norm="l2",
+        alternate_sign=True,
+        dtype=np.float64,
+    ):
         self.input = input
         self.encoding = encoding
         self.decode_error = decode_error
@@ -704,35 +798,56 @@ def __init__(self, *, input='content', encoding='utf-8',
         self.alternate_sign = alternate_sign
         self.dtype = dtype
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def partial_fit(self, X, y=None):
-        """Does nothing: this transformer is stateless.
+        """Only validates estimator's parameters.
 
-        This method is just there to mark the fact that this transformer
-        can work in a streaming setup.
+        This method allows to: (i) validate the estimator's parameters and
+        (ii) be consistent with the scikit-learn transformer API.
 
         Parameters
         ----------
         X : ndarray of shape [n_samples, n_features]
             Training data.
+
+        y : Ignored
+            Not used, present for API consistency by convention.
+
+        Returns
+        -------
+        self : object
+            HashingVectorizer instance.
         """
         return self
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y=None):
-        """Does nothing: this transformer is stateless.
+        """Only validates estimator's parameters.
+
+        This method allows to: (i) validate the estimator's parameters and
+        (ii) be consistent with the scikit-learn transformer API.
 
         Parameters
         ----------
         X : ndarray of shape [n_samples, n_features]
             Training data.
+
+        y : Ignored
+            Not used, present for API consistency by convention.
+
+        Returns
+        -------
+        self : object
+            HashingVectorizer instance.
         """
         # triggers a parameter validation
         if isinstance(X, str):
             raise ValueError(
-                "Iterable over raw text documents expected, "
-                "string object received.")
+                "Iterable over raw text documents expected, string object received."
+            )
 
         self._warn_for_unused_params()
-        self._validate_params()
+        self._validate_ngram_range()
 
         self._get_hasher().fit(X, y=y)
         return self
@@ -754,10 +869,10 @@ def transform(self, X):
         """
         if isinstance(X, str):
             raise ValueError(
-                "Iterable over raw text documents expected, "
-                "string object received.")
+                "Iterable over raw text documents expected, string object received."
+            )
 
-        self._validate_params()
+        self._validate_ngram_range()
 
         analyzer = self.build_analyzer()
         X = self._get_hasher().transform(analyzer(doc) for doc in X)
@@ -788,24 +903,30 @@ def fit_transform(self, X, y=None):
         return self.fit(X, y).transform(X)
 
     def _get_hasher(self):
-        return FeatureHasher(n_features=self.n_features,
-                             input_type='string', dtype=self.dtype,
-                             alternate_sign=self.alternate_sign)
+        return FeatureHasher(
+            n_features=self.n_features,
+            input_type="string",
+            dtype=self.dtype,
+            alternate_sign=self.alternate_sign,
+        )
 
-    def _more_tags(self):
-        return {'X_types': ['string']}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.string = True
+        tags.input_tags.two_d_array = False
+        return tags
 
 
 def _document_frequency(X):
     """Count the number of non-zero values for each feature in sparse X."""
-    if sp.isspmatrix_csr(X):
+    if sp.issparse(X) and X.format == "csr":
         return np.bincount(X.indices, minlength=X.shape[1])
     else:
         return np.diff(X.indptr)
 
 
 class CountVectorizer(_VectorizerMixin, BaseEstimator):
-    r"""Convert a collection of text documents to a matrix of token counts
+    r"""Convert a collection of text documents to a matrix of token counts.
 
     This implementation produces a sparse representation of the counts using
     scipy.sparse.csr_matrix.
@@ -814,6 +935,9 @@ class CountVectorizer(_VectorizerMixin, BaseEstimator):
     that does some kind of feature selection then the number of features will
     be equal to the vocabulary size found by analyzing the data.
 
+    For an efficiency comparison of the different feature extractors, see
+    :ref:`sphx_glr_auto_examples_text_plot_hashing_vs_dict_vectorizer.py`.
+
     Read more in the :ref:`User Guide `.
 
     Parameters
@@ -829,7 +953,7 @@ class CountVectorizer(_VectorizerMixin, BaseEstimator):
         - If `'content'`, the input is expected to be a sequence of items that
           can be of type string or byte.
 
-    encoding : string, default='utf-8'
+    encoding : str, default='utf-8'
         If bytes or files are given to analyze, this encoding is used to
         decode.
 
@@ -839,13 +963,13 @@ class CountVectorizer(_VectorizerMixin, BaseEstimator):
         'strict', meaning that a UnicodeDecodeError will be raised. Other
         values are 'ignore' and 'replace'.
 
-    strip_accents : {'ascii', 'unicode'}, default=None
+    strip_accents : {'ascii', 'unicode'} or callable, default=None
         Remove accents and perform other character normalization
         during the preprocessing step.
         'ascii' is a fast method that only works on characters that have
-        an direct ASCII mapping.
+        a direct ASCII mapping.
         'unicode' is a slightly slower method that works on any characters.
-        None (default) does nothing.
+        None (default) means no character normalization is performed.
 
         Both 'ascii' and 'unicode' use NFKD normalization from
         :func:`unicodedata.normalize`.
@@ -856,7 +980,7 @@ class CountVectorizer(_VectorizerMixin, BaseEstimator):
     preprocessor : callable, default=None
         Override the preprocessing (strip_accents and lowercase) stage while
         preserving the tokenizing and n-grams generation steps.
-        Only applies if ``analyzer is not callable``.
+        Only applies if ``analyzer`` is not callable.
 
     tokenizer : callable, default=None
         Override the string tokenization step while preserving the
@@ -872,11 +996,11 @@ class CountVectorizer(_VectorizerMixin, BaseEstimator):
         will be removed from the resulting tokens.
         Only applies if ``analyzer == 'word'``.
 
-        If None, no stop words will be used. max_df can be set to a value
-        in the range [0.7, 1.0) to automatically detect and filter stop
-        words based on intra corpus document frequency of terms.
+        If None, no stop words will be used. In this case, setting `max_df`
+        to a higher value, such as in the range (0.7, 1.0), can automatically detect
+        and filter stop words based on intra corpus document frequency of terms.
 
-    token_pattern : str, default=r"(?u)\\b\\w\\w+\\b"
+    token_pattern : str or None, default=r"(?u)\\b\\w\\w+\\b"
         Regular expression denoting what constitutes a "token", only used
         if ``analyzer == 'word'``. The default regexp select tokens of 2
         or more alphanumeric characters (punctuation is completely ignored
@@ -892,7 +1016,7 @@ class CountVectorizer(_VectorizerMixin, BaseEstimator):
         such that min_n <= n <= max_n will be used. For example an
         ``ngram_range`` of ``(1, 1)`` means only unigrams, ``(1, 2)`` means
         unigrams and bigrams, and ``(2, 2)`` means only bigrams.
-        Only applies if ``analyzer is not callable``.
+        Only applies if ``analyzer`` is not callable.
 
     analyzer : {'word', 'char', 'char_wb'} or callable, default='word'
         Whether the feature should be made of word n-gram or character
@@ -927,7 +1051,8 @@ class CountVectorizer(_VectorizerMixin, BaseEstimator):
 
     max_features : int, default=None
         If not None, build a vocabulary that only consider the top
-        max_features ordered by term frequency across the corpus.
+        `max_features` ordered by term frequency across the corpus.
+        Otherwise, all features are used.
 
         This parameter is ignored if vocabulary is not None.
 
@@ -943,7 +1068,7 @@ class CountVectorizer(_VectorizerMixin, BaseEstimator):
         probabilistic models that model binary events rather than integer
         counts.
 
-    dtype : type, default=np.int64
+    dtype : dtype, default=np.int64
         Type of the matrix returned by fit_transform() or transform().
 
     Attributes
@@ -951,18 +1076,17 @@ class CountVectorizer(_VectorizerMixin, BaseEstimator):
     vocabulary_ : dict
         A mapping of terms to feature indices.
 
-    fixed_vocabulary_: boolean
+    fixed_vocabulary_ : bool
         True if a fixed vocabulary of term to indices mapping
-        is provided by the user
-
-    stop_words_ : set
-        Terms that were ignored because they either:
+        is provided by the user.
 
-          - occurred in too many documents (`max_df`)
-          - occurred in too few documents (`min_df`)
-          - were cut off by feature selection (`max_features`).
+    See Also
+    --------
+    HashingVectorizer : Convert a collection of text documents to a
+        matrix of token counts.
 
-        This is only available if no vocabulary was given.
+    TfidfVectorizer : Convert a collection of raw documents to a matrix
+        of TF-IDF features.
 
     Examples
     --------
@@ -975,8 +1099,9 @@ class CountVectorizer(_VectorizerMixin, BaseEstimator):
     ... ]
     >>> vectorizer = CountVectorizer()
     >>> X = vectorizer.fit_transform(corpus)
-    >>> print(vectorizer.get_feature_names())
-    ['and', 'document', 'first', 'is', 'one', 'second', 'the', 'third', 'this']
+    >>> vectorizer.get_feature_names_out()
+    array(['and', 'document', 'first', 'is', 'one', 'second', 'the', 'third',
+           'this'], ...)
     >>> print(X.toarray())
     [[0 1 1 1 0 0 1 0 1]
      [0 2 0 1 0 1 1 0 1]
@@ -984,34 +1109,69 @@ class CountVectorizer(_VectorizerMixin, BaseEstimator):
      [0 1 1 1 0 0 1 0 1]]
     >>> vectorizer2 = CountVectorizer(analyzer='word', ngram_range=(2, 2))
     >>> X2 = vectorizer2.fit_transform(corpus)
-    >>> print(vectorizer2.get_feature_names())
-    ['and this', 'document is', 'first document', 'is the', 'is this',
-    'second document', 'the first', 'the second', 'the third', 'third one',
-     'this document', 'this is', 'this the']
+    >>> vectorizer2.get_feature_names_out()
+    array(['and this', 'document is', 'first document', 'is the', 'is this',
+           'second document', 'the first', 'the second', 'the third', 'third one',
+           'this document', 'this is', 'this the'], ...)
      >>> print(X2.toarray())
      [[0 0 1 1 0 0 1 0 0 0 0 1 0]
      [0 1 0 1 0 1 0 1 0 0 1 0 0]
      [1 0 0 1 0 0 0 0 1 1 0 1 0]
      [0 0 1 0 1 0 1 0 0 0 0 0 1]]
-
-    See Also
-    --------
-    HashingVectorizer, TfidfVectorizer
-
-    Notes
-    -----
-    The ``stop_words_`` attribute can get large and increase the model size
-    when pickling. This attribute is provided only for introspection and can
-    be safely removed using delattr or set to None before pickling.
     """
-    @_deprecate_positional_args
-    def __init__(self, *, input='content', encoding='utf-8',
-                 decode_error='strict', strip_accents=None,
-                 lowercase=True, preprocessor=None, tokenizer=None,
-                 stop_words=None, token_pattern=r"(?u)\b\w\w+\b",
-                 ngram_range=(1, 1), analyzer='word',
-                 max_df=1.0, min_df=1, max_features=None,
-                 vocabulary=None, binary=False, dtype=np.int64):
+
+    # raw_documents should not be in the routing mechanism. It should have been
+    # called X in the first place.
+    __metadata_request__fit = {"raw_documents": metadata_routing.UNUSED}
+    __metadata_request__transform = {"raw_documents": metadata_routing.UNUSED}
+
+    _parameter_constraints: dict = {
+        "input": [StrOptions({"filename", "file", "content"})],
+        "encoding": [str],
+        "decode_error": [StrOptions({"strict", "ignore", "replace"})],
+        "strip_accents": [StrOptions({"ascii", "unicode"}), None, callable],
+        "lowercase": ["boolean"],
+        "preprocessor": [callable, None],
+        "tokenizer": [callable, None],
+        "stop_words": [StrOptions({"english"}), list, None],
+        "token_pattern": [str, None],
+        "ngram_range": [tuple],
+        "analyzer": [StrOptions({"word", "char", "char_wb"}), callable],
+        "max_df": [
+            Interval(RealNotInt, 0, 1, closed="both"),
+            Interval(Integral, 1, None, closed="left"),
+        ],
+        "min_df": [
+            Interval(RealNotInt, 0, 1, closed="both"),
+            Interval(Integral, 1, None, closed="left"),
+        ],
+        "max_features": [Interval(Integral, 1, None, closed="left"), None],
+        "vocabulary": [Mapping, HasMethods("__iter__"), None],
+        "binary": ["boolean"],
+        "dtype": "no_validation",  # delegate to numpy
+    }
+
+    def __init__(
+        self,
+        *,
+        input="content",
+        encoding="utf-8",
+        decode_error="strict",
+        strip_accents=None,
+        lowercase=True,
+        preprocessor=None,
+        tokenizer=None,
+        stop_words=None,
+        token_pattern=r"(?u)\b\w\w+\b",
+        ngram_range=(1, 1),
+        analyzer="word",
+        max_df=1.0,
+        min_df=1,
+        max_features=None,
+        vocabulary=None,
+        binary=False,
+        dtype=np.int64,
+    ):
         self.input = input
         self.encoding = encoding
         self.decode_error = decode_error
@@ -1024,15 +1184,7 @@ def __init__(self, *, input='content', encoding='utf-8',
         self.stop_words = stop_words
         self.max_df = max_df
         self.min_df = min_df
-        if max_df < 0 or min_df < 0:
-            raise ValueError("negative value for max_df or min_df")
         self.max_features = max_features
-        if max_features is not None:
-            if (not isinstance(max_features, numbers.Integral) or
-                    max_features <= 0):
-                raise ValueError(
-                    "max_features=%r, neither a positive integer nor None"
-                    % max_features)
         self.ngram_range = ngram_range
         self.vocabulary = vocabulary
         self.binary = binary
@@ -1049,11 +1201,10 @@ def _sort_features(self, X, vocabulary):
             vocabulary[term] = new_val
             map_index[old_val] = new_val
 
-        X.indices = map_index.take(X.indices, mode='clip')
+        X.indices = map_index.take(X.indices, mode="clip")
         return X
 
-    def _limit_features(self, X, vocabulary, high=None, low=None,
-                        limit=None):
+    def _limit_features(self, X, vocabulary, high=None, low=None, limit=None):
         """Remove too rare or too common features.
 
         Prune features that are non zero in more samples than high or less
@@ -1080,22 +1231,20 @@ def _limit_features(self, X, vocabulary, high=None, low=None,
             mask = new_mask
 
         new_indices = np.cumsum(mask) - 1  # maps old indices to new
-        removed_terms = set()
         for term, old_index in list(vocabulary.items()):
             if mask[old_index]:
                 vocabulary[term] = new_indices[old_index]
             else:
                 del vocabulary[term]
-                removed_terms.add(term)
         kept_indices = np.where(mask)[0]
         if len(kept_indices) == 0:
-            raise ValueError("After pruning, no terms remain. Try a lower"
-                             " min_df or a higher max_df.")
-        return X[:, kept_indices], removed_terms
+            raise ValueError(
+                "After pruning, no terms remain. Try a lower min_df or a higher max_df."
+            )
+        return X[:, kept_indices]
 
     def _count_vocab(self, raw_documents, fixed_vocab):
-        """Create sparse feature matrix, and vocabulary where fixed_vocab=False
-        """
+        """Create sparse feature matrix, and vocabulary where fixed_vocab=False"""
         if fixed_vocab:
             vocabulary = self.vocabulary_
         else:
@@ -1107,15 +1256,6 @@ def _count_vocab(self, raw_documents, fixed_vocab):
         j_indices = []
         indptr = []
 
-        if self.lowercase:
-            for vocab in vocabulary:
-                if any(map(str.isupper, vocab)):
-                    warnings.warn("Upper case characters found in"
-                                  " vocabulary while 'lowercase'"
-                                  " is True. These entries will not"
-                                  " be matched with any documents")
-                    break
-
         values = _make_int_array()
         indptr.append(0)
         for doc in raw_documents:
@@ -1139,15 +1279,19 @@ def _count_vocab(self, raw_documents, fixed_vocab):
             # disable defaultdict behaviour
             vocabulary = dict(vocabulary)
             if not vocabulary:
-                raise ValueError("empty vocabulary; perhaps the documents only"
-                                 " contain stop words")
+                raise ValueError(
+                    "empty vocabulary; perhaps the documents only contain stop words"
+                )
 
         if indptr[-1] > np.iinfo(np.int32).max:  # = 2**31 - 1
             if _IS_32BIT:
-                raise ValueError(('sparse CSR array has {} non-zero '
-                                  'elements and requires 64 bit indexing, '
-                                  'which is unsupported with 32 bit Python.')
-                                 .format(indptr[-1]))
+                raise ValueError(
+                    (
+                        "sparse CSR array has {} non-zero "
+                        "elements and requires 64 bit indexing, "
+                        "which is unsupported with 32 bit Python."
+                    ).format(indptr[-1])
+                )
             indices_dtype = np.int64
 
         else:
@@ -1156,9 +1300,11 @@ def _count_vocab(self, raw_documents, fixed_vocab):
         indptr = np.asarray(indptr, dtype=indices_dtype)
         values = np.frombuffer(values, dtype=np.intc)
 
-        X = sp.csr_matrix((values, j_indices, indptr),
-                          shape=(len(indptr) - 1, len(vocabulary)),
-                          dtype=self.dtype)
+        X = sp.csr_matrix(
+            (values, j_indices, indptr),
+            shape=(len(indptr) - 1, len(vocabulary)),
+            dtype=self.dtype,
+        )
         X.sort_indices()
         return vocabulary, X
 
@@ -1168,16 +1314,20 @@ def fit(self, raw_documents, y=None):
         Parameters
         ----------
         raw_documents : iterable
-            An iterable which yields either str, unicode or file objects.
+            An iterable which generates either str, unicode or file objects.
+
+        y : None
+            This parameter is ignored.
 
         Returns
         -------
-        self
+        self : object
+            Fitted vectorizer.
         """
-        self._warn_for_unused_params()
         self.fit_transform(raw_documents)
         return self
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit_transform(self, raw_documents, y=None):
         """Learn the vocabulary dictionary and return document-term matrix.
 
@@ -1187,7 +1337,10 @@ def fit_transform(self, raw_documents, y=None):
         Parameters
         ----------
         raw_documents : iterable
-            An iterable which yields either str, unicode or file objects.
+            An iterable which generates either str, unicode or file objects.
+
+        y : None
+            This parameter is ignored.
 
         Returns
         -------
@@ -1199,38 +1352,43 @@ def fit_transform(self, raw_documents, y=None):
         # TfidfVectorizer.
         if isinstance(raw_documents, str):
             raise ValueError(
-                "Iterable over raw text documents expected, "
-                "string object received.")
+                "Iterable over raw text documents expected, string object received."
+            )
 
-        self._validate_params()
+        self._validate_ngram_range()
+        self._warn_for_unused_params()
         self._validate_vocabulary()
         max_df = self.max_df
         min_df = self.min_df
         max_features = self.max_features
 
-        vocabulary, X = self._count_vocab(raw_documents,
-                                          self.fixed_vocabulary_)
+        if self.fixed_vocabulary_ and self.lowercase:
+            for term in self.vocabulary:
+                if any(map(str.isupper, term)):
+                    warnings.warn(
+                        "Upper case characters found in"
+                        " vocabulary while 'lowercase'"
+                        " is True. These entries will not"
+                        " be matched with any documents"
+                    )
+                    break
+
+        vocabulary, X = self._count_vocab(raw_documents, self.fixed_vocabulary_)
 
         if self.binary:
             X.data.fill(1)
 
         if not self.fixed_vocabulary_:
             n_doc = X.shape[0]
-            max_doc_count = (max_df
-                             if isinstance(max_df, numbers.Integral)
-                             else max_df * n_doc)
-            min_doc_count = (min_df
-                             if isinstance(min_df, numbers.Integral)
-                             else min_df * n_doc)
+            max_doc_count = max_df if isinstance(max_df, Integral) else max_df * n_doc
+            min_doc_count = min_df if isinstance(min_df, Integral) else min_df * n_doc
             if max_doc_count < min_doc_count:
-                raise ValueError(
-                    "max_df corresponds to < documents than min_df")
+                raise ValueError("max_df corresponds to < documents than min_df")
             if max_features is not None:
                 X = self._sort_features(X, vocabulary)
-            X, self.stop_words_ = self._limit_features(X, vocabulary,
-                                                       max_doc_count,
-                                                       min_doc_count,
-                                                       max_features)
+            X = self._limit_features(
+                X, vocabulary, max_doc_count, min_doc_count, max_features
+            )
             if max_features is None:
                 X = self._sort_features(X, vocabulary)
             self.vocabulary_ = vocabulary
@@ -1246,7 +1404,7 @@ def transform(self, raw_documents):
         Parameters
         ----------
         raw_documents : iterable
-            An iterable which yields either str, unicode or file objects.
+            An iterable which generates either str, unicode or file objects.
 
         Returns
         -------
@@ -1255,8 +1413,8 @@ def transform(self, raw_documents):
         """
         if isinstance(raw_documents, str):
             raise ValueError(
-                "Iterable over raw text documents expected, "
-                "string object received.")
+                "Iterable over raw text documents expected, string object received."
+            )
         self._check_vocabulary()
 
         # use the same matrix-building strategy as fit_transform
@@ -1275,12 +1433,12 @@ def inverse_transform(self, X):
 
         Returns
         -------
-        X_inv : list of arrays of shape (n_samples,)
+        X_original : list of arrays of shape (n_samples,)
             List of arrays of terms.
         """
         self._check_vocabulary()
         # We need CSR format for fast row manipulations.
-        X = check_array(X, accept_sparse='csr')
+        X = check_array(X, accept_sparse="csr")
         n_samples = X.shape[0]
 
         terms = np.array(list(self.vocabulary_.keys()))
@@ -1288,28 +1446,40 @@ def inverse_transform(self, X):
         inverse_vocabulary = terms[np.argsort(indices)]
 
         if sp.issparse(X):
-            return [inverse_vocabulary[X[i, :].nonzero()[1]].ravel()
-                    for i in range(n_samples)]
+            return [
+                inverse_vocabulary[X[i, :].nonzero()[1]].ravel()
+                for i in range(n_samples)
+            ]
         else:
-            return [inverse_vocabulary[np.flatnonzero(X[i, :])].ravel()
-                    for i in range(n_samples)]
+            return [
+                inverse_vocabulary[np.flatnonzero(X[i, :])].ravel()
+                for i in range(n_samples)
+            ]
 
-    def get_feature_names(self):
-        """Array mapping from feature integer indices to feature name.
+    def get_feature_names_out(self, input_features=None):
+        """Get output feature names for transformation.
+
+        Parameters
+        ----------
+        input_features : array-like of str or None, default=None
+            Not used, present here for API consistency by convention.
 
         Returns
         -------
-        feature_names : list
-            A list of feature names.
+        feature_names_out : ndarray of str objects
+            Transformed feature names.
         """
-
         self._check_vocabulary()
+        return np.asarray(
+            [t for t, i in sorted(self.vocabulary_.items(), key=itemgetter(1))],
+            dtype=object,
+        )
 
-        return [t for t, i in sorted(self.vocabulary_.items(),
-                                     key=itemgetter(1))]
-
-    def _more_tags(self):
-        return {'X_types': ['string']}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.string = True
+        tags.input_tags.two_d_array = False
+        return tags
 
 
 def _make_int_array():
@@ -1317,8 +1487,10 @@ def _make_int_array():
     return array.array(str("i"))
 
 
-class TfidfTransformer(TransformerMixin, BaseEstimator):
-    """Transform a count matrix to a normalized tf or tf-idf representation
+class TfidfTransformer(
+    OneToOneFeatureMixin, TransformerMixin, BaseEstimator, auto_wrap_output_keys=None
+):
+    """Transform a count matrix to a normalized tf or tf-idf representation.
 
     Tf means term-frequency while tf-idf means term-frequency times inverse
     document-frequency. This is a common term weighting scheme in information
@@ -1362,16 +1534,18 @@ class TfidfTransformer(TransformerMixin, BaseEstimator):
 
     Parameters
     ----------
-    norm : {'l1', 'l2'}, default='l2'
+    norm : {'l1', 'l2'} or None, default='l2'
         Each output row will have unit norm, either:
-        * 'l2': Sum of squares of vector elements is 1. The cosine
-        similarity between two vectors is their dot product when l2 norm has
-        been applied.
-        * 'l1': Sum of absolute values of vector elements is 1.
-        See :func:`preprocessing.normalize`
+
+        - 'l2': Sum of squares of vector elements is 1. The cosine
+          similarity between two vectors is their dot product when l2 norm has
+          been applied.
+        - 'l1': Sum of absolute values of vector elements is 1.
+          See :func:`~sklearn.preprocessing.normalize`.
+        - None: No normalization.
 
     use_idf : bool, default=True
-        Enable inverse-document-frequency reweighting.
+        Enable inverse-document-frequency reweighting. If False, idf(t) = 1.
 
     smooth_idf : bool, default=True
         Smooth idf weights by adding one to document frequencies, as if an
@@ -1389,12 +1563,41 @@ class TfidfTransformer(TransformerMixin, BaseEstimator):
 
         .. versionadded:: 0.20
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 1.0
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    CountVectorizer : Transforms text into a sparse matrix of n-gram counts.
+
+    TfidfVectorizer : Convert a collection of raw documents to a matrix of
+        TF-IDF features.
+
+    HashingVectorizer : Convert a collection of text documents to a matrix
+        of token occurrences.
+
+    References
+    ----------
+    .. [Yates2011] R. Baeza-Yates and B. Ribeiro-Neto (2011). Modern
+                   Information Retrieval. Addison Wesley, pp. 68-74.
+
+    .. [MRS2008] C.D. Manning, P. Raghavan and H. SchÃŧtze  (2008).
+                   Introduction to Information Retrieval. Cambridge University
+                   Press, pp. 118-120.
+
     Examples
     --------
     >>> from sklearn.feature_extraction.text import TfidfTransformer
     >>> from sklearn.feature_extraction.text import CountVectorizer
     >>> from sklearn.pipeline import Pipeline
-    >>> import numpy as np
     >>> corpus = ['this is the first document',
     ...           'this document is the second document',
     ...           'and this is the third one',
@@ -1413,118 +1616,119 @@ class TfidfTransformer(TransformerMixin, BaseEstimator):
            1.        , 1.91629073, 1.91629073])
     >>> pipe.transform(corpus).shape
     (4, 8)
+    """
 
-    References
-    ----------
-
-    .. [Yates2011] R. Baeza-Yates and B. Ribeiro-Neto (2011). Modern
-                   Information Retrieval. Addison Wesley, pp. 68-74.
+    _parameter_constraints: dict = {
+        "norm": [StrOptions({"l1", "l2"}), None],
+        "use_idf": ["boolean"],
+        "smooth_idf": ["boolean"],
+        "sublinear_tf": ["boolean"],
+    }
 
-    .. [MRS2008] C.D. Manning, P. Raghavan and H. SchÃŧtze  (2008).
-                   Introduction to Information Retrieval. Cambridge University
-                   Press, pp. 118-120.
-    """
-    @_deprecate_positional_args
-    def __init__(self, *, norm='l2', use_idf=True, smooth_idf=True,
-                 sublinear_tf=False):
+    def __init__(self, *, norm="l2", use_idf=True, smooth_idf=True, sublinear_tf=False):
         self.norm = norm
         self.use_idf = use_idf
         self.smooth_idf = smooth_idf
         self.sublinear_tf = sublinear_tf
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y=None):
         """Learn the idf vector (global term weights).
 
         Parameters
         ----------
-        X : sparse matrix of shape n_samples, n_features)
+        X : sparse matrix of shape (n_samples, n_features)
             A matrix of term/token counts.
+
+        y : None
+            This parameter is not needed to compute tf-idf.
+
+        Returns
+        -------
+        self : object
+            Fitted transformer.
         """
-        X = check_array(X, accept_sparse=('csr', 'csc'))
+        # large sparse data is not supported for 32bit platforms because
+        # _document_frequency uses np.bincount which works on arrays of
+        # dtype NPY_INTP which is int32 for 32bit platforms. See #20923
+        X = validate_data(
+            self, X, accept_sparse=("csr", "csc"), accept_large_sparse=not _IS_32BIT
+        )
         if not sp.issparse(X):
             X = sp.csr_matrix(X)
-        dtype = X.dtype if X.dtype in FLOAT_DTYPES else np.float64
+        dtype = X.dtype if X.dtype in (np.float64, np.float32) else np.float64
 
         if self.use_idf:
-            n_samples, n_features = X.shape
+            n_samples, _ = X.shape
             df = _document_frequency(X)
-            df = df.astype(dtype, **_astype_copy_false(df))
+            df = df.astype(dtype, copy=False)
 
             # perform idf smoothing if required
-            df += int(self.smooth_idf)
+            df += float(self.smooth_idf)
             n_samples += int(self.smooth_idf)
 
             # log+1 instead of log makes sure terms with zero idf don't get
             # suppressed entirely.
-            idf = np.log(n_samples / df) + 1
-            self._idf_diag = sp.diags(idf, offsets=0,
-                                      shape=(n_features, n_features),
-                                      format='csr',
-                                      dtype=dtype)
+            # Force the dtype of `idf_` to be the same as `df`. In NumPy < 2, the dtype
+            # was depending on the value of `n_samples`.
+            self.idf_ = np.full_like(df, fill_value=n_samples, dtype=dtype)
+            self.idf_ /= df
+            # `np.log` preserves the dtype of `df` and thus `dtype`.
+            np.log(self.idf_, out=self.idf_)
+            self.idf_ += 1.0
 
         return self
 
     def transform(self, X, copy=True):
-        """Transform a count matrix to a tf or tf-idf representation
+        """Transform a count matrix to a tf or tf-idf representation.
 
         Parameters
         ----------
         X : sparse matrix of (n_samples, n_features)
-            a matrix of term/token counts
+            A matrix of term/token counts.
 
         copy : bool, default=True
             Whether to copy X and operate on the copy or perform in-place
-            operations.
+            operations. `copy=False` will only be effective with CSR sparse matrix.
 
         Returns
         -------
         vectors : sparse matrix of shape (n_samples, n_features)
+            Tf-idf-weighted document-term matrix.
         """
-        X = check_array(X, accept_sparse='csr', dtype=FLOAT_DTYPES, copy=copy)
+        check_is_fitted(self)
+        X = validate_data(
+            self,
+            X,
+            accept_sparse="csr",
+            dtype=[np.float64, np.float32],
+            copy=copy,
+            reset=False,
+        )
         if not sp.issparse(X):
-            X = sp.csr_matrix(X, dtype=np.float64)
-
-        n_samples, n_features = X.shape
+            X = sp.csr_matrix(X, dtype=X.dtype)
 
         if self.sublinear_tf:
             np.log(X.data, X.data)
-            X.data += 1
+            X.data += 1.0
 
-        if self.use_idf:
-            # idf_ being a property, the automatic attributes detection
-            # does not work as usual and we need to specify the attribute
-            # name:
-            check_is_fitted(self, attributes=["idf_"],
-                            msg='idf vector is not fitted')
-
-            expected_n_features = self._idf_diag.shape[0]
-            if n_features != expected_n_features:
-                raise ValueError("Input has n_features=%d while the model"
-                                 " has been trained with n_features=%d" % (
-                                     n_features, expected_n_features))
-            # *= doesn't work
-            X = X * self._idf_diag
-
-        if self.norm:
+        if hasattr(self, "idf_"):
+            # the columns of X (CSR matrix) can be accessed with `X.indices `and
+            # multiplied with the corresponding `idf` value
+            X.data *= self.idf_[X.indices]
+
+        if self.norm is not None:
             X = normalize(X, norm=self.norm, copy=False)
 
         return X
 
-    @property
-    def idf_(self):
-        # if _idf_diag is not set, this will raise an attribute error,
-        # which means hasattr(self, "idf_") is False
-        return np.ravel(self._idf_diag.sum(axis=0))
-
-    @idf_.setter
-    def idf_(self, value):
-        value = np.asarray(value, dtype=np.float64)
-        n_features = value.shape[0]
-        self._idf_diag = sp.spdiags(value, diags=0, m=n_features,
-                                    n=n_features, format='csr')
-
-    def _more_tags(self):
-        return {'X_types': 'sparse'}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
+        # FIXME: np.float16 could be preserved if _inplace_csr_row_normalize_l2
+        # accepted it.
+        tags.transformer_tags.preserves_dtype = ["float64", "float32"]
+        return tags
 
 
 class TfidfVectorizer(CountVectorizer):
@@ -1533,6 +1737,16 @@ class TfidfVectorizer(CountVectorizer):
     Equivalent to :class:`CountVectorizer` followed by
     :class:`TfidfTransformer`.
 
+    For an example of usage, see
+    :ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py`.
+
+    For an efficiency comparison of the different feature extractors, see
+    :ref:`sphx_glr_auto_examples_text_plot_hashing_vs_dict_vectorizer.py`.
+
+    For an example of document clustering and comparison with
+    :class:`~sklearn.feature_extraction.text.HashingVectorizer`, see
+    :ref:`sphx_glr_auto_examples_text_plot_document_clustering.py`.
+
     Read more in the :ref:`User Guide `.
 
     Parameters
@@ -1558,13 +1772,13 @@ class TfidfVectorizer(CountVectorizer):
         'strict', meaning that a UnicodeDecodeError will be raised. Other
         values are 'ignore' and 'replace'.
 
-    strip_accents : {'ascii', 'unicode'}, default=None
+    strip_accents : {'ascii', 'unicode'} or callable, default=None
         Remove accents and perform other character normalization
         during the preprocessing step.
         'ascii' is a fast method that only works on characters that have
-        an direct ASCII mapping.
+        a direct ASCII mapping.
         'unicode' is a slightly slower method that works on any characters.
-        None (default) does nothing.
+        None (default) means no character normalization is performed.
 
         Both 'ascii' and 'unicode' use NFKD normalization from
         :func:`unicodedata.normalize`.
@@ -1575,7 +1789,7 @@ class TfidfVectorizer(CountVectorizer):
     preprocessor : callable, default=None
         Override the preprocessing (string transformation) stage while
         preserving the tokenizing and n-grams generation steps.
-        Only applies if ``analyzer is not callable``.
+        Only applies if ``analyzer`` is not callable.
 
     tokenizer : callable, default=None
         Override the string tokenization step while preserving the
@@ -1606,9 +1820,9 @@ class TfidfVectorizer(CountVectorizer):
         will be removed from the resulting tokens.
         Only applies if ``analyzer == 'word'``.
 
-        If None, no stop words will be used. max_df can be set to a value
-        in the range [0.7, 1.0) to automatically detect and filter stop
-        words based on intra corpus document frequency of terms.
+        If None, no stop words will be used. In this case, setting `max_df`
+        to a higher value, such as in the range (0.7, 1.0), can automatically detect
+        and filter stop words based on intra corpus document frequency of terms.
 
     token_pattern : str, default=r"(?u)\\b\\w\\w+\\b"
         Regular expression denoting what constitutes a "token", only used
@@ -1626,7 +1840,7 @@ class TfidfVectorizer(CountVectorizer):
         will be used. For example an ``ngram_range`` of ``(1, 1)`` means only
         unigrams, ``(1, 2)`` means unigrams and bigrams, and ``(2, 2)`` means
         only bigrams.
-        Only applies if ``analyzer is not callable``.
+        Only applies if ``analyzer`` is not callable.
 
     max_df : float or int, default=1.0
         When building the vocabulary ignore terms that have a document
@@ -1646,7 +1860,8 @@ class TfidfVectorizer(CountVectorizer):
 
     max_features : int, default=None
         If not None, build a vocabulary that only consider the top
-        max_features ordered by term frequency across the corpus.
+        `max_features` ordered by term frequency across the corpus.
+        Otherwise, all features are used.
 
         This parameter is ignored if vocabulary is not None.
 
@@ -1658,21 +1873,24 @@ class TfidfVectorizer(CountVectorizer):
     binary : bool, default=False
         If True, all non-zero term counts are set to 1. This does not mean
         outputs will have only 0/1 values, only that the tf term in tf-idf
-        is binary. (Set idf and normalization to False to get 0/1 outputs).
+        is binary. (Set `binary` to True, `use_idf` to False and
+        `norm` to None to get 0/1 outputs).
 
     dtype : dtype, default=float64
         Type of the matrix returned by fit_transform() or transform().
 
-    norm : {'l1', 'l2'}, default='l2'
+    norm : {'l1', 'l2'} or None, default='l2'
         Each output row will have unit norm, either:
-        * 'l2': Sum of squares of vector elements is 1. The cosine
-        similarity between two vectors is their dot product when l2 norm has
-        been applied.
-        * 'l1': Sum of absolute values of vector elements is 1.
-        See :func:`preprocessing.normalize`.
+
+        - 'l2': Sum of squares of vector elements is 1. The cosine
+          similarity between two vectors is their dot product when l2 norm has
+          been applied.
+        - 'l1': Sum of absolute values of vector elements is 1.
+          See :func:`~sklearn.preprocessing.normalize`.
+        - None: No normalization.
 
     use_idf : bool, default=True
-        Enable inverse-document-frequency reweighting.
+        Enable inverse-document-frequency reweighting. If False, idf(t) = 1.
 
     smooth_idf : bool, default=True
         Smooth idf weights by adding one to document frequencies, as if an
@@ -1687,23 +1905,14 @@ class TfidfVectorizer(CountVectorizer):
     vocabulary_ : dict
         A mapping of terms to feature indices.
 
-    fixed_vocabulary_: bool
+    fixed_vocabulary_ : bool
         True if a fixed vocabulary of term to indices mapping
-        is provided by the user
+        is provided by the user.
 
     idf_ : array of shape (n_features,)
         The inverse document frequency (IDF) vector; only defined
         if ``use_idf`` is True.
 
-    stop_words_ : set
-        Terms that were ignored because they either:
-
-          - occurred in too many documents (`max_df`)
-          - occurred in too few documents (`min_df`)
-          - were cut off by feature selection (`max_features`).
-
-        This is only available if no vocabulary was given.
-
     See Also
     --------
     CountVectorizer : Transforms text into a sparse matrix of n-gram counts.
@@ -1711,12 +1920,6 @@ class TfidfVectorizer(CountVectorizer):
     TfidfTransformer : Performs the TF-IDF transformation from a provided
         matrix of counts.
 
-    Notes
-    -----
-    The ``stop_words_`` attribute can get large and increase the model size
-    when pickling. This attribute is provided only for introspection and can
-    be safely removed using delattr or set to None before pickling.
-
     Examples
     --------
     >>> from sklearn.feature_extraction.text import TfidfVectorizer
@@ -1728,97 +1931,130 @@ class TfidfVectorizer(CountVectorizer):
     ... ]
     >>> vectorizer = TfidfVectorizer()
     >>> X = vectorizer.fit_transform(corpus)
-    >>> print(vectorizer.get_feature_names())
-    ['and', 'document', 'first', 'is', 'one', 'second', 'the', 'third', 'this']
+    >>> vectorizer.get_feature_names_out()
+    array(['and', 'document', 'first', 'is', 'one', 'second', 'the', 'third',
+           'this'], ...)
     >>> print(X.shape)
     (4, 9)
     """
-    @_deprecate_positional_args
-    def __init__(self, *, input='content', encoding='utf-8',
-                 decode_error='strict', strip_accents=None, lowercase=True,
-                 preprocessor=None, tokenizer=None, analyzer='word',
-                 stop_words=None, token_pattern=r"(?u)\b\w\w+\b",
-                 ngram_range=(1, 1), max_df=1.0, min_df=1,
-                 max_features=None, vocabulary=None, binary=False,
-                 dtype=np.float64, norm='l2', use_idf=True, smooth_idf=True,
-                 sublinear_tf=False):
 
+    _parameter_constraints: dict = {**CountVectorizer._parameter_constraints}
+    _parameter_constraints.update(
+        {
+            "norm": [StrOptions({"l1", "l2"}), None],
+            "use_idf": ["boolean"],
+            "smooth_idf": ["boolean"],
+            "sublinear_tf": ["boolean"],
+        }
+    )
+
+    def __init__(
+        self,
+        *,
+        input="content",
+        encoding="utf-8",
+        decode_error="strict",
+        strip_accents=None,
+        lowercase=True,
+        preprocessor=None,
+        tokenizer=None,
+        analyzer="word",
+        stop_words=None,
+        token_pattern=r"(?u)\b\w\w+\b",
+        ngram_range=(1, 1),
+        max_df=1.0,
+        min_df=1,
+        max_features=None,
+        vocabulary=None,
+        binary=False,
+        dtype=np.float64,
+        norm="l2",
+        use_idf=True,
+        smooth_idf=True,
+        sublinear_tf=False,
+    ):
         super().__init__(
-            input=input, encoding=encoding, decode_error=decode_error,
-            strip_accents=strip_accents, lowercase=lowercase,
-            preprocessor=preprocessor, tokenizer=tokenizer, analyzer=analyzer,
-            stop_words=stop_words, token_pattern=token_pattern,
-            ngram_range=ngram_range, max_df=max_df, min_df=min_df,
-            max_features=max_features, vocabulary=vocabulary, binary=binary,
-            dtype=dtype)
-
-        self._tfidf = TfidfTransformer(norm=norm, use_idf=use_idf,
-                                       smooth_idf=smooth_idf,
-                                       sublinear_tf=sublinear_tf)
+            input=input,
+            encoding=encoding,
+            decode_error=decode_error,
+            strip_accents=strip_accents,
+            lowercase=lowercase,
+            preprocessor=preprocessor,
+            tokenizer=tokenizer,
+            analyzer=analyzer,
+            stop_words=stop_words,
+            token_pattern=token_pattern,
+            ngram_range=ngram_range,
+            max_df=max_df,
+            min_df=min_df,
+            max_features=max_features,
+            vocabulary=vocabulary,
+            binary=binary,
+            dtype=dtype,
+        )
+        self.norm = norm
+        self.use_idf = use_idf
+        self.smooth_idf = smooth_idf
+        self.sublinear_tf = sublinear_tf
 
     # Broadcast the TF-IDF parameters to the underlying transformer instance
     # for easy grid search and repr
 
-    @property
-    def norm(self):
-        return self._tfidf.norm
-
-    @norm.setter
-    def norm(self, value):
-        self._tfidf.norm = value
-
-    @property
-    def use_idf(self):
-        return self._tfidf.use_idf
-
-    @use_idf.setter
-    def use_idf(self, value):
-        self._tfidf.use_idf = value
-
-    @property
-    def smooth_idf(self):
-        return self._tfidf.smooth_idf
-
-    @smooth_idf.setter
-    def smooth_idf(self, value):
-        self._tfidf.smooth_idf = value
-
-    @property
-    def sublinear_tf(self):
-        return self._tfidf.sublinear_tf
-
-    @sublinear_tf.setter
-    def sublinear_tf(self, value):
-        self._tfidf.sublinear_tf = value
-
     @property
     def idf_(self):
+        """Inverse document frequency vector, only defined if `use_idf=True`.
+
+        Returns
+        -------
+        ndarray of shape (n_features,)
+        """
+        if not hasattr(self, "_tfidf"):
+            raise NotFittedError(
+                f"{self.__class__.__name__} is not fitted yet. Call 'fit' with "
+                "appropriate arguments before using this attribute."
+            )
         return self._tfidf.idf_
 
     @idf_.setter
     def idf_(self, value):
+        if not self.use_idf:
+            raise ValueError("`idf_` cannot be set when `user_idf=False`.")
+        if not hasattr(self, "_tfidf"):
+            # We should support transferring `idf_` from another `TfidfTransformer`
+            # and therefore, we need to create the transformer instance it does not
+            # exist yet.
+            self._tfidf = TfidfTransformer(
+                norm=self.norm,
+                use_idf=self.use_idf,
+                smooth_idf=self.smooth_idf,
+                sublinear_tf=self.sublinear_tf,
+            )
         self._validate_vocabulary()
-        if hasattr(self, 'vocabulary_'):
+        if hasattr(self, "vocabulary_"):
             if len(self.vocabulary_) != len(value):
-                raise ValueError("idf length = %d must be equal "
-                                 "to vocabulary size = %d" %
-                                 (len(value), len(self.vocabulary)))
+                raise ValueError(
+                    "idf length = %d must be equal to vocabulary size = %d"
+                    % (len(value), len(self.vocabulary))
+                )
         self._tfidf.idf_ = value
 
     def _check_params(self):
         if self.dtype not in FLOAT_DTYPES:
-            warnings.warn("Only {} 'dtype' should be used. {} 'dtype' will "
-                          "be converted to np.float64."
-                          .format(FLOAT_DTYPES, self.dtype),
-                          UserWarning)
+            warnings.warn(
+                "Only {} 'dtype' should be used. {} 'dtype' will "
+                "be converted to np.float64.".format(FLOAT_DTYPES, self.dtype),
+                UserWarning,
+            )
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, raw_documents, y=None):
         """Learn vocabulary and idf from training set.
 
         Parameters
         ----------
         raw_documents : iterable
-            An iterable which yields either str, unicode or file objects.
+            An iterable which generates either str, unicode or file objects.
+
         y : None
             This parameter is not needed to compute tfidf.
 
@@ -1829,6 +2065,12 @@ def fit(self, raw_documents, y=None):
         """
         self._check_params()
         self._warn_for_unused_params()
+        self._tfidf = TfidfTransformer(
+            norm=self.norm,
+            use_idf=self.use_idf,
+            smooth_idf=self.smooth_idf,
+            sublinear_tf=self.sublinear_tf,
+        )
         X = super().fit_transform(raw_documents)
         self._tfidf.fit(X)
         return self
@@ -1842,7 +2084,8 @@ def fit_transform(self, raw_documents, y=None):
         Parameters
         ----------
         raw_documents : iterable
-            An iterable which yields either str, unicode or file objects.
+            An iterable which generates either str, unicode or file objects.
+
         y : None
             This parameter is ignored.
 
@@ -1852,6 +2095,12 @@ def fit_transform(self, raw_documents, y=None):
             Tf-idf-weighted document-term matrix.
         """
         self._check_params()
+        self._tfidf = TfidfTransformer(
+            norm=self.norm,
+            use_idf=self.use_idf,
+            smooth_idf=self.smooth_idf,
+            sublinear_tf=self.sublinear_tf,
+        )
         X = super().fit_transform(raw_documents)
         self._tfidf.fit(X)
         # X is already a transformed view of raw_documents so
@@ -1867,17 +2116,21 @@ def transform(self, raw_documents):
         Parameters
         ----------
         raw_documents : iterable
-            An iterable which yields either str, unicode or file objects.
+            An iterable which generates either str, unicode or file objects.
 
         Returns
         -------
         X : sparse matrix of (n_samples, n_features)
             Tf-idf-weighted document-term matrix.
         """
-        check_is_fitted(self, msg='The TF-IDF vectorizer is not fitted')
+        check_is_fitted(self, msg="The TF-IDF vectorizer is not fitted")
 
         X = super().transform(raw_documents)
         return self._tfidf.transform(X, copy=False)
 
-    def _more_tags(self):
-        return {'X_types': ['string'], '_skip_test': True}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.string = True
+        tags.input_tags.two_d_array = False
+        tags._skip_test = True
+        return tags
diff --git a/sklearn/feature_selection/__init__.py b/sklearn/feature_selection/__init__.py
index 86e8a2af39084..d0d2dcee909f4 100644
--- a/sklearn/feature_selection/__init__.py
+++ b/sklearn/feature_selection/__init__.py
@@ -1,49 +1,50 @@
-"""
-The :mod:`sklearn.feature_selection` module implements feature selection
-algorithms. It currently includes univariate filter selection methods and the
-recursive feature elimination algorithm.
-"""
-
-from ._univariate_selection import chi2
-from ._univariate_selection import f_classif
-from ._univariate_selection import f_oneway
-from ._univariate_selection import f_regression
-from ._univariate_selection import SelectPercentile
-from ._univariate_selection import SelectKBest
-from ._univariate_selection import SelectFpr
-from ._univariate_selection import SelectFdr
-from ._univariate_selection import SelectFwe
-from ._univariate_selection import GenericUnivariateSelect
+"""Feature selection algorithms.
 
-from ._variance_threshold import VarianceThreshold
+These include univariate filter selection methods and the recursive feature elimination
+algorithm.
+"""
 
-from ._rfe import RFE
-from ._rfe import RFECV
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
+from ._base import SelectorMixin
 from ._from_model import SelectFromModel
-
+from ._mutual_info import mutual_info_classif, mutual_info_regression
+from ._rfe import RFE, RFECV
 from ._sequential import SequentialFeatureSelector
+from ._univariate_selection import (
+    GenericUnivariateSelect,
+    SelectFdr,
+    SelectFpr,
+    SelectFwe,
+    SelectKBest,
+    SelectPercentile,
+    chi2,
+    f_classif,
+    f_oneway,
+    f_regression,
+    r_regression,
+)
+from ._variance_threshold import VarianceThreshold
 
-from ._mutual_info import mutual_info_regression, mutual_info_classif
-
-from ._base import SelectorMixin
-
-
-__all__ = ['GenericUnivariateSelect',
-           'SequentialFeatureSelector',
-           'RFE',
-           'RFECV',
-           'SelectFdr',
-           'SelectFpr',
-           'SelectFwe',
-           'SelectKBest',
-           'SelectFromModel',
-           'SelectPercentile',
-           'VarianceThreshold',
-           'chi2',
-           'f_classif',
-           'f_oneway',
-           'f_regression',
-           'mutual_info_classif',
-           'mutual_info_regression',
-           'SelectorMixin']
+__all__ = [
+    "RFE",
+    "RFECV",
+    "GenericUnivariateSelect",
+    "SelectFdr",
+    "SelectFpr",
+    "SelectFromModel",
+    "SelectFwe",
+    "SelectKBest",
+    "SelectPercentile",
+    "SelectorMixin",
+    "SequentialFeatureSelector",
+    "VarianceThreshold",
+    "chi2",
+    "f_classif",
+    "f_oneway",
+    "f_regression",
+    "mutual_info_classif",
+    "mutual_info_regression",
+    "r_regression",
+]
diff --git a/sklearn/feature_selection/_base.py b/sklearn/feature_selection/_base.py
index 4f0756e7ee020..56e50e49ca30c 100644
--- a/sklearn/feature_selection/_base.py
+++ b/sklearn/feature_selection/_base.py
@@ -1,23 +1,25 @@
-# -*- coding: utf-8 -*-
 """Generic feature selection mixin"""
 
-# Authors: G. Varoquaux, A. Gramfort, L. Buitinck, J. Nothman
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
+import warnings
 from abc import ABCMeta, abstractmethod
-from warnings import warn
 from operator import attrgetter
 
 import numpy as np
-from scipy.sparse import issparse, csc_matrix
+from scipy.sparse import csc_matrix, issparse
 
 from ..base import TransformerMixin
-from ..utils import (
-    check_array,
-    safe_mask,
-    safe_sqr,
+from ..utils import _safe_indexing, check_array, safe_sqr
+from ..utils._set_output import _get_output_config
+from ..utils._tags import get_tags
+from ..utils.validation import (
+    _check_feature_names_in,
+    _is_pandas_df,
+    check_is_fitted,
+    validate_data,
 )
-from ..utils._tags import _safe_tags
 
 
 class SelectorMixin(TransformerMixin, metaclass=ABCMeta):
@@ -27,11 +29,29 @@ class SelectorMixin(TransformerMixin, metaclass=ABCMeta):
     This mixin provides a feature selector implementation with `transform` and
     `inverse_transform` functionality given an implementation of
     `_get_support_mask`.
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from sklearn.datasets import load_iris
+    >>> from sklearn.base import BaseEstimator
+    >>> from sklearn.feature_selection import SelectorMixin
+    >>> class FeatureSelector(SelectorMixin, BaseEstimator):
+    ...    def fit(self, X, y=None):
+    ...        self.n_features_in_ = X.shape[1]
+    ...        return self
+    ...    def _get_support_mask(self):
+    ...        mask = np.zeros(self.n_features_in_, dtype=bool)
+    ...        mask[:2] = True  # select the first two features
+    ...        return mask
+    >>> X, y = load_iris(return_X_y=True)
+    >>> FeatureSelector().fit_transform(X, y).shape
+    (150, 2)
     """
 
     def get_support(self, indices=False):
         """
-        Get a mask, or integer index, of the features selected
+        Get a mask, or integer index, of the features selected.
 
         Parameters
         ----------
@@ -50,7 +70,7 @@ def get_support(self, indices=False):
             values are indices into the input feature vector.
         """
         mask = self._get_support_mask()
-        return mask if not indices else np.where(mask)[0]
+        return mask if not indices else np.nonzero(mask)[0]
 
     @abstractmethod
     def _get_support_mask(self):
@@ -77,28 +97,42 @@ def transform(self, X):
         X_r : array of shape [n_samples, n_selected_features]
             The input samples with only the selected features.
         """
-        # note: we use _safe_tags instead of _get_tags because this is a
+        # Preserve X when X is a dataframe and the output is configured to
+        # be pandas.
+        output_config_dense = _get_output_config("transform", estimator=self)["dense"]
+        preserve_X = output_config_dense != "default" and _is_pandas_df(X)
+
+        # note: we use get_tags instead of __sklearn_tags__ because this is a
         # public Mixin.
-        X = self._validate_data(
+        X = validate_data(
+            self,
             X,
             dtype=None,
             accept_sparse="csr",
-            force_all_finite=not _safe_tags(self, key="allow_nan"),
+            ensure_all_finite=not get_tags(self).input_tags.allow_nan,
+            skip_check_array=preserve_X,
             reset=False,
         )
+        return self._transform(X)
+
+    def _transform(self, X):
+        """Reduce X to the selected features."""
         mask = self.get_support()
         if not mask.any():
-            warn("No features were selected: either the data is"
-                 " too noisy or the selection test too strict.",
-                 UserWarning)
-            return np.empty(0).reshape((X.shape[0], 0))
-        if len(mask) != X.shape[1]:
-            raise ValueError("X has a different shape than during fitting.")
-        return X[:, safe_mask(X, mask)]
+            warnings.warn(
+                (
+                    "No features were selected: either the data is"
+                    " too noisy or the selection test too strict."
+                ),
+                UserWarning,
+            )
+            if hasattr(X, "iloc"):
+                return X.iloc[:, :0]
+            return np.empty(0, dtype=X.dtype).reshape((X.shape[0], 0))
+        return _safe_indexing(X, mask, axis=1)
 
     def inverse_transform(self, X):
-        """
-        Reverse the transformation operation
+        """Reverse the transformation operation.
 
         Parameters
         ----------
@@ -107,7 +141,7 @@ def inverse_transform(self, X):
 
         Returns
         -------
-        X_r : array of shape [n_samples, n_original_features]
+        X_original : array of shape [n_samples, n_original_features]
             `X` with columns of zeros inserted where features would have
             been removed by :meth:`transform`.
         """
@@ -119,8 +153,11 @@ def inverse_transform(self, X):
             it = self.inverse_transform(np.diff(X.indptr).reshape(1, -1))
             col_nonzeros = it.ravel()
             indptr = np.concatenate([[0], np.cumsum(col_nonzeros)])
-            Xt = csc_matrix((X.data, X.indices, indptr),
-                            shape=(X.shape[0], len(indptr) - 1), dtype=X.dtype)
+            Xt = csc_matrix(
+                (X.data, X.indices, indptr),
+                shape=(X.shape[0], len(indptr) - 1),
+                dtype=X.dtype,
+            )
             return Xt
 
         support = self.get_support()
@@ -134,9 +171,32 @@ def inverse_transform(self, X):
         Xt[:, support] = X
         return Xt
 
+    def get_feature_names_out(self, input_features=None):
+        """Mask feature names according to selected features.
 
-def _get_feature_importances(estimator, getter, transform_func=None,
-                             norm_order=1):
+        Parameters
+        ----------
+        input_features : array-like of str or None, default=None
+            Input features.
+
+            - If `input_features` is `None`, then `feature_names_in_` is
+              used as feature names in. If `feature_names_in_` is not defined,
+              then the following input feature names are generated:
+              `["x0", "x1", ..., "x(n_features_in_ - 1)"]`.
+            - If `input_features` is an array-like, then `input_features` must
+              match `feature_names_in_` if `feature_names_in_` is defined.
+
+        Returns
+        -------
+        feature_names_out : ndarray of str objects
+            Transformed feature names.
+        """
+        check_is_fitted(self)
+        input_features = _check_feature_names_in(self, input_features)
+        return input_features[self.get_support()]
+
+
+def _get_feature_importances(estimator, getter, transform_func=None, norm_order=1):
     """
     Retrieve and aggregate (ndim > 1)  the feature importances
     from an estimator. Also optionally applies transformation.
@@ -165,25 +225,24 @@ def _get_feature_importances(estimator, getter, transform_func=None,
         The features importances, optionally transformed.
     """
     if isinstance(getter, str):
-        if getter == 'auto':
-            if hasattr(estimator, 'coef_'):
-                getter = attrgetter('coef_')
-            elif hasattr(estimator, 'feature_importances_'):
-                getter = attrgetter('feature_importances_')
+        if getter == "auto":
+            if hasattr(estimator, "coef_"):
+                getter = attrgetter("coef_")
+            elif hasattr(estimator, "feature_importances_"):
+                getter = attrgetter("feature_importances_")
             else:
                 raise ValueError(
-                    f"when `importance_getter=='auto'`, the underlying "
+                    "when `importance_getter=='auto'`, the underlying "
                     f"estimator {estimator.__class__.__name__} should have "
-                    f"`coef_` or `feature_importances_` attribute. Either "
-                    f"pass a fitted estimator to feature selector or call fit "
-                    f"before calling transform."
+                    "`coef_` or `feature_importances_` attribute. Either "
+                    "pass a fitted estimator to feature selector or call fit "
+                    "before calling transform."
                 )
         else:
             getter = attrgetter(getter)
     elif not callable(getter):
-        raise ValueError(
-            '`importance_getter` has to be a string or `callable`'
-        )
+        raise ValueError("`importance_getter` has to be a string or `callable`")
+
     importances = getter(estimator)
 
     if transform_func is None:
@@ -192,16 +251,17 @@ def _get_feature_importances(estimator, getter, transform_func=None,
         if importances.ndim == 1:
             importances = np.abs(importances)
         else:
-            importances = np.linalg.norm(importances, axis=0,
-                                         ord=norm_order)
+            importances = np.linalg.norm(importances, axis=0, ord=norm_order)
     elif transform_func == "square":
         if importances.ndim == 1:
             importances = safe_sqr(importances)
         else:
             importances = safe_sqr(importances).sum(axis=0)
     else:
-        raise ValueError("Valid values for `transform_func` are " +
-                         "None, 'norm' and 'square'. Those two " +
-                         "transformation are only supported now")
+        raise ValueError(
+            "Valid values for `transform_func` are "
+            "None, 'norm' and 'square'. Those two "
+            "transformation are only supported now"
+        )
 
     return importances
diff --git a/sklearn/feature_selection/_from_model.py b/sklearn/feature_selection/_from_model.py
index 4889f73518fe9..3b2c73c6cbfae 100644
--- a/sklearn/feature_selection/_from_model.py
+++ b/sklearn/feature_selection/_from_model.py
@@ -1,18 +1,30 @@
-# Authors: Gilles Louppe, Mathieu Blondel, Maheshakya Wijewardena
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-import numpy as np
-import numbers
+from copy import deepcopy
+from numbers import Integral, Real
 
-from ._base import SelectorMixin
-from ._base import _get_feature_importances
-from ..base import BaseEstimator, clone, MetaEstimatorMixin
-from ..utils._tags import _safe_tags
-from ..utils.validation import check_is_fitted
+import numpy as np
 
+from ..base import BaseEstimator, MetaEstimatorMixin, _fit_context, clone
 from ..exceptions import NotFittedError
-from ..utils.metaestimators import if_delegate_has_method
-from ..utils.validation import _deprecate_positional_args
+from ..utils._param_validation import HasMethods, Interval, Options
+from ..utils._tags import get_tags
+from ..utils.metadata_routing import (
+    MetadataRouter,
+    MethodMapping,
+    _routing_enabled,
+    process_routing,
+)
+from ..utils.metaestimators import available_if
+from ..utils.validation import (
+    _check_feature_names,
+    _estimator_has,
+    _num_features,
+    check_is_fitted,
+    check_scalar,
+)
+from ._base import SelectorMixin, _get_feature_importances
 
 
 def _calculate_threshold(estimator, importances, threshold):
@@ -21,8 +33,20 @@ def _calculate_threshold(estimator, importances, threshold):
     if threshold is None:
         # determine default from estimator
         est_name = estimator.__class__.__name__
-        if ((hasattr(estimator, "penalty") and estimator.penalty == "l1") or
-                "Lasso" in est_name):
+        is_l1_penalized = hasattr(estimator, "penalty") and estimator.penalty == "l1"
+        is_lasso = "Lasso" in est_name
+        is_elasticnet_l1_penalized = est_name == "ElasticNet" and (
+            hasattr(estimator, "l1_ratio") and np.isclose(estimator.l1_ratio, 1.0)
+        )
+        is_elasticnetcv_l1_penalized = est_name == "ElasticNetCV" and (
+            hasattr(estimator, "l1_ratio_") and np.isclose(estimator.l1_ratio_, 1.0)
+        )
+        if (
+            is_l1_penalized
+            or is_lasso
+            or is_elasticnet_l1_penalized
+            or is_elasticnetcv_l1_penalized
+        ):
             # the natural default threshold is 0 when l1 penalty was used
             threshold = 1e-5
         else:
@@ -50,8 +74,9 @@ def _calculate_threshold(estimator, importances, threshold):
             threshold = np.mean(importances)
 
         else:
-            raise ValueError("Expected threshold='mean' or threshold='median' "
-                             "got %s" % threshold)
+            raise ValueError(
+                "Expected threshold='mean' or threshold='median' got %s" % threshold
+            )
 
     else:
         threshold = float(threshold)
@@ -75,11 +100,11 @@ class SelectFromModel(MetaEstimatorMixin, SelectorMixin, BaseEstimator):
         ``feature_importances_`` or ``coef_`` attribute after fitting.
         Otherwise, the ``importance_getter`` parameter should be used.
 
-    threshold : string or float, default=None
+    threshold : str or float, default=None
         The threshold value to use for feature selection. Features whose
-        importance is greater or equal are kept while the others are
-        discarded. If "median" (resp. "mean"), then the ``threshold`` value is
-        the median (resp. the mean) of the feature importances. A scaling
+        absolute importance value is greater or equal are kept while the others
+        are discarded. If "median" (resp. "mean"), then the ``threshold`` value
+        is the median (resp. the mean) of the feature importances. A scaling
         factor (e.g., "1.25*mean") may also be used. If None and if the
         estimator has a parameter penalty set to l1, either explicitly
         or implicitly (e.g, Lasso), the threshold used is 1e-5.
@@ -87,22 +112,30 @@ class SelectFromModel(MetaEstimatorMixin, SelectorMixin, BaseEstimator):
 
     prefit : bool, default=False
         Whether a prefit model is expected to be passed into the constructor
-        directly or not. If True, ``transform`` must be called directly
-        and SelectFromModel cannot be used with ``cross_val_score``,
-        ``GridSearchCV`` and similar utilities that clone the estimator.
-        Otherwise train the model using ``fit`` and then ``transform`` to do
-        feature selection.
+        directly or not.
+        If `True`, `estimator` must be a fitted estimator.
+        If `False`, `estimator` is fitted and updated by calling
+        `fit` and `partial_fit`, respectively.
 
     norm_order : non-zero int, inf, -inf, default=1
         Order of the norm used to filter the vectors of coefficients below
         ``threshold`` in the case where the ``coef_`` attribute of the
         estimator is of dimension 2.
 
-    max_features : int, default=None
+    max_features : int, callable, default=None
         The maximum number of features to select.
+
+        - If an integer, then it specifies the maximum number of features to
+          allow.
+        - If a callable, then it specifies how to calculate the maximum number of
+          features allowed by using the output of `max_features(X)`.
+        - If `None`, then all features are kept.
+
         To only select based on ``max_features``, set ``threshold=-np.inf``.
 
         .. versionadded:: 0.20
+        .. versionchanged:: 1.1
+           `max_features` accepts a callable.
 
     importance_getter : str or callable, default='auto'
         If 'auto', uses the feature importance either through a ``coef_``
@@ -123,14 +156,46 @@ class SelectFromModel(MetaEstimatorMixin, SelectorMixin, BaseEstimator):
 
     Attributes
     ----------
-    estimator_ : an estimator
-        The base estimator from which the transformer is built.
-        This is stored only when a non-fitted estimator is passed to the
-        ``SelectFromModel``, i.e when prefit is False.
+    estimator_ : estimator
+        The base estimator from which the transformer is built. This attribute
+        exist only when `fit` has been called.
+
+        - If `prefit=True`, it is a deep copy of `estimator`.
+        - If `prefit=False`, it is a clone of `estimator` and fit on the data
+          passed to `fit` or `partial_fit`.
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`. Only defined if the
+        underlying estimator exposes such an attribute when fit.
+
+        .. versionadded:: 0.24
+
+    max_features_ : int
+        Maximum number of features calculated during :term:`fit`. Only defined
+        if the ``max_features`` is not `None`.
+
+        - If `max_features` is an `int`, then `max_features_ = max_features`.
+        - If `max_features` is a callable, then `max_features_ = max_features(X)`.
+
+        .. versionadded:: 1.1
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
 
     threshold_ : float
         The threshold value used for feature selection.
 
+    See Also
+    --------
+    RFE : Recursive feature elimination based on importance weights.
+    RFECV : Recursive feature elimination with built-in cross-validated
+        selection of the best number of features.
+    SequentialFeatureSelector : Sequential cross-validation based feature
+        selection. Does not rely on importance weights.
+
     Notes
     -----
     Allows NaN/Inf in the input if the underlying estimator does as well.
@@ -146,9 +211,9 @@ class SelectFromModel(MetaEstimatorMixin, SelectorMixin, BaseEstimator):
     >>> y = [0, 1, 0, 1]
     >>> selector = SelectFromModel(estimator=LogisticRegression()).fit(X, y)
     >>> selector.estimator_.coef_
-    array([[-0.3252302 ,  0.83462377,  0.49750423]])
+    array([[-0.3252,  0.8345,  0.4976]])
     >>> selector.threshold_
-    0.55245...
+    np.float64(0.55249)
     >>> selector.get_support()
     array([False,  True, False])
     >>> selector.transform(X)
@@ -157,18 +222,41 @@ class SelectFromModel(MetaEstimatorMixin, SelectorMixin, BaseEstimator):
            [-0.48],
            [ 1.48]])
 
-    See Also
-    --------
-    RFE : Recursive feature elimination based on importance weights.
-    RFECV : Recursive feature elimination with built-in cross-validated
-        selection of the best number of features.
-    SequentialFeatureSelector : Sequential cross-validation based feature
-        selection. Does not rely on importance weights.
+    Using a callable to create a selector that can use no more than half
+    of the input features.
+
+    >>> def half_callable(X):
+    ...     return round(len(X[0]) / 2)
+    >>> half_selector = SelectFromModel(estimator=LogisticRegression(),
+    ...                                 max_features=half_callable)
+    >>> _ = half_selector.fit(X, y)
+    >>> half_selector.max_features_
+    2
     """
-    @_deprecate_positional_args
-    def __init__(self, estimator, *, threshold=None, prefit=False,
-                 norm_order=1, max_features=None,
-                 importance_getter='auto'):
+
+    _parameter_constraints: dict = {
+        "estimator": [HasMethods("fit")],
+        "threshold": [Interval(Real, None, None, closed="both"), str, None],
+        "prefit": ["boolean"],
+        "norm_order": [
+            Interval(Integral, None, -1, closed="right"),
+            Interval(Integral, 1, None, closed="left"),
+            Options(Real, {np.inf, -np.inf}),
+        ],
+        "max_features": [Interval(Integral, 0, None, closed="left"), callable, None],
+        "importance_getter": [str, callable],
+    }
+
+    def __init__(
+        self,
+        estimator,
+        *,
+        threshold=None,
+        prefit=False,
+        norm_order=1,
+        max_features=None,
+        importance_getter="auto",
+    ):
         self.estimator = estimator
         self.threshold = threshold
         self.prefit = prefit
@@ -177,29 +265,69 @@ def __init__(self, estimator, *, threshold=None, prefit=False,
         self.max_features = max_features
 
     def _get_support_mask(self):
-        # SelectFromModel can directly call on transform.
+        estimator = getattr(self, "estimator_", self.estimator)
+        max_features = getattr(self, "max_features_", self.max_features)
+
         if self.prefit:
-            estimator = self.estimator
-        elif hasattr(self, 'estimator_'):
-            estimator = self.estimator_
-        else:
-            raise ValueError('Either fit the model before transform or set'
-                             ' "prefit=True" while passing the fitted'
-                             ' estimator to the constructor.')
+            try:
+                check_is_fitted(self.estimator)
+            except NotFittedError as exc:
+                raise NotFittedError(
+                    "When `prefit=True`, `estimator` is expected to be a fitted "
+                    "estimator."
+                ) from exc
+        if callable(max_features):
+            # This branch is executed when `transform` is called directly and thus
+            # `max_features_` is not set and we fallback using `self.max_features`
+            # that is not validated
+            raise NotFittedError(
+                "When `prefit=True` and `max_features` is a callable, call `fit` "
+                "before calling `transform`."
+            )
+        elif max_features is not None and not isinstance(max_features, Integral):
+            raise ValueError(
+                f"`max_features` must be an integer. Got `max_features={max_features}` "
+                "instead."
+            )
+
         scores = _get_feature_importances(
-            estimator=estimator, getter=self.importance_getter,
-            transform_func='norm', norm_order=self.norm_order)
+            estimator=estimator,
+            getter=self.importance_getter,
+            transform_func="norm",
+            norm_order=self.norm_order,
+        )
         threshold = _calculate_threshold(estimator, scores, self.threshold)
         if self.max_features is not None:
             mask = np.zeros_like(scores, dtype=bool)
-            candidate_indices = \
-                np.argsort(-scores, kind='mergesort')[:self.max_features]
+            candidate_indices = np.argsort(-scores, kind="mergesort")[:max_features]
             mask[candidate_indices] = True
         else:
             mask = np.ones_like(scores, dtype=bool)
         mask[scores < threshold] = False
         return mask
 
+    def _check_max_features(self, X):
+        if self.max_features is not None:
+            n_features = _num_features(X)
+
+            if callable(self.max_features):
+                max_features = self.max_features(X)
+            else:  # int
+                max_features = self.max_features
+
+            check_scalar(
+                max_features,
+                "max_features",
+                Integral,
+                min_val=0,
+                max_val=n_features,
+            )
+            self.max_features_ = max_features
+
+    @_fit_context(
+        # SelectFromModel.estimator is not validated yet
+        prefer_skip_nested_validation=False
+    )
     def fit(self, X, y=None, **fit_params):
         """Fit the SelectFromModel meta-transformer.
 
@@ -212,39 +340,68 @@ def fit(self, X, y=None, **fit_params):
             The target values (integers that correspond to classes in
             classification, real numbers in regression).
 
-        **fit_params : Other estimator specific parameters
+        **fit_params : dict
+            - If `enable_metadata_routing=False` (default): Parameters directly passed
+              to the `fit` method of the sub-estimator. They are ignored if
+              `prefit=True`.
+
+            - If `enable_metadata_routing=True`: Parameters safely routed to the `fit`
+              method of the sub-estimator. They are ignored if `prefit=True`.
+
+            .. versionchanged:: 1.4
+                See :ref:`Metadata Routing User Guide ` for
+                more details.
 
         Returns
         -------
         self : object
+            Fitted estimator.
         """
-        if self.max_features is not None:
-            if not isinstance(self.max_features, numbers.Integral):
-                raise TypeError("'max_features' should be an integer between"
-                                " 0 and {} features. Got {!r} instead."
-                                .format(X.shape[1], self.max_features))
-            elif self.max_features < 0 or self.max_features > X.shape[1]:
-                raise ValueError("'max_features' should be 0 and {} features."
-                                 "Got {} instead."
-                                 .format(X.shape[1], self.max_features))
+        self._check_max_features(X)
 
         if self.prefit:
-            raise NotFittedError(
-                "Since 'prefit=True', call transform directly")
-        self.estimator_ = clone(self.estimator)
-        self.estimator_.fit(X, y, **fit_params)
+            try:
+                check_is_fitted(self.estimator)
+            except NotFittedError as exc:
+                raise NotFittedError(
+                    "When `prefit=True`, `estimator` is expected to be a fitted "
+                    "estimator."
+                ) from exc
+            self.estimator_ = deepcopy(self.estimator)
+        else:
+            if _routing_enabled():
+                routed_params = process_routing(self, "fit", **fit_params)
+                self.estimator_ = clone(self.estimator)
+                self.estimator_.fit(X, y, **routed_params.estimator.fit)
+            else:
+                # TODO(SLEP6): remove when metadata routing cannot be disabled.
+                self.estimator_ = clone(self.estimator)
+                self.estimator_.fit(X, y, **fit_params)
+
+        if hasattr(self.estimator_, "feature_names_in_"):
+            self.feature_names_in_ = self.estimator_.feature_names_in_
+        else:
+            _check_feature_names(self, X, reset=True)
+
         return self
 
     @property
     def threshold_(self):
-        scores = _get_feature_importances(estimator=self.estimator_,
-                                          getter=self.importance_getter,
-                                          transform_func='norm',
-                                          norm_order=self.norm_order)
+        """Threshold value used for feature selection."""
+        scores = _get_feature_importances(
+            estimator=self.estimator_,
+            getter=self.importance_getter,
+            transform_func="norm",
+            norm_order=self.norm_order,
+        )
         return _calculate_threshold(self.estimator, scores, self.threshold)
 
-    @if_delegate_has_method('estimator')
-    def partial_fit(self, X, y=None, **fit_params):
+    @available_if(_estimator_has("partial_fit"))
+    @_fit_context(
+        # SelectFromModel.estimator is not validated yet
+        prefer_skip_nested_validation=False
+    )
+    def partial_fit(self, X, y=None, **partial_fit_params):
         """Fit the SelectFromModel meta-transformer only once.
 
         Parameters
@@ -256,35 +413,101 @@ def partial_fit(self, X, y=None, **fit_params):
             The target values (integers that correspond to classes in
             classification, real numbers in regression).
 
-        **fit_params : Other estimator specific parameters
+        **partial_fit_params : dict
+            - If `enable_metadata_routing=False` (default): Parameters directly passed
+              to the `partial_fit` method of the sub-estimator.
+
+            - If `enable_metadata_routing=True`: Parameters passed to the `partial_fit`
+              method of the sub-estimator. They are ignored if `prefit=True`.
+
+            .. versionchanged:: 1.4
+
+                `**partial_fit_params` are routed to the sub-estimator, if
+                `enable_metadata_routing=True` is set via
+                :func:`~sklearn.set_config`, which allows for aliasing.
+
+                See :ref:`Metadata Routing User Guide ` for
+                more details.
 
         Returns
         -------
         self : object
+            Fitted estimator.
         """
+        first_call = not hasattr(self, "estimator_")
+
+        if first_call:
+            self._check_max_features(X)
+
         if self.prefit:
-            raise NotFittedError(
-                "Since 'prefit=True', call transform directly")
-        if not hasattr(self, "estimator_"):
+            if first_call:
+                try:
+                    check_is_fitted(self.estimator)
+                except NotFittedError as exc:
+                    raise NotFittedError(
+                        "When `prefit=True`, `estimator` is expected to be a fitted "
+                        "estimator."
+                    ) from exc
+                self.estimator_ = deepcopy(self.estimator)
+            return self
+
+        if first_call:
+            self.estimator_ = clone(self.estimator)
+        if _routing_enabled():
+            routed_params = process_routing(self, "partial_fit", **partial_fit_params)
             self.estimator_ = clone(self.estimator)
-        self.estimator_.partial_fit(X, y, **fit_params)
+            self.estimator_.partial_fit(X, y, **routed_params.estimator.partial_fit)
+        else:
+            # TODO(SLEP6): remove when metadata routing cannot be disabled.
+            self.estimator_.partial_fit(X, y, **partial_fit_params)
+
+        if hasattr(self.estimator_, "feature_names_in_"):
+            self.feature_names_in_ = self.estimator_.feature_names_in_
+        else:
+            _check_feature_names(self, X, reset=first_call)
+
         return self
 
     @property
     def n_features_in_(self):
+        """Number of features seen during `fit`."""
         # For consistency with other estimators we raise a AttributeError so
         # that hasattr() fails if the estimator isn't fitted.
         try:
             check_is_fitted(self)
         except NotFittedError as nfe:
             raise AttributeError(
-                "{} object has no n_features_in_ attribute."
-                .format(self.__class__.__name__)
+                "{} object has no n_features_in_ attribute.".format(
+                    self.__class__.__name__
+                )
             ) from nfe
 
         return self.estimator_.n_features_in_
 
-    def _more_tags(self):
-        return {
-            'allow_nan': _safe_tags(self.estimator, key="allow_nan")
-        }
+    def get_metadata_routing(self):
+        """Get metadata routing of this object.
+
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
+
+        .. versionadded:: 1.4
+
+        Returns
+        -------
+        routing : MetadataRouter
+            A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
+            routing information.
+        """
+        router = MetadataRouter(owner=self.__class__.__name__).add(
+            estimator=self.estimator,
+            method_mapping=MethodMapping()
+            .add(caller="partial_fit", callee="partial_fit")
+            .add(caller="fit", callee="fit"),
+        )
+        return router
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = get_tags(self.estimator).input_tags.sparse
+        tags.input_tags.allow_nan = get_tags(self.estimator).input_tags.allow_nan
+        return tags
diff --git a/sklearn/feature_selection/_mutual_info.py b/sklearn/feature_selection/_mutual_info.py
index c5205fc68f370..aef9097879fca 100644
--- a/sklearn/feature_selection/_mutual_info.py
+++ b/sklearn/feature_selection/_mutual_info.py
@@ -1,18 +1,20 @@
-# Author: Nikolay Mayorov 
-# License: 3-clause BSD
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+from numbers import Integral
 
 import numpy as np
 from scipy.sparse import issparse
 from scipy.special import digamma
 
 from ..metrics.cluster import mutual_info_score
-from ..neighbors import NearestNeighbors, KDTree
+from ..neighbors import KDTree, NearestNeighbors
 from ..preprocessing import scale
 from ..utils import check_random_state
-from ..utils.fixes import _astype_copy_false
-from ..utils.validation import check_array, check_X_y
-from ..utils.validation import _deprecate_positional_args
+from ..utils._param_validation import Interval, StrOptions, validate_params
 from ..utils.multiclass import check_classification_targets
+from ..utils.parallel import Parallel, delayed
+from ..utils.validation import check_array, check_X_y
 
 
 def _compute_mi_cc(x, y, n_neighbors):
@@ -30,8 +32,8 @@ def _compute_mi_cc(x, y, n_neighbors):
     Returns
     -------
     mi : float
-        Estimated mutual information. If it turned out to be negative it is
-        replace by 0.
+        Estimated mutual information in nat units. If it turned out to be
+        negative it is replaced by 0.
 
     Notes
     -----
@@ -52,7 +54,7 @@ def _compute_mi_cc(x, y, n_neighbors):
     xy = np.hstack((x, y))
 
     # Here we rely on NearestNeighbors to select the fastest algorithm.
-    nn = NearestNeighbors(metric='chebyshev', n_neighbors=n_neighbors)
+    nn = NearestNeighbors(metric="chebyshev", n_neighbors=n_neighbors)
 
     nn.fit(xy)
     radius = nn.kneighbors()[0]
@@ -60,16 +62,20 @@ def _compute_mi_cc(x, y, n_neighbors):
 
     # KDTree is explicitly fit to allow for the querying of number of
     # neighbors within a specified radius
-    kd = KDTree(x, metric='chebyshev')
+    kd = KDTree(x, metric="chebyshev")
     nx = kd.query_radius(x, radius, count_only=True, return_distance=False)
     nx = np.array(nx) - 1.0
 
-    kd = KDTree(y, metric='chebyshev')
+    kd = KDTree(y, metric="chebyshev")
     ny = kd.query_radius(y, radius, count_only=True, return_distance=False)
     ny = np.array(ny) - 1.0
 
-    mi = (digamma(n_samples) + digamma(n_neighbors) -
-          np.mean(digamma(nx + 1)) - np.mean(digamma(ny + 1)))
+    mi = (
+        digamma(n_samples)
+        + digamma(n_neighbors)
+        - np.mean(digamma(nx + 1))
+        - np.mean(digamma(ny + 1))
+    )
 
     return max(0, mi)
 
@@ -91,8 +97,8 @@ def _compute_mi_cd(c, d, n_neighbors):
     Returns
     -------
     mi : float
-        Estimated mutual information. If it turned out to be negative it is
-        replace by 0.
+        Estimated mutual information in nat units. If it turned out to be
+        negative it is replaced by 0.
 
     Notes
     -----
@@ -135,11 +141,14 @@ def _compute_mi_cd(c, d, n_neighbors):
 
     kd = KDTree(c)
     m_all = kd.query_radius(c, radius, count_only=True, return_distance=False)
-    m_all = np.array(m_all) - 1.0
+    m_all = np.array(m_all)
 
-    mi = (digamma(n_samples) + np.mean(digamma(k_all)) -
-          np.mean(digamma(label_counts)) -
-          np.mean(digamma(m_all + 1)))
+    mi = (
+        digamma(n_samples)
+        + np.mean(digamma(k_all))
+        - np.mean(digamma(label_counts))
+        - np.mean(digamma(m_all))
+    )
 
     return max(0, mi)
 
@@ -190,8 +199,17 @@ def _iterate_columns(X, columns=None):
             yield X[:, i]
 
 
-def _estimate_mi(X, y, discrete_features='auto', discrete_target=False,
-                 n_neighbors=3, copy=True, random_state=None):
+def _estimate_mi(
+    X,
+    y,
+    *,
+    discrete_features="auto",
+    discrete_target=False,
+    n_neighbors=3,
+    copy=True,
+    random_state=None,
+    n_jobs=None,
+):
     """Estimate mutual information between the features and the target.
 
     Parameters
@@ -227,11 +245,21 @@ def _estimate_mi(X, y, discrete_features='auto', discrete_target=False,
         Pass an int for reproducible results across multiple function calls.
         See :term:`Glossary `.
 
+    n_jobs : int, default=None
+        The number of jobs to use for computing the mutual information.
+        The parallelization is done on the columns of `X`.
+        ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
+        ``-1`` means using all processors. See :term:`Glossary `
+        for more details.
+
+        .. versionadded:: 1.5
+
+
     Returns
     -------
     mi : ndarray, shape (n_features,)
-        Estimated mutual information between each feature and the target.
-        A negative value will be replaced by 0.
+        Estimated mutual information between each feature and the target in
+        nat units. A negative value will be replaced by 0.
 
     References
     ----------
@@ -240,12 +268,12 @@ def _estimate_mi(X, y, discrete_features='auto', discrete_target=False,
     .. [2] B. C. Ross "Mutual Information between Discrete and Continuous
            Data Sets". PLoS ONE 9(2), 2014.
     """
-    X, y = check_X_y(X, y, accept_sparse='csc', y_numeric=not discrete_target)
+    X, y = check_X_y(X, y, accept_sparse="csc", y_numeric=not discrete_target)
     n_samples, n_features = X.shape
 
     if isinstance(discrete_features, (str, bool)):
         if isinstance(discrete_features, str):
-            if discrete_features == 'auto':
+            if discrete_features == "auto":
                 discrete_features = issparse(X)
             else:
                 raise ValueError("Invalid string value for discrete_features.")
@@ -253,7 +281,7 @@ def _estimate_mi(X, y, discrete_features='auto', discrete_target=False,
         discrete_mask.fill(discrete_features)
     else:
         discrete_features = check_array(discrete_features, ensure_2d=False)
-        if discrete_features.dtype != 'bool':
+        if discrete_features.dtype != "bool":
             discrete_mask = np.zeros(n_features, dtype=bool)
             discrete_mask[discrete_features] = True
         else:
@@ -265,32 +293,57 @@ def _estimate_mi(X, y, discrete_features='auto', discrete_target=False,
 
     rng = check_random_state(random_state)
     if np.any(continuous_mask):
-        if copy:
-            X = X.copy()
-
-        if not discrete_target:
-            X[:, continuous_mask] = scale(X[:, continuous_mask],
-                                          with_mean=False, copy=False)
+        X = X.astype(np.float64, copy=copy)
+        X[:, continuous_mask] = scale(
+            X[:, continuous_mask], with_mean=False, copy=False
+        )
 
         # Add small noise to continuous features as advised in Kraskov et. al.
-        X = X.astype(float, **_astype_copy_false(X))
         means = np.maximum(1, np.mean(np.abs(X[:, continuous_mask]), axis=0))
-        X[:, continuous_mask] += 1e-10 * means * rng.randn(
-                n_samples, np.sum(continuous_mask))
+        X[:, continuous_mask] += (
+            1e-10
+            * means
+            * rng.standard_normal(size=(n_samples, np.sum(continuous_mask)))
+        )
 
     if not discrete_target:
         y = scale(y, with_mean=False)
-        y += 1e-10 * np.maximum(1, np.mean(np.abs(y))) * rng.randn(n_samples)
+        y += (
+            1e-10
+            * np.maximum(1, np.mean(np.abs(y)))
+            * rng.standard_normal(size=n_samples)
+        )
 
-    mi = [_compute_mi(x, y, discrete_feature, discrete_target, n_neighbors) for
-          x, discrete_feature in zip(_iterate_columns(X), discrete_mask)]
+    mi = Parallel(n_jobs=n_jobs)(
+        delayed(_compute_mi)(x, y, discrete_feature, discrete_target, n_neighbors)
+        for x, discrete_feature in zip(_iterate_columns(X), discrete_mask)
+    )
 
     return np.array(mi)
 
 
-@_deprecate_positional_args
-def mutual_info_regression(X, y, *, discrete_features='auto', n_neighbors=3,
-                           copy=True, random_state=None):
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "y": ["array-like"],
+        "discrete_features": [StrOptions({"auto"}), "boolean", "array-like"],
+        "n_neighbors": [Interval(Integral, 1, None, closed="left")],
+        "copy": ["boolean"],
+        "random_state": ["random_state"],
+        "n_jobs": [Integral, None],
+    },
+    prefer_skip_nested_validation=True,
+)
+def mutual_info_regression(
+    X,
+    y,
+    *,
+    discrete_features="auto",
+    n_neighbors=3,
+    copy=True,
+    random_state=None,
+    n_jobs=None,
+):
     """Estimate mutual information for a continuous target variable.
 
     Mutual information (MI) [1]_ between two random variables is a non-negative
@@ -335,10 +388,21 @@ def mutual_info_regression(X, y, *, discrete_features='auto', n_neighbors=3,
         Pass an int for reproducible results across multiple function calls.
         See :term:`Glossary `.
 
+    n_jobs : int, default=None
+        The number of jobs to use for computing the mutual information.
+        The parallelization is done on the columns of `X`.
+
+        ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
+        ``-1`` means using all processors. See :term:`Glossary `
+        for more details.
+
+        .. versionadded:: 1.5
+
     Returns
     -------
     mi : ndarray, shape (n_features,)
-        Estimated mutual information between each feature and the target.
+        Estimated mutual information between each feature and the target in
+        nat units.
 
     Notes
     -----
@@ -363,14 +427,51 @@ def mutual_info_regression(X, y, *, discrete_features='auto', n_neighbors=3,
            Data Sets". PLoS ONE 9(2), 2014.
     .. [4] L. F. Kozachenko, N. N. Leonenko, "Sample Estimate of the Entropy
            of a Random Vector", Probl. Peredachi Inf., 23:2 (1987), 9-16
-    """
-    return _estimate_mi(X, y, discrete_features, False, n_neighbors,
-                        copy, random_state)
-
 
-@_deprecate_positional_args
-def mutual_info_classif(X, y, *, discrete_features='auto', n_neighbors=3,
-                        copy=True, random_state=None):
+    Examples
+    --------
+    >>> from sklearn.datasets import make_regression
+    >>> from sklearn.feature_selection import mutual_info_regression
+    >>> X, y = make_regression(
+    ...     n_samples=50, n_features=3, n_informative=1, noise=1e-4, random_state=42
+    ... )
+    >>> mutual_info_regression(X, y)
+    array([0.117, 2.645, 0.0287])
+    """
+    return _estimate_mi(
+        X,
+        y,
+        discrete_features=discrete_features,
+        discrete_target=False,
+        n_neighbors=n_neighbors,
+        copy=copy,
+        random_state=random_state,
+        n_jobs=n_jobs,
+    )
+
+
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "y": ["array-like"],
+        "discrete_features": [StrOptions({"auto"}), "boolean", "array-like"],
+        "n_neighbors": [Interval(Integral, 1, None, closed="left")],
+        "copy": ["boolean"],
+        "random_state": ["random_state"],
+        "n_jobs": [Integral, None],
+    },
+    prefer_skip_nested_validation=True,
+)
+def mutual_info_classif(
+    X,
+    y,
+    *,
+    discrete_features="auto",
+    n_neighbors=3,
+    copy=True,
+    random_state=None,
+    n_jobs=None,
+):
     """Estimate mutual information for a discrete target variable.
 
     Mutual information (MI) [1]_ between two random variables is a non-negative
@@ -387,13 +488,13 @@ def mutual_info_classif(X, y, *, discrete_features='auto', n_neighbors=3,
 
     Parameters
     ----------
-    X : array-like or sparse matrix, shape (n_samples, n_features)
+    X : {array-like, sparse matrix} of shape (n_samples, n_features)
         Feature matrix.
 
     y : array-like of shape (n_samples,)
         Target vector.
 
-    discrete_features : {'auto', bool, array-like}, default='auto'
+    discrete_features : 'auto', bool or array-like, default='auto'
         If bool, then determines whether to consider all features discrete
         or continuous. If array, then it should be either a boolean mask
         with shape (n_features,) or array with indices of discrete features.
@@ -415,10 +516,20 @@ def mutual_info_classif(X, y, *, discrete_features='auto', n_neighbors=3,
         Pass an int for reproducible results across multiple function calls.
         See :term:`Glossary `.
 
+    n_jobs : int, default=None
+        The number of jobs to use for computing the mutual information.
+        The parallelization is done on the columns of `X`.
+        ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
+        ``-1`` means using all processors. See :term:`Glossary `
+        for more details.
+
+        .. versionadded:: 1.5
+
     Returns
     -------
     mi : ndarray, shape (n_features,)
-        Estimated mutual information between each feature and the target.
+        Estimated mutual information between each feature and the target in
+        nat units.
 
     Notes
     -----
@@ -443,7 +554,27 @@ def mutual_info_classif(X, y, *, discrete_features='auto', n_neighbors=3,
            Data Sets". PLoS ONE 9(2), 2014.
     .. [4] L. F. Kozachenko, N. N. Leonenko, "Sample Estimate of the Entropy
            of a Random Vector:, Probl. Peredachi Inf., 23:2 (1987), 9-16
+
+    Examples
+    --------
+    >>> from sklearn.datasets import make_classification
+    >>> from sklearn.feature_selection import mutual_info_classif
+    >>> X, y = make_classification(
+    ...     n_samples=100, n_features=10, n_informative=2, n_clusters_per_class=1,
+    ...     shuffle=False, random_state=42
+    ... )
+    >>> mutual_info_classif(X, y)
+    array([0.589, 0.107, 0.196, 0.0968 , 0.,
+           0.   , 0.   , 0.   , 0.     , 0.])
     """
     check_classification_targets(y)
-    return _estimate_mi(X, y, discrete_features, True, n_neighbors,
-                        copy, random_state)
+    return _estimate_mi(
+        X,
+        y,
+        discrete_features=discrete_features,
+        discrete_target=True,
+        n_neighbors=n_neighbors,
+        copy=copy,
+        random_state=random_state,
+        n_jobs=n_jobs,
+    )
diff --git a/sklearn/feature_selection/_rfe.py b/sklearn/feature_selection/_rfe.py
index 16519dfba6761..d2bd78e225a54 100644
--- a/sklearn/feature_selection/_rfe.py
+++ b/sklearn/feature_selection/_rfe.py
@@ -1,44 +1,68 @@
-# Authors: Alexandre Gramfort 
-#          Vincent Michel 
-#          Gilles Louppe 
-#
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 """Recursive feature elimination for feature ranking"""
 
+import warnings
+from copy import deepcopy
+from numbers import Integral
+
 import numpy as np
-import numbers
-from joblib import Parallel, effective_n_jobs
-
-
-from ..utils.metaestimators import if_delegate_has_method
-from ..utils.metaestimators import _safe_split
-from ..utils._tags import _safe_tags
-from ..utils.validation import check_is_fitted
-from ..utils.validation import _deprecate_positional_args
-from ..utils.fixes import delayed
-from ..base import BaseEstimator
-from ..base import MetaEstimatorMixin
-from ..base import clone
-from ..base import is_classifier
+from joblib import effective_n_jobs
+
+from ..base import BaseEstimator, MetaEstimatorMixin, _fit_context, clone, is_classifier
+from ..metrics import get_scorer
 from ..model_selection import check_cv
 from ..model_selection._validation import _score
-from ..metrics import check_scoring
-from ._base import SelectorMixin
-from ._base import _get_feature_importances
-
-
-def _rfe_single_fit(rfe, estimator, X, y, train, test, scorer):
+from ..utils import Bunch, metadata_routing
+from ..utils._metadata_requests import (
+    MetadataRouter,
+    MethodMapping,
+    _raise_for_params,
+    _routing_enabled,
+    process_routing,
+)
+from ..utils._param_validation import HasMethods, Interval, RealNotInt
+from ..utils._tags import get_tags
+from ..utils.metaestimators import _safe_split, available_if
+from ..utils.parallel import Parallel, delayed
+from ..utils.validation import (
+    _check_method_params,
+    _deprecate_positional_args,
+    _estimator_has,
+    check_is_fitted,
+    validate_data,
+)
+from ._base import SelectorMixin, _get_feature_importances
+
+
+def _rfe_single_fit(rfe, estimator, X, y, train, test, scorer, routed_params):
     """
-    Return the score for a fit across one fold.
+    Return the score and n_features per step for a fit across one fold.
     """
     X_train, y_train = _safe_split(estimator, X, y, train)
     X_test, y_test = _safe_split(estimator, X, y, test, train)
-    return rfe._fit(
-        X_train, y_train,
+    fit_params = _check_method_params(
+        X, params=routed_params.estimator.fit, indices=train
+    )
+    score_params = _check_method_params(
+        X=X, params=routed_params.scorer.score, indices=test
+    )
+
+    rfe._fit(
+        X_train,
+        y_train,
         lambda estimator, features: _score(
-            estimator, X_test[:, features], y_test, scorer
-        )).scores_
+            estimator,
+            X_test[:, features],
+            y_test,
+            scorer,
+            score_params=score_params,
+        ),
+        **fit_params,
+    )
+
+    return rfe.step_scores_, rfe.step_support_, rfe.step_ranking_, rfe.step_n_features_
 
 
 class RFE(SelectorMixin, MetaEstimatorMixin, BaseEstimator):
@@ -100,12 +124,27 @@ class RFE(SelectorMixin, MetaEstimatorMixin, BaseEstimator):
 
     Attributes
     ----------
+    classes_ : ndarray of shape (n_classes,)
+        The classes labels. Only available when `estimator` is a classifier.
+
     estimator_ : ``Estimator`` instance
         The fitted estimator used to select features.
 
     n_features_ : int
         The number of selected features.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`. Only defined if the
+        underlying estimator exposes such an attribute when fit.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     ranking_ : ndarray of shape (n_features,)
         The feature ranking, such that ``ranking_[i]`` corresponds to the
         ranking position of the i-th feature. Selected (i.e., estimated
@@ -114,6 +153,26 @@ class RFE(SelectorMixin, MetaEstimatorMixin, BaseEstimator):
     support_ : ndarray of shape (n_features,)
         The mask of selected features.
 
+    See Also
+    --------
+    RFECV : Recursive feature elimination with built-in cross-validated
+        selection of the best number of features.
+    SelectFromModel : Feature selection based on thresholds of importance
+        weights.
+    SequentialFeatureSelector : Sequential cross-validation based feature
+        selection. Does not rely on importance weights.
+
+    Notes
+    -----
+    Allows NaN/Inf in the input if the underlying estimator does as well.
+
+    References
+    ----------
+
+    .. [1] Guyon, I., Weston, J., Barnhill, S., & Vapnik, V., "Gene selection
+           for cancer classification using support vector machines",
+           Mach. Learn., 46(1-3), 389--422, 2002.
+
     Examples
     --------
     The following example shows how to retrieve the 5 most informative
@@ -131,47 +190,59 @@ class RFE(SelectorMixin, MetaEstimatorMixin, BaseEstimator):
            False])
     >>> selector.ranking_
     array([1, 1, 1, 1, 1, 6, 4, 3, 2, 5])
-
-    Notes
-    -----
-    Allows NaN/Inf in the input if the underlying estimator does as well.
-
-    See Also
-    --------
-    RFECV : Recursive feature elimination with built-in cross-validated
-        selection of the best number of features.
-    SelectFromModel : Feature selection based on thresholds of importance
-        weights.
-    SequentialFeatureSelector : Sequential cross-validation based feature
-        selection. Does not rely on importance weights.
-
-    References
-    ----------
-
-    .. [1] Guyon, I., Weston, J., Barnhill, S., & Vapnik, V., "Gene selection
-           for cancer classification using support vector machines",
-           Mach. Learn., 46(1-3), 389--422, 2002.
     """
-    @_deprecate_positional_args
-    def __init__(self, estimator, *, n_features_to_select=None, step=1,
-                 verbose=0, importance_getter='auto'):
+
+    _parameter_constraints: dict = {
+        "estimator": [HasMethods(["fit"])],
+        "n_features_to_select": [
+            None,
+            Interval(RealNotInt, 0, 1, closed="right"),
+            Interval(Integral, 0, None, closed="neither"),
+        ],
+        "step": [
+            Interval(Integral, 0, None, closed="neither"),
+            Interval(RealNotInt, 0, 1, closed="neither"),
+        ],
+        "verbose": ["verbose"],
+        "importance_getter": [str, callable],
+    }
+
+    def __init__(
+        self,
+        estimator,
+        *,
+        n_features_to_select=None,
+        step=1,
+        verbose=0,
+        importance_getter="auto",
+    ):
         self.estimator = estimator
         self.n_features_to_select = n_features_to_select
         self.step = step
         self.importance_getter = importance_getter
         self.verbose = verbose
 
+    # TODO(1.8) remove this property
     @property
     def _estimator_type(self):
         return self.estimator._estimator_type
 
     @property
     def classes_(self):
+        """Classes labels available when `estimator` is a classifier.
+
+        Returns
+        -------
+        ndarray of shape (n_classes,)
+        """
         return self.estimator_.classes_
 
-    def fit(self, X, y):
-        """Fit the RFE model and then the underlying estimator on the selected
-           features.
+    @_fit_context(
+        # RFE.estimator is not validated yet
+        prefer_skip_nested_validation=False
+    )
+    def fit(self, X, y, **fit_params):
+        """Fit the RFE model and then the underlying estimator on the selected features.
 
         Parameters
         ----------
@@ -180,38 +251,59 @@ def fit(self, X, y):
 
         y : array-like of shape (n_samples,)
             The target values.
+
+        **fit_params : dict
+            - If `enable_metadata_routing=False` (default): Parameters directly passed
+              to the ``fit`` method of the underlying estimator.
+
+            - If `enable_metadata_routing=True`: Parameters safely routed to the ``fit``
+              method of the underlying estimator.
+
+            .. versionchanged:: 1.6
+                See :ref:`Metadata Routing User Guide `
+                for more details.
+
+        Returns
+        -------
+        self : object
+            Fitted estimator.
         """
-        return self._fit(X, y)
+        if _routing_enabled():
+            routed_params = process_routing(self, "fit", **fit_params)
+        else:
+            routed_params = Bunch(estimator=Bunch(fit=fit_params))
 
-    def _fit(self, X, y, step_score=None):
-        # Parameter step_score controls the calculation of self.scores_
-        # step_score is not exposed to users
-        # and is used when implementing RFECV
-        # self.scores_ will not be calculated when calling _fit through fit
+        return self._fit(X, y, **routed_params.estimator.fit)
 
-        tags = self._get_tags()
-        X, y = self._validate_data(
-            X, y, accept_sparse="csc",
+    def _fit(self, X, y, step_score=None, **fit_params):
+        # Parameter step_score controls the calculation of self.step_scores_
+        # step_score is not exposed to users and is used when implementing RFECV
+        # self.step_scores_ will not be calculated when calling _fit through fit
+
+        X, y = validate_data(
+            self,
+            X,
+            y,
+            accept_sparse="csc",
             ensure_min_features=2,
-            force_all_finite=not tags.get("allow_nan", True),
-            multi_output=True
+            ensure_all_finite=False,
+            multi_output=True,
         )
-        error_msg = ("n_features_to_select must be either None, a "
-                     "positive integer representing the absolute "
-                     "number of features or a float in (0.0, 1.0] "
-                     "representing a percentage of features to "
-                     f"select. Got {self.n_features_to_select}")
 
         # Initialization
         n_features = X.shape[1]
         if self.n_features_to_select is None:
             n_features_to_select = n_features // 2
-        elif self.n_features_to_select < 0:
-            raise ValueError(error_msg)
-        elif isinstance(self.n_features_to_select, numbers.Integral):  # int
+        elif isinstance(self.n_features_to_select, Integral):  # int
             n_features_to_select = self.n_features_to_select
-        elif self.n_features_to_select > 1.0:  # float > 1
-            raise ValueError(error_msg)
+            if n_features_to_select > n_features:
+                warnings.warn(
+                    (
+                        f"Found {n_features_to_select=} > {n_features=}. There will be"
+                        " no feature selection and all features will be kept."
+                    ),
+                    UserWarning,
+                )
         else:  # float
             n_features_to_select = int(n_features * self.n_features_to_select)
 
@@ -219,14 +311,15 @@ def _fit(self, X, y, step_score=None):
             step = int(max(1, self.step * n_features))
         else:
             step = int(self.step)
-        if step <= 0:
-            raise ValueError("Step must be >0")
 
         support_ = np.ones(n_features, dtype=bool)
         ranking_ = np.ones(n_features, dtype=int)
 
         if step_score:
-            self.scores_ = []
+            self.step_n_features_ = []
+            self.step_scores_ = []
+            self.step_support_ = []
+            self.step_ranking_ = []
 
         # Elimination
         while np.sum(support_) > n_features_to_select:
@@ -238,11 +331,21 @@ def _fit(self, X, y, step_score=None):
             if self.verbose > 0:
                 print("Fitting estimator with %d features." % np.sum(support_))
 
-            estimator.fit(X[:, features], y)
+            estimator.fit(X[:, features], y, **fit_params)
+
+            # Compute step values on the previous selection iteration because
+            # 'estimator' must use features that have not been eliminated yet
+            if step_score:
+                self.step_n_features_.append(len(features))
+                self.step_scores_.append(step_score(estimator, features))
+                self.step_support_.append(list(support_))
+                self.step_ranking_.append(list(ranking_))
 
             # Get importance and rank them
             importances = _get_feature_importances(
-                estimator, self.importance_getter, transform_func="square",
+                estimator,
+                self.importance_getter,
+                transform_func="square",
             )
             ranks = np.argsort(importances)
 
@@ -252,50 +355,65 @@ def _fit(self, X, y, step_score=None):
             # Eliminate the worse features
             threshold = min(step, np.sum(support_) - n_features_to_select)
 
-            # Compute step score on the previous selection iteration
-            # because 'estimator' must use features
-            # that have not been eliminated yet
-            if step_score:
-                self.scores_.append(step_score(estimator, features))
             support_[features[ranks][:threshold]] = False
             ranking_[np.logical_not(support_)] += 1
 
         # Set final attributes
         features = np.arange(n_features)[support_]
         self.estimator_ = clone(self.estimator)
-        self.estimator_.fit(X[:, features], y)
+        self.estimator_.fit(X[:, features], y, **fit_params)
 
-        # Compute step score when only n_features_to_select features left
+        # Compute step values when only n_features_to_select features left
         if step_score:
-            self.scores_.append(step_score(self.estimator_, features))
+            self.step_n_features_.append(len(features))
+            self.step_scores_.append(step_score(self.estimator_, features))
+            self.step_support_.append(support_)
+            self.step_ranking_.append(ranking_)
         self.n_features_ = support_.sum()
         self.support_ = support_
         self.ranking_ = ranking_
 
         return self
 
-    @if_delegate_has_method(delegate='estimator')
-    def predict(self, X):
-        """Reduce X to the selected features and then predict using the
-           underlying estimator.
+    @available_if(_estimator_has("predict"))
+    def predict(self, X, **predict_params):
+        """Reduce X to the selected features and predict using the estimator.
 
         Parameters
         ----------
         X : array of shape [n_samples, n_features]
             The input samples.
 
+        **predict_params : dict
+            Parameters to route to the ``predict`` method of the
+            underlying estimator.
+
+            .. versionadded:: 1.6
+                Only available if `enable_metadata_routing=True`,
+                which can be set by using
+                ``sklearn.set_config(enable_metadata_routing=True)``.
+                See :ref:`Metadata Routing User Guide `
+                for more details.
+
         Returns
         -------
         y : array of shape [n_samples]
             The predicted target values.
         """
+        _raise_for_params(predict_params, self, "predict")
         check_is_fitted(self)
-        return self.estimator_.predict(self.transform(X))
+        if _routing_enabled():
+            routed_params = process_routing(self, "predict", **predict_params)
+        else:
+            routed_params = Bunch(estimator=Bunch(predict={}))
+
+        return self.estimator_.predict(
+            self.transform(X), **routed_params.estimator.predict
+        )
 
-    @if_delegate_has_method(delegate='estimator')
-    def score(self, X, y):
-        """Reduce X to the selected features and then return the score of the
-           underlying estimator.
+    @available_if(_estimator_has("score"))
+    def score(self, X, y, **score_params):
+        """Reduce X to the selected features and return the score of the estimator.
 
         Parameters
         ----------
@@ -304,15 +422,41 @@ def score(self, X, y):
 
         y : array of shape [n_samples]
             The target values.
+
+        **score_params : dict
+            - If `enable_metadata_routing=False` (default): Parameters directly passed
+              to the ``score`` method of the underlying estimator.
+
+            - If `enable_metadata_routing=True`: Parameters safely routed to the `score`
+              method of the underlying estimator.
+
+            .. versionadded:: 1.0
+
+            .. versionchanged:: 1.6
+                See :ref:`Metadata Routing User Guide `
+                for more details.
+
+        Returns
+        -------
+        score : float
+            Score of the underlying base estimator computed with the selected
+            features returned by `rfe.transform(X)` and `y`.
         """
         check_is_fitted(self)
-        return self.estimator_.score(self.transform(X), y)
+        if _routing_enabled():
+            routed_params = process_routing(self, "score", **score_params)
+        else:
+            routed_params = Bunch(estimator=Bunch(score=score_params))
+
+        return self.estimator_.score(
+            self.transform(X), y, **routed_params.estimator.score
+        )
 
     def _get_support_mask(self):
         check_is_fitted(self)
         return self.support_
 
-    @if_delegate_has_method(delegate='estimator')
+    @available_if(_estimator_has("decision_function"))
     def decision_function(self, X):
         """Compute the decision function of ``X``.
 
@@ -334,7 +478,7 @@ def decision_function(self, X):
         check_is_fitted(self)
         return self.estimator_.decision_function(self.transform(X))
 
-    @if_delegate_has_method(delegate='estimator')
+    @available_if(_estimator_has("predict_proba"))
     def predict_proba(self, X):
         """Predict class probabilities for X.
 
@@ -354,7 +498,7 @@ def predict_proba(self, X):
         check_is_fitted(self)
         return self.estimator_.predict_proba(self.transform(X))
 
-    @if_delegate_has_method(delegate='estimator')
+    @available_if(_estimator_has("predict_log_proba"))
     def predict_log_proba(self, X):
         """Predict class log-probabilities for X.
 
@@ -372,18 +516,54 @@ def predict_log_proba(self, X):
         check_is_fitted(self)
         return self.estimator_.predict_log_proba(self.transform(X))
 
-    def _more_tags(self):
-        return {
-            'poor_score': True,
-            'allow_nan': _safe_tags(self.estimator, key='allow_nan'),
-            'requires_y': True,
-        }
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        sub_estimator_tags = get_tags(self.estimator)
+        tags.estimator_type = sub_estimator_tags.estimator_type
+        tags.classifier_tags = deepcopy(sub_estimator_tags.classifier_tags)
+        tags.regressor_tags = deepcopy(sub_estimator_tags.regressor_tags)
+        if tags.classifier_tags is not None:
+            tags.classifier_tags.poor_score = True
+        if tags.regressor_tags is not None:
+            tags.regressor_tags.poor_score = True
+        tags.target_tags.required = True
+        tags.input_tags.sparse = sub_estimator_tags.input_tags.sparse
+        tags.input_tags.allow_nan = sub_estimator_tags.input_tags.allow_nan
+        return tags
+
+    def get_metadata_routing(self):
+        """Get metadata routing of this object.
+
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
+
+        .. versionadded:: 1.6
 
+        Returns
+        -------
+        routing : MetadataRouter
+            A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
+            routing information.
+        """
+        router = MetadataRouter(owner=self.__class__.__name__).add(
+            estimator=self.estimator,
+            method_mapping=MethodMapping()
+            .add(caller="fit", callee="fit")
+            .add(caller="predict", callee="predict")
+            .add(caller="score", callee="score"),
+        )
+        return router
 
-class RFECV(RFE):
-    """Feature ranking with recursive feature elimination and cross-validated
-    selection of the best number of features.
 
+class RFECV(RFE):
+    """Recursive feature elimination with cross-validation to select features.
+
+    The number of features selected is tuned automatically by fitting an :class:`RFE`
+    selector on the different cross-validation splits (provided by the `cv` parameter).
+    The performance of each :class:`RFE` selector is evaluated using `scoring` for
+    different numbers of selected features and aggregated together. Finally, the scores
+    are averaged across folds and the number of features selected is set to the number
+    of features that maximize the cross-validation score.
     See glossary entry for :term:`cross-validation estimator`.
 
     Read more in the :ref:`User Guide `.
@@ -422,7 +602,7 @@ class RFECV(RFE):
 
         For integer/None inputs, if ``y`` is binary or multiclass,
         :class:`~sklearn.model_selection.StratifiedKFold` is used. If the
-        estimator is a classifier or if ``y`` is neither binary nor multiclass,
+        estimator is not a classifier or if ``y`` is neither binary nor multiclass,
         :class:`~sklearn.model_selection.KFold` is used.
 
         Refer :ref:`User Guide ` for the various
@@ -431,10 +611,14 @@ class RFECV(RFE):
         .. versionchanged:: 0.22
             ``cv`` default value of None changed from 3-fold to 5-fold.
 
-    scoring : string, callable or None, default=None
-        A string (see model evaluation documentation) or
-        a scorer callable object / function with signature
-        ``scorer(estimator, X, y)``.
+    scoring : str or callable, default=None
+        Scoring method to evaluate the :class:`RFE` selectors' performance. Options:
+
+        - str: see :ref:`scoring_string_names` for options.
+        - callable: a scorer callable object (e.g., function) with signature
+          ``scorer(estimator, X, y)``. See :ref:`scoring_callable` for details.
+        - `None`: the `estimator`'s
+          :ref:`default evaluation criterion ` is used.
 
     verbose : int, default=0
         Controls verbosity of output.
@@ -466,17 +650,65 @@ class RFECV(RFE):
 
     Attributes
     ----------
+    classes_ : ndarray of shape (n_classes,)
+        The classes labels. Only available when `estimator` is a classifier.
+
     estimator_ : ``Estimator`` instance
         The fitted estimator used to select features.
 
-    grid_scores_ : ndarray of shape (n_subsets_of_features,)
-        The cross-validation scores such that
-        ``grid_scores_[i]`` corresponds to
-        the CV score of the i-th subset of features.
+    cv_results_ : dict of ndarrays
+        All arrays (values of the dictionary) are sorted in ascending order
+        by the number of features used (i.e., the first element of the array
+        represents the models that used the least number of features, while the
+        last element represents the models that used all available features).
+
+        .. versionadded:: 1.0
+
+        This dictionary contains the following keys:
+
+        split(k)_test_score : ndarray of shape (n_subsets_of_features,)
+            The cross-validation scores across (k)th fold.
+
+        mean_test_score : ndarray of shape (n_subsets_of_features,)
+            Mean of scores over the folds.
+
+        std_test_score : ndarray of shape (n_subsets_of_features,)
+            Standard deviation of scores over the folds.
+
+        n_features : ndarray of shape (n_subsets_of_features,)
+            Number of features used at each step.
+
+            .. versionadded:: 1.5
+
+        split(k)_ranking : ndarray of shape (n_subsets_of_features,)
+            The cross-validation rankings across (k)th fold.
+            Selected (i.e., estimated best) features are assigned rank 1.
+            Illustration in
+            :ref:`sphx_glr_auto_examples_feature_selection_plot_rfe_with_cross_validation.py`
+
+            .. versionadded:: 1.7
+
+        split(k)_support : ndarray of shape (n_subsets_of_features,)
+            The cross-validation supports across (k)th fold. The support
+            is the mask of selected features.
+
+            .. versionadded:: 1.7
 
     n_features_ : int
         The number of selected features with cross-validation.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`. Only defined if the
+        underlying estimator exposes such an attribute when fit.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     ranking_ : narray of shape (n_features,)
         The feature ranking, such that `ranking_[i]`
         corresponds to the ranking
@@ -487,14 +719,25 @@ class RFECV(RFE):
     support_ : ndarray of shape (n_features,)
         The mask of selected features.
 
+    See Also
+    --------
+    RFE : Recursive feature elimination.
+
     Notes
     -----
-    The size of ``grid_scores_`` is equal to
+    The size of all values in ``cv_results_`` is equal to
     ``ceil((n_features - min_features_to_select) / step) + 1``,
     where step is the number of features removed at each iteration.
 
     Allows NaN/Inf in the input if the underlying estimator does as well.
 
+    References
+    ----------
+
+    .. [1] Guyon, I., Weston, J., Barnhill, S., & Vapnik, V., "Gene selection
+           for cancer classification using support vector machines",
+           Mach. Learn., 46(1-3), 389--422, 2002.
+
     Examples
     --------
     The following example shows how to retrieve the a-priori not known 5
@@ -512,22 +755,30 @@ class RFECV(RFE):
            False])
     >>> selector.ranking_
     array([1, 1, 1, 1, 1, 6, 4, 3, 2, 5])
-
-    See Also
-    --------
-    RFE : Recursive feature elimination.
-
-    References
-    ----------
-
-    .. [1] Guyon, I., Weston, J., Barnhill, S., & Vapnik, V., "Gene selection
-           for cancer classification using support vector machines",
-           Mach. Learn., 46(1-3), 389--422, 2002.
     """
-    @_deprecate_positional_args
-    def __init__(self, estimator, *, step=1, min_features_to_select=1,
-                 cv=None, scoring=None, verbose=0, n_jobs=None,
-                 importance_getter='auto'):
+
+    _parameter_constraints: dict = {
+        **RFE._parameter_constraints,
+        "min_features_to_select": [Interval(Integral, 0, None, closed="neither")],
+        "cv": ["cv_object"],
+        "scoring": [None, str, callable],
+        "n_jobs": [None, Integral],
+    }
+    _parameter_constraints.pop("n_features_to_select")
+    __metadata_request__fit = {"groups": metadata_routing.UNUSED}
+
+    def __init__(
+        self,
+        estimator,
+        *,
+        step=1,
+        min_features_to_select=1,
+        cv=None,
+        scoring=None,
+        verbose=0,
+        n_jobs=None,
+        importance_getter="auto",
+    ):
         self.estimator = estimator
         self.step = step
         self.importance_getter = importance_getter
@@ -537,9 +788,14 @@ def __init__(self, estimator, *, step=1, min_features_to_select=1,
         self.n_jobs = n_jobs
         self.min_features_to_select = min_features_to_select
 
-    def fit(self, X, y, groups=None):
-        """Fit the RFE model and automatically tune the number of selected
-           features.
+    # TODO(1.8): remove `groups` from the signature after deprecation cycle.
+    @_deprecate_positional_args(version="1.8")
+    @_fit_context(
+        # RFECV.estimator is not validated yet
+        prefer_skip_nested_validation=False
+    )
+    def fit(self, X, y, *, groups=None, **params):
+        """Fit the RFE model and automatically tune the number of selected features.
 
         Parameters
         ----------
@@ -557,32 +813,68 @@ def fit(self, X, y, groups=None):
             instance (e.g., :class:`~sklearn.model_selection.GroupKFold`).
 
             .. versionadded:: 0.20
+
+        **params : dict of str -> object
+            Parameters passed to the ``fit`` method of the estimator,
+            the scorer, and the CV splitter.
+
+            .. versionadded:: 1.6
+                Only available if `enable_metadata_routing=True`,
+                which can be set by using
+                ``sklearn.set_config(enable_metadata_routing=True)``.
+                See :ref:`Metadata Routing User Guide `
+                for more details.
+
+        Returns
+        -------
+        self : object
+            Fitted estimator.
         """
-        tags = self._get_tags()
-        X, y = self._validate_data(
-            X, y, accept_sparse="csr", ensure_min_features=2,
-            force_all_finite=not tags.get('allow_nan', True),
-            multi_output=True
+        _raise_for_params(params, self, "fit")
+        X, y = validate_data(
+            self,
+            X,
+            y,
+            accept_sparse="csr",
+            ensure_min_features=2,
+            ensure_all_finite=False,
+            multi_output=True,
         )
 
+        if _routing_enabled():
+            if groups is not None:
+                params.update({"groups": groups})
+            routed_params = process_routing(self, "fit", **params)
+        else:
+            routed_params = Bunch(
+                estimator=Bunch(fit={}),
+                splitter=Bunch(split={"groups": groups}),
+                scorer=Bunch(score={}),
+            )
+
         # Initialization
         cv = check_cv(self.cv, y, classifier=is_classifier(self.estimator))
-        scorer = check_scoring(self.estimator, scoring=self.scoring)
-        n_features = X.shape[1]
-
-        if 0.0 < self.step < 1.0:
-            step = int(max(1, self.step * n_features))
-        else:
-            step = int(self.step)
-        if step <= 0:
-            raise ValueError("Step must be >0")
+        scorer = self._get_scorer()
 
         # Build an RFE object, which will evaluate and score each possible
         # feature count, down to self.min_features_to_select
-        rfe = RFE(estimator=self.estimator,
-                  n_features_to_select=self.min_features_to_select,
-                  importance_getter=self.importance_getter,
-                  step=self.step, verbose=self.verbose)
+        n_features = X.shape[1]
+        if self.min_features_to_select > n_features:
+            warnings.warn(
+                (
+                    f"Found min_features_to_select={self.min_features_to_select} > "
+                    f"{n_features=}. There will be no feature selection and all "
+                    "features will be kept."
+                ),
+                UserWarning,
+            )
+        rfe = RFE(
+            estimator=self.estimator,
+            n_features_to_select=min(self.min_features_to_select, n_features),
+            importance_getter=self.importance_getter,
+            step=self.step,
+            verbose=self.verbose,
+        )
 
         # Determine the number of subsets of features by fitting across
         # the train folds and choosing the "features_to_select" parameter
@@ -602,33 +894,127 @@ def fit(self, X, y, groups=None):
             parallel = Parallel(n_jobs=self.n_jobs)
             func = delayed(_rfe_single_fit)
 
-        scores = parallel(
-            func(rfe, self.estimator, X, y, train, test, scorer)
-            for train, test in cv.split(X, y, groups))
+        step_results = parallel(
+            func(clone(rfe), self.estimator, X, y, train, test, scorer, routed_params)
+            for train, test in cv.split(X, y, **routed_params.splitter.split)
+        )
+        scores, supports, rankings, step_n_features = zip(*step_results)
+
+        step_n_features_rev = np.array(step_n_features[0])[::-1]
+        scores = np.array(scores)
+        rankings = np.array(rankings)
+        supports = np.array(supports)
 
-        scores = np.sum(scores, axis=0)
-        scores_rev = scores[::-1]
-        argmax_idx = len(scores) - np.argmax(scores_rev) - 1
-        n_features_to_select = max(
-            n_features - (argmax_idx * step),
-            self.min_features_to_select)
+        # Reverse order such that lowest number of features is selected in case of tie.
+        scores_sum_rev = np.sum(scores, axis=0)[::-1]
+        n_features_to_select = step_n_features_rev[np.argmax(scores_sum_rev)]
 
         # Re-execute an elimination with best_k over the whole set
-        rfe = RFE(estimator=self.estimator,
-                  n_features_to_select=n_features_to_select, step=self.step,
-                  importance_getter=self.importance_getter,
-                  verbose=self.verbose)
+        rfe = RFE(
+            estimator=self.estimator,
+            n_features_to_select=n_features_to_select,
+            step=self.step,
+            importance_getter=self.importance_getter,
+            verbose=self.verbose,
+        )
 
-        rfe.fit(X, y)
+        rfe.fit(X, y, **routed_params.estimator.fit)
 
         # Set final attributes
         self.support_ = rfe.support_
         self.n_features_ = rfe.n_features_
         self.ranking_ = rfe.ranking_
         self.estimator_ = clone(self.estimator)
-        self.estimator_.fit(self.transform(X), y)
-
-        # Fixing a normalization error, n is equal to get_n_splits(X, y) - 1
-        # here, the scores are normalized by get_n_splits(X, y)
-        self.grid_scores_ = scores[::-1] / cv.get_n_splits(X, y, groups)
+        self.estimator_.fit(self._transform(X), y, **routed_params.estimator.fit)
+
+        # reverse to stay consistent with before
+        scores_rev = scores[:, ::-1]
+        supports_rev = supports[:, ::-1]
+        rankings_rev = rankings[:, ::-1]
+        self.cv_results_ = {
+            "mean_test_score": np.mean(scores_rev, axis=0),
+            "std_test_score": np.std(scores_rev, axis=0),
+            **{f"split{i}_test_score": scores_rev[i] for i in range(scores.shape[0])},
+            **{f"split{i}_ranking": rankings_rev[i] for i in range(rankings.shape[0])},
+            **{f"split{i}_support": supports_rev[i] for i in range(supports.shape[0])},
+            "n_features": step_n_features_rev,
+        }
         return self
+
+    def score(self, X, y, **score_params):
+        """Score using the `scoring` option on the given test data and labels.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            Test samples.
+
+        y : array-like of shape (n_samples,)
+            True labels for X.
+
+        **score_params : dict
+            Parameters to pass to the `score` method of the underlying scorer.
+
+            .. versionadded:: 1.6
+                Only available if `enable_metadata_routing=True`,
+                which can be set by using
+                ``sklearn.set_config(enable_metadata_routing=True)``.
+                See :ref:`Metadata Routing User Guide `
+                for more details.
+
+        Returns
+        -------
+        score : float
+            Score of self.predict(X) w.r.t. y defined by `scoring`.
+        """
+        _raise_for_params(score_params, self, "score")
+        scoring = self._get_scorer()
+        if _routing_enabled():
+            routed_params = process_routing(self, "score", **score_params)
+        else:
+            routed_params = Bunch()
+            routed_params.scorer = Bunch(score={})
+
+        return scoring(self, X, y, **routed_params.scorer.score)
+
+    def get_metadata_routing(self):
+        """Get metadata routing of this object.
+
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
+
+        .. versionadded:: 1.6
+
+        Returns
+        -------
+        routing : MetadataRouter
+            A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
+            routing information.
+        """
+        router = MetadataRouter(owner=self.__class__.__name__)
+        router.add(
+            estimator=self.estimator,
+            method_mapping=MethodMapping().add(caller="fit", callee="fit"),
+        )
+        router.add(
+            splitter=check_cv(self.cv),
+            method_mapping=MethodMapping().add(
+                caller="fit",
+                callee="split",
+            ),
+        )
+        router.add(
+            scorer=self._get_scorer(),
+            method_mapping=MethodMapping()
+            .add(caller="fit", callee="score")
+            .add(caller="score", callee="score"),
+        )
+
+        return router
+
+    def _get_scorer(self):
+        if self.scoring is None:
+            scoring = "accuracy" if is_classifier(self.estimator) else "r2"
+        else:
+            scoring = self.scoring
+        return get_scorer(scoring)
diff --git a/sklearn/feature_selection/_sequential.py b/sklearn/feature_selection/_sequential.py
index 7ee6b043a0df1..c6d6ed9e2e72e 100644
--- a/sklearn/feature_selection/_sequential.py
+++ b/sklearn/feature_selection/_sequential.py
@@ -1,25 +1,39 @@
 """
 Sequential feature selection
 """
-import numbers
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+from numbers import Integral, Real
 
 import numpy as np
 
+from ..base import BaseEstimator, MetaEstimatorMixin, _fit_context, clone, is_classifier
+from ..metrics import check_scoring, get_scorer_names
+from ..model_selection import check_cv, cross_val_score
+from ..utils._metadata_requests import (
+    MetadataRouter,
+    MethodMapping,
+    _raise_for_params,
+    _routing_enabled,
+    process_routing,
+)
+from ..utils._param_validation import HasMethods, Interval, RealNotInt, StrOptions
+from ..utils._tags import get_tags
+from ..utils.validation import check_is_fitted, validate_data
 from ._base import SelectorMixin
-from ..base import BaseEstimator, MetaEstimatorMixin, clone
-from ..utils._tags import _safe_tags
-from ..utils.validation import check_is_fitted
-from ..model_selection import cross_val_score
 
 
-class SequentialFeatureSelector(SelectorMixin, MetaEstimatorMixin,
-                                BaseEstimator):
+class SequentialFeatureSelector(SelectorMixin, MetaEstimatorMixin, BaseEstimator):
     """Transformer that performs Sequential Feature Selection.
 
     This Sequential Feature Selector adds (forward selection) or
     removes (backward selection) features to form a feature subset in a
     greedy fashion. At each stage, this estimator chooses the best feature to
-    add or remove based on the cross-validation score of an estimator.
+    add or remove based on the cross-validation score of an estimator. In
+    the case of unsupervised learning, this Sequential Feature Selector
+    looks only at the features (X), not the desired outputs (y).
 
     Read more in the :ref:`User Guide `.
 
@@ -30,24 +44,47 @@ class SequentialFeatureSelector(SelectorMixin, MetaEstimatorMixin,
     estimator : estimator instance
         An unfitted estimator.
 
-    n_features_to_select : int or float, default=None
-        The number of features to select. If `None`, half of the features are
-        selected. If integer, the parameter is the absolute number of features
-        to select. If float between 0 and 1, it is the fraction of features to
-        select.
+    n_features_to_select : "auto", int or float, default="auto"
+        If `"auto"`, the behaviour depends on the `tol` parameter:
+
+        - if `tol` is not `None`, then features are selected while the score
+          change does not exceed `tol`.
+        - otherwise, half of the features are selected.
+
+        If integer, the parameter is the absolute number of features to select.
+        If float between 0 and 1, it is the fraction of features to select.
+
+        .. versionadded:: 1.1
+           The option `"auto"` was added in version 1.1.
+
+        .. versionchanged:: 1.3
+           The default changed from `"warn"` to `"auto"` in 1.3.
+
+    tol : float, default=None
+        If the score is not incremented by at least `tol` between two
+        consecutive feature additions or removals, stop adding or removing.
+
+        `tol` can be negative when removing features using `direction="backward"`.
+        `tol` is required to be strictly positive when doing forward selection.
+        It can be useful to reduce the number of features at the cost of a small
+        decrease in the score.
+
+        `tol` is enabled only when `n_features_to_select` is `"auto"`.
+
+        .. versionadded:: 1.1
 
     direction : {'forward', 'backward'}, default='forward'
         Whether to perform forward selection or backward selection.
 
-    scoring : str, callable, list/tuple or dict, default=None
-        A single str (see :ref:`scoring_parameter`) or a callable
-        (see :ref:`scoring`) to evaluate the predictions on the test set.
-
-        NOTE that when using custom scorers, each scorer should return a single
-        value. Metric functions returning a list/array of values can be wrapped
-        into multiple scorers that return one value each.
+    scoring : str or callable, default=None
+        Scoring method to use for cross-validation. Options:
 
-        If None, the estimator's score method is used.
+        - str: see :ref:`scoring_string_names` for options.
+        - callable: a scorer callable object (e.g., function) with signature
+          ``scorer(estimator, X, y)`` that returns a single value.
+          See :ref:`scoring_callable` for details.
+        - `None`: the `estimator`'s
+          :ref:`default evaluation criterion ` is used.
 
     cv : int, cross-validation generator or an iterable, default=None
         Determines the cross-validation splitting strategy.
@@ -59,8 +96,11 @@ class SequentialFeatureSelector(SelectorMixin, MetaEstimatorMixin,
         - An iterable yielding (train, test) splits as arrays of indices.
 
         For integer/None inputs, if the estimator is a classifier and ``y`` is
-        either binary or multiclass, :class:`StratifiedKFold` is used. In all
-        other cases, :class:`KFold` is used.
+        either binary or multiclass,
+        :class:`~sklearn.model_selection.StratifiedKFold` is used. In all other
+        cases, :class:`~sklearn.model_selection.KFold` is used. These splitters
+        are instantiated with `shuffle=False` so the splits will be the same
+        across calls.
 
         Refer :ref:`User Guide ` for the various
         cross-validation strategies that can be used here.
@@ -75,6 +115,18 @@ class SequentialFeatureSelector(SelectorMixin, MetaEstimatorMixin,
 
     Attributes
     ----------
+    n_features_in_ : int
+        Number of features seen during :term:`fit`. Only defined if the
+        underlying estimator exposes such an attribute when fit.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     n_features_to_select_ : int
         The number of features that were selected.
 
@@ -83,6 +135,8 @@ class SequentialFeatureSelector(SelectorMixin, MetaEstimatorMixin,
 
     See Also
     --------
+    GenericUnivariateSelect : Univariate feature selector with configurable
+        strategy.
     RFE : Recursive feature elimination based on importance weights.
     RFECV : Recursive feature elimination based on importance weights, with
         automatic selection of the number of features.
@@ -105,65 +159,106 @@ class SequentialFeatureSelector(SelectorMixin, MetaEstimatorMixin,
     >>> sfs.transform(X).shape
     (150, 3)
     """
-    def __init__(self, estimator, *, n_features_to_select=None,
-                 direction='forward', scoring=None, cv=5, n_jobs=None):
 
+    _parameter_constraints: dict = {
+        "estimator": [HasMethods(["fit"])],
+        "n_features_to_select": [
+            StrOptions({"auto"}),
+            Interval(RealNotInt, 0, 1, closed="right"),
+            Interval(Integral, 0, None, closed="neither"),
+        ],
+        "tol": [None, Interval(Real, None, None, closed="neither")],
+        "direction": [StrOptions({"forward", "backward"})],
+        "scoring": [None, StrOptions(set(get_scorer_names())), callable],
+        "cv": ["cv_object"],
+        "n_jobs": [None, Integral],
+    }
+
+    def __init__(
+        self,
+        estimator,
+        *,
+        n_features_to_select="auto",
+        tol=None,
+        direction="forward",
+        scoring=None,
+        cv=5,
+        n_jobs=None,
+    ):
         self.estimator = estimator
         self.n_features_to_select = n_features_to_select
+        self.tol = tol
         self.direction = direction
         self.scoring = scoring
         self.cv = cv
         self.n_jobs = n_jobs
 
-    def fit(self, X, y):
-        """Learn the features to select.
+    @_fit_context(
+        # SequentialFeatureSelector.estimator is not validated yet
+        prefer_skip_nested_validation=False
+    )
+    def fit(self, X, y=None, **params):
+        """Learn the features to select from X.
 
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
-            Training vectors.
-        y : array-like of shape (n_samples,)
-            Target values.
+            Training vectors, where `n_samples` is the number of samples and
+            `n_features` is the number of predictors.
+
+        y : array-like of shape (n_samples,), default=None
+            Target values. This parameter may be ignored for
+            unsupervised learning.
+
+        **params : dict, default=None
+            Parameters to be passed to the underlying `estimator`, `cv`
+            and `scorer` objects.
+
+            .. versionadded:: 1.6
+
+                Only available if `enable_metadata_routing=True`,
+                which can be set by using
+                ``sklearn.set_config(enable_metadata_routing=True)``.
+                See :ref:`Metadata Routing User Guide ` for
+                more details.
 
         Returns
         -------
         self : object
+            Returns the instance itself.
         """
-        tags = self._get_tags()
-        X, y = self._validate_data(
-            X, y, accept_sparse="csc",
+        _raise_for_params(params, self, "fit")
+        tags = self.__sklearn_tags__()
+        X = validate_data(
+            self,
+            X,
+            accept_sparse="csc",
             ensure_min_features=2,
-            force_all_finite=not tags.get("allow_nan", True),
-            multi_output=True
+            ensure_all_finite=not tags.input_tags.allow_nan,
         )
         n_features = X.shape[1]
 
-        error_msg = ("n_features_to_select must be either None, an "
-                     "integer in [1, n_features - 1] "
-                     "representing the absolute "
-                     "number of features, or a float in (0, 1] "
-                     "representing a percentage of features to "
-                     f"select. Got {self.n_features_to_select}")
-        if self.n_features_to_select is None:
-            self.n_features_to_select_ = n_features // 2
-        elif isinstance(self.n_features_to_select, numbers.Integral):
-            if not 0 < self.n_features_to_select < n_features:
-                raise ValueError(error_msg)
+        if self.n_features_to_select == "auto":
+            if self.tol is not None:
+                # With auto feature selection, `n_features_to_select_` will be updated
+                # to `support_.sum()` after features are selected.
+                self.n_features_to_select_ = n_features - 1
+            else:
+                self.n_features_to_select_ = n_features // 2
+        elif isinstance(self.n_features_to_select, Integral):
+            if self.n_features_to_select >= n_features:
+                raise ValueError("n_features_to_select must be < n_features.")
             self.n_features_to_select_ = self.n_features_to_select
-        elif isinstance(self.n_features_to_select, numbers.Real):
-            if not 0 < self.n_features_to_select <= 1:
-                raise ValueError(error_msg)
-            self.n_features_to_select_ = int(n_features *
-                                             self.n_features_to_select)
-        else:
-            raise ValueError(error_msg)
-
-        if self.direction not in ('forward', 'backward'):
+        elif isinstance(self.n_features_to_select, Real):
+            self.n_features_to_select_ = int(n_features * self.n_features_to_select)
+
+        if self.tol is not None and self.tol < 0 and self.direction == "forward":
             raise ValueError(
-                "direction must be either 'forward' or 'backward'. "
-                f"Got {self.direction}."
+                "tol must be strictly positive when doing forward selection"
             )
 
+        cv = check_cv(self.cv, y, classifier=is_classifier(self.estimator))
+
         cloned_estimator = clone(self.estimator)
 
         # the current mask corresponds to the set of features:
@@ -171,43 +266,98 @@ def fit(self, X, y):
         # - that we have already *excluded* if we do backward selection
         current_mask = np.zeros(shape=n_features, dtype=bool)
         n_iterations = (
-            self.n_features_to_select_ if self.direction == 'forward'
+            self.n_features_to_select_
+            if self.n_features_to_select == "auto" or self.direction == "forward"
             else n_features - self.n_features_to_select_
         )
+
+        old_score = -np.inf
+        is_auto_select = self.tol is not None and self.n_features_to_select == "auto"
+
+        # We only need to verify the routing here and not use the routed params
+        # because internally the actual routing will also take place inside the
+        # `cross_val_score` function.
+        if _routing_enabled():
+            process_routing(self, "fit", **params)
         for _ in range(n_iterations):
-            new_feature_idx = self._get_best_new_feature(cloned_estimator, X,
-                                                         y, current_mask)
+            new_feature_idx, new_score = self._get_best_new_feature_score(
+                cloned_estimator, X, y, cv, current_mask, **params
+            )
+            if is_auto_select and ((new_score - old_score) < self.tol):
+                break
+
+            old_score = new_score
             current_mask[new_feature_idx] = True
 
-        if self.direction == 'backward':
+        if self.direction == "backward":
             current_mask = ~current_mask
+
         self.support_ = current_mask
+        self.n_features_to_select_ = self.support_.sum()
 
         return self
 
-    def _get_best_new_feature(self, estimator, X, y, current_mask):
-        # Return the best new feature to add to the current_mask, i.e. return
-        # the best new feature to add (resp. remove) when doing forward
-        # selection (resp. backward selection)
+    def _get_best_new_feature_score(self, estimator, X, y, cv, current_mask, **params):
+        # Return the best new feature and its score to add to the current_mask,
+        # i.e. return the best new feature and its score to add (resp. remove)
+        # when doing forward selection (resp. backward selection).
+        # Feature will be added if the current score and past score are greater
+        # than tol when n_feature is auto,
         candidate_feature_indices = np.flatnonzero(~current_mask)
         scores = {}
         for feature_idx in candidate_feature_indices:
             candidate_mask = current_mask.copy()
             candidate_mask[feature_idx] = True
-            if self.direction == 'backward':
+            if self.direction == "backward":
                 candidate_mask = ~candidate_mask
             X_new = X[:, candidate_mask]
             scores[feature_idx] = cross_val_score(
-                estimator, X_new, y, cv=self.cv, scoring=self.scoring,
-                n_jobs=self.n_jobs).mean()
-        return max(scores, key=lambda feature_idx: scores[feature_idx])
+                estimator,
+                X_new,
+                y,
+                cv=cv,
+                scoring=self.scoring,
+                n_jobs=self.n_jobs,
+                params=params,
+            ).mean()
+        new_feature_idx = max(scores, key=lambda feature_idx: scores[feature_idx])
+        return new_feature_idx, scores[new_feature_idx]
 
     def _get_support_mask(self):
         check_is_fitted(self)
         return self.support_
 
-    def _more_tags(self):
-        return {
-            'allow_nan': _safe_tags(self.estimator, key="allow_nan"),
-            'requires_y': True,
-        }
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.allow_nan = get_tags(self.estimator).input_tags.allow_nan
+        tags.input_tags.sparse = get_tags(self.estimator).input_tags.sparse
+        return tags
+
+    def get_metadata_routing(self):
+        """Get metadata routing of this object.
+
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
+
+        .. versionadded:: 1.6
+
+        Returns
+        -------
+        routing : MetadataRouter
+            A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
+            routing information.
+        """
+        router = MetadataRouter(owner=self.__class__.__name__)
+        router.add(
+            estimator=self.estimator,
+            method_mapping=MethodMapping().add(caller="fit", callee="fit"),
+        )
+        router.add(
+            splitter=check_cv(self.cv, classifier=is_classifier(self.estimator)),
+            method_mapping=MethodMapping().add(caller="fit", callee="split"),
+        )
+        router.add(
+            scorer=check_scoring(self.estimator, scoring=self.scoring),
+            method_mapping=MethodMapping().add(caller="fit", callee="score"),
+        )
+        return router
diff --git a/sklearn/feature_selection/_univariate_selection.py b/sklearn/feature_selection/_univariate_selection.py
index 0656e27d6e30f..7671a7ad7921d 100644
--- a/sklearn/feature_selection/_univariate_selection.py
+++ b/sklearn/feature_selection/_univariate_selection.py
@@ -1,23 +1,21 @@
 """Univariate features selection."""
 
-# Authors: V. Michel, B. Thirion, G. Varoquaux, A. Gramfort, E. Duchesnay.
-#          L. Buitinck, A. Joly
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-
-import numpy as np
 import warnings
+from numbers import Integral, Real
 
+import numpy as np
 from scipy import special, stats
 from scipy.sparse import issparse
 
-from ..base import BaseEstimator
+from ..base import BaseEstimator, _fit_context
 from ..preprocessing import LabelBinarizer
-from ..utils import (as_float_array, check_array, check_X_y, safe_sqr,
-                     safe_mask)
-from ..utils.extmath import safe_sparse_dot, row_norms
-from ..utils.validation import check_is_fitted
-from ..utils.validation import _deprecate_positional_args
+from ..utils import as_float_array, check_array, check_X_y, safe_mask, safe_sqr
+from ..utils._param_validation import Interval, StrOptions, validate_params
+from ..utils.extmath import row_norms, safe_sparse_dot
+from ..utils.validation import check_is_fitted, validate_data
 from ._base import SelectorMixin
 
 
@@ -41,7 +39,7 @@ def _clean_nans(scores):
 # Contrary to the scipy.stats.f_oneway implementation it does not
 # copy the data while keeping the inputs unchanged.
 def f_oneway(*args):
-    """Performs a 1-way ANOVA.
+    """Perform a 1-way ANOVA.
 
     The one-way ANOVA tests the null hypothesis that 2 or more groups have
     the same population mean. The test is applied to samples from two or
@@ -51,15 +49,15 @@ def f_oneway(*args):
 
     Parameters
     ----------
-    *args : array-like, sparse matrices
-        sample1, sample2... The sample measurements should be given as
+    *args : {array-like, sparse matrix}
+        Sample1, sample2... The sample measurements should be given as
         arguments.
 
     Returns
     -------
-    F-value : float
+    f_statistic : float
         The computed F-value of the test.
-    p-value : float
+    p_value : float
         The associated p-value from the F-distribution.
 
     Notes
@@ -83,13 +81,11 @@ def f_oneway(*args):
 
     References
     ----------
-
     .. [1] Lowry, Richard.  "Concepts and Applications of Inferential
            Statistics". Chapter 14.
-           http://faculty.vassar.edu/lowry/ch14pt1.html
+           http://vassarstats.net/textbook
 
     .. [2] Heiman, G.W.  Research Methods in Statistics. 2002.
-
     """
     n_classes = len(args)
     args = [as_float_array(a) for a in args]
@@ -98,9 +94,9 @@ def f_oneway(*args):
     ss_alldata = sum(safe_sqr(a).sum(axis=0) for a in args)
     sums_args = [np.asarray(a.sum(axis=0)) for a in args]
     square_of_sums_alldata = sum(sums_args) ** 2
-    square_of_sums_args = [s ** 2 for s in sums_args]
+    square_of_sums_args = [s**2 for s in sums_args]
     sstot = ss_alldata - square_of_sums_alldata / float(n_samples)
-    ssbn = 0.
+    ssbn = 0.0
     for k, _ in enumerate(args):
         ssbn += square_of_sums_args[k] / n_samples_per_class[k]
     ssbn -= square_of_sums_alldata / float(n_samples)
@@ -109,10 +105,9 @@ def f_oneway(*args):
     dfwn = n_samples - n_classes
     msb = ssbn / float(dfbn)
     msw = sswn / float(dfwn)
-    constant_features_idx = np.where(msw == 0.)[0]
-    if (np.nonzero(msb)[0].size != msb.size and constant_features_idx.size):
-        warnings.warn("Features %s are constant." % constant_features_idx,
-                      UserWarning)
+    constant_features_idx = np.where(msw == 0.0)[0]
+    if np.nonzero(msb)[0].size != msb.size and constant_features_idx.size:
+        warnings.warn("Features %s are constant." % constant_features_idx, UserWarning)
     f = msb / msw
     # flatten matrix to vector in sparse case
     f = np.asarray(f).ravel()
@@ -120,6 +115,13 @@ def f_oneway(*args):
     return f, prob
 
 
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "y": ["array-like"],
+    },
+    prefer_skip_nested_validation=True,
+)
 def f_classif(X, y):
     """Compute the ANOVA F-value for the provided sample.
 
@@ -127,26 +129,44 @@ def f_classif(X, y):
 
     Parameters
     ----------
-    X : {array-like, sparse matrix} shape = [n_samples, n_features]
+    X : {array-like, sparse matrix} of shape (n_samples, n_features)
         The set of regressors that will be tested sequentially.
 
-    y : array of shape(n_samples)
-        The data matrix.
+    y : array-like of shape (n_samples,)
+        The target vector.
 
     Returns
     -------
-    F : array, shape = [n_features,]
-        The set of F values.
+    f_statistic : ndarray of shape (n_features,)
+        F-statistic for each feature.
 
-    pval : array, shape = [n_features,]
-        The set of p-values.
+    p_values : ndarray of shape (n_features,)
+        P-values associated with the F-statistic.
 
     See Also
     --------
     chi2 : Chi-squared stats of non-negative features for classification tasks.
     f_regression : F-value between label/feature for regression tasks.
+
+    Examples
+    --------
+    >>> from sklearn.datasets import make_classification
+    >>> from sklearn.feature_selection import f_classif
+    >>> X, y = make_classification(
+    ...     n_samples=100, n_features=10, n_informative=2, n_clusters_per_class=1,
+    ...     shuffle=False, random_state=42
+    ... )
+    >>> f_statistic, p_values = f_classif(X, y)
+    >>> f_statistic
+    array([2.21e+02, 7.02e-01, 1.70e+00, 9.31e-01,
+           5.41e+00, 3.25e-01, 4.71e-02, 5.72e-01,
+           7.54e-01, 8.90e-02])
+    >>> p_values
+    array([7.14e-27, 4.04e-01, 1.96e-01, 3.37e-01,
+           2.21e-02, 5.70e-01, 8.29e-01, 4.51e-01,
+           3.87e-01, 7.66e-01])
     """
-    X, y = check_X_y(X, y, accept_sparse=['csr', 'csc', 'coo'])
+    X, y = check_X_y(X, y, accept_sparse=["csr", "csc", "coo"])
     args = [X[safe_mask(X, y == k)] for k in np.unique(y)]
     return f_oneway(*args)
 
@@ -170,14 +190,24 @@ def _chisquare(f_obs, f_exp):
     return chisq, special.chdtrc(k - 1, chisq)
 
 
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "y": ["array-like"],
+    },
+    prefer_skip_nested_validation=True,
+)
 def chi2(X, y):
     """Compute chi-squared stats between each non-negative feature and class.
 
-    This score can be used to select the n_features features with the
+    This score can be used to select the `n_features` features with the
     highest values for the test chi-squared statistic from X, which must
-    contain only non-negative features such as booleans or frequencies
+    contain only **non-negative integer feature values** such as booleans or frequencies
     (e.g., term counts in document classification), relative to the classes.
 
+    If some of your features are continuous, you need to bin them, for
+    example by using :class:`~sklearn.preprocessing.KBinsDiscretizer`.
+
     Recall that the chi-square test measures dependence between stochastic
     variables, so using this function "weeds out" the features that are the
     most likely to be independent of class and therefore irrelevant for
@@ -195,32 +225,62 @@ def chi2(X, y):
 
     Returns
     -------
-    chi2 : array, shape = (n_features,)
-        chi2 statistics of each feature.
-    pval : array, shape = (n_features,)
-        p-values of each feature.
+    chi2 : ndarray of shape (n_features,)
+        Chi2 statistics for each feature.
 
-    Notes
-    -----
-    Complexity of this algorithm is O(n_classes * n_features).
+    p_values : ndarray of shape (n_features,)
+        P-values for each feature.
 
     See Also
     --------
     f_classif : ANOVA F-value between label/feature for classification tasks.
     f_regression : F-value between label/feature for regression tasks.
+
+    Notes
+    -----
+    Complexity of this algorithm is O(n_classes * n_features).
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from sklearn.feature_selection import chi2
+    >>> X = np.array([[1, 1, 3],
+    ...               [0, 1, 5],
+    ...               [5, 4, 1],
+    ...               [6, 6, 2],
+    ...               [1, 4, 0],
+    ...               [0, 0, 0]])
+    >>> y = np.array([1, 1, 0, 0, 2, 2])
+    >>> chi2_stats, p_values = chi2(X, y)
+    >>> chi2_stats
+    array([15.3,  6.5       ,  8.9])
+    >>> p_values
+    array([0.000456, 0.0387, 0.0116 ])
     """
 
     # XXX: we might want to do some of the following in logspace instead for
     # numerical stability.
-    X = check_array(X, accept_sparse='csr')
+    # Converting X to float allows getting better performance for the
+    # safe_sparse_dot call made below.
+    X = check_array(X, accept_sparse="csr", dtype=(np.float64, np.float32))
     if np.any((X.data if issparse(X) else X) < 0):
         raise ValueError("Input X must be non-negative.")
 
-    Y = LabelBinarizer().fit_transform(y)
+    # Use a sparse representation for Y by default to reduce memory usage when
+    # y has many unique classes.
+    Y = LabelBinarizer(sparse_output=True).fit_transform(y)
     if Y.shape[1] == 1:
+        Y = Y.toarray()
         Y = np.append(1 - Y, Y, axis=1)
 
-    observed = safe_sparse_dot(Y.T, X)          # n_classes * n_features
+    observed = safe_sparse_dot(Y.T, X)  # n_classes * n_features
+
+    if issparse(observed):
+        # convert back to a dense array before calling _chisquare
+        # XXX: could _chisquare be reimplement to accept sparse matrices for
+        # cases where both n_classes and n_features are large (and X is
+        # sparse)?
+        observed = observed.toarray()
 
     feature_count = X.sum(axis=0).reshape(1, -1)
     class_prob = Y.mean(axis=0).reshape(1, -1)
@@ -229,88 +289,241 @@ def chi2(X, y):
     return _chisquare(observed, expected)
 
 
-@_deprecate_positional_args
-def f_regression(X, y, *, center=True):
-    """Univariate linear regression tests.
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "y": ["array-like"],
+        "center": ["boolean"],
+        "force_finite": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def r_regression(X, y, *, center=True, force_finite=True):
+    """Compute Pearson's r for each features and the target.
+
+    Pearson's r is also known as the Pearson correlation coefficient.
 
     Linear model for testing the individual effect of each of many regressors.
     This is a scoring function to be used in a feature selection procedure, not
     a free standing feature selection procedure.
 
-    This is done in 2 steps:
+    The cross correlation between each regressor and the target is computed
+    as::
 
-    1. The correlation between each regressor and the target is computed,
-       that is, ((X[:, i] - mean(X[:, i])) * (y - mean_y)) / (std(X[:, i]) *
-       std(y)).
-    2. It is converted to an F score then to a p-value.
+        E[(X[:, i] - mean(X[:, i])) * (y - mean(y))] / (std(X[:, i]) * std(y))
 
     For more on usage see the :ref:`User Guide `.
 
+    .. versionadded:: 1.0
+
     Parameters
     ----------
-    X : {array-like, sparse matrix}  shape = (n_samples, n_features)
-        The set of regressors that will be tested sequentially.
+    X : {array-like, sparse matrix} of shape (n_samples, n_features)
+        The data matrix.
 
-    y : array of shape(n_samples).
-        The data matrix
+    y : array-like of shape (n_samples,)
+        The target vector.
 
     center : bool, default=True
-        If true, X and y will be centered.
+        Whether or not to center the data matrix `X` and the target vector `y`.
+        By default, `X` and `y` will be centered.
+
+    force_finite : bool, default=True
+        Whether or not to force the Pearson's R correlation to be finite.
+        In the particular case where some features in `X` or the target `y`
+        are constant, the Pearson's R correlation is not defined. When
+        `force_finite=False`, a correlation of `np.nan` is returned to
+        acknowledge this case. When `force_finite=True`, this value will be
+        forced to a minimal correlation of `0.0`.
+
+        .. versionadded:: 1.1
 
     Returns
     -------
-    F : array, shape=(n_features,)
-        F values of features.
-
-    pval : array, shape=(n_features,)
-        p-values of F-scores.
+    correlation_coefficient : ndarray of shape (n_features,)
+        Pearson's R correlation coefficients of features.
 
     See Also
     --------
-    mutual_info_regression : Mutual information for a continuous target.
-    f_classif : ANOVA F-value between label/feature for classification tasks.
-    chi2 : Chi-squared stats of non-negative features for classification tasks.
-    SelectKBest : Select features based on the k highest scores.
-    SelectFpr : Select features based on a false positive rate test.
-    SelectFdr : Select features based on an estimated false discovery rate.
-    SelectFwe : Select features based on family-wise error rate.
-    SelectPercentile : Select features based on percentile of the highest
-        scores.
+    f_regression: Univariate linear regression tests returning f-statistic
+        and p-values.
+    mutual_info_regression: Mutual information for a continuous target.
+    f_classif: ANOVA F-value between label/feature for classification tasks.
+    chi2: Chi-squared stats of non-negative features for classification tasks.
+
+    Examples
+    --------
+    >>> from sklearn.datasets import make_regression
+    >>> from sklearn.feature_selection import r_regression
+    >>> X, y = make_regression(
+    ...     n_samples=50, n_features=3, n_informative=1, noise=1e-4, random_state=42
+    ... )
+    >>> r_regression(X, y)
+    array([-0.157,  1.        , -0.229])
     """
-    X, y = check_X_y(X, y, accept_sparse=['csr', 'csc', 'coo'],
-                     dtype=np.float64)
+    X, y = check_X_y(X, y, accept_sparse=["csr", "csc", "coo"], dtype=np.float64)
     n_samples = X.shape[0]
 
-    # compute centered values
-    # note that E[(x - mean(x))*(y - mean(y))] = E[x*(y - mean(y))], so we
+    # Compute centered values
+    # Note that E[(x - mean(x))*(y - mean(y))] = E[x*(y - mean(y))], so we
     # need not center X
     if center:
         y = y - np.mean(y)
-        if issparse(X):
-            X_means = X.mean(axis=0).getA1()
-        else:
-            X_means = X.mean(axis=0)
-        # compute the scaled standard deviations via moments
-        X_norms = np.sqrt(row_norms(X.T, squared=True) -
-                          n_samples * X_means ** 2)
+        # TODO: for Scipy <= 1.10, `isspmatrix(X)` returns `True` for sparse arrays.
+        # Here, we check the output of the `.mean` operation that returns a `np.matrix`
+        # for sparse matrices while a `np.array` for dense and sparse arrays.
+        # We can reconsider using `isspmatrix` when the minimum version is
+        # SciPy >= 1.11
+        X_means = X.mean(axis=0)
+        X_means = X_means.getA1() if isinstance(X_means, np.matrix) else X_means
+        # Compute the scaled standard deviations via moments
+        X_norms = np.sqrt(row_norms(X.T, squared=True) - n_samples * X_means**2)
     else:
         X_norms = row_norms(X.T)
 
-    # compute the correlation
-    corr = safe_sparse_dot(y, X)
-    corr /= X_norms
-    corr /= np.linalg.norm(y)
+    correlation_coefficient = safe_sparse_dot(y, X)
+    with np.errstate(divide="ignore", invalid="ignore"):
+        correlation_coefficient /= X_norms
+        correlation_coefficient /= np.linalg.norm(y)
+
+    if force_finite and not np.isfinite(correlation_coefficient).all():
+        # case where the target or some features are constant
+        # the correlation coefficient(s) is/are set to the minimum (i.e. 0.0)
+        nan_mask = np.isnan(correlation_coefficient)
+        correlation_coefficient[nan_mask] = 0.0
+    return correlation_coefficient
+
+
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "y": ["array-like"],
+        "center": ["boolean"],
+        "force_finite": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def f_regression(X, y, *, center=True, force_finite=True):
+    """Univariate linear regression tests returning F-statistic and p-values.
+
+    Quick linear model for testing the effect of a single regressor,
+    sequentially for many regressors.
+
+    This is done in 2 steps:
+
+    1. The cross correlation between each regressor and the target is computed
+       using :func:`r_regression` as::
 
-    # convert to p-value
-    degrees_of_freedom = y.size - (2 if center else 1)
-    F = corr ** 2 / (1 - corr ** 2) * degrees_of_freedom
-    pv = stats.f.sf(F, 1, degrees_of_freedom)
-    return F, pv
+           E[(X[:, i] - mean(X[:, i])) * (y - mean(y))] / (std(X[:, i]) * std(y))
+
+    2. It is converted to an F score and then to a p-value.
+
+    :func:`f_regression` is derived from :func:`r_regression` and will rank
+    features in the same order if all the features are positively correlated
+    with the target.
+
+    Note however that contrary to :func:`f_regression`, :func:`r_regression`
+    values lie in [-1, 1] and can thus be negative. :func:`f_regression` is
+    therefore recommended as a feature selection criterion to identify
+    potentially predictive feature for a downstream classifier, irrespective of
+    the sign of the association with the target variable.
+
+    Furthermore :func:`f_regression` returns p-values while
+    :func:`r_regression` does not.
+
+    Read more in the :ref:`User Guide `.
+
+    Parameters
+    ----------
+    X : {array-like, sparse matrix} of shape (n_samples, n_features)
+        The data matrix.
+
+    y : array-like of shape (n_samples,)
+        The target vector.
+
+    center : bool, default=True
+        Whether or not to center the data matrix `X` and the target vector `y`.
+        By default, `X` and `y` will be centered.
+
+    force_finite : bool, default=True
+        Whether or not to force the F-statistics and associated p-values to
+        be finite. There are two cases where the F-statistic is expected to not
+        be finite:
+
+        - when the target `y` or some features in `X` are constant. In this
+          case, the Pearson's R correlation is not defined leading to obtain
+          `np.nan` values in the F-statistic and p-value. When
+          `force_finite=True`, the F-statistic is set to `0.0` and the
+          associated p-value is set to `1.0`.
+        - when a feature in `X` is perfectly correlated (or
+          anti-correlated) with the target `y`. In this case, the F-statistic
+          is expected to be `np.inf`. When `force_finite=True`, the F-statistic
+          is set to `np.finfo(dtype).max` and the associated p-value is set to
+          `0.0`.
+
+        .. versionadded:: 1.1
+
+    Returns
+    -------
+    f_statistic : ndarray of shape (n_features,)
+        F-statistic for each feature.
+
+    p_values : ndarray of shape (n_features,)
+        P-values associated with the F-statistic.
+
+    See Also
+    --------
+    r_regression: Pearson's R between label/feature for regression tasks.
+    f_classif: ANOVA F-value between label/feature for classification tasks.
+    chi2: Chi-squared stats of non-negative features for classification tasks.
+    SelectKBest: Select features based on the k highest scores.
+    SelectFpr: Select features based on a false positive rate test.
+    SelectFdr: Select features based on an estimated false discovery rate.
+    SelectFwe: Select features based on family-wise error rate.
+    SelectPercentile: Select features based on percentile of the highest
+        scores.
+
+    Examples
+    --------
+    >>> from sklearn.datasets import make_regression
+    >>> from sklearn.feature_selection import f_regression
+    >>> X, y = make_regression(
+    ...     n_samples=50, n_features=3, n_informative=1, noise=1e-4, random_state=42
+    ... )
+    >>> f_statistic, p_values = f_regression(X, y)
+    >>> f_statistic
+    array([1.21, 2.67e13, 2.66])
+    >>> p_values
+    array([0.276, 1.54e-283, 0.11])
+    """
+    correlation_coefficient = r_regression(
+        X, y, center=center, force_finite=force_finite
+    )
+    deg_of_freedom = y.size - (2 if center else 1)
+
+    corr_coef_squared = correlation_coefficient**2
+
+    with np.errstate(divide="ignore", invalid="ignore"):
+        f_statistic = corr_coef_squared / (1 - corr_coef_squared) * deg_of_freedom
+        p_values = stats.f.sf(f_statistic, 1, deg_of_freedom)
+
+    if force_finite and not np.isfinite(f_statistic).all():
+        # case where there is a perfect (anti-)correlation
+        # f-statistics can be set to the maximum and p-values to zero
+        mask_inf = np.isinf(f_statistic)
+        f_statistic[mask_inf] = np.finfo(f_statistic.dtype).max
+        # case where the target or some features are constant
+        # f-statistics would be minimum and thus p-values large
+        mask_nan = np.isnan(f_statistic)
+        f_statistic[mask_nan] = 0.0
+        p_values[mask_nan] = 1.0
+    return f_statistic, p_values
 
 
 ######################################################################
 # Base classes
 
+
 class _BaseFilter(SelectorMixin, BaseEstimator):
     """Initialize the univariate feature selection.
 
@@ -321,10 +534,13 @@ class _BaseFilter(SelectorMixin, BaseEstimator):
         (scores, pvalues) or a single array with scores.
     """
 
+    _parameter_constraints: dict = {"score_func": [callable]}
+
     def __init__(self, score_func):
         self.score_func = score_func
 
-    def fit(self, X, y):
+    @_fit_context(prefer_skip_nested_validation=True)
+    def fit(self, X, y=None):
         """Run score function on (X, y) and get the appropriate features.
 
         Parameters
@@ -332,21 +548,21 @@ def fit(self, X, y):
         X : array-like of shape (n_samples, n_features)
             The training input samples.
 
-        y : array-like of shape (n_samples,)
+        y : array-like of shape (n_samples,) or None
             The target values (class labels in classification, real numbers in
-            regression).
+            regression). If the selector is unsupervised then `y` can be set to `None`.
 
         Returns
         -------
         self : object
+            Returns the instance itself.
         """
-        X, y = self._validate_data(X, y, accept_sparse=['csr', 'csc'],
-                                   multi_output=True)
-
-        if not callable(self.score_func):
-            raise TypeError("The score function should be a callable, %s (%s) "
-                            "was passed."
-                            % (self.score_func, type(self.score_func)))
+        if y is None:
+            X = validate_data(self, X, accept_sparse=["csr", "csc"])
+        else:
+            X, y = validate_data(
+                self, X, y, accept_sparse=["csr", "csc"], multi_output=True
+            )
 
         self._check_params(X, y)
         score_func_ret = self.score_func(X, y)
@@ -364,8 +580,11 @@ def fit(self, X, y):
     def _check_params(self, X, y):
         pass
 
-    def _more_tags(self):
-        return {'requires_y': True}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.target_tags.required = True
+        tags.input_tags.sparse = True
+        return tags
 
 
 ######################################################################
@@ -397,21 +616,16 @@ class SelectPercentile(_BaseFilter):
     pvalues_ : array-like of shape (n_features,)
         p-values of feature scores, None if `score_func` returned only scores.
 
-    Examples
-    --------
-    >>> from sklearn.datasets import load_digits
-    >>> from sklearn.feature_selection import SelectPercentile, chi2
-    >>> X, y = load_digits(return_X_y=True)
-    >>> X.shape
-    (1797, 64)
-    >>> X_new = SelectPercentile(chi2, percentile=10).fit_transform(X, y)
-    >>> X_new.shape
-    (1797, 7)
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
 
-    Notes
-    -----
-    Ties between features with equal scores will be broken in an unspecified
-    way.
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
 
     See Also
     --------
@@ -426,17 +640,36 @@ class SelectPercentile(_BaseFilter):
     SelectFwe : Select features based on family-wise error rate.
     GenericUnivariateSelect : Univariate feature selector with configurable
         mode.
+
+    Notes
+    -----
+    Ties between features with equal scores will be broken in an unspecified
+    way.
+
+    This filter supports unsupervised feature selection that only requests `X` for
+    computing the scores.
+
+    Examples
+    --------
+    >>> from sklearn.datasets import load_digits
+    >>> from sklearn.feature_selection import SelectPercentile, chi2
+    >>> X, y = load_digits(return_X_y=True)
+    >>> X.shape
+    (1797, 64)
+    >>> X_new = SelectPercentile(chi2, percentile=10).fit_transform(X, y)
+    >>> X_new.shape
+    (1797, 7)
     """
-    @_deprecate_positional_args
+
+    _parameter_constraints: dict = {
+        **_BaseFilter._parameter_constraints,
+        "percentile": [Interval(Real, 0, 100, closed="both")],
+    }
+
     def __init__(self, score_func=f_classif, *, percentile=10):
         super().__init__(score_func=score_func)
         self.percentile = percentile
 
-    def _check_params(self, X, y):
-        if not 0 <= self.percentile <= 100:
-            raise ValueError("percentile should be >=0, <=100; got %r"
-                             % self.percentile)
-
     def _get_support_mask(self):
         check_is_fitted(self)
 
@@ -452,10 +685,15 @@ def _get_support_mask(self):
         ties = np.where(scores == threshold)[0]
         if len(ties):
             max_feats = int(len(scores) * self.percentile / 100)
-            kept_ties = ties[:max_feats - mask.sum()]
+            kept_ties = ties[: max_feats - mask.sum()]
             mask[kept_ties] = True
         return mask
 
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.target_tags.required = False
+        return tags
+
 
 class SelectKBest(_BaseFilter):
     """Select features according to the k highest scores.
@@ -484,6 +722,40 @@ class SelectKBest(_BaseFilter):
     pvalues_ : array-like of shape (n_features,)
         p-values of feature scores, None if `score_func` returned only scores.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    f_classif: ANOVA F-value between label/feature for classification tasks.
+    mutual_info_classif: Mutual information for a discrete target.
+    chi2: Chi-squared stats of non-negative features for classification tasks.
+    f_regression: F-value between label/feature for regression tasks.
+    mutual_info_regression: Mutual information for a continuous target.
+    SelectPercentile: Select features based on percentile of the highest
+        scores.
+    SelectFpr : Select features based on a false positive rate test.
+    SelectFdr : Select features based on an estimated false discovery rate.
+    SelectFwe : Select features based on family-wise error rate.
+    GenericUnivariateSelect : Univariate feature selector with configurable
+        mode.
+
+    Notes
+    -----
+    Ties between features with equal scores will be broken in an unspecified
+    way.
+
+    This filter supports unsupervised feature selection that only requests `X` for
+    computing the scores.
+
     Examples
     --------
     >>> from sklearn.datasets import load_digits
@@ -494,42 +766,28 @@ class SelectKBest(_BaseFilter):
     >>> X_new = SelectKBest(chi2, k=20).fit_transform(X, y)
     >>> X_new.shape
     (1797, 20)
+    """
 
-    Notes
-    -----
-    Ties between features with equal scores will be broken in an unspecified
-    way.
+    _parameter_constraints: dict = {
+        **_BaseFilter._parameter_constraints,
+        "k": [StrOptions({"all"}), Interval(Integral, 0, None, closed="left")],
+    }
 
-    See Also
-    --------
-    f_classif : ANOVA F-value between label/feature for classification tasks.
-    mutual_info_classif : Mutual information for a discrete target.
-    chi2 : Chi-squared stats of non-negative features for classification tasks.
-    f_regression : F-value between label/feature for regression tasks.
-    mutual_info_regression : Mutual information for a continuous target.
-    SelectPercentile : Select features based on percentile of the highest
-        scores.
-    SelectFpr : Select features based on a false positive rate test.
-    SelectFdr : Select features based on an estimated false discovery rate.
-    SelectFwe : Select features based on family-wise error rate.
-    GenericUnivariateSelect : Univariate feature selector with configurable
-        mode.
-    """
-    @_deprecate_positional_args
     def __init__(self, score_func=f_classif, *, k=10):
         super().__init__(score_func=score_func)
         self.k = k
 
     def _check_params(self, X, y):
-        if not (self.k == "all" or 0 <= self.k <= X.shape[1]):
-            raise ValueError("k should be >=0, <= n_features = %d; got %r. "
-                             "Use k='all' to return all features."
-                             % (X.shape[1], self.k))
+        if not isinstance(self.k, str) and self.k > X.shape[1]:
+            warnings.warn(
+                f"k={self.k} is greater than n_features={X.shape[1]}. "
+                "All the features will be returned."
+            )
 
     def _get_support_mask(self):
         check_is_fitted(self)
 
-        if self.k == 'all':
+        if self.k == "all":
             return np.ones(self.scores_.shape, dtype=bool)
         elif self.k == 0:
             return np.zeros(self.scores_.shape, dtype=bool)
@@ -539,9 +797,14 @@ def _get_support_mask(self):
 
             # Request a stable sort. Mergesort takes more memory (~40MB per
             # megafeature on x86-64).
-            mask[np.argsort(scores, kind="mergesort")[-self.k:]] = 1
+            mask[np.argsort(scores, kind="mergesort")[-self.k :]] = 1
             return mask
 
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.target_tags.required = False
+        return tags
+
 
 class SelectFpr(_BaseFilter):
     """Filter: Select the pvalues below alpha based on a FPR test.
@@ -560,7 +823,7 @@ class SelectFpr(_BaseFilter):
         works with classification tasks.
 
     alpha : float, default=5e-2
-        The highest p-value for features to be kept.
+        Features with p-values less than `alpha` are selected.
 
     Attributes
     ----------
@@ -570,16 +833,16 @@ class SelectFpr(_BaseFilter):
     pvalues_ : array-like of shape (n_features,)
         p-values of feature scores.
 
-    Examples
-    --------
-    >>> from sklearn.datasets import load_breast_cancer
-    >>> from sklearn.feature_selection import SelectFpr, chi2
-    >>> X, y = load_breast_cancer(return_X_y=True)
-    >>> X.shape
-    (569, 30)
-    >>> X_new = SelectFpr(chi2, alpha=0.01).fit_transform(X, y)
-    >>> X_new.shape
-    (569, 16)
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
 
     See Also
     --------
@@ -595,8 +858,24 @@ class SelectFpr(_BaseFilter):
     SelectFwe : Select features based on family-wise error rate.
     GenericUnivariateSelect : Univariate feature selector with configurable
         mode.
+
+    Examples
+    --------
+    >>> from sklearn.datasets import load_breast_cancer
+    >>> from sklearn.feature_selection import SelectFpr, chi2
+    >>> X, y = load_breast_cancer(return_X_y=True)
+    >>> X.shape
+    (569, 30)
+    >>> X_new = SelectFpr(chi2, alpha=0.01).fit_transform(X, y)
+    >>> X_new.shape
+    (569, 16)
     """
-    @_deprecate_positional_args
+
+    _parameter_constraints: dict = {
+        **_BaseFilter._parameter_constraints,
+        "alpha": [Interval(Real, 0, 1, closed="both")],
+    }
+
     def __init__(self, score_func=f_classif, *, alpha=5e-2):
         super().__init__(score_func=score_func)
         self.alpha = alpha
@@ -608,7 +887,7 @@ def _get_support_mask(self):
 
 
 class SelectFdr(_BaseFilter):
-    """Filter: Select the p-values for an estimated false discovery rate
+    """Filter: Select the p-values for an estimated false discovery rate.
 
     This uses the Benjamini-Hochberg procedure. ``alpha`` is an upper bound
     on the expected false discovery rate.
@@ -626,17 +905,6 @@ class SelectFdr(_BaseFilter):
     alpha : float, default=5e-2
         The highest uncorrected p-value for features to keep.
 
-    Examples
-    --------
-    >>> from sklearn.datasets import load_breast_cancer
-    >>> from sklearn.feature_selection import SelectFdr, chi2
-    >>> X, y = load_breast_cancer(return_X_y=True)
-    >>> X.shape
-    (569, 30)
-    >>> X_new = SelectFdr(chi2, alpha=0.01).fit_transform(X, y)
-    >>> X_new.shape
-    (569, 16)
-
     Attributes
     ----------
     scores_ : array-like of shape (n_features,)
@@ -645,9 +913,16 @@ class SelectFdr(_BaseFilter):
     pvalues_ : array-like of shape (n_features,)
         p-values of feature scores.
 
-    References
-    ----------
-    https://en.wikipedia.org/wiki/False_discovery_rate
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
 
     See Also
     --------
@@ -655,7 +930,7 @@ class SelectFdr(_BaseFilter):
     mutual_info_classif : Mutual information for a discrete target.
     chi2 : Chi-squared stats of non-negative features for classification tasks.
     f_regression : F-value between label/feature for regression tasks.
-    mutual_info_regression : Mutual information for a contnuous target.
+    mutual_info_regression : Mutual information for a continuous target.
     SelectPercentile : Select features based on percentile of the highest
         scores.
     SelectKBest : Select features based on the k highest scores.
@@ -663,8 +938,28 @@ class SelectFdr(_BaseFilter):
     SelectFwe : Select features based on family-wise error rate.
     GenericUnivariateSelect : Univariate feature selector with configurable
         mode.
+
+    References
+    ----------
+    https://en.wikipedia.org/wiki/False_discovery_rate
+
+    Examples
+    --------
+    >>> from sklearn.datasets import load_breast_cancer
+    >>> from sklearn.feature_selection import SelectFdr, chi2
+    >>> X, y = load_breast_cancer(return_X_y=True)
+    >>> X.shape
+    (569, 30)
+    >>> X_new = SelectFdr(chi2, alpha=0.01).fit_transform(X, y)
+    >>> X_new.shape
+    (569, 16)
     """
-    @_deprecate_positional_args
+
+    _parameter_constraints: dict = {
+        **_BaseFilter._parameter_constraints,
+        "alpha": [Interval(Real, 0, 1, closed="both")],
+    }
+
     def __init__(self, score_func=f_classif, *, alpha=5e-2):
         super().__init__(score_func=score_func)
         self.alpha = alpha
@@ -674,15 +969,16 @@ def _get_support_mask(self):
 
         n_features = len(self.pvalues_)
         sv = np.sort(self.pvalues_)
-        selected = sv[sv <= float(self.alpha) / n_features *
-                      np.arange(1, n_features + 1)]
+        selected = sv[
+            sv <= float(self.alpha) / n_features * np.arange(1, n_features + 1)
+        ]
         if selected.size == 0:
             return np.zeros_like(self.pvalues_, dtype=bool)
         return self.pvalues_ <= selected.max()
 
 
 class SelectFwe(_BaseFilter):
-    """Filter: Select the p-values corresponding to Family-wise error rate
+    """Filter: Select the p-values corresponding to Family-wise error rate.
 
     Read more in the :ref:`User Guide `.
 
@@ -697,17 +993,6 @@ class SelectFwe(_BaseFilter):
     alpha : float, default=5e-2
         The highest uncorrected p-value for features to keep.
 
-    Examples
-    --------
-    >>> from sklearn.datasets import load_breast_cancer
-    >>> from sklearn.feature_selection import SelectFwe, chi2
-    >>> X, y = load_breast_cancer(return_X_y=True)
-    >>> X.shape
-    (569, 30)
-    >>> X_new = SelectFwe(chi2, alpha=0.01).fit_transform(X, y)
-    >>> X_new.shape
-    (569, 15)
-
     Attributes
     ----------
     scores_ : array-like of shape (n_features,)
@@ -716,6 +1001,17 @@ class SelectFwe(_BaseFilter):
     pvalues_ : array-like of shape (n_features,)
         p-values of feature scores.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     See Also
     --------
     f_classif : ANOVA F-value between label/feature for classification tasks.
@@ -728,8 +1024,24 @@ class SelectFwe(_BaseFilter):
     SelectFdr : Select features based on an estimated false discovery rate.
     GenericUnivariateSelect : Univariate feature selector with configurable
         mode.
+
+    Examples
+    --------
+    >>> from sklearn.datasets import load_breast_cancer
+    >>> from sklearn.feature_selection import SelectFwe, chi2
+    >>> X, y = load_breast_cancer(return_X_y=True)
+    >>> X.shape
+    (569, 30)
+    >>> X_new = SelectFwe(chi2, alpha=0.01).fit_transform(X, y)
+    >>> X_new.shape
+    (569, 15)
     """
-    @_deprecate_positional_args
+
+    _parameter_constraints: dict = {
+        **_BaseFilter._parameter_constraints,
+        "alpha": [Interval(Real, 0, 1, closed="both")],
+    }
+
     def __init__(self, score_func=f_classif, *, alpha=5e-2):
         super().__init__(score_func=score_func)
         self.alpha = alpha
@@ -737,13 +1049,14 @@ def __init__(self, score_func=f_classif, *, alpha=5e-2):
     def _get_support_mask(self):
         check_is_fitted(self)
 
-        return (self.pvalues_ < self.alpha / len(self.pvalues_))
+        return self.pvalues_ < self.alpha / len(self.pvalues_)
 
 
 ######################################################################
 # Generic filter
 ######################################################################
 
+
 # TODO this class should fit on either p-values or scores,
 # depending on the mode.
 class GenericUnivariateSelect(_BaseFilter):
@@ -759,9 +1072,10 @@ class GenericUnivariateSelect(_BaseFilter):
         a single array scores.
 
     mode : {'percentile', 'k_best', 'fpr', 'fdr', 'fwe'}, default='percentile'
-        Feature selection mode.
+        Feature selection mode. Note that the `'percentile'` and `'kbest'`
+        modes are supporting unsupervised feature selection (when `y` is `None`).
 
-    param : float or int depending on the feature selection mode, default=1e-5
+    param : "all", float or int, default=1e-5
         Parameter of the corresponding mode.
 
     Attributes
@@ -772,17 +1086,16 @@ class GenericUnivariateSelect(_BaseFilter):
     pvalues_ : array-like of shape (n_features,)
         p-values of feature scores, None if `score_func` returned scores only.
 
-    Examples
-    --------
-    >>> from sklearn.datasets import load_breast_cancer
-    >>> from sklearn.feature_selection import GenericUnivariateSelect, chi2
-    >>> X, y = load_breast_cancer(return_X_y=True)
-    >>> X.shape
-    (569, 30)
-    >>> transformer = GenericUnivariateSelect(chi2, mode='k_best', param=20)
-    >>> X_new = transformer.fit_transform(X, y)
-    >>> X_new.shape
-    (569, 20)
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
 
     See Also
     --------
@@ -797,16 +1110,35 @@ class GenericUnivariateSelect(_BaseFilter):
     SelectFpr : Select features based on a false positive rate test.
     SelectFdr : Select features based on an estimated false discovery rate.
     SelectFwe : Select features based on family-wise error rate.
-    """
 
-    _selection_modes = {'percentile': SelectPercentile,
-                        'k_best': SelectKBest,
-                        'fpr': SelectFpr,
-                        'fdr': SelectFdr,
-                        'fwe': SelectFwe}
+    Examples
+    --------
+    >>> from sklearn.datasets import load_breast_cancer
+    >>> from sklearn.feature_selection import GenericUnivariateSelect, chi2
+    >>> X, y = load_breast_cancer(return_X_y=True)
+    >>> X.shape
+    (569, 30)
+    >>> transformer = GenericUnivariateSelect(chi2, mode='k_best', param=20)
+    >>> X_new = transformer.fit_transform(X, y)
+    >>> X_new.shape
+    (569, 20)
+    """
 
-    @_deprecate_positional_args
-    def __init__(self, score_func=f_classif, *, mode='percentile', param=1e-5):
+    _selection_modes: dict = {
+        "percentile": SelectPercentile,
+        "k_best": SelectKBest,
+        "fpr": SelectFpr,
+        "fdr": SelectFdr,
+        "fwe": SelectFwe,
+    }
+
+    _parameter_constraints: dict = {
+        **_BaseFilter._parameter_constraints,
+        "mode": [StrOptions(set(_selection_modes.keys()))],
+        "param": [Interval(Real, 0, None, closed="left"), StrOptions({"all"})],
+    }
+
+    def __init__(self, score_func=f_classif, *, mode="percentile", param=1e-5):
         super().__init__(score_func=score_func)
         self.mode = mode
         self.param = param
@@ -817,18 +1149,17 @@ def _make_selector(self):
         # Now perform some acrobatics to set the right named parameter in
         # the selector
         possible_params = selector._get_param_names()
-        possible_params.remove('score_func')
+        possible_params.remove("score_func")
         selector.set_params(**{possible_params[0]: self.param})
 
         return selector
 
-    def _check_params(self, X, y):
-        if self.mode not in self._selection_modes:
-            raise ValueError("The mode passed should be one of %s, %r,"
-                             " (type %s) was passed."
-                             % (self._selection_modes.keys(), self.mode,
-                                type(self.mode)))
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.transformer_tags.preserves_dtype = ["float64", "float32"]
+        return tags
 
+    def _check_params(self, X, y):
         self._make_selector()._check_params(X, y)
 
     def _get_support_mask(self):
diff --git a/sklearn/feature_selection/_variance_threshold.py b/sklearn/feature_selection/_variance_threshold.py
index 39892876a6478..f26d70ecf8f82 100644
--- a/sklearn/feature_selection/_variance_threshold.py
+++ b/sklearn/feature_selection/_variance_threshold.py
@@ -1,11 +1,15 @@
-# Author: Lars Buitinck
-# License: 3-clause BSD
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+from numbers import Real
 
 import numpy as np
-from ..base import BaseEstimator
-from ._base import SelectorMixin
+
+from ..base import BaseEstimator, _fit_context
+from ..utils._param_validation import Interval
 from ..utils.sparsefuncs import mean_variance_axis, min_max_axis
-from ..utils.validation import check_is_fitted
+from ..utils.validation import check_is_fitted, validate_data
+from ._base import SelectorMixin
 
 
 class VarianceThreshold(SelectorMixin, BaseEstimator):
@@ -28,6 +32,26 @@ class VarianceThreshold(SelectorMixin, BaseEstimator):
     variances_ : array, shape (n_features,)
         Variances of individual features.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    SelectFromModel: Meta-transformer for selecting features based on
+        importance weights.
+    SelectPercentile : Select features according to a percentile of the highest
+        scores.
+    SequentialFeatureSelector : Transformer that performs Sequential Feature
+        Selection.
+
     Notes
     -----
     Allows NaN in the input.
@@ -38,6 +62,7 @@ class VarianceThreshold(SelectorMixin, BaseEstimator):
     The following dataset has integer features, two of which are the same
     in every sample. These are removed with the default setting for threshold::
 
+        >>> from sklearn.feature_selection import VarianceThreshold
         >>> X = [[0, 2, 0, 3], [0, 1, 4, 3], [0, 1, 1, 3]]
         >>> selector = VarianceThreshold()
         >>> selector.fit_transform(X)
@@ -46,16 +71,22 @@ class VarianceThreshold(SelectorMixin, BaseEstimator):
                [1, 1]])
     """
 
-    def __init__(self, threshold=0.):
+    _parameter_constraints: dict = {
+        "threshold": [Interval(Real, 0, None, closed="left")]
+    }
+
+    def __init__(self, threshold=0.0):
         self.threshold = threshold
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y=None):
         """Learn empirical variances from X.
 
         Parameters
         ----------
         X : {array-like, sparse matrix}, shape (n_samples, n_features)
-            Sample vectors from which to compute variances.
+            Data from which to compute variances, where `n_samples` is
+            the number of samples and `n_features` is the number of features.
 
         y : any, default=None
             Ignored. This parameter exists only for compatibility with
@@ -63,13 +94,18 @@ def fit(self, X, y=None):
 
         Returns
         -------
-        self
+        self : object
+            Returns the instance itself.
         """
-        X = self._validate_data(X, accept_sparse=('csr', 'csc'),
-                                dtype=np.float64,
-                                force_all_finite='allow-nan')
-
-        if hasattr(X, "toarray"):   # sparse matrix
+        X = validate_data(
+            self,
+            X,
+            accept_sparse=("csr", "csc"),
+            dtype=np.float64,
+            ensure_all_finite="allow-nan",
+        )
+
+        if hasattr(X, "toarray"):  # sparse matrix
             _, self.variances_ = mean_variance_axis(X, axis=0)
             if self.threshold == 0:
                 mins, maxes = min_max_axis(X, axis=0)
@@ -85,8 +121,7 @@ def fit(self, X, y=None):
             compare_arr = np.array([self.variances_, peak_to_peaks])
             self.variances_ = np.nanmin(compare_arr, axis=0)
 
-        if np.all(~np.isfinite(self.variances_) |
-                  (self.variances_ <= self.threshold)):
+        if np.all(~np.isfinite(self.variances_) | (self.variances_ <= self.threshold)):
             msg = "No feature in X meets the variance threshold {0:.5f}"
             if X.shape[0] == 1:
                 msg += " (X contains only one sample)"
@@ -99,5 +134,8 @@ def _get_support_mask(self):
 
         return self.variances_ > self.threshold
 
-    def _more_tags(self):
-        return {'allow_nan': True}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.allow_nan = True
+        tags.input_tags.sparse = True
+        return tags
diff --git a/sklearn/feature_selection/tests/test_base.py b/sklearn/feature_selection/tests/test_base.py
index 9515bdc32c600..0bf51a80f01ba 100644
--- a/sklearn/feature_selection/tests/test_base.py
+++ b/sklearn/feature_selection/tests/test_base.py
@@ -1,27 +1,30 @@
 import numpy as np
 import pytest
-from scipy import sparse as sp
-
 from numpy.testing import assert_array_equal
 
 from sklearn.base import BaseEstimator
 from sklearn.feature_selection._base import SelectorMixin
-from sklearn.utils import check_array
+from sklearn.utils.fixes import CSC_CONTAINERS
+from sklearn.utils.validation import validate_data
 
 
 class StepSelector(SelectorMixin, BaseEstimator):
-    """Retain every `step` features (beginning with 0)"""
+    """Retain every `step` features (beginning with 0).
+
+    If `step < 1`, then no features are selected.
+    """
+
     def __init__(self, step=2):
         self.step = step
 
     def fit(self, X, y=None):
-        X = check_array(X, accept_sparse='csc')
-        self.n_input_feats = X.shape[1]
+        X = validate_data(self, X, accept_sparse="csc")
         return self
 
     def _get_support_mask(self):
-        mask = np.zeros(self.n_input_feats, dtype=bool)
-        mask[::self.step] = True
+        mask = np.zeros(self.n_features_in_, dtype=bool)
+        if self.step >= 1:
+            mask[:: self.step] = True
         return mask
 
 
@@ -32,10 +35,10 @@ def _get_support_mask(self):
 Xinv = X.copy()
 Xinv[:, 1::2] = 0
 y = [0, 1]
-feature_names = list('ABCDEFGHIJ')
+feature_names = list("ABCDEFGHIJ")
 feature_names_t = feature_names[::2]
 feature_names_inv = np.array(feature_names)
-feature_names_inv[1::2] = ''
+feature_names_inv[1::2] = ""
 
 
 def test_transform_dense():
@@ -58,17 +61,18 @@ def test_transform_dense():
         sel.transform(np.array([[1], [2]]))
 
 
-def test_transform_sparse():
-    sparse = sp.csc_matrix
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+def test_transform_sparse(csc_container):
+    X_sp = csc_container(X)
     sel = StepSelector()
-    Xt_actual = sel.fit(sparse(X)).transform(sparse(X))
-    Xt_actual2 = sel.fit_transform(sparse(X))
+    Xt_actual = sel.fit(X_sp).transform(X_sp)
+    Xt_actual2 = sel.fit_transform(X_sp)
     assert_array_equal(Xt, Xt_actual.toarray())
     assert_array_equal(Xt, Xt_actual2.toarray())
 
     # Check dtype matches
-    assert np.int32 == sel.transform(sparse(X).astype(np.int32)).dtype
-    assert np.float32 == sel.transform(sparse(X).astype(np.float32)).dtype
+    assert np.int32 == sel.transform(X_sp.astype(np.int32)).dtype
+    assert np.float32 == sel.transform(X_sp.astype(np.float32)).dtype
 
     # Check wrong shape raises error
     with pytest.raises(ValueError):
@@ -81,10 +85,8 @@ def test_inverse_transform_dense():
     assert_array_equal(Xinv, Xinv_actual)
 
     # Check dtype matches
-    assert (np.int32 ==
-                 sel.inverse_transform(Xt.astype(np.int32)).dtype)
-    assert (np.float32 ==
-                 sel.inverse_transform(Xt.astype(np.float32)).dtype)
+    assert np.int32 == sel.inverse_transform(Xt.astype(np.int32)).dtype
+    assert np.float32 == sel.inverse_transform(Xt.astype(np.float32)).dtype
 
     # Check 1d list and other dtype:
     names_inv_actual = sel.inverse_transform([feature_names_t])
@@ -95,17 +97,17 @@ def test_inverse_transform_dense():
         sel.inverse_transform(np.array([[1], [2]]))
 
 
-def test_inverse_transform_sparse():
-    sparse = sp.csc_matrix
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+def test_inverse_transform_sparse(csc_container):
+    X_sp = csc_container(X)
+    Xt_sp = csc_container(Xt)
     sel = StepSelector()
-    Xinv_actual = sel.fit(sparse(X)).inverse_transform(sparse(Xt))
+    Xinv_actual = sel.fit(X_sp).inverse_transform(Xt_sp)
     assert_array_equal(Xinv, Xinv_actual.toarray())
 
     # Check dtype matches
-    assert (np.int32 ==
-                 sel.inverse_transform(sparse(Xt).astype(np.int32)).dtype)
-    assert (np.float32 ==
-                 sel.inverse_transform(sparse(Xt).astype(np.float32)).dtype)
+    assert np.int32 == sel.inverse_transform(Xt_sp.astype(np.int32)).dtype
+    assert np.float32 == sel.inverse_transform(Xt_sp.astype(np.float32)).dtype
 
     # Check wrong shape raises error
     with pytest.raises(ValueError):
@@ -117,3 +119,36 @@ def test_get_support():
     sel.fit(X, y)
     assert_array_equal(support, sel.get_support())
     assert_array_equal(support_inds, sel.get_support(indices=True))
+
+
+def test_output_dataframe():
+    """Check output dtypes for dataframes is consistent with the input dtypes."""
+    pd = pytest.importorskip("pandas")
+
+    X = pd.DataFrame(
+        {
+            "a": pd.Series([1.0, 2.4, 4.5], dtype=np.float32),
+            "b": pd.Series(["a", "b", "a"], dtype="category"),
+            "c": pd.Series(["j", "b", "b"], dtype="category"),
+            "d": pd.Series([3.0, 2.4, 1.2], dtype=np.float64),
+        }
+    )
+
+    for step in [2, 3]:
+        sel = StepSelector(step=step).set_output(transform="pandas")
+        sel.fit(X)
+
+        output = sel.transform(X)
+        for name, dtype in output.dtypes.items():
+            assert dtype == X.dtypes[name]
+
+    # step=0 will select nothing
+    sel0 = StepSelector(step=0).set_output(transform="pandas")
+    sel0.fit(X, y)
+
+    msg = "No features were selected"
+    with pytest.warns(UserWarning, match=msg):
+        output0 = sel0.transform(X)
+
+    assert_array_equal(output0.index, X.index)
+    assert output0.shape == (X.shape[0], 0)
diff --git a/sklearn/feature_selection/tests/test_chi2.py b/sklearn/feature_selection/tests/test_chi2.py
index 29a027bdb27a2..c50def36f1b6c 100644
--- a/sklearn/feature_selection/tests/test_chi2.py
+++ b/sklearn/feature_selection/tests/test_chi2.py
@@ -7,21 +7,17 @@
 
 import numpy as np
 import pytest
-from scipy.sparse import coo_matrix, csr_matrix
 import scipy.stats
 
 from sklearn.feature_selection import SelectKBest, chi2
 from sklearn.feature_selection._univariate_selection import _chisquare
-from sklearn.utils._testing import assert_array_almost_equal
-from sklearn.utils._testing import assert_array_equal
+from sklearn.utils._testing import assert_array_almost_equal, assert_array_equal
+from sklearn.utils.fixes import COO_CONTAINERS, CSR_CONTAINERS
 
 # Feature 0 is highly informative for class 1;
 # feature 1 is the same everywhere;
 # feature 2 is a bit informative for class 2.
-X = [[2, 1, 2],
-     [9, 1, 1],
-     [6, 1, 2],
-     [0, 1, 2]]
+X = [[2, 1, 2], [9, 1, 1], [6, 1, 2], [0, 1, 2]]
 y = [0, 1, 2, 2]
 
 
@@ -30,7 +26,8 @@ def mkchi2(k):
     return SelectKBest(chi2, k=k)
 
 
-def test_chi2():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_chi2(csr_container):
     # Test Chi2 feature extraction
 
     chi2 = mkchi2(k=1).fit(X, y)
@@ -41,7 +38,7 @@ def test_chi2():
     chi2 = mkchi2(k=2).fit(X, y)
     assert_array_equal(sorted(chi2.get_support(indices=True)), [0, 2])
 
-    Xsp = csr_matrix(X, dtype=np.float64)
+    Xsp = csr_container(X, dtype=np.float64)
     chi2 = mkchi2(k=2).fit(Xsp, y)
     assert_array_equal(sorted(chi2.get_support(indices=True)), [0, 2])
     Xtrans = chi2.transform(Xsp)
@@ -53,18 +50,20 @@ def test_chi2():
     assert_array_almost_equal(Xtrans, Xtrans2)
 
 
-def test_chi2_coo():
+@pytest.mark.parametrize("coo_container", COO_CONTAINERS)
+def test_chi2_coo(coo_container):
     # Check that chi2 works with a COO matrix
     # (as returned by CountVectorizer, DictVectorizer)
-    Xcoo = coo_matrix(X)
+    Xcoo = coo_container(X)
     mkchi2(k=2).fit_transform(Xcoo, y)
     # if we got here without an exception, we're safe
 
 
-def test_chi2_negative():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_chi2_negative(csr_container):
     # Check for proper error on negative numbers in the input X.
     X, y = [[0, 1], [-1e-20, 1]], [0, 1]
-    for X in (X, np.array(X), csr_matrix(X)):
+    for X in (X, np.array(X), csr_container(X)):
         with pytest.raises(ValueError):
             chi2(X, y)
 
@@ -73,21 +72,19 @@ def test_chi2_unused_feature():
     # Unused feature should evaluate to NaN
     # and should issue no runtime warning
     with warnings.catch_warnings(record=True) as warned:
-        warnings.simplefilter('always')
+        warnings.simplefilter("always")
         chi, p = chi2([[1, 0], [0, 0]], [1, 0])
         for w in warned:
-            if 'divide by zero' in repr(w):
-                raise AssertionError('Found unexpected warning %s' % w)
+            if "divide by zero" in repr(w):
+                raise AssertionError("Found unexpected warning %s" % w)
     assert_array_equal(chi, [1, np.nan])
     assert_array_equal(p[1], np.nan)
 
 
 def test_chisquare():
     # Test replacement for scipy.stats.chisquare against the original.
-    obs = np.array([[2., 2.],
-                    [1., 1.]])
-    exp = np.array([[1.5, 1.5],
-                    [1.5, 1.5]])
+    obs = np.array([[2.0, 2.0], [1.0, 1.0]])
+    exp = np.array([[1.5, 1.5], [1.5, 1.5]])
     # call SciPy first because our version overwrites obs
     chi_scp, p_scp = scipy.stats.chisquare(obs, exp)
     chi_our, p_our = _chisquare(obs, exp)
diff --git a/sklearn/feature_selection/tests/test_feature_select.py b/sklearn/feature_selection/tests/test_feature_select.py
index 61f709094147e..d7bffec5159bf 100644
--- a/sklearn/feature_selection/tests/test_feature_select.py
+++ b/sklearn/feature_selection/tests/test_feature_select.py
@@ -1,31 +1,45 @@
 """
 Todo: cross-check the F-value with stats model
 """
+
 import itertools
 import warnings
-import numpy as np
-from scipy import stats, sparse
 
+import numpy as np
 import pytest
+from numpy.testing import assert_allclose
+from scipy import sparse, stats
 
-from sklearn.utils._testing import assert_almost_equal
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils._testing import assert_array_almost_equal
-from sklearn.utils._testing import assert_warns
-from sklearn.utils._testing import ignore_warnings
-from sklearn.utils._testing import assert_warns_message
-from sklearn.utils import safe_mask
-
-from sklearn.datasets import make_classification, make_regression
+from sklearn.datasets import load_iris, make_classification, make_regression
 from sklearn.feature_selection import (
-    chi2, f_classif, f_oneway, f_regression, mutual_info_classif,
-    mutual_info_regression, SelectPercentile, SelectKBest, SelectFpr,
-    SelectFdr, SelectFwe, GenericUnivariateSelect)
-
+    GenericUnivariateSelect,
+    SelectFdr,
+    SelectFpr,
+    SelectFwe,
+    SelectKBest,
+    SelectPercentile,
+    chi2,
+    f_classif,
+    f_oneway,
+    f_regression,
+    mutual_info_classif,
+    mutual_info_regression,
+    r_regression,
+)
+from sklearn.utils import safe_mask
+from sklearn.utils._testing import (
+    _convert_container,
+    assert_almost_equal,
+    assert_array_almost_equal,
+    assert_array_equal,
+    ignore_warnings,
+)
+from sklearn.utils.fixes import CSR_CONTAINERS
 
 ##############################################################################
 # Test the score functions
 
+
 def test_f_oneway_vs_scipy_stats():
     # Test that our f_oneway gives the same result as scipy.stats
     rng = np.random.RandomState(0)
@@ -51,50 +65,83 @@ def test_f_oneway_ints():
     assert_array_almost_equal(p, pint, decimal=4)
 
 
-def test_f_classif():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_f_classif(csr_container):
     # Test whether the F test yields meaningful results
     # on a simple simulated classification problem
-    X, y = make_classification(n_samples=200, n_features=20,
-                               n_informative=3, n_redundant=2,
-                               n_repeated=0, n_classes=8,
-                               n_clusters_per_class=1, flip_y=0.0,
-                               class_sep=10, shuffle=False, random_state=0)
+    X, y = make_classification(
+        n_samples=200,
+        n_features=20,
+        n_informative=3,
+        n_redundant=2,
+        n_repeated=0,
+        n_classes=8,
+        n_clusters_per_class=1,
+        flip_y=0.0,
+        class_sep=10,
+        shuffle=False,
+        random_state=0,
+    )
 
     F, pv = f_classif(X, y)
-    F_sparse, pv_sparse = f_classif(sparse.csr_matrix(X), y)
+    F_sparse, pv_sparse = f_classif(csr_container(X), y)
     assert (F > 0).all()
     assert (pv > 0).all()
     assert (pv < 1).all()
     assert (pv[:5] < 0.05).all()
-    assert (pv[5:] > 1.e-4).all()
+    assert (pv[5:] > 1.0e-4).all()
     assert_array_almost_equal(F_sparse, F)
     assert_array_almost_equal(pv_sparse, pv)
 
 
-def test_f_regression():
+@pytest.mark.parametrize("center", [True, False])
+def test_r_regression(center):
+    X, y = make_regression(
+        n_samples=2000, n_features=20, n_informative=5, shuffle=False, random_state=0
+    )
+
+    corr_coeffs = r_regression(X, y, center=center)
+    assert (-1 < corr_coeffs).all()
+    assert (corr_coeffs < 1).all()
+
+    sparse_X = _convert_container(X, "sparse")
+
+    sparse_corr_coeffs = r_regression(sparse_X, y, center=center)
+    assert_allclose(sparse_corr_coeffs, corr_coeffs)
+
+    # Testing against numpy for reference
+    Z = np.hstack((X, y[:, np.newaxis]))
+    correlation_matrix = np.corrcoef(Z, rowvar=False)
+    np_corr_coeffs = correlation_matrix[:-1, -1]
+    assert_array_almost_equal(np_corr_coeffs, corr_coeffs, decimal=3)
+
+
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_f_regression(csr_container):
     # Test whether the F test yields meaningful results
     # on a simple simulated regression problem
-    X, y = make_regression(n_samples=200, n_features=20, n_informative=5,
-                           shuffle=False, random_state=0)
+    X, y = make_regression(
+        n_samples=200, n_features=20, n_informative=5, shuffle=False, random_state=0
+    )
 
     F, pv = f_regression(X, y)
     assert (F > 0).all()
     assert (pv > 0).all()
     assert (pv < 1).all()
     assert (pv[:5] < 0.05).all()
-    assert (pv[5:] > 1.e-4).all()
+    assert (pv[5:] > 1.0e-4).all()
 
     # with centering, compare with sparse
     F, pv = f_regression(X, y, center=True)
-    F_sparse, pv_sparse = f_regression(sparse.csr_matrix(X), y, center=True)
-    assert_array_almost_equal(F_sparse, F)
-    assert_array_almost_equal(pv_sparse, pv)
+    F_sparse, pv_sparse = f_regression(csr_container(X), y, center=True)
+    assert_allclose(F_sparse, F)
+    assert_allclose(pv_sparse, pv)
 
     # again without centering, compare with sparse
     F, pv = f_regression(X, y, center=False)
-    F_sparse, pv_sparse = f_regression(sparse.csr_matrix(X), y, center=False)
-    assert_array_almost_equal(F_sparse, F)
-    assert_array_almost_equal(pv_sparse, pv)
+    F_sparse, pv_sparse = f_regression(csr_container(X), y, center=False)
+    assert_allclose(F_sparse, F)
+    assert_allclose(pv_sparse, pv)
 
 
 def test_f_regression_input_dtype():
@@ -106,8 +153,8 @@ def test_f_regression_input_dtype():
 
     F1, pv1 = f_regression(X, y)
     F2, pv2 = f_regression(X, y.astype(float))
-    assert_array_almost_equal(F1, F2, 5)
-    assert_array_almost_equal(pv1, pv2, 5)
+    assert_allclose(F1, F2, 5)
+    assert_allclose(pv1, pv2, 5)
 
 
 def test_f_regression_center():
@@ -118,46 +165,194 @@ def test_f_regression_center():
     X = np.arange(-5, 6).reshape(-1, 1)  # X has zero mean
     n_samples = X.size
     Y = np.ones(n_samples)
-    Y[::2] *= -1.
-    Y[0] = 0.  # have Y mean being null
+    Y[::2] *= -1.0
+    Y[0] = 0.0  # have Y mean being null
 
     F1, _ = f_regression(X, Y, center=True)
     F2, _ = f_regression(X, Y, center=False)
-    assert_array_almost_equal(F1 * (n_samples - 1.) / (n_samples - 2.), F2)
+    assert_allclose(F1 * (n_samples - 1.0) / (n_samples - 2.0), F2)
     assert_almost_equal(F2[0], 0.232558139)  # value from statsmodels OLS
 
 
+@pytest.mark.parametrize(
+    "X, y, expected_corr_coef, force_finite",
+    [
+        (
+            # A feature in X is constant - forcing finite
+            np.array([[2, 1], [2, 0], [2, 10], [2, 4]]),
+            np.array([0, 1, 1, 0]),
+            np.array([0.0, 0.32075]),
+            True,
+        ),
+        (
+            # The target y is constant - forcing finite
+            np.array([[5, 1], [3, 0], [2, 10], [8, 4]]),
+            np.array([0, 0, 0, 0]),
+            np.array([0.0, 0.0]),
+            True,
+        ),
+        (
+            # A feature in X is constant - not forcing finite
+            np.array([[2, 1], [2, 0], [2, 10], [2, 4]]),
+            np.array([0, 1, 1, 0]),
+            np.array([np.nan, 0.32075]),
+            False,
+        ),
+        (
+            # The target y is constant - not forcing finite
+            np.array([[5, 1], [3, 0], [2, 10], [8, 4]]),
+            np.array([0, 0, 0, 0]),
+            np.array([np.nan, np.nan]),
+            False,
+        ),
+    ],
+)
+def test_r_regression_force_finite(X, y, expected_corr_coef, force_finite):
+    """Check the behaviour of `force_finite` for some corner cases with `r_regression`.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/15672
+    """
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", RuntimeWarning)
+        corr_coef = r_regression(X, y, force_finite=force_finite)
+    np.testing.assert_array_almost_equal(corr_coef, expected_corr_coef)
+
+
+@pytest.mark.parametrize(
+    "X, y, expected_f_statistic, expected_p_values, force_finite",
+    [
+        (
+            # A feature in X is constant - forcing finite
+            np.array([[2, 1], [2, 0], [2, 10], [2, 4]]),
+            np.array([0, 1, 1, 0]),
+            np.array([0.0, 0.2293578]),
+            np.array([1.0, 0.67924985]),
+            True,
+        ),
+        (
+            # The target y is constant - forcing finite
+            np.array([[5, 1], [3, 0], [2, 10], [8, 4]]),
+            np.array([0, 0, 0, 0]),
+            np.array([0.0, 0.0]),
+            np.array([1.0, 1.0]),
+            True,
+        ),
+        (
+            # Feature in X correlated with y - forcing finite
+            np.array([[0, 1], [1, 0], [2, 10], [3, 4]]),
+            np.array([0, 1, 2, 3]),
+            np.array([np.finfo(np.float64).max, 0.845433]),
+            np.array([0.0, 0.454913]),
+            True,
+        ),
+        (
+            # Feature in X anti-correlated with y - forcing finite
+            np.array([[3, 1], [2, 0], [1, 10], [0, 4]]),
+            np.array([0, 1, 2, 3]),
+            np.array([np.finfo(np.float64).max, 0.845433]),
+            np.array([0.0, 0.454913]),
+            True,
+        ),
+        (
+            # A feature in X is constant - not forcing finite
+            np.array([[2, 1], [2, 0], [2, 10], [2, 4]]),
+            np.array([0, 1, 1, 0]),
+            np.array([np.nan, 0.2293578]),
+            np.array([np.nan, 0.67924985]),
+            False,
+        ),
+        (
+            # The target y is constant - not forcing finite
+            np.array([[5, 1], [3, 0], [2, 10], [8, 4]]),
+            np.array([0, 0, 0, 0]),
+            np.array([np.nan, np.nan]),
+            np.array([np.nan, np.nan]),
+            False,
+        ),
+        (
+            # Feature in X correlated with y - not forcing finite
+            np.array([[0, 1], [1, 0], [2, 10], [3, 4]]),
+            np.array([0, 1, 2, 3]),
+            np.array([np.inf, 0.845433]),
+            np.array([0.0, 0.454913]),
+            False,
+        ),
+        (
+            # Feature in X anti-correlated with y - not forcing finite
+            np.array([[3, 1], [2, 0], [1, 10], [0, 4]]),
+            np.array([0, 1, 2, 3]),
+            np.array([np.inf, 0.845433]),
+            np.array([0.0, 0.454913]),
+            False,
+        ),
+    ],
+)
+def test_f_regression_corner_case(
+    X, y, expected_f_statistic, expected_p_values, force_finite
+):
+    """Check the behaviour of `force_finite` for some corner cases with `f_regression`.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/15672
+    """
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", RuntimeWarning)
+        f_statistic, p_values = f_regression(X, y, force_finite=force_finite)
+    np.testing.assert_array_almost_equal(f_statistic, expected_f_statistic)
+    np.testing.assert_array_almost_equal(p_values, expected_p_values)
+
+
 def test_f_classif_multi_class():
     # Test whether the F test yields meaningful results
     # on a simple simulated classification problem
-    X, y = make_classification(n_samples=200, n_features=20,
-                               n_informative=3, n_redundant=2,
-                               n_repeated=0, n_classes=8,
-                               n_clusters_per_class=1, flip_y=0.0,
-                               class_sep=10, shuffle=False, random_state=0)
+    X, y = make_classification(
+        n_samples=200,
+        n_features=20,
+        n_informative=3,
+        n_redundant=2,
+        n_repeated=0,
+        n_classes=8,
+        n_clusters_per_class=1,
+        flip_y=0.0,
+        class_sep=10,
+        shuffle=False,
+        random_state=0,
+    )
 
     F, pv = f_classif(X, y)
     assert (F > 0).all()
     assert (pv > 0).all()
     assert (pv < 1).all()
     assert (pv[:5] < 0.05).all()
-    assert (pv[5:] > 1.e-4).all()
+    assert (pv[5:] > 1.0e-4).all()
 
 
 def test_select_percentile_classif():
     # Test whether the relative univariate feature selection
     # gets the correct items in a simple classification problem
     # with the percentile heuristic
-    X, y = make_classification(n_samples=200, n_features=20,
-                               n_informative=3, n_redundant=2,
-                               n_repeated=0, n_classes=8,
-                               n_clusters_per_class=1, flip_y=0.0,
-                               class_sep=10, shuffle=False, random_state=0)
+    X, y = make_classification(
+        n_samples=200,
+        n_features=20,
+        n_informative=3,
+        n_redundant=2,
+        n_repeated=0,
+        n_classes=8,
+        n_clusters_per_class=1,
+        flip_y=0.0,
+        class_sep=10,
+        shuffle=False,
+        random_state=0,
+    )
 
     univariate_filter = SelectPercentile(f_classif, percentile=25)
     X_r = univariate_filter.fit(X, y).transform(X)
-    X_r2 = GenericUnivariateSelect(f_classif, mode='percentile',
-                                   param=25).fit(X, y).transform(X)
+    X_r2 = (
+        GenericUnivariateSelect(f_classif, mode="percentile", param=25)
+        .fit(X, y)
+        .transform(X)
+    )
     assert_array_equal(X_r, X_r2)
     support = univariate_filter.get_support()
     gtruth = np.zeros(20)
@@ -165,20 +360,32 @@ def test_select_percentile_classif():
     assert_array_equal(support, gtruth)
 
 
-def test_select_percentile_classif_sparse():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_select_percentile_classif_sparse(csr_container):
     # Test whether the relative univariate feature selection
     # gets the correct items in a simple classification problem
     # with the percentile heuristic
-    X, y = make_classification(n_samples=200, n_features=20,
-                               n_informative=3, n_redundant=2,
-                               n_repeated=0, n_classes=8,
-                               n_clusters_per_class=1, flip_y=0.0,
-                               class_sep=10, shuffle=False, random_state=0)
-    X = sparse.csr_matrix(X)
+    X, y = make_classification(
+        n_samples=200,
+        n_features=20,
+        n_informative=3,
+        n_redundant=2,
+        n_repeated=0,
+        n_classes=8,
+        n_clusters_per_class=1,
+        flip_y=0.0,
+        class_sep=10,
+        shuffle=False,
+        random_state=0,
+    )
+    X = csr_container(X)
     univariate_filter = SelectPercentile(f_classif, percentile=25)
     X_r = univariate_filter.fit(X, y).transform(X)
-    X_r2 = GenericUnivariateSelect(f_classif, mode='percentile',
-                                   param=25).fit(X, y).transform(X)
+    X_r2 = (
+        GenericUnivariateSelect(f_classif, mode="percentile", param=25)
+        .fit(X, y)
+        .transform(X)
+    )
     assert_array_equal(X_r.toarray(), X_r2.toarray())
     support = univariate_filter.get_support()
     gtruth = np.zeros(20)
@@ -191,26 +398,38 @@ def test_select_percentile_classif_sparse():
     assert X_r2inv.shape == X.shape
     assert_array_equal(X_r2inv[:, support_mask].toarray(), X_r.toarray())
     # Check other columns are empty
-    assert X_r2inv.getnnz() == X_r.getnnz()
+    assert X_r2inv.nnz == X_r.nnz
 
 
 ##############################################################################
 # Test univariate selection in classification settings
 
+
 def test_select_kbest_classif():
     # Test whether the relative univariate feature selection
     # gets the correct items in a simple classification problem
     # with the k best heuristic
-    X, y = make_classification(n_samples=200, n_features=20,
-                               n_informative=3, n_redundant=2,
-                               n_repeated=0, n_classes=8,
-                               n_clusters_per_class=1, flip_y=0.0,
-                               class_sep=10, shuffle=False, random_state=0)
+    X, y = make_classification(
+        n_samples=200,
+        n_features=20,
+        n_informative=3,
+        n_redundant=2,
+        n_repeated=0,
+        n_classes=8,
+        n_clusters_per_class=1,
+        flip_y=0.0,
+        class_sep=10,
+        shuffle=False,
+        random_state=0,
+    )
 
     univariate_filter = SelectKBest(f_classif, k=5)
     X_r = univariate_filter.fit(X, y).transform(X)
-    X_r2 = GenericUnivariateSelect(
-        f_classif, mode='k_best', param=5).fit(X, y).transform(X)
+    X_r2 = (
+        GenericUnivariateSelect(f_classif, mode="k_best", param=5)
+        .fit(X, y)
+        .transform(X)
+    )
     assert_array_equal(X_r, X_r2)
     support = univariate_filter.get_support()
     gtruth = np.zeros(20)
@@ -220,49 +439,73 @@ def test_select_kbest_classif():
 
 def test_select_kbest_all():
     # Test whether k="all" correctly returns all features.
-    X, y = make_classification(n_samples=20, n_features=10,
-                               shuffle=False, random_state=0)
+    X, y = make_classification(
+        n_samples=20, n_features=10, shuffle=False, random_state=0
+    )
 
-    univariate_filter = SelectKBest(f_classif, k='all')
+    univariate_filter = SelectKBest(f_classif, k="all")
     X_r = univariate_filter.fit(X, y).transform(X)
     assert_array_equal(X, X_r)
+    # Non-regression test for:
+    # https://github.com/scikit-learn/scikit-learn/issues/24949
+    X_r2 = (
+        GenericUnivariateSelect(f_classif, mode="k_best", param="all")
+        .fit(X, y)
+        .transform(X)
+    )
+    assert_array_equal(X_r, X_r2)
 
 
-def test_select_kbest_zero():
+@pytest.mark.parametrize("dtype_in", [np.float32, np.float64])
+def test_select_kbest_zero(dtype_in):
     # Test whether k=0 correctly returns no features.
-    X, y = make_classification(n_samples=20, n_features=10,
-                               shuffle=False, random_state=0)
+    X, y = make_classification(
+        n_samples=20, n_features=10, shuffle=False, random_state=0
+    )
+    X = X.astype(dtype_in)
 
     univariate_filter = SelectKBest(f_classif, k=0)
     univariate_filter.fit(X, y)
     support = univariate_filter.get_support()
     gtruth = np.zeros(10, dtype=bool)
     assert_array_equal(support, gtruth)
-    X_selected = assert_warns_message(UserWarning, 'No features were selected',
-                                      univariate_filter.transform, X)
+    with pytest.warns(UserWarning, match="No features were selected"):
+        X_selected = univariate_filter.transform(X)
     assert X_selected.shape == (20, 0)
+    assert X_selected.dtype == dtype_in
 
 
 def test_select_heuristics_classif():
     # Test whether the relative univariate feature selection
     # gets the correct items in a simple classification problem
     # with the fdr, fwe and fpr heuristics
-    X, y = make_classification(n_samples=200, n_features=20,
-                               n_informative=3, n_redundant=2,
-                               n_repeated=0, n_classes=8,
-                               n_clusters_per_class=1, flip_y=0.0,
-                               class_sep=10, shuffle=False, random_state=0)
+    X, y = make_classification(
+        n_samples=200,
+        n_features=20,
+        n_informative=3,
+        n_redundant=2,
+        n_repeated=0,
+        n_classes=8,
+        n_clusters_per_class=1,
+        flip_y=0.0,
+        class_sep=10,
+        shuffle=False,
+        random_state=0,
+    )
 
     univariate_filter = SelectFwe(f_classif, alpha=0.01)
     X_r = univariate_filter.fit(X, y).transform(X)
     gtruth = np.zeros(20)
     gtruth[:5] = 1
-    for mode in ['fdr', 'fpr', 'fwe']:
-        X_r2 = GenericUnivariateSelect(
-            f_classif, mode=mode, param=0.01).fit(X, y).transform(X)
+    for mode in ["fdr", "fpr", "fwe"]:
+        X_r2 = (
+            GenericUnivariateSelect(f_classif, mode=mode, param=0.01)
+            .fit(X, y)
+            .transform(X)
+        )
         assert_array_equal(X_r, X_r2)
         support = univariate_filter.get_support()
-        assert_array_almost_equal(support, gtruth)
+        assert_allclose(support, gtruth)
 
 
 ##############################################################################
@@ -272,22 +515,25 @@ def test_select_heuristics_classif():
 def assert_best_scores_kept(score_filter):
     scores = score_filter.scores_
     support = score_filter.get_support()
-    assert_array_almost_equal(np.sort(scores[support]),
-                              np.sort(scores)[-support.sum():])
+    assert_allclose(np.sort(scores[support]), np.sort(scores)[-support.sum() :])
 
 
 def test_select_percentile_regression():
     # Test whether the relative univariate feature selection
     # gets the correct items in a simple regression problem
     # with the percentile heuristic
-    X, y = make_regression(n_samples=200, n_features=20,
-                           n_informative=5, shuffle=False, random_state=0)
+    X, y = make_regression(
+        n_samples=200, n_features=20, n_informative=5, shuffle=False, random_state=0
+    )
 
     univariate_filter = SelectPercentile(f_regression, percentile=25)
     X_r = univariate_filter.fit(X, y).transform(X)
     assert_best_scores_kept(univariate_filter)
-    X_r2 = GenericUnivariateSelect(
-        f_regression, mode='percentile', param=25).fit(X, y).transform(X)
+    X_r2 = (
+        GenericUnivariateSelect(f_regression, mode="percentile", param=25)
+        .fit(X, y)
+        .transform(X)
+    )
     assert_array_equal(X_r, X_r2)
     support = univariate_filter.get_support()
     gtruth = np.zeros(20)
@@ -297,53 +543,53 @@ def test_select_percentile_regression():
     X_2[:, np.logical_not(support)] = 0
     assert_array_equal(X_2, univariate_filter.inverse_transform(X_r))
     # Check inverse_transform respects dtype
-    assert_array_equal(X_2.astype(bool),
-                       univariate_filter.inverse_transform(X_r.astype(bool)))
+    assert_array_equal(
+        X_2.astype(bool), univariate_filter.inverse_transform(X_r.astype(bool))
+    )
 
 
 def test_select_percentile_regression_full():
     # Test whether the relative univariate feature selection
     # selects all features when '100%' is asked.
-    X, y = make_regression(n_samples=200, n_features=20,
-                           n_informative=5, shuffle=False, random_state=0)
+    X, y = make_regression(
+        n_samples=200, n_features=20, n_informative=5, shuffle=False, random_state=0
+    )
 
     univariate_filter = SelectPercentile(f_regression, percentile=100)
     X_r = univariate_filter.fit(X, y).transform(X)
     assert_best_scores_kept(univariate_filter)
-    X_r2 = GenericUnivariateSelect(
-        f_regression, mode='percentile', param=100).fit(X, y).transform(X)
+    X_r2 = (
+        GenericUnivariateSelect(f_regression, mode="percentile", param=100)
+        .fit(X, y)
+        .transform(X)
+    )
     assert_array_equal(X_r, X_r2)
     support = univariate_filter.get_support()
     gtruth = np.ones(20)
     assert_array_equal(support, gtruth)
 
 
-def test_invalid_percentile():
-    X, y = make_regression(n_samples=10, n_features=20,
-                           n_informative=2, shuffle=False, random_state=0)
-
-    with pytest.raises(ValueError):
-        SelectPercentile(percentile=-1).fit(X, y)
-    with pytest.raises(ValueError):
-        SelectPercentile(percentile=101).fit(X, y)
-    with pytest.raises(ValueError):
-        GenericUnivariateSelect(mode='percentile', param=-1).fit(X, y)
-    with pytest.raises(ValueError):
-        GenericUnivariateSelect(mode='percentile', param=101).fit(X, y)
-
-
 def test_select_kbest_regression():
     # Test whether the relative univariate feature selection
     # gets the correct items in a simple regression problem
     # with the k best heuristic
-    X, y = make_regression(n_samples=200, n_features=20, n_informative=5,
-                           shuffle=False, random_state=0, noise=10)
+    X, y = make_regression(
+        n_samples=200,
+        n_features=20,
+        n_informative=5,
+        shuffle=False,
+        random_state=0,
+        noise=10,
+    )
 
     univariate_filter = SelectKBest(f_regression, k=5)
     X_r = univariate_filter.fit(X, y).transform(X)
     assert_best_scores_kept(univariate_filter)
-    X_r2 = GenericUnivariateSelect(
-        f_regression, mode='k_best', param=5).fit(X, y).transform(X)
+    X_r2 = (
+        GenericUnivariateSelect(f_regression, mode="k_best", param=5)
+        .fit(X, y)
+        .transform(X)
+    )
     assert_array_equal(X_r, X_r2)
     support = univariate_filter.get_support()
     gtruth = np.zeros(20)
@@ -355,19 +601,28 @@ def test_select_heuristics_regression():
     # Test whether the relative univariate feature selection
     # gets the correct items in a simple regression problem
     # with the fpr, fdr or fwe heuristics
-    X, y = make_regression(n_samples=200, n_features=20, n_informative=5,
-                           shuffle=False, random_state=0, noise=10)
+    X, y = make_regression(
+        n_samples=200,
+        n_features=20,
+        n_informative=5,
+        shuffle=False,
+        random_state=0,
+        noise=10,
+    )
 
     univariate_filter = SelectFpr(f_regression, alpha=0.01)
     X_r = univariate_filter.fit(X, y).transform(X)
     gtruth = np.zeros(20)
     gtruth[:5] = 1
-    for mode in ['fdr', 'fpr', 'fwe']:
-        X_r2 = GenericUnivariateSelect(
-            f_regression, mode=mode, param=0.01).fit(X, y).transform(X)
+    for mode in ["fdr", "fpr", "fwe"]:
+        X_r2 = (
+            GenericUnivariateSelect(f_regression, mode=mode, param=0.01)
+            .fit(X, y)
+            .transform(X)
+        )
         assert_array_equal(X_r, X_r2)
         support = univariate_filter.get_support()
-        assert_array_equal(support[:5], np.ones((5, ), dtype=bool))
+        assert_array_equal(support[:5], np.ones((5,), dtype=bool))
         assert np.sum(support[5:] == 1) < 3
 
 
@@ -376,7 +631,7 @@ def test_boundary_case_ch2():
     X = np.array([[10, 20], [20, 20], [20, 30]])
     y = np.array([[1], [0], [0]])
     scores, pvalues = chi2(X, y)
-    assert_array_almost_equal(scores, np.array([4., 0.71428571]))
+    assert_array_almost_equal(scores, np.array([4.0, 0.71428571]))
     assert_array_almost_equal(pvalues, np.array([0.04550026, 0.39802472]))
 
     filter_fdr = SelectFdr(chi2, alpha=0.1)
@@ -410,17 +665,25 @@ def test_boundary_case_ch2():
 def test_select_fdr_regression(alpha, n_informative):
     # Test that fdr heuristic actually has low FDR.
     def single_fdr(alpha, n_informative, random_state):
-        X, y = make_regression(n_samples=150, n_features=20,
-                               n_informative=n_informative, shuffle=False,
-                               random_state=random_state, noise=10)
+        X, y = make_regression(
+            n_samples=150,
+            n_features=20,
+            n_informative=n_informative,
+            shuffle=False,
+            random_state=random_state,
+            noise=10,
+        )
 
         with warnings.catch_warnings(record=True):
             # Warnings can be raised when no features are selected
             # (low alpha or very noisy data)
             univariate_filter = SelectFdr(f_regression, alpha=alpha)
             X_r = univariate_filter.fit(X, y).transform(X)
-            X_r2 = GenericUnivariateSelect(
-                f_regression, mode='fdr', param=alpha).fit(X, y).transform(X)
+            X_r2 = (
+                GenericUnivariateSelect(f_regression, mode="fdr", param=alpha)
+                .fit(X, y)
+                .transform(X)
+            )
 
         assert_array_equal(X_r, X_r2)
         support = univariate_filter.get_support()
@@ -428,17 +691,18 @@ def single_fdr(alpha, n_informative, random_state):
         num_true_positives = np.sum(support[:n_informative] == 1)
 
         if num_false_positives == 0:
-            return 0.
-        false_discovery_rate = (num_false_positives /
-                                (num_true_positives + num_false_positives))
+            return 0.0
+        false_discovery_rate = num_false_positives / (
+            num_true_positives + num_false_positives
+        )
         return false_discovery_rate
 
     # As per Benjamini-Hochberg, the expected false discovery rate
     # should be lower than alpha:
     # FDR = E(FP / (TP + FP)) <= alpha
-    false_discovery_rate = np.mean([single_fdr(alpha, n_informative,
-                                               random_state) for
-                                    random_state in range(100)])
+    false_discovery_rate = np.mean(
+        [single_fdr(alpha, n_informative, random_state) for random_state in range(100)]
+    )
     assert alpha >= false_discovery_rate
 
     # Make sure that the empirical false discovery rate increases
@@ -451,18 +715,22 @@ def test_select_fwe_regression():
     # Test whether the relative univariate feature selection
     # gets the correct items in a simple regression problem
     # with the fwe heuristic
-    X, y = make_regression(n_samples=200, n_features=20,
-                           n_informative=5, shuffle=False, random_state=0)
+    X, y = make_regression(
+        n_samples=200, n_features=20, n_informative=5, shuffle=False, random_state=0
+    )
 
     univariate_filter = SelectFwe(f_regression, alpha=0.01)
     X_r = univariate_filter.fit(X, y).transform(X)
-    X_r2 = GenericUnivariateSelect(
-        f_regression, mode='fwe', param=0.01).fit(X, y).transform(X)
+    X_r2 = (
+        GenericUnivariateSelect(f_regression, mode="fwe", param=0.01)
+        .fit(X, y)
+        .transform(X)
+    )
     assert_array_equal(X_r, X_r2)
     support = univariate_filter.get_support()
     gtruth = np.zeros(20)
     gtruth[:5] = 1
-    assert_array_equal(support[:5], np.ones((5, ), dtype=bool))
+    assert_array_equal(support[:5], np.ones((5,), dtype=bool))
     assert np.sum(support[5:] == 1) < 2
 
 
@@ -549,37 +817,26 @@ def test_nans():
     # Assert that SelectKBest and SelectPercentile can handle NaNs.
     # First feature has zero variance to confuse f_classif (ANOVA) and
     # make it return a NaN.
-    X = [[0, 1, 0], [0, -1, -1], [0, .5, .5]]
+    X = [[0, 1, 0], [0, -1, -1], [0, 0.5, 0.5]]
     y = [1, 0, 1]
 
-    for select in (SelectKBest(f_classif, k=2),
-                   SelectPercentile(f_classif, percentile=67)):
+    for select in (
+        SelectKBest(f_classif, k=2),
+        SelectPercentile(f_classif, percentile=67),
+    ):
         ignore_warnings(select.fit)(X, y)
         assert_array_equal(select.get_support(indices=True), np.array([1, 2]))
 
 
-def test_score_func_error():
-    X = [[0, 1, 0], [0, -1, -1], [0, .5, .5]]
-    y = [1, 0, 1]
-
-    for SelectFeatures in [SelectKBest, SelectPercentile, SelectFwe,
-                           SelectFdr, SelectFpr, GenericUnivariateSelect]:
-        with pytest.raises(TypeError):
-            SelectFeatures(score_func=10).fit(X, y)
-
-
 def test_invalid_k():
-    X = [[0, 1, 0], [0, -1, -1], [0, .5, .5]]
+    X = [[0, 1, 0], [0, -1, -1], [0, 0.5, 0.5]]
     y = [1, 0, 1]
 
-    with pytest.raises(ValueError):
-        SelectKBest(k=-1).fit(X, y)
-    with pytest.raises(ValueError):
+    msg = "k=4 is greater than n_features=3. All the features will be returned."
+    with pytest.warns(UserWarning, match=msg):
         SelectKBest(k=4).fit(X, y)
-    with pytest.raises(ValueError):
-        GenericUnivariateSelect(mode='k_best', param=-1).fit(X, y)
-    with pytest.raises(ValueError):
-        GenericUnivariateSelect(mode='k_best', param=4).fit(X, y)
+    with pytest.warns(UserWarning, match=msg):
+        GenericUnivariateSelect(mode="k_best", param=4).fit(X, y)
 
 
 def test_f_classif_constant_feature():
@@ -587,7 +844,8 @@ def test_f_classif_constant_feature():
 
     X, y = make_classification(n_samples=10, n_features=5)
     X[:, 0] = 2.0
-    assert_warns(UserWarning, f_classif, X, y)
+    with pytest.warns(UserWarning):
+        f_classif(X, y)
 
 
 def test_no_feature_selected():
@@ -606,23 +864,34 @@ def test_no_feature_selected():
     ]
     for selector in strict_selectors:
         assert_array_equal(selector.get_support(), np.zeros(10))
-        X_selected = assert_warns_message(
-            UserWarning, 'No features were selected', selector.transform, X)
+        with pytest.warns(UserWarning, match="No features were selected"):
+            X_selected = selector.transform(X)
         assert X_selected.shape == (40, 0)
 
 
 def test_mutual_info_classif():
-    X, y = make_classification(n_samples=100, n_features=5,
-                               n_informative=1, n_redundant=1,
-                               n_repeated=0, n_classes=2,
-                               n_clusters_per_class=1, flip_y=0.0,
-                               class_sep=10, shuffle=False, random_state=0)
+    X, y = make_classification(
+        n_samples=100,
+        n_features=5,
+        n_informative=1,
+        n_redundant=1,
+        n_repeated=0,
+        n_classes=2,
+        n_clusters_per_class=1,
+        flip_y=0.0,
+        class_sep=10,
+        shuffle=False,
+        random_state=0,
+    )
 
     # Test in KBest mode.
     univariate_filter = SelectKBest(mutual_info_classif, k=2)
     X_r = univariate_filter.fit(X, y).transform(X)
-    X_r2 = GenericUnivariateSelect(
-        mutual_info_classif, mode='k_best', param=2).fit(X, y).transform(X)
+    X_r2 = (
+        GenericUnivariateSelect(mutual_info_classif, mode="k_best", param=2)
+        .fit(X, y)
+        .transform(X)
+    )
     assert_array_equal(X_r, X_r2)
     support = univariate_filter.get_support()
     gtruth = np.zeros(5)
@@ -632,8 +901,11 @@ def test_mutual_info_classif():
     # Test in Percentile mode.
     univariate_filter = SelectPercentile(mutual_info_classif, percentile=40)
     X_r = univariate_filter.fit(X, y).transform(X)
-    X_r2 = GenericUnivariateSelect(
-        mutual_info_classif, mode='percentile', param=40).fit(X, y).transform(X)
+    X_r2 = (
+        GenericUnivariateSelect(mutual_info_classif, mode="percentile", param=40)
+        .fit(X, y)
+        .transform(X)
+    )
     assert_array_equal(X_r, X_r2)
     support = univariate_filter.get_support()
     gtruth = np.zeros(5)
@@ -642,15 +914,24 @@ def test_mutual_info_classif():
 
 
 def test_mutual_info_regression():
-    X, y = make_regression(n_samples=100, n_features=10, n_informative=2,
-                           shuffle=False, random_state=0, noise=10)
+    X, y = make_regression(
+        n_samples=100,
+        n_features=10,
+        n_informative=2,
+        shuffle=False,
+        random_state=0,
+        noise=10,
+    )
 
     # Test in KBest mode.
     univariate_filter = SelectKBest(mutual_info_regression, k=2)
     X_r = univariate_filter.fit(X, y).transform(X)
     assert_best_scores_kept(univariate_filter)
-    X_r2 = GenericUnivariateSelect(
-        mutual_info_regression, mode='k_best', param=2).fit(X, y).transform(X)
+    X_r2 = (
+        GenericUnivariateSelect(mutual_info_regression, mode="k_best", param=2)
+        .fit(X, y)
+        .transform(X)
+    )
     assert_array_equal(X_r, X_r2)
     support = univariate_filter.get_support()
     gtruth = np.zeros(10)
@@ -660,10 +941,78 @@ def test_mutual_info_regression():
     # Test in Percentile mode.
     univariate_filter = SelectPercentile(mutual_info_regression, percentile=20)
     X_r = univariate_filter.fit(X, y).transform(X)
-    X_r2 = GenericUnivariateSelect(mutual_info_regression, mode='percentile',
-                                   param=20).fit(X, y).transform(X)
+    X_r2 = (
+        GenericUnivariateSelect(mutual_info_regression, mode="percentile", param=20)
+        .fit(X, y)
+        .transform(X)
+    )
     assert_array_equal(X_r, X_r2)
     support = univariate_filter.get_support()
     gtruth = np.zeros(10)
     gtruth[:2] = 1
     assert_array_equal(support, gtruth)
+
+
+def test_dataframe_output_dtypes():
+    """Check that the output datafarme dtypes are the same as the input.
+
+    Non-regression test for gh-24860.
+    """
+    pd = pytest.importorskip("pandas")
+
+    X, y = load_iris(return_X_y=True, as_frame=True)
+    X = X.astype(
+        {
+            "petal length (cm)": np.float32,
+            "petal width (cm)": np.float64,
+        }
+    )
+    X["petal_width_binned"] = pd.cut(X["petal width (cm)"], bins=10)
+
+    column_order = X.columns
+
+    def selector(X, y):
+        ranking = {
+            "sepal length (cm)": 1,
+            "sepal width (cm)": 2,
+            "petal length (cm)": 3,
+            "petal width (cm)": 4,
+            "petal_width_binned": 5,
+        }
+        return np.asarray([ranking[name] for name in column_order])
+
+    univariate_filter = SelectKBest(selector, k=3).set_output(transform="pandas")
+    output = univariate_filter.fit_transform(X, y)
+
+    assert_array_equal(
+        output.columns, ["petal length (cm)", "petal width (cm)", "petal_width_binned"]
+    )
+    for name, dtype in output.dtypes.items():
+        assert dtype == X.dtypes[name]
+
+
+@pytest.mark.parametrize(
+    "selector",
+    [
+        SelectKBest(k=4),
+        SelectPercentile(percentile=80),
+        GenericUnivariateSelect(mode="k_best", param=4),
+        GenericUnivariateSelect(mode="percentile", param=80),
+    ],
+)
+def test_unsupervised_filter(selector):
+    """Check support for unsupervised feature selection for the filter that could
+    require only `X`.
+    """
+    rng = np.random.RandomState(0)
+    X = rng.randn(10, 5)
+
+    def score_func(X, y=None):
+        return np.array([1, 1, 1, 1, 0])
+
+    selector.set_params(score_func=score_func)
+    selector.fit(X)
+    X_trans = selector.transform(X)
+    assert_allclose(X_trans, X[:, :4])
+    X_trans = selector.fit_transform(X)
+    assert_allclose(X_trans, X[:, :4])
diff --git a/sklearn/feature_selection/tests/test_from_model.py b/sklearn/feature_selection/tests/test_from_model.py
index 37b5c105e1daa..17bedf44748fb 100644
--- a/sklearn/feature_selection/tests/test_from_model.py
+++ b/sklearn/feature_selection/tests/test_from_model.py
@@ -1,47 +1,68 @@
-import pytest
-import numpy as np
+import re
+import warnings
+from unittest.mock import Mock
 
-from sklearn.utils._testing import assert_array_almost_equal
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils._testing import assert_allclose
-from sklearn.utils._testing import skip_if_32bit
+import numpy as np
+import pytest
 
 from sklearn import datasets
-from sklearn.linear_model import LogisticRegression, SGDClassifier, Lasso
-from sklearn.svm import LinearSVC
-from sklearn.feature_selection import SelectFromModel
-from sklearn.experimental import enable_hist_gradient_boosting  # noqa
-from sklearn.ensemble import (RandomForestClassifier,
-                              HistGradientBoostingClassifier)
-from sklearn.linear_model import PassiveAggressiveClassifier
 from sklearn.base import BaseEstimator
-from sklearn.pipeline import make_pipeline
+from sklearn.cross_decomposition import CCA, PLSCanonical, PLSRegression
+from sklearn.datasets import make_friedman1, make_regression
 from sklearn.decomposition import PCA
+from sklearn.ensemble import HistGradientBoostingClassifier, RandomForestClassifier
+from sklearn.exceptions import NotFittedError
+from sklearn.feature_selection import SelectFromModel
+from sklearn.linear_model import (
+    ElasticNet,
+    ElasticNetCV,
+    Lasso,
+    LassoCV,
+    LinearRegression,
+    LogisticRegression,
+    PassiveAggressiveClassifier,
+    SGDClassifier,
+)
+from sklearn.pipeline import make_pipeline
+from sklearn.svm import LinearSVC
+from sklearn.utils._testing import (
+    MinimalClassifier,
+    assert_allclose,
+    assert_array_almost_equal,
+    assert_array_equal,
+    skip_if_32bit,
+)
 
 
 class NaNTag(BaseEstimator):
-    def _more_tags(self):
-        return {'allow_nan': True}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.allow_nan = True
+        return tags
 
 
 class NoNaNTag(BaseEstimator):
-    def _more_tags(self):
-        return {'allow_nan': False}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.allow_nan = False
+        return tags
 
 
 class NaNTagRandomForest(RandomForestClassifier):
-    def _more_tags(self):
-        return {'allow_nan': True}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.allow_nan = True
+        return tags
 
 
 iris = datasets.load_iris()
 data, y = iris.data, iris.target
-rng = np.random.RandomState(0)
 
 
 def test_invalid_input():
-    clf = SGDClassifier(alpha=0.1, max_iter=10, shuffle=True,
-                        random_state=None, tol=None)
+    clf = SGDClassifier(
+        alpha=0.1, max_iter=10, shuffle=True, random_state=None, tol=None
+    )
     for threshold in ["gobbledigook", ".5 * gobbledigook"]:
         model = SelectFromModel(clf, threshold=threshold)
         model.fit(data, y)
@@ -59,29 +80,100 @@ def test_input_estimator_unchanged():
 
 @pytest.mark.parametrize(
     "max_features, err_type, err_msg",
-    [(-1, ValueError, "'max_features' should be 0 and"),
-     (data.shape[1] + 1, ValueError, "'max_features' should be 0 and"),
-     ('gobbledigook', TypeError, "should be an integer"),
-     ('all', TypeError, "should be an integer")]
+    [
+        (
+            data.shape[1] + 1,
+            ValueError,
+            "max_features ==",
+        ),
+        (
+            lambda X: 1.5,
+            TypeError,
+            "max_features must be an instance of int, not float.",
+        ),
+        (
+            lambda X: data.shape[1] + 1,
+            ValueError,
+            "max_features ==",
+        ),
+        (
+            lambda X: -1,
+            ValueError,
+            "max_features ==",
+        ),
+    ],
 )
 def test_max_features_error(max_features, err_type, err_msg):
-    clf = RandomForestClassifier(n_estimators=50, random_state=0)
+    err_msg = re.escape(err_msg)
+    clf = RandomForestClassifier(n_estimators=5, random_state=0)
 
-    transformer = SelectFromModel(estimator=clf,
-                                  max_features=max_features,
-                                  threshold=-np.inf)
+    transformer = SelectFromModel(
+        estimator=clf, max_features=max_features, threshold=-np.inf
+    )
     with pytest.raises(err_type, match=err_msg):
         transformer.fit(data, y)
 
 
-@pytest.mark.parametrize("max_features", [0, 2, data.shape[1]])
-def test_max_features_dim(max_features):
-    clf = RandomForestClassifier(n_estimators=50, random_state=0)
-    transformer = SelectFromModel(estimator=clf,
-                                  max_features=max_features,
-                                  threshold=-np.inf)
+@pytest.mark.parametrize("max_features", [0, 2, data.shape[1], None])
+def test_inferred_max_features_integer(max_features):
+    """Check max_features_ and output shape for integer max_features."""
+    clf = RandomForestClassifier(n_estimators=5, random_state=0)
+    transformer = SelectFromModel(
+        estimator=clf, max_features=max_features, threshold=-np.inf
+    )
     X_trans = transformer.fit_transform(data, y)
-    assert X_trans.shape[1] == max_features
+    if max_features is not None:
+        assert transformer.max_features_ == max_features
+        assert X_trans.shape[1] == transformer.max_features_
+    else:
+        assert not hasattr(transformer, "max_features_")
+        assert X_trans.shape[1] == data.shape[1]
+
+
+@pytest.mark.parametrize(
+    "max_features",
+    [lambda X: 1, lambda X: X.shape[1], lambda X: min(X.shape[1], 10000)],
+)
+def test_inferred_max_features_callable(max_features):
+    """Check max_features_ and output shape for callable max_features."""
+    clf = RandomForestClassifier(n_estimators=5, random_state=0)
+    transformer = SelectFromModel(
+        estimator=clf, max_features=max_features, threshold=-np.inf
+    )
+    X_trans = transformer.fit_transform(data, y)
+    assert transformer.max_features_ == max_features(data)
+    assert X_trans.shape[1] == transformer.max_features_
+
+
+@pytest.mark.parametrize("max_features", [lambda X: round(len(X[0]) / 2), 2])
+def test_max_features_array_like(max_features):
+    X = [
+        [0.87, -1.34, 0.31],
+        [-2.79, -0.02, -0.85],
+        [-1.34, -0.48, -2.55],
+        [1.92, 1.48, 0.65],
+    ]
+    y = [0, 1, 0, 1]
+
+    clf = RandomForestClassifier(n_estimators=5, random_state=0)
+    transformer = SelectFromModel(
+        estimator=clf, max_features=max_features, threshold=-np.inf
+    )
+    X_trans = transformer.fit_transform(X, y)
+    assert X_trans.shape[1] == transformer.max_features_
+
+
+@pytest.mark.parametrize(
+    "max_features",
+    [lambda X: min(X.shape[1], 10000), lambda X: X.shape[1], lambda X: 1],
+)
+def test_max_features_callable_data(max_features):
+    """Tests that the callable passed to `fit` is called on X."""
+    clf = RandomForestClassifier(n_estimators=50, random_state=0)
+    m = Mock(side_effect=max_features)
+    transformer = SelectFromModel(estimator=clf, max_features=m, threshold=-np.inf)
+    transformer.fit_transform(data, y)
+    m.assert_called_with(data)
 
 
 class FixedImportanceEstimator(BaseEstimator):
@@ -95,46 +187,57 @@ def fit(self, X, y=None):
 def test_max_features():
     # Test max_features parameter using various values
     X, y = datasets.make_classification(
-        n_samples=1000, n_features=10, n_informative=3, n_redundant=0,
-        n_repeated=0, shuffle=False, random_state=0)
+        n_samples=1000,
+        n_features=10,
+        n_informative=3,
+        n_redundant=0,
+        n_repeated=0,
+        shuffle=False,
+        random_state=0,
+    )
     max_features = X.shape[1]
     est = RandomForestClassifier(n_estimators=50, random_state=0)
 
-    transformer1 = SelectFromModel(estimator=est,
-                                   threshold=-np.inf)
-    transformer2 = SelectFromModel(estimator=est,
-                                   max_features=max_features,
-                                   threshold=-np.inf)
+    transformer1 = SelectFromModel(estimator=est, threshold=-np.inf)
+    transformer2 = SelectFromModel(
+        estimator=est, max_features=max_features, threshold=-np.inf
+    )
     X_new1 = transformer1.fit_transform(X, y)
     X_new2 = transformer2.fit_transform(X, y)
     assert_allclose(X_new1, X_new2)
 
     # Test max_features against actual model.
-    transformer1 = SelectFromModel(estimator=Lasso(alpha=0.025,
-                                                   random_state=42))
+    transformer1 = SelectFromModel(estimator=Lasso(alpha=0.025, random_state=42))
     X_new1 = transformer1.fit_transform(X, y)
     scores1 = np.abs(transformer1.estimator_.coef_)
-    candidate_indices1 = np.argsort(-scores1, kind='mergesort')
+    candidate_indices1 = np.argsort(-scores1, kind="mergesort")
 
     for n_features in range(1, X_new1.shape[1] + 1):
-        transformer2 = SelectFromModel(estimator=Lasso(alpha=0.025,
-                                                       random_state=42),
-                                       max_features=n_features,
-                                       threshold=-np.inf)
+        transformer2 = SelectFromModel(
+            estimator=Lasso(alpha=0.025, random_state=42),
+            max_features=n_features,
+            threshold=-np.inf,
+        )
         X_new2 = transformer2.fit_transform(X, y)
         scores2 = np.abs(transformer2.estimator_.coef_)
-        candidate_indices2 = np.argsort(-scores2, kind='mergesort')
-        assert_allclose(X[:, candidate_indices1[:n_features]],
-                        X[:, candidate_indices2[:n_features]])
-    assert_allclose(transformer1.estimator_.coef_,
-                    transformer2.estimator_.coef_)
+        candidate_indices2 = np.argsort(-scores2, kind="mergesort")
+        assert_allclose(
+            X[:, candidate_indices1[:n_features]], X[:, candidate_indices2[:n_features]]
+        )
+    assert_allclose(transformer1.estimator_.coef_, transformer2.estimator_.coef_)
 
 
 def test_max_features_tiebreak():
     # Test if max_features can break tie among feature importance
     X, y = datasets.make_classification(
-        n_samples=1000, n_features=10, n_informative=3, n_redundant=0,
-        n_repeated=0, shuffle=False, random_state=0)
+        n_samples=1000,
+        n_features=10,
+        n_informative=3,
+        n_redundant=0,
+        n_repeated=0,
+        shuffle=False,
+        random_state=0,
+    )
     max_features = X.shape[1]
 
     feature_importances = np.array([4, 4, 4, 4, 3, 3, 3, 2, 2, 1])
@@ -142,7 +245,8 @@ def test_max_features_tiebreak():
         transformer = SelectFromModel(
             FixedImportanceEstimator(feature_importances),
             max_features=n_features,
-            threshold=-np.inf)
+            threshold=-np.inf,
+        )
         X_new = transformer.fit_transform(X, y)
         selected_feature_indices = np.where(transformer._get_support_mask())[0]
         assert_array_equal(selected_feature_indices, np.arange(n_features))
@@ -151,37 +255,46 @@ def test_max_features_tiebreak():
 
 def test_threshold_and_max_features():
     X, y = datasets.make_classification(
-        n_samples=1000, n_features=10, n_informative=3, n_redundant=0,
-        n_repeated=0, shuffle=False, random_state=0)
+        n_samples=1000,
+        n_features=10,
+        n_informative=3,
+        n_redundant=0,
+        n_repeated=0,
+        shuffle=False,
+        random_state=0,
+    )
     est = RandomForestClassifier(n_estimators=50, random_state=0)
 
-    transformer1 = SelectFromModel(estimator=est, max_features=3,
-                                   threshold=-np.inf)
+    transformer1 = SelectFromModel(estimator=est, max_features=3, threshold=-np.inf)
     X_new1 = transformer1.fit_transform(X, y)
 
     transformer2 = SelectFromModel(estimator=est, threshold=0.04)
     X_new2 = transformer2.fit_transform(X, y)
 
-    transformer3 = SelectFromModel(estimator=est, max_features=3,
-                                   threshold=0.04)
+    transformer3 = SelectFromModel(estimator=est, max_features=3, threshold=0.04)
     X_new3 = transformer3.fit_transform(X, y)
     assert X_new3.shape[1] == min(X_new1.shape[1], X_new2.shape[1])
-    selected_indices = transformer3.transform(
-        np.arange(X.shape[1])[np.newaxis, :])
+    selected_indices = transformer3.transform(np.arange(X.shape[1])[np.newaxis, :])
     assert_allclose(X_new3, X[:, selected_indices[0]])
 
 
 @skip_if_32bit
 def test_feature_importances():
     X, y = datasets.make_classification(
-        n_samples=1000, n_features=10, n_informative=3, n_redundant=0,
-        n_repeated=0, shuffle=False, random_state=0)
+        n_samples=1000,
+        n_features=10,
+        n_informative=3,
+        n_redundant=0,
+        n_repeated=0,
+        shuffle=False,
+        random_state=0,
+    )
 
     est = RandomForestClassifier(n_estimators=50, random_state=0)
     for threshold, func in zip(["mean", "median"], [np.mean, np.median]):
         transformer = SelectFromModel(estimator=est, threshold=threshold)
         transformer.fit(X, y)
-        assert hasattr(transformer.estimator_, 'feature_importances_')
+        assert hasattr(transformer.estimator_, "feature_importances_")
 
         X_new = transformer.transform(X)
         assert X_new.shape[1] < X.shape[1]
@@ -194,8 +307,14 @@ def test_feature_importances():
 def test_sample_weight():
     # Ensure sample weights are passed to underlying estimator
     X, y = datasets.make_classification(
-        n_samples=100, n_features=10, n_informative=3, n_redundant=0,
-        n_repeated=0, shuffle=False, random_state=0)
+        n_samples=100,
+        n_features=10,
+        n_informative=3,
+        n_redundant=0,
+        n_repeated=0,
+        shuffle=False,
+        random_state=0,
+    )
 
     # Check with sample weights
     sample_weight = np.ones(y.shape)
@@ -213,14 +332,28 @@ def test_sample_weight():
     assert np.all(weighted_mask == reweighted_mask)
 
 
-def test_coef_default_threshold():
+@pytest.mark.parametrize(
+    "estimator",
+    [
+        Lasso(alpha=0.1, random_state=42),
+        LassoCV(random_state=42),
+        ElasticNet(l1_ratio=1, random_state=42),
+        ElasticNetCV(l1_ratio=[1], random_state=42),
+    ],
+)
+def test_coef_default_threshold(estimator):
     X, y = datasets.make_classification(
-        n_samples=100, n_features=10, n_informative=3, n_redundant=0,
-        n_repeated=0, shuffle=False, random_state=0)
+        n_samples=100,
+        n_features=10,
+        n_informative=3,
+        n_redundant=0,
+        n_repeated=0,
+        shuffle=False,
+        random_state=0,
+    )
 
     # For the Lasso and related models, the threshold defaults to 1e-5
-    transformer = SelectFromModel(estimator=Lasso(alpha=0.1,
-                                                  random_state=42))
+    transformer = SelectFromModel(estimator=estimator)
     transformer.fit(X, y)
     X_new = transformer.transform(X)
     mask = np.abs(transformer.estimator_.coef_) > 1e-5
@@ -230,18 +363,25 @@ def test_coef_default_threshold():
 @skip_if_32bit
 def test_2d_coef():
     X, y = datasets.make_classification(
-        n_samples=1000, n_features=10, n_informative=3, n_redundant=0,
-        n_repeated=0, shuffle=False, random_state=0, n_classes=4)
+        n_samples=1000,
+        n_features=10,
+        n_informative=3,
+        n_redundant=0,
+        n_repeated=0,
+        shuffle=False,
+        random_state=0,
+        n_classes=4,
+    )
 
     est = LogisticRegression()
     for threshold, func in zip(["mean", "median"], [np.mean, np.median]):
         for order in [1, 2, np.inf]:
             # Fit SelectFromModel a multi-class problem
-            transformer = SelectFromModel(estimator=LogisticRegression(),
-                                          threshold=threshold,
-                                          norm_order=order)
+            transformer = SelectFromModel(
+                estimator=LogisticRegression(), threshold=threshold, norm_order=order
+            )
             transformer.fit(X, y)
-            assert hasattr(transformer.estimator_, 'coef_')
+            assert hasattr(transformer.estimator_, "coef_")
             X_new = transformer.transform(X)
             assert X_new.shape[1] < X.shape[1]
 
@@ -253,14 +393,13 @@ def test_2d_coef():
 
 
 def test_partial_fit():
-    est = PassiveAggressiveClassifier(random_state=0, shuffle=False,
-                                      max_iter=5, tol=None)
+    est = PassiveAggressiveClassifier(
+        random_state=0, shuffle=False, max_iter=5, tol=None
+    )
     transformer = SelectFromModel(estimator=est)
-    transformer.partial_fit(data, y,
-                            classes=np.unique(y))
+    transformer.partial_fit(data, y, classes=np.unique(y))
     old_model = transformer.estimator_
-    transformer.partial_fit(data, y,
-                            classes=np.unique(y))
+    transformer.partial_fit(data, y, classes=np.unique(y))
     new_model = transformer.estimator_
     assert old_model is new_model
 
@@ -287,14 +426,15 @@ def test_prefit():
 
     # Passing a prefit parameter with the selected model
     # and fitting a unfit model with prefit=False should give same results.
-    clf = SGDClassifier(alpha=0.1, max_iter=10, shuffle=True,
-                        random_state=0, tol=None)
+    clf = SGDClassifier(alpha=0.1, max_iter=10, shuffle=True, random_state=0, tol=None)
     model = SelectFromModel(clf)
     model.fit(data, y)
     X_transform = model.transform(data)
     clf.fit(data, y)
     model = SelectFromModel(clf, prefit=True)
     assert_array_almost_equal(model.transform(data), X_transform)
+    model.fit(data, y)
+    assert model.estimator_ is not clf
 
     # Check that the model is rewritten if prefit=False and a fitted model is
     # passed
@@ -302,10 +442,85 @@ def test_prefit():
     model.fit(data, y)
     assert_array_almost_equal(model.transform(data), X_transform)
 
-    # Check that prefit=True and calling fit raises a ValueError
+    # Check that passing an unfitted estimator with `prefit=True` raises a
+    # `ValueError`
+    clf = SGDClassifier(alpha=0.1, max_iter=10, shuffle=True, random_state=0, tol=None)
     model = SelectFromModel(clf, prefit=True)
-    with pytest.raises(ValueError):
+    err_msg = "When `prefit=True`, `estimator` is expected to be a fitted estimator."
+    with pytest.raises(NotFittedError, match=err_msg):
         model.fit(data, y)
+    with pytest.raises(NotFittedError, match=err_msg):
+        model.partial_fit(data, y)
+    with pytest.raises(NotFittedError, match=err_msg):
+        model.transform(data)
+
+    # Check that the internal parameters of prefitted model are not changed
+    # when calling `fit` or `partial_fit` with `prefit=True`
+    clf = SGDClassifier(alpha=0.1, max_iter=10, shuffle=True, tol=None).fit(data, y)
+    model = SelectFromModel(clf, prefit=True)
+    model.fit(data, y)
+    assert_allclose(model.estimator_.coef_, clf.coef_)
+    model.partial_fit(data, y)
+    assert_allclose(model.estimator_.coef_, clf.coef_)
+
+
+def test_prefit_max_features():
+    """Check the interaction between `prefit` and `max_features`."""
+    # case 1: an error should be raised at `transform` if `fit` was not called to
+    # validate the attributes
+    estimator = RandomForestClassifier(n_estimators=5, random_state=0)
+    estimator.fit(data, y)
+    model = SelectFromModel(estimator, prefit=True, max_features=lambda X: X.shape[1])
+
+    err_msg = (
+        "When `prefit=True` and `max_features` is a callable, call `fit` "
+        "before calling `transform`."
+    )
+    with pytest.raises(NotFittedError, match=err_msg):
+        model.transform(data)
+
+    # case 2: `max_features` is not validated and different from an integer
+    # FIXME: we cannot validate the upper bound of the attribute at transform
+    # and we should force calling `fit` if we intend to force the attribute
+    # to have such an upper bound.
+    max_features = 2.5
+    model.set_params(max_features=max_features)
+    with pytest.raises(ValueError, match="`max_features` must be an integer"):
+        model.transform(data)
+
+
+def test_get_feature_names_out_elasticnetcv():
+    """Check if ElasticNetCV works with a list of floats.
+
+    Non-regression test for #30936."""
+    X, y = make_regression(n_features=5, n_informative=3, random_state=0)
+    estimator = ElasticNetCV(l1_ratio=[0.25, 0.5, 0.75], random_state=0)
+    selector = SelectFromModel(estimator=estimator)
+    selector.fit(X, y)
+
+    names_out = selector.get_feature_names_out()
+    mask = selector.get_support()
+    expected = np.array([f"x{i}" for i in range(X.shape[1])])[mask]
+    assert_array_equal(names_out, expected)
+
+
+def test_prefit_get_feature_names_out():
+    """Check the interaction between prefit and the feature names."""
+    clf = RandomForestClassifier(n_estimators=2, random_state=0)
+    clf.fit(data, y)
+    model = SelectFromModel(clf, prefit=True, max_features=1)
+
+    name = type(model).__name__
+    err_msg = (
+        f"This {name} instance is not fitted yet. Call 'fit' with "
+        "appropriate arguments before using this estimator."
+    )
+    with pytest.raises(NotFittedError, match=err_msg):
+        model.get_feature_names_out()
+
+    model.fit(data, y)
+    feature_names = model.get_feature_names_out()
+    assert feature_names == ["x3"]
 
 
 def test_threshold_string():
@@ -323,8 +538,7 @@ def test_threshold_string():
 
 def test_threshold_without_refitting():
     # Test that the threshold can be set without refitting the model.
-    clf = SGDClassifier(alpha=0.1, max_iter=10, shuffle=True,
-                        random_state=0, tol=None)
+    clf = SGDClassifier(alpha=0.1, max_iter=10, shuffle=True, random_state=0, tol=None)
     model = SelectFromModel(clf, threshold="0.1 * mean")
     model.fit(data, y)
     X_transform = model.transform(data)
@@ -341,8 +555,8 @@ def test_fit_accepts_nan_inf():
     model = SelectFromModel(estimator=clf)
 
     nan_data = data.copy()
-    nan_data[0] = np.NaN
-    nan_data[1] = np.Inf
+    nan_data[0] = np.nan
+    nan_data[1] = np.inf
 
     model.fit(data, y)
 
@@ -355,8 +569,8 @@ def test_transform_accepts_nan_inf():
     model = SelectFromModel(estimator=clf)
     model.fit(nan_data, y)
 
-    nan_data[0] = np.NaN
-    nan_data[1] = np.Inf
+    nan_data[0] = np.nan
+    nan_data[1] = np.inf
 
     model.transform(nan_data)
 
@@ -364,11 +578,11 @@ def test_transform_accepts_nan_inf():
 def test_allow_nan_tag_comes_from_estimator():
     allow_nan_est = NaNTag()
     model = SelectFromModel(estimator=allow_nan_est)
-    assert model._get_tags()['allow_nan'] is True
+    assert model.__sklearn_tags__().input_tags.allow_nan is True
 
     no_nan_est = NoNaNTag()
     model = SelectFromModel(estimator=no_nan_est)
-    assert model._get_tags()['allow_nan'] is False
+    assert model.__sklearn_tags__().input_tags.allow_nan is False
 
 
 def _pca_importances(pca_estimator):
@@ -377,9 +591,13 @@ def _pca_importances(pca_estimator):
 
 @pytest.mark.parametrize(
     "estimator, importance_getter",
-    [(make_pipeline(PCA(random_state=0), LogisticRegression()),
-      'named_steps.logisticregression.coef_'),
-     (PCA(random_state=0), _pca_importances)]
+    [
+        (
+            make_pipeline(PCA(random_state=0), LogisticRegression()),
+            "named_steps.logisticregression.coef_",
+        ),
+        (PCA(random_state=0), _pca_importances),
+    ],
 )
 def test_importance_getter(estimator, importance_getter):
     selector = SelectFromModel(
@@ -387,3 +605,100 @@ def test_importance_getter(estimator, importance_getter):
     )
     selector.fit(data, y)
     assert selector.transform(data).shape[1] == 1
+
+
+@pytest.mark.parametrize("PLSEstimator", [CCA, PLSCanonical, PLSRegression])
+def test_select_from_model_pls(PLSEstimator):
+    """Check the behaviour of SelectFromModel with PLS estimators.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/12410
+    """
+    X, y = make_friedman1(n_samples=50, n_features=10, random_state=0)
+    estimator = PLSEstimator(n_components=1)
+    model = make_pipeline(SelectFromModel(estimator), estimator).fit(X, y)
+    assert model.score(X, y) > 0.5
+
+
+def test_estimator_does_not_support_feature_names():
+    """SelectFromModel works with estimators that do not support feature_names_in_.
+
+    Non-regression test for #21949.
+    """
+    pytest.importorskip("pandas")
+    X, y = datasets.load_iris(as_frame=True, return_X_y=True)
+    all_feature_names = set(X.columns)
+
+    def importance_getter(estimator):
+        return np.arange(X.shape[1])
+
+    selector = SelectFromModel(
+        MinimalClassifier(), importance_getter=importance_getter
+    ).fit(X, y)
+
+    # selector learns the feature names itself
+    assert_array_equal(selector.feature_names_in_, X.columns)
+
+    feature_names_out = set(selector.get_feature_names_out())
+    assert feature_names_out < all_feature_names
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", UserWarning)
+
+        selector.transform(X.iloc[1:3])
+
+
+@pytest.mark.parametrize(
+    "error, err_msg, max_features",
+    (
+        [ValueError, "max_features == 10, must be <= 4", 10],
+        [ValueError, "max_features == 5, must be <= 4", lambda x: x.shape[1] + 1],
+    ),
+)
+def test_partial_fit_validate_max_features(error, err_msg, max_features):
+    """Test that partial_fit from SelectFromModel validates `max_features`."""
+    X, y = datasets.make_classification(
+        n_samples=100,
+        n_features=4,
+        random_state=0,
+    )
+
+    with pytest.raises(error, match=err_msg):
+        SelectFromModel(
+            estimator=SGDClassifier(), max_features=max_features
+        ).partial_fit(X, y, classes=[0, 1])
+
+
+@pytest.mark.parametrize("as_frame", [True, False])
+def test_partial_fit_validate_feature_names(as_frame):
+    """Test that partial_fit from SelectFromModel validates `feature_names_in_`."""
+    pytest.importorskip("pandas")
+    X, y = datasets.load_iris(as_frame=as_frame, return_X_y=True)
+
+    selector = SelectFromModel(estimator=SGDClassifier(), max_features=4).partial_fit(
+        X, y, classes=[0, 1, 2]
+    )
+    if as_frame:
+        assert_array_equal(selector.feature_names_in_, X.columns)
+    else:
+        assert not hasattr(selector, "feature_names_in_")
+
+
+def test_from_model_estimator_attribute_error():
+    """Check that we raise the proper AttributeError when the estimator
+    does not implement the `partial_fit` method, which is decorated with
+    `available_if`.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/28108
+    """
+    # `LinearRegression` does not implement 'partial_fit' and should raise an
+    # AttributeError
+    from_model = SelectFromModel(estimator=LinearRegression())
+
+    outer_msg = "This 'SelectFromModel' has no attribute 'partial_fit'"
+    inner_msg = "'LinearRegression' object has no attribute 'partial_fit'"
+    with pytest.raises(AttributeError, match=outer_msg) as exec_info:
+        from_model.fit(data, y).partial_fit(data)
+    assert isinstance(exec_info.value.__cause__, AttributeError)
+    assert inner_msg in str(exec_info.value.__cause__)
diff --git a/sklearn/feature_selection/tests/test_mutual_info.py b/sklearn/feature_selection/tests/test_mutual_info.py
index ca2459f365ba4..4922b7e4e57b3 100644
--- a/sklearn/feature_selection/tests/test_mutual_info.py
+++ b/sklearn/feature_selection/tests/test_mutual_info.py
@@ -1,13 +1,15 @@
-
 import numpy as np
 import pytest
-from scipy.sparse import csr_matrix
 
-from sklearn.utils import check_random_state
-from sklearn.utils._testing import assert_array_equal, assert_almost_equal
+from sklearn.datasets import make_classification, make_regression
+from sklearn.feature_selection import mutual_info_classif, mutual_info_regression
 from sklearn.feature_selection._mutual_info import _compute_mi
-from sklearn.feature_selection import (mutual_info_regression,
-                                       mutual_info_classif)
+from sklearn.utils import check_random_state
+from sklearn.utils._testing import (
+    assert_allclose,
+    assert_array_equal,
+)
+from sklearn.utils.fixes import CSR_CONTAINERS
 
 
 def test_compute_mi_dd():
@@ -16,14 +18,14 @@ def test_compute_mi_dd():
     x = np.array([0, 1, 1, 0, 0])
     y = np.array([1, 0, 0, 0, 1])
 
-    H_x = H_y = -(3/5) * np.log(3/5) - (2/5) * np.log(2/5)
-    H_xy = -1/5 * np.log(1/5) - 2/5 * np.log(2/5) - 2/5 * np.log(2/5)
+    H_x = H_y = -(3 / 5) * np.log(3 / 5) - (2 / 5) * np.log(2 / 5)
+    H_xy = -1 / 5 * np.log(1 / 5) - 2 / 5 * np.log(2 / 5) - 2 / 5 * np.log(2 / 5)
     I_xy = H_x + H_y - H_xy
 
-    assert_almost_equal(_compute_mi(x, y, True, True), I_xy)
+    assert_allclose(_compute_mi(x, y, x_discrete=True, y_discrete=True), I_xy)
 
 
-def test_compute_mi_cc():
+def test_compute_mi_cc(global_dtype):
     # For two continuous variables a good approach is to test on bivariate
     # normal distribution, where mutual information is known.
 
@@ -34,28 +36,31 @@ def test_compute_mi_cc():
     sigma_1 = 1
     sigma_2 = 10
     corr = 0.5
-    cov = np.array([
-        [sigma_1**2, corr * sigma_1 * sigma_2],
-        [corr * sigma_1 * sigma_2, sigma_2**2]
-    ])
+    cov = np.array(
+        [
+            [sigma_1**2, corr * sigma_1 * sigma_2],
+            [corr * sigma_1 * sigma_2, sigma_2**2],
+        ]
+    )
 
     # True theoretical mutual information.
-    I_theory = (np.log(sigma_1) + np.log(sigma_2) -
-                0.5 * np.log(np.linalg.det(cov)))
+    I_theory = np.log(sigma_1) + np.log(sigma_2) - 0.5 * np.log(np.linalg.det(cov))
 
     rng = check_random_state(0)
-    Z = rng.multivariate_normal(mean, cov, size=1000)
+    Z = rng.multivariate_normal(mean, cov, size=1000).astype(global_dtype, copy=False)
 
     x, y = Z[:, 0], Z[:, 1]
 
-    # Theory and computed values won't be very close, assert that the
-    # first figures after decimal point match.
+    # Theory and computed values won't be very close
+    # We here check with a large relative tolerance
     for n_neighbors in [3, 5, 7]:
-        I_computed = _compute_mi(x, y, False, False, n_neighbors)
-        assert_almost_equal(I_computed, I_theory, 1)
+        I_computed = _compute_mi(
+            x, y, x_discrete=False, y_discrete=False, n_neighbors=n_neighbors
+        )
+        assert_allclose(I_computed, I_theory, rtol=1e-1)
 
 
-def test_compute_mi_cd():
+def test_compute_mi_cd(global_dtype):
     # To test define a joint distribution as follows:
     # p(x, y) = p(x) p(y | x)
     # X ~ Bernoulli(p)
@@ -77,46 +82,47 @@ def test_compute_mi_cd():
     for p in [0.3, 0.5, 0.7]:
         x = rng.uniform(size=n_samples) > p
 
-        y = np.empty(n_samples)
+        y = np.empty(n_samples, global_dtype)
         mask = x == 0
         y[mask] = rng.uniform(-1, 1, size=np.sum(mask))
         y[~mask] = rng.uniform(0, 2, size=np.sum(~mask))
 
-        I_theory = -0.5 * ((1 - p) * np.log(0.5 * (1 - p)) +
-                           p * np.log(0.5 * p) + np.log(0.5)) - np.log(2)
+        I_theory = -0.5 * (
+            (1 - p) * np.log(0.5 * (1 - p)) + p * np.log(0.5 * p) + np.log(0.5)
+        ) - np.log(2)
 
         # Assert the same tolerance.
         for n_neighbors in [3, 5, 7]:
-            I_computed = _compute_mi(x, y, True, False, n_neighbors)
-            assert_almost_equal(I_computed, I_theory, 1)
+            I_computed = _compute_mi(
+                x, y, x_discrete=True, y_discrete=False, n_neighbors=n_neighbors
+            )
+            assert_allclose(I_computed, I_theory, rtol=1e-1)
 
 
-def test_compute_mi_cd_unique_label():
+def test_compute_mi_cd_unique_label(global_dtype):
     # Test that adding unique label doesn't change MI.
     n_samples = 100
     x = np.random.uniform(size=n_samples) > 0.5
 
-    y = np.empty(n_samples)
+    y = np.empty(n_samples, global_dtype)
     mask = x == 0
     y[mask] = np.random.uniform(-1, 1, size=np.sum(mask))
     y[~mask] = np.random.uniform(0, 2, size=np.sum(~mask))
 
-    mi_1 = _compute_mi(x, y, True, False)
+    mi_1 = _compute_mi(x, y, x_discrete=True, y_discrete=False)
 
     x = np.hstack((x, 2))
     y = np.hstack((y, 10))
-    mi_2 = _compute_mi(x, y, True, False)
+    mi_2 = _compute_mi(x, y, x_discrete=True, y_discrete=False)
 
-    assert mi_1 == mi_2
+    assert_allclose(mi_1, mi_2)
 
 
 # We are going test that feature ordering by MI matches our expectations.
-def test_mutual_info_classif_discrete():
-    X = np.array([[0, 0, 0],
-                  [1, 1, 0],
-                  [2, 0, 1],
-                  [2, 0, 1],
-                  [2, 0, 1]])
+def test_mutual_info_classif_discrete(global_dtype):
+    X = np.array(
+        [[0, 0, 0], [1, 1, 0], [2, 0, 1], [2, 0, 1], [2, 0, 1]], dtype=global_dtype
+    )
     y = np.array([0, 1, 2, 2, 1])
 
     # Here X[:, 0] is the most informative feature, and X[:, 1] is weakly
@@ -125,45 +131,43 @@ def test_mutual_info_classif_discrete():
     assert_array_equal(np.argsort(-mi), np.array([0, 2, 1]))
 
 
-def test_mutual_info_regression():
+def test_mutual_info_regression(global_dtype):
     # We generate sample from multivariate normal distribution, using
     # transformation from initially uncorrelated variables. The zero
     # variables after transformation is selected as the target vector,
     # it has the strongest correlation with the variable 2, and
     # the weakest correlation with the variable 1.
-    T = np.array([
-        [1, 0.5, 2, 1],
-        [0, 1, 0.1, 0.0],
-        [0, 0.1, 1, 0.1],
-        [0, 0.1, 0.1, 1]
-    ])
+    T = np.array([[1, 0.5, 2, 1], [0, 1, 0.1, 0.0], [0, 0.1, 1, 0.1], [0, 0.1, 0.1, 1]])
     cov = T.dot(T.T)
     mean = np.zeros(4)
 
     rng = check_random_state(0)
-    Z = rng.multivariate_normal(mean, cov, size=1000)
+    Z = rng.multivariate_normal(mean, cov, size=1000).astype(global_dtype, copy=False)
     X = Z[:, 1:]
     y = Z[:, 0]
 
     mi = mutual_info_regression(X, y, random_state=0)
     assert_array_equal(np.argsort(-mi), np.array([1, 2, 0]))
+    # XXX: should mutual_info_regression be fixed to avoid
+    # up-casting float32 inputs to float64?
+    assert mi.dtype == np.float64
 
 
-def test_mutual_info_classif_mixed():
+def test_mutual_info_classif_mixed(global_dtype):
     # Here the target is discrete and there are two continuous and one
     # discrete feature. The idea of this test is clear from the code.
     rng = check_random_state(0)
-    X = rng.rand(1000, 3)
+    X = rng.rand(1000, 3).astype(global_dtype, copy=False)
     X[:, 1] += X[:, 0]
     y = ((0.5 * X[:, 0] + X[:, 2]) > 0.5).astype(int)
     X[:, 2] = X[:, 2] > 0.5
 
-    mi = mutual_info_classif(X, y, discrete_features=[2], n_neighbors=3,
-                             random_state=0)
+    mi = mutual_info_classif(X, y, discrete_features=[2], n_neighbors=3, random_state=0)
     assert_array_equal(np.argsort(-mi), [2, 0, 1])
     for n_neighbors in [5, 7, 9]:
-        mi_nn = mutual_info_classif(X, y, discrete_features=[2],
-                                    n_neighbors=n_neighbors, random_state=0)
+        mi_nn = mutual_info_classif(
+            X, y, discrete_features=[2], n_neighbors=n_neighbors, random_state=0
+        )
         # Check that the continuous values have an higher MI with greater
         # n_neighbors
         assert mi_nn[0] > mi[0]
@@ -173,20 +177,19 @@ def test_mutual_info_classif_mixed():
         assert mi_nn[2] == mi[2]
 
 
-def test_mutual_info_options():
-    X = np.array([[0, 0, 0],
-                  [1, 1, 0],
-                  [2, 0, 1],
-                  [2, 0, 1],
-                  [2, 0, 1]], dtype=float)
-    y = np.array([0, 1, 2, 2, 1], dtype=float)
-    X_csr = csr_matrix(X)
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_mutual_info_options(global_dtype, csr_container):
+    X = np.array(
+        [[0, 0, 0], [1, 1, 0], [2, 0, 1], [2, 0, 1], [2, 0, 1]], dtype=global_dtype
+    )
+    y = np.array([0, 1, 2, 2, 1], dtype=global_dtype)
+    X_csr = csr_container(X)
 
     for mutual_info in (mutual_info_regression, mutual_info_classif):
         with pytest.raises(ValueError):
             mutual_info(X_csr, y, discrete_features=False)
         with pytest.raises(ValueError):
-            mutual_info(X, y, discrete_features='manual')
+            mutual_info(X, y, discrete_features="manual")
         with pytest.raises(ValueError):
             mutual_info(X_csr, y, discrete_features=[True, False, True])
         with pytest.raises(IndexError):
@@ -194,16 +197,74 @@ def test_mutual_info_options():
         with pytest.raises(IndexError):
             mutual_info(X, y, discrete_features=[1, 4])
 
-        mi_1 = mutual_info(X, y, discrete_features='auto', random_state=0)
+        mi_1 = mutual_info(X, y, discrete_features="auto", random_state=0)
         mi_2 = mutual_info(X, y, discrete_features=False, random_state=0)
-        mi_3 = mutual_info(X_csr, y, discrete_features='auto', random_state=0)
+        mi_3 = mutual_info(X_csr, y, discrete_features="auto", random_state=0)
         mi_4 = mutual_info(X_csr, y, discrete_features=True, random_state=0)
-        mi_5 = mutual_info(X, y, discrete_features=[True, False, True],
-                           random_state=0)
+        mi_5 = mutual_info(X, y, discrete_features=[True, False, True], random_state=0)
         mi_6 = mutual_info(X, y, discrete_features=[0, 2], random_state=0)
 
-        assert_array_equal(mi_1, mi_2)
-        assert_array_equal(mi_3, mi_4)
-        assert_array_equal(mi_5, mi_6)
-
-    assert not np.allclose(mi_1, mi_3)
+        assert_allclose(mi_1, mi_2)
+        assert_allclose(mi_3, mi_4)
+        assert_allclose(mi_5, mi_6)
+
+        assert not np.allclose(mi_1, mi_3)
+
+
+@pytest.mark.parametrize("correlated", [True, False])
+def test_mutual_information_symmetry_classif_regression(correlated, global_random_seed):
+    """Check that `mutual_info_classif` and `mutual_info_regression` are
+    symmetric by switching the target `y` as `feature` in `X` and vice
+    versa.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/23720
+    """
+    rng = np.random.RandomState(global_random_seed)
+    n = 100
+    d = rng.randint(10, size=n)
+
+    if correlated:
+        c = d.astype(np.float64)
+    else:
+        c = rng.normal(0, 1, size=n)
+
+    mi_classif = mutual_info_classif(
+        c[:, None], d, discrete_features=[False], random_state=global_random_seed
+    )
+
+    mi_regression = mutual_info_regression(
+        d[:, None], c, discrete_features=[True], random_state=global_random_seed
+    )
+
+    assert mi_classif == pytest.approx(mi_regression)
+
+
+def test_mutual_info_regression_X_int_dtype(global_random_seed):
+    """Check that results agree when X is integer dtype and float dtype.
+
+    Non-regression test for Issue #26696.
+    """
+    rng = np.random.RandomState(global_random_seed)
+    X = rng.randint(100, size=(100, 10))
+    X_float = X.astype(np.float64, copy=True)
+    y = rng.randint(100, size=100)
+
+    expected = mutual_info_regression(X_float, y, random_state=global_random_seed)
+    result = mutual_info_regression(X, y, random_state=global_random_seed)
+    assert_allclose(result, expected)
+
+
+@pytest.mark.parametrize(
+    "mutual_info_func, data_generator",
+    [
+        (mutual_info_regression, make_regression),
+        (mutual_info_classif, make_classification),
+    ],
+)
+def test_mutual_info_n_jobs(global_random_seed, mutual_info_func, data_generator):
+    """Check that results are consistent with different `n_jobs`."""
+    X, y = data_generator(random_state=global_random_seed)
+    single_job = mutual_info_func(X, y, random_state=global_random_seed, n_jobs=1)
+    multi_job = mutual_info_func(X, y, random_state=global_random_seed, n_jobs=2)
+    assert_allclose(single_job, multi_job)
diff --git a/sklearn/feature_selection/tests/test_rfe.py b/sklearn/feature_selection/tests/test_rfe.py
index 9e6dfdbbd593a..1f5672545874c 100644
--- a/sklearn/feature_selection/tests/test_rfe.py
+++ b/sklearn/feature_selection/tests/test_rfe.py
@@ -2,32 +2,33 @@
 Testing Recursive feature elimination
 """
 
+import re
 from operator import attrgetter
-import pytest
+
 import numpy as np
-from numpy.testing import assert_array_almost_equal, assert_array_equal
-from scipy import sparse
+import pytest
+from joblib import parallel_backend
+from numpy.testing import assert_allclose, assert_array_almost_equal, assert_array_equal
 
-from sklearn.feature_selection import RFE, RFECV
-from sklearn.datasets import load_iris, make_friedman1
-from sklearn.metrics import zero_one_loss
-from sklearn.svm import SVC, SVR, LinearSVR
-from sklearn.linear_model import LogisticRegression
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.model_selection import cross_val_score
-from sklearn.model_selection import GroupKFold
+from sklearn.base import BaseEstimator, ClassifierMixin, is_classifier
 from sklearn.compose import TransformedTargetRegressor
+from sklearn.cross_decomposition import CCA, PLSCanonical, PLSRegression
+from sklearn.datasets import load_iris, make_classification, make_friedman1
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.feature_selection import RFE, RFECV
+from sklearn.impute import SimpleImputer
+from sklearn.linear_model import LinearRegression, LogisticRegression
+from sklearn.metrics import get_scorer, make_scorer, zero_one_loss
+from sklearn.model_selection import GroupKFold, cross_val_score
 from sklearn.pipeline import make_pipeline
 from sklearn.preprocessing import StandardScaler
-
+from sklearn.svm import SVC, SVR, LinearSVR
 from sklearn.utils import check_random_state
 from sklearn.utils._testing import ignore_warnings
-
-from sklearn.metrics import make_scorer
-from sklearn.metrics import get_scorer
+from sklearn.utils.fixes import CSR_CONTAINERS
 
 
-class MockClassifier:
+class MockClassifier(ClassifierMixin, BaseEstimator):
     """
     Dummy classifier to test recursive feature elimination
     """
@@ -38,36 +39,40 @@ def __init__(self, foo_param=0):
     def fit(self, X, y):
         assert len(X) == len(y)
         self.coef_ = np.ones(X.shape[1], dtype=np.float64)
+        self.classes_ = sorted(set(y))
         return self
 
     def predict(self, T):
-        return T.shape[0]
+        return np.ones(T.shape[0])
 
     predict_proba = predict
     decision_function = predict
     transform = predict
 
     def score(self, X=None, y=None):
-        return 0.
+        return 0.0
 
     def get_params(self, deep=True):
-        return {'foo_param': self.foo_param}
+        return {"foo_param": self.foo_param}
 
     def set_params(self, **params):
         return self
 
-    def _more_tags(self):
-        return {"allow_nan": True}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.allow_nan = True
+        return tags
 
 
 def test_rfe_features_importance():
     generator = check_random_state(0)
     iris = load_iris()
+    # Add some irrelevant features. Random seed is set to make sure that
+    # irrelevant features are always irrelevant.
     X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
     y = iris.target
 
-    clf = RandomForestClassifier(n_estimators=20,
-                                 random_state=generator, max_depth=2)
+    clf = RandomForestClassifier(n_estimators=20, random_state=generator, max_depth=2)
     rfe = RFE(estimator=clf, n_features_to_select=4, step=0.1)
     rfe.fit(X, y)
     assert len(rfe.ranking_) == X.shape[1]
@@ -80,11 +85,14 @@ def test_rfe_features_importance():
     assert_array_equal(rfe.get_support(), rfe_svc.get_support())
 
 
-def test_rfe():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_rfe(csr_container):
     generator = check_random_state(0)
     iris = load_iris()
+    # Add some irrelevant features. Random seed is set to make sure that
+    # irrelevant features are always irrelevant.
     X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
-    X_sparse = sparse.csr_matrix(X)
+    X_sparse = csr_container(X)
     y = iris.target
 
     # dense model
@@ -109,22 +117,37 @@ def test_rfe():
     assert_array_almost_equal(X_r, X_r_sparse.toarray())
 
 
-@pytest.mark.parametrize("n_features_to_select", [-1, 2.1])
-def test_rfe_invalid_n_features_errors(n_features_to_select):
-    clf = SVC(kernel="linear")
+def test_RFE_fit_score_params():
+    # Make sure RFE passes the metadata down to fit and score methods of the
+    # underlying estimator
+    class TestEstimator(BaseEstimator, ClassifierMixin):
+        def fit(self, X, y, prop=None):
+            if prop is None:
+                raise ValueError("fit: prop cannot be None")
+            self.svc_ = SVC(kernel="linear").fit(X, y)
+            self.coef_ = self.svc_.coef_
+            return self
 
-    iris = load_iris()
-    rfe = RFE(estimator=clf, n_features_to_select=n_features_to_select,
-              step=0.1)
-    msg = f"n_features_to_select must be .+ Got {n_features_to_select}"
-    with pytest.raises(ValueError, match=msg):
-        rfe.fit(iris.data, iris.target)
+        def score(self, X, y, prop=None):
+            if prop is None:
+                raise ValueError("score: prop cannot be None")
+            return self.svc_.score(X, y)
+
+    X, y = load_iris(return_X_y=True)
+    with pytest.raises(ValueError, match="fit: prop cannot be None"):
+        RFE(estimator=TestEstimator()).fit(X, y)
+    with pytest.raises(ValueError, match="score: prop cannot be None"):
+        RFE(estimator=TestEstimator()).fit(X, y, prop="foo").score(X, y)
+
+    RFE(estimator=TestEstimator()).fit(X, y, prop="foo").score(X, y, prop="foo")
 
 
 def test_rfe_percent_n_features():
     # test that the results are the same
     generator = check_random_state(0)
     iris = load_iris()
+    # Add some irrelevant features. Random seed is set to make sure that
+    # irrelevant features are always irrelevant.
     X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
     y = iris.target
     # there are 10 features in the data. We select 40%.
@@ -142,6 +165,8 @@ def test_rfe_percent_n_features():
 def test_rfe_mockclassifier():
     generator = check_random_state(0)
     iris = load_iris()
+    # Add some irrelevant features. Random seed is set to make sure that
+    # irrelevant features are always irrelevant.
     X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
     y = iris.target
 
@@ -155,17 +180,23 @@ def test_rfe_mockclassifier():
     assert X_r.shape == iris.data.shape
 
 
-def test_rfecv():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_rfecv(csr_container):
     generator = check_random_state(0)
     iris = load_iris()
+    # Add some irrelevant features. Random seed is set to make sure that
+    # irrelevant features are always irrelevant.
     X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
-    y = list(iris.target)   # regression test: list should be supported
+    y = list(iris.target)  # regression test: list should be supported
 
     # Test using the score function
     rfecv = RFECV(estimator=SVC(kernel="linear"), step=1)
     rfecv.fit(X, y)
     # non-regression test for missing worst feature:
-    assert len(rfecv.grid_scores_) == X.shape[1]
+
+    for key in rfecv.cv_results_.keys():
+        assert len(rfecv.cv_results_[key]) == X.shape[1]
+
     assert len(rfecv.ranking_) == X.shape[1]
     X_r = rfecv.transform(X)
 
@@ -174,7 +205,7 @@ def test_rfecv():
 
     # same in sparse
     rfecv_sparse = RFECV(estimator=SVC(kernel="linear"), step=1)
-    X_sparse = sparse.csr_matrix(X)
+    X_sparse = csr_container(X)
     rfecv_sparse.fit(X_sparse, y)
     X_r_sparse = rfecv_sparse.transform(X_sparse)
     assert_array_equal(X_r_sparse.toarray(), iris.data)
@@ -187,18 +218,19 @@ def test_rfecv():
     assert_array_equal(X_r, iris.data)
 
     # Test using a scorer
-    scorer = get_scorer('accuracy')
+    scorer = get_scorer("accuracy")
     rfecv = RFECV(estimator=SVC(kernel="linear"), step=1, scoring=scorer)
     rfecv.fit(X, y)
     X_r = rfecv.transform(X)
     assert_array_equal(X_r, iris.data)
 
-    # Test fix on grid_scores
+    # Test fix on cv_results_
     def test_scorer(estimator, X, y):
         return 1.0
+
     rfecv = RFECV(estimator=SVC(kernel="linear"), step=1, scoring=test_scorer)
     rfecv.fit(X, y)
-    assert_array_equal(rfecv.grid_scores_, np.ones(len(rfecv.grid_scores_)))
+
     # In the event of cross validation score ties, the expected behavior of
     # RFECV is to return the FEWEST features that maximize the CV score.
     # Because test_scorer always returns 1.0 in this example, RFECV should
@@ -208,20 +240,23 @@ def test_scorer(estimator, X, y):
     # Same as the first two tests, but with step=2
     rfecv = RFECV(estimator=SVC(kernel="linear"), step=2)
     rfecv.fit(X, y)
-    assert len(rfecv.grid_scores_) == 6
+
+    for key in rfecv.cv_results_.keys():
+        assert len(rfecv.cv_results_[key]) == 6
+
     assert len(rfecv.ranking_) == X.shape[1]
     X_r = rfecv.transform(X)
     assert_array_equal(X_r, iris.data)
 
     rfecv_sparse = RFECV(estimator=SVC(kernel="linear"), step=2)
-    X_sparse = sparse.csr_matrix(X)
+    X_sparse = csr_container(X)
     rfecv_sparse.fit(X_sparse, y)
     X_r_sparse = rfecv_sparse.transform(X_sparse)
     assert_array_equal(X_r_sparse.toarray(), iris.data)
 
     # Verifying that steps < 1 don't blow up.
-    rfecv_sparse = RFECV(estimator=SVC(kernel="linear"), step=.2)
-    X_sparse = sparse.csr_matrix(X)
+    rfecv_sparse = RFECV(estimator=SVC(kernel="linear"), step=0.2)
+    X_sparse = csr_container(X)
     rfecv_sparse.fit(X_sparse, y)
     X_r_sparse = rfecv_sparse.transform(X_sparse)
     assert_array_equal(X_r_sparse.toarray(), iris.data)
@@ -231,20 +266,24 @@ def test_rfecv_mockclassifier():
     generator = check_random_state(0)
     iris = load_iris()
     X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
-    y = list(iris.target)   # regression test: list should be supported
+    y = list(iris.target)  # regression test: list should be supported
 
     # Test using the score function
     rfecv = RFECV(estimator=MockClassifier(), step=1)
     rfecv.fit(X, y)
     # non-regression test for missing worst feature:
-    assert len(rfecv.grid_scores_) == X.shape[1]
+
+    for key in rfecv.cv_results_.keys():
+        assert len(rfecv.cv_results_[key]) == X.shape[1]
+
     assert len(rfecv.ranking_) == X.shape[1]
 
 
 def test_rfecv_verbose_output():
     # Check verbose=1 is producing an output.
-    from io import StringIO
     import sys
+    from io import StringIO
+
     sys.stdout = StringIO()
 
     generator = check_random_state(0)
@@ -260,38 +299,45 @@ def test_rfecv_verbose_output():
     assert len(verbose_output.readline()) > 0
 
 
-def test_rfecv_grid_scores_size():
-    generator = check_random_state(0)
+def test_rfecv_cv_results_size(global_random_seed):
+    generator = check_random_state(global_random_seed)
     iris = load_iris()
     X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
-    y = list(iris.target)   # regression test: list should be supported
+    y = list(iris.target)  # regression test: list should be supported
 
     # Non-regression test for varying combinations of step and
     # min_features_to_select.
     for step, min_features_to_select in [[2, 1], [2, 2], [3, 3]]:
-        rfecv = RFECV(estimator=MockClassifier(), step=step,
-                      min_features_to_select=min_features_to_select)
+        rfecv = RFECV(
+            estimator=MockClassifier(),
+            step=step,
+            min_features_to_select=min_features_to_select,
+        )
         rfecv.fit(X, y)
 
-        score_len = np.ceil(
-            (X.shape[1] - min_features_to_select) / step) + 1
-        assert len(rfecv.grid_scores_) == score_len
+        score_len = np.ceil((X.shape[1] - min_features_to_select) / step) + 1
+
+        for key in rfecv.cv_results_.keys():
+            assert len(rfecv.cv_results_[key]) == score_len
+
         assert len(rfecv.ranking_) == X.shape[1]
         assert rfecv.n_features_ >= min_features_to_select
 
 
 def test_rfe_estimator_tags():
-    rfe = RFE(SVC(kernel='linear'))
-    assert rfe._estimator_type == "classifier"
+    rfe = RFE(SVC(kernel="linear"))
+    assert is_classifier(rfe)
     # make sure that cross-validation is stratified
     iris = load_iris()
     score = cross_val_score(rfe, iris.data, iris.target)
-    assert score.min() > .7
+    assert score.min() > 0.7
 
 
-def test_rfe_min_step():
+def test_rfe_min_step(global_random_seed):
     n_features = 10
-    X, y = make_friedman1(n_samples=50, n_features=n_features, random_state=0)
+    X, y = make_friedman1(
+        n_samples=50, n_features=n_features, random_state=global_random_seed
+    )
     n_samples, n_features = X.shape
     estimator = SVR(kernel="linear")
 
@@ -311,7 +357,7 @@ def test_rfe_min_step():
     assert sel.support_.sum() == n_features // 2
 
 
-def test_number_of_subsets_of_features():
+def test_number_of_subsets_of_features(global_random_seed):
     # In RFE, 'number_of_subsets_of_features'
     # = the number of iterations in '_fit'
     # = max(ranking_)
@@ -333,22 +379,24 @@ def formula2(n_features, n_features_to_select, step):
     n_features_to_select_list = [3, 3]
     step_list = [2, 3]
     for n_features, n_features_to_select, step in zip(
-            n_features_list, n_features_to_select_list, step_list):
-        generator = check_random_state(43)
+        n_features_list, n_features_to_select_list, step_list
+    ):
+        generator = check_random_state(global_random_seed)
         X = generator.normal(size=(100, n_features))
         y = generator.rand(100).round()
-        rfe = RFE(estimator=SVC(kernel="linear"),
-                  n_features_to_select=n_features_to_select, step=step)
+        rfe = RFE(
+            estimator=SVC(kernel="linear"),
+            n_features_to_select=n_features_to_select,
+            step=step,
+        )
         rfe.fit(X, y)
         # this number also equals to the maximum of ranking_
-        assert (np.max(rfe.ranking_) ==
-                formula1(n_features, n_features_to_select, step))
-        assert (np.max(rfe.ranking_) ==
-                formula2(n_features, n_features_to_select, step))
+        assert np.max(rfe.ranking_) == formula1(n_features, n_features_to_select, step)
+        assert np.max(rfe.ranking_) == formula2(n_features, n_features_to_select, step)
 
     # In RFECV, 'fit' calls 'RFE._fit'
     # 'number_of_subsets_of_features' of RFE
-    # = the size of 'grid_scores' of RFECV
+    # = the size of each score in 'cv_results_' of RFECV
     # = the number of iterations of the for loop before optimization #4534
 
     # RFECV, n_features_to_select = 1
@@ -359,33 +407,40 @@ def formula2(n_features, n_features_to_select, step):
     n_features_list = [11, 10]
     step_list = [2, 2]
     for n_features, step in zip(n_features_list, step_list):
-        generator = check_random_state(43)
+        generator = check_random_state(global_random_seed)
         X = generator.normal(size=(100, n_features))
         y = generator.rand(100).round()
         rfecv = RFECV(estimator=SVC(kernel="linear"), step=step)
         rfecv.fit(X, y)
 
-        assert (rfecv.grid_scores_.shape[0] ==
-                formula1(n_features, n_features_to_select, step))
-        assert (rfecv.grid_scores_.shape[0] ==
-                formula2(n_features, n_features_to_select, step))
+        for key in rfecv.cv_results_.keys():
+            assert len(rfecv.cv_results_[key]) == formula1(
+                n_features, n_features_to_select, step
+            )
+            assert len(rfecv.cv_results_[key]) == formula2(
+                n_features, n_features_to_select, step
+            )
 
 
-def test_rfe_cv_n_jobs():
-    generator = check_random_state(0)
+def test_rfe_cv_n_jobs(global_random_seed):
+    generator = check_random_state(global_random_seed)
     iris = load_iris()
     X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
     y = iris.target
 
-    rfecv = RFECV(estimator=SVC(kernel='linear'))
+    rfecv = RFECV(estimator=SVC(kernel="linear"))
     rfecv.fit(X, y)
     rfecv_ranking = rfecv.ranking_
-    rfecv_grid_scores = rfecv.grid_scores_
+
+    rfecv_cv_results_ = rfecv.cv_results_
 
     rfecv.set_params(n_jobs=2)
     rfecv.fit(X, y)
     assert_array_almost_equal(rfecv.ranking_, rfecv_ranking)
-    assert_array_almost_equal(rfecv.grid_scores_, rfecv_grid_scores)
+
+    assert rfecv_cv_results_.keys() == rfecv.cv_results_.keys()
+    for key in rfecv_cv_results_.keys():
+        assert rfecv_cv_results_[key] == pytest.approx(rfecv.cv_results_[key])
 
 
 def test_rfe_cv_groups():
@@ -399,28 +454,26 @@ def test_rfe_cv_groups():
     est_groups = RFECV(
         estimator=RandomForestClassifier(random_state=generator),
         step=1,
-        scoring='accuracy',
-        cv=GroupKFold(n_splits=2)
+        scoring="accuracy",
+        cv=GroupKFold(n_splits=2),
     )
     est_groups.fit(X, y, groups=groups)
     assert est_groups.n_features_ > 0
 
 
 @pytest.mark.parametrize(
-    'importance_getter',
-    [attrgetter('regressor_.coef_'), 'regressor_.coef_'])
-@pytest.mark.parametrize('selector, expected_n_features',
-                         [(RFE, 5), (RFECV, 4)])
-def test_rfe_wrapped_estimator(importance_getter, selector,
-                               expected_n_features):
+    "importance_getter", [attrgetter("regressor_.coef_"), "regressor_.coef_"]
+)
+@pytest.mark.parametrize("selector, expected_n_features", [(RFE, 5), (RFECV, 4)])
+def test_rfe_wrapped_estimator(importance_getter, selector, expected_n_features):
     # Non-regression test for
     # https://github.com/scikit-learn/scikit-learn/issues/15312
     X, y = make_friedman1(n_samples=50, n_features=10, random_state=0)
     estimator = LinearSVR(random_state=0)
 
-    log_estimator = TransformedTargetRegressor(regressor=estimator,
-                                               func=np.log,
-                                               inverse_func=np.exp)
+    log_estimator = TransformedTargetRegressor(
+        regressor=estimator, func=np.log, inverse_func=np.exp
+    )
 
     selector = selector(log_estimator, importance_getter=importance_getter)
     sel = selector.fit(X, y)
@@ -429,14 +482,14 @@ def test_rfe_wrapped_estimator(importance_getter, selector,
 
 @pytest.mark.parametrize(
     "importance_getter, err_type",
-    [("auto", ValueError),
-     ("random", AttributeError),
-     (lambda x: x.importance, AttributeError),
-     ([0], ValueError)]
+    [
+        ("auto", ValueError),
+        ("random", AttributeError),
+        (lambda x: x.importance, AttributeError),
+    ],
 )
 @pytest.mark.parametrize("Selector", [RFE, RFECV])
-def test_rfe_importance_getter_validation(importance_getter, err_type,
-                                          Selector):
+def test_rfe_importance_getter_validation(importance_getter, err_type, Selector):
     X, y = make_friedman1(n_samples=50, n_features=10, random_state=42)
     estimator = LinearSVR()
     log_estimator = TransformedTargetRegressor(
@@ -455,8 +508,8 @@ def test_rfe_allow_nan_inf_in_x(cv):
     y = iris.target
 
     # add nan and inf value to X
-    X[0][0] = np.NaN
-    X[0][1] = np.Inf
+    X[0][0] = np.nan
+    X[0][1] = np.inf
 
     clf = MockClassifier()
     if cv is not None:
@@ -471,20 +524,232 @@ def test_w_pipeline_2d_coef_():
     pipeline = make_pipeline(StandardScaler(), LogisticRegression())
 
     data, y = load_iris(return_X_y=True)
-    sfm = RFE(pipeline, n_features_to_select=2,
-              importance_getter='named_steps.logisticregression.coef_')
+    sfm = RFE(
+        pipeline,
+        n_features_to_select=2,
+        importance_getter="named_steps.logisticregression.coef_",
+    )
 
     sfm.fit(data, y)
     assert sfm.transform(data).shape[1] == 2
 
 
-@pytest.mark.parametrize('ClsRFE', [
-    RFE,
-    RFECV
-    ])
+def test_rfecv_std_and_mean(global_random_seed):
+    generator = check_random_state(global_random_seed)
+    iris = load_iris()
+    X = np.c_[iris.data, generator.normal(size=(len(iris.data), 6))]
+    y = iris.target
+
+    rfecv = RFECV(estimator=SVC(kernel="linear"))
+    rfecv.fit(X, y)
+    split_keys = [
+        key
+        for key in rfecv.cv_results_.keys()
+        if re.search(r"split\d+_test_score", key)
+    ]
+    cv_scores = np.asarray([rfecv.cv_results_[key] for key in split_keys])
+    expected_mean = np.mean(cv_scores, axis=0)
+    expected_std = np.std(cv_scores, axis=0)
+
+    assert_allclose(rfecv.cv_results_["mean_test_score"], expected_mean)
+    assert_allclose(rfecv.cv_results_["std_test_score"], expected_std)
+
+
+@pytest.mark.parametrize(
+    ["min_features_to_select", "n_features", "step", "cv_results_n_features"],
+    [
+        [1, 4, 1, np.array([1, 2, 3, 4])],
+        [1, 5, 1, np.array([1, 2, 3, 4, 5])],
+        [1, 4, 2, np.array([1, 2, 4])],
+        [1, 5, 2, np.array([1, 3, 5])],
+        [1, 4, 3, np.array([1, 4])],
+        [1, 5, 3, np.array([1, 2, 5])],
+        [1, 4, 4, np.array([1, 4])],
+        [1, 5, 4, np.array([1, 5])],
+        [4, 4, 2, np.array([4])],
+        [4, 5, 1, np.array([4, 5])],
+        [4, 5, 2, np.array([4, 5])],
+    ],
+)
+def test_rfecv_cv_results_n_features(
+    min_features_to_select,
+    n_features,
+    step,
+    cv_results_n_features,
+):
+    X, y = make_classification(
+        n_samples=20, n_features=n_features, n_informative=n_features, n_redundant=0
+    )
+    rfecv = RFECV(
+        estimator=SVC(kernel="linear"),
+        step=step,
+        min_features_to_select=min_features_to_select,
+    )
+    rfecv.fit(X, y)
+    assert_array_equal(rfecv.cv_results_["n_features"], cv_results_n_features)
+    assert all(
+        len(value) == len(rfecv.cv_results_["n_features"])
+        for value in rfecv.cv_results_.values()
+    )
+
+
+@pytest.mark.parametrize("ClsRFE", [RFE, RFECV])
 def test_multioutput(ClsRFE):
     X = np.random.normal(size=(10, 3))
     y = np.random.randint(2, size=(10, 2))
     clf = RandomForestClassifier(n_estimators=5)
     rfe_test = ClsRFE(clf)
     rfe_test.fit(X, y)
+
+
+@pytest.mark.parametrize("ClsRFE", [RFE, RFECV])
+def test_pipeline_with_nans(ClsRFE):
+    """Check that RFE works with pipeline that accept nans.
+
+    Non-regression test for gh-21743.
+    """
+    X, y = load_iris(return_X_y=True)
+    X[0, 0] = np.nan
+
+    pipe = make_pipeline(
+        SimpleImputer(),
+        StandardScaler(),
+        LogisticRegression(),
+    )
+
+    fs = ClsRFE(
+        estimator=pipe,
+        importance_getter="named_steps.logisticregression.coef_",
+    )
+    fs.fit(X, y)
+
+
+@pytest.mark.parametrize("ClsRFE", [RFE, RFECV])
+@pytest.mark.parametrize("PLSEstimator", [CCA, PLSCanonical, PLSRegression])
+def test_rfe_pls(ClsRFE, PLSEstimator):
+    """Check the behaviour of RFE with PLS estimators.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/12410
+    """
+    X, y = make_friedman1(n_samples=50, n_features=10, random_state=0)
+    estimator = PLSEstimator(n_components=1)
+    selector = ClsRFE(estimator, step=1).fit(X, y)
+    assert selector.score(X, y) > 0.5
+
+
+def test_rfe_estimator_attribute_error():
+    """Check that we raise the proper AttributeError when the estimator
+    does not implement the `decision_function` method, which is decorated with
+    `available_if`.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/28108
+    """
+    iris = load_iris()
+
+    # `LinearRegression` does not implement 'decision_function' and should raise an
+    # AttributeError
+    rfe = RFE(estimator=LinearRegression())
+
+    outer_msg = "This 'RFE' has no attribute 'decision_function'"
+    inner_msg = "'LinearRegression' object has no attribute 'decision_function'"
+    with pytest.raises(AttributeError, match=outer_msg) as exec_info:
+        rfe.fit(iris.data, iris.target).decision_function(iris.data)
+    assert isinstance(exec_info.value.__cause__, AttributeError)
+    assert inner_msg in str(exec_info.value.__cause__)
+
+
+@pytest.mark.parametrize(
+    "ClsRFE, param", [(RFE, "n_features_to_select"), (RFECV, "min_features_to_select")]
+)
+def test_rfe_n_features_to_select_warning(ClsRFE, param):
+    """Check if the correct warning is raised when trying to initialize a RFE
+    object with a n_features_to_select attribute larger than the number of
+    features present in the X variable that is passed to the fit method
+    """
+    X, y = make_classification(n_features=20, random_state=0)
+
+    with pytest.warns(UserWarning, match=f"{param}=21 > n_features=20"):
+        # Create RFE/RFECV with n_features_to_select/min_features_to_select
+        # larger than the number of features present in the X variable
+        clsrfe = ClsRFE(estimator=LogisticRegression(), **{param: 21})
+        clsrfe.fit(X, y)
+
+
+def test_rfe_with_sample_weight():
+    """Test that `RFE` works correctly with sample weights."""
+    X, y = make_classification(random_state=0)
+    n_samples = X.shape[0]
+
+    # Assign the first half of the samples with twice the weight
+    sample_weight = np.ones_like(y)
+    sample_weight[: n_samples // 2] = 2
+
+    # Duplicate the first half of the data samples to replicate the effect
+    # of sample weights for comparison
+    X2 = np.concatenate([X, X[: n_samples // 2]], axis=0)
+    y2 = np.concatenate([y, y[: n_samples // 2]])
+
+    estimator = SVC(kernel="linear")
+
+    rfe_sw = RFE(estimator=estimator, step=0.1)
+    rfe_sw.fit(X, y, sample_weight=sample_weight)
+
+    rfe = RFE(estimator=estimator, step=0.1)
+    rfe.fit(X2, y2)
+
+    assert_array_equal(rfe_sw.ranking_, rfe.ranking_)
+
+    # Also verify that when sample weights are not doubled the results
+    # are different from the duplicated data
+    rfe_sw_2 = RFE(estimator=estimator, step=0.1)
+    sample_weight_2 = np.ones_like(y)
+    rfe_sw_2.fit(X, y, sample_weight=sample_weight_2)
+
+    assert not np.array_equal(rfe_sw_2.ranking_, rfe.ranking_)
+
+
+def test_rfe_with_joblib_threading_backend(global_random_seed):
+    X, y = make_classification(random_state=global_random_seed)
+
+    clf = LogisticRegression()
+    rfe = RFECV(
+        estimator=clf,
+        n_jobs=2,
+    )
+
+    rfe.fit(X, y)
+    ranking_ref = rfe.ranking_
+
+    with parallel_backend("threading"):
+        rfe.fit(X, y)
+
+    assert_array_equal(ranking_ref, rfe.ranking_)
+
+
+def test_results_per_cv_in_rfecv(global_random_seed):
+    """
+    Test that the results of RFECV are consistent across the different folds
+    in terms of length of the arrays.
+    """
+    X, y = make_classification(random_state=global_random_seed)
+
+    clf = LogisticRegression()
+    rfecv = RFECV(
+        estimator=clf,
+        n_jobs=2,
+        cv=5,
+    )
+
+    rfecv.fit(X, y)
+
+    assert len(rfecv.cv_results_["split1_test_score"]) == len(
+        rfecv.cv_results_["split2_test_score"]
+    )
+    assert len(rfecv.cv_results_["split1_support"]) == len(
+        rfecv.cv_results_["split2_support"]
+    )
+    assert len(rfecv.cv_results_["split1_ranking"]) == len(
+        rfecv.cv_results_["split2_ranking"]
+    )
diff --git a/sklearn/feature_selection/tests/test_sequential.py b/sklearn/feature_selection/tests/test_sequential.py
index 2ca22517ef956..b98d5b400b84e 100644
--- a/sklearn/feature_selection/tests/test_sequential.py
+++ b/sklearn/feature_selection/tests/test_sequential.py
@@ -1,75 +1,170 @@
-import pytest
-import scipy
 import numpy as np
+import pytest
 from numpy.testing import assert_array_equal
 
-from sklearn.preprocessing import StandardScaler
-from sklearn.pipeline import make_pipeline
+from sklearn.cluster import KMeans
+from sklearn.datasets import make_blobs, make_classification, make_regression
+from sklearn.ensemble import HistGradientBoostingRegressor
 from sklearn.feature_selection import SequentialFeatureSelector
-from sklearn.datasets import make_regression
 from sklearn.linear_model import LinearRegression
-from sklearn.experimental import enable_hist_gradient_boosting  # noqa
-from sklearn.ensemble import HistGradientBoostingRegressor
-
-
-@pytest.mark.parametrize('n_features_to_select', (0, 5, 0., -1, 1.1))
-def test_bad_n_features_to_select(n_features_to_select):
-    X, y = make_regression(n_features=5)
-    sfs = SequentialFeatureSelector(LinearRegression(),
-                                    n_features_to_select=n_features_to_select)
-    with pytest.raises(ValueError, match="must be either None"):
-        sfs.fit(X, y)
+from sklearn.model_selection import LeaveOneGroupOut, cross_val_score
+from sklearn.neighbors import KNeighborsClassifier
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import StandardScaler
+from sklearn.utils.fixes import CSR_CONTAINERS
 
 
-def test_bad_direction():
-    X, y = make_regression(n_features=5)
-    sfs = SequentialFeatureSelector(LinearRegression(), direction='bad')
-    with pytest.raises(ValueError, match="must be either 'forward' or"):
+def test_bad_n_features_to_select():
+    n_features = 5
+    X, y = make_regression(n_features=n_features)
+    sfs = SequentialFeatureSelector(LinearRegression(), n_features_to_select=n_features)
+    with pytest.raises(ValueError, match="n_features_to_select must be < n_features"):
         sfs.fit(X, y)
 
 
-@pytest.mark.parametrize('direction', ('forward', 'backward'))
-@pytest.mark.parametrize('n_features_to_select', (1, 5, 9, None))
+@pytest.mark.parametrize("direction", ("forward", "backward"))
+@pytest.mark.parametrize("n_features_to_select", (1, 5, 9, "auto"))
 def test_n_features_to_select(direction, n_features_to_select):
     # Make sure n_features_to_select is respected
 
-    X, y = make_regression(n_features=10)
-    sfs = SequentialFeatureSelector(LinearRegression(),
-                                    n_features_to_select=n_features_to_select,
-                                    direction=direction, cv=2)
+    n_features = 10
+    X, y = make_regression(n_features=n_features, random_state=0)
+    sfs = SequentialFeatureSelector(
+        LinearRegression(),
+        n_features_to_select=n_features_to_select,
+        direction=direction,
+        cv=2,
+    )
     sfs.fit(X, y)
-    if n_features_to_select is None:
-        n_features_to_select = 5  # n_features // 2
+
+    if n_features_to_select == "auto":
+        n_features_to_select = n_features // 2
+
     assert sfs.get_support(indices=True).shape[0] == n_features_to_select
     assert sfs.n_features_to_select_ == n_features_to_select
     assert sfs.transform(X).shape[1] == n_features_to_select
 
 
-@pytest.mark.parametrize('direction', ('forward', 'backward'))
-@pytest.mark.parametrize('n_features_to_select, expected', (
-    (.1, 1),
-    (1., 10),
-    (.5, 5),
-    (None, 5),  # just to make sure .5 is equivalent to passing None
-))
+@pytest.mark.parametrize("direction", ("forward", "backward"))
+def test_n_features_to_select_auto(direction):
+    """Check the behaviour of `n_features_to_select="auto"` with different
+    values for the parameter `tol`.
+    """
+
+    n_features = 10
+    tol = 1e-3
+    X, y = make_regression(n_features=n_features, random_state=0)
+    sfs = SequentialFeatureSelector(
+        LinearRegression(),
+        n_features_to_select="auto",
+        tol=tol,
+        direction=direction,
+        cv=2,
+    )
+    sfs.fit(X, y)
+
+    max_features_to_select = n_features - 1
+
+    assert sfs.get_support(indices=True).shape[0] <= max_features_to_select
+    assert sfs.n_features_to_select_ <= max_features_to_select
+    assert sfs.transform(X).shape[1] <= max_features_to_select
+    assert sfs.get_support(indices=True).shape[0] == sfs.n_features_to_select_
+
+
+@pytest.mark.parametrize("direction", ("forward", "backward"))
+def test_n_features_to_select_stopping_criterion(direction):
+    """Check the behaviour stopping criterion for feature selection
+    depending on the values of `n_features_to_select` and `tol`.
+
+    When `direction` is `'forward'`, select a new features at random
+    among those not currently selected in selector.support_,
+    build a new version of the data that includes all the features
+    in selector.support_ + this newly selected feature.
+    And check that the cross-validation score of the model trained on
+    this new dataset variant is lower than the model with
+    the selected forward selected features or at least does not improve
+    by more than the tol margin.
+
+    When `direction` is `'backward'`, instead of adding a new feature
+    to selector.support_, try to remove one of those selected features at random
+    And check that the cross-validation score is either decreasing or
+    not improving by more than the tol margin.
+    """
+
+    X, y = make_regression(n_features=50, n_informative=10, random_state=0)
+
+    tol = 1e-3
+
+    sfs = SequentialFeatureSelector(
+        LinearRegression(),
+        n_features_to_select="auto",
+        tol=tol,
+        direction=direction,
+        cv=2,
+    )
+    sfs.fit(X, y)
+    selected_X = sfs.transform(X)
+
+    rng = np.random.RandomState(0)
+
+    added_candidates = list(set(range(X.shape[1])) - set(sfs.get_support(indices=True)))
+    added_X = np.hstack(
+        [
+            selected_X,
+            (X[:, rng.choice(added_candidates)])[:, np.newaxis],
+        ]
+    )
+
+    removed_candidate = rng.choice(list(range(sfs.n_features_to_select_)))
+    removed_X = np.delete(selected_X, removed_candidate, axis=1)
+
+    plain_cv_score = cross_val_score(LinearRegression(), X, y, cv=2).mean()
+    sfs_cv_score = cross_val_score(LinearRegression(), selected_X, y, cv=2).mean()
+    added_cv_score = cross_val_score(LinearRegression(), added_X, y, cv=2).mean()
+    removed_cv_score = cross_val_score(LinearRegression(), removed_X, y, cv=2).mean()
+
+    assert sfs_cv_score >= plain_cv_score
+
+    if direction == "forward":
+        assert (sfs_cv_score - added_cv_score) <= tol
+        assert (sfs_cv_score - removed_cv_score) >= tol
+    else:
+        assert (added_cv_score - sfs_cv_score) <= tol
+        assert (removed_cv_score - sfs_cv_score) <= tol
+
+
+@pytest.mark.parametrize("direction", ("forward", "backward"))
+@pytest.mark.parametrize(
+    "n_features_to_select, expected",
+    (
+        (0.1, 1),
+        (1.0, 10),
+        (0.5, 5),
+    ),
+)
 def test_n_features_to_select_float(direction, n_features_to_select, expected):
     # Test passing a float as n_features_to_select
     X, y = make_regression(n_features=10)
-    sfs = SequentialFeatureSelector(LinearRegression(),
-                                    n_features_to_select=n_features_to_select,
-                                    direction=direction, cv=2)
+    sfs = SequentialFeatureSelector(
+        LinearRegression(),
+        n_features_to_select=n_features_to_select,
+        direction=direction,
+        cv=2,
+    )
     sfs.fit(X, y)
     assert sfs.n_features_to_select_ == expected
 
 
-@pytest.mark.parametrize('seed', range(10))
-@pytest.mark.parametrize('direction', ('forward', 'backward'))
-@pytest.mark.parametrize('n_features_to_select, expected_selected_features', [
-    (2, [0, 2]),  # f1 is dropped since it has no predictive power
-    (1, [2]),  # f2 is more predictive than f0 so it's kept
-])
-def test_sanity(seed, direction, n_features_to_select,
-                expected_selected_features):
+@pytest.mark.parametrize("seed", range(10))
+@pytest.mark.parametrize("direction", ("forward", "backward"))
+@pytest.mark.parametrize(
+    "n_features_to_select, expected_selected_features",
+    [
+        (2, [0, 2]),  # f1 is dropped since it has no predictive power
+        (1, [2]),  # f2 is more predictive than f0 so it's kept
+    ],
+)
+def test_sanity(seed, direction, n_features_to_select, expected_selected_features):
     # Basic sanity check: 3 features, only f0 and f2 are correlated with the
     # target, f2 having a stronger correlation than f0. We expect f1 to be
     # dropped, and f2 to always be selected.
@@ -79,20 +174,25 @@ def test_sanity(seed, direction, n_features_to_select,
     X = rng.randn(n_samples, 3)
     y = 3 * X[:, 0] - 10 * X[:, 2]
 
-    sfs = SequentialFeatureSelector(LinearRegression(),
-                                    n_features_to_select=n_features_to_select,
-                                    direction=direction, cv=2)
+    sfs = SequentialFeatureSelector(
+        LinearRegression(),
+        n_features_to_select=n_features_to_select,
+        direction=direction,
+        cv=2,
+    )
     sfs.fit(X, y)
-    assert_array_equal(sfs.get_support(indices=True),
-                       expected_selected_features)
+    assert_array_equal(sfs.get_support(indices=True), expected_selected_features)
 
 
-def test_sparse_support():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sparse_support(csr_container):
     # Make sure sparse data is supported
 
     X, y = make_regression(n_features=10)
-    X = scipy.sparse.csr_matrix(X)
-    sfs = SequentialFeatureSelector(LinearRegression(), cv=2)
+    X = csr_container(X)
+    sfs = SequentialFeatureSelector(
+        LinearRegression(), n_features_to_select="auto", cv=2
+    )
     sfs.fit(X, y)
     sfs.transform(X)
 
@@ -101,17 +201,21 @@ def test_nan_support():
     # Make sure nans are OK if the underlying estimator supports nans
 
     rng = np.random.RandomState(0)
-    n_samples, n_features = 100, 10
+    n_samples, n_features = 40, 4
     X, y = make_regression(n_samples, n_features, random_state=0)
     nan_mask = rng.randint(0, 2, size=(n_samples, n_features), dtype=bool)
     X[nan_mask] = np.nan
-    sfs = SequentialFeatureSelector(HistGradientBoostingRegressor(), cv=2)
+    sfs = SequentialFeatureSelector(
+        HistGradientBoostingRegressor(), n_features_to_select="auto", cv=2
+    )
     sfs.fit(X, y)
     sfs.transform(X)
 
-    with pytest.raises(ValueError, match='Input contains NaN'):
+    with pytest.raises(ValueError, match="Input X contains NaN"):
         # LinearRegression does not support nans
-        SequentialFeatureSelector(LinearRegression(), cv=2).fit(X, y)
+        SequentialFeatureSelector(
+            LinearRegression(), n_features_to_select="auto", cv=2
+        ).fit(X, y)
 
 
 def test_pipeline_support():
@@ -123,12 +227,106 @@ def test_pipeline_support():
 
     # pipeline in SFS
     pipe = make_pipeline(StandardScaler(), LinearRegression())
-    sfs = SequentialFeatureSelector(pipe, cv=2)
+    sfs = SequentialFeatureSelector(pipe, n_features_to_select="auto", cv=2)
     sfs.fit(X, y)
     sfs.transform(X)
 
     # SFS in pipeline
-    sfs = SequentialFeatureSelector(LinearRegression(), cv=2)
+    sfs = SequentialFeatureSelector(
+        LinearRegression(), n_features_to_select="auto", cv=2
+    )
     pipe = make_pipeline(StandardScaler(), sfs)
     pipe.fit(X, y)
     pipe.transform(X)
+
+
+@pytest.mark.parametrize("n_features_to_select", (2, 3))
+def test_unsupervised_model_fit(n_features_to_select):
+    # Make sure that models without classification labels are not being
+    # validated
+
+    X, y = make_blobs(n_features=4)
+    sfs = SequentialFeatureSelector(
+        KMeans(n_init=1),
+        n_features_to_select=n_features_to_select,
+    )
+    sfs.fit(X)
+    assert sfs.transform(X).shape[1] == n_features_to_select
+
+
+@pytest.mark.parametrize("y", ("no_validation", 1j, 99.9, np.nan, 3))
+def test_no_y_validation_model_fit(y):
+    # Make sure that other non-conventional y labels are not accepted
+
+    X, clusters = make_blobs(n_features=6)
+    sfs = SequentialFeatureSelector(
+        KMeans(),
+        n_features_to_select=3,
+    )
+
+    with pytest.raises((TypeError, ValueError)):
+        sfs.fit(X, y)
+
+
+def test_forward_neg_tol_error():
+    """Check that we raise an error when tol<0 and direction='forward'"""
+    X, y = make_regression(n_features=10, random_state=0)
+    sfs = SequentialFeatureSelector(
+        LinearRegression(),
+        n_features_to_select="auto",
+        direction="forward",
+        tol=-1e-3,
+    )
+
+    with pytest.raises(ValueError, match="tol must be strictly positive"):
+        sfs.fit(X, y)
+
+
+def test_backward_neg_tol():
+    """Check that SequentialFeatureSelector works negative tol
+
+    non-regression test for #25525
+    """
+    X, y = make_regression(n_features=10, random_state=0)
+    lr = LinearRegression()
+    initial_score = lr.fit(X, y).score(X, y)
+
+    sfs = SequentialFeatureSelector(
+        lr,
+        n_features_to_select="auto",
+        direction="backward",
+        tol=-1e-3,
+    )
+    Xr = sfs.fit_transform(X, y)
+    new_score = lr.fit(Xr, y).score(Xr, y)
+
+    assert 0 < sfs.get_support().sum() < X.shape[1]
+    assert new_score < initial_score
+
+
+def test_cv_generator_support():
+    """Check that no exception raised when cv is generator
+
+    non-regression test for #25957
+    """
+    X, y = make_classification(random_state=0)
+
+    groups = np.zeros_like(y, dtype=int)
+    groups[y.size // 2 :] = 1
+
+    cv = LeaveOneGroupOut()
+    splits = cv.split(X, y, groups=groups)
+
+    knc = KNeighborsClassifier(n_neighbors=5)
+
+    sfs = SequentialFeatureSelector(knc, n_features_to_select=5, cv=splits)
+    sfs.fit(X, y)
+
+
+def test_fit_rejects_params_with_no_routing_enabled():
+    X, y = make_classification(random_state=42)
+    est = LinearRegression()
+    sfs = SequentialFeatureSelector(estimator=est)
+
+    with pytest.raises(ValueError, match="is only supported if"):
+        sfs.fit(X, y, sample_weight=np.ones_like(y))
diff --git a/sklearn/feature_selection/tests/test_variance_threshold.py b/sklearn/feature_selection/tests/test_variance_threshold.py
index 23e7703708984..45e66cb338a4b 100644
--- a/sklearn/feature_selection/tests/test_variance_threshold.py
+++ b/sklearn/feature_selection/tests/test_variance_threshold.py
@@ -1,60 +1,72 @@
 import numpy as np
 import pytest
 
-from sklearn.utils._testing import assert_array_equal
-
-from scipy.sparse import bsr_matrix, csc_matrix, csr_matrix
-
 from sklearn.feature_selection import VarianceThreshold
+from sklearn.utils._testing import assert_array_equal
+from sklearn.utils.fixes import BSR_CONTAINERS, CSC_CONTAINERS, CSR_CONTAINERS
 
-data = [[0, 1, 2, 3, 4],
-        [0, 2, 2, 3, 5],
-        [1, 1, 2, 4, 0]]
+data = [[0, 1, 2, 3, 4], [0, 2, 2, 3, 5], [1, 1, 2, 4, 0]]
 
 data2 = [[-0.13725701]] * 10
 
-def test_zero_variance():
+
+@pytest.mark.parametrize(
+    "sparse_container", [None] + BSR_CONTAINERS + CSC_CONTAINERS + CSR_CONTAINERS
+)
+def test_zero_variance(sparse_container):
     # Test VarianceThreshold with default setting, zero variance.
+    X = data if sparse_container is None else sparse_container(data)
+    sel = VarianceThreshold().fit(X)
+    assert_array_equal([0, 1, 3, 4], sel.get_support(indices=True))
 
-    for X in [data, csr_matrix(data), csc_matrix(data), bsr_matrix(data)]:
-        sel = VarianceThreshold().fit(X)
-        assert_array_equal([0, 1, 3, 4], sel.get_support(indices=True))
 
+def test_zero_variance_value_error():
+    # Test VarianceThreshold with default setting, zero variance, error cases.
     with pytest.raises(ValueError):
         VarianceThreshold().fit([[0, 1, 2, 3]])
     with pytest.raises(ValueError):
         VarianceThreshold().fit([[0, 1], [0, 1]])
 
 
-def test_variance_threshold():
+@pytest.mark.parametrize("sparse_container", [None] + CSR_CONTAINERS)
+def test_variance_threshold(sparse_container):
     # Test VarianceThreshold with custom variance.
-    for X in [data, csr_matrix(data)]:
-        X = VarianceThreshold(threshold=.4).fit_transform(X)
-        assert (len(data), 1) == X.shape
-
-
-@pytest.mark.skipif(np.var(data2) == 0,
-                    reason=('This test is not valid for this platform, '
-                            'as it relies on numerical instabilities.'))
-def test_zero_variance_floating_point_error():
+    X = data if sparse_container is None else sparse_container(data)
+    X = VarianceThreshold(threshold=0.4).fit_transform(X)
+    assert (len(data), 1) == X.shape
+
+
+@pytest.mark.skipif(
+    np.var(data2) == 0,
+    reason=(
+        "This test is not valid for this platform, "
+        "as it relies on numerical instabilities."
+    ),
+)
+@pytest.mark.parametrize(
+    "sparse_container", [None] + BSR_CONTAINERS + CSC_CONTAINERS + CSR_CONTAINERS
+)
+def test_zero_variance_floating_point_error(sparse_container):
     # Test that VarianceThreshold(0.0).fit eliminates features that have
     # the same value in every sample, even when floating point errors
     # cause np.var not to be 0 for the feature.
     # See #13691
-
-    for X in [data2, csr_matrix(data2), csc_matrix(data2), bsr_matrix(data2)]:
-        msg = "No feature in X meets the variance threshold 0.00000"
-        with pytest.raises(ValueError, match=msg):
-            VarianceThreshold().fit(X)
+    X = data2 if sparse_container is None else sparse_container(data2)
+    msg = "No feature in X meets the variance threshold 0.00000"
+    with pytest.raises(ValueError, match=msg):
+        VarianceThreshold().fit(X)
 
 
-def test_variance_nan():
+@pytest.mark.parametrize(
+    "sparse_container", [None] + BSR_CONTAINERS + CSC_CONTAINERS + CSR_CONTAINERS
+)
+def test_variance_nan(sparse_container):
     arr = np.array(data, dtype=np.float64)
     # add single NaN and feature should still be included
-    arr[0, 0] = np.NaN
+    arr[0, 0] = np.nan
     # make all values in feature NaN and feature should be rejected
-    arr[:, 1] = np.NaN
+    arr[:, 1] = np.nan
 
-    for X in [arr, csr_matrix(arr), csc_matrix(arr), bsr_matrix(arr)]:
-        sel = VarianceThreshold().fit(X)
-        assert_array_equal([0, 3, 4], sel.get_support(indices=True))
+    X = arr if sparse_container is None else sparse_container(arr)
+    sel = VarianceThreshold().fit(X)
+    assert_array_equal([0, 3, 4], sel.get_support(indices=True))
diff --git a/sklearn/frozen/__init__.py b/sklearn/frozen/__init__.py
new file mode 100644
index 0000000000000..8ca540b79229c
--- /dev/null
+++ b/sklearn/frozen/__init__.py
@@ -0,0 +1,6 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+from ._frozen import FrozenEstimator
+
+__all__ = ["FrozenEstimator"]
diff --git a/sklearn/frozen/_frozen.py b/sklearn/frozen/_frozen.py
new file mode 100644
index 0000000000000..7585ea2597b59
--- /dev/null
+++ b/sklearn/frozen/_frozen.py
@@ -0,0 +1,166 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+from copy import deepcopy
+
+from ..base import BaseEstimator
+from ..exceptions import NotFittedError
+from ..utils import get_tags
+from ..utils.metaestimators import available_if
+from ..utils.validation import check_is_fitted
+
+
+def _estimator_has(attr):
+    """Check that final_estimator has `attr`.
+
+    Used together with `available_if`.
+    """
+
+    def check(self):
+        # raise original `AttributeError` if `attr` does not exist
+        getattr(self.estimator, attr)
+        return True
+
+    return check
+
+
+class FrozenEstimator(BaseEstimator):
+    """Estimator that wraps a fitted estimator to prevent re-fitting.
+
+    This meta-estimator takes an estimator and freezes it, in the sense that calling
+    `fit` on it has no effect. `fit_predict` and `fit_transform` are also disabled.
+    All other methods are delegated to the original estimator and original estimator's
+    attributes are accessible as well.
+
+    This is particularly useful when you have a fitted or a pre-trained model as a
+    transformer in a pipeline, and you'd like `pipeline.fit` to have no effect on this
+    step.
+
+    Parameters
+    ----------
+    estimator : estimator
+        The estimator which is to be kept frozen.
+
+    See Also
+    --------
+    None: No similar entry in the scikit-learn documentation.
+
+    Examples
+    --------
+    >>> from sklearn.datasets import make_classification
+    >>> from sklearn.frozen import FrozenEstimator
+    >>> from sklearn.linear_model import LogisticRegression
+    >>> X, y = make_classification(random_state=0)
+    >>> clf = LogisticRegression(random_state=0).fit(X, y)
+    >>> frozen_clf = FrozenEstimator(clf)
+    >>> frozen_clf.fit(X, y)  # No-op
+    FrozenEstimator(estimator=LogisticRegression(random_state=0))
+    >>> frozen_clf.predict(X)  # Predictions from `clf.predict`
+    array(...)
+    """
+
+    def __init__(self, estimator):
+        self.estimator = estimator
+
+    @available_if(_estimator_has("__getitem__"))
+    def __getitem__(self, *args, **kwargs):
+        """__getitem__ is defined in :class:`~sklearn.pipeline.Pipeline` and \
+            :class:`~sklearn.compose.ColumnTransformer`.
+        """
+        return self.estimator.__getitem__(*args, **kwargs)
+
+    def __getattr__(self, name):
+        # `estimator`'s attributes are now accessible except `fit_predict` and
+        # `fit_transform`
+        if name in ["fit_predict", "fit_transform"]:
+            raise AttributeError(f"{name} is not available for frozen estimators.")
+        return getattr(self.estimator, name)
+
+    def __sklearn_clone__(self):
+        return self
+
+    def __sklearn_is_fitted__(self):
+        try:
+            check_is_fitted(self.estimator)
+            return True
+        except NotFittedError:
+            return False
+
+    def fit(self, X, y, *args, **kwargs):
+        """No-op.
+
+        As a frozen estimator, calling `fit` has no effect.
+
+        Parameters
+        ----------
+        X : object
+            Ignored.
+
+        y : object
+            Ignored.
+
+        *args : tuple
+            Additional positional arguments. Ignored, but present for API compatibility
+            with `self.estimator`.
+
+        **kwargs : dict
+            Additional keyword arguments. Ignored, but present for API compatibility
+            with `self.estimator`.
+
+        Returns
+        -------
+        self : object
+            Returns the instance itself.
+        """
+        check_is_fitted(self.estimator)
+        return self
+
+    def set_params(self, **kwargs):
+        """Set the parameters of this estimator.
+
+        The only valid key here is `estimator`. You cannot set the parameters of the
+        inner estimator.
+
+        Parameters
+        ----------
+        **kwargs : dict
+            Estimator parameters.
+
+        Returns
+        -------
+        self : FrozenEstimator
+            This estimator.
+        """
+        estimator = kwargs.pop("estimator", None)
+        if estimator is not None:
+            self.estimator = estimator
+        if kwargs:
+            raise ValueError(
+                "You cannot set parameters of the inner estimator in a frozen "
+                "estimator since calling `fit` has no effect. You can use "
+                "`frozenestimator.estimator.set_params` to set parameters of the inner "
+                "estimator."
+            )
+
+    def get_params(self, deep=True):
+        """Get parameters for this estimator.
+
+        Returns a `{"estimator": estimator}` dict. The parameters of the inner
+        estimator are not included.
+
+        Parameters
+        ----------
+        deep : bool, default=True
+            Ignored.
+
+        Returns
+        -------
+        params : dict
+            Parameter names mapped to their values.
+        """
+        return {"estimator": self.estimator}
+
+    def __sklearn_tags__(self):
+        tags = deepcopy(get_tags(self.estimator))
+        tags._skip_test = True
+        return tags
diff --git a/sklearn/frozen/tests/__init__.py b/sklearn/frozen/tests/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/sklearn/frozen/tests/test_frozen.py b/sklearn/frozen/tests/test_frozen.py
new file mode 100644
index 0000000000000..b304d3ac0aa2c
--- /dev/null
+++ b/sklearn/frozen/tests/test_frozen.py
@@ -0,0 +1,223 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+import re
+
+import numpy as np
+import pytest
+from numpy.testing import assert_array_equal
+
+from sklearn import config_context
+from sklearn.base import (
+    BaseEstimator,
+    clone,
+    is_classifier,
+    is_clusterer,
+    is_outlier_detector,
+    is_regressor,
+)
+from sklearn.cluster import KMeans
+from sklearn.compose import make_column_transformer
+from sklearn.datasets import make_classification, make_regression
+from sklearn.exceptions import NotFittedError, UnsetMetadataPassedError
+from sklearn.frozen import FrozenEstimator
+from sklearn.linear_model import LinearRegression, LogisticRegression
+from sklearn.neighbors import LocalOutlierFactor
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import RobustScaler, StandardScaler
+from sklearn.utils._testing import set_random_state
+from sklearn.utils.validation import check_is_fitted
+
+
+@pytest.fixture
+def regression_dataset():
+    return make_regression()
+
+
+@pytest.fixture
+def classification_dataset():
+    return make_classification()
+
+
+@pytest.mark.parametrize(
+    "estimator, dataset",
+    [
+        (LinearRegression(), "regression_dataset"),
+        (LogisticRegression(), "classification_dataset"),
+        (make_pipeline(StandardScaler(), LinearRegression()), "regression_dataset"),
+        (
+            make_pipeline(StandardScaler(), LogisticRegression()),
+            "classification_dataset",
+        ),
+        (StandardScaler(), "regression_dataset"),
+        (KMeans(), "regression_dataset"),
+        (LocalOutlierFactor(), "regression_dataset"),
+        (
+            make_column_transformer(
+                (StandardScaler(), [0]),
+                (RobustScaler(), [1]),
+            ),
+            "regression_dataset",
+        ),
+    ],
+)
+@pytest.mark.parametrize(
+    "method",
+    ["predict", "predict_proba", "predict_log_proba", "decision_function", "transform"],
+)
+def test_frozen_methods(estimator, dataset, request, method):
+    """Test that frozen.fit doesn't do anything, and that all other methods are
+    exposed by the frozen estimator and return the same values as the estimator.
+    """
+    X, y = request.getfixturevalue(dataset)
+    set_random_state(estimator)
+    estimator.fit(X, y)
+    frozen = FrozenEstimator(estimator)
+    # this should be no-op
+    frozen.fit([[1]], [1])
+
+    if hasattr(estimator, method):
+        assert_array_equal(getattr(estimator, method)(X), getattr(frozen, method)(X))
+
+    assert is_classifier(estimator) == is_classifier(frozen)
+    assert is_regressor(estimator) == is_regressor(frozen)
+    assert is_clusterer(estimator) == is_clusterer(frozen)
+    assert is_outlier_detector(estimator) == is_outlier_detector(frozen)
+
+
+@config_context(enable_metadata_routing=True)
+def test_frozen_metadata_routing(regression_dataset):
+    """Test that metadata routing works with frozen estimators."""
+
+    class ConsumesMetadata(BaseEstimator):
+        def __init__(self, on_fit=None, on_predict=None):
+            self.on_fit = on_fit
+            self.on_predict = on_predict
+
+        def fit(self, X, y, metadata=None):
+            if self.on_fit:
+                assert metadata is not None
+            self.fitted_ = True
+            return self
+
+        def predict(self, X, metadata=None):
+            if self.on_predict:
+                assert metadata is not None
+            return np.ones(len(X))
+
+    X, y = regression_dataset
+    pipeline = make_pipeline(
+        ConsumesMetadata(on_fit=True, on_predict=True)
+        .set_fit_request(metadata=True)
+        .set_predict_request(metadata=True)
+    )
+
+    pipeline.fit(X, y, metadata="test")
+    frozen = FrozenEstimator(pipeline)
+    pipeline.predict(X, metadata="test")
+    frozen.predict(X, metadata="test")
+
+    frozen["consumesmetadata"].set_predict_request(metadata=False)
+    with pytest.raises(
+        TypeError,
+        match=re.escape(
+            "Pipeline.predict got unexpected argument(s) {'metadata'}, which are not "
+            "routed to any object."
+        ),
+    ):
+        frozen.predict(X, metadata="test")
+
+    frozen["consumesmetadata"].set_predict_request(metadata=None)
+    with pytest.raises(UnsetMetadataPassedError):
+        frozen.predict(X, metadata="test")
+
+
+def test_composite_fit(classification_dataset):
+    """Test that calling fit_transform and fit_predict doesn't call fit."""
+
+    class Estimator(BaseEstimator):
+        def fit(self, X, y):
+            try:
+                self._fit_counter += 1
+            except AttributeError:
+                self._fit_counter = 1
+            return self
+
+        def fit_transform(self, X, y=None):
+            # only here to test that it doesn't get called
+            ...  # pragma: no cover
+
+        def fit_predict(self, X, y=None):
+            # only here to test that it doesn't get called
+            ...  # pragma: no cover
+
+    X, y = classification_dataset
+    est = Estimator().fit(X, y)
+    frozen = FrozenEstimator(est)
+
+    with pytest.raises(AttributeError):
+        frozen.fit_predict(X, y)
+    with pytest.raises(AttributeError):
+        frozen.fit_transform(X, y)
+
+    assert frozen._fit_counter == 1
+
+
+def test_clone_frozen(regression_dataset):
+    """Test that cloning a frozen estimator keeps the frozen state."""
+    X, y = regression_dataset
+    estimator = LinearRegression().fit(X, y)
+    frozen = FrozenEstimator(estimator)
+    cloned = clone(frozen)
+    assert cloned.estimator is estimator
+
+
+def test_check_is_fitted(regression_dataset):
+    """Test that check_is_fitted works on frozen estimators."""
+    X, y = regression_dataset
+
+    estimator = LinearRegression()
+    frozen = FrozenEstimator(estimator)
+    with pytest.raises(NotFittedError):
+        check_is_fitted(frozen)
+
+    estimator = LinearRegression().fit(X, y)
+    frozen = FrozenEstimator(estimator)
+    check_is_fitted(frozen)
+
+
+def test_frozen_tags():
+    """Test that frozen estimators have the same tags as the original estimator
+    except for the skip_test tag."""
+
+    class Estimator(BaseEstimator):
+        def __sklearn_tags__(self):
+            tags = super().__sklearn_tags__()
+            tags.input_tags.categorical = True
+            return tags
+
+    estimator = Estimator()
+    frozen = FrozenEstimator(estimator)
+    frozen_tags = frozen.__sklearn_tags__()
+    estimator_tags = estimator.__sklearn_tags__()
+
+    assert frozen_tags._skip_test is True
+    assert estimator_tags._skip_test is False
+
+    assert estimator_tags.input_tags.categorical is True
+    assert frozen_tags.input_tags.categorical is True
+
+
+def test_frozen_params():
+    """Test that FrozenEstimator only exposes the estimator parameter."""
+    est = LogisticRegression()
+    frozen = FrozenEstimator(est)
+
+    with pytest.raises(ValueError, match="You cannot set parameters of the inner"):
+        frozen.set_params(estimator__C=1)
+
+    assert frozen.get_params() == {"estimator": est}
+
+    other_est = LocalOutlierFactor()
+    frozen.set_params(estimator=other_est)
+    assert frozen.get_params() == {"estimator": other_est}
diff --git a/sklearn/gaussian_process/__init__.py b/sklearn/gaussian_process/__init__.py
index 62ea8216deab2..9fafaf67e4ed0 100644
--- a/sklearn/gaussian_process/__init__.py
+++ b/sklearn/gaussian_process/__init__.py
@@ -1,19 +1,10 @@
-# -*- coding: utf-8 -*-
+"""Gaussian process based regression and classification."""
 
-# Author: Jan Hendrik Metzen 
-#         Vincent Dubourg 
-#         (mostly translation, see implementation details)
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-"""
-The :mod:`sklearn.gaussian_process` module implements Gaussian Process
-based regression and classification.
-"""
-
-from ._gpr import GaussianProcessRegressor
-from ._gpc import GaussianProcessClassifier
 from . import kernels
+from ._gpc import GaussianProcessClassifier
+from ._gpr import GaussianProcessRegressor
 
-
-__all__ = ['GaussianProcessRegressor', 'GaussianProcessClassifier',
-           'kernels']
+__all__ = ["GaussianProcessClassifier", "GaussianProcessRegressor", "kernels"]
diff --git a/sklearn/gaussian_process/_gpc.py b/sklearn/gaussian_process/_gpc.py
index e6fe3eb26df49..0ecceb47de905 100644
--- a/sklearn/gaussian_process/_gpc.py
+++ b/sklearn/gaussian_process/_gpc.py
@@ -1,26 +1,25 @@
 """Gaussian processes classification."""
 
-# Authors: Jan Hendrik Metzen 
-#
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
+from numbers import Integral
 from operator import itemgetter
 
 import numpy as np
-from scipy.linalg import cholesky, cho_solve, solve
 import scipy.optimize
+from scipy.linalg import cho_solve, cholesky, solve
 from scipy.special import erf, expit
 
-from ..base import BaseEstimator, ClassifierMixin, clone
-from .kernels \
-    import RBF, CompoundKernel, ConstantKernel as C
-from ..utils.validation import check_is_fitted
+from ..base import BaseEstimator, ClassifierMixin, _fit_context, clone
+from ..multiclass import OneVsOneClassifier, OneVsRestClassifier
+from ..preprocessing import LabelEncoder
 from ..utils import check_random_state
+from ..utils._param_validation import Interval, StrOptions
 from ..utils.optimize import _check_optimize_result
-from ..preprocessing import LabelEncoder
-from ..multiclass import OneVsRestClassifier, OneVsOneClassifier
-from ..utils.validation import _deprecate_positional_args
-
+from ..utils.validation import check_is_fitted, validate_data
+from .kernels import RBF, CompoundKernel, Kernel
+from .kernels import ConstantKernel as C
 
 # Values required for approximating the logistic sigmoid by
 # error functions. coefs are obtained via:
@@ -29,16 +28,15 @@
 # A = (erf(np.dot(x, self.lambdas)) + 1) / 2
 # coefs = lstsq(A, b)[0]
 LAMBDAS = np.array([0.41, 0.4, 0.37, 0.44, 0.39])[:, np.newaxis]
-COEFS = np.array([-1854.8214151, 3516.89893646, 221.29346712,
-                  128.12323805, -2010.49422654])[:, np.newaxis]
+COEFS = np.array(
+    [-1854.8214151, 3516.89893646, 221.29346712, 128.12323805, -2010.49422654]
+)[:, np.newaxis]
 
 
 class _BinaryGaussianProcessClassifierLaplace(BaseEstimator):
     """Binary Gaussian process classification based on Laplace approximation.
 
-    The implementation is based on Algorithm 3.1, 3.2, and 5.1 of
-    ``Gaussian Processes for Machine Learning'' (GPML) by Rasmussen and
-    Williams.
+    The implementation is based on Algorithm 3.1, 3.2, and 5.1 from [RW2006]_.
 
     Internally, the Laplace approximation is used for approximating the
     non-Gaussian posterior by a Gaussian.
@@ -111,7 +109,7 @@ def optimizer(obj_func, initial_theta, bounds):
     random_state : int, RandomState instance or None, default=None
         Determines random number generation used to initialize the centers.
         Pass an int for reproducible results across multiple function calls.
-        See :term: `Glossary `.
+        See :term:`Glossary `.
 
     Attributes
     ----------
@@ -144,11 +142,24 @@ def optimizer(obj_func, initial_theta, bounds):
     log_marginal_likelihood_value_ : float
         The log-marginal-likelihood of ``self.kernel_.theta``
 
+    References
+    ----------
+    .. [RW2006] `Carl E. Rasmussen and Christopher K.I. Williams,
+       "Gaussian Processes for Machine Learning",
+       MIT Press 2006 `_
     """
-    @_deprecate_positional_args
-    def __init__(self, kernel=None, *, optimizer="fmin_l_bfgs_b",
-                 n_restarts_optimizer=0, max_iter_predict=100,
-                 warm_start=False, copy_X_train=True, random_state=None):
+
+    def __init__(
+        self,
+        kernel=None,
+        *,
+        optimizer="fmin_l_bfgs_b",
+        n_restarts_optimizer=0,
+        max_iter_predict=100,
+        warm_start=False,
+        copy_X_train=True,
+        random_state=None,
+    ):
         self.kernel = kernel
         self.optimizer = optimizer
         self.n_restarts_optimizer = n_restarts_optimizer
@@ -158,7 +169,7 @@ def __init__(self, kernel=None, *, optimizer="fmin_l_bfgs_b",
         self.random_state = random_state
 
     def fit(self, X, y):
-        """Fit Gaussian process classification model
+        """Fit Gaussian process classification model.
 
         Parameters
         ----------
@@ -166,15 +177,16 @@ def fit(self, X, y):
             Feature vectors or other representations of training data.
 
         y : array-like of shape (n_samples,)
-            Target values, must be binary
+            Target values, must be binary.
 
         Returns
         -------
         self : returns an instance of self.
         """
         if self.kernel is None:  # Use an RBF kernel as default
-            self.kernel_ = C(1.0, constant_value_bounds="fixed") \
-                * RBF(1.0, length_scale_bounds="fixed")
+            self.kernel_ = C(1.0, constant_value_bounds="fixed") * RBF(
+                1.0, length_scale_bounds="fixed"
+            )
         else:
             self.kernel_ = clone(self.kernel)
 
@@ -188,13 +200,16 @@ def fit(self, X, y):
         self.y_train_ = label_encoder.fit_transform(y)
         self.classes_ = label_encoder.classes_
         if self.classes_.size > 2:
-            raise ValueError("%s supports only binary classification. "
-                             "y contains classes %s"
-                             % (self.__class__.__name__, self.classes_))
+            raise ValueError(
+                "%s supports only binary classification. y contains classes %s"
+                % (self.__class__.__name__, self.classes_)
+            )
         elif self.classes_.size == 1:
-            raise ValueError("{0:s} requires 2 classes; got {1:d} class"
-                             .format(self.__class__.__name__,
-                                     self.classes_.size))
+            raise ValueError(
+                "{0:s} requires 2 classes; got {1:d} class".format(
+                    self.__class__.__name__, self.classes_.size
+                )
+            )
 
         if self.optimizer is not None and self.kernel_.n_dims > 0:
             # Choose hyperparameters based on maximizing the log-marginal
@@ -202,16 +217,18 @@ def fit(self, X, y):
             def obj_func(theta, eval_gradient=True):
                 if eval_gradient:
                     lml, grad = self.log_marginal_likelihood(
-                        theta, eval_gradient=True, clone_kernel=False)
+                        theta, eval_gradient=True, clone_kernel=False
+                    )
                     return -lml, -grad
                 else:
-                    return -self.log_marginal_likelihood(theta,
-                                                         clone_kernel=False)
+                    return -self.log_marginal_likelihood(theta, clone_kernel=False)
 
             # First optimize starting from theta specified in kernel
-            optima = [self._constrained_optimization(obj_func,
-                                                     self.kernel_.theta,
-                                                     self.kernel_.bounds)]
+            optima = [
+                self._constrained_optimization(
+                    obj_func, self.kernel_.theta, self.kernel_.bounds
+                )
+            ]
 
             # Additional runs are performed from log-uniform chosen initial
             # theta
@@ -219,14 +236,14 @@ def obj_func(theta, eval_gradient=True):
                 if not np.isfinite(self.kernel_.bounds).all():
                     raise ValueError(
                         "Multiple optimizer restarts (n_restarts_optimizer>0) "
-                        "requires that all bounds are finite.")
+                        "requires that all bounds are finite."
+                    )
                 bounds = self.kernel_.bounds
                 for iteration in range(self.n_restarts_optimizer):
-                    theta_initial = np.exp(self.rng.uniform(bounds[:, 0],
-                                                            bounds[:, 1]))
+                    theta_initial = np.exp(self.rng.uniform(bounds[:, 0], bounds[:, 1]))
                     optima.append(
-                        self._constrained_optimization(obj_func, theta_initial,
-                                                       bounds))
+                        self._constrained_optimization(obj_func, theta_initial, bounds)
+                    )
             # Select result from run with minimal (negative) log-marginal
             # likelihood
             lml_values = list(map(itemgetter(1), optima))
@@ -235,15 +252,17 @@ def obj_func(theta, eval_gradient=True):
 
             self.log_marginal_likelihood_value_ = -np.min(lml_values)
         else:
-            self.log_marginal_likelihood_value_ = \
-                self.log_marginal_likelihood(self.kernel_.theta)
+            self.log_marginal_likelihood_value_ = self.log_marginal_likelihood(
+                self.kernel_.theta
+            )
 
         # Precompute quantities required for predictions which are independent
         # of actual query points
         K = self.kernel_(self.X_train_)
 
-        _, (self.pi_, self.W_sr_, self.L_, _, _) = \
-            self._posterior_mode(K, return_temporaries=True)
+        _, (self.pi_, self.W_sr_, self.L_, _, _) = self._posterior_mode(
+            K, return_temporaries=True
+        )
 
         return self
 
@@ -287,12 +306,9 @@ def predict_proba(self, X):
         """
         check_is_fitted(self)
 
-        # Based on Algorithm 3.2 of GPML
-        K_star = self.kernel_(self.X_train_, X)  # K_star =k(x_star)
-        f_star = K_star.T.dot(self.y_train_ - self.pi_)  # Line 4
-        v = solve(self.L_, self.W_sr_[:, np.newaxis] * K_star)  # Line 5
-        # Line 6 (compute np.diag(v.T.dot(v)) via einsum)
-        var_f_star = self.kernel_.diag(X) - np.einsum("ij,ij->j", v, v)
+        # Compute the mean and variance of the latent function
+        # (Lines 4-6 of Algorithm 3.2 of GPML)
+        latent_mean, latent_var = self.latent_mean_and_variance(X)
 
         # Line 7:
         # Approximate \int log(z) * N(z | f_star, var_f_star)
@@ -301,17 +317,20 @@ def predict_proba(self, X):
         # sigmoid by a linear combination of 5 error functions.
         # For information on how this integral can be computed see
         # blitiri.blogspot.de/2012/11/gaussian-integral-of-error-function.html
-        alpha = 1 / (2 * var_f_star)
-        gamma = LAMBDAS * f_star
-        integrals = np.sqrt(np.pi / alpha) \
-            * erf(gamma * np.sqrt(alpha / (alpha + LAMBDAS**2))) \
-            / (2 * np.sqrt(var_f_star * 2 * np.pi))
-        pi_star = (COEFS * integrals).sum(axis=0) + .5 * COEFS.sum()
+        alpha = 1 / (2 * latent_var)
+        gamma = LAMBDAS * latent_mean
+        integrals = (
+            np.sqrt(np.pi / alpha)
+            * erf(gamma * np.sqrt(alpha / (alpha + LAMBDAS**2)))
+            / (2 * np.sqrt(latent_var * 2 * np.pi))
+        )
+        pi_star = (COEFS * integrals).sum(axis=0) + 0.5 * COEFS.sum()
 
         return np.vstack((1 - pi_star, pi_star)).T
 
-    def log_marginal_likelihood(self, theta=None, eval_gradient=False,
-                                clone_kernel=True):
+    def log_marginal_likelihood(
+        self, theta=None, eval_gradient=False, clone_kernel=True
+    ):
         """Returns log-marginal likelihood of theta for training data.
 
         Parameters
@@ -343,8 +362,7 @@ def log_marginal_likelihood(self, theta=None, eval_gradient=False,
         """
         if theta is None:
             if eval_gradient:
-                raise ValueError(
-                    "Gradient can only be evaluated for theta!=None")
+                raise ValueError("Gradient can only be evaluated for theta!=None")
             return self.log_marginal_likelihood_value_
 
         if clone_kernel:
@@ -360,8 +378,7 @@ def log_marginal_likelihood(self, theta=None, eval_gradient=False,
 
         # Compute log-marginal-likelihood Z and also store some temporaries
         # which can be reused for computing Z's gradient
-        Z, (pi, W_sr, L, b, a) = \
-            self._posterior_mode(K, return_temporaries=True)
+        Z, (pi, W_sr, L, b, a) = self._posterior_mode(K, return_temporaries=True)
 
         if not eval_gradient:
             return Z
@@ -372,13 +389,16 @@ def log_marginal_likelihood(self, theta=None, eval_gradient=False,
         R = W_sr[:, np.newaxis] * cho_solve((L, True), np.diag(W_sr))  # Line 7
         C = solve(L, W_sr[:, np.newaxis] * K)  # Line 8
         # Line 9: (use einsum to compute np.diag(C.T.dot(C))))
-        s_2 = -0.5 * (np.diag(K) - np.einsum('ij, ij -> j', C, C)) \
-            * (pi * (1 - pi) * (1 - 2 * pi))  # third derivative
+        s_2 = (
+            -0.5
+            * (np.diag(K) - np.einsum("ij, ij -> j", C, C))
+            * (pi * (1 - pi) * (1 - 2 * pi))
+        )  # third derivative
 
         for j in range(d_Z.shape[0]):
-            C = K_gradient[:, :, j]   # Line 11
+            C = K_gradient[:, :, j]  # Line 11
             # Line 12: (R.T.ravel().dot(C.ravel()) = np.trace(R.dot(C)))
-            s_1 = .5 * a.T.dot(C).dot(a) - .5 * R.T.ravel().dot(C.ravel())
+            s_1 = 0.5 * a.T.dot(C).dot(a) - 0.5 * R.T.ravel().dot(C.ravel())
 
             b = C.dot(self.y_train_ - pi)  # Line 13
             s_3 = b - K.dot(R.dot(b))  # Line 14
@@ -387,6 +407,39 @@ def log_marginal_likelihood(self, theta=None, eval_gradient=False,
 
         return Z, d_Z
 
+    def latent_mean_and_variance(self, X):
+        """Compute the mean and variance of the latent function values.
+
+        Based on algorithm 3.2 of [RW2006]_, this function returns the latent
+        mean (Line 4) and variance (Line 6) of the Gaussian process
+        classification model.
+
+        Note that this function is only supported for binary classification.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features) or list of object
+            Query points where the GP is evaluated for classification.
+
+        Returns
+        -------
+        latent_mean : array-like of shape (n_samples,)
+            Mean of the latent function values at the query points.
+
+        latent_var : array-like of shape (n_samples,)
+            Variance of the latent function values at the query points.
+        """
+        check_is_fitted(self)
+
+        # Based on Algorithm 3.2 of GPML
+        K_star = self.kernel_(self.X_train_, X)  # K_star =k(x_star)
+        latent_mean = K_star.T.dot(self.y_train_ - self.pi_)  # Line 4
+        v = solve(self.L_, self.W_sr_[:, np.newaxis] * K_star)  # Line 5
+        # Line 6 (compute np.diag(v.T.dot(v)) via einsum)
+        latent_var = self.kernel_.diag(X) - np.einsum("ij,ij->j", v, v)
+
+        return latent_mean, latent_var
+
     def _posterior_mode(self, K, return_temporaries=False):
         """Mode-finding for binary Laplace GPC and fixed kernel.
 
@@ -398,8 +451,11 @@ def _posterior_mode(self, K, return_temporaries=False):
 
         # If warm_start are enabled, we reuse the last solution for the
         # posterior mode as initialization; otherwise, we initialize with 0
-        if self.warm_start and hasattr(self, "f_cached") \
-           and self.f_cached.shape == self.y_train_.shape:
+        if (
+            self.warm_start
+            and hasattr(self, "f_cached")
+            and self.f_cached.shape == self.y_train_.shape
+        ):
             f = self.f_cached
         else:
             f = np.zeros_like(self.y_train_, dtype=np.float64)
@@ -424,9 +480,11 @@ def _posterior_mode(self, K, return_temporaries=False):
 
             # Line 10: Compute log marginal likelihood in loop and use as
             #          convergence criterion
-            lml = -0.5 * a.T.dot(f) \
-                - np.log1p(np.exp(-(self.y_train_ * 2 - 1) * f)).sum() \
+            lml = (
+                -0.5 * a.T.dot(f)
+                - np.log1p(np.exp(-(self.y_train_ * 2 - 1) * f)).sum()
                 - np.log(np.diag(L)).sum()
+            )
             # Check if we have converged (log marginal likelihood does
             # not decrease)
             # XXX: more complex convergence criterion
@@ -443,13 +501,12 @@ def _posterior_mode(self, K, return_temporaries=False):
     def _constrained_optimization(self, obj_func, initial_theta, bounds):
         if self.optimizer == "fmin_l_bfgs_b":
             opt_res = scipy.optimize.minimize(
-                obj_func, initial_theta, method="L-BFGS-B", jac=True,
-                bounds=bounds)
+                obj_func, initial_theta, method="L-BFGS-B", jac=True, bounds=bounds
+            )
             _check_optimize_result("lbfgs", opt_res)
             theta_opt, func_min = opt_res.x, opt_res.fun
         elif callable(self.optimizer):
-            theta_opt, func_min = \
-                self.optimizer(obj_func, initial_theta, bounds=bounds)
+            theta_opt, func_min = self.optimizer(obj_func, initial_theta, bounds=bounds)
         else:
             raise ValueError("Unknown optimizer %s." % self.optimizer)
 
@@ -459,9 +516,7 @@ def _constrained_optimization(self, obj_func, initial_theta, bounds):
 class GaussianProcessClassifier(ClassifierMixin, BaseEstimator):
     """Gaussian process classification (GPC) based on Laplace approximation.
 
-    The implementation is based on Algorithm 3.1, 3.2, and 5.1 of
-    Gaussian Processes for Machine Learning (GPML) by Rasmussen and
-    Williams.
+    The implementation is based on Algorithm 3.1, 3.2, and 5.1 from [RW2006]_.
 
     Internally, the Laplace approximation is used for approximating the
     non-Gaussian posterior by a Gaussian.
@@ -473,14 +528,17 @@ class GaussianProcessClassifier(ClassifierMixin, BaseEstimator):
 
     Read more in the :ref:`User Guide `.
 
+    .. versionadded:: 0.18
+
     Parameters
     ----------
     kernel : kernel instance, default=None
         The kernel specifying the covariance function of the GP. If None is
         passed, the kernel "1.0 * RBF(1.0)" is used as default. Note that
-        the kernel's hyperparameters are optimized during fitting.
+        the kernel's hyperparameters are optimized during fitting. Also kernel
+        cannot be a `CompoundKernel`.
 
-    optimizer : 'fmin_l_bfgs_b' or callable, default='fmin_l_bfgs_b'
+    optimizer : 'fmin_l_bfgs_b', callable or None, default='fmin_l_bfgs_b'
         Can either be one of the internally supported optimizers for optimizing
         the kernel's parameters, specified by a string, or an externally
         defined optimizer passed as a callable. If a callable is passed, it
@@ -536,7 +594,7 @@ def optimizer(obj_func, initial_theta, bounds):
     random_state : int, RandomState instance or None, default=None
         Determines random number generation used to initialize the centers.
         Pass an int for reproducible results across multiple function calls.
-        See :term: `Glossary `.
+        See :term:`Glossary `.
 
     multi_class : {'one_vs_rest', 'one_vs_one'}, default='one_vs_rest'
         Specifies how multi-class classification problems are handled.
@@ -578,6 +636,27 @@ def optimizer(obj_func, initial_theta, bounds):
     n_classes_ : int
         The number of classes in the training data
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    GaussianProcessRegressor : Gaussian process regression (GPR).
+
+    References
+    ----------
+    .. [RW2006] `Carl E. Rasmussen and Christopher K.I. Williams,
+       "Gaussian Processes for Machine Learning",
+       MIT Press 2006 `_
+
     Examples
     --------
     >>> from sklearn.datasets import load_iris
@@ -593,13 +672,35 @@ def optimizer(obj_func, initial_theta, bounds):
     array([[0.83548752, 0.03228706, 0.13222543],
            [0.79064206, 0.06525643, 0.14410151]])
 
-    .. versionadded:: 0.18
+    For a comparison of the GaussianProcessClassifier with other classifiers see:
+    :ref:`sphx_glr_auto_examples_classification_plot_classification_probability.py`.
     """
-    @_deprecate_positional_args
-    def __init__(self, kernel=None, *, optimizer="fmin_l_bfgs_b",
-                 n_restarts_optimizer=0, max_iter_predict=100,
-                 warm_start=False, copy_X_train=True, random_state=None,
-                 multi_class="one_vs_rest", n_jobs=None):
+
+    _parameter_constraints: dict = {
+        "kernel": [Kernel, None],
+        "optimizer": [StrOptions({"fmin_l_bfgs_b"}), callable, None],
+        "n_restarts_optimizer": [Interval(Integral, 0, None, closed="left")],
+        "max_iter_predict": [Interval(Integral, 1, None, closed="left")],
+        "warm_start": ["boolean"],
+        "copy_X_train": ["boolean"],
+        "random_state": ["random_state"],
+        "multi_class": [StrOptions({"one_vs_rest", "one_vs_one"})],
+        "n_jobs": [Integral, None],
+    }
+
+    def __init__(
+        self,
+        kernel=None,
+        *,
+        optimizer="fmin_l_bfgs_b",
+        n_restarts_optimizer=0,
+        max_iter_predict=100,
+        warm_start=False,
+        copy_X_train=True,
+        random_state=None,
+        multi_class="one_vs_rest",
+        n_jobs=None,
+    ):
         self.kernel = kernel
         self.optimizer = optimizer
         self.n_restarts_optimizer = n_restarts_optimizer
@@ -610,8 +711,9 @@ def __init__(self, kernel=None, *, optimizer="fmin_l_bfgs_b",
         self.multi_class = multi_class
         self.n_jobs = n_jobs
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y):
-        """Fit Gaussian process classification model
+        """Fit Gaussian process classification model.
 
         Parameters
         ----------
@@ -619,18 +721,24 @@ def fit(self, X, y):
             Feature vectors or other representations of training data.
 
         y : array-like of shape (n_samples,)
-            Target values, must be binary
+            Target values, must be binary.
 
         Returns
         -------
-        self : returns an instance of self.
+        self : object
+            Returns an instance of self.
         """
+        if isinstance(self.kernel, CompoundKernel):
+            raise ValueError("kernel cannot be a CompoundKernel")
+
         if self.kernel is None or self.kernel.requires_vector_input:
-            X, y = self._validate_data(X, y, multi_output=False,
-                                       ensure_2d=True, dtype="numeric")
+            X, y = validate_data(
+                self, X, y, multi_output=False, ensure_2d=True, dtype="numeric"
+            )
         else:
-            X, y = self._validate_data(X, y, multi_output=False,
-                                       ensure_2d=False, dtype=None)
+            X, y = validate_data(
+                self, X, y, multi_output=False, ensure_2d=False, dtype=None
+            )
 
         self.base_estimator_ = _BinaryGaussianProcessClassifierLaplace(
             kernel=self.kernel,
@@ -639,37 +747,42 @@ def fit(self, X, y):
             max_iter_predict=self.max_iter_predict,
             warm_start=self.warm_start,
             copy_X_train=self.copy_X_train,
-            random_state=self.random_state)
+            random_state=self.random_state,
+        )
 
         self.classes_ = np.unique(y)
         self.n_classes_ = self.classes_.size
         if self.n_classes_ == 1:
-            raise ValueError("GaussianProcessClassifier requires 2 or more "
-                             "distinct classes; got %d class (only class %s "
-                             "is present)"
-                             % (self.n_classes_, self.classes_[0]))
+            raise ValueError(
+                "GaussianProcessClassifier requires 2 or more "
+                "distinct classes; got %d class (only class %s "
+                "is present)" % (self.n_classes_, self.classes_[0])
+            )
         if self.n_classes_ > 2:
             if self.multi_class == "one_vs_rest":
-                self.base_estimator_ = \
-                    OneVsRestClassifier(self.base_estimator_,
-                                        n_jobs=self.n_jobs)
+                self.base_estimator_ = OneVsRestClassifier(
+                    self.base_estimator_, n_jobs=self.n_jobs
+                )
             elif self.multi_class == "one_vs_one":
-                self.base_estimator_ = \
-                    OneVsOneClassifier(self.base_estimator_,
-                                       n_jobs=self.n_jobs)
+                self.base_estimator_ = OneVsOneClassifier(
+                    self.base_estimator_, n_jobs=self.n_jobs
+                )
             else:
-                raise ValueError("Unknown multi-class mode %s"
-                                 % self.multi_class)
+                raise ValueError("Unknown multi-class mode %s" % self.multi_class)
 
         self.base_estimator_.fit(X, y)
 
         if self.n_classes_ > 2:
             self.log_marginal_likelihood_value_ = np.mean(
-                [estimator.log_marginal_likelihood()
-                 for estimator in self.base_estimator_.estimators_])
+                [
+                    estimator.log_marginal_likelihood()
+                    for estimator in self.base_estimator_.estimators_
+                ]
+            )
         else:
-            self.log_marginal_likelihood_value_ = \
+            self.log_marginal_likelihood_value_ = (
                 self.base_estimator_.log_marginal_likelihood()
+            )
 
         return self
 
@@ -684,16 +797,14 @@ def predict(self, X):
         Returns
         -------
         C : ndarray of shape (n_samples,)
-            Predicted target values for X, values are from ``classes_``
+            Predicted target values for X, values are from ``classes_``.
         """
         check_is_fitted(self)
 
         if self.kernel is None or self.kernel.requires_vector_input:
-            X = self._validate_data(X, ensure_2d=True, dtype="numeric",
-                                    reset=False)
+            X = validate_data(self, X, ensure_2d=True, dtype="numeric", reset=False)
         else:
-            X = self._validate_data(X, ensure_2d=False, dtype=None,
-                                    reset=False)
+            X = validate_data(self, X, ensure_2d=False, dtype=None, reset=False)
 
         return self.base_estimator_.predict(X)
 
@@ -714,31 +825,33 @@ def predict_proba(self, X):
         """
         check_is_fitted(self)
         if self.n_classes_ > 2 and self.multi_class == "one_vs_one":
-            raise ValueError("one_vs_one multi-class mode does not support "
-                             "predicting probability estimates. Use "
-                             "one_vs_rest mode instead.")
+            raise ValueError(
+                "one_vs_one multi-class mode does not support "
+                "predicting probability estimates. Use "
+                "one_vs_rest mode instead."
+            )
 
         if self.kernel is None or self.kernel.requires_vector_input:
-            X = self._validate_data(X, ensure_2d=True, dtype="numeric",
-                                    reset=False)
+            X = validate_data(self, X, ensure_2d=True, dtype="numeric", reset=False)
         else:
-            X = self._validate_data(X, ensure_2d=False, dtype=None,
-                                    reset=False)
+            X = validate_data(self, X, ensure_2d=False, dtype=None, reset=False)
 
         return self.base_estimator_.predict_proba(X)
 
     @property
     def kernel_(self):
+        """Return the kernel of the base estimator."""
         if self.n_classes_ == 2:
             return self.base_estimator_.kernel_
         else:
             return CompoundKernel(
-                [estimator.kernel_
-                 for estimator in self.base_estimator_.estimators_])
+                [estimator.kernel_ for estimator in self.base_estimator_.estimators_]
+            )
 
-    def log_marginal_likelihood(self, theta=None, eval_gradient=False,
-                                clone_kernel=True):
-        """Returns log-marginal likelihood of theta for training data.
+    def log_marginal_likelihood(
+        self, theta=None, eval_gradient=False, clone_kernel=True
+    ):
+        """Return log-marginal likelihood of theta for training data.
 
         In the case of multi-class classification, the mean log-marginal
         likelihood of the one-versus-rest classifiers are returned.
@@ -777,35 +890,84 @@ def log_marginal_likelihood(self, theta=None, eval_gradient=False,
 
         if theta is None:
             if eval_gradient:
-                raise ValueError(
-                    "Gradient can only be evaluated for theta!=None")
+                raise ValueError("Gradient can only be evaluated for theta!=None")
             return self.log_marginal_likelihood_value_
 
         theta = np.asarray(theta)
         if self.n_classes_ == 2:
             return self.base_estimator_.log_marginal_likelihood(
-                theta, eval_gradient, clone_kernel=clone_kernel)
+                theta, eval_gradient, clone_kernel=clone_kernel
+            )
         else:
             if eval_gradient:
                 raise NotImplementedError(
                     "Gradient of log-marginal-likelihood not implemented for "
-                    "multi-class GPC.")
+                    "multi-class GPC."
+                )
             estimators = self.base_estimator_.estimators_
             n_dims = estimators[0].kernel_.n_dims
             if theta.shape[0] == n_dims:  # use same theta for all sub-kernels
                 return np.mean(
-                    [estimator.log_marginal_likelihood(
-                        theta, clone_kernel=clone_kernel)
-                     for i, estimator in enumerate(estimators)])
+                    [
+                        estimator.log_marginal_likelihood(
+                            theta, clone_kernel=clone_kernel
+                        )
+                        for i, estimator in enumerate(estimators)
+                    ]
+                )
             elif theta.shape[0] == n_dims * self.classes_.shape[0]:
                 # theta for compound kernel
                 return np.mean(
-                    [estimator.log_marginal_likelihood(
-                        theta[n_dims * i:n_dims * (i + 1)],
-                        clone_kernel=clone_kernel)
-                     for i, estimator in enumerate(estimators)])
+                    [
+                        estimator.log_marginal_likelihood(
+                            theta[n_dims * i : n_dims * (i + 1)],
+                            clone_kernel=clone_kernel,
+                        )
+                        for i, estimator in enumerate(estimators)
+                    ]
+                )
             else:
-                raise ValueError("Shape of theta must be either %d or %d. "
-                                 "Obtained theta with shape %d."
-                                 % (n_dims, n_dims * self.classes_.shape[0],
-                                    theta.shape[0]))
+                raise ValueError(
+                    "Shape of theta must be either %d or %d. "
+                    "Obtained theta with shape %d."
+                    % (n_dims, n_dims * self.classes_.shape[0], theta.shape[0])
+                )
+
+    def latent_mean_and_variance(self, X):
+        """Compute the mean and variance of the latent function.
+
+        Based on algorithm 3.2 of [RW2006]_, this function returns the latent
+        mean (Line 4) and variance (Line 6) of the Gaussian process
+        classification model.
+
+        Note that this function is only supported for binary classification.
+
+        .. versionadded:: 1.7
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features) or list of object
+            Query points where the GP is evaluated for classification.
+
+        Returns
+        -------
+        latent_mean : array-like of shape (n_samples,)
+            Mean of the latent function values at the query points.
+
+        latent_var : array-like of shape (n_samples,)
+            Variance of the latent function values at the query points.
+        """
+        if self.n_classes_ > 2:
+            raise ValueError(
+                "Returning the mean and variance of the latent function f "
+                "is only supported for binary classification, received "
+                f"{self.n_classes_} classes."
+            )
+        check_is_fitted(self)
+
+        if self.kernel is None or self.kernel.requires_vector_input:
+            X = validate_data(self, X, ensure_2d=True, dtype="numeric", reset=False)
+        else:
+            X = validate_data(self, X, ensure_2d=False, dtype=None, reset=False)
+
+        return self.base_estimator_.latent_mean_and_variance(X)
diff --git a/sklearn/gaussian_process/_gpr.py b/sklearn/gaussian_process/_gpr.py
index b4ab0441efc71..d56e7735be787 100644
--- a/sklearn/gaussian_process/_gpr.py
+++ b/sklearn/gaussian_process/_gpr.py
@@ -1,40 +1,46 @@
 """Gaussian processes regression."""
 
-# Authors: Jan Hendrik Metzen 
-# Modified by: Pete Green 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import warnings
+from numbers import Integral, Real
 from operator import itemgetter
 
 import numpy as np
-from scipy.linalg import cholesky, cho_solve, solve_triangular
 import scipy.optimize
+from scipy.linalg import cho_solve, cholesky, solve_triangular
 
-from ..base import BaseEstimator, RegressorMixin, clone
-from ..base import MultiOutputMixin
-from .kernels import RBF, ConstantKernel as C
+from ..base import BaseEstimator, MultiOutputMixin, RegressorMixin, _fit_context, clone
+from ..preprocessing._data import _handle_zeros_in_scale
 from ..utils import check_random_state
+from ..utils._param_validation import Interval, StrOptions
 from ..utils.optimize import _check_optimize_result
-from ..utils.validation import _deprecate_positional_args
+from ..utils.validation import validate_data
+from .kernels import RBF, Kernel
+from .kernels import ConstantKernel as C
 
+GPR_CHOLESKY_LOWER = True
 
-class GaussianProcessRegressor(MultiOutputMixin,
-                               RegressorMixin, BaseEstimator):
+
+class GaussianProcessRegressor(MultiOutputMixin, RegressorMixin, BaseEstimator):
     """Gaussian process regression (GPR).
 
-    The implementation is based on Algorithm 2.1 of Gaussian Processes
-    for Machine Learning (GPML) by Rasmussen and Williams.
+    The implementation is based on Algorithm 2.1 of [RW2006]_.
 
     In addition to standard scikit-learn estimator API,
-    GaussianProcessRegressor:
+    :class:`GaussianProcessRegressor`:
+
+    * allows prediction without prior fitting (based on the GP prior)
+    * provides an additional method `sample_y(X)`, which evaluates samples
+      drawn from the GPR (prior or posterior) at given inputs
+    * exposes a method `log_marginal_likelihood(theta)`, which can be used
+      externally for other ways of selecting hyperparameters, e.g., via
+      Markov chain Monte Carlo.
 
-       * allows prediction without prior fitting (based on the GP prior)
-       * provides an additional method sample_y(X), which evaluates samples
-         drawn from the GPR (prior or posterior) at given inputs
-       * exposes a method log_marginal_likelihood(theta), which can be used
-         externally for other ways of selecting hyperparameters, e.g., via
-         Markov chain Monte Carlo.
+    To learn the difference between a point-estimate approach vs. a more
+    Bayesian modelling approach, refer to the example entitled
+    :ref:`sphx_glr_auto_examples_gaussian_process_plot_compare_gpr_krr.py`.
 
     Read more in the :ref:`User Guide `.
 
@@ -44,7 +50,7 @@ class GaussianProcessRegressor(MultiOutputMixin,
     ----------
     kernel : kernel instance, default=None
         The kernel specifying the covariance function of the GP. If None is
-        passed, the kernel ``ConstantKernel(1.0, constant_value_bounds="fixed"
+        passed, the kernel ``ConstantKernel(1.0, constant_value_bounds="fixed")
         * RBF(1.0, length_scale_bounds="fixed")`` is used as default. Note that
         the kernel hyperparameters are optimized during fitting unless the
         bounds are marked as "fixed".
@@ -59,17 +65,20 @@ class GaussianProcessRegressor(MultiOutputMixin,
         have the same number of entries as the data used for fitting and is
         used as datapoint-dependent noise level. Allowing to specify the
         noise level directly as a parameter is mainly for convenience and
-        for consistency with Ridge.
+        for consistency with :class:`~sklearn.linear_model.Ridge`.
+        For an example illustrating how the alpha parameter controls
+        the noise variance in Gaussian Process Regression, see
+        :ref:`sphx_glr_auto_examples_gaussian_process_plot_gpr_noisy_targets.py`.
 
-    optimizer : "fmin_l_bfgs_b" or callable, default="fmin_l_bfgs_b"
+    optimizer : "fmin_l_bfgs_b", callable or None, default="fmin_l_bfgs_b"
         Can either be one of the internally supported optimizers for optimizing
         the kernel's parameters, specified by a string, or an externally
         defined optimizer passed as a callable. If a callable is passed, it
         must have the signature::
 
             def optimizer(obj_func, initial_theta, bounds):
-                # * 'obj_func' is the objective function to be minimized, which
-                #   takes the hyperparameters theta as parameter and an
+                # * 'obj_func': the objective function to be minimized, which
+                #   takes the hyperparameters theta as a parameter and an
                 #   optional flag eval_gradient, which determines if the
                 #   gradient is returned additionally to the function value
                 # * 'initial_theta': the initial value for theta, which can be
@@ -80,11 +89,9 @@ def optimizer(obj_func, initial_theta, bounds):
                 # the corresponding value of the target function.
                 return theta_opt, func_min
 
-        Per default, the 'L-BGFS-B' algorithm from scipy.optimize.minimize
+        Per default, the L-BFGS-B algorithm from `scipy.optimize.minimize`
         is used. If None is passed, the kernel's parameters are kept fixed.
-        Available internal optimizers are::
-
-            'fmin_l_bfgs_b'
+        Available internal optimizers are: `{'fmin_l_bfgs_b'}`.
 
     n_restarts_optimizer : int, default=0
         The number of restarts of the optimizer for finding the kernel's
@@ -92,15 +99,15 @@ def optimizer(obj_func, initial_theta, bounds):
         of the optimizer is performed from the kernel's initial parameters,
         the remaining ones (if any) from thetas sampled log-uniform randomly
         from the space of allowed theta-values. If greater than 0, all bounds
-        must be finite. Note that n_restarts_optimizer == 0 implies that one
+        must be finite. Note that `n_restarts_optimizer == 0` implies that one
         run is performed.
 
     normalize_y : bool, default=False
-        Whether the target values y are normalized, the mean and variance of
-        the target values are set equal to 0 and 1 respectively. This is
-        recommended for cases where zero-mean, unit-variance priors are used.
-        Note that, in this implementation, the normalisation is reversed
-        before the GP predictions are reported.
+        Whether or not to normalize the target values `y` by removing the mean
+        and scaling to unit-variance. This is recommended for cases where
+        zero-mean, unit-variance priors are used. Note that, in this
+        implementation, the normalisation is reversed before the GP predictions
+        are reported.
 
         .. versionchanged:: 0.23
 
@@ -110,10 +117,18 @@ def optimizer(obj_func, initial_theta, bounds):
         which might cause predictions to change if the data is modified
         externally.
 
+    n_targets : int, default=None
+        The number of dimensions of the target values. Used to decide the number
+        of outputs when sampling from the prior distributions (i.e. calling
+        :meth:`sample_y` before :meth:`fit`). This parameter is ignored once
+        :meth:`fit` has been called.
+
+        .. versionadded:: 1.3
+
     random_state : int, RandomState instance or None, default=None
         Determines random number generation used to initialize the centers.
         Pass an int for reproducible results across multiple function calls.
-        See :term: `Glossary `.
+        See :term:`Glossary `.
 
     Attributes
     ----------
@@ -122,20 +137,42 @@ def optimizer(obj_func, initial_theta, bounds):
         required for prediction).
 
     y_train_ : array-like of shape (n_samples,) or (n_samples, n_targets)
-        Target values in training data (also required for prediction)
+        Target values in training data (also required for prediction).
 
     kernel_ : kernel instance
         The kernel used for prediction. The structure of the kernel is the
-        same as the one passed as parameter but with optimized hyperparameters
+        same as the one passed as parameter but with optimized hyperparameters.
 
     L_ : array-like of shape (n_samples, n_samples)
-        Lower-triangular Cholesky decomposition of the kernel in ``X_train_``
+        Lower-triangular Cholesky decomposition of the kernel in ``X_train_``.
 
     alpha_ : array-like of shape (n_samples,)
-        Dual coefficients of training data points in kernel space
+        Dual coefficients of training data points in kernel space.
 
     log_marginal_likelihood_value_ : float
-        The log-marginal-likelihood of ``self.kernel_.theta``
+        The log-marginal-likelihood of ``self.kernel_.theta``.
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    GaussianProcessClassifier : Gaussian process classification (GPC)
+        based on Laplace approximation.
+
+    References
+    ----------
+    .. [RW2006] `Carl E. Rasmussen and Christopher K.I. Williams,
+       "Gaussian Processes for Machine Learning",
+       MIT Press 2006 `_
 
     Examples
     --------
@@ -149,21 +186,42 @@ def optimizer(obj_func, initial_theta, bounds):
     >>> gpr.score(X, y)
     0.3680...
     >>> gpr.predict(X[:2,:], return_std=True)
-    (array([653.0..., 592.1...]), array([316.6..., 316.6...]))
-
+    (array([653.0, 592.1]), array([316.6, 316.6]))
     """
-    @_deprecate_positional_args
-    def __init__(self, kernel=None, *, alpha=1e-10,
-                 optimizer="fmin_l_bfgs_b", n_restarts_optimizer=0,
-                 normalize_y=False, copy_X_train=True, random_state=None):
+
+    _parameter_constraints: dict = {
+        "kernel": [None, Kernel],
+        "alpha": [Interval(Real, 0, None, closed="left"), np.ndarray],
+        "optimizer": [StrOptions({"fmin_l_bfgs_b"}), callable, None],
+        "n_restarts_optimizer": [Interval(Integral, 0, None, closed="left")],
+        "normalize_y": ["boolean"],
+        "copy_X_train": ["boolean"],
+        "n_targets": [Interval(Integral, 1, None, closed="left"), None],
+        "random_state": ["random_state"],
+    }
+
+    def __init__(
+        self,
+        kernel=None,
+        *,
+        alpha=1e-10,
+        optimizer="fmin_l_bfgs_b",
+        n_restarts_optimizer=0,
+        normalize_y=False,
+        copy_X_train=True,
+        n_targets=None,
+        random_state=None,
+    ):
         self.kernel = kernel
         self.alpha = alpha
         self.optimizer = optimizer
         self.n_restarts_optimizer = n_restarts_optimizer
         self.normalize_y = normalize_y
         self.copy_X_train = copy_X_train
+        self.n_targets = n_targets
         self.random_state = random_state
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y):
         """Fit Gaussian process regression model.
 
@@ -173,47 +231,64 @@ def fit(self, X, y):
             Feature vectors or other representations of training data.
 
         y : array-like of shape (n_samples,) or (n_samples, n_targets)
-            Target values
+            Target values.
 
         Returns
         -------
-        self : returns an instance of self.
+        self : object
+            GaussianProcessRegressor class instance.
         """
         if self.kernel is None:  # Use an RBF kernel as default
-            self.kernel_ = C(1.0, constant_value_bounds="fixed") \
-                * RBF(1.0, length_scale_bounds="fixed")
+            self.kernel_ = C(1.0, constant_value_bounds="fixed") * RBF(
+                1.0, length_scale_bounds="fixed"
+            )
         else:
             self.kernel_ = clone(self.kernel)
 
         self._rng = check_random_state(self.random_state)
 
         if self.kernel_.requires_vector_input:
-            X, y = self._validate_data(X, y, multi_output=True, y_numeric=True,
-                                       ensure_2d=True, dtype="numeric")
+            dtype, ensure_2d = "numeric", True
         else:
-            X, y = self._validate_data(X, y, multi_output=True, y_numeric=True,
-                                       ensure_2d=False, dtype=None)
+            dtype, ensure_2d = None, False
+        X, y = validate_data(
+            self,
+            X,
+            y,
+            multi_output=True,
+            y_numeric=True,
+            ensure_2d=ensure_2d,
+            dtype=dtype,
+        )
+
+        n_targets_seen = y.shape[1] if y.ndim > 1 else 1
+        if self.n_targets is not None and n_targets_seen != self.n_targets:
+            raise ValueError(
+                "The number of targets seen in `y` is different from the parameter "
+                f"`n_targets`. Got {n_targets_seen} != {self.n_targets}."
+            )
 
         # Normalize target value
         if self.normalize_y:
             self._y_train_mean = np.mean(y, axis=0)
-            self._y_train_std = np.std(y, axis=0)
+            self._y_train_std = _handle_zeros_in_scale(np.std(y, axis=0), copy=False)
 
             # Remove mean and make unit variance
             y = (y - self._y_train_mean) / self._y_train_std
 
         else:
-            self._y_train_mean = np.zeros(1)
-            self._y_train_std = 1
+            shape_y_stats = (y.shape[1],) if y.ndim == 2 else 1
+            self._y_train_mean = np.zeros(shape=shape_y_stats)
+            self._y_train_std = np.ones(shape=shape_y_stats)
 
-        if np.iterable(self.alpha) \
-           and self.alpha.shape[0] != y.shape[0]:
+        if np.iterable(self.alpha) and self.alpha.shape[0] != y.shape[0]:
             if self.alpha.shape[0] == 1:
                 self.alpha = self.alpha[0]
             else:
-                raise ValueError("alpha must be a scalar or an array"
-                                 " with same number of entries as y.(%d != %d)"
-                                 % (self.alpha.shape[0], y.shape[0]))
+                raise ValueError(
+                    "alpha must be a scalar or an array with same number of "
+                    f"entries as y. ({self.alpha.shape[0]} != {y.shape[0]})"
+                )
 
         self.X_train_ = np.copy(X) if self.copy_X_train else X
         self.y_train_ = np.copy(y) if self.copy_X_train else y
@@ -224,16 +299,20 @@ def fit(self, X, y):
             def obj_func(theta, eval_gradient=True):
                 if eval_gradient:
                     lml, grad = self.log_marginal_likelihood(
-                        theta, eval_gradient=True, clone_kernel=False)
+                        theta, eval_gradient=True, clone_kernel=False
+                    )
                     return -lml, -grad
                 else:
-                    return -self.log_marginal_likelihood(theta,
-                                                         clone_kernel=False)
+                    return -self.log_marginal_likelihood(theta, clone_kernel=False)
 
             # First optimize starting from theta specified in kernel
-            optima = [(self._constrained_optimization(obj_func,
-                                                      self.kernel_.theta,
-                                                      self.kernel_.bounds))]
+            optima = [
+                (
+                    self._constrained_optimization(
+                        obj_func, self.kernel_.theta, self.kernel_.bounds
+                    )
+                )
+            ]
 
             # Additional runs are performed from log-uniform chosen initial
             # theta
@@ -241,14 +320,14 @@ def obj_func(theta, eval_gradient=True):
                 if not np.isfinite(self.kernel_.bounds).all():
                     raise ValueError(
                         "Multiple optimizer restarts (n_restarts_optimizer>0) "
-                        "requires that all bounds are finite.")
+                        "requires that all bounds are finite."
+                    )
                 bounds = self.kernel_.bounds
                 for iteration in range(self.n_restarts_optimizer):
-                    theta_initial = \
-                        self._rng.uniform(bounds[:, 0], bounds[:, 1])
+                    theta_initial = self._rng.uniform(bounds[:, 0], bounds[:, 1])
                     optima.append(
-                        self._constrained_optimization(obj_func, theta_initial,
-                                                       bounds))
+                        self._constrained_optimization(obj_func, theta_initial, bounds)
+                    )
             # Select result from run with minimal (negative) log-marginal
             # likelihood
             lml_values = list(map(itemgetter(1), optima))
@@ -257,35 +336,41 @@ def obj_func(theta, eval_gradient=True):
 
             self.log_marginal_likelihood_value_ = -np.min(lml_values)
         else:
-            self.log_marginal_likelihood_value_ = \
-                self.log_marginal_likelihood(self.kernel_.theta,
-                                             clone_kernel=False)
+            self.log_marginal_likelihood_value_ = self.log_marginal_likelihood(
+                self.kernel_.theta, clone_kernel=False
+            )
 
         # Precompute quantities required for predictions which are independent
         # of actual query points
+        # Alg. 2.1, page 19, line 2 -> L = cholesky(K + sigma^2 I)
         K = self.kernel_(self.X_train_)
         K[np.diag_indices_from(K)] += self.alpha
         try:
-            self.L_ = cholesky(K, lower=True)  # Line 2
-            # self.L_ changed, self._K_inv needs to be recomputed
-            self._K_inv = None
+            self.L_ = cholesky(K, lower=GPR_CHOLESKY_LOWER, check_finite=False)
         except np.linalg.LinAlgError as exc:
-            exc.args = ("The kernel, %s, is not returning a "
-                        "positive definite matrix. Try gradually "
-                        "increasing the 'alpha' parameter of your "
-                        "GaussianProcessRegressor estimator."
-                        % self.kernel_,) + exc.args
+            exc.args = (
+                (
+                    f"The kernel, {self.kernel_}, is not returning a positive "
+                    "definite matrix. Try gradually increasing the 'alpha' "
+                    "parameter of your GaussianProcessRegressor estimator."
+                ),
+            ) + exc.args
             raise
-        self.alpha_ = cho_solve((self.L_, True), self.y_train_)  # Line 3
+        # Alg 2.1, page 19, line 3 -> alpha = L^T \ (L \ y)
+        self.alpha_ = cho_solve(
+            (self.L_, GPR_CHOLESKY_LOWER),
+            self.y_train_,
+            check_finite=False,
+        )
         return self
 
     def predict(self, X, return_std=False, return_cov=False):
-        """Predict using the Gaussian process regression model
+        """Predict using the Gaussian process regression model.
 
         We can also predict based on an unfitted model by using the GP prior.
-        In addition to the mean of the predictive distribution, also its
-        standard deviation (return_std=True) or covariance (return_cov=True).
-        Note that at most one of the two can be requested.
+        In addition to the mean of the predictive distribution, optionally also
+        returns its standard deviation (`return_std=True`) or covariance
+        (`return_cov=True`). Note that at most one of the two can be requested.
 
         Parameters
         ----------
@@ -302,83 +387,109 @@ def predict(self, X, return_std=False, return_cov=False):
 
         Returns
         -------
-        y_mean : ndarray of shape (n_samples, [n_output_dims])
-            Mean of predictive distribution a query points.
+        y_mean : ndarray of shape (n_samples,) or (n_samples, n_targets)
+            Mean of predictive distribution at query points.
 
-        y_std : ndarray of shape (n_samples,), optional
+        y_std : ndarray of shape (n_samples,) or (n_samples, n_targets), optional
             Standard deviation of predictive distribution at query points.
             Only returned when `return_std` is True.
 
-        y_cov : ndarray of shape (n_samples, n_samples), optional
-            Covariance of joint predictive distribution a query points.
+        y_cov : ndarray of shape (n_samples, n_samples) or \
+                (n_samples, n_samples, n_targets), optional
+            Covariance of joint predictive distribution at query points.
             Only returned when `return_cov` is True.
         """
         if return_std and return_cov:
             raise RuntimeError(
-                "Not returning standard deviation of predictions when "
-                "returning full covariance.")
+                "At most one of return_std or return_cov can be requested."
+            )
 
         if self.kernel is None or self.kernel.requires_vector_input:
-            X = self._validate_data(X, ensure_2d=True, dtype="numeric",
-                                    reset=False)
+            dtype, ensure_2d = "numeric", True
         else:
-            X = self._validate_data(X, ensure_2d=False, dtype=None,
-                                    reset=False)
+            dtype, ensure_2d = None, False
+
+        X = validate_data(self, X, ensure_2d=ensure_2d, dtype=dtype, reset=False)
 
         if not hasattr(self, "X_train_"):  # Unfitted;predict based on GP prior
             if self.kernel is None:
-                kernel = (C(1.0, constant_value_bounds="fixed") *
-                          RBF(1.0, length_scale_bounds="fixed"))
+                kernel = C(1.0, constant_value_bounds="fixed") * RBF(
+                    1.0, length_scale_bounds="fixed"
+                )
             else:
                 kernel = self.kernel
-            y_mean = np.zeros(X.shape[0])
+
+            n_targets = self.n_targets if self.n_targets is not None else 1
+            y_mean = np.zeros(shape=(X.shape[0], n_targets)).squeeze()
+
             if return_cov:
                 y_cov = kernel(X)
+                if n_targets > 1:
+                    y_cov = np.repeat(
+                        np.expand_dims(y_cov, -1), repeats=n_targets, axis=-1
+                    )
                 return y_mean, y_cov
             elif return_std:
                 y_var = kernel.diag(X)
+                if n_targets > 1:
+                    y_var = np.repeat(
+                        np.expand_dims(y_var, -1), repeats=n_targets, axis=-1
+                    )
                 return y_mean, np.sqrt(y_var)
             else:
                 return y_mean
         else:  # Predict based on GP posterior
+            # Alg 2.1, page 19, line 4 -> f*_bar = K(X_test, X_train) . alpha
             K_trans = self.kernel_(X, self.X_train_)
-            y_mean = K_trans.dot(self.alpha_)  # Line 4 (y_mean = f_star)
+            y_mean = K_trans @ self.alpha_
 
             # undo normalisation
             y_mean = self._y_train_std * y_mean + self._y_train_mean
 
+            # if y_mean has shape (n_samples, 1), reshape to (n_samples,)
+            if y_mean.ndim > 1 and y_mean.shape[1] == 1:
+                y_mean = np.squeeze(y_mean, axis=1)
+
+            # Alg 2.1, page 19, line 5 -> v = L \ K(X_test, X_train)^T
+            V = solve_triangular(
+                self.L_, K_trans.T, lower=GPR_CHOLESKY_LOWER, check_finite=False
+            )
+
             if return_cov:
-                v = cho_solve((self.L_, True), K_trans.T)  # Line 5
-                y_cov = self.kernel_(X) - K_trans.dot(v)  # Line 6
+                # Alg 2.1, page 19, line 6 -> K(X_test, X_test) - v^T. v
+                y_cov = self.kernel_(X) - V.T @ V
 
                 # undo normalisation
-                y_cov = y_cov * self._y_train_std**2
+                y_cov = np.outer(y_cov, self._y_train_std**2).reshape(*y_cov.shape, -1)
+                # if y_cov has shape (n_samples, n_samples, 1), reshape to
+                # (n_samples, n_samples)
+                if y_cov.shape[2] == 1:
+                    y_cov = np.squeeze(y_cov, axis=2)
 
                 return y_mean, y_cov
             elif return_std:
-                # cache result of K_inv computation
-                if self._K_inv is None:
-                    # compute inverse K_inv of K based on its Cholesky
-                    # decomposition L and its inverse L_inv
-                    L_inv = solve_triangular(self.L_.T,
-                                             np.eye(self.L_.shape[0]))
-                    self._K_inv = L_inv.dot(L_inv.T)
-
                 # Compute variance of predictive distribution
-                y_var = self.kernel_.diag(X)
-                y_var -= np.einsum("ij,ij->i",
-                                   np.dot(K_trans, self._K_inv), K_trans)
+                # Use einsum to avoid explicitly forming the large matrix
+                # V^T @ V just to extract its diagonal afterward.
+                y_var = self.kernel_.diag(X).copy()
+                y_var -= np.einsum("ij,ji->i", V.T, V)
 
                 # Check if any of the variances is negative because of
                 # numerical issues. If yes: set the variance to 0.
                 y_var_negative = y_var < 0
                 if np.any(y_var_negative):
-                    warnings.warn("Predicted variances smaller than 0. "
-                                  "Setting those variances to 0.")
+                    warnings.warn(
+                        "Predicted variances smaller than 0. "
+                        "Setting those variances to 0."
+                    )
                     y_var[y_var_negative] = 0.0
 
                 # undo normalisation
-                y_var = y_var * self._y_train_std**2
+                y_var = np.outer(y_var, self._y_train_std**2).reshape(*y_var.shape, -1)
+
+                # if y_var has shape (n_samples, 1), reshape to (n_samples,)
+                if y_var.shape[1] == 1:
+                    y_var = np.squeeze(y_var, axis=1)
 
                 return y_mean, np.sqrt(y_var)
             else:
@@ -389,21 +500,22 @@ def sample_y(self, X, n_samples=1, random_state=0):
 
         Parameters
         ----------
-        X : array-like of shape (n_samples, n_features) or list of object
+        X : array-like of shape (n_samples_X, n_features) or list of object
             Query points where the GP is evaluated.
 
         n_samples : int, default=1
-            The number of samples drawn from the Gaussian process
+            Number of samples drawn from the Gaussian process per query point.
 
         random_state : int, RandomState instance or None, default=0
             Determines random number generation to randomly draw samples.
             Pass an int for reproducible results across multiple function
             calls.
-            See :term: `Glossary `.
+            See :term:`Glossary `.
 
         Returns
         -------
-        y_samples : ndarray of shape (n_samples_X, [n_output_dims], n_samples)
+        y_samples : ndarray of shape (n_samples_X, n_samples), or \
+            (n_samples_X, n_targets, n_samples)
             Values of n_samples samples drawn from Gaussian process and
             evaluated at query points.
         """
@@ -413,16 +525,19 @@ def sample_y(self, X, n_samples=1, random_state=0):
         if y_mean.ndim == 1:
             y_samples = rng.multivariate_normal(y_mean, y_cov, n_samples).T
         else:
-            y_samples = \
-                [rng.multivariate_normal(y_mean[:, i], y_cov,
-                                         n_samples).T[:, np.newaxis]
-                 for i in range(y_mean.shape[1])]
+            y_samples = [
+                rng.multivariate_normal(
+                    y_mean[:, target], y_cov[..., target], n_samples
+                ).T[:, np.newaxis]
+                for target in range(y_mean.shape[1])
+            ]
             y_samples = np.hstack(y_samples)
         return y_samples
 
-    def log_marginal_likelihood(self, theta=None, eval_gradient=False,
-                                clone_kernel=True):
-        """Returns log-marginal likelihood of theta for training data.
+    def log_marginal_likelihood(
+        self, theta=None, eval_gradient=False, clone_kernel=True
+    ):
+        """Return log-marginal likelihood of theta for training data.
 
         Parameters
         ----------
@@ -452,8 +567,7 @@ def log_marginal_likelihood(self, theta=None, eval_gradient=False,
         """
         if theta is None:
             if eval_gradient:
-                raise ValueError(
-                    "Gradient can only be evaluated for theta!=None")
+                raise ValueError("Gradient can only be evaluated for theta!=None")
             return self.log_marginal_likelihood_value_
 
         if clone_kernel:
@@ -467,35 +581,70 @@ def log_marginal_likelihood(self, theta=None, eval_gradient=False,
         else:
             K = kernel(self.X_train_)
 
+        # Alg. 2.1, page 19, line 2 -> L = cholesky(K + sigma^2 I)
         K[np.diag_indices_from(K)] += self.alpha
         try:
-            L = cholesky(K, lower=True)  # Line 2
+            L = cholesky(K, lower=GPR_CHOLESKY_LOWER, check_finite=False)
         except np.linalg.LinAlgError:
-            return (-np.inf, np.zeros_like(theta)) \
-                if eval_gradient else -np.inf
+            return (-np.inf, np.zeros_like(theta)) if eval_gradient else -np.inf
 
         # Support multi-dimensional output of self.y_train_
         y_train = self.y_train_
         if y_train.ndim == 1:
             y_train = y_train[:, np.newaxis]
 
-        alpha = cho_solve((L, True), y_train)  # Line 3
-
-        # Compute log-likelihood (compare line 7)
+        # Alg 2.1, page 19, line 3 -> alpha = L^T \ (L \ y)
+        alpha = cho_solve((L, GPR_CHOLESKY_LOWER), y_train, check_finite=False)
+
+        # Alg 2.1, page 19, line 7
+        # -0.5 . y^T . alpha - sum(log(diag(L))) - n_samples / 2 log(2*pi)
+        # y is originally thought to be a (1, n_samples) row vector. However,
+        # in multioutputs, y is of shape (n_samples, 2) and we need to compute
+        # y^T . alpha for each output, independently using einsum. Thus, it
+        # is equivalent to:
+        # for output_idx in range(n_outputs):
+        #     log_likelihood_dims[output_idx] = (
+        #         y_train[:, [output_idx]] @ alpha[:, [output_idx]]
+        #     )
         log_likelihood_dims = -0.5 * np.einsum("ik,ik->k", y_train, alpha)
         log_likelihood_dims -= np.log(np.diag(L)).sum()
         log_likelihood_dims -= K.shape[0] / 2 * np.log(2 * np.pi)
-        log_likelihood = log_likelihood_dims.sum(-1)  # sum over dimensions
-
-        if eval_gradient:  # compare Equation 5.9 from GPML
-            tmp = np.einsum("ik,jk->ijk", alpha, alpha)  # k: output-dimension
-            tmp -= cho_solve((L, True), np.eye(K.shape[0]))[:, :, np.newaxis]
-            # Compute "0.5 * trace(tmp.dot(K_gradient))" without
-            # constructing the full matrix tmp.dot(K_gradient) since only
-            # its diagonal is required
-            log_likelihood_gradient_dims = \
-                0.5 * np.einsum("ijl,jik->kl", tmp, K_gradient)
-            log_likelihood_gradient = log_likelihood_gradient_dims.sum(-1)
+        # the log likelihood is sum-up across the outputs
+        log_likelihood = log_likelihood_dims.sum(axis=-1)
+
+        if eval_gradient:
+            # Eq. 5.9, p. 114, and footnote 5 in p. 114
+            # 0.5 * trace((alpha . alpha^T - K^-1) . K_gradient)
+            # alpha is supposed to be a vector of (n_samples,) elements. With
+            # multioutputs, alpha is a matrix of size (n_samples, n_outputs).
+            # Therefore, we want to construct a matrix of
+            # (n_samples, n_samples, n_outputs) equivalent to
+            # for output_idx in range(n_outputs):
+            #     output_alpha = alpha[:, [output_idx]]
+            #     inner_term[..., output_idx] = output_alpha @ output_alpha.T
+            inner_term = np.einsum("ik,jk->ijk", alpha, alpha)
+            # compute K^-1 of shape (n_samples, n_samples)
+            K_inv = cho_solve(
+                (L, GPR_CHOLESKY_LOWER), np.eye(K.shape[0]), check_finite=False
+            )
+            # create a new axis to use broadcasting between inner_term and
+            # K_inv
+            inner_term -= K_inv[..., np.newaxis]
+            # Since we are interested about the trace of
+            # inner_term @ K_gradient, we don't explicitly compute the
+            # matrix-by-matrix operation and instead use an einsum. Therefore
+            # it is equivalent to:
+            # for param_idx in range(n_kernel_params):
+            #     for output_idx in range(n_output):
+            #         log_likehood_gradient_dims[param_idx, output_idx] = (
+            #             inner_term[..., output_idx] @
+            #             K_gradient[..., param_idx]
+            #         )
+            log_likelihood_gradient_dims = 0.5 * np.einsum(
+                "ijl,jik->kl", inner_term, K_gradient
+            )
+            # the log likelihood gradient is the sum-up across the outputs
+            log_likelihood_gradient = log_likelihood_gradient_dims.sum(axis=-1)
 
         if eval_gradient:
             return log_likelihood, log_likelihood_gradient
@@ -505,17 +654,22 @@ def log_marginal_likelihood(self, theta=None, eval_gradient=False,
     def _constrained_optimization(self, obj_func, initial_theta, bounds):
         if self.optimizer == "fmin_l_bfgs_b":
             opt_res = scipy.optimize.minimize(
-                obj_func, initial_theta, method="L-BFGS-B", jac=True,
-                bounds=bounds)
+                obj_func,
+                initial_theta,
+                method="L-BFGS-B",
+                jac=True,
+                bounds=bounds,
+            )
             _check_optimize_result("lbfgs", opt_res)
             theta_opt, func_min = opt_res.x, opt_res.fun
         elif callable(self.optimizer):
-            theta_opt, func_min = \
-                self.optimizer(obj_func, initial_theta, bounds=bounds)
+            theta_opt, func_min = self.optimizer(obj_func, initial_theta, bounds=bounds)
         else:
-            raise ValueError("Unknown optimizer %s." % self.optimizer)
+            raise ValueError(f"Unknown optimizer {self.optimizer}.")
 
         return theta_opt, func_min
 
-    def _more_tags(self):
-        return {'requires_fit': False}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.requires_fit = False
+        return tags
diff --git a/sklearn/gaussian_process/kernels.py b/sklearn/gaussian_process/kernels.py
index 008c24f294737..4a0a6ec667be4 100644
--- a/sklearn/gaussian_process/kernels.py
+++ b/sklearn/gaussian_process/kernels.py
@@ -1,39 +1,40 @@
-"""Kernels for Gaussian process regression and classification.
-
-The kernels in this module allow kernel-engineering, i.e., they can be
-combined via the "+" and "*" operators or be exponentiated with a scalar
-via "**". These sum and product expressions can also contain scalar values,
-which are automatically converted to a constant kernel.
-
-All kernels allow (analytic) gradient-based hyperparameter optimization.
-The space of hyperparameters can be specified by giving lower und upper
-boundaries for the value of each hyperparameter (the search space is thus
-rectangular). Instead of specifying bounds, hyperparameters can also be
-declared to be "fixed", which causes these hyperparameters to be excluded from
-optimization.
-"""
-
-# Author: Jan Hendrik Metzen 
-# License: BSD 3 clause
+"""A set of kernels that can be combined by operators and used in Gaussian processes."""
+
+# Kernels for Gaussian process regression and classification.
+#
+# The kernels in this module allow kernel-engineering, i.e., they can be
+# combined via the "+" and "*" operators or be exponentiated with a scalar
+# via "**". These sum and product expressions can also contain scalar values,
+# which are automatically converted to a constant kernel.
+#
+# All kernels allow (analytic) gradient-based hyperparameter optimization.
+# The space of hyperparameters can be specified by giving lower und upper
+# boundaries for the value of each hyperparameter (the search space is thus
+# rectangular). Instead of specifying bounds, hyperparameters can also be
+# declared to be "fixed", which causes these hyperparameters to be excluded from
+# optimization.
+
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 # Note: this module is strongly inspired by the kernel module of the george
 #       package.
 
+import math
+import warnings
 from abc import ABCMeta, abstractmethod
 from collections import namedtuple
-import math
 from inspect import signature
 
 import numpy as np
-from scipy.special import kv, gamma
-from scipy.spatial.distance import pdist, cdist, squareform
+from scipy.spatial.distance import cdist, pdist, squareform
+from scipy.special import gamma, kv
 
-from ..metrics.pairwise import pairwise_kernels
 from ..base import clone
-from ..utils.validation import _num_samples
 from ..exceptions import ConvergenceWarning
-
-import warnings
+from ..metrics.pairwise import pairwise_kernels
+from ..utils.validation import _num_samples
 
 
 def _check_length_scale(X, length_scale):
@@ -41,15 +42,18 @@ def _check_length_scale(X, length_scale):
     if np.ndim(length_scale) > 1:
         raise ValueError("length_scale cannot be of dimension greater than 1")
     if np.ndim(length_scale) == 1 and X.shape[1] != length_scale.shape[0]:
-        raise ValueError("Anisotropic kernel must have the same number of "
-                         "dimensions as data (%d!=%d)"
-                         % (length_scale.shape[0], X.shape[1]))
+        raise ValueError(
+            "Anisotropic kernel must have the same number of "
+            "dimensions as data (%d!=%d)" % (length_scale.shape[0], X.shape[1])
+        )
     return length_scale
 
 
-class Hyperparameter(namedtuple('Hyperparameter',
-                                ('name', 'value_type', 'bounds',
-                                 'n_elements', 'fixed'))):
+class Hyperparameter(
+    namedtuple(
+        "Hyperparameter", ("name", "value_type", "bounds", "n_elements", "fixed")
+    )
+):
     """A kernel hyperparameter's specification in form of a namedtuple.
 
     .. versionadded:: 0.18
@@ -122,29 +126,53 @@ def __new__(cls, name, value_type, bounds, n_elements=1, fixed=None):
                 if bounds.shape[0] == 1:
                     bounds = np.repeat(bounds, n_elements, 0)
                 elif bounds.shape[0] != n_elements:
-                    raise ValueError("Bounds on %s should have either 1 or "
-                                     "%d dimensions. Given are %d"
-                                     % (name, n_elements, bounds.shape[0]))
+                    raise ValueError(
+                        "Bounds on %s should have either 1 or "
+                        "%d dimensions. Given are %d"
+                        % (name, n_elements, bounds.shape[0])
+                    )
 
         if fixed is None:
             fixed = isinstance(bounds, str) and bounds == "fixed"
-        return super(Hyperparameter, cls).__new__(
-            cls, name, value_type, bounds, n_elements, fixed)
+        return super().__new__(cls, name, value_type, bounds, n_elements, fixed)
 
     # This is mainly a testing utility to check that two hyperparameters
     # are equal.
     def __eq__(self, other):
-        return (self.name == other.name and
-                self.value_type == other.value_type and
-                np.all(self.bounds == other.bounds) and
-                self.n_elements == other.n_elements and
-                self.fixed == other.fixed)
+        return (
+            self.name == other.name
+            and self.value_type == other.value_type
+            and np.all(self.bounds == other.bounds)
+            and self.n_elements == other.n_elements
+            and self.fixed == other.fixed
+        )
 
 
 class Kernel(metaclass=ABCMeta):
     """Base class for all kernels.
 
     .. versionadded:: 0.18
+
+    Examples
+    --------
+    >>> from sklearn.gaussian_process.kernels import Kernel, RBF
+    >>> import numpy as np
+    >>> class CustomKernel(Kernel):
+    ...     def __init__(self, length_scale=1.0):
+    ...         self.length_scale = length_scale
+    ...     def __call__(self, X, Y=None):
+    ...         if Y is None:
+    ...             Y = X
+    ...         return np.inner(X, X if Y is None else Y) ** 2
+    ...     def diag(self, X):
+    ...         return np.ones(X.shape[0])
+    ...     def is_stationary(self):
+    ...         return True
+    >>> kernel = CustomKernel(length_scale=2.0)
+    >>> X = np.array([[1, 2], [3, 4]])
+    >>> print(kernel(X))
+    [[ 25 121]
+     [121 625]]
     """
 
     def get_params(self, deep=True):
@@ -166,22 +194,22 @@ def get_params(self, deep=True):
         # introspect the constructor arguments to find the model parameters
         # to represent
         cls = self.__class__
-        init = getattr(cls.__init__, 'deprecated_original', cls.__init__)
+        init = getattr(cls.__init__, "deprecated_original", cls.__init__)
         init_sign = signature(init)
         args, varargs = [], []
         for parameter in init_sign.parameters.values():
-            if (parameter.kind != parameter.VAR_KEYWORD and
-                    parameter.name != 'self'):
+            if parameter.kind != parameter.VAR_KEYWORD and parameter.name != "self":
                 args.append(parameter.name)
             if parameter.kind == parameter.VAR_POSITIONAL:
                 varargs.append(parameter.name)
 
         if len(varargs) != 0:
-            raise RuntimeError("scikit-learn kernels should always "
-                               "specify their parameters in the signature"
-                               " of their __init__ (no varargs)."
-                               " %s doesn't follow this convention."
-                               % (cls, ))
+            raise RuntimeError(
+                "scikit-learn kernels should always "
+                "specify their parameters in the signature"
+                " of their __init__ (no varargs)."
+                " %s doesn't follow this convention." % (cls,)
+            )
         for arg in args:
             params[arg] = getattr(self, arg)
 
@@ -203,24 +231,27 @@ def set_params(self, **params):
             return self
         valid_params = self.get_params(deep=True)
         for key, value in params.items():
-            split = key.split('__', 1)
+            split = key.split("__", 1)
             if len(split) > 1:
                 # nested objects case
                 name, sub_name = split
                 if name not in valid_params:
-                    raise ValueError('Invalid parameter %s for kernel %s. '
-                                     'Check the list of available parameters '
-                                     'with `kernel.get_params().keys()`.' %
-                                     (name, self))
+                    raise ValueError(
+                        "Invalid parameter %s for kernel %s. "
+                        "Check the list of available parameters "
+                        "with `kernel.get_params().keys()`." % (name, self)
+                    )
                 sub_object = valid_params[name]
                 sub_object.set_params(**{sub_name: value})
             else:
                 # simple objects case
                 if key not in valid_params:
-                    raise ValueError('Invalid parameter %s for kernel %s. '
-                                     'Check the list of available parameters '
-                                     'with `kernel.get_params().keys()`.' %
-                                     (key, self.__class__.__name__))
+                    raise ValueError(
+                        "Invalid parameter %s for kernel %s. "
+                        "Check the list of available parameters "
+                        "with `kernel.get_params().keys()`."
+                        % (key, self.__class__.__name__)
+                    )
                 setattr(self, key, value)
         return self
 
@@ -244,8 +275,11 @@ def n_dims(self):
     @property
     def hyperparameters(self):
         """Returns a list of all hyperparameter specifications."""
-        r = [getattr(self, attr) for attr in dir(self)
-             if attr.startswith("hyperparameter_")]
+        r = [
+            getattr(self, attr)
+            for attr in dir(self)
+            if attr.startswith("hyperparameter_")
+        ]
         return r
 
     @property
@@ -289,16 +323,18 @@ def theta(self, theta):
             if hyperparameter.n_elements > 1:
                 # vector-valued parameter
                 params[hyperparameter.name] = np.exp(
-                    theta[i:i + hyperparameter.n_elements])
+                    theta[i : i + hyperparameter.n_elements]
+                )
                 i += hyperparameter.n_elements
             else:
                 params[hyperparameter.name] = np.exp(theta[i])
                 i += 1
 
         if i != len(theta):
-            raise ValueError("theta has not the correct number of entries."
-                             " Should be %d; given are %d"
-                             % (i, len(theta)))
+            raise ValueError(
+                "theta has not the correct number of entries."
+                " Should be %d; given are %d" % (i, len(theta))
+            )
         self.set_params(**params)
 
     @property
@@ -310,9 +346,11 @@ def bounds(self):
         bounds : ndarray of shape (n_dims, 2)
             The log-transformed bounds on the kernel's hyperparameters theta
         """
-        bounds = [hyperparameter.bounds
-                  for hyperparameter in self.hyperparameters
-                  if not hyperparameter.fixed]
+        bounds = [
+            hyperparameter.bounds
+            for hyperparameter in self.hyperparameters
+            if not hyperparameter.fixed
+        ]
         if len(bounds) > 0:
             return np.log(np.vstack(bounds))
         else:
@@ -352,8 +390,9 @@ def __eq__(self, b):
         return True
 
     def __repr__(self):
-        return "{0}({1})".format(self.__class__.__name__,
-                                 ", ".join(map("{0:.3g}".format, self.theta)))
+        return "{0}({1})".format(
+            self.__class__.__name__, ", ".join(map("{0:.3g}".format, self.theta))
+        )
 
     @abstractmethod
     def __call__(self, X, Y=None, eval_gradient=False):
@@ -380,7 +419,7 @@ def diag(self, X):
 
     @abstractmethod
     def is_stationary(self):
-        """Returns whether the kernel is stationary. """
+        """Returns whether the kernel is stationary."""
 
     @property
     def requires_vector_input(self):
@@ -391,31 +430,32 @@ def requires_vector_input(self):
 
     def _check_bounds_params(self):
         """Called after fitting to warn if bounds may have been too tight."""
-        list_close = np.isclose(self.bounds,
-                                np.atleast_2d(self.theta).T)
+        list_close = np.isclose(self.bounds, np.atleast_2d(self.theta).T)
         idx = 0
         for hyp in self.hyperparameters:
             if hyp.fixed:
                 continue
             for dim in range(hyp.n_elements):
                 if list_close[idx, 0]:
-                    warnings.warn("The optimal value found for "
-                                  "dimension %s of parameter %s is "
-                                  "close to the specified lower "
-                                  "bound %s. Decreasing the bound and"
-                                  " calling fit again may find a "
-                                  "better value." %
-                                  (dim, hyp.name, hyp.bounds[dim][0]),
-                                  ConvergenceWarning)
+                    warnings.warn(
+                        "The optimal value found for "
+                        "dimension %s of parameter %s is "
+                        "close to the specified lower "
+                        "bound %s. Decreasing the bound and"
+                        " calling fit again may find a "
+                        "better value." % (dim, hyp.name, hyp.bounds[dim][0]),
+                        ConvergenceWarning,
+                    )
                 elif list_close[idx, 1]:
-                    warnings.warn("The optimal value found for "
-                                  "dimension %s of parameter %s is "
-                                  "close to the specified upper "
-                                  "bound %s. Increasing the bound and"
-                                  " calling fit again may find a "
-                                  "better value." %
-                                  (dim, hyp.name, hyp.bounds[dim][1]),
-                                  ConvergenceWarning)
+                    warnings.warn(
+                        "The optimal value found for "
+                        "dimension %s of parameter %s is "
+                        "close to the specified upper "
+                        "bound %s. Increasing the bound and"
+                        " calling fit again may find a "
+                        "better value." % (dim, hyp.name, hyp.bounds[dim][1]),
+                        ConvergenceWarning,
+                    )
                 idx += 1
 
 
@@ -452,7 +492,7 @@ class StationaryKernelMixin:
     """
 
     def is_stationary(self):
-        """Returns whether the kernel is stationary. """
+        """Returns whether the kernel is stationary."""
         return True
 
 
@@ -541,7 +581,7 @@ def theta(self, theta):
         """
         k_dims = self.k1.n_dims
         for i, kernel in enumerate(self.kernels):
-            kernel.theta = theta[i * k_dims:(i + 1) * k_dims]
+            kernel.theta = theta[i * k_dims : (i + 1) * k_dims]
 
     @property
     def bounds(self):
@@ -595,24 +635,23 @@ def __call__(self, X, Y=None, eval_gradient=False):
                 K_grad.append(K_grad_single[..., np.newaxis])
             return np.dstack(K), np.concatenate(K_grad, 3)
         else:
-            return np.dstack([kernel(X, Y, eval_gradient)
-                              for kernel in self.kernels])
+            return np.dstack([kernel(X, Y, eval_gradient) for kernel in self.kernels])
 
     def __eq__(self, b):
         if type(self) != type(b) or len(self.kernels) != len(b.kernels):
             return False
-        return np.all([self.kernels[i] == b.kernels[i]
-                       for i in range(len(self.kernels))])
+        return np.all(
+            [self.kernels[i] == b.kernels[i] for i in range(len(self.kernels))]
+        )
 
     def is_stationary(self):
-        """Returns whether the kernel is stationary. """
+        """Returns whether the kernel is stationary."""
         return np.all([kernel.is_stationary() for kernel in self.kernels])
 
     @property
     def requires_vector_input(self):
-        """Returns whether the kernel is defined on discrete structures. """
-        return np.any([kernel.requires_vector_input
-                       for kernel in self.kernels])
+        """Returns whether the kernel is defined on discrete structures."""
+        return np.any([kernel.requires_vector_input for kernel in self.kernels])
 
     def diag(self, X):
         """Returns the diagonal of the kernel k(X, X).
@@ -661,25 +700,34 @@ def get_params(self, deep=True):
         params = dict(k1=self.k1, k2=self.k2)
         if deep:
             deep_items = self.k1.get_params().items()
-            params.update(('k1__' + k, val) for k, val in deep_items)
+            params.update(("k1__" + k, val) for k, val in deep_items)
             deep_items = self.k2.get_params().items()
-            params.update(('k2__' + k, val) for k, val in deep_items)
+            params.update(("k2__" + k, val) for k, val in deep_items)
 
         return params
 
     @property
     def hyperparameters(self):
         """Returns a list of all hyperparameter."""
-        r = [Hyperparameter("k1__" + hyperparameter.name,
-                            hyperparameter.value_type,
-                            hyperparameter.bounds, hyperparameter.n_elements)
-             for hyperparameter in self.k1.hyperparameters]
+        r = [
+            Hyperparameter(
+                "k1__" + hyperparameter.name,
+                hyperparameter.value_type,
+                hyperparameter.bounds,
+                hyperparameter.n_elements,
+            )
+            for hyperparameter in self.k1.hyperparameters
+        ]
 
         for hyperparameter in self.k2.hyperparameters:
-            r.append(Hyperparameter("k2__" + hyperparameter.name,
-                                    hyperparameter.value_type,
-                                    hyperparameter.bounds,
-                                    hyperparameter.n_elements))
+            r.append(
+                Hyperparameter(
+                    "k2__" + hyperparameter.name,
+                    hyperparameter.value_type,
+                    hyperparameter.bounds,
+                    hyperparameter.n_elements,
+                )
+            )
         return r
 
     @property
@@ -729,18 +777,18 @@ def bounds(self):
     def __eq__(self, b):
         if type(self) != type(b):
             return False
-        return (self.k1 == b.k1 and self.k2 == b.k2) \
-            or (self.k1 == b.k2 and self.k2 == b.k1)
+        return (self.k1 == b.k1 and self.k2 == b.k2) or (
+            self.k1 == b.k2 and self.k2 == b.k1
+        )
 
     def is_stationary(self):
-        """Returns whether the kernel is stationary. """
+        """Returns whether the kernel is stationary."""
         return self.k1.is_stationary() and self.k2.is_stationary()
 
     @property
     def requires_vector_input(self):
-        """Returns whether the kernel is stationary. """
-        return (self.k1.requires_vector_input or
-                self.k2.requires_vector_input)
+        """Returns whether the kernel is stationary."""
+        return self.k1.requires_vector_input or self.k2.requires_vector_input
 
 
 class Sum(KernelOperator):
@@ -911,8 +959,9 @@ def __call__(self, X, Y=None, eval_gradient=False):
         if eval_gradient:
             K1, K1_gradient = self.k1(X, Y, eval_gradient=True)
             K2, K2_gradient = self.k2(X, Y, eval_gradient=True)
-            return K1 * K2, np.dstack((K1_gradient * K2[:, :, np.newaxis],
-                                       K2_gradient * K1[:, :, np.newaxis]))
+            return K1 * K2, np.dstack(
+                (K1_gradient * K2[:, :, np.newaxis], K2_gradient * K1[:, :, np.newaxis])
+            )
         else:
             return self.k1(X, Y) * self.k2(X, Y)
 
@@ -975,9 +1024,9 @@ class Exponentiation(Kernel):
     >>> gpr = GaussianProcessRegressor(kernel=kernel, alpha=5,
     ...         random_state=0).fit(X, y)
     >>> gpr.score(X, y)
-    0.419...
+    0.419
     >>> gpr.predict(X[:1,:], return_std=True)
-    (array([635.5...]), array([0.559...]))
+    (array([635.5]), array([0.559]))
     """
 
     def __init__(self, kernel, exponent):
@@ -1001,7 +1050,7 @@ def get_params(self, deep=True):
         params = dict(kernel=self.kernel, exponent=self.exponent)
         if deep:
             deep_items = self.kernel.get_params().items()
-            params.update(('kernel__' + k, val) for k, val in deep_items)
+            params.update(("kernel__" + k, val) for k, val in deep_items)
         return params
 
     @property
@@ -1009,10 +1058,14 @@ def hyperparameters(self):
         """Returns a list of all hyperparameter."""
         r = []
         for hyperparameter in self.kernel.hyperparameters:
-            r.append(Hyperparameter("kernel__" + hyperparameter.name,
-                                    hyperparameter.value_type,
-                                    hyperparameter.bounds,
-                                    hyperparameter.n_elements))
+            r.append(
+                Hyperparameter(
+                    "kernel__" + hyperparameter.name,
+                    hyperparameter.value_type,
+                    hyperparameter.bounds,
+                    hyperparameter.n_elements,
+                )
+            )
         return r
 
     @property
@@ -1056,7 +1109,7 @@ def bounds(self):
     def __eq__(self, b):
         if type(self) != type(b):
             return False
-        return (self.kernel == b.kernel and self.exponent == b.exponent)
+        return self.kernel == b.kernel and self.exponent == b.exponent
 
     def __call__(self, X, Y=None, eval_gradient=False):
         """Return the kernel k(X, Y) and optionally its gradient.
@@ -1088,12 +1141,11 @@ def __call__(self, X, Y=None, eval_gradient=False):
         """
         if eval_gradient:
             K, K_gradient = self.kernel(X, Y, eval_gradient=True)
-            K_gradient *= \
-                self.exponent * K[:, :, np.newaxis] ** (self.exponent - 1)
-            return K ** self.exponent, K_gradient
+            K_gradient *= self.exponent * K[:, :, np.newaxis] ** (self.exponent - 1)
+            return K**self.exponent, K_gradient
         else:
             K = self.kernel(X, Y, eval_gradient=False)
-            return K ** self.exponent
+            return K**self.exponent
 
     def diag(self, X):
         """Returns the diagonal of the kernel k(X, X).
@@ -1118,17 +1170,16 @@ def __repr__(self):
         return "{0} ** {1}".format(self.kernel, self.exponent)
 
     def is_stationary(self):
-        """Returns whether the kernel is stationary. """
+        """Returns whether the kernel is stationary."""
         return self.kernel.is_stationary()
 
     @property
     def requires_vector_input(self):
-        """Returns whether the kernel is defined on discrete structures. """
+        """Returns whether the kernel is defined on discrete structures."""
         return self.kernel.requires_vector_input
 
 
-class ConstantKernel(StationaryKernelMixin, GenericKernelMixin,
-                     Kernel):
+class ConstantKernel(StationaryKernelMixin, GenericKernelMixin, Kernel):
     """Constant kernel.
 
     Can be used as part of a product-kernel where it scales the magnitude of
@@ -1172,9 +1223,9 @@ class ConstantKernel(StationaryKernelMixin, GenericKernelMixin,
     >>> gpr = GaussianProcessRegressor(kernel=kernel, alpha=5,
     ...         random_state=0).fit(X, y)
     >>> gpr.score(X, y)
-    0.3696...
+    0.3696
     >>> gpr.predict(X[:1,:], return_std=True)
-    (array([606.1...]), array([0.24...]))
+    (array([606.1]), array([0.248]))
     """
 
     def __init__(self, constant_value=1.0, constant_value_bounds=(1e-5, 1e5)):
@@ -1183,8 +1234,7 @@ def __init__(self, constant_value=1.0, constant_value_bounds=(1e-5, 1e5)):
 
     @property
     def hyperparameter_constant_value(self):
-        return Hyperparameter(
-            "constant_value", "numeric", self.constant_value_bounds)
+        return Hyperparameter("constant_value", "numeric", self.constant_value_bounds)
 
     def __call__(self, X, Y=None, eval_gradient=False):
         """Return the kernel k(X, Y) and optionally its gradient.
@@ -1220,13 +1270,21 @@ def __call__(self, X, Y=None, eval_gradient=False):
         elif eval_gradient:
             raise ValueError("Gradient can only be evaluated when Y is None.")
 
-        K = np.full((_num_samples(X), _num_samples(Y)), self.constant_value,
-                    dtype=np.array(self.constant_value).dtype)
+        K = np.full(
+            (_num_samples(X), _num_samples(Y)),
+            self.constant_value,
+            dtype=np.array(self.constant_value).dtype,
+        )
         if eval_gradient:
             if not self.hyperparameter_constant_value.fixed:
-                return (K, np.full((_num_samples(X), _num_samples(X), 1),
-                                   self.constant_value,
-                                   dtype=np.array(self.constant_value).dtype))
+                return (
+                    K,
+                    np.full(
+                        (_num_samples(X), _num_samples(X), 1),
+                        self.constant_value,
+                        dtype=np.array(self.constant_value).dtype,
+                    ),
+                )
             else:
                 return K, np.empty((_num_samples(X), _num_samples(X), 0))
         else:
@@ -1249,15 +1307,17 @@ def diag(self, X):
         K_diag : ndarray of shape (n_samples_X,)
             Diagonal of kernel k(X, X)
         """
-        return np.full(_num_samples(X), self.constant_value,
-                       dtype=np.array(self.constant_value).dtype)
+        return np.full(
+            _num_samples(X),
+            self.constant_value,
+            dtype=np.array(self.constant_value).dtype,
+        )
 
     def __repr__(self):
         return "{0:.3g}**2".format(np.sqrt(self.constant_value))
 
 
-class WhiteKernel(StationaryKernelMixin, GenericKernelMixin,
-                  Kernel):
+class WhiteKernel(StationaryKernelMixin, GenericKernelMixin, Kernel):
     """White kernel.
 
     The main use-case of this kernel is as part of a sum-kernel where it
@@ -1293,18 +1353,18 @@ class WhiteKernel(StationaryKernelMixin, GenericKernelMixin,
     >>> gpr = GaussianProcessRegressor(kernel=kernel,
     ...         random_state=0).fit(X, y)
     >>> gpr.score(X, y)
-    0.3680...
+    0.3680
     >>> gpr.predict(X[:2,:], return_std=True)
-    (array([653.0..., 592.1... ]), array([316.6..., 316.6...]))
+    (array([653.0, 592.1 ]), array([316.6, 316.6]))
     """
+
     def __init__(self, noise_level=1.0, noise_level_bounds=(1e-5, 1e5)):
         self.noise_level = noise_level
         self.noise_level_bounds = noise_level_bounds
 
     @property
     def hyperparameter_noise_level(self):
-        return Hyperparameter(
-            "noise_level", "numeric", self.noise_level_bounds)
+        return Hyperparameter("noise_level", "numeric", self.noise_level_bounds)
 
     def __call__(self, X, Y=None, eval_gradient=False):
         """Return the kernel k(X, Y) and optionally its gradient.
@@ -1342,8 +1402,10 @@ def __call__(self, X, Y=None, eval_gradient=False):
             K = self.noise_level * np.eye(_num_samples(X))
             if eval_gradient:
                 if not self.hyperparameter_noise_level.fixed:
-                    return (K, self.noise_level
-                            * np.eye(_num_samples(X))[:, :, np.newaxis])
+                    return (
+                        K,
+                        self.noise_level * np.eye(_num_samples(X))[:, :, np.newaxis],
+                    )
                 else:
                     return K, np.empty((_num_samples(X), _num_samples(X), 0))
             else:
@@ -1368,16 +1430,18 @@ def diag(self, X):
         K_diag : ndarray of shape (n_samples_X,)
             Diagonal of kernel k(X, X)
         """
-        return np.full(_num_samples(X), self.noise_level,
-                       dtype=np.array(self.noise_level).dtype)
+        return np.full(
+            _num_samples(X), self.noise_level, dtype=np.array(self.noise_level).dtype
+        )
 
     def __repr__(self):
-        return "{0}(noise_level={1:.3g})".format(self.__class__.__name__,
-                                                 self.noise_level)
+        return "{0}(noise_level={1:.3g})".format(
+            self.__class__.__name__, self.noise_level
+        )
 
 
 class RBF(StationaryKernelMixin, NormalizedKernelMixin, Kernel):
-    """Radial-basis function kernel (aka squared-exponential kernel).
+    """Radial basis function kernel (aka squared-exponential kernel).
 
     The RBF kernel is a stationary kernel. It is also known as the
     "squared exponential" kernel. It is parameterized by a length scale
@@ -1433,11 +1497,12 @@ class RBF(StationaryKernelMixin, NormalizedKernelMixin, Kernel):
     >>> gpc = GaussianProcessClassifier(kernel=kernel,
     ...         random_state=0).fit(X, y)
     >>> gpc.score(X, y)
-    0.9866...
+    0.9866
     >>> gpc.predict_proba(X[:2,:])
-    array([[0.8354..., 0.03228..., 0.1322...],
-           [0.7906..., 0.0652..., 0.1441...]])
+    array([[0.8354, 0.03228, 0.1322],
+           [0.7906, 0.0652, 0.1441]])
     """
+
     def __init__(self, length_scale=1.0, length_scale_bounds=(1e-5, 1e5)):
         self.length_scale = length_scale
         self.length_scale_bounds = length_scale_bounds
@@ -1449,11 +1514,13 @@ def anisotropic(self):
     @property
     def hyperparameter_length_scale(self):
         if self.anisotropic:
-            return Hyperparameter("length_scale", "numeric",
-                                  self.length_scale_bounds,
-                                  len(self.length_scale))
-        return Hyperparameter(
-            "length_scale", "numeric", self.length_scale_bounds)
+            return Hyperparameter(
+                "length_scale",
+                "numeric",
+                self.length_scale_bounds,
+                len(self.length_scale),
+            )
+        return Hyperparameter("length_scale", "numeric", self.length_scale_bounds)
 
     def __call__(self, X, Y=None, eval_gradient=False):
         """Return the kernel k(X, Y) and optionally its gradient.
@@ -1486,31 +1553,29 @@ def __call__(self, X, Y=None, eval_gradient=False):
         X = np.atleast_2d(X)
         length_scale = _check_length_scale(X, self.length_scale)
         if Y is None:
-            dists = pdist(X / length_scale, metric='sqeuclidean')
-            K = np.exp(-.5 * dists)
+            dists = pdist(X / length_scale, metric="sqeuclidean")
+            K = np.exp(-0.5 * dists)
             # convert from upper-triangular matrix to square matrix
             K = squareform(K)
             np.fill_diagonal(K, 1)
         else:
             if eval_gradient:
-                raise ValueError(
-                    "Gradient can only be evaluated when Y is None.")
-            dists = cdist(X / length_scale, Y / length_scale,
-                          metric='sqeuclidean')
-            K = np.exp(-.5 * dists)
+                raise ValueError("Gradient can only be evaluated when Y is None.")
+            dists = cdist(X / length_scale, Y / length_scale, metric="sqeuclidean")
+            K = np.exp(-0.5 * dists)
 
         if eval_gradient:
             if self.hyperparameter_length_scale.fixed:
                 # Hyperparameter l kept fixed
                 return K, np.empty((X.shape[0], X.shape[0], 0))
             elif not self.anisotropic or length_scale.shape[0] == 1:
-                K_gradient = \
-                    (K * squareform(dists))[:, :, np.newaxis]
+                K_gradient = (K * squareform(dists))[:, :, np.newaxis]
                 return K, K_gradient
             elif self.anisotropic:
                 # We need to recompute the pairwise dimension-wise distances
-                K_gradient = (X[:, np.newaxis, :] - X[np.newaxis, :, :]) ** 2 \
-                    / (length_scale ** 2)
+                K_gradient = (X[:, np.newaxis, :] - X[np.newaxis, :, :]) ** 2 / (
+                    length_scale**2
+                )
                 K_gradient *= K[..., np.newaxis]
                 return K, K_gradient
         else:
@@ -1519,15 +1584,17 @@ def __call__(self, X, Y=None, eval_gradient=False):
     def __repr__(self):
         if self.anisotropic:
             return "{0}(length_scale=[{1}])".format(
-                self.__class__.__name__, ", ".join(map("{0:.3g}".format,
-                                                   self.length_scale)))
+                self.__class__.__name__,
+                ", ".join(map("{0:.3g}".format, self.length_scale)),
+            )
         else:  # isotropic
             return "{0}(length_scale={1:.3g})".format(
-                self.__class__.__name__, np.ravel(self.length_scale)[0])
+                self.__class__.__name__, np.ravel(self.length_scale)[0]
+            )
 
 
 class Matern(RBF):
-    """ Matern kernel.
+    """Matern kernel.
 
     The class of Matern kernels is a generalization of the :class:`RBF`.
     It has an additional parameter :math:`\\nu` which controls the
@@ -1600,13 +1667,13 @@ class Matern(RBF):
     >>> gpc = GaussianProcessClassifier(kernel=kernel,
     ...         random_state=0).fit(X, y)
     >>> gpc.score(X, y)
-    0.9866...
+    0.9866
     >>> gpc.predict_proba(X[:2,:])
-    array([[0.8513..., 0.0368..., 0.1117...],
-            [0.8086..., 0.0693..., 0.1220...]])
+    array([[0.8513, 0.0368, 0.1117],
+            [0.8086, 0.0693, 0.1220]])
     """
-    def __init__(self, length_scale=1.0, length_scale_bounds=(1e-5, 1e5),
-                 nu=1.5):
+
+    def __init__(self, length_scale=1.0, length_scale_bounds=(1e-5, 1e5), nu=1.5):
         super().__init__(length_scale, length_scale_bounds)
         self.nu = nu
 
@@ -1641,30 +1708,28 @@ def __call__(self, X, Y=None, eval_gradient=False):
         X = np.atleast_2d(X)
         length_scale = _check_length_scale(X, self.length_scale)
         if Y is None:
-            dists = pdist(X / length_scale, metric='euclidean')
+            dists = pdist(X / length_scale, metric="euclidean")
         else:
             if eval_gradient:
-                raise ValueError(
-                    "Gradient can only be evaluated when Y is None.")
-            dists = cdist(X / length_scale, Y / length_scale,
-                          metric='euclidean')
+                raise ValueError("Gradient can only be evaluated when Y is None.")
+            dists = cdist(X / length_scale, Y / length_scale, metric="euclidean")
 
         if self.nu == 0.5:
             K = np.exp(-dists)
         elif self.nu == 1.5:
             K = dists * math.sqrt(3)
-            K = (1. + K) * np.exp(-K)
+            K = (1.0 + K) * np.exp(-K)
         elif self.nu == 2.5:
             K = dists * math.sqrt(5)
-            K = (1. + K + K ** 2 / 3.0) * np.exp(-K)
+            K = (1.0 + K + K**2 / 3.0) * np.exp(-K)
         elif self.nu == np.inf:
-            K = np.exp(-dists ** 2 / 2.0)
+            K = np.exp(-(dists**2) / 2.0)
         else:  # general case; expensive to evaluate
             K = dists
             K[K == 0.0] += np.finfo(float).eps  # strict zeros result in nan
-            tmp = (math.sqrt(2 * self.nu) * K)
-            K.fill((2 ** (1. - self.nu)) / gamma(self.nu))
-            K *= tmp ** self.nu
+            tmp = math.sqrt(2 * self.nu) * K
+            K.fill((2 ** (1.0 - self.nu)) / gamma(self.nu))
+            K *= tmp**self.nu
             K *= kv(self.nu, tmp)
 
         if Y is None:
@@ -1680,18 +1745,22 @@ def __call__(self, X, Y=None, eval_gradient=False):
 
             # We need to recompute the pairwise dimension-wise distances
             if self.anisotropic:
-                D = (X[:, np.newaxis, :] - X[np.newaxis, :, :])**2 \
-                    / (length_scale ** 2)
+                D = (X[:, np.newaxis, :] - X[np.newaxis, :, :]) ** 2 / (length_scale**2)
             else:
                 D = squareform(dists**2)[:, :, np.newaxis]
 
             if self.nu == 0.5:
                 denominator = np.sqrt(D.sum(axis=2))[:, :, np.newaxis]
-                K_gradient = K[..., np.newaxis] * \
-                    np.divide(D, denominator, where=denominator != 0)
+                divide_result = np.zeros_like(D)
+                np.divide(
+                    D,
+                    denominator,
+                    out=divide_result,
+                    where=denominator != 0,
+                )
+                K_gradient = K[..., np.newaxis] * divide_result
             elif self.nu == 1.5:
-                K_gradient = \
-                    3 * D * np.exp(-np.sqrt(3 * D.sum(-1)))[..., np.newaxis]
+                K_gradient = 3 * D * np.exp(-np.sqrt(3 * D.sum(-1)))[..., np.newaxis]
             elif self.nu == 2.5:
                 tmp = np.sqrt(5 * D.sum(-1))[..., np.newaxis]
                 K_gradient = 5.0 / 3.0 * D * (tmp + 1) * np.exp(-tmp)
@@ -1701,6 +1770,7 @@ def __call__(self, X, Y=None, eval_gradient=False):
                 # approximate gradient numerically
                 def f(theta):  # helper function
                     return self.clone_with_theta(theta)(X, Y)
+
                 return K, _approx_fprime(self.theta, f, 1e-10)
 
             if not self.anisotropic:
@@ -1715,11 +1785,12 @@ def __repr__(self):
             return "{0}(length_scale=[{1}], nu={2:.3g})".format(
                 self.__class__.__name__,
                 ", ".join(map("{0:.3g}".format, self.length_scale)),
-                self.nu)
+                self.nu,
+            )
         else:
             return "{0}(length_scale={1:.3g}, nu={2:.3g})".format(
-                self.__class__.__name__, np.ravel(self.length_scale)[0],
-                self.nu)
+                self.__class__.__name__, np.ravel(self.length_scale)[0], self.nu
+            )
 
 
 class RationalQuadratic(StationaryKernelMixin, NormalizedKernelMixin, Kernel):
@@ -1773,19 +1844,25 @@ class RationalQuadratic(StationaryKernelMixin, NormalizedKernelMixin, Kernel):
     --------
     >>> from sklearn.datasets import load_iris
     >>> from sklearn.gaussian_process import GaussianProcessClassifier
-    >>> from sklearn.gaussian_process.kernels import Matern
+    >>> from sklearn.gaussian_process.kernels import RationalQuadratic
     >>> X, y = load_iris(return_X_y=True)
     >>> kernel = RationalQuadratic(length_scale=1.0, alpha=1.5)
     >>> gpc = GaussianProcessClassifier(kernel=kernel,
     ...         random_state=0).fit(X, y)
     >>> gpc.score(X, y)
-    0.9733...
+    0.9733
     >>> gpc.predict_proba(X[:2,:])
-    array([[0.8881..., 0.0566..., 0.05518...],
-            [0.8678..., 0.0707... , 0.0614...]])
+    array([[0.8881, 0.0566, 0.05518],
+            [0.8678, 0.0707 , 0.0614]])
     """
-    def __init__(self, length_scale=1.0, alpha=1.0,
-                 length_scale_bounds=(1e-5, 1e5), alpha_bounds=(1e-5, 1e5)):
+
+    def __init__(
+        self,
+        length_scale=1.0,
+        alpha=1.0,
+        length_scale_bounds=(1e-5, 1e5),
+        alpha_bounds=(1e-5, 1e5),
+    ):
         self.length_scale = length_scale
         self.alpha = alpha
         self.length_scale_bounds = length_scale_bounds
@@ -1793,8 +1870,7 @@ def __init__(self, length_scale=1.0, alpha=1.0,
 
     @property
     def hyperparameter_length_scale(self):
-        return Hyperparameter(
-            "length_scale", "numeric", self.length_scale_bounds)
+        return Hyperparameter("length_scale", "numeric", self.length_scale_bounds)
 
     @property
     def hyperparameter_alpha(self):
@@ -1830,36 +1906,35 @@ def __call__(self, X, Y=None, eval_gradient=False):
         if len(np.atleast_1d(self.length_scale)) > 1:
             raise AttributeError(
                 "RationalQuadratic kernel only supports isotropic version, "
-                "please use a single scalar for length_scale")
+                "please use a single scalar for length_scale"
+            )
         X = np.atleast_2d(X)
         if Y is None:
-            dists = squareform(pdist(X, metric='sqeuclidean'))
-            tmp = dists / (2 * self.alpha * self.length_scale ** 2)
-            base = (1 + tmp)
-            K = base ** -self.alpha
+            dists = squareform(pdist(X, metric="sqeuclidean"))
+            tmp = dists / (2 * self.alpha * self.length_scale**2)
+            base = 1 + tmp
+            K = base**-self.alpha
             np.fill_diagonal(K, 1)
         else:
             if eval_gradient:
-                raise ValueError(
-                    "Gradient can only be evaluated when Y is None.")
-            dists = cdist(X, Y, metric='sqeuclidean')
-            K = (1 + dists / (2 * self.alpha * self.length_scale ** 2)) \
-                ** -self.alpha
+                raise ValueError("Gradient can only be evaluated when Y is None.")
+            dists = cdist(X, Y, metric="sqeuclidean")
+            K = (1 + dists / (2 * self.alpha * self.length_scale**2)) ** -self.alpha
 
         if eval_gradient:
             # gradient with respect to length_scale
             if not self.hyperparameter_length_scale.fixed:
-                length_scale_gradient = \
-                    dists * K / (self.length_scale ** 2 * base)
+                length_scale_gradient = dists * K / (self.length_scale**2 * base)
                 length_scale_gradient = length_scale_gradient[:, :, np.newaxis]
             else:  # l is kept fixed
                 length_scale_gradient = np.empty((K.shape[0], K.shape[1], 0))
 
             # gradient with respect to alpha
             if not self.hyperparameter_alpha.fixed:
-                alpha_gradient = \
-                    K * (-self.alpha * np.log(base)
-                         + dists / (2 * self.length_scale ** 2 * base))
+                alpha_gradient = K * (
+                    -self.alpha * np.log(base)
+                    + dists / (2 * self.length_scale**2 * base)
+                )
                 alpha_gradient = alpha_gradient[:, :, np.newaxis]
             else:  # alpha is kept fixed
                 alpha_gradient = np.empty((K.shape[0], K.shape[1], 0))
@@ -1870,7 +1945,8 @@ def __call__(self, X, Y=None, eval_gradient=False):
 
     def __repr__(self):
         return "{0}(alpha={1:.3g}, length_scale={2:.3g})".format(
-            self.__class__.__name__, self.alpha, self.length_scale)
+            self.__class__.__name__, self.alpha, self.length_scale
+        )
 
 
 class ExpSineSquared(StationaryKernelMixin, NormalizedKernelMixin, Kernel):
@@ -1887,7 +1963,7 @@ class ExpSineSquared(StationaryKernelMixin, NormalizedKernelMixin, Kernel):
         \frac{ 2\sin^2(\pi d(x_i, x_j)/p) }{ l^ 2} \right)
 
     where :math:`l` is the length scale of the kernel, :math:`p` the
-    periodicity of the kernel and :math:`d(\\cdot,\\cdot)` is the
+    periodicity of the kernel and :math:`d(\cdot,\cdot)` is the
     Euclidean distance.
 
     Read more in the :ref:`User Guide `.
@@ -1923,13 +1999,18 @@ class ExpSineSquared(StationaryKernelMixin, NormalizedKernelMixin, Kernel):
     >>> gpr = GaussianProcessRegressor(kernel=kernel, alpha=5,
     ...         random_state=0).fit(X, y)
     >>> gpr.score(X, y)
-    0.0144...
+    0.0144
     >>> gpr.predict(X[:2,:], return_std=True)
-    (array([425.6..., 457.5...]), array([0.3894..., 0.3467...]))
+    (array([425.6, 457.5]), array([0.3894, 0.3467]))
     """
-    def __init__(self, length_scale=1.0, periodicity=1.0,
-                 length_scale_bounds=(1e-5, 1e5),
-                 periodicity_bounds=(1e-5, 1e5)):
+
+    def __init__(
+        self,
+        length_scale=1.0,
+        periodicity=1.0,
+        length_scale_bounds=(1e-5, 1e5),
+        periodicity_bounds=(1e-5, 1e5),
+    ):
         self.length_scale = length_scale
         self.periodicity = periodicity
         self.length_scale_bounds = length_scale_bounds
@@ -1938,13 +2019,11 @@ def __init__(self, length_scale=1.0, periodicity=1.0,
     @property
     def hyperparameter_length_scale(self):
         """Returns the length scale"""
-        return Hyperparameter(
-            "length_scale", "numeric", self.length_scale_bounds)
+        return Hyperparameter("length_scale", "numeric", self.length_scale_bounds)
 
     @property
     def hyperparameter_periodicity(self):
-        return Hyperparameter(
-            "periodicity", "numeric", self.periodicity_bounds)
+        return Hyperparameter("periodicity", "numeric", self.periodicity_bounds)
 
     def __call__(self, X, Y=None, eval_gradient=False):
         """Return the kernel k(X, Y) and optionally its gradient.
@@ -1976,32 +2055,31 @@ def __call__(self, X, Y=None, eval_gradient=False):
         """
         X = np.atleast_2d(X)
         if Y is None:
-            dists = squareform(pdist(X, metric='euclidean'))
+            dists = squareform(pdist(X, metric="euclidean"))
             arg = np.pi * dists / self.periodicity
             sin_of_arg = np.sin(arg)
-            K = np.exp(- 2 * (sin_of_arg / self.length_scale) ** 2)
+            K = np.exp(-2 * (sin_of_arg / self.length_scale) ** 2)
         else:
             if eval_gradient:
-                raise ValueError(
-                    "Gradient can only be evaluated when Y is None.")
-            dists = cdist(X, Y, metric='euclidean')
-            K = np.exp(- 2 * (np.sin(np.pi / self.periodicity * dists)
-                              / self.length_scale) ** 2)
+                raise ValueError("Gradient can only be evaluated when Y is None.")
+            dists = cdist(X, Y, metric="euclidean")
+            K = np.exp(
+                -2 * (np.sin(np.pi / self.periodicity * dists) / self.length_scale) ** 2
+            )
 
         if eval_gradient:
             cos_of_arg = np.cos(arg)
             # gradient with respect to length_scale
             if not self.hyperparameter_length_scale.fixed:
-                length_scale_gradient = \
-                    4 / self.length_scale**2 * sin_of_arg**2 * K
+                length_scale_gradient = 4 / self.length_scale**2 * sin_of_arg**2 * K
                 length_scale_gradient = length_scale_gradient[:, :, np.newaxis]
             else:  # length_scale is kept fixed
                 length_scale_gradient = np.empty((K.shape[0], K.shape[1], 0))
             # gradient with respect to p
             if not self.hyperparameter_periodicity.fixed:
-                periodicity_gradient = \
-                    4 * arg / self.length_scale**2 * cos_of_arg \
-                    * sin_of_arg * K
+                periodicity_gradient = (
+                    4 * arg / self.length_scale**2 * cos_of_arg * sin_of_arg * K
+                )
                 periodicity_gradient = periodicity_gradient[:, :, np.newaxis]
             else:  # p is kept fixed
                 periodicity_gradient = np.empty((K.shape[0], K.shape[1], 0))
@@ -2012,7 +2090,8 @@ def __call__(self, X, Y=None, eval_gradient=False):
 
     def __repr__(self):
         return "{0}(length_scale={1:.3g}, periodicity={2:.3g})".format(
-            self.__class__.__name__, self.length_scale, self.periodicity)
+            self.__class__.__name__, self.length_scale, self.periodicity
+        )
 
 
 class DotProduct(Kernel):
@@ -2044,7 +2123,7 @@ class DotProduct(Kernel):
     ----------
     sigma_0 : float >= 0, default=1.0
         Parameter controlling the inhomogenity of the kernel. If sigma_0=0,
-        the kernel is homogenous.
+        the kernel is homogeneous.
 
     sigma_0_bounds : pair of floats >= 0 or "fixed", default=(1e-5, 1e5)
         The lower and upper bound on 'sigma_0'.
@@ -2067,10 +2146,11 @@ class DotProduct(Kernel):
     >>> gpr = GaussianProcessRegressor(kernel=kernel,
     ...         random_state=0).fit(X, y)
     >>> gpr.score(X, y)
-    0.3680...
+    0.3680
     >>> gpr.predict(X[:2,:], return_std=True)
-    (array([653.0..., 592.1...]), array([316.6..., 316.6...]))
+    (array([653.0, 592.1]), array([316.6, 316.6]))
     """
+
     def __init__(self, sigma_0=1.0, sigma_0_bounds=(1e-5, 1e5)):
         self.sigma_0 = sigma_0
         self.sigma_0_bounds = sigma_0_bounds
@@ -2109,17 +2189,16 @@ def __call__(self, X, Y=None, eval_gradient=False):
         """
         X = np.atleast_2d(X)
         if Y is None:
-            K = np.inner(X, X) + self.sigma_0 ** 2
+            K = np.inner(X, X) + self.sigma_0**2
         else:
             if eval_gradient:
-                raise ValueError(
-                    "Gradient can only be evaluated when Y is None.")
-            K = np.inner(X, Y) + self.sigma_0 ** 2
+                raise ValueError("Gradient can only be evaluated when Y is None.")
+            K = np.inner(X, Y) + self.sigma_0**2
 
         if eval_gradient:
             if not self.hyperparameter_sigma_0.fixed:
                 K_gradient = np.empty((K.shape[0], K.shape[1], 1))
-                K_gradient[..., 0] = 2 * self.sigma_0 ** 2
+                K_gradient[..., 0] = 2 * self.sigma_0**2
                 return K, K_gradient
             else:
                 return K, np.empty((X.shape[0], X.shape[0], 0))
@@ -2143,22 +2222,21 @@ def diag(self, X):
         K_diag : ndarray of shape (n_samples_X,)
             Diagonal of kernel k(X, X).
         """
-        return np.einsum('ij,ij->i', X, X) + self.sigma_0 ** 2
+        return np.einsum("ij,ij->i", X, X) + self.sigma_0**2
 
     def is_stationary(self):
-        """Returns whether the kernel is stationary. """
+        """Returns whether the kernel is stationary."""
         return False
 
     def __repr__(self):
-        return "{0}(sigma_0={1:.3g})".format(
-            self.__class__.__name__, self.sigma_0)
+        return "{0}(sigma_0={1:.3g})".format(self.__class__.__name__, self.sigma_0)
 
 
 # adapted from scipy/optimize/optimize.py for functions with 2d output
 def _approx_fprime(xk, f, epsilon, args=()):
     f0 = f(*((xk,) + args))
     grad = np.zeros((f0.shape[0], f0.shape[1], len(xk)), float)
-    ei = np.zeros((len(xk), ), float)
+    ei = np.zeros((len(xk),), float)
     for k in range(len(xk)):
         ei[k] = 1.0
         d = epsilon * ei
@@ -2218,13 +2296,19 @@ class PairwiseKernel(Kernel):
     >>> gpc = GaussianProcessClassifier(kernel=kernel,
     ...         random_state=0).fit(X, y)
     >>> gpc.score(X, y)
-    0.9733...
+    0.9733
     >>> gpc.predict_proba(X[:2,:])
-    array([[0.8880..., 0.05663..., 0.05532...],
-           [0.8676..., 0.07073..., 0.06165...]])
+    array([[0.8880, 0.05663, 0.05532],
+           [0.8676, 0.07073, 0.06165]])
     """
-    def __init__(self, gamma=1.0, gamma_bounds=(1e-5, 1e5), metric="linear",
-                 pairwise_kernels_kwargs=None):
+
+    def __init__(
+        self,
+        gamma=1.0,
+        gamma_bounds=(1e-5, 1e5),
+        metric="linear",
+        pairwise_kernels_kwargs=None,
+    ):
         self.gamma = gamma
         self.gamma_bounds = gamma_bounds
         self.metric = metric
@@ -2267,9 +2351,14 @@ def __call__(self, X, Y=None, eval_gradient=False):
             pairwise_kernels_kwargs = {}
 
         X = np.atleast_2d(X)
-        K = pairwise_kernels(X, Y, metric=self.metric, gamma=self.gamma,
-                             filter_params=True,
-                             **pairwise_kernels_kwargs)
+        K = pairwise_kernels(
+            X,
+            Y,
+            metric=self.metric,
+            gamma=self.gamma,
+            filter_params=True,
+            **pairwise_kernels_kwargs,
+        )
         if eval_gradient:
             if self.hyperparameter_gamma.fixed:
                 return K, np.empty((X.shape[0], X.shape[0], 0))
@@ -2277,8 +2366,14 @@ def __call__(self, X, Y=None, eval_gradient=False):
                 # approximate gradient numerically
                 def f(gamma):  # helper function
                     return pairwise_kernels(
-                        X, Y, metric=self.metric, gamma=np.exp(gamma),
-                        filter_params=True, **pairwise_kernels_kwargs)
+                        X,
+                        Y,
+                        metric=self.metric,
+                        gamma=np.exp(gamma),
+                        filter_params=True,
+                        **pairwise_kernels_kwargs,
+                    )
+
                 return K, _approx_fprime(self.theta, f, 1e-10)
         else:
             return K
@@ -2304,9 +2399,10 @@ def diag(self, X):
         return np.apply_along_axis(self, 1, X).ravel()
 
     def is_stationary(self):
-        """Returns whether the kernel is stationary. """
+        """Returns whether the kernel is stationary."""
         return self.metric in ["rbf"]
 
     def __repr__(self):
         return "{0}(gamma={1}, metric={2})".format(
-            self.__class__.__name__, self.gamma, self.metric)
+            self.__class__.__name__, self.gamma, self.metric
+        )
diff --git a/sklearn/gaussian_process/tests/_mini_sequence_kernel.py b/sklearn/gaussian_process/tests/_mini_sequence_kernel.py
index c260a361e1e71..4667329aff9b8 100644
--- a/sklearn/gaussian_process/tests/_mini_sequence_kernel.py
+++ b/sklearn/gaussian_process/tests/_mini_sequence_kernel.py
@@ -1,33 +1,34 @@
-from sklearn.gaussian_process.kernels import Kernel, Hyperparameter
-from sklearn.gaussian_process.kernels import GenericKernelMixin
-from sklearn.gaussian_process.kernels import StationaryKernelMixin
 import numpy as np
+
 from sklearn.base import clone
+from sklearn.gaussian_process.kernels import (
+    GenericKernelMixin,
+    Hyperparameter,
+    Kernel,
+    StationaryKernelMixin,
+)
 
 
-class MiniSeqKernel(GenericKernelMixin,
-                    StationaryKernelMixin,
-                    Kernel):
-    '''
+class MiniSeqKernel(GenericKernelMixin, StationaryKernelMixin, Kernel):
+    """
     A minimal (but valid) convolutional kernel for sequences of variable
     length.
-    '''
-    def __init__(self,
-                 baseline_similarity=0.5,
-                 baseline_similarity_bounds=(1e-5, 1)):
+    """
+
+    def __init__(self, baseline_similarity=0.5, baseline_similarity_bounds=(1e-5, 1)):
         self.baseline_similarity = baseline_similarity
         self.baseline_similarity_bounds = baseline_similarity_bounds
 
     @property
     def hyperparameter_baseline_similarity(self):
-        return Hyperparameter("baseline_similarity",
-                              "numeric",
-                              self.baseline_similarity_bounds)
+        return Hyperparameter(
+            "baseline_similarity", "numeric", self.baseline_similarity_bounds
+        )
 
     def _f(self, s1, s2):
-        return sum([1.0 if c1 == c2 else self.baseline_similarity
-                   for c1 in s1
-                   for c2 in s2])
+        return sum(
+            [1.0 if c1 == c2 else self.baseline_similarity for c1 in s1 for c2 in s2]
+        )
 
     def _g(self, s1, s2):
         return sum([0.0 if c1 == c2 else 1.0 for c1 in s1 for c2 in s2])
@@ -37,8 +38,10 @@ def __call__(self, X, Y=None, eval_gradient=False):
             Y = X
 
         if eval_gradient:
-            return (np.array([[self._f(x, y) for y in Y] for x in X]),
-                    np.array([[[self._g(x, y)] for y in Y] for x in X]))
+            return (
+                np.array([[self._f(x, y) for y in Y] for x in X]),
+                np.array([[[self._g(x, y)] for y in Y] for x in X]),
+            )
         else:
             return np.array([[self._f(x, y) for y in Y] for x in X])
 
diff --git a/sklearn/gaussian_process/tests/test_gpc.py b/sklearn/gaussian_process/tests/test_gpc.py
index 76804906f7fb4..365b8f5a11441 100644
--- a/sklearn/gaussian_process/tests/test_gpc.py
+++ b/sklearn/gaussian_process/tests/test_gpc.py
@@ -1,23 +1,26 @@
-"""Testing for Gaussian process classification """
+"""Testing for Gaussian process classification"""
 
-# Author: Jan Hendrik Metzen 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import warnings
-import numpy as np
-
-from scipy.optimize import approx_fprime
 
+import numpy as np
 import pytest
+from scipy.optimize import approx_fprime
 
+from sklearn.exceptions import ConvergenceWarning
 from sklearn.gaussian_process import GaussianProcessClassifier
-from sklearn.gaussian_process.kernels \
-    import RBF, ConstantKernel as C, WhiteKernel
+from sklearn.gaussian_process.kernels import (
+    RBF,
+    CompoundKernel,
+    WhiteKernel,
+)
+from sklearn.gaussian_process.kernels import (
+    ConstantKernel as C,
+)
 from sklearn.gaussian_process.tests._mini_sequence_kernel import MiniSeqKernel
-from sklearn.exceptions import ConvergenceWarning
-
-from sklearn.utils._testing \
-    import assert_almost_equal, assert_array_equal, assert_warns_message
+from sklearn.utils._testing import assert_almost_equal, assert_array_equal
 
 
 def f(x):
@@ -25,7 +28,7 @@ def f(x):
 
 
 X = np.atleast_2d(np.linspace(0, 10, 30)).T
-X2 = np.atleast_2d([2., 4., 5.5, 6.5, 7.5]).T
+X2 = np.atleast_2d([2.0, 4.0, 5.5, 6.5, 7.5]).T
 y = np.array(f(X).ravel() > 0, dtype=int)
 fX = f(X).ravel()
 y_mc = np.empty(y.shape, dtype=int)  # multi-class
@@ -35,49 +38,50 @@ def f(x):
 
 
 fixed_kernel = RBF(length_scale=1.0, length_scale_bounds="fixed")
-kernels = [RBF(length_scale=0.1), fixed_kernel,
-           RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)),
-           C(1.0, (1e-2, 1e2)) *
-           RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3))]
-non_fixed_kernels = [kernel for kernel in kernels
-                     if kernel != fixed_kernel]
+kernels = [
+    RBF(length_scale=0.1),
+    fixed_kernel,
+    RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)),
+    C(1.0, (1e-2, 1e2)) * RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)),
+]
+non_fixed_kernels = [kernel for kernel in kernels if kernel != fixed_kernel]
 
 
-@pytest.mark.parametrize('kernel', kernels)
+@pytest.mark.parametrize("kernel", kernels)
 def test_predict_consistent(kernel):
     # Check binary predict decision has also predicted probability above 0.5.
     gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
-    assert_array_equal(gpc.predict(X),
-                       gpc.predict_proba(X)[:, 1] >= 0.5)
+    assert_array_equal(gpc.predict(X), gpc.predict_proba(X)[:, 1] >= 0.5)
 
 
 def test_predict_consistent_structured():
     # Check binary predict decision has also predicted probability above 0.5.
-    X = ['A', 'AB', 'B']
+    X = ["A", "AB", "B"]
     y = np.array([True, False, True])
-    kernel = MiniSeqKernel(baseline_similarity_bounds='fixed')
+    kernel = MiniSeqKernel(baseline_similarity_bounds="fixed")
     gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
-    assert_array_equal(gpc.predict(X),
-                       gpc.predict_proba(X)[:, 1] >= 0.5)
+    assert_array_equal(gpc.predict(X), gpc.predict_proba(X)[:, 1] >= 0.5)
 
 
-@pytest.mark.parametrize('kernel', non_fixed_kernels)
+@pytest.mark.parametrize("kernel", non_fixed_kernels)
 def test_lml_improving(kernel):
     # Test that hyperparameter-tuning improves log-marginal likelihood.
     gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
-    assert (gpc.log_marginal_likelihood(gpc.kernel_.theta) >
-            gpc.log_marginal_likelihood(kernel.theta))
+    assert gpc.log_marginal_likelihood(gpc.kernel_.theta) > gpc.log_marginal_likelihood(
+        kernel.theta
+    )
 
 
-@pytest.mark.parametrize('kernel', kernels)
+@pytest.mark.parametrize("kernel", kernels)
 def test_lml_precomputed(kernel):
     # Test that lml of optimized kernel is stored correctly.
     gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
-    assert_almost_equal(gpc.log_marginal_likelihood(gpc.kernel_.theta),
-                        gpc.log_marginal_likelihood(), 7)
+    assert_almost_equal(
+        gpc.log_marginal_likelihood(gpc.kernel_.theta), gpc.log_marginal_likelihood(), 7
+    )
 
 
-@pytest.mark.parametrize('kernel', kernels)
+@pytest.mark.parametrize("kernel", kernels)
 def test_lml_without_cloning_kernel(kernel):
     # Test that clone_kernel=False has side-effects of kernel.theta.
     gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
@@ -87,66 +91,70 @@ def test_lml_without_cloning_kernel(kernel):
     assert_almost_equal(gpc.kernel_.theta, input_theta, 7)
 
 
-@pytest.mark.parametrize('kernel', non_fixed_kernels)
+@pytest.mark.parametrize("kernel", non_fixed_kernels)
 def test_converged_to_local_maximum(kernel):
     # Test that we are in local maximum after hyperparameter-optimization.
     gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
 
-    lml, lml_gradient = \
-        gpc.log_marginal_likelihood(gpc.kernel_.theta, True)
+    lml, lml_gradient = gpc.log_marginal_likelihood(gpc.kernel_.theta, True)
 
-    assert np.all((np.abs(lml_gradient) < 1e-4) |
-                  (gpc.kernel_.theta == gpc.kernel_.bounds[:, 0]) |
-                  (gpc.kernel_.theta == gpc.kernel_.bounds[:, 1]))
+    assert np.all(
+        (np.abs(lml_gradient) < 1e-4)
+        | (gpc.kernel_.theta == gpc.kernel_.bounds[:, 0])
+        | (gpc.kernel_.theta == gpc.kernel_.bounds[:, 1])
+    )
 
 
-@pytest.mark.parametrize('kernel', kernels)
+@pytest.mark.parametrize("kernel", kernels)
 def test_lml_gradient(kernel):
     # Compare analytic and numeric gradient of log marginal likelihood.
     gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
 
     lml, lml_gradient = gpc.log_marginal_likelihood(kernel.theta, True)
-    lml_gradient_approx = \
-        approx_fprime(kernel.theta,
-                      lambda theta: gpc.log_marginal_likelihood(theta,
-                                                                False),
-                      1e-10)
+    lml_gradient_approx = approx_fprime(
+        kernel.theta, lambda theta: gpc.log_marginal_likelihood(theta, False), 1e-10
+    )
 
     assert_almost_equal(lml_gradient, lml_gradient_approx, 3)
 
 
-def test_random_starts():
+def test_random_starts(global_random_seed):
     # Test that an increasing number of random-starts of GP fitting only
     # increases the log marginal likelihood of the chosen theta.
     n_samples, n_features = 25, 2
-    rng = np.random.RandomState(0)
+    rng = np.random.RandomState(global_random_seed)
     X = rng.randn(n_samples, n_features) * 2 - 1
     y = (np.sin(X).sum(axis=1) + np.sin(3 * X).sum(axis=1)) > 0
 
-    kernel = C(1.0, (1e-2, 1e2)) \
-        * RBF(length_scale=[1e-3] * n_features,
-              length_scale_bounds=[(1e-4, 1e+2)] * n_features)
+    kernel = C(1.0, (1e-2, 1e2)) * RBF(
+        length_scale=[1e-3] * n_features, length_scale_bounds=[(1e-4, 1e2)] * n_features
+    )
     last_lml = -np.inf
     for n_restarts_optimizer in range(5):
         gp = GaussianProcessClassifier(
-            kernel=kernel, n_restarts_optimizer=n_restarts_optimizer,
-            random_state=0).fit(X, y)
+            kernel=kernel,
+            n_restarts_optimizer=n_restarts_optimizer,
+            random_state=global_random_seed,
+        ).fit(X, y)
         lml = gp.log_marginal_likelihood(gp.kernel_.theta)
         assert lml > last_lml - np.finfo(np.float32).eps
         last_lml = lml
 
 
-@pytest.mark.parametrize('kernel', non_fixed_kernels)
-def test_custom_optimizer(kernel):
+@pytest.mark.parametrize("kernel", non_fixed_kernels)
+def test_custom_optimizer(kernel, global_random_seed):
     # Test that GPC can use externally defined optimizers.
     # Define a dummy optimizer that simply tests 10 random hyperparameters
     def optimizer(obj_func, initial_theta, bounds):
-        rng = np.random.RandomState(0)
-        theta_opt, func_min = \
-            initial_theta, obj_func(initial_theta, eval_gradient=False)
+        rng = np.random.RandomState(global_random_seed)
+        theta_opt, func_min = (
+            initial_theta,
+            obj_func(initial_theta, eval_gradient=False),
+        )
         for _ in range(10):
-            theta = np.atleast_1d(rng.uniform(np.maximum(-2, bounds[:, 0]),
-                                              np.minimum(1, bounds[:, 1])))
+            theta = np.atleast_1d(
+                rng.uniform(np.maximum(-2, bounds[:, 0]), np.minimum(1, bounds[:, 1]))
+            )
             f = obj_func(theta, eval_gradient=False)
             if f < func_min:
                 theta_opt, func_min = theta, f
@@ -155,11 +163,12 @@ def optimizer(obj_func, initial_theta, bounds):
     gpc = GaussianProcessClassifier(kernel=kernel, optimizer=optimizer)
     gpc.fit(X, y_mc)
     # Checks that optimizer improved marginal likelihood
-    assert (gpc.log_marginal_likelihood(gpc.kernel_.theta) >
-            gpc.log_marginal_likelihood(kernel.theta))
+    assert gpc.log_marginal_likelihood(
+        gpc.kernel_.theta
+    ) >= gpc.log_marginal_likelihood(kernel.theta)
 
 
-@pytest.mark.parametrize('kernel', kernels)
+@pytest.mark.parametrize("kernel", kernels)
 def test_multi_class(kernel):
     # Test GPC for multi-class classification problems.
     gpc = GaussianProcessClassifier(kernel=kernel)
@@ -172,7 +181,7 @@ def test_multi_class(kernel):
     assert_array_equal(np.argmax(y_prob, 1), y_pred)
 
 
-@pytest.mark.parametrize('kernel', kernels)
+@pytest.mark.parametrize("kernel", kernels)
 def test_multi_class_n_jobs(kernel):
     # Test that multi-class GPC produces identical results with n_jobs>1.
     gpc = GaussianProcessClassifier(kernel=kernel)
@@ -189,61 +198,123 @@ def test_multi_class_n_jobs(kernel):
 def test_warning_bounds():
     kernel = RBF(length_scale_bounds=[1e-5, 1e-3])
     gpc = GaussianProcessClassifier(kernel=kernel)
-    assert_warns_message(ConvergenceWarning, "The optimal value found for "
-                                             "dimension 0 of parameter "
-                                             "length_scale is close to "
-                                             "the specified upper bound "
-                                             "0.001. Increasing the bound "
-                                             "and calling fit again may "
-                                             "find a better value.",
-                         gpc.fit, X, y)
-
-    kernel_sum = (WhiteKernel(noise_level_bounds=[1e-5, 1e-3]) +
-                  RBF(length_scale_bounds=[1e3, 1e5]))
+    warning_message = (
+        "The optimal value found for dimension 0 of parameter "
+        "length_scale is close to the specified upper bound "
+        "0.001. Increasing the bound and calling fit again may "
+        "find a better value."
+    )
+    with pytest.warns(ConvergenceWarning, match=warning_message):
+        gpc.fit(X, y)
+
+    kernel_sum = WhiteKernel(noise_level_bounds=[1e-5, 1e-3]) + RBF(
+        length_scale_bounds=[1e3, 1e5]
+    )
     gpc_sum = GaussianProcessClassifier(kernel=kernel_sum)
-    with pytest.warns(None) as record:
-        with warnings.catch_warnings():
-            # scipy 1.3.0 uses tostring which is deprecated in numpy
-            warnings.filterwarnings("ignore", "tostring", DeprecationWarning)
-            gpc_sum.fit(X, y)
-
-    assert len(record) == 2
-    assert record[0].message.args[0] == ("The optimal value found for "
-                                         "dimension 0 of parameter "
-                                         "k1__noise_level is close to the "
-                                         "specified upper bound 0.001. "
-                                         "Increasing the bound and calling "
-                                         "fit again may find a better value.")
-
-    assert record[1].message.args[0] == ("The optimal value found for "
-                                         "dimension 0 of parameter "
-                                         "k2__length_scale is close to the "
-                                         "specified lower bound 1000.0. "
-                                         "Decreasing the bound and calling "
-                                         "fit again may find a better value.")
+    with warnings.catch_warnings(record=True) as record:
+        warnings.simplefilter("always")
+        gpc_sum.fit(X, y)
+
+        assert len(record) == 2
+
+        assert issubclass(record[0].category, ConvergenceWarning)
+        assert (
+            record[0].message.args[0] == "The optimal value found for "
+            "dimension 0 of parameter "
+            "k1__noise_level is close to the "
+            "specified upper bound 0.001. "
+            "Increasing the bound and calling "
+            "fit again may find a better value."
+        )
+
+        assert issubclass(record[1].category, ConvergenceWarning)
+        assert (
+            record[1].message.args[0] == "The optimal value found for "
+            "dimension 0 of parameter "
+            "k2__length_scale is close to the "
+            "specified lower bound 1000.0. "
+            "Decreasing the bound and calling "
+            "fit again may find a better value."
+        )
 
     X_tile = np.tile(X, 2)
-    kernel_dims = RBF(length_scale=[1., 2.],
-                      length_scale_bounds=[1e1, 1e2])
+    kernel_dims = RBF(length_scale=[1.0, 2.0], length_scale_bounds=[1e1, 1e2])
     gpc_dims = GaussianProcessClassifier(kernel=kernel_dims)
 
-    with pytest.warns(None) as record:
-        with warnings.catch_warnings():
-            # scipy 1.3.0 uses tostring which is deprecated in numpy
-            warnings.filterwarnings("ignore", "tostring", DeprecationWarning)
-            gpc_dims.fit(X_tile, y)
-
-    assert len(record) == 2
-    assert record[0].message.args[0] == ("The optimal value found for "
-                                         "dimension 0 of parameter "
-                                         "length_scale is close to the "
-                                         "specified upper bound 100.0. "
-                                         "Increasing the bound and calling "
-                                         "fit again may find a better value.")
-
-    assert record[1].message.args[0] == ("The optimal value found for "
-                                         "dimension 1 of parameter "
-                                         "length_scale is close to the "
-                                         "specified upper bound 100.0. "
-                                         "Increasing the bound and calling "
-                                         "fit again may find a better value.")
+    with warnings.catch_warnings(record=True) as record:
+        warnings.simplefilter("always")
+        gpc_dims.fit(X_tile, y)
+
+        assert len(record) == 2
+
+        assert issubclass(record[0].category, ConvergenceWarning)
+        assert (
+            record[0].message.args[0] == "The optimal value found for "
+            "dimension 0 of parameter "
+            "length_scale is close to the "
+            "specified upper bound 100.0. "
+            "Increasing the bound and calling "
+            "fit again may find a better value."
+        )
+
+        assert issubclass(record[1].category, ConvergenceWarning)
+        assert (
+            record[1].message.args[0] == "The optimal value found for "
+            "dimension 1 of parameter "
+            "length_scale is close to the "
+            "specified upper bound 100.0. "
+            "Increasing the bound and calling "
+            "fit again may find a better value."
+        )
+
+
+@pytest.mark.parametrize(
+    "params, error_type, err_msg",
+    [
+        (
+            {"kernel": CompoundKernel(0)},
+            ValueError,
+            "kernel cannot be a CompoundKernel",
+        )
+    ],
+)
+def test_gpc_fit_error(params, error_type, err_msg):
+    """Check that expected error are raised during fit."""
+    gpc = GaussianProcessClassifier(**params)
+    with pytest.raises(error_type, match=err_msg):
+        gpc.fit(X, y)
+
+
+@pytest.mark.parametrize("kernel", kernels)
+def test_gpc_latent_mean_and_variance_shape(kernel):
+    """Checks that the latent mean and variance have the right shape."""
+    gpc = GaussianProcessClassifier(kernel=kernel)
+    gpc.fit(X, y)
+
+    # Check that the latent mean and variance have the right shape
+    latent_mean, latent_variance = gpc.latent_mean_and_variance(X)
+    assert latent_mean.shape == (X.shape[0],)
+    assert latent_variance.shape == (X.shape[0],)
+
+
+def test_gpc_latent_mean_and_variance_complain_on_more_than_2_classes():
+    """Checks that the latent mean and variance have the right shape."""
+    gpc = GaussianProcessClassifier(kernel=RBF())
+    gpc.fit(X, y_mc)
+
+    # Check that the latent mean and variance have the right shape
+    with pytest.raises(
+        ValueError,
+        match="Returning the mean and variance of the latent function f "
+        "is only supported for binary classification",
+    ):
+        gpc.latent_mean_and_variance(X)
+
+
+def test_latent_mean_and_variance_works_on_structured_kernels():
+    X = ["A", "AB", "B"]
+    y = np.array([True, False, True])
+    kernel = MiniSeqKernel(baseline_similarity_bounds="fixed")
+    gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
+
+    gpc.latent_mean_and_variance(X)
diff --git a/sklearn/gaussian_process/tests/test_gpr.py b/sklearn/gaussian_process/tests/test_gpr.py
index 83c24c7cc8573..f43cc3613b3ff 100644
--- a/sklearn/gaussian_process/tests/test_gpr.py
+++ b/sklearn/gaussian_process/tests/test_gpr.py
@@ -1,101 +1,108 @@
-"""Testing for Gaussian process regression """
+"""Testing for Gaussian process regression"""
 
-# Author: Jan Hendrik Metzen 
-# Modified by: Pete Green 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
+import re
 import sys
-import numpy as np
 import warnings
 
-from scipy.optimize import approx_fprime
-
+import numpy as np
 import pytest
+from scipy.optimize import approx_fprime
 
+from sklearn.exceptions import ConvergenceWarning
 from sklearn.gaussian_process import GaussianProcessRegressor
-from sklearn.gaussian_process.kernels \
-    import RBF, ConstantKernel as C, WhiteKernel
-from sklearn.gaussian_process.kernels import DotProduct, ExpSineSquared
+from sklearn.gaussian_process.kernels import (
+    RBF,
+    DotProduct,
+    ExpSineSquared,
+    WhiteKernel,
+)
+from sklearn.gaussian_process.kernels import (
+    ConstantKernel as C,
+)
 from sklearn.gaussian_process.tests._mini_sequence_kernel import MiniSeqKernel
-from sklearn.exceptions import ConvergenceWarning
-
-from sklearn.utils._testing \
-    import (assert_array_less,
-            assert_almost_equal, assert_raise_message,
-            assert_array_almost_equal, assert_array_equal,
-            assert_allclose, assert_warns_message)
+from sklearn.utils._testing import (
+    assert_allclose,
+    assert_almost_equal,
+    assert_array_almost_equal,
+    assert_array_less,
+)
 
 
 def f(x):
     return x * np.sin(x)
 
 
-X = np.atleast_2d([1., 3., 5., 6., 7., 8.]).T
-X2 = np.atleast_2d([2., 4., 5.5, 6.5, 7.5]).T
+X = np.atleast_2d([1.0, 3.0, 5.0, 6.0, 7.0, 8.0]).T
+X2 = np.atleast_2d([2.0, 4.0, 5.5, 6.5, 7.5]).T
 y = f(X).ravel()
 
 fixed_kernel = RBF(length_scale=1.0, length_scale_bounds="fixed")
-kernels = [RBF(length_scale=1.0), fixed_kernel,
-           RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)),
-           C(1.0, (1e-2, 1e2)) *
-           RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)),
-           C(1.0, (1e-2, 1e2)) *
-           RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)) +
-           C(1e-5, (1e-5, 1e2)),
-           C(0.1, (1e-2, 1e2)) *
-           RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)) +
-           C(1e-5, (1e-5, 1e2))]
-non_fixed_kernels = [kernel for kernel in kernels
-                     if kernel != fixed_kernel]
-
-
-@pytest.mark.parametrize('kernel', kernels)
+kernels = [
+    RBF(length_scale=1.0),
+    fixed_kernel,
+    RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)),
+    C(1.0, (1e-2, 1e2)) * RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)),
+    C(1.0, (1e-2, 1e2)) * RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3))
+    + C(1e-5, (1e-5, 1e2)),
+    C(0.1, (1e-2, 1e2)) * RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3))
+    + C(1e-5, (1e-5, 1e2)),
+]
+non_fixed_kernels = [kernel for kernel in kernels if kernel != fixed_kernel]
+
+
+@pytest.mark.parametrize("kernel", kernels)
 def test_gpr_interpolation(kernel):
-    if sys.maxsize <= 2 ** 32 and sys.version_info[:2] == (3, 6):
-        pytest.xfail("This test may fail on 32bit Py3.6")
+    if sys.maxsize <= 2**32:
+        pytest.xfail("This test may fail on 32 bit Python")
 
     # Test the interpolating property for different kernels.
     gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
     y_pred, y_cov = gpr.predict(X, return_cov=True)
 
     assert_almost_equal(y_pred, y)
-    assert_almost_equal(np.diag(y_cov), 0.)
+    assert_almost_equal(np.diag(y_cov), 0.0)
 
 
 def test_gpr_interpolation_structured():
     # Test the interpolating property for different kernels.
-    kernel = MiniSeqKernel(baseline_similarity_bounds='fixed')
-    X = ['A', 'B', 'C']
+    kernel = MiniSeqKernel(baseline_similarity_bounds="fixed")
+    X = ["A", "B", "C"]
     y = np.array([1, 2, 3])
     gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
     y_pred, y_cov = gpr.predict(X, return_cov=True)
 
-    assert_almost_equal(kernel(X, eval_gradient=True)[1].ravel(),
-                        (1 - np.eye(len(X))).ravel())
+    assert_almost_equal(
+        kernel(X, eval_gradient=True)[1].ravel(), (1 - np.eye(len(X))).ravel()
+    )
     assert_almost_equal(y_pred, y)
-    assert_almost_equal(np.diag(y_cov), 0.)
+    assert_almost_equal(np.diag(y_cov), 0.0)
 
 
-@pytest.mark.parametrize('kernel', non_fixed_kernels)
+@pytest.mark.parametrize("kernel", non_fixed_kernels)
 def test_lml_improving(kernel):
-    if sys.maxsize <= 2 ** 32 and sys.version_info[:2] == (3, 6):
-        pytest.xfail("This test may fail on 32bit Py3.6")
+    if sys.maxsize <= 2**32:
+        pytest.xfail("This test may fail on 32 bit Python")
 
     # Test that hyperparameter-tuning improves log-marginal likelihood.
     gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
-    assert (gpr.log_marginal_likelihood(gpr.kernel_.theta) >
-            gpr.log_marginal_likelihood(kernel.theta))
+    assert gpr.log_marginal_likelihood(gpr.kernel_.theta) > gpr.log_marginal_likelihood(
+        kernel.theta
+    )
 
 
-@pytest.mark.parametrize('kernel', kernels)
+@pytest.mark.parametrize("kernel", kernels)
 def test_lml_precomputed(kernel):
     # Test that lml of optimized kernel is stored correctly.
     gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
-    assert (gpr.log_marginal_likelihood(gpr.kernel_.theta) ==
-            gpr.log_marginal_likelihood())
+    assert gpr.log_marginal_likelihood(gpr.kernel_.theta) == pytest.approx(
+        gpr.log_marginal_likelihood()
+    )
 
 
-@pytest.mark.parametrize('kernel', kernels)
+@pytest.mark.parametrize("kernel", kernels)
 def test_lml_without_cloning_kernel(kernel):
     # Test that lml of optimized kernel is stored correctly.
     gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
@@ -105,20 +112,21 @@ def test_lml_without_cloning_kernel(kernel):
     assert_almost_equal(gpr.kernel_.theta, input_theta, 7)
 
 
-@pytest.mark.parametrize('kernel', non_fixed_kernels)
+@pytest.mark.parametrize("kernel", non_fixed_kernels)
 def test_converged_to_local_maximum(kernel):
     # Test that we are in local maximum after hyperparameter-optimization.
     gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
 
-    lml, lml_gradient = \
-        gpr.log_marginal_likelihood(gpr.kernel_.theta, True)
+    lml, lml_gradient = gpr.log_marginal_likelihood(gpr.kernel_.theta, True)
 
-    assert np.all((np.abs(lml_gradient) < 1e-4) |
-                  (gpr.kernel_.theta == gpr.kernel_.bounds[:, 0]) |
-                  (gpr.kernel_.theta == gpr.kernel_.bounds[:, 1]))
+    assert np.all(
+        (np.abs(lml_gradient) < 1e-4)
+        | (gpr.kernel_.theta == gpr.kernel_.bounds[:, 0])
+        | (gpr.kernel_.theta == gpr.kernel_.bounds[:, 1])
+    )
 
 
-@pytest.mark.parametrize('kernel', non_fixed_kernels)
+@pytest.mark.parametrize("kernel", non_fixed_kernels)
 def test_solution_inside_bounds(kernel):
     # Test that hyperparameter-optimization remains in bounds#
     gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
@@ -132,22 +140,20 @@ def test_solution_inside_bounds(kernel):
     assert_array_less(gpr.kernel_.theta, bounds[:, 1] + tiny)
 
 
-@pytest.mark.parametrize('kernel', kernels)
+@pytest.mark.parametrize("kernel", kernels)
 def test_lml_gradient(kernel):
     # Compare analytic and numeric gradient of log marginal likelihood.
     gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
 
     lml, lml_gradient = gpr.log_marginal_likelihood(kernel.theta, True)
-    lml_gradient_approx = \
-        approx_fprime(kernel.theta,
-                      lambda theta: gpr.log_marginal_likelihood(theta,
-                                                                False),
-                      1e-10)
+    lml_gradient_approx = approx_fprime(
+        kernel.theta, lambda theta: gpr.log_marginal_likelihood(theta, False), 1e-10
+    )
 
     assert_almost_equal(lml_gradient, lml_gradient_approx, 3)
 
 
-@pytest.mark.parametrize('kernel', kernels)
+@pytest.mark.parametrize("kernel", kernels)
 def test_prior(kernel):
     # Test that GP prior has mean 0 and identical variances.
     gpr = GaussianProcessRegressor(kernel=kernel)
@@ -162,7 +168,7 @@ def test_prior(kernel):
         assert_almost_equal(np.diag(y_cov), 1, 5)
 
 
-@pytest.mark.parametrize('kernel', kernels)
+@pytest.mark.parametrize("kernel", kernels)
 def test_sample_statistics(kernel):
     # Test that statistics of samples drawn from GP are correct.
     gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
@@ -173,8 +179,11 @@ def test_sample_statistics(kernel):
 
     # More digits accuracy would require many more samples
     assert_almost_equal(y_mean, np.mean(samples, 1), 1)
-    assert_almost_equal(np.diag(y_cov) / np.diag(y_cov).max(),
-                        np.var(samples, 1) / np.diag(y_cov).max(), 1)
+    assert_almost_equal(
+        np.diag(y_cov) / np.diag(y_cov).max(),
+        np.var(samples, 1) / np.diag(y_cov).max(),
+        1,
+    )
 
 
 def test_no_optimizer():
@@ -184,10 +193,11 @@ def test_no_optimizer():
     assert np.exp(gpr.kernel_.theta) == 1.0
 
 
-@pytest.mark.parametrize('kernel', kernels)
-def test_predict_cov_vs_std(kernel):
-    if sys.maxsize <= 2 ** 32 and sys.version_info[:2] == (3, 6):
-        pytest.xfail("This test may fail on 32bit Py3.6")
+@pytest.mark.parametrize("kernel", kernels)
+@pytest.mark.parametrize("target", [y, np.ones(X.shape[0], dtype=np.float64)])
+def test_predict_cov_vs_std(kernel, target):
+    if sys.maxsize <= 2**32:
+        pytest.xfail("This test may fail on 32 bit Python")
 
     # Test that predicted std.-dev. is consistent with cov's diagonal.
     gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
@@ -207,8 +217,7 @@ def test_anisotropic_kernel():
 
     kernel = RBF([1.0, 1.0])
     gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
-    assert (np.exp(gpr.kernel_.theta[1]) >
-            np.exp(gpr.kernel_.theta[0]) * 5)
+    assert np.exp(gpr.kernel_.theta[1]) > np.exp(gpr.kernel_.theta[0]) * 5
 
 
 def test_random_starts():
@@ -217,24 +226,28 @@ def test_random_starts():
     n_samples, n_features = 25, 2
     rng = np.random.RandomState(0)
     X = rng.randn(n_samples, n_features) * 2 - 1
-    y = np.sin(X).sum(axis=1) + np.sin(3 * X).sum(axis=1) \
+    y = (
+        np.sin(X).sum(axis=1)
+        + np.sin(3 * X).sum(axis=1)
         + rng.normal(scale=0.1, size=n_samples)
+    )
 
-    kernel = C(1.0, (1e-2, 1e2)) \
-        * RBF(length_scale=[1.0] * n_features,
-              length_scale_bounds=[(1e-4, 1e+2)] * n_features) \
-        + WhiteKernel(noise_level=1e-5, noise_level_bounds=(1e-5, 1e1))
+    kernel = C(1.0, (1e-2, 1e2)) * RBF(
+        length_scale=[1.0] * n_features, length_scale_bounds=[(1e-4, 1e2)] * n_features
+    ) + WhiteKernel(noise_level=1e-5, noise_level_bounds=(1e-5, 1e1))
     last_lml = -np.inf
     for n_restarts_optimizer in range(5):
         gp = GaussianProcessRegressor(
-            kernel=kernel, n_restarts_optimizer=n_restarts_optimizer,
-            random_state=0,).fit(X, y)
+            kernel=kernel,
+            n_restarts_optimizer=n_restarts_optimizer,
+            random_state=0,
+        ).fit(X, y)
         lml = gp.log_marginal_likelihood(gp.kernel_.theta)
         assert lml > last_lml - np.finfo(np.float32).eps
         last_lml = lml
 
 
-@pytest.mark.parametrize('kernel', kernels)
+@pytest.mark.parametrize("kernel", kernels)
 def test_y_normalization(kernel):
     """
     Test normalization of the target values in GP
@@ -302,25 +315,21 @@ def test_large_variance_y():
     y_large = 10 * y
 
     # Standard GP with normalize_y=True
-    RBF_params = {'length_scale': 1.0}
+    RBF_params = {"length_scale": 1.0}
     kernel = RBF(**RBF_params)
     gpr = GaussianProcessRegressor(kernel=kernel, normalize_y=True)
     gpr.fit(X, y_large)
     y_pred, y_pred_std = gpr.predict(X2, return_std=True)
 
     # 'Gold standard' mean predictions from GPy
-    y_pred_gpy = np.array([15.16918303,
-                           -27.98707845,
-                           -39.31636019,
-                           14.52605515,
-                           69.18503589])
+    y_pred_gpy = np.array(
+        [15.16918303, -27.98707845, -39.31636019, 14.52605515, 69.18503589]
+    )
 
     # 'Gold standard' std predictions from GPy
-    y_pred_std_gpy = np.array([7.78860962,
-                               3.83179178,
-                               0.63149951,
-                               0.52745188,
-                               0.86170042])
+    y_pred_std_gpy = np.array(
+        [7.78860962, 3.83179178, 0.63149951, 0.52745188, 0.86170042]
+    )
 
     # Based on numerical experiments, it's reasonable to expect our
     # GP's mean predictions to get within 7% of predictions of those
@@ -341,12 +350,10 @@ def test_y_multioutput():
     # of 1d GP and that second dimension is twice as large
     kernel = RBF(length_scale=1.0)
 
-    gpr = GaussianProcessRegressor(kernel=kernel, optimizer=None,
-                                   normalize_y=False)
+    gpr = GaussianProcessRegressor(kernel=kernel, optimizer=None, normalize_y=False)
     gpr.fit(X, y)
 
-    gpr_2d = GaussianProcessRegressor(kernel=kernel, optimizer=None,
-                                      normalize_y=False)
+    gpr_2d = GaussianProcessRegressor(kernel=kernel, optimizer=None, normalize_y=False)
     gpr_2d.fit(X, y_2d)
 
     y_pred_1d, y_std_1d = gpr.predict(X2, return_std=True)
@@ -358,12 +365,17 @@ def test_y_multioutput():
     assert_almost_equal(y_pred_1d, y_pred_2d[:, 1] / 2)
 
     # Standard deviation and covariance do not depend on output
-    assert_almost_equal(y_std_1d, y_std_2d)
-    assert_almost_equal(y_cov_1d, y_cov_2d)
+    for target in range(y_2d.shape[1]):
+        assert_almost_equal(y_std_1d, y_std_2d[..., target])
+        assert_almost_equal(y_cov_1d, y_cov_2d[..., target])
 
     y_sample_1d = gpr.sample_y(X2, n_samples=10)
     y_sample_2d = gpr_2d.sample_y(X2, n_samples=10)
-    assert_almost_equal(y_sample_1d, y_sample_2d[:, 0])
+
+    assert y_sample_1d.shape == (5, 10)
+    assert y_sample_2d.shape == (5, 2, 10)
+    # Only the first target will be equal
+    assert_almost_equal(y_sample_1d, y_sample_2d[:, 0, :])
 
     # Test hyperparameter optimization
     for kernel in kernels:
@@ -376,17 +388,20 @@ def test_y_multioutput():
         assert_almost_equal(gpr.kernel_.theta, gpr_2d.kernel_.theta, 4)
 
 
-@pytest.mark.parametrize('kernel', non_fixed_kernels)
+@pytest.mark.parametrize("kernel", non_fixed_kernels)
 def test_custom_optimizer(kernel):
     # Test that GPR can use externally defined optimizers.
     # Define a dummy optimizer that simply tests 50 random hyperparameters
     def optimizer(obj_func, initial_theta, bounds):
         rng = np.random.RandomState(0)
-        theta_opt, func_min = \
-            initial_theta, obj_func(initial_theta, eval_gradient=False)
+        theta_opt, func_min = (
+            initial_theta,
+            obj_func(initial_theta, eval_gradient=False),
+        )
         for _ in range(50):
-            theta = np.atleast_1d(rng.uniform(np.maximum(-2, bounds[:, 0]),
-                                              np.minimum(1, bounds[:, 1])))
+            theta = np.atleast_1d(
+                rng.uniform(np.maximum(-2, bounds[:, 0]), np.minimum(1, bounds[:, 1]))
+            )
             f = obj_func(theta, eval_gradient=False)
             if f < func_min:
                 theta_opt, func_min = theta, f
@@ -395,8 +410,9 @@ def optimizer(obj_func, initial_theta, bounds):
     gpr = GaussianProcessRegressor(kernel=kernel, optimizer=optimizer)
     gpr.fit(X, y)
     # Checks that optimizer improved marginal likelihood
-    assert (gpr.log_marginal_likelihood(gpr.kernel_.theta) >
-            gpr.log_marginal_likelihood(gpr.kernel.theta))
+    assert gpr.log_marginal_likelihood(gpr.kernel_.theta) > gpr.log_marginal_likelihood(
+        gpr.kernel.theta
+    )
 
 
 def test_gpr_correct_error_message():
@@ -404,15 +420,17 @@ def test_gpr_correct_error_message():
     y = np.ones(6)
     kernel = DotProduct()
     gpr = GaussianProcessRegressor(kernel=kernel, alpha=0.0)
-    assert_raise_message(np.linalg.LinAlgError,
-                         "The kernel, %s, is not returning a "
-                         "positive definite matrix. Try gradually increasing "
-                         "the 'alpha' parameter of your "
-                         "GaussianProcessRegressor estimator."
-                         % kernel, gpr.fit, X, y)
+    message = (
+        "The kernel, %s, is not returning a "
+        "positive definite matrix. Try gradually increasing "
+        "the 'alpha' parameter of your "
+        "GaussianProcessRegressor estimator." % kernel
+    )
+    with pytest.raises(np.linalg.LinAlgError, match=re.escape(message)):
+        gpr.fit(X, y)
 
 
-@pytest.mark.parametrize('kernel', kernels)
+@pytest.mark.parametrize("kernel", kernels)
 def test_duplicate_input(kernel):
     # Test GPR can handle two different output-values for the same input.
     gpr_equal_inputs = GaussianProcessRegressor(kernel=kernel, alpha=1e-2)
@@ -427,10 +445,8 @@ def test_duplicate_input(kernel):
     gpr_similar_inputs.fit(X_, y_)
 
     X_test = np.linspace(0, 10, 100)[:, None]
-    y_pred_equal, y_std_equal = \
-        gpr_equal_inputs.predict(X_test, return_std=True)
-    y_pred_similar, y_std_similar = \
-        gpr_similar_inputs.predict(X_test, return_std=True)
+    y_pred_equal, y_std_equal = gpr_equal_inputs.predict(X_test, return_std=True)
+    y_pred_similar, y_std_similar = gpr_similar_inputs.predict(X_test, return_std=True)
 
     assert_almost_equal(y_pred_equal, y_pred_similar)
     assert_almost_equal(y_std_equal, y_std_similar)
@@ -438,8 +454,9 @@ def test_duplicate_input(kernel):
 
 def test_no_fit_default_predict():
     # Test that GPR predictions without fit does not break by default.
-    default_kernel = (C(1.0, constant_value_bounds="fixed") *
-                      RBF(1.0, length_scale_bounds="fixed"))
+    default_kernel = C(1.0, constant_value_bounds="fixed") * RBF(
+        1.0, length_scale_bounds="fixed"
+    )
     gpr1 = GaussianProcessRegressor()
     _, y_std1 = gpr1.predict(X, return_std=True)
     _, y_cov1 = gpr1.predict(X, return_cov=True)
@@ -452,86 +469,77 @@ def test_no_fit_default_predict():
     assert_array_almost_equal(y_cov1, y_cov2)
 
 
-@pytest.mark.parametrize('kernel', kernels)
-def test_K_inv_reset(kernel):
-    y2 = f(X2).ravel()
-
-    # Test that self._K_inv is reset after a new fit
-    gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
-    assert hasattr(gpr, '_K_inv')
-    assert gpr._K_inv is None
-    gpr.predict(X, return_std=True)
-    assert gpr._K_inv is not None
-    gpr.fit(X2, y2)
-    assert gpr._K_inv is None
-    gpr.predict(X2, return_std=True)
-    gpr2 = GaussianProcessRegressor(kernel=kernel).fit(X2, y2)
-    gpr2.predict(X2, return_std=True)
-    # the value of K_inv should be independent of the first fit
-    assert_array_equal(gpr._K_inv, gpr2._K_inv)
-
-
 def test_warning_bounds():
     kernel = RBF(length_scale_bounds=[1e-5, 1e-3])
     gpr = GaussianProcessRegressor(kernel=kernel)
-    assert_warns_message(ConvergenceWarning, "The optimal value found for "
-                                             "dimension 0 of parameter "
-                                             "length_scale is close to "
-                                             "the specified upper bound "
-                                             "0.001. Increasing the bound "
-                                             "and calling fit again may "
-                                             "find a better value.",
-                         gpr.fit, X, y)
-
-    kernel_sum = (WhiteKernel(noise_level_bounds=[1e-5, 1e-3]) +
-                  RBF(length_scale_bounds=[1e3, 1e5]))
+    warning_message = (
+        "The optimal value found for dimension 0 of parameter "
+        "length_scale is close to the specified upper bound "
+        "0.001. Increasing the bound and calling fit again may "
+        "find a better value."
+    )
+    with pytest.warns(ConvergenceWarning, match=warning_message):
+        gpr.fit(X, y)
+
+    kernel_sum = WhiteKernel(noise_level_bounds=[1e-5, 1e-3]) + RBF(
+        length_scale_bounds=[1e3, 1e5]
+    )
     gpr_sum = GaussianProcessRegressor(kernel=kernel_sum)
-    with pytest.warns(None) as record:
-        with warnings.catch_warnings():
-            # scipy 1.3.0 uses tostring which is deprecated in numpy
-            warnings.filterwarnings("ignore", "tostring", DeprecationWarning)
-            gpr_sum.fit(X, y)
-
-    assert len(record) == 2
-    assert record[0].message.args[0] == ("The optimal value found for "
-                                         "dimension 0 of parameter "
-                                         "k1__noise_level is close to the "
-                                         "specified upper bound 0.001. "
-                                         "Increasing the bound and calling "
-                                         "fit again may find a better value.")
-
-    assert record[1].message.args[0] == ("The optimal value found for "
-                                         "dimension 0 of parameter "
-                                         "k2__length_scale is close to the "
-                                         "specified lower bound 1000.0. "
-                                         "Decreasing the bound and calling "
-                                         "fit again may find a better value.")
+    with warnings.catch_warnings(record=True) as record:
+        warnings.simplefilter("always")
+        gpr_sum.fit(X, y)
+
+        assert len(record) == 2
+
+        assert issubclass(record[0].category, ConvergenceWarning)
+        assert (
+            record[0].message.args[0] == "The optimal value found for "
+            "dimension 0 of parameter "
+            "k1__noise_level is close to the "
+            "specified upper bound 0.001. "
+            "Increasing the bound and calling "
+            "fit again may find a better value."
+        )
+
+        assert issubclass(record[1].category, ConvergenceWarning)
+        assert (
+            record[1].message.args[0] == "The optimal value found for "
+            "dimension 0 of parameter "
+            "k2__length_scale is close to the "
+            "specified lower bound 1000.0. "
+            "Decreasing the bound and calling "
+            "fit again may find a better value."
+        )
 
     X_tile = np.tile(X, 2)
-    kernel_dims = RBF(length_scale=[1., 2.],
-                      length_scale_bounds=[1e1, 1e2])
+    kernel_dims = RBF(length_scale=[1.0, 2.0], length_scale_bounds=[1e1, 1e2])
     gpr_dims = GaussianProcessRegressor(kernel=kernel_dims)
 
-    with pytest.warns(None) as record:
-        with warnings.catch_warnings():
-            # scipy 1.3.0 uses tostring which is deprecated in numpy
-            warnings.filterwarnings("ignore", "tostring", DeprecationWarning)
-            gpr_dims.fit(X_tile, y)
-
-    assert len(record) == 2
-    assert record[0].message.args[0] == ("The optimal value found for "
-                                         "dimension 0 of parameter "
-                                         "length_scale is close to the "
-                                         "specified lower bound 10.0. "
-                                         "Decreasing the bound and calling "
-                                         "fit again may find a better value.")
-
-    assert record[1].message.args[0] == ("The optimal value found for "
-                                         "dimension 1 of parameter "
-                                         "length_scale is close to the "
-                                         "specified lower bound 10.0. "
-                                         "Decreasing the bound and calling "
-                                         "fit again may find a better value.")
+    with warnings.catch_warnings(record=True) as record:
+        warnings.simplefilter("always")
+        gpr_dims.fit(X_tile, y)
+
+        assert len(record) == 2
+
+        assert issubclass(record[0].category, ConvergenceWarning)
+        assert (
+            record[0].message.args[0] == "The optimal value found for "
+            "dimension 0 of parameter "
+            "length_scale is close to the "
+            "specified lower bound 10.0. "
+            "Decreasing the bound and calling "
+            "fit again may find a better value."
+        )
+
+        assert issubclass(record[1].category, ConvergenceWarning)
+        assert (
+            record[1].message.args[0] == "The optimal value found for "
+            "dimension 1 of parameter "
+            "length_scale is close to the "
+            "specified lower bound 10.0. "
+            "Decreasing the bound and calling "
+            "fit again may find a better value."
+        )
 
 
 def test_bound_check_fixed_hyperparameter():
@@ -539,7 +547,303 @@ def test_bound_check_fixed_hyperparameter():
     # Check that having a hyperparameter with fixed bounds doesn't cause an
     # error
     k1 = 50.0**2 * RBF(length_scale=50.0)  # long term smooth rising trend
-    k2 = ExpSineSquared(length_scale=1.0, periodicity=1.0,
-                        periodicity_bounds="fixed")  # seasonal component
+    k2 = ExpSineSquared(
+        length_scale=1.0, periodicity=1.0, periodicity_bounds="fixed"
+    )  # seasonal component
     kernel = k1 + k2
     GaussianProcessRegressor(kernel=kernel).fit(X, y)
+
+
+@pytest.mark.parametrize("kernel", kernels)
+def test_constant_target(kernel):
+    """Check that the std. dev. is affected to 1 when normalizing a constant
+    feature.
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/18318
+    NaN where affected to the target when scaling due to null std. dev. with
+    constant target.
+    """
+    y_constant = np.ones(X.shape[0], dtype=np.float64)
+
+    gpr = GaussianProcessRegressor(kernel=kernel, normalize_y=True)
+    gpr.fit(X, y_constant)
+    assert gpr._y_train_std == pytest.approx(1.0)
+
+    y_pred, y_cov = gpr.predict(X, return_cov=True)
+    assert_allclose(y_pred, y_constant)
+    # set atol because we compare to zero
+    assert_allclose(np.diag(y_cov), 0.0, atol=1e-9)
+
+    # Test multi-target data
+    n_samples, n_targets = X.shape[0], 2
+    rng = np.random.RandomState(0)
+    y = np.concatenate(
+        [
+            rng.normal(size=(n_samples, 1)),  # non-constant target
+            np.full(shape=(n_samples, 1), fill_value=2),  # constant target
+        ],
+        axis=1,
+    )
+
+    gpr.fit(X, y)
+    Y_pred, Y_cov = gpr.predict(X, return_cov=True)
+
+    assert_allclose(Y_pred[:, 1], 2)
+    assert_allclose(np.diag(Y_cov[..., 1]), 0.0, atol=1e-9)
+
+    assert Y_pred.shape == (n_samples, n_targets)
+    assert Y_cov.shape == (n_samples, n_samples, n_targets)
+
+
+def test_gpr_consistency_std_cov_non_invertible_kernel():
+    """Check the consistency between the returned std. dev. and the covariance.
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/19936
+    Inconsistencies were observed when the kernel cannot be inverted (or
+    numerically stable).
+    """
+    kernel = C(8.98576054e05, (1e-12, 1e12)) * RBF(
+        [5.91326520e02, 1.32584051e03], (1e-12, 1e12)
+    ) + WhiteKernel(noise_level=1e-5)
+    gpr = GaussianProcessRegressor(kernel=kernel, alpha=0, optimizer=None)
+    X_train = np.array(
+        [
+            [0.0, 0.0],
+            [1.54919334, -0.77459667],
+            [-1.54919334, 0.0],
+            [0.0, -1.54919334],
+            [0.77459667, 0.77459667],
+            [-0.77459667, 1.54919334],
+        ]
+    )
+    y_train = np.array(
+        [
+            [-2.14882017e-10],
+            [-4.66975823e00],
+            [4.01823986e00],
+            [-1.30303674e00],
+            [-1.35760156e00],
+            [3.31215668e00],
+        ]
+    )
+    gpr.fit(X_train, y_train)
+    X_test = np.array(
+        [
+            [-1.93649167, -1.93649167],
+            [1.93649167, -1.93649167],
+            [-1.93649167, 1.93649167],
+            [1.93649167, 1.93649167],
+        ]
+    )
+    pred1, std = gpr.predict(X_test, return_std=True)
+    pred2, cov = gpr.predict(X_test, return_cov=True)
+    assert_allclose(std, np.sqrt(np.diagonal(cov)), rtol=1e-5)
+
+
+@pytest.mark.parametrize(
+    "params, TypeError, err_msg",
+    [
+        (
+            {"alpha": np.zeros(100)},
+            ValueError,
+            "alpha must be a scalar or an array with same number of entries as y",
+        ),
+        (
+            {
+                "kernel": WhiteKernel(noise_level_bounds=(-np.inf, np.inf)),
+                "n_restarts_optimizer": 2,
+            },
+            ValueError,
+            "requires that all bounds are finite",
+        ),
+    ],
+)
+def test_gpr_fit_error(params, TypeError, err_msg):
+    """Check that expected error are raised during fit."""
+    gpr = GaussianProcessRegressor(**params)
+    with pytest.raises(TypeError, match=err_msg):
+        gpr.fit(X, y)
+
+
+def test_gpr_lml_error():
+    """Check that we raise the proper error in the LML method."""
+    gpr = GaussianProcessRegressor(kernel=RBF()).fit(X, y)
+
+    err_msg = "Gradient can only be evaluated for theta!=None"
+    with pytest.raises(ValueError, match=err_msg):
+        gpr.log_marginal_likelihood(eval_gradient=True)
+
+
+def test_gpr_predict_error():
+    """Check that we raise the proper error during predict."""
+    gpr = GaussianProcessRegressor(kernel=RBF()).fit(X, y)
+
+    err_msg = "At most one of return_std or return_cov can be requested."
+    with pytest.raises(RuntimeError, match=err_msg):
+        gpr.predict(X, return_cov=True, return_std=True)
+
+
+@pytest.mark.parametrize("normalize_y", [True, False])
+@pytest.mark.parametrize("n_targets", [None, 1, 10])
+def test_predict_shapes(normalize_y, n_targets):
+    """Check the shapes of y_mean, y_std, and y_cov in single-output
+    (n_targets=None) and multi-output settings, including the edge case when
+    n_targets=1, where the sklearn convention is to squeeze the predictions.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/17394
+    https://github.com/scikit-learn/scikit-learn/issues/18065
+    https://github.com/scikit-learn/scikit-learn/issues/22174
+    """
+    rng = np.random.RandomState(1234)
+
+    n_features, n_samples_train, n_samples_test = 6, 9, 7
+
+    y_train_shape = (n_samples_train,)
+    if n_targets is not None:
+        y_train_shape = y_train_shape + (n_targets,)
+
+    # By convention single-output data is squeezed upon prediction
+    y_test_shape = (n_samples_test,)
+    if n_targets is not None and n_targets > 1:
+        y_test_shape = y_test_shape + (n_targets,)
+
+    X_train = rng.randn(n_samples_train, n_features)
+    X_test = rng.randn(n_samples_test, n_features)
+    y_train = rng.randn(*y_train_shape)
+
+    model = GaussianProcessRegressor(normalize_y=normalize_y)
+    model.fit(X_train, y_train)
+
+    y_pred, y_std = model.predict(X_test, return_std=True)
+    _, y_cov = model.predict(X_test, return_cov=True)
+
+    assert y_pred.shape == y_test_shape
+    assert y_std.shape == y_test_shape
+    assert y_cov.shape == (n_samples_test,) + y_test_shape
+
+
+@pytest.mark.parametrize("normalize_y", [True, False])
+@pytest.mark.parametrize("n_targets", [None, 1, 10])
+def test_sample_y_shapes(normalize_y, n_targets):
+    """Check the shapes of y_samples in single-output (n_targets=0) and
+    multi-output settings, including the edge case when n_targets=1, where the
+    sklearn convention is to squeeze the predictions.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/22175
+    """
+    rng = np.random.RandomState(1234)
+
+    n_features, n_samples_train = 6, 9
+    # Number of spatial locations to predict at
+    n_samples_X_test = 7
+    # Number of sample predictions per test point
+    n_samples_y_test = 5
+
+    y_train_shape = (n_samples_train,)
+    if n_targets is not None:
+        y_train_shape = y_train_shape + (n_targets,)
+
+    # By convention single-output data is squeezed upon prediction
+    if n_targets is not None and n_targets > 1:
+        y_test_shape = (n_samples_X_test, n_targets, n_samples_y_test)
+    else:
+        y_test_shape = (n_samples_X_test, n_samples_y_test)
+
+    X_train = rng.randn(n_samples_train, n_features)
+    X_test = rng.randn(n_samples_X_test, n_features)
+    y_train = rng.randn(*y_train_shape)
+
+    model = GaussianProcessRegressor(normalize_y=normalize_y)
+
+    # FIXME: before fitting, the estimator does not have information regarding
+    # the number of targets and default to 1. This is inconsistent with the shape
+    # provided after `fit`. This assert should be made once the following issue
+    # is fixed:
+    # https://github.com/scikit-learn/scikit-learn/issues/22430
+    # y_samples = model.sample_y(X_test, n_samples=n_samples_y_test)
+    # assert y_samples.shape == y_test_shape
+
+    model.fit(X_train, y_train)
+
+    y_samples = model.sample_y(X_test, n_samples=n_samples_y_test)
+    assert y_samples.shape == y_test_shape
+
+
+@pytest.mark.parametrize("n_targets", [None, 1, 2, 3])
+@pytest.mark.parametrize("n_samples", [1, 5])
+def test_sample_y_shape_with_prior(n_targets, n_samples):
+    """Check the output shape of `sample_y` is consistent before and after `fit`."""
+    rng = np.random.RandomState(1024)
+
+    X = rng.randn(10, 3)
+    y = rng.randn(10, n_targets if n_targets is not None else 1)
+
+    model = GaussianProcessRegressor(n_targets=n_targets)
+    shape_before_fit = model.sample_y(X, n_samples=n_samples).shape
+    model.fit(X, y)
+    shape_after_fit = model.sample_y(X, n_samples=n_samples).shape
+    assert shape_before_fit == shape_after_fit
+
+
+@pytest.mark.parametrize("n_targets", [None, 1, 2, 3])
+def test_predict_shape_with_prior(n_targets):
+    """Check the output shape of `predict` with prior distribution."""
+    rng = np.random.RandomState(1024)
+
+    n_sample = 10
+    X = rng.randn(n_sample, 3)
+    y = rng.randn(n_sample, n_targets if n_targets is not None else 1)
+
+    model = GaussianProcessRegressor(n_targets=n_targets)
+    mean_prior, cov_prior = model.predict(X, return_cov=True)
+    _, std_prior = model.predict(X, return_std=True)
+
+    model.fit(X, y)
+    mean_post, cov_post = model.predict(X, return_cov=True)
+    _, std_post = model.predict(X, return_std=True)
+
+    assert mean_prior.shape == mean_post.shape
+    assert cov_prior.shape == cov_post.shape
+    assert std_prior.shape == std_post.shape
+
+
+def test_n_targets_error():
+    """Check that an error is raised when the number of targets seen at fit is
+    inconsistent with n_targets.
+    """
+    rng = np.random.RandomState(0)
+    X = rng.randn(10, 3)
+    y = rng.randn(10, 2)
+
+    model = GaussianProcessRegressor(n_targets=1)
+    with pytest.raises(ValueError, match="The number of targets seen in `y`"):
+        model.fit(X, y)
+
+
+class CustomKernel(C):
+    """
+    A custom kernel that has a diag method that returns the first column of the
+    input matrix X. This is a helper for the test to check that the input
+    matrix X is not mutated.
+    """
+
+    def diag(self, X):
+        return X[:, 0]
+
+
+def test_gpr_predict_input_not_modified():
+    """
+    Check that the input X is not modified by the predict method of the
+    GaussianProcessRegressor when setting return_std=True.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/24340
+    """
+    gpr = GaussianProcessRegressor(kernel=CustomKernel()).fit(X, y)
+
+    X2_copy = np.copy(X2)
+    _, _ = gpr.predict(X2, return_std=True)
+
+    assert_allclose(X2, X2_copy)
diff --git a/sklearn/gaussian_process/tests/test_kernels.py b/sklearn/gaussian_process/tests/test_kernels.py
index 1f8e196104e75..5174d50b7df92 100644
--- a/sklearn/gaussian_process/tests/test_kernels.py
+++ b/sklearn/gaussian_process/tests/test_kernels.py
@@ -1,60 +1,80 @@
 """Testing for kernels for Gaussian processes."""
 
-# Author: Jan Hendrik Metzen 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-import pytest
-import numpy as np
 from inspect import signature
 
-from sklearn.gaussian_process.kernels import _approx_fprime
+import numpy as np
+import pytest
 
-from sklearn.metrics.pairwise \
-    import PAIRWISE_KERNEL_FUNCTIONS, euclidean_distances, pairwise_kernels
-from sklearn.gaussian_process.kernels \
-    import (RBF, Matern, RationalQuadratic, ExpSineSquared, DotProduct,
-            ConstantKernel, WhiteKernel, PairwiseKernel, KernelOperator,
-            Exponentiation, CompoundKernel)
 from sklearn.base import clone
-
-from sklearn.utils._testing import (assert_almost_equal, assert_array_equal,
-                                    assert_array_almost_equal,
-                                    assert_allclose,
-                                    assert_raise_message,
-                                    fails_if_pypy)
-
+from sklearn.gaussian_process.kernels import (
+    RBF,
+    CompoundKernel,
+    ConstantKernel,
+    DotProduct,
+    Exponentiation,
+    ExpSineSquared,
+    KernelOperator,
+    Matern,
+    PairwiseKernel,
+    RationalQuadratic,
+    WhiteKernel,
+    _approx_fprime,
+)
+from sklearn.metrics.pairwise import (
+    PAIRWISE_KERNEL_FUNCTIONS,
+    euclidean_distances,
+    pairwise_kernels,
+)
+from sklearn.utils._testing import (
+    assert_allclose,
+    assert_almost_equal,
+    assert_array_almost_equal,
+    assert_array_equal,
+)
 
 X = np.random.RandomState(0).normal(0, 1, (5, 2))
 Y = np.random.RandomState(0).normal(0, 1, (6, 2))
+# Set shared test data as read-only to avoid unintentional in-place
+# modifications that would introduce side-effects between tests.
+X.flags.writeable = False
+Y.flags.writeable = False
 
 kernel_rbf_plus_white = RBF(length_scale=2.0) + WhiteKernel(noise_level=3.0)
-kernels = [RBF(length_scale=2.0), RBF(length_scale_bounds=(0.5, 2.0)),
-           ConstantKernel(constant_value=10.0),
-           2.0 * RBF(length_scale=0.33, length_scale_bounds="fixed"),
-           2.0 * RBF(length_scale=0.5), kernel_rbf_plus_white,
-           2.0 * RBF(length_scale=[0.5, 2.0]),
-           2.0 * Matern(length_scale=0.33, length_scale_bounds="fixed"),
-           2.0 * Matern(length_scale=0.5, nu=0.5),
-           2.0 * Matern(length_scale=1.5, nu=1.5),
-           2.0 * Matern(length_scale=2.5, nu=2.5),
-           2.0 * Matern(length_scale=[0.5, 2.0], nu=0.5),
-           3.0 * Matern(length_scale=[2.0, 0.5], nu=1.5),
-           4.0 * Matern(length_scale=[0.5, 0.5], nu=2.5),
-           RationalQuadratic(length_scale=0.5, alpha=1.5),
-           ExpSineSquared(length_scale=0.5, periodicity=1.5),
-           DotProduct(sigma_0=2.0), DotProduct(sigma_0=2.0) ** 2,
-           RBF(length_scale=[2.0]), Matern(length_scale=[2.0])]
+kernels = [
+    RBF(length_scale=2.0),
+    RBF(length_scale_bounds=(0.5, 2.0)),
+    ConstantKernel(constant_value=10.0),
+    2.0 * RBF(length_scale=0.33, length_scale_bounds="fixed"),
+    2.0 * RBF(length_scale=0.5),
+    kernel_rbf_plus_white,
+    2.0 * RBF(length_scale=[0.5, 2.0]),
+    2.0 * Matern(length_scale=0.33, length_scale_bounds="fixed"),
+    2.0 * Matern(length_scale=0.5, nu=0.5),
+    2.0 * Matern(length_scale=1.5, nu=1.5),
+    2.0 * Matern(length_scale=2.5, nu=2.5),
+    2.0 * Matern(length_scale=[0.5, 2.0], nu=0.5),
+    3.0 * Matern(length_scale=[2.0, 0.5], nu=1.5),
+    4.0 * Matern(length_scale=[0.5, 0.5], nu=2.5),
+    RationalQuadratic(length_scale=0.5, alpha=1.5),
+    ExpSineSquared(length_scale=0.5, periodicity=1.5),
+    DotProduct(sigma_0=2.0),
+    DotProduct(sigma_0=2.0) ** 2,
+    RBF(length_scale=[2.0]),
+    Matern(length_scale=[2.0]),
+]
 for metric in PAIRWISE_KERNEL_FUNCTIONS:
     if metric in ["additive_chi2", "chi2"]:
         continue
     kernels.append(PairwiseKernel(gamma=1.0, metric=metric))
 
 
-# Numerical precisions errors in PyPy
-@fails_if_pypy
-@pytest.mark.parametrize('kernel', kernels)
+@pytest.mark.parametrize("kernel", kernels)
 def test_kernel_gradient(kernel):
     # Compare analytic and numeric gradient of kernels.
+    kernel = clone(kernel)  # make tests independent of one-another
     K, K_gradient = kernel(X, eval_gradient=True)
 
     assert K_gradient.shape[0] == X.shape[0]
@@ -66,37 +86,40 @@ def eval_kernel_for_theta(theta):
         K = kernel_clone(X, eval_gradient=False)
         return K
 
-    K_gradient_approx = \
-        _approx_fprime(kernel.theta, eval_kernel_for_theta, 1e-10)
+    K_gradient_approx = _approx_fprime(kernel.theta, eval_kernel_for_theta, 1e-10)
 
     assert_almost_equal(K_gradient, K_gradient_approx, 4)
 
 
 @pytest.mark.parametrize(
-        'kernel',
-        [kernel for kernel in kernels
-         # skip non-basic kernels
-         if not (isinstance(kernel, KernelOperator)
-                 or isinstance(kernel, Exponentiation))])
+    "kernel",
+    [
+        kernel
+        for kernel in kernels
+        # skip non-basic kernels
+        if not (isinstance(kernel, (KernelOperator, Exponentiation)))
+    ],
+)
 def test_kernel_theta(kernel):
     # Check that parameter vector theta of kernel is set correctly.
+    kernel = clone(kernel)  # make tests independent of one-another
     theta = kernel.theta
     _, K_gradient = kernel(X, eval_gradient=True)
 
     # Determine kernel parameters that contribute to theta
     init_sign = signature(kernel.__class__.__init__).parameters.values()
-    args = [p.name for p in init_sign if p.name != 'self']
-    theta_vars = map(lambda s: s[0:-len("_bounds")],
-                     filter(lambda s: s.endswith("_bounds"), args))
-    assert (
-        set(hyperparameter.name
-            for hyperparameter in kernel.hyperparameters) ==
-        set(theta_vars))
+    args = [p.name for p in init_sign if p.name != "self"]
+    theta_vars = map(
+        lambda s: s[0 : -len("_bounds")], filter(lambda s: s.endswith("_bounds"), args)
+    )
+    assert set(hyperparameter.name for hyperparameter in kernel.hyperparameters) == set(
+        theta_vars
+    )
 
     # Check that values returned in theta are consistent with
     # hyperparameter values (being their logarithms)
     for i, hyperparameter in enumerate(kernel.hyperparameters):
-        assert (theta[i] == np.log(getattr(kernel, hyperparameter.name)))
+        assert theta[i] == np.log(getattr(kernel, hyperparameter.name))
 
     # Fixed kernel parameters must be excluded from theta and gradient.
     for i, hyperparameter in enumerate(kernel.hyperparameters):
@@ -112,12 +135,10 @@ def test_kernel_theta(kernel):
         assert K_gradient.shape[2] == K_gradient_new.shape[2] + 1
         if i > 0:
             assert theta[:i] == new_kernel.theta[:i]
-            assert_array_equal(K_gradient[..., :i],
-                               K_gradient_new[..., :i])
+            assert_array_equal(K_gradient[..., :i], K_gradient_new[..., :i])
         if i + 1 < len(kernel.hyperparameters):
-            assert theta[i + 1:] == new_kernel.theta[i:]
-            assert_array_equal(K_gradient[..., i + 1:],
-                               K_gradient_new[..., i:])
+            assert theta[i + 1 :] == new_kernel.theta[i:]
+            assert_array_equal(K_gradient[..., i + 1 :], K_gradient_new[..., i:])
 
     # Check that values of theta are modified correctly
     for i, hyperparameter in enumerate(kernel.hyperparameters):
@@ -129,19 +150,26 @@ def test_kernel_theta(kernel):
         assert_almost_equal(kernel.theta[i], np.log(43))
 
 
-@pytest.mark.parametrize('kernel',
-                         [kernel for kernel in kernels
-                          # Identity is not satisfied on diagonal
-                          if kernel != kernel_rbf_plus_white])
+@pytest.mark.parametrize(
+    "kernel",
+    [
+        kernel
+        for kernel in kernels
+        # Identity is not satisfied on diagonal
+        if kernel != kernel_rbf_plus_white
+    ],
+)
 def test_auto_vs_cross(kernel):
+    kernel = clone(kernel)  # make tests independent of one-another
     # Auto-correlation and cross-correlation should be consistent.
     K_auto = kernel(X)
     K_cross = kernel(X, X)
     assert_almost_equal(K_auto, K_cross, 5)
 
 
-@pytest.mark.parametrize('kernel', kernels)
+@pytest.mark.parametrize("kernel", kernels)
 def test_kernel_diag(kernel):
+    kernel = clone(kernel)  # make tests independent of one-another
     # Test that diag method of kernel returns consistent results.
     K_call_diag = np.diag(kernel(X))
     K_diag = kernel.diag(X)
@@ -151,12 +179,10 @@ def test_kernel_diag(kernel):
 def test_kernel_operator_commutative():
     # Adding kernels and multiplying kernels should be commutative.
     # Check addition
-    assert_almost_equal((RBF(2.0) + 1.0)(X),
-                        (1.0 + RBF(2.0))(X))
+    assert_almost_equal((RBF(2.0) + 1.0)(X), (1.0 + RBF(2.0))(X))
 
     # Check multiplication
-    assert_almost_equal((3.0 * RBF(2.0))(X),
-                        (RBF(2.0) * 3.0)(X))
+    assert_almost_equal((3.0 * RBF(2.0))(X), (RBF(2.0) * 3.0)(X))
 
 
 def test_kernel_anisotropic():
@@ -164,12 +190,12 @@ def test_kernel_anisotropic():
     kernel = 3.0 * RBF([0.5, 2.0])
 
     K = kernel(X)
-    X1 = np.array(X)
+    X1 = X.copy()
     X1[:, 0] *= 4
     K1 = 3.0 * RBF(2.0)(X1)
     assert_almost_equal(K, K1)
 
-    X2 = np.array(X)
+    X2 = X.copy()
     X2[:, 1] /= 4
     K2 = 3.0 * RBF(0.5)(X2)
     assert_almost_equal(K, K2)
@@ -180,33 +206,33 @@ def test_kernel_anisotropic():
     assert_array_equal(kernel.k2.length_scale, [1.0, 4.0])
 
 
-@pytest.mark.parametrize('kernel',
-                         [kernel for kernel in kernels
-                          if kernel.is_stationary()])
+@pytest.mark.parametrize(
+    "kernel", [kernel for kernel in kernels if kernel.is_stationary()]
+)
 def test_kernel_stationary(kernel):
+    kernel = clone(kernel)  # make tests independent of one-another
     # Test stationarity of kernels.
     K = kernel(X, X + 1)
     assert_almost_equal(K[0, 0], np.diag(K))
 
 
-@pytest.mark.parametrize('kernel',  kernels)
+@pytest.mark.parametrize("kernel", kernels)
 def test_kernel_input_type(kernel):
+    kernel = clone(kernel)  # make tests independent of one-another
     # Test whether kernels is for vectors or structured data
     if isinstance(kernel, Exponentiation):
-        assert(kernel.requires_vector_input ==
-               kernel.kernel.requires_vector_input)
+        assert kernel.requires_vector_input == kernel.kernel.requires_vector_input
     if isinstance(kernel, KernelOperator):
-        assert(kernel.requires_vector_input ==
-               (kernel.k1.requires_vector_input or
-                kernel.k2.requires_vector_input))
+        assert kernel.requires_vector_input == (
+            kernel.k1.requires_vector_input or kernel.k2.requires_vector_input
+        )
 
 
 def test_compound_kernel_input_type():
     kernel = CompoundKernel([WhiteKernel(noise_level=3.0)])
     assert not kernel.requires_vector_input
 
-    kernel = CompoundKernel([WhiteKernel(noise_level=3.0),
-                             RBF(length_scale=2.0)])
+    kernel = CompoundKernel([WhiteKernel(noise_level=3.0), RBF(length_scale=2.0)])
     assert kernel.requires_vector_input
 
 
@@ -221,6 +247,7 @@ def check_hyperparameters_equal(kernel1, kernel2):
 
 @pytest.mark.parametrize("kernel", kernels)
 def test_kernel_clone(kernel):
+    kernel = clone(kernel)  # make tests independent of one-another
     # Test that sklearn's clone works correctly on kernels.
     kernel_cloned = clone(kernel)
 
@@ -236,8 +263,9 @@ def test_kernel_clone(kernel):
     check_hyperparameters_equal(kernel, kernel_cloned)
 
 
-@pytest.mark.parametrize('kernel', kernels)
+@pytest.mark.parametrize("kernel", kernels)
 def test_kernel_clone_after_set_params(kernel):
+    kernel = clone(kernel)  # make tests independent of one-another
     # This test is to verify that using set_params does not
     # break clone on kernels.
     # This used to break because in kernels such as the RBF, non-trivial
@@ -249,19 +277,18 @@ def test_kernel_clone_after_set_params(kernel):
     params = kernel.get_params()
     # RationalQuadratic kernel is isotropic.
     isotropic_kernels = (ExpSineSquared, RationalQuadratic)
-    if 'length_scale' in params and not isinstance(kernel,
-                                                   isotropic_kernels):
-        length_scale = params['length_scale']
+    if "length_scale" in params and not isinstance(kernel, isotropic_kernels):
+        length_scale = params["length_scale"]
         if np.iterable(length_scale):
             # XXX unreached code as of v0.22
-            params['length_scale'] = length_scale[0]
-            params['length_scale_bounds'] = bounds
+            params["length_scale"] = length_scale[0]
+            params["length_scale_bounds"] = bounds
         else:
-            params['length_scale'] = [length_scale] * 2
-            params['length_scale_bounds'] = bounds * 2
+            params["length_scale"] = [length_scale] * 2
+            params["length_scale_bounds"] = bounds * 2
         kernel_cloned.set_params(**params)
         kernel_cloned_clone = clone(kernel_cloned)
-        assert (kernel_cloned_clone.get_params() == kernel_cloned.get_params())
+        assert kernel_cloned_clone.get_params() == kernel_cloned.get_params()
         assert id(kernel_cloned_clone) != id(kernel_cloned)
         check_hyperparameters_equal(kernel_cloned, kernel_cloned_clone)
 
@@ -297,6 +324,7 @@ def test_matern_kernel():
 
 @pytest.mark.parametrize("kernel", kernels)
 def test_kernel_versus_pairwise(kernel):
+    kernel = clone(kernel)  # make tests independent of one-another
     # Check that GP kernels can also be used as pairwise kernels.
 
     # Test auto-kernel
@@ -315,6 +343,7 @@ def test_kernel_versus_pairwise(kernel):
 
 @pytest.mark.parametrize("kernel", kernels)
 def test_set_get_params(kernel):
+    kernel = clone(kernel)  # make tests independent of one-another
     # Check that set_params()/get_params() is consistent with kernel.theta.
 
     # Test get_params()
@@ -326,12 +355,14 @@ def test_set_get_params(kernel):
                 continue
         size = hyperparameter.n_elements
         if size > 1:  # anisotropic kernels
-            assert_almost_equal(np.exp(kernel.theta[index:index + size]),
-                                params[hyperparameter.name])
+            assert_almost_equal(
+                np.exp(kernel.theta[index : index + size]), params[hyperparameter.name]
+            )
             index += size
         else:
-            assert_almost_equal(np.exp(kernel.theta[index]),
-                                params[hyperparameter.name])
+            assert_almost_equal(
+                np.exp(kernel.theta[index]), params[hyperparameter.name]
+            )
             index += 1
     # Test set_params()
     index = 0
@@ -343,8 +374,9 @@ def test_set_get_params(kernel):
         size = hyperparameter.n_elements
         if size > 1:  # anisotropic kernels
             kernel.set_params(**{hyperparameter.name: [value] * size})
-            assert_almost_equal(np.exp(kernel.theta[index:index + size]),
-                                [value] * size)
+            assert_almost_equal(
+                np.exp(kernel.theta[index : index + size]), [value] * size
+            )
             index += size
         else:
             kernel.set_params(**{hyperparameter.name: value})
@@ -354,14 +386,18 @@ def test_set_get_params(kernel):
 
 @pytest.mark.parametrize("kernel", kernels)
 def test_repr_kernels(kernel):
+    kernel = clone(kernel)  # make tests independent of one-another
     # Smoke-test for repr in kernels.
 
     repr(kernel)
 
 
 def test_rational_quadratic_kernel():
-    kernel = RationalQuadratic(length_scale=[1., 1.])
-    assert_raise_message(AttributeError,
-                         "RationalQuadratic kernel only supports isotropic "
-                         "version, please use a single "
-                         "scalar for length_scale", kernel, X)
+    kernel = RationalQuadratic(length_scale=[1.0, 1.0])
+    message = (
+        "RationalQuadratic kernel only supports isotropic "
+        "version, please use a single "
+        "scalar for length_scale"
+    )
+    with pytest.raises(AttributeError, match=message):
+        kernel(X)
diff --git a/sklearn/impute/__init__.py b/sklearn/impute/__init__.py
index 940035ae58589..aaa81d73c34a1 100644
--- a/sklearn/impute/__init__.py
+++ b/sklearn/impute/__init__.py
@@ -1,4 +1,8 @@
-"""Transformers for missing value imputation"""
+"""Transformers for missing value imputation."""
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
 import typing
 
 from ._base import MissingIndicator, SimpleImputer
@@ -7,10 +11,18 @@
 if typing.TYPE_CHECKING:
     # Avoid errors in type checkers (e.g. mypy) for experimental estimators.
     # TODO: remove this check once the estimator is no longer experimental.
-    from ._iterative import IterativeImputer  # noqa
+    from ._iterative import IterativeImputer  # noqa: F401
+
+__all__ = ["KNNImputer", "MissingIndicator", "SimpleImputer"]
+
 
-__all__ = [
-    'MissingIndicator',
-    'SimpleImputer',
-    'KNNImputer'
-]
+# TODO: remove this check once the estimator is no longer experimental.
+def __getattr__(name):
+    if name == "IterativeImputer":
+        raise ImportError(
+            f"{name} is experimental and the API might change without any "
+            "deprecation cycle. To use it, you need to explicitly import "
+            "enable_iterative_imputer:\n"
+            "from sklearn.experimental import enable_iterative_imputer"
+        )
+    raise AttributeError(f"module {__name__} has no attribute {name}")
diff --git a/sklearn/impute/_base.py b/sklearn/impute/_base.py
index 46f2301a1879d..689ba8aceeaf6 100644
--- a/sklearn/impute/_base.py
+++ b/sklearn/impute/_base.py
@@ -1,38 +1,47 @@
-# Authors: Nicolas Tresegnie 
-#          Sergey Feldman 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import numbers
 import warnings
 from collections import Counter
+from functools import partial
+from typing import Callable
 
 import numpy as np
 import numpy.ma as ma
 from scipy import sparse as sp
-from scipy import stats
 
-from ..base import BaseEstimator, TransformerMixin
-from ..utils.sparsefuncs import _get_median
-from ..utils.validation import check_is_fitted
-from ..utils.validation import FLOAT_DTYPES
-from ..utils.validation import _deprecate_positional_args
+from ..base import BaseEstimator, TransformerMixin, _fit_context
 from ..utils._mask import _get_mask
-from ..utils import is_scalar_nan
+from ..utils._missing import is_pandas_na, is_scalar_nan
+from ..utils._param_validation import MissingValues, StrOptions
+from ..utils.fixes import _mode
+from ..utils.sparsefuncs import _get_median
+from ..utils.validation import (
+    FLOAT_DTYPES,
+    _check_feature_names_in,
+    _check_n_features,
+    check_is_fitted,
+    validate_data,
+)
 
 
 def _check_inputs_dtype(X, missing_values):
-    if (X.dtype.kind in ("f", "i", "u") and
-            not isinstance(missing_values, numbers.Real)):
-        raise ValueError("'X' and 'missing_values' types are expected to be"
-                         " both numerical. Got X.dtype={} and "
-                         " type(missing_values)={}."
-                         .format(X.dtype, type(missing_values)))
+    if is_pandas_na(missing_values):
+        # Allow using `pd.NA` as missing values to impute numerical arrays.
+        return
+    if X.dtype.kind in ("f", "i", "u") and not isinstance(missing_values, numbers.Real):
+        raise ValueError(
+            "'X' and 'missing_values' types are expected to be"
+            " both numerical. Got X.dtype={} and "
+            " type(missing_values)={}.".format(X.dtype, type(missing_values))
+        )
 
 
 def _most_frequent(array, extra_value, n_repeat):
     """Compute the most frequent value in a 1d array extended with
-       [extra_value] * n_repeat, where extra_value is assumed to be not part
-       of the array."""
+    [extra_value] * n_repeat, where extra_value is assumed to be not part
+    of the array."""
     # Compute the most frequent value in array only
     if array.size > 0:
         if array.dtype == object:
@@ -42,11 +51,12 @@ def _most_frequent(array, extra_value, n_repeat):
             most_frequent_count = counter.most_common(1)[0][1]
             # tie breaking similarly to scipy.stats.mode
             most_frequent_value = min(
-                value for value, count in counter.items()
+                value
+                for value, count in counter.items()
                 if count == most_frequent_count
             )
         else:
-            mode = stats.mode(array)
+            mode = _mode(array)
             most_frequent_value = mode[0][0]
             most_frequent_count = mode[1][0]
     else:
@@ -71,15 +81,25 @@ class _BaseImputer(TransformerMixin, BaseEstimator):
     It adds automatically support for `add_indicator`.
     """
 
-    def __init__(self, *, missing_values=np.nan, add_indicator=False):
+    _parameter_constraints: dict = {
+        "missing_values": [MissingValues()],
+        "add_indicator": ["boolean"],
+        "keep_empty_features": ["boolean"],
+    }
+
+    def __init__(
+        self, *, missing_values=np.nan, add_indicator=False, keep_empty_features=False
+    ):
         self.missing_values = missing_values
         self.add_indicator = add_indicator
+        self.keep_empty_features = keep_empty_features
 
     def _fit_indicator(self, X):
         """Fit a MissingIndicator."""
         if self.add_indicator:
             self.indicator_ = MissingIndicator(
-                missing_values=self.missing_values, error_on_new=False)
+                missing_values=self.missing_values, error_on_new=False
+            )
             self.indicator_._fit(X, precomputed=True)
         else:
             self.indicator_ = None
@@ -91,10 +111,9 @@ def _transform_indicator(self, X):
         any imputation, since imputation may be done inplace in some cases.
         """
         if self.add_indicator:
-            if not hasattr(self, 'indicator_'):
+            if not hasattr(self, "indicator_"):
                 raise ValueError(
-                    "Make sure to call _fit_indicator before "
-                    "_transform_indicator"
+                    "Make sure to call _fit_indicator before _transform_indicator"
                 )
             return self.indicator_.transform(X)
 
@@ -103,22 +122,40 @@ def _concatenate_indicator(self, X_imputed, X_indicator):
         if not self.add_indicator:
             return X_imputed
 
-        hstack = sp.hstack if sp.issparse(X_imputed) else np.hstack
+        if sp.issparse(X_imputed):
+            # sp.hstack may result in different formats between sparse arrays and
+            # matrices; specify the format to keep consistent behavior
+            hstack = partial(sp.hstack, format=X_imputed.format)
+        else:
+            hstack = np.hstack
+
         if X_indicator is None:
             raise ValueError(
                 "Data from the missing indicator are not provided. Call "
                 "_fit_indicator and _transform_indicator in the imputer "
                 "implementation."
-                )
+            )
 
         return hstack((X_imputed, X_indicator))
 
-    def _more_tags(self):
-        return {'allow_nan': is_scalar_nan(self.missing_values)}
+    def _concatenate_indicator_feature_names_out(self, names, input_features):
+        if not self.add_indicator:
+            return names
+
+        indicator_names = self.indicator_.get_feature_names_out(input_features)
+        return np.concatenate([names, indicator_names])
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.allow_nan = is_scalar_nan(self.missing_values)
+        return tags
 
 
 class SimpleImputer(_BaseImputer):
-    """Imputation transformer for completing missing values.
+    """Univariate imputer for completing missing values with simple strategies.
+
+    Replace missing values using a descriptive statistic (e.g. mean, median, or
+    most frequent) along each column, or using a constant value.
 
     Read more in the :ref:`User Guide `.
 
@@ -128,13 +165,13 @@ class SimpleImputer(_BaseImputer):
 
     Parameters
     ----------
-    missing_values : int, float, str, np.nan or None, default=np.nan
+    missing_values : int, float, str, np.nan, None or pandas.NA, default=np.nan
         The placeholder for the missing values. All occurrences of
         `missing_values` will be imputed. For pandas' dataframes with
         nullable integer dtypes with missing values, `missing_values`
-        should be set to `np.nan`, since `pd.NA` will be converted to `np.nan`.
+        can be set to either `np.nan` or `pd.NA`.
 
-    strategy : string, default='mean'
+    strategy : str or Callable, default='mean'
         The imputation strategy.
 
         - If "mean", then replace missing values using the mean along
@@ -146,29 +183,33 @@ class SimpleImputer(_BaseImputer):
           If there is more than one such value, only the smallest is returned.
         - If "constant", then replace missing values with fill_value. Can be
           used with strings or numeric data.
+        - If an instance of Callable, then replace missing values using the
+          scalar statistic returned by running the callable over a dense 1d
+          array containing non-missing values of each column.
 
         .. versionadded:: 0.20
            strategy="constant" for fixed value imputation.
 
-    fill_value : string or numerical value, default=None
-        When strategy == "constant", fill_value is used to replace all
-        occurrences of missing_values.
-        If left to the default, fill_value will be 0 when imputing numerical
-        data and "missing_value" for strings or object data types.
+        .. versionadded:: 1.5
+           strategy=callable for custom value imputation.
 
-    verbose : integer, default=0
-        Controls the verbosity of the imputer.
+    fill_value : str or numerical value, default=None
+        When strategy == "constant", `fill_value` is used to replace all
+        occurrences of missing_values. For string or object data types,
+        `fill_value` must be a string.
+        If `None`, `fill_value` will be 0 when imputing numerical
+        data and "missing_value" for strings or object data types.
 
-    copy : boolean, default=True
+    copy : bool, default=True
         If True, a copy of X will be created. If False, imputation will
         be done in-place whenever possible. Note that, in the following cases,
         a new copy will always be made, even if `copy=False`:
 
-        - If X is not an array of floating values;
-        - If X is encoded as a CSR matrix;
-        - If add_indicator=True.
+        - If `X` is not an array of floating values;
+        - If `X` is encoded as a CSR matrix;
+        - If `add_indicator=True`.
 
-    add_indicator : boolean, default=False
+    add_indicator : bool, default=False
         If True, a :class:`MissingIndicator` transform will stack onto output
         of the imputer's transform. This allows a predictive estimator
         to account for missingness despite imputation. If a feature has no
@@ -176,6 +217,19 @@ class SimpleImputer(_BaseImputer):
         the missing indicator even if there are missing values at
         transform/test time.
 
+    keep_empty_features : bool, default=False
+        If True, features that consist exclusively of missing values when
+        `fit` is called are returned in results when `transform` is called.
+        The imputed value is always `0` except when `strategy="constant"`
+        in which case `fill_value` will be used instead.
+
+        .. versionadded:: 1.2
+
+        .. versionchanged:: 1.6
+            Currently, when `keep_empty_feature=False` and `strategy="constant"`,
+            empty features are not dropped. This behaviour will change in version
+            1.8. Set `keep_empty_feature=True` to preserve this behaviour.
+
     Attributes
     ----------
     statistics_ : array of shape (n_features,)
@@ -186,11 +240,35 @@ class SimpleImputer(_BaseImputer):
 
     indicator_ : :class:`~sklearn.impute.MissingIndicator`
         Indicator used to add binary indicators for missing values.
-        ``None`` if add_indicator is False.
+        `None` if `add_indicator=False`.
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
 
     See Also
     --------
-    IterativeImputer : Multivariate imputation of missing values.
+    IterativeImputer : Multivariate imputer that estimates values to impute for
+        each feature with missing values from all the others.
+    KNNImputer : Multivariate imputer that estimates missing features using
+        nearest samples.
+
+    Notes
+    -----
+    Columns which only contained missing values at :meth:`fit` are discarded
+    upon :meth:`transform` if strategy is not `"constant"`.
+
+    In a prediction context, simple imputation usually performs poorly when
+    associated with a weak learner. However, with a powerful learner, it can
+    lead to as good or better performance than complex imputation such as
+    :class:`~sklearn.impute.IterativeImputer` or :class:`~sklearn.impute.KNNImputer`.
 
     Examples
     --------
@@ -205,85 +283,155 @@ class SimpleImputer(_BaseImputer):
      [ 4.   3.5  6. ]
      [10.   3.5  9. ]]
 
-    Notes
-    -----
-    Columns which only contained missing values at :meth:`fit` are discarded
-    upon :meth:`transform` if strategy is not "constant".
-
+    For a more detailed example see
+    :ref:`sphx_glr_auto_examples_impute_plot_missing_values.py`.
     """
-    @_deprecate_positional_args
-    def __init__(self, *, missing_values=np.nan, strategy="mean",
-                 fill_value=None, verbose=0, copy=True, add_indicator=False):
+
+    _parameter_constraints: dict = {
+        **_BaseImputer._parameter_constraints,
+        "strategy": [
+            StrOptions({"mean", "median", "most_frequent", "constant"}),
+            callable,
+        ],
+        "fill_value": "no_validation",  # any object is valid
+        "copy": ["boolean"],
+    }
+
+    def __init__(
+        self,
+        *,
+        missing_values=np.nan,
+        strategy="mean",
+        fill_value=None,
+        copy=True,
+        add_indicator=False,
+        keep_empty_features=False,
+    ):
         super().__init__(
             missing_values=missing_values,
-            add_indicator=add_indicator
+            add_indicator=add_indicator,
+            keep_empty_features=keep_empty_features,
         )
         self.strategy = strategy
         self.fill_value = fill_value
-        self.verbose = verbose
         self.copy = copy
 
     def _validate_input(self, X, in_fit):
-        allowed_strategies = ["mean", "median", "most_frequent", "constant"]
-        if self.strategy not in allowed_strategies:
-            raise ValueError("Can only use these strategies: {0} "
-                             " got strategy={1}".format(allowed_strategies,
-                                                        self.strategy))
-
         if self.strategy in ("most_frequent", "constant"):
             # If input is a list of strings, dtype = object.
             # Otherwise ValueError is raised in SimpleImputer
             # with strategy='most_frequent' or 'constant'
             # because the list is converted to Unicode numpy array
-            if isinstance(X, list) and \
-               any(isinstance(elem, str) for row in X for elem in row):
+            if isinstance(X, list) and any(
+                isinstance(elem, str) for row in X for elem in row
+            ):
                 dtype = object
             else:
                 dtype = None
         else:
             dtype = FLOAT_DTYPES
 
-        if not is_scalar_nan(self.missing_values):
-            force_all_finite = True
+        if not in_fit and self._fit_dtype.kind == "O":
+            # Use object dtype if fitted on object dtypes
+            dtype = self._fit_dtype
+
+        if is_pandas_na(self.missing_values) or is_scalar_nan(self.missing_values):
+            ensure_all_finite = "allow-nan"
         else:
-            force_all_finite = "allow-nan"
+            ensure_all_finite = True
 
         try:
-            X = self._validate_data(X, reset=in_fit,
-                                    accept_sparse='csc', dtype=dtype,
-                                    force_all_finite=force_all_finite,
-                                    copy=self.copy)
+            X = validate_data(
+                self,
+                X,
+                reset=in_fit,
+                accept_sparse="csc",
+                dtype=dtype,
+                force_writeable=True if not in_fit else None,
+                ensure_all_finite=ensure_all_finite,
+                copy=self.copy,
+            )
         except ValueError as ve:
             if "could not convert" in str(ve):
-                new_ve = ValueError("Cannot use {} strategy with non-numeric "
-                                    "data:\n{}".format(self.strategy, ve))
+                new_ve = ValueError(
+                    "Cannot use {} strategy with non-numeric data:\n{}".format(
+                        self.strategy, ve
+                    )
+                )
                 raise new_ve from None
             else:
                 raise ve
 
+        if in_fit:
+            # Use the dtype seen in `fit` for non-`fit` conversion
+            self._fit_dtype = X.dtype
+
         _check_inputs_dtype(X, self.missing_values)
         if X.dtype.kind not in ("i", "u", "f", "O"):
-            raise ValueError("SimpleImputer does not support data with dtype "
-                             "{0}. Please provide either a numeric array (with"
-                             " a floating point or integer dtype) or "
-                             "categorical data represented either as an array "
-                             "with integer dtype or an array of string values "
-                             "with an object dtype.".format(X.dtype))
+            raise ValueError(
+                "SimpleImputer does not support data with dtype "
+                "{0}. Please provide either a numeric array (with"
+                " a floating point or integer dtype) or "
+                "categorical data represented either as an array "
+                "with integer dtype or an array of string values "
+                "with an object dtype.".format(X.dtype)
+            )
+
+        if sp.issparse(X) and self.missing_values == 0:
+            # missing_values = 0 not allowed with sparse data as it would
+            # force densification
+            raise ValueError(
+                "Imputation not possible when missing_values "
+                "== 0 and input is sparse. Provide a dense "
+                "array instead."
+            )
+
+        if self.strategy == "constant":
+            if in_fit and self.fill_value is not None:
+                fill_value_dtype = type(self.fill_value)
+                err_msg = (
+                    f"fill_value={self.fill_value!r} (of type {fill_value_dtype!r}) "
+                    f"cannot be cast to the input data that is {X.dtype!r}. "
+                    "If fill_value is a Python scalar, instead pass  a numpy scalar "
+                    "(e.g. fill_value=np.uint8(0) if your data is of type np.uint8). "
+                    "Make sure that both dtypes are of the same kind."
+                )
+            elif not in_fit:
+                fill_value_dtype = self.statistics_.dtype
+                err_msg = (
+                    f"The dtype of the filling value (i.e. {fill_value_dtype!r}) "
+                    f"cannot be cast to the input data that is {X.dtype!r}. "
+                    "Make sure that the dtypes of the input data are of the same kind "
+                    "between fit and transform."
+                )
+            else:
+                # By default, fill_value=None, and the replacement is always
+                # compatible with the input data
+                fill_value_dtype = X.dtype
+
+            # Make sure we can safely cast fill_value dtype to the input data dtype
+            if not np.can_cast(fill_value_dtype, X.dtype, casting="same_kind"):
+                raise ValueError(err_msg)
 
         return X
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y=None):
-        """Fit the imputer on X.
+        """Fit the imputer on `X`.
 
         Parameters
         ----------
         X : {array-like, sparse matrix}, shape (n_samples, n_features)
-            Input data, where ``n_samples`` is the number of samples and
-            ``n_features`` is the number of features.
+            Input data, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
+
+        y : Ignored
+            Not used, present here for API consistency by convention.
 
         Returns
         -------
-        self : SimpleImputer
+        self : object
+            Fitted estimator.
         """
         X = self._validate_input(X, in_fit=True)
 
@@ -297,32 +445,14 @@ def fit(self, X, y=None):
         else:
             fill_value = self.fill_value
 
-        # fill_value should be numerical in case of numerical input
-        if (self.strategy == "constant" and
-                X.dtype.kind in ("i", "u", "f") and
-                not isinstance(fill_value, numbers.Real)):
-            raise ValueError("'fill_value'={0} is invalid. Expected a "
-                             "numerical value when imputing numerical "
-                             "data".format(fill_value))
-
         if sp.issparse(X):
-            # missing_values = 0 not allowed with sparse data as it would
-            # force densification
-            if self.missing_values == 0:
-                raise ValueError("Imputation not possible when missing_values "
-                                 "== 0 and input is sparse. Provide a dense "
-                                 "array instead.")
-            else:
-                self.statistics_ = self._sparse_fit(X,
-                                                    self.strategy,
-                                                    self.missing_values,
-                                                    fill_value)
-
+            self.statistics_ = self._sparse_fit(
+                X, self.strategy, self.missing_values, fill_value
+            )
         else:
-            self.statistics_ = self._dense_fit(X,
-                                               self.strategy,
-                                               self.missing_values,
-                                               fill_value)
+            self.statistics_ = self._dense_fit(
+                X, self.strategy, self.missing_values, fill_value
+            )
 
         return self
 
@@ -335,13 +465,26 @@ def _sparse_fit(self, X, strategy, missing_values, fill_value):
         statistics = np.empty(X.shape[1])
 
         if strategy == "constant":
-            # for constant strategy, self.statistcs_ is used to store
+            # TODO(1.8): Remove FutureWarning and add `np.nan` as a statistic
+            # for empty features to drop them later.
+            if not self.keep_empty_features and any(
+                [all(missing_mask[:, i].data) for i in range(missing_mask.shape[1])]
+            ):
+                warnings.warn(
+                    "Currently, when `keep_empty_feature=False` and "
+                    '`strategy="constant"`, empty features are not dropped. '
+                    "This behaviour will change in version 1.8. Set "
+                    "`keep_empty_feature=True` to preserve this behaviour.",
+                    FutureWarning,
+                )
+
+            # for constant strategy, self.statistics_ is used to store
             # fill_value in each column
             statistics.fill(fill_value)
         else:
             for i in range(X.shape[1]):
-                column = X.data[X.indptr[i]:X.indptr[i + 1]]
-                mask_column = mask_data[X.indptr[i]:X.indptr[i + 1]]
+                column = X.data[X.indptr[i] : X.indptr[i + 1]]
+                mask_column = mask_data[X.indptr[i] : X.indptr[i + 1]]
                 column = column[~mask_column]
 
                 # combine explicit and implicit zeros
@@ -350,18 +493,23 @@ def _sparse_fit(self, X, strategy, missing_values, fill_value):
                 n_explicit_zeros = mask_zeros.sum()
                 n_zeros = n_implicit_zeros[i] + n_explicit_zeros
 
-                if strategy == "mean":
-                    s = column.size + n_zeros
-                    statistics[i] = np.nan if s == 0 else column.sum() / s
+                if len(column) == 0 and self.keep_empty_features:
+                    # in case we want to keep columns with only missing values.
+                    statistics[i] = 0
+                else:
+                    if strategy == "mean":
+                        s = column.size + n_zeros
+                        statistics[i] = np.nan if s == 0 else column.sum() / s
+
+                    elif strategy == "median":
+                        statistics[i] = _get_median(column, n_zeros)
 
-                elif strategy == "median":
-                    statistics[i] = _get_median(column,
-                                                n_zeros)
+                    elif strategy == "most_frequent":
+                        statistics[i] = _most_frequent(column, 0, n_zeros)
+
+                    elif isinstance(strategy, Callable):
+                        statistics[i] = self.strategy(column)
 
-                elif strategy == "most_frequent":
-                    statistics[i] = _most_frequent(column,
-                                                   0,
-                                                   n_zeros)
         super()._fit_indicator(missing_mask)
 
         return statistics
@@ -378,7 +526,7 @@ def _dense_fit(self, X, strategy, missing_values, fill_value):
             mean_masked = np.ma.mean(masked_X, axis=0)
             # Avoid the warning "Warning: converting a masked element to nan."
             mean = np.ma.getdata(mean_masked)
-            mean[np.ma.getmask(mean_masked)] = np.nan
+            mean[np.ma.getmask(mean_masked)] = 0 if self.keep_empty_features else np.nan
 
             return mean
 
@@ -387,7 +535,9 @@ def _dense_fit(self, X, strategy, missing_values, fill_value):
             median_masked = np.ma.median(masked_X, axis=0)
             # Avoid the warning "Warning: converting a masked element to nan."
             median = np.ma.getdata(median_masked)
-            median[np.ma.getmaskarray(median_masked)] = np.nan
+            median[np.ma.getmaskarray(median_masked)] = (
+                0 if self.keep_empty_features else np.nan
+            )
 
             return median
 
@@ -409,23 +559,50 @@ def _dense_fit(self, X, strategy, missing_values, fill_value):
             for i, (row, row_mask) in enumerate(zip(X[:], mask[:])):
                 row_mask = np.logical_not(row_mask).astype(bool)
                 row = row[row_mask]
-                most_frequent[i] = _most_frequent(row, np.nan, 0)
+                if len(row) == 0 and self.keep_empty_features:
+                    most_frequent[i] = 0
+                else:
+                    most_frequent[i] = _most_frequent(row, np.nan, 0)
 
             return most_frequent
 
         # Constant
         elif strategy == "constant":
+            # TODO(1.8): Remove FutureWarning and add `np.nan` as a statistic
+            # for empty features to drop them later.
+            if not self.keep_empty_features and ma.getmask(masked_X).all(axis=0).any():
+                warnings.warn(
+                    "Currently, when `keep_empty_feature=False` and "
+                    '`strategy="constant"`, empty features are not dropped. '
+                    "This behaviour will change in version 1.8. Set "
+                    "`keep_empty_feature=True` to preserve this behaviour.",
+                    FutureWarning,
+                )
+
             # for constant strategy, self.statistcs_ is used to store
             # fill_value in each column
             return np.full(X.shape[1], fill_value, dtype=X.dtype)
 
+        # Custom
+        elif isinstance(strategy, Callable):
+            statistics = np.empty(masked_X.shape[1])
+            for i in range(masked_X.shape[1]):
+                statistics[i] = self.strategy(masked_X[:, i].compressed())
+            return statistics
+
     def transform(self, X):
-        """Impute all missing values in X.
+        """Impute all missing values in `X`.
 
         Parameters
         ----------
         X : {array-like, sparse matrix}, shape (n_samples, n_features)
             The input data to complete.
+
+        Returns
+        -------
+        X_imputed : {ndarray, sparse matrix} of shape \
+                (n_samples, n_features_out)
+            `X` with imputed values.
         """
         check_is_fitted(self)
 
@@ -433,14 +610,16 @@ def transform(self, X):
         statistics = self.statistics_
 
         if X.shape[1] != statistics.shape[0]:
-            raise ValueError("X has %d features per sample, expected %d"
-                             % (X.shape[1], self.statistics_.shape[0]))
+            raise ValueError(
+                "X has %d features per sample, expected %d"
+                % (X.shape[1], self.statistics_.shape[0])
+            )
 
         # compute mask before eliminating invalid features
         missing_mask = _get_mask(X, self.missing_values)
 
-        # Delete the invalid columns if strategy is not constant
-        if self.strategy == "constant":
+        # Decide whether to keep missing features
+        if self.strategy == "constant" or self.keep_empty_features:
             valid_statistics = statistics
             valid_statistics_indexes = None
         else:
@@ -451,18 +630,25 @@ def transform(self, X):
             valid_statistics_indexes = np.flatnonzero(valid_mask)
 
             if invalid_mask.any():
-                missing = np.arange(X.shape[1])[invalid_mask]
-                if self.verbose:
-                    warnings.warn("Deleting features without "
-                                  "observed values: %s" % missing)
+                invalid_features = np.arange(X.shape[1])[invalid_mask]
+                # use feature names warning if features are provided
+                if hasattr(self, "feature_names_in_"):
+                    invalid_features = self.feature_names_in_[invalid_features]
+                warnings.warn(
+                    "Skipping features without any observed values:"
+                    f" {invalid_features}. At least one non-missing value is needed"
+                    f" for imputation with strategy='{self.strategy}'."
+                )
                 X = X[:, valid_statistics_indexes]
 
         # Do actual imputation
         if sp.issparse(X):
             if self.missing_values == 0:
-                raise ValueError("Imputation not possible when missing_values "
-                                 "== 0 and input is sparse. Provide a dense "
-                                 "array instead.")
+                raise ValueError(
+                    "Imputation not possible when missing_values "
+                    "== 0 and input is sparse. Provide a dense "
+                    "array instead."
+                )
             else:
                 # if no invalid statistics are found, use the mask computed
                 # before, else recompute mask
@@ -471,11 +657,10 @@ def transform(self, X):
                 else:
                     mask = _get_mask(X.data, self.missing_values)
                 indexes = np.repeat(
-                    np.arange(len(X.indptr) - 1, dtype=int),
-                    np.diff(X.indptr))[mask]
+                    np.arange(len(X.indptr) - 1, dtype=int), np.diff(X.indptr)
+                )[mask]
 
-                X.data[mask] = valid_statistics[indexes].astype(X.dtype,
-                                                                copy=False)
+                X.data[mask] = valid_statistics[indexes].astype(X.dtype, copy=False)
         else:
             # use mask computed before eliminating invalid mask
             if valid_statistics_indexes is None:
@@ -499,10 +684,10 @@ def inverse_transform(self, X):
         This operation can only be performed after :class:`SimpleImputer` is
         instantiated with `add_indicator=True`.
 
-        Note that ``inverse_transform`` can only invert the transform in
+        Note that `inverse_transform` can only invert the transform in
         features that have binary indicators for missing values. If a feature
-        has no missing values at ``fit`` time, the feature won't have a binary
-        indicator, and the imputation done at ``transform`` time won't be
+        has no missing values at `fit` time, the feature won't have a binary
+        indicator, and the imputation done at `transform` time won't be
         inverted.
 
         .. versionadded:: 0.24
@@ -517,17 +702,19 @@ def inverse_transform(self, X):
         Returns
         -------
         X_original : ndarray of shape (n_samples, n_features)
-            The original X with missing values as it was prior
+            The original `X` with missing values as it was prior
             to imputation.
         """
         check_is_fitted(self)
 
         if not self.add_indicator:
-            raise ValueError("'inverse_transform' works only when "
-                             "'SimpleImputer' is instantiated with "
-                             "'add_indicator=True'. "
-                             f"Got 'add_indicator={self.add_indicator}' "
-                             "instead.")
+            raise ValueError(
+                "'inverse_transform' works only when "
+                "'SimpleImputer' is instantiated with "
+                "'add_indicator=True'. "
+                f"Got 'add_indicator={self.add_indicator}' "
+                "instead."
+            )
 
         n_features_missing = len(self.indicator_.features_)
         non_empty_feature_count = X.shape[1] - n_features_missing
@@ -552,13 +739,49 @@ def inverse_transform(self, X):
         X_original[full_mask] = self.missing_values
         return X_original
 
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
+        tags.input_tags.allow_nan = is_pandas_na(self.missing_values) or is_scalar_nan(
+            self.missing_values
+        )
+        return tags
+
+    def get_feature_names_out(self, input_features=None):
+        """Get output feature names for transformation.
+
+        Parameters
+        ----------
+        input_features : array-like of str or None, default=None
+            Input features.
+
+            - If `input_features` is `None`, then `feature_names_in_` is
+              used as feature names in. If `feature_names_in_` is not defined,
+              then the following input feature names are generated:
+              `["x0", "x1", ..., "x(n_features_in_ - 1)"]`.
+            - If `input_features` is an array-like, then `input_features` must
+              match `feature_names_in_` if `feature_names_in_` is defined.
+
+        Returns
+        -------
+        feature_names_out : ndarray of str objects
+            Transformed feature names.
+        """
+        check_is_fitted(self, "n_features_in_")
+        input_features = _check_feature_names_in(self, input_features)
+        non_missing_mask = np.logical_not(_get_mask(self.statistics_, np.nan))
+        names = input_features[non_missing_mask]
+        return self._concatenate_indicator_feature_names_out(names, input_features)
+
 
 class MissingIndicator(TransformerMixin, BaseEstimator):
     """Binary indicators for missing values.
 
     Note that this component typically should not be used in a vanilla
-    :class:`Pipeline` consisting of transformers and a classifier, but rather
-    could be added using a :class:`FeatureUnion` or :class:`ColumnTransformer`.
+    :class:`~sklearn.pipeline.Pipeline` consisting of transformers and a
+    classifier, but rather could be added using a
+    :class:`~sklearn.pipeline.FeatureUnion` or
+    :class:`~sklearn.compose.ColumnTransformer`.
 
     Read more in the :ref:`User Guide `.
 
@@ -566,7 +789,7 @@ class MissingIndicator(TransformerMixin, BaseEstimator):
 
     Parameters
     ----------
-    missing_values : int, float, string, np.nan or None, default=np.nan
+    missing_values : int, float, str, np.nan or None, default=np.nan
         The placeholder for the missing values. All occurrences of
         `missing_values` will be imputed. For pandas' dataframes with
         nullable integer dtypes with missing values, `missing_values`
@@ -576,29 +799,45 @@ class MissingIndicator(TransformerMixin, BaseEstimator):
         Whether the imputer mask should represent all or a subset of
         features.
 
-        - If 'missing-only' (default), the imputer mask will only represent
+        - If `'missing-only'` (default), the imputer mask will only represent
           features containing missing values during fit time.
-        - If 'all', the imputer mask will represent all features.
+        - If `'all'`, the imputer mask will represent all features.
 
     sparse : bool or 'auto', default='auto'
         Whether the imputer mask format should be sparse or dense.
 
-        - If 'auto' (default), the imputer mask will be of same type as
+        - If `'auto'` (default), the imputer mask will be of same type as
           input.
-        - If True, the imputer mask will be a sparse matrix.
-        - If False, the imputer mask will be a numpy array.
+        - If `True`, the imputer mask will be a sparse matrix.
+        - If `False`, the imputer mask will be a numpy array.
 
     error_on_new : bool, default=True
-        If True, transform will raise an error when there are features with
-        missing values in transform that have no missing values in fit. This is
-        applicable only when `features='missing-only'`.
+        If `True`, :meth:`transform` will raise an error when there are
+        features with missing values that have no missing values in
+        :meth:`fit`. This is applicable only when `features='missing-only'`.
 
     Attributes
     ----------
-    features_ : ndarray, shape (n_missing_features,) or (n_features,)
-        The features indices which will be returned when calling ``transform``.
-        They are computed during ``fit``. For ``features='all'``, it is
-        to ``range(n_features)``.
+    features_ : ndarray of shape (n_missing_features,) or (n_features,)
+        The features indices which will be returned when calling
+        :meth:`transform`. They are computed during :meth:`fit`. If
+        `features='all'`, `features_` is equal to `range(n_features)`.
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    SimpleImputer : Univariate imputation of missing values.
+    IterativeImputer : Multivariate imputation of missing values.
 
     Examples
     --------
@@ -618,11 +857,23 @@ class MissingIndicator(TransformerMixin, BaseEstimator):
     array([[False,  True],
            [ True, False],
            [False, False]])
-
     """
-    @_deprecate_positional_args
-    def __init__(self, *, missing_values=np.nan, features="missing-only",
-                 sparse="auto", error_on_new=True):
+
+    _parameter_constraints: dict = {
+        "missing_values": [MissingValues()],
+        "features": [StrOptions({"missing-only", "all"})],
+        "sparse": ["boolean", StrOptions({"auto"})],
+        "error_on_new": ["boolean"],
+    }
+
+    def __init__(
+        self,
+        *,
+        missing_values=np.nan,
+        features="missing-only",
+        sparse="auto",
+        error_on_new=True,
+    ):
         self.missing_values = missing_values
         self.features = features
         self.sparse = sparse
@@ -634,19 +885,19 @@ def _get_missing_features_info(self, X):
 
         Parameters
         ----------
-        X : {ndarray or sparse matrix}, shape (n_samples, n_features)
-            The input data with missing values. Note that ``X`` has been
-            checked in ``fit`` and ``transform`` before to call this function.
+        X : {ndarray, sparse matrix} of shape (n_samples, n_features)
+            The input data with missing values. Note that `X` has been
+            checked in :meth:`fit` and :meth:`transform` before to call this
+            function.
 
         Returns
         -------
-        imputer_mask : {ndarray or sparse matrix}, shape \
+        imputer_mask : {ndarray, sparse matrix} of shape \
         (n_samples, n_features)
             The imputer mask of the original data.
 
-        features_with_missing : ndarray, shape (n_features_with_missing)
+        features_with_missing : ndarray of shape (n_features_with_missing)
             The features containing missing values.
-
         """
         if not self._precomputed:
             imputer_mask = _get_mask(X, self.missing_values)
@@ -656,12 +907,13 @@ def _get_missing_features_info(self, X):
         if sp.issparse(X):
             imputer_mask.eliminate_zeros()
 
-            if self.features == 'missing-only':
-                n_missing = imputer_mask.getnnz(axis=0)
+            if self.features == "missing-only":
+                # count number of True values in each row.
+                n_missing = imputer_mask.sum(axis=0)
 
             if self.sparse is False:
                 imputer_mask = imputer_mask.toarray()
-            elif imputer_mask.format == 'csr':
+            elif imputer_mask.format == "csr":
                 imputer_mask = imputer_mask.tocsc()
         else:
             if not self._precomputed:
@@ -669,13 +921,13 @@ def _get_missing_features_info(self, X):
             else:
                 imputer_mask = X
 
-            if self.features == 'missing-only':
+            if self.features == "missing-only":
                 n_missing = imputer_mask.sum(axis=0)
 
             if self.sparse is True:
                 imputer_mask = sp.csc_matrix(imputer_mask)
 
-        if self.features == 'all':
+        if self.features == "all":
             features_indices = np.arange(X.shape[1])
         else:
             features_indices = np.flatnonzero(n_missing)
@@ -684,55 +936,61 @@ def _get_missing_features_info(self, X):
 
     def _validate_input(self, X, in_fit):
         if not is_scalar_nan(self.missing_values):
-            force_all_finite = True
+            ensure_all_finite = True
         else:
-            force_all_finite = "allow-nan"
-        X = self._validate_data(X, reset=in_fit,
-                                accept_sparse=('csc', 'csr'), dtype=None,
-                                force_all_finite=force_all_finite)
+            ensure_all_finite = "allow-nan"
+        X = validate_data(
+            self,
+            X,
+            reset=in_fit,
+            accept_sparse=("csc", "csr"),
+            dtype=None,
+            ensure_all_finite=ensure_all_finite,
+        )
         _check_inputs_dtype(X, self.missing_values)
         if X.dtype.kind not in ("i", "u", "f", "O"):
-            raise ValueError("MissingIndicator does not support data with "
-                             "dtype {0}. Please provide either a numeric array"
-                             " (with a floating point or integer dtype) or "
-                             "categorical data represented either as an array "
-                             "with integer dtype or an array of string values "
-                             "with an object dtype.".format(X.dtype))
+            raise ValueError(
+                "MissingIndicator does not support data with "
+                "dtype {0}. Please provide either a numeric array"
+                " (with a floating point or integer dtype) or "
+                "categorical data represented either as an array "
+                "with integer dtype or an array of string values "
+                "with an object dtype.".format(X.dtype)
+            )
 
         if sp.issparse(X) and self.missing_values == 0:
             # missing_values = 0 not allowed with sparse data as it would
             # force densification
-            raise ValueError("Sparse input with missing_values=0 is "
-                             "not supported. Provide a dense "
-                             "array instead.")
+            raise ValueError(
+                "Sparse input with missing_values=0 is "
+                "not supported. Provide a dense "
+                "array instead."
+            )
 
         return X
 
     def _fit(self, X, y=None, precomputed=False):
-        """Fit the transformer on X.
+        """Fit the transformer on `X`.
 
         Parameters
         ----------
-        X : {array-like, sparse matrix}, shape (n_samples, n_features)
-            Input data, where ``n_samples`` is the number of samples and
-            ``n_features`` is the number of features.
-            If `precomputed` is True, then `X` is a mask of the
-            input data.
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Input data, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
+            If `precomputed=True`, then `X` is a mask of the input data.
 
         precomputed : bool
             Whether the input data is a mask.
 
         Returns
         -------
-        imputer_mask : {ndarray or sparse matrix}, shape (n_samples, \
+        imputer_mask : {ndarray, sparse matrix} of shape (n_samples, \
         n_features)
             The imputer mask of the original data.
-
         """
         if precomputed:
-            if not (hasattr(X, 'dtype') and X.dtype.kind == 'b'):
-                raise ValueError("precomputed is True but the input data is "
-                                 "not a mask")
+            if not (hasattr(X, "dtype") and X.dtype.kind == "b"):
+                raise ValueError("precomputed is True but the input data is not a mask")
             self._precomputed = True
         else:
             self._precomputed = False
@@ -741,56 +999,53 @@ def _fit(self, X, y=None, precomputed=False):
         # in the Imputer calling MissingIndicator
         if not self._precomputed:
             X = self._validate_input(X, in_fit=True)
+        else:
+            # only create `n_features_in_` in the precomputed case
+            _check_n_features(self, X, reset=True)
 
         self._n_features = X.shape[1]
 
-        if self.features not in ('missing-only', 'all'):
-            raise ValueError("'features' has to be either 'missing-only' or "
-                             "'all'. Got {} instead.".format(self.features))
-
-        if not ((isinstance(self.sparse, str) and
-                self.sparse == "auto") or isinstance(self.sparse, bool)):
-            raise ValueError("'sparse' has to be a boolean or 'auto'. "
-                             "Got {!r} instead.".format(self.sparse))
-
         missing_features_info = self._get_missing_features_info(X)
         self.features_ = missing_features_info[1]
 
         return missing_features_info[0]
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y=None):
-        """Fit the transformer on X.
+        """Fit the transformer on `X`.
 
         Parameters
         ----------
-        X : {array-like, sparse matrix}, shape (n_samples, n_features)
-            Input data, where ``n_samples`` is the number of samples and
-            ``n_features`` is the number of features.
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Input data, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
+
+        y : Ignored
+            Not used, present for API consistency by convention.
 
         Returns
         -------
         self : object
-            Returns self.
+            Fitted estimator.
         """
         self._fit(X, y)
 
         return self
 
     def transform(self, X):
-        """Generate missing values indicator for X.
+        """Generate missing values indicator for `X`.
 
         Parameters
         ----------
-        X : {array-like, sparse matrix}, shape (n_samples, n_features)
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
             The input data to complete.
 
         Returns
         -------
-        Xt : {ndarray or sparse matrix}, shape (n_samples, n_features) \
+        Xt : {ndarray, sparse matrix} of shape (n_samples, n_features) \
         or (n_samples, n_features_with_missing)
-            The missing indicator for input data. The data type of ``Xt``
+            The missing indicator for input data. The data type of `Xt`
             will be boolean.
-
         """
         check_is_fitted(self)
 
@@ -799,39 +1054,43 @@ def transform(self, X):
         if not self._precomputed:
             X = self._validate_input(X, in_fit=False)
         else:
-            if not (hasattr(X, 'dtype') and X.dtype.kind == 'b'):
-                raise ValueError("precomputed is True but the input data is "
-                                 "not a mask")
+            if not (hasattr(X, "dtype") and X.dtype.kind == "b"):
+                raise ValueError("precomputed is True but the input data is not a mask")
 
         imputer_mask, features = self._get_missing_features_info(X)
 
         if self.features == "missing-only":
             features_diff_fit_trans = np.setdiff1d(features, self.features_)
-            if (self.error_on_new and features_diff_fit_trans.size > 0):
-                raise ValueError("The features {} have missing values "
-                                 "in transform but have no missing values "
-                                 "in fit.".format(features_diff_fit_trans))
+            if self.error_on_new and features_diff_fit_trans.size > 0:
+                raise ValueError(
+                    "The features {} have missing values "
+                    "in transform but have no missing values "
+                    "in fit.".format(features_diff_fit_trans)
+                )
 
             if self.features_.size < self._n_features:
                 imputer_mask = imputer_mask[:, self.features_]
 
         return imputer_mask
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit_transform(self, X, y=None):
-        """Generate missing values indicator for X.
+        """Generate missing values indicator for `X`.
 
         Parameters
         ----------
-        X : {array-like, sparse matrix}, shape (n_samples, n_features)
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
             The input data to complete.
 
+        y : Ignored
+            Not used, present for API consistency by convention.
+
         Returns
         -------
-        Xt : {ndarray or sparse matrix}, shape (n_samples, n_features) \
+        Xt : {ndarray, sparse matrix} of shape (n_samples, n_features) \
         or (n_samples, n_features_with_missing)
-            The missing indicator for input data. The data type of ``Xt``
+            The missing indicator for input data. The data type of `Xt`
             will be boolean.
-
         """
         imputer_mask = self._fit(X, y)
 
@@ -840,9 +1099,41 @@ def fit_transform(self, X, y=None):
 
         return imputer_mask
 
-    def _more_tags(self):
-        return {
-            "allow_nan": True,
-            "X_types": ["2darray", "string"],
-            "preserves_dtype": [],
-        }
+    def get_feature_names_out(self, input_features=None):
+        """Get output feature names for transformation.
+
+        Parameters
+        ----------
+        input_features : array-like of str or None, default=None
+            Input features.
+
+            - If `input_features` is `None`, then `feature_names_in_` is
+              used as feature names in. If `feature_names_in_` is not defined,
+              then the following input feature names are generated:
+              `["x0", "x1", ..., "x(n_features_in_ - 1)"]`.
+            - If `input_features` is an array-like, then `input_features` must
+              match `feature_names_in_` if `feature_names_in_` is defined.
+
+        Returns
+        -------
+        feature_names_out : ndarray of str objects
+            Transformed feature names.
+        """
+        check_is_fitted(self, "n_features_in_")
+        input_features = _check_feature_names_in(self, input_features)
+        prefix = self.__class__.__name__.lower()
+        return np.asarray(
+            [
+                f"{prefix}_{feature_name}"
+                for feature_name in input_features[self.features_]
+            ],
+            dtype=object,
+        )
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.allow_nan = True
+        tags.input_tags.string = True
+        tags.input_tags.sparse = True
+        tags.transformer_tags.preserves_dtype = []
+        return tags
diff --git a/sklearn/impute/_iterative.py b/sklearn/impute/_iterative.py
index f5688fa96d238..ddae5373c5460 100644
--- a/sklearn/impute/_iterative.py
+++ b/sklearn/impute/_iterative.py
@@ -1,27 +1,60 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-from time import time
-from collections import namedtuple
 import warnings
+from collections import namedtuple
+from numbers import Integral, Real
+from time import time
 
-from scipy import stats
 import numpy as np
+from scipy import stats
 
-from ..base import clone
+from ..base import _fit_context, clone
 from ..exceptions import ConvergenceWarning
 from ..preprocessing import normalize
-from ..utils import (check_array, check_random_state, _safe_indexing,
-                     is_scalar_nan)
-from ..utils.validation import FLOAT_DTYPES, check_is_fitted
+from ..utils import _safe_indexing, check_array, check_random_state
+from ..utils._indexing import _safe_assign
 from ..utils._mask import _get_mask
+from ..utils._missing import is_scalar_nan
+from ..utils._param_validation import HasMethods, Interval, StrOptions
+from ..utils.metadata_routing import (
+    MetadataRouter,
+    MethodMapping,
+    _raise_for_params,
+    process_routing,
+)
+from ..utils.validation import (
+    FLOAT_DTYPES,
+    _check_feature_names_in,
+    _num_samples,
+    check_is_fitted,
+    validate_data,
+)
+from ._base import SimpleImputer, _BaseImputer, _check_inputs_dtype
+
+_ImputerTriplet = namedtuple(
+    "_ImputerTriplet", ["feat_idx", "neighbor_feat_idx", "estimator"]
+)
+
+
+def _assign_where(X1, X2, cond):
+    """Assign X2 to X1 where cond is True.
 
-from ._base import _BaseImputer
-from ._base import SimpleImputer
-from ._base import _check_inputs_dtype
+    Parameters
+    ----------
+    X1 : ndarray or dataframe of shape (n_samples, n_features)
+        Data.
 
+    X2 : ndarray of shape (n_samples, n_features)
+        Data to be assigned.
 
-_ImputerTriplet = namedtuple('_ImputerTriplet', ['feat_idx',
-                                                 'neighbor_feat_idx',
-                                                 'estimator'])
+    cond : ndarray of shape (n_samples, n_features)
+        Boolean mask to assign data.
+    """
+    if hasattr(X1, "mask"):  # pandas dataframes
+        X1.mask(cond=cond, other=X2, inplace=True)
+    else:  # ndarrays
+        X1[cond] = X2[cond]
 
 
 class IterativeImputer(_BaseImputer):
@@ -38,7 +71,7 @@ class IterativeImputer(_BaseImputer):
 
       This estimator is still **experimental** for now: the predictions
       and the API might change without any deprecation cycle. To use it,
-      you need to explicitly import ``enable_iterative_imputer``::
+      you need to explicitly import `enable_iterative_imputer`::
 
         >>> # explicitly require this experimental feature
         >>> from sklearn.experimental import enable_iterative_imputer  # noqa
@@ -49,20 +82,20 @@ class IterativeImputer(_BaseImputer):
     ----------
     estimator : estimator object, default=BayesianRidge()
         The estimator to use at each step of the round-robin imputation.
-        If ``sample_posterior`` is True, the estimator must support
-        ``return_std`` in its ``predict`` method.
+        If `sample_posterior=True`, the estimator must support
+        `return_std` in its `predict` method.
 
-    missing_values : int, np.nan, default=np.nan
+    missing_values : int or np.nan, default=np.nan
         The placeholder for the missing values. All occurrences of
         `missing_values` will be imputed. For pandas' dataframes with
         nullable integer dtypes with missing values, `missing_values`
         should be set to `np.nan`, since `pd.NA` will be converted to `np.nan`.
 
-    sample_posterior : boolean, default=False
+    sample_posterior : bool, default=False
         Whether to sample from the (Gaussian) predictive posterior of the
         fitted estimator for each imputation. Estimator must support
-        ``return_std`` in its ``predict`` method if set to ``True``. Set to
-        ``True`` if using ``IterativeImputer`` for multiple imputations.
+        `return_std` in its `predict` method if set to `True`. Set to
+        `True` if using `IterativeImputer` for multiple imputations.
 
     max_iter : int, default=10
         Maximum number of imputation rounds to perform before returning the
@@ -70,7 +103,7 @@ class IterativeImputer(_BaseImputer):
         imputation of each feature with missing values. The stopping criterion
         is met once `max(abs(X_t - X_{t-1}))/max(abs(X[known_vals])) < tol`,
         where `X_t` is `X` at iteration `t`. Note that early stopping is only
-        applied if ``sample_posterior=False``.
+        applied if `sample_posterior=False`.
 
     tol : float, default=1e-3
         Tolerance of the stopping condition.
@@ -83,45 +116,50 @@ class IterativeImputer(_BaseImputer):
         imputation process, the neighbor features are not necessarily nearest,
         but are drawn with probability proportional to correlation for each
         imputed target feature. Can provide significant speed-up when the
-        number of features is huge. If ``None``, all features will be used.
+        number of features is huge. If `None`, all features will be used.
 
-    initial_strategy : str, default='mean'
+    initial_strategy : {'mean', 'median', 'most_frequent', 'constant'}, \
+            default='mean'
         Which strategy to use to initialize the missing values. Same as the
-        ``strategy`` parameter in :class:`~sklearn.impute.SimpleImputer`
-        Valid values: {"mean", "median", "most_frequent", or "constant"}.
+        `strategy` parameter in :class:`~sklearn.impute.SimpleImputer`.
+
+    fill_value : str or numerical value, default=None
+        When `strategy="constant"`, `fill_value` is used to replace all
+        occurrences of missing_values. For string or object data types,
+        `fill_value` must be a string.
+        If `None`, `fill_value` will be 0 when imputing numerical
+        data and "missing_value" for strings or object data types.
+
+        .. versionadded:: 1.3
 
-    imputation_order : str, default='ascending'
+    imputation_order : {'ascending', 'descending', 'roman', 'arabic', \
+            'random'}, default='ascending'
         The order in which the features will be imputed. Possible values:
 
-        "ascending"
-            From features with fewest missing values to most.
-        "descending"
-            From features with most missing values to fewest.
-        "roman"
-            Left to right.
-        "arabic"
-            Right to left.
-        "random"
-            A random order for each round.
-
-    skip_complete : boolean, default=False
-        If ``True`` then features with missing values during ``transform``
-        which did not have any missing values during ``fit`` will be imputed
-        with the initial imputation method only. Set to ``True`` if you have
-        many features with no missing values at both ``fit`` and ``transform``
-        time to save compute.
+        - `'ascending'`: From features with fewest missing values to most.
+        - `'descending'`: From features with most missing values to fewest.
+        - `'roman'`: Left to right.
+        - `'arabic'`: Right to left.
+        - `'random'`: A random order for each round.
+
+    skip_complete : bool, default=False
+        If `True` then features with missing values during :meth:`transform`
+        which did not have any missing values during :meth:`fit` will be
+        imputed with the initial imputation method only. Set to `True` if you
+        have many features with no missing values at both :meth:`fit` and
+        :meth:`transform` time to save compute.
 
     min_value : float or array-like of shape (n_features,), default=-np.inf
-        Minimum possible imputed value. Broadcast to shape (n_features,) if
-        scalar. If array-like, expects shape (n_features,), one min value for
+        Minimum possible imputed value. Broadcast to shape `(n_features,)` if
+        scalar. If array-like, expects shape `(n_features,)`, one min value for
         each feature. The default is `-np.inf`.
 
         .. versionchanged:: 0.23
            Added support for array-like.
 
     max_value : float or array-like of shape (n_features,), default=np.inf
-        Maximum possible imputed value. Broadcast to shape (n_features,) if
-        scalar. If array-like, expects shape (n_features,), one max value for
+        Maximum possible imputed value. Broadcast to shape `(n_features,)` if
+        scalar. If array-like, expects shape `(n_features,)`, one max value for
         each feature. The default is `np.inf`.
 
         .. versionchanged:: 0.23
@@ -134,42 +172,62 @@ class IterativeImputer(_BaseImputer):
 
     random_state : int, RandomState instance or None, default=None
         The seed of the pseudo random number generator to use. Randomizes
-        selection of estimator features if n_nearest_features is not None, the
-        ``imputation_order`` if ``random``, and the sampling from posterior if
-        ``sample_posterior`` is True. Use an integer for determinism.
+        selection of estimator features if `n_nearest_features` is not `None`,
+        the `imputation_order` if `random`, and the sampling from posterior if
+        `sample_posterior=True`. Use an integer for determinism.
         See :term:`the Glossary `.
 
-    add_indicator : boolean, default=False
-        If True, a :class:`MissingIndicator` transform will stack onto output
+    add_indicator : bool, default=False
+        If `True`, a :class:`MissingIndicator` transform will stack onto output
         of the imputer's transform. This allows a predictive estimator
         to account for missingness despite imputation. If a feature has no
         missing values at fit/train time, the feature won't appear on
         the missing indicator even if there are missing values at
         transform/test time.
 
+    keep_empty_features : bool, default=False
+        If True, features that consist exclusively of missing values when
+        `fit` is called are returned in results when `transform` is called.
+        The imputed value is always `0` except when
+        `initial_strategy="constant"` in which case `fill_value` will be
+        used instead.
+
+        .. versionadded:: 1.2
+
     Attributes
     ----------
     initial_imputer_ : object of type :class:`~sklearn.impute.SimpleImputer`
         Imputer used to initialize the missing values.
 
     imputation_sequence_ : list of tuples
-        Each tuple has ``(feat_idx, neighbor_feat_idx, estimator)``, where
-        ``feat_idx`` is the current feature to be imputed,
-        ``neighbor_feat_idx`` is the array of other features used to impute the
-        current feature, and ``estimator`` is the trained estimator used for
-        the imputation. Length is ``self.n_features_with_missing_ *
-        self.n_iter_``.
+        Each tuple has `(feat_idx, neighbor_feat_idx, estimator)`, where
+        `feat_idx` is the current feature to be imputed,
+        `neighbor_feat_idx` is the array of other features used to impute the
+        current feature, and `estimator` is the trained estimator used for
+        the imputation. Length is `self.n_features_with_missing_ *
+        self.n_iter_`.
 
     n_iter_ : int
         Number of iteration rounds that occurred. Will be less than
-        ``self.max_iter`` if early stopping criterion was reached.
+        `self.max_iter` if early stopping criterion was reached.
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
 
     n_features_with_missing_ : int
         Number of features with missing values.
 
     indicator_ : :class:`~sklearn.impute.MissingIndicator`
         Indicator used to add binary indicators for missing values.
-        ``None`` if add_indicator is False.
+        `None` if `add_indicator=False`.
 
     random_state_ : RandomState instance
         RandomState instance that is generated either from a seed, the random
@@ -177,30 +235,29 @@ class IterativeImputer(_BaseImputer):
 
     See Also
     --------
-    SimpleImputer : Univariate imputation of missing values.
-
-    Examples
-    --------
-    >>> import numpy as np
-    >>> from sklearn.experimental import enable_iterative_imputer
-    >>> from sklearn.impute import IterativeImputer
-    >>> imp_mean = IterativeImputer(random_state=0)
-    >>> imp_mean.fit([[7, 2, 3], [4, np.nan, 6], [10, 5, 9]])
-    IterativeImputer(random_state=0)
-    >>> X = [[np.nan, 2, 3], [4, np.nan, 6], [10, np.nan, 9]]
-    >>> imp_mean.transform(X)
-    array([[ 6.9584...,  2.       ,  3.        ],
-           [ 4.       ,  2.6000...,  6.        ],
-           [10.       ,  4.9999...,  9.        ]])
+    SimpleImputer : Univariate imputer for completing missing values
+        with simple strategies.
+    KNNImputer : Multivariate imputer that estimates missing features using
+        nearest samples.
 
     Notes
     -----
     To support imputation in inductive mode we store each feature's estimator
-    during the ``fit`` phase, and predict without refitting (in order) during
-    the ``transform`` phase.
+    during the :meth:`fit` phase, and predict without refitting (in order)
+    during the :meth:`transform` phase.
+
+    Features which contain all missing values at :meth:`fit` are discarded upon
+    :meth:`transform`.
+
+    Using defaults, the imputer scales in :math:`\\mathcal{O}(knp^3\\min(n,p))`
+    where :math:`k` = `max_iter`, :math:`n` the number of samples and
+    :math:`p` the number of features. It thus becomes prohibitively costly when
+    the number of features increases. Setting
+    `n_nearest_features << n_features`, `skip_complete=True` or increasing `tol`
+    can help to reduce its computational cost.
 
-    Features which contain all missing values at ``fit`` are discarded upon
-    ``transform``.
+    Depending on the nature of missing values, simple imputers can be
+    preferable in a prediction context.
 
     References
     ----------
@@ -213,25 +270,71 @@ class IterativeImputer(_BaseImputer):
         Multivariate Data Suitable for use with an Electronic Computer".
         Journal of the Royal Statistical Society 22(2): 302-306.
         `_
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from sklearn.experimental import enable_iterative_imputer
+    >>> from sklearn.impute import IterativeImputer
+    >>> imp_mean = IterativeImputer(random_state=0)
+    >>> imp_mean.fit([[7, 2, 3], [4, np.nan, 6], [10, 5, 9]])
+    IterativeImputer(random_state=0)
+    >>> X = [[np.nan, 2, 3], [4, np.nan, 6], [10, np.nan, 9]]
+    >>> imp_mean.transform(X)
+    array([[ 6.9584,  2.       ,  3.        ],
+           [ 4.       ,  2.6000,  6.        ],
+           [10.       ,  4.9999,  9.        ]])
+
+    For a more detailed example see
+    :ref:`sphx_glr_auto_examples_impute_plot_missing_values.py` or
+    :ref:`sphx_glr_auto_examples_impute_plot_iterative_imputer_variants_comparison.py`.
     """
-    def __init__(self,
-                 estimator=None, *,
-                 missing_values=np.nan,
-                 sample_posterior=False,
-                 max_iter=10,
-                 tol=1e-3,
-                 n_nearest_features=None,
-                 initial_strategy="mean",
-                 imputation_order='ascending',
-                 skip_complete=False,
-                 min_value=-np.inf,
-                 max_value=np.inf,
-                 verbose=0,
-                 random_state=None,
-                 add_indicator=False):
+
+    _parameter_constraints: dict = {
+        **_BaseImputer._parameter_constraints,
+        "estimator": [None, HasMethods(["fit", "predict"])],
+        "sample_posterior": ["boolean"],
+        "max_iter": [Interval(Integral, 0, None, closed="left")],
+        "tol": [Interval(Real, 0, None, closed="left")],
+        "n_nearest_features": [None, Interval(Integral, 1, None, closed="left")],
+        "initial_strategy": [
+            StrOptions({"mean", "median", "most_frequent", "constant"})
+        ],
+        "fill_value": "no_validation",  # any object is valid
+        "imputation_order": [
+            StrOptions({"ascending", "descending", "roman", "arabic", "random"})
+        ],
+        "skip_complete": ["boolean"],
+        "min_value": [None, Interval(Real, None, None, closed="both"), "array-like"],
+        "max_value": [None, Interval(Real, None, None, closed="both"), "array-like"],
+        "verbose": ["verbose"],
+        "random_state": ["random_state"],
+    }
+
+    def __init__(
+        self,
+        estimator=None,
+        *,
+        missing_values=np.nan,
+        sample_posterior=False,
+        max_iter=10,
+        tol=1e-3,
+        n_nearest_features=None,
+        initial_strategy="mean",
+        fill_value=None,
+        imputation_order="ascending",
+        skip_complete=False,
+        min_value=-np.inf,
+        max_value=np.inf,
+        verbose=0,
+        random_state=None,
+        add_indicator=False,
+        keep_empty_features=False,
+    ):
         super().__init__(
             missing_values=missing_values,
-            add_indicator=add_indicator
+            add_indicator=add_indicator,
+            keep_empty_features=keep_empty_features,
         )
 
         self.estimator = estimator
@@ -240,6 +343,7 @@ def __init__(self,
         self.tol = tol
         self.n_nearest_features = n_nearest_features
         self.initial_strategy = initial_strategy
+        self.fill_value = fill_value
         self.imputation_order = imputation_order
         self.skip_complete = skip_complete
         self.min_value = min_value
@@ -247,18 +351,21 @@ def __init__(self,
         self.verbose = verbose
         self.random_state = random_state
 
-    def _impute_one_feature(self,
-                            X_filled,
-                            mask_missing_values,
-                            feat_idx,
-                            neighbor_feat_idx,
-                            estimator=None,
-                            fit_mode=True):
+    def _impute_one_feature(
+        self,
+        X_filled,
+        mask_missing_values,
+        feat_idx,
+        neighbor_feat_idx,
+        estimator=None,
+        fit_mode=True,
+        params=None,
+    ):
         """Impute a single feature from the others provided.
 
         This function predicts the missing values of one of the features using
-        the current estimates of all the other features. The ``estimator`` must
-        support ``return_std=True`` in its ``predict`` method for this function
+        the current estimates of all the other features. The `estimator` must
+        support `return_std=True` in its `predict` method for this function
         to work.
 
         Parameters
@@ -273,48 +380,62 @@ def _impute_one_feature(self,
             Index of the feature currently being imputed.
 
         neighbor_feat_idx : ndarray
-            Indices of the features to be used in imputing ``feat_idx``.
+            Indices of the features to be used in imputing `feat_idx`.
 
         estimator : object
             The estimator to use at this step of the round-robin imputation.
-            If ``sample_posterior`` is True, the estimator must support
-            ``return_std`` in its ``predict`` method.
+            If `sample_posterior=True`, the estimator must support
+            `return_std` in its `predict` method.
             If None, it will be cloned from self._estimator.
 
         fit_mode : boolean, default=True
             Whether to fit and predict with the estimator or just predict.
 
+        params : dict
+            Additional params routed to the individual estimator.
+
         Returns
         -------
         X_filled : ndarray
-            Input data with ``X_filled[missing_row_mask, feat_idx]`` updated.
+            Input data with `X_filled[missing_row_mask, feat_idx]` updated.
 
         estimator : estimator with sklearn API
             The fitted estimator used to impute
-            ``X_filled[missing_row_mask, feat_idx]``.
+            `X_filled[missing_row_mask, feat_idx]`.
         """
         if estimator is None and fit_mode is False:
-            raise ValueError("If fit_mode is False, then an already-fitted "
-                             "estimator should be passed in.")
+            raise ValueError(
+                "If fit_mode is False, then an already-fitted "
+                "estimator should be passed in."
+            )
 
         if estimator is None:
             estimator = clone(self._estimator)
 
         missing_row_mask = mask_missing_values[:, feat_idx]
         if fit_mode:
-            X_train = _safe_indexing(X_filled[:, neighbor_feat_idx],
-                                     ~missing_row_mask)
-            y_train = _safe_indexing(X_filled[:, feat_idx],
-                                     ~missing_row_mask)
-            estimator.fit(X_train, y_train)
+            X_train = _safe_indexing(
+                _safe_indexing(X_filled, neighbor_feat_idx, axis=1),
+                ~missing_row_mask,
+                axis=0,
+            )
+            y_train = _safe_indexing(
+                _safe_indexing(X_filled, feat_idx, axis=1),
+                ~missing_row_mask,
+                axis=0,
+            )
+            estimator.fit(X_train, y_train, **params)
 
         # if no missing values, don't predict
         if np.sum(missing_row_mask) == 0:
             return X_filled, estimator
 
         # get posterior samples if there is at least one missing value
-        X_test = _safe_indexing(X_filled[:, neighbor_feat_idx],
-                                missing_row_mask)
+        X_test = _safe_indexing(
+            _safe_indexing(X_filled, neighbor_feat_idx, axis=1),
+            missing_row_mask,
+            axis=0,
+        )
         if self.sample_posterior:
             mus, sigmas = estimator.predict(X_test, return_std=True)
             imputed_values = np.zeros(mus.shape, dtype=X_filled.dtype)
@@ -334,54 +455,55 @@ def _impute_one_feature(self,
             a = (self._min_value[feat_idx] - mus) / sigmas
             b = (self._max_value[feat_idx] - mus) / sigmas
 
-            truncated_normal = stats.truncnorm(a=a, b=b,
-                                               loc=mus, scale=sigmas)
+            truncated_normal = stats.truncnorm(a=a, b=b, loc=mus, scale=sigmas)
             imputed_values[inrange_mask] = truncated_normal.rvs(
-                random_state=self.random_state_)
+                random_state=self.random_state_
+            )
         else:
             imputed_values = estimator.predict(X_test)
-            imputed_values = np.clip(imputed_values,
-                                     self._min_value[feat_idx],
-                                     self._max_value[feat_idx])
+            imputed_values = np.clip(
+                imputed_values, self._min_value[feat_idx], self._max_value[feat_idx]
+            )
 
         # update the feature
-        X_filled[missing_row_mask, feat_idx] = imputed_values
+        _safe_assign(
+            X_filled,
+            imputed_values,
+            row_indexer=missing_row_mask,
+            column_indexer=feat_idx,
+        )
         return X_filled, estimator
 
-    def _get_neighbor_feat_idx(self,
-                               n_features,
-                               feat_idx,
-                               abs_corr_mat):
-        """Get a list of other features to predict ``feat_idx``.
+    def _get_neighbor_feat_idx(self, n_features, feat_idx, abs_corr_mat):
+        """Get a list of other features to predict `feat_idx`.
 
-        If self.n_nearest_features is less than or equal to the total
+        If `self.n_nearest_features` is less than or equal to the total
         number of features, then use a probability proportional to the absolute
-        correlation between ``feat_idx`` and each other feature to randomly
+        correlation between `feat_idx` and each other feature to randomly
         choose a subsample of the other features (without replacement).
 
         Parameters
         ----------
         n_features : int
-            Number of features in ``X``.
+            Number of features in `X`.
 
         feat_idx : int
             Index of the feature currently being imputed.
 
         abs_corr_mat : ndarray, shape (n_features, n_features)
-            Absolute correlation matrix of ``X``. The diagonal has been zeroed
+            Absolute correlation matrix of `X`. The diagonal has been zeroed
             out and each feature has been normalized to sum to 1. Can be None.
 
         Returns
         -------
         neighbor_feat_idx : array-like
-            The features to use to impute ``feat_idx``.
+            The features to use to impute `feat_idx`.
         """
-        if (self.n_nearest_features is not None and
-                self.n_nearest_features < n_features):
+        if self.n_nearest_features is not None and self.n_nearest_features < n_features:
             p = abs_corr_mat[:, feat_idx]
             neighbor_feat_idx = self.random_state_.choice(
-                np.arange(n_features), self.n_nearest_features, replace=False,
-                p=p)
+                np.arange(n_features), self.n_nearest_features, replace=False, p=p
+            )
         else:
             inds_left = np.arange(feat_idx)
             inds_right = np.arange(feat_idx + 1, n_features)
@@ -400,8 +522,8 @@ def _get_ordered_idx(self, mask_missing_values):
         Parameters
         ----------
         mask_missing_values : array-like, shape (n_samples, n_features)
-            Input data's missing indicator matrix, where "n_samples" is the
-            number of samples and "n_features" is the number of features.
+            Input data's missing indicator matrix, where `n_samples` is the
+            number of samples and `n_features` is the number of features.
 
         Returns
         -------
@@ -413,26 +535,19 @@ def _get_ordered_idx(self, mask_missing_values):
             missing_values_idx = np.flatnonzero(frac_of_missing_values)
         else:
             missing_values_idx = np.arange(np.shape(frac_of_missing_values)[0])
-        if self.imputation_order == 'roman':
+        if self.imputation_order == "roman":
             ordered_idx = missing_values_idx
-        elif self.imputation_order == 'arabic':
+        elif self.imputation_order == "arabic":
             ordered_idx = missing_values_idx[::-1]
-        elif self.imputation_order == 'ascending':
+        elif self.imputation_order == "ascending":
             n = len(frac_of_missing_values) - len(missing_values_idx)
-            ordered_idx = np.argsort(frac_of_missing_values,
-                                     kind='mergesort')[n:]
-        elif self.imputation_order == 'descending':
+            ordered_idx = np.argsort(frac_of_missing_values, kind="mergesort")[n:]
+        elif self.imputation_order == "descending":
             n = len(frac_of_missing_values) - len(missing_values_idx)
-            ordered_idx = np.argsort(frac_of_missing_values,
-                                     kind='mergesort')[n:][::-1]
-        elif self.imputation_order == 'random':
+            ordered_idx = np.argsort(frac_of_missing_values, kind="mergesort")[n:][::-1]
+        elif self.imputation_order == "random":
             ordered_idx = missing_values_idx
             self.random_state_.shuffle(ordered_idx)
-        else:
-            raise ValueError("Got an invalid imputation order: '{0}'. It must "
-                             "be one of the following: 'roman', 'arabic', "
-                             "'ascending', 'descending', or "
-                             "'random'.".format(self.imputation_order))
         return ordered_idx
 
     def _get_abs_corr_mat(self, X_filled, tolerance=1e-6):
@@ -444,23 +559,22 @@ def _get_abs_corr_mat(self, X_filled, tolerance=1e-6):
             Input data with the most recent imputations.
 
         tolerance : float, default=1e-6
-            ``abs_corr_mat`` can have nans, which will be replaced
-            with ``tolerance``.
+            `abs_corr_mat` can have nans, which will be replaced
+            with `tolerance`.
 
         Returns
         -------
         abs_corr_mat : ndarray, shape (n_features, n_features)
-            Absolute correlation matrix of ``X`` at the beginning of the
+            Absolute correlation matrix of `X` at the beginning of the
             current round. The diagonal has been zeroed out and each feature's
             absolute correlations with all others have been normalized to sum
             to 1.
         """
         n_features = X_filled.shape[1]
-        if (self.n_nearest_features is None or
-                self.n_nearest_features >= n_features):
+        if self.n_nearest_features is None or self.n_nearest_features >= n_features:
             return None
-        with np.errstate(invalid='ignore'):
-            # if a feature in the neighboorhood has only a single value
+        with np.errstate(invalid="ignore"):
+            # if a feature in the neighborhood has only a single value
             # (e.g., categorical feature), the std. dev. will be null and
             # np.corrcoef will raise a warning due to a division by zero
             abs_corr_mat = np.abs(np.corrcoef(X_filled.T))
@@ -471,130 +585,211 @@ def _get_abs_corr_mat(self, X_filled, tolerance=1e-6):
         # features are not their own neighbors
         np.fill_diagonal(abs_corr_mat, 0)
         # needs to sum to 1 for np.random.choice sampling
-        abs_corr_mat = normalize(abs_corr_mat, norm='l1', axis=0, copy=False)
+        abs_corr_mat = normalize(abs_corr_mat, norm="l1", axis=0, copy=False)
         return abs_corr_mat
 
     def _initial_imputation(self, X, in_fit=False):
-        """Perform initial imputation for input X.
+        """Perform initial imputation for input `X`.
 
         Parameters
         ----------
-        X : ndarray, shape (n_samples, n_features)
-            Input data, where "n_samples" is the number of samples and
-            "n_features" is the number of features.
+        X : ndarray of shape (n_samples, n_features)
+            Input data, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
 
         in_fit : bool, default=False
-            Whether function is called in fit.
+            Whether function is called in :meth:`fit`.
 
         Returns
         -------
-        Xt : ndarray, shape (n_samples, n_features)
-            Input data, where "n_samples" is the number of samples and
-            "n_features" is the number of features.
+        Xt : ndarray of shape (n_samples, n_features)
+            Input data, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
 
-        X_filled : ndarray, shape (n_samples, n_features)
+        X_filled : ndarray of shape (n_samples, n_features)
             Input data with the most recent imputations.
 
-        mask_missing_values : ndarray, shape (n_samples, n_features)
-            Input data's missing indicator matrix, where "n_samples" is the
-            number of samples and "n_features" is the number of features.
+        mask_missing_values : ndarray of shape (n_samples, n_features)
+            Input data's missing indicator matrix, where `n_samples` is the
+            number of samples and `n_features` is the number of features,
+            masked by non-missing features.
 
         X_missing_mask : ndarray, shape (n_samples, n_features)
             Input data's mask matrix indicating missing datapoints, where
-            "n_samples" is the number of samples and "n_features" is the
+            `n_samples` is the number of samples and `n_features` is the
             number of features.
         """
         if is_scalar_nan(self.missing_values):
-            force_all_finite = "allow-nan"
+            ensure_all_finite = "allow-nan"
         else:
-            force_all_finite = True
-
-        X = self._validate_data(X, dtype=FLOAT_DTYPES, order="F", reset=in_fit,
-                                force_all_finite=force_all_finite)
+            ensure_all_finite = True
+
+        X = validate_data(
+            self,
+            X,
+            dtype=FLOAT_DTYPES,
+            order="F",
+            reset=in_fit,
+            ensure_all_finite=ensure_all_finite,
+        )
         _check_inputs_dtype(X, self.missing_values)
 
         X_missing_mask = _get_mask(X, self.missing_values)
         mask_missing_values = X_missing_mask.copy()
+
+        # TODO (1.8): remove this once the deprecation is removed. In the meantime,
+        # we need to catch the warning to avoid false positives.
+        catch_warning = (
+            self.initial_strategy == "constant" and not self.keep_empty_features
+        )
+
         if self.initial_imputer_ is None:
             self.initial_imputer_ = SimpleImputer(
                 missing_values=self.missing_values,
-                strategy=self.initial_strategy
-            )
-            X_filled = self.initial_imputer_.fit_transform(X)
+                strategy=self.initial_strategy,
+                fill_value=self.fill_value,
+                keep_empty_features=self.keep_empty_features,
+            ).set_output(transform="default")
+
+            # TODO (1.8): remove this once the deprecation is removed to keep only
+            # the code in the else case.
+            if catch_warning:
+                with warnings.catch_warnings():
+                    warnings.simplefilter("ignore", FutureWarning)
+                    X_filled = self.initial_imputer_.fit_transform(X)
+            else:
+                X_filled = self.initial_imputer_.fit_transform(X)
         else:
-            X_filled = self.initial_imputer_.transform(X)
+            # TODO (1.8): remove this once the deprecation is removed to keep only
+            # the code in the else case.
+            if catch_warning:
+                with warnings.catch_warnings():
+                    warnings.simplefilter("ignore", FutureWarning)
+                    X_filled = self.initial_imputer_.transform(X)
+            else:
+                X_filled = self.initial_imputer_.transform(X)
+
+        if in_fit:
+            self._is_empty_feature = np.all(mask_missing_values, axis=0)
+
+        if not self.keep_empty_features:
+            # drop empty features
+            Xt = X[:, ~self._is_empty_feature]
+            mask_missing_values = mask_missing_values[:, ~self._is_empty_feature]
+
+            if self.initial_imputer_.get_params()["strategy"] == "constant":
+                # The constant strategy has a specific behavior and preserve empty
+                # features even with ``keep_empty_features=False``. We need to drop
+                # the column for consistency.
+                # TODO (1.8): remove this `if` branch once the following issue is
+                # addressed:
+                # https://github.com/scikit-learn/scikit-learn/issues/29827
+                X_filled = X_filled[:, ~self._is_empty_feature]
 
-        valid_mask = np.flatnonzero(np.logical_not(
-            np.isnan(self.initial_imputer_.statistics_)))
-        Xt = X[:, valid_mask]
-        mask_missing_values = mask_missing_values[:, valid_mask]
+        else:
+            # mark empty features as not missing and keep the original
+            # imputation
+            mask_missing_values[:, self._is_empty_feature] = False
+            Xt = X
+            Xt[:, self._is_empty_feature] = X_filled[:, self._is_empty_feature]
 
         return Xt, X_filled, mask_missing_values, X_missing_mask
 
     @staticmethod
-    def _validate_limit(limit, limit_type, n_features):
-        """Validate the limits (min/max) of the feature values
-        Converts scalar min/max limits to vectors of shape (n_features,)
+    def _validate_limit(
+        limit, limit_type, n_features, is_empty_feature, keep_empty_feature
+    ):
+        """Validate the limits (min/max) of the feature values.
+
+        Converts scalar min/max limits to vectors of shape `(n_features,)`.
 
         Parameters
         ----------
         limit: scalar or array-like
-            The user-specified limit (i.e, min_value or max_value)
-        limit_type: string, "max" or "min"
-            n_features: Number of features in the dataset
+            The user-specified limit (i.e, min_value or max_value).
+        limit_type: {'max', 'min'}
+            Type of limit to validate.
+        n_features: int
+            Number of features in the dataset.
+        is_empty_feature: ndarray, shape (n_features, )
+            Mask array indicating empty feature imputer has seen during fit.
+        keep_empty_feature: bool
+            If False, remove empty-feature indices from the limit.
 
         Returns
         -------
         limit: ndarray, shape(n_features,)
-            Array of limits, one for each feature
+            Array of limits, one for each feature.
         """
+        n_features_in = _num_samples(is_empty_feature)
+        if (
+            limit is not None
+            and not np.isscalar(limit)
+            and _num_samples(limit) != n_features_in
+        ):
+            raise ValueError(
+                f"'{limit_type}_value' should be of shape ({n_features_in},) when an"
+                f" array-like is provided. Got {len(limit)}, instead."
+            )
+
         limit_bound = np.inf if limit_type == "max" else -np.inf
         limit = limit_bound if limit is None else limit
         if np.isscalar(limit):
             limit = np.full(n_features, limit)
-        limit = check_array(
-            limit, force_all_finite=False, copy=False, ensure_2d=False
-        )
-        if not limit.shape[0] == n_features:
-            raise ValueError(
-                f"'{limit_type}_value' should be of "
-                f"shape ({n_features},) when an array-like "
-                f"is provided. Got {limit.shape}, instead."
-            )
+        limit = check_array(limit, ensure_all_finite=False, copy=False, ensure_2d=False)
+
+        # Make sure to remove the empty feature elements from the bounds
+        if not keep_empty_feature and len(limit) == len(is_empty_feature):
+            limit = limit[~is_empty_feature]
+
         return limit
 
-    def fit_transform(self, X, y=None):
-        """Fits the imputer on X and return the transformed X.
+    @_fit_context(
+        # IterativeImputer.estimator is not validated yet
+        prefer_skip_nested_validation=False
+    )
+    def fit_transform(self, X, y=None, **params):
+        """Fit the imputer on `X` and return the transformed `X`.
 
         Parameters
         ----------
         X : array-like, shape (n_samples, n_features)
-            Input data, where "n_samples" is the number of samples and
-            "n_features" is the number of features.
+            Input data, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
+
+        y : Ignored
+            Not used, present for API consistency by convention.
+
+        **params : dict
+            Parameters routed to the `fit` method of the sub-estimator via the
+            metadata routing API.
 
-        y : ignored.
+            .. versionadded:: 1.5
+              Only available if
+              `sklearn.set_config(enable_metadata_routing=True)` is set. See
+              :ref:`Metadata Routing User Guide ` for more
+              details.
 
         Returns
         -------
         Xt : array-like, shape (n_samples, n_features)
             The imputed input data.
         """
-        self.random_state_ = getattr(self, "random_state_",
-                                     check_random_state(self.random_state))
+        _raise_for_params(params, self, "fit")
 
-        if self.max_iter < 0:
-            raise ValueError(
-                "'max_iter' should be a positive integer. Got {} instead."
-                .format(self.max_iter))
+        routed_params = process_routing(
+            self,
+            "fit",
+            **params,
+        )
 
-        if self.tol < 0:
-            raise ValueError(
-                "'tol' should be a non-negative float. Got {} instead."
-                .format(self.tol)
-            )
+        self.random_state_ = getattr(
+            self, "random_state_", check_random_state(self.random_state)
+        )
 
         if self.estimator is None:
             from ..linear_model import BayesianRidge
+
             self._estimator = BayesianRidge()
         else:
             self._estimator = clone(self.estimator)
@@ -603,8 +798,9 @@ def fit_transform(self, X, y=None):
 
         self.initial_imputer_ = None
 
-        X, Xt, mask_missing_values, complete_mask = (
-            self._initial_imputation(X, in_fit=True))
+        X, Xt, mask_missing_values, complete_mask = self._initial_imputation(
+            X, in_fit=True
+        )
 
         super()._fit_indicator(complete_mask)
         X_indicator = super()._transform_indicator(complete_mask)
@@ -613,19 +809,28 @@ def fit_transform(self, X, y=None):
             self.n_iter_ = 0
             return super()._concatenate_indicator(Xt, X_indicator)
 
-        # Edge case: a single feature. We return the initial ...
+        # Edge case: a single feature, we return the initial imputation.
         if Xt.shape[1] == 1:
             self.n_iter_ = 0
             return super()._concatenate_indicator(Xt, X_indicator)
 
         self._min_value = self._validate_limit(
-            self.min_value, "min", X.shape[1])
+            self.min_value,
+            "min",
+            X.shape[1],
+            self._is_empty_feature,
+            self.keep_empty_features,
+        )
         self._max_value = self._validate_limit(
-            self.max_value, "max", X.shape[1])
+            self.max_value,
+            "max",
+            X.shape[1],
+            self._is_empty_feature,
+            self.keep_empty_features,
+        )
 
         if not np.all(np.greater(self._max_value, self._min_value)):
-            raise ValueError(
-                "One (or more) features have min_value >= max_value.")
+            raise ValueError("One (or more) features have min_value >= max_value.")
 
         # order in which to impute
         # note this is probably too slow for large feature data (d > 100000)
@@ -638,60 +843,68 @@ def fit_transform(self, X, y=None):
 
         n_samples, n_features = Xt.shape
         if self.verbose > 0:
-            print("[IterativeImputer] Completing matrix with shape %s"
-                  % (X.shape,))
+            print("[IterativeImputer] Completing matrix with shape %s" % (X.shape,))
         start_t = time()
         if not self.sample_posterior:
             Xt_previous = Xt.copy()
-            normalized_tol = self.tol * np.max(
-                np.abs(X[~mask_missing_values])
-            )
+            normalized_tol = self.tol * np.max(np.abs(X[~mask_missing_values]))
         for self.n_iter_ in range(1, self.max_iter + 1):
-            if self.imputation_order == 'random':
+            if self.imputation_order == "random":
                 ordered_idx = self._get_ordered_idx(mask_missing_values)
 
             for feat_idx in ordered_idx:
-                neighbor_feat_idx = self._get_neighbor_feat_idx(n_features,
-                                                                feat_idx,
-                                                                abs_corr_mat)
+                neighbor_feat_idx = self._get_neighbor_feat_idx(
+                    n_features, feat_idx, abs_corr_mat
+                )
                 Xt, estimator = self._impute_one_feature(
-                    Xt, mask_missing_values, feat_idx, neighbor_feat_idx,
-                    estimator=None, fit_mode=True)
-                estimator_triplet = _ImputerTriplet(feat_idx,
-                                                    neighbor_feat_idx,
-                                                    estimator)
+                    Xt,
+                    mask_missing_values,
+                    feat_idx,
+                    neighbor_feat_idx,
+                    estimator=None,
+                    fit_mode=True,
+                    params=routed_params.estimator.fit,
+                )
+                estimator_triplet = _ImputerTriplet(
+                    feat_idx, neighbor_feat_idx, estimator
+                )
                 self.imputation_sequence_.append(estimator_triplet)
 
             if self.verbose > 1:
-                print('[IterativeImputer] Ending imputation round '
-                      '%d/%d, elapsed time %0.2f'
-                      % (self.n_iter_, self.max_iter, time() - start_t))
+                print(
+                    "[IterativeImputer] Ending imputation round "
+                    "%d/%d, elapsed time %0.2f"
+                    % (self.n_iter_, self.max_iter, time() - start_t)
+                )
 
             if not self.sample_posterior:
-                inf_norm = np.linalg.norm(Xt - Xt_previous, ord=np.inf,
-                                          axis=None)
+                inf_norm = np.linalg.norm(Xt - Xt_previous, ord=np.inf, axis=None)
                 if self.verbose > 0:
-                    print('[IterativeImputer] '
-                          'Change: {}, scaled tolerance: {} '.format(
-                              inf_norm, normalized_tol))
+                    print(
+                        "[IterativeImputer] Change: {}, scaled tolerance: {} ".format(
+                            inf_norm, normalized_tol
+                        )
+                    )
                 if inf_norm < normalized_tol:
                     if self.verbose > 0:
-                        print('[IterativeImputer] Early stopping criterion '
-                              'reached.')
+                        print("[IterativeImputer] Early stopping criterion reached.")
                     break
                 Xt_previous = Xt.copy()
         else:
             if not self.sample_posterior:
-                warnings.warn("[IterativeImputer] Early stopping criterion not"
-                              " reached.", ConvergenceWarning)
-        Xt[~mask_missing_values] = X[~mask_missing_values]
+                warnings.warn(
+                    "[IterativeImputer] Early stopping criterion not reached.",
+                    ConvergenceWarning,
+                )
+        _assign_where(Xt, X, cond=~mask_missing_values)
+
         return super()._concatenate_indicator(Xt, X_indicator)
 
     def transform(self, X):
-        """Imputes all missing values in X.
+        """Impute all missing values in `X`.
 
-        Note that this is stochastic, and that if random_state is not fixed,
-        repeated calls, or permuted input, will yield different results.
+        Note that this is stochastic, and that if `random_state` is not fixed,
+        repeated calls, or permuted input, results will differ.
 
         Parameters
         ----------
@@ -705,7 +918,9 @@ def transform(self, X):
         """
         check_is_fitted(self)
 
-        X, Xt, mask_missing_values, complete_mask = self._initial_imputation(X)
+        X, Xt, mask_missing_values, complete_mask = self._initial_imputation(
+            X, in_fit=False
+        )
 
         X_indicator = super()._transform_indicator(complete_mask)
 
@@ -715,8 +930,7 @@ def transform(self, X):
         imputations_per_round = len(self.imputation_sequence_) // self.n_iter_
         i_rnd = 0
         if self.verbose > 0:
-            print("[IterativeImputer] Completing matrix with shape %s"
-                  % (X.shape,))
+            print("[IterativeImputer] Completing matrix with shape %s" % (X.shape,))
         start_t = time()
         for it, estimator_triplet in enumerate(self.imputation_sequence_):
             Xt, _ = self._impute_one_feature(
@@ -725,34 +939,92 @@ def transform(self, X):
                 estimator_triplet.feat_idx,
                 estimator_triplet.neighbor_feat_idx,
                 estimator=estimator_triplet.estimator,
-                fit_mode=False
+                fit_mode=False,
             )
             if not (it + 1) % imputations_per_round:
                 if self.verbose > 1:
-                    print('[IterativeImputer] Ending imputation round '
-                          '%d/%d, elapsed time %0.2f'
-                          % (i_rnd + 1, self.n_iter_, time() - start_t))
+                    print(
+                        "[IterativeImputer] Ending imputation round "
+                        "%d/%d, elapsed time %0.2f"
+                        % (i_rnd + 1, self.n_iter_, time() - start_t)
+                    )
                 i_rnd += 1
 
-        Xt[~mask_missing_values] = X[~mask_missing_values]
+        _assign_where(Xt, X, cond=~mask_missing_values)
 
         return super()._concatenate_indicator(Xt, X_indicator)
 
-    def fit(self, X, y=None):
-        """Fits the imputer on X and return self.
+    def fit(self, X, y=None, **fit_params):
+        """Fit the imputer on `X` and return self.
 
         Parameters
         ----------
         X : array-like, shape (n_samples, n_features)
-            Input data, where "n_samples" is the number of samples and
-            "n_features" is the number of features.
+            Input data, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
 
-        y : ignored
+        y : Ignored
+            Not used, present for API consistency by convention.
+
+        **fit_params : dict
+            Parameters routed to the `fit` method of the sub-estimator via the
+            metadata routing API.
+
+            .. versionadded:: 1.5
+              Only available if
+              `sklearn.set_config(enable_metadata_routing=True)` is set. See
+              :ref:`Metadata Routing User Guide ` for more
+              details.
 
         Returns
         -------
         self : object
-            Returns self.
+            Fitted estimator.
         """
-        self.fit_transform(X)
+        self.fit_transform(X, **fit_params)
         return self
+
+    def get_feature_names_out(self, input_features=None):
+        """Get output feature names for transformation.
+
+        Parameters
+        ----------
+        input_features : array-like of str or None, default=None
+            Input features.
+
+            - If `input_features` is `None`, then `feature_names_in_` is
+              used as feature names in. If `feature_names_in_` is not defined,
+              then the following input feature names are generated:
+              `["x0", "x1", ..., "x(n_features_in_ - 1)"]`.
+            - If `input_features` is an array-like, then `input_features` must
+              match `feature_names_in_` if `feature_names_in_` is defined.
+
+        Returns
+        -------
+        feature_names_out : ndarray of str objects
+            Transformed feature names.
+        """
+        check_is_fitted(self, "n_features_in_")
+        input_features = _check_feature_names_in(self, input_features)
+        names = self.initial_imputer_.get_feature_names_out(input_features)
+        return self._concatenate_indicator_feature_names_out(names, input_features)
+
+    def get_metadata_routing(self):
+        """Get metadata routing of this object.
+
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
+
+        .. versionadded:: 1.5
+
+        Returns
+        -------
+        routing : MetadataRouter
+            A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
+            routing information.
+        """
+        router = MetadataRouter(owner=self.__class__.__name__).add(
+            estimator=self.estimator,
+            method_mapping=MethodMapping().add(callee="fit", caller="fit"),
+        )
+        return router
diff --git a/sklearn/impute/_knn.py b/sklearn/impute/_knn.py
index c4b407fdd66e7..1b7ef06edc256 100644
--- a/sklearn/impute/_knn.py
+++ b/sklearn/impute/_knn.py
@@ -1,19 +1,24 @@
-# Authors: Ashim Bhattarai 
-#          Thomas J Fan 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+from numbers import Integral
 
 import numpy as np
 
-from ._base import _BaseImputer
-from ..utils.validation import FLOAT_DTYPES
+from ..base import _fit_context
 from ..metrics import pairwise_distances_chunked
 from ..metrics.pairwise import _NAN_METRICS
 from ..neighbors._base import _get_weights
-from ..neighbors._base import _check_weights
-from ..utils import is_scalar_nan
 from ..utils._mask import _get_mask
-from ..utils.validation import check_is_fitted
-from ..utils.validation import _deprecate_positional_args
+from ..utils._missing import is_scalar_nan
+from ..utils._param_validation import Hidden, Interval, StrOptions
+from ..utils.validation import (
+    FLOAT_DTYPES,
+    _check_feature_names_in,
+    check_is_fitted,
+    validate_data,
+)
+from ._base import _BaseImputer
 
 
 class KNNImputer(_BaseImputer):
@@ -55,9 +60,9 @@ class KNNImputer(_BaseImputer):
 
         - 'nan_euclidean'
         - callable : a user-defined function which conforms to the definition
-          of ``_pairwise_callable(X, Y, metric, **kwds)``. The function
-          accepts two arrays, X and Y, and a `missing_values` keyword in
-          `kwds` and returns a scalar distance value.
+          of ``func_metric(x, y, *, missing_values=np.nan)``. `x` and `y`
+          corresponds to a row (i.e. 1-D arrays) of `X` and `Y`, respectively.
+          The callable should returns a scalar distance value.
 
     copy : bool, default=True
         If True, a copy of X will be created. If False, imputation will
@@ -71,18 +76,44 @@ class KNNImputer(_BaseImputer):
         missing indicator even if there are missing values at transform/test
         time.
 
+    keep_empty_features : bool, default=False
+        If True, features that consist exclusively of missing values when
+        `fit` is called are returned in results when `transform` is called.
+        The imputed value is always `0`.
+
+        .. versionadded:: 1.2
+
     Attributes
     ----------
     indicator_ : :class:`~sklearn.impute.MissingIndicator`
         Indicator used to add binary indicators for missing values.
         ``None`` if add_indicator is False.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    SimpleImputer : Univariate imputer for completing missing values
+        with simple strategies.
+    IterativeImputer : Multivariate imputer that estimates values to impute for
+        each feature with missing values from all the others.
+
     References
     ----------
-    * Olga Troyanskaya, Michael Cantor, Gavin Sherlock, Pat Brown, Trevor
+    * `Olga Troyanskaya, Michael Cantor, Gavin Sherlock, Pat Brown, Trevor
       Hastie, Robert Tibshirani, David Botstein and Russ B. Altman, Missing
       value estimation methods for DNA microarrays, BIOINFORMATICS Vol. 17
       no. 6, 2001 Pages 520-525.
+      `_
 
     Examples
     --------
@@ -95,22 +126,41 @@ class KNNImputer(_BaseImputer):
            [3. , 4. , 3. ],
            [5.5, 6. , 5. ],
            [8. , 8. , 7. ]])
+
+    For a more detailed example see
+    :ref:`sphx_glr_auto_examples_impute_plot_missing_values.py`.
     """
-    @_deprecate_positional_args
-    def __init__(self, *, missing_values=np.nan, n_neighbors=5,
-                 weights="uniform", metric="nan_euclidean", copy=True,
-                 add_indicator=False):
+
+    _parameter_constraints: dict = {
+        **_BaseImputer._parameter_constraints,
+        "n_neighbors": [Interval(Integral, 1, None, closed="left")],
+        "weights": [StrOptions({"uniform", "distance"}), callable, Hidden(None)],
+        "metric": [StrOptions(set(_NAN_METRICS)), callable],
+        "copy": ["boolean"],
+    }
+
+    def __init__(
+        self,
+        *,
+        missing_values=np.nan,
+        n_neighbors=5,
+        weights="uniform",
+        metric="nan_euclidean",
+        copy=True,
+        add_indicator=False,
+        keep_empty_features=False,
+    ):
         super().__init__(
             missing_values=missing_values,
-            add_indicator=add_indicator
+            add_indicator=add_indicator,
+            keep_empty_features=keep_empty_features,
         )
         self.n_neighbors = n_neighbors
         self.weights = weights
         self.metric = metric
         self.copy = copy
 
-    def _calc_impute(self, dist_pot_donors, n_neighbors,
-                     fit_X_col, mask_fit_X_col):
+    def _calc_impute(self, dist_pot_donors, n_neighbors, fit_X_col, mask_fit_X_col):
         """Helper function to impute a single column.
 
         Parameters
@@ -135,18 +185,23 @@ def _calc_impute(self, dist_pot_donors, n_neighbors,
             Imputed values for receiver.
         """
         # Get donors
-        donors_idx = np.argpartition(dist_pot_donors, n_neighbors - 1,
-                                     axis=1)[:, :n_neighbors]
+        donors_idx = np.argpartition(dist_pot_donors, n_neighbors - 1, axis=1)[
+            :, :n_neighbors
+        ]
 
-        # Get weight matrix from from distance matrix
+        # Get weight matrix from distance matrix
         donors_dist = dist_pot_donors[
-            np.arange(donors_idx.shape[0])[:, None], donors_idx]
+            np.arange(donors_idx.shape[0])[:, None], donors_idx
+        ]
 
         weight_matrix = _get_weights(donors_dist, self.weights)
 
         # fill nans with zeros
         if weight_matrix is not None:
             weight_matrix[np.isnan(weight_matrix)] = 0.0
+        else:
+            weight_matrix = np.ones_like(donors_dist)
+            weight_matrix[np.isnan(donors_dist)] = 0.0
 
         # Retrieve donor values and calculate kNN average
         donors = fit_X_col.take(donors_idx)
@@ -155,6 +210,7 @@ def _calc_impute(self, dist_pot_donors, n_neighbors,
 
         return np.ma.average(donors, axis=1, weights=weight_matrix).data
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y=None):
         """Fit the imputer on X.
 
@@ -164,29 +220,32 @@ def fit(self, X, y=None):
             Input data, where `n_samples` is the number of samples and
             `n_features` is the number of features.
 
+        y : Ignored
+            Not used, present here for API consistency by convention.
+
         Returns
         -------
         self : object
+            The fitted `KNNImputer` class instance.
         """
         # Check data integrity and calling arguments
         if not is_scalar_nan(self.missing_values):
-            force_all_finite = True
+            ensure_all_finite = True
         else:
-            force_all_finite = "allow-nan"
-            if self.metric not in _NAN_METRICS and not callable(self.metric):
-                raise ValueError(
-                    "The selected metric does not support NaN values")
-        if self.n_neighbors <= 0:
-            raise ValueError(
-                "Expected n_neighbors > 0. Got {}".format(self.n_neighbors))
-
-        X = self._validate_data(X, accept_sparse=False, dtype=FLOAT_DTYPES,
-                                force_all_finite=force_all_finite,
-                                copy=self.copy)
-
-        _check_weights(self.weights)
+            ensure_all_finite = "allow-nan"
+
+        X = validate_data(
+            self,
+            X,
+            accept_sparse=False,
+            dtype=FLOAT_DTYPES,
+            ensure_all_finite=ensure_all_finite,
+            copy=self.copy,
+        )
+
         self._fit_X = X
         self._mask_fit_X = _get_mask(self._fit_X, self.missing_values)
+        self._valid_mask = ~np.all(self._mask_fit_X, axis=0)
 
         super()._fit_indicator(self._mask_fit_X)
 
@@ -209,26 +268,42 @@ def transform(self, X):
 
         check_is_fitted(self)
         if not is_scalar_nan(self.missing_values):
-            force_all_finite = True
+            ensure_all_finite = True
         else:
-            force_all_finite = "allow-nan"
-        X = self._validate_data(X, accept_sparse=False, dtype=FLOAT_DTYPES,
-                                force_all_finite=force_all_finite,
-                                copy=self.copy, reset=False)
+            ensure_all_finite = "allow-nan"
+        X = validate_data(
+            self,
+            X,
+            accept_sparse=False,
+            dtype=FLOAT_DTYPES,
+            force_writeable=True,
+            ensure_all_finite=ensure_all_finite,
+            copy=self.copy,
+            reset=False,
+        )
 
         mask = _get_mask(X, self.missing_values)
         mask_fit_X = self._mask_fit_X
-        valid_mask = ~np.all(mask_fit_X, axis=0)
+        valid_mask = self._valid_mask
 
         X_indicator = super()._transform_indicator(mask)
 
         # Removes columns where the training data is all nan
-        if not np.any(mask):
+        if not np.any(mask[:, valid_mask]):
             # No missing values in X
-            # Remove columns where the training data is all nan
-            return X[:, valid_mask]
+            if self.keep_empty_features:
+                Xc = X
+                Xc[:, ~valid_mask] = 0
+            else:
+                Xc = X[:, valid_mask]
 
-        row_missing_idx = np.flatnonzero(mask.any(axis=1))
+            # Even if there are no missing values in X, we still concatenate Xc
+            # with the missing value indicator matrix, X_indicator.
+            # This is to ensure that the output maintains consistency in terms
+            # of columns, regardless of whether missing values exist in X or not.
+            return super()._concatenate_indicator(Xc, X_indicator)
+
+        row_missing_idx = np.flatnonzero(mask[:, valid_mask].any(axis=1))
 
         non_missing_fix_X = np.logical_not(mask_fit_X)
 
@@ -237,7 +312,7 @@ def transform(self, X):
         dist_idx_map[row_missing_idx] = np.arange(row_missing_idx.shape[0])
 
         def process_chunk(dist_chunk, start):
-            row_missing_chunk = row_missing_idx[start:start + len(dist_chunk)]
+            row_missing_chunk = row_missing_idx[start : start + len(dist_chunk)]
 
             # Find and impute missing by column
             for col in range(X.shape[1]):
@@ -250,22 +325,24 @@ def process_chunk(dist_chunk, start):
                     # column has no missing values
                     continue
 
-                potential_donors_idx, = np.nonzero(non_missing_fix_X[:, col])
+                (potential_donors_idx,) = np.nonzero(non_missing_fix_X[:, col])
 
                 # receivers_idx are indices in X
                 receivers_idx = row_missing_chunk[np.flatnonzero(col_mask)]
 
                 # distances for samples that needed imputation for column
-                dist_subset = (dist_chunk[dist_idx_map[receivers_idx] - start]
-                               [:, potential_donors_idx])
+                dist_subset = dist_chunk[dist_idx_map[receivers_idx] - start][
+                    :, potential_donors_idx
+                ]
 
                 # receivers with all nan distances impute with mean
                 all_nan_dist_mask = np.isnan(dist_subset).all(axis=1)
                 all_nan_receivers_idx = receivers_idx[all_nan_dist_mask]
 
                 if all_nan_receivers_idx.size:
-                    col_mean = np.ma.array(self._fit_X[:, col],
-                                           mask=mask_fit_X[:, col]).mean()
+                    col_mean = np.ma.array(
+                        self._fit_X[:, col], mask=mask_fit_X[:, col]
+                    ).mean()
                     X[all_nan_receivers_idx, col] = col_mean
 
                     if len(all_nan_receivers_idx) == len(receivers_idx):
@@ -274,16 +351,17 @@ def process_chunk(dist_chunk, start):
 
                     # receivers with at least one defined distance
                     receivers_idx = receivers_idx[~all_nan_dist_mask]
-                    dist_subset = (dist_chunk[dist_idx_map[receivers_idx]
-                                              - start]
-                                   [:, potential_donors_idx])
+                    dist_subset = dist_chunk[dist_idx_map[receivers_idx] - start][
+                        :, potential_donors_idx
+                    ]
 
                 n_neighbors = min(self.n_neighbors, len(potential_donors_idx))
                 value = self._calc_impute(
                     dist_subset,
                     n_neighbors,
                     self._fit_X[potential_donors_idx, col],
-                    mask_fit_X[potential_donors_idx, col])
+                    mask_fit_X[potential_donors_idx, col],
+                )
                 X[receivers_idx, col] = value
 
         # process in fixed-memory chunks
@@ -292,10 +370,42 @@ def process_chunk(dist_chunk, start):
             self._fit_X,
             metric=self.metric,
             missing_values=self.missing_values,
-            force_all_finite=force_all_finite,
-            reduce_func=process_chunk)
+            ensure_all_finite=ensure_all_finite,
+            reduce_func=process_chunk,
+        )
         for chunk in gen:
             # process_chunk modifies X in place. No return value.
             pass
 
-        return super()._concatenate_indicator(X[:, valid_mask], X_indicator)
+        if self.keep_empty_features:
+            Xc = X
+            Xc[:, ~valid_mask] = 0
+        else:
+            Xc = X[:, valid_mask]
+
+        return super()._concatenate_indicator(Xc, X_indicator)
+
+    def get_feature_names_out(self, input_features=None):
+        """Get output feature names for transformation.
+
+        Parameters
+        ----------
+        input_features : array-like of str or None, default=None
+            Input features.
+
+            - If `input_features` is `None`, then `feature_names_in_` is
+              used as feature names in. If `feature_names_in_` is not defined,
+              then the following input feature names are generated:
+              `["x0", "x1", ..., "x(n_features_in_ - 1)"]`.
+            - If `input_features` is an array-like, then `input_features` must
+              match `feature_names_in_` if `feature_names_in_` is defined.
+
+        Returns
+        -------
+        feature_names_out : ndarray of str objects
+            Transformed feature names.
+        """
+        check_is_fitted(self, "n_features_in_")
+        input_features = _check_feature_names_in(self, input_features)
+        names = input_features[self._valid_mask]
+        return self._concatenate_indicator_feature_names_out(names, input_features)
diff --git a/sklearn/impute/tests/test_base.py b/sklearn/impute/tests/test_base.py
index 8786e77523ab4..0c1bd83f7ca9e 100644
--- a/sklearn/impute/tests/test_base.py
+++ b/sklearn/impute/tests/test_base.py
@@ -1,9 +1,10 @@
-import pytest
-
 import numpy as np
+import pytest
 
 from sklearn.impute._base import _BaseImputer
+from sklearn.impute._iterative import _assign_where
 from sklearn.utils._mask import _get_mask
+from sklearn.utils._testing import _convert_container, assert_allclose
 
 
 @pytest.fixture
@@ -61,8 +62,9 @@ def test_base_imputer_not_fit(data):
 
 def test_base_imputer_not_transform(data):
     imputer = NoTransformIndicatorImputer(add_indicator=True)
-    err_msg = ("Call _fit_indicator and _transform_indicator in the "
-               "imputer implementation")
+    err_msg = (
+        "Call _fit_indicator and _transform_indicator in the imputer implementation"
+    )
     with pytest.raises(ValueError, match=err_msg):
         imputer.fit(data).transform(data)
     with pytest.raises(ValueError, match=err_msg):
@@ -86,3 +88,20 @@ def test_base_no_precomputed_mask_transform(data):
         imputer.transform(data)
     with pytest.raises(ValueError, match=err_msg):
         imputer.fit_transform(data)
+
+
+@pytest.mark.parametrize("X1_type", ["array", "dataframe"])
+def test_assign_where(X1_type):
+    """Check the behaviour of the private helpers `_assign_where`."""
+    rng = np.random.RandomState(0)
+
+    n_samples, n_features = 10, 5
+    X1 = _convert_container(rng.randn(n_samples, n_features), constructor_name=X1_type)
+    X2 = rng.randn(n_samples, n_features)
+    mask = rng.randint(0, 2, size=(n_samples, n_features)).astype(bool)
+
+    _assign_where(X1, X2, mask)
+
+    if X1_type == "dataframe":
+        X1 = X1.to_numpy()
+    assert_allclose(X1[mask], X2[mask])
diff --git a/sklearn/impute/tests/test_common.py b/sklearn/impute/tests/test_common.py
index 220a335c15285..afebc96ac035c 100644
--- a/sklearn/impute/tests/test_common.py
+++ b/sklearn/impute/tests/test_common.py
@@ -1,26 +1,27 @@
-import pytest
-
 import numpy as np
-from scipy import sparse
+import pytest
 
-from sklearn.utils._testing import assert_allclose
-from sklearn.utils._testing import assert_allclose_dense_sparse
-from sklearn.utils._testing import assert_array_equal
+from sklearn.experimental import enable_iterative_imputer  # noqa: F401
+from sklearn.impute import IterativeImputer, KNNImputer, SimpleImputer
+from sklearn.utils._testing import (
+    assert_allclose,
+    assert_allclose_dense_sparse,
+    assert_array_equal,
+)
+from sklearn.utils.fixes import CSR_CONTAINERS
 
-from sklearn.experimental import enable_iterative_imputer  # noqa
 
-from sklearn.impute import IterativeImputer
-from sklearn.impute import KNNImputer
-from sklearn.impute import SimpleImputer
+def imputers():
+    return [IterativeImputer(tol=0.1), KNNImputer(), SimpleImputer()]
 
 
-IMPUTERS = [IterativeImputer(), KNNImputer(), SimpleImputer()]
-SPARSE_IMPUTERS = [SimpleImputer()]
+def sparse_imputers():
+    return [SimpleImputer()]
 
 
 # ConvergenceWarning will be raised by the IterativeImputer
 @pytest.mark.filterwarnings("ignore::sklearn.exceptions.ConvergenceWarning")
-@pytest.mark.parametrize("imputer", IMPUTERS)
+@pytest.mark.parametrize("imputer", imputers(), ids=lambda x: x.__class__.__name__)
 def test_imputation_missing_value_in_test_array(imputer):
     # [Non Regression Test for issue #13968] Missing value in test set should
     # not throw an error and return a finite dataset
@@ -33,20 +34,24 @@ def test_imputation_missing_value_in_test_array(imputer):
 # ConvergenceWarning will be raised by the IterativeImputer
 @pytest.mark.filterwarnings("ignore::sklearn.exceptions.ConvergenceWarning")
 @pytest.mark.parametrize("marker", [np.nan, -1, 0])
-@pytest.mark.parametrize("imputer", IMPUTERS)
+@pytest.mark.parametrize("imputer", imputers(), ids=lambda x: x.__class__.__name__)
 def test_imputers_add_indicator(marker, imputer):
-    X = np.array([
-        [marker, 1,      5,      marker, 1],
-        [2,      marker, 1,      marker, 2],
-        [6,      3,      marker, marker, 3],
-        [1,      2,      9,      marker, 4]
-    ])
-    X_true_indicator = np.array([
-        [1., 0., 0., 1.],
-        [0., 1., 0., 1.],
-        [0., 0., 1., 1.],
-        [0., 0., 0., 1.]
-    ])
+    X = np.array(
+        [
+            [marker, 1, 5, marker, 1],
+            [2, marker, 1, marker, 2],
+            [6, 3, marker, marker, 3],
+            [1, 2, 9, marker, 4],
+        ]
+    )
+    X_true_indicator = np.array(
+        [
+            [1.0, 0.0, 0.0, 1.0],
+            [0.0, 1.0, 0.0, 1.0],
+            [0.0, 0.0, 1.0, 1.0],
+            [0.0, 0.0, 0.0, 1.0],
+        ]
+    )
     imputer.set_params(missing_values=marker, add_indicator=True)
 
     X_trans = imputer.fit_transform(X)
@@ -61,20 +66,27 @@ def test_imputers_add_indicator(marker, imputer):
 # ConvergenceWarning will be raised by the IterativeImputer
 @pytest.mark.filterwarnings("ignore::sklearn.exceptions.ConvergenceWarning")
 @pytest.mark.parametrize("marker", [np.nan, -1])
-@pytest.mark.parametrize("imputer", SPARSE_IMPUTERS)
-def test_imputers_add_indicator_sparse(imputer, marker):
-    X = sparse.csr_matrix([
-        [marker, 1,      5,      marker, 1],
-        [2,      marker, 1,      marker, 2],
-        [6,      3,      marker, marker, 3],
-        [1,      2,      9,      marker, 4]
-    ])
-    X_true_indicator = sparse.csr_matrix([
-        [1., 0., 0., 1.],
-        [0., 1., 0., 1.],
-        [0., 0., 1., 1.],
-        [0., 0., 0., 1.]
-    ])
+@pytest.mark.parametrize(
+    "imputer", sparse_imputers(), ids=lambda x: x.__class__.__name__
+)
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_imputers_add_indicator_sparse(imputer, marker, csr_container):
+    X = csr_container(
+        [
+            [marker, 1, 5, marker, 1],
+            [2, marker, 1, marker, 2],
+            [6, 3, marker, marker, 3],
+            [1, 2, 9, marker, 4],
+        ]
+    )
+    X_true_indicator = csr_container(
+        [
+            [1.0, 0.0, 0.0, 1.0],
+            [0.0, 1.0, 0.0, 1.0],
+            [0.0, 0.0, 1.0, 1.0],
+            [0.0, 0.0, 0.0, 1.0],
+        ]
+    )
     imputer.set_params(missing_values=marker, add_indicator=True)
 
     X_trans = imputer.fit_transform(X)
@@ -88,21 +100,22 @@ def test_imputers_add_indicator_sparse(imputer, marker):
 
 # ConvergenceWarning will be raised by the IterativeImputer
 @pytest.mark.filterwarnings("ignore::sklearn.exceptions.ConvergenceWarning")
-@pytest.mark.parametrize("imputer", IMPUTERS)
+@pytest.mark.parametrize("imputer", imputers(), ids=lambda x: x.__class__.__name__)
 @pytest.mark.parametrize("add_indicator", [True, False])
 def test_imputers_pandas_na_integer_array_support(imputer, add_indicator):
     # Test pandas IntegerArray with pd.NA
-    pd = pytest.importorskip('pandas', minversion="1.0")
+    pd = pytest.importorskip("pandas")
     marker = np.nan
-    imputer = imputer.set_params(add_indicator=add_indicator,
-                                 missing_values=marker)
-
-    X = np.array([
-        [marker, 1,      5,      marker, 1],
-        [2,      marker, 1,      marker, 2],
-        [6,      3,      marker, marker, 3],
-        [1,      2,      9,      marker, 4]
-    ])
+    imputer = imputer.set_params(add_indicator=add_indicator, missing_values=marker)
+
+    X = np.array(
+        [
+            [marker, 1, 5, marker, 1],
+            [2, marker, 1, marker, 2],
+            [6, 3, marker, marker, 3],
+            [1, 2, 9, marker, 4],
+        ]
+    )
     # fit on numpy array
     X_trans_expected = imputer.fit_transform(X)
 
@@ -113,3 +126,95 @@ def test_imputers_pandas_na_integer_array_support(imputer, add_indicator):
     X_trans = imputer.fit_transform(X_df)
 
     assert_allclose(X_trans_expected, X_trans)
+
+
+@pytest.mark.parametrize("imputer", imputers(), ids=lambda x: x.__class__.__name__)
+@pytest.mark.parametrize("add_indicator", [True, False])
+def test_imputers_feature_names_out_pandas(imputer, add_indicator):
+    """Check feature names out for imputers."""
+    pd = pytest.importorskip("pandas")
+    marker = np.nan
+    imputer = imputer.set_params(add_indicator=add_indicator, missing_values=marker)
+
+    X = np.array(
+        [
+            [marker, 1, 5, 3, marker, 1],
+            [2, marker, 1, 4, marker, 2],
+            [6, 3, 7, marker, marker, 3],
+            [1, 2, 9, 8, marker, 4],
+        ]
+    )
+    X_df = pd.DataFrame(X, columns=["a", "b", "c", "d", "e", "f"])
+    imputer.fit(X_df)
+
+    names = imputer.get_feature_names_out()
+
+    if add_indicator:
+        expected_names = [
+            "a",
+            "b",
+            "c",
+            "d",
+            "f",
+            "missingindicator_a",
+            "missingindicator_b",
+            "missingindicator_d",
+            "missingindicator_e",
+        ]
+        assert_array_equal(expected_names, names)
+    else:
+        expected_names = ["a", "b", "c", "d", "f"]
+        assert_array_equal(expected_names, names)
+
+
+@pytest.mark.parametrize("keep_empty_features", [True, False])
+@pytest.mark.parametrize("imputer", imputers(), ids=lambda x: x.__class__.__name__)
+def test_keep_empty_features(imputer, keep_empty_features):
+    """Check that the imputer keeps features with only missing values."""
+    X = np.array([[np.nan, 1], [np.nan, 2], [np.nan, 3]])
+    imputer = imputer.set_params(
+        add_indicator=False, keep_empty_features=keep_empty_features
+    )
+
+    for method in ["fit_transform", "transform"]:
+        X_imputed = getattr(imputer, method)(X)
+        if keep_empty_features:
+            assert X_imputed.shape == X.shape
+        else:
+            assert X_imputed.shape == (X.shape[0], X.shape[1] - 1)
+
+
+@pytest.mark.parametrize("imputer", imputers(), ids=lambda x: x.__class__.__name__)
+@pytest.mark.parametrize("missing_value_test", [np.nan, 1])
+def test_imputation_adds_missing_indicator_if_add_indicator_is_true(
+    imputer, missing_value_test
+):
+    """Check that missing indicator always exists when add_indicator=True.
+
+    Non-regression test for gh-26590.
+    """
+    X_train = np.array([[0, np.nan], [1, 2]])
+
+    # Test data where missing_value_test variable can be set to np.nan or 1.
+    X_test = np.array([[0, missing_value_test], [1, 2]])
+
+    imputer.set_params(add_indicator=True)
+    imputer.fit(X_train)
+
+    X_test_imputed_with_indicator = imputer.transform(X_test)
+    assert X_test_imputed_with_indicator.shape == (2, 3)
+
+    imputer.set_params(add_indicator=False)
+    imputer.fit(X_train)
+    X_test_imputed_without_indicator = imputer.transform(X_test)
+    assert X_test_imputed_without_indicator.shape == (2, 2)
+
+    assert_allclose(
+        X_test_imputed_with_indicator[:, :-1], X_test_imputed_without_indicator
+    )
+    if np.isnan(missing_value_test):
+        expected_missing_indicator = [1, 0]
+    else:
+        expected_missing_indicator = [0, 0]
+
+    assert_allclose(X_test_imputed_with_indicator[:, -1], expected_missing_indicator)
diff --git a/sklearn/impute/tests/test_impute.py b/sklearn/impute/tests/test_impute.py
index d7e9ef30cbf72..16501b0550364 100644
--- a/sklearn/impute/tests/test_impute.py
+++ b/sklearn/impute/tests/test_impute.py
@@ -1,37 +1,55 @@
-from __future__ import division
-
-import pytest
+import io
+import re
+import warnings
+from itertools import product
 
 import numpy as np
+import pytest
 from scipy import sparse
 from scipy.stats import kstest
 
-import io
-
-from sklearn.utils._testing import assert_allclose
-from sklearn.utils._testing import assert_allclose_dense_sparse
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils._testing import assert_array_almost_equal
-
-# make IterativeImputer available
-from sklearn.experimental import enable_iterative_imputer  # noqa
-
+from sklearn import tree
 from sklearn.datasets import load_diabetes
-from sklearn.impute import MissingIndicator
-from sklearn.impute import SimpleImputer, IterativeImputer
 from sklearn.dummy import DummyRegressor
-from sklearn.linear_model import BayesianRidge, ARDRegression, RidgeCV
-from sklearn.pipeline import Pipeline
-from sklearn.pipeline import make_union
-from sklearn.model_selection import GridSearchCV
-from sklearn import tree
-from sklearn.random_projection import _sparse_random_matrix
 from sklearn.exceptions import ConvergenceWarning
+
+# make IterativeImputer available
+from sklearn.experimental import enable_iterative_imputer  # noqa: F401
+from sklearn.impute import IterativeImputer, KNNImputer, MissingIndicator, SimpleImputer
 from sklearn.impute._base import _most_frequent
+from sklearn.linear_model import ARDRegression, BayesianRidge, RidgeCV
+from sklearn.model_selection import GridSearchCV
+from sklearn.pipeline import Pipeline, make_union
+from sklearn.random_projection import _sparse_random_matrix
+from sklearn.utils._testing import (
+    _convert_container,
+    assert_allclose,
+    assert_allclose_dense_sparse,
+    assert_array_almost_equal,
+    assert_array_equal,
+)
+from sklearn.utils.fixes import (
+    BSR_CONTAINERS,
+    COO_CONTAINERS,
+    CSC_CONTAINERS,
+    CSR_CONTAINERS,
+    LIL_CONTAINERS,
+)
+
+
+def _assert_array_equal_and_same_dtype(x, y):
+    assert_array_equal(x, y)
+    assert x.dtype == y.dtype
+
 
+def _assert_allclose_and_same_dtype(x, y):
+    assert_allclose(x, y)
+    assert x.dtype == y.dtype
 
-def _check_statistics(X, X_true,
-                      strategy, statistics, missing_values):
+
+def _check_statistics(
+    X, X_true, strategy, statistics, missing_values, sparse_container
+):
     """Utility function for testing imputation for a given strategy.
 
     Test with dense and sparse arrays
@@ -40,43 +58,43 @@ def _check_statistics(X, X_true,
         - the statistics (mean, median, mode) are correct
         - the missing values are imputed correctly"""
 
-    err_msg = "Parameters: strategy = %s, missing_values = %s, " \
-              "sparse = {0}" % (strategy, missing_values)
+    err_msg = "Parameters: strategy = %s, missing_values = %s, sparse = {0}" % (
+        strategy,
+        missing_values,
+    )
 
     assert_ae = assert_array_equal
 
-    if X.dtype.kind == 'f' or X_true.dtype.kind == 'f':
+    if X.dtype.kind == "f" or X_true.dtype.kind == "f":
         assert_ae = assert_array_almost_equal
 
     # Normal matrix
     imputer = SimpleImputer(missing_values=missing_values, strategy=strategy)
     X_trans = imputer.fit(X).transform(X.copy())
-    assert_ae(imputer.statistics_, statistics,
-              err_msg=err_msg.format(False))
+    assert_ae(imputer.statistics_, statistics, err_msg=err_msg.format(False))
     assert_ae(X_trans, X_true, err_msg=err_msg.format(False))
 
     # Sparse matrix
     imputer = SimpleImputer(missing_values=missing_values, strategy=strategy)
-    imputer.fit(sparse.csc_matrix(X))
-    X_trans = imputer.transform(sparse.csc_matrix(X.copy()))
+    imputer.fit(sparse_container(X))
+    X_trans = imputer.transform(sparse_container(X.copy()))
 
     if sparse.issparse(X_trans):
         X_trans = X_trans.toarray()
 
-    assert_ae(imputer.statistics_, statistics,
-              err_msg=err_msg.format(True))
+    assert_ae(imputer.statistics_, statistics, err_msg=err_msg.format(True))
     assert_ae(X_trans, X_true, err_msg=err_msg.format(True))
 
 
-@pytest.mark.parametrize("strategy",
-                         ['mean', 'median', 'most_frequent', "constant"])
-def test_imputation_shape(strategy):
+@pytest.mark.parametrize("strategy", ["mean", "median", "most_frequent", "constant"])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_imputation_shape(strategy, csr_container):
     # Verify the shapes of the imputed matrix for different strategies.
     X = np.random.randn(10, 2)
     X[::2] = np.nan
 
     imputer = SimpleImputer(strategy=strategy)
-    X_imputed = imputer.fit_transform(sparse.csr_matrix(X))
+    X_imputed = imputer.fit_transform(csr_container(X))
     assert X_imputed.shape == (10, 2)
     X_imputed = imputer.fit_transform(X)
     assert X_imputed.shape == (10, 2)
@@ -86,33 +104,49 @@ def test_imputation_shape(strategy):
     assert X_imputed.shape == (10, 2)
 
 
-@pytest.mark.parametrize("strategy", ["const", 101, None])
-def test_imputation_error_invalid_strategy(strategy):
+@pytest.mark.parametrize("strategy", ["mean", "median", "most_frequent"])
+def test_imputation_deletion_warning(strategy):
     X = np.ones((3, 5))
-    X[0, 0] = np.nan
+    X[:, 0] = np.nan
+    imputer = SimpleImputer(strategy=strategy).fit(X)
 
-    with pytest.raises(ValueError, match=str(strategy)):
-        imputer = SimpleImputer(strategy=strategy)
-        imputer.fit_transform(X)
+    with pytest.warns(UserWarning, match="Skipping"):
+        imputer.transform(X)
 
 
 @pytest.mark.parametrize("strategy", ["mean", "median", "most_frequent"])
-def test_imputation_deletion_warning(strategy):
-    X = np.ones((3, 5))
-    X[:, 0] = np.nan
+def test_imputation_deletion_warning_feature_names(strategy):
+    pd = pytest.importorskip("pandas")
 
-    with pytest.warns(UserWarning, match="Deleting"):
-        imputer = SimpleImputer(strategy=strategy, verbose=True)
-        imputer.fit_transform(X)
+    missing_values = np.nan
+    feature_names = np.array(["a", "b", "c", "d"], dtype=object)
+    X = pd.DataFrame(
+        [
+            [missing_values, missing_values, 1, missing_values],
+            [4, missing_values, 2, 10],
+        ],
+        columns=feature_names,
+    )
+
+    imputer = SimpleImputer(strategy=strategy).fit(X)
 
+    # check SimpleImputer returning feature name attribute correctly
+    assert_array_equal(imputer.feature_names_in_, feature_names)
+
+    # ensure that skipped feature warning includes feature name
+    with pytest.warns(
+        UserWarning, match=r"Skipping features without any observed values: \['b'\]"
+    ):
+        imputer.transform(X)
 
-@pytest.mark.parametrize("strategy", ["mean", "median",
-                                      "most_frequent", "constant"])
-def test_imputation_error_sparse_0(strategy):
+
+@pytest.mark.parametrize("strategy", ["mean", "median", "most_frequent", "constant"])
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+def test_imputation_error_sparse_0(strategy, csc_container):
     # check that error are raised when missing_values = 0 and input is sparse
     X = np.ones((3, 5))
     X[0] = 0
-    X = sparse.csc_matrix(X)
+    X = csc_container(X)
 
     imputer = SimpleImputer(strategy=strategy, missing_values=0)
     with pytest.raises(ValueError, match="Provide a dense array"):
@@ -125,17 +159,18 @@ def test_imputation_error_sparse_0(strategy):
 
 def safe_median(arr, *args, **kwargs):
     # np.median([]) raises a TypeError for numpy >= 1.10.1
-    length = arr.size if hasattr(arr, 'size') else len(arr)
+    length = arr.size if hasattr(arr, "size") else len(arr)
     return np.nan if length == 0 else np.median(arr, *args, **kwargs)
 
 
 def safe_mean(arr, *args, **kwargs):
     # np.mean([]) raises a RuntimeWarning for numpy >= 1.10.1
-    length = arr.size if hasattr(arr, 'size') else len(arr)
+    length = arr.size if hasattr(arr, "size") else len(arr)
     return np.nan if length == 0 else np.mean(arr, *args, **kwargs)
 
 
-def test_imputation_mean_median():
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+def test_imputation_mean_median(csc_container):
     # Test imputation using the mean and median strategies, when
     # missing_values != 0.
     rng = np.random.RandomState(0)
@@ -146,11 +181,12 @@ def test_imputation_mean_median():
 
     zeros = np.zeros(shape[0])
     values = np.arange(1, shape[0] + 1)
-    values[4::2] = - values[4::2]
+    values[4::2] = -values[4::2]
 
-    tests = [("mean", np.nan, lambda z, v, p: safe_mean(np.hstack((z, v)))),
-             ("median", np.nan,
-              lambda z, v, p: safe_median(np.hstack((z, v))))]
+    tests = [
+        ("mean", np.nan, lambda z, v, p: safe_mean(np.hstack((z, v)))),
+        ("median", np.nan, lambda z, v, p: safe_median(np.hstack((z, v)))),
+    ]
 
     for strategy, test_missing_values, true_value_fun in tests:
         X = np.empty(shape)
@@ -164,8 +200,7 @@ def test_imputation_mean_median():
         # And a matrix X_true containing all true values
         for j in range(shape[1]):
             nb_zeros = (j - dec + 1 > 0) * (j - dec + 1) * (j - dec + 1)
-            nb_missing_values = max(shape[0] + dec * dec
-                                    - (j + dec) * (j + dec), 0)
+            nb_missing_values = max(shape[0] + dec * dec - (j + dec) * (j + dec), 0)
             nb_values = shape[0] - nb_zeros - nb_missing_values
 
             z = zeros[:nb_zeros]
@@ -179,15 +214,13 @@ def test_imputation_mean_median():
 
             if 0 == test_missing_values:
                 # XXX unreached code as of v0.22
-                X_true[:, j] = np.hstack((v,
-                                          np.repeat(
-                                              true_statistics[j],
-                                              nb_missing_values + nb_zeros)))
+                X_true[:, j] = np.hstack(
+                    (v, np.repeat(true_statistics[j], nb_missing_values + nb_zeros))
+                )
             else:
-                X_true[:, j] = np.hstack((v,
-                                          z,
-                                          np.repeat(true_statistics[j],
-                                                    nb_missing_values)))
+                X_true[:, j] = np.hstack(
+                    (v, z, np.repeat(true_statistics[j], nb_missing_values))
+                )
 
             # Shuffle them the same way
             np.random.RandomState(j).shuffle(X[:, j])
@@ -201,77 +234,83 @@ def test_imputation_mean_median():
 
         X_true = X_true[:, cols_to_keep]
 
-        _check_statistics(X, X_true, strategy,
-                          true_statistics, test_missing_values)
+        _check_statistics(
+            X, X_true, strategy, true_statistics, test_missing_values, csc_container
+        )
 
 
-def test_imputation_median_special_cases():
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+def test_imputation_median_special_cases(csc_container):
     # Test median imputation with sparse boundary cases
-    X = np.array([
-        [0, np.nan, np.nan],  # odd: implicit zero
-        [5, np.nan, np.nan],  # odd: explicit nonzero
-        [0, 0, np.nan],    # even: average two zeros
-        [-5, 0, np.nan],   # even: avg zero and neg
-        [0, 5, np.nan],    # even: avg zero and pos
-        [4, 5, np.nan],    # even: avg nonzeros
-        [-4, -5, np.nan],  # even: avg negatives
-        [-1, 2, np.nan],   # even: crossing neg and pos
-    ]).transpose()
-
-    X_imputed_median = np.array([
-        [0, 0, 0],
-        [5, 5, 5],
-        [0, 0, 0],
-        [-5, 0, -2.5],
-        [0, 5, 2.5],
-        [4, 5, 4.5],
-        [-4, -5, -4.5],
-        [-1, 2, .5],
-    ]).transpose()
-    statistics_median = [0, 5, 0, -2.5, 2.5, 4.5, -4.5, .5]
-
-    _check_statistics(X, X_imputed_median, "median",
-                      statistics_median, np.nan)
+    X = np.array(
+        [
+            [0, np.nan, np.nan],  # odd: implicit zero
+            [5, np.nan, np.nan],  # odd: explicit nonzero
+            [0, 0, np.nan],  # even: average two zeros
+            [-5, 0, np.nan],  # even: avg zero and neg
+            [0, 5, np.nan],  # even: avg zero and pos
+            [4, 5, np.nan],  # even: avg nonzeros
+            [-4, -5, np.nan],  # even: avg negatives
+            [-1, 2, np.nan],  # even: crossing neg and pos
+        ]
+    ).transpose()
+
+    X_imputed_median = np.array(
+        [
+            [0, 0, 0],
+            [5, 5, 5],
+            [0, 0, 0],
+            [-5, 0, -2.5],
+            [0, 5, 2.5],
+            [4, 5, 4.5],
+            [-4, -5, -4.5],
+            [-1, 2, 0.5],
+        ]
+    ).transpose()
+    statistics_median = [0, 5, 0, -2.5, 2.5, 4.5, -4.5, 0.5]
+
+    _check_statistics(
+        X, X_imputed_median, "median", statistics_median, np.nan, csc_container
+    )
 
 
 @pytest.mark.parametrize("strategy", ["mean", "median"])
 @pytest.mark.parametrize("dtype", [None, object, str])
 def test_imputation_mean_median_error_invalid_type(strategy, dtype):
-    X = np.array([["a", "b", 3],
-                  [4, "e", 6],
-                  ["g", "h", 9]], dtype=dtype)
-    msg = "non-numeric data:\ncould not convert string to float: '"
+    X = np.array([["a", "b", 3], [4, "e", 6], ["g", "h", 9]], dtype=dtype)
+    msg = "non-numeric data:\ncould not convert string to float:"
     with pytest.raises(ValueError, match=msg):
         imputer = SimpleImputer(strategy=strategy)
         imputer.fit_transform(X)
 
 
 @pytest.mark.parametrize("strategy", ["mean", "median"])
-@pytest.mark.parametrize("type", ['list', 'dataframe'])
+@pytest.mark.parametrize("type", ["list", "dataframe"])
 def test_imputation_mean_median_error_invalid_type_list_pandas(strategy, type):
-    X = [["a", "b", 3],
-         [4, "e", 6],
-         ["g", "h", 9]]
-    if type == 'dataframe':
+    X = [["a", "b", 3], [4, "e", 6], ["g", "h", 9]]
+    if type == "dataframe":
         pd = pytest.importorskip("pandas")
         X = pd.DataFrame(X)
-    msg = "non-numeric data:\ncould not convert string to float: '"
+    msg = "non-numeric data:\ncould not convert string to float:"
     with pytest.raises(ValueError, match=msg):
         imputer = SimpleImputer(strategy=strategy)
         imputer.fit_transform(X)
 
 
 @pytest.mark.parametrize("strategy", ["constant", "most_frequent"])
-@pytest.mark.parametrize("dtype", [str, np.dtype('U'), np.dtype('S')])
+@pytest.mark.parametrize("dtype", [str, np.dtype("U"), np.dtype("S")])
 def test_imputation_const_mostf_error_invalid_types(strategy, dtype):
     # Test imputation on non-numeric data using "most_frequent" and "constant"
     # strategy
-    X = np.array([
-        [np.nan, np.nan, "a", "f"],
-        [np.nan, "c", np.nan, "d"],
-        [np.nan, "b", "d", np.nan],
-        [np.nan, "c", "d", "h"],
-    ], dtype=dtype)
+    X = np.array(
+        [
+            [np.nan, np.nan, "a", "f"],
+            [np.nan, "c", np.nan, "d"],
+            [np.nan, "b", "d", np.nan],
+            [np.nan, "c", "d", "h"],
+        ],
+        dtype=dtype,
+    )
 
     err_msg = "SimpleImputer does not support data"
     with pytest.raises(ValueError, match=err_msg):
@@ -279,48 +318,58 @@ def test_imputation_const_mostf_error_invalid_types(strategy, dtype):
         imputer.fit(X).transform(X)
 
 
-def test_imputation_most_frequent():
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+def test_imputation_most_frequent(csc_container):
     # Test imputation using the most-frequent strategy.
-    X = np.array([
-        [-1, -1, 0, 5],
-        [-1, 2, -1, 3],
-        [-1, 1, 3, -1],
-        [-1, 2, 3, 7],
-    ])
-
-    X_true = np.array([
-        [2, 0, 5],
-        [2, 3, 3],
-        [1, 3, 3],
-        [2, 3, 7],
-    ])
+    X = np.array(
+        [
+            [-1, -1, 0, 5],
+            [-1, 2, -1, 3],
+            [-1, 1, 3, -1],
+            [-1, 2, 3, 7],
+        ]
+    )
+
+    X_true = np.array(
+        [
+            [2, 0, 5],
+            [2, 3, 3],
+            [1, 3, 3],
+            [2, 3, 7],
+        ]
+    )
 
     # scipy.stats.mode, used in SimpleImputer, doesn't return the first most
     # frequent as promised in the doc but the lowest most frequent. When this
     # test will fail after an update of scipy, SimpleImputer will need to be
     # updated to be consistent with the new (correct) behaviour
-    _check_statistics(X, X_true, "most_frequent", [np.nan, 2, 3, 3], -1)
+    _check_statistics(X, X_true, "most_frequent", [np.nan, 2, 3, 3], -1, csc_container)
 
 
 @pytest.mark.parametrize("marker", [None, np.nan, "NAN", "", 0])
 def test_imputation_most_frequent_objects(marker):
     # Test imputation using the most-frequent strategy.
-    X = np.array([
-        [marker, marker, "a", "f"],
-        [marker, "c", marker, "d"],
-        [marker, "b", "d", marker],
-        [marker, "c", "d", "h"],
-    ], dtype=object)
-
-    X_true = np.array([
-        ["c", "a", "f"],
-        ["c", "d", "d"],
-        ["b", "d", "d"],
-        ["c", "d", "h"],
-    ], dtype=object)
-
-    imputer = SimpleImputer(missing_values=marker,
-                            strategy="most_frequent")
+    X = np.array(
+        [
+            [marker, marker, "a", "f"],
+            [marker, "c", marker, "d"],
+            [marker, "b", "d", marker],
+            [marker, "c", "d", "h"],
+        ],
+        dtype=object,
+    )
+
+    X_true = np.array(
+        [
+            ["c", "a", "f"],
+            ["c", "d", "d"],
+            ["b", "d", "d"],
+            ["c", "d", "h"],
+        ],
+        dtype=object,
+    )
+
+    imputer = SimpleImputer(missing_values=marker, strategy="most_frequent")
     X_trans = imputer.fit(X).transform(X)
 
     assert_array_equal(X_trans, X_true)
@@ -331,20 +380,14 @@ def test_imputation_most_frequent_pandas(dtype):
     # Test imputation using the most frequent strategy on pandas df
     pd = pytest.importorskip("pandas")
 
-    f = io.StringIO("Cat1,Cat2,Cat3,Cat4\n"
-                    ",i,x,\n"
-                    "a,,y,\n"
-                    "a,j,,\n"
-                    "b,j,x,")
+    f = io.StringIO("Cat1,Cat2,Cat3,Cat4\n,i,x,\na,,y,\na,j,,\nb,j,x,")
 
     df = pd.read_csv(f, dtype=dtype)
 
-    X_true = np.array([
-        ["a", "i", "x"],
-        ["a", "j", "y"],
-        ["a", "j", "x"],
-        ["b", "j", "x"]
-    ], dtype=object)
+    X_true = np.array(
+        [["a", "i", "x"], ["a", "j", "y"], ["a", "j", "x"], ["b", "j", "x"]],
+        dtype=object,
+    )
 
     imputer = SimpleImputer(strategy="most_frequent")
     X_trans = imputer.fit_transform(df)
@@ -352,114 +395,125 @@ def test_imputation_most_frequent_pandas(dtype):
     assert_array_equal(X_trans, X_true)
 
 
-@pytest.mark.parametrize("X_data, missing_value", [(1, 0), (1., np.nan)])
+@pytest.mark.parametrize("X_data, missing_value", [(1, 0), (1.0, np.nan)])
 def test_imputation_constant_error_invalid_type(X_data, missing_value):
     # Verify that exceptions are raised on invalid fill_value type
     X = np.full((3, 5), X_data, dtype=float)
     X[0, 0] = missing_value
 
-    with pytest.raises(ValueError, match="imputing numerical"):
-        imputer = SimpleImputer(missing_values=missing_value,
-                                strategy="constant",
-                                fill_value="x")
+    fill_value = "x"
+    err_msg = f"fill_value={fill_value!r} (of type {type(fill_value)!r}) cannot be cast"
+    with pytest.raises(ValueError, match=re.escape(err_msg)):
+        imputer = SimpleImputer(
+            missing_values=missing_value, strategy="constant", fill_value=fill_value
+        )
         imputer.fit_transform(X)
 
 
+# TODO (1.8): check that `keep_empty_features=False` drop the
+# empty features due to the behaviour change.
 def test_imputation_constant_integer():
     # Test imputation using the constant strategy on integers
-    X = np.array([
-        [-1, 2, 3, -1],
-        [4, -1, 5, -1],
-        [6, 7, -1, -1],
-        [8, 9, 0, -1]
-    ])
-
-    X_true = np.array([
-        [0, 2, 3, 0],
-        [4, 0, 5, 0],
-        [6, 7, 0, 0],
-        [8, 9, 0, 0]
-    ])
-
-    imputer = SimpleImputer(missing_values=-1, strategy="constant",
-                            fill_value=0)
+    X = np.array([[-1, 2, 3, -1], [4, -1, 5, -1], [6, 7, -1, -1], [8, 9, 0, -1]])
+
+    X_true = np.array([[0, 2, 3, 0], [4, 0, 5, 0], [6, 7, 0, 0], [8, 9, 0, 0]])
+
+    imputer = SimpleImputer(
+        missing_values=-1, strategy="constant", fill_value=0, keep_empty_features=True
+    )
     X_trans = imputer.fit_transform(X)
 
     assert_array_equal(X_trans, X_true)
 
 
-@pytest.mark.parametrize("array_constructor", [sparse.csr_matrix, np.asarray])
+# TODO (1.8): check that `keep_empty_features=False` drop the
+# empty features due to the behaviour change.
+@pytest.mark.parametrize("array_constructor", CSR_CONTAINERS + [np.asarray])
 def test_imputation_constant_float(array_constructor):
     # Test imputation using the constant strategy on floats
-    X = np.array([
-        [np.nan, 1.1, 0, np.nan],
-        [1.2, np.nan, 1.3, np.nan],
-        [0, 0, np.nan, np.nan],
-        [1.4, 1.5, 0, np.nan]
-    ])
-
-    X_true = np.array([
-        [-1, 1.1, 0, -1],
-        [1.2, -1, 1.3, -1],
-        [0, 0, -1, -1],
-        [1.4, 1.5, 0, -1]
-    ])
+    X = np.array(
+        [
+            [np.nan, 1.1, 0, np.nan],
+            [1.2, np.nan, 1.3, np.nan],
+            [0, 0, np.nan, np.nan],
+            [1.4, 1.5, 0, np.nan],
+        ]
+    )
+
+    X_true = np.array(
+        [[-1, 1.1, 0, -1], [1.2, -1, 1.3, -1], [0, 0, -1, -1], [1.4, 1.5, 0, -1]]
+    )
 
     X = array_constructor(X)
 
     X_true = array_constructor(X_true)
 
-    imputer = SimpleImputer(strategy="constant", fill_value=-1)
+    imputer = SimpleImputer(
+        strategy="constant", fill_value=-1, keep_empty_features=True
+    )
     X_trans = imputer.fit_transform(X)
 
     assert_allclose_dense_sparse(X_trans, X_true)
 
 
+# TODO (1.8): check that `keep_empty_features=False` drop the
+# empty features due to the behaviour change.
 @pytest.mark.parametrize("marker", [None, np.nan, "NAN", "", 0])
 def test_imputation_constant_object(marker):
     # Test imputation using the constant strategy on objects
-    X = np.array([
-        [marker, "a", "b", marker],
-        ["c", marker, "d", marker],
-        ["e", "f", marker, marker],
-        ["g", "h", "i", marker]
-    ], dtype=object)
-
-    X_true = np.array([
-        ["missing", "a", "b", "missing"],
-        ["c", "missing", "d", "missing"],
-        ["e", "f", "missing", "missing"],
-        ["g", "h", "i", "missing"]
-    ], dtype=object)
-
-    imputer = SimpleImputer(missing_values=marker, strategy="constant",
-                            fill_value="missing")
+    X = np.array(
+        [
+            [marker, "a", "b", marker],
+            ["c", marker, "d", marker],
+            ["e", "f", marker, marker],
+            ["g", "h", "i", marker],
+        ],
+        dtype=object,
+    )
+
+    X_true = np.array(
+        [
+            ["missing", "a", "b", "missing"],
+            ["c", "missing", "d", "missing"],
+            ["e", "f", "missing", "missing"],
+            ["g", "h", "i", "missing"],
+        ],
+        dtype=object,
+    )
+
+    imputer = SimpleImputer(
+        missing_values=marker,
+        strategy="constant",
+        fill_value="missing",
+        keep_empty_features=True,
+    )
     X_trans = imputer.fit_transform(X)
 
     assert_array_equal(X_trans, X_true)
 
 
+# TODO (1.8): check that `keep_empty_features=False` drop the
+# empty features due to the behaviour change.
 @pytest.mark.parametrize("dtype", [object, "category"])
 def test_imputation_constant_pandas(dtype):
     # Test imputation using the constant strategy on pandas df
     pd = pytest.importorskip("pandas")
 
-    f = io.StringIO("Cat1,Cat2,Cat3,Cat4\n"
-                    ",i,x,\n"
-                    "a,,y,\n"
-                    "a,j,,\n"
-                    "b,j,x,")
+    f = io.StringIO("Cat1,Cat2,Cat3,Cat4\n,i,x,\na,,y,\na,j,,\nb,j,x,")
 
     df = pd.read_csv(f, dtype=dtype)
 
-    X_true = np.array([
-        ["missing_value", "i", "x", "missing_value"],
-        ["a", "missing_value", "y", "missing_value"],
-        ["a", "j", "missing_value", "missing_value"],
-        ["b", "j", "x", "missing_value"]
-    ], dtype=object)
+    X_true = np.array(
+        [
+            ["missing_value", "i", "x", "missing_value"],
+            ["a", "missing_value", "y", "missing_value"],
+            ["a", "j", "missing_value", "missing_value"],
+            ["b", "j", "x", "missing_value"],
+        ],
+        dtype=object,
+    )
 
-    imputer = SimpleImputer(strategy="constant")
+    imputer = SimpleImputer(strategy="constant", keep_empty_features=True)
     X_trans = imputer.fit_transform(df)
 
     assert_array_equal(X_trans, X_true)
@@ -482,14 +536,14 @@ def test_imputation_pipeline_grid_search():
     X = _sparse_random_matrix(100, 100, density=0.10)
     missing_values = X.data[0]
 
-    pipeline = Pipeline([('imputer',
-                          SimpleImputer(missing_values=missing_values)),
-                         ('tree',
-                          tree.DecisionTreeRegressor(random_state=0))])
+    pipeline = Pipeline(
+        [
+            ("imputer", SimpleImputer(missing_values=missing_values)),
+            ("tree", tree.DecisionTreeRegressor(random_state=0)),
+        ]
+    )
 
-    parameters = {
-        'imputer__strategy': ["mean", "median", "most_frequent"]
-    }
+    parameters = {"imputer__strategy": ["mean", "median", "most_frequent"]}
 
     Y = _sparse_random_matrix(100, 1, density=0.10).toarray()
     gs = GridSearchCV(pipeline, parameters)
@@ -509,8 +563,7 @@ def test_imputation_copy():
 
     # copy=True, sparse csr => copy
     X = X_orig.copy()
-    imputer = SimpleImputer(missing_values=X.data[0], strategy="mean",
-                            copy=True)
+    imputer = SimpleImputer(missing_values=X.data[0], strategy="mean", copy=True)
     Xt = imputer.fit(X).transform(X)
     Xt.data[0] = -1
     assert not np.all(X.data == Xt.data)
@@ -524,16 +577,14 @@ def test_imputation_copy():
 
     # copy=False, sparse csc => no copy
     X = X_orig.copy().tocsc()
-    imputer = SimpleImputer(missing_values=X.data[0], strategy="mean",
-                            copy=False)
+    imputer = SimpleImputer(missing_values=X.data[0], strategy="mean", copy=False)
     Xt = imputer.fit(X).transform(X)
     Xt.data[0] = -1
     assert_array_almost_equal(X.data, Xt.data)
 
     # copy=False, sparse csr => copy
     X = X_orig.copy()
-    imputer = SimpleImputer(missing_values=X.data[0], strategy="mean",
-                            copy=False)
+    imputer = SimpleImputer(missing_values=X.data[0], strategy="mean", copy=False)
     Xt = imputer.fit(X).transform(X)
     Xt.data[0] = -1
     assert not np.all(X.data == Xt.data)
@@ -559,13 +610,11 @@ def test_iterative_imputer_zero_iters():
     # repeat but force n_iter_ to 0
     imputer = IterativeImputer(max_iter=5).fit(X)
     # transformed should not be equal to initial imputation
-    assert not np.all(imputer.transform(X) ==
-                      imputer.initial_imputer_.transform(X))
+    assert not np.all(imputer.transform(X) == imputer.initial_imputer_.transform(X))
 
     imputer.n_iter_ = 0
     # now they should be equal as only initial imputation is done
-    assert_allclose(imputer.transform(X),
-                    imputer.initial_imputer_.transform(X))
+    assert_allclose(imputer.transform(X), imputer.initial_imputer_.transform(X))
 
 
 def test_iterative_imputer_verbose():
@@ -592,8 +641,7 @@ def test_iterative_imputer_all_missing():
 
 
 @pytest.mark.parametrize(
-    "imputation_order",
-    ['random', 'roman', 'ascending', 'descending', 'arabic']
+    "imputation_order", ["random", "roman", "ascending", "descending", "arabic"]
 )
 def test_iterative_imputer_imputation_order(imputation_order):
     rng = np.random.RandomState(0)
@@ -603,37 +651,37 @@ def test_iterative_imputer_imputation_order(imputation_order):
     X = _sparse_random_matrix(n, d, density=0.10, random_state=rng).toarray()
     X[:, 0] = 1  # this column should not be discarded by IterativeImputer
 
-    imputer = IterativeImputer(missing_values=0,
-                               max_iter=max_iter,
-                               n_nearest_features=5,
-                               sample_posterior=False,
-                               skip_complete=True,
-                               min_value=0,
-                               max_value=1,
-                               verbose=1,
-                               imputation_order=imputation_order,
-                               random_state=rng)
+    imputer = IterativeImputer(
+        missing_values=0,
+        max_iter=max_iter,
+        n_nearest_features=5,
+        sample_posterior=False,
+        skip_complete=True,
+        min_value=0,
+        max_value=1,
+        verbose=1,
+        imputation_order=imputation_order,
+        random_state=rng,
+    )
     imputer.fit_transform(X)
     ordered_idx = [i.feat_idx for i in imputer.imputation_sequence_]
 
-    assert (len(ordered_idx) // imputer.n_iter_ ==
-            imputer.n_features_with_missing_)
+    assert len(ordered_idx) // imputer.n_iter_ == imputer.n_features_with_missing_
 
-    if imputation_order == 'roman':
-        assert np.all(ordered_idx[:d-1] == np.arange(1, d))
-    elif imputation_order == 'arabic':
-        assert np.all(ordered_idx[:d-1] == np.arange(d-1, 0, -1))
-    elif imputation_order == 'random':
-        ordered_idx_round_1 = ordered_idx[:d-1]
-        ordered_idx_round_2 = ordered_idx[d-1:]
+    if imputation_order == "roman":
+        assert np.all(ordered_idx[: d - 1] == np.arange(1, d))
+    elif imputation_order == "arabic":
+        assert np.all(ordered_idx[: d - 1] == np.arange(d - 1, 0, -1))
+    elif imputation_order == "random":
+        ordered_idx_round_1 = ordered_idx[: d - 1]
+        ordered_idx_round_2 = ordered_idx[d - 1 :]
         assert ordered_idx_round_1 != ordered_idx_round_2
-    elif 'ending' in imputation_order:
+    elif "ending" in imputation_order:
         assert len(ordered_idx) == max_iter * (d - 1)
 
 
 @pytest.mark.parametrize(
-    "estimator",
-    [None, DummyRegressor(), BayesianRidge(), ARDRegression(), RidgeCV()]
+    "estimator", [None, DummyRegressor(), BayesianRidge(), ARDRegression(), RidgeCV()]
 )
 def test_iterative_imputer_estimators(estimator):
     rng = np.random.RandomState(0)
@@ -642,17 +690,17 @@ def test_iterative_imputer_estimators(estimator):
     d = 10
     X = _sparse_random_matrix(n, d, density=0.10, random_state=rng).toarray()
 
-    imputer = IterativeImputer(missing_values=0,
-                               max_iter=1,
-                               estimator=estimator,
-                               random_state=rng)
+    imputer = IterativeImputer(
+        missing_values=0, max_iter=1, estimator=estimator, random_state=rng
+    )
     imputer.fit_transform(X)
 
     # check that types are correct for estimators
     hashes = []
     for triplet in imputer.imputation_sequence_:
-        expected_type = (type(estimator) if estimator is not None
-                         else type(BayesianRidge()))
+        expected_type = (
+            type(estimator) if estimator is not None else type(BayesianRidge())
+        )
         assert isinstance(triplet.estimator, expected_type)
         hashes.append(id(triplet.estimator))
 
@@ -664,14 +712,11 @@ def test_iterative_imputer_clip():
     rng = np.random.RandomState(0)
     n = 100
     d = 10
-    X = _sparse_random_matrix(n, d, density=0.10,
-                             random_state=rng).toarray()
+    X = _sparse_random_matrix(n, d, density=0.10, random_state=rng).toarray()
 
-    imputer = IterativeImputer(missing_values=0,
-                               max_iter=1,
-                               min_value=0.1,
-                               max_value=0.2,
-                               random_state=rng)
+    imputer = IterativeImputer(
+        missing_values=0, max_iter=1, min_value=0.1, max_value=0.2, random_state=rng
+    )
 
     Xt = imputer.fit_transform(X)
     assert_allclose(np.min(Xt[X == 0]), 0.1)
@@ -686,15 +731,17 @@ def test_iterative_imputer_clip_truncnorm():
     X = _sparse_random_matrix(n, d, density=0.10, random_state=rng).toarray()
     X[:, 0] = 1
 
-    imputer = IterativeImputer(missing_values=0,
-                               max_iter=2,
-                               n_nearest_features=5,
-                               sample_posterior=True,
-                               min_value=0.1,
-                               max_value=0.2,
-                               verbose=1,
-                               imputation_order='random',
-                               random_state=rng)
+    imputer = IterativeImputer(
+        missing_values=0,
+        max_iter=2,
+        n_nearest_features=5,
+        sample_posterior=True,
+        min_value=0.1,
+        max_value=0.2,
+        verbose=1,
+        imputation_order="random",
+        random_state=rng,
+    )
     Xt = imputer.fit_transform(X)
     assert_allclose(np.min(Xt[X == 0]), 0.1)
     assert_allclose(np.max(Xt[X == 0]), 0.2)
@@ -713,10 +760,9 @@ def test_iterative_imputer_truncated_normal_posterior():
     X = rng.normal(size=(5, 5))
     X[0][0] = np.nan
 
-    imputer = IterativeImputer(min_value=0,
-                               max_value=0.5,
-                               sample_posterior=True,
-                               random_state=rng)
+    imputer = IterativeImputer(
+        min_value=0, max_value=0.5, sample_posterior=True, random_state=rng
+    )
 
     imputer.fit_transform(X)
     # generate multiple imputations for the single missing value
@@ -726,20 +772,16 @@ def test_iterative_imputer_truncated_normal_posterior():
     assert all(imputations <= 0.5)
 
     mu, sigma = imputations.mean(), imputations.std()
-    ks_statistic, p_value = kstest((imputations - mu) / sigma, 'norm')
+    ks_statistic, p_value = kstest((imputations - mu) / sigma, "norm")
     if sigma == 0:
         sigma += 1e-12
-    ks_statistic, p_value = kstest((imputations - mu) / sigma, 'norm')
+    ks_statistic, p_value = kstest((imputations - mu) / sigma, "norm")
     # we want to fail to reject null hypothesis
     # null hypothesis: distributions are the same
-    assert ks_statistic < 0.2 or p_value > 0.1, \
-        "The posterior does appear to be normal"
+    assert ks_statistic < 0.2 or p_value > 0.1, "The posterior does appear to be normal"
 
 
-@pytest.mark.parametrize(
-    "strategy",
-    ["mean", "median", "most_frequent"]
-)
+@pytest.mark.parametrize("strategy", ["mean", "median", "most_frequent"])
 def test_iterative_imputer_missing_at_transform(strategy):
     rng = np.random.RandomState(0)
     n = 100
@@ -750,17 +792,16 @@ def test_iterative_imputer_missing_at_transform(strategy):
     X_train[:, 0] = 1  # definitely no missing values in 0th column
     X_test[0, 0] = 0  # definitely missing value in 0th column
 
-    imputer = IterativeImputer(missing_values=0,
-                               max_iter=1,
-                               initial_strategy=strategy,
-                               random_state=rng).fit(X_train)
-    initial_imputer = SimpleImputer(missing_values=0,
-                                    strategy=strategy).fit(X_train)
+    imputer = IterativeImputer(
+        missing_values=0, max_iter=1, initial_strategy=strategy, random_state=rng
+    ).fit(X_train)
+    initial_imputer = SimpleImputer(missing_values=0, strategy=strategy).fit(X_train)
 
     # if there were no missing values at time of fit, then imputer will
     # only use the initial imputer for that feature at transform
-    assert_allclose(imputer.transform(X_test)[:, 0],
-                    initial_imputer.transform(X_test)[:, 0])
+    assert_allclose(
+        imputer.transform(X_test)[:, 0], initial_imputer.transform(X_test)[:, 0]
+    )
 
 
 def test_iterative_imputer_transform_stochasticity():
@@ -768,14 +809,12 @@ def test_iterative_imputer_transform_stochasticity():
     rng2 = np.random.RandomState(1)
     n = 100
     d = 10
-    X = _sparse_random_matrix(n, d, density=0.10,
-                             random_state=rng1).toarray()
+    X = _sparse_random_matrix(n, d, density=0.10, random_state=rng1).toarray()
 
     # when sample_posterior=True, two transforms shouldn't be equal
-    imputer = IterativeImputer(missing_values=0,
-                               max_iter=1,
-                               sample_posterior=True,
-                               random_state=rng1)
+    imputer = IterativeImputer(
+        missing_values=0, max_iter=1, sample_posterior=True, random_state=rng1
+    )
     imputer.fit(X)
 
     X_fitted_1 = imputer.transform(X)
@@ -787,19 +826,23 @@ def test_iterative_imputer_transform_stochasticity():
     # when sample_posterior=False, and n_nearest_features=None
     # and imputation_order is not random
     # the two transforms should be identical even if rng are different
-    imputer1 = IterativeImputer(missing_values=0,
-                                max_iter=1,
-                                sample_posterior=False,
-                                n_nearest_features=None,
-                                imputation_order='ascending',
-                                random_state=rng1)
-
-    imputer2 = IterativeImputer(missing_values=0,
-                                max_iter=1,
-                                sample_posterior=False,
-                                n_nearest_features=None,
-                                imputation_order='ascending',
-                                random_state=rng2)
+    imputer1 = IterativeImputer(
+        missing_values=0,
+        max_iter=1,
+        sample_posterior=False,
+        n_nearest_features=None,
+        imputation_order="ascending",
+        random_state=rng1,
+    )
+
+    imputer2 = IterativeImputer(
+        missing_values=0,
+        max_iter=1,
+        sample_posterior=False,
+        n_nearest_features=None,
+        imputation_order="ascending",
+        random_state=rng2,
+    )
     imputer1.fit(X)
     imputer2.fit(X)
 
@@ -835,17 +878,12 @@ def test_iterative_imputer_rank_one():
     X_missing = X.copy()
     X_missing[nan_mask] = np.nan
 
-    imputer = IterativeImputer(max_iter=5,
-                               verbose=1,
-                               random_state=rng)
+    imputer = IterativeImputer(max_iter=5, verbose=1, random_state=rng)
     X_filled = imputer.fit_transform(X_missing)
     assert_allclose(X_filled, X, atol=0.02)
 
 
-@pytest.mark.parametrize(
-    "rank",
-    [3, 5]
-)
+@pytest.mark.parametrize("rank", [3, 5])
 def test_iterative_imputer_transform_recovery(rank):
     rng = np.random.RandomState(0)
     n = 70
@@ -863,10 +901,9 @@ def test_iterative_imputer_transform_recovery(rank):
     X_test_filled = X_filled[n:]
     X_test = X_missing[n:]
 
-    imputer = IterativeImputer(max_iter=5,
-                               imputation_order='descending',
-                               verbose=1,
-                               random_state=rng).fit(X_train)
+    imputer = IterativeImputer(
+        max_iter=5, imputation_order="descending", verbose=1, random_state=rng
+    ).fit(X_train)
     X_test_est = imputer.transform(X_test)
     assert_allclose(X_test_filled, X_test_est, atol=0.1)
 
@@ -880,7 +917,7 @@ def test_iterative_imputer_additive_matrix():
     X_filled = np.zeros(A.shape)
     for i in range(d):
         for j in range(d):
-            X_filled[:, (i+j) % d] += (A[:, i] + B[:, j]) / 2
+            X_filled[:, (i + j) % d] += (A[:, i] + B[:, j]) / 2
     # a quarter is randomly missing
     nan_mask = rng.rand(n, d) < 0.25
     X_missing = X_filled.copy()
@@ -892,24 +929,11 @@ def test_iterative_imputer_additive_matrix():
     X_test_filled = X_filled[n:]
     X_test = X_missing[n:]
 
-    imputer = IterativeImputer(max_iter=10,
-                               verbose=1,
-                               random_state=rng).fit(X_train)
+    imputer = IterativeImputer(max_iter=10, verbose=1, random_state=rng).fit(X_train)
     X_test_est = imputer.transform(X_test)
     assert_allclose(X_test_filled, X_test_est, rtol=1e-3, atol=0.01)
 
 
-@pytest.mark.parametrize("max_iter, tol, error_type, warning", [
-    (-1, 1e-3, ValueError, 'should be a positive integer'),
-    (1, -1e-3, ValueError, 'should be a non-negative float')
-])
-def test_iterative_imputer_error_param(max_iter, tol, error_type, warning):
-    X = np.zeros((100, 2))
-    imputer = IterativeImputer(max_iter=max_iter, tol=tol)
-    with pytest.raises(error_type, match=warning):
-        imputer.fit_transform(X)
-
-
 def test_iterative_imputer_early_stopping():
     rng = np.random.RandomState(0)
     n = 50
@@ -921,26 +945,21 @@ def test_iterative_imputer_early_stopping():
     X_missing = X.copy()
     X_missing[nan_mask] = np.nan
 
-    imputer = IterativeImputer(max_iter=100,
-                               tol=1e-2,
-                               sample_posterior=False,
-                               verbose=1,
-                               random_state=rng)
+    imputer = IterativeImputer(
+        max_iter=100, tol=1e-2, sample_posterior=False, verbose=1, random_state=rng
+    )
     X_filled_100 = imputer.fit_transform(X_missing)
     assert len(imputer.imputation_sequence_) == d * imputer.n_iter_
 
-    imputer = IterativeImputer(max_iter=imputer.n_iter_,
-                               sample_posterior=False,
-                               verbose=1,
-                               random_state=rng)
+    imputer = IterativeImputer(
+        max_iter=imputer.n_iter_, sample_posterior=False, verbose=1, random_state=rng
+    )
     X_filled_early = imputer.fit_transform(X_missing)
     assert_allclose(X_filled_100, X_filled_early, atol=1e-7)
 
-    imputer = IterativeImputer(max_iter=100,
-                               tol=0,
-                               sample_posterior=False,
-                               verbose=1,
-                               random_state=rng)
+    imputer = IterativeImputer(
+        max_iter=100, tol=0, sample_posterior=False, verbose=1, random_state=rng
+    )
     imputer.fit(X_missing)
     assert imputer.n_iter_ == imputer.max_iter
 
@@ -959,40 +978,45 @@ def test_iterative_imputer_catch_warning():
     missing_rate = 0.15
     for feat in range(n_features):
         sample_idx = rng.choice(
-            np.arange(n_samples), size=int(n_samples * missing_rate),
-            replace=False
+            np.arange(n_samples), size=int(n_samples * missing_rate), replace=False
         )
         X[sample_idx, feat] = np.nan
 
     imputer = IterativeImputer(n_nearest_features=5, sample_posterior=True)
-    with pytest.warns(None) as record:
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", RuntimeWarning)
         X_fill = imputer.fit_transform(X, y)
-    assert not record.list
     assert not np.any(np.isnan(X_fill))
 
 
 @pytest.mark.parametrize(
     "min_value, max_value, correct_output",
-    [(0, 100, np.array([[0] * 3, [100] * 3])),
-     (None, None, np.array([[-np.inf] * 3, [np.inf] * 3])),
-     (-np.inf, np.inf, np.array([[-np.inf] * 3, [np.inf] * 3])),
-     ([-5, 5, 10], [100, 200, 300], np.array([[-5, 5, 10], [100, 200, 300]])),
-     ([-5, -np.inf, 10], [100, 200, np.inf],
-      np.array([[-5, -np.inf, 10], [100, 200, np.inf]]))],
-    ids=["scalars", "None-default", "inf", "lists", "lists-with-inf"])
-def test_iterative_imputer_min_max_array_like(min_value,
-                                              max_value,
-                                              correct_output):
+    [
+        (0, 100, np.array([[0] * 3, [100] * 3])),
+        (None, None, np.array([[-np.inf] * 3, [np.inf] * 3])),
+        (-np.inf, np.inf, np.array([[-np.inf] * 3, [np.inf] * 3])),
+        ([-5, 5, 10], [100, 200, 300], np.array([[-5, 5, 10], [100, 200, 300]])),
+        (
+            [-5, -np.inf, 10],
+            [100, 200, np.inf],
+            np.array([[-5, -np.inf, 10], [100, 200, np.inf]]),
+        ),
+    ],
+    ids=["scalars", "None-default", "inf", "lists", "lists-with-inf"],
+)
+def test_iterative_imputer_min_max_array_like(min_value, max_value, correct_output):
     # check that passing scalar or array-like
     # for min_value and max_value in IterativeImputer works
     X = np.random.RandomState(0).randn(10, 3)
     imputer = IterativeImputer(min_value=min_value, max_value=max_value)
     imputer.fit(X)
 
-    assert (isinstance(imputer._min_value, np.ndarray) and
-            isinstance(imputer._max_value, np.ndarray))
-    assert ((imputer._min_value.shape[0] == X.shape[1]) and
-            (imputer._max_value.shape[0] == X.shape[1]))
+    assert isinstance(imputer._min_value, np.ndarray) and isinstance(
+        imputer._max_value, np.ndarray
+    )
+    assert (imputer._min_value.shape[0] == X.shape[1]) and (
+        imputer._max_value.shape[0] == X.shape[1]
+    )
 
     assert_allclose(correct_output[0, :], imputer._min_value)
     assert_allclose(correct_output[1, :], imputer._max_value)
@@ -1000,9 +1024,13 @@ def test_iterative_imputer_min_max_array_like(min_value,
 
 @pytest.mark.parametrize(
     "min_value, max_value, err_msg",
-    [(100, 0, "min_value >= max_value."),
-     (np.inf, -np.inf, "min_value >= max_value."),
-     ([-5, 5], [100, 200, 0], "_value' should be of shape")])
+    [
+        (100, 0, "min_value >= max_value."),
+        (np.inf, -np.inf, "min_value >= max_value."),
+        ([-5, 5], [100, 200, 0], "_value' should be of shape"),
+        ([-5, 5, 5], [100, 200], "_value' should be of shape"),
+    ],
+)
 def test_iterative_imputer_catch_min_max_error(min_value, max_value, err_msg):
     # check that passing scalar or array-like
     # for min_value and max_value in IterativeImputer works
@@ -1014,52 +1042,43 @@ def test_iterative_imputer_catch_min_max_error(min_value, max_value, err_msg):
 
 @pytest.mark.parametrize(
     "min_max_1, min_max_2",
-    [([None, None], [-np.inf, np.inf]),
-     ([-10, 10], [[-10] * 4, [10] * 4])],
-    ids=["None-vs-inf", "Scalar-vs-vector"])
+    [([None, None], [-np.inf, np.inf]), ([-10, 10], [[-10] * 4, [10] * 4])],
+    ids=["None-vs-inf", "Scalar-vs-vector"],
+)
 def test_iterative_imputer_min_max_array_like_imputation(min_max_1, min_max_2):
     # Test that None/inf and scalar/vector give the same imputation
-    X_train = np.array([
-        [np.nan, 2, 2, 1],
-        [10, np.nan, np.nan, 7],
-        [3, 1, np.nan, 1],
-        [np.nan, 4, 2, np.nan]])
-    X_test = np.array([
-        [np.nan, 2, np.nan, 5],
-        [2, 4, np.nan, np.nan],
-        [np.nan, 1, 10, 1]])
-    imputer1 = IterativeImputer(min_value=min_max_1[0],
-                                max_value=min_max_1[1],
-                                random_state=0)
-    imputer2 = IterativeImputer(min_value=min_max_2[0],
-                                max_value=min_max_2[1],
-                                random_state=0)
+    X_train = np.array(
+        [
+            [np.nan, 2, 2, 1],
+            [10, np.nan, np.nan, 7],
+            [3, 1, np.nan, 1],
+            [np.nan, 4, 2, np.nan],
+        ]
+    )
+    X_test = np.array(
+        [[np.nan, 2, np.nan, 5], [2, 4, np.nan, np.nan], [np.nan, 1, 10, 1]]
+    )
+    imputer1 = IterativeImputer(
+        min_value=min_max_1[0], max_value=min_max_1[1], random_state=0
+    )
+    imputer2 = IterativeImputer(
+        min_value=min_max_2[0], max_value=min_max_2[1], random_state=0
+    )
     X_test_imputed1 = imputer1.fit(X_train).transform(X_test)
     X_test_imputed2 = imputer2.fit(X_train).transform(X_test)
     assert_allclose(X_test_imputed1[:, 0], X_test_imputed2[:, 0])
 
 
-@pytest.mark.parametrize(
-    "skip_complete", [True, False]
-)
+@pytest.mark.parametrize("skip_complete", [True, False])
 def test_iterative_imputer_skip_non_missing(skip_complete):
     # check the imputing strategy when missing data are present in the
     # testing set only.
     # taken from: https://github.com/scikit-learn/scikit-learn/issues/14383
     rng = np.random.RandomState(0)
-    X_train = np.array([
-        [5, 2, 2, 1],
-        [10, 1, 2, 7],
-        [3, 1, 1, 1],
-        [8, 4, 2, 2]
-    ])
-    X_test = np.array([
-        [np.nan, 2, 4, 5],
-        [np.nan, 4, 1, 2],
-        [np.nan, 1, 10, 1]
-    ])
+    X_train = np.array([[5, 2, 2, 1], [10, 1, 2, 7], [3, 1, 1, 1], [8, 4, 2, 2]])
+    X_test = np.array([[np.nan, 2, 4, 5], [np.nan, 4, 1, 2], [np.nan, 1, 10, 1]])
     imputer = IterativeImputer(
-        initial_strategy='mean', skip_complete=skip_complete, random_state=rng
+        initial_strategy="mean", skip_complete=skip_complete, random_state=rng
     )
     X_test_est = imputer.fit(X_train).transform(X_test)
     if skip_complete:
@@ -1069,14 +1088,8 @@ def test_iterative_imputer_skip_non_missing(skip_complete):
         assert_allclose(X_test_est[:, 0], [11, 7, 12], rtol=1e-4)
 
 
-@pytest.mark.parametrize(
-    "rs_imputer",
-    [None, 1, np.random.RandomState(seed=1)]
-)
-@pytest.mark.parametrize(
-    "rs_estimator",
-    [None, 1, np.random.RandomState(seed=1)]
-)
+@pytest.mark.parametrize("rs_imputer", [None, 1, np.random.RandomState(seed=1)])
+@pytest.mark.parametrize("rs_estimator", [None, 1, np.random.RandomState(seed=1)])
 def test_iterative_imputer_dont_set_random_state(rs_imputer, rs_estimator):
     class ZeroEstimator:
         def __init__(self, random_state):
@@ -1097,18 +1110,20 @@ def predict(self, X):
 
 @pytest.mark.parametrize(
     "X_fit, X_trans, params, msg_err",
-    [(np.array([[-1, 1], [1, 2]]), np.array([[-1, 1], [1, -1]]),
-      {'features': 'missing-only', 'sparse': 'auto'},
-      'have missing values in transform but have no missing values in fit'),
-     (np.array([[-1, 1], [1, 2]]), np.array([[-1, 1], [1, 2]]),
-      {'features': 'random', 'sparse': 'auto'},
-      "'features' has to be either 'missing-only' or 'all'"),
-     (np.array([[-1, 1], [1, 2]]), np.array([[-1, 1], [1, 2]]),
-      {'features': 'all', 'sparse': 'random'},
-      "'sparse' has to be a boolean or 'auto'"),
-     (np.array([['a', 'b'], ['c', 'a']], dtype=str),
-      np.array([['a', 'b'], ['c', 'a']], dtype=str),
-      {}, "MissingIndicator does not support data with dtype")]
+    [
+        (
+            np.array([[-1, 1], [1, 2]]),
+            np.array([[-1, 1], [1, -1]]),
+            {"features": "missing-only", "sparse": "auto"},
+            "have missing values in transform but have no missing values in fit",
+        ),
+        (
+            np.array([["a", "b"], ["c", "a"]], dtype=str),
+            np.array([["a", "b"], ["c", "a"]], dtype=str),
+            {},
+            "MissingIndicator does not support data with dtype",
+        ),
+    ],
 )
 def test_missing_indicator_error(X_fit, X_trans, params, msg_err):
     indicator = MissingIndicator(missing_values=-1)
@@ -1117,32 +1132,37 @@ def test_missing_indicator_error(X_fit, X_trans, params, msg_err):
         indicator.fit(X_fit).transform(X_trans)
 
 
+def _generate_missing_indicator_cases():
+    missing_values_dtypes = [(0, np.int32), (np.nan, np.float64), (-1, np.int32)]
+    arr_types = (
+        [np.array]
+        + CSC_CONTAINERS
+        + CSR_CONTAINERS
+        + COO_CONTAINERS
+        + LIL_CONTAINERS
+        + BSR_CONTAINERS
+    )
+    return [
+        (arr_type, missing_values, dtype)
+        for arr_type, (missing_values, dtype) in product(
+            arr_types, missing_values_dtypes
+        )
+        if not (missing_values == 0 and arr_type is not np.array)
+    ]
+
+
 @pytest.mark.parametrize(
-    "missing_values, dtype, arr_type",
-    [(np.nan, np.float64, np.array),
-     (0,      np.int32,   np.array),
-     (-1,     np.int32,   np.array),
-     (np.nan, np.float64, sparse.csc_matrix),
-     (-1,     np.int32,   sparse.csc_matrix),
-     (np.nan, np.float64, sparse.csr_matrix),
-     (-1,     np.int32,   sparse.csr_matrix),
-     (np.nan, np.float64, sparse.coo_matrix),
-     (-1,     np.int32,   sparse.coo_matrix),
-     (np.nan, np.float64, sparse.lil_matrix),
-     (-1,     np.int32,   sparse.lil_matrix),
-     (np.nan, np.float64, sparse.bsr_matrix),
-     (-1,     np.int32,   sparse.bsr_matrix)
-     ])
+    "arr_type, missing_values, dtype", _generate_missing_indicator_cases()
+)
 @pytest.mark.parametrize(
     "param_features, n_features, features_indices",
-    [('missing-only', 3, np.array([0, 1, 2])),
-     ('all', 3, np.array([0, 1, 2]))])
-def test_missing_indicator_new(missing_values, arr_type, dtype, param_features,
-                               n_features, features_indices):
-    X_fit = np.array([[missing_values, missing_values, 1],
-                      [4, 2, missing_values]])
-    X_trans = np.array([[missing_values, missing_values, 1],
-                        [4, 12, 10]])
+    [("missing-only", 3, np.array([0, 1, 2])), ("all", 3, np.array([0, 1, 2]))],
+)
+def test_missing_indicator_new(
+    missing_values, arr_type, dtype, param_features, n_features, features_indices
+):
+    X_fit = np.array([[missing_values, missing_values, 1], [4, 2, missing_values]])
+    X_trans = np.array([[missing_values, missing_values, 1], [4, 12, 10]])
     X_fit_expected = np.array([[1, 1, 0], [0, 0, 1]])
     X_trans_expected = np.array([[1, 1, 0], [0, 0, 0]])
 
@@ -1152,9 +1172,9 @@ def test_missing_indicator_new(missing_values, arr_type, dtype, param_features,
     X_fit_expected = X_fit_expected.astype(dtype)
     X_trans_expected = X_trans_expected.astype(dtype)
 
-    indicator = MissingIndicator(missing_values=missing_values,
-                                 features=param_features,
-                                 sparse=False)
+    indicator = MissingIndicator(
+        missing_values=missing_values, features=param_features, sparse=False
+    )
     X_fit_mask = indicator.fit_transform(X_fit)
     X_trans_mask = indicator.transform(X_trans)
 
@@ -1176,24 +1196,22 @@ def test_missing_indicator_new(missing_values, arr_type, dtype, param_features,
 
     assert X_fit_mask_sparse.dtype == bool
     assert X_trans_mask_sparse.dtype == bool
-    assert X_fit_mask_sparse.format == 'csc'
-    assert X_trans_mask_sparse.format == 'csc'
+    assert X_fit_mask_sparse.format == "csc"
+    assert X_trans_mask_sparse.format == "csc"
     assert_allclose(X_fit_mask_sparse.toarray(), X_fit_mask)
     assert_allclose(X_trans_mask_sparse.toarray(), X_trans_mask)
 
 
 @pytest.mark.parametrize(
     "arr_type",
-    [sparse.csc_matrix, sparse.csr_matrix, sparse.coo_matrix,
-     sparse.lil_matrix, sparse.bsr_matrix])
+    CSC_CONTAINERS + CSR_CONTAINERS + COO_CONTAINERS + LIL_CONTAINERS + BSR_CONTAINERS,
+)
 def test_missing_indicator_raise_on_sparse_with_missing_0(arr_type):
     # test for sparse input and missing_value == 0
 
     missing_values = 0
-    X_fit = np.array([[missing_values, missing_values, 1],
-                      [4, missing_values, 2]])
-    X_trans = np.array([[missing_values, missing_values, 1],
-                        [4, 12, 10]])
+    X_fit = np.array([[missing_values, missing_values, 1], [4, missing_values, 2]])
+    X_trans = np.array([[missing_values, missing_values, 1], [4, 12, 10]])
 
     # convert the input to the right array format
     X_fit_sparse = arr_type(X_fit)
@@ -1209,34 +1227,36 @@ def test_missing_indicator_raise_on_sparse_with_missing_0(arr_type):
         indicator.transform(X_trans_sparse)
 
 
-@pytest.mark.parametrize("param_sparse", [True, False, 'auto'])
-@pytest.mark.parametrize("missing_values, arr_type",
-                         [(np.nan, np.array),
-                          (0,      np.array),
-                          (np.nan, sparse.csc_matrix),
-                          (np.nan, sparse.csr_matrix),
-                          (np.nan, sparse.coo_matrix),
-                          (np.nan, sparse.lil_matrix)
-                          ])
-def test_missing_indicator_sparse_param(arr_type, missing_values,
-                                        param_sparse):
+@pytest.mark.parametrize("param_sparse", [True, False, "auto"])
+@pytest.mark.parametrize(
+    "arr_type, missing_values",
+    [(np.array, 0)]
+    + list(
+        product(
+            CSC_CONTAINERS
+            + CSR_CONTAINERS
+            + COO_CONTAINERS
+            + LIL_CONTAINERS
+            + BSR_CONTAINERS,
+            [np.nan],
+        )
+    ),
+)
+def test_missing_indicator_sparse_param(arr_type, missing_values, param_sparse):
     # check the format of the output with different sparse parameter
-    X_fit = np.array([[missing_values, missing_values, 1],
-                      [4, missing_values, 2]])
-    X_trans = np.array([[missing_values, missing_values, 1],
-                        [4, 12, 10]])
+    X_fit = np.array([[missing_values, missing_values, 1], [4, missing_values, 2]])
+    X_trans = np.array([[missing_values, missing_values, 1], [4, 12, 10]])
     X_fit = arr_type(X_fit).astype(np.float64)
     X_trans = arr_type(X_trans).astype(np.float64)
 
-    indicator = MissingIndicator(missing_values=missing_values,
-                                 sparse=param_sparse)
+    indicator = MissingIndicator(missing_values=missing_values, sparse=param_sparse)
     X_fit_mask = indicator.fit_transform(X_fit)
     X_trans_mask = indicator.transform(X_trans)
 
     if param_sparse is True:
-        assert X_fit_mask.format == 'csc'
-        assert X_trans_mask.format == 'csc'
-    elif param_sparse == 'auto' and missing_values == 0:
+        assert X_fit_mask.format == "csc"
+        assert X_trans_mask.format == "csc"
+    elif param_sparse == "auto" and missing_values == 0:
         assert isinstance(X_fit_mask, np.ndarray)
         assert isinstance(X_trans_mask, np.ndarray)
     elif param_sparse is False:
@@ -1244,54 +1264,65 @@ def test_missing_indicator_sparse_param(arr_type, missing_values,
         assert isinstance(X_trans_mask, np.ndarray)
     else:
         if sparse.issparse(X_fit):
-            assert X_fit_mask.format == 'csc'
-            assert X_trans_mask.format == 'csc'
+            assert X_fit_mask.format == "csc"
+            assert X_trans_mask.format == "csc"
         else:
             assert isinstance(X_fit_mask, np.ndarray)
             assert isinstance(X_trans_mask, np.ndarray)
 
 
 def test_missing_indicator_string():
-    X = np.array([['a', 'b', 'c'], ['b', 'c', 'a']], dtype=object)
-    indicator = MissingIndicator(missing_values='a', features='all')
+    X = np.array([["a", "b", "c"], ["b", "c", "a"]], dtype=object)
+    indicator = MissingIndicator(missing_values="a", features="all")
     X_trans = indicator.fit_transform(X)
-    assert_array_equal(X_trans, np.array([[True, False, False],
-                                          [False, False, True]]))
+    assert_array_equal(X_trans, np.array([[True, False, False], [False, False, True]]))
 
 
 @pytest.mark.parametrize(
     "X, missing_values, X_trans_exp",
-    [(np.array([['a', 'b'], ['b', 'a']], dtype=object), 'a',
-      np.array([['b', 'b', True, False], ['b', 'b', False, True]],
-               dtype=object)),
-     (np.array([[np.nan, 1.], [1., np.nan]]), np.nan,
-      np.array([[1., 1., True, False], [1., 1., False, True]])),
-     (np.array([[np.nan, 'b'], ['b', np.nan]], dtype=object), np.nan,
-      np.array([['b', 'b', True, False], ['b', 'b', False, True]],
-               dtype=object)),
-     (np.array([[None, 'b'], ['b', None]], dtype=object), None,
-      np.array([['b', 'b', True, False], ['b', 'b', False, True]],
-               dtype=object))]
+    [
+        (
+            np.array([["a", "b"], ["b", "a"]], dtype=object),
+            "a",
+            np.array([["b", "b", True, False], ["b", "b", False, True]], dtype=object),
+        ),
+        (
+            np.array([[np.nan, 1.0], [1.0, np.nan]]),
+            np.nan,
+            np.array([[1.0, 1.0, True, False], [1.0, 1.0, False, True]]),
+        ),
+        (
+            np.array([[np.nan, "b"], ["b", np.nan]], dtype=object),
+            np.nan,
+            np.array([["b", "b", True, False], ["b", "b", False, True]], dtype=object),
+        ),
+        (
+            np.array([[None, "b"], ["b", None]], dtype=object),
+            None,
+            np.array([["b", "b", True, False], ["b", "b", False, True]], dtype=object),
+        ),
+    ],
 )
 def test_missing_indicator_with_imputer(X, missing_values, X_trans_exp):
     trans = make_union(
-        SimpleImputer(missing_values=missing_values, strategy='most_frequent'),
-        MissingIndicator(missing_values=missing_values)
+        SimpleImputer(missing_values=missing_values, strategy="most_frequent"),
+        MissingIndicator(missing_values=missing_values),
     )
     X_trans = trans.fit_transform(X)
     assert_array_equal(X_trans, X_trans_exp)
 
 
-@pytest.mark.parametrize("imputer_constructor",
-                         [SimpleImputer, IterativeImputer])
+@pytest.mark.parametrize("imputer_constructor", [SimpleImputer, IterativeImputer])
 @pytest.mark.parametrize(
     "imputer_missing_values, missing_value, err_msg",
-    [("NaN", np.nan, "Input contains NaN"),
-     ("-1", -1, "types are expected to be both numerical.")])
-def test_inconsistent_dtype_X_missing_values(imputer_constructor,
-                                             imputer_missing_values,
-                                             missing_value,
-                                             err_msg):
+    [
+        ("NaN", np.nan, "Input X contains NaN"),
+        ("-1", -1, "types are expected to be both numerical."),
+    ],
+)
+def test_inconsistent_dtype_X_missing_values(
+    imputer_constructor, imputer_missing_values, missing_value, err_msg
+):
     # regression test for issue #11390. Comparison between incoherent dtype
     # for X and missing_values was not raising a proper error.
     rng = np.random.RandomState(42)
@@ -1307,33 +1338,29 @@ def test_inconsistent_dtype_X_missing_values(imputer_constructor,
 def test_missing_indicator_no_missing():
     # check that all features are dropped if there are no missing values when
     # features='missing-only' (#13491)
-    X = np.array([[1, 1],
-                  [1, 1]])
+    X = np.array([[1, 1], [1, 1]])
 
-    mi = MissingIndicator(features='missing-only', missing_values=-1)
+    mi = MissingIndicator(features="missing-only", missing_values=-1)
     Xt = mi.fit_transform(X)
 
     assert Xt.shape[1] == 0
 
 
-def test_missing_indicator_sparse_no_explicit_zeros():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_missing_indicator_sparse_no_explicit_zeros(csr_container):
     # Check that non missing values don't become explicit zeros in the mask
     # generated by missing indicator when X is sparse. (#13491)
-    X = sparse.csr_matrix([[0, 1, 2],
-                           [1, 2, 0],
-                           [2, 0, 1]])
+    X = csr_container([[0, 1, 2], [1, 2, 0], [2, 0, 1]])
 
-    mi = MissingIndicator(features='all', missing_values=1)
+    mi = MissingIndicator(features="all", missing_values=1)
     Xt = mi.fit_transform(X)
 
-    assert Xt.getnnz() == Xt.sum()
+    assert Xt.nnz == Xt.sum()
 
 
-@pytest.mark.parametrize("imputer_constructor",
-                         [SimpleImputer, IterativeImputer])
+@pytest.mark.parametrize("imputer_constructor", [SimpleImputer, IterativeImputer])
 def test_imputer_without_indicator(imputer_constructor):
-    X = np.array([[1, 1],
-                  [1, 1]])
+    X = np.array([[1, 1], [1, 1]])
     imputer = imputer_constructor()
     imputer.fit(X)
 
@@ -1342,24 +1369,18 @@ def test_imputer_without_indicator(imputer_constructor):
 
 @pytest.mark.parametrize(
     "arr_type",
-    [
-        sparse.csc_matrix, sparse.csr_matrix, sparse.coo_matrix,
-        sparse.lil_matrix, sparse.bsr_matrix
-    ]
+    CSC_CONTAINERS + CSR_CONTAINERS + COO_CONTAINERS + LIL_CONTAINERS + BSR_CONTAINERS,
 )
 def test_simple_imputation_add_indicator_sparse_matrix(arr_type):
-    X_sparse = arr_type([
-        [np.nan, 1, 5],
-        [2, np.nan, 1],
-        [6, 3, np.nan],
-        [1, 2, 9]
-    ])
-    X_true = np.array([
-        [3., 1., 5., 1., 0., 0.],
-        [2., 2., 1., 0., 1., 0.],
-        [6., 3., 5., 0., 0., 1.],
-        [1., 2., 9., 0., 0., 0.],
-    ])
+    X_sparse = arr_type([[np.nan, 1, 5], [2, np.nan, 1], [6, 3, np.nan], [1, 2, 9]])
+    X_true = np.array(
+        [
+            [3.0, 1.0, 5.0, 1.0, 0.0, 0.0],
+            [2.0, 2.0, 1.0, 0.0, 1.0, 0.0],
+            [6.0, 3.0, 5.0, 0.0, 0.0, 1.0],
+            [1.0, 2.0, 9.0, 0.0, 0.0, 0.0],
+        ]
+    )
 
     imputer = SimpleImputer(missing_values=np.nan, add_indicator=True)
     X_trans = imputer.fit_transform(X_sparse)
@@ -1370,17 +1391,12 @@ def test_simple_imputation_add_indicator_sparse_matrix(arr_type):
 
 
 @pytest.mark.parametrize(
-    'strategy, expected',
-    [('most_frequent', 'b'), ('constant', 'missing_value')]
+    "strategy, expected", [("most_frequent", "b"), ("constant", "missing_value")]
 )
 def test_simple_imputation_string_list(strategy, expected):
-    X = [['a', 'b'],
-         ['c', np.nan]]
+    X = [["a", "b"], ["c", np.nan]]
 
-    X_true = np.array([
-        ['a', 'b'],
-        ['c', expected]
-    ], dtype=object)
+    X_true = np.array([["a", "b"], ["c", expected]], dtype=object)
 
     imputer = SimpleImputer(strategy=strategy)
     X_trans = imputer.fit_transform(X)
@@ -1390,10 +1406,7 @@ def test_simple_imputation_string_list(strategy, expected):
 
 @pytest.mark.parametrize(
     "order, idx_order",
-    [
-        ("ascending", [3, 4, 2, 0, 1]),
-        ("descending", [1, 0, 2, 4, 3])
-    ]
+    [("ascending", [3, 4, 2, 0, 1]), ("descending", [1, 0, 2, 4, 3])],
 )
 def test_imputation_order(order, idx_order):
     # regression test for #15393
@@ -1405,9 +1418,9 @@ def test_imputation_order(order, idx_order):
     X[:10, 4] = np.nan
 
     with pytest.warns(ConvergenceWarning):
-        trs = IterativeImputer(max_iter=1,
-                               imputation_order=order,
-                               random_state=0).fit(X)
+        trs = IterativeImputer(max_iter=1, imputation_order=order, random_state=0).fit(
+            X
+        )
         idx = [x.feat_idx for x in trs.imputation_sequence_]
         assert idx == idx_order
 
@@ -1415,36 +1428,45 @@ def test_imputation_order(order, idx_order):
 @pytest.mark.parametrize("missing_value", [-1, np.nan])
 def test_simple_imputation_inverse_transform(missing_value):
     # Test inverse_transform feature for np.nan
-    X_1 = np.array([
-        [9, missing_value, 3, -1],
-        [4, -1, 5, 4],
-        [6, 7, missing_value, -1],
-        [8, 9, 0, missing_value]
-    ])
-
-    X_2 = np.array([
-        [5, 4, 2, 1],
-        [2, 1, missing_value, 3],
-        [9, missing_value, 7, 1],
-        [6, 4, 2, missing_value]
-    ])
-
-    X_3 = np.array([
-        [1, missing_value, 5, 9],
-        [missing_value, 4, missing_value, missing_value],
-        [2, missing_value, 7, missing_value],
-        [missing_value, 3, missing_value, 8]
-    ])
-
-    X_4 = np.array([
-        [1, 1, 1, 3],
-        [missing_value, 2, missing_value, 1],
-        [2, 3, 3, 4],
-        [missing_value, 4, missing_value, 2]
-    ])
-
-    imputer = SimpleImputer(missing_values=missing_value, strategy='mean',
-                            add_indicator=True)
+    X_1 = np.array(
+        [
+            [9, missing_value, 3, -1],
+            [4, -1, 5, 4],
+            [6, 7, missing_value, -1],
+            [8, 9, 0, missing_value],
+        ]
+    )
+
+    X_2 = np.array(
+        [
+            [5, 4, 2, 1],
+            [2, 1, missing_value, 3],
+            [9, missing_value, 7, 1],
+            [6, 4, 2, missing_value],
+        ]
+    )
+
+    X_3 = np.array(
+        [
+            [1, missing_value, 5, 9],
+            [missing_value, 4, missing_value, missing_value],
+            [2, missing_value, 7, missing_value],
+            [missing_value, 3, missing_value, 8],
+        ]
+    )
+
+    X_4 = np.array(
+        [
+            [1, 1, 1, 3],
+            [missing_value, 2, missing_value, 1],
+            [2, 3, 3, 4],
+            [missing_value, 4, missing_value, 2],
+        ]
+    )
+
+    imputer = SimpleImputer(
+        missing_values=missing_value, strategy="mean", add_indicator=True
+    )
 
     X_1_trans = imputer.fit_transform(X_1)
     X_1_inv_trans = imputer.inverse_transform(X_1_trans)
@@ -1463,17 +1485,20 @@ def test_simple_imputation_inverse_transform(missing_value):
 
 @pytest.mark.parametrize("missing_value", [-1, np.nan])
 def test_simple_imputation_inverse_transform_exceptions(missing_value):
-    X_1 = np.array([
-        [9, missing_value, 3, -1],
-        [4, -1, 5, 4],
-        [6, 7, missing_value, -1],
-        [8, 9, 0, missing_value]
-    ])
+    X_1 = np.array(
+        [
+            [9, missing_value, 3, -1],
+            [4, -1, 5, 4],
+            [6, 7, missing_value, -1],
+            [8, 9, 0, missing_value],
+        ]
+    )
 
     imputer = SimpleImputer(missing_values=missing_value, strategy="mean")
     X_1_trans = imputer.fit_transform(X_1)
-    with pytest.raises(ValueError,
-                       match=f"Got 'add_indicator={imputer.add_indicator}'"):
+    with pytest.raises(
+        ValueError, match=f"Got 'add_indicator={imputer.add_indicator}'"
+    ):
         imputer.inverse_transform(X_1_trans)
 
 
@@ -1481,22 +1506,430 @@ def test_simple_imputation_inverse_transform_exceptions(missing_value):
     "expected,array,dtype,extra_value,n_repeat",
     [
         # array of object dtype
-        ("extra_value", ['a', 'b', 'c'], object, "extra_value", 2),
+        ("extra_value", ["a", "b", "c"], object, "extra_value", 2),
         (
             "most_frequent_value",
-            ['most_frequent_value', 'most_frequent_value', 'value'],
-            object, "extra_value", 1
+            ["most_frequent_value", "most_frequent_value", "value"],
+            object,
+            "extra_value",
+            1,
         ),
-        ("a", ['min_value', 'min_value' 'value'], object, "a", 2),
-        ("min_value", ['min_value', 'min_value', 'value'], object, "z", 2),
+        ("a", ["min_value", "min_valuevalue"], object, "a", 2),
+        ("min_value", ["min_value", "min_value", "value"], object, "z", 2),
         # array of numeric dtype
         (10, [1, 2, 3], int, 10, 2),
         (1, [1, 1, 2], int, 10, 1),
         (10, [20, 20, 1], int, 10, 2),
         (1, [1, 1, 20], int, 10, 2),
-    ]
+    ],
 )
 def test_most_frequent(expected, array, dtype, extra_value, n_repeat):
     assert expected == _most_frequent(
         np.array(array, dtype=dtype), extra_value, n_repeat
     )
+
+
+@pytest.mark.parametrize(
+    "initial_strategy", ["mean", "median", "most_frequent", "constant"]
+)
+def test_iterative_imputer_keep_empty_features(initial_strategy):
+    """Check the behaviour of the iterative imputer with different initial strategy
+    and keeping empty features (i.e. features containing only missing values).
+    """
+    X = np.array([[1, np.nan, 2], [3, np.nan, np.nan]])
+
+    imputer = IterativeImputer(
+        initial_strategy=initial_strategy, keep_empty_features=True
+    )
+    X_imputed = imputer.fit_transform(X)
+    assert_allclose(X_imputed[:, 1], 0)
+    X_imputed = imputer.transform(X)
+    assert_allclose(X_imputed[:, 1], 0)
+
+
+# TODO (1.8): check that `keep_empty_features=False` drop the
+# empty features due to the behaviour change.
+def test_iterative_imputer_constant_fill_value():
+    """Check that we propagate properly the parameter `fill_value`."""
+    X = np.array([[-1, 2, 3, -1], [4, -1, 5, -1], [6, 7, -1, -1], [8, 9, 0, -1]])
+
+    fill_value = 100
+    imputer = IterativeImputer(
+        missing_values=-1,
+        initial_strategy="constant",
+        fill_value=fill_value,
+        max_iter=0,
+        keep_empty_features=True,
+    )
+    imputer.fit_transform(X)
+    assert_array_equal(imputer.initial_imputer_.statistics_, fill_value)
+
+
+def test_iterative_imputer_min_max_value_remove_empty():
+    """Check that we properly apply the empty feature mask to `min_value` and
+    `max_value`.
+
+    Non-regression test for https://github.com/scikit-learn/scikit-learn/issues/29355
+    """
+    # Intentionally make column 2 as a missing column, then the bound of the imputed
+    # value of column 3 should be (4, 5)
+    X = np.array(
+        [
+            [1, 2, np.nan, np.nan],
+            [4, 5, np.nan, 6],
+            [7, 8, np.nan, np.nan],
+            [10, 11, np.nan, 12],
+        ]
+    )
+    min_value = [-np.inf, -np.inf, -np.inf, 4]
+    max_value = [np.inf, np.inf, np.inf, 5]
+
+    X_imputed = IterativeImputer(
+        min_value=min_value,
+        max_value=max_value,
+        keep_empty_features=False,
+    ).fit_transform(X)
+
+    X_without_missing_column = np.delete(X, 2, axis=1)
+    assert X_imputed.shape == X_without_missing_column.shape
+    assert np.min(X_imputed[np.isnan(X_without_missing_column)]) == pytest.approx(4)
+    assert np.max(X_imputed[np.isnan(X_without_missing_column)]) == pytest.approx(5)
+
+    # Intentionally make column 3 as a missing column, then the bound of the imputed
+    # value of column 2 should be (3.5, 6)
+    X = np.array(
+        [
+            [1, 2, np.nan, np.nan],
+            [4, 5, 6, np.nan],
+            [7, 8, np.nan, np.nan],
+            [10, 11, 12, np.nan],
+        ]
+    )
+    min_value = [-np.inf, -np.inf, 3.5, -np.inf]
+    max_value = [np.inf, np.inf, 6, np.inf]
+
+    X_imputed = IterativeImputer(
+        min_value=min_value,
+        max_value=max_value,
+        keep_empty_features=False,
+    ).fit_transform(X)
+
+    X_without_missing_column = X[:, :3]
+    assert X_imputed.shape == X_without_missing_column.shape
+    assert np.min(X_imputed[np.isnan(X_without_missing_column)]) == pytest.approx(3.5)
+    assert np.max(X_imputed[np.isnan(X_without_missing_column)]) == pytest.approx(6)
+
+
+@pytest.mark.parametrize("keep_empty_features", [True, False])
+def test_knn_imputer_keep_empty_features(keep_empty_features):
+    """Check the behaviour of `keep_empty_features` for `KNNImputer`."""
+    X = np.array([[1, np.nan, 2], [3, np.nan, np.nan]])
+
+    imputer = KNNImputer(keep_empty_features=keep_empty_features)
+
+    for method in ["fit_transform", "transform"]:
+        X_imputed = getattr(imputer, method)(X)
+        if keep_empty_features:
+            assert X_imputed.shape == X.shape
+            assert_array_equal(X_imputed[:, 1], 0)
+        else:
+            assert X_imputed.shape == (X.shape[0], X.shape[1] - 1)
+
+
+def test_simple_impute_pd_na():
+    pd = pytest.importorskip("pandas")
+
+    # Impute pandas array of string types.
+    df = pd.DataFrame({"feature": pd.Series(["abc", None, "de"], dtype="string")})
+    imputer = SimpleImputer(missing_values=pd.NA, strategy="constant", fill_value="na")
+    _assert_array_equal_and_same_dtype(
+        imputer.fit_transform(df), np.array([["abc"], ["na"], ["de"]], dtype=object)
+    )
+
+    # Impute pandas array of string types without any missing values.
+    df = pd.DataFrame({"feature": pd.Series(["abc", "de", "fgh"], dtype="string")})
+    imputer = SimpleImputer(fill_value="ok", strategy="constant")
+    _assert_array_equal_and_same_dtype(
+        imputer.fit_transform(df), np.array([["abc"], ["de"], ["fgh"]], dtype=object)
+    )
+
+    # Impute pandas array of integer types.
+    df = pd.DataFrame({"feature": pd.Series([1, None, 3], dtype="Int64")})
+    imputer = SimpleImputer(missing_values=pd.NA, strategy="constant", fill_value=-1)
+    _assert_allclose_and_same_dtype(
+        imputer.fit_transform(df), np.array([[1], [-1], [3]], dtype="float64")
+    )
+
+    # Use `np.nan` also works.
+    imputer = SimpleImputer(missing_values=np.nan, strategy="constant", fill_value=-1)
+    _assert_allclose_and_same_dtype(
+        imputer.fit_transform(df), np.array([[1], [-1], [3]], dtype="float64")
+    )
+
+    # Impute pandas array of integer types with 'median' strategy.
+    df = pd.DataFrame({"feature": pd.Series([1, None, 2, 3], dtype="Int64")})
+    imputer = SimpleImputer(missing_values=pd.NA, strategy="median")
+    _assert_allclose_and_same_dtype(
+        imputer.fit_transform(df), np.array([[1], [2], [2], [3]], dtype="float64")
+    )
+
+    # Impute pandas array of integer types with 'mean' strategy.
+    df = pd.DataFrame({"feature": pd.Series([1, None, 2], dtype="Int64")})
+    imputer = SimpleImputer(missing_values=pd.NA, strategy="mean")
+    _assert_allclose_and_same_dtype(
+        imputer.fit_transform(df), np.array([[1], [1.5], [2]], dtype="float64")
+    )
+
+    # Impute pandas array of float types.
+    df = pd.DataFrame({"feature": pd.Series([1.0, None, 3.0], dtype="float64")})
+    imputer = SimpleImputer(missing_values=pd.NA, strategy="constant", fill_value=-2.0)
+    _assert_allclose_and_same_dtype(
+        imputer.fit_transform(df), np.array([[1.0], [-2.0], [3.0]], dtype="float64")
+    )
+
+    # Impute pandas array of float types with 'median' strategy.
+    df = pd.DataFrame({"feature": pd.Series([1.0, None, 2.0, 3.0], dtype="float64")})
+    imputer = SimpleImputer(missing_values=pd.NA, strategy="median")
+    _assert_allclose_and_same_dtype(
+        imputer.fit_transform(df),
+        np.array([[1.0], [2.0], [2.0], [3.0]], dtype="float64"),
+    )
+
+
+def test_missing_indicator_feature_names_out():
+    """Check that missing indicator return the feature names with a prefix."""
+    pd = pytest.importorskip("pandas")
+
+    missing_values = np.nan
+    X = pd.DataFrame(
+        [
+            [missing_values, missing_values, 1, missing_values],
+            [4, missing_values, 2, 10],
+        ],
+        columns=["a", "b", "c", "d"],
+    )
+
+    indicator = MissingIndicator(missing_values=missing_values).fit(X)
+    feature_names = indicator.get_feature_names_out()
+    expected_names = ["missingindicator_a", "missingindicator_b", "missingindicator_d"]
+    assert_array_equal(expected_names, feature_names)
+
+
+def test_imputer_lists_fit_transform():
+    """Check transform uses object dtype when fitted on an object dtype.
+
+    Non-regression test for #19572.
+    """
+
+    X = [["a", "b"], ["c", "b"], ["a", "a"]]
+    imp_frequent = SimpleImputer(strategy="most_frequent").fit(X)
+    X_trans = imp_frequent.transform([[np.nan, np.nan]])
+    assert X_trans.dtype == object
+    assert_array_equal(X_trans, [["a", "b"]])
+
+
+@pytest.mark.parametrize("dtype_test", [np.float32, np.float64])
+def test_imputer_transform_preserves_numeric_dtype(dtype_test):
+    """Check transform preserves numeric dtype independent of fit dtype."""
+    X = np.asarray(
+        [[1.2, 3.4, np.nan], [np.nan, 1.2, 1.3], [4.2, 2, 1]], dtype=np.float64
+    )
+    imp = SimpleImputer().fit(X)
+
+    X_test = np.asarray([[np.nan, np.nan, np.nan]], dtype=dtype_test)
+    X_trans = imp.transform(X_test)
+    assert X_trans.dtype == dtype_test
+
+
+@pytest.mark.parametrize("array_type", ["array", "sparse"])
+@pytest.mark.parametrize("keep_empty_features", [True, False])
+def test_simple_imputer_constant_keep_empty_features(array_type, keep_empty_features):
+    """Check the behaviour of `keep_empty_features` with `strategy='constant'.
+    For backward compatibility, a column full of missing values will always be
+    fill and never dropped.
+    """
+    X = np.array([[np.nan, 2], [np.nan, 3], [np.nan, 6]])
+    X = _convert_container(X, array_type)
+    fill_value = 10
+    imputer = SimpleImputer(
+        strategy="constant",
+        fill_value=fill_value,
+        keep_empty_features=keep_empty_features,
+    )
+
+    for method in ["fit_transform", "transform"]:
+        # TODO(1.8): Remove the condition and still call getattr(imputer, method)(X)
+        if method.startswith("fit") and not keep_empty_features:
+            warn_msg = '`strategy="constant"`, empty features are not dropped. '
+            with pytest.warns(FutureWarning, match=warn_msg):
+                X_imputed = getattr(imputer, method)(X)
+        else:
+            X_imputed = getattr(imputer, method)(X)
+        assert X_imputed.shape == X.shape
+        constant_feature = (
+            X_imputed[:, 0].toarray() if array_type == "sparse" else X_imputed[:, 0]
+        )
+        assert_array_equal(constant_feature, fill_value)
+
+
+@pytest.mark.parametrize("array_type", ["array", "sparse"])
+@pytest.mark.parametrize("strategy", ["mean", "median", "most_frequent"])
+@pytest.mark.parametrize("keep_empty_features", [True, False])
+def test_simple_imputer_keep_empty_features(strategy, array_type, keep_empty_features):
+    """Check the behaviour of `keep_empty_features` with all strategies but
+    'constant'.
+    """
+    X = np.array([[np.nan, 2], [np.nan, 3], [np.nan, 6]])
+    X = _convert_container(X, array_type)
+    imputer = SimpleImputer(strategy=strategy, keep_empty_features=keep_empty_features)
+
+    for method in ["fit_transform", "transform"]:
+        X_imputed = getattr(imputer, method)(X)
+        if keep_empty_features:
+            assert X_imputed.shape == X.shape
+            constant_feature = (
+                X_imputed[:, 0].toarray() if array_type == "sparse" else X_imputed[:, 0]
+            )
+            assert_array_equal(constant_feature, 0)
+        else:
+            assert X_imputed.shape == (X.shape[0], X.shape[1] - 1)
+
+
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+def test_imputation_custom(csc_container):
+    X = np.array(
+        [
+            [1.1, 1.1, 1.1],
+            [3.9, 1.2, np.nan],
+            [np.nan, 1.3, np.nan],
+            [0.1, 1.4, 1.4],
+            [4.9, 1.5, 1.5],
+            [np.nan, 1.6, 1.6],
+        ]
+    )
+
+    X_true = np.array(
+        [
+            [1.1, 1.1, 1.1],
+            [3.9, 1.2, 1.1],
+            [0.1, 1.3, 1.1],
+            [0.1, 1.4, 1.4],
+            [4.9, 1.5, 1.5],
+            [0.1, 1.6, 1.6],
+        ]
+    )
+
+    imputer = SimpleImputer(missing_values=np.nan, strategy=np.min)
+    X_trans = imputer.fit_transform(X)
+    assert_array_equal(X_trans, X_true)
+
+    # Sparse matrix
+    imputer = SimpleImputer(missing_values=np.nan, strategy=np.min)
+    X_trans = imputer.fit_transform(csc_container(X))
+    assert_array_equal(X_trans.toarray(), X_true)
+
+
+def test_simple_imputer_constant_fill_value_casting():
+    """Check that we raise a proper error message when we cannot cast the fill value
+    to the input data type. Otherwise, check that the casting is done properly.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/28309
+    """
+    # cannot cast fill_value at fit
+    fill_value = 1.5
+    X_int64 = np.array([[1, 2, 3], [2, 3, 4]], dtype=np.int64)
+    imputer = SimpleImputer(
+        strategy="constant", fill_value=fill_value, missing_values=2
+    )
+    err_msg = f"fill_value={fill_value!r} (of type {type(fill_value)!r}) cannot be cast"
+    with pytest.raises(ValueError, match=re.escape(err_msg)):
+        imputer.fit(X_int64)
+
+    # cannot cast fill_value at transform
+    X_float64 = np.array([[1, 2, 3], [2, 3, 4]], dtype=np.float64)
+    imputer.fit(X_float64)
+    err_msg = (
+        f"The dtype of the filling value (i.e. {imputer.statistics_.dtype!r}) "
+        "cannot be cast"
+    )
+    with pytest.raises(ValueError, match=re.escape(err_msg)):
+        imputer.transform(X_int64)
+
+    # check that no error is raised when having the same kind of dtype
+    fill_value_list = [np.float64(1.5), 1.5, 1]
+    X_float32 = X_float64.astype(np.float32)
+
+    for fill_value in fill_value_list:
+        imputer = SimpleImputer(
+            strategy="constant", fill_value=fill_value, missing_values=2
+        )
+        X_trans = imputer.fit_transform(X_float32)
+        assert X_trans.dtype == X_float32.dtype
+
+
+@pytest.mark.parametrize("strategy", ["mean", "median", "most_frequent", "constant"])
+def test_iterative_imputer_no_empty_features(strategy):
+    """Check the behaviour of `keep_empty_features` with no empty features.
+
+    With no-empty features, we should get the same imputation whatever the
+    parameter `keep_empty_features`.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/29375
+    """
+    X = np.array([[np.nan, 0, 1], [2, np.nan, 3], [4, 5, np.nan]])
+
+    imputer_drop_empty_features = IterativeImputer(
+        initial_strategy=strategy, fill_value=1, keep_empty_features=False
+    )
+
+    imputer_keep_empty_features = IterativeImputer(
+        initial_strategy=strategy, fill_value=1, keep_empty_features=True
+    )
+
+    assert_allclose(
+        imputer_drop_empty_features.fit_transform(X),
+        imputer_keep_empty_features.fit_transform(X),
+    )
+
+
+@pytest.mark.parametrize("strategy", ["mean", "median", "most_frequent", "constant"])
+@pytest.mark.parametrize(
+    "X_test",
+    [
+        np.array([[1, 2, 3, 4], [5, 6, 7, 8]]),  # without empty feature
+        np.array([[np.nan, 2, 3, 4], [np.nan, 6, 7, 8]]),  # empty feature at column 0
+        np.array([[1, 2, 3, np.nan], [5, 6, 7, np.nan]]),  # empty feature at column 3
+    ],
+)
+def test_iterative_imputer_with_empty_features(strategy, X_test):
+    """Check the behaviour of `keep_empty_features` in the presence of empty features.
+
+    With `keep_empty_features=True`, the empty feature will be imputed with the value
+    defined by the initial imputation.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/29375
+    """
+    X_train = np.array(
+        [[np.nan, np.nan, 0, 1], [np.nan, 2, np.nan, 3], [np.nan, 4, 5, np.nan]]
+    )
+
+    imputer_drop_empty_features = IterativeImputer(
+        initial_strategy=strategy, fill_value=0, keep_empty_features=False
+    )
+    X_train_drop_empty_features = imputer_drop_empty_features.fit_transform(X_train)
+    X_test_drop_empty_features = imputer_drop_empty_features.transform(X_test)
+
+    imputer_keep_empty_features = IterativeImputer(
+        initial_strategy=strategy, fill_value=0, keep_empty_features=True
+    )
+    X_train_keep_empty_features = imputer_keep_empty_features.fit_transform(X_train)
+    X_test_keep_empty_features = imputer_keep_empty_features.transform(X_test)
+
+    assert_allclose(X_train_drop_empty_features, X_train_keep_empty_features[:, 1:])
+    assert_allclose(X_train_keep_empty_features[:, 0], 0)
+
+    assert X_train_drop_empty_features.shape[1] == X_test_drop_empty_features.shape[1]
+    assert X_train_keep_empty_features.shape[1] == X_test_keep_empty_features.shape[1]
diff --git a/sklearn/impute/tests/test_knn.py b/sklearn/impute/tests/test_knn.py
index 68c4d9f3cc54a..34244d628600f 100644
--- a/sklearn/impute/tests/test_knn.py
+++ b/sklearn/impute/tests/test_knn.py
@@ -3,8 +3,7 @@
 
 from sklearn import config_context
 from sklearn.impute import KNNImputer
-from sklearn.metrics.pairwise import nan_euclidean_distances
-from sklearn.metrics.pairwise import pairwise_distances
+from sklearn.metrics.pairwise import nan_euclidean_distances, pairwise_distances
 from sklearn.neighbors import KNeighborsRegressor
 from sklearn.utils._testing import assert_allclose
 
@@ -29,76 +28,77 @@ def test_knn_imputer_default_with_invalid_input(na):
     # Test imputation with default values and invalid input
 
     # Test with inf present
-    X = np.array([
-        [np.inf, 1, 1, 2, na],
-        [2, 1, 2, 2, 3],
-        [3, 2, 3, 3, 8],
-        [na, 6, 0, 5, 13],
-        [na, 7, 0, 7, 8],
-        [6, 6, 2, 5, 7],
-    ])
-    with pytest.raises(ValueError, match="Input contains (infinity|NaN)"):
+    X = np.array(
+        [
+            [np.inf, 1, 1, 2, na],
+            [2, 1, 2, 2, 3],
+            [3, 2, 3, 3, 8],
+            [na, 6, 0, 5, 13],
+            [na, 7, 0, 7, 8],
+            [6, 6, 2, 5, 7],
+        ]
+    )
+    with pytest.raises(ValueError, match="Input X contains (infinity|NaN)"):
         KNNImputer(missing_values=na).fit(X)
 
     # Test with inf present in matrix passed in transform()
-    X = np.array([
-        [np.inf, 1, 1, 2, na],
-        [2, 1, 2, 2, 3],
-        [3, 2, 3, 3, 8],
-        [na, 6, 0, 5, 13],
-        [na, 7, 0, 7, 8],
-        [6, 6, 2, 5, 7],
-    ])
-
-    X_fit = np.array([
-        [0, 1, 1, 2, na],
-        [2, 1, 2, 2, 3],
-        [3, 2, 3, 3, 8],
-        [na, 6, 0, 5, 13],
-        [na, 7, 0, 7, 8],
-        [6, 6, 2, 5, 7],
-    ])
+    X = np.array(
+        [
+            [np.inf, 1, 1, 2, na],
+            [2, 1, 2, 2, 3],
+            [3, 2, 3, 3, 8],
+            [na, 6, 0, 5, 13],
+            [na, 7, 0, 7, 8],
+            [6, 6, 2, 5, 7],
+        ]
+    )
+
+    X_fit = np.array(
+        [
+            [0, 1, 1, 2, na],
+            [2, 1, 2, 2, 3],
+            [3, 2, 3, 3, 8],
+            [na, 6, 0, 5, 13],
+            [na, 7, 0, 7, 8],
+            [6, 6, 2, 5, 7],
+        ]
+    )
     imputer = KNNImputer(missing_values=na).fit(X_fit)
-    with pytest.raises(ValueError, match="Input contains (infinity|NaN)"):
+    with pytest.raises(ValueError, match="Input X contains (infinity|NaN)"):
         imputer.transform(X)
 
-    # negative n_neighbors
-    with pytest.raises(ValueError, match="Expected n_neighbors > 0"):
-        KNNImputer(missing_values=na, n_neighbors=0).fit(X_fit)
-
     # Test with missing_values=0 when NaN present
     imputer = KNNImputer(missing_values=0, n_neighbors=2, weights="uniform")
-    X = np.array([
-        [np.nan, 0, 0, 0, 5],
-        [np.nan, 1, 0, np.nan, 3],
-        [np.nan, 2, 0, 0, 0],
-        [np.nan, 6, 0, 5, 13],
-    ])
-    msg = (r"Input contains NaN, infinity or a value too large for "
-           r"dtype\('float64'\)")
+    X = np.array(
+        [
+            [np.nan, 0, 0, 0, 5],
+            [np.nan, 1, 0, np.nan, 3],
+            [np.nan, 2, 0, 0, 0],
+            [np.nan, 6, 0, 5, 13],
+        ]
+    )
+    msg = "Input X contains NaN"
     with pytest.raises(ValueError, match=msg):
         imputer.fit(X)
 
-    X = np.array([
-        [0, 0],
-        [np.nan, 2],
-    ])
-
-    # Test with a metric type without NaN support
-    imputer = KNNImputer(metric="euclidean")
-    bad_metric_msg = "The selected metric does not support NaN values"
-    with pytest.raises(ValueError, match=bad_metric_msg):
-        imputer.fit(X)
+    X = np.array(
+        [
+            [0, 0],
+            [np.nan, 2],
+        ]
+    )
 
 
 @pytest.mark.parametrize("na", [np.nan, -1])
 def test_knn_imputer_removes_all_na_features(na):
-    X = np.array([
-        [1, 1, na, 1, 1, 1.],
-        [2, 3, na, 2, 2, 2],
-        [3, 4, na, 3, 3, na],
-        [6, 4, na, na, 6, 6],
-    ])
+    X = np.array(
+        [
+            [1, 1, na, 1, 1, 1.0],
+            [2, 3, na, 2, 2, 2],
+            [3, 4, na, 3, 3, na],
+            [6, 4, na, na, 6, 6],
+        ]
+    )
     knn = KNNImputer(missing_values=na, n_neighbors=2).fit(X)
 
     X_transform = knn.transform(X)
@@ -113,115 +113,112 @@ def test_knn_imputer_removes_all_na_features(na):
 @pytest.mark.parametrize("na", [np.nan, -1])
 def test_knn_imputer_zero_nan_imputes_the_same(na):
     # Test with an imputable matrix and compare with different missing_values
-    X_zero = np.array([
-        [1, 0, 1, 1, 1.],
-        [2, 2, 2, 2, 2],
-        [3, 3, 3, 3, 0],
-        [6, 6, 0, 6, 6],
-    ])
-
-    X_nan = np.array([
-        [1, na, 1, 1, 1.],
-        [2, 2, 2, 2, 2],
-        [3, 3, 3, 3, na],
-        [6, 6, na, 6, 6],
-    ])
-
-    X_imputed = np.array([
-        [1, 2.5, 1, 1, 1.],
-        [2, 2, 2, 2, 2],
-        [3, 3, 3, 3, 1.5],
-        [6, 6, 2.5, 6, 6],
-    ])
-
-    imputer_zero = KNNImputer(missing_values=0, n_neighbors=2,
-                              weights="uniform")
-
-    imputer_nan = KNNImputer(missing_values=na, n_neighbors=2,
-                             weights="uniform")
+    X_zero = np.array(
+        [
+            [1, 0, 1, 1, 1.0],
+            [2, 2, 2, 2, 2],
+            [3, 3, 3, 3, 0],
+            [6, 6, 0, 6, 6],
+        ]
+    )
+
+    X_nan = np.array(
+        [
+            [1, na, 1, 1, 1.0],
+            [2, 2, 2, 2, 2],
+            [3, 3, 3, 3, na],
+            [6, 6, na, 6, 6],
+        ]
+    )
+
+    X_imputed = np.array(
+        [
+            [1, 2.5, 1, 1, 1.0],
+            [2, 2, 2, 2, 2],
+            [3, 3, 3, 3, 1.5],
+            [6, 6, 2.5, 6, 6],
+        ]
+    )
+
+    imputer_zero = KNNImputer(missing_values=0, n_neighbors=2, weights="uniform")
+
+    imputer_nan = KNNImputer(missing_values=na, n_neighbors=2, weights="uniform")
 
     assert_allclose(imputer_zero.fit_transform(X_zero), X_imputed)
-    assert_allclose(imputer_zero.fit_transform(X_zero),
-                    imputer_nan.fit_transform(X_nan))
+    assert_allclose(
+        imputer_zero.fit_transform(X_zero), imputer_nan.fit_transform(X_nan)
+    )
 
 
 @pytest.mark.parametrize("na", [np.nan, -1])
 def test_knn_imputer_verify(na):
     # Test with an imputable matrix
-    X = np.array([
-        [1, 0, 0, 1],
-        [2, 1, 2, na],
-        [3, 2, 3, na],
-        [na, 4, 5, 5],
-        [6, na, 6, 7],
-        [8, 8, 8, 8],
-        [16, 15, 18, 19],
-    ])
-
-    X_imputed = np.array([
-        [1, 0, 0, 1],
-        [2, 1, 2, 8],
-        [3, 2, 3, 8],
-        [4, 4, 5, 5],
-        [6, 3, 6, 7],
-        [8, 8, 8, 8],
-        [16, 15, 18, 19],
-    ])
+    X = np.array(
+        [
+            [1, 0, 0, 1],
+            [2, 1, 2, na],
+            [3, 2, 3, na],
+            [na, 4, 5, 5],
+            [6, na, 6, 7],
+            [8, 8, 8, 8],
+            [16, 15, 18, 19],
+        ]
+    )
+
+    X_imputed = np.array(
+        [
+            [1, 0, 0, 1],
+            [2, 1, 2, 8],
+            [3, 2, 3, 8],
+            [4, 4, 5, 5],
+            [6, 3, 6, 7],
+            [8, 8, 8, 8],
+            [16, 15, 18, 19],
+        ]
+    )
 
     imputer = KNNImputer(missing_values=na)
     assert_allclose(imputer.fit_transform(X), X_imputed)
 
     # Test when there is not enough neighbors
-    X = np.array([
-        [1, 0, 0, na],
-        [2, 1, 2, na],
-        [3, 2, 3, na],
-        [4, 4, 5, na],
-        [6, 7, 6, na],
-        [8, 8, 8, na],
-        [20, 20, 20, 20],
-        [22, 22, 22, 22]
-    ])
+    X = np.array(
+        [
+            [1, 0, 0, na],
+            [2, 1, 2, na],
+            [3, 2, 3, na],
+            [4, 4, 5, na],
+            [6, 7, 6, na],
+            [8, 8, 8, na],
+            [20, 20, 20, 20],
+            [22, 22, 22, 22],
+        ]
+    )
 
     # Not enough neighbors, use column mean from training
     X_impute_value = (20 + 22) / 2
-    X_imputed = np.array([
-        [1, 0, 0, X_impute_value],
-        [2, 1, 2, X_impute_value],
-        [3, 2, 3, X_impute_value],
-        [4, 4, 5, X_impute_value],
-        [6, 7, 6, X_impute_value],
-        [8, 8, 8, X_impute_value],
-        [20, 20, 20, 20],
-        [22, 22, 22, 22]
-    ])
+    X_imputed = np.array(
+        [
+            [1, 0, 0, X_impute_value],
+            [2, 1, 2, X_impute_value],
+            [3, 2, 3, X_impute_value],
+            [4, 4, 5, X_impute_value],
+            [6, 7, 6, X_impute_value],
+            [8, 8, 8, X_impute_value],
+            [20, 20, 20, 20],
+            [22, 22, 22, 22],
+        ]
+    )
 
     imputer = KNNImputer(missing_values=na)
     assert_allclose(imputer.fit_transform(X), X_imputed)
 
     # Test when data in fit() and transform() are different
-    X = np.array([
-        [0, 0],
-        [na, 2],
-        [4, 3],
-        [5, 6],
-        [7, 7],
-        [9, 8],
-        [11, 16]
-    ])
-
-    X1 = np.array([
-        [1, 0],
-        [3, 2],
-        [4, na]
-    ])
+    X = np.array([[0, 0], [na, 2], [4, 3], [5, 6], [7, 7], [9, 8], [11, 16]])
+
+    X1 = np.array([[1, 0], [3, 2], [4, na]])
 
     X_2_1 = (0 + 3 + 6 + 7 + 8) / 5
-    X1_imputed = np.array([
-        [1, 0],
-        [3, 2],
-        [4, X_2_1]
-    ])
+    X1_imputed = np.array([[1, 0], [3, 2], [4, X_2_1]])
 
     imputer = KNNImputer(missing_values=na)
     assert_allclose(imputer.fit(X).transform(X1), X1_imputed)
@@ -229,26 +226,9 @@ def test_knn_imputer_verify(na):
 
 @pytest.mark.parametrize("na", [np.nan, -1])
 def test_knn_imputer_one_n_neighbors(na):
+    X = np.array([[0, 0], [na, 2], [4, 3], [5, na], [7, 7], [na, 8], [14, 13]])
 
-    X = np.array([
-        [0, 0],
-        [na, 2],
-        [4, 3],
-        [5, na],
-        [7, 7],
-        [na, 8],
-        [14, 13]
-    ])
-
-    X_imputed = np.array([
-        [0, 0],
-        [4, 2],
-        [4, 3],
-        [5, 3],
-        [7, 7],
-        [7, 8],
-        [14, 13]
-    ])
+    X_imputed = np.array([[0, 0], [4, 2], [4, 3], [5, 3], [7, 7], [7, 8], [14, 13]])
 
     imputer = KNNImputer(n_neighbors=1, missing_values=na)
 
@@ -257,25 +237,11 @@ def test_knn_imputer_one_n_neighbors(na):
 
 @pytest.mark.parametrize("na", [np.nan, -1])
 def test_knn_imputer_all_samples_are_neighbors(na):
-    X = np.array([
-        [0, 0],
-        [na, 2],
-        [4, 3],
-        [5, na],
-        [7, 7],
-        [na, 8],
-        [14, 13]
-    ])
-
-    X_imputed = np.array([
-        [0, 0],
-        [6, 2],
-        [4, 3],
-        [5, 5.5],
-        [7, 7],
-        [6, 8],
-        [14, 13]
-    ])
+    X = np.array([[0, 0], [na, 2], [4, 3], [5, na], [7, 7], [na, 8], [14, 13]])
+
+    X_imputed = np.array(
+        [[0, 0], [6.25, 2], [4, 3], [5, 5.75], [7, 7], [6.25, 8], [14, 13]]
+    )
 
     n_neighbors = X.shape[0] - 1
     imputer = KNNImputer(n_neighbors=n_neighbors, missing_values=na)
@@ -289,27 +255,12 @@ def test_knn_imputer_all_samples_are_neighbors(na):
 
 @pytest.mark.parametrize("na", [np.nan, -1])
 def test_knn_imputer_weight_uniform(na):
-
-    X = np.array([
-        [0, 0],
-        [na, 2],
-        [4, 3],
-        [5, 6],
-        [7, 7],
-        [9, 8],
-        [11, 10]
-    ])
+    X = np.array([[0, 0], [na, 2], [4, 3], [5, 6], [7, 7], [9, 8], [11, 10]])
 
     # Test with "uniform" weight (or unweighted)
-    X_imputed_uniform = np.array([
-        [0, 0],
-        [5, 2],
-        [4, 3],
-        [5, 6],
-        [7, 7],
-        [9, 8],
-        [11, 10]
-    ])
+    X_imputed_uniform = np.array(
+        [[0, 0], [5, 2], [4, 3], [5, 6], [7, 7], [9, 8], [11, 10]]
+    )
 
     imputer = KNNImputer(weights="uniform", missing_values=na)
     assert_allclose(imputer.fit_transform(X), X_imputed_uniform)
@@ -331,15 +282,7 @@ def uniform_weight(dist):
 
 @pytest.mark.parametrize("na", [np.nan, -1])
 def test_knn_imputer_weight_distance(na):
-    X = np.array([
-        [0, 0],
-        [na, 2],
-        [4, 3],
-        [5, 6],
-        [7, 7],
-        [9, 8],
-        [11, 10]
-    ])
+    X = np.array([[0, 0], [na, 2], [4, 3], [5, 6], [7, 7], [9, 8], [11, 10]])
 
     # Test with "distance" weight
     nn = KNeighborsRegressor(metric="euclidean", weights="distance")
@@ -353,64 +296,58 @@ def test_knn_imputer_weight_distance(na):
     weights = 1 / dist[:, X_neighbors_idx].ravel()
     manual_imputed_value = np.average(X[X_neighbors_idx, 0], weights=weights)
 
-    X_imputed_distance1 = np.array([
-        [0, 0],
-        [manual_imputed_value, 2],
-        [4, 3],
-        [5, 6],
-        [7, 7],
-        [9, 8],
-        [11, 10]
-    ])
+    X_imputed_distance1 = np.array(
+        [[0, 0], [manual_imputed_value, 2], [4, 3], [5, 6], [7, 7], [9, 8], [11, 10]]
+    )
 
     # NearestNeighbor calculation
-    X_imputed_distance2 = np.array([
-        [0, 0],
-        [knn_imputed_value, 2],
-        [4, 3],
-        [5, 6],
-        [7, 7],
-        [9, 8],
-        [11, 10]
-    ])
+    X_imputed_distance2 = np.array(
+        [[0, 0], [knn_imputed_value, 2], [4, 3], [5, 6], [7, 7], [9, 8], [11, 10]]
+    )
 
     imputer = KNNImputer(weights="distance", missing_values=na)
     assert_allclose(imputer.fit_transform(X), X_imputed_distance1)
     assert_allclose(imputer.fit_transform(X), X_imputed_distance2)
 
     # Test with weights = "distance" and n_neighbors=2
-    X = np.array([
-        [na, 0, 0],
-        [2, 1, 2],
-        [3, 2, 3],
-        [4, 5, 5],
-    ])
+    X = np.array(
+        [
+            [na, 0, 0],
+            [2, 1, 2],
+            [3, 2, 3],
+            [4, 5, 5],
+        ]
+    )
 
     # neighbors are rows 1, 2, the nan_euclidean_distances are:
-    dist_0_1 = np.sqrt((3/2)*((1 - 0)**2 + (2 - 0)**2))
-    dist_0_2 = np.sqrt((3/2)*((2 - 0)**2 + (3 - 0)**2))
+    dist_0_1 = np.sqrt((3 / 2) * ((1 - 0) ** 2 + (2 - 0) ** 2))
+    dist_0_2 = np.sqrt((3 / 2) * ((2 - 0) ** 2 + (3 - 0) ** 2))
     imputed_value = np.average([2, 3], weights=[1 / dist_0_1, 1 / dist_0_2])
 
-    X_imputed = np.array([
-        [imputed_value, 0, 0],
-        [2, 1, 2],
-        [3, 2, 3],
-        [4, 5, 5],
-    ])
+    X_imputed = np.array(
+        [
+            [imputed_value, 0, 0],
+            [2, 1, 2],
+            [3, 2, 3],
+            [4, 5, 5],
+        ]
+    )
 
     imputer = KNNImputer(n_neighbors=2, weights="distance", missing_values=na)
     assert_allclose(imputer.fit_transform(X), X_imputed)
 
     # Test with varying missingness patterns
-    X = np.array([
-        [1, 0, 0, 1],
-        [0, na, 1, na],
-        [1, 1, 1, na],
-        [0, 1, 0, 0],
-        [0, 0, 0, 0],
-        [1, 0, 1, 1],
-        [10, 10, 10, 10],
-    ])
+    X = np.array(
+        [
+            [1, 0, 0, 1],
+            [0, na, 1, na],
+            [1, 1, 1, na],
+            [0, 1, 0, 0],
+            [0, 0, 0, 0],
+            [1, 0, 1, 1],
+            [10, 10, 10, 10],
+        ]
+    )
 
     # Get weights of donor neighbors
     dist = nan_euclidean_distances(X, missing_values=na)
@@ -431,32 +368,37 @@ def test_knn_imputer_weight_distance(na):
     r1c3_imp = np.ma.average(col3_donor_values, weights=r1c3_nbor_wt)
     r2c3_imp = np.ma.average(col3_donor_values, weights=r2c3_nbor_wt)
 
-    X_imputed = np.array([
-        [1, 0, 0, 1],
-        [0, r1c1_imp, 1, r1c3_imp],
-        [1, 1, 1, r2c3_imp],
-        [0, 1, 0, 0],
-        [0, 0, 0, 0],
-        [1, 0, 1, 1],
-        [10, 10, 10, 10],
-    ])
+    X_imputed = np.array(
+        [
+            [1, 0, 0, 1],
+            [0, r1c1_imp, 1, r1c3_imp],
+            [1, 1, 1, r2c3_imp],
+            [0, 1, 0, 0],
+            [0, 0, 0, 0],
+            [1, 0, 1, 1],
+            [10, 10, 10, 10],
+        ]
+    )
 
     imputer = KNNImputer(weights="distance", missing_values=na)
     assert_allclose(imputer.fit_transform(X), X_imputed)
 
-    X = np.array([
-        [0, 0, 0, na],
-        [1, 1, 1, na],
-        [2, 2, na, 2],
-        [3, 3, 3, 3],
-        [4, 4, 4, 4],
-        [5, 5, 5, 5],
-        [6, 6, 6, 6],
-        [na, 7, 7, 7]
-    ])
-
-    dist = pairwise_distances(X, metric="nan_euclidean", squared=False,
-                              missing_values=na)
+    X = np.array(
+        [
+            [0, 0, 0, na],
+            [1, 1, 1, na],
+            [2, 2, na, 2],
+            [3, 3, 3, 3],
+            [4, 4, 4, 4],
+            [5, 5, 5, 5],
+            [6, 6, 6, 6],
+            [na, 7, 7, 7],
+        ]
+    )
+
+    dist = pairwise_distances(
+        X, metric="nan_euclidean", squared=False, missing_values=na
+    )
 
     # Calculate weights
     r0c3_w = 1.0 / dist[0, 2:-1]
@@ -470,45 +412,38 @@ def test_knn_imputer_weight_distance(na):
     r2c2 = np.average(X[(0, 1, 3, 4, 5), 2], weights=r2c2_w)
     r7c0 = np.average(X[2:7, 0], weights=r7c0_w)
 
-    X_imputed = np.array([
-        [0, 0, 0, r0c3],
-        [1, 1, 1, r1c3],
-        [2, 2, r2c2, 2],
-        [3, 3, 3, 3],
-        [4, 4, 4, 4],
-        [5, 5, 5, 5],
-        [6, 6, 6, 6],
-        [r7c0, 7, 7, 7]
-    ])
+    X_imputed = np.array(
+        [
+            [0, 0, 0, r0c3],
+            [1, 1, 1, r1c3],
+            [2, 2, r2c2, 2],
+            [3, 3, 3, 3],
+            [4, 4, 4, 4],
+            [5, 5, 5, 5],
+            [6, 6, 6, 6],
+            [r7c0, 7, 7, 7],
+        ]
+    )
 
     imputer_comp_wt = KNNImputer(missing_values=na, weights="distance")
     assert_allclose(imputer_comp_wt.fit_transform(X), X_imputed)
 
 
 def test_knn_imputer_callable_metric():
-
     # Define callable metric that returns the l1 norm:
     def custom_callable(x, y, missing_values=np.nan, squared=False):
         x = np.ma.array(x, mask=np.isnan(x))
         y = np.ma.array(y, mask=np.isnan(y))
-        dist = np.nansum(np.abs(x-y))
+        dist = np.nansum(np.abs(x - y))
         return dist
 
-    X = np.array([
-        [4, 3, 3, np.nan],
-        [6, 9, 6, 9],
-        [4, 8, 6, 9],
-        [np.nan, 9, 11, 10.]
-    ])
+    X = np.array([[4, 3, 3, np.nan], [6, 9, 6, 9], [4, 8, 6, 9], [np.nan, 9, 11, 10.0]])
 
     X_0_3 = (9 + 9) / 2
     X_3_0 = (6 + 4) / 2
-    X_imputed = np.array([
-        [4, 3, 3, X_0_3],
-        [6, 9, 6, 9],
-        [4, 8, 6, 9],
-        [X_3_0, 9, 11, 10.]
-    ])
+    X_imputed = np.array(
+        [[4, 3, 3, X_0_3], [6, 9, 6, 9], [4, 8, 6, 9], [X_3_0, 9, 11, 10.0]]
+    )
 
     imputer = KNNImputer(n_neighbors=2, metric=custom_callable)
     assert_allclose(imputer.fit_transform(X), X_imputed)
@@ -520,17 +455,18 @@ def custom_callable(x, y, missing_values=np.nan, squared=False):
 # for a small dataset. However, it should raise a UserWarning that we ignore.
 @pytest.mark.filterwarnings("ignore:adhere to working_memory")
 def test_knn_imputer_with_simple_example(na, working_memory):
-
-    X = np.array([
-        [0, na, 0, na],
-        [1, 1, 1, na],
-        [2, 2, na,  2],
-        [3, 3, 3, 3],
-        [4, 4, 4, 4],
-        [5, 5, 5, 5],
-        [6, 6, 6, 6],
-        [na, 7, 7, 7]
-    ])
+    X = np.array(
+        [
+            [0, na, 0, na],
+            [1, 1, 1, na],
+            [2, 2, na, 2],
+            [3, 3, 3, 3],
+            [4, 4, 4, 4],
+            [5, 5, 5, 5],
+            [6, 6, 6, 6],
+            [na, 7, 7, 7],
+        ]
+    )
 
     r0c1 = np.mean(X[1:6, 1])
     r0c3 = np.mean(X[2:-1, -1])
@@ -538,16 +474,18 @@ def test_knn_imputer_with_simple_example(na, working_memory):
     r2c2 = np.mean(X[[0, 1, 3, 4, 5], 2])
     r7c0 = np.mean(X[2:-1, 0])
 
-    X_imputed = np.array([
-        [0, r0c1, 0, r0c3],
-        [1, 1, 1, r1c3],
-        [2, 2, r2c2, 2],
-        [3, 3, 3, 3],
-        [4, 4, 4, 4],
-        [5, 5, 5, 5],
-        [6, 6, 6, 6],
-        [r7c0, 7, 7, 7]
-    ])
+    X_imputed = np.array(
+        [
+            [0, r0c1, 0, r0c3],
+            [1, 1, 1, r1c3],
+            [2, 2, r2c2, 2],
+            [3, 3, 3, 3],
+            [4, 4, 4, 4],
+            [5, 5, 5, 5],
+            [6, 6, 6, 6],
+            [r7c0, 7, 7, 7],
+        ]
+    )
 
     with config_context(working_memory=working_memory):
         imputer_comp = KNNImputer(missing_values=na)
@@ -555,19 +493,11 @@ def test_knn_imputer_with_simple_example(na, working_memory):
 
 
 @pytest.mark.parametrize("na", [-1, np.nan])
-@pytest.mark.parametrize("weights", ['uniform', 'distance'])
+@pytest.mark.parametrize("weights", ["uniform", "distance"])
 def test_knn_imputer_not_enough_valid_distances(na, weights):
     # Samples with needed feature has nan distance
-    X1 = np.array([
-        [na, 11],
-        [na, 1],
-        [3, na]
-    ])
-    X1_imputed = np.array([
-        [3, 11],
-        [3, 1],
-        [3, 6]
-    ])
+    X1 = np.array([[na, 11], [na, 1], [3, na]])
+    X1_imputed = np.array([[3, 11], [3, 1], [3, 6]])
 
     knn = KNNImputer(missing_values=na, n_neighbors=1, weights=weights)
     assert_allclose(knn.fit_transform(X1), X1_imputed)
@@ -577,65 +507,64 @@ def test_knn_imputer_not_enough_valid_distances(na, weights):
     assert_allclose(knn.transform(X2), X2_imputed)
 
 
+@pytest.mark.parametrize("na", [-1, np.nan])
+@pytest.mark.parametrize("weights", ["uniform", "distance"])
+def test_knn_imputer_nan_distance(na, weights):
+    # Samples with nan distance should be excluded from the mean computation
+    X1_train = np.array([[1, 1], [na, 2]])
+    X1_test = np.array([[0, na]])
+    X1_test_expected = np.array([[0, 1]])
+
+    knn1 = KNNImputer(n_neighbors=2, missing_values=na, weights=weights)
+    knn1.fit(X1_train)
+    assert_allclose(knn1.transform(X1_test), X1_test_expected)
+
+    X2_train = np.array([[na, 1, 1], [2, na, 2], [3, 3, na]])
+    X2_test = np.array([[na, 0, na], [0, na, na], [na, na, 0]])
+    X2_test_expected = np.array([[3, 0, 1], [0, 3, 2], [2, 1, 0]])
+
+    knn2 = KNNImputer(n_neighbors=2, missing_values=na, weights=weights)
+    knn2.fit(X2_train)
+    assert_allclose(knn2.transform(X2_test), X2_test_expected)
+
+
 @pytest.mark.parametrize("na", [-1, np.nan])
 def test_knn_imputer_drops_all_nan_features(na):
-    X1 = np.array([
-        [na, 1],
-        [na, 2]
-    ])
+    X1 = np.array([[na, 1], [na, 2]])
     knn = KNNImputer(missing_values=na, n_neighbors=1)
     X1_expected = np.array([[1], [2]])
     assert_allclose(knn.fit_transform(X1), X1_expected)
 
-    X2 = np.array([
-        [1, 2],
-        [3, na]
-    ])
+    X2 = np.array([[1, 2], [3, na]])
     X2_expected = np.array([[2], [1.5]])
     assert_allclose(knn.transform(X2), X2_expected)
 
 
 @pytest.mark.parametrize("working_memory", [None, 0])
 @pytest.mark.parametrize("na", [-1, np.nan])
-def test_knn_imputer_distance_weighted_not_enough_neighbors(na,
-                                                            working_memory):
-    X = np.array([
-        [3, na],
-        [2, na],
-        [na, 4],
-        [5, 6],
-        [6, 8],
-        [na, 5]
-    ])
-
-    dist = pairwise_distances(X, metric="nan_euclidean", squared=False,
-                              missing_values=na)
-
-    X_01 = np.average(X[3:5, 1], weights=1/dist[0, 3:5])
-    X_11 = np.average(X[3:5, 1], weights=1/dist[1, 3:5])
-    X_20 = np.average(X[3:5, 0], weights=1/dist[2, 3:5])
-    X_50 = np.average(X[3:5, 0], weights=1/dist[5, 3:5])
-
-    X_expected = np.array([
-        [3, X_01],
-        [2, X_11],
-        [X_20, 4],
-        [5, 6],
-        [6, 8],
-        [X_50, 5]
-    ])
+def test_knn_imputer_distance_weighted_not_enough_neighbors(na, working_memory):
+    X = np.array([[3, na], [2, na], [na, 4], [5, 6], [6, 8], [na, 5]])
+
+    dist = pairwise_distances(
+        X, metric="nan_euclidean", squared=False, missing_values=na
+    )
+
+    X_01 = np.average(X[3:5, 1], weights=1 / dist[0, 3:5])
+    X_11 = np.average(X[3:5, 1], weights=1 / dist[1, 3:5])
+    X_20 = np.average(X[3:5, 0], weights=1 / dist[2, 3:5])
+    X_50 = np.average(X[3:5, 0], weights=1 / dist[5, 3:5])
+
+    X_expected = np.array([[3, X_01], [2, X_11], [X_20, 4], [5, 6], [6, 8], [X_50, 5]])
 
     with config_context(working_memory=working_memory):
-        knn_3 = KNNImputer(missing_values=na, n_neighbors=3,
-                           weights='distance')
+        knn_3 = KNNImputer(missing_values=na, n_neighbors=3, weights="distance")
         assert_allclose(knn_3.fit_transform(X), X_expected)
 
-        knn_4 = KNNImputer(missing_values=na, n_neighbors=4,
-                           weights='distance')
+        knn_4 = KNNImputer(missing_values=na, n_neighbors=4, weights="distance")
         assert_allclose(knn_4.fit_transform(X), X_expected)
 
 
 @pytest.mark.parametrize("na, allow_nan", [(-1, False), (np.nan, True)])
 def test_knn_tags(na, allow_nan):
     knn = KNNImputer(missing_values=na)
-    assert knn._get_tags()["allow_nan"] == allow_nan
+    assert knn.__sklearn_tags__().input_tags.allow_nan == allow_nan
diff --git a/sklearn/inspection/__init__.py b/sklearn/inspection/__init__.py
index e3b5bbe69a2ae..8e0a1125ef041 100644
--- a/sklearn/inspection/__init__.py
+++ b/sklearn/inspection/__init__.py
@@ -1,16 +1,16 @@
-"""The :mod:`sklearn.inspection` module includes tools for model inspection."""
+"""Tools for model inspection."""
 
-
-from ._permutation_importance import permutation_importance
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 from ._partial_dependence import partial_dependence
-from ._plot.partial_dependence import plot_partial_dependence
+from ._permutation_importance import permutation_importance
+from ._plot.decision_boundary import DecisionBoundaryDisplay
 from ._plot.partial_dependence import PartialDependenceDisplay
 
-
 __all__ = [
-    'partial_dependence',
-    'plot_partial_dependence',
-    'permutation_importance',
-    'PartialDependenceDisplay'
+    "DecisionBoundaryDisplay",
+    "PartialDependenceDisplay",
+    "partial_dependence",
+    "permutation_importance",
 ]
diff --git a/sklearn/inspection/_partial_dependence.py b/sklearn/inspection/_partial_dependence.py
index 0736130f41524..ad352c45cc03b 100644
--- a/sklearn/inspection/_partial_dependence.py
+++ b/sklearn/inspection/_partial_dependence.py
@@ -1,112 +1,212 @@
 """Partial dependence plots for regression and classification models."""
 
-# Authors: Peter Prettenhofer
-#          Trevor Stephens
-#          Nicolas Hug
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-from collections.abc import Iterable
 import warnings
+from collections.abc import Iterable
 
 import numpy as np
 from scipy import sparse
 from scipy.stats.mstats import mquantiles
 
 from ..base import is_classifier, is_regressor
-from ..pipeline import Pipeline
-from ..utils.extmath import cartesian
-from ..utils import check_array
-from ..utils import check_matplotlib_support  # noqa
-from ..utils import _safe_indexing
-from ..utils import _determine_key_type
-from ..utils import _get_column_indices
-from ..utils.validation import check_is_fitted
-from ..utils import Bunch
-from ..utils.validation import _deprecate_positional_args
-from ..tree import DecisionTreeRegressor
 from ..ensemble import RandomForestRegressor
-from ..exceptions import NotFittedError
 from ..ensemble._gb import BaseGradientBoosting
 from ..ensemble._hist_gradient_boosting.gradient_boosting import (
-    BaseHistGradientBoosting)
-
+    BaseHistGradientBoosting,
+)
+from ..tree import DecisionTreeRegressor
+from ..utils import Bunch, _safe_indexing, check_array
+from ..utils._indexing import _determine_key_type, _get_column_indices, _safe_assign
+from ..utils._optional_dependencies import check_matplotlib_support  # noqa: F401
+from ..utils._param_validation import (
+    HasMethods,
+    Integral,
+    Interval,
+    StrOptions,
+    validate_params,
+)
+from ..utils._response import _get_response_values
+from ..utils.extmath import cartesian
+from ..utils.validation import _check_sample_weight, check_is_fitted
+from ._pd_utils import _check_feature_names, _get_feature_index
 
 __all__ = [
-    'partial_dependence',
+    "partial_dependence",
 ]
 
 
-def _grid_from_X(X, percentiles, grid_resolution):
+def _grid_from_X(X, percentiles, is_categorical, grid_resolution, custom_values):
     """Generate a grid of points based on the percentiles of X.
 
     The grid is a cartesian product between the columns of ``values``. The
     ith column of ``values`` consists in ``grid_resolution`` equally-spaced
     points between the percentiles of the jth column of X.
+
     If ``grid_resolution`` is bigger than the number of unique values in the
-    jth column of X, then those unique values will be used instead.
+    j-th column of X or if the feature is a categorical feature (by inspecting
+    `is_categorical`) , then those unique values will be used instead.
 
     Parameters
     ----------
-    X : ndarray, shape (n_samples, n_target_features)
+    X : array-like of shape (n_samples, n_target_features)
         The data.
 
-    percentiles : tuple of floats
+    percentiles : tuple of float
         The percentiles which are used to construct the extreme values of
         the grid. Must be in [0, 1].
 
+    is_categorical : list of bool
+        For each feature, tells whether it is categorical or not. If a feature
+        is categorical, then the values used will be the unique ones
+        (i.e. categories) instead of the percentiles.
+
     grid_resolution : int
         The number of equally spaced points to be placed on the grid for each
         feature.
 
+    custom_values: dict
+        Mapping from column index of X to an array-like of values where
+        the partial dependence should be calculated for that feature
+
     Returns
     -------
-    grid : ndarray, shape (n_points, n_target_features)
+    grid : ndarray of shape (n_points, n_target_features)
         A value for each feature at each point in the grid. ``n_points`` is
         always ``<= grid_resolution ** X.shape[1]``.
 
     values : list of 1d ndarrays
         The values with which the grid has been created. The size of each
-        array ``values[j]`` is either ``grid_resolution``, or the number of
-        unique values in ``X[:, j]``, whichever is smaller.
+        array ``values[j]`` is either ``grid_resolution``, the number of
+        unique values in ``X[:, j]``, if j is not in ``custom_range``.
+        If j is in ``custom_range``, then it is the length of ``custom_range[j]``.
     """
     if not isinstance(percentiles, Iterable) or len(percentiles) != 2:
         raise ValueError("'percentiles' must be a sequence of 2 elements.")
     if not all(0 <= x <= 1 for x in percentiles):
         raise ValueError("'percentiles' values must be in [0, 1].")
     if percentiles[0] >= percentiles[1]:
-        raise ValueError('percentiles[0] must be strictly less '
-                         'than percentiles[1].')
+        raise ValueError("percentiles[0] must be strictly less than percentiles[1].")
 
     if grid_resolution <= 1:
         raise ValueError("'grid_resolution' must be strictly greater than 1.")
 
+    def _convert_custom_values(values):
+        # Convert custom types such that object types are always used for string arrays
+        dtype = object if any(isinstance(v, str) for v in values) else None
+        return np.asarray(values, dtype=dtype)
+
+    custom_values = {k: _convert_custom_values(v) for k, v in custom_values.items()}
+    if any(v.ndim != 1 for v in custom_values.values()):
+        error_string = ", ".join(
+            f"Feature {k}: {v.ndim} dimensions"
+            for k, v in custom_values.items()
+            if v.ndim != 1
+        )
+
+        raise ValueError(
+            "The custom grid for some features is not a one-dimensional array. "
+            f"{error_string}"
+        )
+
     values = []
-    for feature in range(X.shape[1]):
-        uniques = np.unique(_safe_indexing(X, feature, axis=1))
-        if uniques.shape[0] < grid_resolution:
-            # feature has low resolution use unique vals
-            axis = uniques
+    # TODO: we should handle missing values (i.e. `np.nan`) specifically and store them
+    # in a different Bunch attribute.
+    for feature, is_cat in enumerate(is_categorical):
+        if feature in custom_values:
+            # Use values in the custom range
+            axis = custom_values[feature]
         else:
-            # create axis based on percentiles and grid resolution
-            emp_percentiles = mquantiles(
-                _safe_indexing(X, feature, axis=1), prob=percentiles, axis=0
-            )
-            if np.allclose(emp_percentiles[0], emp_percentiles[1]):
+            try:
+                uniques = np.unique(_safe_indexing(X, feature, axis=1))
+            except TypeError as exc:
+                # `np.unique` will fail in the presence of `np.nan` and `str` categories
+                # due to sorting. Temporary, we reraise an error explaining the problem.
                 raise ValueError(
-                    'percentiles are too close to each other, '
-                    'unable to build the grid. Please choose percentiles '
-                    'that are further apart.')
-            axis = np.linspace(emp_percentiles[0],
-                               emp_percentiles[1],
-                               num=grid_resolution, endpoint=True)
+                    f"The column #{feature} contains mixed data types. Finding unique "
+                    "categories fail due to sorting. It usually means that the column "
+                    "contains `np.nan` values together with `str` categories. Such use "
+                    "case is not yet supported in scikit-learn."
+                ) from exc
+
+            if is_cat or uniques.shape[0] < grid_resolution:
+                # Use the unique values either because:
+                # - feature has low resolution use unique values
+                # - feature is categorical
+                axis = uniques
+            else:
+                # create axis based on percentiles and grid resolution
+                emp_percentiles = mquantiles(
+                    _safe_indexing(X, feature, axis=1), prob=percentiles, axis=0
+                )
+                if np.allclose(emp_percentiles[0], emp_percentiles[1]):
+                    raise ValueError(
+                        "percentiles are too close to each other, "
+                        "unable to build the grid. Please choose percentiles "
+                        "that are further apart."
+                    )
+                axis = np.linspace(
+                    emp_percentiles[0],
+                    emp_percentiles[1],
+                    num=grid_resolution,
+                    endpoint=True,
+                )
         values.append(axis)
 
     return cartesian(values), values
 
 
 def _partial_dependence_recursion(est, grid, features):
-    averaged_predictions = est._compute_partial_dependence_recursion(grid,
-                                                                     features)
+    """Calculate partial dependence via the recursion method.
+
+    The recursion method is in particular enabled for tree-based estimators.
+
+    For each `grid` value, a weighted tree traversal is performed: if a split node
+    involves an input feature of interest, the corresponding left or right branch
+    is followed; otherwise both branches are followed, each branch being weighted
+    by the fraction of training samples that entered that branch. Finally, the
+    partial dependence is given by a weighted average of all the visited leaves
+    values.
+
+    This method is more efficient in terms of speed than the `'brute'` method
+    (:func:`~sklearn.inspection._partial_dependence._partial_dependence_brute`).
+    However, here, the partial dependence computation is done explicitly with the
+    `X` used during training of `est`.
+
+    Parameters
+    ----------
+    est : BaseEstimator
+        A fitted estimator object implementing :term:`predict` or
+        :term:`decision_function`. Multioutput-multiclass classifiers are not
+        supported. Note that `'recursion'` is only supported for some tree-based
+        estimators (namely
+        :class:`~sklearn.ensemble.GradientBoostingClassifier`,
+        :class:`~sklearn.ensemble.GradientBoostingRegressor`,
+        :class:`~sklearn.ensemble.HistGradientBoostingClassifier`,
+        :class:`~sklearn.ensemble.HistGradientBoostingRegressor`,
+        :class:`~sklearn.tree.DecisionTreeRegressor`,
+        :class:`~sklearn.ensemble.RandomForestRegressor`,
+        ).
+
+    grid : array-like of shape (n_points, n_target_features)
+        The grid of feature values for which the partial dependence is calculated.
+        Note that `n_points` is the number of points in the grid and `n_target_features`
+        is the number of features you are doing partial dependence at.
+
+    features : array-like of {int, str}
+        The feature (e.g. `[0]`) or pair of interacting features
+        (e.g. `[(0, 1)]`) for which the partial dependency should be computed.
+
+    Returns
+    -------
+    averaged_predictions : array-like of shape (n_targets, n_points)
+        The averaged predictions for the given `grid` of features values.
+        Note that `n_targets` is the number of targets (e.g. 1 for binary
+        classification, `n_tasks` for multi-output regression, and `n_classes` for
+        multiclass classification) and `n_points` is the number of points in the `grid`.
+    """
+    averaged_predictions = est._compute_partial_dependence_recursion(grid, features)
     if averaged_predictions.ndim == 1:
         # reshape to (1, n_points) for consistency with
         # _partial_dependence_brute
@@ -115,58 +215,102 @@ def _partial_dependence_recursion(est, grid, features):
     return averaged_predictions
 
 
-def _partial_dependence_brute(est, grid, features, X, response_method):
+def _partial_dependence_brute(
+    est, grid, features, X, response_method, sample_weight=None
+):
+    """Calculate partial dependence via the brute force method.
 
+    The brute method explicitly averages the predictions of an estimator over a
+    grid of feature values.
+
+    For each `grid` value, all the samples from `X` have their variables of
+    interest replaced by that specific `grid` value. The predictions are then made
+    and averaged across the samples.
+
+    This method is slower than the `'recursion'`
+    (:func:`~sklearn.inspection._partial_dependence._partial_dependence_recursion`)
+    version for estimators with this second option. However, with the `'brute'`
+    force method, the average will be done with the given `X` and not the `X`
+    used during training, as it is done in the `'recursion'` version. Therefore
+    the average can always accept `sample_weight` (even when the estimator was
+    fitted without).
+
+    Parameters
+    ----------
+    est : BaseEstimator
+        A fitted estimator object implementing :term:`predict`,
+        :term:`predict_proba`, or :term:`decision_function`.
+        Multioutput-multiclass classifiers are not supported.
+
+    grid : array-like of shape (n_points, n_target_features)
+        The grid of feature values for which the partial dependence is calculated.
+        Note that `n_points` is the number of points in the grid and `n_target_features`
+        is the number of features you are doing partial dependence at.
+
+    features : array-like of {int, str}
+        The feature (e.g. `[0]`) or pair of interacting features
+        (e.g. `[(0, 1)]`) for which the partial dependency should be computed.
+
+    X : array-like of shape (n_samples, n_features)
+        `X` is used to generate values for the complement features. That is, for
+        each value in `grid`, the method will average the prediction of each
+        sample from `X` having that grid value for `features`.
+
+    response_method : {'auto', 'predict_proba', 'decision_function'}, \
+            default='auto'
+        Specifies whether to use :term:`predict_proba` or
+        :term:`decision_function` as the target response. For regressors
+        this parameter is ignored and the response is always the output of
+        :term:`predict`. By default, :term:`predict_proba` is tried first
+        and we revert to :term:`decision_function` if it doesn't exist.
+
+    sample_weight : array-like of shape (n_samples,), default=None
+        Sample weights are used to calculate weighted means when averaging the
+        model output. If `None`, then samples are equally weighted. Note that
+        `sample_weight` does not change the individual predictions.
+
+    Returns
+    -------
+    averaged_predictions : array-like of shape (n_targets, n_points)
+        The averaged predictions for the given `grid` of features values.
+        Note that `n_targets` is the number of targets (e.g. 1 for binary
+        classification, `n_tasks` for multi-output regression, and `n_classes` for
+        multiclass classification) and `n_points` is the number of points in the `grid`.
+
+    predictions : array-like
+        The predictions for the given `grid` of features values over the samples
+        from `X`. For non-multioutput regression and binary classification the
+        shape is `(n_instances, n_points)` and for multi-output regression and
+        multiclass classification the shape is `(n_targets, n_instances, n_points)`,
+        where `n_targets` is the number of targets (`n_tasks` for multi-output
+        regression, and `n_classes` for multiclass classification), `n_instances`
+        is the number of instances in `X`, and `n_points` is the number of points
+        in the `grid`.
+    """
     predictions = []
     averaged_predictions = []
 
-    # define the prediction_method (predict, predict_proba, decision_function).
-    if is_regressor(est):
-        prediction_method = est.predict
-    else:
-        predict_proba = getattr(est, 'predict_proba', None)
-        decision_function = getattr(est, 'decision_function', None)
-        if response_method == 'auto':
-            # try predict_proba, then decision_function if it doesn't exist
-            prediction_method = predict_proba or decision_function
-        else:
-            prediction_method = (predict_proba if response_method ==
-                                 'predict_proba' else decision_function)
-        if prediction_method is None:
-            if response_method == 'auto':
-                raise ValueError(
-                    'The estimator has no predict_proba and no '
-                    'decision_function method.'
-                )
-            elif response_method == 'predict_proba':
-                raise ValueError('The estimator has no predict_proba method.')
-            else:
-                raise ValueError(
-                    'The estimator has no decision_function method.')
+    if response_method == "auto":
+        response_method = (
+            "predict" if is_regressor(est) else ["predict_proba", "decision_function"]
+        )
 
+    X_eval = X.copy()
     for new_values in grid:
-        X_eval = X.copy()
         for i, variable in enumerate(features):
-            if hasattr(X_eval, 'iloc'):
-                X_eval.iloc[:, variable] = new_values[i]
-            else:
-                X_eval[:, variable] = new_values[i]
-
-        try:
-            # Note: predictions is of shape
-            # (n_points,) for non-multioutput regressors
-            # (n_points, n_tasks) for multioutput regressors
-            # (n_points, 1) for the regressors in cross_decomposition (I think)
-            # (n_points, 2) for binary classification
-            # (n_points, n_classes) for multiclass classification
-            pred = prediction_method(X_eval)
-
-            predictions.append(pred)
-            # average over samples
-            averaged_predictions.append(np.mean(pred, axis=0))
-        except NotFittedError as e:
-            raise ValueError(
-                "'estimator' parameter must be a fitted estimator") from e
+            _safe_assign(X_eval, new_values[i], column_indexer=variable)
+
+        # Note: predictions is of shape
+        # (n_points,) for non-multioutput regressors
+        # (n_points, n_tasks) for multioutput regressors
+        # (n_points, 1) for the regressors in cross_decomposition (I think)
+        # (n_points, 1) for binary classification (positive class already selected)
+        # (n_points, n_classes) for multiclass classification
+        pred, _ = _get_response_values(est, X_eval, response_method=response_method)
+
+        predictions.append(pred)
+        # average over samples
+        averaged_predictions.append(np.average(pred, axis=0, weights=sample_weight))
 
     n_samples = X.shape[0]
 
@@ -191,29 +335,59 @@ def _partial_dependence_brute(est, grid, features, X, response_method):
     # - n_tasks for multi-output regression
     # - n_classes for multiclass classification.
     averaged_predictions = np.array(averaged_predictions).T
-    if is_regressor(est) and averaged_predictions.ndim == 1:
-        # non-multioutput regression, shape is (n_points,)
-        averaged_predictions = averaged_predictions.reshape(1, -1)
-    elif is_classifier(est) and averaged_predictions.shape[0] == 2:
-        # Binary classification, shape is (2, n_points).
-        # we output the effect of **positive** class
-        averaged_predictions = averaged_predictions[1]
+    if averaged_predictions.ndim == 1:
+        # reshape to (1, n_points) for consistency with
+        # _partial_dependence_recursion
         averaged_predictions = averaged_predictions.reshape(1, -1)
 
     return averaged_predictions, predictions
 
 
-@_deprecate_positional_args
-def partial_dependence(estimator, X, features, *, response_method='auto',
-                       percentiles=(0.05, 0.95), grid_resolution=100,
-                       method='auto', kind='legacy'):
+@validate_params(
+    {
+        "estimator": [
+            HasMethods(["fit", "predict"]),
+            HasMethods(["fit", "predict_proba"]),
+            HasMethods(["fit", "decision_function"]),
+        ],
+        "X": ["array-like", "sparse matrix"],
+        "features": ["array-like", Integral, str],
+        "sample_weight": ["array-like", None],
+        "categorical_features": ["array-like", None],
+        "feature_names": ["array-like", None],
+        "response_method": [StrOptions({"auto", "predict_proba", "decision_function"})],
+        "percentiles": [tuple],
+        "grid_resolution": [Interval(Integral, 1, None, closed="left")],
+        "method": [StrOptions({"auto", "recursion", "brute"})],
+        "kind": [StrOptions({"average", "individual", "both"})],
+        "custom_values": [dict, None],
+    },
+    prefer_skip_nested_validation=True,
+)
+def partial_dependence(
+    estimator,
+    X,
+    features,
+    *,
+    sample_weight=None,
+    categorical_features=None,
+    feature_names=None,
+    response_method="auto",
+    percentiles=(0.05, 0.95),
+    grid_resolution=100,
+    custom_values=None,
+    method="auto",
+    kind="average",
+):
     """Partial dependence of ``features``.
 
     Partial dependence of a feature (or a set of features) corresponds to
     the average response of an estimator for each possible value of the
     feature.
 
-    Read more in the :ref:`User Guide `.
+    Read more in
+    :ref:`sphx_glr_auto_examples_inspection_plot_partial_dependence.py`
+    and the :ref:`User Guide `.
 
     .. warning::
 
@@ -239,16 +413,45 @@ def partial_dependence(estimator, X, features, *, response_method='auto',
         :term:`predict_proba`, or :term:`decision_function`.
         Multioutput-multiclass classifiers are not supported.
 
-    X : {array-like or dataframe} of shape (n_samples, n_features)
+    X : {array-like, sparse matrix or dataframe} of shape (n_samples, n_features)
         ``X`` is used to generate a grid of values for the target
         ``features`` (where the partial dependence will be evaluated), and
         also to generate values for the complement features when the
         `method` is 'brute'.
 
-    features : array-like of {int, str}
+    features : array-like of {int, str, bool} or int or str
         The feature (e.g. `[0]`) or pair of interacting features
         (e.g. `[(0, 1)]`) for which the partial dependency should be computed.
 
+    sample_weight : array-like of shape (n_samples,), default=None
+        Sample weights are used to calculate weighted means when averaging the
+        model output. If `None`, then samples are equally weighted. If
+        `sample_weight` is not `None`, then `method` will be set to `'brute'`.
+        Note that `sample_weight` is ignored for `kind='individual'`.
+
+        .. versionadded:: 1.3
+
+    categorical_features : array-like of shape (n_features,) or shape \
+            (n_categorical_features,), dtype={bool, int, str}, default=None
+        Indicates the categorical features.
+
+        - `None`: no feature will be considered categorical;
+        - boolean array-like: boolean mask of shape `(n_features,)`
+            indicating which features are categorical. Thus, this array has
+            the same shape has `X.shape[1]`;
+        - integer or string array-like: integer indices or strings
+            indicating categorical features.
+
+        .. versionadded:: 1.2
+
+    feature_names : array-like of shape (n_features,), dtype=str, default=None
+        Name of each feature; `feature_names[i]` holds the name of the feature
+        with index `i`.
+        By default, the name of the feature corresponds to their numerical
+        index for NumPy array and their column name for pandas dataframe.
+
+        .. versionadded:: 1.2
+
     response_method : {'auto', 'predict_proba', 'decision_function'}, \
             default='auto'
         Specifies whether to use :term:`predict_proba` or
@@ -262,10 +465,24 @@ def partial_dependence(estimator, X, features, *, response_method='auto',
     percentiles : tuple of float, default=(0.05, 0.95)
         The lower and upper percentile used to create the extreme values
         for the grid. Must be in [0, 1].
+        This parameter is overridden by `custom_values` if that parameter is set.
 
     grid_resolution : int, default=100
         The number of equally spaced points on the grid, for each target
         feature.
+        This parameter is overridden by `custom_values` if that parameter is set.
+
+    custom_values : dict
+        A dictionary mapping the index of an element of `features` to an array
+        of values where the partial dependence should be calculated
+        for that feature. Setting a range of values for a feature overrides
+        `grid_resolution` and `percentiles`.
+
+        See :ref:`how to use partial_dependence
+        ` for an example of how this parameter can
+        be used.
+
+        .. versionadded:: 1.7
 
     method : {'auto', 'recursion', 'brute'}, default='auto'
         The method used to calculate the averaged predictions:
@@ -282,7 +499,7 @@ def partial_dependence(estimator, X, features, *, response_method='auto',
           This is more efficient in terms of speed.
           With this method, the target response of a
           classifier is always the decision function, not the predicted
-          probabilities. Since the `'recursion'` method implicitely computes
+          probabilities. Since the `'recursion'` method implicitly computes
           the average of the Individual Conditional Expectation (ICE) by
           design, it is not compatible with ICE and thus `kind` must be
           `'average'`.
@@ -291,76 +508,60 @@ def partial_dependence(estimator, X, features, *, response_method='auto',
           computationally intensive.
 
         - `'auto'`: the `'recursion'` is used for estimators that support it,
-          and `'brute'` is used otherwise.
+          and `'brute'` is used otherwise. If `sample_weight` is not `None`,
+          then `'brute'` is used regardless of the estimator.
 
         Please see :ref:`this note ` for
         differences between the `'brute'` and `'recursion'` method.
 
-    kind : {'legacy', 'average', 'individual', 'both'}, default='legacy'
+    kind : {'average', 'individual', 'both'}, default='average'
         Whether to return the partial dependence averaged across all the
-        samples in the dataset or one line per sample or both.
+        samples in the dataset or one value per sample or both.
         See Returns below.
 
         Note that the fast `method='recursion'` option is only available for
-        `kind='average'`. Plotting individual dependencies requires using the
-        slower `method='brute'` option.
+        `kind='average'` and `sample_weights=None`. Computing individual
+        dependencies and doing weighted averages requires using the slower
+        `method='brute'`.
 
         .. versionadded:: 0.24
-        .. deprecated:: 0.24
-            `kind='legacy'` is deprecated and will be removed in version 1.1.
-            `kind='average'` will be the new default. It is intended to migrate
-            from the ndarray output to :class:`~sklearn.utils.Bunch` output.
-
 
     Returns
     -------
-    predictions : ndarray or :class:`~sklearn.utils.Bunch`
+    predictions : :class:`~sklearn.utils.Bunch`
+        Dictionary-like object, with the following attributes.
 
-        - if `kind='legacy'`, return value is ndarray of shape (n_outputs, \
+        individual : ndarray of shape (n_outputs, n_instances, \
                 len(values[0]), len(values[1]), ...)
+            The predictions for all the points in the grid for all
+            samples in X. This is also known as Individual
+            Conditional Expectation (ICE).
+            Only available when `kind='individual'` or `kind='both'`.
+
+        average : ndarray of shape (n_outputs, len(values[0]), \
+                len(values[1]), ...)
             The predictions for all the points in the grid, averaged
-            over all samples in X (or over the training data if ``method``
-            is 'recursion').
-
-        - if `kind='individual'`, `'average'` or `'both'`, return value is \
-                :class:`~sklearn.utils.Bunch`
-            Dictionary-like object, with the following attributes.
-
-            individual : ndarray of shape (n_outputs, n_instances, \
-                    len(values[0]), len(values[1]), ...)
-                The predictions for all the points in the grid for all
-                samples in X. This is also known as Individual
-                Conditional Expectation (ICE)
-
-            average : ndarray of shape (n_outputs, len(values[0]), \
-                    len(values[1]), ...)
-                The predictions for all the points in the grid, averaged
-                over all samples in X (or over the training data if
-                ``method`` is 'recursion').
-                Only available when kind='both'.
-
-            values : seq of 1d ndarrays
-                The values with which the grid has been created. The generated
-                grid is a cartesian product of the arrays in ``values``.
-                ``len(values) == len(features)``. The size of each array
-                ``values[j]`` is either ``grid_resolution``, or the number of
-                unique values in ``X[:, j]``, whichever is smaller.
-
-        ``n_outputs`` corresponds to the number of classes in a multi-class
-        setting, or to the number of tasks for multi-output regression.
-        For classical regression and binary classification ``n_outputs==1``.
-        ``n_values_feature_j`` corresponds to the size ``values[j]``.
+            over all samples in X (or over the training data if
+            `method` is 'recursion').
+            Only available when `kind='average'` or `kind='both'`.
+
+        grid_values : seq of 1d ndarrays
+            The values with which the grid has been created. The generated
+            grid is a cartesian product of the arrays in `grid_values` where
+            `len(grid_values) == len(features)`. The size of each array
+            `grid_values[j]` is either `grid_resolution`, or the number of
+            unique values in `X[:, j]`, whichever is smaller.
 
-    values : seq of 1d ndarrays
-        The values with which the grid has been created. The generated grid
-        is a cartesian product of the arrays in ``values``. ``len(values) ==
-        len(features)``. The size of each array ``values[j]`` is either
-        ``grid_resolution``, or the number of unique values in ``X[:, j]``,
-        whichever is smaller. Only available when `kind="legacy"`.
+            .. versionadded:: 1.3
+
+        `n_outputs` corresponds to the number of classes in a multi-class
+        setting, or to the number of tasks for multi-output regression.
+        For classical regression and binary classification `n_outputs==1`.
+        `n_values_feature_j` corresponds to the size `grid_values[j]`.
 
     See Also
     --------
-    plot_partial_dependence : Plot Partial Dependence.
+    PartialDependenceDisplay.from_estimator : Plot Partial Dependence.
     PartialDependenceDisplay : Partial Dependence visualization.
 
     Examples
@@ -371,119 +572,179 @@ def partial_dependence(estimator, X, features, *, response_method='auto',
     >>> gb = GradientBoostingClassifier(random_state=0).fit(X, y)
     >>> partial_dependence(gb, features=[0], X=X, percentiles=(0, 1),
     ...                    grid_resolution=2) # doctest: +SKIP
-    (array([[-4.52...,  4.52...]]), [array([ 0.,  1.])])
+    (array([[-4.52,  4.52]]), [array([ 0.,  1.])])
     """
-    if not (is_classifier(estimator) or is_regressor(estimator)):
-        raise ValueError(
-            "'estimator' must be a fitted regressor or classifier."
-        )
+    check_is_fitted(estimator)
 
-    if isinstance(estimator, Pipeline):
-        # TODO: to be removed if/when pipeline get a `steps_` attributes
-        # assuming Pipeline is the only estimator that does not store a new
-        # attribute
-        for est in estimator:
-            # FIXME: remove the None option when it will be deprecated
-            if est not in (None, 'drop'):
-                check_is_fitted(est)
-    else:
-        check_is_fitted(estimator)
+    if not (is_classifier(estimator) or is_regressor(estimator)):
+        raise ValueError("'estimator' must be a fitted regressor or classifier.")
 
-    if (is_classifier(estimator) and
-            isinstance(estimator.classes_[0], np.ndarray)):
-        raise ValueError(
-            'Multiclass-multioutput estimators are not supported'
-        )
+    if is_classifier(estimator) and isinstance(estimator.classes_[0], np.ndarray):
+        raise ValueError("Multiclass-multioutput estimators are not supported")
 
     # Use check_array only on lists and other non-array-likes / sparse. Do not
     # convert DataFrame into a NumPy array.
-    if not(hasattr(X, '__array__') or sparse.issparse(X)):
-        X = check_array(X, force_all_finite='allow-nan', dtype=object)
+    if not (hasattr(X, "__array__") or sparse.issparse(X)):
+        X = check_array(X, ensure_all_finite="allow-nan", dtype=object)
 
-    accepted_responses = ('auto', 'predict_proba', 'decision_function')
-    if response_method not in accepted_responses:
-        raise ValueError(
-            'response_method {} is invalid. Accepted response_method names '
-            'are {}.'.format(response_method, ', '.join(accepted_responses)))
-
-    if is_regressor(estimator) and response_method != 'auto':
+    if is_regressor(estimator) and response_method != "auto":
         raise ValueError(
             "The response_method parameter is ignored for regressors and "
             "must be 'auto'."
         )
 
-    accepted_methods = ('brute', 'recursion', 'auto')
-    if method not in accepted_methods:
-        raise ValueError(
-            'method {} is invalid. Accepted method names are {}.'.format(
-                method, ', '.join(accepted_methods)))
-
-    if kind != 'average' and kind != 'legacy':
-        if method == 'recursion':
+    if kind != "average":
+        if method == "recursion":
             raise ValueError(
-                "The 'recursion' method only applies when 'kind' is set "
-                "to 'average'"
+                "The 'recursion' method only applies when 'kind' is set to 'average'"
             )
-        method = 'brute'
-
-    if method == 'auto':
-        if (isinstance(estimator, BaseGradientBoosting) and
-                estimator.init is None):
-            method = 'recursion'
-        elif isinstance(estimator, (BaseHistGradientBoosting,
-                                    DecisionTreeRegressor,
-                                    RandomForestRegressor)):
-            method = 'recursion'
-        else:
-            method = 'brute'
+        method = "brute"
 
-    if method == 'recursion':
-        if not isinstance(estimator,
-                          (BaseGradientBoosting, BaseHistGradientBoosting,
-                           DecisionTreeRegressor, RandomForestRegressor)):
+    if method == "recursion" and sample_weight is not None:
+        raise ValueError(
+            "The 'recursion' method can only be applied when sample_weight is None."
+        )
+
+    if method == "auto":
+        if sample_weight is not None:
+            method = "brute"
+        elif isinstance(estimator, BaseGradientBoosting) and estimator.init is None:
+            method = "recursion"
+        elif isinstance(
+            estimator,
+            (BaseHistGradientBoosting, DecisionTreeRegressor, RandomForestRegressor),
+        ):
+            method = "recursion"
+        else:
+            method = "brute"
+
+    if method == "recursion":
+        if not isinstance(
+            estimator,
+            (
+                BaseGradientBoosting,
+                BaseHistGradientBoosting,
+                DecisionTreeRegressor,
+                RandomForestRegressor,
+            ),
+        ):
             supported_classes_recursion = (
-                'GradientBoostingClassifier',
-                'GradientBoostingRegressor',
-                'HistGradientBoostingClassifier',
-                'HistGradientBoostingRegressor',
-                'HistGradientBoostingRegressor',
-                'DecisionTreeRegressor',
-                'RandomForestRegressor',
+                "GradientBoostingClassifier",
+                "GradientBoostingRegressor",
+                "HistGradientBoostingClassifier",
+                "HistGradientBoostingRegressor",
+                "HistGradientBoostingRegressor",
+                "DecisionTreeRegressor",
+                "RandomForestRegressor",
             )
             raise ValueError(
                 "Only the following estimators support the 'recursion' "
-                "method: {}. Try using method='brute'."
-                .format(', '.join(supported_classes_recursion)))
-        if response_method == 'auto':
-            response_method = 'decision_function'
+                "method: {}. Try using method='brute'.".format(
+                    ", ".join(supported_classes_recursion)
+                )
+            )
+        if response_method == "auto":
+            response_method = "decision_function"
 
-        if response_method != 'decision_function':
+        if response_method != "decision_function":
             raise ValueError(
                 "With the 'recursion' method, the response_method must be "
                 "'decision_function'. Got {}.".format(response_method)
             )
 
-    if _determine_key_type(features, accept_slice=False) == 'int':
+    if sample_weight is not None:
+        sample_weight = _check_sample_weight(sample_weight, X)
+
+    if _determine_key_type(features, accept_slice=False) == "int":
         # _get_column_indices() supports negative indexing. Here, we limit
         # the indexing to be positive. The upper bound will be checked
         # by _get_column_indices()
         if np.any(np.less(features, 0)):
-            raise ValueError(
-                'all features must be in [0, {}]'.format(X.shape[1] - 1)
-            )
+            raise ValueError("all features must be in [0, {}]".format(X.shape[1] - 1))
 
     features_indices = np.asarray(
-        _get_column_indices(X, features), dtype=np.int32, order='C'
+        _get_column_indices(X, features), dtype=np.intp, order="C"
     ).ravel()
 
+    feature_names = _check_feature_names(X, feature_names)
+
+    n_features = X.shape[1]
+    if categorical_features is None:
+        is_categorical = [False] * len(features_indices)
+    else:
+        categorical_features = np.asarray(categorical_features)
+        if categorical_features.size == 0:
+            raise ValueError(
+                "Passing an empty list (`[]`) to `categorical_features` is not "
+                "supported. Use `None` instead to indicate that there are no "
+                "categorical features."
+            )
+        if categorical_features.dtype.kind == "b":
+            # categorical features provided as a list of boolean
+            if categorical_features.size != n_features:
+                raise ValueError(
+                    "When `categorical_features` is a boolean array-like, "
+                    "the array should be of shape (n_features,). Got "
+                    f"{categorical_features.size} elements while `X` contains "
+                    f"{n_features} features."
+                )
+            is_categorical = [categorical_features[idx] for idx in features_indices]
+        elif categorical_features.dtype.kind in ("i", "O", "U"):
+            # categorical features provided as a list of indices or feature names
+            categorical_features_idx = [
+                _get_feature_index(cat, feature_names=feature_names)
+                for cat in categorical_features
+            ]
+            is_categorical = [
+                idx in categorical_features_idx for idx in features_indices
+            ]
+        else:
+            raise ValueError(
+                "Expected `categorical_features` to be an array-like of boolean,"
+                f" integer, or string. Got {categorical_features.dtype} instead."
+            )
+
+    custom_values = custom_values or {}
+    if isinstance(features, (str, int)):
+        features = [features]
+
+    for feature_idx, feature, is_cat in zip(features_indices, features, is_categorical):
+        if is_cat:
+            continue
+
+        if _safe_indexing(X, feature_idx, axis=1).dtype.kind in "iu":
+            # TODO(1.9): raise a ValueError instead.
+            warnings.warn(
+                f"The column {feature!r} contains integer data. Partial "
+                "dependence plots are not supported for integer data: this "
+                "can lead to implicit rounding with NumPy arrays or even errors "
+                "with newer pandas versions. Please convert numerical features"
+                "to floating point dtypes ahead of time to avoid problems. "
+                "This will raise ValueError in scikit-learn 1.9.",
+                FutureWarning,
+            )
+            # Do not warn again for other features to avoid spamming the caller.
+            break
+
+    X_subset = _safe_indexing(X, features_indices, axis=1)
+
+    custom_values_for_X_subset = {
+        index: custom_values.get(feature)
+        for index, feature in enumerate(features)
+        if feature in custom_values
+    }
+
     grid, values = _grid_from_X(
-        _safe_indexing(X, features_indices, axis=1), percentiles,
-        grid_resolution
+        X_subset,
+        percentiles,
+        is_categorical,
+        grid_resolution,
+        custom_values_for_X_subset,
     )
 
-    if method == 'brute':
+    if method == "brute":
         averaged_predictions, predictions = _partial_dependence_brute(
-            estimator, grid, features_indices, X, response_method
+            estimator, grid, features_indices, X, response_method, sample_weight
         )
 
         # reshape predictions to
@@ -499,24 +760,16 @@ def partial_dependence(estimator, X, features, *, response_method='auto',
     # reshape averaged_predictions to
     # (n_outputs, n_values_feature_0, n_values_feature_1, ...)
     averaged_predictions = averaged_predictions.reshape(
-        -1, *[val.shape[0] for val in values])
-
-    if kind == 'legacy':
-        warnings.warn(
-            "A Bunch will be returned in place of 'predictions' from version"
-            " 1.1 (renaming of 0.26) with partial dependence results "
-            "accessible via the 'average' key. In the meantime, pass "
-            "kind='average' to get the future behaviour.",
-            FutureWarning
-        )
-        # TODO 1.1: Remove kind == 'legacy' section
-        return averaged_predictions, values
-    elif kind == 'average':
-        return Bunch(average=averaged_predictions, values=values)
-    elif kind == 'individual':
-        return Bunch(individual=predictions, values=values)
+        -1, *[val.shape[0] for val in values]
+    )
+    pdp_results = Bunch(grid_values=values)
+
+    if kind == "average":
+        pdp_results["average"] = averaged_predictions
+    elif kind == "individual":
+        pdp_results["individual"] = predictions
     else:  # kind='both'
-        return Bunch(
-            average=averaged_predictions, individual=predictions,
-            values=values,
-        )
+        pdp_results["average"] = averaged_predictions
+        pdp_results["individual"] = predictions
+
+    return pdp_results
diff --git a/sklearn/inspection/_pd_utils.py b/sklearn/inspection/_pd_utils.py
new file mode 100644
index 0000000000000..a48ba4d9a4490
--- /dev/null
+++ b/sklearn/inspection/_pd_utils.py
@@ -0,0 +1,68 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+
+def _check_feature_names(X, feature_names=None):
+    """Check feature names.
+
+    Parameters
+    ----------
+    X : array-like of shape (n_samples, n_features)
+        Input data.
+
+    feature_names : None or array-like of shape (n_names,), dtype=str
+        Feature names to check or `None`.
+
+    Returns
+    -------
+    feature_names : list of str
+        Feature names validated. If `feature_names` is `None`, then a list of
+        feature names is provided, i.e. the column names of a pandas dataframe
+        or a generic list of feature names (e.g. `["x0", "x1", ...]`) for a
+        NumPy array.
+    """
+    if feature_names is None:
+        if hasattr(X, "columns") and hasattr(X.columns, "tolist"):
+            # get the column names for a pandas dataframe
+            feature_names = X.columns.tolist()
+        else:
+            # define a list of numbered indices for a numpy array
+            feature_names = [f"x{i}" for i in range(X.shape[1])]
+    elif hasattr(feature_names, "tolist"):
+        # convert numpy array or pandas index to a list
+        feature_names = feature_names.tolist()
+    if len(set(feature_names)) != len(feature_names):
+        raise ValueError("feature_names should not contain duplicates.")
+
+    return feature_names
+
+
+def _get_feature_index(fx, feature_names=None):
+    """Get feature index.
+
+    Parameters
+    ----------
+    fx : int or str
+        Feature index or name.
+
+    feature_names : list of str, default=None
+        All feature names from which to search the indices.
+
+    Returns
+    -------
+    idx : int
+        Feature index.
+    """
+    if isinstance(fx, str):
+        if feature_names is None:
+            raise ValueError(
+                f"Cannot plot partial dependence for feature {fx!r} since "
+                "the list of feature names was not provided, neither as "
+                "column names of a pandas data-frame nor via the feature_names "
+                "parameter."
+            )
+        try:
+            return feature_names.index(fx)
+        except ValueError as e:
+            raise ValueError(f"Feature {fx!r} not in feature_names") from e
+    return fx
diff --git a/sklearn/inspection/_permutation_importance.py b/sklearn/inspection/_permutation_importance.py
index 2a7b6cd23147b..451062fbe272e 100644
--- a/sklearn/inspection/_permutation_importance.py
+++ b/sklearn/inspection/_permutation_importance.py
@@ -1,49 +1,78 @@
 """Permutation importance for estimators."""
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+import numbers
+
 import numpy as np
-from joblib import Parallel
 
-from ..metrics import check_scoring
-from ..metrics._scorer import _check_multimetric_scoring, _MultimetricScorer
+from ..ensemble._bagging import _generate_indices
+from ..metrics import check_scoring, get_scorer_names
 from ..model_selection._validation import _aggregate_score_dicts
-from ..utils import Bunch
-from ..utils import check_random_state
-from ..utils import check_array
-from ..utils.validation import _deprecate_positional_args
-from ..utils.fixes import delayed
+from ..utils import Bunch, _safe_indexing, check_array, check_random_state
+from ..utils._param_validation import (
+    HasMethods,
+    Integral,
+    Interval,
+    RealNotInt,
+    StrOptions,
+    validate_params,
+)
+from ..utils.parallel import Parallel, delayed
 
 
 def _weights_scorer(scorer, estimator, X, y, sample_weight):
     if sample_weight is not None:
-        return scorer(estimator, X, y, sample_weight)
+        return scorer(estimator, X, y, sample_weight=sample_weight)
     return scorer(estimator, X, y)
 
 
-def _calculate_permutation_scores(estimator, X, y, sample_weight, col_idx,
-                                  random_state, n_repeats, scorer):
+def _calculate_permutation_scores(
+    estimator,
+    X,
+    y,
+    sample_weight,
+    col_idx,
+    random_state,
+    n_repeats,
+    scorer,
+    max_samples,
+):
     """Calculate score when `col_idx` is permuted."""
     random_state = check_random_state(random_state)
 
-    # Work on a copy of X to to ensure thread-safety in case of threading based
+    # Work on a copy of X to ensure thread-safety in case of threading based
     # parallelism. Furthermore, making a copy is also useful when the joblib
     # backend is 'loky' (default) or the old 'multiprocessing': in those cases,
     # if X is large it will be automatically be backed by a readonly memory map
     # (memmap). X.copy() on the other hand is always guaranteed to return a
     # writable data-structure whose columns can be shuffled inplace.
-    X_permuted = X.copy()
+    if max_samples < X.shape[0]:
+        row_indices = _generate_indices(
+            random_state=random_state,
+            bootstrap=False,
+            n_population=X.shape[0],
+            n_samples=max_samples,
+        )
+        X_permuted = _safe_indexing(X, row_indices, axis=0)
+        y = _safe_indexing(y, row_indices, axis=0)
+        if sample_weight is not None:
+            sample_weight = _safe_indexing(sample_weight, row_indices, axis=0)
+    else:
+        X_permuted = X.copy()
 
     scores = []
-    shuffling_idx = np.arange(X.shape[0])
+    shuffling_idx = np.arange(X_permuted.shape[0])
     for _ in range(n_repeats):
         random_state.shuffle(shuffling_idx)
         if hasattr(X_permuted, "iloc"):
             col = X_permuted.iloc[shuffling_idx, col_idx]
             col.index = X_permuted.index
-            X_permuted.iloc[:, col_idx] = col
+            X_permuted[X_permuted.columns[col_idx]] = col
         else:
             X_permuted[:, col_idx] = X_permuted[shuffling_idx, col_idx]
-        scores.append(
-            _weights_scorer(scorer, estimator, X_permuted, y, sample_weight)
-        )
+        scores.append(_weights_scorer(scorer, estimator, X_permuted, y, sample_weight))
 
     if isinstance(scores[0], dict):
         scores = _aggregate_score_dicts(scores)
@@ -75,14 +104,49 @@ def _create_importances_bunch(baseline_score, permuted_score):
             Raw permutation importance scores.
     """
     importances = baseline_score - permuted_score
-    return Bunch(importances_mean=np.mean(importances, axis=1),
-                 importances_std=np.std(importances, axis=1),
-                 importances=importances)
-
-
-@_deprecate_positional_args
-def permutation_importance(estimator, X, y, *, scoring=None, n_repeats=5,
-                           n_jobs=None, random_state=None, sample_weight=None):
+    return Bunch(
+        importances_mean=np.mean(importances, axis=1),
+        importances_std=np.std(importances, axis=1),
+        importances=importances,
+    )
+
+
+@validate_params(
+    {
+        "estimator": [HasMethods(["fit"])],
+        "X": ["array-like"],
+        "y": ["array-like", None],
+        "scoring": [
+            StrOptions(set(get_scorer_names())),
+            callable,
+            list,
+            tuple,
+            dict,
+            None,
+        ],
+        "n_repeats": [Interval(Integral, 1, None, closed="left")],
+        "n_jobs": [Integral, None],
+        "random_state": ["random_state"],
+        "sample_weight": ["array-like", None],
+        "max_samples": [
+            Interval(Integral, 1, None, closed="left"),
+            Interval(RealNotInt, 0, 1, closed="right"),
+        ],
+    },
+    prefer_skip_nested_validation=True,
+)
+def permutation_importance(
+    estimator,
+    X,
+    y,
+    *,
+    scoring=None,
+    n_repeats=5,
+    n_jobs=None,
+    random_state=None,
+    sample_weight=None,
+    max_samples=1.0,
+):
     """Permutation importance for feature evaluation [BRE]_.
 
     The :term:`estimator` is required to be a fitted estimator. `X` can be the
@@ -112,8 +176,11 @@ def permutation_importance(estimator, X, y, *, scoring=None, n_repeats=5,
         Scorer to use.
         If `scoring` represents a single score, one can use:
 
-        - a single string (see :ref:`scoring_parameter`);
-        - a callable (see :ref:`scoring`) that returns a single value.
+        - str: see :ref:`scoring_string_names` for options.
+        - callable: a scorer callable object (e.g., function) with signature
+          ``scorer(estimator, X, y)``. See :ref:`scoring_callable` for details.
+        - `None`: the `estimator`'s
+          :ref:`default evaluation criterion ` is used.
 
         If `scoring` represents multiple scores, one can use:
 
@@ -126,8 +193,6 @@ def permutation_importance(estimator, X, y, *, scoring=None, n_repeats=5,
         `permutation_importance` for each of the scores as it reuses
         predictions to avoid redundant computation.
 
-        If None, the estimator's default scorer is used.
-
     n_repeats : int, default=5
         Number of times to permute a feature.
 
@@ -142,13 +207,29 @@ def permutation_importance(estimator, X, y, *, scoring=None, n_repeats=5,
         Pseudo-random number generator to control the permutations of each
         feature.
         Pass an int to get reproducible results across function calls.
-        See :term: `Glossary `.
+        See :term:`Glossary `.
 
     sample_weight : array-like of shape (n_samples,), default=None
         Sample weights used in scoring.
 
         .. versionadded:: 0.24
 
+    max_samples : int or float, default=1.0
+        The number of samples to draw from X to compute feature importance
+        in each repeat (without replacement).
+
+        - If int, then draw `max_samples` samples.
+        - If float, then draw `max_samples * X.shape[0]` samples.
+        - If `max_samples` is equal to `1.0` or `X.shape[0]`, all samples
+          will be used.
+
+        While using this option may provide less accurate importance estimates,
+        it keeps the method tractable when evaluating feature importance on
+        large datasets. In combination with `n_repeats`, this allows to control
+        the computational speed vs statistical accuracy trade-off of this method.
+
+        .. versionadded:: 1.0
+
     Returns
     -------
     result : :class:`~sklearn.utils.Bunch` or dict of such instances
@@ -167,8 +248,8 @@ def permutation_importance(estimator, X, y, *, scoring=None, n_repeats=5,
 
     References
     ----------
-    .. [BRE] L. Breiman, "Random Forests", Machine Learning, 45(1), 5-32,
-             2001. https://doi.org/10.1023/A:1010933404324
+    .. [BRE] :doi:`L. Breiman, "Random Forests", Machine Learning, 45(1), 5-32,
+             2001. <10.1023/A:1010933404324>`
 
     Examples
     --------
@@ -181,12 +262,12 @@ def permutation_importance(estimator, X, y, *, scoring=None, n_repeats=5,
     >>> result = permutation_importance(clf, X, y, n_repeats=10,
     ...                                 random_state=0)
     >>> result.importances_mean
-    array([0.4666..., 0.       , 0.       ])
+    array([0.4666, 0.       , 0.       ])
     >>> result.importances_std
-    array([0.2211..., 0.       , 0.       ])
+    array([0.2211, 0.       , 0.       ])
     """
     if not hasattr(X, "iloc"):
-        X = check_array(X, force_all_finite='allow-nan', dtype=None)
+        X = check_array(X, ensure_all_finite="allow-nan", dtype=None)
 
     # Precompute random seed from the random state to be used
     # to get a fresh independent RandomState instance for each
@@ -196,31 +277,35 @@ def permutation_importance(estimator, X, y, *, scoring=None, n_repeats=5,
     random_state = check_random_state(random_state)
     random_seed = random_state.randint(np.iinfo(np.int32).max + 1)
 
-    if callable(scoring):
-        scorer = scoring
-    elif scoring is None or isinstance(scoring, str):
-        scorer = check_scoring(estimator, scoring=scoring)
-    else:
-        scorers_dict = _check_multimetric_scoring(estimator, scoring)
-        scorer = _MultimetricScorer(**scorers_dict)
+    if not isinstance(max_samples, numbers.Integral):
+        max_samples = int(max_samples * X.shape[0])
+    elif max_samples > X.shape[0]:
+        raise ValueError("max_samples must be <= n_samples")
 
-    baseline_score = _weights_scorer(scorer, estimator, X, y,
-                                     sample_weight)
+    scorer = check_scoring(estimator, scoring=scoring)
+    baseline_score = _weights_scorer(scorer, estimator, X, y, sample_weight)
 
     scores = Parallel(n_jobs=n_jobs)(
         delayed(_calculate_permutation_scores)(
-            estimator, X, y, sample_weight, col_idx, random_seed,
-            n_repeats, scorer
-        ) for col_idx in range(X.shape[1]))
+            estimator,
+            X,
+            y,
+            sample_weight,
+            col_idx,
+            random_seed,
+            n_repeats,
+            scorer,
+            max_samples,
+        )
+        for col_idx in range(X.shape[1])
+    )
 
     if isinstance(baseline_score, dict):
         return {
             name: _create_importances_bunch(
                 baseline_score[name],
                 # unpack the permuted scores
-                np.array([
-                    scores[col_idx][name] for col_idx in range(X.shape[1])
-                ])
+                np.array([scores[col_idx][name] for col_idx in range(X.shape[1])]),
             )
             for name in baseline_score
         }
diff --git a/sklearn/inspection/_plot/__init__.py b/sklearn/inspection/_plot/__init__.py
index e69de29bb2d1d..67dd18fb94b59 100644
--- a/sklearn/inspection/_plot/__init__.py
+++ b/sklearn/inspection/_plot/__init__.py
@@ -0,0 +1,2 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
diff --git a/sklearn/inspection/_plot/decision_boundary.py b/sklearn/inspection/_plot/decision_boundary.py
new file mode 100644
index 0000000000000..bc28708d7c488
--- /dev/null
+++ b/sklearn/inspection/_plot/decision_boundary.py
@@ -0,0 +1,558 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+import warnings
+
+import numpy as np
+
+from ...base import is_regressor
+from ...preprocessing import LabelEncoder
+from ...utils import _safe_indexing
+from ...utils._optional_dependencies import check_matplotlib_support
+from ...utils._response import _get_response_values
+from ...utils._set_output import _get_adapter_from_container
+from ...utils.validation import (
+    _is_arraylike_not_scalar,
+    _is_pandas_df,
+    _is_polars_df,
+    _num_features,
+    check_is_fitted,
+)
+
+
+def _check_boundary_response_method(estimator, response_method, class_of_interest):
+    """Validate the response methods to be used with the fitted estimator.
+
+    Parameters
+    ----------
+    estimator : object
+        Fitted estimator to check.
+
+    response_method : {'auto', 'decision_function', 'predict_proba', 'predict'}
+        Specifies whether to use :term:`decision_function`, :term:`predict_proba`,
+        :term:`predict` as the target response. If set to 'auto', the response method is
+        tried in the before mentioned order.
+
+    class_of_interest : int, float, bool, str or None
+        The class considered when plotting the decision. Cannot be None if
+        multiclass and `response_method` is 'predict_proba' or 'decision_function'.
+
+        .. versionadded:: 1.4
+
+    Returns
+    -------
+    prediction_method : list of str or str
+        The name or list of names of the response methods to use.
+    """
+    has_classes = hasattr(estimator, "classes_")
+    if has_classes and _is_arraylike_not_scalar(estimator.classes_[0]):
+        msg = "Multi-label and multi-output multi-class classifiers are not supported"
+        raise ValueError(msg)
+
+    if response_method == "auto":
+        if is_regressor(estimator):
+            prediction_method = "predict"
+        else:
+            prediction_method = ["decision_function", "predict_proba", "predict"]
+    else:
+        prediction_method = response_method
+
+    return prediction_method
+
+
+class DecisionBoundaryDisplay:
+    """Decisions boundary visualization.
+
+    It is recommended to use
+    :func:`~sklearn.inspection.DecisionBoundaryDisplay.from_estimator`
+    to create a :class:`DecisionBoundaryDisplay`. All parameters are stored as
+    attributes.
+
+    Read more in the :ref:`User Guide `.
+
+    For a detailed example comparing the decision boundaries of multinomial and
+    one-vs-rest logistic regression, please see
+    :ref:`sphx_glr_auto_examples_linear_model_plot_logistic_multinomial.py`.
+
+    .. versionadded:: 1.1
+
+    Parameters
+    ----------
+    xx0 : ndarray of shape (grid_resolution, grid_resolution)
+        First output of :func:`meshgrid `.
+
+    xx1 : ndarray of shape (grid_resolution, grid_resolution)
+        Second output of :func:`meshgrid `.
+
+    response : ndarray of shape (grid_resolution, grid_resolution) or \
+            (grid_resolution, grid_resolution, n_classes)
+        Values of the response function.
+
+    multiclass_colors : list of str or str, default=None
+        Specifies how to color each class when plotting all classes of multiclass
+        problem. Ignored for binary problems and multiclass problems when plotting a
+        single prediction value per point.
+        Possible inputs are:
+
+        * list: list of Matplotlib
+          `color `_
+          strings, of length `n_classes`
+        * str: name of :class:`matplotlib.colors.Colormap`
+        * None: 'viridis' colormap is used to sample colors
+
+        Single color colormaps will be generated from the colors in the list or
+        colors taken from the colormap and passed to the `cmap` parameter of
+        the `plot_method`.
+
+        .. versionadded:: 1.7
+
+    xlabel : str, default=None
+        Default label to place on x axis.
+
+    ylabel : str, default=None
+        Default label to place on y axis.
+
+    Attributes
+    ----------
+    surface_ : matplotlib `QuadContourSet` or `QuadMesh` or list of such objects
+        If `plot_method` is 'contour' or 'contourf', `surface_` is
+        :class:`QuadContourSet `. If
+        `plot_method` is 'pcolormesh', `surface_` is
+        :class:`QuadMesh `.
+
+    multiclass_colors_ : array of shape (n_classes, 4)
+        Colors used to plot each class in multiclass problems.
+        Only defined when `color_of_interest` is None.
+
+        .. versionadded:: 1.7
+
+    ax_ : matplotlib Axes
+        Axes with decision boundary.
+
+    figure_ : matplotlib Figure
+        Figure containing the decision boundary.
+
+    See Also
+    --------
+    DecisionBoundaryDisplay.from_estimator : Plot decision boundary given an estimator.
+
+    Examples
+    --------
+    >>> import matplotlib.pyplot as plt
+    >>> import numpy as np
+    >>> from sklearn.datasets import load_iris
+    >>> from sklearn.inspection import DecisionBoundaryDisplay
+    >>> from sklearn.tree import DecisionTreeClassifier
+    >>> iris = load_iris()
+    >>> feature_1, feature_2 = np.meshgrid(
+    ...     np.linspace(iris.data[:, 0].min(), iris.data[:, 0].max()),
+    ...     np.linspace(iris.data[:, 1].min(), iris.data[:, 1].max())
+    ... )
+    >>> grid = np.vstack([feature_1.ravel(), feature_2.ravel()]).T
+    >>> tree = DecisionTreeClassifier().fit(iris.data[:, :2], iris.target)
+    >>> y_pred = np.reshape(tree.predict(grid), feature_1.shape)
+    >>> display = DecisionBoundaryDisplay(
+    ...     xx0=feature_1, xx1=feature_2, response=y_pred
+    ... )
+    >>> display.plot()
+    <...>
+    >>> display.ax_.scatter(
+    ...     iris.data[:, 0], iris.data[:, 1], c=iris.target, edgecolor="black"
+    ... )
+    <...>
+    >>> plt.show()
+    """
+
+    def __init__(
+        self, *, xx0, xx1, response, multiclass_colors=None, xlabel=None, ylabel=None
+    ):
+        self.xx0 = xx0
+        self.xx1 = xx1
+        self.response = response
+        self.multiclass_colors = multiclass_colors
+        self.xlabel = xlabel
+        self.ylabel = ylabel
+
+    def plot(self, plot_method="contourf", ax=None, xlabel=None, ylabel=None, **kwargs):
+        """Plot visualization.
+
+        Parameters
+        ----------
+        plot_method : {'contourf', 'contour', 'pcolormesh'}, default='contourf'
+            Plotting method to call when plotting the response. Please refer
+            to the following matplotlib documentation for details:
+            :func:`contourf `,
+            :func:`contour `,
+            :func:`pcolormesh `.
+
+        ax : Matplotlib axes, default=None
+            Axes object to plot on. If `None`, a new figure and axes is
+            created.
+
+        xlabel : str, default=None
+            Overwrite the x-axis label.
+
+        ylabel : str, default=None
+            Overwrite the y-axis label.
+
+        **kwargs : dict
+            Additional keyword arguments to be passed to the `plot_method`.
+
+        Returns
+        -------
+        display: :class:`~sklearn.inspection.DecisionBoundaryDisplay`
+            Object that stores computed values.
+        """
+        check_matplotlib_support("DecisionBoundaryDisplay.plot")
+        import matplotlib as mpl
+        import matplotlib.pyplot as plt
+
+        if plot_method not in ("contourf", "contour", "pcolormesh"):
+            raise ValueError(
+                "plot_method must be 'contourf', 'contour', or 'pcolormesh'. "
+                f"Got {plot_method} instead."
+            )
+
+        if ax is None:
+            _, ax = plt.subplots()
+
+        plot_func = getattr(ax, plot_method)
+        if self.response.ndim == 2:
+            self.surface_ = plot_func(self.xx0, self.xx1, self.response, **kwargs)
+        else:  # self.response.ndim == 3
+            n_responses = self.response.shape[-1]
+            if (
+                isinstance(self.multiclass_colors, str)
+                or self.multiclass_colors is None
+            ):
+                if isinstance(self.multiclass_colors, str):
+                    cmap = self.multiclass_colors
+                else:
+                    if n_responses <= 10:
+                        cmap = "tab10"
+                    else:
+                        cmap = "gist_rainbow"
+
+                # Special case for the tab10 and tab20 colormaps that encode a
+                # discrete set of colors that are easily distinguishable
+                # contrary to other colormaps that are continuous.
+                if cmap == "tab10" and n_responses <= 10:
+                    colors = plt.get_cmap("tab10", 10).colors[:n_responses]
+                elif cmap == "tab20" and n_responses <= 20:
+                    colors = plt.get_cmap("tab20", 20).colors[:n_responses]
+                else:
+                    colors = plt.get_cmap(cmap, n_responses).colors
+            elif isinstance(self.multiclass_colors, str):
+                colors = colors = plt.get_cmap(
+                    self.multiclass_colors, n_responses
+                ).colors
+            else:
+                colors = [mpl.colors.to_rgba(color) for color in self.multiclass_colors]
+
+            self.multiclass_colors_ = colors
+            multiclass_cmaps = [
+                mpl.colors.LinearSegmentedColormap.from_list(
+                    f"colormap_{class_idx}", [(1.0, 1.0, 1.0, 1.0), (r, g, b, 1.0)]
+                )
+                for class_idx, (r, g, b, _) in enumerate(colors)
+            ]
+
+            self.surface_ = []
+            for class_idx, cmap in enumerate(multiclass_cmaps):
+                response = np.ma.array(
+                    self.response[:, :, class_idx],
+                    mask=~(self.response.argmax(axis=2) == class_idx),
+                )
+                # `cmap` should not be in kwargs
+                safe_kwargs = kwargs.copy()
+                if "cmap" in safe_kwargs:
+                    del safe_kwargs["cmap"]
+                    warnings.warn(
+                        "Plotting max class of multiclass 'decision_function' or "
+                        "'predict_proba', thus 'multiclass_colors' used and "
+                        "'cmap' kwarg ignored."
+                    )
+                self.surface_.append(
+                    plot_func(self.xx0, self.xx1, response, cmap=cmap, **safe_kwargs)
+                )
+
+        if xlabel is not None or not ax.get_xlabel():
+            xlabel = self.xlabel if xlabel is None else xlabel
+            ax.set_xlabel(xlabel)
+        if ylabel is not None or not ax.get_ylabel():
+            ylabel = self.ylabel if ylabel is None else ylabel
+            ax.set_ylabel(ylabel)
+
+        self.ax_ = ax
+        self.figure_ = ax.figure
+        return self
+
+    @classmethod
+    def from_estimator(
+        cls,
+        estimator,
+        X,
+        *,
+        grid_resolution=100,
+        eps=1.0,
+        plot_method="contourf",
+        response_method="auto",
+        class_of_interest=None,
+        multiclass_colors=None,
+        xlabel=None,
+        ylabel=None,
+        ax=None,
+        **kwargs,
+    ):
+        """Plot decision boundary given an estimator.
+
+        Read more in the :ref:`User Guide `.
+
+        Parameters
+        ----------
+        estimator : object
+            Trained estimator used to plot the decision boundary.
+
+        X : {array-like, sparse matrix, dataframe} of shape (n_samples, 2)
+            Input data that should be only 2-dimensional.
+
+        grid_resolution : int, default=100
+            Number of grid points to use for plotting decision boundary.
+            Higher values will make the plot look nicer but be slower to
+            render.
+
+        eps : float, default=1.0
+            Extends the minimum and maximum values of X for evaluating the
+            response function.
+
+        plot_method : {'contourf', 'contour', 'pcolormesh'}, default='contourf'
+            Plotting method to call when plotting the response. Please refer
+            to the following matplotlib documentation for details:
+            :func:`contourf `,
+            :func:`contour `,
+            :func:`pcolormesh `.
+
+        response_method : {'auto', 'decision_function', 'predict_proba', \
+                'predict'}, default='auto'
+            Specifies whether to use :term:`decision_function`,
+            :term:`predict_proba` or :term:`predict` as the target response.
+            If set to 'auto', the response method is tried in the order as
+            listed above.
+
+            .. versionchanged:: 1.6
+                For multiclass problems, 'auto' no longer defaults to 'predict'.
+
+        class_of_interest : int, float, bool or str, default=None
+            The class to be plotted when `response_method` is 'predict_proba'
+            or 'decision_function'. If None, `estimator.classes_[1]` is considered
+            the positive class for binary classifiers. For multiclass
+            classifiers, if None, all classes will be represented in the
+            decision boundary plot; the class with the highest response value
+            at each point is plotted. The color of each class can be set via
+            `multiclass_colors`.
+
+            .. versionadded:: 1.4
+
+        multiclass_colors : list of str, or str, default=None
+            Specifies how to color each class when plotting multiclass
+            'predict_proba' or 'decision_function' and `class_of_interest` is
+            None. Ignored in all other cases.
+
+            Possible inputs are:
+
+            * list: list of Matplotlib
+              `color `_
+              strings, of length `n_classes`
+            * str: name of :class:`matplotlib.colors.Colormap`
+            * None: 'tab10' colormap is used to sample colors if the number of
+                classes is less than or equal to 10, otherwise 'gist_rainbow'
+                colormap.
+
+            Single color colormaps will be generated from the colors in the list or
+            colors taken from the colormap, and passed to the `cmap` parameter of
+            the `plot_method`.
+
+            .. versionadded:: 1.7
+
+        xlabel : str, default=None
+            The label used for the x-axis. If `None`, an attempt is made to
+            extract a label from `X` if it is a dataframe, otherwise an empty
+            string is used.
+
+        ylabel : str, default=None
+            The label used for the y-axis. If `None`, an attempt is made to
+            extract a label from `X` if it is a dataframe, otherwise an empty
+            string is used.
+
+        ax : Matplotlib axes, default=None
+            Axes object to plot on. If `None`, a new figure and axes is
+            created.
+
+        **kwargs : dict
+            Additional keyword arguments to be passed to the
+            `plot_method`.
+
+        Returns
+        -------
+        display : :class:`~sklearn.inspection.DecisionBoundaryDisplay`
+            Object that stores the result.
+
+        See Also
+        --------
+        DecisionBoundaryDisplay : Decision boundary visualization.
+        sklearn.metrics.ConfusionMatrixDisplay.from_estimator : Plot the
+            confusion matrix given an estimator, the data, and the label.
+        sklearn.metrics.ConfusionMatrixDisplay.from_predictions : Plot the
+            confusion matrix given the true and predicted labels.
+
+        Examples
+        --------
+        >>> import matplotlib.pyplot as plt
+        >>> from sklearn.datasets import load_iris
+        >>> from sklearn.linear_model import LogisticRegression
+        >>> from sklearn.inspection import DecisionBoundaryDisplay
+        >>> iris = load_iris()
+        >>> X = iris.data[:, :2]
+        >>> classifier = LogisticRegression().fit(X, iris.target)
+        >>> disp = DecisionBoundaryDisplay.from_estimator(
+        ...     classifier, X, response_method="predict",
+        ...     xlabel=iris.feature_names[0], ylabel=iris.feature_names[1],
+        ...     alpha=0.5,
+        ... )
+        >>> disp.ax_.scatter(X[:, 0], X[:, 1], c=iris.target, edgecolor="k")
+        <...>
+        >>> plt.show()
+        """
+        check_matplotlib_support(f"{cls.__name__}.from_estimator")
+        check_is_fitted(estimator)
+        import matplotlib as mpl
+
+        if not grid_resolution > 1:
+            raise ValueError(
+                "grid_resolution must be greater than 1. Got"
+                f" {grid_resolution} instead."
+            )
+
+        if not eps >= 0:
+            raise ValueError(
+                f"eps must be greater than or equal to 0. Got {eps} instead."
+            )
+
+        possible_plot_methods = ("contourf", "contour", "pcolormesh")
+        if plot_method not in possible_plot_methods:
+            available_methods = ", ".join(possible_plot_methods)
+            raise ValueError(
+                f"plot_method must be one of {available_methods}. "
+                f"Got {plot_method} instead."
+            )
+
+        num_features = _num_features(X)
+        if num_features != 2:
+            raise ValueError(
+                f"n_features must be equal to 2. Got {num_features} instead."
+            )
+
+        if (
+            response_method in ("predict_proba", "decision_function", "auto")
+            and multiclass_colors is not None
+            and hasattr(estimator, "classes_")
+            and (n_classes := len(estimator.classes_)) > 2
+        ):
+            if isinstance(multiclass_colors, list):
+                if len(multiclass_colors) != n_classes:
+                    raise ValueError(
+                        "When 'multiclass_colors' is a list, it must be of the same "
+                        f"length as 'estimator.classes_' ({n_classes}), got: "
+                        f"{len(multiclass_colors)}."
+                    )
+                elif any(
+                    not mpl.colors.is_color_like(col) for col in multiclass_colors
+                ):
+                    raise ValueError(
+                        "When 'multiclass_colors' is a list, it can only contain valid"
+                        f" Matplotlib color names. Got: {multiclass_colors}"
+                    )
+            if isinstance(multiclass_colors, str):
+                if multiclass_colors not in mpl.pyplot.colormaps():
+                    raise ValueError(
+                        "When 'multiclass_colors' is a string, it must be a valid "
+                        f"Matplotlib colormap. Got: {multiclass_colors}"
+                    )
+
+        x0, x1 = _safe_indexing(X, 0, axis=1), _safe_indexing(X, 1, axis=1)
+
+        x0_min, x0_max = x0.min() - eps, x0.max() + eps
+        x1_min, x1_max = x1.min() - eps, x1.max() + eps
+
+        xx0, xx1 = np.meshgrid(
+            np.linspace(x0_min, x0_max, grid_resolution),
+            np.linspace(x1_min, x1_max, grid_resolution),
+        )
+
+        X_grid = np.c_[xx0.ravel(), xx1.ravel()]
+        if _is_pandas_df(X) or _is_polars_df(X):
+            adapter = _get_adapter_from_container(X)
+            X_grid = adapter.create_container(
+                X_grid,
+                X_grid,
+                columns=X.columns,
+            )
+
+        prediction_method = _check_boundary_response_method(
+            estimator, response_method, class_of_interest
+        )
+        try:
+            response, _, response_method_used = _get_response_values(
+                estimator,
+                X_grid,
+                response_method=prediction_method,
+                pos_label=class_of_interest,
+                return_response_method_used=True,
+            )
+        except ValueError as exc:
+            if "is not a valid label" in str(exc):
+                # re-raise a more informative error message since `pos_label` is unknown
+                # to our user when interacting with
+                # `DecisionBoundaryDisplay.from_estimator`
+                raise ValueError(
+                    f"class_of_interest={class_of_interest} is not a valid label: It "
+                    f"should be one of {estimator.classes_}"
+                ) from exc
+            raise
+
+        # convert classes predictions into integers
+        if response_method_used == "predict" and hasattr(estimator, "classes_"):
+            encoder = LabelEncoder()
+            encoder.classes_ = estimator.classes_
+            response = encoder.transform(response)
+
+        if response.ndim == 1:
+            response = response.reshape(*xx0.shape)
+        else:
+            if is_regressor(estimator):
+                raise ValueError("Multi-output regressors are not supported")
+
+            if class_of_interest is not None:
+                # For the multiclass case, `_get_response_values` returns the response
+                # as-is. Thus, we have a column per class and we need to select the
+                # column corresponding to the positive class.
+                col_idx = np.flatnonzero(estimator.classes_ == class_of_interest)[0]
+                response = response[:, col_idx].reshape(*xx0.shape)
+            else:
+                response = response.reshape(*xx0.shape, response.shape[-1])
+
+        if xlabel is None:
+            xlabel = X.columns[0] if hasattr(X, "columns") else ""
+
+        if ylabel is None:
+            ylabel = X.columns[1] if hasattr(X, "columns") else ""
+
+        display = cls(
+            xx0=xx0,
+            xx1=xx1,
+            response=response,
+            multiclass_colors=multiclass_colors,
+            xlabel=xlabel,
+            ylabel=ylabel,
+        )
+        return display.plot(ax=ax, plot_method=plot_method, **kwargs)
diff --git a/sklearn/inspection/_plot/partial_dependence.py b/sklearn/inspection/_plot/partial_dependence.py
index d6604d7ae675f..b31a5070b236b 100644
--- a/sklearn/inspection/_plot/partial_dependence.py
+++ b/sklearn/inspection/_plot/partial_dependence.py
@@ -1,3 +1,6 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
 import numbers
 from itertools import chain
 from math import ceil
@@ -5,420 +8,39 @@
 import numpy as np
 from scipy import sparse
 from scipy.stats.mstats import mquantiles
-from joblib import Parallel
 
-from .. import partial_dependence
 from ...base import is_regressor
-from ...utils import check_array
-from ...utils import check_matplotlib_support  # noqa
-from ...utils import check_random_state
-from ...utils import _safe_indexing
-from ...utils.validation import _deprecate_positional_args
-from ...utils.fixes import delayed
-
-
-@_deprecate_positional_args
-def plot_partial_dependence(
-    estimator,
-    X,
-    features,
-    *,
-    feature_names=None,
-    target=None,
-    response_method="auto",
-    n_cols=3,
-    grid_resolution=100,
-    percentiles=(0.05, 0.95),
-    method="auto",
-    n_jobs=None,
-    verbose=0,
-    line_kw=None,
-    contour_kw=None,
-    ax=None,
-    kind="average",
-    subsample=1000,
-    random_state=None,
-):
-    """Partial dependence (PD) and individual conditional expectation (ICE)
-    plots.
-
-    Partial dependence plots, individual conditional expectation plots or an
-    overlay of both of them can be plotted by setting the ``kind``
-    parameter.
-    The ``len(features)`` plots are arranged in a grid with ``n_cols``
-    columns. Two-way partial dependence plots are plotted as contour plots. The
-    deciles of the feature values will be shown with tick marks on the x-axes
-    for one-way plots, and on both axes for two-way plots.
-
-    Read more in the :ref:`User Guide `.
-
-    .. note::
-
-        :func:`plot_partial_dependence` does not support using the same axes
-        with multiple calls. To plot the the partial dependence for multiple
-        estimators, please pass the axes created by the first call to the
-        second call::
-
-          >>> from sklearn.inspection import plot_partial_dependence
-          >>> from sklearn.datasets import make_friedman1
-          >>> from sklearn.linear_model import LinearRegression
-          >>> from sklearn.ensemble import RandomForestRegressor
-          >>> X, y = make_friedman1()
-          >>> est1 = LinearRegression().fit(X, y)
-          >>> est2 = RandomForestRegressor().fit(X, y)
-          >>> disp1 = plot_partial_dependence(est1, X,
-          ...                                 [1, 2])  # doctest: +SKIP
-          >>> disp2 = plot_partial_dependence(est2, X, [1, 2],
-          ...                                 ax=disp1.axes_)  # doctest: +SKIP
-
-    .. warning::
-
-        For :class:`~sklearn.ensemble.GradientBoostingClassifier` and
-        :class:`~sklearn.ensemble.GradientBoostingRegressor`, the
-        `'recursion'` method (used by default) will not account for the `init`
-        predictor of the boosting process. In practice, this will produce
-        the same values as `'brute'` up to a constant offset in the target
-        response, provided that `init` is a constant estimator (which is the
-        default). However, if `init` is not a constant estimator, the
-        partial dependence values are incorrect for `'recursion'` because the
-        offset will be sample-dependent. It is preferable to use the `'brute'`
-        method. Note that this only applies to
-        :class:`~sklearn.ensemble.GradientBoostingClassifier` and
-        :class:`~sklearn.ensemble.GradientBoostingRegressor`, not to
-        :class:`~sklearn.ensemble.HistGradientBoostingClassifier` and
-        :class:`~sklearn.ensemble.HistGradientBoostingRegressor`.
-
-    Parameters
-    ----------
-    estimator : BaseEstimator
-        A fitted estimator object implementing :term:`predict`,
-        :term:`predict_proba`, or :term:`decision_function`.
-        Multioutput-multiclass classifiers are not supported.
-
-    X : {array-like or dataframe} of shape (n_samples, n_features)
-        ``X`` is used to generate a grid of values for the target
-        ``features`` (where the partial dependence will be evaluated), and
-        also to generate values for the complement features when the
-        `method` is `'brute'`.
-
-    features : list of {int, str, pair of int, pair of str}
-        The target features for which to create the PDPs.
-        If `features[i]` is an integer or a string, a one-way PDP is created;
-        if `features[i]` is a tuple, a two-way PDP is created (only supported
-        with `kind='average'`). Each tuple must be of size 2.
-        if any entry is a string, then it must be in ``feature_names``.
-
-    feature_names : array-like of shape (n_features,), dtype=str, default=None
-        Name of each feature; `feature_names[i]` holds the name of the feature
-        with index `i`.
-        By default, the name of the feature corresponds to their numerical
-        index for NumPy array and their column name for pandas dataframe.
-
-    target : int, default=None
-        - In a multiclass setting, specifies the class for which the PDPs
-          should be computed. Note that for binary classification, the
-          positive class (index 1) is always used.
-        - In a multioutput setting, specifies the task for which the PDPs
-          should be computed.
-
-        Ignored in binary classification or classical regression settings.
-
-    response_method : {'auto', 'predict_proba', 'decision_function'}, \
-            default='auto'
-        Specifies whether to use :term:`predict_proba` or
-        :term:`decision_function` as the target response. For regressors
-        this parameter is ignored and the response is always the output of
-        :term:`predict`. By default, :term:`predict_proba` is tried first
-        and we revert to :term:`decision_function` if it doesn't exist. If
-        ``method`` is `'recursion'`, the response is always the output of
-        :term:`decision_function`.
-
-    n_cols : int, default=3
-        The maximum number of columns in the grid plot. Only active when `ax`
-        is a single axis or `None`.
-
-    grid_resolution : int, default=100
-        The number of equally spaced points on the axes of the plots, for each
-        target feature.
-
-    percentiles : tuple of float, default=(0.05, 0.95)
-        The lower and upper percentile used to create the extreme values
-        for the PDP axes. Must be in [0, 1].
-
-    method : str, default='auto'
-        The method used to calculate the averaged predictions:
-
-        - `'recursion'` is only supported for some tree-based estimators
-          (namely
-          :class:`~sklearn.ensemble.GradientBoostingClassifier`,
-          :class:`~sklearn.ensemble.GradientBoostingRegressor`,
-          :class:`~sklearn.ensemble.HistGradientBoostingClassifier`,
-          :class:`~sklearn.ensemble.HistGradientBoostingRegressor`,
-          :class:`~sklearn.tree.DecisionTreeRegressor`,
-          :class:`~sklearn.ensemble.RandomForestRegressor`
-          but is more efficient in terms of speed.
-          With this method, the target response of a
-          classifier is always the decision function, not the predicted
-          probabilities. Since the `'recursion'` method implicitely computes
-          the average of the ICEs by design, it is not compatible with ICE and
-          thus `kind` must be `'average'`.
-
-        - `'brute'` is supported for any estimator, but is more
-          computationally intensive.
-
-        - `'auto'`: the `'recursion'` is used for estimators that support it,
-          and `'brute'` is used otherwise.
-
-        Please see :ref:`this note ` for
-        differences between the `'brute'` and `'recursion'` method.
-
-    n_jobs : int, default=None
-        The number of CPUs to use to compute the partial dependences.
-        ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
-        ``-1`` means using all processors. See :term:`Glossary `
-        for more details.
-
-    verbose : int, default=0
-        Verbose output during PD computations.
-
-    line_kw : dict, default=None
-        Dict with keywords passed to the ``matplotlib.pyplot.plot`` call.
-        For one-way partial dependence plots.
-
-    contour_kw : dict, default=None
-        Dict with keywords passed to the ``matplotlib.pyplot.contourf`` call.
-        For two-way partial dependence plots.
-
-    ax : Matplotlib axes or array-like of Matplotlib axes, default=None
-        - If a single axis is passed in, it is treated as a bounding axes
-          and a grid of partial dependence plots will be drawn within
-          these bounds. The `n_cols` parameter controls the number of
-          columns in the grid.
-        - If an array-like of axes are passed in, the partial dependence
-          plots will be drawn directly into these axes.
-        - If `None`, a figure and a bounding axes is created and treated
-          as the single axes case.
-
-        .. versionadded:: 0.22
-
-    kind : {'average', 'individual', 'both'}, default='average'
-        Whether to plot the partial dependence averaged across all the samples
-        in the dataset or one line per sample or both.
-
-        - ``kind='average'`` results in the traditional PD plot;
-        - ``kind='individual'`` results in the ICE plot.
-
-       Note that the fast ``method='recursion'`` option is only available for
-       ``kind='average'``. Plotting individual dependencies requires using the
-       slower ``method='brute'`` option.
-
-        .. versionadded:: 0.24
-
-    subsample : float, int or None, default=1000
-        Sampling for ICE curves when `kind` is 'individual' or 'both'.
-        If `float`, should be between 0.0 and 1.0 and represent the proportion
-        of the dataset to be used to plot ICE curves. If `int`, represents the
-        absolute number samples to use.
-
-        Note that the full dataset is still used to calculate averaged partial
-        dependence when `kind='both'`.
-
-        .. versionadded:: 0.24
-
-    random_state : int, RandomState instance or None, default=None
-        Controls the randomness of the selected samples when subsamples is not
-        `None` and `kind` is either `'both'` or `'individual'`.
-        See :term:`Glossary ` for details.
-
-        .. versionadded:: 0.24
-
-    Returns
-    -------
-    display : :class:`~sklearn.inspection.PartialDependenceDisplay`
-
-    See Also
-    --------
-    partial_dependence : Compute Partial Dependence values.
-    PartialDependenceDisplay : Partial Dependence visualization.
-
-    Examples
-    --------
-    >>> from sklearn.datasets import make_friedman1
-    >>> from sklearn.ensemble import GradientBoostingRegressor
-    >>> X, y = make_friedman1()
-    >>> clf = GradientBoostingRegressor(n_estimators=10).fit(X, y)
-    >>> plot_partial_dependence(clf, X, [0, (0, 1)]) #doctest: +SKIP
-    """
-    check_matplotlib_support('plot_partial_dependence')  # noqa
-    import matplotlib.pyplot as plt  # noqa
-
-    # set target_idx for multi-class estimators
-    if hasattr(estimator, 'classes_') and np.size(estimator.classes_) > 2:
-        if target is None:
-            raise ValueError('target must be specified for multi-class')
-        target_idx = np.searchsorted(estimator.classes_, target)
-        if (not (0 <= target_idx < len(estimator.classes_)) or
-                estimator.classes_[target_idx] != target):
-            raise ValueError('target not in est.classes_, got {}'.format(
-                target))
-    else:
-        # regression and binary classification
-        target_idx = 0
-
-    # Use check_array only on lists and other non-array-likes / sparse. Do not
-    # convert DataFrame into a NumPy array.
-    if not(hasattr(X, '__array__') or sparse.issparse(X)):
-        X = check_array(X, force_all_finite='allow-nan', dtype=object)
-    n_features = X.shape[1]
-
-    # convert feature_names to list
-    if feature_names is None:
-        if hasattr(X, "loc"):
-            # get the column names for a pandas dataframe
-            feature_names = X.columns.tolist()
-        else:
-            # define a list of numbered indices for a numpy array
-            feature_names = [str(i) for i in range(n_features)]
-    elif hasattr(feature_names, "tolist"):
-        # convert numpy array or pandas index to a list
-        feature_names = feature_names.tolist()
-    if len(set(feature_names)) != len(feature_names):
-        raise ValueError('feature_names should not contain duplicates.')
-
-    def convert_feature(fx):
-        if isinstance(fx, str):
-            try:
-                fx = feature_names.index(fx)
-            except ValueError as e:
-                raise ValueError('Feature %s not in feature_names' % fx) from e
-        return int(fx)
-
-    # convert features into a seq of int tuples
-    tmp_features = []
-    for fxs in features:
-        if isinstance(fxs, (numbers.Integral, str)):
-            fxs = (fxs,)
-        try:
-            fxs = tuple(convert_feature(fx) for fx in fxs)
-        except TypeError as e:
-            raise ValueError(
-                'Each entry in features must be either an int, '
-                'a string, or an iterable of size at most 2.'
-            ) from e
-        if not 1 <= np.size(fxs) <= 2:
-            raise ValueError('Each entry in features must be either an int, '
-                             'a string, or an iterable of size at most 2.')
-        if kind != 'average' and np.size(fxs) > 1:
-            raise ValueError(
-                f"It is not possible to display individual effects for more "
-                f"than one feature at a time. Got: features={features}.")
-        tmp_features.append(fxs)
-
-    features = tmp_features
-
-    # Early exit if the axes does not have the correct number of axes
-    if ax is not None and not isinstance(ax, plt.Axes):
-        axes = np.asarray(ax, dtype=object)
-        if axes.size != len(features):
-            raise ValueError("Expected ax to have {} axes, got {}".format(
-                             len(features), axes.size))
-
-    for i in chain.from_iterable(features):
-        if i >= len(feature_names):
-            raise ValueError('All entries of features must be less than '
-                             'len(feature_names) = {0}, got {1}.'
-                             .format(len(feature_names), i))
-
-    if isinstance(subsample, numbers.Integral):
-        if subsample <= 0:
-            raise ValueError(
-                f"When an integer, subsample={subsample} should be positive."
-            )
-    elif isinstance(subsample, numbers.Real):
-        if subsample <= 0 or subsample >= 1:
-            raise ValueError(
-                f"When a floating-point, subsample={subsample} should be in "
-                f"the (0, 1) range."
-            )
-
-    # compute predictions and/or averaged predictions
-    pd_results = Parallel(n_jobs=n_jobs, verbose=verbose)(
-        delayed(partial_dependence)(estimator, X, fxs,
-                                    response_method=response_method,
-                                    method=method,
-                                    grid_resolution=grid_resolution,
-                                    percentiles=percentiles,
-                                    kind=kind)
-        for fxs in features)
-
-    # For multioutput regression, we can only check the validity of target
-    # now that we have the predictions.
-    # Also note: as multiclass-multioutput classifiers are not supported,
-    # multiclass and multioutput scenario are mutually exclusive. So there is
-    # no risk of overwriting target_idx here.
-    pd_result = pd_results[0]  # checking the first result is enough
-    n_tasks = (pd_result.average.shape[0] if kind == 'average'
-               else pd_result.individual.shape[0])
-    if is_regressor(estimator) and n_tasks > 1:
-        if target is None:
-            raise ValueError(
-                'target must be specified for multi-output regressors')
-        if not 0 <= target <= n_tasks:
-            raise ValueError(
-                'target must be in [0, n_tasks], got {}.'.format(target))
-        target_idx = target
-
-    # get global min and max average predictions of PD grouped by plot type
-    pdp_lim = {}
-    for pdp in pd_results:
-        values = pdp["values"]
-        preds = (pdp.average if kind == 'average' else pdp.individual)
-        min_pd = preds[target_idx].min()
-        max_pd = preds[target_idx].max()
-        n_fx = len(values)
-        old_min_pd, old_max_pd = pdp_lim.get(n_fx, (min_pd, max_pd))
-        min_pd = min(min_pd, old_min_pd)
-        max_pd = max(max_pd, old_max_pd)
-        pdp_lim[n_fx] = (min_pd, max_pd)
-
-    deciles = {}
-    for fx in chain.from_iterable(features):
-        if fx not in deciles:
-            X_col = _safe_indexing(X, fx, axis=1)
-            deciles[fx] = mquantiles(X_col, prob=np.arange(0.1, 1.0, 0.1))
-
-    display = PartialDependenceDisplay(
-        pd_results=pd_results,
-        features=features,
-        feature_names=feature_names,
-        target_idx=target_idx,
-        pdp_lim=pdp_lim,
-        deciles=deciles,
-        kind=kind,
-        subsample=subsample,
-        random_state=random_state,
-    )
-    return display.plot(
-        ax=ax, n_cols=n_cols, line_kw=line_kw, contour_kw=contour_kw
-    )
+from ...utils import (
+    Bunch,
+    _safe_indexing,
+    check_array,
+    check_random_state,
+)
+from ...utils._encode import _unique
+from ...utils._optional_dependencies import check_matplotlib_support
+from ...utils._plotting import _validate_style_kwargs
+from ...utils.parallel import Parallel, delayed
+from .. import partial_dependence
+from .._pd_utils import _check_feature_names, _get_feature_index
 
 
 class PartialDependenceDisplay:
-    """Partial Dependence Plot (PDP).
-
-    This can also display individual partial dependencies which are often
-    referred to as: Individual Condition Expectation (ICE).
+    """Partial Dependence Plot (PDP) and Individual Conditional Expectation (ICE).
 
     It is recommended to use
-    :func:`~sklearn.inspection.plot_partial_dependence` to create a
-    :class:`~sklearn.inspection.PartialDependenceDisplay`. All parameters are
-    stored as attributes.
+    :func:`~sklearn.inspection.PartialDependenceDisplay.from_estimator` to create a
+    :class:`~sklearn.inspection.PartialDependenceDisplay`. All parameters are stored
+    as attributes.
+
+    For general information regarding `scikit-learn` visualization tools, see
+    the :ref:`Visualization Guide `.
+    For guidance on interpreting these plots, refer to the
+    :ref:`Inspection Guide `.
 
-    Read more in
-    :ref:`sphx_glr_auto_examples_miscellaneous_plot_partial_dependence_visualization_api.py`
-    and the :ref:`User Guide `.
+    For an example on how to use this class, see the following example:
+    :ref:`sphx_glr_auto_examples_miscellaneous_plot_partial_dependence_visualization_api.py`.
 
-        .. versionadded:: 0.22
+    .. versionadded:: 0.22
 
     Parameters
     ----------
@@ -444,27 +66,42 @@ class PartialDependenceDisplay:
 
         Ignored in binary classification or classical regression settings.
 
-    pdp_lim : dict
-        Global min and max average predictions, such that all plots will have
-        the same scale and y limits. `pdp_lim[1]` is the global min and max for
-        single partial dependence curves. `pdp_lim[2]` is the global min and
-        max for two-way partial dependence curves.
-
     deciles : dict
         Deciles for feature indices in ``features``.
 
-    kind : {'average', 'individual', 'both'}, default='average'
+    kind : {'average', 'individual', 'both'} or list of such str, \
+            default='average'
         Whether to plot the partial dependence averaged across all the samples
         in the dataset or one line per sample or both.
 
         - ``kind='average'`` results in the traditional PD plot;
-        - ``kind='individual'`` results in the ICE plot.
-
-       Note that the fast ``method='recursion'`` option is only available for
-       ``kind='average'``. Plotting individual dependencies requires using the
-       slower ``method='brute'`` option.
+        - ``kind='individual'`` results in the ICE plot;
+        - ``kind='both'`` results in plotting both the ICE and PD on the same
+          plot.
+
+        A list of such strings can be provided to specify `kind` on a per-plot
+        basis. The length of the list should be the same as the number of
+        interaction requested in `features`.
+
+        .. note::
+           ICE ('individual' or 'both') is not a valid option for 2-ways
+           interactions plot. As a result, an error will be raised.
+           2-ways interaction plots should always be configured to
+           use the 'average' kind instead.
+
+        .. note::
+           The fast ``method='recursion'`` option is only available for
+           `kind='average'` and `sample_weights=None`. Computing individual
+           dependencies and doing weighted averages requires using the slower
+           `method='brute'`.
 
         .. versionadded:: 0.24
+           Add `kind` parameter with `'average'`, `'individual'`, and `'both'`
+           options.
+
+        .. versionadded:: 1.1
+           Add the possibility to pass a list of string specifying `kind`
+           for each plot.
 
     subsample : float, int or None, default=1000
         Sampling for ICE curves when `kind` is 'individual' or 'both'.
@@ -483,6 +120,13 @@ class PartialDependenceDisplay:
 
         .. versionadded:: 0.24
 
+    is_categorical : list of (bool,) or list of (bool, bool), default=None
+        Whether each target feature in `features` is categorical or not.
+        The list should be same size as `features`. If `None`, all features
+        are assumed to be continuous.
+
+        .. versionadded:: 1.2
+
     Attributes
     ----------
     bounding_ax_ : matplotlib Axes or None
@@ -528,15 +172,57 @@ class PartialDependenceDisplay:
         item in `ax`. Elements that are None correspond to a nonexisting axes
         or an axes that does not include a contour plot.
 
+    bars_ : ndarray of matplotlib Artists
+        If `ax` is an axes or None, `bars_[i, j]` is the partial dependence bar
+        plot on the i-th row and j-th column (for a categorical feature).
+        If `ax` is a list of axes, `bars_[i]` is the partial dependence bar
+        plot corresponding to the i-th item in `ax`. Elements that are None
+        correspond to a nonexisting axes or an axes that does not include a
+        bar plot.
+
+        .. versionadded:: 1.2
+
+    heatmaps_ : ndarray of matplotlib Artists
+        If `ax` is an axes or None, `heatmaps_[i, j]` is the partial dependence
+        heatmap on the i-th row and j-th column (for a pair of categorical
+        features) . If `ax` is a list of axes, `heatmaps_[i]` is the partial
+        dependence heatmap corresponding to the i-th item in `ax`. Elements
+        that are None correspond to a nonexisting axes or an axes that does not
+        include a heatmap.
+
+        .. versionadded:: 1.2
+
     figure_ : matplotlib Figure
         Figure containing partial dependence plots.
 
     See Also
     --------
     partial_dependence : Compute Partial Dependence values.
-    plot_partial_dependence : Plot Partial Dependence.
+    PartialDependenceDisplay.from_estimator : Plot Partial Dependence.
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> import matplotlib.pyplot as plt
+    >>> from sklearn.datasets import make_friedman1
+    >>> from sklearn.ensemble import GradientBoostingRegressor
+    >>> from sklearn.inspection import PartialDependenceDisplay
+    >>> from sklearn.inspection import partial_dependence
+    >>> X, y = make_friedman1()
+    >>> clf = GradientBoostingRegressor(n_estimators=10).fit(X, y)
+    >>> features, feature_names = [(0,)], [f"Features #{i}" for i in range(X.shape[1])]
+    >>> deciles = {0: np.linspace(0, 1, num=5)}
+    >>> pd_results = partial_dependence(
+    ...     clf, X, features=0, kind="average", grid_resolution=5)
+    >>> display = PartialDependenceDisplay(
+    ...     [pd_results], features=features, feature_names=feature_names,
+    ...     target_idx=0, deciles=deciles
+    ... )
+    >>> display.plot(pdp_lim={1: (-1.38, 0.66)})
+    <...>
+    >>> plt.show()
     """
-    @_deprecate_positional_args
+
     def __init__(
         self,
         pd_results,
@@ -544,21 +230,563 @@ def __init__(
         features,
         feature_names,
         target_idx,
-        pdp_lim,
         deciles,
         kind="average",
         subsample=1000,
         random_state=None,
+        is_categorical=None,
     ):
         self.pd_results = pd_results
         self.features = features
         self.feature_names = feature_names
         self.target_idx = target_idx
-        self.pdp_lim = pdp_lim
         self.deciles = deciles
         self.kind = kind
         self.subsample = subsample
         self.random_state = random_state
+        self.is_categorical = is_categorical
+
+    @classmethod
+    def from_estimator(
+        cls,
+        estimator,
+        X,
+        features,
+        *,
+        sample_weight=None,
+        categorical_features=None,
+        feature_names=None,
+        target=None,
+        response_method="auto",
+        n_cols=3,
+        grid_resolution=100,
+        percentiles=(0.05, 0.95),
+        custom_values=None,
+        method="auto",
+        n_jobs=None,
+        verbose=0,
+        line_kw=None,
+        ice_lines_kw=None,
+        pd_line_kw=None,
+        contour_kw=None,
+        ax=None,
+        kind="average",
+        centered=False,
+        subsample=1000,
+        random_state=None,
+    ):
+        """Partial dependence (PD) and individual conditional expectation (ICE) plots.
+
+        Partial dependence plots, individual conditional expectation plots, or an
+        overlay of both can be plotted by setting the `kind` parameter.
+        This method generates one plot for each entry in `features`. The plots
+        are arranged in a grid with `n_cols` columns. For one-way partial
+        dependence plots, the deciles of the feature values are shown on the
+        x-axis. For two-way plots, the deciles are shown on both axes and PDPs
+        are contour plots.
+
+        For general information regarding `scikit-learn` visualization tools, see
+        the :ref:`Visualization Guide `.
+        For guidance on interpreting these plots, refer to the
+        :ref:`Inspection Guide `.
+
+        For an example on how to use this class method, see
+        :ref:`sphx_glr_auto_examples_inspection_plot_partial_dependence.py`.
+
+        .. note::
+
+            :func:`PartialDependenceDisplay.from_estimator` does not support using the
+            same axes with multiple calls. To plot the partial dependence for
+            multiple estimators, please pass the axes created by the first call to the
+            second call::
+
+               >>> from sklearn.inspection import PartialDependenceDisplay
+               >>> from sklearn.datasets import make_friedman1
+               >>> from sklearn.linear_model import LinearRegression
+               >>> from sklearn.ensemble import RandomForestRegressor
+               >>> X, y = make_friedman1()
+               >>> est1 = LinearRegression().fit(X, y)
+               >>> est2 = RandomForestRegressor().fit(X, y)
+               >>> disp1 = PartialDependenceDisplay.from_estimator(est1, X,
+               ...                                                 [1, 2])
+               >>> disp2 = PartialDependenceDisplay.from_estimator(est2, X, [1, 2],
+               ...                                                 ax=disp1.axes_)
+
+        .. warning::
+
+            For :class:`~sklearn.ensemble.GradientBoostingClassifier` and
+            :class:`~sklearn.ensemble.GradientBoostingRegressor`, the
+            `'recursion'` method (used by default) will not account for the `init`
+            predictor of the boosting process. In practice, this will produce
+            the same values as `'brute'` up to a constant offset in the target
+            response, provided that `init` is a constant estimator (which is the
+            default). However, if `init` is not a constant estimator, the
+            partial dependence values are incorrect for `'recursion'` because the
+            offset will be sample-dependent. It is preferable to use the `'brute'`
+            method. Note that this only applies to
+            :class:`~sklearn.ensemble.GradientBoostingClassifier` and
+            :class:`~sklearn.ensemble.GradientBoostingRegressor`, not to
+            :class:`~sklearn.ensemble.HistGradientBoostingClassifier` and
+            :class:`~sklearn.ensemble.HistGradientBoostingRegressor`.
+
+        .. versionadded:: 1.0
+
+        Parameters
+        ----------
+        estimator : BaseEstimator
+            A fitted estimator object implementing :term:`predict`,
+            :term:`predict_proba`, or :term:`decision_function`.
+            Multioutput-multiclass classifiers are not supported.
+
+        X : {array-like, dataframe} of shape (n_samples, n_features)
+            ``X`` is used to generate a grid of values for the target
+            ``features`` (where the partial dependence will be evaluated), and
+            also to generate values for the complement features when the
+            `method` is `'brute'`.
+
+        features : list of {int, str, pair of int, pair of str}
+            The target features for which to create the PDPs.
+            If `features[i]` is an integer or a string, a one-way PDP is created;
+            if `features[i]` is a tuple, a two-way PDP is created (only supported
+            with `kind='average'`). Each tuple must be of size 2.
+            If any entry is a string, then it must be in ``feature_names``.
+
+        sample_weight : array-like of shape (n_samples,), default=None
+            Sample weights are used to calculate weighted means when averaging the
+            model output. If `None`, then samples are equally weighted. If
+            `sample_weight` is not `None`, then `method` will be set to `'brute'`.
+            Note that `sample_weight` is ignored for `kind='individual'`.
+
+            .. versionadded:: 1.3
+
+        categorical_features : array-like of shape (n_features,) or shape \
+                (n_categorical_features,), dtype={bool, int, str}, default=None
+            Indicates the categorical features.
+
+            - `None`: no feature will be considered categorical;
+            - boolean array-like: boolean mask of shape `(n_features,)`
+              indicating which features are categorical. Thus, this array has
+              the same shape has `X.shape[1]`;
+            - integer or string array-like: integer indices or strings
+              indicating categorical features.
+
+            .. versionadded:: 1.2
+
+        feature_names : array-like of shape (n_features,), dtype=str, default=None
+            Name of each feature; `feature_names[i]` holds the name of the feature
+            with index `i`.
+            By default, the name of the feature corresponds to their numerical
+            index for NumPy array and their column name for pandas dataframe.
+
+        target : int, default=None
+            - In a multiclass setting, specifies the class for which the PDPs
+              should be computed. Note that for binary classification, the
+              positive class (index 1) is always used.
+            - In a multioutput setting, specifies the task for which the PDPs
+              should be computed.
+
+            Ignored in binary classification or classical regression settings.
+
+        response_method : {'auto', 'predict_proba', 'decision_function'}, \
+                default='auto'
+            Specifies whether to use :term:`predict_proba` or
+            :term:`decision_function` as the target response. For regressors
+            this parameter is ignored and the response is always the output of
+            :term:`predict`. By default, :term:`predict_proba` is tried first
+            and we revert to :term:`decision_function` if it doesn't exist. If
+            ``method`` is `'recursion'`, the response is always the output of
+            :term:`decision_function`.
+
+        n_cols : int, default=3
+            The maximum number of columns in the grid plot. Only active when `ax`
+            is a single axis or `None`.
+
+        grid_resolution : int, default=100
+            The number of equally spaced points on the axes of the plots, for each
+            target feature.
+            This parameter is overridden by `custom_values` if that parameter is set.
+
+        percentiles : tuple of float, default=(0.05, 0.95)
+            The lower and upper percentile used to create the extreme values
+            for the PDP axes. Must be in [0, 1].
+            This parameter is overridden by `custom_values` if that parameter is set.
+
+        custom_values : dict
+            A dictionary mapping the index of an element of `features` to an
+            array of values where the partial dependence should be calculated
+            for that feature. Setting a range of values for a feature overrides
+            `grid_resolution` and `percentiles`.
+
+            .. versionadded:: 1.7
+
+        method : str, default='auto'
+            The method used to calculate the averaged predictions:
+
+            - `'recursion'` is only supported for some tree-based estimators
+              (namely
+              :class:`~sklearn.ensemble.GradientBoostingClassifier`,
+              :class:`~sklearn.ensemble.GradientBoostingRegressor`,
+              :class:`~sklearn.ensemble.HistGradientBoostingClassifier`,
+              :class:`~sklearn.ensemble.HistGradientBoostingRegressor`,
+              :class:`~sklearn.tree.DecisionTreeRegressor`,
+              :class:`~sklearn.ensemble.RandomForestRegressor`
+              but is more efficient in terms of speed.
+              With this method, the target response of a
+              classifier is always the decision function, not the predicted
+              probabilities. Since the `'recursion'` method implicitly computes
+              the average of the ICEs by design, it is not compatible with ICE and
+              thus `kind` must be `'average'`.
+
+            - `'brute'` is supported for any estimator, but is more
+              computationally intensive.
+
+            - `'auto'`: the `'recursion'` is used for estimators that support it,
+              and `'brute'` is used otherwise. If `sample_weight` is not `None`,
+              then `'brute'` is used regardless of the estimator.
+
+            Please see :ref:`this note ` for
+            differences between the `'brute'` and `'recursion'` method.
+
+        n_jobs : int, default=None
+            The number of CPUs to use to compute the partial dependences.
+            Computation is parallelized over features specified by the `features`
+            parameter.
+
+            ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
+            ``-1`` means using all processors. See :term:`Glossary `
+            for more details.
+
+        verbose : int, default=0
+            Verbose output during PD computations.
+
+        line_kw : dict, default=None
+            Dict with keywords passed to the ``matplotlib.pyplot.plot`` call.
+            For one-way partial dependence plots. It can be used to define common
+            properties for both `ice_lines_kw` and `pdp_line_kw`.
+
+        ice_lines_kw : dict, default=None
+            Dictionary with keywords passed to the `matplotlib.pyplot.plot` call.
+            For ICE lines in the one-way partial dependence plots.
+            The key value pairs defined in `ice_lines_kw` takes priority over
+            `line_kw`.
+
+        pd_line_kw : dict, default=None
+            Dictionary with keywords passed to the `matplotlib.pyplot.plot` call.
+            For partial dependence in one-way partial dependence plots.
+            The key value pairs defined in `pd_line_kw` takes priority over
+            `line_kw`.
+
+        contour_kw : dict, default=None
+            Dict with keywords passed to the ``matplotlib.pyplot.contourf`` call.
+            For two-way partial dependence plots.
+
+        ax : Matplotlib axes or array-like of Matplotlib axes, default=None
+            - If a single axis is passed in, it is treated as a bounding axes
+              and a grid of partial dependence plots will be drawn within
+              these bounds. The `n_cols` parameter controls the number of
+              columns in the grid.
+            - If an array-like of axes are passed in, the partial dependence
+              plots will be drawn directly into these axes.
+            - If `None`, a figure and a bounding axes is created and treated
+              as the single axes case.
+
+        kind : {'average', 'individual', 'both'}, default='average'
+            Whether to plot the partial dependence averaged across all the samples
+            in the dataset or one line per sample or both.
+
+            - ``kind='average'`` results in the traditional PD plot;
+            - ``kind='individual'`` results in the ICE plot.
+
+            Note that the fast `method='recursion'` option is only available for
+            `kind='average'` and `sample_weights=None`. Computing individual
+            dependencies and doing weighted averages requires using the slower
+            `method='brute'`.
+
+        centered : bool, default=False
+            If `True`, the ICE and PD lines will start at the origin of the
+            y-axis. By default, no centering is done.
+
+            .. versionadded:: 1.1
+
+        subsample : float, int or None, default=1000
+            Sampling for ICE curves when `kind` is 'individual' or 'both'.
+            If `float`, should be between 0.0 and 1.0 and represent the proportion
+            of the dataset to be used to plot ICE curves. If `int`, represents the
+            absolute number samples to use.
+
+            Note that the full dataset is still used to calculate averaged partial
+            dependence when `kind='both'`.
+
+        random_state : int, RandomState instance or None, default=None
+            Controls the randomness of the selected samples when subsamples is not
+            `None` and `kind` is either `'both'` or `'individual'`.
+            See :term:`Glossary ` for details.
+
+        Returns
+        -------
+        display : :class:`~sklearn.inspection.PartialDependenceDisplay`
+
+        See Also
+        --------
+        partial_dependence : Compute Partial Dependence values.
+
+        Examples
+        --------
+        >>> import matplotlib.pyplot as plt
+        >>> from sklearn.datasets import make_friedman1
+        >>> from sklearn.ensemble import GradientBoostingRegressor
+        >>> from sklearn.inspection import PartialDependenceDisplay
+        >>> X, y = make_friedman1()
+        >>> clf = GradientBoostingRegressor(n_estimators=10).fit(X, y)
+        >>> PartialDependenceDisplay.from_estimator(clf, X, [0, (0, 1)])
+        <...>
+        >>> plt.show()
+        """
+        check_matplotlib_support(f"{cls.__name__}.from_estimator")
+        import matplotlib.pyplot as plt
+
+        # set target_idx for multi-class estimators
+        if hasattr(estimator, "classes_") and np.size(estimator.classes_) > 2:
+            if target is None:
+                raise ValueError("target must be specified for multi-class")
+            target_idx = np.searchsorted(estimator.classes_, target)
+            if (
+                not (0 <= target_idx < len(estimator.classes_))
+                or estimator.classes_[target_idx] != target
+            ):
+                raise ValueError("target not in est.classes_, got {}".format(target))
+        else:
+            # regression and binary classification
+            target_idx = 0
+
+        # Use check_array only on lists and other non-array-likes / sparse. Do not
+        # convert DataFrame into a NumPy array.
+        if not (hasattr(X, "__array__") or sparse.issparse(X)):
+            X = check_array(X, ensure_all_finite="allow-nan", dtype=object)
+        n_features = X.shape[1]
+
+        feature_names = _check_feature_names(X, feature_names)
+        # expand kind to always be a list of str
+        kind_ = [kind] * len(features) if isinstance(kind, str) else kind
+        if len(kind_) != len(features):
+            raise ValueError(
+                "When `kind` is provided as a list of strings, it should contain "
+                f"as many elements as `features`. `kind` contains {len(kind_)} "
+                f"element(s) and `features` contains {len(features)} element(s)."
+            )
+
+        # convert features into a seq of int tuples
+        tmp_features, ice_for_two_way_pd = [], []
+        for kind_plot, fxs in zip(kind_, features):
+            if isinstance(fxs, (numbers.Integral, str)):
+                fxs = (fxs,)
+            try:
+                fxs = tuple(
+                    _get_feature_index(fx, feature_names=feature_names) for fx in fxs
+                )
+            except TypeError as e:
+                raise ValueError(
+                    "Each entry in features must be either an int, "
+                    "a string, or an iterable of size at most 2."
+                ) from e
+            if not 1 <= np.size(fxs) <= 2:
+                raise ValueError(
+                    "Each entry in features must be either an int, "
+                    "a string, or an iterable of size at most 2."
+                )
+            # store the information if 2-way PD was requested with ICE to later
+            # raise a ValueError with an exhaustive list of problematic
+            # settings.
+            ice_for_two_way_pd.append(kind_plot != "average" and np.size(fxs) > 1)
+
+            tmp_features.append(fxs)
+
+        if any(ice_for_two_way_pd):
+            # raise an error and be specific regarding the parameter values
+            # when 1- and 2-way PD were requested
+            kind_ = [
+                "average" if forcing_average else kind_plot
+                for forcing_average, kind_plot in zip(ice_for_two_way_pd, kind_)
+            ]
+            raise ValueError(
+                "ICE plot cannot be rendered for 2-way feature interactions. "
+                "2-way feature interactions mandates PD plots using the "
+                "'average' kind: "
+                f"features={features!r} should be configured to use "
+                f"kind={kind_!r} explicitly."
+            )
+        features = tmp_features
+
+        if categorical_features is None:
+            is_categorical = [
+                (False,) if len(fxs) == 1 else (False, False) for fxs in features
+            ]
+        else:
+            # we need to create a boolean indicator of which features are
+            # categorical from the categorical_features list.
+            categorical_features = np.asarray(categorical_features)
+            if categorical_features.dtype.kind == "b":
+                # categorical features provided as a list of boolean
+                if categorical_features.size != n_features:
+                    raise ValueError(
+                        "When `categorical_features` is a boolean array-like, "
+                        "the array should be of shape (n_features,). Got "
+                        f"{categorical_features.size} elements while `X` contains "
+                        f"{n_features} features."
+                    )
+                is_categorical = [
+                    tuple(categorical_features[fx] for fx in fxs) for fxs in features
+                ]
+            elif categorical_features.dtype.kind in ("i", "O", "U"):
+                # categorical features provided as a list of indices or feature names
+                categorical_features_idx = [
+                    _get_feature_index(cat, feature_names=feature_names)
+                    for cat in categorical_features
+                ]
+                is_categorical = [
+                    tuple([idx in categorical_features_idx for idx in fxs])
+                    for fxs in features
+                ]
+            else:
+                raise ValueError(
+                    "Expected `categorical_features` to be an array-like of boolean,"
+                    f" integer, or string. Got {categorical_features.dtype} instead."
+                )
+
+            for cats in is_categorical:
+                if np.size(cats) == 2 and (cats[0] != cats[1]):
+                    raise ValueError(
+                        "Two-way partial dependence plots are not supported for pairs"
+                        " of continuous and categorical features."
+                    )
+
+            # collect the indices of the categorical features targeted by the partial
+            # dependence computation
+            categorical_features_targeted = set(
+                [
+                    fx
+                    for fxs, cats in zip(features, is_categorical)
+                    for fx in fxs
+                    if any(cats)
+                ]
+            )
+            if categorical_features_targeted:
+                min_n_cats = min(
+                    [
+                        len(_unique(_safe_indexing(X, idx, axis=1)))
+                        for idx in categorical_features_targeted
+                    ]
+                )
+                if grid_resolution < min_n_cats:
+                    raise ValueError(
+                        "The resolution of the computed grid is less than the "
+                        "minimum number of categories in the targeted categorical "
+                        "features. Expect the `grid_resolution` to be greater than "
+                        f"{min_n_cats}. Got {grid_resolution} instead."
+                    )
+
+            for is_cat, kind_plot in zip(is_categorical, kind_):
+                if any(is_cat) and kind_plot != "average":
+                    raise ValueError(
+                        "It is not possible to display individual effects for"
+                        " categorical features."
+                    )
+
+        # Early exit if the axes does not have the correct number of axes
+        if ax is not None and not isinstance(ax, plt.Axes):
+            axes = np.asarray(ax, dtype=object)
+            if axes.size != len(features):
+                raise ValueError(
+                    "Expected ax to have {} axes, got {}".format(
+                        len(features), axes.size
+                    )
+                )
+
+        for i in chain.from_iterable(features):
+            if i >= len(feature_names):
+                raise ValueError(
+                    "All entries of features must be less than "
+                    "len(feature_names) = {0}, got {1}.".format(len(feature_names), i)
+                )
+
+        if isinstance(subsample, numbers.Integral):
+            if subsample <= 0:
+                raise ValueError(
+                    f"When an integer, subsample={subsample} should be positive."
+                )
+        elif isinstance(subsample, numbers.Real):
+            if subsample <= 0 or subsample >= 1:
+                raise ValueError(
+                    f"When a floating-point, subsample={subsample} should be in "
+                    "the (0, 1) range."
+                )
+
+        # compute predictions and/or averaged predictions
+        pd_results = Parallel(n_jobs=n_jobs, verbose=verbose)(
+            delayed(partial_dependence)(
+                estimator,
+                X,
+                fxs,
+                sample_weight=sample_weight,
+                feature_names=feature_names,
+                categorical_features=categorical_features,
+                response_method=response_method,
+                method=method,
+                grid_resolution=grid_resolution,
+                percentiles=percentiles,
+                kind=kind_plot,
+                custom_values=custom_values,
+            )
+            for kind_plot, fxs in zip(kind_, features)
+        )
+
+        # For multioutput regression, we can only check the validity of target
+        # now that we have the predictions.
+        # Also note: as multiclass-multioutput classifiers are not supported,
+        # multiclass and multioutput scenario are mutually exclusive. So there is
+        # no risk of overwriting target_idx here.
+        pd_result = pd_results[0]  # checking the first result is enough
+        n_tasks = (
+            pd_result.average.shape[0]
+            if kind_[0] == "average"
+            else pd_result.individual.shape[0]
+        )
+        if is_regressor(estimator) and n_tasks > 1:
+            if target is None:
+                raise ValueError("target must be specified for multi-output regressors")
+            if not 0 <= target <= n_tasks:
+                raise ValueError(
+                    "target must be in [0, n_tasks], got {}.".format(target)
+                )
+            target_idx = target
+
+        deciles = {}
+        for fxs, cats in zip(features, is_categorical):
+            for fx, cat in zip(fxs, cats):
+                if not cat and fx not in deciles:
+                    X_col = _safe_indexing(X, fx, axis=1)
+                    deciles[fx] = mquantiles(X_col, prob=np.arange(0.1, 1.0, 0.1))
+
+        display = cls(
+            pd_results=pd_results,
+            features=features,
+            feature_names=feature_names,
+            target_idx=target_idx,
+            deciles=deciles,
+            kind=kind,
+            subsample=subsample,
+            random_state=random_state,
+            is_categorical=is_categorical,
+        )
+        return display.plot(
+            ax=ax,
+            n_cols=n_cols,
+            line_kw=line_kw,
+            ice_lines_kw=ice_lines_kw,
+            pd_line_kw=pd_line_kw,
+            contour_kw=contour_kw,
+            centered=centered,
+        )
 
     def _get_sample_count(self, n_samples):
         """Compute the number of samples as an integer."""
@@ -571,8 +799,14 @@ def _get_sample_count(self, n_samples):
         return n_samples
 
     def _plot_ice_lines(
-        self, preds, feature_values, n_ice_to_plot,
-        ax, pd_plot_idx, n_total_lines_by_plot, individual_line_kw
+        self,
+        preds,
+        feature_values,
+        n_ice_to_plot,
+        ax,
+        pd_plot_idx,
+        n_total_lines_by_plot,
+        individual_line_kw,
     ):
         """Plot the ICE lines.
 
@@ -599,14 +833,15 @@ def _plot_ice_lines(
         rng = check_random_state(self.random_state)
         # subsample ice
         ice_lines_idx = rng.choice(
-            preds.shape[0], n_ice_to_plot, replace=False,
+            preds.shape[0],
+            n_ice_to_plot,
+            replace=False,
         )
         ice_lines_subsampled = preds[ice_lines_idx, :]
         # plot the subsampled ice
         for ice_idx, ice in enumerate(ice_lines_subsampled):
             line_idx = np.unravel_index(
-                pd_plot_idx * n_total_lines_by_plot + ice_idx,
-                self.lines_.shape
+                pd_plot_idx * n_total_lines_by_plot + ice_idx, self.lines_.shape
             )
             self.lines_[line_idx] = ax.plot(
                 feature_values, ice.ravel(), **individual_line_kw
@@ -619,6 +854,8 @@ def _plot_average_dependence(
         ax,
         pd_line_idx,
         line_kw,
+        categorical,
+        bar_kw,
     ):
         """Plot the average partial dependence.
 
@@ -630,22 +867,32 @@ def _plot_average_dependence(
         feature_values : ndarray of shape (n_grid_points,)
             The feature values for which the predictions have been computed.
         ax : Matplotlib axes
-            The axis on which to plot the ICE lines.
+            The axis on which to plot the average PD.
         pd_line_idx : int
             The sequential index of the plot. It will be unraveled to find the
             matching 2D position in the grid layout.
         line_kw : dict
             Dict with keywords passed when plotting the PD plot.
+        categorical : bool
+            Whether feature is categorical.
+        bar_kw: dict
+            Dict with keywords passed when plotting the PD bars (categorical).
         """
-        line_idx = np.unravel_index(pd_line_idx, self.lines_.shape)
-        self.lines_[line_idx] = ax.plot(
-            feature_values,
-            avg_preds,
-            **line_kw,
-        )[0]
+        if categorical:
+            bar_idx = np.unravel_index(pd_line_idx, self.bars_.shape)
+            self.bars_[bar_idx] = ax.bar(feature_values, avg_preds, **bar_kw)[0]
+            ax.tick_params(axis="x", rotation=90)
+        else:
+            line_idx = np.unravel_index(pd_line_idx, self.lines_.shape)
+            self.lines_[line_idx] = ax.plot(
+                feature_values,
+                avg_preds,
+                **line_kw,
+            )[0]
 
     def _plot_one_way_partial_dependence(
         self,
+        kind,
         preds,
         avg_preds,
         feature_values,
@@ -655,13 +902,18 @@ def _plot_one_way_partial_dependence(
         n_cols,
         pd_plot_idx,
         n_lines,
-        individual_line_kw,
-        line_kw,
+        ice_lines_kw,
+        pd_line_kw,
+        categorical,
+        bar_kw,
+        pdp_lim,
     ):
         """Plot 1-way partial dependence: ICE and PDP.
 
         Parameters
         ----------
+        kind : str
+            The kind of partial plot to draw.
         preds : ndarray of shape \
                 (n_instances, n_grid_points) or None
             The predictions computed for all points of `feature_values` for a
@@ -684,14 +936,22 @@ def _plot_one_way_partial_dependence(
             matching 2D position in the grid layout.
         n_lines : int
             The total number of lines expected to be plot on the axis.
-        individual_line_kw : dict
+        ice_lines_kw : dict
             Dict with keywords passed when plotting the ICE lines.
-        line_kw : dict
+        pd_line_kw : dict
             Dict with keywords passed when plotting the PD plot.
+        categorical : bool
+            Whether feature is categorical.
+        bar_kw: dict
+            Dict with keywords passed when plotting the PD bars (categorical).
+        pdp_lim : dict
+            Global min and max average predictions, such that all plots will
+            have the same scale and y limits. `pdp_lim[1]` is the global min
+            and max for single partial dependence curves.
         """
-        from matplotlib import transforms  # noqa
+        from matplotlib import transforms
 
-        if self.kind in ("individual", "both"):
+        if kind in ("individual", "both"):
             self._plot_ice_lines(
                 preds[self.target_idx],
                 feature_values,
@@ -699,12 +959,12 @@ def _plot_one_way_partial_dependence(
                 ax,
                 pd_plot_idx,
                 n_lines,
-                individual_line_kw,
+                ice_lines_kw,
             )
 
-        if self.kind in ("average", "both"):
+        if kind in ("average", "both"):
             # the average is stored as the last line
-            if self.kind == "average":
+            if kind == "average":
                 pd_line_idx = pd_plot_idx
             else:
                 pd_line_idx = pd_plot_idx * n_lines + n_ice_lines
@@ -713,23 +973,26 @@ def _plot_one_way_partial_dependence(
                 feature_values,
                 ax,
                 pd_line_idx,
-                line_kw,
+                pd_line_kw,
+                categorical,
+                bar_kw,
             )
 
-        trans = transforms.blended_transform_factory(
-            ax.transData, ax.transAxes
-        )
+        trans = transforms.blended_transform_factory(ax.transData, ax.transAxes)
         # create the decile line for the vertical axis
         vlines_idx = np.unravel_index(pd_plot_idx, self.deciles_vlines_.shape)
-        self.deciles_vlines_[vlines_idx] = ax.vlines(
-            self.deciles[feature_idx[0]],
-            0,
-            0.05,
-            transform=trans,
-            color="k",
-        )
+        if self.deciles.get(feature_idx[0], None) is not None:
+            self.deciles_vlines_[vlines_idx] = ax.vlines(
+                self.deciles[feature_idx[0]],
+                0,
+                0.05,
+                transform=trans,
+                color="k",
+            )
         # reset ylim which was overwritten by vlines
-        ax.set_ylim(self.pdp_lim[1])
+        min_val = min(val[0] for val in pdp_lim.values())
+        max_val = max(val[1] for val in pdp_lim.values())
+        ax.set_ylim([min_val, max_val])
 
         # Set xlabel if it is not already set
         if not ax.get_xlabel():
@@ -737,11 +1000,11 @@ def _plot_one_way_partial_dependence(
 
         if n_cols is None or pd_plot_idx % n_cols == 0:
             if not ax.get_ylabel():
-                ax.set_ylabel('Partial dependence')
+                ax.set_ylabel("Partial dependence")
         else:
             ax.set_yticklabels([])
 
-        if line_kw.get("label", None) and self.kind != 'individual':
+        if pd_line_kw.get("label", None) and kind != "individual" and not categorical:
             ax.legend()
 
     def _plot_two_way_partial_dependence(
@@ -753,6 +1016,8 @@ def _plot_two_way_partial_dependence(
         pd_plot_idx,
         Z_level,
         contour_kw,
+        categorical,
+        heatmap_kw,
     ):
         """Plot 2-way partial dependence.
 
@@ -776,49 +1041,114 @@ def _plot_two_way_partial_dependence(
             The Z-level used to encode the average predictions.
         contour_kw : dict
             Dict with keywords passed when plotting the contours.
+        categorical : bool
+            Whether features are categorical.
+        heatmap_kw: dict
+            Dict with keywords passed when plotting the PD heatmap
+            (categorical).
         """
-        from matplotlib import transforms  # noqa
-
-        XX, YY = np.meshgrid(feature_values[0], feature_values[1])
-        Z = avg_preds[self.target_idx].T
-        CS = ax.contour(XX, YY, Z, levels=Z_level, linewidths=0.5, colors="k")
-        contour_idx = np.unravel_index(pd_plot_idx, self.contours_.shape)
-        self.contours_[contour_idx] = ax.contourf(
-            XX,
-            YY,
-            Z,
-            levels=Z_level,
-            vmax=Z_level[-1],
-            vmin=Z_level[0],
-            **contour_kw,
-        )
-        ax.clabel(CS, fmt="%2.2f", colors="k", fontsize=10, inline=True)
+        if categorical:
+            import matplotlib.pyplot as plt
+
+            default_im_kw = dict(interpolation="nearest", cmap="viridis")
+            im_kw = {**default_im_kw, **heatmap_kw}
+
+            data = avg_preds[self.target_idx]
+            im = ax.imshow(data, **im_kw)
+            text = None
+            cmap_min, cmap_max = im.cmap(0), im.cmap(1.0)
+
+            text = np.empty_like(data, dtype=object)
+            # print text with appropriate color depending on background
+            thresh = (data.max() + data.min()) / 2.0
+
+            for flat_index in range(data.size):
+                row, col = np.unravel_index(flat_index, data.shape)
+                color = cmap_max if data[row, col] < thresh else cmap_min
+
+                values_format = ".2f"
+                text_data = format(data[row, col], values_format)
+
+                text_kwargs = dict(ha="center", va="center", color=color)
+                text[row, col] = ax.text(col, row, text_data, **text_kwargs)
+
+            fig = ax.figure
+            fig.colorbar(im, ax=ax)
+            ax.set(
+                xticks=np.arange(len(feature_values[1])),
+                yticks=np.arange(len(feature_values[0])),
+                xticklabels=feature_values[1],
+                yticklabels=feature_values[0],
+                xlabel=self.feature_names[feature_idx[1]],
+                ylabel=self.feature_names[feature_idx[0]],
+            )
 
-        trans = transforms.blended_transform_factory(
-            ax.transData, ax.transAxes
-        )
-        # create the decile line for the vertical axis
-        xlim, ylim = ax.get_xlim(), ax.get_ylim()
-        vlines_idx = np.unravel_index(pd_plot_idx, self.deciles_vlines_.shape)
-        self.deciles_vlines_[vlines_idx] = ax.vlines(
-            self.deciles[feature_idx[0]], 0, 0.05, transform=trans, color="k",
-        )
-        # create the decile line for the horizontal axis
-        hlines_idx = np.unravel_index(pd_plot_idx, self.deciles_hlines_.shape)
-        self.deciles_hlines_[hlines_idx] = ax.hlines(
-            self.deciles[feature_idx[1]], 0, 0.05, transform=trans, color="k",
-        )
-        # reset xlim and ylim since they are overwritten by hlines and vlines
-        ax.set_xlim(xlim)
-        ax.set_ylim(ylim)
+            plt.setp(ax.get_xticklabels(), rotation="vertical")
 
-        # set xlabel if it is not already set
-        if not ax.get_xlabel():
-            ax.set_xlabel(self.feature_names[feature_idx[0]])
-        ax.set_ylabel(self.feature_names[feature_idx[1]])
+            heatmap_idx = np.unravel_index(pd_plot_idx, self.heatmaps_.shape)
+            self.heatmaps_[heatmap_idx] = im
+        else:
+            from matplotlib import transforms
+
+            XX, YY = np.meshgrid(feature_values[0], feature_values[1])
+            Z = avg_preds[self.target_idx].T
+            CS = ax.contour(XX, YY, Z, levels=Z_level, linewidths=0.5, colors="k")
+            contour_idx = np.unravel_index(pd_plot_idx, self.contours_.shape)
+            self.contours_[contour_idx] = ax.contourf(
+                XX,
+                YY,
+                Z,
+                levels=Z_level,
+                vmax=Z_level[-1],
+                vmin=Z_level[0],
+                **contour_kw,
+            )
+            ax.clabel(CS, fmt="%2.2f", colors="k", fontsize=10, inline=True)
+
+            trans = transforms.blended_transform_factory(ax.transData, ax.transAxes)
+            # create the decile line for the vertical axis
+            xlim, ylim = ax.get_xlim(), ax.get_ylim()
+            vlines_idx = np.unravel_index(pd_plot_idx, self.deciles_vlines_.shape)
+            self.deciles_vlines_[vlines_idx] = ax.vlines(
+                self.deciles[feature_idx[0]],
+                0,
+                0.05,
+                transform=trans,
+                color="k",
+            )
+            # create the decile line for the horizontal axis
+            hlines_idx = np.unravel_index(pd_plot_idx, self.deciles_hlines_.shape)
+            self.deciles_hlines_[hlines_idx] = ax.hlines(
+                self.deciles[feature_idx[1]],
+                0,
+                0.05,
+                transform=trans,
+                color="k",
+            )
+            # reset xlim and ylim since they are overwritten by hlines and
+            # vlines
+            ax.set_xlim(xlim)
+            ax.set_ylim(ylim)
+
+            # set xlabel if it is not already set
+            if not ax.get_xlabel():
+                ax.set_xlabel(self.feature_names[feature_idx[0]])
+            ax.set_ylabel(self.feature_names[feature_idx[1]])
 
-    @_deprecate_positional_args(version="1.1")
-    def plot(self, *, ax=None, n_cols=3, line_kw=None, contour_kw=None):
+    def plot(
+        self,
+        *,
+        ax=None,
+        n_cols=3,
+        line_kw=None,
+        ice_lines_kw=None,
+        pd_line_kw=None,
+        contour_kw=None,
+        bar_kw=None,
+        heatmap_kw=None,
+        pdp_lim=None,
+        centered=False,
+    ):
         """Plot partial dependence plots.
 
         Parameters
@@ -841,63 +1171,177 @@ def plot(self, *, ax=None, n_cols=3, line_kw=None, contour_kw=None):
             Dict with keywords passed to the `matplotlib.pyplot.plot` call.
             For one-way partial dependence plots.
 
+        ice_lines_kw : dict, default=None
+            Dictionary with keywords passed to the `matplotlib.pyplot.plot` call.
+            For ICE lines in the one-way partial dependence plots.
+            The key value pairs defined in `ice_lines_kw` takes priority over
+            `line_kw`.
+
+            .. versionadded:: 1.0
+
+        pd_line_kw : dict, default=None
+            Dictionary with keywords passed to the `matplotlib.pyplot.plot` call.
+            For partial dependence in one-way partial dependence plots.
+            The key value pairs defined in `pd_line_kw` takes priority over
+            `line_kw`.
+
+            .. versionadded:: 1.0
+
         contour_kw : dict, default=None
             Dict with keywords passed to the `matplotlib.pyplot.contourf`
             call for two-way partial dependence plots.
 
+        bar_kw : dict, default=None
+            Dict with keywords passed to the `matplotlib.pyplot.bar`
+            call for one-way categorical partial dependence plots.
+
+            .. versionadded:: 1.2
+
+        heatmap_kw : dict, default=None
+            Dict with keywords passed to the `matplotlib.pyplot.imshow`
+            call for two-way categorical partial dependence plots.
+
+            .. versionadded:: 1.2
+
+        pdp_lim : dict, default=None
+            Global min and max average predictions, such that all plots will have the
+            same scale and y limits. `pdp_lim[1]` is the global min and max for single
+            partial dependence curves. `pdp_lim[2]` is the global min and max for
+            two-way partial dependence curves. If `None` (default), the limit will be
+            inferred from the global minimum and maximum of all predictions.
+
+            .. versionadded:: 1.1
+
+        centered : bool, default=False
+            If `True`, the ICE and PD lines will start at the origin of the
+            y-axis. By default, no centering is done.
+
+            .. versionadded:: 1.1
+
         Returns
         -------
         display : :class:`~sklearn.inspection.PartialDependenceDisplay`
+            Returns a :class:`~sklearn.inspection.PartialDependenceDisplay`
+            object that contains the partial dependence plots.
         """
 
         check_matplotlib_support("plot_partial_dependence")
-        import matplotlib.pyplot as plt  # noqa
-        from matplotlib.gridspec import GridSpecFromSubplotSpec  # noqa
+        import matplotlib.pyplot as plt
+        from matplotlib.gridspec import GridSpecFromSubplotSpec
+
+        if isinstance(self.kind, str):
+            kind = [self.kind] * len(self.features)
+        else:
+            kind = self.kind
+
+        if self.is_categorical is None:
+            is_categorical = [
+                (False,) if len(fx) == 1 else (False, False) for fx in self.features
+            ]
+        else:
+            is_categorical = self.is_categorical
+
+        if len(kind) != len(self.features):
+            raise ValueError(
+                "When `kind` is provided as a list of strings, it should "
+                "contain as many elements as `features`. `kind` contains "
+                f"{len(kind)} element(s) and `features` contains "
+                f"{len(self.features)} element(s)."
+            )
+
+        valid_kinds = {"average", "individual", "both"}
+        if any([k not in valid_kinds for k in kind]):
+            raise ValueError(
+                f"Values provided to `kind` must be one of: {valid_kinds!r} or a list"
+                f" of such values. Currently, kind={self.kind!r}"
+            )
+
+        # Center results before plotting
+        if not centered:
+            pd_results_ = self.pd_results
+        else:
+            pd_results_ = []
+            for kind_plot, pd_result in zip(kind, self.pd_results):
+                current_results = {"grid_values": pd_result["grid_values"]}
+
+                if kind_plot in ("individual", "both"):
+                    preds = pd_result.individual
+                    preds = preds - preds[self.target_idx, :, 0, None]
+                    current_results["individual"] = preds
+
+                if kind_plot in ("average", "both"):
+                    avg_preds = pd_result.average
+                    avg_preds = avg_preds - avg_preds[self.target_idx, 0, None]
+                    current_results["average"] = avg_preds
+
+                pd_results_.append(Bunch(**current_results))
+
+        if pdp_lim is None:
+            # get global min and max average predictions of PD grouped by plot type
+            pdp_lim = {}
+            for kind_plot, pdp in zip(kind, pd_results_):
+                values = pdp["grid_values"]
+                preds = pdp.average if kind_plot == "average" else pdp.individual
+                min_pd = preds[self.target_idx].min()
+                max_pd = preds[self.target_idx].max()
+
+                # expand the limits to account so that the plotted lines do not touch
+                # the edges of the plot
+                span = max_pd - min_pd
+                min_pd -= 0.05 * span
+                max_pd += 0.05 * span
+
+                n_fx = len(values)
+                old_min_pd, old_max_pd = pdp_lim.get(n_fx, (min_pd, max_pd))
+                min_pd = min(min_pd, old_min_pd)
+                max_pd = max(max_pd, old_max_pd)
+                pdp_lim[n_fx] = (min_pd, max_pd)
 
         if line_kw is None:
             line_kw = {}
-        if contour_kw is None:
-            contour_kw = {}
+        if ice_lines_kw is None:
+            ice_lines_kw = {}
+        if pd_line_kw is None:
+            pd_line_kw = {}
+        if bar_kw is None:
+            bar_kw = {}
+        if heatmap_kw is None:
+            heatmap_kw = {}
 
         if ax is None:
             _, ax = plt.subplots()
 
+        if contour_kw is None:
+            contour_kw = {}
         default_contour_kws = {"alpha": 0.75}
-        contour_kw = {**default_contour_kws, **contour_kw}
-
-        default_line_kws = {
-            "color": "C0",
-            "label": "average" if self.kind == "both" else None,
-        }
-        line_kw = {**default_line_kws, **line_kw}
-
-        individual_line_kw = line_kw.copy()
-        del individual_line_kw["label"]
-
-        if self.kind == 'individual' or self.kind == 'both':
-            individual_line_kw['alpha'] = 0.3
-            individual_line_kw['linewidth'] = 0.5
+        contour_kw = _validate_style_kwargs(default_contour_kws, contour_kw)
 
         n_features = len(self.features)
-        if self.kind in ("individual", "both"):
+        is_average_plot = [kind_plot == "average" for kind_plot in kind]
+        if all(is_average_plot):
+            # only average plots are requested
+            n_ice_lines = 0
+            n_lines = 1
+        else:
+            # we need to determine the number of ICE samples computed
+            ice_plot_idx = is_average_plot.index(False)
             n_ice_lines = self._get_sample_count(
-                len(self.pd_results[0].individual[0])
+                len(pd_results_[ice_plot_idx].individual[0])
             )
-            if self.kind == "individual":
-                n_lines = n_ice_lines
+            if any([kind_plot == "both" for kind_plot in kind]):
+                n_lines = n_ice_lines + 1  # account for the average line
             else:
-                n_lines = n_ice_lines + 1
-        else:
-            n_ice_lines = 0
-            n_lines = 1
+                n_lines = n_ice_lines
 
         if isinstance(ax, plt.Axes):
             # If ax was set off, it has most likely been set to off
             # by a previous call to plot.
             if not ax.axison:
-                raise ValueError("The ax was already used in another plot "
-                                 "function, please set ax=display.axes_ "
-                                 "instead")
+                raise ValueError(
+                    "The ax was already used in another plot "
+                    "function, please set ax=display.axes_ "
+                    "instead"
+                )
 
             ax.set_axis_off()
             self.bounding_ax_ = ax
@@ -907,24 +1351,28 @@ def plot(self, *, ax=None, n_cols=3, line_kw=None, contour_kw=None):
             n_rows = int(np.ceil(n_features / float(n_cols)))
 
             self.axes_ = np.empty((n_rows, n_cols), dtype=object)
-            if self.kind == 'average':
+            if all(is_average_plot):
                 self.lines_ = np.empty((n_rows, n_cols), dtype=object)
             else:
                 self.lines_ = np.empty((n_rows, n_cols, n_lines), dtype=object)
             self.contours_ = np.empty((n_rows, n_cols), dtype=object)
+            self.bars_ = np.empty((n_rows, n_cols), dtype=object)
+            self.heatmaps_ = np.empty((n_rows, n_cols), dtype=object)
 
             axes_ravel = self.axes_.ravel()
 
-            gs = GridSpecFromSubplotSpec(n_rows, n_cols,
-                                         subplot_spec=ax.get_subplotspec())
+            gs = GridSpecFromSubplotSpec(
+                n_rows, n_cols, subplot_spec=ax.get_subplotspec()
+            )
             for i, spec in zip(range(n_features), gs):
                 axes_ravel[i] = self.figure_.add_subplot(spec)
 
         else:  # array-like
             ax = np.asarray(ax, dtype=object)
             if ax.size != n_features:
-                raise ValueError("Expected ax to have {} axes, got {}"
-                                 .format(n_features, ax.size))
+                raise ValueError(
+                    "Expected ax to have {} axes, got {}".format(n_features, ax.size)
+                )
 
             if ax.ndim == 2:
                 n_cols = ax.shape[1]
@@ -934,35 +1382,88 @@ def plot(self, *, ax=None, n_cols=3, line_kw=None, contour_kw=None):
             self.bounding_ax_ = None
             self.figure_ = ax.ravel()[0].figure
             self.axes_ = ax
-            if self.kind == 'average':
+            if all(is_average_plot):
                 self.lines_ = np.empty_like(ax, dtype=object)
             else:
                 self.lines_ = np.empty(ax.shape + (n_lines,), dtype=object)
             self.contours_ = np.empty_like(ax, dtype=object)
+            self.bars_ = np.empty_like(ax, dtype=object)
+            self.heatmaps_ = np.empty_like(ax, dtype=object)
 
         # create contour levels for two-way plots
-        if 2 in self.pdp_lim:
-            Z_level = np.linspace(*self.pdp_lim[2], num=8)
+        if 2 in pdp_lim:
+            Z_level = np.linspace(*pdp_lim[2], num=8)
 
         self.deciles_vlines_ = np.empty_like(self.axes_, dtype=object)
         self.deciles_hlines_ = np.empty_like(self.axes_, dtype=object)
 
-        for pd_plot_idx, (axi, feature_idx, pd_result) in enumerate(
-            zip(self.axes_.ravel(), self.features, self.pd_results)
+        for pd_plot_idx, (axi, feature_idx, cat, pd_result, kind_plot) in enumerate(
+            zip(
+                self.axes_.ravel(),
+                self.features,
+                is_categorical,
+                pd_results_,
+                kind,
+            )
         ):
             avg_preds = None
             preds = None
-            feature_values = pd_result["values"]
-            if self.kind == 'individual':
+            feature_values = pd_result["grid_values"]
+            if kind_plot == "individual":
                 preds = pd_result.individual
-            elif self.kind == 'average':
+            elif kind_plot == "average":
                 avg_preds = pd_result.average
-            else:  # kind='both'
+            else:  # kind_plot == 'both'
                 avg_preds = pd_result.average
                 preds = pd_result.individual
 
             if len(feature_values) == 1:
+                # define the line-style for the current plot
+                default_line_kws = {
+                    "color": "C0",
+                    "label": "average" if kind_plot == "both" else None,
+                }
+                if kind_plot == "individual":
+                    default_ice_lines_kws = {"alpha": 0.3, "linewidth": 0.5}
+                    default_pd_lines_kws = {}
+                elif kind_plot == "both":
+                    # by default, we need to distinguish the average line from
+                    # the individual lines via color and line style
+                    default_ice_lines_kws = {
+                        "alpha": 0.3,
+                        "linewidth": 0.5,
+                        "color": "tab:blue",
+                    }
+                    default_pd_lines_kws = {
+                        "color": "tab:orange",
+                        "linestyle": "--",
+                    }
+                else:
+                    default_ice_lines_kws = {}
+                    default_pd_lines_kws = {}
+
+                default_ice_lines_kws = {**default_line_kws, **default_ice_lines_kws}
+                default_pd_lines_kws = {**default_line_kws, **default_pd_lines_kws}
+
+                line_kw = _validate_style_kwargs(default_line_kws, line_kw)
+
+                ice_lines_kw = _validate_style_kwargs(
+                    _validate_style_kwargs(default_ice_lines_kws, line_kw), ice_lines_kw
+                )
+                del ice_lines_kw["label"]
+
+                pd_line_kw = _validate_style_kwargs(
+                    _validate_style_kwargs(default_pd_lines_kws, line_kw), pd_line_kw
+                )
+
+                default_bar_kws = {"color": "C0"}
+                bar_kw = _validate_style_kwargs(default_bar_kws, bar_kw)
+
+                default_heatmap_kw = {}
+                heatmap_kw = _validate_style_kwargs(default_heatmap_kw, heatmap_kw)
+
                 self._plot_one_way_partial_dependence(
+                    kind_plot,
                     preds,
                     avg_preds,
                     feature_values[0],
@@ -972,8 +1473,11 @@ def plot(self, *, ax=None, n_cols=3, line_kw=None, contour_kw=None):
                     n_cols,
                     pd_plot_idx,
                     n_lines,
-                    individual_line_kw,
-                    line_kw,
+                    ice_lines_kw,
+                    pd_line_kw,
+                    cat[0],
+                    bar_kw,
+                    pdp_lim,
                 )
             else:
                 self._plot_two_way_partial_dependence(
@@ -984,6 +1488,8 @@ def plot(self, *, ax=None, n_cols=3, line_kw=None, contour_kw=None):
                     pd_plot_idx,
                     Z_level,
                     contour_kw,
+                    cat[0] and cat[1],
+                    heatmap_kw,
                 )
 
         return self
diff --git a/sklearn/inspection/_plot/tests/test_boundary_decision_display.py b/sklearn/inspection/_plot/tests/test_boundary_decision_display.py
new file mode 100644
index 0000000000000..3284f42241fa5
--- /dev/null
+++ b/sklearn/inspection/_plot/tests/test_boundary_decision_display.py
@@ -0,0 +1,685 @@
+import warnings
+
+import numpy as np
+import pytest
+
+from sklearn.base import BaseEstimator, ClassifierMixin
+from sklearn.datasets import (
+    load_diabetes,
+    load_iris,
+    make_classification,
+    make_multilabel_classification,
+)
+from sklearn.ensemble import IsolationForest
+from sklearn.inspection import DecisionBoundaryDisplay
+from sklearn.inspection._plot.decision_boundary import _check_boundary_response_method
+from sklearn.linear_model import LogisticRegression
+from sklearn.preprocessing import scale
+from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
+from sklearn.utils._testing import (
+    _convert_container,
+    assert_allclose,
+    assert_array_equal,
+)
+from sklearn.utils.fixes import parse_version
+
+X, y = make_classification(
+    n_informative=1,
+    n_redundant=1,
+    n_clusters_per_class=1,
+    n_features=2,
+    random_state=42,
+)
+
+
+def load_iris_2d_scaled():
+    X, y = load_iris(return_X_y=True)
+    X = scale(X)[:, :2]
+    return X, y
+
+
+@pytest.fixture(scope="module")
+def fitted_clf():
+    return LogisticRegression().fit(X, y)
+
+
+def test_input_data_dimension(pyplot):
+    """Check that we raise an error when `X` does not have exactly 2 features."""
+    X, y = make_classification(n_samples=10, n_features=4, random_state=0)
+
+    clf = LogisticRegression().fit(X, y)
+    msg = "n_features must be equal to 2. Got 4 instead."
+    with pytest.raises(ValueError, match=msg):
+        DecisionBoundaryDisplay.from_estimator(estimator=clf, X=X)
+
+
+def test_check_boundary_response_method_error():
+    """Check error raised for multi-output multi-class classifiers by
+    `_check_boundary_response_method`.
+    """
+
+    class MultiLabelClassifier:
+        classes_ = [np.array([0, 1]), np.array([0, 1])]
+
+    err_msg = "Multi-label and multi-output multi-class classifiers are not supported"
+    with pytest.raises(ValueError, match=err_msg):
+        _check_boundary_response_method(MultiLabelClassifier(), "predict", None)
+
+
+@pytest.mark.parametrize(
+    "estimator, response_method, class_of_interest, expected_prediction_method",
+    [
+        (DecisionTreeRegressor(), "predict", None, "predict"),
+        (DecisionTreeRegressor(), "auto", None, "predict"),
+        (LogisticRegression().fit(*load_iris_2d_scaled()), "predict", None, "predict"),
+        (
+            LogisticRegression().fit(*load_iris_2d_scaled()),
+            "auto",
+            None,
+            ["decision_function", "predict_proba", "predict"],
+        ),
+        (
+            LogisticRegression().fit(*load_iris_2d_scaled()),
+            "predict_proba",
+            0,
+            "predict_proba",
+        ),
+        (
+            LogisticRegression().fit(*load_iris_2d_scaled()),
+            "decision_function",
+            0,
+            "decision_function",
+        ),
+        (
+            LogisticRegression().fit(X, y),
+            "auto",
+            None,
+            ["decision_function", "predict_proba", "predict"],
+        ),
+        (LogisticRegression().fit(X, y), "predict", None, "predict"),
+        (
+            LogisticRegression().fit(X, y),
+            ["predict_proba", "decision_function"],
+            None,
+            ["predict_proba", "decision_function"],
+        ),
+    ],
+)
+def test_check_boundary_response_method(
+    estimator, response_method, class_of_interest, expected_prediction_method
+):
+    """Check the behaviour of `_check_boundary_response_method` for the supported
+    cases.
+    """
+    prediction_method = _check_boundary_response_method(
+        estimator, response_method, class_of_interest
+    )
+    assert prediction_method == expected_prediction_method
+
+
+def test_multiclass_predict(pyplot):
+    """Check multiclass `response=predict` gives expected results."""
+    grid_resolution = 10
+    eps = 1.0
+    X, y = make_classification(n_classes=3, n_informative=3, random_state=0)
+    X = X[:, [0, 1]]
+    lr = LogisticRegression(random_state=0).fit(X, y)
+
+    disp = DecisionBoundaryDisplay.from_estimator(
+        lr, X, response_method="predict", grid_resolution=grid_resolution, eps=1.0
+    )
+
+    x0_min, x0_max = X[:, 0].min() - eps, X[:, 0].max() + eps
+    x1_min, x1_max = X[:, 1].min() - eps, X[:, 1].max() + eps
+    xx0, xx1 = np.meshgrid(
+        np.linspace(x0_min, x0_max, grid_resolution),
+        np.linspace(x1_min, x1_max, grid_resolution),
+    )
+    response = lr.predict(np.c_[xx0.ravel(), xx1.ravel()])
+    assert_allclose(disp.response, response.reshape(xx0.shape))
+    assert_allclose(disp.xx0, xx0)
+    assert_allclose(disp.xx1, xx1)
+
+
+@pytest.mark.parametrize(
+    "kwargs, error_msg",
+    [
+        (
+            {"plot_method": "hello_world"},
+            r"plot_method must be one of contourf, contour, pcolormesh. Got hello_world"
+            r" instead.",
+        ),
+        (
+            {"grid_resolution": 1},
+            r"grid_resolution must be greater than 1. Got 1 instead",
+        ),
+        (
+            {"grid_resolution": -1},
+            r"grid_resolution must be greater than 1. Got -1 instead",
+        ),
+        ({"eps": -1.1}, r"eps must be greater than or equal to 0. Got -1.1 instead"),
+    ],
+)
+def test_input_validation_errors(pyplot, kwargs, error_msg, fitted_clf):
+    """Check input validation from_estimator."""
+    with pytest.raises(ValueError, match=error_msg):
+        DecisionBoundaryDisplay.from_estimator(fitted_clf, X, **kwargs)
+
+
+@pytest.mark.parametrize(
+    "kwargs, error_msg",
+    [
+        ({"multiclass_colors": "not_cmap"}, "it must be a valid Matplotlib colormap"),
+        ({"multiclass_colors": ["red", "green"]}, "it must be of the same length"),
+        (
+            {"multiclass_colors": ["red", "green", "not color"]},
+            "it can only contain valid Matplotlib color names",
+        ),
+    ],
+)
+def test_input_validation_errors_multiclass_colors(pyplot, kwargs, error_msg):
+    """Check input validation for `multiclass_colors` in `from_estimator`."""
+    X, y = load_iris_2d_scaled()
+    clf = LogisticRegression().fit(X, y)
+    with pytest.raises(ValueError, match=error_msg):
+        DecisionBoundaryDisplay.from_estimator(clf, X, **kwargs)
+
+
+def test_display_plot_input_error(pyplot, fitted_clf):
+    """Check input validation for `plot`."""
+    disp = DecisionBoundaryDisplay.from_estimator(fitted_clf, X, grid_resolution=5)
+
+    with pytest.raises(ValueError, match="plot_method must be 'contourf'"):
+        disp.plot(plot_method="hello_world")
+
+
+@pytest.mark.parametrize(
+    "response_method", ["auto", "predict", "predict_proba", "decision_function"]
+)
+@pytest.mark.parametrize("plot_method", ["contourf", "contour"])
+def test_decision_boundary_display_classifier(
+    pyplot, fitted_clf, response_method, plot_method
+):
+    """Check that decision boundary is correct."""
+    fig, ax = pyplot.subplots()
+    eps = 2.0
+    disp = DecisionBoundaryDisplay.from_estimator(
+        fitted_clf,
+        X,
+        grid_resolution=5,
+        response_method=response_method,
+        plot_method=plot_method,
+        eps=eps,
+        ax=ax,
+    )
+    assert isinstance(disp.surface_, pyplot.matplotlib.contour.QuadContourSet)
+    assert disp.ax_ == ax
+    assert disp.figure_ == fig
+
+    x0, x1 = X[:, 0], X[:, 1]
+
+    x0_min, x0_max = x0.min() - eps, x0.max() + eps
+    x1_min, x1_max = x1.min() - eps, x1.max() + eps
+
+    assert disp.xx0.min() == pytest.approx(x0_min)
+    assert disp.xx0.max() == pytest.approx(x0_max)
+    assert disp.xx1.min() == pytest.approx(x1_min)
+    assert disp.xx1.max() == pytest.approx(x1_max)
+
+    fig2, ax2 = pyplot.subplots()
+    # change plotting method for second plot
+    disp.plot(plot_method="pcolormesh", ax=ax2, shading="auto")
+    assert isinstance(disp.surface_, pyplot.matplotlib.collections.QuadMesh)
+    assert disp.ax_ == ax2
+    assert disp.figure_ == fig2
+
+
+@pytest.mark.parametrize("response_method", ["auto", "predict", "decision_function"])
+@pytest.mark.parametrize("plot_method", ["contourf", "contour"])
+def test_decision_boundary_display_outlier_detector(
+    pyplot, response_method, plot_method
+):
+    """Check that decision boundary is correct for outlier detector."""
+    fig, ax = pyplot.subplots()
+    eps = 2.0
+    outlier_detector = IsolationForest(random_state=0).fit(X, y)
+    disp = DecisionBoundaryDisplay.from_estimator(
+        outlier_detector,
+        X,
+        grid_resolution=5,
+        response_method=response_method,
+        plot_method=plot_method,
+        eps=eps,
+        ax=ax,
+    )
+    assert isinstance(disp.surface_, pyplot.matplotlib.contour.QuadContourSet)
+    assert disp.ax_ == ax
+    assert disp.figure_ == fig
+
+    x0, x1 = X[:, 0], X[:, 1]
+
+    x0_min, x0_max = x0.min() - eps, x0.max() + eps
+    x1_min, x1_max = x1.min() - eps, x1.max() + eps
+
+    assert disp.xx0.min() == pytest.approx(x0_min)
+    assert disp.xx0.max() == pytest.approx(x0_max)
+    assert disp.xx1.min() == pytest.approx(x1_min)
+    assert disp.xx1.max() == pytest.approx(x1_max)
+
+
+@pytest.mark.parametrize("response_method", ["auto", "predict"])
+@pytest.mark.parametrize("plot_method", ["contourf", "contour"])
+def test_decision_boundary_display_regressor(pyplot, response_method, plot_method):
+    """Check that we can display the decision boundary for a regressor."""
+    X, y = load_diabetes(return_X_y=True)
+    X = X[:, :2]
+    tree = DecisionTreeRegressor().fit(X, y)
+    fig, ax = pyplot.subplots()
+    eps = 2.0
+    disp = DecisionBoundaryDisplay.from_estimator(
+        tree,
+        X,
+        response_method=response_method,
+        ax=ax,
+        eps=eps,
+        plot_method=plot_method,
+    )
+    assert isinstance(disp.surface_, pyplot.matplotlib.contour.QuadContourSet)
+    assert disp.ax_ == ax
+    assert disp.figure_ == fig
+
+    x0, x1 = X[:, 0], X[:, 1]
+
+    x0_min, x0_max = x0.min() - eps, x0.max() + eps
+    x1_min, x1_max = x1.min() - eps, x1.max() + eps
+
+    assert disp.xx0.min() == pytest.approx(x0_min)
+    assert disp.xx0.max() == pytest.approx(x0_max)
+    assert disp.xx1.min() == pytest.approx(x1_min)
+    assert disp.xx1.max() == pytest.approx(x1_max)
+
+    fig2, ax2 = pyplot.subplots()
+    # change plotting method for second plot
+    disp.plot(plot_method="pcolormesh", ax=ax2, shading="auto")
+    assert isinstance(disp.surface_, pyplot.matplotlib.collections.QuadMesh)
+    assert disp.ax_ == ax2
+    assert disp.figure_ == fig2
+
+
+@pytest.mark.parametrize(
+    "response_method, msg",
+    [
+        (
+            "predict_proba",
+            "MyClassifier has none of the following attributes: predict_proba",
+        ),
+        (
+            "decision_function",
+            "MyClassifier has none of the following attributes: decision_function",
+        ),
+        (
+            "auto",
+            (
+                "MyClassifier has none of the following attributes: decision_function, "
+                "predict_proba, predict"
+            ),
+        ),
+        (
+            "bad_method",
+            "MyClassifier has none of the following attributes: bad_method",
+        ),
+    ],
+)
+def test_error_bad_response(pyplot, response_method, msg):
+    """Check errors for bad response."""
+
+    class MyClassifier(ClassifierMixin, BaseEstimator):
+        def fit(self, X, y):
+            self.fitted_ = True
+            self.classes_ = [0, 1]
+            return self
+
+    clf = MyClassifier().fit(X, y)
+
+    with pytest.raises(AttributeError, match=msg):
+        DecisionBoundaryDisplay.from_estimator(clf, X, response_method=response_method)
+
+
+@pytest.mark.parametrize("response_method", ["auto", "predict", "predict_proba"])
+def test_multilabel_classifier_error(pyplot, response_method):
+    """Check that multilabel classifier raises correct error."""
+    X, y = make_multilabel_classification(random_state=0)
+    X = X[:, :2]
+    tree = DecisionTreeClassifier().fit(X, y)
+
+    msg = "Multi-label and multi-output multi-class classifiers are not supported"
+    with pytest.raises(ValueError, match=msg):
+        DecisionBoundaryDisplay.from_estimator(
+            tree,
+            X,
+            response_method=response_method,
+        )
+
+
+@pytest.mark.parametrize("response_method", ["auto", "predict", "predict_proba"])
+def test_multi_output_multi_class_classifier_error(pyplot, response_method):
+    """Check that multi-output multi-class classifier raises correct error."""
+    X = np.asarray([[0, 1], [1, 2]])
+    y = np.asarray([["tree", "cat"], ["cat", "tree"]])
+    tree = DecisionTreeClassifier().fit(X, y)
+
+    msg = "Multi-label and multi-output multi-class classifiers are not supported"
+    with pytest.raises(ValueError, match=msg):
+        DecisionBoundaryDisplay.from_estimator(
+            tree,
+            X,
+            response_method=response_method,
+        )
+
+
+def test_multioutput_regressor_error(pyplot):
+    """Check that multioutput regressor raises correct error."""
+    X = np.asarray([[0, 1], [1, 2]])
+    y = np.asarray([[0, 1], [4, 1]])
+    tree = DecisionTreeRegressor().fit(X, y)
+    with pytest.raises(ValueError, match="Multi-output regressors are not supported"):
+        DecisionBoundaryDisplay.from_estimator(tree, X, response_method="predict")
+
+
+@pytest.mark.parametrize(
+    "response_method",
+    ["predict_proba", "decision_function", ["predict_proba", "predict"]],
+)
+def test_regressor_unsupported_response(pyplot, response_method):
+    """Check that we can display the decision boundary for a regressor."""
+    X, y = load_diabetes(return_X_y=True)
+    X = X[:, :2]
+    tree = DecisionTreeRegressor().fit(X, y)
+    err_msg = "should either be a classifier to be used with response_method"
+    with pytest.raises(ValueError, match=err_msg):
+        DecisionBoundaryDisplay.from_estimator(tree, X, response_method=response_method)
+
+
+@pytest.mark.filterwarnings(
+    # We expect to raise the following warning because the classifier is fit on a
+    # NumPy array
+    "ignore:X has feature names, but LogisticRegression was fitted without"
+)
+def test_dataframe_labels_used(pyplot, fitted_clf):
+    """Check that column names are used for pandas."""
+    pd = pytest.importorskip("pandas")
+    df = pd.DataFrame(X, columns=["col_x", "col_y"])
+
+    # pandas column names are used by default
+    _, ax = pyplot.subplots()
+    disp = DecisionBoundaryDisplay.from_estimator(fitted_clf, df, ax=ax)
+    assert ax.get_xlabel() == "col_x"
+    assert ax.get_ylabel() == "col_y"
+
+    # second call to plot will have the names
+    fig, ax = pyplot.subplots()
+    disp.plot(ax=ax)
+    assert ax.get_xlabel() == "col_x"
+    assert ax.get_ylabel() == "col_y"
+
+    # axes with a label will not get overridden
+    fig, ax = pyplot.subplots()
+    ax.set(xlabel="hello", ylabel="world")
+    disp.plot(ax=ax)
+    assert ax.get_xlabel() == "hello"
+    assert ax.get_ylabel() == "world"
+
+    # labels get overridden only if provided to the `plot` method
+    disp.plot(ax=ax, xlabel="overwritten_x", ylabel="overwritten_y")
+    assert ax.get_xlabel() == "overwritten_x"
+    assert ax.get_ylabel() == "overwritten_y"
+
+    # labels do not get inferred if provided to `from_estimator`
+    _, ax = pyplot.subplots()
+    disp = DecisionBoundaryDisplay.from_estimator(
+        fitted_clf, df, ax=ax, xlabel="overwritten_x", ylabel="overwritten_y"
+    )
+    assert ax.get_xlabel() == "overwritten_x"
+    assert ax.get_ylabel() == "overwritten_y"
+
+
+def test_string_target(pyplot):
+    """Check that decision boundary works with classifiers trained on string labels."""
+    iris = load_iris()
+    X = iris.data[:, [0, 1]]
+
+    # Use strings as target
+    y = iris.target_names[iris.target]
+    log_reg = LogisticRegression().fit(X, y)
+
+    # Does not raise
+    DecisionBoundaryDisplay.from_estimator(
+        log_reg,
+        X,
+        grid_resolution=5,
+        response_method="predict",
+    )
+
+
+@pytest.mark.parametrize("constructor_name", ["pandas", "polars"])
+def test_dataframe_support(pyplot, constructor_name):
+    """Check that passing a dataframe at fit and to the Display does not
+    raise warnings.
+
+    Non-regression test for:
+    * https://github.com/scikit-learn/scikit-learn/issues/23311
+    * https://github.com/scikit-learn/scikit-learn/issues/28717
+    """
+    df = _convert_container(
+        X, constructor_name=constructor_name, columns_name=["col_x", "col_y"]
+    )
+    estimator = LogisticRegression().fit(df, y)
+
+    with warnings.catch_warnings():
+        # no warnings linked to feature names validation should be raised
+        warnings.simplefilter("error", UserWarning)
+        DecisionBoundaryDisplay.from_estimator(estimator, df, response_method="predict")
+
+
+@pytest.mark.parametrize("response_method", ["predict_proba", "decision_function"])
+def test_class_of_interest_binary(pyplot, response_method):
+    """Check the behaviour of passing `class_of_interest` for plotting the output of
+    `predict_proba` and `decision_function` in the binary case.
+    """
+    iris = load_iris()
+    X = iris.data[:100, :2]
+    y = iris.target[:100]
+    assert_array_equal(np.unique(y), [0, 1])
+
+    estimator = LogisticRegression().fit(X, y)
+    # We will check that `class_of_interest=None` is equivalent to
+    # `class_of_interest=estimator.classes_[1]`
+    disp_default = DecisionBoundaryDisplay.from_estimator(
+        estimator,
+        X,
+        response_method=response_method,
+        class_of_interest=None,
+    )
+    disp_class_1 = DecisionBoundaryDisplay.from_estimator(
+        estimator,
+        X,
+        response_method=response_method,
+        class_of_interest=estimator.classes_[1],
+    )
+
+    assert_allclose(disp_default.response, disp_class_1.response)
+
+    # we can check that `_get_response_values` modifies the response when targeting
+    # the other class, i.e. 1 - p(y=1|x) for `predict_proba` and -decision_function
+    # for `decision_function`.
+    disp_class_0 = DecisionBoundaryDisplay.from_estimator(
+        estimator,
+        X,
+        response_method=response_method,
+        class_of_interest=estimator.classes_[0],
+    )
+
+    if response_method == "predict_proba":
+        assert_allclose(disp_default.response, 1 - disp_class_0.response)
+    else:
+        assert response_method == "decision_function"
+        assert_allclose(disp_default.response, -disp_class_0.response)
+
+
+@pytest.mark.parametrize("response_method", ["predict_proba", "decision_function"])
+def test_class_of_interest_multiclass(pyplot, response_method):
+    """Check the behaviour of passing `class_of_interest` for plotting the output of
+    `predict_proba` and `decision_function` in the multiclass case.
+    """
+    iris = load_iris()
+    X = iris.data[:, :2]
+    y = iris.target  # the target are numerical labels
+    class_of_interest_idx = 2
+
+    estimator = LogisticRegression().fit(X, y)
+    disp = DecisionBoundaryDisplay.from_estimator(
+        estimator,
+        X,
+        response_method=response_method,
+        class_of_interest=class_of_interest_idx,
+    )
+
+    # we will check that we plot the expected values as response
+    grid = np.concatenate([disp.xx0.reshape(-1, 1), disp.xx1.reshape(-1, 1)], axis=1)
+    response = getattr(estimator, response_method)(grid)[:, class_of_interest_idx]
+    assert_allclose(response.reshape(*disp.response.shape), disp.response)
+
+    # make the same test but this time using target as strings
+    y = iris.target_names[iris.target]
+    estimator = LogisticRegression().fit(X, y)
+
+    disp = DecisionBoundaryDisplay.from_estimator(
+        estimator,
+        X,
+        response_method=response_method,
+        class_of_interest=iris.target_names[class_of_interest_idx],
+    )
+
+    grid = np.concatenate([disp.xx0.reshape(-1, 1), disp.xx1.reshape(-1, 1)], axis=1)
+    response = getattr(estimator, response_method)(grid)[:, class_of_interest_idx]
+    assert_allclose(response.reshape(*disp.response.shape), disp.response)
+
+    # check that we raise an error for unknown labels
+    # this test should already be handled in `_get_response_values` but we can have this
+    # test here as well
+    err_msg = "class_of_interest=2 is not a valid label: It should be one of"
+    with pytest.raises(ValueError, match=err_msg):
+        DecisionBoundaryDisplay.from_estimator(
+            estimator,
+            X,
+            response_method=response_method,
+            class_of_interest=class_of_interest_idx,
+        )
+
+
+@pytest.mark.parametrize("response_method", ["predict_proba", "decision_function"])
+def test_multiclass_plot_max_class(pyplot, response_method):
+    """Check plot correct when plotting max multiclass class."""
+    import matplotlib as mpl
+
+    # In matplotlib < v3.5, default value of `pcolormesh(shading)` is 'flat', which
+    # results in the last row and column being dropped. Thus older versions produce
+    # a 99x99 grid, while newer versions produce a 100x100 grid.
+    if parse_version(mpl.__version__) < parse_version("3.5"):
+        pytest.skip("`pcolormesh` in Matplotlib >= 3.5 gives smaller grid size.")
+
+    X, y = load_iris_2d_scaled()
+    clf = LogisticRegression().fit(X, y)
+
+    disp = DecisionBoundaryDisplay.from_estimator(
+        clf,
+        X,
+        plot_method="pcolormesh",
+        response_method=response_method,
+    )
+
+    grid = np.concatenate([disp.xx0.reshape(-1, 1), disp.xx1.reshape(-1, 1)], axis=1)
+    response = getattr(clf, response_method)(grid).reshape(*disp.response.shape)
+    assert_allclose(response, disp.response)
+
+    assert len(disp.surface_) == len(clf.classes_)
+    # Get which class has highest response and check it is plotted
+    highest_class = np.argmax(response, axis=2)
+    for idx, quadmesh in enumerate(disp.surface_):
+        # Note quadmesh mask is True (i.e. masked) when `idx` is NOT the highest class
+        assert_array_equal(
+            highest_class != idx,
+            quadmesh.get_array().mask.reshape(*highest_class.shape),
+        )
+
+
+@pytest.mark.parametrize(
+    "multiclass_colors",
+    [
+        "plasma",
+        ["red", "green", "blue"],
+    ],
+)
+@pytest.mark.parametrize("plot_method", ["contourf", "contour", "pcolormesh"])
+def test_multiclass_colors_cmap(pyplot, plot_method, multiclass_colors):
+    """Check correct cmap used for all `multiclass_colors` inputs."""
+    import matplotlib as mpl
+
+    if parse_version(mpl.__version__) < parse_version("3.5"):
+        pytest.skip(
+            "Matplotlib >= 3.5 is needed for `==` to check equivalence of colormaps"
+        )
+
+    X, y = load_iris_2d_scaled()
+    clf = LogisticRegression().fit(X, y)
+
+    disp = DecisionBoundaryDisplay.from_estimator(
+        clf,
+        X,
+        plot_method=plot_method,
+        multiclass_colors=multiclass_colors,
+    )
+
+    if multiclass_colors == "plasma":
+        colors = mpl.pyplot.get_cmap(multiclass_colors, len(clf.classes_)).colors
+    else:
+        colors = [mpl.colors.to_rgba(color) for color in multiclass_colors]
+
+    cmaps = [
+        mpl.colors.LinearSegmentedColormap.from_list(
+            f"colormap_{class_idx}", [(1.0, 1.0, 1.0, 1.0), (r, g, b, 1.0)]
+        )
+        for class_idx, (r, g, b, _) in enumerate(colors)
+    ]
+
+    for idx, quad in enumerate(disp.surface_):
+        assert quad.cmap == cmaps[idx]
+
+
+def test_multiclass_plot_max_class_cmap_kwarg(pyplot):
+    """Check `cmap` kwarg ignored when using plotting max multiclass class."""
+    X, y = load_iris_2d_scaled()
+    clf = LogisticRegression().fit(X, y)
+
+    msg = (
+        "Plotting max class of multiclass 'decision_function' or 'predict_proba', "
+        "thus 'multiclass_colors' used and 'cmap' kwarg ignored."
+    )
+    with pytest.warns(UserWarning, match=msg):
+        DecisionBoundaryDisplay.from_estimator(clf, X, cmap="viridis")
+
+
+def test_subclass_named_constructors_return_type_is_subclass(pyplot):
+    """Check that named constructors return the correct type when subclassed.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/pull/27675
+    """
+    clf = LogisticRegression().fit(X, y)
+
+    class SubclassOfDisplay(DecisionBoundaryDisplay):
+        pass
+
+    curve = SubclassOfDisplay.from_estimator(estimator=clf, X=X)
+
+    assert isinstance(curve, SubclassOfDisplay)
diff --git a/sklearn/inspection/_plot/tests/test_plot_partial_dependence.py b/sklearn/inspection/_plot/tests/test_plot_partial_dependence.py
index 6ec0fde9775af..75869079be9cc 100644
--- a/sklearn/inspection/_plot/tests/test_plot_partial_dependence.py
+++ b/sklearn/inspection/_plot/tests/test_plot_partial_dependence.py
@@ -1,29 +1,30 @@
 import numpy as np
-from scipy.stats.mstats import mquantiles
-
 import pytest
 from numpy.testing import assert_allclose
+from scipy.stats.mstats import mquantiles
 
-from sklearn.datasets import load_diabetes
-from sklearn.datasets import load_iris
-from sklearn.datasets import make_classification, make_regression
-from sklearn.ensemble import GradientBoostingRegressor
-from sklearn.ensemble import GradientBoostingClassifier
+from sklearn.compose import make_column_transformer
+from sklearn.datasets import (
+    load_diabetes,
+    load_iris,
+    make_classification,
+    make_regression,
+)
+from sklearn.ensemble import GradientBoostingClassifier, GradientBoostingRegressor
+from sklearn.inspection import PartialDependenceDisplay
 from sklearn.linear_model import LinearRegression
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import OneHotEncoder
 from sklearn.utils._testing import _convert_container
 
-from sklearn.inspection import plot_partial_dependence
-
-
-# TODO: Remove when https://github.com/numpy/numpy/issues/14397 is resolved
-pytestmark = pytest.mark.filterwarnings(
-    "ignore:In future, it will be an error for 'np.bool_':DeprecationWarning:"
-    "matplotlib.*")
-
 
 @pytest.fixture(scope="module")
 def diabetes():
-    return load_diabetes()
+    # diabetes dataset, subsampled for speed
+    data = load_diabetes()
+    data.data = data.data[:50]
+    data.target = data.target[:50]
+    return data
 
 
 @pytest.fixture(scope="module")
@@ -33,18 +34,40 @@ def clf_diabetes(diabetes):
     return clf
 
 
+def custom_values_helper(feature, grid_resolution):
+    return np.linspace(
+        *mquantiles(feature, (0.05, 0.95), axis=0), num=grid_resolution, endpoint=True
+    )
+
+
 @pytest.mark.filterwarnings("ignore:A Bunch will be returned")
 @pytest.mark.parametrize("grid_resolution", [10, 20])
-def test_plot_partial_dependence(grid_resolution, pyplot, clf_diabetes,
-                                 diabetes):
+@pytest.mark.parametrize("use_custom_values", [True, False])
+def test_plot_partial_dependence(
+    use_custom_values,
+    grid_resolution,
+    pyplot,
+    clf_diabetes,
+    diabetes,
+):
     # Test partial dependence plot function.
     # Use columns 0 & 2 as 1 is not quantitative (sex)
     feature_names = diabetes.feature_names
-    disp = plot_partial_dependence(clf_diabetes, diabetes.data,
-                                   [0, 2, (0, 2)],
-                                   grid_resolution=grid_resolution,
-                                   feature_names=feature_names,
-                                   contour_kw={"cmap": "jet"})
+    custom_values = None
+    if use_custom_values:
+        custom_values = {
+            0: custom_values_helper(diabetes.data[:, 0], grid_resolution),
+            2: custom_values_helper(diabetes.data[:, 2], grid_resolution),
+        }
+    disp = PartialDependenceDisplay.from_estimator(
+        clf_diabetes,
+        diabetes.data,
+        [0, 2, (0, 2)],
+        grid_resolution=grid_resolution,
+        feature_names=feature_names,
+        contour_kw={"cmap": "jet"},
+        custom_values=custom_values,
+    )
     fig = pyplot.gcf()
     axs = fig.get_axes()
     assert disp.figure_ is fig
@@ -68,13 +91,14 @@ def test_plot_partial_dependence(grid_resolution, pyplot, clf_diabetes,
     assert disp.deciles_hlines_[0, 1] is None
     assert disp.deciles_hlines_[0, 2] is not None
 
-    assert disp.features == [(0, ), (2, ), (0, 2)]
+    assert disp.features == [(0,), (2,), (0, 2)]
     assert np.all(disp.feature_names == feature_names)
     assert len(disp.deciles) == 2
     for i in [0, 2]:
-        assert_allclose(disp.deciles[i],
-                        mquantiles(diabetes.data[:, i],
-                                   prob=np.arange(0.1, 1.0, 0.1)))
+        assert_allclose(
+            disp.deciles[i],
+            mquantiles(diabetes.data[:, i], prob=np.arange(0.1, 1.0, 0.1)),
+        )
 
     single_feature_positions = [(0, (0, 0)), (2, (0, 1))]
     expected_ylabels = ["Partial dependence", ""]
@@ -83,7 +107,6 @@ def test_plot_partial_dependence(grid_resolution, pyplot, clf_diabetes,
         ax = disp.axes_[pos]
         assert ax.get_ylabel() == expected_ylabels[i]
         assert ax.get_xlabel() == diabetes.feature_names[feat_col]
-        assert_allclose(ax.get_ylim(), disp.pdp_lim[1])
 
         line = disp.lines_[pos]
 
@@ -92,33 +115,51 @@ def test_plot_partial_dependence(grid_resolution, pyplot, clf_diabetes,
         target_idx = disp.target_idx
 
         line_data = line.get_data()
-        assert_allclose(line_data[0], avg_preds["values"][0])
+        assert_allclose(line_data[0], avg_preds["grid_values"][0])
         assert_allclose(line_data[1], avg_preds.average[target_idx].ravel())
 
     # two feature position
     ax = disp.axes_[0, 2]
     coutour = disp.contours_[0, 2]
-    expected_levels = np.linspace(*disp.pdp_lim[2], num=8)
-    assert_allclose(coutour.levels, expected_levels)
     assert coutour.get_cmap().name == "jet"
     assert ax.get_xlabel() == diabetes.feature_names[0]
     assert ax.get_ylabel() == diabetes.feature_names[2]
 
 
-@pytest.mark.filterwarnings("ignore:A Bunch will be returned")
-@pytest.mark.parametrize("kind, subsample, shape", [
-    ('average', None, (1, 3)),
-    ('individual', None, (1, 3, 442)),
-    ('both', None, (1, 3, 443)),
-    ('individual', 50, (1, 3, 50)),
-    ('both', 50, (1, 3, 51)),
-    ('individual', 0.5, (1, 3, 221)),
-    ('both', 0.5, (1, 3, 222))
-])
-def test_plot_partial_dependence_kind(pyplot, kind, subsample, shape,
-                                      clf_diabetes, diabetes):
-    disp = plot_partial_dependence(clf_diabetes, diabetes.data, [0, 1, 2],
-                                   kind=kind, subsample=subsample)
+@pytest.mark.parametrize(
+    "kind, centered, subsample, shape",
+    [
+        ("average", False, None, (1, 3)),
+        ("individual", False, None, (1, 3, 50)),
+        ("both", False, None, (1, 3, 51)),
+        ("individual", False, 20, (1, 3, 20)),
+        ("both", False, 20, (1, 3, 21)),
+        ("individual", False, 0.5, (1, 3, 25)),
+        ("both", False, 0.5, (1, 3, 26)),
+        ("average", True, None, (1, 3)),
+        ("individual", True, None, (1, 3, 50)),
+        ("both", True, None, (1, 3, 51)),
+        ("individual", True, 20, (1, 3, 20)),
+        ("both", True, 20, (1, 3, 21)),
+    ],
+)
+def test_plot_partial_dependence_kind(
+    pyplot,
+    kind,
+    centered,
+    subsample,
+    shape,
+    clf_diabetes,
+    diabetes,
+):
+    disp = PartialDependenceDisplay.from_estimator(
+        clf_diabetes,
+        diabetes.data,
+        [0, 1, 2],
+        kind=kind,
+        centered=centered,
+        subsample=subsample,
+    )
 
     assert disp.axes_.shape == (1, 3)
     assert disp.lines_.shape == shape
@@ -128,22 +169,46 @@ def test_plot_partial_dependence_kind(pyplot, kind, subsample, shape,
     assert disp.contours_[0, 1] is None
     assert disp.contours_[0, 2] is None
 
+    if centered:
+        assert all([ln._y[0] == 0.0 for ln in disp.lines_.ravel() if ln is not None])
+    else:
+        assert all([ln._y[0] != 0.0 for ln in disp.lines_.ravel() if ln is not None])
+
 
-@pytest.mark.filterwarnings("ignore:A Bunch will be returned")
 @pytest.mark.parametrize(
     "input_type, feature_names_type",
-    [('dataframe', None),
-     ('dataframe', 'list'), ('list', 'list'), ('array', 'list'),
-     ('dataframe', 'array'), ('list', 'array'), ('array', 'array'),
-     ('dataframe', 'series'), ('list', 'series'), ('array', 'series'),
-     ('dataframe', 'index'), ('list', 'index'), ('array', 'index')]
+    [
+        ("dataframe", None),
+        ("dataframe", "list"),
+        ("list", "list"),
+        ("array", "list"),
+        ("dataframe", "array"),
+        ("list", "array"),
+        ("array", "array"),
+        ("dataframe", "series"),
+        ("list", "series"),
+        ("array", "series"),
+        ("dataframe", "index"),
+        ("list", "index"),
+        ("array", "index"),
+    ],
 )
-def test_plot_partial_dependence_str_features(pyplot, clf_diabetes, diabetes,
-                                              input_type, feature_names_type):
-    if input_type == 'dataframe':
+@pytest.mark.parametrize("use_custom_values", [True, False])
+def test_plot_partial_dependence_str_features(
+    pyplot,
+    use_custom_values,
+    clf_diabetes,
+    diabetes,
+    input_type,
+    feature_names_type,
+):
+    age = diabetes.data[:, diabetes.feature_names.index("age")]
+    bmi = diabetes.data[:, diabetes.feature_names.index("bmi")]
+
+    if input_type == "dataframe":
         pd = pytest.importorskip("pandas")
         X = pd.DataFrame(diabetes.data, columns=diabetes.feature_names)
-    elif input_type == 'list':
+    elif input_type == "list":
         X = diabetes.data.tolist()
     else:
         X = diabetes.data
@@ -151,16 +216,26 @@ def test_plot_partial_dependence_str_features(pyplot, clf_diabetes, diabetes,
     if feature_names_type is None:
         feature_names = None
     else:
-        feature_names = _convert_container(diabetes.feature_names,
-                                           feature_names_type)
+        feature_names = _convert_container(diabetes.feature_names, feature_names_type)
 
     grid_resolution = 25
+    custom_values = None
+    if use_custom_values:
+        custom_values = {
+            "age": custom_values_helper(age, grid_resolution),
+            "bmi": custom_values_helper(bmi, grid_resolution),
+        }
     # check with str features and array feature names and single column
-    disp = plot_partial_dependence(clf_diabetes, X,
-                                   [('age', 'bmi'), 'bmi'],
-                                   grid_resolution=grid_resolution,
-                                   feature_names=feature_names,
-                                   n_cols=1, line_kw={"alpha": 0.8})
+    disp = PartialDependenceDisplay.from_estimator(
+        clf_diabetes,
+        X,
+        [("age", "bmi"), "bmi"],
+        grid_resolution=grid_resolution,
+        feature_names=feature_names,
+        n_cols=1,
+        line_kw={"alpha": 0.8},
+        custom_values=custom_values,
+    )
     fig = pyplot.gcf()
     axs = fig.get_axes()
     assert len(axs) == 3
@@ -190,30 +265,44 @@ def test_plot_partial_dependence_str_features(pyplot, clf_diabetes, diabetes,
     assert line.get_alpha() == 0.8
 
     line_data = line.get_data()
-    assert_allclose(line_data[0], avg_preds["values"][0])
+    assert_allclose(line_data[0], avg_preds["grid_values"][0])
     assert_allclose(line_data[1], avg_preds.average[target_idx].ravel())
 
     # contour
     ax = disp.axes_[0, 0]
-    coutour = disp.contours_[0, 0]
-    expect_levels = np.linspace(*disp.pdp_lim[2], num=8)
-    assert_allclose(coutour.levels, expect_levels)
     assert ax.get_xlabel() == "age"
     assert ax.get_ylabel() == "bmi"
 
 
 @pytest.mark.filterwarnings("ignore:A Bunch will be returned")
-def test_plot_partial_dependence_custom_axes(pyplot, clf_diabetes, diabetes):
+@pytest.mark.parametrize("use_custom_values", [True, False])
+def test_plot_partial_dependence_custom_axes(
+    use_custom_values, pyplot, clf_diabetes, diabetes
+):
     grid_resolution = 25
     fig, (ax1, ax2) = pyplot.subplots(1, 2)
-    disp = plot_partial_dependence(clf_diabetes, diabetes.data,
-                                   ['age', ('age', 'bmi')],
-                                   grid_resolution=grid_resolution,
-                                   feature_names=diabetes.feature_names,
-                                   ax=[ax1, ax2])
+
+    age = diabetes.data[:, diabetes.feature_names.index("age")]
+    bmi = diabetes.data[:, diabetes.feature_names.index("bmi")]
+    custom_values = None
+    if use_custom_values:
+        custom_values = {
+            "age": custom_values_helper(age, grid_resolution),
+            "bmi": custom_values_helper(bmi, grid_resolution),
+        }
+
+    disp = PartialDependenceDisplay.from_estimator(
+        clf_diabetes,
+        diabetes.data,
+        ["age", ("age", "bmi")],
+        grid_resolution=grid_resolution,
+        feature_names=diabetes.feature_names,
+        ax=[ax1, ax2],
+        custom_values=custom_values,
+    )
     assert fig is disp.figure_
     assert disp.bounding_ax_ is None
-    assert disp.axes_.shape == (2, )
+    assert disp.axes_.shape == (2,)
     assert disp.axes_[0] is ax1
     assert disp.axes_[1] is ax2
 
@@ -226,30 +315,48 @@ def test_plot_partial_dependence_custom_axes(pyplot, clf_diabetes, diabetes):
     target_idx = disp.target_idx
 
     line_data = line.get_data()
-    assert_allclose(line_data[0], avg_preds["values"][0])
+    assert_allclose(line_data[0], avg_preds["grid_values"][0])
     assert_allclose(line_data[1], avg_preds.average[target_idx].ravel())
 
     # contour
     ax = disp.axes_[1]
-    coutour = disp.contours_[1]
-    expect_levels = np.linspace(*disp.pdp_lim[2], num=8)
-    assert_allclose(coutour.levels, expect_levels)
     assert ax.get_xlabel() == "age"
     assert ax.get_ylabel() == "bmi"
 
 
-@pytest.mark.filterwarnings("ignore:A Bunch will be returned")
-@pytest.mark.parametrize("kind, lines", [
-    ('average', 1), ('individual', 442), ('both', 443)
-])
-def test_plot_partial_dependence_passing_numpy_axes(pyplot, clf_diabetes,
-                                                    diabetes, kind, lines):
+@pytest.mark.parametrize(
+    "kind, lines", [("average", 1), ("individual", 50), ("both", 51)]
+)
+@pytest.mark.parametrize("use_custom_values", [True, False])
+def test_plot_partial_dependence_passing_numpy_axes(
+    pyplot,
+    clf_diabetes,
+    diabetes,
+    use_custom_values,
+    kind,
+    lines,
+):
     grid_resolution = 25
     feature_names = diabetes.feature_names
-    disp1 = plot_partial_dependence(clf_diabetes, diabetes.data,
-                                    ['age', 'bmi'], kind=kind,
-                                    grid_resolution=grid_resolution,
-                                    feature_names=feature_names)
+
+    age = diabetes.data[:, diabetes.feature_names.index("age")]
+    bmi = diabetes.data[:, diabetes.feature_names.index("bmi")]
+    custom_values = None
+    if use_custom_values:
+        custom_values = {
+            "age": custom_values_helper(age, grid_resolution),
+            "bmi": custom_values_helper(bmi, grid_resolution),
+        }
+
+    disp1 = PartialDependenceDisplay.from_estimator(
+        clf_diabetes,
+        diabetes.data,
+        ["age", "bmi"],
+        kind=kind,
+        grid_resolution=grid_resolution,
+        feature_names=feature_names,
+        custom_values=custom_values,
+    )
     assert disp1.axes_.shape == (1, 2)
     assert disp1.axes_[0, 0].get_ylabel() == "Partial dependence"
     assert disp1.axes_[0, 1].get_ylabel() == ""
@@ -259,39 +366,75 @@ def test_plot_partial_dependence_passing_numpy_axes(pyplot, clf_diabetes,
     lr = LinearRegression()
     lr.fit(diabetes.data, diabetes.target)
 
-    disp2 = plot_partial_dependence(lr, diabetes.data,
-                                    ['age', 'bmi'], kind=kind,
-                                    grid_resolution=grid_resolution,
-                                    feature_names=feature_names,
-                                    ax=disp1.axes_)
+    disp2 = PartialDependenceDisplay.from_estimator(
+        lr,
+        diabetes.data,
+        ["age", "bmi"],
+        kind=kind,
+        grid_resolution=grid_resolution,
+        feature_names=feature_names,
+        ax=disp1.axes_,
+    )
 
     assert np.all(disp1.axes_ == disp2.axes_)
     assert len(disp2.axes_[0, 0].get_lines()) == 2 * lines
     assert len(disp2.axes_[0, 1].get_lines()) == 2 * lines
 
 
-@pytest.mark.filterwarnings("ignore:A Bunch will be returned")
 @pytest.mark.parametrize("nrows, ncols", [(2, 2), (3, 1)])
-def test_plot_partial_dependence_incorrent_num_axes(pyplot, clf_diabetes,
-                                                    diabetes, nrows, ncols):
+@pytest.mark.parametrize("use_custom_values", [True, False])
+def test_plot_partial_dependence_incorrent_num_axes(
+    pyplot,
+    clf_diabetes,
+    diabetes,
+    use_custom_values,
+    nrows,
+    ncols,
+):
     grid_resolution = 5
     fig, axes = pyplot.subplots(nrows, ncols)
     axes_formats = [list(axes.ravel()), tuple(axes.ravel()), axes]
 
     msg = "Expected ax to have 2 axes, got {}".format(nrows * ncols)
 
-    disp = plot_partial_dependence(clf_diabetes, diabetes.data,
-                                   ['age', 'bmi'],
-                                   grid_resolution=grid_resolution,
-                                   feature_names=diabetes.feature_names)
+    age = diabetes.data[:, diabetes.feature_names.index("age")]
+    bmi = diabetes.data[:, diabetes.feature_names.index("bmi")]
+    custom_values = None
+    if use_custom_values:
+        custom_values = {
+            "age": custom_values_helper(age, grid_resolution),
+            "bmi": custom_values_helper(bmi, grid_resolution),
+        }
+
+    age = diabetes.data[:, diabetes.feature_names.index("age")]
+    bmi = diabetes.data[:, diabetes.feature_names.index("bmi")]
+    custom_values = None
+    if use_custom_values:
+        custom_values = {
+            "age": custom_values_helper(age, grid_resolution),
+            "bmi": custom_values_helper(bmi, grid_resolution),
+        }
+
+    disp = PartialDependenceDisplay.from_estimator(
+        clf_diabetes,
+        diabetes.data,
+        ["age", "bmi"],
+        grid_resolution=grid_resolution,
+        feature_names=diabetes.feature_names,
+        custom_values=custom_values,
+    )
 
     for ax_format in axes_formats:
         with pytest.raises(ValueError, match=msg):
-            plot_partial_dependence(clf_diabetes, diabetes.data,
-                                    ['age', 'bmi'],
-                                    grid_resolution=grid_resolution,
-                                    feature_names=diabetes.feature_names,
-                                    ax=ax_format)
+            PartialDependenceDisplay.from_estimator(
+                clf_diabetes,
+                diabetes.data,
+                ["age", "bmi"],
+                grid_resolution=grid_resolution,
+                feature_names=diabetes.feature_names,
+                ax=ax_format,
+                custom_values=custom_values,
+            )
 
         # with axes object
         with pytest.raises(ValueError, match=msg):
@@ -299,8 +442,10 @@ def test_plot_partial_dependence_incorrent_num_axes(pyplot, clf_diabetes,
 
 
 @pytest.mark.filterwarnings("ignore:A Bunch will be returned")
-def test_plot_partial_dependence_with_same_axes(pyplot, clf_diabetes,
-                                                diabetes):
+@pytest.mark.parametrize("use_custom_values", [True, False])
+def test_plot_partial_dependence_with_same_axes(
+    use_custom_values, pyplot, clf_diabetes, diabetes
+):
     # The first call to plot_partial_dependence will create two new axes to
     # place in the space of the passed in axes, which results in a total of
     # three axes in the figure.
@@ -313,51 +458,108 @@ def test_plot_partial_dependence_with_same_axes(pyplot, clf_diabetes,
     # disp2 = plot_partial_dependence(..., ax=disp1.axes_)
 
     grid_resolution = 25
+
+    age = diabetes.data[:, diabetes.feature_names.index("age")]
+    bmi = diabetes.data[:, diabetes.feature_names.index("bmi")]
+    custom_values = None
+    if use_custom_values:
+        custom_values = {
+            "age": custom_values_helper(age, grid_resolution),
+            "bmi": custom_values_helper(bmi, grid_resolution),
+        }
+
     fig, ax = pyplot.subplots()
-    plot_partial_dependence(clf_diabetes, diabetes.data, ['age', 'bmi'],
-                            grid_resolution=grid_resolution,
-                            feature_names=diabetes.feature_names, ax=ax)
+    PartialDependenceDisplay.from_estimator(
+        clf_diabetes,
+        diabetes.data,
+        ["age", "bmi"],
+        grid_resolution=grid_resolution,
+        feature_names=diabetes.feature_names,
+        ax=ax,
+        custom_values=custom_values,
+    )
 
-    msg = ("The ax was already used in another plot function, please set "
-           "ax=display.axes_ instead")
+    msg = (
+        "The ax was already used in another plot function, please set "
+        "ax=display.axes_ instead"
+    )
 
     with pytest.raises(ValueError, match=msg):
-        plot_partial_dependence(clf_diabetes, diabetes.data,
-                                ['age', 'bmi'],
-                                grid_resolution=grid_resolution,
-                                feature_names=diabetes.feature_names, ax=ax)
+        PartialDependenceDisplay.from_estimator(
+            clf_diabetes,
+            diabetes.data,
+            ["age", "bmi"],
+            grid_resolution=grid_resolution,
+            feature_names=diabetes.feature_names,
+            custom_values=custom_values,
+            ax=ax,
+        )
 
 
 @pytest.mark.filterwarnings("ignore:A Bunch will be returned")
-def test_plot_partial_dependence_feature_name_reuse(pyplot, clf_diabetes,
-                                                    diabetes):
+@pytest.mark.parametrize("use_custom_values", [True, False])
+def test_plot_partial_dependence_feature_name_reuse(
+    use_custom_values, pyplot, clf_diabetes, diabetes
+):
     # second call to plot does not change the feature names from the first
     # call
+    grid_resolution = 10
+
+    custom_values = None
+    if use_custom_values:
+        custom_values = {
+            0: custom_values_helper(diabetes.data[:, 0], grid_resolution),
+            1: custom_values_helper(diabetes.data[:, 1], grid_resolution),
+        }
 
     feature_names = diabetes.feature_names
-    disp = plot_partial_dependence(clf_diabetes, diabetes.data,
-                                   [0, 1],
-                                   grid_resolution=10,
-                                   feature_names=feature_names)
+    disp = PartialDependenceDisplay.from_estimator(
+        clf_diabetes,
+        diabetes.data,
+        [0, 1],
+        grid_resolution=grid_resolution,
+        feature_names=feature_names,
+        custom_values=custom_values,
+    )
 
-    plot_partial_dependence(clf_diabetes, diabetes.data, [0, 1],
-                            grid_resolution=10, ax=disp.axes_)
+    PartialDependenceDisplay.from_estimator(
+        clf_diabetes,
+        diabetes.data,
+        [0, 1],
+        grid_resolution=grid_resolution,
+        ax=disp.axes_,
+        custom_values=custom_values,
+    )
 
     for i, ax in enumerate(disp.axes_.ravel()):
         assert ax.get_xlabel() == feature_names[i]
 
 
 @pytest.mark.filterwarnings("ignore:A Bunch will be returned")
-def test_plot_partial_dependence_multiclass(pyplot):
+@pytest.mark.parametrize("use_custom_values", [True, False])
+def test_plot_partial_dependence_multiclass(use_custom_values, pyplot):
     grid_resolution = 25
     clf_int = GradientBoostingClassifier(n_estimators=10, random_state=1)
     iris = load_iris()
 
+    custom_values = None
+    if use_custom_values:
+        custom_values = {
+            0: custom_values_helper(iris.data[:, 0], grid_resolution),
+            1: custom_values_helper(iris.data[:, 1], grid_resolution),
+        }
+
     # Test partial dependence plot function on multi-class input.
     clf_int.fit(iris.data, iris.target)
-    disp_target_0 = plot_partial_dependence(clf_int, iris.data, [0, 1],
-                                            target=0,
-                                            grid_resolution=grid_resolution)
+
+    disp_target_0 = PartialDependenceDisplay.from_estimator(
+        clf_int,
+        iris.data,
+        [0, 1],
+        target=0,
+        grid_resolution=grid_resolution,
+        custom_values=custom_values,
+    )
     assert disp_target_0.figure_ is pyplot.gcf()
     assert disp_target_0.axes_.shape == (1, 2)
     assert disp_target_0.lines_.shape == (1, 2)
@@ -371,9 +573,15 @@ def test_plot_partial_dependence_multiclass(pyplot):
     target = iris.target_names[iris.target]
     clf_symbol = GradientBoostingClassifier(n_estimators=10, random_state=1)
     clf_symbol.fit(iris.data, target)
-    disp_symbol = plot_partial_dependence(clf_symbol, iris.data, [0, 1],
-                                          target='setosa',
-                                          grid_resolution=grid_resolution)
+
+    disp_symbol = PartialDependenceDisplay.from_estimator(
+        clf_symbol,
+        iris.data,
+        [0, 1],
+        target="setosa",
+        grid_resolution=grid_resolution,
+        custom_values=custom_values,
+    )
     assert disp_symbol.figure_ is pyplot.gcf()
     assert disp_symbol.axes_.shape == (1, 2)
     assert disp_symbol.lines_.shape == (1, 2)
@@ -383,34 +591,54 @@ def test_plot_partial_dependence_multiclass(pyplot):
     assert all(c is None for c in disp_symbol.contours_.flat)
     assert disp_symbol.target_idx == 0
 
-    for int_result, symbol_result in zip(disp_target_0.pd_results,
-                                         disp_symbol.pd_results):
+    for int_result, symbol_result in zip(
+        disp_target_0.pd_results, disp_symbol.pd_results
+    ):
         assert_allclose(int_result.average, symbol_result.average)
-        assert_allclose(int_result["values"], symbol_result["values"])
+        assert_allclose(int_result["grid_values"], symbol_result["grid_values"])
 
     # check that the pd plots are different for another target
-    disp_target_1 = plot_partial_dependence(clf_int, iris.data, [0, 1],
-                                            target=1,
-                                            grid_resolution=grid_resolution)
+
+    disp_target_1 = PartialDependenceDisplay.from_estimator(
+        clf_int,
+        iris.data,
+        [0, 3],
+        target=1,
+        grid_resolution=grid_resolution,
+        custom_values=custom_values,
+    )
     target_0_data_y = disp_target_0.lines_[0, 0].get_data()[1]
     target_1_data_y = disp_target_1.lines_[0, 0].get_data()[1]
     assert any(target_0_data_y != target_1_data_y)
 
 
-multioutput_regression_data = make_regression(n_samples=50, n_targets=2,
-                                              random_state=0)
+multioutput_regression_data = make_regression(n_samples=50, n_targets=2, random_state=0)
 
 
-@pytest.mark.filterwarnings("ignore:A Bunch will be returned")
 @pytest.mark.parametrize("target", [0, 1])
-def test_plot_partial_dependence_multioutput(pyplot, target):
+@pytest.mark.parametrize("use_custom_values", [True, False])
+def test_plot_partial_dependence_multioutput(use_custom_values, pyplot, target):
     # Test partial dependence plot function on multi-output input.
     X, y = multioutput_regression_data
     clf = LinearRegression().fit(X, y)
 
     grid_resolution = 25
-    disp = plot_partial_dependence(clf, X, [0, 1], target=target,
-                                   grid_resolution=grid_resolution)
+
+    custom_values = None
+    if use_custom_values:
+        custom_values = {
+            0: custom_values_helper(X[:, 0], grid_resolution),
+            1: custom_values_helper(X[:, 1], grid_resolution),
+        }
+
+    disp = PartialDependenceDisplay.from_estimator(
+        clf,
+        X,
+        [0, 1],
+        target=target,
+        grid_resolution=grid_resolution,
+        custom_values=custom_values,
+    )
     fig = pyplot.gcf()
     axs = fig.get_axes()
     assert len(axs) == 3
@@ -423,106 +651,262 @@ def test_plot_partial_dependence_multioutput(pyplot, target):
     for i, pos in enumerate(positions):
         ax = disp.axes_[pos]
         assert ax.get_ylabel() == expected_label[i]
-        assert ax.get_xlabel() == "{}".format(i)
+        assert ax.get_xlabel() == f"x{i}"
 
 
-@pytest.mark.filterwarnings("ignore:A Bunch will be returned")
 def test_plot_partial_dependence_dataframe(pyplot, clf_diabetes, diabetes):
-    pd = pytest.importorskip('pandas')
+    pd = pytest.importorskip("pandas")
     df = pd.DataFrame(diabetes.data, columns=diabetes.feature_names)
 
     grid_resolution = 25
 
-    plot_partial_dependence(
-        clf_diabetes, df, ['bp', 's1'], grid_resolution=grid_resolution,
-        feature_names=df.columns.tolist()
+    PartialDependenceDisplay.from_estimator(
+        clf_diabetes,
+        df,
+        ["bp", "s1"],
+        grid_resolution=grid_resolution,
+        feature_names=df.columns.tolist(),
     )
 
 
 dummy_classification_data = make_classification(random_state=0)
 
 
-@pytest.mark.filterwarnings("ignore:A Bunch will be returned")
 @pytest.mark.parametrize(
     "data, params, err_msg",
-    [(multioutput_regression_data, {"target": None, 'features': [0]},
-      "target must be specified for multi-output"),
-     (multioutput_regression_data, {"target": -1, 'features': [0]},
-      r'target must be in \[0, n_tasks\]'),
-     (multioutput_regression_data, {"target": 100, 'features': [0]},
-      r'target must be in \[0, n_tasks\]'),
-     (dummy_classification_data,
-     {'features': ['foobar'], 'feature_names': None},
-     'Feature foobar not in feature_names'),
-     (dummy_classification_data,
-     {'features': ['foobar'], 'feature_names': ['abcd', 'def']},
-      'Feature foobar not in feature_names'),
-     (dummy_classification_data, {'features': [(1, 2, 3)]},
-      'Each entry in features must be either an int, '),
-     (dummy_classification_data, {'features': [1, {}]},
-      'Each entry in features must be either an int, '),
-     (dummy_classification_data, {'features': [tuple()]},
-      'Each entry in features must be either an int, '),
-     (dummy_classification_data,
-      {'features': [123], 'feature_names': ['blahblah']},
-      'All entries of features must be less than '),
-     (dummy_classification_data,
-      {'features': [0, 1, 2], 'feature_names': ['a', 'b', 'a']},
-      'feature_names should not contain duplicates'),
-     (dummy_classification_data, {'features': [(1, 2)], 'kind': 'individual'},
-      'It is not possible to display individual effects for more than one'),
-     (dummy_classification_data, {'features': [(1, 2)], 'kind': 'both'},
-      'It is not possible to display individual effects for more than one'),
-     (dummy_classification_data, {'features': [1], 'subsample': -1},
-      'When an integer, subsample=-1 should be positive.'),
-     (dummy_classification_data, {'features': [1], 'subsample': 1.2},
-      r'When a floating-point, subsample=1.2 should be in the \(0, 1\) range')]
+    [
+        (
+            multioutput_regression_data,
+            {"target": None, "features": [0]},
+            "target must be specified for multi-output",
+        ),
+        (
+            multioutput_regression_data,
+            {"target": -1, "features": [0]},
+            r"target must be in \[0, n_tasks\]",
+        ),
+        (
+            multioutput_regression_data,
+            {"target": 100, "features": [0]},
+            r"target must be in \[0, n_tasks\]",
+        ),
+        (
+            dummy_classification_data,
+            {"features": ["foobar"], "feature_names": None},
+            "Feature 'foobar' not in feature_names",
+        ),
+        (
+            dummy_classification_data,
+            {"features": ["foobar"], "feature_names": ["abcd", "def"]},
+            "Feature 'foobar' not in feature_names",
+        ),
+        (
+            dummy_classification_data,
+            {"features": [(1, 2, 3)]},
+            "Each entry in features must be either an int, ",
+        ),
+        (
+            dummy_classification_data,
+            {"features": [1, {}]},
+            "Each entry in features must be either an int, ",
+        ),
+        (
+            dummy_classification_data,
+            {"features": [tuple()]},
+            "Each entry in features must be either an int, ",
+        ),
+        (
+            dummy_classification_data,
+            {"features": [123], "feature_names": ["blahblah"]},
+            "All entries of features must be less than ",
+        ),
+        (
+            dummy_classification_data,
+            {"features": [0, 1, 2], "feature_names": ["a", "b", "a"]},
+            "feature_names should not contain duplicates",
+        ),
+        (
+            dummy_classification_data,
+            {"features": [1, 2], "kind": ["both"]},
+            "When `kind` is provided as a list of strings, it should contain",
+        ),
+        (
+            dummy_classification_data,
+            {"features": [1], "subsample": -1},
+            "When an integer, subsample=-1 should be positive.",
+        ),
+        (
+            dummy_classification_data,
+            {"features": [1], "subsample": 1.2},
+            r"When a floating-point, subsample=1.2 should be in the \(0, 1\) range",
+        ),
+        (
+            dummy_classification_data,
+            {"features": [1, 2], "categorical_features": [1.0, 2.0]},
+            "Expected `categorical_features` to be an array-like of boolean,",
+        ),
+        (
+            dummy_classification_data,
+            {"features": [(1, 2)], "categorical_features": [2]},
+            "Two-way partial dependence plots are not supported for pairs",
+        ),
+        (
+            dummy_classification_data,
+            {"features": [1], "categorical_features": [1], "kind": "individual"},
+            "It is not possible to display individual effects",
+        ),
+    ],
 )
 def test_plot_partial_dependence_error(pyplot, data, params, err_msg):
     X, y = data
     estimator = LinearRegression().fit(X, y)
 
     with pytest.raises(ValueError, match=err_msg):
-        plot_partial_dependence(estimator, X, **params)
+        PartialDependenceDisplay.from_estimator(estimator, X, **params)
 
 
-@pytest.mark.filterwarnings("ignore:A Bunch will be returned")
-@pytest.mark.parametrize("params, err_msg", [
-    ({'target': 4, 'features': [0]},
-     'target not in est.classes_, got 4'),
-    ({'target': None, 'features': [0]},
-     'target must be specified for multi-class'),
-    ({'target': 1, 'features': [4.5]},
-     'Each entry in features must be either an int,'),
-])
+@pytest.mark.parametrize(
+    "params, err_msg",
+    [
+        ({"target": 4, "features": [0]}, "target not in est.classes_, got 4"),
+        ({"target": None, "features": [0]}, "target must be specified for multi-class"),
+        (
+            {"target": 1, "features": [4.5]},
+            "Each entry in features must be either an int,",
+        ),
+    ],
+)
 def test_plot_partial_dependence_multiclass_error(pyplot, params, err_msg):
     iris = load_iris()
     clf = GradientBoostingClassifier(n_estimators=10, random_state=1)
     clf.fit(iris.data, iris.target)
 
     with pytest.raises(ValueError, match=err_msg):
-        plot_partial_dependence(clf, iris.data, **params)
+        PartialDependenceDisplay.from_estimator(clf, iris.data, **params)
 
 
-def test_plot_partial_dependence_does_not_override_ylabel(pyplot, clf_diabetes,
-                                                          diabetes):
+def test_plot_partial_dependence_does_not_override_ylabel(
+    pyplot, clf_diabetes, diabetes
+):
     # Non-regression test to be sure to not override the ylabel if it has been
     # See https://github.com/scikit-learn/scikit-learn/issues/15772
     _, axes = pyplot.subplots(1, 2)
     axes[0].set_ylabel("Hello world")
-    plot_partial_dependence(clf_diabetes, diabetes.data,
-                            [0, 1], ax=axes)
+    PartialDependenceDisplay.from_estimator(
+        clf_diabetes, diabetes.data, [0, 1], ax=axes
+    )
 
     assert axes[0].get_ylabel() == "Hello world"
     assert axes[1].get_ylabel() == "Partial dependence"
 
 
+@pytest.mark.parametrize(
+    "categorical_features, array_type",
+    [
+        (["col_A", "col_C"], "dataframe"),
+        ([0, 2], "array"),
+        ([True, False, True], "array"),
+    ],
+)
+def test_plot_partial_dependence_with_categorical(
+    pyplot, categorical_features, array_type
+):
+    X = [[1, 1, "A"], [2, 0, "C"], [3, 2, "B"]]
+    column_name = ["col_A", "col_B", "col_C"]
+    X = _convert_container(X, array_type, columns_name=column_name)
+    y = np.array([1.2, 0.5, 0.45]).T
+
+    preprocessor = make_column_transformer((OneHotEncoder(), categorical_features))
+    model = make_pipeline(preprocessor, LinearRegression())
+    model.fit(X, y)
+
+    # single feature
+    disp = PartialDependenceDisplay.from_estimator(
+        model,
+        X,
+        features=["col_C"],
+        feature_names=column_name,
+        categorical_features=categorical_features,
+    )
+
+    assert disp.figure_ is pyplot.gcf()
+    assert disp.bars_.shape == (1, 1)
+    assert disp.bars_[0][0] is not None
+    assert disp.lines_.shape == (1, 1)
+    assert disp.lines_[0][0] is None
+    assert disp.contours_.shape == (1, 1)
+    assert disp.contours_[0][0] is None
+    assert disp.deciles_vlines_.shape == (1, 1)
+    assert disp.deciles_vlines_[0][0] is None
+    assert disp.deciles_hlines_.shape == (1, 1)
+    assert disp.deciles_hlines_[0][0] is None
+    assert disp.axes_[0, 0].get_legend() is None
+
+    # interaction between two features
+    disp = PartialDependenceDisplay.from_estimator(
+        model,
+        X,
+        features=[("col_A", "col_C")],
+        feature_names=column_name,
+        categorical_features=categorical_features,
+    )
+
+    assert disp.figure_ is pyplot.gcf()
+    assert disp.bars_.shape == (1, 1)
+    assert disp.bars_[0][0] is None
+    assert disp.lines_.shape == (1, 1)
+    assert disp.lines_[0][0] is None
+    assert disp.contours_.shape == (1, 1)
+    assert disp.contours_[0][0] is None
+    assert disp.deciles_vlines_.shape == (1, 1)
+    assert disp.deciles_vlines_[0][0] is None
+    assert disp.deciles_hlines_.shape == (1, 1)
+    assert disp.deciles_hlines_[0][0] is None
+    assert disp.axes_[0, 0].get_legend() is None
+
+
+def test_plot_partial_dependence_legend(pyplot):
+    pd = pytest.importorskip("pandas")
+    X = pd.DataFrame(
+        {
+            "col_A": ["A", "B", "C"],
+            "col_B": [1.0, 0.0, 2.0],
+            "col_C": ["C", "B", "A"],
+        }
+    )
+    y = np.array([1.2, 0.5, 0.45]).T
+
+    categorical_features = ["col_A", "col_C"]
+    preprocessor = make_column_transformer((OneHotEncoder(), categorical_features))
+    model = make_pipeline(preprocessor, LinearRegression())
+    model.fit(X, y)
+
+    disp = PartialDependenceDisplay.from_estimator(
+        model,
+        X,
+        features=["col_B", "col_C"],
+        categorical_features=categorical_features,
+        kind=["both", "average"],
+    )
+
+    legend_text = disp.axes_[0, 0].get_legend().get_texts()
+    assert len(legend_text) == 1
+    assert legend_text[0].get_text() == "average"
+    assert disp.axes_[0, 1].get_legend() is None
+
+
 @pytest.mark.parametrize(
     "kind, expected_shape",
-    [("average", (1, 2)), ("individual", (1, 2, 50)), ("both", (1, 2, 51))],
+    [("average", (1, 2)), ("individual", (1, 2, 20)), ("both", (1, 2, 21))],
 )
+@pytest.mark.parametrize("use_custom_values", [True, False])
 def test_plot_partial_dependence_subsampling(
-    pyplot, clf_diabetes, diabetes, kind, expected_shape
+    pyplot,
+    clf_diabetes,
+    diabetes,
+    use_custom_values,
+    kind,
+    expected_shape,
 ):
     # check that the subsampling is properly working
     # non-regression test for:
@@ -531,23 +915,31 @@ def test_plot_partial_dependence_subsampling(
     grid_resolution = 25
     feature_names = diabetes.feature_names
 
-    disp1 = plot_partial_dependence(
+    age = diabetes.data[:, diabetes.feature_names.index("age")]
+    bmi = diabetes.data[:, diabetes.feature_names.index("bmi")]
+
+    custom_values = None
+    if use_custom_values:
+        custom_values = {
+            "age": custom_values_helper(age, grid_resolution),
+            "bmi": custom_values_helper(bmi, grid_resolution),
+        }
+
+    disp1 = PartialDependenceDisplay.from_estimator(
         clf_diabetes,
         diabetes.data,
         ["age", "bmi"],
         kind=kind,
         grid_resolution=grid_resolution,
         feature_names=feature_names,
-        subsample=50,
+        subsample=20,
         random_state=0,
+        custom_values=custom_values,
     )
 
     assert disp1.lines_.shape == expected_shape
     assert all(
-        [
-            isinstance(line, matplotlib.lines.Line2D)
-            for line in disp1.lines_.ravel()
-        ]
+        [isinstance(line, matplotlib.lines.Line2D) for line in disp1.lines_.ravel()]
     )
 
 
@@ -571,7 +963,7 @@ def test_partial_dependence_overwrite_labels(
     label,
 ):
     """Test that make sure that we can overwrite the label of the PDP plot"""
-    disp = plot_partial_dependence(
+    disp = PartialDependenceDisplay.from_estimator(
         clf_diabetes,
         diabetes.data,
         [0, 2],
@@ -588,3 +980,336 @@ def test_partial_dependence_overwrite_labels(
             legend_text = ax.get_legend().get_texts()
             assert len(legend_text) == 1
             assert legend_text[0].get_text() == label
+
+
+@pytest.mark.parametrize(
+    "categorical_features, array_type",
+    [
+        (["col_A", "col_C"], "dataframe"),
+        ([0, 2], "array"),
+        ([True, False, True], "array"),
+    ],
+)
+def test_grid_resolution_with_categorical(pyplot, categorical_features, array_type):
+    """Check that we raise a ValueError when the grid_resolution is too small
+    respect to the number of categories in the categorical features targeted.
+    """
+    X = [["A", 1, "A"], ["B", 0, "C"], ["C", 2, "B"]]
+    column_name = ["col_A", "col_B", "col_C"]
+    X = _convert_container(X, array_type, columns_name=column_name)
+    y = np.array([1.2, 0.5, 0.45]).T
+
+    preprocessor = make_column_transformer((OneHotEncoder(), categorical_features))
+    model = make_pipeline(preprocessor, LinearRegression())
+    model.fit(X, y)
+
+    err_msg = (
+        "resolution of the computed grid is less than the minimum number of categories"
+    )
+    with pytest.raises(ValueError, match=err_msg):
+        PartialDependenceDisplay.from_estimator(
+            model,
+            X,
+            features=["col_C"],
+            feature_names=column_name,
+            categorical_features=categorical_features,
+            grid_resolution=2,
+        )
+
+
+@pytest.mark.parametrize("kind", ["individual", "average", "both"])
+@pytest.mark.parametrize("centered", [True, False])
+def test_partial_dependence_plot_limits_one_way(
+    pyplot, clf_diabetes, diabetes, kind, centered
+):
+    """Check that the PD limit on the plots are properly set on one-way plots."""
+    disp = PartialDependenceDisplay.from_estimator(
+        clf_diabetes,
+        diabetes.data,
+        features=(0, 1),
+        kind=kind,
+        grid_resolution=25,
+        feature_names=diabetes.feature_names,
+    )
+
+    range_pd = np.array([-1, 1], dtype=np.float64)
+    for pd in disp.pd_results:
+        if "average" in pd:
+            pd["average"][...] = range_pd[1]
+            pd["average"][0, 0] = range_pd[0]
+        if "individual" in pd:
+            pd["individual"][...] = range_pd[1]
+            pd["individual"][0, 0, 0] = range_pd[0]
+
+    disp.plot(centered=centered)
+    # check that we anchor to zero x-axis when centering
+    y_lim = range_pd - range_pd[0] if centered else range_pd
+    padding = 0.05 * (y_lim[1] - y_lim[0])
+    y_lim[0] -= padding
+    y_lim[1] += padding
+    for ax in disp.axes_.ravel():
+        assert_allclose(ax.get_ylim(), y_lim)
+
+
+@pytest.mark.parametrize("centered", [True, False])
+def test_partial_dependence_plot_limits_two_way(
+    pyplot, clf_diabetes, diabetes, centered
+):
+    """Check that the PD limit on the plots are properly set on two-way plots."""
+    disp = PartialDependenceDisplay.from_estimator(
+        clf_diabetes,
+        diabetes.data,
+        features=[(0, 1)],
+        kind="average",
+        grid_resolution=25,
+        feature_names=diabetes.feature_names,
+    )
+
+    range_pd = np.array([-1, 1], dtype=np.float64)
+    for pd in disp.pd_results:
+        pd["average"][...] = range_pd[1]
+        pd["average"][0, 0] = range_pd[0]
+
+    disp.plot(centered=centered)
+    contours = disp.contours_[0, 0]
+    levels = range_pd - range_pd[0] if centered else range_pd
+
+    padding = 0.05 * (levels[1] - levels[0])
+    levels[0] -= padding
+    levels[1] += padding
+    expect_levels = np.linspace(*levels, num=8)
+    assert_allclose(contours.levels, expect_levels)
+
+
+def test_partial_dependence_kind_list(
+    pyplot,
+    clf_diabetes,
+    diabetes,
+):
+    """Check that we can provide a list of strings to kind parameter."""
+    matplotlib = pytest.importorskip("matplotlib")
+
+    disp = PartialDependenceDisplay.from_estimator(
+        clf_diabetes,
+        diabetes.data,
+        features=[0, 2, (1, 2)],
+        grid_resolution=20,
+        kind=["both", "both", "average"],
+    )
+
+    for idx in [0, 1]:
+        assert all(
+            [
+                isinstance(line, matplotlib.lines.Line2D)
+                for line in disp.lines_[0, idx].ravel()
+            ]
+        )
+        assert disp.contours_[0, idx] is None
+
+    assert disp.contours_[0, 2] is not None
+    assert all([line is None for line in disp.lines_[0, 2].ravel()])
+
+
+@pytest.mark.parametrize(
+    "features, kind",
+    [
+        ([0, 2, (1, 2)], "individual"),
+        ([0, 2, (1, 2)], "both"),
+        ([(0, 1), (0, 2), (1, 2)], "individual"),
+        ([(0, 1), (0, 2), (1, 2)], "both"),
+        ([0, 2, (1, 2)], ["individual", "individual", "individual"]),
+        ([0, 2, (1, 2)], ["both", "both", "both"]),
+    ],
+)
+def test_partial_dependence_kind_error(
+    pyplot,
+    clf_diabetes,
+    diabetes,
+    features,
+    kind,
+):
+    """Check that we raise an informative error when 2-way PD is requested
+    together with 1-way PD/ICE"""
+    warn_msg = (
+        "ICE plot cannot be rendered for 2-way feature interactions. 2-way "
+        "feature interactions mandates PD plots using the 'average' kind"
+    )
+    with pytest.raises(ValueError, match=warn_msg):
+        PartialDependenceDisplay.from_estimator(
+            clf_diabetes,
+            diabetes.data,
+            features=features,
+            grid_resolution=20,
+            kind=kind,
+        )
+
+
+@pytest.mark.parametrize(
+    "line_kw, pd_line_kw, ice_lines_kw, expected_colors",
+    [
+        ({"color": "r"}, {"color": "g"}, {"color": "b"}, ("g", "b")),
+        (None, {"color": "g"}, {"color": "b"}, ("g", "b")),
+        ({"color": "r"}, None, {"color": "b"}, ("r", "b")),
+        ({"color": "r"}, {"color": "g"}, None, ("g", "r")),
+        ({"color": "r"}, None, None, ("r", "r")),
+        ({"color": "r"}, {"linestyle": "--"}, {"linestyle": "-."}, ("r", "r")),
+        ({"c": "r"}, None, None, ("r", "r")),
+        ({"c": "r", "ls": "-."}, {"color": "g"}, {"color": "b"}, ("g", "b")),
+        ({"c": "r"}, {"c": "g"}, {"c": "b"}, ("g", "b")),
+        ({"c": "r"}, {"ls": "--"}, {"ls": "-."}, ("r", "r")),
+    ],
+)
+def test_plot_partial_dependence_lines_kw(
+    pyplot,
+    clf_diabetes,
+    diabetes,
+    line_kw,
+    pd_line_kw,
+    ice_lines_kw,
+    expected_colors,
+):
+    """Check that passing `pd_line_kw` and `ice_lines_kw` will act on the
+    specific lines in the plot.
+    """
+
+    disp = PartialDependenceDisplay.from_estimator(
+        clf_diabetes,
+        diabetes.data,
+        [0, 2],
+        grid_resolution=20,
+        feature_names=diabetes.feature_names,
+        n_cols=2,
+        kind="both",
+        line_kw=line_kw,
+        pd_line_kw=pd_line_kw,
+        ice_lines_kw=ice_lines_kw,
+    )
+
+    line = disp.lines_[0, 0, -1]
+    assert line.get_color() == expected_colors[0], (
+        f"{line.get_color()}!={expected_colors[0]}\n{line_kw} and {pd_line_kw}"
+    )
+    if pd_line_kw is not None:
+        if "linestyle" in pd_line_kw:
+            assert line.get_linestyle() == pd_line_kw["linestyle"]
+        elif "ls" in pd_line_kw:
+            assert line.get_linestyle() == pd_line_kw["ls"]
+    else:
+        assert line.get_linestyle() == "--"
+
+    line = disp.lines_[0, 0, 0]
+    assert line.get_color() == expected_colors[1], (
+        f"{line.get_color()}!={expected_colors[1]}"
+    )
+    if ice_lines_kw is not None:
+        if "linestyle" in ice_lines_kw:
+            assert line.get_linestyle() == ice_lines_kw["linestyle"]
+        elif "ls" in ice_lines_kw:
+            assert line.get_linestyle() == ice_lines_kw["ls"]
+    else:
+        assert line.get_linestyle() == "-"
+
+
+def test_partial_dependence_display_wrong_len_kind(
+    pyplot,
+    clf_diabetes,
+    diabetes,
+):
+    """Check that we raise an error when `kind` is a list with a wrong length.
+
+    This case can only be triggered using the `PartialDependenceDisplay.from_estimator`
+    method.
+    """
+    disp = PartialDependenceDisplay.from_estimator(
+        clf_diabetes,
+        diabetes.data,
+        features=[0, 2],
+        grid_resolution=20,
+        kind="average",  # len(kind) != len(features)
+    )
+
+    # alter `kind` to be a list with a length different from length of `features`
+    disp.kind = ["average"]
+    err_msg = (
+        r"When `kind` is provided as a list of strings, it should contain as many"
+        r" elements as `features`. `kind` contains 1 element\(s\) and `features`"
+        r" contains 2 element\(s\)."
+    )
+    with pytest.raises(ValueError, match=err_msg):
+        disp.plot()
+
+
+@pytest.mark.parametrize(
+    "kind",
+    ["individual", "both", "average", ["average", "both"], ["individual", "both"]],
+)
+def test_partial_dependence_display_kind_centered_interaction(
+    pyplot,
+    kind,
+    clf_diabetes,
+    diabetes,
+):
+    """Check that we properly center ICE and PD when passing kind as a string and as a
+    list."""
+    disp = PartialDependenceDisplay.from_estimator(
+        clf_diabetes,
+        diabetes.data,
+        [0, 1],
+        kind=kind,
+        centered=True,
+        subsample=5,
+    )
+
+    assert all([ln._y[0] == 0.0 for ln in disp.lines_.ravel() if ln is not None])
+
+
+def test_partial_dependence_display_with_constant_sample_weight(
+    pyplot,
+    clf_diabetes,
+    diabetes,
+):
+    """Check that the utilization of a constant sample weight maintains the
+    standard behavior.
+    """
+    disp = PartialDependenceDisplay.from_estimator(
+        clf_diabetes,
+        diabetes.data,
+        [0, 1],
+        kind="average",
+        method="brute",
+    )
+
+    sample_weight = np.ones_like(diabetes.target)
+    disp_sw = PartialDependenceDisplay.from_estimator(
+        clf_diabetes,
+        diabetes.data,
+        [0, 1],
+        sample_weight=sample_weight,
+        kind="average",
+        method="brute",
+    )
+
+    assert np.array_equal(
+        disp.pd_results[0]["average"], disp_sw.pd_results[0]["average"]
+    )
+
+
+def test_subclass_named_constructors_return_type_is_subclass(
+    pyplot, diabetes, clf_diabetes
+):
+    """Check that named constructors return the correct type when subclassed.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/pull/27675
+    """
+
+    class SubclassOfDisplay(PartialDependenceDisplay):
+        pass
+
+    curve = SubclassOfDisplay.from_estimator(
+        clf_diabetes,
+        diabetes.data,
+        [0, 2, (0, 2)],
+    )
+
+    assert isinstance(curve, SubclassOfDisplay)
diff --git a/sklearn/inspection/setup.py b/sklearn/inspection/setup.py
deleted file mode 100644
index e4f629d9ba0f0..0000000000000
--- a/sklearn/inspection/setup.py
+++ /dev/null
@@ -1,17 +0,0 @@
-from numpy.distutils.misc_util import Configuration
-
-
-def configuration(parent_package="", top_path=None):
-    config = Configuration("inspection", parent_package, top_path)
-
-    config.add_subpackage('_plot')
-    config.add_subpackage('_plot.tests')
-
-    config.add_subpackage('tests')
-
-    return config
-
-
-if __name__ == "__main__":
-    from numpy.distutils.core import setup
-    setup(**configuration().todict())
diff --git a/sklearn/inspection/tests/test_partial_dependence.py b/sklearn/inspection/tests/test_partial_dependence.py
index f7727210148c6..816fe5512edc4 100644
--- a/sklearn/inspection/tests/test_partial_dependence.py
+++ b/sklearn/inspection/tests/test_partial_dependence.py
@@ -2,45 +2,48 @@
 Testing for the partial dependence module.
 """
 
+import re
+import warnings
+
 import numpy as np
 import pytest
 
 import sklearn
+from sklearn.base import BaseEstimator, ClassifierMixin, clone, is_regressor
+from sklearn.cluster import KMeans
+from sklearn.compose import make_column_transformer
+from sklearn.datasets import load_iris, make_classification, make_regression
+from sklearn.dummy import DummyClassifier
+from sklearn.ensemble import (
+    GradientBoostingClassifier,
+    GradientBoostingRegressor,
+    HistGradientBoostingClassifier,
+    HistGradientBoostingRegressor,
+    RandomForestRegressor,
+)
+from sklearn.exceptions import NotFittedError
+from sklearn.impute import SimpleImputer
 from sklearn.inspection import partial_dependence
 from sklearn.inspection._partial_dependence import (
     _grid_from_X,
     _partial_dependence_brute,
-    _partial_dependence_recursion
+    _partial_dependence_recursion,
 )
-from sklearn.ensemble import GradientBoostingClassifier
-from sklearn.ensemble import GradientBoostingRegressor
-from sklearn.ensemble import RandomForestRegressor
-from sklearn.experimental import enable_hist_gradient_boosting  # noqa
-from sklearn.ensemble import HistGradientBoostingClassifier
-from sklearn.ensemble import HistGradientBoostingRegressor
-from sklearn.linear_model import LinearRegression
-from sklearn.linear_model import LogisticRegression
-from sklearn.linear_model import MultiTaskLasso
-from sklearn.tree import DecisionTreeRegressor
-from sklearn.datasets import load_iris
-from sklearn.datasets import make_classification, make_regression
-from sklearn.cluster import KMeans
-from sklearn.compose import make_column_transformer
+from sklearn.linear_model import LinearRegression, LogisticRegression, MultiTaskLasso
 from sklearn.metrics import r2_score
-from sklearn.preprocessing import PolynomialFeatures
-from sklearn.preprocessing import StandardScaler
-from sklearn.preprocessing import RobustScaler
-from sklearn.preprocessing import scale
 from sklearn.pipeline import make_pipeline
-from sklearn.dummy import DummyClassifier
-from sklearn.base import BaseEstimator, ClassifierMixin, clone
-from sklearn.exceptions import NotFittedError
-from sklearn.utils._testing import assert_allclose
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils import _IS_32BIT
-from sklearn.utils.validation import check_random_state
+from sklearn.preprocessing import (
+    OneHotEncoder,
+    PolynomialFeatures,
+    RobustScaler,
+    StandardScaler,
+    scale,
+)
+from sklearn.tree import DecisionTreeRegressor
 from sklearn.tree.tests.test_tree import assert_is_subtree
-
+from sklearn.utils._testing import assert_allclose, assert_array_equal
+from sklearn.utils.fixes import _IS_32BIT
+from sklearn.utils.validation import check_random_state
 
 # toy sample
 X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]]
@@ -48,40 +51,47 @@
 
 
 # (X, y), n_targets  <-- as expected in the output of partial_dep()
-binary_classification_data = (make_classification(n_samples=50,
-                                                  random_state=0), 1)
-multiclass_classification_data = (make_classification(n_samples=50,
-                                                      n_classes=3,
-                                                      n_clusters_per_class=1,
-                                                      random_state=0), 3)
+binary_classification_data = (make_classification(n_samples=50, random_state=0), 1)
+multiclass_classification_data = (
+    make_classification(
+        n_samples=50, n_classes=3, n_clusters_per_class=1, random_state=0
+    ),
+    3,
+)
 regression_data = (make_regression(n_samples=50, random_state=0), 1)
-multioutput_regression_data = (make_regression(n_samples=50, n_targets=2,
-                                               random_state=0), 2)
+multioutput_regression_data = (
+    make_regression(n_samples=50, n_targets=2, random_state=0),
+    2,
+)
 
 # iris
 iris = load_iris()
 
 
-@pytest.mark.filterwarnings("ignore:A Bunch will be returned")
-@pytest.mark.parametrize('Estimator, method, data', [
-    (GradientBoostingClassifier, 'auto', binary_classification_data),
-    (GradientBoostingClassifier, 'auto', multiclass_classification_data),
-    (GradientBoostingClassifier, 'brute', binary_classification_data),
-    (GradientBoostingClassifier, 'brute', multiclass_classification_data),
-    (GradientBoostingRegressor, 'auto', regression_data),
-    (GradientBoostingRegressor, 'brute', regression_data),
-    (DecisionTreeRegressor, 'brute', regression_data),
-    (LinearRegression, 'brute', regression_data),
-    (LinearRegression, 'brute', multioutput_regression_data),
-    (LogisticRegression, 'brute', binary_classification_data),
-    (LogisticRegression, 'brute', multiclass_classification_data),
-    (MultiTaskLasso, 'brute', multioutput_regression_data),
-    ])
-@pytest.mark.parametrize('grid_resolution', (5, 10))
-@pytest.mark.parametrize('features', ([1], [1, 2]))
-@pytest.mark.parametrize('kind', ('legacy', 'average', 'individual', 'both'))
-def test_output_shape(Estimator, method, data, grid_resolution,
-                      features, kind):
+@pytest.mark.parametrize(
+    "Estimator, method, data",
+    [
+        (GradientBoostingClassifier, "auto", binary_classification_data),
+        (GradientBoostingClassifier, "auto", multiclass_classification_data),
+        (GradientBoostingClassifier, "brute", binary_classification_data),
+        (GradientBoostingClassifier, "brute", multiclass_classification_data),
+        (GradientBoostingRegressor, "auto", regression_data),
+        (GradientBoostingRegressor, "brute", regression_data),
+        (DecisionTreeRegressor, "brute", regression_data),
+        (LinearRegression, "brute", regression_data),
+        (LinearRegression, "brute", multioutput_regression_data),
+        (LogisticRegression, "brute", binary_classification_data),
+        (LogisticRegression, "brute", multiclass_classification_data),
+        (MultiTaskLasso, "brute", multioutput_regression_data),
+    ],
+)
+@pytest.mark.parametrize("grid_resolution", (5, 10))
+@pytest.mark.parametrize("features", ([1], [1, 2]))
+@pytest.mark.parametrize("kind", ("average", "individual", "both"))
+@pytest.mark.parametrize("use_custom_values", [True, False])
+def test_output_shape(
+    Estimator, method, data, grid_resolution, features, kind, use_custom_values
+):
     # Check that partial_dependence has consistent output shape for different
     # kinds of estimators:
     # - classifiers with binary and multiclass settings
@@ -89,6 +99,8 @@ def test_output_shape(Estimator, method, data, grid_resolution,
     # - multi-task regressors
 
     est = Estimator()
+    if hasattr(est, "n_estimators"):
+        est.set_params(n_estimators=2)  # speed-up computations
 
     # n_target corresponds to the number of classes (1 for binary classif) or
     # the number of tasks / outputs in multi task settings. It's equal to 1 for
@@ -96,23 +108,32 @@ def test_output_shape(Estimator, method, data, grid_resolution,
     (X, y), n_targets = data
     n_instances = X.shape[0]
 
+    custom_values = None
+    if use_custom_values:
+        grid_resolution = 5
+        custom_values = {f: X[:grid_resolution, f] for f in features}
+
     est.fit(X, y)
     result = partial_dependence(
-        est, X=X, features=features, method=method, kind=kind,
-        grid_resolution=grid_resolution
-    )
-    # FIXME: Remove 'legacy' support in 1.1
-    pdp, axes = result if kind == 'legacy' else (result, result["values"])
-
-    expected_pdp_shape = (n_targets,
-                          *[grid_resolution for _ in range(len(features))])
-    expected_ice_shape = (n_targets, n_instances,
-                          *[grid_resolution for _ in range(len(features))])
-    if kind == 'legacy':
-        assert pdp.shape == expected_pdp_shape
-    elif kind == 'average':
+        est,
+        X=X,
+        features=features,
+        method=method,
+        kind=kind,
+        grid_resolution=grid_resolution,
+        custom_values=custom_values,
+    )
+    pdp, axes = result, result["grid_values"]
+
+    expected_pdp_shape = (n_targets, *[grid_resolution for _ in range(len(features))])
+    expected_ice_shape = (
+        n_targets,
+        n_instances,
+        *[grid_resolution for _ in range(len(features))],
+    )
+    if kind == "average":
         assert pdp.average.shape == expected_pdp_shape
-    elif kind == 'individual':
+    elif kind == "individual":
         assert pdp.individual.shape == expected_ice_shape
     else:  # 'both'
         assert pdp.average.shape == expected_pdp_shape
@@ -128,15 +149,12 @@ def test_grid_from_X():
 
     # Make sure that the grid is a cartesian product of the input (it will use
     # the unique values instead of the percentiles)
-    percentiles = (.05, .95)
+    percentiles = (0.05, 0.95)
     grid_resolution = 100
-    X = np.asarray([[1, 2],
-                    [3, 4]])
-    grid, axes = _grid_from_X(X, percentiles, grid_resolution)
-    assert_array_equal(grid, [[1, 2],
-                              [1, 4],
-                              [3, 2],
-                              [3, 4]])
+    is_categorical = [False, False]
+    X = np.asarray([[1, 2], [3, 4]])
+    grid, axes = _grid_from_X(X, percentiles, is_categorical, grid_resolution, {})
+    assert_array_equal(grid, [[1, 2], [1, 4], [3, 2], [3, 4]])
     assert_array_equal(axes, X.T)
 
     # test shapes of returned objects depending on the number of unique values
@@ -146,46 +164,168 @@ def test_grid_from_X():
 
     # n_unique_values > grid_resolution
     X = rng.normal(size=(20, 2))
-    grid, axes = _grid_from_X(X, percentiles, grid_resolution=grid_resolution)
+    grid, axes = _grid_from_X(
+        X,
+        percentiles,
+        is_categorical,
+        grid_resolution=grid_resolution,
+        custom_values={},
+    )
     assert grid.shape == (grid_resolution * grid_resolution, X.shape[1])
     assert np.asarray(axes).shape == (2, grid_resolution)
+    assert grid.dtype == X.dtype
 
     # n_unique_values < grid_resolution, will use actual values
     n_unique_values = 12
-    X[n_unique_values - 1:, 0] = 12345
+    X[n_unique_values - 1 :, 0] = 12345
     rng.shuffle(X)  # just to make sure the order is irrelevant
-    grid, axes = _grid_from_X(X, percentiles, grid_resolution=grid_resolution)
+    grid, axes = _grid_from_X(
+        X,
+        percentiles,
+        is_categorical,
+        grid_resolution=grid_resolution,
+        custom_values={},
+    )
     assert grid.shape == (n_unique_values * grid_resolution, X.shape[1])
     # axes is a list of arrays of different shapes
     assert axes[0].shape == (n_unique_values,)
     assert axes[1].shape == (grid_resolution,)
+    assert grid.dtype == X.dtype
+
+    # Check that uses custom_range
+    X = rng.normal(size=(20, 2))
+    X[n_unique_values - 1 :, 0] = 12345
+    col_1_range = [0, 2, 3]
+    grid, axes = _grid_from_X(
+        X,
+        percentiles,
+        is_categorical=is_categorical,
+        grid_resolution=grid_resolution,
+        custom_values={1: col_1_range},
+    )
+    assert grid.shape == (n_unique_values * len(col_1_range), X.shape[1])
+    # axes is a list of arrays of different shapes
+    assert axes[0].shape == (n_unique_values,)
+    assert axes[1].shape == (len(col_1_range),)
+    assert grid.dtype == X.dtype
+
+    # Check that grid_resolution does not impact custom_range
+    X = rng.normal(size=(20, 2))
+    col_0_range = [0, 2, 3, 4, 5, 6]
+    grid_resolution = 5
+    grid, axes = _grid_from_X(
+        X,
+        percentiles,
+        is_categorical=is_categorical,
+        grid_resolution=grid_resolution,
+        custom_values={0: col_0_range},
+    )
+    assert grid.shape == (grid_resolution * len(col_0_range), X.shape[1])
+    # axes is a list of arrays of different shapes
+    assert axes[0].shape == (len(col_0_range),)
+    assert axes[1].shape == (grid_resolution,)
+    assert grid.dtype == np.result_type(X, np.asarray(col_0_range).dtype)
+
+    X = np.array([[0, "a"], [1, "b"], [2, "c"]])
+
+    grid, axes = _grid_from_X(
+        X,
+        percentiles,
+        is_categorical=is_categorical,
+        grid_resolution=grid_resolution,
+        custom_values={1: ["a", "b", "c"]},
+    )
+    assert grid.dtype == object
+
+
+@pytest.mark.parametrize(
+    "grid_resolution",
+    [
+        2,  # since n_categories > 2, we should not use quantiles resampling
+        100,
+    ],
+)
+def test_grid_from_X_with_categorical(grid_resolution):
+    """Check that `_grid_from_X` always sample from categories and does not
+    depend from the percentiles.
+    """
+    pd = pytest.importorskip("pandas")
+    percentiles = (0.05, 0.95)
+    is_categorical = [True]
+    X = pd.DataFrame({"cat_feature": ["A", "B", "C", "A", "B", "D", "E"]})
+    grid, axes = _grid_from_X(
+        X,
+        percentiles,
+        is_categorical,
+        grid_resolution=grid_resolution,
+        custom_values={},
+    )
+    assert grid.shape == (5, X.shape[1])
+    assert axes[0].shape == (5,)
+
+
+@pytest.mark.parametrize("grid_resolution", [3, 100])
+def test_grid_from_X_heterogeneous_type(grid_resolution):
+    """Check that `_grid_from_X` always sample from categories and does not
+    depend from the percentiles.
+    """
+    pd = pytest.importorskip("pandas")
+    percentiles = (0.05, 0.95)
+    is_categorical = [True, False]
+    X = pd.DataFrame(
+        {
+            "cat": ["A", "B", "C", "A", "B", "D", "E", "A", "B", "D"],
+            "num": [1, 1, 1, 2, 5, 6, 6, 6, 6, 8],
+        }
+    )
+    nunique = X.nunique()
+
+    grid, axes = _grid_from_X(
+        X,
+        percentiles,
+        is_categorical,
+        grid_resolution=grid_resolution,
+        custom_values={},
+    )
+    if grid_resolution == 3:
+        assert grid.shape == (15, 2)
+        assert axes[0].shape[0] == nunique["num"]
+        assert axes[1].shape[0] == grid_resolution
+    else:
+        assert grid.shape == (25, 2)
+        assert axes[0].shape[0] == nunique["cat"]
+        assert axes[1].shape[0] == nunique["cat"]
 
 
 @pytest.mark.parametrize(
     "grid_resolution, percentiles, err_msg",
-    [(2, (0, 0.0001), "percentiles are too close"),
-     (100, (1, 2, 3, 4), "'percentiles' must be a sequence of 2 elements"),
-     (100, 12345, "'percentiles' must be a sequence of 2 elements"),
-     (100, (-1, .95), r"'percentiles' values must be in \[0, 1\]"),
-     (100, (.05, 2), r"'percentiles' values must be in \[0, 1\]"),
-     (100, (.9, .1), r"percentiles\[0\] must be strictly less than"),
-     (1, (0.05, 0.95), "'grid_resolution' must be strictly greater than 1")]
+    [
+        (2, (0, 0.0001), "percentiles are too close"),
+        (100, (1, 2, 3, 4), "'percentiles' must be a sequence of 2 elements"),
+        (100, 12345, "'percentiles' must be a sequence of 2 elements"),
+        (100, (-1, 0.95), r"'percentiles' values must be in \[0, 1\]"),
+        (100, (0.05, 2), r"'percentiles' values must be in \[0, 1\]"),
+        (100, (0.9, 0.1), r"percentiles\[0\] must be strictly less than"),
+        (1, (0.05, 0.95), "'grid_resolution' must be strictly greater than 1"),
+    ],
 )
 def test_grid_from_X_error(grid_resolution, percentiles, err_msg):
     X = np.asarray([[1, 2], [3, 4]])
+    is_categorical = [False]
     with pytest.raises(ValueError, match=err_msg):
-        _grid_from_X(
-            X, grid_resolution=grid_resolution, percentiles=percentiles
-        )
+        _grid_from_X(X, percentiles, is_categorical, grid_resolution, custom_values={})
 
 
-@pytest.mark.parametrize('target_feature', range(5))
-@pytest.mark.parametrize('est, method', [
-    (LinearRegression(), 'brute'),
-    (GradientBoostingRegressor(random_state=0), 'brute'),
-    (GradientBoostingRegressor(random_state=0), 'recursion'),
-    (HistGradientBoostingRegressor(random_state=0), 'brute'),
-    (HistGradientBoostingRegressor(random_state=0), 'recursion')]
+@pytest.mark.parametrize("target_feature", range(5))
+@pytest.mark.parametrize(
+    "est, method",
+    [
+        (LinearRegression(), "brute"),
+        (GradientBoostingRegressor(random_state=0), "brute"),
+        (GradientBoostingRegressor(random_state=0), "recursion"),
+        (HistGradientBoostingRegressor(random_state=0), "brute"),
+        (HistGradientBoostingRegressor(random_state=0), "recursion"),
+    ],
 )
 def test_partial_dependence_helpers(est, method, target_feature):
     # Check that what is returned by _partial_dependence_brute or
@@ -205,21 +345,23 @@ def test_partial_dependence_helpers(est, method, target_feature):
     # into account with the recursion method, for technical reasons. We set
     # the mean to 0 to that this 'bug' doesn't have any effect.
     y = y - y.mean()
-    est.fit(X, y)
+
+    # Clone is necessary to make the test thread-safe.
+    est = clone(est).fit(X, y)
 
     # target feature will be set to .5 and then to 123
-    features = np.array([target_feature], dtype=np.int32)
-    grid = np.array([[.5],
-                     [123]])
+    features = np.array([target_feature], dtype=np.intp)
+    grid = np.array([[0.5], [123]])
 
-    if method == 'brute':
-        pdp, predictions = _partial_dependence_brute(est, grid, features, X,
-                                                     response_method='auto')
+    if method == "brute":
+        pdp, predictions = _partial_dependence_brute(
+            est, grid, features, X, response_method="auto"
+        )
     else:
         pdp = _partial_dependence_recursion(est, grid, features)
 
     mean_predictions = []
-    for val in (.5, 123):
+    for val in (0.5, 123):
         X_ = X.copy()
         X_[:, target_feature] = val
         mean_predictions.append(est.predict(X_).mean())
@@ -227,11 +369,11 @@ def test_partial_dependence_helpers(est, method, target_feature):
     pdp = pdp[0]  # (shape is (1, 2) so make it (2,))
 
     # allow for greater margin for error with recursion method
-    rtol = 1e-1 if method == 'recursion' else 1e-3
+    rtol = 1e-1 if method == "recursion" else 1e-3
     assert np.allclose(pdp, mean_predictions, rtol=rtol)
 
 
-@pytest.mark.parametrize('seed', range(1))
+@pytest.mark.parametrize("seed", range(1))
 def test_recursion_decision_tree_vs_forest_and_gbdt(seed):
     # Make sure that the recursion method gives the same results on a
     # DecisionTreeRegressor and a GradientBoostingRegressor or a
@@ -255,19 +397,25 @@ def test_recursion_decision_tree_vs_forest_and_gbdt(seed):
     max_depth = 5
 
     tree_seed = 0
-    forest = RandomForestRegressor(n_estimators=1, max_features=None,
-                                   bootstrap=False, max_depth=max_depth,
-                                   random_state=tree_seed)
+    forest = RandomForestRegressor(
+        n_estimators=1,
+        max_features=None,
+        bootstrap=False,
+        max_depth=max_depth,
+        random_state=tree_seed,
+    )
     # The forest will use ensemble.base._set_random_states to set the
     # random_state of the tree sub-estimator. We simulate this here to have
     # equivalent estimators.
-    equiv_random_state = check_random_state(tree_seed).randint(
-        np.iinfo(np.int32).max)
-    gbdt = GradientBoostingRegressor(n_estimators=1, learning_rate=1,
-                                     criterion='mse', max_depth=max_depth,
-                                     random_state=equiv_random_state)
-    tree = DecisionTreeRegressor(max_depth=max_depth,
-                                 random_state=equiv_random_state)
+    equiv_random_state = check_random_state(tree_seed).randint(np.iinfo(np.int32).max)
+    gbdt = GradientBoostingRegressor(
+        n_estimators=1,
+        learning_rate=1,
+        criterion="squared_error",
+        max_depth=max_depth,
+        random_state=equiv_random_state,
+    )
+    tree = DecisionTreeRegressor(max_depth=max_depth, random_state=equiv_random_state)
 
     forest.fit(X, y)
     gbdt.fit(X, y)
@@ -286,7 +434,7 @@ def test_recursion_decision_tree_vs_forest_and_gbdt(seed):
 
     grid = rng.randn(50).reshape(-1, 1)
     for f in range(n_features):
-        features = np.array([f], dtype=np.int32)
+        features = np.array([f], dtype=np.intp)
 
         pdp_forest = _partial_dependence_recursion(forest, grid, features)
         pdp_gbdt = _partial_dependence_recursion(gbdt, grid, features)
@@ -296,42 +444,56 @@ def test_recursion_decision_tree_vs_forest_and_gbdt(seed):
         np.testing.assert_allclose(pdp_forest, pdp_tree)
 
 
-@pytest.mark.parametrize('est', (
-    GradientBoostingClassifier(random_state=0),
-    HistGradientBoostingClassifier(random_state=0),
-))
-@pytest.mark.parametrize('target_feature', (0, 1, 2, 3, 4, 5))
+@pytest.mark.parametrize(
+    "est",
+    (
+        GradientBoostingClassifier(random_state=0),
+        HistGradientBoostingClassifier(random_state=0),
+    ),
+)
+@pytest.mark.parametrize("target_feature", (0, 1, 2, 3, 4, 5))
 def test_recursion_decision_function(est, target_feature):
     # Make sure the recursion method (implicitly uses decision_function) has
     # the same result as using brute method with
     # response_method=decision_function
 
-    X, y = make_classification(n_classes=2, n_clusters_per_class=1,
-                               random_state=1)
-    assert np.mean(y) == .5  # make sure the init estimator predicts 0 anyway
+    X, y = make_classification(n_classes=2, n_clusters_per_class=1, random_state=1)
+    assert np.mean(y) == 0.5  # make sure the init estimator predicts 0 anyway
 
-    est.fit(X, y)
+    est = clone(est).fit(X, y)
 
     preds_1 = partial_dependence(
-        est, X, [target_feature], response_method='decision_function',
-        method='recursion', kind='average'
+        est,
+        X,
+        [target_feature],
+        response_method="decision_function",
+        method="recursion",
+        kind="average",
     )
     preds_2 = partial_dependence(
-        est, X, [target_feature], response_method='decision_function',
-        method='brute', kind='average'
+        est,
+        X,
+        [target_feature],
+        response_method="decision_function",
+        method="brute",
+        kind="average",
     )
 
-    assert_allclose(preds_1['average'], preds_2['average'], atol=1e-7)
+    assert_allclose(preds_1["average"], preds_2["average"], atol=1e-7)
 
 
-@pytest.mark.parametrize('est', (
-    LinearRegression(),
-    GradientBoostingRegressor(random_state=0),
-    HistGradientBoostingRegressor(random_state=0, min_samples_leaf=1,
-                                  max_leaf_nodes=None, max_iter=1),
-    DecisionTreeRegressor(random_state=0),
-))
-@pytest.mark.parametrize('power', (1, 2))
+@pytest.mark.parametrize(
+    "est",
+    (
+        LinearRegression(),
+        GradientBoostingRegressor(random_state=0),
+        HistGradientBoostingRegressor(
+            random_state=0, min_samples_leaf=1, max_leaf_nodes=None, max_iter=1
+        ),
+        DecisionTreeRegressor(random_state=0),
+    ),
+)
+@pytest.mark.parametrize("power", (1, 2))
 def test_partial_dependence_easy_target(est, power):
     # If the target y only depends on one feature in an obvious way (linear or
     # quadratic) then the partial dependence for that feature should reflect
@@ -344,47 +506,49 @@ def test_partial_dependence_easy_target(est, power):
     n_samples = 200
     target_variable = 2
     X = rng.normal(size=(n_samples, 5))
-    y = X[:, target_variable]**power
+    y = X[:, target_variable] ** power
 
-    est.fit(X, y)
+    est = clone(est).fit(X, y)
 
     pdp = partial_dependence(
-        est, features=[target_variable], X=X, grid_resolution=1000,
-        kind='average'
+        est, features=[target_variable], X=X, grid_resolution=1000, kind="average"
     )
 
-    new_X = pdp["values"][0].reshape(-1, 1)
-    new_y = pdp['average'][0]
+    new_X = pdp["grid_values"][0].reshape(-1, 1)
+    new_y = pdp["average"][0]
     # add polynomial features if needed
     new_X = PolynomialFeatures(degree=power).fit_transform(new_X)
 
     lr = LinearRegression().fit(new_X, new_y)
     r2 = r2_score(new_y, lr.predict(new_X))
 
-    assert r2 > .99
+    assert r2 > 0.99
 
 
-@pytest.mark.parametrize('Estimator',
-                         (sklearn.tree.DecisionTreeClassifier,
-                          sklearn.tree.ExtraTreeClassifier,
-                          sklearn.ensemble.ExtraTreesClassifier,
-                          sklearn.neighbors.KNeighborsClassifier,
-                          sklearn.neighbors.RadiusNeighborsClassifier,
-                          sklearn.ensemble.RandomForestClassifier))
+@pytest.mark.parametrize(
+    "Estimator",
+    (
+        sklearn.tree.DecisionTreeClassifier,
+        sklearn.tree.ExtraTreeClassifier,
+        sklearn.ensemble.ExtraTreesClassifier,
+        sklearn.neighbors.KNeighborsClassifier,
+        sklearn.neighbors.RadiusNeighborsClassifier,
+        sklearn.ensemble.RandomForestClassifier,
+    ),
+)
 def test_multiclass_multioutput(Estimator):
     # Make sure error is raised for multiclass-multioutput classifiers
 
     # make multiclass-multioutput dataset
-    X, y = make_classification(n_classes=3, n_clusters_per_class=1,
-                               random_state=0)
+    X, y = make_classification(n_classes=3, n_clusters_per_class=1, random_state=0)
     y = np.array([y, y]).T
 
     est = Estimator()
     est.fit(X, y)
 
     with pytest.raises(
-            ValueError,
-            match="Multiclass-multioutput estimators are not supported"):
+        ValueError, match="Multiclass-multioutput estimators are not supported"
+    ):
         partial_dependence(est, X, [0])
 
 
@@ -395,110 +559,102 @@ def fit(self, X, y):
         return self
 
 
-@pytest.mark.filterwarnings("ignore:A Bunch will be returned")
 @pytest.mark.parametrize(
     "estimator, params, err_msg",
-    [(KMeans(),
-      {'features': [0]},
-      "'estimator' must be a fitted regressor or classifier"),
-     (LinearRegression(),
-      {'features': [0], 'response_method': 'predict_proba'},
-      'The response_method parameter is ignored for regressors'),
-     (GradientBoostingClassifier(random_state=0),
-      {'features': [0], 'response_method': 'predict_proba',
-       'method': 'recursion'},
-      "'recursion' method, the response_method must be 'decision_function'"),
-     (GradientBoostingClassifier(random_state=0),
-      {'features': [0], 'response_method': 'predict_proba', 'method': 'auto'},
-      "'recursion' method, the response_method must be 'decision_function'"),
-     (GradientBoostingClassifier(random_state=0),
-      {'features': [0], 'response_method': 'blahblah'},
-      'response_method blahblah is invalid. Accepted response_method'),
-     (NoPredictProbaNoDecisionFunction(),
-      {'features': [0], 'response_method': 'auto'},
-      'The estimator has no predict_proba and no decision_function method'),
-     (NoPredictProbaNoDecisionFunction(),
-      {'features': [0], 'response_method': 'predict_proba'},
-      'The estimator has no predict_proba method.'),
-     (NoPredictProbaNoDecisionFunction(),
-      {'features': [0], 'response_method': 'decision_function'},
-      'The estimator has no decision_function method.'),
-     (LinearRegression(),
-      {'features': [0], 'method': 'blahblah'},
-      'blahblah is invalid. Accepted method names are brute, recursion, auto'),
-     (LinearRegression(),
-      {'features': [0], 'method': 'recursion', 'kind': 'individual'},
-      "The 'recursion' method only applies when 'kind' is set to 'average'"),
-     (LinearRegression(),
-      {'features': [0], 'method': 'recursion', 'kind': 'both'},
-      "The 'recursion' method only applies when 'kind' is set to 'average'"),
-     (LinearRegression(),
-      {'features': [0], 'method': 'recursion'},
-      "Only the following estimators support the 'recursion' method:")]
+    [
+        (
+            KMeans(random_state=0, n_init="auto"),
+            {"features": [0]},
+            "'estimator' must be a fitted regressor or classifier",
+        ),
+        (
+            LinearRegression(),
+            {"features": [0], "response_method": "predict_proba"},
+            "The response_method parameter is ignored for regressors",
+        ),
+        (
+            GradientBoostingClassifier(random_state=0),
+            {
+                "features": [0],
+                "response_method": "predict_proba",
+                "method": "recursion",
+            },
+            "'recursion' method, the response_method must be 'decision_function'",
+        ),
+        (
+            GradientBoostingClassifier(random_state=0),
+            {"features": [0], "response_method": "predict_proba", "method": "auto"},
+            "'recursion' method, the response_method must be 'decision_function'",
+        ),
+        (
+            LinearRegression(),
+            {"features": [0], "method": "recursion", "kind": "individual"},
+            "The 'recursion' method only applies when 'kind' is set to 'average'",
+        ),
+        (
+            LinearRegression(),
+            {"features": [0], "method": "recursion", "kind": "both"},
+            "The 'recursion' method only applies when 'kind' is set to 'average'",
+        ),
+        (
+            LinearRegression(),
+            {"features": [0], "method": "recursion"},
+            "Only the following estimators support the 'recursion' method:",
+        ),
+        (
+            LinearRegression(),
+            {"features": [0, 1], "custom_values": {0: [1, 2, 3], 1: np.ones((3, 3))}},
+            (
+                "The custom grid for some features is not a one-dimensional array. "
+                "Feature 1: 2 dimensions"
+            ),
+        ),
+    ],
 )
 def test_partial_dependence_error(estimator, params, err_msg):
     X, y = make_classification(random_state=0)
-    estimator.fit(X, y)
+    estimator = clone(estimator).fit(X, y)
 
     with pytest.raises(ValueError, match=err_msg):
         partial_dependence(estimator, X, **params)
 
 
 @pytest.mark.parametrize(
-    "with_dataframe, err_msg",
-    [(True, "Only array-like or scalar are supported"),
-     (False, "Only array-like or scalar are supported")]
+    "estimator", [LinearRegression(), GradientBoostingClassifier(random_state=0)]
 )
-def test_partial_dependence_slice_error(with_dataframe, err_msg):
-    X, y = make_classification(random_state=0)
-    if with_dataframe:
-        pd = pytest.importorskip('pandas')
-        X = pd.DataFrame(X)
-    estimator = LogisticRegression().fit(X, y)
-
-    with pytest.raises(TypeError, match=err_msg):
-        partial_dependence(estimator, X, features=slice(0, 2, 1))
-
-
-@pytest.mark.parametrize(
-    'estimator',
-    [LinearRegression(), GradientBoostingClassifier(random_state=0)]
-)
-@pytest.mark.parametrize('features', [-1, 10000])
+@pytest.mark.parametrize("features", [-1, 10000])
 def test_partial_dependence_unknown_feature_indices(estimator, features):
     X, y = make_classification(random_state=0)
-    estimator.fit(X, y)
+    estimator = clone(estimator).fit(X, y)
 
-    err_msg = 'all features must be in'
+    err_msg = "all features must be in"
     with pytest.raises(ValueError, match=err_msg):
         partial_dependence(estimator, X, [features])
 
 
 @pytest.mark.parametrize(
-    'estimator',
-    [LinearRegression(), GradientBoostingClassifier(random_state=0)]
+    "estimator", [LinearRegression(), GradientBoostingClassifier(random_state=0)]
 )
 def test_partial_dependence_unknown_feature_string(estimator):
     pd = pytest.importorskip("pandas")
     X, y = make_classification(random_state=0)
     df = pd.DataFrame(X)
-    estimator.fit(df, y)
+    estimator = clone(estimator).fit(df, y)
 
-    features = ['random']
-    err_msg = 'A given column is not a column of the dataframe'
+    features = ["random"]
+    err_msg = "A given column is not a column of the dataframe"
     with pytest.raises(ValueError, match=err_msg):
         partial_dependence(estimator, df, features)
 
 
 @pytest.mark.parametrize(
-    'estimator',
-    [LinearRegression(), GradientBoostingClassifier(random_state=0)]
+    "estimator", [LinearRegression(), GradientBoostingClassifier(random_state=0)]
 )
 def test_partial_dependence_X_list(estimator):
     # check that array-like objects are accepted
     X, y = make_classification(random_state=0)
-    estimator.fit(X, y)
-    partial_dependence(estimator, list(X), [0], kind='average')
+    estimator = clone(estimator).fit(X, y)
+    partial_dependence(estimator, list(X), [0], kind="average")
 
 
 def test_warning_recursion_non_constant_init():
@@ -509,17 +665,17 @@ def test_warning_recursion_non_constant_init():
     gbc.fit(X, y)
 
     with pytest.warns(
-            UserWarning,
-            match='Using recursion method with a non-constant init predictor'):
-        partial_dependence(gbc, X, [0], method='recursion', kind='average')
+        UserWarning, match="Using recursion method with a non-constant init predictor"
+    ):
+        partial_dependence(gbc, X, [0], method="recursion", kind="average")
 
     with pytest.warns(
-            UserWarning,
-            match='Using recursion method with a non-constant init predictor'):
-        partial_dependence(gbc, X, [0], method='recursion', kind='average')
+        UserWarning, match="Using recursion method with a non-constant init predictor"
+    ):
+        partial_dependence(gbc, X, [0], method="recursion", kind="average")
 
 
-def test_partial_dependence_sample_weight():
+def test_partial_dependence_sample_weight_of_fitted_estimator():
     # Test near perfect correlation between partial dependence and diagonal
     # when sample weights emphasize y = x predictions
     # non-regression test for #13193
@@ -535,14 +691,14 @@ def test_partial_dependence_sample_weight():
     X = np.c_[mask, x]
     # sample weights to emphasize data points where y = x
     sample_weight = np.ones(N)
-    sample_weight[mask] = 1000.
+    sample_weight[mask] = 1000.0
 
     clf = GradientBoostingRegressor(n_estimators=10, random_state=1)
     clf.fit(X, y, sample_weight=sample_weight)
 
-    pdp = partial_dependence(clf, X, features=[1], kind='average')
+    pdp = partial_dependence(clf, X, features=[1], kind="average")
 
-    assert np.corrcoef(pdp['average'], pdp["values"])[0, 1] > 0.99
+    assert np.corrcoef(pdp["average"], pdp["grid_values"])[0, 1] > 0.99
 
 
 def test_hist_gbdt_sw_not_supported():
@@ -550,8 +706,9 @@ def test_hist_gbdt_sw_not_supported():
     clf = HistGradientBoostingRegressor(random_state=1)
     clf.fit(X, y, sample_weight=np.ones(len(X)))
 
-    with pytest.raises(NotImplementedError,
-                       match="does not support partial dependence"):
+    with pytest.raises(
+        NotImplementedError, match="does not support partial dependence"
+    ):
         partial_dependence(clf, X, features=[1])
 
 
@@ -568,41 +725,143 @@ def test_partial_dependence_pipeline():
 
     features = 0
     pdp_pipe = partial_dependence(
-        pipe, iris.data, features=[features], grid_resolution=10,
-        kind='average'
+        pipe, iris.data, features=[features], grid_resolution=10, kind="average"
     )
     pdp_clf = partial_dependence(
-        clf, scaler.transform(iris.data), features=[features],
-        grid_resolution=10, kind='average'
+        clf,
+        scaler.transform(iris.data),
+        features=[features],
+        grid_resolution=10,
+        kind="average",
     )
-    assert_allclose(pdp_pipe['average'], pdp_clf['average'])
+    assert_allclose(pdp_pipe["average"], pdp_clf["average"])
     assert_allclose(
-        pdp_pipe["values"][0],
-        pdp_clf["values"][0] * scaler.scale_[features] + scaler.mean_[features]
+        pdp_pipe["grid_values"][0],
+        pdp_clf["grid_values"][0] * scaler.scale_[features] + scaler.mean_[features],
+    )
+
+
+@pytest.mark.parametrize(
+    "features, grid_resolution, n_vals_expected",
+    [
+        (["a"], 10, 10),
+        (["a"], 2, 2),
+    ],
+)
+def test_partial_dependence_binary_model_grid_resolution(
+    features, grid_resolution, n_vals_expected
+):
+    pd = pytest.importorskip("pandas")
+    model = DummyClassifier()
+
+    rng = np.random.RandomState(0)
+    X = pd.DataFrame(
+        {
+            "a": rng.randint(0, 10, size=100).astype(np.float64),
+            "b": rng.randint(0, 10, size=100).astype(np.float64),
+        }
+    )
+    y = pd.Series(rng.randint(0, 2, size=100))
+    model.fit(X, y)
+
+    part_dep = partial_dependence(
+        model,
+        X,
+        features=features,
+        grid_resolution=grid_resolution,
+        kind="average",
+    )
+    assert part_dep["average"].size == n_vals_expected
+
+
+@pytest.mark.parametrize(
+    "features, custom_values, n_vals_expected",
+    [
+        (["a"], {"a": [1.0, 2.0, 3.0, 4.0]}, 4),
+        (["a"], {"a": [1.0, 2.0]}, 2),
+        (["a"], {"a": [1.0]}, 1),
+    ],
+)
+def test_partial_dependence_binary_model_custom_values(
+    features, custom_values, n_vals_expected
+):
+    pd = pytest.importorskip("pandas")
+    model = DummyClassifier()
+
+    X = pd.DataFrame({"a": [1.0, 2.0, 3.0, 4.0], "b": [6.0, 7.0, 8.0, 9.0]})
+    y = pd.Series([0, 1, 0, 1])
+    model.fit(X, y)
+
+    part_dep = partial_dependence(
+        model,
+        X,
+        features=features,
+        grid_resolution=3,
+        custom_values=custom_values,
+        kind="average",
     )
+    assert part_dep["average"].size == n_vals_expected
+
+
+@pytest.mark.parametrize(
+    "features, custom_values, n_vals_expected",
+    [
+        (["b"], {"b": ["a", "b"]}, 2),
+        (["b"], {"b": ["a"]}, 1),
+        (["a", "b"], {"a": [1.0, 2.0], "b": ["a", "b"]}, 4),
+    ],
+)
+def test_partial_dependence_pipeline_custom_values(
+    features, custom_values, n_vals_expected
+):
+    pd = pytest.importorskip("pandas")
+    pl = make_pipeline(
+        SimpleImputer(strategy="most_frequent"), OneHotEncoder(), DummyClassifier()
+    )
+
+    X = pd.DataFrame({"a": [1.0, 2.0, 3.0, 4.0], "b": ["a", "b", "a", "b"]})
+    y = pd.Series([0, 1, 0, 1])
+    pl.fit(X, y)
+
+    X_holdout = pd.DataFrame({"a": [1.0, 2.0, 3.0, 4.0], "b": ["a", "b", "a", None]})
+    part_dep = partial_dependence(
+        pl,
+        X_holdout,
+        features=features,
+        grid_resolution=3,
+        custom_values=custom_values,
+        kind="average",
+    )
+    assert part_dep["average"].size == n_vals_expected
 
 
 @pytest.mark.parametrize(
     "estimator",
-    [LogisticRegression(max_iter=1000, random_state=0),
-     GradientBoostingClassifier(random_state=0, n_estimators=5)],
-    ids=['estimator-brute', 'estimator-recursion']
+    [
+        LogisticRegression(max_iter=1000, random_state=0),
+        GradientBoostingClassifier(random_state=0, n_estimators=5),
+    ],
+    ids=["estimator-brute", "estimator-recursion"],
 )
 @pytest.mark.parametrize(
     "preprocessor",
-    [None,
-     make_column_transformer(
-         (StandardScaler(), [iris.feature_names[i] for i in (0, 2)]),
-         (RobustScaler(), [iris.feature_names[i] for i in (1, 3)])),
-     make_column_transformer(
-         (StandardScaler(), [iris.feature_names[i] for i in (0, 2)]),
-         remainder='passthrough')],
-    ids=['None', 'column-transformer', 'column-transformer-passthrough']
+    [
+        None,
+        make_column_transformer(
+            (StandardScaler(), [iris.feature_names[i] for i in (0, 2)]),
+            (RobustScaler(), [iris.feature_names[i] for i in (1, 3)]),
+        ),
+        make_column_transformer(
+            (StandardScaler(), [iris.feature_names[i] for i in (0, 2)]),
+            remainder="passthrough",
+        ),
+    ],
+    ids=["None", "column-transformer", "column-transformer-passthrough"],
 )
 @pytest.mark.parametrize(
     "features",
     [[0, 2], [iris.feature_names[i] for i in (0, 2)]],
-    ids=['features-integer', 'features-string']
+    ids=["features-integer", "features-string"],
 )
 def test_partial_dependence_dataframe(estimator, preprocessor, features):
     # check that the partial dependence support dataframe and pipeline
@@ -610,10 +869,10 @@ def test_partial_dependence_dataframe(estimator, preprocessor, features):
     pd = pytest.importorskip("pandas")
     df = pd.DataFrame(scale(iris.data), columns=iris.feature_names)
 
-    pipe = make_pipeline(preprocessor, estimator)
+    pipe = make_pipeline(preprocessor, clone(estimator))
     pipe.fit(df, iris.target)
     pdp_pipe = partial_dependence(
-        pipe, df, features=features, grid_resolution=10, kind='average'
+        pipe, df, features=features, grid_resolution=10, kind="average"
     )
 
     # the column transformer will reorder the column when transforming
@@ -628,53 +887,95 @@ def test_partial_dependence_dataframe(estimator, preprocessor, features):
 
     clf = clone(estimator).fit(X_proc, iris.target)
     pdp_clf = partial_dependence(
-        clf, X_proc, features=features_clf, method='brute', grid_resolution=10,
-        kind='average'
+        clf,
+        X_proc,
+        features=features_clf,
+        method="brute",
+        grid_resolution=10,
+        kind="average",
     )
 
-    assert_allclose(pdp_pipe['average'], pdp_clf['average'])
+    assert_allclose(pdp_pipe["average"], pdp_clf["average"])
     if preprocessor is not None:
-        scaler = preprocessor.named_transformers_['standardscaler']
+        scaler = preprocessor.named_transformers_["standardscaler"]
         assert_allclose(
-            pdp_pipe["values"][1],
-            pdp_clf["values"][1] * scaler.scale_[1] + scaler.mean_[1]
+            pdp_pipe["grid_values"][1],
+            pdp_clf["grid_values"][1] * scaler.scale_[1] + scaler.mean_[1],
         )
     else:
-        assert_allclose(pdp_pipe["values"][1], pdp_clf["values"][1])
+        assert_allclose(pdp_pipe["grid_values"][1], pdp_clf["grid_values"][1])
 
 
 @pytest.mark.parametrize(
-    "features, expected_pd_shape",
-    [(0, (3, 10)),
-     (iris.feature_names[0], (3, 10)),
-     ([0, 2], (3, 10, 10)),
-     ([iris.feature_names[i] for i in (0, 2)], (3, 10, 10)),
-     ([True, False, True, False], (3, 10, 10))],
-    ids=['scalar-int', 'scalar-str', 'list-int', 'list-str', 'mask']
+    "features, custom_values, expected_pd_shape",
+    [
+        (0, None, (3, 10)),
+        (0, {0: [1.0, 2.0, 3.0]}, (3, 3)),
+        (iris.feature_names[0], None, (3, 10)),
+        (iris.feature_names[0], {iris.feature_names[0]: np.array([1.0, 2.0])}, (3, 2)),
+        ([0, 2], None, (3, 10, 10)),
+        ([0, 2], {2: [7, 8, 9, 10]}, (3, 10, 4)),
+        ([iris.feature_names[i] for i in (0, 2)], None, (3, 10, 10)),
+        (
+            [iris.feature_names[i] for i in (0, 2)],
+            {iris.feature_names[2]: [1, 2, 3, 10]},
+            (3, 10, 4),
+        ),
+        ([iris.feature_names[i] for i in (0, 2)], {2: [1, 2, 3, 10]}, (3, 10, 10)),
+        (
+            [iris.feature_names[i] for i in (0, 2, 3)],
+            {iris.feature_names[2]: [1, 10]},
+            (3, 10, 2, 10),
+        ),
+        ([True, False, True, False], None, (3, 10, 10)),
+    ],
+    ids=[
+        "scalar-int",
+        "scalar-int-custom-values",
+        "scalar-str",
+        "scalar-str-custom-values",
+        "list-int",
+        "list-int-custom-values",
+        "list-str",
+        "list-str-custom-values",
+        "list-str-custom-values-incorrect",
+        "list-str-three-features",
+        "mask",
+    ],
 )
-def test_partial_dependence_feature_type(features, expected_pd_shape):
+def test_partial_dependence_feature_type(features, custom_values, expected_pd_shape):
     # check all possible features type supported in PDP
     pd = pytest.importorskip("pandas")
     df = pd.DataFrame(iris.data, columns=iris.feature_names)
 
     preprocessor = make_column_transformer(
         (StandardScaler(), [iris.feature_names[i] for i in (0, 2)]),
-        (RobustScaler(), [iris.feature_names[i] for i in (1, 3)])
+        (RobustScaler(), [iris.feature_names[i] for i in (1, 3)]),
     )
     pipe = make_pipeline(
         preprocessor, LogisticRegression(max_iter=1000, random_state=0)
     )
     pipe.fit(df, iris.target)
     pdp_pipe = partial_dependence(
-        pipe, df, features=features, grid_resolution=10, kind='average'
+        pipe,
+        df,
+        features=features,
+        grid_resolution=10,
+        kind="average",
+        custom_values=custom_values,
     )
-    assert pdp_pipe['average'].shape == expected_pd_shape
-    assert len(pdp_pipe["values"]) == len(pdp_pipe['average'].shape) - 1
+    assert pdp_pipe["average"].shape == expected_pd_shape
+    assert len(pdp_pipe["grid_values"]) == len(pdp_pipe["average"].shape) - 1
 
 
 @pytest.mark.parametrize(
-    "estimator", [LinearRegression(), LogisticRegression(),
-                  GradientBoostingRegressor(), GradientBoostingClassifier()]
+    "estimator",
+    [
+        LinearRegression(),
+        LogisticRegression(),
+        GradientBoostingRegressor(),
+        GradientBoostingClassifier(),
+    ],
 )
 def test_partial_dependence_unfitted(estimator):
     X = iris.data
@@ -688,33 +989,229 @@ def test_partial_dependence_unfitted(estimator):
         partial_dependence(estimator, X, features=[0, 2], grid_resolution=10)
 
 
-@pytest.mark.parametrize('Estimator, data', [
-    (LinearRegression, multioutput_regression_data),
-    (LogisticRegression, binary_classification_data)])
+@pytest.mark.parametrize(
+    "Estimator, data",
+    [
+        (LinearRegression, multioutput_regression_data),
+        (LogisticRegression, binary_classification_data),
+    ],
+)
 def test_kind_average_and_average_of_individual(Estimator, data):
     est = Estimator()
     (X, y), n_targets = data
     est.fit(X, y)
 
-    pdp_avg = partial_dependence(
-            est, X=X, features=[1, 2], kind='average'
+    pdp_avg = partial_dependence(est, X=X, features=[1, 2], kind="average")
+    pdp_ind = partial_dependence(est, X=X, features=[1, 2], kind="individual")
+    avg_ind = np.mean(pdp_ind["individual"], axis=1)
+    assert_allclose(avg_ind, pdp_avg["average"])
+
+
+@pytest.mark.parametrize(
+    "Estimator, data",
+    [
+        (LinearRegression, multioutput_regression_data),
+        (LogisticRegression, binary_classification_data),
+    ],
+)
+def test_partial_dependence_kind_individual_ignores_sample_weight(Estimator, data):
+    """Check that `sample_weight` does not have any effect on reported ICE."""
+    est = Estimator()
+    (X, y), n_targets = data
+    sample_weight = np.arange(X.shape[0])
+    est.fit(X, y)
+
+    pdp_nsw = partial_dependence(est, X=X, features=[1, 2], kind="individual")
+    pdp_sw = partial_dependence(
+        est, X=X, features=[1, 2], kind="individual", sample_weight=sample_weight
     )
-    pdp_ind = partial_dependence(
-        est, X=X, features=[1, 2], kind='individual'
+    assert_allclose(pdp_nsw["individual"], pdp_sw["individual"])
+    assert_allclose(pdp_nsw["grid_values"], pdp_sw["grid_values"])
+
+
+@pytest.mark.parametrize(
+    "estimator",
+    [
+        LinearRegression(),
+        LogisticRegression(),
+        RandomForestRegressor(),
+        GradientBoostingClassifier(),
+    ],
+)
+@pytest.mark.parametrize("non_null_weight_idx", [0, 1, -1])
+def test_partial_dependence_non_null_weight_idx(estimator, non_null_weight_idx):
+    """Check that if we pass a `sample_weight` of zeros with only one index with
+    sample weight equals one, then the average `partial_dependence` with this
+    `sample_weight` is equal to the individual `partial_dependence` of the
+    corresponding index.
+    """
+    X, y = iris.data, iris.target
+    preprocessor = make_column_transformer(
+        (StandardScaler(), [0, 2]), (RobustScaler(), [1, 3])
+    )
+    pipe = make_pipeline(preprocessor, clone(estimator)).fit(X, y)
+
+    sample_weight = np.zeros_like(y)
+    sample_weight[non_null_weight_idx] = 1
+    pdp_sw = partial_dependence(
+        pipe,
+        X,
+        [2, 3],
+        kind="average",
+        sample_weight=sample_weight,
+        grid_resolution=10,
     )
-    avg_ind = np.mean(pdp_ind['individual'], axis=1)
-    assert_allclose(avg_ind, pdp_avg['average'])
+    pdp_ind = partial_dependence(pipe, X, [2, 3], kind="individual", grid_resolution=10)
+    output_dim = 1 if is_regressor(pipe) else len(np.unique(y))
+    for i in range(output_dim):
+        assert_allclose(
+            pdp_ind["individual"][i][non_null_weight_idx],
+            pdp_sw["average"][i],
+        )
 
 
-def test_warning_for_kind_legacy():
+@pytest.mark.parametrize(
+    "Estimator, data",
+    [
+        (LinearRegression, multioutput_regression_data),
+        (LogisticRegression, binary_classification_data),
+    ],
+)
+def test_partial_dependence_equivalence_equal_sample_weight(Estimator, data):
+    """Check that `sample_weight=None` is equivalent to having equal weights."""
+
+    est = Estimator()
+    (X, y), n_targets = data
+    est.fit(X, y)
+
+    sample_weight, params = None, {"X": X, "features": [1, 2], "kind": "average"}
+    pdp_sw_none = partial_dependence(est, **params, sample_weight=sample_weight)
+    sample_weight = np.ones(len(y))
+    pdp_sw_unit = partial_dependence(est, **params, sample_weight=sample_weight)
+    assert_allclose(pdp_sw_none["average"], pdp_sw_unit["average"])
+    sample_weight = 2 * np.ones(len(y))
+    pdp_sw_doubling = partial_dependence(est, **params, sample_weight=sample_weight)
+    assert_allclose(pdp_sw_none["average"], pdp_sw_doubling["average"])
+
+
+def test_partial_dependence_sample_weight_size_error():
+    """Check that we raise an error when the size of `sample_weight` is not
+    consistent with `X` and `y`.
+    """
     est = LogisticRegression()
     (X, y), n_targets = binary_classification_data
+    sample_weight = np.ones_like(y)
     est.fit(X, y)
 
-    err_msg = ("A Bunch will be returned in place of 'predictions' from "
-               "version 1.1")
-    with pytest.warns(FutureWarning, match=err_msg):
-        partial_dependence(est, X=X, features=[1, 2])
+    with pytest.raises(ValueError, match="sample_weight.shape =="):
+        partial_dependence(
+            est, X, features=[0], sample_weight=sample_weight[1:], grid_resolution=10
+        )
+
+
+def test_partial_dependence_sample_weight_with_recursion():
+    """Check that we raise an error when `sample_weight` is provided with
+    `"recursion"` method.
+    """
+    est = RandomForestRegressor()
+    (X, y), n_targets = regression_data
+    sample_weight = np.ones_like(y)
+    est.fit(X, y, sample_weight=sample_weight)
+
+    with pytest.raises(ValueError, match="'recursion' method can only be applied when"):
+        partial_dependence(
+            est, X, features=[0], method="recursion", sample_weight=sample_weight
+        )
+
 
-    with pytest.warns(FutureWarning, match=err_msg):
-        partial_dependence(est, X=X, features=[1, 2], kind='legacy')
+def test_mixed_type_categorical():
+    """Check that we raise a proper error when a column has mixed types and
+    the sorting of `np.unique` will fail."""
+    X = np.array(["A", "B", "C", np.nan], dtype=object).reshape(-1, 1)
+    y = np.array([0, 1, 0, 1])
+
+    from sklearn.preprocessing import OrdinalEncoder
+
+    clf = make_pipeline(
+        OrdinalEncoder(encoded_missing_value=-1),
+        LogisticRegression(),
+    ).fit(X, y)
+    with pytest.raises(ValueError, match="The column #0 contains mixed data types"):
+        partial_dependence(clf, X, features=[0])
+
+
+def test_reject_array_with_integer_dtype():
+    X = np.arange(8).reshape(4, 2)
+    y = np.array([0, 1, 0, 1])
+    clf = DummyClassifier()
+    clf.fit(X, y)
+    with pytest.warns(
+        FutureWarning, match=re.escape("The column 0 contains integer data.")
+    ):
+        partial_dependence(clf, X, features=0)
+
+    with pytest.warns(
+        FutureWarning, match=re.escape("The column 1 contains integer data.")
+    ):
+        partial_dependence(clf, X, features=[1], categorical_features=[0])
+
+    with pytest.warns(
+        FutureWarning, match=re.escape("The column 0 contains integer data.")
+    ):
+        partial_dependence(clf, X, features=[0, 1])
+
+    # The following should not raise as we do not compute numerical partial
+    # dependence on integer columns.
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        partial_dependence(clf, X, features=1, categorical_features=[1])
+
+
+def test_reject_pandas_with_integer_dtype():
+    pd = pytest.importorskip("pandas")
+    X = pd.DataFrame(
+        {
+            "a": [1.0, 2.0, 3.0],
+            "b": [1, 2, 3],
+            "c": [1, 2, 3],
+        }
+    )
+    y = np.array([0, 1, 0])
+    clf = DummyClassifier()
+    clf.fit(X, y)
+
+    with pytest.warns(
+        FutureWarning, match=re.escape("The column 'c' contains integer data.")
+    ):
+        partial_dependence(clf, X, features="c")
+
+    with pytest.warns(
+        FutureWarning, match=re.escape("The column 'c' contains integer data.")
+    ):
+        partial_dependence(clf, X, features=["a", "c"])
+
+    # The following should not raise as we do not compute numerical partial
+    # dependence on integer columns.
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        partial_dependence(clf, X, features=["a"])
+        partial_dependence(clf, X, features=["c"], categorical_features=["c"])
+
+
+def test_partial_dependence_empty_categorical_features():
+    """Check that we raise the proper exception when `categorical_features`
+    is an empty list"""
+    clf = make_pipeline(StandardScaler(), LogisticRegression())
+    clf.fit(iris.data, iris.target)
+
+    with pytest.raises(
+        ValueError,
+        match=re.escape(
+            "Passing an empty list (`[]`) to `categorical_features` is not "
+            "supported. Use `None` instead to indicate that there are no "
+            "categorical features."
+        ),
+    ):
+        partial_dependence(
+            estimator=clf, X=iris.data, features=[0], categorical_features=[]
+        )
diff --git a/sklearn/inspection/tests/test_pd_utils.py b/sklearn/inspection/tests/test_pd_utils.py
new file mode 100644
index 0000000000000..5dea3834a77a7
--- /dev/null
+++ b/sklearn/inspection/tests/test_pd_utils.py
@@ -0,0 +1,47 @@
+import numpy as np
+import pytest
+
+from sklearn.inspection._pd_utils import _check_feature_names, _get_feature_index
+from sklearn.utils._testing import _convert_container
+
+
+@pytest.mark.parametrize(
+    "feature_names, array_type, expected_feature_names",
+    [
+        (None, "array", ["x0", "x1", "x2"]),
+        (None, "dataframe", ["a", "b", "c"]),
+        (np.array(["a", "b", "c"]), "array", ["a", "b", "c"]),
+    ],
+)
+def test_check_feature_names(feature_names, array_type, expected_feature_names):
+    X = np.random.randn(10, 3)
+    column_names = ["a", "b", "c"]
+    X = _convert_container(X, constructor_name=array_type, columns_name=column_names)
+    feature_names_validated = _check_feature_names(X, feature_names)
+    assert feature_names_validated == expected_feature_names
+
+
+def test_check_feature_names_error():
+    X = np.random.randn(10, 3)
+    feature_names = ["a", "b", "c", "a"]
+    msg = "feature_names should not contain duplicates."
+    with pytest.raises(ValueError, match=msg):
+        _check_feature_names(X, feature_names)
+
+
+@pytest.mark.parametrize("fx, idx", [(0, 0), (1, 1), ("a", 0), ("b", 1), ("c", 2)])
+def test_get_feature_index(fx, idx):
+    feature_names = ["a", "b", "c"]
+    assert _get_feature_index(fx, feature_names) == idx
+
+
+@pytest.mark.parametrize(
+    "fx, feature_names, err_msg",
+    [
+        ("a", None, "Cannot plot partial dependence for feature 'a'"),
+        ("d", ["a", "b", "c"], "Feature 'd' not in feature_names"),
+    ],
+)
+def test_get_feature_names_error(fx, feature_names, err_msg):
+    with pytest.raises(ValueError, match=err_msg):
+        _get_feature_index(fx, feature_names)
diff --git a/sklearn/inspection/tests/test_permutation_importance.py b/sklearn/inspection/tests/test_permutation_importance.py
index e0c877d3f9a03..b51ad7b71f66d 100644
--- a/sklearn/inspection/tests/test_permutation_importance.py
+++ b/sklearn/inspection/tests/test_permutation_importance.py
@@ -1,64 +1,74 @@
-import pytest
 import numpy as np
-
+import pytest
+from joblib import parallel_backend
 from numpy.testing import assert_allclose
 
 from sklearn.compose import ColumnTransformer
-from sklearn.datasets import load_diabetes
-from sklearn.datasets import load_iris
-from sklearn.datasets import make_classification
-from sklearn.datasets import make_regression
+from sklearn.datasets import (
+    load_diabetes,
+    load_iris,
+    make_classification,
+    make_regression,
+)
 from sklearn.dummy import DummyClassifier
-from sklearn.ensemble import RandomForestRegressor
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.linear_model import LinearRegression
-from sklearn.linear_model import LogisticRegression
+from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
 from sklearn.impute import SimpleImputer
 from sklearn.inspection import permutation_importance
-from sklearn.model_selection import train_test_split
+from sklearn.linear_model import LinearRegression, LogisticRegression
 from sklearn.metrics import (
     get_scorer,
     mean_squared_error,
     r2_score,
 )
+from sklearn.model_selection import train_test_split
 from sklearn.pipeline import make_pipeline
-from sklearn.preprocessing import KBinsDiscretizer
-from sklearn.preprocessing import OneHotEncoder
-from sklearn.preprocessing import StandardScaler
-from sklearn.preprocessing import scale
-from sklearn.utils import parallel_backend
+from sklearn.preprocessing import KBinsDiscretizer, OneHotEncoder, StandardScaler, scale
 from sklearn.utils._testing import _convert_container
 
 
 @pytest.mark.parametrize("n_jobs", [1, 2])
-def test_permutation_importance_correlated_feature_regression(n_jobs):
+@pytest.mark.parametrize("max_samples", [0.5, 1.0])
+@pytest.mark.parametrize("sample_weight", [None, "ones"])
+def test_permutation_importance_correlated_feature_regression(
+    n_jobs, max_samples, sample_weight
+):
     # Make sure that feature highly correlated to the target have a higher
     # importance
     rng = np.random.RandomState(42)
     n_repeats = 5
 
     X, y = load_diabetes(return_X_y=True)
-    y_with_little_noise = (
-        y + rng.normal(scale=0.001, size=y.shape[0])).reshape(-1, 1)
+    y_with_little_noise = (y + rng.normal(scale=0.001, size=y.shape[0])).reshape(-1, 1)
 
     X = np.hstack([X, y_with_little_noise])
 
+    weights = np.ones_like(y) if sample_weight == "ones" else sample_weight
     clf = RandomForestRegressor(n_estimators=10, random_state=42)
     clf.fit(X, y)
 
-    result = permutation_importance(clf, X, y, n_repeats=n_repeats,
-                                    random_state=rng, n_jobs=n_jobs)
+    result = permutation_importance(
+        clf,
+        X,
+        y,
+        sample_weight=weights,
+        n_repeats=n_repeats,
+        random_state=rng,
+        n_jobs=n_jobs,
+        max_samples=max_samples,
+    )
 
     assert result.importances.shape == (X.shape[1], n_repeats)
 
     # the correlated feature with y was added as the last column and should
     # have the highest importance
-    assert np.all(result.importances_mean[-1] >
-                  result.importances_mean[:-1])
+    assert np.all(result.importances_mean[-1] > result.importances_mean[:-1])
 
 
 @pytest.mark.parametrize("n_jobs", [1, 2])
-def test_permutation_importance_correlated_feature_regression_pandas(n_jobs):
+@pytest.mark.parametrize("max_samples", [0.5, 1.0])
+def test_permutation_importance_correlated_feature_regression_pandas(
+    n_jobs, max_samples
+):
     pd = pytest.importorskip("pandas")
 
     # Make sure that feature highly correlated to the target have a higher
@@ -68,18 +78,24 @@ def test_permutation_importance_correlated_feature_regression_pandas(n_jobs):
 
     dataset = load_iris()
     X, y = dataset.data, dataset.target
-    y_with_little_noise = (
-        y + rng.normal(scale=0.001, size=y.shape[0])).reshape(-1, 1)
+    y_with_little_noise = (y + rng.normal(scale=0.001, size=y.shape[0])).reshape(-1, 1)
 
     # Adds feature correlated with y as the last column
     X = pd.DataFrame(X, columns=dataset.feature_names)
-    X['correlated_feature'] = y_with_little_noise
+    X["correlated_feature"] = y_with_little_noise
 
     clf = RandomForestClassifier(n_estimators=10, random_state=42)
     clf.fit(X, y)
 
-    result = permutation_importance(clf, X, y, n_repeats=n_repeats,
-                                    random_state=rng, n_jobs=n_jobs)
+    result = permutation_importance(
+        clf,
+        X,
+        y,
+        n_repeats=n_repeats,
+        random_state=rng,
+        n_jobs=n_jobs,
+        max_samples=max_samples,
+    )
 
     assert result.importances.shape == (X.shape[1], n_repeats)
 
@@ -89,7 +105,8 @@ def test_permutation_importance_correlated_feature_regression_pandas(n_jobs):
 
 
 @pytest.mark.parametrize("n_jobs", [1, 2])
-def test_robustness_to_high_cardinality_noisy_feature(n_jobs, seed=42):
+@pytest.mark.parametrize("max_samples", [0.5, 1.0])
+def test_robustness_to_high_cardinality_noisy_feature(n_jobs, max_samples, seed=42):
     # Permutation variable importance should not be affected by the high
     # cardinality bias of traditional feature importances, especially when
     # computed on a held-out test set:
@@ -106,8 +123,7 @@ def test_robustness_to_high_cardinality_noisy_feature(n_jobs, seed=42):
     # while leaving some classes unexplained to make the problem harder.
     classes = np.arange(n_classes)
     y = rng.choice(classes, size=n_samples)
-    X = np.hstack([(y == c).reshape(-1, 1)
-                   for c in classes[:n_informative_features]])
+    X = np.hstack([(y == c).reshape(-1, 1) for c in classes[:n_informative_features]])
     X = X.astype(np.float32)
 
     # Not all target classes are explained by the binary class indicator
@@ -123,7 +139,8 @@ def test_robustness_to_high_cardinality_noisy_feature(n_jobs, seed=42):
     # Test size should be large enough for importance measurements to be
     # stable:
     X_train, X_test, y_train, y_test = train_test_split(
-        X, y, test_size=0.5, random_state=rng)
+        X, y, test_size=0.5, random_state=rng
+    )
     clf = RandomForestClassifier(n_estimators=5, random_state=rng)
     clf.fit(X_train, y_train)
 
@@ -137,8 +154,15 @@ def test_robustness_to_high_cardinality_noisy_feature(n_jobs, seed=42):
 
     # Let's check that permutation-based feature importances do not have this
     # problem.
-    r = permutation_importance(clf, X_test, y_test, n_repeats=n_repeats,
-                               random_state=rng, n_jobs=n_jobs)
+    r = permutation_importance(
+        clf,
+        X_test,
+        y_test,
+        n_repeats=n_repeats,
+        random_state=rng,
+        n_jobs=n_jobs,
+        max_samples=max_samples,
+    )
 
     assert r.importances.shape == (X.shape[1], n_repeats)
 
@@ -169,10 +193,9 @@ def test_permutation_importance_mixed_types():
     X = np.array([[1.0, 2.0, 3.0, np.nan], [2, 1, 2, 1]]).T
     y = np.array([0, 1, 0, 1])
 
-    clf = make_pipeline(SimpleImputer(), LogisticRegression(solver='lbfgs'))
+    clf = make_pipeline(SimpleImputer(), LogisticRegression(solver="lbfgs"))
     clf.fit(X, y)
-    result = permutation_importance(clf, X, y, n_repeats=n_repeats,
-                                    random_state=rng)
+    result = permutation_importance(clf, X, y, n_repeats=n_repeats, random_state=rng)
 
     assert result.importances.shape == (X.shape[1], n_repeats)
 
@@ -182,8 +205,7 @@ def test_permutation_importance_mixed_types():
 
     # use another random state
     rng = np.random.RandomState(0)
-    result2 = permutation_importance(clf, X, y, n_repeats=n_repeats,
-                                     random_state=rng)
+    result2 = permutation_importance(clf, X, y, n_repeats=n_repeats, random_state=rng)
     assert result2.importances.shape == (X.shape[1], n_repeats)
 
     assert not np.allclose(result.importances, result2.importances)
@@ -199,20 +221,17 @@ def test_permutation_importance_mixed_types_pandas():
     n_repeats = 5
 
     # Last column is correlated with y
-    X = pd.DataFrame({'col1': [1.0, 2.0, 3.0, np.nan],
-                      'col2': ['a', 'b', 'a', 'b']})
+    X = pd.DataFrame({"col1": [1.0, 2.0, 3.0, np.nan], "col2": ["a", "b", "a", "b"]})
     y = np.array([0, 1, 0, 1])
 
     num_preprocess = make_pipeline(SimpleImputer(), StandardScaler())
-    preprocess = ColumnTransformer([
-        ('num', num_preprocess, ['col1']),
-        ('cat', OneHotEncoder(), ['col2'])
-    ])
-    clf = make_pipeline(preprocess, LogisticRegression(solver='lbfgs'))
+    preprocess = ColumnTransformer(
+        [("num", num_preprocess, ["col1"]), ("cat", OneHotEncoder(), ["col2"])]
+    )
+    clf = make_pipeline(preprocess, LogisticRegression(solver="lbfgs"))
     clf.fit(X, y)
 
-    result = permutation_importance(clf, X, y, n_repeats=n_repeats,
-                                    random_state=rng)
+    result = permutation_importance(clf, X, y, n_repeats=n_repeats, random_state=rng)
 
     assert result.importances.shape == (X.shape[1], n_repeats)
     # the correlated feature with y is the last column and should
@@ -230,27 +249,30 @@ def test_permutation_importance_linear_regresssion():
 
     # this relationship can be computed in closed form
     expected_importances = 2 * lr.coef_**2
-    results = permutation_importance(lr, X, y,
-                                     n_repeats=50,
-                                     scoring='neg_mean_squared_error')
-    assert_allclose(expected_importances, results.importances_mean,
-                    rtol=1e-1, atol=1e-6)
+    results = permutation_importance(
+        lr, X, y, n_repeats=50, scoring="neg_mean_squared_error"
+    )
+    assert_allclose(
+        expected_importances, results.importances_mean, rtol=1e-1, atol=1e-6
+    )
 
 
-def test_permutation_importance_equivalence_sequential_parallel():
+@pytest.mark.parametrize("max_samples", [500, 1.0])
+def test_permutation_importance_equivalence_sequential_parallel(max_samples):
     # regression test to make sure that sequential and parallel calls will
     # output the same results.
+    # Also tests that max_samples equal to number of samples is equivalent to 1.0
     X, y = make_regression(n_samples=500, n_features=10, random_state=0)
     lr = LinearRegression().fit(X, y)
 
     importance_sequential = permutation_importance(
-        lr, X, y, n_repeats=5, random_state=0, n_jobs=1
+        lr, X, y, n_repeats=5, random_state=0, n_jobs=1, max_samples=max_samples
     )
 
     # First check that the problem is structured enough and that the model is
     # complex enough to not yield trivial, constant importances:
-    imp_min = importance_sequential['importances'].min()
-    imp_max = importance_sequential['importances'].max()
+    imp_min = importance_sequential["importances"].min()
+    imp_max = importance_sequential["importances"].max()
     assert imp_max - imp_min > 0.3
 
     # The actually check that parallelism does not impact the results
@@ -260,10 +282,10 @@ def test_permutation_importance_equivalence_sequential_parallel():
 
     # process-based parallelism (by default):
     importance_processes = permutation_importance(
-        lr, X, y, n_repeats=5, random_state=0, n_jobs=2)
+        lr, X, y, n_repeats=5, random_state=0, n_jobs=2
+    )
     assert_allclose(
-        importance_processes['importances'],
-        importance_sequential['importances']
+        importance_processes["importances"], importance_sequential["importances"]
     )
 
     # thread-based parallelism:
@@ -272,16 +294,16 @@ def test_permutation_importance_equivalence_sequential_parallel():
             lr, X, y, n_repeats=5, random_state=0, n_jobs=2
         )
     assert_allclose(
-        importance_threading['importances'],
-        importance_sequential['importances']
+        importance_threading["importances"], importance_sequential["importances"]
     )
 
 
 @pytest.mark.parametrize("n_jobs", [None, 1, 2])
-def test_permutation_importance_equivalence_array_dataframe(n_jobs):
+@pytest.mark.parametrize("max_samples", [0.5, 1.0])
+def test_permutation_importance_equivalence_array_dataframe(n_jobs, max_samples):
     # This test checks that the column shuffling logic has the same behavior
     # both a dataframe and a simple numpy array.
-    pd = pytest.importorskip('pandas')
+    pd = pytest.importorskip("pandas")
 
     # regression test to make sure that sequential and parallel calls will
     # output the same results.
@@ -289,7 +311,11 @@ def test_permutation_importance_equivalence_array_dataframe(n_jobs):
     X_df = pd.DataFrame(X)
 
     # Add a categorical feature that is statistically linked to y:
-    binner = KBinsDiscretizer(n_bins=3, encode="ordinal")
+    binner = KBinsDiscretizer(
+        n_bins=3,
+        encode="ordinal",
+        quantile_method="averaged_inverted_cdf",
+    )
     cat_column = binner.fit_transform(y.reshape(-1, 1))
 
     # Concatenate the extra column to the numpy array: integers will be
@@ -297,17 +323,13 @@ def test_permutation_importance_equivalence_array_dataframe(n_jobs):
     X = np.hstack([X, cat_column])
     assert X.dtype.kind == "f"
 
-    # Insert extra column as a non-numpy-native dtype (while keeping backward
-    # compat for old pandas versions):
-    if hasattr(pd, "Categorical"):
-        cat_column = pd.Categorical(cat_column.ravel())
-    else:
-        cat_column = cat_column.ravel()
+    # Insert extra column as a non-numpy-native dtype:
+    cat_column = pd.Categorical(cat_column.ravel())
     new_col_idx = len(X_df.columns)
     X_df[new_col_idx] = cat_column
     assert X_df[new_col_idx].dtype == cat_column.dtype
 
-    # Stich an aribtrary index to the dataframe:
+    # Stich an arbitrary index to the dataframe:
     X_df.index = np.arange(len(X_df)).astype(str)
 
     rf = RandomForestRegressor(n_estimators=5, max_depth=3, random_state=0)
@@ -315,23 +337,34 @@ def test_permutation_importance_equivalence_array_dataframe(n_jobs):
 
     n_repeats = 3
     importance_array = permutation_importance(
-        rf, X, y, n_repeats=n_repeats, random_state=0, n_jobs=n_jobs
+        rf,
+        X,
+        y,
+        n_repeats=n_repeats,
+        random_state=0,
+        n_jobs=n_jobs,
+        max_samples=max_samples,
     )
 
     # First check that the problem is structured enough and that the model is
     # complex enough to not yield trivial, constant importances:
-    imp_min = importance_array['importances'].min()
-    imp_max = importance_array['importances'].max()
+    imp_min = importance_array["importances"].min()
+    imp_max = importance_array["importances"].max()
     assert imp_max - imp_min > 0.3
 
     # Now check that importances computed on dataframe matche the values
     # of those computed on the array with the same data.
     importance_dataframe = permutation_importance(
-        rf, X_df, y, n_repeats=n_repeats, random_state=0, n_jobs=n_jobs
+        rf,
+        X_df,
+        y,
+        n_repeats=n_repeats,
+        random_state=0,
+        n_jobs=n_jobs,
+        max_samples=max_samples,
     )
     assert_allclose(
-        importance_array['importances'],
-        importance_dataframe['importances']
+        importance_array["importances"], importance_dataframe["importances"]
     )
 
 
@@ -340,12 +373,13 @@ def test_permutation_importance_large_memmaped_data(input_type):
     # Smoke, non-regression test for:
     # https://github.com/scikit-learn/scikit-learn/issues/15810
     n_samples, n_features = int(5e4), 4
-    X, y = make_classification(n_samples=n_samples, n_features=n_features,
-                               random_state=0)
+    X, y = make_classification(
+        n_samples=n_samples, n_features=n_features, random_state=0
+    )
     assert X.nbytes > 1e6  # trigger joblib memmaping
 
     X = _convert_container(X, input_type)
-    clf = DummyClassifier(strategy='prior').fit(X, y)
+    clf = DummyClassifier(strategy="prior").fit(X, y)
 
     # Actual smoke test: should not raise any error:
     n_repeats = 5
@@ -378,33 +412,42 @@ def test_permutation_importance_sample_weight():
     # When all samples are weighted with the same weights, the ratio of
     # the two features importance should equal to 1 on expectation (when using
     # mean absolutes error as the loss function).
-    pi = permutation_importance(lr, x, y, random_state=1,
-                                scoring='neg_mean_absolute_error',
-                                n_repeats=200)
+    pi = permutation_importance(
+        lr, x, y, random_state=1, scoring="neg_mean_absolute_error", n_repeats=200
+    )
     x1_x2_imp_ratio_w_none = pi.importances_mean[0] / pi.importances_mean[1]
     assert x1_x2_imp_ratio_w_none == pytest.approx(1, 0.01)
 
     # When passing a vector of ones as the sample_weight, results should be
     # the same as in the case that sample_weight=None.
     w = np.ones(n_samples)
-    pi = permutation_importance(lr, x, y, random_state=1,
-                                scoring='neg_mean_absolute_error',
-                                n_repeats=200, sample_weight=w)
+    pi = permutation_importance(
+        lr,
+        x,
+        y,
+        random_state=1,
+        scoring="neg_mean_absolute_error",
+        n_repeats=200,
+        sample_weight=w,
+    )
     x1_x2_imp_ratio_w_ones = pi.importances_mean[0] / pi.importances_mean[1]
-    assert x1_x2_imp_ratio_w_ones == pytest.approx(
-        x1_x2_imp_ratio_w_none, 0.01)
+    assert x1_x2_imp_ratio_w_ones == pytest.approx(x1_x2_imp_ratio_w_none, 0.01)
 
     # When the ratio between the weights of the first half of the samples and
     # the second half of the samples approaches to infinity, the ratio of
     # the two features importance should equal to 2 on expectation (when using
     # mean absolutes error as the loss function).
-    w = np.hstack([np.repeat(10.0 ** 10, n_half_samples),
-                   np.repeat(1.0, n_half_samples)])
+    w = np.hstack([np.repeat(10.0**10, n_half_samples), np.repeat(1.0, n_half_samples)])
     lr.fit(x, y, w)
-    pi = permutation_importance(lr, x, y, random_state=1,
-                                scoring='neg_mean_absolute_error',
-                                n_repeats=200,
-                                sample_weight=w)
+    pi = permutation_importance(
+        lr,
+        x,
+        y,
+        random_state=1,
+        scoring="neg_mean_absolute_error",
+        n_repeats=200,
+        sample_weight=w,
+    )
     x1_x2_imp_ratio_w = pi.importances_mean[0] / pi.importances_mean[1]
     assert x1_x2_imp_ratio_w / x1_x2_imp_ratio_w_none == pytest.approx(2, 0.01)
 
@@ -424,21 +467,20 @@ def my_scorer(estimator, X, y):
     # test that permutation_importance does not return error when
     # sample_weight is None
     try:
-        permutation_importance(lr, x, y, random_state=1,
-                               scoring=my_scorer,
-                               n_repeats=1)
+        permutation_importance(lr, x, y, random_state=1, scoring=my_scorer, n_repeats=1)
     except TypeError:
-        pytest.fail("permutation_test raised an error when using a scorer "
-                    "function that does not accept sample_weight even though "
-                    "sample_weight was None")
+        pytest.fail(
+            "permutation_test raised an error when using a scorer "
+            "function that does not accept sample_weight even though "
+            "sample_weight was None"
+        )
 
     # test that permutation_importance raise exception when sample_weight is
     # not None
     with pytest.raises(TypeError):
-        permutation_importance(lr, x, y, random_state=1,
-                               scoring=my_scorer,
-                               n_repeats=1,
-                               sample_weight=w)
+        permutation_importance(
+            lr, x, y, random_state=1, scoring=my_scorer, n_repeats=1, sample_weight=w
+        )
 
 
 @pytest.mark.parametrize(
@@ -456,9 +498,7 @@ def my_scorer(estimator, X, y):
             ["r2", "neg_mean_squared_error"],
             lambda estimator, X, y: {
                 "r2": r2_score(y, estimator.predict(X)),
-                "neg_mean_squared_error": -mean_squared_error(
-                    y, estimator.predict(X)
-                ),
+                "neg_mean_squared_error": -mean_squared_error(y, estimator.predict(X)),
             },
         ),
     ],
@@ -482,3 +522,19 @@ def test_permutation_importance_multi_metric(list_single_scorer, multi_scorer):
         )
 
         assert_allclose(multi_result.importances, single_result.importances)
+
+
+def test_permutation_importance_max_samples_error():
+    """Check that a proper error message is raised when `max_samples` is not
+    set to a valid input value.
+    """
+    X = np.array([(1.0, 2.0, 3.0, 4.0)]).T
+    y = np.array([0, 1, 0, 1])
+
+    clf = LogisticRegression()
+    clf.fit(X, y)
+
+    err_msg = r"max_samples must be <= n_samples"
+
+    with pytest.raises(ValueError, match=err_msg):
+        permutation_importance(clf, X, y, max_samples=5)
diff --git a/sklearn/isotonic.py b/sklearn/isotonic.py
index b57ce23f8cc52..2f2c56ae5d13c 100644
--- a/sklearn/isotonic.py
+++ b/sklearn/isotonic.py
@@ -1,24 +1,35 @@
-# Authors: Fabian Pedregosa 
-#          Alexandre Gramfort 
-#          Nelle Varoquaux 
-# License: BSD 3 clause
+"""Isotonic regression for obtaining monotonic fit to data."""
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+import math
+import warnings
+from numbers import Real
 
 import numpy as np
-from scipy import interpolate
+from scipy import interpolate, optimize
 from scipy.stats import spearmanr
-import warnings
-import math
 
-from .base import BaseEstimator, TransformerMixin, RegressorMixin
-from .utils import check_array, check_consistent_length
-from .utils.validation import _check_sample_weight, _deprecate_positional_args
-from ._isotonic import _inplace_contiguous_isotonic_regression, _make_unique
+from sklearn.utils import metadata_routing
 
+from ._isotonic import _inplace_contiguous_isotonic_regression, _make_unique
+from .base import BaseEstimator, RegressorMixin, TransformerMixin, _fit_context
+from .utils import check_array, check_consistent_length
+from .utils._param_validation import Interval, StrOptions, validate_params
+from .utils.fixes import parse_version, sp_base_version
+from .utils.validation import _check_sample_weight, check_is_fitted
 
-__all__ = ['check_increasing', 'isotonic_regression',
-           'IsotonicRegression']
+__all__ = ["IsotonicRegression", "check_increasing", "isotonic_regression"]
 
 
+@validate_params(
+    {
+        "x": ["array-like"],
+        "y": ["array-like"],
+    },
+    prefer_skip_nested_validation=True,
+)
 def check_increasing(x, y):
     """Determine whether y is monotonically correlated with x.
 
@@ -50,6 +61,16 @@ def check_increasing(x, y):
     ----------
     Fisher transformation. Wikipedia.
     https://en.wikipedia.org/wiki/Fisher_transformation
+
+    Examples
+    --------
+    >>> from sklearn.isotonic import check_increasing
+    >>> x, y = [1, 2, 3, 4, 5], [2, 4, 6, 8, 10]
+    >>> check_increasing(x, y)
+    np.True_
+    >>> y = [10, 8, 6, 4, 2]
+    >>> check_increasing(x, y)
+    np.False_
     """
 
     # Calculate Spearman rho estimate and set return accordingly.
@@ -58,7 +79,7 @@ def check_increasing(x, y):
 
     # Run Fisher transform to get the rho CI, but handle rho=+/-1
     if rho not in [-1.0, 1.0] and len(x) > 3:
-        F = 0.5 * math.log((1. + rho) / (1. - rho))
+        F = 0.5 * math.log((1.0 + rho) / (1.0 - rho))
         F_se = 1 / math.sqrt(len(x) - 3)
 
         # Use a 95% CI, i.e., +/-1.96 S.E.
@@ -68,17 +89,29 @@ def check_increasing(x, y):
 
         # Warn if the CI spans zero.
         if np.sign(rho_0) != np.sign(rho_1):
-            warnings.warn("Confidence interval of the Spearman "
-                          "correlation coefficient spans zero. "
-                          "Determination of ``increasing`` may be "
-                          "suspect.")
+            warnings.warn(
+                "Confidence interval of the Spearman "
+                "correlation coefficient spans zero. "
+                "Determination of ``increasing`` may be "
+                "suspect."
+            )
 
     return increasing_bool
 
 
-@_deprecate_positional_args
-def isotonic_regression(y, *, sample_weight=None, y_min=None, y_max=None,
-                        increasing=True):
+@validate_params(
+    {
+        "y": ["array-like"],
+        "sample_weight": ["array-like", None],
+        "y_min": [Interval(Real, None, None, closed="both"), None],
+        "y_max": [Interval(Real, None, None, closed="both"), None],
+        "increasing": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def isotonic_regression(
+    y, *, sample_weight=None, y_min=None, y_max=None, increasing=True
+):
     """Solve the isotonic regression model.
 
     Read more in the :ref:`User Guide `.
@@ -102,25 +135,41 @@ def isotonic_regression(y, *, sample_weight=None, y_min=None, y_max=None,
 
     increasing : bool, default=True
         Whether to compute ``y_`` is increasing (if set to True) or decreasing
-        (if set to False)
+        (if set to False).
 
     Returns
     -------
-    y_ : list of floats
+    y_ : ndarray of shape (n_samples,)
         Isotonic fit of y.
 
     References
     ----------
     "Active set algorithms for isotonic regression; A unifying framework"
     by Michael J. Best and Nilotpal Chakravarti, section 3.
+
+    Examples
+    --------
+    >>> from sklearn.isotonic import isotonic_regression
+    >>> isotonic_regression([5, 3, 1, 2, 8, 10, 7, 9, 6, 4])
+    array([2.75   , 2.75   , 2.75   , 2.75   , 7.33,
+           7.33, 7.33, 7.33, 7.33, 7.33])
     """
-    order = np.s_[:] if increasing else np.s_[::-1]
-    y = check_array(y, ensure_2d=False, dtype=[np.float64, np.float32])
-    y = np.array(y[order], dtype=y.dtype)
-    sample_weight = _check_sample_weight(sample_weight, y, dtype=y.dtype)
-    sample_weight = np.ascontiguousarray(sample_weight[order])
+    y = check_array(y, ensure_2d=False, input_name="y", dtype=[np.float64, np.float32])
+    if sp_base_version >= parse_version("1.12.0"):
+        res = optimize.isotonic_regression(
+            y=y, weights=sample_weight, increasing=increasing
+        )
+        y = np.asarray(res.x, dtype=y.dtype)
+    else:
+        # TODO: remove this branch when Scipy 1.12 is the minimum supported version
+        # Also remove _inplace_contiguous_isotonic_regression.
+        order = np.s_[:] if increasing else np.s_[::-1]
+        y = np.array(y[order], dtype=y.dtype)
+        sample_weight = _check_sample_weight(sample_weight, y, dtype=y.dtype, copy=True)
+        sample_weight = np.ascontiguousarray(sample_weight[order])
+        _inplace_contiguous_isotonic_regression(y, sample_weight)
+        y = y[order]
 
-    _inplace_contiguous_isotonic_regression(y, sample_weight)
     if y_min is not None or y_max is not None:
         # Older versions of np.clip don't accept None as a bound, so use np.inf
         if y_min is None:
@@ -128,7 +177,7 @@ def isotonic_regression(y, *, sample_weight=None, y_min=None, y_max=None,
         if y_max is None:
             y_max = np.inf
         np.clip(y, y_min, y_max, y)
-    return y[order]
+    return y
 
 
 class IsotonicRegression(RegressorMixin, TransformerMixin, BaseEstimator):
@@ -188,6 +237,14 @@ class IsotonicRegression(RegressorMixin, TransformerMixin, BaseEstimator):
     increasing_ : bool
         Inferred value for ``increasing``.
 
+    See Also
+    --------
+    sklearn.linear_model.LinearRegression : Ordinary least squares Linear
+        Regression.
+    sklearn.ensemble.HistGradientBoostingRegressor : Gradient boosting that
+        is a non-parametric model accepting monotonicity constraints.
+    isotonic_regression : Function to solve the isotonic regression model.
+
     Notes
     -----
     Ties are broken using the secondary method from de Leeuw, 1977.
@@ -214,11 +271,21 @@ class IsotonicRegression(RegressorMixin, TransformerMixin, BaseEstimator):
     >>> X, y = make_regression(n_samples=10, n_features=1, random_state=41)
     >>> iso_reg = IsotonicRegression().fit(X, y)
     >>> iso_reg.predict([.1, .2])
-    array([1.8628..., 3.7256...])
+    array([1.8628, 3.7256])
     """
-    @_deprecate_positional_args
-    def __init__(self, *, y_min=None, y_max=None, increasing=True,
-                 out_of_bounds='nan'):
+
+    # T should have been called X
+    __metadata_request__predict = {"T": metadata_routing.UNUSED}
+    __metadata_request__transform = {"T": metadata_routing.UNUSED}
+
+    _parameter_constraints: dict = {
+        "y_min": [Interval(Real, None, None, closed="both"), None],
+        "y_max": [Interval(Real, None, None, closed="both"), None],
+        "increasing": ["boolean", StrOptions({"auto"})],
+        "out_of_bounds": [StrOptions({"nan", "clip", "raise"})],
+    }
+
+    def __init__(self, *, y_min=None, y_max=None, increasing=True, out_of_bounds="nan"):
         self.y_min = y_min
         self.y_max = y_max
         self.increasing = increasing
@@ -226,26 +293,23 @@ def __init__(self, *, y_min=None, y_max=None, increasing=True,
 
     def _check_input_data_shape(self, X):
         if not (X.ndim == 1 or (X.ndim == 2 and X.shape[1] == 1)):
-            msg = "Isotonic regression input X should be a 1d array or " \
-                  "2d array with 1 feature"
+            msg = (
+                "Isotonic regression input X should be a 1d array or "
+                "2d array with 1 feature"
+            )
             raise ValueError(msg)
 
     def _build_f(self, X, y):
         """Build the f_ interp1d function."""
 
-        # Handle the out_of_bounds argument by setting bounds_error
-        if self.out_of_bounds not in ["raise", "nan", "clip"]:
-            raise ValueError("The argument ``out_of_bounds`` must be in "
-                             "'nan', 'clip', 'raise'; got {0}"
-                             .format(self.out_of_bounds))
-
         bounds_error = self.out_of_bounds == "raise"
         if len(y) == 1:
             # single y, constant prediction
             self.f_ = lambda x: y.repeat(x.shape)
         else:
-            self.f_ = interpolate.interp1d(X, y, kind='linear',
-                                           bounds_error=bounds_error)
+            self.f_ = interpolate.interp1d(
+                X, y, kind="linear", bounds_error=bounds_error
+            )
 
     def _build_y(self, X, y, sample_weight, trim_duplicates=True):
         """Build the y_ IsotonicRegression."""
@@ -253,7 +317,7 @@ def _build_y(self, X, y, sample_weight, trim_duplicates=True):
         X = X.reshape(-1)  # use 1d view
 
         # Determine increasing if auto-determination requested
-        if self.increasing == 'auto':
+        if self.increasing == "auto":
             self.increasing_ = check_increasing(X, y)
         else:
             self.increasing_ = self.increasing
@@ -266,13 +330,16 @@ def _build_y(self, X, y, sample_weight, trim_duplicates=True):
 
         order = np.lexsort((y, X))
         X, y, sample_weight = [array[order] for array in [X, y, sample_weight]]
-        unique_X, unique_y, unique_sample_weight = _make_unique(
-            X, y, sample_weight)
+        unique_X, unique_y, unique_sample_weight = _make_unique(X, y, sample_weight)
 
         X = unique_X
-        y = isotonic_regression(unique_y, sample_weight=unique_sample_weight,
-                                y_min=self.y_min, y_max=self.y_max,
-                                increasing=self.increasing_)
+        y = isotonic_regression(
+            unique_y,
+            sample_weight=unique_sample_weight,
+            y_min=self.y_min,
+            y_max=self.y_max,
+            increasing=self.increasing_,
+        )
 
         # Handle the left and right bounds on X
         self.X_min_, self.X_max_ = np.min(X), np.max(X)
@@ -283,8 +350,7 @@ def _build_y(self, X, y, sample_weight, trim_duplicates=True):
             # Aside from the 1st and last point, remove points whose y values
             # are equal to both the point before and the point after it.
             keep_data[1:-1] = np.logical_or(
-                np.not_equal(y[1:-1], y[:-2]),
-                np.not_equal(y[1:-1], y[2:])
+                np.not_equal(y[1:-1], y[:-2]), np.not_equal(y[1:-1], y[2:])
             )
             return X[keep_data], y[keep_data]
         else:
@@ -294,6 +360,7 @@ def _build_y(self, X, y, sample_weight, trim_duplicates=True):
             # prediction speed).
             return X, y
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y, sample_weight=None):
         """Fit the model using X, y as training data.
 
@@ -323,8 +390,10 @@ def fit(self, X, y, sample_weight=None):
         new input data.
         """
         check_params = dict(accept_sparse=False, ensure_2d=False)
-        X = check_array(X, dtype=[np.float64, np.float32], **check_params)
-        y = check_array(y, dtype=X.dtype, **check_params)
+        X = check_array(
+            X, input_name="X", dtype=[np.float64, np.float32], **check_params
+        )
+        y = check_array(y, input_name="y", dtype=X.dtype, **check_params)
         check_consistent_length(X, y, sample_weight)
 
         # Transform y by running the isotonic regression algorithm and
@@ -341,24 +410,17 @@ def fit(self, X, y, sample_weight=None):
         self._build_f(X, y)
         return self
 
-    def transform(self, T):
-        """Transform new data by linear interpolation
+    def _transform(self, T):
+        """`_transform` is called by both `transform` and `predict` methods.
 
-        Parameters
-        ----------
-        T : array-like of shape (n_samples,) or (n_samples, 1)
-            Data to transform.
+        Since `transform` is wrapped to output arrays of specific types (e.g.
+        NumPy arrays, pandas DataFrame), we cannot make `predict` call `transform`
+        directly.
 
-            .. versionchanged:: 0.24
-               Also accepts 2d array with 1 feature.
-
-        Returns
-        -------
-        y_pred : ndarray of shape (n_samples,)
-            The transformed data
+        The above behaviour could be changed in the future, if we decide to output
+        other type of arrays when calling `predict`.
         """
-
-        if hasattr(self, 'X_thresholds_'):
+        if hasattr(self, "X_thresholds_"):
             dtype = self.X_thresholds_.dtype
         else:
             dtype = np.float64
@@ -368,12 +430,6 @@ def transform(self, T):
         self._check_input_data_shape(T)
         T = T.reshape(-1)  # use 1d view
 
-        # Handle the out_of_bounds argument by clipping if needed
-        if self.out_of_bounds not in ["raise", "nan", "clip"]:
-            raise ValueError("The argument ``out_of_bounds`` must be in "
-                             "'nan', 'clip', 'raise'; got {0}"
-                             .format(self.out_of_bounds))
-
         if self.out_of_bounds == "clip":
             T = np.clip(T, self.X_min_, self.X_max_)
 
@@ -384,6 +440,24 @@ def transform(self, T):
 
         return res
 
+    def transform(self, T):
+        """Transform new data by linear interpolation.
+
+        Parameters
+        ----------
+        T : array-like of shape (n_samples,) or (n_samples, 1)
+            Data to transform.
+
+            .. versionchanged:: 0.24
+               Also accepts 2d array with 1 feature.
+
+        Returns
+        -------
+        y_pred : ndarray of shape (n_samples,)
+            The transformed data.
+        """
+        return self._transform(T)
+
     def predict(self, T):
         """Predict new data by linear interpolation.
 
@@ -397,13 +471,34 @@ def predict(self, T):
         y_pred : ndarray of shape (n_samples,)
             Transformed data.
         """
-        return self.transform(T)
+        return self._transform(T)
+
+    # We implement get_feature_names_out here instead of using
+    # `ClassNamePrefixFeaturesOutMixin`` because `input_features` are ignored.
+    # `input_features` are ignored because `IsotonicRegression` accepts 1d
+    # arrays and the semantics of `feature_names_in_` are not clear for 1d arrays.
+    def get_feature_names_out(self, input_features=None):
+        """Get output feature names for transformation.
+
+        Parameters
+        ----------
+        input_features : array-like of str or None, default=None
+            Ignored.
+
+        Returns
+        -------
+        feature_names_out : ndarray of str objects
+            An ndarray with one string i.e. ["isotonicregression0"].
+        """
+        check_is_fitted(self, "f_")
+        class_name = self.__class__.__name__.lower()
+        return np.asarray([f"{class_name}0"], dtype=object)
 
     def __getstate__(self):
-        """Pickle-protocol - return state of the estimator. """
+        """Pickle-protocol - return state of the estimator."""
         state = super().__getstate__()
         # remove interpolation method
-        state.pop('f_', None)
+        state.pop("f_", None)
         return state
 
     def __setstate__(self, state):
@@ -412,8 +507,11 @@ def __setstate__(self, state):
         We need to rebuild the interpolation function.
         """
         super().__setstate__(state)
-        if hasattr(self, 'X_thresholds_') and hasattr(self, 'y_thresholds_'):
+        if hasattr(self, "X_thresholds_") and hasattr(self, "y_thresholds_"):
             self._build_f(self.X_thresholds_, self.y_thresholds_)
 
-    def _more_tags(self):
-        return {'X_types': ['1darray']}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.one_d_array = True
+        tags.input_tags.two_d_array = False
+        return tags
diff --git a/sklearn/kernel_approximation.py b/sklearn/kernel_approximation.py
index ca02aac3e982c..02c8af755baea 100644
--- a/sklearn/kernel_approximation.py
+++ b/sklearn/kernel_approximation.py
@@ -1,33 +1,36 @@
-"""
-The :mod:`sklearn.kernel_approximation` module implements several
-approximate kernel feature maps based on Fourier transforms and Count Sketches.
-"""
+"""Approximate kernel feature maps based on Fourier transforms and count sketches."""
 
-# Author: Andreas Mueller 
-#         Daniel Lopez-Sanchez (TensorSketch) 
-
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import warnings
+from numbers import Integral, Real
 
 import numpy as np
 import scipy.sparse as sp
+from scipy.fft import fft, ifft
 from scipy.linalg import svd
-try:
-    from scipy.fft import fft, ifft
-except ImportError:   # scipy < 1.4
-    from scipy.fftpack import fft, ifft
-
-from .base import BaseEstimator
-from .base import TransformerMixin
-from .utils import check_random_state, as_float_array
+
+from .base import (
+    BaseEstimator,
+    ClassNamePrefixFeaturesOutMixin,
+    TransformerMixin,
+    _fit_context,
+)
+from .metrics.pairwise import KERNEL_PARAMS, PAIRWISE_KERNEL_FUNCTIONS, pairwise_kernels
+from .utils import check_random_state
+from .utils._param_validation import Interval, StrOptions
 from .utils.extmath import safe_sparse_dot
-from .utils.validation import check_is_fitted
-from .metrics.pairwise import pairwise_kernels, KERNEL_PARAMS
-from .utils.validation import check_non_negative, _deprecate_positional_args
+from .utils.validation import (
+    _check_feature_names_in,
+    check_is_fitted,
+    validate_data,
+)
 
 
-class PolynomialCountSketch(BaseEstimator, TransformerMixin):
+class PolynomialCountSketch(
+    ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator
+):
     """Polynomial kernel approximation via Tensor Sketch.
 
     Implements Tensor Sketch, which approximates the feature map
@@ -56,10 +59,10 @@ class PolynomialCountSketch(BaseEstimator, TransformerMixin):
         will be approximated.
 
     n_components : int, default=100
-        Dimensionality of the output feature space. Usually, n_components
+        Dimensionality of the output feature space. Usually, `n_components`
         should be greater than the number of features in input samples in
         order to achieve good performance. The optimal score / run time
-        balance is typically achieved around n_components = 10 * n_features,
+        balance is typically achieved around `n_components` = 10 * `n_features`,
         but this depends on the specific dataset being used.
 
     random_state : int, RandomState instance, default=None
@@ -77,6 +80,26 @@ class PolynomialCountSketch(BaseEstimator, TransformerMixin):
         Array with random entries in {+1, -1}, used to represent
         the 2-wise independent hash functions for Count Sketch computation.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    AdditiveChi2Sampler : Approximate feature map for additive chi2 kernel.
+    Nystroem : Approximate a kernel map using a subset of the training data.
+    RBFSampler : Approximate a RBF kernel feature map using random Fourier
+        features.
+    SkewedChi2Sampler : Approximate feature map for "skewed chi-squared" kernel.
+    sklearn.metrics.pairwise.kernel_metrics : List of built-in kernels.
+
     Examples
     --------
     >>> from sklearn.kernel_approximation import PolynomialCountSketch
@@ -90,16 +113,29 @@ class PolynomialCountSketch(BaseEstimator, TransformerMixin):
     SGDClassifier(max_iter=10)
     >>> clf.score(X_features, y)
     1.0
+
+    For a more detailed example of usage, see
+    :ref:`sphx_glr_auto_examples_kernel_approximation_plot_scalable_poly_kernels.py`
     """
 
-    def __init__(self, *, gamma=1., degree=2, coef0=0, n_components=100,
-                 random_state=None):
+    _parameter_constraints: dict = {
+        "gamma": [Interval(Real, 0, None, closed="left")],
+        "degree": [Interval(Integral, 1, None, closed="left")],
+        "coef0": [Interval(Real, None, None, closed="neither")],
+        "n_components": [Interval(Integral, 1, None, closed="left")],
+        "random_state": ["random_state"],
+    }
+
+    def __init__(
+        self, *, gamma=1.0, degree=2, coef0=0, n_components=100, random_state=None
+    ):
         self.gamma = gamma
         self.degree = degree
         self.coef0 = coef0
         self.n_components = n_components
         self.random_state = random_state
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y=None):
         """Fit the model with X.
 
@@ -109,29 +145,31 @@ def fit(self, X, y=None):
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
-            Training data, where n_samples in the number of samples
-            and n_features is the number of features.
+            Training data, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
+
+        y : array-like of shape (n_samples,) or (n_samples, n_outputs), \
+                default=None
+            Target values (None for unsupervised transformations).
 
         Returns
         -------
         self : object
-            Returns the transformer.
+            Returns the instance itself.
         """
-        if not self.degree >= 1:
-            raise ValueError(f"degree={self.degree} should be >=1.")
-
-        X = self._validate_data(X, accept_sparse="csc")
+        X = validate_data(self, X, accept_sparse="csc")
         random_state = check_random_state(self.random_state)
 
         n_features = X.shape[1]
         if self.coef0 != 0:
             n_features += 1
 
-        self.indexHash_ = random_state.randint(0, high=self.n_components,
-                                               size=(self.degree, n_features))
+        self.indexHash_ = random_state.randint(
+            0, high=self.n_components, size=(self.degree, n_features)
+        )
 
-        self.bitHash_ = random_state.choice(a=[-1, 1],
-                                            size=(self.degree, n_features))
+        self.bitHash_ = random_state.choice(a=[-1, 1], size=(self.degree, n_features))
+        self._n_features_out = self.n_components
         return self
 
     def transform(self, X):
@@ -140,50 +178,54 @@ def transform(self, X):
         Parameters
         ----------
         X : {array-like}, shape (n_samples, n_features)
-            New data, where n_samples in the number of samples
-            and n_features is the number of features.
+            New data, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
 
         Returns
         -------
         X_new : array-like, shape (n_samples, n_components)
+            Returns the instance itself.
         """
 
         check_is_fitted(self)
-        X = self._validate_data(X, accept_sparse="csc", reset=False)
+        X = validate_data(self, X, accept_sparse="csc", reset=False)
 
         X_gamma = np.sqrt(self.gamma) * X
 
         if sp.issparse(X_gamma) and self.coef0 != 0:
-            X_gamma = sp.hstack([X_gamma, np.sqrt(self.coef0) *
-                                 np.ones((X_gamma.shape[0], 1))],
-                                format="csc")
+            X_gamma = sp.hstack(
+                [X_gamma, np.sqrt(self.coef0) * np.ones((X_gamma.shape[0], 1))],
+                format="csc",
+            )
 
         elif not sp.issparse(X_gamma) and self.coef0 != 0:
-            X_gamma = np.hstack([X_gamma, np.sqrt(self.coef0) *
-                                 np.ones((X_gamma.shape[0], 1))])
+            X_gamma = np.hstack(
+                [X_gamma, np.sqrt(self.coef0) * np.ones((X_gamma.shape[0], 1))]
+            )
 
         if X_gamma.shape[1] != self.indexHash_.shape[1]:
-            raise ValueError("Number of features of test samples does not"
-                             " match that of training samples.")
+            raise ValueError(
+                "Number of features of test samples does not"
+                " match that of training samples."
+            )
 
-        count_sketches = np.zeros(
-            (X_gamma.shape[0], self.degree, self.n_components))
+        count_sketches = np.zeros((X_gamma.shape[0], self.degree, self.n_components))
 
         if sp.issparse(X_gamma):
             for j in range(X_gamma.shape[1]):
                 for d in range(self.degree):
                     iHashIndex = self.indexHash_[d, j]
                     iHashBit = self.bitHash_[d, j]
-                    count_sketches[:, d, iHashIndex] += \
-                        (iHashBit * X_gamma[:, j]).toarray().ravel()
+                    count_sketches[:, d, iHashIndex] += (
+                        (iHashBit * X_gamma[:, [j]]).toarray().ravel()
+                    )
 
         else:
             for j in range(X_gamma.shape[1]):
                 for d in range(self.degree):
                     iHashIndex = self.indexHash_[d, j]
                     iHashBit = self.bitHash_[d, j]
-                    count_sketches[:, d, iHashIndex] += \
-                        iHashBit * X_gamma[:, j]
+                    count_sketches[:, d, iHashIndex] += iHashBit * X_gamma[:, j]
 
         # For each same, compute a count sketch of phi(x) using the polynomial
         # multiplication (via FFT) of p count sketches of x.
@@ -193,10 +235,14 @@ def transform(self, X):
 
         return data_sketch
 
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
+        return tags
 
-class RBFSampler(TransformerMixin, BaseEstimator):
-    """Approximates feature map of an RBF kernel by Monte Carlo approximation
-    of its Fourier transform.
+
+class RBFSampler(ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator):
+    """Approximate a RBF kernel feature map using random Fourier features.
 
     It implements a variant of Random Kitchen Sinks.[1]
 
@@ -204,8 +250,13 @@ class RBFSampler(TransformerMixin, BaseEstimator):
 
     Parameters
     ----------
-    gamma : float, default=1.0
-        Parameter of RBF kernel: exp(-gamma * x^2)
+    gamma : 'scale' or float, default=1.0
+        Parameter of RBF kernel: exp(-gamma * x^2).
+        If ``gamma='scale'`` is passed then it uses
+        1 / (n_features * X.var()) as value of gamma.
+
+        .. versionadded:: 1.2
+           The option `"scale"` was added in 1.2.
 
     n_components : int, default=100
         Number of Monte Carlo samples per original feature.
@@ -219,15 +270,44 @@ class RBFSampler(TransformerMixin, BaseEstimator):
 
     Attributes
     ----------
-    random_offset_ : ndarray of shape (n_components,), dtype=float64
+    random_offset_ : ndarray of shape (n_components,), dtype={np.float64, np.float32}
         Random offset used to compute the projection in the `n_components`
         dimensions of the feature space.
 
     random_weights_ : ndarray of shape (n_features, n_components),\
-        dtype=float64
+        dtype={np.float64, np.float32}
         Random projection directions drawn from the Fourier transform
         of the RBF kernel.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    AdditiveChi2Sampler : Approximate feature map for additive chi2 kernel.
+    Nystroem : Approximate a kernel map using a subset of the training data.
+    PolynomialCountSketch : Polynomial kernel approximation via Tensor Sketch.
+    SkewedChi2Sampler : Approximate feature map for
+        "skewed chi-squared" kernel.
+    sklearn.metrics.pairwise.kernel_metrics : List of built-in kernels.
+
+    Notes
+    -----
+    See "Random Features for Large-Scale Kernel Machines" by A. Rahimi and
+    Benjamin Recht.
+
+    [1] "Weighted Sums of Random Kitchen Sinks: Replacing
+    minimization with randomization in learning" by A. Rahimi and
+    Benjamin Recht.
+    (https://people.eecs.berkeley.edu/~brecht/papers/08.rah.rec.nips.pdf)
 
     Examples
     --------
@@ -242,23 +322,23 @@ class RBFSampler(TransformerMixin, BaseEstimator):
     SGDClassifier(max_iter=5)
     >>> clf.score(X_features, y)
     1.0
+    """
 
-    Notes
-    -----
-    See "Random Features for Large-Scale Kernel Machines" by A. Rahimi and
-    Benjamin Recht.
+    _parameter_constraints: dict = {
+        "gamma": [
+            StrOptions({"scale"}),
+            Interval(Real, 0.0, None, closed="left"),
+        ],
+        "n_components": [Interval(Integral, 1, None, closed="left")],
+        "random_state": ["random_state"],
+    }
 
-    [1] "Weighted Sums of Random Kitchen Sinks: Replacing
-    minimization with randomization in learning" by A. Rahimi and
-    Benjamin Recht.
-    (https://people.eecs.berkeley.edu/~brecht/papers/08.rah.rec.nips.pdf)
-    """
-    @_deprecate_positional_args
-    def __init__(self, *, gamma=1., n_components=100, random_state=None):
+    def __init__(self, *, gamma=1.0, n_components=100, random_state=None):
         self.gamma = gamma
         self.n_components = n_components
         self.random_state = random_state
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y=None):
         """Fit the model with X.
 
@@ -267,24 +347,41 @@ def fit(self, X, y=None):
         Parameters
         ----------
         X : {array-like, sparse matrix}, shape (n_samples, n_features)
-            Training data, where n_samples in the number of samples
-            and n_features is the number of features.
+            Training data, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
+
+        y : array-like, shape (n_samples,) or (n_samples, n_outputs), \
+                default=None
+            Target values (None for unsupervised transformations).
 
         Returns
         -------
         self : object
-            Returns the transformer.
+            Returns the instance itself.
         """
-
-        X = self._validate_data(X, accept_sparse='csr')
+        X = validate_data(self, X, accept_sparse="csr")
         random_state = check_random_state(self.random_state)
         n_features = X.shape[1]
+        sparse = sp.issparse(X)
+        if self.gamma == "scale":
+            # var = E[X^2] - E[X]^2 if sparse
+            X_var = (X.multiply(X)).mean() - (X.mean()) ** 2 if sparse else X.var()
+            self._gamma = 1.0 / (n_features * X_var) if X_var != 0 else 1.0
+        else:
+            self._gamma = self.gamma
+        self.random_weights_ = (2.0 * self._gamma) ** 0.5 * random_state.normal(
+            size=(n_features, self.n_components)
+        )
+
+        self.random_offset_ = random_state.uniform(0, 2 * np.pi, size=self.n_components)
 
-        self.random_weights_ = (np.sqrt(2 * self.gamma) * random_state.normal(
-            size=(n_features, self.n_components)))
+        if X.dtype == np.float32:
+            # Setting the data type of the fitted attribute will ensure the
+            # output data type during `transform`.
+            self.random_weights_ = self.random_weights_.astype(X.dtype, copy=False)
+            self.random_offset_ = self.random_offset_.astype(X.dtype, copy=False)
 
-        self.random_offset_ = random_state.uniform(0, 2 * np.pi,
-                                                   size=self.n_components)
+        self._n_features_out = self.n_components
         return self
 
     def transform(self, X):
@@ -293,26 +390,34 @@ def transform(self, X):
         Parameters
         ----------
         X : {array-like, sparse matrix}, shape (n_samples, n_features)
-            New data, where n_samples in the number of samples
-            and n_features is the number of features.
+            New data, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
 
         Returns
         -------
         X_new : array-like, shape (n_samples, n_components)
+            Returns the instance itself.
         """
         check_is_fitted(self)
 
-        X = self._validate_data(X, accept_sparse='csr', reset=False)
+        X = validate_data(self, X, accept_sparse="csr", reset=False)
         projection = safe_sparse_dot(X, self.random_weights_)
         projection += self.random_offset_
         np.cos(projection, projection)
-        projection *= np.sqrt(2.) / np.sqrt(self.n_components)
+        projection *= (2.0 / self.n_components) ** 0.5
         return projection
 
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
+        tags.transformer_tags.preserves_dtype = ["float64", "float32"]
+        return tags
 
-class SkewedChi2Sampler(TransformerMixin, BaseEstimator):
-    """Approximates feature map of the "skewed chi-squared" kernel by Monte
-    Carlo approximation of its Fourier transform.
+
+class SkewedChi2Sampler(
+    ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator
+):
+    """Approximate feature map for "skewed chi-squared" kernel.
 
     Read more in the :ref:`User Guide `.
 
@@ -322,7 +427,7 @@ class SkewedChi2Sampler(TransformerMixin, BaseEstimator):
         "skewedness" parameter of the kernel. Needs to be cross-validated.
 
     n_components : int, default=100
-        number of Monte Carlo samples per original feature.
+        Number of Monte Carlo samples per original feature.
         Equals the dimensionality of the computed feature space.
 
     random_state : int, RandomState instance or None, default=None
@@ -341,6 +446,32 @@ class SkewedChi2Sampler(TransformerMixin, BaseEstimator):
         Bias term, which will be added to the data. It is uniformly distributed
         between 0 and 2*pi.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    AdditiveChi2Sampler : Approximate feature map for additive chi2 kernel.
+    Nystroem : Approximate a kernel map using a subset of the training data.
+    RBFSampler : Approximate a RBF kernel feature map using random Fourier
+        features.
+    SkewedChi2Sampler : Approximate feature map for "skewed chi-squared" kernel.
+    sklearn.metrics.pairwise.chi2_kernel : The exact chi squared kernel.
+    sklearn.metrics.pairwise.kernel_metrics : List of built-in kernels.
+
+    References
+    ----------
+    See "Random Fourier Approximations for Skewed Multiplicative Histogram
+    Kernels" by Fuxin Li, Catalin Ionescu and Cristian Sminchisescu.
+
     Examples
     --------
     >>> from sklearn.kernel_approximation import SkewedChi2Sampler
@@ -356,25 +487,20 @@ class SkewedChi2Sampler(TransformerMixin, BaseEstimator):
     SGDClassifier(max_iter=10)
     >>> clf.score(X_features, y)
     1.0
+    """
 
-    References
-    ----------
-    See "Random Fourier Approximations for Skewed Multiplicative Histogram
-    Kernels" by Fuxin Li, Catalin Ionescu and Cristian Sminchisescu.
-
-    See Also
-    --------
-    AdditiveChi2Sampler : A different approach for approximating an additive
-        variant of the chi squared kernel.
+    _parameter_constraints: dict = {
+        "skewedness": [Interval(Real, None, None, closed="neither")],
+        "n_components": [Interval(Integral, 1, None, closed="left")],
+        "random_state": ["random_state"],
+    }
 
-    sklearn.metrics.pairwise.chi2_kernel : The exact chi squared kernel.
-    """
-    @_deprecate_positional_args
-    def __init__(self, *, skewedness=1., n_components=100, random_state=None):
+    def __init__(self, *, skewedness=1.0, n_components=100, random_state=None):
         self.skewedness = skewedness
         self.n_components = n_components
         self.random_state = random_state
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y=None):
         """Fit the model with X.
 
@@ -383,24 +509,33 @@ def fit(self, X, y=None):
         Parameters
         ----------
         X : array-like, shape (n_samples, n_features)
-            Training data, where n_samples in the number of samples
-            and n_features is the number of features.
+            Training data, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
+
+        y : array-like, shape (n_samples,) or (n_samples, n_outputs), \
+                default=None
+            Target values (None for unsupervised transformations).
 
         Returns
         -------
         self : object
-            Returns the transformer.
+            Returns the instance itself.
         """
-
-        X = self._validate_data(X)
+        X = validate_data(self, X)
         random_state = check_random_state(self.random_state)
         n_features = X.shape[1]
         uniform = random_state.uniform(size=(n_features, self.n_components))
         # transform by inverse CDF of sech
-        self.random_weights_ = (1. / np.pi
-                                * np.log(np.tan(np.pi / 2. * uniform)))
-        self.random_offset_ = random_state.uniform(0, 2 * np.pi,
-                                                   size=self.n_components)
+        self.random_weights_ = 1.0 / np.pi * np.log(np.tan(np.pi / 2.0 * uniform))
+        self.random_offset_ = random_state.uniform(0, 2 * np.pi, size=self.n_components)
+
+        if X.dtype == np.float32:
+            # Setting the data type of the fitted attribute will ensure the
+            # output data type during `transform`.
+            self.random_weights_ = self.random_weights_.astype(X.dtype, copy=False)
+            self.random_offset_ = self.random_offset_.astype(X.dtype, copy=False)
+
+        self._n_features_out = self.n_components
         return self
 
     def transform(self, X):
@@ -409,30 +544,35 @@ def transform(self, X):
         Parameters
         ----------
         X : array-like, shape (n_samples, n_features)
-            New data, where n_samples in the number of samples
-            and n_features is the number of features. All values of X must be
+            New data, where `n_samples` is the number of samples
+            and `n_features` is the number of features. All values of X must be
             strictly greater than "-skewedness".
 
         Returns
         -------
         X_new : array-like, shape (n_samples, n_components)
+            Returns the instance itself.
         """
         check_is_fitted(self)
-
-        X = as_float_array(X, copy=True)
-        X = self._validate_data(X, copy=False, reset=False)
+        X = validate_data(
+            self, X, copy=True, dtype=[np.float64, np.float32], reset=False
+        )
         if (X <= -self.skewedness).any():
-            raise ValueError("X may not contain entries smaller than"
-                             " -skewedness.")
+            raise ValueError("X may not contain entries smaller than -skewedness.")
 
         X += self.skewedness
         np.log(X, X)
         projection = safe_sparse_dot(X, self.random_weights_)
         projection += self.random_offset_
         np.cos(projection, projection)
-        projection *= np.sqrt(2.) / np.sqrt(self.n_components)
+        projection *= np.sqrt(2.0) / np.sqrt(self.n_components)
         return projection
 
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.transformer_tags.preserves_dtype = ["float64", "float32"]
+        return tags
+
 
 class AdditiveChi2Sampler(TransformerMixin, BaseEstimator):
     """Approximate feature map for additive chi2 kernel.
@@ -442,7 +582,7 @@ class AdditiveChi2Sampler(TransformerMixin, BaseEstimator):
 
     Since the kernel that is to be approximated is additive, the components of
     the input vectors can be treated separately.  Each entry in the original
-    space is transformed into 2*sample_steps+1 features, where sample_steps is
+    space is transformed into 2*sample_steps-1 features, where sample_steps is
     a parameter of the method. Typical values of sample_steps include 1, 2 and
     3.
 
@@ -455,33 +595,22 @@ class AdditiveChi2Sampler(TransformerMixin, BaseEstimator):
     ----------
     sample_steps : int, default=2
         Gives the number of (complex) sampling points.
+
     sample_interval : float, default=None
         Sampling interval. Must be specified when sample_steps not in {1,2,3}.
 
     Attributes
     ----------
-    sample_interval_ : float
-        Stored sampling interval. Specified as a parameter if sample_steps not
-        in {1,2,3}.
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
 
-    Examples
-    --------
-    >>> from sklearn.datasets import load_digits
-    >>> from sklearn.linear_model import SGDClassifier
-    >>> from sklearn.kernel_approximation import AdditiveChi2Sampler
-    >>> X, y = load_digits(return_X_y=True)
-    >>> chi2sampler = AdditiveChi2Sampler(sample_steps=2)
-    >>> X_transformed = chi2sampler.fit_transform(X, y)
-    >>> clf = SGDClassifier(max_iter=5, random_state=0, tol=1e-3)
-    >>> clf.fit(X_transformed, y)
-    SGDClassifier(max_iter=5, random_state=0)
-    >>> clf.score(X_transformed, y)
-    0.9499...
+        .. versionadded:: 0.24
 
-    Notes
-    -----
-    This estimator approximates a slightly different version of the additive
-    chi squared kernel then ``metric.additive_chi2`` computes.
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
 
     See Also
     --------
@@ -493,48 +622,76 @@ class AdditiveChi2Sampler(TransformerMixin, BaseEstimator):
     sklearn.metrics.pairwise.additive_chi2_kernel : The exact additive chi
         squared kernel.
 
+    Notes
+    -----
+    This estimator approximates a slightly different version of the additive
+    chi squared kernel then ``metric.additive_chi2`` computes.
+
+    This estimator is stateless and does not need to be fitted. However, we
+    recommend to call :meth:`fit_transform` instead of :meth:`transform`, as
+    parameter validation is only performed in :meth:`fit`.
+
     References
     ----------
     See `"Efficient additive kernels via explicit feature maps"
     `_
     A. Vedaldi and A. Zisserman, Pattern Analysis and Machine Intelligence,
     2011
+
+    Examples
+    --------
+    >>> from sklearn.datasets import load_digits
+    >>> from sklearn.linear_model import SGDClassifier
+    >>> from sklearn.kernel_approximation import AdditiveChi2Sampler
+    >>> X, y = load_digits(return_X_y=True)
+    >>> chi2sampler = AdditiveChi2Sampler(sample_steps=2)
+    >>> X_transformed = chi2sampler.fit_transform(X, y)
+    >>> clf = SGDClassifier(max_iter=5, random_state=0, tol=1e-3)
+    >>> clf.fit(X_transformed, y)
+    SGDClassifier(max_iter=5, random_state=0)
+    >>> clf.score(X_transformed, y)
+    0.9499...
     """
-    @_deprecate_positional_args
+
+    _parameter_constraints: dict = {
+        "sample_steps": [Interval(Integral, 1, None, closed="left")],
+        "sample_interval": [Interval(Real, 0, None, closed="left"), None],
+    }
+
     def __init__(self, *, sample_steps=2, sample_interval=None):
         self.sample_steps = sample_steps
         self.sample_interval = sample_interval
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y=None):
-        """Set the parameters
+        """Only validates estimator's parameters.
+
+        This method allows to: (i) validate the estimator's parameters and
+        (ii) be consistent with the scikit-learn transformer API.
 
         Parameters
         ----------
         X : array-like, shape (n_samples, n_features)
-            Training data, where n_samples in the number of samples
-            and n_features is the number of features.
+            Training data, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
+
+        y : array-like, shape (n_samples,) or (n_samples, n_outputs), \
+                default=None
+            Target values (None for unsupervised transformations).
 
         Returns
         -------
         self : object
             Returns the transformer.
         """
-        X = self._validate_data(X, accept_sparse='csr')
-        check_non_negative(X, 'X in AdditiveChi2Sampler.fit')
+        X = validate_data(self, X, accept_sparse="csr", ensure_non_negative=True)
+
+        if self.sample_interval is None and self.sample_steps not in (1, 2, 3):
+            raise ValueError(
+                "If sample_steps is not in [1, 2, 3],"
+                " you need to provide sample_interval"
+            )
 
-        if self.sample_interval is None:
-            # See reference, figure 2 c)
-            if self.sample_steps == 1:
-                self.sample_interval_ = 0.8
-            elif self.sample_steps == 2:
-                self.sample_interval_ = 0.5
-            elif self.sample_steps == 3:
-                self.sample_interval_ = 0.4
-            else:
-                raise ValueError("If sample_steps is not in [1, 2, 3],"
-                                 " you need to provide sample_interval")
-        else:
-            self.sample_interval_ = self.sample_interval
         return self
 
     def transform(self, X):
@@ -542,45 +699,93 @@ def transform(self, X):
 
         Parameters
         ----------
-        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+        X : {array-like, sparse matrix}, shape (n_samples, n_features)
+            Training data, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
 
         Returns
         -------
         X_new : {ndarray, sparse matrix}, \
-               shape = (n_samples, n_features * (2*sample_steps + 1))
-            Whether the return value is an array of sparse matrix depends on
+               shape = (n_samples, n_features * (2*sample_steps - 1))
+            Whether the return value is an array or sparse matrix depends on
             the type of the input X.
         """
-        msg = ("%(name)s is not fitted. Call fit to set the parameters before"
-               " calling transform")
-        check_is_fitted(self, msg=msg)
-
-        X = self._validate_data(X, accept_sparse='csr', reset=False)
-        check_non_negative(X, 'X in AdditiveChi2Sampler.transform')
+        X = validate_data(
+            self, X, accept_sparse="csr", reset=False, ensure_non_negative=True
+        )
         sparse = sp.issparse(X)
 
+        if self.sample_interval is None:
+            # See figure 2 c) of "Efficient additive kernels via explicit feature maps"
+            # 
+            # A. Vedaldi and A. Zisserman, Pattern Analysis and Machine Intelligence,
+            # 2011
+            if self.sample_steps == 1:
+                sample_interval = 0.8
+            elif self.sample_steps == 2:
+                sample_interval = 0.5
+            elif self.sample_steps == 3:
+                sample_interval = 0.4
+            else:
+                raise ValueError(
+                    "If sample_steps is not in [1, 2, 3],"
+                    " you need to provide sample_interval"
+                )
+        else:
+            sample_interval = self.sample_interval
+
         # zeroth component
         # 1/cosh = sech
         # cosh(0) = 1.0
-
         transf = self._transform_sparse if sparse else self._transform_dense
-        return transf(X)
+        return transf(X, self.sample_steps, sample_interval)
+
+    def get_feature_names_out(self, input_features=None):
+        """Get output feature names for transformation.
+
+        Parameters
+        ----------
+        input_features : array-like of str or None, default=None
+            Only used to validate feature names with the names seen in :meth:`fit`.
+
+        Returns
+        -------
+        feature_names_out : ndarray of str objects
+            Transformed feature names.
+        """
+        # Note that passing attributes="n_features_in_" forces check_is_fitted
+        # to check if the attribute is present. Otherwise it will pass on this
+        # stateless estimator (requires_fit=False)
+        check_is_fitted(self, attributes="n_features_in_")
+        input_features = _check_feature_names_in(
+            self, input_features, generate_names=True
+        )
+        est_name = self.__class__.__name__.lower()
 
-    def _transform_dense(self, X):
-        non_zero = (X != 0.0)
+        names_list = [f"{est_name}_{name}_sqrt" for name in input_features]
+
+        for j in range(1, self.sample_steps):
+            cos_names = [f"{est_name}_{name}_cos{j}" for name in input_features]
+            sin_names = [f"{est_name}_{name}_sin{j}" for name in input_features]
+            names_list.extend(cos_names + sin_names)
+
+        return np.asarray(names_list, dtype=object)
+
+    @staticmethod
+    def _transform_dense(X, sample_steps, sample_interval):
+        non_zero = X != 0.0
         X_nz = X[non_zero]
 
         X_step = np.zeros_like(X)
-        X_step[non_zero] = np.sqrt(X_nz * self.sample_interval_)
+        X_step[non_zero] = np.sqrt(X_nz * sample_interval)
 
         X_new = [X_step]
 
-        log_step_nz = self.sample_interval_ * np.log(X_nz)
-        step_nz = 2 * X_nz * self.sample_interval_
+        log_step_nz = sample_interval * np.log(X_nz)
+        step_nz = 2 * X_nz * sample_interval
 
-        for j in range(1, self.sample_steps):
-            factor_nz = np.sqrt(step_nz /
-                                np.cosh(np.pi * j * self.sample_interval_))
+        for j in range(1, sample_steps):
+            factor_nz = np.sqrt(step_nz / np.cosh(np.pi * j * sample_interval))
 
             X_step = np.zeros_like(X)
             X_step[non_zero] = factor_nz * np.cos(j * log_step_nz)
@@ -592,40 +797,46 @@ def _transform_dense(self, X):
 
         return np.hstack(X_new)
 
-    def _transform_sparse(self, X):
+    @staticmethod
+    def _transform_sparse(X, sample_steps, sample_interval):
         indices = X.indices.copy()
         indptr = X.indptr.copy()
 
-        data_step = np.sqrt(X.data * self.sample_interval_)
-        X_step = sp.csr_matrix((data_step, indices, indptr),
-                               shape=X.shape, dtype=X.dtype, copy=False)
+        data_step = np.sqrt(X.data * sample_interval)
+        X_step = sp.csr_matrix(
+            (data_step, indices, indptr), shape=X.shape, dtype=X.dtype, copy=False
+        )
         X_new = [X_step]
 
-        log_step_nz = self.sample_interval_ * np.log(X.data)
-        step_nz = 2 * X.data * self.sample_interval_
+        log_step_nz = sample_interval * np.log(X.data)
+        step_nz = 2 * X.data * sample_interval
 
-        for j in range(1, self.sample_steps):
-            factor_nz = np.sqrt(step_nz /
-                                np.cosh(np.pi * j * self.sample_interval_))
+        for j in range(1, sample_steps):
+            factor_nz = np.sqrt(step_nz / np.cosh(np.pi * j * sample_interval))
 
             data_step = factor_nz * np.cos(j * log_step_nz)
-            X_step = sp.csr_matrix((data_step, indices, indptr),
-                                   shape=X.shape, dtype=X.dtype, copy=False)
+            X_step = sp.csr_matrix(
+                (data_step, indices, indptr), shape=X.shape, dtype=X.dtype, copy=False
+            )
             X_new.append(X_step)
 
             data_step = factor_nz * np.sin(j * log_step_nz)
-            X_step = sp.csr_matrix((data_step, indices, indptr),
-                                   shape=X.shape, dtype=X.dtype, copy=False)
+            X_step = sp.csr_matrix(
+                (data_step, indices, indptr), shape=X.shape, dtype=X.dtype, copy=False
+            )
             X_new.append(X_step)
 
         return sp.hstack(X_new)
 
-    def _more_tags(self):
-        return {'stateless': True,
-                'requires_positive_X': True}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.requires_fit = False
+        tags.input_tags.positive_only = True
+        tags.input_tags.sparse = True
+        return tags
 
 
-class Nystroem(TransformerMixin, BaseEstimator):
+class Nystroem(ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator):
     """Approximate a kernel map using a subset of the training data.
 
     Constructs an approximate feature map for an arbitrary kernel
@@ -637,9 +848,9 @@ class Nystroem(TransformerMixin, BaseEstimator):
 
     Parameters
     ----------
-    kernel : string or callable, default='rbf'
+    kernel : str or callable, default='rbf'
         Kernel map to be approximated. A callable should accept two arguments
-        and the keyword arguments passed to this object as kernel_params, and
+        and the keyword arguments passed to this object as `kernel_params`, and
         should return a floating point number.
 
     gamma : float, default=None
@@ -665,14 +876,14 @@ class Nystroem(TransformerMixin, BaseEstimator):
 
     random_state : int, RandomState instance or None, default=None
         Pseudo-random number generator to control the uniform sampling without
-        replacement of n_components of the training data to construct the basis
-        kernel.
+        replacement of `n_components` of the training data to construct the
+        basis kernel.
         Pass an int for reproducible output across multiple function calls.
         See :term:`Glossary `.
 
     n_jobs : int, default=None
         The number of jobs to use for the computation. This works by breaking
-        down the kernel matrix into n_jobs even slices and computing them in
+        down the kernel matrix into `n_jobs` even slices and computing them in
         parallel.
 
         ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
@@ -693,6 +904,37 @@ class Nystroem(TransformerMixin, BaseEstimator):
         Normalization matrix needed for embedding.
         Square root of the kernel matrix on ``components_``.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    AdditiveChi2Sampler : Approximate feature map for additive chi2 kernel.
+    PolynomialCountSketch : Polynomial kernel approximation via Tensor Sketch.
+    RBFSampler : Approximate a RBF kernel feature map using random Fourier
+        features.
+    SkewedChi2Sampler : Approximate feature map for "skewed chi-squared" kernel.
+    sklearn.metrics.pairwise.kernel_metrics : List of built-in kernels.
+
+    References
+    ----------
+    * Williams, C.K.I. and Seeger, M.
+      "Using the Nystroem method to speed up kernel machines",
+      Advances in neural information processing systems 2001
+
+    * T. Yang, Y. Li, M. Mahdavi, R. Jin and Z. Zhou
+      "Nystroem Method vs Random Fourier Features: A Theoretical and Empirical
+      Comparison",
+      Advances in Neural Information Processing Systems 2012
+
     Examples
     --------
     >>> from sklearn import datasets, svm
@@ -708,31 +950,34 @@ class Nystroem(TransformerMixin, BaseEstimator):
     LinearSVC()
     >>> clf.score(data_transformed, y)
     0.9987...
-
-    References
-    ----------
-    * Williams, C.K.I. and Seeger, M.
-      "Using the Nystroem method to speed up kernel machines",
-      Advances in neural information processing systems 2001
-
-    * T. Yang, Y. Li, M. Mahdavi, R. Jin and Z. Zhou
-      "Nystroem Method vs Random Fourier Features: A Theoretical and Empirical
-      Comparison",
-      Advances in Neural Information Processing Systems 2012
-
-
-    See Also
-    --------
-    RBFSampler : An approximation to the RBF kernel using random Fourier
-        features.
-
-    sklearn.metrics.pairwise.kernel_metrics : List of built-in kernels.
     """
-    @_deprecate_positional_args
-    def __init__(self, kernel="rbf", *, gamma=None, coef0=None, degree=None,
-                 kernel_params=None, n_components=100, random_state=None,
-                 n_jobs=None):
 
+    _parameter_constraints: dict = {
+        "kernel": [
+            StrOptions(set(PAIRWISE_KERNEL_FUNCTIONS.keys()) | {"precomputed"}),
+            callable,
+        ],
+        "gamma": [Interval(Real, 0, None, closed="left"), None],
+        "coef0": [Interval(Real, None, None, closed="neither"), None],
+        "degree": [Interval(Real, 1, None, closed="left"), None],
+        "kernel_params": [dict, None],
+        "n_components": [Interval(Integral, 1, None, closed="left")],
+        "random_state": ["random_state"],
+        "n_jobs": [Integral, None],
+    }
+
+    def __init__(
+        self,
+        kernel="rbf",
+        *,
+        gamma=None,
+        coef0=None,
+        degree=None,
+        kernel_params=None,
+        n_components=100,
+        random_state=None,
+        n_jobs=None,
+    ):
         self.kernel = kernel
         self.gamma = gamma
         self.coef0 = coef0
@@ -742,6 +987,7 @@ def __init__(self, kernel="rbf", *, gamma=None, coef0=None, degree=None,
         self.random_state = random_state
         self.n_jobs = n_jobs
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y=None):
         """Fit estimator to data.
 
@@ -750,10 +996,20 @@ def fit(self, X, y=None):
 
         Parameters
         ----------
-        X : array-like of shape (n_samples, n_features)
-            Training data.
+        X : array-like, shape (n_samples, n_features)
+            Training data, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
+
+        y : array-like, shape (n_samples,) or (n_samples, n_outputs), \
+                default=None
+            Target values (None for unsupervised transformations).
+
+        Returns
+        -------
+        self : object
+            Returns the instance itself.
         """
-        X = self._validate_data(X, accept_sparse='csr')
+        X = validate_data(self, X, accept_sparse="csr")
         rnd = check_random_state(self.random_state)
         n_samples = X.shape[0]
 
@@ -761,9 +1017,11 @@ def fit(self, X, y=None):
         if self.n_components > n_samples:
             # XXX should we just bail?
             n_components = n_samples
-            warnings.warn("n_components > n_samples. This is not possible.\n"
-                          "n_components was set to n_samples, which results"
-                          " in inefficient evaluation of the full kernel.")
+            warnings.warn(
+                "n_components > n_samples. This is not possible.\n"
+                "n_components was set to n_samples, which results"
+                " in inefficient evaluation of the full kernel."
+            )
 
         else:
             n_components = self.n_components
@@ -772,17 +1030,21 @@ def fit(self, X, y=None):
         basis_inds = inds[:n_components]
         basis = X[basis_inds]
 
-        basis_kernel = pairwise_kernels(basis, metric=self.kernel,
-                                        filter_params=True,
-                                        n_jobs=self.n_jobs,
-                                        **self._get_kernel_params())
+        basis_kernel = pairwise_kernels(
+            basis,
+            metric=self.kernel,
+            filter_params=True,
+            n_jobs=self.n_jobs,
+            **self._get_kernel_params(),
+        )
 
         # sqrt of kernel matrix on basis vectors
         U, S, V = svd(basis_kernel)
         S = np.maximum(S, 1e-12)
         self.normalization_ = np.dot(U / np.sqrt(S), V)
         self.components_ = basis
-        self.component_indices_ = inds
+        self.component_indices_ = basis_inds
+        self._n_features_out = n_components
         return self
 
     def transform(self, X):
@@ -802,39 +1064,43 @@ def transform(self, X):
             Transformed data.
         """
         check_is_fitted(self)
-        X = self._validate_data(X, accept_sparse='csr', reset=False)
+        X = validate_data(self, X, accept_sparse="csr", reset=False)
 
         kernel_params = self._get_kernel_params()
-        embedded = pairwise_kernels(X, self.components_,
-                                    metric=self.kernel,
-                                    filter_params=True,
-                                    n_jobs=self.n_jobs,
-                                    **kernel_params)
+        embedded = pairwise_kernels(
+            X,
+            self.components_,
+            metric=self.kernel,
+            filter_params=True,
+            n_jobs=self.n_jobs,
+            **kernel_params,
+        )
         return np.dot(embedded, self.normalization_.T)
 
     def _get_kernel_params(self):
         params = self.kernel_params
         if params is None:
             params = {}
-        if not callable(self.kernel) and self.kernel != 'precomputed':
-            for param in (KERNEL_PARAMS[self.kernel]):
+        if not callable(self.kernel) and self.kernel != "precomputed":
+            for param in KERNEL_PARAMS[self.kernel]:
                 if getattr(self, param) is not None:
                     params[param] = getattr(self, param)
         else:
-            if (self.gamma is not None or
-                    self.coef0 is not None or
-                    self.degree is not None):
-                raise ValueError("Don't pass gamma, coef0 or degree to "
-                                 "Nystroem if using a callable "
-                                 "or precomputed kernel")
+            if (
+                self.gamma is not None
+                or self.coef0 is not None
+                or self.degree is not None
+            ):
+                raise ValueError(
+                    "Don't pass gamma, coef0 or degree to "
+                    "Nystroem if using a callable "
+                    "or precomputed kernel"
+                )
 
         return params
 
-    def _more_tags(self):
-        return {
-            '_xfail_checks': {
-                'check_transformer_preserve_dtypes':
-                'dtypes are preserved but not at a close enough precision',
-            },
-            'preserves_dtype': [np.float64, np.float32]
-        }
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
+        tags.transformer_tags.preserves_dtype = ["float64", "float32"]
+        return tags
diff --git a/sklearn/kernel_ridge.py b/sklearn/kernel_ridge.py
index 8a27ea572b344..29e744647acc9 100644
--- a/sklearn/kernel_ridge.py
+++ b/sklearn/kernel_ridge.py
@@ -1,17 +1,17 @@
-"""Module :mod:`sklearn.kernel_ridge` implements kernel ridge regression."""
+"""Kernel ridge regression."""
 
-# Authors: Mathieu Blondel 
-#          Jan Hendrik Metzen 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+from numbers import Real
 
 import numpy as np
 
-from .base import BaseEstimator, RegressorMixin, MultiOutputMixin
-from .metrics.pairwise import pairwise_kernels
+from .base import BaseEstimator, MultiOutputMixin, RegressorMixin, _fit_context
 from .linear_model._ridge import _solve_cholesky_kernel
-from .utils.validation import check_is_fitted, _check_sample_weight
-from .utils.validation import _deprecate_positional_args
-from .utils.deprecation import deprecated
+from .metrics.pairwise import PAIRWISE_KERNEL_FUNCTIONS, pairwise_kernels
+from .utils._param_validation import Interval, StrOptions
+from .utils.validation import _check_sample_weight, check_is_fitted, validate_data
 
 
 class KernelRidge(MultiOutputMixin, RegressorMixin, BaseEstimator):
@@ -49,11 +49,11 @@ class KernelRidge(MultiOutputMixin, RegressorMixin, BaseEstimator):
         assumed to be specific to the targets. Hence they must correspond in
         number. See :ref:`ridge_regression` for formula.
 
-    kernel : string or callable, default="linear"
+    kernel : str or callable, default="linear"
         Kernel mapping used internally. This parameter is directly passed to
-        :class:`~sklearn.metrics.pairwise.pairwise_kernel`.
+        :class:`~sklearn.metrics.pairwise.pairwise_kernels`.
         If `kernel` is a string, it must be one of the metrics
-        in `pairwise.PAIRWISE_KERNEL_FUNCTIONS`.
+        in `pairwise.PAIRWISE_KERNEL_FUNCTIONS` or "precomputed".
         If `kernel` is "precomputed", X is assumed to be a kernel matrix.
         Alternatively, if `kernel` is a callable function, it is called on
         each pair of instances (rows) and the resulting value recorded. The
@@ -76,7 +76,7 @@ class KernelRidge(MultiOutputMixin, RegressorMixin, BaseEstimator):
         Zero coefficient for polynomial and sigmoid kernels.
         Ignored by other kernels.
 
-    kernel_params : mapping of string to any, default=None
+    kernel_params : dict, default=None
         Additional parameters (keyword arguments) for kernel function passed
         as callable object.
 
@@ -90,17 +90,34 @@ class KernelRidge(MultiOutputMixin, RegressorMixin, BaseEstimator):
         kernel == "precomputed" this is instead the precomputed
         training matrix, of shape (n_samples, n_samples).
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    sklearn.gaussian_process.GaussianProcessRegressor : Gaussian
+        Process regressor providing automatic kernel hyperparameters
+        tuning and predictions uncertainty.
+    sklearn.linear_model.Ridge : Linear ridge regression.
+    sklearn.linear_model.RidgeCV : Ridge regression with built-in
+        cross-validation.
+    sklearn.svm.SVR : Support Vector Regression accepting a large variety
+        of kernels.
+
     References
     ----------
     * Kevin P. Murphy
       "Machine Learning: A Probabilistic Perspective", The MIT Press
       chapter 14.4.3, pp. 492-493
 
-    See Also
-    --------
-    sklearn.linear_model.Ridge : Linear ridge regression.
-    sklearn.svm.SVR : Support Vector Regression implemented using libsvm.
-
     Examples
     --------
     >>> from sklearn.kernel_ridge import KernelRidge
@@ -109,13 +126,33 @@ class KernelRidge(MultiOutputMixin, RegressorMixin, BaseEstimator):
     >>> rng = np.random.RandomState(0)
     >>> y = rng.randn(n_samples)
     >>> X = rng.randn(n_samples, n_features)
-    >>> clf = KernelRidge(alpha=1.0)
-    >>> clf.fit(X, y)
+    >>> krr = KernelRidge(alpha=1.0)
+    >>> krr.fit(X, y)
     KernelRidge(alpha=1.0)
     """
-    @_deprecate_positional_args
-    def __init__(self, alpha=1, *, kernel="linear", gamma=None, degree=3,
-                 coef0=1, kernel_params=None):
+
+    _parameter_constraints: dict = {
+        "alpha": [Interval(Real, 0, None, closed="left"), "array-like"],
+        "kernel": [
+            StrOptions(set(PAIRWISE_KERNEL_FUNCTIONS.keys()) | {"precomputed"}),
+            callable,
+        ],
+        "gamma": [Interval(Real, 0, None, closed="left"), None],
+        "degree": [Interval(Real, 0, None, closed="left")],
+        "coef0": [Interval(Real, None, None, closed="neither")],
+        "kernel_params": [dict, None],
+    }
+
+    def __init__(
+        self,
+        alpha=1,
+        *,
+        kernel="linear",
+        gamma=None,
+        degree=3,
+        coef0=1,
+        kernel_params=None,
+    ):
         self.alpha = alpha
         self.kernel = kernel
         self.gamma = gamma
@@ -127,25 +164,18 @@ def _get_kernel(self, X, Y=None):
         if callable(self.kernel):
             params = self.kernel_params or {}
         else:
-            params = {"gamma": self.gamma,
-                      "degree": self.degree,
-                      "coef0": self.coef0}
-        return pairwise_kernels(X, Y, metric=self.kernel,
-                                filter_params=True, **params)
-
-    def _more_tags(self):
-        return {'pairwise': self.kernel == 'precomputed'}
-
-    # TODO: Remove in 1.1
-    # mypy error: Decorated property not supported
-    @deprecated("Attribute _pairwise was deprecated in "  # type: ignore
-                "version 0.24 and will be removed in 1.1 (renaming of 0.26).")
-    @property
-    def _pairwise(self):
-        return self.kernel == "precomputed"
+            params = {"gamma": self.gamma, "degree": self.degree, "coef0": self.coef0}
+        return pairwise_kernels(X, Y, metric=self.kernel, filter_params=True, **params)
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
+        tags.input_tags.pairwise = self.kernel == "precomputed"
+        return tags
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y, sample_weight=None):
-        """Fit Kernel Ridge regression model
+        """Fit Kernel Ridge regression model.
 
         Parameters
         ----------
@@ -154,18 +184,20 @@ def fit(self, X, y, sample_weight=None):
             a precomputed kernel matrix, of shape (n_samples, n_samples).
 
         y : array-like of shape (n_samples,) or (n_samples, n_targets)
-            Target values
+            Target values.
 
         sample_weight : float or array-like of shape (n_samples,), default=None
             Individual weights for each sample, ignored if None is passed.
 
         Returns
         -------
-        self : returns an instance of self.
+        self : object
+            Returns the instance itself.
         """
         # Convert data
-        X, y = self._validate_data(X, y, accept_sparse=("csr", "csc"),
-                                   multi_output=True, y_numeric=True)
+        X, y = validate_data(
+            self, X, y, accept_sparse=("csr", "csc"), multi_output=True, y_numeric=True
+        )
         if sample_weight is not None and not isinstance(sample_weight, float):
             sample_weight = _check_sample_weight(sample_weight, X)
 
@@ -178,9 +210,7 @@ def fit(self, X, y, sample_weight=None):
             ravel = True
 
         copy = self.kernel == "precomputed"
-        self.dual_coef_ = _solve_cholesky_kernel(K, y, alpha,
-                                                 sample_weight,
-                                                 copy)
+        self.dual_coef_ = _solve_cholesky_kernel(K, y, alpha, sample_weight, copy)
         if ravel:
             self.dual_coef_ = self.dual_coef_.ravel()
 
@@ -189,7 +219,7 @@ def fit(self, X, y, sample_weight=None):
         return self
 
     def predict(self, X):
-        """Predict using the kernel ridge model
+        """Predict using the kernel ridge model.
 
         Parameters
         ----------
@@ -205,6 +235,6 @@ def predict(self, X):
             Returns predicted values.
         """
         check_is_fitted(self)
-        X = self._validate_data(X, accept_sparse=("csr", "csc"), reset=False)
+        X = validate_data(self, X, accept_sparse=("csr", "csc"), reset=False)
         K = self._get_kernel(X, self.X_fit_)
         return np.dot(K, self.dual_coef_)
diff --git a/sklearn/linear_model/__init__.py b/sklearn/linear_model/__init__.py
index 110e0008bccc9..541f164daf46a 100644
--- a/sklearn/linear_model/__init__.py
+++ b/sklearn/linear_model/__init__.py
@@ -1,80 +1,95 @@
-"""
-The :mod:`sklearn.linear_model` module implements a variety of linear models.
-"""
+"""A variety of linear models."""
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 # See http://scikit-learn.sourceforge.net/modules/sgd.html and
 # http://scikit-learn.sourceforge.net/modules/linear_model.html for
 # complete documentation.
 
 from ._base import LinearRegression
-from ._bayes import BayesianRidge, ARDRegression
-from ._least_angle import (Lars, LassoLars, lars_path, lars_path_gram, LarsCV,
-                           LassoLarsCV, LassoLarsIC)
-from ._coordinate_descent import (Lasso, ElasticNet, LassoCV, ElasticNetCV,
-                                  lasso_path, enet_path, MultiTaskLasso,
-                                  MultiTaskElasticNet, MultiTaskElasticNetCV,
-                                  MultiTaskLassoCV)
-from ._glm import (PoissonRegressor,
-                   GammaRegressor, TweedieRegressor)
+from ._bayes import ARDRegression, BayesianRidge
+from ._coordinate_descent import (
+    ElasticNet,
+    ElasticNetCV,
+    Lasso,
+    LassoCV,
+    MultiTaskElasticNet,
+    MultiTaskElasticNetCV,
+    MultiTaskLasso,
+    MultiTaskLassoCV,
+    enet_path,
+    lasso_path,
+)
+from ._glm import GammaRegressor, PoissonRegressor, TweedieRegressor
 from ._huber import HuberRegressor
-from ._sgd_fast import Hinge, Log, ModifiedHuber, SquaredLoss, Huber
-from ._stochastic_gradient import SGDClassifier, SGDRegressor
-from ._ridge import (Ridge, RidgeCV, RidgeClassifier, RidgeClassifierCV,
-                     ridge_regression)
+from ._least_angle import (
+    Lars,
+    LarsCV,
+    LassoLars,
+    LassoLarsCV,
+    LassoLarsIC,
+    lars_path,
+    lars_path_gram,
+)
 from ._logistic import LogisticRegression, LogisticRegressionCV
-from ._omp import (orthogonal_mp, orthogonal_mp_gram,
-                   OrthogonalMatchingPursuit, OrthogonalMatchingPursuitCV)
-from ._passive_aggressive import PassiveAggressiveClassifier
-from ._passive_aggressive import PassiveAggressiveRegressor
+from ._omp import (
+    OrthogonalMatchingPursuit,
+    OrthogonalMatchingPursuitCV,
+    orthogonal_mp,
+    orthogonal_mp_gram,
+)
+from ._passive_aggressive import PassiveAggressiveClassifier, PassiveAggressiveRegressor
 from ._perceptron import Perceptron
-
+from ._quantile import QuantileRegressor
 from ._ransac import RANSACRegressor
+from ._ridge import Ridge, RidgeClassifier, RidgeClassifierCV, RidgeCV, ridge_regression
+from ._stochastic_gradient import SGDClassifier, SGDOneClassSVM, SGDRegressor
 from ._theil_sen import TheilSenRegressor
 
-__all__ = ['ARDRegression',
-           'BayesianRidge',
-           'ElasticNet',
-           'ElasticNetCV',
-           'Hinge',
-           'Huber',
-           'HuberRegressor',
-           'Lars',
-           'LarsCV',
-           'Lasso',
-           'LassoCV',
-           'LassoLars',
-           'LassoLarsCV',
-           'LassoLarsIC',
-           'LinearRegression',
-           'Log',
-           'LogisticRegression',
-           'LogisticRegressionCV',
-           'ModifiedHuber',
-           'MultiTaskElasticNet',
-           'MultiTaskElasticNetCV',
-           'MultiTaskLasso',
-           'MultiTaskLassoCV',
-           'OrthogonalMatchingPursuit',
-           'OrthogonalMatchingPursuitCV',
-           'PassiveAggressiveClassifier',
-           'PassiveAggressiveRegressor',
-           'Perceptron',
-           'Ridge',
-           'RidgeCV',
-           'RidgeClassifier',
-           'RidgeClassifierCV',
-           'SGDClassifier',
-           'SGDRegressor',
-           'SquaredLoss',
-           'TheilSenRegressor',
-           'enet_path',
-           'lars_path',
-           'lars_path_gram',
-           'lasso_path',
-           'orthogonal_mp',
-           'orthogonal_mp_gram',
-           'ridge_regression',
-           'RANSACRegressor',
-           'PoissonRegressor',
-           'GammaRegressor',
-           'TweedieRegressor']
+__all__ = [
+    "ARDRegression",
+    "BayesianRidge",
+    "ElasticNet",
+    "ElasticNetCV",
+    "GammaRegressor",
+    "HuberRegressor",
+    "Lars",
+    "LarsCV",
+    "Lasso",
+    "LassoCV",
+    "LassoLars",
+    "LassoLarsCV",
+    "LassoLarsIC",
+    "LinearRegression",
+    "LogisticRegression",
+    "LogisticRegressionCV",
+    "MultiTaskElasticNet",
+    "MultiTaskElasticNetCV",
+    "MultiTaskLasso",
+    "MultiTaskLassoCV",
+    "OrthogonalMatchingPursuit",
+    "OrthogonalMatchingPursuitCV",
+    "PassiveAggressiveClassifier",
+    "PassiveAggressiveRegressor",
+    "Perceptron",
+    "PoissonRegressor",
+    "QuantileRegressor",
+    "RANSACRegressor",
+    "Ridge",
+    "RidgeCV",
+    "RidgeClassifier",
+    "RidgeClassifierCV",
+    "SGDClassifier",
+    "SGDOneClassSVM",
+    "SGDRegressor",
+    "TheilSenRegressor",
+    "TweedieRegressor",
+    "enet_path",
+    "lars_path",
+    "lars_path_gram",
+    "lasso_path",
+    "orthogonal_mp",
+    "orthogonal_mp_gram",
+    "ridge_regression",
+]
diff --git a/sklearn/linear_model/_base.py b/sklearn/linear_model/_base.py
index 1842620dfa105..c059e3fa84310 100644
--- a/sklearn/linear_model/_base.py
+++ b/sklearn/linear_model/_base.py
@@ -2,45 +2,47 @@
 Generalized Linear Models.
 """
 
-# Author: Alexandre Gramfort 
-# Fabian Pedregosa 
-# Olivier Grisel 
-#         Vincent Michel 
-#         Peter Prettenhofer 
-#         Mathieu Blondel 
-#         Lars Buitinck
-#         Maryan Morel 
-#         Giorgio Patrini 
-#         Maria Telenczuk 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-from abc import ABCMeta, abstractmethod
 import numbers
 import warnings
+from abc import ABCMeta, abstractmethod
+from numbers import Integral, Real
 
 import numpy as np
 import scipy.sparse as sp
-from scipy import linalg
-from scipy import optimize
-from scipy import sparse
+from scipy import linalg, optimize, sparse
+from scipy.sparse.linalg import lsqr
 from scipy.special import expit
-from joblib import Parallel
-
-from ..base import (BaseEstimator, ClassifierMixin, RegressorMixin,
-                    MultiOutputMixin)
-from ..utils import check_array
-from ..utils.validation import FLOAT_DTYPES
-from ..utils.validation import _deprecate_positional_args
-from ..utils import check_random_state
-from ..utils.extmath import safe_sparse_dot
-from ..utils.extmath import _incremental_mean_and_var
-from ..utils.sparsefuncs import mean_variance_axis, inplace_column_scale
-from ..utils.fixes import sparse_lsqr
-from ..utils._seq_dataset import ArrayDataset32, CSRDataset32
-from ..utils._seq_dataset import ArrayDataset64, CSRDataset64
-from ..utils.validation import check_is_fitted, _check_sample_weight
 
-from ..utils.fixes import delayed
+from ..base import (
+    BaseEstimator,
+    ClassifierMixin,
+    MultiOutputMixin,
+    RegressorMixin,
+    _fit_context,
+)
+from ..utils import check_array, check_random_state
+from ..utils._array_api import (
+    _asarray_with_order,
+    _average,
+    get_namespace,
+    get_namespace_and_device,
+    indexing_dtype,
+    supported_float_dtypes,
+)
+from ..utils._param_validation import Interval
+from ..utils._seq_dataset import (
+    ArrayDataset32,
+    ArrayDataset64,
+    CSRDataset32,
+    CSRDataset64,
+)
+from ..utils.extmath import safe_sparse_dot
+from ..utils.parallel import Parallel, delayed
+from ..utils.sparsefuncs import mean_variance_axis
+from ..utils.validation import _check_sample_weight, check_is_fitted, validate_data
 
 # TODO: bayesian_ridge_regression and bayesian_regression_ard
 # should be squashed into its respective objects.
@@ -50,94 +52,6 @@
 # intercept oscillation.
 
 
-# FIXME in 1.2: parameter 'normalize' should be removed from linear models
-# in cases where now normalize=False. The default value of 'normalize' should
-# be changed to False in linear models where now normalize=True
-def _deprecate_normalize(normalize, default, estimator_name):
-    """ Normalize is to be deprecated from linear models and a use of
-    a pipeline with a StandardScaler is to be recommended instead.
-    Here the appropriate message is selected to be displayed to the user
-    depending on the default normalize value (as it varies between the linear
-    models and normalize value selected by the user).
-
-    Parameters
-    ----------
-    normalize : bool,
-        normalize value passed by the user
-
-    default : bool,
-        default normalize value used by the estimator
-
-    estimator_name : string,
-        name of the linear estimator which calls this function.
-        The name will be used for writing the deprecation warnings
-
-    Returns
-    -------
-    normalize : bool,
-        normalize value which should further be used by the estimator at this
-        stage of the depreciation process
-
-    Notes
-    -----
-    This function should be updated in 1.2 depending on the value of
-    `normalize`:
-    - True, warning: `normalize` was deprecated in 1.2 and will be removed in
-      1.4. Suggest to use pipeline instead.
-    - False, `normalize` was deprecated in 1.2 and it will be removed in 1.4.
-      Leave normalize to its default value.
-    - `deprecated` - this should only be possible with default == False as from
-      1.2 `normalize` in all the linear models should be either removed or the
-      default should be set to False.
-    This function should be completely removed in 1.4.
-    """
-
-    if normalize not in [True, False, 'deprecated']:
-        raise ValueError("Leave 'normalize' to its default value or set it "
-                         "to True or False")
-
-    if normalize == 'deprecated':
-        _normalize = default
-    else:
-        _normalize = normalize
-
-    if default and normalize == 'deprecated':
-        warnings.warn(
-            "The default of 'normalize' will be set to False in version 1.2 "
-            "and deprecated in version 1.4. \nPass normalize=False and use "
-            "Pipeline with a StandardScaler in a preprocessing stage if you "
-            "wish to reproduce the previous behavior:\n"
-            "model = make_pipeline(StandardScaler(with_mean=False), \n"
-            f"{estimator_name}(normalize=False))\n"
-            "If you wish to use additional parameters in "
-            "the fit() you can include them as follows:\n"
-            "kwargs = {model.steps[-1][0] + "
-            "'__': }\n"
-            "model.fit(X, y, **kwargs)", FutureWarning
-        )
-    elif normalize != 'deprecated' and normalize and not default:
-        warnings.warn(
-            "'normalize' was deprecated in version 1.0 and will be "
-            "removed in 1.2 \nIf you still wish to normalize use "
-            "Pipeline with a StandardScaler in a preprocessing stage if you "
-            "wish to reproduce the previous behavior:\n"
-            "model = make_pipeline(StandardScaler(with_mean=False), "
-            f"{estimator_name}()). \nIf you wish to use additional "
-            "parameters in the fit() you can include them as follows: "
-            "kwargs = {model.steps[-1][0] + "
-            "'__': }\n"
-            "model.fit(X, y, **kwargs)", FutureWarning
-        )
-    elif not normalize and not default:
-        warnings.warn(
-            "'normalize' was deprecated in version 1.0 and will be"
-            " removed in 1.2 Don't set 'normalize' parameter"
-            " and leave it to its default value", FutureWarning
-        )
-
-    return _normalize
-
-
 def make_dataset(X, y, sample_weight, random_state=None):
     """Create ``Dataset`` abstraction for sparse and dense inputs.
 
@@ -156,7 +70,8 @@ def make_dataset(X, y, sample_weight, random_state=None):
         The weight of each sample
 
     random_state : int, RandomState instance or None (default)
-        Determines random number generation for dataset shuffling and noise.
+        Determines random number generation for dataset random sampling. It is not
+        used for dataset shuffling.
         Pass an int for reproducible output across multiple function calls.
         See :term:`Glossary `.
 
@@ -180,8 +95,7 @@ def make_dataset(X, y, sample_weight, random_state=None):
         ArrayData = ArrayDataset64
 
     if sp.issparse(X):
-        dataset = CSRData(X.data, X.indptr, X.indices, y, sample_weight,
-                          seed=seed)
+        dataset = CSRData(X.data, X.indptr, X.indices, y, sample_weight, seed=seed)
         intercept_decay = SPARSE_INTERCEPT_DECAY
     else:
         X = np.ascontiguousarray(X)
@@ -191,87 +105,100 @@ def make_dataset(X, y, sample_weight, random_state=None):
     return dataset, intercept_decay
 
 
-def _preprocess_data(X, y, fit_intercept, normalize=False, copy=True,
-                     sample_weight=None, return_mean=False, check_input=True):
-    """Center and scale data.
-
-    Centers data to have mean zero along axis 0. If fit_intercept=False or if
-    the X is a sparse matrix, no centering is done, but normalization can still
-    be applied. The function returns the statistics necessary to reconstruct
-    the input data, which are X_offset, y_offset, X_scale, such that the output
-
-        X = (X - X_offset) / X_scale
-
-    X_scale is the L2 norm of X - X_offset. If sample_weight is not None,
-    then the weighted mean of X and y is zero, and not the mean itself. If
-    return_mean=True, the mean, eventually weighted, is returned, independently
-    of whether X was centered (option used for optimization with sparse data in
-    coordinate_descend).
+def _preprocess_data(
+    X,
+    y,
+    *,
+    fit_intercept,
+    copy=True,
+    copy_y=True,
+    sample_weight=None,
+    check_input=True,
+):
+    """Common data preprocessing for fitting linear models.
+
+    This helper is in charge of the following steps:
+
+    - Ensure that `sample_weight` is an array or `None`.
+    - If `check_input=True`, perform standard input validation of `X`, `y`.
+    - Perform copies if requested to avoid side-effects in case of inplace
+      modifications of the input.
+
+    Then, if `fit_intercept=True` this preprocessing centers both `X` and `y` as
+    follows:
+        - if `X` is dense, center the data and
+        store the mean vector in `X_offset`.
+        - if `X` is sparse, store the mean in `X_offset`
+        without centering `X`. The centering is expected to be handled by the
+        linear solver where appropriate.
+        - in either case, always center `y` and store the mean in `y_offset`.
+        - both `X_offset` and `y_offset` are always weighted by `sample_weight`
+          if not set to `None`.
+
+    If `fit_intercept=False`, no centering is performed and `X_offset`, `y_offset`
+    are set to zero.
 
-    This is here because nearly all linear models will want their data to be
-    centered. This function also systematically makes y consistent with X.dtype
+    Returns
+    -------
+    X_out : {ndarray, sparse matrix} of shape (n_samples, n_features)
+        If copy=True a copy of the input X is triggered, otherwise operations are
+        inplace.
+        If input X is dense, then X_out is centered.
+    y_out : {ndarray, sparse matrix} of shape (n_samples,) or (n_samples, n_targets)
+        Centered version of y. Possibly performed inplace on input y depending
+        on the copy_y parameter.
+    X_offset : ndarray of shape (n_features,)
+        The mean per column of input X.
+    y_offset : float or ndarray of shape (n_features,)
+    X_scale : ndarray of shape (n_features,)
+        Always an array of ones. TODO: refactor the code base to make it
+        possible to remove this unused variable.
     """
+    xp, _, device_ = get_namespace_and_device(X, y, sample_weight)
+    n_samples, n_features = X.shape
+    X_is_sparse = sp.issparse(X)
+
     if isinstance(sample_weight, numbers.Number):
         sample_weight = None
     if sample_weight is not None:
-        sample_weight = np.asarray(sample_weight)
+        sample_weight = xp.asarray(sample_weight)
 
     if check_input:
-        X = check_array(X, copy=copy, accept_sparse=['csr', 'csc'],
-                        dtype=FLOAT_DTYPES)
-    elif copy:
-        if sp.issparse(X):
-            X = X.copy()
-        else:
-            X = X.copy(order='K')
+        X = check_array(
+            X, copy=copy, accept_sparse=["csr", "csc"], dtype=supported_float_dtypes(xp)
+        )
+        y = check_array(y, dtype=X.dtype, copy=copy_y, ensure_2d=False)
+    else:
+        y = xp.astype(y, X.dtype, copy=copy_y)
+        if copy:
+            if X_is_sparse:
+                X = X.copy()
+            else:
+                X = _asarray_with_order(X, order="K", copy=True, xp=xp)
 
-    y = np.asarray(y, dtype=X.dtype)
+    dtype_ = X.dtype
 
     if fit_intercept:
-        if sp.issparse(X):
-            X_offset, X_var = mean_variance_axis(
-                X, axis=0, weights=sample_weight
-            )
-            if not return_mean:
-                X_offset[:] = X.dtype.type(0)
+        if X_is_sparse:
+            X_offset, X_var = mean_variance_axis(X, axis=0, weights=sample_weight)
         else:
-            if normalize:
-                X_offset, X_var, _ = _incremental_mean_and_var(
-                    X, last_mean=0., last_variance=0., last_sample_count=0.,
-                    sample_weight=sample_weight
-                )
-            else:
-                X_offset = np.average(X, axis=0, weights=sample_weight)
+            X_offset = _average(X, axis=0, weights=sample_weight, xp=xp)
 
-            X_offset = X_offset.astype(X.dtype, copy=False)
+            X_offset = xp.astype(X_offset, X.dtype, copy=False)
             X -= X_offset
 
-        if normalize:
-            X_var = X_var.astype(X.dtype, copy=False)
-            # Detect constant features on the computed variance, before taking
-            # the np.sqrt. Otherwise constant features cannot be detected with
-            # sample_weights.
-            constant_mask = X_var < 10 * np.finfo(X.dtype).eps
-            X_var *= X.shape[0]
-            X_scale = np.sqrt(X_var, out=X_var)
-            X_scale[constant_mask] = 1.
-            if sp.issparse(X):
-                inplace_column_scale(X, 1. / X_scale)
-            else:
-                X /= X_scale
-        else:
-            X_scale = np.ones(X.shape[1], dtype=X.dtype)
-
-        y_offset = np.average(y, axis=0, weights=sample_weight)
-        y = y - y_offset
+        y_offset = _average(y, axis=0, weights=sample_weight, xp=xp)
+        y -= y_offset
     else:
-        X_offset = np.zeros(X.shape[1], dtype=X.dtype)
-        X_scale = np.ones(X.shape[1], dtype=X.dtype)
+        X_offset = xp.zeros(n_features, dtype=X.dtype, device=device_)
         if y.ndim == 1:
-            y_offset = X.dtype.type(0)
+            y_offset = xp.asarray(0.0, dtype=dtype_, device=device_)
         else:
-            y_offset = np.zeros(y.shape[1], dtype=X.dtype)
+            y_offset = xp.zeros(y.shape[1], dtype=dtype_, device=device_)
 
+    # XXX: X_scale is no longer needed. It is an historic artifact from the
+    # time where linear model exposed the normalize parameter.
+    X_scale = xp.ones(n_features, dtype=X.dtype, device=device_)
     return X, y, X_offset, y_offset, X_scale
 
 
@@ -279,10 +206,22 @@ def _preprocess_data(X, y, fit_intercept, normalize=False, copy=True,
 # Currently, the fact that sag implements its own way to deal with
 # sample_weight makes the refactoring tricky.
 
-def _rescale_data(X, y, sample_weight):
+
+def _rescale_data(X, y, sample_weight, inplace=False):
     """Rescale data sample-wise by square root of sample_weight.
 
-    For many linear models, this enables easy support for sample_weight.
+    For many linear models, this enables easy support for sample_weight because
+
+        (y - X w)' S (y - X w)
+
+    with S = diag(sample_weight) becomes
+
+        ||y_rescaled - X_rescaled w||_2^2
+
+    when setting
+
+        y_rescaled = sqrt(S) y
+        X_rescaled = sqrt(S) X
 
     Returns
     -------
@@ -290,17 +229,39 @@ def _rescale_data(X, y, sample_weight):
 
     y_rescaled : {array-like, sparse matrix}
     """
+    # Assume that _validate_data and _check_sample_weight have been called by
+    # the caller.
+    xp, _ = get_namespace(X, y, sample_weight)
     n_samples = X.shape[0]
-    sample_weight = np.asarray(sample_weight)
-    if sample_weight.ndim == 0:
-        sample_weight = np.full(n_samples, sample_weight,
-                                dtype=sample_weight.dtype)
-    sample_weight = np.sqrt(sample_weight)
-    sw_matrix = sparse.dia_matrix((sample_weight, 0),
-                                  shape=(n_samples, n_samples))
-    X = safe_sparse_dot(sw_matrix, X)
-    y = safe_sparse_dot(sw_matrix, y)
-    return X, y
+    sample_weight_sqrt = xp.sqrt(sample_weight)
+
+    if sp.issparse(X) or sp.issparse(y):
+        sw_matrix = sparse.dia_matrix(
+            (sample_weight_sqrt, 0), shape=(n_samples, n_samples)
+        )
+
+    if sp.issparse(X):
+        X = safe_sparse_dot(sw_matrix, X)
+    else:
+        if inplace:
+            X *= sample_weight_sqrt[:, None]
+        else:
+            X = X * sample_weight_sqrt[:, None]
+
+    if sp.issparse(y):
+        y = safe_sparse_dot(sw_matrix, y)
+    else:
+        if inplace:
+            if y.ndim == 1:
+                y *= sample_weight_sqrt
+            else:
+                y *= sample_weight_sqrt[:, None]
+        else:
+            if y.ndim == 1:
+                y = y * sample_weight_sqrt
+            else:
+                y = y * sample_weight_sqrt[:, None]
+    return X, y, sample_weight_sqrt
 
 
 class LinearModel(BaseEstimator, metaclass=ABCMeta):
@@ -313,10 +274,12 @@ def fit(self, X, y):
     def _decision_function(self, X):
         check_is_fitted(self)
 
-        X = self._validate_data(X, accept_sparse=['csr', 'csc', 'coo'],
-                                reset=False)
-        return safe_sparse_dot(X, self.coef_.T,
-                               dense_output=True) + self.intercept_
+        X = validate_data(self, X, accept_sparse=["csr", "csc", "coo"], reset=False)
+        coef_ = self.coef_
+        if coef_.ndim == 1:
+            return X @ coef_ + self.intercept_
+        else:
+            return X @ coef_.T + self.intercept_
 
     def predict(self, X):
         """
@@ -334,19 +297,26 @@ def predict(self, X):
         """
         return self._decision_function(X)
 
-    _preprocess_data = staticmethod(_preprocess_data)
-
     def _set_intercept(self, X_offset, y_offset, X_scale):
-        """Set the intercept_
-        """
+        """Set the intercept_"""
+
+        xp, _ = get_namespace(X_offset, y_offset, X_scale)
+
         if self.fit_intercept:
-            self.coef_ = self.coef_ / X_scale
-            self.intercept_ = y_offset - np.dot(X_offset, self.coef_.T)
-        else:
-            self.intercept_ = 0.
+            # We always want coef_.dtype=X.dtype. For instance, X.dtype can differ from
+            # coef_.dtype if warm_start=True.
+            coef_ = xp.astype(self.coef_, X_scale.dtype, copy=False)
+            coef_ = self.coef_ = xp.divide(coef_, X_scale)
 
-    def _more_tags(self):
-        return {'requires_y': True}
+            if coef_.ndim == 1:
+                intercept_ = y_offset - X_offset @ coef_
+            else:
+                intercept_ = y_offset - X_offset @ coef_.T
+
+            self.intercept_ = intercept_
+
+        else:
+            self.intercept_ = 0.0
 
 
 # XXX Should this derive from LinearModel? It should be a mixin, not an ABC.
@@ -366,22 +336,26 @@ def decision_function(self, X):
 
         Parameters
         ----------
-        X : array-like or sparse matrix, shape (n_samples, n_features)
-            Samples.
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            The data matrix for which we want to get the confidence scores.
 
         Returns
         -------
-        array, shape=(n_samples,) if n_classes == 2 else (n_samples, n_classes)
-            Confidence scores per (sample, class) combination. In the binary
-            case, confidence score for self.classes_[1] where >0 means this
-            class would be predicted.
+        scores : ndarray of shape (n_samples,) or (n_samples, n_classes)
+            Confidence scores per `(n_samples, n_classes)` combination. In the
+            binary case, confidence score for `self.classes_[1]` where >0 means
+            this class would be predicted.
         """
         check_is_fitted(self)
-
-        X = self._validate_data(X, accept_sparse='csr', reset=False)
-        scores = safe_sparse_dot(X, self.coef_.T,
-                                 dense_output=True) + self.intercept_
-        return scores.ravel() if scores.shape[1] == 1 else scores
+        xp, _ = get_namespace(X)
+
+        X = validate_data(self, X, accept_sparse="csr", reset=False)
+        scores = safe_sparse_dot(X, self.coef_.T, dense_output=True) + self.intercept_
+        return (
+            xp.reshape(scores, (-1,))
+            if (scores.ndim > 1 and scores.shape[1] == 1)
+            else scores
+        )
 
     def predict(self, X):
         """
@@ -389,20 +363,22 @@ def predict(self, X):
 
         Parameters
         ----------
-        X : array-like or sparse matrix, shape (n_samples, n_features)
-            Samples.
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            The data matrix for which we want to get the predictions.
 
         Returns
         -------
-        C : array, shape [n_samples]
-            Predicted class label per sample.
+        y_pred : ndarray of shape (n_samples,)
+            Vector containing the class labels for each sample.
         """
+        xp, _ = get_namespace(X)
         scores = self.decision_function(X)
         if len(scores.shape) == 1:
-            indices = (scores > 0).astype(int)
+            indices = xp.astype(scores > 0, indexing_dtype(xp))
         else:
-            indices = scores.argmax(axis=1)
-        return self.classes_[indices]
+            indices = xp.argmax(scores, axis=1)
+
+        return xp.take(self.classes_, indices, axis=0)
 
     def _predict_proba_lr(self, X):
         """Probability estimation for OvR logistic regression.
@@ -494,32 +470,34 @@ class LinearRegression(MultiOutputMixin, RegressorMixin, LinearModel):
         to False, no intercept will be used in calculations
         (i.e. data is expected to be centered).
 
-    normalize : bool, default=False
-        This parameter is ignored when ``fit_intercept`` is set to False.
-        If True, the regressors X will be normalized before regression by
-        subtracting the mean and dividing by the l2-norm.
-        If you wish to standardize, please use
-        :class:`~sklearn.preprocessing.StandardScaler` before calling ``fit``
-        on an estimator with ``normalize=False``.
-
-        .. deprecated:: 1.0
-           `normalize` was deprecated in version 1.0 and will be
-           removed in 1.2.
-
     copy_X : bool, default=True
         If True, X will be copied; else, it may be overwritten.
 
+    tol : float, default=1e-6
+        The precision of the solution (`coef_`) is determined by `tol` which
+        specifies a different convergence criterion for the `lsqr` solver.
+        `tol` is set as `atol` and `btol` of `scipy.sparse.linalg.lsqr` when
+        fitting on sparse training data. This parameter has no effect when fitting
+        on dense data.
+
+        .. versionadded:: 1.7
+
     n_jobs : int, default=None
         The number of jobs to use for the computation. This will only provide
-        speedup for n_targets > 1 and sufficient large problems.
-        ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
-        ``-1`` means using all processors. See :term:`Glossary `
-        for more details.
+        speedup in case of sufficiently large problems, that is if firstly
+        `n_targets > 1` and secondly `X` is sparse or if `positive` is set
+        to `True`. ``None`` means 1 unless in a
+        :obj:`joblib.parallel_backend` context. ``-1`` means using all
+        processors. See :term:`Glossary ` for more details.
 
     positive : bool, default=False
         When set to ``True``, forces the coefficients to be positive. This
         option is only supported for dense arrays.
 
+        For a comparison between a linear regression model with positive constraints
+        on the regression coefficients and a linear regression without such constraints,
+        see :ref:`sphx_glr_auto_examples_linear_model_plot_nnls.py`.
+
         .. versionadded:: 0.24
 
     Attributes
@@ -540,6 +518,17 @@ class LinearRegression(MultiOutputMixin, RegressorMixin, LinearModel):
         Independent term in the linear model. Set to 0.0 if
         `fit_intercept = False`.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     See Also
     --------
     Ridge : Ridge regression addresses some of the
@@ -570,19 +559,35 @@ class LinearRegression(MultiOutputMixin, RegressorMixin, LinearModel):
     >>> reg.coef_
     array([1., 2.])
     >>> reg.intercept_
-    3.0000...
+    np.float64(3.0)
     >>> reg.predict(np.array([[3, 5]]))
     array([16.])
     """
-    @_deprecate_positional_args
-    def __init__(self, *, fit_intercept=True, normalize='deprecated',
-                 copy_X=True, n_jobs=None, positive=False):
+
+    _parameter_constraints: dict = {
+        "fit_intercept": ["boolean"],
+        "copy_X": ["boolean"],
+        "n_jobs": [None, Integral],
+        "positive": ["boolean"],
+        "tol": [Interval(Real, 0, None, closed="left")],
+    }
+
+    def __init__(
+        self,
+        *,
+        fit_intercept=True,
+        copy_X=True,
+        tol=1e-6,
+        n_jobs=None,
+        positive=False,
+    ):
         self.fit_intercept = fit_intercept
-        self.normalize = normalize
         self.copy_X = copy_X
+        self.tol = tol
         self.n_jobs = n_jobs
         self.positive = positive
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y, sample_weight=None):
         """
         Fit linear model.
@@ -590,83 +595,110 @@ def fit(self, X, y, sample_weight=None):
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
-            Training data
+            Training data.
 
         y : array-like of shape (n_samples,) or (n_samples, n_targets)
-            Target values. Will be cast to X's dtype if necessary
+            Target values. Will be cast to X's dtype if necessary.
 
         sample_weight : array-like of shape (n_samples,), default=None
-            Individual weights for each sample
+            Individual weights for each sample.
 
             .. versionadded:: 0.17
                parameter *sample_weight* support to LinearRegression.
 
         Returns
         -------
-        self : returns an instance of self.
+        self : object
+            Fitted Estimator.
         """
-
-        _normalize = _deprecate_normalize(
-            self.normalize, default=False,
-            estimator_name=self.__class__.__name__
-        )
-
         n_jobs_ = self.n_jobs
 
-        accept_sparse = False if self.positive else ['csr', 'csc', 'coo']
+        accept_sparse = False if self.positive else ["csr", "csc", "coo"]
 
-        X, y = self._validate_data(X, y, accept_sparse=accept_sparse,
-                                   y_numeric=True, multi_output=True)
+        X, y = validate_data(
+            self,
+            X,
+            y,
+            accept_sparse=accept_sparse,
+            y_numeric=True,
+            multi_output=True,
+            force_writeable=True,
+        )
 
-        if sample_weight is not None:
-            sample_weight = _check_sample_weight(sample_weight, X,
-                                                 dtype=X.dtype)
+        has_sw = sample_weight is not None
+        if has_sw:
+            sample_weight = _check_sample_weight(
+                sample_weight, X, dtype=X.dtype, ensure_non_negative=True
+            )
 
-        X, y, X_offset, y_offset, X_scale = self._preprocess_data(
-            X, y, fit_intercept=self.fit_intercept, normalize=_normalize,
-            copy=self.copy_X, sample_weight=sample_weight,
-            return_mean=True)
+        # Note that neither _rescale_data nor the rest of the fit method of
+        # LinearRegression can benefit from in-place operations when X is a
+        # sparse matrix. Therefore, let's not copy X when it is sparse.
+        copy_X_in_preprocess_data = self.copy_X and not sp.issparse(X)
 
-        if sample_weight is not None:
-            # Sample weight can be implemented via a simple rescaling.
-            X, y = _rescale_data(X, y, sample_weight)
+        X, y, X_offset, y_offset, X_scale = _preprocess_data(
+            X,
+            y,
+            fit_intercept=self.fit_intercept,
+            copy=copy_X_in_preprocess_data,
+            sample_weight=sample_weight,
+        )
+
+        if has_sw:
+            # Sample weight can be implemented via a simple rescaling. Note
+            # that we safely do inplace rescaling when _preprocess_data has
+            # already made a copy if requested.
+            X, y, sample_weight_sqrt = _rescale_data(
+                X, y, sample_weight, inplace=copy_X_in_preprocess_data
+            )
 
         if self.positive:
             if y.ndim < 2:
-                self.coef_, self._residues = optimize.nnls(X, y)
+                self.coef_ = optimize.nnls(X, y)[0]
             else:
                 # scipy.optimize.nnls cannot handle y with shape (M, K)
                 outs = Parallel(n_jobs=n_jobs_)(
-                        delayed(optimize.nnls)(X, y[:, j])
-                        for j in range(y.shape[1]))
-                self.coef_, self._residues = map(np.vstack, zip(*outs))
+                    delayed(optimize.nnls)(X, y[:, j]) for j in range(y.shape[1])
+                )
+                self.coef_ = np.vstack([out[0] for out in outs])
         elif sp.issparse(X):
             X_offset_scale = X_offset / X_scale
 
-            def matvec(b):
-                return X.dot(b) - b.dot(X_offset_scale)
+            if has_sw:
+
+                def matvec(b):
+                    return X.dot(b) - sample_weight_sqrt * b.dot(X_offset_scale)
+
+                def rmatvec(b):
+                    return X.T.dot(b) - X_offset_scale * b.dot(sample_weight_sqrt)
 
-            def rmatvec(b):
-                return X.T.dot(b) - X_offset_scale * np.sum(b)
+            else:
+
+                def matvec(b):
+                    return X.dot(b) - b.dot(X_offset_scale)
 
-            X_centered = sparse.linalg.LinearOperator(shape=X.shape,
-                                                      matvec=matvec,
-                                                      rmatvec=rmatvec)
+                def rmatvec(b):
+                    return X.T.dot(b) - X_offset_scale * b.sum()
+
+            X_centered = sparse.linalg.LinearOperator(
+                shape=X.shape, matvec=matvec, rmatvec=rmatvec
+            )
 
             if y.ndim < 2:
-                out = sparse_lsqr(X_centered, y)
-                self.coef_ = out[0]
-                self._residues = out[3]
+                self.coef_ = lsqr(X_centered, y, atol=self.tol, btol=self.tol)[0]
             else:
                 # sparse_lstsq cannot handle y with shape (M, K)
                 outs = Parallel(n_jobs=n_jobs_)(
-                    delayed(sparse_lsqr)(X_centered, y[:, j].ravel())
-                    for j in range(y.shape[1]))
+                    delayed(lsqr)(
+                        X_centered, y[:, j].ravel(), atol=self.tol, btol=self.tol
+                    )
+                    for j in range(y.shape[1])
+                )
                 self.coef_ = np.vstack([out[0] for out in outs])
-                self._residues = np.vstack([out[3] for out in outs])
         else:
-            self.coef_, self._residues, self.rank_, self.singular_ = \
-                linalg.lstsq(X, y)
+            # cut-off ratio for small singular values
+            cond = max(X.shape) * np.finfo(X.dtype).eps
+            self.coef_, _, self.rank_, self.singular_ = linalg.lstsq(X, y, cond=cond)
             self.coef_ = self.coef_.T
 
         if y.ndim == 1:
@@ -674,10 +706,15 @@ def rmatvec(b):
         self._set_intercept(X_offset, y_offset, X_scale)
         return self
 
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = not self.positive
+        return tags
+
 
-def _check_precomputed_gram_matrix(X, precompute, X_offset, X_scale,
-                                   rtol=1e-7,
-                                   atol=1e-5):
+def _check_precomputed_gram_matrix(
+    X, precompute, X_offset, X_scale, rtol=None, atol=1e-5
+):
     """Computes a single element of the gram matrix and compares it to
     the corresponding element of the user supplied gram matrix.
 
@@ -697,8 +734,10 @@ def _check_precomputed_gram_matrix(X, precompute, X_offset, X_scale,
     X_scale : ndarray of shape (n_features,)
         Array of feature scale factors used to normalize design matrix.
 
-    rtol : float, default=1e-7
-        Relative tolerance; see numpy.allclose.
+    rtol : float, default=None
+        Relative tolerance; see numpy.allclose
+        If None, it is set to 1e-4 for arrays of dtype numpy.float32 and 1e-7
+        otherwise.
 
     atol : float, default=1e-5
         absolute tolerance; see :func`numpy.allclose`. Note that the default
@@ -713,7 +752,7 @@ def _check_precomputed_gram_matrix(X, precompute, X_offset, X_scale,
 
     n_features = X.shape[1]
     f1 = n_features // 2
-    f2 = min(f1+1, n_features-1)
+    f2 = min(f1 + 1, n_features - 1)
 
     v1 = (X[:, f1] - X_offset[f1]) * X_scale[f1]
     v2 = (X[:, f2] - X_offset[f2]) * X_scale[f2]
@@ -721,53 +760,80 @@ def _check_precomputed_gram_matrix(X, precompute, X_offset, X_scale,
     expected = np.dot(v1, v2)
     actual = precompute[f1, f2]
 
-    if not np.isclose(expected, actual, rtol=rtol, atol=atol):
-        raise ValueError("Gram matrix passed in via 'precompute' parameter "
-                         "did not pass validation when a single element was "
-                         "checked - please check that it was computed "
-                         f"properly. For element ({f1},{f2}) we computed "
-                         f"{expected} but the user-supplied value was "
-                         f"{actual}.")
+    dtypes = [precompute.dtype, expected.dtype]
+    if rtol is None:
+        rtols = [1e-4 if dtype == np.float32 else 1e-7 for dtype in dtypes]
+        rtol = max(rtols)
 
+    if not np.isclose(expected, actual, rtol=rtol, atol=atol):
+        raise ValueError(
+            "Gram matrix passed in via 'precompute' parameter "
+            "did not pass validation when a single element was "
+            "checked - please check that it was computed "
+            f"properly. For element ({f1},{f2}) we computed "
+            f"{expected} but the user-supplied value was "
+            f"{actual}."
+        )
 
-def _pre_fit(X, y, Xy, precompute, normalize, fit_intercept, copy,
-             check_input=True, sample_weight=None):
-    """Aux function used at beginning of fit in linear models
 
-    Parameters
-    ----------
-    order : 'F', 'C' or None, default=None
-        Whether X and y will be forced to be fortran or c-style. Only relevant
-        if sample_weight is not None.
+def _pre_fit(
+    X,
+    y,
+    Xy,
+    precompute,
+    fit_intercept,
+    copy,
+    check_input=True,
+    sample_weight=None,
+):
+    """Function used at beginning of fit in linear models with L1 or L0 penalty.
+
+    This function applies _preprocess_data and additionally computes the gram matrix
+    `precompute` as needed as well as `Xy`.
     """
     n_samples, n_features = X.shape
 
-    if sparse.isspmatrix(X):
+    if sparse.issparse(X):
         # copy is not needed here as X is not modified inplace when X is sparse
         precompute = False
         X, y, X_offset, y_offset, X_scale = _preprocess_data(
-            X, y, fit_intercept=fit_intercept, normalize=normalize,
-            copy=False, return_mean=True, check_input=check_input)
+            X,
+            y,
+            fit_intercept=fit_intercept,
+            copy=False,
+            check_input=check_input,
+            sample_weight=sample_weight,
+        )
     else:
         # copy was done in fit if necessary
         X, y, X_offset, y_offset, X_scale = _preprocess_data(
-            X, y, fit_intercept=fit_intercept, normalize=normalize, copy=copy,
-            check_input=check_input, sample_weight=sample_weight)
-    if sample_weight is not None:
-        X, y = _rescale_data(X, y, sample_weight=sample_weight)
+            X,
+            y,
+            fit_intercept=fit_intercept,
+            copy=copy,
+            check_input=check_input,
+            sample_weight=sample_weight,
+        )
+        # Rescale only in dense case. Sparse cd solver directly deals with
+        # sample_weight.
+        if sample_weight is not None:
+            # This triggers copies anyway.
+            X, y, _ = _rescale_data(X, y, sample_weight=sample_weight)
 
-    # FIXME: 'normalize' to be removed in 1.2
-    if hasattr(precompute, '__array__'):
-        if (fit_intercept and not np.allclose(X_offset, np.zeros(n_features))
-                or normalize and not np.allclose(X_scale, np.ones(n_features)
-                                                 )):
+    if hasattr(precompute, "__array__"):
+        if fit_intercept and not np.allclose(X_offset, np.zeros(n_features)):
             warnings.warn(
-                "Gram matrix was provided but X was centered to fit "
-                "intercept, or X was normalized : recomputing Gram matrix.",
-                UserWarning
+                (
+                    "Gram matrix was provided but X was centered to fit "
+                    "intercept: recomputing Gram matrix."
+                ),
+                UserWarning,
             )
+            # TODO: instead of warning and recomputing, we could just center
+            # the user provided Gram matrix a-posteriori (after making a copy
+            # when `copy=True`).
             # recompute Gram
-            precompute = 'auto'
+            precompute = "auto"
             Xy = None
         elif check_input:
             # If we're going to use the user's precomputed gram matrix, we
@@ -775,31 +841,29 @@ def _pre_fit(X, y, Xy, precompute, normalize, fit_intercept, copy,
             _check_precomputed_gram_matrix(X, precompute, X_offset, X_scale)
 
     # precompute if n_samples > n_features
-    if isinstance(precompute, str) and precompute == 'auto':
-        precompute = (n_samples > n_features)
+    if isinstance(precompute, str) and precompute == "auto":
+        precompute = n_samples > n_features
 
     if precompute is True:
         # make sure that the 'precompute' array is contiguous.
-        precompute = np.empty(shape=(n_features, n_features), dtype=X.dtype,
-                              order='C')
+        precompute = np.empty(shape=(n_features, n_features), dtype=X.dtype, order="C")
         np.dot(X.T, X, out=precompute)
 
-    if not hasattr(precompute, '__array__'):
+    if not hasattr(precompute, "__array__"):
         Xy = None  # cannot use Xy if precompute is not Gram
 
-    if hasattr(precompute, '__array__') and Xy is None:
-        common_dtype = np.find_common_type([X.dtype, y.dtype], [])
+    if hasattr(precompute, "__array__") and Xy is None:
+        common_dtype = np.result_type(X.dtype, y.dtype)
         if y.ndim == 1:
             # Xy is 1d, make sure it is contiguous.
-            Xy = np.empty(shape=n_features, dtype=common_dtype, order='C')
+            Xy = np.empty(shape=n_features, dtype=common_dtype, order="C")
             np.dot(X.T, y, out=Xy)
         else:
             # Make sure that Xy is always F contiguous even if X or y are not
             # contiguous: the goal is to make it fast to extract the data for a
             # specific target.
             n_targets = y.shape[1]
-            Xy = np.empty(shape=(n_features, n_targets), dtype=common_dtype,
-                          order='F')
+            Xy = np.empty(shape=(n_features, n_targets), dtype=common_dtype, order="F")
             np.dot(y.T, X, out=Xy.T)
 
     return X, y, X_offset, y_offset, X_scale, precompute, Xy
diff --git a/sklearn/linear_model/_bayes.py b/sklearn/linear_model/_bayes.py
index 634417e2b0efa..adf515d44d1d9 100644
--- a/sklearn/linear_model/_bayes.py
+++ b/sklearn/linear_model/_bayes.py
@@ -2,24 +2,27 @@
 Various bayesian regression
 """
 
-# Authors: V. Michel, F. Pedregosa, A. Gramfort
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 from math import log
+from numbers import Integral, Real
+
 import numpy as np
 from scipy import linalg
-
-from ._base import LinearModel, _rescale_data
-from ..base import RegressorMixin
-from ..utils.extmath import fast_logdet
 from scipy.linalg import pinvh
-from ..utils.validation import _check_sample_weight
-from ..utils.validation import _deprecate_positional_args
 
+from ..base import RegressorMixin, _fit_context
+from ..utils import _safe_indexing
+from ..utils._param_validation import Interval
+from ..utils.extmath import fast_logdet
+from ..utils.validation import _check_sample_weight, validate_data
+from ._base import LinearModel, _preprocess_data, _rescale_data
 
 ###############################################################################
 # BayesianRidge regression
 
+
 class BayesianRidge(RegressorMixin, LinearModel):
     """Bayesian ridge regression.
 
@@ -28,11 +31,17 @@ class BayesianRidge(RegressorMixin, LinearModel):
     lambda (precision of the weights) and alpha (precision of the noise).
 
     Read more in the :ref:`User Guide `.
+    For an intuitive visualization of how the sinusoid is approximated by
+    a polynomial using different pairs of initial values, see
+    :ref:`sphx_glr_auto_examples_linear_model_plot_bayesian_ridge_curvefit.py`.
 
     Parameters
     ----------
-    n_iter : int, default=300
-        Maximum number of iterations. Should be greater than or equal to 1.
+    max_iter : int, default=300
+        Maximum number of iterations over the complete dataset before
+        stopping independently of any early stopping criterion.
+
+        .. versionchanged:: 1.3
 
     tol : float, default=1e-3
         Stop the algorithm if w has converged.
@@ -57,13 +66,13 @@ class BayesianRidge(RegressorMixin, LinearModel):
         Initial value for alpha (precision of the noise).
         If not set, alpha_init is 1/Var(y).
 
-            .. versionadded:: 0.22
+        .. versionadded:: 0.22
 
     lambda_init : float, default=None
         Initial value for lambda (precision of the weights).
         If not set, lambda_init is 1.
 
-            .. versionadded:: 0.22
+        .. versionadded:: 0.22
 
     compute_score : bool, default=False
         If True, compute the log marginal likelihood at each iteration of the
@@ -76,21 +85,12 @@ class BayesianRidge(RegressorMixin, LinearModel):
         to False, no intercept will be used in calculations
         (i.e. data is expected to be centered).
 
-    normalize : bool, default=False
-        This parameter is ignored when ``fit_intercept`` is set to False.
-        If True, the regressors X will be normalized before regression by
-        subtracting the mean and dividing by the l2-norm.
-        If you wish to standardize, please use
-        :class:`~sklearn.preprocessing.StandardScaler` before calling ``fit``
-        on an estimator with ``normalize=False``.
-
     copy_X : bool, default=True
         If True, X will be copied; else, it may be overwritten.
 
     verbose : bool, default=False
         Verbose mode when fitting the model.
 
-
     Attributes
     ----------
     coef_ : array-like of shape (n_features,)
@@ -98,7 +98,7 @@ class BayesianRidge(RegressorMixin, LinearModel):
 
     intercept_ : float
         Independent term in decision function. Set to 0.0 if
-        ``fit_intercept = False``.
+        `fit_intercept = False`.
 
     alpha_ : float
        Estimated precision of the noise.
@@ -119,22 +119,27 @@ class BayesianRidge(RegressorMixin, LinearModel):
     n_iter_ : int
         The actual number of iterations to reach the stopping criterion.
 
-    X_offset_ : float
-        If `normalize=True`, offset subtracted for centering data to a
-        zero mean.
+    X_offset_ : ndarray of shape (n_features,)
+        If `fit_intercept=True`, offset subtracted for centering data to a
+        zero mean. Set to np.zeros(n_features) otherwise.
 
-    X_scale_ : float
-        If `normalize=True`, parameter used to scale data to a unit
-        standard deviation.
+    X_scale_ : ndarray of shape (n_features,)
+        Set to np.ones(n_features).
 
-    Examples
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
     --------
-    >>> from sklearn import linear_model
-    >>> clf = linear_model.BayesianRidge()
-    >>> clf.fit([[0,0], [1, 1], [2, 2]], [0, 1, 2])
-    BayesianRidge()
-    >>> clf.predict([[1, 1]])
-    array([1.])
+    ARDRegression : Bayesian ARD regression.
 
     Notes
     -----
@@ -153,13 +158,49 @@ class BayesianRidge(RegressorMixin, LinearModel):
 
     M. E. Tipping, Sparse Bayesian Learning and the Relevance Vector Machine,
     Journal of Machine Learning Research, Vol. 1, 2001.
+
+    Examples
+    --------
+    >>> from sklearn import linear_model
+    >>> clf = linear_model.BayesianRidge()
+    >>> clf.fit([[0,0], [1, 1], [2, 2]], [0, 1, 2])
+    BayesianRidge()
+    >>> clf.predict([[1, 1]])
+    array([1.])
     """
-    @_deprecate_positional_args
-    def __init__(self, *, n_iter=300, tol=1.e-3, alpha_1=1.e-6, alpha_2=1.e-6,
-                 lambda_1=1.e-6, lambda_2=1.e-6, alpha_init=None,
-                 lambda_init=None, compute_score=False, fit_intercept=True,
-                 normalize=False, copy_X=True, verbose=False):
-        self.n_iter = n_iter
+
+    _parameter_constraints: dict = {
+        "max_iter": [Interval(Integral, 1, None, closed="left")],
+        "tol": [Interval(Real, 0, None, closed="neither")],
+        "alpha_1": [Interval(Real, 0, None, closed="left")],
+        "alpha_2": [Interval(Real, 0, None, closed="left")],
+        "lambda_1": [Interval(Real, 0, None, closed="left")],
+        "lambda_2": [Interval(Real, 0, None, closed="left")],
+        "alpha_init": [None, Interval(Real, 0, None, closed="left")],
+        "lambda_init": [None, Interval(Real, 0, None, closed="left")],
+        "compute_score": ["boolean"],
+        "fit_intercept": ["boolean"],
+        "copy_X": ["boolean"],
+        "verbose": ["verbose"],
+    }
+
+    def __init__(
+        self,
+        *,
+        max_iter=300,
+        tol=1.0e-3,
+        alpha_1=1.0e-6,
+        alpha_2=1.0e-6,
+        lambda_1=1.0e-6,
+        lambda_2=1.0e-6,
+        alpha_init=None,
+        lambda_init=None,
+        compute_score=False,
+        fit_intercept=True,
+        copy_X=True,
+        verbose=False,
+    ):
+        self.max_iter = max_iter
         self.tol = tol
         self.alpha_1 = alpha_1
         self.alpha_2 = alpha_2
@@ -169,63 +210,78 @@ def __init__(self, *, n_iter=300, tol=1.e-3, alpha_1=1.e-6, alpha_2=1.e-6,
         self.lambda_init = lambda_init
         self.compute_score = compute_score
         self.fit_intercept = fit_intercept
-        self.normalize = normalize
         self.copy_X = copy_X
         self.verbose = verbose
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y, sample_weight=None):
-        """Fit the model
+        """Fit the model.
 
         Parameters
         ----------
         X : ndarray of shape (n_samples, n_features)
-            Training data
+            Training data.
         y : ndarray of shape (n_samples,)
-            Target values. Will be cast to X's dtype if necessary
+            Target values. Will be cast to X's dtype if necessary.
 
         sample_weight : ndarray of shape (n_samples,), default=None
-            Individual weights for each sample
+            Individual weights for each sample.
 
             .. versionadded:: 0.20
                parameter *sample_weight* support to BayesianRidge.
 
         Returns
         -------
-        self : returns an instance of self.
+        self : object
+            Returns the instance itself.
         """
+        X, y = validate_data(
+            self,
+            X,
+            y,
+            dtype=[np.float64, np.float32],
+            force_writeable=True,
+            y_numeric=True,
+        )
+        dtype = X.dtype
+        n_samples, n_features = X.shape
 
-        if self.n_iter < 1:
-            raise ValueError('n_iter should be greater than or equal to 1.'
-                             ' Got {!r}.'.format(self.n_iter))
-
-        X, y = self._validate_data(X, y, dtype=np.float64, y_numeric=True)
-
+        sw_sum = n_samples
+        y_var = y.var()
         if sample_weight is not None:
-            sample_weight = _check_sample_weight(sample_weight, X,
-                                                 dtype=X.dtype)
-
-        X, y, X_offset_, y_offset_, X_scale_ = self._preprocess_data(
-            X, y, self.fit_intercept, self.normalize, self.copy_X,
-            sample_weight=sample_weight)
+            sample_weight = _check_sample_weight(sample_weight, X, dtype=dtype)
+            sw_sum = sample_weight.sum()
+            y_mean = np.average(y, weights=sample_weight)
+            y_var = np.average((y - y_mean) ** 2, weights=sample_weight)
+
+        X, y, X_offset_, y_offset_, X_scale_ = _preprocess_data(
+            X,
+            y,
+            fit_intercept=self.fit_intercept,
+            copy=self.copy_X,
+            sample_weight=sample_weight,
+        )
 
         if sample_weight is not None:
             # Sample weight can be implemented via a simple rescaling.
-            X, y = _rescale_data(X, y, sample_weight)
+            X, y, _ = _rescale_data(X, y, sample_weight)
 
         self.X_offset_ = X_offset_
         self.X_scale_ = X_scale_
-        n_samples, n_features = X.shape
 
         # Initialization of the values of the parameters
         eps = np.finfo(np.float64).eps
-        # Add `eps` in the denominator to omit division by zero if `np.var(y)`
-        # is zero
+        # Add `eps` in the denominator to omit division by zero
         alpha_ = self.alpha_init
         lambda_ = self.lambda_init
         if alpha_ is None:
-            alpha_ = 1. / (np.var(y) + eps)
+            alpha_ = 1.0 / (y_var + eps)
         if lambda_ is None:
-            lambda_ = 1.
+            lambda_ = 1.0
+
+        # Avoid unintended type promotion to float64 with numpy 2
+        alpha_ = np.asarray(alpha_, dtype=dtype)
+        lambda_ = np.asarray(lambda_, dtype=dtype)
 
         verbose = self.verbose
         lambda_1 = self.lambda_1
@@ -237,32 +293,45 @@ def fit(self, X, y, sample_weight=None):
         coef_old_ = None
 
         XT_y = np.dot(X.T, y)
-        U, S, Vh = linalg.svd(X, full_matrices=False)
-        eigen_vals_ = S ** 2
+        # Let M, N = n_samples, n_features and K = min(M, N).
+        # The posterior covariance matrix needs Vh_full: (N, N).
+        # The full SVD is only required when n_samples < n_features.
+        # When n_samples < n_features, K=M and full_matrices=True
+        # U: (M, M), S: M, Vh_full: (N, N), Vh: (M, N)
+        # When n_samples > n_features, K=N and full_matrices=False
+        # U: (M, N), S: N, Vh_full: (N, N), Vh: (N, N)
+        U, S, Vh_full = linalg.svd(X, full_matrices=(n_samples < n_features))
+        K = len(S)
+        eigen_vals_ = S**2
+        eigen_vals_full = np.zeros(n_features, dtype=dtype)
+        eigen_vals_full[0:K] = eigen_vals_
+        Vh = Vh_full[0:K, :]
 
         # Convergence loop of the bayesian ridge regression
-        for iter_ in range(self.n_iter):
-
+        for iter_ in range(self.max_iter):
             # update posterior mean coef_ based on alpha_ and lambda_ and
-            # compute corresponding rmse
-            coef_, rmse_ = self._update_coef_(X, y, n_samples, n_features,
-                                              XT_y, U, Vh, eigen_vals_,
-                                              alpha_, lambda_)
+            # compute corresponding sse (sum of squared errors)
+            coef_, sse_ = self._update_coef_(
+                X, y, n_samples, n_features, XT_y, U, Vh, eigen_vals_, alpha_, lambda_
+            )
             if self.compute_score:
                 # compute the log marginal likelihood
-                s = self._log_marginal_likelihood(n_samples, n_features,
-                                                  eigen_vals_,
-                                                  alpha_, lambda_,
-                                                  coef_, rmse_)
+                s = self._log_marginal_likelihood(
+                    n_samples,
+                    n_features,
+                    sw_sum,
+                    eigen_vals_,
+                    alpha_,
+                    lambda_,
+                    coef_,
+                    sse_,
+                )
                 self.scores_.append(s)
 
             # Update alpha and lambda according to (MacKay, 1992)
-            gamma_ = np.sum((alpha_ * eigen_vals_) /
-                            (lambda_ + alpha_ * eigen_vals_))
-            lambda_ = ((gamma_ + 2 * lambda_1) /
-                       (np.sum(coef_ ** 2) + 2 * lambda_2))
-            alpha_ = ((n_samples - gamma_ + 2 * alpha_1) /
-                      (rmse_ + 2 * alpha_2))
+            gamma_ = np.sum((alpha_ * eigen_vals_) / (lambda_ + alpha_ * eigen_vals_))
+            lambda_ = (gamma_ + 2 * lambda_1) / (np.sum(coef_**2) + 2 * lambda_2)
+            alpha_ = (sw_sum - gamma_ + 2 * alpha_1) / (sse_ + 2 * alpha_2)
 
             # Check for convergence
             if iter_ != 0 and np.sum(np.abs(coef_old_ - coef_)) < self.tol:
@@ -277,23 +346,28 @@ def fit(self, X, y, sample_weight=None):
         # log marginal likelihood and posterior covariance
         self.alpha_ = alpha_
         self.lambda_ = lambda_
-        self.coef_, rmse_ = self._update_coef_(X, y, n_samples, n_features,
-                                               XT_y, U, Vh, eigen_vals_,
-                                               alpha_, lambda_)
+        self.coef_, sse_ = self._update_coef_(
+            X, y, n_samples, n_features, XT_y, U, Vh, eigen_vals_, alpha_, lambda_
+        )
         if self.compute_score:
             # compute the log marginal likelihood
-            s = self._log_marginal_likelihood(n_samples, n_features,
-                                              eigen_vals_,
-                                              alpha_, lambda_,
-                                              coef_, rmse_)
+            s = self._log_marginal_likelihood(
+                n_samples,
+                n_features,
+                sw_sum,
+                eigen_vals_,
+                alpha_,
+                lambda_,
+                coef_,
+                sse_,
+            )
             self.scores_.append(s)
             self.scores_ = np.array(self.scores_)
 
-        # posterior covariance is given by 1/alpha_ * scaled_sigma_
-        scaled_sigma_ = np.dot(Vh.T,
-                               Vh / (eigen_vals_ +
-                                     lambda_ / alpha_)[:, np.newaxis])
-        self.sigma_ = (1. / alpha_) * scaled_sigma_
+        # posterior covariance
+        self.sigma_ = np.dot(
+            Vh_full.T, Vh_full / (alpha_ * eigen_vals_full + lambda_)[:, np.newaxis]
+        )
 
         self._set_intercept(X_offset_, y_offset_, X_scale_)
 
@@ -322,18 +396,17 @@ def predict(self, X, return_std=False):
             Standard deviation of predictive distribution of query points.
         """
         y_mean = self._decision_function(X)
-        if return_std is False:
+        if not return_std:
             return y_mean
         else:
-            if self.normalize:
-                X = (X - self.X_offset_) / self.X_scale_
             sigmas_squared_data = (np.dot(X, self.sigma_) * X).sum(axis=1)
-            y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))
+            y_std = np.sqrt(sigmas_squared_data + (1.0 / self.alpha_))
             return y_mean, y_std
 
-    def _update_coef_(self, X, y, n_samples, n_features, XT_y, U, Vh,
-                      eigen_vals_, alpha_, lambda_):
-        """Update posterior mean and compute corresponding rmse.
+    def _update_coef_(
+        self, X, y, n_samples, n_features, XT_y, U, Vh, eigen_vals_, alpha_, lambda_
+    ):
+        """Update posterior mean and compute corresponding sse (sum of squared errors).
 
         Posterior mean is given by coef_ = scaled_sigma_ * X.T * y where
         scaled_sigma_ = (lambda_/alpha_ * np.eye(n_features)
@@ -341,22 +414,23 @@ def _update_coef_(self, X, y, n_samples, n_features, XT_y, U, Vh,
         """
 
         if n_samples > n_features:
-            coef_ = np.linalg.multi_dot([Vh.T,
-                                         Vh / (eigen_vals_ + lambda_ /
-                                               alpha_)[:, np.newaxis],
-                                         XT_y])
+            coef_ = np.linalg.multi_dot(
+                [Vh.T, Vh / (eigen_vals_ + lambda_ / alpha_)[:, np.newaxis], XT_y]
+            )
         else:
-            coef_ = np.linalg.multi_dot([X.T,
-                                         U / (eigen_vals_ + lambda_ /
-                                              alpha_)[None, :],
-                                         U.T, y])
+            coef_ = np.linalg.multi_dot(
+                [X.T, U / (eigen_vals_ + lambda_ / alpha_)[None, :], U.T, y]
+            )
 
-        rmse_ = np.sum((y - np.dot(X, coef_)) ** 2)
+        # Note: we do not need to explicitly use the weights in this sum because
+        # y and X were preprocessed by _rescale_data to handle the weights.
+        sse_ = np.sum((y - np.dot(X, coef_)) ** 2)
 
-        return coef_, rmse_
+        return coef_, sse_
 
-    def _log_marginal_likelihood(self, n_samples, n_features, eigen_vals,
-                                 alpha_, lambda_, coef, rmse):
+    def _log_marginal_likelihood(
+        self, n_samples, n_features, sw_sum, eigen_vals, alpha_, lambda_, coef, sse
+    ):
         """Log marginal likelihood."""
         alpha_1 = self.alpha_1
         alpha_2 = self.alpha_2
@@ -367,21 +441,22 @@ def _log_marginal_likelihood(self, n_samples, n_features, eigen_vals,
         # posterior covariance is given by
         # sigma = (lambda_ * np.eye(n_features) + alpha_ * np.dot(X.T, X))^-1
         if n_samples > n_features:
-            logdet_sigma = - np.sum(np.log(lambda_ + alpha_ * eigen_vals))
+            logdet_sigma = -np.sum(np.log(lambda_ + alpha_ * eigen_vals))
         else:
-            logdet_sigma = np.full(n_features, lambda_,
-                                   dtype=np.array(lambda_).dtype)
+            logdet_sigma = np.full(n_features, lambda_, dtype=np.array(lambda_).dtype)
             logdet_sigma[:n_samples] += alpha_ * eigen_vals
-            logdet_sigma = - np.sum(np.log(logdet_sigma))
+            logdet_sigma = -np.sum(np.log(logdet_sigma))
 
         score = lambda_1 * log(lambda_) - lambda_2 * lambda_
         score += alpha_1 * log(alpha_) - alpha_2 * alpha_
-        score += 0.5 * (n_features * log(lambda_) +
-                        n_samples * log(alpha_) -
-                        alpha_ * rmse -
-                        lambda_ * np.sum(coef ** 2) +
-                        logdet_sigma -
-                        n_samples * log(2 * np.pi))
+        score += 0.5 * (
+            n_features * log(lambda_)
+            + sw_sum * log(alpha_)
+            - alpha_ * sse
+            - lambda_ * np.sum(coef**2)
+            + logdet_sigma
+            - sw_sum * log(2 * np.pi)
+        )
 
         return score
 
@@ -403,9 +478,11 @@ class ARDRegression(RegressorMixin, LinearModel):
 
     Parameters
     ----------
-    n_iter : int, default=300
+    max_iter : int, default=300
         Maximum number of iterations.
 
+        .. versionchanged:: 1.3
+
     tol : float, default=1e-3
         Stop the algorithm if w has converged.
 
@@ -429,22 +506,14 @@ class ARDRegression(RegressorMixin, LinearModel):
         If True, compute the objective function at each step of the model.
 
     threshold_lambda : float, default=10 000
-        threshold for removing (pruning) weights with high precision from
+        Threshold for removing (pruning) weights with high precision from
         the computation.
 
     fit_intercept : bool, default=True
-        whether to calculate the intercept for this model. If set
+        Whether to calculate the intercept for this model. If set
         to false, no intercept will be used in calculations
         (i.e. data is expected to be centered).
 
-    normalize : bool, default=False
-        This parameter is ignored when ``fit_intercept`` is set to False.
-        If True, the regressors X will be normalized before regression by
-        subtracting the mean and dividing by the l2-norm.
-        If you wish to standardize, please use
-        :class:`~sklearn.preprocessing.StandardScaler` before calling ``fit``
-        on an estimator with ``normalize=False``.
-
     copy_X : bool, default=True
         If True, X will be copied; else, it may be overwritten.
 
@@ -468,26 +537,36 @@ class ARDRegression(RegressorMixin, LinearModel):
     scores_ : float
         if computed, value of the objective function (to be maximized)
 
+    n_iter_ : int
+        The actual number of iterations to reach the stopping criterion.
+
+        .. versionadded:: 1.3
+
     intercept_ : float
         Independent term in decision function. Set to 0.0 if
         ``fit_intercept = False``.
 
     X_offset_ : float
-        If `normalize=True`, offset subtracted for centering data to a
-        zero mean.
+        If `fit_intercept=True`, offset subtracted for centering data to a
+        zero mean. Set to np.zeros(n_features) otherwise.
 
     X_scale_ : float
-        If `normalize=True`, parameter used to scale data to a unit
-        standard deviation.
+        Set to np.ones(n_features).
 
-    Examples
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
     --------
-    >>> from sklearn import linear_model
-    >>> clf = linear_model.ARDRegression()
-    >>> clf.fit([[0,0], [1, 1], [2, 2]], [0, 1, 2])
-    ARDRegression()
-    >>> clf.predict([[1, 1]])
-    array([1.])
+    BayesianRidge : Bayesian ridge regression.
 
     Notes
     -----
@@ -506,16 +585,49 @@ class ARDRegression(RegressorMixin, LinearModel):
     ARD is a little different than the slide: only dimensions/features for
     which ``self.lambda_ < self.threshold_lambda`` are kept and the rest are
     discarded.
+
+    Examples
+    --------
+    >>> from sklearn import linear_model
+    >>> clf = linear_model.ARDRegression()
+    >>> clf.fit([[0,0], [1, 1], [2, 2]], [0, 1, 2])
+    ARDRegression()
+    >>> clf.predict([[1, 1]])
+    array([1.])
     """
-    @_deprecate_positional_args
-    def __init__(self, *, n_iter=300, tol=1.e-3, alpha_1=1.e-6, alpha_2=1.e-6,
-                 lambda_1=1.e-6, lambda_2=1.e-6, compute_score=False,
-                 threshold_lambda=1.e+4, fit_intercept=True, normalize=False,
-                 copy_X=True, verbose=False):
-        self.n_iter = n_iter
+
+    _parameter_constraints: dict = {
+        "max_iter": [Interval(Integral, 1, None, closed="left")],
+        "tol": [Interval(Real, 0, None, closed="left")],
+        "alpha_1": [Interval(Real, 0, None, closed="left")],
+        "alpha_2": [Interval(Real, 0, None, closed="left")],
+        "lambda_1": [Interval(Real, 0, None, closed="left")],
+        "lambda_2": [Interval(Real, 0, None, closed="left")],
+        "compute_score": ["boolean"],
+        "threshold_lambda": [Interval(Real, 0, None, closed="left")],
+        "fit_intercept": ["boolean"],
+        "copy_X": ["boolean"],
+        "verbose": ["verbose"],
+    }
+
+    def __init__(
+        self,
+        *,
+        max_iter=300,
+        tol=1.0e-3,
+        alpha_1=1.0e-6,
+        alpha_2=1.0e-6,
+        lambda_1=1.0e-6,
+        lambda_2=1.0e-6,
+        compute_score=False,
+        threshold_lambda=1.0e4,
+        fit_intercept=True,
+        copy_X=True,
+        verbose=False,
+    ):
+        self.max_iter = max_iter
         self.tol = tol
         self.fit_intercept = fit_intercept
-        self.normalize = normalize
         self.alpha_1 = alpha_1
         self.alpha_2 = alpha_2
         self.lambda_1 = lambda_1
@@ -525,32 +637,42 @@ def __init__(self, *, n_iter=300, tol=1.e-3, alpha_1=1.e-6, alpha_2=1.e-6,
         self.copy_X = copy_X
         self.verbose = verbose
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y):
-        """Fit the ARDRegression model according to the given training data
-        and parameters.
+        """Fit the model according to the given training data and parameters.
 
         Iterative procedure to maximize the evidence
 
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
-            Training vector, where n_samples in the number of samples and
-            n_features is the number of features.
+            Training vector, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
         y : array-like of shape (n_samples,)
-            Target values (integers). Will be cast to X's dtype if necessary
+            Target values (integers). Will be cast to X's dtype if necessary.
 
         Returns
         -------
-        self : returns an instance of self.
+        self : object
+            Fitted estimator.
         """
-        X, y = self._validate_data(X, y, dtype=np.float64, y_numeric=True,
-                                   ensure_min_samples=2)
+        X, y = validate_data(
+            self,
+            X,
+            y,
+            dtype=[np.float64, np.float32],
+            force_writeable=True,
+            y_numeric=True,
+            ensure_min_samples=2,
+        )
+        dtype = X.dtype
 
         n_samples, n_features = X.shape
-        coef_ = np.zeros(n_features)
+        coef_ = np.zeros(n_features, dtype=dtype)
 
-        X, y, X_offset_, y_offset_, X_scale_ = self._preprocess_data(
-            X, y, self.fit_intercept, self.normalize, self.copy_X)
+        X, y, X_offset_, y_offset_, X_scale_ = _preprocess_data(
+            X, y, fit_intercept=self.fit_intercept, copy=self.copy_X
+        )
 
         self.X_offset_ = X_offset_
         self.X_scale_ = X_scale_
@@ -567,33 +689,37 @@ def fit(self, X, y):
         # Initialization of the values of the parameters
         eps = np.finfo(np.float64).eps
         # Add `eps` in the denominator to omit division by zero if `np.var(y)`
-        # is zero
-        alpha_ = 1. / (np.var(y) + eps)
-        lambda_ = np.ones(n_features)
+        # is zero.
+        # Explicitly set dtype to avoid unintended type promotion with numpy 2.
+        alpha_ = np.asarray(1.0 / (np.var(y) + eps), dtype=dtype)
+        lambda_ = np.ones(n_features, dtype=dtype)
 
         self.scores_ = list()
         coef_old_ = None
 
         def update_coeff(X, y, coef_, alpha_, keep_lambda, sigma_):
-            coef_[keep_lambda] = alpha_ * np.linalg.multi_dot([
-                sigma_, X[:, keep_lambda].T, y])
+            coef_[keep_lambda] = alpha_ * np.linalg.multi_dot(
+                [sigma_, X[:, keep_lambda].T, y]
+            )
             return coef_
 
-        update_sigma = (self._update_sigma if n_samples >= n_features
-                        else self._update_sigma_woodbury)
+        update_sigma = (
+            self._update_sigma
+            if n_samples >= n_features
+            else self._update_sigma_woodbury
+        )
         # Iterative procedure of ARDRegression
-        for iter_ in range(self.n_iter):
+        for iter_ in range(self.max_iter):
             sigma_ = update_sigma(X, alpha_, lambda_, keep_lambda)
             coef_ = update_coeff(X, y, coef_, alpha_, keep_lambda, sigma_)
 
             # Update alpha and lambda
-            rmse_ = np.sum((y - np.dot(X, coef_)) ** 2)
-            gamma_ = 1. - lambda_[keep_lambda] * np.diag(sigma_)
-            lambda_[keep_lambda] = ((gamma_ + 2. * lambda_1) /
-                                    ((coef_[keep_lambda]) ** 2 +
-                                     2. * lambda_2))
-            alpha_ = ((n_samples - gamma_.sum() + 2. * alpha_1) /
-                      (rmse_ + 2. * alpha_2))
+            sse_ = np.sum((y - np.dot(X, coef_)) ** 2)
+            gamma_ = 1.0 - lambda_[keep_lambda] * np.diag(sigma_)
+            lambda_[keep_lambda] = (gamma_ + 2.0 * lambda_1) / (
+                (coef_[keep_lambda]) ** 2 + 2.0 * lambda_2
+            )
+            alpha_ = (n_samples - gamma_.sum() + 2.0 * alpha_1) / (sse_ + 2.0 * alpha_2)
 
             # Prune the weights with a precision over a threshold
             keep_lambda = lambda_ < self.threshold_lambda
@@ -603,9 +729,12 @@ def update_coeff(X, y, coef_, alpha_, keep_lambda, sigma_):
             if self.compute_score:
                 s = (lambda_1 * np.log(lambda_) - lambda_2 * lambda_).sum()
                 s += alpha_1 * log(alpha_) - alpha_2 * alpha_
-                s += 0.5 * (fast_logdet(sigma_) + n_samples * log(alpha_) +
-                            np.sum(np.log(lambda_)))
-                s -= 0.5 * (alpha_ * rmse_ + (lambda_ * coef_ ** 2).sum())
+                s += 0.5 * (
+                    fast_logdet(sigma_)
+                    + n_samples * log(alpha_)
+                    + np.sum(np.log(lambda_))
+                )
+                s -= 0.5 * (alpha_ * sse_ + (lambda_ * coef_**2).sum())
                 self.scores_.append(s)
 
             # Check for convergence
@@ -618,6 +747,8 @@ def update_coeff(X, y, coef_, alpha_, keep_lambda, sigma_):
             if not keep_lambda.any():
                 break
 
+        self.n_iter_ = iter_ + 1
+
         if keep_lambda.any():
             # update sigma and mu using updated params from the last iteration
             sigma_ = update_sigma(X, alpha_, lambda_, keep_lambda)
@@ -642,11 +773,12 @@ def _update_sigma_woodbury(self, X, alpha_, lambda_, keep_lambda):
         X_keep = X[:, keep_lambda]
         inv_lambda = 1 / lambda_[keep_lambda].reshape(1, -1)
         sigma_ = pinvh(
-            np.eye(n_samples) / alpha_ + np.dot(X_keep * inv_lambda, X_keep.T)
+            np.eye(n_samples, dtype=X.dtype) / alpha_
+            + np.dot(X_keep * inv_lambda, X_keep.T)
         )
         sigma_ = np.dot(sigma_, X_keep * inv_lambda)
-        sigma_ = - np.dot(inv_lambda.reshape(-1, 1) * X_keep.T, sigma_)
-        sigma_[np.diag_indices(sigma_.shape[1])] += 1. / lambda_[keep_lambda]
+        sigma_ = -np.dot(inv_lambda.reshape(-1, 1) * X_keep.T, sigma_)
+        sigma_[np.diag_indices(sigma_.shape[1])] += 1.0 / lambda_[keep_lambda]
         return sigma_
 
     def _update_sigma(self, X, alpha_, lambda_, keep_lambda):
@@ -655,7 +787,7 @@ def _update_sigma(self, X, alpha_, lambda_, keep_lambda):
         # invert a matrix of shape (n_features, n_features)
         X_keep = X[:, keep_lambda]
         gram = np.dot(X_keep.T, X_keep)
-        eye = np.eye(gram.shape[0])
+        eye = np.eye(gram.shape[0], dtype=X.dtype)
         sigma_inv = lambda_[keep_lambda] * eye + alpha_ * gram
         sigma_ = pinvh(sigma_inv)
         return sigma_
@@ -686,9 +818,8 @@ def predict(self, X, return_std=False):
         if return_std is False:
             return y_mean
         else:
-            if self.normalize:
-                X = (X - self.X_offset_) / self.X_scale_
-            X = X[:, self.lambda_ < self.threshold_lambda]
+            col_index = self.lambda_ < self.threshold_lambda
+            X = _safe_indexing(X, indices=col_index, axis=1)
             sigmas_squared_data = (np.dot(X, self.sigma_) * X).sum(axis=1)
-            y_std = np.sqrt(sigmas_squared_data + (1. / self.alpha_))
+            y_std = np.sqrt(sigmas_squared_data + (1.0 / self.alpha_))
             return y_mean, y_std
diff --git a/sklearn/linear_model/_cd_fast.pyx b/sklearn/linear_model/_cd_fast.pyx
index 84e4b4a49df01..ce598ebb011d2 100644
--- a/sklearn/linear_model/_cd_fast.pyx
+++ b/sklearn/linear_model/_cd_fast.pyx
@@ -1,35 +1,20 @@
-# Author: Alexandre Gramfort 
-#         Fabian Pedregosa 
-#         Olivier Grisel 
-#         Alexis Mignon 
-#         Manoj Kumar 
-#
-# License: BSD 3 clause
-#
-# cython: boundscheck=False, wraparound=False, cdivision=True
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 from libc.math cimport fabs
-cimport numpy as np
 import numpy as np
-import numpy.linalg as linalg
 
-cimport cython
-from cpython cimport bool
 from cython cimport floating
 import warnings
 from ..exceptions import ConvergenceWarning
 
-from ..utils._cython_blas cimport (_axpy, _dot, _asum, _ger, _gemv, _nrm2,
-                                   _copy, _scal)
-from ..utils._cython_blas cimport RowMajor, ColMajor, Trans, NoTrans
-
-
+from ..utils._cython_blas cimport (
+    _axpy, _dot, _asum, _gemv, _nrm2, _copy, _scal
+)
+from ..utils._cython_blas cimport ColMajor, Trans, NoTrans
+from ..utils._typedefs cimport uint32_t
 from ..utils._random cimport our_rand_r
 
-ctypedef np.float64_t DOUBLE
-ctypedef np.uint32_t UINT32_t
-
-np.import_array()
 
 # The following two functions are shamelessly copied from the tree code.
 
@@ -37,21 +22,23 @@ cdef enum:
     # Max value for our rand_r replacement (near the bottom).
     # We don't use RAND_MAX because it's different across platforms and
     # particularly tiny on Windows/MSVC.
-    RAND_R_MAX = 0x7FFFFFFF
+    # It corresponds to the maximum representable value for
+    # 32-bit signed integers (i.e. 2^31 - 1).
+    RAND_R_MAX = 2147483647
 
 
-cdef inline UINT32_t rand_int(UINT32_t end, UINT32_t* random_state) nogil:
+cdef inline uint32_t rand_int(uint32_t end, uint32_t* random_state) noexcept nogil:
     """Generate a random integer in [0; end)."""
     return our_rand_r(random_state) % end
 
 
-cdef inline floating fmax(floating x, floating y) nogil:
+cdef inline floating fmax(floating x, floating y) noexcept nogil:
     if x > y:
         return x
     return y
 
 
-cdef inline floating fsign(floating f) nogil:
+cdef inline floating fsign(floating f) noexcept nogil:
     if f == 0:
         return 0
     elif f > 0:
@@ -60,7 +47,7 @@ cdef inline floating fsign(floating f) nogil:
         return -1.0
 
 
-cdef floating abs_max(int n, floating* a) nogil:
+cdef floating abs_max(int n, const floating* a) noexcept nogil:
     """np.max(np.abs(a))"""
     cdef int i
     cdef floating m = fabs(a[0])
@@ -72,7 +59,7 @@ cdef floating abs_max(int n, floating* a) nogil:
     return m
 
 
-cdef floating max(int n, floating* a) nogil:
+cdef floating max(int n, floating* a) noexcept nogil:
     """np.max(a)"""
     cdef int i
     cdef floating m = a[0]
@@ -84,7 +71,7 @@ cdef floating max(int n, floating* a) nogil:
     return m
 
 
-cdef floating diff_abs_max(int n, floating* a, floating* b) nogil:
+cdef floating diff_abs_max(int n, const floating* a, floating* b) noexcept nogil:
     """np.max(np.abs(a - b))"""
     cdef int i
     cdef floating m = fabs(a[0] - b[0])
@@ -96,12 +83,33 @@ cdef floating diff_abs_max(int n, floating* a, floating* b) nogil:
     return m
 
 
-def enet_coordinate_descent(floating[::1] w,
-                            floating alpha, floating beta,
-                            floating[::1, :] X,
-                            floating[::1] y,
-                            int max_iter, floating tol,
-                            object rng, bint random=0, bint positive=0):
+message_conv = (
+    "Objective did not converge. You might want to increase "
+    "the number of iterations, check the scale of the "
+    "features or consider increasing regularisation."
+)
+
+
+message_ridge = (
+    "Linear regression models with a zero l1 penalization "
+    "strength are more efficiently fitted using one of the "
+    "solvers implemented in "
+    "sklearn.linear_model.Ridge/RidgeCV instead."
+)
+
+
+def enet_coordinate_descent(
+    floating[::1] w,
+    floating alpha,
+    floating beta,
+    const floating[::1, :] X,
+    const floating[::1] y,
+    unsigned int max_iter,
+    floating tol,
+    object rng,
+    bint random=0,
+    bint positive=0
+):
     """Cython version of the coordinate descent algorithm
         for Elastic-Net regression
 
@@ -109,6 +117,16 @@ def enet_coordinate_descent(floating[::1] w,
 
         (1/2) * norm(y - X w, 2)^2 + alpha norm(w, 1) + (beta/2) norm(w, 2)^2
 
+    Returns
+    -------
+    w : ndarray of shape (n_features,)
+        ElasticNet coefficients.
+    gap : float
+        Achieved dual gap.
+    tol : float
+        Equals input `tol` times `np.dot(y, y)`. The tolerance used for the dual gap.
+    n_iter : int
+        Number of coordinate descent iterations.
     """
 
     if floating is float:
@@ -138,14 +156,13 @@ def enet_coordinate_descent(floating[::1] w,
     cdef floating R_norm2
     cdef floating w_norm2
     cdef floating l1_norm
-    cdef floating const
+    cdef floating const_
     cdef floating A_norm2
     cdef unsigned int ii
-    cdef unsigned int i
     cdef unsigned int n_iter = 0
     cdef unsigned int f_iter
-    cdef UINT32_t rand_r_state_seed = rng.randint(0, RAND_R_MAX)
-    cdef UINT32_t* rand_r_state = &rand_r_state_seed
+    cdef uint32_t rand_r_state_seed = rng.randint(0, RAND_R_MAX)
+    cdef uint32_t* rand_r_state = &rand_r_state_seed
 
     if alpha == 0 and beta == 0:
         warnings.warn("Coordinate descent with no regularization may lead to "
@@ -197,9 +214,11 @@ def enet_coordinate_descent(floating[::1] w,
 
                 w_max = fmax(w_max, fabs(w[ii]))
 
-            if (w_max == 0.0 or
-                d_w_max / w_max < d_w_tol or
-                n_iter == max_iter - 1):
+            if (
+                w_max == 0.0
+                or d_w_max / w_max < d_w_tol
+                or n_iter == max_iter - 1
+            ):
                 # the biggest coordinate update of this iteration was smaller
                 # than the tolerance: check the duality gap as ultimate
                 # stopping criterion
@@ -223,19 +242,18 @@ def enet_coordinate_descent(floating[::1] w,
                 w_norm2 = _dot(n_features, &w[0], 1, &w[0], 1)
 
                 if (dual_norm_XtA > alpha):
-                    const = alpha / dual_norm_XtA
-                    A_norm2 = R_norm2 * (const ** 2)
+                    const_ = alpha / dual_norm_XtA
+                    A_norm2 = R_norm2 * (const_ ** 2)
                     gap = 0.5 * (R_norm2 + A_norm2)
                 else:
-                    const = 1.0
+                    const_ = 1.0
                     gap = R_norm2
 
                 l1_norm = _asum(n_features, &w[0], 1)
 
-                # np.dot(R.T, y)
                 gap += (alpha * l1_norm
-                        - const * _dot(n_samples, &R[0], 1, &y[0], 1)
-                        + 0.5 * beta * (1 + const ** 2) * (w_norm2))
+                        - const_ * _dot(n_samples, &R[0], 1, &y[0], 1)  # np.dot(R.T, y)
+                        + 0.5 * beta * (1 + const_ ** 2) * (w_norm2))
 
                 if gap < tol:
                     # return if we reached desired tolerance
@@ -244,78 +262,120 @@ def enet_coordinate_descent(floating[::1] w,
         else:
             # for/else, runs if for doesn't end with a `break`
             with gil:
-                warnings.warn("Objective did not converge. You might want to "
-                              "increase the number of iterations. Duality "
-                              "gap: {}, tolerance: {}".format(gap, tol),
-                              ConvergenceWarning)
-
-    return w, gap, tol, n_iter + 1
-
-
-def sparse_enet_coordinate_descent(floating [::1] w,
-                            floating alpha, floating beta,
-                            np.ndarray[floating, ndim=1, mode='c'] X_data,
-                            np.ndarray[int, ndim=1, mode='c'] X_indices,
-                            np.ndarray[int, ndim=1, mode='c'] X_indptr,
-                            np.ndarray[floating, ndim=1] y,
-                            floating[:] X_mean, int max_iter,
-                            floating tol, object rng, bint random=0,
-                            bint positive=0):
-    """Cython version of the coordinate descent algorithm for Elastic-Net
+                message = (
+                    message_conv +
+                    f" Duality gap: {gap:.3e}, tolerance: {tol:.3e}"
+                )
+                if alpha < np.finfo(np.float64).eps:
+                    message += "\n" + message_ridge
+                warnings.warn(message, ConvergenceWarning)
 
-    We minimize:
+    return np.asarray(w), gap, tol, n_iter + 1
 
-        (1/2) * norm(y - X w, 2)^2 + alpha norm(w, 1) + (beta/2) * norm(w, 2)^2
 
-    """
+def sparse_enet_coordinate_descent(
+    floating[::1] w,
+    floating alpha,
+    floating beta,
+    const floating[::1] X_data,
+    const int[::1] X_indices,
+    const int[::1] X_indptr,
+    const floating[::1] y,
+    const floating[::1] sample_weight,
+    const floating[::1] X_mean,
+    unsigned int max_iter,
+    floating tol,
+    object rng,
+    bint random=0,
+    bint positive=0,
+):
+    """Cython version of the coordinate descent algorithm for Elastic-Net
 
-    # get the data information into easy vars
-    cdef unsigned int n_samples = y.shape[0]
-    cdef unsigned int n_features = w.shape[0]
+    We minimize:
 
-    # compute norms of the columns of X
-    cdef unsigned int ii
-    cdef floating[:] norm_cols_X
+        1/2 * norm(y - Z w, 2)^2 + alpha * norm(w, 1) + (beta/2) * norm(w, 2)^2
 
-    cdef unsigned int startptr = X_indptr[0]
-    cdef unsigned int endptr
+    where Z = X - X_mean.
+    With sample weights sw, this becomes
 
-    # initial value of the residuals
-    cdef floating[:] R = y.copy()
+        1/2 * sum(sw * (y - Z w)^2, axis=0) + alpha * norm(w, 1)
+        + (beta/2) * norm(w, 2)^2
 
-    cdef floating[:] X_T_R
-    cdef floating[:] XtA
+    and X_mean is the weighted average of X (per column).
+
+    Returns
+    -------
+    w : ndarray of shape (n_features,)
+        ElasticNet coefficients.
+    gap : float
+        Achieved dual gap.
+    tol : float
+        Equals input `tol` times `np.dot(y, y)`. The tolerance used for the dual gap.
+    n_iter : int
+        Number of coordinate descent iterations.
+    """
+    # Notes for sample_weight:
+    # For dense X, one centers X and y and then rescales them by sqrt(sample_weight).
+    # Here, for sparse X, we get the sample_weight averaged center X_mean. We take care
+    # that every calculation results as if we had rescaled y and X (and therefore also
+    # X_mean) by sqrt(sample_weight) without actually calculating the square root.
+    # We work with:
+    #     yw = sample_weight * y
+    #     R = sample_weight * residual
+    #     norm_cols_X = np.sum(sample_weight * (X - X_mean)**2, axis=0)
 
     if floating is float:
         dtype = np.float32
     else:
         dtype = np.float64
 
-    norm_cols_X = np.zeros(n_features, dtype=dtype)
-    X_T_R = np.zeros(n_features, dtype=dtype)
-    XtA = np.zeros(n_features, dtype=dtype)
+    # get the data information into easy vars
+    cdef unsigned int n_samples = y.shape[0]
+    cdef unsigned int n_features = w.shape[0]
+
+    # compute norms of the columns of X
+    cdef floating[:] norm_cols_X = np.zeros(n_features, dtype=dtype)
+
+    # initial value of the residuals
+    # R = y - Zw, weighted version R = sample_weight * (y - Zw)
+    cdef floating[::1] R
+    cdef floating[::1] XtA = np.empty(n_features, dtype=dtype)
+    cdef const floating[::1] yw
 
     cdef floating tmp
     cdef floating w_ii
     cdef floating d_w_max
     cdef floating w_max
     cdef floating d_w_ii
+    cdef floating gap = tol + 1.0
+    cdef floating d_w_tol = tol
+    cdef floating dual_norm_XtA
     cdef floating X_mean_ii
     cdef floating R_sum = 0.0
     cdef floating R_norm2
     cdef floating w_norm2
-    cdef floating A_norm2
     cdef floating l1_norm
+    cdef floating const_
+    cdef floating A_norm2
     cdef floating normalize_sum
-    cdef floating gap = tol + 1.0
-    cdef floating d_w_tol = tol
-    cdef floating dual_norm_XtA
+    cdef unsigned int ii
     cdef unsigned int jj
     cdef unsigned int n_iter = 0
     cdef unsigned int f_iter
-    cdef UINT32_t rand_r_state_seed = rng.randint(0, RAND_R_MAX)
-    cdef UINT32_t* rand_r_state = &rand_r_state_seed
+    cdef unsigned int startptr = X_indptr[0]
+    cdef unsigned int endptr
+    cdef uint32_t rand_r_state_seed = rng.randint(0, RAND_R_MAX)
+    cdef uint32_t* rand_r_state = &rand_r_state_seed
     cdef bint center = False
+    cdef bint no_sample_weights = sample_weight is None
+    cdef int kk
+
+    if no_sample_weights:
+        yw = y
+        R = y.copy()
+    else:
+        yw = np.multiply(sample_weight, y)
+        R = yw.copy()
 
     with nogil:
         # center = (X_mean != 0).any()
@@ -324,25 +384,46 @@ def sparse_enet_coordinate_descent(floating [::1] w,
                 center = True
                 break
 
+        # R = y - np.dot(X, w)
         for ii in range(n_features):
             X_mean_ii = X_mean[ii]
             endptr = X_indptr[ii + 1]
             normalize_sum = 0.0
             w_ii = w[ii]
 
-            for jj in range(startptr, endptr):
-                normalize_sum += (X_data[jj] - X_mean_ii) ** 2
-                R[X_indices[jj]] -= X_data[jj] * w_ii
-            norm_cols_X[ii] = normalize_sum + \
-                (n_samples - endptr + startptr) * X_mean_ii ** 2
-
-            if center:
-                for jj in range(n_samples):
-                    R[jj] += X_mean_ii * w_ii
+            if no_sample_weights:
+                for jj in range(startptr, endptr):
+                    normalize_sum += (X_data[jj] - X_mean_ii) ** 2
+                    R[X_indices[jj]] -= X_data[jj] * w_ii
+                norm_cols_X[ii] = normalize_sum + \
+                    (n_samples - endptr + startptr) * X_mean_ii ** 2
+                if center:
+                    for jj in range(n_samples):
+                        R[jj] += X_mean_ii * w_ii
+                        R_sum += R[jj]
+            else:
+                # R = sw * (y - np.dot(X, w))
+                for jj in range(startptr, endptr):
+                    tmp = sample_weight[X_indices[jj]]
+                    # second term will be subtracted by loop over range(n_samples)
+                    normalize_sum += (tmp * (X_data[jj] - X_mean_ii) ** 2
+                                      - tmp * X_mean_ii ** 2)
+                    R[X_indices[jj]] -= tmp * X_data[jj] * w_ii
+                if center:
+                    for jj in range(n_samples):
+                        normalize_sum += sample_weight[jj] * X_mean_ii ** 2
+                        R[jj] += sample_weight[jj] * X_mean_ii * w_ii
+                        R_sum += R[jj]
+                norm_cols_X[ii] = normalize_sum
             startptr = endptr
 
+        # Note: No need to update R_sum from here on because the update terms cancel
+        # each other: w_ii * np.sum(X[:,ii] - X_mean[ii]) = 0. R_sum is only ever
+        # needed and calculated if X_mean is provided.
+
         # tol *= np.dot(y, y)
-        tol *= _dot(n_samples, &y[0], 1, &y[0], 1)
+        # with sample weights: tol *= y @ (sw * y)
+        tol *= _dot(n_samples, &y[0], 1, &yw[0], 1)
 
         for n_iter in range(max_iter):
 
@@ -365,11 +446,19 @@ def sparse_enet_coordinate_descent(floating [::1] w,
 
                 if w_ii != 0.0:
                     # R += w_ii * X[:,ii]
-                    for jj in range(startptr, endptr):
-                        R[X_indices[jj]] += X_data[jj] * w_ii
-                    if center:
-                        for jj in range(n_samples):
-                            R[jj] -= X_mean_ii * w_ii
+                    if no_sample_weights:
+                        for jj in range(startptr, endptr):
+                            R[X_indices[jj]] += X_data[jj] * w_ii
+                        if center:
+                            for jj in range(n_samples):
+                                R[jj] -= X_mean_ii * w_ii
+                    else:
+                        for jj in range(startptr, endptr):
+                            tmp = sample_weight[X_indices[jj]]
+                            R[X_indices[jj]] += tmp * X_data[jj] * w_ii
+                        if center:
+                            for jj in range(n_samples):
+                                R[jj] -= sample_weight[jj] * X_mean_ii * w_ii
 
                 # tmp = (X[:,ii] * R).sum()
                 tmp = 0.0
@@ -377,9 +466,6 @@ def sparse_enet_coordinate_descent(floating [::1] w,
                     tmp += R[X_indices[jj]] * X_data[jj]
 
                 if center:
-                    R_sum = 0.0
-                    for jj in range(n_samples):
-                        R_sum += R[jj]
                     tmp -= R_sum * X_mean_ii
 
                 if positive and tmp < 0.0:
@@ -390,40 +476,41 @@ def sparse_enet_coordinate_descent(floating [::1] w,
 
                 if w[ii] != 0.0:
                     # R -=  w[ii] * X[:,ii] # Update residual
-                    for jj in range(startptr, endptr):
-                        R[X_indices[jj]] -= X_data[jj] * w[ii]
-
-                    if center:
-                        for jj in range(n_samples):
-                            R[jj] += X_mean_ii * w[ii]
+                    if no_sample_weights:
+                        for jj in range(startptr, endptr):
+                            R[X_indices[jj]] -= X_data[jj] * w[ii]
+                        if center:
+                            for jj in range(n_samples):
+                                R[jj] += X_mean_ii * w[ii]
+                    else:
+                        for jj in range(startptr, endptr):
+                            tmp = sample_weight[X_indices[jj]]
+                            R[X_indices[jj]] -= tmp * X_data[jj] * w[ii]
+                        if center:
+                            for jj in range(n_samples):
+                                R[jj] += sample_weight[jj] * X_mean_ii * w[ii]
 
                 # update the maximum absolute coefficient update
                 d_w_ii = fabs(w[ii] - w_ii)
-                if d_w_ii > d_w_max:
-                    d_w_max = d_w_ii
+                d_w_max = fmax(d_w_max, d_w_ii)
 
-                if fabs(w[ii]) > w_max:
-                    w_max = fabs(w[ii])
+                w_max = fmax(w_max, fabs(w[ii]))
 
             if w_max == 0.0 or d_w_max / w_max < d_w_tol or n_iter == max_iter - 1:
                 # the biggest coordinate update of this iteration was smaller than
                 # the tolerance: check the duality gap as ultimate stopping
                 # criterion
 
+                # XtA = X.T @ R - beta * w
                 # sparse X.T / dense R dot product
-                if center:
-                    R_sum = 0.0
-                    for jj in range(n_samples):
-                        R_sum += R[jj]
-
                 for ii in range(n_features):
-                    X_T_R[ii] = 0.0
-                    for jj in range(X_indptr[ii], X_indptr[ii + 1]):
-                        X_T_R[ii] += X_data[jj] * R[X_indices[jj]]
+                    XtA[ii] = 0.0
+                    for kk in range(X_indptr[ii], X_indptr[ii + 1]):
+                        XtA[ii] += X_data[kk] * R[X_indices[kk]]
 
                     if center:
-                        X_T_R[ii] -= X_mean[ii] * R_sum
-                    XtA[ii] = X_T_R[ii] - beta * w[ii]
+                        XtA[ii] -= X_mean[ii] * R_sum
+                    XtA[ii] -= beta * w[ii]
 
                 if positive:
                     dual_norm_XtA = max(n_features, &XtA[0])
@@ -431,26 +518,30 @@ def sparse_enet_coordinate_descent(floating [::1] w,
                     dual_norm_XtA = abs_max(n_features, &XtA[0])
 
                 # R_norm2 = np.dot(R, R)
-                R_norm2 = _dot(n_samples, &R[0], 1, &R[0], 1)
+                if no_sample_weights:
+                    R_norm2 = _dot(n_samples, &R[0], 1, &R[0], 1)
+                else:
+                    R_norm2 = 0.0
+                    for jj in range(n_samples):
+                        # R is already multiplied by sample_weight
+                        if sample_weight[jj] != 0:
+                            R_norm2 += (R[jj] ** 2) / sample_weight[jj]
 
                 # w_norm2 = np.dot(w, w)
                 w_norm2 = _dot(n_features, &w[0], 1, &w[0], 1)
                 if (dual_norm_XtA > alpha):
-                    const = alpha / dual_norm_XtA
-                    A_norm2 = R_norm2 * const**2
+                    const_ = alpha / dual_norm_XtA
+                    A_norm2 = R_norm2 * const_**2
                     gap = 0.5 * (R_norm2 + A_norm2)
                 else:
-                    const = 1.0
+                    const_ = 1.0
                     gap = R_norm2
 
                 l1_norm = _asum(n_features, &w[0], 1)
 
-                gap += (alpha * l1_norm - const * _dot(
-                            n_samples,
-                            &R[0], 1,
-                            &y[0], 1
-                            )
-                        + 0.5 * beta * (1 + const ** 2) * w_norm2)
+                gap += (alpha * l1_norm
+                        - const_ * _dot(n_samples, &R[0], 1, &y[0], 1)  # np.dot(R.T, y)
+                        + 0.5 * beta * (1 + const_ ** 2) * w_norm2)
 
                 if gap < tol:
                     # return if we reached desired tolerance
@@ -459,21 +550,30 @@ def sparse_enet_coordinate_descent(floating [::1] w,
         else:
             # for/else, runs if for doesn't end with a `break`
             with gil:
-                warnings.warn("Objective did not converge. You might want to "
-                              "increase the number of iterations. Duality "
-                              "gap: {}, tolerance: {}".format(gap, tol),
-                              ConvergenceWarning)
+                message = (
+                    message_conv +
+                    f" Duality gap: {gap:.3e}, tolerance: {tol:.3e}"
+                )
+                if alpha < np.finfo(np.float64).eps:
+                    message += "\n" + message_ridge
+                warnings.warn(message, ConvergenceWarning)
 
-    return w, gap, tol, n_iter + 1
+    return np.asarray(w), gap, tol, n_iter + 1
 
 
-def enet_coordinate_descent_gram(floating[::1] w,
-                                 floating alpha, floating beta,
-                                 np.ndarray[floating, ndim=2, mode='c'] Q,
-                                 np.ndarray[floating, ndim=1, mode='c'] q,
-                                 np.ndarray[floating, ndim=1] y,
-                                 int max_iter, floating tol, object rng,
-                                 bint random=0, bint positive=0):
+def enet_coordinate_descent_gram(
+    floating[::1] w,
+    floating alpha,
+    floating beta,
+    const floating[:, ::1] Q,
+    const floating[::1] q,
+    const floating[:] y,
+    unsigned int max_iter,
+    floating tol,
+    object rng,
+    bint random=0,
+    bint positive=0
+):
     """Cython version of the coordinate descent algorithm
         for Elastic-Net regression
 
@@ -484,6 +584,17 @@ def enet_coordinate_descent_gram(floating[::1] w,
         which amount to the Elastic-Net problem when:
         Q = X^T X (Gram matrix)
         q = X^T y
+
+    Returns
+    -------
+    w : ndarray of shape (n_features,)
+        ElasticNet coefficients.
+    gap : float
+        Achieved dual gap.
+    tol : float
+        Equals input `tol` times `np.dot(y, y)`. The tolerance used for the dual gap.
+    n_iter : int
+        Number of coordinate descent iterations.
     """
 
     if floating is float:
@@ -492,7 +603,6 @@ def enet_coordinate_descent_gram(floating[::1] w,
         dtype = np.float64
 
     # get the data information into easy vars
-    cdef unsigned int n_samples = y.shape[0]
     cdef unsigned int n_features = Q.shape[0]
 
     # initial value "Q w" which will be kept of up to date in the iterations
@@ -512,20 +622,23 @@ def enet_coordinate_descent_gram(floating[::1] w,
     cdef unsigned int ii
     cdef unsigned int n_iter = 0
     cdef unsigned int f_iter
-    cdef UINT32_t rand_r_state_seed = rng.randint(0, RAND_R_MAX)
-    cdef UINT32_t* rand_r_state = &rand_r_state_seed
+    cdef uint32_t rand_r_state_seed = rng.randint(0, RAND_R_MAX)
+    cdef uint32_t* rand_r_state = &rand_r_state_seed
 
     cdef floating y_norm2 = np.dot(y, y)
-    cdef floating* w_ptr = &w[0]
-    cdef floating* Q_ptr = &Q[0, 0]
-    cdef floating* q_ptr = q.data
+    cdef floating* w_ptr = &w[0]
+    cdef const floating* Q_ptr = &Q[0, 0]
+    cdef const floating* q_ptr = &q[0]
     cdef floating* H_ptr = &H[0]
     cdef floating* XtA_ptr = &XtA[0]
     tol = tol * y_norm2
 
     if alpha == 0:
-        warnings.warn("Coordinate descent with alpha=0 may lead to unexpected"
-            " results and is discouraged.")
+        warnings.warn(
+            "Coordinate descent without L1 regularization may "
+            "lead to unexpected results and is discouraged. "
+            "Set l1_ratio > 0 to add L1 regularization."
+        )
 
     with nogil:
         for n_iter in range(max_iter):
@@ -593,17 +706,20 @@ def enet_coordinate_descent_gram(floating[::1] w,
                 w_norm2 = _dot(n_features, &w[0], 1, &w[0], 1)
 
                 if (dual_norm_XtA > alpha):
-                    const = alpha / dual_norm_XtA
-                    A_norm2 = R_norm2 * (const ** 2)
+                    const_ = alpha / dual_norm_XtA
+                    A_norm2 = R_norm2 * (const_ ** 2)
                     gap = 0.5 * (R_norm2 + A_norm2)
                 else:
-                    const = 1.0
+                    const_ = 1.0
                     gap = R_norm2
 
                 # The call to asum is equivalent to the L1 norm of w
-                gap += (alpha * _asum(n_features, &w[0], 1) -
-                        const * y_norm2 +  const * q_dot_w +
-                        0.5 * beta * (1 + const ** 2) * w_norm2)
+                gap += (
+                    alpha * _asum(n_features, &w[0], 1)
+                    - const_ * y_norm2
+                    + const_ * q_dot_w
+                    + 0.5 * beta * (1 + const_ ** 2) * w_norm2
+                )
 
                 if gap < tol:
                     # return if we reached desired tolerance
@@ -612,26 +728,43 @@ def enet_coordinate_descent_gram(floating[::1] w,
         else:
             # for/else, runs if for doesn't end with a `break`
             with gil:
-                warnings.warn("Objective did not converge. You might want to "
-                              "increase the number of iterations. Duality "
-                              "gap: {}, tolerance: {}".format(gap, tol),
-                              ConvergenceWarning)
+                message = (
+                    message_conv +
+                    f" Duality gap: {gap:.3e}, tolerance: {tol:.3e}"
+                )
+                warnings.warn(message, ConvergenceWarning)
 
     return np.asarray(w), gap, tol, n_iter + 1
 
 
 def enet_coordinate_descent_multi_task(
-        floating[::1, :] W, floating l1_reg, floating l2_reg,
-        np.ndarray[floating, ndim=2, mode='fortran'] X,  # TODO: use views with Cython 3.0
-        np.ndarray[floating, ndim=2, mode='fortran'] Y,  # hopefully with skl 1.0
-        int max_iter, floating tol, object rng, bint random=0):
+    const floating[::1, :] W,
+    floating l1_reg,
+    floating l2_reg,
+    const floating[::1, :] X,
+    const floating[::1, :] Y,
+    unsigned int max_iter,
+    floating tol,
+    object rng,
+    bint random=0
+):
     """Cython version of the coordinate descent algorithm
-        for Elastic-Net mult-task regression
+        for Elastic-Net multi-task regression
 
         We minimize
 
         0.5 * norm(Y - X W.T, 2)^2 + l1_reg ||W.T||_21 + 0.5 * l2_reg norm(W.T, 2)^2
 
+    Returns
+    -------
+    W : ndarray of shape (n_tasks, n_features)
+        ElasticNet coefficients.
+    gap : float
+        Achieved dual gap.
+    tol : float
+        Equals input `tol` times `np.dot(y, y)`. The tolerance used for the dual gap.
+    n_iter : int
+        Number of coordinate descent iterations.
     """
 
     if floating is float:
@@ -670,15 +803,17 @@ def enet_coordinate_descent_multi_task(
     cdef unsigned int jj
     cdef unsigned int n_iter = 0
     cdef unsigned int f_iter
-    cdef UINT32_t rand_r_state_seed = rng.randint(0, RAND_R_MAX)
-    cdef UINT32_t* rand_r_state = &rand_r_state_seed
+    cdef uint32_t rand_r_state_seed = rng.randint(0, RAND_R_MAX)
+    cdef uint32_t* rand_r_state = &rand_r_state_seed
 
-    cdef floating* X_ptr = &X[0, 0]
-    cdef floating* Y_ptr = &Y[0, 0]
+    cdef const floating* X_ptr = &X[0, 0]
+    cdef const floating* Y_ptr = &Y[0, 0]
 
     if l1_reg == 0:
-        warnings.warn("Coordinate descent with l1_reg=0 may lead to unexpected"
-            " results and is discouraged.")
+        warnings.warn(
+            "Coordinate descent with l1_reg=0 may lead to unexpected"
+            " results and is discouraged."
+        )
 
     with nogil:
         # norm_cols_X = (np.asarray(X) ** 2).sum(axis=0)
@@ -791,11 +926,11 @@ def enet_coordinate_descent_multi_task(
                 R_norm = _nrm2(n_samples * n_tasks, &R[0, 0], 1)
                 w_norm = _nrm2(n_features * n_tasks, &W[0, 0], 1)
                 if (dual_norm_XtA > l1_reg):
-                    const =  l1_reg / dual_norm_XtA
-                    A_norm = R_norm * const
+                    const_ = l1_reg / dual_norm_XtA
+                    A_norm = R_norm * const_
                     gap = 0.5 * (R_norm ** 2 + A_norm ** 2)
                 else:
-                    const = 1.0
+                    const_ = 1.0
                     gap = R_norm ** 2
 
                 # ry_sum = np.sum(R * y)
@@ -806,18 +941,22 @@ def enet_coordinate_descent_multi_task(
                 for ii in range(n_features):
                     l21_norm += _nrm2(n_tasks, &W[0, ii], 1)
 
-                gap += l1_reg * l21_norm - const * ry_sum + \
-                     0.5 * l2_reg * (1 + const ** 2) * (w_norm ** 2)
+                gap += (
+                    l1_reg * l21_norm
+                    - const_ * ry_sum
+                    + 0.5 * l2_reg * (1 + const_ ** 2) * (w_norm ** 2)
+                )
 
-                if gap < tol:
+                if gap <= tol:
                     # return if we reached desired tolerance
                     break
         else:
             # for/else, runs if for doesn't end with a `break`
             with gil:
-                warnings.warn("Objective did not converge. You might want to "
-                              "increase the number of iterations. Duality "
-                              "gap: {}, tolerance: {}".format(gap, tol),
-                              ConvergenceWarning)
+                message = (
+                    message_conv +
+                    f" Duality gap: {gap:.3e}, tolerance: {tol:.3e}"
+                )
+                warnings.warn(message, ConvergenceWarning)
 
     return np.asarray(W), gap, tol, n_iter + 1
diff --git a/sklearn/linear_model/_coordinate_descent.py b/sklearn/linear_model/_coordinate_descent.py
index 4fdeb783db194..62096133ada2f 100644
--- a/sklearn/linear_model/_coordinate_descent.py
+++ b/sklearn/linear_model/_coordinate_descent.py
@@ -1,37 +1,51 @@
-# Author: Alexandre Gramfort 
-#         Fabian Pedregosa 
-#         Olivier Grisel 
-#         Gael Varoquaux 
-#
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
+import numbers
 import sys
 import warnings
-import numbers
-from abc import ABCMeta, abstractmethod
+from abc import ABC, abstractmethod
+from functools import partial
+from numbers import Integral, Real
 
 import numpy as np
+from joblib import effective_n_jobs
 from scipy import sparse
-from joblib import Parallel, effective_n_jobs
 
-from ._base import LinearModel, _pre_fit
-from ..base import RegressorMixin, MultiOutputMixin
-from ._base import _preprocess_data
-from ..utils import check_array
-from ..utils.validation import check_random_state
+from sklearn.utils import metadata_routing
+
+from ..base import MultiOutputMixin, RegressorMixin, _fit_context
 from ..model_selection import check_cv
+from ..utils import Bunch, check_array, check_scalar
+from ..utils._metadata_requests import (
+    MetadataRouter,
+    MethodMapping,
+    _raise_for_params,
+    get_routing_for_object,
+)
+from ..utils._param_validation import Hidden, Interval, StrOptions, validate_params
 from ..utils.extmath import safe_sparse_dot
-from ..utils.fixes import _astype_copy_false, _joblib_parallel_args
-from ..utils.validation import check_is_fitted, _check_sample_weight
-from ..utils.validation import column_or_1d
-from ..utils.validation import _deprecate_positional_args
-from ..utils.fixes import delayed
+from ..utils.metadata_routing import (
+    _routing_enabled,
+    process_routing,
+)
+from ..utils.parallel import Parallel, delayed
+from ..utils.validation import (
+    _check_sample_weight,
+    check_consistent_length,
+    check_is_fitted,
+    check_random_state,
+    column_or_1d,
+    has_fit_parameter,
+    validate_data,
+)
 
 # mypy error: Module 'sklearn.linear_model' has no attribute '_cd_fast'
-from . import _cd_fast as cd_fast  # type: ignore
+from . import _cd_fast as cd_fast  # type: ignore[attr-defined]
+from ._base import LinearModel, _pre_fit, _preprocess_data
 
 
-def _set_order(X, y, order='C'):
+def _set_order(X, y, order="C"):
     """Change the order of X and y if necessary.
 
     Parameters
@@ -55,17 +69,16 @@ def _set_order(X, y, order='C'):
     y : ndarray of shape (n_samples,)
         Target values with guaranteed order.
     """
-    if order not in [None, 'C', 'F']:
-        raise ValueError("Unknown value for order. Got {} instead of "
-                         "None, 'C' or 'F'.".format(order))
+    if order not in [None, "C", "F"]:
+        raise ValueError(
+            "Unknown value for order. Got {} instead of None, 'C' or 'F'.".format(order)
+        )
     sparse_X = sparse.issparse(X)
     sparse_y = sparse.issparse(y)
     if order is not None:
         sparse_format = "csc" if order == "F" else "csr"
         if sparse_X:
-            # As of scipy 1.1.0, new argument copy=False by default.
-            # This is what we want.
-            X = X.asformat(sparse_format, **_astype_copy_false(X))
+            X = X.asformat(sparse_format, copy=False)
         else:
             X = np.asarray(X, order=order)
         if sparse_y:
@@ -78,9 +91,19 @@ def _set_order(X, y, order='C'):
 ###############################################################################
 # Paths functions
 
-def _alpha_grid(X, y, Xy=None, l1_ratio=1.0, fit_intercept=True,
-                eps=1e-3, n_alphas=100, normalize=False, copy_X=True):
-    """ Compute the grid of alpha values for elastic net parameter search
+
+def _alpha_grid(
+    X,
+    y,
+    Xy=None,
+    l1_ratio=1.0,
+    fit_intercept=True,
+    eps=1e-3,
+    n_alphas=100,
+    copy_X=True,
+    sample_weight=None,
+):
+    """Compute the grid of alpha values for elastic net parameter search
 
     Parameters
     ----------
@@ -88,10 +111,11 @@ def _alpha_grid(X, y, Xy=None, l1_ratio=1.0, fit_intercept=True,
         Training data. Pass directly as Fortran-contiguous data to avoid
         unnecessary memory duplication
 
-    y : ndarray of shape (n_samples,)
+    y : ndarray of shape (n_samples,) or (n_samples, n_outputs)
         Target values
 
-    Xy : array-like of shape (n_features,), default=None
+    Xy : array-like of shape (n_features,) or (n_features, n_outputs),\
+         default=None
         Xy = np.dot(X.T, y) that can be precomputed.
 
     l1_ratio : float, default=1.0
@@ -110,70 +134,89 @@ def _alpha_grid(X, y, Xy=None, l1_ratio=1.0, fit_intercept=True,
     fit_intercept : bool, default=True
         Whether to fit an intercept or not
 
-    normalize : bool, default=False
-        This parameter is ignored when ``fit_intercept`` is set to False.
-        If True, the regressors X will be normalized before regression by
-        subtracting the mean and dividing by the l2-norm.
-        If you wish to standardize, please use
-        :class:`~sklearn.preprocessing.StandardScaler` before calling ``fit``
-        on an estimator with ``normalize=False``.
-
     copy_X : bool, default=True
         If ``True``, X will be copied; else, it may be overwritten.
+
+    sample_weight : ndarray of shape (n_samples,), default=None
     """
     if l1_ratio == 0:
-        raise ValueError("Automatic alpha grid generation is not supported for"
-                         " l1_ratio=0. Please supply a grid by providing "
-                         "your estimator with the appropriate `alphas=` "
-                         "argument.")
-    n_samples = len(y)
-
-    sparse_center = False
-    if Xy is None:
-        X_sparse = sparse.isspmatrix(X)
-        sparse_center = X_sparse and (fit_intercept or normalize)
-        X = check_array(X, accept_sparse='csc',
-                        copy=(copy_X and fit_intercept and not X_sparse))
-        if not X_sparse:
-            # X can be touched inplace thanks to the above line
-            X, y, _, _, _ = _preprocess_data(X, y, fit_intercept,
-                                             normalize, copy=False)
-        Xy = safe_sparse_dot(X.T, y, dense_output=True)
-
-        if sparse_center:
-            # Workaround to find alpha_max for sparse matrices.
-            # since we should not destroy the sparsity of such matrices.
-            _, _, X_offset, _, X_scale = _preprocess_data(X, y, fit_intercept,
-                                                          normalize,
-                                                          return_mean=True)
-            mean_dot = X_offset * np.sum(y)
-
-    if Xy.ndim == 1:
-        Xy = Xy[:, np.newaxis]
-
-    if sparse_center:
-        if fit_intercept:
-            Xy -= mean_dot[:, np.newaxis]
-        if normalize:
-            Xy /= X_scale[:, np.newaxis]
-
-    alpha_max = (np.sqrt(np.sum(Xy ** 2, axis=1)).max() /
-                 (n_samples * l1_ratio))
-
-    if alpha_max <= np.finfo(float).resolution:
-        alphas = np.empty(n_alphas)
-        alphas.fill(np.finfo(float).resolution)
-        return alphas
-
-    return np.logspace(np.log10(alpha_max * eps), np.log10(alpha_max),
-                       num=n_alphas)[::-1]
-
-
-@_deprecate_positional_args
-def lasso_path(X, y, *, eps=1e-3, n_alphas=100, alphas=None,
-               precompute='auto', Xy=None, copy_X=True, coef_init=None,
-               verbose=False, return_n_iter=False, positive=False, **params):
-    """Compute Lasso path with coordinate descent
+        raise ValueError(
+            "Automatic alpha grid generation is not supported for"
+            " l1_ratio=0. Please supply a grid by providing "
+            "your estimator with the appropriate `alphas=` "
+            "argument."
+        )
+    if Xy is not None:
+        Xyw = Xy
+    else:
+        X, y, X_offset, _, _ = _preprocess_data(
+            X,
+            y,
+            fit_intercept=fit_intercept,
+            copy=copy_X,
+            sample_weight=sample_weight,
+            check_input=False,
+        )
+        if sample_weight is not None:
+            if y.ndim > 1:
+                yw = y * sample_weight.reshape(-1, 1)
+            else:
+                yw = y * sample_weight
+        else:
+            yw = y
+        if sparse.issparse(X):
+            Xyw = safe_sparse_dot(X.T, yw, dense_output=True) - np.sum(yw) * X_offset
+        else:
+            Xyw = np.dot(X.T, yw)
+
+    if Xyw.ndim == 1:
+        Xyw = Xyw[:, np.newaxis]
+    if sample_weight is not None:
+        n_samples = sample_weight.sum()
+    else:
+        n_samples = X.shape[0]
+    alpha_max = np.sqrt(np.sum(Xyw**2, axis=1)).max() / (n_samples * l1_ratio)
+
+    if alpha_max <= np.finfo(np.float64).resolution:
+        return np.full(n_alphas, np.finfo(np.float64).resolution)
+
+    return np.geomspace(alpha_max, alpha_max * eps, num=n_alphas)
+
+
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "y": ["array-like", "sparse matrix"],
+        "eps": [Interval(Real, 0, None, closed="neither")],
+        "n_alphas": [Interval(Integral, 1, None, closed="left")],
+        "alphas": ["array-like", None],
+        "precompute": [StrOptions({"auto"}), "boolean", "array-like"],
+        "Xy": ["array-like", None],
+        "copy_X": ["boolean"],
+        "coef_init": ["array-like", None],
+        "verbose": ["verbose"],
+        "return_n_iter": ["boolean"],
+        "positive": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def lasso_path(
+    X,
+    y,
+    *,
+    eps=1e-3,
+    n_alphas=100,
+    alphas=None,
+    precompute="auto",
+    Xy=None,
+    copy_X=True,
+    coef_init=None,
+    verbose=False,
+    return_n_iter=False,
+    positive=False,
+    **params,
+):
+    """Compute Lasso path with coordinate descent.
 
     The Lasso optimization function varies for mono and multi-outputs.
 
@@ -201,27 +244,27 @@ def lasso_path(X, y, *, eps=1e-3, n_alphas=100, alphas=None,
         can be sparse.
 
     y : {array-like, sparse matrix} of shape (n_samples,) or \
-        (n_samples, n_outputs)
-        Target values
+        (n_samples, n_targets)
+        Target values.
 
     eps : float, default=1e-3
         Length of the path. ``eps=1e-3`` means that
-        ``alpha_min / alpha_max = 1e-3``
+        ``alpha_min / alpha_max = 1e-3``.
 
     n_alphas : int, default=100
-        Number of alphas along the regularization path
+        Number of alphas along the regularization path.
 
-    alphas : ndarray, default=None
+    alphas : array-like, default=None
         List of alphas where to compute the models.
-        If ``None`` alphas are set automatically
+        If ``None`` alphas are set automatically.
 
-    precompute : 'auto', bool or array-like of shape (n_features, n_features),\
-                 default='auto'
+    precompute : 'auto', bool or array-like of shape \
+            (n_features, n_features), default='auto'
         Whether to use a precomputed Gram matrix to speed up
         calculations. If set to ``'auto'`` let us decide. The Gram
         matrix can also be passed as argument.
 
-    Xy : array-like of shape (n_features,) or (n_features, n_outputs),\
+    Xy : array-like of shape (n_features,) or (n_features, n_targets),\
          default=None
         Xy = np.dot(X.T, y) that can be precomputed. It is useful
         only when the Gram matrix is precomputed.
@@ -229,21 +272,21 @@ def lasso_path(X, y, *, eps=1e-3, n_alphas=100, alphas=None,
     copy_X : bool, default=True
         If ``True``, X will be copied; else, it may be overwritten.
 
-    coef_init : ndarray of shape (n_features, ), default=None
+    coef_init : array-like of shape (n_features, ), default=None
         The initial values of the coefficients.
 
     verbose : bool or int, default=False
         Amount of verbosity.
 
     return_n_iter : bool, default=False
-        whether to return the number of iterations or not.
+        Whether to return the number of iterations or not.
 
     positive : bool, default=False
         If set to True, forces coefficients to be positive.
         (Only allowed when ``y.ndim == 1``).
 
     **params : kwargs
-        keyword arguments passed to the coordinate descent solver.
+        Keyword arguments passed to the coordinate descent solver.
 
     Returns
     -------
@@ -251,7 +294,7 @@ def lasso_path(X, y, *, eps=1e-3, n_alphas=100, alphas=None,
         The alphas along the path where models are computed.
 
     coefs : ndarray of shape (n_features, n_alphas) or \
-            (n_outputs, n_features, n_alphas)
+            (n_targets, n_features, n_alphas)
         Coefficients along the path.
 
     dual_gaps : ndarray of shape (n_alphas,)
@@ -261,11 +304,23 @@ def lasso_path(X, y, *, eps=1e-3, n_alphas=100, alphas=None,
         The number of iterations taken by the coordinate descent optimizer to
         reach the specified tolerance for each alpha.
 
+    See Also
+    --------
+    lars_path : Compute Least Angle Regression or Lasso path using LARS
+        algorithm.
+    Lasso : The Lasso is a linear model that estimates sparse coefficients.
+    LassoLars : Lasso model fit with Least Angle Regression a.k.a. Lars.
+    LassoCV : Lasso linear model with iterative fitting along a regularization
+        path.
+    LassoLarsCV : Cross-validated Lasso using the LARS algorithm.
+    sklearn.decomposition.sparse_encode : Estimator that can be used to
+        transform signals into sparse linear combination of atoms from a fixed.
+
     Notes
     -----
     For an example, see
-    :ref:`examples/linear_model/plot_lasso_coordinate_descent_path.py
-    `.
+    :ref:`examples/linear_model/plot_lasso_lasso_lars_elasticnet_path.py
+    `.
 
     To avoid unnecessary memory duplication the X argument of the fit method
     should be directly passed as a Fortran-contiguous numpy array.
@@ -280,6 +335,8 @@ def lasso_path(X, y, *, eps=1e-3, n_alphas=100, alphas=None,
 
     Comparing lasso_path and lars_path with interpolation:
 
+    >>> import numpy as np
+    >>> from sklearn.linear_model import lasso_path
     >>> X = np.array([[1, 2, 3.1], [2.3, 5.4, 4.3]]).T
     >>> y = np.array([1, 2, 3.1])
     >>> # Use lasso_path to compute a coefficient path
@@ -298,29 +355,63 @@ def lasso_path(X, y, *, eps=1e-3, n_alphas=100, alphas=None,
     >>> print(coef_path_continuous([5., 1., .5]))
     [[0.         0.         0.46915237]
      [0.2159048  0.4425765  0.23668876]]
-
-    See Also
-    --------
-    lars_path
-    Lasso
-    LassoLars
-    LassoCV
-    LassoLarsCV
-    sklearn.decomposition.sparse_encode
-    """
-    return enet_path(X, y, l1_ratio=1., eps=eps, n_alphas=n_alphas,
-                     alphas=alphas, precompute=precompute, Xy=Xy,
-                     copy_X=copy_X, coef_init=coef_init, verbose=verbose,
-                     positive=positive, return_n_iter=return_n_iter, **params)
-
-
-@_deprecate_positional_args
-def enet_path(X, y, *, l1_ratio=0.5, eps=1e-3, n_alphas=100, alphas=None,
-              precompute='auto', Xy=None, copy_X=True, coef_init=None,
-              verbose=False, return_n_iter=False, positive=False,
-              check_input=True, **params):
     """
-    Compute elastic net path with coordinate descent.
+    return enet_path(
+        X,
+        y,
+        l1_ratio=1.0,
+        eps=eps,
+        n_alphas=n_alphas,
+        alphas=alphas,
+        precompute=precompute,
+        Xy=Xy,
+        copy_X=copy_X,
+        coef_init=coef_init,
+        verbose=verbose,
+        positive=positive,
+        return_n_iter=return_n_iter,
+        **params,
+    )
+
+
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "y": ["array-like", "sparse matrix"],
+        "l1_ratio": [Interval(Real, 0.0, 1.0, closed="both")],
+        "eps": [Interval(Real, 0.0, None, closed="neither")],
+        "n_alphas": [Interval(Integral, 1, None, closed="left")],
+        "alphas": ["array-like", None],
+        "precompute": [StrOptions({"auto"}), "boolean", "array-like"],
+        "Xy": ["array-like", None],
+        "copy_X": ["boolean"],
+        "coef_init": ["array-like", None],
+        "verbose": ["verbose"],
+        "return_n_iter": ["boolean"],
+        "positive": ["boolean"],
+        "check_input": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def enet_path(
+    X,
+    y,
+    *,
+    l1_ratio=0.5,
+    eps=1e-3,
+    n_alphas=100,
+    alphas=None,
+    precompute="auto",
+    Xy=None,
+    copy_X=True,
+    coef_init=None,
+    verbose=False,
+    return_n_iter=False,
+    positive=False,
+    check_input=True,
+    **params,
+):
+    """Compute elastic net path with coordinate descent.
 
     The elastic net optimization function varies for mono and multi-outputs.
 
@@ -352,7 +443,7 @@ def enet_path(X, y, *, l1_ratio=0.5, eps=1e-3, n_alphas=100, alphas=None,
         can be sparse.
 
     y : {array-like, sparse matrix} of shape (n_samples,) or \
-        (n_samples, n_outputs)
+        (n_samples, n_targets)
         Target values.
 
     l1_ratio : float, default=0.5
@@ -366,17 +457,17 @@ def enet_path(X, y, *, l1_ratio=0.5, eps=1e-3, n_alphas=100, alphas=None,
     n_alphas : int, default=100
         Number of alphas along the regularization path.
 
-    alphas : ndarray, default=None
+    alphas : array-like, default=None
         List of alphas where to compute the models.
         If None alphas are set automatically.
 
-    precompute : 'auto', bool or array-like of shape (n_features, n_features),\
-                 default='auto'
+    precompute : 'auto', bool or array-like of shape \
+            (n_features, n_features), default='auto'
         Whether to use a precomputed Gram matrix to speed up
         calculations. If set to ``'auto'`` let us decide. The Gram
         matrix can also be passed as argument.
 
-    Xy : array-like of shape (n_features,) or (n_features, n_outputs),\
+    Xy : array-like of shape (n_features,) or (n_features, n_targets),\
          default=None
         Xy = np.dot(X.T, y) that can be precomputed. It is useful
         only when the Gram matrix is precomputed.
@@ -384,7 +475,7 @@ def enet_path(X, y, *, l1_ratio=0.5, eps=1e-3, n_alphas=100, alphas=None,
     copy_X : bool, default=True
         If ``True``, X will be copied; else, it may be overwritten.
 
-    coef_init : ndarray of shape (n_features, ), default=None
+    coef_init : array-like of shape (n_features, ), default=None
         The initial values of the coefficients.
 
     verbose : bool or int, default=False
@@ -411,7 +502,7 @@ def enet_path(X, y, *, l1_ratio=0.5, eps=1e-3, n_alphas=100, alphas=None,
         The alphas along the path where models are computed.
 
     coefs : ndarray of shape (n_features, n_alphas) or \
-            (n_outputs, n_features, n_alphas)
+            (n_targets, n_features, n_alphas)
         Coefficients along the path.
 
     dual_gaps : ndarray of shape (n_alphas,)
@@ -424,85 +515,136 @@ def enet_path(X, y, *, l1_ratio=0.5, eps=1e-3, n_alphas=100, alphas=None,
 
     See Also
     --------
-    MultiTaskElasticNet
-    MultiTaskElasticNetCV
-    ElasticNet
-    ElasticNetCV
+    MultiTaskElasticNet : Multi-task ElasticNet model trained with L1/L2 mixed-norm \
+    as regularizer.
+    MultiTaskElasticNetCV : Multi-task L1/L2 ElasticNet with built-in cross-validation.
+    ElasticNet : Linear regression with combined L1 and L2 priors as regularizer.
+    ElasticNetCV : Elastic Net model with iterative fitting along a regularization path.
 
     Notes
     -----
     For an example, see
-    :ref:`examples/linear_model/plot_lasso_coordinate_descent_path.py
-    `.
+    :ref:`examples/linear_model/plot_lasso_lasso_lars_elasticnet_path.py
+    `.
+
+    Examples
+    --------
+    >>> from sklearn.linear_model import enet_path
+    >>> from sklearn.datasets import make_regression
+    >>> X, y, true_coef = make_regression(
+    ...    n_samples=100, n_features=5, n_informative=2, coef=True, random_state=0
+    ... )
+    >>> true_coef
+    array([ 0.        ,  0.        ,  0.        , 97.9, 45.7])
+    >>> alphas, estimated_coef, _ = enet_path(X, y, n_alphas=3)
+    >>> alphas.shape
+    (3,)
+    >>> estimated_coef
+     array([[ 0.,  0.787,  0.568],
+            [ 0.,  1.120,  0.620],
+            [-0., -2.129, -1.128],
+            [ 0., 23.046, 88.939],
+            [ 0., 10.637, 41.566]])
     """
+    X_offset_param = params.pop("X_offset", None)
+    X_scale_param = params.pop("X_scale", None)
+    sample_weight = params.pop("sample_weight", None)
+    tol = params.pop("tol", 1e-4)
+    max_iter = params.pop("max_iter", 1000)
+    random_state = params.pop("random_state", None)
+    selection = params.pop("selection", "cyclic")
+
+    if len(params) > 0:
+        raise ValueError("Unexpected parameters in params", params.keys())
+
     # We expect X and y to be already Fortran ordered when bypassing
     # checks
     if check_input:
-        X = check_array(X, accept_sparse='csc', dtype=[np.float64, np.float32],
-                        order='F', copy=copy_X)
-        y = check_array(y, accept_sparse='csc', dtype=X.dtype.type,
-                        order='F', copy=False, ensure_2d=False)
+        X = check_array(
+            X,
+            accept_sparse="csc",
+            dtype=[np.float64, np.float32],
+            order="F",
+            copy=copy_X,
+        )
+        y = check_array(
+            y,
+            accept_sparse="csc",
+            dtype=X.dtype.type,
+            order="F",
+            copy=False,
+            ensure_2d=False,
+        )
         if Xy is not None:
             # Xy should be a 1d contiguous array or a 2D C ordered array
-            Xy = check_array(Xy, dtype=X.dtype.type, order='C', copy=False,
-                             ensure_2d=False)
+            Xy = check_array(
+                Xy, dtype=X.dtype.type, order="C", copy=False, ensure_2d=False
+            )
 
     n_samples, n_features = X.shape
 
     multi_output = False
     if y.ndim != 1:
         multi_output = True
-        _, n_outputs = y.shape
+        n_targets = y.shape[1]
 
     if multi_output and positive:
-        raise ValueError('positive=True is not allowed for multi-output'
-                         ' (y.ndim != 1)')
+        raise ValueError("positive=True is not allowed for multi-output (y.ndim != 1)")
 
     # MultiTaskElasticNet does not support sparse matrices
-    if not multi_output and sparse.isspmatrix(X):
-        if 'X_offset' in params:
-            # As sparse matrices are not actually centered we need this
-            # to be passed to the CD solver.
-            X_sparse_scaling = params['X_offset'] / params['X_scale']
+    if not multi_output and sparse.issparse(X):
+        if X_offset_param is not None:
+            # As sparse matrices are not actually centered we need this to be passed to
+            # the CD solver.
+            X_sparse_scaling = X_offset_param / X_scale_param
             X_sparse_scaling = np.asarray(X_sparse_scaling, dtype=X.dtype)
         else:
             X_sparse_scaling = np.zeros(n_features, dtype=X.dtype)
 
-    # X should be normalized and fit already if function is called
+    # X should have been passed through _pre_fit already if function is called
     # from ElasticNet.fit
     if check_input:
-        X, y, X_offset, y_offset, X_scale, precompute, Xy = \
-            _pre_fit(X, y, Xy, precompute, normalize=False,
-                     fit_intercept=False, copy=False, check_input=check_input)
+        X, y, _, _, _, precompute, Xy = _pre_fit(
+            X,
+            y,
+            Xy,
+            precompute,
+            fit_intercept=False,
+            copy=False,
+            check_input=check_input,
+        )
     if alphas is None:
         # No need to normalize of fit_intercept: it has been done
         # above
-        alphas = _alpha_grid(X, y, Xy=Xy, l1_ratio=l1_ratio,
-                             fit_intercept=False, eps=eps, n_alphas=n_alphas,
-                             normalize=False, copy_X=False)
-    else:
+        alphas = _alpha_grid(
+            X,
+            y,
+            Xy=Xy,
+            l1_ratio=l1_ratio,
+            fit_intercept=False,
+            eps=eps,
+            n_alphas=n_alphas,
+            copy_X=False,
+        )
+    elif len(alphas) > 1:
         alphas = np.sort(alphas)[::-1]  # make sure alphas are properly ordered
 
     n_alphas = len(alphas)
-    tol = params.get('tol', 1e-4)
-    max_iter = params.get('max_iter', 1000)
     dual_gaps = np.empty(n_alphas)
     n_iters = []
 
-    rng = check_random_state(params.get('random_state', None))
-    selection = params.get('selection', 'cyclic')
-    if selection not in ['random', 'cyclic']:
+    rng = check_random_state(random_state)
+    if selection not in ["random", "cyclic"]:
         raise ValueError("selection should be either random or cyclic.")
-    random = (selection == 'random')
+    random = selection == "random"
 
     if not multi_output:
         coefs = np.empty((n_features, n_alphas), dtype=X.dtype)
     else:
-        coefs = np.empty((n_outputs, n_features, n_alphas),
-                         dtype=X.dtype)
+        coefs = np.empty((n_targets, n_features, n_alphas), dtype=X.dtype)
 
     if coef_init is None:
-        coef_ = np.zeros(coefs.shape[:-1], dtype=X.dtype, order='F')
+        coef_ = np.zeros(coefs.shape[:-1], dtype=X.dtype, order="F")
     else:
         coef_ = np.asfortranarray(coef_init, dtype=X.dtype)
 
@@ -510,30 +652,54 @@ def enet_path(X, y, *, l1_ratio=0.5, eps=1e-3, n_alphas=100, alphas=None,
         # account for n_samples scaling in objectives between here and cd_fast
         l1_reg = alpha * l1_ratio * n_samples
         l2_reg = alpha * (1.0 - l1_ratio) * n_samples
-        if not multi_output and sparse.isspmatrix(X):
+        if not multi_output and sparse.issparse(X):
             model = cd_fast.sparse_enet_coordinate_descent(
-                coef_, l1_reg, l2_reg, X.data, X.indices,
-                X.indptr, y, X_sparse_scaling,
-                max_iter, tol, rng, random, positive)
+                w=coef_,
+                alpha=l1_reg,
+                beta=l2_reg,
+                X_data=X.data,
+                X_indices=X.indices,
+                X_indptr=X.indptr,
+                y=y,
+                sample_weight=sample_weight,
+                X_mean=X_sparse_scaling,
+                max_iter=max_iter,
+                tol=tol,
+                rng=rng,
+                random=random,
+                positive=positive,
+            )
         elif multi_output:
             model = cd_fast.enet_coordinate_descent_multi_task(
-                coef_, l1_reg, l2_reg, X, y, max_iter, tol, rng, random)
+                coef_, l1_reg, l2_reg, X, y, max_iter, tol, rng, random
+            )
         elif isinstance(precompute, np.ndarray):
             # We expect precompute to be already Fortran ordered when bypassing
             # checks
             if check_input:
-                precompute = check_array(precompute, dtype=X.dtype.type,
-                                         order='C')
+                precompute = check_array(precompute, dtype=X.dtype.type, order="C")
             model = cd_fast.enet_coordinate_descent_gram(
-                coef_, l1_reg, l2_reg, precompute, Xy, y, max_iter,
-                tol, rng, random, positive)
+                coef_,
+                l1_reg,
+                l2_reg,
+                precompute,
+                Xy,
+                y,
+                max_iter,
+                tol,
+                rng,
+                random,
+                positive,
+            )
         elif precompute is False:
             model = cd_fast.enet_coordinate_descent(
-                coef_, l1_reg, l2_reg, X, y, max_iter, tol, rng, random,
-                positive)
+                coef_, l1_reg, l2_reg, X, y, max_iter, tol, rng, random, positive
+            )
         else:
-            raise ValueError("Precompute should be one of True, False, "
-                             "'auto' or array-like. Got %r" % precompute)
+            raise ValueError(
+                "Precompute should be one of True, False, 'auto' or array-like. Got %r"
+                % precompute
+            )
         coef_, dual_gap_, eps_, n_iter_ = model
         coefs[..., i] = coef_
         # we correct the scale of the returned dual gap, as the objective
@@ -545,9 +711,9 @@ def enet_path(X, y, *, l1_ratio=0.5, eps=1e-3, n_alphas=100, alphas=None,
             if verbose > 2:
                 print(model)
             elif verbose > 1:
-                print('Path: %03i out of %03i' % (i, n_alphas))
+                print("Path: %03i out of %03i" % (i, n_alphas))
             else:
-                sys.stderr.write('.')
+                sys.stderr.write(".")
 
     if return_n_iter:
         return alphas, coefs, dual_gaps, n_iters
@@ -570,7 +736,7 @@ class ElasticNet(MultiOutputMixin, RegressorMixin, LinearModel):
     If you are interested in controlling the L1 and L2 penalty
     separately, keep in mind that this is equivalent to::
 
-            a * L1 + b * L2
+            a * ||w||_1 + 0.5 * b * ||w||_2^2
 
     where::
 
@@ -603,19 +769,14 @@ class ElasticNet(MultiOutputMixin, RegressorMixin, LinearModel):
         Whether the intercept should be estimated or not. If ``False``, the
         data is assumed to be already centered.
 
-    normalize : bool, default=False
-        This parameter is ignored when ``fit_intercept`` is set to False.
-        If True, the regressors X will be normalized before regression by
-        subtracting the mean and dividing by the l2-norm.
-        If you wish to standardize, please use
-        :class:`~sklearn.preprocessing.StandardScaler` before calling ``fit``
-        on an estimator with ``normalize=False``.
-
     precompute : bool or array-like of shape (n_features, n_features),\
                  default=False
         Whether to use a precomputed Gram matrix to speed up
         calculations. The Gram matrix can also be passed as argument.
         For sparse input this option is always ``False`` to preserve sparsity.
+        Check :ref:`an example on how to use a precomputed Gram Matrix in ElasticNet
+        `
+        for details.
 
     max_iter : int, default=1000
         The maximum number of iterations.
@@ -627,7 +788,7 @@ class ElasticNet(MultiOutputMixin, RegressorMixin, LinearModel):
         The tolerance for the optimization: if the updates are
         smaller than ``tol``, the optimization code checks the
         dual gap for optimality and continues until it is smaller
-        than ``tol``.
+        than ``tol``, see Notes below.
 
     warm_start : bool, default=False
         When set to ``True``, reuse the solution of the previous call to fit as
@@ -655,7 +816,7 @@ class ElasticNet(MultiOutputMixin, RegressorMixin, LinearModel):
         Parameter vector (w in the cost function formula).
 
     sparse_coef_ : sparse matrix of shape (n_features,) or \
-            (n_tasks, n_features)
+            (n_targets, n_features)
         Sparse representation of the `coef_`.
 
     intercept_ : float or ndarray of shape (n_targets,)
@@ -669,6 +830,36 @@ class ElasticNet(MultiOutputMixin, RegressorMixin, LinearModel):
         Given param alpha, the dual gaps at the end of the optimization,
         same shape as each observation of y.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    ElasticNetCV : Elastic net model with best model selection by
+        cross-validation.
+    SGDRegressor : Implements elastic net regression with incremental training.
+    SGDClassifier : Implements logistic regression with elastic net penalty
+        (``SGDClassifier(loss="log_loss", penalty="elasticnet")``).
+
+    Notes
+    -----
+    To avoid unnecessary memory duplication the X argument of the fit method
+    should be directly passed as a Fortran-contiguous numpy array.
+
+    The precise stopping criteria based on `tol` are the following: First, check that
+    that maximum coordinate update, i.e. :math:`\\max_j |w_j^{new} - w_j^{old}|`
+    is smaller than `tol` times the maximum absolute coefficient, :math:`\\max_j |w_j|`.
+    If so, then additionally check whether the dual gap is smaller than `tol` times
+    :math:`||y||_2^2 / n_{\text{samples}}`.
+
     Examples
     --------
     >>> from sklearn.linear_model import ElasticNet
@@ -681,35 +872,49 @@ class ElasticNet(MultiOutputMixin, RegressorMixin, LinearModel):
     >>> print(regr.coef_)
     [18.83816048 64.55968825]
     >>> print(regr.intercept_)
-    1.451...
+    1.451
     >>> print(regr.predict([[0, 0]]))
-    [1.451...]
+    [1.451]
+    """
 
+    # "check_input" is used for optimisation and isn't something to be passed
+    # around in a pipeline.
+    __metadata_request__fit = {"check_input": metadata_routing.UNUSED}
+
+    _parameter_constraints: dict = {
+        "alpha": [Interval(Real, 0, None, closed="left")],
+        "l1_ratio": [Interval(Real, 0, 1, closed="both")],
+        "fit_intercept": ["boolean"],
+        "precompute": ["boolean", "array-like"],
+        "max_iter": [Interval(Integral, 1, None, closed="left"), None],
+        "copy_X": ["boolean"],
+        "tol": [Interval(Real, 0, None, closed="left")],
+        "warm_start": ["boolean"],
+        "positive": ["boolean"],
+        "random_state": ["random_state"],
+        "selection": [StrOptions({"cyclic", "random"})],
+    }
 
-    Notes
-    -----
-    To avoid unnecessary memory duplication the X argument of the fit method
-    should be directly passed as a Fortran-contiguous numpy array.
-
-    See Also
-    --------
-    ElasticNetCV : Elastic net model with best model selection by
-        cross-validation.
-    SGDRegressor : Implements elastic net regression with incremental training.
-    SGDClassifier : Implements logistic regression with elastic net penalty
-        (``SGDClassifier(loss="log", penalty="elasticnet")``).
-    """
     path = staticmethod(enet_path)
 
-    @_deprecate_positional_args
-    def __init__(self, alpha=1.0, *, l1_ratio=0.5, fit_intercept=True,
-                 normalize=False, precompute=False, max_iter=1000,
-                 copy_X=True, tol=1e-4, warm_start=False, positive=False,
-                 random_state=None, selection='cyclic'):
+    def __init__(
+        self,
+        alpha=1.0,
+        *,
+        l1_ratio=0.5,
+        fit_intercept=True,
+        precompute=False,
+        max_iter=1000,
+        copy_X=True,
+        tol=1e-4,
+        warm_start=False,
+        positive=False,
+        random_state=None,
+        selection="cyclic",
+    ):
         self.alpha = alpha
         self.l1_ratio = l1_ratio
         self.fit_intercept = fit_intercept
-        self.normalize = normalize
         self.precompute = precompute
         self.max_iter = max_iter
         self.copy_X = copy_X
@@ -719,20 +924,23 @@ def __init__(self, alpha=1.0, *, l1_ratio=0.5, fit_intercept=True,
         self.random_state = random_state
         self.selection = selection
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y, sample_weight=None, check_input=True):
         """Fit model with coordinate descent.
 
         Parameters
         ----------
-        X : {ndarray, sparse matrix} of (n_samples, n_features)
+        X : {ndarray, sparse matrix, sparse array} of (n_samples, n_features)
             Data.
 
-        y : {ndarray, sparse matrix} of shape (n_samples,) or \
-            (n_samples, n_targets)
+            Note that large sparse matrices and arrays requiring `int64`
+            indices are not accepted.
+
+        y : ndarray of shape (n_samples,) or (n_samples, n_targets)
             Target. Will be cast to X's dtype if necessary.
 
         sample_weight : float or array-like of shape (n_samples,), default=None
-            Sample weight. Internally, the `sample_weight` vector will be
+            Sample weights. Internally, the `sample_weight` vector will be
             rescaled to sum to `n_samples`.
 
             .. versionadded:: 0.23
@@ -741,9 +949,13 @@ def fit(self, X, y, sample_weight=None, check_input=True):
             Allow to bypass several input checking.
             Don't use this parameter unless you know what you do.
 
+        Returns
+        -------
+        self : object
+            Fitted estimator.
+
         Notes
         -----
-
         Coordinate descent is an algorithm that considers each column of
         data at a time hence it will automatically convert the X input
         as a Fortran-contiguous numpy array if necessary.
@@ -751,20 +963,15 @@ def fit(self, X, y, sample_weight=None, check_input=True):
         To avoid memory re-allocation it is advised to allocate the
         initial data in memory directly using that format.
         """
-
         if self.alpha == 0:
-            warnings.warn("With alpha=0, this algorithm does not converge "
-                          "well. You are advised to use the LinearRegression "
-                          "estimator", stacklevel=2)
-
-        if isinstance(self.precompute, str):
-            raise ValueError('precompute should be one of True, False or'
-                             ' array-like. Got %r' % self.precompute)
-
-        if (not isinstance(self.l1_ratio, numbers.Number) or
-                self.l1_ratio < 0 or self.l1_ratio > 1):
-            raise ValueError("l1_ratio must be between 0 and 1; "
-                             f"got l1_ratio={self.l1_ratio}")
+            warnings.warn(
+                (
+                    "With alpha=0, this algorithm does not converge "
+                    "well. You are advised to use the LinearRegression "
+                    "estimator"
+                ),
+                stacklevel=2,
+            )
 
         # Remember if X is copied
         X_copied = False
@@ -772,13 +979,22 @@ def fit(self, X, y, sample_weight=None, check_input=True):
         # when bypassing checks
         if check_input:
             X_copied = self.copy_X and self.fit_intercept
-            X, y = self._validate_data(X, y, accept_sparse='csc',
-                                       order='F',
-                                       dtype=[np.float64, np.float32],
-                                       copy=X_copied, multi_output=True,
-                                       y_numeric=True)
-            y = check_array(y, order='F', copy=False, dtype=X.dtype.type,
-                            ensure_2d=False)
+            X, y = validate_data(
+                self,
+                X,
+                y,
+                accept_sparse="csc",
+                order="F",
+                dtype=[np.float64, np.float32],
+                force_writeable=True,
+                accept_large_sparse=False,
+                copy=X_copied,
+                multi_output=True,
+                y_numeric=True,
+            )
+            y = check_array(
+                y, order="F", copy=False, dtype=X.dtype.type, ensure_2d=False
+            )
 
         n_samples, n_features = X.shape
         alpha = self.alpha
@@ -787,37 +1003,58 @@ def fit(self, X, y, sample_weight=None, check_input=True):
             sample_weight = None
         if sample_weight is not None:
             if check_input:
-                if sparse.issparse(X):
-                    raise ValueError("Sample weights do not (yet) support "
-                                     "sparse matrices.")
-                sample_weight = _check_sample_weight(sample_weight, X,
-                                                     dtype=X.dtype)
-            # simplify things by rescaling sw to sum up to n_samples
-            # => np.average(x, weights=sw) = np.mean(sw * x)
+                sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype)
+            # TLDR: Rescale sw to sum up to n_samples.
+            # Long: The objective function of Enet
+            #
+            #    1/2 * np.average(squared error, weights=sw)
+            #    + alpha * penalty                                             (1)
+            #
+            # is invariant under rescaling of sw.
+            # But enet_path coordinate descent minimizes
+            #
+            #     1/2 * sum(squared error) + alpha' * penalty                  (2)
+            #
+            # and therefore sets
+            #
+            #     alpha' = n_samples * alpha                                   (3)
+            #
+            # inside its function body, which results in objective (2) being
+            # equivalent to (1) in case of no sw.
+            # With sw, however, enet_path should set
+            #
+            #     alpha' = sum(sw) * alpha                                     (4)
+            #
+            # Therefore, we use the freedom of Eq. (1) to rescale sw before
+            # calling enet_path, i.e.
+            #
+            #     sw *= n_samples / sum(sw)
+            #
+            # such that sum(sw) = n_samples. This way, (3) and (4) are the same.
             sample_weight = sample_weight * (n_samples / np.sum(sample_weight))
-            # Objective function is:
-            # 1/2 * np.average(squared error, weights=sw) + alpha * penalty
-            # but coordinate descent minimizes:
-            # 1/2 * sum(squared error) + alpha * penalty
-            # enet_path therefore sets alpha = n_samples * alpha
-            # With sw, enet_path should set alpha = sum(sw) * alpha
-            # Therefore, we rescale alpha = sum(sw) / n_samples * alpha
-            # Note: As we rescaled sample_weights to sum up to n_samples,
-            #       we don't need this
-            # alpha *= np.sum(sample_weight) / n_samples
+            # Note: Alternatively, we could also have rescaled alpha instead
+            # of sample_weight:
+            #
+            #     alpha *= np.sum(sample_weight) / n_samples
 
         # Ensure copying happens only once, don't do it again if done above.
         # X and y will be rescaled if sample_weight is not None, order='F'
         # ensures that the returned X and y are still F-contiguous.
         should_copy = self.copy_X and not X_copied
-        X, y, X_offset, y_offset, X_scale, precompute, Xy = \
-            _pre_fit(X, y, None, self.precompute, self.normalize,
-                     self.fit_intercept, copy=should_copy,
-                     check_input=check_input, sample_weight=sample_weight)
+        X, y, X_offset, y_offset, X_scale, precompute, Xy = _pre_fit(
+            X,
+            y,
+            None,
+            self.precompute,
+            fit_intercept=self.fit_intercept,
+            copy=should_copy,
+            check_input=check_input,
+            sample_weight=sample_weight,
+        )
         # coordinate descent needs F-ordered arrays and _pre_fit might have
         # called _rescale_data
         if check_input or sample_weight is not None:
-            X, y = _set_order(X, y, order='F')
+            X, y = _set_order(X, y, order="F")
         if y.ndim == 1:
             y = y[:, np.newaxis]
         if Xy is not None and Xy.ndim == 1:
@@ -825,12 +1062,8 @@ def fit(self, X, y, sample_weight=None, check_input=True):
 
         n_targets = y.shape[1]
 
-        if self.selection not in ['cyclic', 'random']:
-            raise ValueError("selection should be either random or cyclic.")
-
         if not self.warm_start or not hasattr(self, "coef_"):
-            coef_ = np.zeros((n_targets, n_features), dtype=X.dtype,
-                             order='F')
+            coef_ = np.zeros((n_targets, n_features), dtype=X.dtype, order="F")
         else:
             coef_ = self.coef_
             if coef_.ndim == 1:
@@ -844,19 +1077,30 @@ def fit(self, X, y, sample_weight=None, check_input=True):
                 this_Xy = Xy[:, k]
             else:
                 this_Xy = None
-            _, this_coef, this_dual_gap, this_iter = \
-                self.path(X, y[:, k],
-                          l1_ratio=self.l1_ratio, eps=None,
-                          n_alphas=None, alphas=[alpha],
-                          precompute=precompute, Xy=this_Xy,
-                          fit_intercept=False, normalize=False, copy_X=True,
-                          verbose=False, tol=self.tol, positive=self.positive,
-                          X_offset=X_offset, X_scale=X_scale,
-                          return_n_iter=True, coef_init=coef_[k],
-                          max_iter=self.max_iter,
-                          random_state=self.random_state,
-                          selection=self.selection,
-                          check_input=False)
+            _, this_coef, this_dual_gap, this_iter = self.path(
+                X,
+                y[:, k],
+                l1_ratio=self.l1_ratio,
+                eps=None,
+                n_alphas=None,
+                alphas=[alpha],
+                precompute=precompute,
+                Xy=this_Xy,
+                copy_X=True,
+                coef_init=coef_[k],
+                verbose=False,
+                return_n_iter=True,
+                positive=self.positive,
+                check_input=False,
+                # from here on **params
+                tol=self.tol,
+                X_offset=X_offset,
+                X_scale=X_scale,
+                max_iter=self.max_iter,
+                random_state=self.random_state,
+                selection=self.selection,
+                sample_weight=sample_weight,
+            )
             coef_[k] = this_coef[:, 0]
             dual_gaps_[k] = this_dual_gap[0]
             self.n_iter_.append(this_iter[0])
@@ -871,8 +1115,13 @@ def fit(self, X, y, sample_weight=None, check_input=True):
 
         self._set_intercept(X_offset, y_offset, X_scale)
 
-        # workaround since _set_intercept will cast self.coef_ into X.dtype
-        self.coef_ = np.asarray(self.coef_, dtype=X.dtype)
+        # check for finiteness of coefficients
+        if not all(np.isfinite(w).all() for w in [self.coef_, self.intercept_]):
+            raise ValueError(
+                "Coordinate descent iterations resulted in non-finite parameter"
+                " values. The input data may contain large values and need to"
+                " be preprocessed."
+            )
 
         # return self for chaining fit and predict calls
         return self
@@ -895,18 +1144,23 @@ def _decision_function(self, X):
             The predicted decision function.
         """
         check_is_fitted(self)
-        if sparse.isspmatrix(X):
-            return safe_sparse_dot(X, self.coef_.T,
-                                   dense_output=True) + self.intercept_
+        if sparse.issparse(X):
+            return safe_sparse_dot(X, self.coef_.T, dense_output=True) + self.intercept_
         else:
             return super()._decision_function(X)
 
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
+        return tags
+
 
 ###############################################################################
 # Lasso model
 
+
 class Lasso(ElasticNet):
-    """Linear Model trained with L1 prior as regularizer (aka the Lasso)
+    """Linear Model trained with L1 prior as regularizer (aka the Lasso).
 
     The optimization objective for Lasso is::
 
@@ -920,25 +1174,19 @@ class Lasso(ElasticNet):
     Parameters
     ----------
     alpha : float, default=1.0
-        Constant that multiplies the L1 term. Defaults to 1.0.
-        ``alpha = 0`` is equivalent to an ordinary least square, solved
-        by the :class:`LinearRegression` object. For numerical
-        reasons, using ``alpha = 0`` with the ``Lasso`` object is not advised.
-        Given this, you should use the :class:`LinearRegression` object.
+        Constant that multiplies the L1 term, controlling regularization
+        strength. `alpha` must be a non-negative float i.e. in `[0, inf)`.
+
+        When `alpha = 0`, the objective is equivalent to ordinary least
+        squares, solved by the :class:`LinearRegression` object. For numerical
+        reasons, using `alpha = 0` with the `Lasso` object is not advised.
+        Instead, you should use the :class:`LinearRegression` object.
 
     fit_intercept : bool, default=True
         Whether to calculate the intercept for this model. If set
         to False, no intercept will be used in calculations
         (i.e. data is expected to be centered).
 
-    normalize : bool, default=False
-        This parameter is ignored when ``fit_intercept`` is set to False.
-        If True, the regressors X will be normalized before regression by
-        subtracting the mean and dividing by the l2-norm.
-        If you wish to standardize, please use
-        :class:`~sklearn.preprocessing.StandardScaler` before calling ``fit``
-        on an estimator with ``normalize=False``.
-
     precompute : bool or array-like of shape (n_features, n_features),\
                  default=False
         Whether to use a precomputed Gram matrix to speed up
@@ -955,7 +1203,7 @@ class Lasso(ElasticNet):
         The tolerance for the optimization: if the updates are
         smaller than ``tol``, the optimization code checks the
         dual gap for optimality and continues until it is smaller
-        than ``tol``.
+        than ``tol``, see Notes below.
 
     warm_start : bool, default=False
         When set to True, reuse the solution of the previous call to fit as
@@ -997,25 +1245,25 @@ class Lasso(ElasticNet):
         Number of iterations run by the coordinate descent solver to reach
         the specified tolerance.
 
-    Examples
-    --------
-    >>> from sklearn import linear_model
-    >>> clf = linear_model.Lasso(alpha=0.1)
-    >>> clf.fit([[0,0], [1, 1], [2, 2]], [0, 1, 2])
-    Lasso(alpha=0.1)
-    >>> print(clf.coef_)
-    [0.85 0.  ]
-    >>> print(clf.intercept_)
-    0.15...
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
 
     See Also
     --------
-    lars_path
-    lasso_path
-    LassoLars
-    LassoCV
-    LassoLarsCV
-    sklearn.decomposition.sparse_encode
+    lars_path : Regularization path using LARS.
+    lasso_path : Regularization path using Lasso.
+    LassoLars : Lasso Path along the regularization parameter using LARS algorithm.
+    LassoCV : Lasso alpha parameter by cross-validation.
+    LassoLarsCV : Lasso least angle parameter algorithm by cross-validation.
+    sklearn.decomposition.sparse_encode : Sparse coding array estimator.
 
     Notes
     -----
@@ -1023,27 +1271,95 @@ class Lasso(ElasticNet):
 
     To avoid unnecessary memory duplication the X argument of the fit method
     should be directly passed as a Fortran-contiguous numpy array.
+
+    Regularization improves the conditioning of the problem and
+    reduces the variance of the estimates. Larger values specify stronger
+    regularization. Alpha corresponds to `1 / (2C)` in other linear
+    models such as :class:`~sklearn.linear_model.LogisticRegression` or
+    :class:`~sklearn.svm.LinearSVC`.
+
+    The precise stopping criteria based on `tol` are the following: First, check that
+    that maximum coordinate update, i.e. :math:`\\max_j |w_j^{new} - w_j^{old}|`
+    is smaller than `tol` times the maximum absolute coefficient, :math:`\\max_j |w_j|`.
+    If so, then additionally check whether the dual gap is smaller than `tol` times
+    :math:`||y||_2^2 / n_{\\text{samples}}`.
+
+    The target can be a 2-dimensional array, resulting in the optimization of the
+    following objective::
+
+        (1 / (2 * n_samples)) * ||Y - XW||^2_F + alpha * ||W||_11
+
+    where :math:`||W||_{1,1}` is the sum of the magnitude of the matrix coefficients.
+    It should not be confused with :class:`~sklearn.linear_model.MultiTaskLasso` which
+    instead penalizes the :math:`L_{2,1}` norm of the coefficients, yielding row-wise
+    sparsity in the coefficients.
+
+    Examples
+    --------
+    >>> from sklearn import linear_model
+    >>> clf = linear_model.Lasso(alpha=0.1)
+    >>> clf.fit([[0,0], [1, 1], [2, 2]], [0, 1, 2])
+    Lasso(alpha=0.1)
+    >>> print(clf.coef_)
+    [0.85 0.  ]
+    >>> print(clf.intercept_)
+    0.15
     """
+
+    _parameter_constraints: dict = {
+        **ElasticNet._parameter_constraints,
+    }
+    _parameter_constraints.pop("l1_ratio")
+
     path = staticmethod(enet_path)
 
-    @_deprecate_positional_args
-    def __init__(self, alpha=1.0, *, fit_intercept=True, normalize=False,
-                 precompute=False, copy_X=True, max_iter=1000,
-                 tol=1e-4, warm_start=False, positive=False,
-                 random_state=None, selection='cyclic'):
+    def __init__(
+        self,
+        alpha=1.0,
+        *,
+        fit_intercept=True,
+        precompute=False,
+        copy_X=True,
+        max_iter=1000,
+        tol=1e-4,
+        warm_start=False,
+        positive=False,
+        random_state=None,
+        selection="cyclic",
+    ):
         super().__init__(
-            alpha=alpha, l1_ratio=1.0, fit_intercept=fit_intercept,
-            normalize=normalize, precompute=precompute, copy_X=copy_X,
-            max_iter=max_iter, tol=tol, warm_start=warm_start,
-            positive=positive, random_state=random_state,
-            selection=selection)
+            alpha=alpha,
+            l1_ratio=1.0,
+            fit_intercept=fit_intercept,
+            precompute=precompute,
+            copy_X=copy_X,
+            max_iter=max_iter,
+            tol=tol,
+            warm_start=warm_start,
+            positive=positive,
+            random_state=random_state,
+            selection=selection,
+        )
 
 
 ###############################################################################
 # Functions for CV with paths functions
 
-def _path_residuals(X, y, train, test, path, path_params, alphas=None,
-                    l1_ratio=1, X_order=None, dtype=None):
+
+def _path_residuals(
+    X,
+    y,
+    sample_weight,
+    train,
+    test,
+    fit_intercept,
+    path,
+    path_params,
+    alphas=None,
+    l1_ratio=1,
+    X_order=None,
+    dtype=None,
+):
     """Returns the MSE for the models computed by 'path'.
 
     Parameters
@@ -1054,6 +1370,9 @@ def _path_residuals(X, y, train, test, path, path_params, alphas=None,
     y : array-like of shape (n_samples,) or (n_samples, n_targets)
         Target values.
 
+    sample_weight : None or array-like of shape (n_samples,)
+        Sample weights.
+
     train : list of indices
         The indices of the train set.
 
@@ -1089,44 +1408,65 @@ def _path_residuals(X, y, train, test, path, path_params, alphas=None,
     y_train = y[train]
     X_test = X[test]
     y_test = y[test]
+    if sample_weight is None:
+        sw_train, sw_test = None, None
+    else:
+        sw_train = sample_weight[train]
+        sw_test = sample_weight[test]
+        n_samples = X_train.shape[0]
+        # TLDR: Rescale sw_train to sum up to n_samples on the training set.
+        # See TLDR and long comment inside ElasticNet.fit.
+        sw_train *= n_samples / np.sum(sw_train)
+        # Note: Alternatively, we could also have rescaled alpha instead
+        # of sample_weight:
+        #
+        #     alpha *= np.sum(sample_weight) / n_samples
 
     if not sparse.issparse(X):
-        for array, array_input in ((X_train, X), (y_train, y),
-                                   (X_test, X), (y_test, y)):
-            if array.base is not array_input and not array.flags['WRITEABLE']:
+        for array, array_input in (
+            (X_train, X),
+            (y_train, y),
+            (X_test, X),
+            (y_test, y),
+        ):
+            if array.base is not array_input and not array.flags["WRITEABLE"]:
                 # fancy indexing should create a writable copy but it doesn't
                 # for read-only memmaps (cf. numpy#14132).
                 array.setflags(write=True)
 
-    fit_intercept = path_params['fit_intercept']
-    normalize = path_params['normalize']
-
     if y.ndim == 1:
-        precompute = path_params['precompute']
+        precompute = path_params["precompute"]
     else:
         # No Gram variant of multi-task exists right now.
         # Fall back to default enet_multitask
         precompute = False
 
-    X_train, y_train, X_offset, y_offset, X_scale, precompute, Xy = \
-        _pre_fit(X_train, y_train, None, precompute, normalize, fit_intercept,
-                 copy=False)
+    X_train, y_train, X_offset, y_offset, X_scale, precompute, Xy = _pre_fit(
+        X_train,
+        y_train,
+        None,
+        precompute,
+        fit_intercept=fit_intercept,
+        copy=False,
+        sample_weight=sw_train,
+    )
 
     path_params = path_params.copy()
-    path_params['Xy'] = Xy
-    path_params['X_offset'] = X_offset
-    path_params['X_scale'] = X_scale
-    path_params['precompute'] = precompute
-    path_params['copy_X'] = False
-    path_params['alphas'] = alphas
-
-    if 'l1_ratio' in path_params:
-        path_params['l1_ratio'] = l1_ratio
+    path_params["Xy"] = Xy
+    path_params["X_offset"] = X_offset
+    path_params["X_scale"] = X_scale
+    path_params["precompute"] = precompute
+    path_params["copy_X"] = False
+    path_params["alphas"] = alphas
+    # needed for sparse cd solver
+    path_params["sample_weight"] = sw_train
+
+    if "l1_ratio" in path_params:
+        path_params["l1_ratio"] = l1_ratio
 
     # Do the ordering and type casting here, as if it is done in the path,
     # X is copied and a reference is kept here
-    X_train = check_array(X_train, accept_sparse='csc', dtype=dtype,
-                          order=X_order)
+    X_train = check_array(X_train, accept_sparse="csc", dtype=dtype, order=X_order)
     alphas, coefs, _ = path(X_train, y_train, **path_params)
     del X_train, y_train
 
@@ -1136,32 +1476,69 @@ def _path_residuals(X, y, train, test, path, path_params, alphas=None,
         y_offset = np.atleast_1d(y_offset)
         y_test = y_test[:, np.newaxis]
 
-    if normalize:
-        nonzeros = np.flatnonzero(X_scale)
-        coefs[:, nonzeros] /= X_scale[nonzeros][:, np.newaxis]
-
     intercepts = y_offset[:, np.newaxis] - np.dot(X_offset, coefs)
     X_test_coefs = safe_sparse_dot(X_test, coefs)
     residues = X_test_coefs - y_test[:, :, np.newaxis]
     residues += intercepts
-    this_mses = ((residues ** 2).mean(axis=0)).mean(axis=0)
+    if sample_weight is None:
+        this_mse = (residues**2).mean(axis=0)
+    else:
+        this_mse = np.average(residues**2, weights=sw_test, axis=0)
 
-    return this_mses
+    return this_mse.mean(axis=0)
 
 
-class LinearModelCV(MultiOutputMixin, LinearModel, metaclass=ABCMeta):
+class LinearModelCV(MultiOutputMixin, LinearModel, ABC):
     """Base class for iterative model fitting along a regularization path."""
 
+    _parameter_constraints: dict = {
+        "eps": [Interval(Real, 0, None, closed="neither")],
+        "n_alphas": [
+            Interval(Integral, 1, None, closed="left"),
+            Hidden(StrOptions({"deprecated"})),
+        ],
+        # TODO(1.9): remove "warn" and None options.
+        "alphas": [
+            Interval(Integral, 1, None, closed="left"),
+            "array-like",
+            None,
+            Hidden(StrOptions({"warn"})),
+        ],
+        "fit_intercept": ["boolean"],
+        "precompute": [StrOptions({"auto"}), "array-like", "boolean"],
+        "max_iter": [Interval(Integral, 1, None, closed="left")],
+        "tol": [Interval(Real, 0, None, closed="left")],
+        "copy_X": ["boolean"],
+        "cv": ["cv_object"],
+        "verbose": ["verbose"],
+        "n_jobs": [Integral, None],
+        "positive": ["boolean"],
+        "random_state": ["random_state"],
+        "selection": [StrOptions({"cyclic", "random"})],
+    }
+
     @abstractmethod
-    def __init__(self, eps=1e-3, n_alphas=100, alphas=None, fit_intercept=True,
-                 normalize=False, precompute='auto', max_iter=1000, tol=1e-4,
-                 copy_X=True, cv=None, verbose=False, n_jobs=None,
-                 positive=False, random_state=None, selection='cyclic'):
+    def __init__(
+        self,
+        eps=1e-3,
+        n_alphas="deprecated",
+        alphas="warn",
+        fit_intercept=True,
+        precompute="auto",
+        max_iter=1000,
+        tol=1e-4,
+        copy_X=True,
+        cv=None,
+        verbose=False,
+        n_jobs=None,
+        positive=False,
+        random_state=None,
+        selection="cyclic",
+    ):
         self.eps = eps
         self.n_alphas = n_alphas
         self.alphas = alphas
         self.fit_intercept = fit_intercept
-        self.normalize = normalize
         self.precompute = precompute
         self.max_iter = max_iter
         self.tol = tol
@@ -1181,7 +1558,13 @@ def _get_estimator(self):
     def _is_multitask(self):
         """Bool indicating if class is meant for multidimensional target."""
 
-    def fit(self, X, y):
+    @staticmethod
+    @abstractmethod
+    def path(X, y, **kwargs):
+        """Compute path with coordinate descent."""
+
+    @_fit_context(prefer_skip_nested_validation=True)
+    def fit(self, X, y, sample_weight=None, **params):
         """Fit linear model with coordinate descent.
 
         Fit is on grid of alphas and best alpha estimated by cross-validation.
@@ -1191,20 +1574,80 @@ def fit(self, X, y):
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
             Training data. Pass directly as Fortran-contiguous data
             to avoid unnecessary memory duplication. If y is mono-output,
-            X can be sparse.
+            X can be sparse. Note that large sparse matrices and arrays
+            requiring `int64` indices are not accepted.
 
         y : array-like of shape (n_samples,) or (n_samples, n_targets)
             Target values.
+
+        sample_weight : float or array-like of shape (n_samples,), \
+                default=None
+            Sample weights used for fitting and evaluation of the weighted
+            mean squared error of each cv-fold. Note that the cross validated
+            MSE that is finally used to find the best model is the unweighted
+            mean over the (weighted) MSEs of each test fold.
+
+        **params : dict, default=None
+            Parameters to be passed to the CV splitter.
+
+            .. versionadded:: 1.4
+                Only available if `enable_metadata_routing=True`,
+                which can be set by using
+                ``sklearn.set_config(enable_metadata_routing=True)``.
+                See :ref:`Metadata Routing User Guide ` for
+                more details.
+
+        Returns
+        -------
+        self : object
+            Returns an instance of fitted model.
         """
+        _raise_for_params(params, self, "fit")
+
+        # TODO(1.9): remove n_alphas and alphas={"warn", None}; set alphas=100 by
+        # default. Remove these deprecations messages and use self.alphas directly
+        # instead of self._alphas.
+        if self.n_alphas == "deprecated":
+            self._alphas = 100
+        else:
+            warnings.warn(
+                "'n_alphas' was deprecated in 1.7 and will be removed in 1.9. "
+                "'alphas' now accepts an integer value which removes the need to pass "
+                "'n_alphas'. The default value of 'alphas' will change from None to "
+                "100 in 1.9. Pass an explicit value to 'alphas' and leave 'n_alphas' "
+                "to its default value to silence this warning.",
+                FutureWarning,
+            )
+            self._alphas = self.n_alphas
+
+        if isinstance(self.alphas, str) and self.alphas == "warn":
+            # - If self.n_alphas == "deprecated", both are left to their default values
+            #   so we don't warn since the future default behavior will be the same as
+            #   the current default behavior.
+            # - If self.n_alphas != "deprecated", then we already warned about it
+            #   and the warning message mentions the future self.alphas default, so
+            #   no need to warn a second time.
+            pass
+        elif self.alphas is None:
+            warnings.warn(
+                "'alphas=None' is deprecated and will be removed in 1.9, at which "
+                "point the default value will be set to 100. Set 'alphas=100' "
+                "to silence this warning.",
+                FutureWarning,
+            )
+        else:
+            self._alphas = self.alphas
+
         # This makes sure that there is no duplication in memory.
         # Dealing right with copy_X is important in the following:
         # Multiple functions touch X and subsamples of X and can induce a
         # lot of duplication of memory
         copy_X = self.copy_X and self.fit_intercept
 
-        check_y_params = dict(copy=False, dtype=[np.float64, np.float32],
-                              ensure_2d=False)
-        if isinstance(X, np.ndarray) or sparse.isspmatrix(X):
+        check_y_params = dict(
+            copy=False, dtype=[np.float64, np.float32], ensure_2d=False
+        )
+        if isinstance(X, np.ndarray) or sparse.issparse(X):
             # Keep a reference to X
             reference_to_old_X = X
             # Let us not impose fortran ordering so far: it is
@@ -1212,17 +1655,23 @@ def fit(self, X, y):
             # by the model fitting itself
 
             # Need to validate separately here.
-            # We can't pass multi_ouput=True because that would allow y to be
+            # We can't pass multi_output=True because that would allow y to be
             # csr. We also want to allow y to be 64 or 32 but check_X_y only
             # allows to convert for 64.
-            check_X_params = dict(accept_sparse='csc',
-                                  dtype=[np.float64, np.float32], copy=False)
-            X, y = self._validate_data(X, y,
-                                       validate_separately=(check_X_params,
-                                                            check_y_params))
-            if sparse.isspmatrix(X):
-                if (hasattr(reference_to_old_X, "data") and
-                   not np.may_share_memory(reference_to_old_X.data, X.data)):
+            check_X_params = dict(
+                accept_sparse="csc",
+                dtype=[np.float64, np.float32],
+                force_writeable=True,
+                copy=False,
+                accept_large_sparse=False,
+            )
+            X, y = validate_data(
+                self, X, y, validate_separately=(check_X_params, check_y_params)
+            )
+            if sparse.issparse(X):
+                if hasattr(reference_to_old_X, "data") and not np.may_share_memory(
+                    reference_to_old_X.data, X.data
+                ):
                     # X is a sparse matrix and has been copied
                     copy_X = False
             elif not np.may_share_memory(reference_to_old_X, X):
@@ -1231,96 +1680,166 @@ def fit(self, X, y):
             del reference_to_old_X
         else:
             # Need to validate separately here.
-            # We can't pass multi_ouput=True because that would allow y to be
+            # We can't pass multi_output=True because that would allow y to be
             # csr. We also want to allow y to be 64 or 32 but check_X_y only
             # allows to convert for 64.
-            check_X_params = dict(accept_sparse='csc',
-                                  dtype=[np.float64, np.float32], order='F',
-                                  copy=copy_X)
-            X, y = self._validate_data(X, y,
-                                       validate_separately=(check_X_params,
-                                                            check_y_params))
+            check_X_params = dict(
+                accept_sparse="csc",
+                dtype=[np.float64, np.float32],
+                order="F",
+                force_writeable=True,
+                copy=copy_X,
+            )
+            X, y = validate_data(
+                self, X, y, validate_separately=(check_X_params, check_y_params)
+            )
             copy_X = False
 
-        if y.shape[0] == 0:
-            raise ValueError("y has 0 samples: %r" % y)
+        check_consistent_length(X, y)
 
         if not self._is_multitask():
             if y.ndim > 1 and y.shape[1] > 1:
-                raise ValueError("For multi-task outputs, use "
-                                 "MultiTask%s" % self.__class__.__name__)
+                raise ValueError(
+                    "For multi-task outputs, use MultiTask%s" % self.__class__.__name__
+                )
             y = column_or_1d(y, warn=True)
         else:
-            if sparse.isspmatrix(X):
-                raise TypeError("X should be dense but a sparse matrix was"
-                                "passed")
+            if sparse.issparse(X):
+                raise TypeError("X should be dense but a sparse matrix waspassed")
             elif y.ndim == 1:
-                raise ValueError("For mono-task outputs, use "
-                                 "%sCV" % self.__class__.__name__[9:])
-
-        model = self._get_estimator()
+                raise ValueError(
+                    "For mono-task outputs, use %sCV" % self.__class__.__name__[9:]
+                )
 
-        if self.selection not in ["random", "cyclic"]:
-            raise ValueError("selection should be either random or cyclic.")
+        if isinstance(sample_weight, numbers.Number):
+            sample_weight = None
+        if sample_weight is not None:
+            sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype)
 
-        if X.shape[0] != y.shape[0]:
-            raise ValueError("X and y have inconsistent dimensions (%d != %d)"
-                             % (X.shape[0], y.shape[0]))
+        model = self._get_estimator()
 
         # All LinearModelCV parameters except 'cv' are acceptable
         path_params = self.get_params()
-        if 'l1_ratio' in path_params:
-            l1_ratios = np.atleast_1d(path_params['l1_ratio'])
+
+        # Pop `intercept` that is not parameter of the path function
+        path_params.pop("fit_intercept", None)
+
+        if "l1_ratio" in path_params:
+            l1_ratios = np.atleast_1d(path_params["l1_ratio"])
             # For the first path, we need to set l1_ratio
-            path_params['l1_ratio'] = l1_ratios[0]
+            path_params["l1_ratio"] = l1_ratios[0]
         else:
-            l1_ratios = [1, ]
-        path_params.pop('cv', None)
-        path_params.pop('n_jobs', None)
+            l1_ratios = [
+                1,
+            ]
+        path_params.pop("cv", None)
+        path_params.pop("n_jobs", None)
 
-        alphas = self.alphas
         n_l1_ratio = len(l1_ratios)
-        if alphas is None:
-            alphas = [_alpha_grid(X, y, l1_ratio=l1_ratio,
-                                  fit_intercept=self.fit_intercept,
-                                  eps=self.eps, n_alphas=self.n_alphas,
-                                  normalize=self.normalize, copy_X=self.copy_X)
-                      for l1_ratio in l1_ratios]
+
+        check_scalar_alpha = partial(
+            check_scalar,
+            target_type=Real,
+            min_val=0.0,
+            include_boundaries="left",
+        )
+
+        if isinstance(self._alphas, Integral):
+            alphas = [
+                _alpha_grid(
+                    X,
+                    y,
+                    l1_ratio=l1_ratio,
+                    fit_intercept=self.fit_intercept,
+                    eps=self.eps,
+                    n_alphas=self._alphas,
+                    copy_X=self.copy_X,
+                    sample_weight=sample_weight,
+                )
+                for l1_ratio in l1_ratios
+            ]
         else:
+            # Making sure alphas entries are scalars.
+            for index, alpha in enumerate(self._alphas):
+                check_scalar_alpha(alpha, f"alphas[{index}]")
             # Making sure alphas is properly ordered.
-            alphas = np.tile(np.sort(alphas)[::-1], (n_l1_ratio, 1))
+            alphas = np.tile(np.sort(self._alphas)[::-1], (n_l1_ratio, 1))
+
         # We want n_alphas to be the number of alphas used for each l1_ratio.
         n_alphas = len(alphas[0])
-        path_params.update({'n_alphas': n_alphas})
+        path_params.update({"n_alphas": n_alphas})
 
-        path_params['copy_X'] = copy_X
+        path_params["copy_X"] = copy_X
         # We are not computing in parallel, we can modify X
         # inplace in the folds
         if effective_n_jobs(self.n_jobs) > 1:
-            path_params['copy_X'] = False
+            path_params["copy_X"] = False
 
         # init cross-validation generator
         cv = check_cv(self.cv)
 
+        if _routing_enabled():
+            splitter_supports_sample_weight = get_routing_for_object(cv).consumes(
+                method="split", params=["sample_weight"]
+            )
+            if (
+                sample_weight is not None
+                and not splitter_supports_sample_weight
+                and not has_fit_parameter(self, "sample_weight")
+            ):
+                raise ValueError(
+                    "The CV splitter and underlying estimator do not support"
+                    " sample weights."
+                )
+
+            if splitter_supports_sample_weight:
+                params["sample_weight"] = sample_weight
+
+            routed_params = process_routing(self, "fit", **params)
+
+            if sample_weight is not None and not has_fit_parameter(
+                self, "sample_weight"
+            ):
+                # MultiTaskElasticNetCV does not (yet) support sample_weight
+                sample_weight = None
+        else:
+            routed_params = Bunch()
+            routed_params.splitter = Bunch(split=Bunch())
+
         # Compute path for all folds and compute MSE to get the best alpha
-        folds = list(cv.split(X, y))
+        folds = list(cv.split(X, y, **routed_params.splitter.split))
         best_mse = np.inf
 
         # We do a double for loop folded in one, in order to be able to
         # iterate in parallel on l1_ratio and folds
-        jobs = (delayed(_path_residuals)(X, y, train, test, self.path,
-                                         path_params, alphas=this_alphas,
-                                         l1_ratio=this_l1_ratio, X_order='F',
-                                         dtype=X.dtype.type)
-                for this_l1_ratio, this_alphas in zip(l1_ratios, alphas)
-                for train, test in folds)
-        mse_paths = Parallel(n_jobs=self.n_jobs, verbose=self.verbose,
-                             **_joblib_parallel_args(prefer="threads"))(jobs)
+        jobs = (
+            delayed(_path_residuals)(
+                X,
+                y,
+                sample_weight,
+                train,
+                test,
+                self.fit_intercept,
+                self.path,
+                path_params,
+                alphas=this_alphas,
+                l1_ratio=this_l1_ratio,
+                X_order="F",
+                dtype=X.dtype.type,
+            )
+            for this_l1_ratio, this_alphas in zip(l1_ratios, alphas)
+            for train, test in folds
+        )
+        mse_paths = Parallel(
+            n_jobs=self.n_jobs,
+            verbose=self.verbose,
+            prefer="threads",
+        )(jobs)
         mse_paths = np.reshape(mse_paths, (n_l1_ratio, len(folds), -1))
+        # The mean is computed over folds.
         mean_mse = np.mean(mse_paths, axis=1)
-        self.mse_path_ = np.squeeze(np.rollaxis(mse_paths, 2, 1))
-        for l1_ratio, l1_alphas, mse_alphas in zip(l1_ratios, alphas,
-                                                   mean_mse):
+        self.mse_path_ = np.squeeze(np.moveaxis(mse_paths, 2, 1))
+        for l1_ratio, l1_alphas, mse_alphas in zip(l1_ratios, alphas, mean_mse):
             i_best_alpha = np.argmin(mse_alphas)
             this_best_mse = mse_alphas[i_best_alpha]
             if this_best_mse < best_mse:
@@ -1330,7 +1849,7 @@ def fit(self, X, y):
 
         self.l1_ratio_ = best_l1_ratio
         self.alpha_ = best_alpha
-        if self.alphas is None:
+        if isinstance(self._alphas, Integral):
             self.alphas_ = np.asarray(alphas)
             if n_l1_ratio == 1:
                 self.alphas_ = self.alphas_[0]
@@ -1339,9 +1858,11 @@ def fit(self, X, y):
             self.alphas_ = np.asarray(alphas[0])
 
         # Refit the model with the parameters selected
-        common_params = {name: value
-                         for name, value in self.get_params().items()
-                         if name in model.get_params()}
+        common_params = {
+            name: value
+            for name, value in self.get_params().items()
+            if name in model.get_params()
+        }
         model.set_params(**common_params)
         model.alpha = best_alpha
         model.l1_ratio = best_l1_ratio
@@ -1349,8 +1870,14 @@ def fit(self, X, y):
         precompute = getattr(self, "precompute", None)
         if isinstance(precompute, str) and precompute == "auto":
             model.precompute = False
-        model.fit(X, y)
-        if not hasattr(self, 'l1_ratio'):
+
+        if sample_weight is None:
+            # MultiTaskElasticNetCV does not (yet) support sample_weight, even
+            # not sample_weight=None.
+            model.fit(X, y)
+        else:
+            model.fit(X, y, sample_weight=sample_weight)
+        if not hasattr(self, "l1_ratio"):
             del self.l1_ratio_
         self.coef_ = model.coef_
         self.intercept_ = model.intercept_
@@ -1358,6 +1885,37 @@ def fit(self, X, y):
         self.n_iter_ = model.n_iter_
         return self
 
+    def get_metadata_routing(self):
+        """Get metadata routing of this object.
+
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
+
+        .. versionadded:: 1.4
+
+        Returns
+        -------
+        routing : MetadataRouter
+            A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
+            routing information.
+        """
+        router = (
+            MetadataRouter(owner=self.__class__.__name__)
+            .add_self_request(self)
+            .add(
+                splitter=check_cv(self.cv),
+                method_mapping=MethodMapping().add(caller="fit", callee="split"),
+            )
+        )
+        return router
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        multitask = self._is_multitask()
+        tags.input_tags.sparse = not multitask
+        tags.target_tags.multi_output = multitask
+        return tags
+
 
 class LassoCV(RegressorMixin, LinearModelCV):
     """Lasso linear model with iterative fitting along a regularization path.
@@ -1381,25 +1939,30 @@ class LassoCV(RegressorMixin, LinearModelCV):
     n_alphas : int, default=100
         Number of alphas along the regularization path.
 
-    alphas : ndarray, default=None
-        List of alphas where to compute the models.
-        If ``None`` alphas are set automatically.
+        .. deprecated:: 1.7
+            `n_alphas` was deprecated in 1.7 and will be removed in 1.9. Use `alphas`
+            instead.
+
+    alphas : array-like or int, default=None
+        Values of alphas to test along the regularization path.
+        If int, `alphas` values are generated automatically.
+        If array-like, list of alpha values to use.
+
+        .. versionchanged:: 1.7
+            `alphas` accepts an integer value which removes the need to pass
+            `n_alphas`.
+
+        .. deprecated:: 1.7
+            `alphas=None` was deprecated in 1.7 and will be removed in 1.9, at which
+            point the default value will be set to 100.
 
     fit_intercept : bool, default=True
         Whether to calculate the intercept for this model. If set
         to false, no intercept will be used in calculations
         (i.e. data is expected to be centered).
 
-    normalize : bool, default=False
-        This parameter is ignored when ``fit_intercept`` is set to False.
-        If True, the regressors X will be normalized before regression by
-        subtracting the mean and dividing by the l2-norm.
-        If you wish to standardize, please use
-        :class:`~sklearn.preprocessing.StandardScaler` before calling ``fit``
-        on an estimator with ``normalize=False``.
-
-    precompute : 'auto', bool or array-like of shape (n_features, n_features),\
-                 default='auto'
+    precompute : 'auto', bool or array-like of shape \
+            (n_features, n_features), default='auto'
         Whether to use a precomputed Gram matrix to speed up
         calculations. If set to ``'auto'`` let us decide. The Gram
         matrix can also be passed as argument.
@@ -1425,7 +1988,7 @@ class LassoCV(RegressorMixin, LinearModelCV):
         - :term:`CV splitter`,
         - An iterable yielding (train, test) splits as arrays of indices.
 
-        For int/None inputs, :class:`KFold` is used.
+        For int/None inputs, :class:`~sklearn.model_selection.KFold` is used.
 
         Refer :ref:`User Guide ` for the various
         cross-validation strategies that can be used here.
@@ -1482,48 +2045,95 @@ class LassoCV(RegressorMixin, LinearModelCV):
         Number of iterations run by the coordinate descent solver to reach
         the specified tolerance for the optimal alpha.
 
-    Examples
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
     --------
-    >>> from sklearn.linear_model import LassoCV
-    >>> from sklearn.datasets import make_regression
-    >>> X, y = make_regression(noise=4, random_state=0)
-    >>> reg = LassoCV(cv=5, random_state=0).fit(X, y)
-    >>> reg.score(X, y)
-    0.9993...
-    >>> reg.predict(X[:1,])
-    array([-78.4951...])
+    lars_path : Compute Least Angle Regression or Lasso path using LARS
+        algorithm.
+    lasso_path : Compute Lasso path with coordinate descent.
+    Lasso : The Lasso is a linear model that estimates sparse coefficients.
+    LassoLars : Lasso model fit with Least Angle Regression a.k.a. Lars.
+    LassoCV : Lasso linear model with iterative fitting along a regularization
+        path.
+    LassoLarsCV : Cross-validated Lasso using the LARS algorithm.
 
     Notes
     -----
-    For an example, see
-    :ref:`examples/linear_model/plot_lasso_model_selection.py
+    In `fit`, once the best parameter `alpha` is found through
+    cross-validation, the model is fit again using the entire training set.
+
+    To avoid unnecessary memory duplication the `X` argument of the `fit`
+    method should be directly passed as a Fortran-contiguous numpy array.
+
+    For an example, see :ref:`examples/linear_model/plot_lasso_model_selection.py
     `.
 
-    To avoid unnecessary memory duplication the X argument of the fit method
-    should be directly passed as a Fortran-contiguous numpy array.
+    :class:`LassoCV` leads to different results than a hyperparameter
+    search using :class:`~sklearn.model_selection.GridSearchCV` with a
+    :class:`Lasso` model. In :class:`LassoCV`, a model for a given
+    penalty `alpha` is warm started using the coefficients of the
+    closest model (trained at the previous iteration) on the
+    regularization path. It tends to speed up the hyperparameter
+    search.
 
-    See Also
+    Examples
     --------
-    lars_path
-    lasso_path
-    LassoLars
-    Lasso
-    LassoLarsCV
+    >>> from sklearn.linear_model import LassoCV
+    >>> from sklearn.datasets import make_regression
+    >>> X, y = make_regression(noise=4, random_state=0)
+    >>> reg = LassoCV(cv=5, random_state=0).fit(X, y)
+    >>> reg.score(X, y)
+    0.9993
+    >>> reg.predict(X[:1,])
+    array([-78.4951])
     """
+
     path = staticmethod(lasso_path)
 
-    @_deprecate_positional_args
-    def __init__(self, *, eps=1e-3, n_alphas=100, alphas=None,
-                 fit_intercept=True,
-                 normalize=False, precompute='auto', max_iter=1000, tol=1e-4,
-                 copy_X=True, cv=None, verbose=False, n_jobs=None,
-                 positive=False, random_state=None, selection='cyclic'):
+    def __init__(
+        self,
+        *,
+        eps=1e-3,
+        n_alphas="deprecated",
+        alphas="warn",
+        fit_intercept=True,
+        precompute="auto",
+        max_iter=1000,
+        tol=1e-4,
+        copy_X=True,
+        cv=None,
+        verbose=False,
+        n_jobs=None,
+        positive=False,
+        random_state=None,
+        selection="cyclic",
+    ):
         super().__init__(
-            eps=eps, n_alphas=n_alphas, alphas=alphas,
-            fit_intercept=fit_intercept, normalize=normalize,
-            precompute=precompute, max_iter=max_iter, tol=tol, copy_X=copy_X,
-            cv=cv, verbose=verbose, n_jobs=n_jobs, positive=positive,
-            random_state=random_state, selection=selection)
+            eps=eps,
+            n_alphas=n_alphas,
+            alphas=alphas,
+            fit_intercept=fit_intercept,
+            precompute=precompute,
+            max_iter=max_iter,
+            tol=tol,
+            copy_X=copy_X,
+            cv=cv,
+            verbose=verbose,
+            n_jobs=n_jobs,
+            positive=positive,
+            random_state=random_state,
+            selection=selection,
+        )
 
     def _get_estimator(self):
         return Lasso()
@@ -1531,8 +2141,45 @@ def _get_estimator(self):
     def _is_multitask(self):
         return False
 
-    def _more_tags(self):
-        return {'multioutput': False}
+    def fit(self, X, y, sample_weight=None, **params):
+        """Fit Lasso model with coordinate descent.
+
+        Fit is on grid of alphas and best alpha estimated by cross-validation.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Training data. Pass directly as Fortran-contiguous data
+            to avoid unnecessary memory duplication. If y is mono-output,
+            X can be sparse. Note that large sparse matrices and arrays
+            requiring `int64` indices are not accepted.
+
+        y : array-like of shape (n_samples,)
+            Target values.
+
+        sample_weight : float or array-like of shape (n_samples,), \
+                default=None
+            Sample weights used for fitting and evaluation of the weighted
+            mean squared error of each cv-fold. Note that the cross validated
+            MSE that is finally used to find the best model is the unweighted
+            mean over the (weighted) MSEs of each test fold.
+
+        **params : dict, default=None
+            Parameters to be passed to the CV splitter.
+
+            .. versionadded:: 1.4
+                Only available if `enable_metadata_routing=True`,
+                which can be set by using
+                ``sklearn.set_config(enable_metadata_routing=True)``.
+                See :ref:`Metadata Routing User Guide ` for
+                more details.
+
+        Returns
+        -------
+        self : object
+            Returns an instance of fitted model.
+        """
+        return super().fit(X, y, sample_weight=sample_weight, **params)
 
 
 class ElasticNetCV(RegressorMixin, LinearModelCV):
@@ -1545,7 +2192,7 @@ class ElasticNetCV(RegressorMixin, LinearModelCV):
     Parameters
     ----------
     l1_ratio : float or list of float, default=0.5
-        float between 0 and 1 passed to ElasticNet (scaling between
+        Float between 0 and 1 passed to ElasticNet (scaling between
         l1 and l2 penalties). For ``l1_ratio = 0``
         the penalty is an L2 penalty. For ``l1_ratio = 1`` it is an L1 penalty.
         For ``0 < l1_ratio < 1``, the penalty is a combination of L1 and L2
@@ -1563,25 +2210,30 @@ class ElasticNetCV(RegressorMixin, LinearModelCV):
     n_alphas : int, default=100
         Number of alphas along the regularization path, used for each l1_ratio.
 
-    alphas : ndarray, default=None
-        List of alphas where to compute the models.
-        If None alphas are set automatically.
+        .. deprecated:: 1.7
+            `n_alphas` was deprecated in 1.7 and will be removed in 1.9. Use `alphas`
+            instead.
+
+    alphas : array-like or int, default=None
+        Values of alphas to test along the regularization path, used for each l1_ratio.
+        If int, `alphas` values are generated automatically.
+        If array-like, list of alpha values to use.
+
+        .. versionchanged:: 1.7
+            `alphas` accepts an integer value which removes the need to pass
+            `n_alphas`.
+
+        .. deprecated:: 1.7
+            `alphas=None` was deprecated in 1.7 and will be removed in 1.9, at which
+            point the default value will be set to 100.
 
     fit_intercept : bool, default=True
         Whether to calculate the intercept for this model. If set
         to false, no intercept will be used in calculations
         (i.e. data is expected to be centered).
 
-    normalize : bool, default=False
-        This parameter is ignored when ``fit_intercept`` is set to False.
-        If True, the regressors X will be normalized before regression by
-        subtracting the mean and dividing by the l2-norm.
-        If you wish to standardize, please use
-        :class:`~sklearn.preprocessing.StandardScaler` before calling ``fit``
-        on an estimator with ``normalize=False``.
-
-    precompute : 'auto', bool or array-like of shape (n_features, n_features),\
-                 default='auto'
+    precompute : 'auto', bool or array-like of shape \
+            (n_features, n_features), default='auto'
         Whether to use a precomputed Gram matrix to speed up
         calculations. If set to ``'auto'`` let us decide. The Gram
         matrix can also be passed as argument.
@@ -1604,7 +2256,7 @@ class ElasticNetCV(RegressorMixin, LinearModelCV):
         - :term:`CV splitter`,
         - An iterable yielding (train, test) splits as arrays of indices.
 
-        For int/None inputs, :class:`KFold` is used.
+        For int/None inputs, :class:`~sklearn.model_selection.KFold` is used.
 
         Refer :ref:`User Guide ` for the various
         cross-validation strategies that can be used here.
@@ -1668,33 +2320,31 @@ class ElasticNetCV(RegressorMixin, LinearModelCV):
         Number of iterations run by the coordinate descent solver to reach
         the specified tolerance for the optimal alpha.
 
-    Examples
-    --------
-    >>> from sklearn.linear_model import ElasticNetCV
-    >>> from sklearn.datasets import make_regression
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
 
-    >>> X, y = make_regression(n_features=2, random_state=0)
-    >>> regr = ElasticNetCV(cv=5, random_state=0)
-    >>> regr.fit(X, y)
-    ElasticNetCV(cv=5, random_state=0)
-    >>> print(regr.alpha_)
-    0.199...
-    >>> print(regr.intercept_)
-    0.398...
-    >>> print(regr.predict([[0, 0]]))
-    [0.398...]
+        .. versionadded:: 0.24
 
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    enet_path : Compute elastic net path with coordinate descent.
+    ElasticNet : Linear regression with combined L1 and L2 priors as regularizer.
 
     Notes
     -----
-    For an example, see
-    :ref:`examples/linear_model/plot_lasso_model_selection.py
-    `.
+    In `fit`, once the best parameters `l1_ratio` and `alpha` are found through
+    cross-validation, the model is fit again using the entire training set.
 
-    To avoid unnecessary memory duplication the X argument of the fit method
-    should be directly passed as a Fortran-contiguous numpy array.
+    To avoid unnecessary memory duplication the `X` argument of the `fit`
+    method should be directly passed as a Fortran-contiguous numpy array.
 
-    The parameter l1_ratio corresponds to alpha in the glmnet R package
+    The parameter `l1_ratio` corresponds to alpha in the glmnet R package
     while alpha corresponds to the lambda parameter in glmnet.
     More specifically, the optimization objective is::
 
@@ -1711,26 +2361,58 @@ class ElasticNetCV(RegressorMixin, LinearModelCV):
 
         alpha = a + b and l1_ratio = a / (a + b).
 
-    See Also
+    For an example, see
+    :ref:`examples/linear_model/plot_lasso_model_selection.py
+    `.
+
+    Examples
     --------
-    enet_path
-    ElasticNet
+    >>> from sklearn.linear_model import ElasticNetCV
+    >>> from sklearn.datasets import make_regression
 
+    >>> X, y = make_regression(n_features=2, random_state=0)
+    >>> regr = ElasticNetCV(cv=5, random_state=0)
+    >>> regr.fit(X, y)
+    ElasticNetCV(cv=5, random_state=0)
+    >>> print(regr.alpha_)
+    0.199
+    >>> print(regr.intercept_)
+    0.398
+    >>> print(regr.predict([[0, 0]]))
+    [0.398]
     """
+
+    _parameter_constraints: dict = {
+        **LinearModelCV._parameter_constraints,
+        "l1_ratio": [Interval(Real, 0, 1, closed="both"), "array-like"],
+    }
+
     path = staticmethod(enet_path)
 
-    @_deprecate_positional_args
-    def __init__(self, *, l1_ratio=0.5, eps=1e-3, n_alphas=100, alphas=None,
-                 fit_intercept=True, normalize=False, precompute='auto',
-                 max_iter=1000, tol=1e-4, cv=None, copy_X=True,
-                 verbose=0, n_jobs=None, positive=False, random_state=None,
-                 selection='cyclic'):
+    def __init__(
+        self,
+        *,
+        l1_ratio=0.5,
+        eps=1e-3,
+        n_alphas="deprecated",
+        alphas="warn",
+        fit_intercept=True,
+        precompute="auto",
+        max_iter=1000,
+        tol=1e-4,
+        cv=None,
+        copy_X=True,
+        verbose=0,
+        n_jobs=None,
+        positive=False,
+        random_state=None,
+        selection="cyclic",
+    ):
         self.l1_ratio = l1_ratio
         self.eps = eps
         self.n_alphas = n_alphas
         self.alphas = alphas
         self.fit_intercept = fit_intercept
-        self.normalize = normalize
         self.precompute = precompute
         self.max_iter = max_iter
         self.tol = tol
@@ -1748,16 +2430,53 @@ def _get_estimator(self):
     def _is_multitask(self):
         return False
 
-    def _more_tags(self):
-        return {'multioutput': False}
+    def fit(self, X, y, sample_weight=None, **params):
+        """Fit ElasticNet model with coordinate descent.
+
+        Fit is on grid of alphas and best alpha estimated by cross-validation.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Training data. Pass directly as Fortran-contiguous data
+            to avoid unnecessary memory duplication. If y is mono-output,
+            X can be sparse. Note that large sparse matrices and arrays
+            requiring `int64` indices are not accepted.
+
+        y : array-like of shape (n_samples,)
+            Target values.
+
+        sample_weight : float or array-like of shape (n_samples,), \
+                default=None
+            Sample weights used for fitting and evaluation of the weighted
+            mean squared error of each cv-fold. Note that the cross validated
+            MSE that is finally used to find the best model is the unweighted
+            mean over the (weighted) MSEs of each test fold.
+
+        **params : dict, default=None
+            Parameters to be passed to the CV splitter.
+
+            .. versionadded:: 1.4
+                Only available if `enable_metadata_routing=True`,
+                which can be set by using
+                ``sklearn.set_config(enable_metadata_routing=True)``.
+                See :ref:`Metadata Routing User Guide ` for
+                more details.
+
+        Returns
+        -------
+        self : object
+            Returns an instance of fitted model.
+        """
+        return super().fit(X, y, sample_weight=sample_weight, **params)
+
 
 ###############################################################################
 # Multi Task ElasticNet and Lasso models (with joint feature selection)
 
 
 class MultiTaskElasticNet(Lasso):
-    """Multi-task ElasticNet model trained with L1/L2 mixed-norm as
-    regularizer.
+    """Multi-task ElasticNet model trained with L1/L2 mixed-norm as regularizer.
 
     The optimization objective for MultiTaskElasticNet is::
 
@@ -1789,14 +2508,6 @@ class MultiTaskElasticNet(Lasso):
         to false, no intercept will be used in calculations
         (i.e. data is expected to be centered).
 
-    normalize : bool, default=False
-        This parameter is ignored when ``fit_intercept`` is set to False.
-        If True, the regressors X will be normalized before regression by
-        subtracting the mean and dividing by the l2-norm.
-        If you wish to standardize, please use
-        :class:`~sklearn.preprocessing.StandardScaler` before calling ``fit``
-        on an estimator with ``normalize=False``.
-
     copy_X : bool, default=True
         If ``True``, X will be copied; else, it may be overwritten.
 
@@ -1828,10 +2539,10 @@ class MultiTaskElasticNet(Lasso):
 
     Attributes
     ----------
-    intercept_ : ndarray of shape (n_tasks,)
+    intercept_ : ndarray of shape (n_targets,)
         Independent term in decision function.
 
-    coef_ : ndarray of shape (n_tasks, n_features)
+    coef_ : ndarray of shape (n_targets, n_features)
         Parameter vector (W in the cost function formula). If a 1D y is
         passed in at fit (non multi-task usage), ``coef_`` is then a 1D array.
         Note that ``coef_`` stores the transpose of ``W``, ``W.T``.
@@ -1847,27 +2558,27 @@ class MultiTaskElasticNet(Lasso):
         The tolerance scaled scaled by the variance of the target `y`.
 
     sparse_coef_ : sparse matrix of shape (n_features,) or \
-            (n_tasks, n_features)
+            (n_targets, n_features)
         Sparse representation of the `coef_`.
 
-    Examples
-    --------
-    >>> from sklearn import linear_model
-    >>> clf = linear_model.MultiTaskElasticNet(alpha=0.1)
-    >>> clf.fit([[0,0], [1, 1], [2, 2]], [[0, 0], [1, 1], [2, 2]])
-    MultiTaskElasticNet(alpha=0.1)
-    >>> print(clf.coef_)
-    [[0.45663524 0.45612256]
-     [0.45663524 0.45612256]]
-    >>> print(clf.intercept_)
-    [0.0872422 0.0872422]
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
 
     See Also
     --------
     MultiTaskElasticNetCV : Multi-task L1/L2 ElasticNet with built-in
         cross-validation.
-    ElasticNet
-    MultiTaskLasso
+    ElasticNet : Linear regression with combined L1 and L2 priors as regularizer.
+    MultiTaskLasso : Multi-task Lasso model trained with L1/L2
+        mixed-norm as regularizer.
 
     Notes
     -----
@@ -1875,15 +2586,42 @@ class MultiTaskElasticNet(Lasso):
 
     To avoid unnecessary memory duplication the X and y arguments of the fit
     method should be directly passed as Fortran-contiguous numpy arrays.
+
+    Examples
+    --------
+    >>> from sklearn import linear_model
+    >>> clf = linear_model.MultiTaskElasticNet(alpha=0.1)
+    >>> clf.fit([[0,0], [1, 1], [2, 2]], [[0, 0], [1, 1], [2, 2]])
+    MultiTaskElasticNet(alpha=0.1)
+    >>> print(clf.coef_)
+    [[0.45663524 0.45612256]
+     [0.45663524 0.45612256]]
+    >>> print(clf.intercept_)
+    [0.0872422 0.0872422]
     """
-    @_deprecate_positional_args
-    def __init__(self, alpha=1.0, *, l1_ratio=0.5, fit_intercept=True,
-                 normalize=False, copy_X=True, max_iter=1000, tol=1e-4,
-                 warm_start=False, random_state=None, selection='cyclic'):
+
+    _parameter_constraints: dict = {
+        **ElasticNet._parameter_constraints,
+    }
+    for param in ("precompute", "positive"):
+        _parameter_constraints.pop(param)
+
+    def __init__(
+        self,
+        alpha=1.0,
+        *,
+        l1_ratio=0.5,
+        fit_intercept=True,
+        copy_X=True,
+        max_iter=1000,
+        tol=1e-4,
+        warm_start=False,
+        random_state=None,
+        selection="cyclic",
+    ):
         self.l1_ratio = l1_ratio
         self.alpha = alpha
         self.fit_intercept = fit_intercept
-        self.normalize = normalize
         self.max_iter = max_iter
         self.copy_X = copy_X
         self.tol = tol
@@ -1891,19 +2629,24 @@ def __init__(self, alpha=1.0, *, l1_ratio=0.5, fit_intercept=True,
         self.random_state = random_state
         self.selection = selection
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y):
-        """Fit MultiTaskElasticNet model with coordinate descent
+        """Fit MultiTaskElasticNet model with coordinate descent.
 
         Parameters
         ----------
         X : ndarray of shape (n_samples, n_features)
             Data.
-        y : ndarray of shape (n_samples, n_tasks)
+        y : ndarray of shape (n_samples, n_targets)
             Target. Will be cast to X's dtype if necessary.
 
+        Returns
+        -------
+        self : object
+            Fitted estimator.
+
         Notes
         -----
-
         Coordinate descent is an algorithm that considers each column of
         data at a time hence it will automatically convert the X input
         as a Fortran-contiguous numpy array if necessary.
@@ -1912,48 +2655,62 @@ def fit(self, X, y):
         initial data in memory directly using that format.
         """
         # Need to validate separately here.
-        # We can't pass multi_ouput=True because that would allow y to be csr.
-        check_X_params = dict(dtype=[np.float64, np.float32], order='F',
-                              copy=self.copy_X and self.fit_intercept)
-        check_y_params = dict(ensure_2d=False, order='F')
-        X, y = self._validate_data(X, y, validate_separately=(check_X_params,
-                                                              check_y_params))
+        # We can't pass multi_output=True because that would allow y to be csr.
+        check_X_params = dict(
+            dtype=[np.float64, np.float32],
+            order="F",
+            force_writeable=True,
+            copy=self.copy_X and self.fit_intercept,
+        )
+        check_y_params = dict(ensure_2d=False, order="F")
+        X, y = validate_data(
+            self, X, y, validate_separately=(check_X_params, check_y_params)
+        )
+        check_consistent_length(X, y)
         y = y.astype(X.dtype)
 
-        if hasattr(self, 'l1_ratio'):
-            model_str = 'ElasticNet'
+        if hasattr(self, "l1_ratio"):
+            model_str = "ElasticNet"
         else:
-            model_str = 'Lasso'
+            model_str = "Lasso"
         if y.ndim == 1:
             raise ValueError("For mono-task outputs, use %s" % model_str)
 
         n_samples, n_features = X.shape
-        _, n_tasks = y.shape
-
-        if n_samples != y.shape[0]:
-            raise ValueError("X and y have inconsistent dimensions (%d != %d)"
-                             % (n_samples, y.shape[0]))
+        n_targets = y.shape[1]
 
         X, y, X_offset, y_offset, X_scale = _preprocess_data(
-            X, y, self.fit_intercept, self.normalize, copy=False)
+            X, y, fit_intercept=self.fit_intercept, copy=False
+        )
 
         if not self.warm_start or not hasattr(self, "coef_"):
-            self.coef_ = np.zeros((n_tasks, n_features), dtype=X.dtype.type,
-                                  order='F')
+            self.coef_ = np.zeros(
+                (n_targets, n_features), dtype=X.dtype.type, order="F"
+            )
 
         l1_reg = self.alpha * self.l1_ratio * n_samples
         l2_reg = self.alpha * (1.0 - self.l1_ratio) * n_samples
 
         self.coef_ = np.asfortranarray(self.coef_)  # coef contiguous in memory
 
-        if self.selection not in ['random', 'cyclic']:
-            raise ValueError("selection should be either random or cyclic.")
-        random = (self.selection == 'random')
-
-        self.coef_, self.dual_gap_, self.eps_, self.n_iter_ = \
-            cd_fast.enet_coordinate_descent_multi_task(
-                self.coef_, l1_reg, l2_reg, X, y, self.max_iter, self.tol,
-                check_random_state(self.random_state), random)
+        random = self.selection == "random"
+
+        (
+            self.coef_,
+            self.dual_gap_,
+            self.eps_,
+            self.n_iter_,
+        ) = cd_fast.enet_coordinate_descent_multi_task(
+            self.coef_,
+            l1_reg,
+            l2_reg,
+            X,
+            y,
+            self.max_iter,
+            self.tol,
+            check_random_state(self.random_state),
+            random,
+        )
 
         # account for different objective scaling here and in cd_fast
         self.dual_gap_ /= n_samples
@@ -1963,8 +2720,12 @@ def fit(self, X, y):
         # return self for chaining fit and predict calls
         return self
 
-    def _more_tags(self):
-        return {'multioutput_only': True}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = False
+        tags.target_tags.multi_output = True
+        tags.target_tags.single_output = False
+        return tags
 
 
 class MultiTaskLasso(MultiTaskElasticNet):
@@ -1992,14 +2753,6 @@ class MultiTaskLasso(MultiTaskElasticNet):
         to false, no intercept will be used in calculations
         (i.e. data is expected to be centered).
 
-    normalize : bool, default=False
-        This parameter is ignored when ``fit_intercept`` is set to False.
-        If True, the regressors X will be normalized before regression by
-        subtracting the mean and dividing by the l2-norm.
-        If you wish to standardize, please use
-        :class:`~sklearn.preprocessing.StandardScaler` before calling ``fit``
-        on an estimator with ``normalize=False``.
-
     copy_X : bool, default=True
         If ``True``, X will be copied; else, it may be overwritten.
 
@@ -2027,15 +2780,15 @@ class MultiTaskLasso(MultiTaskElasticNet):
         If set to 'random', a random coefficient is updated every iteration
         rather than looping over features sequentially by default. This
         (setting to 'random') often leads to significantly faster convergence
-        especially when tol is higher than 1e-4
+        especially when tol is higher than 1e-4.
 
     Attributes
     ----------
-    coef_ : ndarray of shape (n_tasks, n_features)
+    coef_ : ndarray of shape (n_targets, n_features)
         Parameter vector (W in the cost function formula).
         Note that ``coef_`` stores the transpose of ``W``, ``W.T``.
 
-    intercept_ : ndarray of shape (n_tasks,)
+    intercept_ : ndarray of shape (n_targets,)
         Independent term in decision function.
 
     n_iter_ : int
@@ -2049,9 +2802,34 @@ class MultiTaskLasso(MultiTaskElasticNet):
         The tolerance scaled scaled by the variance of the target `y`.
 
     sparse_coef_ : sparse matrix of shape (n_features,) or \
-            (n_tasks, n_features)
+            (n_targets, n_features)
         Sparse representation of the `coef_`.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    Lasso: Linear Model trained with L1 prior as regularizer (aka the Lasso).
+    MultiTaskLassoCV: Multi-task L1 regularized linear model with built-in
+        cross-validation.
+    MultiTaskElasticNetCV: Multi-task L1/L2 ElasticNet with built-in cross-validation.
+
+    Notes
+    -----
+    The algorithm used to fit the model is coordinate descent.
+
+    To avoid unnecessary memory duplication the X and y arguments of the fit
+    method should be directly passed as Fortran-contiguous numpy arrays.
+
     Examples
     --------
     >>> from sklearn import linear_model
@@ -2063,27 +2841,27 @@ class MultiTaskLasso(MultiTaskElasticNet):
     [0.         0.94592424]]
     >>> print(clf.intercept_)
     [-0.41888636 -0.87382323]
-
-    See Also
-    --------
-    MultiTaskLasso : Multi-task L1/L2 Lasso with built-in cross-validation.
-    Lasso
-    MultiTaskElasticNet
-
-    Notes
-    -----
-    The algorithm used to fit the model is coordinate descent.
-
-    To avoid unnecessary memory duplication the X and y arguments of the fit
-    method should be directly passed as Fortran-contiguous numpy arrays.
     """
-    @_deprecate_positional_args
-    def __init__(self, alpha=1.0, *, fit_intercept=True, normalize=False,
-                 copy_X=True, max_iter=1000, tol=1e-4, warm_start=False,
-                 random_state=None, selection='cyclic'):
+
+    _parameter_constraints: dict = {
+        **MultiTaskElasticNet._parameter_constraints,
+    }
+    _parameter_constraints.pop("l1_ratio")
+
+    def __init__(
+        self,
+        alpha=1.0,
+        *,
+        fit_intercept=True,
+        copy_X=True,
+        max_iter=1000,
+        tol=1e-4,
+        warm_start=False,
+        random_state=None,
+        selection="cyclic",
+    ):
         self.alpha = alpha
         self.fit_intercept = fit_intercept
-        self.normalize = normalize
         self.max_iter = max_iter
         self.copy_X = copy_X
         self.tol = tol
@@ -2126,7 +2904,7 @@ class MultiTaskElasticNetCV(RegressorMixin, LinearModelCV):
         prediction score is used. Note that a good choice of list of
         values for l1_ratio is often to put more values close to 1
         (i.e. Lasso) and less close to 0 (i.e. Ridge), as in ``[.1, .5, .7,
-        .9, .95, .99, 1]``
+        .9, .95, .99, 1]``.
 
     eps : float, default=1e-3
         Length of the path. ``eps=1e-3`` means that
@@ -2135,23 +2913,28 @@ class MultiTaskElasticNetCV(RegressorMixin, LinearModelCV):
     n_alphas : int, default=100
         Number of alphas along the regularization path.
 
-    alphas : array-like, default=None
-        List of alphas where to compute the models.
-        If not provided, set automatically.
+        .. deprecated:: 1.7
+            `n_alphas` was deprecated in 1.7 and will be removed in 1.9. Use `alphas`
+            instead.
+
+    alphas : array-like or int, default=None
+        Values of alphas to test along the regularization path, used for each l1_ratio.
+        If int, `alphas` values are generated automatically.
+        If array-like, list of alpha values to use.
+
+        .. versionchanged:: 1.7
+            `alphas` accepts an integer value which removes the need to pass
+            `n_alphas`.
+
+        .. deprecated:: 1.7
+            `alphas=None` was deprecated in 1.7 and will be removed in 1.9, at which
+            point the default value will be set to 100.
 
     fit_intercept : bool, default=True
         Whether to calculate the intercept for this model. If set
         to false, no intercept will be used in calculations
         (i.e. data is expected to be centered).
 
-    normalize : bool, default=False
-        This parameter is ignored when ``fit_intercept`` is set to False.
-        If True, the regressors X will be normalized before regression by
-        subtracting the mean and dividing by the l2-norm.
-        If you wish to standardize, please use
-        :class:`~sklearn.preprocessing.StandardScaler` before calling ``fit``
-        on an estimator with ``normalize=False``.
-
     max_iter : int, default=1000
         The maximum number of iterations.
 
@@ -2170,7 +2953,7 @@ class MultiTaskElasticNetCV(RegressorMixin, LinearModelCV):
         - :term:`CV splitter`,
         - An iterable yielding (train, test) splits as arrays of indices.
 
-        For int/None inputs, :class:`KFold` is used.
+        For int/None inputs, :class:`~sklearn.model_selection.KFold` is used.
 
         Refer :ref:`User Guide ` for the various
         cross-validation strategies that can be used here.
@@ -2205,10 +2988,10 @@ class MultiTaskElasticNetCV(RegressorMixin, LinearModelCV):
 
     Attributes
     ----------
-    intercept_ : ndarray of shape (n_tasks,)
+    intercept_ : ndarray of shape (n_targets,)
         Independent term in decision function.
 
-    coef_ : ndarray of shape (n_tasks, n_features)
+    coef_ : ndarray of shape (n_targets, n_features)
         Parameter vector (W in the cost function formula).
         Note that ``coef_`` stores the transpose of ``W``, ``W.T``.
 
@@ -2232,6 +3015,35 @@ class MultiTaskElasticNetCV(RegressorMixin, LinearModelCV):
     dual_gap_ : float
         The dual gap at the end of the optimization for the optimal alpha.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    MultiTaskElasticNet : Multi-task L1/L2 ElasticNet with built-in cross-validation.
+    ElasticNetCV : Elastic net model with best model selection by
+        cross-validation.
+    MultiTaskLassoCV : Multi-task Lasso model trained with L1 norm
+        as regularizer and built-in cross-validation.
+
+    Notes
+    -----
+    The algorithm used to fit the model is coordinate descent.
+
+    In `fit`, once the best parameters `l1_ratio` and `alpha` are found through
+    cross-validation, the model is fit again using the entire training set.
+
+    To avoid unnecessary memory duplication the `X` and `y` arguments of the
+    `fit` method should be directly passed as Fortran-contiguous numpy arrays.
+
     Examples
     --------
     >>> from sklearn import linear_model
@@ -2244,34 +3056,39 @@ class MultiTaskElasticNetCV(RegressorMixin, LinearModelCV):
      [0.52875032 0.46958558]]
     >>> print(clf.intercept_)
     [0.00166409 0.00166409]
+    """
 
-    See Also
-    --------
-    MultiTaskElasticNet
-    ElasticNetCV
-    MultiTaskLassoCV
+    _parameter_constraints: dict = {
+        **LinearModelCV._parameter_constraints,
+        "l1_ratio": [Interval(Real, 0, 1, closed="both"), "array-like"],
+    }
+    _parameter_constraints.pop("precompute")
+    _parameter_constraints.pop("positive")
 
-    Notes
-    -----
-    The algorithm used to fit the model is coordinate descent.
-
-    To avoid unnecessary memory duplication the X and y arguments of the fit
-    method should be directly passed as Fortran-contiguous numpy arrays.
-    """
     path = staticmethod(enet_path)
 
-    @_deprecate_positional_args
-    def __init__(self, *, l1_ratio=0.5, eps=1e-3, n_alphas=100, alphas=None,
-                 fit_intercept=True, normalize=False,
-                 max_iter=1000, tol=1e-4, cv=None, copy_X=True,
-                 verbose=0, n_jobs=None, random_state=None,
-                 selection='cyclic'):
+    def __init__(
+        self,
+        *,
+        l1_ratio=0.5,
+        eps=1e-3,
+        n_alphas="deprecated",
+        alphas="warn",
+        fit_intercept=True,
+        max_iter=1000,
+        tol=1e-4,
+        cv=None,
+        copy_X=True,
+        verbose=0,
+        n_jobs=None,
+        random_state=None,
+        selection="cyclic",
+    ):
         self.l1_ratio = l1_ratio
         self.eps = eps
         self.n_alphas = n_alphas
         self.alphas = alphas
         self.fit_intercept = fit_intercept
-        self.normalize = normalize
         self.max_iter = max_iter
         self.tol = tol
         self.cv = cv
@@ -2287,8 +3104,41 @@ def _get_estimator(self):
     def _is_multitask(self):
         return True
 
-    def _more_tags(self):
-        return {'multioutput_only': True}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.target_tags.single_output = False
+        return tags
+
+    # This is necessary as LinearModelCV now supports sample_weight while
+    # MultiTaskElasticNetCV does not (yet).
+    def fit(self, X, y, **params):
+        """Fit MultiTaskElasticNet model with coordinate descent.
+
+        Fit is on grid of alphas and best alpha estimated by cross-validation.
+
+        Parameters
+        ----------
+        X : ndarray of shape (n_samples, n_features)
+            Training data.
+        y : ndarray of shape (n_samples, n_targets)
+            Training target variable. Will be cast to X's dtype if necessary.
+
+        **params : dict, default=None
+            Parameters to be passed to the CV splitter.
+
+            .. versionadded:: 1.4
+                Only available if `enable_metadata_routing=True`,
+                which can be set by using
+                ``sklearn.set_config(enable_metadata_routing=True)``.
+                See :ref:`Metadata Routing User Guide ` for
+                more details.
+
+        Returns
+        -------
+        self : object
+            Returns MultiTaskElasticNet instance.
+        """
+        return super().fit(X, y, **params)
 
 
 class MultiTaskLassoCV(RegressorMixin, LinearModelCV):
@@ -2319,23 +3169,28 @@ class MultiTaskLassoCV(RegressorMixin, LinearModelCV):
     n_alphas : int, default=100
         Number of alphas along the regularization path.
 
-    alphas : array-like, default=None
-        List of alphas where to compute the models.
-        If not provided, set automatically.
+        .. deprecated:: 1.7
+            `n_alphas` was deprecated in 1.7 and will be removed in 1.9. Use `alphas`
+            instead.
+
+    alphas : array-like or int, default=None
+        Values of alphas to test along the regularization path.
+        If int, `alphas` values are generated automatically.
+        If array-like, list of alpha values to use.
+
+        .. versionchanged:: 1.7
+            `alphas` accepts an integer value which removes the need to pass
+            `n_alphas`.
+
+        .. deprecated:: 1.7
+            `alphas=None` was deprecated in 1.7 and will be removed in 1.9, at which
+            point the default value will be set to 100.
 
     fit_intercept : bool, default=True
         Whether to calculate the intercept for this model. If set
         to false, no intercept will be used in calculations
         (i.e. data is expected to be centered).
 
-    normalize : bool, default=False
-        This parameter is ignored when ``fit_intercept`` is set to False.
-        If True, the regressors X will be normalized before regression by
-        subtracting the mean and dividing by the l2-norm.
-        If you wish to standardize, please use
-        :class:`~sklearn.preprocessing.StandardScaler` before calling ``fit``
-        on an estimator with ``normalize=False``.
-
     max_iter : int, default=1000
         The maximum number of iterations.
 
@@ -2357,7 +3212,7 @@ class MultiTaskLassoCV(RegressorMixin, LinearModelCV):
         - :term:`CV splitter`,
         - An iterable yielding (train, test) splits as arrays of indices.
 
-        For int/None inputs, :class:`KFold` is used.
+        For int/None inputs, :class:`~sklearn.model_selection.KFold` is used.
 
         Refer :ref:`User Guide ` for the various
         cross-validation strategies that can be used here.
@@ -2389,10 +3244,10 @@ class MultiTaskLassoCV(RegressorMixin, LinearModelCV):
 
     Attributes
     ----------
-    intercept_ : ndarray of shape (n_tasks,)
+    intercept_ : ndarray of shape (n_targets,)
         Independent term in decision function.
 
-    coef_ : ndarray of shape (n_tasks, n_features)
+    coef_ : ndarray of shape (n_targets, n_features)
         Parameter vector (W in the cost function formula).
         Note that ``coef_`` stores the transpose of ``W``, ``W.T``.
 
@@ -2412,6 +3267,36 @@ class MultiTaskLassoCV(RegressorMixin, LinearModelCV):
     dual_gap_ : float
         The dual gap at the end of the optimization for the optimal alpha.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    MultiTaskElasticNet : Multi-task ElasticNet model trained with L1/L2
+        mixed-norm as regularizer.
+    ElasticNetCV : Elastic net model with best model selection by
+        cross-validation.
+    MultiTaskElasticNetCV : Multi-task L1/L2 ElasticNet with built-in
+        cross-validation.
+
+    Notes
+    -----
+    The algorithm used to fit the model is coordinate descent.
+
+    In `fit`, once the best parameter `alpha` is found through
+    cross-validation, the model is fit again using the entire training set.
+
+    To avoid unnecessary memory duplication the `X` and `y` arguments of the
+    `fit` method should be directly passed as Fortran-contiguous numpy arrays.
+
     Examples
     --------
     >>> from sklearn.linear_model import MultiTaskLassoCV
@@ -2420,39 +3305,51 @@ class MultiTaskLassoCV(RegressorMixin, LinearModelCV):
     >>> X, y = make_regression(n_targets=2, noise=4, random_state=0)
     >>> reg = MultiTaskLassoCV(cv=5, random_state=0).fit(X, y)
     >>> r2_score(y, reg.predict(X))
-    0.9994...
+    0.9994
     >>> reg.alpha_
-    0.5713...
+    np.float64(0.5713)
     >>> reg.predict(X[:1,])
-    array([[153.7971...,  94.9015...]])
-
-    See Also
-    --------
-    MultiTaskElasticNet
-    ElasticNetCV
-    MultiTaskElasticNetCV
+    array([[153.7971,  94.9015]])
+    """
 
-    Notes
-    -----
-    The algorithm used to fit the model is coordinate descent.
+    _parameter_constraints: dict = {
+        **LinearModelCV._parameter_constraints,
+    }
+    _parameter_constraints.pop("precompute")
+    _parameter_constraints.pop("positive")
 
-    To avoid unnecessary memory duplication the X and y arguments of the fit
-    method should be directly passed as Fortran-contiguous numpy arrays.
-    """
     path = staticmethod(lasso_path)
 
-    @_deprecate_positional_args
-    def __init__(self, *, eps=1e-3, n_alphas=100, alphas=None,
-                 fit_intercept=True,
-                 normalize=False, max_iter=1000, tol=1e-4, copy_X=True,
-                 cv=None, verbose=False, n_jobs=None, random_state=None,
-                 selection='cyclic'):
+    def __init__(
+        self,
+        *,
+        eps=1e-3,
+        n_alphas="deprecated",
+        alphas="warn",
+        fit_intercept=True,
+        max_iter=1000,
+        tol=1e-4,
+        copy_X=True,
+        cv=None,
+        verbose=False,
+        n_jobs=None,
+        random_state=None,
+        selection="cyclic",
+    ):
         super().__init__(
-            eps=eps, n_alphas=n_alphas, alphas=alphas,
-            fit_intercept=fit_intercept, normalize=normalize,
-            max_iter=max_iter, tol=tol, copy_X=copy_X,
-            cv=cv, verbose=verbose, n_jobs=n_jobs, random_state=random_state,
-            selection=selection)
+            eps=eps,
+            n_alphas=n_alphas,
+            alphas=alphas,
+            fit_intercept=fit_intercept,
+            max_iter=max_iter,
+            tol=tol,
+            copy_X=copy_X,
+            cv=cv,
+            verbose=verbose,
+            n_jobs=n_jobs,
+            random_state=random_state,
+            selection=selection,
+        )
 
     def _get_estimator(self):
         return MultiTaskLasso()
@@ -2460,5 +3357,38 @@ def _get_estimator(self):
     def _is_multitask(self):
         return True
 
-    def _more_tags(self):
-        return {'multioutput_only': True}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.target_tags.single_output = False
+        return tags
+
+    # This is necessary as LinearModelCV now supports sample_weight while
+    # MultiTaskLassoCV does not (yet).
+    def fit(self, X, y, **params):
+        """Fit MultiTaskLasso model with coordinate descent.
+
+        Fit is on grid of alphas and best alpha estimated by cross-validation.
+
+        Parameters
+        ----------
+        X : ndarray of shape (n_samples, n_features)
+            Data.
+        y : ndarray of shape (n_samples, n_targets)
+            Target. Will be cast to X's dtype if necessary.
+
+        **params : dict, default=None
+            Parameters to be passed to the CV splitter.
+
+            .. versionadded:: 1.4
+                Only available if `enable_metadata_routing=True`,
+                which can be set by using
+                ``sklearn.set_config(enable_metadata_routing=True)``.
+                See :ref:`Metadata Routing User Guide ` for
+                more details.
+
+        Returns
+        -------
+        self : object
+            Returns an instance of fitted model.
+        """
+        return super().fit(X, y, **params)
diff --git a/sklearn/linear_model/_glm/__init__.py b/sklearn/linear_model/_glm/__init__.py
index 3b5c0d95d6124..5c471c35096f8 100644
--- a/sklearn/linear_model/_glm/__init__.py
+++ b/sklearn/linear_model/_glm/__init__.py
@@ -1,15 +1,16 @@
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 from .glm import (
-    GeneralizedLinearRegressor,
-    PoissonRegressor,
     GammaRegressor,
-    TweedieRegressor
+    PoissonRegressor,
+    TweedieRegressor,
+    _GeneralizedLinearRegressor,
 )
 
 __all__ = [
-    "GeneralizedLinearRegressor",
-    "PoissonRegressor",
     "GammaRegressor",
-    "TweedieRegressor"
+    "PoissonRegressor",
+    "TweedieRegressor",
+    "_GeneralizedLinearRegressor",
 ]
diff --git a/sklearn/linear_model/_glm/_newton_solver.py b/sklearn/linear_model/_glm/_newton_solver.py
new file mode 100644
index 0000000000000..c5c940bed6c39
--- /dev/null
+++ b/sklearn/linear_model/_glm/_newton_solver.py
@@ -0,0 +1,617 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""
+Newton solver for Generalized Linear Models
+"""
+
+import warnings
+from abc import ABC, abstractmethod
+
+import numpy as np
+import scipy.linalg
+import scipy.optimize
+
+from ..._loss.loss import HalfSquaredError
+from ...exceptions import ConvergenceWarning
+from ...utils.optimize import _check_optimize_result
+from .._linear_loss import LinearModelLoss
+
+
+class NewtonSolver(ABC):
+    """Newton solver for GLMs.
+
+    This class implements Newton/2nd-order optimization routines for GLMs. Each Newton
+    iteration aims at finding the Newton step which is done by the inner solver. With
+    Hessian H, gradient g and coefficients coef, one step solves:
+
+        H @ coef_newton = -g
+
+    For our GLM / LinearModelLoss, we have gradient g and Hessian H:
+
+        g = X.T @ loss.gradient + l2_reg_strength * coef
+        H = X.T @ diag(loss.hessian) @ X + l2_reg_strength * identity
+
+    Backtracking line search updates coef = coef_old + t * coef_newton for some t in
+    (0, 1].
+
+    This is a base class, actual implementations (child classes) may deviate from the
+    above pattern and use structure specific tricks.
+
+    Usage pattern:
+        - initialize solver: sol = NewtonSolver(...)
+        - solve the problem: sol.solve(X, y, sample_weight)
+
+    References
+    ----------
+    - Jorge Nocedal, Stephen J. Wright. (2006) "Numerical Optimization"
+      2nd edition
+      https://doi.org/10.1007/978-0-387-40065-5
+
+    - Stephen P. Boyd, Lieven Vandenberghe. (2004) "Convex Optimization."
+      Cambridge University Press, 2004.
+      https://web.stanford.edu/~boyd/cvxbook/bv_cvxbook.pdf
+
+    Parameters
+    ----------
+    coef : ndarray of shape (n_dof,), (n_classes, n_dof) or (n_classes * n_dof,)
+        Initial coefficients of a linear model.
+        If shape (n_classes * n_dof,), the classes of one feature are contiguous,
+        i.e. one reconstructs the 2d-array via
+        coef.reshape((n_classes, -1), order="F").
+
+    linear_loss : LinearModelLoss
+        The loss to be minimized.
+
+    l2_reg_strength : float, default=0.0
+        L2 regularization strength.
+
+    tol : float, default=1e-4
+        The optimization problem is solved when each of the following condition is
+        fulfilled:
+        1. maximum |gradient| <= tol
+        2. Newton decrement d: 1/2 * d^2 <= tol
+
+    max_iter : int, default=100
+        Maximum number of Newton steps allowed.
+
+    n_threads : int, default=1
+        Number of OpenMP threads to use for the computation of the Hessian and gradient
+        of the loss function.
+
+    Attributes
+    ----------
+    coef_old : ndarray of shape coef.shape
+        Coefficient of previous iteration.
+
+    coef_newton : ndarray of shape coef.shape
+        Newton step.
+
+    gradient : ndarray of shape coef.shape
+        Gradient of the loss w.r.t. the coefficients.
+
+    gradient_old : ndarray of shape coef.shape
+        Gradient of previous iteration.
+
+    loss_value : float
+        Value of objective function = loss + penalty.
+
+    loss_value_old : float
+        Value of objective function of previous itertion.
+
+    raw_prediction : ndarray of shape (n_samples,) or (n_samples, n_classes)
+
+    converged : bool
+        Indicator for convergence of the solver.
+
+    iteration : int
+        Number of Newton steps, i.e. calls to inner_solve
+
+    use_fallback_lbfgs_solve : bool
+        If set to True, the solver will resort to call LBFGS to finish the optimisation
+        procedure in case of convergence issues.
+
+    gradient_times_newton : float
+        gradient @ coef_newton, set in inner_solve and used by line_search. If the
+        Newton step is a descent direction, this is negative.
+    """
+
+    def __init__(
+        self,
+        *,
+        coef,
+        linear_loss=LinearModelLoss(base_loss=HalfSquaredError(), fit_intercept=True),
+        l2_reg_strength=0.0,
+        tol=1e-4,
+        max_iter=100,
+        n_threads=1,
+        verbose=0,
+    ):
+        self.coef = coef
+        self.linear_loss = linear_loss
+        self.l2_reg_strength = l2_reg_strength
+        self.tol = tol
+        self.max_iter = max_iter
+        self.n_threads = n_threads
+        self.verbose = verbose
+
+    def setup(self, X, y, sample_weight):
+        """Precomputations
+
+        If None, initializes:
+            - self.coef
+        Sets:
+            - self.raw_prediction
+            - self.loss_value
+        """
+        _, _, self.raw_prediction = self.linear_loss.weight_intercept_raw(self.coef, X)
+        self.loss_value = self.linear_loss.loss(
+            coef=self.coef,
+            X=X,
+            y=y,
+            sample_weight=sample_weight,
+            l2_reg_strength=self.l2_reg_strength,
+            n_threads=self.n_threads,
+            raw_prediction=self.raw_prediction,
+        )
+
+    @abstractmethod
+    def update_gradient_hessian(self, X, y, sample_weight):
+        """Update gradient and Hessian."""
+
+    @abstractmethod
+    def inner_solve(self, X, y, sample_weight):
+        """Compute Newton step.
+
+        Sets:
+            - self.coef_newton
+            - self.gradient_times_newton
+        """
+
+    def fallback_lbfgs_solve(self, X, y, sample_weight):
+        """Fallback solver in case of emergency.
+
+        If a solver detects convergence problems, it may fall back to this methods in
+        the hope to exit with success instead of raising an error.
+
+        Sets:
+            - self.coef
+            - self.converged
+        """
+        max_iter = self.max_iter - self.iteration
+        opt_res = scipy.optimize.minimize(
+            self.linear_loss.loss_gradient,
+            self.coef,
+            method="L-BFGS-B",
+            jac=True,
+            options={
+                "maxiter": max_iter,
+                "maxls": 50,  # default is 20
+                "iprint": self.verbose - 1,
+                "gtol": self.tol,
+                "ftol": 64 * np.finfo(np.float64).eps,
+            },
+            args=(X, y, sample_weight, self.l2_reg_strength, self.n_threads),
+        )
+        self.iteration += _check_optimize_result("lbfgs", opt_res, max_iter=max_iter)
+        self.coef = opt_res.x
+        self.converged = opt_res.status == 0
+
+    def line_search(self, X, y, sample_weight):
+        """Backtracking line search.
+
+        Sets:
+            - self.coef_old
+            - self.coef
+            - self.loss_value_old
+            - self.loss_value
+            - self.gradient_old
+            - self.gradient
+            - self.raw_prediction
+        """
+        # line search parameters
+        beta, sigma = 0.5, 0.00048828125  # 1/2, 1/2**11
+        eps = 16 * np.finfo(self.loss_value.dtype).eps
+        t = 1  # step size
+
+        # gradient_times_newton = self.gradient @ self.coef_newton
+        # was computed in inner_solve.
+        armijo_term = sigma * self.gradient_times_newton
+        _, _, raw_prediction_newton = self.linear_loss.weight_intercept_raw(
+            self.coef_newton, X
+        )
+
+        self.coef_old = self.coef
+        self.loss_value_old = self.loss_value
+        self.gradient_old = self.gradient
+
+        # np.sum(np.abs(self.gradient_old))
+        sum_abs_grad_old = -1
+
+        is_verbose = self.verbose >= 2
+        if is_verbose:
+            print("  Backtracking Line Search")
+            print(f"    eps=16 * finfo.eps={eps}")
+
+        for i in range(21):  # until and including t = beta**20 ~ 1e-6
+            self.coef = self.coef_old + t * self.coef_newton
+            raw = self.raw_prediction + t * raw_prediction_newton
+            self.loss_value, self.gradient = self.linear_loss.loss_gradient(
+                coef=self.coef,
+                X=X,
+                y=y,
+                sample_weight=sample_weight,
+                l2_reg_strength=self.l2_reg_strength,
+                n_threads=self.n_threads,
+                raw_prediction=raw,
+            )
+            # Note: If coef_newton is too large, loss_gradient may produce inf values,
+            # potentially accompanied by a RuntimeWarning.
+            # This case will be captured by the Armijo condition.
+
+            # 1. Check Armijo / sufficient decrease condition.
+            # The smaller (more negative) the better.
+            loss_improvement = self.loss_value - self.loss_value_old
+            check = loss_improvement <= t * armijo_term
+            if is_verbose:
+                print(
+                    f"    line search iteration={i + 1}, step size={t}\n"
+                    f"      check loss improvement <= armijo term: {loss_improvement} "
+                    f"<= {t * armijo_term} {check}"
+                )
+            if check:
+                break
+            # 2. Deal with relative loss differences around machine precision.
+            tiny_loss = np.abs(self.loss_value_old * eps)
+            check = np.abs(loss_improvement) <= tiny_loss
+            if is_verbose:
+                print(
+                    "      check loss |improvement| <= eps * |loss_old|:"
+                    f" {np.abs(loss_improvement)} <= {tiny_loss} {check}"
+                )
+            if check:
+                if sum_abs_grad_old < 0:
+                    sum_abs_grad_old = scipy.linalg.norm(self.gradient_old, ord=1)
+                # 2.1 Check sum of absolute gradients as alternative condition.
+                sum_abs_grad = scipy.linalg.norm(self.gradient, ord=1)
+                check = sum_abs_grad < sum_abs_grad_old
+                if is_verbose:
+                    print(
+                        "      check sum(|gradient|) < sum(|gradient_old|): "
+                        f"{sum_abs_grad} < {sum_abs_grad_old} {check}"
+                    )
+                if check:
+                    break
+
+            t *= beta
+        else:
+            warnings.warn(
+                (
+                    f"Line search of Newton solver {self.__class__.__name__} at"
+                    f" iteration #{self.iteration} did no converge after 21 line search"
+                    " refinement iterations. It will now resort to lbfgs instead."
+                ),
+                ConvergenceWarning,
+            )
+            if self.verbose:
+                print("  Line search did not converge and resorts to lbfgs instead.")
+            self.use_fallback_lbfgs_solve = True
+            return
+
+        self.raw_prediction = raw
+        if is_verbose:
+            print(
+                f"    line search successful after {i + 1} iterations with "
+                f"loss={self.loss_value}."
+            )
+
+    def check_convergence(self, X, y, sample_weight):
+        """Check for convergence.
+
+        Sets self.converged.
+        """
+        if self.verbose:
+            print("  Check Convergence")
+        # Note: Checking maximum relative change of coefficient <= tol is a bad
+        # convergence criterion because even a large step could have brought us close
+        # to the true minimum.
+        # coef_step = self.coef - self.coef_old
+        # change = np.max(np.abs(coef_step) / np.maximum(1, np.abs(self.coef_old)))
+        # check = change <= tol
+
+        # 1. Criterion: maximum |gradient| <= tol
+        #    The gradient was already updated in line_search()
+        g_max_abs = np.max(np.abs(self.gradient))
+        check = g_max_abs <= self.tol
+        if self.verbose:
+            print(f"    1. max |gradient| {g_max_abs} <= {self.tol} {check}")
+        if not check:
+            return
+
+        # 2. Criterion: For Newton decrement d, check 1/2 * d^2 <= tol
+        #       d = sqrt(grad @ hessian^-1 @ grad)
+        #         = sqrt(coef_newton @ hessian @ coef_newton)
+        #    See Boyd, Vanderberghe (2009) "Convex Optimization" Chapter 9.5.1.
+        d2 = self.coef_newton @ self.hessian @ self.coef_newton
+        check = 0.5 * d2 <= self.tol
+        if self.verbose:
+            print(f"    2. Newton decrement {0.5 * d2} <= {self.tol} {check}")
+        if not check:
+            return
+
+        if self.verbose:
+            loss_value = self.linear_loss.loss(
+                coef=self.coef,
+                X=X,
+                y=y,
+                sample_weight=sample_weight,
+                l2_reg_strength=self.l2_reg_strength,
+                n_threads=self.n_threads,
+            )
+            print(f"  Solver did converge at loss = {loss_value}.")
+        self.converged = True
+
+    def finalize(self, X, y, sample_weight):
+        """Finalize the solvers results.
+
+        Some solvers may need this, others not.
+        """
+        pass
+
+    def solve(self, X, y, sample_weight):
+        """Solve the optimization problem.
+
+        This is the main routine.
+
+        Order of calls:
+            self.setup()
+            while iteration:
+                self.update_gradient_hessian()
+                self.inner_solve()
+                self.line_search()
+                self.check_convergence()
+            self.finalize()
+
+        Returns
+        -------
+        coef : ndarray of shape (n_dof,), (n_classes, n_dof) or (n_classes * n_dof,)
+            Solution of the optimization problem.
+        """
+        # setup usually:
+        #   - initializes self.coef if needed
+        #   - initializes and calculates self.raw_predictions, self.loss_value
+        self.setup(X=X, y=y, sample_weight=sample_weight)
+
+        self.iteration = 1
+        self.converged = False
+        self.use_fallback_lbfgs_solve = False
+
+        while self.iteration <= self.max_iter and not self.converged:
+            if self.verbose:
+                print(f"Newton iter={self.iteration}")
+
+            self.use_fallback_lbfgs_solve = False  # Fallback solver.
+
+            # 1. Update Hessian and gradient
+            self.update_gradient_hessian(X=X, y=y, sample_weight=sample_weight)
+
+            # TODO:
+            # if iteration == 1:
+            # We might stop early, e.g. we already are close to the optimum,
+            # usually detected by zero gradients at this stage.
+
+            # 2. Inner solver
+            #    Calculate Newton step/direction
+            #    This usually sets self.coef_newton and self.gradient_times_newton.
+            self.inner_solve(X=X, y=y, sample_weight=sample_weight)
+            if self.use_fallback_lbfgs_solve:
+                break
+
+            # 3. Backtracking line search
+            #    This usually sets self.coef_old, self.coef, self.loss_value_old
+            #    self.loss_value, self.gradient_old, self.gradient,
+            #    self.raw_prediction.
+            self.line_search(X=X, y=y, sample_weight=sample_weight)
+            if self.use_fallback_lbfgs_solve:
+                break
+
+            # 4. Check convergence
+            #    Sets self.converged.
+            self.check_convergence(X=X, y=y, sample_weight=sample_weight)
+
+            # 5. Next iteration
+            self.iteration += 1
+
+        if not self.converged:
+            if self.use_fallback_lbfgs_solve:
+                # Note: The fallback solver circumvents check_convergence and relies on
+                # the convergence checks of lbfgs instead. Enough warnings have been
+                # raised on the way.
+                self.fallback_lbfgs_solve(X=X, y=y, sample_weight=sample_weight)
+            else:
+                warnings.warn(
+                    (
+                        f"Newton solver did not converge after {self.iteration - 1} "
+                        "iterations."
+                    ),
+                    ConvergenceWarning,
+                )
+
+        self.iteration -= 1
+        self.finalize(X=X, y=y, sample_weight=sample_weight)
+        return self.coef
+
+
+class NewtonCholeskySolver(NewtonSolver):
+    """Cholesky based Newton solver.
+
+    Inner solver for finding the Newton step H w_newton = -g uses Cholesky based linear
+    solver.
+    """
+
+    def setup(self, X, y, sample_weight):
+        super().setup(X=X, y=y, sample_weight=sample_weight)
+        if self.linear_loss.base_loss.is_multiclass:
+            # Easier with ravelled arrays, e.g., for scipy.linalg.solve.
+            # As with LinearModelLoss, we always are contiguous in n_classes.
+            self.coef = self.coef.ravel(order="F")
+        # Note that the computation of gradient in LinearModelLoss follows the shape of
+        # coef.
+        self.gradient = np.empty_like(self.coef)
+        # But the hessian is always 2d.
+        n = self.coef.size
+        self.hessian = np.empty_like(self.coef, shape=(n, n))
+        # To help case distinctions.
+        self.is_multinomial_with_intercept = (
+            self.linear_loss.base_loss.is_multiclass and self.linear_loss.fit_intercept
+        )
+        self.is_multinomial_no_penalty = (
+            self.linear_loss.base_loss.is_multiclass and self.l2_reg_strength == 0
+        )
+
+    def update_gradient_hessian(self, X, y, sample_weight):
+        _, _, self.hessian_warning = self.linear_loss.gradient_hessian(
+            coef=self.coef,
+            X=X,
+            y=y,
+            sample_weight=sample_weight,
+            l2_reg_strength=self.l2_reg_strength,
+            n_threads=self.n_threads,
+            gradient_out=self.gradient,
+            hessian_out=self.hessian,
+            raw_prediction=self.raw_prediction,  # this was updated in line_search
+        )
+
+    def inner_solve(self, X, y, sample_weight):
+        if self.hessian_warning:
+            warnings.warn(
+                (
+                    f"The inner solver of {self.__class__.__name__} detected a "
+                    "pointwise hessian with many negative values at iteration "
+                    f"#{self.iteration}. It will now resort to lbfgs instead."
+                ),
+                ConvergenceWarning,
+            )
+            if self.verbose:
+                print(
+                    "  The inner solver detected a pointwise Hessian with many "
+                    "negative values and resorts to lbfgs instead."
+                )
+            self.use_fallback_lbfgs_solve = True
+            return
+
+        # Note: The following case distinction could also be shifted to the
+        # implementation of HalfMultinomialLoss instead of here within the solver.
+        if self.is_multinomial_no_penalty:
+            # The multinomial loss is overparametrized for each unpenalized feature, so
+            # at least the intercepts. This can be seen by noting that predicted
+            # probabilities are invariant under shifting all coefficients of a single
+            # feature j for all classes by the same amount c:
+            #   coef[k, :] -> coef[k, :] + c    =>    proba stays the same
+            # where we have assumed coef.shape = (n_classes, n_features).
+            # Therefore, also the loss (-log-likelihood), gradient and hessian stay the
+            # same, see
+            # Noah Simon and Jerome Friedman and Trevor Hastie. (2013) "A Blockwise
+            # Descent Algorithm for Group-penalized Multiresponse and Multinomial
+            # Regression". https://doi.org/10.48550/arXiv.1311.6529
+            #
+            # We choose the standard approach and set all the coefficients of the last
+            # class to zero, for all features including the intercept.
+            n_classes = self.linear_loss.base_loss.n_classes
+            n_dof = self.coef.size // n_classes  # degree of freedom per class
+            n = self.coef.size - n_dof  # effective size
+            self.coef[n_classes - 1 :: n_classes] = 0
+            self.gradient[n_classes - 1 :: n_classes] = 0
+            self.hessian[n_classes - 1 :: n_classes, :] = 0
+            self.hessian[:, n_classes - 1 :: n_classes] = 0
+            # We also need the reduced variants of gradient and hessian where the
+            # entries set to zero are removed. For 2 features and 3 classes with
+            # arbitrary values, "x" means removed:
+            #   gradient = [0, 1, x, 3, 4, x]
+            #
+            #   hessian = [0,  1, x,  3,  4, x]
+            #             [1,  7, x,  9, 10, x]
+            #             [x,  x, x,  x,  x, x]
+            #             [3,  9, x, 21, 22, x]
+            #             [4, 10, x, 22, 28, x]
+            #             [x,  x, x,  x, x,  x]
+            # The following slicing triggers copies of gradient and hessian.
+            gradient = self.gradient.reshape(-1, n_classes)[:, :-1].flatten()
+            hessian = self.hessian.reshape(n_dof, n_classes, n_dof, n_classes)[
+                :, :-1, :, :-1
+            ].reshape(n, n)
+        elif self.is_multinomial_with_intercept:
+            # Here, only intercepts are unpenalized. We again choose the last class and
+            # set its intercept to zero.
+            self.coef[-1] = 0
+            self.gradient[-1] = 0
+            self.hessian[-1, :] = 0
+            self.hessian[:, -1] = 0
+            gradient, hessian = self.gradient[:-1], self.hessian[:-1, :-1]
+        else:
+            gradient, hessian = self.gradient, self.hessian
+
+        try:
+            with warnings.catch_warnings():
+                warnings.simplefilter("error", scipy.linalg.LinAlgWarning)
+                self.coef_newton = scipy.linalg.solve(
+                    hessian, -gradient, check_finite=False, assume_a="sym"
+                )
+                if self.is_multinomial_no_penalty:
+                    self.coef_newton = np.c_[
+                        self.coef_newton.reshape(n_dof, n_classes - 1), np.zeros(n_dof)
+                    ].reshape(-1)
+                    assert self.coef_newton.flags.f_contiguous
+                elif self.is_multinomial_with_intercept:
+                    self.coef_newton = np.r_[self.coef_newton, 0]
+                self.gradient_times_newton = self.gradient @ self.coef_newton
+                if self.gradient_times_newton > 0:
+                    if self.verbose:
+                        print(
+                            "  The inner solver found a Newton step that is not a "
+                            "descent direction and resorts to LBFGS steps instead."
+                        )
+                    self.use_fallback_lbfgs_solve = True
+                    return
+        except (np.linalg.LinAlgError, scipy.linalg.LinAlgWarning) as e:
+            warnings.warn(
+                f"The inner solver of {self.__class__.__name__} stumbled upon a "
+                "singular or very ill-conditioned Hessian matrix at iteration "
+                f"{self.iteration}. It will now resort to lbfgs instead.\n"
+                "Further options are to use another solver or to avoid such situation "
+                "in the first place. Possible remedies are removing collinear features"
+                " of X or increasing the penalization strengths.\n"
+                "The original Linear Algebra message was:\n" + str(e),
+                scipy.linalg.LinAlgWarning,
+            )
+            # Possible causes:
+            # 1. hess_pointwise is negative. But this is already taken care in
+            #    LinearModelLoss.gradient_hessian.
+            # 2. X is singular or ill-conditioned
+            #    This might be the most probable cause.
+            #
+            # There are many possible ways to deal with this situation. Most of them
+            # add, explicitly or implicitly, a matrix to the hessian to make it
+            # positive definite, confer to Chapter 3.4 of Nocedal & Wright 2nd ed.
+            # Instead, we resort to lbfgs.
+            if self.verbose:
+                print(
+                    "  The inner solver stumbled upon an singular or ill-conditioned "
+                    "Hessian matrix and resorts to LBFGS instead."
+                )
+            self.use_fallback_lbfgs_solve = True
+            return
+
+    def finalize(self, X, y, sample_weight):
+        if self.is_multinomial_no_penalty:
+            # Our convention is usually the symmetric parametrization where
+            # sum(coef[classes, features], axis=0) = 0.
+            # We convert now to this convention. Note that it does not change
+            # the predicted probabilities.
+            n_classes = self.linear_loss.base_loss.n_classes
+            self.coef = self.coef.reshape(n_classes, -1, order="F")
+            self.coef -= np.mean(self.coef, axis=0)
+        elif self.is_multinomial_with_intercept:
+            # Only the intercept needs an update to the symmetric parametrization.
+            n_classes = self.linear_loss.base_loss.n_classes
+            self.coef[-n_classes:] -= np.mean(self.coef[-n_classes:])
diff --git a/sklearn/linear_model/_glm/glm.py b/sklearn/linear_model/_glm/glm.py
index 7d98f7734b322..7f138f420dc36 100644
--- a/sklearn/linear_model/_glm/glm.py
+++ b/sklearn/linear_model/_glm/glm.py
@@ -1,67 +1,53 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
 """
 Generalized Linear Models with Exponential Dispersion Family
 """
 
-# Author: Christian Lorentzen 
-# some parts and tricks stolen from other sklearn files.
-# License: BSD 3 clause
-
-import numbers
+from numbers import Integral, Real
 
 import numpy as np
 import scipy.optimize
 
-from ...base import BaseEstimator, RegressorMixin
-from ...utils.optimize import _check_optimize_result
-from ...utils.validation import check_is_fitted, _check_sample_weight
-from ..._loss.glm_distribution import (
-        ExponentialDispersionModel,
-        TweedieDistribution,
-        EDM_DISTRIBUTIONS
-)
-from .link import (
-        BaseLink,
-        IdentityLink,
-        LogLink,
+from ..._loss.loss import (
+    HalfGammaLoss,
+    HalfPoissonLoss,
+    HalfSquaredError,
+    HalfTweedieLoss,
+    HalfTweedieLossIdentity,
 )
+from ...base import BaseEstimator, RegressorMixin, _fit_context
+from ...utils import check_array
+from ...utils._openmp_helpers import _openmp_effective_n_threads
+from ...utils._param_validation import Hidden, Interval, StrOptions
+from ...utils.optimize import _check_optimize_result
+from ...utils.validation import _check_sample_weight, check_is_fitted, validate_data
+from .._linear_loss import LinearModelLoss
+from ._newton_solver import NewtonCholeskySolver, NewtonSolver
 
 
-def _safe_lin_pred(X, coef):
-    """Compute the linear predictor taking care if intercept is present."""
-    if coef.size == X.shape[1] + 1:
-        return X @ coef[1:] + coef[0]
-    else:
-        return X @ coef
-
-
-def _y_pred_deviance_derivative(coef, X, y, weights, family, link):
-    """Compute y_pred and the derivative of the deviance w.r.t coef."""
-    lin_pred = _safe_lin_pred(X, coef)
-    y_pred = link.inverse(lin_pred)
-    d1 = link.inverse_derivative(lin_pred)
-    temp = d1 * family.deviance_derivative(y, y_pred, weights)
-    if coef.size == X.shape[1] + 1:
-        devp = np.concatenate(([temp.sum()], temp @ X))
-    else:
-        devp = temp @ X  # same as X.T @ temp
-    return y_pred, devp
-
-
-class GeneralizedLinearRegressor(RegressorMixin, BaseEstimator):
+class _GeneralizedLinearRegressor(RegressorMixin, BaseEstimator):
     """Regression via a penalized Generalized Linear Model (GLM).
 
-    GLMs based on a reproductive Exponential Dispersion Model (EDM) aim at
-    fitting and predicting the mean of the target y as y_pred=h(X*w).
-    Therefore, the fit minimizes the following objective function with L2
-    priors as regularizer::
+    GLMs based on a reproductive Exponential Dispersion Model (EDM) aim at fitting and
+    predicting the mean of the target y as y_pred=h(X*w) with coefficients w.
+    Therefore, the fit minimizes the following objective function with L2 priors as
+    regularizer::
 
-            1/(2*sum(s)) * deviance(y, h(X*w); s)
-            + 1/2 * alpha * |w|_2
+        1/(2*sum(s_i)) * sum(s_i * deviance(y_i, h(x_i*w)) + 1/2 * alpha * ||w||_2^2
 
-    with inverse link function h and s=sample_weight.
+    with inverse link function h, s=sample_weight and per observation (unit) deviance
+    deviance(y_i, h(x_i*w)). Note that for an EDM, 1/2 * deviance is the negative
+    log-likelihood up to a constant (in w) term.
     The parameter ``alpha`` corresponds to the lambda parameter in glmnet.
 
-    Read more in the :ref:`User Guide `.
+    Instead of implementing the EDM family and a link function separately, we directly
+    use the loss functions `from sklearn._loss` which have the link functions included
+    in them for performance reasons. We pick the loss functions that implement
+    (1/2 times) EDM deviances.
+
+    Read more in the :ref:`User Guide `.
 
     .. versionadded:: 0.23
 
@@ -72,39 +58,38 @@ class GeneralizedLinearRegressor(RegressorMixin, BaseEstimator):
         regularization strength. ``alpha = 0`` is equivalent to unpenalized
         GLMs. In this case, the design matrix `X` must have full column rank
         (no collinearities).
+        Values must be in the range `[0.0, inf)`.
 
     fit_intercept : bool, default=True
         Specifies if a constant (a.k.a. bias or intercept) should be
         added to the linear predictor (X @ coef + intercept).
 
-    family : {'normal', 'poisson', 'gamma', 'inverse-gaussian'} \
-            or an ExponentialDispersionModel instance, default='normal'
-        The distributional assumption of the GLM, i.e. which distribution from
-        the EDM, specifies the loss function to be minimized.
-
-    link : {'auto', 'identity', 'log'} or an instance of class BaseLink, \
-            default='auto'
-        The link function of the GLM, i.e. mapping from linear predictor
-        `X @ coeff + intercept` to prediction `y_pred`. Option 'auto' sets
-        the link depending on the chosen family as follows:
-
-        - 'identity' for Normal distribution
-        - 'log' for Poisson,  Gamma and Inverse Gaussian distributions
-
-    solver : 'lbfgs', default='lbfgs'
+    solver : {'lbfgs', 'newton-cholesky'}, default='lbfgs'
         Algorithm to use in the optimization problem:
 
         'lbfgs'
             Calls scipy's L-BFGS-B optimizer.
 
+        'newton-cholesky'
+            Uses Newton-Raphson steps (in arbitrary precision arithmetic equivalent to
+            iterated reweighted least squares) with an inner Cholesky based solver.
+            This solver is a good choice for `n_samples` >> `n_features`, especially
+            with one-hot encoded categorical features with rare categories. Be aware
+            that the memory usage of this solver has a quadratic dependency on
+            `n_features` because it explicitly computes the Hessian matrix.
+
+            .. versionadded:: 1.2
+
     max_iter : int, default=100
         The maximal number of iterations for the solver.
+        Values must be in the range `[1, inf)`.
 
     tol : float, default=1e-4
         Stopping criterion. For the lbfgs solver,
         the iteration will stop when ``max{|g_j|, j = 1, ..., d} <= tol``
         where ``g_j`` is the j-th component of the gradient (derivative) of
         the objective function.
+        Values must be in the range `(0.0, inf)`.
 
     warm_start : bool, default=False
         If set to ``True``, reuse the solution of the previous call to ``fit``
@@ -112,6 +97,7 @@ class GeneralizedLinearRegressor(RegressorMixin, BaseEstimator):
 
     verbose : int, default=0
         For the lbfgs solver set verbose to any positive number for verbosity.
+        Values must be in the range `[0, inf)`.
 
     Attributes
     ----------
@@ -124,21 +110,64 @@ class GeneralizedLinearRegressor(RegressorMixin, BaseEstimator):
 
     n_iter_ : int
         Actual number of iterations used in the solver.
+
+    _base_loss : BaseLoss, default=HalfSquaredError()
+        This is set during fit via `self._get_loss()`.
+        A `_base_loss` contains a specific loss function as well as the link
+        function. The loss to be minimized specifies the distributional assumption of
+        the GLM, i.e. the distribution from the EDM. Here are some examples:
+
+        =======================  ========  ==========================
+        _base_loss               Link      Target Domain
+        =======================  ========  ==========================
+        HalfSquaredError         identity  y any real number
+        HalfPoissonLoss          log       0 <= y
+        HalfGammaLoss            log       0 < y
+        HalfTweedieLoss          log       dependent on tweedie power
+        HalfTweedieLossIdentity  identity  dependent on tweedie power
+        =======================  ========  ==========================
+
+        The link function of the GLM, i.e. mapping from linear predictor
+        `X @ coeff + intercept` to prediction `y_pred`. For instance, with a log link,
+        we have `y_pred = exp(X @ coeff + intercept)`.
     """
-    def __init__(self, *, alpha=1.0,
-                 fit_intercept=True, family='normal', link='auto',
-                 solver='lbfgs', max_iter=100, tol=1e-4, warm_start=False,
-                 verbose=0):
+
+    # We allow for NewtonSolver classes for the "solver" parameter but do not
+    # make them public in the docstrings. This facilitates testing and
+    # benchmarking.
+    _parameter_constraints: dict = {
+        "alpha": [Interval(Real, 0.0, None, closed="left")],
+        "fit_intercept": ["boolean"],
+        "solver": [
+            StrOptions({"lbfgs", "newton-cholesky"}),
+            Hidden(type),
+        ],
+        "max_iter": [Interval(Integral, 1, None, closed="left")],
+        "tol": [Interval(Real, 0.0, None, closed="neither")],
+        "warm_start": ["boolean"],
+        "verbose": ["verbose"],
+    }
+
+    def __init__(
+        self,
+        *,
+        alpha=1.0,
+        fit_intercept=True,
+        solver="lbfgs",
+        max_iter=100,
+        tol=1e-4,
+        warm_start=False,
+        verbose=0,
+    ):
         self.alpha = alpha
         self.fit_intercept = fit_intercept
-        self.family = family
-        self.link = link
         self.solver = solver
         self.max_iter = max_iter
         self.tol = tol
         self.warm_start = warm_start
         self.verbose = verbose
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y, sample_weight=None):
         """Fit a Generalized Linear Model.
 
@@ -155,143 +184,140 @@ def fit(self, X, y, sample_weight=None):
 
         Returns
         -------
-        self : returns an instance of self.
+        self : object
+            Fitted model.
         """
-        if isinstance(self.family, ExponentialDispersionModel):
-            self._family_instance = self.family
-        elif self.family in EDM_DISTRIBUTIONS:
-            self._family_instance = EDM_DISTRIBUTIONS[self.family]()
+        X, y = validate_data(
+            self,
+            X,
+            y,
+            accept_sparse=["csc", "csr"],
+            dtype=[np.float64, np.float32],
+            y_numeric=True,
+            multi_output=False,
+        )
+
+        # required by losses
+        if self.solver == "lbfgs":
+            # lbfgs will force coef and therefore raw_prediction to be float64. The
+            # base_loss needs y, X @ coef and sample_weight all of same dtype
+            # (and contiguous).
+            loss_dtype = np.float64
         else:
+            loss_dtype = min(max(y.dtype, X.dtype), np.float64)
+        y = check_array(y, dtype=loss_dtype, order="C", ensure_2d=False)
+
+        if sample_weight is not None:
+            # Note that _check_sample_weight calls check_array(order="C") required by
+            # losses.
+            sample_weight = _check_sample_weight(sample_weight, X, dtype=loss_dtype)
+
+        n_samples, n_features = X.shape
+        self._base_loss = self._get_loss()
+
+        linear_loss = LinearModelLoss(
+            base_loss=self._base_loss,
+            fit_intercept=self.fit_intercept,
+        )
+
+        if not linear_loss.base_loss.in_y_true_range(y):
             raise ValueError(
-                "The family must be an instance of class"
-                " ExponentialDispersionModel or an element of"
-                " ['normal', 'poisson', 'gamma', 'inverse-gaussian']"
-                "; got (family={0})".format(self.family))
-
-        # Guarantee that self._link_instance is set to an instance of
-        # class BaseLink
-        if isinstance(self.link, BaseLink):
-            self._link_instance = self.link
-        else:
-            if self.link == 'auto':
-                if isinstance(self._family_instance, TweedieDistribution):
-                    if self._family_instance.power <= 0:
-                        self._link_instance = IdentityLink()
-                    if self._family_instance.power >= 1:
-                        self._link_instance = LogLink()
-                else:
-                    raise ValueError("No default link known for the "
-                                     "specified distribution family. Please "
-                                     "set link manually, i.e. not to 'auto'; "
-                                     "got (link='auto', family={})"
-                                     .format(self.family))
-            elif self.link == 'identity':
-                self._link_instance = IdentityLink()
-            elif self.link == 'log':
-                self._link_instance = LogLink()
-            else:
-                raise ValueError(
-                    "The link must be an instance of class Link or "
-                    "an element of ['auto', 'identity', 'log']; "
-                    "got (link={0})".format(self.link))
-
-        if not isinstance(self.alpha, numbers.Number) or self.alpha < 0:
-            raise ValueError("Penalty term must be a non-negative number;"
-                             " got (alpha={0})".format(self.alpha))
-        if not isinstance(self.fit_intercept, bool):
-            raise ValueError("The argument fit_intercept must be bool;"
-                             " got {0}".format(self.fit_intercept))
-        if self.solver not in ['lbfgs']:
-            raise ValueError("GeneralizedLinearRegressor supports only solvers"
-                             "'lbfgs'; got {0}".format(self.solver))
-        solver = self.solver
-        if (not isinstance(self.max_iter, numbers.Integral)
-                or self.max_iter <= 0):
-            raise ValueError("Maximum number of iteration must be a positive "
-                             "integer;"
-                             " got (max_iter={0!r})".format(self.max_iter))
-        if not isinstance(self.tol, numbers.Number) or self.tol <= 0:
-            raise ValueError("Tolerance for stopping criteria must be "
-                             "positive; got (tol={0!r})".format(self.tol))
-        if not isinstance(self.warm_start, bool):
-            raise ValueError("The argument warm_start must be bool;"
-                             " got {0}".format(self.warm_start))
-
-        family = self._family_instance
-        link = self._link_instance
-
-        X, y = self._validate_data(X, y, accept_sparse=['csc', 'csr'],
-                                   dtype=[np.float64, np.float32],
-                                   y_numeric=True, multi_output=False)
-
-        weights = _check_sample_weight(sample_weight, X)
-
-        _, n_features = X.shape
-
-        if not np.all(family.in_y_range(y)):
-            raise ValueError("Some value(s) of y are out of the valid "
-                             "range for family {0}"
-                             .format(family.__class__.__name__))
+                "Some value(s) of y are out of the valid range of the loss"
+                f" {self._base_loss.__class__.__name__!r}."
+            )
+
         # TODO: if alpha=0 check that X is not rank deficient
 
-        # rescaling of sample_weight
-        #
-        # IMPORTANT NOTE: Since we want to minimize
-        # 1/(2*sum(sample_weight)) * deviance + L2,
-        # deviance = sum(sample_weight * unit_deviance),
-        # we rescale weights such that sum(weights) = 1 and this becomes
-        # 1/2*deviance + L2 with deviance=sum(weights * unit_deviance)
-        weights = weights / weights.sum()
-
-        if self.warm_start and hasattr(self, 'coef_'):
+        # NOTE: Rescaling of sample_weight:
+        # We want to minimize
+        #     obj = 1/(2 * sum(sample_weight)) * sum(sample_weight * deviance)
+        #         + 1/2 * alpha * L2,
+        # with
+        #     deviance = 2 * loss.
+        # The objective is invariant to multiplying sample_weight by a constant. We
+        # could choose this constant such that sum(sample_weight) = 1 in order to end
+        # up with
+        #     obj = sum(sample_weight * loss) + 1/2 * alpha * L2.
+        # But LinearModelLoss.loss() already computes
+        #     average(loss, weights=sample_weight)
+        # Thus, without rescaling, we have
+        #     obj = LinearModelLoss.loss(...)
+
+        if self.warm_start and hasattr(self, "coef_"):
             if self.fit_intercept:
-                coef = np.concatenate((np.array([self.intercept_]),
-                                       self.coef_))
+                # LinearModelLoss needs intercept at the end of coefficient array.
+                coef = np.concatenate((self.coef_, np.array([self.intercept_])))
             else:
                 coef = self.coef_
+            coef = coef.astype(loss_dtype, copy=False)
         else:
+            coef = linear_loss.init_zero_coef(X, dtype=loss_dtype)
             if self.fit_intercept:
-                coef = np.zeros(n_features+1)
-                coef[0] = link(np.average(y, weights=weights))
-            else:
-                coef = np.zeros(n_features)
-
-        # algorithms for optimization
-
-        if solver == 'lbfgs':
-            def func(coef, X, y, weights, alpha, family, link):
-                y_pred, devp = _y_pred_deviance_derivative(
-                    coef, X, y, weights, family, link
+                coef[-1] = linear_loss.base_loss.link.link(
+                    np.average(y, weights=sample_weight)
                 )
-                dev = family.deviance(y, y_pred, weights)
-                # offset if coef[0] is intercept
-                offset = 1 if self.fit_intercept else 0
-                coef_scaled = alpha * coef[offset:]
-                obj = 0.5 * dev + 0.5 * (coef[offset:] @ coef_scaled)
-                objp = 0.5 * devp
-                objp[offset:] += coef_scaled
-                return obj, objp
 
-            args = (X, y, weights, self.alpha, family, link)
+        l2_reg_strength = self.alpha
+        n_threads = _openmp_effective_n_threads()
+
+        # Algorithms for optimization:
+        # Note again that our losses implement 1/2 * deviance.
+        if self.solver == "lbfgs":
+            func = linear_loss.loss_gradient
 
             opt_res = scipy.optimize.minimize(
-                func, coef, method="L-BFGS-B", jac=True,
+                func,
+                coef,
+                method="L-BFGS-B",
+                jac=True,
                 options={
                     "maxiter": self.max_iter,
-                    "iprint": (self.verbose > 0) - 1,
+                    "maxls": 50,  # default is 20
+                    "iprint": self.verbose - 1,
                     "gtol": self.tol,
-                    "ftol": 1e3*np.finfo(float).eps,
+                    # The constant 64 was found empirically to pass the test suite.
+                    # The point is that ftol is very small, but a bit larger than
+                    # machine precision for float64, which is the dtype used by lbfgs.
+                    "ftol": 64 * np.finfo(float).eps,
                 },
-                args=args)
-            self.n_iter_ = _check_optimize_result("lbfgs", opt_res)
+                args=(X, y, sample_weight, l2_reg_strength, n_threads),
+            )
+            self.n_iter_ = _check_optimize_result(
+                "lbfgs", opt_res, max_iter=self.max_iter
+            )
             coef = opt_res.x
+        elif self.solver == "newton-cholesky":
+            sol = NewtonCholeskySolver(
+                coef=coef,
+                linear_loss=linear_loss,
+                l2_reg_strength=l2_reg_strength,
+                tol=self.tol,
+                max_iter=self.max_iter,
+                n_threads=n_threads,
+                verbose=self.verbose,
+            )
+            coef = sol.solve(X, y, sample_weight)
+            self.n_iter_ = sol.iteration
+        elif issubclass(self.solver, NewtonSolver):
+            sol = self.solver(
+                coef=coef,
+                linear_loss=linear_loss,
+                l2_reg_strength=l2_reg_strength,
+                tol=self.tol,
+                max_iter=self.max_iter,
+                n_threads=n_threads,
+            )
+            coef = sol.solve(X, y, sample_weight)
+            self.n_iter_ = sol.iteration
+        else:
+            raise ValueError(f"Invalid solver={self.solver}.")
 
         if self.fit_intercept:
-            self.intercept_ = coef[0]
-            self.coef_ = coef[1:]
+            self.intercept_ = coef[-1]
+            self.coef_ = coef[:-1]
         else:
             # set intercept to zero as the other linear models do
-            self.intercept_ = 0.
+            self.intercept_ = 0.0
             self.coef_ = coef
 
         return self
@@ -299,6 +325,8 @@ def func(coef, X, y, weights, alpha, family, link):
     def _linear_predictor(self, X):
         """Compute the linear_predictor = `X @ coef_ + intercept_`.
 
+        Note that we often use the term raw_prediction instead of linear predictor.
+
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
@@ -310,9 +338,15 @@ def _linear_predictor(self, X):
             Returns predicted values of linear predictor.
         """
         check_is_fitted(self)
-        X = self._validate_data(X, accept_sparse=['csr', 'csc', 'coo'],
-                                dtype=[np.float64, np.float32], ensure_2d=True,
-                                allow_nd=False, reset=False)
+        X = validate_data(
+            self,
+            X,
+            accept_sparse=["csr", "csc", "coo"],
+            dtype=[np.float64, np.float32],
+            ensure_2d=True,
+            allow_nd=False,
+            reset=False,
+        )
         return X @ self.coef_ + self.intercept_
 
     def predict(self, X):
@@ -329,16 +363,16 @@ def predict(self, X):
             Returns predicted values.
         """
         # check_array is done in _linear_predictor
-        eta = self._linear_predictor(X)
-        y_pred = self._link_instance.inverse(eta)
+        raw_prediction = self._linear_predictor(X)
+        y_pred = self._base_loss.link.inverse(raw_prediction)
         return y_pred
 
     def score(self, X, y, sample_weight=None):
         """Compute D^2, the percentage of deviance explained.
 
         D^2 is a generalization of the coefficient of determination R^2.
-        R^2 uses squared error and D^2 deviance. Note that those two are equal
-        for ``family='normal'``.
+        R^2 uses squared error and D^2 uses the deviance of this GLM, see the
+        :ref:`User Guide `.
 
         D^2 is defined as
         :math:`D^2 = 1-\\frac{D(y_{true},y_{pred})}{D_{null}}`,
@@ -364,58 +398,123 @@ def score(self, X, y, sample_weight=None):
         score : float
             D^2 of self.predict(X) w.r.t. y.
         """
+        # TODO: Adapt link to User Guide in the docstring, once
+        # https://github.com/scikit-learn/scikit-learn/pull/22118 is merged.
+        #
         # Note, default score defined in RegressorMixin is R^2 score.
         # TODO: make D^2 a score function in module metrics (and thereby get
         #       input validation and so on)
-        weights = _check_sample_weight(sample_weight, X)
-        y_pred = self.predict(X)
-        dev = self._family_instance.deviance(y, y_pred, weights=weights)
-        y_mean = np.average(y, weights=weights)
-        dev_null = self._family_instance.deviance(y, y_mean, weights=weights)
-        return 1 - dev / dev_null
-
-    def _more_tags(self):
-        # create the _family_instance if fit wasn't called yet.
-        if hasattr(self, '_family_instance'):
-            _family_instance = self._family_instance
-        elif isinstance(self.family, ExponentialDispersionModel):
-            _family_instance = self.family
-        elif self.family in EDM_DISTRIBUTIONS:
-            _family_instance = EDM_DISTRIBUTIONS[self.family]()
-        else:
-            raise ValueError
-        return {"requires_positive_y": not _family_instance.in_y_range(-1.0)}
+        raw_prediction = self._linear_predictor(X)  # validates X
+        # required by losses
+        y = check_array(y, dtype=raw_prediction.dtype, order="C", ensure_2d=False)
+
+        if sample_weight is not None:
+            # Note that _check_sample_weight calls check_array(order="C") required by
+            # losses.
+            sample_weight = _check_sample_weight(sample_weight, X, dtype=y.dtype)
 
+        base_loss = self._base_loss
+
+        if not base_loss.in_y_true_range(y):
+            raise ValueError(
+                "Some value(s) of y are out of the valid range of the loss"
+                f" {base_loss.__name__}."
+            )
+
+        constant = np.average(
+            base_loss.constant_to_optimal_zero(y_true=y, sample_weight=None),
+            weights=sample_weight,
+        )
+
+        # Missing factor of 2 in deviance cancels out.
+        deviance = base_loss(
+            y_true=y,
+            raw_prediction=raw_prediction,
+            sample_weight=sample_weight,
+            n_threads=1,
+        )
+        y_mean = base_loss.link.link(np.average(y, weights=sample_weight))
+        deviance_null = base_loss(
+            y_true=y,
+            raw_prediction=np.tile(y_mean, y.shape[0]),
+            sample_weight=sample_weight,
+            n_threads=1,
+        )
+        return 1 - (deviance + constant) / (deviance_null + constant)
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
+        try:
+            # Create instance of BaseLoss if fit wasn't called yet. This is necessary as
+            # TweedieRegressor might set the used loss during fit different from
+            # self._base_loss.
+            base_loss = self._get_loss()
+            tags.target_tags.positive_only = not base_loss.in_y_true_range(-1.0)
+        except (ValueError, AttributeError, TypeError):
+            # This happens when the link or power parameter of TweedieRegressor is
+            # invalid. We fallback on the default tags in that case.
+            pass  # pragma: no cover
+        return tags
+
+    def _get_loss(self):
+        """This is only necessary because of the link and power arguments of the
+        TweedieRegressor.
+
+        Note that we do not need to pass sample_weight to the loss class as this is
+        only needed to set loss.constant_hessian on which GLMs do not rely.
+        """
+        return HalfSquaredError()
 
-class PoissonRegressor(GeneralizedLinearRegressor):
+
+class PoissonRegressor(_GeneralizedLinearRegressor):
     """Generalized Linear Model with a Poisson distribution.
 
     This regressor uses the 'log' link function.
 
-    Read more in the :ref:`User Guide `.
+    Read more in the :ref:`User Guide `.
 
     .. versionadded:: 0.23
 
     Parameters
     ----------
     alpha : float, default=1
-        Constant that multiplies the penalty term and thus determines the
+        Constant that multiplies the L2 penalty term and determines the
         regularization strength. ``alpha = 0`` is equivalent to unpenalized
         GLMs. In this case, the design matrix `X` must have full column rank
         (no collinearities).
+        Values of `alpha` must be in the range `[0.0, inf)`.
 
     fit_intercept : bool, default=True
         Specifies if a constant (a.k.a. bias or intercept) should be
-        added to the linear predictor (X @ coef + intercept).
+        added to the linear predictor (`X @ coef + intercept`).
+
+    solver : {'lbfgs', 'newton-cholesky'}, default='lbfgs'
+        Algorithm to use in the optimization problem:
+
+        'lbfgs'
+            Calls scipy's L-BFGS-B optimizer.
+
+        'newton-cholesky'
+            Uses Newton-Raphson steps (in arbitrary precision arithmetic equivalent to
+            iterated reweighted least squares) with an inner Cholesky based solver.
+            This solver is a good choice for `n_samples` >> `n_features`, especially
+            with one-hot encoded categorical features with rare categories. Be aware
+            that the memory usage of this solver has a quadratic dependency on
+            `n_features` because it explicitly computes the Hessian matrix.
+
+            .. versionadded:: 1.2
 
     max_iter : int, default=100
         The maximal number of iterations for the solver.
+        Values must be in the range `[1, inf)`.
 
     tol : float, default=1e-4
         Stopping criterion. For the lbfgs solver,
         the iteration will stop when ``max{|g_j|, j = 1, ..., d} <= tol``
         where ``g_j`` is the j-th component of the gradient (derivative) of
         the objective function.
+        Values must be in the range `(0.0, inf)`.
 
     warm_start : bool, default=False
         If set to ``True``, reuse the solution of the previous call to ``fit``
@@ -423,6 +522,7 @@ class PoissonRegressor(GeneralizedLinearRegressor):
 
     verbose : int, default=0
         For the lbfgs solver set verbose to any positive number for verbosity.
+        Values must be in the range `[0, inf)`.
 
     Attributes
     ----------
@@ -433,11 +533,26 @@ class PoissonRegressor(GeneralizedLinearRegressor):
     intercept_ : float
         Intercept (a.k.a. bias) added to linear predictor.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     n_iter_ : int
         Actual number of iterations used in the solver.
 
+    See Also
+    --------
+    TweedieRegressor : Generalized Linear Model with a Tweedie distribution.
+
     Examples
-    ----------
+    --------
     >>> from sklearn import linear_model
     >>> clf = linear_model.PoissonRegressor()
     >>> X = [[1, 2], [2, 3], [3, 4], [4, 3]]
@@ -445,81 +560,129 @@ class PoissonRegressor(GeneralizedLinearRegressor):
     >>> clf.fit(X, y)
     PoissonRegressor()
     >>> clf.score(X, y)
-    0.990...
+    np.float64(0.990)
     >>> clf.coef_
-    array([0.121..., 0.158...])
+    array([0.121, 0.158])
     >>> clf.intercept_
-    2.088...
+    np.float64(2.088)
     >>> clf.predict([[1, 1], [3, 4]])
-    array([10.676..., 21.875...])
+    array([10.676, 21.875])
     """
-    def __init__(self, *, alpha=1.0, fit_intercept=True, max_iter=100,
-                 tol=1e-4, warm_start=False, verbose=0):
-
-        super().__init__(alpha=alpha, fit_intercept=fit_intercept,
-                         family="poisson", link='log', max_iter=max_iter,
-                         tol=tol, warm_start=warm_start, verbose=verbose)
-
-    @property
-    def family(self):
-        # Make this attribute read-only to avoid mis-uses e.g. in GridSearch.
-        return "poisson"
 
-    @family.setter
-    def family(self, value):
-        if value != "poisson":
-            raise ValueError("PoissonRegressor.family must be 'poisson'!")
-
-
-class GammaRegressor(GeneralizedLinearRegressor):
+    _parameter_constraints: dict = {
+        **_GeneralizedLinearRegressor._parameter_constraints
+    }
+
+    def __init__(
+        self,
+        *,
+        alpha=1.0,
+        fit_intercept=True,
+        solver="lbfgs",
+        max_iter=100,
+        tol=1e-4,
+        warm_start=False,
+        verbose=0,
+    ):
+        super().__init__(
+            alpha=alpha,
+            fit_intercept=fit_intercept,
+            solver=solver,
+            max_iter=max_iter,
+            tol=tol,
+            warm_start=warm_start,
+            verbose=verbose,
+        )
+
+    def _get_loss(self):
+        return HalfPoissonLoss()
+
+
+class GammaRegressor(_GeneralizedLinearRegressor):
     """Generalized Linear Model with a Gamma distribution.
 
     This regressor uses the 'log' link function.
 
-    Read more in the :ref:`User Guide `.
+    Read more in the :ref:`User Guide `.
 
     .. versionadded:: 0.23
 
     Parameters
     ----------
     alpha : float, default=1
-        Constant that multiplies the penalty term and thus determines the
+        Constant that multiplies the L2 penalty term and determines the
         regularization strength. ``alpha = 0`` is equivalent to unpenalized
         GLMs. In this case, the design matrix `X` must have full column rank
         (no collinearities).
+        Values of `alpha` must be in the range `[0.0, inf)`.
 
     fit_intercept : bool, default=True
         Specifies if a constant (a.k.a. bias or intercept) should be
-        added to the linear predictor (X @ coef + intercept).
+        added to the linear predictor `X @ coef_ + intercept_`.
+
+    solver : {'lbfgs', 'newton-cholesky'}, default='lbfgs'
+        Algorithm to use in the optimization problem:
+
+        'lbfgs'
+            Calls scipy's L-BFGS-B optimizer.
+
+        'newton-cholesky'
+            Uses Newton-Raphson steps (in arbitrary precision arithmetic equivalent to
+            iterated reweighted least squares) with an inner Cholesky based solver.
+            This solver is a good choice for `n_samples` >> `n_features`, especially
+            with one-hot encoded categorical features with rare categories. Be aware
+            that the memory usage of this solver has a quadratic dependency on
+            `n_features` because it explicitly computes the Hessian matrix.
+
+            .. versionadded:: 1.2
 
     max_iter : int, default=100
         The maximal number of iterations for the solver.
+        Values must be in the range `[1, inf)`.
 
     tol : float, default=1e-4
         Stopping criterion. For the lbfgs solver,
         the iteration will stop when ``max{|g_j|, j = 1, ..., d} <= tol``
         where ``g_j`` is the j-th component of the gradient (derivative) of
         the objective function.
+        Values must be in the range `(0.0, inf)`.
 
     warm_start : bool, default=False
         If set to ``True``, reuse the solution of the previous call to ``fit``
-        as initialization for ``coef_`` and ``intercept_`` .
+        as initialization for `coef_` and `intercept_`.
 
     verbose : int, default=0
         For the lbfgs solver set verbose to any positive number for verbosity.
+        Values must be in the range `[0, inf)`.
 
     Attributes
     ----------
     coef_ : array of shape (n_features,)
-        Estimated coefficients for the linear predictor (`X * coef_ +
+        Estimated coefficients for the linear predictor (`X @ coef_ +
         intercept_`) in the GLM.
 
     intercept_ : float
         Intercept (a.k.a. bias) added to linear predictor.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
     n_iter_ : int
         Actual number of iterations used in the solver.
 
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    PoissonRegressor : Generalized Linear Model with a Poisson distribution.
+    TweedieRegressor : Generalized Linear Model with a Tweedie distribution.
+
     Examples
     --------
     >>> from sklearn import linear_model
@@ -529,39 +692,51 @@ class GammaRegressor(GeneralizedLinearRegressor):
     >>> clf.fit(X, y)
     GammaRegressor()
     >>> clf.score(X, y)
-    0.773...
+    np.float64(0.773)
     >>> clf.coef_
-    array([0.072..., 0.066...])
+    array([0.073, 0.067])
     >>> clf.intercept_
-    2.896...
+    np.float64(2.896)
     >>> clf.predict([[1, 0], [2, 8]])
-    array([19.483..., 35.795...])
+    array([19.483, 35.795])
     """
-    def __init__(self, *, alpha=1.0, fit_intercept=True, max_iter=100,
-                 tol=1e-4, warm_start=False, verbose=0):
-
-        super().__init__(alpha=alpha, fit_intercept=fit_intercept,
-                         family="gamma", link='log', max_iter=max_iter,
-                         tol=tol, warm_start=warm_start, verbose=verbose)
 
-    @property
-    def family(self):
-        # Make this attribute read-only to avoid mis-uses e.g. in GridSearch.
-        return "gamma"
-
-    @family.setter
-    def family(self, value):
-        if value != "gamma":
-            raise ValueError("GammaRegressor.family must be 'gamma'!")
-
-
-class TweedieRegressor(GeneralizedLinearRegressor):
+    _parameter_constraints: dict = {
+        **_GeneralizedLinearRegressor._parameter_constraints
+    }
+
+    def __init__(
+        self,
+        *,
+        alpha=1.0,
+        fit_intercept=True,
+        solver="lbfgs",
+        max_iter=100,
+        tol=1e-4,
+        warm_start=False,
+        verbose=0,
+    ):
+        super().__init__(
+            alpha=alpha,
+            fit_intercept=fit_intercept,
+            solver=solver,
+            max_iter=max_iter,
+            tol=tol,
+            warm_start=warm_start,
+            verbose=verbose,
+        )
+
+    def _get_loss(self):
+        return HalfGammaLoss()
+
+
+class TweedieRegressor(_GeneralizedLinearRegressor):
     """Generalized Linear Model with a Tweedie distribution.
 
     This estimator can be used to model different GLMs depending on the
     ``power`` parameter, which determines the underlying distribution.
 
-    Read more in the :ref:`User Guide `.
+    Read more in the :ref:`User Guide `.
 
     .. versionadded:: 0.23
 
@@ -588,31 +763,51 @@ class TweedieRegressor(GeneralizedLinearRegressor):
             For ``0 < power < 1``, no distribution exists.
 
     alpha : float, default=1
-        Constant that multiplies the penalty term and thus determines the
+        Constant that multiplies the L2 penalty term and determines the
         regularization strength. ``alpha = 0`` is equivalent to unpenalized
         GLMs. In this case, the design matrix `X` must have full column rank
         (no collinearities).
+        Values of `alpha` must be in the range `[0.0, inf)`.
 
     fit_intercept : bool, default=True
         Specifies if a constant (a.k.a. bias or intercept) should be
-        added to the linear predictor (X @ coef + intercept).
+        added to the linear predictor (`X @ coef + intercept`).
 
     link : {'auto', 'identity', 'log'}, default='auto'
         The link function of the GLM, i.e. mapping from linear predictor
         `X @ coeff + intercept` to prediction `y_pred`. Option 'auto' sets
-        the link depending on the chosen family as follows:
+        the link depending on the chosen `power` parameter as follows:
+
+        - 'identity' for ``power <= 0``, e.g. for the Normal distribution
+        - 'log' for ``power > 0``, e.g. for Poisson, Gamma and Inverse Gaussian
+          distributions
+
+    solver : {'lbfgs', 'newton-cholesky'}, default='lbfgs'
+        Algorithm to use in the optimization problem:
+
+        'lbfgs'
+            Calls scipy's L-BFGS-B optimizer.
 
-        - 'identity' for Normal distribution
-        - 'log' for Poisson,  Gamma and Inverse Gaussian distributions
+        'newton-cholesky'
+            Uses Newton-Raphson steps (in arbitrary precision arithmetic equivalent to
+            iterated reweighted least squares) with an inner Cholesky based solver.
+            This solver is a good choice for `n_samples` >> `n_features`, especially
+            with one-hot encoded categorical features with rare categories. Be aware
+            that the memory usage of this solver has a quadratic dependency on
+            `n_features` because it explicitly computes the Hessian matrix.
+
+            .. versionadded:: 1.2
 
     max_iter : int, default=100
         The maximal number of iterations for the solver.
+        Values must be in the range `[1, inf)`.
 
     tol : float, default=1e-4
         Stopping criterion. For the lbfgs solver,
         the iteration will stop when ``max{|g_j|, j = 1, ..., d} <= tol``
         where ``g_j`` is the j-th component of the gradient (derivative) of
         the objective function.
+        Values must be in the range `(0.0, inf)`.
 
     warm_start : bool, default=False
         If set to ``True``, reuse the solution of the previous call to ``fit``
@@ -620,6 +815,7 @@ class TweedieRegressor(GeneralizedLinearRegressor):
 
     verbose : int, default=0
         For the lbfgs solver set verbose to any positive number for verbosity.
+        Values must be in the range `[0, inf)`.
 
     Attributes
     ----------
@@ -633,8 +829,24 @@ class TweedieRegressor(GeneralizedLinearRegressor):
     n_iter_ : int
         Actual number of iterations used in the solver.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    PoissonRegressor : Generalized Linear Model with a Poisson distribution.
+    GammaRegressor : Generalized Linear Model with a Gamma distribution.
+
     Examples
-    ----------
+    --------
     >>> from sklearn import linear_model
     >>> clf = linear_model.TweedieRegressor()
     >>> X = [[1, 2], [2, 3], [3, 4], [4, 3]]
@@ -642,36 +854,57 @@ class TweedieRegressor(GeneralizedLinearRegressor):
     >>> clf.fit(X, y)
     TweedieRegressor()
     >>> clf.score(X, y)
-    0.839...
+    np.float64(0.839)
     >>> clf.coef_
-    array([0.599..., 0.299...])
+    array([0.599, 0.299])
     >>> clf.intercept_
-    1.600...
+    np.float64(1.600)
     >>> clf.predict([[1, 1], [3, 4]])
-    array([2.500..., 4.599...])
+    array([2.500, 4.599])
     """
-    def __init__(self, *, power=0.0, alpha=1.0, fit_intercept=True,
-                 link='auto', max_iter=100, tol=1e-4,
-                 warm_start=False, verbose=0):
-
-        super().__init__(alpha=alpha, fit_intercept=fit_intercept,
-                         family=TweedieDistribution(power=power), link=link,
-                         max_iter=max_iter, tol=tol,
-                         warm_start=warm_start, verbose=verbose)
-
-    @property
-    def family(self):
-        # We use a property with a setter to make sure that the family is
-        # always a Tweedie distribution, and that self.power and
-        # self.family.power are identical by construction.
-        dist = TweedieDistribution(power=self.power)
-        # TODO: make the returned object immutable
-        return dist
-
-    @family.setter
-    def family(self, value):
-        if isinstance(value, TweedieDistribution):
-            self.power = value.power
-        else:
-            raise TypeError("TweedieRegressor.family must be of type "
-                            "TweedieDistribution!")
+
+    _parameter_constraints: dict = {
+        **_GeneralizedLinearRegressor._parameter_constraints,
+        "power": [Interval(Real, None, None, closed="neither")],
+        "link": [StrOptions({"auto", "identity", "log"})],
+    }
+
+    def __init__(
+        self,
+        *,
+        power=0.0,
+        alpha=1.0,
+        fit_intercept=True,
+        link="auto",
+        solver="lbfgs",
+        max_iter=100,
+        tol=1e-4,
+        warm_start=False,
+        verbose=0,
+    ):
+        super().__init__(
+            alpha=alpha,
+            fit_intercept=fit_intercept,
+            solver=solver,
+            max_iter=max_iter,
+            tol=tol,
+            warm_start=warm_start,
+            verbose=verbose,
+        )
+        self.link = link
+        self.power = power
+
+    def _get_loss(self):
+        if self.link == "auto":
+            if self.power <= 0:
+                # identity link
+                return HalfTweedieLossIdentity(power=self.power)
+            else:
+                # log link
+                return HalfTweedieLoss(power=self.power)
+
+        if self.link == "log":
+            return HalfTweedieLoss(power=self.power)
+
+        if self.link == "identity":
+            return HalfTweedieLossIdentity(power=self.power)
diff --git a/sklearn/linear_model/_glm/link.py b/sklearn/linear_model/_glm/link.py
deleted file mode 100644
index 878d8e835bc42..0000000000000
--- a/sklearn/linear_model/_glm/link.py
+++ /dev/null
@@ -1,110 +0,0 @@
-"""
-Link functions used in GLM
-"""
-
-# Author: Christian Lorentzen 
-# License: BSD 3 clause
-
-from abc import ABCMeta, abstractmethod
-
-import numpy as np
-from scipy.special import expit, logit
-
-
-class BaseLink(metaclass=ABCMeta):
-    """Abstract base class for Link functions."""
-
-    @abstractmethod
-    def __call__(self, y_pred):
-        """Compute the link function g(y_pred).
-
-        The link function links the mean y_pred=E[Y] to the so called linear
-        predictor (X*w), i.e. g(y_pred) = linear predictor.
-
-        Parameters
-        ----------
-        y_pred : array of shape (n_samples,)
-            Usually the (predicted) mean.
-        """
-
-    @abstractmethod
-    def derivative(self, y_pred):
-        """Compute the derivative of the link g'(y_pred).
-
-        Parameters
-        ----------
-        y_pred : array of shape (n_samples,)
-            Usually the (predicted) mean.
-        """
-
-    @abstractmethod
-    def inverse(self, lin_pred):
-        """Compute the inverse link function h(lin_pred).
-
-        Gives the inverse relationship between linear predictor and the mean
-        y_pred=E[Y], i.e. h(linear predictor) = y_pred.
-
-        Parameters
-        ----------
-        lin_pred : array of shape (n_samples,)
-            Usually the (fitted) linear predictor.
-        """
-
-    @abstractmethod
-    def inverse_derivative(self, lin_pred):
-        """Compute the derivative of the inverse link function h'(lin_pred).
-
-        Parameters
-        ----------
-        lin_pred : array of shape (n_samples,)
-            Usually the (fitted) linear predictor.
-        """
-
-
-class IdentityLink(BaseLink):
-    """The identity link function g(x)=x."""
-
-    def __call__(self, y_pred):
-        return y_pred
-
-    def derivative(self, y_pred):
-        return np.ones_like(y_pred)
-
-    def inverse(self, lin_pred):
-        return lin_pred
-
-    def inverse_derivative(self, lin_pred):
-        return np.ones_like(lin_pred)
-
-
-class LogLink(BaseLink):
-    """The log link function g(x)=log(x)."""
-
-    def __call__(self, y_pred):
-        return np.log(y_pred)
-
-    def derivative(self, y_pred):
-        return 1 / y_pred
-
-    def inverse(self, lin_pred):
-        return np.exp(lin_pred)
-
-    def inverse_derivative(self, lin_pred):
-        return np.exp(lin_pred)
-
-
-class LogitLink(BaseLink):
-    """The logit link function g(x)=logit(x)."""
-
-    def __call__(self, y_pred):
-        return logit(y_pred)
-
-    def derivative(self, y_pred):
-        return 1 / (y_pred * (1 - y_pred))
-
-    def inverse(self, lin_pred):
-        return expit(lin_pred)
-
-    def inverse_derivative(self, lin_pred):
-        ep = expit(lin_pred)
-        return ep * (1 - ep)
diff --git a/sklearn/linear_model/_glm/tests/__init__.py b/sklearn/linear_model/_glm/tests/__init__.py
index 588cf7e93eef0..67dd18fb94b59 100644
--- a/sklearn/linear_model/_glm/tests/__init__.py
+++ b/sklearn/linear_model/_glm/tests/__init__.py
@@ -1 +1,2 @@
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
diff --git a/sklearn/linear_model/_glm/tests/test_glm.py b/sklearn/linear_model/_glm/tests/test_glm.py
index d6fc4e14b12fa..fbcc4d61a8e1c 100644
--- a/sklearn/linear_model/_glm/tests/test_glm.py
+++ b/sklearn/linear_model/_glm/tests/test_glm.py
@@ -1,181 +1,639 @@
-# Authors: Christian Lorentzen 
-#
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+import itertools
+import warnings
+from functools import partial
 
 import numpy as np
-from numpy.testing import assert_allclose
 import pytest
-import warnings
+import scipy
+from numpy.testing import assert_allclose
+from scipy import linalg
+from scipy.optimize import minimize, root
 
-from sklearn.datasets import make_regression
-from sklearn.linear_model._glm import GeneralizedLinearRegressor
+from sklearn._loss import HalfBinomialLoss, HalfPoissonLoss, HalfTweedieLoss
+from sklearn._loss.link import IdentityLink, LogLink
+from sklearn.base import clone
+from sklearn.datasets import make_low_rank_matrix, make_regression
+from sklearn.exceptions import ConvergenceWarning
 from sklearn.linear_model import (
-    TweedieRegressor,
+    GammaRegressor,
     PoissonRegressor,
-    GammaRegressor
-)
-from sklearn.linear_model._glm.link import (
-    IdentityLink,
-    LogLink,
-)
-from sklearn._loss.glm_distribution import (
-    TweedieDistribution,
-    NormalDistribution, PoissonDistribution,
-    GammaDistribution, InverseGaussianDistribution,
+    Ridge,
+    TweedieRegressor,
 )
-from sklearn.linear_model import Ridge
-from sklearn.exceptions import ConvergenceWarning
+from sklearn.linear_model._glm import _GeneralizedLinearRegressor
+from sklearn.linear_model._glm._newton_solver import NewtonCholeskySolver
+from sklearn.linear_model._linear_loss import LinearModelLoss
+from sklearn.metrics import d2_tweedie_score, mean_poisson_deviance
 from sklearn.model_selection import train_test_split
 
+SOLVERS = ["lbfgs", "newton-cholesky"]
+
+
+class BinomialRegressor(_GeneralizedLinearRegressor):
+    def _get_loss(self):
+        return HalfBinomialLoss()
+
+
+def _special_minimize(fun, grad, x, tol_NM, tol):
+    # Find good starting point by Nelder-Mead
+    res_NM = minimize(
+        fun, x, method="Nelder-Mead", options={"xatol": tol_NM, "fatol": tol_NM}
+    )
+    # Now refine via root finding on the gradient of the function, which is
+    # more precise than minimizing the function itself.
+    res = root(
+        grad,
+        res_NM.x,
+        method="lm",
+        options={"ftol": tol, "xtol": tol, "gtol": tol},
+    )
+    return res.x
+
 
 @pytest.fixture(scope="module")
 def regression_data():
-    X, y = make_regression(n_samples=107,
-                           n_features=10,
-                           n_informative=80, noise=0.5,
-                           random_state=2)
+    X, y = make_regression(
+        n_samples=107, n_features=10, n_informative=80, noise=0.5, random_state=2
+    )
     return X, y
 
 
-def test_sample_weights_validation():
-    """Test the raised errors in the validation of sample_weight."""
-    # scalar value but not positive
-    X = [[1]]
-    y = [1]
-    weights = 0
-    glm = GeneralizedLinearRegressor()
+@pytest.fixture(
+    params=itertools.product(
+        ["long", "wide"],
+        [
+            BinomialRegressor(),
+            PoissonRegressor(),
+            GammaRegressor(),
+            # TweedieRegressor(power=3.0),  # too difficult
+            # TweedieRegressor(power=0, link="log"),  # too difficult
+            TweedieRegressor(power=1.5),
+        ],
+    ),
+    ids=lambda param: f"{param[0]}-{param[1]}",
+)
+def glm_dataset(global_random_seed, request):
+    """Dataset with GLM solutions, well conditioned X.
+
+    This is inspired by ols_ridge_dataset in test_ridge.py.
+
+    The construction is based on the SVD decomposition of X = U S V'.
+
+    Parameters
+    ----------
+    type : {"long", "wide"}
+        If "long", then n_samples > n_features.
+        If "wide", then n_features > n_samples.
+    model : a GLM model
+
+    For "wide", we return the minimum norm solution:
+
+        min ||w||_2 subject to w = argmin deviance(X, y, w)
+
+    Note that the deviance is always minimized if y = inverse_link(X w) is possible to
+    achieve, which it is in the wide data case. Therefore, we can construct the
+    solution with minimum norm like (wide) OLS:
+
+        min ||w||_2 subject to link(y) = raw_prediction = X w
+
+    Returns
+    -------
+    model : GLM model
+    X : ndarray
+        Last column of 1, i.e. intercept.
+    y : ndarray
+    coef_unpenalized : ndarray
+        Minimum norm solutions, i.e. min sum(loss(w)) (with minimum ||w||_2 in
+        case of ambiguity)
+        Last coefficient is intercept.
+    coef_penalized : ndarray
+        GLM solution with alpha=l2_reg_strength=1, i.e.
+        min 1/n * sum(loss) + ||w[:-1]||_2^2.
+        Last coefficient is intercept.
+    l2_reg_strength : float
+        Always equal 1.
+    """
+    data_type, model = request.param
+    # Make larger dim more than double as big as the smaller one.
+    # This helps when constructing singular matrices like (X, X).
+    if data_type == "long":
+        n_samples, n_features = 12, 4
+    else:
+        n_samples, n_features = 4, 12
+    k = min(n_samples, n_features)
+    rng = np.random.RandomState(global_random_seed)
+    X = make_low_rank_matrix(
+        n_samples=n_samples,
+        n_features=n_features,
+        effective_rank=k,
+        tail_strength=0.1,
+        random_state=rng,
+    )
+    X[:, -1] = 1  # last columns acts as intercept
+    U, s, Vt = linalg.svd(X, full_matrices=False)
+    assert np.all(s > 1e-3)  # to be sure
+    assert np.max(s) / np.min(s) < 100  # condition number of X
+
+    if data_type == "long":
+        coef_unpenalized = rng.uniform(low=1, high=3, size=n_features)
+        coef_unpenalized *= rng.choice([-1, 1], size=n_features)
+        raw_prediction = X @ coef_unpenalized
+    else:
+        raw_prediction = rng.uniform(low=-3, high=3, size=n_samples)
+        # minimum norm solution min ||w||_2 such that raw_prediction = X w:
+        # w = X'(XX')^-1 raw_prediction = V s^-1 U' raw_prediction
+        coef_unpenalized = Vt.T @ np.diag(1 / s) @ U.T @ raw_prediction
+
+    linear_loss = LinearModelLoss(base_loss=model._get_loss(), fit_intercept=True)
+    sw = np.full(shape=n_samples, fill_value=1 / n_samples)
+    y = linear_loss.base_loss.link.inverse(raw_prediction)
+
+    # Add penalty l2_reg_strength * ||coef||_2^2 for l2_reg_strength=1 and solve with
+    # optimizer. Note that the problem is well conditioned such that we get accurate
+    # results.
+    l2_reg_strength = 1
+    fun = partial(
+        linear_loss.loss,
+        X=X[:, :-1],
+        y=y,
+        sample_weight=sw,
+        l2_reg_strength=l2_reg_strength,
+    )
+    grad = partial(
+        linear_loss.gradient,
+        X=X[:, :-1],
+        y=y,
+        sample_weight=sw,
+        l2_reg_strength=l2_reg_strength,
+    )
+    coef_penalized_with_intercept = _special_minimize(
+        fun, grad, coef_unpenalized, tol_NM=1e-6, tol=1e-14
+    )
 
-    # Positive weights are accepted
-    glm.fit(X, y, sample_weight=1)
+    linear_loss = LinearModelLoss(base_loss=model._get_loss(), fit_intercept=False)
+    fun = partial(
+        linear_loss.loss,
+        X=X[:, :-1],
+        y=y,
+        sample_weight=sw,
+        l2_reg_strength=l2_reg_strength,
+    )
+    grad = partial(
+        linear_loss.gradient,
+        X=X[:, :-1],
+        y=y,
+        sample_weight=sw,
+        l2_reg_strength=l2_reg_strength,
+    )
+    coef_penalized_without_intercept = _special_minimize(
+        fun, grad, coef_unpenalized[:-1], tol_NM=1e-6, tol=1e-14
+    )
 
-    # 2d array
-    weights = [[0]]
-    with pytest.raises(ValueError, match="must be 1D array or scalar"):
-        glm.fit(X, y, weights)
+    # To be sure
+    assert np.linalg.norm(coef_penalized_with_intercept) < np.linalg.norm(
+        coef_unpenalized
+    )
 
-    # 1d but wrong length
-    weights = [1, 0]
-    msg = r"sample_weight.shape == \(2,\), expected \(1,\)!"
-    with pytest.raises(ValueError, match=msg):
-        glm.fit(X, y, weights)
+    return (
+        model,
+        X,
+        y,
+        coef_unpenalized,
+        coef_penalized_with_intercept,
+        coef_penalized_without_intercept,
+        l2_reg_strength,
+    )
 
 
-@pytest.mark.parametrize('name, instance',
-                         [('normal', NormalDistribution()),
-                          ('poisson', PoissonDistribution()),
-                          ('gamma', GammaDistribution()),
-                          ('inverse-gaussian', InverseGaussianDistribution())])
-def test_glm_family_argument(name, instance):
-    """Test GLM family argument set as string."""
-    y = np.array([0.1, 0.5])  # in range of all distributions
-    X = np.array([[1], [2]])
-    glm = GeneralizedLinearRegressor(family=name, alpha=0).fit(X, y)
-    assert isinstance(glm._family_instance, instance.__class__)
+@pytest.mark.parametrize("solver", SOLVERS)
+@pytest.mark.parametrize("fit_intercept", [False, True])
+def test_glm_regression(solver, fit_intercept, glm_dataset):
+    """Test that GLM converges for all solvers to correct solution.
 
-    glm = GeneralizedLinearRegressor(family='not a family')
-    with pytest.raises(ValueError, match="family must be"):
-        glm.fit(X, y)
+    We work with a simple constructed data set with known solution.
+    """
+    model, X, y, _, coef_with_intercept, coef_without_intercept, alpha = glm_dataset
+    params = dict(
+        alpha=alpha,
+        fit_intercept=fit_intercept,
+        solver=solver,
+        tol=1e-12,
+        max_iter=1000,
+    )
+
+    model = clone(model).set_params(**params)
+    X = X[:, :-1]  # remove intercept
+    if fit_intercept:
+        coef = coef_with_intercept
+        intercept = coef[-1]
+        coef = coef[:-1]
+    else:
+        coef = coef_without_intercept
+        intercept = 0
 
+    model.fit(X, y)
 
-@pytest.mark.parametrize('name, instance',
-                         [('identity', IdentityLink()),
-                          ('log', LogLink())])
-def test_glm_link_argument(name, instance):
-    """Test GLM link argument set as string."""
-    y = np.array([0.1, 0.5])  # in range of all distributions
-    X = np.array([[1], [2]])
-    glm = GeneralizedLinearRegressor(family='normal', link=name).fit(X, y)
-    assert isinstance(glm._link_instance, instance.__class__)
+    rtol = 5e-5 if solver == "lbfgs" else 1e-9
+    assert model.intercept_ == pytest.approx(intercept, rel=rtol)
+    assert_allclose(model.coef_, coef, rtol=rtol)
 
-    glm = GeneralizedLinearRegressor(family='normal', link='not a link')
-    with pytest.raises(ValueError, match="link must be"):
-        glm.fit(X, y)
+    # Same with sample_weight.
+    model = (
+        clone(model).set_params(**params).fit(X, y, sample_weight=np.ones(X.shape[0]))
+    )
+    assert model.intercept_ == pytest.approx(intercept, rel=rtol)
+    assert_allclose(model.coef_, coef, rtol=rtol)
 
 
-@pytest.mark.parametrize('family, expected_link_class', [
-    ('normal', IdentityLink),
-    ('poisson', LogLink),
-    ('gamma', LogLink),
-    ('inverse-gaussian', LogLink),
-])
-def test_glm_link_auto(family, expected_link_class):
-    # Make sure link='auto' delivers the expected link function
-    y = np.array([0.1, 0.5])  # in range of all distributions
-    X = np.array([[1], [2]])
-    glm = GeneralizedLinearRegressor(family=family, link='auto').fit(X, y)
-    assert isinstance(glm._link_instance, expected_link_class)
+@pytest.mark.parametrize("solver", SOLVERS)
+@pytest.mark.parametrize("fit_intercept", [True, False])
+def test_glm_regression_hstacked_X(solver, fit_intercept, glm_dataset):
+    """Test that GLM converges for all solvers to correct solution on hstacked data.
 
+    We work with a simple constructed data set with known solution.
+    Fit on [X] with alpha is the same as fit on [X, X]/2 with alpha/2.
+    For long X, [X, X] is still a long but singular matrix.
+    """
+    model, X, y, _, coef_with_intercept, coef_without_intercept, alpha = glm_dataset
+    n_samples, n_features = X.shape
+    params = dict(
+        alpha=alpha / 2,
+        fit_intercept=fit_intercept,
+        solver=solver,
+        tol=1e-12,
+        max_iter=1000,
+    )
 
-@pytest.mark.parametrize('alpha', ['not a number', -4.2])
-def test_glm_alpha_argument(alpha):
-    """Test GLM for invalid alpha argument."""
-    y = np.array([1, 2])
-    X = np.array([[1], [2]])
-    glm = GeneralizedLinearRegressor(family='normal', alpha=alpha)
-    with pytest.raises(ValueError,
-                       match="Penalty term must be a non-negative"):
-        glm.fit(X, y)
+    model = clone(model).set_params(**params)
+    X = X[:, :-1]  # remove intercept
+    X = 0.5 * np.concatenate((X, X), axis=1)
+    assert np.linalg.matrix_rank(X) <= min(n_samples, n_features - 1)
+    if fit_intercept:
+        coef = coef_with_intercept
+        intercept = coef[-1]
+        coef = coef[:-1]
+    else:
+        coef = coef_without_intercept
+        intercept = 0
 
+    with warnings.catch_warnings():
+        # XXX: Investigate if the ConvergenceWarning that can appear in some
+        # cases should be considered a bug or not. In the mean time we don't
+        # fail when the assertions below pass irrespective of the presence of
+        # the warning.
+        warnings.simplefilter("ignore", ConvergenceWarning)
+        model.fit(X, y)
+
+    rtol = 2e-4 if solver == "lbfgs" else 5e-9
+    assert model.intercept_ == pytest.approx(intercept, rel=rtol)
+    assert_allclose(model.coef_, np.r_[coef, coef], rtol=rtol)
+
+
+@pytest.mark.parametrize("solver", SOLVERS)
+@pytest.mark.parametrize("fit_intercept", [True, False])
+def test_glm_regression_vstacked_X(solver, fit_intercept, glm_dataset):
+    """Test that GLM converges for all solvers to correct solution on vstacked data.
+
+    We work with a simple constructed data set with known solution.
+    Fit on [X] with alpha is the same as fit on [X], [y]
+                                                [X], [y] with 1 * alpha.
+    It is the same alpha as the average loss stays the same.
+    For wide X, [X', X'] is a singular matrix.
+    """
+    model, X, y, _, coef_with_intercept, coef_without_intercept, alpha = glm_dataset
+    n_samples, n_features = X.shape
+    params = dict(
+        alpha=alpha,
+        fit_intercept=fit_intercept,
+        solver=solver,
+        tol=1e-12,
+        max_iter=1000,
+    )
 
-@pytest.mark.parametrize('fit_intercept', ['not bool', 1, 0, [True]])
-def test_glm_fit_intercept_argument(fit_intercept):
-    """Test GLM for invalid fit_intercept argument."""
-    y = np.array([1, 2])
-    X = np.array([[1], [1]])
-    glm = GeneralizedLinearRegressor(fit_intercept=fit_intercept)
-    with pytest.raises(ValueError, match="fit_intercept must be bool"):
-        glm.fit(X, y)
+    model = clone(model).set_params(**params)
+    X = X[:, :-1]  # remove intercept
+    X = np.concatenate((X, X), axis=0)
+    assert np.linalg.matrix_rank(X) <= min(n_samples, n_features)
+    y = np.r_[y, y]
+    if fit_intercept:
+        coef = coef_with_intercept
+        intercept = coef[-1]
+        coef = coef[:-1]
+    else:
+        coef = coef_without_intercept
+        intercept = 0
+    model.fit(X, y)
 
+    rtol = 3e-5 if solver == "lbfgs" else 5e-9
+    assert model.intercept_ == pytest.approx(intercept, rel=rtol)
+    assert_allclose(model.coef_, coef, rtol=rtol)
 
-@pytest.mark.parametrize('solver',
-                         ['not a solver', 1, [1]])
-def test_glm_solver_argument(solver):
-    """Test GLM for invalid solver argument."""
-    y = np.array([1, 2])
-    X = np.array([[1], [2]])
-    glm = GeneralizedLinearRegressor(solver=solver)
-    with pytest.raises(ValueError):
-        glm.fit(X, y)
 
+@pytest.mark.parametrize("solver", SOLVERS)
+@pytest.mark.parametrize("fit_intercept", [True, False])
+def test_glm_regression_unpenalized(solver, fit_intercept, glm_dataset):
+    """Test that unpenalized GLM converges for all solvers to correct solution.
 
-@pytest.mark.parametrize('max_iter', ['not a number', 0, -1, 5.5, [1]])
-def test_glm_max_iter_argument(max_iter):
-    """Test GLM for invalid max_iter argument."""
-    y = np.array([1, 2])
-    X = np.array([[1], [2]])
-    glm = GeneralizedLinearRegressor(max_iter=max_iter)
-    with pytest.raises(ValueError, match="must be a positive integer"):
-        glm.fit(X, y)
+    We work with a simple constructed data set with known solution.
+    Note: This checks the minimum norm solution for wide X, i.e.
+    n_samples < n_features:
+        min ||w||_2 subject to w = argmin deviance(X, y, w)
+    """
+    model, X, y, coef, _, _, _ = glm_dataset
+    n_samples, n_features = X.shape
+    alpha = 0  # unpenalized
+    params = dict(
+        alpha=alpha,
+        fit_intercept=fit_intercept,
+        solver=solver,
+        tol=1e-12,
+        max_iter=1000,
+    )
 
+    model = clone(model).set_params(**params)
+    if fit_intercept:
+        X = X[:, :-1]  # remove intercept
+        intercept = coef[-1]
+        coef = coef[:-1]
+    else:
+        intercept = 0
 
-@pytest.mark.parametrize('tol', ['not a number', 0, -1.0, [1e-3]])
-def test_glm_tol_argument(tol):
-    """Test GLM for invalid tol argument."""
-    y = np.array([1, 2])
-    X = np.array([[1], [2]])
-    glm = GeneralizedLinearRegressor(tol=tol)
-    with pytest.raises(ValueError, match="stopping criteria must be positive"):
-        glm.fit(X, y)
+    with warnings.catch_warnings():
+        if solver.startswith("newton") and n_samples < n_features:
+            # The newton solvers should warn and automatically fallback to LBFGS
+            # in this case. The model should still converge.
+            warnings.filterwarnings("ignore", category=scipy.linalg.LinAlgWarning)
+        # XXX: Investigate if the ConvergenceWarning that can appear in some
+        # cases should be considered a bug or not. In the mean time we don't
+        # fail when the assertions below pass irrespective of the presence of
+        # the warning.
+        warnings.filterwarnings("ignore", category=ConvergenceWarning)
+        model.fit(X, y)
+
+    # FIXME: `assert_allclose(model.coef_, coef)` should work for all cases but fails
+    # for the wide/fat case with n_features > n_samples. Most current GLM solvers do
+    # NOT return the minimum norm solution with fit_intercept=True.
+    if n_samples > n_features:
+        rtol = 5e-5 if solver == "lbfgs" else 1e-7
+        assert model.intercept_ == pytest.approx(intercept)
+        assert_allclose(model.coef_, coef, rtol=rtol)
+    else:
+        # As it is an underdetermined problem, prediction = y. The following shows that
+        # we get a solution, i.e. a (non-unique) minimum of the objective function ...
+        rtol = 5e-5
+        if solver == "newton-cholesky":
+            rtol = 5e-4
+        assert_allclose(model.predict(X), y, rtol=rtol)
+
+        norm_solution = np.linalg.norm(np.r_[intercept, coef])
+        norm_model = np.linalg.norm(np.r_[model.intercept_, model.coef_])
+        if solver == "newton-cholesky":
+            # XXX: This solver shows random behaviour. Sometimes it finds solutions
+            # with norm_model <= norm_solution! So we check conditionally.
+            if norm_model < (1 + 1e-12) * norm_solution:
+                assert model.intercept_ == pytest.approx(intercept)
+                assert_allclose(model.coef_, coef, rtol=rtol)
+        elif solver == "lbfgs" and fit_intercept:
+            # But it is not the minimum norm solution. Otherwise the norms would be
+            # equal.
+            assert norm_model > (1 + 1e-12) * norm_solution
+
+            # See https://github.com/scikit-learn/scikit-learn/issues/23670.
+            # Note: Even adding a tiny penalty does not give the minimal norm solution.
+            # XXX: We could have naively expected LBFGS to find the minimal norm
+            # solution by adding a very small penalty. Even that fails for a reason we
+            # do not properly understand at this point.
+        else:
+            # When `fit_intercept=False`, LBFGS naturally converges to the minimum norm
+            # solution on this problem.
+            # XXX: Do we have any theoretical guarantees why this should be the case?
+            assert model.intercept_ == pytest.approx(intercept, rel=rtol)
+            assert_allclose(model.coef_, coef, rtol=rtol)
+
+
+@pytest.mark.parametrize("solver", SOLVERS)
+@pytest.mark.parametrize("fit_intercept", [True, False])
+def test_glm_regression_unpenalized_hstacked_X(solver, fit_intercept, glm_dataset):
+    """Test that unpenalized GLM converges for all solvers to correct solution.
+
+    We work with a simple constructed data set with known solution.
+    GLM fit on [X] is the same as fit on [X, X]/2.
+    For long X, [X, X] is a singular matrix and we check against the minimum norm
+    solution:
+        min ||w||_2 subject to w = argmin deviance(X, y, w)
+    """
+    model, X, y, coef, _, _, _ = glm_dataset
+    n_samples, n_features = X.shape
+    alpha = 0  # unpenalized
+    params = dict(
+        alpha=alpha,
+        fit_intercept=fit_intercept,
+        solver=solver,
+        tol=1e-12,
+        max_iter=1000,
+    )
+
+    model = clone(model).set_params(**params)
+    if fit_intercept:
+        intercept = coef[-1]
+        coef = coef[:-1]
+        if n_samples > n_features:
+            X = X[:, :-1]  # remove intercept
+            X = 0.5 * np.concatenate((X, X), axis=1)
+        else:
+            # To know the minimum norm solution, we keep one intercept column and do
+            # not divide by 2. Later on, we must take special care.
+            X = np.c_[X[:, :-1], X[:, :-1], X[:, -1]]
+    else:
+        intercept = 0
+        X = 0.5 * np.concatenate((X, X), axis=1)
+    assert np.linalg.matrix_rank(X) <= min(n_samples, n_features)
+
+    with warnings.catch_warnings():
+        if solver.startswith("newton"):
+            # The newton solvers should warn and automatically fallback to LBFGS
+            # in this case. The model should still converge.
+            warnings.filterwarnings("ignore", category=scipy.linalg.LinAlgWarning)
+        # XXX: Investigate if the ConvergenceWarning that can appear in some
+        # cases should be considered a bug or not. In the mean time we don't
+        # fail when the assertions below pass irrespective of the presence of
+        # the warning.
+        warnings.filterwarnings("ignore", category=ConvergenceWarning)
+        model.fit(X, y)
+
+    if fit_intercept and n_samples < n_features:
+        # Here we take special care.
+        model_intercept = 2 * model.intercept_
+        model_coef = 2 * model.coef_[:-1]  # exclude the other intercept term.
+        # For minimum norm solution, we would have
+        # assert model.intercept_ == pytest.approx(model.coef_[-1])
+    else:
+        model_intercept = model.intercept_
+        model_coef = model.coef_
+
+    if n_samples > n_features:
+        assert model_intercept == pytest.approx(intercept)
+        rtol = 1e-4
+        assert_allclose(model_coef, np.r_[coef, coef], rtol=rtol)
+    else:
+        # As it is an underdetermined problem, prediction = y. The following shows that
+        # we get a solution, i.e. a (non-unique) minimum of the objective function ...
+        rtol = 1e-6 if solver == "lbfgs" else 5e-6
+        assert_allclose(model.predict(X), y, rtol=rtol)
+        if (solver == "lbfgs" and fit_intercept) or solver == "newton-cholesky":
+            # Same as in test_glm_regression_unpenalized.
+            # But it is not the minimum norm solution. Otherwise the norms would be
+            # equal.
+            norm_solution = np.linalg.norm(
+                0.5 * np.r_[intercept, intercept, coef, coef]
+            )
+            norm_model = np.linalg.norm(np.r_[model.intercept_, model.coef_])
+            assert norm_model > (1 + 1e-12) * norm_solution
+            # For minimum norm solution, we would have
+            # assert model.intercept_ == pytest.approx(model.coef_[-1])
+        else:
+            assert model_intercept == pytest.approx(intercept, rel=5e-6)
+            assert_allclose(model_coef, np.r_[coef, coef], rtol=1e-4)
+
+
+@pytest.mark.parametrize("solver", SOLVERS)
+@pytest.mark.parametrize("fit_intercept", [True, False])
+def test_glm_regression_unpenalized_vstacked_X(solver, fit_intercept, glm_dataset):
+    """Test that unpenalized GLM converges for all solvers to correct solution.
+
+    We work with a simple constructed data set with known solution.
+    GLM fit on [X] is the same as fit on [X], [y]
+                                         [X], [y].
+    For wide X, [X', X'] is a singular matrix and we check against the minimum norm
+    solution:
+        min ||w||_2 subject to w = argmin deviance(X, y, w)
+    """
+    model, X, y, coef, _, _, _ = glm_dataset
+    n_samples, n_features = X.shape
+    alpha = 0  # unpenalized
+    params = dict(
+        alpha=alpha,
+        fit_intercept=fit_intercept,
+        solver=solver,
+        tol=1e-12,
+        max_iter=1000,
+    )
 
+    model = clone(model).set_params(**params)
+    if fit_intercept:
+        X = X[:, :-1]  # remove intercept
+        intercept = coef[-1]
+        coef = coef[:-1]
+    else:
+        intercept = 0
+    X = np.concatenate((X, X), axis=0)
+    assert np.linalg.matrix_rank(X) <= min(n_samples, n_features)
+    y = np.r_[y, y]
 
-@pytest.mark.parametrize('warm_start', ['not bool', 1, 0, [True]])
-def test_glm_warm_start_argument(warm_start):
-    """Test GLM for invalid warm_start argument."""
-    y = np.array([1, 2])
+    with warnings.catch_warnings():
+        if solver.startswith("newton") and n_samples < n_features:
+            # The newton solvers should warn and automatically fallback to LBFGS
+            # in this case. The model should still converge.
+            warnings.filterwarnings("ignore", category=scipy.linalg.LinAlgWarning)
+        # XXX: Investigate if the ConvergenceWarning that can appear in some
+        # cases should be considered a bug or not. In the mean time we don't
+        # fail when the assertions below pass irrespective of the presence of
+        # the warning.
+        warnings.filterwarnings("ignore", category=ConvergenceWarning)
+        model.fit(X, y)
+
+    if n_samples > n_features:
+        rtol = 5e-5 if solver == "lbfgs" else 1e-6
+        assert model.intercept_ == pytest.approx(intercept)
+        assert_allclose(model.coef_, coef, rtol=rtol)
+    else:
+        # As it is an underdetermined problem, prediction = y. The following shows that
+        # we get a solution, i.e. a (non-unique) minimum of the objective function ...
+        rtol = 1e-6 if solver == "lbfgs" else 5e-6
+        assert_allclose(model.predict(X), y, rtol=rtol)
+
+        norm_solution = np.linalg.norm(np.r_[intercept, coef])
+        norm_model = np.linalg.norm(np.r_[model.intercept_, model.coef_])
+        if solver == "newton-cholesky":
+            # XXX: This solver shows random behaviour. Sometimes it finds solutions
+            # with norm_model <= norm_solution! So we check conditionally.
+            if not (norm_model > (1 + 1e-12) * norm_solution):
+                assert model.intercept_ == pytest.approx(intercept)
+                assert_allclose(model.coef_, coef, rtol=1e-4)
+        elif solver == "lbfgs" and fit_intercept:
+            # Same as in test_glm_regression_unpenalized.
+            # But it is not the minimum norm solution. Otherwise the norms would be
+            # equal.
+            assert norm_model > (1 + 1e-12) * norm_solution
+        else:
+            rtol = 1e-5 if solver == "newton-cholesky" else 1e-4
+            assert model.intercept_ == pytest.approx(intercept, rel=rtol)
+            assert_allclose(model.coef_, coef, rtol=rtol)
+
+
+def test_sample_weights_validation():
+    """Test the raised errors in the validation of sample_weight."""
+    # scalar value but not positive
+    X = [[1]]
+    y = [1]
+    weights = 0
+    glm = _GeneralizedLinearRegressor()
+
+    # Positive weights are accepted
+    glm.fit(X, y, sample_weight=1)
+
+    # 2d array
+    weights = [[0]]
+    with pytest.raises(ValueError, match="must be 1D array or scalar"):
+        glm.fit(X, y, weights)
+
+    # 1d but wrong length
+    weights = [1, 0]
+    msg = r"sample_weight.shape == \(2,\), expected \(1,\)!"
+    with pytest.raises(ValueError, match=msg):
+        glm.fit(X, y, weights)
+
+
+@pytest.mark.parametrize(
+    "glm",
+    [
+        TweedieRegressor(power=3),
+        PoissonRegressor(),
+        GammaRegressor(),
+        TweedieRegressor(power=1.5),
+    ],
+)
+def test_glm_wrong_y_range(glm):
+    """
+    Test that fitting a GLM model raises a ValueError when `y` contains
+    values outside the valid range for the given distribution.
+
+    Generalized Linear Models (GLMs) with certain distributions, such as
+    Poisson, Gamma, and Tweedie (with power > 1), require `y` to be
+    non-negative. This test ensures that passing a `y` array containing
+    negative values triggers the expected ValueError with the correct message.
+    """
+    y = np.array([-1, 2])
     X = np.array([[1], [1]])
-    glm = GeneralizedLinearRegressor(warm_start=warm_start)
-    with pytest.raises(ValueError, match="warm_start must be bool"):
+    msg = r"Some value\(s\) of y are out of the valid range of the loss"
+    with pytest.raises(ValueError, match=msg):
         glm.fit(X, y)
 
 
-@pytest.mark.parametrize('fit_intercept', [False, True])
+@pytest.mark.parametrize("fit_intercept", [False, True])
 def test_glm_identity_regression(fit_intercept):
     """Test GLM regression with identity link on a simple dataset."""
-    coef = [1., 2.]
+    coef = [1.0, 2.0]
     X = np.array([[1, 1, 1, 1, 1], [0, 1, 2, 3, 4]]).T
     y = np.dot(X, coef)
-    glm = GeneralizedLinearRegressor(alpha=0, family='normal', link='identity',
-                                     fit_intercept=fit_intercept, tol=1e-12)
+    glm = _GeneralizedLinearRegressor(
+        alpha=0,
+        fit_intercept=fit_intercept,
+        tol=1e-12,
+    )
     if fit_intercept:
         glm.fit(X[:, 1:], y)
         assert_allclose(glm.coef_, coef[1:], rtol=1e-10)
@@ -185,20 +643,21 @@ def test_glm_identity_regression(fit_intercept):
         assert_allclose(glm.coef_, coef, rtol=1e-12)
 
 
-@pytest.mark.parametrize('fit_intercept', [False, True])
-@pytest.mark.parametrize('alpha', [0.0, 1.0])
-@pytest.mark.parametrize('family', ['normal', 'poisson', 'gamma'])
-def test_glm_sample_weight_consistentcy(fit_intercept, alpha, family):
+@pytest.mark.parametrize("fit_intercept", [False, True])
+@pytest.mark.parametrize("alpha", [0.0, 1.0])
+@pytest.mark.parametrize(
+    "GLMEstimator", [_GeneralizedLinearRegressor, PoissonRegressor, GammaRegressor]
+)
+def test_glm_sample_weight_consistency(fit_intercept, alpha, GLMEstimator):
     """Test that the impact of sample_weight is consistent"""
     rng = np.random.RandomState(0)
     n_samples, n_features = 10, 5
 
     X = rng.rand(n_samples, n_features)
     y = rng.rand(n_samples)
-    glm_params = dict(alpha=alpha, family=family, link='auto',
-                      fit_intercept=fit_intercept)
+    glm_params = dict(alpha=alpha, fit_intercept=fit_intercept)
 
-    glm = GeneralizedLinearRegressor(**glm_params).fit(X, y)
+    glm = GLMEstimator(**glm_params).fit(X, y)
     coef = glm.coef_.copy()
 
     # sample_weight=np.ones(..) should be equivalent to sample_weight=None
@@ -207,12 +666,12 @@ def test_glm_sample_weight_consistentcy(fit_intercept, alpha, family):
     assert_allclose(glm.coef_, coef, rtol=1e-12)
 
     # sample_weight are normalized to 1 so, scaling them has no effect
-    sample_weight = 2*np.ones(y.shape)
+    sample_weight = 2 * np.ones(y.shape)
     glm.fit(X, y, sample_weight=sample_weight)
     assert_allclose(glm.coef_, coef, rtol=1e-12)
 
     # setting one element of sample_weight to 0 is equivalent to removing
-    # the correspoding sample
+    # the corresponding sample
     sample_weight = np.ones(y.shape)
     sample_weight[-1] = 0
     glm.fit(X, y, sample_weight=sample_weight)
@@ -221,99 +680,149 @@ def test_glm_sample_weight_consistentcy(fit_intercept, alpha, family):
     assert_allclose(glm.coef_, coef1, rtol=1e-12)
 
     # check that multiplying sample_weight by 2 is equivalent
-    # to repeating correspoding samples twice
-    X2 = np.concatenate([X, X[:n_samples//2]], axis=0)
-    y2 = np.concatenate([y, y[:n_samples//2]])
+    # to repeating corresponding samples twice
+    X2 = np.concatenate([X, X[: n_samples // 2]], axis=0)
+    y2 = np.concatenate([y, y[: n_samples // 2]])
     sample_weight_1 = np.ones(len(y))
-    sample_weight_1[:n_samples//2] = 2
+    sample_weight_1[: n_samples // 2] = 2
 
-    glm1 = GeneralizedLinearRegressor(**glm_params).fit(
-            X, y, sample_weight=sample_weight_1
-    )
+    glm1 = GLMEstimator(**glm_params).fit(X, y, sample_weight=sample_weight_1)
 
-    glm2 = GeneralizedLinearRegressor(**glm_params).fit(
-            X2, y2, sample_weight=None
-    )
+    glm2 = GLMEstimator(**glm_params).fit(X2, y2, sample_weight=None)
     assert_allclose(glm1.coef_, glm2.coef_)
 
 
-@pytest.mark.parametrize('fit_intercept', [True, False])
+@pytest.mark.parametrize("solver", SOLVERS)
+@pytest.mark.parametrize("fit_intercept", [True, False])
 @pytest.mark.parametrize(
-    'family',
-    [NormalDistribution(), PoissonDistribution(),
-     GammaDistribution(), InverseGaussianDistribution(),
-     TweedieDistribution(power=1.5), TweedieDistribution(power=4.5)])
-def test_glm_log_regression(fit_intercept, family):
+    "estimator",
+    [
+        PoissonRegressor(),
+        GammaRegressor(),
+        TweedieRegressor(power=3.0),
+        TweedieRegressor(power=0, link="log"),
+        TweedieRegressor(power=1.5),
+        TweedieRegressor(power=4.5),
+    ],
+)
+def test_glm_log_regression(solver, fit_intercept, estimator):
     """Test GLM regression with log link on a simple dataset."""
     coef = [0.2, -0.1]
-    X = np.array([[1, 1, 1, 1, 1], [0, 1, 2, 3, 4]]).T
+    X = np.array([[0, 1, 2, 3, 4], [1, 1, 1, 1, 1]]).T
     y = np.exp(np.dot(X, coef))
-    glm = GeneralizedLinearRegressor(
-                alpha=0, family=family, link='log',
-                fit_intercept=fit_intercept, tol=1e-7)
+    glm = clone(estimator).set_params(
+        alpha=0,
+        fit_intercept=fit_intercept,
+        solver=solver,
+        tol=1e-8,
+    )
     if fit_intercept:
-        res = glm.fit(X[:, 1:], y)
-        assert_allclose(res.coef_, coef[1:], rtol=1e-6)
-        assert_allclose(res.intercept_, coef[0], rtol=1e-6)
+        res = glm.fit(X[:, :-1], y)
+        assert_allclose(res.coef_, coef[:-1], rtol=1e-6)
+        assert_allclose(res.intercept_, coef[-1], rtol=1e-6)
     else:
         res = glm.fit(X, y)
         assert_allclose(res.coef_, coef, rtol=2e-6)
 
 
-@pytest.mark.parametrize('fit_intercept', [True, False])
-def test_warm_start(fit_intercept):
-    n_samples, n_features = 110, 10
-    X, y = make_regression(n_samples=n_samples, n_features=n_features,
-                           n_informative=n_features-2, noise=0.5,
-                           random_state=42)
-
-    glm1 = GeneralizedLinearRegressor(
-        warm_start=False,
-        fit_intercept=fit_intercept,
-        max_iter=1000
+@pytest.mark.parametrize("solver", SOLVERS)
+@pytest.mark.parametrize("fit_intercept", [True, False])
+def test_warm_start(solver, fit_intercept, global_random_seed):
+    """
+    Test that `warm_start=True` enables incremental fitting in PoissonRegressor.
+
+    This test verifies that when using `warm_start=True`, the model continues
+    optimizing from previous coefficients instead of restarting from scratch.
+    It ensures that after an initial fit with `max_iter=1`, the model has a
+    higher objective function value (indicating incomplete optimization).
+    The test then checks whether allowing additional iterations enables
+    convergence to a solution comparable to a fresh training run (`warm_start=False`).
+    """
+    n_samples, n_features = 100, 10
+    X, y = make_regression(
+        n_samples=n_samples,
+        n_features=n_features,
+        n_informative=n_features - 2,
+        bias=fit_intercept * 1.0,
+        noise=1.0,
+        random_state=global_random_seed,
     )
+    y = np.abs(y)  # Poisson requires non-negative targets.
+    alpha = 1
+    params = {
+        "solver": solver,
+        "fit_intercept": fit_intercept,
+        "tol": 1e-10,
+    }
+
+    glm1 = PoissonRegressor(warm_start=False, max_iter=1000, alpha=alpha, **params)
     glm1.fit(X, y)
 
-    glm2 = GeneralizedLinearRegressor(
-        warm_start=True,
+    glm2 = PoissonRegressor(warm_start=True, max_iter=1, alpha=alpha, **params)
+    # As we intentionally set max_iter=1 such that the solver should raise a
+    # ConvergenceWarning.
+    with pytest.warns(ConvergenceWarning):
+        glm2.fit(X, y)
+
+    linear_loss = LinearModelLoss(
+        base_loss=glm1._get_loss(),
         fit_intercept=fit_intercept,
-        max_iter=1
     )
-    # As we intentionally set max_iter=1, L-BFGS-B will issue a
-    # ConvergenceWarning which we here simply ignore.
-    with warnings.catch_warnings():
-        warnings.filterwarnings('ignore', category=ConvergenceWarning)
-        glm2.fit(X, y)
-    assert glm1.score(X, y) > glm2.score(X, y)
+    sw = np.full_like(y, fill_value=1 / n_samples)
+
+    objective_glm1 = linear_loss.loss(
+        coef=np.r_[glm1.coef_, glm1.intercept_] if fit_intercept else glm1.coef_,
+        X=X,
+        y=y,
+        sample_weight=sw,
+        l2_reg_strength=alpha,
+    )
+    objective_glm2 = linear_loss.loss(
+        coef=np.r_[glm2.coef_, glm2.intercept_] if fit_intercept else glm2.coef_,
+        X=X,
+        y=y,
+        sample_weight=sw,
+        l2_reg_strength=alpha,
+    )
+    assert objective_glm1 < objective_glm2
+
     glm2.set_params(max_iter=1000)
     glm2.fit(X, y)
-    # The two model are not exactly identical since the lbfgs solver
+    # The two models are not exactly identical since the lbfgs solver
     # computes the approximate hessian from previous iterations, which
     # will not be strictly identical in the case of a warm start.
-    assert_allclose(glm1.coef_, glm2.coef_, rtol=1e-5)
-    assert_allclose(glm1.score(X, y), glm2.score(X, y), rtol=1e-4)
+    rtol = 2e-4 if solver == "lbfgs" else 1e-9
+    assert_allclose(glm1.coef_, glm2.coef_, rtol=rtol)
+    assert_allclose(glm1.score(X, y), glm2.score(X, y), rtol=1e-5)
 
 
-@pytest.mark.parametrize('n_samples, n_features', [(100, 10), (10, 100)])
-@pytest.mark.parametrize('fit_intercept', [True, False])
-@pytest.mark.parametrize('sample_weight', [None, True])
-def test_normal_ridge_comparison(n_samples, n_features, fit_intercept,
-                                 sample_weight, request):
+@pytest.mark.parametrize("n_samples, n_features", [(100, 10), (10, 100)])
+@pytest.mark.parametrize("fit_intercept", [True, False])
+@pytest.mark.parametrize("sample_weight", [None, True])
+def test_normal_ridge_comparison(
+    n_samples, n_features, fit_intercept, sample_weight, request
+):
     """Compare with Ridge regression for Normal distributions."""
     test_size = 10
-    X, y = make_regression(n_samples=n_samples + test_size,
-                           n_features=n_features,
-                           n_informative=n_features-2, noise=0.5,
-                           random_state=42)
+    X, y = make_regression(
+        n_samples=n_samples + test_size,
+        n_features=n_features,
+        n_informative=n_features - 2,
+        noise=0.5,
+        random_state=42,
+    )
 
     if n_samples > n_features:
         ridge_params = {"solver": "svd"}
     else:
         ridge_params = {"solver": "saga", "max_iter": 1000000, "tol": 1e-7}
 
-    X_train, X_test, y_train, y_test, = train_test_split(
-        X, y, test_size=test_size, random_state=0
-    )
+    (
+        X_train,
+        X_test,
+        y_train,
+        y_test,
+    ) = train_test_split(X, y, test_size=test_size, random_state=0)
 
     alpha = 1.0
     if sample_weight is None:
@@ -324,27 +833,31 @@ def test_normal_ridge_comparison(n_samples, n_features, fit_intercept,
         alpha_ridge = alpha * sw_train.sum()
 
     # GLM has 1/(2*n) * Loss + 1/2*L2, Ridge has Loss + L2
-    ridge = Ridge(alpha=alpha_ridge, normalize=False,
-                  random_state=42, fit_intercept=fit_intercept,
-                  **ridge_params)
+    ridge = Ridge(
+        alpha=alpha_ridge,
+        random_state=42,
+        fit_intercept=fit_intercept,
+        **ridge_params,
+    )
     ridge.fit(X_train, y_train, sample_weight=sw_train)
 
-    glm = GeneralizedLinearRegressor(alpha=alpha, family='normal',
-                                     link='identity',
-                                     fit_intercept=fit_intercept,
-                                     max_iter=300,
-                                     tol=1e-5)
+    glm = _GeneralizedLinearRegressor(
+        alpha=alpha,
+        fit_intercept=fit_intercept,
+        max_iter=300,
+        tol=1e-5,
+    )
     glm.fit(X_train, y_train, sample_weight=sw_train)
-    assert glm.coef_.shape == (X.shape[1], )
+    assert glm.coef_.shape == (X.shape[1],)
     assert_allclose(glm.coef_, ridge.coef_, atol=5e-5)
     assert_allclose(glm.intercept_, ridge.intercept_, rtol=1e-5)
     assert_allclose(glm.predict(X_train), ridge.predict(X_train), rtol=2e-4)
     assert_allclose(glm.predict(X_test), ridge.predict(X_test), rtol=2e-4)
 
 
-def test_poisson_glmnet():
-    """Compare Poisson regression with L2 regularization and LogLink to glmnet
-    """
+@pytest.mark.parametrize("solver", ["lbfgs", "newton-cholesky"])
+def test_poisson_glmnet(solver):
+    """Compare Poisson regression with L2 regularization and LogLink to glmnet"""
     # library("glmnet")
     # options(digits=10)
     # df <- data.frame(a=c(-2,-1,1,2), b=c(0,0,1,1), y=c(0,1,1,2))
@@ -358,10 +871,13 @@ def test_poisson_glmnet():
     # b            0.03741173122
     X = np.array([[-2, -1, 1, 2], [0, 0, 1, 1]]).T
     y = np.array([0, 1, 1, 2])
-    glm = GeneralizedLinearRegressor(alpha=1,
-                                     fit_intercept=True, family='poisson',
-                                     link='log', tol=1e-7,
-                                     max_iter=300)
+    glm = PoissonRegressor(
+        alpha=1,
+        fit_intercept=True,
+        tol=1e-7,
+        max_iter=300,
+        solver=solver,
+    )
     glm.fit(X, y)
     assert_allclose(glm.intercept_, -0.12889386979, rtol=1e-5)
     assert_allclose(glm.coef_, [0.29019207995, 0.03741173122], rtol=1e-5)
@@ -370,56 +886,54 @@ def test_poisson_glmnet():
 def test_convergence_warning(regression_data):
     X, y = regression_data
 
-    est = GeneralizedLinearRegressor(max_iter=1, tol=1e-20)
+    est = _GeneralizedLinearRegressor(max_iter=1, tol=1e-20)
     with pytest.warns(ConvergenceWarning):
         est.fit(X, y)
 
 
-def test_poisson_regression_family(regression_data):
-    # Make sure the family attribute is read-only to prevent searching over it
-    # e.g. in a grid search
-    est = PoissonRegressor()
-    est.family == "poisson"
-
-    msg = "PoissonRegressor.family must be 'poisson'!"
-    with pytest.raises(ValueError, match=msg):
-        est.family = 0
-
-
-def test_gamma_regression_family(regression_data):
-    # Make sure the family attribute is read-only to prevent searching over it
-    # e.g. in a grid search
-    est = GammaRegressor()
-    est.family == "gamma"
-
-    msg = "GammaRegressor.family must be 'gamma'!"
-    with pytest.raises(ValueError, match=msg):
-        est.family = 0
+@pytest.mark.parametrize(
+    "name, link_class", [("identity", IdentityLink), ("log", LogLink)]
+)
+def test_tweedie_link_argument(name, link_class):
+    """Test GLM link argument set as string."""
+    y = np.array([0.1, 0.5])  # in range of all distributions
+    X = np.array([[1], [2]])
+    glm = TweedieRegressor(power=1, link=name).fit(X, y)
+    assert isinstance(glm._base_loss.link, link_class)
 
 
-def test_tweedie_regression_family(regression_data):
-    # Make sure the family attribute is always a TweedieDistribution and that
-    # the power attribute is properly updated
-    power = 2.0
-    est = TweedieRegressor(power=power)
-    assert isinstance(est.family, TweedieDistribution)
-    assert est.family.power == power
-    assert est.power == power
+@pytest.mark.parametrize(
+    "power, expected_link_class",
+    [
+        (0, IdentityLink),  # normal
+        (1, LogLink),  # poisson
+        (2, LogLink),  # gamma
+        (3, LogLink),  # inverse-gaussian
+    ],
+)
+def test_tweedie_link_auto(power, expected_link_class):
+    """Test that link='auto' delivers the expected link function"""
+    y = np.array([0.1, 0.5])  # in range of all distributions
+    X = np.array([[1], [2]])
+    glm = TweedieRegressor(link="auto", power=power).fit(X, y)
+    assert isinstance(glm._base_loss.link, expected_link_class)
 
-    new_power = 0
-    new_family = TweedieDistribution(power=new_power)
-    est.family = new_family
-    assert isinstance(est.family, TweedieDistribution)
-    assert est.family.power == new_power
-    assert est.power == new_power
 
-    msg = "TweedieRegressor.family must be of type TweedieDistribution!"
-    with pytest.raises(TypeError, match=msg):
-        est.family = None
+@pytest.mark.parametrize("power", [0, 1, 1.5, 2, 3])
+@pytest.mark.parametrize("link", ["log", "identity"])
+def test_tweedie_score(regression_data, power, link):
+    """Test that GLM score equals d2_tweedie_score for Tweedie losses."""
+    X, y = regression_data
+    # make y positive
+    y = np.abs(y) + 1.0
+    glm = TweedieRegressor(power=power, link=link).fit(X, y)
+    assert glm.score(X, y) == pytest.approx(
+        d2_tweedie_score(y, glm.predict(X), power=power)
+    )
 
 
 @pytest.mark.parametrize(
-    'estimator, value',
+    "estimator, value",
     [
         (PoissonRegressor(), True),
         (GammaRegressor(), True),
@@ -428,4 +942,201 @@ def test_tweedie_regression_family(regression_data):
     ],
 )
 def test_tags(estimator, value):
-    assert estimator._get_tags()['requires_positive_y'] is value
+    """Test that `positive_only` tag is correctly set for different estimators."""
+    assert estimator.__sklearn_tags__().target_tags.positive_only is value
+
+
+def test_linalg_warning_with_newton_solver(global_random_seed):
+    """
+    Test that the Newton solver raises a warning and falls back to LBFGS when
+    encountering a singular or ill-conditioned Hessian matrix.
+
+    This test assess the behavior of `PoissonRegressor` with the "newton-cholesky"
+    solver.
+    It verifies the following:-
+    - The model significantly improves upon the constant baseline deviance.
+    - LBFGS remains robust on collinear data.
+    - The Newton solver raises a `LinAlgWarning` on collinear data and falls
+      back to LBFGS.
+    """
+    newton_solver = "newton-cholesky"
+    rng = np.random.RandomState(global_random_seed)
+    # Use at least 20 samples to reduce the likelihood of getting a degenerate
+    # dataset for any global_random_seed.
+    X_orig = rng.normal(size=(20, 3))
+    y = rng.poisson(
+        np.exp(X_orig @ np.ones(X_orig.shape[1])), size=X_orig.shape[0]
+    ).astype(np.float64)
+
+    # Collinear variation of the same input features.
+    X_collinear = np.hstack([X_orig] * 10)
+
+    # Let's consider the deviance of a constant baseline on this problem.
+    baseline_pred = np.full_like(y, y.mean())
+    constant_model_deviance = mean_poisson_deviance(y, baseline_pred)
+    assert constant_model_deviance > 1.0
+
+    # No warning raised on well-conditioned design, even without regularization.
+    tol = 1e-10
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        reg = PoissonRegressor(solver=newton_solver, alpha=0.0, tol=tol).fit(X_orig, y)
+    original_newton_deviance = mean_poisson_deviance(y, reg.predict(X_orig))
+
+    # On this dataset, we should have enough data points to not make it
+    # possible to get a near zero deviance (for the any of the admissible
+    # random seeds). This will make it easier to interpret meaning of rtol in
+    # the subsequent assertions:
+    assert original_newton_deviance > 0.2
+
+    # We check that the model could successfully fit information in X_orig to
+    # improve upon the constant baseline by a large margin (when evaluated on
+    # the traing set).
+    assert constant_model_deviance - original_newton_deviance > 0.1
+
+    # LBFGS is robust to a collinear design because its approximation of the
+    # Hessian is Symmeric Positive Definite by construction. Let's record its
+    # solution
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        reg = PoissonRegressor(solver="lbfgs", alpha=0.0, tol=tol).fit(X_collinear, y)
+    collinear_lbfgs_deviance = mean_poisson_deviance(y, reg.predict(X_collinear))
+
+    # The LBFGS solution on the collinear is expected to reach a comparable
+    # solution to the Newton solution on the original data.
+    rtol = 1e-6
+    assert collinear_lbfgs_deviance == pytest.approx(original_newton_deviance, rel=rtol)
+
+    # Fitting a Newton solver on the collinear version of the training data
+    # without regularization should raise an informative warning and fallback
+    # to the LBFGS solver.
+    msg = (
+        "The inner solver of .*Newton.*Solver stumbled upon a singular or very "
+        "ill-conditioned Hessian matrix"
+    )
+    with pytest.warns(scipy.linalg.LinAlgWarning, match=msg):
+        reg = PoissonRegressor(solver=newton_solver, alpha=0.0, tol=tol).fit(
+            X_collinear, y
+        )
+    # As a result we should still automatically converge to a good solution.
+    collinear_newton_deviance = mean_poisson_deviance(y, reg.predict(X_collinear))
+    assert collinear_newton_deviance == pytest.approx(
+        original_newton_deviance, rel=rtol
+    )
+
+    # Increasing the regularization slightly should make the problem go away:
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", scipy.linalg.LinAlgWarning)
+        reg = PoissonRegressor(solver=newton_solver, alpha=1e-10).fit(X_collinear, y)
+
+    # The slightly penalized model on the collinear data should be close enough
+    # to the unpenalized model on the original data.
+    penalized_collinear_newton_deviance = mean_poisson_deviance(
+        y, reg.predict(X_collinear)
+    )
+    assert penalized_collinear_newton_deviance == pytest.approx(
+        original_newton_deviance, rel=rtol
+    )
+
+
+@pytest.mark.parametrize("verbose", [0, 1, 2])
+def test_newton_solver_verbosity(capsys, verbose):
+    """Test the std output of verbose newton solvers."""
+    y = np.array([1, 2], dtype=float)
+    X = np.array([[1.0, 0], [0, 1]], dtype=float)
+    linear_loss = LinearModelLoss(base_loss=HalfPoissonLoss(), fit_intercept=False)
+    sol = NewtonCholeskySolver(
+        coef=linear_loss.init_zero_coef(X),
+        linear_loss=linear_loss,
+        l2_reg_strength=0,
+        verbose=verbose,
+    )
+    sol.solve(X, y, None)  # returns array([0., 0.69314758])
+    captured = capsys.readouterr()
+
+    if verbose == 0:
+        assert captured.out == ""
+    else:
+        msg = [
+            "Newton iter=1",
+            "Check Convergence",
+            "1. max |gradient|",
+            "2. Newton decrement",
+            "Solver did converge at loss = ",
+        ]
+        for m in msg:
+            assert m in captured.out
+
+    if verbose >= 2:
+        msg = ["Backtracking Line Search", "line search iteration="]
+        for m in msg:
+            assert m in captured.out
+
+    # Set the Newton solver to a state with a completely wrong Newton step.
+    sol = NewtonCholeskySolver(
+        coef=linear_loss.init_zero_coef(X),
+        linear_loss=linear_loss,
+        l2_reg_strength=0,
+        verbose=verbose,
+    )
+    sol.setup(X=X, y=y, sample_weight=None)
+    sol.iteration = 1
+    sol.update_gradient_hessian(X=X, y=y, sample_weight=None)
+    sol.coef_newton = np.array([1.0, 0])
+    sol.gradient_times_newton = sol.gradient @ sol.coef_newton
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", ConvergenceWarning)
+        sol.line_search(X=X, y=y, sample_weight=None)
+        captured = capsys.readouterr()
+    if verbose >= 1:
+        assert (
+            "Line search did not converge and resorts to lbfgs instead." in captured.out
+        )
+
+    # Set the Newton solver to a state with bad Newton step such that the loss
+    # improvement in line search is tiny.
+    sol = NewtonCholeskySolver(
+        coef=np.array([1e-12, 0.69314758]),
+        linear_loss=linear_loss,
+        l2_reg_strength=0,
+        verbose=verbose,
+    )
+    sol.setup(X=X, y=y, sample_weight=None)
+    sol.iteration = 1
+    sol.update_gradient_hessian(X=X, y=y, sample_weight=None)
+    sol.coef_newton = np.array([1e-6, 0])
+    sol.gradient_times_newton = sol.gradient @ sol.coef_newton
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", ConvergenceWarning)
+        sol.line_search(X=X, y=y, sample_weight=None)
+        captured = capsys.readouterr()
+    if verbose >= 2:
+        msg = [
+            "line search iteration=",
+            "check loss improvement <= armijo term:",
+            "check loss |improvement| <= eps * |loss_old|:",
+            "check sum(|gradient|) < sum(|gradient_old|):",
+        ]
+        for m in msg:
+            assert m in captured.out
+
+    # Test for a case with negative hessian. We badly initialize coef for a Tweedie
+    # loss with non-canonical link, e.g. Inverse Gaussian deviance with a log link.
+    linear_loss = LinearModelLoss(
+        base_loss=HalfTweedieLoss(power=3), fit_intercept=False
+    )
+    sol = NewtonCholeskySolver(
+        coef=linear_loss.init_zero_coef(X) + 1,
+        linear_loss=linear_loss,
+        l2_reg_strength=0,
+        verbose=verbose,
+    )
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", ConvergenceWarning)
+        sol.solve(X, y, None)
+    captured = capsys.readouterr()
+    if verbose >= 1:
+        assert (
+            "The inner solver detected a pointwise Hessian with many negative values"
+            " and resorts to lbfgs instead." in captured.out
+        )
diff --git a/sklearn/linear_model/_glm/tests/test_link.py b/sklearn/linear_model/_glm/tests/test_link.py
deleted file mode 100644
index 27ec4ed19bdc2..0000000000000
--- a/sklearn/linear_model/_glm/tests/test_link.py
+++ /dev/null
@@ -1,45 +0,0 @@
-# Authors: Christian Lorentzen 
-#
-# License: BSD 3 clause
-import numpy as np
-from numpy.testing import assert_allclose
-import pytest
-from scipy.optimize import check_grad
-
-from sklearn.linear_model._glm.link import (
-    IdentityLink,
-    LogLink,
-    LogitLink,
-)
-
-
-LINK_FUNCTIONS = [IdentityLink, LogLink, LogitLink]
-
-
-@pytest.mark.parametrize('Link', LINK_FUNCTIONS)
-def test_link_properties(Link):
-    """Test link inverse and derivative."""
-    rng = np.random.RandomState(42)
-    x = rng.rand(100) * 100
-    link = Link()
-    if isinstance(link, LogitLink):
-        # careful for large x, note expit(36) = 1
-        # limit max eta to 15
-        x = x / 100 * 15
-    assert_allclose(link(link.inverse(x)), x)
-    # if g(h(x)) = x, then g'(h(x)) = 1/h'(x)
-    # g = link, h = link.inverse
-    assert_allclose(link.derivative(link.inverse(x)),
-                    1 / link.inverse_derivative(x))
-
-
-@pytest.mark.parametrize('Link', LINK_FUNCTIONS)
-def test_link_derivative(Link):
-    link = Link()
-    x = np.random.RandomState(0).rand(1)
-    err = check_grad(link, link.derivative, x) / link.derivative(x)
-    assert abs(err) < 1e-6
-
-    err = (check_grad(link.inverse, link.inverse_derivative, x)
-           / link.derivative(x))
-    assert abs(err) < 1e-6
diff --git a/sklearn/linear_model/_huber.py b/sklearn/linear_model/_huber.py
index 56062fa783eb8..51f24035a3c83 100644
--- a/sklearn/linear_model/_huber.py
+++ b/sklearn/linear_model/_huber.py
@@ -1,17 +1,18 @@
-# Authors: Manoj Kumar mks542@nyu.edu
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-import numpy as np
+from numbers import Integral, Real
 
+import numpy as np
 from scipy import optimize
 
-from ..base import BaseEstimator, RegressorMixin
-from ._base import LinearModel
-from ..utils import axis0_safe_slice
-from ..utils.validation import _check_sample_weight
-from ..utils.validation import _deprecate_positional_args
+from ..base import BaseEstimator, RegressorMixin, _fit_context
+from ..utils._mask import axis0_safe_slice
+from ..utils._param_validation import Interval
 from ..utils.extmath import safe_sparse_dot
 from ..utils.optimize import _check_optimize_result
+from ..utils.validation import _check_sample_weight, validate_data
+from ._base import LinearModel
 
 
 def _huber_loss_and_gradient(w, X, y, epsilon, alpha, sample_weight=None):
@@ -50,7 +51,7 @@ def _huber_loss_and_gradient(w, X, y, epsilon, alpha, sample_weight=None):
         coefficient, intercept and the scale as a vector.
     """
     _, n_features = X.shape
-    fit_intercept = (n_features + 2 == w.shape[0])
+    fit_intercept = n_features + 2 == w.shape[0]
     if fit_intercept:
         intercept = w[-2]
     sigma = w[-1]
@@ -75,8 +76,10 @@ def _huber_loss_and_gradient(w, X, y, epsilon, alpha, sample_weight=None):
     # num_outliers is just the number of outliers.
     outliers_sw = sample_weight[outliers_mask]
     n_sw_outliers = np.sum(outliers_sw)
-    outlier_loss = (2. * epsilon * np.sum(outliers_sw * outliers) -
-                    sigma * n_sw_outliers * epsilon ** 2)
+    outlier_loss = (
+        2.0 * epsilon * np.sum(outliers_sw * outliers)
+        - sigma * n_sw_outliers * epsilon**2
+    )
 
     # Calculate the quadratic loss due to the non-outliers.-
     # This is equal to |(y - X'w - c)**2 / sigma**2| * sigma
@@ -93,7 +96,8 @@ def _huber_loss_and_gradient(w, X, y, epsilon, alpha, sample_weight=None):
     # Gradient due to the squared loss.
     X_non_outliers = -axis0_safe_slice(X, ~outliers_mask, n_non_outliers)
     grad[:n_features] = (
-        2. / sigma * safe_sparse_dot(weighted_non_outliers, X_non_outliers))
+        2.0 / sigma * safe_sparse_dot(weighted_non_outliers, X_non_outliers)
+    )
 
     # Gradient due to the linear loss.
     signed_outliers = np.ones_like(outliers)
@@ -101,21 +105,20 @@ def _huber_loss_and_gradient(w, X, y, epsilon, alpha, sample_weight=None):
     signed_outliers[signed_outliers_mask] = -1.0
     X_outliers = axis0_safe_slice(X, outliers_mask, num_outliers)
     sw_outliers = sample_weight[outliers_mask] * signed_outliers
-    grad[:n_features] -= 2. * epsilon * (
-        safe_sparse_dot(sw_outliers, X_outliers))
+    grad[:n_features] -= 2.0 * epsilon * (safe_sparse_dot(sw_outliers, X_outliers))
 
     # Gradient due to the penalty.
-    grad[:n_features] += alpha * 2. * w
+    grad[:n_features] += alpha * 2.0 * w
 
     # Gradient due to sigma.
     grad[-1] = n_samples
-    grad[-1] -= n_sw_outliers * epsilon ** 2
+    grad[-1] -= n_sw_outliers * epsilon**2
     grad[-1] -= squared_loss / sigma
 
     # Gradient due to the intercept.
     if fit_intercept:
-        grad[-2] = -2. * np.sum(weighted_non_outliers) / sigma
-        grad[-2] -= 2. * epsilon * np.sum(sw_outliers)
+        grad[-2] = -2.0 * np.sum(weighted_non_outliers) / sigma
+        grad[-2] -= 2.0 * epsilon * np.sum(sw_outliers)
 
     loss = n_samples * sigma + squared_loss + outlier_loss
     loss += alpha * np.dot(w, w)
@@ -123,18 +126,19 @@ def _huber_loss_and_gradient(w, X, y, epsilon, alpha, sample_weight=None):
 
 
 class HuberRegressor(LinearModel, RegressorMixin, BaseEstimator):
-    """Linear regression model that is robust to outliers.
+    """L2-regularized linear regression model that is robust to outliers.
 
     The Huber Regressor optimizes the squared loss for the samples where
-    ``|(y - X'w) / sigma| < epsilon`` and the absolute loss for the samples
-    where ``|(y - X'w) / sigma| > epsilon``, where w and sigma are parameters
-    to be optimized. The parameter sigma makes sure that if y is scaled up
-    or down by a certain factor, one does not need to rescale epsilon to
+    ``|(y - Xw - c) / sigma| < epsilon`` and the absolute loss for the samples
+    where ``|(y - Xw - c) / sigma| > epsilon``, where the model coefficients
+    ``w``, the intercept ``c`` and the scale ``sigma`` are parameters
+    to be optimized. The parameter `sigma` makes sure that if `y` is scaled up
+    or down by a certain factor, one does not need to rescale `epsilon` to
     achieve the same robustness. Note that this does not take into account
-    the fact that the different features of X may be of different scales.
+    the fact that the different features of `X` may be of different scales.
 
-    This makes sure that the loss function is not heavily influenced by the
-    outliers while not completely ignoring their effect.
+    The Huber loss function has the advantage of not being heavily influenced
+    by the outliers while not completely ignoring their effect.
 
     Read more in the :ref:`User Guide `
 
@@ -142,17 +146,19 @@ class HuberRegressor(LinearModel, RegressorMixin, BaseEstimator):
 
     Parameters
     ----------
-    epsilon : float, greater than 1.0, default=1.35
+    epsilon : float, default=1.35
         The parameter epsilon controls the number of samples that should be
         classified as outliers. The smaller the epsilon, the more robust it is
-        to outliers.
+        to outliers. Epsilon must be in the range `[1, inf)`.
 
     max_iter : int, default=100
         Maximum number of iterations that
         ``scipy.optimize.minimize(method="L-BFGS-B")`` should run for.
 
     alpha : float, default=0.0001
-        Regularization parameter.
+        Strength of the squared L2 regularization. Note that the penalty is
+        equal to ``alpha * ||w||^2``.
+        Must be in the range `[0, inf)`.
 
     warm_start : bool, default=False
         This is useful if the stored attributes of a previously used model
@@ -172,13 +178,24 @@ class HuberRegressor(LinearModel, RegressorMixin, BaseEstimator):
     Attributes
     ----------
     coef_ : array, shape (n_features,)
-        Features got by optimizing the Huber loss.
+        Features got by optimizing the L2-regularized Huber loss.
 
     intercept_ : float
         Bias.
 
     scale_ : float
-        The value by which ``|y - X'w - c|`` is scaled down.
+        The value by which ``|y - Xw - c|`` is scaled down.
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
 
     n_iter_ : int
         Number of iterations that
@@ -193,6 +210,19 @@ class HuberRegressor(LinearModel, RegressorMixin, BaseEstimator):
         A boolean mask which is set to True where the samples are identified
         as outliers.
 
+    See Also
+    --------
+    RANSACRegressor : RANSAC (RANdom SAmple Consensus) algorithm.
+    TheilSenRegressor : Theil-Sen Estimator robust multivariate regression model.
+    SGDRegressor : Fitted by minimizing a regularized empirical loss with SGD.
+
+    References
+    ----------
+    .. [1] Peter J. Huber, Elvezio M. Ronchetti, Robust Statistics
+           Concomitant scale estimates, p. 172
+    .. [2] Art B. Owen (2006), `A robust hybrid of lasso and ridge regression.
+           `_
+
     Examples
     --------
     >>> import numpy as np
@@ -205,9 +235,9 @@ class HuberRegressor(LinearModel, RegressorMixin, BaseEstimator):
     >>> y[:4] = rng.uniform(10, 20, 4)
     >>> huber = HuberRegressor().fit(X, y)
     >>> huber.score(X, y)
-    -7.284...
+    -7.284
     >>> huber.predict(X[:1,])
-    array([806.7200...])
+    array([806.7200])
     >>> linear = LinearRegression().fit(X, y)
     >>> print("True coefficients:", coef)
     True coefficients: [20.4923...  34.1698...]
@@ -215,17 +245,27 @@ class HuberRegressor(LinearModel, RegressorMixin, BaseEstimator):
     Huber coefficients: [17.7906... 31.0106...]
     >>> print("Linear Regression coefficients:", linear.coef_)
     Linear Regression coefficients: [-1.9221...  7.0226...]
-
-    References
-    ----------
-    .. [1] Peter J. Huber, Elvezio M. Ronchetti, Robust Statistics
-           Concomitant scale estimates, pg 172
-    .. [2] Art B. Owen (2006), A robust hybrid of lasso and ridge regression.
-           https://statweb.stanford.edu/~owen/reports/hhu.pdf
     """
-    @_deprecate_positional_args
-    def __init__(self, *, epsilon=1.35, max_iter=100, alpha=0.0001,
-                 warm_start=False, fit_intercept=True, tol=1e-05):
+
+    _parameter_constraints: dict = {
+        "epsilon": [Interval(Real, 1.0, None, closed="left")],
+        "max_iter": [Interval(Integral, 0, None, closed="left")],
+        "alpha": [Interval(Real, 0, None, closed="left")],
+        "warm_start": ["boolean"],
+        "fit_intercept": ["boolean"],
+        "tol": [Interval(Real, 0.0, None, closed="left")],
+    }
+
+    def __init__(
+        self,
+        *,
+        epsilon=1.35,
+        max_iter=100,
+        alpha=0.0001,
+        warm_start=False,
+        fit_intercept=True,
+        tol=1e-05,
+    ):
         self.epsilon = epsilon
         self.max_iter = max_iter
         self.alpha = alpha
@@ -233,14 +273,15 @@ def __init__(self, *, epsilon=1.35, max_iter=100, alpha=0.0001,
         self.fit_intercept = fit_intercept
         self.tol = tol
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y, sample_weight=None):
         """Fit the model according to the given training data.
 
         Parameters
         ----------
         X : array-like, shape (n_samples, n_features)
-            Training vector, where n_samples in the number of samples and
-            n_features is the number of features.
+            Training vector, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
 
         y : array-like, shape (n_samples,)
             Target vector relative to X.
@@ -251,21 +292,22 @@ def fit(self, X, y, sample_weight=None):
         Returns
         -------
         self : object
+            Fitted `HuberRegressor` estimator.
         """
-        X, y = self._validate_data(
-            X, y, copy=False, accept_sparse=['csr'], y_numeric=True,
-            dtype=[np.float64, np.float32])
+        X, y = validate_data(
+            self,
+            X,
+            y,
+            copy=False,
+            accept_sparse=["csr"],
+            y_numeric=True,
+            dtype=[np.float64, np.float32],
+        )
 
         sample_weight = _check_sample_weight(sample_weight, X)
 
-        if self.epsilon < 1.0:
-            raise ValueError(
-                "epsilon should be greater than or equal to 1.0, got %f"
-                % self.epsilon)
-
-        if self.warm_start and hasattr(self, 'coef_'):
-            parameters = np.concatenate(
-                (self.coef_, [self.intercept_, self.scale_]))
+        if self.warm_start and hasattr(self, "coef_"):
+            parameters = np.concatenate((self.coef_, [self.intercept_, self.scale_]))
         else:
             if self.fit_intercept:
                 parameters = np.zeros(X.shape[1] + 2)
@@ -282,26 +324,35 @@ def fit(self, X, y, sample_weight=None):
         bounds[-1][0] = np.finfo(np.float64).eps * 10
 
         opt_res = optimize.minimize(
-            _huber_loss_and_gradient, parameters, method="L-BFGS-B", jac=True,
+            _huber_loss_and_gradient,
+            parameters,
+            method="L-BFGS-B",
+            jac=True,
             args=(X, y, self.epsilon, self.alpha, sample_weight),
             options={"maxiter": self.max_iter, "gtol": self.tol, "iprint": -1},
-            bounds=bounds)
+            bounds=bounds,
+        )
 
         parameters = opt_res.x
 
         if opt_res.status == 2:
-            raise ValueError("HuberRegressor convergence failed:"
-                             " l-BFGS-b solver terminated with %s"
-                             % opt_res.message)
+            raise ValueError(
+                "HuberRegressor convergence failed: l-BFGS-b solver terminated with %s"
+                % opt_res.message
+            )
         self.n_iter_ = _check_optimize_result("lbfgs", opt_res, self.max_iter)
         self.scale_ = parameters[-1]
         if self.fit_intercept:
             self.intercept_ = parameters[-2]
         else:
             self.intercept_ = 0.0
-        self.coef_ = parameters[:X.shape[1]]
+        self.coef_ = parameters[: X.shape[1]]
 
-        residual = np.abs(
-            y - safe_sparse_dot(X, self.coef_) - self.intercept_)
+        residual = np.abs(y - safe_sparse_dot(X, self.coef_) - self.intercept_)
         self.outliers_ = residual > self.scale_ * self.epsilon
         return self
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
+        return tags
diff --git a/sklearn/linear_model/_least_angle.py b/sklearn/linear_model/_least_angle.py
index 55e37ff51fc6a..4bffe5f6e8c0d 100644
--- a/sklearn/linear_model/_least_angle.py
+++ b/sklearn/linear_model/_least_angle.py
@@ -2,35 +2,64 @@
 Least Angle Regression algorithm. See the documentation on the
 Generalized Linear Model for a complete discussion.
 """
-# Author: Fabian Pedregosa 
-#         Alexandre Gramfort 
-#         Gael Varoquaux
-#
-# License: BSD 3 clause
 
-from math import log
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
 import sys
 import warnings
+from math import log
+from numbers import Integral, Real
 
 import numpy as np
-from scipy import linalg, interpolate
+from scipy import interpolate, linalg
 from scipy.linalg.lapack import get_lapack_funcs
-from joblib import Parallel
 
-from ._base import LinearModel
-from ..base import RegressorMixin, MultiOutputMixin
-# mypy error: Module 'sklearn.utils' has no attribute 'arrayfuncs'
-from ..utils import arrayfuncs, as_float_array  # type: ignore
-from ..utils import check_random_state
-from ..model_selection import check_cv
+from ..base import MultiOutputMixin, RegressorMixin, _fit_context
 from ..exceptions import ConvergenceWarning
-from ..utils.validation import _deprecate_positional_args
-from ..utils.fixes import delayed
-
-SOLVE_TRIANGULAR_ARGS = {'check_finite': False}
-
+from ..model_selection import check_cv
 
-@_deprecate_positional_args
+# mypy error: Module 'sklearn.utils' has no attribute 'arrayfuncs'
+from ..utils import (
+    Bunch,
+    arrayfuncs,
+    as_float_array,
+    check_random_state,
+)
+from ..utils._metadata_requests import (
+    MetadataRouter,
+    MethodMapping,
+    _raise_for_params,
+    _routing_enabled,
+    process_routing,
+)
+from ..utils._param_validation import Hidden, Interval, StrOptions, validate_params
+from ..utils.parallel import Parallel, delayed
+from ..utils.validation import validate_data
+from ._base import LinearModel, LinearRegression, _preprocess_data
+
+SOLVE_TRIANGULAR_ARGS = {"check_finite": False}
+
+
+@validate_params(
+    {
+        "X": [np.ndarray, None],
+        "y": [np.ndarray, None],
+        "Xy": [np.ndarray, None],
+        "Gram": [StrOptions({"auto"}), "boolean", np.ndarray, None],
+        "max_iter": [Interval(Integral, 0, None, closed="left")],
+        "alpha_min": [Interval(Real, 0, None, closed="left")],
+        "method": [StrOptions({"lar", "lasso"})],
+        "copy_X": ["boolean"],
+        "eps": [Interval(Real, 0, None, closed="neither"), None],
+        "copy_Gram": ["boolean"],
+        "verbose": ["verbose"],
+        "return_path": ["boolean"],
+        "return_n_iter": ["boolean"],
+        "positive": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
 def lars_path(
     X,
     y,
@@ -46,36 +75,35 @@ def lars_path(
     verbose=0,
     return_path=True,
     return_n_iter=False,
-    positive=False
+    positive=False,
 ):
-    """Compute Least Angle Regression or Lasso path using LARS algorithm [1]
+    """Compute Least Angle Regression or Lasso path using the LARS algorithm.
 
     The optimization objective for the case method='lasso' is::
 
     (1 / (2 * n_samples)) * ||y - Xw||^2_2 + alpha * ||w||_1
 
-    in the case of method='lars', the objective function is only known in
-    the form of an implicit equation (see discussion in [1])
+    in the case of method='lar', the objective function is only known in
+    the form of an implicit equation (see discussion in [1]_).
 
     Read more in the :ref:`User Guide `.
 
     Parameters
     ----------
-    X : None or array-like of shape (n_samples, n_features)
-        Input data. Note that if X is None then the Gram matrix must be
-        specified, i.e., cannot be None or False.
+    X : None or ndarray of shape (n_samples, n_features)
+        Input data. If X is `None`, Gram must also be `None`.
+        If only the Gram matrix is available, use `lars_path_gram` instead.
 
-    y : None or array-like of shape (n_samples,)
+    y : None or ndarray of shape (n_samples,)
         Input targets.
 
-    Xy : array-like of shape (n_samples,) or (n_samples, n_targets), \
-            default=None
-        Xy = np.dot(X.T, y) that can be precomputed. It is useful
+    Xy : array-like of shape (n_features,), default=None
+        `Xy = X.T @ y` that can be precomputed. It is useful
         only when the Gram matrix is precomputed.
 
-    Gram : None, 'auto', array-like of shape (n_features, n_features), \
+    Gram : None, 'auto', bool, ndarray of shape (n_features, n_features), \
             default=None
-        Precomputed Gram matrix (X' * X), if ``'auto'``, the Gram
+        Precomputed Gram matrix `X.T @ X`, if `'auto'`, the Gram
         matrix is precomputed from the given X, if there are more samples
         than features.
 
@@ -84,30 +112,30 @@ def lars_path(
 
     alpha_min : float, default=0
         Minimum correlation along the path. It corresponds to the
-        regularization parameter alpha parameter in the Lasso.
+        regularization parameter `alpha` in the Lasso.
 
     method : {'lar', 'lasso'}, default='lar'
-        Specifies the returned model. Select ``'lar'`` for Least Angle
-        Regression, ``'lasso'`` for the Lasso.
+        Specifies the returned model. Select `'lar'` for Least Angle
+        Regression, `'lasso'` for the Lasso.
 
     copy_X : bool, default=True
-        If ``False``, ``X`` is overwritten.
+        If `False`, `X` is overwritten.
 
     eps : float, default=np.finfo(float).eps
         The machine-precision regularization in the computation of the
         Cholesky diagonal factors. Increase this for very ill-conditioned
-        systems. Unlike the ``tol`` parameter in some iterative
+        systems. Unlike the `tol` parameter in some iterative
         optimization-based algorithms, this parameter does not control
         the tolerance of the optimization.
 
     copy_Gram : bool, default=True
-        If ``False``, ``Gram`` is overwritten.
+        If `False`, `Gram` is overwritten.
 
     verbose : int, default=0
         Controls output verbosity.
 
     return_path : bool, default=True
-        If ``return_path==True`` returns the entire path, else returns only the
+        If `True`, returns the entire path, else returns only the
         last point of the path.
 
     return_n_iter : bool, default=False
@@ -118,38 +146,37 @@ def lars_path(
         This option is only allowed with method 'lasso'. Note that the model
         coefficients will not converge to the ordinary-least-squares solution
         for small values of alpha. Only coefficients up to the smallest alpha
-        value (``alphas_[alphas_ > 0.].min()`` when fit_path=True) reached by
+        value (`alphas_[alphas_ > 0.].min()` when fit_path=True) reached by
         the stepwise Lars-Lasso algorithm are typically in congruence with the
-        solution of the coordinate descent lasso_path function.
+        solution of the coordinate descent `lasso_path` function.
 
     Returns
     -------
-    alphas : array-like of shape (n_alphas + 1,)
+    alphas : ndarray of shape (n_alphas + 1,)
         Maximum of covariances (in absolute value) at each iteration.
-        ``n_alphas`` is either ``max_iter``, ``n_features`` or the
-        number of nodes in the path with ``alpha >= alpha_min``, whichever
+        `n_alphas` is either `max_iter`, `n_features`, or the
+        number of nodes in the path with `alpha >= alpha_min`, whichever
         is smaller.
 
-    active : array-like of shape (n_alphas,)
+    active : ndarray of shape (n_alphas,)
         Indices of active variables at the end of the path.
 
-    coefs : array-like of shape (n_features, n_alphas + 1)
-        Coefficients along the path
+    coefs : ndarray of shape (n_features, n_alphas + 1)
+        Coefficients along the path.
 
     n_iter : int
-        Number of iterations run. Returned only if return_n_iter is set
+        Number of iterations run. Returned only if `return_n_iter` is set
         to True.
 
     See Also
     --------
-    lars_path_gram
-    lasso_path
-    lasso_path_gram
-    LassoLars
-    Lars
-    LassoLarsCV
-    LarsCV
-    sklearn.decomposition.sparse_encode
+    lars_path_gram : Compute LARS path in the sufficient stats mode.
+    lasso_path : Compute Lasso path with coordinate descent.
+    LassoLars : Lasso model fit with Least Angle Regression a.k.a. Lars.
+    Lars : Least Angle Regression model a.k.a. LAR.
+    LassoLarsCV : Cross-validated Lasso, using the LARS algorithm.
+    LarsCV : Cross-validated Least Angle Regression model.
+    sklearn.decomposition.sparse_encode : Sparse coding.
 
     References
     ----------
@@ -162,20 +189,67 @@ def lars_path(
     .. [3] `Wikipedia entry on the Lasso
            `_
 
+    Examples
+    --------
+    >>> from sklearn.linear_model import lars_path
+    >>> from sklearn.datasets import make_regression
+    >>> X, y, true_coef = make_regression(
+    ...    n_samples=100, n_features=5, n_informative=2, coef=True, random_state=0
+    ... )
+    >>> true_coef
+    array([ 0.        ,  0.        ,  0.        , 97.9, 45.7])
+    >>> alphas, _, estimated_coef = lars_path(X, y)
+    >>> alphas.shape
+    (3,)
+    >>> estimated_coef
+    array([[ 0.     ,  0.     ,  0.     ],
+           [ 0.     ,  0.     ,  0.     ],
+           [ 0.     ,  0.     ,  0.     ],
+           [ 0.     , 46.96, 97.99],
+           [ 0.     ,  0.     , 45.70]])
     """
     if X is None and Gram is not None:
         raise ValueError(
-            'X cannot be None if Gram is not None'
-            'Use lars_path_gram to avoid passing X and y.'
+            "X cannot be None if Gram is not None"
+            "Use lars_path_gram to avoid passing X and y."
         )
     return _lars_path_solver(
-        X=X, y=y, Xy=Xy, Gram=Gram, n_samples=None, max_iter=max_iter,
-        alpha_min=alpha_min, method=method, copy_X=copy_X,
-        eps=eps, copy_Gram=copy_Gram, verbose=verbose, return_path=return_path,
-        return_n_iter=return_n_iter, positive=positive)
-
-
-@_deprecate_positional_args
+        X=X,
+        y=y,
+        Xy=Xy,
+        Gram=Gram,
+        n_samples=None,
+        max_iter=max_iter,
+        alpha_min=alpha_min,
+        method=method,
+        copy_X=copy_X,
+        eps=eps,
+        copy_Gram=copy_Gram,
+        verbose=verbose,
+        return_path=return_path,
+        return_n_iter=return_n_iter,
+        positive=positive,
+    )
+
+
+@validate_params(
+    {
+        "Xy": [np.ndarray],
+        "Gram": [np.ndarray],
+        "n_samples": [Interval(Integral, 0, None, closed="left")],
+        "max_iter": [Interval(Integral, 0, None, closed="left")],
+        "alpha_min": [Interval(Real, 0, None, closed="left")],
+        "method": [StrOptions({"lar", "lasso"})],
+        "copy_X": ["boolean"],
+        "eps": [Interval(Real, 0, None, closed="neither"), None],
+        "copy_Gram": ["boolean"],
+        "verbose": ["verbose"],
+        "return_path": ["boolean"],
+        "return_n_iter": ["boolean"],
+        "positive": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
 def lars_path_gram(
     Xy,
     Gram,
@@ -190,28 +264,28 @@ def lars_path_gram(
     verbose=0,
     return_path=True,
     return_n_iter=False,
-    positive=False
+    positive=False,
 ):
-    """lars_path in the sufficient stats mode [1]
+    """The lars_path in the sufficient stats mode.
 
     The optimization objective for the case method='lasso' is::
 
     (1 / (2 * n_samples)) * ||y - Xw||^2_2 + alpha * ||w||_1
 
-    in the case of method='lars', the objective function is only known in
-    the form of an implicit equation (see discussion in [1])
+    in the case of method='lar', the objective function is only known in
+    the form of an implicit equation (see discussion in [1]_).
 
     Read more in the :ref:`User Guide `.
 
     Parameters
     ----------
-    Xy : array-like of shape (n_samples,) or (n_samples, n_targets)
-        Xy = np.dot(X.T, y).
+    Xy : ndarray of shape (n_features,)
+        `Xy = X.T @ y`.
 
-    Gram : array-like of shape (n_features, n_features)
-        Gram = np.dot(X.T * X).
+    Gram : ndarray of shape (n_features, n_features)
+        `Gram = X.T @ X`.
 
-    n_samples : int or float
+    n_samples : int
         Equivalent size of sample.
 
     max_iter : int, default=500
@@ -222,27 +296,27 @@ def lars_path_gram(
         regularization parameter alpha parameter in the Lasso.
 
     method : {'lar', 'lasso'}, default='lar'
-        Specifies the returned model. Select ``'lar'`` for Least Angle
+        Specifies the returned model. Select `'lar'` for Least Angle
         Regression, ``'lasso'`` for the Lasso.
 
     copy_X : bool, default=True
-        If ``False``, ``X`` is overwritten.
+        If `False`, `X` is overwritten.
 
     eps : float, default=np.finfo(float).eps
         The machine-precision regularization in the computation of the
         Cholesky diagonal factors. Increase this for very ill-conditioned
-        systems. Unlike the ``tol`` parameter in some iterative
+        systems. Unlike the `tol` parameter in some iterative
         optimization-based algorithms, this parameter does not control
         the tolerance of the optimization.
 
     copy_Gram : bool, default=True
-        If ``False``, ``Gram`` is overwritten.
+        If `False`, `Gram` is overwritten.
 
     verbose : int, default=0
         Controls output verbosity.
 
     return_path : bool, default=True
-        If ``return_path==True`` returns the entire path, else returns only the
+        If `return_path==True` returns the entire path, else returns only the
         last point of the path.
 
     return_n_iter : bool, default=False
@@ -253,38 +327,37 @@ def lars_path_gram(
         This option is only allowed with method 'lasso'. Note that the model
         coefficients will not converge to the ordinary-least-squares solution
         for small values of alpha. Only coefficients up to the smallest alpha
-        value (``alphas_[alphas_ > 0.].min()`` when fit_path=True) reached by
+        value (`alphas_[alphas_ > 0.].min()` when `fit_path=True`) reached by
         the stepwise Lars-Lasso algorithm are typically in congruence with the
         solution of the coordinate descent lasso_path function.
 
     Returns
     -------
-    alphas : array-like of shape (n_alphas + 1,)
+    alphas : ndarray of shape (n_alphas + 1,)
         Maximum of covariances (in absolute value) at each iteration.
-        ``n_alphas`` is either ``max_iter``, ``n_features`` or the
-        number of nodes in the path with ``alpha >= alpha_min``, whichever
+        `n_alphas` is either `max_iter`, `n_features` or the
+        number of nodes in the path with `alpha >= alpha_min`, whichever
         is smaller.
 
-    active : array-like of shape (n_alphas,)
+    active : ndarray of shape (n_alphas,)
         Indices of active variables at the end of the path.
 
-    coefs : array-like of shape (n_features, n_alphas + 1)
-        Coefficients along the path
+    coefs : ndarray of shape (n_features, n_alphas + 1)
+        Coefficients along the path.
 
     n_iter : int
-        Number of iterations run. Returned only if return_n_iter is set
+        Number of iterations run. Returned only if `return_n_iter` is set
         to True.
 
     See Also
     --------
-    lars_path
-    lasso_path
-    lasso_path_gram
-    LassoLars
-    Lars
-    LassoLarsCV
-    LarsCV
-    sklearn.decomposition.sparse_encode
+    lars_path_gram : Compute LARS path.
+    lasso_path : Compute Lasso path with coordinate descent.
+    LassoLars : Lasso model fit with Least Angle Regression a.k.a. Lars.
+    Lars : Least Angle Regression model a.k.a. LAR.
+    LassoLarsCV : Cross-validated Lasso, using the LARS algorithm.
+    LarsCV : Cross-validated Least Angle Regression model.
+    sklearn.decomposition.sparse_encode : Sparse coding.
 
     References
     ----------
@@ -297,13 +370,42 @@ def lars_path_gram(
     .. [3] `Wikipedia entry on the Lasso
            `_
 
+    Examples
+    --------
+    >>> from sklearn.linear_model import lars_path_gram
+    >>> from sklearn.datasets import make_regression
+    >>> X, y, true_coef = make_regression(
+    ...    n_samples=100, n_features=5, n_informative=2, coef=True, random_state=0
+    ... )
+    >>> true_coef
+    array([ 0.        ,  0.        ,  0.        , 97.9, 45.7])
+    >>> alphas, _, estimated_coef = lars_path_gram(X.T @ y, X.T @ X, n_samples=100)
+    >>> alphas.shape
+    (3,)
+    >>> estimated_coef
+    array([[ 0.     ,  0.     ,  0.     ],
+           [ 0.     ,  0.     ,  0.     ],
+           [ 0.     ,  0.     ,  0.     ],
+           [ 0.     , 46.96, 97.99],
+           [ 0.     ,  0.     , 45.70]])
     """
     return _lars_path_solver(
-        X=None, y=None, Xy=Xy, Gram=Gram, n_samples=n_samples,
-        max_iter=max_iter, alpha_min=alpha_min, method=method,
-        copy_X=copy_X, eps=eps, copy_Gram=copy_Gram,
-        verbose=verbose, return_path=return_path,
-        return_n_iter=return_n_iter, positive=positive)
+        X=None,
+        y=None,
+        Xy=Xy,
+        Gram=Gram,
+        n_samples=n_samples,
+        max_iter=max_iter,
+        alpha_min=alpha_min,
+        method=method,
+        copy_X=copy_X,
+        eps=eps,
+        copy_Gram=copy_Gram,
+        verbose=verbose,
+        return_path=return_path,
+        return_n_iter=return_n_iter,
+        positive=positive,
+    )
 
 
 def _lars_path_solver(
@@ -329,7 +431,7 @@ def _lars_path_solver(
 
     (1 / (2 * n_samples)) * ||y - Xw||^2_2 + alpha * ||w||_1
 
-    in the case of method='lars', the objective function is only known in
+    in the case of method='lar', the objective function is only known in
     the form of an implicit equation (see discussion in [1])
 
     Read more in the :ref:`User Guide `.
@@ -343,8 +445,7 @@ def _lars_path_solver(
     y : None or ndarray of shape (n_samples,)
         Input targets.
 
-    Xy : array-like of shape (n_samples,) or (n_samples, n_targets), \
-            default=None
+    Xy : array-like of shape (n_features,), default=None
         `Xy = np.dot(X.T, y)` that can be precomputed. It is useful
         only when the Gram matrix is precomputed.
 
@@ -440,9 +541,7 @@ def _lars_path_solver(
 
     """
     if method == "lar" and positive:
-        raise ValueError(
-            "Positive constraint not supported for 'lar' " "coding method."
-        )
+        raise ValueError("Positive constraint not supported for 'lar' coding method.")
 
     n_samples = n_samples if n_samples is not None else y.size
 
@@ -454,8 +553,8 @@ def _lars_path_solver(
     if Gram is None or Gram is False:
         Gram = None
         if X is None:
-            raise ValueError('X and Gram cannot both be unspecified.')
-    elif isinstance(Gram, str) and Gram == 'auto' or Gram is True:
+            raise ValueError("X and Gram cannot both be unspecified.")
+    elif (isinstance(Gram, str) and Gram == "auto") or Gram is True:
         if Gram is True or X.shape[0] > X.shape[1]:
             Gram = np.dot(X.T, X)
         else:
@@ -468,23 +567,37 @@ def _lars_path_solver(
     else:
         n_features = Cov.shape[0]
         if Gram.shape != (n_features, n_features):
-            raise ValueError('The shapes of the inputs Gram and Xy'
-                             ' do not match.')
+            raise ValueError("The shapes of the inputs Gram and Xy do not match.")
 
     if copy_X and X is not None and Gram is None:
         # force copy. setting the array to be fortran-ordered
         # speeds up the calculation of the (partial) Gram matrix
         # and allows to easily swap columns
-        X = X.copy('F')
+        X = X.copy("F")
 
     max_features = min(max_iter, n_features)
 
+    dtypes = set(a.dtype for a in (X, y, Xy, Gram) if a is not None)
+    if len(dtypes) == 1:
+        # use the precision level of input data if it is consistent
+        return_dtype = next(iter(dtypes))
+    else:
+        # fallback to double precision otherwise
+        return_dtype = np.float64
+
     if return_path:
-        coefs = np.zeros((max_features + 1, n_features))
-        alphas = np.zeros(max_features + 1)
+        coefs = np.zeros((max_features + 1, n_features), dtype=return_dtype)
+        alphas = np.zeros(max_features + 1, dtype=return_dtype)
     else:
-        coef, prev_coef = np.zeros(n_features), np.zeros(n_features)
-        alpha, prev_alpha = np.array([0.]), np.array([0.])  # better ideas?
+        coef, prev_coef = (
+            np.zeros(n_features, dtype=return_dtype),
+            np.zeros(n_features, dtype=return_dtype),
+        )
+        alpha, prev_alpha = (
+            np.array([0.0], dtype=return_dtype),
+            np.array([0.0], dtype=return_dtype),
+        )
+        # above better ideas?
 
     n_iter, n_active = 0, 0
     active, indices = list(), np.arange(n_features)
@@ -496,20 +609,21 @@ def _lars_path_solver(
     # referenced.
     if Gram is None:
         L = np.empty((max_features, max_features), dtype=X.dtype)
-        swap, nrm2 = linalg.get_blas_funcs(('swap', 'nrm2'), (X,))
+        swap, nrm2 = linalg.get_blas_funcs(("swap", "nrm2"), (X,))
     else:
         L = np.empty((max_features, max_features), dtype=Gram.dtype)
-        swap, nrm2 = linalg.get_blas_funcs(('swap', 'nrm2'), (Cov,))
-    solve_cholesky, = get_lapack_funcs(('potrs',), (L,))
+        swap, nrm2 = linalg.get_blas_funcs(("swap", "nrm2"), (Cov,))
+    (solve_cholesky,) = get_lapack_funcs(("potrs",), (L,))
 
     if verbose:
         if verbose > 1:
             print("Step\t\tAdded\t\tDropped\t\tActive set size\t\tC")
         else:
-            sys.stdout.write('.')
+            sys.stdout.write(".")
             sys.stdout.flush()
 
     tiny32 = np.finfo(np.float32).tiny  # to avoid division by 0 warning
+    cov_precision = np.finfo(Cov.dtype).precision
     equality_tolerance = np.finfo(np.float32).eps
 
     if Gram is not None:
@@ -530,7 +644,7 @@ def _lars_path_solver(
             else:
                 C = np.fabs(C_)
         else:
-            C = 0.
+            C = 0.0
 
         if return_path:
             alpha = alphas[n_iter, np.newaxis]
@@ -545,8 +659,7 @@ def _lars_path_solver(
                 if n_iter > 0:
                     # In the first iteration, all alphas are zero, the formula
                     # below would make ss a NaN
-                    ss = ((prev_alpha[0] - alpha_min) /
-                          (prev_alpha[0] - alpha[0]))
+                    ss = (prev_alpha[0] - alpha_min) / (prev_alpha[0] - alpha[0])
                     coef[:] = prev_coef + ss * (coef - prev_coef)
                 alpha[0] = alpha_min
             if return_path:
@@ -556,7 +669,6 @@ def _lars_path_solver(
         if n_iter >= max_iter or n_active >= n_features:
             break
         if not drop:
-
             ##########################################################
             # Append x_j to the Cholesky factorization of (Xa * Xa') #
             #                                                        #
@@ -580,8 +692,7 @@ def _lars_path_solver(
             if Gram is None:
                 X.T[n], X.T[m] = swap(X.T[n], X.T[m])
                 c = nrm2(X.T[n_active]) ** 2
-                L[n_active, :n_active] = \
-                    np.dot(X.T[n_active], X.T[:n_active].T)
+                L[n_active, :n_active] = np.dot(X.T[n_active], X.T[:n_active].T)
             else:
                 # swap does only work inplace if matrix is fortran
                 # contiguous ...
@@ -592,11 +703,14 @@ def _lars_path_solver(
 
             # Update the cholesky decomposition for the Gram matrix
             if n_active:
-                linalg.solve_triangular(L[:n_active, :n_active],
-                                        L[n_active, :n_active],
-                                        trans=0, lower=1,
-                                        overwrite_b=True,
-                                        **SOLVE_TRIANGULAR_ARGS)
+                linalg.solve_triangular(
+                    L[:n_active, :n_active],
+                    L[n_active, :n_active],
+                    trans=0,
+                    lower=1,
+                    overwrite_b=True,
+                    **SOLVE_TRIANGULAR_ARGS,
+                )
 
             v = np.dot(L[n_active, :n_active], L[n_active, :n_active])
             diag = max(np.sqrt(np.abs(c - v)), eps)
@@ -606,20 +720,16 @@ def _lars_path_solver(
                 # The system is becoming too ill-conditioned.
                 # We have degenerate vectors in our active set.
                 # We'll 'drop for good' the last regressor added.
-
-                # Note: this case is very rare. It is no longer triggered by
-                # the test suite. The `equality_tolerance` margin added in 0.16
-                # to get early stopping to work consistently on all versions of
-                # Python including 32 bit Python under Windows seems to make it
-                # very difficult to trigger the 'drop for good' strategy.
-                warnings.warn('Regressors in active set degenerate. '
-                              'Dropping a regressor, after %i iterations, '
-                              'i.e. alpha=%.3e, '
-                              'with an active set of %i regressors, and '
-                              'the smallest cholesky pivot element being %.3e.'
-                              ' Reduce max_iter or increase eps parameters.'
-                              % (n_iter, alpha, n_active, diag),
-                              ConvergenceWarning)
+                warnings.warn(
+                    "Regressors in active set degenerate. "
+                    "Dropping a regressor, after %i iterations, "
+                    "i.e. alpha=%.3e, "
+                    "with an active set of %i regressors, and "
+                    "the smallest cholesky pivot element being %.3e."
+                    " Reduce max_iter or increase eps parameters."
+                    % (n_iter, alpha.item(), n_active, diag),
+                    ConvergenceWarning,
+                )
 
                 # XXX: need to figure a 'drop for good' way
                 Cov = Cov_not_shortened
@@ -631,47 +741,49 @@ def _lars_path_solver(
             n_active += 1
 
             if verbose > 1:
-                print("%s\t\t%s\t\t%s\t\t%s\t\t%s" % (n_iter, active[-1], '',
-                                                      n_active, C))
+                print(
+                    "%s\t\t%s\t\t%s\t\t%s\t\t%s" % (n_iter, active[-1], "", n_active, C)
+                )
 
-        if method == 'lasso' and n_iter > 0 and prev_alpha[0] < alpha[0]:
+        if method == "lasso" and n_iter > 0 and prev_alpha[0] < alpha[0]:
             # alpha is increasing. This is because the updates of Cov are
             # bringing in too much numerical error that is greater than
             # than the remaining correlation with the
             # regressors. Time to bail out
-            warnings.warn('Early stopping the lars path, as the residues '
-                          'are small and the current value of alpha is no '
-                          'longer well controlled. %i iterations, alpha=%.3e, '
-                          'previous alpha=%.3e, with an active set of %i '
-                          'regressors.'
-                          % (n_iter, alpha, prev_alpha, n_active),
-                          ConvergenceWarning)
+            warnings.warn(
+                "Early stopping the lars path, as the residues "
+                "are small and the current value of alpha is no "
+                "longer well controlled. %i iterations, alpha=%.3e, "
+                "previous alpha=%.3e, with an active set of %i "
+                "regressors." % (n_iter, alpha.item(), prev_alpha.item(), n_active),
+                ConvergenceWarning,
+            )
             break
 
         # least squares solution
-        least_squares, _ = solve_cholesky(L[:n_active, :n_active],
-                                          sign_active[:n_active],
-                                          lower=True)
+        least_squares, _ = solve_cholesky(
+            L[:n_active, :n_active], sign_active[:n_active], lower=True
+        )
 
         if least_squares.size == 1 and least_squares == 0:
             # This happens because sign_active[:n_active] = 0
             least_squares[...] = 1
-            AA = 1.
+            AA = 1.0
         else:
             # is this really needed ?
-            AA = 1. / np.sqrt(np.sum(least_squares * sign_active[:n_active]))
+            AA = 1.0 / np.sqrt(np.sum(least_squares * sign_active[:n_active]))
 
             if not np.isfinite(AA):
                 # L is too ill-conditioned
                 i = 0
                 L_ = L[:n_active, :n_active].copy()
                 while not np.isfinite(AA):
-                    L_.flat[::n_active + 1] += (2 ** i) * eps
+                    L_.flat[:: n_active + 1] += (2**i) * eps
                     least_squares, _ = solve_cholesky(
-                        L_, sign_active[:n_active], lower=True)
-                    tmp = max(np.sum(least_squares * sign_active[:n_active]),
-                              eps)
-                    AA = 1. / np.sqrt(tmp)
+                        L_, sign_active[:n_active], lower=True
+                    )
+                    tmp = max(np.sum(least_squares * sign_active[:n_active]), eps)
+                    AA = 1.0 / np.sqrt(tmp)
                     i += 1
             least_squares *= AA
 
@@ -685,8 +797,11 @@ def _lars_path_solver(
             # if huge number of features, this takes 50% of time, I
             # think could be avoided if we just update it using an
             # orthogonal (QR) decomposition of X
-            corr_eq_dir = np.dot(Gram[:n_active, n_active:].T,
-                                 least_squares)
+            corr_eq_dir = np.dot(Gram[:n_active, n_active:].T, least_squares)
+
+        # Explicit rounding can be necessary to avoid `np.argmax(Cov)` yielding
+        # unstable results because of rounding errors.
+        np.around(corr_eq_dir, decimals=cov_precision, out=corr_eq_dir)
 
         g1 = arrayfuncs.min_pos((C - Cov) / (AA - corr_eq_dir + tiny32))
         if positive:
@@ -706,7 +821,7 @@ def _lars_path_solver(
             # update the sign, important for LAR
             sign_active[idx] = -sign_active[idx]
 
-            if method == 'lasso':
+            if method == "lasso":
                 gamma_ = z_pos
             drop = True
 
@@ -735,8 +850,7 @@ def _lars_path_solver(
         Cov -= gamma_ * corr_eq_dir
 
         # See if any coefficient has changed sign
-        if drop and method == 'lasso':
-
+        if drop and method == "lasso":
             # handle the case when idx is not length of 1
             for ii in idx:
                 arrayfuncs.cholesky_delete(L[:n_active, :n_active], ii)
@@ -763,8 +877,7 @@ def _lars_path_solver(
                     for i in range(ii, n_active):
                         indices[i], indices[i + 1] = indices[i + 1], indices[i]
                         Gram[i], Gram[i + 1] = swap(Gram[i], Gram[i + 1])
-                        Gram[:, i], Gram[:, i + 1] = swap(Gram[:, i],
-                                                          Gram[:, i + 1])
+                        Gram[:, i], Gram[:, i + 1] = swap(Gram[:, i], Gram[:, i + 1])
 
                 # Cov_n = Cov_j + x_j * X + increment(betas) TODO:
                 # will this still work with multiple drops ?
@@ -777,15 +890,17 @@ def _lars_path_solver(
                 Cov = np.r_[temp, Cov]
 
             sign_active = np.delete(sign_active, idx)
-            sign_active = np.append(sign_active, 0.)  # just to maintain size
+            sign_active = np.append(sign_active, 0.0)  # just to maintain size
             if verbose > 1:
-                print("%s\t\t%s\t\t%s\t\t%s\t\t%s" % (n_iter, '', drop_idx,
-                                                      n_active, abs(temp)))
+                print(
+                    "%s\t\t%s\t\t%s\t\t%s\t\t%s"
+                    % (n_iter, "", drop_idx, n_active, abs(temp))
+                )
 
     if return_path:
         # resize coefs in case of early stop
-        alphas = alphas[:n_iter + 1]
-        coefs = coefs[:n_iter + 1]
+        alphas = alphas[: n_iter + 1]
+        coefs = coefs[: n_iter + 1]
 
         if return_n_iter:
             return alphas, active, coefs.T, n_iter
@@ -801,8 +916,9 @@ def _lars_path_solver(
 ###############################################################################
 # Estimator classes
 
+
 class Lars(MultiOutputMixin, RegressorMixin, LinearModel):
-    """Least Angle Regression model a.k.a. LAR
+    """Least Angle Regression model a.k.a. LAR.
 
     Read more in the :ref:`User Guide `.
 
@@ -816,14 +932,6 @@ class Lars(MultiOutputMixin, RegressorMixin, LinearModel):
     verbose : bool or int, default=False
         Sets the verbosity amount.
 
-    normalize : bool, default=True
-        This parameter is ignored when ``fit_intercept`` is set to False.
-        If True, the regressors X will be normalized before regression by
-        subtracting the mean and dividing by the l2-norm.
-        If you wish to standardize, please use
-        :class:`~sklearn.preprocessing.StandardScaler` before calling ``fit``
-        on an estimator with ``normalize=False``.
-
     precompute : bool, 'auto' or array-like , default='auto'
         Whether to use a precomputed Gram matrix to speed up
         calculations. If set to ``'auto'`` let us decide. The Gram
@@ -891,6 +999,24 @@ class Lars(MultiOutputMixin, RegressorMixin, LinearModel):
         The number of iterations taken by lars_path to find the
         grid of alphas for each target.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    lars_path: Compute Least Angle Regression or Lasso
+        path using LARS algorithm.
+    LarsCV : Cross-validated Least Angle Regression model.
+    sklearn.decomposition.sparse_encode : Sparse coding.
+
     Examples
     --------
     >>> from sklearn import linear_model
@@ -898,26 +1024,39 @@ class Lars(MultiOutputMixin, RegressorMixin, LinearModel):
     >>> reg.fit([[-1, 1], [0, 0], [1, 1]], [-1.1111, 0, -1.1111])
     Lars(n_nonzero_coefs=1)
     >>> print(reg.coef_)
-    [ 0. -1.11...]
-
-    See Also
-    --------
-    lars_path, LarsCV
-    sklearn.decomposition.sparse_encode
-
+    [ 0. -1.11]
     """
 
+    _parameter_constraints: dict = {
+        "fit_intercept": ["boolean"],
+        "verbose": ["verbose"],
+        "precompute": ["boolean", StrOptions({"auto"}), np.ndarray, Hidden(None)],
+        "n_nonzero_coefs": [Interval(Integral, 1, None, closed="left")],
+        "eps": [Interval(Real, 0, None, closed="left")],
+        "copy_X": ["boolean"],
+        "fit_path": ["boolean"],
+        "jitter": [Interval(Real, 0, None, closed="left"), None],
+        "random_state": ["random_state"],
+    }
+
     method = "lar"
     positive = False
 
-    @_deprecate_positional_args
-    def __init__(self, *, fit_intercept=True, verbose=False, normalize=True,
-                 precompute='auto', n_nonzero_coefs=500,
-                 eps=np.finfo(float).eps, copy_X=True, fit_path=True,
-                 jitter=None, random_state=None):
+    def __init__(
+        self,
+        *,
+        fit_intercept=True,
+        verbose=False,
+        precompute="auto",
+        n_nonzero_coefs=500,
+        eps=np.finfo(float).eps,
+        copy_X=True,
+        fit_path=True,
+        jitter=None,
+        random_state=None,
+    ):
         self.fit_intercept = fit_intercept
         self.verbose = verbose
-        self.normalize = normalize
         self.precompute = precompute
         self.n_nonzero_coefs = n_nonzero_coefs
         self.eps = eps
@@ -928,10 +1067,11 @@ def __init__(self, *, fit_intercept=True, verbose=False, normalize=True,
 
     @staticmethod
     def _get_gram(precompute, X, y):
-        if (not hasattr(precompute, '__array__')) and (
-                (precompute is True) or
-                (precompute == 'auto' and X.shape[0] > X.shape[1]) or
-                (precompute == 'auto' and y.shape[1] > 1)):
+        if (not hasattr(precompute, "__array__")) and (
+            (precompute is True)
+            or (precompute == "auto" and X.shape[0] > X.shape[1])
+            or (precompute == "auto" and y.shape[1] > 1)
+        ):
             precompute = np.dot(X.T, X)
 
         return precompute
@@ -940,8 +1080,9 @@ def _fit(self, X, y, max_iter, alpha, fit_path, Xy=None):
         """Auxiliary method to fit the model using X, y as training data"""
         n_features = X.shape[1]
 
-        X, y, X_offset, y_offset, X_scale = self._preprocess_data(
-            X, y, self.fit_intercept, self.normalize, self.copy_X)
+        X, y, X_offset, y_offset, X_scale = _preprocess_data(
+            X, y, fit_intercept=self.fit_intercept, copy=self.copy_X
+        )
 
         if y.ndim == 1:
             y = y[:, np.newaxis]
@@ -952,7 +1093,7 @@ def _fit(self, X, y, max_iter, alpha, fit_path, Xy=None):
 
         self.alphas_ = []
         self.n_iter_ = []
-        self.coef_ = np.empty((n_targets, n_features))
+        self.coef_ = np.empty((n_targets, n_features), dtype=X.dtype)
 
         if fit_path:
             self.active_ = []
@@ -960,11 +1101,21 @@ def _fit(self, X, y, max_iter, alpha, fit_path, Xy=None):
             for k in range(n_targets):
                 this_Xy = None if Xy is None else Xy[:, k]
                 alphas, active, coef_path, n_iter_ = lars_path(
-                    X, y[:, k], Gram=Gram, Xy=this_Xy, copy_X=self.copy_X,
-                    copy_Gram=True, alpha_min=alpha, method=self.method,
-                    verbose=max(0, self.verbose - 1), max_iter=max_iter,
-                    eps=self.eps, return_path=True,
-                    return_n_iter=True, positive=self.positive)
+                    X,
+                    y[:, k],
+                    Gram=Gram,
+                    Xy=this_Xy,
+                    copy_X=self.copy_X,
+                    copy_Gram=True,
+                    alpha_min=alpha,
+                    method=self.method,
+                    verbose=max(0, self.verbose - 1),
+                    max_iter=max_iter,
+                    eps=self.eps,
+                    return_path=True,
+                    return_n_iter=True,
+                    positive=self.positive,
+                )
                 self.alphas_.append(alphas)
                 self.active_.append(active)
                 self.n_iter_.append(n_iter_)
@@ -973,18 +1124,29 @@ def _fit(self, X, y, max_iter, alpha, fit_path, Xy=None):
 
             if n_targets == 1:
                 self.alphas_, self.active_, self.coef_path_, self.coef_ = [
-                    a[0] for a in (self.alphas_, self.active_, self.coef_path_,
-                                   self.coef_)]
+                    a[0]
+                    for a in (self.alphas_, self.active_, self.coef_path_, self.coef_)
+                ]
                 self.n_iter_ = self.n_iter_[0]
         else:
             for k in range(n_targets):
                 this_Xy = None if Xy is None else Xy[:, k]
                 alphas, _, self.coef_[k], n_iter_ = lars_path(
-                    X, y[:, k], Gram=Gram, Xy=this_Xy, copy_X=self.copy_X,
-                    copy_Gram=True, alpha_min=alpha, method=self.method,
-                    verbose=max(0, self.verbose - 1), max_iter=max_iter,
-                    eps=self.eps, return_path=False, return_n_iter=True,
-                    positive=self.positive)
+                    X,
+                    y[:, k],
+                    Gram=Gram,
+                    Xy=this_Xy,
+                    copy_X=self.copy_X,
+                    copy_Gram=True,
+                    alpha_min=alpha,
+                    method=self.method,
+                    verbose=max(0, self.verbose - 1),
+                    max_iter=max_iter,
+                    eps=self.eps,
+                    return_path=False,
+                    return_n_iter=True,
+                    positive=self.positive,
+                )
                 self.alphas_.append(alphas)
                 self.n_iter_.append(n_iter_)
             if n_targets == 1:
@@ -994,6 +1156,7 @@ def _fit(self, X, y, max_iter, alpha, fit_path, Xy=None):
         self._set_intercept(X_offset, y_offset, X_scale)
         return self
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y, Xy=None):
         """Fit the model using X, y as training data.
 
@@ -1005,7 +1168,7 @@ def fit(self, X, y, Xy=None):
         y : array-like of shape (n_samples,) or (n_samples, n_targets)
             Target values.
 
-        Xy : array-like of shape (n_samples,) or (n_samples, n_targets), \
+        Xy : array-like of shape (n_features,) or (n_features, n_targets), \
                 default=None
             Xy = np.dot(X.T, y) that can be precomputed. It is useful
             only when the Gram matrix is precomputed.
@@ -1013,13 +1176,15 @@ def fit(self, X, y, Xy=None):
         Returns
         -------
         self : object
-            returns an instance of self.
+            Returns an instance of self.
         """
-        X, y = self._validate_data(X, y, y_numeric=True, multi_output=True)
+        X, y = validate_data(
+            self, X, y, force_writeable=True, y_numeric=True, multi_output=True
+        )
 
-        alpha = getattr(self, 'alpha', 0.)
-        if hasattr(self, 'n_nonzero_coefs'):
-            alpha = 0.  # n_nonzero_coefs parametrization takes priority
+        alpha = getattr(self, "alpha", 0.0)
+        if hasattr(self, "n_nonzero_coefs"):
+            alpha = 0.0  # n_nonzero_coefs parametrization takes priority
             max_iter = self.n_nonzero_coefs
         else:
             max_iter = self.max_iter
@@ -1030,14 +1195,20 @@ def fit(self, X, y, Xy=None):
             noise = rng.uniform(high=self.jitter, size=len(y))
             y = y + noise
 
-        self._fit(X, y, max_iter=max_iter, alpha=alpha, fit_path=self.fit_path,
-                  Xy=Xy)
+        self._fit(
+            X,
+            y,
+            max_iter=max_iter,
+            alpha=alpha,
+            fit_path=self.fit_path,
+            Xy=Xy,
+        )
 
         return self
 
 
 class LassoLars(Lars):
-    """Lasso model fit with Least Angle Regression a.k.a. Lars
+    """Lasso model fit with Least Angle Regression a.k.a. Lars.
 
     It is a Linear Model trained with an L1 prior as regularizer.
 
@@ -1057,21 +1228,13 @@ class LassoLars(Lars):
         should prefer the LinearRegression object.
 
     fit_intercept : bool, default=True
-        whether to calculate the intercept for this model. If set
+        Whether to calculate the intercept for this model. If set
         to false, no intercept will be used in calculations
         (i.e. data is expected to be centered).
 
     verbose : bool or int, default=False
         Sets the verbosity amount.
 
-    normalize : bool, default=True
-        This parameter is ignored when ``fit_intercept`` is set to False.
-        If True, the regressors X will be normalized before regression by
-        subtracting the mean and dividing by the l2-norm.
-        If you wish to standardize, please use
-        :class:`~sklearn.preprocessing.StandardScaler` before calling ``fit``
-        on an estimator with ``normalize=False``.
-
     precompute : bool, 'auto' or array-like, default='auto'
         Whether to use a precomputed Gram matrix to speed up
         calculations. If set to ``'auto'`` let us decide. The Gram
@@ -1150,6 +1313,31 @@ class LassoLars(Lars):
         The number of iterations taken by lars_path to find the
         grid of alphas for each target.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    lars_path : Compute Least Angle Regression or Lasso
+        path using LARS algorithm.
+    lasso_path : Compute Lasso path with coordinate descent.
+    Lasso : Linear Model trained with L1 prior as
+        regularizer (aka the Lasso).
+    LassoCV : Lasso linear model with iterative fitting
+        along a regularization path.
+    LassoLarsCV: Cross-validated Lasso, using the LARS algorithm.
+    LassoLarsIC : Lasso model fit with Lars using BIC
+        or AIC for model selection.
+    sklearn.decomposition.sparse_encode : Sparse coding.
+
     Examples
     --------
     >>> from sklearn import linear_model
@@ -1157,31 +1345,38 @@ class LassoLars(Lars):
     >>> reg.fit([[-1, 1], [0, 0], [1, 1]], [-1, 0, -1])
     LassoLars(alpha=0.01)
     >>> print(reg.coef_)
-    [ 0.         -0.963257...]
-
-    See Also
-    --------
-    lars_path
-    lasso_path
-    Lasso
-    LassoCV
-    LassoLarsCV
-    LassoLarsIC
-    sklearn.decomposition.sparse_encode
-
+    [ 0.         -0.955]
     """
-    method = 'lasso'
 
-    @_deprecate_positional_args
-    def __init__(self, alpha=1.0, *, fit_intercept=True, verbose=False,
-                 normalize=True, precompute='auto', max_iter=500,
-                 eps=np.finfo(float).eps, copy_X=True, fit_path=True,
-                 positive=False, jitter=None, random_state=None):
+    _parameter_constraints: dict = {
+        **Lars._parameter_constraints,
+        "alpha": [Interval(Real, 0, None, closed="left")],
+        "max_iter": [Interval(Integral, 0, None, closed="left")],
+        "positive": ["boolean"],
+    }
+    _parameter_constraints.pop("n_nonzero_coefs")
+
+    method = "lasso"
+
+    def __init__(
+        self,
+        alpha=1.0,
+        *,
+        fit_intercept=True,
+        verbose=False,
+        precompute="auto",
+        max_iter=500,
+        eps=np.finfo(float).eps,
+        copy_X=True,
+        fit_path=True,
+        positive=False,
+        jitter=None,
+        random_state=None,
+    ):
         self.alpha = alpha
         self.fit_intercept = fit_intercept
         self.max_iter = max_iter
         self.verbose = verbose
-        self.normalize = normalize
         self.positive = positive
         self.precompute = precompute
         self.copy_X = copy_X
@@ -1194,16 +1389,27 @@ def __init__(self, alpha=1.0, *, fit_intercept=True, verbose=False,
 ###############################################################################
 # Cross-validated estimator classes
 
+
 def _check_copy_and_writeable(array, copy=False):
     if copy or not array.flags.writeable:
         return array.copy()
     return array
 
 
-def _lars_path_residues(X_train, y_train, X_test, y_test, Gram=None,
-                        copy=True, method='lars', verbose=False,
-                        fit_intercept=True, normalize=True, max_iter=500,
-                        eps=np.finfo(float).eps, positive=False):
+def _lars_path_residues(
+    X_train,
+    y_train,
+    X_test,
+    y_test,
+    Gram=None,
+    copy=True,
+    method="lar",
+    verbose=False,
+    fit_intercept=True,
+    max_iter=500,
+    eps=np.finfo(float).eps,
+    positive=False,
+):
     """Compute the residues on left-out data for a full LARS path
 
     Parameters
@@ -1249,14 +1455,6 @@ def _lars_path_residues(X_train, y_train, X_test, y_test, Gram=None,
         'lasso' for expected small values of alpha in the doc of LassoLarsCV
         and LassoLarsIC.
 
-    normalize : bool, default=True
-        This parameter is ignored when ``fit_intercept`` is set to False.
-        If True, the regressors X will be normalized before regression by
-        subtracting the mean and dividing by the l2-norm.
-        If you wish to standardize, please use
-        :class:`~sklearn.preprocessing.StandardScaler` before calling ``fit``
-        on an estimator with ``normalize=False``.
-
     max_iter : int, default=500
         Maximum number of iterations to perform.
 
@@ -1298,17 +1496,18 @@ def _lars_path_residues(X_train, y_train, X_test, y_test, Gram=None,
         y_test = as_float_array(y_test, copy=False)
         y_test -= y_mean
 
-    if normalize:
-        norms = np.sqrt(np.sum(X_train ** 2, axis=0))
-        nonzeros = np.flatnonzero(norms)
-        X_train[:, nonzeros] /= norms[nonzeros]
-
     alphas, active, coefs = lars_path(
-        X_train, y_train, Gram=Gram, copy_X=False, copy_Gram=False,
-        method=method, verbose=max(0, verbose - 1), max_iter=max_iter, eps=eps,
-        positive=positive)
-    if normalize:
-        coefs[nonzeros] /= norms[nonzeros][:, np.newaxis]
+        X_train,
+        y_train,
+        Gram=Gram,
+        copy_X=False,
+        copy_Gram=False,
+        method=method,
+        verbose=max(0, verbose - 1),
+        max_iter=max_iter,
+        eps=eps,
+        positive=positive,
+    )
     residues = np.dot(X_test, coefs) - y_test[:, np.newaxis]
     return alphas, active, coefs, residues.T
 
@@ -1323,7 +1522,7 @@ class LarsCV(Lars):
     Parameters
     ----------
     fit_intercept : bool, default=True
-        whether to calculate the intercept for this model. If set
+        Whether to calculate the intercept for this model. If set
         to false, no intercept will be used in calculations
         (i.e. data is expected to be centered).
 
@@ -1333,14 +1532,6 @@ class LarsCV(Lars):
     max_iter : int, default=500
         Maximum number of iterations to perform.
 
-    normalize : bool, default=True
-        This parameter is ignored when ``fit_intercept`` is set to False.
-        If True, the regressors X will be normalized before regression by
-        subtracting the mean and dividing by the l2-norm.
-        If you wish to standardize, please use
-        :class:`~sklearn.preprocessing.StandardScaler` before calling ``fit``
-        on an estimator with ``normalize=False``.
-
     precompute : bool, 'auto' or array-like , default='auto'
         Whether to use a precomputed Gram matrix to speed up
         calculations. If set to ``'auto'`` let us decide. The Gram matrix
@@ -1355,7 +1546,7 @@ class LarsCV(Lars):
         - :term:`CV splitter`,
         - An iterable yielding (train, test) splits as arrays of indices.
 
-        For integer/None inputs, :class:`KFold` is used.
+        For integer/None inputs, :class:`~sklearn.model_selection.KFold` is used.
 
         Refer :ref:`User Guide ` for the various
         cross-validation strategies that can be used here.
@@ -1365,7 +1556,7 @@ class LarsCV(Lars):
 
     max_n_alphas : int, default=1000
         The maximum number of points on the path used to compute the
-        residuals in the cross-validation
+        residuals in the cross-validation.
 
     n_jobs : int or None, default=None
         Number of CPUs to use during the cross validation.
@@ -1414,6 +1605,36 @@ class LarsCV(Lars):
     n_iter_ : array-like or int
         the number of iterations run by Lars with the optimal alpha.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    lars_path : Compute Least Angle Regression or Lasso
+        path using LARS algorithm.
+    lasso_path : Compute Lasso path with coordinate descent.
+    Lasso : Linear Model trained with L1 prior as
+        regularizer (aka the Lasso).
+    LassoCV : Lasso linear model with iterative fitting
+        along a regularization path.
+    LassoLars : Lasso model fit with Least Angle Regression a.k.a. Lars.
+    LassoLarsIC : Lasso model fit with Lars using BIC
+        or AIC for model selection.
+    sklearn.decomposition.sparse_encode : Sparse coding.
+
+    Notes
+    -----
+    In `fit`, once the best parameter `alpha` is found through
+    cross-validation, the model is fit again using the entire training set.
+
     Examples
     --------
     >>> from sklearn.linear_model import LarsCV
@@ -1421,38 +1642,60 @@ class LarsCV(Lars):
     >>> X, y = make_regression(n_samples=200, noise=4.0, random_state=0)
     >>> reg = LarsCV(cv=5).fit(X, y)
     >>> reg.score(X, y)
-    0.9996...
+    0.9996
     >>> reg.alpha_
-    0.0254...
+    np.float64(0.2961)
     >>> reg.predict(X[:1,])
-    array([154.0842...])
-
-    See Also
-    --------
-    lars_path, LassoLars, LassoLarsCV
+    array([154.3996])
     """
 
+    _parameter_constraints: dict = {
+        **Lars._parameter_constraints,
+        "max_iter": [Interval(Integral, 0, None, closed="left")],
+        "cv": ["cv_object"],
+        "max_n_alphas": [Interval(Integral, 1, None, closed="left")],
+        "n_jobs": [Integral, None],
+    }
+
+    for parameter in ["n_nonzero_coefs", "jitter", "fit_path", "random_state"]:
+        _parameter_constraints.pop(parameter)
+
     method = "lar"
 
-    @_deprecate_positional_args
-    def __init__(self, *, fit_intercept=True, verbose=False, max_iter=500,
-                 normalize=True, precompute='auto', cv=None,
-                 max_n_alphas=1000, n_jobs=None, eps=np.finfo(float).eps,
-                 copy_X=True):
+    def __init__(
+        self,
+        *,
+        fit_intercept=True,
+        verbose=False,
+        max_iter=500,
+        precompute="auto",
+        cv=None,
+        max_n_alphas=1000,
+        n_jobs=None,
+        eps=np.finfo(float).eps,
+        copy_X=True,
+    ):
         self.max_iter = max_iter
         self.cv = cv
         self.max_n_alphas = max_n_alphas
         self.n_jobs = n_jobs
-        super().__init__(fit_intercept=fit_intercept,
-                         verbose=verbose, normalize=normalize,
-                         precompute=precompute,
-                         n_nonzero_coefs=500,
-                         eps=eps, copy_X=copy_X, fit_path=True)
+        super().__init__(
+            fit_intercept=fit_intercept,
+            verbose=verbose,
+            precompute=precompute,
+            n_nonzero_coefs=500,
+            eps=eps,
+            copy_X=copy_X,
+            fit_path=True,
+        )
 
-    def _more_tags(self):
-        return {'multioutput': False}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.target_tags.multi_output = False
+        return tags
 
-    def fit(self, X, y):
+    @_fit_context(prefer_skip_nested_validation=True)
+    def fit(self, X, y, **params):
         """Fit the model using X, y as training data.
 
         Parameters
@@ -1463,34 +1706,62 @@ def fit(self, X, y):
         y : array-like of shape (n_samples,)
             Target values.
 
+        **params : dict, default=None
+            Parameters to be passed to the CV splitter.
+
+            .. versionadded:: 1.4
+                Only available if `enable_metadata_routing=True`,
+                which can be set by using
+                ``sklearn.set_config(enable_metadata_routing=True)``.
+                See :ref:`Metadata Routing User Guide ` for
+                more details.
+
         Returns
         -------
         self : object
-            returns an instance of self.
+            Returns an instance of self.
         """
-        X, y = self._validate_data(X, y, y_numeric=True)
+        _raise_for_params(params, self, "fit")
+
+        X, y = validate_data(self, X, y, force_writeable=True, y_numeric=True)
         X = as_float_array(X, copy=self.copy_X)
         y = as_float_array(y, copy=self.copy_X)
 
         # init cross-validation generator
         cv = check_cv(self.cv, classifier=False)
 
+        if _routing_enabled():
+            routed_params = process_routing(self, "fit", **params)
+        else:
+            routed_params = Bunch(splitter=Bunch(split={}))
+
         # As we use cross-validation, the Gram matrix is not precomputed here
         Gram = self.precompute
-        if hasattr(Gram, '__array__'):
-            warnings.warn('Parameter "precompute" cannot be an array in '
-                          '%s. Automatically switch to "auto" instead.'
-                          % self.__class__.__name__)
-            Gram = 'auto'
+        if hasattr(Gram, "__array__"):
+            warnings.warn(
+                'Parameter "precompute" cannot be an array in '
+                '%s. Automatically switch to "auto" instead.' % self.__class__.__name__
+            )
+            Gram = "auto"
 
         cv_paths = Parallel(n_jobs=self.n_jobs, verbose=self.verbose)(
             delayed(_lars_path_residues)(
-                X[train], y[train], X[test], y[test], Gram=Gram, copy=False,
-                method=self.method, verbose=max(0, self.verbose - 1),
-                normalize=self.normalize, fit_intercept=self.fit_intercept,
-                max_iter=self.max_iter, eps=self.eps, positive=self.positive)
-            for train, test in cv.split(X, y))
-        all_alphas = np.concatenate(list(zip(*cv_paths))[0])
+                X[train],
+                y[train],
+                X[test],
+                y[test],
+                Gram=Gram,
+                copy=False,
+                method=self.method,
+                verbose=max(0, self.verbose - 1),
+                fit_intercept=self.fit_intercept,
+                max_iter=self.max_iter,
+                eps=self.eps,
+                positive=self.positive,
+            )
+            for train, test in cv.split(X, y, **routed_params.splitter.split)
+        )
+        all_alphas = np.concatenate(next(zip(*cv_paths)))
         # Unique also sorts
         all_alphas = np.unique(all_alphas)
         # Take at most max_n_alphas values
@@ -1507,9 +1778,7 @@ def fit(self, X, y):
             if alphas[-1] != all_alphas[-1]:
                 alphas = np.r_[alphas, all_alphas[-1]]
                 residues = np.r_[residues, residues[-1, np.newaxis]]
-            this_residues = interpolate.interp1d(alphas,
-                                                 residues,
-                                                 axis=0)(all_alphas)
+            this_residues = interpolate.interp1d(alphas, residues, axis=0)(all_alphas)
             this_residues **= 2
             mse_path[:, index] = np.mean(this_residues, axis=-1)
 
@@ -1525,13 +1794,39 @@ def fit(self, X, y):
         self.cv_alphas_ = all_alphas
         self.mse_path_ = mse_path
 
-        # Now compute the full model
+        # Now compute the full model using best_alpha
         # it will call a lasso internally when self if LassoLarsCV
         # as self.method == 'lasso'
-        self._fit(X, y, max_iter=self.max_iter, alpha=best_alpha,
-                  Xy=None, fit_path=True)
+        self._fit(
+            X,
+            y,
+            max_iter=self.max_iter,
+            alpha=best_alpha,
+            Xy=None,
+            fit_path=True,
+        )
         return self
 
+    def get_metadata_routing(self):
+        """Get metadata routing of this object.
+
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
+
+        .. versionadded:: 1.4
+
+        Returns
+        -------
+        routing : MetadataRouter
+            A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
+            routing information.
+        """
+        router = MetadataRouter(owner=self.__class__.__name__).add(
+            splitter=check_cv(self.cv),
+            method_mapping=MethodMapping().add(caller="fit", callee="split"),
+        )
+        return router
+
 
 class LassoLarsCV(LarsCV):
     """Cross-validated Lasso, using the LARS algorithm.
@@ -1547,7 +1842,7 @@ class LassoLarsCV(LarsCV):
     Parameters
     ----------
     fit_intercept : bool, default=True
-        whether to calculate the intercept for this model. If set
+        Whether to calculate the intercept for this model. If set
         to false, no intercept will be used in calculations
         (i.e. data is expected to be centered).
 
@@ -1557,14 +1852,6 @@ class LassoLarsCV(LarsCV):
     max_iter : int, default=500
         Maximum number of iterations to perform.
 
-    normalize : bool, default=True
-        This parameter is ignored when ``fit_intercept`` is set to False.
-        If True, the regressors X will be normalized before regression by
-        subtracting the mean and dividing by the l2-norm.
-        If you wish to standardize, please use
-        :class:`~sklearn.preprocessing.StandardScaler` before calling ``fit``
-        on an estimator with ``normalize=False``.
-
     precompute : bool or 'auto' , default='auto'
         Whether to use a precomputed Gram matrix to speed up
         calculations. If set to ``'auto'`` let us decide. The Gram matrix
@@ -1579,7 +1866,7 @@ class LassoLarsCV(LarsCV):
         - :term:`CV splitter`,
         - An iterable yielding (train, test) splits as arrays of indices.
 
-        For integer/None inputs, :class:`KFold` is used.
+        For integer/None inputs, :class:`~sklearn.model_selection.KFold` is used.
 
         Refer :ref:`User Guide ` for the various
         cross-validation strategies that can be used here.
@@ -1589,7 +1876,7 @@ class LassoLarsCV(LarsCV):
 
     max_n_alphas : int, default=1000
         The maximum number of points on the path used to compute the
-        residuals in the cross-validation
+        residuals in the cross-validation.
 
     n_jobs : int or None, default=None
         Number of CPUs to use during the cross validation.
@@ -1649,6 +1936,47 @@ class LassoLarsCV(LarsCV):
     active_ : list of int
         Indices of active variables at the end of the path.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    lars_path : Compute Least Angle Regression or Lasso
+        path using LARS algorithm.
+    lasso_path : Compute Lasso path with coordinate descent.
+    Lasso : Linear Model trained with L1 prior as
+        regularizer (aka the Lasso).
+    LassoCV : Lasso linear model with iterative fitting
+        along a regularization path.
+    LassoLars : Lasso model fit with Least Angle Regression a.k.a. Lars.
+    LassoLarsIC : Lasso model fit with Lars using BIC
+        or AIC for model selection.
+    sklearn.decomposition.sparse_encode : Sparse coding.
+
+    Notes
+    -----
+    The object solves the same problem as the
+    :class:`~sklearn.linear_model.LassoCV` object. However, unlike the
+    :class:`~sklearn.linear_model.LassoCV`, it find the relevant alphas values
+    by itself. In general, because of this property, it will be more stable.
+    However, it is more fragile to heavily multicollinear datasets.
+
+    It is more efficient than the :class:`~sklearn.linear_model.LassoCV` if
+    only a small number of features are selected compared to the total number,
+    for instance if there are very few samples compared to the number of
+    features.
+
+    In `fit`, once the best parameter `alpha` is found through
+    cross-validation, the model is fit again using the entire training set.
+
     Examples
     --------
     >>> from sklearn.linear_model import LassoLarsCV
@@ -1656,40 +1984,37 @@ class LassoLarsCV(LarsCV):
     >>> X, y = make_regression(noise=4.0, random_state=0)
     >>> reg = LassoLarsCV(cv=5).fit(X, y)
     >>> reg.score(X, y)
-    0.9992...
+    0.9993
     >>> reg.alpha_
-    0.0484...
+    np.float64(0.3972)
     >>> reg.predict(X[:1,])
-    array([-77.8723...])
-
-    Notes
-    -----
-
-    The object solves the same problem as the LassoCV object. However,
-    unlike the LassoCV, it find the relevant alphas values by itself.
-    In general, because of this property, it will be more stable.
-    However, it is more fragile to heavily multicollinear datasets.
-
-    It is more efficient than the LassoCV if only a small number of
-    features are selected compared to the total number, for instance if
-    there are very few samples compared to the number of features.
-
-    See Also
-    --------
-    lars_path, LassoLars, LarsCV, LassoCV
+    array([-78.4831])
     """
 
-    method = 'lasso'
-
-    @_deprecate_positional_args
-    def __init__(self, *, fit_intercept=True, verbose=False, max_iter=500,
-                 normalize=True, precompute='auto', cv=None,
-                 max_n_alphas=1000, n_jobs=None, eps=np.finfo(float).eps,
-                 copy_X=True, positive=False):
+    _parameter_constraints = {
+        **LarsCV._parameter_constraints,
+        "positive": ["boolean"],
+    }
+
+    method = "lasso"
+
+    def __init__(
+        self,
+        *,
+        fit_intercept=True,
+        verbose=False,
+        max_iter=500,
+        precompute="auto",
+        cv=None,
+        max_n_alphas=1000,
+        n_jobs=None,
+        eps=np.finfo(float).eps,
+        copy_X=True,
+        positive=False,
+    ):
         self.fit_intercept = fit_intercept
         self.verbose = verbose
         self.max_iter = max_iter
-        self.normalize = normalize
         self.precompute = precompute
         self.cv = cv
         self.max_n_alphas = max_n_alphas
@@ -1702,41 +2027,33 @@ def __init__(self, *, fit_intercept=True, verbose=False, max_iter=500,
 
 
 class LassoLarsIC(LassoLars):
-    """Lasso model fit with Lars using BIC or AIC for model selection
+    """Lasso model fit with Lars using BIC or AIC for model selection.
 
     The optimization objective for Lasso is::
 
     (1 / (2 * n_samples)) * ||y - Xw||^2_2 + alpha * ||w||_1
 
-    AIC is the Akaike information criterion and BIC is the Bayes
-    Information criterion. Such criteria are useful to select the value
+    AIC is the Akaike information criterion [2]_ and BIC is the Bayes
+    Information criterion [3]_. Such criteria are useful to select the value
     of the regularization parameter by making a trade-off between the
     goodness of fit and the complexity of the model. A good model should
     explain well the data while being simple.
 
-    Read more in the :ref:`User Guide `.
+    Read more in the :ref:`User Guide `.
 
     Parameters
     ----------
-    criterion : {'bic' , 'aic'}, default='aic'
+    criterion : {'aic', 'bic'}, default='aic'
         The type of criterion to use.
 
     fit_intercept : bool, default=True
-        whether to calculate the intercept for this model. If set
+        Whether to calculate the intercept for this model. If set
         to false, no intercept will be used in calculations
         (i.e. data is expected to be centered).
 
     verbose : bool or int, default=False
         Sets the verbosity amount.
 
-    normalize : bool, default=True
-        This parameter is ignored when ``fit_intercept`` is set to False.
-        If True, the regressors X will be normalized before regression by
-        subtracting the mean and dividing by the l2-norm.
-        If you wish to standardize, please use
-        :class:`~sklearn.preprocessing.StandardScaler` before calling ``fit``
-        on an estimator with ``normalize=False``.
-
     precompute : bool, 'auto' or array-like, default='auto'
         Whether to use a precomputed Gram matrix to speed up
         calculations. If set to ``'auto'`` let us decide. The Gram
@@ -1768,6 +2085,13 @@ class LassoLarsIC(LassoLars):
         As a consequence using LassoLarsIC only makes sense for problems where
         a sparse solution is expected and/or reached.
 
+    noise_variance : float, default=None
+        The estimated noise variance of the data. If `None`, an unbiased
+        estimate is computed by an OLS model. However, it is only possible
+        in the case where `n_samples > n_features + fit_intercept`.
+
+        .. versionadded:: 1.1
+
     Attributes
     ----------
     coef_ : array-like of shape (n_features,)
@@ -1792,62 +2116,119 @@ class LassoLarsIC(LassoLars):
     criterion_ : array-like of shape (n_alphas,)
         The value of the information criteria ('aic', 'bic') across all
         alphas. The alpha which has the smallest information criterion is
-        chosen. This value is larger by a factor of ``n_samples`` compared to
-        Eqns. 2.15 and 2.16 in (Zou et al, 2007).
+        chosen, as specified in [1]_.
 
+    noise_variance_ : float
+        The estimated noise variance from the data used to compute the
+        criterion.
 
-    Examples
+        .. versionadded:: 1.1
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
     --------
-    >>> from sklearn import linear_model
-    >>> reg = linear_model.LassoLarsIC(criterion='bic')
-    >>> reg.fit([[-1, 1], [0, 0], [1, 1]], [-1.1111, 0, -1.1111])
-    LassoLarsIC(criterion='bic')
-    >>> print(reg.coef_)
-    [ 0.  -1.11...]
+    lars_path : Compute Least Angle Regression or Lasso
+        path using LARS algorithm.
+    lasso_path : Compute Lasso path with coordinate descent.
+    Lasso : Linear Model trained with L1 prior as
+        regularizer (aka the Lasso).
+    LassoCV : Lasso linear model with iterative fitting
+        along a regularization path.
+    LassoLars : Lasso model fit with Least Angle Regression a.k.a. Lars.
+    LassoLarsCV: Cross-validated Lasso, using the LARS algorithm.
+    sklearn.decomposition.sparse_encode : Sparse coding.
 
     Notes
     -----
-    The estimation of the number of degrees of freedom is given by:
+    The number of degrees of freedom is computed as in [1]_.
 
-    "On the degrees of freedom of the lasso"
-    Hui Zou, Trevor Hastie, and Robert Tibshirani
-    Ann. Statist. Volume 35, Number 5 (2007), 2173-2192.
+    To have more details regarding the mathematical formulation of the
+    AIC and BIC criteria, please refer to :ref:`User Guide `.
 
-    https://en.wikipedia.org/wiki/Akaike_information_criterion
-    https://en.wikipedia.org/wiki/Bayesian_information_criterion
+    References
+    ----------
+    .. [1] :arxiv:`Zou, Hui, Trevor Hastie, and Robert Tibshirani.
+            "On the degrees of freedom of the lasso."
+            The Annals of Statistics 35.5 (2007): 2173-2192.
+            <0712.0881>`
 
-    See Also
+    .. [2] `Wikipedia entry on the Akaike information criterion
+            `_
+
+    .. [3] `Wikipedia entry on the Bayesian information criterion
+            `_
+
+    Examples
     --------
-    lars_path, LassoLars, LassoLarsCV
+    >>> from sklearn import linear_model
+    >>> reg = linear_model.LassoLarsIC(criterion='bic')
+    >>> X = [[-2, 2], [-1, 1], [0, 0], [1, 1], [2, 2]]
+    >>> y = [-2.2222, -1.1111, 0, -1.1111, -2.2222]
+    >>> reg.fit(X, y)
+    LassoLarsIC(criterion='bic')
+    >>> print(reg.coef_)
+    [ 0.  -1.11]
     """
-    @_deprecate_positional_args
-    def __init__(self, criterion='aic', *, fit_intercept=True, verbose=False,
-                 normalize=True, precompute='auto', max_iter=500,
-                 eps=np.finfo(float).eps, copy_X=True, positive=False):
+
+    _parameter_constraints: dict = {
+        **LassoLars._parameter_constraints,
+        "criterion": [StrOptions({"aic", "bic"})],
+        "noise_variance": [Interval(Real, 0, None, closed="left"), None],
+    }
+
+    for parameter in ["jitter", "fit_path", "alpha", "random_state"]:
+        _parameter_constraints.pop(parameter)
+
+    def __init__(
+        self,
+        criterion="aic",
+        *,
+        fit_intercept=True,
+        verbose=False,
+        precompute="auto",
+        max_iter=500,
+        eps=np.finfo(float).eps,
+        copy_X=True,
+        positive=False,
+        noise_variance=None,
+    ):
         self.criterion = criterion
         self.fit_intercept = fit_intercept
         self.positive = positive
         self.max_iter = max_iter
         self.verbose = verbose
-        self.normalize = normalize
         self.copy_X = copy_X
         self.precompute = precompute
         self.eps = eps
         self.fit_path = True
+        self.noise_variance = noise_variance
 
-    def _more_tags(self):
-        return {'multioutput': False}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.target_tags.multi_output = False
+        return tags
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y, copy_X=None):
         """Fit the model using X, y as training data.
 
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
-            training data.
+            Training data.
 
         y : array-like of shape (n_samples,)
-            target values. Will be cast to X's dtype if necessary
+            Target values. Will be cast to X's dtype if necessary.
 
         copy_X : bool, default=None
             If provided, this parameter will override the choice
@@ -1857,36 +2238,47 @@ def fit(self, X, y, copy_X=None):
         Returns
         -------
         self : object
-            returns an instance of self.
+            Returns an instance of self.
         """
         if copy_X is None:
             copy_X = self.copy_X
-        X, y = self._validate_data(X, y, y_numeric=True)
+        X, y = validate_data(self, X, y, force_writeable=True, y_numeric=True)
 
-        X, y, Xmean, ymean, Xstd = LinearModel._preprocess_data(
-            X, y, self.fit_intercept, self.normalize, copy_X)
+        X, y, Xmean, ymean, Xstd = _preprocess_data(
+            X, y, fit_intercept=self.fit_intercept, copy=copy_X
+        )
 
         Gram = self.precompute
 
         alphas_, _, coef_path_, self.n_iter_ = lars_path(
-            X, y, Gram=Gram, copy_X=copy_X, copy_Gram=True, alpha_min=0.0,
-            method='lasso', verbose=self.verbose, max_iter=self.max_iter,
-            eps=self.eps, return_n_iter=True, positive=self.positive)
+            X,
+            y,
+            Gram=Gram,
+            copy_X=copy_X,
+            copy_Gram=True,
+            alpha_min=0.0,
+            method="lasso",
+            verbose=self.verbose,
+            max_iter=self.max_iter,
+            eps=self.eps,
+            return_n_iter=True,
+            positive=self.positive,
+        )
 
         n_samples = X.shape[0]
 
-        if self.criterion == 'aic':
-            K = 2  # AIC
-        elif self.criterion == 'bic':
-            K = log(n_samples)  # BIC
+        if self.criterion == "aic":
+            criterion_factor = 2
+        elif self.criterion == "bic":
+            criterion_factor = log(n_samples)
         else:
-            raise ValueError('criterion should be either bic or aic')
-
-        R = y[:, np.newaxis] - np.dot(X, coef_path_)  # residuals
-        mean_squared_error = np.mean(R ** 2, axis=0)
-        sigma2 = np.var(y)
+            raise ValueError(
+                f"criterion should be either bic or aic, got {self.criterion!r}"
+            )
 
-        df = np.zeros(coef_path_.shape[1], dtype=int)  # Degrees of freedom
+        residuals = y[:, np.newaxis] - np.dot(X, coef_path_)
+        residuals_sum_squares = np.sum(residuals**2, axis=0)
+        degrees_of_freedom = np.zeros(coef_path_.shape[1], dtype=int)
         for k, coef in enumerate(coef_path_.T):
             mask = np.abs(coef) > np.finfo(coef.dtype).eps
             if not np.any(mask):
@@ -1894,15 +2286,61 @@ def fit(self, X, y, copy_X=None):
             # get the number of degrees of freedom equal to:
             # Xc = X[:, mask]
             # Trace(Xc * inv(Xc.T, Xc) * Xc.T) ie the number of non-zero coefs
-            df[k] = np.sum(mask)
+            degrees_of_freedom[k] = np.sum(mask)
 
         self.alphas_ = alphas_
-        eps64 = np.finfo('float64').eps
-        self.criterion_ = (n_samples * mean_squared_error / (sigma2 + eps64) +
-                           K * df)  # Eqns. 2.15--16 in (Zou et al, 2007)
+
+        if self.noise_variance is None:
+            self.noise_variance_ = self._estimate_noise_variance(
+                X, y, positive=self.positive
+            )
+        else:
+            self.noise_variance_ = self.noise_variance
+
+        self.criterion_ = (
+            n_samples * np.log(2 * np.pi * self.noise_variance_)
+            + residuals_sum_squares / self.noise_variance_
+            + criterion_factor * degrees_of_freedom
+        )
         n_best = np.argmin(self.criterion_)
 
         self.alpha_ = alphas_[n_best]
         self.coef_ = coef_path_[:, n_best]
         self._set_intercept(Xmean, ymean, Xstd)
         return self
+
+    def _estimate_noise_variance(self, X, y, positive):
+        """Compute an estimate of the variance with an OLS model.
+
+        Parameters
+        ----------
+        X : ndarray of shape (n_samples, n_features)
+            Data to be fitted by the OLS model. We expect the data to be
+            centered.
+
+        y : ndarray of shape (n_samples,)
+            Associated target.
+
+        positive : bool, default=False
+            Restrict coefficients to be >= 0. This should be inline with
+            the `positive` parameter from `LassoLarsIC`.
+
+        Returns
+        -------
+        noise_variance : float
+            An estimator of the noise variance of an OLS model.
+        """
+        if X.shape[0] <= X.shape[1] + self.fit_intercept:
+            raise ValueError(
+                f"You are using {self.__class__.__name__} in the case where the number "
+                "of samples is smaller than the number of features. In this setting, "
+                "getting a good estimate for the variance of the noise is not "
+                "possible. Provide an estimate of the noise variance in the "
+                "constructor."
+            )
+        # X and y are already centered and we don't need to fit with an intercept
+        ols_model = LinearRegression(positive=positive, fit_intercept=False)
+        y_pred = ols_model.fit(X, y).predict(X)
+        return np.sum((y - y_pred) ** 2) / (
+            X.shape[0] - X.shape[1] - self.fit_intercept
+        )
diff --git a/sklearn/linear_model/_linear_loss.py b/sklearn/linear_model/_linear_loss.py
new file mode 100644
index 0000000000000..9213008a19841
--- /dev/null
+++ b/sklearn/linear_model/_linear_loss.py
@@ -0,0 +1,825 @@
+"""
+Loss functions for linear models with raw_prediction = X @ coef
+"""
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+import numpy as np
+from scipy import sparse
+
+from ..utils.extmath import squared_norm
+
+
+def sandwich_dot(X, W):
+    """Compute the sandwich product X.T @ diag(W) @ X."""
+    # TODO: This "sandwich product" is the main computational bottleneck for solvers
+    # that use the full hessian matrix. Here, thread parallelism would pay-off the
+    # most.
+    # While a dedicated Cython routine could exploit the symmetry, it is very hard to
+    # beat BLAS GEMM, even thought the latter cannot exploit the symmetry, unless one
+    # pays the price of taking square roots and implements
+    #    sqrtWX = sqrt(W)[: None] * X
+    #    return sqrtWX.T @ sqrtWX
+    # which (might) detect the symmetry and use BLAS SYRK under the hood.
+    n_samples = X.shape[0]
+    if sparse.issparse(X):
+        return (
+            X.T @ sparse.dia_matrix((W, 0), shape=(n_samples, n_samples)) @ X
+        ).toarray()
+    else:
+        # np.einsum may use less memory but the following, using BLAS matrix
+        # multiplication (gemm), is by far faster.
+        WX = W[:, None] * X
+        return X.T @ WX
+
+
+class LinearModelLoss:
+    """General class for loss functions with raw_prediction = X @ coef + intercept.
+
+    Note that raw_prediction is also known as linear predictor.
+
+    The loss is the average of per sample losses and includes a term for L2
+    regularization::
+
+        loss = 1 / s_sum * sum_i s_i loss(y_i, X_i @ coef + intercept)
+               + 1/2 * l2_reg_strength * ||coef||_2^2
+
+    with sample weights s_i=1 if sample_weight=None and s_sum=sum_i s_i.
+
+    Gradient and hessian, for simplicity without intercept, are::
+
+        gradient = 1 / s_sum * X.T @ loss.gradient + l2_reg_strength * coef
+        hessian = 1 / s_sum * X.T @ diag(loss.hessian) @ X
+                  + l2_reg_strength * identity
+
+    Conventions:
+        if fit_intercept:
+            n_dof =  n_features + 1
+        else:
+            n_dof = n_features
+
+        if base_loss.is_multiclass:
+            coef.shape = (n_classes, n_dof) or ravelled (n_classes * n_dof,)
+        else:
+            coef.shape = (n_dof,)
+
+        The intercept term is at the end of the coef array:
+        if base_loss.is_multiclass:
+            if coef.shape (n_classes, n_dof):
+                intercept = coef[:, -1]
+            if coef.shape (n_classes * n_dof,)
+                intercept = coef[n_features::n_dof] = coef[(n_dof-1)::n_dof]
+            intercept.shape = (n_classes,)
+        else:
+            intercept = coef[-1]
+
+        Shape of gradient follows shape of coef.
+        gradient.shape = coef.shape
+
+        But hessian (to make our lives simpler) are always 2-d:
+        if base_loss.is_multiclass:
+            hessian.shape = (n_classes * n_dof, n_classes * n_dof)
+        else:
+            hessian.shape = (n_dof, n_dof)
+
+    Note: If coef has shape (n_classes * n_dof,), the 2d-array can be reconstructed as
+
+        coef.reshape((n_classes, -1), order="F")
+
+    The option order="F" makes coef[:, i] contiguous. This, in turn, makes the
+    coefficients without intercept, coef[:, :-1], contiguous and speeds up
+    matrix-vector computations.
+
+    Note: If the average loss per sample is wanted instead of the sum of the loss per
+    sample, one can simply use a rescaled sample_weight such that
+    sum(sample_weight) = 1.
+
+    Parameters
+    ----------
+    base_loss : instance of class BaseLoss from sklearn._loss.
+    fit_intercept : bool
+    """
+
+    def __init__(self, base_loss, fit_intercept):
+        self.base_loss = base_loss
+        self.fit_intercept = fit_intercept
+
+    def init_zero_coef(self, X, dtype=None):
+        """Allocate coef of correct shape with zeros.
+
+        Parameters:
+        -----------
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Training data.
+        dtype : data-type, default=None
+            Overrides the data type of coef. With dtype=None, coef will have the same
+            dtype as X.
+
+        Returns
+        -------
+        coef : ndarray of shape (n_dof,) or (n_classes, n_dof)
+            Coefficients of a linear model.
+        """
+        n_features = X.shape[1]
+        n_classes = self.base_loss.n_classes
+        if self.fit_intercept:
+            n_dof = n_features + 1
+        else:
+            n_dof = n_features
+        if self.base_loss.is_multiclass:
+            coef = np.zeros_like(X, shape=(n_classes, n_dof), dtype=dtype, order="F")
+        else:
+            coef = np.zeros_like(X, shape=n_dof, dtype=dtype)
+        return coef
+
+    def weight_intercept(self, coef):
+        """Helper function to get coefficients and intercept.
+
+        Parameters
+        ----------
+        coef : ndarray of shape (n_dof,), (n_classes, n_dof) or (n_classes * n_dof,)
+            Coefficients of a linear model.
+            If shape (n_classes * n_dof,), the classes of one feature are contiguous,
+            i.e. one reconstructs the 2d-array via
+            coef.reshape((n_classes, -1), order="F").
+
+        Returns
+        -------
+        weights : ndarray of shape (n_features,) or (n_classes, n_features)
+            Coefficients without intercept term.
+        intercept : float or ndarray of shape (n_classes,)
+            Intercept terms.
+        """
+        if not self.base_loss.is_multiclass:
+            if self.fit_intercept:
+                intercept = coef[-1]
+                weights = coef[:-1]
+            else:
+                intercept = 0.0
+                weights = coef
+        else:
+            # reshape to (n_classes, n_dof)
+            if coef.ndim == 1:
+                weights = coef.reshape((self.base_loss.n_classes, -1), order="F")
+            else:
+                weights = coef
+            if self.fit_intercept:
+                intercept = weights[:, -1]
+                weights = weights[:, :-1]
+            else:
+                intercept = 0.0
+
+        return weights, intercept
+
+    def weight_intercept_raw(self, coef, X):
+        """Helper function to get coefficients, intercept and raw_prediction.
+
+        Parameters
+        ----------
+        coef : ndarray of shape (n_dof,), (n_classes, n_dof) or (n_classes * n_dof,)
+            Coefficients of a linear model.
+            If shape (n_classes * n_dof,), the classes of one feature are contiguous,
+            i.e. one reconstructs the 2d-array via
+            coef.reshape((n_classes, -1), order="F").
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Training data.
+
+        Returns
+        -------
+        weights : ndarray of shape (n_features,) or (n_classes, n_features)
+            Coefficients without intercept term.
+        intercept : float or ndarray of shape (n_classes,)
+            Intercept terms.
+        raw_prediction : ndarray of shape (n_samples,) or \
+            (n_samples, n_classes)
+        """
+        weights, intercept = self.weight_intercept(coef)
+
+        if not self.base_loss.is_multiclass:
+            raw_prediction = X @ weights + intercept
+        else:
+            # weights has shape (n_classes, n_dof)
+            raw_prediction = X @ weights.T + intercept  # ndarray, likely C-contiguous
+
+        return weights, intercept, raw_prediction
+
+    def l2_penalty(self, weights, l2_reg_strength):
+        """Compute L2 penalty term l2_reg_strength/2 *||w||_2^2."""
+        norm2_w = weights @ weights if weights.ndim == 1 else squared_norm(weights)
+        return 0.5 * l2_reg_strength * norm2_w
+
+    def loss(
+        self,
+        coef,
+        X,
+        y,
+        sample_weight=None,
+        l2_reg_strength=0.0,
+        n_threads=1,
+        raw_prediction=None,
+    ):
+        """Compute the loss as weighted average over point-wise losses.
+
+        Parameters
+        ----------
+        coef : ndarray of shape (n_dof,), (n_classes, n_dof) or (n_classes * n_dof,)
+            Coefficients of a linear model.
+            If shape (n_classes * n_dof,), the classes of one feature are contiguous,
+            i.e. one reconstructs the 2d-array via
+            coef.reshape((n_classes, -1), order="F").
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Training data.
+        y : contiguous array of shape (n_samples,)
+            Observed, true target values.
+        sample_weight : None or contiguous array of shape (n_samples,), default=None
+            Sample weights.
+        l2_reg_strength : float, default=0.0
+            L2 regularization strength
+        n_threads : int, default=1
+            Number of OpenMP threads to use.
+        raw_prediction : C-contiguous array of shape (n_samples,) or array of \
+            shape (n_samples, n_classes)
+            Raw prediction values (in link space). If provided, these are used. If
+            None, then raw_prediction = X @ coef + intercept is calculated.
+
+        Returns
+        -------
+        loss : float
+            Weighted average of losses per sample, plus penalty.
+        """
+        if raw_prediction is None:
+            weights, intercept, raw_prediction = self.weight_intercept_raw(coef, X)
+        else:
+            weights, intercept = self.weight_intercept(coef)
+
+        loss = self.base_loss.loss(
+            y_true=y,
+            raw_prediction=raw_prediction,
+            sample_weight=None,
+            n_threads=n_threads,
+        )
+        loss = np.average(loss, weights=sample_weight)
+
+        return loss + self.l2_penalty(weights, l2_reg_strength)
+
+    def loss_gradient(
+        self,
+        coef,
+        X,
+        y,
+        sample_weight=None,
+        l2_reg_strength=0.0,
+        n_threads=1,
+        raw_prediction=None,
+    ):
+        """Computes the sum of loss and gradient w.r.t. coef.
+
+        Parameters
+        ----------
+        coef : ndarray of shape (n_dof,), (n_classes, n_dof) or (n_classes * n_dof,)
+            Coefficients of a linear model.
+            If shape (n_classes * n_dof,), the classes of one feature are contiguous,
+            i.e. one reconstructs the 2d-array via
+            coef.reshape((n_classes, -1), order="F").
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Training data.
+        y : contiguous array of shape (n_samples,)
+            Observed, true target values.
+        sample_weight : None or contiguous array of shape (n_samples,), default=None
+            Sample weights.
+        l2_reg_strength : float, default=0.0
+            L2 regularization strength
+        n_threads : int, default=1
+            Number of OpenMP threads to use.
+        raw_prediction : C-contiguous array of shape (n_samples,) or array of \
+            shape (n_samples, n_classes)
+            Raw prediction values (in link space). If provided, these are used. If
+            None, then raw_prediction = X @ coef + intercept is calculated.
+
+        Returns
+        -------
+        loss : float
+            Weighted average of losses per sample, plus penalty.
+
+        gradient : ndarray of shape coef.shape
+             The gradient of the loss.
+        """
+        (n_samples, n_features), n_classes = X.shape, self.base_loss.n_classes
+        n_dof = n_features + int(self.fit_intercept)
+
+        if raw_prediction is None:
+            weights, intercept, raw_prediction = self.weight_intercept_raw(coef, X)
+        else:
+            weights, intercept = self.weight_intercept(coef)
+
+        loss, grad_pointwise = self.base_loss.loss_gradient(
+            y_true=y,
+            raw_prediction=raw_prediction,
+            sample_weight=sample_weight,
+            n_threads=n_threads,
+        )
+        sw_sum = n_samples if sample_weight is None else np.sum(sample_weight)
+        loss = loss.sum() / sw_sum
+        loss += self.l2_penalty(weights, l2_reg_strength)
+
+        grad_pointwise /= sw_sum
+
+        if not self.base_loss.is_multiclass:
+            grad = np.empty_like(coef, dtype=weights.dtype)
+            grad[:n_features] = X.T @ grad_pointwise + l2_reg_strength * weights
+            if self.fit_intercept:
+                grad[-1] = grad_pointwise.sum()
+        else:
+            grad = np.empty((n_classes, n_dof), dtype=weights.dtype, order="F")
+            # grad_pointwise.shape = (n_samples, n_classes)
+            grad[:, :n_features] = grad_pointwise.T @ X + l2_reg_strength * weights
+            if self.fit_intercept:
+                grad[:, -1] = grad_pointwise.sum(axis=0)
+            if coef.ndim == 1:
+                grad = grad.ravel(order="F")
+
+        return loss, grad
+
+    def gradient(
+        self,
+        coef,
+        X,
+        y,
+        sample_weight=None,
+        l2_reg_strength=0.0,
+        n_threads=1,
+        raw_prediction=None,
+    ):
+        """Computes the gradient w.r.t. coef.
+
+        Parameters
+        ----------
+        coef : ndarray of shape (n_dof,), (n_classes, n_dof) or (n_classes * n_dof,)
+            Coefficients of a linear model.
+            If shape (n_classes * n_dof,), the classes of one feature are contiguous,
+            i.e. one reconstructs the 2d-array via
+            coef.reshape((n_classes, -1), order="F").
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Training data.
+        y : contiguous array of shape (n_samples,)
+            Observed, true target values.
+        sample_weight : None or contiguous array of shape (n_samples,), default=None
+            Sample weights.
+        l2_reg_strength : float, default=0.0
+            L2 regularization strength
+        n_threads : int, default=1
+            Number of OpenMP threads to use.
+        raw_prediction : C-contiguous array of shape (n_samples,) or array of \
+            shape (n_samples, n_classes)
+            Raw prediction values (in link space). If provided, these are used. If
+            None, then raw_prediction = X @ coef + intercept is calculated.
+
+        Returns
+        -------
+        gradient : ndarray of shape coef.shape
+             The gradient of the loss.
+        """
+        (n_samples, n_features), n_classes = X.shape, self.base_loss.n_classes
+        n_dof = n_features + int(self.fit_intercept)
+
+        if raw_prediction is None:
+            weights, intercept, raw_prediction = self.weight_intercept_raw(coef, X)
+        else:
+            weights, intercept = self.weight_intercept(coef)
+
+        grad_pointwise = self.base_loss.gradient(
+            y_true=y,
+            raw_prediction=raw_prediction,
+            sample_weight=sample_weight,
+            n_threads=n_threads,
+        )
+        sw_sum = n_samples if sample_weight is None else np.sum(sample_weight)
+        grad_pointwise /= sw_sum
+
+        if not self.base_loss.is_multiclass:
+            grad = np.empty_like(coef, dtype=weights.dtype)
+            grad[:n_features] = X.T @ grad_pointwise + l2_reg_strength * weights
+            if self.fit_intercept:
+                grad[-1] = grad_pointwise.sum()
+            return grad
+        else:
+            grad = np.empty((n_classes, n_dof), dtype=weights.dtype, order="F")
+            # gradient.shape = (n_samples, n_classes)
+            grad[:, :n_features] = grad_pointwise.T @ X + l2_reg_strength * weights
+            if self.fit_intercept:
+                grad[:, -1] = grad_pointwise.sum(axis=0)
+            if coef.ndim == 1:
+                return grad.ravel(order="F")
+            else:
+                return grad
+
+    def gradient_hessian(
+        self,
+        coef,
+        X,
+        y,
+        sample_weight=None,
+        l2_reg_strength=0.0,
+        n_threads=1,
+        gradient_out=None,
+        hessian_out=None,
+        raw_prediction=None,
+    ):
+        """Computes gradient and hessian w.r.t. coef.
+
+        Parameters
+        ----------
+        coef : ndarray of shape (n_dof,), (n_classes, n_dof) or (n_classes * n_dof,)
+            Coefficients of a linear model.
+            If shape (n_classes * n_dof,), the classes of one feature are contiguous,
+            i.e. one reconstructs the 2d-array via
+            coef.reshape((n_classes, -1), order="F").
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Training data.
+        y : contiguous array of shape (n_samples,)
+            Observed, true target values.
+        sample_weight : None or contiguous array of shape (n_samples,), default=None
+            Sample weights.
+        l2_reg_strength : float, default=0.0
+            L2 regularization strength
+        n_threads : int, default=1
+            Number of OpenMP threads to use.
+        gradient_out : None or ndarray of shape coef.shape
+            A location into which the gradient is stored. If None, a new array
+            might be created.
+        hessian_out : None or ndarray of shape (n_dof, n_dof) or \
+            (n_classes * n_dof, n_classes * n_dof)
+            A location into which the hessian is stored. If None, a new array
+            might be created.
+        raw_prediction : C-contiguous array of shape (n_samples,) or array of \
+            shape (n_samples, n_classes)
+            Raw prediction values (in link space). If provided, these are used. If
+            None, then raw_prediction = X @ coef + intercept is calculated.
+
+        Returns
+        -------
+        gradient : ndarray of shape coef.shape
+             The gradient of the loss.
+
+        hessian : ndarray of shape (n_dof, n_dof) or \
+            (n_classes, n_dof, n_dof, n_classes)
+            Hessian matrix.
+
+        hessian_warning : bool
+            True if pointwise hessian has more than 25% of its elements non-positive.
+        """
+        (n_samples, n_features), n_classes = X.shape, self.base_loss.n_classes
+        n_dof = n_features + int(self.fit_intercept)
+        if raw_prediction is None:
+            weights, intercept, raw_prediction = self.weight_intercept_raw(coef, X)
+        else:
+            weights, intercept = self.weight_intercept(coef)
+        sw_sum = n_samples if sample_weight is None else np.sum(sample_weight)
+
+        # Allocate gradient.
+        if gradient_out is None:
+            grad = np.empty_like(coef, dtype=weights.dtype, order="F")
+        elif gradient_out.shape != coef.shape:
+            raise ValueError(
+                f"gradient_out is required to have shape coef.shape = {coef.shape}; "
+                f"got {gradient_out.shape}."
+            )
+        elif self.base_loss.is_multiclass and not gradient_out.flags.f_contiguous:
+            raise ValueError("gradient_out must be F-contiguous.")
+        else:
+            grad = gradient_out
+        # Allocate hessian.
+        n = coef.size  # for multinomial this equals n_dof * n_classes
+        if hessian_out is None:
+            hess = np.empty((n, n), dtype=weights.dtype)
+        elif hessian_out.shape != (n, n):
+            raise ValueError(
+                f"hessian_out is required to have shape ({n, n}); got "
+                f"{hessian_out.shape=}."
+            )
+        elif self.base_loss.is_multiclass and (
+            not hessian_out.flags.c_contiguous and not hessian_out.flags.f_contiguous
+        ):
+            raise ValueError("hessian_out must be contiguous.")
+        else:
+            hess = hessian_out
+
+        if not self.base_loss.is_multiclass:
+            grad_pointwise, hess_pointwise = self.base_loss.gradient_hessian(
+                y_true=y,
+                raw_prediction=raw_prediction,
+                sample_weight=sample_weight,
+                n_threads=n_threads,
+            )
+            grad_pointwise /= sw_sum
+            hess_pointwise /= sw_sum
+
+            # For non-canonical link functions and far away from the optimum, the
+            # pointwise hessian can be negative. We take care that 75% of the hessian
+            # entries are positive.
+            hessian_warning = (
+                np.average(hess_pointwise <= 0, weights=sample_weight) > 0.25
+            )
+            hess_pointwise = np.abs(hess_pointwise)
+
+            grad[:n_features] = X.T @ grad_pointwise + l2_reg_strength * weights
+            if self.fit_intercept:
+                grad[-1] = grad_pointwise.sum()
+
+            if hessian_warning:
+                # Exit early without computing the hessian.
+                return grad, hess, hessian_warning
+
+            hess[:n_features, :n_features] = sandwich_dot(X, hess_pointwise)
+
+            if l2_reg_strength > 0:
+                # The L2 penalty enters the Hessian on the diagonal only. To add those
+                # terms, we use a flattened view of the array.
+                order = "C" if hess.flags.c_contiguous else "F"
+                hess.reshape(-1, order=order)[: (n_features * n_dof) : (n_dof + 1)] += (
+                    l2_reg_strength
+                )
+
+            if self.fit_intercept:
+                # With intercept included as added column to X, the hessian becomes
+                # hess = (X, 1)' @ diag(h) @ (X, 1)
+                #      = (X' @ diag(h) @ X, X' @ h)
+                #        (           h @ X, sum(h))
+                # The left upper part has already been filled, it remains to compute
+                # the last row and the last column.
+                Xh = X.T @ hess_pointwise
+                hess[:-1, -1] = Xh
+                hess[-1, :-1] = Xh
+                hess[-1, -1] = hess_pointwise.sum()
+        else:
+            # Here we may safely assume HalfMultinomialLoss aka categorical
+            # cross-entropy.
+            # HalfMultinomialLoss computes only the diagonal part of the hessian, i.e.
+            # diagonal in the classes. Here, we want the full hessian. Therefore, we
+            # call gradient_proba.
+            grad_pointwise, proba = self.base_loss.gradient_proba(
+                y_true=y,
+                raw_prediction=raw_prediction,
+                sample_weight=sample_weight,
+                n_threads=n_threads,
+            )
+            grad_pointwise /= sw_sum
+            grad = grad.reshape((n_classes, n_dof), order="F")
+            grad[:, :n_features] = grad_pointwise.T @ X + l2_reg_strength * weights
+            if self.fit_intercept:
+                grad[:, -1] = grad_pointwise.sum(axis=0)
+            if coef.ndim == 1:
+                grad = grad.ravel(order="F")
+
+            # The full hessian matrix, i.e. not only the diagonal part, dropping most
+            # indices, is given by:
+            #
+            #   hess = X' @ h @ X
+            #
+            # Here, h is a priori a 4-dimensional matrix of shape
+            # (n_samples, n_samples, n_classes, n_classes). It is diagonal its first
+            # two dimensions (the ones with n_samples), i.e. it is
+            # effectively a 3-dimensional matrix (n_samples, n_classes, n_classes).
+            #
+            #   h = diag(p) - p' p
+            #
+            # or with indices k and l for classes
+            #
+            #   h_kl = p_k * delta_kl - p_k * p_l
+            #
+            # with p_k the (predicted) probability for class k. Only the dimension in
+            # n_samples multiplies with X.
+            # For 3 classes and n_samples = 1, this looks like ("@" is a bit misused
+            # here):
+            #
+            #   hess = X' @ (h00 h10 h20) @ X
+            #               (h10 h11 h12)
+            #               (h20 h12 h22)
+            #        = (X' @ diag(h00) @ X, X' @ diag(h10), X' @ diag(h20))
+            #          (X' @ diag(h10) @ X, X' @ diag(h11), X' @ diag(h12))
+            #          (X' @ diag(h20) @ X, X' @ diag(h12), X' @ diag(h22))
+            #
+            # Now coef of shape (n_classes * n_dof) is contiguous in n_classes.
+            # Therefore, we want the hessian to follow this convention, too, i.e.
+            #     hess[:n_classes, :n_classes] = (x0' @ h00 @ x0, x0' @ h10 @ x0, ..)
+            #                                    (x0' @ h10 @ x0, x0' @ h11 @ x0, ..)
+            #                                    (x0' @ h20 @ x0, x0' @ h12 @ x0, ..)
+            # is the first feature, x0, for all classes. In our implementation, we
+            # still want to take advantage of BLAS "X.T @ X". Therefore, we have some
+            # index/slicing battle to fight.
+            if sample_weight is not None:
+                sw = sample_weight / sw_sum
+            else:
+                sw = 1.0 / sw_sum
+
+            for k in range(n_classes):
+                # Diagonal terms (in classes) hess_kk.
+                # Note that this also writes to some of the lower triangular part.
+                h = proba[:, k] * (1 - proba[:, k]) * sw
+                hess[
+                    k : n_classes * n_features : n_classes,
+                    k : n_classes * n_features : n_classes,
+                ] = sandwich_dot(X, h)
+                if self.fit_intercept:
+                    # See above in the non multiclass case.
+                    Xh = X.T @ h
+                    hess[
+                        k : n_classes * n_features : n_classes,
+                        n_classes * n_features + k,
+                    ] = Xh
+                    hess[
+                        n_classes * n_features + k,
+                        k : n_classes * n_features : n_classes,
+                    ] = Xh
+                    hess[n_classes * n_features + k, n_classes * n_features + k] = (
+                        h.sum()
+                    )
+                # Off diagonal terms (in classes) hess_kl.
+                for l in range(k + 1, n_classes):
+                    # Upper triangle (in classes).
+                    h = -proba[:, k] * proba[:, l] * sw
+                    hess[
+                        k : n_classes * n_features : n_classes,
+                        l : n_classes * n_features : n_classes,
+                    ] = sandwich_dot(X, h)
+                    if self.fit_intercept:
+                        Xh = X.T @ h
+                        hess[
+                            k : n_classes * n_features : n_classes,
+                            n_classes * n_features + l,
+                        ] = Xh
+                        hess[
+                            n_classes * n_features + k,
+                            l : n_classes * n_features : n_classes,
+                        ] = Xh
+                        hess[n_classes * n_features + k, n_classes * n_features + l] = (
+                            h.sum()
+                        )
+                    # Fill lower triangle (in classes).
+                    hess[l::n_classes, k::n_classes] = hess[k::n_classes, l::n_classes]
+
+            if l2_reg_strength > 0:
+                # See above in the non multiclass case.
+                order = "C" if hess.flags.c_contiguous else "F"
+                hess.reshape(-1, order=order)[
+                    : (n_classes**2 * n_features * n_dof) : (n_classes * n_dof + 1)
+                ] += l2_reg_strength
+
+            # The pointwise hessian is always non-negative for the multinomial loss.
+            hessian_warning = False
+
+        return grad, hess, hessian_warning
+
+    def gradient_hessian_product(
+        self, coef, X, y, sample_weight=None, l2_reg_strength=0.0, n_threads=1
+    ):
+        """Computes gradient and hessp (hessian product function) w.r.t. coef.
+
+        Parameters
+        ----------
+        coef : ndarray of shape (n_dof,), (n_classes, n_dof) or (n_classes * n_dof,)
+            Coefficients of a linear model.
+            If shape (n_classes * n_dof,), the classes of one feature are contiguous,
+            i.e. one reconstructs the 2d-array via
+            coef.reshape((n_classes, -1), order="F").
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Training data.
+        y : contiguous array of shape (n_samples,)
+            Observed, true target values.
+        sample_weight : None or contiguous array of shape (n_samples,), default=None
+            Sample weights.
+        l2_reg_strength : float, default=0.0
+            L2 regularization strength
+        n_threads : int, default=1
+            Number of OpenMP threads to use.
+
+        Returns
+        -------
+        gradient : ndarray of shape coef.shape
+             The gradient of the loss.
+
+        hessp : callable
+            Function that takes in a vector input of shape of gradient and
+            and returns matrix-vector product with hessian.
+        """
+        (n_samples, n_features), n_classes = X.shape, self.base_loss.n_classes
+        n_dof = n_features + int(self.fit_intercept)
+        weights, intercept, raw_prediction = self.weight_intercept_raw(coef, X)
+        sw_sum = n_samples if sample_weight is None else np.sum(sample_weight)
+
+        if not self.base_loss.is_multiclass:
+            grad_pointwise, hess_pointwise = self.base_loss.gradient_hessian(
+                y_true=y,
+                raw_prediction=raw_prediction,
+                sample_weight=sample_weight,
+                n_threads=n_threads,
+            )
+            grad_pointwise /= sw_sum
+            hess_pointwise /= sw_sum
+            grad = np.empty_like(coef, dtype=weights.dtype)
+            grad[:n_features] = X.T @ grad_pointwise + l2_reg_strength * weights
+            if self.fit_intercept:
+                grad[-1] = grad_pointwise.sum()
+
+            # Precompute as much as possible: hX, hX_sum and hessian_sum
+            hessian_sum = hess_pointwise.sum()
+            if sparse.issparse(X):
+                hX = (
+                    sparse.dia_matrix((hess_pointwise, 0), shape=(n_samples, n_samples))
+                    @ X
+                )
+            else:
+                hX = hess_pointwise[:, np.newaxis] * X
+
+            if self.fit_intercept:
+                # Calculate the double derivative with respect to intercept.
+                # Note: In case hX is sparse, hX.sum is a matrix object.
+                hX_sum = np.squeeze(np.asarray(hX.sum(axis=0)))
+                # prevent squeezing to zero-dim array if n_features == 1
+                hX_sum = np.atleast_1d(hX_sum)
+
+            # With intercept included and l2_reg_strength = 0, hessp returns
+            # res = (X, 1)' @ diag(h) @ (X, 1) @ s
+            #     = (X, 1)' @ (hX @ s[:n_features], sum(h) * s[-1])
+            # res[:n_features] = X' @ hX @ s[:n_features] + sum(h) * s[-1]
+            # res[-1] = 1' @ hX @ s[:n_features] + sum(h) * s[-1]
+            def hessp(s):
+                ret = np.empty_like(s)
+                if sparse.issparse(X):
+                    ret[:n_features] = X.T @ (hX @ s[:n_features])
+                else:
+                    ret[:n_features] = np.linalg.multi_dot([X.T, hX, s[:n_features]])
+                ret[:n_features] += l2_reg_strength * s[:n_features]
+
+                if self.fit_intercept:
+                    ret[:n_features] += s[-1] * hX_sum
+                    ret[-1] = hX_sum @ s[:n_features] + hessian_sum * s[-1]
+                return ret
+
+        else:
+            # Here we may safely assume HalfMultinomialLoss aka categorical
+            # cross-entropy.
+            # HalfMultinomialLoss computes only the diagonal part of the hessian, i.e.
+            # diagonal in the classes. Here, we want the matrix-vector product of the
+            # full hessian. Therefore, we call gradient_proba.
+            grad_pointwise, proba = self.base_loss.gradient_proba(
+                y_true=y,
+                raw_prediction=raw_prediction,
+                sample_weight=sample_weight,
+                n_threads=n_threads,
+            )
+            grad_pointwise /= sw_sum
+            grad = np.empty((n_classes, n_dof), dtype=weights.dtype, order="F")
+            grad[:, :n_features] = grad_pointwise.T @ X + l2_reg_strength * weights
+            if self.fit_intercept:
+                grad[:, -1] = grad_pointwise.sum(axis=0)
+
+            # Full hessian-vector product, i.e. not only the diagonal part of the
+            # hessian. Derivation with some index battle for input vector s:
+            #   - sample index i
+            #   - feature indices j, m
+            #   - class indices k, l
+            #   - 1_{k=l} is one if k=l else 0
+            #   - p_i_k is the (predicted) probability that sample i belongs to class k
+            #     for all i: sum_k p_i_k = 1
+            #   - s_l_m is input vector for class l and feature m
+            #   - X' = X transposed
+            #
+            # Note: Hessian with dropping most indices is just:
+            #       X' @ p_k (1(k=l) - p_l) @ X
+            #
+            # result_{k j} = sum_{i, l, m} Hessian_{i, k j, m l} * s_l_m
+            #   = sum_{i, l, m} (X')_{ji} * p_i_k * (1_{k=l} - p_i_l)
+            #                   * X_{im} s_l_m
+            #   = sum_{i, m} (X')_{ji} * p_i_k
+            #                * (X_{im} * s_k_m - sum_l p_i_l * X_{im} * s_l_m)
+            #
+            # See also https://github.com/scikit-learn/scikit-learn/pull/3646#discussion_r17461411
+            def hessp(s):
+                s = s.reshape((n_classes, -1), order="F")  # shape = (n_classes, n_dof)
+                if self.fit_intercept:
+                    s_intercept = s[:, -1]
+                    s = s[:, :-1]  # shape = (n_classes, n_features)
+                else:
+                    s_intercept = 0
+                tmp = X @ s.T + s_intercept  # X_{im} * s_k_m
+                tmp += (-proba * tmp).sum(axis=1)[:, np.newaxis]  # - sum_l ..
+                tmp *= proba  # * p_i_k
+                if sample_weight is not None:
+                    tmp *= sample_weight[:, np.newaxis]
+                # hess_prod = empty_like(grad), but we ravel grad below and this
+                # function is run after that.
+                hess_prod = np.empty((n_classes, n_dof), dtype=weights.dtype, order="F")
+                hess_prod[:, :n_features] = (tmp.T @ X) / sw_sum + l2_reg_strength * s
+                if self.fit_intercept:
+                    hess_prod[:, -1] = tmp.sum(axis=0) / sw_sum
+                if coef.ndim == 1:
+                    return hess_prod.ravel(order="F")
+                else:
+                    return hess_prod
+
+            if coef.ndim == 1:
+                return grad.ravel(order="F"), hessp
+
+        return grad, hessp
diff --git a/sklearn/linear_model/_logistic.py b/sklearn/linear_model/_logistic.py
index 2b8b6a716cbf7..f0c97268c612d 100644
--- a/sklearn/linear_model/_logistic.py
+++ b/sklearn/linear_model/_logistic.py
@@ -2,487 +2,123 @@
 Logistic Regression
 """
 
-# Author: Gael Varoquaux 
-#         Fabian Pedregosa 
-#         Alexandre Gramfort 
-#         Manoj Kumar 
-#         Lars Buitinck
-#         Simon Wu 
-#         Arthur Mensch  n_features:
-        grad[-1] = z0.sum()
-    return out, grad
-
-
-def _logistic_loss(w, X, y, alpha, sample_weight=None):
-    """Computes the logistic loss.
-
-    Parameters
-    ----------
-    w : ndarray of shape (n_features,) or (n_features + 1,)
-        Coefficient vector.
-
-    X : {array-like, sparse matrix} of shape (n_samples, n_features)
-        Training data.
-
-    y : ndarray of shape (n_samples,)
-        Array of labels.
-
-    alpha : float
-        Regularization parameter. alpha is equal to 1 / C.
-
-    sample_weight : array-like of shape (n_samples,) default=None
-        Array of weights that are assigned to individual samples.
-        If not provided, then each sample is given unit weight.
-
-    Returns
-    -------
-    out : float
-        Logistic loss.
-    """
-    w, c, yz = _intercept_dot(w, X, y)
-
-    if sample_weight is None:
-        sample_weight = np.ones(y.shape[0])
-
-    # Logistic loss is the negative of the log of the logistic function.
-    out = -np.sum(sample_weight * log_logistic(yz)) + .5 * alpha * np.dot(w, w)
-    return out
-
-
-def _logistic_grad_hess(w, X, y, alpha, sample_weight=None):
-    """Computes the gradient and the Hessian, in the case of a logistic loss.
-
-    Parameters
-    ----------
-    w : ndarray of shape (n_features,) or (n_features + 1,)
-        Coefficient vector.
-
-    X : {array-like, sparse matrix} of shape (n_samples, n_features)
-        Training data.
-
-    y : ndarray of shape (n_samples,)
-        Array of labels.
-
-    alpha : float
-        Regularization parameter. alpha is equal to 1 / C.
-
-    sample_weight : array-like of shape (n_samples,) default=None
-        Array of weights that are assigned to individual samples.
-        If not provided, then each sample is given unit weight.
-
-    Returns
-    -------
-    grad : ndarray of shape (n_features,) or (n_features + 1,)
-        Logistic gradient.
-
-    Hs : callable
-        Function that takes the gradient as a parameter and returns the
-        matrix product of the Hessian and gradient.
-    """
-    n_samples, n_features = X.shape
-    grad = np.empty_like(w)
-    fit_intercept = grad.shape[0] > n_features
-
-    w, c, yz = _intercept_dot(w, X, y)
-
-    if sample_weight is None:
-        sample_weight = np.ones(y.shape[0])
-
-    z = expit(yz)
-    z0 = sample_weight * (z - 1) * y
-
-    grad[:n_features] = safe_sparse_dot(X.T, z0) + alpha * w
-
-    # Case where we fit the intercept.
-    if fit_intercept:
-        grad[-1] = z0.sum()
-
-    # The mat-vec product of the Hessian
-    d = sample_weight * z * (1 - z)
-    if sparse.issparse(X):
-        dX = safe_sparse_dot(sparse.dia_matrix((d, 0),
-                             shape=(n_samples, n_samples)), X)
-    else:
-        # Precompute as much as possible
-        dX = d[:, np.newaxis] * X
-
-    if fit_intercept:
-        # Calculate the double derivative with respect to intercept
-        # In the case of sparse matrices this returns a matrix object.
-        dd_intercept = np.squeeze(np.array(dX.sum(axis=0)))
-
-    def Hs(s):
-        ret = np.empty_like(s)
-        ret[:n_features] = X.T.dot(dX.dot(s[:n_features]))
-        ret[:n_features] += alpha * s[:n_features]
-
-        # For the fit intercept case.
-        if fit_intercept:
-            ret[:n_features] += s[-1] * dd_intercept
-            ret[-1] = dd_intercept.dot(s[:n_features])
-            ret[-1] += d.sum() * s[-1]
-        return ret
-
-    return grad, Hs
-
-
-def _multinomial_loss(w, X, Y, alpha, sample_weight):
-    """Computes multinomial loss and class probabilities.
-
-    Parameters
-    ----------
-    w : ndarray of shape (n_classes * n_features,) or
-        (n_classes * (n_features + 1),)
-        Coefficient vector.
-
-    X : {array-like, sparse matrix} of shape (n_samples, n_features)
-        Training data.
-
-    Y : ndarray of shape (n_samples, n_classes)
-        Transformed labels according to the output of LabelBinarizer.
-
-    alpha : float
-        Regularization parameter. alpha is equal to 1 / C.
-
-    sample_weight : array-like of shape (n_samples,)
-        Array of weights that are assigned to individual samples.
-
-    Returns
-    -------
-    loss : float
-        Multinomial loss.
-
-    p : ndarray of shape (n_samples, n_classes)
-        Estimated class probabilities.
-
-    w : ndarray of shape (n_classes, n_features)
-        Reshaped param vector excluding intercept terms.
-
-    Reference
-    ---------
-    Bishop, C. M. (2006). Pattern recognition and machine learning.
-    Springer. (Chapter 4.3.4)
-    """
-    n_classes = Y.shape[1]
-    n_features = X.shape[1]
-    fit_intercept = w.size == (n_classes * (n_features + 1))
-    w = w.reshape(n_classes, -1)
-    sample_weight = sample_weight[:, np.newaxis]
-    if fit_intercept:
-        intercept = w[:, -1]
-        w = w[:, :-1]
-    else:
-        intercept = 0
-    p = safe_sparse_dot(X, w.T)
-    p += intercept
-    p -= logsumexp(p, axis=1)[:, np.newaxis]
-    loss = -(sample_weight * Y * p).sum()
-    loss += 0.5 * alpha * squared_norm(w)
-    p = np.exp(p, p)
-    return loss, p, w
-
-
-def _multinomial_loss_grad(w, X, Y, alpha, sample_weight):
-    """Computes the multinomial loss, gradient and class probabilities.
-
-    Parameters
-    ----------
-    w : ndarray of shape (n_classes * n_features,) or
-        (n_classes * (n_features + 1),)
-        Coefficient vector.
-
-    X : {array-like, sparse matrix} of shape (n_samples, n_features)
-        Training data.
-
-    Y : ndarray of shape (n_samples, n_classes)
-        Transformed labels according to the output of LabelBinarizer.
-
-    alpha : float
-        Regularization parameter. alpha is equal to 1 / C.
-
-    sample_weight : array-like of shape (n_samples,)
-        Array of weights that are assigned to individual samples.
-
-    Returns
-    -------
-    loss : float
-        Multinomial loss.
-
-    grad : ndarray of shape (n_classes * n_features,) or \
-            (n_classes * (n_features + 1),)
-        Ravelled gradient of the multinomial loss.
-
-    p : ndarray of shape (n_samples, n_classes)
-        Estimated class probabilities
-
-    Reference
-    ---------
-    Bishop, C. M. (2006). Pattern recognition and machine learning.
-    Springer. (Chapter 4.3.4)
-    """
-    n_classes = Y.shape[1]
-    n_features = X.shape[1]
-    fit_intercept = (w.size == n_classes * (n_features + 1))
-    grad = np.zeros((n_classes, n_features + bool(fit_intercept)),
-                    dtype=X.dtype)
-    loss, p, w = _multinomial_loss(w, X, Y, alpha, sample_weight)
-    sample_weight = sample_weight[:, np.newaxis]
-    diff = sample_weight * (p - Y)
-    grad[:, :n_features] = safe_sparse_dot(diff.T, X)
-    grad[:, :n_features] += alpha * w
-    if fit_intercept:
-        grad[:, -1] = diff.sum(axis=0)
-    return loss, grad.ravel(), p
-
-
-def _multinomial_grad_hess(w, X, Y, alpha, sample_weight):
-    """
-    Computes the gradient and the Hessian, in the case of a multinomial loss.
-
-    Parameters
-    ----------
-    w : ndarray of shape (n_classes * n_features,) or
-        (n_classes * (n_features + 1),)
-        Coefficient vector.
-
-    X : {array-like, sparse matrix} of shape (n_samples, n_features)
-        Training data.
-
-    Y : ndarray of shape (n_samples, n_classes)
-        Transformed labels according to the output of LabelBinarizer.
-
-    alpha : float
-        Regularization parameter. alpha is equal to 1 / C.
-
-    sample_weight : array-like of shape (n_samples,)
-        Array of weights that are assigned to individual samples.
-
-    Returns
-    -------
-    grad : ndarray of shape (n_classes * n_features,) or \
-            (n_classes * (n_features + 1),)
-        Ravelled gradient of the multinomial loss.
-
-    hessp : callable
-        Function that takes in a vector input of shape (n_classes * n_features)
-        or (n_classes * (n_features + 1)) and returns matrix-vector product
-        with hessian.
-
-    References
-    ----------
-    Barak A. Pearlmutter (1993). Fast Exact Multiplication by the Hessian.
-        http://www.bcl.hamilton.ie/~barak/papers/nc-hessian.pdf
-    """
-    n_features = X.shape[1]
-    n_classes = Y.shape[1]
-    fit_intercept = w.size == (n_classes * (n_features + 1))
-
-    # `loss` is unused. Refactoring to avoid computing it does not
-    # significantly speed up the computation and decreases readability
-    loss, grad, p = _multinomial_loss_grad(w, X, Y, alpha, sample_weight)
-    sample_weight = sample_weight[:, np.newaxis]
-
-    # Hessian-vector product derived by applying the R-operator on the gradient
-    # of the multinomial loss function.
-    def hessp(v):
-        v = v.reshape(n_classes, -1)
-        if fit_intercept:
-            inter_terms = v[:, -1]
-            v = v[:, :-1]
-        else:
-            inter_terms = 0
-        # r_yhat holds the result of applying the R-operator on the multinomial
-        # estimator.
-        r_yhat = safe_sparse_dot(X, v.T)
-        r_yhat += inter_terms
-        r_yhat += (-p * r_yhat).sum(axis=1)[:, np.newaxis]
-        r_yhat *= p
-        r_yhat *= sample_weight
-        hessProd = np.zeros((n_classes, n_features + bool(fit_intercept)))
-        hessProd[:, :n_features] = safe_sparse_dot(r_yhat.T, X)
-        hessProd[:, :n_features] += v * alpha
-        if fit_intercept:
-            hessProd[:, -1] = r_yhat.sum(axis=0)
-        return hessProd.ravel()
-
-    return grad, hessp
+    "#logistic-regression"
+)
 
 
 def _check_solver(solver, penalty, dual):
-    all_solvers = ['liblinear', 'newton-cg', 'lbfgs', 'sag', 'saga']
-    if solver not in all_solvers:
-        raise ValueError("Logistic Regression supports only solvers in %s, got"
-                         " %s." % (all_solvers, solver))
-
-    all_penalties = ['l1', 'l2', 'elasticnet', 'none']
-    if penalty not in all_penalties:
-        raise ValueError("Logistic Regression supports only penalties in %s,"
-                         " got %s." % (all_penalties, penalty))
-
-    if solver not in ['liblinear', 'saga'] and penalty not in ('l2', 'none'):
-        raise ValueError("Solver %s supports only 'l2' or 'none' penalties, "
-                         "got %s penalty." % (solver, penalty))
-    if solver != 'liblinear' and dual:
-        raise ValueError("Solver %s supports only "
-                         "dual=False, got dual=%s" % (solver, dual))
-
-    if penalty == 'elasticnet' and solver != 'saga':
-        raise ValueError("Only 'saga' solver supports elasticnet penalty,"
-                         " got solver={}.".format(solver))
-
-    if solver == 'liblinear' and penalty == 'none':
+    if solver not in ["liblinear", "saga"] and penalty not in ("l2", None):
         raise ValueError(
-            "penalty='none' is not supported for the liblinear solver"
+            f"Solver {solver} supports only 'l2' or None penalties, got {penalty} "
+            "penalty."
         )
+    if solver != "liblinear" and dual:
+        raise ValueError(f"Solver {solver} supports only dual=False, got dual={dual}")
+
+    if penalty == "elasticnet" and solver != "saga":
+        raise ValueError(
+            f"Only 'saga' solver supports elasticnet penalty, got solver={solver}."
+        )
+
+    if solver == "liblinear" and penalty is None:
+        raise ValueError("penalty=None is not supported for the liblinear solver")
 
     return solver
 
 
 def _check_multi_class(multi_class, solver, n_classes):
-    if multi_class == 'auto':
-        if solver == 'liblinear':
-            multi_class = 'ovr'
+    """Computes the multi class type, either "multinomial" or "ovr".
+
+    For `n_classes` > 2 and a solver that supports it, returns "multinomial".
+    For all other cases, in particular binary classification, return "ovr".
+    """
+    if multi_class == "auto":
+        if solver in ("liblinear",):
+            multi_class = "ovr"
         elif n_classes > 2:
-            multi_class = 'multinomial'
+            multi_class = "multinomial"
         else:
-            multi_class = 'ovr'
-    if multi_class not in ('multinomial', 'ovr'):
-        raise ValueError("multi_class should be 'multinomial', 'ovr' or "
-                         "'auto'. Got %s." % multi_class)
-    if multi_class == 'multinomial' and solver == 'liblinear':
-        raise ValueError("Solver %s does not support "
-                         "a multinomial backend." % solver)
+            multi_class = "ovr"
+    if multi_class == "multinomial" and solver in ("liblinear",):
+        raise ValueError("Solver %s does not support a multinomial backend." % solver)
     return multi_class
 
 
-def _logistic_regression_path(X, y, pos_class=None, Cs=10, fit_intercept=True,
-                              max_iter=100, tol=1e-4, verbose=0,
-                              solver='lbfgs', coef=None,
-                              class_weight=None, dual=False, penalty='l2',
-                              intercept_scaling=1., multi_class='auto',
-                              random_state=None, check_input=True,
-                              max_squared_sum=None, sample_weight=None,
-                              l1_ratio=None):
+def _logistic_regression_path(
+    X,
+    y,
+    pos_class=None,
+    Cs=10,
+    fit_intercept=True,
+    max_iter=100,
+    tol=1e-4,
+    verbose=0,
+    solver="lbfgs",
+    coef=None,
+    class_weight=None,
+    dual=False,
+    penalty="l2",
+    intercept_scaling=1.0,
+    multi_class="auto",
+    random_state=None,
+    check_input=True,
+    max_squared_sum=None,
+    sample_weight=None,
+    l1_ratio=None,
+    n_threads=1,
+):
     """Compute a Logistic Regression model for a list of regularization
     parameters.
 
@@ -528,7 +164,7 @@ def _logistic_regression_path(X, y, pos_class=None, Cs=10, fit_intercept=True,
         For the liblinear and lbfgs solvers set verbose to any positive
         number for verbosity.
 
-    solver : {'lbfgs', 'newton-cg', 'liblinear', 'sag', 'saga'}, \
+    solver : {'lbfgs', 'liblinear', 'newton-cg', 'newton-cholesky', 'sag', 'saga'}, \
             default='lbfgs'
         Numerical solver to use.
 
@@ -606,6 +242,9 @@ def _logistic_regression_path(X, y, pos_class=None, Cs=10, fit_intercept=True,
         to using ``penalty='l1'``. For ``0 < l1_ratio <1``, the penalty is a
         combination of L1 and L2.
 
+    n_threads : int, default=1
+       Number of OpenMP threads to use.
+
     Returns
     -------
     coefs : ndarray of shape (n_cs, n_features) or (n_cs, n_features + 1)
@@ -636,75 +275,107 @@ def _logistic_regression_path(X, y, pos_class=None, Cs=10, fit_intercept=True,
 
     # Preprocessing.
     if check_input:
-        X = check_array(X, accept_sparse='csr', dtype=np.float64,
-                        accept_large_sparse=solver != 'liblinear')
+        X = check_array(
+            X,
+            accept_sparse="csr",
+            dtype=np.float64,
+            accept_large_sparse=solver not in ["liblinear", "sag", "saga"],
+        )
         y = check_array(y, ensure_2d=False, dtype=None)
         check_consistent_length(X, y)
-    _, n_features = X.shape
+    n_samples, n_features = X.shape
 
     classes = np.unique(y)
     random_state = check_random_state(random_state)
 
     multi_class = _check_multi_class(multi_class, solver, len(classes))
-    if pos_class is None and multi_class != 'multinomial':
-        if (classes.size > 2):
-            raise ValueError('To fit OvR, use the pos_class argument')
+    if pos_class is None and multi_class != "multinomial":
+        if classes.size > 2:
+            raise ValueError("To fit OvR, use the pos_class argument")
         # np.unique(y) gives labels in sorted order.
         pos_class = classes[1]
 
-    # If sample weights exist, convert them to array (support for lists)
-    # and check length
-    # Otherwise set them to 1 for all examples
-    sample_weight = _check_sample_weight(sample_weight, X,
-                                         dtype=X.dtype, copy=True)
+    if sample_weight is not None or class_weight is not None:
+        sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype, copy=True)
 
     # If class_weights is a dict (provided by the user), the weights
     # are assigned to the original labels. If it is "balanced", then
     # the class_weights are assigned after masking the labels with a OvR.
     le = LabelEncoder()
-    if isinstance(class_weight, dict) or multi_class == 'multinomial':
-        class_weight_ = compute_class_weight(class_weight,
-                                             classes=classes, y=y)
+    if isinstance(class_weight, dict) or (
+        multi_class == "multinomial" and class_weight is not None
+    ):
+        class_weight_ = compute_class_weight(
+            class_weight, classes=classes, y=y, sample_weight=sample_weight
+        )
         sample_weight *= class_weight_[le.fit_transform(y)]
 
-    # For doing a ovr, we need to mask the labels first. for the
+    # For doing a ovr, we need to mask the labels first. For the
     # multinomial case this is not necessary.
-    if multi_class == 'ovr':
+    if multi_class == "ovr":
         w0 = np.zeros(n_features + int(fit_intercept), dtype=X.dtype)
-        mask_classes = np.array([-1, 1])
-        mask = (y == pos_class)
+        mask = y == pos_class
         y_bin = np.ones(y.shape, dtype=X.dtype)
-        y_bin[~mask] = -1.
-        # for compute_class_weight
+        if solver == "liblinear":
+            mask_classes = np.array([-1, 1])
+            y_bin[~mask] = -1.0
+        else:
+            # HalfBinomialLoss, used for those solvers, represents y in [0, 1] instead
+            # of in [-1, 1].
+            mask_classes = np.array([0, 1])
+            y_bin[~mask] = 0.0
 
+        # for compute_class_weight
         if class_weight == "balanced":
-            class_weight_ = compute_class_weight(class_weight,
-                                                 classes=mask_classes,
-                                                 y=y_bin)
+            class_weight_ = compute_class_weight(
+                class_weight,
+                classes=mask_classes,
+                y=y_bin,
+                sample_weight=sample_weight,
+            )
             sample_weight *= class_weight_[le.fit_transform(y_bin)]
 
     else:
-        if solver not in ['sag', 'saga']:
+        if solver in ["sag", "saga", "lbfgs", "newton-cg", "newton-cholesky"]:
+            # SAG, lbfgs, newton-cg and newton-cholesky multinomial solvers need
+            # LabelEncoder, not LabelBinarizer, i.e. y as a 1d-array of integers.
+            # LabelEncoder also saves memory compared to LabelBinarizer, especially
+            # when n_classes is large.
+            le = LabelEncoder()
+            Y_multi = le.fit_transform(y).astype(X.dtype, copy=False)
+        else:
+            # For liblinear solver, apply LabelBinarizer, i.e. y is one-hot encoded.
             lbin = LabelBinarizer()
             Y_multi = lbin.fit_transform(y)
             if Y_multi.shape[1] == 1:
                 Y_multi = np.hstack([1 - Y_multi, Y_multi])
-        else:
-            # SAG multinomial solver needs LabelEncoder, not LabelBinarizer
-            le = LabelEncoder()
-            Y_multi = le.fit_transform(y).astype(X.dtype, copy=False)
 
-        w0 = np.zeros((classes.size, n_features + int(fit_intercept)),
-                      order='F', dtype=X.dtype)
+        w0 = np.zeros(
+            (classes.size, n_features + int(fit_intercept)), order="F", dtype=X.dtype
+        )
+
+    # IMPORTANT NOTE:
+    # All solvers relying on LinearModelLoss need to scale the penalty with n_samples
+    # or the sum of sample weights because the implemented logistic regression
+    # objective here is (unfortunately)
+    #     C * sum(pointwise_loss) + penalty
+    # instead of (as LinearModelLoss does)
+    #     mean(pointwise_loss) + 1/C * penalty
+    if solver in ["lbfgs", "newton-cg", "newton-cholesky"]:
+        # This needs to be calculated after sample_weight is multiplied by
+        # class_weight. It is even tested that passing class_weight is equivalent to
+        # passing sample_weights according to class_weight.
+        sw_sum = n_samples if sample_weight is None else np.sum(sample_weight)
 
     if coef is not None:
         # it must work both giving the bias term and not
-        if multi_class == 'ovr':
+        if multi_class == "ovr":
             if coef.size not in (n_features, w0.size):
                 raise ValueError(
-                    'Initialization coef is of shape %d, expected shape '
-                    '%d or %d' % (coef.size, n_features, w0.size))
-            w0[:coef.size] = coef
+                    "Initialization coef is of shape %d, expected shape %d or %d"
+                    % (coef.size, n_features, w0.size)
+                )
+            w0[: coef.size] = coef
         else:
             # For binary problems coef.shape[0] should be 1, otherwise it
             # should be classes.size.
@@ -712,102 +383,206 @@ def _logistic_regression_path(X, y, pos_class=None, Cs=10, fit_intercept=True,
             if n_classes == 2:
                 n_classes = 1
 
-            if (coef.shape[0] != n_classes or
-                    coef.shape[1] not in (n_features, n_features + 1)):
+            if coef.shape[0] != n_classes or coef.shape[1] not in (
+                n_features,
+                n_features + 1,
+            ):
                 raise ValueError(
-                    'Initialization coef is of shape (%d, %d), expected '
-                    'shape (%d, %d) or (%d, %d)' % (
-                        coef.shape[0], coef.shape[1], classes.size,
-                        n_features, classes.size, n_features + 1))
+                    "Initialization coef is of shape (%d, %d), expected "
+                    "shape (%d, %d) or (%d, %d)"
+                    % (
+                        coef.shape[0],
+                        coef.shape[1],
+                        classes.size,
+                        n_features,
+                        classes.size,
+                        n_features + 1,
+                    )
+                )
 
             if n_classes == 1:
-                w0[0, :coef.shape[1]] = -coef
-                w0[1, :coef.shape[1]] = coef
+                w0[0, : coef.shape[1]] = -coef
+                w0[1, : coef.shape[1]] = coef
             else:
-                w0[:, :coef.shape[1]] = coef
-
-    if multi_class == 'multinomial':
-        # scipy.optimize.minimize and newton-cg accepts only
-        # ravelled parameters.
-        if solver in ['lbfgs', 'newton-cg']:
-            w0 = w0.ravel()
+                w0[:, : coef.shape[1]] = coef
+
+    if multi_class == "multinomial":
+        if solver in ["lbfgs", "newton-cg", "newton-cholesky"]:
+            # scipy.optimize.minimize and newton-cg accept only ravelled parameters,
+            # i.e. 1d-arrays. LinearModelLoss expects classes to be contiguous and
+            # reconstructs the 2d-array via w0.reshape((n_classes, -1), order="F").
+            # As w0 is F-contiguous, ravel(order="F") also avoids a copy.
+            w0 = w0.ravel(order="F")
+        loss = LinearModelLoss(
+            base_loss=HalfMultinomialLoss(n_classes=classes.size),
+            fit_intercept=fit_intercept,
+        )
         target = Y_multi
-        if solver == 'lbfgs':
-            def func(x, *args): return _multinomial_loss_grad(x, *args)[0:2]
-        elif solver == 'newton-cg':
-            def func(x, *args): return _multinomial_loss(x, *args)[0]
-            def grad(x, *args): return _multinomial_loss_grad(x, *args)[1]
-            hess = _multinomial_grad_hess
-        warm_start_sag = {'coef': w0.T}
+        if solver == "lbfgs":
+            func = loss.loss_gradient
+        elif solver == "newton-cg":
+            func = loss.loss
+            grad = loss.gradient
+            hess = loss.gradient_hessian_product  # hess = [gradient, hessp]
+        warm_start_sag = {"coef": w0.T}
     else:
         target = y_bin
-        if solver == 'lbfgs':
-            func = _logistic_loss_and_grad
-        elif solver == 'newton-cg':
-            func = _logistic_loss
-            def grad(x, *args): return _logistic_loss_and_grad(x, *args)[1]
-            hess = _logistic_grad_hess
-        warm_start_sag = {'coef': np.expand_dims(w0, axis=1)}
+        if solver == "lbfgs":
+            loss = LinearModelLoss(
+                base_loss=HalfBinomialLoss(), fit_intercept=fit_intercept
+            )
+            func = loss.loss_gradient
+        elif solver == "newton-cg":
+            loss = LinearModelLoss(
+                base_loss=HalfBinomialLoss(), fit_intercept=fit_intercept
+            )
+            func = loss.loss
+            grad = loss.gradient
+            hess = loss.gradient_hessian_product  # hess = [gradient, hessp]
+        elif solver == "newton-cholesky":
+            loss = LinearModelLoss(
+                base_loss=HalfBinomialLoss(), fit_intercept=fit_intercept
+            )
+        warm_start_sag = {"coef": np.expand_dims(w0, axis=1)}
 
     coefs = list()
     n_iter = np.zeros(len(Cs), dtype=np.int32)
     for i, C in enumerate(Cs):
-        if solver == 'lbfgs':
+        if solver == "lbfgs":
+            l2_reg_strength = 1.0 / (C * sw_sum)
             iprint = [-1, 50, 1, 100, 101][
-                np.searchsorted(np.array([0, 1, 2, 3]), verbose)]
+                np.searchsorted(np.array([0, 1, 2, 3]), verbose)
+            ]
             opt_res = optimize.minimize(
-                func, w0, method="L-BFGS-B", jac=True,
-                args=(X, target, 1. / C, sample_weight),
-                options={"iprint": iprint, "gtol": tol, "maxiter": max_iter}
+                func,
+                w0,
+                method="L-BFGS-B",
+                jac=True,
+                args=(X, target, sample_weight, l2_reg_strength, n_threads),
+                options={
+                    "maxiter": max_iter,
+                    "maxls": 50,  # default is 20
+                    "iprint": iprint,
+                    "gtol": tol,
+                    "ftol": 64 * np.finfo(float).eps,
+                },
             )
             n_iter_i = _check_optimize_result(
-                solver, opt_res, max_iter,
-                extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)
+                solver,
+                opt_res,
+                max_iter,
+                extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,
+            )
             w0, loss = opt_res.x, opt_res.fun
-        elif solver == 'newton-cg':
-            args = (X, target, 1. / C, sample_weight)
-            w0, n_iter_i = _newton_cg(hess, func, grad, w0, args=args,
-                                      maxiter=max_iter, tol=tol)
-        elif solver == 'liblinear':
-            coef_, intercept_, n_iter_i, = _fit_liblinear(
-                X, target, C, fit_intercept, intercept_scaling, None,
-                penalty, dual, verbose, max_iter, tol, random_state,
-                sample_weight=sample_weight)
+        elif solver == "newton-cg":
+            l2_reg_strength = 1.0 / (C * sw_sum)
+            args = (X, target, sample_weight, l2_reg_strength, n_threads)
+            w0, n_iter_i = _newton_cg(
+                grad_hess=hess,
+                func=func,
+                grad=grad,
+                x0=w0,
+                args=args,
+                maxiter=max_iter,
+                tol=tol,
+                verbose=verbose,
+            )
+        elif solver == "newton-cholesky":
+            l2_reg_strength = 1.0 / (C * sw_sum)
+            sol = NewtonCholeskySolver(
+                coef=w0,
+                linear_loss=loss,
+                l2_reg_strength=l2_reg_strength,
+                tol=tol,
+                max_iter=max_iter,
+                n_threads=n_threads,
+                verbose=verbose,
+            )
+            w0 = sol.solve(X=X, y=target, sample_weight=sample_weight)
+            n_iter_i = sol.iteration
+        elif solver == "liblinear":
+            if len(classes) > 2:
+                warnings.warn(
+                    "Using the 'liblinear' solver for multiclass classification is "
+                    "deprecated. An error will be raised in 1.8. Either use another "
+                    "solver which supports the multinomial loss or wrap the estimator "
+                    "in a OneVsRestClassifier to keep applying a one-versus-rest "
+                    "scheme.",
+                    FutureWarning,
+                )
+            (
+                coef_,
+                intercept_,
+                n_iter_i,
+            ) = _fit_liblinear(
+                X,
+                target,
+                C,
+                fit_intercept,
+                intercept_scaling,
+                None,
+                penalty,
+                dual,
+                verbose,
+                max_iter,
+                tol,
+                random_state,
+                sample_weight=sample_weight,
+            )
             if fit_intercept:
                 w0 = np.concatenate([coef_.ravel(), intercept_])
             else:
                 w0 = coef_.ravel()
+            # n_iter_i is an array for each class. However, `target` is always encoded
+            # in {-1, 1}, so we only take the first element of n_iter_i.
+            n_iter_i = n_iter_i.item()
 
-        elif solver in ['sag', 'saga']:
-            if multi_class == 'multinomial':
+        elif solver in ["sag", "saga"]:
+            if multi_class == "multinomial":
                 target = target.astype(X.dtype, copy=False)
-                loss = 'multinomial'
+                loss = "multinomial"
             else:
-                loss = 'log'
+                loss = "log"
             # alpha is for L2-norm, beta is for L1-norm
-            if penalty == 'l1':
-                alpha = 0.
-                beta = 1. / C
-            elif penalty == 'l2':
-                alpha = 1. / C
-                beta = 0.
+            if penalty == "l1":
+                alpha = 0.0
+                beta = 1.0 / C
+            elif penalty == "l2":
+                alpha = 1.0 / C
+                beta = 0.0
             else:  # Elastic-Net penalty
-                alpha = (1. / C) * (1 - l1_ratio)
-                beta = (1. / C) * l1_ratio
+                alpha = (1.0 / C) * (1 - l1_ratio)
+                beta = (1.0 / C) * l1_ratio
 
             w0, n_iter_i, warm_start_sag = sag_solver(
-                X, target, sample_weight, loss, alpha,
-                beta, max_iter, tol,
-                verbose, random_state, False, max_squared_sum, warm_start_sag,
-                is_saga=(solver == 'saga'))
+                X,
+                target,
+                sample_weight,
+                loss,
+                alpha,
+                beta,
+                max_iter,
+                tol,
+                verbose,
+                random_state,
+                False,
+                max_squared_sum,
+                warm_start_sag,
+                is_saga=(solver == "saga"),
+            )
 
         else:
-            raise ValueError("solver must be one of {'liblinear', 'lbfgs', "
-                             "'newton-cg', 'sag'}, got '%s' instead" % solver)
+            raise ValueError(
+                "solver must be one of {'liblinear', 'lbfgs', "
+                "'newton-cg', 'sag'}, got '%s' instead" % solver
+            )
 
-        if multi_class == 'multinomial':
+        if multi_class == "multinomial":
             n_classes = max(2, classes.size)
-            multi_w0 = np.reshape(w0, (n_classes, -1))
+            if solver in ["lbfgs", "newton-cg", "newton-cholesky"]:
+                multi_w0 = np.reshape(w0, (n_classes, -1), order="F")
+            else:
+                multi_w0 = w0
             if n_classes == 2:
                 multi_w0 = multi_w0[1][np.newaxis, :]
             coefs.append(multi_w0.copy())
@@ -820,14 +595,31 @@ def grad(x, *args): return _logistic_loss_and_grad(x, *args)[1]
 
 
 # helper function for LogisticCV
-def _log_reg_scoring_path(X, y, train, test, pos_class=None, Cs=10,
-                          scoring=None, fit_intercept=False,
-                          max_iter=100, tol=1e-4, class_weight=None,
-                          verbose=0, solver='lbfgs', penalty='l2',
-                          dual=False, intercept_scaling=1.,
-                          multi_class='auto', random_state=None,
-                          max_squared_sum=None, sample_weight=None,
-                          l1_ratio=None):
+def _log_reg_scoring_path(
+    X,
+    y,
+    train,
+    test,
+    *,
+    pos_class,
+    Cs,
+    scoring,
+    fit_intercept,
+    max_iter,
+    tol,
+    class_weight,
+    verbose,
+    solver,
+    penalty,
+    dual,
+    intercept_scaling,
+    multi_class,
+    random_state,
+    max_squared_sum,
+    sample_weight,
+    l1_ratio,
+    score_params,
+):
     """Computes scores across logistic_regression_path
 
     Parameters
@@ -844,34 +636,34 @@ def _log_reg_scoring_path(X, y, train, test, pos_class=None, Cs=10,
     test : list of indices
         The indices of the test set.
 
-    pos_class : int, default=None
+    pos_class : int
         The class with respect to which we perform a one-vs-all fit.
         If None, then it is assumed that the given problem is binary.
 
-    Cs : int or list of floats, default=10
+    Cs : int or list of floats
         Each of the values in Cs describes the inverse of
         regularization strength. If Cs is as an int, then a grid of Cs
         values are chosen in a logarithmic scale between 1e-4 and 1e4.
-        If not provided, then a fixed set of values for Cs are used.
 
-    scoring : callable, default=None
-        A string (see model evaluation documentation) or
-        a scorer callable object / function with signature
-        ``scorer(estimator, X, y)``. For a list of scoring functions
-        that can be used, look at :mod:`sklearn.metrics`. The
-        default scoring option used is accuracy_score.
+    scoring : str, callable or None
+        The scoring method to use for cross-validation. Options:
+
+        - str: see :ref:`scoring_string_names` for options.
+        - callable: a scorer callable object (e.g., function) with signature
+          ``scorer(estimator, X, y)``. See :ref:`scoring_callable` for details.
+        - `None`: :ref:`accuracy ` is used.
 
-    fit_intercept : bool, default=False
+    fit_intercept : bool
         If False, then the bias term is set to zero. Else the last
         term of each coef_ gives us the intercept.
 
-    max_iter : int, default=100
+    max_iter : int
         Maximum number of iterations for the solver.
 
-    tol : float, default=1e-4
+    tol : float
         Tolerance for stopping criteria.
 
-    class_weight : dict or 'balanced', default=None
+    class_weight : dict or 'balanced'
         Weights associated with classes in the form ``{class_label: weight}``.
         If not given, all classes are supposed to have weight one.
 
@@ -882,25 +674,24 @@ def _log_reg_scoring_path(X, y, train, test, pos_class=None, Cs=10,
         Note that these weights will be multiplied with sample_weight (passed
         through the fit method) if sample_weight is specified.
 
-    verbose : int, default=0
+    verbose : int
         For the liblinear and lbfgs solvers set verbose to any positive
         number for verbosity.
 
-    solver : {'lbfgs', 'newton-cg', 'liblinear', 'sag', 'saga'}, \
-            default='lbfgs'
+    solver : {'lbfgs', 'liblinear', 'newton-cg', 'newton-cholesky', 'sag', 'saga'}
         Decides which solver to use.
 
-    penalty : {'l1', 'l2', 'elasticnet'}, default='l2'
+    penalty : {'l1', 'l2', 'elasticnet'}
         Used to specify the norm used in the penalization. The 'newton-cg',
         'sag' and 'lbfgs' solvers support only l2 penalties. 'elasticnet' is
         only supported by the 'saga' solver.
 
-    dual : bool, default=False
+    dual : bool
         Dual or primal formulation. Dual formulation is only implemented for
         l2 penalty with liblinear solver. Prefer dual=False when
         n_samples > n_features.
 
-    intercept_scaling : float, default=1.
+    intercept_scaling : float
         Useful only when the solver 'liblinear' is used
         and self.fit_intercept is set to True. In this case, x becomes
         [x, self.intercept_scaling],
@@ -912,32 +703,35 @@ def _log_reg_scoring_path(X, y, train, test, pos_class=None, Cs=10,
         To lessen the effect of regularization on synthetic feature weight
         (and therefore on the intercept) intercept_scaling has to be increased.
 
-    multi_class : {'auto', 'ovr', 'multinomial'}, default='auto'
+    multi_class : {'auto', 'ovr', 'multinomial'}
         If the option chosen is 'ovr', then a binary problem is fit for each
         label. For 'multinomial' the loss minimised is the multinomial loss fit
         across the entire probability distribution, *even when the data is
         binary*. 'multinomial' is unavailable when solver='liblinear'.
 
-    random_state : int, RandomState instance, default=None
+    random_state : int, RandomState instance
         Used when ``solver`` == 'sag', 'saga' or 'liblinear' to shuffle the
         data. See :term:`Glossary ` for details.
 
-    max_squared_sum : float, default=None
+    max_squared_sum : float
         Maximum squared sum of X over samples. Used only in SAG solver.
         If None, it will be computed, going through all the samples.
         The value should be precomputed to speed up cross validation.
 
-    sample_weight : array-like of shape(n_samples,), default=None
+    sample_weight : array-like of shape(n_samples,)
         Array of weights that are assigned to individual samples.
         If not provided, then each sample is given unit weight.
 
-    l1_ratio : float, default=None
+    l1_ratio : float
         The Elastic-Net mixing parameter, with ``0 <= l1_ratio <= 1``. Only
         used if ``penalty='elasticnet'``. Setting ``l1_ratio=0`` is equivalent
         to using ``penalty='l2'``, while setting ``l1_ratio=1`` is equivalent
         to using ``penalty='l1'``. For ``0 < l1_ratio <1``, the penalty is a
         combination of L1 and L2.
 
+    score_params : dict
+        Parameters to pass to the `score` method of the underlying scorer.
+
     Returns
     -------
     coefs : ndarray of shape (n_cs, n_features) or (n_cs, n_features + 1)
@@ -959,68 +753,77 @@ def _log_reg_scoring_path(X, y, train, test, pos_class=None, Cs=10,
     y_train = y[train]
     y_test = y[test]
 
+    sw_train, sw_test = None, None
     if sample_weight is not None:
         sample_weight = _check_sample_weight(sample_weight, X)
-        sample_weight = sample_weight[train]
+        sw_train = sample_weight[train]
+        sw_test = sample_weight[test]
 
     coefs, Cs, n_iter = _logistic_regression_path(
-        X_train, y_train, Cs=Cs, l1_ratio=l1_ratio,
-        fit_intercept=fit_intercept, solver=solver, max_iter=max_iter,
-        class_weight=class_weight, pos_class=pos_class,
-        multi_class=multi_class, tol=tol, verbose=verbose, dual=dual,
-        penalty=penalty, intercept_scaling=intercept_scaling,
-        random_state=random_state, check_input=False,
-        max_squared_sum=max_squared_sum, sample_weight=sample_weight)
+        X_train,
+        y_train,
+        Cs=Cs,
+        l1_ratio=l1_ratio,
+        fit_intercept=fit_intercept,
+        solver=solver,
+        max_iter=max_iter,
+        class_weight=class_weight,
+        pos_class=pos_class,
+        multi_class=multi_class,
+        tol=tol,
+        verbose=verbose,
+        dual=dual,
+        penalty=penalty,
+        intercept_scaling=intercept_scaling,
+        random_state=random_state,
+        check_input=False,
+        max_squared_sum=max_squared_sum,
+        sample_weight=sw_train,
+    )
 
     log_reg = LogisticRegression(solver=solver, multi_class=multi_class)
 
     # The score method of Logistic Regression has a classes_ attribute.
-    if multi_class == 'ovr':
+    if multi_class == "ovr":
         log_reg.classes_ = np.array([-1, 1])
-    elif multi_class == 'multinomial':
+    elif multi_class == "multinomial":
         log_reg.classes_ = np.unique(y_train)
     else:
-        raise ValueError("multi_class should be either multinomial or ovr, "
-                         "got %d" % multi_class)
+        raise ValueError(
+            "multi_class should be either multinomial or ovr, got %d" % multi_class
+        )
 
     if pos_class is not None:
-        mask = (y_test == pos_class)
+        mask = y_test == pos_class
         y_test = np.ones(y_test.shape, dtype=np.float64)
-        y_test[~mask] = -1.
+        y_test[~mask] = -1.0
 
     scores = list()
 
     scoring = get_scorer(scoring)
     for w in coefs:
-        if multi_class == 'ovr':
+        if multi_class == "ovr":
             w = w[np.newaxis, :]
         if fit_intercept:
             log_reg.coef_ = w[:, :-1]
             log_reg.intercept_ = w[:, -1]
         else:
             log_reg.coef_ = w
-            log_reg.intercept_ = 0.
+            log_reg.intercept_ = 0.0
 
         if scoring is None:
-            scores.append(log_reg.score(X_test, y_test))
+            scores.append(log_reg.score(X_test, y_test, sample_weight=sw_test))
         else:
-            scores.append(scoring(log_reg, X_test, y_test))
-
+            score_params = score_params or {}
+            score_params = _check_method_params(X=X, params=score_params, indices=test)
+            scores.append(scoring(log_reg, X_test, y_test, **score_params))
     return coefs, Cs, np.array(scores), n_iter
 
 
-class LogisticRegression(LinearClassifierMixin,
-                         SparseCoefMixin,
-                         BaseEstimator):
+class LogisticRegression(LinearClassifierMixin, SparseCoefMixin, BaseEstimator):
     """
     Logistic Regression (aka logit, MaxEnt) classifier.
 
-    In the multiclass case, the training algorithm uses the one-vs-rest (OvR)
-    scheme if the 'multi_class' option is set to 'ovr', and uses the
-    cross-entropy loss if the 'multi_class' option is set to 'multinomial'.
-    (Currently the 'multinomial' option is supported only by the 'lbfgs',
-    'sag', 'saga' and 'newton-cg' solvers.)
-
     This class implements regularized logistic regression using the
     'liblinear' library, 'newton-cg', 'sag', 'saga' and 'lbfgs' solvers. **Note
     that regularization is applied by default**. It can handle both dense
@@ -1034,22 +837,35 @@ class LogisticRegression(LinearClassifierMixin,
     the L2 penalty. The Elastic-Net regularization is only supported by the
     'saga' solver.
 
+    For :term:`multiclass` problems, all solvers but 'liblinear' optimize the
+    (penalized) multinomial loss. 'liblinear' only handle binary classification but can
+    be extended to handle multiclass by using
+    :class:`~sklearn.multiclass.OneVsRestClassifier`.
+
     Read more in the :ref:`User Guide `.
 
     Parameters
     ----------
-    penalty : {'l1', 'l2', 'elasticnet', 'none'}, default='l2'
-        Used to specify the norm used in the penalization. The 'newton-cg',
-        'sag' and 'lbfgs' solvers support only l2 penalties. 'elasticnet' is
-        only supported by the 'saga' solver. If 'none' (not supported by the
-        liblinear solver), no regularization is applied.
+    penalty : {'l1', 'l2', 'elasticnet', None}, default='l2'
+        Specify the norm of the penalty:
+
+        - `None`: no penalty is added;
+        - `'l2'`: add a L2 penalty term and it is the default choice;
+        - `'l1'`: add a L1 penalty term;
+        - `'elasticnet'`: both L1 and L2 penalty terms are added.
+
+        .. warning::
+           Some penalties may not work with some solvers. See the parameter
+           `solver` below, to know the compatibility between the penalty and
+           solver.
 
         .. versionadded:: 0.19
            l1 penalty with SAGA solver (allowing 'multinomial' + L1)
 
     dual : bool, default=False
-        Dual or primal formulation. Dual formulation is only implemented for
-        l2 penalty with liblinear solver. Prefer dual=False when
+        Dual (constrained) or primal (regularized, see also
+        :ref:`this equation `) formulation. Dual formulation
+        is only implemented for l2 penalty with liblinear solver. Prefer dual=False when
         n_samples > n_features.
 
     tol : float, default=1e-4
@@ -1095,31 +911,60 @@ class LogisticRegression(LinearClassifierMixin,
         Used when ``solver`` == 'sag', 'saga' or 'liblinear' to shuffle the
         data. See :term:`Glossary ` for details.
 
-    solver : {'newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'}, \
+    solver : {'lbfgs', 'liblinear', 'newton-cg', 'newton-cholesky', 'sag', 'saga'}, \
             default='lbfgs'
 
-        Algorithm to use in the optimization problem.
-
-        - For small datasets, 'liblinear' is a good choice, whereas 'sag' and
-          'saga' are faster for large ones.
-        - For multiclass problems, only 'newton-cg', 'sag', 'saga' and 'lbfgs'
-          handle multinomial loss; 'liblinear' is limited to one-versus-rest
-          schemes.
-        - 'newton-cg', 'lbfgs', 'sag' and 'saga' handle L2 or no penalty
-        - 'liblinear' and 'saga' also handle L1 penalty
-        - 'saga' also supports 'elasticnet' penalty
-        - 'liblinear' does not support setting ``penalty='none'``
-
-        Note that 'sag' and 'saga' fast convergence is only guaranteed on
-        features with approximately the same scale. You can
-        preprocess the data with a scaler from sklearn.preprocessing.
+        Algorithm to use in the optimization problem. Default is 'lbfgs'.
+        To choose a solver, you might want to consider the following aspects:
+
+        - For small datasets, 'liblinear' is a good choice, whereas 'sag'
+          and 'saga' are faster for large ones;
+        - For :term:`multiclass` problems, all solvers except 'liblinear' minimize the
+          full multinomial loss;
+        - 'liblinear' can only handle binary classification by default. To apply a
+          one-versus-rest scheme for the multiclass setting one can wrap it with the
+          :class:`~sklearn.multiclass.OneVsRestClassifier`.
+        - 'newton-cholesky' is a good choice for
+          `n_samples` >> `n_features * n_classes`, especially with one-hot encoded
+          categorical features with rare categories. Be aware that the memory usage
+          of this solver has a quadratic dependency on `n_features * n_classes`
+          because it explicitly computes the full Hessian matrix.
+
+        .. warning::
+           The choice of the algorithm depends on the penalty chosen and on
+           (multinomial) multiclass support:
+
+           ================= ============================== ======================
+           solver            penalty                        multinomial multiclass
+           ================= ============================== ======================
+           'lbfgs'           'l2', None                     yes
+           'liblinear'       'l1', 'l2'                     no
+           'newton-cg'       'l2', None                     yes
+           'newton-cholesky' 'l2', None                     yes
+           'sag'             'l2', None                     yes
+           'saga'            'elasticnet', 'l1', 'l2', None yes
+           ================= ============================== ======================
+
+        .. note::
+           'sag' and 'saga' fast convergence is only guaranteed on features
+           with approximately the same scale. You can preprocess the data with
+           a scaler from :mod:`sklearn.preprocessing`.
+
+        .. seealso::
+           Refer to the :ref:`User Guide ` for more
+           information regarding :class:`LogisticRegression` and more specifically the
+           :ref:`Table `
+           summarizing solver/penalty supports.
 
         .. versionadded:: 0.17
-           Stochastic Average Gradient descent solver.
+           Stochastic Average Gradient (SAG) descent solver. Multinomial support in
+           version 0.18.
         .. versionadded:: 0.19
            SAGA solver.
         .. versionchanged:: 0.22
-            The default solver changed from 'liblinear' to 'lbfgs' in 0.22.
+           The default solver changed from 'liblinear' to 'lbfgs' in 0.22.
+        .. versionadded:: 1.2
+           newton-cholesky solver. Multinomial support in version 1.6.
 
     max_iter : int, default=100
         Maximum number of iterations taken for the solvers to converge.
@@ -1136,6 +981,13 @@ class LogisticRegression(LinearClassifierMixin,
            Stochastic Average Gradient descent solver for 'multinomial' case.
         .. versionchanged:: 0.22
             Default changed from 'ovr' to 'auto' in 0.22.
+        .. deprecated:: 1.5
+           ``multi_class`` was deprecated in version 1.5 and will be removed in 1.7.
+           From then on, the recommended 'multinomial' will always be used for
+           `n_classes >= 3`.
+           Solvers that do not support 'multinomial' will raise an error.
+           Use `sklearn.multiclass.OneVsRestClassifier(LogisticRegression())` if you
+           still want to use OvR.
 
     verbose : int, default=0
         For the liblinear and lbfgs solvers set verbose to any positive
@@ -1186,6 +1038,17 @@ class LogisticRegression(LinearClassifierMixin,
         corresponds to outcome 1 (True) and `-intercept_` corresponds to
         outcome 0 (False).
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     n_iter_ : ndarray of shape (n_classes,) or (1, )
         Actual number of iterations for all classes. If binary or multinomial,
         it returns only 1 element. For liblinear solver, only the maximum
@@ -1199,7 +1062,7 @@ class LogisticRegression(LinearClassifierMixin,
     See Also
     --------
     SGDClassifier : Incrementally trained logistic regression (when given
-        the parameter ``loss="log"``).
+        the parameter ``loss="log_loss"``).
     LogisticRegressionCV : Logistic regression with built-in cross validation.
 
     Notes
@@ -1228,9 +1091,8 @@ class LogisticRegression(LinearClassifierMixin,
         https://hal.inria.fr/hal-00860051/document
 
     SAGA -- Defazio, A., Bach F. & Lacoste-Julien S. (2014).
-        SAGA: A Fast Incremental Gradient Method With Support
-        for Non-Strongly Convex Composite Objectives
-        https://arxiv.org/abs/1407.0202
+            :arxiv:`"SAGA: A Fast Incremental Gradient Method With Support
+            for Non-Strongly Convex Composite Objectives" <1407.0202>`
 
     Hsiang-Fu Yu, Fang-Lan Huang, Chih-Jen Lin (2011). Dual coordinate descent
         methods for logistic regression and maximum entropy models.
@@ -1246,18 +1108,59 @@ class LogisticRegression(LinearClassifierMixin,
     >>> clf.predict(X[:2, :])
     array([0, 0])
     >>> clf.predict_proba(X[:2, :])
-    array([[9.8...e-01, 1.8...e-02, 1.4...e-08],
-           [9.7...e-01, 2.8...e-02, ...e-08]])
+    array([[9.82e-01, 1.82e-02, 1.44e-08],
+           [9.72e-01, 2.82e-02, 3.02e-08]])
     >>> clf.score(X, y)
-    0.97...
+    0.97
+
+    For a comparison of the LogisticRegression with other classifiers see:
+    :ref:`sphx_glr_auto_examples_classification_plot_classification_probability.py`.
     """
-    @_deprecate_positional_args
-    def __init__(self, penalty='l2', *, dual=False, tol=1e-4, C=1.0,
-                 fit_intercept=True, intercept_scaling=1, class_weight=None,
-                 random_state=None, solver='lbfgs', max_iter=100,
-                 multi_class='auto', verbose=0, warm_start=False, n_jobs=None,
-                 l1_ratio=None):
 
+    _parameter_constraints: dict = {
+        "penalty": [StrOptions({"l1", "l2", "elasticnet"}), None],
+        "dual": ["boolean"],
+        "tol": [Interval(Real, 0, None, closed="left")],
+        "C": [Interval(Real, 0, None, closed="right")],
+        "fit_intercept": ["boolean"],
+        "intercept_scaling": [Interval(Real, 0, None, closed="neither")],
+        "class_weight": [dict, StrOptions({"balanced"}), None],
+        "random_state": ["random_state"],
+        "solver": [
+            StrOptions(
+                {"lbfgs", "liblinear", "newton-cg", "newton-cholesky", "sag", "saga"}
+            )
+        ],
+        "max_iter": [Interval(Integral, 0, None, closed="left")],
+        "verbose": ["verbose"],
+        "warm_start": ["boolean"],
+        "n_jobs": [None, Integral],
+        "l1_ratio": [Interval(Real, 0, 1, closed="both"), None],
+        "multi_class": [
+            StrOptions({"auto", "ovr", "multinomial"}),
+            Hidden(StrOptions({"deprecated"})),
+        ],
+    }
+
+    def __init__(
+        self,
+        penalty="l2",
+        *,
+        dual=False,
+        tol=1e-4,
+        C=1.0,
+        fit_intercept=True,
+        intercept_scaling=1,
+        class_weight=None,
+        random_state=None,
+        solver="lbfgs",
+        max_iter=100,
+        multi_class="deprecated",
+        verbose=0,
+        warm_start=False,
+        n_jobs=None,
+        l1_ratio=None,
+    ):
         self.penalty = penalty
         self.dual = dual
         self.tol = tol
@@ -1274,6 +1177,7 @@ def __init__(self, penalty='l2', *, dual=False, tol=1e-4, C=1.0,
         self.n_jobs = n_jobs
         self.l1_ratio = l1_ratio
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y, sample_weight=None):
         """
         Fit the model according to the given training data.
@@ -1281,8 +1185,8 @@ def fit(self, X, y, sample_weight=None):
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
-            Training vector, where n_samples is the number of samples and
-            n_features is the number of features.
+            Training vector, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
 
         y : array-like of shape (n_samples,)
             Target vector relative to X.
@@ -1305,65 +1209,114 @@ def fit(self, X, y, sample_weight=None):
         """
         solver = _check_solver(self.solver, self.penalty, self.dual)
 
-        if not isinstance(self.C, numbers.Number) or self.C < 0:
-            raise ValueError("Penalty term must be positive; got (C=%r)"
-                             % self.C)
-        if self.penalty == 'elasticnet':
-            if (not isinstance(self.l1_ratio, numbers.Number) or
-                    self.l1_ratio < 0 or self.l1_ratio > 1):
-                raise ValueError("l1_ratio must be between 0 and 1;"
-                                 " got (l1_ratio=%r)" % self.l1_ratio)
-        elif self.l1_ratio is not None:
-            warnings.warn("l1_ratio parameter is only used when penalty is "
-                          "'elasticnet'. Got "
-                          "(penalty={})".format(self.penalty))
-        if self.penalty == 'none':
+        if self.penalty != "elasticnet" and self.l1_ratio is not None:
+            warnings.warn(
+                "l1_ratio parameter is only used when penalty is "
+                "'elasticnet'. Got "
+                "(penalty={})".format(self.penalty)
+            )
+
+        if self.penalty == "elasticnet" and self.l1_ratio is None:
+            raise ValueError("l1_ratio must be specified when penalty is elasticnet.")
+
+        if self.penalty is None:
             if self.C != 1.0:  # default values
                 warnings.warn(
-                    "Setting penalty='none' will ignore the C and l1_ratio "
-                    "parameters"
+                    "Setting penalty=None will ignore the C and l1_ratio parameters"
                 )
                 # Note that check for l1_ratio is done right above
             C_ = np.inf
-            penalty = 'l2'
+            penalty = "l2"
         else:
             C_ = self.C
             penalty = self.penalty
-        if not isinstance(self.max_iter, numbers.Number) or self.max_iter < 0:
-            raise ValueError("Maximum number of iteration must be positive;"
-                             " got (max_iter=%r)" % self.max_iter)
-        if not isinstance(self.tol, numbers.Number) or self.tol < 0:
-            raise ValueError("Tolerance for stopping criteria must be "
-                             "positive; got (tol=%r)" % self.tol)
-
-        if solver == 'lbfgs':
+
+        if solver == "lbfgs":
             _dtype = np.float64
         else:
             _dtype = [np.float64, np.float32]
 
-        X, y = self._validate_data(X, y, accept_sparse='csr', dtype=_dtype,
-                                   order="C",
-                                   accept_large_sparse=solver != 'liblinear')
+        X, y = validate_data(
+            self,
+            X,
+            y,
+            accept_sparse="csr",
+            dtype=_dtype,
+            order="C",
+            accept_large_sparse=solver not in ["liblinear", "sag", "saga"],
+        )
         check_classification_targets(y)
         self.classes_ = np.unique(y)
 
-        multi_class = _check_multi_class(self.multi_class, solver,
-                                         len(self.classes_))
+        # TODO(1.8) remove multi_class
+        multi_class = self.multi_class
+        if self.multi_class == "multinomial" and len(self.classes_) == 2:
+            warnings.warn(
+                (
+                    "'multi_class' was deprecated in version 1.5 and will be removed in"
+                    " 1.7. From then on, binary problems will be fit as proper binary "
+                    " logistic regression models (as if multi_class='ovr' were set)."
+                    " Leave it to its default value to avoid this warning."
+                ),
+                FutureWarning,
+            )
+        elif self.multi_class in ("multinomial", "auto"):
+            warnings.warn(
+                (
+                    "'multi_class' was deprecated in version 1.5 and will be removed in"
+                    " 1.7. From then on, it will always use 'multinomial'."
+                    " Leave it to its default value to avoid this warning."
+                ),
+                FutureWarning,
+            )
+        elif self.multi_class == "ovr":
+            warnings.warn(
+                (
+                    "'multi_class' was deprecated in version 1.5 and will be removed in"
+                    " 1.7. Use OneVsRestClassifier(LogisticRegression(..)) instead."
+                    " Leave it to its default value to avoid this warning."
+                ),
+                FutureWarning,
+            )
+        else:
+            # Set to old default value.
+            multi_class = "auto"
+        multi_class = _check_multi_class(multi_class, solver, len(self.classes_))
 
-        if solver == 'liblinear':
+        if solver == "liblinear":
+            if len(self.classes_) > 2:
+                warnings.warn(
+                    "Using the 'liblinear' solver for multiclass classification is "
+                    "deprecated. An error will be raised in 1.8. Either use another "
+                    "solver which supports the multinomial loss or wrap the estimator "
+                    "in a OneVsRestClassifier to keep applying a one-versus-rest "
+                    "scheme.",
+                    FutureWarning,
+                )
             if effective_n_jobs(self.n_jobs) != 1:
-                warnings.warn("'n_jobs' > 1 does not have any effect when"
-                              " 'solver' is set to 'liblinear'. Got 'n_jobs'"
-                              " = {}.".format(effective_n_jobs(self.n_jobs)))
-            self.coef_, self.intercept_, n_iter_ = _fit_liblinear(
-                X, y, self.C, self.fit_intercept, self.intercept_scaling,
-                self.class_weight, self.penalty, self.dual, self.verbose,
-                self.max_iter, self.tol, self.random_state,
-                sample_weight=sample_weight)
-            self.n_iter_ = np.array([n_iter_])
+                warnings.warn(
+                    "'n_jobs' > 1 does not have any effect when"
+                    " 'solver' is set to 'liblinear'. Got 'n_jobs'"
+                    " = {}.".format(effective_n_jobs(self.n_jobs))
+                )
+            self.coef_, self.intercept_, self.n_iter_ = _fit_liblinear(
+                X,
+                y,
+                self.C,
+                self.fit_intercept,
+                self.intercept_scaling,
+                self.class_weight,
+                self.penalty,
+                self.dual,
+                self.verbose,
+                self.max_iter,
+                self.tol,
+                self.random_state,
+                sample_weight=sample_weight,
+            )
             return self
 
-        if solver in ['sag', 'saga']:
+        if solver in ["sag", "saga"]:
             max_squared_sum = row_norms(X, squared=True).max()
         else:
             max_squared_sum = None
@@ -1371,25 +1324,27 @@ def fit(self, X, y, sample_weight=None):
         n_classes = len(self.classes_)
         classes_ = self.classes_
         if n_classes < 2:
-            raise ValueError("This solver needs samples of at least 2 classes"
-                             " in the data, but the data contains only one"
-                             " class: %r" % classes_[0])
+            raise ValueError(
+                "This solver needs samples of at least 2 classes"
+                " in the data, but the data contains only one"
+                " class: %r" % classes_[0]
+            )
 
         if len(self.classes_) == 2:
             n_classes = 1
             classes_ = classes_[1:]
 
         if self.warm_start:
-            warm_start_coef = getattr(self, 'coef_', None)
+            warm_start_coef = getattr(self, "coef_", None)
         else:
             warm_start_coef = None
         if warm_start_coef is not None and self.fit_intercept:
-            warm_start_coef = np.append(warm_start_coef,
-                                        self.intercept_[:, np.newaxis],
-                                        axis=1)
+            warm_start_coef = np.append(
+                warm_start_coef, self.intercept_[:, np.newaxis], axis=1
+            )
 
         # Hack so that we iterate only once for the multinomial case.
-        if multi_class == 'multinomial':
+        if multi_class == "multinomial":
             classes_ = [None]
             warm_start_coef = [warm_start_coef]
         if warm_start_coef is None:
@@ -1399,32 +1354,61 @@ def fit(self, X, y, sample_weight=None):
 
         # The SAG solver releases the GIL so it's more efficient to use
         # threads for this solver.
-        if solver in ['sag', 'saga']:
-            prefer = 'threads'
+        if solver in ["sag", "saga"]:
+            prefer = "threads"
+        else:
+            prefer = "processes"
+
+        # TODO: Refactor this to avoid joblib parallelism entirely when doing binary
+        # and multinomial multiclass classification and use joblib only for the
+        # one-vs-rest multiclass case.
+        if (
+            solver in ["lbfgs", "newton-cg", "newton-cholesky"]
+            and len(classes_) == 1
+            and effective_n_jobs(self.n_jobs) == 1
+        ):
+            # In the future, we would like n_threads = _openmp_effective_n_threads()
+            # For the time being, we just do
+            n_threads = 1
         else:
-            prefer = 'processes'
-        fold_coefs_ = Parallel(n_jobs=self.n_jobs, verbose=self.verbose,
-                               **_joblib_parallel_args(prefer=prefer))(
-            path_func(X, y, pos_class=class_, Cs=[C_],
-                      l1_ratio=self.l1_ratio, fit_intercept=self.fit_intercept,
-                      tol=self.tol, verbose=self.verbose, solver=solver,
-                      multi_class=multi_class, max_iter=self.max_iter,
-                      class_weight=self.class_weight, check_input=False,
-                      random_state=self.random_state, coef=warm_start_coef_,
-                      penalty=penalty, max_squared_sum=max_squared_sum,
-                      sample_weight=sample_weight)
-            for class_, warm_start_coef_ in zip(classes_, warm_start_coef))
+            n_threads = 1
+
+        fold_coefs_ = Parallel(n_jobs=self.n_jobs, verbose=self.verbose, prefer=prefer)(
+            path_func(
+                X,
+                y,
+                pos_class=class_,
+                Cs=[C_],
+                l1_ratio=self.l1_ratio,
+                fit_intercept=self.fit_intercept,
+                tol=self.tol,
+                verbose=self.verbose,
+                solver=solver,
+                multi_class=multi_class,
+                max_iter=self.max_iter,
+                class_weight=self.class_weight,
+                check_input=False,
+                random_state=self.random_state,
+                coef=warm_start_coef_,
+                penalty=penalty,
+                max_squared_sum=max_squared_sum,
+                sample_weight=sample_weight,
+                n_threads=n_threads,
+            )
+            for class_, warm_start_coef_ in zip(classes_, warm_start_coef)
+        )
 
         fold_coefs_, _, n_iter_ = zip(*fold_coefs_)
         self.n_iter_ = np.asarray(n_iter_, dtype=np.int32)[:, 0]
 
         n_features = X.shape[1]
-        if multi_class == 'multinomial':
+        if multi_class == "multinomial":
             self.coef_ = fold_coefs_[0][0]
         else:
             self.coef_ = np.asarray(fold_coefs_)
-            self.coef_ = self.coef_.reshape(n_classes, n_features +
-                                            int(self.fit_intercept))
+            self.coef_ = self.coef_.reshape(
+                n_classes, n_features + int(self.fit_intercept)
+            )
 
         if self.fit_intercept:
             self.intercept_ = self.coef_[:, -1]
@@ -1444,8 +1428,8 @@ def predict_proba(self, X):
         For a multi_class problem, if multi_class is set to be "multinomial"
         the softmax function is used to find the predicted probability of
         each class.
-        Else use a one-vs-rest approach, i.e calculate the probability
-        of each class assuming it to be positive using the logistic function.
+        Else use a one-vs-rest approach, i.e. calculate the probability
+        of each class assuming it to be positive using the logistic function
         and normalize these values across all the classes.
 
         Parameters
@@ -1462,9 +1446,10 @@ def predict_proba(self, X):
         """
         check_is_fitted(self)
 
-        ovr = (self.multi_class in ["ovr", "warn"] or
-               (self.multi_class == 'auto' and (self.classes_.size <= 2 or
-                                                self.solver == 'liblinear')))
+        ovr = self.multi_class in ["ovr", "warn"] or (
+            self.multi_class in ["auto", "deprecated"]
+            and (self.classes_.size <= 2 or self.solver == "liblinear")
+        )
         if ovr:
             return super()._predict_proba_lr(X)
         else:
@@ -1498,16 +1483,19 @@ def predict_log_proba(self, X):
         """
         return np.log(self.predict_proba(X))
 
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
+        return tags
 
-class LogisticRegressionCV(LogisticRegression,
-                           LinearClassifierMixin,
-                           BaseEstimator):
+
+class LogisticRegressionCV(LogisticRegression, LinearClassifierMixin, BaseEstimator):
     """Logistic Regression CV (aka logit, MaxEnt) classifier.
 
     See glossary entry for :term:`cross-validation estimator`.
 
     This class implements logistic regression using liblinear, newton-cg, sag
-    of lbfgs optimizer. The newton-cg, sag and lbfgs solvers support only L2
+    or lbfgs optimizer. The newton-cg, sag and lbfgs solvers support only L2
     regularization with primal formulation. The liblinear solver supports both
     L1 and L2 regularization, with a dual formulation only for the L2 penalty.
     Elastic-Net penalty is only supported by the saga solver.
@@ -1543,45 +1531,79 @@ class LogisticRegressionCV(LogisticRegression,
             ``cv`` default value if None changed from 3-fold to 5-fold.
 
     dual : bool, default=False
-        Dual or primal formulation. Dual formulation is only implemented for
-        l2 penalty with liblinear solver. Prefer dual=False when
+        Dual (constrained) or primal (regularized, see also
+        :ref:`this equation `) formulation. Dual formulation
+        is only implemented for l2 penalty with liblinear solver. Prefer dual=False when
         n_samples > n_features.
 
     penalty : {'l1', 'l2', 'elasticnet'}, default='l2'
-        Used to specify the norm used in the penalization. The 'newton-cg',
-        'sag' and 'lbfgs' solvers support only l2 penalties. 'elasticnet' is
-        only supported by the 'saga' solver.
+        Specify the norm of the penalty:
 
-    scoring : str or callable, default=None
-        A string (see model evaluation documentation) or
-        a scorer callable object / function with signature
-        ``scorer(estimator, X, y)``. For a list of scoring functions
-        that can be used, look at :mod:`sklearn.metrics`. The
-        default scoring option used is 'accuracy'.
+        - `'l2'`: add a L2 penalty term (used by default);
+        - `'l1'`: add a L1 penalty term;
+        - `'elasticnet'`: both L1 and L2 penalty terms are added.
 
-    solver : {'newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'}, \
-            default='lbfgs'
+        .. warning::
+           Some penalties may not work with some solvers. See the parameter
+           `solver` below, to know the compatibility between the penalty and
+           solver.
 
-        Algorithm to use in the optimization problem.
+    scoring : str or callable, default=None
+        The scoring method to use for cross-validation. Options:
 
-        - For small datasets, 'liblinear' is a good choice, whereas 'sag' and
-          'saga' are faster for large ones.
-        - For multiclass problems, only 'newton-cg', 'sag', 'saga' and 'lbfgs'
-          handle multinomial loss; 'liblinear' is limited to one-versus-rest
-          schemes.
-        - 'newton-cg', 'lbfgs' and 'sag' only handle L2 penalty, whereas
-          'liblinear' and 'saga' handle L1 penalty.
-        - 'liblinear' might be slower in LogisticRegressionCV because it does
-          not handle warm-starting.
+        - str: see :ref:`scoring_string_names` for options.
+        - callable: a scorer callable object (e.g., function) with signature
+          ``scorer(estimator, X, y)``. See :ref:`scoring_callable` for details.
+        - `None`: :ref:`accuracy ` is used.
 
-        Note that 'sag' and 'saga' fast convergence is only guaranteed on
-        features with approximately the same scale. You can preprocess the data
-        with a scaler from sklearn.preprocessing.
+    solver : {'lbfgs', 'liblinear', 'newton-cg', 'newton-cholesky', 'sag', 'saga'}, \
+            default='lbfgs'
+
+        Algorithm to use in the optimization problem. Default is 'lbfgs'.
+        To choose a solver, you might want to consider the following aspects:
+
+        - For small datasets, 'liblinear' is a good choice, whereas 'sag'
+          and 'saga' are faster for large ones;
+        - For multiclass problems, all solvers except 'liblinear' minimize the full
+          multinomial loss;
+        - 'liblinear' might be slower in :class:`LogisticRegressionCV`
+          because it does not handle warm-starting.
+        - 'liblinear' can only handle binary classification by default. To apply a
+          one-versus-rest scheme for the multiclass setting one can wrap it with the
+          :class:`~sklearn.multiclass.OneVsRestClassifier`.
+        - 'newton-cholesky' is a good choice for
+          `n_samples` >> `n_features * n_classes`, especially with one-hot encoded
+          categorical features with rare categories. Be aware that the memory usage
+          of this solver has a quadratic dependency on `n_features * n_classes`
+          because it explicitly computes the full Hessian matrix.
+
+        .. warning::
+           The choice of the algorithm depends on the penalty chosen and on
+           (multinomial) multiclass support:
+
+           ================= ============================== ======================
+           solver            penalty                        multinomial multiclass
+           ================= ============================== ======================
+           'lbfgs'           'l2'                           yes
+           'liblinear'       'l1', 'l2'                     no
+           'newton-cg'       'l2'                           yes
+           'newton-cholesky' 'l2',                          yes
+           'sag'             'l2',                          yes
+           'saga'            'elasticnet', 'l1', 'l2'       yes
+           ================= ============================== ======================
+
+        .. note::
+           'sag' and 'saga' fast convergence is only guaranteed on features
+           with approximately the same scale. You can preprocess the data with
+           a scaler from :mod:`sklearn.preprocessing`.
 
         .. versionadded:: 0.17
-           Stochastic Average Gradient descent solver.
+           Stochastic Average Gradient (SAG) descent solver. Multinomial support in
+           version 0.18.
         .. versionadded:: 0.19
            SAGA solver.
+        .. versionadded:: 1.2
+           newton-cholesky solver. Multinomial support in version 1.6.
 
     tol : float, default=1e-4
         Tolerance for stopping criteria.
@@ -1645,6 +1667,13 @@ class LogisticRegressionCV(LogisticRegression,
            Stochastic Average Gradient descent solver for 'multinomial' case.
         .. versionchanged:: 0.22
             Default changed from 'ovr' to 'auto' in 0.22.
+        .. deprecated:: 1.5
+           ``multi_class`` was deprecated in version 1.5 and will be removed in 1.7.
+           From then on, the recommended 'multinomial' will always be used for
+           `n_classes >= 3`.
+           Solvers that do not support 'multinomial' will raise an error.
+           Use `sklearn.multiclass.OneVsRestClassifier(LogisticRegressionCV())` if you
+           still want to use OvR.
 
     random_state : int, RandomState instance, default=None
         Used when `solver='sag'`, 'saga' or 'liblinear' to shuffle the data.
@@ -1702,7 +1731,7 @@ class LogisticRegressionCV(LogisticRegression,
         an OvR for the corresponding class. If the 'multi_class' option
         given is 'multinomial' then the same scores are repeated across
         all classes, since this is the multinomial class. Each dict value
-        has shape ``(n_folds, n_cs`` or ``(n_folds, n_cs, n_l1_ratios)`` if
+        has shape ``(n_folds, n_cs)`` or ``(n_folds, n_cs, n_l1_ratios)`` if
         ``penalty='elasticnet'``.
 
     C_ : ndarray of shape (n_classes,) or (n_classes - 1,)
@@ -1723,6 +1752,21 @@ class LogisticRegressionCV(LogisticRegression,
         If ``penalty='elasticnet'``, the shape is ``(n_classes, n_folds,
         n_cs, n_l1_ratios)`` or ``(1, n_folds, n_cs, n_l1_ratios)``.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    LogisticRegression : Logistic regression without tuning the
+        hyperparameter `C`.
 
     Examples
     --------
@@ -1736,18 +1780,45 @@ class LogisticRegressionCV(LogisticRegression,
     (2, 3)
     >>> clf.score(X, y)
     0.98...
+    """
 
-    See Also
-    --------
-    LogisticRegression
+    _parameter_constraints: dict = {**LogisticRegression._parameter_constraints}
 
-    """
-    @_deprecate_positional_args
-    def __init__(self, *, Cs=10, fit_intercept=True, cv=None, dual=False,
-                 penalty='l2', scoring=None, solver='lbfgs', tol=1e-4,
-                 max_iter=100, class_weight=None, n_jobs=None, verbose=0,
-                 refit=True, intercept_scaling=1., multi_class='auto',
-                 random_state=None, l1_ratios=None):
+    for param in ["C", "warm_start", "l1_ratio"]:
+        _parameter_constraints.pop(param)
+
+    _parameter_constraints.update(
+        {
+            "Cs": [Interval(Integral, 1, None, closed="left"), "array-like"],
+            "cv": ["cv_object"],
+            "scoring": [StrOptions(set(get_scorer_names())), callable, None],
+            "l1_ratios": ["array-like", None],
+            "refit": ["boolean"],
+            "penalty": [StrOptions({"l1", "l2", "elasticnet"})],
+        }
+    )
+
+    def __init__(
+        self,
+        *,
+        Cs=10,
+        fit_intercept=True,
+        cv=None,
+        dual=False,
+        penalty="l2",
+        scoring=None,
+        solver="lbfgs",
+        tol=1e-4,
+        max_iter=100,
+        class_weight=None,
+        n_jobs=None,
+        verbose=0,
+        refit=True,
+        intercept_scaling=1.0,
+        multi_class="deprecated",
+        random_state=None,
+        l1_ratios=None,
+    ):
         self.Cs = Cs
         self.fit_intercept = fit_intercept
         self.cv = cv
@@ -1766,14 +1837,15 @@ def __init__(self, *, Cs=10, fit_intercept=True, cv=None, dual=False,
         self.random_state = random_state
         self.l1_ratios = l1_ratios
 
-    def fit(self, X, y, sample_weight=None):
+    @_fit_context(prefer_skip_nested_validation=True)
+    def fit(self, X, y, sample_weight=None, **params):
         """Fit the model according to the given training data.
 
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
-            Training vector, where n_samples is the number of samples and
-            n_features is the number of features.
+            Training vector, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
 
         y : array-like of shape (n_samples,)
             Target vector relative to X.
@@ -1782,43 +1854,56 @@ def fit(self, X, y, sample_weight=None):
             Array of weights that are assigned to individual samples.
             If not provided, then each sample is given unit weight.
 
+        **params : dict
+            Parameters to pass to the underlying splitter and scorer.
+
+            .. versionadded:: 1.4
+
         Returns
         -------
         self : object
+            Fitted LogisticRegressionCV estimator.
         """
+        _raise_for_params(params, self, "fit")
+
         solver = _check_solver(self.solver, self.penalty, self.dual)
 
-        if not isinstance(self.max_iter, numbers.Number) or self.max_iter < 0:
-            raise ValueError("Maximum number of iteration must be positive;"
-                             " got (max_iter=%r)" % self.max_iter)
-        if not isinstance(self.tol, numbers.Number) or self.tol < 0:
-            raise ValueError("Tolerance for stopping criteria must be "
-                             "positive; got (tol=%r)" % self.tol)
-        if self.penalty == 'elasticnet':
-            if self.l1_ratios is None or len(self.l1_ratios) == 0 or any(
-                    (not isinstance(l1_ratio, numbers.Number) or l1_ratio < 0
-                     or l1_ratio > 1) for l1_ratio in self.l1_ratios):
-                raise ValueError("l1_ratios must be a list of numbers between "
-                                 "0 and 1; got (l1_ratios=%r)" %
-                                 self.l1_ratios)
+        if self.penalty == "elasticnet":
+            if (
+                self.l1_ratios is None
+                or len(self.l1_ratios) == 0
+                or any(
+                    (
+                        not isinstance(l1_ratio, numbers.Number)
+                        or l1_ratio < 0
+                        or l1_ratio > 1
+                    )
+                    for l1_ratio in self.l1_ratios
+                )
+            ):
+                raise ValueError(
+                    "l1_ratios must be a list of numbers between "
+                    "0 and 1; got (l1_ratios=%r)" % self.l1_ratios
+                )
             l1_ratios_ = self.l1_ratios
         else:
             if self.l1_ratios is not None:
-                warnings.warn("l1_ratios parameter is only used when penalty "
-                              "is 'elasticnet'. Got (penalty={})".format(
-                                  self.penalty))
+                warnings.warn(
+                    "l1_ratios parameter is only used when penalty "
+                    "is 'elasticnet'. Got (penalty={})".format(self.penalty)
+                )
 
             l1_ratios_ = [None]
 
-        if self.penalty == 'none':
-            raise ValueError(
-                "penalty='none' is not useful and not supported by "
-                "LogisticRegressionCV."
-            )
-
-        X, y = self._validate_data(X, y, accept_sparse='csr', dtype=np.float64,
-                                   order="C",
-                                   accept_large_sparse=solver != 'liblinear')
+        X, y = validate_data(
+            self,
+            X,
+            y,
+            accept_sparse="csr",
+            dtype=np.float64,
+            order="C",
+            accept_large_sparse=solver not in ["liblinear", "sag", "saga"],
+        )
         check_classification_targets(y)
 
         class_weight = self.class_weight
@@ -1827,32 +1912,81 @@ def fit(self, X, y, sample_weight=None):
         label_encoder = LabelEncoder().fit(y)
         y = label_encoder.transform(y)
         if isinstance(class_weight, dict):
-            class_weight = {label_encoder.transform([cls])[0]: v
-                            for cls, v in class_weight.items()}
+            class_weight = {
+                label_encoder.transform([cls])[0]: v for cls, v in class_weight.items()
+            }
 
         # The original class labels
         classes = self.classes_ = label_encoder.classes_
         encoded_labels = label_encoder.transform(label_encoder.classes_)
 
-        multi_class = _check_multi_class(self.multi_class, solver,
-                                         len(classes))
+        # TODO(1.8) remove multi_class
+        multi_class = self.multi_class
+        if self.multi_class == "multinomial" and len(self.classes_) == 2:
+            warnings.warn(
+                (
+                    "'multi_class' was deprecated in version 1.5 and will be removed in"
+                    " 1.7. From then on, binary problems will be fit as proper binary "
+                    " logistic regression models (as if multi_class='ovr' were set)."
+                    " Leave it to its default value to avoid this warning."
+                ),
+                FutureWarning,
+            )
+        elif self.multi_class in ("multinomial", "auto"):
+            warnings.warn(
+                (
+                    "'multi_class' was deprecated in version 1.5 and will be removed in"
+                    " 1.7. From then on, it will always use 'multinomial'."
+                    " Leave it to its default value to avoid this warning."
+                ),
+                FutureWarning,
+            )
+        elif self.multi_class == "ovr":
+            warnings.warn(
+                (
+                    "'multi_class' was deprecated in version 1.5 and will be removed in"
+                    " 1.7. Use OneVsRestClassifier(LogisticRegressionCV(..)) instead."
+                    " Leave it to its default value to avoid this warning."
+                ),
+                FutureWarning,
+            )
+        else:
+            # Set to old default value.
+            multi_class = "auto"
+        multi_class = _check_multi_class(multi_class, solver, len(classes))
 
-        if solver in ['sag', 'saga']:
+        if solver in ["sag", "saga"]:
             max_squared_sum = row_norms(X, squared=True).max()
         else:
             max_squared_sum = None
 
+        if _routing_enabled():
+            routed_params = process_routing(
+                self,
+                "fit",
+                sample_weight=sample_weight,
+                **params,
+            )
+        else:
+            routed_params = Bunch()
+            routed_params.splitter = Bunch(split={})
+            routed_params.scorer = Bunch(score=params)
+            if sample_weight is not None:
+                routed_params.scorer.score["sample_weight"] = sample_weight
+
         # init cross-validation generator
         cv = check_cv(self.cv, y, classifier=True)
-        folds = list(cv.split(X, y))
+        folds = list(cv.split(X, y, **routed_params.splitter.split))
 
         # Use the label encoded classes
         n_classes = len(encoded_labels)
 
         if n_classes < 2:
-            raise ValueError("This solver needs samples of at least 2 classes"
-                             " in the data, but the data contains only one"
-                             " class: %r" % classes[0])
+            raise ValueError(
+                "This solver needs samples of at least 2 classes"
+                " in the data, but the data contains only one"
+                " class: %r" % classes[0]
+            )
 
         if n_classes == 2:
             # OvR in case of binary problems is as good as fitting
@@ -1863,7 +1997,7 @@ def fit(self, X, y, sample_weight=None):
 
         # We need this hack to iterate only once over labels, in the case of
         # multi_class = multinomial, without changing the value of the labels.
-        if multi_class == 'multinomial':
+        if multi_class == "multinomial":
             iter_encoded_labels = iter_classes = [None]
         else:
             iter_encoded_labels = encoded_labels
@@ -1872,35 +2006,51 @@ def fit(self, X, y, sample_weight=None):
         # compute the class weights for the entire dataset y
         if class_weight == "balanced":
             class_weight = compute_class_weight(
-                class_weight, classes=np.arange(len(self.classes_)), y=y)
+                class_weight,
+                classes=np.arange(len(self.classes_)),
+                y=y,
+                sample_weight=sample_weight,
+            )
             class_weight = dict(enumerate(class_weight))
 
         path_func = delayed(_log_reg_scoring_path)
 
         # The SAG solver releases the GIL so it's more efficient to use
         # threads for this solver.
-        if self.solver in ['sag', 'saga']:
-            prefer = 'threads'
+        if self.solver in ["sag", "saga"]:
+            prefer = "threads"
         else:
-            prefer = 'processes'
-
-        fold_coefs_ = Parallel(n_jobs=self.n_jobs, verbose=self.verbose,
-                               **_joblib_parallel_args(prefer=prefer))(
-            path_func(X, y, train, test, pos_class=label, Cs=self.Cs,
-                      fit_intercept=self.fit_intercept, penalty=self.penalty,
-                      dual=self.dual, solver=solver, tol=self.tol,
-                      max_iter=self.max_iter, verbose=self.verbose,
-                      class_weight=class_weight, scoring=self.scoring,
-                      multi_class=multi_class,
-                      intercept_scaling=self.intercept_scaling,
-                      random_state=self.random_state,
-                      max_squared_sum=max_squared_sum,
-                      sample_weight=sample_weight,
-                      l1_ratio=l1_ratio
-                      )
+            prefer = "processes"
+
+        fold_coefs_ = Parallel(n_jobs=self.n_jobs, verbose=self.verbose, prefer=prefer)(
+            path_func(
+                X,
+                y,
+                train,
+                test,
+                pos_class=label,
+                Cs=self.Cs,
+                fit_intercept=self.fit_intercept,
+                penalty=self.penalty,
+                dual=self.dual,
+                solver=solver,
+                tol=self.tol,
+                max_iter=self.max_iter,
+                verbose=self.verbose,
+                class_weight=class_weight,
+                scoring=self.scoring,
+                multi_class=multi_class,
+                intercept_scaling=self.intercept_scaling,
+                random_state=self.random_state,
+                max_squared_sum=max_squared_sum,
+                sample_weight=sample_weight,
+                l1_ratio=l1_ratio,
+                score_params=routed_params.scorer.score,
+            )
             for label in iter_encoded_labels
             for train, test in folds
-            for l1_ratio in l1_ratios_)
+            for l1_ratio in l1_ratios_
+        )
 
         # _log_reg_scoring_path will output different shapes depending on the
         # multi_class param, so we need to reshape the outputs accordingly.
@@ -1915,30 +2065,27 @@ def fit(self, X, y, sample_weight=None):
         #  (1, n_folds, n_Cs . n_l1_ratios)
         coefs_paths, Cs, scores, n_iter_ = zip(*fold_coefs_)
         self.Cs_ = Cs[0]
-        if multi_class == 'multinomial':
+        if multi_class == "multinomial":
             coefs_paths = np.reshape(
                 coefs_paths,
-                (len(folds),  len(l1_ratios_) * len(self.Cs_), n_classes, -1)
+                (len(folds), len(l1_ratios_) * len(self.Cs_), n_classes, -1),
             )
             # equiv to coefs_paths = np.moveaxis(coefs_paths, (0, 1, 2, 3),
             #                                                 (1, 2, 0, 3))
             coefs_paths = np.swapaxes(coefs_paths, 0, 1)
             coefs_paths = np.swapaxes(coefs_paths, 0, 2)
             self.n_iter_ = np.reshape(
-                n_iter_,
-                (1, len(folds), len(self.Cs_) * len(l1_ratios_))
+                n_iter_, (1, len(folds), len(self.Cs_) * len(l1_ratios_))
             )
             # repeat same scores across all classes
             scores = np.tile(scores, (n_classes, 1, 1))
         else:
             coefs_paths = np.reshape(
                 coefs_paths,
-                (n_classes, len(folds), len(self.Cs_) * len(l1_ratios_),
-                 -1)
+                (n_classes, len(folds), len(self.Cs_) * len(l1_ratios_), -1),
             )
             self.n_iter_ = np.reshape(
-                n_iter_,
-                (n_classes, len(folds), len(self.Cs_) * len(l1_ratios_))
+                n_iter_, (n_classes, len(folds), len(self.Cs_) * len(l1_ratios_))
             )
         scores = np.reshape(scores, (n_classes, len(folds), -1))
         self.scores_ = dict(zip(classes, scores))
@@ -1949,9 +2096,9 @@ def fit(self, X, y, sample_weight=None):
         self.coef_ = np.empty((n_classes, X.shape[1]))
         self.intercept_ = np.zeros(n_classes)
         for index, (cls, encoded_label) in enumerate(
-                zip(iter_classes, iter_encoded_labels)):
-
-            if multi_class == 'ovr':
+            zip(iter_classes, iter_encoded_labels)
+        ):
+            if multi_class == "ovr":
                 scores = self.scores_[cls]
                 coefs_paths = self.coefs_paths_[cls]
             else:
@@ -1976,52 +2123,66 @@ def fit(self, X, y, sample_weight=None):
                 l1_ratio_ = l1_ratios_[best_index_l1]
                 self.l1_ratio_.append(l1_ratio_)
 
-                if multi_class == 'multinomial':
-                    coef_init = np.mean(coefs_paths[:, :, best_index, :],
-                                        axis=1)
+                if multi_class == "multinomial":
+                    coef_init = np.mean(coefs_paths[:, :, best_index, :], axis=1)
                 else:
                     coef_init = np.mean(coefs_paths[:, best_index, :], axis=0)
 
                 # Note that y is label encoded and hence pos_class must be
                 # the encoded label / None (for 'multinomial')
                 w, _, _ = _logistic_regression_path(
-                    X, y, pos_class=encoded_label, Cs=[C_], solver=solver,
-                    fit_intercept=self.fit_intercept, coef=coef_init,
-                    max_iter=self.max_iter, tol=self.tol,
+                    X,
+                    y,
+                    pos_class=encoded_label,
+                    Cs=[C_],
+                    solver=solver,
+                    fit_intercept=self.fit_intercept,
+                    coef=coef_init,
+                    max_iter=self.max_iter,
+                    tol=self.tol,
                     penalty=self.penalty,
                     class_weight=class_weight,
                     multi_class=multi_class,
                     verbose=max(0, self.verbose - 1),
                     random_state=self.random_state,
-                    check_input=False, max_squared_sum=max_squared_sum,
+                    check_input=False,
+                    max_squared_sum=max_squared_sum,
                     sample_weight=sample_weight,
-                    l1_ratio=l1_ratio_)
+                    l1_ratio=l1_ratio_,
+                )
                 w = w[0]
 
             else:
                 # Take the best scores across every fold and the average of
                 # all coefficients corresponding to the best scores.
                 best_indices = np.argmax(scores, axis=1)
-                if multi_class == 'ovr':
-                    w = np.mean([coefs_paths[i, best_indices[i], :]
-                                 for i in range(len(folds))], axis=0)
+                if multi_class == "ovr":
+                    w = np.mean(
+                        [coefs_paths[i, best_indices[i], :] for i in range(len(folds))],
+                        axis=0,
+                    )
                 else:
-                    w = np.mean([coefs_paths[:, i, best_indices[i], :]
-                                 for i in range(len(folds))], axis=0)
+                    w = np.mean(
+                        [
+                            coefs_paths[:, i, best_indices[i], :]
+                            for i in range(len(folds))
+                        ],
+                        axis=0,
+                    )
 
                 best_indices_C = best_indices % len(self.Cs_)
                 self.C_.append(np.mean(self.Cs_[best_indices_C]))
 
-                if self.penalty == 'elasticnet':
+                if self.penalty == "elasticnet":
                     best_indices_l1 = best_indices // len(self.Cs_)
                     self.l1_ratio_.append(np.mean(l1_ratios_[best_indices_l1]))
                 else:
                     self.l1_ratio_.append(None)
 
-            if multi_class == 'multinomial':
+            if multi_class == "multinomial":
                 self.C_ = np.tile(self.C_, n_classes)
                 self.l1_ratio_ = np.tile(self.l1_ratio_, n_classes)
-                self.coef_ = w[:, :X.shape[1]]
+                self.coef_ = w[:, : X.shape[1]]
                 if self.fit_intercept:
                     self.intercept_ = w[:, -1]
             else:
@@ -2047,23 +2208,26 @@ def fit(self, X, y, sample_weight=None):
             # The same goes for the other arrays
             for cls, coefs_path in self.coefs_paths_.items():
                 self.coefs_paths_[cls] = coefs_path.reshape(
-                    (len(folds), self.l1_ratios_.size, self.Cs_.size, -1))
-                self.coefs_paths_[cls] = np.transpose(self.coefs_paths_[cls],
-                                                      (0, 2, 1, 3))
+                    (len(folds), self.l1_ratios_.size, self.Cs_.size, -1)
+                )
+                self.coefs_paths_[cls] = np.transpose(
+                    self.coefs_paths_[cls], (0, 2, 1, 3)
+                )
             for cls, score in self.scores_.items():
                 self.scores_[cls] = score.reshape(
-                    (len(folds), self.l1_ratios_.size, self.Cs_.size))
+                    (len(folds), self.l1_ratios_.size, self.Cs_.size)
+                )
                 self.scores_[cls] = np.transpose(self.scores_[cls], (0, 2, 1))
 
             self.n_iter_ = self.n_iter_.reshape(
-                (-1, len(folds), self.l1_ratios_.size, self.Cs_.size))
+                (-1, len(folds), self.l1_ratios_.size, self.Cs_.size)
+            )
             self.n_iter_ = np.transpose(self.n_iter_, (0, 1, 3, 2))
 
         return self
 
-    def score(self, X, y, sample_weight=None):
-        """Returns the score using the `scoring` option on the given
-        test data and labels.
+    def score(self, X, y, sample_weight=None, **score_params):
+        """Score using the `scoring` option on the given test data and labels.
 
         Parameters
         ----------
@@ -2076,21 +2240,78 @@ def score(self, X, y, sample_weight=None):
         sample_weight : array-like of shape (n_samples,), default=None
             Sample weights.
 
+        **score_params : dict
+            Parameters to pass to the `score` method of the underlying scorer.
+
+            .. versionadded:: 1.4
+
         Returns
         -------
         score : float
-            Score of self.predict(X) wrt. y.
+            Score of self.predict(X) w.r.t. y.
+        """
+        _raise_for_params(score_params, self, "score")
+
+        scoring = self._get_scorer()
+        if _routing_enabled():
+            routed_params = process_routing(
+                self,
+                "score",
+                sample_weight=sample_weight,
+                **score_params,
+            )
+        else:
+            routed_params = Bunch()
+            routed_params.scorer = Bunch(score={})
+            if sample_weight is not None:
+                routed_params.scorer.score["sample_weight"] = sample_weight
+
+        return scoring(
+            self,
+            X,
+            y,
+            **routed_params.scorer.score,
+        )
+
+    def get_metadata_routing(self):
+        """Get metadata routing of this object.
+
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
 
+        .. versionadded:: 1.4
+
+        Returns
+        -------
+        routing : MetadataRouter
+            A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
+            routing information.
         """
-        scoring = self.scoring or 'accuracy'
-        scoring = get_scorer(scoring)
 
-        return scoring(self, X, y, sample_weight=sample_weight)
+        router = (
+            MetadataRouter(owner=self.__class__.__name__)
+            .add_self_request(self)
+            .add(
+                splitter=self.cv,
+                method_mapping=MethodMapping().add(caller="fit", callee="split"),
+            )
+            .add(
+                scorer=self._get_scorer(),
+                method_mapping=MethodMapping()
+                .add(caller="score", callee="score")
+                .add(caller="fit", callee="score"),
+            )
+        )
+        return router
+
+    def _get_scorer(self):
+        """Get the scorer based on the scoring method specified.
+        The default scoring method is `accuracy`.
+        """
+        scoring = self.scoring or "accuracy"
+        return get_scorer(scoring)
 
-    def _more_tags(self):
-        return {
-            '_xfail_checks': {
-                'check_sample_weights_invariance':
-                'zero sample_weight is not equivalent to removing samples',
-            }
-        }
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
+        return tags
diff --git a/sklearn/linear_model/_omp.py b/sklearn/linear_model/_omp.py
index 3f995f0f34318..2f4dbac2d7634 100644
--- a/sklearn/linear_model/_omp.py
+++ b/sklearn/linear_model/_omp.py
@@ -1,24 +1,30 @@
-"""Orthogonal matching pursuit algorithms
-"""
+"""Orthogonal matching pursuit algorithms"""
 
-# Author: Vlad Niculae
-#
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import warnings
 from math import sqrt
+from numbers import Integral, Real
 
 import numpy as np
 from scipy import linalg
 from scipy.linalg.lapack import get_lapack_funcs
-from joblib import Parallel
 
-from ._base import LinearModel, _pre_fit
-from ..base import RegressorMixin, MultiOutputMixin
-from ..utils import as_float_array, check_array
-from ..utils.validation import _deprecate_positional_args
-from ..utils.fixes import delayed
+from ..base import MultiOutputMixin, RegressorMixin, _fit_context
 from ..model_selection import check_cv
+from ..utils import Bunch, as_float_array, check_array
+from ..utils._param_validation import Interval, StrOptions, validate_params
+from ..utils.metadata_routing import (
+    MetadataRouter,
+    MethodMapping,
+    _raise_for_params,
+    _routing_enabled,
+    process_routing,
+)
+from ..utils.parallel import Parallel, delayed
+from ..utils.validation import validate_data
+from ._base import LinearModel, _pre_fit
 
 premature = (
     "Orthogonal matching pursuit ended prematurely due to linear"
@@ -27,8 +33,7 @@
 )
 
 
-def _cholesky_omp(X, y, n_nonzero_coefs, tol=None, copy_X=True,
-                  return_path=False):
+def _cholesky_omp(X, y, n_nonzero_coefs, tol=None, copy_X=True, return_path=False):
     """Orthogonal Matching Pursuit step using the Cholesky decomposition.
 
     Parameters
@@ -72,13 +77,13 @@ def _cholesky_omp(X, y, n_nonzero_coefs, tol=None, copy_X=True,
         Number of active features at convergence.
     """
     if copy_X:
-        X = X.copy('F')
+        X = X.copy("F")
     else:  # even if we are allowed to overwrite, still copy it if bad order
         X = np.asfortranarray(X)
 
     min_float = np.finfo(X.dtype).eps
-    nrm2, swap = linalg.get_blas_funcs(('nrm2', 'swap'), (X,))
-    potrs, = get_lapack_funcs(('potrs',), (X,))
+    nrm2, swap = linalg.get_blas_funcs(("nrm2", "swap"), (X,))
+    (potrs,) = get_lapack_funcs(("potrs",), (X,))
 
     alpha = np.dot(X.T, y)
     residual = y
@@ -103,11 +108,14 @@ def _cholesky_omp(X, y, n_nonzero_coefs, tol=None, copy_X=True,
         if n_active > 0:
             # Updates the Cholesky decomposition of X' X
             L[n_active, :n_active] = np.dot(X[:, :n_active].T, X[:, lam])
-            linalg.solve_triangular(L[:n_active, :n_active],
-                                    L[n_active, :n_active],
-                                    trans=0, lower=1,
-                                    overwrite_b=True,
-                                    check_finite=False)
+            linalg.solve_triangular(
+                L[:n_active, :n_active],
+                L[n_active, :n_active],
+                trans=0,
+                lower=1,
+                overwrite_b=True,
+                check_finite=False,
+            )
             v = nrm2(L[n_active, :n_active]) ** 2
             Lkk = linalg.norm(X[:, lam]) ** 2 - v
             if Lkk <= min_float:  # selected atoms are dependent
@@ -123,8 +131,9 @@ def _cholesky_omp(X, y, n_nonzero_coefs, tol=None, copy_X=True,
         n_active += 1
 
         # solves LL'x = X'y as a composition of two triangular systems
-        gamma, _ = potrs(L[:n_active, :n_active], alpha[:n_active], lower=True,
-                         overwrite_b=False)
+        gamma, _ = potrs(
+            L[:n_active, :n_active], alpha[:n_active], lower=True, overwrite_b=False
+        )
 
         if return_path:
             coefs[:n_active, n_active - 1] = gamma
@@ -140,8 +149,16 @@ def _cholesky_omp(X, y, n_nonzero_coefs, tol=None, copy_X=True,
         return gamma, indices[:n_active], n_active
 
 
-def _gram_omp(Gram, Xy, n_nonzero_coefs, tol_0=None, tol=None,
-              copy_Gram=True, copy_Xy=True, return_path=False):
+def _gram_omp(
+    Gram,
+    Xy,
+    n_nonzero_coefs,
+    tol_0=None,
+    tol=None,
+    copy_Gram=True,
+    copy_Xy=True,
+    return_path=False,
+):
     """Orthogonal Matching Pursuit step on a precomputed Gram matrix.
 
     This function uses the Cholesky decomposition method.
@@ -193,14 +210,14 @@ def _gram_omp(Gram, Xy, n_nonzero_coefs, tol_0=None, tol=None,
     n_active : int
         Number of active features at convergence.
     """
-    Gram = Gram.copy('F') if copy_Gram else np.asfortranarray(Gram)
+    Gram = Gram.copy("F") if copy_Gram else np.asfortranarray(Gram)
 
     if copy_Xy or not Xy.flags.writeable:
         Xy = Xy.copy()
 
     min_float = np.finfo(Gram.dtype).eps
-    nrm2, swap = linalg.get_blas_funcs(('nrm2', 'swap'), (Gram,))
-    potrs, = get_lapack_funcs(('potrs',), (Gram,))
+    nrm2, swap = linalg.get_blas_funcs(("nrm2", "swap"), (Gram,))
+    (potrs,) = get_lapack_funcs(("potrs",), (Gram,))
 
     indices = np.arange(len(Gram))  # keeping track of swapping
     alpha = Xy
@@ -213,7 +230,7 @@ def _gram_omp(Gram, Xy, n_nonzero_coefs, tol_0=None, tol=None,
 
     L = np.empty((max_features, max_features), dtype=Gram.dtype)
 
-    L[0, 0] = 1.
+    L[0, 0] = 1.0
     if return_path:
         coefs = np.empty_like(L)
 
@@ -225,11 +242,14 @@ def _gram_omp(Gram, Xy, n_nonzero_coefs, tol_0=None, tol=None,
             break
         if n_active > 0:
             L[n_active, :n_active] = Gram[lam, :n_active]
-            linalg.solve_triangular(L[:n_active, :n_active],
-                                    L[n_active, :n_active],
-                                    trans=0, lower=1,
-                                    overwrite_b=True,
-                                    check_finite=False)
+            linalg.solve_triangular(
+                L[:n_active, :n_active],
+                L[n_active, :n_active],
+                trans=0,
+                lower=1,
+                overwrite_b=True,
+                check_finite=False,
+            )
             v = nrm2(L[n_active, :n_active]) ** 2
             Lkk = Gram[lam, lam] - v
             if Lkk <= min_float:  # selected atoms are dependent
@@ -245,8 +265,9 @@ def _gram_omp(Gram, Xy, n_nonzero_coefs, tol_0=None, tol=None,
         Xy[n_active], Xy[lam] = Xy[lam], Xy[n_active]
         n_active += 1
         # solves LL'x = X'y as a composition of two triangular systems
-        gamma, _ = potrs(L[:n_active, :n_active], Xy[:n_active], lower=True,
-                         overwrite_b=False)
+        gamma, _ = potrs(
+            L[:n_active, :n_active], Xy[:n_active], lower=True, overwrite_b=False
+        )
         if return_path:
             coefs[:n_active, n_active - 1] = gamma
         beta = np.dot(Gram[:, :n_active], gamma)
@@ -266,10 +287,30 @@ def _gram_omp(Gram, Xy, n_nonzero_coefs, tol_0=None, tol=None,
         return gamma, indices[:n_active], n_active
 
 
-@_deprecate_positional_args
-def orthogonal_mp(X, y, *, n_nonzero_coefs=None, tol=None, precompute=False,
-                  copy_X=True, return_path=False,
-                  return_n_iter=False):
+@validate_params(
+    {
+        "X": ["array-like"],
+        "y": [np.ndarray],
+        "n_nonzero_coefs": [Interval(Integral, 1, None, closed="left"), None],
+        "tol": [Interval(Real, 0, None, closed="left"), None],
+        "precompute": ["boolean", StrOptions({"auto"})],
+        "copy_X": ["boolean"],
+        "return_path": ["boolean"],
+        "return_n_iter": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def orthogonal_mp(
+    X,
+    y,
+    *,
+    n_nonzero_coefs=None,
+    tol=None,
+    precompute=False,
+    copy_X=True,
+    return_path=False,
+    return_n_iter=False,
+):
     r"""Orthogonal Matching Pursuit (OMP).
 
     Solves n_targets Orthogonal Matching Pursuit problems.
@@ -286,7 +327,7 @@ def orthogonal_mp(X, y, *, n_nonzero_coefs=None, tol=None, precompute=False,
 
     Parameters
     ----------
-    X : ndarray of shape (n_samples, n_features)
+    X : array-like of shape (n_samples, n_features)
         Input data. Columns are assumed to have unit norm.
 
     y : ndarray of shape (n_samples,) or (n_samples, n_targets)
@@ -297,7 +338,7 @@ def orthogonal_mp(X, y, *, n_nonzero_coefs=None, tol=None, precompute=False,
         default) this value is set to 10% of n_features.
 
     tol : float, default=None
-        Maximum norm of the residual. If not None, overrides n_nonzero_coefs.
+        Maximum squared norm of the residual. If not None, overrides n_nonzero_coefs.
 
     precompute : 'auto' or bool, default=False
         Whether to perform precomputations. Improves performance when n_targets
@@ -321,7 +362,7 @@ def orthogonal_mp(X, y, *, n_nonzero_coefs=None, tol=None, precompute=False,
         Coefficients of the OMP solution. If `return_path=True`, this contains
         the whole coefficient path. In this case its shape is
         (n_features, n_features) or (n_features, n_targets, n_features) and
-        iterating over the last axis yields coefficients in increasing order
+        iterating over the last axis generates coefficients in increasing order
         of active features.
 
     n_iters : array-like or int
@@ -330,25 +371,35 @@ def orthogonal_mp(X, y, *, n_nonzero_coefs=None, tol=None, precompute=False,
 
     See Also
     --------
-    OrthogonalMatchingPursuit
-    orthogonal_mp_gram
-    lars_path
-    sklearn.decomposition.sparse_encode
+    OrthogonalMatchingPursuit : Orthogonal Matching Pursuit model.
+    orthogonal_mp_gram : Solve OMP problems using Gram matrix and the product X.T * y.
+    lars_path : Compute Least Angle Regression or Lasso path using LARS algorithm.
+    sklearn.decomposition.sparse_encode : Sparse coding.
 
     Notes
     -----
     Orthogonal matching pursuit was introduced in S. Mallat, Z. Zhang,
     Matching pursuits with time-frequency dictionaries, IEEE Transactions on
     Signal Processing, Vol. 41, No. 12. (December 1993), pp. 3397-3415.
-    (http://blanche.polytechnique.fr/~mallat/papiers/MallatPursuit93.pdf)
+    (https://www.di.ens.fr/~mallat/papiers/MallatPursuit93.pdf)
 
     This implementation is based on Rubinstein, R., Zibulevsky, M. and Elad,
     M., Efficient Implementation of the K-SVD Algorithm using Batch Orthogonal
     Matching Pursuit Technical Report - CS Technion, April 2008.
     https://www.cs.technion.ac.il/~ronrubin/Publications/KSVD-OMP-v2.pdf
 
+    Examples
+    --------
+    >>> from sklearn.datasets import make_regression
+    >>> from sklearn.linear_model import orthogonal_mp
+    >>> X, y = make_regression(noise=4, random_state=0)
+    >>> coef = orthogonal_mp(X, y)
+    >>> coef.shape
+    (100,)
+    >>> X[:1,] @ coef
+    array([-78.68])
     """
-    X = check_array(X, order='F', copy=copy_X)
+    X = check_array(X, order="F", copy=copy_X)
     copy_X = False
     if y.ndim == 1:
         y = y.reshape(-1, 1)
@@ -359,27 +410,30 @@ def orthogonal_mp(X, y, *, n_nonzero_coefs=None, tol=None, precompute=False,
         # default for n_nonzero_coefs is 0.1 * n_features
         # but at least one.
         n_nonzero_coefs = max(int(0.1 * X.shape[1]), 1)
-    if tol is not None and tol < 0:
-        raise ValueError("Epsilon cannot be negative")
-    if tol is None and n_nonzero_coefs <= 0:
-        raise ValueError("The number of atoms must be positive")
     if tol is None and n_nonzero_coefs > X.shape[1]:
-        raise ValueError("The number of atoms cannot be more than the number "
-                         "of features")
-    if precompute == 'auto':
+        raise ValueError(
+            "The number of atoms cannot be more than the number of features"
+        )
+    if precompute == "auto":
         precompute = X.shape[0] > X.shape[1]
     if precompute:
         G = np.dot(X.T, X)
         G = np.asfortranarray(G)
         Xy = np.dot(X.T, y)
         if tol is not None:
-            norms_squared = np.sum((y ** 2), axis=0)
+            norms_squared = np.sum((y**2), axis=0)
         else:
             norms_squared = None
-        return orthogonal_mp_gram(G, Xy, n_nonzero_coefs=n_nonzero_coefs,
-                                  tol=tol, norms_squared=norms_squared,
-                                  copy_Gram=copy_X, copy_Xy=False,
-                                  return_path=return_path)
+        return orthogonal_mp_gram(
+            G,
+            Xy,
+            n_nonzero_coefs=n_nonzero_coefs,
+            tol=tol,
+            norms_squared=norms_squared,
+            copy_Gram=copy_X,
+            copy_Xy=False,
+            return_path=return_path,
+        )
 
     if return_path:
         coef = np.zeros((X.shape[1], y.shape[1], X.shape[1]))
@@ -389,13 +443,13 @@ def orthogonal_mp(X, y, *, n_nonzero_coefs=None, tol=None, precompute=False,
 
     for k in range(y.shape[1]):
         out = _cholesky_omp(
-            X, y[:, k], n_nonzero_coefs, tol,
-            copy_X=copy_X, return_path=return_path)
+            X, y[:, k], n_nonzero_coefs, tol, copy_X=copy_X, return_path=return_path
+        )
         if return_path:
             _, idx, coefs, n_iter = out
-            coef = coef[:, :, :len(idx)]
+            coef = coef[:, :, : len(idx)]
             for n_active, x in enumerate(coefs.T):
-                coef[idx[:n_active + 1], k, n_active] = x[:n_active + 1]
+                coef[idx[: n_active + 1], k, n_active] = x[: n_active + 1]
         else:
             x, idx, n_iter = out
             coef[idx, k] = x
@@ -410,11 +464,32 @@ def orthogonal_mp(X, y, *, n_nonzero_coefs=None, tol=None, precompute=False,
         return np.squeeze(coef)
 
 
-@_deprecate_positional_args
-def orthogonal_mp_gram(Gram, Xy, *, n_nonzero_coefs=None, tol=None,
-                       norms_squared=None, copy_Gram=True,
-                       copy_Xy=True, return_path=False,
-                       return_n_iter=False):
+@validate_params(
+    {
+        "Gram": ["array-like"],
+        "Xy": ["array-like"],
+        "n_nonzero_coefs": [Interval(Integral, 0, None, closed="neither"), None],
+        "tol": [Interval(Real, 0, None, closed="left"), None],
+        "norms_squared": ["array-like", None],
+        "copy_Gram": ["boolean"],
+        "copy_Xy": ["boolean"],
+        "return_path": ["boolean"],
+        "return_n_iter": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def orthogonal_mp_gram(
+    Gram,
+    Xy,
+    *,
+    n_nonzero_coefs=None,
+    tol=None,
+    norms_squared=None,
+    copy_Gram=True,
+    copy_Xy=True,
+    return_path=False,
+    return_n_iter=False,
+):
     """Gram Orthogonal Matching Pursuit (OMP).
 
     Solves n_targets Orthogonal Matching Pursuit problems using only
@@ -424,30 +499,31 @@ def orthogonal_mp_gram(Gram, Xy, *, n_nonzero_coefs=None, tol=None,
 
     Parameters
     ----------
-    Gram : ndarray of shape (n_features, n_features)
-        Gram matrix of the input data: X.T * X.
+    Gram : array-like of shape (n_features, n_features)
+        Gram matrix of the input data: `X.T * X`.
 
-    Xy : ndarray of shape (n_features,) or (n_features, n_targets)
-        Input targets multiplied by X: X.T * y.
+    Xy : array-like of shape (n_features,) or (n_features, n_targets)
+        Input targets multiplied by `X`: `X.T * y`.
 
     n_nonzero_coefs : int, default=None
-        Desired number of non-zero entries in the solution. If None (by
+        Desired number of non-zero entries in the solution. If `None` (by
         default) this value is set to 10% of n_features.
 
     tol : float, default=None
-        Maximum norm of the residual. If not None, overrides n_nonzero_coefs.
+        Maximum squared norm of the residual. If not `None`,
+        overrides `n_nonzero_coefs`.
 
     norms_squared : array-like of shape (n_targets,), default=None
-        Squared L2 norms of the lines of y. Required if tol is not None.
+        Squared L2 norms of the lines of `y`. Required if `tol` is not None.
 
     copy_Gram : bool, default=True
-        Whether the gram matrix must be copied by the algorithm. A false
+        Whether the gram matrix must be copied by the algorithm. A `False`
         value is only helpful if it is already Fortran-ordered, otherwise a
         copy is made anyway.
 
     copy_Xy : bool, default=True
-        Whether the covariance vector Xy must be copied by the algorithm.
-        If False, it may be overwritten.
+        Whether the covariance vector `Xy` must be copied by the algorithm.
+        If `False`, it may be overwritten.
 
     return_path : bool, default=False
         Whether to return every value of the nonzero coefficients along the
@@ -461,35 +537,47 @@ def orthogonal_mp_gram(Gram, Xy, *, n_nonzero_coefs=None, tol=None,
     coef : ndarray of shape (n_features,) or (n_features, n_targets)
         Coefficients of the OMP solution. If `return_path=True`, this contains
         the whole coefficient path. In this case its shape is
-        (n_features, n_features) or (n_features, n_targets, n_features) and
+        `(n_features, n_features)` or `(n_features, n_targets, n_features)` and
         iterating over the last axis yields coefficients in increasing order
         of active features.
 
-    n_iters : array-like or int
+    n_iters : list or int
         Number of active features across every target. Returned only if
         `return_n_iter` is set to True.
 
     See Also
     --------
-    OrthogonalMatchingPursuit
-    orthogonal_mp
-    lars_path
-    sklearn.decomposition.sparse_encode
+    OrthogonalMatchingPursuit : Orthogonal Matching Pursuit model (OMP).
+    orthogonal_mp : Solves n_targets Orthogonal Matching Pursuit problems.
+    lars_path : Compute Least Angle Regression or Lasso path using
+        LARS algorithm.
+    sklearn.decomposition.sparse_encode : Generic sparse coding.
+        Each column of the result is the solution to a Lasso problem.
 
     Notes
     -----
     Orthogonal matching pursuit was introduced in G. Mallat, Z. Zhang,
     Matching pursuits with time-frequency dictionaries, IEEE Transactions on
     Signal Processing, Vol. 41, No. 12. (December 1993), pp. 3397-3415.
-    (http://blanche.polytechnique.fr/~mallat/papiers/MallatPursuit93.pdf)
+    (https://www.di.ens.fr/~mallat/papiers/MallatPursuit93.pdf)
 
     This implementation is based on Rubinstein, R., Zibulevsky, M. and Elad,
     M., Efficient Implementation of the K-SVD Algorithm using Batch Orthogonal
     Matching Pursuit Technical Report - CS Technion, April 2008.
     https://www.cs.technion.ac.il/~ronrubin/Publications/KSVD-OMP-v2.pdf
 
+    Examples
+    --------
+    >>> from sklearn.datasets import make_regression
+    >>> from sklearn.linear_model import orthogonal_mp_gram
+    >>> X, y = make_regression(noise=4, random_state=0)
+    >>> coef = orthogonal_mp_gram(X.T @ X, X.T @ y)
+    >>> coef.shape
+    (100,)
+    >>> X[:1,] @ coef
+    array([-78.68])
     """
-    Gram = check_array(Gram, order='F', copy=copy_Gram)
+    Gram = check_array(Gram, order="F", copy=copy_Gram)
     Xy = np.asarray(Xy)
     if Xy.ndim > 1 and Xy.shape[1] > 1:
         # or subsequent target will be affected
@@ -505,33 +593,41 @@ def orthogonal_mp_gram(Gram, Xy, *, n_nonzero_coefs=None, tol=None,
     if n_nonzero_coefs is None and tol is None:
         n_nonzero_coefs = int(0.1 * len(Gram))
     if tol is not None and norms_squared is None:
-        raise ValueError('Gram OMP needs the precomputed norms in order '
-                         'to evaluate the error sum of squares.')
+        raise ValueError(
+            "Gram OMP needs the precomputed norms in order "
+            "to evaluate the error sum of squares."
+        )
     if tol is not None and tol < 0:
         raise ValueError("Epsilon cannot be negative")
     if tol is None and n_nonzero_coefs <= 0:
         raise ValueError("The number of atoms must be positive")
     if tol is None and n_nonzero_coefs > len(Gram):
-        raise ValueError("The number of atoms cannot be more than the number "
-                         "of features")
+        raise ValueError(
+            "The number of atoms cannot be more than the number of features"
+        )
 
     if return_path:
-        coef = np.zeros((len(Gram), Xy.shape[1], len(Gram)))
+        coef = np.zeros((len(Gram), Xy.shape[1], len(Gram)), dtype=Gram.dtype)
     else:
-        coef = np.zeros((len(Gram), Xy.shape[1]))
+        coef = np.zeros((len(Gram), Xy.shape[1]), dtype=Gram.dtype)
 
     n_iters = []
     for k in range(Xy.shape[1]):
         out = _gram_omp(
-            Gram, Xy[:, k], n_nonzero_coefs,
-            norms_squared[k] if tol is not None else None, tol,
-            copy_Gram=copy_Gram, copy_Xy=False,
-            return_path=return_path)
+            Gram,
+            Xy[:, k],
+            n_nonzero_coefs,
+            norms_squared[k] if tol is not None else None,
+            tol,
+            copy_Gram=copy_Gram,
+            copy_Xy=False,
+            return_path=return_path,
+        )
         if return_path:
             _, idx, coefs, n_iter = out
-            coef = coef[:, :, :len(idx)]
+            coef = coef[:, :, : len(idx)]
             for n_active, x in enumerate(coefs.T):
-                coef[idx[:n_active + 1], k, n_active] = x[:n_active + 1]
+                coef[idx[: n_active + 1], k, n_active] = x[: n_active + 1]
         else:
             x, idx, n_iter = out
             coef[idx, k] = x
@@ -554,25 +650,18 @@ class OrthogonalMatchingPursuit(MultiOutputMixin, RegressorMixin, LinearModel):
     Parameters
     ----------
     n_nonzero_coefs : int, default=None
-        Desired number of non-zero entries in the solution. If None (by
-        default) this value is set to 10% of n_features.
+        Desired number of non-zero entries in the solution. Ignored if `tol` is set.
+        When `None` and `tol` is also `None`, this value is either set to 10% of
+        `n_features` or 1, whichever is greater.
 
     tol : float, default=None
-        Maximum norm of the residual. If not None, overrides n_nonzero_coefs.
+        Maximum squared norm of the residual. If not None, overrides n_nonzero_coefs.
 
     fit_intercept : bool, default=True
-        whether to calculate the intercept for this model. If set
+        Whether to calculate the intercept for this model. If set
         to false, no intercept will be used in calculations
         (i.e. data is expected to be centered).
 
-    normalize : bool, default=True
-        This parameter is ignored when ``fit_intercept`` is set to False.
-        If True, the regressors X will be normalized before regression by
-        subtracting the mean and dividing by the l2-norm.
-        If you wish to standardize, please use
-        :class:`~sklearn.preprocessing.StandardScaler` before calling ``fit``
-        on an estimator with ``normalize=False``.
-
     precompute : 'auto' or bool, default='auto'
         Whether to use a precomputed Gram and Xy matrix to speed up
         calculations. Improves performance when :term:`n_targets` or
@@ -590,53 +679,80 @@ class OrthogonalMatchingPursuit(MultiOutputMixin, RegressorMixin, LinearModel):
     n_iter_ : int or array-like
         Number of active features across every target.
 
-    n_nonzero_coefs_ : int
-        The number of non-zero coefficients in the solution. If
-        `n_nonzero_coefs` is None and `tol` is None this value is either set
+    n_nonzero_coefs_ : int or None
+        The number of non-zero coefficients in the solution or `None` when `tol` is
+        set. If `n_nonzero_coefs` is None and `tol` is None this value is either set
         to 10% of `n_features` or 1, whichever is greater.
 
-    Examples
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
     --------
-    >>> from sklearn.linear_model import OrthogonalMatchingPursuit
-    >>> from sklearn.datasets import make_regression
-    >>> X, y = make_regression(noise=4, random_state=0)
-    >>> reg = OrthogonalMatchingPursuit().fit(X, y)
-    >>> reg.score(X, y)
-    0.9991...
-    >>> reg.predict(X[:1,])
-    array([-78.3854...])
+    orthogonal_mp : Solves n_targets Orthogonal Matching Pursuit problems.
+    orthogonal_mp_gram :  Solves n_targets Orthogonal Matching Pursuit
+        problems using only the Gram matrix X.T * X and the product X.T * y.
+    lars_path : Compute Least Angle Regression or Lasso path using LARS algorithm.
+    Lars : Least Angle Regression model a.k.a. LAR.
+    LassoLars : Lasso model fit with Least Angle Regression a.k.a. Lars.
+    sklearn.decomposition.sparse_encode : Generic sparse coding.
+        Each column of the result is the solution to a Lasso problem.
+    OrthogonalMatchingPursuitCV : Cross-validated
+        Orthogonal Matching Pursuit model (OMP).
 
     Notes
     -----
     Orthogonal matching pursuit was introduced in G. Mallat, Z. Zhang,
     Matching pursuits with time-frequency dictionaries, IEEE Transactions on
     Signal Processing, Vol. 41, No. 12. (December 1993), pp. 3397-3415.
-    (http://blanche.polytechnique.fr/~mallat/papiers/MallatPursuit93.pdf)
+    (https://www.di.ens.fr/~mallat/papiers/MallatPursuit93.pdf)
 
     This implementation is based on Rubinstein, R., Zibulevsky, M. and Elad,
     M., Efficient Implementation of the K-SVD Algorithm using Batch Orthogonal
     Matching Pursuit Technical Report - CS Technion, April 2008.
     https://www.cs.technion.ac.il/~ronrubin/Publications/KSVD-OMP-v2.pdf
 
-    See Also
+    Examples
     --------
-    orthogonal_mp
-    orthogonal_mp_gram
-    lars_path
-    Lars
-    LassoLars
-    sklearn.decomposition.sparse_encode
-    OrthogonalMatchingPursuitCV
+    >>> from sklearn.linear_model import OrthogonalMatchingPursuit
+    >>> from sklearn.datasets import make_regression
+    >>> X, y = make_regression(noise=4, random_state=0)
+    >>> reg = OrthogonalMatchingPursuit().fit(X, y)
+    >>> reg.score(X, y)
+    0.9991
+    >>> reg.predict(X[:1,])
+    array([-78.3854])
     """
-    @_deprecate_positional_args
-    def __init__(self, *, n_nonzero_coefs=None, tol=None, fit_intercept=True,
-                 normalize=True, precompute='auto'):
+
+    _parameter_constraints: dict = {
+        "n_nonzero_coefs": [Interval(Integral, 1, None, closed="left"), None],
+        "tol": [Interval(Real, 0, None, closed="left"), None],
+        "fit_intercept": ["boolean"],
+        "precompute": [StrOptions({"auto"}), "boolean"],
+    }
+
+    def __init__(
+        self,
+        *,
+        n_nonzero_coefs=None,
+        tol=None,
+        fit_intercept=True,
+        precompute="auto",
+    ):
         self.n_nonzero_coefs = n_nonzero_coefs
         self.tol = tol
         self.fit_intercept = fit_intercept
-        self.normalize = normalize
         self.precompute = precompute
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y):
         """Fit the model using X, y as training data.
 
@@ -646,20 +762,19 @@ def fit(self, X, y):
             Training data.
 
         y : array-like of shape (n_samples,) or (n_samples, n_targets)
-            Target values. Will be cast to X's dtype if necessary
-
+            Target values. Will be cast to X's dtype if necessary.
 
         Returns
         -------
         self : object
-            returns an instance of self.
+            Returns an instance of self.
         """
-        X, y = self._validate_data(X, y, multi_output=True, y_numeric=True)
+        X, y = validate_data(self, X, y, multi_output=True, y_numeric=True)
         n_features = X.shape[1]
 
-        X, y, X_offset, y_offset, X_scale, Gram, Xy = \
-            _pre_fit(X, y, None, self.precompute, self.normalize,
-                     self.fit_intercept, copy=True)
+        X, y, X_offset, y_offset, X_scale, Gram, Xy = _pre_fit(
+            X, y, None, self.precompute, self.fit_intercept, copy=True
+        )
 
         if y.ndim == 1:
             y = y[:, np.newaxis]
@@ -668,29 +783,48 @@ def fit(self, X, y):
             # default for n_nonzero_coefs is 0.1 * n_features
             # but at least one.
             self.n_nonzero_coefs_ = max(int(0.1 * n_features), 1)
+        elif self.tol is not None:
+            self.n_nonzero_coefs_ = None
         else:
             self.n_nonzero_coefs_ = self.n_nonzero_coefs
 
         if Gram is False:
             coef_, self.n_iter_ = orthogonal_mp(
-                X, y, n_nonzero_coefs=self.n_nonzero_coefs_, tol=self.tol,
-                precompute=False, copy_X=True,
-                return_n_iter=True)
+                X,
+                y,
+                n_nonzero_coefs=self.n_nonzero_coefs_,
+                tol=self.tol,
+                precompute=False,
+                copy_X=True,
+                return_n_iter=True,
+            )
         else:
-            norms_sq = np.sum(y ** 2, axis=0) if self.tol is not None else None
+            norms_sq = np.sum(y**2, axis=0) if self.tol is not None else None
 
             coef_, self.n_iter_ = orthogonal_mp_gram(
-                Gram, Xy=Xy, n_nonzero_coefs=self.n_nonzero_coefs_,
-                tol=self.tol, norms_squared=norms_sq,
-                copy_Gram=True, copy_Xy=True,
-                return_n_iter=True)
+                Gram,
+                Xy=Xy,
+                n_nonzero_coefs=self.n_nonzero_coefs_,
+                tol=self.tol,
+                norms_squared=norms_sq,
+                copy_Gram=True,
+                copy_Xy=True,
+                return_n_iter=True,
+            )
         self.coef_ = coef_.T
         self._set_intercept(X_offset, y_offset, X_scale)
         return self
 
 
-def _omp_path_residues(X_train, y_train, X_test, y_test, copy=True,
-                       fit_intercept=True, normalize=True, max_iter=100):
+def _omp_path_residues(
+    X_train,
+    y_train,
+    X_test,
+    y_test,
+    copy=True,
+    fit_intercept=True,
+    max_iter=100,
+):
     """Compute the residues on left-out data for a full LARS path.
 
     Parameters
@@ -716,14 +850,6 @@ def _omp_path_residues(X_train, y_train, X_test, y_test, copy=True,
         to false, no intercept will be used in calculations
         (i.e. data is expected to be centered).
 
-    normalize : bool, default=True
-        This parameter is ignored when ``fit_intercept`` is set to False.
-        If True, the regressors X will be normalized before regression by
-        subtracting the mean and dividing by the l2-norm.
-        If you wish to standardize, please use
-        :class:`~sklearn.preprocessing.StandardScaler` before calling ``fit``
-        on an estimator with ``normalize=False``.
-
     max_iter : int, default=100
         Maximum numbers of iterations to perform, therefore maximum features
         to include. 100 by default.
@@ -750,18 +876,17 @@ def _omp_path_residues(X_train, y_train, X_test, y_test, copy=True,
         y_test = as_float_array(y_test, copy=False)
         y_test -= y_mean
 
-    if normalize:
-        norms = np.sqrt(np.sum(X_train ** 2, axis=0))
-        nonzeros = np.flatnonzero(norms)
-        X_train[:, nonzeros] /= norms[nonzeros]
-
-    coefs = orthogonal_mp(X_train, y_train, n_nonzero_coefs=max_iter, tol=None,
-                          precompute=False, copy_X=False,
-                          return_path=True)
+    coefs = orthogonal_mp(
+        X_train,
+        y_train,
+        n_nonzero_coefs=max_iter,
+        tol=None,
+        precompute=False,
+        copy_X=False,
+        return_path=True,
+    )
     if coefs.ndim == 1:
         coefs = coefs[:, np.newaxis]
-    if normalize:
-        coefs[nonzeros] /= norms[nonzeros][:, np.newaxis]
 
     return np.dot(coefs.T, X_test.T) - y_test
 
@@ -781,18 +906,10 @@ class OrthogonalMatchingPursuitCV(RegressorMixin, LinearModel):
         copy is made anyway.
 
     fit_intercept : bool, default=True
-        whether to calculate the intercept for this model. If set
+        Whether to calculate the intercept for this model. If set
         to false, no intercept will be used in calculations
         (i.e. data is expected to be centered).
 
-    normalize : bool, default=True
-        This parameter is ignored when ``fit_intercept`` is set to False.
-        If True, the regressors X will be normalized before regression by
-        subtracting the mean and dividing by the l2-norm.
-        If you wish to standardize, please use
-        :class:`~sklearn.preprocessing.StandardScaler` before calling ``fit``
-        on an estimator with ``normalize=False``.
-
     max_iter : int, default=None
         Maximum numbers of iterations to perform, therefore maximum features
         to include. 10% of ``n_features`` but at least 5 if available.
@@ -806,7 +923,7 @@ class OrthogonalMatchingPursuitCV(RegressorMixin, LinearModel):
         - :term:`CV splitter`,
         - An iterable yielding (train, test) splits as arrays of indices.
 
-        For integer/None inputs, :class:`KFold` is used.
+        For integer/None inputs, :class:`~sklearn.model_selection.KFold` is used.
 
         Refer :ref:`User Guide ` for the various
         cross-validation strategies that can be used here.
@@ -839,6 +956,36 @@ class OrthogonalMatchingPursuitCV(RegressorMixin, LinearModel):
         Number of active features across every target for the model refit with
         the best hyperparameters got by cross-validating across all folds.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    orthogonal_mp : Solves n_targets Orthogonal Matching Pursuit problems.
+    orthogonal_mp_gram : Solves n_targets Orthogonal Matching Pursuit
+        problems using only the Gram matrix X.T * X and the product X.T * y.
+    lars_path : Compute Least Angle Regression or Lasso path using LARS algorithm.
+    Lars : Least Angle Regression model a.k.a. LAR.
+    LassoLars : Lasso model fit with Least Angle Regression a.k.a. Lars.
+    OrthogonalMatchingPursuit : Orthogonal Matching Pursuit model (OMP).
+    LarsCV : Cross-validated Least Angle Regression model.
+    LassoLarsCV : Cross-validated Lasso model fit with Least Angle Regression.
+    sklearn.decomposition.sparse_encode : Generic sparse coding.
+        Each column of the result is the solution to a Lasso problem.
+
+    Notes
+    -----
+    In `fit`, once the optimal number of non-zero coefficients is found through
+    cross-validation, the model is fit again using the entire training set.
+
     Examples
     --------
     >>> from sklearn.linear_model import OrthogonalMatchingPursuitCV
@@ -847,37 +994,41 @@ class OrthogonalMatchingPursuitCV(RegressorMixin, LinearModel):
     ...                        noise=4, random_state=0)
     >>> reg = OrthogonalMatchingPursuitCV(cv=5).fit(X, y)
     >>> reg.score(X, y)
-    0.9991...
+    0.9991
     >>> reg.n_nonzero_coefs_
-    10
+    np.int64(10)
     >>> reg.predict(X[:1,])
-    array([-78.3854...])
-
-    See Also
-    --------
-    orthogonal_mp
-    orthogonal_mp_gram
-    lars_path
-    Lars
-    LassoLars
-    OrthogonalMatchingPursuit
-    LarsCV
-    LassoLarsCV
-    sklearn.decomposition.sparse_encode
-
+    array([-78.3854])
     """
-    @_deprecate_positional_args
-    def __init__(self, *, copy=True, fit_intercept=True, normalize=True,
-                 max_iter=None, cv=None, n_jobs=None, verbose=False):
+
+    _parameter_constraints: dict = {
+        "copy": ["boolean"],
+        "fit_intercept": ["boolean"],
+        "max_iter": [Interval(Integral, 0, None, closed="left"), None],
+        "cv": ["cv_object"],
+        "n_jobs": [Integral, None],
+        "verbose": ["verbose"],
+    }
+
+    def __init__(
+        self,
+        *,
+        copy=True,
+        fit_intercept=True,
+        max_iter=None,
+        cv=None,
+        n_jobs=None,
+        verbose=False,
+    ):
         self.copy = copy
         self.fit_intercept = fit_intercept
-        self.normalize = normalize
         self.max_iter = max_iter
         self.cv = cv
         self.n_jobs = n_jobs
         self.verbose = verbose
 
-    def fit(self, X, y):
+    @_fit_context(prefer_skip_nested_validation=True)
+    def fit(self, X, y, **fit_params):
         """Fit the model using X, y as training data.
 
         Parameters
@@ -888,34 +1039,83 @@ def fit(self, X, y):
         y : array-like of shape (n_samples,)
             Target values. Will be cast to X's dtype if necessary.
 
+        **fit_params : dict
+            Parameters to pass to the underlying splitter.
+
+            .. versionadded:: 1.4
+                Only available if `enable_metadata_routing=True`,
+                which can be set by using
+                ``sklearn.set_config(enable_metadata_routing=True)``.
+                See :ref:`Metadata Routing User Guide ` for
+                more details.
+
         Returns
         -------
         self : object
-            returns an instance of self.
+            Returns an instance of self.
         """
-        X, y = self._validate_data(X, y, y_numeric=True, ensure_min_features=2,
-                                   estimator=self)
-        X = as_float_array(X, copy=False, force_all_finite=False)
+        _raise_for_params(fit_params, self, "fit")
+
+        X, y = validate_data(self, X, y, y_numeric=True, ensure_min_features=2)
+        X = as_float_array(X, copy=False, ensure_all_finite=False)
         cv = check_cv(self.cv, classifier=False)
-        max_iter = (min(max(int(0.1 * X.shape[1]), 5), X.shape[1])
-                    if not self.max_iter
-                    else self.max_iter)
+        if _routing_enabled():
+            routed_params = process_routing(self, "fit", **fit_params)
+        else:
+            # TODO(SLEP6): remove when metadata routing cannot be disabled.
+            routed_params = Bunch()
+            routed_params.splitter = Bunch(split={})
+        max_iter = (
+            min(max(int(0.1 * X.shape[1]), 5), X.shape[1])
+            if not self.max_iter
+            else self.max_iter
+        )
         cv_paths = Parallel(n_jobs=self.n_jobs, verbose=self.verbose)(
             delayed(_omp_path_residues)(
-                X[train], y[train], X[test], y[test], self.copy,
-                self.fit_intercept, self.normalize, max_iter)
-            for train, test in cv.split(X))
+                X[train],
+                y[train],
+                X[test],
+                y[test],
+                self.copy,
+                self.fit_intercept,
+                max_iter,
+            )
+            for train, test in cv.split(X, **routed_params.splitter.split)
+        )
 
         min_early_stop = min(fold.shape[0] for fold in cv_paths)
-        mse_folds = np.array([(fold[:min_early_stop] ** 2).mean(axis=1)
-                              for fold in cv_paths])
+        mse_folds = np.array(
+            [(fold[:min_early_stop] ** 2).mean(axis=1) for fold in cv_paths]
+        )
         best_n_nonzero_coefs = np.argmin(mse_folds.mean(axis=0)) + 1
         self.n_nonzero_coefs_ = best_n_nonzero_coefs
-        omp = OrthogonalMatchingPursuit(n_nonzero_coefs=best_n_nonzero_coefs,
-                                        fit_intercept=self.fit_intercept,
-                                        normalize=self.normalize)
-        omp.fit(X, y)
+        omp = OrthogonalMatchingPursuit(
+            n_nonzero_coefs=best_n_nonzero_coefs,
+            fit_intercept=self.fit_intercept,
+        ).fit(X, y)
+
         self.coef_ = omp.coef_
         self.intercept_ = omp.intercept_
         self.n_iter_ = omp.n_iter_
         return self
+
+    def get_metadata_routing(self):
+        """Get metadata routing of this object.
+
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
+
+        .. versionadded:: 1.4
+
+        Returns
+        -------
+        routing : MetadataRouter
+            A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
+            routing information.
+        """
+
+        router = MetadataRouter(owner=self.__class__.__name__).add(
+            splitter=self.cv,
+            method_mapping=MethodMapping().add(caller="fit", callee="split"),
+        )
+        return router
diff --git a/sklearn/linear_model/_passive_aggressive.py b/sklearn/linear_model/_passive_aggressive.py
index f3fa17ad1325e..61eb06edae85f 100644
--- a/sklearn/linear_model/_passive_aggressive.py
+++ b/sklearn/linear_model/_passive_aggressive.py
@@ -1,20 +1,20 @@
-# Authors: Rob Zinkov, Mathieu Blondel
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-from ..utils.validation import _deprecate_positional_args
-from ._stochastic_gradient import BaseSGDClassifier
-from ._stochastic_gradient import BaseSGDRegressor
-from ._stochastic_gradient import DEFAULT_EPSILON
+from numbers import Real
+
+from ..base import _fit_context
+from ..utils._param_validation import Interval, StrOptions
+from ._stochastic_gradient import DEFAULT_EPSILON, BaseSGDClassifier, BaseSGDRegressor
 
 
 class PassiveAggressiveClassifier(BaseSGDClassifier):
-    """Passive Aggressive Classifier
+    """Passive Aggressive Classifier.
 
     Read more in the :ref:`User Guide `.
 
     Parameters
     ----------
-
     C : float, default=1.0
         Maximum step size (regularization). Defaults to 1.0.
 
@@ -25,7 +25,7 @@ class PassiveAggressiveClassifier(BaseSGDClassifier):
     max_iter : int, default=1000
         The maximum number of passes over the training data (aka epochs).
         It only impacts the behavior in the ``fit`` method, and not the
-        :meth:`partial_fit` method.
+        :meth:`~sklearn.linear_model.PassiveAggressiveClassifier.partial_fit` method.
 
         .. versionadded:: 0.19
 
@@ -36,11 +36,11 @@ class PassiveAggressiveClassifier(BaseSGDClassifier):
         .. versionadded:: 0.19
 
     early_stopping : bool, default=False
-        Whether to use early stopping to terminate training when validation.
+        Whether to use early stopping to terminate training when validation
         score is not improving. If set to True, it will automatically set aside
         a stratified fraction of training data as validation and terminate
-        training when validation score is not improving by at least tol for
-        n_iter_no_change consecutive epochs.
+        training when validation score is not improving by at least `tol` for
+        `n_iter_no_change` consecutive epochs.
 
         .. versionadded:: 0.20
 
@@ -59,10 +59,10 @@ class PassiveAggressiveClassifier(BaseSGDClassifier):
     shuffle : bool, default=True
         Whether or not the training data should be shuffled after each epoch.
 
-    verbose : integer, default=0
-        The verbosity level
+    verbose : int, default=0
+        The verbosity level.
 
-    loss : string, default="hinge"
+    loss : str, default="hinge"
         The loss function to be used:
         hinge: equivalent to PA-I in the reference paper.
         squared_hinge: equivalent to PA-II in the reference paper.
@@ -98,7 +98,7 @@ class PassiveAggressiveClassifier(BaseSGDClassifier):
 
         The "balanced" mode uses the values of y to automatically adjust
         weights inversely proportional to class frequencies in the input data
-        as ``n_samples / (n_classes * np.bincount(y))``
+        as ``n_samples / (n_classes * np.bincount(y))``.
 
         .. versionadded:: 0.17
            parameter *class_weight* to automatically weight samples.
@@ -110,36 +110,54 @@ class PassiveAggressiveClassifier(BaseSGDClassifier):
         average. So average=10 will begin averaging after seeing 10 samples.
 
         .. versionadded:: 0.19
-           parameter *average* to use weights averaging in SGD
+           parameter *average* to use weights averaging in SGD.
 
     Attributes
     ----------
-    coef_ : array, shape = [1, n_features] if n_classes == 2 else [n_classes,\
-            n_features]
+    coef_ : ndarray of shape (1, n_features) if n_classes == 2 else \
+            (n_classes, n_features)
         Weights assigned to the features.
 
-    intercept_ : array, shape = [1] if n_classes == 2 else [n_classes]
+    intercept_ : ndarray of shape (1,) if n_classes == 2 else (n_classes,)
         Constants in decision function.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     n_iter_ : int
         The actual number of iterations to reach the stopping criterion.
         For multiclass fits, it is the maximum over every binary fit.
 
-    classes_ : array of shape (n_classes,)
+    classes_ : ndarray of shape (n_classes,)
         The unique classes labels.
 
     t_ : int
         Number of weight updates performed during training.
-        Same as ``(n_iter_ * n_samples)``.
+        Same as ``(n_iter_ * n_samples + 1)``.
 
-    loss_function_ : callable
-        Loss function used by the algorithm.
+    See Also
+    --------
+    SGDClassifier : Incrementally trained logistic regression.
+    Perceptron : Linear perceptron classifier.
+
+    References
+    ----------
+    Online Passive-Aggressive Algorithms
+    
+    K. Crammer, O. Dekel, J. Keshat, S. Shalev-Shwartz, Y. Singer - JMLR (2006)
 
     Examples
     --------
     >>> from sklearn.linear_model import PassiveAggressiveClassifier
     >>> from sklearn.datasets import make_classification
-
     >>> X, y = make_classification(n_features=4, random_state=0)
     >>> clf = PassiveAggressiveClassifier(max_iter=1000, random_state=0,
     ... tol=1e-3)
@@ -151,25 +169,33 @@ class PassiveAggressiveClassifier(BaseSGDClassifier):
     [1.84127814]
     >>> print(clf.predict([[0, 0, 0, 0]]))
     [1]
-
-    See Also
-    --------
-    SGDClassifier
-    Perceptron
-
-    References
-    ----------
-    Online Passive-Aggressive Algorithms
-    
-    K. Crammer, O. Dekel, J. Keshat, S. Shalev-Shwartz, Y. Singer - JMLR (2006)
-
     """
-    @_deprecate_positional_args
-    def __init__(self, *, C=1.0, fit_intercept=True, max_iter=1000, tol=1e-3,
-                 early_stopping=False, validation_fraction=0.1,
-                 n_iter_no_change=5, shuffle=True, verbose=0, loss="hinge",
-                 n_jobs=None, random_state=None, warm_start=False,
-                 class_weight=None, average=False):
+
+    _parameter_constraints: dict = {
+        **BaseSGDClassifier._parameter_constraints,
+        "loss": [StrOptions({"hinge", "squared_hinge"})],
+        "C": [Interval(Real, 0, None, closed="right")],
+    }
+
+    def __init__(
+        self,
+        *,
+        C=1.0,
+        fit_intercept=True,
+        max_iter=1000,
+        tol=1e-3,
+        early_stopping=False,
+        validation_fraction=0.1,
+        n_iter_no_change=5,
+        shuffle=True,
+        verbose=0,
+        loss="hinge",
+        n_jobs=None,
+        random_state=None,
+        warm_start=False,
+        class_weight=None,
+        average=False,
+    ):
         super().__init__(
             penalty=None,
             fit_intercept=fit_intercept,
@@ -185,23 +211,25 @@ def __init__(self, *, C=1.0, fit_intercept=True, max_iter=1000, tol=1e-3,
             warm_start=warm_start,
             class_weight=class_weight,
             average=average,
-            n_jobs=n_jobs)
+            n_jobs=n_jobs,
+        )
 
         self.C = C
         self.loss = loss
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def partial_fit(self, X, y, classes=None):
         """Fit linear model with Passive Aggressive algorithm.
 
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
-            Subset of the training data
+            Subset of the training data.
 
-        y : numpy array of shape [n_samples]
-            Subset of the target values
+        y : array-like of shape (n_samples,)
+            Subset of the target values.
 
-        classes : array, shape = [n_classes]
+        classes : ndarray of shape (n_classes,)
             Classes across all calls to partial_fit.
             Can be obtained by via `np.unique(y_all)`, where y_all is the
             target vector of the entire dataset.
@@ -211,55 +239,80 @@ def partial_fit(self, X, y, classes=None):
 
         Returns
         -------
-        self : returns an instance of self.
+        self : object
+            Fitted estimator.
         """
-        self._validate_params(for_partial_fit=True)
-        if self.class_weight == 'balanced':
-            raise ValueError("class_weight 'balanced' is not supported for "
-                             "partial_fit. For 'balanced' weights, use "
-                             "`sklearn.utils.compute_class_weight` with "
-                             "`class_weight='balanced'`. In place of y you "
-                             "can use a large enough subset of the full "
-                             "training set target to properly estimate the "
-                             "class frequency distributions. Pass the "
-                             "resulting weights as the class_weight "
-                             "parameter.")
-        lr = "pa1" if self.loss == "hinge" else "pa2"
-        return self._partial_fit(X, y, alpha=1.0, C=self.C,
-                                 loss="hinge", learning_rate=lr, max_iter=1,
-                                 classes=classes, sample_weight=None,
-                                 coef_init=None, intercept_init=None)
+        if not hasattr(self, "classes_"):
+            self._more_validate_params(for_partial_fit=True)
+
+            if self.class_weight == "balanced":
+                raise ValueError(
+                    "class_weight 'balanced' is not supported for "
+                    "partial_fit. For 'balanced' weights, use "
+                    "`sklearn.utils.compute_class_weight` with "
+                    "`class_weight='balanced'`. In place of y you "
+                    "can use a large enough subset of the full "
+                    "training set target to properly estimate the "
+                    "class frequency distributions. Pass the "
+                    "resulting weights as the class_weight "
+                    "parameter."
+                )
 
+        lr = "pa1" if self.loss == "hinge" else "pa2"
+        return self._partial_fit(
+            X,
+            y,
+            alpha=1.0,
+            C=self.C,
+            loss="hinge",
+            learning_rate=lr,
+            max_iter=1,
+            classes=classes,
+            sample_weight=None,
+            coef_init=None,
+            intercept_init=None,
+        )
+
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y, coef_init=None, intercept_init=None):
         """Fit linear model with Passive Aggressive algorithm.
 
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
-            Training data
+            Training data.
 
-        y : numpy array of shape [n_samples]
-            Target values
+        y : array-like of shape (n_samples,)
+            Target values.
 
-        coef_init : array, shape = [n_classes,n_features]
+        coef_init : ndarray of shape (n_classes, n_features)
             The initial coefficients to warm-start the optimization.
 
-        intercept_init : array, shape = [n_classes]
+        intercept_init : ndarray of shape (n_classes,)
             The initial intercept to warm-start the optimization.
 
         Returns
         -------
-        self : returns an instance of self.
+        self : object
+            Fitted estimator.
         """
-        self._validate_params()
+        self._more_validate_params()
+
         lr = "pa1" if self.loss == "hinge" else "pa2"
-        return self._fit(X, y, alpha=1.0, C=self.C,
-                         loss="hinge", learning_rate=lr,
-                         coef_init=coef_init, intercept_init=intercept_init)
+        return self._fit(
+            X,
+            y,
+            alpha=1.0,
+            C=self.C,
+            loss="hinge",
+            learning_rate=lr,
+            coef_init=coef_init,
+            intercept_init=intercept_init,
+        )
 
 
 class PassiveAggressiveRegressor(BaseSGDRegressor):
-    """Passive Aggressive Regressor
+    """Passive Aggressive Regressor.
 
     Read more in the :ref:`User Guide `.
 
@@ -276,7 +329,7 @@ class PassiveAggressiveRegressor(BaseSGDRegressor):
     max_iter : int, default=1000
         The maximum number of passes over the training data (aka epochs).
         It only impacts the behavior in the ``fit`` method, and not the
-        :meth:`partial_fit` method.
+        :meth:`~sklearn.linear_model.PassiveAggressiveRegressor.partial_fit` method.
 
         .. versionadded:: 0.19
 
@@ -310,10 +363,10 @@ class PassiveAggressiveRegressor(BaseSGDRegressor):
     shuffle : bool, default=True
         Whether or not the training data should be shuffled after each epoch.
 
-    verbose : integer, default=0
-        The verbosity level
+    verbose : int, default=0
+        The verbosity level.
 
-    loss : string, default="epsilon_insensitive"
+    loss : str, default="epsilon_insensitive"
         The loss function to be used:
         epsilon_insensitive: equivalent to PA-I in the reference paper.
         squared_epsilon_insensitive: equivalent to PA-II in the reference
@@ -345,7 +398,7 @@ class PassiveAggressiveRegressor(BaseSGDRegressor):
         average. So average=10 will begin averaging after seeing 10 samples.
 
         .. versionadded:: 0.19
-           parameter *average* to use weights averaging in SGD
+           parameter *average* to use weights averaging in SGD.
 
     Attributes
     ----------
@@ -356,12 +409,34 @@ class PassiveAggressiveRegressor(BaseSGDRegressor):
     intercept_ : array, shape = [1] if n_classes == 2 else [n_classes]
         Constants in decision function.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     n_iter_ : int
         The actual number of iterations to reach the stopping criterion.
 
     t_ : int
         Number of weight updates performed during training.
-        Same as ``(n_iter_ * n_samples)``.
+        Same as ``(n_iter_ * n_samples + 1)``.
+
+    See Also
+    --------
+    SGDRegressor : Linear model fitted by minimizing a regularized
+        empirical loss with SGD.
+
+    References
+    ----------
+    Online Passive-Aggressive Algorithms
+    
+    K. Crammer, O. Dekel, J. Keshat, S. Shalev-Shwartz, Y. Singer - JMLR (2006).
 
     Examples
     --------
@@ -379,25 +454,33 @@ class PassiveAggressiveRegressor(BaseSGDRegressor):
     [-0.02306214]
     >>> print(regr.predict([[0, 0, 0, 0]]))
     [-0.02306214]
-
-    See Also
-    --------
-    SGDRegressor
-
-    References
-    ----------
-    Online Passive-Aggressive Algorithms
-    
-    K. Crammer, O. Dekel, J. Keshat, S. Shalev-Shwartz, Y. Singer - JMLR (2006)
-
     """
-    @_deprecate_positional_args
-    def __init__(self, *, C=1.0, fit_intercept=True, max_iter=1000, tol=1e-3,
-                 early_stopping=False, validation_fraction=0.1,
-                 n_iter_no_change=5, shuffle=True, verbose=0,
-                 loss="epsilon_insensitive", epsilon=DEFAULT_EPSILON,
-                 random_state=None, warm_start=False,
-                 average=False):
+
+    _parameter_constraints: dict = {
+        **BaseSGDRegressor._parameter_constraints,
+        "loss": [StrOptions({"epsilon_insensitive", "squared_epsilon_insensitive"})],
+        "C": [Interval(Real, 0, None, closed="right")],
+        "epsilon": [Interval(Real, 0, None, closed="left")],
+    }
+
+    def __init__(
+        self,
+        *,
+        C=1.0,
+        fit_intercept=True,
+        max_iter=1000,
+        tol=1e-3,
+        early_stopping=False,
+        validation_fraction=0.1,
+        n_iter_no_change=5,
+        shuffle=True,
+        verbose=0,
+        loss="epsilon_insensitive",
+        epsilon=DEFAULT_EPSILON,
+        random_state=None,
+        warm_start=False,
+        average=False,
+    ):
         super().__init__(
             penalty=None,
             l1_ratio=0,
@@ -413,43 +496,56 @@ def __init__(self, *, C=1.0, fit_intercept=True, max_iter=1000, tol=1e-3,
             verbose=verbose,
             random_state=random_state,
             warm_start=warm_start,
-            average=average)
+            average=average,
+        )
         self.C = C
         self.loss = loss
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def partial_fit(self, X, y):
         """Fit linear model with Passive Aggressive algorithm.
 
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
-            Subset of training data
+            Subset of training data.
 
         y : numpy array of shape [n_samples]
-            Subset of target values
+            Subset of target values.
 
         Returns
         -------
-        self : returns an instance of self.
+        self : object
+            Fitted estimator.
         """
-        self._validate_params(for_partial_fit=True)
-        lr = "pa1" if self.loss == "epsilon_insensitive" else "pa2"
-        return self._partial_fit(X, y, alpha=1.0, C=self.C,
-                                 loss="epsilon_insensitive",
-                                 learning_rate=lr, max_iter=1,
-                                 sample_weight=None,
-                                 coef_init=None, intercept_init=None)
+        if not hasattr(self, "coef_"):
+            self._more_validate_params(for_partial_fit=True)
 
+        lr = "pa1" if self.loss == "epsilon_insensitive" else "pa2"
+        return self._partial_fit(
+            X,
+            y,
+            alpha=1.0,
+            C=self.C,
+            loss="epsilon_insensitive",
+            learning_rate=lr,
+            max_iter=1,
+            sample_weight=None,
+            coef_init=None,
+            intercept_init=None,
+        )
+
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y, coef_init=None, intercept_init=None):
         """Fit linear model with Passive Aggressive algorithm.
 
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
-            Training data
+            Training data.
 
         y : numpy array of shape [n_samples]
-            Target values
+            Target values.
 
         coef_init : array, shape = [n_features]
             The initial coefficients to warm-start the optimization.
@@ -459,12 +555,19 @@ def fit(self, X, y, coef_init=None, intercept_init=None):
 
         Returns
         -------
-        self : returns an instance of self.
+        self : object
+            Fitted estimator.
         """
-        self._validate_params()
+        self._more_validate_params()
+
         lr = "pa1" if self.loss == "epsilon_insensitive" else "pa2"
-        return self._fit(X, y, alpha=1.0, C=self.C,
-                         loss="epsilon_insensitive",
-                         learning_rate=lr,
-                         coef_init=coef_init,
-                         intercept_init=intercept_init)
+        return self._fit(
+            X,
+            y,
+            alpha=1.0,
+            C=self.C,
+            loss="epsilon_insensitive",
+            learning_rate=lr,
+            coef_init=coef_init,
+            intercept_init=intercept_init,
+        )
diff --git a/sklearn/linear_model/_perceptron.py b/sklearn/linear_model/_perceptron.py
index 9d886eb1ca065..e93200ba385fa 100644
--- a/sklearn/linear_model/_perceptron.py
+++ b/sklearn/linear_model/_perceptron.py
@@ -1,12 +1,22 @@
-# Author: Mathieu Blondel
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-from ..utils.validation import _deprecate_positional_args
+from numbers import Real
+
+from ..utils._param_validation import Interval, StrOptions
 from ._stochastic_gradient import BaseSGDClassifier
 
 
 class Perceptron(BaseSGDClassifier):
-    """Perceptron
+    """Linear perceptron classifier.
+
+    The implementation is a wrapper around :class:`~sklearn.linear_model.SGDClassifier`
+    by fixing the `loss` and `learning_rate` parameters as::
+
+        SGDClassifier(loss="perceptron", learning_rate="constant")
+
+    Other available parameters are described below and are forwarded to
+    :class:`~sklearn.linear_model.SGDClassifier`.
 
     Read more in the :ref:`User Guide `.
 
@@ -38,7 +48,7 @@ class Perceptron(BaseSGDClassifier):
 
         .. versionadded:: 0.19
 
-    tol : float, default=1e-3
+    tol : float or None, default=1e-3
         The stopping criterion. If it is not None, the iterations will stop
         when (loss > previous_loss - tol).
 
@@ -48,9 +58,9 @@ class Perceptron(BaseSGDClassifier):
         Whether or not the training data should be shuffled after each epoch.
 
     verbose : int, default=0
-        The verbosity level
+        The verbosity level.
 
-    eta0 : double, default=1
+    eta0 : float, default=1
         Constant by which the updates are multiplied.
 
     n_jobs : int, default=None
@@ -60,18 +70,18 @@ class Perceptron(BaseSGDClassifier):
         ``-1`` means using all processors. See :term:`Glossary `
         for more details.
 
-    random_state : int, RandomState instance, default=None
+    random_state : int, RandomState instance or None, default=0
         Used to shuffle the training data, when ``shuffle`` is set to
         ``True``. Pass an int for reproducible output across multiple
         function calls.
         See :term:`Glossary `.
 
     early_stopping : bool, default=False
-        Whether to use early stopping to terminate training when validation.
+        Whether to use early stopping to terminate training when validation
         score is not improving. If set to True, it will automatically set aside
         a stratified fraction of training data as validation and terminate
-        training when validation score is not improving by at least tol for
-        n_iter_no_change consecutive epochs.
+        training when validation score is not improving by at least `tol` for
+        `n_iter_no_change` consecutive epochs.
 
         .. versionadded:: 0.20
 
@@ -95,7 +105,7 @@ class Perceptron(BaseSGDClassifier):
 
         The "balanced" mode uses the values of y to automatically adjust
         weights inversely proportional to class frequencies in the input data
-        as ``n_samples / (n_classes * np.bincount(y))``
+        as ``n_samples / (n_classes * np.bincount(y))``.
 
     warm_start : bool, default=False
         When set to True, reuse the solution of the previous call to fit as
@@ -114,9 +124,16 @@ class Perceptron(BaseSGDClassifier):
     intercept_ : ndarray of shape (1,) if n_classes == 2 else (n_classes,)
         Constants in decision function.
 
-    loss_function_ : concrete LossFunction
-        The function that determines the loss, or difference between the
-        output of the algorithm and the target values.
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
 
     n_iter_ : int
         The actual number of iterations to reach the stopping criterion.
@@ -124,16 +141,24 @@ class Perceptron(BaseSGDClassifier):
 
     t_ : int
         Number of weight updates performed during training.
-        Same as ``(n_iter_ * n_samples)``.
+        Same as ``(n_iter_ * n_samples + 1)``.
+
+    See Also
+    --------
+    sklearn.linear_model.SGDClassifier : Linear classifiers
+        (SVM, logistic regression, etc.) with SGD training.
 
     Notes
     -----
-
     ``Perceptron`` is a classification algorithm which shares the same
     underlying implementation with ``SGDClassifier``. In fact,
     ``Perceptron()`` is equivalent to `SGDClassifier(loss="perceptron",
     eta0=1, learning_rate="constant", penalty=None)`.
 
+    References
+    ----------
+    https://en.wikipedia.org/wiki/Perceptron and references therein.
+
     Examples
     --------
     >>> from sklearn.datasets import load_digits
@@ -144,28 +169,58 @@ class Perceptron(BaseSGDClassifier):
     Perceptron()
     >>> clf.score(X, y)
     0.939...
-
-    See Also
-    --------
-    SGDClassifier
-
-    References
-    ----------
-
-    https://en.wikipedia.org/wiki/Perceptron and references therein.
     """
-    @_deprecate_positional_args
-    def __init__(self, *, penalty=None, alpha=0.0001, l1_ratio=0.15,
-                 fit_intercept=True,
-                 max_iter=1000, tol=1e-3, shuffle=True, verbose=0, eta0=1.0,
-                 n_jobs=None, random_state=0, early_stopping=False,
-                 validation_fraction=0.1, n_iter_no_change=5,
-                 class_weight=None, warm_start=False):
+
+    _parameter_constraints: dict = {**BaseSGDClassifier._parameter_constraints}
+    _parameter_constraints.pop("loss")
+    _parameter_constraints.pop("average")
+    _parameter_constraints.update(
+        {
+            "penalty": [StrOptions({"l2", "l1", "elasticnet"}), None],
+            "alpha": [Interval(Real, 0, None, closed="left")],
+            "l1_ratio": [Interval(Real, 0, 1, closed="both")],
+            "eta0": [Interval(Real, 0, None, closed="left")],
+        }
+    )
+
+    def __init__(
+        self,
+        *,
+        penalty=None,
+        alpha=0.0001,
+        l1_ratio=0.15,
+        fit_intercept=True,
+        max_iter=1000,
+        tol=1e-3,
+        shuffle=True,
+        verbose=0,
+        eta0=1.0,
+        n_jobs=None,
+        random_state=0,
+        early_stopping=False,
+        validation_fraction=0.1,
+        n_iter_no_change=5,
+        class_weight=None,
+        warm_start=False,
+    ):
         super().__init__(
-            loss="perceptron", penalty=penalty, alpha=alpha, l1_ratio=l1_ratio,
-            fit_intercept=fit_intercept, max_iter=max_iter, tol=tol,
-            shuffle=shuffle, verbose=verbose, random_state=random_state,
-            learning_rate="constant", eta0=eta0, early_stopping=early_stopping,
+            loss="perceptron",
+            penalty=penalty,
+            alpha=alpha,
+            l1_ratio=l1_ratio,
+            fit_intercept=fit_intercept,
+            max_iter=max_iter,
+            tol=tol,
+            shuffle=shuffle,
+            verbose=verbose,
+            random_state=random_state,
+            learning_rate="constant",
+            eta0=eta0,
+            early_stopping=early_stopping,
             validation_fraction=validation_fraction,
-            n_iter_no_change=n_iter_no_change, power_t=0.5,
-            warm_start=warm_start, class_weight=class_weight, n_jobs=n_jobs)
+            n_iter_no_change=n_iter_no_change,
+            power_t=0.5,
+            warm_start=warm_start,
+            class_weight=class_weight,
+            n_jobs=n_jobs,
+        )
diff --git a/sklearn/linear_model/_quantile.py b/sklearn/linear_model/_quantile.py
new file mode 100644
index 0000000000000..446d232958e8d
--- /dev/null
+++ b/sklearn/linear_model/_quantile.py
@@ -0,0 +1,301 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+import warnings
+from numbers import Real
+
+import numpy as np
+from scipy import sparse
+from scipy.optimize import linprog
+
+from ..base import BaseEstimator, RegressorMixin, _fit_context
+from ..exceptions import ConvergenceWarning
+from ..utils import _safe_indexing
+from ..utils._param_validation import Interval, StrOptions
+from ..utils.fixes import parse_version, sp_version
+from ..utils.validation import _check_sample_weight, validate_data
+from ._base import LinearModel
+
+
+class QuantileRegressor(LinearModel, RegressorMixin, BaseEstimator):
+    """Linear regression model that predicts conditional quantiles.
+
+    The linear :class:`QuantileRegressor` optimizes the pinball loss for a
+    desired `quantile` and is robust to outliers.
+
+    This model uses an L1 regularization like
+    :class:`~sklearn.linear_model.Lasso`.
+
+    Read more in the :ref:`User Guide `.
+
+    .. versionadded:: 1.0
+
+    Parameters
+    ----------
+    quantile : float, default=0.5
+        The quantile that the model tries to predict. It must be strictly
+        between 0 and 1. If 0.5 (default), the model predicts the 50%
+        quantile, i.e. the median.
+
+    alpha : float, default=1.0
+        Regularization constant that multiplies the L1 penalty term.
+
+    fit_intercept : bool, default=True
+        Whether or not to fit the intercept.
+
+    solver : {'highs-ds', 'highs-ipm', 'highs', 'interior-point', \
+            'revised simplex'}, default='highs'
+        Method used by :func:`scipy.optimize.linprog` to solve the linear
+        programming formulation.
+
+        It is recommended to use the highs methods because
+        they are the fastest ones. Solvers "highs-ds", "highs-ipm" and "highs"
+        support sparse input data and, in fact, always convert to sparse csc.
+
+        From `scipy>=1.11.0`, "interior-point" is not available anymore.
+
+        .. versionchanged:: 1.4
+           The default of `solver` changed to `"highs"` in version 1.4.
+
+    solver_options : dict, default=None
+        Additional parameters passed to :func:`scipy.optimize.linprog` as
+        options. If `None` and if `solver='interior-point'`, then
+        `{"lstsq": True}` is passed to :func:`scipy.optimize.linprog` for the
+        sake of stability.
+
+    Attributes
+    ----------
+    coef_ : array of shape (n_features,)
+        Estimated coefficients for the features.
+
+    intercept_ : float
+        The intercept of the model, aka bias term.
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    n_iter_ : int
+        The actual number of iterations performed by the solver.
+
+    See Also
+    --------
+    Lasso : The Lasso is a linear model that estimates sparse coefficients
+        with l1 regularization.
+    HuberRegressor : Linear regression model that is robust to outliers.
+
+    Examples
+    --------
+    >>> from sklearn.linear_model import QuantileRegressor
+    >>> import numpy as np
+    >>> n_samples, n_features = 10, 2
+    >>> rng = np.random.RandomState(0)
+    >>> y = rng.randn(n_samples)
+    >>> X = rng.randn(n_samples, n_features)
+    >>> # the two following lines are optional in practice
+    >>> from sklearn.utils.fixes import sp_version, parse_version
+    >>> reg = QuantileRegressor(quantile=0.8).fit(X, y)
+    >>> np.mean(y <= reg.predict(X))
+    np.float64(0.8)
+    """
+
+    _parameter_constraints: dict = {
+        "quantile": [Interval(Real, 0, 1, closed="neither")],
+        "alpha": [Interval(Real, 0, None, closed="left")],
+        "fit_intercept": ["boolean"],
+        "solver": [
+            StrOptions(
+                {
+                    "highs-ds",
+                    "highs-ipm",
+                    "highs",
+                    "interior-point",
+                    "revised simplex",
+                }
+            ),
+        ],
+        "solver_options": [dict, None],
+    }
+
+    def __init__(
+        self,
+        *,
+        quantile=0.5,
+        alpha=1.0,
+        fit_intercept=True,
+        solver="highs",
+        solver_options=None,
+    ):
+        self.quantile = quantile
+        self.alpha = alpha
+        self.fit_intercept = fit_intercept
+        self.solver = solver
+        self.solver_options = solver_options
+
+    @_fit_context(prefer_skip_nested_validation=True)
+    def fit(self, X, y, sample_weight=None):
+        """Fit the model according to the given training data.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Training data.
+
+        y : array-like of shape (n_samples,)
+            Target values.
+
+        sample_weight : array-like of shape (n_samples,), default=None
+            Sample weights.
+
+        Returns
+        -------
+        self : object
+            Returns self.
+        """
+        X, y = validate_data(
+            self,
+            X,
+            y,
+            accept_sparse=["csc", "csr", "coo"],
+            y_numeric=True,
+            multi_output=False,
+        )
+        sample_weight = _check_sample_weight(sample_weight, X)
+
+        n_features = X.shape[1]
+        n_params = n_features
+
+        if self.fit_intercept:
+            n_params += 1
+            # Note that centering y and X with _preprocess_data does not work
+            # for quantile regression.
+
+        # The objective is defined as 1/n * sum(pinball loss) + alpha * L1.
+        # So we rescale the penalty term, which is equivalent.
+        alpha = np.sum(sample_weight) * self.alpha
+
+        if self.solver == "interior-point" and sp_version >= parse_version("1.11.0"):
+            raise ValueError(
+                f"Solver {self.solver} is not anymore available in SciPy >= 1.11.0."
+            )
+
+        if sparse.issparse(X) and self.solver not in ["highs", "highs-ds", "highs-ipm"]:
+            raise ValueError(
+                f"Solver {self.solver} does not support sparse X. "
+                "Use solver 'highs' for example."
+            )
+        # make default solver more stable
+        if self.solver_options is None and self.solver == "interior-point":
+            solver_options = {"lstsq": True}
+        else:
+            solver_options = self.solver_options
+
+        # After rescaling alpha, the minimization problem is
+        #     min sum(pinball loss) + alpha * L1
+        # Use linear programming formulation of quantile regression
+        #     min_x c x
+        #           A_eq x = b_eq
+        #                0 <= x
+        # x = (s0, s, t0, t, u, v) = slack variables >= 0
+        # intercept = s0 - t0
+        # coef = s - t
+        # c = (0, alpha * 1_p, 0, alpha * 1_p, quantile * 1_n, (1-quantile) * 1_n)
+        # residual = y - X@coef - intercept = u - v
+        # A_eq = (1_n, X, -1_n, -X, diag(1_n), -diag(1_n))
+        # b_eq = y
+        # p = n_features
+        # n = n_samples
+        # 1_n = vector of length n with entries equal one
+        # see https://stats.stackexchange.com/questions/384909/
+        #
+        # Filtering out zero sample weights from the beginning makes life
+        # easier for the linprog solver.
+        indices = np.nonzero(sample_weight)[0]
+        n_indices = len(indices)  # use n_mask instead of n_samples
+        if n_indices < len(sample_weight):
+            sample_weight = sample_weight[indices]
+            X = _safe_indexing(X, indices)
+            y = _safe_indexing(y, indices)
+        c = np.concatenate(
+            [
+                np.full(2 * n_params, fill_value=alpha),
+                sample_weight * self.quantile,
+                sample_weight * (1 - self.quantile),
+            ]
+        )
+        if self.fit_intercept:
+            # do not penalize the intercept
+            c[0] = 0
+            c[n_params] = 0
+
+        if self.solver in ["highs", "highs-ds", "highs-ipm"]:
+            # Note that highs methods always use a sparse CSC memory layout internally,
+            # even for optimization problems parametrized using dense numpy arrays.
+            # Therefore, we work with CSC matrices as early as possible to limit
+            # unnecessary repeated memory copies.
+            eye = sparse.eye(n_indices, dtype=X.dtype, format="csc")
+            if self.fit_intercept:
+                ones = sparse.csc_matrix(np.ones(shape=(n_indices, 1), dtype=X.dtype))
+                A_eq = sparse.hstack([ones, X, -ones, -X, eye, -eye], format="csc")
+            else:
+                A_eq = sparse.hstack([X, -X, eye, -eye], format="csc")
+        else:
+            eye = np.eye(n_indices)
+            if self.fit_intercept:
+                ones = np.ones((n_indices, 1))
+                A_eq = np.concatenate([ones, X, -ones, -X, eye, -eye], axis=1)
+            else:
+                A_eq = np.concatenate([X, -X, eye, -eye], axis=1)
+
+        b_eq = y
+
+        result = linprog(
+            c=c,
+            A_eq=A_eq,
+            b_eq=b_eq,
+            method=self.solver,
+            options=solver_options,
+        )
+        solution = result.x
+        if not result.success:
+            failure = {
+                1: "Iteration limit reached.",
+                2: "Problem appears to be infeasible.",
+                3: "Problem appears to be unbounded.",
+                4: "Numerical difficulties encountered.",
+            }
+            warnings.warn(
+                "Linear programming for QuantileRegressor did not succeed.\n"
+                f"Status is {result.status}: "
+                + failure.setdefault(result.status, "unknown reason")
+                + "\n"
+                + "Result message of linprog:\n"
+                + result.message,
+                ConvergenceWarning,
+            )
+
+        # positive slack - negative slack
+        # solution is an array with (params_pos, params_neg, u, v)
+        params = solution[:n_params] - solution[n_params : 2 * n_params]
+
+        self.n_iter_ = result.nit
+
+        if self.fit_intercept:
+            self.coef_ = params[1:]
+            self.intercept_ = params[0]
+        else:
+            self.coef_ = params
+            self.intercept_ = 0.0
+        return self
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
+        return tags
diff --git a/sklearn/linear_model/_ransac.py b/sklearn/linear_model/_ransac.py
index c9246c121c387..c18065436dc35 100644
--- a/sklearn/linear_model/_ransac.py
+++ b/sklearn/linear_model/_ransac.py
@@ -1,21 +1,45 @@
-# coding: utf-8
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-# Author: Johannes SchÃļnberger
-#
-# License: BSD 3 clause
+import warnings
+from numbers import Integral, Real
 
 import numpy as np
-import warnings
 
-from ..base import BaseEstimator, MetaEstimatorMixin, RegressorMixin, clone
-from ..base import MultiOutputMixin
-from ..utils import check_random_state, check_consistent_length
+from ..base import (
+    BaseEstimator,
+    MetaEstimatorMixin,
+    MultiOutputMixin,
+    RegressorMixin,
+    _fit_context,
+    clone,
+)
+from ..exceptions import ConvergenceWarning
+from ..utils import check_consistent_length, check_random_state, get_tags
+from ..utils._bunch import Bunch
+from ..utils._param_validation import (
+    HasMethods,
+    Interval,
+    Options,
+    RealNotInt,
+    StrOptions,
+)
+from ..utils.metadata_routing import (
+    MetadataRouter,
+    MethodMapping,
+    _raise_for_params,
+    _routing_enabled,
+    process_routing,
+)
 from ..utils.random import sample_without_replacement
-from ..utils.validation import check_is_fitted, _check_sample_weight
-from ..utils.validation import _deprecate_positional_args
+from ..utils.validation import (
+    _check_method_params,
+    _check_sample_weight,
+    check_is_fitted,
+    has_fit_parameter,
+    validate_data,
+)
 from ._base import LinearRegression
-from ..utils.validation import has_fit_parameter
-from ..exceptions import ConvergenceWarning
 
 _EPSILON = np.spacing(1)
 
@@ -46,16 +70,20 @@ def _dynamic_max_trials(n_inliers, n_samples, min_samples, probability):
     """
     inlier_ratio = n_inliers / float(n_samples)
     nom = max(_EPSILON, 1 - probability)
-    denom = max(_EPSILON, 1 - inlier_ratio ** min_samples)
+    denom = max(_EPSILON, 1 - inlier_ratio**min_samples)
     if nom == 1:
         return 0
     if denom == 1:
-        return float('inf')
+        return float("inf")
     return abs(float(np.ceil(np.log(nom) / np.log(denom))))
 
 
-class RANSACRegressor(MetaEstimatorMixin, RegressorMixin,
-                      MultiOutputMixin, BaseEstimator):
+class RANSACRegressor(
+    MetaEstimatorMixin,
+    RegressorMixin,
+    MultiOutputMixin,
+    BaseEstimator,
+):
     """RANSAC (RANdom SAmple Consensus) algorithm.
 
     RANSAC is an iterative algorithm for the robust estimation of parameters
@@ -65,18 +93,18 @@ class RANSACRegressor(MetaEstimatorMixin, RegressorMixin,
 
     Parameters
     ----------
-    base_estimator : object, default=None
+    estimator : object, default=None
         Base estimator object which implements the following methods:
 
-         * `fit(X, y)`: Fit model to given training data and target values.
-         * `score(X, y)`: Returns the mean accuracy on the given test data,
-           which is used for the stop criterion defined by `stop_score`.
-           Additionally, the score is used to decide which of two equally
-           large consensus sets is chosen as the better one.
-         * `predict(X)`: Returns predicted values using the linear model,
-           which is used to compute residual error using loss function.
+        * `fit(X, y)`: Fit model to given training data and target values.
+        * `score(X, y)`: Returns the mean accuracy on the given test data,
+          which is used for the stop criterion defined by `stop_score`.
+          Additionally, the score is used to decide which of two equally
+          large consensus sets is chosen as the better one.
+        * `predict(X)`: Returns predicted values using the linear model,
+          which is used to compute residual error using loss function.
 
-        If `base_estimator` is None, then
+        If `estimator` is None, then
         :class:`~sklearn.linear_model.LinearRegression` is used for
         target values of dtype float.
 
@@ -86,16 +114,20 @@ class RANSACRegressor(MetaEstimatorMixin, RegressorMixin,
     min_samples : int (>= 1) or float ([0, 1]), default=None
         Minimum number of samples chosen randomly from original data. Treated
         as an absolute number of samples for `min_samples >= 1`, treated as a
-        relative number `ceil(min_samples * X.shape[0]`) for
+        relative number `ceil(min_samples * X.shape[0])` for
         `min_samples < 1`. This is typically chosen as the minimal number of
-        samples necessary to estimate the given `base_estimator`. By default a
-        ``sklearn.linear_model.LinearRegression()`` estimator is assumed and
-        `min_samples` is chosen as ``X.shape[1] + 1``.
+        samples necessary to estimate the given `estimator`. By default a
+        :class:`~sklearn.linear_model.LinearRegression` estimator is assumed and
+        `min_samples` is chosen as ``X.shape[1] + 1``. This parameter is highly
+        dependent upon the model, so if a `estimator` other than
+        :class:`~sklearn.linear_model.LinearRegression` is used, the user must
+        provide a value.
 
     residual_threshold : float, default=None
         Maximum residual for a data sample to be classified as an inlier.
         By default the threshold is chosen as the MAD (median absolute
-        deviation) of the target values `y`.
+        deviation) of the target values `y`. Points whose residuals are
+        strictly equal to the threshold are considered as inliers.
 
     is_data_valid : callable, default=None
         This function is called with the randomly selected data before the
@@ -137,10 +169,9 @@ class RANSACRegressor(MetaEstimatorMixin, RegressorMixin,
         as 0.99 (the default) and e is the current fraction of inliers w.r.t.
         the total number of samples.
 
-    loss : string, callable, default='absolute_loss'
-        String inputs, "absolute_loss" and "squared_loss" are supported which
-        find the absolute loss and squared loss per sample
-        respectively.
+    loss : str, callable, default='absolute_error'
+        String inputs, 'absolute_error' and 'squared_error' are supported which
+        find the absolute error and squared error per sample respectively.
 
         If ``loss`` is a callable, then it should be a function that takes
         two arrays as inputs, the true and predicted value and returns a 1-D
@@ -160,7 +191,8 @@ class RANSACRegressor(MetaEstimatorMixin, RegressorMixin,
     Attributes
     ----------
     estimator_ : object
-        Best fitted model (copy of the `base_estimator` object).
+        Final model fitted on the inliers predicted by the "best" model found
+        during RANSAC sampling (copy of the `estimator` object).
 
     n_trials_ : int
         Number of random selection trials until one of the stop criteria is
@@ -186,6 +218,29 @@ class RANSACRegressor(MetaEstimatorMixin, RegressorMixin,
 
         .. versionadded:: 0.19
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    HuberRegressor : Linear regression model that is robust to outliers.
+    TheilSenRegressor : Theil-Sen Estimator robust multivariate regression model.
+    SGDRegressor : Fitted by minimizing a regularized empirical loss with SGD.
+
+    References
+    ----------
+    .. [1] https://en.wikipedia.org/wiki/RANSAC
+    .. [2] https://www.sri.com/wp-content/uploads/2021/12/ransac-publication.pdf
+    .. [3] https://bmva-archive.org.uk/bmvc/2009/Papers/Paper355/Paper355.pdf
+
     Examples
     --------
     >>> from sklearn.linear_model import RANSACRegressor
@@ -194,25 +249,59 @@ class RANSACRegressor(MetaEstimatorMixin, RegressorMixin,
     ...     n_samples=200, n_features=2, noise=4.0, random_state=0)
     >>> reg = RANSACRegressor(random_state=0).fit(X, y)
     >>> reg.score(X, y)
-    0.9885...
+    0.9885
     >>> reg.predict(X[:1,])
-    array([-31.9417...])
+    array([-31.9417])
 
-    References
-    ----------
-    .. [1] https://en.wikipedia.org/wiki/RANSAC
-    .. [2] https://www.sri.com/sites/default/files/publications/ransac-publication.pdf
-    .. [3] http://www.bmva.org/bmvc/2009/Papers/Paper355/Paper355.pdf
+    For a more detailed example, see
+    :ref:`sphx_glr_auto_examples_linear_model_plot_ransac.py`
     """
-    @_deprecate_positional_args
-    def __init__(self, base_estimator=None, *, min_samples=None,
-                 residual_threshold=None, is_data_valid=None,
-                 is_model_valid=None, max_trials=100, max_skips=np.inf,
-                 stop_n_inliers=np.inf, stop_score=np.inf,
-                 stop_probability=0.99, loss='absolute_loss',
-                 random_state=None):
-
-        self.base_estimator = base_estimator
+
+    _parameter_constraints: dict = {
+        "estimator": [HasMethods(["fit", "score", "predict"]), None],
+        "min_samples": [
+            Interval(Integral, 1, None, closed="left"),
+            Interval(RealNotInt, 0, 1, closed="both"),
+            None,
+        ],
+        "residual_threshold": [Interval(Real, 0, None, closed="left"), None],
+        "is_data_valid": [callable, None],
+        "is_model_valid": [callable, None],
+        "max_trials": [
+            Interval(Integral, 0, None, closed="left"),
+            Options(Real, {np.inf}),
+        ],
+        "max_skips": [
+            Interval(Integral, 0, None, closed="left"),
+            Options(Real, {np.inf}),
+        ],
+        "stop_n_inliers": [
+            Interval(Integral, 0, None, closed="left"),
+            Options(Real, {np.inf}),
+        ],
+        "stop_score": [Interval(Real, None, None, closed="both")],
+        "stop_probability": [Interval(Real, 0, 1, closed="both")],
+        "loss": [StrOptions({"absolute_error", "squared_error"}), callable],
+        "random_state": ["random_state"],
+    }
+
+    def __init__(
+        self,
+        estimator=None,
+        *,
+        min_samples=None,
+        residual_threshold=None,
+        is_data_valid=None,
+        is_model_valid=None,
+        max_trials=100,
+        max_skips=np.inf,
+        stop_n_inliers=np.inf,
+        stop_score=np.inf,
+        stop_probability=0.99,
+        loss="absolute_error",
+        random_state=None,
+    ):
+        self.estimator = estimator
         self.min_samples = min_samples
         self.residual_threshold = residual_threshold
         self.is_data_valid = is_data_valid
@@ -225,12 +314,16 @@ def __init__(self, base_estimator=None, *, min_samples=None,
         self.random_state = random_state
         self.loss = loss
 
-    def fit(self, X, y, sample_weight=None):
+    @_fit_context(
+        # RansacRegressor.estimator is not validated yet
+        prefer_skip_nested_validation=False
+    )
+    def fit(self, X, y, sample_weight=None, **fit_params):
         """Fit estimator using RANSAC algorithm.
 
         Parameters
         ----------
-        X : array-like or sparse matrix, shape [n_samples, n_features]
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
             Training data.
 
         y : array-like of shape (n_samples,) or (n_samples, n_targets)
@@ -238,51 +331,66 @@ def fit(self, X, y, sample_weight=None):
 
         sample_weight : array-like of shape (n_samples,), default=None
             Individual weights for each sample
-            raises error if sample_weight is passed and base_estimator
+            raises error if sample_weight is passed and estimator
             fit method does not support it.
 
             .. versionadded:: 0.18
 
+        **fit_params : dict
+            Parameters routed to the `fit` method of the sub-estimator via the
+            metadata routing API.
+
+            .. versionadded:: 1.5
+
+                Only available if
+                `sklearn.set_config(enable_metadata_routing=True)` is set. See
+                :ref:`Metadata Routing User Guide ` for more
+                details.
+
+        Returns
+        -------
+        self : object
+            Fitted `RANSACRegressor` estimator.
+
         Raises
         ------
         ValueError
             If no valid consensus set could be found. This occurs if
             `is_data_valid` and `is_model_valid` return False for all
             `max_trials` randomly chosen sub-samples.
-
         """
-        # Need to validate separately here.
-        # We can't pass multi_ouput=True because that would allow y to be csr.
-        check_X_params = dict(accept_sparse='csr')
+        # Need to validate separately here. We can't pass multi_output=True
+        # because that would allow y to be csr. Delay expensive finiteness
+        # check to the estimator's own input validation.
+        _raise_for_params(fit_params, self, "fit")
+        check_X_params = dict(accept_sparse="csr", ensure_all_finite=False)
         check_y_params = dict(ensure_2d=False)
-        X, y = self._validate_data(X, y, validate_separately=(check_X_params,
-                                                              check_y_params))
+        X, y = validate_data(
+            self, X, y, validate_separately=(check_X_params, check_y_params)
+        )
         check_consistent_length(X, y)
 
-        if self.base_estimator is not None:
-            base_estimator = clone(self.base_estimator)
+        if self.estimator is not None:
+            estimator = clone(self.estimator)
         else:
-            base_estimator = LinearRegression()
+            estimator = LinearRegression()
 
         if self.min_samples is None:
-            # assume linear model by default
+            if not isinstance(estimator, LinearRegression):
+                raise ValueError(
+                    "`min_samples` needs to be explicitly set when estimator "
+                    "is not a LinearRegression."
+                )
             min_samples = X.shape[1] + 1
         elif 0 < self.min_samples < 1:
             min_samples = np.ceil(self.min_samples * X.shape[0])
         elif self.min_samples >= 1:
-            if self.min_samples % 1 != 0:
-                raise ValueError("Absolute number of samples must be an "
-                                 "integer value.")
             min_samples = self.min_samples
-        else:
-            raise ValueError("Value for `min_samples` must be scalar and "
-                             "positive.")
         if min_samples > X.shape[0]:
-            raise ValueError("`min_samples` may not be larger than number "
-                             "of samples: n_samples = %d." % (X.shape[0]))
-
-        if self.stop_probability < 0 or self.stop_probability > 1:
-            raise ValueError("`stop_probability` must be in range [0, 1].")
+            raise ValueError(
+                "`min_samples` may not be larger than number "
+                "of samples: n_samples = %d." % (X.shape[0])
+            )
 
         if self.residual_threshold is None:
             # MAD (median absolute deviation)
@@ -290,46 +398,51 @@ def fit(self, X, y, sample_weight=None):
         else:
             residual_threshold = self.residual_threshold
 
-        if self.loss == "absolute_loss":
+        if self.loss == "absolute_error":
             if y.ndim == 1:
                 loss_function = lambda y_true, y_pred: np.abs(y_true - y_pred)
             else:
-                loss_function = lambda \
-                    y_true, y_pred: np.sum(np.abs(y_true - y_pred), axis=1)
-
-        elif self.loss == "squared_loss":
+                loss_function = lambda y_true, y_pred: np.sum(
+                    np.abs(y_true - y_pred), axis=1
+                )
+        elif self.loss == "squared_error":
             if y.ndim == 1:
                 loss_function = lambda y_true, y_pred: (y_true - y_pred) ** 2
             else:
-                loss_function = lambda \
-                    y_true, y_pred: np.sum((y_true - y_pred) ** 2, axis=1)
+                loss_function = lambda y_true, y_pred: np.sum(
+                    (y_true - y_pred) ** 2, axis=1
+                )
 
         elif callable(self.loss):
             loss_function = self.loss
 
-        else:
-            raise ValueError(
-                "loss should be 'absolute_loss', 'squared_loss' or a callable."
-                "Got %s. " % self.loss)
-
-
         random_state = check_random_state(self.random_state)
 
         try:  # Not all estimator accept a random_state
-            base_estimator.set_params(random_state=random_state)
+            estimator.set_params(random_state=random_state)
         except ValueError:
             pass
 
-        estimator_fit_has_sample_weight = has_fit_parameter(base_estimator,
-                                                            "sample_weight")
-        estimator_name = type(base_estimator).__name__
-        if (sample_weight is not None and not
-                estimator_fit_has_sample_weight):
-            raise ValueError("%s does not support sample_weight. Samples"
-                             " weights are only used for the calibration"
-                             " itself." % estimator_name)
+        estimator_fit_has_sample_weight = has_fit_parameter(estimator, "sample_weight")
+        estimator_name = type(estimator).__name__
+        if sample_weight is not None and not estimator_fit_has_sample_weight:
+            raise ValueError(
+                "%s does not support sample_weight. Sample"
+                " weights are only used for the calibration"
+                " itself." % estimator_name
+            )
+
         if sample_weight is not None:
-            sample_weight = _check_sample_weight(sample_weight, X)
+            fit_params["sample_weight"] = sample_weight
+
+        if _routing_enabled():
+            routed_params = process_routing(self, "fit", **fit_params)
+        else:
+            routed_params = Bunch()
+            routed_params.estimator = Bunch(fit={}, predict={}, score={})
+            if sample_weight is not None:
+                sample_weight = _check_sample_weight(sample_weight, X)
+                routed_params.estimator.fit = {"sample_weight": sample_weight}
 
         n_inliers_best = 1
         score_best = -np.inf
@@ -350,41 +463,48 @@ def fit(self, X, y, sample_weight=None):
         while self.n_trials_ < max_trials:
             self.n_trials_ += 1
 
-            if (self.n_skips_no_inliers_ + self.n_skips_invalid_data_ +
-                    self.n_skips_invalid_model_) > self.max_skips:
+            if (
+                self.n_skips_no_inliers_
+                + self.n_skips_invalid_data_
+                + self.n_skips_invalid_model_
+            ) > self.max_skips:
                 break
 
             # choose random sample set
-            subset_idxs = sample_without_replacement(n_samples, min_samples,
-                                                     random_state=random_state)
+            subset_idxs = sample_without_replacement(
+                n_samples, min_samples, random_state=random_state
+            )
             X_subset = X[subset_idxs]
             y_subset = y[subset_idxs]
 
             # check if random sample set is valid
-            if (self.is_data_valid is not None
-                    and not self.is_data_valid(X_subset, y_subset)):
+            if self.is_data_valid is not None and not self.is_data_valid(
+                X_subset, y_subset
+            ):
                 self.n_skips_invalid_data_ += 1
                 continue
 
+            # cut `fit_params` down to `subset_idxs`
+            fit_params_subset = _check_method_params(
+                X, params=routed_params.estimator.fit, indices=subset_idxs
+            )
+
             # fit model for current random sample set
-            if sample_weight is None:
-                base_estimator.fit(X_subset, y_subset)
-            else:
-                base_estimator.fit(X_subset, y_subset,
-                                   sample_weight=sample_weight[subset_idxs])
+            estimator.fit(X_subset, y_subset, **fit_params_subset)
 
             # check if estimated model is valid
-            if (self.is_model_valid is not None and not
-                    self.is_model_valid(base_estimator, X_subset, y_subset)):
+            if self.is_model_valid is not None and not self.is_model_valid(
+                estimator, X_subset, y_subset
+            ):
                 self.n_skips_invalid_model_ += 1
                 continue
 
             # residuals of all data for current random sample model
-            y_pred = base_estimator.predict(X)
+            y_pred = estimator.predict(X)
             residuals_subset = loss_function(y, y_pred)
 
             # classify data into inliers and outliers
-            inlier_mask_subset = residuals_subset < residual_threshold
+            inlier_mask_subset = residuals_subset <= residual_threshold
             n_inliers_subset = np.sum(inlier_mask_subset)
 
             # less inliers -> skip current random sample
@@ -397,14 +517,21 @@ def fit(self, X, y, sample_weight=None):
             X_inlier_subset = X[inlier_idxs_subset]
             y_inlier_subset = y[inlier_idxs_subset]
 
+            # cut `fit_params` down to `inlier_idxs_subset`
+            score_params_inlier_subset = _check_method_params(
+                X, params=routed_params.estimator.score, indices=inlier_idxs_subset
+            )
+
             # score of inlier data set
-            score_subset = base_estimator.score(X_inlier_subset,
-                                                y_inlier_subset)
+            score_subset = estimator.score(
+                X_inlier_subset,
+                y_inlier_subset,
+                **score_params_inlier_subset,
+            )
 
             # same number of inliers but worse score -> skip current random
             # sample
-            if (n_inliers_subset == n_inliers_best
-                    and score_subset < score_best):
+            if n_inliers_subset == n_inliers_best and score_subset < score_best:
                 continue
 
             # save current random sample as best sample
@@ -417,60 +544,83 @@ def fit(self, X, y, sample_weight=None):
 
             max_trials = min(
                 max_trials,
-                _dynamic_max_trials(n_inliers_best, n_samples,
-                                    min_samples, self.stop_probability))
+                _dynamic_max_trials(
+                    n_inliers_best, n_samples, min_samples, self.stop_probability
+                ),
+            )
 
             # break if sufficient number of inliers or score is reached
-            if n_inliers_best >= self.stop_n_inliers or \
-                            score_best >= self.stop_score:
+            if n_inliers_best >= self.stop_n_inliers or score_best >= self.stop_score:
                 break
 
         # if none of the iterations met the required criteria
         if inlier_mask_best is None:
-            if ((self.n_skips_no_inliers_ + self.n_skips_invalid_data_ +
-                    self.n_skips_invalid_model_) > self.max_skips):
+            if (
+                self.n_skips_no_inliers_
+                + self.n_skips_invalid_data_
+                + self.n_skips_invalid_model_
+            ) > self.max_skips:
                 raise ValueError(
                     "RANSAC skipped more iterations than `max_skips` without"
                     " finding a valid consensus set. Iterations were skipped"
                     " because each randomly chosen sub-sample failed the"
                     " passing criteria. See estimator attributes for"
-                    " diagnostics (n_skips*).")
+                    " diagnostics (n_skips*)."
+                )
             else:
                 raise ValueError(
                     "RANSAC could not find a valid consensus set. All"
                     " `max_trials` iterations were skipped because each"
                     " randomly chosen sub-sample failed the passing criteria."
-                    " See estimator attributes for diagnostics (n_skips*).")
+                    " See estimator attributes for diagnostics (n_skips*)."
+                )
         else:
-            if (self.n_skips_no_inliers_ + self.n_skips_invalid_data_ +
-                    self.n_skips_invalid_model_) > self.max_skips:
-                warnings.warn("RANSAC found a valid consensus set but exited"
-                              " early due to skipping more iterations than"
-                              " `max_skips`. See estimator attributes for"
-                              " diagnostics (n_skips*).",
-                              ConvergenceWarning)
+            if (
+                self.n_skips_no_inliers_
+                + self.n_skips_invalid_data_
+                + self.n_skips_invalid_model_
+            ) > self.max_skips:
+                warnings.warn(
+                    (
+                        "RANSAC found a valid consensus set but exited"
+                        " early due to skipping more iterations than"
+                        " `max_skips`. See estimator attributes for"
+                        " diagnostics (n_skips*)."
+                    ),
+                    ConvergenceWarning,
+                )
 
         # estimate final model using all inliers
-        if sample_weight is None:
-            base_estimator.fit(X_inlier_best, y_inlier_best)
-        else:
-            base_estimator.fit(
-                X_inlier_best,
-                y_inlier_best,
-                sample_weight=sample_weight[inlier_best_idxs_subset])
+        fit_params_best_idxs_subset = _check_method_params(
+            X, params=routed_params.estimator.fit, indices=inlier_best_idxs_subset
+        )
 
-        self.estimator_ = base_estimator
+        estimator.fit(X_inlier_best, y_inlier_best, **fit_params_best_idxs_subset)
+
+        self.estimator_ = estimator
         self.inlier_mask_ = inlier_mask_best
         return self
 
-    def predict(self, X):
+    def predict(self, X, **params):
         """Predict using the estimated model.
 
         This is a wrapper for `estimator_.predict(X)`.
 
         Parameters
         ----------
-        X : numpy array of shape [n_samples, n_features]
+        X : {array-like or sparse matrix} of shape (n_samples, n_features)
+            Input data.
+
+        **params : dict
+            Parameters routed to the `predict` method of the sub-estimator via
+            the metadata routing API.
+
+            .. versionadded:: 1.5
+
+                Only available if
+                `sklearn.set_config(enable_metadata_routing=True)` is set. See
+                :ref:`Metadata Routing User Guide ` for more
+                details.
 
         Returns
         -------
@@ -478,35 +628,99 @@ def predict(self, X):
             Returns predicted values.
         """
         check_is_fitted(self)
+        X = validate_data(
+            self,
+            X,
+            ensure_all_finite=False,
+            accept_sparse=True,
+            reset=False,
+        )
+
+        _raise_for_params(params, self, "predict")
+
+        if _routing_enabled():
+            predict_params = process_routing(self, "predict", **params).estimator[
+                "predict"
+            ]
+        else:
+            predict_params = {}
 
-        return self.estimator_.predict(X)
+        return self.estimator_.predict(X, **predict_params)
 
-    def score(self, X, y):
-        """Returns the score of the prediction.
+    def score(self, X, y, **params):
+        """Return the score of the prediction.
 
         This is a wrapper for `estimator_.score(X, y)`.
 
         Parameters
         ----------
-        X : numpy array or sparse matrix of shape [n_samples, n_features]
+        X : (array-like or sparse matrix} of shape (n_samples, n_features)
             Training data.
 
-        y : array, shape = [n_samples] or [n_samples, n_targets]
+        y : array-like of shape (n_samples,) or (n_samples, n_targets)
             Target values.
 
+        **params : dict
+            Parameters routed to the `score` method of the sub-estimator via
+            the metadata routing API.
+
+            .. versionadded:: 1.5
+
+                Only available if
+                `sklearn.set_config(enable_metadata_routing=True)` is set. See
+                :ref:`Metadata Routing User Guide ` for more
+                details.
+
         Returns
         -------
         z : float
             Score of the prediction.
         """
         check_is_fitted(self)
+        X = validate_data(
+            self,
+            X,
+            ensure_all_finite=False,
+            accept_sparse=True,
+            reset=False,
+        )
+
+        _raise_for_params(params, self, "score")
+        if _routing_enabled():
+            score_params = process_routing(self, "score", **params).estimator["score"]
+        else:
+            score_params = {}
 
-        return self.estimator_.score(X, y)
+        return self.estimator_.score(X, y, **score_params)
 
-    def _more_tags(self):
-        return {
-            '_xfail_checks': {
-                'check_sample_weights_invariance':
-                'zero sample_weight is not equivalent to removing samples',
-            }
-        }
+    def get_metadata_routing(self):
+        """Get metadata routing of this object.
+
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
+
+        .. versionadded:: 1.5
+
+        Returns
+        -------
+        routing : MetadataRouter
+            A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
+            routing information.
+        """
+        router = MetadataRouter(owner=self.__class__.__name__).add(
+            estimator=self.estimator,
+            method_mapping=MethodMapping()
+            .add(caller="fit", callee="fit")
+            .add(caller="fit", callee="score")
+            .add(caller="score", callee="score")
+            .add(caller="predict", callee="predict"),
+        )
+        return router
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        if self.estimator is None:
+            tags.input_tags.sparse = True  # default estimator is LinearRegression
+        else:
+            tags.input_tags.sparse = get_tags(self.estimator).input_tags.sparse
+        return tags
diff --git a/sklearn/linear_model/_ridge.py b/sklearn/linear_model/_ridge.py
index 2d360c6edbc58..0a55291a70ace 100644
--- a/sklearn/linear_model/_ridge.py
+++ b/sklearn/linear_model/_ridge.py
@@ -2,75 +2,111 @@
 Ridge regression
 """
 
-# Author: Mathieu Blondel 
-#         Reuben Fletcher-Costin 
-#         Fabian Pedregosa 
-#         Michael Eickenberg 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-
-from abc import ABCMeta, abstractmethod
+import numbers
 import warnings
+from abc import ABCMeta, abstractmethod
+from functools import partial
+from numbers import Integral, Real
 
 import numpy as np
-from scipy import linalg
-from scipy import sparse
+from scipy import linalg, optimize, sparse
 from scipy.sparse import linalg as sp_linalg
 
-from ._base import LinearClassifierMixin, LinearModel, _rescale_data
-from ._sag import sag_solver
-from ..base import RegressorMixin, MultiOutputMixin, is_classifier
-from ..utils.extmath import safe_sparse_dot
-from ..utils.extmath import row_norms
-from ..utils import check_array
-from ..utils import check_consistent_length
-from ..utils import compute_sample_weight
-from ..utils import column_or_1d
-from ..utils.validation import _check_sample_weight
-from ..utils.validation import _deprecate_positional_args
-from ..preprocessing import LabelBinarizer
-from ..model_selection import GridSearchCV
-from ..metrics import check_scoring
+from sklearn.base import BaseEstimator
+
+from ..base import MultiOutputMixin, RegressorMixin, _fit_context, is_classifier
 from ..exceptions import ConvergenceWarning
+from ..metrics import check_scoring, get_scorer_names
+from ..model_selection import GridSearchCV
+from ..preprocessing import LabelBinarizer
+from ..utils import (
+    Bunch,
+    check_array,
+    check_consistent_length,
+    check_scalar,
+    column_or_1d,
+    compute_sample_weight,
+)
+from ..utils._array_api import (
+    _is_numpy_namespace,
+    _ravel,
+    device,
+    get_namespace,
+    get_namespace_and_device,
+)
+from ..utils._param_validation import Interval, StrOptions, validate_params
+from ..utils.extmath import row_norms, safe_sparse_dot
+from ..utils.fixes import _sparse_linalg_cg
+from ..utils.metadata_routing import (
+    MetadataRouter,
+    MethodMapping,
+    _raise_for_params,
+    _routing_enabled,
+    process_routing,
+)
 from ..utils.sparsefuncs import mean_variance_axis
+from ..utils.validation import _check_sample_weight, check_is_fitted, validate_data
+from ._base import LinearClassifierMixin, LinearModel, _preprocess_data, _rescale_data
+from ._sag import sag_solver
 
 
-def _solve_sparse_cg(X, y, alpha, max_iter=None, tol=1e-3, verbose=0,
-                     X_offset=None, X_scale=None):
+def _get_rescaled_operator(X, X_offset, sample_weight_sqrt):
+    """Create LinearOperator for matrix products with implicit centering.
 
-    def _get_rescaled_operator(X):
+    Matrix product `LinearOperator @ coef` returns `(X - X_offset) @ coef`.
+    """
 
-        X_offset_scale = X_offset / X_scale
+    def matvec(b):
+        return X.dot(b) - sample_weight_sqrt * b.dot(X_offset)
+
+    def rmatvec(b):
+        return X.T.dot(b) - X_offset * b.dot(sample_weight_sqrt)
 
-        def matvec(b):
-            return X.dot(b) - b.dot(X_offset_scale)
+    X1 = sparse.linalg.LinearOperator(shape=X.shape, matvec=matvec, rmatvec=rmatvec)
+    return X1
 
-        def rmatvec(b):
-            return X.T.dot(b) - X_offset_scale * np.sum(b)
 
-        X1 = sparse.linalg.LinearOperator(shape=X.shape,
-                                          matvec=matvec,
-                                          rmatvec=rmatvec)
-        return X1
+def _solve_sparse_cg(
+    X,
+    y,
+    alpha,
+    max_iter=None,
+    tol=1e-4,
+    verbose=0,
+    X_offset=None,
+    X_scale=None,
+    sample_weight_sqrt=None,
+):
+    if sample_weight_sqrt is None:
+        sample_weight_sqrt = np.ones(X.shape[0], dtype=X.dtype)
 
     n_samples, n_features = X.shape
 
     if X_offset is None or X_scale is None:
         X1 = sp_linalg.aslinearoperator(X)
     else:
-        X1 = _get_rescaled_operator(X)
+        X_offset_scale = X_offset / X_scale
+        X1 = _get_rescaled_operator(X, X_offset_scale, sample_weight_sqrt)
 
     coefs = np.empty((y.shape[1], n_features), dtype=X.dtype)
 
     if n_features > n_samples:
+
         def create_mv(curr_alpha):
             def _mv(x):
                 return X1.matvec(X1.rmatvec(x)) + curr_alpha * x
+
             return _mv
+
     else:
+
         def create_mv(curr_alpha):
             def _mv(x):
                 return X1.rmatvec(X1.matvec(x)) + curr_alpha * x
+
             return _mv
 
     for i in range(y.shape[1]):
@@ -81,40 +117,69 @@ def _mv(x):
             # kernel ridge
             # w = X.T * inv(X X^t + alpha*Id) y
             C = sp_linalg.LinearOperator(
-                (n_samples, n_samples), matvec=mv, dtype=X.dtype)
-            # FIXME atol
-            try:
-                coef, info = sp_linalg.cg(C, y_column, tol=tol, atol='legacy')
-            except TypeError:
-                # old scipy
-                coef, info = sp_linalg.cg(C, y_column, tol=tol)
+                (n_samples, n_samples), matvec=mv, dtype=X.dtype
+            )
+            coef, info = _sparse_linalg_cg(C, y_column, rtol=tol)
             coefs[i] = X1.rmatvec(coef)
         else:
             # linear ridge
             # w = inv(X^t X + alpha*Id) * X.T y
             y_column = X1.rmatvec(y_column)
             C = sp_linalg.LinearOperator(
-                (n_features, n_features), matvec=mv, dtype=X.dtype)
-            # FIXME atol
-            try:
-                coefs[i], info = sp_linalg.cg(C, y_column, maxiter=max_iter,
-                                              tol=tol, atol='legacy')
-            except TypeError:
-                # old scipy
-                coefs[i], info = sp_linalg.cg(C, y_column, maxiter=max_iter,
-                                              tol=tol)
+                (n_features, n_features), matvec=mv, dtype=X.dtype
+            )
+            coefs[i], info = _sparse_linalg_cg(C, y_column, maxiter=max_iter, rtol=tol)
 
         if info < 0:
             raise ValueError("Failed with error code %d" % info)
 
         if max_iter is None and info > 0 and verbose:
-            warnings.warn("sparse_cg did not converge after %d iterations." %
-                          info, ConvergenceWarning)
+            warnings.warn(
+                "sparse_cg did not converge after %d iterations." % info,
+                ConvergenceWarning,
+            )
 
     return coefs
 
 
-def _solve_lsqr(X, y, alpha, max_iter=None, tol=1e-3):
+def _solve_lsqr(
+    X,
+    y,
+    *,
+    alpha,
+    fit_intercept=True,
+    max_iter=None,
+    tol=1e-4,
+    X_offset=None,
+    X_scale=None,
+    sample_weight_sqrt=None,
+):
+    """Solve Ridge regression via LSQR.
+
+    We expect that y is always mean centered.
+    If X is dense, we expect it to be mean centered such that we can solve
+        ||y - Xw||_2^2 + alpha * ||w||_2^2
+
+    If X is sparse, we expect X_offset to be given such that we can solve
+        ||y - (X - X_offset)w||_2^2 + alpha * ||w||_2^2
+
+    With sample weights S=diag(sample_weight), this becomes
+        ||sqrt(S) (y - (X - X_offset) w)||_2^2 + alpha * ||w||_2^2
+    and we expect y and X to already be rescaled, i.e. sqrt(S) @ y, sqrt(S) @ X. In
+    this case, X_offset is the sample_weight weighted mean of X before scaling by
+    sqrt(S). The objective then reads
+       ||y - (X - sqrt(S) X_offset) w)||_2^2 + alpha * ||w||_2^2
+    """
+    if sample_weight_sqrt is None:
+        sample_weight_sqrt = np.ones(X.shape[0], dtype=X.dtype)
+
+    if sparse.issparse(X) and fit_intercept:
+        X_offset_scale = X_offset / X_scale
+        X1 = _get_rescaled_operator(X, X_offset_scale, sample_weight_sqrt)
+    else:
+        # No need to touch anything
+        X1 = X
+
     n_samples, n_features = X.shape
     coefs = np.empty((y.shape[1], n_features), dtype=X.dtype)
     n_iter = np.empty(y.shape[1], dtype=np.int32)
@@ -124,8 +189,9 @@ def _solve_lsqr(X, y, alpha, max_iter=None, tol=1e-3):
 
     for i in range(y.shape[1]):
         y_column = y[:, i]
-        info = sp_linalg.lsqr(X, y_column, damp=sqrt_alpha[i],
-                              atol=tol, btol=tol, iter_lim=max_iter)
+        info = sp_linalg.lsqr(
+            X1, y_column, damp=sqrt_alpha[i], atol=tol, btol=tol, iter_lim=max_iter
+        )
         coefs[i] = info[0]
         n_iter[i] = info[2]
 
@@ -143,16 +209,14 @@ def _solve_cholesky(X, y, alpha):
     one_alpha = np.array_equal(alpha, len(alpha) * [alpha[0]])
 
     if one_alpha:
-        A.flat[::n_features + 1] += alpha[0]
-        return linalg.solve(A, Xy, sym_pos=True,
-                            overwrite_a=True).T
+        A.flat[:: n_features + 1] += alpha[0]
+        return linalg.solve(A, Xy, assume_a="pos", overwrite_a=True).T
     else:
         coefs = np.empty([n_targets, n_features], dtype=X.dtype)
         for coef, target, current_alpha in zip(coefs, Xy.T, alpha):
-            A.flat[::n_features + 1] += current_alpha
-            coef[:] = linalg.solve(A, target, sym_pos=True,
-                                   overwrite_a=False).ravel()
-            A.flat[::n_features + 1] -= current_alpha
+            A.flat[:: n_features + 1] += current_alpha
+            coef[:] = linalg.solve(A, target, assume_a="pos", overwrite_a=False).ravel()
+            A.flat[:: n_features + 1] -= current_alpha
         return coefs
 
 
@@ -166,8 +230,7 @@ def _solve_cholesky_kernel(K, y, alpha, sample_weight=None, copy=False):
 
     alpha = np.atleast_1d(alpha)
     one_alpha = (alpha == alpha[0]).all()
-    has_sw = isinstance(sample_weight, np.ndarray) \
-        or sample_weight not in [1.0, None]
+    has_sw = isinstance(sample_weight, np.ndarray) or sample_weight not in [1.0, None]
 
     if has_sw:
         # Unlike other solvers, we need to support sample_weight directly
@@ -178,22 +241,23 @@ def _solve_cholesky_kernel(K, y, alpha, sample_weight=None, copy=False):
 
     if one_alpha:
         # Only one penalty, we can solve multi-target problems in one time.
-        K.flat[::n_samples + 1] += alpha[0]
+        K.flat[:: n_samples + 1] += alpha[0]
 
         try:
             # Note: we must use overwrite_a=False in order to be able to
             #       use the fall-back solution below in case a LinAlgError
             #       is raised
-            dual_coef = linalg.solve(K, y, sym_pos=True,
-                                     overwrite_a=False)
+            dual_coef = linalg.solve(K, y, assume_a="pos", overwrite_a=False)
         except np.linalg.LinAlgError:
-            warnings.warn("Singular matrix in solving dual problem. Using "
-                          "least-squares solution instead.")
+            warnings.warn(
+                "Singular matrix in solving dual problem. Using "
+                "least-squares solution instead."
+            )
             dual_coef = linalg.lstsq(K, y)[0]
 
         # K is expensive to compute and store in memory so change it back in
         # case it was user-given.
-        K.flat[::n_samples + 1] -= alpha[0]
+        K.flat[:: n_samples + 1] -= alpha[0]
 
         if has_sw:
             dual_coef *= sw[:, np.newaxis]
@@ -204,12 +268,13 @@ def _solve_cholesky_kernel(K, y, alpha, sample_weight=None, copy=False):
         dual_coefs = np.empty([n_targets, n_samples], K.dtype)
 
         for dual_coef, target, current_alpha in zip(dual_coefs, y.T, alpha):
-            K.flat[::n_samples + 1] += current_alpha
+            K.flat[:: n_samples + 1] += current_alpha
 
-            dual_coef[:] = linalg.solve(K, target, sym_pos=True,
-                                        overwrite_a=False).ravel()
+            dual_coef[:] = linalg.solve(
+                K, target, assume_a="pos", overwrite_a=False
+            ).ravel()
 
-            K.flat[::n_samples + 1] -= current_alpha
+            K.flat[:: n_samples + 1] -= current_alpha
 
         if has_sw:
             dual_coefs *= sw[np.newaxis, :]
@@ -217,51 +282,162 @@ def _solve_cholesky_kernel(K, y, alpha, sample_weight=None, copy=False):
         return dual_coefs.T
 
 
-def _solve_svd(X, y, alpha):
-    U, s, Vt = linalg.svd(X, full_matrices=False)
+def _solve_svd(X, y, alpha, xp=None):
+    xp, _ = get_namespace(X, xp=xp)
+    U, s, Vt = xp.linalg.svd(X, full_matrices=False)
     idx = s > 1e-15  # same default value as scipy.linalg.pinv
-    s_nnz = s[idx][:, np.newaxis]
-    UTy = np.dot(U.T, y)
-    d = np.zeros((s.size, alpha.size), dtype=X.dtype)
-    d[idx] = s_nnz / (s_nnz ** 2 + alpha)
+    s_nnz = s[idx][:, None]
+    UTy = U.T @ y
+    d = xp.zeros((s.shape[0], alpha.shape[0]), dtype=X.dtype, device=device(X))
+    d[idx] = s_nnz / (s_nnz**2 + alpha)
     d_UT_y = d * UTy
-    return np.dot(Vt.T, d_UT_y).T
-
+    return (Vt.T @ d_UT_y).T
+
+
+def _solve_lbfgs(
+    X,
+    y,
+    alpha,
+    positive=True,
+    max_iter=None,
+    tol=1e-4,
+    X_offset=None,
+    X_scale=None,
+    sample_weight_sqrt=None,
+):
+    """Solve ridge regression with LBFGS.
+
+    The main purpose is fitting with forcing coefficients to be positive.
+    For unconstrained ridge regression, there are faster dedicated solver methods.
+    Note that with positive bounds on the coefficients, LBFGS seems faster
+    than scipy.optimize.lsq_linear.
+    """
+    n_samples, n_features = X.shape
 
-def _get_valid_accept_sparse(is_X_sparse, solver):
-    if is_X_sparse and solver in ['auto', 'sag', 'saga']:
-        return 'csr'
+    options = {}
+    if max_iter is not None:
+        options["maxiter"] = max_iter
+    config = {
+        "method": "L-BFGS-B",
+        "tol": tol,
+        "jac": True,
+        "options": options,
+    }
+    if positive:
+        config["bounds"] = [(0, np.inf)] * n_features
+
+    if X_offset is not None and X_scale is not None:
+        X_offset_scale = X_offset / X_scale
     else:
-        return ['csr', 'csc', 'coo']
+        X_offset_scale = None
 
+    if sample_weight_sqrt is None:
+        sample_weight_sqrt = np.ones(X.shape[0], dtype=X.dtype)
 
-@_deprecate_positional_args
-def ridge_regression(X, y, alpha, *, sample_weight=None, solver='auto',
-                     max_iter=None, tol=1e-3, verbose=0, random_state=None,
-                     return_n_iter=False, return_intercept=False,
-                     check_input=True):
+    coefs = np.empty((y.shape[1], n_features), dtype=X.dtype)
+
+    for i in range(y.shape[1]):
+        x0 = np.zeros((n_features,))
+        y_column = y[:, i]
+
+        def func(w):
+            residual = X.dot(w) - y_column
+            if X_offset_scale is not None:
+                residual -= sample_weight_sqrt * w.dot(X_offset_scale)
+            f = 0.5 * residual.dot(residual) + 0.5 * alpha[i] * w.dot(w)
+            grad = X.T @ residual + alpha[i] * w
+            if X_offset_scale is not None:
+                grad -= X_offset_scale * residual.dot(sample_weight_sqrt)
+
+            return f, grad
+
+        result = optimize.minimize(func, x0, **config)
+        if not result["success"]:
+            warnings.warn(
+                (
+                    "The lbfgs solver did not converge. Try increasing max_iter "
+                    f"or tol. Currently: max_iter={max_iter} and tol={tol}"
+                ),
+                ConvergenceWarning,
+            )
+        coefs[i] = result["x"]
+
+    return coefs
+
+
+def _get_valid_accept_sparse(is_X_sparse, solver):
+    if is_X_sparse and solver in ["auto", "sag", "saga"]:
+        return "csr"
+    else:
+        return ["csr", "csc", "coo"]
+
+
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix", sp_linalg.LinearOperator],
+        "y": ["array-like"],
+        "alpha": [Interval(Real, 0, None, closed="left"), "array-like"],
+        "sample_weight": [
+            Interval(Real, None, None, closed="neither"),
+            "array-like",
+            None,
+        ],
+        "solver": [
+            StrOptions(
+                {"auto", "svd", "cholesky", "lsqr", "sparse_cg", "sag", "saga", "lbfgs"}
+            )
+        ],
+        "max_iter": [Interval(Integral, 0, None, closed="left"), None],
+        "tol": [Interval(Real, 0, None, closed="left")],
+        "verbose": ["verbose"],
+        "positive": ["boolean"],
+        "random_state": ["random_state"],
+        "return_n_iter": ["boolean"],
+        "return_intercept": ["boolean"],
+        "check_input": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def ridge_regression(
+    X,
+    y,
+    alpha,
+    *,
+    sample_weight=None,
+    solver="auto",
+    max_iter=None,
+    tol=1e-4,
+    verbose=0,
+    positive=False,
+    random_state=None,
+    return_n_iter=False,
+    return_intercept=False,
+    check_input=True,
+):
     """Solve the ridge equation by the method of normal equations.
 
     Read more in the :ref:`User Guide `.
 
     Parameters
     ----------
-    X : {ndarray, sparse matrix, LinearOperator} of shape \
+    X : {array-like, sparse matrix, LinearOperator} of shape \
         (n_samples, n_features)
-        Training data
+        Training data.
 
-    y : ndarray of shape (n_samples,) or (n_samples, n_targets)
-        Target values
+    y : array-like of shape (n_samples,) or (n_samples, n_targets)
+        Target values.
 
     alpha : float or array-like of shape (n_targets,)
-        Regularization strength; must be a positive float. Regularization
-        improves the conditioning of the problem and reduces the variance of
-        the estimates. Larger values specify stronger regularization.
-        Alpha corresponds to ``1 / (2C)`` in other linear models such as
-        :class:`~sklearn.linear_model.LogisticRegression` or
-        :class:`~sklearn.svm.LinearSVC`. If an array is passed, penalties are
-        assumed to be specific to the targets. Hence they must correspond in
-        number.
+        Constant that multiplies the L2 term, controlling regularization
+        strength. `alpha` must be a non-negative float i.e. in `[0, inf)`.
+
+        When `alpha = 0`, the objective is equivalent to ordinary least
+        squares, solved by the :class:`LinearRegression` object. For numerical
+        reasons, using `alpha = 0` with the `Ridge` object is not advised.
+        Instead, you should use the :class:`LinearRegression` object.
+
+        If an array is passed, penalties are assumed to be specific to the
+        targets. Hence they must correspond in number.
 
     sample_weight : float or array-like of shape (n_samples,), default=None
         Individual weights for each sample. If given a float, every sample
@@ -270,14 +446,15 @@ def ridge_regression(X, y, alpha, *, sample_weight=None, solver='auto',
 
         .. versionadded:: 0.17
 
-    solver : {'auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag', 'saga'}, \
-        default='auto'
+    solver : {'auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg', \
+            'sag', 'saga', 'lbfgs'}, default='auto'
         Solver to use in the computational routines:
 
         - 'auto' chooses the solver automatically based on the type of data.
 
         - 'svd' uses a Singular Value Decomposition of X to compute the Ridge
-          coefficients. More stable for singular matrices than 'cholesky'.
+          coefficients. It is the most stable solver, in particular more stable
+          for singular matrices than 'cholesky' at the cost of being slower.
 
         - 'cholesky' uses the standard scipy.linalg.solve function to
           obtain a closed-form solution via a Cholesky decomposition of
@@ -300,10 +477,13 @@ def ridge_regression(X, y, alpha, *, sample_weight=None, solver='auto',
           approximately the same scale. You can preprocess the data with a
           scaler from sklearn.preprocessing.
 
+        - 'lbfgs' uses L-BFGS-B algorithm implemented in
+          `scipy.optimize.minimize`. It can be used only when `positive`
+          is True.
 
-        All last five solvers support both dense and sparse data. However, only
-        'sag' and 'sparse_cg' supports sparse input when `fit_intercept` is
-        True.
+        All solvers except 'svd' support both dense and sparse data. However, only
+        'lsqr', 'sag', 'sparse_cg', and 'lbfgs' support sparse input when
+        `fit_intercept` is True.
 
         .. versionadded:: 0.17
            Stochastic Average Gradient descent solver.
@@ -314,15 +494,24 @@ def ridge_regression(X, y, alpha, *, sample_weight=None, solver='auto',
         Maximum number of iterations for conjugate gradient solver.
         For the 'sparse_cg' and 'lsqr' solvers, the default value is determined
         by scipy.sparse.linalg. For 'sag' and saga solver, the default value is
-        1000.
+        1000. For 'lbfgs' solver, the default value is 15000.
+
+    tol : float, default=1e-4
+        Precision of the solution. Note that `tol` has no effect for solvers 'svd' and
+        'cholesky'.
 
-    tol : float, default=1e-3
-        Precision of the solution.
+        .. versionchanged:: 1.2
+           Default value changed from 1e-3 to 1e-4 for consistency with other linear
+           models.
 
     verbose : int, default=0
         Verbosity level. Setting verbose > 0 will display additional
         information depending on the solver used.
 
+    positive : bool, default=False
+        When set to ``True``, forces the coefficients to be positive.
+        Only 'lbfgs' solver is supported in this case.
+
     random_state : int, RandomState instance, default=None
         Used when ``solver`` == 'sag' or 'saga' to shuffle the data.
         See :term:`Glossary ` for details.
@@ -362,51 +551,111 @@ def ridge_regression(X, y, alpha, *, sample_weight=None, solver='auto',
     Notes
     -----
     This function won't compute the intercept.
+
+    Regularization improves the conditioning of the problem and
+    reduces the variance of the estimates. Larger values specify stronger
+    regularization. Alpha corresponds to ``1 / (2C)`` in other linear
+    models such as :class:`~sklearn.linear_model.LogisticRegression` or
+    :class:`~sklearn.svm.LinearSVC`. If an array is passed, penalties are
+    assumed to be specific to the targets. Hence they must correspond in
+    number.
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from sklearn.datasets import make_regression
+    >>> from sklearn.linear_model import ridge_regression
+    >>> rng = np.random.RandomState(0)
+    >>> X = rng.randn(100, 4)
+    >>> y = 2.0 * X[:, 0] - 1.0 * X[:, 1] + 0.1 * rng.standard_normal(100)
+    >>> coef, intercept = ridge_regression(X, y, alpha=1.0, return_intercept=True,
+    ...                                    random_state=0)
+    >>> coef
+    array([ 1.97, -1., -2.69e-3, -9.27e-4 ])
+    >>> intercept
+    np.float64(-.0012)
     """
-    return _ridge_regression(X, y, alpha,
-                             sample_weight=sample_weight,
-                             solver=solver,
-                             max_iter=max_iter,
-                             tol=tol,
-                             verbose=verbose,
-                             random_state=random_state,
-                             return_n_iter=return_n_iter,
-                             return_intercept=return_intercept,
-                             X_scale=None,
-                             X_offset=None,
-                             check_input=check_input)
-
-
-def _ridge_regression(X, y, alpha, sample_weight=None, solver='auto',
-                      max_iter=None, tol=1e-3, verbose=0, random_state=None,
-                      return_n_iter=False, return_intercept=False,
-                      X_scale=None, X_offset=None, check_input=True):
+    return _ridge_regression(
+        X,
+        y,
+        alpha,
+        sample_weight=sample_weight,
+        solver=solver,
+        max_iter=max_iter,
+        tol=tol,
+        verbose=verbose,
+        positive=positive,
+        random_state=random_state,
+        return_n_iter=return_n_iter,
+        return_intercept=return_intercept,
+        X_scale=None,
+        X_offset=None,
+        check_input=check_input,
+    )
+
+
+def _ridge_regression(
+    X,
+    y,
+    alpha,
+    sample_weight=None,
+    solver="auto",
+    max_iter=None,
+    tol=1e-4,
+    verbose=0,
+    positive=False,
+    random_state=None,
+    return_n_iter=False,
+    return_intercept=False,
+    return_solver=False,
+    X_scale=None,
+    X_offset=None,
+    check_input=True,
+    fit_intercept=False,
+):
+    xp, is_array_api_compliant, device_ = get_namespace_and_device(
+        X, y, sample_weight, X_scale, X_offset
+    )
+    is_numpy_namespace = _is_numpy_namespace(xp)
+    X_is_sparse = sparse.issparse(X)
 
     has_sw = sample_weight is not None
 
-    if solver == 'auto':
-        if return_intercept:
-            # only sag supports fitting intercept directly
-            solver = "sag"
-        elif not sparse.issparse(X):
-            solver = "cholesky"
-        else:
-            solver = "sparse_cg"
+    solver = resolve_solver(solver, positive, return_intercept, X_is_sparse, xp)
+
+    if is_numpy_namespace and not X_is_sparse:
+        X = np.asarray(X)
+
+    if not is_numpy_namespace and solver != "svd":
+        raise ValueError(
+            f"Array API dispatch to namespace {xp.__name__} only supports "
+            f"solver 'svd'. Got '{solver}'."
+        )
+
+    if positive and solver != "lbfgs":
+        raise ValueError(
+            "When positive=True, only 'lbfgs' solver can be used. "
+            f"Please change solver {solver} to 'lbfgs' "
+            "or set positive=False."
+        )
 
-    if solver not in ('sparse_cg', 'cholesky', 'svd', 'lsqr', 'sag', 'saga'):
-        raise ValueError("Known solvers are 'sparse_cg', 'cholesky', 'svd'"
-                         " 'lsqr', 'sag' or 'saga'. Got %s." % solver)
+    if solver == "lbfgs" and not positive:
+        raise ValueError(
+            "'lbfgs' solver can be used only when positive=True. "
+            "Please use another solver."
+        )
 
-    if return_intercept and solver != 'sag':
-        raise ValueError("In Ridge, only 'sag' solver can directly fit the "
-                         "intercept. Please change solver to 'sag' or set "
-                         "return_intercept=False.")
+    if return_intercept and solver != "sag":
+        raise ValueError(
+            "In Ridge, only 'sag' solver can directly fit the "
+            "intercept. Please change solver to 'sag' or set "
+            "return_intercept=False."
+        )
 
     if check_input:
-        _dtype = [np.float64, np.float32]
-        _accept_sparse = _get_valid_accept_sparse(sparse.issparse(X), solver)
-        X = check_array(X, accept_sparse=_accept_sparse, dtype=_dtype,
-                        order="C")
+        _dtype = [xp.float64, xp.float32]
+        _accept_sparse = _get_valid_accept_sparse(X_is_sparse, solver)
+        X = check_array(X, accept_sparse=_accept_sparse, dtype=_dtype, order="C")
         y = check_array(y, dtype=X.dtype, ensure_2d=False, order=None)
     check_consistent_length(X, y)
 
@@ -415,48 +664,77 @@ def _ridge_regression(X, y, alpha, sample_weight=None, solver='auto',
     if y.ndim > 2:
         raise ValueError("Target y has the wrong shape %s" % str(y.shape))
 
-    ravel = False
     if y.ndim == 1:
-        y = y.reshape(-1, 1)
-        ravel = True
+        y = xp.reshape(y, (-1, 1))
 
     n_samples_, n_targets = y.shape
 
     if n_samples != n_samples_:
-        raise ValueError("Number of samples in X and y does not correspond:"
-                         " %d != %d" % (n_samples, n_samples_))
+        raise ValueError(
+            "Number of samples in X and y does not correspond: %d != %d"
+            % (n_samples, n_samples_)
+        )
 
     if has_sw:
         sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype)
 
-        if solver not in ['sag', 'saga']:
+        if solver not in ["sag", "saga"]:
             # SAG supports sample_weight directly. For other solvers,
             # we implement sample_weight via a simple rescaling.
-            X, y = _rescale_data(X, y, sample_weight)
+            X, y, sample_weight_sqrt = _rescale_data(X, y, sample_weight)
+
+    # Some callers of this method might pass alpha as single
+    # element array which already has been validated.
+    if alpha is not None and not isinstance(alpha, type(xp.asarray([0.0]))):
+        alpha = check_scalar(
+            alpha,
+            "alpha",
+            target_type=numbers.Real,
+            min_val=0.0,
+            include_boundaries="left",
+        )
 
     # There should be either 1 or n_targets penalties
-    alpha = np.asarray(alpha, dtype=X.dtype).ravel()
-    if alpha.size not in [1, n_targets]:
-        raise ValueError("Number of targets and number of penalties "
-                         "do not correspond: %d != %d"
-                         % (alpha.size, n_targets))
+    alpha = _ravel(xp.asarray(alpha, device=device_, dtype=X.dtype), xp=xp)
+    if alpha.shape[0] not in [1, n_targets]:
+        raise ValueError(
+            "Number of targets and number of penalties do not correspond: %d != %d"
+            % (alpha.shape[0], n_targets)
+        )
 
-    if alpha.size == 1 and n_targets > 1:
-        alpha = np.repeat(alpha, n_targets)
+    if alpha.shape[0] == 1 and n_targets > 1:
+        alpha = xp.full(
+            shape=(n_targets,), fill_value=alpha[0], dtype=alpha.dtype, device=device_
+        )
 
     n_iter = None
-    if solver == 'sparse_cg':
-        coef = _solve_sparse_cg(X, y, alpha,
-                                max_iter=max_iter,
-                                tol=tol,
-                                verbose=verbose,
-                                X_offset=X_offset,
-                                X_scale=X_scale)
-
-    elif solver == 'lsqr':
-        coef, n_iter = _solve_lsqr(X, y, alpha, max_iter, tol)
-
-    elif solver == 'cholesky':
+    if solver == "sparse_cg":
+        coef = _solve_sparse_cg(
+            X,
+            y,
+            alpha,
+            max_iter=max_iter,
+            tol=tol,
+            verbose=verbose,
+            X_offset=X_offset,
+            X_scale=X_scale,
+            sample_weight_sqrt=sample_weight_sqrt if has_sw else None,
+        )
+
+    elif solver == "lsqr":
+        coef, n_iter = _solve_lsqr(
+            X,
+            y,
+            alpha=alpha,
+            fit_intercept=fit_intercept,
+            max_iter=max_iter,
+            tol=tol,
+            X_offset=X_offset,
+            X_scale=X_scale,
+            sample_weight_sqrt=sample_weight_sqrt if has_sw else None,
+        )
+
+    elif solver == "cholesky":
         if n_features > n_samples:
             K = safe_sparse_dot(X, X.T, dense_output=True)
             try:
@@ -465,28 +743,41 @@ def _ridge_regression(X, y, alpha, sample_weight=None, solver='auto',
                 coef = safe_sparse_dot(X.T, dual_coef, dense_output=True).T
             except linalg.LinAlgError:
                 # use SVD solver if matrix is singular
-                solver = 'svd'
+                solver = "svd"
         else:
             try:
                 coef = _solve_cholesky(X, y, alpha)
             except linalg.LinAlgError:
                 # use SVD solver if matrix is singular
-                solver = 'svd'
+                solver = "svd"
 
-    elif solver in ['sag', 'saga']:
+    elif solver in ["sag", "saga"]:
         # precompute max_squared_sum for all targets
         max_squared_sum = row_norms(X, squared=True).max()
 
         coef = np.empty((y.shape[1], n_features), dtype=X.dtype)
         n_iter = np.empty(y.shape[1], dtype=np.int32)
-        intercept = np.zeros((y.shape[1], ), dtype=X.dtype)
+        intercept = np.zeros((y.shape[1],), dtype=X.dtype)
         for i, (alpha_i, target) in enumerate(zip(alpha, y.T)):
-            init = {'coef': np.zeros((n_features + int(return_intercept), 1),
-                                     dtype=X.dtype)}
+            init = {
+                "coef": np.zeros((n_features + int(return_intercept), 1), dtype=X.dtype)
+            }
             coef_, n_iter_, _ = sag_solver(
-                X, target.ravel(), sample_weight, 'squared', alpha_i, 0,
-                max_iter, tol, verbose, random_state, False, max_squared_sum,
-                init, is_saga=solver == 'saga')
+                X,
+                target.ravel(),
+                sample_weight,
+                "squared",
+                alpha_i,
+                0,
+                max_iter,
+                tol,
+                verbose,
+                random_state,
+                False,
+                max_squared_sum,
+                init,
+                is_saga=solver == "saga",
+            )
             if return_intercept:
                 coef[i] = coef_[:-1]
                 intercept[i] = coef_[-1]
@@ -496,105 +787,223 @@ def _ridge_regression(X, y, alpha, sample_weight=None, solver='auto',
 
         if intercept.shape[0] == 1:
             intercept = intercept[0]
-        coef = np.asarray(coef)
 
-    if solver == 'svd':
-        if sparse.issparse(X):
-            raise TypeError('SVD solver does not support sparse'
-                            ' inputs currently')
-        coef = _solve_svd(X, y, alpha)
+    elif solver == "lbfgs":
+        coef = _solve_lbfgs(
+            X,
+            y,
+            alpha,
+            positive=positive,
+            tol=tol,
+            max_iter=max_iter,
+            X_offset=X_offset,
+            X_scale=X_scale,
+            sample_weight_sqrt=sample_weight_sqrt if has_sw else None,
+        )
+
+    if solver == "svd":
+        if X_is_sparse:
+            raise TypeError("SVD solver does not support sparse inputs currently")
+        coef = _solve_svd(X, y, alpha, xp)
+
+    if n_targets == 1:
+        coef = _ravel(coef)
 
-    if ravel:
-        # When y was passed as a 1d-array, we flatten the coefficients.
-        coef = coef.ravel()
+    coef = xp.asarray(coef)
 
     if return_n_iter and return_intercept:
-        return coef, n_iter, intercept
+        res = coef, n_iter, intercept
     elif return_intercept:
-        return coef, intercept
+        res = coef, intercept
     elif return_n_iter:
-        return coef, n_iter
+        res = coef, n_iter
     else:
-        return coef
+        res = coef
+
+    return (*res, solver) if return_solver else res
+
+
+def resolve_solver(solver, positive, return_intercept, is_sparse, xp):
+    if solver != "auto":
+        return solver
+
+    is_numpy_namespace = _is_numpy_namespace(xp)
+
+    auto_solver_np = resolve_solver_for_numpy(positive, return_intercept, is_sparse)
+    if is_numpy_namespace:
+        return auto_solver_np
+
+    if positive:
+        raise ValueError(
+            "The solvers that support positive fitting do not support "
+            f"Array API dispatch to namespace {xp.__name__}. Please "
+            "either disable Array API dispatch, or use a numpy-like "
+            "namespace, or set `positive=False`."
+        )
+
+    # At the moment, Array API dispatch only supports the "svd" solver.
+    solver = "svd"
+    if solver != auto_solver_np:
+        warnings.warn(
+            f"Using Array API dispatch to namespace {xp.__name__} with "
+            f"`solver='auto'` will result in using the solver '{solver}'. "
+            "The results may differ from those when using a Numpy array, "
+            f"because in that case the preferred solver would be {auto_solver_np}. "
+            f"Set `solver='{solver}'` to suppress this warning."
+        )
+
+    return solver
+
+
+def resolve_solver_for_numpy(positive, return_intercept, is_sparse):
+    if positive:
+        return "lbfgs"
+
+    if return_intercept:
+        # sag supports fitting intercept directly
+        return "sag"
+
+    if not is_sparse:
+        return "cholesky"
+
+    return "sparse_cg"
 
 
 class _BaseRidge(LinearModel, metaclass=ABCMeta):
+    _parameter_constraints: dict = {
+        "alpha": [Interval(Real, 0, None, closed="left"), np.ndarray],
+        "fit_intercept": ["boolean"],
+        "copy_X": ["boolean"],
+        "max_iter": [Interval(Integral, 1, None, closed="left"), None],
+        "tol": [Interval(Real, 0, None, closed="left")],
+        "solver": [
+            StrOptions(
+                {"auto", "svd", "cholesky", "lsqr", "sparse_cg", "sag", "saga", "lbfgs"}
+            )
+        ],
+        "positive": ["boolean"],
+        "random_state": ["random_state"],
+    }
+
     @abstractmethod
-    @_deprecate_positional_args
-    def __init__(self, alpha=1.0, *, fit_intercept=True, normalize=False,
-                 copy_X=True, max_iter=None, tol=1e-3, solver="auto",
-                 random_state=None):
+    def __init__(
+        self,
+        alpha=1.0,
+        *,
+        fit_intercept=True,
+        copy_X=True,
+        max_iter=None,
+        tol=1e-4,
+        solver="auto",
+        positive=False,
+        random_state=None,
+    ):
         self.alpha = alpha
         self.fit_intercept = fit_intercept
-        self.normalize = normalize
         self.copy_X = copy_X
         self.max_iter = max_iter
         self.tol = tol
         self.solver = solver
+        self.positive = positive
         self.random_state = random_state
 
     def fit(self, X, y, sample_weight=None):
+        xp, is_array_api_compliant = get_namespace(X, y, sample_weight)
+
+        if self.solver == "lbfgs" and not self.positive:
+            raise ValueError(
+                "'lbfgs' solver can be used only when positive=True. "
+                "Please use another solver."
+            )
 
-        # all other solvers work at both float precision levels
-        _dtype = [np.float64, np.float32]
-        _accept_sparse = _get_valid_accept_sparse(sparse.issparse(X),
-                                                  self.solver)
-        X, y = self._validate_data(X, y,
-                                   accept_sparse=_accept_sparse,
-                                   dtype=_dtype,
-                                   multi_output=True, y_numeric=True)
-        if sparse.issparse(X) and self.fit_intercept:
-            if self.solver not in ['auto', 'sparse_cg', 'sag']:
+        if self.positive:
+            if self.solver not in ["auto", "lbfgs"]:
+                raise ValueError(
+                    f"solver='{self.solver}' does not support positive fitting. Please"
+                    " set the solver to 'auto' or 'lbfgs', or set `positive=False`"
+                )
+            else:
+                solver = self.solver
+        elif sparse.issparse(X) and self.fit_intercept:
+            if self.solver not in ["auto", "lbfgs", "lsqr", "sag", "sparse_cg"]:
                 raise ValueError(
                     "solver='{}' does not support fitting the intercept "
                     "on sparse data. Please set the solver to 'auto' or "
-                    "'sparse_cg', 'sag', or set `fit_intercept=False`"
-                    .format(self.solver))
-            if (self.solver == 'sag' and self.max_iter is None and
-                    self.tol > 1e-4):
+                    "'lsqr', 'sparse_cg', 'sag', 'lbfgs' "
+                    "or set `fit_intercept=False`".format(self.solver)
+                )
+            if self.solver in ["lsqr", "lbfgs"]:
+                solver = self.solver
+            elif self.solver == "sag" and self.max_iter is None and self.tol > 1e-4:
                 warnings.warn(
                     '"sag" solver requires many iterations to fit '
-                    'an intercept with sparse inputs. Either set the '
+                    "an intercept with sparse inputs. Either set the "
                     'solver to "auto" or "sparse_cg", or set a low '
                     '"tol" and a high "max_iter" (especially if inputs are '
-                    'not standardized).')
-                solver = 'sag'
+                    "not standardized)."
+                )
+                solver = "sag"
             else:
-                solver = 'sparse_cg'
+                solver = "sparse_cg"
         else:
             solver = self.solver
 
         if sample_weight is not None:
-            sample_weight = _check_sample_weight(sample_weight, X,
-                                                 dtype=X.dtype)
+            sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype)
 
         # when X is sparse we only remove offset from y
-        X, y, X_offset, y_offset, X_scale = self._preprocess_data(
-            X, y, self.fit_intercept, self.normalize, self.copy_X,
-            sample_weight=sample_weight, return_mean=True)
-
-        if solver == 'sag' and sparse.issparse(X) and self.fit_intercept:
-            self.coef_, self.n_iter_, self.intercept_ = _ridge_regression(
-                X, y, alpha=self.alpha, sample_weight=sample_weight,
-                max_iter=self.max_iter, tol=self.tol, solver='sag',
-                random_state=self.random_state, return_n_iter=True,
-                return_intercept=True, check_input=False)
+        X, y, X_offset, y_offset, X_scale = _preprocess_data(
+            X,
+            y,
+            fit_intercept=self.fit_intercept,
+            copy=self.copy_X,
+            sample_weight=sample_weight,
+        )
+
+        if solver == "sag" and sparse.issparse(X) and self.fit_intercept:
+            self.coef_, self.n_iter_, self.intercept_, self.solver_ = _ridge_regression(
+                X,
+                y,
+                alpha=self.alpha,
+                sample_weight=sample_weight,
+                max_iter=self.max_iter,
+                tol=self.tol,
+                solver="sag",
+                positive=self.positive,
+                random_state=self.random_state,
+                return_n_iter=True,
+                return_intercept=True,
+                return_solver=True,
+                check_input=False,
+            )
             # add the offset which was subtracted by _preprocess_data
             self.intercept_ += y_offset
 
         else:
             if sparse.issparse(X) and self.fit_intercept:
-                # required to fit intercept with sparse_cg solver
-                params = {'X_offset': X_offset, 'X_scale': X_scale}
+                # required to fit intercept with sparse_cg and lbfgs solver
+                params = {"X_offset": X_offset, "X_scale": X_scale}
             else:
                 # for dense matrices or when intercept is set to 0
                 params = {}
 
-            self.coef_, self.n_iter_ = _ridge_regression(
-                X, y, alpha=self.alpha, sample_weight=sample_weight,
-                max_iter=self.max_iter, tol=self.tol, solver=solver,
-                random_state=self.random_state, return_n_iter=True,
-                return_intercept=False, check_input=False, **params)
+            self.coef_, self.n_iter_, self.solver_ = _ridge_regression(
+                X,
+                y,
+                alpha=self.alpha,
+                sample_weight=sample_weight,
+                max_iter=self.max_iter,
+                tol=self.tol,
+                solver=solver,
+                positive=self.positive,
+                random_state=self.random_state,
+                return_n_iter=True,
+                return_intercept=False,
+                return_solver=True,
+                check_input=False,
+                fit_intercept=self.fit_intercept,
+                **params,
+            )
             self._set_intercept(X_offset, y_offset, X_scale)
 
         return self
@@ -618,28 +1027,22 @@ class Ridge(MultiOutputMixin, RegressorMixin, _BaseRidge):
     Parameters
     ----------
     alpha : {float, ndarray of shape (n_targets,)}, default=1.0
-        Regularization strength; must be a positive float. Regularization
-        improves the conditioning of the problem and reduces the variance of
-        the estimates. Larger values specify stronger regularization.
-        Alpha corresponds to ``1 / (2C)`` in other linear models such as
-        :class:`~sklearn.linear_model.LogisticRegression` or
-        :class:`~sklearn.svm.LinearSVC`. If an array is passed, penalties are
-        assumed to be specific to the targets. Hence they must correspond in
-        number.
+        Constant that multiplies the L2 term, controlling regularization
+        strength. `alpha` must be a non-negative float i.e. in `[0, inf)`.
+
+        When `alpha = 0`, the objective is equivalent to ordinary least
+        squares, solved by the :class:`LinearRegression` object. For numerical
+        reasons, using `alpha = 0` with the `Ridge` object is not advised.
+        Instead, you should use the :class:`LinearRegression` object.
+
+        If an array is passed, penalties are assumed to be specific to the
+        targets. Hence they must correspond in number.
 
     fit_intercept : bool, default=True
         Whether to fit the intercept for this model. If set
         to false, no intercept will be used in calculations
         (i.e. ``X`` and ``y`` are expected to be centered).
 
-    normalize : bool, default=False
-        This parameter is ignored when ``fit_intercept`` is set to False.
-        If True, the regressors X will be normalized before regression by
-        subtracting the mean and dividing by the l2-norm.
-        If you wish to standardize, please use
-        :class:`~sklearn.preprocessing.StandardScaler` before calling ``fit``
-        on an estimator with ``normalize=False``.
-
     copy_X : bool, default=True
         If True, X will be copied; else, it may be overwritten.
 
@@ -647,18 +1050,40 @@ class Ridge(MultiOutputMixin, RegressorMixin, _BaseRidge):
         Maximum number of iterations for conjugate gradient solver.
         For 'sparse_cg' and 'lsqr' solvers, the default value is determined
         by scipy.sparse.linalg. For 'sag' solver, the default value is 1000.
+        For 'lbfgs' solver, the default value is 15000.
+
+    tol : float, default=1e-4
+        The precision of the solution (`coef_`) is determined by `tol` which
+        specifies a different convergence criterion for each solver:
+
+        - 'svd': `tol` has no impact.
+
+        - 'cholesky': `tol` has no impact.
+
+        - 'sparse_cg': norm of residuals smaller than `tol`.
+
+        - 'lsqr': `tol` is set as atol and btol of scipy.sparse.linalg.lsqr,
+          which control the norm of the residual vector in terms of the norms of
+          matrix and coefficients.
+
+        - 'sag' and 'saga': relative change of coef smaller than `tol`.
 
-    tol : float, default=1e-3
-        Precision of the solution.
+        - 'lbfgs': maximum of the absolute (projected) gradient=max|residuals|
+          smaller than `tol`.
 
-    solver : {'auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag', 'saga'}, \
-        default='auto'
+        .. versionchanged:: 1.2
+           Default value changed from 1e-3 to 1e-4 for consistency with other linear
+           models.
+
+    solver : {'auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg', \
+            'sag', 'saga', 'lbfgs'}, default='auto'
         Solver to use in the computational routines:
 
         - 'auto' chooses the solver automatically based on the type of data.
 
         - 'svd' uses a Singular Value Decomposition of X to compute the Ridge
-          coefficients. More stable for singular matrices than 'cholesky'.
+          coefficients. It is the most stable solver, in particular more stable
+          for singular matrices than 'cholesky' at the cost of being slower.
 
         - 'cholesky' uses the standard scipy.linalg.solve function to
           obtain a closed-form solution.
@@ -680,15 +1105,23 @@ class Ridge(MultiOutputMixin, RegressorMixin, _BaseRidge):
           approximately the same scale. You can preprocess the data with a
           scaler from sklearn.preprocessing.
 
-        All last five solvers support both dense and sparse data. However, only
-        'sag' and 'sparse_cg' supports sparse input when `fit_intercept` is
-        True.
+        - 'lbfgs' uses L-BFGS-B algorithm implemented in
+          `scipy.optimize.minimize`. It can be used only when `positive`
+          is True.
+
+        All solvers except 'svd' support both dense and sparse data. However, only
+        'lsqr', 'sag', 'sparse_cg', and 'lbfgs' support sparse input when
+        `fit_intercept` is True.
 
         .. versionadded:: 0.17
            Stochastic Average Gradient descent solver.
         .. versionadded:: 0.19
            SAGA solver.
 
+    positive : bool, default=False
+        When set to ``True``, forces the coefficients to be positive.
+        Only 'lbfgs' solver is supported in this case.
+
     random_state : int, RandomState instance, default=None
         Used when ``solver`` == 'sag' or 'saga' to shuffle the data.
         See :term:`Glossary ` for details.
@@ -711,6 +1144,23 @@ class Ridge(MultiOutputMixin, RegressorMixin, _BaseRidge):
 
         .. versionadded:: 0.17
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    solver_ : str
+        The solver that was used at fit time by the computational
+        routines.
+
+        .. versionadded:: 1.5
+
     See Also
     --------
     RidgeClassifier : Ridge classifier.
@@ -718,6 +1168,14 @@ class Ridge(MultiOutputMixin, RegressorMixin, _BaseRidge):
     :class:`~sklearn.kernel_ridge.KernelRidge` : Kernel ridge regression
         combines ridge regression with the kernel trick.
 
+    Notes
+    -----
+    Regularization improves the conditioning of the problem and
+    reduces the variance of the estimates. Larger values specify stronger
+    regularization. Alpha corresponds to ``1 / (2C)`` in other linear
+    models such as :class:`~sklearn.linear_model.LogisticRegression` or
+    :class:`~sklearn.svm.LinearSVC`.
+
     Examples
     --------
     >>> from sklearn.linear_model import Ridge
@@ -730,26 +1188,41 @@ class Ridge(MultiOutputMixin, RegressorMixin, _BaseRidge):
     >>> clf.fit(X, y)
     Ridge()
     """
-    @_deprecate_positional_args
-    def __init__(self, alpha=1.0, *, fit_intercept=True, normalize=False,
-                 copy_X=True, max_iter=None, tol=1e-3, solver="auto",
-                 random_state=None):
+
+    def __init__(
+        self,
+        alpha=1.0,
+        *,
+        fit_intercept=True,
+        copy_X=True,
+        max_iter=None,
+        tol=1e-4,
+        solver="auto",
+        positive=False,
+        random_state=None,
+    ):
         super().__init__(
-            alpha=alpha, fit_intercept=fit_intercept,
-            normalize=normalize, copy_X=copy_X,
-            max_iter=max_iter, tol=tol, solver=solver,
-            random_state=random_state)
+            alpha=alpha,
+            fit_intercept=fit_intercept,
+            copy_X=copy_X,
+            max_iter=max_iter,
+            tol=tol,
+            solver=solver,
+            positive=positive,
+            random_state=random_state,
+        )
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y, sample_weight=None):
         """Fit Ridge regression model.
 
         Parameters
         ----------
         X : {ndarray, sparse matrix} of shape (n_samples, n_features)
-            Training data
+            Training data.
 
         y : ndarray of shape (n_samples,) or (n_samples, n_targets)
-            Target values
+            Target values.
 
         sample_weight : float or ndarray of shape (n_samples,), default=None
             Individual weights for each sample. If given a float, every sample
@@ -757,12 +1230,123 @@ def fit(self, X, y, sample_weight=None):
 
         Returns
         -------
-        self : returns an instance of self.
+        self : object
+            Fitted estimator.
         """
+        _accept_sparse = _get_valid_accept_sparse(sparse.issparse(X), self.solver)
+        xp, _ = get_namespace(X, y, sample_weight)
+        X, y = validate_data(
+            self,
+            X,
+            y,
+            accept_sparse=_accept_sparse,
+            dtype=[xp.float64, xp.float32],
+            force_writeable=True,
+            multi_output=True,
+            y_numeric=True,
+        )
         return super().fit(X, y, sample_weight=sample_weight)
 
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.array_api_support = True
+        tags.input_tags.sparse = (self.solver != "svd") and (
+            self.solver != "cholesky" or not self.fit_intercept
+        )
+        return tags
+
+
+class _RidgeClassifierMixin(LinearClassifierMixin):
+    def _prepare_data(self, X, y, sample_weight, solver):
+        """Validate `X` and `y` and binarize `y`.
+
+        Parameters
+        ----------
+        X : {ndarray, sparse matrix} of shape (n_samples, n_features)
+            Training data.
+
+        y : ndarray of shape (n_samples,)
+            Target values.
+
+        sample_weight : float or ndarray of shape (n_samples,), default=None
+            Individual weights for each sample. If given a float, every sample
+            will have the same weight.
+
+        solver : str
+            The solver used in `Ridge` to know which sparse format to support.
+
+        Returns
+        -------
+        X : {ndarray, sparse matrix} of shape (n_samples, n_features)
+            Validated training data.
+
+        y : ndarray of shape (n_samples,)
+            Validated target values.
+
+        sample_weight : ndarray of shape (n_samples,)
+            Validated sample weights.
+
+        Y : ndarray of shape (n_samples, n_classes)
+            The binarized version of `y`.
+        """
+        accept_sparse = _get_valid_accept_sparse(sparse.issparse(X), solver)
+        X, y = validate_data(
+            self,
+            X,
+            y,
+            accept_sparse=accept_sparse,
+            multi_output=True,
+            y_numeric=False,
+            force_writeable=True,
+        )
+
+        self._label_binarizer = LabelBinarizer(pos_label=1, neg_label=-1)
+        Y = self._label_binarizer.fit_transform(y)
+        if not self._label_binarizer.y_type_.startswith("multilabel"):
+            y = column_or_1d(y, warn=True)
+
+        sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype)
+        if self.class_weight:
+            sample_weight = sample_weight * compute_sample_weight(self.class_weight, y)
+        return X, y, sample_weight, Y
+
+    def predict(self, X):
+        """Predict class labels for samples in `X`.
+
+        Parameters
+        ----------
+        X : {array-like, spare matrix} of shape (n_samples, n_features)
+            The data matrix for which we want to predict the targets.
 
-class RidgeClassifier(LinearClassifierMixin, _BaseRidge):
+        Returns
+        -------
+        y_pred : ndarray of shape (n_samples,) or (n_samples, n_outputs)
+            Vector or matrix containing the predictions. In binary and
+            multiclass problems, this is a vector containing `n_samples`. In
+            a multilabel problem, it returns a matrix of shape
+            `(n_samples, n_outputs)`.
+        """
+        check_is_fitted(self, attributes=["_label_binarizer"])
+        if self._label_binarizer.y_type_.startswith("multilabel"):
+            # Threshold such that the negative label is -1 and positive label
+            # is 1 to use the inverse transform of the label binarizer fitted
+            # during fit.
+            scores = 2 * (self.decision_function(X) > 0) - 1
+            return self._label_binarizer.inverse_transform(scores)
+        return super().predict(X)
+
+    @property
+    def classes_(self):
+        """Classes labels."""
+        return self._label_binarizer.classes_
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.classifier_tags.multi_label = True
+        return tags
+
+
+class RidgeClassifier(_RidgeClassifierMixin, _BaseRidge):
     """Classifier using Ridge regression.
 
     This classifier first converts the target values into ``{-1, 1}`` and
@@ -786,14 +1370,6 @@ class RidgeClassifier(LinearClassifierMixin, _BaseRidge):
         intercept will be used in calculations (e.g. data is expected to be
         already centered).
 
-    normalize : bool, default=False
-        This parameter is ignored when ``fit_intercept`` is set to False.
-        If True, the regressors X will be normalized before regression by
-        subtracting the mean and dividing by the l2-norm.
-        If you wish to standardize, please use
-        :class:`~sklearn.preprocessing.StandardScaler` before calling ``fit``
-        on an estimator with ``normalize=False``.
-
     copy_X : bool, default=True
         If True, X will be copied; else, it may be overwritten.
 
@@ -801,8 +1377,28 @@ class RidgeClassifier(LinearClassifierMixin, _BaseRidge):
         Maximum number of iterations for conjugate gradient solver.
         The default value is determined by scipy.sparse.linalg.
 
-    tol : float, default=1e-3
-        Precision of the solution.
+    tol : float, default=1e-4
+        The precision of the solution (`coef_`) is determined by `tol` which
+        specifies a different convergence criterion for each solver:
+
+        - 'svd': `tol` has no impact.
+
+        - 'cholesky': `tol` has no impact.
+
+        - 'sparse_cg': norm of residuals smaller than `tol`.
+
+        - 'lsqr': `tol` is set as atol and btol of scipy.sparse.linalg.lsqr,
+          which control the norm of the residual vector in terms of the norms of
+          matrix and coefficients.
+
+        - 'sag' and 'saga': relative change of coef smaller than `tol`.
+
+        - 'lbfgs': maximum of the absolute (projected) gradient=max|residuals|
+          smaller than `tol`.
+
+        .. versionchanged:: 1.2
+           Default value changed from 1e-3 to 1e-4 for consistency with other linear
+           models.
 
     class_weight : dict or 'balanced', default=None
         Weights associated with classes in the form ``{class_label: weight}``.
@@ -812,14 +1408,15 @@ class RidgeClassifier(LinearClassifierMixin, _BaseRidge):
         weights inversely proportional to class frequencies in the input data
         as ``n_samples / (n_classes * np.bincount(y))``.
 
-    solver : {'auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag', 'saga'}, \
-        default='auto'
+    solver : {'auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg', \
+            'sag', 'saga', 'lbfgs'}, default='auto'
         Solver to use in the computational routines:
 
         - 'auto' chooses the solver automatically based on the type of data.
 
         - 'svd' uses a Singular Value Decomposition of X to compute the Ridge
-          coefficients. More stable for singular matrices than 'cholesky'.
+          coefficients. It is the most stable solver, in particular more stable
+          for singular matrices than 'cholesky' at the cost of being slower.
 
         - 'cholesky' uses the standard scipy.linalg.solve function to
           obtain a closed-form solution.
@@ -844,7 +1441,15 @@ class RidgeClassifier(LinearClassifierMixin, _BaseRidge):
           .. versionadded:: 0.17
              Stochastic Average Gradient descent solver.
           .. versionadded:: 0.19
-           SAGA solver.
+             SAGA solver.
+
+        - 'lbfgs' uses L-BFGS-B algorithm implemented in
+          `scipy.optimize.minimize`. It can be used only when `positive`
+          is True.
+
+    positive : bool, default=False
+        When set to ``True``, forces the coefficients to be positive.
+        Only 'lbfgs' solver is supported in this case.
 
     random_state : int, RandomState instance, default=None
         Used when ``solver`` == 'sag' or 'saga' to shuffle the data.
@@ -868,6 +1473,23 @@ class RidgeClassifier(LinearClassifierMixin, _BaseRidge):
     classes_ : ndarray of shape (n_classes,)
         The classes labels.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    solver_ : str
+        The solver that was used at fit time by the computational
+        routines.
+
+        .. versionadded:: 1.5
+
     See Also
     --------
     Ridge : Ridge regression.
@@ -888,16 +1510,38 @@ class RidgeClassifier(LinearClassifierMixin, _BaseRidge):
     >>> clf.score(X, y)
     0.9595...
     """
-    @_deprecate_positional_args
-    def __init__(self, alpha=1.0, *, fit_intercept=True, normalize=False,
-                 copy_X=True, max_iter=None, tol=1e-3, class_weight=None,
-                 solver="auto", random_state=None):
+
+    _parameter_constraints: dict = {
+        **_BaseRidge._parameter_constraints,
+        "class_weight": [dict, StrOptions({"balanced"}), None],
+    }
+
+    def __init__(
+        self,
+        alpha=1.0,
+        *,
+        fit_intercept=True,
+        copy_X=True,
+        max_iter=None,
+        tol=1e-4,
+        class_weight=None,
+        solver="auto",
+        positive=False,
+        random_state=None,
+    ):
         super().__init__(
-            alpha=alpha, fit_intercept=fit_intercept, normalize=normalize,
-            copy_X=copy_X, max_iter=max_iter, tol=tol, solver=solver,
-            random_state=random_state)
+            alpha=alpha,
+            fit_intercept=fit_intercept,
+            copy_X=copy_X,
+            max_iter=max_iter,
+            tol=tol,
+            solver=solver,
+            positive=positive,
+            random_state=random_state,
+        )
         self.class_weight = class_weight
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y, sample_weight=None):
         """Fit Ridge classifier model.
 
@@ -914,56 +1558,34 @@ def fit(self, X, y, sample_weight=None):
             will have the same weight.
 
             .. versionadded:: 0.17
-               *sample_weight* support to Classifier.
+               *sample_weight* support to RidgeClassifier.
 
         Returns
         -------
         self : object
             Instance of the estimator.
         """
-        _accept_sparse = _get_valid_accept_sparse(sparse.issparse(X),
-                                                  self.solver)
-        X, y = self._validate_data(X, y, accept_sparse=_accept_sparse,
-                                   multi_output=True, y_numeric=False)
-        sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype)
-
-        self._label_binarizer = LabelBinarizer(pos_label=1, neg_label=-1)
-        Y = self._label_binarizer.fit_transform(y)
-        if not self._label_binarizer.y_type_.startswith('multilabel'):
-            y = column_or_1d(y, warn=True)
-        else:
-            # we don't (yet) support multi-label classification in Ridge
-            raise ValueError(
-                "%s doesn't support multi-label classification" % (
-                    self.__class__.__name__))
-
-        if self.class_weight:
-            # modify the sample weights with the corresponding class weight
-            sample_weight = (sample_weight *
-                             compute_sample_weight(self.class_weight, y))
+        X, y, sample_weight, Y = self._prepare_data(X, y, sample_weight, self.solver)
 
         super().fit(X, Y, sample_weight=sample_weight)
         return self
 
-    @property
-    def classes_(self):
-        return self._label_binarizer.classes_
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = (self.solver != "svd") and (
+            self.solver != "cholesky" or not self.fit_intercept
+        )
+        return tags
 
 
 def _check_gcv_mode(X, gcv_mode):
-    possible_gcv_modes = [None, 'auto', 'svd', 'eigen']
-    if gcv_mode not in possible_gcv_modes:
-        raise ValueError(
-            "Unknown value for 'gcv_mode'. "
-            "Got {} instead of one of {}" .format(
-                gcv_mode, possible_gcv_modes))
-    if gcv_mode in ['eigen', 'svd']:
+    if gcv_mode in ["eigen", "svd"]:
         return gcv_mode
     # if X has more rows than columns, use decomposition of X^T.X,
     # otherwise X.X^T
     if X.shape[0] > X.shape[1]:
-        return 'svd'
-    return 'eigen'
+        return "svd"
+    return "eigen"
 
 
 def _find_smallest_angle(query, vectors):
@@ -1000,15 +1622,18 @@ def __init__(self, X, X_mean, sqrt_sw):
 
     def _matvec(self, v):
         v = v.ravel()
-        return safe_sparse_dot(
-            self.X, v[:-1], dense_output=True
-        ) - self.sqrt_sw * self.X_mean.dot(v[:-1]) + v[-1] * self.sqrt_sw
+        return (
+            safe_sparse_dot(self.X, v[:-1], dense_output=True)
+            - self.sqrt_sw * self.X_mean.dot(v[:-1])
+            + v[-1] * self.sqrt_sw
+        )
 
     def _matmat(self, v):
         return (
-            safe_sparse_dot(self.X, v[:-1], dense_output=True) -
-            self.sqrt_sw[:, None] * self.X_mean.dot(v[:-1]) + v[-1] *
-            self.sqrt_sw[:, None])
+            safe_sparse_dot(self.X, v[:-1], dense_output=True)
+            - self.sqrt_sw[:, None] * self.X_mean.dot(v[:-1])
+            + v[-1] * self.sqrt_sw[:, None]
+        )
 
     def _transpose(self):
         return _XT_CenterStackOp(self.X, self.X_mean, self.sqrt_sw)
@@ -1032,9 +1657,8 @@ def _matvec(self, v):
         v = v.ravel()
         n_features = self.shape[0]
         res = np.empty(n_features, dtype=self.X.dtype)
-        res[:-1] = (
-            safe_sparse_dot(self.X.T, v, dense_output=True) -
-            (self.X_mean * self.sqrt_sw.dot(v))
+        res[:-1] = safe_sparse_dot(self.X.T, v, dense_output=True) - (
+            self.X_mean * self.sqrt_sw.dot(v)
         )
         res[-1] = np.dot(v, self.sqrt_sw)
         return res
@@ -1042,15 +1666,14 @@ def _matvec(self, v):
     def _matmat(self, v):
         n_features = self.shape[0]
         res = np.empty((n_features, v.shape[1]), dtype=self.X.dtype)
-        res[:-1] = (
-            safe_sparse_dot(self.X.T, v, dense_output=True) -
-            self.X_mean[:, None] * self.sqrt_sw.dot(v)
-        )
+        res[:-1] = safe_sparse_dot(self.X.T, v, dense_output=True) - self.X_mean[
+            :, None
+        ] * self.sqrt_sw.dot(v)
         res[-1] = np.dot(self.sqrt_sw, v)
         return res
 
 
-class _IdentityRegressor:
+class _IdentityRegressor(RegressorMixin, BaseEstimator):
     """Fake regressor which will directly output the prediction."""
 
     def decision_function(self, y_predict):
@@ -1060,12 +1683,13 @@ def predict(self, y_predict):
         return y_predict
 
 
-class _IdentityClassifier(LinearClassifierMixin):
+class _IdentityClassifier(LinearClassifierMixin, BaseEstimator):
     """Fake classifier which will directly output the prediction.
 
     We inherit from LinearClassifierMixin to get the proper shape for the
     output `y`.
     """
+
     def __init__(self, classes):
         self.classes_ = classes
 
@@ -1078,6 +1702,28 @@ class _RidgeGCV(LinearModel):
 
     This class is not intended to be used directly. Use RidgeCV instead.
 
+    `_RidgeGCV` uses a Generalized Cross-Validation for model selection. It's an
+    efficient approximation of leave-one-out cross-validation (LOO-CV), where instead of
+    computing multiple models by excluding one data point at a time, it uses an
+    algebraic shortcut to approximate the LOO-CV error, making it faster and
+    computationally more efficient.
+
+    Using a naive grid-search approach with a leave-one-out cross-validation in contrast
+    requires to fit `n_samples` models to compute the prediction error for each sample
+    and then to repeat this process for each alpha in the grid.
+
+    Here, the prediction error for each sample is computed by solving a **single**
+    linear system (in other words a single model) via a matrix factorization (i.e.
+    eigendecomposition or SVD) solving the problem stated in the Notes section. Finally,
+    we need to repeat this process for each alpha in the grid. The detailed complexity
+    is further discussed in Sect. 4 in [1].
+
+    This algebraic approach is only applicable for regularized least squares
+    problems. It could potentially be extended to kernel ridge regression.
+
+    See the Notes section and references for more details regarding the formulation
+    and the linear system that is solved.
+
     Notes
     -----
 
@@ -1110,36 +1756,42 @@ class _RidgeGCV(LinearModel):
 
     References
     ----------
-    http://cbcl.mit.edu/publications/ps/MIT-CSAIL-TR-2007-025.pdf
-    https://www.mit.edu/~9.520/spring07/Classes/rlsslides.pdf
+    [1] http://cbcl.mit.edu/publications/ps/MIT-CSAIL-TR-2007-025.pdf
+    [2] https://www.mit.edu/~9.520/spring07/Classes/rlsslides.pdf
     """
-    @_deprecate_positional_args
-    def __init__(self, alphas=(0.1, 1.0, 10.0), *,
-                 fit_intercept=True, normalize=False,
-                 scoring=None, copy_X=True,
-                 gcv_mode=None, store_cv_values=False,
-                 is_clf=False, alpha_per_target=False):
-        self.alphas = np.asarray(alphas)
+
+    def __init__(
+        self,
+        alphas=(0.1, 1.0, 10.0),
+        *,
+        fit_intercept=True,
+        scoring=None,
+        copy_X=True,
+        gcv_mode=None,
+        store_cv_results=False,
+        is_clf=False,
+        alpha_per_target=False,
+    ):
+        self.alphas = alphas
         self.fit_intercept = fit_intercept
-        self.normalize = normalize
         self.scoring = scoring
         self.copy_X = copy_X
         self.gcv_mode = gcv_mode
-        self.store_cv_values = store_cv_values
+        self.store_cv_results = store_cv_results
         self.is_clf = is_clf
         self.alpha_per_target = alpha_per_target
 
     @staticmethod
     def _decomp_diag(v_prime, Q):
         # compute diagonal of the matrix: dot(Q, dot(diag(v_prime), Q^T))
-        return (v_prime * Q ** 2).sum(axis=-1)
+        return (v_prime * Q**2).sum(axis=-1)
 
     @staticmethod
     def _diag_dot(D, B):
         # compute dot(diag(D), B)
         if len(B.shape) > 1:
             # handle case where B is > 1-d
-            D = D[(slice(None), ) + (np.newaxis, ) * (len(B.shape) - 1)]
+            D = D[(slice(None),) + (np.newaxis,) * (len(B.shape) - 1)]
         return D * B
 
     def _compute_gram(self, X, sqrt_sw):
@@ -1182,15 +1834,17 @@ def _compute_gram(self, X, sqrt_sw):
         # X is sparse
         n_samples = X.shape[0]
         sample_weight_matrix = sparse.dia_matrix(
-            (sqrt_sw, 0), shape=(n_samples, n_samples))
+            (sqrt_sw, 0), shape=(n_samples, n_samples)
+        )
         X_weighted = sample_weight_matrix.dot(X)
         X_mean, _ = mean_variance_axis(X_weighted, axis=0)
         X_mean *= n_samples / sqrt_sw.dot(sqrt_sw)
-        X_mX = sqrt_sw[:, None] * safe_sparse_dot(
-            X_mean, X.T, dense_output=True)
+        X_mX = sqrt_sw[:, None] * safe_sparse_dot(X_mean, X.T, dense_output=True)
         X_mX_m = np.outer(sqrt_sw, sqrt_sw) * np.dot(X_mean, X_mean)
-        return (safe_sparse_dot(X, X.T, dense_output=True) + X_mX_m
-                - X_mX - X_mX.T, X_mean)
+        return (
+            safe_sparse_dot(X, X.T, dense_output=True) + X_mX_m - X_mX - X_mX.T,
+            X_mean,
+        )
 
     def _compute_covariance(self, X, sqrt_sw):
         """Computes covariance matrix X^TX with possible centering.
@@ -1228,18 +1882,21 @@ def _compute_covariance(self, X, sqrt_sw):
         # this function only gets called for sparse X
         n_samples = X.shape[0]
         sample_weight_matrix = sparse.dia_matrix(
-            (sqrt_sw, 0), shape=(n_samples, n_samples))
+            (sqrt_sw, 0), shape=(n_samples, n_samples)
+        )
         X_weighted = sample_weight_matrix.dot(X)
         X_mean, _ = mean_variance_axis(X_weighted, axis=0)
         X_mean = X_mean * n_samples / sqrt_sw.dot(sqrt_sw)
         weight_sum = sqrt_sw.dot(sqrt_sw)
-        return (safe_sparse_dot(X.T, X, dense_output=True) -
-                weight_sum * np.outer(X_mean, X_mean),
-                X_mean)
+        return (
+            safe_sparse_dot(X.T, X, dense_output=True)
+            - weight_sum * np.outer(X_mean, X_mean),
+            X_mean,
+        )
 
     def _sparse_multidot_diag(self, X, A, X_mean, sqrt_sw):
         """Compute the diagonal of (X - X_mean).dot(A).dot((X - X_mean).T)
-        without explicitely centering X nor computing X.dot(A)
+        without explicitly centering X nor computing X.dot(A)
         when X is sparse.
 
         Parameters
@@ -1264,14 +1921,13 @@ def _sparse_multidot_diag(self, X, A, X_mean, sqrt_sw):
         for start in range(0, X.shape[0], batch_size):
             batch = slice(start, min(X.shape[0], start + batch_size), 1)
             X_batch = np.empty(
-                (X[batch].shape[0], X.shape[1] + self.fit_intercept),
-                dtype=X.dtype
+                (X[batch].shape[0], X.shape[1] + self.fit_intercept), dtype=X.dtype
             )
             if self.fit_intercept:
-                X_batch[:, :-1] = X[batch].A - X_mean * scale[batch][:, None]
+                X_batch[:, :-1] = X[batch].toarray() - X_mean * scale[batch][:, None]
                 X_batch[:, -1] = intercept_col[batch]
             else:
-                X_batch = X[batch].A
+                X_batch = X[batch].toarray()
             diag[batch] = (X_batch.dot(A) * X_batch).sum(axis=1)
         return diag
 
@@ -1294,7 +1950,7 @@ def _solve_eigen_gram(self, alpha, y, sqrt_sw, X_mean, eigvals, Q, QT_y):
 
         Used when we have a decomposition of X.X^T (n_samples <= n_features).
         """
-        w = 1. / (eigvals + alpha)
+        w = 1.0 / (eigvals + alpha)
         if self.fit_intercept:
             # the vector containing the square roots of the sample weights (1
             # when no sample weights) is the eigenvector of XX^T which
@@ -1338,7 +1994,8 @@ def _eigen_decompose_covariance(self, X, y, sqrt_sw):
         return X_mean, eigvals, V, X
 
     def _solve_eigen_covariance_no_intercept(
-            self, alpha, y, sqrt_sw, X_mean, eigvals, V, X):
+        self, alpha, y, sqrt_sw, X_mean, eigvals, V, X
+    ):
         """Compute dual coefficients and diagonal of G^-1.
 
         Used when we have a decomposition of X^T.X
@@ -1355,7 +2012,8 @@ def _solve_eigen_covariance_no_intercept(
         return (1 - hat_diag) / alpha, (y - y_hat) / alpha
 
     def _solve_eigen_covariance_intercept(
-            self, alpha, y, sqrt_sw, X_mean, eigvals, V, X):
+        self, alpha, y, sqrt_sw, X_mean, eigvals, V, X
+    ):
         """Compute dual coefficients and diagonal of G^-1.
 
         Used when we have a decomposition of X^T.X
@@ -1384,8 +2042,7 @@ def _solve_eigen_covariance_intercept(
             hat_diag = hat_diag[:, np.newaxis]
         return (1 - hat_diag) / alpha, (y - y_hat) / alpha
 
-    def _solve_eigen_covariance(
-            self, alpha, y, sqrt_sw, X_mean, eigvals, V, X):
+    def _solve_eigen_covariance(self, alpha, y, sqrt_sw, X_mean, eigvals, V, X):
         """Compute dual coefficients and diagonal of G^-1.
 
         Used when we have a decomposition of X^T.X
@@ -1393,9 +2050,11 @@ def _solve_eigen_covariance(
         """
         if self.fit_intercept:
             return self._solve_eigen_covariance_intercept(
-                alpha, y, sqrt_sw, X_mean, eigvals, V, X)
+                alpha, y, sqrt_sw, X_mean, eigvals, V, X
+            )
         return self._solve_eigen_covariance_no_intercept(
-            alpha, y, sqrt_sw, X_mean, eigvals, V, X)
+            alpha, y, sqrt_sw, X_mean, eigvals, V, X
+        )
 
     def _svd_decompose_design_matrix(self, X, y, sqrt_sw):
         # X already centered
@@ -1407,32 +2066,31 @@ def _svd_decompose_design_matrix(self, X, y, sqrt_sw):
             intercept_column = sqrt_sw[:, None]
             X = np.hstack((X, intercept_column))
         U, singvals, _ = linalg.svd(X, full_matrices=0)
-        singvals_sq = singvals ** 2
+        singvals_sq = singvals**2
         UT_y = np.dot(U.T, y)
         return X_mean, singvals_sq, U, UT_y
 
-    def _solve_svd_design_matrix(
-            self, alpha, y, sqrt_sw, X_mean, singvals_sq, U, UT_y):
+    def _solve_svd_design_matrix(self, alpha, y, sqrt_sw, X_mean, singvals_sq, U, UT_y):
         """Compute dual coefficients and diagonal of G^-1.
 
         Used when we have an SVD decomposition of X
         (n_samples > n_features and X is dense).
         """
-        w = ((singvals_sq + alpha) ** -1) - (alpha ** -1)
+        w = ((singvals_sq + alpha) ** -1) - (alpha**-1)
         if self.fit_intercept:
             # detect intercept column
             normalized_sw = sqrt_sw / np.linalg.norm(sqrt_sw)
             intercept_dim = _find_smallest_angle(normalized_sw, U)
             # cancel the regularization for the intercept
-            w[intercept_dim] = - (alpha ** -1)
-        c = np.dot(U, self._diag_dot(w, UT_y)) + (alpha ** -1) * y
-        G_inverse_diag = self._decomp_diag(w, U) + (alpha ** -1)
+            w[intercept_dim] = -(alpha**-1)
+        c = np.dot(U, self._diag_dot(w, UT_y)) + (alpha**-1) * y
+        G_inverse_diag = self._decomp_diag(w, U) + (alpha**-1)
         if len(y.shape) != 1:
             # handle case where y is 2-d
             G_inverse_diag = G_inverse_diag[:, np.newaxis]
         return G_inverse_diag, c
 
-    def fit(self, X, y, sample_weight=None):
+    def fit(self, X, y, sample_weight=None, score_params=None):
         """Fit Ridge regression model with gcv.
 
         Parameters
@@ -1445,15 +2103,30 @@ def fit(self, X, y, sample_weight=None):
 
         sample_weight : float or ndarray of shape (n_samples,), default=None
             Individual weights for each sample. If given a float, every sample
-            will have the same weight.
+            will have the same weight. Note that the scale of `sample_weight`
+            has an impact on the loss; i.e. multiplying all weights by `k`
+            is equivalent to setting `alpha / k`.
+
+        score_params : dict, default=None
+            Parameters to be passed to the underlying scorer.
+
+            .. versionadded:: 1.5
+                See :ref:`Metadata Routing User Guide ` for
+                more details.
 
         Returns
         -------
         self : object
         """
-        X, y = self._validate_data(X, y, accept_sparse=['csr', 'csc', 'coo'],
-                                   dtype=[np.float64],
-                                   multi_output=True, y_numeric=True)
+        X, y = validate_data(
+            self,
+            X,
+            y,
+            accept_sparse=["csr", "csc", "coo"],
+            dtype=[np.float64],
+            multi_output=True,
+            y_numeric=True,
+        )
 
         # alpha_per_target cannot be used in classifier mode. All subclasses
         # of _RidgeGCV that are classifiers keep alpha_per_target at its
@@ -1461,24 +2134,25 @@ def fit(self, X, y, sample_weight=None):
         assert not (self.is_clf and self.alpha_per_target)
 
         if sample_weight is not None:
-            sample_weight = _check_sample_weight(sample_weight, X,
-                                                 dtype=X.dtype)
+            sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype)
 
-        if np.any(self.alphas <= 0):
-            raise ValueError(
-                "alphas must be strictly positive. Got {} containing some "
-                "negative or null value instead.".format(self.alphas))
+        self.alphas = np.asarray(self.alphas)
 
-        X, y, X_offset, y_offset, X_scale = LinearModel._preprocess_data(
-            X, y, self.fit_intercept, self.normalize, self.copy_X,
-            sample_weight=sample_weight)
+        unscaled_y = y
+        X, y, X_offset, y_offset, X_scale = _preprocess_data(
+            X,
+            y,
+            fit_intercept=self.fit_intercept,
+            copy=self.copy_X,
+            sample_weight=sample_weight,
+        )
 
         gcv_mode = _check_gcv_mode(X, self.gcv_mode)
 
-        if gcv_mode == 'eigen':
+        if gcv_mode == "eigen":
             decompose = self._eigen_decompose_gram
             solve = self._solve_eigen_gram
-        elif gcv_mode == 'svd':
+        elif gcv_mode == "svd":
             if sparse.issparse(X):
                 decompose = self._eigen_decompose_covariance
                 solve = self._solve_eigen_covariance
@@ -1489,58 +2163,48 @@ def fit(self, X, y, sample_weight=None):
         n_samples = X.shape[0]
 
         if sample_weight is not None:
-            X, y = _rescale_data(X, y, sample_weight)
-            sqrt_sw = np.sqrt(sample_weight)
+            X, y, sqrt_sw = _rescale_data(X, y, sample_weight)
         else:
             sqrt_sw = np.ones(n_samples, dtype=X.dtype)
 
         X_mean, *decomposition = decompose(X, y, sqrt_sw)
 
-        scorer = check_scoring(self, scoring=self.scoring, allow_none=True)
-        error = scorer is None
-
         n_y = 1 if len(y.shape) == 1 else y.shape[1]
         n_alphas = 1 if np.ndim(self.alphas) == 0 else len(self.alphas)
 
-        if self.store_cv_values:
-            self.cv_values_ = np.empty(
-                (n_samples * n_y, n_alphas), dtype=X.dtype)
+        if self.store_cv_results:
+            self.cv_results_ = np.empty((n_samples * n_y, n_alphas), dtype=X.dtype)
 
         best_coef, best_score, best_alpha = None, None, None
 
         for i, alpha in enumerate(np.atleast_1d(self.alphas)):
-            G_inverse_diag, c = solve(
-                float(alpha), y, sqrt_sw, X_mean, *decomposition)
-            if error:
+            G_inverse_diag, c = solve(float(alpha), y, sqrt_sw, X_mean, *decomposition)
+            if self.scoring is None:
                 squared_errors = (c / G_inverse_diag) ** 2
-                if self.alpha_per_target:
-                    alpha_score = -squared_errors.mean(axis=0)
-                else:
-                    alpha_score = -squared_errors.mean()
-                if self.store_cv_values:
-                    self.cv_values_[:, i] = squared_errors.ravel()
+                alpha_score = self._score_without_scorer(squared_errors=squared_errors)
+                if self.store_cv_results:
+                    self.cv_results_[:, i] = squared_errors.ravel()
             else:
                 predictions = y - (c / G_inverse_diag)
-                if self.store_cv_values:
-                    self.cv_values_[:, i] = predictions.ravel()
-
-                if self.is_clf:
-                    identity_estimator = _IdentityClassifier(
-                        classes=np.arange(n_y)
-                    )
-                    alpha_score = scorer(identity_estimator,
-                                         predictions, y.argmax(axis=1))
-                else:
-                    identity_estimator = _IdentityRegressor()
-                    if self.alpha_per_target:
-                        alpha_score = np.array([
-                            scorer(identity_estimator,
-                                   predictions[:, j], y[:, j])
-                            for j in range(n_y)
-                        ])
+                # Rescale predictions back to original scale
+                if sample_weight is not None:  # avoid the unnecessary division by ones
+                    if predictions.ndim > 1:
+                        predictions /= sqrt_sw[:, None]
                     else:
-                        alpha_score = scorer(identity_estimator,
-                                             predictions.ravel(), y.ravel())
+                        predictions /= sqrt_sw
+                predictions += y_offset
+
+                if self.store_cv_results:
+                    self.cv_results_[:, i] = predictions.ravel()
+
+                score_params = score_params or {}
+                alpha_score = self._score(
+                    predictions=predictions,
+                    y=unscaled_y,
+                    n_y=n_y,
+                    scorer=self.scoring,
+                    score_params=score_params,
+                )
 
             # Keep track of the best model
             if best_score is None:
@@ -1567,36 +2231,102 @@ def fit(self, X, y, sample_weight=None):
         self.best_score_ = best_score
         self.dual_coef_ = best_coef
         self.coef_ = safe_sparse_dot(self.dual_coef_.T, X)
+        if y.ndim == 1 or y.shape[1] == 1:
+            self.coef_ = self.coef_.ravel()
 
-        X_offset += X_mean * X_scale
+        if sparse.issparse(X):
+            X_offset = X_mean * X_scale
+        else:
+            X_offset += X_mean * X_scale
         self._set_intercept(X_offset, y_offset, X_scale)
 
-        if self.store_cv_values:
+        if self.store_cv_results:
             if len(y.shape) == 1:
-                cv_values_shape = n_samples, n_alphas
+                cv_results_shape = n_samples, n_alphas
             else:
-                cv_values_shape = n_samples, n_y, n_alphas
-            self.cv_values_ = self.cv_values_.reshape(cv_values_shape)
+                cv_results_shape = n_samples, n_y, n_alphas
+            self.cv_results_ = self.cv_results_.reshape(cv_results_shape)
 
         return self
 
+    def _score_without_scorer(self, squared_errors):
+        """Performs scoring using squared errors when the scorer is None."""
+        if self.alpha_per_target:
+            _score = -squared_errors.mean(axis=0)
+        else:
+            _score = -squared_errors.mean()
+
+        return _score
+
+    def _score(self, *, predictions, y, n_y, scorer, score_params):
+        """Performs scoring with the specified scorer using the
+        predictions and the true y values.
+        """
+        if self.is_clf:
+            identity_estimator = _IdentityClassifier(classes=np.arange(n_y))
+            _score = scorer(
+                identity_estimator,
+                predictions,
+                y.argmax(axis=1),
+                **score_params,
+            )
+        else:
+            identity_estimator = _IdentityRegressor()
+            if self.alpha_per_target:
+                _score = np.array(
+                    [
+                        scorer(
+                            identity_estimator,
+                            predictions[:, j],
+                            y[:, j],
+                            **score_params,
+                        )
+                        for j in range(n_y)
+                    ]
+                )
+            else:
+                _score = scorer(identity_estimator, predictions, y, **score_params)
+
+        return _score
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        # Required since this is neither a RegressorMixin nor a ClassifierMixin
+        tags.target_tags.required = True
+        return tags
+
 
 class _BaseRidgeCV(LinearModel):
-    @_deprecate_positional_args
-    def __init__(self, alphas=(0.1, 1.0, 10.0), *,
-                 fit_intercept=True, normalize=False, scoring=None,
-                 cv=None, gcv_mode=None, store_cv_values=False,
-                 alpha_per_target=False):
-        self.alphas = np.asarray(alphas)
+    _parameter_constraints: dict = {
+        "alphas": ["array-like", Interval(Real, 0, None, closed="neither")],
+        "fit_intercept": ["boolean"],
+        "scoring": [StrOptions(set(get_scorer_names())), callable, None],
+        "cv": ["cv_object"],
+        "gcv_mode": [StrOptions({"auto", "svd", "eigen"}), None],
+        "store_cv_results": ["boolean"],
+        "alpha_per_target": ["boolean"],
+    }
+
+    def __init__(
+        self,
+        alphas=(0.1, 1.0, 10.0),
+        *,
+        fit_intercept=True,
+        scoring=None,
+        cv=None,
+        gcv_mode=None,
+        store_cv_results=False,
+        alpha_per_target=False,
+    ):
+        self.alphas = alphas
         self.fit_intercept = fit_intercept
-        self.normalize = normalize
         self.scoring = scoring
         self.cv = cv
         self.gcv_mode = gcv_mode
-        self.store_cv_values = store_cv_values
+        self.store_cv_results = store_cv_results
         self.alpha_per_target = alpha_per_target
 
-    def fit(self, X, y, sample_weight=None):
+    def fit(self, X, y, sample_weight=None, **params):
         """Fit Ridge regression model with cv.
 
         Parameters
@@ -1612,58 +2342,175 @@ def fit(self, X, y, sample_weight=None):
             Individual weights for each sample. If given a float, every sample
             will have the same weight.
 
+        **params : dict, default=None
+            Extra parameters for the underlying scorer.
+
+            .. versionadded:: 1.5
+                Only available if `enable_metadata_routing=True`,
+                which can be set by using
+                ``sklearn.set_config(enable_metadata_routing=True)``.
+                See :ref:`Metadata Routing User Guide ` for
+                more details.
+
         Returns
         -------
         self : object
+            Fitted estimator.
 
         Notes
         -----
         When sample_weight is provided, the selected hyperparameter may depend
-        on whether we use leave-one-out cross-validation (cv=None or cv='auto')
+        on whether we use leave-one-out cross-validation (cv=None)
         or another form of cross-validation, because only leave-one-out
         cross-validation takes the sample weights into account when computing
         the validation score.
         """
+        _raise_for_params(params, self, "fit")
         cv = self.cv
+        scorer = self._get_scorer()
+
+        # `_RidgeGCV` does not work for alpha = 0
+        if cv is None:
+            check_scalar_alpha = partial(
+                check_scalar,
+                target_type=numbers.Real,
+                min_val=0.0,
+                include_boundaries="neither",
+            )
+        else:
+            check_scalar_alpha = partial(
+                check_scalar,
+                target_type=numbers.Real,
+                min_val=0.0,
+                include_boundaries="left",
+            )
+
+        if isinstance(self.alphas, (np.ndarray, list, tuple)):
+            n_alphas = 1 if np.ndim(self.alphas) == 0 else len(self.alphas)
+            if n_alphas != 1:
+                for index, alpha in enumerate(self.alphas):
+                    alpha = check_scalar_alpha(alpha, f"alphas[{index}]")
+            else:
+                self.alphas[0] = check_scalar_alpha(self.alphas[0], "alphas")
+        alphas = np.asarray(self.alphas)
+
+        if sample_weight is not None:
+            params["sample_weight"] = sample_weight
+
         if cv is None:
-            estimator = _RidgeGCV(self.alphas,
-                                  fit_intercept=self.fit_intercept,
-                                  normalize=self.normalize,
-                                  scoring=self.scoring,
-                                  gcv_mode=self.gcv_mode,
-                                  store_cv_values=self.store_cv_values,
-                                  is_clf=is_classifier(self),
-                                  alpha_per_target=self.alpha_per_target)
-            estimator.fit(X, y, sample_weight=sample_weight)
+            if _routing_enabled():
+                routed_params = process_routing(
+                    self,
+                    "fit",
+                    **params,
+                )
+            else:
+                routed_params = Bunch(scorer=Bunch(score={}))
+                if sample_weight is not None:
+                    routed_params.scorer.score["sample_weight"] = sample_weight
+
+            # reset `scorer` variable to original user-intend if no scoring is passed
+            if self.scoring is None:
+                scorer = None
+
+            estimator = _RidgeGCV(
+                alphas,
+                fit_intercept=self.fit_intercept,
+                scoring=scorer,
+                gcv_mode=self.gcv_mode,
+                store_cv_results=self.store_cv_results,
+                is_clf=is_classifier(self),
+                alpha_per_target=self.alpha_per_target,
+            )
+            estimator.fit(
+                X,
+                y,
+                sample_weight=sample_weight,
+                score_params=routed_params.scorer.score,
+            )
             self.alpha_ = estimator.alpha_
             self.best_score_ = estimator.best_score_
-            if self.store_cv_values:
-                self.cv_values_ = estimator.cv_values_
+            if self.store_cv_results:
+                self.cv_results_ = estimator.cv_results_
         else:
-            if self.store_cv_values:
-                raise ValueError("cv!=None and store_cv_values=True"
-                                 " are incompatible")
+            if self.store_cv_results:
+                raise ValueError("cv!=None and store_cv_results=True are incompatible")
             if self.alpha_per_target:
-                raise ValueError("cv!=None and alpha_per_target=True"
-                                 " are incompatible")
-            parameters = {'alpha': self.alphas}
-            solver = 'sparse_cg' if sparse.issparse(X) else 'auto'
+                raise ValueError("cv!=None and alpha_per_target=True are incompatible")
+
+            parameters = {"alpha": alphas}
+            solver = "sparse_cg" if sparse.issparse(X) else "auto"
             model = RidgeClassifier if is_classifier(self) else Ridge
-            gs = GridSearchCV(model(fit_intercept=self.fit_intercept,
-                                    normalize=self.normalize,
-                                    solver=solver),
-                              parameters, cv=cv, scoring=self.scoring)
-            gs.fit(X, y, sample_weight=sample_weight)
-            estimator = gs.best_estimator_
-            self.alpha_ = gs.best_estimator_.alpha
-            self.best_score_ = gs.best_score_
+            estimator = model(
+                fit_intercept=self.fit_intercept,
+                solver=solver,
+            )
+            if _routing_enabled():
+                estimator.set_fit_request(sample_weight=True)
+
+            grid_search = GridSearchCV(
+                estimator,
+                parameters,
+                cv=cv,
+                scoring=scorer,
+            )
+
+            grid_search.fit(X, y, **params)
+            estimator = grid_search.best_estimator_
+            self.alpha_ = grid_search.best_estimator_.alpha
+            self.best_score_ = grid_search.best_score_
 
         self.coef_ = estimator.coef_
         self.intercept_ = estimator.intercept_
         self.n_features_in_ = estimator.n_features_in_
+        if hasattr(estimator, "feature_names_in_"):
+            self.feature_names_in_ = estimator.feature_names_in_
 
         return self
 
+    def get_metadata_routing(self):
+        """Get metadata routing of this object.
+
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
+
+        .. versionadded:: 1.5
+
+        Returns
+        -------
+        routing : MetadataRouter
+            A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
+            routing information.
+        """
+        router = (
+            MetadataRouter(owner=self.__class__.__name__)
+            .add_self_request(self)
+            .add(
+                scorer=self.scoring,
+                method_mapping=MethodMapping().add(caller="fit", callee="score"),
+            )
+            .add(
+                splitter=self.cv,
+                method_mapping=MethodMapping().add(caller="fit", callee="split"),
+            )
+        )
+        return router
+
+    def _get_scorer(self):
+        scorer = check_scoring(estimator=self, scoring=self.scoring, allow_none=True)
+        if _routing_enabled() and self.scoring is None:
+            # This estimator passes an array of 1s as sample_weight even if
+            # sample_weight is not provided by the user. Therefore we need to
+            # always request it. But we don't set it if it's passed explicitly
+            # by the user.
+            scorer.set_score_request(sample_weight=True)
+        return scorer
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
+        return tags
+
 
 class RidgeCV(MultiOutputMixin, RegressorMixin, _BaseRidgeCV):
     """Ridge regression with built-in cross-validation.
@@ -1676,7 +2523,7 @@ class RidgeCV(MultiOutputMixin, RegressorMixin, _BaseRidgeCV):
 
     Parameters
     ----------
-    alphas : ndarray of shape (n_alphas,), default=(0.1, 1.0, 10.0)
+    alphas : array-like of shape (n_alphas,), default=(0.1, 1.0, 10.0)
         Array of alpha values to try.
         Regularization strength; must be a positive float. Regularization
         improves the conditioning of the problem and reduces the variance of
@@ -1684,28 +2531,22 @@ class RidgeCV(MultiOutputMixin, RegressorMixin, _BaseRidgeCV):
         Alpha corresponds to ``1 / (2C)`` in other linear models such as
         :class:`~sklearn.linear_model.LogisticRegression` or
         :class:`~sklearn.svm.LinearSVC`.
-        If using Leave-One-Out cross-validation, alphas must be positive.
+        If using Leave-One-Out cross-validation, alphas must be strictly positive.
 
     fit_intercept : bool, default=True
         Whether to calculate the intercept for this model. If set
         to false, no intercept will be used in calculations
         (i.e. data is expected to be centered).
 
-    normalize : bool, default=False
-        This parameter is ignored when ``fit_intercept`` is set to False.
-        If True, the regressors X will be normalized before regression by
-        subtracting the mean and dividing by the l2-norm.
-        If you wish to standardize, please use
-        :class:`~sklearn.preprocessing.StandardScaler` before calling ``fit``
-        on an estimator with ``normalize=False``.
-
-    scoring : string, callable, default=None
-        A string (see model evaluation documentation) or
-        a scorer callable object / function with signature
-        ``scorer(estimator, X, y)``.
-        If None, the negative mean squared error if cv is 'auto' or None
-        (i.e. when using leave-one-out cross-validation), and r2 score
-        otherwise.
+    scoring : str, callable, default=None
+        The scoring method to use for cross-validation. Options:
+
+        - str: see :ref:`scoring_string_names` for options.
+        - callable: a scorer callable object (e.g., function) with signature
+          ``scorer(estimator, X, y)``. See :ref:`scoring_callable` for details.
+        - `None`: negative :ref:`mean squared error ` if cv is
+          None (i.e. when using leave-one-out cross-validation), or
+          :ref:`coefficient of determination ` (:math:`R^2`) otherwise.
 
     cv : int, cross-validation generator or an iterable, default=None
         Determines the cross-validation splitting strategy.
@@ -1723,7 +2564,7 @@ class RidgeCV(MultiOutputMixin, RegressorMixin, _BaseRidgeCV):
         Refer :ref:`User Guide ` for the various
         cross-validation strategies that can be used here.
 
-    gcv_mode : {'auto', 'svd', eigen'}, default='auto'
+    gcv_mode : {'auto', 'svd', 'eigen'}, default='auto'
         Flag indicating which strategy to use when performing
         Leave-One-Out Cross-Validation. Options are::
 
@@ -1735,12 +2576,15 @@ class RidgeCV(MultiOutputMixin, RegressorMixin, _BaseRidgeCV):
         The 'auto' mode is the default and is intended to pick the cheaper
         option of the two depending on the shape of the training data.
 
-    store_cv_values : bool, default=False
+    store_cv_results : bool, default=False
         Flag indicating if the cross-validation values corresponding to
-        each alpha should be stored in the ``cv_values_`` attribute (see
+        each alpha should be stored in the ``cv_results_`` attribute (see
         below). This flag is only compatible with ``cv=None`` (i.e. using
         Leave-One-Out Cross-Validation).
 
+        .. versionchanged:: 1.5
+            Parameter name changed from `store_cv_values` to `store_cv_results`.
+
     alpha_per_target : bool, default=False
         Flag indicating whether to optimize the alpha value (picked from the
         `alphas` parameter list) for each target separately (for multi-output
@@ -1752,13 +2596,16 @@ class RidgeCV(MultiOutputMixin, RegressorMixin, _BaseRidgeCV):
 
     Attributes
     ----------
-    cv_values_ : ndarray of shape (n_samples, n_alphas) or \
-        shape (n_samples, n_targets, n_alphas), optional
+    cv_results_ : ndarray of shape (n_samples, n_alphas) or \
+            shape (n_samples, n_targets, n_alphas), optional
         Cross-validation values for each alpha (only available if
-        ``store_cv_values=True`` and ``cv=None``). After ``fit()`` has been
-        called, this attribute will contain the mean squared errors
-        (by default) or the values of the ``{loss,score}_func`` function
-        (if provided in the constructor).
+        ``store_cv_results=True`` and ``cv=None``). After ``fit()`` has been
+        called, this attribute will contain the mean squared errors if
+        `scoring is None` otherwise it will contain standardized per point
+        prediction values.
+
+        .. versionchanged:: 1.5
+            `cv_values_` changed to `cv_results_`.
 
     coef_ : ndarray of shape (n_features) or (n_targets, n_features)
         Weight vector(s).
@@ -1777,6 +2624,23 @@ class RidgeCV(MultiOutputMixin, RegressorMixin, _BaseRidgeCV):
 
         .. versionadded:: 0.23
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    Ridge : Ridge regression.
+    RidgeClassifier : Classifier based on ridge regression on {-1, 1} labels.
+    RidgeClassifierCV : Ridge classifier with built-in cross validation.
+
     Examples
     --------
     >>> from sklearn.datasets import load_diabetes
@@ -1785,16 +2649,53 @@ class RidgeCV(MultiOutputMixin, RegressorMixin, _BaseRidgeCV):
     >>> clf = RidgeCV(alphas=[1e-3, 1e-2, 1e-1, 1]).fit(X, y)
     >>> clf.score(X, y)
     0.5166...
-
-    See Also
-    --------
-    Ridge : Ridge regression.
-    RidgeClassifier : Ridge classifier.
-    RidgeClassifierCV : Ridge classifier with built-in cross validation.
     """
 
+    @_fit_context(prefer_skip_nested_validation=True)
+    def fit(self, X, y, sample_weight=None, **params):
+        """Fit Ridge regression model with cv.
+
+        Parameters
+        ----------
+        X : ndarray of shape (n_samples, n_features)
+            Training data. If using GCV, will be cast to float64
+            if necessary.
+
+        y : ndarray of shape (n_samples,) or (n_samples, n_targets)
+            Target values. Will be cast to X's dtype if necessary.
+
+        sample_weight : float or ndarray of shape (n_samples,), default=None
+            Individual weights for each sample. If given a float, every sample
+            will have the same weight.
+
+        **params : dict, default=None
+            Parameters to be passed to the underlying scorer.
+
+            .. versionadded:: 1.5
+                Only available if `enable_metadata_routing=True`,
+                which can be set by using
+                ``sklearn.set_config(enable_metadata_routing=True)``.
+                See :ref:`Metadata Routing User Guide ` for
+                more details.
+
+        Returns
+        -------
+        self : object
+            Fitted estimator.
+
+        Notes
+        -----
+        When sample_weight is provided, the selected hyperparameter may depend
+        on whether we use leave-one-out cross-validation (cv=None)
+        or another form of cross-validation, because only leave-one-out
+        cross-validation takes the sample weights into account when computing
+        the validation score.
+        """
+        super().fit(X, y, sample_weight=sample_weight, **params)
+        return self
+
 
-class RidgeClassifierCV(LinearClassifierMixin, _BaseRidgeCV):
+class RidgeClassifierCV(_RidgeClassifierMixin, _BaseRidgeCV):
     """Ridge classifier with built-in cross-validation.
 
     See glossary entry for :term:`cross-validation estimator`.
@@ -1806,7 +2707,7 @@ class RidgeClassifierCV(LinearClassifierMixin, _BaseRidgeCV):
 
     Parameters
     ----------
-    alphas : ndarray of shape (n_alphas,), default=(0.1, 1.0, 10.0)
+    alphas : array-like of shape (n_alphas,), default=(0.1, 1.0, 10.0)
         Array of alpha values to try.
         Regularization strength; must be a positive float. Regularization
         improves the conditioning of the problem and reduces the variance of
@@ -1814,24 +2715,22 @@ class RidgeClassifierCV(LinearClassifierMixin, _BaseRidgeCV):
         Alpha corresponds to ``1 / (2C)`` in other linear models such as
         :class:`~sklearn.linear_model.LogisticRegression` or
         :class:`~sklearn.svm.LinearSVC`.
+        If using Leave-One-Out cross-validation, alphas must be strictly positive.
 
     fit_intercept : bool, default=True
         Whether to calculate the intercept for this model. If set
         to false, no intercept will be used in calculations
         (i.e. data is expected to be centered).
 
-    normalize : bool, default=False
-        This parameter is ignored when ``fit_intercept`` is set to False.
-        If True, the regressors X will be normalized before regression by
-        subtracting the mean and dividing by the l2-norm.
-        If you wish to standardize, please use
-        :class:`~sklearn.preprocessing.StandardScaler` before calling ``fit``
-        on an estimator with ``normalize=False``.
+    scoring : str, callable, default=None
+        The scoring method to use for cross-validation. Options:
 
-    scoring : string, callable, default=None
-        A string (see model evaluation documentation) or
-        a scorer callable object / function with signature
-        ``scorer(estimator, X, y)``.
+        - str: see :ref:`scoring_string_names` for options.
+        - callable: a scorer callable object (e.g., function) with signature
+          ``scorer(estimator, X, y)``. See :ref:`scoring_callable` for details.
+        - `None`: negative :ref:`mean squared error ` if cv is
+          None (i.e. when using leave-one-out cross-validation), or
+          :ref:`accuracy ` otherwise.
 
     cv : int, cross-validation generator or an iterable, default=None
         Determines the cross-validation splitting strategy.
@@ -1851,22 +2750,27 @@ class RidgeClassifierCV(LinearClassifierMixin, _BaseRidgeCV):
 
         The "balanced" mode uses the values of y to automatically adjust
         weights inversely proportional to class frequencies in the input data
-        as ``n_samples / (n_classes * np.bincount(y))``
+        as ``n_samples / (n_classes * np.bincount(y))``.
 
-    store_cv_values : bool, default=False
-        Flag indicating if the cross-validation values corresponding to
-        each alpha should be stored in the ``cv_values_`` attribute (see
+    store_cv_results : bool, default=False
+        Flag indicating if the cross-validation results corresponding to
+        each alpha should be stored in the ``cv_results_`` attribute (see
         below). This flag is only compatible with ``cv=None`` (i.e. using
         Leave-One-Out Cross-Validation).
 
+        .. versionchanged:: 1.5
+            Parameter name changed from `store_cv_values` to `store_cv_results`.
+
     Attributes
     ----------
-    cv_values_ : ndarray of shape (n_samples, n_targets, n_alphas), optional
-        Cross-validation values for each alpha (if ``store_cv_values=True`` and
+    cv_results_ : ndarray of shape (n_samples, n_targets, n_alphas), optional
+        Cross-validation results for each alpha (only if ``store_cv_results=True`` and
         ``cv=None``). After ``fit()`` has been called, this attribute will
-        contain the mean squared errors (by default) or the values of the
-        ``{loss,score}_func`` function (if provided in the constructor). This
-        attribute exists only when ``store_cv_values`` is True.
+        contain the mean squared errors if `scoring is None` otherwise it
+        will contain standardized per point prediction values.
+
+        .. versionchanged:: 1.5
+            `cv_values_` changed to `cv_results_`.
 
     coef_ : ndarray of shape (1, n_features) or (n_targets, n_features)
         Coefficient of the features in the decision function.
@@ -1888,14 +2792,16 @@ class RidgeClassifierCV(LinearClassifierMixin, _BaseRidgeCV):
     classes_ : ndarray of shape (n_classes,)
         The classes labels.
 
-    Examples
-    --------
-    >>> from sklearn.datasets import load_breast_cancer
-    >>> from sklearn.linear_model import RidgeClassifierCV
-    >>> X, y = load_breast_cancer(return_X_y=True)
-    >>> clf = RidgeClassifierCV(alphas=[1e-3, 1e-2, 1e-1, 1]).fit(X, y)
-    >>> clf.score(X, y)
-    0.9630...
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
 
     See Also
     --------
@@ -1908,24 +2814,52 @@ class RidgeClassifierCV(LinearClassifierMixin, _BaseRidgeCV):
     For multi-class classification, n_class classifiers are trained in
     a one-versus-all approach. Concretely, this is implemented by taking
     advantage of the multi-variate response support in Ridge.
+
+    Examples
+    --------
+    >>> from sklearn.datasets import load_breast_cancer
+    >>> from sklearn.linear_model import RidgeClassifierCV
+    >>> X, y = load_breast_cancer(return_X_y=True)
+    >>> clf = RidgeClassifierCV(alphas=[1e-3, 1e-2, 1e-1, 1]).fit(X, y)
+    >>> clf.score(X, y)
+    0.9630...
     """
-    @_deprecate_positional_args
-    def __init__(self, alphas=(0.1, 1.0, 10.0), *, fit_intercept=True,
-                 normalize=False, scoring=None, cv=None, class_weight=None,
-                 store_cv_values=False):
+
+    _parameter_constraints: dict = {
+        **_BaseRidgeCV._parameter_constraints,
+        "class_weight": [dict, StrOptions({"balanced"}), None],
+    }
+    for param in ("gcv_mode", "alpha_per_target"):
+        _parameter_constraints.pop(param)
+
+    def __init__(
+        self,
+        alphas=(0.1, 1.0, 10.0),
+        *,
+        fit_intercept=True,
+        scoring=None,
+        cv=None,
+        class_weight=None,
+        store_cv_results=False,
+    ):
         super().__init__(
-            alphas=alphas, fit_intercept=fit_intercept, normalize=normalize,
-            scoring=scoring, cv=cv, store_cv_values=store_cv_values)
+            alphas=alphas,
+            fit_intercept=fit_intercept,
+            scoring=scoring,
+            cv=cv,
+            store_cv_results=store_cv_results,
+        )
         self.class_weight = class_weight
 
-    def fit(self, X, y, sample_weight=None):
+    @_fit_context(prefer_skip_nested_validation=True)
+    def fit(self, X, y, sample_weight=None, **params):
         """Fit Ridge classifier with cv.
 
         Parameters
         ----------
         X : ndarray of shape (n_samples, n_features)
-            Training vectors, where n_samples is the number of samples
-            and n_features is the number of features. When using GCV,
+            Training vectors, where `n_samples` is the number of samples
+            and `n_features` is the number of features. When using GCV,
             will be cast to float64 if necessary.
 
         y : ndarray of shape (n_samples,)
@@ -1935,36 +2869,31 @@ def fit(self, X, y, sample_weight=None):
             Individual weights for each sample. If given a float, every sample
             will have the same weight.
 
+        **params : dict, default=None
+            Parameters to be passed to the underlying scorer.
+
+            .. versionadded:: 1.5
+                Only available if `enable_metadata_routing=True`,
+                which can be set by using
+                ``sklearn.set_config(enable_metadata_routing=True)``.
+                See :ref:`Metadata Routing User Guide ` for
+                more details.
+
         Returns
         -------
         self : object
+            Fitted estimator.
         """
-        X, y = self._validate_data(X, y, accept_sparse=['csr', 'csc', 'coo'],
-                                   multi_output=True, y_numeric=False)
-        sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype)
-
-        self._label_binarizer = LabelBinarizer(pos_label=1, neg_label=-1)
-        Y = self._label_binarizer.fit_transform(y)
-        if not self._label_binarizer.y_type_.startswith('multilabel'):
-            y = column_or_1d(y, warn=True)
-
-        if self.class_weight:
-            # modify the sample weights with the corresponding class weight
-            sample_weight = (sample_weight *
-                             compute_sample_weight(self.class_weight, y))
-
+        # `RidgeClassifier` does not accept "sag" or "saga" solver and thus support
+        # csr, csc, and coo sparse matrices. By using solver="eigen" we force to accept
+        # all sparse format.
+        X, y, sample_weight, Y = self._prepare_data(X, y, sample_weight, solver="eigen")
+
+        # If cv is None, gcv mode will be used and we used the binarized Y
+        # since y will not be binarized in _RidgeGCV estimator.
+        # If cv is not None, a GridSearchCV with some RidgeClassifier
+        # estimators are used where y will be binarized. Thus, we pass y
+        # instead of the binarized Y.
         target = Y if self.cv is None else y
-        _BaseRidgeCV.fit(self, X, target, sample_weight=sample_weight)
+        super().fit(X, target, sample_weight=sample_weight, **params)
         return self
-
-    @property
-    def classes_(self):
-        return self._label_binarizer.classes_
-
-    def _more_tags(self):
-        return {
-            '_xfail_checks': {
-                'check_sample_weights_invariance':
-                'zero sample_weight is not equivalent to removing samples',
-            }
-        }
diff --git a/sklearn/linear_model/_sag.py b/sklearn/linear_model/_sag.py
index d0bd001081d61..12e5d049b0b1f 100644
--- a/sklearn/linear_model/_sag.py
+++ b/sklearn/linear_model/_sag.py
@@ -1,25 +1,23 @@
 """Solvers for Ridge and LogisticRegression using SAG algorithm"""
 
-# Authors: Tom Dupre la Tour 
-#
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import warnings
 
 import numpy as np
 
-from ._base import make_dataset
-from ._sag_fast import sag32, sag64
 from ..exceptions import ConvergenceWarning
 from ..utils import check_array
-from ..utils.validation import _check_sample_weight
-from ..utils.validation import _deprecate_positional_args
 from ..utils.extmath import row_norms
+from ..utils.validation import _check_sample_weight
+from ._base import make_dataset
+from ._sag_fast import sag32, sag64
 
 
-def get_auto_step_size(max_squared_sum, alpha_scaled, loss, fit_intercept,
-                       n_samples=None,
-                       is_saga=False):
+def get_auto_step_size(
+    max_squared_sum, alpha_scaled, loss, fit_intercept, n_samples=None, is_saga=False
+):
     """Compute automatic step size for SAG solver.
 
     The step size is set to 1 / (alpha_scaled + L + fit_intercept) where L is
@@ -59,38 +57,49 @@ def get_auto_step_size(max_squared_sum, alpha_scaled, loss, fit_intercept,
     Minimizing finite sums with the stochastic average gradient
     https://hal.inria.fr/hal-00860051/document
 
-    Defazio, A., Bach F. & Lacoste-Julien S. (2014).
-    SAGA: A Fast Incremental Gradient Method With Support
-    for Non-Strongly Convex Composite Objectives
-    https://arxiv.org/abs/1407.0202
+    :arxiv:`Defazio, A., Bach F. & Lacoste-Julien S. (2014).
+    "SAGA: A Fast Incremental Gradient Method With Support
+    for Non-Strongly Convex Composite Objectives" <1407.0202>`
     """
-    if loss in ('log', 'multinomial'):
-        L = (0.25 * (max_squared_sum + int(fit_intercept)) + alpha_scaled)
-    elif loss == 'squared':
+    if loss in ("log", "multinomial"):
+        L = 0.25 * (max_squared_sum + int(fit_intercept)) + alpha_scaled
+    elif loss == "squared":
         # inverse Lipschitz constant for squared loss
         L = max_squared_sum + int(fit_intercept) + alpha_scaled
     else:
-        raise ValueError("Unknown loss function for SAG solver, got %s "
-                         "instead of 'log' or 'squared'" % loss)
+        raise ValueError(
+            "Unknown loss function for SAG solver, got %s instead of 'log' or 'squared'"
+            % loss
+        )
     if is_saga:
         # SAGA theoretical step size is 1/3L or 1 / (2 * (L + mu n))
         # See Defazio et al. 2014
         mun = min(2 * n_samples * alpha_scaled, L)
-        step = 1. / (2 * L + mun)
+        step = 1.0 / (2 * L + mun)
     else:
         # SAG theoretical step size is 1/16L but it is recommended to use 1 / L
         # see http://www.birs.ca//workshops//2014/14w5003/files/schmidt.pdf,
         # slide 65
-        step = 1. / L
+        step = 1.0 / L
     return step
 
 
-@_deprecate_positional_args
-def sag_solver(X, y, sample_weight=None, loss='log', alpha=1., beta=0.,
-               max_iter=1000, tol=0.001, verbose=0, random_state=None,
-               check_input=True, max_squared_sum=None,
-               warm_start_mem=None,
-               is_saga=False):
+def sag_solver(
+    X,
+    y,
+    sample_weight=None,
+    loss="log",
+    alpha=1.0,
+    beta=0.0,
+    max_iter=1000,
+    tol=0.001,
+    verbose=0,
+    random_state=None,
+    check_input=True,
+    max_squared_sum=None,
+    warm_start_mem=None,
+    is_saga=False,
+):
     """SAG solver for Ridge and LogisticRegression.
 
     SAG stands for Stochastic Average Gradient: the gradient of the loss is
@@ -118,7 +127,7 @@ def sag_solver(X, y, sample_weight=None, loss='log', alpha=1., beta=0.,
 
     y : ndarray of shape (n_samples,)
         Target values. With loss='multinomial', y must be label encoded
-        (see preprocessing.LabelEncoder).
+        (see preprocessing.LabelEncoder). For loss='log' it must be in [0, 1].
 
     sample_weight : array-like of shape (n_samples,), default=None
         Weights applied to individual samples (1. for unweighted).
@@ -145,7 +154,7 @@ def sag_solver(X, y, sample_weight=None, loss='log', alpha=1., beta=0.,
         The max number of passes over the training data if the stopping
         criteria is not reached.
 
-    tol : double, default=0.001
+    tol : float, default=0.001
         The stopping criteria for the weights. The iterations will stop when
         max(change in weights) / max(weights) < tol.
 
@@ -210,10 +219,9 @@ def sag_solver(X, y, sample_weight=None, loss='log', alpha=1., beta=0.,
 
     >>> X = np.array([[-1, -1], [-2, -1], [1, 1], [2, 1]])
     >>> y = np.array([1, 1, 2, 2])
-    >>> clf = linear_model.LogisticRegression(
-    ...     solver='sag', multi_class='multinomial')
+    >>> clf = linear_model.LogisticRegression(solver='sag')
     >>> clf.fit(X, y)
-    LogisticRegression(multi_class='multinomial', solver='sag')
+    LogisticRegression(solver='sag')
 
     References
     ----------
@@ -221,10 +229,9 @@ def sag_solver(X, y, sample_weight=None, loss='log', alpha=1., beta=0.,
     Minimizing finite sums with the stochastic average gradient
     https://hal.inria.fr/hal-00860051/document
 
-    Defazio, A., Bach F. & Lacoste-Julien S. (2014).
-    SAGA: A Fast Incremental Gradient Method With Support
-    for Non-Strongly Convex Composite Objectives
-    https://arxiv.org/abs/1407.0202
+    :arxiv:`Defazio, A., Bach F. & Lacoste-Julien S. (2014).
+    "SAGA: A Fast Incremental Gradient Method With Support
+    for Non-Strongly Convex Composite Objectives" <1407.0202>`
 
     See Also
     --------
@@ -239,8 +246,8 @@ def sag_solver(X, y, sample_weight=None, loss='log', alpha=1., beta=0.,
 
     if check_input:
         _dtype = [np.float64, np.float32]
-        X = check_array(X, dtype=_dtype, accept_sparse='csr', order='C')
-        y = check_array(y, dtype=_dtype, ensure_2d=False, order='C')
+        X = check_array(X, dtype=_dtype, accept_sparse="csr", order="C")
+        y = check_array(y, dtype=_dtype, ensure_2d=False, order="C")
 
     n_samples, n_features = X.shape[0], X.shape[1]
     # As in SGD, the alpha is scaled by n_samples.
@@ -248,17 +255,16 @@ def sag_solver(X, y, sample_weight=None, loss='log', alpha=1., beta=0.,
     beta_scaled = float(beta) / n_samples
 
     # if loss == 'multinomial', y should be label encoded.
-    n_classes = int(y.max()) + 1 if loss == 'multinomial' else 1
+    n_classes = int(y.max()) + 1 if loss == "multinomial" else 1
 
     # initialization
     sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype)
 
-    if 'coef' in warm_start_mem.keys():
-        coef_init = warm_start_mem['coef']
+    if "coef" in warm_start_mem.keys():
+        coef_init = warm_start_mem["coef"]
     else:
         # assume fit_intercept is False
-        coef_init = np.zeros((n_features, n_classes), dtype=X.dtype,
-                             order='C')
+        coef_init = np.zeros((n_features, n_classes), dtype=X.dtype, order="C")
 
     # coef_init contains possibly the intercept_init at the end.
     # Note that Ridge centers the data before fitting, so fit_intercept=False.
@@ -269,29 +275,29 @@ def sag_solver(X, y, sample_weight=None, loss='log', alpha=1., beta=0.,
     else:
         intercept_init = np.zeros(n_classes, dtype=X.dtype)
 
-    if 'intercept_sum_gradient' in warm_start_mem.keys():
-        intercept_sum_gradient = warm_start_mem['intercept_sum_gradient']
+    if "intercept_sum_gradient" in warm_start_mem.keys():
+        intercept_sum_gradient = warm_start_mem["intercept_sum_gradient"]
     else:
         intercept_sum_gradient = np.zeros(n_classes, dtype=X.dtype)
 
-    if 'gradient_memory' in warm_start_mem.keys():
-        gradient_memory_init = warm_start_mem['gradient_memory']
+    if "gradient_memory" in warm_start_mem.keys():
+        gradient_memory_init = warm_start_mem["gradient_memory"]
     else:
-        gradient_memory_init = np.zeros((n_samples, n_classes),
-                                        dtype=X.dtype, order='C')
-    if 'sum_gradient' in warm_start_mem.keys():
-        sum_gradient_init = warm_start_mem['sum_gradient']
+        gradient_memory_init = np.zeros(
+            (n_samples, n_classes), dtype=X.dtype, order="C"
+        )
+    if "sum_gradient" in warm_start_mem.keys():
+        sum_gradient_init = warm_start_mem["sum_gradient"]
     else:
-        sum_gradient_init = np.zeros((n_features, n_classes),
-                                     dtype=X.dtype, order='C')
+        sum_gradient_init = np.zeros((n_features, n_classes), dtype=X.dtype, order="C")
 
-    if 'seen' in warm_start_mem.keys():
-        seen_init = warm_start_mem['seen']
+    if "seen" in warm_start_mem.keys():
+        seen_init = warm_start_mem["seen"]
     else:
-        seen_init = np.zeros(n_samples, dtype=np.int32, order='C')
+        seen_init = np.zeros(n_samples, dtype=np.int32, order="C")
 
-    if 'num_seen' in warm_start_mem.keys():
-        num_seen_init = warm_start_mem['num_seen']
+    if "num_seen" in warm_start_mem.keys():
+        num_seen_init = warm_start_mem["num_seen"]
     else:
         num_seen_init = 0
 
@@ -299,44 +305,64 @@ def sag_solver(X, y, sample_weight=None, loss='log', alpha=1., beta=0.,
 
     if max_squared_sum is None:
         max_squared_sum = row_norms(X, squared=True).max()
-    step_size = get_auto_step_size(max_squared_sum, alpha_scaled, loss,
-                                   fit_intercept, n_samples=n_samples,
-                                   is_saga=is_saga)
+    step_size = get_auto_step_size(
+        max_squared_sum,
+        alpha_scaled,
+        loss,
+        fit_intercept,
+        n_samples=n_samples,
+        is_saga=is_saga,
+    )
     if step_size * alpha_scaled == 1:
-        raise ZeroDivisionError("Current sag implementation does not handle "
-                                "the case step_size * alpha_scaled == 1")
+        raise ZeroDivisionError(
+            "Current sag implementation does not handle "
+            "the case step_size * alpha_scaled == 1"
+        )
 
     sag = sag64 if X.dtype == np.float64 else sag32
-    num_seen, n_iter_ = sag(dataset, coef_init,
-                            intercept_init, n_samples,
-                            n_features, n_classes, tol,
-                            max_iter,
-                            loss,
-                            step_size, alpha_scaled,
-                            beta_scaled,
-                            sum_gradient_init,
-                            gradient_memory_init,
-                            seen_init,
-                            num_seen_init,
-                            fit_intercept,
-                            intercept_sum_gradient,
-                            intercept_decay,
-                            is_saga,
-                            verbose)
+    num_seen, n_iter_ = sag(
+        dataset,
+        coef_init,
+        intercept_init,
+        n_samples,
+        n_features,
+        n_classes,
+        tol,
+        max_iter,
+        loss,
+        step_size,
+        alpha_scaled,
+        beta_scaled,
+        sum_gradient_init,
+        gradient_memory_init,
+        seen_init,
+        num_seen_init,
+        fit_intercept,
+        intercept_sum_gradient,
+        intercept_decay,
+        is_saga,
+        verbose,
+    )
 
     if n_iter_ == max_iter:
-        warnings.warn("The max_iter was reached which means "
-                      "the coef_ did not converge", ConvergenceWarning)
+        warnings.warn(
+            "The max_iter was reached which means the coef_ did not converge",
+            ConvergenceWarning,
+        )
 
     if fit_intercept:
         coef_init = np.vstack((coef_init, intercept_init))
 
-    warm_start_mem = {'coef': coef_init, 'sum_gradient': sum_gradient_init,
-                      'intercept_sum_gradient': intercept_sum_gradient,
-                      'gradient_memory': gradient_memory_init,
-                      'seen': seen_init, 'num_seen': num_seen}
+    warm_start_mem = {
+        "coef": coef_init,
+        "sum_gradient": sum_gradient_init,
+        "intercept_sum_gradient": intercept_sum_gradient,
+        "gradient_memory": gradient_memory_init,
+        "seen": seen_init,
+        "num_seen": num_seen,
+    }
 
-    if loss == 'multinomial':
+    if loss == "multinomial":
         coef_ = coef_init.T
     else:
         coef_ = coef_init[:, 0]
diff --git a/sklearn/linear_model/_sag_fast.pyx.tp b/sklearn/linear_model/_sag_fast.pyx.tp
index 8508340e3b329..906928673b0b7 100644
--- a/sklearn/linear_model/_sag_fast.pyx.tp
+++ b/sklearn/linear_model/_sag_fast.pyx.tp
@@ -8,242 +8,102 @@ Template file for easily generate fused types consistent code using Tempita
 Generated file: sag_fast.pyx
 
 Each class is duplicated for all dtypes (float and double). The keywords
-between double braces are substituted in setup.py.
-
-Authors: Danny Sullivan 
-         Tom Dupre la Tour 
-         Arthur Mensch 
-         Joan Massich 
-
-License: BSD 3 clause
+between double braces are substituted during the build.
 """
 
-# name, c_type
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+# name_suffix, c_type, np_type
 dtypes = [('64', 'double', 'np.float64'),
           ('32', 'float', 'np.float32')]
 
-def get_dispatch(dtypes):
-    for name, c_type, np_type in dtypes:
-        yield name, c_type, np_type
-
 }}
+"""SAG and SAGA implementation"""
 
-#------------------------------------------------------------------------------
-
-# cython: cdivision=True
-# cython: boundscheck=False
-# cython: wraparound=False
-#
-# Authors: Danny Sullivan 
-#          Tom Dupre la Tour 
-#          Arthur Mensch  y:
         return x
     return y
 
 {{endfor}}
 
+{{for name_suffix, c_type, np_type in dtypes}}
 
-{{for name, c_type, np_type in get_dispatch(dtypes)}}
-
-cdef {{c_type}} _logsumexp{{name}}({{c_type}}* arr, int n_classes) nogil:
-    """Computes the sum of arr assuming arr is in the log domain.
-
-    Returns log(sum(exp(arr))) while minimizing the possibility of
-    over/underflow.
-    """
-    # Use the max to normalize, as with the log this is what accumulates
-    # the less errors
-    cdef {{c_type}} vmax = arr[0]
-    cdef {{c_type}} out = 0.0
-    cdef int i
-
-    for i in range(1, n_classes):
-        if vmax < arr[i]:
-            vmax = arr[i]
-
-    for i in range(n_classes):
-        out += exp(arr[i] - vmax)
-
-    return log(out) + vmax
+cdef inline {{c_type}} _soft_thresholding{{name_suffix}}({{c_type}} x, {{c_type}} shrinkage) noexcept nogil:
+    return fmax{{name_suffix}}(x - shrinkage, 0) - fmax{{name_suffix}}(- x - shrinkage, 0)
 
 {{endfor}}
 
 
-{{for name, c_type, np_type in get_dispatch(dtypes)}}
-
-cdef class MultinomialLogLoss{{name}}:
-    cdef {{c_type}} _loss(self, {{c_type}}* prediction, {{c_type}} y, int n_classes,
-                      {{c_type}} sample_weight) nogil:
-        r"""Multinomial Logistic regression loss.
-
-        The multinomial logistic loss for one sample is:
-        loss = - sw \sum_c \delta_{y,c} (prediction[c] - logsumexp(prediction))
-             = sw (logsumexp(prediction) - prediction[y])
-
-        where:
-            prediction = dot(x_sample, weights) + intercept
-            \delta_{y,c} = 1 if (y == c) else 0
-            sw = sample_weight
-
-        Parameters
-        ----------
-        prediction : pointer to a np.ndarray[{{c_type}}] of shape (n_classes,)
-            Prediction of the multinomial classifier, for current sample.
-
-        y : {{c_type}}, between 0 and n_classes - 1
-            Indice of the correct class for current sample (i.e. label encoded).
-
-        n_classes : integer
-            Total number of classes.
-
-        sample_weight : {{c_type}}
-            Weight of current sample.
-
-        Returns
-        -------
-        loss : {{c_type}}
-            Multinomial loss for current sample.
-
-        Reference
-        ---------
-        Bishop, C. M. (2006). Pattern recognition and machine learning.
-        Springer. (Chapter 4.3.4)
-        """
-        cdef {{c_type}} logsumexp_prediction = _logsumexp{{name}}(prediction, n_classes)
-        cdef {{c_type}} loss
-
-        # y is the indice of the correct class of current sample.
-        loss = (logsumexp_prediction - prediction[int(y)]) * sample_weight
-        return loss
-
-    cdef void dloss(self, {{c_type}}* prediction, {{c_type}} y, int n_classes,
-                     {{c_type}} sample_weight, {{c_type}}* gradient_ptr) nogil:
-        r"""Multinomial Logistic regression gradient of the loss.
-
-        The gradient of the multinomial logistic loss with respect to a class c,
-        and for one sample is:
-        grad_c = - sw * (p[c] - \delta_{y,c})
-
-        where:
-            p[c] = exp(logsumexp(prediction) - prediction[c])
-            prediction = dot(sample, weights) + intercept
-            \delta_{y,c} = 1 if (y == c) else 0
-            sw = sample_weight
-
-        Note that to obtain the true gradient, this value has to be multiplied
-        by the sample vector x.
-
-        Parameters
-        ----------
-        prediction : pointer to a np.ndarray[{{c_type}}] of shape (n_classes,)
-            Prediction of the multinomial classifier, for current sample.
-
-        y : {{c_type}}, between 0 and n_classes - 1
-            Indice of the correct class for current sample (i.e. label encoded)
-
-        n_classes : integer
-            Total number of classes.
-
-        sample_weight : {{c_type}}
-            Weight of current sample.
-
-        gradient_ptr : pointer to a np.ndarray[{{c_type}}] of shape (n_classes,)
-            Gradient vector to be filled.
-
-        Reference
-        ---------
-        Bishop, C. M. (2006). Pattern recognition and machine learning.
-        Springer. (Chapter 4.3.4)
-        """
-        cdef {{c_type}} logsumexp_prediction = _logsumexp{{name}}(prediction, n_classes)
-        cdef int class_ind
-
-        for class_ind in range(n_classes):
-            gradient_ptr[class_ind] = exp(prediction[class_ind] -
-                                          logsumexp_prediction)
-
-            # y is the indice of the correct class of current sample.
-            if class_ind == y:
-                gradient_ptr[class_ind] -= 1.0
-
-            gradient_ptr[class_ind] *= sample_weight
-
-    def __reduce__(self):
-        return MultinomialLogLoss{{name}}, ()
-
-{{endfor}}
-
-{{for name, c_type, np_type in get_dispatch(dtypes)}}
-
-cdef inline {{c_type}} _soft_thresholding{{name}}({{c_type}} x, {{c_type}} shrinkage) nogil:
-    return fmax{{name}}(x - shrinkage, 0) - fmax{{name}}(- x - shrinkage, 0)
-
-{{endfor}}
-
-
-{{for name, c_type, np_type in get_dispatch(dtypes)}}
-
-def sag{{name}}(SequentialDataset{{name}} dataset,
-        np.ndarray[{{c_type}}, ndim=2, mode='c'] weights_array,
-        np.ndarray[{{c_type}}, ndim=1, mode='c'] intercept_array,
-        int n_samples,
-        int n_features,
-        int n_classes,
-        double tol,
-        int max_iter,
-        str loss_function,
-        double step_size,
-        double alpha,
-        double beta,
-        np.ndarray[{{c_type}}, ndim=2, mode='c'] sum_gradient_init,
-        np.ndarray[{{c_type}}, ndim=2, mode='c'] gradient_memory_init,
-        np.ndarray[bint, ndim=1, mode='c'] seen_init,
-        int num_seen,
-        bint fit_intercept,
-        np.ndarray[{{c_type}}, ndim=1, mode='c'] intercept_sum_gradient_init,
-        double intercept_decay,
-        bint saga,
-        bint verbose):
+{{for name_suffix, c_type, np_type in dtypes}}
+
+def sag{{name_suffix}}(
+    SequentialDataset{{name_suffix}} dataset,
+    {{c_type}}[:, ::1] weights_array,
+    {{c_type}}[::1] intercept_array,
+    int n_samples,
+    int n_features,
+    int n_classes,
+    double tol,
+    int max_iter,
+    str loss_function,
+    double step_size,
+    double alpha,
+    double beta,
+    {{c_type}}[:, ::1] sum_gradient_init,
+    {{c_type}}[:, ::1] gradient_memory_init,
+    bint[::1] seen_init,
+    int num_seen,
+    bint fit_intercept,
+    {{c_type}}[::1] intercept_sum_gradient_init,
+    double intercept_decay,
+    bint saga,
+    bint verbose
+):
     """Stochastic Average Gradient (SAG) and SAGA solvers.
 
     Used in Ridge and LogisticRegression.
 
+    Some implementation details:
+
+    - Just-in-time (JIT) update: In SAG(A), the average-gradient update is
+    collinear with the drawn sample X_i. Therefore, if the data is sparse, the
+    random sample X_i will change the average gradient only on features j where
+    X_ij != 0. In some cases, the average gradient on feature j might change
+    only after k random samples with no change. In these cases, instead of
+    applying k times the same gradient step on feature j, we apply the gradient
+    step only once, scaled by k. This is called the "just-in-time update", and
+    it is performed in `lagged_update{{name_suffix}}`. This function also
+    applies the proximal operator after the gradient step (if L1 regularization
+    is used in SAGA).
+
+    - Weight scale: In SAG(A), the weights are scaled down at each iteration
+    due to the L2 regularization. To avoid updating all the weights at each
+    iteration, the weight scale is factored out in a separate variable `wscale`
+    which is only used in the JIT update. When this variable is too small, it
+    is reset for numerical stability using the function
+    `scale_weights{{name_suffix}}`. This reset requires applying all remaining
+    JIT updates. This reset is also performed every `n_samples` iterations
+    before each convergence check, so when the algorithm stops, we are sure
+    that there is no remaining JIT updates.
+
     Reference
     ---------
     Schmidt, M., Roux, N. L., & Bach, F. (2013).
@@ -251,11 +111,9 @@ def sag{{name}}(SequentialDataset{{name}} dataset,
     https://hal.inria.fr/hal-00860051/document
     (section 4.3)
 
-    Defazio, A., Bach, F., Lacoste-Julien, S. (2014),
-    SAGA: A Fast Incremental Gradient Method With Support
-    for Non-Strongly Convex Composite Objectives
-    https://arxiv.org/abs/1407.0202
-
+    :arxiv:`Defazio, A., Bach F. & Lacoste-Julien S. (2014).
+    "SAGA: A Fast Incremental Gradient Method With Support
+    for Non-Strongly Convex Composite Objectives" <1407.0202>`
     """
     # the data pointer for x, the current sample
     cdef {{c_type}} *x_data_ptr = NULL
@@ -264,7 +122,7 @@ def sag{{name}}(SequentialDataset{{name}} dataset,
     # the number of non-zero features for current sample
     cdef int xnnz = -1
     # the label value for current sample
-    # the label value for curent sample
+    # the label value for current sample
     cdef {{c_type}} y
     # the sample weight
     cdef {{c_type}} sample_weight
@@ -291,48 +149,31 @@ def sag{{name}}(SequentialDataset{{name}} dataset,
     # precomputation since the step size does not change in this implementation
     cdef {{c_type}} wscale_update = 1.0 - step_size * alpha
 
-    # vector of booleans indicating whether this sample has been seen
-    cdef bint* seen =  seen_init.data
-
     # helper for cumulative sum
     cdef {{c_type}} cum_sum
 
     # the pointer to the coef_ or weights
-    cdef {{c_type}}* weights = <{{c_type}} * >weights_array.data
-    # the pointer to the intercept_array
-    cdef {{c_type}}* intercept = <{{c_type}} * >intercept_array.data
-
-    # the pointer to the intercept_sum_gradient
-    cdef {{c_type}}* intercept_sum_gradient = \
-        <{{c_type}} * >intercept_sum_gradient_init.data
+    cdef {{c_type}}* weights = &weights_array[0, 0]
 
     # the sum of gradients for each feature
-    cdef {{c_type}}* sum_gradient = <{{c_type}}*> sum_gradient_init.data
+    cdef {{c_type}}* sum_gradient = &sum_gradient_init[0, 0]
+
     # the previously seen gradient for each sample
-    cdef {{c_type}}* gradient_memory = <{{c_type}}*> gradient_memory_init.data
+    cdef {{c_type}}* gradient_memory = &gradient_memory_init[0, 0]
 
     # the cumulative sums needed for JIT params
-    cdef np.ndarray[{{c_type}}, ndim=1] cumulative_sums_array = \
-        np.empty(n_samples, dtype={{np_type}}, order="c")
-    cdef {{c_type}}* cumulative_sums = <{{c_type}}*> cumulative_sums_array.data
+    cdef {{c_type}}[::1] cumulative_sums = np.empty(n_samples, dtype={{np_type}}, order="c")
 
     # the index for the last time this feature was updated
-    cdef np.ndarray[int, ndim=1] feature_hist_array = \
-        np.zeros(n_features, dtype=np.int32, order="c")
-    cdef int* feature_hist =  feature_hist_array.data
+    cdef int[::1] feature_hist = np.zeros(n_features, dtype=np.int32, order="c")
 
     # the previous weights to use to compute stopping criteria
-    cdef np.ndarray[{{c_type}}, ndim=2] previous_weights_array = \
-        np.zeros((n_features, n_classes), dtype={{np_type}}, order="c")
-    cdef {{c_type}}* previous_weights = <{{c_type}}*> previous_weights_array.data
+    cdef {{c_type}}[:, ::1] previous_weights_array = np.zeros((n_features, n_classes), dtype={{np_type}}, order="c")
+    cdef {{c_type}}* previous_weights = &previous_weights_array[0, 0]
 
-    cdef np.ndarray[{{c_type}}, ndim=1] prediction_array = \
-        np.zeros(n_classes, dtype={{np_type}}, order="c")
-    cdef {{c_type}}* prediction = <{{c_type}}*> prediction_array.data
+    cdef {{c_type}}[::1] prediction = np.zeros(n_classes, dtype={{np_type}}, order="c")
 
-    cdef np.ndarray[{{c_type}}, ndim=1] gradient_array = \
-        np.zeros(n_classes, dtype={{np_type}}, order="c")
-    cdef {{c_type}}* gradient = <{{c_type}}*> gradient_array.data
+    cdef {{c_type}}[::1] gradient = np.zeros(n_classes, dtype={{np_type}}, order="c")
 
     # Intermediate variable that need declaration since cython cannot infer when templating
     cdef {{c_type}} val
@@ -350,79 +191,95 @@ def sag{{name}}(SequentialDataset{{name}} dataset,
     cumulative_sums[0] = 0.0
 
     # the multipliative scale needed for JIT params
-    cdef np.ndarray[{{c_type}}, ndim=1] cumulative_sums_prox_array
-    cdef {{c_type}}* cumulative_sums_prox
+    cdef {{c_type}}[::1] cumulative_sums_prox
+    cdef {{c_type}}* cumulative_sums_prox_ptr
 
     cdef bint prox = beta > 0 and saga
 
     # Loss function to optimize
-    cdef LossFunction loss
-    # Wether the loss function is multinomial
+    cdef CyLossFunction loss
+    # Whether the loss function is multinomial
     cdef bint multinomial = False
     # Multinomial loss function
-    cdef MultinomialLogLoss{{name}} multiloss
+    cdef CyHalfMultinomialLoss multiloss
 
     if loss_function == "multinomial":
         multinomial = True
-        multiloss = MultinomialLogLoss{{name}}()
+        multiloss = CyHalfMultinomialLoss()
     elif loss_function == "log":
-        loss = Log()
+        loss = CyHalfBinomialLoss()
     elif loss_function == "squared":
-        loss = SquaredLoss()
+        loss = CyHalfSquaredError()
     else:
         raise ValueError("Invalid loss parameter: got %s instead of "
                          "one of ('log', 'squared', 'multinomial')"
                          % loss_function)
 
     if prox:
-        cumulative_sums_prox_array = np.empty(n_samples,
-                                              dtype={{np_type}}, order="c")
-        cumulative_sums_prox = <{{c_type}}*> cumulative_sums_prox_array.data
+        cumulative_sums_prox = np.empty(n_samples, dtype={{np_type}}, order="c")
+        cumulative_sums_prox_ptr = &cumulative_sums_prox[0]
     else:
-        cumulative_sums_prox = NULL
+        cumulative_sums_prox = None
+        cumulative_sums_prox_ptr = NULL
 
     with nogil:
         start_time = time(NULL)
         for n_iter in range(max_iter):
             for sample_itr in range(n_samples):
                 # extract a random sample
-                sample_ind = dataset.random(&x_data_ptr, &x_ind_ptr, &xnnz,
-                                              &y, &sample_weight)
+                sample_ind = dataset.random(&x_data_ptr, &x_ind_ptr, &xnnz, &y, &sample_weight)
 
                 # cached index for gradient_memory
                 s_idx = sample_ind * n_classes
 
                 # update the number of samples seen and the seen array
-                if seen[sample_ind] == 0:
+                if seen_init[sample_ind] == 0:
                     num_seen += 1
-                    seen[sample_ind] = 1
+                    seen_init[sample_ind] = 1
 
-                # make the weight updates
+                # make the weight updates (just-in-time gradient step, and prox operator)
                 if sample_itr > 0:
-                   status = lagged_update{{name}}(weights, wscale, xnnz,
-                                                  n_samples, n_classes,
-                                                  sample_itr,
-                                                  cumulative_sums,
-                                                  cumulative_sums_prox,
-                                                  feature_hist,
-                                                  prox,
-                                                  sum_gradient,
-                                                  x_ind_ptr,
-                                                  False,
-                                                  n_iter)
+                   status = lagged_update{{name_suffix}}(
+                       weights=weights,
+                       wscale=wscale,
+                       xnnz=xnnz,
+                       n_samples=n_samples,
+                       n_classes=n_classes,
+                       sample_itr=sample_itr,
+                       cumulative_sums=&cumulative_sums[0],
+                       cumulative_sums_prox=cumulative_sums_prox_ptr,
+                       feature_hist=&feature_hist[0],
+                       prox=prox,
+                       sum_gradient=sum_gradient,
+                       x_ind_ptr=x_ind_ptr,
+                       reset=False,
+                       n_iter=n_iter
+                   )
                    if status == -1:
                        break
 
                 # find the current prediction
-                predict_sample{{name}}(x_data_ptr, x_ind_ptr, xnnz, weights, wscale,
-                                       intercept, prediction, n_classes)
+                predict_sample{{name_suffix}}(
+                    x_data_ptr=x_data_ptr,
+                    x_ind_ptr=x_ind_ptr,
+                    xnnz=xnnz,
+                    w_data_ptr=weights,
+                    wscale=wscale,
+                    intercept=&intercept_array[0],
+                    prediction=&prediction[0],
+                    n_classes=n_classes
+                )
 
                 # compute the gradient for this sample, given the prediction
                 if multinomial:
-                    multiloss.dloss(prediction, y, n_classes, sample_weight,
-                                     gradient)
+                    multiloss.cy_gradient(
+                        y_true=y,
+                        raw_prediction=prediction,
+                        sample_weight=sample_weight,
+                        gradient_out=gradient,
+                    )
                 else:
-                    gradient[0] = loss.dloss(prediction[0], y) * sample_weight
+                    gradient[0] = loss.cy_gradient(y, prediction[0]) * sample_weight
 
                 # L2 regularization by simply rescaling the weights
                 wscale *= wscale_update
@@ -437,6 +294,11 @@ def sag{{name}}(SequentialDataset{{name}} dataset,
                             val * (gradient[class_ind] -
                                    gradient_memory[s_idx + class_ind])
                         if saga:
+                            # Note that this is not the main gradient step,
+                            # which is performed just-in-time in lagged_update.
+                            # This part is done outside the JIT update
+                            # as it does not depend on the average gradient.
+                            # The prox operator is applied after the JIT update
                             weights[f_idx + class_ind] -= \
                                 (gradient_correction * step_size
                                  * (1 - 1. / num_seen) / wscale)
@@ -447,19 +309,19 @@ def sag{{name}}(SequentialDataset{{name}} dataset,
                     for class_ind in range(n_classes):
                         gradient_correction = (gradient[class_ind] -
                                                gradient_memory[s_idx + class_ind])
-                        intercept_sum_gradient[class_ind] += gradient_correction
+                        intercept_sum_gradient_init[class_ind] += gradient_correction
                         gradient_correction *= step_size * (1. - 1. / num_seen)
                         if saga:
-                            intercept[class_ind] -= \
-                                (step_size * intercept_sum_gradient[class_ind] /
+                            intercept_array[class_ind] -= \
+                                (step_size * intercept_sum_gradient_init[class_ind] /
                                  num_seen * intercept_decay) + gradient_correction
                         else:
-                            intercept[class_ind] -= \
-                                (step_size * intercept_sum_gradient[class_ind] /
+                            intercept_array[class_ind] -= \
+                                (step_size * intercept_sum_gradient_init[class_ind] /
                                  num_seen * intercept_decay)
 
                         # check to see that the intercept is not inf or NaN
-                        if not skl_isfinite{{name}}(intercept[class_ind]):
+                        if not isfinite(intercept_array[class_ind]):
                             status = -1
                             break
                     # Break from the n_samples outer loop if an error happened
@@ -484,16 +346,25 @@ def sag{{name}}(SequentialDataset{{name}} dataset,
                         (cumulative_sums_prox[sample_itr - 1] +
                              step_size * beta / wscale)
                 # If wscale gets too small, we need to reset the scale.
+                # This also resets the just-in-time update system.
                 if wscale < 1e-9:
                     if verbose:
                         with gil:
                             print("rescaling...")
-                    status = scale_weights{{name}}(
-                        weights, &wscale, n_features, n_samples, n_classes,
-                        sample_itr, cumulative_sums,
-                        cumulative_sums_prox,
-                        feature_hist,
-                        prox, sum_gradient, n_iter)
+                    status = scale_weights{{name_suffix}}(
+                        weights=weights,
+                        wscale=&wscale,
+                        n_features=n_features,
+                        n_samples=n_samples,
+                        n_classes=n_classes,
+                        sample_itr=sample_itr,
+                        cumulative_sums=&cumulative_sums[0],
+                        cumulative_sums_prox=cumulative_sums_prox_ptr,
+                        feature_hist=&feature_hist[0],
+                        prox=prox,
+                        sum_gradient=sum_gradient,
+                        n_iter=n_iter
+                    )
                     if status == -1:
                         break
 
@@ -502,26 +373,33 @@ def sag{{name}}(SequentialDataset{{name}} dataset,
             if status == -1:
                 break
 
-            # we scale the weights every n_samples iterations and reset the
+            # We scale the weights every n_samples iterations and reset the
             # just-in-time update system for numerical stability.
-            status = scale_weights{{name}}(weights, &wscale, n_features,
-                                           n_samples,
-                                           n_classes, n_samples - 1,
-                                           cumulative_sums,
-                                           cumulative_sums_prox,
-                                           feature_hist,
-                                           prox, sum_gradient, n_iter)
-
+            # Because this reset is done before every convergence check, we are
+            # sure there is no remaining lagged update when the algorithm stops.
+            status = scale_weights{{name_suffix}}(
+                weights=weights,
+                wscale=&wscale,
+                n_features=n_features,
+                n_samples=n_samples,
+                n_classes=n_classes,
+                sample_itr=n_samples - 1,
+                cumulative_sums=&cumulative_sums[0],
+                cumulative_sums_prox=cumulative_sums_prox_ptr,
+                feature_hist=&feature_hist[0],
+                prox=prox,
+                sum_gradient=sum_gradient,
+                n_iter=n_iter
+            )
             if status == -1:
                 break
+
             # check if the stopping criteria is reached
             max_change = 0.0
             max_weight = 0.0
             for idx in range(n_features * n_classes):
-                max_weight = fmax{{name}}(max_weight, fabs(weights[idx]))
-                max_change = fmax{{name}}(max_change,
-                                  fabs(weights[idx] -
-                                       previous_weights[idx]))
+                max_weight = fmax{{name_suffix}}(max_weight, fabs(weights[idx]))
+                max_change = fmax{{name_suffix}}(max_change, fabs(weights[idx] - previous_weights[idx]))
                 previous_weights[idx] = weights[idx]
             if ((max_weight != 0 and max_change / max_weight <= tol)
                 or max_weight == 0 and max_change == 0):
@@ -532,7 +410,7 @@ def sag{{name}}(SequentialDataset{{name}} dataset,
                               (n_iter + 1, end_time - start_time))
                 break
             elif verbose:
-                printf('Epoch %d, change: %.8f\n', n_iter + 1,
+                printf('Epoch %d, change: %.8g\n', n_iter + 1,
                                                   max_change / max_weight)
     n_iter += 1
     # We do the error treatment here based on error code in status to avoid
@@ -553,18 +431,26 @@ def sag{{name}}(SequentialDataset{{name}} dataset,
 {{endfor}}
 
 
-{{for name, c_type, np_type in get_dispatch(dtypes)}}
+{{for name_suffix, c_type, np_type in dtypes}}
 
-cdef int scale_weights{{name}}({{c_type}}* weights, {{c_type}}* wscale,
-                               int n_features,
-                               int n_samples, int n_classes, int sample_itr,
-                               {{c_type}}* cumulative_sums,
-                               {{c_type}}* cumulative_sums_prox,
-                               int* feature_hist,
-                               bint prox,
-                               {{c_type}}* sum_gradient,
-                               int n_iter) nogil:
-    """Scale the weights with wscale for numerical stability.
+cdef int scale_weights{{name_suffix}}(
+    {{c_type}}* weights,
+    {{c_type}}* wscale,
+    int n_features,
+    int n_samples,
+    int n_classes,
+    int sample_itr,
+    {{c_type}}* cumulative_sums,
+    {{c_type}}* cumulative_sums_prox,
+    int* feature_hist,
+    bint prox,
+    {{c_type}}* sum_gradient,
+    int n_iter
+) noexcept nogil:
+    """Scale the weights and reset wscale to 1.0 for numerical stability, and
+    reset the just-in-time (JIT) update system.
+
+    See `sag{{name_suffix}}`'s docstring about the JIT update system.
 
     wscale = (1 - step_size * alpha) ** (n_iter * n_samples + sample_itr)
     can become very small, so we reset it every n_samples iterations to 1.0 for
@@ -574,16 +460,22 @@ cdef int scale_weights{{name}}({{c_type}}* weights, {{c_type}}* wscale,
     """
 
     cdef int status
-    status = lagged_update{{name}}(weights, wscale[0], n_features,
-                                   n_samples, n_classes, sample_itr + 1,
-                                   cumulative_sums,
-                                   cumulative_sums_prox,
-                                   feature_hist,
-                                   prox,
-                                   sum_gradient,
-                                   NULL,
-                                   True,
-                                   n_iter)
+    status = lagged_update{{name_suffix}}(
+        weights,
+        wscale[0],
+        n_features,
+        n_samples,
+        n_classes,
+        sample_itr + 1,
+        cumulative_sums,
+        cumulative_sums_prox,
+        feature_hist,
+        prox,
+        sum_gradient,
+        NULL,
+        True,
+        n_iter
+    )
     # if lagged update succeeded, reset wscale to 1.0
     if status == 0:
         wscale[0] = 1.0
@@ -592,19 +484,28 @@ cdef int scale_weights{{name}}({{c_type}}* weights, {{c_type}}* wscale,
 {{endfor}}
 
 
-{{for name, c_type, np_type in get_dispatch(dtypes)}}
-
-cdef int lagged_update{{name}}({{c_type}}* weights, {{c_type}} wscale, int xnnz,
-                               int n_samples, int n_classes, int sample_itr,
-                               {{c_type}}* cumulative_sums,
-                               {{c_type}}* cumulative_sums_prox,
-                               int* feature_hist,
-                               bint prox,
-                               {{c_type}}* sum_gradient,
-                               int* x_ind_ptr,
-                               bint reset,
-                               int n_iter) nogil:
+{{for name_suffix, c_type, np_type in dtypes}}
+
+cdef int lagged_update{{name_suffix}}(
+    {{c_type}}* weights,
+    {{c_type}} wscale,
+    int xnnz,
+    int n_samples,
+    int n_classes,
+    int sample_itr,
+    {{c_type}}* cumulative_sums,
+    {{c_type}}* cumulative_sums_prox,
+    int* feature_hist,
+    bint prox,
+    {{c_type}}* sum_gradient,
+    int* x_ind_ptr,
+    bint reset,
+    int n_iter
+) noexcept nogil:
     """Hard perform the JIT updates for non-zero features of present sample.
+
+    See `sag{{name_suffix}}`'s docstring about the JIT update system.
+
     The updates that awaits are kept in memory using cumulative_sums,
     cumulative_sums_prox, wscale and feature_hist. See original SAGA paper
     (Defazio et al. 2014) for details. If reset=True, we also reset wscale to
@@ -630,7 +531,7 @@ cdef int lagged_update{{name}}({{c_type}}* weights, {{c_type}} wscale, int xnnz,
                 weights[idx] -= cum_sum * sum_gradient[idx]
                 if reset:
                     weights[idx] *= wscale
-                    if not skl_isfinite{{name}}(weights[idx]):
+                    if not isfinite(weights[idx]):
                         # returning here does not require the gil as the return
                         # type is a C integer
                         return -1
@@ -643,7 +544,7 @@ cdef int lagged_update{{name}}({{c_type}}* weights, {{c_type}} wscale, int xnnz,
                     # efficient than unrolling all the lagged updates.
                     # Idea taken from scikit-learn-contrib/lightning.
                     weights[idx] -= cum_sum * sum_gradient[idx]
-                    weights[idx] = _soft_thresholding{{name}}(weights[idx],
+                    weights[idx] = _soft_thresholding{{name_suffix}}(weights[idx],
                                                       cum_sum_prox)
                 else:
                     last_update_ind = feature_hist[feature_ind]
@@ -660,13 +561,13 @@ cdef int lagged_update{{name}}({{c_type}}* weights, {{c_type}} wscale, int xnnz,
                             grad_step = cumulative_sums[lagged_ind]
                             prox_step = cumulative_sums_prox[lagged_ind]
                         weights[idx] -= sum_gradient[idx] * grad_step
-                        weights[idx] = _soft_thresholding{{name}}(weights[idx],
+                        weights[idx] = _soft_thresholding{{name_suffix}}(weights[idx],
                                                           prox_step)
 
                 if reset:
                     weights[idx] *= wscale
                     # check to see that the weight is not inf or NaN
-                    if not skl_isfinite{{name}}(weights[idx]):
+                    if not isfinite(weights[idx]):
                         return -1
         if reset:
             feature_hist[feature_ind] = sample_itr % n_samples
@@ -683,12 +584,18 @@ cdef int lagged_update{{name}}({{c_type}}* weights, {{c_type}} wscale, int xnnz,
 {{endfor}}
 
 
-{{for name, c_type, np_type in get_dispatch(dtypes)}}
+{{for name_suffix, c_type, np_type in dtypes}}
 
-cdef void predict_sample{{name}}({{c_type}}* x_data_ptr, int* x_ind_ptr, int xnnz,
-                                 {{c_type}}* w_data_ptr, {{c_type}} wscale,
-                                 {{c_type}}* intercept, {{c_type}}* prediction,
-                                 int n_classes) nogil:
+cdef void predict_sample{{name_suffix}}(
+    {{c_type}}* x_data_ptr,
+    int* x_ind_ptr,
+    int xnnz,
+    {{c_type}}* w_data_ptr,
+    {{c_type}} wscale,
+    {{c_type}}* intercept,
+    {{c_type}}* prediction,
+    int n_classes
+) noexcept nogil:
     """Compute the prediction given sparse sample x and dense weight w.
 
     Parameters
@@ -733,67 +640,3 @@ cdef void predict_sample{{name}}({{c_type}}* x_data_ptr, int* x_ind_ptr, int xnn
 
 
 {{endfor}}
-
-
-def _multinomial_grad_loss_all_samples(
-        SequentialDataset64 dataset,
-        np.ndarray[double, ndim=2, mode='c'] weights_array,
-        np.ndarray[double, ndim=1, mode='c'] intercept_array,
-        int n_samples, int n_features, int n_classes):
-    """Compute multinomial gradient and loss across all samples.
-
-    Used for testing purpose only.
-    """
-    cdef double* weights = weights_array.data
-    cdef double* intercept = intercept_array.data
-
-    cdef double *x_data_ptr = NULL
-    cdef int *x_ind_ptr = NULL
-    cdef int xnnz = -1
-    cdef double y
-    cdef double sample_weight
-
-    cdef double wscale = 1.0
-    cdef int i, j, class_ind, feature_ind
-    cdef double val
-    cdef double sum_loss = 0.0
-
-    cdef MultinomialLogLoss64 multiloss = MultinomialLogLoss64()
-
-    cdef np.ndarray[double, ndim=2] sum_gradient_array = \
-        np.zeros((n_features, n_classes), dtype=np.double, order="c")
-    cdef double* sum_gradient =  sum_gradient_array.data
-
-    cdef np.ndarray[double, ndim=1] prediction_array = \
-        np.zeros(n_classes, dtype=np.double, order="c")
-    cdef double* prediction =  prediction_array.data
-
-    cdef np.ndarray[double, ndim=1] gradient_array = \
-        np.zeros(n_classes, dtype=np.double, order="c")
-    cdef double* gradient =  gradient_array.data
-
-    with nogil:
-        for i in range(n_samples):
-            # get next sample on the dataset
-            dataset.next(&x_data_ptr, &x_ind_ptr, &xnnz,
-                         &y, &sample_weight)
-
-            # prediction of the multinomial classifier for the sample
-            predict_sample64(x_data_ptr, x_ind_ptr, xnnz, weights, wscale,
-                           intercept, prediction, n_classes)
-
-            # compute the gradient for this sample, given the prediction
-            multiloss.dloss(prediction, y, n_classes, sample_weight, gradient)
-
-            # compute the loss for this sample, given the prediction
-            sum_loss += multiloss._loss(prediction, y, n_classes, sample_weight)
-
-            # update the sum of the gradient
-            for j in range(xnnz):
-                feature_ind = x_ind_ptr[j]
-                val = x_data_ptr[j]
-                for class_ind in range(n_classes):
-                    sum_gradient[feature_ind * n_classes + class_ind] += \
-                        gradient[class_ind] * val
-
-    return sum_loss, sum_gradient_array
diff --git a/sklearn/linear_model/_sgd_fast.pxd b/sklearn/linear_model/_sgd_fast.pxd
deleted file mode 100644
index 3c02f5ab1a834..0000000000000
--- a/sklearn/linear_model/_sgd_fast.pxd
+++ /dev/null
@@ -1,26 +0,0 @@
-# License: BSD 3 clause
-"""Helper to load LossFunction from sgd_fast.pyx to sag_fast.pyx"""
-
-cdef class LossFunction:
-    cdef double loss(self, double p, double y) nogil
-    cdef double dloss(self, double p, double y) nogil
-
-
-cdef class Regression(LossFunction):
-    cdef double loss(self, double p, double y) nogil
-    cdef double dloss(self, double p, double y) nogil
-
-
-cdef class Classification(LossFunction):
-    cdef double loss(self, double p, double y) nogil
-    cdef double dloss(self, double p, double y) nogil
-
-
-cdef class Log(Classification):
-    cdef double loss(self, double p, double y) nogil
-    cdef double dloss(self, double p, double y) nogil
-
-
-cdef class SquaredLoss(Regression):
-    cdef double loss(self, double p, double y) nogil
-    cdef double dloss(self, double p, double y) nogil
diff --git a/sklearn/linear_model/_sgd_fast.pyx b/sklearn/linear_model/_sgd_fast.pyx.tp
similarity index 66%
rename from sklearn/linear_model/_sgd_fast.pyx
rename to sklearn/linear_model/_sgd_fast.pyx.tp
index 3940e5d873669..45cdf9172d8c4 100644
--- a/sklearn/linear_model/_sgd_fast.pyx
+++ b/sklearn/linear_model/_sgd_fast.pyx.tp
@@ -1,121 +1,124 @@
-# cython: cdivision=True
-# cython: boundscheck=False
-# cython: wraparound=False
-#
-# Author: Peter Prettenhofer 
-#         Mathieu Blondel (partial_fit support)
-#         Rob Zinkov (passive-aggressive)
-#         Lars Buitinck
-#
-# License: BSD 3 clause
+{{py:
 
+"""
+Template file to easily generate fused types consistent code using Tempita
+(https://github.com/cython/cython/blob/master/Cython/Tempita/_tempita.py).
+
+Generated file: _sgd_fast.pyx
+
+Each relevant function is duplicated for the dtypes float and double.
+The keywords between double braces are substituted during the build.
+"""
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+# The dtypes are defined as follows (name_suffix, c_type, np_type)
+dtypes = [
+    ("64", "double", "np.float64"),
+    ("32", "float", "np.float32"),
+]
+
+}}
+"""SGD implementation"""
 
 import numpy as np
-import sys
 from time import time
 
-cimport cython
-from libc.math cimport exp, log, sqrt, pow, fabs
-cimport numpy as np
-from numpy.math cimport INFINITY
-cdef extern from "_sgd_fast_helpers.h":
-    bint skl_isfinite(double) nogil
-
-from ..utils._weight_vector cimport WeightVector
-from ..utils._seq_dataset cimport SequentialDataset64 as SequentialDataset
+from cython cimport floating
+from libc.math cimport exp, fabs, isfinite, log, pow, INFINITY
 
-np.import_array()
+from .._loss._loss cimport CyLossFunction
+from ..utils._typedefs cimport uint32_t, uint8_t
+from ..utils._weight_vector cimport WeightVector32, WeightVector64
+from ..utils._seq_dataset cimport SequentialDataset32, SequentialDataset64
 
-# Penalty constants
-DEF NO_PENALTY = 0
-DEF L1 = 1
-DEF L2 = 2
-DEF ELASTICNET = 3
 
-# Learning rate constants
-DEF CONSTANT = 1
-DEF OPTIMAL = 2
-DEF INVSCALING = 3
-DEF ADAPTIVE = 4
-DEF PA1 = 5
-DEF PA2 = 6
+cdef extern from *:
+    """
+    /* Penalty constants */
+    #define NO_PENALTY 0
+    #define L1 1
+    #define L2 2
+    #define ELASTICNET 3
+
+    /* Learning rate constants */
+    #define CONSTANT 1
+    #define OPTIMAL 2
+    #define INVSCALING 3
+    #define ADAPTIVE 4
+    #define PA1 5
+    #define PA2 6
+    """
+    int NO_PENALTY = 0
+    int L1 = 1
+    int L2 = 2
+    int ELASTICNET = 3
 
+    int CONSTANT = 1
+    int OPTIMAL = 2
+    int INVSCALING = 3
+    int ADAPTIVE = 4
+    int PA1 = 5
+    int PA2 = 6
 
 
 # ----------------------------------------
 # Extension Types for Loss Functions
 # ----------------------------------------
 
-cdef class LossFunction:
-    """Base class for convex loss functions"""
+cdef class Regression(CyLossFunction):
+    """Base class for loss functions for regression"""
+
+    def py_loss(self, double p, double y):
+        """Python version of `loss` for testing only.
 
-    cdef double loss(self, double p, double y) nogil:
-        """Evaluate the loss function.
+        Pytest needs a python function and can't use cdef functions.
 
         Parameters
         ----------
         p : double
-            The prediction, p = w^T x
+            The prediction, `p = w^T x + intercept`.
         y : double
-            The true value (aka target)
+            The true value (aka target).
 
         Returns
         -------
         double
             The loss evaluated at `p` and `y`.
         """
-        return 0.
+        return self.cy_loss(y, p)
 
     def py_dloss(self, double p, double y):
-        """Python version of `dloss` for testing.
-
-        Pytest needs a python function and can't use cdef functions.
-        """
-        return self.dloss(p, y)
+        """Python version of `dloss` for testing only.
 
-    def py_loss(self, double p, double y):
-        """Python version of `loss` for testing.
-        
         Pytest needs a python function and can't use cdef functions.
-        """
-        return self.loss(p, y)
-
-    cdef double dloss(self, double p, double y) nogil:
-        """Evaluate the derivative of the loss function with respect to
-        the prediction `p`.
 
         Parameters
         ----------
         p : double
-            The prediction, p = w^T x
+            The prediction, `p = w^T x`.
         y : double
-            The true value (aka target)
+            The true value (aka target).
+
         Returns
         -------
         double
             The derivative of the loss function with regards to `p`.
         """
-        return 0.
-
-
-cdef class Regression(LossFunction):
-    """Base class for loss functions for regression"""
-
-    cdef double loss(self, double p, double y) nogil:
-        return 0.
+        return self.cy_gradient(y, p)
 
-    cdef double dloss(self, double p, double y) nogil:
-        return 0.
 
-
-cdef class Classification(LossFunction):
+cdef class Classification(CyLossFunction):
     """Base class for loss functions for classification"""
 
-    cdef double loss(self, double p, double y) nogil:
-        return 0.
+    def py_loss(self, double p, double y):
+        """Python version of `loss` for testing only."""
+        return self.cy_loss(y, p)
 
-    cdef double dloss(self, double p, double y) nogil:
-        return 0.
+    def py_dloss(self, double p, double y):
+        """Python version of `dloss` for testing only."""
+        return self.cy_gradient(y, p)
 
 
 cdef class ModifiedHuber(Classification):
@@ -126,7 +129,7 @@ cdef class ModifiedHuber(Classification):
     See T. Zhang 'Solving Large Scale Linear Prediction Problems Using
     Stochastic Gradient Descent', ICML'04.
     """
-    cdef double loss(self, double p, double y) nogil:
+    cdef double cy_loss(self, double y, double p) noexcept nogil:
         cdef double z = p * y
         if z >= 1.0:
             return 0.0
@@ -135,7 +138,7 @@ cdef class ModifiedHuber(Classification):
         else:
             return -4.0 * z
 
-    cdef double dloss(self, double p, double y) nogil:
+    cdef double cy_gradient(self, double y, double p) noexcept nogil:
         cdef double z = p * y
         if z >= 1.0:
             return 0.0
@@ -164,13 +167,13 @@ cdef class Hinge(Classification):
     def __init__(self, double threshold=1.0):
         self.threshold = threshold
 
-    cdef double loss(self, double p, double y) nogil:
+    cdef double cy_loss(self, double y, double p) noexcept nogil:
         cdef double z = p * y
         if z <= self.threshold:
             return self.threshold - z
         return 0.0
 
-    cdef double dloss(self, double p, double y) nogil:
+    cdef double cy_gradient(self, double y, double p) noexcept nogil:
         cdef double z = p * y
         if z <= self.threshold:
             return -y
@@ -196,13 +199,13 @@ cdef class SquaredHinge(Classification):
     def __init__(self, double threshold=1.0):
         self.threshold = threshold
 
-    cdef double loss(self, double p, double y) nogil:
+    cdef double cy_loss(self, double y, double p) noexcept nogil:
         cdef double z = self.threshold - p * y
         if z > 0:
             return z * z
         return 0.0
 
-    cdef double dloss(self, double p, double y) nogil:
+    cdef double cy_gradient(self, double y, double p) noexcept nogil:
         cdef double z = self.threshold - p * y
         if z > 0:
             return -2 * y * z
@@ -212,79 +215,6 @@ cdef class SquaredHinge(Classification):
         return SquaredHinge, (self.threshold,)
 
 
-cdef class Log(Classification):
-    """Logistic regression loss for binary classification with y in {-1, 1}"""
-
-    cdef double loss(self, double p, double y) nogil:
-        cdef double z = p * y
-        # approximately equal and saves the computation of the log
-        if z > 18:
-            return exp(-z)
-        if z < -18:
-            return -z
-        return log(1.0 + exp(-z))
-
-    cdef double dloss(self, double p, double y) nogil:
-        cdef double z = p * y
-        # approximately equal and saves the computation of the log
-        if z > 18.0:
-            return exp(-z) * -y
-        if z < -18.0:
-            return -y
-        return -y / (exp(z) + 1.0)
-
-    def __reduce__(self):
-        return Log, ()
-
-
-cdef class SquaredLoss(Regression):
-    """Squared loss traditional used in linear regression."""
-    cdef double loss(self, double p, double y) nogil:
-        return 0.5 * (p - y) * (p - y)
-
-    cdef double dloss(self, double p, double y) nogil:
-        return p - y
-
-    def __reduce__(self):
-        return SquaredLoss, ()
-
-
-cdef class Huber(Regression):
-    """Huber regression loss
-
-    Variant of the SquaredLoss that is robust to outliers (quadratic near zero,
-    linear in for large errors).
-
-    https://en.wikipedia.org/wiki/Huber_Loss_Function
-    """
-
-    cdef double c
-
-    def __init__(self, double c):
-        self.c = c
-
-    cdef double loss(self, double p, double y) nogil:
-        cdef double r = p - y
-        cdef double abs_r = fabs(r)
-        if abs_r <= self.c:
-            return 0.5 * r * r
-        else:
-            return self.c * abs_r - (0.5 * self.c * self.c)
-
-    cdef double dloss(self, double p, double y) nogil:
-        cdef double r = p - y
-        cdef double abs_r = fabs(r)
-        if abs_r <= self.c:
-            return r
-        elif r > 0.0:
-            return self.c
-        else:
-            return -self.c
-
-    def __reduce__(self):
-        return Huber, (self.c,)
-
-
 cdef class EpsilonInsensitive(Regression):
     """Epsilon-Insensitive loss (used by SVR).
 
@@ -296,11 +226,11 @@ cdef class EpsilonInsensitive(Regression):
     def __init__(self, double epsilon):
         self.epsilon = epsilon
 
-    cdef double loss(self, double p, double y) nogil:
+    cdef double cy_loss(self, double y, double p) noexcept nogil:
         cdef double ret = fabs(y - p) - self.epsilon
         return ret if ret > 0 else 0
 
-    cdef double dloss(self, double p, double y) nogil:
+    cdef double cy_gradient(self, double y, double p) noexcept nogil:
         if y - p > self.epsilon:
             return -1
         elif p - y > self.epsilon:
@@ -323,11 +253,11 @@ cdef class SquaredEpsilonInsensitive(Regression):
     def __init__(self, double epsilon):
         self.epsilon = epsilon
 
-    cdef double loss(self, double p, double y) nogil:
+    cdef double cy_loss(self, double y, double p) noexcept nogil:
         cdef double ret = fabs(y - p) - self.epsilon
         return ret * ret if ret > 0 else 0
 
-    cdef double dloss(self, double p, double y) nogil:
+    cdef double cy_gradient(self, double y, double p) noexcept nogil:
         cdef double z
         z = y - p
         if z > self.epsilon:
@@ -340,42 +270,54 @@ cdef class SquaredEpsilonInsensitive(Regression):
     def __reduce__(self):
         return SquaredEpsilonInsensitive, (self.epsilon,)
 
-
-def _plain_sgd(np.ndarray[double, ndim=1, mode='c'] weights,
-               double intercept,
-               np.ndarray[double, ndim=1, mode='c'] average_weights,
-               double average_intercept,
-               LossFunction loss,
-               int penalty_type,
-               double alpha, double C,
-               double l1_ratio,
-               SequentialDataset dataset,
-               np.ndarray[unsigned char, ndim=1, mode='c'] validation_mask,
-               bint early_stopping, validation_score_cb,
-               int n_iter_no_change,
-               int max_iter, double tol, int fit_intercept,
-               int verbose, bint shuffle, np.uint32_t seed,
-               double weight_pos, double weight_neg,
-               int learning_rate, double eta0,
-               double power_t,
-               double t=1.0,
-               double intercept_decay=1.0,
-               int average=0):
+{{for name_suffix, c_type, np_type in dtypes}}
+
+def _plain_sgd{{name_suffix}}(
+    const {{c_type}}[::1] weights,
+    double intercept,
+    const {{c_type}}[::1] average_weights,
+    double average_intercept,
+    CyLossFunction loss,
+    int penalty_type,
+    double alpha,
+    double C,
+    double l1_ratio,
+    SequentialDataset{{name_suffix}} dataset,
+    const uint8_t[::1] validation_mask,
+    bint early_stopping,
+    validation_score_cb,
+    int n_iter_no_change,
+    unsigned int max_iter,
+    double tol,
+    int fit_intercept,
+    int verbose,
+    bint shuffle,
+    uint32_t seed,
+    double weight_pos,
+    double weight_neg,
+    int learning_rate,
+    double eta0,
+    double power_t,
+    bint one_class,
+    double t=1.0,
+    double intercept_decay=1.0,
+    int average=0,
+):
     """SGD for generic loss functions and penalties with optional averaging
 
     Parameters
     ----------
-    weights : ndarray[double, ndim=1]
+    weights : ndarray[{{c_type}}, ndim=1]
         The allocated vector of weights.
     intercept : double
         The initial intercept.
-    average_weights : ndarray[double, ndim=1]
+    average_weights : ndarray[{{c_type}}, ndim=1]
         The average weights as computed for ASGD. Should be None if average
         is 0.
     average_intercept : double
         The average intercept for ASGD. Should be 0 if average is 0.
-    loss : LossFunction
-        A concrete ``LossFunction`` object.
+    loss : CyLossFunction
+        A concrete ``CyLossFunction`` object.
     penalty_type : int
         The penalty 2 for L2, 1 for L1, and 3 for Elastic-Net.
     alpha : float
@@ -387,7 +329,7 @@ def _plain_sgd(np.ndarray[double, ndim=1, mode='c'] weights,
         l1_ratio=0 corresponds to L2 penalty, l1_ratio=1 to L1.
     dataset : SequentialDataset
         A concrete ``SequentialDataset`` object.
-    validation_mask : ndarray[unsigned char, ndim=1]
+    validation_mask : ndarray[uint8_t, ndim=1]
         Equal to True on the validation set.
     early_stopping : boolean
         Whether to use a stopping criterion based on the validation set.
@@ -401,8 +343,6 @@ def _plain_sgd(np.ndarray[double, ndim=1, mode='c'] weights,
         The maximum number of iterations (epochs).
     tol: double
         The tolerance for the stopping criterion.
-    dataset : SequentialDataset
-        A concrete ``SequentialDataset`` object.
     fit_intercept : int
         Whether or not to fit the intercept (1 or 0).
     verbose : int
@@ -413,7 +353,7 @@ def _plain_sgd(np.ndarray[double, ndim=1, mode='c'] weights,
         The weight of the positive class.
     weight_neg : float
         The weight of the negative class.
-    seed : np.uint32_t
+    seed : uint32_t
         Seed of the pseudorandom number generator used to shuffle the data.
     learning_rate : int
         The learning rate:
@@ -427,6 +367,8 @@ def _plain_sgd(np.ndarray[double, ndim=1, mode='c'] weights,
         The initial learning rate.
     power_t : double
         The exponent for inverse scaling learning rate.
+    one_class : boolean
+        Whether to solve the One-Class SVM optimization problem.
     t : double
         Initial state of the learning rate. This value is equal to the
         iteration count except when the learning rate is set to `optimal`.
@@ -435,6 +377,7 @@ def _plain_sgd(np.ndarray[double, ndim=1, mode='c'] weights,
         The number of iterations before averaging starts. average=1 is
         equivalent to averaging for all iterations.
 
+
     Returns
     -------
     weights : array, shape=[n_features]
@@ -455,11 +398,9 @@ def _plain_sgd(np.ndarray[double, ndim=1, mode='c'] weights,
     cdef Py_ssize_t n_samples = dataset.n_samples
     cdef Py_ssize_t n_features = weights.shape[0]
 
-    cdef WeightVector w = WeightVector(weights, average_weights)
-    cdef double* w_ptr = &weights[0]
-    cdef double *x_data_ptr = NULL
+    cdef WeightVector{{name_suffix}} w = WeightVector{{name_suffix}}(weights, average_weights)
+    cdef {{c_type}} *x_data_ptr = NULL
     cdef int *x_ind_ptr = NULL
-    cdef double* ps_ptr = NULL
 
     # helper variables
     cdef int no_improvement_count = 0
@@ -468,32 +409,31 @@ def _plain_sgd(np.ndarray[double, ndim=1, mode='c'] weights,
     cdef double eta = 0.0
     cdef double p = 0.0
     cdef double update = 0.0
+    cdef double intercept_update = 0.0
     cdef double sumloss = 0.0
     cdef double score = 0.0
     cdef double best_loss = INFINITY
     cdef double best_score = -INFINITY
-    cdef double y = 0.0
-    cdef double sample_weight
-    cdef double class_weight = 1.0
+    cdef {{c_type}} y = 0.0
+    cdef {{c_type}} sample_weight
+    cdef {{c_type}} class_weight = 1.0
     cdef unsigned int count = 0
+    cdef unsigned int train_count = n_samples - np.sum(validation_mask)
     cdef unsigned int epoch = 0
     cdef unsigned int i = 0
     cdef int is_hinge = isinstance(loss, Hinge)
     cdef double optimal_init = 0.0
     cdef double dloss = 0.0
     cdef double MAX_DLOSS = 1e12
-    cdef double max_change = 0.0
-    cdef double max_weight = 0.0
 
     cdef long long sample_index
-    cdef unsigned char [:] validation_mask_view = validation_mask
 
     # q vector is only used for L1 regularization
-    cdef np.ndarray[double, ndim = 1, mode = "c"] q = None
-    cdef double * q_data_ptr = NULL
+    cdef {{c_type}}[::1] q = None
+    cdef {{c_type}} * q_data_ptr = NULL
     if penalty_type == L1 or penalty_type == ELASTICNET:
-        q = np.zeros((n_features,), dtype=np.float64, order="c")
-        q_data_ptr =  q.data
+        q = np.zeros((n_features,), dtype={{np_type}}, order="c")
+        q_data_ptr = &q[0]
     cdef double u = 0.0
 
     if penalty_type == L2:
@@ -506,7 +446,7 @@ def _plain_sgd(np.ndarray[double, ndim=1, mode='c'] weights,
     if learning_rate == OPTIMAL:
         typw = np.sqrt(1.0 / np.sqrt(alpha))
         # computing eta0, the initial learning rate
-        initial_eta0 = typw / max(1.0, loss.dloss(-typw, 1.0))
+        initial_eta0 = typw / max(1.0, loss.cy_gradient(1.0, -typw))
         # initialize t such that eta at first sample equals eta0
         optimal_init = 1.0 / (initial_eta0 * alpha)
 
@@ -524,7 +464,7 @@ def _plain_sgd(np.ndarray[double, ndim=1, mode='c'] weights,
                              &y, &sample_weight)
 
                 sample_index = dataset.index_data_ptr[dataset.current_index]
-                if validation_mask_view[sample_index]:
+                if validation_mask[sample_index]:
                     # do not learn on the validation set
                     continue
 
@@ -535,7 +475,7 @@ def _plain_sgd(np.ndarray[double, ndim=1, mode='c'] weights,
                     eta = eta0 / pow(t, power_t)
 
                 if verbose or not early_stopping:
-                    sumloss += loss.loss(p, y)
+                    sumloss += loss.cy_loss(y, p)
 
                 if y > 0.0:
                     class_weight = weight_pos
@@ -546,12 +486,12 @@ def _plain_sgd(np.ndarray[double, ndim=1, mode='c'] weights,
                     update = sqnorm(x_data_ptr, x_ind_ptr, xnnz)
                     if update == 0:
                         continue
-                    update = min(C, loss.loss(p, y) / update)
+                    update = min(C, loss.cy_loss(y, p) / update)
                 elif learning_rate == PA2:
                     update = sqnorm(x_data_ptr, x_ind_ptr, xnnz)
-                    update = loss.loss(p, y) / (update + 0.5 / C)
+                    update = loss.cy_loss(y, p) / (update + 0.5 / C)
                 else:
-                    dloss = loss.dloss(p, y)
+                    dloss = loss.cy_gradient(y, p)
                     # clip dloss with large values to avoid numerical
                     # instabilities
                     if dloss < -MAX_DLOSS:
@@ -574,10 +514,15 @@ def _plain_sgd(np.ndarray[double, ndim=1, mode='c'] weights,
                     # do not scale to negative values when eta or alpha are too
                     # big: instead set the weights to zero
                     w.scale(max(0, 1.0 - ((1.0 - l1_ratio) * eta * alpha)))
+
                 if update != 0.0:
                     w.add(x_data_ptr, x_ind_ptr, xnnz, update)
-                    if fit_intercept == 1:
-                        intercept += update * intercept_decay
+                if fit_intercept == 1:
+                    intercept_update = update
+                    if one_class:  # specific for One-Class SVM
+                        intercept_update -= 2. * eta * alpha
+                    if intercept_update != 0:
+                        intercept += intercept_update * intercept_decay
 
                 if 0 < average <= t:
                     # compute the average for the intercept and update the
@@ -591,7 +536,7 @@ def _plain_sgd(np.ndarray[double, ndim=1, mode='c'] weights,
 
                 if penalty_type == L1 or penalty_type == ELASTICNET:
                     u += (l1_ratio * eta * alpha)
-                    l1penalty(w, q_data_ptr, x_ind_ptr, xnnz, u)
+                    l1penalty{{name_suffix}}(w, q_data_ptr, x_ind_ptr, xnnz, u)
 
                 t += 1
                 count += 1
@@ -601,21 +546,20 @@ def _plain_sgd(np.ndarray[double, ndim=1, mode='c'] weights,
                 with gil:
                     print("Norm: %.2f, NNZs: %d, Bias: %.6f, T: %d, "
                           "Avg. loss: %f"
-                          % (w.norm(), weights.nonzero()[0].shape[0],
-                             intercept, count, sumloss / n_samples))
+                          % (w.norm(), np.nonzero(weights)[0].shape[0],
+                             intercept, count, sumloss / train_count))
                     print("Total training time: %.2f seconds."
                           % (time() - t_start))
 
             # floating-point under-/overflow check.
-            if (not skl_isfinite(intercept)
-                or any_nonfinite(weights.data, n_features)):
+            if (not isfinite(intercept) or any_nonfinite(weights)):
                 infinity = True
                 break
 
             # evaluate the score on the validation set
             if early_stopping:
                 with gil:
-                    score = validation_score_cb(weights, intercept)
+                    score = validation_score_cb(weights.base, intercept)
                 if tol > -INFINITY and score < best_score + tol:
                     no_improvement_count += 1
                 else:
@@ -624,7 +568,7 @@ def _plain_sgd(np.ndarray[double, ndim=1, mode='c'] weights,
                     best_score = score
             # or evaluate the loss on the training set
             else:
-                if tol > -INFINITY and sumloss > best_loss - tol * n_samples:
+                if tol > -INFINITY and sumloss > best_loss - tol * train_count:
                     no_improvement_count += 1
                 else:
                     no_improvement_count = 0
@@ -650,17 +594,29 @@ def _plain_sgd(np.ndarray[double, ndim=1, mode='c'] weights,
 
     w.reset_wscale()
 
-    return weights, intercept, average_weights, average_intercept, epoch + 1
+    return (
+        weights.base,
+        intercept,
+        None if average_weights is None else average_weights.base,
+        average_intercept,
+        epoch + 1
+    )
 
+{{endfor}}
 
-cdef bint any_nonfinite(double *w, int n) nogil:
-    for i in range(n):
-        if not skl_isfinite(w[i]):
+
+cdef inline bint any_nonfinite(const floating[::1] w) noexcept nogil:
+    for i in range(w.shape[0]):
+        if not isfinite(w[i]):
             return True
     return 0
 
 
-cdef double sqnorm(double * x_data_ptr, int * x_ind_ptr, int xnnz) nogil:
+cdef inline double sqnorm(
+    floating * x_data_ptr,
+    int * x_ind_ptr,
+    int xnnz,
+) noexcept nogil:
     cdef double x_norm = 0.0
     cdef int j
     cdef double z
@@ -670,8 +626,15 @@ cdef double sqnorm(double * x_data_ptr, int * x_ind_ptr, int xnnz) nogil:
     return x_norm
 
 
-cdef void l1penalty(WeightVector w, double * q_data_ptr,
-                    int *x_ind_ptr, int xnnz, double u) nogil:
+{{for name_suffix, c_type, np_type in dtypes}}
+
+cdef void l1penalty{{name_suffix}}(
+    WeightVector{{name_suffix}} w,
+    {{c_type}} * q_data_ptr,
+    int *x_ind_ptr,
+    int xnnz,
+    double u,
+) noexcept nogil:
     """Apply the L1 penalty to each updated feature
 
     This implements the truncated gradient approach by
@@ -681,7 +644,7 @@ cdef void l1penalty(WeightVector w, double * q_data_ptr,
     cdef int j = 0
     cdef int idx = 0
     cdef double wscale = w.wscale
-    cdef double *w_data_ptr = w.w_data_ptr
+    cdef {{c_type}} *w_data_ptr = w.w_data_ptr
     for j in range(xnnz):
         idx = x_ind_ptr[j]
         z = w_data_ptr[idx]
@@ -694,3 +657,5 @@ cdef void l1penalty(WeightVector w, double * q_data_ptr,
                 0.0, w_data_ptr[idx] + ((u - q_data_ptr[idx]) / wscale))
 
         q_data_ptr[idx] += wscale * (w_data_ptr[idx] - z)
+
+{{endfor}}
diff --git a/sklearn/linear_model/_sgd_fast_helpers.h b/sklearn/linear_model/_sgd_fast_helpers.h
deleted file mode 100644
index 819c6b63b2e00..0000000000000
--- a/sklearn/linear_model/_sgd_fast_helpers.h
+++ /dev/null
@@ -1,16 +0,0 @@
-// We cannot directly reuse the npy_isfinite from npy_math.h as numpy
-// and scikit-learn are not necessarily built with the same compiler.
-// When re-declaring the functions in the template for cython
-// specific for each parameter input type, it needs to be 2 different functions
-// as cython doesn't support function overloading.
-#ifdef _MSC_VER
-# include 
-# define skl_isfinite _finite
-# define skl_isfinite32 _finite
-# define skl_isfinite64 _finite
-#else
-# include 
-# define skl_isfinite npy_isfinite
-# define skl_isfinite32 npy_isfinite
-# define skl_isfinite64 npy_isfinite
-#endif
diff --git a/sklearn/linear_model/_stochastic_gradient.py b/sklearn/linear_model/_stochastic_gradient.py
index 65f6cc6966ba4..8f7c814000614 100644
--- a/sklearn/linear_model/_stochastic_gradient.py
+++ b/sklearn/linear_model/_stochastic_gradient.py
@@ -1,44 +1,53 @@
-# Authors: Peter Prettenhofer  (main author)
-#          Mathieu Blondel (partial_fit support)
-#
-# License: BSD 3 clause
-"""Classification and regression using Stochastic Gradient Descent (SGD)."""
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-import numpy as np
-import warnings
+"""Classification, regression and One-Class SVM using Stochastic Gradient
+Descent (SGD).
+"""
 
+import warnings
 from abc import ABCMeta, abstractmethod
+from numbers import Integral, Real
 
-from joblib import Parallel
+import numpy as np
 
-from ..base import clone, is_classifier
-from ._base import LinearClassifierMixin, SparseCoefMixin
-from ._base import make_dataset
-from ..base import BaseEstimator, RegressorMixin
-from ..utils import check_random_state
+from .._loss._loss import CyHalfBinomialLoss, CyHalfSquaredError, CyHuberLoss
+from ..base import (
+    BaseEstimator,
+    OutlierMixin,
+    RegressorMixin,
+    _fit_context,
+    clone,
+    is_classifier,
+)
+from ..exceptions import ConvergenceWarning
+from ..model_selection import ShuffleSplit, StratifiedShuffleSplit
+from ..utils import check_random_state, compute_class_weight
+from ..utils._param_validation import Hidden, Interval, StrOptions
 from ..utils.extmath import safe_sparse_dot
+from ..utils.metaestimators import available_if
 from ..utils.multiclass import _check_partial_fit_first_call
-from ..utils.validation import check_is_fitted, _check_sample_weight
-from ..utils.validation import _deprecate_positional_args
-from ..utils.fixes import delayed
-from ..exceptions import ConvergenceWarning
-from ..model_selection import StratifiedShuffleSplit, ShuffleSplit
-
-from ._sgd_fast import _plain_sgd
-from ..utils import compute_class_weight
-from ._sgd_fast import Hinge
-from ._sgd_fast import SquaredHinge
-from ._sgd_fast import Log
-from ._sgd_fast import ModifiedHuber
-from ._sgd_fast import SquaredLoss
-from ._sgd_fast import Huber
-from ._sgd_fast import EpsilonInsensitive
-from ._sgd_fast import SquaredEpsilonInsensitive
-from ..utils.fixes import _joblib_parallel_args
-from ..utils import deprecated
-
-LEARNING_RATE_TYPES = {"constant": 1, "optimal": 2, "invscaling": 3,
-                       "adaptive": 4, "pa1": 5, "pa2": 6}
+from ..utils.parallel import Parallel, delayed
+from ..utils.validation import _check_sample_weight, check_is_fitted, validate_data
+from ._base import LinearClassifierMixin, SparseCoefMixin, make_dataset
+from ._sgd_fast import (
+    EpsilonInsensitive,
+    Hinge,
+    ModifiedHuber,
+    SquaredEpsilonInsensitive,
+    SquaredHinge,
+    _plain_sgd32,
+    _plain_sgd64,
+)
+
+LEARNING_RATE_TYPES = {
+    "constant": 1,
+    "optimal": 2,
+    "invscaling": 3,
+    "adaptive": 4,
+    "pa1": 5,
+    "pa2": 6,
+}
 
 PENALTY_TYPES = {"none": 0, "l2": 2, "l1": 1, "elasticnet": 3}
 
@@ -51,8 +60,7 @@
 class _ValidationScoreCallback:
     """Callback for early stopping based on validation score"""
 
-    def __init__(self, estimator, X_val, y_val, sample_weight_val,
-                 classes=None):
+    def __init__(self, estimator, X_val, y_val, sample_weight_val, classes=None):
         self.estimator = clone(estimator)
         self.estimator.t_ = 1  # to pass check_is_fitted
         if classes is not None:
@@ -70,13 +78,42 @@ def __call__(self, coef, intercept):
 
 class BaseSGD(SparseCoefMixin, BaseEstimator, metaclass=ABCMeta):
     """Base class for SGD classification and regression."""
-    @_deprecate_positional_args
-    def __init__(self, loss, *, penalty='l2', alpha=0.0001, C=1.0,
-                 l1_ratio=0.15, fit_intercept=True, max_iter=1000, tol=1e-3,
-                 shuffle=True, verbose=0, epsilon=0.1, random_state=None,
-                 learning_rate="optimal", eta0=0.0, power_t=0.5,
-                 early_stopping=False, validation_fraction=0.1,
-                 n_iter_no_change=5, warm_start=False, average=False):
+
+    _parameter_constraints: dict = {
+        "fit_intercept": ["boolean"],
+        "max_iter": [Interval(Integral, 1, None, closed="left")],
+        "tol": [Interval(Real, 0, None, closed="left"), None],
+        "shuffle": ["boolean"],
+        "verbose": ["verbose"],
+        "random_state": ["random_state"],
+        "warm_start": ["boolean"],
+        "average": [Interval(Integral, 0, None, closed="neither"), "boolean"],
+    }
+
+    def __init__(
+        self,
+        loss,
+        *,
+        penalty="l2",
+        alpha=0.0001,
+        C=1.0,
+        l1_ratio=0.15,
+        fit_intercept=True,
+        max_iter=1000,
+        tol=1e-3,
+        shuffle=True,
+        verbose=0,
+        epsilon=0.1,
+        random_state=None,
+        learning_rate="optimal",
+        eta0=0.0,
+        power_t=0.5,
+        early_stopping=False,
+        validation_fraction=0.1,
+        n_iter_no_change=5,
+        warm_start=False,
+        average=False,
+    ):
         self.loss = loss
         self.penalty = penalty
         self.learning_rate = learning_rate
@@ -97,152 +134,133 @@ def __init__(self, loss, *, penalty='l2', alpha=0.0001, C=1.0,
         self.average = average
         self.max_iter = max_iter
         self.tol = tol
-        # current tests expect init to do parameter validation
-        # but we are not allowed to set attributes
-        self._validate_params()
-
-    def set_params(self, **kwargs):
-        """Set and validate the parameters of estimator.
-
-        Parameters
-        ----------
-        **kwargs : dict
-            Estimator parameters.
-
-        Returns
-        -------
-        self : object
-            Estimator instance.
-        """
-        super().set_params(**kwargs)
-        self._validate_params()
-        return self
 
     @abstractmethod
     def fit(self, X, y):
         """Fit model."""
 
-    def _validate_params(self, for_partial_fit=False):
-        """Validate input params. """
-        if not isinstance(self.shuffle, bool):
-            raise ValueError("shuffle must be either True or False")
-        if not isinstance(self.early_stopping, bool):
-            raise ValueError("early_stopping must be either True or False")
+    def _more_validate_params(self, for_partial_fit=False):
+        """Validate input params."""
         if self.early_stopping and for_partial_fit:
             raise ValueError("early_stopping should be False with partial_fit")
-        if self.max_iter is not None and self.max_iter <= 0:
-            raise ValueError("max_iter must be > zero. Got %f" % self.max_iter)
-        if not (0.0 <= self.l1_ratio <= 1.0):
-            raise ValueError("l1_ratio must be in [0, 1]")
-        if self.alpha < 0.0:
-            raise ValueError("alpha must be >= 0")
-        if self.n_iter_no_change < 1:
-            raise ValueError("n_iter_no_change must be >= 1")
-        if not (0.0 < self.validation_fraction < 1.0):
-            raise ValueError("validation_fraction must be in range (0, 1)")
-        if self.learning_rate in ("constant", "invscaling", "adaptive"):
-            if self.eta0 <= 0.0:
-                raise ValueError("eta0 must be > 0")
+        if (
+            self.learning_rate in ("constant", "invscaling", "adaptive")
+            and self.eta0 <= 0.0
+        ):
+            raise ValueError("eta0 must be > 0")
         if self.learning_rate == "optimal" and self.alpha == 0:
-            raise ValueError("alpha must be > 0 since "
-                             "learning_rate is 'optimal'. alpha is used "
-                             "to compute the optimal learning rate.")
+            raise ValueError(
+                "alpha must be > 0 since "
+                "learning_rate is 'optimal'. alpha is used "
+                "to compute the optimal learning rate."
+            )
+        if self.penalty == "elasticnet" and self.l1_ratio is None:
+            raise ValueError("l1_ratio must be set when penalty is 'elasticnet'")
 
         # raises ValueError if not registered
         self._get_penalty_type(self.penalty)
         self._get_learning_rate_type(self.learning_rate)
 
-        if self.loss not in self.loss_functions:
-            raise ValueError("The loss %s is not supported. " % self.loss)
+    def _get_l1_ratio(self):
+        if self.l1_ratio is None:
+            # plain_sgd expects a float. Any value is fine since at this point
+            # penalty can't be "elsaticnet" so l1_ratio is not used.
+            return 0.0
+        return self.l1_ratio
 
     def _get_loss_function(self, loss):
-        """Get concrete ``LossFunction`` object for str ``loss``. """
-        try:
-            loss_ = self.loss_functions[loss]
-            loss_class, args = loss_[0], loss_[1:]
-            if loss in ('huber', 'epsilon_insensitive',
-                        'squared_epsilon_insensitive'):
-                args = (self.epsilon, )
-            return loss_class(*args)
-        except KeyError as e:
-            raise ValueError("The loss %s is not supported. " % loss) from e
+        """Get concrete ``LossFunction`` object for str ``loss``."""
+        loss_ = self.loss_functions[loss]
+        loss_class, args = loss_[0], loss_[1:]
+        if loss in ("huber", "epsilon_insensitive", "squared_epsilon_insensitive"):
+            args = (self.epsilon,)
+        return loss_class(*args)
 
     def _get_learning_rate_type(self, learning_rate):
-        try:
-            return LEARNING_RATE_TYPES[learning_rate]
-        except KeyError as e:
-            raise ValueError("learning rate %s "
-                             "is not supported. " % learning_rate) from e
+        return LEARNING_RATE_TYPES[learning_rate]
 
     def _get_penalty_type(self, penalty):
         penalty = str(penalty).lower()
-        try:
-            return PENALTY_TYPES[penalty]
-        except KeyError as e:
-            raise ValueError("Penalty %s is not supported. " % penalty) from e
-
-    def _allocate_parameter_mem(self, n_classes, n_features, coef_init=None,
-                                intercept_init=None):
+        return PENALTY_TYPES[penalty]
+
+    def _allocate_parameter_mem(
+        self,
+        n_classes,
+        n_features,
+        input_dtype,
+        coef_init=None,
+        intercept_init=None,
+        one_class=0,
+    ):
         """Allocate mem for parameters; initialize if provided."""
         if n_classes > 2:
             # allocate coef_ for multi-class
             if coef_init is not None:
-                coef_init = np.asarray(coef_init, order="C")
+                coef_init = np.asarray(coef_init, dtype=input_dtype, order="C")
                 if coef_init.shape != (n_classes, n_features):
-                    raise ValueError("Provided ``coef_`` does not match "
-                                     "dataset. ")
+                    raise ValueError("Provided ``coef_`` does not match dataset. ")
                 self.coef_ = coef_init
             else:
-                self.coef_ = np.zeros((n_classes, n_features),
-                                      dtype=np.float64, order="C")
+                self.coef_ = np.zeros(
+                    (n_classes, n_features), dtype=input_dtype, order="C"
+                )
 
             # allocate intercept_ for multi-class
             if intercept_init is not None:
-                intercept_init = np.asarray(intercept_init, order="C")
-                if intercept_init.shape != (n_classes, ):
-                    raise ValueError("Provided intercept_init "
-                                     "does not match dataset.")
+                intercept_init = np.asarray(
+                    intercept_init, order="C", dtype=input_dtype
+                )
+                if intercept_init.shape != (n_classes,):
+                    raise ValueError("Provided intercept_init does not match dataset.")
                 self.intercept_ = intercept_init
             else:
-                self.intercept_ = np.zeros(n_classes, dtype=np.float64,
-                                           order="C")
+                self.intercept_ = np.zeros(n_classes, dtype=input_dtype, order="C")
         else:
-            # allocate coef_ for binary problem
+            # allocate coef_
             if coef_init is not None:
-                coef_init = np.asarray(coef_init, dtype=np.float64,
-                                       order="C")
+                coef_init = np.asarray(coef_init, dtype=input_dtype, order="C")
                 coef_init = coef_init.ravel()
                 if coef_init.shape != (n_features,):
-                    raise ValueError("Provided coef_init does not "
-                                     "match dataset.")
+                    raise ValueError("Provided coef_init does not match dataset.")
                 self.coef_ = coef_init
             else:
-                self.coef_ = np.zeros(n_features,
-                                      dtype=np.float64,
-                                      order="C")
+                self.coef_ = np.zeros(n_features, dtype=input_dtype, order="C")
 
-            # allocate intercept_ for binary problem
+            # allocate intercept_
             if intercept_init is not None:
-                intercept_init = np.asarray(intercept_init, dtype=np.float64)
+                intercept_init = np.asarray(intercept_init, dtype=input_dtype)
                 if intercept_init.shape != (1,) and intercept_init.shape != ():
-                    raise ValueError("Provided intercept_init "
-                                     "does not match dataset.")
-                self.intercept_ = intercept_init.reshape(1,)
+                    raise ValueError("Provided intercept_init does not match dataset.")
+                if one_class:
+                    self.offset_ = intercept_init.reshape(
+                        1,
+                    )
+                else:
+                    self.intercept_ = intercept_init.reshape(
+                        1,
+                    )
             else:
-                self.intercept_ = np.zeros(1, dtype=np.float64, order="C")
+                if one_class:
+                    self.offset_ = np.zeros(1, dtype=input_dtype, order="C")
+                else:
+                    self.intercept_ = np.zeros(1, dtype=input_dtype, order="C")
 
         # initialize average parameters
         if self.average > 0:
             self._standard_coef = self.coef_
-            self._standard_intercept = self.intercept_
-            self._average_coef = np.zeros(self.coef_.shape,
-                                          dtype=np.float64,
-                                          order="C")
-            self._average_intercept = np.zeros(self._standard_intercept.shape,
-                                               dtype=np.float64,
-                                               order="C")
-
-    def _make_validation_split(self, y):
+            self._average_coef = np.zeros(
+                self.coef_.shape, dtype=input_dtype, order="C"
+            )
+            if one_class:
+                self._standard_intercept = 1 - self.offset_
+            else:
+                self._standard_intercept = self.intercept_
+
+            self._average_intercept = np.zeros(
+                self._standard_intercept.shape, dtype=input_dtype, order="C"
+            )
+
+    def _make_validation_split(self, y, sample_mask):
         """Split the dataset between training set and validation set.
 
         Parameters
@@ -250,13 +268,17 @@ def _make_validation_split(self, y):
         y : ndarray of shape (n_samples, )
             Target values.
 
+        sample_mask : ndarray of shape (n_samples, )
+            A boolean array indicating whether each sample should be included
+            for validation set.
+
         Returns
         -------
         validation_mask : ndarray of shape (n_samples, )
-            Equal to 1 on the validation set, 0 on the training set.
+            Equal to True on the validation set, False on the training set.
         """
         n_samples = y.shape[0]
-        validation_mask = np.zeros(n_samples, dtype=np.uint8)
+        validation_mask = np.zeros(n_samples, dtype=np.bool_)
         if not self.early_stopping:
             # use the full set for training, with an empty validation set
             return validation_mask
@@ -265,71 +287,61 @@ def _make_validation_split(self, y):
             splitter_type = StratifiedShuffleSplit
         else:
             splitter_type = ShuffleSplit
-        cv = splitter_type(test_size=self.validation_fraction,
-                           random_state=self.random_state)
+        cv = splitter_type(
+            test_size=self.validation_fraction, random_state=self.random_state
+        )
         idx_train, idx_val = next(cv.split(np.zeros(shape=(y.shape[0], 1)), y))
+
+        if not np.any(sample_mask[idx_val]):
+            raise ValueError(
+                "The sample weights for validation set are all zero, consider using a"
+                " different random state."
+            )
+
         if idx_train.shape[0] == 0 or idx_val.shape[0] == 0:
             raise ValueError(
                 "Splitting %d samples into a train set and a validation set "
                 "with validation_fraction=%r led to an empty set (%d and %d "
                 "samples). Please either change validation_fraction, increase "
                 "number of samples, or disable early_stopping."
-                % (n_samples, self.validation_fraction, idx_train.shape[0],
-                   idx_val.shape[0]))
-
-        validation_mask[idx_val] = 1
+                % (
+                    n_samples,
+                    self.validation_fraction,
+                    idx_train.shape[0],
+                    idx_val.shape[0],
+                )
+            )
+
+        validation_mask[idx_val] = True
         return validation_mask
 
-    def _make_validation_score_cb(self, validation_mask, X, y, sample_weight,
-                                  classes=None):
+    def _make_validation_score_cb(
+        self, validation_mask, X, y, sample_weight, classes=None
+    ):
         if not self.early_stopping:
             return None
 
         return _ValidationScoreCallback(
-            self, X[validation_mask], y[validation_mask],
-            sample_weight[validation_mask], classes=classes)
-
-    # mypy error: Decorated property not supported
-    @deprecated("Attribute standard_coef_ was deprecated "  # type: ignore
-                "in version 0.23 and will be removed in 1.0 "
-                "(renaming of 0.25).")
-    @property
-    def standard_coef_(self):
-        return self._standard_coef
-
-    # mypy error: Decorated property not supported
-    @deprecated(  # type: ignore
-        "Attribute standard_intercept_ was deprecated "
-        "in version 0.23 and will be removed in 1.0 (renaming of 0.25)."
-    )
-    @property
-    def standard_intercept_(self):
-        return self._standard_intercept
-
-    # mypy error: Decorated property not supported
-    @deprecated("Attribute average_coef_ was deprecated "  # type: ignore
-                "in version 0.23 and will be removed in 1.0 "
-                "(renaming of 0.25).")
-    @property
-    def average_coef_(self):
-        return self._average_coef
-
-    # mypy error: Decorated property not supported
-    @deprecated("Attribute average_intercept_ was deprecated "  # type: ignore
-                "in version 0.23 and will be removed in 1.0 "
-                "(renaming of 0.25).")
-    @property
-    def average_intercept_(self):
-        return self._average_intercept
-
-
-def _prepare_fit_binary(est, y, i):
+            self,
+            X[validation_mask],
+            y[validation_mask],
+            sample_weight[validation_mask],
+            classes=classes,
+        )
+
+
+def _prepare_fit_binary(est, y, i, input_dtype, label_encode=True):
     """Initialization for fit_binary.
 
     Returns y, coef, intercept, average_coef, average_intercept.
     """
-    y_i = np.ones(y.shape, dtype=np.float64, order="C")
-    y_i[y != est.classes_[i]] = -1.0
+    y_i = np.ones(y.shape, dtype=input_dtype, order="C")
+    if label_encode:
+        # y in {0, 1}
+        y_i[y != est.classes_[i]] = 0.0
+    else:
+        # y in {-1, +1}
+        y_i[y != est.classes_[i]] = -1.0
     average_intercept = 0
     average_coef = None
 
@@ -355,9 +367,21 @@ def _prepare_fit_binary(est, y, i):
     return y_i, coef, intercept, average_coef, average_intercept
 
 
-def fit_binary(est, i, X, y, alpha, C, learning_rate, max_iter,
-               pos_weight, neg_weight, sample_weight, validation_mask=None,
-               random_state=None):
+def fit_binary(
+    est,
+    i,
+    X,
+    y,
+    alpha,
+    C,
+    learning_rate,
+    max_iter,
+    pos_weight,
+    neg_weight,
+    sample_weight,
+    validation_mask=None,
+    random_state=None,
+):
     """Fit a single binary classifier.
 
     The i'th class is considered the "positive" class.
@@ -382,7 +406,7 @@ def fit_binary(est, i, X, y, alpha, C, learning_rate, max_iter,
     C : float
         Maximum step size for passive aggressive
 
-    learning_rate : string
+    learning_rate : str
         The learning rate. Accepted values are 'constant', 'optimal',
         'invscaling', 'pa1' and 'pa2'.
 
@@ -410,22 +434,26 @@ def fit_binary(est, i, X, y, alpha, C, learning_rate, max_iter,
     """
     # if average is not true, average_coef, and average_intercept will be
     # unused
-    y_i, coef, intercept, average_coef, average_intercept = \
-        _prepare_fit_binary(est, y, i)
+    label_encode = isinstance(est._loss_function_, CyHalfBinomialLoss)
+    y_i, coef, intercept, average_coef, average_intercept = _prepare_fit_binary(
+        est, y, i, input_dtype=X.dtype, label_encode=label_encode
+    )
     assert y_i.shape[0] == y.shape[0] == sample_weight.shape[0]
 
     random_state = check_random_state(random_state)
     dataset, intercept_decay = make_dataset(
-        X, y_i, sample_weight, random_state=random_state)
+        X, y_i, sample_weight, random_state=random_state
+    )
 
     penalty_type = est._get_penalty_type(est.penalty)
     learning_rate_type = est._get_learning_rate_type(learning_rate)
 
     if validation_mask is None:
-        validation_mask = est._make_validation_split(y_i)
+        validation_mask = est._make_validation_split(y_i, sample_mask=sample_weight > 0)
     classes = np.array([-1, 1], dtype=y_i.dtype)
     validation_score_cb = est._make_validation_score_cb(
-        validation_mask, X, y_i, sample_weight, classes=classes)
+        validation_mask, X, y_i, sample_weight, classes=classes
+    )
 
     # numpy mtrand expects a C long which is a signed 32 bit integer under
     # Windows
@@ -433,13 +461,38 @@ def fit_binary(est, i, X, y, alpha, C, learning_rate, max_iter,
 
     tol = est.tol if est.tol is not None else -np.inf
 
+    _plain_sgd = _get_plain_sgd_function(input_dtype=coef.dtype)
     coef, intercept, average_coef, average_intercept, n_iter_ = _plain_sgd(
-        coef, intercept, average_coef, average_intercept, est.loss_function_,
-        penalty_type, alpha, C, est.l1_ratio, dataset, validation_mask,
-        est.early_stopping, validation_score_cb, int(est.n_iter_no_change),
-        max_iter, tol, int(est.fit_intercept), int(est.verbose),
-        int(est.shuffle), seed, pos_weight, neg_weight, learning_rate_type,
-        est.eta0, est.power_t, est.t_, intercept_decay, est.average)
+        coef,
+        intercept,
+        average_coef,
+        average_intercept,
+        est._loss_function_,
+        penalty_type,
+        alpha,
+        C,
+        est._get_l1_ratio(),
+        dataset,
+        validation_mask,
+        est.early_stopping,
+        validation_score_cb,
+        int(est.n_iter_no_change),
+        max_iter,
+        tol,
+        int(est.fit_intercept),
+        int(est.verbose),
+        int(est.shuffle),
+        seed,
+        pos_weight,
+        neg_weight,
+        learning_rate_type,
+        est.eta0,
+        est.power_t,
+        0,
+        est.t_,
+        intercept_decay,
+        est.average,
+    )
 
     if est.average:
         if len(est.classes_) == 2:
@@ -450,51 +503,108 @@ def fit_binary(est, i, X, y, alpha, C, learning_rate, max_iter,
     return coef, intercept, n_iter_
 
 
-class BaseSGDClassifier(LinearClassifierMixin, BaseSGD, metaclass=ABCMeta):
+def _get_plain_sgd_function(input_dtype):
+    return _plain_sgd32 if input_dtype == np.float32 else _plain_sgd64
 
+
+class BaseSGDClassifier(LinearClassifierMixin, BaseSGD, metaclass=ABCMeta):
     loss_functions = {
         "hinge": (Hinge, 1.0),
         "squared_hinge": (SquaredHinge, 1.0),
         "perceptron": (Hinge, 0.0),
-        "log": (Log, ),
-        "modified_huber": (ModifiedHuber, ),
-        "squared_loss": (SquaredLoss, ),
-        "huber": (Huber, DEFAULT_EPSILON),
+        "log_loss": (CyHalfBinomialLoss,),
+        "modified_huber": (ModifiedHuber,),
+        "squared_error": (CyHalfSquaredError,),
+        "huber": (CyHuberLoss, DEFAULT_EPSILON),
         "epsilon_insensitive": (EpsilonInsensitive, DEFAULT_EPSILON),
-        "squared_epsilon_insensitive": (SquaredEpsilonInsensitive,
-                                        DEFAULT_EPSILON),
+        "squared_epsilon_insensitive": (SquaredEpsilonInsensitive, DEFAULT_EPSILON),
     }
 
-    @abstractmethod
-    @_deprecate_positional_args
-    def __init__(self, loss="hinge", *, penalty='l2', alpha=0.0001,
-                 l1_ratio=0.15, fit_intercept=True, max_iter=1000, tol=1e-3,
-                 shuffle=True, verbose=0, epsilon=DEFAULT_EPSILON, n_jobs=None,
-                 random_state=None, learning_rate="optimal", eta0=0.0,
-                 power_t=0.5, early_stopping=False,
-                 validation_fraction=0.1, n_iter_no_change=5,
-                 class_weight=None, warm_start=False, average=False):
+    _parameter_constraints: dict = {
+        **BaseSGD._parameter_constraints,
+        "loss": [StrOptions(set(loss_functions))],
+        "early_stopping": ["boolean"],
+        "validation_fraction": [Interval(Real, 0, 1, closed="neither")],
+        "n_iter_no_change": [Interval(Integral, 1, None, closed="left")],
+        "n_jobs": [Integral, None],
+        "class_weight": [StrOptions({"balanced"}), dict, None],
+    }
 
+    @abstractmethod
+    def __init__(
+        self,
+        loss="hinge",
+        *,
+        penalty="l2",
+        alpha=0.0001,
+        l1_ratio=0.15,
+        fit_intercept=True,
+        max_iter=1000,
+        tol=1e-3,
+        shuffle=True,
+        verbose=0,
+        epsilon=DEFAULT_EPSILON,
+        n_jobs=None,
+        random_state=None,
+        learning_rate="optimal",
+        eta0=0.0,
+        power_t=0.5,
+        early_stopping=False,
+        validation_fraction=0.1,
+        n_iter_no_change=5,
+        class_weight=None,
+        warm_start=False,
+        average=False,
+    ):
         super().__init__(
-            loss=loss, penalty=penalty, alpha=alpha, l1_ratio=l1_ratio,
-            fit_intercept=fit_intercept, max_iter=max_iter, tol=tol,
-            shuffle=shuffle, verbose=verbose, epsilon=epsilon,
-            random_state=random_state, learning_rate=learning_rate, eta0=eta0,
-            power_t=power_t, early_stopping=early_stopping,
+            loss=loss,
+            penalty=penalty,
+            alpha=alpha,
+            l1_ratio=l1_ratio,
+            fit_intercept=fit_intercept,
+            max_iter=max_iter,
+            tol=tol,
+            shuffle=shuffle,
+            verbose=verbose,
+            epsilon=epsilon,
+            random_state=random_state,
+            learning_rate=learning_rate,
+            eta0=eta0,
+            power_t=power_t,
+            early_stopping=early_stopping,
             validation_fraction=validation_fraction,
-            n_iter_no_change=n_iter_no_change, warm_start=warm_start,
-            average=average)
+            n_iter_no_change=n_iter_no_change,
+            warm_start=warm_start,
+            average=average,
+        )
         self.class_weight = class_weight
         self.n_jobs = n_jobs
 
-    def _partial_fit(self, X, y, alpha, C,
-                     loss, learning_rate, max_iter,
-                     classes, sample_weight,
-                     coef_init, intercept_init):
+    def _partial_fit(
+        self,
+        X,
+        y,
+        alpha,
+        C,
+        loss,
+        learning_rate,
+        max_iter,
+        classes,
+        sample_weight,
+        coef_init,
+        intercept_init,
+    ):
         first_call = not hasattr(self, "classes_")
-        X, y = self._validate_data(X, y, accept_sparse='csr', dtype=np.float64,
-                                   order="C", accept_large_sparse=False,
-                                   reset=first_call)
+        X, y = validate_data(
+            self,
+            X,
+            y,
+            accept_sparse="csr",
+            dtype=[np.float64, np.float32],
+            order="C",
+            accept_large_sparse=False,
+            reset=first_call,
+        )
 
         n_samples, n_features = X.shape
 
@@ -504,50 +614,76 @@ def _partial_fit(self, X, y, alpha, C,
 
         # Allocate datastructures from input arguments
         self._expanded_class_weight = compute_class_weight(
-            self.class_weight, classes=self.classes_, y=y)
-        sample_weight = _check_sample_weight(sample_weight, X)
+            self.class_weight, classes=self.classes_, y=y
+        )
+        sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype)
 
         if getattr(self, "coef_", None) is None or coef_init is not None:
-            self._allocate_parameter_mem(n_classes, n_features,
-                                         coef_init, intercept_init)
+            self._allocate_parameter_mem(
+                n_classes=n_classes,
+                n_features=n_features,
+                input_dtype=X.dtype,
+                coef_init=coef_init,
+                intercept_init=intercept_init,
+            )
         elif n_features != self.coef_.shape[-1]:
-            raise ValueError("Number of features %d does not match previous "
-                             "data %d." % (n_features, self.coef_.shape[-1]))
+            raise ValueError(
+                "Number of features %d does not match previous data %d."
+                % (n_features, self.coef_.shape[-1])
+            )
 
-        self.loss_function_ = self._get_loss_function(loss)
+        self._loss_function_ = self._get_loss_function(loss)
         if not hasattr(self, "t_"):
             self.t_ = 1.0
 
         # delegate to concrete training procedure
         if n_classes > 2:
-            self._fit_multiclass(X, y, alpha=alpha, C=C,
-                                 learning_rate=learning_rate,
-                                 sample_weight=sample_weight,
-                                 max_iter=max_iter)
+            self._fit_multiclass(
+                X,
+                y,
+                alpha=alpha,
+                C=C,
+                learning_rate=learning_rate,
+                sample_weight=sample_weight,
+                max_iter=max_iter,
+            )
         elif n_classes == 2:
-            self._fit_binary(X, y, alpha=alpha, C=C,
-                             learning_rate=learning_rate,
-                             sample_weight=sample_weight,
-                             max_iter=max_iter)
+            self._fit_binary(
+                X,
+                y,
+                alpha=alpha,
+                C=C,
+                learning_rate=learning_rate,
+                sample_weight=sample_weight,
+                max_iter=max_iter,
+            )
         else:
             raise ValueError(
-                "The number of classes has to be greater than one;"
-                " got %d class" % n_classes)
+                "The number of classes has to be greater than one; got %d class"
+                % n_classes
+            )
 
         return self
 
-    def _fit(self, X, y, alpha, C, loss, learning_rate, coef_init=None,
-             intercept_init=None, sample_weight=None):
-        self._validate_params()
+    def _fit(
+        self,
+        X,
+        y,
+        alpha,
+        C,
+        loss,
+        learning_rate,
+        coef_init=None,
+        intercept_init=None,
+        sample_weight=None,
+    ):
         if hasattr(self, "classes_"):
-            self.classes_ = None
-
-        X, y = self._validate_data(X, y, accept_sparse='csr',
-                                   dtype=np.float64, order="C",
-                                   accept_large_sparse=False)
+            # delete the attribute otherwise _partial_fit thinks it's not the first call
+            delattr(self, "classes_")
 
         # labels can be encoded as float, int, or string literals
         # np.unique sorts in asc order; largest class id is positive class
+        y = validate_data(self, y=y)
         classes = np.unique(y)
 
         if self.warm_start and hasattr(self, "coef_"):
@@ -568,26 +704,51 @@ def _fit(self, X, y, alpha, C, loss, learning_rate, coef_init=None,
         # Clear iteration count for multiple call to fit.
         self.t_ = 1.0
 
-        self._partial_fit(X, y, alpha, C, loss, learning_rate, self.max_iter,
-                          classes, sample_weight, coef_init, intercept_init)
-
-        if (self.tol is not None and self.tol > -np.inf
-                and self.n_iter_ == self.max_iter):
-            warnings.warn("Maximum number of iteration reached before "
-                          "convergence. Consider increasing max_iter to "
-                          "improve the fit.",
-                          ConvergenceWarning)
+        self._partial_fit(
+            X,
+            y,
+            alpha,
+            C,
+            loss,
+            learning_rate,
+            self.max_iter,
+            classes,
+            sample_weight,
+            coef_init,
+            intercept_init,
+        )
+
+        if (
+            self.tol is not None
+            and self.tol > -np.inf
+            and self.n_iter_ == self.max_iter
+        ):
+            warnings.warn(
+                (
+                    "Maximum number of iteration reached before "
+                    "convergence. Consider increasing max_iter to "
+                    "improve the fit."
+                ),
+                ConvergenceWarning,
+            )
         return self
 
-    def _fit_binary(self, X, y, alpha, C, sample_weight,
-                    learning_rate, max_iter):
-        """Fit a binary classifier on X and y. """
-        coef, intercept, n_iter_ = fit_binary(self, 1, X, y, alpha, C,
-                                              learning_rate, max_iter,
-                                              self._expanded_class_weight[1],
-                                              self._expanded_class_weight[0],
-                                              sample_weight,
-                                              random_state=self.random_state)
+    def _fit_binary(self, X, y, alpha, C, sample_weight, learning_rate, max_iter):
+        """Fit a binary classifier on X and y."""
+        coef, intercept, n_iter_ = fit_binary(
+            self,
+            1,
+            X,
+            y,
+            alpha,
+            C,
+            learning_rate,
+            max_iter,
+            self._expanded_class_weight[1],
+            self._expanded_class_weight[0],
+            sample_weight,
+            random_state=self.random_state,
+        )
 
         self.t_ += n_iter_ * X.shape[0]
         self.n_iter_ = n_iter_
@@ -606,8 +767,7 @@ def _fit_binary(self, X, y, alpha, C, sample_weight,
             # intercept is a float, need to convert it to an array of length 1
             self.intercept_ = np.atleast_1d(intercept)
 
-    def _fit_multiclass(self, X, y, alpha, C, learning_rate,
-                        sample_weight, max_iter):
+    def _fit_multiclass(self, X, y, alpha, C, learning_rate, sample_weight, max_iter):
         """Fit a multi-class classifier by combining binary classifiers
 
         Each binary classifier predicts one class versus all others. This
@@ -615,7 +775,7 @@ def _fit_multiclass(self, X, y, alpha, C, learning_rate,
         """
         # Precompute the validation split using the multiclass labels
         # to ensure proper balancing of the classes.
-        validation_mask = self._make_validation_split(y)
+        validation_mask = self._make_validation_split(y, sample_mask=sample_weight > 0)
 
         # Use joblib to fit OvA in parallel.
         # Pick the random seed for each job outside of fit_binary to avoid
@@ -623,17 +783,29 @@ def _fit_multiclass(self, X, y, alpha, C, learning_rate,
         # to non-deterministic behavior
         random_state = check_random_state(self.random_state)
         seeds = random_state.randint(MAX_INT, size=len(self.classes_))
-        result = Parallel(n_jobs=self.n_jobs, verbose=self.verbose,
-                          **_joblib_parallel_args(require="sharedmem"))(
-            delayed(fit_binary)(self, i, X, y, alpha, C, learning_rate,
-                                max_iter, self._expanded_class_weight[i],
-                                1., sample_weight,
-                                validation_mask=validation_mask,
-                                random_state=seed)
-            for i, seed in enumerate(seeds))
+        result = Parallel(
+            n_jobs=self.n_jobs, verbose=self.verbose, require="sharedmem"
+        )(
+            delayed(fit_binary)(
+                self,
+                i,
+                X,
+                y,
+                alpha,
+                C,
+                learning_rate,
+                max_iter,
+                self._expanded_class_weight[i],
+                1.0,
+                sample_weight,
+                validation_mask=validation_mask,
+                random_state=seed,
+            )
+            for i, seed in enumerate(seeds)
+        )
 
         # take the maximum of n_iter_ over every binary fit
-        n_iter_ = 0.
+        n_iter_ = 0.0
         for i, (_, intercept, n_iter_i) in enumerate(result):
             self.intercept_[i] = intercept
             n_iter_ = max(n_iter_, n_iter_i)
@@ -650,13 +822,14 @@ def _fit_multiclass(self, X, y, alpha, C, learning_rate,
                 self._standard_intercept = np.atleast_1d(self.intercept_)
                 self.intercept_ = self._standard_intercept
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def partial_fit(self, X, y, classes=None, sample_weight=None):
         """Perform one epoch of stochastic gradient descent on given samples.
 
         Internally, this method uses ``max_iter = 1``. Therefore, it is not
         guaranteed that a minimum of the cost function is reached after calling
-        it once. Matters such as objective convergence and early stopping
-        should be handled by the user.
+        it once. Matters such as objective convergence, early stopping, and
+        learning rate adjustments should be handled by the user.
 
         Parameters
         ----------
@@ -680,27 +853,41 @@ def partial_fit(self, X, y, classes=None, sample_weight=None):
 
         Returns
         -------
-        self :
+        self : object
             Returns an instance of self.
         """
-        self._validate_params(for_partial_fit=True)
-        if self.class_weight in ['balanced']:
-            raise ValueError("class_weight '{0}' is not supported for "
-                             "partial_fit. In order to use 'balanced' weights,"
-                             " use compute_class_weight('{0}', "
-                             "classes=classes, y=y). "
-                             "In place of y you can us a large enough sample "
-                             "of the full training set target to properly "
-                             "estimate the class frequency distributions. "
-                             "Pass the resulting weights as the class_weight "
-                             "parameter.".format(self.class_weight))
-        return self._partial_fit(X, y, alpha=self.alpha, C=1.0, loss=self.loss,
-                                 learning_rate=self.learning_rate, max_iter=1,
-                                 classes=classes, sample_weight=sample_weight,
-                                 coef_init=None, intercept_init=None)
-
-    def fit(self, X, y, coef_init=None, intercept_init=None,
-            sample_weight=None):
+        if not hasattr(self, "classes_"):
+            self._more_validate_params(for_partial_fit=True)
+
+            if self.class_weight == "balanced":
+                raise ValueError(
+                    "class_weight '{0}' is not supported for "
+                    "partial_fit. In order to use 'balanced' weights,"
+                    " use compute_class_weight('{0}', "
+                    "classes=classes, y=y). "
+                    "In place of y you can use a large enough sample "
+                    "of the full training set target to properly "
+                    "estimate the class frequency distributions. "
+                    "Pass the resulting weights as the class_weight "
+                    "parameter.".format(self.class_weight)
+                )
+
+        return self._partial_fit(
+            X,
+            y,
+            alpha=self.alpha,
+            C=1.0,
+            loss=self.loss,
+            learning_rate=self.learning_rate,
+            max_iter=1,
+            classes=classes,
+            sample_weight=sample_weight,
+            coef_init=None,
+            intercept_init=None,
+        )
+
+    @_fit_context(prefer_skip_nested_validation=True)
+    def fit(self, X, y, coef_init=None, intercept_init=None, sample_weight=None):
         """Fit linear model with Stochastic Gradient Descent.
 
         Parameters
@@ -725,13 +912,27 @@ def fit(self, X, y, coef_init=None, intercept_init=None,
 
         Returns
         -------
-        self :
+        self : object
             Returns an instance of self.
         """
-        return self._fit(X, y, alpha=self.alpha, C=1.0,
-                         loss=self.loss, learning_rate=self.learning_rate,
-                         coef_init=coef_init, intercept_init=intercept_init,
-                         sample_weight=sample_weight)
+        self._more_validate_params()
+
+        return self._fit(
+            X,
+            y,
+            alpha=self.alpha,
+            C=1.0,
+            loss=self.loss,
+            learning_rate=self.learning_rate,
+            coef_init=coef_init,
+            intercept_init=intercept_init,
+            sample_weight=sample_weight,
+        )
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
+        return tags
 
 
 class SGDClassifier(BaseSGDClassifier):
@@ -761,42 +962,51 @@ class SGDClassifier(BaseSGDClassifier):
 
     Parameters
     ----------
-    loss : str, default='hinge'
-        The loss function to be used. Defaults to 'hinge', which gives a
-        linear SVM.
-
-        The possible options are 'hinge', 'log', 'modified_huber',
-        'squared_hinge', 'perceptron', or a regression loss: 'squared_loss',
-        'huber', 'epsilon_insensitive', or 'squared_epsilon_insensitive'.
-
-        The 'log' loss gives logistic regression, a probabilistic classifier.
-        'modified_huber' is another smooth loss that brings tolerance to
-        outliers as well as probability estimates.
-        'squared_hinge' is like hinge but is quadratically penalized.
-        'perceptron' is the linear loss used by the perceptron algorithm.
-        The other losses are designed for regression but can be useful in
-        classification as well; see
-        :class:`~sklearn.linear_model.SGDRegressor` for a description.
-
-        More details about the losses formulas can be found in the
-        :ref:`User Guide `.
-
-    penalty : {'l2', 'l1', 'elasticnet'}, default='l2'
+    loss : {'hinge', 'log_loss', 'modified_huber', 'squared_hinge',\
+        'perceptron', 'squared_error', 'huber', 'epsilon_insensitive',\
+        'squared_epsilon_insensitive'}, default='hinge'
+        The loss function to be used.
+
+        - 'hinge' gives a linear SVM.
+        - 'log_loss' gives logistic regression, a probabilistic classifier.
+        - 'modified_huber' is another smooth loss that brings tolerance to
+          outliers as well as probability estimates.
+        - 'squared_hinge' is like hinge but is quadratically penalized.
+        - 'perceptron' is the linear loss used by the perceptron algorithm.
+        - The other losses, 'squared_error', 'huber', 'epsilon_insensitive' and
+          'squared_epsilon_insensitive' are designed for regression but can be useful
+          in classification as well; see
+          :class:`~sklearn.linear_model.SGDRegressor` for a description.
+
+        More details about the losses formulas can be found in the :ref:`User Guide
+        ` and you can find a visualisation of the loss
+        functions in
+        :ref:`sphx_glr_auto_examples_linear_model_plot_sgd_loss_functions.py`.
+
+    penalty : {'l2', 'l1', 'elasticnet', None}, default='l2'
         The penalty (aka regularization term) to be used. Defaults to 'l2'
         which is the standard regularizer for linear SVM models. 'l1' and
         'elasticnet' might bring sparsity to the model (feature selection)
-        not achievable with 'l2'.
+        not achievable with 'l2'. No penalty is added when set to `None`.
+
+        You can see a visualisation of the penalties in
+        :ref:`sphx_glr_auto_examples_linear_model_plot_sgd_penalties.py`.
 
     alpha : float, default=0.0001
         Constant that multiplies the regularization term. The higher the
-        value, the stronger the regularization.
-        Also used to compute the learning rate when set to `learning_rate` is
-        set to 'optimal'.
+        value, the stronger the regularization. Also used to compute the
+        learning rate when `learning_rate` is set to 'optimal'.
+        Values must be in the range `[0.0, inf)`.
 
     l1_ratio : float, default=0.15
         The Elastic Net mixing parameter, with 0 <= l1_ratio <= 1.
         l1_ratio=0 corresponds to L2 penalty, l1_ratio=1 to L1.
         Only used if `penalty` is 'elasticnet'.
+        Values must be in the range `[0.0, 1.0]` or can be `None` if
+        `penalty` is not `elasticnet`.
+
+        .. versionchanged:: 1.7
+            `l1_ratio` can be `None` when `penalty` is not "elasticnet".
 
     fit_intercept : bool, default=True
         Whether the intercept should be estimated or not. If False, the
@@ -806,15 +1016,17 @@ class SGDClassifier(BaseSGDClassifier):
         The maximum number of passes over the training data (aka epochs).
         It only impacts the behavior in the ``fit`` method, and not the
         :meth:`partial_fit` method.
+        Values must be in the range `[1, inf)`.
 
         .. versionadded:: 0.19
 
-    tol : float, default=1e-3
+    tol : float or None, default=1e-3
         The stopping criterion. If it is not None, training will stop
         when (loss > best_loss - tol) for ``n_iter_no_change`` consecutive
         epochs.
         Convergence is checked against the training loss or the
         validation loss depending on the `early_stopping` parameter.
+        Values must be in the range `[0.0, inf)`.
 
         .. versionadded:: 0.19
 
@@ -823,6 +1035,7 @@ class SGDClassifier(BaseSGDClassifier):
 
     verbose : int, default=0
         The verbosity level.
+        Values must be in the range `[0, inf)`.
 
     epsilon : float, default=0.1
         Epsilon in the epsilon-insensitive loss functions; only if `loss` is
@@ -831,6 +1044,7 @@ class SGDClassifier(BaseSGDClassifier):
         important to get the prediction exactly right.
         For epsilon-insensitive, any differences between the current prediction
         and the correct label are ignored if they are less than this threshold.
+        Values must be in the range `[0.0, inf)`.
 
     n_jobs : int, default=None
         The number of CPUs to use to do the OVA (One Versus All, for
@@ -843,37 +1057,43 @@ class SGDClassifier(BaseSGDClassifier):
         Used for shuffling the data, when ``shuffle`` is set to ``True``.
         Pass an int for reproducible output across multiple function calls.
         See :term:`Glossary `.
+        Integer values must be in the range `[0, 2**32 - 1]`.
 
     learning_rate : str, default='optimal'
         The learning rate schedule:
 
         - 'constant': `eta = eta0`
         - 'optimal': `eta = 1.0 / (alpha * (t + t0))`
-          where t0 is chosen by a heuristic proposed by Leon Bottou.
+          where `t0` is chosen by a heuristic proposed by Leon Bottou.
         - 'invscaling': `eta = eta0 / pow(t, power_t)`
-        - 'adaptive': eta = eta0, as long as the training keeps decreasing.
+        - 'adaptive': `eta = eta0`, as long as the training keeps decreasing.
           Each time n_iter_no_change consecutive epochs fail to decrease the
           training loss by tol or fail to increase validation score by tol if
-          early_stopping is True, the current learning rate is divided by 5.
+          `early_stopping` is `True`, the current learning rate is divided by 5.
 
-            .. versionadded:: 0.20
-                Added 'adaptive' option
+        .. versionadded:: 0.20
+            Added 'adaptive' option.
 
-    eta0 : double, default=0.0
+    eta0 : float, default=0.0
         The initial learning rate for the 'constant', 'invscaling' or
         'adaptive' schedules. The default value is 0.0 as eta0 is not used by
         the default schedule 'optimal'.
+        Values must be in the range `[0.0, inf)`.
 
-    power_t : double, default=0.5
-        The exponent for inverse scaling learning rate [default 0.5].
+    power_t : float, default=0.5
+        The exponent for inverse scaling learning rate.
+        Values must be in the range `(-inf, inf)`.
 
     early_stopping : bool, default=False
         Whether to use early stopping to terminate training when validation
-        score is not improving. If set to True, it will automatically set aside
+        score is not improving. If set to `True`, it will automatically set aside
         a stratified fraction of training data as validation and terminate
         training when validation score returned by the `score` method is not
         improving by at least tol for n_iter_no_change consecutive epochs.
 
+        See :ref:`sphx_glr_auto_examples_linear_model_plot_sgd_early_stopping.py` for an
+        example of the effects of early stopping.
+
         .. versionadded:: 0.20
             Added 'early_stopping' option
 
@@ -881,6 +1101,7 @@ class SGDClassifier(BaseSGDClassifier):
         The proportion of training data to set aside as validation set for
         early stopping. Must be between 0 and 1.
         Only used if `early_stopping` is True.
+        Values must be in the range `(0.0, 1.0)`.
 
         .. versionadded:: 0.20
             Added 'validation_fraction' option
@@ -890,6 +1111,7 @@ class SGDClassifier(BaseSGDClassifier):
         fitting.
         Convergence is checked against the training loss or the
         validation loss depending on the `early_stopping` parameter.
+        Integer values must be in the range `[1, max_iter)`.
 
         .. versionadded:: 0.20
             Added 'n_iter_no_change' option
@@ -918,11 +1140,12 @@ class SGDClassifier(BaseSGDClassifier):
         existing counter.
 
     average : bool or int, default=False
-        When set to True, computes the averaged SGD weights accross all
+        When set to `True`, computes the averaged SGD weights across all
         updates and stores the result in the ``coef_`` attribute. If set to
         an int greater than 1, averaging will begin once the total number of
         samples seen reaches `average`. So ``average=10`` will begin
         averaging after seeing 10 samples.
+        Integer values must be in the range `[1, n_samples]`.
 
     Attributes
     ----------
@@ -937,13 +1160,22 @@ class SGDClassifier(BaseSGDClassifier):
         The actual number of iterations before reaching the stopping criterion.
         For multiclass fits, it is the maximum over every binary fit.
 
-    loss_function_ : concrete ``LossFunction``
-
     classes_ : array of shape (n_classes,)
 
     t_ : int
         Number of weight updates performed during training.
-        Same as ``(n_iter_ * n_samples)``.
+        Same as ``(n_iter_ * n_samples + 1)``.
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
 
     See Also
     --------
@@ -970,32 +1202,79 @@ class SGDClassifier(BaseSGDClassifier):
     >>> print(clf.predict([[-0.8, -1]]))
     [1]
     """
-    @_deprecate_positional_args
-    def __init__(self, loss="hinge", *, penalty='l2', alpha=0.0001,
-                 l1_ratio=0.15,
-                 fit_intercept=True, max_iter=1000, tol=1e-3, shuffle=True,
-                 verbose=0, epsilon=DEFAULT_EPSILON, n_jobs=None,
-                 random_state=None, learning_rate="optimal", eta0=0.0,
-                 power_t=0.5, early_stopping=False, validation_fraction=0.1,
-                 n_iter_no_change=5, class_weight=None, warm_start=False,
-                 average=False):
+
+    _parameter_constraints: dict = {
+        **BaseSGDClassifier._parameter_constraints,
+        "penalty": [StrOptions({"l2", "l1", "elasticnet"}), None],
+        "alpha": [Interval(Real, 0, None, closed="left")],
+        "l1_ratio": [Interval(Real, 0, 1, closed="both"), None],
+        "power_t": [Interval(Real, None, None, closed="neither")],
+        "epsilon": [Interval(Real, 0, None, closed="left")],
+        "learning_rate": [
+            StrOptions({"constant", "optimal", "invscaling", "adaptive"}),
+            Hidden(StrOptions({"pa1", "pa2"})),
+        ],
+        "eta0": [Interval(Real, 0, None, closed="left")],
+    }
+
+    def __init__(
+        self,
+        loss="hinge",
+        *,
+        penalty="l2",
+        alpha=0.0001,
+        l1_ratio=0.15,
+        fit_intercept=True,
+        max_iter=1000,
+        tol=1e-3,
+        shuffle=True,
+        verbose=0,
+        epsilon=DEFAULT_EPSILON,
+        n_jobs=None,
+        random_state=None,
+        learning_rate="optimal",
+        eta0=0.0,
+        power_t=0.5,
+        early_stopping=False,
+        validation_fraction=0.1,
+        n_iter_no_change=5,
+        class_weight=None,
+        warm_start=False,
+        average=False,
+    ):
         super().__init__(
-            loss=loss, penalty=penalty, alpha=alpha, l1_ratio=l1_ratio,
-            fit_intercept=fit_intercept, max_iter=max_iter, tol=tol,
-            shuffle=shuffle, verbose=verbose, epsilon=epsilon, n_jobs=n_jobs,
-            random_state=random_state, learning_rate=learning_rate, eta0=eta0,
-            power_t=power_t, early_stopping=early_stopping,
+            loss=loss,
+            penalty=penalty,
+            alpha=alpha,
+            l1_ratio=l1_ratio,
+            fit_intercept=fit_intercept,
+            max_iter=max_iter,
+            tol=tol,
+            shuffle=shuffle,
+            verbose=verbose,
+            epsilon=epsilon,
+            n_jobs=n_jobs,
+            random_state=random_state,
+            learning_rate=learning_rate,
+            eta0=eta0,
+            power_t=power_t,
+            early_stopping=early_stopping,
             validation_fraction=validation_fraction,
-            n_iter_no_change=n_iter_no_change, class_weight=class_weight,
-            warm_start=warm_start, average=average)
+            n_iter_no_change=n_iter_no_change,
+            class_weight=class_weight,
+            warm_start=warm_start,
+            average=average,
+        )
 
     def _check_proba(self):
-        if self.loss not in ("log", "modified_huber"):
-            raise AttributeError("probability estimates are not available for"
-                                 " loss=%r" % self.loss)
-
-    @property
-    def predict_proba(self):
+        if self.loss not in ("log_loss", "modified_huber"):
+            raise AttributeError(
+                "probability estimates are not available for loss=%r" % self.loss
+            )
+        return True
+
+    @available_if(_check_proba)
+    def predict_proba(self, X):
         """Probability estimates.
 
         This method is only available for log loss and modified Huber loss.
@@ -1025,23 +1304,19 @@ def predict_proba(self):
         ----------
         Zadrozny and Elkan, "Transforming classifier scores into multiclass
         probability estimates", SIGKDD'02,
-        http://www.research.ibm.com/people/z/zadrozny/kdd2002-Transf.pdf
+        https://dl.acm.org/doi/pdf/10.1145/775047.775151
 
         The justification for the formula in the loss="modified_huber"
         case is in the appendix B in:
         http://jmlr.csail.mit.edu/papers/volume2/zhang02c/zhang02c.pdf
         """
-        self._check_proba()
-        return self._predict_proba
-
-    def _predict_proba(self, X):
         check_is_fitted(self)
 
-        if self.loss == "log":
+        if self.loss == "log_loss":
             return self._predict_proba_lr(X)
 
         elif self.loss == "modified_huber":
-            binary = (len(self.classes_) == 2)
+            binary = len(self.classes_) == 2
             scores = self.decision_function(X)
 
             if binary:
@@ -1051,8 +1326,8 @@ def _predict_proba(self, X):
                 prob = scores
 
             np.clip(scores, -1, 1, prob)
-            prob += 1.
-            prob /= 2.
+            prob += 1.0
+            prob /= 2.0
 
             if binary:
                 prob2[:, 0] -= prob
@@ -1062,7 +1337,7 @@ def _predict_proba(self, X):
                 # normalize neatly; work around this to produce uniform
                 # probabilities
                 prob_sum = prob.sum(axis=1)
-                all_zero = (prob_sum == 0)
+                all_zero = prob_sum == 0
                 if np.any(all_zero):
                     prob[all_zero, :] = 1
                     prob_sum[all_zero] = len(self.classes_)
@@ -1073,12 +1348,14 @@ def _predict_proba(self, X):
             return prob
 
         else:
-            raise NotImplementedError("predict_(log_)proba only supported when"
-                                      " loss='log' or loss='modified_huber' "
-                                      "(%r given)" % self.loss)
-
-    @property
-    def predict_log_proba(self):
+            raise NotImplementedError(
+                "predict_(log_)proba only supported when"
+                " loss='log_loss' or loss='modified_huber' "
+                "(%r given)" % self.loss
+            )
+
+    @available_if(_check_proba)
+    def predict_log_proba(self, X):
         """Log of probability estimates.
 
         This method is only available for log loss and modified Huber loss.
@@ -1100,76 +1377,122 @@ def predict_log_proba(self):
             model, where classes are ordered as they are in
             `self.classes_`.
         """
-        self._check_proba()
-        return self._predict_log_proba
-
-    def _predict_log_proba(self, X):
         return np.log(self.predict_proba(X))
 
-    def _more_tags(self):
-        return {
-            '_xfail_checks': {
-                'check_sample_weights_invariance':
-                'zero sample_weight is not equivalent to removing samples',
-            }
-        }
-
 
 class BaseSGDRegressor(RegressorMixin, BaseSGD):
-
     loss_functions = {
-        "squared_loss": (SquaredLoss, ),
-        "huber": (Huber, DEFAULT_EPSILON),
+        "squared_error": (CyHalfSquaredError,),
+        "huber": (CyHuberLoss, DEFAULT_EPSILON),
         "epsilon_insensitive": (EpsilonInsensitive, DEFAULT_EPSILON),
-        "squared_epsilon_insensitive": (SquaredEpsilonInsensitive,
-                                        DEFAULT_EPSILON),
+        "squared_epsilon_insensitive": (SquaredEpsilonInsensitive, DEFAULT_EPSILON),
+    }
+
+    _parameter_constraints: dict = {
+        **BaseSGD._parameter_constraints,
+        "loss": [StrOptions(set(loss_functions))],
+        "early_stopping": ["boolean"],
+        "validation_fraction": [Interval(Real, 0, 1, closed="neither")],
+        "n_iter_no_change": [Interval(Integral, 1, None, closed="left")],
     }
 
     @abstractmethod
-    @_deprecate_positional_args
-    def __init__(self, loss="squared_loss", *, penalty="l2", alpha=0.0001,
-                 l1_ratio=0.15, fit_intercept=True, max_iter=1000, tol=1e-3,
-                 shuffle=True, verbose=0, epsilon=DEFAULT_EPSILON,
-                 random_state=None, learning_rate="invscaling", eta0=0.01,
-                 power_t=0.25, early_stopping=False, validation_fraction=0.1,
-                 n_iter_no_change=5, warm_start=False, average=False):
+    def __init__(
+        self,
+        loss="squared_error",
+        *,
+        penalty="l2",
+        alpha=0.0001,
+        l1_ratio=0.15,
+        fit_intercept=True,
+        max_iter=1000,
+        tol=1e-3,
+        shuffle=True,
+        verbose=0,
+        epsilon=DEFAULT_EPSILON,
+        random_state=None,
+        learning_rate="invscaling",
+        eta0=0.01,
+        power_t=0.25,
+        early_stopping=False,
+        validation_fraction=0.1,
+        n_iter_no_change=5,
+        warm_start=False,
+        average=False,
+    ):
         super().__init__(
-            loss=loss, penalty=penalty, alpha=alpha, l1_ratio=l1_ratio,
-            fit_intercept=fit_intercept, max_iter=max_iter, tol=tol,
-            shuffle=shuffle, verbose=verbose, epsilon=epsilon,
-            random_state=random_state, learning_rate=learning_rate, eta0=eta0,
-            power_t=power_t, early_stopping=early_stopping,
+            loss=loss,
+            penalty=penalty,
+            alpha=alpha,
+            l1_ratio=l1_ratio,
+            fit_intercept=fit_intercept,
+            max_iter=max_iter,
+            tol=tol,
+            shuffle=shuffle,
+            verbose=verbose,
+            epsilon=epsilon,
+            random_state=random_state,
+            learning_rate=learning_rate,
+            eta0=eta0,
+            power_t=power_t,
+            early_stopping=early_stopping,
             validation_fraction=validation_fraction,
-            n_iter_no_change=n_iter_no_change, warm_start=warm_start,
-            average=average)
-
-    def _partial_fit(self, X, y, alpha, C, loss, learning_rate,
-                     max_iter, sample_weight, coef_init, intercept_init):
+            n_iter_no_change=n_iter_no_change,
+            warm_start=warm_start,
+            average=average,
+        )
+
+    def _partial_fit(
+        self,
+        X,
+        y,
+        alpha,
+        C,
+        loss,
+        learning_rate,
+        max_iter,
+        sample_weight,
+        coef_init,
+        intercept_init,
+    ):
         first_call = getattr(self, "coef_", None) is None
-        X, y = self._validate_data(X, y, accept_sparse="csr", copy=False,
-                                   order='C', dtype=np.float64,
-                                   accept_large_sparse=False, reset=first_call)
-        y = y.astype(np.float64, copy=False)
+        X, y = validate_data(
+            self,
+            X,
+            y,
+            accept_sparse="csr",
+            copy=False,
+            order="C",
+            dtype=[np.float64, np.float32],
+            accept_large_sparse=False,
+            reset=first_call,
+        )
+        y = y.astype(X.dtype, copy=False)
 
         n_samples, n_features = X.shape
 
-        sample_weight = _check_sample_weight(sample_weight, X)
+        sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype)
 
         # Allocate datastructures from input arguments
         if first_call:
-            self._allocate_parameter_mem(1, n_features, coef_init,
-                                         intercept_init)
+            self._allocate_parameter_mem(
+                n_classes=1,
+                n_features=n_features,
+                input_dtype=X.dtype,
+                coef_init=coef_init,
+                intercept_init=intercept_init,
+            )
         if self.average > 0 and getattr(self, "_average_coef", None) is None:
-            self._average_coef = np.zeros(n_features,
-                                          dtype=np.float64,
-                                          order="C")
-            self._average_intercept = np.zeros(1, dtype=np.float64, order="C")
+            self._average_coef = np.zeros(n_features, dtype=X.dtype, order="C")
+            self._average_intercept = np.zeros(1, dtype=X.dtype, order="C")
 
-        self._fit_regressor(X, y, alpha, C, loss, learning_rate,
-                            sample_weight, max_iter)
+        self._fit_regressor(
+            X, y, alpha, C, loss, learning_rate, sample_weight, max_iter
+        )
 
         return self
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def partial_fit(self, X, y, sample_weight=None):
         """Perform one epoch of stochastic gradient descent on given samples.
 
@@ -1181,10 +1504,10 @@ def partial_fit(self, X, y, sample_weight=None):
         Parameters
         ----------
         X : {array-like, sparse matrix}, shape (n_samples, n_features)
-            Subset of training data
+            Subset of training data.
 
         y : numpy array of shape (n_samples,)
-            Subset of target values
+            Subset of target values.
 
         sample_weight : array-like, shape (n_samples,), default=None
             Weights applied to individual samples.
@@ -1192,18 +1515,37 @@ def partial_fit(self, X, y, sample_weight=None):
 
         Returns
         -------
-        self : returns an instance of self.
+        self : object
+            Returns an instance of self.
         """
-        self._validate_params(for_partial_fit=True)
-        return self._partial_fit(X, y, self.alpha, C=1.0,
-                                 loss=self.loss,
-                                 learning_rate=self.learning_rate, max_iter=1,
-                                 sample_weight=sample_weight, coef_init=None,
-                                 intercept_init=None)
-
-    def _fit(self, X, y, alpha, C, loss, learning_rate, coef_init=None,
-             intercept_init=None, sample_weight=None):
-        self._validate_params()
+        if not hasattr(self, "coef_"):
+            self._more_validate_params(for_partial_fit=True)
+
+        return self._partial_fit(
+            X,
+            y,
+            self.alpha,
+            C=1.0,
+            loss=self.loss,
+            learning_rate=self.learning_rate,
+            max_iter=1,
+            sample_weight=sample_weight,
+            coef_init=None,
+            intercept_init=None,
+        )
+
+    def _fit(
+        self,
+        X,
+        y,
+        alpha,
+        C,
+        loss,
+        learning_rate,
+        coef_init=None,
+        intercept_init=None,
+        sample_weight=None,
+    ):
         if self.warm_start and getattr(self, "coef_", None) is not None:
             if coef_init is None:
                 coef_init = self.coef_
@@ -1216,30 +1558,46 @@ def _fit(self, X, y, alpha, C, loss, learning_rate, coef_init=None,
         # Clear iteration count for multiple call to fit.
         self.t_ = 1.0
 
-        self._partial_fit(X, y, alpha, C, loss, learning_rate,
-                          self.max_iter, sample_weight, coef_init,
-                          intercept_init)
-
-        if (self.tol is not None and self.tol > -np.inf
-                and self.n_iter_ == self.max_iter):
-            warnings.warn("Maximum number of iteration reached before "
-                          "convergence. Consider increasing max_iter to "
-                          "improve the fit.",
-                          ConvergenceWarning)
+        self._partial_fit(
+            X,
+            y,
+            alpha,
+            C,
+            loss,
+            learning_rate,
+            self.max_iter,
+            sample_weight,
+            coef_init,
+            intercept_init,
+        )
+
+        if (
+            self.tol is not None
+            and self.tol > -np.inf
+            and self.n_iter_ == self.max_iter
+        ):
+            warnings.warn(
+                (
+                    "Maximum number of iteration reached before "
+                    "convergence. Consider increasing max_iter to "
+                    "improve the fit."
+                ),
+                ConvergenceWarning,
+            )
 
         return self
 
-    def fit(self, X, y, coef_init=None, intercept_init=None,
-            sample_weight=None):
+    @_fit_context(prefer_skip_nested_validation=True)
+    def fit(self, X, y, coef_init=None, intercept_init=None, sample_weight=None):
         """Fit linear model with Stochastic Gradient Descent.
 
         Parameters
         ----------
         X : {array-like, sparse matrix}, shape (n_samples, n_features)
-            Training data
+            Training data.
 
         y : ndarray of shape (n_samples,)
-            Target values
+            Target values.
 
         coef_init : ndarray of shape (n_features,), default=None
             The initial coefficients to warm-start the optimization.
@@ -1252,13 +1610,22 @@ def fit(self, X, y, coef_init=None, intercept_init=None,
 
         Returns
         -------
-        self : returns an instance of self.
+        self : object
+            Fitted `SGDRegressor` estimator.
         """
-        return self._fit(X, y, alpha=self.alpha, C=1.0,
-                         loss=self.loss, learning_rate=self.learning_rate,
-                         coef_init=coef_init,
-                         intercept_init=intercept_init,
-                         sample_weight=sample_weight)
+        self._more_validate_params()
+
+        return self._fit(
+            X,
+            y,
+            alpha=self.alpha,
+            C=1.0,
+            loss=self.loss,
+            learning_rate=self.learning_rate,
+            coef_init=coef_init,
+            intercept_init=intercept_init,
+            sample_weight=sample_weight,
+        )
 
     def _decision_function(self, X):
         """Predict using the linear model
@@ -1274,18 +1641,18 @@ def _decision_function(self, X):
         """
         check_is_fitted(self)
 
-        X = self._validate_data(X, accept_sparse='csr', reset=False)
+        X = validate_data(self, X, accept_sparse="csr", reset=False)
 
-        scores = safe_sparse_dot(X, self.coef_.T,
-                                 dense_output=True) + self.intercept_
+        scores = safe_sparse_dot(X, self.coef_.T, dense_output=True) + self.intercept_
         return scores.ravel()
 
     def predict(self, X):
-        """Predict using the linear model
+        """Predict using the linear model.
 
         Parameters
         ----------
         X : {array-like, sparse matrix}, shape (n_samples, n_features)
+            Input data.
 
         Returns
         -------
@@ -1294,10 +1661,9 @@ def predict(self, X):
         """
         return self._decision_function(X)
 
-    def _fit_regressor(self, X, y, alpha, C, loss, learning_rate,
-                       sample_weight, max_iter):
-        dataset, intercept_decay = make_dataset(X, y, sample_weight)
-
+    def _fit_regressor(
+        self, X, y, alpha, C, loss, learning_rate, sample_weight, max_iter
+    ):
         loss_function = self._get_loss_function(loss)
         penalty_type = self._get_penalty_type(self.penalty)
         learning_rate_type = self._get_learning_rate_type(learning_rate)
@@ -1305,14 +1671,19 @@ def _fit_regressor(self, X, y, alpha, C, loss, learning_rate,
         if not hasattr(self, "t_"):
             self.t_ = 1.0
 
-        validation_mask = self._make_validation_split(y)
+        validation_mask = self._make_validation_split(y, sample_mask=sample_weight > 0)
         validation_score_cb = self._make_validation_score_cb(
-            validation_mask, X, y, sample_weight)
+            validation_mask, X, y, sample_weight
+        )
 
         random_state = check_random_state(self.random_state)
         # numpy mtrand expects a C long which is a signed 32 bit integer under
         # Windows
-        seed = random_state.randint(0, np.iinfo(np.int32).max)
+        seed = random_state.randint(0, MAX_INT)
+
+        dataset, intercept_decay = make_dataset(
+            X, y, sample_weight, random_state=random_state
+        )
 
         tol = self.tol if self.tol is not None else -np.inf
 
@@ -1327,28 +1698,38 @@ def _fit_regressor(self, X, y, alpha, C, loss, learning_rate,
             average_coef = None  # Not used
             average_intercept = [0]  # Not used
 
-        coef, intercept, average_coef, average_intercept, self.n_iter_ = \
-            _plain_sgd(coef,
-                       intercept[0],
-                       average_coef,
-                       average_intercept[0],
-                       loss_function,
-                       penalty_type,
-                       alpha, C,
-                       self.l1_ratio,
-                       dataset,
-                       validation_mask, self.early_stopping,
-                       validation_score_cb,
-                       int(self.n_iter_no_change),
-                       max_iter, tol,
-                       int(self.fit_intercept),
-                       int(self.verbose),
-                       int(self.shuffle),
-                       seed,
-                       1.0, 1.0,
-                       learning_rate_type,
-                       self.eta0, self.power_t, self.t_,
-                       intercept_decay, self.average)
+        _plain_sgd = _get_plain_sgd_function(input_dtype=coef.dtype)
+        coef, intercept, average_coef, average_intercept, self.n_iter_ = _plain_sgd(
+            coef,
+            intercept[0],
+            average_coef,
+            average_intercept[0],
+            loss_function,
+            penalty_type,
+            alpha,
+            C,
+            self._get_l1_ratio(),
+            dataset,
+            validation_mask,
+            self.early_stopping,
+            validation_score_cb,
+            int(self.n_iter_no_change),
+            max_iter,
+            tol,
+            int(self.fit_intercept),
+            int(self.verbose),
+            int(self.shuffle),
+            seed,
+            1.0,
+            1.0,
+            learning_rate_type,
+            self.eta0,
+            self.power_t,
+            0,
+            self.t_,
+            intercept_decay,
+            self.average,
+        )
 
         self.t_ += self.n_iter_ * X.shape[0]
 
@@ -1367,9 +1748,14 @@ def _fit_regressor(self, X, y, alpha, C, loss, learning_rate,
         else:
             self.intercept_ = np.atleast_1d(intercept)
 
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
+        return tags
+
 
 class SGDRegressor(BaseSGDRegressor):
-    """Linear model fitted by minimizing a regularized empirical loss with SGD
+    """Linear model fitted by minimizing a regularized empirical loss with SGD.
 
     SGD stands for Stochastic Gradient Descent: the gradient of the loss is
     estimated each sample at a time and the model is updated along the way with
@@ -1389,12 +1775,12 @@ class SGDRegressor(BaseSGDRegressor):
 
     Parameters
     ----------
-    loss : str, default='squared_loss'
-        The loss function to be used. The possible values are 'squared_loss',
+    loss : str, default='squared_error'
+        The loss function to be used. The possible values are 'squared_error',
         'huber', 'epsilon_insensitive', or 'squared_epsilon_insensitive'
 
-        The 'squared_loss' refers to the ordinary least squares fit.
-        'huber' modifies 'squared_loss' to focus less on getting outliers
+        The 'squared_error' refers to the ordinary least squares fit.
+        'huber' modifies 'squared_error' to focus less on getting outliers
         correct by switching from squared to linear loss past a distance of
         epsilon. 'epsilon_insensitive' ignores errors less than epsilon and is
         linear past that; this is the loss function used in SVR.
@@ -1404,22 +1790,30 @@ class SGDRegressor(BaseSGDRegressor):
         More details about the losses formulas can be found in the
         :ref:`User Guide `.
 
-    penalty : {'l2', 'l1', 'elasticnet'}, default='l2'
+    penalty : {'l2', 'l1', 'elasticnet', None}, default='l2'
         The penalty (aka regularization term) to be used. Defaults to 'l2'
         which is the standard regularizer for linear SVM models. 'l1' and
         'elasticnet' might bring sparsity to the model (feature selection)
-        not achievable with 'l2'.
+        not achievable with 'l2'. No penalty is added when set to `None`.
+
+        You can see a visualisation of the penalties in
+        :ref:`sphx_glr_auto_examples_linear_model_plot_sgd_penalties.py`.
 
     alpha : float, default=0.0001
         Constant that multiplies the regularization term. The higher the
-        value, the stronger the regularization.
-        Also used to compute the learning rate when set to `learning_rate` is
-        set to 'optimal'.
+        value, the stronger the regularization. Also used to compute the
+        learning rate when `learning_rate` is set to 'optimal'.
+        Values must be in the range `[0.0, inf)`.
 
     l1_ratio : float, default=0.15
         The Elastic Net mixing parameter, with 0 <= l1_ratio <= 1.
         l1_ratio=0 corresponds to L2 penalty, l1_ratio=1 to L1.
         Only used if `penalty` is 'elasticnet'.
+        Values must be in the range `[0.0, 1.0]` or can be `None` if
+        `penalty` is not `elasticnet`.
+
+        .. versionchanged:: 1.7
+            `l1_ratio` can be `None` when `penalty` is not "elasticnet".
 
     fit_intercept : bool, default=True
         Whether the intercept should be estimated or not. If False, the
@@ -1429,15 +1823,17 @@ class SGDRegressor(BaseSGDRegressor):
         The maximum number of passes over the training data (aka epochs).
         It only impacts the behavior in the ``fit`` method, and not the
         :meth:`partial_fit` method.
+        Values must be in the range `[1, inf)`.
 
         .. versionadded:: 0.19
 
-    tol : float, default=1e-3
+    tol : float or None, default=1e-3
         The stopping criterion. If it is not None, training will stop
         when (loss > best_loss - tol) for ``n_iter_no_change`` consecutive
         epochs.
         Convergence is checked against the training loss or the
         validation loss depending on the `early_stopping` parameter.
+        Values must be in the range `[0.0, inf)`.
 
         .. versionadded:: 0.19
 
@@ -1446,6 +1842,7 @@ class SGDRegressor(BaseSGDRegressor):
 
     verbose : int, default=0
         The verbosity level.
+        Values must be in the range `[0, inf)`.
 
     epsilon : float, default=0.1
         Epsilon in the epsilon-insensitive loss functions; only if `loss` is
@@ -1454,13 +1851,14 @@ class SGDRegressor(BaseSGDRegressor):
         important to get the prediction exactly right.
         For epsilon-insensitive, any differences between the current prediction
         and the correct label are ignored if they are less than this threshold.
+        Values must be in the range `[0.0, inf)`.
 
     random_state : int, RandomState instance, default=None
         Used for shuffling the data, when ``shuffle`` is set to ``True``.
         Pass an int for reproducible output across multiple function calls.
         See :term:`Glossary `.
 
-    learning_rate : string, default='invscaling'
+    learning_rate : str, default='invscaling'
         The learning rate schedule:
 
         - 'constant': `eta = eta0`
@@ -1472,15 +1870,17 @@ class SGDRegressor(BaseSGDRegressor):
           training loss by tol or fail to increase validation score by tol if
           early_stopping is True, the current learning rate is divided by 5.
 
-            .. versionadded:: 0.20
-                Added 'adaptive' option
+        .. versionadded:: 0.20
+            Added 'adaptive' option.
 
-    eta0 : double, default=0.01
+    eta0 : float, default=0.01
         The initial learning rate for the 'constant', 'invscaling' or
         'adaptive' schedules. The default value is 0.01.
+        Values must be in the range `[0.0, inf)`.
 
-    power_t : double, default=0.25
+    power_t : float, default=0.25
         The exponent for inverse scaling learning rate.
+        Values must be in the range `(-inf, inf)`.
 
     early_stopping : bool, default=False
         Whether to use early stopping to terminate training when validation
@@ -1490,6 +1890,9 @@ class SGDRegressor(BaseSGDRegressor):
         improving by at least `tol` for `n_iter_no_change` consecutive
         epochs.
 
+        See :ref:`sphx_glr_auto_examples_linear_model_plot_sgd_early_stopping.py` for an
+        example of the effects of early stopping.
+
         .. versionadded:: 0.20
             Added 'early_stopping' option
 
@@ -1497,6 +1900,7 @@ class SGDRegressor(BaseSGDRegressor):
         The proportion of training data to set aside as validation set for
         early stopping. Must be between 0 and 1.
         Only used if `early_stopping` is True.
+        Values must be in the range `(0.0, 1.0)`.
 
         .. versionadded:: 0.20
             Added 'validation_fraction' option
@@ -1506,6 +1910,7 @@ class SGDRegressor(BaseSGDRegressor):
         fitting.
         Convergence is checked against the training loss or the
         validation loss depending on the `early_stopping` parameter.
+        Integer values must be in the range `[1, max_iter)`.
 
         .. versionadded:: 0.20
             Added 'n_iter_no_change' option
@@ -1524,7 +1929,7 @@ class SGDRegressor(BaseSGDRegressor):
         existing counter.
 
     average : bool or int, default=False
-        When set to True, computes the averaged SGD weights accross all
+        When set to True, computes the averaged SGD weights across all
         updates and stores the result in the ``coef_`` attribute. If set to
         an int greater than 1, averaging will begin once the total number of
         samples seen reaches `average`. So ``average=10`` will begin
@@ -1538,27 +1943,33 @@ class SGDRegressor(BaseSGDRegressor):
     intercept_ : ndarray of shape (1,)
         The intercept term.
 
-    average_coef_ : ndarray of shape (n_features,)
-        Averaged weights assigned to the features. Only available
-        if ``average=True``.
-
-        .. deprecated:: 0.23
-            Attribute ``average_coef_`` was deprecated
-            in version 0.23 and will be removed in 1.0 (renaming of 0.25).
-
-    average_intercept_ : ndarray of shape (1,)
-        The averaged intercept term. Only available if ``average=True``.
-
-        .. deprecated:: 0.23
-            Attribute ``average_intercept_`` was deprecated
-            in version 0.23 and will be removed in 1.0 (renaming of 0.25).
-
     n_iter_ : int
         The actual number of iterations before reaching the stopping criterion.
 
     t_ : int
         Number of weight updates performed during training.
-        Same as ``(n_iter_ * n_samples)``.
+        Same as ``(n_iter_ * n_samples + 1)``.
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    HuberRegressor : Linear regression model that is robust to outliers.
+    Lars : Least Angle Regression model.
+    Lasso : Linear Model trained with L1 prior as regularizer.
+    RANSACRegressor : RANSAC (RANdom SAmple Consensus) algorithm.
+    Ridge : Linear least squares with l2 regularization.
+    sklearn.svm.SVR : Epsilon-Support Vector Regression.
+    TheilSenRegressor : Theil-Sen Estimator robust multivariate regression model.
 
     Examples
     --------
@@ -1576,33 +1987,618 @@ class SGDRegressor(BaseSGDRegressor):
     >>> reg.fit(X, y)
     Pipeline(steps=[('standardscaler', StandardScaler()),
                     ('sgdregressor', SGDRegressor())])
+    """
+
+    _parameter_constraints: dict = {
+        **BaseSGDRegressor._parameter_constraints,
+        "penalty": [StrOptions({"l2", "l1", "elasticnet"}), None],
+        "alpha": [Interval(Real, 0, None, closed="left")],
+        "l1_ratio": [Interval(Real, 0, 1, closed="both"), None],
+        "power_t": [Interval(Real, None, None, closed="neither")],
+        "learning_rate": [
+            StrOptions({"constant", "optimal", "invscaling", "adaptive"}),
+            Hidden(StrOptions({"pa1", "pa2"})),
+        ],
+        "epsilon": [Interval(Real, 0, None, closed="left")],
+        "eta0": [Interval(Real, 0, None, closed="left")],
+    }
+
+    def __init__(
+        self,
+        loss="squared_error",
+        *,
+        penalty="l2",
+        alpha=0.0001,
+        l1_ratio=0.15,
+        fit_intercept=True,
+        max_iter=1000,
+        tol=1e-3,
+        shuffle=True,
+        verbose=0,
+        epsilon=DEFAULT_EPSILON,
+        random_state=None,
+        learning_rate="invscaling",
+        eta0=0.01,
+        power_t=0.25,
+        early_stopping=False,
+        validation_fraction=0.1,
+        n_iter_no_change=5,
+        warm_start=False,
+        average=False,
+    ):
+        super().__init__(
+            loss=loss,
+            penalty=penalty,
+            alpha=alpha,
+            l1_ratio=l1_ratio,
+            fit_intercept=fit_intercept,
+            max_iter=max_iter,
+            tol=tol,
+            shuffle=shuffle,
+            verbose=verbose,
+            epsilon=epsilon,
+            random_state=random_state,
+            learning_rate=learning_rate,
+            eta0=eta0,
+            power_t=power_t,
+            early_stopping=early_stopping,
+            validation_fraction=validation_fraction,
+            n_iter_no_change=n_iter_no_change,
+            warm_start=warm_start,
+            average=average,
+        )
+
+
+class SGDOneClassSVM(OutlierMixin, BaseSGD):
+    """Solves linear One-Class SVM using Stochastic Gradient Descent.
+
+    This implementation is meant to be used with a kernel approximation
+    technique (e.g. `sklearn.kernel_approximation.Nystroem`) to obtain results
+    similar to `sklearn.svm.OneClassSVM` which uses a Gaussian kernel by
+    default.
+
+    Read more in the :ref:`User Guide `.
+
+    .. versionadded:: 1.0
+
+    Parameters
+    ----------
+    nu : float, default=0.5
+        The nu parameter of the One Class SVM: an upper bound on the
+        fraction of training errors and a lower bound of the fraction of
+        support vectors. Should be in the interval (0, 1]. By default 0.5
+        will be taken.
+
+    fit_intercept : bool, default=True
+        Whether the intercept should be estimated or not. Defaults to True.
+
+    max_iter : int, default=1000
+        The maximum number of passes over the training data (aka epochs).
+        It only impacts the behavior in the ``fit`` method, and not the
+        `partial_fit`. Defaults to 1000.
+        Values must be in the range `[1, inf)`.
+
+    tol : float or None, default=1e-3
+        The stopping criterion. If it is not None, the iterations will stop
+        when (loss > previous_loss - tol). Defaults to 1e-3.
+        Values must be in the range `[0.0, inf)`.
+
+    shuffle : bool, default=True
+        Whether or not the training data should be shuffled after each epoch.
+        Defaults to True.
+
+    verbose : int, default=0
+        The verbosity level.
+
+    random_state : int, RandomState instance or None, default=None
+        The seed of the pseudo random number generator to use when shuffling
+        the data.  If int, random_state is the seed used by the random number
+        generator; If RandomState instance, random_state is the random number
+        generator; If None, the random number generator is the RandomState
+        instance used by `np.random`.
+
+    learning_rate : {'constant', 'optimal', 'invscaling', 'adaptive'}, default='optimal'
+        The learning rate schedule to use with `fit`. (If using `partial_fit`,
+        learning rate must be controlled directly).
+
+        - 'constant': `eta = eta0`
+        - 'optimal': `eta = 1.0 / (alpha * (t + t0))`
+          where t0 is chosen by a heuristic proposed by Leon Bottou.
+        - 'invscaling': `eta = eta0 / pow(t, power_t)`
+        - 'adaptive': eta = eta0, as long as the training keeps decreasing.
+          Each time n_iter_no_change consecutive epochs fail to decrease the
+          training loss by tol or fail to increase validation score by tol if
+          early_stopping is True, the current learning rate is divided by 5.
+
+    eta0 : float, default=0.0
+        The initial learning rate for the 'constant', 'invscaling' or
+        'adaptive' schedules. The default value is 0.0 as eta0 is not used by
+        the default schedule 'optimal'.
+        Values must be in the range `[0.0, inf)`.
+
+    power_t : float, default=0.5
+        The exponent for inverse scaling learning rate.
+        Values must be in the range `(-inf, inf)`.
+
+    warm_start : bool, default=False
+        When set to True, reuse the solution of the previous call to fit as
+        initialization, otherwise, just erase the previous solution.
+        See :term:`the Glossary `.
+
+        Repeatedly calling fit or partial_fit when warm_start is True can
+        result in a different solution than when calling fit a single time
+        because of the way the data is shuffled.
+        If a dynamic learning rate is used, the learning rate is adapted
+        depending on the number of samples already seen. Calling ``fit`` resets
+        this counter, while ``partial_fit``  will result in increasing the
+        existing counter.
+
+    average : bool or int, default=False
+        When set to True, computes the averaged SGD weights and stores the
+        result in the ``coef_`` attribute. If set to an int greater than 1,
+        averaging will begin once the total number of samples seen reaches
+        average. So ``average=10`` will begin averaging after seeing 10
+        samples.
+
+    Attributes
+    ----------
+    coef_ : ndarray of shape (1, n_features)
+        Weights assigned to the features.
+
+    offset_ : ndarray of shape (1,)
+        Offset used to define the decision function from the raw scores.
+        We have the relation: decision_function = score_samples - offset.
+
+    n_iter_ : int
+        The actual number of iterations to reach the stopping criterion.
+
+    t_ : int
+        Number of weight updates performed during training.
+        Same as ``(n_iter_ * n_samples + 1)``.
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
 
     See Also
     --------
-    Ridge, ElasticNet, Lasso, sklearn.svm.SVR
+    sklearn.svm.OneClassSVM : Unsupervised Outlier Detection.
 
+    Notes
+    -----
+    This estimator has a linear complexity in the number of training samples
+    and is thus better suited than the `sklearn.svm.OneClassSVM`
+    implementation for datasets with a large number of training samples (say
+    > 10,000).
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from sklearn import linear_model
+    >>> X = np.array([[-1, -1], [-2, -1], [1, 1], [2, 1]])
+    >>> clf = linear_model.SGDOneClassSVM(random_state=42)
+    >>> clf.fit(X)
+    SGDOneClassSVM(random_state=42)
+
+    >>> print(clf.predict([[4, 4]]))
+    [1]
     """
-    @_deprecate_positional_args
-    def __init__(self, loss="squared_loss", *, penalty="l2", alpha=0.0001,
-                 l1_ratio=0.15, fit_intercept=True, max_iter=1000, tol=1e-3,
-                 shuffle=True, verbose=0, epsilon=DEFAULT_EPSILON,
-                 random_state=None, learning_rate="invscaling", eta0=0.01,
-                 power_t=0.25, early_stopping=False, validation_fraction=0.1,
-                 n_iter_no_change=5, warm_start=False, average=False):
+
+    loss_functions = {"hinge": (Hinge, 1.0)}
+
+    _parameter_constraints: dict = {
+        **BaseSGD._parameter_constraints,
+        "nu": [Interval(Real, 0.0, 1.0, closed="right")],
+        "learning_rate": [
+            StrOptions({"constant", "optimal", "invscaling", "adaptive"}),
+            Hidden(StrOptions({"pa1", "pa2"})),
+        ],
+        "eta0": [Interval(Real, 0, None, closed="left")],
+        "power_t": [Interval(Real, None, None, closed="neither")],
+    }
+
+    def __init__(
+        self,
+        nu=0.5,
+        fit_intercept=True,
+        max_iter=1000,
+        tol=1e-3,
+        shuffle=True,
+        verbose=0,
+        random_state=None,
+        learning_rate="optimal",
+        eta0=0.0,
+        power_t=0.5,
+        warm_start=False,
+        average=False,
+    ):
+        self.nu = nu
         super().__init__(
-            loss=loss, penalty=penalty, alpha=alpha, l1_ratio=l1_ratio,
-            fit_intercept=fit_intercept, max_iter=max_iter, tol=tol,
-            shuffle=shuffle, verbose=verbose, epsilon=epsilon,
-            random_state=random_state, learning_rate=learning_rate, eta0=eta0,
-            power_t=power_t, early_stopping=early_stopping,
-            validation_fraction=validation_fraction,
-            n_iter_no_change=n_iter_no_change, warm_start=warm_start,
-            average=average)
-
-    def _more_tags(self):
-        return {
-            '_xfail_checks': {
-                'check_sample_weights_invariance':
-                'zero sample_weight is not equivalent to removing samples',
-            }
-        }
+            loss="hinge",
+            penalty="l2",
+            C=1.0,
+            l1_ratio=0,
+            fit_intercept=fit_intercept,
+            max_iter=max_iter,
+            tol=tol,
+            shuffle=shuffle,
+            verbose=verbose,
+            epsilon=DEFAULT_EPSILON,
+            random_state=random_state,
+            learning_rate=learning_rate,
+            eta0=eta0,
+            power_t=power_t,
+            early_stopping=False,
+            validation_fraction=0.1,
+            n_iter_no_change=5,
+            warm_start=warm_start,
+            average=average,
+        )
+
+    def _fit_one_class(self, X, alpha, C, sample_weight, learning_rate, max_iter):
+        """Uses SGD implementation with X and y=np.ones(n_samples)."""
+
+        # The One-Class SVM uses the SGD implementation with
+        # y=np.ones(n_samples).
+        n_samples = X.shape[0]
+        y = np.ones(n_samples, dtype=X.dtype, order="C")
+
+        dataset, offset_decay = make_dataset(X, y, sample_weight)
+
+        penalty_type = self._get_penalty_type(self.penalty)
+        learning_rate_type = self._get_learning_rate_type(learning_rate)
+
+        # early stopping is set to False for the One-Class SVM. thus
+        # validation_mask and validation_score_cb will be set to values
+        # associated to early_stopping=False in _make_validation_split and
+        # _make_validation_score_cb respectively.
+        validation_mask = self._make_validation_split(y, sample_mask=sample_weight > 0)
+        validation_score_cb = self._make_validation_score_cb(
+            validation_mask, X, y, sample_weight
+        )
+
+        random_state = check_random_state(self.random_state)
+        # numpy mtrand expects a C long which is a signed 32 bit integer under
+        # Windows
+        seed = random_state.randint(0, np.iinfo(np.int32).max)
+
+        tol = self.tol if self.tol is not None else -np.inf
+
+        one_class = 1
+        # There are no class weights for the One-Class SVM and they are
+        # therefore set to 1.
+        pos_weight = 1
+        neg_weight = 1
+
+        if self.average:
+            coef = self._standard_coef
+            intercept = self._standard_intercept
+            average_coef = self._average_coef
+            average_intercept = self._average_intercept
+        else:
+            coef = self.coef_
+            intercept = 1 - self.offset_
+            average_coef = None  # Not used
+            average_intercept = [0]  # Not used
+
+        _plain_sgd = _get_plain_sgd_function(input_dtype=coef.dtype)
+        coef, intercept, average_coef, average_intercept, self.n_iter_ = _plain_sgd(
+            coef,
+            intercept[0],
+            average_coef,
+            average_intercept[0],
+            self._loss_function_,
+            penalty_type,
+            alpha,
+            C,
+            self.l1_ratio,
+            dataset,
+            validation_mask,
+            self.early_stopping,
+            validation_score_cb,
+            int(self.n_iter_no_change),
+            max_iter,
+            tol,
+            int(self.fit_intercept),
+            int(self.verbose),
+            int(self.shuffle),
+            seed,
+            neg_weight,
+            pos_weight,
+            learning_rate_type,
+            self.eta0,
+            self.power_t,
+            one_class,
+            self.t_,
+            offset_decay,
+            self.average,
+        )
+
+        self.t_ += self.n_iter_ * n_samples
+
+        if self.average > 0:
+            self._average_intercept = np.atleast_1d(average_intercept)
+            self._standard_intercept = np.atleast_1d(intercept)
+
+            if self.average <= self.t_ - 1.0:
+                # made enough updates for averaging to be taken into account
+                self.coef_ = average_coef
+                self.offset_ = 1 - np.atleast_1d(average_intercept)
+            else:
+                self.coef_ = coef
+                self.offset_ = 1 - np.atleast_1d(intercept)
+
+        else:
+            self.offset_ = 1 - np.atleast_1d(intercept)
+
+    def _partial_fit(
+        self,
+        X,
+        alpha,
+        C,
+        loss,
+        learning_rate,
+        max_iter,
+        sample_weight,
+        coef_init,
+        offset_init,
+    ):
+        first_call = getattr(self, "coef_", None) is None
+        X = validate_data(
+            self,
+            X,
+            None,
+            accept_sparse="csr",
+            dtype=[np.float64, np.float32],
+            order="C",
+            accept_large_sparse=False,
+            reset=first_call,
+        )
+
+        n_features = X.shape[1]
+
+        # Allocate datastructures from input arguments
+        sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype)
+
+        # We use intercept = 1 - offset where intercept is the intercept of
+        # the SGD implementation and offset is the offset of the One-Class SVM
+        # optimization problem.
+        if getattr(self, "coef_", None) is None or coef_init is not None:
+            self._allocate_parameter_mem(
+                n_classes=1,
+                n_features=n_features,
+                input_dtype=X.dtype,
+                coef_init=coef_init,
+                intercept_init=offset_init,
+                one_class=1,
+            )
+        elif n_features != self.coef_.shape[-1]:
+            raise ValueError(
+                "Number of features %d does not match previous data %d."
+                % (n_features, self.coef_.shape[-1])
+            )
+
+        if self.average and getattr(self, "_average_coef", None) is None:
+            self._average_coef = np.zeros(n_features, dtype=X.dtype, order="C")
+            self._average_intercept = np.zeros(1, dtype=X.dtype, order="C")
+
+        self._loss_function_ = self._get_loss_function(loss)
+        if not hasattr(self, "t_"):
+            self.t_ = 1.0
+
+        # delegate to concrete training procedure
+        self._fit_one_class(
+            X,
+            alpha=alpha,
+            C=C,
+            learning_rate=learning_rate,
+            sample_weight=sample_weight,
+            max_iter=max_iter,
+        )
+
+        return self
+
+    @_fit_context(prefer_skip_nested_validation=True)
+    def partial_fit(self, X, y=None, sample_weight=None):
+        """Fit linear One-Class SVM with Stochastic Gradient Descent.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix}, shape (n_samples, n_features)
+            Subset of the training data.
+        y : Ignored
+            Not used, present for API consistency by convention.
+
+        sample_weight : array-like, shape (n_samples,), optional
+            Weights applied to individual samples.
+            If not provided, uniform weights are assumed.
+
+        Returns
+        -------
+        self : object
+            Returns a fitted instance of self.
+        """
+        if not hasattr(self, "coef_"):
+            self._more_validate_params(for_partial_fit=True)
+
+        alpha = self.nu / 2
+        return self._partial_fit(
+            X,
+            alpha,
+            C=1.0,
+            loss=self.loss,
+            learning_rate=self.learning_rate,
+            max_iter=1,
+            sample_weight=sample_weight,
+            coef_init=None,
+            offset_init=None,
+        )
+
+    def _fit(
+        self,
+        X,
+        alpha,
+        C,
+        loss,
+        learning_rate,
+        coef_init=None,
+        offset_init=None,
+        sample_weight=None,
+    ):
+        if self.warm_start and hasattr(self, "coef_"):
+            if coef_init is None:
+                coef_init = self.coef_
+            if offset_init is None:
+                offset_init = self.offset_
+        else:
+            self.coef_ = None
+            self.offset_ = None
+
+        # Clear iteration count for multiple call to fit.
+        self.t_ = 1.0
+
+        self._partial_fit(
+            X,
+            alpha,
+            C,
+            loss,
+            learning_rate,
+            self.max_iter,
+            sample_weight,
+            coef_init,
+            offset_init,
+        )
+
+        if (
+            self.tol is not None
+            and self.tol > -np.inf
+            and self.n_iter_ == self.max_iter
+        ):
+            warnings.warn(
+                (
+                    "Maximum number of iteration reached before "
+                    "convergence. Consider increasing max_iter to "
+                    "improve the fit."
+                ),
+                ConvergenceWarning,
+            )
+
+        return self
+
+    @_fit_context(prefer_skip_nested_validation=True)
+    def fit(self, X, y=None, coef_init=None, offset_init=None, sample_weight=None):
+        """Fit linear One-Class SVM with Stochastic Gradient Descent.
+
+        This solves an equivalent optimization problem of the
+        One-Class SVM primal optimization problem and returns a weight vector
+        w and an offset rho such that the decision function is given by
+         - rho.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix}, shape (n_samples, n_features)
+            Training data.
+        y : Ignored
+            Not used, present for API consistency by convention.
+
+        coef_init : array, shape (n_classes, n_features)
+            The initial coefficients to warm-start the optimization.
+
+        offset_init : array, shape (n_classes,)
+            The initial offset to warm-start the optimization.
+
+        sample_weight : array-like, shape (n_samples,), optional
+            Weights applied to individual samples.
+            If not provided, uniform weights are assumed. These weights will
+            be multiplied with class_weight (passed through the
+            constructor) if class_weight is specified.
+
+        Returns
+        -------
+        self : object
+            Returns a fitted instance of self.
+        """
+        self._more_validate_params()
+
+        alpha = self.nu / 2
+        self._fit(
+            X,
+            alpha=alpha,
+            C=1.0,
+            loss=self.loss,
+            learning_rate=self.learning_rate,
+            coef_init=coef_init,
+            offset_init=offset_init,
+            sample_weight=sample_weight,
+        )
+
+        return self
+
+    def decision_function(self, X):
+        """Signed distance to the separating hyperplane.
+
+        Signed distance is positive for an inlier and negative for an
+        outlier.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix}, shape (n_samples, n_features)
+            Testing data.
+
+        Returns
+        -------
+        dec : array-like, shape (n_samples,)
+            Decision function values of the samples.
+        """
+
+        check_is_fitted(self, "coef_")
+
+        X = validate_data(self, X, accept_sparse="csr", reset=False)
+        decisions = safe_sparse_dot(X, self.coef_.T, dense_output=True) - self.offset_
+
+        return decisions.ravel()
+
+    def score_samples(self, X):
+        """Raw scoring function of the samples.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix}, shape (n_samples, n_features)
+            Testing data.
+
+        Returns
+        -------
+        score_samples : array-like, shape (n_samples,)
+            Unshiffted scoring function values of the samples.
+        """
+        score_samples = self.decision_function(X) + self.offset_
+        return score_samples
+
+    def predict(self, X):
+        """Return labels (1 inlier, -1 outlier) of the samples.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix}, shape (n_samples, n_features)
+            Testing data.
+
+        Returns
+        -------
+        y : array, shape (n_samples,)
+            Labels of the samples.
+        """
+        y = (self.decision_function(X) >= 0).astype(np.int32)
+        y[y == 0] = -1  # for consistency with outlier detectors
+        return y
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
+        return tags
diff --git a/sklearn/linear_model/_theil_sen.py b/sklearn/linear_model/_theil_sen.py
index 39f3d5c69fb00..4b25145a8ca55 100644
--- a/sklearn/linear_model/_theil_sen.py
+++ b/sklearn/linear_model/_theil_sen.py
@@ -1,28 +1,27 @@
-# -*- coding: utf-8 -*-
 """
 A Theil-Sen Estimator for Multiple Linear Regression Model
 """
 
-# Author: Florian Wilhelm 
-#
-# License: BSD 3 clause
-
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import warnings
 from itertools import combinations
+from numbers import Integral, Real
 
 import numpy as np
+from joblib import effective_n_jobs
 from scipy import linalg
-from scipy.special import binom
 from scipy.linalg.lapack import get_lapack_funcs
-from joblib import Parallel, effective_n_jobs
+from scipy.special import binom
 
-from ._base import LinearModel
-from ..base import RegressorMixin
-from ..utils import check_random_state
-from ..utils.validation import _deprecate_positional_args
-from ..utils.fixes import delayed
+from ..base import RegressorMixin, _fit_context
 from ..exceptions import ConvergenceWarning
+from ..utils import check_random_state
+from ..utils._param_validation import Hidden, Interval, StrOptions
+from ..utils.parallel import Parallel, delayed
+from ..utils.validation import validate_data
+from ._base import LinearModel
 
 _EPSILON = np.finfo(np.double).eps
 
@@ -37,8 +36,8 @@ def _modified_weiszfeld_step(X, x_old):
     Parameters
     ----------
     X : array-like of shape (n_samples, n_features)
-        Training vector, where n_samples is the number of samples and
-        n_features is the number of features.
+        Training vector, where `n_samples` is the number of samples and
+        `n_features` is the number of features.
 
     x_old : ndarray of shape = (n_features,)
         Current start vector.
@@ -55,7 +54,7 @@ def _modified_weiszfeld_step(X, x_old):
       http://users.jyu.fi/~samiayr/pdf/ayramo_eurogen05.pdf
     """
     diff = X - x_old
-    diff_norm = np.sqrt(np.sum(diff ** 2, axis=1))
+    diff_norm = np.sqrt(np.sum(diff**2, axis=1))
     mask = diff_norm >= _EPSILON
     # x_old equals one of our samples
     is_x_old_in_X = int(mask.sum() < X.shape[0])
@@ -65,17 +64,20 @@ def _modified_weiszfeld_step(X, x_old):
     quotient_norm = linalg.norm(np.sum(diff / diff_norm, axis=0))
 
     if quotient_norm > _EPSILON:  # to avoid division by zero
-        new_direction = (np.sum(X[mask, :] / diff_norm, axis=0)
-                         / np.sum(1 / diff_norm, axis=0))
+        new_direction = np.sum(X[mask, :] / diff_norm, axis=0) / np.sum(
+            1 / diff_norm, axis=0
+        )
     else:
-        new_direction = 1.
-        quotient_norm = 1.
+        new_direction = 1.0
+        quotient_norm = 1.0
 
-    return (max(0., 1. - is_x_old_in_X / quotient_norm) * new_direction
-            + min(1., is_x_old_in_X / quotient_norm) * x_old)
+    return (
+        max(0.0, 1.0 - is_x_old_in_X / quotient_norm) * new_direction
+        + min(1.0, is_x_old_in_X / quotient_norm) * x_old
+    )
 
 
-def _spatial_median(X, max_iter=300, tol=1.e-3):
+def _spatial_median(X, max_iter=300, tol=1.0e-3):
     """Spatial median (L1 median).
 
     The spatial median is member of a class of so-called M-estimators which
@@ -86,8 +88,8 @@ def _spatial_median(X, max_iter=300, tol=1.e-3):
     Parameters
     ----------
     X : array-like of shape (n_samples, n_features)
-        Training vector, where n_samples is the number of samples and
-        n_features is the number of features.
+        Training vector, where `n_samples` is the number of samples and
+        `n_features` is the number of features.
 
     max_iter : int, default=300
         Maximum number of iterations.
@@ -122,9 +124,12 @@ def _spatial_median(X, max_iter=300, tol=1.e-3):
         else:
             spatial_median_old = spatial_median
     else:
-        warnings.warn("Maximum number of iterations {max_iter} reached in "
-                      "spatial median for TheilSen regressor."
-                      "".format(max_iter=max_iter), ConvergenceWarning)
+        warnings.warn(
+            "Maximum number of iterations {max_iter} reached in "
+            "spatial median for TheilSen regressor."
+            "".format(max_iter=max_iter),
+            ConvergenceWarning,
+        )
     return n_iter, spatial_median
 
 
@@ -144,8 +149,15 @@ def _breakdown_point(n_samples, n_subsamples):
     breakdown_point : float
         Approximation of breakdown point.
     """
-    return 1 - (0.5 ** (1 / n_subsamples) * (n_samples - n_subsamples + 1) +
-                n_subsamples - 1) / n_samples
+    return (
+        1
+        - (
+            0.5 ** (1 / n_subsamples) * (n_samples - n_subsamples + 1)
+            + n_subsamples
+            - 1
+        )
+        / n_samples
+    )
 
 
 def _lstsq(X, y, indices, fit_intercept):
@@ -158,11 +170,11 @@ def _lstsq(X, y, indices, fit_intercept):
     Parameters
     ----------
     X : array-like of shape (n_samples, n_features)
-        Design matrix, where n_samples is the number of samples and
-        n_features is the number of features.
+        Design matrix, where `n_samples` is the number of samples and
+        `n_features` is the number of features.
 
     y : ndarray of shape (n_samples,)
-        Target vector, where n_samples is the number of samples.
+        Target vector, where `n_samples` is the number of samples.
 
     indices : ndarray of shape (n_subpopulation, n_subsamples)
         Indices of all subsamples with respect to the chosen subpopulation.
@@ -182,13 +194,12 @@ def _lstsq(X, y, indices, fit_intercept):
     X_subpopulation = np.ones((n_subsamples, n_features))
     # gelss need to pad y_subpopulation to be of the max dim of X_subpopulation
     y_subpopulation = np.zeros((max(n_subsamples, n_features)))
-    lstsq, = get_lapack_funcs(('gelss',), (X_subpopulation, y_subpopulation))
+    (lstsq,) = get_lapack_funcs(("gelss",), (X_subpopulation, y_subpopulation))
 
     for index, subset in enumerate(indices):
         X_subpopulation[:, fit_intercept:] = X[subset, :]
         y_subpopulation[:n_subsamples] = y[subset]
-        weights[index] = lstsq(X_subpopulation,
-                               y_subpopulation)[1][:n_features]
+        weights[index] = lstsq(X_subpopulation, y_subpopulation)[1][:n_features]
 
     return weights
 
@@ -216,13 +227,18 @@ class TheilSenRegressor(RegressorMixin, LinearModel):
     copy_X : bool, default=True
         If True, X will be copied; else, it may be overwritten.
 
+        .. deprecated:: 1.6
+            `copy_X` was deprecated in 1.6 and will be removed in 1.8.
+            It has no effect as a copy is always made.
+
     max_subpopulation : int, default=1e4
         Instead of computing with a set of cardinality 'n choose k', where n is
         the number of samples and k is the number of subsamples (at least
         number of features), consider only a stochastic subpopulation of a
         given maximal size if 'n choose k' is larger than max_subpopulation.
         For other than small problem sizes this parameter will determine
-        memory usage and runtime if n_subsamples is not changed.
+        memory usage and runtime if n_subsamples is not changed. Note that the
+        data type should be int but floats such as 1e4 can be accepted too.
 
     n_subsamples : int, default=None
         Number of samples to calculate the parameters. This is at least the
@@ -237,14 +253,14 @@ class TheilSenRegressor(RegressorMixin, LinearModel):
     max_iter : int, default=300
         Maximum number of iterations for the calculation of spatial median.
 
-    tol : float, default=1.e-3
+    tol : float, default=1e-3
         Tolerance when calculating spatial median.
 
     random_state : int, RandomState instance or None, default=None
         A random number generator instance to define the state of the random
         permutations generator. Pass an int for reproducible output across
         multiple function calls.
-        See :term:`Glossary `
+        See :term:`Glossary `.
 
     n_jobs : int, default=None
         Number of CPUs to use during the cross validation.
@@ -273,6 +289,29 @@ class TheilSenRegressor(RegressorMixin, LinearModel):
         Number of combinations taken into account from 'n choose k', where n is
         the number of samples and k is the number of subsamples.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    HuberRegressor : Linear regression model that is robust to outliers.
+    RANSACRegressor : RANSAC (RANdom SAmple Consensus) algorithm.
+    SGDRegressor : Fitted by minimizing a regularized empirical loss with SGD.
+
+    References
+    ----------
+    - Theil-Sen Estimators in a Multiple Linear Regression Model, 2009
+      Xin Dang, Hanxiang Peng, Xueqin Wang and Heping Zhang
+      http://home.olemiss.edu/~xdang/papers/MTSE.pdf
+
     Examples
     --------
     >>> from sklearn.linear_model import TheilSenRegressor
@@ -281,23 +320,40 @@ class TheilSenRegressor(RegressorMixin, LinearModel):
     ...     n_samples=200, n_features=2, noise=4.0, random_state=0)
     >>> reg = TheilSenRegressor(random_state=0).fit(X, y)
     >>> reg.score(X, y)
-    0.9884...
+    0.9884
     >>> reg.predict(X[:1,])
-    array([-31.5871...])
-
-    References
-    ----------
-    - Theil-Sen Estimators in a Multiple Linear Regression Model, 2009
-      Xin Dang, Hanxiang Peng, Xueqin Wang and Heping Zhang
-      http://home.olemiss.edu/~xdang/papers/MTSE.pdf
+    array([-31.5871])
     """
-    @_deprecate_positional_args
-    def __init__(self, *, fit_intercept=True, copy_X=True,
-                 max_subpopulation=1e4, n_subsamples=None, max_iter=300,
-                 tol=1.e-3, random_state=None, n_jobs=None, verbose=False):
+
+    _parameter_constraints: dict = {
+        "fit_intercept": ["boolean"],
+        "copy_X": ["boolean", Hidden(StrOptions({"deprecated"}))],
+        # target_type should be Integral but can accept Real for backward compatibility
+        "max_subpopulation": [Interval(Real, 1, None, closed="left")],
+        "n_subsamples": [None, Integral],
+        "max_iter": [Interval(Integral, 0, None, closed="left")],
+        "tol": [Interval(Real, 0.0, None, closed="left")],
+        "random_state": ["random_state"],
+        "n_jobs": [None, Integral],
+        "verbose": ["verbose"],
+    }
+
+    def __init__(
+        self,
+        *,
+        fit_intercept=True,
+        copy_X="deprecated",
+        max_subpopulation=1e4,
+        n_subsamples=None,
+        max_iter=300,
+        tol=1.0e-3,
+        random_state=None,
+        n_jobs=None,
+        verbose=False,
+    ):
         self.fit_intercept = fit_intercept
         self.copy_X = copy_X
-        self.max_subpopulation = int(max_subpopulation)
+        self.max_subpopulation = max_subpopulation
         self.n_subsamples = n_subsamples
         self.max_iter = max_iter
         self.tol = tol
@@ -315,33 +371,34 @@ def _check_subparams(self, n_samples, n_features):
 
         if n_subsamples is not None:
             if n_subsamples > n_samples:
-                raise ValueError("Invalid parameter since n_subsamples > "
-                                 "n_samples ({0} > {1}).".format(n_subsamples,
-                                                                 n_samples))
+                raise ValueError(
+                    "Invalid parameter since n_subsamples > "
+                    "n_samples ({0} > {1}).".format(n_subsamples, n_samples)
+                )
             if n_samples >= n_features:
                 if n_dim > n_subsamples:
                     plus_1 = "+1" if self.fit_intercept else ""
-                    raise ValueError("Invalid parameter since n_features{0} "
-                                     "> n_subsamples ({1} > {2})."
-                                     "".format(plus_1, n_dim, n_samples))
+                    raise ValueError(
+                        "Invalid parameter since n_features{0} "
+                        "> n_subsamples ({1} > {2})."
+                        "".format(plus_1, n_dim, n_subsamples)
+                    )
             else:  # if n_samples < n_features
                 if n_subsamples != n_samples:
-                    raise ValueError("Invalid parameter since n_subsamples != "
-                                     "n_samples ({0} != {1}) while n_samples "
-                                     "< n_features.".format(n_subsamples,
-                                                            n_samples))
+                    raise ValueError(
+                        "Invalid parameter since n_subsamples != "
+                        "n_samples ({0} != {1}) while n_samples "
+                        "< n_features.".format(n_subsamples, n_samples)
+                    )
         else:
             n_subsamples = min(n_dim, n_samples)
 
-        if self.max_subpopulation <= 0:
-            raise ValueError("Subpopulation must be strictly positive "
-                             "({0} <= 0).".format(self.max_subpopulation))
-
         all_combinations = max(1, np.rint(binom(n_samples, n_subsamples)))
         n_subpopulation = int(min(self.max_subpopulation, all_combinations))
 
         return n_subsamples, n_subpopulation
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y):
         """Fit linear model.
 
@@ -355,12 +412,22 @@ def fit(self, X, y):
         Returns
         -------
         self : returns an instance of self.
+            Fitted `TheilSenRegressor` estimator.
         """
+        if self.copy_X != "deprecated":
+            warnings.warn(
+                "`copy_X` was deprecated in 1.6 and will be removed in 1.8 since it "
+                "has no effect internally. Simply leave this parameter to its default "
+                "value to avoid this warning.",
+                FutureWarning,
+            )
+
         random_state = check_random_state(self.random_state)
-        X, y = self._validate_data(X, y, y_numeric=True)
+        X, y = validate_data(self, X, y, y_numeric=True)
         n_samples, n_features = X.shape
-        n_subsamples, self.n_subpopulation_ = self._check_subparams(n_samples,
-                                                                    n_features)
+        n_subsamples, self.n_subpopulation_ = self._check_subparams(
+            n_samples, n_features
+        )
         self.breakdown_ = _breakdown_point(n_samples, n_subsamples)
 
         if self.verbose:
@@ -368,33 +435,33 @@ def fit(self, X, y):
             print("Number of samples: {0}".format(n_samples))
             tol_outliers = int(self.breakdown_ * n_samples)
             print("Tolerable outliers: {0}".format(tol_outliers))
-            print("Number of subpopulations: {0}".format(
-                self.n_subpopulation_))
+            print("Number of subpopulations: {0}".format(self.n_subpopulation_))
 
         # Determine indices of subpopulation
         if np.rint(binom(n_samples, n_subsamples)) <= self.max_subpopulation:
             indices = list(combinations(range(n_samples), n_subsamples))
         else:
-            indices = [random_state.choice(n_samples, size=n_subsamples,
-                                           replace=False)
-                       for _ in range(self.n_subpopulation_)]
+            indices = [
+                random_state.choice(n_samples, size=n_subsamples, replace=False)
+                for _ in range(self.n_subpopulation_)
+            ]
 
         n_jobs = effective_n_jobs(self.n_jobs)
         index_list = np.array_split(indices, n_jobs)
-        weights = Parallel(n_jobs=n_jobs,
-                           verbose=self.verbose)(
+        weights = Parallel(n_jobs=n_jobs, verbose=self.verbose)(
             delayed(_lstsq)(X, y, index_list[job], self.fit_intercept)
-            for job in range(n_jobs))
+            for job in range(n_jobs)
+        )
         weights = np.vstack(weights)
-        self.n_iter_, coefs = _spatial_median(weights,
-                                              max_iter=self.max_iter,
-                                              tol=self.tol)
+        self.n_iter_, coefs = _spatial_median(
+            weights, max_iter=self.max_iter, tol=self.tol
+        )
 
         if self.fit_intercept:
             self.intercept_ = coefs[0]
             self.coef_ = coefs[1:]
         else:
-            self.intercept_ = 0.
+            self.intercept_ = 0.0
             self.coef_ = coefs
 
         return self
diff --git a/sklearn/linear_model/meson.build b/sklearn/linear_model/meson.build
new file mode 100644
index 0000000000000..6d8405c793389
--- /dev/null
+++ b/sklearn/linear_model/meson.build
@@ -0,0 +1,32 @@
+# .pyx is generated, so this is needed to make Cython compilation work
+linear_model_cython_tree = [
+  fs.copyfile('__init__.py'),
+]
+
+py.extension_module(
+  '_cd_fast',
+  [cython_gen.process('_cd_fast.pyx'), utils_cython_tree],
+  subdir: 'sklearn/linear_model',
+  install: true
+)
+
+name_list = ['_sgd_fast', '_sag_fast']
+
+foreach name: name_list
+  pyx = custom_target(
+    name + '_pyx',
+    output: name + '.pyx',
+    input: name + '.pyx.tp',
+    command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
+    # TODO in principle this should go in py.exension_module below. This is
+    # temporary work-around for dependency issue with .pyx.tp files. For more
+    # details, see https://github.com/mesonbuild/meson/issues/13212
+    depends: [linear_model_cython_tree, utils_cython_tree, _loss_cython_tree],
+  )
+  py.extension_module(
+    name,
+    cython_gen.process(pyx),
+    subdir: 'sklearn/linear_model',
+    install: true
+)
+endforeach
diff --git a/sklearn/linear_model/setup.py b/sklearn/linear_model/setup.py
deleted file mode 100644
index d0c9e8c04c16d..0000000000000
--- a/sklearn/linear_model/setup.py
+++ /dev/null
@@ -1,44 +0,0 @@
-import os
-import numpy
-
-from sklearn._build_utils import gen_from_templates
-
-
-def configuration(parent_package='', top_path=None):
-    from numpy.distutils.misc_util import Configuration
-
-    config = Configuration('linear_model', parent_package, top_path)
-
-    libraries = []
-    if os.name == 'posix':
-        libraries.append('m')
-
-    config.add_extension('_cd_fast',
-                         sources=['_cd_fast.pyx'],
-                         include_dirs=numpy.get_include(),
-                         libraries=libraries)
-
-    config.add_extension('_sgd_fast',
-                         sources=['_sgd_fast.pyx'],
-                         include_dirs=numpy.get_include(),
-                         libraries=libraries)
-
-    # generate sag_fast from template
-    templates = ['sklearn/linear_model/_sag_fast.pyx.tp']
-    gen_from_templates(templates, top_path)
-
-    config.add_extension('_sag_fast',
-                         sources=['_sag_fast.pyx'],
-                         include_dirs=numpy.get_include())
-
-    # add other directories
-    config.add_subpackage('tests')
-    config.add_subpackage('_glm')
-    config.add_subpackage('_glm/tests')
-
-    return config
-
-
-if __name__ == '__main__':
-    from numpy.distutils.core import setup
-    setup(**configuration(top_path='').todict())
diff --git a/sklearn/linear_model/tests/test_base.py b/sklearn/linear_model/tests/test_base.py
index bf7a2696fcda2..cf8dfdf4e4712 100644
--- a/sklearn/linear_model/tests/test_base.py
+++ b/sklearn/linear_model/tests/test_base.py
@@ -1,33 +1,32 @@
-# Author: Alexandre Gramfort 
-#         Fabian Pedregosa 
-#         Maria Telenczuk 
-#
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-import pytest
+import warnings
 
 import numpy as np
-from scipy import sparse
-from scipy import linalg
-
-from sklearn.utils._testing import assert_array_almost_equal
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils._testing import assert_almost_equal
-from sklearn.utils._testing import assert_allclose
-from sklearn.utils import check_random_state
-from sklearn.utils.fixes import parse_version
+import pytest
+from scipy import linalg, sparse
 
+from sklearn.datasets import load_iris, make_regression, make_sparse_uncorrelated
 from sklearn.linear_model import LinearRegression
-from sklearn.linear_model._base import _deprecate_normalize
-from sklearn.linear_model._base import _preprocess_data
-from sklearn.linear_model._base import _rescale_data
-from sklearn.linear_model._base import make_dataset
-from sklearn.datasets import make_sparse_uncorrelated
-from sklearn.datasets import make_regression
-from sklearn.datasets import load_iris
-from sklearn.preprocessing import StandardScaler
-
-rng = np.random.RandomState(0)
+from sklearn.linear_model._base import (
+    _preprocess_data,
+    _rescale_data,
+    make_dataset,
+)
+from sklearn.preprocessing import add_dummy_feature
+from sklearn.utils._testing import (
+    assert_allclose,
+    assert_array_almost_equal,
+    assert_array_equal,
+)
+from sklearn.utils.fixes import (
+    COO_CONTAINERS,
+    CSC_CONTAINERS,
+    CSR_CONTAINERS,
+    LIL_CONTAINERS,
+)
+
 rtol = 1e-6
 
 
@@ -55,51 +54,49 @@ def test_linear_regression():
     assert_array_almost_equal(reg.predict(X), [0])
 
 
-def test_linear_regression_sample_weights():
-    # TODO: loop over sparse data as well
-
-    rng = np.random.RandomState(0)
+@pytest.mark.parametrize("sparse_container", [None] + CSR_CONTAINERS)
+@pytest.mark.parametrize("fit_intercept", [True, False])
+def test_linear_regression_sample_weights(
+    sparse_container, fit_intercept, global_random_seed
+):
+    rng = np.random.RandomState(global_random_seed)
 
     # It would not work with under-determined systems
-    for n_samples, n_features in ((6, 5), ):
+    n_samples, n_features = 6, 5
 
-        y = rng.randn(n_samples)
-        X = rng.randn(n_samples, n_features)
-        sample_weight = 1.0 + rng.rand(n_samples)
+    X = rng.normal(size=(n_samples, n_features))
+    if sparse_container is not None:
+        X = sparse_container(X)
+    y = rng.normal(size=n_samples)
 
-        for intercept in (True, False):
+    sample_weight = 1.0 + rng.uniform(size=n_samples)
 
-            # LinearRegression with explicit sample_weight
-            reg = LinearRegression(fit_intercept=intercept)
-            reg.fit(X, y, sample_weight=sample_weight)
-            coefs1 = reg.coef_
-            inter1 = reg.intercept_
+    # LinearRegression with explicit sample_weight
+    reg = LinearRegression(fit_intercept=fit_intercept, tol=1e-16)
+    reg.fit(X, y, sample_weight=sample_weight)
+    coefs1 = reg.coef_
+    inter1 = reg.intercept_
 
-            assert reg.coef_.shape == (X.shape[1], )  # sanity checks
-            assert reg.score(X, y) > 0.5
+    assert reg.coef_.shape == (X.shape[1],)  # sanity checks
 
-            # Closed form of the weighted least square
-            # theta = (X^T W X)^(-1) * X^T W y
-            W = np.diag(sample_weight)
-            if intercept is False:
-                X_aug = X
-            else:
-                dummy_column = np.ones(shape=(n_samples, 1))
-                X_aug = np.concatenate((dummy_column, X), axis=1)
+    # Closed form of the weighted least square
+    # theta = (X^T W X)^(-1) @ X^T W y
+    W = np.diag(sample_weight)
+    X_aug = X if not fit_intercept else add_dummy_feature(X)
 
-            coefs2 = linalg.solve(X_aug.T.dot(W).dot(X_aug),
-                                  X_aug.T.dot(W).dot(y))
+    Xw = X_aug.T @ W @ X_aug
+    yw = X_aug.T @ W @ y
+    coefs2 = linalg.solve(Xw, yw)
 
-            if intercept is False:
-                assert_array_almost_equal(coefs1, coefs2)
-            else:
-                assert_array_almost_equal(coefs1, coefs2[1:])
-                assert_almost_equal(inter1, coefs2[0])
+    if not fit_intercept:
+        assert_allclose(coefs1, coefs2)
+    else:
+        assert_allclose(coefs1, coefs2[1:])
+        assert_allclose(inter1, coefs2[0])
 
 
 def test_raises_value_error_if_positive_and_sparse():
-    error_msg = ('A sparse matrix was passed, '
-                 'but dense data is required.')
+    error_msg = "Sparse data was passed for X, but dense data is required."
     # X must not be sparse if positive == True
     X = sparse.eye(10)
     y = np.ones(10)
@@ -110,33 +107,30 @@ def test_raises_value_error_if_positive_and_sparse():
         reg.fit(X, y)
 
 
-def test_raises_value_error_if_sample_weights_greater_than_1d():
+@pytest.mark.parametrize("n_samples, n_features", [(2, 3), (3, 2)])
+def test_raises_value_error_if_sample_weights_greater_than_1d(n_samples, n_features):
     # Sample weights must be either scalar or 1D
+    rng = np.random.RandomState(0)
+    X = rng.randn(n_samples, n_features)
+    y = rng.randn(n_samples)
+    sample_weights_OK = rng.randn(n_samples) ** 2 + 1
+    sample_weights_OK_1 = 1.0
+    sample_weights_OK_2 = 2.0
 
-    n_sampless = [2, 3]
-    n_featuress = [3, 2]
-
-    for n_samples, n_features in zip(n_sampless, n_featuress):
-        X = rng.randn(n_samples, n_features)
-        y = rng.randn(n_samples)
-        sample_weights_OK = rng.randn(n_samples) ** 2 + 1
-        sample_weights_OK_1 = 1.
-        sample_weights_OK_2 = 2.
-
-        reg = LinearRegression()
+    reg = LinearRegression()
 
-        # make sure the "OK" sample weights actually work
-        reg.fit(X, y, sample_weights_OK)
-        reg.fit(X, y, sample_weights_OK_1)
-        reg.fit(X, y, sample_weights_OK_2)
+    # make sure the "OK" sample weights actually work
+    reg.fit(X, y, sample_weights_OK)
+    reg.fit(X, y, sample_weights_OK_1)
+    reg.fit(X, y, sample_weights_OK_2)
 
 
 def test_fit_intercept():
     # Test assertions on betas shape.
-    X2 = np.array([[0.38349978, 0.61650022],
-                   [0.58853682, 0.41146318]])
-    X3 = np.array([[0.27677969, 0.70693172, 0.01628859],
-                   [0.08385139, 0.20692515, 0.70922346]])
+    X2 = np.array([[0.38349978, 0.61650022], [0.58853682, 0.41146318]])
+    X3 = np.array(
+        [[0.27677969, 0.70693172, 0.01628859], [0.08385139, 0.20692515, 0.70922346]]
+    )
     y = np.array([1, 1])
 
     lr2_without_intercept = LinearRegression(fit_intercept=False).fit(X2, y)
@@ -145,124 +139,38 @@ def test_fit_intercept():
     lr3_without_intercept = LinearRegression(fit_intercept=False).fit(X3, y)
     lr3_with_intercept = LinearRegression().fit(X3, y)
 
-    assert (lr2_with_intercept.coef_.shape ==
-            lr2_without_intercept.coef_.shape)
-    assert (lr3_with_intercept.coef_.shape ==
-            lr3_without_intercept.coef_.shape)
-    assert (lr2_without_intercept.coef_.ndim ==
-            lr3_without_intercept.coef_.ndim)
-
-
-def test_error_on_wrong_normalize():
-    normalize = 'wrong'
-    default = True
-    error_msg = "Leave 'normalize' to its default"
-    with pytest.raises(ValueError, match=error_msg):
-        _deprecate_normalize(normalize, default, 'estimator')
-    ValueError
-
-
-@pytest.mark.parametrize('normalize', [True, False, 'deprecated'])
-@pytest.mark.parametrize('default', [True, False])
-# FIXME update test in 1.2 for new versions
-def test_deprecate_normalize(normalize, default):
-    # test all possible case of the normalize parameter deprecation
-    if not default:
-        if normalize == 'deprecated':
-            # no warning
-            output = default
-            expected = None
-            warning_msg = []
-        else:
-            output = normalize
-            expected = FutureWarning
-            warning_msg = ['1.2']
-            if not normalize:
-                warning_msg.append('default value')
-            else:
-                warning_msg.append('StandardScaler(')
-    elif default:
-        if normalize == 'deprecated':
-            # warning to pass False and use StandardScaler
-            output = default
-            expected = FutureWarning
-            warning_msg = ['False', '1.2', 'StandardScaler(']
-        else:
-            # no warning
-            output = normalize
-            expected = None
-            warning_msg = []
-
-    with pytest.warns(expected) as record:
-        _normalize = _deprecate_normalize(normalize, default, 'estimator')
-    assert _normalize == output
+    assert lr2_with_intercept.coef_.shape == lr2_without_intercept.coef_.shape
+    assert lr3_with_intercept.coef_.shape == lr3_without_intercept.coef_.shape
+    assert lr2_without_intercept.coef_.ndim == lr3_without_intercept.coef_.ndim
 
-    n_warnings = 0 if expected is None else 1
-    assert len(record) == n_warnings
-    if n_warnings:
-        assert all([
-            warning in str(record[0].message)
-            for warning in warning_msg
-        ])
 
-
-def test_linear_regression_sparse(random_state=0):
+def test_linear_regression_sparse(global_random_seed):
     # Test that linear regression also works with sparse data
-    random_state = check_random_state(random_state)
-    for i in range(10):
-        n = 100
-        X = sparse.eye(n, n)
-        beta = random_state.rand(n)
-        y = X * beta[:, np.newaxis]
-
-        ols = LinearRegression()
-        ols.fit(X, y.ravel())
-        assert_array_almost_equal(beta, ols.coef_ + ols.intercept_)
-
-        assert_array_almost_equal(ols.predict(X) - y.ravel(), 0)
+    rng = np.random.RandomState(global_random_seed)
+    n = 100
+    X = sparse.eye(n, n)
+    beta = rng.rand(n)
+    y = X @ beta
 
+    ols = LinearRegression()
+    ols.fit(X, y.ravel())
+    assert_array_almost_equal(beta, ols.coef_ + ols.intercept_)
 
-@pytest.mark.parametrize(
-    'normalize, n_warnings, warning',
-    [(True, 1, FutureWarning),
-     (False, 1, FutureWarning),
-     ("deprecated", 0, None)]
-)
-# FIXME remove test in 1.4
-def test_linear_regression_normalize_deprecation(
-     normalize, n_warnings, warning
-):
-    # check that we issue a FutureWarning when normalize was set in
-    # LinearRegression
-    rng = check_random_state(0)
-    n_samples = 200
-    n_features = 2
-    X = rng.randn(n_samples, n_features)
-    X[X < 0.1] = 0.0
-    y = rng.rand(n_samples)
-
-    model = LinearRegression(normalize=normalize)
-    with pytest.warns(warning) as record:
-        model.fit(X, y)
-    assert len(record) == n_warnings
-    if n_warnings:
-        assert "'normalize' was deprecated" in str(record[0].message)
+    assert_array_almost_equal(ols.predict(X) - y.ravel(), 0)
 
 
-# FIXME: 'normalize' to be removed in 1.2 in LinearRegression
-@pytest.mark.filterwarnings("ignore:'normalize' was deprecated")
-@pytest.mark.parametrize('normalize', [True, False])
-@pytest.mark.parametrize('fit_intercept', [True, False])
-def test_linear_regression_sparse_equal_dense(normalize, fit_intercept):
+@pytest.mark.parametrize("fit_intercept", [True, False])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_linear_regression_sparse_equal_dense(fit_intercept, csr_container):
     # Test that linear regression agrees between sparse and dense
-    rng = check_random_state(0)
+    rng = np.random.RandomState(0)
     n_samples = 200
     n_features = 2
     X = rng.randn(n_samples, n_features)
-    X[X < 0.1] = 0.
-    Xcsr = sparse.csr_matrix(X)
+    X[X < 0.1] = 0.0
+    Xcsr = csr_container(X)
     y = rng.rand(n_samples)
-    params = dict(normalize=normalize, fit_intercept=fit_intercept)
+    params = dict(fit_intercept=fit_intercept)
     clf_dense = LinearRegression(**params)
     clf_sparse = LinearRegression(**params)
     clf_dense.fit(X, y)
@@ -271,9 +179,10 @@ def test_linear_regression_sparse_equal_dense(normalize, fit_intercept):
     assert_allclose(clf_dense.coef_, clf_sparse.coef_)
 
 
-def test_linear_regression_multiple_outcome(random_state=0):
+def test_linear_regression_multiple_outcome():
     # Test multiple-outcome linear regressions
-    X, y = make_regression(random_state=random_state)
+    rng = np.random.RandomState(0)
+    X, y = make_regression(random_state=rng)
 
     Y = np.vstack((y, y)).T
     n_features = X.shape[1]
@@ -287,11 +196,12 @@ def test_linear_regression_multiple_outcome(random_state=0):
     assert_array_almost_equal(np.vstack((y_pred, y_pred)).T, Y_pred, decimal=3)
 
 
-def test_linear_regression_sparse_multiple_outcome(random_state=0):
+@pytest.mark.parametrize("coo_container", COO_CONTAINERS)
+def test_linear_regression_sparse_multiple_outcome(global_random_seed, coo_container):
     # Test multiple-outcome linear regressions with sparse data
-    random_state = check_random_state(random_state)
-    X, y = make_sparse_uncorrelated(random_state=random_state)
-    X = sparse.coo_matrix(X)
+    rng = np.random.RandomState(global_random_seed)
+    X, y = make_sparse_uncorrelated(random_state=rng)
+    X = coo_container(X)
     Y = np.vstack((y, y)).T
     n_features = X.shape[1]
 
@@ -327,38 +237,40 @@ def test_linear_regression_positive():
     assert_allclose(reg.predict(X), [0])
 
 
-def test_linear_regression_positive_multiple_outcome(random_state=0):
+def test_linear_regression_positive_multiple_outcome(global_random_seed):
     # Test multiple-outcome nonnegative linear regressions
-    random_state = check_random_state(random_state)
-    X, y = make_sparse_uncorrelated(random_state=random_state)
+    rng = np.random.RandomState(global_random_seed)
+    X, y = make_sparse_uncorrelated(random_state=rng)
     Y = np.vstack((y, y)).T
     n_features = X.shape[1]
 
     ols = LinearRegression(positive=True)
     ols.fit(X, Y)
     assert ols.coef_.shape == (2, n_features)
-    assert np.all(ols.coef_ >= 0.)
+    assert np.all(ols.coef_ >= 0.0)
     Y_pred = ols.predict(X)
     ols.fit(X, y.ravel())
     y_pred = ols.predict(X)
     assert_allclose(np.vstack((y_pred, y_pred)).T, Y_pred)
 
 
-def test_linear_regression_positive_vs_nonpositive():
+def test_linear_regression_positive_vs_nonpositive(global_random_seed):
     # Test differences with LinearRegression when positive=False.
-    X, y = make_sparse_uncorrelated(random_state=0)
+    rng = np.random.RandomState(global_random_seed)
+    X, y = make_sparse_uncorrelated(random_state=rng)
 
     reg = LinearRegression(positive=True)
     reg.fit(X, y)
     regn = LinearRegression(positive=False)
     regn.fit(X, y)
 
-    assert np.mean((reg.coef_ - regn.coef_)**2) > 1e-3
+    assert np.mean((reg.coef_ - regn.coef_) ** 2) > 1e-3
 
 
-def test_linear_regression_positive_vs_nonpositive_when_positive():
+def test_linear_regression_positive_vs_nonpositive_when_positive(global_random_seed):
     # Test LinearRegression fitted coefficients
     # when the problem is positive.
+    rng = np.random.RandomState(global_random_seed)
     n_samples = 200
     n_features = 4
     X = rng.rand(n_samples, n_features)
@@ -369,17 +281,70 @@ def test_linear_regression_positive_vs_nonpositive_when_positive():
     regn = LinearRegression(positive=False)
     regn.fit(X, y)
 
-    assert np.mean((reg.coef_ - regn.coef_)**2) < 1e-6
+    assert np.mean((reg.coef_ - regn.coef_) ** 2) < 1e-6
+
+
+@pytest.mark.parametrize("sparse_container", [None] + CSR_CONTAINERS)
+@pytest.mark.parametrize("use_sw", [True, False])
+def test_inplace_data_preprocessing(sparse_container, use_sw, global_random_seed):
+    # Check that the data is not modified inplace by the linear regression
+    # estimator.
+    rng = np.random.RandomState(global_random_seed)
+    original_X_data = rng.randn(10, 12)
+    original_y_data = rng.randn(10, 2)
+    orginal_sw_data = rng.rand(10)
+
+    if sparse_container is not None:
+        X = sparse_container(original_X_data)
+    else:
+        X = original_X_data.copy()
+    y = original_y_data.copy()
+    # XXX: Note hat y_sparse is not supported (broken?) in the current
+    # implementation of LinearRegression.
+
+    if use_sw:
+        sample_weight = orginal_sw_data.copy()
+    else:
+        sample_weight = None
+
+    # Do not allow inplace preprocessing of X and y:
+    reg = LinearRegression()
+    reg.fit(X, y, sample_weight=sample_weight)
+    if sparse_container is not None:
+        assert_allclose(X.toarray(), original_X_data)
+    else:
+        assert_allclose(X, original_X_data)
+    assert_allclose(y, original_y_data)
+
+    if use_sw:
+        assert_allclose(sample_weight, orginal_sw_data)
+
+    # Allow inplace preprocessing of X and y
+    reg = LinearRegression(copy_X=False)
+    reg.fit(X, y, sample_weight=sample_weight)
+    if sparse_container is not None:
+        # No optimization relying on the inplace modification of sparse input
+        # data has been implemented at this time.
+        assert_allclose(X.toarray(), original_X_data)
+    else:
+        # X has been offset (and optionally rescaled by sample weights)
+        # inplace. The 0.42 threshold is arbitrary and has been found to be
+        # robust to any random seed in the admissible range.
+        assert np.linalg.norm(X - original_X_data) > 0.42
+
+    # y should not have been modified inplace by LinearRegression.fit.
+    assert_allclose(y, original_y_data)
+
+    if use_sw:
+        # Sample weights have no reason to ever be modified inplace.
+        assert_allclose(sample_weight, orginal_sw_data)
 
 
 def test_linear_regression_pd_sparse_dataframe_warning():
-    pd = pytest.importorskip('pandas')
-    # restrict the pd versions < '0.24.0' as they have a bug in is_sparse func
-    if parse_version(pd.__version__) < parse_version('0.24.0'):
-        pytest.skip("pandas 0.24+ required.")
+    pd = pytest.importorskip("pandas")
 
     # Warning is raised only when some of the columns is sparse
-    df = pd.DataFrame({'0': np.random.randn(10)})
+    df = pd.DataFrame({"0": np.random.randn(10)})
     for col in range(1, 4):
         arr = np.random.randn(10)
         arr[:8] = 0
@@ -395,49 +360,41 @@ def test_linear_regression_pd_sparse_dataframe_warning():
         reg.fit(df.iloc[:, 0:2], df.iloc[:, 3])
 
     # does not warn when the whole dataframe is sparse
-    df['0'] = pd.arrays.SparseArray(df['0'], fill_value=0)
+    df["0"] = pd.arrays.SparseArray(df["0"], fill_value=0)
     assert hasattr(df, "sparse")
 
-    with pytest.warns(None) as record:
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", UserWarning)
         reg.fit(df.iloc[:, 0:2], df.iloc[:, 3])
-    assert not record
 
 
-def test_preprocess_data():
+def test_preprocess_data(global_random_seed):
+    rng = np.random.RandomState(global_random_seed)
     n_samples = 200
     n_features = 2
     X = rng.rand(n_samples, n_features)
     y = rng.rand(n_samples)
     expected_X_mean = np.mean(X, axis=0)
-    expected_X_scale = np.std(X, axis=0) * np.sqrt(X.shape[0])
     expected_y_mean = np.mean(y, axis=0)
 
-    Xt, yt, X_mean, y_mean, X_scale = \
-        _preprocess_data(X, y, fit_intercept=False, normalize=False)
+    Xt, yt, X_mean, y_mean, X_scale = _preprocess_data(X, y, fit_intercept=False)
     assert_array_almost_equal(X_mean, np.zeros(n_features))
     assert_array_almost_equal(y_mean, 0)
     assert_array_almost_equal(X_scale, np.ones(n_features))
     assert_array_almost_equal(Xt, X)
     assert_array_almost_equal(yt, y)
 
-    Xt, yt, X_mean, y_mean, X_scale = \
-        _preprocess_data(X, y, fit_intercept=True, normalize=False)
+    Xt, yt, X_mean, y_mean, X_scale = _preprocess_data(X, y, fit_intercept=True)
     assert_array_almost_equal(X_mean, expected_X_mean)
     assert_array_almost_equal(y_mean, expected_y_mean)
     assert_array_almost_equal(X_scale, np.ones(n_features))
     assert_array_almost_equal(Xt, X - expected_X_mean)
     assert_array_almost_equal(yt, y - expected_y_mean)
 
-    Xt, yt, X_mean, y_mean, X_scale = \
-        _preprocess_data(X, y, fit_intercept=True, normalize=True)
-    assert_array_almost_equal(X_mean, expected_X_mean)
-    assert_array_almost_equal(y_mean, expected_y_mean)
-    assert_array_almost_equal(X_scale, expected_X_scale)
-    assert_array_almost_equal(Xt, (X - expected_X_mean) / expected_X_scale)
-    assert_array_almost_equal(yt, y - expected_y_mean)
-
 
-def test_preprocess_data_multioutput():
+@pytest.mark.parametrize("sparse_container", [None] + CSC_CONTAINERS)
+def test_preprocess_data_multioutput(global_random_seed, sparse_container):
+    rng = np.random.RandomState(global_random_seed)
     n_samples = 200
     n_features = 3
     n_outputs = 2
@@ -445,26 +402,21 @@ def test_preprocess_data_multioutput():
     y = rng.rand(n_samples, n_outputs)
     expected_y_mean = np.mean(y, axis=0)
 
-    args = [X, sparse.csc_matrix(X)]
-    for X in args:
-        _, yt, _, y_mean, _ = _preprocess_data(X, y, fit_intercept=False,
-                                               normalize=False)
-        assert_array_almost_equal(y_mean, np.zeros(n_outputs))
-        assert_array_almost_equal(yt, y)
+    if sparse_container is not None:
+        X = sparse_container(X)
 
-        _, yt, _, y_mean, _ = _preprocess_data(X, y, fit_intercept=True,
-                                               normalize=False)
-        assert_array_almost_equal(y_mean, expected_y_mean)
-        assert_array_almost_equal(yt, y - y_mean)
+    _, yt, _, y_mean, _ = _preprocess_data(X, y, fit_intercept=False)
+    assert_array_almost_equal(y_mean, np.zeros(n_outputs))
+    assert_array_almost_equal(yt, y)
 
-        _, yt, _, y_mean, _ = _preprocess_data(X, y, fit_intercept=True,
-                                               normalize=True)
-        assert_array_almost_equal(y_mean, expected_y_mean)
-        assert_array_almost_equal(yt, y - y_mean)
+    _, yt, _, y_mean, _ = _preprocess_data(X, y, fit_intercept=True)
+    assert_array_almost_equal(y_mean, expected_y_mean)
+    assert_array_almost_equal(yt, y - y_mean)
 
 
-@pytest.mark.parametrize("is_sparse", [False, True])
-def test_preprocess_data_weighted(is_sparse):
+@pytest.mark.parametrize("sparse_container", [None] + CSR_CONTAINERS)
+def test_preprocess_data_weighted(sparse_container, global_random_seed):
+    rng = np.random.RandomState(global_random_seed)
     n_samples = 200
     n_features = 4
     # Generate random data with 50% of zero values to make sure
@@ -472,17 +424,17 @@ def test_preprocess_data_weighted(is_sparse):
     # shifts the mean value for each columns in X further away from
     # zero.
     X = rng.rand(n_samples, n_features)
-    X[X < 0.5] = 0.
+    X[X < 0.5] = 0.0
 
     # Scale the first feature of X to be 10 larger than the other to
     # better check the impact of feature scaling.
     X[:, 0] *= 10
 
     # Constant non-zero feature.
-    X[:, 2] = 1.
+    X[:, 2] = 1.0
 
     # Constant zero feature (non-materialized in the sparse case)
-    X[:, 3] = 0.
+    X[:, 3] = 0.0
     y = rng.rand(n_samples)
 
     sample_weight = rng.rand(n_samples)
@@ -490,148 +442,96 @@ def test_preprocess_data_weighted(is_sparse):
     expected_y_mean = np.average(y, axis=0, weights=sample_weight)
 
     X_sample_weight_avg = np.average(X, weights=sample_weight, axis=0)
-    X_sample_weight_var = np.average((X - X_sample_weight_avg)**2,
-                                     weights=sample_weight,
-                                     axis=0)
+    X_sample_weight_var = np.average(
+        (X - X_sample_weight_avg) ** 2, weights=sample_weight, axis=0
+    )
     constant_mask = X_sample_weight_var < 10 * np.finfo(X.dtype).eps
     assert_array_equal(constant_mask, [0, 0, 1, 1])
-    expected_X_scale = np.sqrt(X_sample_weight_var) * np.sqrt(n_samples)
+    expected_X_scale = np.sqrt(X_sample_weight_var) * np.sqrt(sample_weight.sum())
 
     # near constant features should not be scaled
     expected_X_scale[constant_mask] = 1
 
-    if is_sparse:
-        X = sparse.csr_matrix(X)
+    if sparse_container is not None:
+        X = sparse_container(X)
 
     # normalize is False
-    Xt, yt, X_mean, y_mean, X_scale = \
-        _preprocess_data(X, y, fit_intercept=True, normalize=False,
-                         sample_weight=sample_weight, return_mean=True)
+    Xt, yt, X_mean, y_mean, X_scale = _preprocess_data(
+        X,
+        y,
+        fit_intercept=True,
+        sample_weight=sample_weight,
+    )
     assert_array_almost_equal(X_mean, expected_X_mean)
     assert_array_almost_equal(y_mean, expected_y_mean)
     assert_array_almost_equal(X_scale, np.ones(n_features))
-    if is_sparse:
+    if sparse_container is not None:
         assert_array_almost_equal(Xt.toarray(), X.toarray())
     else:
         assert_array_almost_equal(Xt, X - expected_X_mean)
     assert_array_almost_equal(yt, y - expected_y_mean)
 
-    # normalize is True
-    Xt, yt, X_mean, y_mean, X_scale = \
-        _preprocess_data(X, y, fit_intercept=True, normalize=True,
-                         sample_weight=sample_weight, return_mean=True)
-
-    assert_array_almost_equal(X_mean, expected_X_mean)
-    assert_array_almost_equal(y_mean, expected_y_mean)
-    assert_array_almost_equal(X_scale, expected_X_scale)
-
-    if is_sparse:
-        # X is not centered
-        assert_array_almost_equal(
-            Xt.toarray(), X.toarray() / expected_X_scale
-        )
-    else:
-        assert_array_almost_equal(
-            Xt, (X - expected_X_mean) / expected_X_scale
-        )
-
-    # _preprocess_data with normalize=True scales the data by the feature-wise
-    # euclidean norms while StandardScaler scales the data by the feature-wise
-    # standard deviations.
-    # The two are equivalent up to a ratio of np.sqrt(n_samples).
-    if is_sparse:
-        scaler = StandardScaler(with_mean=False).fit(
-            X, sample_weight=sample_weight)
-
-        # Non-constant features are scaled similarly with np.sqrt(n_samples)
-        assert_array_almost_equal(
-            scaler.transform(X).toarray()[:, :2] / np.sqrt(n_samples),
-            Xt.toarray()[:, :2]
-        )
 
-        # Constant features go through un-scaled.
-        assert_array_almost_equal(
-            scaler.transform(X).toarray()[:, 2:],
-            Xt.toarray()[:, 2:]
-        )
-    else:
-        scaler = StandardScaler(with_mean=True).fit(
-            X, sample_weight=sample_weight)
-        assert_array_almost_equal(scaler.mean_, X_mean)
-        assert_array_almost_equal(scaler.transform(X) / np.sqrt(n_samples), Xt)
-    assert_array_almost_equal(yt, y - expected_y_mean)
-
-
-def test_sparse_preprocess_data_with_return_mean():
+@pytest.mark.parametrize("lil_container", LIL_CONTAINERS)
+def test_sparse_preprocess_data_offsets(global_random_seed, lil_container):
+    rng = np.random.RandomState(global_random_seed)
     n_samples = 200
     n_features = 2
-    # random_state not supported yet in sparse.rand
-    X = sparse.rand(n_samples, n_features, density=.5)  # , random_state=rng
-    X = X.tolil()
+    X = sparse.rand(n_samples, n_features, density=0.5, random_state=rng)
+    X = lil_container(X)
     y = rng.rand(n_samples)
     XA = X.toarray()
-    expected_X_scale = np.std(XA, axis=0) * np.sqrt(X.shape[0])
 
-    Xt, yt, X_mean, y_mean, X_scale = \
-        _preprocess_data(X, y, fit_intercept=False, normalize=False,
-                         return_mean=True)
+    Xt, yt, X_mean, y_mean, X_scale = _preprocess_data(X, y, fit_intercept=False)
     assert_array_almost_equal(X_mean, np.zeros(n_features))
     assert_array_almost_equal(y_mean, 0)
     assert_array_almost_equal(X_scale, np.ones(n_features))
-    assert_array_almost_equal(Xt.A, XA)
+    assert_array_almost_equal(Xt.toarray(), XA)
     assert_array_almost_equal(yt, y)
 
-    Xt, yt, X_mean, y_mean, X_scale = \
-        _preprocess_data(X, y, fit_intercept=True, normalize=False,
-                         return_mean=True)
+    Xt, yt, X_mean, y_mean, X_scale = _preprocess_data(X, y, fit_intercept=True)
     assert_array_almost_equal(X_mean, np.mean(XA, axis=0))
     assert_array_almost_equal(y_mean, np.mean(y, axis=0))
     assert_array_almost_equal(X_scale, np.ones(n_features))
-    assert_array_almost_equal(Xt.A, XA)
-    assert_array_almost_equal(yt, y - np.mean(y, axis=0))
-
-    Xt, yt, X_mean, y_mean, X_scale = \
-        _preprocess_data(X, y, fit_intercept=True, normalize=True,
-                         return_mean=True)
-    assert_array_almost_equal(X_mean, np.mean(XA, axis=0))
-    assert_array_almost_equal(y_mean, np.mean(y, axis=0))
-    assert_array_almost_equal(X_scale, expected_X_scale)
-    assert_array_almost_equal(Xt.A, XA / expected_X_scale)
+    assert_array_almost_equal(Xt.toarray(), XA)
     assert_array_almost_equal(yt, y - np.mean(y, axis=0))
 
 
-def test_csr_preprocess_data():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_csr_preprocess_data(csr_container):
     # Test output format of _preprocess_data, when input is csr
     X, y = make_regression()
     X[X < 2.5] = 0.0
-    csr = sparse.csr_matrix(X)
-    csr_, y, _, _, _ = _preprocess_data(csr, y, True)
-    assert csr_.getformat() == 'csr'
+    csr = csr_container(X)
+    csr_, y, _, _, _ = _preprocess_data(csr, y, fit_intercept=True)
+    assert csr_.format == "csr"
 
 
-@pytest.mark.parametrize('is_sparse', (True, False))
-@pytest.mark.parametrize('to_copy', (True, False))
-def test_preprocess_copy_data_no_checks(is_sparse, to_copy):
+@pytest.mark.parametrize("sparse_container", [None] + CSR_CONTAINERS)
+@pytest.mark.parametrize("to_copy", (True, False))
+def test_preprocess_copy_data_no_checks(sparse_container, to_copy):
     X, y = make_regression()
     X[X < 2.5] = 0.0
 
-    if is_sparse:
-        X = sparse.csr_matrix(X)
+    if sparse_container is not None:
+        X = sparse_container(X)
 
-    X_, y_, _, _, _ = _preprocess_data(X, y, True,
-                                       copy=to_copy, check_input=False)
+    X_, y_, _, _, _ = _preprocess_data(
+        X, y, fit_intercept=True, copy=to_copy, check_input=False
+    )
 
-    if to_copy and is_sparse:
+    if to_copy and sparse_container is not None:
         assert not np.may_share_memory(X_.data, X.data)
     elif to_copy:
         assert not np.may_share_memory(X_, X)
-    elif is_sparse:
+    elif sparse_container is not None:
         assert np.may_share_memory(X_.data, X.data)
     else:
         assert np.may_share_memory(X_, X)
 
 
-def test_dtype_preprocess_data():
+def test_dtype_preprocess_data(global_random_seed):
+    rng = np.random.RandomState(global_random_seed)
     n_samples = 200
     n_features = 2
     X = rng.rand(n_samples, n_features)
@@ -643,62 +543,70 @@ def test_dtype_preprocess_data():
     y_64 = np.asarray(y, dtype=np.float64)
 
     for fit_intercept in [True, False]:
-        for normalize in [True, False]:
-
-            Xt_32, yt_32, X_mean_32, y_mean_32, X_scale_32 = _preprocess_data(
-                X_32, y_32, fit_intercept=fit_intercept, normalize=normalize,
-                return_mean=True)
-
-            Xt_64, yt_64, X_mean_64, y_mean_64, X_scale_64 = _preprocess_data(
-                X_64, y_64, fit_intercept=fit_intercept, normalize=normalize,
-                return_mean=True)
-
-            Xt_3264, yt_3264, X_mean_3264, y_mean_3264, X_scale_3264 = (
-                _preprocess_data(X_32, y_64, fit_intercept=fit_intercept,
-                                 normalize=normalize, return_mean=True))
-
-            Xt_6432, yt_6432, X_mean_6432, y_mean_6432, X_scale_6432 = (
-                _preprocess_data(X_64, y_32, fit_intercept=fit_intercept,
-                                 normalize=normalize, return_mean=True))
-
-            assert Xt_32.dtype == np.float32
-            assert yt_32.dtype == np.float32
-            assert X_mean_32.dtype == np.float32
-            assert y_mean_32.dtype == np.float32
-            assert X_scale_32.dtype == np.float32
-
-            assert Xt_64.dtype == np.float64
-            assert yt_64.dtype == np.float64
-            assert X_mean_64.dtype == np.float64
-            assert y_mean_64.dtype == np.float64
-            assert X_scale_64.dtype == np.float64
-
-            assert Xt_3264.dtype == np.float32
-            assert yt_3264.dtype == np.float32
-            assert X_mean_3264.dtype == np.float32
-            assert y_mean_3264.dtype == np.float32
-            assert X_scale_3264.dtype == np.float32
-
-            assert Xt_6432.dtype == np.float64
-            assert yt_6432.dtype == np.float64
-            assert X_mean_6432.dtype == np.float64
-            assert y_mean_6432.dtype == np.float64
-            assert X_scale_6432.dtype == np.float64
-
-            assert X_32.dtype == np.float32
-            assert y_32.dtype == np.float32
-            assert X_64.dtype == np.float64
-            assert y_64.dtype == np.float64
-
-            assert_array_almost_equal(Xt_32, Xt_64)
-            assert_array_almost_equal(yt_32, yt_64)
-            assert_array_almost_equal(X_mean_32, X_mean_64)
-            assert_array_almost_equal(y_mean_32, y_mean_64)
-            assert_array_almost_equal(X_scale_32, X_scale_64)
-
-
-@pytest.mark.parametrize('n_targets', [None, 2])
-def test_rescale_data_dense(n_targets):
+        Xt_32, yt_32, X_mean_32, y_mean_32, X_scale_32 = _preprocess_data(
+            X_32,
+            y_32,
+            fit_intercept=fit_intercept,
+        )
+
+        Xt_64, yt_64, X_mean_64, y_mean_64, X_scale_64 = _preprocess_data(
+            X_64,
+            y_64,
+            fit_intercept=fit_intercept,
+        )
+
+        Xt_3264, yt_3264, X_mean_3264, y_mean_3264, X_scale_3264 = _preprocess_data(
+            X_32,
+            y_64,
+            fit_intercept=fit_intercept,
+        )
+
+        Xt_6432, yt_6432, X_mean_6432, y_mean_6432, X_scale_6432 = _preprocess_data(
+            X_64,
+            y_32,
+            fit_intercept=fit_intercept,
+        )
+
+        assert Xt_32.dtype == np.float32
+        assert yt_32.dtype == np.float32
+        assert X_mean_32.dtype == np.float32
+        assert y_mean_32.dtype == np.float32
+        assert X_scale_32.dtype == np.float32
+
+        assert Xt_64.dtype == np.float64
+        assert yt_64.dtype == np.float64
+        assert X_mean_64.dtype == np.float64
+        assert y_mean_64.dtype == np.float64
+        assert X_scale_64.dtype == np.float64
+
+        assert Xt_3264.dtype == np.float32
+        assert yt_3264.dtype == np.float32
+        assert X_mean_3264.dtype == np.float32
+        assert y_mean_3264.dtype == np.float32
+        assert X_scale_3264.dtype == np.float32
+
+        assert Xt_6432.dtype == np.float64
+        assert yt_6432.dtype == np.float64
+        assert X_mean_6432.dtype == np.float64
+        assert y_mean_6432.dtype == np.float64
+        assert X_scale_6432.dtype == np.float64
+
+        assert X_32.dtype == np.float32
+        assert y_32.dtype == np.float32
+        assert X_64.dtype == np.float64
+        assert y_64.dtype == np.float64
+
+        assert_array_almost_equal(Xt_32, Xt_64)
+        assert_array_almost_equal(yt_32, yt_64)
+        assert_array_almost_equal(X_mean_32, X_mean_64)
+        assert_array_almost_equal(y_mean_32, y_mean_64)
+        assert_array_almost_equal(X_scale_32, X_scale_64)
+
+
+@pytest.mark.parametrize("n_targets", [None, 2])
+@pytest.mark.parametrize("sparse_container", [None] + CSR_CONTAINERS)
+def test_rescale_data(n_targets, sparse_container, global_random_seed):
+    rng = np.random.RandomState(global_random_seed)
     n_samples = 200
     n_features = 2
 
@@ -708,27 +616,48 @@ def test_rescale_data_dense(n_targets):
         y = rng.rand(n_samples)
     else:
         y = rng.rand(n_samples, n_targets)
-    rescaled_X, rescaled_y = _rescale_data(X, y, sample_weight)
-    rescaled_X2 = X * np.sqrt(sample_weight)[:, np.newaxis]
+
+    expected_sqrt_sw = np.sqrt(sample_weight)
+    expected_rescaled_X = X * expected_sqrt_sw[:, np.newaxis]
+
     if n_targets is None:
-        rescaled_y2 = y * np.sqrt(sample_weight)
+        expected_rescaled_y = y * expected_sqrt_sw
     else:
-        rescaled_y2 = y * np.sqrt(sample_weight)[:, np.newaxis]
-    assert_array_almost_equal(rescaled_X, rescaled_X2)
-    assert_array_almost_equal(rescaled_y, rescaled_y2)
+        expected_rescaled_y = y * expected_sqrt_sw[:, np.newaxis]
+
+    if sparse_container is not None:
+        X = sparse_container(X)
+        if n_targets is None:
+            y = sparse_container(y.reshape(-1, 1))
+        else:
+            y = sparse_container(y)
+
+    rescaled_X, rescaled_y, sqrt_sw = _rescale_data(X, y, sample_weight)
+
+    assert_allclose(sqrt_sw, expected_sqrt_sw)
+
+    if sparse_container is not None:
+        rescaled_X = rescaled_X.toarray()
+        rescaled_y = rescaled_y.toarray()
+        if n_targets is None:
+            rescaled_y = rescaled_y.ravel()
 
+    assert_allclose(rescaled_X, expected_rescaled_X)
+    assert_allclose(rescaled_y, expected_rescaled_y)
 
-def test_fused_types_make_dataset():
+
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_fused_types_make_dataset(csr_container):
     iris = load_iris()
 
     X_32 = iris.data.astype(np.float32)
     y_32 = iris.target.astype(np.float32)
-    X_csr_32 = sparse.csr_matrix(X_32)
+    X_csr_32 = csr_container(X_32)
     sample_weight_32 = np.arange(y_32.size, dtype=np.float32)
 
     X_64 = iris.data.astype(np.float64)
     y_64 = iris.target.astype(np.float64)
-    X_csr_64 = sparse.csr_matrix(X_64)
+    X_csr_64 = csr_container(X_64)
     sample_weight_64 = np.arange(y_64.size, dtype=np.float64)
 
     # array
@@ -761,3 +690,102 @@ def test_fused_types_make_dataset():
     assert_array_equal(xi_data_64, xicsr_data_64)
     assert_array_equal(yi_32, yicsr_32)
     assert_array_equal(yi_64, yicsr_64)
+
+
+@pytest.mark.parametrize("X_shape", [(10, 5), (10, 20), (100, 100)])
+@pytest.mark.parametrize(
+    "sparse_container",
+    [None]
+    + [
+        pytest.param(
+            container,
+            marks=pytest.mark.xfail(
+                reason="Known to fail for CSR arrays, see issue #30131."
+            ),
+        )
+        for container in CSR_CONTAINERS
+    ],
+)
+@pytest.mark.parametrize("fit_intercept", [False, True])
+def test_linear_regression_sample_weight_consistency(
+    X_shape, sparse_container, fit_intercept, global_random_seed
+):
+    """Test that the impact of sample_weight is consistent.
+
+    Note that this test is stricter than the common test
+    check_sample_weight_equivalence alone and also tests sparse X.
+    It is very similar to test_enet_sample_weight_consistency.
+    """
+    rng = np.random.RandomState(global_random_seed)
+    n_samples, n_features = X_shape
+
+    X = rng.rand(n_samples, n_features)
+    y = rng.rand(n_samples)
+    if sparse_container is not None:
+        X = sparse_container(X)
+    params = dict(fit_intercept=fit_intercept)
+
+    reg = LinearRegression(**params).fit(X, y, sample_weight=None)
+    coef = reg.coef_.copy()
+    if fit_intercept:
+        intercept = reg.intercept_
+
+    # 1) sample_weight=np.ones(..) must be equivalent to sample_weight=None,
+    # a special case of check_sample_weight_equivalence(name, reg), but we also
+    # test with sparse input.
+    sample_weight = np.ones_like(y)
+    reg.fit(X, y, sample_weight=sample_weight)
+    assert_allclose(reg.coef_, coef, rtol=1e-6)
+    if fit_intercept:
+        assert_allclose(reg.intercept_, intercept)
+
+    # 2) sample_weight=None should be equivalent to sample_weight = number
+    sample_weight = 123.0
+    reg.fit(X, y, sample_weight=sample_weight)
+    assert_allclose(reg.coef_, coef, rtol=1e-6)
+    if fit_intercept:
+        assert_allclose(reg.intercept_, intercept)
+
+    # 3) scaling of sample_weight should have no effect, cf. np.average()
+    sample_weight = rng.uniform(low=0.01, high=2, size=X.shape[0])
+    reg = reg.fit(X, y, sample_weight=sample_weight)
+    coef = reg.coef_.copy()
+    if fit_intercept:
+        intercept = reg.intercept_
+
+    reg.fit(X, y, sample_weight=np.pi * sample_weight)
+    assert_allclose(reg.coef_, coef, rtol=1e-6 if sparse_container is None else 1e-5)
+    if fit_intercept:
+        assert_allclose(reg.intercept_, intercept)
+
+    # 4) setting elements of sample_weight to 0 is equivalent to removing these samples
+    sample_weight_0 = sample_weight.copy()
+    sample_weight_0[-5:] = 0
+    y[-5:] *= 1000  # to make excluding those samples important
+    reg.fit(X, y, sample_weight=sample_weight_0)
+    coef_0 = reg.coef_.copy()
+    if fit_intercept:
+        intercept_0 = reg.intercept_
+    reg.fit(X[:-5], y[:-5], sample_weight=sample_weight[:-5])
+    assert_allclose(reg.coef_, coef_0, rtol=1e-5)
+    if fit_intercept:
+        assert_allclose(reg.intercept_, intercept_0)
+
+    # 5) check that multiplying sample_weight by 2 is equivalent to repeating
+    # corresponding samples twice
+    if sparse_container is not None:
+        X2 = sparse.vstack([X, X[: n_samples // 2]], format="csc")
+    else:
+        X2 = np.concatenate([X, X[: n_samples // 2]], axis=0)
+    y2 = np.concatenate([y, y[: n_samples // 2]])
+    sample_weight_1 = sample_weight.copy()
+    sample_weight_1[: n_samples // 2] *= 2
+    sample_weight_2 = np.concatenate(
+        [sample_weight, sample_weight[: n_samples // 2]], axis=0
+    )
+
+    reg1 = LinearRegression(**params).fit(X, y, sample_weight=sample_weight_1)
+    reg2 = LinearRegression(**params).fit(X2, y2, sample_weight=sample_weight_2)
+    assert_allclose(reg1.coef_, reg2.coef_, rtol=1e-6)
+    if fit_intercept:
+        assert_allclose(reg1.intercept_, reg2.intercept_)
diff --git a/sklearn/linear_model/tests/test_bayes.py b/sklearn/linear_model/tests/test_bayes.py
index 529702ff752ac..9f7fabb749f52 100644
--- a/sklearn/linear_model/tests/test_bayes.py
+++ b/sklearn/linear_model/tests/test_bayes.py
@@ -1,37 +1,26 @@
-# Author: Alexandre Gramfort 
-#         Fabian Pedregosa 
-#
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 from math import log
 
 import numpy as np
-from scipy.linalg import pinvh
 import pytest
 
-
-from sklearn.utils._testing import assert_array_almost_equal
-from sklearn.utils._testing import assert_almost_equal
-from sklearn.utils._testing import assert_array_less
-from sklearn.utils._testing import assert_raise_message
-from sklearn.utils import check_random_state
-from sklearn.linear_model import BayesianRidge, ARDRegression
-from sklearn.linear_model import Ridge
 from sklearn import datasets
+from sklearn.linear_model import ARDRegression, BayesianRidge, Ridge
+from sklearn.utils import check_random_state
+from sklearn.utils._testing import (
+    _convert_container,
+    assert_allclose,
+    assert_almost_equal,
+    assert_array_almost_equal,
+    assert_array_less,
+)
 from sklearn.utils.extmath import fast_logdet
 
 diabetes = datasets.load_diabetes()
 
 
-def test_n_iter():
-    """Check value of n_iter."""
-    X = np.array([[1], [2], [6], [8], [10]])
-    y = np.array([1, 2, 6, 8, 10])
-    clf = BayesianRidge(n_iter=0)
-    msg = "n_iter should be greater than or equal to 1."
-    assert_raise_message(ValueError, msg, clf.fit, X, y)
-
-
 def test_bayesian_ridge_scores():
     """Check scores attribute shape"""
     X, y = diabetes.data, diabetes.target
@@ -60,8 +49,8 @@ def test_bayesian_ridge_score_values():
     n_samples = X.shape[0]
     # check with initial values of alpha and lambda (see code for the values)
     eps = np.finfo(np.float64).eps
-    alpha_ = 1. / (np.var(y) + eps)
-    lambda_ = 1.
+    alpha_ = 1.0 / (np.var(y) + eps)
+    lambda_ = 1.0
 
     # value of the parameters of the Gamma hyperpriors
     alpha_1 = 0.1
@@ -72,15 +61,22 @@ def test_bayesian_ridge_score_values():
     # compute score using formula of docstring
     score = lambda_1 * log(lambda_) - lambda_2 * lambda_
     score += alpha_1 * log(alpha_) - alpha_2 * alpha_
-    M = 1. / alpha_ * np.eye(n_samples) + 1. / lambda_ * np.dot(X, X.T)
-    M_inv = pinvh(M)
-    score += - 0.5 * (fast_logdet(M) + np.dot(y.T, np.dot(M_inv, y)) +
-                      n_samples * log(2 * np.pi))
+    M = 1.0 / alpha_ * np.eye(n_samples) + 1.0 / lambda_ * np.dot(X, X.T)
+    M_inv_dot_y = np.linalg.solve(M, y)
+    score += -0.5 * (
+        fast_logdet(M) + np.dot(y.T, M_inv_dot_y) + n_samples * log(2 * np.pi)
+    )
 
     # compute score with BayesianRidge
-    clf = BayesianRidge(alpha_1=alpha_1, alpha_2=alpha_2,
-                        lambda_1=lambda_1, lambda_2=lambda_2,
-                        n_iter=1, fit_intercept=False, compute_score=True)
+    clf = BayesianRidge(
+        alpha_1=alpha_1,
+        alpha_2=alpha_2,
+        lambda_1=lambda_1,
+        lambda_2=lambda_2,
+        max_iter=1,
+        fit_intercept=False,
+        compute_score=True,
+    )
     clf.fit(X, y)
 
     assert_almost_equal(clf.scores_[0], score, decimal=9)
@@ -99,6 +95,22 @@ def test_bayesian_ridge_parameter():
     assert_almost_equal(rr_model.intercept_, br_model.intercept_)
 
 
+@pytest.mark.parametrize("n_samples, n_features", [(10, 20), (20, 10)])
+def test_bayesian_covariance_matrix(n_samples, n_features, global_random_seed):
+    """Check the posterior covariance matrix sigma_
+
+    Non-regression test for https://github.com/scikit-learn/scikit-learn/issues/31093
+    """
+    X, y = datasets.make_regression(
+        n_samples, n_features, random_state=global_random_seed
+    )
+    reg = BayesianRidge(fit_intercept=False).fit(X, y)
+    covariance_matrix = np.linalg.inv(
+        reg.lambda_ * np.identity(n_features) + reg.alpha_ * np.dot(X.T, X)
+    )
+    assert_allclose(reg.sigma_, covariance_matrix, rtol=1e-6)
+
+
 def test_bayesian_sample_weights():
     # Test correctness of the sample_weights method
     X = np.array([[1, 1], [3, 4], [5, 7], [4, 1], [2, 6], [3, 10], [3, 2]])
@@ -109,7 +121,8 @@ def test_bayesian_sample_weights():
     # lambda_ and alpha_ from the Bayesian Ridge model must be identical
     br_model = BayesianRidge(compute_score=True).fit(X, y, sample_weight=w)
     rr_model = Ridge(alpha=br_model.lambda_ / br_model.alpha_).fit(
-        X, y, sample_weight=w)
+        X, y, sample_weight=w
+    )
     assert_array_almost_equal(rr_model.coef_, br_model.coef_)
     assert_almost_equal(rr_model.intercept_, br_model.intercept_)
 
@@ -129,14 +142,14 @@ def test_toy_bayesian_ridge_object():
 def test_bayesian_initial_params():
     # Test BayesianRidge with initial values (alpha_init, lambda_init)
     X = np.vander(np.linspace(0, 4, 5), 4)
-    y = np.array([0., 1., 0., -1., 0.])    # y = (x^3 - 6x^2 + 8x) / 3
+    y = np.array([0.0, 1.0, 0.0, -1.0, 0.0])  # y = (x^3 - 6x^2 + 8x) / 3
 
     # In this case, starting from the default initial values will increase
     # the bias of the fitted curve. So, lambda_init should be small.
-    reg = BayesianRidge(alpha_init=1., lambda_init=1e-3)
+    reg = BayesianRidge(alpha_init=1.0, lambda_init=1e-3)
     # Check the R2 score nearly equals to one.
     r2 = reg.fit(X, y).score(X, y)
-    assert_almost_equal(r2, 1.)
+    assert_almost_equal(r2, 1.0)
 
 
 def test_prediction_bayesian_ridge_ard_with_constant_input():
@@ -147,10 +160,8 @@ def test_prediction_bayesian_ridge_ard_with_constant_input():
     random_state = check_random_state(42)
     constant_value = random_state.rand()
     X = random_state.random_sample((n_samples, n_features))
-    y = np.full(n_samples, constant_value,
-                dtype=np.array(constant_value).dtype)
-    expected = np.full(n_samples, constant_value,
-                       dtype=np.array(constant_value).dtype)
+    y = np.full(n_samples, constant_value, dtype=np.array(constant_value).dtype)
+    expected = np.full(n_samples, constant_value, dtype=np.array(constant_value).dtype)
 
     for clf in [BayesianRidge(), ARDRegression()]:
         y_pred = clf.fit(X, y).predict(X)
@@ -166,8 +177,7 @@ def test_std_bayesian_ridge_ard_with_constant_input():
     random_state = check_random_state(42)
     constant_value = random_state.rand()
     X = random_state.random_sample((n_samples, n_features))
-    y = np.full(n_samples, constant_value,
-                dtype=np.array(constant_value).dtype)
+    y = np.full(n_samples, constant_value, dtype=np.array(constant_value).dtype)
     expected_upper_boundary = 0.01
 
     for clf in [BayesianRidge(), ARDRegression()]:
@@ -178,10 +188,9 @@ def test_std_bayesian_ridge_ard_with_constant_input():
 def test_update_of_sigma_in_ard():
     # Checks that `sigma_` is updated correctly after the last iteration
     # of the ARDRegression algorithm. See issue #10128.
-    X = np.array([[1, 0],
-                  [0, 0]])
+    X = np.array([[1, 0], [0, 0]])
     y = np.array([0, 0])
-    clf = ARDRegression(n_iter=1)
+    clf = ARDRegression(max_iter=1)
     clf.fit(X, y)
     # With the inputs above, ARDRegression prunes both of the two coefficients
     # in the first iteration. Hence, the expected shape of `sigma_` is (0, 0).
@@ -202,12 +211,11 @@ def test_toy_ard_object():
     assert_array_almost_equal(clf.predict(test), [1, 3, 4], 2)
 
 
-@pytest.mark.parametrize('seed', range(100))
-@pytest.mark.parametrize('n_samples, n_features', ((10, 100), (100, 10)))
-def test_ard_accuracy_on_easy_problem(seed, n_samples, n_features):
+@pytest.mark.parametrize("n_samples, n_features", ((10, 100), (100, 10)))
+def test_ard_accuracy_on_easy_problem(global_random_seed, n_samples, n_features):
     # Check that ARD converges with reasonable accuracy on an easy problem
     # (Github issue #14055)
-    X = np.random.RandomState(seed=seed).normal(size=(250, 3))
+    X = np.random.RandomState(global_random_seed).normal(size=(250, 3))
     y = X[:, 1]
 
     regressor = ARDRegression()
@@ -217,7 +225,8 @@ def test_ard_accuracy_on_easy_problem(seed, n_samples, n_features):
     assert abs_coef_error < 1e-10
 
 
-def test_return_std():
+@pytest.mark.parametrize("constructor_name", ["array", "dataframe"])
+def test_return_std(constructor_name):
     # Test return_std option for both Bayesian regressors
     def f(X):
         return np.dot(X, w) + b
@@ -233,7 +242,10 @@ def f_noise(X, noise_mult):
     b = 1.0
 
     X = np.random.random((n_train, d))
+    X = _convert_container(X, constructor_name)
+
     X_test = np.random.random((n_test, d))
+    X_test = _convert_container(X_test, constructor_name)
 
     for decimal, noise_mult in enumerate([1, 0.1, 0.01]):
         y = f_noise(X, noise_mult)
@@ -249,13 +261,12 @@ def f_noise(X, noise_mult):
         assert_array_almost_equal(y_std2, noise_mult, decimal=decimal)
 
 
-@pytest.mark.parametrize('seed', range(10))
-def test_update_sigma(seed):
+def test_update_sigma(global_random_seed):
     # make sure the two update_sigma() helpers are equivalent. The woodbury
     # formula is used when n_samples < n_features, and the other one is used
     # otherwise.
 
-    rng = np.random.RandomState(seed)
+    rng = np.random.RandomState(global_random_seed)
 
     # set n_samples == n_features to avoid instability issues when inverting
     # the matrices. Using the woodbury formula would be unstable when
@@ -274,11 +285,30 @@ def test_update_sigma(seed):
     np.testing.assert_allclose(sigma, sigma_woodbury)
 
 
-def test_ard_regression_predict_normalize_true():
-    """Check that we can predict with `normalize=True` and `return_std=True`.
-    Non-regression test for:
-    https://github.com/scikit-learn/scikit-learn/issues/18605
-    """
-    clf = ARDRegression(normalize=True)
-    clf.fit([[0, 0], [1, 1], [2, 2]], [0, 1, 2])
-    clf.predict([[1, 1]], return_std=True)
+@pytest.mark.parametrize("dtype", [np.float32, np.float64])
+@pytest.mark.parametrize("Estimator", [BayesianRidge, ARDRegression])
+def test_dtype_match(dtype, Estimator):
+    # Test that np.float32 input data is not cast to np.float64 when possible
+    X = np.array([[1, 1], [3, 4], [5, 7], [4, 1], [2, 6], [3, 10], [3, 2]], dtype=dtype)
+    y = np.array([1, 2, 3, 2, 0, 4, 5]).T
+
+    model = Estimator()
+    # check type consistency
+    model.fit(X, y)
+    attributes = ["coef_", "sigma_"]
+    for attribute in attributes:
+        assert getattr(model, attribute).dtype == X.dtype
+
+    y_mean, y_std = model.predict(X, return_std=True)
+    assert y_mean.dtype == X.dtype
+    assert y_std.dtype == X.dtype
+
+
+@pytest.mark.parametrize("Estimator", [BayesianRidge, ARDRegression])
+def test_dtype_correctness(Estimator):
+    X = np.array([[1, 1], [3, 4], [5, 7], [4, 1], [2, 6], [3, 10], [3, 2]])
+    y = np.array([1, 2, 3, 2, 0, 4, 5]).T
+    model = Estimator()
+    coef_32 = model.fit(X.astype(np.float32), y).coef_
+    coef_64 = model.fit(X.astype(np.float64), y).coef_
+    np.testing.assert_allclose(coef_32, coef_64, rtol=1e-4)
diff --git a/sklearn/linear_model/tests/test_common.py b/sklearn/linear_model/tests/test_common.py
new file mode 100644
index 0000000000000..2483a26644cbb
--- /dev/null
+++ b/sklearn/linear_model/tests/test_common.py
@@ -0,0 +1,234 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+import inspect
+
+import numpy as np
+import pytest
+
+from sklearn.base import is_classifier
+from sklearn.datasets import make_classification, make_low_rank_matrix, make_regression
+from sklearn.linear_model import (
+    ARDRegression,
+    BayesianRidge,
+    ElasticNet,
+    ElasticNetCV,
+    GammaRegressor,
+    HuberRegressor,
+    Lars,
+    LarsCV,
+    Lasso,
+    LassoCV,
+    LassoLars,
+    LassoLarsCV,
+    LassoLarsIC,
+    LinearRegression,
+    LogisticRegression,
+    LogisticRegressionCV,
+    MultiTaskElasticNet,
+    MultiTaskElasticNetCV,
+    MultiTaskLasso,
+    MultiTaskLassoCV,
+    OrthogonalMatchingPursuit,
+    OrthogonalMatchingPursuitCV,
+    PassiveAggressiveClassifier,
+    PassiveAggressiveRegressor,
+    Perceptron,
+    PoissonRegressor,
+    Ridge,
+    RidgeClassifier,
+    RidgeClassifierCV,
+    RidgeCV,
+    SGDClassifier,
+    SGDRegressor,
+    TheilSenRegressor,
+    TweedieRegressor,
+)
+from sklearn.preprocessing import MinMaxScaler
+from sklearn.svm import LinearSVC, LinearSVR
+from sklearn.utils._testing import set_random_state
+
+
+# Note: GammaRegressor() and TweedieRegressor(power != 1) have a non-canonical link.
+@pytest.mark.parametrize(
+    "model",
+    [
+        ARDRegression(),
+        BayesianRidge(),
+        ElasticNet(),
+        ElasticNetCV(),
+        Lars(),
+        LarsCV(),
+        Lasso(),
+        LassoCV(),
+        LassoLarsCV(),
+        LassoLarsIC(),
+        LinearRegression(),
+        # TODO: FIx SAGA which fails badly with sample_weights.
+        # This is a known limitation, see:
+        # https://github.com/scikit-learn/scikit-learn/issues/21305
+        pytest.param(
+            LogisticRegression(
+                penalty="elasticnet", solver="saga", l1_ratio=0.5, tol=1e-15
+            ),
+            marks=pytest.mark.xfail(reason="Missing importance sampling scheme"),
+        ),
+        LogisticRegressionCV(tol=1e-6),
+        MultiTaskElasticNet(),
+        MultiTaskElasticNetCV(),
+        MultiTaskLasso(),
+        MultiTaskLassoCV(),
+        OrthogonalMatchingPursuit(),
+        OrthogonalMatchingPursuitCV(),
+        PoissonRegressor(),
+        Ridge(),
+        RidgeCV(),
+        pytest.param(
+            SGDRegressor(tol=1e-15),
+            marks=pytest.mark.xfail(reason="Insufficient precision."),
+        ),
+        SGDRegressor(penalty="elasticnet", max_iter=10_000),
+        TweedieRegressor(power=0),  # same as Ridge
+    ],
+    ids=lambda x: x.__class__.__name__,
+)
+@pytest.mark.parametrize("with_sample_weight", [False, True])
+def test_balance_property(model, with_sample_weight, global_random_seed):
+    # Test that sum(y_predicted) == sum(y_observed) on the training set.
+    # This must hold for all linear models with deviance of an exponential disperson
+    # family as loss and the corresponding canonical link if fit_intercept=True.
+    # Examples:
+    #     - squared error and identity link (most linear models)
+    #     - Poisson deviance with log link
+    #     - log loss with logit link
+    # This is known as balance property or unconditional calibration/unbiasedness.
+    # For reference, see Corollary 3.18, 3.20 and Chapter 5.1.5 of
+    # M.V. Wuthrich and M. Merz, "Statistical Foundations of Actuarial Learning and its
+    # Applications" (June 3, 2022). http://doi.org/10.2139/ssrn.3822407
+
+    if (
+        with_sample_weight
+        and "sample_weight" not in inspect.signature(model.fit).parameters.keys()
+    ):
+        pytest.skip("Estimator does not support sample_weight.")
+
+    rel = 2e-4  # test precision
+    if isinstance(model, SGDRegressor):
+        rel = 1e-1
+    elif hasattr(model, "solver") and model.solver == "saga":
+        rel = 1e-2
+
+    rng = np.random.RandomState(global_random_seed)
+    n_train, n_features, n_targets = 100, 10, None
+    if isinstance(
+        model,
+        (MultiTaskElasticNet, MultiTaskElasticNetCV, MultiTaskLasso, MultiTaskLassoCV),
+    ):
+        n_targets = 3
+    X = make_low_rank_matrix(n_samples=n_train, n_features=n_features, random_state=rng)
+    if n_targets:
+        coef = (
+            rng.uniform(low=-2, high=2, size=(n_features, n_targets))
+            / np.max(X, axis=0)[:, None]
+        )
+    else:
+        coef = rng.uniform(low=-2, high=2, size=n_features) / np.max(X, axis=0)
+
+    expectation = np.exp(X @ coef + 0.5)
+    y = rng.poisson(lam=expectation) + 1  # strict positive, i.e. y > 0
+    if is_classifier(model):
+        y = (y > expectation + 1).astype(np.float64)
+
+    if with_sample_weight:
+        sw = rng.uniform(low=1, high=10, size=y.shape[0])
+    else:
+        sw = None
+
+    model.set_params(fit_intercept=True)  # to be sure
+    if with_sample_weight:
+        model.fit(X, y, sample_weight=sw)
+    else:
+        model.fit(X, y)
+    # Assert balance property.
+    if is_classifier(model):
+        assert np.average(model.predict_proba(X)[:, 1], weights=sw) == pytest.approx(
+            np.average(y, weights=sw), rel=rel
+        )
+    else:
+        assert np.average(model.predict(X), weights=sw, axis=0) == pytest.approx(
+            np.average(y, weights=sw, axis=0), rel=rel
+        )
+
+
+@pytest.mark.filterwarnings("ignore:The default of 'normalize'")
+@pytest.mark.filterwarnings("ignore:lbfgs failed to converge")
+@pytest.mark.parametrize(
+    "Regressor",
+    [
+        ARDRegression,
+        BayesianRidge,
+        ElasticNet,
+        ElasticNetCV,
+        GammaRegressor,
+        HuberRegressor,
+        Lars,
+        LarsCV,
+        Lasso,
+        LassoCV,
+        LassoLars,
+        LassoLarsCV,
+        LassoLarsIC,
+        LinearSVR,
+        LinearRegression,
+        OrthogonalMatchingPursuit,
+        OrthogonalMatchingPursuitCV,
+        PassiveAggressiveRegressor,
+        PoissonRegressor,
+        Ridge,
+        RidgeCV,
+        SGDRegressor,
+        TheilSenRegressor,
+        TweedieRegressor,
+    ],
+)
+@pytest.mark.parametrize("ndim", [1, 2])
+def test_linear_model_regressor_coef_shape(Regressor, ndim):
+    """Check the consistency of linear models `coef` shape."""
+    if Regressor is LinearRegression:
+        pytest.xfail("LinearRegression does not follow `coef_` shape contract!")
+
+    X, y = make_regression(random_state=0, n_samples=200, n_features=20)
+    y = MinMaxScaler().fit_transform(y.reshape(-1, 1))[:, 0] + 1
+    y = y[:, np.newaxis] if ndim == 2 else y
+
+    regressor = Regressor()
+    set_random_state(regressor)
+    regressor.fit(X, y)
+    assert regressor.coef_.shape == (X.shape[1],)
+
+
+@pytest.mark.parametrize(
+    "Classifier",
+    [
+        LinearSVC,
+        LogisticRegression,
+        LogisticRegressionCV,
+        PassiveAggressiveClassifier,
+        Perceptron,
+        RidgeClassifier,
+        RidgeClassifierCV,
+        SGDClassifier,
+    ],
+)
+@pytest.mark.parametrize("n_classes", [2, 3])
+def test_linear_model_classifier_coef_shape(Classifier, n_classes):
+    if Classifier in (RidgeClassifier, RidgeClassifierCV):
+        pytest.xfail(f"{Classifier} does not follow `coef_` shape contract!")
+
+    X, y = make_classification(n_informative=10, n_classes=n_classes, random_state=0)
+    n_features = X.shape[1]
+
+    classifier = Classifier()
+    set_random_state(classifier)
+    classifier.fit(X, y)
+    expected_shape = (1, n_features) if n_classes == 2 else (n_classes, n_features)
+    assert classifier.coef_.shape == expected_shape
diff --git a/sklearn/linear_model/tests/test_coordinate_descent.py b/sklearn/linear_model/tests/test_coordinate_descent.py
index ebddb6a7e47c6..70226210c010d 100644
--- a/sklearn/linear_model/tests/test_coordinate_descent.py
+++ b/sklearn/linear_model/tests/test_coordinate_descent.py
@@ -1,128 +1,132 @@
-# Authors: Olivier Grisel 
-#          Alexandre Gramfort 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
+import warnings
+from copy import deepcopy
+
+import joblib
 import numpy as np
 import pytest
 from scipy import interpolate, sparse
-from copy import deepcopy
-import joblib
 
-from sklearn.base import is_classifier
-from sklearn.datasets import load_diabetes
-from sklearn.datasets import make_regression
-from sklearn.model_selection import train_test_split
-from sklearn.pipeline import make_pipeline
-from sklearn.pipeline import Pipeline
-from sklearn.preprocessing import StandardScaler
+from sklearn.base import clone, config_context, is_classifier
+from sklearn.datasets import load_diabetes, make_regression
 from sklearn.exceptions import ConvergenceWarning
-from sklearn.utils._testing import assert_allclose
-from sklearn.utils._testing import assert_array_almost_equal
-from sklearn.utils._testing import assert_almost_equal
-from sklearn.utils._testing import ignore_warnings
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils._testing import _convert_container
-from sklearn.utils._testing import TempMemmap
-from sklearn.utils.fixes import parse_version
-from sklearn.utils.sparsefuncs import mean_variance_axis
-
 from sklearn.linear_model import (
-    ARDRegression,
-    BayesianRidge,
     ElasticNet,
     ElasticNetCV,
-    enet_path,
-    Lars,
-    lars_path,
     Lasso,
     LassoCV,
     LassoLars,
     LassoLarsCV,
-    LassoLarsIC,
-    lasso_path,
     LinearRegression,
     MultiTaskElasticNet,
     MultiTaskElasticNetCV,
     MultiTaskLasso,
     MultiTaskLassoCV,
-    OrthogonalMatchingPursuit,
     Ridge,
     RidgeClassifier,
+    RidgeClassifierCV,
     RidgeCV,
+    enet_path,
+    lars_path,
+    lasso_path,
 )
-
 from sklearn.linear_model._coordinate_descent import _set_order
+from sklearn.model_selection import (
+    BaseCrossValidator,
+    GridSearchCV,
+    LeaveOneGroupOut,
+)
+from sklearn.model_selection._split import GroupsConsumerMixin
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import StandardScaler
 from sklearn.utils import check_array
+from sklearn.utils._testing import (
+    TempMemmap,
+    assert_allclose,
+    assert_almost_equal,
+    assert_array_almost_equal,
+    assert_array_equal,
+    assert_array_less,
+    ignore_warnings,
+)
+from sklearn.utils.fixes import COO_CONTAINERS, CSC_CONTAINERS, CSR_CONTAINERS
 
 
-@pytest.mark.parametrize('l1_ratio', (-1, 2, None, 10, 'something_wrong'))
-def test_l1_ratio_param_invalid(l1_ratio):
-    # Check that correct error is raised when l1_ratio in ElasticNet
-    # is outside the correct range
-    X = np.array([[-1.], [0.], [1.]])
-    Y = [-1, 0, 1]       # just a straight line
-
-    msg = "l1_ratio must be between 0 and 1; got l1_ratio="
-    clf = ElasticNet(alpha=0.1, l1_ratio=l1_ratio)
-    with pytest.raises(ValueError, match=msg):
-        clf.fit(X, Y)
-
-
-@pytest.mark.parametrize('order', ['C', 'F'])
-@pytest.mark.parametrize('input_order', ['C', 'F'])
+@pytest.mark.parametrize("order", ["C", "F"])
+@pytest.mark.parametrize("input_order", ["C", "F"])
 def test_set_order_dense(order, input_order):
     """Check that _set_order returns arrays with promised order."""
     X = np.array([[0], [0], [0]], order=input_order)
     y = np.array([0, 0, 0], order=input_order)
     X2, y2 = _set_order(X, y, order=order)
-    if order == 'C':
-        assert X2.flags['C_CONTIGUOUS']
-        assert y2.flags['C_CONTIGUOUS']
-    elif order == 'F':
-        assert X2.flags['F_CONTIGUOUS']
-        assert y2.flags['F_CONTIGUOUS']
+    if order == "C":
+        assert X2.flags["C_CONTIGUOUS"]
+        assert y2.flags["C_CONTIGUOUS"]
+    elif order == "F":
+        assert X2.flags["F_CONTIGUOUS"]
+        assert y2.flags["F_CONTIGUOUS"]
 
     if order == input_order:
         assert X is X2
         assert y is y2
 
 
-@pytest.mark.parametrize('order', ['C', 'F'])
-@pytest.mark.parametrize('input_order', ['C', 'F'])
-def test_set_order_sparse(order, input_order):
+@pytest.mark.parametrize("order", ["C", "F"])
+@pytest.mark.parametrize("input_order", ["C", "F"])
+@pytest.mark.parametrize("coo_container", COO_CONTAINERS)
+def test_set_order_sparse(order, input_order, coo_container):
     """Check that _set_order returns sparse matrices in promised format."""
-    X = sparse.coo_matrix(np.array([[0], [0], [0]]))
-    y = sparse.coo_matrix(np.array([0, 0, 0]))
+    X = coo_container(np.array([[0], [0], [0]]))
+    y = coo_container(np.array([0, 0, 0]))
     sparse_format = "csc" if input_order == "F" else "csr"
     X = X.asformat(sparse_format)
     y = X.asformat(sparse_format)
     X2, y2 = _set_order(X, y, order=order)
-    if order == 'C':
-        assert sparse.isspmatrix_csr(X2)
-        assert sparse.isspmatrix_csr(y2)
-    elif order == 'F':
-        assert sparse.isspmatrix_csc(X2)
-        assert sparse.isspmatrix_csc(y2)
+
+    format = "csc" if order == "F" else "csr"
+    assert sparse.issparse(X2) and X2.format == format
+    assert sparse.issparse(y2) and y2.format == format
 
 
 def test_lasso_zero():
     # Check that the lasso can handle zero data without crashing
     X = [[0], [0], [0]]
     y = [0, 0, 0]
-    clf = Lasso(alpha=0.1).fit(X, y)
+    # _cd_fast.pyx tests for gap < tol, but here we get 0.0 < 0.0
+    # should probably be changed to gap <= tol ?
+    with ignore_warnings(category=ConvergenceWarning):
+        clf = Lasso(alpha=0.1).fit(X, y)
     pred = clf.predict([[1], [2], [3]])
     assert_array_almost_equal(clf.coef_, [0])
     assert_array_almost_equal(pred, [0, 0, 0])
     assert_almost_equal(clf.dual_gap_, 0)
 
 
+@pytest.mark.filterwarnings("ignore::sklearn.exceptions.ConvergenceWarning")
+def test_enet_nonfinite_params():
+    # Check ElasticNet throws ValueError when dealing with non-finite parameter
+    # values
+    rng = np.random.RandomState(0)
+    n_samples = 10
+    fmax = np.finfo(np.float64).max
+    X = fmax * rng.uniform(size=(n_samples, 2))
+    y = rng.randint(0, 2, size=n_samples)
+
+    clf = ElasticNet(alpha=0.1)
+    msg = "Coordinate descent iterations resulted in non-finite parameter values"
+    with pytest.raises(ValueError, match=msg):
+        clf.fit(X, y)
+
+
 def test_lasso_toy():
     # Test Lasso on a toy example for various values of alpha.
     # When validating this against glmnet notice that glmnet divides it
     # against nobs.
 
     X = [[-1], [0], [1]]
-    Y = [-1, 0, 1]       # just a straight line
+    Y = [-1, 0, 1]  # just a straight line
     T = [[2], [3], [4]]  # test sample
 
     clf = Lasso(alpha=1e-8)
@@ -135,21 +139,21 @@ def test_lasso_toy():
     clf = Lasso(alpha=0.1)
     clf.fit(X, Y)
     pred = clf.predict(T)
-    assert_array_almost_equal(clf.coef_, [.85])
+    assert_array_almost_equal(clf.coef_, [0.85])
     assert_array_almost_equal(pred, [1.7, 2.55, 3.4])
     assert_almost_equal(clf.dual_gap_, 0)
 
     clf = Lasso(alpha=0.5)
     clf.fit(X, Y)
     pred = clf.predict(T)
-    assert_array_almost_equal(clf.coef_, [.25])
-    assert_array_almost_equal(pred, [0.5, 0.75, 1.])
+    assert_array_almost_equal(clf.coef_, [0.25])
+    assert_array_almost_equal(pred, [0.5, 0.75, 1.0])
     assert_almost_equal(clf.dual_gap_, 0)
 
     clf = Lasso(alpha=1)
     clf.fit(X, Y)
     pred = clf.predict(T)
-    assert_array_almost_equal(clf.coef_, [.0])
+    assert_array_almost_equal(clf.coef_, [0.0])
     assert_array_almost_equal(pred, [0, 0, 0])
     assert_almost_equal(clf.dual_gap_, 0)
 
@@ -160,9 +164,9 @@ def test_enet_toy():
     # we test it as a border case.
     # ElasticNet is tested with and without precomputed Gram matrix
 
-    X = np.array([[-1.], [0.], [1.]])
-    Y = [-1, 0, 1]       # just a straight line
-    T = [[2.], [3.], [4.]]  # test sample
+    X = np.array([[-1.0], [0.0], [1.0]])
+    Y = [-1, 0, 1]  # just a straight line
+    T = [[2.0], [3.0], [4.0]]  # test sample
 
     # this should be the same as lasso
     clf = ElasticNet(alpha=1e-8, l1_ratio=1.0)
@@ -172,8 +176,7 @@ def test_enet_toy():
     assert_array_almost_equal(pred, [2, 3, 4])
     assert_almost_equal(clf.dual_gap_, 0)
 
-    clf = ElasticNet(alpha=0.5, l1_ratio=0.3, max_iter=100,
-                     precompute=False)
+    clf = ElasticNet(alpha=0.5, l1_ratio=0.3, max_iter=100, precompute=False)
     clf.fit(X, Y)
     pred = clf.predict(T)
     assert_array_almost_equal(clf.coef_, [0.50819], decimal=3)
@@ -213,15 +216,14 @@ def test_lasso_dual_gap():
     clf = Lasso(alpha=alpha, fit_intercept=False).fit(X, y)
     w = clf.coef_
     R = y - X @ w
-    primal = 0.5 * np.mean(R ** 2) + clf.alpha * np.sum(np.abs(w))
+    primal = 0.5 * np.mean(R**2) + clf.alpha * np.sum(np.abs(w))
     # dual pt: R / n_samples, dual constraint: norm(X.T @ theta, inf) <= alpha
     R /= np.max(np.abs(X.T @ R) / (n_samples * alpha))
-    dual = 0.5 * (np.mean(y ** 2) - np.mean((y - R) ** 2))
+    dual = 0.5 * (np.mean(y**2) - np.mean((y - R) ** 2))
     assert_allclose(clf.dual_gap_, primal - dual)
 
 
-def build_dataset(n_samples=50, n_features=200, n_informative_features=10,
-                  n_targets=1):
+def build_dataset(n_samples=50, n_features=200, n_informative_features=10, n_targets=1):
     """
     build an ill-posed linear regression problem with many noisy features and
     comparatively few samples
@@ -242,42 +244,42 @@ def build_dataset(n_samples=50, n_features=200, n_informative_features=10,
 def test_lasso_cv():
     X, y, X_test, y_test = build_dataset()
     max_iter = 150
-    clf = LassoCV(n_alphas=10, eps=1e-3, max_iter=max_iter, cv=3).fit(X, y)
+    clf = LassoCV(alphas=10, eps=1e-3, max_iter=max_iter, cv=3).fit(X, y)
     assert_almost_equal(clf.alpha_, 0.056, 2)
 
-    clf = LassoCV(n_alphas=10, eps=1e-3, max_iter=max_iter, precompute=True,
-                  cv=3)
+    clf = LassoCV(alphas=10, eps=1e-3, max_iter=max_iter, precompute=True, cv=3)
     clf.fit(X, y)
     assert_almost_equal(clf.alpha_, 0.056, 2)
 
     # Check that the lars and the coordinate descent implementation
     # select a similar alpha
-    lars = LassoLarsCV(normalize=False, max_iter=30, cv=3).fit(X, y)
+    lars = LassoLarsCV(max_iter=30, cv=3).fit(X, y)
     # for this we check that they don't fall in the grid of
     # clf.alphas further than 1
-    assert np.abs(np.searchsorted(clf.alphas_[::-1], lars.alpha_) -
-                  np.searchsorted(clf.alphas_[::-1], clf.alpha_)) <= 1
+    assert (
+        np.abs(
+            np.searchsorted(clf.alphas_[::-1], lars.alpha_)
+            - np.searchsorted(clf.alphas_[::-1], clf.alpha_)
+        )
+        <= 1
+    )
     # check that they also give a similar MSE
     mse_lars = interpolate.interp1d(lars.cv_alphas_, lars.mse_path_.T)
-    np.testing.assert_approx_equal(mse_lars(clf.alphas_[5]).mean(),
-                                   clf.mse_path_[5].mean(), significant=2)
+    assert_allclose(mse_lars(clf.alphas_[5]).mean(), clf.mse_path_[5].mean(), rtol=1e-2)
 
     # test set
     assert clf.score(X_test, y_test) > 0.99
 
 
 def test_lasso_cv_with_some_model_selection():
-    from sklearn.model_selection import ShuffleSplit
     from sklearn import datasets
+    from sklearn.model_selection import ShuffleSplit
 
     diabetes = datasets.load_diabetes()
     X = diabetes.data
     y = diabetes.target
 
-    pipe = make_pipeline(
-        StandardScaler(),
-        LassoCV(cv=ShuffleSplit(random_state=0))
-    )
+    pipe = make_pipeline(StandardScaler(), LassoCV(cv=ShuffleSplit(random_state=0)))
     pipe.fit(X, y)
 
 
@@ -286,26 +288,62 @@ def test_lasso_cv_positive_constraint():
     max_iter = 500
 
     # Ensure the unconstrained fit has a negative coefficient
-    clf_unconstrained = LassoCV(n_alphas=3, eps=1e-1, max_iter=max_iter, cv=2,
-                                n_jobs=1)
+    clf_unconstrained = LassoCV(alphas=3, eps=1e-1, max_iter=max_iter, cv=2, n_jobs=1)
     clf_unconstrained.fit(X, y)
     assert min(clf_unconstrained.coef_) < 0
 
     # On same data, constrained fit has non-negative coefficients
-    clf_constrained = LassoCV(n_alphas=3, eps=1e-1, max_iter=max_iter,
-                              positive=True, cv=2, n_jobs=1)
+    clf_constrained = LassoCV(
+        alphas=3, eps=1e-1, max_iter=max_iter, positive=True, cv=2, n_jobs=1
+    )
     clf_constrained.fit(X, y)
     assert min(clf_constrained.coef_) >= 0
 
 
+@pytest.mark.parametrize(
+    "alphas, err_type, err_msg",
+    [
+        ((1, -1, -100), ValueError, r"alphas\[1\] == -1, must be >= 0.0."),
+        (
+            (-0.1, -1.0, -10.0),
+            ValueError,
+            r"alphas\[0\] == -0.1, must be >= 0.0.",
+        ),
+        (
+            (1, 1.0, "1"),
+            TypeError,
+            r"alphas\[2\] must be an instance of float, not str",
+        ),
+    ],
+)
+def test_lassocv_alphas_validation(alphas, err_type, err_msg):
+    """Check the `alphas` validation in LassoCV."""
+
+    n_samples, n_features = 5, 5
+    rng = np.random.RandomState(0)
+    X = rng.randn(n_samples, n_features)
+    y = rng.randint(0, 2, n_samples)
+    lassocv = LassoCV(alphas=alphas)
+    with pytest.raises(err_type, match=err_msg):
+        lassocv.fit(X, y)
+
+
 def _scale_alpha_inplace(estimator, n_samples):
     """Rescale the parameter alpha from when the estimator is evoked with
-    normalize set to True to when it is evoked in a Pipeline with normalize set
+    normalize set to True as if it were evoked in a Pipeline with normalize set
     to False and with a StandardScaler.
     """
-    if 'alpha' not in estimator.get_params():
+    if ("alpha" not in estimator.get_params()) and (
+        "alphas" not in estimator.get_params()
+    ):
         return
 
+    if isinstance(estimator, (RidgeCV, RidgeClassifierCV)):
+        # alphas is not validated at this point and can be a list.
+        # We convert it to a np.ndarray to make sure broadcasting
+        # is used.
+        alphas = np.asarray(estimator.alphas) * n_samples
+        return estimator.set_params(alphas=alphas)
     if isinstance(estimator, (Lasso, LassoLars, MultiTaskLasso)):
         alpha = estimator.alpha * np.sqrt(n_samples)
     if isinstance(estimator, (Ridge, RidgeClassifier)):
@@ -322,203 +360,40 @@ def _scale_alpha_inplace(estimator, n_samples):
     estimator.set_params(alpha=alpha)
 
 
-# FIXME: 'normalize' to be removed in 1.2 for all the models excluding:
-# OrthogonalMatchingPursuit, Lars, LassoLars, LarsCV, LassoLarsCV
-# for which it is to be removed in 1.4
-@pytest.mark.filterwarnings("ignore:'normalize' was deprecated")
+@pytest.mark.filterwarnings("ignore::sklearn.exceptions.ConvergenceWarning")
 @pytest.mark.parametrize(
     "LinearModel, params",
-    [(Lasso, {"tol": 1e-16, "alpha": 0.1}),
-     (LassoLars, {"alpha": 0.1}),
-     (RidgeClassifier, {"solver": 'sparse_cg', "alpha": 0.1}),
-     (ElasticNet, {"tol": 1e-16, 'l1_ratio': 1, "alpha": 0.1}),
-     (ElasticNet, {"tol": 1e-16, 'l1_ratio': 0, "alpha": 0.1}),
-     (Ridge, {"solver": 'sparse_cg', 'tol': 1e-12, "alpha": 0.1}),
-     (BayesianRidge, {}),
-     (ARDRegression, {}),
-     (OrthogonalMatchingPursuit, {}),
-     (MultiTaskElasticNet, {"tol": 1e-16, 'l1_ratio': 1, "alpha": 0.1}),
-     (MultiTaskElasticNet, {"tol": 1e-16, 'l1_ratio': 0, "alpha": 0.1}),
-     (MultiTaskLasso, {"tol": 1e-16, "alpha": 0.1}),
-     (Lars, {}),
-     (LinearRegression, {}),
-     (LassoLarsIC, {})]
-)
-def test_model_pipeline_same_as_normalize_true(LinearModel, params):
-    # Test that linear models (LinearModel) set with normalize set to True are
-    # doing the same as the same linear model preceeded by StandardScaler
-    # in the pipeline and with normalize set to False
-
-    # normalize is True
-    model_normalize = LinearModel(normalize=True, fit_intercept=True, **params)
-
-    pipeline = make_pipeline(
-        StandardScaler(),
-        LinearModel(normalize=False, fit_intercept=True, **params)
-    )
-
-    is_multitask = model_normalize._get_tags()["multioutput_only"]
-
-    # prepare the data
-    n_samples, n_features = 100, 2
-    rng = np.random.RandomState(0)
-    w = rng.randn(n_features)
-    X = rng.randn(n_samples, n_features)
-    X += 20  # make features non-zero mean
-    y = X.dot(w)
-
-    # make classes out of regression
-    if is_classifier(model_normalize):
-        y[y > np.mean(y)] = -1
-        y[y > 0] = 1
-    if is_multitask:
-        y = np.stack((y, y), axis=1)
-
-    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
-
-    _scale_alpha_inplace(pipeline[1], X_train.shape[0])
-
-    model_normalize.fit(X_train, y_train)
-    y_pred_normalize = model_normalize.predict(X_test)
-
-    pipeline.fit(X_train, y_train)
-    y_pred_standardize = pipeline.predict(X_test)
-
-    assert_allclose(
-        model_normalize.coef_ * pipeline[0].scale_, pipeline[1].coef_)
-    assert pipeline[1].intercept_ == pytest.approx(y_train.mean())
-    assert (model_normalize.intercept_ ==
-            pytest.approx(y_train.mean() -
-                          model_normalize.coef_.dot(X_train.mean(0))))
-    assert_allclose(y_pred_normalize, y_pred_standardize)
-
-
-# FIXME: 'normalize' to be removed in 1.2
-@pytest.mark.filterwarnings("ignore:'normalize' was deprecated")
-@pytest.mark.parametrize(
-    "estimator, params",
     [
-         (Lasso, {"tol": 1e-16, "alpha": 0.1}),
-         (RidgeClassifier, {"solver": 'sparse_cg', "alpha": 0.1}),
-         (ElasticNet, {"tol": 1e-16, 'l1_ratio': 1, "alpha": 0.1}),
-         (ElasticNet, {"tol": 1e-16, 'l1_ratio': 0, "alpha": 0.1}),
-         (Ridge, {"solver": 'sparse_cg', 'tol': 1e-12, "alpha": 0.1}),
-         (LinearRegression, {}),
-     ]
-)
-@pytest.mark.parametrize(
-    "is_sparse, with_mean", [
-        (False, True),
-        (False, False),
-        (True, False)
-        # No need to test sparse and with_mean=True
-    ]
+        (Lasso, {"tol": 1e-16, "alpha": 0.1}),
+        (LassoCV, {"tol": 1e-16}),
+        (ElasticNetCV, {}),
+        (RidgeClassifier, {"solver": "sparse_cg", "alpha": 0.1}),
+        (ElasticNet, {"tol": 1e-16, "l1_ratio": 1, "alpha": 0.01}),
+        (ElasticNet, {"tol": 1e-16, "l1_ratio": 0, "alpha": 0.01}),
+        (Ridge, {"solver": "sparse_cg", "tol": 1e-12, "alpha": 0.1}),
+        (LinearRegression, {}),
+        (RidgeCV, {}),
+        (RidgeClassifierCV, {}),
+    ],
 )
-def test_linear_model_sample_weights_normalize_in_pipeline(
-        is_sparse, with_mean, estimator, params
-):
-    # Test that the results for running linear model with sample_weight
-    # and with normalize set to True gives similar results as the same linear
-    # model with normalize set to False in a pipeline with
-    # a StandardScaler and sample_weight.
-    model_name = estimator.__name__
-
-    if model_name in ['Lasso', 'ElasticNet'] and is_sparse:
-        pytest.skip(f'{model_name} does not support sample_weight with sparse')
-
-    rng = np.random.RandomState(0)
-    X, y = make_regression(n_samples=20, n_features=5, noise=1e-2,
-                           random_state=rng)
-
-    if is_classifier(estimator):
-        y = np.sign(y)
-
-    # make sure the data is not centered to make the problem more
-    # difficult + add 0s for the sparse case
-    X[X < 0] = 0
-
-    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5,
-                                                        random_state=rng)
-    if is_sparse:
-        X_train = sparse.csr_matrix(X_train)
-        X_test = _convert_container(X_train, 'sparse')
-
-    sample_weight = rng.rand(X_train.shape[0])
-
-    # linear estimator with built-in feature normalization
-    reg_with_normalize = estimator(normalize=True, fit_intercept=True,
-                                   **params)
-    reg_with_normalize.fit(X_train, y_train, sample_weight=sample_weight)
-
-    # linear estimator in a pipeline with a StandardScaler, normalize=False
-    linear_regressor = estimator(normalize=False, fit_intercept=True, **params)
-    _scale_alpha_inplace(linear_regressor, X_train.shape[0])  # rescale alpha
-    reg_with_scaler = Pipeline([
-        ("scaler", StandardScaler(with_mean=with_mean)),
-        ("linear_regressor", linear_regressor)
-    ])
-
-    fit_params = {
-        "scaler__sample_weight":  sample_weight,
-        "linear_regressor__sample_weight": sample_weight,
-    }
-
-    reg_with_scaler.fit(X_train, y_train, **fit_params)
-
-    # Check that the 2 regressions models are exactly equivalent in the
-    # sense that they predict exactly the same outcome.
-    y_pred_normalize = reg_with_normalize.predict(X_test)
-    y_pred_scaler = reg_with_scaler.predict(X_test)
-    assert_allclose(y_pred_normalize,  y_pred_scaler)
-    # Check intercept computation when normalize is True
-    y_train_mean = np.average(y_train, weights=sample_weight)
-    if is_sparse:
-        X_train_mean, _ = mean_variance_axis(X_train, axis=0,
-                                             weights=sample_weight)
-    else:
-        X_train_mean = np.average(X_train, weights=sample_weight, axis=0)
-    assert (reg_with_normalize.intercept_ ==
-            pytest.approx(y_train_mean -
-                          reg_with_normalize.coef_.dot(X_train_mean)))
-
-
-# FIXME: 'normalize' to be removed in 1.2
-@pytest.mark.filterwarnings("ignore:'normalize' was deprecated")
-@pytest.mark.parametrize(
-    "LinearModel, params",
-    [(Lasso, {"tol": 1e-16, "alpha": 0.1}),
-     (LassoCV, {"tol": 1e-16}),
-     (ElasticNetCV, {}),
-     (RidgeClassifier, {"solver": 'sparse_cg', "alpha": 0.1}),
-     (ElasticNet, {"tol": 1e-16, 'l1_ratio': 1, "alpha": 0.01}),
-     (ElasticNet, {"tol": 1e-16, 'l1_ratio': 0, "alpha": 0.01}),
-     (Ridge, {"solver": 'sparse_cg', 'tol': 1e-12, "alpha": 0.1}),
-     (LinearRegression, {}),
-     (RidgeCV, {})]
- )
-def test_model_pipeline_same_dense_and_sparse(LinearModel, params):
-    # Test that linear model preceeded by StandardScaler in the pipeline and
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_model_pipeline_same_dense_and_sparse(LinearModel, params, csr_container):
+    # Test that linear model preceded by StandardScaler in the pipeline and
     # with normalize set to False gives the same y_pred and the same .coef_
     # given X sparse or dense
 
-    model_dense = make_pipeline(
-        StandardScaler(with_mean=False),
-        LinearModel(normalize=False, **params)
-    )
+    model_dense = make_pipeline(StandardScaler(with_mean=False), LinearModel(**params))
 
-    model_sparse = make_pipeline(
-        StandardScaler(with_mean=False),
-        LinearModel(normalize=False, **params)
-    )
+    model_sparse = make_pipeline(StandardScaler(with_mean=False), LinearModel(**params))
 
     # prepare the data
     rng = np.random.RandomState(0)
     n_samples = 200
     n_features = 2
     X = rng.randn(n_samples, n_features)
-    X[X < 0.1] = 0.
+    X[X < 0.1] = 0.0
 
-    X_sparse = sparse.csr_matrix(X)
+    X_sparse = csr_container(X)
     y = rng.rand(n_samples)
 
     if is_classifier(model_dense):
@@ -542,35 +417,37 @@ def test_lasso_path_return_models_vs_new_return_gives_same_coefficients():
     # Some toy data
     X = np.array([[1, 2, 3.1], [2.3, 5.4, 4.3]]).T
     y = np.array([1, 2, 3.1])
-    alphas = [5., 1., .5]
+    alphas = [5.0, 1.0, 0.5]
 
     # Use lars_path and lasso_path(new output) with 1D linear interpolation
     # to compute the same path
-    alphas_lars, _, coef_path_lars = lars_path(X, y, method='lasso')
-    coef_path_cont_lars = interpolate.interp1d(alphas_lars[::-1],
-                                               coef_path_lars[:, ::-1])
-    alphas_lasso2, coef_path_lasso2, _ = lasso_path(X, y, alphas=alphas,
-                                                    return_models=False)
-    coef_path_cont_lasso = interpolate.interp1d(alphas_lasso2[::-1],
-                                                coef_path_lasso2[:, ::-1])
+    alphas_lars, _, coef_path_lars = lars_path(X, y, method="lasso")
+    coef_path_cont_lars = interpolate.interp1d(
+        alphas_lars[::-1], coef_path_lars[:, ::-1]
+    )
+    alphas_lasso2, coef_path_lasso2, _ = lasso_path(X, y, alphas=alphas)
+    coef_path_cont_lasso = interpolate.interp1d(
+        alphas_lasso2[::-1], coef_path_lasso2[:, ::-1]
+    )
 
     assert_array_almost_equal(
-        coef_path_cont_lasso(alphas), coef_path_cont_lars(alphas),
-        decimal=1)
+        coef_path_cont_lasso(alphas), coef_path_cont_lars(alphas), decimal=1
+    )
 
 
 def test_enet_path():
     # We use a large number of samples and of informative features so that
     # the l1_ratio selected is more toward ridge than lasso
-    X, y, X_test, y_test = build_dataset(n_samples=200, n_features=100,
-                                         n_informative_features=100)
+    X, y, X_test, y_test = build_dataset(
+        n_samples=200, n_features=100, n_informative_features=100
+    )
     max_iter = 150
 
     # Here we have a small number of iterations, and thus the
     # ElasticNet might not converge. This is to speed up tests
-    clf = ElasticNetCV(alphas=[0.01, 0.05, 0.1], eps=2e-3,
-                       l1_ratio=[0.5, 0.7], cv=3,
-                       max_iter=max_iter)
+    clf = ElasticNetCV(
+        alphas=[0.01, 0.05, 0.1], eps=2e-3, l1_ratio=[0.5, 0.7], cv=3, max_iter=max_iter
+    )
     ignore_warnings(clf.fit)(X, y)
     # Well-conditioned settings, we should have selected our
     # smallest penalty
@@ -579,9 +456,14 @@ def test_enet_path():
     # that is closer to ridge than to lasso
     assert clf.l1_ratio_ == min(clf.l1_ratio)
 
-    clf = ElasticNetCV(alphas=[0.01, 0.05, 0.1], eps=2e-3,
-                       l1_ratio=[0.5, 0.7], cv=3,
-                       max_iter=max_iter, precompute=True)
+    clf = ElasticNetCV(
+        alphas=[0.01, 0.05, 0.1],
+        eps=2e-3,
+        l1_ratio=[0.5, 0.7],
+        cv=3,
+        max_iter=max_iter,
+        precompute=True,
+    )
     ignore_warnings(clf.fit)(X, y)
 
     # Well-conditioned settings, we should have selected our
@@ -597,8 +479,9 @@ def test_enet_path():
 
     # Multi-output/target case
     X, y, X_test, y_test = build_dataset(n_features=10, n_targets=3)
-    clf = MultiTaskElasticNetCV(n_alphas=5, eps=2e-3, l1_ratio=[0.5, 0.7],
-                                cv=3, max_iter=max_iter)
+    clf = MultiTaskElasticNetCV(
+        alphas=5, eps=2e-3, l1_ratio=[0.5, 0.7], cv=3, max_iter=max_iter
+    )
     ignore_warnings(clf.fit)(X, y)
     # We are in well-conditioned settings with low noise: we should
     # have a good test-set performance
@@ -608,9 +491,9 @@ def test_enet_path():
     # Mono-output should have same cross-validated alpha_ and l1_ratio_
     # in both cases.
     X, y, _, _ = build_dataset(n_features=10)
-    clf1 = ElasticNetCV(n_alphas=5, eps=2e-3, l1_ratio=[0.5, 0.7])
+    clf1 = ElasticNetCV(alphas=5, eps=2e-3, l1_ratio=[0.5, 0.7])
     clf1.fit(X, y)
-    clf2 = MultiTaskElasticNetCV(n_alphas=5, eps=2e-3, l1_ratio=[0.5, 0.7])
+    clf2 = MultiTaskElasticNetCV(alphas=5, eps=2e-3, l1_ratio=[0.5, 0.7])
     clf2.fit(X, y[:, np.newaxis])
     assert_almost_equal(clf1.l1_ratio_, clf2.l1_ratio_)
     assert_almost_equal(clf1.alpha_, clf2.alpha_)
@@ -620,11 +503,10 @@ def test_path_parameters():
     X, y, _, _ = build_dataset()
     max_iter = 100
 
-    clf = ElasticNetCV(n_alphas=50, eps=1e-3, max_iter=max_iter,
-                       l1_ratio=0.5, tol=1e-3)
+    clf = ElasticNetCV(alphas=50, eps=1e-3, max_iter=max_iter, l1_ratio=0.5, tol=1e-3)
     clf.fit(X, y)  # new params
     assert_almost_equal(0.5, clf.l1_ratio)
-    assert 50 == clf.n_alphas
+    assert 50 == clf._alphas
     assert 50 == len(clf.alphas_)
 
 
@@ -641,7 +523,7 @@ def test_warm_start():
 
 def test_lasso_alpha_warning():
     X = [[-1], [0], [1]]
-    Y = [-1, 0, 1]       # just a straight line
+    Y = [-1, 0, 1]  # just a straight line
 
     clf = Lasso(alpha=0)
     warning_message = (
@@ -655,22 +537,22 @@ def test_lasso_alpha_warning():
 
 def test_lasso_positive_constraint():
     X = [[-1], [0], [1]]
-    y = [1, 0, -1]       # just a straight line with negative slope
+    y = [1, 0, -1]  # just a straight line with negative slope
 
-    lasso = Lasso(alpha=0.1, max_iter=1000, positive=True)
+    lasso = Lasso(alpha=0.1, positive=True)
     lasso.fit(X, y)
     assert min(lasso.coef_) >= 0
 
-    lasso = Lasso(alpha=0.1, max_iter=1000, precompute=True, positive=True)
+    lasso = Lasso(alpha=0.1, precompute=True, positive=True)
     lasso.fit(X, y)
     assert min(lasso.coef_) >= 0
 
 
 def test_enet_positive_constraint():
     X = [[-1], [0], [1]]
-    y = [1, 0, -1]       # just a straight line with negative slope
+    y = [1, 0, -1]  # just a straight line with negative slope
 
-    enet = ElasticNet(alpha=0.1, max_iter=1000, positive=True)
+    enet = ElasticNet(alpha=0.1, positive=True)
     enet.fit(X, y)
     assert min(enet.coef_) >= 0
 
@@ -680,24 +562,25 @@ def test_enet_cv_positive_constraint():
     max_iter = 500
 
     # Ensure the unconstrained fit has a negative coefficient
-    enetcv_unconstrained = ElasticNetCV(n_alphas=3, eps=1e-1,
-                                        max_iter=max_iter,
-                                        cv=2, n_jobs=1)
+    enetcv_unconstrained = ElasticNetCV(
+        alphas=3, eps=1e-1, max_iter=max_iter, cv=2, n_jobs=1
+    )
     enetcv_unconstrained.fit(X, y)
     assert min(enetcv_unconstrained.coef_) < 0
 
     # On same data, constrained fit has non-negative coefficients
-    enetcv_constrained = ElasticNetCV(n_alphas=3, eps=1e-1, max_iter=max_iter,
-                                      cv=2, positive=True, n_jobs=1)
+    enetcv_constrained = ElasticNetCV(
+        alphas=3, eps=1e-1, max_iter=max_iter, cv=2, positive=True, n_jobs=1
+    )
     enetcv_constrained.fit(X, y)
     assert min(enetcv_constrained.coef_) >= 0
 
 
 def test_uniform_targets():
-    enet = ElasticNetCV(n_alphas=3)
-    m_enet = MultiTaskElasticNetCV(n_alphas=3)
-    lasso = LassoCV(n_alphas=3)
-    m_lasso = MultiTaskLassoCV(n_alphas=3)
+    enet = ElasticNetCV(alphas=3)
+    m_enet = MultiTaskElasticNetCV(alphas=3)
+    lasso = LassoCV(alphas=3)
+    m_lasso = MultiTaskLassoCV(alphas=3)
 
     models_single_task = (enet, lasso)
     models_multi_task = (m_enet, m_lasso)
@@ -713,15 +596,17 @@ def test_uniform_targets():
     for model in models_single_task:
         for y_values in (0, 5):
             y1.fill(y_values)
-            assert_array_equal(model.fit(X_train, y1).predict(X_test), y1)
-            assert_array_equal(model.alphas_, [np.finfo(float).resolution]*3)
+            with ignore_warnings(category=ConvergenceWarning):
+                assert_array_equal(model.fit(X_train, y1).predict(X_test), y1)
+            assert_array_equal(model.alphas_, [np.finfo(float).resolution] * 3)
 
     for model in models_multi_task:
         for y_values in (0, 5):
             y2[:, 0].fill(y_values)
             y2[:, 1].fill(2 * y_values)
-            assert_array_equal(model.fit(X_train, y2).predict(X_test), y2)
-            assert_array_equal(model.alphas_, [np.finfo(float).resolution]*3)
+            with ignore_warnings(category=ConvergenceWarning):
+                assert_array_equal(model.fit(X_train, y2).predict(X_test), y2)
+            assert_array_equal(model.alphas_, [np.finfo(float).resolution] * 3)
 
 
 def test_multi_task_lasso_and_enet():
@@ -747,14 +632,14 @@ def test_multi_task_lasso_and_enet():
 
 def test_lasso_readonly_data():
     X = np.array([[-1], [0], [1]])
-    Y = np.array([-1, 0, 1])   # just a straight line
+    Y = np.array([-1, 0, 1])  # just a straight line
     T = np.array([[2], [3], [4]])  # test sample
     with TempMemmap((X, Y)) as (X, Y):
         clf = Lasso(alpha=0.5)
         clf.fit(X, Y)
         pred = clf.predict(T)
-        assert_array_almost_equal(clf.coef_, [.25])
-        assert_array_almost_equal(pred, [0.5, 0.75, 1.])
+        assert_array_almost_equal(clf.coef_, [0.25])
+        assert_array_almost_equal(pred, [0.5, 0.75, 1.0])
         assert_almost_equal(clf.dual_gap_, 0)
 
 
@@ -770,12 +655,16 @@ def test_multi_task_lasso_readonly_data():
 
 def test_enet_multitarget():
     n_targets = 3
-    X, y, _, _ = build_dataset(n_samples=10, n_features=8,
-                               n_informative_features=10, n_targets=n_targets)
+    X, y, _, _ = build_dataset(
+        n_samples=10, n_features=8, n_informative_features=10, n_targets=n_targets
+    )
     estimator = ElasticNet(alpha=0.01)
     estimator.fit(X, y)
-    coef, intercept, dual_gap = (estimator.coef_, estimator.intercept_,
-                                 estimator.dual_gap_)
+    coef, intercept, dual_gap = (
+        estimator.coef_,
+        estimator.intercept_,
+        estimator.dual_gap_,
+    )
 
     for k in range(n_targets):
         estimator.fit(X, y[:, k])
@@ -801,20 +690,21 @@ def test_multitask_enet_and_lasso_cv():
     assert_almost_equal(clf.alpha_, 0.00278, 3)
 
     X, y, _, _ = build_dataset(n_targets=3)
-    clf = MultiTaskElasticNetCV(n_alphas=10, eps=1e-3, max_iter=100,
-                                l1_ratio=[0.3, 0.5], tol=1e-3, cv=3)
+    clf = MultiTaskElasticNetCV(
+        alphas=10, eps=1e-3, max_iter=200, l1_ratio=[0.3, 0.5], tol=1e-3, cv=3
+    )
     clf.fit(X, y)
     assert 0.5 == clf.l1_ratio_
     assert (3, X.shape[1]) == clf.coef_.shape
-    assert (3, ) == clf.intercept_.shape
+    assert (3,) == clf.intercept_.shape
     assert (2, 10, 3) == clf.mse_path_.shape
     assert (2, 10) == clf.alphas_.shape
 
     X, y, _, _ = build_dataset(n_targets=3)
-    clf = MultiTaskLassoCV(n_alphas=10, eps=1e-3, max_iter=100, tol=1e-3, cv=3)
+    clf = MultiTaskLassoCV(alphas=10, eps=1e-3, max_iter=500, tol=1e-3, cv=3)
     clf.fit(X, y)
     assert (3, X.shape[1]) == clf.coef_.shape
-    assert (3, ) == clf.intercept_.shape
+    assert (3,) == clf.intercept_.shape
     assert (10, 3) == clf.mse_path_.shape
     assert 10 == len(clf.alphas_)
 
@@ -822,9 +712,9 @@ def test_multitask_enet_and_lasso_cv():
 def test_1d_multioutput_enet_and_multitask_enet_cv():
     X, y, _, _ = build_dataset(n_features=10)
     y = y[:, np.newaxis]
-    clf = ElasticNetCV(n_alphas=5, eps=2e-3, l1_ratio=[0.5, 0.7])
+    clf = ElasticNetCV(alphas=5, eps=2e-3, l1_ratio=[0.5, 0.7])
     clf.fit(X, y[:, 0])
-    clf1 = MultiTaskElasticNetCV(n_alphas=5, eps=2e-3, l1_ratio=[0.5, 0.7])
+    clf1 = MultiTaskElasticNetCV(alphas=5, eps=2e-3, l1_ratio=[0.5, 0.7])
     clf1.fit(X, y)
     assert_almost_equal(clf.l1_ratio_, clf1.l1_ratio_)
     assert_almost_equal(clf.alpha_, clf1.alpha_)
@@ -835,50 +725,33 @@ def test_1d_multioutput_enet_and_multitask_enet_cv():
 def test_1d_multioutput_lasso_and_multitask_lasso_cv():
     X, y, _, _ = build_dataset(n_features=10)
     y = y[:, np.newaxis]
-    clf = LassoCV(n_alphas=5, eps=2e-3)
+    clf = LassoCV(alphas=5, eps=2e-3)
     clf.fit(X, y[:, 0])
-    clf1 = MultiTaskLassoCV(n_alphas=5, eps=2e-3)
+    clf1 = MultiTaskLassoCV(alphas=5, eps=2e-3)
     clf1.fit(X, y)
     assert_almost_equal(clf.alpha_, clf1.alpha_)
     assert_almost_equal(clf.coef_, clf1.coef_[0])
     assert_almost_equal(clf.intercept_, clf1.intercept_[0])
 
 
-def test_sparse_input_dtype_enet_and_lassocv():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sparse_input_dtype_enet_and_lassocv(csr_container):
     X, y, _, _ = build_dataset(n_features=10)
-    clf = ElasticNetCV(n_alphas=5)
-    clf.fit(sparse.csr_matrix(X), y)
-    clf1 = ElasticNetCV(n_alphas=5)
-    clf1.fit(sparse.csr_matrix(X, dtype=np.float32), y)
+    clf = ElasticNetCV(alphas=5)
+    clf.fit(csr_container(X), y)
+    clf1 = ElasticNetCV(alphas=5)
+    clf1.fit(csr_container(X, dtype=np.float32), y)
     assert_almost_equal(clf.alpha_, clf1.alpha_, decimal=6)
     assert_almost_equal(clf.coef_, clf1.coef_, decimal=6)
 
-    clf = LassoCV(n_alphas=5)
-    clf.fit(sparse.csr_matrix(X), y)
-    clf1 = LassoCV(n_alphas=5)
-    clf1.fit(sparse.csr_matrix(X, dtype=np.float32), y)
+    clf = LassoCV(alphas=5)
+    clf.fit(csr_container(X), y)
+    clf1 = LassoCV(alphas=5)
+    clf1.fit(csr_container(X, dtype=np.float32), y)
     assert_almost_equal(clf.alpha_, clf1.alpha_, decimal=6)
     assert_almost_equal(clf.coef_, clf1.coef_, decimal=6)
 
 
-def test_precompute_invalid_argument():
-    X, y, _, _ = build_dataset()
-    for clf in [ElasticNetCV(precompute="invalid"),
-                LassoCV(precompute="invalid")]:
-        err_msg = ".*should be.*True.*False.*auto.* array-like.*Got 'invalid'"
-        with pytest.raises(ValueError, match=err_msg):
-            clf.fit(X, y)
-
-    # Precompute = 'auto' is not supported for ElasticNet and Lasso
-    err_msg = ".*should be.*True.*False.*array-like.*Got 'auto'"
-    with pytest.raises(ValueError, match=err_msg):
-        ElasticNet(precompute='auto').fit(X, y)
-
-    err_msg = ".*should be.*True.*False.*array-like.*Got 'auto'"
-    with pytest.raises(ValueError, match=err_msg):
-        Lasso(precompute='auto').fit(X, y)
-
-
 def test_elasticnet_precompute_incorrect_gram():
     # check that passing an invalid precomputed Gram matrix will raise an
     # error.
@@ -905,7 +778,7 @@ def test_elasticnet_precompute_gram_weighted_samples():
     sample_weight = rng.lognormal(size=y.shape)
 
     w_norm = sample_weight * (y.shape / np.sum(sample_weight))
-    X_c = (X - np.average(X, axis=0, weights=w_norm))
+    X_c = X - np.average(X, axis=0, weights=w_norm)
     X_r = X_c * np.sqrt(w_norm)[:, np.newaxis]
     gram = np.dot(X_r.T, X_r)
 
@@ -918,6 +791,29 @@ def test_elasticnet_precompute_gram_weighted_samples():
     assert_allclose(clf1.coef_, clf2.coef_)
 
 
+def test_elasticnet_precompute_gram():
+    # Check the dtype-aware check for a precomputed Gram matrix
+    # (see https://github.com/scikit-learn/scikit-learn/pull/22059
+    # and https://github.com/scikit-learn/scikit-learn/issues/21997).
+    # Here: (X_c.T, X_c)[2, 3] is not equal to np.dot(X_c[:, 2], X_c[:, 3])
+    # but within tolerance for np.float32
+
+    rng = np.random.RandomState(58)
+    X = rng.binomial(1, 0.25, (1000, 4)).astype(np.float32)
+    y = rng.rand(1000).astype(np.float32)
+
+    X_c = X - np.average(X, axis=0)
+    gram = np.dot(X_c.T, X_c)
+
+    clf1 = ElasticNet(alpha=0.01, precompute=gram)
+    clf1.fit(X_c, y)
+
+    clf2 = ElasticNet(alpha=0.01, precompute=False)
+    clf2.fit(X, y)
+
+    assert_allclose(clf1.coef_, clf2.coef_)
+
+
 def test_warm_start_convergence():
     X, y, _, _ = build_dataset()
     model = ElasticNet(alpha=1e-3, tol=1e-3).fit(X, y)
@@ -963,51 +859,46 @@ def test_warm_start_convergence_with_regularizer_decrement():
     assert low_reg_model.n_iter_ > warm_low_reg_model.n_iter_
 
 
-def test_random_descent():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_random_descent(csr_container):
     # Test that both random and cyclic selection give the same results.
     # Ensure that the test models fully converge and check a wide
     # range of conditions.
 
     # This uses the coordinate descent algo using the gram trick.
     X, y, _, _ = build_dataset(n_samples=50, n_features=20)
-    clf_cyclic = ElasticNet(selection='cyclic', tol=1e-8)
+    clf_cyclic = ElasticNet(selection="cyclic", tol=1e-8)
     clf_cyclic.fit(X, y)
-    clf_random = ElasticNet(selection='random', tol=1e-8, random_state=42)
+    clf_random = ElasticNet(selection="random", tol=1e-8, random_state=42)
     clf_random.fit(X, y)
     assert_array_almost_equal(clf_cyclic.coef_, clf_random.coef_)
     assert_almost_equal(clf_cyclic.intercept_, clf_random.intercept_)
 
     # This uses the descent algo without the gram trick
-    clf_cyclic = ElasticNet(selection='cyclic', tol=1e-8)
+    clf_cyclic = ElasticNet(selection="cyclic", tol=1e-8)
     clf_cyclic.fit(X.T, y[:20])
-    clf_random = ElasticNet(selection='random', tol=1e-8, random_state=42)
+    clf_random = ElasticNet(selection="random", tol=1e-8, random_state=42)
     clf_random.fit(X.T, y[:20])
     assert_array_almost_equal(clf_cyclic.coef_, clf_random.coef_)
     assert_almost_equal(clf_cyclic.intercept_, clf_random.intercept_)
 
     # Sparse Case
-    clf_cyclic = ElasticNet(selection='cyclic', tol=1e-8)
-    clf_cyclic.fit(sparse.csr_matrix(X), y)
-    clf_random = ElasticNet(selection='random', tol=1e-8, random_state=42)
-    clf_random.fit(sparse.csr_matrix(X), y)
+    clf_cyclic = ElasticNet(selection="cyclic", tol=1e-8)
+    clf_cyclic.fit(csr_container(X), y)
+    clf_random = ElasticNet(selection="random", tol=1e-8, random_state=42)
+    clf_random.fit(csr_container(X), y)
     assert_array_almost_equal(clf_cyclic.coef_, clf_random.coef_)
     assert_almost_equal(clf_cyclic.intercept_, clf_random.intercept_)
 
     # Multioutput case.
     new_y = np.hstack((y[:, np.newaxis], y[:, np.newaxis]))
-    clf_cyclic = MultiTaskElasticNet(selection='cyclic', tol=1e-8)
+    clf_cyclic = MultiTaskElasticNet(selection="cyclic", tol=1e-8)
     clf_cyclic.fit(X, new_y)
-    clf_random = MultiTaskElasticNet(selection='random', tol=1e-8,
-                                     random_state=42)
+    clf_random = MultiTaskElasticNet(selection="random", tol=1e-8, random_state=42)
     clf_random.fit(X, new_y)
     assert_array_almost_equal(clf_cyclic.coef_, clf_random.coef_)
     assert_almost_equal(clf_cyclic.intercept_, clf_random.intercept_)
 
-    # Raise error when selection is not in cyclic or random.
-    clf_random = ElasticNet(selection='invalid')
-    with pytest.raises(ValueError):
-        clf_random.fit(X, y)
-
 
 def test_enet_path_positive():
     # Test positive parameter
@@ -1027,31 +918,43 @@ def test_enet_path_positive():
             path(X, Y, positive=True)
 
 
-def test_sparse_dense_descent_paths():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sparse_dense_descent_paths(csr_container):
     # Test that dense and sparse input give the same input for descent paths.
     X, y, _, _ = build_dataset(n_samples=50, n_features=20)
-    csr = sparse.csr_matrix(X)
+    csr = csr_container(X)
     for path in [enet_path, lasso_path]:
-        _, coefs, _ = path(X, y, fit_intercept=False)
-        _, sparse_coefs, _ = path(csr, y, fit_intercept=False)
+        _, coefs, _ = path(X, y)
+        _, sparse_coefs, _ = path(csr, y)
         assert_array_almost_equal(coefs, sparse_coefs)
 
 
+@pytest.mark.parametrize("path_func", [enet_path, lasso_path])
+def test_path_unknown_parameter(path_func):
+    """Check that passing parameter not used by the coordinate descent solver
+    will raise an error."""
+    X, y, _, _ = build_dataset(n_samples=50, n_features=20)
+    err_msg = "Unexpected parameters in params"
+    with pytest.raises(ValueError, match=err_msg):
+        path_func(X, y, normalize=True, fit_intercept=True)
+
+
 def test_check_input_false():
     X, y, _, _ = build_dataset(n_samples=20, n_features=10)
-    X = check_array(X, order='F', dtype='float64')
-    y = check_array(X, order='F', dtype='float64')
-    clf = ElasticNet(selection='cyclic', tol=1e-8)
+    X = check_array(X, order="F", dtype="float64")
+    y = check_array(X, order="F", dtype="float64")
+    clf = ElasticNet(selection="cyclic", tol=1e-8)
     # Check that no error is raised if data is provided in the right format
     clf.fit(X, y, check_input=False)
     # With check_input=False, an exhaustive check is not made on y but its
     # dtype is still cast in _preprocess_data to X's dtype. So the test should
     # pass anyway
-    X = check_array(X, order='F', dtype='float32')
-    clf.fit(X, y, check_input=False)
+    X = check_array(X, order="F", dtype="float32")
+    with ignore_warnings(category=ConvergenceWarning):
+        clf.fit(X, y, check_input=False)
     # With no input checking, providing X in C order should result in false
     # computation
-    X = check_array(X, order='C', dtype='float64')
+    X = check_array(X, order="C", dtype="float64")
     with pytest.raises(ValueError):
         clf.fit(X, y, check_input=False)
 
@@ -1059,7 +962,7 @@ def test_check_input_false():
 @pytest.mark.parametrize("check_input", [True, False])
 def test_enet_copy_X_True(check_input):
     X, y, _, _ = build_dataset()
-    X = X.copy(order='F')
+    X = X.copy(order="F")
 
     original_X = X.copy()
     enet = ElasticNet(copy_X=True)
@@ -1070,7 +973,7 @@ def test_enet_copy_X_True(check_input):
 
 def test_enet_copy_X_False_check_input_False():
     X, y, _, _ = build_dataset()
-    X = X.copy(order='F')
+    X = X.copy(order="F")
 
     original_X = X.copy()
     enet = ElasticNet(copy_X=False)
@@ -1083,17 +986,16 @@ def test_enet_copy_X_False_check_input_False():
 def test_overrided_gram_matrix():
     X, y, _, _ = build_dataset(n_samples=20, n_features=10)
     Gram = X.T.dot(X)
-    clf = ElasticNet(selection='cyclic', tol=1e-8, precompute=Gram)
+    clf = ElasticNet(selection="cyclic", tol=1e-8, precompute=Gram)
     warning_message = (
         "Gram matrix was provided but X was centered"
-        " to fit intercept, "
-        "or X was normalized : recomputing Gram matrix."
+        " to fit intercept: recomputing Gram matrix."
     )
     with pytest.warns(UserWarning, match=warning_message):
         clf.fit(X, y)
 
 
-@pytest.mark.parametrize('model', [ElasticNet, Lasso])
+@pytest.mark.parametrize("model", [ElasticNet, Lasso])
 def test_lasso_non_float_y(model):
     X = [[0, 0], [1, 1], [-1, -1]]
     y = [0, 1, 2]
@@ -1112,60 +1014,68 @@ def test_enet_float_precision():
     # Here we have a small number of iterations, and thus the
     # ElasticNet might not converge. This is to speed up tests
 
-    for normalize in [True, False]:
-        for fit_intercept in [True, False]:
-            coef = {}
-            intercept = {}
-            for dtype in [np.float64, np.float32]:
-                clf = ElasticNet(alpha=0.5, max_iter=100, precompute=False,
-                                 fit_intercept=fit_intercept,
-                                 normalize=normalize)
-
-                X = dtype(X)
-                y = dtype(y)
-                ignore_warnings(clf.fit)(X, y)
-
-                coef[('simple', dtype)] = clf.coef_
-                intercept[('simple', dtype)] = clf.intercept_
-
-                assert clf.coef_.dtype == dtype
-
-                # test precompute Gram array
-                Gram = X.T.dot(X)
-                clf_precompute = ElasticNet(alpha=0.5, max_iter=100,
-                                            precompute=Gram,
-                                            fit_intercept=fit_intercept,
-                                            normalize=normalize)
-                ignore_warnings(clf_precompute.fit)(X, y)
-                assert_array_almost_equal(clf.coef_, clf_precompute.coef_)
-                assert_array_almost_equal(clf.intercept_,
-                                          clf_precompute.intercept_)
-
-                # test multi task enet
-                multi_y = np.hstack((y[:, np.newaxis], y[:, np.newaxis]))
-                clf_multioutput = MultiTaskElasticNet(
-                    alpha=0.5, max_iter=100, fit_intercept=fit_intercept,
-                    normalize=normalize)
-                clf_multioutput.fit(X, multi_y)
-                coef[('multi', dtype)] = clf_multioutput.coef_
-                intercept[('multi', dtype)] = clf_multioutput.intercept_
-                assert clf.coef_.dtype == dtype
-
-            for v in ['simple', 'multi']:
-                assert_array_almost_equal(coef[(v, np.float32)],
-                                          coef[(v, np.float64)],
-                                          decimal=4)
-                assert_array_almost_equal(intercept[(v, np.float32)],
-                                          intercept[(v, np.float64)],
-                                          decimal=4)
-
-
+    for fit_intercept in [True, False]:
+        coef = {}
+        intercept = {}
+        for dtype in [np.float64, np.float32]:
+            clf = ElasticNet(
+                alpha=0.5,
+                max_iter=100,
+                precompute=False,
+                fit_intercept=fit_intercept,
+            )
+
+            X = dtype(X)
+            y = dtype(y)
+            ignore_warnings(clf.fit)(X, y)
+
+            coef[("simple", dtype)] = clf.coef_
+            intercept[("simple", dtype)] = clf.intercept_
+
+            assert clf.coef_.dtype == dtype
+
+            # test precompute Gram array
+            Gram = X.T.dot(X)
+            clf_precompute = ElasticNet(
+                alpha=0.5,
+                max_iter=100,
+                precompute=Gram,
+                fit_intercept=fit_intercept,
+            )
+            ignore_warnings(clf_precompute.fit)(X, y)
+            assert_array_almost_equal(clf.coef_, clf_precompute.coef_)
+            assert_array_almost_equal(clf.intercept_, clf_precompute.intercept_)
+
+            # test multi task enet
+            multi_y = np.hstack((y[:, np.newaxis], y[:, np.newaxis]))
+            clf_multioutput = MultiTaskElasticNet(
+                alpha=0.5,
+                max_iter=100,
+                fit_intercept=fit_intercept,
+            )
+            clf_multioutput.fit(X, multi_y)
+            coef[("multi", dtype)] = clf_multioutput.coef_
+            intercept[("multi", dtype)] = clf_multioutput.intercept_
+            assert clf.coef_.dtype == dtype
+
+        for v in ["simple", "multi"]:
+            assert_array_almost_equal(
+                coef[(v, np.float32)], coef[(v, np.float64)], decimal=4
+            )
+            assert_array_almost_equal(
+                intercept[(v, np.float32)], intercept[(v, np.float64)], decimal=4
+            )
+
+
+@pytest.mark.filterwarnings("ignore::sklearn.exceptions.ConvergenceWarning")
 def test_enet_l1_ratio():
     # Test that an error message is raised if an estimator that
     # uses _alpha_grid is called with l1_ratio=0
-    msg = ("Automatic alpha grid generation is not supported for l1_ratio=0. "
-           "Please supply a grid by providing your estimator with the "
-           "appropriate `alphas=` argument.")
+    msg = (
+        "Automatic alpha grid generation is not supported for l1_ratio=0. "
+        "Please supply a grid by providing your estimator with the "
+        "appropriate `alphas=` argument."
+    )
     X = np.array([[1, 2, 4, 5, 8], [3, 5, 7, 7, 8]]).T
     y = np.array([12, 10, 11, 21, 5])
 
@@ -1175,9 +1085,19 @@ def test_enet_l1_ratio():
     with pytest.raises(ValueError, match=msg):
         MultiTaskElasticNetCV(l1_ratio=0, random_state=42).fit(X, y[:, None])
 
+    # Test that l1_ratio=0 with alpha>0 produces user warning
+    warning_message = (
+        "Coordinate descent without L1 regularization may "
+        "lead to unexpected results and is discouraged. "
+        "Set l1_ratio > 0 to add L1 regularization."
+    )
+    est = ElasticNetCV(l1_ratio=[0], alphas=[1])
+    with pytest.warns(UserWarning, match=warning_message):
+        est.fit(X, y)
+
     # Test that l1_ratio=0 is allowed if we supply a grid manually
     alphas = [0.1, 10]
-    estkwds = {'alphas': alphas, 'random_state': 42}
+    estkwds = {"alphas": alphas, "random_state": 42}
     est_desired = ElasticNetCV(l1_ratio=0.00001, **estkwds)
     est = ElasticNetCV(l1_ratio=0, **estkwds)
     with ignore_warnings():
@@ -1211,11 +1131,13 @@ def test_warm_start_multitask_lasso():
     assert_array_almost_equal(clf2.coef_, clf.coef_)
 
 
-@pytest.mark.parametrize('klass, n_classes, kwargs',
-                         [(Lasso, 1, dict(precompute=True)),
-                          (Lasso, 1, dict(precompute=False)),
-                          (MultiTaskLasso, 2, dict()),
-                          (MultiTaskLasso, 2, dict())])
+@pytest.mark.parametrize(
+    "klass, n_classes, kwargs",
+    [
+        (Lasso, 1, dict(precompute=True)),
+        (Lasso, 1, dict(precompute=False)),
+    ],
+)
 def test_enet_coordinate_descent(klass, n_classes, kwargs):
     """Test that a warning is issued if model does not converge"""
     clf = klass(max_iter=2, **kwargs)
@@ -1238,38 +1160,34 @@ def test_convergence_warnings():
     X = random_state.standard_normal((1000, 500))
     y = random_state.standard_normal((1000, 3))
 
-    # check that the model fails to converge (a negative dual gap cannot occur)
-    with pytest.warns(ConvergenceWarning):
-        MultiTaskElasticNet(max_iter=1, tol=-1).fit(X, y)
-
-    # check that the model converges w/o warnings
-    with pytest.warns(None) as record:
-        MultiTaskElasticNet(max_iter=1000).fit(X, y)
-
-    assert not record.list
+    # check that the model converges w/o convergence warnings
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", ConvergenceWarning)
+        MultiTaskElasticNet().fit(X, y)
 
 
-def test_sparse_input_convergence_warning():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sparse_input_convergence_warning(csr_container):
     X, y, _, _ = build_dataset(n_samples=1000, n_features=500)
 
     with pytest.warns(ConvergenceWarning):
-        ElasticNet(max_iter=1, tol=0).fit(
-            sparse.csr_matrix(X, dtype=np.float32), y)
+        ElasticNet(max_iter=1, tol=0).fit(csr_container(X, dtype=np.float32), y)
 
-    # check that the model converges w/o warnings
-    with pytest.warns(None) as record:
-        Lasso(max_iter=1000).fit(sparse.csr_matrix(X, dtype=np.float32), y)
+    # check that the model converges w/o convergence warnings
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", ConvergenceWarning)
+        Lasso().fit(csr_container(X, dtype=np.float32), y)
 
-    assert not record.list
 
-
-@pytest.mark.parametrize("precompute, inner_precompute", [
-    (True, True),
-    ('auto', False),
-    (False, False),
-])
-def test_lassoCV_does_not_set_precompute(monkeypatch, precompute,
-                                         inner_precompute):
+@pytest.mark.parametrize(
+    "precompute, inner_precompute",
+    [
+        (True, True),
+        ("auto", False),
+        (False, False),
+    ],
+)
+def test_lassoCV_does_not_set_precompute(monkeypatch, precompute, inner_precompute):
     X, y, _, _ = build_dataset()
     calls = 0
 
@@ -1280,8 +1198,7 @@ def fit(self, X, y):
             calls += 1
             assert self.precompute == inner_precompute
 
-    monkeypatch.setattr("sklearn.linear_model._coordinate_descent.Lasso",
-                        LassoMock)
+    monkeypatch.setattr("sklearn.linear_model._coordinate_descent.Lasso", LassoMock)
     clf = LassoCV(precompute=precompute)
     clf.fit(X, y)
     assert calls > 0
@@ -1290,115 +1207,299 @@ def fit(self, X, y):
 def test_multi_task_lasso_cv_dtype():
     n_samples, n_features = 10, 3
     rng = np.random.RandomState(42)
-    X = rng.binomial(1, .5, size=(n_samples, n_features))
+    X = rng.binomial(1, 0.5, size=(n_samples, n_features))
     X = X.astype(int)  # make it explicit that X is int
     y = X[:, [0, 0]].copy()
-    est = MultiTaskLassoCV(n_alphas=5, fit_intercept=True).fit(X, y)
+    est = MultiTaskLassoCV(alphas=5, fit_intercept=True).fit(X, y)
     assert_array_almost_equal(est.coef_, [[1, 0, 0]] * 2, decimal=3)
 
 
-@pytest.mark.parametrize('fit_intercept', [True, False])
-@pytest.mark.parametrize('alpha', [0.01])
-@pytest.mark.parametrize('normalize', [False, True])
-@pytest.mark.parametrize('precompute', [False, True])
-def test_enet_sample_weight_consistency(fit_intercept, alpha, normalize,
-                                        precompute):
-    """Test that the impact of sample_weight is consistent."""
-    rng = np.random.RandomState(0)
+@pytest.mark.parametrize("fit_intercept", [True, False])
+@pytest.mark.parametrize("alpha", [0.01])
+@pytest.mark.parametrize("precompute", [False, True])
+@pytest.mark.parametrize("sparse_container", [None] + CSR_CONTAINERS)
+def test_enet_sample_weight_consistency(
+    fit_intercept, alpha, precompute, sparse_container, global_random_seed
+):
+    """Test that the impact of sample_weight is consistent.
+
+    Note that this test is stricter than the common test
+    check_sample_weight_equivalence alone and also tests sparse X.
+    """
+    rng = np.random.RandomState(global_random_seed)
     n_samples, n_features = 10, 5
 
     X = rng.rand(n_samples, n_features)
     y = rng.rand(n_samples)
-    params = dict(alpha=alpha, fit_intercept=fit_intercept,
-                  precompute=precompute, tol=1e-6, l1_ratio=0.5)
+    if sparse_container is not None:
+        X = sparse_container(X)
+    params = dict(
+        alpha=alpha,
+        fit_intercept=fit_intercept,
+        precompute=precompute,
+        tol=1e-6,
+        l1_ratio=0.5,
+    )
 
     reg = ElasticNet(**params).fit(X, y)
     coef = reg.coef_.copy()
     if fit_intercept:
         intercept = reg.intercept_
 
-    # sample_weight=np.ones(..) should be equivalent to sample_weight=None
+    # 1) sample_weight=np.ones(..) should be equivalent to sample_weight=None
     sample_weight = np.ones_like(y)
     reg.fit(X, y, sample_weight=sample_weight)
     assert_allclose(reg.coef_, coef, rtol=1e-6)
     if fit_intercept:
         assert_allclose(reg.intercept_, intercept)
 
-    # sample_weight=None should be equivalent to sample_weight = number
-    sample_weight = 123.
+    # 2) sample_weight=None should be equivalent to sample_weight = number
+    sample_weight = 123.0
     reg.fit(X, y, sample_weight=sample_weight)
     assert_allclose(reg.coef_, coef, rtol=1e-6)
     if fit_intercept:
         assert_allclose(reg.intercept_, intercept)
 
-    # scaling of sample_weight should have no effect, cf. np.average()
-    sample_weight = 2 * np.ones_like(y)
-    reg.fit(X, y, sample_weight=sample_weight)
+    # 3) scaling of sample_weight should have no effect, cf. np.average()
+    sample_weight = rng.uniform(low=0.01, high=2, size=X.shape[0])
+    reg = reg.fit(X, y, sample_weight=sample_weight)
+    coef = reg.coef_.copy()
+    if fit_intercept:
+        intercept = reg.intercept_
+
+    reg.fit(X, y, sample_weight=np.pi * sample_weight)
     assert_allclose(reg.coef_, coef, rtol=1e-6)
     if fit_intercept:
         assert_allclose(reg.intercept_, intercept)
 
-    # setting one element of sample_weight to 0 is equivalent to removing
-    # the corresponding sample
-    sample_weight = np.ones_like(y)
-    sample_weight[-1] = 0
-    reg.fit(X, y, sample_weight=sample_weight)
-    coef1 = reg.coef_.copy()
+    # 4) setting elements of sample_weight to 0 is equivalent to removing these samples
+    sample_weight_0 = sample_weight.copy()
+    sample_weight_0[-5:] = 0
+    y[-5:] *= 1000  # to make excluding those samples important
+    reg.fit(X, y, sample_weight=sample_weight_0)
+    coef_0 = reg.coef_.copy()
     if fit_intercept:
-        intercept1 = reg.intercept_
-    reg.fit(X[:-1], y[:-1])
-    assert_allclose(reg.coef_, coef1, rtol=1e-6)
+        intercept_0 = reg.intercept_
+    reg.fit(X[:-5], y[:-5], sample_weight=sample_weight[:-5])
+    assert_allclose(reg.coef_, coef_0, rtol=1e-6)
     if fit_intercept:
-        assert_allclose(reg.intercept_, intercept1)
+        assert_allclose(reg.intercept_, intercept_0)
+
+    # 5) check that multiplying sample_weight by 2 is equivalent to repeating
+    # corresponding samples twice
+    if sparse_container is not None:
+        X2 = sparse.vstack([X, X[: n_samples // 2]], format="csc")
+    else:
+        X2 = np.concatenate([X, X[: n_samples // 2]], axis=0)
+    y2 = np.concatenate([y, y[: n_samples // 2]])
+    sample_weight_1 = sample_weight.copy()
+    sample_weight_1[: n_samples // 2] *= 2
+    sample_weight_2 = np.concatenate(
+        [sample_weight, sample_weight[: n_samples // 2]], axis=0
+    )
 
-    # check that multiplying sample_weight by 2 is equivalent
-    # to repeating corresponding samples twice
-    if sparse.issparse(X):
-        X = X.toarray()
+    reg1 = ElasticNet(**params).fit(X, y, sample_weight=sample_weight_1)
+    reg2 = ElasticNet(**params).fit(X2, y2, sample_weight=sample_weight_2)
+    assert_allclose(reg1.coef_, reg2.coef_, rtol=1e-6)
 
-    X2 = np.concatenate([X, X[:n_samples//2]], axis=0)
-    y2 = np.concatenate([y, y[:n_samples//2]])
-    sample_weight_1 = np.ones(len(y))
-    sample_weight_1[:n_samples//2] = 2
 
-    reg1 = ElasticNet(**params).fit(
-            X, y, sample_weight=sample_weight_1
+@pytest.mark.parametrize("fit_intercept", [True, False])
+@pytest.mark.parametrize("sparse_container", [None] + CSC_CONTAINERS)
+def test_enet_cv_sample_weight_correctness(
+    fit_intercept, sparse_container, global_random_seed
+):
+    """Test that ElasticNetCV with sample weights gives correct results.
+
+    We fit the same model twice, once with weighted training data, once with repeated
+    data points in the training data and check that both models converge to the
+    same solution.
+
+    Since this model uses an internal cross-validation scheme to tune the alpha
+    regularization parameter, we make sure that the repetitions only occur within
+    a specific CV group. Data points belonging to other CV groups stay
+    unit-weighted / "unrepeated".
+    """
+    rng = np.random.RandomState(global_random_seed)
+    n_splits, n_samples_per_cv, n_features = 3, 10, 5
+    X_with_weights = rng.rand(n_splits * n_samples_per_cv, n_features)
+    beta = rng.rand(n_features)
+    beta[0:2] = 0
+    y_with_weights = X_with_weights @ beta + rng.rand(n_splits * n_samples_per_cv)
+
+    if sparse_container is not None:
+        X_with_weights = sparse_container(X_with_weights)
+    params = dict(tol=1e-6)
+
+    # Assign random integer weights only to the first cross-validation group.
+    # The samples in the other cross-validation groups are left with unit
+    # weights.
+
+    sw = np.ones_like(y_with_weights)
+    sw[:n_samples_per_cv] = rng.randint(0, 5, size=n_samples_per_cv)
+    groups_with_weights = np.concatenate(
+        [
+            np.full(n_samples_per_cv, 0),
+            np.full(n_samples_per_cv, 1),
+            np.full(n_samples_per_cv, 2),
+        ]
+    )
+    splits_with_weights = list(
+        LeaveOneGroupOut().split(X_with_weights, groups=groups_with_weights)
     )
+    reg_with_weights = ElasticNetCV(
+        cv=splits_with_weights, fit_intercept=fit_intercept, **params
+    )
+
+    reg_with_weights.fit(X_with_weights, y_with_weights, sample_weight=sw)
+
+    if sparse_container is not None:
+        X_with_weights = X_with_weights.toarray()
+    X_with_repetitions = np.repeat(X_with_weights, sw.astype(int), axis=0)
+    if sparse_container is not None:
+        X_with_repetitions = sparse_container(X_with_repetitions)
+
+    y_with_repetitions = np.repeat(y_with_weights, sw.astype(int), axis=0)
+    groups_with_repetitions = np.repeat(groups_with_weights, sw.astype(int), axis=0)
 
-    reg2 = ElasticNet(**params).fit(
-            X2, y2, sample_weight=None
+    splits_with_repetitions = list(
+        LeaveOneGroupOut().split(X_with_repetitions, groups=groups_with_repetitions)
     )
-    assert_allclose(reg1.coef_, reg2.coef_)
+    reg_with_repetitions = ElasticNetCV(
+        cv=splits_with_repetitions, fit_intercept=fit_intercept, **params
+    )
+    reg_with_repetitions.fit(X_with_repetitions, y_with_repetitions)
+
+    # Check that the alpha selection process is the same:
+    assert_allclose(reg_with_weights.mse_path_, reg_with_repetitions.mse_path_)
+    assert_allclose(reg_with_weights.alphas_, reg_with_repetitions.alphas_)
+    assert reg_with_weights.alpha_ == pytest.approx(reg_with_repetitions.alpha_)
+
+    # Check that the final model coefficients are the same:
+    assert_allclose(reg_with_weights.coef_, reg_with_repetitions.coef_, atol=1e-10)
+    assert reg_with_weights.intercept_ == pytest.approx(reg_with_repetitions.intercept_)
+
+
+@pytest.mark.parametrize("sample_weight", [False, True])
+def test_enet_cv_grid_search(sample_weight):
+    """Test that ElasticNetCV gives same result as GridSearchCV."""
+    n_samples, n_features = 200, 10
+    cv = 5
+    X, y = make_regression(
+        n_samples=n_samples,
+        n_features=n_features,
+        effective_rank=10,
+        n_informative=n_features - 4,
+        noise=10,
+        random_state=0,
+    )
+    if sample_weight:
+        sample_weight = np.linspace(1, 5, num=n_samples)
+    else:
+        sample_weight = None
 
+    alphas = np.logspace(np.log10(1e-5), np.log10(1), num=10)
+    l1_ratios = [0.1, 0.5, 0.9]
+    reg = ElasticNetCV(cv=cv, alphas=alphas, l1_ratio=l1_ratios)
+    reg.fit(X, y, sample_weight=sample_weight)
 
-def test_enet_sample_weight_sparse():
-    reg = ElasticNet()
-    X = sparse.csc_matrix(np.zeros((3, 2)))
-    y = np.array([-1, 0, 1])
-    sw = np.array([1, 2, 3])
-    with pytest.raises(ValueError, match="Sample weights do not.*support "
-                                         "sparse matrices"):
-        reg.fit(X, y, sample_weight=sw, check_input=True)
-
-
-@pytest.mark.parametrize("backend", ["loky", "threading"])
-@pytest.mark.parametrize("estimator",
-                         [ElasticNetCV, MultiTaskElasticNetCV,
-                          LassoCV, MultiTaskLassoCV])
-def test_linear_models_cv_fit_for_all_backends(backend, estimator):
-    # LinearModelsCV.fit performs inplace operations on input data which is
-    # memmapped when using loky backend, causing an error due to unexpected
-    # behavior of fancy indexing of read-only memmaps (cf. numpy#14132).
+    param = {"alpha": alphas, "l1_ratio": l1_ratios}
+    gs = GridSearchCV(
+        estimator=ElasticNet(),
+        param_grid=param,
+        cv=cv,
+        scoring="neg_mean_squared_error",
+    ).fit(X, y, sample_weight=sample_weight)
 
-    if (parse_version(joblib.__version__) < parse_version('0.12')
-            and backend == 'loky'):
-        pytest.skip('loky backend does not exist in joblib <0.12')
+    assert reg.l1_ratio_ == pytest.approx(gs.best_params_["l1_ratio"])
+    assert reg.alpha_ == pytest.approx(gs.best_params_["alpha"])
 
-    # Create a problem sufficiently large to cause memmapping (1MB).
-    n_targets = 1 + (estimator in (MultiTaskElasticNetCV, MultiTaskLassoCV))
-    X, y = make_regression(20000, 10, n_targets=n_targets)
 
-    with joblib.parallel_backend(backend=backend):
+@pytest.mark.parametrize("fit_intercept", [True, False])
+@pytest.mark.parametrize("l1_ratio", [0, 0.5, 1])
+@pytest.mark.parametrize("precompute", [False, True])
+@pytest.mark.parametrize("sparse_container", [None] + CSC_CONTAINERS)
+def test_enet_cv_sample_weight_consistency(
+    fit_intercept, l1_ratio, precompute, sparse_container
+):
+    """Test that the impact of sample_weight is consistent."""
+    rng = np.random.RandomState(0)
+    n_samples, n_features = 10, 5
+
+    X = rng.rand(n_samples, n_features)
+    y = X.sum(axis=1) + rng.rand(n_samples)
+    params = dict(
+        l1_ratio=l1_ratio,
+        fit_intercept=fit_intercept,
+        precompute=precompute,
+        tol=1e-6,
+        cv=3,
+    )
+    if sparse_container is not None:
+        X = sparse_container(X)
+
+    if l1_ratio == 0:
+        params.pop("l1_ratio", None)
+        reg = LassoCV(**params).fit(X, y)
+    else:
+        reg = ElasticNetCV(**params).fit(X, y)
+    coef = reg.coef_.copy()
+    if fit_intercept:
+        intercept = reg.intercept_
+
+    # sample_weight=np.ones(..) should be equivalent to sample_weight=None
+    sample_weight = np.ones_like(y)
+    reg.fit(X, y, sample_weight=sample_weight)
+    assert_allclose(reg.coef_, coef, rtol=1e-6)
+    if fit_intercept:
+        assert_allclose(reg.intercept_, intercept)
+
+    # sample_weight=None should be equivalent to sample_weight = number
+    sample_weight = 123.0
+    reg.fit(X, y, sample_weight=sample_weight)
+    assert_allclose(reg.coef_, coef, rtol=1e-6)
+    if fit_intercept:
+        assert_allclose(reg.intercept_, intercept)
+
+    # scaling of sample_weight should have no effect, cf. np.average()
+    sample_weight = 2 * np.ones_like(y)
+    reg.fit(X, y, sample_weight=sample_weight)
+    assert_allclose(reg.coef_, coef, rtol=1e-6)
+    if fit_intercept:
+        assert_allclose(reg.intercept_, intercept)
+
+
+@pytest.mark.parametrize("X_is_sparse", [False, True])
+@pytest.mark.parametrize("fit_intercept", [False, True])
+@pytest.mark.parametrize("sample_weight", [np.array([10, 1, 10, 1]), None])
+def test_enet_alpha_max_sample_weight(X_is_sparse, fit_intercept, sample_weight):
+    X = np.array([[3.0, 1.0], [2.0, 5.0], [5.0, 3.0], [1.0, 4.0]])
+    beta = np.array([1, 1])
+    y = X @ beta
+    if X_is_sparse:
+        X = sparse.csc_matrix(X)
+    # Test alpha_max makes coefs zero.
+    reg = ElasticNetCV(alphas=1, cv=2, eps=1, fit_intercept=fit_intercept)
+    reg.fit(X, y, sample_weight=sample_weight)
+    assert_allclose(reg.coef_, 0, atol=1e-5)
+    alpha_max = reg.alpha_
+    # Test smaller alpha makes coefs nonzero.
+    reg = ElasticNet(alpha=0.99 * alpha_max, fit_intercept=fit_intercept)
+    reg.fit(X, y, sample_weight=sample_weight)
+    assert_array_less(1e-3, np.max(np.abs(reg.coef_)))
+
+
+@pytest.mark.parametrize("estimator", [ElasticNetCV, LassoCV])
+def test_linear_models_cv_fit_with_loky(estimator):
+    # LinearModelsCV.fit performs operations on fancy-indexed memmapped
+    # data when using the loky backend, causing an error due to unexpected
+    # behavior of fancy indexing of read-only memmaps (cf. numpy#14132).
+
+    # Create a problem sufficiently large to cause memmapping (1MB).
+    # Unfortunately the scikit-learn and joblib APIs do not make it possible to
+    # change the max_nbyte of the inner Parallel call.
+    X, y = make_regression(int(1e6) // 8 + 1, 1)
+    assert X.nbytes > 1e6  # 1 MB
+    with joblib.parallel_backend("loky"):
         estimator(n_jobs=2, cv=3).fit(X, y)
 
 
@@ -1419,3 +1520,286 @@ def test_enet_sample_weight_does_not_overwrite_sample_weight(check_input):
     reg.fit(X, y, sample_weight=sample_weight, check_input=check_input)
 
     assert_array_equal(sample_weight, sample_weight_1_25)
+
+
+@pytest.mark.filterwarnings("ignore::sklearn.exceptions.ConvergenceWarning")
+@pytest.mark.parametrize("ridge_alpha", [1e-1, 1.0, 1e6])
+def test_enet_ridge_consistency(ridge_alpha):
+    # Check that ElasticNet(l1_ratio=0) converges to the same solution as Ridge
+    # provided that the value of alpha is adapted.
+    #
+    # XXX: this test does not pass for weaker regularization (lower values of
+    # ridge_alpha): it could be either a problem of ElasticNet or Ridge (less
+    # likely) and depends on the dataset statistics: lower values for
+    # effective_rank are more problematic in particular.
+
+    rng = np.random.RandomState(42)
+    n_samples = 300
+    X, y = make_regression(
+        n_samples=n_samples,
+        n_features=100,
+        effective_rank=10,
+        n_informative=50,
+        random_state=rng,
+    )
+    sw = rng.uniform(low=0.01, high=10, size=X.shape[0])
+    alpha = 1.0
+    common_params = dict(
+        tol=1e-12,
+    )
+    ridge = Ridge(alpha=alpha, **common_params).fit(X, y, sample_weight=sw)
+
+    alpha_enet = alpha / sw.sum()
+    enet = ElasticNet(alpha=alpha_enet, l1_ratio=0, **common_params).fit(
+        X, y, sample_weight=sw
+    )
+    assert_allclose(ridge.coef_, enet.coef_)
+    assert_allclose(ridge.intercept_, enet.intercept_)
+
+
+@pytest.mark.parametrize(
+    "estimator",
+    [
+        Lasso(alpha=1.0),
+        ElasticNet(alpha=1.0, l1_ratio=0.1),
+    ],
+)
+def test_sample_weight_invariance(estimator):
+    rng = np.random.RandomState(42)
+    X, y = make_regression(
+        n_samples=100,
+        n_features=300,
+        effective_rank=10,
+        n_informative=50,
+        random_state=rng,
+    )
+    sw = rng.uniform(low=0.01, high=2, size=X.shape[0])
+    params = dict(tol=1e-12)
+
+    # Check that setting some weights to 0 is equivalent to trimming the
+    # samples:
+    cutoff = X.shape[0] // 3
+    sw_with_null = sw.copy()
+    sw_with_null[:cutoff] = 0.0
+    X_trimmed, y_trimmed = X[cutoff:, :], y[cutoff:]
+    sw_trimmed = sw[cutoff:]
+
+    reg_trimmed = (
+        clone(estimator)
+        .set_params(**params)
+        .fit(X_trimmed, y_trimmed, sample_weight=sw_trimmed)
+    )
+    reg_null_weighted = (
+        clone(estimator).set_params(**params).fit(X, y, sample_weight=sw_with_null)
+    )
+    assert_allclose(reg_null_weighted.coef_, reg_trimmed.coef_)
+    assert_allclose(reg_null_weighted.intercept_, reg_trimmed.intercept_)
+
+    # Check that duplicating the training dataset is equivalent to multiplying
+    # the weights by 2:
+    X_dup = np.concatenate([X, X], axis=0)
+    y_dup = np.concatenate([y, y], axis=0)
+    sw_dup = np.concatenate([sw, sw], axis=0)
+
+    reg_2sw = clone(estimator).set_params(**params).fit(X, y, sample_weight=2 * sw)
+    reg_dup = (
+        clone(estimator).set_params(**params).fit(X_dup, y_dup, sample_weight=sw_dup)
+    )
+
+    assert_allclose(reg_2sw.coef_, reg_dup.coef_)
+    assert_allclose(reg_2sw.intercept_, reg_dup.intercept_)
+
+
+def test_read_only_buffer():
+    """Test that sparse coordinate descent works for read-only buffers"""
+
+    rng = np.random.RandomState(0)
+    clf = ElasticNet(alpha=0.1, copy_X=True, random_state=rng)
+    X = np.asfortranarray(rng.uniform(size=(100, 10)))
+    X.setflags(write=False)
+
+    y = rng.rand(100)
+    clf.fit(X, y)
+
+
+@pytest.mark.parametrize(
+    "EstimatorCV",
+    [ElasticNetCV, LassoCV, MultiTaskElasticNetCV, MultiTaskLassoCV],
+)
+def test_cv_estimators_reject_params_with_no_routing_enabled(EstimatorCV):
+    """Check that the models inheriting from class:`LinearModelCV` raise an
+    error when any `params` are passed when routing is not enabled.
+    """
+    X, y = make_regression(random_state=42)
+    groups = np.array([0, 1] * (len(y) // 2))
+    estimator = EstimatorCV()
+    msg = "is only supported if enable_metadata_routing=True"
+    with pytest.raises(ValueError, match=msg):
+        estimator.fit(X, y, groups=groups)
+
+
+@pytest.mark.parametrize(
+    "MultiTaskEstimatorCV",
+    [MultiTaskElasticNetCV, MultiTaskLassoCV],
+)
+@config_context(enable_metadata_routing=True)
+def test_multitask_cv_estimators_with_sample_weight(MultiTaskEstimatorCV):
+    """Check that for :class:`MultiTaskElasticNetCV` and
+    class:`MultiTaskLassoCV` if `sample_weight` is passed and the
+    CV splitter does not support `sample_weight` an error is raised.
+    On the other hand if the splitter does support `sample_weight`
+    while `sample_weight` is passed there is no error and process
+    completes smoothly as before.
+    """
+
+    class CVSplitter(GroupsConsumerMixin, BaseCrossValidator):
+        def get_n_splits(self, X=None, y=None, groups=None, metadata=None):
+            pass  # pragma: nocover
+
+    class CVSplitterSampleWeight(CVSplitter):
+        def split(self, X, y=None, groups=None, sample_weight=None):
+            split_index = len(X) // 2
+            train_indices = list(range(0, split_index))
+            test_indices = list(range(split_index, len(X)))
+            yield test_indices, train_indices
+            yield train_indices, test_indices
+
+    X, y = make_regression(random_state=42, n_targets=2)
+    sample_weight = np.ones(X.shape[0])
+
+    # If CV splitter does not support sample_weight an error is raised
+    splitter = CVSplitter().set_split_request(groups=True)
+    estimator = MultiTaskEstimatorCV(cv=splitter)
+    msg = "do not support sample weights"
+    with pytest.raises(ValueError, match=msg):
+        estimator.fit(X, y, sample_weight=sample_weight)
+
+    # If CV splitter does support sample_weight no error is raised
+    splitter = CVSplitterSampleWeight().set_split_request(
+        groups=True, sample_weight=True
+    )
+    estimator = MultiTaskEstimatorCV(cv=splitter)
+    estimator.fit(X, y, sample_weight=sample_weight)
+
+
+# TODO(1.9): remove
+@pytest.mark.parametrize(
+    "Estimator", [LassoCV, ElasticNetCV, MultiTaskLassoCV, MultiTaskElasticNetCV]
+)
+def test_linear_model_cv_deprecated_n_alphas(Estimator):
+    """Check the deprecation of n_alphas in favor of alphas."""
+    X, y = make_regression(n_targets=2, random_state=42)
+
+    # Asses warning message raised by LinearModelCV when n_alphas is used
+    with pytest.warns(
+        FutureWarning,
+        match="'n_alphas' was deprecated in 1.7 and will be removed in 1.9",
+    ):
+        clf = Estimator(n_alphas=5)
+        if clf._is_multitask():
+            clf = clf.fit(X, y)
+        else:
+            clf = clf.fit(X, y[:, 0])
+
+    # Asses no warning message raised when n_alphas is not used
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        clf = Estimator(alphas=5)
+        if clf._is_multitask():
+            clf = clf.fit(X, y)
+        else:
+            clf = clf.fit(X, y[:, 0])
+
+
+# TODO(1.9): remove
+@pytest.mark.parametrize(
+    "Estimator", [ElasticNetCV, LassoCV, MultiTaskLassoCV, MultiTaskElasticNetCV]
+)
+def test_linear_model_cv_deprecated_alphas_none(Estimator):
+    """Check the deprecation of alphas=None."""
+    X, y = make_regression(n_targets=2, random_state=42)
+
+    with pytest.warns(
+        FutureWarning, match="'alphas=None' is deprecated and will be removed in 1.9"
+    ):
+        clf = Estimator(alphas=None)
+        if clf._is_multitask():
+            clf.fit(X, y)
+        else:
+            clf.fit(X, y[:, 0])
+
+
+# TODO(1.9): remove
+@pytest.mark.parametrize(
+    "Estimator", [ElasticNetCV, LassoCV, MultiTaskLassoCV, MultiTaskElasticNetCV]
+)
+def test_linear_model_cv_alphas_n_alphas_unset(Estimator):
+    """Check that no warning is raised when both n_alphas and alphas are unset."""
+    X, y = make_regression(n_targets=2, random_state=42)
+
+    # Asses no warning message raised when n_alphas is not used
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        clf = Estimator()
+        if clf._is_multitask():
+            clf = clf.fit(X, y)
+        else:
+            clf = clf.fit(X, y[:, 0])
+
+
+# TODO(1.9): remove
+@pytest.mark.filterwarnings("ignore:'n_alphas' was deprecated in 1.7")
+@pytest.mark.parametrize(
+    "Estimator", [ElasticNetCV, LassoCV, MultiTaskLassoCV, MultiTaskElasticNetCV]
+)
+def test_linear_model_cv_alphas(Estimator):
+    """Check that the behavior of alphas is consistent with n_alphas."""
+    X, y = make_regression(n_targets=2, random_state=42)
+
+    # n_alphas is set, alphas is not => n_alphas is used
+    clf = Estimator(n_alphas=5)
+    if clf._is_multitask():
+        clf.fit(X, y)
+    else:
+        clf.fit(X, y[:, 0])
+    assert len(clf.alphas_) == 5
+
+    # n_alphas is set, alphas is set => alphas has priority
+    clf = Estimator(n_alphas=5, alphas=10)
+    if clf._is_multitask():
+        clf.fit(X, y)
+    else:
+        clf.fit(X, y[:, 0])
+    assert len(clf.alphas_) == 10
+
+    # same with alphas array-like
+    clf = Estimator(n_alphas=5, alphas=np.arange(10))
+    if clf._is_multitask():
+        clf.fit(X, y)
+    else:
+        clf.fit(X, y[:, 0])
+    assert len(clf.alphas_) == 10
+
+    # n_alphas is not set, alphas is set => alphas is used
+    clf = Estimator(alphas=10)
+    if clf._is_multitask():
+        clf.fit(X, y)
+    else:
+        clf.fit(X, y[:, 0])
+    assert len(clf.alphas_) == 10
+
+    # same with alphas array-like
+    clf = Estimator(alphas=np.arange(10))
+    if clf._is_multitask():
+        clf.fit(X, y)
+    else:
+        clf.fit(X, y[:, 0])
+    assert len(clf.alphas_) == 10
+
+    # both are not set => default = 100
+    clf = Estimator()
+    if clf._is_multitask():
+        clf.fit(X, y)
+    else:
+        clf.fit(X, y[:, 0])
+    assert len(clf.alphas_) == 100
diff --git a/sklearn/linear_model/tests/test_huber.py b/sklearn/linear_model/tests/test_huber.py
index 156cd4b57dbc8..9c0c7d213ee27 100644
--- a/sklearn/linear_model/tests/test_huber.py
+++ b/sklearn/linear_model/tests/test_huber.py
@@ -1,25 +1,27 @@
-# Authors: Manoj Kumar mks542@nyu.edu
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import numpy as np
-from scipy import optimize, sparse
-
-from sklearn.utils._testing import assert_almost_equal
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils._testing import assert_array_almost_equal
+import pytest
+from scipy import optimize
 
 from sklearn.datasets import make_regression
-from sklearn.linear_model import (
-    HuberRegressor, LinearRegression, SGDRegressor, Ridge)
+from sklearn.linear_model import HuberRegressor, LinearRegression, Ridge, SGDRegressor
 from sklearn.linear_model._huber import _huber_loss_and_gradient
+from sklearn.utils._testing import (
+    assert_almost_equal,
+    assert_array_almost_equal,
+    assert_array_equal,
+)
+from sklearn.utils.fixes import CSR_CONTAINERS
 
 
 def make_regression_with_outliers(n_samples=50, n_features=20):
     rng = np.random.RandomState(0)
     # Generate data with outliers by replacing 10% of the samples with noise.
     X, y = make_regression(
-        n_samples=n_samples, n_features=n_features,
-        random_state=0, noise=0.05)
+        n_samples=n_samples, n_features=n_features, random_state=0, noise=0.05
+    )
 
     # Replace 10% of the sample with noise.
     num_noise = int(0.1 * n_samples)
@@ -65,11 +67,13 @@ def grad_func(x, *args):
             w = rng.randn(n_features)
             w[-1] = np.abs(w[-1])
             grad_same = optimize.check_grad(
-                loss_func, grad_func, w, X, y, 0.01, 0.1, sample_weight)
+                loss_func, grad_func, w, X, y, 0.01, 0.1, sample_weight
+            )
             assert_almost_equal(grad_same, 1e-6, 4)
 
 
-def test_huber_sample_weights():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_huber_sample_weights(csr_container):
     # Test sample_weights implementation in HuberRegressor"""
 
     X, y = make_regression_with_outliers()
@@ -82,13 +86,11 @@ def test_huber_sample_weights():
     # sure that the number of decimal places used is somewhat insensitive to
     # the amplitude of the coefficients and therefore to the scale of the
     # data and the regularization parameter
-    scale = max(np.mean(np.abs(huber.coef_)),
-                np.mean(np.abs(huber.intercept_)))
+    scale = max(np.mean(np.abs(huber.coef_)), np.mean(np.abs(huber.intercept_)))
 
     huber.fit(X, y, sample_weight=np.ones(y.shape[0]))
     assert_array_almost_equal(huber.coef_ / scale, huber_coef / scale)
-    assert_array_almost_equal(huber.intercept_ / scale,
-                              huber_intercept / scale)
+    assert_array_almost_equal(huber.intercept_ / scale, huber_intercept / scale)
 
     X, y = make_regression_with_outliers(n_samples=5, n_features=20)
     X_new = np.vstack((X, np.vstack((X[1], X[1], X[3]))))
@@ -102,23 +104,22 @@ def test_huber_sample_weights():
     huber.fit(X, y, sample_weight=sample_weight)
 
     assert_array_almost_equal(huber.coef_ / scale, huber_coef / scale)
-    assert_array_almost_equal(huber.intercept_ / scale,
-                              huber_intercept / scale)
+    assert_array_almost_equal(huber.intercept_ / scale, huber_intercept / scale)
 
     # Test sparse implementation with sample weights.
-    X_csr = sparse.csr_matrix(X)
+    X_csr = csr_container(X)
     huber_sparse = HuberRegressor()
     huber_sparse.fit(X_csr, y, sample_weight=sample_weight)
-    assert_array_almost_equal(huber_sparse.coef_ / scale,
-                              huber_coef / scale)
+    assert_array_almost_equal(huber_sparse.coef_ / scale, huber_coef / scale)
 
 
-def test_huber_sparse():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_huber_sparse(csr_container):
     X, y = make_regression_with_outliers()
     huber = HuberRegressor(alpha=0.1)
     huber.fit(X, y)
 
-    X_csr = sparse.csr_matrix(X)
+    X_csr = csr_container(X)
     huber_sparse = HuberRegressor(alpha=0.1)
     huber_sparse.fit(X_csr, y)
     assert_array_almost_equal(huber_sparse.coef_, huber.coef_)
@@ -128,16 +129,16 @@ def test_huber_sparse():
 def test_huber_scaling_invariant():
     # Test that outliers filtering is scaling independent.
     X, y = make_regression_with_outliers()
-    huber = HuberRegressor(fit_intercept=False, alpha=0.0, max_iter=100)
+    huber = HuberRegressor(fit_intercept=False, alpha=0.0)
     huber.fit(X, y)
     n_outliers_mask_1 = huber.outliers_
     assert not np.all(n_outliers_mask_1)
 
-    huber.fit(X, 2. * y)
+    huber.fit(X, 2.0 * y)
     n_outliers_mask_2 = huber.outliers_
     assert_array_equal(n_outliers_mask_2, n_outliers_mask_1)
 
-    huber.fit(2. * X, 2. * y)
+    huber.fit(2.0 * X, 2.0 * y)
     n_outliers_mask_3 = huber.outliers_
     assert_array_equal(n_outliers_mask_3, n_outliers_mask_1)
 
@@ -149,8 +150,7 @@ def test_huber_and_sgd_same_results():
 
     # Fit once to find out the scale parameter. Scale down X and y by scale
     # so that the scale parameter is optimized to 1.0
-    huber = HuberRegressor(fit_intercept=False, alpha=0.0, max_iter=100,
-                           epsilon=1.35)
+    huber = HuberRegressor(fit_intercept=False, alpha=0.0, epsilon=1.35)
     huber.fit(X, y)
     X_scale = X / huber.scale_
     y_scale = y / huber.scale_
@@ -158,16 +158,22 @@ def test_huber_and_sgd_same_results():
     assert_almost_equal(huber.scale_, 1.0, 3)
 
     sgdreg = SGDRegressor(
-        alpha=0.0, loss="huber", shuffle=True, random_state=0, max_iter=10000,
-        fit_intercept=False, epsilon=1.35, tol=None)
+        alpha=0.0,
+        loss="huber",
+        shuffle=True,
+        random_state=0,
+        max_iter=10000,
+        fit_intercept=False,
+        epsilon=1.35,
+        tol=None,
+    )
     sgdreg.fit(X_scale, y_scale)
     assert_array_almost_equal(huber.coef_, sgdreg.coef_, 1)
 
 
 def test_huber_warm_start():
     X, y = make_regression_with_outliers()
-    huber_warm = HuberRegressor(
-        alpha=1.0, max_iter=10000, warm_start=True, tol=1e-1)
+    huber_warm = HuberRegressor(alpha=1.0, max_iter=10000, warm_start=True, tol=1e-1)
 
     huber_warm.fit(X, y)
     huber_warm_coef = huber_warm.coef_.copy()
@@ -205,7 +211,6 @@ def test_huber_better_r2_score():
 
 def test_huber_bool():
     # Test that it does not crash with bool data
-    X, y = make_regression(n_samples=200, n_features=2, noise=4.0,
-                           random_state=0)
+    X, y = make_regression(n_samples=200, n_features=2, noise=4.0, random_state=0)
     X_bool = X > 0
     HuberRegressor().fit(X_bool, y)
diff --git a/sklearn/linear_model/tests/test_least_angle.py b/sklearn/linear_model/tests/test_least_angle.py
index a8b0e939c080d..9b4a39750e03a 100644
--- a/sklearn/linear_model/tests/test_least_angle.py
+++ b/sklearn/linear_model/tests/test_least_angle.py
@@ -4,19 +4,27 @@
 import pytest
 from scipy import linalg
 
+from sklearn import datasets, linear_model
 from sklearn.base import clone
-from sklearn.model_selection import train_test_split
-from sklearn.utils._testing import assert_allclose
-from sklearn.utils._testing import assert_array_almost_equal
-from sklearn.utils._testing import assert_raises
-from sklearn.utils._testing import ignore_warnings
-from sklearn.utils._testing import TempMemmap
-from sklearn.utils.fixes import np_version, parse_version
 from sklearn.exceptions import ConvergenceWarning
-from sklearn import linear_model, datasets
+from sklearn.linear_model import (
+    Lars,
+    LarsCV,
+    LassoLars,
+    LassoLarsCV,
+    LassoLarsIC,
+    lars_path,
+)
 from sklearn.linear_model._least_angle import _lars_path_residues
-from sklearn.linear_model import LassoLarsIC, lars_path
-from sklearn.linear_model import Lars, LassoLars
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import StandardScaler
+from sklearn.utils._testing import (
+    TempMemmap,
+    assert_allclose,
+    assert_array_almost_equal,
+    ignore_warnings,
+)
 
 # TODO: use another dataset that has multiple drops
 diabetes = datasets.load_diabetes()
@@ -30,15 +38,14 @@ def test_simple():
     # Principle of Lars is to keep covariances tied and decreasing
 
     # also test verbose output
-    from io import StringIO
     import sys
+    from io import StringIO
+
     old_stdout = sys.stdout
     try:
         sys.stdout = StringIO()
 
-        _, _, coef_path_ = linear_model.lars_path(
-            X, y, method="lar", verbose=10
-        )
+        _, _, coef_path_ = linear_model.lars_path(X, y, method="lar", verbose=10)
 
         sys.stdout = old_stdout
 
@@ -86,18 +93,17 @@ def _assert_same_lars_path_result(output1, output2):
 def test_lars_path_gram_equivalent(method, return_path):
     _assert_same_lars_path_result(
         linear_model.lars_path_gram(
-            Xy=Xy, Gram=G, n_samples=n_samples, method=method,
-            return_path=return_path),
-        linear_model.lars_path(
-            X, y, Gram=G, method=method,
-            return_path=return_path))
+            Xy=Xy, Gram=G, n_samples=n_samples, method=method, return_path=return_path
+        ),
+        linear_model.lars_path(X, y, Gram=G, method=method, return_path=return_path),
+    )
 
 
 def test_x_none_gram_none_raises_value_error():
     # Test that lars_path with no X and Gram raises exception
     Xy = np.dot(X.T, y)
-    assert_raises(ValueError, linear_model.lars_path, None, y, Gram=None,
-                  Xy=Xy)
+    with pytest.raises(ValueError, match="X and Gram cannot both be unspecified"):
+        linear_model.lars_path(None, y, Gram=None, Xy=Xy)
 
 
 def test_all_precomputed():
@@ -106,66 +112,66 @@ def test_all_precomputed():
     Xy = np.dot(X.T, y)
     for method in "lar", "lasso":
         output = linear_model.lars_path(X, y, method=method)
-        output_pre = linear_model.lars_path(X, y, Gram=G, Xy=Xy,
-                                            method=method)
+        output_pre = linear_model.lars_path(X, y, Gram=G, Xy=Xy, method=method)
         for expected, got in zip(output, output_pre):
             assert_array_almost_equal(expected, got)
 
 
-@pytest.mark.filterwarnings('ignore: `rcond` parameter will change')
-# numpy deprecation
+# TODO: remove warning filter when numpy min version >= 2.0.0
+@pytest.mark.filterwarnings("ignore: `rcond` parameter will change")
 def test_lars_lstsq():
     # Test that Lars gives least square solution at the end
     # of the path
     X1 = 3 * X  # use un-normalized dataset
-    clf = linear_model.LassoLars(alpha=0.)
+    clf = linear_model.LassoLars(alpha=0.0)
     clf.fit(X1, y)
-    # Avoid FutureWarning about default value change when numpy >= 1.14
-    rcond = None if np_version >= parse_version('1.14') else -1
-    coef_lstsq = np.linalg.lstsq(X1, y, rcond=rcond)[0]
+    coef_lstsq = np.linalg.lstsq(X1, y)[0]
     assert_array_almost_equal(clf.coef_, coef_lstsq)
 
 
-@pytest.mark.filterwarnings('ignore:`rcond` parameter will change')
-# numpy deprecation
+# TODO: remove warning filter when numpy min version >= 2.0.0
+@pytest.mark.filterwarnings("ignore: `rcond` parameter will change")
 def test_lasso_gives_lstsq_solution():
     # Test that Lars Lasso gives least square solution at the end
     # of the path
-    _, _, coef_path_ = linear_model.lars_path(X, y, method='lasso')
+    _, _, coef_path_ = linear_model.lars_path(X, y, method="lasso")
     coef_lstsq = np.linalg.lstsq(X, y)[0]
     assert_array_almost_equal(coef_lstsq, coef_path_[:, -1])
 
 
 def test_collinearity():
     # Check that lars_path is robust to collinearity in input
-    X = np.array([[3., 3., 1.],
-                  [2., 2., 0.],
-                  [1., 1., 0]])
-    y = np.array([1., 0., 0])
+    X = np.array([[3.0, 3.0, 1.0], [2.0, 2.0, 0.0], [1.0, 1.0, 0]])
+    y = np.array([1.0, 0.0, 0])
     rng = np.random.RandomState(0)
 
     f = ignore_warnings
     _, _, coef_path_ = f(linear_model.lars_path)(X, y, alpha_min=0.01)
     assert not np.isnan(coef_path_).any()
     residual = np.dot(X, coef_path_[:, -1]) - y
-    assert (residual ** 2).sum() < 1.  # just make sure it's bounded
+    assert (residual**2).sum() < 1.0  # just make sure it's bounded
 
     n_samples = 10
     X = rng.rand(n_samples, 5)
     y = np.zeros(n_samples)
-    _, _, coef_path_ = linear_model.lars_path(X, y, Gram='auto', copy_X=False,
-                                              copy_Gram=False, alpha_min=0.,
-                                              method='lasso', verbose=0,
-                                              max_iter=500)
+    _, _, coef_path_ = linear_model.lars_path(
+        X,
+        y,
+        Gram="auto",
+        copy_X=False,
+        copy_Gram=False,
+        alpha_min=0.0,
+        method="lasso",
+        verbose=0,
+        max_iter=500,
+    )
     assert_array_almost_equal(coef_path_, np.zeros_like(coef_path_))
 
 
 def test_no_path():
     # Test that the ``return_path=False`` option returns the correct output
     alphas_, _, coef_path_ = linear_model.lars_path(X, y, method="lar")
-    alpha_, _, coef = linear_model.lars_path(
-        X, y, method="lar", return_path=False
-    )
+    alpha_, _, coef = linear_model.lars_path(X, y, method="lar", return_path=False)
 
     assert_array_almost_equal(coef, coef_path_[:, -1])
     assert alpha_ == alphas_[-1]
@@ -189,24 +195,26 @@ def test_no_path_all_precomputed():
     G = np.dot(X.T, X)
     Xy = np.dot(X.T, y)
     alphas_, _, coef_path_ = linear_model.lars_path(
-        X, y, method='lasso', Xy=Xy, Gram=G, alpha_min=0.9)
+        X, y, method="lasso", Xy=Xy, Gram=G, alpha_min=0.9
+    )
     alpha_, _, coef = linear_model.lars_path(
-        X, y, method='lasso', Gram=G, Xy=Xy, alpha_min=0.9, return_path=False)
+        X, y, method="lasso", Gram=G, Xy=Xy, alpha_min=0.9, return_path=False
+    )
 
     assert_array_almost_equal(coef, coef_path_[:, -1])
     assert alpha_ == alphas_[-1]
 
 
 @pytest.mark.parametrize(
-        'classifier',
-        [linear_model.Lars, linear_model.LarsCV, linear_model.LassoLarsIC])
+    "classifier", [linear_model.Lars, linear_model.LarsCV, linear_model.LassoLarsIC]
+)
 def test_lars_precompute(classifier):
     # Check for different values of precompute
     G = np.dot(X.T, X)
 
     clf = classifier(precompute=G)
     output_1 = ignore_warnings(clf.fit)(X, y).coef_
-    for precompute in [True, False, 'auto', None]:
+    for precompute in [True, False, "auto", None]:
         clf = classifier(precompute=precompute)
         output_2 = clf.fit(X, y).coef_
         assert_array_almost_equal(output_1, output_2, decimal=8)
@@ -214,7 +222,7 @@ def test_lars_precompute(classifier):
 
 def test_singular_matrix():
     # Test when input is a singular matrix
-    X1 = np.array([[1, 1.], [1., 1.]])
+    X1 = np.array([[1, 1.0], [1.0, 1.0]])
     y1 = np.array([1, 1])
     _, _, coef_path = linear_model.lars_path(X1, y1)
     assert_array_almost_equal(coef_path.T, [[0, 0], [1, 0]])
@@ -225,26 +233,20 @@ def test_rank_deficient_design():
     # deficient input data (with n_features < rank) in the same way
     # as coordinate descent Lasso
     y = [5, 0, 5]
-    for X in (
-              [[5, 0],
-               [0, 5],
-               [10, 10]],
-              [[10, 10, 0],
-               [1e-32, 0, 0],
-               [0, 0, 1]]
-             ):
+    for X in ([[5, 0], [0, 5], [10, 10]], [[10, 10, 0], [1e-32, 0, 0], [0, 0, 1]]):
         # To be able to use the coefs to compute the objective function,
         # we need to turn off normalization
-        lars = linear_model.LassoLars(.1, normalize=False)
+        lars = linear_model.LassoLars(0.1)
         coef_lars_ = lars.fit(X, y).coef_
-        obj_lars = (1. / (2. * 3.)
-                    * linalg.norm(y - np.dot(X, coef_lars_)) ** 2
-                    + .1 * linalg.norm(coef_lars_, 1))
-        coord_descent = linear_model.Lasso(.1, tol=1e-6, normalize=False)
+        obj_lars = 1.0 / (2.0 * 3.0) * linalg.norm(
+            y - np.dot(X, coef_lars_)
+        ) ** 2 + 0.1 * linalg.norm(coef_lars_, 1)
+        coord_descent = linear_model.Lasso(0.1, tol=1e-6)
         coef_cd_ = coord_descent.fit(X, y).coef_
-        obj_cd = ((1. / (2. * 3.)) * linalg.norm(y - np.dot(X, coef_cd_)) ** 2
-                  + .1 * linalg.norm(coef_cd_, 1))
-        assert obj_lars < obj_cd * (1. + 1e-8)
+        obj_cd = (1.0 / (2.0 * 3.0)) * linalg.norm(
+            y - np.dot(X, coef_cd_)
+        ) ** 2 + 0.1 * linalg.norm(coef_cd_, 1)
+        assert obj_lars < obj_cd * (1.0 + 1e-8)
 
 
 def test_lasso_lars_vs_lasso_cd():
@@ -252,7 +254,7 @@ def test_lasso_lars_vs_lasso_cd():
     # same results.
     X = 3 * diabetes.data
 
-    alphas, _, lasso_path = linear_model.lars_path(X, y, method='lasso')
+    alphas, _, lasso_path = linear_model.lars_path(X, y, method="lasso")
     lasso_cd = linear_model.Lasso(fit_intercept=False, tol=1e-8)
     for c, a in zip(lasso_path.T, alphas):
         if a == 0:
@@ -264,17 +266,17 @@ def test_lasso_lars_vs_lasso_cd():
 
     # similar test, with the classifiers
     for alpha in np.linspace(1e-2, 1 - 1e-2, 20):
-        clf1 = linear_model.LassoLars(alpha=alpha, normalize=False).fit(X, y)
-        clf2 = linear_model.Lasso(alpha=alpha, tol=1e-8,
-                                  normalize=False).fit(X, y)
+        clf1 = linear_model.LassoLars(alpha=alpha).fit(X, y)
+        clf2 = linear_model.Lasso(alpha=alpha, tol=1e-8).fit(X, y)
         err = linalg.norm(clf1.coef_ - clf2.coef_)
         assert err < 1e-3
 
     # same test, with normalized data
     X = diabetes.data
-    alphas, _, lasso_path = linear_model.lars_path(X, y, method='lasso')
-    lasso_cd = linear_model.Lasso(fit_intercept=False, normalize=True,
-                                  tol=1e-8)
+    X = X - X.sum(axis=0)
+    X /= np.linalg.norm(X, axis=0)
+    alphas, _, lasso_path = linear_model.lars_path(X, y, method="lasso")
+    lasso_cd = linear_model.Lasso(fit_intercept=False, tol=1e-8)
     for c, a in zip(lasso_path.T, alphas):
         if a == 0:
             continue
@@ -290,9 +292,12 @@ def test_lasso_lars_vs_lasso_cd_early_stopping():
     # (test : before, in the middle, and in the last part of the path)
     alphas_min = [10, 0.9, 1e-4]
 
+    X = diabetes.data
+
     for alpha_min in alphas_min:
-        alphas, _, lasso_path = linear_model.lars_path(X, y, method='lasso',
-                                                       alpha_min=alpha_min)
+        alphas, _, lasso_path = linear_model.lars_path(
+            X, y, method="lasso", alpha_min=alpha_min
+        )
         lasso_cd = linear_model.Lasso(fit_intercept=False, tol=1e-8)
         lasso_cd.alpha = alphas[-1]
         lasso_cd.fit(X, y)
@@ -300,10 +305,14 @@ def test_lasso_lars_vs_lasso_cd_early_stopping():
         assert error < 0.01
 
     # same test, with normalization
+    X = diabetes.data - diabetes.data.sum(axis=0)
+    X /= np.linalg.norm(X, axis=0)
+
     for alpha_min in alphas_min:
-        alphas, _, lasso_path = linear_model.lars_path(X, y, method='lasso',
-                                                       alpha_min=alpha_min)
-        lasso_cd = linear_model.Lasso(normalize=True, tol=1e-8)
+        alphas, _, lasso_path = linear_model.lars_path(
+            X, y, method="lasso", alpha_min=alpha_min
+        )
+        lasso_cd = linear_model.Lasso(tol=1e-8)
         lasso_cd.alpha = alphas[-1]
         lasso_cd.fit(X, y)
         error = linalg.norm(lasso_path[:, -1] - lasso_cd.coef_)
@@ -343,12 +352,9 @@ def test_lasso_lars_vs_lasso_cd_ill_conditioned():
     sigma = 0.2
     y += sigma * rng.rand(*y.shape)
     y = y.squeeze()
-    lars_alphas, _, lars_coef = linear_model.lars_path(X, y, method='lasso')
+    lars_alphas, _, lars_coef = linear_model.lars_path(X, y, method="lasso")
 
-    _, lasso_coef2, _ = linear_model.lasso_path(X, y,
-                                                alphas=lars_alphas,
-                                                tol=1e-6,
-                                                fit_intercept=False)
+    _, lasso_coef2, _ = linear_model.lasso_path(X, y, alphas=lars_alphas, tol=1e-6)
 
     assert_array_almost_equal(lars_coef, lasso_coef2, decimal=1)
 
@@ -360,30 +366,27 @@ def test_lasso_lars_vs_lasso_cd_ill_conditioned2():
     # Note it used to be the case that Lars had to use the drop for good
     # strategy for this but this is no longer the case with the
     # equality_tolerance checks
-    X = [[1e20, 1e20, 0],
-         [-1e-32, 0, 0],
-         [1, 1, 1]]
+    X = [[1e20, 1e20, 0], [-1e-32, 0, 0], [1, 1, 1]]
     y = [10, 10, 1]
-    alpha = .0001
+    alpha = 0.0001
 
     def objective_function(coef):
-        return (1. / (2. * len(X)) * linalg.norm(y - np.dot(X, coef)) ** 2
-                + alpha * linalg.norm(coef, 1))
+        return 1.0 / (2.0 * len(X)) * linalg.norm(
+            y - np.dot(X, coef)
+        ) ** 2 + alpha * linalg.norm(coef, 1)
 
-    lars = linear_model.LassoLars(alpha=alpha, normalize=False)
-    warning_message = (
-        "Regressors in active set degenerate."
-    )
+    lars = linear_model.LassoLars(alpha=alpha)
+    warning_message = "Regressors in active set degenerate."
     with pytest.warns(ConvergenceWarning, match=warning_message):
         lars.fit(X, y)
     lars_coef_ = lars.coef_
     lars_obj = objective_function(lars_coef_)
 
-    coord_descent = linear_model.Lasso(alpha=alpha, tol=1e-4, normalize=False)
+    coord_descent = linear_model.Lasso(alpha=alpha, tol=1e-4)
     cd_coef_ = coord_descent.fit(X, y).coef_
     cd_obj = objective_function(cd_coef_)
 
-    assert lars_obj < cd_obj * (1. + 1e-8)
+    assert lars_obj < cd_obj * (1.0 + 1e-8)
 
 
 def test_lars_add_features():
@@ -391,9 +394,8 @@ def test_lars_add_features():
     # test for 6d2b4c
     # Hilbert matrix
     n = 5
-    H = 1. / (np.arange(1, n + 1) + np.arange(n)[:, np.newaxis])
-    clf = linear_model.Lars(fit_intercept=False).fit(
-        H, np.arange(n))
+    H = 1.0 / (np.arange(1, n + 1) + np.arange(n)[:, np.newaxis])
+    clf = linear_model.Lars(fit_intercept=False).fit(H, np.arange(n))
     assert np.all(np.isfinite(clf.coef_))
 
 
@@ -406,10 +408,9 @@ def test_lars_n_nonzero_coefs(verbose=False):
     assert len(lars.alphas_) == 7
 
 
-@ignore_warnings
 def test_multitarget():
     # Assure that estimators receiving multidimensional y do the right thing
-    Y = np.vstack([y, y ** 2]).T
+    Y = np.vstack([y, y**2]).T
     n_targets = Y.shape[1]
     estimators = [
         linear_model.LassoLars(),
@@ -422,8 +423,12 @@ def test_multitarget():
     for estimator in estimators:
         estimator.fit(X, Y)
         Y_pred = estimator.predict(X)
-        alphas, active, coef, path = (estimator.alphas_, estimator.active_,
-                                      estimator.coef_, estimator.coef_path_)
+        alphas, active, coef, path = (
+            estimator.alphas_,
+            estimator.active_,
+            estimator.coef_,
+            estimator.coef_path_,
+        )
         for k in range(n_targets):
             estimator.fit(X, Y[:, k])
             y_pred = estimator.predict(X)
@@ -447,26 +452,28 @@ def test_lars_cv():
         lars_cv.fit(X, y)
         np.testing.assert_array_less(old_alpha, lars_cv.alpha_)
         old_alpha = lars_cv.alpha_
-    assert not hasattr(lars_cv, 'n_nonzero_coefs')
+    assert not hasattr(lars_cv, "n_nonzero_coefs")
 
 
 def test_lars_cv_max_iter(recwarn):
-    warnings.simplefilter('always')
-    with np.errstate(divide='raise', invalid='raise'):
+    warnings.simplefilter("always")
+    with np.errstate(divide="raise", invalid="raise"):
         X = diabetes.data
         y = diabetes.target
         rng = np.random.RandomState(42)
         x = rng.randn(len(y))
         X = diabetes.data
         X = np.c_[X, x, x]  # add correlated features
+        X = StandardScaler().fit_transform(X)
         lars_cv = linear_model.LassoLarsCV(max_iter=5, cv=5)
         lars_cv.fit(X, y)
+
     # Check that there is no warning in general and no ConvergenceWarning
     # in particular.
     # Materialize the string representation of the warning to get a more
     # informative error message in case of AssertionError.
     recorded_warnings = [str(w) for w in recwarn]
-    assert recorded_warnings == []
+    assert len(recorded_warnings) == 0
 
 
 def test_lasso_lars_ic():
@@ -474,11 +481,12 @@ def test_lasso_lars_ic():
     # - some good features are selected.
     # - alpha_bic > alpha_aic
     # - n_nonzero_bic < n_nonzero_aic
-    lars_bic = linear_model.LassoLarsIC('bic')
-    lars_aic = linear_model.LassoLarsIC('aic')
+    lars_bic = linear_model.LassoLarsIC("bic")
+    lars_aic = linear_model.LassoLarsIC("aic")
     rng = np.random.RandomState(42)
     X = diabetes.data
     X = np.c_[X, rng.randn(X.shape[0], 5)]  # add 5 bad features
+    X = StandardScaler().fit_transform(X)
     lars_bic.fit(X, y)
     lars_aic.fit(X, y)
     nonzero_bic = np.where(lars_bic.coef_)[0]
@@ -487,10 +495,6 @@ def test_lasso_lars_ic():
     assert len(nonzero_bic) < len(nonzero_aic)
     assert np.max(nonzero_bic) < diabetes.data.shape[1]
 
-    # test error on unknown IC
-    lars_broken = linear_model.LassoLarsIC('')
-    assert_raises(ValueError, lars_broken.fit, X, y)
-
 
 def test_lars_path_readonly_data():
     # When using automated memory mapping on large input, the
@@ -519,35 +523,39 @@ def test_lars_path_positive_constraint():
             diabetes["data"], diabetes["target"], method="lar", positive=True
         )
 
-    method = 'lasso'
-    _, _, coefs = \
-        linear_model.lars_path(X, y, return_path=True, method=method,
-                               positive=False)
+    method = "lasso"
+    _, _, coefs = linear_model.lars_path(
+        X, y, return_path=True, method=method, positive=False
+    )
     assert coefs.min() < 0
 
-    _, _, coefs = \
-        linear_model.lars_path(X, y, return_path=True, method=method,
-                               positive=True)
+    _, _, coefs = linear_model.lars_path(
+        X, y, return_path=True, method=method, positive=True
+    )
     assert coefs.min() >= 0
 
 
 # now we gonna test the positive option for all estimator classes
 
-default_parameter = {'fit_intercept': False}
+default_parameter = {"fit_intercept": False}
 
-estimator_parameter_map = {'LassoLars': {'alpha': 0.1},
-                           'LassoLarsCV': {},
-                           'LassoLarsIC': {}}
+estimator_parameter_map = {
+    "LassoLars": {"alpha": 0.1},
+    "LassoLarsCV": {},
+    "LassoLarsIC": {},
+}
 
 
 def test_estimatorclasses_positive_constraint():
     # testing the transmissibility for the positive option of all estimator
     # classes in this same function here
-    default_parameter = {'fit_intercept': False}
+    default_parameter = {"fit_intercept": False}
 
-    estimator_parameter_map = {'LassoLars': {'alpha': 0.1},
-                               'LassoLarsCV': {},
-                               'LassoLarsIC': {}}
+    estimator_parameter_map = {
+        "LassoLars": {"alpha": 0.1},
+        "LassoLarsCV": {},
+        "LassoLarsIC": {},
+    }
     for estname in estimator_parameter_map:
         params = default_parameter.copy()
         params.update(estimator_parameter_map[estname])
@@ -570,8 +578,7 @@ def test_lasso_lars_vs_lasso_cd_positive():
     # not normalized data
     X = 3 * diabetes.data
 
-    alphas, _, lasso_path = linear_model.lars_path(X, y, method='lasso',
-                                                   positive=True)
+    alphas, _, lasso_path = linear_model.lars_path(X, y, method="lasso", positive=True)
     lasso_cd = linear_model.Lasso(fit_intercept=False, tol=1e-8, positive=True)
     for c, a in zip(lasso_path.T, alphas):
         if a == 0:
@@ -591,19 +598,20 @@ def test_lasso_lars_vs_lasso_cd_positive():
     # https://gist.github.com/michigraber/7e7d7c75eca694c7a6ff
 
     for alpha in np.linspace(6e-1, 1 - 1e-2, 20):
-        clf1 = linear_model.LassoLars(fit_intercept=False, alpha=alpha,
-                                      normalize=False, positive=True).fit(X, y)
-        clf2 = linear_model.Lasso(fit_intercept=False, alpha=alpha, tol=1e-8,
-                                  normalize=False, positive=True).fit(X, y)
+        clf1 = linear_model.LassoLars(
+            fit_intercept=False, alpha=alpha, positive=True
+        ).fit(X, y)
+        clf2 = linear_model.Lasso(
+            fit_intercept=False, alpha=alpha, tol=1e-8, positive=True
+        ).fit(X, y)
         err = linalg.norm(clf1.coef_ - clf2.coef_)
         assert err < 1e-3
 
     # normalized data
-    X = diabetes.data
-    alphas, _, lasso_path = linear_model.lars_path(X, y, method='lasso',
-                                                   positive=True)
-    lasso_cd = linear_model.Lasso(fit_intercept=False, normalize=True,
-                                  tol=1e-8, positive=True)
+    X = diabetes.data - diabetes.data.sum(axis=0)
+    X /= np.linalg.norm(X, axis=0)
+    alphas, _, lasso_path = linear_model.lars_path(X, y, method="lasso", positive=True)
+    lasso_cd = linear_model.Lasso(fit_intercept=False, tol=1e-8, positive=True)
     for c, a in zip(lasso_path.T[:-1], alphas[:-1]):  # don't include alpha=0
         lasso_cd.alpha = a
         lasso_cd.fit(X, y)
@@ -613,28 +621,22 @@ def test_lasso_lars_vs_lasso_cd_positive():
 
 def test_lasso_lars_vs_R_implementation():
     # Test that sklearn LassoLars implementation agrees with the LassoLars
-    # implementation available in R (lars library) under the following
-    # scenarios:
-    # 1) fit_intercept=False and normalize=False
-    # 2) fit_intercept=True and normalize=True
+    # implementation available in R (lars library) when fit_intercept=False.
 
     # Let's generate the data used in the bug report 7778
-    y = np.array([-6.45006793, -3.51251449, -8.52445396, 6.12277822,
-                  -19.42109366])
-    x = np.array([[0.47299829, 0, 0, 0, 0],
-                  [0.08239882, 0.85784863, 0, 0, 0],
-                  [0.30114139, -0.07501577, 0.80895216, 0, 0],
-                  [-0.01460346, -0.1015233, 0.0407278, 0.80338378, 0],
-                  [-0.69363927, 0.06754067, 0.18064514, -0.0803561,
-                   0.40427291]])
+    y = np.array([-6.45006793, -3.51251449, -8.52445396, 6.12277822, -19.42109366])
+    x = np.array(
+        [
+            [0.47299829, 0, 0, 0, 0],
+            [0.08239882, 0.85784863, 0, 0, 0],
+            [0.30114139, -0.07501577, 0.80895216, 0, 0],
+            [-0.01460346, -0.1015233, 0.0407278, 0.80338378, 0],
+            [-0.69363927, 0.06754067, 0.18064514, -0.0803561, 0.40427291],
+        ]
+    )
 
     X = x.T
 
-    ###########################################################################
-    # Scenario 1: Let's compare R vs sklearn when fit_intercept=False and
-    # normalize=False
-    ###########################################################################
-    #
     # The R result was obtained using the following code:
     #
     # library(lars)
@@ -643,71 +645,68 @@ def test_lasso_lars_vs_R_implementation():
     # r = t(model_lasso_lars$beta)
     #
 
-    r = np.array([[0, 0, 0, 0, 0, -79.810362809499026, -83.528788732782829,
-                   -83.777653739190711, -83.784156932888934,
-                   -84.033390591756657],
-                  [0, 0, 0, 0, -0.476624256777266, 0, 0, 0, 0,
-                   0.025219751009936],
-                  [0, -3.577397088285891, -4.702795355871871,
-                   -7.016748621359461, -7.614898471899412, -0.336938391359179,
-                   0, 0, 0.001213370600853, 0.048162321585148],
-                  [0, 0, 0, 2.231558436628169, 2.723267514525966,
-                   2.811549786389614, 2.813766976061531, 2.817462468949557,
-                   2.817368178703816, 2.816221090636795],
-                  [0, 0, -1.218422599914637, -3.457726183014808,
-                   -4.021304522060710, -45.827461592423745,
-                   -47.776608869312305,
-                   -47.911561610746404, -47.914845922736234,
-                   -48.039562334265717]])
-
-    model_lasso_lars = linear_model.LassoLars(alpha=0, fit_intercept=False,
-                                              normalize=False)
+    r = np.array(
+        [
+            [
+                0,
+                0,
+                0,
+                0,
+                0,
+                -79.810362809499026,
+                -83.528788732782829,
+                -83.777653739190711,
+                -83.784156932888934,
+                -84.033390591756657,
+            ],
+            [0, 0, 0, 0, -0.476624256777266, 0, 0, 0, 0, 0.025219751009936],
+            [
+                0,
+                -3.577397088285891,
+                -4.702795355871871,
+                -7.016748621359461,
+                -7.614898471899412,
+                -0.336938391359179,
+                0,
+                0,
+                0.001213370600853,
+                0.048162321585148,
+            ],
+            [
+                0,
+                0,
+                0,
+                2.231558436628169,
+                2.723267514525966,
+                2.811549786389614,
+                2.813766976061531,
+                2.817462468949557,
+                2.817368178703816,
+                2.816221090636795,
+            ],
+            [
+                0,
+                0,
+                -1.218422599914637,
+                -3.457726183014808,
+                -4.021304522060710,
+                -45.827461592423745,
+                -47.776608869312305,
+                -47.911561610746404,
+                -47.914845922736234,
+                -48.039562334265717,
+            ],
+        ]
+    )
+
+    model_lasso_lars = linear_model.LassoLars(alpha=0, fit_intercept=False)
     model_lasso_lars.fit(X, y)
     skl_betas = model_lasso_lars.coef_path_
 
     assert_array_almost_equal(r, skl_betas, decimal=12)
-    ###########################################################################
 
-    ###########################################################################
-    # Scenario 2: Let's compare R vs sklearn when fit_intercept=True and
-    # normalize=True
-    #
-    # Note: When normalize is equal to True, R returns the coefficients in
-    # their original units, that is, they are rescaled back, whereas sklearn
-    # does not do that, therefore, we need to do this step before comparing
-    # their results.
-    ###########################################################################
-    #
-    # The R result was obtained using the following code:
-    #
-    # library(lars)
-    # model_lasso_lars2 = lars(X, t(y), type="lasso", intercept=TRUE,
-    #                           trace=TRUE, normalize=TRUE)
-    # r2 = t(model_lasso_lars2$beta)
-
-    r2 = np.array([[0, 0, 0, 0, 0],
-                   [0, 0, 0, 8.371887668009453, 19.463768371044026],
-                   [0, 0, 0, 0, 9.901611055290553],
-                   [0, 7.495923132833733, 9.245133544334507,
-                    17.389369207545062, 26.971656815643499],
-                   [0, 0, -1.569380717440311, -5.924804108067312,
-                    -7.996385265061972]])
-
-    model_lasso_lars2 = linear_model.LassoLars(alpha=0, normalize=True)
-    model_lasso_lars2.fit(X, y)
-    skl_betas2 = model_lasso_lars2.coef_path_
-
-    # Let's rescale back the coefficients returned by sklearn before comparing
-    # against the R result (read the note above)
-    temp = X - np.mean(X, axis=0)
-    normx = np.sqrt(np.sum(temp ** 2, axis=0))
-    skl_betas2 /= normx[:, np.newaxis]
 
-    assert_array_almost_equal(r2, skl_betas2, decimal=12)
-    ###########################################################################
-
-
-@pytest.mark.parametrize('copy_X', [True, False])
+@pytest.mark.parametrize("copy_X", [True, False])
 def test_lasso_lars_copyX_behaviour(copy_X):
     """
     Test that user input regarding copy_X is not being overridden (it was until
@@ -723,7 +722,7 @@ def test_lasso_lars_copyX_behaviour(copy_X):
     assert copy_X == np.array_equal(X, X_copy)
 
 
-@pytest.mark.parametrize('copy_X', [True, False])
+@pytest.mark.parametrize("copy_X", [True, False])
 def test_lasso_lars_fit_copyX_behaviour(copy_X):
     """
     Test that user input to .fit for copy_X overrides default __init__ value
@@ -738,13 +737,12 @@ def test_lasso_lars_fit_copyX_behaviour(copy_X):
     assert copy_X == np.array_equal(X, X_copy)
 
 
-@pytest.mark.parametrize('est', (LassoLars(alpha=1e-3), Lars()))
+@pytest.mark.parametrize("est", (LassoLars(alpha=1e-3), Lars()))
 def test_lars_with_jitter(est):
     # Test that a small amount of jitter helps stability,
     # using example provided in issue #2746
 
-    X = np.array([[0.0, 0.0, 0.0, -1.0, 0.0],
-                  [0.0, -1.0, 0.0, 0.0, 0.0]])
+    X = np.array([[0.0, 0.0, 0.0, -1.0, 0.0], [0.0, -1.0, 0.0, 0.0, 0.0]])
     y = [-2.5, -2.5]
     expected_coef = [0, 2.5, 0, 2.5, 0]
 
@@ -756,14 +754,13 @@ def test_lars_with_jitter(est):
     est.fit(X, y)
     est_jitter.fit(X, y)
 
-    assert np.mean((est.coef_ - est_jitter.coef_)**2) > .1
+    assert np.mean((est.coef_ - est_jitter.coef_) ** 2) > 0.1
     np.testing.assert_allclose(est_jitter.coef_, expected_coef, rtol=1e-3)
 
 
 def test_X_none_gram_not_none():
-    with pytest.raises(ValueError,
-                       match="X cannot be None if Gram is not None"):
-        lars_path(X=None, y=[1], Gram='not None')
+    with pytest.raises(ValueError, match="X cannot be None if Gram is not None"):
+        lars_path(X=None, y=np.array([1]), Gram=True)
 
 
 def test_copy_X_with_auto_gram():
@@ -774,6 +771,99 @@ def test_copy_X_with_auto_gram():
     y = rng.rand(6)
 
     X_before = X.copy()
-    linear_model.lars_path(X, y, Gram='auto', copy_X=True, method='lasso')
+    linear_model.lars_path(X, y, Gram="auto", copy_X=True, method="lasso")
     # X did not change
     assert_allclose(X, X_before)
+
+
+@pytest.mark.parametrize(
+    "LARS, has_coef_path, args",
+    (
+        (Lars, True, {}),
+        (LassoLars, True, {}),
+        (LassoLarsIC, False, {}),
+        (LarsCV, True, {}),
+        # max_iter=5 is for avoiding ConvergenceWarning
+        (LassoLarsCV, True, {"max_iter": 5}),
+    ),
+)
+@pytest.mark.parametrize("dtype", (np.float32, np.float64))
+def test_lars_dtype_match(LARS, has_coef_path, args, dtype):
+    # The test ensures that the fit method preserves input dtype
+    rng = np.random.RandomState(0)
+    X = rng.rand(20, 6).astype(dtype)
+    y = rng.rand(20).astype(dtype)
+
+    model = LARS(**args)
+    model.fit(X, y)
+    assert model.coef_.dtype == dtype
+    if has_coef_path:
+        assert model.coef_path_.dtype == dtype
+    assert model.intercept_.dtype == dtype
+
+
+@pytest.mark.parametrize(
+    "LARS, has_coef_path, args",
+    (
+        (Lars, True, {}),
+        (LassoLars, True, {}),
+        (LassoLarsIC, False, {}),
+        (LarsCV, True, {}),
+        # max_iter=5 is for avoiding ConvergenceWarning
+        (LassoLarsCV, True, {"max_iter": 5}),
+    ),
+)
+def test_lars_numeric_consistency(LARS, has_coef_path, args):
+    # The test ensures numerical consistency between trained coefficients
+    # of float32 and float64.
+    rtol = 1e-5
+    atol = 1e-5
+
+    rng = np.random.RandomState(0)
+    X_64 = rng.rand(10, 6)
+    y_64 = rng.rand(10)
+
+    model_64 = LARS(**args).fit(X_64, y_64)
+    model_32 = LARS(**args).fit(X_64.astype(np.float32), y_64.astype(np.float32))
+
+    assert_allclose(model_64.coef_, model_32.coef_, rtol=rtol, atol=atol)
+    if has_coef_path:
+        assert_allclose(model_64.coef_path_, model_32.coef_path_, rtol=rtol, atol=atol)
+    assert_allclose(model_64.intercept_, model_32.intercept_, rtol=rtol, atol=atol)
+
+
+@pytest.mark.parametrize("criterion", ["aic", "bic"])
+def test_lassolarsic_alpha_selection(criterion):
+    """Check that we properly compute the AIC and BIC score.
+
+    In this test, we reproduce the example of the Fig. 2 of Zou et al.
+    (reference [1] in LassoLarsIC) In this example, only 7 features should be
+    selected.
+    """
+    model = make_pipeline(StandardScaler(), LassoLarsIC(criterion=criterion))
+    model.fit(X, y)
+
+    best_alpha_selected = np.argmin(model[-1].criterion_)
+    assert best_alpha_selected == 7
+
+
+@pytest.mark.parametrize("fit_intercept", [True, False])
+def test_lassolarsic_noise_variance(fit_intercept):
+    """Check the behaviour when `n_samples` < `n_features` and that one needs
+    to provide the noise variance."""
+    rng = np.random.RandomState(0)
+    X, y = datasets.make_regression(
+        n_samples=10, n_features=11 - fit_intercept, random_state=rng
+    )
+
+    model = make_pipeline(StandardScaler(), LassoLarsIC(fit_intercept=fit_intercept))
+
+    err_msg = (
+        "You are using LassoLarsIC in the case where the number of samples is smaller"
+        " than the number of features"
+    )
+    with pytest.raises(ValueError, match=err_msg):
+        model.fit(X, y)
+
+    model.set_params(lassolarsic__noise_variance=1.0)
+    model.fit(X, y).predict(X)
diff --git a/sklearn/linear_model/tests/test_linear_loss.py b/sklearn/linear_model/tests/test_linear_loss.py
new file mode 100644
index 0000000000000..a273656b3dbb8
--- /dev/null
+++ b/sklearn/linear_model/tests/test_linear_loss.py
@@ -0,0 +1,510 @@
+"""
+Tests for LinearModelLoss
+
+Note that correctness of losses (which compose LinearModelLoss) is already well
+covered in the _loss module.
+"""
+
+import numpy as np
+import pytest
+from numpy.testing import assert_allclose
+from scipy import linalg, optimize
+
+from sklearn._loss.loss import (
+    HalfBinomialLoss,
+    HalfMultinomialLoss,
+    HalfPoissonLoss,
+)
+from sklearn.datasets import make_low_rank_matrix
+from sklearn.linear_model._linear_loss import LinearModelLoss
+from sklearn.utils.extmath import squared_norm
+from sklearn.utils.fixes import CSR_CONTAINERS
+
+# We do not need to test all losses, just what LinearModelLoss does on top of the
+# base losses.
+LOSSES = [HalfBinomialLoss, HalfMultinomialLoss, HalfPoissonLoss]
+
+
+def random_X_y_coef(
+    linear_model_loss, n_samples, n_features, coef_bound=(-2, 2), seed=42
+):
+    """Random generate y, X and coef in valid range."""
+    rng = np.random.RandomState(seed)
+    n_dof = n_features + linear_model_loss.fit_intercept
+    X = make_low_rank_matrix(
+        n_samples=n_samples,
+        n_features=n_features,
+        random_state=rng,
+    )
+    coef = linear_model_loss.init_zero_coef(X)
+
+    if linear_model_loss.base_loss.is_multiclass:
+        n_classes = linear_model_loss.base_loss.n_classes
+        coef.flat[:] = rng.uniform(
+            low=coef_bound[0],
+            high=coef_bound[1],
+            size=n_classes * n_dof,
+        )
+        if linear_model_loss.fit_intercept:
+            raw_prediction = X @ coef[:, :-1].T + coef[:, -1]
+        else:
+            raw_prediction = X @ coef.T
+        proba = linear_model_loss.base_loss.link.inverse(raw_prediction)
+
+        # y = rng.choice(np.arange(n_classes), p=proba) does not work.
+        # See https://stackoverflow.com/a/34190035/16761084
+        def choice_vectorized(items, p):
+            s = p.cumsum(axis=1)
+            r = rng.rand(p.shape[0])[:, None]
+            k = (s < r).sum(axis=1)
+            return items[k]
+
+        y = choice_vectorized(np.arange(n_classes), p=proba).astype(np.float64)
+    else:
+        coef.flat[:] = rng.uniform(
+            low=coef_bound[0],
+            high=coef_bound[1],
+            size=n_dof,
+        )
+        if linear_model_loss.fit_intercept:
+            raw_prediction = X @ coef[:-1] + coef[-1]
+        else:
+            raw_prediction = X @ coef
+        y = linear_model_loss.base_loss.link.inverse(
+            raw_prediction + rng.uniform(low=-1, high=1, size=n_samples)
+        )
+
+    return X, y, coef
+
+
+@pytest.mark.parametrize("base_loss", LOSSES)
+@pytest.mark.parametrize("fit_intercept", [False, True])
+@pytest.mark.parametrize("n_features", [0, 1, 10])
+@pytest.mark.parametrize("dtype", [None, np.float32, np.float64, np.int64])
+def test_init_zero_coef(
+    base_loss, fit_intercept, n_features, dtype, global_random_seed
+):
+    """Test that init_zero_coef initializes coef correctly."""
+    loss = LinearModelLoss(base_loss=base_loss(), fit_intercept=fit_intercept)
+    rng = np.random.RandomState(global_random_seed)
+    X = rng.normal(size=(5, n_features))
+    coef = loss.init_zero_coef(X, dtype=dtype)
+    if loss.base_loss.is_multiclass:
+        n_classes = loss.base_loss.n_classes
+        assert coef.shape == (n_classes, n_features + fit_intercept)
+        assert coef.flags["F_CONTIGUOUS"]
+    else:
+        assert coef.shape == (n_features + fit_intercept,)
+
+    if dtype is None:
+        assert coef.dtype == X.dtype
+    else:
+        assert coef.dtype == dtype
+
+    assert np.count_nonzero(coef) == 0
+
+
+@pytest.mark.parametrize("base_loss", LOSSES)
+@pytest.mark.parametrize("fit_intercept", [False, True])
+@pytest.mark.parametrize("sample_weight", [None, "range"])
+@pytest.mark.parametrize("l2_reg_strength", [0, 1])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_loss_grad_hess_are_the_same(
+    base_loss,
+    fit_intercept,
+    sample_weight,
+    l2_reg_strength,
+    csr_container,
+    global_random_seed,
+):
+    """Test that loss and gradient are the same across different functions."""
+    loss = LinearModelLoss(base_loss=base_loss(), fit_intercept=fit_intercept)
+    X, y, coef = random_X_y_coef(
+        linear_model_loss=loss, n_samples=10, n_features=5, seed=global_random_seed
+    )
+    X_old, y_old, coef_old = X.copy(), y.copy(), coef.copy()
+
+    if sample_weight == "range":
+        sample_weight = np.linspace(1, y.shape[0], num=y.shape[0])
+
+    l1 = loss.loss(
+        coef, X, y, sample_weight=sample_weight, l2_reg_strength=l2_reg_strength
+    )
+    g1 = loss.gradient(
+        coef, X, y, sample_weight=sample_weight, l2_reg_strength=l2_reg_strength
+    )
+    l2, g2 = loss.loss_gradient(
+        coef, X, y, sample_weight=sample_weight, l2_reg_strength=l2_reg_strength
+    )
+    g3, h3 = loss.gradient_hessian_product(
+        coef, X, y, sample_weight=sample_weight, l2_reg_strength=l2_reg_strength
+    )
+    g4, h4, _ = loss.gradient_hessian(
+        coef, X, y, sample_weight=sample_weight, l2_reg_strength=l2_reg_strength
+    )
+    assert_allclose(l1, l2)
+    assert_allclose(g1, g2)
+    assert_allclose(g1, g3)
+    assert_allclose(g1, g4)
+    # The ravelling only takes effect for multiclass.
+    assert_allclose(h4 @ g4.ravel(order="F"), h3(g3).ravel(order="F"))
+    # Test that gradient_out and hessian_out are considered properly.
+    g_out = np.empty_like(coef)
+    h_out = np.empty_like(coef, shape=(coef.size, coef.size))
+    g5, h5, _ = loss.gradient_hessian(
+        coef,
+        X,
+        y,
+        sample_weight=sample_weight,
+        l2_reg_strength=l2_reg_strength,
+        gradient_out=g_out,
+        hessian_out=h_out,
+    )
+    assert np.shares_memory(g5, g_out)
+    assert np.shares_memory(h5, h_out)
+    assert_allclose(g5, g_out)
+    assert_allclose(h5, h_out)
+    assert_allclose(g1, g5)
+    assert_allclose(h5, h4)
+
+    # same for sparse X
+    Xs = csr_container(X)
+    l1_sp = loss.loss(
+        coef, Xs, y, sample_weight=sample_weight, l2_reg_strength=l2_reg_strength
+    )
+    g1_sp = loss.gradient(
+        coef, Xs, y, sample_weight=sample_weight, l2_reg_strength=l2_reg_strength
+    )
+    l2_sp, g2_sp = loss.loss_gradient(
+        coef, Xs, y, sample_weight=sample_weight, l2_reg_strength=l2_reg_strength
+    )
+    g3_sp, h3_sp = loss.gradient_hessian_product(
+        coef, Xs, y, sample_weight=sample_weight, l2_reg_strength=l2_reg_strength
+    )
+    g4_sp, h4_sp, _ = loss.gradient_hessian(
+        coef, Xs, y, sample_weight=sample_weight, l2_reg_strength=l2_reg_strength
+    )
+    assert_allclose(l1, l1_sp)
+    assert_allclose(l1, l2_sp)
+    assert_allclose(g1, g1_sp)
+    assert_allclose(g1, g2_sp)
+    assert_allclose(g1, g3_sp)
+    assert_allclose(h3(g1), h3_sp(g1_sp))
+    assert_allclose(g1, g4_sp)
+    assert_allclose(h4, h4_sp)
+
+    # X, y and coef should not have changed
+    assert_allclose(X, X_old)
+    assert_allclose(Xs.toarray(), X_old)
+    assert_allclose(y, y_old)
+    assert_allclose(coef, coef_old)
+
+
+@pytest.mark.parametrize("base_loss", LOSSES)
+@pytest.mark.parametrize("sample_weight", [None, "range"])
+@pytest.mark.parametrize("l2_reg_strength", [0, 1])
+@pytest.mark.parametrize("X_container", CSR_CONTAINERS + [None])
+def test_loss_gradients_hessp_intercept(
+    base_loss, sample_weight, l2_reg_strength, X_container, global_random_seed
+):
+    """Test that loss and gradient handle intercept correctly."""
+    loss = LinearModelLoss(base_loss=base_loss(), fit_intercept=False)
+    loss_inter = LinearModelLoss(base_loss=base_loss(), fit_intercept=True)
+    n_samples, n_features = 10, 5
+    X, y, coef = random_X_y_coef(
+        linear_model_loss=loss,
+        n_samples=n_samples,
+        n_features=n_features,
+        seed=global_random_seed,
+    )
+
+    X[:, -1] = 1  # make last column of 1 to mimic intercept term
+    X_inter = X[
+        :, :-1
+    ]  # exclude intercept column as it is added automatically by loss_inter
+
+    if X_container is not None:
+        X = X_container(X)
+
+    if sample_weight == "range":
+        sample_weight = np.linspace(1, y.shape[0], num=y.shape[0])
+
+    l, g = loss.loss_gradient(
+        coef, X, y, sample_weight=sample_weight, l2_reg_strength=l2_reg_strength
+    )
+    _, hessp = loss.gradient_hessian_product(
+        coef, X, y, sample_weight=sample_weight, l2_reg_strength=l2_reg_strength
+    )
+    l_inter, g_inter = loss_inter.loss_gradient(
+        coef, X_inter, y, sample_weight=sample_weight, l2_reg_strength=l2_reg_strength
+    )
+    _, hessp_inter = loss_inter.gradient_hessian_product(
+        coef, X_inter, y, sample_weight=sample_weight, l2_reg_strength=l2_reg_strength
+    )
+
+    # Note, that intercept gets no L2 penalty.
+    assert l == pytest.approx(
+        l_inter + 0.5 * l2_reg_strength * squared_norm(coef.T[-1])
+    )
+
+    g_inter_corrected = g_inter
+    g_inter_corrected.T[-1] += l2_reg_strength * coef.T[-1]
+    assert_allclose(g, g_inter_corrected)
+
+    s = np.random.RandomState(global_random_seed).randn(*coef.shape)
+    h = hessp(s)
+    h_inter = hessp_inter(s)
+    h_inter_corrected = h_inter
+    h_inter_corrected.T[-1] += l2_reg_strength * s.T[-1]
+    assert_allclose(h, h_inter_corrected)
+
+
+@pytest.mark.parametrize("base_loss", LOSSES)
+@pytest.mark.parametrize("fit_intercept", [False, True])
+@pytest.mark.parametrize("sample_weight", [None, "range"])
+@pytest.mark.parametrize("l2_reg_strength", [0, 1])
+def test_gradients_hessians_numerically(
+    base_loss, fit_intercept, sample_weight, l2_reg_strength, global_random_seed
+):
+    """Test gradients and hessians with numerical derivatives.
+
+    Gradient should equal the numerical derivatives of the loss function.
+    Hessians should equal the numerical derivatives of gradients.
+    """
+    loss = LinearModelLoss(base_loss=base_loss(), fit_intercept=fit_intercept)
+    n_samples, n_features = 10, 5
+    X, y, coef = random_X_y_coef(
+        linear_model_loss=loss,
+        n_samples=n_samples,
+        n_features=n_features,
+        seed=global_random_seed,
+    )
+    coef = coef.ravel(order="F")  # this is important only for multinomial loss
+
+    if sample_weight == "range":
+        sample_weight = np.linspace(1, y.shape[0], num=y.shape[0])
+
+    # 1. Check gradients numerically
+    eps = 1e-6
+    g, hessp = loss.gradient_hessian_product(
+        coef, X, y, sample_weight=sample_weight, l2_reg_strength=l2_reg_strength
+    )
+    # Use a trick to get central finite difference of accuracy 4 (five-point stencil)
+    # https://en.wikipedia.org/wiki/Numerical_differentiation
+    # https://en.wikipedia.org/wiki/Finite_difference_coefficient
+    # approx_g1 = (f(x + eps) - f(x - eps)) / (2*eps)
+    approx_g1 = optimize.approx_fprime(
+        coef,
+        lambda coef: loss.loss(
+            coef - eps,
+            X,
+            y,
+            sample_weight=sample_weight,
+            l2_reg_strength=l2_reg_strength,
+        ),
+        2 * eps,
+    )
+    # approx_g2 = (f(x + 2*eps) - f(x - 2*eps)) / (4*eps)
+    approx_g2 = optimize.approx_fprime(
+        coef,
+        lambda coef: loss.loss(
+            coef - 2 * eps,
+            X,
+            y,
+            sample_weight=sample_weight,
+            l2_reg_strength=l2_reg_strength,
+        ),
+        4 * eps,
+    )
+    # Five-point stencil approximation
+    # See: https://en.wikipedia.org/wiki/Five-point_stencil#1D_first_derivative
+    approx_g = (4 * approx_g1 - approx_g2) / 3
+    assert_allclose(g, approx_g, rtol=1e-2, atol=1e-8)
+
+    # 2. Check hessp numerically along the second direction of the gradient
+    vector = np.zeros_like(g)
+    vector[1] = 1
+    hess_col = hessp(vector)
+    # Computation of the Hessian is particularly fragile to numerical errors when doing
+    # simple finite differences. Here we compute the grad along a path in the direction
+    # of the vector and then use a least-square regression to estimate the slope
+    eps = 1e-3
+    d_x = np.linspace(-eps, eps, 30)
+    d_grad = np.array(
+        [
+            loss.gradient(
+                coef + t * vector,
+                X,
+                y,
+                sample_weight=sample_weight,
+                l2_reg_strength=l2_reg_strength,
+            )
+            for t in d_x
+        ]
+    )
+    d_grad -= d_grad.mean(axis=0)
+    approx_hess_col = linalg.lstsq(d_x[:, np.newaxis], d_grad)[0].ravel()
+    assert_allclose(approx_hess_col, hess_col, rtol=1e-3)
+
+
+@pytest.mark.parametrize("fit_intercept", [False, True])
+def test_multinomial_coef_shape(fit_intercept, global_random_seed):
+    """Test that multinomial LinearModelLoss respects shape of coef."""
+    loss = LinearModelLoss(base_loss=HalfMultinomialLoss(), fit_intercept=fit_intercept)
+    n_samples, n_features = 10, 5
+    X, y, coef = random_X_y_coef(
+        linear_model_loss=loss,
+        n_samples=n_samples,
+        n_features=n_features,
+        seed=global_random_seed,
+    )
+    s = np.random.RandomState(global_random_seed).randn(*coef.shape)
+
+    l, g = loss.loss_gradient(coef, X, y)
+    g1 = loss.gradient(coef, X, y)
+    g2, hessp = loss.gradient_hessian_product(coef, X, y)
+    h = hessp(s)
+    assert g.shape == coef.shape
+    assert h.shape == coef.shape
+    assert_allclose(g, g1)
+    assert_allclose(g, g2)
+    g3, hess, _ = loss.gradient_hessian(coef, X, y)
+    assert g3.shape == coef.shape
+    # But full hessian is always 2d.
+    assert hess.shape == (coef.size, coef.size)
+
+    coef_r = coef.ravel(order="F")
+    s_r = s.ravel(order="F")
+    l_r, g_r = loss.loss_gradient(coef_r, X, y)
+    g1_r = loss.gradient(coef_r, X, y)
+    g2_r, hessp_r = loss.gradient_hessian_product(coef_r, X, y)
+    h_r = hessp_r(s_r)
+    assert g_r.shape == coef_r.shape
+    assert h_r.shape == coef_r.shape
+    assert_allclose(g_r, g1_r)
+    assert_allclose(g_r, g2_r)
+
+    assert_allclose(g, g_r.reshape(loss.base_loss.n_classes, -1, order="F"))
+    assert_allclose(h, h_r.reshape(loss.base_loss.n_classes, -1, order="F"))
+
+
+@pytest.mark.parametrize("sample_weight", [None, "range"])
+def test_multinomial_hessian_3_classes(sample_weight, global_random_seed):
+    """Test multinomial hessian for 3 classes and 2 points.
+
+    For n_classes = 3 and n_samples = 2, we have
+      p0 = [p0_0, p0_1]
+      p1 = [p1_0, p1_1]
+      p2 = [p2_0, p2_1]
+    and with 2 x 2 diagonal subblocks
+      H = [p0 * (1-p0),    -p0 * p1,    -p0 * p2]
+          [   -p0 * p1, p1 * (1-p1),    -p1 * p2]
+          [   -p0 * p2,    -p1 * p2, p2 * (1-p2)]
+      hess = X' H X
+    """
+    n_samples, n_features, n_classes = 2, 5, 3
+    loss = LinearModelLoss(
+        base_loss=HalfMultinomialLoss(n_classes=n_classes), fit_intercept=False
+    )
+    X, y, coef = random_X_y_coef(
+        linear_model_loss=loss,
+        n_samples=n_samples,
+        n_features=n_features,
+        seed=global_random_seed,
+    )
+    coef = coef.ravel(order="F")  # this is important only for multinomial loss
+
+    if sample_weight == "range":
+        sample_weight = np.linspace(1, y.shape[0], num=y.shape[0])
+
+    grad, hess, _ = loss.gradient_hessian(
+        coef,
+        X,
+        y,
+        sample_weight=sample_weight,
+        l2_reg_strength=0,
+    )
+    # Hessian must be a symmetrix matrix.
+    assert_allclose(hess, hess.T)
+
+    weights, intercept, raw_prediction = loss.weight_intercept_raw(coef, X)
+    grad_pointwise, proba = loss.base_loss.gradient_proba(
+        y_true=y,
+        raw_prediction=raw_prediction,
+        sample_weight=sample_weight,
+    )
+    p0d, p1d, p2d, oned = (
+        np.diag(proba[:, 0]),
+        np.diag(proba[:, 1]),
+        np.diag(proba[:, 2]),
+        np.diag(np.ones(2)),
+    )
+    h = np.block(
+        [
+            [p0d * (oned - p0d), -p0d * p1d, -p0d * p2d],
+            [-p0d * p1d, p1d * (oned - p1d), -p1d * p2d],
+            [-p0d * p2d, -p1d * p2d, p2d * (oned - p2d)],
+        ]
+    )
+    h = h.reshape((n_classes, n_samples, n_classes, n_samples))
+    if sample_weight is None:
+        h /= n_samples
+    else:
+        h *= sample_weight / np.sum(sample_weight)
+    # hess_expected.shape = (n_features, n_classes, n_classes, n_features)
+    hess_expected = np.einsum("ij, mini, ik->jmnk", X, h, X)
+    hess_expected = np.moveaxis(hess_expected, 2, 3)
+    hess_expected = hess_expected.reshape(
+        n_classes * n_features, n_classes * n_features, order="C"
+    )
+    assert_allclose(hess_expected, hess_expected.T)
+    assert_allclose(hess, hess_expected)
+
+
+def test_linear_loss_gradient_hessian_raises_wrong_out_parameters():
+    """Test that wrong gradient_out and hessian_out raises errors."""
+    n_samples, n_features, n_classes = 5, 2, 3
+    loss = LinearModelLoss(base_loss=HalfBinomialLoss(), fit_intercept=False)
+    X = np.ones((n_samples, n_features))
+    y = np.ones(n_samples)
+    coef = loss.init_zero_coef(X)
+    gradient_out = np.zeros(1)
+    with pytest.raises(
+        ValueError, match="gradient_out is required to have shape coef.shape"
+    ):
+        loss.gradient_hessian(
+            coef=coef,
+            X=X,
+            y=y,
+            gradient_out=gradient_out,
+            hessian_out=None,
+        )
+    hessian_out = np.zeros(1)
+    with pytest.raises(ValueError, match="hessian_out is required to have shape"):
+        loss.gradient_hessian(
+            coef=coef,
+            X=X,
+            y=y,
+            gradient_out=None,
+            hessian_out=hessian_out,
+        )
+
+    loss = LinearModelLoss(base_loss=HalfMultinomialLoss(), fit_intercept=False)
+    coef = loss.init_zero_coef(X)
+    gradient_out = np.zeros((2 * n_classes, n_features))[::2]
+    with pytest.raises(ValueError, match="gradient_out must be F-contiguous"):
+        loss.gradient_hessian(
+            coef=coef,
+            X=X,
+            y=y,
+            gradient_out=gradient_out,
+        )
+    hessian_out = np.zeros((2 * n_classes * n_features, n_classes * n_features))[::2]
+    with pytest.raises(ValueError, match="hessian_out must be contiguous"):
+        loss.gradient_hessian(
+            coef=coef,
+            X=X,
+            y=y,
+            gradient_out=None,
+            hessian_out=hessian_out,
+        )
diff --git a/sklearn/linear_model/tests/test_logistic.py b/sklearn/linear_model/tests/test_logistic.py
index bdc9a4a24914b..e8e41a25c6e2b 100644
--- a/sklearn/linear_model/tests/test_logistic.py
+++ b/sklearn/linear_model/tests/test_logistic.py
@@ -1,40 +1,60 @@
+import itertools
 import os
-import re
 import warnings
-import numpy as np
-from numpy.testing import assert_allclose, assert_almost_equal
-from numpy.testing import assert_array_almost_equal, assert_array_equal
-import scipy.sparse as sp
-from scipy import linalg, optimize, sparse
+from functools import partial
 
+import numpy as np
 import pytest
+from numpy.testing import (
+    assert_allclose,
+    assert_almost_equal,
+    assert_array_almost_equal,
+    assert_array_equal,
+)
+from scipy import sparse
+from scipy.linalg import LinAlgWarning, svd
 
+from sklearn import config_context
+from sklearn._loss import HalfMultinomialLoss
 from sklearn.base import clone
-from sklearn.datasets import load_iris, make_classification
-from sklearn.metrics import log_loss
-from sklearn.metrics import get_scorer
-from sklearn.model_selection import StratifiedKFold
-from sklearn.model_selection import GridSearchCV
-from sklearn.model_selection import train_test_split
-from sklearn.model_selection import cross_val_score
-from sklearn.preprocessing import LabelEncoder, StandardScaler
-from sklearn.utils import compute_class_weight, _IS_32BIT
-from sklearn.utils._testing import ignore_warnings
-from sklearn.utils import shuffle
-from sklearn.linear_model import SGDClassifier
-from sklearn.preprocessing import scale
-from sklearn.utils._testing import skip_if_no_parallel
-
+from sklearn.datasets import load_iris, make_classification, make_low_rank_matrix
 from sklearn.exceptions import ConvergenceWarning
+from sklearn.linear_model import SGDClassifier
 from sklearn.linear_model._logistic import (
-    LogisticRegression,
-    _logistic_regression_path, LogisticRegressionCV,
-    _logistic_loss_and_grad, _logistic_grad_hess,
-    _multinomial_grad_hess, _logistic_loss,
-    _log_reg_scoring_path)
+    LogisticRegression as LogisticRegressionDefault,
+)
+from sklearn.linear_model._logistic import (
+    LogisticRegressionCV as LogisticRegressionCVDefault,
+)
+from sklearn.linear_model._logistic import (
+    _log_reg_scoring_path,
+    _logistic_regression_path,
+)
+from sklearn.metrics import get_scorer, log_loss
+from sklearn.model_selection import (
+    GridSearchCV,
+    LeaveOneGroupOut,
+    StratifiedKFold,
+    cross_val_score,
+    train_test_split,
+)
+from sklearn.multiclass import OneVsRestClassifier
+from sklearn.preprocessing import LabelEncoder, StandardScaler, scale
+from sklearn.svm import l1_min_c
+from sklearn.utils import compute_class_weight, shuffle
+from sklearn.utils._testing import ignore_warnings, skip_if_no_parallel
+from sklearn.utils.fixes import _IS_32BIT, COO_CONTAINERS, CSR_CONTAINERS
+
+pytestmark = pytest.mark.filterwarnings(
+    "error::sklearn.exceptions.ConvergenceWarning:sklearn.*"
+)
+# Fixing random_state helps prevent ConvergenceWarnings
+LogisticRegression = partial(LogisticRegressionDefault, random_state=0)
+LogisticRegressionCV = partial(LogisticRegressionCVDefault, random_state=0)
+
 
+SOLVERS = ("lbfgs", "liblinear", "newton-cg", "newton-cholesky", "sag", "saga")
 X = [[-1, 0], [0, 1], [1, 1]]
-X_sp = sp.csr_matrix(X)
 Y1 = [0, 1, 1]
 Y2 = [2, 1, 0]
 iris = load_iris()
@@ -58,55 +78,23 @@ def check_predictions(clf, X, y):
     assert_array_equal(probabilities.argmax(axis=1), y)
 
 
-def test_predict_2_classes():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_predict_2_classes(csr_container):
     # Simple sanity check on a 2 classes dataset
     # Make sure it predicts the correct result on simple datasets.
     check_predictions(LogisticRegression(random_state=0), X, Y1)
-    check_predictions(LogisticRegression(random_state=0), X_sp, Y1)
+    check_predictions(LogisticRegression(random_state=0), csr_container(X), Y1)
 
     check_predictions(LogisticRegression(C=100, random_state=0), X, Y1)
-    check_predictions(LogisticRegression(C=100, random_state=0), X_sp, Y1)
-
-    check_predictions(LogisticRegression(fit_intercept=False,
-                                         random_state=0), X, Y1)
-    check_predictions(LogisticRegression(fit_intercept=False,
-                                         random_state=0), X_sp, Y1)
-
-
-def test_error():
-    # Test for appropriate exception on errors
-    msg = "Penalty term must be positive"
-
-    with pytest.raises(ValueError, match=msg):
-        LogisticRegression(C=-1).fit(X, Y1)
-
-    with pytest.raises(ValueError, match=msg):
-        LogisticRegression(C="test").fit(X, Y1)
-
-    msg = "is not a valid scoring value"
-    with pytest.raises(ValueError, match=msg):
-        LogisticRegressionCV(scoring='bad-scorer', cv=2).fit(X, Y1)
-
-    for LR in [LogisticRegression, LogisticRegressionCV]:
-        msg = "Tolerance for stopping criteria must be positive"
+    check_predictions(LogisticRegression(C=100, random_state=0), csr_container(X), Y1)
 
-        with pytest.raises(ValueError, match=msg):
-            LR(tol=-1).fit(X, Y1)
-
-        with pytest.raises(ValueError, match=msg):
-            LR(tol="test").fit(X, Y1)
-
-        msg = "Maximum number of iteration must be positive"
-
-        with pytest.raises(ValueError, match=msg):
-            LR(max_iter=-1).fit(X, Y1)
-
-        with pytest.raises(ValueError, match=msg):
-            LR(max_iter="test").fit(X, Y1)
+    check_predictions(LogisticRegression(fit_intercept=False, random_state=0), X, Y1)
+    check_predictions(
+        LogisticRegression(fit_intercept=False, random_state=0), csr_container(X), Y1
+    )
 
 
 def test_logistic_cv_mock_scorer():
-
     class MockScorer:
         def __init__(self):
             self.calls = 0
@@ -122,7 +110,8 @@ def __call__(self, model, X, y, sample_weight=None):
     cv = 2
 
     lr = LogisticRegressionCV(Cs=Cs, scoring=mock_scorer, cv=cv)
-    lr.fit(X, Y1)
+    X, y = make_classification(random_state=0)
+    lr.fit(X, y)
 
     # Cs[2] has the highest score (0.8) from MockScorer
     assert lr.C_[0] == Cs[2]
@@ -138,182 +127,195 @@ def __call__(self, model, X, y, sample_weight=None):
     assert mock_scorer.calls == 1
 
 
-def test_logistic_cv_score_does_not_warn_by_default():
-    lr = LogisticRegressionCV(cv=2)
-    lr.fit(X, Y1)
-
-    with pytest.warns(None) as record:
-        lr.score(X, lr.predict(X))
-    assert len(record) == 0
-
-
 @skip_if_no_parallel
 def test_lr_liblinear_warning():
-    n_samples, n_features = iris.data.shape
-    target = iris.target_names[iris.target]
+    X, y = make_classification(random_state=0)
 
-    lr = LogisticRegression(solver='liblinear', n_jobs=2)
+    lr = LogisticRegression(solver="liblinear", n_jobs=2)
     warning_message = (
         "'n_jobs' > 1 does not have any effect when"
         " 'solver' is set to 'liblinear'. Got 'n_jobs'"
         " = 2."
     )
     with pytest.warns(UserWarning, match=warning_message):
-        lr.fit(iris.data, target)
+        lr.fit(X, y)
 
 
-def test_predict_3_classes():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_predict_3_classes(csr_container):
     check_predictions(LogisticRegression(C=10), X, Y2)
-    check_predictions(LogisticRegression(C=10), X_sp, Y2)
+    check_predictions(LogisticRegression(C=10), csr_container(X), Y2)
 
 
-def test_predict_iris():
-    # Test logistic regression with the iris dataset
-    n_samples, n_features = iris.data.shape
+# TODO(1.8): remove filterwarnings after the deprecation of multi_class
+@pytest.mark.filterwarnings("ignore:.*'multi_class' was deprecated.*:FutureWarning")
+@pytest.mark.filterwarnings(
+    "ignore:.*'liblinear' solver for multiclass classification is deprecated.*"
+)
+@pytest.mark.parametrize(
+    "clf",
+    [
+        LogisticRegression(C=len(iris.data), solver="liblinear", multi_class="ovr"),
+        LogisticRegression(C=len(iris.data), solver="lbfgs"),
+        LogisticRegression(C=len(iris.data), solver="newton-cg"),
+        LogisticRegression(
+            C=len(iris.data), solver="sag", tol=1e-2, multi_class="ovr", random_state=42
+        ),
+        LogisticRegression(
+            C=len(iris.data),
+            solver="saga",
+            tol=1e-2,
+            multi_class="ovr",
+            random_state=42,
+        ),
+        LogisticRegression(C=len(iris.data), solver="newton-cholesky"),
+    ],
+)
+def test_predict_iris(clf):
+    """Test logistic regression with the iris dataset.
 
+    Test that both multinomial and OvR solvers handle multiclass data correctly and
+    give good accuracy score (>0.95) for the training data.
+    """
+    n_samples, n_features = iris.data.shape
     target = iris.target_names[iris.target]
 
-    # Test that both multinomial and OvR solvers handle
-    # multiclass data correctly and give good accuracy
-    # score (>0.95) for the training data.
-    for clf in [LogisticRegression(C=len(iris.data), solver='liblinear',
-                                   multi_class='ovr'),
-                LogisticRegression(C=len(iris.data), solver='lbfgs',
-                                   multi_class='multinomial'),
-                LogisticRegression(C=len(iris.data), solver='newton-cg',
-                                   multi_class='multinomial'),
-                LogisticRegression(C=len(iris.data), solver='sag', tol=1e-2,
-                                   multi_class='ovr', random_state=42),
-                LogisticRegression(C=len(iris.data), solver='saga', tol=1e-2,
-                                   multi_class='ovr', random_state=42)
-                ]:
+    if clf.solver == "lbfgs":
+        # lbfgs has convergence issues on the iris data with its default max_iter=100
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", ConvergenceWarning)
+            clf.fit(iris.data, target)
+    else:
         clf.fit(iris.data, target)
-        assert_array_equal(np.unique(target), clf.classes_)
-
-        pred = clf.predict(iris.data)
-        assert np.mean(pred == target) > .95
+    assert_array_equal(np.unique(target), clf.classes_)
 
-        probabilities = clf.predict_proba(iris.data)
-        assert_array_almost_equal(probabilities.sum(axis=1),
-                                  np.ones(n_samples))
+    pred = clf.predict(iris.data)
+    assert np.mean(pred == target) > 0.95
 
-        pred = iris.target_names[probabilities.argmax(axis=1)]
-        assert np.mean(pred == target) > .95
+    probabilities = clf.predict_proba(iris.data)
+    assert_allclose(probabilities.sum(axis=1), np.ones(n_samples))
 
+    pred = iris.target_names[probabilities.argmax(axis=1)]
+    assert np.mean(pred == target) > 0.95
 
-@pytest.mark.parametrize('solver', ['lbfgs', 'newton-cg', 'sag', 'saga'])
-def test_multinomial_validation(solver):
-    lr = LogisticRegression(C=-1, solver=solver, multi_class='multinomial')
-
-    with pytest.raises(ValueError):
-        lr.fit([[0, 1], [1, 0]], [0, 1])
 
-
-@pytest.mark.parametrize('LR', [LogisticRegression, LogisticRegressionCV])
+# TODO(1.8): remove filterwarnings after the deprecation of multi_class
+@pytest.mark.filterwarnings("ignore:.*'multi_class' was deprecated.*:FutureWarning")
+@pytest.mark.parametrize("LR", [LogisticRegression, LogisticRegressionCV])
 def test_check_solver_option(LR):
     X, y = iris.data, iris.target
 
-    msg = (r"Logistic Regression supports only solvers in \['liblinear', "
-           r"'newton-cg', 'lbfgs', 'sag', 'saga'\], got wrong_name.")
-    lr = LR(solver="wrong_name", multi_class="ovr")
-    with pytest.raises(ValueError, match=msg):
-        lr.fit(X, y)
-
-    msg = ("multi_class should be 'multinomial', 'ovr' or 'auto'. "
-           "Got wrong_name")
-    lr = LR(solver='newton-cg', multi_class="wrong_name")
-    with pytest.raises(ValueError, match=msg):
-        lr.fit(X, y)
-
     # only 'liblinear' solver
-    msg = "Solver liblinear does not support a multinomial backend."
-    lr = LR(solver='liblinear', multi_class='multinomial')
-    with pytest.raises(ValueError, match=msg):
-        lr.fit(X, y)
+    for solver in ["liblinear"]:
+        msg = f"Solver {solver} does not support a multinomial backend."
+        lr = LR(solver=solver, multi_class="multinomial")
+        with pytest.raises(ValueError, match=msg):
+            lr.fit(X, y)
 
     # all solvers except 'liblinear' and 'saga'
-    for solver in ['newton-cg', 'lbfgs', 'sag']:
-        msg = ("Solver %s supports only 'l2' or 'none' penalties," %
-               solver)
-        lr = LR(solver=solver, penalty='l1', multi_class='ovr')
+    for solver in ["lbfgs", "newton-cg", "newton-cholesky", "sag"]:
+        msg = "Solver %s supports only 'l2' or None penalties," % solver
+        lr = LR(solver=solver, penalty="l1", multi_class="ovr")
         with pytest.raises(ValueError, match=msg):
             lr.fit(X, y)
-    for solver in ['newton-cg', 'lbfgs', 'sag', 'saga']:
-        msg = ("Solver %s supports only dual=False, got dual=True" %
-               solver)
-        lr = LR(solver=solver, dual=True, multi_class='ovr')
+    for solver in ["lbfgs", "newton-cg", "newton-cholesky", "sag", "saga"]:
+        msg = "Solver %s supports only dual=False, got dual=True" % solver
+        lr = LR(solver=solver, dual=True, multi_class="ovr")
         with pytest.raises(ValueError, match=msg):
             lr.fit(X, y)
 
     # only saga supports elasticnet. We only test for liblinear because the
     # error is raised before for the other solvers (solver %s supports only l2
     # penalties)
-    for solver in ['liblinear']:
-        msg = ("Only 'saga' solver supports elasticnet penalty, got "
-               "solver={}.".format(solver))
-        lr = LR(solver=solver, penalty='elasticnet')
+    for solver in ["liblinear"]:
+        msg = f"Only 'saga' solver supports elasticnet penalty, got solver={solver}."
+        lr = LR(solver=solver, penalty="elasticnet")
         with pytest.raises(ValueError, match=msg):
             lr.fit(X, y)
 
     # liblinear does not support penalty='none'
-    msg = "penalty='none' is not supported for the liblinear solver"
-    lr = LR(penalty='none', solver='liblinear')
-    with pytest.raises(ValueError, match=msg):
-        lr.fit(X, y)
+    # (LogisticRegressionCV does not supports penalty='none' at all)
+    if LR is LogisticRegression:
+        msg = "penalty=None is not supported for the liblinear solver"
+        lr = LR(penalty=None, solver="liblinear")
+        with pytest.raises(ValueError, match=msg):
+            lr.fit(X, y)
+
+
+@pytest.mark.parametrize("LR", [LogisticRegression, LogisticRegressionCV])
+def test_elasticnet_l1_ratio_err_helpful(LR):
+    # Check that an informative error message is raised when penalty="elasticnet"
+    # but l1_ratio is not specified.
+    model = LR(penalty="elasticnet", solver="saga")
+    with pytest.raises(ValueError, match=r".*l1_ratio.*"):
+        model.fit(np.array([[1, 2], [3, 4]]), np.array([0, 1]))
 
 
-@pytest.mark.parametrize('solver', ['lbfgs', 'newton-cg', 'sag', 'saga'])
+# TODO(1.8): remove whole test with deprecation of multi_class
+@pytest.mark.filterwarnings("ignore:.*'multi_class' was deprecated.*:FutureWarning")
+@pytest.mark.parametrize("solver", ["lbfgs", "newton-cg", "sag", "saga"])
 def test_multinomial_binary(solver):
     # Test multinomial LR on a binary problem.
     target = (iris.target > 0).astype(np.intp)
     target = np.array(["setosa", "not-setosa"])[target]
 
-    clf = LogisticRegression(solver=solver, multi_class='multinomial',
-                             random_state=42, max_iter=2000)
+    clf = LogisticRegression(
+        solver=solver, multi_class="multinomial", random_state=42, max_iter=2000
+    )
     clf.fit(iris.data, target)
 
     assert clf.coef_.shape == (1, iris.data.shape[1])
     assert clf.intercept_.shape == (1,)
     assert_array_equal(clf.predict(iris.data), target)
 
-    mlr = LogisticRegression(solver=solver, multi_class='multinomial',
-                             random_state=42, fit_intercept=False)
+    mlr = LogisticRegression(
+        solver=solver, multi_class="multinomial", random_state=42, fit_intercept=False
+    )
     mlr.fit(iris.data, target)
-    pred = clf.classes_[np.argmax(clf.predict_log_proba(iris.data),
-                                  axis=1)]
-    assert np.mean(pred == target) > .9
+    pred = clf.classes_[np.argmax(clf.predict_log_proba(iris.data), axis=1)]
+    assert np.mean(pred == target) > 0.9
 
 
-def test_multinomial_binary_probabilities():
+# TODO(1.8): remove filterwarnings after the deprecation of multi_class
+# Maybe even remove this whole test as correctness of multinomial loss is tested
+# elsewhere.
+@pytest.mark.filterwarnings("ignore:.*'multi_class' was deprecated.*:FutureWarning")
+def test_multinomial_binary_probabilities(global_random_seed):
     # Test multinomial LR gives expected probabilities based on the
     # decision function, for a binary problem.
-    X, y = make_classification()
-    clf = LogisticRegression(multi_class='multinomial', solver='saga')
+    X, y = make_classification(random_state=global_random_seed)
+    clf = LogisticRegression(
+        multi_class="multinomial",
+        solver="saga",
+        tol=1e-3,
+        random_state=global_random_seed,
+    )
     clf.fit(X, y)
 
     decision = clf.decision_function(X)
     proba = clf.predict_proba(X)
 
-    expected_proba_class_1 = (np.exp(decision) /
-                              (np.exp(decision) + np.exp(-decision)))
+    expected_proba_class_1 = np.exp(decision) / (np.exp(decision) + np.exp(-decision))
     expected_proba = np.c_[1 - expected_proba_class_1, expected_proba_class_1]
 
     assert_almost_equal(proba, expected_proba)
 
 
-def test_sparsify():
+@pytest.mark.parametrize("coo_container", COO_CONTAINERS)
+def test_sparsify(coo_container):
     # Test sparsify and densify members.
     n_samples, n_features = iris.data.shape
     target = iris.target_names[iris.target]
-    clf = LogisticRegression(random_state=0).fit(iris.data, target)
+    X = scale(iris.data)
+    clf = LogisticRegression(random_state=0).fit(X, target)
 
-    pred_d_d = clf.decision_function(iris.data)
+    pred_d_d = clf.decision_function(X)
 
     clf.sparsify()
-    assert sp.issparse(clf.coef_)
-    pred_s_d = clf.decision_function(iris.data)
+    assert sparse.issparse(clf.coef_)
+    pred_s_d = clf.decision_function(X)
 
-    sp_data = sp.coo_matrix(iris.data)
+    sp_data = coo_container(X)
     pred_s_s = clf.decision_function(sp_data)
 
     clf.densify()
@@ -374,32 +376,56 @@ def test_consistency_path():
     f = ignore_warnings
     # can't test with fit_intercept=True since LIBLINEAR
     # penalizes the intercept
-    for solver in ['sag', 'saga']:
+    for solver in ["sag", "saga"]:
         coefs, Cs, _ = f(_logistic_regression_path)(
-            X, y, Cs=Cs, fit_intercept=False, tol=1e-5, solver=solver,
-            max_iter=1000, multi_class='ovr', random_state=0)
+            X,
+            y,
+            Cs=Cs,
+            fit_intercept=False,
+            tol=1e-5,
+            solver=solver,
+            max_iter=1000,
+            random_state=0,
+        )
         for i, C in enumerate(Cs):
-            lr = LogisticRegression(C=C, fit_intercept=False, tol=1e-5,
-                                    solver=solver, multi_class='ovr',
-                                    random_state=0, max_iter=1000)
+            lr = LogisticRegression(
+                C=C,
+                fit_intercept=False,
+                tol=1e-5,
+                solver=solver,
+                random_state=0,
+                max_iter=1000,
+            )
             lr.fit(X, y)
             lr_coef = lr.coef_.ravel()
-            assert_array_almost_equal(lr_coef, coefs[i], decimal=4,
-                                      err_msg="with solver = %s" % solver)
+            assert_array_almost_equal(
+                lr_coef, coefs[i], decimal=4, err_msg="with solver = %s" % solver
+            )
 
     # test for fit_intercept=True
-    for solver in ('lbfgs', 'newton-cg', 'liblinear', 'sag', 'saga'):
+    for solver in ("lbfgs", "newton-cg", "newton-cholesky", "liblinear", "sag", "saga"):
         Cs = [1e3]
         coefs, Cs, _ = f(_logistic_regression_path)(
-            X, y, Cs=Cs, tol=1e-6, solver=solver,
-            intercept_scaling=10000., random_state=0, multi_class='ovr')
-        lr = LogisticRegression(C=Cs[0], tol=1e-4,
-                                intercept_scaling=10000., random_state=0,
-                                multi_class='ovr', solver=solver)
+            X,
+            y,
+            Cs=Cs,
+            tol=1e-6,
+            solver=solver,
+            intercept_scaling=10000.0,
+            random_state=0,
+        )
+        lr = LogisticRegression(
+            C=Cs[0],
+            tol=1e-6,
+            intercept_scaling=10000.0,
+            random_state=0,
+            solver=solver,
+        )
         lr.fit(X, y)
         lr_coef = np.concatenate([lr.coef_.ravel(), lr.intercept_])
-        assert_array_almost_equal(lr_coef, coefs[0], decimal=4,
-                                  err_msg="with solver = %s" % solver)
+        assert_array_almost_equal(
+            lr_coef, coefs[0], decimal=4, err_msg="with solver = %s" % solver
+        )
 
 
 def test_logistic_regression_path_convergence_fail():
@@ -412,15 +438,13 @@ def test_logistic_regression_path_convergence_fail():
     # advice (scaling the data) and to the logistic regression specific
     # documentation that includes hints on the solver configuration.
     with pytest.warns(ConvergenceWarning) as record:
-        with warnings.catch_warnings():
-            # scipy 1.3.0 uses tostring which is deprecated in numpy
-            warnings.filterwarnings("ignore", "tostring", DeprecationWarning)
-            _logistic_regression_path(
-                X, y, Cs=Cs, tol=0., max_iter=1, random_state=0, verbose=0)
+        _logistic_regression_path(
+            X, y, Cs=Cs, tol=0.0, max_iter=1, random_state=0, verbose=0
+        )
 
     assert len(record) == 1
     warn_msg = record[0].message.args[0]
-    assert "lbfgs failed to converge" in warn_msg
+    assert "lbfgs failed to converge after 1 iteration(s)" in warn_msg
     assert "Increase the number of iterations" in warn_msg
     assert "scale the data" in warn_msg
     assert "linear_model.html#logistic-regression" in warn_msg
@@ -429,14 +453,26 @@ def test_logistic_regression_path_convergence_fail():
 def test_liblinear_dual_random_state():
     # random_state is relevant for liblinear solver only if dual=True
     X, y = make_classification(n_samples=20, random_state=0)
-    lr1 = LogisticRegression(random_state=0, dual=True, max_iter=1, tol=1e-15,
-                             solver='liblinear', multi_class='ovr')
+    lr1 = LogisticRegression(
+        random_state=0,
+        dual=True,
+        tol=1e-3,
+        solver="liblinear",
+    )
     lr1.fit(X, y)
-    lr2 = LogisticRegression(random_state=0, dual=True, max_iter=1, tol=1e-15,
-                             solver='liblinear', multi_class='ovr')
+    lr2 = LogisticRegression(
+        random_state=0,
+        dual=True,
+        tol=1e-3,
+        solver="liblinear",
+    )
     lr2.fit(X, y)
-    lr3 = LogisticRegression(random_state=8, dual=True, max_iter=1, tol=1e-15,
-                             solver='liblinear', multi_class='ovr')
+    lr3 = LogisticRegression(
+        random_state=8,
+        dual=True,
+        tol=1e-3,
+        solver="liblinear",
+    )
     lr3.fit(X, y)
 
     # same result for same random state
@@ -447,85 +483,6 @@ def test_liblinear_dual_random_state():
         assert_array_almost_equal(lr1.coef_, lr3.coef_)
 
 
-def test_logistic_loss_and_grad():
-    X_ref, y = make_classification(n_samples=20, random_state=0)
-    n_features = X_ref.shape[1]
-
-    X_sp = X_ref.copy()
-    X_sp[X_sp < .1] = 0
-    X_sp = sp.csr_matrix(X_sp)
-    for X in (X_ref, X_sp):
-        w = np.zeros(n_features)
-
-        # First check that our derivation of the grad is correct
-        loss, grad = _logistic_loss_and_grad(w, X, y, alpha=1.)
-        approx_grad = optimize.approx_fprime(
-            w, lambda w: _logistic_loss_and_grad(w, X, y, alpha=1.)[0], 1e-3
-        )
-        assert_array_almost_equal(grad, approx_grad, decimal=2)
-
-        # Second check that our intercept implementation is good
-        w = np.zeros(n_features + 1)
-        loss_interp, grad_interp = _logistic_loss_and_grad(
-            w, X, y, alpha=1.
-        )
-        assert_array_almost_equal(loss, loss_interp)
-
-        approx_grad = optimize.approx_fprime(
-            w, lambda w: _logistic_loss_and_grad(w, X, y, alpha=1.)[0], 1e-3
-        )
-        assert_array_almost_equal(grad_interp, approx_grad, decimal=2)
-
-
-def test_logistic_grad_hess():
-    rng = np.random.RandomState(0)
-    n_samples, n_features = 50, 5
-    X_ref = rng.randn(n_samples, n_features)
-    y = np.sign(X_ref.dot(5 * rng.randn(n_features)))
-    X_ref -= X_ref.mean()
-    X_ref /= X_ref.std()
-    X_sp = X_ref.copy()
-    X_sp[X_sp < .1] = 0
-    X_sp = sp.csr_matrix(X_sp)
-    for X in (X_ref, X_sp):
-        w = np.full(n_features, .1)
-
-        # First check that _logistic_grad_hess is consistent
-        # with _logistic_loss_and_grad
-        loss, grad = _logistic_loss_and_grad(w, X, y, alpha=1.)
-        grad_2, hess = _logistic_grad_hess(w, X, y, alpha=1.)
-        assert_array_almost_equal(grad, grad_2)
-
-        # Now check our hessian along the second direction of the grad
-        vector = np.zeros_like(grad)
-        vector[1] = 1
-        hess_col = hess(vector)
-
-        # Computation of the Hessian is particularly fragile to numerical
-        # errors when doing simple finite differences. Here we compute the
-        # grad along a path in the direction of the vector and then use a
-        # least-square regression to estimate the slope
-        e = 1e-3
-        d_x = np.linspace(-e, e, 30)
-        d_grad = np.array([
-            _logistic_loss_and_grad(w + t * vector, X, y, alpha=1.)[1]
-            for t in d_x
-        ])
-
-        d_grad -= d_grad.mean(axis=0)
-        approx_hess_col = linalg.lstsq(d_x[:, np.newaxis], d_grad)[0].ravel()
-
-        assert_array_almost_equal(approx_hess_col, hess_col, decimal=3)
-
-        # Second check that our intercept implementation is good
-        w = np.zeros(n_features + 1)
-        loss_interp, grad_interp = _logistic_loss_and_grad(w, X, y, alpha=1.)
-        loss_interp_2 = _logistic_loss(w, X, y, alpha=1.)
-        grad_interp_2, hess = _logistic_grad_hess(w, X, y, alpha=1.)
-        assert_array_almost_equal(loss_interp, loss_interp_2)
-        assert_array_almost_equal(grad_interp, grad_interp_2)
-
-
 def test_logistic_cv():
     # test for LogisticRegressionCV object
     n_samples, n_features = 50, 5
@@ -534,11 +491,11 @@ def test_logistic_cv():
     y = np.sign(X_ref.dot(5 * rng.randn(n_features)))
     X_ref -= X_ref.mean()
     X_ref /= X_ref.std()
-    lr_cv = LogisticRegressionCV(Cs=[1.], fit_intercept=False,
-                                 solver='liblinear', multi_class='ovr', cv=3)
+    lr_cv = LogisticRegressionCV(
+        Cs=[1.0], fit_intercept=False, solver="liblinear", cv=3
+    )
     lr_cv.fit(X_ref, y)
-    lr = LogisticRegression(C=1., fit_intercept=False,
-                            solver='liblinear', multi_class='ovr')
+    lr = LogisticRegression(C=1.0, fit_intercept=False, solver="liblinear")
     lr.fit(X_ref, y)
     assert_array_almost_equal(lr.coef_, lr_cv.coef_)
 
@@ -553,53 +510,74 @@ def test_logistic_cv():
     assert_array_equal(scores.shape, (1, 3, 1))
 
 
-@pytest.mark.parametrize('scoring, multiclass_agg_list',
-                         [('accuracy', ['']),
-                          ('precision', ['_macro', '_weighted']),
-                          # no need to test for micro averaging because it
-                          # is the same as accuracy for f1, precision,
-                          # and recall (see https://github.com/
-                          # scikit-learn/scikit-learn/pull/
-                          # 11578#discussion_r203250062)
-                          ('f1', ['_macro', '_weighted']),
-                          ('neg_log_loss', ['']),
-                          ('recall', ['_macro', '_weighted'])])
+@pytest.mark.parametrize(
+    "scoring, multiclass_agg_list",
+    [
+        ("accuracy", [""]),
+        ("precision", ["_macro", "_weighted"]),
+        # no need to test for micro averaging because it
+        # is the same as accuracy for f1, precision,
+        # and recall (see https://github.com/
+        # scikit-learn/scikit-learn/pull/
+        # 11578#discussion_r203250062)
+        ("f1", ["_macro", "_weighted"]),
+        ("neg_log_loss", [""]),
+        ("recall", ["_macro", "_weighted"]),
+    ],
+)
 def test_logistic_cv_multinomial_score(scoring, multiclass_agg_list):
     # test that LogisticRegressionCV uses the right score to compute its
     # cross-validation scores when using a multinomial scoring
     # see https://github.com/scikit-learn/scikit-learn/issues/8720
-    X, y = make_classification(n_samples=100, random_state=0, n_classes=3,
-                               n_informative=6)
+    X, y = make_classification(
+        n_samples=100, random_state=0, n_classes=3, n_informative=6
+    )
     train, test = np.arange(80), np.arange(80, 100)
-    lr = LogisticRegression(C=1., multi_class='multinomial')
+    lr = LogisticRegression(C=1.0)
     # we use lbfgs to support multinomial
     params = lr.get_params()
     # we store the params to set them further in _log_reg_scoring_path
-    for key in ['C', 'n_jobs', 'warm_start']:
+    for key in ["C", "n_jobs", "warm_start"]:
         del params[key]
     lr.fit(X[train], y[train])
     for averaging in multiclass_agg_list:
         scorer = get_scorer(scoring + averaging)
         assert_array_almost_equal(
-            _log_reg_scoring_path(X, y, train, test, Cs=[1.],
-                                  scoring=scorer, **params)[2][0],
-            scorer(lr, X[test], y[test]))
+            _log_reg_scoring_path(
+                X,
+                y,
+                train,
+                test,
+                Cs=[1.0],
+                scoring=scorer,
+                pos_class=None,
+                max_squared_sum=None,
+                sample_weight=None,
+                score_params=None,
+                **(params | {"multi_class": "multinomial"}),
+            )[2][0],
+            scorer(lr, X[test], y[test]),
+        )
 
 
 def test_multinomial_logistic_regression_string_inputs():
     # Test with string labels for LogisticRegression(CV)
     n_samples, n_features, n_classes = 50, 5, 3
-    X_ref, y = make_classification(n_samples=n_samples, n_features=n_features,
-                                   n_classes=n_classes, n_informative=3,
-                                   random_state=0)
-    y_str = LabelEncoder().fit(['bar', 'baz', 'foo']).inverse_transform(y)
+    X_ref, y = make_classification(
+        n_samples=n_samples,
+        n_features=n_features,
+        n_classes=n_classes,
+        n_informative=3,
+        random_state=0,
+    )
+    y_str = LabelEncoder().fit(["bar", "baz", "foo"]).inverse_transform(y)
     # For numerical labels, let y values be taken from set (-1, 0, 1)
     y = np.array(y) - 1
     # Test for string labels
-    lr = LogisticRegression(multi_class='multinomial')
-    lr_cv = LogisticRegressionCV(multi_class='multinomial', Cs=3)
-    lr_str = LogisticRegression(multi_class='multinomial')
-    lr_cv_str = LogisticRegressionCV(multi_class='multinomial', Cs=3)
+    lr = LogisticRegression()
+    lr_cv = LogisticRegressionCV(Cs=3)
+    lr_str = LogisticRegression()
+    lr_cv_str = LogisticRegressionCV(Cs=3)
 
     lr.fit(X_ref, y)
     lr_cv.fit(X_ref, y)
@@ -607,27 +585,27 @@ def test_multinomial_logistic_regression_string_inputs():
     lr_cv_str.fit(X_ref, y_str)
 
     assert_array_almost_equal(lr.coef_, lr_str.coef_)
-    assert sorted(lr_str.classes_) == ['bar', 'baz', 'foo']
+    assert sorted(lr_str.classes_) == ["bar", "baz", "foo"]
     assert_array_almost_equal(lr_cv.coef_, lr_cv_str.coef_)
-    assert sorted(lr_str.classes_) == ['bar', 'baz', 'foo']
-    assert sorted(lr_cv_str.classes_) == ['bar', 'baz', 'foo']
+    assert sorted(lr_str.classes_) == ["bar", "baz", "foo"]
+    assert sorted(lr_cv_str.classes_) == ["bar", "baz", "foo"]
 
     # The predictions should be in original labels
-    assert sorted(np.unique(lr_str.predict(X_ref))) == ['bar', 'baz', 'foo']
-    assert sorted(np.unique(lr_cv_str.predict(X_ref))) == ['bar', 'baz', 'foo']
+    assert sorted(np.unique(lr_str.predict(X_ref))) == ["bar", "baz", "foo"]
+    assert sorted(np.unique(lr_cv_str.predict(X_ref))) == ["bar", "baz", "foo"]
 
     # Make sure class weights can be given with string labels
-    lr_cv_str = LogisticRegression(
-        class_weight={'bar': 1, 'baz': 2, 'foo': 0},
-        multi_class='multinomial').fit(X_ref, y_str)
-    assert sorted(np.unique(lr_cv_str.predict(X_ref))) == ['bar', 'baz']
+    lr_cv_str = LogisticRegression(class_weight={"bar": 1, "baz": 2, "foo": 0}).fit(
+        X_ref, y_str
+    )
+    assert sorted(np.unique(lr_cv_str.predict(X_ref))) == ["bar", "baz"]
 
 
-def test_logistic_cv_sparse():
-    X, y = make_classification(n_samples=50, n_features=5,
-                               random_state=0)
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_logistic_cv_sparse(csr_container):
+    X, y = make_classification(n_samples=50, n_features=5, random_state=0)
     X[X < 1.0] = 0.0
-    csr = sp.csr_matrix(X)
+    csr = csr_container(X)
 
     clf = LogisticRegressionCV()
     clf.fit(X, y)
@@ -638,39 +616,9 @@ def test_logistic_cv_sparse():
     assert clfs.C_ == clf.C_
 
 
-def test_intercept_logistic_helper():
-    n_samples, n_features = 10, 5
-    X, y = make_classification(n_samples=n_samples, n_features=n_features,
-                               random_state=0)
-
-    # Fit intercept case.
-    alpha = 1.
-    w = np.ones(n_features + 1)
-    grad_interp, hess_interp = _logistic_grad_hess(w, X, y, alpha)
-    loss_interp = _logistic_loss(w, X, y, alpha)
-
-    # Do not fit intercept. This can be considered equivalent to adding
-    # a feature vector of ones, i.e column of one vectors.
-    X_ = np.hstack((X, np.ones(10)[:, np.newaxis]))
-    grad, hess = _logistic_grad_hess(w, X_, y, alpha)
-    loss = _logistic_loss(w, X_, y, alpha)
-
-    # In the fit_intercept=False case, the feature vector of ones is
-    # penalized. This should be taken care of.
-    assert_almost_equal(loss_interp + 0.5 * (w[-1] ** 2), loss)
-
-    # Check gradient.
-    assert_array_almost_equal(grad_interp[:n_features], grad[:n_features])
-    assert_almost_equal(grad_interp[-1] + alpha * w[-1], grad[-1])
-
-    rng = np.random.RandomState(0)
-    grad = rng.rand(n_features + 1)
-    hess_interp = hess_interp(grad)
-    hess = hess(grad)
-    assert_array_almost_equal(hess_interp[:n_features], hess[:n_features])
-    assert_almost_equal(hess_interp[-1] + alpha * grad[-1], hess[-1])
-
-
+# TODO(1.8): remove filterwarnings after the deprecation of multi_class
+# Best remove this whole test.
+@pytest.mark.filterwarnings("ignore:.*'multi_class' was deprecated.*:FutureWarning")
 def test_ovr_multinomial_iris():
     # Test that OvR and multinomial are correct using the iris dataset.
     train, target = iris.data, iris.target
@@ -684,11 +632,11 @@ def test_ovr_multinomial_iris():
     precomputed_folds = list(cv.split(train, target))
 
     # Train clf on the original dataset where classes 0 and 1 are separated
-    clf = LogisticRegressionCV(cv=precomputed_folds, multi_class='ovr')
+    clf = LogisticRegressionCV(cv=precomputed_folds, multi_class="ovr")
     clf.fit(train, target)
 
     # Conflate classes 0 and 1 and train clf1 on this modified dataset
-    clf1 = LogisticRegressionCV(cv=precomputed_folds, multi_class='ovr')
+    clf1 = LogisticRegressionCV(cv=precomputed_folds, multi_class="ovr")
     target_copy = target.copy()
     target_copy[target_copy == 0] = 1
     clf1.fit(train, target_copy)
@@ -709,12 +657,19 @@ def test_ovr_multinomial_iris():
     assert scores.shape == (3, n_cv, 10)
 
     # Test that for the iris data multinomial gives a better accuracy than OvR
-    for solver in ['lbfgs', 'newton-cg', 'sag', 'saga']:
-        max_iter = 500 if solver in ['sag', 'saga'] else 15
+    for solver in ["lbfgs", "newton-cg", "sag", "saga"]:
+        max_iter = 500 if solver in ["sag", "saga"] else 30
         clf_multi = LogisticRegressionCV(
-            solver=solver, multi_class='multinomial', max_iter=max_iter,
-            random_state=42, tol=1e-3 if solver in ['sag', 'saga'] else 1e-2,
-            cv=2)
+            solver=solver,
+            max_iter=max_iter,
+            random_state=42,
+            tol=1e-3 if solver in ["sag", "saga"] else 1e-2,
+            cv=2,
+        )
+        if solver == "lbfgs":
+            # lbfgs requires scaling to avoid convergence warnings
+            train = scale(train)
+
         clf_multi.fit(train, target)
         multi_score = clf_multi.score(train, target)
         ovr_score = clf.score(train, target)
@@ -731,175 +686,339 @@ def test_ovr_multinomial_iris():
 
 
 def test_logistic_regression_solvers():
+    """Test solvers converge to the same result."""
     X, y = make_classification(n_features=10, n_informative=5, random_state=0)
 
-    params = dict(fit_intercept=False, random_state=42, multi_class='ovr')
-    ncg = LogisticRegression(solver='newton-cg', **params)
-    lbf = LogisticRegression(solver='lbfgs', **params)
-    lib = LogisticRegression(solver='liblinear', **params)
-    sag = LogisticRegression(solver='sag', **params)
-    saga = LogisticRegression(solver='saga', **params)
-    ncg.fit(X, y)
-    lbf.fit(X, y)
-    sag.fit(X, y)
-    saga.fit(X, y)
-    lib.fit(X, y)
-    assert_array_almost_equal(ncg.coef_, lib.coef_, decimal=3)
-    assert_array_almost_equal(lib.coef_, lbf.coef_, decimal=3)
-    assert_array_almost_equal(ncg.coef_, lbf.coef_, decimal=3)
-    assert_array_almost_equal(sag.coef_, lib.coef_, decimal=3)
-    assert_array_almost_equal(sag.coef_, ncg.coef_, decimal=3)
-    assert_array_almost_equal(sag.coef_, lbf.coef_, decimal=3)
-    assert_array_almost_equal(saga.coef_, sag.coef_, decimal=3)
-    assert_array_almost_equal(saga.coef_, lbf.coef_, decimal=3)
-    assert_array_almost_equal(saga.coef_, ncg.coef_, decimal=3)
-    assert_array_almost_equal(saga.coef_, lib.coef_, decimal=3)
-
-
-def test_logistic_regression_solvers_multiclass():
-    X, y = make_classification(n_samples=20, n_features=20, n_informative=10,
-                               n_classes=3, random_state=0)
-    tol = 1e-7
-    params = dict(fit_intercept=False, tol=tol, random_state=42,
-                  multi_class='ovr')
-    ncg = LogisticRegression(solver='newton-cg', **params)
-    lbf = LogisticRegression(solver='lbfgs', **params)
-    lib = LogisticRegression(solver='liblinear', **params)
-    sag = LogisticRegression(solver='sag', max_iter=1000, **params)
-    saga = LogisticRegression(solver='saga', max_iter=10000, **params)
-    ncg.fit(X, y)
-    lbf.fit(X, y)
-    sag.fit(X, y)
-    saga.fit(X, y)
-    lib.fit(X, y)
-    assert_array_almost_equal(ncg.coef_, lib.coef_, decimal=4)
-    assert_array_almost_equal(lib.coef_, lbf.coef_, decimal=4)
-    assert_array_almost_equal(ncg.coef_, lbf.coef_, decimal=4)
-    assert_array_almost_equal(sag.coef_, lib.coef_, decimal=4)
-    assert_array_almost_equal(sag.coef_, ncg.coef_, decimal=4)
-    assert_array_almost_equal(sag.coef_, lbf.coef_, decimal=4)
-    assert_array_almost_equal(saga.coef_, sag.coef_, decimal=4)
-    assert_array_almost_equal(saga.coef_, lbf.coef_, decimal=4)
-    assert_array_almost_equal(saga.coef_, ncg.coef_, decimal=4)
-    assert_array_almost_equal(saga.coef_, lib.coef_, decimal=4)
-
-
-def test_logistic_regressioncv_class_weights():
-    for weight in [{0: 0.1, 1: 0.2}, {0: 0.1, 1: 0.2, 2: 0.5}]:
-        n_classes = len(weight)
-        for class_weight in (weight, 'balanced'):
-            X, y = make_classification(n_samples=30, n_features=3,
-                                       n_repeated=0,
-                                       n_informative=3, n_redundant=0,
-                                       n_classes=n_classes, random_state=0)
-
-            clf_lbf = LogisticRegressionCV(solver='lbfgs', Cs=1,
-                                           fit_intercept=False,
-                                           multi_class='ovr',
-                                           class_weight=class_weight)
-            clf_ncg = LogisticRegressionCV(solver='newton-cg', Cs=1,
-                                           fit_intercept=False,
-                                           multi_class='ovr',
-                                           class_weight=class_weight)
-            clf_lib = LogisticRegressionCV(solver='liblinear', Cs=1,
-                                           fit_intercept=False,
-                                           multi_class='ovr',
-                                           class_weight=class_weight)
-            clf_sag = LogisticRegressionCV(solver='sag', Cs=1,
-                                           fit_intercept=False,
-                                           multi_class='ovr',
-                                           class_weight=class_weight,
-                                           tol=1e-5, max_iter=10000,
-                                           random_state=0)
-            clf_saga = LogisticRegressionCV(solver='saga', Cs=1,
-                                            fit_intercept=False,
-                                            multi_class='ovr',
-                                            class_weight=class_weight,
-                                            tol=1e-5, max_iter=10000,
-                                            random_state=0)
-            clf_lbf.fit(X, y)
-            clf_ncg.fit(X, y)
-            clf_lib.fit(X, y)
-            clf_sag.fit(X, y)
-            clf_saga.fit(X, y)
-            assert_array_almost_equal(clf_lib.coef_, clf_lbf.coef_, decimal=4)
-            assert_array_almost_equal(clf_ncg.coef_, clf_lbf.coef_, decimal=4)
-            assert_array_almost_equal(clf_sag.coef_, clf_lbf.coef_, decimal=4)
-            assert_array_almost_equal(clf_saga.coef_, clf_lbf.coef_, decimal=4)
-
-
-def test_logistic_regression_sample_weights():
-    X, y = make_classification(n_samples=20, n_features=5, n_informative=3,
-                               n_classes=2, random_state=0)
-    sample_weight = y + 1
-
-    for LR in [LogisticRegression, LogisticRegressionCV]:
+    params = dict(fit_intercept=False, random_state=42)
 
-        kw = {'random_state': 42, 'fit_intercept': False, 'multi_class': 'ovr'}
-        if LR is LogisticRegressionCV:
-            kw.update({'Cs': 3, 'cv': 3})
+    regressors = {
+        solver: LogisticRegression(solver=solver, **params).fit(X, y)
+        for solver in SOLVERS
+    }
 
-        # Test that passing sample_weight as ones is the same as
-        # not passing them at all (default None)
-        for solver in ['lbfgs', 'liblinear']:
-            clf_sw_none = LR(solver=solver, **kw)
-            clf_sw_ones = LR(solver=solver, **kw)
-            clf_sw_none.fit(X, y)
-            clf_sw_ones.fit(X, y, sample_weight=np.ones(y.shape[0]))
-            assert_array_almost_equal(
-                clf_sw_none.coef_, clf_sw_ones.coef_, decimal=4)
-
-        # Test that sample weights work the same with the lbfgs,
-        # newton-cg, and 'sag' solvers
-        clf_sw_lbfgs = LR(**kw)
-        clf_sw_lbfgs.fit(X, y, sample_weight=sample_weight)
-        clf_sw_n = LR(solver='newton-cg', **kw)
-        clf_sw_n.fit(X, y, sample_weight=sample_weight)
-        clf_sw_sag = LR(solver='sag', tol=1e-10, **kw)
-        # ignore convergence warning due to small dataset
-        with ignore_warnings():
-            clf_sw_sag.fit(X, y, sample_weight=sample_weight)
-        clf_sw_liblinear = LR(solver='liblinear', **kw)
-        clf_sw_liblinear.fit(X, y, sample_weight=sample_weight)
-        assert_array_almost_equal(
-            clf_sw_lbfgs.coef_, clf_sw_n.coef_, decimal=4)
-        assert_array_almost_equal(
-            clf_sw_lbfgs.coef_, clf_sw_sag.coef_, decimal=4)
+    for solver_1, solver_2 in itertools.combinations(regressors, r=2):
         assert_array_almost_equal(
-            clf_sw_lbfgs.coef_, clf_sw_liblinear.coef_, decimal=4)
-
-        # Test that passing class_weight as [1,2] is the same as
-        # passing class weight = [1,1] but adjusting sample weights
-        # to be 2 for all instances of class 2
-        for solver in ['lbfgs', 'liblinear']:
-            clf_cw_12 = LR(solver=solver, class_weight={0: 1, 1: 2}, **kw)
-            clf_cw_12.fit(X, y)
-            clf_sw_12 = LR(solver=solver, **kw)
-            clf_sw_12.fit(X, y, sample_weight=sample_weight)
-            assert_array_almost_equal(
-                clf_cw_12.coef_, clf_sw_12.coef_, decimal=4)
+            regressors[solver_1].coef_, regressors[solver_2].coef_, decimal=3
+        )
+
+
+# TODO(1.8): remove filterwarnings after the deprecation of multi_class
+@pytest.mark.filterwarnings("ignore:.*'multi_class' was deprecated.*:FutureWarning")
+@pytest.mark.parametrize("fit_intercept", [False, True])
+def test_logistic_regression_solvers_multiclass(fit_intercept):
+    """Test solvers converge to the same result for multiclass problems."""
+    X, y = make_classification(
+        n_samples=20, n_features=20, n_informative=10, n_classes=3, random_state=0
+    )
+    tol = 1e-8
+    params = dict(fit_intercept=fit_intercept, tol=tol, random_state=42)
+
+    # Override max iteration count for specific solvers to allow for
+    # proper convergence.
+    solver_max_iter = {"lbfgs": 200, "sag": 10_000, "saga": 10_000}
+
+    regressors = {
+        solver: LogisticRegression(
+            solver=solver, max_iter=solver_max_iter.get(solver, 100), **params
+        ).fit(X, y)
+        for solver in set(SOLVERS) - set(["liblinear"])
+    }
+
+    for solver_1, solver_2 in itertools.combinations(regressors, r=2):
+        assert_allclose(
+            regressors[solver_1].coef_,
+            regressors[solver_2].coef_,
+            rtol=5e-3 if (solver_1 == "saga" or solver_2 == "saga") else 1e-3,
+            err_msg=f"{solver_1} vs {solver_2}",
+        )
+        if fit_intercept:
+            assert_allclose(
+                regressors[solver_1].intercept_,
+                regressors[solver_2].intercept_,
+                rtol=5e-3 if (solver_1 == "saga" or solver_2 == "saga") else 1e-3,
+                err_msg=f"{solver_1} vs {solver_2}",
+            )
+
+
+@pytest.mark.parametrize("fit_intercept", [False, True])
+def test_logistic_regression_solvers_multiclass_unpenalized(
+    fit_intercept, global_random_seed
+):
+    """Test and compare solver results for unpenalized multinomial multiclass."""
+    # We want to avoid perfect separation.
+    n_samples, n_features, n_classes = 100, 4, 3
+    rng = np.random.RandomState(global_random_seed)
+    X = make_low_rank_matrix(
+        n_samples=n_samples,
+        n_features=n_features + fit_intercept,
+        effective_rank=n_features + fit_intercept,
+        tail_strength=0.1,
+        random_state=rng,
+    )
+    if fit_intercept:
+        X[:, -1] = 1
+    U, s, Vt = svd(X)
+    assert np.all(s > 1e-3)  # to be sure that X is not singular
+    assert np.max(s) / np.min(s) < 100  # condition number of X
+    if fit_intercept:
+        X = X[:, :-1]
+    coef = rng.uniform(low=1, high=3, size=n_features * n_classes)
+    coef = coef.reshape(n_classes, n_features)
+    intercept = rng.uniform(low=-1, high=1, size=n_classes) * fit_intercept
+    raw_prediction = X @ coef.T + intercept
+
+    loss = HalfMultinomialLoss(n_classes=n_classes)
+    proba = loss.link.inverse(raw_prediction)
+    # Only newer numpy version (1.22) support more dimensions on pvals.
+    y = np.zeros(n_samples)
+    for i in range(n_samples):
+        y[i] = np.argwhere(rng.multinomial(n=1, pvals=proba[i, :]))[0, 0]
+
+    tol = 1e-9
+    params = dict(fit_intercept=fit_intercept, random_state=42)
+    solver_max_iter = {"lbfgs": 200, "sag": 10_000, "saga": 10_000}
+    solver_tol = {"sag": 1e-8, "saga": 1e-8}
+    regressors = {
+        solver: LogisticRegression(
+            C=np.inf,
+            solver=solver,
+            tol=solver_tol.get(solver, tol),
+            max_iter=solver_max_iter.get(solver, 100),
+            **params,
+        ).fit(X, y)
+        for solver in set(SOLVERS) - set(["liblinear"])
+    }
+    for solver in regressors.keys():
+        # See the docstring of test_multinomial_identifiability_on_iris for reference.
+        assert_allclose(
+            regressors[solver].coef_.sum(axis=0), 0, atol=1e-10, err_msg=solver
+        )
+
+    for solver_1, solver_2 in itertools.combinations(regressors, r=2):
+        assert_allclose(
+            regressors[solver_1].coef_,
+            regressors[solver_2].coef_,
+            rtol=5e-3 if (solver_1 == "saga" or solver_2 == "saga") else 2e-3,
+            err_msg=f"{solver_1} vs {solver_2}",
+        )
+        if fit_intercept:
+            assert_allclose(
+                regressors[solver_1].intercept_,
+                regressors[solver_2].intercept_,
+                rtol=5e-3 if (solver_1 == "saga" or solver_2 == "saga") else 1e-3,
+                err_msg=f"{solver_1} vs {solver_2}",
+            )
+
+
+@pytest.mark.parametrize("weight", [{0: 0.1, 1: 0.2}, {0: 0.1, 1: 0.2, 2: 0.5}])
+@pytest.mark.parametrize("class_weight", ["weight", "balanced"])
+def test_logistic_regressioncv_class_weights(weight, class_weight, global_random_seed):
+    """Test class_weight for LogisticRegressionCV."""
+    n_classes = len(weight)
+    if class_weight == "weight":
+        class_weight = weight
+
+    X, y = make_classification(
+        n_samples=30,
+        n_features=3,
+        n_repeated=0,
+        n_informative=3,
+        n_redundant=0,
+        n_classes=n_classes,
+        random_state=global_random_seed,
+    )
+    params = dict(
+        Cs=1,
+        fit_intercept=False,
+        class_weight=class_weight,
+        tol=1e-8,
+    )
+    clf_lbfgs = LogisticRegressionCV(solver="lbfgs", **params)
+
+    # XXX: lbfgs' line search can fail and cause a ConvergenceWarning for some
+    # 10% of the random seeds, but only on specific platforms (in particular
+    # when using Atlas BLAS/LAPACK implementation). Doubling the maxls internal
+    # parameter of the solver does not help. However this lack of proper
+    # convergence does not seem to prevent the assertion to pass, so we ignore
+    # the warning for now.
+    # See: https://github.com/scikit-learn/scikit-learn/pull/27649
+    with ignore_warnings(category=ConvergenceWarning):
+        clf_lbfgs.fit(X, y)
+
+    for solver in set(SOLVERS) - set(["lbfgs", "liblinear", "newton-cholesky"]):
+        clf = LogisticRegressionCV(solver=solver, **params)
+        if solver in ("sag", "saga"):
+            clf.set_params(
+                tol=1e-18, max_iter=10000, random_state=global_random_seed + 1
+            )
+        clf.fit(X, y)
+
+        assert_allclose(
+            clf.coef_, clf_lbfgs.coef_, rtol=1e-3, err_msg=f"{solver} vs lbfgs"
+        )
+
+
+@pytest.mark.parametrize("problem", ("single", "cv"))
+@pytest.mark.parametrize(
+    "solver", ("lbfgs", "liblinear", "newton-cg", "newton-cholesky", "sag", "saga")
+)
+def test_logistic_regression_sample_weights(problem, solver, global_random_seed):
+    n_samples_per_cv_group = 200
+    n_cv_groups = 3
+
+    X, y = make_classification(
+        n_samples=n_samples_per_cv_group * n_cv_groups,
+        n_features=5,
+        n_informative=3,
+        n_classes=2,
+        n_redundant=0,
+        random_state=global_random_seed,
+    )
+    rng = np.random.RandomState(global_random_seed)
+    sw = np.ones(y.shape[0])
+
+    kw_weighted = {
+        "random_state": global_random_seed,
+        "fit_intercept": False,
+        "max_iter": 100_000 if solver.startswith("sag") else 1_000,
+        "tol": 1e-8,
+    }
+    kw_repeated = kw_weighted.copy()
+    sw[:n_samples_per_cv_group] = rng.randint(0, 5, size=n_samples_per_cv_group)
+    X_repeated = np.repeat(X, sw.astype(int), axis=0)
+    y_repeated = np.repeat(y, sw.astype(int), axis=0)
+
+    if problem == "single":
+        LR = LogisticRegression
+    elif problem == "cv":
+        LR = LogisticRegressionCV
+        # We weight the first fold 2 times more.
+        groups_weighted = np.concatenate(
+            [
+                np.full(n_samples_per_cv_group, 0),
+                np.full(n_samples_per_cv_group, 1),
+                np.full(n_samples_per_cv_group, 2),
+            ]
+        )
+        splits_weighted = list(LeaveOneGroupOut().split(X, groups=groups_weighted))
+        kw_weighted.update({"Cs": 100, "cv": splits_weighted})
+
+        groups_repeated = np.repeat(groups_weighted, sw.astype(int), axis=0)
+        splits_repeated = list(
+            LeaveOneGroupOut().split(X_repeated, groups=groups_repeated)
+        )
+        kw_repeated.update({"Cs": 100, "cv": splits_repeated})
+
+    clf_sw_weighted = LR(solver=solver, **kw_weighted)
+    clf_sw_repeated = LR(solver=solver, **kw_repeated)
+
+    if solver == "lbfgs":
+        # lbfgs has convergence issues on the data but this should not impact
+        # the quality of the results.
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", ConvergenceWarning)
+            clf_sw_weighted.fit(X, y, sample_weight=sw)
+            clf_sw_repeated.fit(X_repeated, y_repeated)
+
+    else:
+        clf_sw_weighted.fit(X, y, sample_weight=sw)
+        clf_sw_repeated.fit(X_repeated, y_repeated)
+
+    if problem == "cv":
+        assert_allclose(clf_sw_weighted.scores_[1], clf_sw_repeated.scores_[1])
+    assert_allclose(clf_sw_weighted.coef_, clf_sw_repeated.coef_, atol=1e-5)
+
+
+@pytest.mark.parametrize(
+    "solver", ("lbfgs", "newton-cg", "newton-cholesky", "sag", "saga")
+)
+def test_logistic_regression_solver_class_weights(solver, global_random_seed):
+    # Test that passing class_weight as [1, 2] is the same as
+    # passing class weight = [1,1] but adjusting sample weights
+    # to be 2 for all instances of class 1.
+
+    X, y = make_classification(
+        n_samples=300,
+        n_features=5,
+        n_informative=3,
+        n_classes=2,
+        random_state=global_random_seed,
+    )
+
+    sample_weight = y + 1
+
+    kw_weighted = {
+        "random_state": global_random_seed,
+        "fit_intercept": False,
+        "max_iter": 100_000,
+        "tol": 1e-8,
+    }
+    clf_cw_12 = LogisticRegression(
+        solver=solver, class_weight={0: 1, 1: 2}, **kw_weighted
+    )
+    clf_cw_12.fit(X, y)
+    clf_sw_12 = LogisticRegression(solver=solver, **kw_weighted)
+    clf_sw_12.fit(X, y, sample_weight=sample_weight)
+    assert_allclose(clf_cw_12.coef_, clf_sw_12.coef_, atol=1e-6)
 
+
+def test_sample_and_class_weight_equivalence_liblinear(global_random_seed):
     # Test the above for l1 penalty and l2 penalty with dual=True.
     # since the patched liblinear code is different.
+
+    X, y = make_classification(
+        n_samples=300,
+        n_features=5,
+        n_informative=3,
+        n_classes=2,
+        random_state=global_random_seed,
+    )
+
+    sample_weight = y + 1
+
     clf_cw = LogisticRegression(
-        solver="liblinear", fit_intercept=False, class_weight={0: 1, 1: 2},
-        penalty="l1", tol=1e-5, random_state=42, multi_class='ovr')
+        solver="liblinear",
+        fit_intercept=False,
+        class_weight={0: 1, 1: 2},
+        penalty="l1",
+        max_iter=10_000,
+        tol=1e-12,
+        random_state=global_random_seed,
+    )
     clf_cw.fit(X, y)
     clf_sw = LogisticRegression(
-        solver="liblinear", fit_intercept=False, penalty="l1", tol=1e-5,
-        random_state=42, multi_class='ovr')
+        solver="liblinear",
+        fit_intercept=False,
+        penalty="l1",
+        max_iter=10_000,
+        tol=1e-12,
+        random_state=global_random_seed,
+    )
     clf_sw.fit(X, y, sample_weight)
-    assert_array_almost_equal(clf_cw.coef_, clf_sw.coef_, decimal=4)
+    assert_allclose(clf_cw.coef_, clf_sw.coef_, atol=1e-10)
 
     clf_cw = LogisticRegression(
-        solver="liblinear", fit_intercept=False, class_weight={0: 1, 1: 2},
-        penalty="l2", dual=True, random_state=42, multi_class='ovr')
+        solver="liblinear",
+        fit_intercept=False,
+        class_weight={0: 1, 1: 2},
+        penalty="l2",
+        max_iter=10_000,
+        tol=1e-12,
+        dual=True,
+        random_state=global_random_seed,
+    )
     clf_cw.fit(X, y)
     clf_sw = LogisticRegression(
-        solver="liblinear", fit_intercept=False, penalty="l2", dual=True,
-        random_state=42, multi_class='ovr')
+        solver="liblinear",
+        fit_intercept=False,
+        penalty="l2",
+        max_iter=10_000,
+        tol=1e-12,
+        dual=True,
+        random_state=global_random_seed,
+    )
     clf_sw.fit(X, y, sample_weight)
-    assert_array_almost_equal(clf_cw.coef_, clf_sw.coef_, decimal=4)
+    assert_allclose(clf_cw.coef_, clf_sw.coef_, atol=1e-10)
 
 
 def _compute_class_weight_dictionary(y):
@@ -910,33 +1029,40 @@ def _compute_class_weight_dictionary(y):
     return class_weight_dict
 
 
-def test_logistic_regression_class_weights():
+@pytest.mark.parametrize("csr_container", [lambda x: x] + CSR_CONTAINERS)
+def test_logistic_regression_class_weights(csr_container):
+    # Scale data to avoid convergence warnings with the lbfgs solver
+    X_iris = scale(iris.data)
     # Multinomial case: remove 90% of class 0
-    X = iris.data[45:, :]
+    X = X_iris[45:, :]
+    X = csr_container(X)
     y = iris.target[45:]
-    solvers = ("lbfgs", "newton-cg")
     class_weight_dict = _compute_class_weight_dictionary(y)
 
-    for solver in solvers:
-        clf1 = LogisticRegression(solver=solver, multi_class="multinomial",
-                                  class_weight="balanced")
-        clf2 = LogisticRegression(solver=solver, multi_class="multinomial",
-                                  class_weight=class_weight_dict)
+    for solver in set(SOLVERS) - set(["liblinear", "newton-cholesky"]):
+        params = dict(solver=solver, max_iter=1000)
+        clf1 = LogisticRegression(class_weight="balanced", **params)
+        clf2 = LogisticRegression(class_weight=class_weight_dict, **params)
         clf1.fit(X, y)
         clf2.fit(X, y)
-        assert_array_almost_equal(clf1.coef_, clf2.coef_, decimal=4)
+        assert len(clf1.classes_) == 3
+        assert_allclose(clf1.coef_, clf2.coef_, rtol=1e-4)
+        # Same as appropriate sample_weight.
+        sw = np.ones(X.shape[0])
+        for c in clf1.classes_:
+            sw[y == c] *= class_weight_dict[c]
+        clf3 = LogisticRegression(**params).fit(X, y, sample_weight=sw)
+        assert_allclose(clf3.coef_, clf2.coef_, rtol=1e-4)
 
     # Binary case: remove 90% of class 0 and 100% of class 2
-    X = iris.data[45:100, :]
+    X = X_iris[45:100, :]
     y = iris.target[45:100]
-    solvers = ("lbfgs", "newton-cg", "liblinear")
     class_weight_dict = _compute_class_weight_dictionary(y)
 
-    for solver in solvers:
-        clf1 = LogisticRegression(solver=solver, multi_class="ovr",
-                                  class_weight="balanced")
-        clf2 = LogisticRegression(solver=solver, multi_class="ovr",
-                                  class_weight=class_weight_dict)
+    for solver in SOLVERS:
+        params = dict(solver=solver, max_iter=1000)
+        clf1 = LogisticRegression(class_weight="balanced", **params)
+        clf2 = LogisticRegression(class_weight=class_weight_dict, **params)
         clf1.fit(X, y)
         clf2.fit(X, y)
         assert_array_almost_equal(clf1.coef_, clf2.coef_, decimal=6)
@@ -947,79 +1073,58 @@ def test_logistic_regression_multinomial():
 
     # Some basic attributes of Logistic Regression
     n_samples, n_features, n_classes = 50, 20, 3
-    X, y = make_classification(n_samples=n_samples,
-                               n_features=n_features,
-                               n_informative=10,
-                               n_classes=n_classes, random_state=0)
+    X, y = make_classification(
+        n_samples=n_samples,
+        n_features=n_features,
+        n_informative=10,
+        n_classes=n_classes,
+        random_state=0,
+    )
 
     X = StandardScaler(with_mean=False).fit_transform(X)
 
     # 'lbfgs' is used as a referenced
-    solver = 'lbfgs'
-    ref_i = LogisticRegression(solver=solver, multi_class='multinomial')
-    ref_w = LogisticRegression(solver=solver, multi_class='multinomial',
-                               fit_intercept=False)
+    solver = "lbfgs"
+    ref_i = LogisticRegression(solver=solver, tol=1e-6)
+    ref_w = LogisticRegression(solver=solver, fit_intercept=False, tol=1e-6)
     ref_i.fit(X, y)
     ref_w.fit(X, y)
     assert ref_i.coef_.shape == (n_classes, n_features)
     assert ref_w.coef_.shape == (n_classes, n_features)
-    for solver in ['sag', 'saga', 'newton-cg']:
-        clf_i = LogisticRegression(solver=solver, multi_class='multinomial',
-                                   random_state=42, max_iter=2000, tol=1e-7,
-                                   )
-        clf_w = LogisticRegression(solver=solver, multi_class='multinomial',
-                                   random_state=42, max_iter=2000, tol=1e-7,
-                                   fit_intercept=False)
+    for solver in ["sag", "saga", "newton-cg"]:
+        clf_i = LogisticRegression(
+            solver=solver,
+            random_state=42,
+            max_iter=2000,
+            tol=1e-7,
+        )
+        clf_w = LogisticRegression(
+            solver=solver,
+            random_state=42,
+            max_iter=2000,
+            tol=1e-7,
+            fit_intercept=False,
+        )
         clf_i.fit(X, y)
         clf_w.fit(X, y)
         assert clf_i.coef_.shape == (n_classes, n_features)
         assert clf_w.coef_.shape == (n_classes, n_features)
 
         # Compare solutions between lbfgs and the other solvers
-        assert_allclose(ref_i.coef_, clf_i.coef_, rtol=1e-2)
+        assert_allclose(ref_i.coef_, clf_i.coef_, rtol=1e-3)
         assert_allclose(ref_w.coef_, clf_w.coef_, rtol=1e-2)
-        assert_allclose(ref_i.intercept_, clf_i.intercept_, rtol=1e-2)
+        assert_allclose(ref_i.intercept_, clf_i.intercept_, rtol=1e-3)
 
     # Test that the path give almost the same results. However since in this
     # case we take the average of the coefs after fitting across all the
     # folds, it need not be exactly the same.
-    for solver in ['lbfgs', 'newton-cg', 'sag', 'saga']:
-        clf_path = LogisticRegressionCV(solver=solver, max_iter=2000, tol=1e-6,
-                                        multi_class='multinomial', Cs=[1.])
+    for solver in ["lbfgs", "newton-cg", "sag", "saga"]:
+        clf_path = LogisticRegressionCV(
+            solver=solver, max_iter=2000, tol=1e-6, Cs=[1.0]
+        )
         clf_path.fit(X, y)
-        assert_allclose(clf_path.coef_, ref_i.coef_, rtol=2e-2)
-        assert_allclose(clf_path.intercept_, ref_i.intercept_, rtol=2e-2)
-
-
-def test_multinomial_grad_hess():
-    rng = np.random.RandomState(0)
-    n_samples, n_features, n_classes = 100, 5, 3
-    X = rng.randn(n_samples, n_features)
-    w = rng.rand(n_classes, n_features)
-    Y = np.zeros((n_samples, n_classes))
-    ind = np.argmax(np.dot(X, w.T), axis=1)
-    Y[range(0, n_samples), ind] = 1
-    w = w.ravel()
-    sample_weights = np.ones(X.shape[0])
-    grad, hessp = _multinomial_grad_hess(w, X, Y, alpha=1.,
-                                         sample_weight=sample_weights)
-    # extract first column of hessian matrix
-    vec = np.zeros(n_features * n_classes)
-    vec[0] = 1
-    hess_col = hessp(vec)
-
-    # Estimate hessian using least squares as done in
-    # test_logistic_grad_hess
-    e = 1e-3
-    d_x = np.linspace(-e, e, 30)
-    d_grad = np.array([
-        _multinomial_grad_hess(w + t * vec, X, Y, alpha=1.,
-                               sample_weight=sample_weights)[0]
-        for t in d_x
-    ])
-    d_grad -= d_grad.mean(axis=0)
-    approx_hess_col = linalg.lstsq(d_x[:, np.newaxis], d_grad)[0].ravel()
-    assert_array_almost_equal(hess_col, approx_hess_col)
+        assert_allclose(clf_path.coef_, ref_i.coef_, rtol=1e-2)
+        assert_allclose(clf_path.intercept_, ref_i.intercept_, rtol=1e-2)
 
 
 def test_liblinear_decision_function_zero():
@@ -1029,8 +1134,7 @@ def test_liblinear_decision_function_zero():
     # See Issue: https://github.com/scikit-learn/scikit-learn/issues/3600
     # and the PR https://github.com/scikit-learn/scikit-learn/pull/3623
     X, y = make_classification(n_samples=5, n_features=5, random_state=0)
-    clf = LogisticRegression(fit_intercept=False, solver='liblinear',
-                             multi_class='ovr')
+    clf = LogisticRegression(fit_intercept=False, solver="liblinear")
     clf.fit(X, y)
 
     # Dummy data such that the decision function becomes zero.
@@ -1038,33 +1142,22 @@ def test_liblinear_decision_function_zero():
     assert_array_equal(clf.predict(X), np.zeros(5))
 
 
-def test_liblinear_logregcv_sparse():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_liblinear_logregcv_sparse(csr_container):
     # Test LogRegCV with solver='liblinear' works for sparse matrices
 
     X, y = make_classification(n_samples=10, n_features=5, random_state=0)
-    clf = LogisticRegressionCV(solver='liblinear', multi_class='ovr')
-    clf.fit(sparse.csr_matrix(X), y)
+    clf = LogisticRegressionCV(solver="liblinear")
+    clf.fit(csr_container(X), y)
 
 
-def test_saga_sparse():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_saga_sparse(csr_container):
     # Test LogRegCV with solver='liblinear' works for sparse matrices
 
     X, y = make_classification(n_samples=10, n_features=5, random_state=0)
-    clf = LogisticRegressionCV(solver='saga')
-    clf.fit(sparse.csr_matrix(X), y)
-
-
-def test_logreg_intercept_scaling():
-    # Test that the right error message is thrown when intercept_scaling <= 0
-
-    for i in [-1, 0]:
-        clf = LogisticRegression(intercept_scaling=i, solver='liblinear',
-                                 multi_class='ovr')
-        msg = ('Intercept scaling is %r but needs to be greater than 0.'
-               ' To disable fitting an intercept,'
-               ' set fit_intercept=False.' % clf.intercept_scaling)
-        with pytest.raises(ValueError, match=msg):
-            clf.fit(X, Y1)
+    clf = LogisticRegressionCV(solver="saga", tol=1e-2)
+    clf.fit(csr_container(X), y)
 
 
 def test_logreg_intercept_scaling_zero():
@@ -1072,7 +1165,7 @@ def test_logreg_intercept_scaling_zero():
 
     clf = LogisticRegression(fit_intercept=False)
     clf.fit(X, Y1)
-    assert clf.intercept_ == 0.
+    assert clf.intercept_ == 0.0
 
 
 def test_logreg_l1():
@@ -1081,19 +1174,27 @@ def test_logreg_l1():
     # the two models at convergence.
     rng = np.random.RandomState(42)
     n_samples = 50
-    X, y = make_classification(n_samples=n_samples, n_features=20,
-                               random_state=0)
+    X, y = make_classification(n_samples=n_samples, n_features=20, random_state=0)
     X_noise = rng.normal(size=(n_samples, 3))
     X_constant = np.ones(shape=(n_samples, 2))
     X = np.concatenate((X, X_noise, X_constant), axis=1)
-    lr_liblinear = LogisticRegression(penalty="l1", C=1.0, solver='liblinear',
-                                      fit_intercept=False, multi_class='ovr',
-                                      tol=1e-10)
+    lr_liblinear = LogisticRegression(
+        penalty="l1",
+        C=1.0,
+        solver="liblinear",
+        fit_intercept=False,
+        tol=1e-10,
+    )
     lr_liblinear.fit(X, y)
 
-    lr_saga = LogisticRegression(penalty="l1", C=1.0, solver='saga',
-                                 fit_intercept=False, multi_class='ovr',
-                                 max_iter=1000, tol=1e-10)
+    lr_saga = LogisticRegression(
+        penalty="l1",
+        C=1.0,
+        solver="saga",
+        fit_intercept=False,
+        max_iter=1000,
+        tol=1e-10,
+    )
     lr_saga.fit(X, y)
     assert_array_almost_equal(lr_saga.coef_, lr_liblinear.coef_)
 
@@ -1103,28 +1204,37 @@ def test_logreg_l1():
     assert_array_almost_equal(lr_saga.coef_[0, -5:], np.zeros(5))
 
 
-def test_logreg_l1_sparse_data():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_logreg_l1_sparse_data(csr_container):
     # Because liblinear penalizes the intercept and saga does not, we do not
     # fit the intercept to make it possible to compare the coefficients of
     # the two models at convergence.
     rng = np.random.RandomState(42)
     n_samples = 50
-    X, y = make_classification(n_samples=n_samples, n_features=20,
-                               random_state=0)
+    X, y = make_classification(n_samples=n_samples, n_features=20, random_state=0)
     X_noise = rng.normal(scale=0.1, size=(n_samples, 3))
     X_constant = np.zeros(shape=(n_samples, 2))
     X = np.concatenate((X, X_noise, X_constant), axis=1)
     X[X < 1] = 0
-    X = sparse.csr_matrix(X)
-
-    lr_liblinear = LogisticRegression(penalty="l1", C=1.0, solver='liblinear',
-                                      fit_intercept=False, multi_class='ovr',
-                                      tol=1e-10)
+    X = csr_container(X)
+
+    lr_liblinear = LogisticRegression(
+        penalty="l1",
+        C=1.0,
+        solver="liblinear",
+        fit_intercept=False,
+        tol=1e-10,
+    )
     lr_liblinear.fit(X, y)
 
-    lr_saga = LogisticRegression(penalty="l1", C=1.0, solver='saga',
-                                 fit_intercept=False, multi_class='ovr',
-                                 max_iter=1000, tol=1e-10)
+    lr_saga = LogisticRegression(
+        penalty="l1",
+        C=1.0,
+        solver="saga",
+        fit_intercept=False,
+        max_iter=1000,
+        tol=1e-10,
+    )
     lr_saga.fit(X, y)
     assert_array_almost_equal(lr_saga.coef_, lr_liblinear.coef_)
     # Noise and constant features should be regularized to zero by the l1
@@ -1133,9 +1243,14 @@ def test_logreg_l1_sparse_data():
     assert_array_almost_equal(lr_saga.coef_[0, -5:], np.zeros(5))
 
     # Check that solving on the sparse and dense data yield the same results
-    lr_saga_dense = LogisticRegression(penalty="l1", C=1.0, solver='saga',
-                                       fit_intercept=False, multi_class='ovr',
-                                       max_iter=1000, tol=1e-10)
+    lr_saga_dense = LogisticRegression(
+        penalty="l1",
+        C=1.0,
+        solver="saga",
+        fit_intercept=False,
+        max_iter=1000,
+        tol=1e-10,
+    )
     lr_saga_dense.fit(X.toarray(), y)
     assert_array_almost_equal(lr_saga.coef_, lr_saga_dense.coef_)
 
@@ -1151,10 +1266,9 @@ def test_logistic_regression_cv_refit(random_seed, penalty):
     # logistic regression loss is convex, we should still recover exactly
     # the same solution as long as the stopping criterion is strict enough (and
     # that there are no exactly duplicated features when penalty='l1').
-    X, y = make_classification(n_samples=100, n_features=20,
-                               random_state=random_seed)
+    X, y = make_classification(n_samples=100, n_features=20, random_state=random_seed)
     common_params = dict(
-        solver='saga',
+        solver="saga",
         penalty=penalty,
         random_state=random_seed,
         max_iter=1000,
@@ -1168,15 +1282,16 @@ def test_logistic_regression_cv_refit(random_seed, penalty):
 
 
 def test_logreg_predict_proba_multinomial():
-    X, y = make_classification(n_samples=10, n_features=20, random_state=0,
-                               n_classes=3, n_informative=10)
+    X, y = make_classification(
+        n_samples=10, n_features=20, random_state=0, n_classes=3, n_informative=10
+    )
 
     # Predicted probabilities using the true-entropy loss should give a
     # smaller loss than those using the ovr method.
-    clf_multi = LogisticRegression(multi_class="multinomial", solver="lbfgs")
+    clf_multi = LogisticRegression(solver="lbfgs")
     clf_multi.fit(X, y)
     clf_multi_loss = log_loss(y, clf_multi.predict_proba(X))
-    clf_ovr = LogisticRegression(multi_class="ovr", solver="lbfgs")
+    clf_ovr = OneVsRestClassifier(LogisticRegression(solver="lbfgs"))
     clf_ovr.fit(X, y)
     clf_ovr_loss = log_loss(y, clf_ovr.predict_proba(X))
     assert clf_ovr_loss > clf_multi_loss
@@ -1188,100 +1303,123 @@ def test_logreg_predict_proba_multinomial():
     assert clf_wrong_loss > clf_multi_loss
 
 
+# TODO(1.8): remove filterwarnings after the deprecation of multi_class
+@pytest.mark.filterwarnings("ignore:.*'multi_class' was deprecated.*:FutureWarning")
 @pytest.mark.parametrize("max_iter", np.arange(1, 5))
-@pytest.mark.parametrize("multi_class", ['ovr', 'multinomial'])
+@pytest.mark.parametrize("multi_class", ["ovr", "multinomial"])
 @pytest.mark.parametrize(
     "solver, message",
-    [("newton-cg", "newton-cg failed to converge. Increase the "
-                   "number of iterations."),
-     ("liblinear", "Liblinear failed to converge, increase the "
-                   "number of iterations."),
-     ("sag", "The max_iter was reached which means the "
-             "coef_ did not converge"),
-     ("saga", "The max_iter was reached which means the "
-              "coef_ did not converge"),
-     ("lbfgs", "lbfgs failed to converge")])
+    [
+        (
+            "newton-cg",
+            "newton-cg failed to converge.* Increase the number of iterations.",
+        ),
+        (
+            "liblinear",
+            "Liblinear failed to converge, increase the number of iterations.",
+        ),
+        ("sag", "The max_iter was reached which means the coef_ did not converge"),
+        ("saga", "The max_iter was reached which means the coef_ did not converge"),
+        ("lbfgs", "lbfgs failed to converge"),
+        ("newton-cholesky", "Newton solver did not converge after [0-9]* iterations"),
+    ],
+)
 def test_max_iter(max_iter, multi_class, solver, message):
     # Test that the maximum number of iteration is reached
     X, y_bin = iris.data, iris.target.copy()
     y_bin[y_bin == 2] = 0
 
-    if solver == 'liblinear' and multi_class == 'multinomial':
-        pytest.skip("'multinomial' is unavailable when solver='liblinear'")
-
-    lr = LogisticRegression(max_iter=max_iter, tol=1e-15,
-                            multi_class=multi_class,
-                            random_state=0, solver=solver)
+    if solver in ("liblinear",) and multi_class == "multinomial":
+        pytest.skip("'multinomial' is not supported by liblinear")
+    if solver == "newton-cholesky" and max_iter > 1:
+        pytest.skip("solver newton-cholesky might converge very fast")
+
+    lr = LogisticRegression(
+        max_iter=max_iter,
+        tol=1e-15,
+        multi_class=multi_class,
+        random_state=0,
+        solver=solver,
+    )
     with pytest.warns(ConvergenceWarning, match=message):
         lr.fit(X, y_bin)
 
     assert lr.n_iter_[0] == max_iter
 
 
-@pytest.mark.parametrize('solver',
-                         ['newton-cg', 'liblinear', 'sag', 'saga', 'lbfgs'])
+# TODO(1.8): remove filterwarnings after the deprecation of multi_class
+@pytest.mark.filterwarnings("ignore:.*'multi_class' was deprecated.*:FutureWarning")
+@pytest.mark.filterwarnings(
+    "ignore:.*'liblinear' solver for multiclass classification is deprecated.*"
+)
+@pytest.mark.parametrize("solver", SOLVERS)
 def test_n_iter(solver):
     # Test that self.n_iter_ has the correct format.
     X, y = iris.data, iris.target
+    if solver == "lbfgs":
+        # lbfgs requires scaling to avoid convergence warnings
+        X = scale(X)
+
+    n_classes = np.unique(y).shape[0]
+    assert n_classes == 3
 
+    # Also generate a binary classification sub-problem.
     y_bin = y.copy()
     y_bin[y_bin == 2] = 0
 
     n_Cs = 4
     n_cv_fold = 2
 
+    # Binary classification case
+    clf = LogisticRegression(tol=1e-2, C=1.0, solver=solver, random_state=42)
+    clf.fit(X, y_bin)
+    assert clf.n_iter_.shape == (1,)
+
+    clf_cv = LogisticRegressionCV(
+        tol=1e-2, solver=solver, Cs=n_Cs, cv=n_cv_fold, random_state=42
+    )
+    clf_cv.fit(X, y_bin)
+    assert clf_cv.n_iter_.shape == (1, n_cv_fold, n_Cs)
+
     # OvR case
-    n_classes = 1 if solver == 'liblinear' else np.unique(y).shape[0]
-    clf = LogisticRegression(tol=1e-2, multi_class='ovr',
-                             solver=solver, C=1.,
-                             random_state=42, max_iter=100)
-    clf.fit(X, y)
+    clf.set_params(multi_class="ovr").fit(X, y)
     assert clf.n_iter_.shape == (n_classes,)
 
-    n_classes = np.unique(y).shape[0]
-    clf = LogisticRegressionCV(tol=1e-2, multi_class='ovr',
-                               solver=solver, Cs=n_Cs, cv=n_cv_fold,
-                               random_state=42, max_iter=100)
-    clf.fit(X, y)
-    assert clf.n_iter_.shape == (n_classes, n_cv_fold, n_Cs)
-    clf.fit(X, y_bin)
-    assert clf.n_iter_.shape == (1, n_cv_fold, n_Cs)
+    clf_cv.set_params(multi_class="ovr").fit(X, y)
+    assert clf_cv.n_iter_.shape == (n_classes, n_cv_fold, n_Cs)
 
     # multinomial case
-    n_classes = 1
-    if solver in ('liblinear', 'sag', 'saga'):
+    if solver in ("liblinear",):
+        # This solver only supports one-vs-rest multiclass classification.
         return
 
-    clf = LogisticRegression(tol=1e-2, multi_class='multinomial',
-                             solver=solver, C=1.,
-                             random_state=42, max_iter=100)
-    clf.fit(X, y)
-    assert clf.n_iter_.shape == (n_classes,)
+    # When using the multinomial objective function, there is a single
+    # optimization problem to solve for all classes at once:
+    clf.set_params(multi_class="multinomial").fit(X, y)
+    assert clf.n_iter_.shape == (1,)
 
-    clf = LogisticRegressionCV(tol=1e-2, multi_class='multinomial',
-                               solver=solver, Cs=n_Cs, cv=n_cv_fold,
-                               random_state=42, max_iter=100)
-    clf.fit(X, y)
-    assert clf.n_iter_.shape == (n_classes, n_cv_fold, n_Cs)
-    clf.fit(X, y_bin)
-    assert clf.n_iter_.shape == (1, n_cv_fold, n_Cs)
+    clf_cv.set_params(multi_class="multinomial").fit(X, y)
+    assert clf_cv.n_iter_.shape == (1, n_cv_fold, n_Cs)
 
 
-@pytest.mark.parametrize('solver', ('newton-cg', 'sag', 'saga', 'lbfgs'))
-@pytest.mark.parametrize('warm_start', (True, False))
-@pytest.mark.parametrize('fit_intercept', (True, False))
-@pytest.mark.parametrize('multi_class', ['ovr', 'multinomial'])
-def test_warm_start(solver, warm_start, fit_intercept, multi_class):
+@pytest.mark.parametrize(
+    "solver", sorted(set(SOLVERS) - set(["liblinear", "newton-cholesky"]))
+)
+@pytest.mark.parametrize("warm_start", (True, False))
+@pytest.mark.parametrize("fit_intercept", (True, False))
+def test_warm_start(solver, warm_start, fit_intercept):
     # A 1-iteration second fit on same data should give almost same result
     # with warm starting, and quite different result without warm starting.
     # Warm starting does not work with liblinear solver.
     X, y = iris.data, iris.target
 
-    clf = LogisticRegression(tol=1e-4, multi_class=multi_class,
-                             warm_start=warm_start,
-                             solver=solver,
-                             random_state=42, max_iter=100,
-                             fit_intercept=fit_intercept)
+    clf = LogisticRegression(
+        tol=1e-4,
+        warm_start=warm_start,
+        solver=solver,
+        random_state=42,
+        fit_intercept=fit_intercept,
+    )
     with ignore_warnings(category=ConvergenceWarning):
         clf.fit(X, y)
         coef_1 = clf.coef_
@@ -1289,17 +1427,18 @@ def test_warm_start(solver, warm_start, fit_intercept, multi_class):
         clf.max_iter = 1
         clf.fit(X, y)
     cum_diff = np.sum(np.abs(coef_1 - clf.coef_))
-    msg = ("Warm starting issue with %s solver in %s mode "
-           "with fit_intercept=%s and warm_start=%s"
-           % (solver, multi_class, str(fit_intercept),
-              str(warm_start)))
+    msg = (
+        f"Warm starting issue with solver {solver}"
+        f"with {fit_intercept=} and {warm_start=}"
+    )
     if warm_start:
         assert 2.0 > cum_diff, msg
     else:
         assert cum_diff > 2.0, msg
 
 
-def test_saga_vs_liblinear():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_saga_vs_liblinear(csr_container):
     iris = load_iris()
     X, y = iris.data, iris.target
     X = np.concatenate([X] * 3)
@@ -1308,30 +1447,35 @@ def test_saga_vs_liblinear():
     X_bin = X[y <= 1]
     y_bin = y[y <= 1] * 2 - 1
 
-    X_sparse, y_sparse = make_classification(n_samples=50, n_features=20,
-                                             random_state=0)
-    X_sparse = sparse.csr_matrix(X_sparse)
+    X_sparse, y_sparse = make_classification(
+        n_samples=50, n_features=20, random_state=0
+    )
+    X_sparse = csr_container(X_sparse)
 
-    for (X, y) in ((X_bin, y_bin), (X_sparse, y_sparse)):
-        for penalty in ['l1', 'l2']:
+    for X, y in ((X_bin, y_bin), (X_sparse, y_sparse)):
+        for penalty in ["l1", "l2"]:
             n_samples = X.shape[0]
             # alpha=1e-3 is time consuming
             for alpha in np.logspace(-1, 1, 3):
                 saga = LogisticRegression(
-                    C=1. / (n_samples * alpha),
-                    solver='saga',
-                    multi_class='ovr',
+                    C=1.0 / (n_samples * alpha),
+                    solver="saga",
                     max_iter=200,
                     fit_intercept=False,
-                    penalty=penalty, random_state=0, tol=1e-24)
+                    penalty=penalty,
+                    random_state=0,
+                    tol=1e-6,
+                )
 
                 liblinear = LogisticRegression(
-                    C=1. / (n_samples * alpha),
-                    solver='liblinear',
-                    multi_class='ovr',
+                    C=1.0 / (n_samples * alpha),
+                    solver="liblinear",
                     max_iter=200,
                     fit_intercept=False,
-                    penalty=penalty, random_state=0, tol=1e-24)
+                    penalty=penalty,
+                    random_state=0,
+                    tol=1e-6,
+                )
 
                 saga.fit(X, y)
                 liblinear.fit(X, y)
@@ -1339,29 +1483,38 @@ def test_saga_vs_liblinear():
                 assert_array_almost_equal(saga.coef_, liblinear.coef_, 3)
 
 
-@pytest.mark.parametrize('multi_class', ['ovr', 'multinomial'])
-@pytest.mark.parametrize('solver', ['newton-cg', 'liblinear', 'saga'])
-@pytest.mark.parametrize('fit_intercept', [False, True])
-def test_dtype_match(solver, multi_class, fit_intercept):
+# TODO(1.8): remove filterwarnings after the deprecation of multi_class
+@pytest.mark.filterwarnings("ignore:.*'multi_class' was deprecated.*:FutureWarning")
+@pytest.mark.parametrize("multi_class", ["ovr", "multinomial"])
+@pytest.mark.parametrize(
+    "solver", ["liblinear", "newton-cg", "newton-cholesky", "saga"]
+)
+@pytest.mark.parametrize("fit_intercept", [False, True])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_dtype_match(solver, multi_class, fit_intercept, csr_container):
     # Test that np.float32 input data is not cast to np.float64 when possible
     # and that the output is approximately the same no matter the input format.
 
-    if solver == 'liblinear' and multi_class == 'multinomial':
-        pytest.skip('liblinear does not support multinomial logistic')
+    if solver == "liblinear" and multi_class == "multinomial":
+        pytest.skip(f"Solver={solver} does not support multinomial logistic.")
 
-    out32_type = np.float64 if solver == 'liblinear' else np.float32
+    out32_type = np.float64 if solver == "liblinear" else np.float32
 
     X_32 = np.array(X).astype(np.float32)
     y_32 = np.array(Y1).astype(np.float32)
     X_64 = np.array(X).astype(np.float64)
     y_64 = np.array(Y1).astype(np.float64)
-    X_sparse_32 = sp.csr_matrix(X, dtype=np.float32)
-    X_sparse_64 = sp.csr_matrix(X, dtype=np.float64)
+    X_sparse_32 = csr_container(X, dtype=np.float32)
+    X_sparse_64 = csr_container(X, dtype=np.float64)
     solver_tol = 5e-4
 
     lr_templ = LogisticRegression(
-        solver=solver, multi_class=multi_class,
-        random_state=42, tol=solver_tol, fit_intercept=fit_intercept)
+        solver=solver,
+        multi_class=multi_class,
+        random_state=42,
+        tol=solver_tol,
+        fit_intercept=fit_intercept,
+    )
 
     # Check 32-bit type consistency
     lr_32 = clone(lr_templ)
@@ -1394,14 +1547,14 @@ def test_dtype_match(solver, multi_class, fit_intercept):
 
     # factor of 2 to get the ball diameter
     atol = 2 * 1.72 * solver_tol
-    if os.name == 'nt' and _IS_32BIT:
+    if os.name == "nt" and _IS_32BIT:
         # FIXME
         atol = 1e-2
 
     # Check accuracy consistency
     assert_allclose(lr_32.coef_, lr_64.coef_.astype(np.float32), atol=atol)
 
-    if solver == 'saga' and fit_intercept:
+    if solver == "saga" and fit_intercept:
         # FIXME: SAGA on sparse data fits the intercept inaccurately with the
         # default tol and max_iter parameters.
         atol = 1e-1
@@ -1417,12 +1570,8 @@ def test_warm_start_converge_LR():
     rng = np.random.RandomState(0)
     X = np.concatenate((rng.randn(100, 2) + [1, 1], rng.randn(100, 2)))
     y = np.array([1] * 100 + [-1] * 100)
-    lr_no_ws = LogisticRegression(multi_class='multinomial',
-                                  solver='sag', warm_start=False,
-                                  random_state=0)
-    lr_ws = LogisticRegression(multi_class='multinomial',
-                               solver='sag', warm_start=True,
-                               random_state=0)
+    lr_no_ws = LogisticRegression(solver="sag", warm_start=False, random_state=0)
+    lr_ws = LogisticRegression(solver="sag", warm_start=True, random_state=0)
 
     lr_no_ws_loss = log_loss(y, lr_no_ws.fit(X, y).predict_proba(X))
     for i in range(5):
@@ -1436,42 +1585,54 @@ def test_elastic_net_coeffs():
     # with saga solver (l1_ratio different from 0 or 1)
     X, y = make_classification(random_state=0)
 
-    C = 2.
-    l1_ratio = .5
+    C = 2.0
+    l1_ratio = 0.5
     coeffs = list()
-    for penalty in ('elasticnet', 'l1', 'l2'):
-        lr = LogisticRegression(penalty=penalty, C=C, solver='saga',
-                                random_state=0, l1_ratio=l1_ratio)
+    for penalty, ratio in (("elasticnet", l1_ratio), ("l1", None), ("l2", None)):
+        lr = LogisticRegression(
+            penalty=penalty,
+            C=C,
+            solver="saga",
+            random_state=0,
+            l1_ratio=ratio,
+            tol=1e-3,
+            max_iter=200,
+        )
         lr.fit(X, y)
         coeffs.append(lr.coef_)
 
     elastic_net_coeffs, l1_coeffs, l2_coeffs = coeffs
     # make sure coeffs differ by at least .1
-    assert not np.allclose(elastic_net_coeffs, l1_coeffs, rtol=0, atol=.1)
-    assert not np.allclose(elastic_net_coeffs, l2_coeffs, rtol=0, atol=.1)
-    assert not np.allclose(l2_coeffs, l1_coeffs, rtol=0, atol=.1)
+    assert not np.allclose(elastic_net_coeffs, l1_coeffs, rtol=0, atol=0.1)
+    assert not np.allclose(elastic_net_coeffs, l2_coeffs, rtol=0, atol=0.1)
+    assert not np.allclose(l2_coeffs, l1_coeffs, rtol=0, atol=0.1)
 
 
-@pytest.mark.parametrize('C', [.001, .1, 1, 10, 100, 1000, 1e6])
-@pytest.mark.parametrize('penalty, l1_ratio',
-                         [('l1', 1),
-                          ('l2', 0)])
+@pytest.mark.parametrize("C", [0.001, 0.1, 1, 10, 100, 1000, 1e6])
+@pytest.mark.parametrize("penalty, l1_ratio", [("l1", 1), ("l2", 0)])
 def test_elastic_net_l1_l2_equivalence(C, penalty, l1_ratio):
     # Make sure elasticnet is equivalent to l1 when l1_ratio=1 and to l2 when
     # l1_ratio=0.
     X, y = make_classification(random_state=0)
 
-    lr_enet = LogisticRegression(penalty='elasticnet', C=C, l1_ratio=l1_ratio,
-                                 solver='saga', random_state=0)
-    lr_expected = LogisticRegression(penalty=penalty, C=C, solver='saga',
-                                     random_state=0)
+    lr_enet = LogisticRegression(
+        penalty="elasticnet",
+        C=C,
+        l1_ratio=l1_ratio,
+        solver="saga",
+        random_state=0,
+        tol=1e-2,
+    )
+    lr_expected = LogisticRegression(
+        penalty=penalty, C=C, solver="saga", random_state=0, tol=1e-2
+    )
     lr_enet.fit(X, y)
     lr_expected.fit(X, y)
 
     assert_array_almost_equal(lr_enet.coef_, lr_expected.coef_)
 
 
-@pytest.mark.parametrize('C', [.001, 1, 100, 1e6])
+@pytest.mark.parametrize("C", [0.001, 1, 100, 1e6])
 def test_elastic_net_vs_l1_l2(C):
     # Make sure that elasticnet with grid search on l1_ratio gives same or
     # better results than just l1 or just l2.
@@ -1479,16 +1640,19 @@ def test_elastic_net_vs_l1_l2(C):
     X, y = make_classification(500, random_state=0)
     X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
 
-    param_grid = {'l1_ratio': np.linspace(0, 1, 5)}
+    param_grid = {"l1_ratio": np.linspace(0, 1, 5)}
 
-    enet_clf = LogisticRegression(penalty='elasticnet', C=C, solver='saga',
-                                  random_state=0)
+    enet_clf = LogisticRegression(
+        penalty="elasticnet", C=C, solver="saga", random_state=0, tol=1e-2
+    )
     gs = GridSearchCV(enet_clf, param_grid, refit=True)
 
-    l1_clf = LogisticRegression(penalty='l1', C=C, solver='saga',
-                                random_state=0)
-    l2_clf = LogisticRegression(penalty='l2', C=C, solver='saga',
-                                random_state=0)
+    l1_clf = LogisticRegression(
+        penalty="l1", C=C, solver="saga", random_state=0, tol=1e-2
+    )
+    l2_clf = LogisticRegression(
+        penalty="l2", C=C, solver="saga", random_state=0, tol=1e-2
+    )
 
     for clf in (gs, l1_clf, l2_clf):
         clf.fit(X_train, y_train)
@@ -1497,24 +1661,36 @@ def test_elastic_net_vs_l1_l2(C):
     assert gs.score(X_test, y_test) >= l2_clf.score(X_test, y_test)
 
 
-@pytest.mark.parametrize('C', np.logspace(-3, 2, 4))
-@pytest.mark.parametrize('l1_ratio', [.1, .5, .9])
+@pytest.mark.parametrize("C", np.logspace(-3, 2, 4))
+@pytest.mark.parametrize("l1_ratio", [0.1, 0.5, 0.9])
 def test_LogisticRegression_elastic_net_objective(C, l1_ratio):
     # Check that training with a penalty matching the objective leads
     # to a lower objective.
     # Here we train a logistic regression with l2 (a) and elasticnet (b)
     # penalties, and compute the elasticnet objective. That of a should be
     # greater than that of b (both objectives are convex).
-    X, y = make_classification(n_samples=1000, n_classes=2, n_features=20,
-                               n_informative=10, n_redundant=0,
-                               n_repeated=0, random_state=0)
+    X, y = make_classification(
+        n_samples=1000,
+        n_classes=2,
+        n_features=20,
+        n_informative=10,
+        n_redundant=0,
+        n_repeated=0,
+        random_state=0,
+    )
     X = scale(X)
 
-    lr_enet = LogisticRegression(penalty='elasticnet', solver='saga',
-                                 random_state=0, C=C, l1_ratio=l1_ratio,
-                                 fit_intercept=False)
-    lr_l2 = LogisticRegression(penalty='l2', solver='saga', random_state=0,
-                               C=C, fit_intercept=False)
+    lr_enet = LogisticRegression(
+        penalty="elasticnet",
+        solver="saga",
+        random_state=0,
+        C=C,
+        l1_ratio=l1_ratio,
+        fit_intercept=False,
+    )
+    lr_l2 = LogisticRegression(
+        penalty="l2", solver="saga", random_state=0, C=C, fit_intercept=False
+    )
     lr_enet.fit(X, y)
     lr_l2.fit(X, y)
 
@@ -1522,45 +1698,54 @@ def enet_objective(lr):
         coef = lr.coef_.ravel()
         obj = C * log_loss(y, lr.predict_proba(X))
         obj += l1_ratio * np.sum(np.abs(coef))
-        obj += (1. - l1_ratio) * 0.5 * np.dot(coef, coef)
+        obj += (1.0 - l1_ratio) * 0.5 * np.dot(coef, coef)
         return obj
 
     assert enet_objective(lr_enet) < enet_objective(lr_l2)
 
 
-@pytest.mark.parametrize('multi_class', ('ovr', 'multinomial'))
-def test_LogisticRegressionCV_GridSearchCV_elastic_net(multi_class):
+@pytest.mark.parametrize("n_classes", (2, 3))
+def test_LogisticRegressionCV_GridSearchCV_elastic_net(n_classes):
     # make sure LogisticRegressionCV gives same best params (l1 and C) as
     # GridSearchCV when penalty is elasticnet
 
-    if multi_class == 'ovr':
-        # This is actually binary classification, ovr multiclass is treated in
-        # test_LogisticRegressionCV_GridSearchCV_elastic_net_ovr
-        X, y = make_classification(random_state=0)
-    else:
-        X, y = make_classification(n_samples=100, n_classes=3, n_informative=3,
-                                   random_state=0)
+    X, y = make_classification(
+        n_samples=100, n_classes=n_classes, n_informative=3, random_state=0
+    )
 
     cv = StratifiedKFold(5)
 
     l1_ratios = np.linspace(0, 1, 3)
     Cs = np.logspace(-4, 4, 3)
 
-    lrcv = LogisticRegressionCV(penalty='elasticnet', Cs=Cs, solver='saga',
-                                cv=cv, l1_ratios=l1_ratios, random_state=0,
-                                multi_class=multi_class)
+    lrcv = LogisticRegressionCV(
+        penalty="elasticnet",
+        Cs=Cs,
+        solver="saga",
+        cv=cv,
+        l1_ratios=l1_ratios,
+        random_state=0,
+        tol=1e-2,
+    )
     lrcv.fit(X, y)
 
-    param_grid = {'C': Cs, 'l1_ratio': l1_ratios}
-    lr = LogisticRegression(penalty='elasticnet', solver='saga',
-                            random_state=0, multi_class=multi_class)
+    param_grid = {"C": Cs, "l1_ratio": l1_ratios}
+    lr = LogisticRegression(
+        penalty="elasticnet",
+        solver="saga",
+        random_state=0,
+        tol=1e-2,
+    )
     gs = GridSearchCV(lr, param_grid, cv=cv)
     gs.fit(X, y)
 
-    assert gs.best_params_['l1_ratio'] == lrcv.l1_ratio_[0]
-    assert gs.best_params_['C'] == lrcv.C_[0]
+    assert gs.best_params_["l1_ratio"] == lrcv.l1_ratio_[0]
+    assert gs.best_params_["C"] == lrcv.C_[0]
 
 
+# TODO(1.8): remove filterwarnings after the deprecation of multi_class
+# Maybe remove whole test after removal of the deprecated multi_class.
+@pytest.mark.filterwarnings("ignore:.*'multi_class' was deprecated.*:FutureWarning")
 def test_LogisticRegressionCV_GridSearchCV_elastic_net_ovr():
     # make sure LogisticRegressionCV gives same best params (l1 and C) as
     # GridSearchCV when penalty is elasticnet and multiclass is ovr. We can't
@@ -1569,135 +1754,174 @@ def test_LogisticRegressionCV_GridSearchCV_elastic_net_ovr():
     # l1_param for each class, while LogisticRegression will share the
     # parameters over the *n_classes* classifiers.
 
-    X, y = make_classification(n_samples=100, n_classes=3, n_informative=3,
-                               random_state=0)
+    X, y = make_classification(
+        n_samples=100, n_classes=3, n_informative=3, random_state=0
+    )
     X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
     cv = StratifiedKFold(5)
 
     l1_ratios = np.linspace(0, 1, 3)
     Cs = np.logspace(-4, 4, 3)
 
-    lrcv = LogisticRegressionCV(penalty='elasticnet', Cs=Cs, solver='saga',
-                                cv=cv, l1_ratios=l1_ratios, random_state=0,
-                                multi_class='ovr')
+    lrcv = LogisticRegressionCV(
+        penalty="elasticnet",
+        Cs=Cs,
+        solver="saga",
+        cv=cv,
+        l1_ratios=l1_ratios,
+        random_state=0,
+        multi_class="ovr",
+        tol=1e-2,
+    )
     lrcv.fit(X_train, y_train)
 
-    param_grid = {'C': Cs, 'l1_ratio': l1_ratios}
-    lr = LogisticRegression(penalty='elasticnet', solver='saga',
-                            random_state=0, multi_class='ovr')
+    param_grid = {"C": Cs, "l1_ratio": l1_ratios}
+    lr = LogisticRegression(
+        penalty="elasticnet",
+        solver="saga",
+        random_state=0,
+        multi_class="ovr",
+        tol=1e-2,
+    )
     gs = GridSearchCV(lr, param_grid, cv=cv)
     gs.fit(X_train, y_train)
 
     # Check that predictions are 80% the same
-    assert (lrcv.predict(X_train) == gs.predict(X_train)).mean() >= .8
-    assert (lrcv.predict(X_test) == gs.predict(X_test)).mean() >= .8
+    assert (lrcv.predict(X_train) == gs.predict(X_train)).mean() >= 0.8
+    assert (lrcv.predict(X_test) == gs.predict(X_test)).mean() >= 0.8
 
 
-@pytest.mark.parametrize('penalty', ('l2', 'elasticnet'))
-@pytest.mark.parametrize('multi_class', ('ovr', 'multinomial', 'auto'))
+# TODO(1.8): remove filterwarnings after the deprecation of multi_class
+@pytest.mark.filterwarnings("ignore:.*'multi_class' was deprecated.*:FutureWarning")
+@pytest.mark.parametrize("penalty", ("l2", "elasticnet"))
+@pytest.mark.parametrize("multi_class", ("ovr", "multinomial", "auto"))
 def test_LogisticRegressionCV_no_refit(penalty, multi_class):
     # Test LogisticRegressionCV attribute shapes when refit is False
 
     n_classes = 3
     n_features = 20
-    X, y = make_classification(n_samples=200, n_classes=n_classes,
-                               n_informative=n_classes, n_features=n_features,
-                               random_state=0)
+    X, y = make_classification(
+        n_samples=200,
+        n_classes=n_classes,
+        n_informative=n_classes,
+        n_features=n_features,
+        random_state=0,
+    )
 
     Cs = np.logspace(-4, 4, 3)
-    if penalty == 'elasticnet':
+    if penalty == "elasticnet":
         l1_ratios = np.linspace(0, 1, 2)
     else:
         l1_ratios = None
 
-    lrcv = LogisticRegressionCV(penalty=penalty, Cs=Cs, solver='saga',
-                                l1_ratios=l1_ratios, random_state=0,
-                                multi_class=multi_class, refit=False)
+    lrcv = LogisticRegressionCV(
+        penalty=penalty,
+        Cs=Cs,
+        solver="saga",
+        l1_ratios=l1_ratios,
+        random_state=0,
+        multi_class=multi_class,
+        tol=1e-2,
+        refit=False,
+    )
     lrcv.fit(X, y)
     assert lrcv.C_.shape == (n_classes,)
     assert lrcv.l1_ratio_.shape == (n_classes,)
     assert lrcv.coef_.shape == (n_classes, n_features)
 
 
+# TODO(1.8): remove filterwarnings after the deprecation of multi_class
+# Remove multi_class an change first element of the expected n_iter_.shape from
+# n_classes to 1 (according to the docstring).
+@pytest.mark.filterwarnings("ignore:.*'multi_class' was deprecated.*:FutureWarning")
 def test_LogisticRegressionCV_elasticnet_attribute_shapes():
     # Make sure the shapes of scores_ and coefs_paths_ attributes are correct
     # when using elasticnet (added one dimension for l1_ratios)
 
     n_classes = 3
     n_features = 20
-    X, y = make_classification(n_samples=200, n_classes=n_classes,
-                               n_informative=n_classes, n_features=n_features,
-                               random_state=0)
+    X, y = make_classification(
+        n_samples=200,
+        n_classes=n_classes,
+        n_informative=n_classes,
+        n_features=n_features,
+        random_state=0,
+    )
 
     Cs = np.logspace(-4, 4, 3)
     l1_ratios = np.linspace(0, 1, 2)
 
     n_folds = 2
-    lrcv = LogisticRegressionCV(penalty='elasticnet', Cs=Cs, solver='saga',
-                                cv=n_folds, l1_ratios=l1_ratios,
-                                multi_class='ovr', random_state=0)
+    lrcv = LogisticRegressionCV(
+        penalty="elasticnet",
+        Cs=Cs,
+        solver="saga",
+        cv=n_folds,
+        l1_ratios=l1_ratios,
+        multi_class="ovr",
+        random_state=0,
+        tol=1e-2,
+    )
     lrcv.fit(X, y)
     coefs_paths = np.asarray(list(lrcv.coefs_paths_.values()))
-    assert coefs_paths.shape == (n_classes, n_folds, Cs.size,
-                                 l1_ratios.size, n_features + 1)
+    assert coefs_paths.shape == (
+        n_classes,
+        n_folds,
+        Cs.size,
+        l1_ratios.size,
+        n_features + 1,
+    )
     scores = np.asarray(list(lrcv.scores_.values()))
     assert scores.shape == (n_classes, n_folds, Cs.size, l1_ratios.size)
 
     assert lrcv.n_iter_.shape == (n_classes, n_folds, Cs.size, l1_ratios.size)
 
 
-@pytest.mark.parametrize('l1_ratio', (-1, 2, None, 'something_wrong'))
-def test_l1_ratio_param(l1_ratio):
-
-    msg = r"l1_ratio must be between 0 and 1; got \(l1_ratio=%r\)" % l1_ratio
-    with pytest.raises(ValueError, match=msg):
-        LogisticRegression(penalty='elasticnet', solver='saga',
-                           l1_ratio=l1_ratio).fit(X, Y1)
-
-    if l1_ratio is not None:
-        msg = (r"l1_ratio parameter is only used when penalty is"
-               r" 'elasticnet'\. Got \(penalty=l1\)")
-        with pytest.warns(UserWarning, match=msg):
-            LogisticRegression(penalty='l1', solver='saga',
-                               l1_ratio=l1_ratio).fit(X, Y1)
-
-
-@pytest.mark.parametrize('l1_ratios', ([], [.5, 2], None, 'something_wrong'))
-def test_l1_ratios_param(l1_ratios):
-
-    msg = ("l1_ratios must be a list of numbers between 0 and 1; got "
-           "(l1_ratios=%r)" % l1_ratios)
-
-    with pytest.raises(ValueError, match=re.escape(msg)):
-        LogisticRegressionCV(penalty='elasticnet',
-                             solver='saga',
-                             l1_ratios=l1_ratios, cv=2).fit(X, Y1)
-
-    if l1_ratios is not None:
-        msg = (r"l1_ratios parameter is only used when penalty"
-               r" is 'elasticnet'. Got \(penalty=l1\)")
-        function = LogisticRegressionCV(penalty='l1', solver='saga',
-                                        l1_ratios=l1_ratios, cv=2).fit
-        with pytest.warns(UserWarning, match=msg):
-            function(X, Y1)
+def test_l1_ratio_non_elasticnet():
+    msg = (
+        r"l1_ratio parameter is only used when penalty is"
+        r" 'elasticnet'\. Got \(penalty=l1\)"
+    )
+    with pytest.warns(UserWarning, match=msg):
+        LogisticRegression(penalty="l1", solver="saga", l1_ratio=0.5).fit(X, Y1)
 
 
-@pytest.mark.parametrize('C', np.logspace(-3, 2, 4))
-@pytest.mark.parametrize('l1_ratio', [.1, .5, .9])
+@pytest.mark.parametrize("C", np.logspace(-3, 2, 4))
+@pytest.mark.parametrize("l1_ratio", [0.1, 0.5, 0.9])
 def test_elastic_net_versus_sgd(C, l1_ratio):
     # Compare elasticnet penalty in LogisticRegression() and SGD(loss='log')
     n_samples = 500
-    X, y = make_classification(n_samples=n_samples, n_classes=2, n_features=5,
-                               n_informative=5, n_redundant=0, n_repeated=0,
-                               random_state=1)
+    X, y = make_classification(
+        n_samples=n_samples,
+        n_classes=2,
+        n_features=5,
+        n_informative=5,
+        n_redundant=0,
+        n_repeated=0,
+        random_state=1,
+    )
     X = scale(X)
 
     sgd = SGDClassifier(
-        penalty='elasticnet', random_state=1, fit_intercept=False, tol=-np.inf,
-        max_iter=2000, l1_ratio=l1_ratio, alpha=1. / C / n_samples, loss='log')
+        penalty="elasticnet",
+        random_state=1,
+        fit_intercept=False,
+        tol=None,
+        max_iter=2000,
+        l1_ratio=l1_ratio,
+        alpha=1.0 / C / n_samples,
+        loss="log_loss",
+    )
     log = LogisticRegression(
-        penalty='elasticnet', random_state=1, fit_intercept=False, tol=1e-5,
-        max_iter=1000, l1_ratio=l1_ratio, C=C, solver='saga')
+        penalty="elasticnet",
+        random_state=1,
+        fit_intercept=False,
+        tol=1e-5,
+        max_iter=1000,
+        l1_ratio=l1_ratio,
+        C=C,
+        solver="saga",
+    )
 
     sgd.fit(X, y)
     log.fit(X, y)
@@ -1708,13 +1932,25 @@ def test_logistic_regression_path_coefs_multinomial():
     # Make sure that the returned coefs by logistic_regression_path when
     # multi_class='multinomial' don't override each other (used to be a
     # bug).
-    X, y = make_classification(n_samples=200, n_classes=3, n_informative=2,
-                               n_redundant=0, n_clusters_per_class=1,
-                               random_state=0, n_features=2)
-    Cs = [.00001, 1, 10000]
-    coefs, _, _ = _logistic_regression_path(X, y, penalty='l1', Cs=Cs,
-                                            solver='saga', random_state=0,
-                                            multi_class='multinomial')
+    X, y = make_classification(
+        n_samples=200,
+        n_classes=3,
+        n_informative=2,
+        n_redundant=0,
+        n_clusters_per_class=1,
+        random_state=0,
+        n_features=2,
+    )
+    Cs = [0.00001, 1, 10000]
+    coefs, _, _ = _logistic_regression_path(
+        X,
+        y,
+        penalty="l1",
+        Cs=Cs,
+        solver="saga",
+        random_state=0,
+        multi_class="multinomial",
+    )
 
     with pytest.raises(AssertionError):
         assert_array_almost_equal(coefs[0], coefs[1], decimal=1)
@@ -1724,15 +1960,23 @@ def test_logistic_regression_path_coefs_multinomial():
         assert_array_almost_equal(coefs[1], coefs[2], decimal=1)
 
 
-@pytest.mark.parametrize('est',
-                         [LogisticRegression(random_state=0, max_iter=500),
-                          LogisticRegressionCV(random_state=0, cv=3,
-                                               Cs=3, tol=1e-3, max_iter=500)],
-                         ids=lambda x: x.__class__.__name__)
-@pytest.mark.parametrize('solver', ['liblinear', 'lbfgs', 'newton-cg', 'sag',
-                                    'saga'])
+# TODO(1.8): remove filterwarnings after the deprecation of multi_class
+@pytest.mark.filterwarnings("ignore:.*'multi_class' was deprecated.*:FutureWarning")
+@pytest.mark.filterwarnings(
+    "ignore:.*'liblinear' solver for multiclass classification is deprecated.*"
+)
+@pytest.mark.parametrize(
+    "est",
+    [
+        LogisticRegression(random_state=0, max_iter=500),
+        LogisticRegressionCV(random_state=0, cv=3, Cs=3, tol=1e-3, max_iter=500),
+    ],
+    ids=lambda x: x.__class__.__name__,
+)
+@pytest.mark.parametrize("solver", SOLVERS)
 def test_logistic_regression_multi_class_auto(est, solver):
-    # check multi_class='auto' => multi_class='ovr' iff binary y or liblinear
+    # check multi_class='auto' => multi_class='ovr'
+    # iff binary y or liblinear
 
     def fit(X, y, **kw):
         return clone(est).set_params(**kw).fit(X, y)
@@ -1742,87 +1986,101 @@ def fit(X, y, **kw):
     X2 = scaled_data[1::10]
     y_multi = iris.target[::10]
     y_bin = y_multi == 0
-    est_auto_bin = fit(X, y_bin, multi_class='auto', solver=solver)
-    est_ovr_bin = fit(X, y_bin, multi_class='ovr', solver=solver)
+    est_auto_bin = fit(X, y_bin, multi_class="auto", solver=solver)
+    est_ovr_bin = fit(X, y_bin, multi_class="ovr", solver=solver)
     assert_allclose(est_auto_bin.coef_, est_ovr_bin.coef_)
-    assert_allclose(est_auto_bin.predict_proba(X2),
-                    est_ovr_bin.predict_proba(X2))
+    assert_allclose(est_auto_bin.predict_proba(X2), est_ovr_bin.predict_proba(X2))
 
-    est_auto_multi = fit(X, y_multi, multi_class='auto', solver=solver)
-    if solver == 'liblinear':
-        est_ovr_multi = fit(X, y_multi, multi_class='ovr', solver=solver)
+    est_auto_multi = fit(X, y_multi, multi_class="auto", solver=solver)
+    if solver == "liblinear":
+        est_ovr_multi = fit(X, y_multi, multi_class="ovr", solver=solver)
         assert_allclose(est_auto_multi.coef_, est_ovr_multi.coef_)
-        assert_allclose(est_auto_multi.predict_proba(X2),
-                        est_ovr_multi.predict_proba(X2))
+        assert_allclose(
+            est_auto_multi.predict_proba(X2), est_ovr_multi.predict_proba(X2)
+        )
     else:
-        est_multi_multi = fit(X, y_multi, multi_class='multinomial',
-                              solver=solver)
+        est_multi_multi = fit(X, y_multi, multi_class="multinomial", solver=solver)
         assert_allclose(est_auto_multi.coef_, est_multi_multi.coef_)
-        assert_allclose(est_auto_multi.predict_proba(X2),
-                        est_multi_multi.predict_proba(X2))
+        assert_allclose(
+            est_auto_multi.predict_proba(X2), est_multi_multi.predict_proba(X2)
+        )
 
         # Make sure multi_class='ovr' is distinct from ='multinomial'
-        assert not np.allclose(est_auto_bin.coef_,
-                               fit(X, y_bin, multi_class='multinomial',
-                                   solver=solver).coef_)
-        assert not np.allclose(est_auto_bin.coef_,
-                               fit(X, y_multi, multi_class='multinomial',
-                                   solver=solver).coef_)
+        assert not np.allclose(
+            est_auto_bin.coef_,
+            fit(X, y_bin, multi_class="multinomial", solver=solver).coef_,
+        )
+        assert not np.allclose(
+            est_auto_bin.coef_,
+            fit(X, y_multi, multi_class="multinomial", solver=solver).coef_,
+        )
 
 
-@pytest.mark.parametrize('solver', ('lbfgs', 'newton-cg', 'sag', 'saga'))
+@pytest.mark.parametrize("solver", sorted(set(SOLVERS) - set(["liblinear"])))
 def test_penalty_none(solver):
-    # - Make sure warning is raised if penalty='none' and C is set to a
+    # - Make sure warning is raised if penalty=None and C is set to a
     #   non-default value.
-    # - Make sure setting penalty='none' is equivalent to setting C=np.inf with
+    # - Make sure setting penalty=None is equivalent to setting C=np.inf with
     #   l2 penalty.
-    X, y = make_classification(n_samples=1000, random_state=0)
+    X, y = make_classification(n_samples=1000, n_redundant=0, random_state=0)
 
-    msg = "Setting penalty='none' will ignore the C"
-    lr = LogisticRegression(penalty='none', solver=solver, C=4)
+    msg = "Setting penalty=None will ignore the C"
+    lr = LogisticRegression(penalty=None, solver=solver, C=4)
     with pytest.warns(UserWarning, match=msg):
         lr.fit(X, y)
 
-    lr_none = LogisticRegression(penalty='none', solver=solver,
-                                 random_state=0)
-    lr_l2_C_inf = LogisticRegression(penalty='l2', C=np.inf, solver=solver,
-                                     random_state=0)
+    lr_none = LogisticRegression(penalty=None, solver=solver, random_state=0)
+    lr_l2_C_inf = LogisticRegression(
+        penalty="l2", C=np.inf, solver=solver, random_state=0
+    )
     pred_none = lr_none.fit(X, y).predict(X)
     pred_l2_C_inf = lr_l2_C_inf.fit(X, y).predict(X)
     assert_array_equal(pred_none, pred_l2_C_inf)
 
-    lr = LogisticRegressionCV(penalty='none')
-    err_msg = (
-        "penalty='none' is not useful and not supported by "
-        "LogisticRegressionCV"
-    )
-    with pytest.raises(ValueError, match=err_msg):
-        lr.fit(X, y)
-
 
 @pytest.mark.parametrize(
     "params",
-    [{'penalty': 'l1', 'dual': False, 'tol': 1e-12, 'max_iter': 1000},
-     {'penalty': 'l2', 'dual': True, 'tol': 1e-12, 'max_iter': 1000},
-     {'penalty': 'l2', 'dual': False, 'tol': 1e-12, 'max_iter': 1000}]
+    [
+        {"penalty": "l1", "dual": False, "tol": 1e-6, "max_iter": 1000},
+        {"penalty": "l2", "dual": True, "tol": 1e-12, "max_iter": 1000},
+        {"penalty": "l2", "dual": False, "tol": 1e-12, "max_iter": 1000},
+    ],
 )
 def test_logisticregression_liblinear_sample_weight(params):
     # check that we support sample_weight with liblinear in all possible cases:
     # l1-primal, l2-primal, l2-dual
-    X = np.array([[1, 3], [1, 3], [1, 3], [1, 3],
-                  [2, 1], [2, 1], [2, 1], [2, 1],
-                  [3, 3], [3, 3], [3, 3], [3, 3],
-                  [4, 1], [4, 1], [4, 1], [4, 1]], dtype=np.dtype('float'))
-    y = np.array([1, 1, 1, 1, 2, 2, 2, 2,
-                  1, 1, 1, 1, 2, 2, 2, 2], dtype=np.dtype('int'))
+    X = np.array(
+        [
+            [1, 3],
+            [1, 3],
+            [1, 3],
+            [1, 3],
+            [2, 1],
+            [2, 1],
+            [2, 1],
+            [2, 1],
+            [3, 3],
+            [3, 3],
+            [3, 3],
+            [3, 3],
+            [4, 1],
+            [4, 1],
+            [4, 1],
+            [4, 1],
+        ],
+        dtype=np.dtype("float"),
+    )
+    y = np.array(
+        [1, 1, 1, 1, 2, 2, 2, 2, 1, 1, 1, 1, 2, 2, 2, 2], dtype=np.dtype("int")
+    )
 
     X2 = np.vstack([X, X])
     y2 = np.hstack([y, 3 - y])
     sample_weight = np.ones(shape=len(y) * 2)
-    sample_weight[len(y):] = 0
+    sample_weight[len(y) :] = 0
     X2, y2, sample_weight = shuffle(X2, y2, sample_weight, random_state=0)
 
-    base_clf = LogisticRegression(solver='liblinear', random_state=42)
+    base_clf = LogisticRegression(solver="liblinear", random_state=42)
     base_clf.set_params(**params)
     clf_no_weight = clone(base_clf).fit(X, y)
     clf_with_weight = clone(base_clf).fit(X2, y2, sample_weight=sample_weight)
@@ -1843,28 +2101,44 @@ def test_scores_attribute_layout_elasticnet():
     X, y = make_classification(n_samples=1000, random_state=0)
     cv = StratifiedKFold(n_splits=5)
 
-    l1_ratios = [.1, .9]
-    Cs = [.1, 1, 10]
-
-    lrcv = LogisticRegressionCV(penalty='elasticnet', solver='saga',
-                                l1_ratios=l1_ratios, Cs=Cs, cv=cv,
-                                random_state=0)
+    l1_ratios = [0.1, 0.9]
+    Cs = [0.1, 1, 10]
+
+    lrcv = LogisticRegressionCV(
+        penalty="elasticnet",
+        solver="saga",
+        l1_ratios=l1_ratios,
+        Cs=Cs,
+        cv=cv,
+        random_state=0,
+        max_iter=250,
+        tol=1e-3,
+    )
     lrcv.fit(X, y)
 
     avg_scores_lrcv = lrcv.scores_[1].mean(axis=0)  # average over folds
 
     for i, C in enumerate(Cs):
         for j, l1_ratio in enumerate(l1_ratios):
-
-            lr = LogisticRegression(penalty='elasticnet', solver='saga', C=C,
-                                    l1_ratio=l1_ratio, random_state=0)
+            lr = LogisticRegression(
+                penalty="elasticnet",
+                solver="saga",
+                C=C,
+                l1_ratio=l1_ratio,
+                random_state=0,
+                max_iter=250,
+                tol=1e-3,
+            )
 
             avg_score_lr = cross_val_score(lr, X, y, cv=cv).mean()
             assert avg_scores_lrcv[i, j] == pytest.approx(avg_score_lr)
 
 
+# TODO(1.8): remove filterwarnings after the deprecation of multi_class
+@pytest.mark.filterwarnings("ignore:.*'multi_class' was deprecated.*:FutureWarning")
+@pytest.mark.parametrize("solver", ["lbfgs", "newton-cg", "newton-cholesky"])
 @pytest.mark.parametrize("fit_intercept", [False, True])
-def test_multinomial_identifiability_on_iris(fit_intercept):
+def test_multinomial_identifiability_on_iris(solver, fit_intercept):
     """Test that the multinomial classification is identifiable.
 
     A multinomial with c classes can be modeled with
@@ -1874,50 +2148,292 @@ def test_multinomial_identifiability_on_iris(fit_intercept):
     satisfies the symmetric constraint:
     sum(coef_k, k=1..c) = 0
 
-    Further details can be found in the appendix of [2].
+    Further details can be found in [2].
 
     Reference
     ---------
-    .. [1] Zhu, Ji and Trevor J. Hastie. "Classification of gene microarrays by
-    penalized logistic regression". Biostatistics 5 3 (2004): 427-43.
-    https://doi.org/10.1093/biostatistics%2Fkxg046
-
-    .. [2] Powers, Scott, Trevor J. Hastie and Robert Tibshirani. "Nuclear
-    penalized multinomial regression with an application to predicting at bat
-    outcomes in baseball." Statistical modelling 18 5-6 (2017): 388-410 .
-    https://arxiv.org/pdf/1706.10272.pdf
+    .. [1] :doi:`Zhu, Ji and Trevor J. Hastie. "Classification of gene microarrays by
+           penalized logistic regression". Biostatistics 5 3 (2004): 427-43.
+           <10.1093/biostatistics/kxg046>`
+
+    .. [2] :arxiv:`Noah Simon and Jerome Friedman and Trevor Hastie. (2013)
+           "A Blockwise Descent Algorithm for Group-penalized Multiresponse and
+           Multinomial Regression". <1311.6529>`
     """
     # Test logistic regression with the iris dataset
     n_samples, n_features = iris.data.shape
     target = iris.target_names[iris.target]
 
-    clf = LogisticRegression(C=len(iris.data), solver='lbfgs', max_iter=300,
-                             multi_class='multinomial',
-                             fit_intercept=fit_intercept
-                             )
-    clf.fit(iris.data, target)
+    clf = LogisticRegression(
+        C=len(iris.data),
+        solver="lbfgs",
+        fit_intercept=fit_intercept,
+    )
+    # Scaling X to ease convergence.
+    X_scaled = scale(iris.data)
+    clf.fit(X_scaled, target)
 
     # axis=0 is sum over classes
     assert_allclose(clf.coef_.sum(axis=0), 0, atol=1e-10)
     if fit_intercept:
-        clf.intercept_.sum(axis=0) == pytest.approx(0, abs=1e-15)
+        assert clf.intercept_.sum(axis=0) == pytest.approx(0, abs=1e-11)
 
 
-@pytest.mark.parametrize("multi_class", ['ovr', 'multinomial', 'auto'])
-@pytest.mark.parametrize("class_weight", [
-    {0: 1.0, 1: 10.0, 2: 1.0}, 'balanced'
-])
+# TODO(1.8): remove filterwarnings after the deprecation of multi_class
+@pytest.mark.filterwarnings("ignore:.*'multi_class' was deprecated.*:FutureWarning")
+@pytest.mark.parametrize("multi_class", ["ovr", "multinomial", "auto"])
+@pytest.mark.parametrize("class_weight", [{0: 1.0, 1: 10.0, 2: 1.0}, "balanced"])
 def test_sample_weight_not_modified(multi_class, class_weight):
     X, y = load_iris(return_X_y=True)
     n_features = len(X)
     W = np.ones(n_features)
-    W[:n_features // 2] = 2
+    W[: n_features // 2] = 2
 
     expected = W.copy()
 
-    clf = LogisticRegression(random_state=0,
-                             class_weight=class_weight,
-                             max_iter=200,
-                             multi_class=multi_class)
+    clf = LogisticRegression(
+        random_state=0, class_weight=class_weight, max_iter=200, multi_class=multi_class
+    )
     clf.fit(X, y, sample_weight=W)
     assert_allclose(expected, W)
+
+
+@pytest.mark.parametrize("solver", SOLVERS)
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_large_sparse_matrix(solver, global_random_seed, csr_container):
+    # Solvers either accept large sparse matrices, or raise helpful error.
+    # Non-regression test for pull-request #21093.
+
+    # generate sparse matrix with int64 indices
+    X = csr_container(sparse.rand(20, 10, random_state=global_random_seed))
+    for attr in ["indices", "indptr"]:
+        setattr(X, attr, getattr(X, attr).astype("int64"))
+    rng = np.random.RandomState(global_random_seed)
+    y = rng.randint(2, size=X.shape[0])
+
+    if solver in ["liblinear", "sag", "saga"]:
+        msg = "Only sparse matrices with 32-bit integer indices"
+        with pytest.raises(ValueError, match=msg):
+            LogisticRegression(solver=solver).fit(X, y)
+    else:
+        LogisticRegression(solver=solver).fit(X, y)
+
+
+def test_single_feature_newton_cg():
+    # Test that Newton-CG works with a single feature and intercept.
+    # Non-regression test for issue #23605.
+
+    X = np.array([[0.5, 0.65, 1.1, 1.25, 0.8, 0.54, 0.95, 0.7]]).T
+    y = np.array([1, 1, 0, 0, 1, 1, 0, 1])
+    assert X.shape[1] == 1
+    LogisticRegression(solver="newton-cg", fit_intercept=True).fit(X, y)
+
+
+def test_liblinear_not_stuck():
+    # Non-regression https://github.com/scikit-learn/scikit-learn/issues/18264
+    X = iris.data.copy()
+    y = iris.target.copy()
+    X = X[y != 2]
+    y = y[y != 2]
+    X_prep = StandardScaler().fit_transform(X)
+
+    C = l1_min_c(X, y, loss="log") * 10 ** (10 / 29)
+    clf = LogisticRegression(
+        penalty="l1",
+        solver="liblinear",
+        tol=1e-6,
+        max_iter=100,
+        intercept_scaling=10000.0,
+        random_state=0,
+        C=C,
+    )
+
+    # test that the fit does not raise a ConvergenceWarning
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", ConvergenceWarning)
+        clf.fit(X_prep, y)
+
+
+@config_context(enable_metadata_routing=True)
+def test_lr_cv_scores_differ_when_sample_weight_is_requested():
+    """Test that `sample_weight` is correctly passed to the scorer in
+    `LogisticRegressionCV.fit` and `LogisticRegressionCV.score` by
+    checking the difference in scores with the case when `sample_weight`
+    is not requested.
+    """
+    rng = np.random.RandomState(10)
+    X, y = make_classification(n_samples=10, random_state=rng)
+    X_t, y_t = make_classification(n_samples=10, random_state=rng)
+    sample_weight = np.ones(len(y))
+    sample_weight[: len(y) // 2] = 2
+    kwargs = {"sample_weight": sample_weight}
+
+    scorer1 = get_scorer("accuracy")
+    lr_cv1 = LogisticRegressionCV(scoring=scorer1)
+    lr_cv1.fit(X, y, **kwargs)
+
+    scorer2 = get_scorer("accuracy")
+    scorer2.set_score_request(sample_weight=True)
+    lr_cv2 = LogisticRegressionCV(scoring=scorer2)
+    lr_cv2.fit(X, y, **kwargs)
+
+    assert not np.allclose(lr_cv1.scores_[1], lr_cv2.scores_[1])
+
+    score_1 = lr_cv1.score(X_t, y_t, **kwargs)
+    score_2 = lr_cv2.score(X_t, y_t, **kwargs)
+
+    assert not np.allclose(score_1, score_2)
+
+
+def test_lr_cv_scores_without_enabling_metadata_routing():
+    """Test that `sample_weight` is passed correctly to the scorer in
+    `LogisticRegressionCV.fit` and `LogisticRegressionCV.score` even
+    when `enable_metadata_routing=False`
+    """
+    rng = np.random.RandomState(10)
+    X, y = make_classification(n_samples=10, random_state=rng)
+    X_t, y_t = make_classification(n_samples=10, random_state=rng)
+    sample_weight = np.ones(len(y))
+    sample_weight[: len(y) // 2] = 2
+    kwargs = {"sample_weight": sample_weight}
+
+    with config_context(enable_metadata_routing=False):
+        scorer1 = get_scorer("accuracy")
+        lr_cv1 = LogisticRegressionCV(scoring=scorer1)
+        lr_cv1.fit(X, y, **kwargs)
+        score_1 = lr_cv1.score(X_t, y_t, **kwargs)
+
+    with config_context(enable_metadata_routing=True):
+        scorer2 = get_scorer("accuracy")
+        scorer2.set_score_request(sample_weight=True)
+        lr_cv2 = LogisticRegressionCV(scoring=scorer2)
+        lr_cv2.fit(X, y, **kwargs)
+        score_2 = lr_cv2.score(X_t, y_t, **kwargs)
+
+    assert_allclose(lr_cv1.scores_[1], lr_cv2.scores_[1])
+    assert_allclose(score_1, score_2)
+
+
+@pytest.mark.parametrize("solver", SOLVERS)
+def test_zero_max_iter(solver):
+    # Make sure we can inspect the state of LogisticRegression right after
+    # initialization (before the first weight update).
+    X, y = load_iris(return_X_y=True)
+    y = y == 2
+    with ignore_warnings(category=ConvergenceWarning):
+        clf = LogisticRegression(solver=solver, max_iter=0).fit(X, y)
+    if solver not in ["saga", "sag"]:
+        # XXX: sag and saga have n_iter_ = [1]...
+        assert clf.n_iter_ == 0
+
+    if solver != "lbfgs":
+        # XXX: lbfgs has already started to update the coefficients...
+        assert_allclose(clf.coef_, np.zeros_like(clf.coef_))
+        assert_allclose(
+            clf.decision_function(X),
+            np.full(shape=X.shape[0], fill_value=clf.intercept_),
+        )
+        assert_allclose(
+            clf.predict_proba(X),
+            np.full(shape=(X.shape[0], 2), fill_value=0.5),
+        )
+    assert clf.score(X, y) < 0.7
+
+
+def test_passing_params_without_enabling_metadata_routing():
+    """Test that the right error message is raised when metadata params
+    are passed while not supported when `enable_metadata_routing=False`."""
+    X, y = make_classification(n_samples=10, random_state=0)
+    lr_cv = LogisticRegressionCV()
+    msg = "is only supported if enable_metadata_routing=True"
+
+    with config_context(enable_metadata_routing=False):
+        params = {"extra_param": 1.0}
+
+        with pytest.raises(ValueError, match=msg):
+            lr_cv.fit(X, y, **params)
+
+        with pytest.raises(ValueError, match=msg):
+            lr_cv.score(X, y, **params)
+
+
+# TODO(1.8): remove
+def test_multi_class_deprecated():
+    """Check `multi_class` parameter deprecated."""
+    X, y = make_classification(n_classes=3, n_samples=50, n_informative=6)
+    lr = LogisticRegression(multi_class="ovr")
+    msg = "'multi_class' was deprecated"
+    with pytest.warns(FutureWarning, match=msg):
+        lr.fit(X, y)
+
+    lrCV = LogisticRegressionCV(multi_class="ovr")
+    with pytest.warns(FutureWarning, match=msg):
+        lrCV.fit(X, y)
+
+    # Special warning for "binary multinomial"
+    X, y = make_classification(n_classes=2, n_samples=50, n_informative=6)
+    lr = LogisticRegression(multi_class="multinomial")
+    msg = "'multi_class' was deprecated.*binary problems"
+    with pytest.warns(FutureWarning, match=msg):
+        lr.fit(X, y)
+
+    lrCV = LogisticRegressionCV(multi_class="multinomial")
+    with pytest.warns(FutureWarning, match=msg):
+        lrCV.fit(X, y)
+
+
+def test_newton_cholesky_fallback_to_lbfgs(global_random_seed):
+    # Wide data matrix should lead to a rank-deficient Hessian matrix
+    # hence make the Newton-Cholesky solver raise a warning and fallback to
+    # lbfgs.
+    X, y = make_classification(
+        n_samples=10, n_features=20, random_state=global_random_seed
+    )
+    C = 1e30  # very high C to nearly disable regularization
+
+    # Check that LBFGS can converge without any warning on this problem.
+    lr_lbfgs = LogisticRegression(solver="lbfgs", C=C)
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        lr_lbfgs.fit(X, y)
+        n_iter_lbfgs = lr_lbfgs.n_iter_[0]
+
+    assert n_iter_lbfgs >= 1
+
+    # Check that the Newton-Cholesky solver raises a warning and falls back to
+    # LBFGS. This should converge with the same number of iterations as the
+    # above call of lbfgs since the Newton-Cholesky triggers the fallback
+    # before completing the first iteration, for the problem setting at hand.
+    lr_nc = LogisticRegression(solver="newton-cholesky", C=C)
+    with ignore_warnings(category=LinAlgWarning):
+        lr_nc.fit(X, y)
+        n_iter_nc = lr_nc.n_iter_[0]
+
+    assert n_iter_nc == n_iter_lbfgs
+
+    # Trying to fit the same model again with a small iteration budget should
+    # therefore raise a ConvergenceWarning:
+    lr_nc_limited = LogisticRegression(
+        solver="newton-cholesky", C=C, max_iter=n_iter_lbfgs - 1
+    )
+    with ignore_warnings(category=LinAlgWarning):
+        with pytest.warns(ConvergenceWarning, match="lbfgs failed to converge"):
+            lr_nc_limited.fit(X, y)
+            n_iter_nc_limited = lr_nc_limited.n_iter_[0]
+
+    assert n_iter_nc_limited == lr_nc_limited.max_iter - 1
+
+
+# TODO(1.8): check for an error instead
+@pytest.mark.parametrize("Estimator", [LogisticRegression, LogisticRegressionCV])
+def test_liblinear_multiclass_warning(Estimator):
+    """Check that liblinear warns on multiclass problems."""
+    msg = (
+        "Using the 'liblinear' solver for multiclass classification is "
+        "deprecated. An error will be raised in 1.8. Either use another "
+        "solver which supports the multinomial loss or wrap the estimator "
+        "in a OneVsRestClassifier to keep applying a one-versus-rest "
+        "scheme."
+    )
+    with pytest.warns(FutureWarning, match=msg):
+        Estimator(solver="liblinear").fit(iris.data, iris.target)
diff --git a/sklearn/linear_model/tests/test_omp.py b/sklearn/linear_model/tests/test_omp.py
index 3cbda003f0148..cfdffe581e034 100644
--- a/sklearn/linear_model/tests/test_omp.py
+++ b/sklearn/linear_model/tests/test_omp.py
@@ -1,28 +1,35 @@
-# Author: Vlad Niculae
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
 
 import numpy as np
 import pytest
 
-from sklearn.utils._testing import assert_raises
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils._testing import assert_array_almost_equal
-from sklearn.utils._testing import ignore_warnings
-
-
-from sklearn.linear_model import (orthogonal_mp, orthogonal_mp_gram,
-                                  OrthogonalMatchingPursuit,
-                                  OrthogonalMatchingPursuitCV,
-                                  LinearRegression)
-from sklearn.utils import check_random_state
 from sklearn.datasets import make_sparse_coded_signal
+from sklearn.linear_model import (
+    LinearRegression,
+    OrthogonalMatchingPursuit,
+    OrthogonalMatchingPursuitCV,
+    orthogonal_mp,
+    orthogonal_mp_gram,
+)
+from sklearn.utils import check_random_state
+from sklearn.utils._testing import (
+    assert_allclose,
+    assert_array_almost_equal,
+    assert_array_equal,
+    ignore_warnings,
+)
 
 n_samples, n_features, n_nonzero_coefs, n_targets = 25, 35, 5, 3
-y, X, gamma = make_sparse_coded_signal(n_samples=n_targets,
-                                       n_components=n_features,
-                                       n_features=n_samples,
-                                       n_nonzero_coefs=n_nonzero_coefs,
-                                       random_state=0)
+y, X, gamma = make_sparse_coded_signal(
+    n_samples=n_targets,
+    n_components=n_features,
+    n_features=n_samples,
+    n_nonzero_coefs=n_nonzero_coefs,
+    random_state=0,
+)
+y, X, gamma = y.T, X.T, gamma.T
 # Make X not of norm 1 for testing
 X *= 10
 y *= 10
@@ -32,24 +39,21 @@
 
 
 def test_correct_shapes():
-    assert (orthogonal_mp(X, y[:, 0], n_nonzero_coefs=5).shape ==
-                 (n_features,))
-    assert (orthogonal_mp(X, y, n_nonzero_coefs=5).shape ==
-                 (n_features, 3))
+    assert orthogonal_mp(X, y[:, 0], n_nonzero_coefs=5).shape == (n_features,)
+    assert orthogonal_mp(X, y, n_nonzero_coefs=5).shape == (n_features, 3)
 
 
 def test_correct_shapes_gram():
-    assert (orthogonal_mp_gram(G, Xy[:, 0], n_nonzero_coefs=5).shape ==
-                 (n_features,))
-    assert (orthogonal_mp_gram(G, Xy, n_nonzero_coefs=5).shape ==
-                 (n_features, 3))
+    assert orthogonal_mp_gram(G, Xy[:, 0], n_nonzero_coefs=5).shape == (n_features,)
+    assert orthogonal_mp_gram(G, Xy, n_nonzero_coefs=5).shape == (n_features, 3)
 
 
 def test_n_nonzero_coefs():
     assert np.count_nonzero(orthogonal_mp(X, y[:, 0], n_nonzero_coefs=5)) <= 5
-    assert np.count_nonzero(orthogonal_mp(X, y[:, 0],
-                                          n_nonzero_coefs=5,
-                                          precompute=True)) <= 5
+    assert (
+        np.count_nonzero(orthogonal_mp(X, y[:, 0], n_nonzero_coefs=5, precompute=True))
+        <= 5
+    )
 
 
 def test_tol():
@@ -63,19 +67,20 @@ def test_tol():
 def test_with_without_gram():
     assert_array_almost_equal(
         orthogonal_mp(X, y, n_nonzero_coefs=5),
-        orthogonal_mp(X, y, n_nonzero_coefs=5, precompute=True))
+        orthogonal_mp(X, y, n_nonzero_coefs=5, precompute=True),
+    )
 
 
 def test_with_without_gram_tol():
     assert_array_almost_equal(
-        orthogonal_mp(X, y, tol=1.),
-        orthogonal_mp(X, y, tol=1., precompute=True))
+        orthogonal_mp(X, y, tol=1.0), orthogonal_mp(X, y, tol=1.0, precompute=True)
+    )
 
 
 def test_unreachable_accuracy():
     assert_array_almost_equal(
-        orthogonal_mp(X, y, tol=0),
-        orthogonal_mp(X, y, n_nonzero_coefs=n_features))
+        orthogonal_mp(X, y, tol=0), orthogonal_mp(X, y, n_nonzero_coefs=n_features)
+    )
     warning_message = (
         "Orthogonal matching pursuit ended prematurely "
         "due to linear dependence in the dictionary. "
@@ -84,23 +89,22 @@ def test_unreachable_accuracy():
     with pytest.warns(RuntimeWarning, match=warning_message):
         assert_array_almost_equal(
             orthogonal_mp(X, y, tol=0, precompute=True),
-            orthogonal_mp(X, y, precompute=True,
-                          n_nonzero_coefs=n_features))
+            orthogonal_mp(X, y, precompute=True, n_nonzero_coefs=n_features),
+        )
 
 
-def test_bad_input():
-    assert_raises(ValueError, orthogonal_mp, X, y, tol=-1)
-    assert_raises(ValueError, orthogonal_mp, X, y, n_nonzero_coefs=-1)
-    assert_raises(ValueError, orthogonal_mp, X, y,
-                  n_nonzero_coefs=n_features + 1)
-    assert_raises(ValueError, orthogonal_mp_gram, G, Xy, tol=-1)
-    assert_raises(ValueError, orthogonal_mp_gram, G, Xy, n_nonzero_coefs=-1)
-    assert_raises(ValueError, orthogonal_mp_gram, G, Xy,
-                  n_nonzero_coefs=n_features + 1)
+@pytest.mark.parametrize("positional_params", [(X, y), (G, Xy)])
+@pytest.mark.parametrize(
+    "keyword_params",
+    [{"n_nonzero_coefs": n_features + 1}],
+)
+def test_bad_input(positional_params, keyword_params):
+    with pytest.raises(ValueError):
+        orthogonal_mp(*positional_params, **keyword_params)
 
 
 def test_perfect_signal_recovery():
-    idx, = gamma[:, 0].nonzero()
+    (idx,) = gamma[:, 0].nonzero()
     gamma_rec = orthogonal_mp(X, y[:, 0], n_nonzero_coefs=5)
     gamma_gram = orthogonal_mp_gram(G, Xy[:, 0], n_nonzero_coefs=5)
     assert_array_equal(idx, np.flatnonzero(gamma_rec))
@@ -112,14 +116,14 @@ def test_perfect_signal_recovery():
 def test_orthogonal_mp_gram_readonly():
     # Non-regression test for:
     # https://github.com/scikit-learn/scikit-learn/issues/5956
-    idx, = gamma[:, 0].nonzero()
+    (idx,) = gamma[:, 0].nonzero()
     G_readonly = G.copy()
     G_readonly.setflags(write=False)
     Xy_readonly = Xy.copy()
     Xy_readonly.setflags(write=False)
-    gamma_gram = orthogonal_mp_gram(G_readonly, Xy_readonly[:, 0],
-                                    n_nonzero_coefs=5,
-                                    copy_Gram=False, copy_Xy=False)
+    gamma_gram = orthogonal_mp_gram(
+        G_readonly, Xy_readonly[:, 0], n_nonzero_coefs=5, copy_Gram=False, copy_Xy=False
+    )
     assert_array_equal(idx, np.flatnonzero(gamma_gram))
     assert_array_almost_equal(gamma[:, 0], gamma_gram, decimal=2)
 
@@ -137,11 +141,11 @@ def test_estimator():
     assert np.count_nonzero(omp.coef_) <= n_targets * n_nonzero_coefs
 
     coef_normalized = omp.coef_[0].copy()
-    omp.set_params(fit_intercept=True, normalize=False)
+    omp.set_params(fit_intercept=True)
     omp.fit(X, y[:, 0])
     assert_array_almost_equal(coef_normalized, omp.coef_)
 
-    omp.set_params(fit_intercept=False, normalize=False)
+    omp.set_params(fit_intercept=False)
     omp.fit(X, y[:, 0])
     assert np.count_nonzero(omp.coef_) <= n_nonzero_coefs
     assert omp.coef_.shape == (n_features,)
@@ -153,11 +157,22 @@ def test_estimator():
     assert np.count_nonzero(omp.coef_) <= n_targets * n_nonzero_coefs
 
 
+def test_estimator_n_nonzero_coefs():
+    """Check `n_nonzero_coefs_` correct when `tol` is and isn't set."""
+    omp = OrthogonalMatchingPursuit(n_nonzero_coefs=n_nonzero_coefs)
+    omp.fit(X, y[:, 0])
+    assert omp.n_nonzero_coefs_ == n_nonzero_coefs
+
+    omp = OrthogonalMatchingPursuit(n_nonzero_coefs=n_nonzero_coefs, tol=0.5)
+    omp.fit(X, y[:, 0])
+    assert omp.n_nonzero_coefs_ is None
+
+
 def test_identical_regressors():
     newX = X.copy()
     newX[:, 1] = newX[:, 0]
     gamma = np.zeros(n_features)
-    gamma[0] = gamma[1] = 1.
+    gamma[0] = gamma[1] = 1.0
     newy = np.dot(newX, gamma)
     warning_message = (
         "Orthogonal matching pursuit ended prematurely "
@@ -165,7 +180,7 @@ def test_identical_regressors():
         "The requested precision might not have been met."
     )
     with pytest.warns(RuntimeWarning, match=warning_message):
-        orthogonal_mp(newX, newy, 2)
+        orthogonal_mp(newX, newy, n_nonzero_coefs=2)
 
 
 def test_swapped_regressors():
@@ -186,10 +201,8 @@ def test_swapped_regressors():
 def test_no_atoms():
     y_empty = np.zeros_like(y)
     Xy_empty = np.dot(X.T, y_empty)
-    gamma_empty = ignore_warnings(orthogonal_mp)(X, y_empty,
-                                                 n_nonzero_coefs=1)
-    gamma_empty_gram = ignore_warnings(orthogonal_mp)(G, Xy_empty,
-                                                      n_nonzero_coefs=1)
+    gamma_empty = ignore_warnings(orthogonal_mp)(X, y_empty, n_nonzero_coefs=1)
+    gamma_empty_gram = ignore_warnings(orthogonal_mp)(G, Xy_empty, n_nonzero_coefs=1)
     assert np.all(gamma_empty == 0)
     assert np.all(gamma_empty_gram == 0)
 
@@ -206,10 +219,8 @@ def test_omp_path():
 
 
 def test_omp_return_path_prop_with_gram():
-    path = orthogonal_mp(X, y, n_nonzero_coefs=5, return_path=True,
-                         precompute=True)
-    last = orthogonal_mp(X, y, n_nonzero_coefs=5, return_path=False,
-                         precompute=True)
+    path = orthogonal_mp(X, y, n_nonzero_coefs=5, return_path=True, precompute=True)
+    last = orthogonal_mp(X, y, n_nonzero_coefs=5, return_path=False, precompute=True)
     assert path.shape == (n_features, n_targets, 5)
     assert_array_almost_equal(path[:, :, -1], last)
 
@@ -217,13 +228,13 @@ def test_omp_return_path_prop_with_gram():
 def test_omp_cv():
     y_ = y[:, 0]
     gamma_ = gamma[:, 0]
-    ompcv = OrthogonalMatchingPursuitCV(normalize=True, fit_intercept=False,
-                                        max_iter=10)
+    ompcv = OrthogonalMatchingPursuitCV(fit_intercept=False, max_iter=10)
     ompcv.fit(X, y_)
     assert ompcv.n_nonzero_coefs_ == n_nonzero_coefs
     assert_array_almost_equal(ompcv.coef_, gamma_)
-    omp = OrthogonalMatchingPursuit(normalize=True, fit_intercept=False,
-                                    n_nonzero_coefs=ompcv.n_nonzero_coefs_)
+    omp = OrthogonalMatchingPursuit(
+        fit_intercept=False, n_nonzero_coefs=ompcv.n_nonzero_coefs_
+    )
     omp.fit(X, y_)
     assert_array_almost_equal(ompcv.coef_, omp.coef_)
 
@@ -240,3 +251,23 @@ def test_omp_reaches_least_squares():
     omp.fit(X, Y)
     lstsq.fit(X, Y)
     assert_array_almost_equal(omp.coef_, lstsq.coef_)
+
+
+@pytest.mark.parametrize("data_type", (np.float32, np.float64))
+def test_omp_gram_dtype_match(data_type):
+    # verify matching input data type and output data type
+    coef = orthogonal_mp_gram(
+        G.astype(data_type), Xy.astype(data_type), n_nonzero_coefs=5
+    )
+    assert coef.dtype == data_type
+
+
+def test_omp_gram_numerical_consistency():
+    # verify numericaly consistency among np.float32 and np.float64
+    coef_32 = orthogonal_mp_gram(
+        G.astype(np.float32), Xy.astype(np.float32), n_nonzero_coefs=5
+    )
+    coef_64 = orthogonal_mp_gram(
+        G.astype(np.float32), Xy.astype(np.float64), n_nonzero_coefs=5
+    )
+    assert_allclose(coef_32, coef_64)
diff --git a/sklearn/linear_model/tests/test_passive_aggressive.py b/sklearn/linear_model/tests/test_passive_aggressive.py
index f2403773277a7..bcfd58b1eab2b 100644
--- a/sklearn/linear_model/tests/test_passive_aggressive.py
+++ b/sklearn/linear_model/tests/test_passive_aggressive.py
@@ -1,16 +1,16 @@
 import numpy as np
-import scipy.sparse as sp
-
 import pytest
 
-from sklearn.utils._testing import assert_array_almost_equal
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils._testing import assert_almost_equal
 from sklearn.base import ClassifierMixin
-from sklearn.utils import check_random_state
 from sklearn.datasets import load_iris
-from sklearn.linear_model import PassiveAggressiveClassifier
-from sklearn.linear_model import PassiveAggressiveRegressor
+from sklearn.linear_model import PassiveAggressiveClassifier, PassiveAggressiveRegressor
+from sklearn.utils import check_random_state
+from sklearn.utils._testing import (
+    assert_almost_equal,
+    assert_array_almost_equal,
+    assert_array_equal,
+)
+from sklearn.utils.fixes import CSR_CONTAINERS
 
 iris = load_iris()
 random_state = check_random_state(12)
@@ -18,13 +18,18 @@
 random_state.shuffle(indices)
 X = iris.data[indices]
 y = iris.target[indices]
-X_csr = sp.csr_matrix(X)
 
 
 class MyPassiveAggressive(ClassifierMixin):
-
-    def __init__(self, C=1.0, epsilon=0.01, loss="hinge",
-                 fit_intercept=True, n_iter=1, random_state=None):
+    def __init__(
+        self,
+        C=1.0,
+        epsilon=0.01,
+        loss="hinge",
+        fit_intercept=True,
+        n_iter=1,
+        random_state=None,
+    ):
         self.C = C
         self.epsilon = epsilon
         self.loss = loss
@@ -48,8 +53,7 @@ def fit(self, X, y):
 
                 if self.loss in ("hinge", "epsilon_insensitive"):
                     step = min(self.C, loss / sqnorm)
-                elif self.loss in ("squared_hinge",
-                                   "squared_epsilon_insensitive"):
+                elif self.loss in ("squared_hinge", "squared_epsilon_insensitive"):
                     step = loss / (sqnorm + 1.0 / (2 * self.C))
 
                 if self.loss in ("hinge", "squared_hinge"):
@@ -65,39 +69,44 @@ def project(self, X):
         return np.dot(X, self.w) + self.b
 
 
-def test_classifier_accuracy():
-    for data in (X, X_csr):
-        for fit_intercept in (True, False):
-            for average in (False, True):
-                clf = PassiveAggressiveClassifier(
-                    C=1.0, max_iter=30, fit_intercept=fit_intercept,
-                    random_state=1, average=average, tol=None)
-                clf.fit(data, y)
-                score = clf.score(data, y)
-                assert score > 0.79
-                if average:
-                    assert hasattr(clf, '_average_coef')
-                    assert hasattr(clf, '_average_intercept')
-                    assert hasattr(clf, '_standard_intercept')
-                    assert hasattr(clf, '_standard_coef')
-
-
-def test_classifier_partial_fit():
+@pytest.mark.parametrize("average", [False, True])
+@pytest.mark.parametrize("fit_intercept", [True, False])
+@pytest.mark.parametrize("csr_container", [None, *CSR_CONTAINERS])
+def test_classifier_accuracy(csr_container, fit_intercept, average):
+    data = csr_container(X) if csr_container is not None else X
+    clf = PassiveAggressiveClassifier(
+        C=1.0,
+        max_iter=30,
+        fit_intercept=fit_intercept,
+        random_state=1,
+        average=average,
+        tol=None,
+    )
+    clf.fit(data, y)
+    score = clf.score(data, y)
+    assert score > 0.79
+    if average:
+        assert hasattr(clf, "_average_coef")
+        assert hasattr(clf, "_average_intercept")
+        assert hasattr(clf, "_standard_intercept")
+        assert hasattr(clf, "_standard_coef")
+
+
+@pytest.mark.parametrize("average", [False, True])
+@pytest.mark.parametrize("csr_container", [None, *CSR_CONTAINERS])
+def test_classifier_partial_fit(csr_container, average):
     classes = np.unique(y)
-    for data in (X, X_csr):
-        for average in (False, True):
-            clf = PassiveAggressiveClassifier(random_state=0,
-                                              average=average,
-                                              max_iter=5)
-            for t in range(30):
-                clf.partial_fit(data, y, classes)
-            score = clf.score(data, y)
-            assert score > 0.79
-            if average:
-                assert hasattr(clf, '_average_coef')
-                assert hasattr(clf, '_average_intercept')
-                assert hasattr(clf, '_standard_intercept')
-                assert hasattr(clf, '_standard_coef')
+    data = csr_container(X) if csr_container is not None else X
+    clf = PassiveAggressiveClassifier(random_state=0, average=average, max_iter=5)
+    for t in range(30):
+        clf.partial_fit(data, y, classes)
+    score = clf.score(data, y)
+    assert score > 0.79
+    if average:
+        assert hasattr(clf, "_average_coef")
+        assert hasattr(clf, "_average_intercept")
+        assert hasattr(clf, "_standard_intercept")
+        assert hasattr(clf, "_standard_coef")
 
 
 def test_classifier_refit():
@@ -109,44 +118,46 @@ def test_classifier_refit():
     assert_array_equal(clf.classes_, iris.target_names)
 
 
-@pytest.mark.parametrize('loss', ("hinge", "squared_hinge"))
-def test_classifier_correctness(loss):
+@pytest.mark.parametrize("csr_container", [None, *CSR_CONTAINERS])
+@pytest.mark.parametrize("loss", ("hinge", "squared_hinge"))
+def test_classifier_correctness(loss, csr_container):
     y_bin = y.copy()
     y_bin[y != 1] = -1
 
     clf1 = MyPassiveAggressive(loss=loss, n_iter=2)
     clf1.fit(X, y_bin)
 
-    for data in (X, X_csr):
-        clf2 = PassiveAggressiveClassifier(loss=loss, max_iter=2,
-                                           shuffle=False, tol=None)
-        clf2.fit(data, y_bin)
+    data = csr_container(X) if csr_container is not None else X
+    clf2 = PassiveAggressiveClassifier(loss=loss, max_iter=2, shuffle=False, tol=None)
+    clf2.fit(data, y_bin)
 
-        assert_array_almost_equal(clf1.w, clf2.coef_.ravel(), decimal=2)
+    assert_array_almost_equal(clf1.w, clf2.coef_.ravel(), decimal=2)
 
 
-def test_classifier_undefined_methods():
+@pytest.mark.parametrize(
+    "response_method", ["predict_proba", "predict_log_proba", "transform"]
+)
+def test_classifier_undefined_methods(response_method):
     clf = PassiveAggressiveClassifier(max_iter=100)
-    for meth in ("predict_proba", "predict_log_proba", "transform"):
-        with pytest.raises(AttributeError):
-            getattr(clf, meth)
+    with pytest.raises(AttributeError):
+        getattr(clf, response_method)
 
 
 def test_class_weights():
     # Test class weights.
-    X2 = np.array([[-1.0, -1.0], [-1.0, 0], [-.8, -1.0],
-                   [1.0, 1.0], [1.0, 0.0]])
+    X2 = np.array([[-1.0, -1.0], [-1.0, 0], [-0.8, -1.0], [1.0, 1.0], [1.0, 0.0]])
     y2 = [1, 1, 1, -1, -1]
 
-    clf = PassiveAggressiveClassifier(C=0.1, max_iter=100, class_weight=None,
-                                      random_state=100)
+    clf = PassiveAggressiveClassifier(
+        C=0.1, max_iter=100, class_weight=None, random_state=100
+    )
     clf.fit(X2, y2)
     assert_array_equal(clf.predict([[0.2, -1.0]]), np.array([1]))
 
     # we give a small weights to class 1
-    clf = PassiveAggressiveClassifier(C=0.1, max_iter=100,
-                                      class_weight={1: 0.001},
-                                      random_state=100)
+    clf = PassiveAggressiveClassifier(
+        C=0.1, max_iter=100, class_weight={1: 0.001}, random_state=100
+    )
     clf.fit(X2, y2)
 
     # now the hyperplane should rotate clock-wise and
@@ -164,17 +175,16 @@ def test_partial_fit_weight_class_balanced():
 def test_equal_class_weight():
     X2 = [[1, 0], [1, 0], [0, 1], [0, 1]]
     y2 = [0, 0, 1, 1]
-    clf = PassiveAggressiveClassifier(
-        C=0.1, max_iter=1000, tol=None, class_weight=None)
+    clf = PassiveAggressiveClassifier(C=0.1, tol=None, class_weight=None)
     clf.fit(X2, y2)
 
     # Already balanced, so "balanced" weights should have no effect
-    clf_balanced = PassiveAggressiveClassifier(
-        C=0.1, max_iter=1000, tol=None, class_weight="balanced")
+    clf_balanced = PassiveAggressiveClassifier(C=0.1, tol=None, class_weight="balanced")
     clf_balanced.fit(X2, y2)
 
     clf_weighted = PassiveAggressiveClassifier(
-        C=0.1, max_iter=1000, tol=None, class_weight={0: 0.5, 1: 0.5})
+        C=0.1, tol=None, class_weight={0: 0.5, 1: 0.5}
+    )
     clf_weighted.fit(X2, y2)
 
     # should be similar up to some epsilon due to learning rate schedule
@@ -184,8 +194,7 @@ def test_equal_class_weight():
 
 def test_wrong_class_weight_label():
     # ValueError due to wrong class_weight label.
-    X2 = np.array([[-1.0, -1.0], [-1.0, 0], [-.8, -1.0],
-                   [1.0, 1.0], [1.0, 0.0]])
+    X2 = np.array([[-1.0, -1.0], [-1.0, 0], [-0.8, -1.0], [1.0, 1.0], [1.0, 0.0]])
     y2 = [1, 1, 1, -1, -1]
 
     clf = PassiveAggressiveClassifier(class_weight={0: 0.5}, max_iter=100)
@@ -193,94 +202,67 @@ def test_wrong_class_weight_label():
         clf.fit(X2, y2)
 
 
-def test_wrong_class_weight_format():
-    # ValueError due to wrong class_weight argument type.
-    X2 = np.array([[-1.0, -1.0], [-1.0, 0], [-.8, -1.0],
-                   [1.0, 1.0], [1.0, 0.0]])
-    y2 = [1, 1, 1, -1, -1]
-
-    clf = PassiveAggressiveClassifier(class_weight=[0.5], max_iter=100)
-    with pytest.raises(ValueError):
-        clf.fit(X2, y2)
-
-    clf = PassiveAggressiveClassifier(class_weight="the larch", max_iter=100)
-    with pytest.raises(ValueError):
-        clf.fit(X2, y2)
-
-
-def test_regressor_mse():
+@pytest.mark.parametrize("average", [False, True])
+@pytest.mark.parametrize("fit_intercept", [True, False])
+@pytest.mark.parametrize("csr_container", [None, *CSR_CONTAINERS])
+def test_regressor_mse(csr_container, fit_intercept, average):
     y_bin = y.copy()
     y_bin[y != 1] = -1
 
-    for data in (X, X_csr):
-        for fit_intercept in (True, False):
-            for average in (False, True):
-                reg = PassiveAggressiveRegressor(
-                    C=1.0, fit_intercept=fit_intercept,
-                    random_state=0, average=average, max_iter=5)
-                reg.fit(data, y_bin)
-                pred = reg.predict(data)
-                assert np.mean((pred - y_bin) ** 2) < 1.7
-                if average:
-                    assert hasattr(reg, '_average_coef')
-                    assert hasattr(reg, '_average_intercept')
-                    assert hasattr(reg, '_standard_intercept')
-                    assert hasattr(reg, '_standard_coef')
-
-
-def test_regressor_partial_fit():
+    data = csr_container(X) if csr_container is not None else X
+    reg = PassiveAggressiveRegressor(
+        C=1.0,
+        fit_intercept=fit_intercept,
+        random_state=0,
+        average=average,
+        max_iter=5,
+    )
+    reg.fit(data, y_bin)
+    pred = reg.predict(data)
+    assert np.mean((pred - y_bin) ** 2) < 1.7
+    if average:
+        assert hasattr(reg, "_average_coef")
+        assert hasattr(reg, "_average_intercept")
+        assert hasattr(reg, "_standard_intercept")
+        assert hasattr(reg, "_standard_coef")
+
+
+@pytest.mark.parametrize("average", [False, True])
+@pytest.mark.parametrize("csr_container", [None, *CSR_CONTAINERS])
+def test_regressor_partial_fit(csr_container, average):
     y_bin = y.copy()
     y_bin[y != 1] = -1
 
-    for data in (X, X_csr):
-        for average in (False, True):
-            reg = PassiveAggressiveRegressor(random_state=0,
-                                             average=average, max_iter=100)
-            for t in range(50):
-                reg.partial_fit(data, y_bin)
-            pred = reg.predict(data)
-            assert np.mean((pred - y_bin) ** 2) < 1.7
-            if average:
-                assert hasattr(reg, '_average_coef')
-                assert hasattr(reg, '_average_intercept')
-                assert hasattr(reg, '_standard_intercept')
-                assert hasattr(reg, '_standard_coef')
-
-
-@pytest.mark.parametrize(
-        'loss',
-        ("epsilon_insensitive", "squared_epsilon_insensitive"))
-def test_regressor_correctness(loss):
+    data = csr_container(X) if csr_container is not None else X
+    reg = PassiveAggressiveRegressor(random_state=0, average=average, max_iter=100)
+    for t in range(50):
+        reg.partial_fit(data, y_bin)
+    pred = reg.predict(data)
+    assert np.mean((pred - y_bin) ** 2) < 1.7
+    if average:
+        assert hasattr(reg, "_average_coef")
+        assert hasattr(reg, "_average_intercept")
+        assert hasattr(reg, "_standard_intercept")
+        assert hasattr(reg, "_standard_coef")
+
+
+@pytest.mark.parametrize("csr_container", [None, *CSR_CONTAINERS])
+@pytest.mark.parametrize("loss", ("epsilon_insensitive", "squared_epsilon_insensitive"))
+def test_regressor_correctness(loss, csr_container):
     y_bin = y.copy()
     y_bin[y != 1] = -1
 
     reg1 = MyPassiveAggressive(loss=loss, n_iter=2)
     reg1.fit(X, y_bin)
 
-    for data in (X, X_csr):
-        reg2 = PassiveAggressiveRegressor(tol=None, loss=loss, max_iter=2,
-                                          shuffle=False)
-        reg2.fit(data, y_bin)
+    data = csr_container(X) if csr_container is not None else X
+    reg2 = PassiveAggressiveRegressor(tol=None, loss=loss, max_iter=2, shuffle=False)
+    reg2.fit(data, y_bin)
 
-        assert_array_almost_equal(reg1.w, reg2.coef_.ravel(), decimal=2)
+    assert_array_almost_equal(reg1.w, reg2.coef_.ravel(), decimal=2)
 
 
 def test_regressor_undefined_methods():
     reg = PassiveAggressiveRegressor(max_iter=100)
-    for meth in ("transform",):
-        with pytest.raises(AttributeError):
-            getattr(reg, meth)
-
-
-# TODO: remove in 1.0
-@pytest.mark.parametrize('klass', [PassiveAggressiveClassifier,
-                                   PassiveAggressiveRegressor])
-def test_passive_aggressive_deprecated_attr(klass):
-    est = klass(average=True)
-    est.fit(X, y)
-
-    msg = "Attribute {} was deprecated"
-    for att in ['average_coef_', 'average_intercept_',
-                'standard_coef_', 'standard_intercept_']:
-        with pytest.warns(FutureWarning, match=msg.format(att)):
-            getattr(est, att)
+    with pytest.raises(AttributeError):
+        reg.transform(X)
diff --git a/sklearn/linear_model/tests/test_perceptron.py b/sklearn/linear_model/tests/test_perceptron.py
index f62595d7bc590..71456ae72132c 100644
--- a/sklearn/linear_model/tests/test_perceptron.py
+++ b/sklearn/linear_model/tests/test_perceptron.py
@@ -1,12 +1,11 @@
 import numpy as np
-import scipy.sparse as sp
 import pytest
 
-from sklearn.utils._testing import assert_allclose
-from sklearn.utils._testing import assert_array_almost_equal
-from sklearn.utils import check_random_state
 from sklearn.datasets import load_iris
 from sklearn.linear_model import Perceptron
+from sklearn.utils import check_random_state
+from sklearn.utils._testing import assert_allclose, assert_array_almost_equal
+from sklearn.utils.fixes import CSR_CONTAINERS
 
 iris = load_iris()
 random_state = check_random_state(12)
@@ -14,12 +13,9 @@
 random_state.shuffle(indices)
 X = iris.data[indices]
 y = iris.target[indices]
-X_csr = sp.csr_matrix(X)
-X_csr.sort_indices()
 
 
 class MyPerceptron:
-
     def __init__(self, n_iter=1):
         self.n_iter = n_iter
 
@@ -42,12 +38,13 @@ def predict(self, X):
         return np.sign(self.project(X))
 
 
-def test_perceptron_accuracy():
-    for data in (X, X_csr):
-        clf = Perceptron(max_iter=100, tol=None, shuffle=False)
-        clf.fit(data, y)
-        score = clf.score(data, y)
-        assert score > 0.7
+@pytest.mark.parametrize("container", CSR_CONTAINERS + [np.array])
+def test_perceptron_accuracy(container):
+    data = container(X)
+    clf = Perceptron(max_iter=100, tol=None, shuffle=False)
+    clf.fit(data, y)
+    score = clf.score(data, y)
+    assert score > 0.7
 
 
 def test_perceptron_correctness():
@@ -72,20 +69,20 @@ def test_undefined_methods():
 
 def test_perceptron_l1_ratio():
     """Check that `l1_ratio` has an impact when `penalty='elasticnet'`"""
-    clf1 = Perceptron(l1_ratio=0, penalty='elasticnet')
+    clf1 = Perceptron(l1_ratio=0, penalty="elasticnet")
     clf1.fit(X, y)
 
-    clf2 = Perceptron(l1_ratio=0.15, penalty='elasticnet')
+    clf2 = Perceptron(l1_ratio=0.15, penalty="elasticnet")
     clf2.fit(X, y)
 
     assert clf1.score(X, y) != clf2.score(X, y)
 
     # check that the bounds of elastic net which should correspond to an l1 or
     # l2 penalty depending of `l1_ratio` value.
-    clf_l1 = Perceptron(penalty='l1').fit(X, y)
-    clf_elasticnet = Perceptron(l1_ratio=1, penalty='elasticnet').fit(X, y)
+    clf_l1 = Perceptron(penalty="l1").fit(X, y)
+    clf_elasticnet = Perceptron(l1_ratio=1, penalty="elasticnet").fit(X, y)
     assert_allclose(clf_l1.coef_, clf_elasticnet.coef_)
 
-    clf_l2 = Perceptron(penalty='l2').fit(X, y)
-    clf_elasticnet = Perceptron(l1_ratio=0, penalty='elasticnet').fit(X, y)
+    clf_l2 = Perceptron(penalty="l2").fit(X, y)
+    clf_elasticnet = Perceptron(l1_ratio=0, penalty="elasticnet").fit(X, y)
     assert_allclose(clf_l2.coef_, clf_elasticnet.coef_)
diff --git a/sklearn/linear_model/tests/test_quantile.py b/sklearn/linear_model/tests/test_quantile.py
new file mode 100644
index 0000000000000..1d166b14091cc
--- /dev/null
+++ b/sklearn/linear_model/tests/test_quantile.py
@@ -0,0 +1,283 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+import numpy as np
+import pytest
+from pytest import approx
+from scipy.optimize import minimize
+
+from sklearn.datasets import make_regression
+from sklearn.exceptions import ConvergenceWarning
+from sklearn.linear_model import HuberRegressor, QuantileRegressor
+from sklearn.metrics import mean_pinball_loss
+from sklearn.utils._testing import assert_allclose
+from sklearn.utils.fixes import (
+    COO_CONTAINERS,
+    CSC_CONTAINERS,
+    CSR_CONTAINERS,
+    parse_version,
+    sp_version,
+)
+
+
+@pytest.fixture
+def X_y_data():
+    X, y = make_regression(n_samples=10, n_features=1, random_state=0, noise=1)
+    return X, y
+
+
+@pytest.mark.skipif(
+    parse_version(sp_version.base_version) >= parse_version("1.11"),
+    reason="interior-point solver is not available in SciPy 1.11",
+)
+@pytest.mark.parametrize("solver", ["interior-point", "revised simplex"])
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+def test_incompatible_solver_for_sparse_input(X_y_data, solver, csc_container):
+    X, y = X_y_data
+    X_sparse = csc_container(X)
+    err_msg = (
+        f"Solver {solver} does not support sparse X. Use solver 'highs' for example."
+    )
+    with pytest.raises(ValueError, match=err_msg):
+        QuantileRegressor(solver=solver).fit(X_sparse, y)
+
+
+@pytest.mark.parametrize(
+    "quantile, alpha, intercept, coef",
+    [
+        # for 50% quantile w/o regularization, any slope in [1, 10] is okay
+        [0.5, 0, 1, None],
+        # if positive error costs more, the slope is maximal
+        [0.51, 0, 1, 10],
+        # if negative error costs more, the slope is minimal
+        [0.49, 0, 1, 1],
+        # for a small lasso penalty, the slope is also minimal
+        [0.5, 0.01, 1, 1],
+        # for a large lasso penalty, the model predicts the constant median
+        [0.5, 100, 2, 0],
+    ],
+)
+def test_quantile_toy_example(quantile, alpha, intercept, coef):
+    # test how different parameters affect a small intuitive example
+    X = [[0], [1], [1]]
+    y = [1, 2, 11]
+    model = QuantileRegressor(quantile=quantile, alpha=alpha).fit(X, y)
+    assert_allclose(model.intercept_, intercept, atol=1e-2)
+    if coef is not None:
+        assert_allclose(model.coef_[0], coef, atol=1e-2)
+    if alpha < 100:
+        assert model.coef_[0] >= 1
+    assert model.coef_[0] <= 10
+
+
+@pytest.mark.parametrize("fit_intercept", [True, False])
+def test_quantile_equals_huber_for_low_epsilon(fit_intercept):
+    X, y = make_regression(n_samples=100, n_features=20, random_state=0, noise=1.0)
+    alpha = 1e-4
+    huber = HuberRegressor(
+        epsilon=1 + 1e-4, alpha=alpha, fit_intercept=fit_intercept
+    ).fit(X, y)
+    quant = QuantileRegressor(alpha=alpha, fit_intercept=fit_intercept).fit(X, y)
+    assert_allclose(huber.coef_, quant.coef_, atol=1e-1)
+    if fit_intercept:
+        assert huber.intercept_ == approx(quant.intercept_, abs=1e-1)
+        # check that we still predict fraction
+        assert np.mean(y < quant.predict(X)) == approx(0.5, abs=1e-1)
+
+
+@pytest.mark.parametrize("q", [0.5, 0.9, 0.05])
+def test_quantile_estimates_calibration(q):
+    # Test that model estimates percentage of points below the prediction
+    X, y = make_regression(n_samples=1000, n_features=20, random_state=0, noise=1.0)
+    quant = QuantileRegressor(quantile=q, alpha=0).fit(X, y)
+    assert np.mean(y < quant.predict(X)) == approx(q, abs=1e-2)
+
+
+def test_quantile_sample_weight():
+    # test that with unequal sample weights we still estimate weighted fraction
+    n = 1000
+    X, y = make_regression(n_samples=n, n_features=5, random_state=0, noise=10.0)
+    weight = np.ones(n)
+    # when we increase weight of upper observations,
+    # estimate of quantile should go up
+    weight[y > y.mean()] = 100
+    quant = QuantileRegressor(quantile=0.5, alpha=1e-8)
+    quant.fit(X, y, sample_weight=weight)
+    fraction_below = np.mean(y < quant.predict(X))
+    assert fraction_below > 0.5
+    weighted_fraction_below = np.average(y < quant.predict(X), weights=weight)
+    assert weighted_fraction_below == approx(0.5, abs=3e-2)
+
+
+@pytest.mark.parametrize("quantile", [0.2, 0.5, 0.8])
+def test_asymmetric_error(quantile):
+    """Test quantile regression for asymmetric distributed targets."""
+    n_samples = 1000
+    rng = np.random.RandomState(42)
+    X = np.concatenate(
+        (
+            np.abs(rng.randn(n_samples)[:, None]),
+            -rng.randint(2, size=(n_samples, 1)),
+        ),
+        axis=1,
+    )
+    intercept = 1.23
+    coef = np.array([0.5, -2])
+    #  Take care that X @ coef + intercept > 0
+    assert np.min(X @ coef + intercept) > 0
+    # For an exponential distribution with rate lambda, e.g. exp(-lambda * x),
+    # the quantile at level q is:
+    #   quantile(q) = - log(1 - q) / lambda
+    #   scale = 1/lambda = -quantile(q) / log(1 - q)
+    y = rng.exponential(
+        scale=-(X @ coef + intercept) / np.log(1 - quantile), size=n_samples
+    )
+    model = QuantileRegressor(
+        quantile=quantile,
+        alpha=0,
+    ).fit(X, y)
+    # This test can be made to pass with any solver but in the interest
+    # of sparing continuous integration resources, the test is performed
+    # with the fastest solver only.
+
+    assert model.intercept_ == approx(intercept, rel=0.2)
+    assert_allclose(model.coef_, coef, rtol=0.6)
+    assert_allclose(np.mean(model.predict(X) > y), quantile, atol=1e-2)
+
+    # Now compare to Nelder-Mead optimization with L1 penalty
+    alpha = 0.01
+    model.set_params(alpha=alpha).fit(X, y)
+    model_coef = np.r_[model.intercept_, model.coef_]
+
+    def func(coef):
+        loss = mean_pinball_loss(y, X @ coef[1:] + coef[0], alpha=quantile)
+        L1 = np.sum(np.abs(coef[1:]))
+        return loss + alpha * L1
+
+    res = minimize(
+        fun=func,
+        x0=[1, 0, -1],
+        method="Nelder-Mead",
+        tol=1e-12,
+        options={"maxiter": 2000},
+    )
+
+    assert func(model_coef) == approx(func(res.x))
+    assert_allclose(model.intercept_, res.x[0])
+    assert_allclose(model.coef_, res.x[1:])
+    assert_allclose(np.mean(model.predict(X) > y), quantile, atol=1e-2)
+
+
+@pytest.mark.parametrize("quantile", [0.2, 0.5, 0.8])
+def test_equivariance(quantile):
+    """Test equivariace of quantile regression.
+
+    See Koenker (2005) Quantile Regression, Chapter 2.2.3.
+    """
+    rng = np.random.RandomState(42)
+    n_samples, n_features = 100, 5
+    X, y = make_regression(
+        n_samples=n_samples,
+        n_features=n_features,
+        n_informative=n_features,
+        noise=0,
+        random_state=rng,
+        shuffle=False,
+    )
+    # make y asymmetric
+    y += rng.exponential(scale=100, size=y.shape)
+    params = dict(alpha=0)
+    model1 = QuantileRegressor(quantile=quantile, **params).fit(X, y)
+
+    # coef(q; a*y, X) = a * coef(q; y, X)
+    a = 2.5
+    model2 = QuantileRegressor(quantile=quantile, **params).fit(X, a * y)
+    assert model2.intercept_ == approx(a * model1.intercept_, rel=1e-5)
+    assert_allclose(model2.coef_, a * model1.coef_, rtol=1e-5)
+
+    # coef(1-q; -a*y, X) = -a * coef(q; y, X)
+    model2 = QuantileRegressor(quantile=1 - quantile, **params).fit(X, -a * y)
+    assert model2.intercept_ == approx(-a * model1.intercept_, rel=1e-5)
+    assert_allclose(model2.coef_, -a * model1.coef_, rtol=1e-5)
+
+    # coef(q; y + X @ g, X) = coef(q; y, X) + g
+    g_intercept, g_coef = rng.randn(), rng.randn(n_features)
+    model2 = QuantileRegressor(quantile=quantile, **params)
+    model2.fit(X, y + X @ g_coef + g_intercept)
+    assert model2.intercept_ == approx(model1.intercept_ + g_intercept)
+    assert_allclose(model2.coef_, model1.coef_ + g_coef, rtol=1e-6)
+
+    # coef(q; y, X @ A) = A^-1 @ coef(q; y, X)
+    A = rng.randn(n_features, n_features)
+    model2 = QuantileRegressor(quantile=quantile, **params)
+    model2.fit(X @ A, y)
+    assert model2.intercept_ == approx(model1.intercept_, rel=1e-5)
+    assert_allclose(model2.coef_, np.linalg.solve(A, model1.coef_), rtol=1e-5)
+
+
+@pytest.mark.skipif(
+    parse_version(sp_version.base_version) >= parse_version("1.11"),
+    reason="interior-point solver is not available in SciPy 1.11",
+)
+@pytest.mark.filterwarnings("ignore:`method='interior-point'` is deprecated")
+def test_linprog_failure():
+    """Test that linprog fails."""
+    X = np.linspace(0, 10, num=10).reshape(-1, 1)
+    y = np.linspace(0, 10, num=10)
+    reg = QuantileRegressor(
+        alpha=0, solver="interior-point", solver_options={"maxiter": 1}
+    )
+
+    msg = "Linear programming for QuantileRegressor did not succeed."
+    with pytest.warns(ConvergenceWarning, match=msg):
+        reg.fit(X, y)
+
+
+@pytest.mark.parametrize(
+    "sparse_container", CSC_CONTAINERS + CSR_CONTAINERS + COO_CONTAINERS
+)
+@pytest.mark.parametrize("solver", ["highs", "highs-ds", "highs-ipm"])
+@pytest.mark.parametrize("fit_intercept", [True, False])
+def test_sparse_input(sparse_container, solver, fit_intercept, global_random_seed):
+    """Test that sparse and dense X give same results."""
+    n_informative = 10
+    quantile_level = 0.6
+    X, y = make_regression(
+        n_samples=300,
+        n_features=20,
+        n_informative=10,
+        random_state=global_random_seed,
+        noise=1.0,
+    )
+    X_sparse = sparse_container(X)
+    alpha = 0.1
+    quant_dense = QuantileRegressor(
+        quantile=quantile_level, alpha=alpha, fit_intercept=fit_intercept
+    ).fit(X, y)
+    quant_sparse = QuantileRegressor(
+        quantile=quantile_level, alpha=alpha, fit_intercept=fit_intercept, solver=solver
+    ).fit(X_sparse, y)
+    assert_allclose(quant_sparse.coef_, quant_dense.coef_, rtol=1e-2)
+    sparse_support = quant_sparse.coef_ != 0
+    dense_support = quant_dense.coef_ != 0
+    assert dense_support.sum() == pytest.approx(n_informative, abs=1)
+    assert sparse_support.sum() == pytest.approx(n_informative, abs=1)
+    if fit_intercept:
+        assert quant_sparse.intercept_ == approx(quant_dense.intercept_)
+        # check that we still predict fraction
+        empirical_coverage = np.mean(y < quant_sparse.predict(X_sparse))
+        assert empirical_coverage == approx(quantile_level, abs=3e-2)
+
+
+def test_error_interior_point_future(X_y_data, monkeypatch):
+    """Check that we will raise a proper error when requesting
+    `solver='interior-point'` in SciPy >= 1.11.
+    """
+    X, y = X_y_data
+    import sklearn.linear_model._quantile
+
+    with monkeypatch.context() as m:
+        m.setattr(sklearn.linear_model._quantile, "sp_version", parse_version("1.11.0"))
+        err_msg = "Solver interior-point is not anymore available in SciPy >= 1.11.0."
+        with pytest.raises(ValueError, match=err_msg):
+            QuantileRegressor(solver="interior-point").fit(X, y)
diff --git a/sklearn/linear_model/tests/test_ransac.py b/sklearn/linear_model/tests/test_ransac.py
index f631199a5d268..7b2bc66160ef3 100644
--- a/sklearn/linear_model/tests/test_ransac.py
+++ b/sklearn/linear_model/tests/test_ransac.py
@@ -1,19 +1,19 @@
 import numpy as np
 import pytest
-from scipy import sparse
+from numpy.testing import assert_array_almost_equal, assert_array_equal
 
-from numpy.testing import assert_array_almost_equal
-from numpy.testing import assert_array_equal
-
-from sklearn.utils import check_random_state
-from sklearn.utils._testing import assert_raises_regexp
-from sklearn.utils._testing import assert_allclose
 from sklearn.datasets import make_regression
-from sklearn.linear_model import LinearRegression, RANSACRegressor
-from sklearn.linear_model import OrthogonalMatchingPursuit
-from sklearn.linear_model._ransac import _dynamic_max_trials
 from sklearn.exceptions import ConvergenceWarning
-
+from sklearn.linear_model import (
+    LinearRegression,
+    OrthogonalMatchingPursuit,
+    RANSACRegressor,
+    Ridge,
+)
+from sklearn.linear_model._ransac import _dynamic_max_trials
+from sklearn.utils import check_random_state
+from sklearn.utils._testing import assert_allclose
+from sklearn.utils.fixes import COO_CONTAINERS, CSC_CONTAINERS, CSR_CONTAINERS
 
 # Generate coordinates of line
 X = np.arange(-200, 200)
@@ -30,17 +30,16 @@
 
 
 def test_ransac_inliers_outliers():
-
-    base_estimator = LinearRegression()
-    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
-                                       residual_threshold=5, random_state=0)
+    estimator = LinearRegression()
+    ransac_estimator = RANSACRegressor(
+        estimator, min_samples=2, residual_threshold=5, random_state=0
+    )
 
     # Estimate parameters of corrupted data
     ransac_estimator.fit(X, y)
 
     # Ground truth / reference inlier mask
-    ref_inlier_mask = np.ones_like(ransac_estimator.inlier_mask_
-                                   ).astype(np.bool_)
+    ref_inlier_mask = np.ones_like(ransac_estimator.inlier_mask_).astype(np.bool_)
     ref_inlier_mask[outliers] = False
 
     assert_array_equal(ransac_estimator.inlier_mask_, ref_inlier_mask)
@@ -56,11 +55,14 @@ def is_data_valid(X, y):
     X = rng.rand(10, 2)
     y = rng.rand(10, 1)
 
-    base_estimator = LinearRegression()
-    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
-                                       residual_threshold=5,
-                                       is_data_valid=is_data_valid,
-                                       random_state=0)
+    estimator = LinearRegression()
+    ransac_estimator = RANSACRegressor(
+        estimator,
+        min_samples=2,
+        residual_threshold=5,
+        is_data_valid=is_data_valid,
+        random_state=0,
+    )
     with pytest.raises(ValueError):
         ransac_estimator.fit(X, y)
 
@@ -71,30 +73,36 @@ def is_model_valid(estimator, X, y):
         assert y.shape[0] == 2
         return False
 
-    base_estimator = LinearRegression()
-    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
-                                       residual_threshold=5,
-                                       is_model_valid=is_model_valid,
-                                       random_state=0)
+    estimator = LinearRegression()
+    ransac_estimator = RANSACRegressor(
+        estimator,
+        min_samples=2,
+        residual_threshold=5,
+        is_model_valid=is_model_valid,
+        random_state=0,
+    )
     with pytest.raises(ValueError):
         ransac_estimator.fit(X, y)
 
 
 def test_ransac_max_trials():
-    base_estimator = LinearRegression()
-
-    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
-                                       residual_threshold=5, max_trials=0,
-                                       random_state=0)
+    estimator = LinearRegression()
+
+    ransac_estimator = RANSACRegressor(
+        estimator,
+        min_samples=2,
+        residual_threshold=5,
+        max_trials=0,
+        random_state=0,
+    )
     with pytest.raises(ValueError):
         ransac_estimator.fit(X, y)
 
     # there is a 1e-9 chance it will take these many trials. No good reason
     # 1e-2 isn't enough, can still happen
     # 2 is the what ransac defines  as min_samples = X.shape[1] + 1
-    max_trials = _dynamic_max_trials(
-        len(X) - len(outliers), X.shape[0], 2, 1 - 1e-9)
-    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2)
+    max_trials = _dynamic_max_trials(len(X) - len(outliers), X.shape[0], 2, 1 - 1e-9)
+    ransac_estimator = RANSACRegressor(estimator, min_samples=2)
     for i in range(50):
         ransac_estimator.set_params(min_samples=2, random_state=i)
         ransac_estimator.fit(X, y)
@@ -102,20 +110,28 @@ def test_ransac_max_trials():
 
 
 def test_ransac_stop_n_inliers():
-    base_estimator = LinearRegression()
-    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
-                                       residual_threshold=5, stop_n_inliers=2,
-                                       random_state=0)
+    estimator = LinearRegression()
+    ransac_estimator = RANSACRegressor(
+        estimator,
+        min_samples=2,
+        residual_threshold=5,
+        stop_n_inliers=2,
+        random_state=0,
+    )
     ransac_estimator.fit(X, y)
 
     assert ransac_estimator.n_trials_ == 1
 
 
 def test_ransac_stop_score():
-    base_estimator = LinearRegression()
-    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
-                                       residual_threshold=5, stop_score=0,
-                                       random_state=0)
+    estimator = LinearRegression()
+    ransac_estimator = RANSACRegressor(
+        estimator,
+        min_samples=2,
+        residual_threshold=5,
+        stop_score=0,
+        random_state=0,
+    )
     ransac_estimator.fit(X, y)
 
     assert ransac_estimator.n_trials_ == 1
@@ -123,13 +139,14 @@ def test_ransac_stop_score():
 
 def test_ransac_score():
     X = np.arange(100)[:, None]
-    y = np.zeros((100, ))
+    y = np.zeros((100,))
     y[0] = 1
     y[1] = 100
 
-    base_estimator = LinearRegression()
-    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
-                                       residual_threshold=0.5, random_state=0)
+    estimator = LinearRegression()
+    ransac_estimator = RANSACRegressor(
+        estimator, min_samples=2, residual_threshold=0.5, random_state=0
+    )
     ransac_estimator.fit(X, y)
 
     assert ransac_estimator.score(X[2:], y[2:]) == 1
@@ -138,44 +155,31 @@ def test_ransac_score():
 
 def test_ransac_predict():
     X = np.arange(100)[:, None]
-    y = np.zeros((100, ))
+    y = np.zeros((100,))
     y[0] = 1
     y[1] = 100
 
-    base_estimator = LinearRegression()
-    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
-                                       residual_threshold=0.5, random_state=0)
+    estimator = LinearRegression()
+    ransac_estimator = RANSACRegressor(
+        estimator, min_samples=2, residual_threshold=0.5, random_state=0
+    )
     ransac_estimator.fit(X, y)
 
     assert_array_equal(ransac_estimator.predict(X), np.zeros(100))
 
 
-def test_ransac_resid_thresh_no_inliers():
-    # When residual_threshold=0.0 there are no inliers and a
-    # ValueError with a message should be raised
-    base_estimator = LinearRegression()
-    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
-                                       residual_threshold=0.0, random_state=0,
-                                       max_trials=5)
-
-    msg = ("RANSAC could not find a valid consensus set")
-    assert_raises_regexp(ValueError, msg, ransac_estimator.fit, X, y)
-    assert ransac_estimator.n_skips_no_inliers_ == 5
-    assert ransac_estimator.n_skips_invalid_data_ == 0
-    assert ransac_estimator.n_skips_invalid_model_ == 0
-
-
 def test_ransac_no_valid_data():
     def is_data_valid(X, y):
         return False
 
-    base_estimator = LinearRegression()
-    ransac_estimator = RANSACRegressor(base_estimator,
-                                       is_data_valid=is_data_valid,
-                                       max_trials=5)
+    estimator = LinearRegression()
+    ransac_estimator = RANSACRegressor(
+        estimator, is_data_valid=is_data_valid, max_trials=5
+    )
 
-    msg = ("RANSAC could not find a valid consensus set")
-    assert_raises_regexp(ValueError, msg, ransac_estimator.fit, X, y)
+    msg = "RANSAC could not find a valid consensus set"
+    with pytest.raises(ValueError, match=msg):
+        ransac_estimator.fit(X, y)
     assert ransac_estimator.n_skips_no_inliers_ == 0
     assert ransac_estimator.n_skips_invalid_data_ == 5
     assert ransac_estimator.n_skips_invalid_model_ == 0
@@ -185,13 +189,14 @@ def test_ransac_no_valid_model():
     def is_model_valid(estimator, X, y):
         return False
 
-    base_estimator = LinearRegression()
-    ransac_estimator = RANSACRegressor(base_estimator,
-                                       is_model_valid=is_model_valid,
-                                       max_trials=5)
+    estimator = LinearRegression()
+    ransac_estimator = RANSACRegressor(
+        estimator, is_model_valid=is_model_valid, max_trials=5
+    )
 
-    msg = ("RANSAC could not find a valid consensus set")
-    assert_raises_regexp(ValueError, msg, ransac_estimator.fit, X, y)
+    msg = "RANSAC could not find a valid consensus set"
+    with pytest.raises(ValueError, match=msg):
+        ransac_estimator.fit(X, y)
     assert ransac_estimator.n_skips_no_inliers_ == 0
     assert ransac_estimator.n_skips_invalid_data_ == 0
     assert ransac_estimator.n_skips_invalid_model_ == 5
@@ -201,14 +206,14 @@ def test_ransac_exceed_max_skips():
     def is_data_valid(X, y):
         return False
 
-    base_estimator = LinearRegression()
-    ransac_estimator = RANSACRegressor(base_estimator,
-                                       is_data_valid=is_data_valid,
-                                       max_trials=5,
-                                       max_skips=3)
+    estimator = LinearRegression()
+    ransac_estimator = RANSACRegressor(
+        estimator, is_data_valid=is_data_valid, max_trials=5, max_skips=3
+    )
 
-    msg = ("RANSAC skipped more iterations than `max_skips`")
-    assert_raises_regexp(ValueError, msg, ransac_estimator.fit, X, y)
+    msg = "RANSAC skipped more iterations than `max_skips`"
+    with pytest.raises(ValueError, match=msg):
+        ransac_estimator.fit(X, y)
     assert ransac_estimator.n_skips_no_inliers_ == 0
     assert ransac_estimator.n_skips_invalid_data_ == 4
     assert ransac_estimator.n_skips_invalid_model_ == 0
@@ -226,11 +231,10 @@ def is_data_valid(X, y):
         else:
             return False
 
-    base_estimator = LinearRegression()
-    ransac_estimator = RANSACRegressor(base_estimator,
-                                       is_data_valid=is_data_valid,
-                                       max_skips=3,
-                                       max_trials=5)
+    estimator = LinearRegression()
+    ransac_estimator = RANSACRegressor(
+        estimator, is_data_valid=is_data_valid, max_skips=3, max_trials=5
+    )
     warning_message = (
         "RANSAC found a valid consensus set but exited "
         "early due to skipping more iterations than "
@@ -244,114 +248,93 @@ def is_data_valid(X, y):
     assert ransac_estimator.n_skips_invalid_model_ == 0
 
 
-def test_ransac_sparse_coo():
-    X_sparse = sparse.coo_matrix(X)
-
-    base_estimator = LinearRegression()
-    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
-                                       residual_threshold=5, random_state=0)
-    ransac_estimator.fit(X_sparse, y)
-
-    ref_inlier_mask = np.ones_like(ransac_estimator.inlier_mask_
-                                   ).astype(np.bool_)
-    ref_inlier_mask[outliers] = False
-
-    assert_array_equal(ransac_estimator.inlier_mask_, ref_inlier_mask)
-
-
-def test_ransac_sparse_csr():
-    X_sparse = sparse.csr_matrix(X)
-
-    base_estimator = LinearRegression()
-    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
-                                       residual_threshold=5, random_state=0)
-    ransac_estimator.fit(X_sparse, y)
-
-    ref_inlier_mask = np.ones_like(ransac_estimator.inlier_mask_
-                                   ).astype(np.bool_)
-    ref_inlier_mask[outliers] = False
-
-    assert_array_equal(ransac_estimator.inlier_mask_, ref_inlier_mask)
-
-
-def test_ransac_sparse_csc():
-    X_sparse = sparse.csc_matrix(X)
+@pytest.mark.parametrize(
+    "sparse_container", COO_CONTAINERS + CSR_CONTAINERS + CSC_CONTAINERS
+)
+def test_ransac_sparse(sparse_container):
+    X_sparse = sparse_container(X)
 
-    base_estimator = LinearRegression()
-    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
-                                       residual_threshold=5, random_state=0)
+    estimator = LinearRegression()
+    ransac_estimator = RANSACRegressor(
+        estimator, min_samples=2, residual_threshold=5, random_state=0
+    )
     ransac_estimator.fit(X_sparse, y)
 
-    ref_inlier_mask = np.ones_like(ransac_estimator.inlier_mask_
-                                   ).astype(np.bool_)
+    ref_inlier_mask = np.ones_like(ransac_estimator.inlier_mask_).astype(np.bool_)
     ref_inlier_mask[outliers] = False
 
     assert_array_equal(ransac_estimator.inlier_mask_, ref_inlier_mask)
 
 
 def test_ransac_none_estimator():
+    estimator = LinearRegression()
 
-    base_estimator = LinearRegression()
-
-    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
-                                       residual_threshold=5, random_state=0)
-    ransac_none_estimator = RANSACRegressor(None, min_samples=2,
-                                            residual_threshold=5,
-                                            random_state=0)
+    ransac_estimator = RANSACRegressor(
+        estimator, min_samples=2, residual_threshold=5, random_state=0
+    )
+    ransac_none_estimator = RANSACRegressor(
+        None, min_samples=2, residual_threshold=5, random_state=0
+    )
 
     ransac_estimator.fit(X, y)
     ransac_none_estimator.fit(X, y)
 
-    assert_array_almost_equal(ransac_estimator.predict(X),
-                              ransac_none_estimator.predict(X))
+    assert_array_almost_equal(
+        ransac_estimator.predict(X), ransac_none_estimator.predict(X)
+    )
 
 
 def test_ransac_min_n_samples():
-    base_estimator = LinearRegression()
-    ransac_estimator1 = RANSACRegressor(base_estimator, min_samples=2,
-                                        residual_threshold=5, random_state=0)
-    ransac_estimator2 = RANSACRegressor(base_estimator,
-                                        min_samples=2. / X.shape[0],
-                                        residual_threshold=5, random_state=0)
-    ransac_estimator3 = RANSACRegressor(base_estimator, min_samples=-1,
-                                        residual_threshold=5, random_state=0)
-    ransac_estimator4 = RANSACRegressor(base_estimator, min_samples=5.2,
-                                        residual_threshold=5, random_state=0)
-    ransac_estimator5 = RANSACRegressor(base_estimator, min_samples=2.0,
-                                        residual_threshold=5, random_state=0)
-    ransac_estimator6 = RANSACRegressor(base_estimator,
-                                        residual_threshold=5, random_state=0)
-    ransac_estimator7 = RANSACRegressor(base_estimator,
-                                        min_samples=X.shape[0] + 1,
-                                        residual_threshold=5, random_state=0)
+    estimator = LinearRegression()
+    ransac_estimator1 = RANSACRegressor(
+        estimator, min_samples=2, residual_threshold=5, random_state=0
+    )
+    ransac_estimator2 = RANSACRegressor(
+        estimator,
+        min_samples=2.0 / X.shape[0],
+        residual_threshold=5,
+        random_state=0,
+    )
+    ransac_estimator5 = RANSACRegressor(
+        estimator, min_samples=2, residual_threshold=5, random_state=0
+    )
+    ransac_estimator6 = RANSACRegressor(estimator, residual_threshold=5, random_state=0)
+    ransac_estimator7 = RANSACRegressor(
+        estimator, min_samples=X.shape[0] + 1, residual_threshold=5, random_state=0
+    )
+    # GH #19390
+    ransac_estimator8 = RANSACRegressor(
+        Ridge(), min_samples=None, residual_threshold=5, random_state=0
+    )
 
     ransac_estimator1.fit(X, y)
     ransac_estimator2.fit(X, y)
     ransac_estimator5.fit(X, y)
     ransac_estimator6.fit(X, y)
 
-    assert_array_almost_equal(ransac_estimator1.predict(X),
-                              ransac_estimator2.predict(X))
-    assert_array_almost_equal(ransac_estimator1.predict(X),
-                              ransac_estimator5.predict(X))
-    assert_array_almost_equal(ransac_estimator1.predict(X),
-                              ransac_estimator6.predict(X))
-
-    with pytest.raises(ValueError):
-        ransac_estimator3.fit(X, y)
-
-    with pytest.raises(ValueError):
-        ransac_estimator4.fit(X, y)
+    assert_array_almost_equal(
+        ransac_estimator1.predict(X), ransac_estimator2.predict(X)
+    )
+    assert_array_almost_equal(
+        ransac_estimator1.predict(X), ransac_estimator5.predict(X)
+    )
+    assert_array_almost_equal(
+        ransac_estimator1.predict(X), ransac_estimator6.predict(X)
+    )
 
     with pytest.raises(ValueError):
         ransac_estimator7.fit(X, y)
 
+    err_msg = "`min_samples` needs to be explicitly set"
+    with pytest.raises(ValueError, match=err_msg):
+        ransac_estimator8.fit(X, y)
 
-def test_ransac_multi_dimensional_targets():
 
-    base_estimator = LinearRegression()
-    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
-                                       residual_threshold=5, random_state=0)
+def test_ransac_multi_dimensional_targets():
+    estimator = LinearRegression()
+    ransac_estimator = RANSACRegressor(
+        estimator, min_samples=2, residual_threshold=5, random_state=0
+    )
 
     # 3-D target values
     yyy = np.column_stack([y, y, y])
@@ -360,8 +343,7 @@ def test_ransac_multi_dimensional_targets():
     ransac_estimator.fit(X, yyy)
 
     # Ground truth / reference inlier mask
-    ref_inlier_mask = np.ones_like(ransac_estimator.inlier_mask_
-                                   ).astype(np.bool_)
+    ref_inlier_mask = np.ones_like(ransac_estimator.inlier_mask_).astype(np.bool_)
     ref_inlier_mask[outliers] = False
 
     assert_array_equal(ransac_estimator.inlier_mask_, ref_inlier_mask)
@@ -379,50 +361,65 @@ def loss_mono(y_true, y_pred):
 
     yyy = np.column_stack([y, y, y])
 
-    base_estimator = LinearRegression()
-    ransac_estimator0 = RANSACRegressor(base_estimator, min_samples=2,
-                                        residual_threshold=5, random_state=0)
-    ransac_estimator1 = RANSACRegressor(base_estimator, min_samples=2,
-                                        residual_threshold=5, random_state=0,
-                                        loss=loss_multi1)
-    ransac_estimator2 = RANSACRegressor(base_estimator, min_samples=2,
-                                        residual_threshold=5, random_state=0,
-                                        loss=loss_multi2)
+    estimator = LinearRegression()
+    ransac_estimator0 = RANSACRegressor(
+        estimator, min_samples=2, residual_threshold=5, random_state=0
+    )
+    ransac_estimator1 = RANSACRegressor(
+        estimator,
+        min_samples=2,
+        residual_threshold=5,
+        random_state=0,
+        loss=loss_multi1,
+    )
+    ransac_estimator2 = RANSACRegressor(
+        estimator,
+        min_samples=2,
+        residual_threshold=5,
+        random_state=0,
+        loss=loss_multi2,
+    )
 
     # multi-dimensional
     ransac_estimator0.fit(X, yyy)
     ransac_estimator1.fit(X, yyy)
     ransac_estimator2.fit(X, yyy)
-    assert_array_almost_equal(ransac_estimator0.predict(X),
-                              ransac_estimator1.predict(X))
-    assert_array_almost_equal(ransac_estimator0.predict(X),
-                              ransac_estimator2.predict(X))
+    assert_array_almost_equal(
+        ransac_estimator0.predict(X), ransac_estimator1.predict(X)
+    )
+    assert_array_almost_equal(
+        ransac_estimator0.predict(X), ransac_estimator2.predict(X)
+    )
 
     # one-dimensional
     ransac_estimator0.fit(X, y)
     ransac_estimator2.loss = loss_mono
     ransac_estimator2.fit(X, y)
-    assert_array_almost_equal(ransac_estimator0.predict(X),
-                              ransac_estimator2.predict(X))
-    ransac_estimator3 = RANSACRegressor(base_estimator, min_samples=2,
-                                        residual_threshold=5, random_state=0,
-                                        loss="squared_loss")
+    assert_array_almost_equal(
+        ransac_estimator0.predict(X), ransac_estimator2.predict(X)
+    )
+    ransac_estimator3 = RANSACRegressor(
+        estimator,
+        min_samples=2,
+        residual_threshold=5,
+        random_state=0,
+        loss="squared_error",
+    )
     ransac_estimator3.fit(X, y)
-    assert_array_almost_equal(ransac_estimator0.predict(X),
-                              ransac_estimator2.predict(X))
+    assert_array_almost_equal(
+        ransac_estimator0.predict(X), ransac_estimator2.predict(X)
+    )
 
 
 def test_ransac_default_residual_threshold():
-    base_estimator = LinearRegression()
-    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
-                                       random_state=0)
+    estimator = LinearRegression()
+    ransac_estimator = RANSACRegressor(estimator, min_samples=2, random_state=0)
 
     # Estimate parameters of corrupted data
     ransac_estimator.fit(X, y)
 
     # Ground truth / reference inlier mask
-    ref_inlier_mask = np.ones_like(ransac_estimator.inlier_mask_
-                                   ).astype(np.bool_)
+    ref_inlier_mask = np.ones_like(ransac_estimator.inlier_mask_).astype(np.bool_)
     ref_inlier_mask[outliers] = False
 
     assert_array_equal(ransac_estimator.inlier_mask_, ref_inlier_mask)
@@ -457,31 +454,18 @@ def test_ransac_dynamic_max_trials():
 
     # e = 0%, min_samples = 10
     assert _dynamic_max_trials(1, 100, 10, 0) == 0
-    assert _dynamic_max_trials(1, 100, 10, 1) == float('inf')
-
-    base_estimator = LinearRegression()
-    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
-                                       stop_probability=-0.1)
-
-    with pytest.raises(ValueError):
-        ransac_estimator.fit(X, y)
-
-    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
-                                       stop_probability=1.1)
-    with pytest.raises(ValueError):
-        ransac_estimator.fit(X, y)
+    assert _dynamic_max_trials(1, 100, 10, 1) == float("inf")
 
 
 def test_ransac_fit_sample_weight():
     ransac_estimator = RANSACRegressor(random_state=0)
     n_samples = y.shape[0]
     weights = np.ones(n_samples)
-    ransac_estimator.fit(X, y, weights)
+    ransac_estimator.fit(X, y, sample_weight=weights)
     # sanity check
     assert ransac_estimator.inlier_mask_.shape[0] == n_samples
 
-    ref_inlier_mask = np.ones_like(ransac_estimator.inlier_mask_
-                                   ).astype(np.bool_)
+    ref_inlier_mask = np.ones_like(ransac_estimator.inlier_mask_).astype(np.bool_)
     ref_inlier_mask[outliers] = False
     # check that mask is correct
     assert_array_equal(ransac_estimator.inlier_mask_, ref_inlier_mask)
@@ -496,28 +480,36 @@ def test_ransac_fit_sample_weight():
     outlier_weight = random_state.randint(0, 10, 1)
     outlier_y = random_state.randint(-1000, 0, 1)
 
-    X_flat = np.append(np.repeat(X_, sample_weight, axis=0),
-                       np.repeat(outlier_X, outlier_weight, axis=0), axis=0)
-    y_flat = np.ndarray.flatten(np.append(np.repeat(y_, sample_weight, axis=0),
-                                np.repeat(outlier_y, outlier_weight, axis=0),
-                                          axis=0))
+    X_flat = np.append(
+        np.repeat(X_, sample_weight, axis=0),
+        np.repeat(outlier_X, outlier_weight, axis=0),
+        axis=0,
+    )
+    y_flat = np.ndarray.flatten(
+        np.append(
+            np.repeat(y_, sample_weight, axis=0),
+            np.repeat(outlier_y, outlier_weight, axis=0),
+            axis=0,
+        )
+    )
     ransac_estimator.fit(X_flat, y_flat)
     ref_coef_ = ransac_estimator.estimator_.coef_
 
     sample_weight = np.append(sample_weight, outlier_weight)
     X_ = np.append(X_, outlier_X, axis=0)
     y_ = np.append(y_, outlier_y)
-    ransac_estimator.fit(X_, y_, sample_weight)
+    ransac_estimator.fit(X_, y_, sample_weight=sample_weight)
 
     assert_allclose(ransac_estimator.estimator_.coef_, ref_coef_)
 
-    # check that if base_estimator.fit doesn't support
+    # check that if estimator.fit doesn't support
     # sample_weight, raises error
-    base_estimator = OrthogonalMatchingPursuit()
-    ransac_estimator = RANSACRegressor(base_estimator)
+    estimator = OrthogonalMatchingPursuit()
+    ransac_estimator = RANSACRegressor(estimator, min_samples=10)
 
-    with pytest.raises(ValueError):
-        ransac_estimator.fit(X, y, weights)
+    err_msg = f"{estimator.__class__.__name__} does not support sample_weight."
+    with pytest.raises(ValueError, match=err_msg):
+        ransac_estimator.fit(X, y, sample_weight=weights)
 
 
 def test_ransac_final_model_fit_sample_weight():
@@ -525,14 +517,29 @@ def test_ransac_final_model_fit_sample_weight():
     rng = check_random_state(42)
     sample_weight = rng.randint(1, 4, size=y.shape[0])
     sample_weight = sample_weight / sample_weight.sum()
-    ransac = RANSACRegressor(base_estimator=LinearRegression(), random_state=0)
+    ransac = RANSACRegressor(random_state=0)
     ransac.fit(X, y, sample_weight=sample_weight)
 
     final_model = LinearRegression()
     mask_samples = ransac.inlier_mask_
     final_model.fit(
-        X[mask_samples], y[mask_samples],
-        sample_weight=sample_weight[mask_samples]
+        X[mask_samples], y[mask_samples], sample_weight=sample_weight[mask_samples]
     )
 
     assert_allclose(ransac.estimator_.coef_, final_model.coef_, atol=1e-12)
+
+
+def test_perfect_horizontal_line():
+    """Check that we can fit a line where all samples are inliers.
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/19497
+    """
+    X = np.arange(100)[:, None]
+    y = np.zeros((100,))
+
+    estimator = LinearRegression()
+    ransac_estimator = RANSACRegressor(estimator, random_state=0)
+    ransac_estimator.fit(X, y)
+
+    assert_allclose(ransac_estimator.estimator_.coef_, 0.0)
+    assert_allclose(ransac_estimator.estimator_.intercept_, 0.0)
diff --git a/sklearn/linear_model/tests/test_ridge.py b/sklearn/linear_model/tests/test_ridge.py
index 01839fe0ba457..24515195fb7cc 100644
--- a/sklearn/linear_model/tests/test_ridge.py
+++ b/sklearn/linear_model/tests/test_ridge.py
@@ -1,46 +1,78 @@
-import numpy as np
-import scipy.sparse as sp
-from scipy import linalg
+import warnings
 from itertools import product
 
+import numpy as np
 import pytest
+from scipy import linalg
 
-from sklearn.utils._testing import assert_almost_equal
-from sklearn.utils._testing import assert_allclose
-from sklearn.utils._testing import assert_array_almost_equal
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils._testing import ignore_warnings
-
+from sklearn import config_context, datasets
+from sklearn.base import clone
+from sklearn.datasets import (
+    make_classification,
+    make_low_rank_matrix,
+    make_multilabel_classification,
+    make_regression,
+)
 from sklearn.exceptions import ConvergenceWarning
-
-from sklearn import datasets
-from sklearn.metrics import mean_squared_error
-from sklearn.metrics import make_scorer
-from sklearn.metrics import get_scorer
-
-from sklearn.linear_model import LinearRegression
-from sklearn.linear_model import ridge_regression
-from sklearn.linear_model import Ridge
-from sklearn.linear_model._ridge import _RidgeGCV
-from sklearn.linear_model import RidgeCV
-from sklearn.linear_model import RidgeClassifier
-from sklearn.linear_model import RidgeClassifierCV
-from sklearn.linear_model._ridge import _solve_cholesky
-from sklearn.linear_model._ridge import _solve_cholesky_kernel
-from sklearn.linear_model._ridge import _check_gcv_mode
-from sklearn.linear_model._ridge import _X_CenterStackOp
-from sklearn.datasets import make_regression
-from sklearn.datasets import make_classification
-
-from sklearn.model_selection import GridSearchCV
-from sklearn.model_selection import KFold
-from sklearn.model_selection import GroupKFold
-from sklearn.model_selection import cross_val_predict
-from sklearn.model_selection import LeaveOneOut
-
+from sklearn.linear_model import (
+    LinearRegression,
+    Ridge,
+    RidgeClassifier,
+    RidgeClassifierCV,
+    RidgeCV,
+    ridge_regression,
+)
+from sklearn.linear_model._ridge import (
+    _check_gcv_mode,
+    _RidgeGCV,
+    _solve_cholesky,
+    _solve_cholesky_kernel,
+    _solve_lbfgs,
+    _solve_svd,
+    _X_CenterStackOp,
+)
+from sklearn.metrics import get_scorer, make_scorer, mean_squared_error
+from sklearn.model_selection import (
+    GridSearchCV,
+    GroupKFold,
+    KFold,
+    LeaveOneOut,
+    cross_val_predict,
+)
 from sklearn.preprocessing import minmax_scale
 from sklearn.utils import check_random_state
-from sklearn.datasets import make_multilabel_classification
+from sklearn.utils._array_api import (
+    _NUMPY_NAMESPACE_NAMES,
+    _atol_for_type,
+    _convert_to_numpy,
+    _get_namespace_device_dtype_ids,
+    yield_namespace_device_dtype_combinations,
+    yield_namespaces,
+)
+from sklearn.utils._test_common.instance_generator import _get_check_estimator_ids
+from sklearn.utils._testing import (
+    assert_allclose,
+    assert_almost_equal,
+    assert_array_almost_equal,
+    assert_array_equal,
+    ignore_warnings,
+)
+from sklearn.utils.estimator_checks import (
+    _array_api_for_tests,
+    check_array_api_input_and_values,
+)
+from sklearn.utils.fixes import (
+    _IS_32BIT,
+    COO_CONTAINERS,
+    CSC_CONTAINERS,
+    CSR_CONTAINERS,
+    DOK_CONTAINERS,
+    LIL_CONTAINERS,
+)
+
+SOLVERS = ["svd", "sparse_cg", "cholesky", "lsqr", "sag", "saga"]
+SPARSE_SOLVERS_WITH_INTERCEPT = ("sparse_cg", "sag")
+SPARSE_SOLVERS_WITHOUT_INTERCEPT = ("sparse_cg", "cholesky", "lsqr", "sag", "saga")
 
 diabetes = datasets.load_diabetes()
 X_diabetes, y_diabetes = diabetes.data, diabetes.target
@@ -51,20 +83,10 @@
 X_diabetes, y_diabetes = X_diabetes[ind], y_diabetes[ind]
 
 iris = datasets.load_iris()
+X_iris, y_iris = iris.data, iris.target
 
-X_iris = sp.csr_matrix(iris.data)
-y_iris = iris.target
-
-
-def DENSE_FILTER(X):
-    return X
 
-
-def SPARSE_FILTER(X):
-    return sp.csr_matrix(X)
-
-
-def _accuracy_callable(y_test, y_pred):
+def _accuracy_callable(y_test, y_pred, **kwargs):
     return np.mean(y_test == y_pred)
 
 
@@ -72,42 +94,422 @@ def _mean_squared_error_callable(y_test, y_pred):
     return ((y_test - y_pred) ** 2).mean()
 
 
-@pytest.mark.parametrize('solver',
-                         ("svd", "sparse_cg", "cholesky", "lsqr", "sag"))
-def test_ridge(solver):
-    # Ridge regression convergence test using score
-    # TODO: for this test to be robust, we should use a dataset instead
-    # of np.random.
-    rng = np.random.RandomState(0)
-    alpha = 1.0
+@pytest.fixture(params=["long", "wide"])
+def ols_ridge_dataset(global_random_seed, request):
+    """Dataset with OLS and Ridge solutions, well conditioned X.
+
+    The construction is based on the SVD decomposition of X = U S V'.
+
+    Parameters
+    ----------
+    type : {"long", "wide"}
+        If "long", then n_samples > n_features.
+        If "wide", then n_features > n_samples.
+
+    For "wide", we return the minimum norm solution w = X' (XX')^-1 y:
+
+        min ||w||_2 subject to X w = y
+
+    Returns
+    -------
+    X : ndarray
+        Last column of 1, i.e. intercept.
+    y : ndarray
+    coef_ols : ndarray of shape
+        Minimum norm OLS solutions, i.e. min ||X w - y||_2_2 (with minimum ||w||_2 in
+        case of ambiguity)
+        Last coefficient is intercept.
+    coef_ridge : ndarray of shape (5,)
+        Ridge solution with alpha=1, i.e. min ||X w - y||_2_2 + ||w||_2^2.
+        Last coefficient is intercept.
+    """
+    # Make larger dim more than double as big as the smaller one.
+    # This helps when constructing singular matrices like (X, X).
+    if request.param == "long":
+        n_samples, n_features = 12, 4
+    else:
+        n_samples, n_features = 4, 12
+    k = min(n_samples, n_features)
+    rng = np.random.RandomState(global_random_seed)
+    X = make_low_rank_matrix(
+        n_samples=n_samples, n_features=n_features, effective_rank=k, random_state=rng
+    )
+    X[:, -1] = 1  # last columns acts as intercept
+    U, s, Vt = linalg.svd(X)
+    assert np.all(s > 1e-3)  # to be sure
+    U1, U2 = U[:, :k], U[:, k:]
+    Vt1, _ = Vt[:k, :], Vt[k:, :]
+
+    if request.param == "long":
+        # Add a term that vanishes in the product X'y
+        coef_ols = rng.uniform(low=-10, high=10, size=n_features)
+        y = X @ coef_ols
+        y += U2 @ rng.normal(size=n_samples - n_features) ** 2
+    else:
+        y = rng.uniform(low=-10, high=10, size=n_samples)
+        # w = X'(XX')^-1 y = V s^-1 U' y
+        coef_ols = Vt1.T @ np.diag(1 / s) @ U1.T @ y
+
+    # Add penalty alpha * ||coef||_2^2 for alpha=1 and solve via normal equations.
+    # Note that the problem is well conditioned such that we get accurate results.
+    alpha = 1
+    d = alpha * np.identity(n_features)
+    d[-1, -1] = 0  # intercept gets no penalty
+    coef_ridge = linalg.solve(X.T @ X + d, X.T @ y)
+
+    # To be sure
+    R_OLS = y - X @ coef_ols
+    R_Ridge = y - X @ coef_ridge
+    assert np.linalg.norm(R_OLS) < np.linalg.norm(R_Ridge)
+
+    return X, y, coef_ols, coef_ridge
+
+
+@pytest.mark.parametrize("solver", SOLVERS)
+@pytest.mark.parametrize("fit_intercept", [True, False])
+def test_ridge_regression(solver, fit_intercept, ols_ridge_dataset, global_random_seed):
+    """Test that Ridge converges for all solvers to correct solution.
+
+    We work with a simple constructed data set with known solution.
+    """
+    X, y, _, coef = ols_ridge_dataset
+    alpha = 1.0  # because ols_ridge_dataset uses this.
+    params = dict(
+        alpha=alpha,
+        fit_intercept=True,
+        solver=solver,
+        tol=1e-15 if solver in ("sag", "saga") else 1e-10,
+        random_state=global_random_seed,
+    )
 
-    # With more samples than features
-    n_samples, n_features = 6, 5
-    y = rng.randn(n_samples)
-    X = rng.randn(n_samples, n_features)
+    # Calculate residuals and R2.
+    res_null = y - np.mean(y)
+    res_Ridge = y - X @ coef
+    R2_Ridge = 1 - np.sum(res_Ridge**2) / np.sum(res_null**2)
 
-    ridge = Ridge(alpha=alpha, solver=solver)
-    ridge.fit(X, y)
-    assert ridge.coef_.shape == (X.shape[1], )
-    assert ridge.score(X, y) > 0.47
+    model = Ridge(**params)
+    X = X[:, :-1]  # remove intercept
+    if fit_intercept:
+        intercept = coef[-1]
+    else:
+        X = X - X.mean(axis=0)
+        y = y - y.mean()
+        intercept = 0
+    model.fit(X, y)
+    coef = coef[:-1]
+
+    assert model.intercept_ == pytest.approx(intercept)
+    assert_allclose(model.coef_, coef)
+    assert model.score(X, y) == pytest.approx(R2_Ridge)
+
+    # Same with sample_weight.
+    model = Ridge(**params).fit(X, y, sample_weight=np.ones(X.shape[0]))
+    assert model.intercept_ == pytest.approx(intercept)
+    assert_allclose(model.coef_, coef)
+    assert model.score(X, y) == pytest.approx(R2_Ridge)
+
+    assert model.solver_ == solver
+
+
+@pytest.mark.parametrize("solver", SOLVERS)
+@pytest.mark.parametrize("fit_intercept", [True, False])
+def test_ridge_regression_hstacked_X(
+    solver, fit_intercept, ols_ridge_dataset, global_random_seed
+):
+    """Test that Ridge converges for all solvers to correct solution on hstacked data.
+
+    We work with a simple constructed data set with known solution.
+    Fit on [X] with alpha is the same as fit on [X, X]/2 with alpha/2.
+    For long X, [X, X] is a singular matrix.
+    """
+    X, y, _, coef = ols_ridge_dataset
+    n_samples, n_features = X.shape
+    alpha = 1.0  # because ols_ridge_dataset uses this.
+
+    model = Ridge(
+        alpha=alpha / 2,
+        fit_intercept=fit_intercept,
+        solver=solver,
+        tol=1e-15 if solver in ("sag", "saga") else 1e-10,
+        random_state=global_random_seed,
+    )
+    X = X[:, :-1]  # remove intercept
+    X = 0.5 * np.concatenate((X, X), axis=1)
+    assert np.linalg.matrix_rank(X) <= min(n_samples, n_features - 1)
+    if fit_intercept:
+        intercept = coef[-1]
+    else:
+        X = X - X.mean(axis=0)
+        y = y - y.mean()
+        intercept = 0
+    model.fit(X, y)
+    coef = coef[:-1]
+
+    assert model.intercept_ == pytest.approx(intercept)
+    # coefficients are not all on the same magnitude, adding a small atol to
+    # make this test less brittle
+    assert_allclose(model.coef_, np.r_[coef, coef], atol=1e-8)
+
+
+@pytest.mark.parametrize("solver", SOLVERS)
+@pytest.mark.parametrize("fit_intercept", [True, False])
+def test_ridge_regression_vstacked_X(
+    solver, fit_intercept, ols_ridge_dataset, global_random_seed
+):
+    """Test that Ridge converges for all solvers to correct solution on vstacked data.
+
+    We work with a simple constructed data set with known solution.
+    Fit on [X] with alpha is the same as fit on [X], [y]
+                                                [X], [y] with 2 * alpha.
+    For wide X, [X', X'] is a singular matrix.
+    """
+    X, y, _, coef = ols_ridge_dataset
+    n_samples, n_features = X.shape
+    alpha = 1.0  # because ols_ridge_dataset uses this.
+
+    model = Ridge(
+        alpha=2 * alpha,
+        fit_intercept=fit_intercept,
+        solver=solver,
+        tol=1e-15 if solver in ("sag", "saga") else 1e-10,
+        random_state=global_random_seed,
+    )
+    X = X[:, :-1]  # remove intercept
+    X = np.concatenate((X, X), axis=0)
+    assert np.linalg.matrix_rank(X) <= min(n_samples, n_features)
+    y = np.r_[y, y]
+    if fit_intercept:
+        intercept = coef[-1]
+    else:
+        X = X - X.mean(axis=0)
+        y = y - y.mean()
+        intercept = 0
+    model.fit(X, y)
+    coef = coef[:-1]
+
+    assert model.intercept_ == pytest.approx(intercept)
+    # coefficients are not all on the same magnitude, adding a small atol to
+    # make this test less brittle
+    assert_allclose(model.coef_, coef, atol=1e-8)
+
+
+@pytest.mark.parametrize("solver", SOLVERS)
+@pytest.mark.parametrize("fit_intercept", [True, False])
+def test_ridge_regression_unpenalized(
+    solver, fit_intercept, ols_ridge_dataset, global_random_seed
+):
+    """Test that unpenalized Ridge = OLS converges for all solvers to correct solution.
+
+    We work with a simple constructed data set with known solution.
+    Note: This checks the minimum norm solution for wide X, i.e.
+    n_samples < n_features:
+        min ||w||_2 subject to X w = y
+    """
+    X, y, coef, _ = ols_ridge_dataset
+    n_samples, n_features = X.shape
+    alpha = 0  # OLS
+    params = dict(
+        alpha=alpha,
+        fit_intercept=fit_intercept,
+        solver=solver,
+        tol=1e-15 if solver in ("sag", "saga") else 1e-10,
+        random_state=global_random_seed,
+    )
 
-    if solver in ("cholesky", "sag"):
-        # Currently the only solvers to support sample_weight.
-        ridge.fit(X, y, sample_weight=np.ones(n_samples))
-        assert ridge.score(X, y) > 0.47
+    model = Ridge(**params)
+    # Note that cholesky might give a warning: "Singular matrix in solving dual
+    # problem. Using least-squares solution instead."
+    if fit_intercept:
+        X = X[:, :-1]  # remove intercept
+        intercept = coef[-1]
+        coef = coef[:-1]
+    else:
+        intercept = 0
+    model.fit(X, y)
+
+    # FIXME: `assert_allclose(model.coef_, coef)` should work for all cases but fails
+    # for the wide/fat case with n_features > n_samples. The current Ridge solvers do
+    # NOT return the minimum norm solution with fit_intercept=True.
+    if n_samples > n_features or not fit_intercept:
+        assert model.intercept_ == pytest.approx(intercept)
+        assert_allclose(model.coef_, coef)
+    else:
+        # As it is an underdetermined problem, residuals = 0. This shows that we get
+        # a solution to X w = y ....
+        assert_allclose(model.predict(X), y)
+        assert_allclose(X @ coef + intercept, y)
+        # But it is not the minimum norm solution. (This should be equal.)
+        assert np.linalg.norm(np.r_[model.intercept_, model.coef_]) > np.linalg.norm(
+            np.r_[intercept, coef]
+        )
+
+        pytest.xfail(reason="Ridge does not provide the minimum norm solution.")
+        assert model.intercept_ == pytest.approx(intercept)
+        assert_allclose(model.coef_, coef)
+
+
+@pytest.mark.parametrize("solver", SOLVERS)
+@pytest.mark.parametrize("fit_intercept", [True, False])
+def test_ridge_regression_unpenalized_hstacked_X(
+    solver, fit_intercept, ols_ridge_dataset, global_random_seed
+):
+    """Test that unpenalized Ridge = OLS converges for all solvers to correct solution.
+
+    We work with a simple constructed data set with known solution.
+    OLS fit on [X] is the same as fit on [X, X]/2.
+    For long X, [X, X] is a singular matrix and we check against the minimum norm
+    solution:
+        min ||w||_2 subject to min ||X w - y||_2
+    """
+    X, y, coef, _ = ols_ridge_dataset
+    n_samples, n_features = X.shape
+    alpha = 0  # OLS
+
+    model = Ridge(
+        alpha=alpha,
+        fit_intercept=fit_intercept,
+        solver=solver,
+        tol=1e-15 if solver in ("sag", "saga") else 1e-10,
+        random_state=global_random_seed,
+    )
+    if fit_intercept:
+        X = X[:, :-1]  # remove intercept
+        intercept = coef[-1]
+        coef = coef[:-1]
+    else:
+        intercept = 0
+    X = 0.5 * np.concatenate((X, X), axis=1)
+    assert np.linalg.matrix_rank(X) <= min(n_samples, n_features)
+    model.fit(X, y)
+
+    if n_samples > n_features or not fit_intercept:
+        assert model.intercept_ == pytest.approx(intercept)
+        if solver == "cholesky":
+            # Cholesky is a bad choice for singular X.
+            pytest.skip()
+        assert_allclose(model.coef_, np.r_[coef, coef])
+    else:
+        # FIXME: Same as in test_ridge_regression_unpenalized.
+        # As it is an underdetermined problem, residuals = 0. This shows that we get
+        # a solution to X w = y ....
+        assert_allclose(model.predict(X), y)
+        # But it is not the minimum norm solution. (This should be equal.)
+        assert np.linalg.norm(np.r_[model.intercept_, model.coef_]) > np.linalg.norm(
+            np.r_[intercept, coef, coef]
+        )
+
+        pytest.xfail(reason="Ridge does not provide the minimum norm solution.")
+        assert model.intercept_ == pytest.approx(intercept)
+        assert_allclose(model.coef_, np.r_[coef, coef])
+
+
+@pytest.mark.parametrize("solver", SOLVERS)
+@pytest.mark.parametrize("fit_intercept", [True, False])
+def test_ridge_regression_unpenalized_vstacked_X(
+    solver, fit_intercept, ols_ridge_dataset, global_random_seed
+):
+    """Test that unpenalized Ridge = OLS converges for all solvers to correct solution.
+
+    We work with a simple constructed data set with known solution.
+    OLS fit on [X] is the same as fit on [X], [y]
+                                         [X], [y].
+    For wide X, [X', X'] is a singular matrix and we check against the minimum norm
+    solution:
+        min ||w||_2 subject to X w = y
+    """
+    X, y, coef, _ = ols_ridge_dataset
+    n_samples, n_features = X.shape
+    alpha = 0  # OLS
+
+    model = Ridge(
+        alpha=alpha,
+        fit_intercept=fit_intercept,
+        solver=solver,
+        tol=1e-15 if solver in ("sag", "saga") else 1e-10,
+        random_state=global_random_seed,
+    )
 
-    # With more features than samples
-    n_samples, n_features = 5, 10
-    y = rng.randn(n_samples)
-    X = rng.randn(n_samples, n_features)
-    ridge = Ridge(alpha=alpha, solver=solver)
-    ridge.fit(X, y)
-    assert ridge.score(X, y) > .9
+    if fit_intercept:
+        X = X[:, :-1]  # remove intercept
+        intercept = coef[-1]
+        coef = coef[:-1]
+    else:
+        intercept = 0
+    X = np.concatenate((X, X), axis=0)
+    assert np.linalg.matrix_rank(X) <= min(n_samples, n_features)
+    y = np.r_[y, y]
+    model.fit(X, y)
+
+    if n_samples > n_features or not fit_intercept:
+        assert model.intercept_ == pytest.approx(intercept)
+        assert_allclose(model.coef_, coef)
+    else:
+        # FIXME: Same as in test_ridge_regression_unpenalized.
+        # As it is an underdetermined problem, residuals = 0. This shows that we get
+        # a solution to X w = y ....
+        assert_allclose(model.predict(X), y)
+        # But it is not the minimum norm solution. (This should be equal.)
+        assert np.linalg.norm(np.r_[model.intercept_, model.coef_]) > np.linalg.norm(
+            np.r_[intercept, coef]
+        )
+
+        pytest.xfail(reason="Ridge does not provide the minimum norm solution.")
+        assert model.intercept_ == pytest.approx(intercept)
+        assert_allclose(model.coef_, coef)
+
+
+@pytest.mark.parametrize("solver", SOLVERS)
+@pytest.mark.parametrize("fit_intercept", [True, False])
+@pytest.mark.parametrize("sparse_container", [None] + CSR_CONTAINERS)
+@pytest.mark.parametrize("alpha", [1.0, 1e-2])
+def test_ridge_regression_sample_weights(
+    solver,
+    fit_intercept,
+    sparse_container,
+    alpha,
+    ols_ridge_dataset,
+    global_random_seed,
+):
+    """Test that Ridge with sample weights gives correct results.
+
+    We use the following trick:
+        ||y - Xw||_2 = (z - Aw)' W (z - Aw)
+    for z=[y, y], A' = [X', X'] (vstacked), and W[:n/2] + W[n/2:] = 1, W=diag(W)
+    """
+    if sparse_container is not None:
+        if fit_intercept and solver not in SPARSE_SOLVERS_WITH_INTERCEPT:
+            pytest.skip()
+        elif not fit_intercept and solver not in SPARSE_SOLVERS_WITHOUT_INTERCEPT:
+            pytest.skip()
+    X, y, _, coef = ols_ridge_dataset
+    n_samples, n_features = X.shape
+    sw = rng.uniform(low=0, high=1, size=n_samples)
+
+    model = Ridge(
+        alpha=alpha,
+        fit_intercept=fit_intercept,
+        solver=solver,
+        tol=1e-15 if solver in ["sag", "saga"] else 1e-10,
+        max_iter=100_000,
+        random_state=global_random_seed,
+    )
+    X = X[:, :-1]  # remove intercept
+    X = np.concatenate((X, X), axis=0)
+    y = np.r_[y, y]
+    sw = np.r_[sw, 1 - sw] * alpha
+    if fit_intercept:
+        intercept = coef[-1]
+    else:
+        X = X - X.mean(axis=0)
+        y = y - y.mean()
+        intercept = 0
+    if sparse_container is not None:
+        X = sparse_container(X)
+    model.fit(X, y, sample_weight=sw)
+    coef = coef[:-1]
 
-    if solver in ("cholesky", "sag"):
-        # Currently the only solvers to support sample_weight.
-        ridge.fit(X, y, sample_weight=np.ones(n_samples))
-        assert ridge.score(X, y) > 0.9
+    assert model.intercept_ == pytest.approx(intercept)
+    assert_allclose(model.coef_, coef)
 
 
 def test_primal_dual_relationship():
@@ -119,105 +521,18 @@ def test_primal_dual_relationship():
     assert_array_almost_equal(coef, coef2)
 
 
-def test_ridge_singular():
-    # test on a singular matrix
-    rng = np.random.RandomState(0)
-    n_samples, n_features = 6, 6
-    y = rng.randn(n_samples // 2)
-    y = np.concatenate((y, y))
-    X = rng.randn(n_samples // 2, n_features)
-    X = np.concatenate((X, X), axis=0)
-
-    ridge = Ridge(alpha=0)
-    ridge.fit(X, y)
-    assert ridge.score(X, y) > 0.9
-
-
-def test_ridge_regression_sample_weights():
-    rng = np.random.RandomState(0)
-
-    for solver in ("cholesky", ):
-        for n_samples, n_features in ((6, 5), (5, 10)):
-            for alpha in (1.0, 1e-2):
-                y = rng.randn(n_samples)
-                X = rng.randn(n_samples, n_features)
-                sample_weight = 1.0 + rng.rand(n_samples)
-
-                coefs = ridge_regression(X, y,
-                                         alpha=alpha,
-                                         sample_weight=sample_weight,
-                                         solver=solver)
-
-                # Sample weight can be implemented via a simple rescaling
-                # for the square loss.
-                coefs2 = ridge_regression(
-                    X * np.sqrt(sample_weight)[:, np.newaxis],
-                    y * np.sqrt(sample_weight),
-                    alpha=alpha, solver=solver)
-                assert_array_almost_equal(coefs, coefs2)
-
-
 def test_ridge_regression_convergence_fail():
     rng = np.random.RandomState(0)
     y = rng.randn(5)
     X = rng.randn(5, 10)
-    warning_message = (
-        r"sparse_cg did not converge after"
-        r" [0-9]+ iterations."
-    )
+    warning_message = r"sparse_cg did not converge after [0-9]+ iterations."
     with pytest.warns(ConvergenceWarning, match=warning_message):
-        ridge_regression(X, y,
-                         alpha=1.0, solver="sparse_cg",
-                         tol=0., max_iter=None, verbose=1)
-
+        ridge_regression(
+            X, y, alpha=1.0, solver="sparse_cg", tol=0.0, max_iter=None, verbose=1
+        )
 
-def test_ridge_sample_weights():
-    # TODO: loop over sparse data as well
-    # Note: parametrizing this test with pytest results in failed
-    #       assertions, meaning that is is not extremely robust
 
-    rng = np.random.RandomState(0)
-    param_grid = product((1.0, 1e-2), (True, False),
-                         ('svd', 'cholesky', 'lsqr', 'sparse_cg'))
-
-    for n_samples, n_features in ((6, 5), (5, 10)):
-
-        y = rng.randn(n_samples)
-        X = rng.randn(n_samples, n_features)
-        sample_weight = 1.0 + rng.rand(n_samples)
-
-        for (alpha, intercept, solver) in param_grid:
-
-            # Ridge with explicit sample_weight
-            est = Ridge(alpha=alpha, fit_intercept=intercept,
-                        solver=solver, tol=1e-6)
-            est.fit(X, y, sample_weight=sample_weight)
-            coefs = est.coef_
-            inter = est.intercept_
-
-            # Closed form of the weighted regularized least square
-            # theta = (X^T W X + alpha I)^(-1) * X^T W y
-            W = np.diag(sample_weight)
-            if intercept is False:
-                X_aug = X
-                D = np.eye(n_features)
-            else:
-                dummy_column = np.ones(shape=(n_samples, 1))
-                X_aug = np.concatenate((dummy_column, X), axis=1)
-                D = np.eye(n_features + 1)
-                D[0, 0] = 0
-
-            cf_coefs = linalg.solve(X_aug.T.dot(W).dot(X_aug) + alpha * D,
-                                    X_aug.T.dot(W).dot(y))
-
-            if intercept is False:
-                assert_array_almost_equal(coefs, cf_coefs)
-            else:
-                assert_array_almost_equal(coefs, cf_coefs[1:])
-                assert_almost_equal(inter, cf_coefs[0])
-
-
-def test_ridge_shapes():
+def test_ridge_shapes_type():
     # Test shape of coef_ and intercept_
     rng = np.random.RandomState(0)
     n_samples, n_features = 5, 10
@@ -231,14 +546,20 @@ def test_ridge_shapes():
     ridge.fit(X, y)
     assert ridge.coef_.shape == (n_features,)
     assert ridge.intercept_.shape == ()
+    assert isinstance(ridge.coef_, np.ndarray)
+    assert isinstance(ridge.intercept_, float)
 
     ridge.fit(X, Y1)
-    assert ridge.coef_.shape == (1, n_features)
-    assert ridge.intercept_.shape == (1, )
+    assert ridge.coef_.shape == (n_features,)
+    assert ridge.intercept_.shape == (1,)
+    assert isinstance(ridge.coef_, np.ndarray)
+    assert isinstance(ridge.intercept_, np.ndarray)
 
     ridge.fit(X, Y)
     assert ridge.coef_.shape == (2, n_features)
-    assert ridge.intercept_.shape == (2, )
+    assert ridge.intercept_.shape == (2,)
+    assert isinstance(ridge.coef_, np.ndarray)
+    assert isinstance(ridge.intercept_, np.ndarray)
 
 
 def test_ridge_intercept():
@@ -247,7 +568,7 @@ def test_ridge_intercept():
     n_samples, n_features = 5, 10
     X = rng.randn(n_samples, n_features)
     y = rng.randn(n_samples)
-    Y = np.c_[y, 1. + y]
+    Y = np.c_[y, 1.0 + y]
 
     ridge = Ridge()
 
@@ -256,29 +577,7 @@ def test_ridge_intercept():
 
     ridge.fit(X, Y)
     assert_almost_equal(ridge.intercept_[0], intercept)
-    assert_almost_equal(ridge.intercept_[1], intercept + 1.)
-
-
-def test_toy_ridge_object():
-    # Test BayesianRegression ridge classifier
-    # TODO: test also n_samples > n_features
-    X = np.array([[1], [2]])
-    Y = np.array([1, 2])
-    reg = Ridge(alpha=0.0)
-    reg.fit(X, Y)
-    X_test = [[1], [2], [3], [4]]
-    assert_almost_equal(reg.predict(X_test), [1., 2, 3, 4])
-
-    assert len(reg.coef_.shape) == 1
-    assert type(reg.intercept_) == np.float64
-
-    Y = np.vstack((Y, Y)).T
-
-    reg.fit(X, Y)
-    X_test = [[1], [2], [3], [4]]
-
-    assert len(reg.coef_.shape) == 2
-    assert type(reg.intercept_) == np.ndarray
+    assert_almost_equal(ridge.intercept_[1], intercept + 1.0)
 
 
 def test_ridge_vs_lstsq():
@@ -290,7 +589,7 @@ def test_ridge_vs_lstsq():
     y = rng.randn(n_samples)
     X = rng.randn(n_samples, n_features)
 
-    ridge = Ridge(alpha=0., fit_intercept=False)
+    ridge = Ridge(alpha=0.0, fit_intercept=False)
     ols = LinearRegression(fit_intercept=False)
 
     ridge.fit(X, y)
@@ -313,40 +612,46 @@ def test_ridge_individual_penalties():
 
     penalties = np.arange(n_targets)
 
-    coef_cholesky = np.array([
-        Ridge(alpha=alpha, solver="cholesky").fit(X, target).coef_
-        for alpha, target in zip(penalties, y.T)])
+    coef_cholesky = np.array(
+        [
+            Ridge(alpha=alpha, solver="cholesky").fit(X, target).coef_
+            for alpha, target in zip(penalties, y.T)
+        ]
+    )
 
     coefs_indiv_pen = [
-        Ridge(alpha=penalties, solver=solver, tol=1e-8).fit(X, y).coef_
-        for solver in ['svd', 'sparse_cg', 'lsqr', 'cholesky', 'sag', 'saga']]
+        Ridge(alpha=penalties, solver=solver, tol=1e-12).fit(X, y).coef_
+        for solver in ["svd", "sparse_cg", "lsqr", "cholesky", "sag", "saga"]
+    ]
     for coef_indiv_pen in coefs_indiv_pen:
         assert_array_almost_equal(coef_cholesky, coef_indiv_pen)
 
     # Test error is raised when number of targets and penalties do not match.
     ridge = Ridge(alpha=penalties[:-1])
-    with pytest.raises(ValueError):
+    err_msg = "Number of targets and number of penalties do not correspond: 4 != 5"
+    with pytest.raises(ValueError, match=err_msg):
         ridge.fit(X, y)
 
 
-@pytest.mark.parametrize('n_col', [(), (1,), (3,)])
-def test_X_CenterStackOp(n_col):
+@pytest.mark.parametrize("n_col", [(), (1,), (3,)])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_X_CenterStackOp(n_col, csr_container):
     rng = np.random.RandomState(0)
     X = rng.randn(11, 8)
     X_m = rng.randn(8)
     sqrt_sw = rng.randn(len(X))
     Y = rng.randn(11, *n_col)
     A = rng.randn(9, *n_col)
-    operator = _X_CenterStackOp(sp.csr_matrix(X), X_m, sqrt_sw)
-    reference_operator = np.hstack(
-        [X - sqrt_sw[:, None] * X_m, sqrt_sw[:, None]])
+    operator = _X_CenterStackOp(csr_container(X), X_m, sqrt_sw)
+    reference_operator = np.hstack([X - sqrt_sw[:, None] * X_m, sqrt_sw[:, None]])
     assert_allclose(reference_operator.dot(A), operator.dot(A))
     assert_allclose(reference_operator.T.dot(Y), operator.T.dot(Y))
 
 
-@pytest.mark.parametrize('shape', [(10, 1), (13, 9), (3, 7), (2, 2), (20, 20)])
-@pytest.mark.parametrize('uniform_weights', [True, False])
-def test_compute_gram(shape, uniform_weights):
+@pytest.mark.parametrize("shape", [(10, 1), (13, 9), (3, 7), (2, 2), (20, 20)])
+@pytest.mark.parametrize("uniform_weights", [True, False])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_compute_gram(shape, uniform_weights, csr_container):
     rng = np.random.RandomState(0)
     X = rng.randn(*shape)
     if uniform_weights:
@@ -357,16 +662,17 @@ def test_compute_gram(shape, uniform_weights):
     X_mean = np.average(X, axis=0, weights=sw)
     X_centered = (X - X_mean) * sqrt_sw[:, None]
     true_gram = X_centered.dot(X_centered.T)
-    X_sparse = sp.csr_matrix(X * sqrt_sw[:, None])
+    X_sparse = csr_container(X * sqrt_sw[:, None])
     gcv = _RidgeGCV(fit_intercept=True)
     computed_gram, computed_mean = gcv._compute_gram(X_sparse, sqrt_sw)
     assert_allclose(X_mean, computed_mean)
     assert_allclose(true_gram, computed_gram)
 
 
-@pytest.mark.parametrize('shape', [(10, 1), (13, 9), (3, 7), (2, 2), (20, 20)])
-@pytest.mark.parametrize('uniform_weights', [True, False])
-def test_compute_covariance(shape, uniform_weights):
+@pytest.mark.parametrize("shape", [(10, 1), (13, 9), (3, 7), (2, 2), (20, 20)])
+@pytest.mark.parametrize("uniform_weights", [True, False])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_compute_covariance(shape, uniform_weights, csr_container):
     rng = np.random.RandomState(0)
     X = rng.randn(*shape)
     if uniform_weights:
@@ -377,7 +683,7 @@ def test_compute_covariance(shape, uniform_weights):
     X_mean = np.average(X, axis=0, weights=sw)
     X_centered = (X - X_mean) * sqrt_sw[:, None]
     true_covariance = X_centered.T.dot(X_centered)
-    X_sparse = sp.csr_matrix(X * sqrt_sw[:, None])
+    X_sparse = csr_container(X * sqrt_sw[:, None])
     gcv = _RidgeGCV(fit_intercept=True)
     computed_cov, computed_mean = gcv._compute_covariance(X_sparse, sqrt_sw)
     assert_allclose(X_mean, computed_mean)
@@ -385,23 +691,43 @@ def test_compute_covariance(shape, uniform_weights):
 
 
 def _make_sparse_offset_regression(
-        n_samples=100, n_features=100, proportion_nonzero=.5,
-        n_informative=10, n_targets=1, bias=13., X_offset=30.,
-        noise=30., shuffle=True, coef=False, random_state=None):
+    n_samples=100,
+    n_features=100,
+    proportion_nonzero=0.5,
+    n_informative=10,
+    n_targets=1,
+    bias=13.0,
+    X_offset=30.0,
+    noise=30.0,
+    shuffle=True,
+    coef=False,
+    positive=False,
+    random_state=None,
+):
     X, y, c = make_regression(
-        n_samples=n_samples, n_features=n_features,
-        n_informative=n_informative, n_targets=n_targets, bias=bias,
-        noise=noise, shuffle=shuffle,
-        coef=True, random_state=random_state)
+        n_samples=n_samples,
+        n_features=n_features,
+        n_informative=n_informative,
+        n_targets=n_targets,
+        bias=bias,
+        noise=noise,
+        shuffle=shuffle,
+        coef=True,
+        random_state=random_state,
+    )
     if n_features == 1:
         c = np.asarray([c])
     X += X_offset
-    mask = np.random.RandomState(random_state).binomial(
-        1, proportion_nonzero, X.shape) > 0
+    mask = (
+        np.random.RandomState(random_state).binomial(1, proportion_nonzero, X.shape) > 0
+    )
     removed_X = X.copy()
-    X[~mask] = 0.
-    removed_X[mask] = 0.
+    X[~mask] = 0.0
+    removed_X[mask] = 0.0
     y -= removed_X.dot(c)
+    if positive:
+        y += X.dot(np.abs(c) + 1 - c)
+        c = np.abs(c) + 1
     if n_features == 1:
         c = c[0]
     if coef:
@@ -410,81 +736,110 @@ def _make_sparse_offset_regression(
 
 
 @pytest.mark.parametrize(
-    'solver, sparse_X',
-    ((solver, sparse_X) for
-     (solver, sparse_X) in product(
-         ['cholesky', 'sag', 'sparse_cg', 'lsqr', 'saga', 'ridgecv'],
-         [False, True])
-     if not (sparse_X and solver not in ['sparse_cg', 'ridgecv'])))
+    "solver, sparse_container",
+    (
+        (solver, sparse_container)
+        for (solver, sparse_container) in product(
+            ["cholesky", "sag", "sparse_cg", "lsqr", "saga", "ridgecv"],
+            [None] + CSR_CONTAINERS,
+        )
+        if sparse_container is None or solver in ["sparse_cg", "ridgecv"]
+    ),
+)
 @pytest.mark.parametrize(
-    'n_samples,dtype,proportion_nonzero',
-    [(20, 'float32', .1), (40, 'float32', 1.), (20, 'float64', .2)])
-@pytest.mark.parametrize('normalize', [True, False])
-@pytest.mark.parametrize('seed', np.arange(3))
+    "n_samples,dtype,proportion_nonzero",
+    [(20, "float32", 0.1), (40, "float32", 1.0), (20, "float64", 0.2)],
+)
 def test_solver_consistency(
-        solver, proportion_nonzero, n_samples, dtype, sparse_X, seed,
-        normalize):
-    alpha = 1.
-    noise = 50. if proportion_nonzero > .9 else 500.
+    solver, proportion_nonzero, n_samples, dtype, sparse_container, global_random_seed
+):
+    alpha = 1.0
+    noise = 50.0 if proportion_nonzero > 0.9 else 500.0
     X, y = _make_sparse_offset_regression(
-        bias=10, n_features=30, proportion_nonzero=proportion_nonzero,
-        noise=noise, random_state=seed, n_samples=n_samples)
-    if not normalize:
-        # Manually scale the data to avoid pathological cases. We use
-        # minmax_scale to deal with the sparse case without breaking
-        # the sparsity pattern.
-        X = minmax_scale(X)
-    svd_ridge = Ridge(
-        solver='svd', normalize=normalize, alpha=alpha).fit(X, y)
+        bias=10,
+        n_features=30,
+        proportion_nonzero=proportion_nonzero,
+        noise=noise,
+        random_state=global_random_seed,
+        n_samples=n_samples,
+    )
+    # Manually scale the data to avoid pathological cases. We use
+    # minmax_scale to deal with the sparse case without breaking
+    # the sparsity pattern.
+    X = minmax_scale(X)
+
+    svd_ridge = Ridge(solver="svd", alpha=alpha).fit(X, y)
     X = X.astype(dtype, copy=False)
     y = y.astype(dtype, copy=False)
-    if sparse_X:
-        X = sp.csr_matrix(X)
-    if solver == 'ridgecv':
-        ridge = RidgeCV(alphas=[alpha], normalize=normalize)
+    if sparse_container is not None:
+        X = sparse_container(X)
+    if solver == "ridgecv":
+        ridge = RidgeCV(alphas=[alpha])
     else:
-        ridge = Ridge(solver=solver, tol=1e-10, normalize=normalize,
-                      alpha=alpha)
+        if solver.startswith("sag"):
+            # Avoid ConvergenceWarning for sag and saga solvers.
+            tol = 1e-7
+            max_iter = 100_000
+        else:
+            tol = 1e-10
+            max_iter = None
+
+        ridge = Ridge(
+            alpha=alpha,
+            solver=solver,
+            max_iter=max_iter,
+            tol=tol,
+            random_state=global_random_seed,
+        )
     ridge.fit(X, y)
-    assert_allclose(
-        ridge.coef_, svd_ridge.coef_, atol=1e-3, rtol=1e-3)
-    assert_allclose(
-        ridge.intercept_, svd_ridge.intercept_, atol=1e-3, rtol=1e-3)
+    assert_allclose(ridge.coef_, svd_ridge.coef_, atol=1e-3, rtol=1e-3)
+    assert_allclose(ridge.intercept_, svd_ridge.intercept_, atol=1e-3, rtol=1e-3)
 
 
-@pytest.mark.parametrize('gcv_mode', ['svd', 'eigen'])
-@pytest.mark.parametrize('X_constructor', [np.asarray, sp.csr_matrix])
-@pytest.mark.parametrize('X_shape', [(11, 8), (11, 20)])
-@pytest.mark.parametrize('fit_intercept', [True, False])
+@pytest.mark.parametrize("gcv_mode", ["svd", "eigen"])
+@pytest.mark.parametrize("X_container", [np.asarray] + CSR_CONTAINERS)
+@pytest.mark.parametrize("X_shape", [(11, 8), (11, 20)])
+@pytest.mark.parametrize("fit_intercept", [True, False])
 @pytest.mark.parametrize(
-    'y_shape, normalize, noise',
+    "y_shape, noise",
     [
-        ((11,), True, 1.),
-        ((11, 1), False, 30.),
-        ((11, 3), False, 150.),
-    ]
+        ((11,), 1.0),
+        ((11, 1), 30.0),
+        ((11, 3), 150.0),
+    ],
 )
 def test_ridge_gcv_vs_ridge_loo_cv(
-        gcv_mode, X_constructor, X_shape, y_shape,
-        fit_intercept, normalize, noise):
+    gcv_mode, X_container, X_shape, y_shape, fit_intercept, noise
+):
     n_samples, n_features = X_shape
     n_targets = y_shape[-1] if len(y_shape) == 2 else 1
     X, y = _make_sparse_offset_regression(
-        n_samples=n_samples, n_features=n_features, n_targets=n_targets,
-        random_state=0, shuffle=False, noise=noise, n_informative=5
+        n_samples=n_samples,
+        n_features=n_features,
+        n_targets=n_targets,
+        random_state=0,
+        shuffle=False,
+        noise=noise,
+        n_informative=5,
     )
     y = y.reshape(y_shape)
 
-    alphas = [1e-3, .1, 1., 10., 1e3]
-    loo_ridge = RidgeCV(cv=n_samples, fit_intercept=fit_intercept,
-                        alphas=alphas, scoring='neg_mean_squared_error',
-                        normalize=normalize)
-    gcv_ridge = RidgeCV(gcv_mode=gcv_mode, fit_intercept=fit_intercept,
-                        alphas=alphas, normalize=normalize)
+    alphas = [1e-3, 0.1, 1.0, 10.0, 1e3]
+    loo_ridge = RidgeCV(
+        cv=n_samples,
+        fit_intercept=fit_intercept,
+        alphas=alphas,
+        scoring="neg_mean_squared_error",
+    )
+    gcv_ridge = RidgeCV(
+        gcv_mode=gcv_mode,
+        fit_intercept=fit_intercept,
+        alphas=alphas,
+    )
 
     loo_ridge.fit(X, y)
 
-    X_gcv = X_constructor(X)
+    X_gcv = X_container(X)
     gcv_ridge.fit(X_gcv, y)
 
     assert gcv_ridge.alpha_ == pytest.approx(loo_ridge.alpha_)
@@ -494,47 +849,62 @@ def test_ridge_gcv_vs_ridge_loo_cv(
 
 def test_ridge_loo_cv_asym_scoring():
     # checking on asymmetric scoring
-    scoring = 'explained_variance'
+    scoring = "explained_variance"
     n_samples, n_features = 10, 5
     n_targets = 1
     X, y = _make_sparse_offset_regression(
-        n_samples=n_samples, n_features=n_features, n_targets=n_targets,
-        random_state=0, shuffle=False, noise=1, n_informative=5
+        n_samples=n_samples,
+        n_features=n_features,
+        n_targets=n_targets,
+        random_state=0,
+        shuffle=False,
+        noise=1,
+        n_informative=5,
     )
 
-    alphas = [1e-3, .1, 1., 10., 1e3]
-    loo_ridge = RidgeCV(cv=n_samples, fit_intercept=True,
-                        alphas=alphas, scoring=scoring,
-                        normalize=True)
+    alphas = [1e-3, 0.1, 1.0, 10.0, 1e3]
+    loo_ridge = RidgeCV(
+        cv=n_samples, fit_intercept=True, alphas=alphas, scoring=scoring
+    )
 
-    gcv_ridge = RidgeCV(fit_intercept=True,
-                        alphas=alphas, scoring=scoring,
-                        normalize=True)
+    gcv_ridge = RidgeCV(fit_intercept=True, alphas=alphas, scoring=scoring)
 
     loo_ridge.fit(X, y)
     gcv_ridge.fit(X, y)
 
-    assert gcv_ridge.alpha_ == pytest.approx(loo_ridge.alpha_)
+    assert gcv_ridge.alpha_ == pytest.approx(loo_ridge.alpha_), (
+        f"{gcv_ridge.alpha_=}, {loo_ridge.alpha_=}"
+    )
     assert_allclose(gcv_ridge.coef_, loo_ridge.coef_, rtol=1e-3)
     assert_allclose(gcv_ridge.intercept_, loo_ridge.intercept_, rtol=1e-3)
 
 
-@pytest.mark.parametrize('gcv_mode', ['svd', 'eigen'])
-@pytest.mark.parametrize('X_constructor', [np.asarray, sp.csr_matrix])
-@pytest.mark.parametrize('n_features', [8, 20])
-@pytest.mark.parametrize('y_shape, fit_intercept, noise',
-                         [((11,), True, 1.),
-                          ((11, 1), True, 20.),
-                          ((11, 3), True, 150.),
-                          ((11, 3), False, 30.)])
+@pytest.mark.parametrize("gcv_mode", ["svd", "eigen"])
+@pytest.mark.parametrize("X_container", [np.asarray] + CSR_CONTAINERS)
+@pytest.mark.parametrize("n_features", [8, 20])
+@pytest.mark.parametrize(
+    "y_shape, fit_intercept, noise",
+    [
+        ((11,), True, 1.0),
+        ((11, 1), True, 20.0),
+        ((11, 3), True, 150.0),
+        ((11, 3), False, 30.0),
+    ],
+)
 def test_ridge_gcv_sample_weights(
-        gcv_mode, X_constructor, fit_intercept, n_features, y_shape, noise):
-    alphas = [1e-3, .1, 1., 10., 1e3]
+    gcv_mode, X_container, fit_intercept, n_features, y_shape, noise
+):
+    alphas = [1e-3, 0.1, 1.0, 10.0, 1e3]
     rng = np.random.RandomState(0)
     n_targets = y_shape[-1] if len(y_shape) == 2 else 1
     X, y = _make_sparse_offset_regression(
-        n_samples=11, n_features=n_features, n_targets=n_targets,
-        random_state=0, shuffle=False, noise=noise)
+        n_samples=11,
+        n_features=n_features,
+        n_targets=n_targets,
+        random_state=0,
+        shuffle=False,
+        noise=noise,
+    )
     y = y.reshape(y_shape)
 
     sample_weight = 3 * rng.randn(len(X))
@@ -546,28 +916,36 @@ def test_ridge_gcv_sample_weights(
     cv = GroupKFold(n_splits=X.shape[0])
     splits = cv.split(X_tiled, y_tiled, groups=indices)
     kfold = RidgeCV(
-        alphas=alphas, cv=splits, scoring='neg_mean_squared_error',
-        fit_intercept=fit_intercept)
+        alphas=alphas,
+        cv=splits,
+        scoring="neg_mean_squared_error",
+        fit_intercept=fit_intercept,
+    )
     kfold.fit(X_tiled, y_tiled)
 
     ridge_reg = Ridge(alpha=kfold.alpha_, fit_intercept=fit_intercept)
     splits = cv.split(X_tiled, y_tiled, groups=indices)
     predictions = cross_val_predict(ridge_reg, X_tiled, y_tiled, cv=splits)
-    kfold_errors = (y_tiled - predictions)**2
+    if predictions.shape != y_tiled.shape:
+        predictions = predictions.reshape(y_tiled.shape)
+    kfold_errors = (y_tiled - predictions) ** 2
     kfold_errors = [
-        np.sum(kfold_errors[indices == i], axis=0) for
-        i in np.arange(X.shape[0])]
+        np.sum(kfold_errors[indices == i], axis=0) for i in np.arange(X.shape[0])
+    ]
     kfold_errors = np.asarray(kfold_errors)
 
-    X_gcv = X_constructor(X)
+    X_gcv = X_container(X)
     gcv_ridge = RidgeCV(
-        alphas=alphas, store_cv_values=True,
-        gcv_mode=gcv_mode, fit_intercept=fit_intercept)
+        alphas=alphas,
+        store_cv_results=True,
+        gcv_mode=gcv_mode,
+        fit_intercept=fit_intercept,
+    )
     gcv_ridge.fit(X_gcv, y, sample_weight=sample_weight)
     if len(y_shape) == 2:
-        gcv_errors = gcv_ridge.cv_values_[:, :, alphas.index(kfold.alpha_)]
+        gcv_errors = gcv_ridge.cv_results_[:, :, alphas.index(kfold.alpha_)]
     else:
-        gcv_errors = gcv_ridge.cv_values_[:, alphas.index(kfold.alpha_)]
+        gcv_errors = gcv_ridge.cv_results_[:, alphas.index(kfold.alpha_)]
 
     assert kfold.alpha_ == pytest.approx(gcv_ridge.alpha_)
     assert_allclose(gcv_errors, kfold_errors, rtol=1e-3)
@@ -575,44 +953,40 @@ def test_ridge_gcv_sample_weights(
     assert_allclose(gcv_ridge.intercept_, kfold.intercept_, rtol=1e-3)
 
 
-@pytest.mark.parametrize('mode', [True, 1, 5, 'bad', 'gcv'])
-def test_check_gcv_mode_error(mode):
-    X, y = make_regression(n_samples=5, n_features=2)
-    gcv = RidgeCV(gcv_mode=mode)
-    with pytest.raises(ValueError, match="Unknown value for 'gcv_mode'"):
-        gcv.fit(X, y)
-    with pytest.raises(ValueError, match="Unknown value for 'gcv_mode'"):
-        _check_gcv_mode(X, mode)
-
-
-@pytest.mark.parametrize("sparse", [True, False])
+@pytest.mark.parametrize("sparse_container", [None] + CSR_CONTAINERS)
 @pytest.mark.parametrize(
-    'mode, mode_n_greater_than_p, mode_p_greater_than_n',
-    [(None, 'svd', 'eigen'),
-     ('auto', 'svd', 'eigen'),
-     ('eigen', 'eigen', 'eigen'),
-     ('svd', 'svd', 'svd')]
+    "mode, mode_n_greater_than_p, mode_p_greater_than_n",
+    [
+        (None, "svd", "eigen"),
+        ("auto", "svd", "eigen"),
+        ("eigen", "eigen", "eigen"),
+        ("svd", "svd", "svd"),
+    ],
 )
-def test_check_gcv_mode_choice(sparse, mode, mode_n_greater_than_p,
-                               mode_p_greater_than_n):
+def test_check_gcv_mode_choice(
+    sparse_container, mode, mode_n_greater_than_p, mode_p_greater_than_n
+):
     X, _ = make_regression(n_samples=5, n_features=2)
-    if sparse:
-        X = sp.csr_matrix(X)
+    if sparse_container is not None:
+        X = sparse_container(X)
     assert _check_gcv_mode(X, mode) == mode_n_greater_than_p
     assert _check_gcv_mode(X.T, mode) == mode_p_greater_than_n
 
 
-def _test_ridge_loo(filter_):
+def _test_ridge_loo(sparse_container):
     # test that can work with both dense or sparse matrices
     n_samples = X_diabetes.shape[0]
 
     ret = []
 
-    fit_intercept = filter_ == DENSE_FILTER
+    if sparse_container is None:
+        X, fit_intercept = X_diabetes, True
+    else:
+        X, fit_intercept = sparse_container(X_diabetes), False
     ridge_gcv = _RidgeGCV(fit_intercept=fit_intercept)
 
     # check best alpha
-    ridge_gcv.fit(filter_(X_diabetes), y_diabetes)
+    ridge_gcv.fit(X, y_diabetes)
     alpha_ = ridge_gcv.alpha_
     ret.append(alpha_)
 
@@ -620,7 +994,7 @@ def _test_ridge_loo(filter_):
     f = ignore_warnings
     scoring = make_scorer(mean_squared_error, greater_is_better=False)
     ridge_gcv2 = RidgeCV(fit_intercept=False, scoring=scoring)
-    f(ridge_gcv2.fit)(filter_(X_diabetes), y_diabetes)
+    f(ridge_gcv2.fit)(X, y_diabetes)
     assert ridge_gcv2.alpha_ == pytest.approx(alpha_)
 
     # check that we get same best alpha with custom score_func
@@ -629,84 +1003,74 @@ def func(x, y):
 
     scoring = make_scorer(func)
     ridge_gcv3 = RidgeCV(fit_intercept=False, scoring=scoring)
-    f(ridge_gcv3.fit)(filter_(X_diabetes), y_diabetes)
+    f(ridge_gcv3.fit)(X, y_diabetes)
     assert ridge_gcv3.alpha_ == pytest.approx(alpha_)
 
     # check that we get same best alpha with a scorer
-    scorer = get_scorer('neg_mean_squared_error')
+    scorer = get_scorer("neg_mean_squared_error")
     ridge_gcv4 = RidgeCV(fit_intercept=False, scoring=scorer)
-    ridge_gcv4.fit(filter_(X_diabetes), y_diabetes)
+    ridge_gcv4.fit(X, y_diabetes)
     assert ridge_gcv4.alpha_ == pytest.approx(alpha_)
 
     # check that we get same best alpha with sample weights
-    if filter_ == DENSE_FILTER:
-        ridge_gcv.fit(filter_(X_diabetes), y_diabetes,
-                      sample_weight=np.ones(n_samples))
+    if sparse_container is None:
+        ridge_gcv.fit(X, y_diabetes, sample_weight=np.ones(n_samples))
         assert ridge_gcv.alpha_ == pytest.approx(alpha_)
 
     # simulate several responses
     Y = np.vstack((y_diabetes, y_diabetes)).T
 
-    ridge_gcv.fit(filter_(X_diabetes), Y)
-    Y_pred = ridge_gcv.predict(filter_(X_diabetes))
-    ridge_gcv.fit(filter_(X_diabetes), y_diabetes)
-    y_pred = ridge_gcv.predict(filter_(X_diabetes))
+    ridge_gcv.fit(X, Y)
+    Y_pred = ridge_gcv.predict(X)
+    ridge_gcv.fit(X, y_diabetes)
+    y_pred = ridge_gcv.predict(X)
 
-    assert_allclose(np.vstack((y_pred, y_pred)).T,
-                    Y_pred, rtol=1e-5)
+    assert_allclose(np.vstack((y_pred, y_pred)).T, Y_pred, rtol=1e-5)
 
     return ret
 
 
-def _test_ridge_cv_normalize(filter_):
-    ridge_cv = RidgeCV(normalize=True, cv=3)
-    ridge_cv.fit(filter_(10. * X_diabetes), y_diabetes)
-
-    gs = GridSearchCV(Ridge(normalize=True, solver='sparse_cg'), cv=3,
-                      param_grid={'alpha': ridge_cv.alphas})
-    gs.fit(filter_(10. * X_diabetes), y_diabetes)
-    assert gs.best_estimator_.alpha == ridge_cv.alpha_
-
-
-def _test_ridge_cv(filter_):
+def _test_ridge_cv(sparse_container):
+    X = X_diabetes if sparse_container is None else sparse_container(X_diabetes)
     ridge_cv = RidgeCV()
-    ridge_cv.fit(filter_(X_diabetes), y_diabetes)
-    ridge_cv.predict(filter_(X_diabetes))
+    ridge_cv.fit(X, y_diabetes)
+    ridge_cv.predict(X)
 
     assert len(ridge_cv.coef_.shape) == 1
-    assert type(ridge_cv.intercept_) == np.float64
+    assert type(ridge_cv.intercept_) is np.float64
 
     cv = KFold(5)
     ridge_cv.set_params(cv=cv)
-    ridge_cv.fit(filter_(X_diabetes), y_diabetes)
-    ridge_cv.predict(filter_(X_diabetes))
+    ridge_cv.fit(X, y_diabetes)
+    ridge_cv.predict(X)
 
     assert len(ridge_cv.coef_.shape) == 1
-    assert type(ridge_cv.intercept_) == np.float64
+    assert type(ridge_cv.intercept_) is np.float64
 
 
 @pytest.mark.parametrize(
     "ridge, make_dataset",
-    [(RidgeCV(store_cv_values=False), make_regression),
-     (RidgeClassifierCV(store_cv_values=False), make_classification)]
+    [
+        (RidgeCV(store_cv_results=False), make_regression),
+        (RidgeClassifierCV(store_cv_results=False), make_classification),
+    ],
 )
-def test_ridge_gcv_cv_values_not_stored(ridge, make_dataset):
-    # Check that `cv_values_` is not stored when store_cv_values is False
+def test_ridge_gcv_cv_results_not_stored(ridge, make_dataset):
+    # Check that `cv_results_` is not stored when store_cv_results is False
     X, y = make_dataset(n_samples=6, random_state=42)
     ridge.fit(X, y)
-    assert not hasattr(ridge, "cv_values_")
+    assert not hasattr(ridge, "cv_results_")
 
 
 @pytest.mark.parametrize(
     "ridge, make_dataset",
-    [(RidgeCV(), make_regression),
-     (RidgeClassifierCV(), make_classification)]
+    [(RidgeCV(), make_regression), (RidgeClassifierCV(), make_classification)],
 )
 @pytest.mark.parametrize("cv", [None, 3])
 def test_ridge_best_score(ridge, make_dataset, cv):
     # check that the best_score_ is store
     X, y = make_dataset(n_samples=6, random_state=42)
-    ridge.set_params(store_cv_values=False, cv=cv)
+    ridge.set_params(store_cv_results=False, cv=cv)
     ridge.fit(X, y)
     assert hasattr(ridge, "best_score_")
     assert isinstance(ridge.best_score_, float)
@@ -721,16 +1085,17 @@ def test_ridge_cv_individual_penalties():
     # a different optimal alpha.
     n_samples, n_features, n_targets = 20, 5, 3
     y = rng.randn(n_samples, n_targets)
-    X = (np.dot(y[:, [0]], np.ones((1, n_features))) +
-         np.dot(y[:, [1]], 0.05 * np.ones((1, n_features))) +
-         np.dot(y[:, [2]], 0.001 * np.ones((1, n_features))) +
-         rng.randn(n_samples, n_features))
+    X = (
+        np.dot(y[:, [0]], np.ones((1, n_features)))
+        + np.dot(y[:, [1]], 0.05 * np.ones((1, n_features)))
+        + np.dot(y[:, [2]], 0.001 * np.ones((1, n_features)))
+        + rng.randn(n_samples, n_features)
+    )
 
     alphas = (1, 100, 1000)
 
     # Find optimal alpha for each target
-    optimal_alphas = [RidgeCV(alphas=alphas).fit(X, target).alpha_
-                      for target in y.T]
+    optimal_alphas = [RidgeCV(alphas=alphas).fit(X, target).alpha_ for target in y.T]
 
     # Find optimal alphas for all targets simultaneously
     ridge_cv = RidgeCV(alphas=alphas, alpha_per_target=True).fit(X, y)
@@ -738,36 +1103,38 @@ def test_ridge_cv_individual_penalties():
 
     # The resulting regression weights should incorporate the different
     # alpha values.
-    assert_array_almost_equal(Ridge(alpha=ridge_cv.alpha_).fit(X, y).coef_,
-                              ridge_cv.coef_)
+    assert_array_almost_equal(
+        Ridge(alpha=ridge_cv.alpha_).fit(X, y).coef_, ridge_cv.coef_
+    )
 
-    # Test shape of alpha_ and cv_values_
-    ridge_cv = RidgeCV(alphas=alphas, alpha_per_target=True,
-                       store_cv_values=True).fit(X, y)
+    # Test shape of alpha_ and cv_results_
+    ridge_cv = RidgeCV(alphas=alphas, alpha_per_target=True, store_cv_results=True).fit(
+        X, y
+    )
     assert ridge_cv.alpha_.shape == (n_targets,)
     assert ridge_cv.best_score_.shape == (n_targets,)
-    assert ridge_cv.cv_values_.shape == (n_samples, len(alphas), n_targets)
+    assert ridge_cv.cv_results_.shape == (n_samples, len(alphas), n_targets)
 
     # Test edge case of there being only one alpha value
-    ridge_cv = RidgeCV(alphas=1, alpha_per_target=True,
-                       store_cv_values=True).fit(X, y)
+    ridge_cv = RidgeCV(alphas=1, alpha_per_target=True, store_cv_results=True).fit(X, y)
     assert ridge_cv.alpha_.shape == (n_targets,)
     assert ridge_cv.best_score_.shape == (n_targets,)
-    assert ridge_cv.cv_values_.shape == (n_samples, n_targets, 1)
+    assert ridge_cv.cv_results_.shape == (n_samples, n_targets, 1)
 
     # Test edge case of there being only one target
-    ridge_cv = RidgeCV(alphas=alphas, alpha_per_target=True,
-                       store_cv_values=True).fit(X, y[:, 0])
+    ridge_cv = RidgeCV(alphas=alphas, alpha_per_target=True, store_cv_results=True).fit(
+        X, y[:, 0]
+    )
     assert np.isscalar(ridge_cv.alpha_)
     assert np.isscalar(ridge_cv.best_score_)
-    assert ridge_cv.cv_values_.shape == (n_samples, len(alphas))
+    assert ridge_cv.cv_results_.shape == (n_samples, len(alphas))
 
     # Try with a custom scoring function
-    ridge_cv = RidgeCV(alphas=alphas, alpha_per_target=True,
-                       scoring='r2').fit(X, y)
+    ridge_cv = RidgeCV(alphas=alphas, alpha_per_target=True, scoring="r2").fit(X, y)
     assert_array_equal(optimal_alphas, ridge_cv.alpha_)
-    assert_array_almost_equal(Ridge(alpha=ridge_cv.alpha_).fit(X, y).coef_,
-                              ridge_cv.coef_)
+    assert_array_almost_equal(
+        Ridge(alpha=ridge_cv.alpha_).fit(X, y).coef_, ridge_cv.coef_
+    )
 
     # Using a custom CV object should throw an error in combination with
     # alpha_per_target=True
@@ -780,118 +1147,243 @@ def test_ridge_cv_individual_penalties():
         ridge_cv.fit(X, y)
 
 
-def _test_ridge_diabetes(filter_):
+def _test_ridge_diabetes(sparse_container):
+    X = X_diabetes if sparse_container is None else sparse_container(X_diabetes)
     ridge = Ridge(fit_intercept=False)
-    ridge.fit(filter_(X_diabetes), y_diabetes)
-    return np.round(ridge.score(filter_(X_diabetes), y_diabetes), 5)
+    ridge.fit(X, y_diabetes)
+    return np.round(ridge.score(X, y_diabetes), 5)
 
 
-def _test_multi_ridge_diabetes(filter_):
+def _test_multi_ridge_diabetes(sparse_container):
     # simulate several responses
+    X = X_diabetes if sparse_container is None else sparse_container(X_diabetes)
     Y = np.vstack((y_diabetes, y_diabetes)).T
     n_features = X_diabetes.shape[1]
 
     ridge = Ridge(fit_intercept=False)
-    ridge.fit(filter_(X_diabetes), Y)
+    ridge.fit(X, Y)
     assert ridge.coef_.shape == (2, n_features)
-    Y_pred = ridge.predict(filter_(X_diabetes))
-    ridge.fit(filter_(X_diabetes), y_diabetes)
-    y_pred = ridge.predict(filter_(X_diabetes))
-    assert_array_almost_equal(np.vstack((y_pred, y_pred)).T,
-                              Y_pred, decimal=3)
+    Y_pred = ridge.predict(X)
+    ridge.fit(X, y_diabetes)
+    y_pred = ridge.predict(X)
+    assert_array_almost_equal(np.vstack((y_pred, y_pred)).T, Y_pred, decimal=3)
 
 
-def _test_ridge_classifiers(filter_):
+def _test_ridge_classifiers(sparse_container):
     n_classes = np.unique(y_iris).shape[0]
     n_features = X_iris.shape[1]
+    X = X_iris if sparse_container is None else sparse_container(X_iris)
+
     for reg in (RidgeClassifier(), RidgeClassifierCV()):
-        reg.fit(filter_(X_iris), y_iris)
+        reg.fit(X, y_iris)
         assert reg.coef_.shape == (n_classes, n_features)
-        y_pred = reg.predict(filter_(X_iris))
-        assert np.mean(y_iris == y_pred) > .79
+        y_pred = reg.predict(X)
+        assert np.mean(y_iris == y_pred) > 0.79
 
     cv = KFold(5)
     reg = RidgeClassifierCV(cv=cv)
-    reg.fit(filter_(X_iris), y_iris)
-    y_pred = reg.predict(filter_(X_iris))
+    reg.fit(X, y_iris)
+    y_pred = reg.predict(X)
     assert np.mean(y_iris == y_pred) >= 0.8
 
 
 @pytest.mark.parametrize("scoring", [None, "accuracy", _accuracy_callable])
 @pytest.mark.parametrize("cv", [None, KFold(5)])
-@pytest.mark.parametrize("filter_", [DENSE_FILTER, SPARSE_FILTER])
-def test_ridge_classifier_with_scoring(filter_, scoring, cv):
+@pytest.mark.parametrize("sparse_container", [None] + CSR_CONTAINERS)
+def test_ridge_classifier_with_scoring(sparse_container, scoring, cv):
     # non-regression test for #14672
     # check that RidgeClassifierCV works with all sort of scoring and
     # cross-validation
+    X = X_iris if sparse_container is None else sparse_container(X_iris)
     scoring_ = make_scorer(scoring) if callable(scoring) else scoring
     clf = RidgeClassifierCV(scoring=scoring_, cv=cv)
     # Smoke test to check that fit/predict does not raise error
-    clf.fit(filter_(X_iris), y_iris).predict(filter_(X_iris))
+    clf.fit(X, y_iris).predict(X)
 
 
 @pytest.mark.parametrize("cv", [None, KFold(5)])
-@pytest.mark.parametrize("filter_", [DENSE_FILTER, SPARSE_FILTER])
-def test_ridge_regression_custom_scoring(filter_, cv):
+@pytest.mark.parametrize("sparse_container", [None] + CSR_CONTAINERS)
+def test_ridge_regression_custom_scoring(sparse_container, cv):
     # check that custom scoring is working as expected
     # check the tie breaking strategy (keep the first alpha tried)
 
-    def _dummy_score(y_test, y_pred):
+    def _dummy_score(y_test, y_pred, **kwargs):
         return 0.42
 
+    X = X_iris if sparse_container is None else sparse_container(X_iris)
     alphas = np.logspace(-2, 2, num=5)
-    clf = RidgeClassifierCV(
-        alphas=alphas, scoring=make_scorer(_dummy_score), cv=cv
-    )
-    clf.fit(filter_(X_iris), y_iris)
+    clf = RidgeClassifierCV(alphas=alphas, scoring=make_scorer(_dummy_score), cv=cv)
+    clf.fit(X, y_iris)
     assert clf.best_score_ == pytest.approx(0.42)
     # In case of tie score, the first alphas will be kept
     assert clf.alpha_ == pytest.approx(alphas[0])
 
 
-def _test_tolerance(filter_):
+def _test_tolerance(sparse_container):
+    X = X_diabetes if sparse_container is None else sparse_container(X_diabetes)
+
     ridge = Ridge(tol=1e-5, fit_intercept=False)
-    ridge.fit(filter_(X_diabetes), y_diabetes)
-    score = ridge.score(filter_(X_diabetes), y_diabetes)
+    ridge.fit(X, y_diabetes)
+    score = ridge.score(X, y_diabetes)
 
     ridge2 = Ridge(tol=1e-3, fit_intercept=False)
-    ridge2.fit(filter_(X_diabetes), y_diabetes)
-    score2 = ridge2.score(filter_(X_diabetes), y_diabetes)
+    ridge2.fit(X, y_diabetes)
+    score2 = ridge2.score(X, y_diabetes)
 
     assert score >= score2
 
 
-def check_dense_sparse(test_func):
+def check_array_api_attributes(name, estimator, array_namespace, device, dtype_name):
+    xp = _array_api_for_tests(array_namespace, device)
+
+    X_iris_np = X_iris.astype(dtype_name)
+    y_iris_np = y_iris.astype(dtype_name)
+
+    X_iris_xp = xp.asarray(X_iris_np, device=device)
+    y_iris_xp = xp.asarray(y_iris_np, device=device)
+
+    estimator.fit(X_iris_np, y_iris_np)
+    coef_np = estimator.coef_
+    intercept_np = estimator.intercept_
+
+    with config_context(array_api_dispatch=True):
+        estimator_xp = clone(estimator).fit(X_iris_xp, y_iris_xp)
+        coef_xp = estimator_xp.coef_
+        assert coef_xp.shape == (4,)
+        assert coef_xp.dtype == X_iris_xp.dtype
+
+        assert_allclose(
+            _convert_to_numpy(coef_xp, xp=xp),
+            coef_np,
+            atol=_atol_for_type(dtype_name),
+        )
+        intercept_xp = estimator_xp.intercept_
+        assert intercept_xp.shape == ()
+        assert intercept_xp.dtype == X_iris_xp.dtype
+
+        assert_allclose(
+            _convert_to_numpy(intercept_xp, xp=xp),
+            intercept_np,
+            atol=_atol_for_type(dtype_name),
+        )
+
+
+@pytest.mark.parametrize(
+    "array_namespace, device, dtype_name",
+    yield_namespace_device_dtype_combinations(),
+    ids=_get_namespace_device_dtype_ids,
+)
+@pytest.mark.parametrize(
+    "check",
+    [check_array_api_input_and_values, check_array_api_attributes],
+    ids=_get_check_estimator_ids,
+)
+@pytest.mark.parametrize(
+    "estimator",
+    [Ridge(solver="svd")],
+    ids=_get_check_estimator_ids,
+)
+def test_ridge_array_api_compliance(
+    estimator, check, array_namespace, device, dtype_name
+):
+    name = estimator.__class__.__name__
+    check(name, estimator, array_namespace, device=device, dtype_name=dtype_name)
+
+
+@pytest.mark.parametrize(
+    "array_namespace", yield_namespaces(include_numpy_namespaces=False)
+)
+def test_array_api_error_and_warnings_for_solver_parameter(array_namespace):
+    xp = _array_api_for_tests(array_namespace, device=None)
+
+    X_iris_xp = xp.asarray(X_iris[:5])
+    y_iris_xp = xp.asarray(y_iris[:5])
+
+    available_solvers = Ridge._parameter_constraints["solver"][0].options
+    for solver in available_solvers - {"auto", "svd"}:
+        ridge = Ridge(solver=solver, positive=solver == "lbfgs")
+        expected_msg = (
+            f"Array API dispatch to namespace {xp.__name__} only supports "
+            f"solver 'svd'. Got '{solver}'."
+        )
+
+        with pytest.raises(ValueError, match=expected_msg):
+            with config_context(array_api_dispatch=True):
+                ridge.fit(X_iris_xp, y_iris_xp)
+
+    ridge = Ridge(solver="auto", positive=True)
+    expected_msg = (
+        "The solvers that support positive fitting do not support "
+        f"Array API dispatch to namespace {xp.__name__}. Please "
+        "either disable Array API dispatch, or use a numpy-like "
+        "namespace, or set `positive=False`."
+    )
+
+    with pytest.raises(ValueError, match=expected_msg):
+        with config_context(array_api_dispatch=True):
+            ridge.fit(X_iris_xp, y_iris_xp)
+
+    ridge = Ridge()
+    expected_msg = (
+        f"Using Array API dispatch to namespace {xp.__name__} with `solver='auto'` "
+        "will result in using the solver 'svd'. The results may differ from those "
+        "when using a Numpy array, because in that case the preferred solver would "
+        "be cholesky. Set `solver='svd'` to suppress this warning."
+    )
+    with pytest.warns(UserWarning, match=expected_msg):
+        with config_context(array_api_dispatch=True):
+            ridge.fit(X_iris_xp, y_iris_xp)
+
+
+@pytest.mark.parametrize("array_namespace", sorted(_NUMPY_NAMESPACE_NAMES))
+def test_array_api_numpy_namespace_no_warning(array_namespace):
+    xp = _array_api_for_tests(array_namespace, device=None)
+
+    X_iris_xp = xp.asarray(X_iris[:5])
+    y_iris_xp = xp.asarray(y_iris[:5])
+
+    ridge = Ridge()
+    expected_msg = (
+        "Results might be different than when Array API dispatch is "
+        "disabled, or when a numpy-like namespace is used"
+    )
+
+    with warnings.catch_warnings():
+        warnings.filterwarnings("error", message=expected_msg, category=UserWarning)
+        with config_context(array_api_dispatch=True):
+            ridge.fit(X_iris_xp, y_iris_xp)
+
+    # All numpy namespaces are compatible with all solver, in particular
+    # solvers that support `positive=True` (like 'lbfgs') should work.
+    with config_context(array_api_dispatch=True):
+        Ridge(solver="auto", positive=True).fit(X_iris_xp, y_iris_xp)
+
+
+@pytest.mark.parametrize(
+    "test_func",
+    (
+        _test_ridge_loo,
+        _test_ridge_cv,
+        _test_ridge_diabetes,
+        _test_multi_ridge_diabetes,
+        _test_ridge_classifiers,
+        _test_tolerance,
+    ),
+)
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_dense_sparse(test_func, csr_container):
     # test dense matrix
-    ret_dense = test_func(DENSE_FILTER)
+    ret_dense = test_func(None)
     # test sparse matrix
-    ret_sparse = test_func(SPARSE_FILTER)
+    ret_sparse = test_func(csr_container)
     # test that the outputs are the same
     if ret_dense is not None and ret_sparse is not None:
         assert_array_almost_equal(ret_dense, ret_sparse, decimal=3)
 
 
-@pytest.mark.parametrize(
-        'test_func',
-        (_test_ridge_loo, _test_ridge_cv, _test_ridge_cv_normalize,
-         _test_ridge_diabetes, _test_multi_ridge_diabetes,
-         _test_ridge_classifiers, _test_tolerance))
-def test_dense_sparse(test_func):
-    check_dense_sparse(test_func)
-
-
-def test_ridge_sparse_svd():
-    X = sp.csc_matrix(rng.rand(100, 10))
-    y = rng.rand(100)
-    ridge = Ridge(solver='svd', fit_intercept=False)
-    with pytest.raises(TypeError):
-        ridge.fit(X, y)
-
-
 def test_class_weights():
     # Test class weights.
-    X = np.array([[-1.0, -1.0], [-1.0, 0], [-.8, -1.0],
-                  [1.0, 1.0], [1.0, 0.0]])
+    X = np.array([[-1.0, -1.0], [-1.0, 0], [-0.8, -1.0], [1.0, 1.0], [1.0, 0.0]])
     y = [1, 1, 1, -1, -1]
 
     reg = RidgeClassifier(class_weight=None)
@@ -907,38 +1399,38 @@ def test_class_weights():
     assert_array_equal(reg.predict([[0.2, -1.0]]), np.array([-1]))
 
     # check if class_weight = 'balanced' can handle negative labels.
-    reg = RidgeClassifier(class_weight='balanced')
+    reg = RidgeClassifier(class_weight="balanced")
     reg.fit(X, y)
     assert_array_equal(reg.predict([[0.2, -1.0]]), np.array([1]))
 
     # class_weight = 'balanced', and class_weight = None should return
     # same values when y has equal number of all labels
-    X = np.array([[-1.0, -1.0], [-1.0, 0], [-.8, -1.0], [1.0, 1.0]])
+    X = np.array([[-1.0, -1.0], [-1.0, 0], [-0.8, -1.0], [1.0, 1.0]])
     y = [1, 1, -1, -1]
     reg = RidgeClassifier(class_weight=None)
     reg.fit(X, y)
-    rega = RidgeClassifier(class_weight='balanced')
+    rega = RidgeClassifier(class_weight="balanced")
     rega.fit(X, y)
     assert len(rega.classes_) == 2
     assert_array_almost_equal(reg.coef_, rega.coef_)
     assert_array_almost_equal(reg.intercept_, rega.intercept_)
 
 
-@pytest.mark.parametrize('reg', (RidgeClassifier, RidgeClassifierCV))
+@pytest.mark.parametrize("reg", (RidgeClassifier, RidgeClassifierCV))
 def test_class_weight_vs_sample_weight(reg):
     """Check class_weights resemble sample_weights behavior."""
 
     # Iris is balanced, so no effect expected for using 'balanced' weights
     reg1 = reg()
     reg1.fit(iris.data, iris.target)
-    reg2 = reg(class_weight='balanced')
+    reg2 = reg(class_weight="balanced")
     reg2.fit(iris.data, iris.target)
     assert_almost_equal(reg1.coef_, reg2.coef_)
 
     # Inflate importance of class 1, check against user-defined weights
     sample_weight = np.ones(iris.target.shape)
     sample_weight[iris.target == 1] *= 100
-    class_weight = {0: 1., 1: 100., 2: 1.}
+    class_weight = {0: 1.0, 1: 100.0, 2: 1.0}
     reg1 = reg()
     reg1.fit(iris.data, iris.target, sample_weight)
     reg2 = reg(class_weight=class_weight)
@@ -947,7 +1439,7 @@ def test_class_weight_vs_sample_weight(reg):
 
     # Check that sample_weight and class_weight are multiplicative
     reg1 = reg()
-    reg1.fit(iris.data, iris.target, sample_weight ** 2)
+    reg1.fit(iris.data, iris.target, sample_weight**2)
     reg2 = reg(class_weight=class_weight)
     reg2.fit(iris.data, iris.target, sample_weight)
     assert_almost_equal(reg1.coef_, reg2.coef_)
@@ -955,24 +1447,23 @@ def test_class_weight_vs_sample_weight(reg):
 
 def test_class_weights_cv():
     # Test class weights for cross validated ridge classifier.
-    X = np.array([[-1.0, -1.0], [-1.0, 0], [-.8, -1.0],
-                  [1.0, 1.0], [1.0, 0.0]])
+    X = np.array([[-1.0, -1.0], [-1.0, 0], [-0.8, -1.0], [1.0, 1.0], [1.0, 0.0]])
     y = [1, 1, 1, -1, -1]
 
-    reg = RidgeClassifierCV(class_weight=None, alphas=[.01, .1, 1])
+    reg = RidgeClassifierCV(class_weight=None, alphas=[0.01, 0.1, 1])
     reg.fit(X, y)
 
     # we give a small weights to class 1
-    reg = RidgeClassifierCV(class_weight={1: 0.001}, alphas=[.01, .1, 1, 10])
+    reg = RidgeClassifierCV(class_weight={1: 0.001}, alphas=[0.01, 0.1, 1, 10])
     reg.fit(X, y)
 
-    assert_array_equal(reg.predict([[-.2, 2]]), np.array([-1]))
+    assert_array_equal(reg.predict([[-0.2, 2]]), np.array([-1]))
 
 
 @pytest.mark.parametrize(
-    "scoring", [None, 'neg_mean_squared_error', _mean_squared_error_callable]
+    "scoring", [None, "neg_mean_squared_error", _mean_squared_error_callable]
 )
-def test_ridgecv_store_cv_values(scoring):
+def test_ridgecv_store_cv_results(scoring):
     rng = np.random.RandomState(42)
 
     n_samples = 8
@@ -983,28 +1474,27 @@ def test_ridgecv_store_cv_values(scoring):
 
     scoring_ = make_scorer(scoring) if callable(scoring) else scoring
 
-    r = RidgeCV(alphas=alphas, cv=None, store_cv_values=True, scoring=scoring_)
+    r = RidgeCV(alphas=alphas, cv=None, store_cv_results=True, scoring=scoring_)
 
     # with len(y.shape) == 1
     y = rng.randn(n_samples)
     r.fit(x, y)
-    assert r.cv_values_.shape == (n_samples, n_alphas)
+    assert r.cv_results_.shape == (n_samples, n_alphas)
 
     # with len(y.shape) == 2
     n_targets = 3
     y = rng.randn(n_samples, n_targets)
     r.fit(x, y)
-    assert r.cv_values_.shape == (n_samples, n_targets, n_alphas)
+    assert r.cv_results_.shape == (n_samples, n_targets, n_alphas)
 
-    r = RidgeCV(cv=3, store_cv_values=True, scoring=scoring)
-    with pytest.raises(ValueError, match='cv!=None and store_cv_values'):
+    r = RidgeCV(cv=3, store_cv_results=True, scoring=scoring)
+    with pytest.raises(ValueError, match="cv!=None and store_cv_results"):
         r.fit(x, y)
 
 
-@pytest.mark.parametrize("scoring", [None, 'accuracy', _accuracy_callable])
-def test_ridge_classifier_cv_store_cv_values(scoring):
-    x = np.array([[-1.0, -1.0], [-1.0, 0], [-.8, -1.0],
-                  [1.0, 1.0], [1.0, 0.0]])
+@pytest.mark.parametrize("scoring", [None, "accuracy", _accuracy_callable])
+def test_ridge_classifier_cv_store_cv_results(scoring):
+    x = np.array([[-1.0, -1.0], [-1.0, 0], [-0.8, -1.0], [1.0, 1.0], [1.0, 0.0]])
     y = np.array([1, 1, 1, -1, -1])
 
     n_samples = x.shape[0]
@@ -1014,21 +1504,64 @@ def test_ridge_classifier_cv_store_cv_values(scoring):
     scoring_ = make_scorer(scoring) if callable(scoring) else scoring
 
     r = RidgeClassifierCV(
-        alphas=alphas, cv=None, store_cv_values=True, scoring=scoring_
+        alphas=alphas, cv=None, store_cv_results=True, scoring=scoring_
     )
 
     # with len(y.shape) == 1
     n_targets = 1
     r.fit(x, y)
-    assert r.cv_values_.shape == (n_samples, n_targets, n_alphas)
+    assert r.cv_results_.shape == (n_samples, n_targets, n_alphas)
 
     # with len(y.shape) == 2
-    y = np.array([[1, 1, 1, -1, -1],
-                  [1, -1, 1, -1, 1],
-                  [-1, -1, 1, -1, -1]]).transpose()
+    y = np.array(
+        [[1, 1, 1, -1, -1], [1, -1, 1, -1, 1], [-1, -1, 1, -1, -1]]
+    ).transpose()
     n_targets = y.shape[1]
     r.fit(x, y)
-    assert r.cv_values_.shape == (n_samples, n_targets, n_alphas)
+    assert r.cv_results_.shape == (n_samples, n_targets, n_alphas)
+
+
+@pytest.mark.parametrize("Estimator", [RidgeCV, RidgeClassifierCV])
+def test_ridgecv_alphas_conversion(Estimator):
+    rng = np.random.RandomState(0)
+    alphas = (0.1, 1.0, 10.0)
+
+    n_samples, n_features = 5, 5
+    if Estimator is RidgeCV:
+        y = rng.randn(n_samples)
+    else:
+        y = rng.randint(0, 2, n_samples)
+    X = rng.randn(n_samples, n_features)
+
+    ridge_est = Estimator(alphas=alphas)
+    assert ridge_est.alphas is alphas, (
+        f"`alphas` was mutated in `{Estimator.__name__}.__init__`"
+    )
+
+    ridge_est.fit(X, y)
+    assert_array_equal(ridge_est.alphas, np.asarray(alphas))
+
+
+@pytest.mark.parametrize("cv", [None, 3])
+@pytest.mark.parametrize("Estimator", [RidgeCV, RidgeClassifierCV])
+def test_ridgecv_alphas_zero(cv, Estimator):
+    """Check alpha=0.0 raises error only when `cv=None`."""
+    rng = np.random.RandomState(0)
+    alphas = (0.0, 1.0, 10.0)
+
+    n_samples, n_features = 5, 5
+    if Estimator is RidgeCV:
+        y = rng.randn(n_samples)
+    else:
+        y = rng.randint(0, 2, n_samples)
+    X = rng.randn(n_samples, n_features)
+
+    ridge_est = Estimator(alphas=alphas, cv=cv)
+    if cv is None:
+        with pytest.raises(ValueError, match=r"alphas\[0\] == 0.0, must be > 0.0."):
+            ridge_est.fit(X, y)
+    else:
+        ridge_est.fit(X, y)
 
 
 def test_ridgecv_sample_weight():
@@ -1047,7 +1580,7 @@ def test_ridgecv_sample_weight():
         ridgecv.fit(X, y, sample_weight=sample_weight)
 
         # Check using GridSearchCV directly
-        parameters = {'alpha': alphas}
+        parameters = {"alpha": alphas}
         gs = GridSearchCV(Ridge(), parameters, cv=cv)
         gs.fit(X, y, sample_weight=sample_weight)
 
@@ -1067,8 +1600,8 @@ def test_raises_value_error_if_sample_weights_greater_than_1d():
         X = rng.randn(n_samples, n_features)
         y = rng.randn(n_samples)
         sample_weights_OK = rng.randn(n_samples) ** 2 + 1
-        sample_weights_OK_1 = 1.
-        sample_weights_OK_2 = 2.
+        sample_weights_OK_1 = 1.0
+        sample_weights_OK_2 = 2.0
         sample_weights_not_OK = sample_weights_OK[:, np.newaxis]
         sample_weights_not_OK_2 = sample_weights_OK[np.newaxis, :]
 
@@ -1094,40 +1627,30 @@ def fit_ridge_not_ok_2():
             fit_ridge_not_ok_2()
 
 
-def test_sparse_design_with_sample_weights():
+@pytest.mark.parametrize("n_samples,n_features", [[2, 3], [3, 2]])
+@pytest.mark.parametrize(
+    "sparse_container",
+    COO_CONTAINERS + CSC_CONTAINERS + CSR_CONTAINERS + DOK_CONTAINERS + LIL_CONTAINERS,
+)
+def test_sparse_design_with_sample_weights(n_samples, n_features, sparse_container):
     # Sample weights must work with sparse matrices
-
-    n_sampless = [2, 3]
-    n_featuress = [3, 2]
-
     rng = np.random.RandomState(42)
 
-    sparse_matrix_converters = [sp.coo_matrix,
-                                sp.csr_matrix,
-                                sp.csc_matrix,
-                                sp.lil_matrix,
-                                sp.dok_matrix
-                                ]
-
-    sparse_ridge = Ridge(alpha=1., fit_intercept=False)
-    dense_ridge = Ridge(alpha=1., fit_intercept=False)
+    sparse_ridge = Ridge(alpha=1.0, fit_intercept=False)
+    dense_ridge = Ridge(alpha=1.0, fit_intercept=False)
 
-    for n_samples, n_features in zip(n_sampless, n_featuress):
-        X = rng.randn(n_samples, n_features)
-        y = rng.randn(n_samples)
-        sample_weights = rng.randn(n_samples) ** 2 + 1
-        for sparse_converter in sparse_matrix_converters:
-            X_sparse = sparse_converter(X)
-            sparse_ridge.fit(X_sparse, y, sample_weight=sample_weights)
-            dense_ridge.fit(X, y, sample_weight=sample_weights)
+    X = rng.randn(n_samples, n_features)
+    y = rng.randn(n_samples)
+    sample_weights = rng.randn(n_samples) ** 2 + 1
+    X_sparse = sparse_container(X)
+    sparse_ridge.fit(X_sparse, y, sample_weight=sample_weights)
+    dense_ridge.fit(X, y, sample_weight=sample_weights)
 
-            assert_array_almost_equal(sparse_ridge.coef_, dense_ridge.coef_,
-                                      decimal=6)
+    assert_array_almost_equal(sparse_ridge.coef_, dense_ridge.coef_, decimal=6)
 
 
 def test_ridgecv_int_alphas():
-    X = np.array([[-1.0, -1.0], [-1.0, 0], [-.8, -1.0],
-                  [1.0, 1.0], [1.0, 0.0]])
+    X = np.array([[-1.0, -1.0], [-1.0, 0], [-0.8, -1.0], [1.0, 1.0], [1.0, 0.0]])
     y = [1, 1, 1, -1, -1]
 
     # Integers
@@ -1135,39 +1658,50 @@ def test_ridgecv_int_alphas():
     ridge.fit(X, y)
 
 
-def test_ridgecv_negative_alphas():
-    X = np.array([[-1.0, -1.0], [-1.0, 0], [-.8, -1.0],
-                  [1.0, 1.0], [1.0, 0.0]])
-    y = [1, 1, 1, -1, -1]
-
-    # Negative integers
-    ridge = RidgeCV(alphas=(-1, -10, -100))
-    with pytest.raises(ValueError, match="alphas must be strictly positive"):
-        ridge.fit(X, y)
-
-    # Negative floats
-    ridge = RidgeCV(alphas=(-0.1, -1.0, -10.0))
-    with pytest.raises(ValueError, match="alphas must be strictly positive"):
-        ridge.fit(X, y)
+@pytest.mark.parametrize("Estimator", [RidgeCV, RidgeClassifierCV])
+@pytest.mark.parametrize(
+    "params, err_type, err_msg",
+    [
+        ({"alphas": (1, -1, -100)}, ValueError, r"alphas\[1\] == -1, must be > 0.0"),
+        (
+            {"alphas": (-0.1, -1.0, -10.0)},
+            ValueError,
+            r"alphas\[0\] == -0.1, must be > 0.0",
+        ),
+        (
+            {"alphas": (1, 1.0, "1")},
+            TypeError,
+            r"alphas\[2\] must be an instance of float, not str",
+        ),
+    ],
+)
+def test_ridgecv_alphas_validation(Estimator, params, err_type, err_msg):
+    """Check the `alphas` validation in RidgeCV and RidgeClassifierCV."""
 
+    n_samples, n_features = 5, 5
+    X = rng.randn(n_samples, n_features)
+    y = rng.randint(0, 2, n_samples)
 
-def test_raises_value_error_if_solver_not_supported():
-    # Tests whether a ValueError is raised if a non-identified solver
-    # is passed to ridge_regression
+    with pytest.raises(err_type, match=err_msg):
+        Estimator(**params).fit(X, y)
 
-    wrong_solver = "This is not a solver (MagritteSolveCV QuantumBitcoin)"
 
-    exception = ValueError
-    message = ("Known solvers are 'sparse_cg', 'cholesky', 'svd'"
-               " 'lsqr', 'sag' or 'saga'. Got %s." % wrong_solver)
+@pytest.mark.parametrize("Estimator", [RidgeCV, RidgeClassifierCV])
+def test_ridgecv_alphas_scalar(Estimator):
+    """Check the case when `alphas` is a scalar.
+    This case was supported in the past when `alphas` where converted
+    into array in `__init__`.
+    We add this test to ensure backward compatibility.
+    """
 
-    def func():
-        X = np.eye(3)
-        y = np.ones(3)
-        ridge_regression(X, y, alpha=1., solver=wrong_solver)
+    n_samples, n_features = 5, 5
+    X = rng.randn(n_samples, n_features)
+    if Estimator is RidgeCV:
+        y = rng.randn(n_samples)
+    else:
+        y = rng.randint(0, 2, n_samples)
 
-        with pytest.raises(exception, match=message):
-            func()
+    Estimator(alphas=1).fit(X, y)
 
 
 def test_sparse_cg_max_iter():
@@ -1176,7 +1710,7 @@ def test_sparse_cg_max_iter():
     assert reg.coef_.shape[0] == X_diabetes.shape[1]
 
 
-@ignore_warnings
+@pytest.mark.filterwarnings("ignore::sklearn.exceptions.ConvergenceWarning")
 def test_n_iter():
     # Test that self.n_iter_ is correct.
     n_targets = 2
@@ -1184,82 +1718,108 @@ def test_n_iter():
     y_n = np.tile(y, (n_targets, 1)).T
 
     for max_iter in range(1, 4):
-        for solver in ('sag', 'saga', 'lsqr'):
+        for solver in ("sag", "saga", "lsqr"):
             reg = Ridge(solver=solver, max_iter=max_iter, tol=1e-12)
             reg.fit(X, y_n)
             assert_array_equal(reg.n_iter_, np.tile(max_iter, n_targets))
 
-    for solver in ('sparse_cg', 'svd', 'cholesky'):
+    for solver in ("sparse_cg", "svd", "cholesky"):
         reg = Ridge(solver=solver, max_iter=1, tol=1e-1)
         reg.fit(X, y_n)
         assert reg.n_iter_ is None
 
 
-@pytest.mark.parametrize('solver', ['sparse_cg', 'auto'])
-def test_ridge_fit_intercept_sparse(solver):
-    X, y = _make_sparse_offset_regression(n_features=20, random_state=0)
-    X_csr = sp.csr_matrix(X)
-
-    # for now only sparse_cg can correctly fit an intercept with sparse X with
-    # default tol and max_iter.
-    # sag is tested separately in test_ridge_fit_intercept_sparse_sag
-    # because it requires more iterations and should raise a warning if default
-    # max_iter is used.
-    # other solvers raise an exception, as checked in
-    # test_ridge_fit_intercept_sparse_error
-    #
+@pytest.mark.parametrize("solver", ["lsqr", "sparse_cg", "lbfgs", "auto"])
+@pytest.mark.parametrize("with_sample_weight", [True, False])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_ridge_fit_intercept_sparse(
+    solver, with_sample_weight, global_random_seed, csr_container
+):
+    """Check that ridge finds the same coefs and intercept on dense and sparse input
+    in the presence of sample weights.
+
+    For now only sparse_cg and lbfgs can correctly fit an intercept
+    with sparse X with default tol and max_iter.
+    'sag' is tested separately in test_ridge_fit_intercept_sparse_sag because it
+    requires more iterations and should raise a warning if default max_iter is used.
+    Other solvers raise an exception, as checked in
+    test_ridge_fit_intercept_sparse_error
+    """
+    positive = solver == "lbfgs"
+    X, y = _make_sparse_offset_regression(
+        n_features=20, random_state=global_random_seed, positive=positive
+    )
+
+    sample_weight = None
+    if with_sample_weight:
+        rng = np.random.RandomState(global_random_seed)
+        sample_weight = 1.0 + rng.uniform(size=X.shape[0])
+
     # "auto" should switch to "sparse_cg" when X is sparse
     # so the reference we use for both ("auto" and "sparse_cg") is
     # Ridge(solver="sparse_cg"), fitted using the dense representation (note
     # that "sparse_cg" can fit sparse or dense data)
-    dense_ridge = Ridge(solver='sparse_cg')
-    sparse_ridge = Ridge(solver=solver)
-    dense_ridge.fit(X, y)
-    with pytest.warns(None) as record:
-        sparse_ridge.fit(X_csr, y)
-    assert len(record) == 0
-    assert np.allclose(dense_ridge.intercept_, sparse_ridge.intercept_)
-    assert np.allclose(dense_ridge.coef_, sparse_ridge.coef_)
+    dense_solver = "sparse_cg" if solver == "auto" else solver
+    dense_ridge = Ridge(solver=dense_solver, tol=1e-12, positive=positive)
+    sparse_ridge = Ridge(solver=solver, tol=1e-12, positive=positive)
+
+    dense_ridge.fit(X, y, sample_weight=sample_weight)
+    sparse_ridge.fit(csr_container(X), y, sample_weight=sample_weight)
+
+    assert_allclose(dense_ridge.intercept_, sparse_ridge.intercept_)
+    assert_allclose(dense_ridge.coef_, sparse_ridge.coef_, rtol=5e-7)
 
 
-@pytest.mark.parametrize('solver', ['saga', 'lsqr', 'svd', 'cholesky'])
-def test_ridge_fit_intercept_sparse_error(solver):
+@pytest.mark.parametrize("solver", ["saga", "svd", "cholesky"])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_ridge_fit_intercept_sparse_error(solver, csr_container):
     X, y = _make_sparse_offset_regression(n_features=20, random_state=0)
-    X_csr = sp.csr_matrix(X)
+    X_csr = csr_container(X)
     sparse_ridge = Ridge(solver=solver)
     err_msg = "solver='{}' does not support".format(solver)
     with pytest.raises(ValueError, match=err_msg):
         sparse_ridge.fit(X_csr, y)
 
 
-def test_ridge_fit_intercept_sparse_sag():
+@pytest.mark.parametrize("with_sample_weight", [True, False])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_ridge_fit_intercept_sparse_sag(
+    with_sample_weight, global_random_seed, csr_container
+):
     X, y = _make_sparse_offset_regression(
-        n_features=5, n_samples=20, random_state=0, X_offset=5.)
-    X_csr = sp.csr_matrix(X)
+        n_features=5, n_samples=20, random_state=global_random_seed, X_offset=5.0
+    )
+    if with_sample_weight:
+        rng = np.random.RandomState(global_random_seed)
+        sample_weight = 1.0 + rng.uniform(size=X.shape[0])
+    else:
+        sample_weight = None
+    X_csr = csr_container(X)
 
-    params = dict(alpha=1., solver='sag', fit_intercept=True,
-                  tol=1e-10, max_iter=100000)
+    params = dict(
+        alpha=1.0, solver="sag", fit_intercept=True, tol=1e-10, max_iter=100000
+    )
     dense_ridge = Ridge(**params)
     sparse_ridge = Ridge(**params)
-    dense_ridge.fit(X, y)
-    with pytest.warns(None) as record:
-        sparse_ridge.fit(X_csr, y)
-    assert len(record) == 0
-    assert np.allclose(dense_ridge.intercept_, sparse_ridge.intercept_,
-                       rtol=1e-4)
-    assert np.allclose(dense_ridge.coef_, sparse_ridge.coef_, rtol=1e-4)
+    dense_ridge.fit(X, y, sample_weight=sample_weight)
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", UserWarning)
+        sparse_ridge.fit(X_csr, y, sample_weight=sample_weight)
+    assert_allclose(dense_ridge.intercept_, sparse_ridge.intercept_, rtol=1e-4)
+    assert_allclose(dense_ridge.coef_, sparse_ridge.coef_, rtol=1e-4)
     with pytest.warns(UserWarning, match='"sag" solver requires.*'):
-        Ridge(solver='sag').fit(X_csr, y)
+        Ridge(solver="sag", fit_intercept=True, tol=1e-3, max_iter=None).fit(X_csr, y)
 
 
-@pytest.mark.parametrize('return_intercept', [False, True])
-@pytest.mark.parametrize('sample_weight', [None, np.ones(1000)])
-@pytest.mark.parametrize('arr_type', [np.array, sp.csr_matrix])
-@pytest.mark.parametrize('solver', ['auto', 'sparse_cg', 'cholesky', 'lsqr',
-                                    'sag', 'saga'])
-def test_ridge_regression_check_arguments_validity(return_intercept,
-                                                   sample_weight, arr_type,
-                                                   solver):
+@pytest.mark.parametrize("return_intercept", [False, True])
+@pytest.mark.parametrize("sample_weight", [None, np.ones(1000)])
+@pytest.mark.parametrize("container", [np.array] + CSR_CONTAINERS)
+@pytest.mark.parametrize(
+    "solver", ["auto", "sparse_cg", "cholesky", "lsqr", "sag", "saga", "lbfgs"]
+)
+def test_ridge_regression_check_arguments_validity(
+    return_intercept, sample_weight, container, solver
+):
     """check if all combinations of arguments give valid estimations"""
 
     # test excludes 'svd' solver because it raises exception for sparse inputs
@@ -1268,30 +1828,41 @@ def test_ridge_regression_check_arguments_validity(return_intercept,
     X = rng.rand(1000, 3)
     true_coefs = [1, 2, 0.1]
     y = np.dot(X, true_coefs)
-    true_intercept = 0.
+    true_intercept = 0.0
     if return_intercept:
-        true_intercept = 10000.
+        true_intercept = 10000.0
     y += true_intercept
-    X_testing = arr_type(X)
+    X_testing = container(X)
 
-    alpha, atol, tol = 1e-3, 1e-4, 1e-6
+    alpha, tol = 1e-3, 1e-6
+    atol = 1e-3 if _IS_32BIT else 1e-4
 
-    if solver not in ['sag', 'auto'] and return_intercept:
+    positive = solver == "lbfgs"
+
+    if solver not in ["sag", "auto"] and return_intercept:
         with pytest.raises(ValueError, match="In Ridge, only 'sag' solver"):
-            ridge_regression(X_testing, y,
-                             alpha=alpha,
-                             solver=solver,
-                             sample_weight=sample_weight,
-                             return_intercept=return_intercept,
-                             tol=tol)
+            ridge_regression(
+                X_testing,
+                y,
+                alpha=alpha,
+                solver=solver,
+                sample_weight=sample_weight,
+                return_intercept=return_intercept,
+                positive=positive,
+                tol=tol,
+            )
         return
 
-    out = ridge_regression(X_testing, y, alpha=alpha,
-                           solver=solver,
-                           sample_weight=sample_weight,
-                           return_intercept=return_intercept,
-                           tol=tol,
-                           )
+    out = ridge_regression(
+        X_testing,
+        y,
+        alpha=alpha,
+        solver=solver,
+        sample_weight=sample_weight,
+        positive=positive,
+        return_intercept=return_intercept,
+        tol=tol,
+    )
 
     if return_intercept:
         coef, intercept = out
@@ -1301,17 +1872,13 @@ def test_ridge_regression_check_arguments_validity(return_intercept,
         assert_allclose(out, true_coefs, rtol=0, atol=atol)
 
 
-def test_ridge_classifier_no_support_multilabel():
-    X, y = make_multilabel_classification(n_samples=10, random_state=0)
-    with pytest.raises(ValueError):
-        RidgeClassifier().fit(X, y)
-
-
 @pytest.mark.parametrize(
-    "solver", ["svd", "sparse_cg", "cholesky", "lsqr", "sag", "saga"])
+    "solver", ["svd", "sparse_cg", "cholesky", "lsqr", "sag", "saga", "lbfgs"]
+)
 def test_dtype_match(solver):
     rng = np.random.RandomState(0)
     alpha = 1.0
+    positive = solver == "lbfgs"
 
     n_samples, n_features = 6, 5
     X_64 = rng.randn(n_samples, n_features)
@@ -1321,12 +1888,16 @@ def test_dtype_match(solver):
 
     tol = 2 * np.finfo(np.float32).resolution
     # Check type consistency 32bits
-    ridge_32 = Ridge(alpha=alpha, solver=solver, max_iter=500, tol=tol)
+    ridge_32 = Ridge(
+        alpha=alpha, solver=solver, max_iter=500, tol=tol, positive=positive
+    )
     ridge_32.fit(X_32, y_32)
     coef_32 = ridge_32.coef_
 
     # Check type consistency 64 bits
-    ridge_64 = Ridge(alpha=alpha, solver=solver, max_iter=500, tol=tol)
+    ridge_64 = Ridge(
+        alpha=alpha, solver=solver, max_iter=500, tol=tol, positive=positive
+    )
     ridge_64.fit(X_64, y_64)
     coef_64 = ridge_64.coef_
 
@@ -1342,7 +1913,7 @@ def test_dtype_match_cholesky():
     # Test different alphas in cholesky solver to ensure full coverage.
     # This test is separated from test_dtype_match for clarity.
     rng = np.random.RandomState(0)
-    alpha = (1.0, 0.5)
+    alpha = np.array([1.0, 0.5])
 
     n_samples, n_features, n_target = 6, 7, 2
     X_64 = rng.randn(n_samples, n_features)
@@ -1351,12 +1922,12 @@ def test_dtype_match_cholesky():
     y_32 = y_64.astype(np.float32)
 
     # Check type consistency 32bits
-    ridge_32 = Ridge(alpha=alpha, solver='cholesky')
+    ridge_32 = Ridge(alpha=alpha, solver="cholesky")
     ridge_32.fit(X_32, y_32)
     coef_32 = ridge_32.coef_
 
     # Check type consistency 64 bits
-    ridge_64 = Ridge(alpha=alpha, solver='cholesky')
+    ridge_64 = Ridge(alpha=alpha, solver="cholesky")
     ridge_64.fit(X_64, y_64)
     coef_64 = ridge_64.coef_
 
@@ -1369,8 +1940,9 @@ def test_dtype_match_cholesky():
 
 
 @pytest.mark.parametrize(
-    'solver', ['svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag', 'saga'])
-@pytest.mark.parametrize('seed', range(1))
+    "solver", ["svd", "cholesky", "lsqr", "sparse_cg", "sag", "saga", "lbfgs"]
+)
+@pytest.mark.parametrize("seed", range(1))
 def test_ridge_regression_dtype_stability(solver, seed):
     random_state = np.random.RandomState(seed)
     n_samples, n_features = 6, 5
@@ -1378,21 +1950,25 @@ def test_ridge_regression_dtype_stability(solver, seed):
     coef = random_state.randn(n_features)
     y = np.dot(X, coef) + 0.01 * random_state.randn(n_samples)
     alpha = 1.0
+    positive = solver == "lbfgs"
     results = dict()
     # XXX: Sparse CG seems to be far less numerically stable than the
     # others, maybe we should not enable float32 for this one.
     atol = 1e-3 if solver == "sparse_cg" else 1e-5
     for current_dtype in (np.float32, np.float64):
-        results[current_dtype] = ridge_regression(X.astype(current_dtype),
-                                                  y.astype(current_dtype),
-                                                  alpha=alpha,
-                                                  solver=solver,
-                                                  random_state=random_state,
-                                                  sample_weight=None,
-                                                  max_iter=500,
-                                                  tol=1e-10,
-                                                  return_n_iter=False,
-                                                  return_intercept=False)
+        results[current_dtype] = ridge_regression(
+            X.astype(current_dtype),
+            y.astype(current_dtype),
+            alpha=alpha,
+            solver=solver,
+            random_state=random_state,
+            sample_weight=None,
+            positive=positive,
+            max_iter=500,
+            tol=1e-10,
+            return_n_iter=False,
+            return_intercept=False,
+        )
 
     assert results[np.float32].dtype == np.float32
     assert results[np.float64].dtype == np.float64
@@ -1406,4 +1982,399 @@ def test_ridge_sag_with_X_fortran():
     X = np.asfortranarray(X)
     X = X[::2, :]
     y = y[::2]
-    Ridge(solver='sag').fit(X, y)
+    Ridge(solver="sag").fit(X, y)
+
+
+@pytest.mark.parametrize(
+    "Classifier, params",
+    [
+        (RidgeClassifier, {}),
+        (RidgeClassifierCV, {"cv": None}),
+        (RidgeClassifierCV, {"cv": 3}),
+    ],
+)
+def test_ridgeclassifier_multilabel(Classifier, params):
+    """Check that multilabel classification is supported and give meaningful
+    results."""
+    X, y = make_multilabel_classification(n_classes=1, random_state=0)
+    y = y.reshape(-1, 1)
+    Y = np.concatenate([y, y], axis=1)
+    clf = Classifier(**params).fit(X, Y)
+    Y_pred = clf.predict(X)
+
+    assert Y_pred.shape == Y.shape
+    assert_array_equal(Y_pred[:, 0], Y_pred[:, 1])
+    Ridge(solver="sag").fit(X, y)
+
+
+@pytest.mark.parametrize("solver", ["auto", "lbfgs"])
+@pytest.mark.parametrize("fit_intercept", [True, False])
+@pytest.mark.parametrize("alpha", [1e-3, 1e-2, 0.1, 1.0])
+def test_ridge_positive_regression_test(solver, fit_intercept, alpha):
+    """Test that positive Ridge finds true positive coefficients."""
+    X = np.array([[1, 2], [3, 4], [5, 6], [7, 8]])
+    coef = np.array([1, -10])
+    if fit_intercept:
+        intercept = 20
+        y = X.dot(coef) + intercept
+    else:
+        y = X.dot(coef)
+
+    model = Ridge(
+        alpha=alpha, positive=True, solver=solver, fit_intercept=fit_intercept
+    )
+    model.fit(X, y)
+    assert np.all(model.coef_ >= 0)
+
+
+@pytest.mark.parametrize("fit_intercept", [True, False])
+@pytest.mark.parametrize("alpha", [1e-3, 1e-2, 0.1, 1.0])
+def test_ridge_ground_truth_positive_test(fit_intercept, alpha):
+    """Test that Ridge w/wo positive converges to the same solution.
+
+    Ridge with positive=True and positive=False must give the same
+    when the ground truth coefs are all positive.
+    """
+    rng = np.random.RandomState(42)
+    X = rng.randn(300, 100)
+    coef = rng.uniform(0.1, 1.0, size=X.shape[1])
+    if fit_intercept:
+        intercept = 1
+        y = X @ coef + intercept
+    else:
+        y = X @ coef
+    y += rng.normal(size=X.shape[0]) * 0.01
+
+    results = []
+    for positive in [True, False]:
+        model = Ridge(
+            alpha=alpha, positive=positive, fit_intercept=fit_intercept, tol=1e-10
+        )
+        results.append(model.fit(X, y).coef_)
+    assert_allclose(*results, atol=1e-6, rtol=0)
+
+
+@pytest.mark.parametrize(
+    "solver", ["svd", "cholesky", "lsqr", "sparse_cg", "sag", "saga"]
+)
+def test_ridge_positive_error_test(solver):
+    """Test input validation for positive argument in Ridge."""
+    alpha = 0.1
+    X = np.array([[1, 2], [3, 4]])
+    coef = np.array([1, -1])
+    y = X @ coef
+
+    model = Ridge(alpha=alpha, positive=True, solver=solver, fit_intercept=False)
+    with pytest.raises(ValueError, match="does not support positive"):
+        model.fit(X, y)
+
+    with pytest.raises(ValueError, match="only 'lbfgs' solver can be used"):
+        _, _ = ridge_regression(
+            X, y, alpha, positive=True, solver=solver, return_intercept=False
+        )
+
+
+@pytest.mark.parametrize("alpha", [1e-3, 1e-2, 0.1, 1.0])
+def test_positive_ridge_loss(alpha):
+    """Check ridge loss consistency when positive argument is enabled."""
+    X, y = make_regression(n_samples=300, n_features=300, random_state=42)
+    alpha = 0.10
+    n_checks = 100
+
+    def ridge_loss(model, random_state=None, noise_scale=1e-8):
+        intercept = model.intercept_
+        if random_state is not None:
+            rng = np.random.RandomState(random_state)
+            coef = model.coef_ + rng.uniform(0, noise_scale, size=model.coef_.shape)
+        else:
+            coef = model.coef_
+
+        return 0.5 * np.sum((y - X @ coef - intercept) ** 2) + 0.5 * alpha * np.sum(
+            coef**2
+        )
+
+    model = Ridge(alpha=alpha).fit(X, y)
+    model_positive = Ridge(alpha=alpha, positive=True).fit(X, y)
+
+    # Check 1:
+    #   Loss for solution found by Ridge(positive=False)
+    #   is lower than that for solution found by Ridge(positive=True)
+    loss = ridge_loss(model)
+    loss_positive = ridge_loss(model_positive)
+    assert loss <= loss_positive
+
+    # Check 2:
+    #   Loss for solution found by Ridge(positive=True)
+    #   is lower than that for small random positive perturbation
+    #   of the positive solution.
+    for random_state in range(n_checks):
+        loss_perturbed = ridge_loss(model_positive, random_state=random_state)
+        assert loss_positive <= loss_perturbed
+
+
+@pytest.mark.parametrize("alpha", [1e-3, 1e-2, 0.1, 1.0])
+def test_lbfgs_solver_consistency(alpha):
+    """Test that LBGFS gets almost the same coef of svd when positive=False."""
+    X, y = make_regression(n_samples=300, n_features=300, random_state=42)
+    y = np.expand_dims(y, 1)
+    alpha = np.asarray([alpha])
+    config = {
+        "positive": False,
+        "tol": 1e-16,
+        "max_iter": 500000,
+    }
+
+    coef_lbfgs = _solve_lbfgs(X, y, alpha, **config)
+    coef_cholesky = _solve_svd(X, y, alpha)
+    assert_allclose(coef_lbfgs, coef_cholesky, atol=1e-4, rtol=0)
+
+
+def test_lbfgs_solver_error():
+    """Test that LBFGS solver raises ConvergenceWarning."""
+    X = np.array([[1, -1], [1, 1]])
+    y = np.array([-1e10, 1e10])
+
+    model = Ridge(
+        alpha=0.01,
+        solver="lbfgs",
+        fit_intercept=False,
+        tol=1e-12,
+        positive=True,
+        max_iter=1,
+    )
+    with pytest.warns(ConvergenceWarning, match="lbfgs solver did not converge"):
+        model.fit(X, y)
+
+
+@pytest.mark.parametrize("fit_intercept", [False, True])
+@pytest.mark.parametrize("sparse_container", [None] + CSR_CONTAINERS)
+@pytest.mark.parametrize("data", ["tall", "wide"])
+@pytest.mark.parametrize("solver", SOLVERS + ["lbfgs"])
+def test_ridge_sample_weight_consistency(
+    fit_intercept, sparse_container, data, solver, global_random_seed
+):
+    """Test that the impact of sample_weight is consistent.
+
+    Note that this test is stricter than the common test
+    check_sample_weight_equivalence alone.
+    """
+    # filter out solver that do not support sparse input
+    if sparse_container is not None:
+        if solver == "svd" or (solver in ("cholesky", "saga") and fit_intercept):
+            pytest.skip("unsupported configuration")
+
+    # XXX: this test is quite sensitive to the seed used to generate the data:
+    # ideally we would like the test to pass for any global_random_seed but this is not
+    # the case at the moment.
+    rng = np.random.RandomState(42)
+    n_samples = 12
+    if data == "tall":
+        n_features = n_samples // 2
+    else:
+        n_features = n_samples * 2
+
+    X = rng.rand(n_samples, n_features)
+    y = rng.rand(n_samples)
+    if sparse_container is not None:
+        X = sparse_container(X)
+    params = dict(
+        fit_intercept=fit_intercept,
+        alpha=1.0,
+        solver=solver,
+        positive=(solver == "lbfgs"),
+        random_state=global_random_seed,  # for sag/saga
+        tol=1e-12,
+    )
+
+    # 1) sample_weight=np.ones(..) should be equivalent to sample_weight=None,
+    # a special case of check_sample_weight_equivalence(name, reg), but we also
+    # test with sparse input.
+    reg = Ridge(**params).fit(X, y, sample_weight=None)
+    coef = reg.coef_.copy()
+    if fit_intercept:
+        intercept = reg.intercept_
+    sample_weight = np.ones_like(y)
+    reg.fit(X, y, sample_weight=sample_weight)
+    assert_allclose(reg.coef_, coef, rtol=1e-6)
+    if fit_intercept:
+        assert_allclose(reg.intercept_, intercept)
+
+    # 2) setting elements of sample_weight to 0 is equivalent to removing these samples,
+    # another special case of check_sample_weight_equivalence(name, reg), but we
+    # also test with sparse input
+    sample_weight = rng.uniform(low=0.01, high=2, size=X.shape[0])
+    sample_weight[-5:] = 0
+    y[-5:] *= 1000  # to make excluding those samples important
+    reg.fit(X, y, sample_weight=sample_weight)
+    coef = reg.coef_.copy()
+    if fit_intercept:
+        intercept = reg.intercept_
+    reg.fit(X[:-5, :], y[:-5], sample_weight=sample_weight[:-5])
+    assert_allclose(reg.coef_, coef, rtol=1e-6)
+    if fit_intercept:
+        assert_allclose(reg.intercept_, intercept)
+
+    # 3) scaling of sample_weight should have no effect
+    # Note: For models with penalty, scaling the penalty term might work.
+    reg2 = Ridge(**params).set_params(alpha=np.pi * params["alpha"])
+    reg2.fit(X, y, sample_weight=np.pi * sample_weight)
+    if solver in ("sag", "saga") and not fit_intercept:
+        pytest.xfail(f"Solver {solver} does fail test for scaling of sample_weight.")
+    assert_allclose(reg2.coef_, coef, rtol=1e-6)
+    if fit_intercept:
+        assert_allclose(reg2.intercept_, intercept)
+
+    # 4) check that multiplying sample_weight by 2 is equivalent
+    # to repeating corresponding samples twice
+    if sparse_container is not None:
+        X = X.toarray()
+    X2 = np.concatenate([X, X[: n_samples // 2]], axis=0)
+    y2 = np.concatenate([y, y[: n_samples // 2]])
+    sample_weight_1 = sample_weight.copy()
+    sample_weight_1[: n_samples // 2] *= 2
+    sample_weight_2 = np.concatenate(
+        [sample_weight, sample_weight[: n_samples // 2]], axis=0
+    )
+    if sparse_container is not None:
+        X = sparse_container(X)
+        X2 = sparse_container(X2)
+    reg1 = Ridge(**params).fit(X, y, sample_weight=sample_weight_1)
+    reg2 = Ridge(**params).fit(X2, y2, sample_weight=sample_weight_2)
+    assert_allclose(reg1.coef_, reg2.coef_)
+    if fit_intercept:
+        assert_allclose(reg1.intercept_, reg2.intercept_)
+
+
+@pytest.mark.parametrize("with_sample_weight", [False, True])
+@pytest.mark.parametrize("fit_intercept", [False, True])
+@pytest.mark.parametrize("n_targets", [1, 2])
+def test_ridge_cv_results_predictions(with_sample_weight, fit_intercept, n_targets):
+    """Check that the predictions stored in `cv_results_` are on the original scale.
+
+    The GCV approach works on scaled data: centered by an offset and scaled by the
+    square root of the sample weights. Thus, prior to computing scores, the
+    predictions need to be scaled back to the original scale. These predictions are
+    the ones stored in `cv_results_` in `RidgeCV`.
+
+    In this test, we check that the internal predictions stored in `cv_results_` are
+    equivalent to a naive LOO-CV grid search with a `Ridge` estimator.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/13998
+    """
+    X, y = make_regression(
+        n_samples=100, n_features=10, n_targets=n_targets, random_state=0
+    )
+    sample_weight = np.ones(shape=(X.shape[0],))
+    if with_sample_weight:
+        sample_weight[::2] = 0.5
+
+    alphas = (0.1, 1.0, 10.0)
+
+    # scoring should be set to store predictions and not the squared error
+    ridge_cv = RidgeCV(
+        alphas=alphas,
+        scoring="neg_mean_squared_error",
+        fit_intercept=fit_intercept,
+        store_cv_results=True,
+    )
+    ridge_cv.fit(X, y, sample_weight=sample_weight)
+
+    # manual grid-search with a `Ridge` estimator
+    predictions = np.empty(shape=(*y.shape, len(alphas)))
+    cv = LeaveOneOut()
+    for alpha_idx, alpha in enumerate(alphas):
+        for idx, (train_idx, test_idx) in enumerate(cv.split(X, y)):
+            ridge = Ridge(alpha=alpha, fit_intercept=fit_intercept)
+            ridge.fit(X[train_idx], y[train_idx], sample_weight[train_idx])
+            predictions[idx, ..., alpha_idx] = ridge.predict(X[test_idx])
+    assert_allclose(ridge_cv.cv_results_, predictions)
+
+
+def test_ridge_cv_multioutput_sample_weight(global_random_seed):
+    """Check that `RidgeCV` works properly with multioutput and sample_weight
+    when `scoring != None`.
+
+    We check the error reported by the RidgeCV is close to a naive LOO-CV using a
+    Ridge estimator.
+    """
+    X, y = make_regression(n_targets=2, random_state=global_random_seed)
+    sample_weight = np.ones(shape=(X.shape[0],))
+
+    ridge_cv = RidgeCV(scoring="neg_mean_squared_error", store_cv_results=True)
+    ridge_cv.fit(X, y, sample_weight=sample_weight)
+
+    cv = LeaveOneOut()
+    ridge = Ridge(alpha=ridge_cv.alpha_)
+    y_pred_loo = np.squeeze(
+        [
+            ridge.fit(X[train], y[train], sample_weight=sample_weight[train]).predict(
+                X[test]
+            )
+            for train, test in cv.split(X)
+        ]
+    )
+    assert_allclose(ridge_cv.best_score_, -mean_squared_error(y, y_pred_loo))
+
+
+def test_ridge_cv_custom_multioutput_scorer():
+    """Check that `RidgeCV` works properly with a custom multioutput scorer."""
+    X, y = make_regression(n_targets=2, random_state=0)
+
+    def custom_error(y_true, y_pred):
+        errors = (y_true - y_pred) ** 2
+        mean_errors = np.mean(errors, axis=0)
+        if mean_errors.ndim == 1:
+            # case of multioutput
+            return -np.average(mean_errors, weights=[2, 1])
+        # single output - this part of the code should not be reached in the case of
+        # multioutput scoring
+        return -mean_errors  # pragma: no cover
+
+    def custom_multioutput_scorer(estimator, X, y):
+        """Multioutput score that give twice more importance to the second target."""
+        return -custom_error(y, estimator.predict(X))
+
+    ridge_cv = RidgeCV(scoring=custom_multioutput_scorer)
+    ridge_cv.fit(X, y)
+
+    cv = LeaveOneOut()
+    ridge = Ridge(alpha=ridge_cv.alpha_)
+    y_pred_loo = np.squeeze(
+        [ridge.fit(X[train], y[train]).predict(X[test]) for train, test in cv.split(X)]
+    )
+
+    assert_allclose(ridge_cv.best_score_, -custom_error(y, y_pred_loo))
+
+
+# Metadata Routing Tests
+# ======================
+
+
+@pytest.mark.parametrize("metaestimator", [RidgeCV, RidgeClassifierCV])
+@config_context(enable_metadata_routing=True)
+def test_metadata_routing_with_default_scoring(metaestimator):
+    """Test that `RidgeCV` or `RidgeClassifierCV` with default `scoring`
+    argument (`None`), don't enter into `RecursionError` when metadata is routed.
+    """
+    metaestimator().get_metadata_routing()
+
+
+@pytest.mark.parametrize(
+    "metaestimator, make_dataset",
+    [
+        (RidgeCV(), make_regression),
+        (RidgeClassifierCV(), make_classification),
+    ],
+)
+@config_context(enable_metadata_routing=True)
+def test_set_score_request_with_default_scoring(metaestimator, make_dataset):
+    """Test that `set_score_request` is set within `RidgeCV.fit()` and
+    `RidgeClassifierCV.fit()` when using the default scoring and no
+    UnsetMetadataPassedError is raised. Regression test for the fix in PR #29634."""
+    X, y = make_dataset(n_samples=100, n_features=5, random_state=42)
+    metaestimator.fit(X, y, sample_weight=np.ones(X.shape[0]))
+
+
+# End of Metadata Routing Tests
+# =============================
diff --git a/sklearn/linear_model/tests/test_sag.py b/sklearn/linear_model/tests/test_sag.py
index 22432185cc09b..575838f8e8497 100644
--- a/sklearn/linear_model/tests/test_sag.py
+++ b/sklearn/linear_model/tests/test_sag.py
@@ -1,30 +1,26 @@
-# Authors: Danny Sullivan 
-#          Tom Dupre la Tour 
-#
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import math
-import pytest
+import re
+
 import numpy as np
-import scipy.sparse as sp
-from scipy.special import logsumexp
+import pytest
 
-from sklearn.linear_model._sag import get_auto_step_size
-from sklearn.linear_model._sag_fast import _multinomial_grad_loss_all_samples
+from sklearn.base import clone
+from sklearn.datasets import load_iris, make_blobs, make_classification
 from sklearn.linear_model import LogisticRegression, Ridge
-from sklearn.linear_model._base import make_dataset
-from sklearn.linear_model._logistic import _multinomial_loss_grad
-
+from sklearn.linear_model._sag import get_auto_step_size
+from sklearn.multiclass import OneVsRestClassifier
+from sklearn.preprocessing import LabelEncoder
+from sklearn.utils import check_random_state, compute_class_weight
+from sklearn.utils._testing import (
+    assert_allclose,
+    assert_almost_equal,
+    assert_array_almost_equal,
+)
 from sklearn.utils.extmath import row_norms
-from sklearn.utils._testing import assert_almost_equal
-from sklearn.utils._testing import assert_array_almost_equal
-from sklearn.utils._testing import assert_allclose
-from sklearn.utils._testing import assert_raise_message
-from sklearn.utils import compute_class_weight
-from sklearn.utils import check_random_state
-from sklearn.preprocessing import LabelEncoder, LabelBinarizer
-from sklearn.datasets import make_blobs, load_iris, make_classification
-from sklearn.base import clone
+from sklearn.utils.fixes import CSR_CONTAINERS
 
 iris = load_iris()
 
@@ -41,7 +37,7 @@ def log_dloss(p, y):
 
 
 def log_loss(p, y):
-    return np.mean(np.log(1. + np.exp(-y * p)))
+    return np.mean(np.log(1.0 + np.exp(-y * p)))
 
 
 # this is used for sag regression
@@ -58,12 +54,22 @@ def get_pobj(w, alpha, myX, myy, loss):
     w = w.ravel()
     pred = np.dot(myX, w)
     p = loss(pred, myy)
-    p += alpha * w.dot(w) / 2.
+    p += alpha * w.dot(w) / 2.0
     return p
 
 
-def sag(X, y, step_size, alpha, n_iter=1, dloss=None, sparse=False,
-        sample_weight=None, fit_intercept=True, saga=False):
+def sag(
+    X,
+    y,
+    step_size,
+    alpha,
+    n_iter=1,
+    dloss=None,
+    sparse=False,
+    sample_weight=None,
+    fit_intercept=True,
+    saga=False,
+):
     n_samples, n_features = X.shape[0], X.shape[1]
 
     weights = np.zeros(X.shape[1])
@@ -80,11 +86,11 @@ def sag(X, y, step_size, alpha, n_iter=1, dloss=None, sparse=False,
 
     # sparse data has a fixed decay of .01
     if sparse:
-        decay = .01
+        decay = 0.01
 
     for epoch in range(n_iter):
         for k in range(n_samples):
-            idx = int(rng.rand(1) * n_samples)
+            idx = int(rng.rand() * n_samples)
             # idx = k
             entry = X[idx]
             seen.add(idx)
@@ -97,33 +103,42 @@ def sag(X, y, step_size, alpha, n_iter=1, dloss=None, sparse=False,
             sum_gradient += gradient_correction
             gradient_memory[idx] = update
             if saga:
-                weights -= (gradient_correction *
-                            step_size * (1 - 1. / len(seen)))
+                weights -= gradient_correction * step_size * (1 - 1.0 / len(seen))
 
             if fit_intercept:
-                gradient_correction = (gradient -
-                                       intercept_gradient_memory[idx])
+                gradient_correction = gradient - intercept_gradient_memory[idx]
                 intercept_gradient_memory[idx] = gradient
                 intercept_sum_gradient += gradient_correction
-                gradient_correction *= step_size * (1. - 1. / len(seen))
+                gradient_correction *= step_size * (1.0 - 1.0 / len(seen))
                 if saga:
-                    intercept -= (step_size * intercept_sum_gradient /
-                                  len(seen) * decay) + gradient_correction
+                    intercept -= (
+                        step_size * intercept_sum_gradient / len(seen) * decay
+                    ) + gradient_correction
                 else:
-                    intercept -= (step_size * intercept_sum_gradient /
-                                  len(seen) * decay)
+                    intercept -= step_size * intercept_sum_gradient / len(seen) * decay
 
             weights -= step_size * sum_gradient / len(seen)
 
     return weights, intercept
 
 
-def sag_sparse(X, y, step_size, alpha, n_iter=1,
-               dloss=None, sample_weight=None, sparse=False,
-               fit_intercept=True, saga=False, random_state=0):
-    if step_size * alpha == 1.:
-        raise ZeroDivisionError("Sparse sag does not handle the case "
-                                "step_size * alpha == 1")
+def sag_sparse(
+    X,
+    y,
+    step_size,
+    alpha,
+    n_iter=1,
+    dloss=None,
+    sample_weight=None,
+    sparse=False,
+    fit_intercept=True,
+    saga=False,
+    random_state=0,
+):
+    if step_size * alpha == 1.0:
+        raise ZeroDivisionError(
+            "Sparse sag does not handle the case step_size * alpha == 1"
+        )
     n_samples, n_features = X.shape[0], X.shape[1]
 
     weights = np.zeros(n_features)
@@ -141,13 +156,13 @@ def sag_sparse(X, y, step_size, alpha, n_iter=1,
 
     # sparse data has a fixed decay of .01
     if sparse:
-        decay = .01
+        decay = 0.01
 
     counter = 0
     for epoch in range(n_iter):
         for k in range(n_samples):
             # idx = k
-            idx = int(rng.rand(1) * n_samples)
+            idx = int(rng.rand() * n_samples)
             entry = X[idx]
             seen.add(idx)
 
@@ -156,9 +171,9 @@ def sag_sparse(X, y, step_size, alpha, n_iter=1,
                     if last_updated[j] == 0:
                         weights[j] -= c_sum[counter - 1] * sum_gradient[j]
                     else:
-                        weights[j] -= ((c_sum[counter - 1] -
-                                        c_sum[last_updated[j] - 1]) *
-                                       sum_gradient[j])
+                        weights[j] -= (
+                            c_sum[counter - 1] - c_sum[last_updated[j] - 1]
+                        ) * sum_gradient[j]
                     last_updated[j] = counter
 
             p = (wscale * np.dot(entry, weights)) + intercept
@@ -172,38 +187,40 @@ def sag_sparse(X, y, step_size, alpha, n_iter=1,
             sum_gradient += gradient_correction
             if saga:
                 for j in range(n_features):
-                    weights[j] -= (gradient_correction[j] * step_size *
-                                   (1 - 1. / len(seen)) / wscale)
+                    weights[j] -= (
+                        gradient_correction[j]
+                        * step_size
+                        * (1 - 1.0 / len(seen))
+                        / wscale
+                    )
 
             if fit_intercept:
                 gradient_correction = gradient - gradient_memory[idx]
                 intercept_sum_gradient += gradient_correction
-                gradient_correction *= step_size * (1. - 1. / len(seen))
+                gradient_correction *= step_size * (1.0 - 1.0 / len(seen))
                 if saga:
-                    intercept -= ((step_size * intercept_sum_gradient /
-                                   len(seen) * decay) +
-                                  gradient_correction)
+                    intercept -= (
+                        step_size * intercept_sum_gradient / len(seen) * decay
+                    ) + gradient_correction
                 else:
-                    intercept -= (step_size * intercept_sum_gradient /
-                                  len(seen) * decay)
+                    intercept -= step_size * intercept_sum_gradient / len(seen) * decay
 
             gradient_memory[idx] = gradient
 
-            wscale *= (1.0 - alpha * step_size)
+            wscale *= 1.0 - alpha * step_size
             if counter == 0:
                 c_sum[0] = step_size / (wscale * len(seen))
             else:
-                c_sum[counter] = (c_sum[counter - 1] +
-                                  step_size / (wscale * len(seen)))
+                c_sum[counter] = c_sum[counter - 1] + step_size / (wscale * len(seen))
 
             if counter >= 1 and wscale < 1e-9:
                 for j in range(n_features):
                     if last_updated[j] == 0:
                         weights[j] -= c_sum[counter] * sum_gradient[j]
                     else:
-                        weights[j] -= ((c_sum[counter] -
-                                        c_sum[last_updated[j] - 1]) *
-                                       sum_gradient[j])
+                        weights[j] -= (
+                            c_sum[counter] - c_sum[last_updated[j] - 1]
+                        ) * sum_gradient[j]
                     last_updated[j] = counter + 1
                 c_sum[counter] = 0
                 weights *= wscale
@@ -215,49 +232,63 @@ def sag_sparse(X, y, step_size, alpha, n_iter=1,
         if last_updated[j] == 0:
             weights[j] -= c_sum[counter - 1] * sum_gradient[j]
         else:
-            weights[j] -= ((c_sum[counter - 1] -
-                            c_sum[last_updated[j] - 1]) *
-                           sum_gradient[j])
+            weights[j] -= (
+                c_sum[counter - 1] - c_sum[last_updated[j] - 1]
+            ) * sum_gradient[j]
     weights *= wscale
     return weights, intercept
 
 
 def get_step_size(X, alpha, fit_intercept, classification=True):
     if classification:
-        return (4.0 / (np.max(np.sum(X * X, axis=1)) +
-                       fit_intercept + 4.0 * alpha))
+        return 4.0 / (np.max(np.sum(X * X, axis=1)) + fit_intercept + 4.0 * alpha)
     else:
         return 1.0 / (np.max(np.sum(X * X, axis=1)) + fit_intercept + alpha)
 
 
 def test_classifier_matching():
     n_samples = 20
-    X, y = make_blobs(n_samples=n_samples, centers=2, random_state=0,
-                      cluster_std=0.1)
-    y[y == 0] = -1
+    X, y = make_blobs(n_samples=n_samples, centers=2, random_state=0, cluster_std=0.1)
+    # y must be 0 or 1
     alpha = 1.1
     fit_intercept = True
     step_size = get_step_size(X, alpha, fit_intercept)
-    for solver in ['sag', 'saga']:
-        if solver == 'sag':
+    for solver in ["sag", "saga"]:
+        if solver == "sag":
             n_iter = 80
         else:
             # SAGA variance w.r.t. stream order is higher
             n_iter = 300
-        clf = LogisticRegression(solver=solver, fit_intercept=fit_intercept,
-                                 tol=1e-11, C=1. / alpha / n_samples,
-                                 max_iter=n_iter, random_state=10,
-                                 multi_class='ovr')
+        clf = LogisticRegression(
+            solver=solver,
+            fit_intercept=fit_intercept,
+            tol=1e-11,
+            C=1.0 / alpha / n_samples,
+            max_iter=n_iter,
+            random_state=10,
+        )
         clf.fit(X, y)
 
-        weights, intercept = sag_sparse(X, y, step_size, alpha, n_iter=n_iter,
-                                        dloss=log_dloss,
-                                        fit_intercept=fit_intercept,
-                                        saga=solver == 'saga')
-        weights2, intercept2 = sag(X, y, step_size, alpha, n_iter=n_iter,
-                                   dloss=log_dloss,
-                                   fit_intercept=fit_intercept,
-                                   saga=solver == 'saga')
+        weights, intercept = sag_sparse(
+            X,
+            2 * y - 1,  # y must be -1 or +1
+            step_size,
+            alpha,
+            n_iter=n_iter,
+            dloss=log_dloss,
+            fit_intercept=fit_intercept,
+            saga=solver == "saga",
+        )
+        weights2, intercept2 = sag(
+            X,
+            2 * y - 1,  # y must be -1 or +1
+            step_size,
+            alpha,
+            n_iter=n_iter,
+            dloss=log_dloss,
+            fit_intercept=fit_intercept,
+            saga=solver == "saga",
+        )
         weights = np.atleast_2d(weights)
         intercept = np.atleast_1d(intercept)
         weights2 = np.atleast_2d(weights2)
@@ -278,21 +309,38 @@ def test_regressor_matching():
     true_w = rng.normal(size=n_features)
     y = X.dot(true_w)
 
-    alpha = 1.
+    alpha = 1.0
     n_iter = 100
     fit_intercept = True
 
     step_size = get_step_size(X, alpha, fit_intercept, classification=False)
-    clf = Ridge(fit_intercept=fit_intercept, tol=.00000000001, solver='sag',
-                alpha=alpha * n_samples, max_iter=n_iter)
+    clf = Ridge(
+        fit_intercept=fit_intercept,
+        tol=0.00000000001,
+        solver="sag",
+        alpha=alpha * n_samples,
+        max_iter=n_iter,
+    )
     clf.fit(X, y)
 
-    weights1, intercept1 = sag_sparse(X, y, step_size, alpha, n_iter=n_iter,
-                                      dloss=squared_dloss,
-                                      fit_intercept=fit_intercept)
-    weights2, intercept2 = sag(X, y, step_size, alpha, n_iter=n_iter,
-                               dloss=squared_dloss,
-                               fit_intercept=fit_intercept)
+    weights1, intercept1 = sag_sparse(
+        X,
+        y,
+        step_size,
+        alpha,
+        n_iter=n_iter,
+        dloss=squared_dloss,
+        fit_intercept=fit_intercept,
+    )
+    weights2, intercept2 = sag(
+        X,
+        y,
+        step_size,
+        alpha,
+        n_iter=n_iter,
+        dloss=squared_dloss,
+        fit_intercept=fit_intercept,
+    )
 
     assert_allclose(weights1, clf.coef_)
     assert_allclose(intercept1, clf.intercept_)
@@ -300,25 +348,34 @@ def test_regressor_matching():
     assert_allclose(intercept2, clf.intercept_)
 
 
-@pytest.mark.filterwarnings('ignore:The max_iter was reached')
-def test_sag_pobj_matches_logistic_regression():
+@pytest.mark.filterwarnings("ignore:The max_iter was reached")
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sag_pobj_matches_logistic_regression(csr_container):
     """tests if the sag pobj matches log reg"""
     n_samples = 100
     alpha = 1.0
     max_iter = 20
-    X, y = make_blobs(n_samples=n_samples, centers=2, random_state=0,
-                      cluster_std=0.1)
-
-    clf1 = LogisticRegression(solver='sag', fit_intercept=False, tol=.0000001,
-                              C=1. / alpha / n_samples, max_iter=max_iter,
-                              random_state=10, multi_class='ovr')
+    X, y = make_blobs(n_samples=n_samples, centers=2, random_state=0, cluster_std=0.1)
+
+    clf1 = LogisticRegression(
+        solver="sag",
+        fit_intercept=False,
+        tol=0.0000001,
+        C=1.0 / alpha / n_samples,
+        max_iter=max_iter,
+        random_state=10,
+    )
     clf2 = clone(clf1)
-    clf3 = LogisticRegression(fit_intercept=False, tol=.0000001,
-                              C=1. / alpha / n_samples, max_iter=max_iter,
-                              random_state=10, multi_class='ovr')
+    clf3 = LogisticRegression(
+        fit_intercept=False,
+        tol=0.0000001,
+        C=1.0 / alpha / n_samples,
+        max_iter=max_iter,
+        random_state=10,
+    )
 
     clf1.fit(X, y)
-    clf2.fit(sp.csr_matrix(X), y)
+    clf2.fit(csr_container(X), y)
     clf3.fit(X, y)
 
     pobj1 = get_pobj(clf1.coef_, alpha, X, y, log_loss)
@@ -330,8 +387,9 @@ def test_sag_pobj_matches_logistic_regression():
     assert_array_almost_equal(pobj3, pobj1, decimal=4)
 
 
-@pytest.mark.filterwarnings('ignore:The max_iter was reached')
-def test_sag_pobj_matches_ridge_regression():
+@pytest.mark.filterwarnings("ignore:The max_iter was reached")
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sag_pobj_matches_ridge_regression(csr_container):
     """tests if the sag pobj matches ridge reg"""
     n_samples = 100
     n_features = 10
@@ -343,14 +401,26 @@ def test_sag_pobj_matches_ridge_regression():
     true_w = rng.normal(size=n_features)
     y = X.dot(true_w)
 
-    clf1 = Ridge(fit_intercept=fit_intercept, tol=.00000000001, solver='sag',
-                 alpha=alpha, max_iter=n_iter, random_state=42)
+    clf1 = Ridge(
+        fit_intercept=fit_intercept,
+        tol=0.00000000001,
+        solver="sag",
+        alpha=alpha,
+        max_iter=n_iter,
+        random_state=42,
+    )
     clf2 = clone(clf1)
-    clf3 = Ridge(fit_intercept=fit_intercept, tol=.00001, solver='lsqr',
-                 alpha=alpha, max_iter=n_iter, random_state=42)
+    clf3 = Ridge(
+        fit_intercept=fit_intercept,
+        tol=0.00001,
+        solver="lsqr",
+        alpha=alpha,
+        max_iter=n_iter,
+        random_state=42,
+    )
 
     clf1.fit(X, y)
-    clf2.fit(sp.csr_matrix(X), y)
+    clf2.fit(csr_container(X), y)
     clf3.fit(X, y)
 
     pobj1 = get_pobj(clf1.coef_, alpha, X, y, squared_loss)
@@ -362,44 +432,59 @@ def test_sag_pobj_matches_ridge_regression():
     assert_array_almost_equal(pobj3, pobj2, decimal=4)
 
 
-@pytest.mark.filterwarnings('ignore:The max_iter was reached')
-def test_sag_regressor_computed_correctly():
+@pytest.mark.filterwarnings("ignore:The max_iter was reached")
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sag_regressor_computed_correctly(csr_container):
     """tests if the sag regressor is computed correctly"""
-    alpha = .1
+    alpha = 0.1
     n_features = 10
     n_samples = 40
     max_iter = 100
-    tol = .000001
+    tol = 0.000001
     fit_intercept = True
     rng = np.random.RandomState(0)
     X = rng.normal(size=(n_samples, n_features))
     w = rng.normal(size=n_features)
-    y = np.dot(X, w) + 2.
+    y = np.dot(X, w) + 2.0
     step_size = get_step_size(X, alpha, fit_intercept, classification=False)
 
-    clf1 = Ridge(fit_intercept=fit_intercept, tol=tol, solver='sag',
-                 alpha=alpha * n_samples, max_iter=max_iter,
-                 random_state=rng)
+    clf1 = Ridge(
+        fit_intercept=fit_intercept,
+        tol=tol,
+        solver="sag",
+        alpha=alpha * n_samples,
+        max_iter=max_iter,
+        random_state=rng,
+    )
     clf2 = clone(clf1)
 
     clf1.fit(X, y)
-    clf2.fit(sp.csr_matrix(X), y)
-
-    spweights1, spintercept1 = sag_sparse(X, y, step_size, alpha,
-                                          n_iter=max_iter,
-                                          dloss=squared_dloss,
-                                          fit_intercept=fit_intercept,
-                                          random_state=rng)
-
-    spweights2, spintercept2 = sag_sparse(X, y, step_size, alpha,
-                                          n_iter=max_iter,
-                                          dloss=squared_dloss, sparse=True,
-                                          fit_intercept=fit_intercept,
-                                          random_state=rng)
-
-    assert_array_almost_equal(clf1.coef_.ravel(),
-                              spweights1.ravel(),
-                              decimal=3)
+    clf2.fit(csr_container(X), y)
+
+    spweights1, spintercept1 = sag_sparse(
+        X,
+        y,
+        step_size,
+        alpha,
+        n_iter=max_iter,
+        dloss=squared_dloss,
+        fit_intercept=fit_intercept,
+        random_state=rng,
+    )
+
+    spweights2, spintercept2 = sag_sparse(
+        X,
+        y,
+        step_size,
+        alpha,
+        n_iter=max_iter,
+        dloss=squared_dloss,
+        sparse=True,
+        fit_intercept=fit_intercept,
+        random_state=rng,
+    )
+
+    assert_array_almost_equal(clf1.coef_.ravel(), spweights1.ravel(), decimal=3)
     assert_almost_equal(clf1.intercept_, spintercept1, decimal=1)
 
     # TODO: uncomment when sparse Ridge with intercept will be fixed (#4710)
@@ -422,43 +507,50 @@ def test_get_auto_step_size():
     for saga in [True, False]:
         for fit_intercept in (True, False):
             if saga:
-                L_sqr = (max_squared_sum + alpha + int(fit_intercept))
-                L_log = (max_squared_sum + 4.0 * alpha +
-                         int(fit_intercept)) / 4.0
+                L_sqr = max_squared_sum + alpha + int(fit_intercept)
+                L_log = (max_squared_sum + 4.0 * alpha + int(fit_intercept)) / 4.0
                 mun_sqr = min(2 * n_samples * alpha, L_sqr)
                 mun_log = min(2 * n_samples * alpha, L_log)
                 step_size_sqr = 1 / (2 * L_sqr + mun_sqr)
                 step_size_log = 1 / (2 * L_log + mun_log)
             else:
-                step_size_sqr = 1.0 / (max_squared_sum +
-                                       alpha + int(fit_intercept))
-                step_size_log = 4.0 / (max_squared_sum + 4.0 * alpha +
-                                       int(fit_intercept))
-
-            step_size_sqr_ = get_auto_step_size(max_squared_sum_, alpha,
-                                                "squared",
-                                                fit_intercept,
-                                                n_samples=n_samples,
-                                                is_saga=saga)
-            step_size_log_ = get_auto_step_size(max_squared_sum_, alpha, "log",
-                                                fit_intercept,
-                                                n_samples=n_samples,
-                                                is_saga=saga)
+                step_size_sqr = 1.0 / (max_squared_sum + alpha + int(fit_intercept))
+                step_size_log = 4.0 / (
+                    max_squared_sum + 4.0 * alpha + int(fit_intercept)
+                )
+
+            step_size_sqr_ = get_auto_step_size(
+                max_squared_sum_,
+                alpha,
+                "squared",
+                fit_intercept,
+                n_samples=n_samples,
+                is_saga=saga,
+            )
+            step_size_log_ = get_auto_step_size(
+                max_squared_sum_,
+                alpha,
+                "log",
+                fit_intercept,
+                n_samples=n_samples,
+                is_saga=saga,
+            )
 
             assert_almost_equal(step_size_sqr, step_size_sqr_, decimal=4)
             assert_almost_equal(step_size_log, step_size_log_, decimal=4)
 
-    msg = 'Unknown loss function for SAG solver, got wrong instead of'
-    assert_raise_message(ValueError, msg, get_auto_step_size,
-                         max_squared_sum_, alpha, "wrong", fit_intercept)
+    msg = "Unknown loss function for SAG solver, got wrong instead of"
+    with pytest.raises(ValueError, match=msg):
+        get_auto_step_size(max_squared_sum_, alpha, "wrong", fit_intercept)
 
 
 @pytest.mark.parametrize("seed", range(3))  # locally tested with 1000 seeds
-def test_sag_regressor(seed):
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sag_regressor(seed, csr_container):
     """tests if the sag regressor performs well"""
     xmin, xmax = -5, 5
     n_samples = 300
-    tol = .001
+    tol = 0.001
     max_iter = 100
     alpha = 0.1
     rng = np.random.RandomState(seed)
@@ -467,11 +559,16 @@ def test_sag_regressor(seed):
     # simple linear function without noise
     y = 0.5 * X.ravel()
 
-    clf1 = Ridge(tol=tol, solver='sag', max_iter=max_iter,
-                 alpha=alpha * n_samples, random_state=rng)
+    clf1 = Ridge(
+        tol=tol,
+        solver="sag",
+        max_iter=max_iter,
+        alpha=alpha * n_samples,
+        random_state=rng,
+    )
     clf2 = clone(clf1)
     clf1.fit(X, y)
-    clf2.fit(sp.csr_matrix(X), y)
+    clf2.fit(csr_container(X), y)
     score1 = clf1.score(X, y)
     score2 = clf2.score(X, y)
     assert score1 > 0.98
@@ -480,80 +577,99 @@ def test_sag_regressor(seed):
     # simple linear function with noise
     y = 0.5 * X.ravel() + rng.randn(n_samples, 1).ravel()
 
-    clf1 = Ridge(tol=tol, solver='sag', max_iter=max_iter,
-                 alpha=alpha * n_samples)
+    clf1 = Ridge(tol=tol, solver="sag", max_iter=max_iter, alpha=alpha * n_samples)
     clf2 = clone(clf1)
     clf1.fit(X, y)
-    clf2.fit(sp.csr_matrix(X), y)
+    clf2.fit(csr_container(X), y)
     score1 = clf1.score(X, y)
     score2 = clf2.score(X, y)
     assert score1 > 0.45
     assert score2 > 0.45
 
 
-@pytest.mark.filterwarnings('ignore:The max_iter was reached')
-def test_sag_classifier_computed_correctly():
+@pytest.mark.filterwarnings("ignore:The max_iter was reached")
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sag_classifier_computed_correctly(csr_container):
     """tests if the binary classifier is computed correctly"""
-    alpha = .1
+    alpha = 0.1
     n_samples = 50
     n_iter = 50
-    tol = .00001
+    tol = 0.00001
     fit_intercept = True
-    X, y = make_blobs(n_samples=n_samples, centers=2, random_state=0,
-                      cluster_std=0.1)
+    X, y = make_blobs(n_samples=n_samples, centers=2, random_state=0, cluster_std=0.1)
     step_size = get_step_size(X, alpha, fit_intercept, classification=True)
     classes = np.unique(y)
     y_tmp = np.ones(n_samples)
     y_tmp[y != classes[1]] = -1
     y = y_tmp
 
-    clf1 = LogisticRegression(solver='sag', C=1. / alpha / n_samples,
-                              max_iter=n_iter, tol=tol, random_state=77,
-                              fit_intercept=fit_intercept, multi_class='ovr')
+    clf1 = LogisticRegression(
+        solver="sag",
+        C=1.0 / alpha / n_samples,
+        max_iter=n_iter,
+        tol=tol,
+        random_state=77,
+        fit_intercept=fit_intercept,
+    )
     clf2 = clone(clf1)
 
     clf1.fit(X, y)
-    clf2.fit(sp.csr_matrix(X), y)
-
-    spweights, spintercept = sag_sparse(X, y, step_size, alpha, n_iter=n_iter,
-                                        dloss=log_dloss,
-                                        fit_intercept=fit_intercept)
-    spweights2, spintercept2 = sag_sparse(X, y, step_size, alpha,
-                                          n_iter=n_iter,
-                                          dloss=log_dloss, sparse=True,
-                                          fit_intercept=fit_intercept)
-
-    assert_array_almost_equal(clf1.coef_.ravel(),
-                              spweights.ravel(),
-                              decimal=2)
+    clf2.fit(csr_container(X), y)
+
+    spweights, spintercept = sag_sparse(
+        X,
+        y,
+        step_size,
+        alpha,
+        n_iter=n_iter,
+        dloss=log_dloss,
+        fit_intercept=fit_intercept,
+    )
+    spweights2, spintercept2 = sag_sparse(
+        X,
+        y,
+        step_size,
+        alpha,
+        n_iter=n_iter,
+        dloss=log_dloss,
+        sparse=True,
+        fit_intercept=fit_intercept,
+    )
+
+    assert_array_almost_equal(clf1.coef_.ravel(), spweights.ravel(), decimal=2)
     assert_almost_equal(clf1.intercept_, spintercept, decimal=1)
 
-    assert_array_almost_equal(clf2.coef_.ravel(),
-                              spweights2.ravel(),
-                              decimal=2)
+    assert_array_almost_equal(clf2.coef_.ravel(), spweights2.ravel(), decimal=2)
     assert_almost_equal(clf2.intercept_, spintercept2, decimal=1)
 
 
-@pytest.mark.filterwarnings('ignore:The max_iter was reached')
-def test_sag_multiclass_computed_correctly():
+@pytest.mark.filterwarnings("ignore:The max_iter was reached")
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sag_multiclass_computed_correctly(csr_container):
     """tests if the multiclass classifier is computed correctly"""
-    alpha = .1
+    alpha = 0.1
     n_samples = 20
-    tol = .00001
-    max_iter = 40
+    tol = 1e-5
+    max_iter = 70
     fit_intercept = True
-    X, y = make_blobs(n_samples=n_samples, centers=3, random_state=0,
-                      cluster_std=0.1)
+    X, y = make_blobs(n_samples=n_samples, centers=3, random_state=0, cluster_std=0.1)
     step_size = get_step_size(X, alpha, fit_intercept, classification=True)
     classes = np.unique(y)
 
-    clf1 = LogisticRegression(solver='sag', C=1. / alpha / n_samples,
-                              max_iter=max_iter, tol=tol, random_state=77,
-                              fit_intercept=fit_intercept, multi_class='ovr')
+    clf1 = OneVsRestClassifier(
+        LogisticRegression(
+            solver="sag",
+            C=1.0 / alpha / n_samples,
+            max_iter=max_iter,
+            tol=tol,
+            random_state=77,
+            fit_intercept=fit_intercept,
+        )
+    )
     clf2 = clone(clf1)
 
     clf1.fit(X, y)
-    clf2.fit(sp.csr_matrix(X), y)
+    clf2.fit(csr_container(X), y)
 
     coef1 = []
     intercept1 = []
@@ -563,13 +679,25 @@ def test_sag_multiclass_computed_correctly():
         y_encoded = np.ones(n_samples)
         y_encoded[y != cl] = -1
 
-        spweights1, spintercept1 = sag_sparse(X, y_encoded, step_size, alpha,
-                                              dloss=log_dloss, n_iter=max_iter,
-                                              fit_intercept=fit_intercept)
-        spweights2, spintercept2 = sag_sparse(X, y_encoded, step_size, alpha,
-                                              dloss=log_dloss, n_iter=max_iter,
-                                              sparse=True,
-                                              fit_intercept=fit_intercept)
+        spweights1, spintercept1 = sag_sparse(
+            X,
+            y_encoded,
+            step_size,
+            alpha,
+            dloss=log_dloss,
+            n_iter=max_iter,
+            fit_intercept=fit_intercept,
+        )
+        spweights2, spintercept2 = sag_sparse(
+            X,
+            y_encoded,
+            step_size,
+            alpha,
+            dloss=log_dloss,
+            n_iter=max_iter,
+            sparse=True,
+            fit_intercept=fit_intercept,
+        )
         coef1.append(spweights1)
         intercept1.append(spintercept1)
 
@@ -582,249 +710,134 @@ def test_sag_multiclass_computed_correctly():
     intercept2 = np.array(intercept2)
 
     for i, cl in enumerate(classes):
-        assert_array_almost_equal(clf1.coef_[i].ravel(),
-                                  coef1[i].ravel(),
-                                  decimal=2)
-        assert_almost_equal(clf1.intercept_[i], intercept1[i], decimal=1)
+        assert_allclose(clf1.estimators_[i].coef_.ravel(), coef1[i], rtol=1e-2)
+        assert_allclose(clf1.estimators_[i].intercept_, intercept1[i], rtol=1e-1)
 
-        assert_array_almost_equal(clf2.coef_[i].ravel(),
-                                  coef2[i].ravel(),
-                                  decimal=2)
-        assert_almost_equal(clf2.intercept_[i], intercept2[i], decimal=1)
+        assert_allclose(clf2.estimators_[i].coef_.ravel(), coef2[i], rtol=1e-2)
+        # Note the very crude accuracy, i.e. high rtol.
+        assert_allclose(clf2.estimators_[i].intercept_, intercept2[i], rtol=5e-1)
 
 
-def test_classifier_results():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_classifier_results(csr_container):
     """tests if classifier results match target"""
-    alpha = .1
+    alpha = 0.1
     n_features = 20
     n_samples = 10
-    tol = .01
+    tol = 0.01
     max_iter = 200
     rng = np.random.RandomState(0)
     X = rng.normal(size=(n_samples, n_features))
     w = rng.normal(size=n_features)
     y = np.dot(X, w)
     y = np.sign(y)
-    clf1 = LogisticRegression(solver='sag', C=1. / alpha / n_samples,
-                              max_iter=max_iter, tol=tol, random_state=77)
+    clf1 = LogisticRegression(
+        solver="sag",
+        C=1.0 / alpha / n_samples,
+        max_iter=max_iter,
+        tol=tol,
+        random_state=77,
+    )
     clf2 = clone(clf1)
 
     clf1.fit(X, y)
-    clf2.fit(sp.csr_matrix(X), y)
+    clf2.fit(csr_container(X), y)
     pred1 = clf1.predict(X)
     pred2 = clf2.predict(X)
     assert_almost_equal(pred1, y, decimal=12)
     assert_almost_equal(pred2, y, decimal=12)
 
 
-@pytest.mark.filterwarnings('ignore:The max_iter was reached')
-def test_binary_classifier_class_weight():
+@pytest.mark.filterwarnings("ignore:The max_iter was reached")
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_binary_classifier_class_weight(csr_container):
     """tests binary classifier with classweights for each class"""
-    alpha = .1
+    alpha = 0.1
     n_samples = 50
     n_iter = 20
-    tol = .00001
+    tol = 0.00001
     fit_intercept = True
-    X, y = make_blobs(n_samples=n_samples, centers=2, random_state=10,
-                      cluster_std=0.1)
+    X, y = make_blobs(n_samples=n_samples, centers=2, random_state=10, cluster_std=0.1)
     step_size = get_step_size(X, alpha, fit_intercept, classification=True)
     classes = np.unique(y)
     y_tmp = np.ones(n_samples)
     y_tmp[y != classes[1]] = -1
     y = y_tmp
 
-    class_weight = {1: .45, -1: .55}
-    clf1 = LogisticRegression(solver='sag', C=1. / alpha / n_samples,
-                              max_iter=n_iter, tol=tol, random_state=77,
-                              fit_intercept=fit_intercept, multi_class='ovr',
-                              class_weight=class_weight)
+    class_weight = {1: 0.45, -1: 0.55}
+    clf1 = LogisticRegression(
+        solver="sag",
+        C=1.0 / alpha / n_samples,
+        max_iter=n_iter,
+        tol=tol,
+        random_state=77,
+        fit_intercept=fit_intercept,
+        class_weight=class_weight,
+    )
     clf2 = clone(clf1)
 
     clf1.fit(X, y)
-    clf2.fit(sp.csr_matrix(X), y)
+    clf2.fit(csr_container(X), y)
 
     le = LabelEncoder()
-    class_weight_ = compute_class_weight(class_weight, classes=np.unique(y),
-                                         y=y)
+    class_weight_ = compute_class_weight(class_weight, classes=np.unique(y), y=y)
     sample_weight = class_weight_[le.fit_transform(y)]
-    spweights, spintercept = sag_sparse(X, y, step_size, alpha, n_iter=n_iter,
-                                        dloss=log_dloss,
-                                        sample_weight=sample_weight,
-                                        fit_intercept=fit_intercept)
-    spweights2, spintercept2 = sag_sparse(X, y, step_size, alpha,
-                                          n_iter=n_iter,
-                                          dloss=log_dloss, sparse=True,
-                                          sample_weight=sample_weight,
-                                          fit_intercept=fit_intercept)
-
-    assert_array_almost_equal(clf1.coef_.ravel(),
-                              spweights.ravel(),
-                              decimal=2)
+    spweights, spintercept = sag_sparse(
+        X,
+        y,
+        step_size,
+        alpha,
+        n_iter=n_iter,
+        dloss=log_dloss,
+        sample_weight=sample_weight,
+        fit_intercept=fit_intercept,
+    )
+    spweights2, spintercept2 = sag_sparse(
+        X,
+        y,
+        step_size,
+        alpha,
+        n_iter=n_iter,
+        dloss=log_dloss,
+        sparse=True,
+        sample_weight=sample_weight,
+        fit_intercept=fit_intercept,
+    )
+
+    assert_array_almost_equal(clf1.coef_.ravel(), spweights.ravel(), decimal=2)
     assert_almost_equal(clf1.intercept_, spintercept, decimal=1)
 
-    assert_array_almost_equal(clf2.coef_.ravel(),
-                              spweights2.ravel(),
-                              decimal=2)
+    assert_array_almost_equal(clf2.coef_.ravel(), spweights2.ravel(), decimal=2)
     assert_almost_equal(clf2.intercept_, spintercept2, decimal=1)
 
 
-@pytest.mark.filterwarnings('ignore:The max_iter was reached')
-def test_multiclass_classifier_class_weight():
-    """tests multiclass with classweights for each class"""
-    alpha = .1
-    n_samples = 20
-    tol = .00001
-    max_iter = 50
-    class_weight = {0: .45, 1: .55, 2: .75}
-    fit_intercept = True
-    X, y = make_blobs(n_samples=n_samples, centers=3, random_state=0,
-                      cluster_std=0.1)
-    step_size = get_step_size(X, alpha, fit_intercept, classification=True)
-    classes = np.unique(y)
-
-    clf1 = LogisticRegression(solver='sag', C=1. / alpha / n_samples,
-                              max_iter=max_iter, tol=tol, random_state=77,
-                              fit_intercept=fit_intercept, multi_class='ovr',
-                              class_weight=class_weight)
-    clf2 = clone(clf1)
-    clf1.fit(X, y)
-    clf2.fit(sp.csr_matrix(X), y)
-
-    le = LabelEncoder()
-    class_weight_ = compute_class_weight(class_weight, classes=np.unique(y),
-                                         y=y)
-    sample_weight = class_weight_[le.fit_transform(y)]
-
-    coef1 = []
-    intercept1 = []
-    coef2 = []
-    intercept2 = []
-    for cl in classes:
-        y_encoded = np.ones(n_samples)
-        y_encoded[y != cl] = -1
-
-        spweights1, spintercept1 = sag_sparse(X, y_encoded, step_size, alpha,
-                                              n_iter=max_iter, dloss=log_dloss,
-                                              sample_weight=sample_weight)
-        spweights2, spintercept2 = sag_sparse(X, y_encoded, step_size, alpha,
-                                              n_iter=max_iter, dloss=log_dloss,
-                                              sample_weight=sample_weight,
-                                              sparse=True)
-        coef1.append(spweights1)
-        intercept1.append(spintercept1)
-        coef2.append(spweights2)
-        intercept2.append(spintercept2)
-
-    coef1 = np.vstack(coef1)
-    intercept1 = np.array(intercept1)
-    coef2 = np.vstack(coef2)
-    intercept2 = np.array(intercept2)
-
-    for i, cl in enumerate(classes):
-        assert_array_almost_equal(clf1.coef_[i].ravel(),
-                                  coef1[i].ravel(),
-                                  decimal=2)
-        assert_almost_equal(clf1.intercept_[i], intercept1[i], decimal=1)
-
-        assert_array_almost_equal(clf2.coef_[i].ravel(),
-                                  coef2[i].ravel(),
-                                  decimal=2)
-        assert_almost_equal(clf2.intercept_[i], intercept2[i], decimal=1)
-
-
 def test_classifier_single_class():
     """tests if ValueError is thrown with only one class"""
     X = [[1, 2], [3, 4]]
     y = [1, 1]
 
-    assert_raise_message(ValueError,
-                         "This solver needs samples of at least 2 classes "
-                         "in the data",
-                         LogisticRegression(solver='sag').fit,
-                         X, y)
+    msg = "This solver needs samples of at least 2 classes in the data"
+    with pytest.raises(ValueError, match=msg):
+        LogisticRegression(solver="sag").fit(X, y)
 
 
 def test_step_size_alpha_error():
     X = [[0, 0], [0, 0]]
     y = [1, -1]
     fit_intercept = False
-    alpha = 1.
-    msg = ("Current sag implementation does not handle the case"
-           " step_size * alpha_scaled == 1")
-
-    clf1 = LogisticRegression(solver='sag', C=1. / alpha,
-                              fit_intercept=fit_intercept)
-    assert_raise_message(ZeroDivisionError, msg, clf1.fit, X, y)
-
-    clf2 = Ridge(fit_intercept=fit_intercept, solver='sag', alpha=alpha)
-    assert_raise_message(ZeroDivisionError, msg, clf2.fit, X, y)
-
-
-def test_multinomial_loss():
-    # test if the multinomial loss and gradient computations are consistent
-    X, y = iris.data, iris.target.astype(np.float64)
-    n_samples, n_features = X.shape
-    n_classes = len(np.unique(y))
-
-    rng = check_random_state(42)
-    weights = rng.randn(n_features, n_classes)
-    intercept = rng.randn(n_classes)
-    sample_weights = rng.randn(n_samples)
-    np.abs(sample_weights, sample_weights)
-
-    # compute loss and gradient like in multinomial SAG
-    dataset, _ = make_dataset(X, y, sample_weights, random_state=42)
-    loss_1, grad_1 = _multinomial_grad_loss_all_samples(dataset, weights,
-                                                        intercept, n_samples,
-                                                        n_features, n_classes)
-    # compute loss and gradient like in multinomial LogisticRegression
-    lbin = LabelBinarizer()
-    Y_bin = lbin.fit_transform(y)
-    weights_intercept = np.vstack((weights, intercept)).T.ravel()
-    loss_2, grad_2, _ = _multinomial_loss_grad(weights_intercept, X, Y_bin,
-                                               0.0, sample_weights)
-    grad_2 = grad_2.reshape(n_classes, -1)
-    grad_2 = grad_2[:, :-1].T
-
-    # comparison
-    assert_array_almost_equal(grad_1, grad_2)
-    assert_almost_equal(loss_1, loss_2)
-
-
-def test_multinomial_loss_ground_truth():
-    # n_samples, n_features, n_classes = 4, 2, 3
-    n_classes = 3
-    X = np.array([[1.1, 2.2], [2.2, -4.4], [3.3, -2.2], [1.1, 1.1]])
-    y = np.array([0, 1, 2, 0])
-    lbin = LabelBinarizer()
-    Y_bin = lbin.fit_transform(y)
-
-    weights = np.array([[0.1, 0.2, 0.3], [1.1, 1.2, -1.3]])
-    intercept = np.array([1., 0, -.2])
-    sample_weights = np.array([0.8, 1, 1, 0.8])
-
-    prediction = np.dot(X, weights) + intercept
-    logsumexp_prediction = logsumexp(prediction, axis=1)
-    p = prediction - logsumexp_prediction[:, np.newaxis]
-    loss_1 = -(sample_weights[:, np.newaxis] * p * Y_bin).sum()
-    diff = sample_weights[:, np.newaxis] * (np.exp(p) - Y_bin)
-    grad_1 = np.dot(X.T, diff)
-
-    weights_intercept = np.vstack((weights, intercept)).T.ravel()
-    loss_2, grad_2, _ = _multinomial_loss_grad(weights_intercept, X, Y_bin,
-                                               0.0, sample_weights)
-    grad_2 = grad_2.reshape(n_classes, -1)
-    grad_2 = grad_2[:, :-1].T
-
-    assert_almost_equal(loss_1, loss_2)
-    assert_array_almost_equal(grad_1, grad_2)
-
-    # ground truth
-    loss_gt = 11.680360354325961
-    grad_gt = np.array([[-0.557487, -1.619151, +2.176638],
-                        [-0.903942, +5.258745, -4.354803]])
-    assert_almost_equal(loss_1, loss_gt)
-    assert_array_almost_equal(grad_1, grad_gt)
+    alpha = 1.0
+    msg = re.escape(
+        "Current sag implementation does not handle the case"
+        " step_size * alpha_scaled == 1"
+    )
+
+    clf1 = LogisticRegression(solver="sag", C=1.0 / alpha, fit_intercept=fit_intercept)
+    with pytest.raises(ZeroDivisionError, match=msg):
+        clf1.fit(X, y)
+
+    clf2 = Ridge(fit_intercept=fit_intercept, solver="sag", alpha=alpha)
+    with pytest.raises(ZeroDivisionError, match=msg):
+        clf2.fit(X, y)
 
 
 @pytest.mark.parametrize("solver", ["sag", "saga"])
diff --git a/sklearn/linear_model/tests/test_sgd.py b/sklearn/linear_model/tests/test_sgd.py
index 217249631390d..26d138ae3649b 100644
--- a/sklearn/linear_model/tests/test_sgd.py
+++ b/sklearn/linear_model/tests/test_sgd.py
@@ -1,25 +1,32 @@
 import pickle
-import pytest
+from unittest.mock import Mock
 
+import joblib
 import numpy as np
+import pytest
 import scipy.sparse as sp
-import joblib
 
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils._testing import assert_almost_equal
-from sklearn.utils._testing import assert_array_almost_equal
-from sklearn.utils._testing import assert_raises_regexp
-from sklearn.utils._testing import ignore_warnings
-from sklearn.utils.fixes import parse_version
-
-from sklearn import linear_model, datasets, metrics
+from sklearn import datasets, linear_model, metrics
 from sklearn.base import clone, is_classifier
-from sklearn.preprocessing import LabelEncoder, scale, MinMaxScaler
-from sklearn.preprocessing import StandardScaler
 from sklearn.exceptions import ConvergenceWarning
-from sklearn.model_selection import StratifiedShuffleSplit, ShuffleSplit
+from sklearn.kernel_approximation import Nystroem
 from sklearn.linear_model import _sgd_fast as sgd_fast
-from sklearn.model_selection import RandomizedSearchCV
+from sklearn.linear_model import _stochastic_gradient
+from sklearn.model_selection import (
+    RandomizedSearchCV,
+    ShuffleSplit,
+    StratifiedShuffleSplit,
+)
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import LabelEncoder, MinMaxScaler, StandardScaler, scale
+from sklearn.svm import OneClassSVM
+from sklearn.utils import get_tags
+from sklearn.utils._testing import (
+    assert_allclose,
+    assert_almost_equal,
+    assert_array_almost_equal,
+    assert_array_equal,
+)
 
 
 def _update_kwargs(kwargs):
@@ -62,8 +69,21 @@ def partial_fit(self, X, y, *args, **kw):
     def decision_function(self, X, *args, **kw):
         # XXX untested as of v0.22
         X = sp.csr_matrix(X)
-        return linear_model.SGDRegressor.decision_function(self, X, *args,
-                                                           **kw)
+        return linear_model.SGDRegressor.decision_function(self, X, *args, **kw)
+
+
+class _SparseSGDOneClassSVM(linear_model.SGDOneClassSVM):
+    def fit(self, X, *args, **kw):
+        X = sp.csr_matrix(X)
+        return linear_model.SGDOneClassSVM.fit(self, X, *args, **kw)
+
+    def partial_fit(self, X, *args, **kw):
+        X = sp.csr_matrix(X)
+        return linear_model.SGDOneClassSVM.partial_fit(self, X, *args, **kw)
+
+    def decision_function(self, X, *args, **kw):
+        X = sp.csr_matrix(X)
+        return linear_model.SGDOneClassSVM.decision_function(self, X, *args, **kw)
 
 
 def SGDClassifier(**kwargs):
@@ -76,6 +96,11 @@ def SGDRegressor(**kwargs):
     return linear_model.SGDRegressor(**kwargs)
 
 
+def SGDOneClassSVM(**kwargs):
+    _update_kwargs(kwargs)
+    return linear_model.SGDOneClassSVM(**kwargs)
+
+
 def SparseSGDClassifier(**kwargs):
     _update_kwargs(kwargs)
     return _SparseSGDClassifier(**kwargs)
@@ -86,6 +111,11 @@ def SparseSGDRegressor(**kwargs):
     return _SparseSGDRegressor(**kwargs)
 
 
+def SparseSGDOneClassSVM(**kwargs):
+    _update_kwargs(kwargs)
+    return _SparseSGDOneClassSVM(**kwargs)
+
+
 # Test Data
 
 # test sample 1
@@ -95,25 +125,51 @@ def SparseSGDRegressor(**kwargs):
 true_result = [1, 2, 2]
 
 # test sample 2; string class labels
-X2 = np.array([[-1, 1], [-0.75, 0.5], [-1.5, 1.5],
-               [1, 1], [0.75, 0.5], [1.5, 1.5],
-               [-1, -1], [0, -0.5], [1, -1]])
+X2 = np.array(
+    [
+        [-1, 1],
+        [-0.75, 0.5],
+        [-1.5, 1.5],
+        [1, 1],
+        [0.75, 0.5],
+        [1.5, 1.5],
+        [-1, -1],
+        [0, -0.5],
+        [1, -1],
+    ]
+)
 Y2 = ["one"] * 3 + ["two"] * 3 + ["three"] * 3
 T2 = np.array([[-1.5, 0.5], [1, 2], [0, -2]])
 true_result2 = ["one", "two", "three"]
 
 # test sample 3
-X3 = np.array([[1, 1, 0, 0, 0, 0], [1, 1, 0, 0, 0, 0],
-               [0, 0, 1, 0, 0, 0], [0, 0, 1, 0, 0, 0],
-               [0, 0, 0, 0, 1, 1], [0, 0, 0, 0, 1, 1],
-               [0, 0, 0, 1, 0, 0], [0, 0, 0, 1, 0, 0]])
+X3 = np.array(
+    [
+        [1, 1, 0, 0, 0, 0],
+        [1, 1, 0, 0, 0, 0],
+        [0, 0, 1, 0, 0, 0],
+        [0, 0, 1, 0, 0, 0],
+        [0, 0, 0, 0, 1, 1],
+        [0, 0, 0, 0, 1, 1],
+        [0, 0, 0, 1, 0, 0],
+        [0, 0, 0, 1, 0, 0],
+    ]
+)
 Y3 = np.array([1, 1, 1, 1, 2, 2, 2, 2])
 
 # test sample 4 - two more or less redundant feature groups
-X4 = np.array([[1, 0.9, 0.8, 0, 0, 0], [1, .84, .98, 0, 0, 0],
-               [1, .96, .88, 0, 0, 0], [1, .91, .99, 0, 0, 0],
-               [0, 0, 0, .89, .91, 1], [0, 0, 0, .79, .84, 1],
-               [0, 0, 0, .91, .95, 1], [0, 0, 0, .93, 1, 1]])
+X4 = np.array(
+    [
+        [1, 0.9, 0.8, 0, 0, 0],
+        [1, 0.84, 0.98, 0, 0, 0],
+        [1, 0.96, 0.88, 0, 0, 0],
+        [1, 0.91, 0.99, 0, 0, 0],
+        [0, 0, 0, 0.89, 0.91, 1],
+        [0, 0, 0, 0.79, 0.84, 1],
+        [0, 0, 0, 0.91, 0.95, 1],
+        [0, 0, 0, 0.93, 1, 1],
+    ]
+)
 Y4 = np.array([1, 1, 1, 1, 2, 2, 2, 2])
 
 iris = datasets.load_iris()
@@ -127,6 +183,7 @@ def SparseSGDRegressor(**kwargs):
 ###############################################################################
 # Common Test Case to classification and regression
 
+
 # a simple implementation of ASGD to use for testing
 # uses squared loss to find the gradient
 def asgd(klass, X, y, eta, alpha, weight_init=None, intercept_init=0.0):
@@ -142,7 +199,7 @@ def asgd(klass, X, y, eta, alpha, weight_init=None, intercept_init=0.0):
 
     # sparse data has a fixed decay of .01
     if klass in (SparseSGDClassifier, SparseSGDRegressor):
-        decay = .01
+        decay = 0.01
 
     for i, entry in enumerate(X):
         p = np.dot(entry, weights)
@@ -163,45 +220,18 @@ def asgd(klass, X, y, eta, alpha, weight_init=None, intercept_init=0.0):
     return average_weights, average_intercept
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier,
-                                   SGDRegressor, SparseSGDRegressor])
-def test_sgd_bad_alpha(klass):
-    # Check whether expected ValueError on bad alpha
-    with pytest.raises(ValueError):
-        klass(alpha=-.1)
-
-
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier,
-                                   SGDRegressor, SparseSGDRegressor])
-def test_sgd_bad_penalty(klass):
-    # Check whether expected ValueError on bad penalty
-    with pytest.raises(ValueError):
-        klass(penalty='foobar', l1_ratio=0.85)
-
-
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier,
-                                   SGDRegressor, SparseSGDRegressor])
-def test_sgd_bad_loss(klass):
-    # Check whether expected ValueError on bad loss
-    with pytest.raises(ValueError):
-        klass(loss="foobar")
-
-
 def _test_warm_start(klass, X, Y, lr):
     # Test that explicit warm restart...
-    clf = klass(alpha=0.01, eta0=0.01, shuffle=False,
-                learning_rate=lr)
+    clf = klass(alpha=0.01, eta0=0.01, shuffle=False, learning_rate=lr)
     clf.fit(X, Y)
 
-    clf2 = klass(alpha=0.001, eta0=0.01, shuffle=False,
-                 learning_rate=lr)
-    clf2.fit(X, Y,
-             coef_init=clf.coef_.copy(),
-             intercept_init=clf.intercept_.copy())
+    clf2 = klass(alpha=0.001, eta0=0.01, shuffle=False, learning_rate=lr)
+    clf2.fit(X, Y, coef_init=clf.coef_.copy(), intercept_init=clf.intercept_.copy())
 
     # ... and implicit warm restart are equivalent.
-    clf3 = klass(alpha=0.01, eta0=0.01, shuffle=False,
-                 warm_start=True, learning_rate=lr)
+    clf3 = klass(
+        alpha=0.01, eta0=0.01, shuffle=False, warm_start=True, learning_rate=lr
+    )
     clf3.fit(X, Y)
 
     assert clf3.t_ == clf.t_
@@ -214,16 +244,17 @@ def _test_warm_start(klass, X, Y, lr):
     assert_array_almost_equal(clf3.coef_, clf2.coef_)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier,
-                                   SGDRegressor, SparseSGDRegressor])
-@pytest.mark.parametrize('lr',
-                         ["constant", "optimal", "invscaling", "adaptive"])
+@pytest.mark.parametrize(
+    "klass", [SGDClassifier, SparseSGDClassifier, SGDRegressor, SparseSGDRegressor]
+)
+@pytest.mark.parametrize("lr", ["constant", "optimal", "invscaling", "adaptive"])
 def test_warm_start(klass, lr):
     _test_warm_start(klass, X, Y, lr)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier,
-                                   SGDRegressor, SparseSGDRegressor])
+@pytest.mark.parametrize(
+    "klass", [SGDClassifier, SparseSGDClassifier, SGDRegressor, SparseSGDRegressor]
+)
 def test_input_format(klass):
     # Input format tests.
     clf = klass(alpha=0.01, shuffle=False)
@@ -235,56 +266,62 @@ def test_input_format(klass):
         clf.fit(X, Y_)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier,
-                                   SGDRegressor, SparseSGDRegressor])
+@pytest.mark.parametrize(
+    "klass", [SGDClassifier, SparseSGDClassifier, SGDRegressor, SparseSGDRegressor]
+)
 def test_clone(klass):
     # Test whether clone works ok.
-    clf = klass(alpha=0.01, penalty='l1')
+    clf = klass(alpha=0.01, penalty="l1")
     clf = clone(clf)
-    clf.set_params(penalty='l2')
+    clf.set_params(penalty="l2")
     clf.fit(X, Y)
 
-    clf2 = klass(alpha=0.01, penalty='l2')
+    clf2 = klass(alpha=0.01, penalty="l2")
     clf2.fit(X, Y)
 
     assert_array_equal(clf.coef_, clf2.coef_)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier,
-                                   SGDRegressor, SparseSGDRegressor])
+@pytest.mark.parametrize(
+    "klass",
+    [
+        SGDClassifier,
+        SparseSGDClassifier,
+        SGDRegressor,
+        SparseSGDRegressor,
+        SGDOneClassSVM,
+        SparseSGDOneClassSVM,
+    ],
+)
 def test_plain_has_no_average_attr(klass):
-    clf = klass(average=True, eta0=.01)
+    clf = klass(average=True, eta0=0.01)
     clf.fit(X, Y)
 
-    assert hasattr(clf, '_average_coef')
-    assert hasattr(clf, '_average_intercept')
-    assert hasattr(clf, '_standard_intercept')
-    assert hasattr(clf, '_standard_coef')
+    assert hasattr(clf, "_average_coef")
+    assert hasattr(clf, "_average_intercept")
+    assert hasattr(clf, "_standard_intercept")
+    assert hasattr(clf, "_standard_coef")
 
     clf = klass()
     clf.fit(X, Y)
 
-    assert not hasattr(clf, '_average_coef')
-    assert not hasattr(clf, '_average_intercept')
-    assert not hasattr(clf, '_standard_intercept')
-    assert not hasattr(clf, '_standard_coef')
-
-
-# TODO: remove in 1.0
-@pytest.mark.parametrize('klass', [SGDClassifier, SGDRegressor])
-def test_sgd_deprecated_attr(klass):
-    est = klass(average=True, eta0=.01)
-    est.fit(X, Y)
-
-    msg = "Attribute {} was deprecated"
-    for att in ['average_coef_', 'average_intercept_',
-                'standard_coef_', 'standard_intercept_']:
-        with pytest.warns(FutureWarning, match=msg.format(att)):
-            getattr(est, att)
-
-
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier,
-                                   SGDRegressor, SparseSGDRegressor])
+    assert not hasattr(clf, "_average_coef")
+    assert not hasattr(clf, "_average_intercept")
+    assert not hasattr(clf, "_standard_intercept")
+    assert not hasattr(clf, "_standard_coef")
+
+
+@pytest.mark.parametrize(
+    "klass",
+    [
+        SGDClassifier,
+        SparseSGDClassifier,
+        SGDRegressor,
+        SparseSGDRegressor,
+        SGDOneClassSVM,
+        SparseSGDOneClassSVM,
+    ],
+)
 def test_late_onset_averaging_not_reached(klass):
     clf1 = klass(average=600)
     clf2 = klass()
@@ -297,99 +334,119 @@ def test_late_onset_averaging_not_reached(klass):
             clf2.partial_fit(X, Y)
 
     assert_array_almost_equal(clf1.coef_, clf2.coef_, decimal=16)
-    assert_almost_equal(clf1.intercept_, clf2.intercept_, decimal=16)
+    if klass in [SGDClassifier, SparseSGDClassifier, SGDRegressor, SparseSGDRegressor]:
+        assert_almost_equal(clf1.intercept_, clf2.intercept_, decimal=16)
+    elif klass in [SGDOneClassSVM, SparseSGDOneClassSVM]:
+        assert_allclose(clf1.offset_, clf2.offset_)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier,
-                                   SGDRegressor, SparseSGDRegressor])
+@pytest.mark.parametrize(
+    "klass", [SGDClassifier, SparseSGDClassifier, SGDRegressor, SparseSGDRegressor]
+)
 def test_late_onset_averaging_reached(klass):
-    eta0 = .001
-    alpha = .0001
+    eta0 = 0.001
+    alpha = 0.0001
     Y_encode = np.array(Y)
     Y_encode[Y_encode == 1] = -1.0
     Y_encode[Y_encode == 2] = 1.0
 
-    clf1 = klass(average=7, learning_rate="constant",
-                 loss='squared_loss', eta0=eta0,
-                 alpha=alpha, max_iter=2, shuffle=False)
-    clf2 = klass(average=0, learning_rate="constant",
-                 loss='squared_loss', eta0=eta0,
-                 alpha=alpha, max_iter=1, shuffle=False)
+    clf1 = klass(
+        average=7,
+        learning_rate="constant",
+        loss="squared_error",
+        eta0=eta0,
+        alpha=alpha,
+        max_iter=2,
+        shuffle=False,
+    )
+    clf2 = klass(
+        average=False,
+        learning_rate="constant",
+        loss="squared_error",
+        eta0=eta0,
+        alpha=alpha,
+        max_iter=1,
+        shuffle=False,
+    )
 
     clf1.fit(X, Y_encode)
     clf2.fit(X, Y_encode)
 
-    average_weights, average_intercept = \
-        asgd(klass, X, Y_encode, eta0, alpha,
-             weight_init=clf2.coef_.ravel(),
-             intercept_init=clf2.intercept_)
+    average_weights, average_intercept = asgd(
+        klass,
+        X,
+        Y_encode,
+        eta0,
+        alpha,
+        weight_init=clf2.coef_.ravel(),
+        intercept_init=clf2.intercept_,
+    )
 
-    assert_array_almost_equal(clf1.coef_.ravel(),
-                              average_weights.ravel(),
-                              decimal=16)
+    assert_array_almost_equal(clf1.coef_.ravel(), average_weights.ravel(), decimal=16)
     assert_almost_equal(clf1.intercept_, average_intercept, decimal=16)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier,
-                                   SGDRegressor, SparseSGDRegressor])
-def test_sgd_bad_alpha_for_optimal_learning_rate(klass):
-    # Check whether expected ValueError on bad alpha, i.e. 0
-    # since alpha is used to compute the optimal learning rate
-    with pytest.raises(ValueError):
-        klass(alpha=0, learning_rate="optimal")
-
-
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier,
-                                   SGDRegressor, SparseSGDRegressor])
+@pytest.mark.parametrize(
+    "klass", [SGDClassifier, SparseSGDClassifier, SGDRegressor, SparseSGDRegressor]
+)
 def test_early_stopping(klass):
     X = iris.data[iris.target > 0]
     Y = iris.target[iris.target > 0]
     for early_stopping in [True, False]:
         max_iter = 1000
-        clf = klass(early_stopping=early_stopping, tol=1e-3,
-                    max_iter=max_iter).fit(X, Y)
+        clf = klass(early_stopping=early_stopping, tol=1e-3, max_iter=max_iter).fit(
+            X, Y
+        )
         assert clf.n_iter_ < max_iter
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier,
-                                   SGDRegressor, SparseSGDRegressor])
+@pytest.mark.parametrize(
+    "klass", [SGDClassifier, SparseSGDClassifier, SGDRegressor, SparseSGDRegressor]
+)
 def test_adaptive_longer_than_constant(klass):
-    clf1 = klass(learning_rate="adaptive", eta0=0.01, tol=1e-3,
-                 max_iter=100)
+    clf1 = klass(learning_rate="adaptive", eta0=0.01, tol=1e-3, max_iter=100)
     clf1.fit(iris.data, iris.target)
-    clf2 = klass(learning_rate="constant", eta0=0.01, tol=1e-3,
-                 max_iter=100)
+    clf2 = klass(learning_rate="constant", eta0=0.01, tol=1e-3, max_iter=100)
     clf2.fit(iris.data, iris.target)
     assert clf1.n_iter_ > clf2.n_iter_
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier,
-                                   SGDRegressor, SparseSGDRegressor])
+@pytest.mark.parametrize(
+    "klass", [SGDClassifier, SparseSGDClassifier, SGDRegressor, SparseSGDRegressor]
+)
 def test_validation_set_not_used_for_training(klass):
     X, Y = iris.data, iris.target
     validation_fraction = 0.4
     seed = 42
     shuffle = False
     max_iter = 10
-    clf1 = klass(early_stopping=True,
-                 random_state=np.random.RandomState(seed),
-                 validation_fraction=validation_fraction,
-                 learning_rate='constant', eta0=0.01,
-                 tol=None, max_iter=max_iter, shuffle=shuffle)
+    clf1 = klass(
+        early_stopping=True,
+        random_state=np.random.RandomState(seed),
+        validation_fraction=validation_fraction,
+        learning_rate="constant",
+        eta0=0.01,
+        tol=None,
+        max_iter=max_iter,
+        shuffle=shuffle,
+    )
     clf1.fit(X, Y)
     assert clf1.n_iter_ == max_iter
 
-    clf2 = klass(early_stopping=False,
-                 random_state=np.random.RandomState(seed),
-                 learning_rate='constant', eta0=0.01,
-                 tol=None, max_iter=max_iter, shuffle=shuffle)
+    clf2 = klass(
+        early_stopping=False,
+        random_state=np.random.RandomState(seed),
+        learning_rate="constant",
+        eta0=0.01,
+        tol=None,
+        max_iter=max_iter,
+        shuffle=shuffle,
+    )
 
     if is_classifier(clf2):
-        cv = StratifiedShuffleSplit(test_size=validation_fraction,
-                                    random_state=seed)
+        cv = StratifiedShuffleSplit(test_size=validation_fraction, random_state=seed)
     else:
-        cv = ShuffleSplit(test_size=validation_fraction,
-                          random_state=seed)
+        cv = ShuffleSplit(test_size=validation_fraction, random_state=seed)
     idx_train, idx_val = next(cv.split(X, Y))
     idx_train = np.sort(idx_train)  # remove shuffling
     clf2.fit(X[idx_train], Y[idx_train])
@@ -398,22 +455,30 @@ def test_validation_set_not_used_for_training(klass):
     assert_array_equal(clf1.coef_, clf2.coef_)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier,
-                                   SGDRegressor, SparseSGDRegressor])
+@pytest.mark.parametrize(
+    "klass", [SGDClassifier, SparseSGDClassifier, SGDRegressor, SparseSGDRegressor]
+)
 def test_n_iter_no_change(klass):
     X, Y = iris.data, iris.target
     # test that n_iter_ increases monotonically with n_iter_no_change
     for early_stopping in [True, False]:
-        n_iter_list = [klass(early_stopping=early_stopping,
-                             n_iter_no_change=n_iter_no_change,
-                             tol=1e-4, max_iter=1000
-                             ).fit(X, Y).n_iter_
-                       for n_iter_no_change in [2, 3, 10]]
+        n_iter_list = [
+            klass(
+                early_stopping=early_stopping,
+                n_iter_no_change=n_iter_no_change,
+                tol=1e-4,
+                max_iter=1000,
+            )
+            .fit(X, Y)
+            .n_iter_
+            for n_iter_no_change in [2, 3, 10]
+        ]
         assert_array_equal(n_iter_list, sorted(n_iter_list))
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier,
-                                   SGDRegressor, SparseSGDRegressor])
+@pytest.mark.parametrize(
+    "klass", [SGDClassifier, SparseSGDClassifier, SGDRegressor, SparseSGDRegressor]
+)
 def test_not_enough_sample_for_early_stopping(klass):
     # test an error is raised if the training or validation set is empty
     clf = klass(early_stopping=True, validation_fraction=0.99)
@@ -421,130 +486,122 @@ def test_not_enough_sample_for_early_stopping(klass):
         clf.fit(X3, Y3)
 
 
+@pytest.mark.parametrize("Estimator", [SGDClassifier, SGDRegressor])
+@pytest.mark.parametrize("l1_ratio", [0, 0.7, 1])
+def test_sgd_l1_ratio_not_used(Estimator, l1_ratio):
+    """Check that l1_ratio is not used when penalty is not 'elasticnet'"""
+    clf1 = Estimator(penalty="l1", l1_ratio=None, random_state=0).fit(X, Y)
+    clf2 = Estimator(penalty="l1", l1_ratio=l1_ratio, random_state=0).fit(X, Y)
+
+    assert_allclose(clf1.coef_, clf2.coef_)
+
+
+@pytest.mark.parametrize(
+    "Estimator", [SGDClassifier, SparseSGDClassifier, SGDRegressor, SparseSGDRegressor]
+)
+def test_sgd_failing_penalty_validation(Estimator):
+    clf = Estimator(penalty="elasticnet", l1_ratio=None)
+    with pytest.raises(
+        ValueError, match="l1_ratio must be set when penalty is 'elasticnet'"
+    ):
+        clf.fit(X, Y)
+
+
 ###############################################################################
 # Classification Test Case
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_sgd_clf(klass):
     # Check that SGD gives any results :-)
 
-    for loss in ("hinge", "squared_hinge", "log", "modified_huber"):
-        clf = klass(penalty='l2', alpha=0.01, fit_intercept=True,
-                    loss=loss, max_iter=10, shuffle=True)
+    for loss in ("hinge", "squared_hinge", "log_loss", "modified_huber"):
+        clf = klass(
+            penalty="l2",
+            alpha=0.01,
+            fit_intercept=True,
+            loss=loss,
+            max_iter=10,
+            shuffle=True,
+        )
         clf.fit(X, Y)
         # assert_almost_equal(clf.coef_[0], clf.coef_[1], decimal=7)
         assert_array_equal(clf.predict(T), true_result)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
-def test_sgd_bad_l1_ratio(klass):
-    # Check whether expected ValueError on bad l1_ratio
-    with pytest.raises(ValueError):
-        klass(l1_ratio=1.1)
-
-
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
-def test_sgd_bad_learning_rate_schedule(klass):
-    # Check whether expected ValueError on bad learning_rate
-    with pytest.raises(ValueError):
-        klass(learning_rate="")
-
-
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
-def test_sgd_bad_eta0(klass):
-    # Check whether expected ValueError on bad eta0
-    with pytest.raises(ValueError):
-        klass(eta0=0, learning_rate="constant")
-
-
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
-def test_sgd_max_iter_param(klass):
-    # Test parameter validity check
-    with pytest.raises(ValueError):
-        klass(max_iter=-10000)
-
-
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
-def test_sgd_shuffle_param(klass):
-    # Test parameter validity check
-    with pytest.raises(ValueError):
-        klass(shuffle="false")
-
-
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
-def test_sgd_early_stopping_param(klass):
-    # Test parameter validity check
-    with pytest.raises(ValueError):
-        klass(early_stopping="false")
-
-
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
-def test_sgd_validation_fraction(klass):
-    # Test parameter validity check
-    with pytest.raises(ValueError):
-        klass(validation_fraction=-.1)
-
-
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
-def test_sgd_n_iter_no_change(klass):
-    # Test parameter validity check
-    with pytest.raises(ValueError):
-        klass(n_iter_no_change=0)
-
-
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
-def test_argument_coef(klass):
-    # Checks coef_init not allowed as model argument (only fit)
-    # Provided coef_ does not match dataset
-    with pytest.raises(TypeError):
-        klass(coef_init=np.zeros((3,)))
-
-
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize(
+    "klass", [SGDClassifier, SparseSGDClassifier, SGDOneClassSVM, SparseSGDOneClassSVM]
+)
 def test_provide_coef(klass):
-    # Checks coef_init shape for the warm starts
-    # Provided coef_ does not match dataset.
-    with pytest.raises(ValueError):
+    """Check that the shape of `coef_init` is validated."""
+    with pytest.raises(ValueError, match="Provided coef_init does not match dataset"):
         klass().fit(X, Y, coef_init=np.zeros((3,)))
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
-def test_set_intercept(klass):
-    # Checks intercept_ shape for the warm starts
-    # Provided intercept_ does not match dataset.
-    with pytest.raises(ValueError):
-        klass().fit(X, Y, intercept_init=np.zeros((3,)))
-
-
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize(
+    "klass, fit_params",
+    [
+        (SGDClassifier, {"intercept_init": np.zeros((3,))}),
+        (SparseSGDClassifier, {"intercept_init": np.zeros((3,))}),
+        (SGDOneClassSVM, {"offset_init": np.zeros((3,))}),
+        (SparseSGDOneClassSVM, {"offset_init": np.zeros((3,))}),
+    ],
+)
+def test_set_intercept_offset(klass, fit_params):
+    """Check that `intercept_init` or `offset_init` is validated."""
+    sgd_estimator = klass()
+    with pytest.raises(ValueError, match="does not match dataset"):
+        sgd_estimator.fit(X, Y, **fit_params)
+
+
+@pytest.mark.parametrize(
+    "klass", [SGDClassifier, SparseSGDClassifier, SGDRegressor, SparseSGDRegressor]
+)
 def test_sgd_early_stopping_with_partial_fit(klass):
-    # Test parameter validity check
-    with pytest.raises(ValueError):
+    """Check that we raise an error for `early_stopping` used with
+    `partial_fit`.
+    """
+    err_msg = "early_stopping should be False with partial_fit"
+    with pytest.raises(ValueError, match=err_msg):
         klass(early_stopping=True).partial_fit(X, Y)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
-def test_set_intercept_binary(klass):
-    # Checks intercept_ shape for the warm starts in binary case
-    klass().fit(X5, Y5, intercept_init=0)
+@pytest.mark.parametrize(
+    "klass, fit_params",
+    [
+        (SGDClassifier, {"intercept_init": 0}),
+        (SparseSGDClassifier, {"intercept_init": 0}),
+        (SGDOneClassSVM, {"offset_init": 0}),
+        (SparseSGDOneClassSVM, {"offset_init": 0}),
+    ],
+)
+def test_set_intercept_offset_binary(klass, fit_params):
+    """Check that we can pass a scaler with binary classification to
+    `intercept_init` or `offset_init`."""
+    klass().fit(X5, Y5, **fit_params)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_average_binary_computed_correctly(klass):
     # Checks the SGDClassifier correctly computes the average weights
-    eta = .1
-    alpha = 2.
+    eta = 0.1
+    alpha = 2.0
     n_samples = 20
     n_features = 10
     rng = np.random.RandomState(0)
     X = rng.normal(size=(n_samples, n_features))
     w = rng.normal(size=n_features)
 
-    clf = klass(loss='squared_loss',
-                learning_rate='constant',
-                eta0=eta, alpha=alpha,
-                fit_intercept=True,
-                max_iter=1, average=True, shuffle=False)
+    clf = klass(
+        loss="squared_error",
+        learning_rate="constant",
+        eta0=eta,
+        alpha=alpha,
+        fit_intercept=True,
+        max_iter=1,
+        average=True,
+        shuffle=False,
+    )
 
     # simple linear function without noise
     y = np.dot(X, w)
@@ -554,13 +611,11 @@ def test_average_binary_computed_correctly(klass):
 
     average_weights, average_intercept = asgd(klass, X, y, eta, alpha)
     average_weights = average_weights.reshape(1, -1)
-    assert_array_almost_equal(clf.coef_,
-                              average_weights,
-                              decimal=14)
+    assert_array_almost_equal(clf.coef_, average_weights, decimal=14)
     assert_almost_equal(clf.intercept_, average_intercept, decimal=14)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_set_intercept_to_intercept(klass):
     # Checks intercept_ shape consistency for the warm starts
     # Inconsistent intercept_ shape.
@@ -570,7 +625,7 @@ def test_set_intercept_to_intercept(klass):
     klass().fit(X, Y, intercept_init=clf.intercept_)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_sgd_at_least_two_labels(klass):
     # Target must have at least two labels
     clf = klass(alpha=0.01, max_iter=20)
@@ -578,24 +633,24 @@ def test_sgd_at_least_two_labels(klass):
         clf.fit(X2, np.ones(9))
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_partial_fit_weight_class_balanced(klass):
     # partial_fit with class_weight='balanced' not supported"""
-    regex = (r"class_weight 'balanced' is not supported for "
-             r"partial_fit\. In order to use 'balanced' weights, "
-             r"use compute_class_weight\('balanced', classes=classes, y=y\). "
-             r"In place of y you can us a large enough sample "
-             r"of the full training set target to properly "
-             r"estimate the class frequency distributions\. "
-             r"Pass the resulting weights as the class_weight "
-             r"parameter\.")
-    assert_raises_regexp(ValueError,
-                         regex,
-                         klass(class_weight='balanced').partial_fit,
-                         X, Y, classes=np.unique(Y))
-
-
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+    regex = (
+        r"class_weight 'balanced' is not supported for "
+        r"partial_fit\. In order to use 'balanced' weights, "
+        r"use compute_class_weight\('balanced', classes=classes, y=y\). "
+        r"In place of y you can use a large enough sample "
+        r"of the full training set target to properly "
+        r"estimate the class frequency distributions\. "
+        r"Pass the resulting weights as the class_weight "
+        r"parameter\."
+    )
+    with pytest.raises(ValueError, match=regex):
+        klass(class_weight="balanced").partial_fit(X, Y, classes=np.unique(Y))
+
+
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_sgd_multiclass(klass):
     # Multi-class test case
     clf = klass(alpha=0.01, max_iter=20).fit(X2, Y2)
@@ -606,16 +661,21 @@ def test_sgd_multiclass(klass):
     assert_array_equal(pred, true_result2)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_sgd_multiclass_average(klass):
-    eta = .001
-    alpha = .01
+    eta = 0.001
+    alpha = 0.01
     # Multi-class average test case
-    clf = klass(loss='squared_loss',
-                learning_rate='constant',
-                eta0=eta, alpha=alpha,
-                fit_intercept=True,
-                max_iter=1, average=True, shuffle=False)
+    clf = klass(
+        loss="squared_error",
+        learning_rate="constant",
+        eta0=eta,
+        alpha=alpha,
+        fit_intercept=True,
+        max_iter=1,
+        average=True,
+        shuffle=False,
+    )
 
     np_Y2 = np.array(Y2)
     clf.fit(X2, np_Y2)
@@ -626,24 +686,21 @@ def test_sgd_multiclass_average(klass):
         y_i[np_Y2 != cl] = -1
         average_coef, average_intercept = asgd(klass, X2, y_i, eta, alpha)
         assert_array_almost_equal(average_coef, clf.coef_[i], decimal=16)
-        assert_almost_equal(average_intercept,
-                            clf.intercept_[i],
-                            decimal=16)
+        assert_almost_equal(average_intercept, clf.intercept_[i], decimal=16)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_sgd_multiclass_with_init_coef(klass):
     # Multi-class test case
     clf = klass(alpha=0.01, max_iter=20)
-    clf.fit(X2, Y2, coef_init=np.zeros((3, 2)),
-            intercept_init=np.zeros(3))
+    clf.fit(X2, Y2, coef_init=np.zeros((3, 2)), intercept_init=np.zeros(3))
     assert clf.coef_.shape == (3, 2)
     assert clf.intercept_.shape, (3,)
     pred = clf.predict(T2)
     assert_array_equal(pred, true_result2)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_sgd_multiclass_njobs(klass):
     # Multi-class test case with multi-core support
     clf = klass(alpha=0.01, max_iter=20, n_jobs=2).fit(X2, Y2)
@@ -654,7 +711,7 @@ def test_sgd_multiclass_njobs(klass):
     assert_array_equal(pred, true_result2)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_set_coef_multiclass(klass):
     # Checks coef_init and intercept_init shape for multi-class
     # problems
@@ -675,7 +732,7 @@ def test_set_coef_multiclass(klass):
     clf = klass().fit(X2, Y2, intercept_init=np.zeros((3,)))
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_sgd_predict_proba_method_access(klass):
     # Checks that SGDClassifier predict_proba and predict_log_proba methods
     # can either be accessed or raise an appropriate error message
@@ -684,37 +741,45 @@ def test_sgd_predict_proba_method_access(klass):
     # details.
     for loss in linear_model.SGDClassifier.loss_functions:
         clf = SGDClassifier(loss=loss)
-        if loss in ('log', 'modified_huber'):
-            assert hasattr(clf, 'predict_proba')
-            assert hasattr(clf, 'predict_log_proba')
+        if loss in ("log_loss", "modified_huber"):
+            assert hasattr(clf, "predict_proba")
+            assert hasattr(clf, "predict_log_proba")
         else:
-            message = ("probability estimates are not "
-                       "available for loss={!r}".format(loss))
-            assert not hasattr(clf, 'predict_proba')
-            assert not hasattr(clf, 'predict_log_proba')
-            with pytest.raises(AttributeError,
-                               match=message):
+            inner_msg = "probability estimates are not available for loss={!r}".format(
+                loss
+            )
+            assert not hasattr(clf, "predict_proba")
+            assert not hasattr(clf, "predict_log_proba")
+            with pytest.raises(
+                AttributeError, match="has no attribute 'predict_proba'"
+            ) as exec_info:
                 clf.predict_proba
-            with pytest.raises(AttributeError,
-                               match=message):
+
+            assert isinstance(exec_info.value.__cause__, AttributeError)
+            assert inner_msg in str(exec_info.value.__cause__)
+
+            with pytest.raises(
+                AttributeError, match="has no attribute 'predict_log_proba'"
+            ) as exec_info:
                 clf.predict_log_proba
+            assert isinstance(exec_info.value.__cause__, AttributeError)
+            assert inner_msg in str(exec_info.value.__cause__)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_sgd_proba(klass):
     # Check SGD.predict_proba
 
     # Hinge loss does not allow for conditional prob estimate.
     # We cannot use the factory here, because it defines predict_proba
     # anyway.
-    clf = SGDClassifier(loss="hinge", alpha=0.01,
-                        max_iter=10, tol=None).fit(X, Y)
+    clf = SGDClassifier(loss="hinge", alpha=0.01, max_iter=10, tol=None).fit(X, Y)
     assert not hasattr(clf, "predict_proba")
     assert not hasattr(clf, "predict_log_proba")
 
     # log and modified_huber losses can output probability estimates
     # binary case
-    for loss in ["log", "modified_huber"]:
+    for loss in ["log_loss", "modified_huber"]:
         clf = klass(loss=loss, alpha=0.01, max_iter=10)
         clf.fit(X, Y)
         p = clf.predict_proba([[3, 2]])
@@ -722,16 +787,19 @@ def test_sgd_proba(klass):
         p = clf.predict_proba([[-1, -1]])
         assert p[0, 1] < 0.5
 
-        p = clf.predict_log_proba([[3, 2]])
-        assert p[0, 1] > p[0, 0]
-        p = clf.predict_log_proba([[-1, -1]])
-        assert p[0, 1] < p[0, 0]
+        # If predict_proba is 0, we get "RuntimeWarning: divide by zero encountered
+        # in log". We avoid it here.
+        with np.errstate(divide="ignore"):
+            p = clf.predict_log_proba([[3, 2]])
+            assert p[0, 1] > p[0, 0]
+            p = clf.predict_log_proba([[-1, -1]])
+            assert p[0, 1] < p[0, 0]
 
     # log loss multiclass probability estimates
-    clf = klass(loss="log", alpha=0.01, max_iter=10).fit(X2, Y2)
+    clf = klass(loss="log_loss", alpha=0.01, max_iter=10).fit(X2, Y2)
 
-    d = clf.decision_function([[.1, -.1], [.3, .2]])
-    p = clf.predict_proba([[.1, -.1], [.3, .2]])
+    d = clf.decision_function([[0.1, -0.1], [0.3, 0.2]])
+    p = clf.predict_proba([[0.1, -0.1], [0.3, 0.2]])
     assert_array_equal(np.argmax(p, axis=1), np.argmax(d, axis=1))
     assert_almost_equal(p[0].sum(), 1)
     assert np.all(p[0] >= 0)
@@ -757,7 +825,7 @@ def test_sgd_proba(klass):
     p = clf.predict_proba([[3, 2]])
     if klass != SparseSGDClassifier:
         assert np.argmax(d, axis=1) == np.argmax(p, axis=1)
-    else:   # XXX the sparse test gets a different X2 (?)
+    else:  # XXX the sparse test gets a different X2 (?)
         assert np.argmin(d, axis=1) == np.argmin(p, axis=1)
 
     # the following sample produces decision_function values < -1,
@@ -767,10 +835,10 @@ def test_sgd_proba(klass):
     d = clf.decision_function([x])
     if np.all(d < -1):  # XXX not true in sparse test case (why?)
         p = clf.predict_proba([x])
-        assert_array_almost_equal(p[0], [1 / 3.] * 3)
+        assert_array_almost_equal(p[0], [1 / 3.0] * 3)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_sgd_l1(klass):
     # Test L1 regularization
     n = len(X4)
@@ -781,8 +849,14 @@ def test_sgd_l1(klass):
     X = X4[idx, :]
     Y = Y4[idx]
 
-    clf = klass(penalty='l1', alpha=.2, fit_intercept=False,
-                max_iter=2000, tol=None, shuffle=False)
+    clf = klass(
+        penalty="l1",
+        alpha=0.2,
+        fit_intercept=False,
+        max_iter=2000,
+        tol=None,
+        shuffle=False,
+    )
     clf.fit(X, Y)
     assert_array_equal(clf.coef_[0, 1:-1], np.zeros((4,)))
     pred = clf.predict(X)
@@ -801,21 +875,18 @@ def test_sgd_l1(klass):
     assert_array_equal(pred, Y)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_class_weights(klass):
     # Test class weights.
-    X = np.array([[-1.0, -1.0], [-1.0, 0], [-.8, -1.0],
-                  [1.0, 1.0], [1.0, 0.0]])
+    X = np.array([[-1.0, -1.0], [-1.0, 0], [-0.8, -1.0], [1.0, 1.0], [1.0, 0.0]])
     y = [1, 1, 1, -1, -1]
 
-    clf = klass(alpha=0.1, max_iter=1000, fit_intercept=False,
-                class_weight=None)
+    clf = klass(alpha=0.1, max_iter=1000, fit_intercept=False, class_weight=None)
     clf.fit(X, y)
     assert_array_equal(clf.predict([[0.2, -1.0]]), np.array([1]))
 
     # we give a small weights to class 1
-    clf = klass(alpha=0.1, max_iter=1000, fit_intercept=False,
-                class_weight={1: 0.001})
+    clf = klass(alpha=0.1, max_iter=1000, fit_intercept=False, class_weight={1: 0.001})
     clf.fit(X, y)
 
     # now the hyperplane should rotate clock-wise and
@@ -823,7 +894,7 @@ def test_class_weights(klass):
     assert_array_equal(clf.predict([[0.2, -1.0]]), np.array([-1]))
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_equal_class_weight(klass):
     # Test if equal class weights approx. equals no class weights.
     X = [[1, 0], [1, 0], [0, 1], [0, 1]]
@@ -833,15 +904,14 @@ def test_equal_class_weight(klass):
 
     X = [[1, 0], [0, 1]]
     y = [0, 1]
-    clf_weighted = klass(alpha=0.1, max_iter=1000,
-                         class_weight={0: 0.5, 1: 0.5})
+    clf_weighted = klass(alpha=0.1, max_iter=1000, class_weight={0: 0.5, 1: 0.5})
     clf_weighted.fit(X, y)
 
     # should be similar up to some epsilon due to learning rate schedule
     assert_almost_equal(clf.coef_, clf_weighted.coef_, decimal=2)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_wrong_class_weight_label(klass):
     # ValueError due to not existing class label.
     clf = klass(alpha=0.1, max_iter=1000, class_weight={0: 0.5})
@@ -849,18 +919,10 @@ def test_wrong_class_weight_label(klass):
         clf.fit(X, Y)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
-def test_wrong_class_weight_format(klass):
-    # ValueError due to wrong class_weight argument type.
-    clf = klass(alpha=0.1, max_iter=1000, class_weight=[0.5])
-    with pytest.raises(ValueError):
-        clf.fit(X, Y)
-
-
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_weights_multiplied(klass):
     # Tests that class_weight and sample_weight are multiplicative
-    class_weights = {1: .6, 2: .3}
+    class_weights = {1: 0.6, 2: 0.3}
     rng = np.random.RandomState(0)
     sample_weights = rng.random_sample(Y4.shape[0])
     multiplied_together = np.copy(sample_weights)
@@ -876,7 +938,7 @@ def test_weights_multiplied(klass):
     assert_almost_equal(clf1.coef_, clf2.coef_)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_balanced_weight(klass):
     # Test class weights for imbalanced data"""
     # compute reference metrics on iris dataset that is quite balanced by
@@ -888,16 +950,15 @@ def test_balanced_weight(klass):
     rng.shuffle(idx)
     X = X[idx]
     y = y[idx]
-    clf = klass(alpha=0.0001, max_iter=1000,
-                class_weight=None, shuffle=False).fit(X, y)
-    f1 = metrics.f1_score(y, clf.predict(X), average='weighted')
+    clf = klass(alpha=0.0001, max_iter=1000, class_weight=None, shuffle=False).fit(X, y)
+    f1 = metrics.f1_score(y, clf.predict(X), average="weighted")
     assert_almost_equal(f1, 0.96, decimal=1)
 
     # make the same prediction using balanced class_weight
-    clf_balanced = klass(alpha=0.0001, max_iter=1000,
-                         class_weight="balanced",
-                         shuffle=False).fit(X, y)
-    f1 = metrics.f1_score(y, clf_balanced.predict(X), average='weighted')
+    clf_balanced = klass(
+        alpha=0.0001, max_iter=1000, class_weight="balanced", shuffle=False
+    ).fit(X, y)
+    f1 = metrics.f1_score(y, clf_balanced.predict(X), average="weighted")
     assert_almost_equal(f1, 0.96, decimal=1)
 
     # Make sure that in the balanced case it does not change anything
@@ -915,21 +976,19 @@ def test_balanced_weight(klass):
     clf = klass(max_iter=1000, class_weight=None, shuffle=False)
     clf.fit(X_imbalanced, y_imbalanced)
     y_pred = clf.predict(X)
-    assert metrics.f1_score(y, y_pred, average='weighted') < 0.96
+    assert metrics.f1_score(y, y_pred, average="weighted") < 0.96
 
     # fit a model with balanced class_weight enabled
-    clf = klass(max_iter=1000, class_weight="balanced",
-                shuffle=False)
+    clf = klass(max_iter=1000, class_weight="balanced", shuffle=False)
     clf.fit(X_imbalanced, y_imbalanced)
     y_pred = clf.predict(X)
-    assert metrics.f1_score(y, y_pred, average='weighted') > 0.96
+    assert metrics.f1_score(y, y_pred, average="weighted") > 0.96
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_sample_weights(klass):
     # Test weights on individual samples
-    X = np.array([[-1.0, -1.0], [-1.0, 0], [-.8, -1.0],
-                  [1.0, 1.0], [1.0, 0.0]])
+    X = np.array([[-1.0, -1.0], [-1.0, 0], [-0.8, -1.0], [1.0, 1.0], [1.0, 0.0]])
     y = [1, 1, 1, -1, -1]
 
     clf = klass(alpha=0.1, max_iter=1000, fit_intercept=False)
@@ -944,16 +1003,21 @@ def test_sample_weights(klass):
     assert_array_equal(clf.predict([[0.2, -1.0]]), np.array([-1]))
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize(
+    "klass", [SGDClassifier, SparseSGDClassifier, SGDOneClassSVM, SparseSGDOneClassSVM]
+)
 def test_wrong_sample_weights(klass):
     # Test if ValueError is raised if sample_weight has wrong shape
-    clf = klass(alpha=0.1, max_iter=1000, fit_intercept=False)
+    if klass in [SGDClassifier, SparseSGDClassifier]:
+        clf = klass(alpha=0.1, max_iter=1000, fit_intercept=False)
+    elif klass in [SGDOneClassSVM, SparseSGDOneClassSVM]:
+        clf = klass(nu=0.1, max_iter=1000, fit_intercept=False)
     # provided sample_weight too long
     with pytest.raises(ValueError):
         clf.fit(X, Y, sample_weight=np.arange(7))
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_partial_fit_exception(klass):
     clf = klass(alpha=0.01)
     # classes was not specified
@@ -961,7 +1025,7 @@ def test_partial_fit_exception(klass):
         clf.partial_fit(X3, Y3)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_partial_fit_binary(klass):
     third = X.shape[0] // 3
     clf = klass(alpha=0.01)
@@ -970,7 +1034,7 @@ def test_partial_fit_binary(klass):
     clf.partial_fit(X[:third], Y[:third], classes=classes)
     assert clf.coef_.shape == (1, X.shape[1])
     assert clf.intercept_.shape == (1,)
-    assert clf.decision_function([[0, 0]]).shape == (1, )
+    assert clf.decision_function([[0, 0]]).shape == (1,)
     id1 = id(clf.coef_.data)
 
     clf.partial_fit(X[third:], Y[third:])
@@ -982,7 +1046,7 @@ def test_partial_fit_binary(klass):
     assert_array_equal(y_pred, true_result)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_partial_fit_multiclass(klass):
     third = X2.shape[0] // 3
     clf = klass(alpha=0.01)
@@ -1000,7 +1064,7 @@ def test_partial_fit_multiclass(klass):
     assert id1, id2
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_partial_fit_multiclass_average(klass):
     third = X2.shape[0] // 3
     clf = klass(alpha=0.01, average=X2.shape[0])
@@ -1015,30 +1079,27 @@ def test_partial_fit_multiclass_average(klass):
     assert clf.intercept_.shape == (3,)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_fit_then_partial_fit(klass):
     # Partial_fit should work after initial fit in the multiclass case.
     # Non-regression test for #2496; fit would previously produce a
     # Fortran-ordered coef_ that subsequent partial_fit couldn't handle.
     clf = klass()
     clf.fit(X2, Y2)
-    clf.partial_fit(X2, Y2)     # no exception here
+    clf.partial_fit(X2, Y2)  # no exception here
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
-@pytest.mark.parametrize('lr',
-                         ["constant", "optimal", "invscaling", "adaptive"])
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize("lr", ["constant", "optimal", "invscaling", "adaptive"])
 def test_partial_fit_equal_fit_classif(klass, lr):
     for X_, Y_, T_ in ((X, Y, T), (X2, Y2, T2)):
-        clf = klass(alpha=0.01, eta0=0.01, max_iter=2,
-                    learning_rate=lr, shuffle=False)
+        clf = klass(alpha=0.01, eta0=0.01, max_iter=2, learning_rate=lr, shuffle=False)
         clf.fit(X_, Y_)
         y_pred = clf.decision_function(T_)
         t = clf.t_
 
         classes = np.unique(Y_)
-        clf = klass(alpha=0.01, eta0=0.01, learning_rate=lr,
-                    shuffle=False)
+        clf = klass(alpha=0.01, eta0=0.01, learning_rate=lr, shuffle=False)
         for i in range(2):
             clf.partial_fit(X_, Y_, classes=classes)
         y_pred2 = clf.decision_function(T_)
@@ -1047,18 +1108,26 @@ def test_partial_fit_equal_fit_classif(klass, lr):
         assert_array_almost_equal(y_pred, y_pred2, decimal=2)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_regression_losses(klass):
     random_state = np.random.RandomState(1)
-    clf = klass(alpha=0.01, learning_rate="constant",
-                eta0=0.1, loss="epsilon_insensitive",
-                random_state=random_state)
+    clf = klass(
+        alpha=0.01,
+        learning_rate="constant",
+        eta0=0.1,
+        loss="epsilon_insensitive",
+        random_state=random_state,
+    )
     clf.fit(X, Y)
     assert 1.0 == np.mean(clf.predict(X) == Y)
 
-    clf = klass(alpha=0.01, learning_rate="constant",
-                eta0=0.1, loss="squared_epsilon_insensitive",
-                random_state=random_state)
+    clf = klass(
+        alpha=0.01,
+        learning_rate="constant",
+        eta0=0.1,
+        loss="squared_epsilon_insensitive",
+        random_state=random_state,
+    )
     clf.fit(X, Y)
     assert 1.0 == np.mean(clf.predict(X) == Y)
 
@@ -1066,18 +1135,23 @@ def test_regression_losses(klass):
     clf.fit(X, Y)
     assert 1.0 == np.mean(clf.predict(X) == Y)
 
-    clf = klass(alpha=0.01, learning_rate="constant", eta0=0.01,
-                loss="squared_loss", random_state=random_state)
+    clf = klass(
+        alpha=0.01,
+        learning_rate="constant",
+        eta0=0.01,
+        loss="squared_error",
+        random_state=random_state,
+    )
     clf.fit(X, Y)
     assert 1.0 == np.mean(clf.predict(X) == Y)
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_warm_start_multiclass(klass):
     _test_warm_start(klass, X2, Y2, "optimal")
 
 
-@pytest.mark.parametrize('klass', [SGDClassifier, SparseSGDClassifier])
+@pytest.mark.parametrize("klass", [SGDClassifier, SparseSGDClassifier])
 def test_multiple_fit(klass):
     # Test multiple calls of fit w/ different shaped inputs.
     clf = klass(alpha=0.01, shuffle=False)
@@ -1092,7 +1166,8 @@ def test_multiple_fit(klass):
 ###############################################################################
 # Regression Test Case
 
-@pytest.mark.parametrize('klass', [SGDRegressor, SparseSGDRegressor])
+
+@pytest.mark.parametrize("klass", [SGDRegressor, SparseSGDRegressor])
 def test_sgd_reg(klass):
     # Check that SGD gives any results.
     clf = klass(alpha=0.1, max_iter=2, fit_intercept=False)
@@ -1100,12 +1175,12 @@ def test_sgd_reg(klass):
     assert clf.coef_[0] == clf.coef_[1]
 
 
-@pytest.mark.parametrize('klass', [SGDRegressor, SparseSGDRegressor])
+@pytest.mark.parametrize("klass", [SGDRegressor, SparseSGDRegressor])
 def test_sgd_averaged_computed_correctly(klass):
     # Tests the average regressor matches the naive implementation
 
-    eta = .001
-    alpha = .01
+    eta = 0.001
+    alpha = 0.01
     n_samples = 20
     n_features = 10
     rng = np.random.RandomState(0)
@@ -1115,26 +1190,29 @@ def test_sgd_averaged_computed_correctly(klass):
     # simple linear function without noise
     y = np.dot(X, w)
 
-    clf = klass(loss='squared_loss',
-                learning_rate='constant',
-                eta0=eta, alpha=alpha,
-                fit_intercept=True,
-                max_iter=1, average=True, shuffle=False)
+    clf = klass(
+        loss="squared_error",
+        learning_rate="constant",
+        eta0=eta,
+        alpha=alpha,
+        fit_intercept=True,
+        max_iter=1,
+        average=True,
+        shuffle=False,
+    )
 
     clf.fit(X, y)
     average_weights, average_intercept = asgd(klass, X, y, eta, alpha)
 
-    assert_array_almost_equal(clf.coef_,
-                              average_weights,
-                              decimal=16)
+    assert_array_almost_equal(clf.coef_, average_weights, decimal=16)
     assert_almost_equal(clf.intercept_, average_intercept, decimal=16)
 
 
-@pytest.mark.parametrize('klass', [SGDRegressor, SparseSGDRegressor])
+@pytest.mark.parametrize("klass", [SGDRegressor, SparseSGDRegressor])
 def test_sgd_averaged_partial_fit(klass):
     # Tests whether the partial fit yields the same average as the fit
-    eta = .001
-    alpha = .01
+    eta = 0.001
+    alpha = 0.01
     n_samples = 20
     n_features = 10
     rng = np.random.RandomState(0)
@@ -1144,47 +1222,53 @@ def test_sgd_averaged_partial_fit(klass):
     # simple linear function without noise
     y = np.dot(X, w)
 
-    clf = klass(loss='squared_loss',
-                learning_rate='constant',
-                eta0=eta, alpha=alpha,
-                fit_intercept=True,
-                max_iter=1, average=True, shuffle=False)
+    clf = klass(
+        loss="squared_error",
+        learning_rate="constant",
+        eta0=eta,
+        alpha=alpha,
+        fit_intercept=True,
+        max_iter=1,
+        average=True,
+        shuffle=False,
+    )
 
-    clf.partial_fit(X[:int(n_samples / 2)][:], y[:int(n_samples / 2)])
-    clf.partial_fit(X[int(n_samples / 2):][:], y[int(n_samples / 2):])
+    clf.partial_fit(X[: int(n_samples / 2)][:], y[: int(n_samples / 2)])
+    clf.partial_fit(X[int(n_samples / 2) :][:], y[int(n_samples / 2) :])
     average_weights, average_intercept = asgd(klass, X, y, eta, alpha)
 
-    assert_array_almost_equal(clf.coef_,
-                              average_weights,
-                              decimal=16)
+    assert_array_almost_equal(clf.coef_, average_weights, decimal=16)
     assert_almost_equal(clf.intercept_[0], average_intercept, decimal=16)
 
 
-@pytest.mark.parametrize('klass', [SGDRegressor, SparseSGDRegressor])
+@pytest.mark.parametrize("klass", [SGDRegressor, SparseSGDRegressor])
 def test_average_sparse(klass):
     # Checks the average weights on data with 0s
 
-    eta = .001
-    alpha = .01
-    clf = klass(loss='squared_loss',
-                learning_rate='constant',
-                eta0=eta, alpha=alpha,
-                fit_intercept=True,
-                max_iter=1, average=True, shuffle=False)
+    eta = 0.001
+    alpha = 0.01
+    clf = klass(
+        loss="squared_error",
+        learning_rate="constant",
+        eta0=eta,
+        alpha=alpha,
+        fit_intercept=True,
+        max_iter=1,
+        average=True,
+        shuffle=False,
+    )
 
     n_samples = Y3.shape[0]
 
-    clf.partial_fit(X3[:int(n_samples / 2)][:], Y3[:int(n_samples / 2)])
-    clf.partial_fit(X3[int(n_samples / 2):][:], Y3[int(n_samples / 2):])
+    clf.partial_fit(X3[: int(n_samples / 2)][:], Y3[: int(n_samples / 2)])
+    clf.partial_fit(X3[int(n_samples / 2) :][:], Y3[int(n_samples / 2) :])
     average_weights, average_intercept = asgd(klass, X3, Y3, eta, alpha)
 
-    assert_array_almost_equal(clf.coef_,
-                              average_weights,
-                              decimal=16)
+    assert_array_almost_equal(clf.coef_, average_weights, decimal=16)
     assert_almost_equal(clf.intercept_, average_intercept, decimal=16)
 
 
-@pytest.mark.parametrize('klass', [SGDRegressor, SparseSGDRegressor])
+@pytest.mark.parametrize("klass", [SGDRegressor, SparseSGDRegressor])
 def test_sgd_least_squares_fit(klass):
     xmin, xmax = -5, 5
     n_samples = 100
@@ -1194,8 +1278,7 @@ def test_sgd_least_squares_fit(klass):
     # simple linear function without noise
     y = 0.5 * X.ravel()
 
-    clf = klass(loss='squared_loss', alpha=0.1, max_iter=20,
-                fit_intercept=False)
+    clf = klass(loss="squared_error", alpha=0.1, max_iter=20, fit_intercept=False)
     clf.fit(X, y)
     score = clf.score(X, y)
     assert score > 0.99
@@ -1203,14 +1286,13 @@ def test_sgd_least_squares_fit(klass):
     # simple linear function with noise
     y = 0.5 * X.ravel() + rng.randn(n_samples, 1).ravel()
 
-    clf = klass(loss='squared_loss', alpha=0.1, max_iter=20,
-                fit_intercept=False)
+    clf = klass(loss="squared_error", alpha=0.1, max_iter=20, fit_intercept=False)
     clf.fit(X, y)
     score = clf.score(X, y)
     assert score > 0.5
 
 
-@pytest.mark.parametrize('klass', [SGDRegressor, SparseSGDRegressor])
+@pytest.mark.parametrize("klass", [SGDRegressor, SparseSGDRegressor])
 def test_sgd_epsilon_insensitive(klass):
     xmin, xmax = -5, 5
     n_samples = 100
@@ -1220,9 +1302,13 @@ def test_sgd_epsilon_insensitive(klass):
     # simple linear function without noise
     y = 0.5 * X.ravel()
 
-    clf = klass(loss='epsilon_insensitive', epsilon=0.01,
-                alpha=0.1, max_iter=20,
-                fit_intercept=False)
+    clf = klass(
+        loss="epsilon_insensitive",
+        epsilon=0.01,
+        alpha=0.1,
+        max_iter=20,
+        fit_intercept=False,
+    )
     clf.fit(X, y)
     score = clf.score(X, y)
     assert score > 0.99
@@ -1230,15 +1316,19 @@ def test_sgd_epsilon_insensitive(klass):
     # simple linear function with noise
     y = 0.5 * X.ravel() + rng.randn(n_samples, 1).ravel()
 
-    clf = klass(loss='epsilon_insensitive', epsilon=0.01,
-                alpha=0.1, max_iter=20,
-                fit_intercept=False)
+    clf = klass(
+        loss="epsilon_insensitive",
+        epsilon=0.01,
+        alpha=0.1,
+        max_iter=20,
+        fit_intercept=False,
+    )
     clf.fit(X, y)
     score = clf.score(X, y)
     assert score > 0.5
 
 
-@pytest.mark.parametrize('klass', [SGDRegressor, SparseSGDRegressor])
+@pytest.mark.parametrize("klass", [SGDRegressor, SparseSGDRegressor])
 def test_sgd_huber_fit(klass):
     xmin, xmax = -5, 5
     n_samples = 100
@@ -1248,8 +1338,7 @@ def test_sgd_huber_fit(klass):
     # simple linear function without noise
     y = 0.5 * X.ravel()
 
-    clf = klass(loss="huber", epsilon=0.1, alpha=0.1, max_iter=20,
-                fit_intercept=False)
+    clf = klass(loss="huber", epsilon=0.1, alpha=0.1, max_iter=20, fit_intercept=False)
     clf.fit(X, y)
     score = clf.score(X, y)
     assert score > 0.99
@@ -1257,14 +1346,13 @@ def test_sgd_huber_fit(klass):
     # simple linear function with noise
     y = 0.5 * X.ravel() + rng.randn(n_samples, 1).ravel()
 
-    clf = klass(loss="huber", epsilon=0.1, alpha=0.1, max_iter=20,
-                fit_intercept=False)
+    clf = klass(loss="huber", epsilon=0.1, alpha=0.1, max_iter=20, fit_intercept=False)
     clf.fit(X, y)
     score = clf.score(X, y)
     assert score > 0.5
 
 
-@pytest.mark.parametrize('klass', [SGDRegressor, SparseSGDRegressor])
+@pytest.mark.parametrize("klass", [SGDRegressor, SparseSGDRegressor])
 def test_elasticnet_convergence(klass):
     # Check that the SGD output is consistent with coordinate descent
 
@@ -1279,30 +1367,34 @@ def test_elasticnet_convergence(klass):
     # XXX: alpha = 0.1 seems to cause convergence problems
     for alpha in [0.01, 0.001]:
         for l1_ratio in [0.5, 0.8, 1.0]:
-            cd = linear_model.ElasticNet(alpha=alpha, l1_ratio=l1_ratio,
-                                         fit_intercept=False)
+            cd = linear_model.ElasticNet(
+                alpha=alpha, l1_ratio=l1_ratio, fit_intercept=False
+            )
             cd.fit(X, y)
-            sgd = klass(penalty='elasticnet', max_iter=50,
-                        alpha=alpha, l1_ratio=l1_ratio,
-                        fit_intercept=False)
+            sgd = klass(
+                penalty="elasticnet",
+                max_iter=50,
+                alpha=alpha,
+                l1_ratio=l1_ratio,
+                fit_intercept=False,
+            )
             sgd.fit(X, y)
-            err_msg = ("cd and sgd did not converge to comparable "
-                       "results for alpha=%f and l1_ratio=%f"
-                       % (alpha, l1_ratio))
-            assert_almost_equal(cd.coef_, sgd.coef_, decimal=2,
-                                err_msg=err_msg)
+            err_msg = (
+                "cd and sgd did not converge to comparable "
+                "results for alpha=%f and l1_ratio=%f" % (alpha, l1_ratio)
+            )
+            assert_almost_equal(cd.coef_, sgd.coef_, decimal=2, err_msg=err_msg)
 
 
-@ignore_warnings
-@pytest.mark.parametrize('klass', [SGDRegressor, SparseSGDRegressor])
+@pytest.mark.parametrize("klass", [SGDRegressor, SparseSGDRegressor])
 def test_partial_fit(klass):
     third = X.shape[0] // 3
     clf = klass(alpha=0.01)
 
     clf.partial_fit(X[:third], Y[:third])
-    assert clf.coef_.shape == (X.shape[1], )
+    assert clf.coef_.shape == (X.shape[1],)
     assert clf.intercept_.shape == (1,)
-    assert clf.predict([[0, 0]]).shape == (1, )
+    assert clf.predict([[0, 0]]).shape == (1,)
     id1 = id(clf.coef_.data)
 
     clf.partial_fit(X[third:], Y[third:])
@@ -1311,18 +1403,15 @@ def test_partial_fit(klass):
     assert id1, id2
 
 
-@pytest.mark.parametrize('klass', [SGDRegressor, SparseSGDRegressor])
-@pytest.mark.parametrize('lr',
-                         ["constant", "optimal", "invscaling", "adaptive"])
+@pytest.mark.parametrize("klass", [SGDRegressor, SparseSGDRegressor])
+@pytest.mark.parametrize("lr", ["constant", "optimal", "invscaling", "adaptive"])
 def test_partial_fit_equal_fit(klass, lr):
-    clf = klass(alpha=0.01, max_iter=2, eta0=0.01,
-                learning_rate=lr, shuffle=False)
+    clf = klass(alpha=0.01, max_iter=2, eta0=0.01, learning_rate=lr, shuffle=False)
     clf.fit(X, Y)
     y_pred = clf.predict(T)
     t = clf.t_
 
-    clf = klass(alpha=0.01, eta0=0.01,
-                learning_rate=lr, shuffle=False)
+    clf = klass(alpha=0.01, eta0=0.01, learning_rate=lr, shuffle=False)
     for i in range(2):
         clf.partial_fit(X, Y)
     y_pred2 = clf.predict(T)
@@ -1331,38 +1420,353 @@ def test_partial_fit_equal_fit(klass, lr):
     assert_array_almost_equal(y_pred, y_pred2, decimal=2)
 
 
-@pytest.mark.parametrize('klass', [SGDRegressor, SparseSGDRegressor])
+@pytest.mark.parametrize("klass", [SGDRegressor, SparseSGDRegressor])
 def test_loss_function_epsilon(klass):
     clf = klass(epsilon=0.9)
     clf.set_params(epsilon=0.1)
-    assert clf.loss_functions['huber'][1] == 0.1
+    assert clf.loss_functions["huber"][1] == 0.1
+
+
+###############################################################################
+# SGD One Class SVM Test Case
+
+
+# a simple implementation of ASGD to use for testing SGDOneClassSVM
+def asgd_oneclass(klass, X, eta, nu, coef_init=None, offset_init=0.0):
+    if coef_init is None:
+        coef = np.zeros(X.shape[1])
+    else:
+        coef = coef_init
+
+    average_coef = np.zeros(X.shape[1])
+    offset = offset_init
+    intercept = 1 - offset
+    average_intercept = 0.0
+    decay = 1.0
+
+    # sparse data has a fixed decay of .01
+    if klass == SparseSGDOneClassSVM:
+        decay = 0.01
+
+    for i, entry in enumerate(X):
+        p = np.dot(entry, coef)
+        p += intercept
+        if p <= 1.0:
+            gradient = -1
+        else:
+            gradient = 0
+        coef *= max(0, 1.0 - (eta * nu / 2))
+        coef += -(eta * gradient * entry)
+        intercept += -(eta * (nu + gradient)) * decay
+
+        average_coef *= i
+        average_coef += coef
+        average_coef /= i + 1.0
+
+        average_intercept *= i
+        average_intercept += intercept
+        average_intercept /= i + 1.0
+
+    return average_coef, 1 - average_intercept
+
+
+@pytest.mark.parametrize("klass", [SGDOneClassSVM, SparseSGDOneClassSVM])
+def _test_warm_start_oneclass(klass, X, lr):
+    # Test that explicit warm restart...
+    clf = klass(nu=0.5, eta0=0.01, shuffle=False, learning_rate=lr)
+    clf.fit(X)
+
+    clf2 = klass(nu=0.1, eta0=0.01, shuffle=False, learning_rate=lr)
+    clf2.fit(X, coef_init=clf.coef_.copy(), offset_init=clf.offset_.copy())
+
+    # ... and implicit warm restart are equivalent.
+    clf3 = klass(nu=0.5, eta0=0.01, shuffle=False, warm_start=True, learning_rate=lr)
+    clf3.fit(X)
+
+    assert clf3.t_ == clf.t_
+    assert_allclose(clf3.coef_, clf.coef_)
+
+    clf3.set_params(nu=0.1)
+    clf3.fit(X)
+
+    assert clf3.t_ == clf2.t_
+    assert_allclose(clf3.coef_, clf2.coef_)
+
+
+@pytest.mark.parametrize("klass", [SGDOneClassSVM, SparseSGDOneClassSVM])
+@pytest.mark.parametrize("lr", ["constant", "optimal", "invscaling", "adaptive"])
+def test_warm_start_oneclass(klass, lr):
+    _test_warm_start_oneclass(klass, X, lr)
+
+
+@pytest.mark.parametrize("klass", [SGDOneClassSVM, SparseSGDOneClassSVM])
+def test_clone_oneclass(klass):
+    # Test whether clone works ok.
+    clf = klass(nu=0.5)
+    clf = clone(clf)
+    clf.set_params(nu=0.1)
+    clf.fit(X)
+
+    clf2 = klass(nu=0.1)
+    clf2.fit(X)
+
+    assert_array_equal(clf.coef_, clf2.coef_)
+
+
+@pytest.mark.parametrize("klass", [SGDOneClassSVM, SparseSGDOneClassSVM])
+def test_partial_fit_oneclass(klass):
+    third = X.shape[0] // 3
+    clf = klass(nu=0.1)
+
+    clf.partial_fit(X[:third])
+    assert clf.coef_.shape == (X.shape[1],)
+    assert clf.offset_.shape == (1,)
+    assert clf.predict([[0, 0]]).shape == (1,)
+    previous_coefs = clf.coef_
+
+    clf.partial_fit(X[third:])
+    # check that coef_ haven't been re-allocated
+    assert clf.coef_ is previous_coefs
+
+    # raises ValueError if number of features does not match previous data
+    with pytest.raises(ValueError):
+        clf.partial_fit(X[:, 1])
+
+
+@pytest.mark.parametrize("klass", [SGDOneClassSVM, SparseSGDOneClassSVM])
+@pytest.mark.parametrize("lr", ["constant", "optimal", "invscaling", "adaptive"])
+def test_partial_fit_equal_fit_oneclass(klass, lr):
+    clf = klass(nu=0.05, max_iter=2, eta0=0.01, learning_rate=lr, shuffle=False)
+    clf.fit(X)
+    y_scores = clf.decision_function(T)
+    t = clf.t_
+    coef = clf.coef_
+    offset = clf.offset_
+
+    clf = klass(nu=0.05, eta0=0.01, max_iter=1, learning_rate=lr, shuffle=False)
+    for _ in range(2):
+        clf.partial_fit(X)
+    y_scores2 = clf.decision_function(T)
+
+    assert clf.t_ == t
+    assert_allclose(y_scores, y_scores2)
+    assert_allclose(clf.coef_, coef)
+    assert_allclose(clf.offset_, offset)
+
+
+@pytest.mark.parametrize("klass", [SGDOneClassSVM, SparseSGDOneClassSVM])
+def test_late_onset_averaging_reached_oneclass(klass):
+    # Test average
+    eta0 = 0.001
+    nu = 0.05
+
+    # 2 passes over the training set but average only at second pass
+    clf1 = klass(
+        average=7, learning_rate="constant", eta0=eta0, nu=nu, max_iter=2, shuffle=False
+    )
+    # 1 pass over the training set with no averaging
+    clf2 = klass(
+        average=False,
+        learning_rate="constant",
+        eta0=eta0,
+        nu=nu,
+        max_iter=1,
+        shuffle=False,
+    )
+
+    clf1.fit(X)
+    clf2.fit(X)
+
+    # Start from clf2 solution, compute averaging using asgd function and
+    # compare with clf1 solution
+    average_coef, average_offset = asgd_oneclass(
+        klass, X, eta0, nu, coef_init=clf2.coef_.ravel(), offset_init=clf2.offset_
+    )
+
+    assert_allclose(clf1.coef_.ravel(), average_coef.ravel())
+    assert_allclose(clf1.offset_, average_offset)
+
+
+@pytest.mark.parametrize("klass", [SGDOneClassSVM, SparseSGDOneClassSVM])
+def test_sgd_averaged_computed_correctly_oneclass(klass):
+    # Tests the average SGD One-Class SVM matches the naive implementation
+    eta = 0.001
+    nu = 0.05
+    n_samples = 20
+    n_features = 10
+    rng = np.random.RandomState(0)
+    X = rng.normal(size=(n_samples, n_features))
+
+    clf = klass(
+        learning_rate="constant",
+        eta0=eta,
+        nu=nu,
+        fit_intercept=True,
+        max_iter=1,
+        average=True,
+        shuffle=False,
+    )
+
+    clf.fit(X)
+    average_coef, average_offset = asgd_oneclass(klass, X, eta, nu)
+
+    assert_allclose(clf.coef_, average_coef)
+    assert_allclose(clf.offset_, average_offset)
+
+
+@pytest.mark.parametrize("klass", [SGDOneClassSVM, SparseSGDOneClassSVM])
+def test_sgd_averaged_partial_fit_oneclass(klass):
+    # Tests whether the partial fit yields the same average as the fit
+    eta = 0.001
+    nu = 0.05
+    n_samples = 20
+    n_features = 10
+    rng = np.random.RandomState(0)
+    X = rng.normal(size=(n_samples, n_features))
+
+    clf = klass(
+        learning_rate="constant",
+        eta0=eta,
+        nu=nu,
+        fit_intercept=True,
+        max_iter=1,
+        average=True,
+        shuffle=False,
+    )
+
+    clf.partial_fit(X[: int(n_samples / 2)][:])
+    clf.partial_fit(X[int(n_samples / 2) :][:])
+    average_coef, average_offset = asgd_oneclass(klass, X, eta, nu)
+
+    assert_allclose(clf.coef_, average_coef)
+    assert_allclose(clf.offset_, average_offset)
+
+
+@pytest.mark.parametrize("klass", [SGDOneClassSVM, SparseSGDOneClassSVM])
+def test_average_sparse_oneclass(klass):
+    # Checks the average coef on data with 0s
+    eta = 0.001
+    nu = 0.01
+    clf = klass(
+        learning_rate="constant",
+        eta0=eta,
+        nu=nu,
+        fit_intercept=True,
+        max_iter=1,
+        average=True,
+        shuffle=False,
+    )
+
+    n_samples = X3.shape[0]
+
+    clf.partial_fit(X3[: int(n_samples / 2)])
+    clf.partial_fit(X3[int(n_samples / 2) :])
+    average_coef, average_offset = asgd_oneclass(klass, X3, eta, nu)
+
+    assert_allclose(clf.coef_, average_coef)
+    assert_allclose(clf.offset_, average_offset)
+
+
+def test_sgd_oneclass():
+    # Test fit, decision_function, predict and score_samples on a toy
+    # dataset
+    X_train = np.array([[-2, -1], [-1, -1], [1, 1]])
+    X_test = np.array([[0.5, -2], [2, 2]])
+    clf = SGDOneClassSVM(
+        nu=0.5, eta0=1, learning_rate="constant", shuffle=False, max_iter=1
+    )
+    clf.fit(X_train)
+    assert_allclose(clf.coef_, np.array([-0.125, 0.4375]))
+    assert clf.offset_[0] == -0.5
+
+    scores = clf.score_samples(X_test)
+    assert_allclose(scores, np.array([-0.9375, 0.625]))
+
+    dec = clf.score_samples(X_test) - clf.offset_
+    assert_allclose(clf.decision_function(X_test), dec)
+
+    pred = clf.predict(X_test)
+    assert_array_equal(pred, np.array([-1, 1]))
+
+
+def test_ocsvm_vs_sgdocsvm():
+    # Checks SGDOneClass SVM gives a good approximation of kernelized
+    # One-Class SVM
+    nu = 0.05
+    gamma = 2.0
+    random_state = 42
+
+    # Generate train and test data
+    rng = np.random.RandomState(random_state)
+    X = 0.3 * rng.randn(500, 2)
+    X_train = np.r_[X + 2, X - 2]
+    X = 0.3 * rng.randn(100, 2)
+    X_test = np.r_[X + 2, X - 2]
+
+    # One-Class SVM
+    clf = OneClassSVM(gamma=gamma, kernel="rbf", nu=nu)
+    clf.fit(X_train)
+    y_pred_ocsvm = clf.predict(X_test)
+    dec_ocsvm = clf.decision_function(X_test).reshape(1, -1)
+
+    # SGDOneClassSVM using kernel approximation
+    max_iter = 15
+    transform = Nystroem(gamma=gamma, random_state=random_state)
+    clf_sgd = SGDOneClassSVM(
+        nu=nu,
+        shuffle=True,
+        fit_intercept=True,
+        max_iter=max_iter,
+        random_state=random_state,
+        tol=None,
+    )
+    pipe_sgd = make_pipeline(transform, clf_sgd)
+    pipe_sgd.fit(X_train)
+    y_pred_sgdocsvm = pipe_sgd.predict(X_test)
+    dec_sgdocsvm = pipe_sgd.decision_function(X_test).reshape(1, -1)
+
+    assert np.mean(y_pred_sgdocsvm == y_pred_ocsvm) >= 0.99
+    corrcoef = np.corrcoef(np.concatenate((dec_ocsvm, dec_sgdocsvm)))[0, 1]
+    assert corrcoef >= 0.9
 
 
 def test_l1_ratio():
     # Test if l1 ratio extremes match L1 and L2 penalty settings.
-    X, y = datasets.make_classification(n_samples=1000,
-                                        n_features=100, n_informative=20,
-                                        random_state=1234)
+    X, y = datasets.make_classification(
+        n_samples=1000, n_features=100, n_informative=20, random_state=1234
+    )
 
     # test if elasticnet with l1_ratio near 1 gives same result as pure l1
-    est_en = SGDClassifier(alpha=0.001, penalty='elasticnet', tol=None,
-                           max_iter=6, l1_ratio=0.9999999999,
-                           random_state=42).fit(X, y)
-    est_l1 = SGDClassifier(alpha=0.001, penalty='l1', max_iter=6,
-                           random_state=42, tol=None).fit(X, y)
+    est_en = SGDClassifier(
+        alpha=0.001,
+        penalty="elasticnet",
+        tol=None,
+        max_iter=6,
+        l1_ratio=0.9999999999,
+        random_state=42,
+    ).fit(X, y)
+    est_l1 = SGDClassifier(
+        alpha=0.001, penalty="l1", max_iter=6, random_state=42, tol=None
+    ).fit(X, y)
     assert_array_almost_equal(est_en.coef_, est_l1.coef_)
 
     # test if elasticnet with l1_ratio near 0 gives same result as pure l2
-    est_en = SGDClassifier(alpha=0.001, penalty='elasticnet', tol=None,
-                           max_iter=6, l1_ratio=0.0000000001,
-                           random_state=42).fit(X, y)
-    est_l2 = SGDClassifier(alpha=0.001, penalty='l2', max_iter=6,
-                           random_state=42, tol=None).fit(X, y)
+    est_en = SGDClassifier(
+        alpha=0.001,
+        penalty="elasticnet",
+        tol=None,
+        max_iter=6,
+        l1_ratio=0.0000000001,
+        random_state=42,
+    ).fit(X, y)
+    est_l2 = SGDClassifier(
+        alpha=0.001, penalty="l2", max_iter=6, random_state=42, tol=None
+    ).fit(X, y)
     assert_array_almost_equal(est_en.coef_, est_l2.coef_)
 
 
 def test_underflow_or_overlow():
-    with np.errstate(all='raise'):
+    with np.errstate(all="raise"):
         # Generate some weird data with hugely unscaled features
         rng = np.random.RandomState(0)
         n_samples = 100
@@ -1380,41 +1784,58 @@ def test_underflow_or_overlow():
 
         # Define a ground truth on the scaled data
         ground_truth = rng.normal(size=n_features)
-        y = (np.dot(X_scaled, ground_truth) > 0.).astype(np.int32)
+        y = (np.dot(X_scaled, ground_truth) > 0.0).astype(np.int32)
         assert_array_equal(np.unique(y), [0, 1])
 
-        model = SGDClassifier(alpha=0.1, loss='squared_hinge', max_iter=500)
+        model = SGDClassifier(alpha=0.1, loss="squared_hinge", max_iter=500)
 
         # smoke test: model is stable on scaled data
         model.fit(X_scaled, y)
         assert np.isfinite(model.coef_).all()
 
         # model is numerically unstable on unscaled data
-        msg_regxp = (r"Floating-point under-/overflow occurred at epoch #.*"
-                     " Scaling input data with StandardScaler or MinMaxScaler"
-                     " might help.")
-        assert_raises_regexp(ValueError, msg_regxp, model.fit, X, y)
+        msg_regxp = (
+            r"Floating-point under-/overflow occurred at epoch #.*"
+            " Scaling input data with StandardScaler or MinMaxScaler"
+            " might help."
+        )
+        with pytest.raises(ValueError, match=msg_regxp):
+            model.fit(X, y)
 
 
 def test_numerical_stability_large_gradient():
     # Non regression test case for numerical stability on scaled problems
     # where the gradient can still explode with some losses
-    model = SGDClassifier(loss='squared_hinge', max_iter=10, shuffle=True,
-                          penalty='elasticnet', l1_ratio=0.3, alpha=0.01,
-                          eta0=0.001, random_state=0, tol=None)
-    with np.errstate(all='raise'):
+    model = SGDClassifier(
+        loss="squared_hinge",
+        max_iter=10,
+        shuffle=True,
+        penalty="elasticnet",
+        l1_ratio=0.3,
+        alpha=0.01,
+        eta0=0.001,
+        random_state=0,
+        tol=None,
+    )
+    with np.errstate(all="raise"):
         model.fit(iris.data, iris.target)
     assert np.isfinite(model.coef_).all()
 
 
-@pytest.mark.parametrize('penalty', ['l2', 'l1', 'elasticnet'])
+@pytest.mark.parametrize("penalty", ["l2", "l1", "elasticnet"])
 def test_large_regularization(penalty):
     # Non regression tests for numerical stability issues caused by large
     # regularization parameters
-    model = SGDClassifier(alpha=1e5, learning_rate='constant', eta0=0.1,
-                          penalty=penalty, shuffle=False,
-                          tol=None, max_iter=6)
-    with np.errstate(all='raise'):
+    model = SGDClassifier(
+        alpha=1e5,
+        learning_rate="constant",
+        eta0=0.1,
+        penalty=penalty,
+        shuffle=False,
+        tol=None,
+        max_iter=6,
+    )
+    with np.errstate(all="raise"):
         model.fit(iris.data, iris.target)
     assert_array_almost_equal(model.coef_, np.zeros_like(model.coef_))
 
@@ -1469,9 +1890,14 @@ def test_loss_hinge():
     loss = sgd_fast.Hinge(1.0)
     cases = [
         # (p, y, expected_loss, expected_dloss)
-        (1.1, 1.0, 0.0, 0.0), (-2.0, -1.0, 0.0, 0.0),
-        (1.0, 1.0, 0.0, -1.0), (-1.0, -1.0, 0.0, 1.0), (0.5, 1.0, 0.5, -1.0),
-        (2.0, -1.0, 3.0, 1.0), (-0.5, -1.0, 0.5, 1.0), (0.0, 1.0, 1, -1.0)
+        (1.1, 1.0, 0.0, 0.0),
+        (-2.0, -1.0, 0.0, 0.0),
+        (1.0, 1.0, 0.0, -1.0),
+        (-1.0, -1.0, 0.0, 1.0),
+        (0.5, 1.0, 0.5, -1.0),
+        (2.0, -1.0, 3.0, 1.0),
+        (-0.5, -1.0, 0.5, 1.0),
+        (0.0, 1.0, 1, -1.0),
     ]
     _test_loss_common(loss, cases)
 
@@ -1479,9 +1905,14 @@ def test_loss_hinge():
     loss = sgd_fast.Hinge(0.0)
     cases = [
         # (p, y, expected_loss, expected_dloss)
-        (1.0, 1.0, 0.0, 0.0), (-0.1, -1.0, 0.0, 0.0),
-        (0.0, 1.0, 0.0, -1.0), (0.0, -1.0, 0.0, 1.0), (0.5, -1.0, 0.5, 1.0),
-        (2.0, -1.0, 2.0, 1.0), (-0.5, 1.0, 0.5, -1.0), (-1.0, 1.0, 1.0, -1.0),
+        (1.0, 1.0, 0.0, 0.0),
+        (-0.1, -1.0, 0.0, 0.0),
+        (0.0, 1.0, 0.0, -1.0),
+        (0.0, -1.0, 0.0, 1.0),
+        (0.5, -1.0, 0.5, 1.0),
+        (2.0, -1.0, 2.0, 1.0),
+        (-0.5, 1.0, 0.5, -1.0),
+        (-1.0, 1.0, 1.0, -1.0),
     ]
     _test_loss_common(loss, cases)
 
@@ -1491,50 +1922,12 @@ def test_gradient_squared_hinge():
     loss = sgd_fast.SquaredHinge(1.0)
     cases = [
         # (p, y, expected_loss, expected_dloss)
-        (1.0, 1.0, 0.0, 0.0), (-2.0, -1.0, 0.0, 0.0), (1.0, -1.0, 4.0, 4.0),
-        (-1.0, 1.0, 4.0, -4.0), (0.5, 1.0, 0.25, -1.0), (0.5, -1.0, 2.25, 3.0)
-    ]
-    _test_loss_common(loss, cases)
-
-
-def test_loss_log():
-    # Test Log (logistic loss)
-    loss = sgd_fast.Log()
-    cases = [
-        # (p, y, expected_loss, expected_dloss)
-        (1.0, 1.0, np.log(1.0 + np.exp(-1.0)), -1.0 / (np.exp(1.0) + 1.0)),
-        (1.0, -1.0, np.log(1.0 + np.exp(1.0)), 1.0 / (np.exp(-1.0) + 1.0)),
-        (-1.0, -1.0, np.log(1.0 + np.exp(-1.0)), 1.0 / (np.exp(1.0) + 1.0)),
-        (-1.0, 1.0, np.log(1.0 + np.exp(1.0)), -1.0 / (np.exp(-1.0) + 1.0)),
-        (0.0, 1.0, np.log(2), -0.5), (0.0, -1.0, np.log(2), 0.5),
-        (17.9, -1.0, 17.9, 1.0), (-17.9, 1.0, 17.9, -1.0),
-    ]
-    _test_loss_common(loss, cases)
-    assert_almost_equal(loss.py_dloss(18.1, 1.0), np.exp(-18.1) * -1.0, 16)
-    assert_almost_equal(loss.py_loss(18.1, 1.0), np.exp(-18.1), 16)
-    assert_almost_equal(loss.py_dloss(-18.1, -1.0), np.exp(-18.1) * 1.0, 16)
-    assert_almost_equal(loss.py_loss(-18.1, 1.0), 18.1, 16)
-
-
-def test_loss_squared_loss():
-    # Test SquaredLoss
-    loss = sgd_fast.SquaredLoss()
-    cases = [
-        # (p, y, expected_loss, expected_dloss)
-        (0.0, 0.0, 0.0, 0.0), (1.0, 1.0, 0.0, 0.0), (1.0, 0.0, 0.5, 1.0),
-        (0.5, -1.0, 1.125, 1.5), (-2.5, 2.0, 10.125, -4.5)
-    ]
-    _test_loss_common(loss, cases)
-
-
-def test_loss_huber():
-    # Test Huber
-    loss = sgd_fast.Huber(0.1)
-    cases = [
-        # (p, y, expected_loss, expected_dloss)
-        (0.0, 0.0, 0.0, 0.0), (0.1, 0.0, 0.005, 0.1), (0.0, 0.1, 0.005, -0.1),
-        (3.95, 4.0, 0.00125, -0.05), (5.0, 2.0, 0.295, 0.1),
-        (-1.0, 5.0, 0.595, -0.1)
+        (1.0, 1.0, 0.0, 0.0),
+        (-2.0, -1.0, 0.0, 0.0),
+        (1.0, -1.0, 4.0, 4.0),
+        (-1.0, 1.0, 4.0, -4.0),
+        (0.5, 1.0, 0.25, -1.0),
+        (0.5, -1.0, 2.25, 3.0),
     ]
     _test_loss_common(loss, cases)
 
@@ -1544,9 +1937,14 @@ def test_loss_modified_huber():
     loss = sgd_fast.ModifiedHuber()
     cases = [
         # (p, y, expected_loss, expected_dloss)
-        (1.0, 1.0, 0.0, 0.0), (-1.0, -1.0, 0.0, 0.0), (2.0, 1.0, 0.0, 0.0),
-        (0.0, 1.0, 1.0, -2.0), (-1.0, 1.0, 4.0, -4.0), (0.5, -1.0, 2.25, 3.0),
-        (-2.0, 1.0, 8, -4.0), (-3.0, 1.0, 12, -4.0)
+        (1.0, 1.0, 0.0, 0.0),
+        (-1.0, -1.0, 0.0, 0.0),
+        (2.0, 1.0, 0.0, 0.0),
+        (0.0, 1.0, 1.0, -2.0),
+        (-1.0, 1.0, 4.0, -4.0),
+        (0.5, -1.0, 2.25, 3.0),
+        (-2.0, 1.0, 8, -4.0),
+        (-3.0, 1.0, 12, -4.0),
     ]
     _test_loss_common(loss, cases)
 
@@ -1556,9 +1954,14 @@ def test_loss_epsilon_insensitive():
     loss = sgd_fast.EpsilonInsensitive(0.1)
     cases = [
         # (p, y, expected_loss, expected_dloss)
-        (0.0, 0.0, 0.0, 0.0), (0.1, 0.0, 0.0, 0.0), (-2.05, -2.0, 0.0, 0.0),
-        (3.05, 3.0, 0.0, 0.0), (2.2, 2.0, 0.1, 1.0), (2.0, -1.0, 2.9, 1.0),
-        (2.0, 2.2, 0.1, -1.0), (-2.0, 1.0, 2.9, -1.0)
+        (0.0, 0.0, 0.0, 0.0),
+        (0.1, 0.0, 0.0, 0.0),
+        (-2.05, -2.0, 0.0, 0.0),
+        (3.05, 3.0, 0.0, 0.0),
+        (2.2, 2.0, 0.1, 1.0),
+        (2.0, -1.0, 2.9, 1.0),
+        (2.0, 2.2, 0.1, -1.0),
+        (-2.0, 1.0, 2.9, -1.0),
     ]
     _test_loss_common(loss, cases)
 
@@ -1568,9 +1971,14 @@ def test_loss_squared_epsilon_insensitive():
     loss = sgd_fast.SquaredEpsilonInsensitive(0.1)
     cases = [
         # (p, y, expected_loss, expected_dloss)
-        (0.0, 0.0, 0.0, 0.0), (0.1, 0.0, 0.0, 0.0), (-2.05, -2.0, 0.0, 0.0),
-        (3.05, 3.0, 0.0, 0.0), (2.2, 2.0, 0.01, 0.2), (2.0, -1.0, 8.41, 5.8),
-        (2.0, 2.2, 0.01, -0.2), (-2.0, 1.0, 8.41, -5.8)
+        (0.0, 0.0, 0.0, 0.0),
+        (0.1, 0.0, 0.0, 0.0),
+        (-2.05, -2.0, 0.0, 0.0),
+        (3.05, 3.0, 0.0, 0.0),
+        (2.2, 2.0, 0.01, 0.2),
+        (2.0, -1.0, 8.41, 5.8),
+        (2.0, 2.2, 0.01, -0.2),
+        (-2.0, 1.0, 8.41, -5.8),
     ]
     _test_loss_common(loss, cases)
 
@@ -1578,9 +1986,15 @@ def test_loss_squared_epsilon_insensitive():
 def test_multi_thread_multi_class_and_early_stopping():
     # This is a non-regression test for a bad interaction between
     # early stopping internal attribute and thread-based parallelism.
-    clf = SGDClassifier(alpha=1e-3, tol=1e-3, max_iter=1000,
-                        early_stopping=True, n_iter_no_change=100,
-                        random_state=0, n_jobs=2)
+    clf = SGDClassifier(
+        alpha=1e-3,
+        tol=1e-3,
+        max_iter=1000,
+        early_stopping=True,
+        n_iter_no_change=100,
+        random_state=0,
+        n_jobs=2,
+    )
     clf.fit(iris.data, iris.target)
     assert clf.n_iter_ > clf.n_iter_no_change
     assert clf.n_iter_ < clf.n_iter_no_change + 20
@@ -1592,20 +2006,17 @@ def test_multi_core_gridsearch_and_early_stopping():
     # early stopping internal attribute and process-based multi-core
     # parallelism.
     param_grid = {
-        'alpha': np.logspace(-4, 4, 9),
-        'n_iter_no_change': [5, 10, 50],
+        "alpha": np.logspace(-4, 4, 9),
+        "n_iter_no_change": [5, 10, 50],
     }
 
-    clf = SGDClassifier(tol=1e-2, max_iter=1000, early_stopping=True,
-                        random_state=0)
-    search = RandomizedSearchCV(clf, param_grid, n_iter=3, n_jobs=2,
-                                random_state=0)
+    clf = SGDClassifier(tol=1e-2, max_iter=1000, early_stopping=True, random_state=0)
+    search = RandomizedSearchCV(clf, param_grid, n_iter=5, n_jobs=2, random_state=0)
     search.fit(iris.data, iris.target)
     assert search.best_score_ > 0.8
 
 
-@pytest.mark.parametrize("backend",
-                         ["loky", "multiprocessing", "threading"])
+@pytest.mark.parametrize("backend", ["loky", "multiprocessing", "threading"])
 def test_SGDClassifier_fit_for_all_backends(backend):
     # This is a non-regression smoke test. In the multi-class case,
     # SGDClassifier.fit fits each class in a one-versus-all fashion using
@@ -1621,28 +2032,164 @@ def test_SGDClassifier_fit_for_all_backends(backend):
     # a segmentation fault when trying to write in a readonly memory mapped
     # buffer.
 
-    if (parse_version(joblib.__version__) < parse_version('0.12')
-            and backend == 'loky'):
-        pytest.skip('loky backend does not exist in joblib <0.12')
-
     random_state = np.random.RandomState(42)
 
     # Create a classification problem with 50000 features and 20 classes. Using
     # loky or multiprocessing this make the clf.coef_ exceed the threshold
     # above which memmaping is used in joblib and loky (1MB as of 2018/11/1).
-    X = sp.random(500, 2000, density=0.02, format='csr',
-                  random_state=random_state)
+    X = sp.random(500, 2000, density=0.02, format="csr", random_state=random_state)
     y = random_state.choice(20, 500)
 
     # Begin by fitting a SGD classifier sequentially
-    clf_sequential = SGDClassifier(max_iter=1000, n_jobs=1,
-                                   random_state=42)
+    clf_sequential = SGDClassifier(max_iter=1000, n_jobs=1, random_state=42)
     clf_sequential.fit(X, y)
 
     # Fit a SGDClassifier using the specified backend, and make sure the
     # coefficients are equal to those obtained using a sequential fit
-    clf_parallel = SGDClassifier(max_iter=1000, n_jobs=4,
-                                 random_state=42)
+    clf_parallel = SGDClassifier(max_iter=1000, n_jobs=4, random_state=42)
     with joblib.parallel_backend(backend=backend):
         clf_parallel.fit(X, y)
     assert_array_almost_equal(clf_sequential.coef_, clf_parallel.coef_)
+
+
+@pytest.mark.parametrize(
+    "Estimator", [linear_model.SGDClassifier, linear_model.SGDRegressor]
+)
+def test_sgd_random_state(Estimator, global_random_seed):
+    # Train the same model on the same data without converging and check that we
+    # get reproducible results by fixing the random seed.
+    if Estimator == linear_model.SGDRegressor:
+        X, y = datasets.make_regression(random_state=global_random_seed)
+    else:
+        X, y = datasets.make_classification(random_state=global_random_seed)
+
+    # Fitting twice a model with the same hyper-parameters on the same training
+    # set with the same seed leads to the same results deterministically.
+
+    est = Estimator(random_state=global_random_seed, max_iter=1)
+    with pytest.warns(ConvergenceWarning):
+        coef_same_seed_a = est.fit(X, y).coef_
+        assert est.n_iter_ == 1
+
+    est = Estimator(random_state=global_random_seed, max_iter=1)
+    with pytest.warns(ConvergenceWarning):
+        coef_same_seed_b = est.fit(X, y).coef_
+        assert est.n_iter_ == 1
+
+    assert_allclose(coef_same_seed_a, coef_same_seed_b)
+
+    # Fitting twice a model with the same hyper-parameters on the same training
+    # set but with different random seed leads to different results after one
+    # epoch because of the random shuffling of the dataset.
+
+    est = Estimator(random_state=global_random_seed + 1, max_iter=1)
+    with pytest.warns(ConvergenceWarning):
+        coef_other_seed = est.fit(X, y).coef_
+        assert est.n_iter_ == 1
+
+    assert np.abs(coef_same_seed_a - coef_other_seed).max() > 1.0
+
+
+def test_validation_mask_correctly_subsets(monkeypatch):
+    """Test that data passed to validation callback correctly subsets.
+
+    Non-regression test for #23255.
+    """
+    X, Y = iris.data, iris.target
+    n_samples = X.shape[0]
+    validation_fraction = 0.2
+    clf = linear_model.SGDClassifier(
+        early_stopping=True,
+        tol=1e-3,
+        max_iter=1000,
+        validation_fraction=validation_fraction,
+    )
+
+    mock = Mock(side_effect=_stochastic_gradient._ValidationScoreCallback)
+    monkeypatch.setattr(_stochastic_gradient, "_ValidationScoreCallback", mock)
+    clf.fit(X, Y)
+
+    X_val, y_val = mock.call_args[0][1:3]
+    assert X_val.shape[0] == int(n_samples * validation_fraction)
+    assert y_val.shape[0] == int(n_samples * validation_fraction)
+
+
+def test_sgd_error_on_zero_validation_weight():
+    # Test that SGDClassifier raises error when all the validation samples
+    # have zero sample_weight. Non-regression test for #17229.
+    X, Y = iris.data, iris.target
+    sample_weight = np.zeros_like(Y)
+    validation_fraction = 0.4
+
+    clf = linear_model.SGDClassifier(
+        early_stopping=True, validation_fraction=validation_fraction, random_state=0
+    )
+
+    error_message = (
+        "The sample weights for validation set are all zero, consider using a"
+        " different random state."
+    )
+    with pytest.raises(ValueError, match=error_message):
+        clf.fit(X, Y, sample_weight=sample_weight)
+
+
+@pytest.mark.parametrize("Estimator", [SGDClassifier, SGDRegressor])
+def test_sgd_verbose(Estimator):
+    """non-regression test for gh #25249"""
+    Estimator(verbose=1).fit(X, Y)
+
+
+@pytest.mark.parametrize(
+    "SGDEstimator",
+    [
+        SGDClassifier,
+        SparseSGDClassifier,
+        SGDRegressor,
+        SparseSGDRegressor,
+        SGDOneClassSVM,
+        SparseSGDOneClassSVM,
+    ],
+)
+@pytest.mark.parametrize("data_type", (np.float32, np.float64))
+def test_sgd_dtype_match(SGDEstimator, data_type):
+    _X = X.astype(data_type)
+    _Y = np.array(Y, dtype=data_type)
+    sgd_model = SGDEstimator()
+    sgd_model.fit(_X, _Y)
+    assert sgd_model.coef_.dtype == data_type
+
+
+@pytest.mark.parametrize(
+    "SGDEstimator",
+    [
+        SGDClassifier,
+        SparseSGDClassifier,
+        SGDRegressor,
+        SparseSGDRegressor,
+        SGDOneClassSVM,
+        SparseSGDOneClassSVM,
+    ],
+)
+def test_sgd_numerical_consistency(SGDEstimator):
+    X_64 = X.astype(dtype=np.float64)
+    Y_64 = np.array(Y, dtype=np.float64)
+
+    X_32 = X.astype(dtype=np.float32)
+    Y_32 = np.array(Y, dtype=np.float32)
+
+    sgd_64 = SGDEstimator(max_iter=20)
+    sgd_64.fit(X_64, Y_64)
+
+    sgd_32 = SGDEstimator(max_iter=20)
+    sgd_32.fit(X_32, Y_32)
+
+    assert_allclose(sgd_64.coef_, sgd_32.coef_)
+
+
+def test_sgd_one_class_svm_estimator_type():
+    """Check that SGDOneClassSVM has the correct estimator type.
+
+    Non-regression test for if the mixin was not on the left.
+    """
+    sgd_ocsvm = SGDOneClassSVM()
+    assert get_tags(sgd_ocsvm).estimator_type == "outlier_detector"
diff --git a/sklearn/linear_model/tests/test_sparse_coordinate_descent.py b/sklearn/linear_model/tests/test_sparse_coordinate_descent.py
index 23b57a699a655..1aab9babeeb40 100644
--- a/sklearn/linear_model/tests/test_sparse_coordinate_descent.py
+++ b/sklearn/linear_model/tests/test_sparse_coordinate_descent.py
@@ -1,14 +1,18 @@
 import numpy as np
 import pytest
 import scipy.sparse as sp
+from numpy.testing import assert_allclose
 
-from sklearn.utils._testing import assert_array_almost_equal
-from sklearn.utils._testing import assert_almost_equal
-
-from sklearn.utils._testing import ignore_warnings
+from sklearn.datasets import make_regression
 from sklearn.exceptions import ConvergenceWarning
-
-from sklearn.linear_model import Lasso, ElasticNet, LassoCV, ElasticNetCV
+from sklearn.linear_model import ElasticNet, ElasticNetCV, Lasso, LassoCV
+from sklearn.utils._testing import (
+    assert_almost_equal,
+    assert_array_almost_equal,
+    create_memmap_backed_data,
+    ignore_warnings,
+)
+from sklearn.utils.fixes import COO_CONTAINERS, CSC_CONTAINERS, LIL_CONTAINERS
 
 
 def test_sparse_coef():
@@ -16,80 +20,75 @@ def test_sparse_coef():
     clf = ElasticNet()
     clf.coef_ = [1, 2, 3]
 
-    assert sp.isspmatrix(clf.sparse_coef_)
+    assert sp.issparse(clf.sparse_coef_)
     assert clf.sparse_coef_.toarray().tolist()[0] == clf.coef_
 
 
-def test_normalize_option():
-    # Check that the normalize option in enet works
-    X = sp.csc_matrix([[-1], [0], [1]])
-    y = [-1, 0, 1]
-    clf_dense = ElasticNet(normalize=True)
-    clf_sparse = ElasticNet(normalize=True)
-    clf_dense.fit(X, y)
-    X = sp.csc_matrix(X)
-    clf_sparse.fit(X, y)
-    assert_almost_equal(clf_dense.dual_gap_, 0)
-    assert_array_almost_equal(clf_dense.coef_, clf_sparse.coef_)
-
-
-def test_lasso_zero():
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+def test_lasso_zero(csc_container):
     # Check that the sparse lasso can handle zero data without crashing
-    X = sp.csc_matrix((3, 1))
+    X = csc_container((3, 1))
     y = [0, 0, 0]
     T = np.array([[1], [2], [3]])
     clf = Lasso().fit(X, y)
     pred = clf.predict(T)
     assert_array_almost_equal(clf.coef_, [0])
     assert_array_almost_equal(pred, [0, 0, 0])
-    assert_almost_equal(clf.dual_gap_,  0)
+    assert_almost_equal(clf.dual_gap_, 0)
 
 
-def test_enet_toy_list_input():
+@pytest.mark.parametrize("with_sample_weight", [True, False])
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+def test_enet_toy_list_input(with_sample_weight, csc_container):
     # Test ElasticNet for various values of alpha and l1_ratio with list X
 
     X = np.array([[-1], [0], [1]])
-    X = sp.csc_matrix(X)
-    Y = [-1, 0, 1]       # just a straight line
+    X = csc_container(X)
+    Y = [-1, 0, 1]  # just a straight line
     T = np.array([[2], [3], [4]])  # test sample
+    if with_sample_weight:
+        sw = np.array([2.0, 2, 2])
+    else:
+        sw = None
 
     # this should be the same as unregularized least squares
     clf = ElasticNet(alpha=0, l1_ratio=1.0)
     # catch warning about alpha=0.
     # this is discouraged but should work.
-    ignore_warnings(clf.fit)(X, Y)
+    ignore_warnings(clf.fit)(X, Y, sample_weight=sw)
     pred = clf.predict(T)
     assert_array_almost_equal(clf.coef_, [1])
     assert_array_almost_equal(pred, [2, 3, 4])
     assert_almost_equal(clf.dual_gap_, 0)
 
-    clf = ElasticNet(alpha=0.5, l1_ratio=0.3, max_iter=1000)
-    clf.fit(X, Y)
+    clf = ElasticNet(alpha=0.5, l1_ratio=0.3)
+    clf.fit(X, Y, sample_weight=sw)
     pred = clf.predict(T)
     assert_array_almost_equal(clf.coef_, [0.50819], decimal=3)
-    assert_array_almost_equal(pred, [1.0163,  1.5245,  2.0327], decimal=3)
+    assert_array_almost_equal(pred, [1.0163, 1.5245, 2.0327], decimal=3)
     assert_almost_equal(clf.dual_gap_, 0)
 
     clf = ElasticNet(alpha=0.5, l1_ratio=0.5)
-    clf.fit(X, Y)
+    clf.fit(X, Y, sample_weight=sw)
     pred = clf.predict(T)
     assert_array_almost_equal(clf.coef_, [0.45454], 3)
-    assert_array_almost_equal(pred, [0.9090,  1.3636,  1.8181], 3)
+    assert_array_almost_equal(pred, [0.9090, 1.3636, 1.8181], 3)
     assert_almost_equal(clf.dual_gap_, 0)
 
 
-def test_enet_toy_explicit_sparse_input():
+@pytest.mark.parametrize("lil_container", LIL_CONTAINERS)
+def test_enet_toy_explicit_sparse_input(lil_container):
     # Test ElasticNet for various values of alpha and l1_ratio with sparse X
     f = ignore_warnings
     # training samples
-    X = sp.lil_matrix((3, 1))
+    X = lil_container((3, 1))
     X[0, 0] = -1
     # X[1, 0] = 0
     X[2, 0] = 1
-    Y = [-1, 0, 1]       # just a straight line (the identity function)
+    Y = [-1, 0, 1]  # just a straight line (the identity function)
 
     # test samples
-    T = sp.lil_matrix((3, 1))
+    T = lil_container((3, 1))
     T[0, 0] = 2
     T[1, 0] = 3
     T[2, 0] = 4
@@ -102,23 +101,30 @@ def test_enet_toy_explicit_sparse_input():
     assert_array_almost_equal(pred, [2, 3, 4])
     assert_almost_equal(clf.dual_gap_, 0)
 
-    clf = ElasticNet(alpha=0.5, l1_ratio=0.3, max_iter=1000)
+    clf = ElasticNet(alpha=0.5, l1_ratio=0.3)
     clf.fit(X, Y)
     pred = clf.predict(T)
     assert_array_almost_equal(clf.coef_, [0.50819], decimal=3)
-    assert_array_almost_equal(pred, [1.0163,  1.5245,  2.0327], decimal=3)
+    assert_array_almost_equal(pred, [1.0163, 1.5245, 2.0327], decimal=3)
     assert_almost_equal(clf.dual_gap_, 0)
 
     clf = ElasticNet(alpha=0.5, l1_ratio=0.5)
     clf.fit(X, Y)
     pred = clf.predict(T)
     assert_array_almost_equal(clf.coef_, [0.45454], 3)
-    assert_array_almost_equal(pred, [0.9090,  1.3636,  1.8181], 3)
+    assert_array_almost_equal(pred, [0.9090, 1.3636, 1.8181], 3)
     assert_almost_equal(clf.dual_gap_, 0)
 
 
-def make_sparse_data(n_samples=100, n_features=100, n_informative=10, seed=42,
-                     positive=False, n_targets=1):
+def make_sparse_data(
+    sparse_container,
+    n_samples=100,
+    n_features=100,
+    n_informative=10,
+    seed=42,
+    positive=False,
+    n_targets=1,
+):
     random_state = np.random.RandomState(seed)
 
     # build an ill-posed linear regression problem with many noisy features and
@@ -136,34 +142,52 @@ def make_sparse_data(n_samples=100, n_features=100, n_informative=10, seed=42,
 
     # generate training ground truth labels
     y = np.dot(X, w)
-    X = sp.csc_matrix(X)
+    X = sparse_container(X)
     if n_targets == 1:
         y = np.ravel(y)
     return X, y
 
 
-def _test_sparse_enet_not_as_toy_dataset(alpha, fit_intercept, positive):
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+@pytest.mark.parametrize(
+    "alpha, fit_intercept, positive",
+    [(0.1, False, False), (0.1, True, False), (1e-3, False, True), (1e-3, True, True)],
+)
+def test_sparse_enet_not_as_toy_dataset(csc_container, alpha, fit_intercept, positive):
     n_samples, n_features, max_iter = 100, 100, 1000
     n_informative = 10
 
-    X, y = make_sparse_data(n_samples, n_features, n_informative,
-                            positive=positive)
-
-    X_train, X_test = X[n_samples // 2:], X[:n_samples // 2]
-    y_train, y_test = y[n_samples // 2:], y[:n_samples // 2]
+    X, y = make_sparse_data(
+        csc_container, n_samples, n_features, n_informative, positive=positive
+    )
 
-    s_clf = ElasticNet(alpha=alpha, l1_ratio=0.8, fit_intercept=fit_intercept,
-                       max_iter=max_iter, tol=1e-7, positive=positive,
-                       warm_start=True)
+    X_train, X_test = X[n_samples // 2 :], X[: n_samples // 2]
+    y_train, y_test = y[n_samples // 2 :], y[: n_samples // 2]
+
+    s_clf = ElasticNet(
+        alpha=alpha,
+        l1_ratio=0.8,
+        fit_intercept=fit_intercept,
+        max_iter=max_iter,
+        tol=1e-7,
+        positive=positive,
+        warm_start=True,
+    )
     s_clf.fit(X_train, y_train)
 
     assert_almost_equal(s_clf.dual_gap_, 0, 4)
     assert s_clf.score(X_test, y_test) > 0.85
 
     # check the convergence is the same as the dense version
-    d_clf = ElasticNet(alpha=alpha, l1_ratio=0.8, fit_intercept=fit_intercept,
-                       max_iter=max_iter, tol=1e-7, positive=positive,
-                       warm_start=True)
+    d_clf = ElasticNet(
+        alpha=alpha,
+        l1_ratio=0.8,
+        fit_intercept=fit_intercept,
+        max_iter=max_iter,
+        tol=1e-7,
+        positive=positive,
+        warm_start=True,
+    )
     d_clf.fit(X_train.toarray(), y_train)
 
     assert_almost_equal(d_clf.dual_gap_, 0, 4)
@@ -176,25 +200,17 @@ def _test_sparse_enet_not_as_toy_dataset(alpha, fit_intercept, positive):
     assert np.sum(s_clf.coef_ != 0.0) < 2 * n_informative
 
 
-def test_sparse_enet_not_as_toy_dataset():
-    _test_sparse_enet_not_as_toy_dataset(alpha=0.1, fit_intercept=False,
-                                         positive=False)
-    _test_sparse_enet_not_as_toy_dataset(alpha=0.1, fit_intercept=True,
-                                         positive=False)
-    _test_sparse_enet_not_as_toy_dataset(alpha=1e-3, fit_intercept=False,
-                                         positive=True)
-    _test_sparse_enet_not_as_toy_dataset(alpha=1e-3, fit_intercept=True,
-                                         positive=True)
-
-
-def test_sparse_lasso_not_as_toy_dataset():
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+def test_sparse_lasso_not_as_toy_dataset(csc_container):
     n_samples = 100
     max_iter = 1000
     n_informative = 10
-    X, y = make_sparse_data(n_samples=n_samples, n_informative=n_informative)
+    X, y = make_sparse_data(
+        csc_container, n_samples=n_samples, n_informative=n_informative
+    )
 
-    X_train, X_test = X[n_samples // 2:], X[:n_samples // 2]
-    y_train, y_test = y[n_samples // 2:], y[:n_samples // 2]
+    X_train, X_test = X[n_samples // 2 :], X[: n_samples // 2]
+    y_train, y_test = y[n_samples // 2 :], y[: n_samples // 2]
 
     s_clf = Lasso(alpha=0.1, fit_intercept=False, max_iter=max_iter, tol=1e-7)
     s_clf.fit(X_train, y_train)
@@ -211,16 +227,19 @@ def test_sparse_lasso_not_as_toy_dataset():
     assert np.sum(s_clf.coef_ != 0.0) == n_informative
 
 
-def test_enet_multitarget():
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+def test_enet_multitarget(csc_container):
     n_targets = 3
-    X, y = make_sparse_data(n_targets=n_targets)
+    X, y = make_sparse_data(csc_container, n_targets=n_targets)
 
-    estimator = ElasticNet(alpha=0.01, precompute=None)
-    # XXX: There is a bug when precompute is not None!
+    estimator = ElasticNet(alpha=0.01, precompute=False)
+    # XXX: There is a bug when precompute is not False!
     estimator.fit(X, y)
-    coef, intercept, dual_gap = (estimator.coef_,
-                                 estimator.intercept_,
-                                 estimator.dual_gap_)
+    coef, intercept, dual_gap = (
+        estimator.coef_,
+        estimator.intercept_,
+        estimator.dual_gap_,
+    )
 
     for k in range(n_targets):
         estimator.fit(X, y[:, k])
@@ -229,12 +248,18 @@ def test_enet_multitarget():
         assert_array_almost_equal(dual_gap[k], estimator.dual_gap_)
 
 
-def test_path_parameters():
-    X, y = make_sparse_data()
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+def test_path_parameters(csc_container):
+    X, y = make_sparse_data(csc_container)
     max_iter = 50
     n_alphas = 10
-    clf = ElasticNetCV(n_alphas=n_alphas, eps=1e-3, max_iter=max_iter,
-                       l1_ratio=0.5, fit_intercept=False)
+    clf = ElasticNetCV(
+        n_alphas=n_alphas,
+        eps=1e-3,
+        max_iter=max_iter,
+        l1_ratio=0.5,
+        fit_intercept=False,
+    )
     ignore_warnings(clf.fit)(X, y)  # new params
     assert_almost_equal(0.5, clf.l1_ratio)
     assert n_alphas == clf.n_alphas
@@ -244,58 +269,97 @@ def test_path_parameters():
     assert_almost_equal(clf.mse_path_, sparse_mse_path)
 
 
-def test_same_output_sparse_dense_lasso_and_enet_cv():
-    X, y = make_sparse_data(n_samples=40, n_features=10)
-    for normalize in [True, False]:
-        clfs = ElasticNetCV(max_iter=100, normalize=normalize)
-        ignore_warnings(clfs.fit)(X, y)
-        clfd = ElasticNetCV(max_iter=100, normalize=normalize)
-        ignore_warnings(clfd.fit)(X.toarray(), y)
-        assert_almost_equal(clfs.alpha_, clfd.alpha_, 7)
-        assert_almost_equal(clfs.intercept_, clfd.intercept_, 7)
-        assert_array_almost_equal(clfs.mse_path_, clfd.mse_path_)
-        assert_array_almost_equal(clfs.alphas_, clfd.alphas_)
-
-        clfs = LassoCV(max_iter=100, cv=4, normalize=normalize)
-        ignore_warnings(clfs.fit)(X, y)
-        clfd = LassoCV(max_iter=100, cv=4, normalize=normalize)
-        ignore_warnings(clfd.fit)(X.toarray(), y)
-        assert_almost_equal(clfs.alpha_, clfd.alpha_, 7)
-        assert_almost_equal(clfs.intercept_, clfd.intercept_, 7)
-        assert_array_almost_equal(clfs.mse_path_, clfd.mse_path_)
-        assert_array_almost_equal(clfs.alphas_, clfd.alphas_)
-
-
-def test_same_multiple_output_sparse_dense():
-    for normalize in [True, False]:
-        l = ElasticNet(normalize=normalize)
-        X = [[0, 1, 2, 3, 4],
-             [0, 2, 5, 8, 11],
-             [9, 10, 11, 12, 13],
-             [10, 11, 12, 13, 14]]
-        y = [[1, 2, 3, 4, 5],
-             [1, 3, 6, 9, 12],
-             [10, 11, 12, 13, 14],
-             [11, 12, 13, 14, 15]]
-        ignore_warnings(l.fit)(X, y)
-        sample = np.array([1, 2, 3, 4, 5]).reshape(1, -1)
-        predict_dense = l.predict(sample)
-
-        l_sp = ElasticNet(normalize=normalize)
-        X_sp = sp.coo_matrix(X)
-        ignore_warnings(l_sp.fit)(X_sp, y)
-        sample_sparse = sp.coo_matrix(sample)
-        predict_sparse = l_sp.predict(sample_sparse)
-
-        assert_array_almost_equal(predict_sparse, predict_dense)
-
-
-def test_sparse_enet_coordinate_descent():
+@pytest.mark.parametrize("Model", [Lasso, ElasticNet, LassoCV, ElasticNetCV])
+@pytest.mark.parametrize("fit_intercept", [False, True])
+@pytest.mark.parametrize("n_samples, n_features", [(24, 6), (6, 24)])
+@pytest.mark.parametrize("with_sample_weight", [True, False])
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+def test_sparse_dense_equality(
+    Model, fit_intercept, n_samples, n_features, with_sample_weight, csc_container
+):
+    X, y = make_regression(
+        n_samples=n_samples,
+        n_features=n_features,
+        effective_rank=n_features // 2,
+        n_informative=n_features // 2,
+        bias=4 * fit_intercept,
+        noise=1,
+        random_state=42,
+    )
+    if with_sample_weight:
+        sw = np.abs(np.random.RandomState(42).normal(scale=10, size=y.shape))
+    else:
+        sw = None
+    Xs = csc_container(X)
+    params = {"fit_intercept": fit_intercept}
+    reg_dense = Model(**params).fit(X, y, sample_weight=sw)
+    reg_sparse = Model(**params).fit(Xs, y, sample_weight=sw)
+    if fit_intercept:
+        assert reg_sparse.intercept_ == pytest.approx(reg_dense.intercept_)
+        # balance property
+        assert np.average(reg_sparse.predict(X), weights=sw) == pytest.approx(
+            np.average(y, weights=sw)
+        )
+    assert_allclose(reg_sparse.coef_, reg_dense.coef_)
+
+
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+def test_same_output_sparse_dense_lasso_and_enet_cv(csc_container):
+    X, y = make_sparse_data(csc_container, n_samples=40, n_features=10)
+    clfs = ElasticNetCV(max_iter=100)
+    clfs.fit(X, y)
+    clfd = ElasticNetCV(max_iter=100)
+    clfd.fit(X.toarray(), y)
+    assert_almost_equal(clfs.alpha_, clfd.alpha_, 7)
+    assert_almost_equal(clfs.intercept_, clfd.intercept_, 7)
+    assert_array_almost_equal(clfs.mse_path_, clfd.mse_path_)
+    assert_array_almost_equal(clfs.alphas_, clfd.alphas_)
+
+    clfs = LassoCV(max_iter=100, cv=4)
+    clfs.fit(X, y)
+    clfd = LassoCV(max_iter=100, cv=4)
+    clfd.fit(X.toarray(), y)
+    assert_almost_equal(clfs.alpha_, clfd.alpha_, 7)
+    assert_almost_equal(clfs.intercept_, clfd.intercept_, 7)
+    assert_array_almost_equal(clfs.mse_path_, clfd.mse_path_)
+    assert_array_almost_equal(clfs.alphas_, clfd.alphas_)
+
+
+@pytest.mark.parametrize("coo_container", COO_CONTAINERS)
+def test_same_multiple_output_sparse_dense(coo_container):
+    l = ElasticNet()
+    X = [
+        [0, 1, 2, 3, 4],
+        [0, 2, 5, 8, 11],
+        [9, 10, 11, 12, 13],
+        [10, 11, 12, 13, 14],
+    ]
+    y = [
+        [1, 2, 3, 4, 5],
+        [1, 3, 6, 9, 12],
+        [10, 11, 12, 13, 14],
+        [11, 12, 13, 14, 15],
+    ]
+    l.fit(X, y)
+    sample = np.array([1, 2, 3, 4, 5]).reshape(1, -1)
+    predict_dense = l.predict(sample)
+
+    l_sp = ElasticNet()
+    X_sp = coo_container(X)
+    l_sp.fit(X_sp, y)
+    sample_sparse = coo_container(sample)
+    predict_sparse = l_sp.predict(sample_sparse)
+
+    assert_array_almost_equal(predict_sparse, predict_dense)
+
+
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+def test_sparse_enet_coordinate_descent(csc_container):
     """Test that a warning is issued if model does not converge"""
     clf = Lasso(max_iter=2)
     n_samples = 5
     n_features = 2
-    X = sp.csc_matrix((n_samples, n_features)) * 1e50
+    X = csc_container((n_samples, n_features)) * 1e50
     y = np.ones(n_samples)
     warning_message = (
         "Objective did not converge. You might want "
@@ -303,3 +367,18 @@ def test_sparse_enet_coordinate_descent():
     )
     with pytest.warns(ConvergenceWarning, match=warning_message):
         clf.fit(X, y)
+
+
+@pytest.mark.parametrize("copy_X", (True, False))
+def test_sparse_read_only_buffer(copy_X):
+    """Test that sparse coordinate descent works for read-only buffers"""
+    rng = np.random.RandomState(0)
+
+    clf = ElasticNet(alpha=0.1, copy_X=copy_X, random_state=rng)
+    X = sp.random(100, 20, format="csc", random_state=rng)
+
+    # Make X.data read-only
+    X.data = create_memmap_backed_data(X.data)
+
+    y = rng.rand(100)
+    clf.fit(X, y)
diff --git a/sklearn/linear_model/tests/test_theil_sen.py b/sklearn/linear_model/tests/test_theil_sen.py
index c670fc3979b80..216415f2ee927 100644
--- a/sklearn/linear_model/tests/test_theil_sen.py
+++ b/sklearn/linear_model/tests/test_theil_sen.py
@@ -2,29 +2,39 @@
 Testing for Theil-Sen module (sklearn.linear_model.theil_sen)
 """
 
-# Author: Florian Wilhelm 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
 import os
+import re
 import sys
 from contextlib import contextmanager
+
 import numpy as np
 import pytest
-from numpy.testing import assert_array_equal, assert_array_less
-from numpy.testing import assert_array_almost_equal
+from numpy.testing import (
+    assert_array_almost_equal,
+    assert_array_equal,
+    assert_array_less,
+)
 from scipy.linalg import norm
 from scipy.optimize import fmin_bfgs
+
 from sklearn.exceptions import ConvergenceWarning
 from sklearn.linear_model import LinearRegression, TheilSenRegressor
-from sklearn.linear_model._theil_sen import _spatial_median, _breakdown_point
-from sklearn.linear_model._theil_sen import _modified_weiszfeld_step
-from sklearn.utils._testing import assert_almost_equal, assert_raises
+from sklearn.linear_model._theil_sen import (
+    _breakdown_point,
+    _modified_weiszfeld_step,
+    _spatial_median,
+)
+from sklearn.utils._testing import assert_almost_equal
 
 
 @contextmanager
 def no_stdout_stderr():
     old_stdout = sys.stdout
     old_stderr = sys.stderr
-    with open(os.devnull, 'w') as devnull:
+    with open(os.devnull, "w") as devnull:
         sys.stdout = devnull
         sys.stderr = devnull
         yield
@@ -36,9 +46,9 @@ def no_stdout_stderr():
 def gen_toy_problem_1d(intercept=True):
     random_state = np.random.RandomState(0)
     # Linear model y = 3*x + N(2, 0.1**2)
-    w = 3.
+    w = 3.0
     if intercept:
-        c = 2.
+        c = 2.0
         n_samples = 50
     else:
         c = 0.1
@@ -66,8 +76,8 @@ def gen_toy_problem_2d():
     n_samples = 100
     # Linear model y = 5*x_1 + 10*x_2 + N(1, 0.1**2)
     X = random_state.normal(size=(n_samples, 2))
-    w = np.array([5., 10.])
-    c = 1.
+    w = np.array([5.0, 10.0])
+    c = 1.0
     noise = 0.1 * random_state.normal(size=n_samples)
     y = np.dot(X, w) + c + noise
     # Add some outliers
@@ -82,8 +92,8 @@ def gen_toy_problem_4d():
     n_samples = 10000
     # Linear model y = 5*x_1 + 10*x_2  + 42*x_3 + 7*x_4 + N(1, 0.1**2)
     X = random_state.normal(size=(n_samples, 4))
-    w = np.array([5., 10., 42., 7.])
-    c = 1.
+    w = np.array([5.0, 10.0, 42.0, 7.0])
+    c = 1.0
     noise = 0.1 * random_state.normal(size=n_samples)
     y = np.dot(X, w) + c + noise
     # Add some outliers
@@ -94,9 +104,9 @@ def gen_toy_problem_4d():
 
 
 def test_modweiszfeld_step_1d():
-    X = np.array([1., 2., 3.]).reshape(3, 1)
+    X = np.array([1.0, 2.0, 3.0]).reshape(3, 1)
     # Check startvalue is element of X and solution
-    median = 2.
+    median = 2.0
     new_y = _modified_weiszfeld_step(X, median)
     assert_array_almost_equal(new_y, median)
     # Check startvalue is not the solution
@@ -105,19 +115,19 @@ def test_modweiszfeld_step_1d():
     assert_array_less(median, new_y)
     assert_array_less(new_y, y)
     # Check startvalue is not the solution but element of X
-    y = 3.
+    y = 3.0
     new_y = _modified_weiszfeld_step(X, y)
     assert_array_less(median, new_y)
     assert_array_less(new_y, y)
     # Check that a single vector is identity
-    X = np.array([1., 2., 3.]).reshape(1, 3)
-    y = X[0, ]
+    X = np.array([1.0, 2.0, 3.0]).reshape(1, 3)
+    y = X[0]
     new_y = _modified_weiszfeld_step(X, y)
     assert_array_equal(y, new_y)
 
 
 def test_modweiszfeld_step_2d():
-    X = np.array([0., 0., 1., 1., 0., 1.]).reshape(3, 2)
+    X = np.array([0.0, 0.0, 1.0, 1.0, 0.0, 1.0]).reshape(3, 2)
     y = np.array([0.5, 0.5])
     # Check first two iterations
     new_y = _modified_weiszfeld_step(X, y)
@@ -131,8 +141,8 @@ def test_modweiszfeld_step_2d():
 
 
 def test_spatial_median_1d():
-    X = np.array([1., 2., 3.]).reshape(3, 1)
-    true_median = 2.
+    X = np.array([1.0, 2.0, 3.0]).reshape(3, 1)
+    true_median = 2.0
     _, median = _spatial_median(X)
     assert_array_almost_equal(median, true_median)
     # Test larger problem and for exact solution in 1d case
@@ -144,8 +154,8 @@ def test_spatial_median_1d():
 
 
 def test_spatial_median_2d():
-    X = np.array([0., 0., 1., 1., 0., 1.]).reshape(3, 2)
-    _, median = _spatial_median(X, max_iter=100, tol=1.e-6)
+    X = np.array([0.0, 0.0, 1.0, 1.0, 0.0, 1.0]).reshape(3, 2)
+    _, median = _spatial_median(X, max_iter=100, tol=1.0e-6)
 
     def cost_func(y):
         dists = np.array([norm(x - y) for x in X])
@@ -155,12 +165,9 @@ def cost_func(y):
     fermat_weber = fmin_bfgs(cost_func, median, disp=False)
     assert_array_almost_equal(median, fermat_weber)
     # Check when maximum iteration is exceeded a warning is emitted
-    warning_message = (
-        "Maximum number of iterations 30 reached"
-        " in spatial median."
-    )
+    warning_message = "Maximum number of iterations 30 reached in spatial median."
     with pytest.warns(ConvergenceWarning, match=warning_message):
-        _spatial_median(X, max_iter=30, tol=0.)
+        _spatial_median(X, max_iter=30, tol=0.0)
 
 
 def test_theil_sen_1d():
@@ -180,10 +187,9 @@ def test_theil_sen_1d_no_intercept():
     lstq = LinearRegression(fit_intercept=False).fit(X, y)
     assert np.abs(lstq.coef_ - w - c) > 0.5
     # Check that Theil-Sen works
-    theil_sen = TheilSenRegressor(fit_intercept=False,
-                                  random_state=0).fit(X, y)
+    theil_sen = TheilSenRegressor(fit_intercept=False, random_state=0).fit(X, y)
     assert_array_almost_equal(theil_sen.coef_, w + c, 1)
-    assert_almost_equal(theil_sen.intercept_, 0.)
+    assert_almost_equal(theil_sen.intercept_, 0.0)
 
     # non-regression test for #18104
     theil_sen.score(X, y)
@@ -195,33 +201,36 @@ def test_theil_sen_2d():
     lstq = LinearRegression().fit(X, y)
     assert norm(lstq.coef_ - w) > 1.0
     # Check that Theil-Sen works
-    theil_sen = TheilSenRegressor(max_subpopulation=1e3,
-                                  random_state=0).fit(X, y)
+    theil_sen = TheilSenRegressor(max_subpopulation=1e3, random_state=0).fit(X, y)
     assert_array_almost_equal(theil_sen.coef_, w, 1)
     assert_array_almost_equal(theil_sen.intercept_, c, 1)
 
 
 def test_calc_breakdown_point():
     bp = _breakdown_point(1e10, 2)
-    assert np.abs(bp - 1 + 1 / (np.sqrt(2))) < 1.e-6
-
-
-def test_checksubparams_negative_subpopulation():
+    assert np.abs(bp - 1 + 1 / (np.sqrt(2))) < 1.0e-6
+
+
+@pytest.mark.parametrize(
+    "param, ExceptionCls, match",
+    [
+        (
+            {"n_subsamples": 1},
+            ValueError,
+            re.escape("Invalid parameter since n_features+1 > n_subsamples (2 > 1)"),
+        ),
+        (
+            {"n_subsamples": 101},
+            ValueError,
+            re.escape("Invalid parameter since n_subsamples > n_samples (101 > 50)"),
+        ),
+    ],
+)
+def test_checksubparams_invalid_input(param, ExceptionCls, match):
     X, y, w, c = gen_toy_problem_1d()
-    theil_sen = TheilSenRegressor(max_subpopulation=-1, random_state=0)
-    assert_raises(ValueError, theil_sen.fit, X, y)
-
-
-def test_checksubparams_too_few_subsamples():
-    X, y, w, c = gen_toy_problem_1d()
-    theil_sen = TheilSenRegressor(n_subsamples=1, random_state=0)
-    assert_raises(ValueError, theil_sen.fit, X, y)
-
-
-def test_checksubparams_too_many_subsamples():
-    X, y, w, c = gen_toy_problem_1d()
-    theil_sen = TheilSenRegressor(n_subsamples=101, random_state=0)
-    assert_raises(ValueError, theil_sen.fit, X, y)
+    theil_sen = TheilSenRegressor(**param, random_state=0)
+    with pytest.raises(ExceptionCls, match=match):
+        theil_sen.fit(X, y)
 
 
 def test_checksubparams_n_subsamples_if_less_samples_than_features():
@@ -230,21 +239,20 @@ def test_checksubparams_n_subsamples_if_less_samples_than_features():
     X = random_state.normal(size=(n_samples, n_features))
     y = random_state.normal(size=n_samples)
     theil_sen = TheilSenRegressor(n_subsamples=9, random_state=0)
-    assert_raises(ValueError, theil_sen.fit, X, y)
+    with pytest.raises(ValueError):
+        theil_sen.fit(X, y)
 
 
 def test_subpopulation():
     X, y, w, c = gen_toy_problem_4d()
-    theil_sen = TheilSenRegressor(max_subpopulation=250,
-                                  random_state=0).fit(X, y)
+    theil_sen = TheilSenRegressor(max_subpopulation=250, random_state=0).fit(X, y)
     assert_array_almost_equal(theil_sen.coef_, w, 1)
     assert_array_almost_equal(theil_sen.intercept_, c, 1)
 
 
 def test_subsamples():
     X, y, w, c = gen_toy_problem_4d()
-    theil_sen = TheilSenRegressor(n_subsamples=X.shape[0],
-                                  random_state=0).fit(X, y)
+    theil_sen = TheilSenRegressor(n_subsamples=X.shape[0], random_state=0).fit(X, y)
     lstq = LinearRegression().fit(X, y)
     # Check for exact the same results as Least Squares
     assert_array_almost_equal(theil_sen.coef_, lstq.coef_, 9)
@@ -255,9 +263,7 @@ def test_verbosity():
     # Check that Theil-Sen can be verbose
     with no_stdout_stderr():
         TheilSenRegressor(verbose=True, random_state=0).fit(X, y)
-        TheilSenRegressor(verbose=True,
-                          max_subpopulation=10,
-                          random_state=0).fit(X, y)
+        TheilSenRegressor(verbose=True, max_subpopulation=10, random_state=0).fit(X, y)
 
 
 def test_theil_sen_parallel():
@@ -266,9 +272,9 @@ def test_theil_sen_parallel():
     lstq = LinearRegression().fit(X, y)
     assert norm(lstq.coef_ - w) > 1.0
     # Check that Theil-Sen works
-    theil_sen = TheilSenRegressor(n_jobs=2,
-                                  random_state=0,
-                                  max_subpopulation=2e3).fit(X, y)
+    theil_sen = TheilSenRegressor(n_jobs=2, random_state=0, max_subpopulation=2e3).fit(
+        X, y
+    )
     assert_array_almost_equal(theil_sen.coef_, w, 1)
     assert_array_almost_equal(theil_sen.intercept_, c, 1)
 
@@ -279,8 +285,7 @@ def test_less_samples_than_features():
     X = random_state.normal(size=(n_samples, n_features))
     y = random_state.normal(size=n_samples)
     # Check that Theil-Sen falls back to Least Squares if fit_intercept=False
-    theil_sen = TheilSenRegressor(fit_intercept=False,
-                                  random_state=0).fit(X, y)
+    theil_sen = TheilSenRegressor(fit_intercept=False, random_state=0).fit(X, y)
     lstq = LinearRegression(fit_intercept=False).fit(X, y)
     assert_array_almost_equal(theil_sen.coef_, lstq.coef_, 12)
     # Check fit_intercept=True case. This will not be equal to the Least
@@ -288,3 +293,11 @@ def test_less_samples_than_features():
     theil_sen = TheilSenRegressor(fit_intercept=True, random_state=0).fit(X, y)
     y_pred = theil_sen.predict(X)
     assert_array_almost_equal(y_pred, y, 12)
+
+
+# TODO(1.8): Remove
+def test_copy_X_deprecated():
+    X, y, _, _ = gen_toy_problem_1d()
+    theil_sen = TheilSenRegressor(copy_X=True, random_state=0)
+    with pytest.warns(FutureWarning, match="`copy_X` was deprecated"):
+        theil_sen.fit(X, y)
diff --git a/sklearn/manifold/__init__.py b/sklearn/manifold/__init__.py
index a04c4f27418fd..349f7c1a4a7c4 100644
--- a/sklearn/manifold/__init__.py
+++ b/sklearn/manifold/__init__.py
@@ -1,13 +1,22 @@
-"""
-The :mod:`sklearn.manifold` module implements data embedding techniques.
-"""
+"""Data embedding techniques."""
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-from ._locally_linear import locally_linear_embedding, LocallyLinearEmbedding
 from ._isomap import Isomap
+from ._locally_linear import LocallyLinearEmbedding, locally_linear_embedding
 from ._mds import MDS, smacof
 from ._spectral_embedding import SpectralEmbedding, spectral_embedding
 from ._t_sne import TSNE, trustworthiness
 
-__all__ = ['locally_linear_embedding', 'LocallyLinearEmbedding', 'Isomap',
-           'MDS', 'smacof', 'SpectralEmbedding', 'spectral_embedding', "TSNE",
-           'trustworthiness']
+__all__ = [
+    "MDS",
+    "TSNE",
+    "Isomap",
+    "LocallyLinearEmbedding",
+    "SpectralEmbedding",
+    "locally_linear_embedding",
+    "smacof",
+    "spectral_embedding",
+    "trustworthiness",
+]
diff --git a/sklearn/manifold/_barnes_hut_tsne.pyx b/sklearn/manifold/_barnes_hut_tsne.pyx
index b15462e597684..e84df4a9074b2 100644
--- a/sklearn/manifold/_barnes_hut_tsne.pyx
+++ b/sklearn/manifold/_barnes_hut_tsne.pyx
@@ -1,31 +1,25 @@
-# cython: boundscheck=False
-# cython: wraparound=False
-# cython: cdivision=True
-#
-# Author: Christopher Moody 
-# Author: Nick Travers 
-# Implementation by Chris Moody & Nick Travers
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
 # See http://homepage.tudelft.nl/19j49/t-SNE.html for reference
 # implementations and papers describing the technique
 
 
 import numpy as np
-cimport numpy as np
+cimport numpy as cnp
 from libc.stdio cimport printf
-from libc.math cimport sqrt, log
+from libc.math cimport log
 from libc.stdlib cimport malloc, free
+from libc.time cimport clock, clock_t
 from cython.parallel cimport prange, parallel
 
 from ..neighbors._quad_tree cimport _QuadTree
 
-np.import_array()
+cnp.import_array()
 
 
 cdef char* EMPTY_STRING = ""
 
-cdef extern from "math.h":
-    float fabsf(float x) nogil
-
 # Smallest strictly positive value that can be represented by floating
 # point numbers for different precision levels. This is useful to avoid
 # taking the log of zero when computing the KL divergence.
@@ -40,25 +34,17 @@ cdef float FLOAT64_EPS = np.finfo(np.float64).eps
 cdef enum:
     DEBUGFLAG = 0
 
-cdef extern from "time.h":
-    # Declare only what is necessary from `tm` structure.
-    ctypedef long clock_t
-    clock_t clock() nogil
-    double CLOCKS_PER_SEC
-
-
 cdef float compute_gradient(float[:] val_P,
                             float[:, :] pos_reference,
-                            np.int64_t[:] neighbors,
-                            np.int64_t[:] indptr,
+                            cnp.int64_t[:] neighbors,
+                            cnp.int64_t[:] indptr,
                             float[:, :] tot_force,
                             _QuadTree qt,
                             float theta,
                             int dof,
                             long start,
-                            long stop,
                             bint compute_error,
-                            int num_threads) nogil:
+                            int num_threads) noexcept nogil:
     # Having created the tree, calculate the gradient
     # in two components, the positive and negative forces
     cdef:
@@ -73,14 +59,14 @@ cdef float compute_gradient(float[:] val_P,
 
     if qt.verbose > 11:
         printf("[t-SNE] Allocating %li elements in force arrays\n",
-                n_samples * n_dimensions * 2)
+               n_samples * n_dimensions * 2)
     cdef float* neg_f =  malloc(sizeof(float) * n_samples * n_dimensions)
     cdef float* pos_f =  malloc(sizeof(float) * n_samples * n_dimensions)
 
     if take_timing:
         t1 = clock()
     sQ = compute_gradient_negative(pos_reference, neg_f, qt, dof, theta, start,
-                                   stop, num_threads)
+                                   num_threads)
     if take_timing:
         t2 = clock()
         printf("[t-SNE] Computing negative gradient: %e ticks\n", ((float) (t2 - t1)))
@@ -107,16 +93,16 @@ cdef float compute_gradient(float[:] val_P,
 
 cdef float compute_gradient_positive(float[:] val_P,
                                      float[:, :] pos_reference,
-                                     np.int64_t[:] neighbors,
-                                     np.int64_t[:] indptr,
+                                     cnp.int64_t[:] neighbors,
+                                     cnp.int64_t[:] indptr,
                                      float* pos_f,
                                      int n_dimensions,
                                      int dof,
                                      double sum_Q,
-                                     np.int64_t start,
+                                     cnp.int64_t start,
                                      int verbose,
                                      bint compute_error,
-                                     int num_threads) nogil:
+                                     int num_threads) noexcept nogil:
     # Sum over the following expression for i not equal to j
     # grad_i = p_ij (1 + ||y_i - y_j||^2)^-1 (y_i - y_j)
     # This is equivalent to compute_edge_forces in the authors' code
@@ -161,8 +147,7 @@ cdef float compute_gradient_positive(float[:] val_P,
                 # only compute the error when needed
                 if compute_error:
                     qij = qij / sum_Q
-                    C += pij * log(max(pij, FLOAT32_TINY) \
-                        / max(qij, FLOAT32_TINY))
+                    C += pij * log(max(pij, FLOAT32_TINY) / max(qij, FLOAT32_TINY))
                 for ax in range(n_dimensions):
                     pos_f[i * n_dimensions + ax] += dij * buff[ax]
 
@@ -180,16 +165,14 @@ cdef double compute_gradient_negative(float[:, :] pos_reference,
                                       int dof,
                                       float theta,
                                       long start,
-                                      long stop,
-                                      int num_threads) nogil:
-    if stop == -1:
-        stop = pos_reference.shape[0]
+                                      int num_threads) noexcept nogil:
     cdef:
         int ax
         int n_dimensions = qt.n_dimensions
         int offset = n_dimensions + 2
         long i, j, idx
-        long n = stop - start
+        long n_samples = pos_reference.shape[0]
+        long n = n_samples - start
         long dta = 0
         long dtb = 0
         float size, dist2s, mult
@@ -202,7 +185,6 @@ cdef double compute_gradient_negative(float[:, :] pos_reference,
         clock_t t1 = 0, t2 = 0, t3 = 0
         int take_timing = 1 if qt.verbose > 20 else 0
 
-
     with nogil, parallel(num_threads=num_threads):
         # Define thread-local buffers
         summary =  malloc(sizeof(float) * n * offset)
@@ -210,7 +192,7 @@ cdef double compute_gradient_negative(float[:, :] pos_reference,
         force =  malloc(sizeof(float) * n_dimensions)
         neg_force =  malloc(sizeof(float) * n_dimensions)
 
-        for i in prange(start, stop, schedule='static'):
+        for i in prange(start, n_samples, schedule='static'):
             # Clear the arrays
             for ax in range(n_dimensions):
                 force[ax] = 0.0
@@ -262,8 +244,8 @@ cdef double compute_gradient_negative(float[:, :] pos_reference,
 
 def gradient(float[:] val_P,
              float[:, :] pos_output,
-             np.int64_t[:] neighbors,
-             np.int64_t[:] indptr,
+             cnp.int64_t[:] neighbors,
+             cnp.int64_t[:] indptr,
              float[:, :] forces,
              float theta,
              int n_dimensions,
@@ -273,7 +255,7 @@ def gradient(float[:] val_P,
              bint compute_error=1,
              int num_threads=1):
     # This function is designed to be called from external Python
-    # it passes the 'forces' array by reference and fills thats array
+    # it passes the 'forces' array by reference and fills that's array
     # up in-place
     cdef float C
     cdef int n
@@ -298,7 +280,7 @@ def gradient(float[:] val_P,
         printf("[t-SNE] Computing gradient\n%s", EMPTY_STRING)
 
     C = compute_gradient(val_P, pos_output, neighbors, indptr, forces,
-                         qt, theta, dof, skip_num_points, -1, compute_error,
+                         qt, theta, dof, skip_num_points, compute_error,
                          num_threads)
 
     if verbose > 10:
diff --git a/sklearn/manifold/_isomap.py b/sklearn/manifold/_isomap.py
index d843c3ddd8462..90154470c18a4 100644
--- a/sklearn/manifold/_isomap.py
+++ b/sklearn/manifold/_isomap.py
@@ -1,20 +1,32 @@
 """Isomap for manifold learning"""
 
-# Author: Jake Vanderplas  -- 
-# License: BSD 3 clause (C) 2011
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+import warnings
+from numbers import Integral, Real
 
 import numpy as np
-from ..base import BaseEstimator, TransformerMixin
-from ..neighbors import NearestNeighbors, kneighbors_graph
-from ..utils.validation import check_is_fitted
-from ..utils.validation import _deprecate_positional_args
-from ..utils.graph import graph_shortest_path
+from scipy.sparse import issparse
+from scipy.sparse.csgraph import connected_components, shortest_path
+
+from ..base import (
+    BaseEstimator,
+    ClassNamePrefixFeaturesOutMixin,
+    TransformerMixin,
+    _fit_context,
+)
 from ..decomposition import KernelPCA
+from ..metrics.pairwise import _VALID_METRICS
+from ..neighbors import NearestNeighbors, kneighbors_graph, radius_neighbors_graph
 from ..preprocessing import KernelCenterer
+from ..utils._param_validation import Interval, StrOptions
+from ..utils.graph import _fix_connected_components
+from ..utils.validation import check_is_fitted
 
 
-class Isomap(TransformerMixin, BaseEstimator):
-    """Isomap Embedding
+class Isomap(ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator):
+    """Isomap Embedding.
 
     Non-linear dimensionality reduction through Isometric Mapping
 
@@ -22,11 +34,18 @@ class Isomap(TransformerMixin, BaseEstimator):
 
     Parameters
     ----------
-    n_neighbors : int, default=5
-        number of neighbors to consider for each point.
+    n_neighbors : int or None, default=5
+        Number of neighbors to consider for each point. If `n_neighbors` is an int,
+        then `radius` must be `None`.
+
+    radius : float or None, default=None
+        Limiting distance of neighbors to return. If `radius` is a float,
+        then `n_neighbors` must be set to `None`.
+
+        .. versionadded:: 1.1
 
     n_components : int, default=2
-        number of coordinates for the manifold
+        Number of coordinates for the manifold.
 
     eigen_solver : {'auto', 'arpack', 'dense'}, default='auto'
         'auto' : Attempt to choose the most efficient solver
@@ -66,7 +85,7 @@ class Isomap(TransformerMixin, BaseEstimator):
         ``-1`` means using all processors. See :term:`Glossary `
         for more details.
 
-    metric : string, or callable, default="minkowski"
+    metric : str, or callable, default="minkowski"
         The metric to use when calculating distance between instances in a
         feature array. If metric is a string or callable, it must be one of
         the options allowed by :func:`sklearn.metrics.pairwise_distances` for
@@ -76,7 +95,7 @@ class Isomap(TransformerMixin, BaseEstimator):
 
         .. versionadded:: 0.22
 
-    p : int, default=2
+    p : float, default=2
         Parameter for the Minkowski metric from
         sklearn.metrics.pairwise.pairwise_distances. When p = 1, this is
         equivalent to using manhattan_distance (l1), and euclidean_distance
@@ -105,6 +124,34 @@ class Isomap(TransformerMixin, BaseEstimator):
     dist_matrix_ : array-like, shape (n_samples, n_samples)
         Stores the geodesic distance matrix of training data.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    sklearn.decomposition.PCA : Principal component analysis that is a linear
+        dimensionality reduction method.
+    sklearn.decomposition.KernelPCA : Non-linear dimensionality reduction using
+        kernels and PCA.
+    MDS : Manifold learning using multidimensional scaling.
+    TSNE : T-distributed Stochastic Neighbor Embedding.
+    LocallyLinearEmbedding : Manifold learning using Locally Linear Embedding.
+    SpectralEmbedding : Spectral embedding for non-linear dimensionality.
+
+    References
+    ----------
+
+    .. [1] Tenenbaum, J.B.; De Silva, V.; & Langford, J.C. A global geometric
+           framework for nonlinear dimensionality reduction. Science 290 (5500)
+
     Examples
     --------
     >>> from sklearn.datasets import load_digits
@@ -116,19 +163,41 @@ class Isomap(TransformerMixin, BaseEstimator):
     >>> X_transformed = embedding.fit_transform(X[:100])
     >>> X_transformed.shape
     (100, 2)
-
-    References
-    ----------
-
-    .. [1] Tenenbaum, J.B.; De Silva, V.; & Langford, J.C. A global geometric
-           framework for nonlinear dimensionality reduction. Science 290 (5500)
     """
-    @_deprecate_positional_args
-    def __init__(self, *, n_neighbors=5, n_components=2, eigen_solver='auto',
-                 tol=0, max_iter=None, path_method='auto',
-                 neighbors_algorithm='auto', n_jobs=None, metric='minkowski',
-                 p=2, metric_params=None):
+
+    _parameter_constraints: dict = {
+        "n_neighbors": [Interval(Integral, 1, None, closed="left"), None],
+        "radius": [Interval(Real, 0, None, closed="both"), None],
+        "n_components": [Interval(Integral, 1, None, closed="left")],
+        "eigen_solver": [StrOptions({"auto", "arpack", "dense"})],
+        "tol": [Interval(Real, 0, None, closed="left")],
+        "max_iter": [Interval(Integral, 1, None, closed="left"), None],
+        "path_method": [StrOptions({"auto", "FW", "D"})],
+        "neighbors_algorithm": [StrOptions({"auto", "brute", "kd_tree", "ball_tree"})],
+        "n_jobs": [Integral, None],
+        "p": [Interval(Real, 1, None, closed="left")],
+        "metric": [StrOptions(set(_VALID_METRICS) | {"precomputed"}), callable],
+        "metric_params": [dict, None],
+    }
+
+    def __init__(
+        self,
+        *,
+        n_neighbors=5,
+        radius=None,
+        n_components=2,
+        eigen_solver="auto",
+        tol=0,
+        max_iter=None,
+        path_method="auto",
+        neighbors_algorithm="auto",
+        n_jobs=None,
+        metric="minkowski",
+        p=2,
+        metric_params=None,
+    ):
         self.n_neighbors = n_neighbors
+        self.radius = radius
         self.n_components = n_components
         self.eigen_solver = eigen_solver
         self.tol = tol
@@ -141,32 +210,105 @@ def __init__(self, *, n_neighbors=5, n_components=2, eigen_solver='auto',
         self.metric_params = metric_params
 
     def _fit_transform(self, X):
-        self.nbrs_ = NearestNeighbors(n_neighbors=self.n_neighbors,
-                                      algorithm=self.neighbors_algorithm,
-                                      metric=self.metric, p=self.p,
-                                      metric_params=self.metric_params,
-                                      n_jobs=self.n_jobs)
+        if self.n_neighbors is not None and self.radius is not None:
+            raise ValueError(
+                "Both n_neighbors and radius are provided. Use"
+                f" Isomap(radius={self.radius}, n_neighbors=None) if intended to use"
+                " radius-based neighbors"
+            )
+
+        self.nbrs_ = NearestNeighbors(
+            n_neighbors=self.n_neighbors,
+            radius=self.radius,
+            algorithm=self.neighbors_algorithm,
+            metric=self.metric,
+            p=self.p,
+            metric_params=self.metric_params,
+            n_jobs=self.n_jobs,
+        )
         self.nbrs_.fit(X)
         self.n_features_in_ = self.nbrs_.n_features_in_
-
-        self.kernel_pca_ = KernelPCA(n_components=self.n_components,
-                                     kernel="precomputed",
-                                     eigen_solver=self.eigen_solver,
-                                     tol=self.tol, max_iter=self.max_iter,
-                                     n_jobs=self.n_jobs)
-
-        kng = kneighbors_graph(self.nbrs_, self.n_neighbors,
-                               metric=self.metric, p=self.p,
-                               metric_params=self.metric_params,
-                               mode='distance', n_jobs=self.n_jobs)
-
-        self.dist_matrix_ = graph_shortest_path(kng,
-                                                method=self.path_method,
-                                                directed=False)
-        G = self.dist_matrix_ ** 2
+        if hasattr(self.nbrs_, "feature_names_in_"):
+            self.feature_names_in_ = self.nbrs_.feature_names_in_
+
+        self.kernel_pca_ = KernelPCA(
+            n_components=self.n_components,
+            kernel="precomputed",
+            eigen_solver=self.eigen_solver,
+            tol=self.tol,
+            max_iter=self.max_iter,
+            n_jobs=self.n_jobs,
+        ).set_output(transform="default")
+
+        if self.n_neighbors is not None:
+            nbg = kneighbors_graph(
+                self.nbrs_,
+                self.n_neighbors,
+                metric=self.metric,
+                p=self.p,
+                metric_params=self.metric_params,
+                mode="distance",
+                n_jobs=self.n_jobs,
+            )
+        else:
+            nbg = radius_neighbors_graph(
+                self.nbrs_,
+                radius=self.radius,
+                metric=self.metric,
+                p=self.p,
+                metric_params=self.metric_params,
+                mode="distance",
+                n_jobs=self.n_jobs,
+            )
+
+        # Compute the number of connected components, and connect the different
+        # components to be able to compute a shortest path between all pairs
+        # of samples in the graph.
+        # Similar fix to cluster._agglomerative._fix_connectivity.
+        n_connected_components, labels = connected_components(nbg)
+        if n_connected_components > 1:
+            if self.metric == "precomputed" and issparse(X):
+                raise RuntimeError(
+                    "The number of connected components of the neighbors graph"
+                    f" is {n_connected_components} > 1. The graph cannot be "
+                    "completed with metric='precomputed', and Isomap cannot be"
+                    "fitted. Increase the number of neighbors to avoid this "
+                    "issue, or precompute the full distance matrix instead "
+                    "of passing a sparse neighbors graph."
+                )
+            warnings.warn(
+                (
+                    "The number of connected components of the neighbors graph "
+                    f"is {n_connected_components} > 1. Completing the graph to fit"
+                    " Isomap might be slow. Increase the number of neighbors to "
+                    "avoid this issue."
+                ),
+                stacklevel=2,
+            )
+
+            # use array validated by NearestNeighbors
+            nbg = _fix_connected_components(
+                X=self.nbrs_._fit_X,
+                graph=nbg,
+                n_connected_components=n_connected_components,
+                component_labels=labels,
+                mode="distance",
+                metric=self.nbrs_.effective_metric_,
+                **self.nbrs_.effective_metric_params_,
+            )
+
+        self.dist_matrix_ = shortest_path(nbg, method=self.path_method, directed=False)
+
+        if self.nbrs_._fit_X.dtype == np.float32:
+            self.dist_matrix_ = self.dist_matrix_.astype(
+                self.nbrs_._fit_X.dtype, copy=False
+            )
+
+        G = self.dist_matrix_**2
         G *= -0.5
 
         self.embedding_ = self.kernel_pca_.fit_transform(G)
+        self._n_features_out = self.embedding_.shape[1]
 
     def reconstruction_error(self):
         """Compute the reconstruction error for the embedding.
@@ -174,6 +316,7 @@ def reconstruction_error(self):
         Returns
         -------
         reconstruction_error : float
+            Reconstruction error.
 
         Notes
         -----
@@ -187,44 +330,56 @@ def reconstruction_error(self):
 
         ``K(D) = -0.5 * (I - 1/n_samples) * D^2 * (I - 1/n_samples)``
         """
-        G = -0.5 * self.dist_matrix_ ** 2
+        G = -0.5 * self.dist_matrix_**2
         G_center = KernelCenterer().fit_transform(G)
-        evals = self.kernel_pca_.lambdas_
-        return np.sqrt(np.sum(G_center ** 2) - np.sum(evals ** 2)) / G.shape[0]
+        evals = self.kernel_pca_.eigenvalues_
+        return np.sqrt(np.sum(G_center**2) - np.sum(evals**2)) / G.shape[0]
 
+    @_fit_context(
+        # Isomap.metric is not validated yet
+        prefer_skip_nested_validation=False
+    )
     def fit(self, X, y=None):
-        """Compute the embedding vectors for data X
+        """Compute the embedding vectors for data X.
 
         Parameters
         ----------
-        X : {array-like, sparse graph, BallTree, KDTree, NearestNeighbors}
+        X : {array-like, sparse matrix, BallTree, KDTree, NearestNeighbors}
             Sample data, shape = (n_samples, n_features), in the form of a
-            numpy array, sparse graph, precomputed tree, or NearestNeighbors
+            numpy array, sparse matrix, precomputed tree, or NearestNeighbors
             object.
 
         y : Ignored
+            Not used, present for API consistency by convention.
 
         Returns
         -------
-        self : returns an instance of self.
+        self : object
+            Returns a fitted instance of self.
         """
         self._fit_transform(X)
         return self
 
+    @_fit_context(
+        # Isomap.metric is not validated yet
+        prefer_skip_nested_validation=False
+    )
     def fit_transform(self, X, y=None):
         """Fit the model from data in X and transform X.
 
         Parameters
         ----------
-        X : {array-like, sparse graph, BallTree, KDTree}
-            Training vector, where n_samples in the number of samples
-            and n_features is the number of features.
+        X : {array-like, sparse matrix, BallTree, KDTree}
+            Training vector, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
 
         y : Ignored
+            Not used, present for API consistency by convention.
 
         Returns
         -------
         X_new : array-like, shape (n_samples, n_components)
+            X transformed in the new space.
         """
         self._fit_transform(X)
         return self.embedding_
@@ -242,7 +397,7 @@ def transform(self, X):
 
         Parameters
         ----------
-        X : array-like, shape (n_queries, n_features)
+        X : {array-like, sparse matrix}, shape (n_queries, n_features)
             If neighbors_algorithm='precomputed', X is assumed to be a
             distance matrix or a sparse graph of shape
             (n_queries, n_samples_fit).
@@ -250,9 +405,13 @@ def transform(self, X):
         Returns
         -------
         X_new : array-like, shape (n_queries, n_components)
+            X transformed in the new space.
         """
         check_is_fitted(self)
-        distances, indices = self.nbrs_.kneighbors(X, return_distance=True)
+        if self.n_neighbors is not None:
+            distances, indices = self.nbrs_.kneighbors(X, return_distance=True)
+        else:
+            distances, indices = self.nbrs_.radius_neighbors(X, return_distance=True)
 
         # Create the graph of shortest distances from X to
         # training data via the nearest neighbors of X.
@@ -261,12 +420,23 @@ def transform(self, X):
 
         n_samples_fit = self.nbrs_.n_samples_fit_
         n_queries = distances.shape[0]
-        G_X = np.zeros((n_queries, n_samples_fit))
+
+        if hasattr(X, "dtype") and X.dtype == np.float32:
+            dtype = np.float32
+        else:
+            dtype = np.float64
+
+        G_X = np.zeros((n_queries, n_samples_fit), dtype)
         for i in range(n_queries):
-            G_X[i] = np.min(self.dist_matrix_[indices[i]] +
-                            distances[i][:, None], 0)
+            G_X[i] = np.min(self.dist_matrix_[indices[i]] + distances[i][:, None], 0)
 
         G_X **= 2
         G_X *= -0.5
 
         return self.kernel_pca_.transform(G_X)
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.transformer_tags.preserves_dtype = ["float64", "float32"]
+        tags.input_tags.sparse = True
+        return tags
diff --git a/sklearn/manifold/_locally_linear.py b/sklearn/manifold/_locally_linear.py
index 7a4e0ace9fccd..e6967446274ad 100644
--- a/sklearn/manifold/_locally_linear.py
+++ b/sklearn/manifold/_locally_linear.py
@@ -1,22 +1,28 @@
 """Locally Linear Embedding"""
 
-# Author: Fabian Pedregosa -- 
-#         Jake Vanderplas  -- 
-# License: BSD 3 clause (C) INRIA 2011
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+from numbers import Integral, Real
 
 import numpy as np
-from scipy.linalg import eigh, svd, qr, solve
-from scipy.sparse import eye, csr_matrix
+from scipy.linalg import eigh, qr, solve, svd
+from scipy.sparse import csr_matrix, eye, lil_matrix
 from scipy.sparse.linalg import eigsh
 
-from ..base import BaseEstimator, TransformerMixin, _UnstableArchMixin
-from ..utils import check_random_state, check_array
+from ..base import (
+    BaseEstimator,
+    ClassNamePrefixFeaturesOutMixin,
+    TransformerMixin,
+    _fit_context,
+    _UnstableArchMixin,
+)
+from ..neighbors import NearestNeighbors
+from ..utils import check_array, check_random_state
 from ..utils._arpack import _init_arpack_v0
+from ..utils._param_validation import Interval, StrOptions, validate_params
 from ..utils.extmath import stable_cumsum
-from ..utils.validation import check_is_fitted
-from ..utils.validation import FLOAT_DTYPES
-from ..utils.validation import _deprecate_positional_args
-from ..neighbors import NearestNeighbors
+from ..utils.validation import FLOAT_DTYPES, check_is_fitted, validate_data
 
 
 def barycenter_weights(X, Y, indices, reg=1e-3):
@@ -35,7 +41,7 @@ def barycenter_weights(X, Y, indices, reg=1e-3):
             Indices of the points in Y used to compute the barycenter
 
     reg : float, default=1e-3
-        amount of regularization to add for the problem to be
+        Amount of regularization to add for the problem to be
         well-posed in the case of n_neighbors > n_dim
 
     Returns
@@ -67,11 +73,12 @@ def barycenter_weights(X, Y, indices, reg=1e-3):
             R = reg * trace
         else:
             R = reg
-        G.flat[::n_neighbors + 1] += R
-        w = solve(G, v, sym_pos=True)
+        G.flat[:: n_neighbors + 1] += R
+        w = solve(G, v, assume_a="pos")
         B[i, :] = w / np.sum(w)
     return B
 
+
 def barycenter_kneighbors_graph(X, n_neighbors, reg=1e-3, n_jobs=None):
     """Computes the barycenter weighted graph of k-Neighbors for points in X
 
@@ -111,12 +118,12 @@ def barycenter_kneighbors_graph(X, n_neighbors, reg=1e-3, n_jobs=None):
     ind = knn.kneighbors(X, return_distance=False)[:, 1:]
     data = barycenter_weights(X, X, ind, reg=reg)
     indptr = np.arange(0, n_samples * n_neighbors + 1, n_neighbors)
-    return csr_matrix((data.ravel(), ind.ravel(), indptr),
-                      shape=(n_samples, n_samples))
+    return csr_matrix((data.ravel(), ind.ravel(), indptr), shape=(n_samples, n_samples))
 
 
-def null_space(M, k, k_skip=1, eigen_solver='arpack', tol=1E-6, max_iter=100,
-               random_state=None):
+def null_space(
+    M, k, k_skip=1, eigen_solver="arpack", tol=1e-6, max_iter=100, random_state=None
+):
     """
     Find the null space of a matrix M.
 
@@ -154,20 +161,20 @@ def null_space(M, k, k_skip=1, eigen_solver='arpack', tol=1E-6, max_iter=100,
     random_state : int, RandomState instance, default=None
         Determines the random number generator when ``solver`` == 'arpack'.
         Pass an int for reproducible results across multiple function calls.
-        See :term: `Glossary `.
+        See :term:`Glossary `.
     """
-    if eigen_solver == 'auto':
+    if eigen_solver == "auto":
         if M.shape[0] > 200 and k + k_skip < 10:
-            eigen_solver = 'arpack'
+            eigen_solver = "arpack"
         else:
-            eigen_solver = 'dense'
+            eigen_solver = "dense"
 
-    if eigen_solver == 'arpack':
+    if eigen_solver == "arpack":
         v0 = _init_arpack_v0(M.shape[0], random_state)
         try:
-            eigen_values, eigen_vectors = eigsh(M, k + k_skip, sigma=0.0,
-                                                tol=tol, maxiter=max_iter,
-                                                v0=v0)
+            eigen_values, eigen_vectors = eigsh(
+                M, k + k_skip, sigma=0.0, tol=tol, maxiter=max_iter, v0=v0
+            )
         except RuntimeError as e:
             raise ValueError(
                 "Error in determining null-space with ARPACK. Error message: "
@@ -178,123 +185,33 @@ def null_space(M, k, k_skip=1, eigen_solver='arpack', tol=1E-6, max_iter=100,
             ) from e
 
         return eigen_vectors[:, k_skip:], np.sum(eigen_values[k_skip:])
-    elif eigen_solver == 'dense':
-        if hasattr(M, 'toarray'):
+    elif eigen_solver == "dense":
+        if hasattr(M, "toarray"):
             M = M.toarray()
         eigen_values, eigen_vectors = eigh(
-            M, eigvals=(k_skip, k + k_skip - 1), overwrite_a=True)
+            M, subset_by_index=(k_skip, k + k_skip - 1), overwrite_a=True
+        )
         index = np.argsort(np.abs(eigen_values))
         return eigen_vectors[:, index], np.sum(eigen_values)
     else:
         raise ValueError("Unrecognized eigen_solver '%s'" % eigen_solver)
 
 
-@_deprecate_positional_args
-def locally_linear_embedding(
-        X, *, n_neighbors, n_components, reg=1e-3, eigen_solver='auto',
-        tol=1e-6, max_iter=100, method='standard', hessian_tol=1E-4,
-        modified_tol=1E-12, random_state=None, n_jobs=None):
-    """Perform a Locally Linear Embedding analysis on the data.
-
-    Read more in the :ref:`User Guide `.
-
-    Parameters
-    ----------
-    X : {array-like, NearestNeighbors}
-        Sample data, shape = (n_samples, n_features), in the form of a
-        numpy array or a NearestNeighbors object.
-
-    n_neighbors : int
-        number of neighbors to consider for each point.
-
-    n_components : int
-        number of coordinates for the manifold.
-
-    reg : float, default=1e-3
-        regularization constant, multiplies the trace of the local covariance
-        matrix of the distances.
-
-    eigen_solver : {'auto', 'arpack', 'dense'}, default='auto'
-        auto : algorithm will attempt to choose the best method for input data
-
-        arpack : use arnoldi iteration in shift-invert mode.
-                    For this method, M may be a dense matrix, sparse matrix,
-                    or general linear operator.
-                    Warning: ARPACK can be unstable for some problems.  It is
-                    best to try several random seeds in order to check results.
-
-        dense  : use standard dense matrix operations for the eigenvalue
-                    decomposition.  For this method, M must be an array
-                    or matrix type.  This method should be avoided for
-                    large problems.
-
-    tol : float, default=1e-6
-        Tolerance for 'arpack' method
-        Not used if eigen_solver=='dense'.
-
-    max_iter : int, default=100
-        maximum number of iterations for the arpack solver.
-
-    method : {'standard', 'hessian', 'modified', 'ltsa'}, default='standard'
-        standard : use the standard locally linear embedding algorithm.
-                   see reference [1]_
-        hessian  : use the Hessian eigenmap method.  This method requires
-                   n_neighbors > n_components * (1 + (n_components + 1) / 2.
-                   see reference [2]_
-        modified : use the modified locally linear embedding algorithm.
-                   see reference [3]_
-        ltsa     : use local tangent space alignment algorithm
-                   see reference [4]_
-
-    hessian_tol : float, default=1e-4
-        Tolerance for Hessian eigenmapping method.
-        Only used if method == 'hessian'
-
-    modified_tol : float, default=1e-12
-        Tolerance for modified LLE method.
-        Only used if method == 'modified'
-
-    random_state : int, RandomState instance, default=None
-        Determines the random number generator when ``solver`` == 'arpack'.
-        Pass an int for reproducible results across multiple function calls.
-        See :term: `Glossary `.
-
-    n_jobs : int or None, default=None
-        The number of parallel jobs to run for neighbors search.
-        ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
-        ``-1`` means using all processors. See :term:`Glossary `
-        for more details.
-
-    Returns
-    -------
-    Y : array-like, shape [n_samples, n_components]
-        Embedding vectors.
-
-    squared_error : float
-        Reconstruction error for the embedding vectors. Equivalent to
-        ``norm(Y - W Y, 'fro')**2``, where W are the reconstruction weights.
-
-    References
-    ----------
-
-    .. [1] Roweis, S. & Saul, L. Nonlinear dimensionality reduction
-        by locally linear embedding.  Science 290:2323 (2000).
-    .. [2] Donoho, D. & Grimes, C. Hessian eigenmaps: Locally
-        linear embedding techniques for high-dimensional data.
-        Proc Natl Acad Sci U S A.  100:5591 (2003).
-    .. [3] Zhang, Z. & Wang, J. MLLE: Modified Locally Linear
-        Embedding Using Multiple Weights.
-        http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.70.382
-    .. [4] Zhang, Z. & Zha, H. Principal manifolds and nonlinear
-        dimensionality reduction via tangent space alignment.
-        Journal of Shanghai Univ.  8:406 (2004)
-    """
-    if eigen_solver not in ('auto', 'arpack', 'dense'):
-        raise ValueError("unrecognized eigen_solver '%s'" % eigen_solver)
-
-    if method not in ('standard', 'hessian', 'modified', 'ltsa'):
-        raise ValueError("unrecognized method '%s'" % method)
-
+def _locally_linear_embedding(
+    X,
+    *,
+    n_neighbors,
+    n_components,
+    reg=1e-3,
+    eigen_solver="auto",
+    tol=1e-6,
+    max_iter=100,
+    method="standard",
+    hessian_tol=1e-4,
+    modified_tol=1e-12,
+    random_state=None,
+    n_jobs=None,
+):
     nbrs = NearestNeighbors(n_neighbors=n_neighbors + 1, n_jobs=n_jobs)
     nbrs.fit(X)
     X = nbrs._fit_X
@@ -302,51 +219,53 @@ def locally_linear_embedding(
     N, d_in = X.shape
 
     if n_components > d_in:
-        raise ValueError("output dimension must be less than or equal "
-                         "to input dimension")
+        raise ValueError(
+            "output dimension must be less than or equal to input dimension"
+        )
     if n_neighbors >= N:
         raise ValueError(
-            "Expected n_neighbors <= n_samples, "
-            " but n_samples = %d, n_neighbors = %d" %
-            (N, n_neighbors)
+            "Expected n_neighbors <= n_samples,  but n_samples = %d, n_neighbors = %d"
+            % (N, n_neighbors)
         )
 
-    if n_neighbors <= 0:
-        raise ValueError("n_neighbors must be positive")
-
-    M_sparse = (eigen_solver != 'dense')
+    M_sparse = eigen_solver != "dense"
+    M_container_constructor = lil_matrix if M_sparse else np.zeros
 
-    if method == 'standard':
+    if method == "standard":
         W = barycenter_kneighbors_graph(
-            nbrs, n_neighbors=n_neighbors, reg=reg, n_jobs=n_jobs)
+            nbrs, n_neighbors=n_neighbors, reg=reg, n_jobs=n_jobs
+        )
 
         # we'll compute M = (I-W)'(I-W)
         # depending on the solver, we'll do this differently
         if M_sparse:
             M = eye(*W.shape, format=W.format) - W
-            M = (M.T * M).tocsr()
+            M = M.T @ M
         else:
-            M = (W.T * W - W.T - W).toarray()
-            M.flat[::M.shape[0] + 1] += 1  # W = W - I = W - I
+            M = (W.T @ W - W.T - W).toarray()
+            M.flat[:: M.shape[0] + 1] += 1  # M = W' W - W' - W + I
 
-    elif method == 'hessian':
+    elif method == "hessian":
         dp = n_components * (n_components + 1) // 2
 
         if n_neighbors <= n_components + dp:
-            raise ValueError("for method='hessian', n_neighbors must be "
-                             "greater than "
-                             "[n_components * (n_components + 3) / 2]")
+            raise ValueError(
+                "for method='hessian', n_neighbors must be "
+                "greater than "
+                "[n_components * (n_components + 3) / 2]"
+            )
 
-        neighbors = nbrs.kneighbors(X, n_neighbors=n_neighbors + 1,
-                                    return_distance=False)
+        neighbors = nbrs.kneighbors(
+            X, n_neighbors=n_neighbors + 1, return_distance=False
+        )
         neighbors = neighbors[:, 1:]
 
         Yi = np.empty((n_neighbors, 1 + n_components + dp), dtype=np.float64)
         Yi[:, 0] = 1
 
-        M = np.zeros((N, N), dtype=np.float64)
+        M = M_container_constructor((N, N), dtype=np.float64)
 
-        use_svd = (n_neighbors > d_in)
+        use_svd = n_neighbors > d_in
 
         for i in range(N):
             Gi = X[neighbors[i]]
@@ -359,17 +278,16 @@ def locally_linear_embedding(
                 Ci = np.dot(Gi, Gi.T)
                 U = eigh(Ci)[1][:, ::-1]
 
-            Yi[:, 1:1 + n_components] = U[:, :n_components]
+            Yi[:, 1 : 1 + n_components] = U[:, :n_components]
 
             j = 1 + n_components
             for k in range(n_components):
-                Yi[:, j:j + n_components - k] = (U[:, k:k + 1] *
-                                                 U[:, k:n_components])
+                Yi[:, j : j + n_components - k] = U[:, k : k + 1] * U[:, k:n_components]
                 j += n_components - k
 
             Q, R = qr(Yi)
 
-            w = Q[:, n_components + 1:]
+            w = Q[:, n_components + 1 :]
             S = w.sum(0)
 
             S[np.where(abs(S) < hessian_tol)] = 1
@@ -378,16 +296,13 @@ def locally_linear_embedding(
             nbrs_x, nbrs_y = np.meshgrid(neighbors[i], neighbors[i])
             M[nbrs_x, nbrs_y] += np.dot(w, w.T)
 
-        if M_sparse:
-            M = csr_matrix(M)
-
-    elif method == 'modified':
+    elif method == "modified":
         if n_neighbors < n_components:
-            raise ValueError("modified LLE requires "
-                             "n_neighbors >= n_components")
+            raise ValueError("modified LLE requires n_neighbors >= n_components")
 
-        neighbors = nbrs.kneighbors(X, n_neighbors=n_neighbors + 1,
-                                    return_distance=False)
+        neighbors = nbrs.kneighbors(
+            X, n_neighbors=n_neighbors + 1, return_distance=False
+        )
         neighbors = neighbors[:, 1:]
 
         # find the eigenvectors and eigenvalues of each local covariance
@@ -398,13 +313,12 @@ def locally_linear_embedding(
         evals = np.zeros([N, nev])
 
         # choose the most efficient way to find the eigenvectors
-        use_svd = (n_neighbors > d_in)
+        use_svd = n_neighbors > d_in
 
         if use_svd:
             for i in range(N):
                 X_nbrs = X[neighbors[i]] - X[i]
-                V[i], evals[i], _ = svd(X_nbrs,
-                                        full_matrices=True)
+                V[i], evals[i], _ = svd(X_nbrs, full_matrices=True)
             evals **= 2
         else:
             for i in range(N):
@@ -417,7 +331,7 @@ def locally_linear_embedding(
         # find regularized weights: this is like normal LLE.
         # because we've already computed the SVD of each covariance matrix,
         # it's faster to use this rather than np.linalg.solve
-        reg = 1E-3 * evals.sum(1)
+        reg = 1e-3 * evals.sum(1)
 
         tmp = np.dot(V.transpose(0, 2, 1), np.ones(n_neighbors))
         tmp[:, :nev] /= evals + reg[:, None]
@@ -445,12 +359,13 @@ def locally_linear_embedding(
 
         # Now calculate M.
         # This is the [N x N] matrix whose null space is the desired embedding
-        M = np.zeros((N, N), dtype=np.float64)
+        M = M_container_constructor((N, N), dtype=np.float64)
+
         for i in range(N):
             s_i = s_range[i]
 
             # select bottom s_i eigenvectors and calculate alpha
-            Vi = V[i, :, n_neighbors - s_i:]
+            Vi = V[i, :, n_neighbors - s_i :]
             alpha_i = np.linalg.norm(Vi.sum(0)) / np.sqrt(s_i)
 
             # compute Householder matrix which satisfies
@@ -469,8 +384,7 @@ def locally_linear_embedding(
             # Then the weight matrix is
             #  >> Wi = np.dot(Vi,Hi) + (1-alpha_i) * w_reg[i,:,None]
             # We do this much more efficiently:
-            Wi = (Vi - 2 * np.outer(np.dot(Vi, h), h) +
-                  (1 - alpha_i) * w_reg[i, :, None])
+            Wi = Vi - 2 * np.outer(np.dot(Vi, h), h) + (1 - alpha_i) * w_reg[i, :, None]
 
             # Update M as follows:
             # >> W_hat = np.zeros( (N,s_i) )
@@ -482,26 +396,24 @@ def locally_linear_embedding(
             M[nbrs_x, nbrs_y] += np.dot(Wi, Wi.T)
             Wi_sum1 = Wi.sum(1)
             M[i, neighbors[i]] -= Wi_sum1
-            M[neighbors[i], i] -= Wi_sum1
+            M[neighbors[i], [i]] -= Wi_sum1
             M[i, i] += s_i
 
-        if M_sparse:
-            M = csr_matrix(M)
-
-    elif method == 'ltsa':
-        neighbors = nbrs.kneighbors(X, n_neighbors=n_neighbors + 1,
-                                    return_distance=False)
+    elif method == "ltsa":
+        neighbors = nbrs.kneighbors(
+            X, n_neighbors=n_neighbors + 1, return_distance=False
+        )
         neighbors = neighbors[:, 1:]
 
-        M = np.zeros((N, N))
+        M = M_container_constructor((N, N), dtype=np.float64)
 
-        use_svd = (n_neighbors > d_in)
+        use_svd = n_neighbors > d_in
 
         for i in range(N):
             Xi = X[neighbors[i]]
             Xi -= Xi.mean(0)
 
-            # compute n_components largest eigenvalues of Xi * Xi^T
+            # compute n_components largest eigenvalues of Xi @ Xi^T
             if use_svd:
                 v = svd(Xi, full_matrices=True)[0]
             else:
@@ -510,34 +422,79 @@ def locally_linear_embedding(
 
             Gi = np.zeros((n_neighbors, n_components + 1))
             Gi[:, 1:] = v[:, :n_components]
-            Gi[:, 0] = 1. / np.sqrt(n_neighbors)
+            Gi[:, 0] = 1.0 / np.sqrt(n_neighbors)
 
             GiGiT = np.dot(Gi, Gi.T)
 
             nbrs_x, nbrs_y = np.meshgrid(neighbors[i], neighbors[i])
             M[nbrs_x, nbrs_y] -= GiGiT
-            M[neighbors[i], neighbors[i]] += 1
-
-    return null_space(M, n_components, k_skip=1, eigen_solver=eigen_solver,
-                      tol=tol, max_iter=max_iter, random_state=random_state)
-
 
-class LocallyLinearEmbedding(TransformerMixin,
-                             _UnstableArchMixin, BaseEstimator):
-    """Locally Linear Embedding
+            M[neighbors[i], neighbors[i]] += np.ones(shape=n_neighbors)
+
+    if M_sparse:
+        M = M.tocsr()
+
+    return null_space(
+        M,
+        n_components,
+        k_skip=1,
+        eigen_solver=eigen_solver,
+        tol=tol,
+        max_iter=max_iter,
+        random_state=random_state,
+    )
+
+
+@validate_params(
+    {
+        "X": ["array-like", NearestNeighbors],
+        "n_neighbors": [Interval(Integral, 1, None, closed="left")],
+        "n_components": [Interval(Integral, 1, None, closed="left")],
+        "reg": [Interval(Real, 0, None, closed="left")],
+        "eigen_solver": [StrOptions({"auto", "arpack", "dense"})],
+        "tol": [Interval(Real, 0, None, closed="left")],
+        "max_iter": [Interval(Integral, 1, None, closed="left")],
+        "method": [StrOptions({"standard", "hessian", "modified", "ltsa"})],
+        "hessian_tol": [Interval(Real, 0, None, closed="left")],
+        "modified_tol": [Interval(Real, 0, None, closed="left")],
+        "random_state": ["random_state"],
+        "n_jobs": [None, Integral],
+    },
+    prefer_skip_nested_validation=True,
+)
+def locally_linear_embedding(
+    X,
+    *,
+    n_neighbors,
+    n_components,
+    reg=1e-3,
+    eigen_solver="auto",
+    tol=1e-6,
+    max_iter=100,
+    method="standard",
+    hessian_tol=1e-4,
+    modified_tol=1e-12,
+    random_state=None,
+    n_jobs=None,
+):
+    """Perform a Locally Linear Embedding analysis on the data.
 
     Read more in the :ref:`User Guide `.
 
     Parameters
     ----------
-    n_neighbors : int, default=5
-        number of neighbors to consider for each point.
+    X : {array-like, NearestNeighbors}
+        Sample data, shape = (n_samples, n_features), in the form of a
+        numpy array or a NearestNeighbors object.
 
-    n_components : int, default=2
-        number of coordinates for the manifold
+    n_neighbors : int
+        Number of neighbors to consider for each point.
+
+    n_components : int
+        Number of coordinates for the manifold.
 
     reg : float, default=1e-3
-        regularization constant, multiplies the trace of the local covariance
+        Regularization constant, multiplies the trace of the local covariance
         matrix of the distances.
 
     eigen_solver : {'auto', 'arpack', 'dense'}, default='auto'
@@ -559,7 +516,133 @@ class LocallyLinearEmbedding(TransformerMixin,
         Not used if eigen_solver=='dense'.
 
     max_iter : int, default=100
-        maximum number of iterations for the arpack solver.
+        Maximum number of iterations for the arpack solver.
+
+    method : {'standard', 'hessian', 'modified', 'ltsa'}, default='standard'
+        standard : use the standard locally linear embedding algorithm.
+                   see reference [1]_
+        hessian  : use the Hessian eigenmap method.  This method requires
+                   n_neighbors > n_components * (1 + (n_components + 1) / 2.
+                   see reference [2]_
+        modified : use the modified locally linear embedding algorithm.
+                   see reference [3]_
+        ltsa     : use local tangent space alignment algorithm
+                   see reference [4]_
+
+    hessian_tol : float, default=1e-4
+        Tolerance for Hessian eigenmapping method.
+        Only used if method == 'hessian'.
+
+    modified_tol : float, default=1e-12
+        Tolerance for modified LLE method.
+        Only used if method == 'modified'.
+
+    random_state : int, RandomState instance, default=None
+        Determines the random number generator when ``solver`` == 'arpack'.
+        Pass an int for reproducible results across multiple function calls.
+        See :term:`Glossary `.
+
+    n_jobs : int or None, default=None
+        The number of parallel jobs to run for neighbors search.
+        ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
+        ``-1`` means using all processors. See :term:`Glossary `
+        for more details.
+
+    Returns
+    -------
+    Y : ndarray of shape (n_samples, n_components)
+        Embedding vectors.
+
+    squared_error : float
+        Reconstruction error for the embedding vectors. Equivalent to
+        ``norm(Y - W Y, 'fro')**2``, where W are the reconstruction weights.
+
+    References
+    ----------
+
+    .. [1] Roweis, S. & Saul, L. Nonlinear dimensionality reduction
+        by locally linear embedding.  Science 290:2323 (2000).
+    .. [2] Donoho, D. & Grimes, C. Hessian eigenmaps: Locally
+        linear embedding techniques for high-dimensional data.
+        Proc Natl Acad Sci U S A.  100:5591 (2003).
+    .. [3] `Zhang, Z. & Wang, J. MLLE: Modified Locally Linear
+        Embedding Using Multiple Weights.
+        `_
+    .. [4] Zhang, Z. & Zha, H. Principal manifolds and nonlinear
+        dimensionality reduction via tangent space alignment.
+        Journal of Shanghai Univ.  8:406 (2004)
+
+    Examples
+    --------
+    >>> from sklearn.datasets import load_digits
+    >>> from sklearn.manifold import locally_linear_embedding
+    >>> X, _ = load_digits(return_X_y=True)
+    >>> X.shape
+    (1797, 64)
+    >>> embedding, _ = locally_linear_embedding(X[:100],n_neighbors=5, n_components=2)
+    >>> embedding.shape
+    (100, 2)
+    """
+    return _locally_linear_embedding(
+        X=X,
+        n_neighbors=n_neighbors,
+        n_components=n_components,
+        reg=reg,
+        eigen_solver=eigen_solver,
+        tol=tol,
+        max_iter=max_iter,
+        method=method,
+        hessian_tol=hessian_tol,
+        modified_tol=modified_tol,
+        random_state=random_state,
+        n_jobs=n_jobs,
+    )
+
+
+class LocallyLinearEmbedding(
+    ClassNamePrefixFeaturesOutMixin,
+    TransformerMixin,
+    _UnstableArchMixin,
+    BaseEstimator,
+):
+    """Locally Linear Embedding.
+
+    Read more in the :ref:`User Guide `.
+
+    Parameters
+    ----------
+    n_neighbors : int, default=5
+        Number of neighbors to consider for each point.
+
+    n_components : int, default=2
+        Number of coordinates for the manifold.
+
+    reg : float, default=1e-3
+        Regularization constant, multiplies the trace of the local covariance
+        matrix of the distances.
+
+    eigen_solver : {'auto', 'arpack', 'dense'}, default='auto'
+        The solver used to compute the eigenvectors. The available options are:
+
+        - `'auto'` : algorithm will attempt to choose the best method for input
+          data.
+        - `'arpack'` : use arnoldi iteration in shift-invert mode. For this
+          method, M may be a dense matrix, sparse matrix, or general linear
+          operator.
+        - `'dense'`  : use standard dense matrix operations for the eigenvalue
+          decomposition. For this method, M must be an array or matrix type.
+          This method should be avoided for large problems.
+
+        .. warning::
+           ARPACK can be unstable for some problems.  It is best to try several
+           random seeds in order to check results.
+
+    tol : float, default=1e-6
+        Tolerance for 'arpack' method
+        Not used if eigen_solver=='dense'.
+
+    max_iter : int, default=100
+        Maximum number of iterations for the arpack solver.
         Not used if eigen_solver=='dense'.
 
     method : {'standard', 'hessian', 'modified', 'ltsa'}, default='standard'
@@ -575,21 +658,21 @@ class LocallyLinearEmbedding(TransformerMixin,
 
     hessian_tol : float, default=1e-4
         Tolerance for Hessian eigenmapping method.
-        Only used if ``method == 'hessian'``
+        Only used if ``method == 'hessian'``.
 
     modified_tol : float, default=1e-12
         Tolerance for modified LLE method.
-        Only used if ``method == 'modified'``
+        Only used if ``method == 'modified'``.
 
     neighbors_algorithm : {'auto', 'brute', 'kd_tree', 'ball_tree'}, \
                           default='auto'
-        algorithm to use for nearest neighbors search,
-        passed to neighbors.NearestNeighbors instance
+        Algorithm to use for nearest neighbors search, passed to
+        :class:`~sklearn.neighbors.NearestNeighbors` instance.
 
     random_state : int, RandomState instance, default=None
         Determines the random number generator when
         ``eigen_solver`` == 'arpack'. Pass an int for reproducible results
-        across multiple function calls. See :term: `Glossary `.
+        across multiple function calls. See :term:`Glossary `.
 
     n_jobs : int or None, default=None
         The number of parallel jobs to run.
@@ -605,21 +688,26 @@ class LocallyLinearEmbedding(TransformerMixin,
     reconstruction_error_ : float
         Reconstruction error associated with `embedding_`
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     nbrs_ : NearestNeighbors object
         Stores nearest neighbors instance, including BallTree or KDtree
         if applicable.
 
-    Examples
+    See Also
     --------
-    >>> from sklearn.datasets import load_digits
-    >>> from sklearn.manifold import LocallyLinearEmbedding
-    >>> X, _ = load_digits(return_X_y=True)
-    >>> X.shape
-    (1797, 64)
-    >>> embedding = LocallyLinearEmbedding(n_components=2)
-    >>> X_transformed = embedding.fit_transform(X[:100])
-    >>> X_transformed.shape
-    (100, 2)
+    SpectralEmbedding : Spectral embedding for non-linear dimensionality
+        reduction.
+    TSNE : Distributed Stochastic Neighbor Embedding.
 
     References
     ----------
@@ -629,18 +717,57 @@ class LocallyLinearEmbedding(TransformerMixin,
     .. [2] Donoho, D. & Grimes, C. Hessian eigenmaps: Locally
         linear embedding techniques for high-dimensional data.
         Proc Natl Acad Sci U S A.  100:5591 (2003).
-    .. [3] Zhang, Z. & Wang, J. MLLE: Modified Locally Linear
+    .. [3] `Zhang, Z. & Wang, J. MLLE: Modified Locally Linear
         Embedding Using Multiple Weights.
-        http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.70.382
+        `_
     .. [4] Zhang, Z. & Zha, H. Principal manifolds and nonlinear
         dimensionality reduction via tangent space alignment.
         Journal of Shanghai Univ.  8:406 (2004)
+
+    Examples
+    --------
+    >>> from sklearn.datasets import load_digits
+    >>> from sklearn.manifold import LocallyLinearEmbedding
+    >>> X, _ = load_digits(return_X_y=True)
+    >>> X.shape
+    (1797, 64)
+    >>> embedding = LocallyLinearEmbedding(n_components=2)
+    >>> X_transformed = embedding.fit_transform(X[:100])
+    >>> X_transformed.shape
+    (100, 2)
     """
-    @_deprecate_positional_args
-    def __init__(self, *, n_neighbors=5, n_components=2, reg=1E-3,
-                 eigen_solver='auto', tol=1E-6, max_iter=100,
-                 method='standard', hessian_tol=1E-4, modified_tol=1E-12,
-                 neighbors_algorithm='auto', random_state=None, n_jobs=None):
+
+    _parameter_constraints: dict = {
+        "n_neighbors": [Interval(Integral, 1, None, closed="left")],
+        "n_components": [Interval(Integral, 1, None, closed="left")],
+        "reg": [Interval(Real, 0, None, closed="left")],
+        "eigen_solver": [StrOptions({"auto", "arpack", "dense"})],
+        "tol": [Interval(Real, 0, None, closed="left")],
+        "max_iter": [Interval(Integral, 1, None, closed="left")],
+        "method": [StrOptions({"standard", "hessian", "modified", "ltsa"})],
+        "hessian_tol": [Interval(Real, 0, None, closed="left")],
+        "modified_tol": [Interval(Real, 0, None, closed="left")],
+        "neighbors_algorithm": [StrOptions({"auto", "brute", "kd_tree", "ball_tree"})],
+        "random_state": ["random_state"],
+        "n_jobs": [None, Integral],
+    }
+
+    def __init__(
+        self,
+        *,
+        n_neighbors=5,
+        n_components=2,
+        reg=1e-3,
+        eigen_solver="auto",
+        tol=1e-6,
+        max_iter=100,
+        method="standard",
+        hessian_tol=1e-4,
+        modified_tol=1e-12,
+        neighbors_algorithm="auto",
+        random_state=None,
+        n_jobs=None,
+    ):
         self.n_neighbors = n_neighbors
         self.n_components = n_components
         self.reg = reg
@@ -655,52 +782,67 @@ def __init__(self, *, n_neighbors=5, n_components=2, reg=1E-3,
         self.n_jobs = n_jobs
 
     def _fit_transform(self, X):
-        self.nbrs_ = NearestNeighbors(n_neighbors=self.n_neighbors,
-                                      algorithm=self.neighbors_algorithm,
-                                      n_jobs=self.n_jobs)
+        self.nbrs_ = NearestNeighbors(
+            n_neighbors=self.n_neighbors,
+            algorithm=self.neighbors_algorithm,
+            n_jobs=self.n_jobs,
+        )
 
         random_state = check_random_state(self.random_state)
-        X = self._validate_data(X, dtype=float)
+        X = validate_data(self, X, dtype=float)
         self.nbrs_.fit(X)
-        self.embedding_, self.reconstruction_error_ = \
-            locally_linear_embedding(
-                X=self.nbrs_, n_neighbors=self.n_neighbors,
-                n_components=self.n_components,
-                eigen_solver=self.eigen_solver, tol=self.tol,
-                max_iter=self.max_iter, method=self.method,
-                hessian_tol=self.hessian_tol, modified_tol=self.modified_tol,
-                random_state=random_state, reg=self.reg, n_jobs=self.n_jobs)
+        self.embedding_, self.reconstruction_error_ = _locally_linear_embedding(
+            X=self.nbrs_,
+            n_neighbors=self.n_neighbors,
+            n_components=self.n_components,
+            eigen_solver=self.eigen_solver,
+            tol=self.tol,
+            max_iter=self.max_iter,
+            method=self.method,
+            hessian_tol=self.hessian_tol,
+            modified_tol=self.modified_tol,
+            random_state=random_state,
+            reg=self.reg,
+            n_jobs=self.n_jobs,
+        )
+        self._n_features_out = self.embedding_.shape[1]
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y=None):
-        """Compute the embedding vectors for data X
+        """Compute the embedding vectors for data X.
 
         Parameters
         ----------
-        X : array-like of shape [n_samples, n_features]
-            training set.
+        X : array-like of shape (n_samples, n_features)
+            Training set.
 
         y : Ignored
+            Not used, present here for API consistency by convention.
 
         Returns
         -------
-        self : returns an instance of self.
+        self : object
+            Fitted `LocallyLinearEmbedding` class instance.
         """
         self._fit_transform(X)
         return self
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit_transform(self, X, y=None):
         """Compute the embedding vectors for data X and transform X.
 
         Parameters
         ----------
-        X : array-like of shape [n_samples, n_features]
-            training set.
+        X : array-like of shape (n_samples, n_features)
+            Training set.
 
         y : Ignored
+            Not used, present here for API consistency by convention.
 
         Returns
         -------
         X_new : array-like, shape (n_samples, n_components)
+            Returns the instance itself.
         """
         self._fit_transform(X)
         return self.embedding_
@@ -712,21 +854,24 @@ def transform(self, X):
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
+            Training set.
 
         Returns
         -------
-        X_new : array, shape = [n_samples, n_components]
+        X_new : ndarray of shape (n_samples, n_components)
+            Returns the instance itself.
 
         Notes
         -----
         Because of scaling performed by this method, it is discouraged to use
-        it together with methods that are not scale-invariant (like SVMs)
+        it together with methods that are not scale-invariant (like SVMs).
         """
         check_is_fitted(self)
 
-        X = check_array(X)
-        ind = self.nbrs_.kneighbors(X, n_neighbors=self.n_neighbors,
-                                    return_distance=False)
+        X = validate_data(self, X, reset=False)
+        ind = self.nbrs_.kneighbors(
+            X, n_neighbors=self.n_neighbors, return_distance=False
+        )
         weights = barycenter_weights(X, self.nbrs_._fit_X, ind, reg=self.reg)
         X_new = np.empty((X.shape[0], self.n_components))
         for i in range(X.shape[0]):
diff --git a/sklearn/manifold/_mds.py b/sklearn/manifold/_mds.py
index 6a144e3033e8e..6c31c72f7ef59 100644
--- a/sklearn/manifold/_mds.py
+++ b/sklearn/manifold/_mds.py
@@ -2,25 +2,35 @@
 Multi-dimensional Scaling (MDS).
 """
 
-# author: Nelle Varoquaux 
-# License: BSD
-
-import numpy as np
-from joblib import Parallel, effective_n_jobs
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import warnings
+from numbers import Integral, Real
 
-from ..base import BaseEstimator
-from ..metrics import euclidean_distances
-from ..utils import check_random_state, check_array, check_symmetric
-from ..isotonic import IsotonicRegression
-from ..utils.validation import _deprecate_positional_args
-from ..utils.deprecation import deprecated
-from ..utils.fixes import delayed
-
+import numpy as np
+from joblib import effective_n_jobs
 
-def _smacof_single(dissimilarities, metric=True, n_components=2, init=None,
-                   max_iter=300, verbose=0, eps=1e-3, random_state=None):
+from ..base import BaseEstimator, _fit_context
+from ..isotonic import IsotonicRegression
+from ..metrics import euclidean_distances
+from ..utils import check_array, check_random_state, check_symmetric
+from ..utils._param_validation import Interval, StrOptions, validate_params
+from ..utils.parallel import Parallel, delayed
+from ..utils.validation import validate_data
+
+
+def _smacof_single(
+    dissimilarities,
+    metric=True,
+    n_components=2,
+    init=None,
+    max_iter=300,
+    verbose=0,
+    eps=1e-6,
+    random_state=None,
+    normalized_stress=False,
+):
     """Computes multidimensional scaling using SMACOF algorithm.
 
     Parameters
@@ -30,6 +40,8 @@ def _smacof_single(dissimilarities, metric=True, n_components=2, init=None,
 
     metric : bool, default=True
         Compute metric or nonmetric SMACOF algorithm.
+        When ``False`` (i.e. non-metric MDS), dissimilarities with 0 are considered as
+        missing values.
 
     n_components : int, default=2
         Number of dimensions in which to immerse the dissimilarities. If an
@@ -47,14 +59,27 @@ def _smacof_single(dissimilarities, metric=True, n_components=2, init=None,
     verbose : int, default=0
         Level of verbosity.
 
-    eps : float, default=1e-3
-        Relative tolerance with respect to stress at which to declare
-        convergence.
+    eps : float, default=1e-6
+        The tolerance with respect to stress (normalized by the sum of squared
+        embedding distances) at which to declare convergence.
+
+        .. versionchanged:: 1.7
+           The default value for `eps` has changed from 1e-3 to 1e-6, as a result
+           of a bugfix in the computation of the convergence criterion.
 
     random_state : int, RandomState instance or None, default=None
         Determines the random number generator used to initialize the centers.
         Pass an int for reproducible results across multiple function calls.
-        See :term: `Glossary `.
+        See :term:`Glossary `.
+
+    normalized_stress : bool, default=False
+        Whether to return normalized stress value (Stress-1) instead of raw
+        stress.
+
+        .. versionadded:: 1.2
+
+        .. versionchanged:: 1.7
+           Normalized stress is now supported for metric MDS as well.
 
     Returns
     -------
@@ -64,79 +89,140 @@ def _smacof_single(dissimilarities, metric=True, n_components=2, init=None,
     stress : float
         The final value of the stress (sum of squared distance of the
         disparities and the distances for all constrained points).
+        If `normalized_stress=True`, returns Stress-1.
+        A value of 0 indicates "perfect" fit, 0.025 excellent, 0.05 good,
+        0.1 fair, and 0.2 poor [1]_.
 
     n_iter : int
         The number of iterations corresponding to the best stress.
+
+    References
+    ----------
+    .. [1] "Nonmetric multidimensional scaling: a numerical method" Kruskal, J.
+           Psychometrika, 29 (1964)
+
+    .. [2] "Multidimensional scaling by optimizing goodness of fit to a nonmetric
+           hypothesis" Kruskal, J. Psychometrika, 29, (1964)
+
+    .. [3] "Modern Multidimensional Scaling - Theory and Applications" Borg, I.;
+           Groenen P. Springer Series in Statistics (1997)
     """
     dissimilarities = check_symmetric(dissimilarities, raise_exception=True)
 
     n_samples = dissimilarities.shape[0]
     random_state = check_random_state(random_state)
 
-    sim_flat = ((1 - np.tri(n_samples)) * dissimilarities).ravel()
-    sim_flat_w = sim_flat[sim_flat != 0]
+    dissimilarities_flat = ((1 - np.tri(n_samples)) * dissimilarities).ravel()
+    dissimilarities_flat_w = dissimilarities_flat[dissimilarities_flat != 0]
     if init is None:
         # Randomly choose initial configuration
-        X = random_state.rand(n_samples * n_components)
+        X = random_state.uniform(size=n_samples * n_components)
         X = X.reshape((n_samples, n_components))
     else:
         # overrides the parameter p
         n_components = init.shape[1]
         if n_samples != init.shape[0]:
-            raise ValueError("init matrix should be of shape (%d, %d)" %
-                             (n_samples, n_components))
+            raise ValueError(
+                "init matrix should be of shape (%d, %d)" % (n_samples, n_components)
+            )
         X = init
+    distances = euclidean_distances(X)
+
+    # Out of bounds condition cannot happen because we are transforming
+    # the training set here, but does sometimes get triggered in
+    # practice due to machine precision issues. Hence "clip".
+    ir = IsotonicRegression(out_of_bounds="clip")
 
     old_stress = None
-    ir = IsotonicRegression()
     for it in range(max_iter):
         # Compute distance and monotonic regression
-        dis = euclidean_distances(X)
-
         if metric:
             disparities = dissimilarities
         else:
-            dis_flat = dis.ravel()
+            distances_flat = distances.ravel()
             # dissimilarities with 0 are considered as missing values
-            dis_flat_w = dis_flat[sim_flat != 0]
-
-            # Compute the disparities using a monotonic regression
-            disparities_flat = ir.fit_transform(sim_flat_w, dis_flat_w)
-            disparities = dis_flat.copy()
-            disparities[sim_flat != 0] = disparities_flat
+            distances_flat_w = distances_flat[dissimilarities_flat != 0]
+
+            # Compute the disparities using isotonic regression.
+            # For the first SMACOF iteration, use scaled original dissimilarities.
+            # (This choice follows the R implementation described in this paper:
+            # https://www.jstatsoft.org/article/view/v102i10)
+            if it < 1:
+                disparities_flat = dissimilarities_flat_w
+            else:
+                disparities_flat = ir.fit_transform(
+                    dissimilarities_flat_w, distances_flat_w
+                )
+            disparities = np.zeros_like(distances_flat)
+            disparities[dissimilarities_flat != 0] = disparities_flat
             disparities = disparities.reshape((n_samples, n_samples))
-            disparities *= np.sqrt((n_samples * (n_samples - 1) / 2) /
-                                   (disparities ** 2).sum())
-
-        # Compute stress
-        stress = ((dis.ravel() - disparities.ravel()) ** 2).sum() / 2
+            disparities *= np.sqrt(
+                (n_samples * (n_samples - 1) / 2) / (disparities**2).sum()
+            )
+            disparities = disparities + disparities.T
 
         # Update X using the Guttman transform
-        dis[dis == 0] = 1e-5
-        ratio = disparities / dis
-        B = - ratio
+        distances[distances == 0] = 1e-5
+        ratio = disparities / distances
+        B = -ratio
         B[np.arange(len(B)), np.arange(len(B))] += ratio.sum(axis=1)
-        X = 1. / n_samples * np.dot(B, X)
+        X = 1.0 / n_samples * np.dot(B, X)
 
-        dis = np.sqrt((X ** 2).sum(axis=1)).sum()
-        if verbose >= 2:
-            print('it: %d, stress %s' % (it, stress))
+        # Compute stress
+        distances = euclidean_distances(X)
+        stress = ((distances.ravel() - disparities.ravel()) ** 2).sum() / 2
+
+        if verbose >= 2:  # pragma: no cover
+            print(f"Iteration {it}, stress {stress:.4f}")
         if old_stress is not None:
-            if(old_stress - stress / dis) < eps:
-                if verbose:
-                    print('breaking at iteration %d with stress %s' % (it,
-                                                                       stress))
+            sum_squared_distances = (distances.ravel() ** 2).sum()
+            if ((old_stress - stress) / (sum_squared_distances / 2)) < eps:
+                if verbose:  # pragma: no cover
+                    print("Convergence criterion reached.")
                 break
-        old_stress = stress / dis
+        old_stress = stress
+
+    if normalized_stress:
+        sum_squared_distances = (distances.ravel() ** 2).sum()
+        stress = np.sqrt(stress / (sum_squared_distances / 2))
 
     return X, stress, it + 1
 
 
-@_deprecate_positional_args
-def smacof(dissimilarities, *, metric=True, n_components=2, init=None,
-           n_init=8, n_jobs=None, max_iter=300, verbose=0, eps=1e-3,
-           random_state=None, return_n_iter=False):
-    """Computes multidimensional scaling using the SMACOF algorithm.
+# TODO(1.9): change default `n_init` to 1, see PR #31117
+@validate_params(
+    {
+        "dissimilarities": ["array-like"],
+        "metric": ["boolean"],
+        "n_components": [Interval(Integral, 1, None, closed="left")],
+        "init": ["array-like", None],
+        "n_init": [Interval(Integral, 1, None, closed="left"), StrOptions({"warn"})],
+        "n_jobs": [Integral, None],
+        "max_iter": [Interval(Integral, 1, None, closed="left")],
+        "verbose": ["verbose"],
+        "eps": [Interval(Real, 0, None, closed="left")],
+        "random_state": ["random_state"],
+        "return_n_iter": ["boolean"],
+        "normalized_stress": ["boolean", StrOptions({"auto"})],
+    },
+    prefer_skip_nested_validation=True,
+)
+def smacof(
+    dissimilarities,
+    *,
+    metric=True,
+    n_components=2,
+    init=None,
+    n_init="warn",
+    n_jobs=None,
+    max_iter=300,
+    verbose=0,
+    eps=1e-6,
+    random_state=None,
+    return_n_iter=False,
+    normalized_stress="auto",
+):
+    """Compute multidimensional scaling using the SMACOF algorithm.
 
     The SMACOF (Scaling by MAjorizing a COmplicated Function) algorithm is a
     multidimensional scaling algorithm which minimizes an objective function
@@ -145,7 +231,8 @@ def smacof(dissimilarities, *, metric=True, n_components=2, init=None,
     stress, and is more powerful than traditional techniques such as gradient
     descent.
 
-    The SMACOF algorithm for metric MDS can summarized by the following steps:
+    The SMACOF algorithm for metric MDS can be summarized by the following
+    steps:
 
     1. Set an initial start configuration, randomly or not.
     2. Compute the stress
@@ -157,11 +244,13 @@ def smacof(dissimilarities, *, metric=True, n_components=2, init=None,
 
     Parameters
     ----------
-    dissimilarities : ndarray of shape (n_samples, n_samples)
+    dissimilarities : array-like of shape (n_samples, n_samples)
         Pairwise dissimilarities between the points. Must be symmetric.
 
     metric : bool, default=True
         Compute metric or nonmetric SMACOF algorithm.
+        When ``False`` (i.e. non-metric MDS), dissimilarities with 0 are considered as
+        missing values.
 
     n_components : int, default=2
         Number of dimensions in which to immerse the dissimilarities. If an
@@ -169,7 +258,7 @@ def smacof(dissimilarities, *, metric=True, n_components=2, init=None,
         ``init`` is used to determine the dimensionality of the embedding
         space.
 
-    init : ndarray of shape (n_samples, n_components), default=None
+    init : array-like of shape (n_samples, n_components), default=None
         Starting configuration of the embedding to initialize the algorithm. By
         default, the algorithm is initialized with a randomly chosen array.
 
@@ -179,6 +268,9 @@ def smacof(dissimilarities, *, metric=True, n_components=2, init=None,
         determined by the run with the smallest final stress. If ``init`` is
         provided, this option is overridden and a single run is performed.
 
+        .. versionchanged:: 1.9
+           The default value for `n_iter` will change from 8 to 1 in version 1.9.
+
     n_jobs : int, default=None
         The number of jobs to use for the computation. If multiple
         initializations are used (``n_init``), each run of the algorithm is
@@ -194,18 +286,35 @@ def smacof(dissimilarities, *, metric=True, n_components=2, init=None,
     verbose : int, default=0
         Level of verbosity.
 
-    eps : float, default=1e-3
-        Relative tolerance with respect to stress at which to declare
-        convergence.
+    eps : float, default=1e-6
+        The tolerance with respect to stress (normalized by the sum of squared
+        embedding distances) at which to declare convergence.
+
+        .. versionchanged:: 1.7
+           The default value for `eps` has changed from 1e-3 to 1e-6, as a result
+           of a bugfix in the computation of the convergence criterion.
 
     random_state : int, RandomState instance or None, default=None
         Determines the random number generator used to initialize the centers.
         Pass an int for reproducible results across multiple function calls.
-        See :term: `Glossary `.
+        See :term:`Glossary `.
 
     return_n_iter : bool, default=False
         Whether or not to return the number of iterations.
 
+    normalized_stress : bool or "auto", default="auto"
+        Whether to return normalized stress value (Stress-1) instead of raw
+        stress. By default, metric MDS returns raw stress while non-metric MDS
+        returns normalized stress.
+
+        .. versionadded:: 1.2
+
+        .. versionchanged:: 1.4
+           The default value changed from `False` to `"auto"` in version 1.4.
+
+        .. versionchanged:: 1.7
+           Normalized stress is now supported for metric MDS as well.
+
     Returns
     -------
     X : ndarray of shape (n_samples, n_components)
@@ -214,33 +323,61 @@ def smacof(dissimilarities, *, metric=True, n_components=2, init=None,
     stress : float
         The final value of the stress (sum of squared distance of the
         disparities and the distances for all constrained points).
+        If `normalized_stress=True`, returns Stress-1.
+        A value of 0 indicates "perfect" fit, 0.025 excellent, 0.05 good,
+        0.1 fair, and 0.2 poor [1]_.
 
     n_iter : int
         The number of iterations corresponding to the best stress. Returned
         only if ``return_n_iter`` is set to ``True``.
 
-    Notes
-    -----
-    "Modern Multidimensional Scaling - Theory and Applications" Borg, I.;
-    Groenen P. Springer Series in Statistics (1997)
+    References
+    ----------
+    .. [1] "Nonmetric multidimensional scaling: a numerical method" Kruskal, J.
+           Psychometrika, 29 (1964)
+
+    .. [2] "Multidimensional scaling by optimizing goodness of fit to a nonmetric
+           hypothesis" Kruskal, J. Psychometrika, 29, (1964)
 
-    "Nonmetric multidimensional scaling: a numerical method" Kruskal, J.
-    Psychometrika, 29 (1964)
+    .. [3] "Modern Multidimensional Scaling - Theory and Applications" Borg, I.;
+           Groenen P. Springer Series in Statistics (1997)
 
-    "Multidimensional scaling by optimizing goodness of fit to a nonmetric
-    hypothesis" Kruskal, J. Psychometrika, 29, (1964)
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from sklearn.manifold import smacof
+    >>> from sklearn.metrics import euclidean_distances
+    >>> X = np.array([[0, 1, 2], [1, 0, 3], [2, 3, 0]])
+    >>> dissimilarities = euclidean_distances(X)
+    >>> Z, stress = smacof(
+    ...     dissimilarities, n_components=2, n_init=1, eps=1e-6, random_state=42
+    ... )
+    >>> Z.shape
+    (3, 2)
+    >>> np.round(stress, 6).item()
+    3.2e-05
     """
 
+    if n_init == "warn":
+        warnings.warn(
+            "The default value of `n_init` will change from 8 to 1 in 1.9.",
+            FutureWarning,
+        )
+        n_init = 8
+
     dissimilarities = check_array(dissimilarities)
     random_state = check_random_state(random_state)
 
-    if hasattr(init, '__array__'):
+    if normalized_stress == "auto":
+        normalized_stress = not metric
+
+    if hasattr(init, "__array__"):
         init = np.asarray(init).copy()
         if not n_init == 1:
             warnings.warn(
-                'Explicit initial positions passed: '
-                'performing only one init of the MDS instead of %d'
-                % n_init)
+                "Explicit initial positions passed: "
+                "performing only one init of the MDS instead of %d" % n_init
+            )
             n_init = 1
 
     best_pos, best_stress = None, None
@@ -248,10 +385,16 @@ def smacof(dissimilarities, *, metric=True, n_components=2, init=None,
     if effective_n_jobs(n_jobs) == 1:
         for it in range(n_init):
             pos, stress, n_iter_ = _smacof_single(
-                dissimilarities, metric=metric,
-                n_components=n_components, init=init,
-                max_iter=max_iter, verbose=verbose,
-                eps=eps, random_state=random_state)
+                dissimilarities,
+                metric=metric,
+                n_components=n_components,
+                init=init,
+                max_iter=max_iter,
+                verbose=verbose,
+                eps=eps,
+                random_state=random_state,
+                normalized_stress=normalized_stress,
+            )
             if best_stress is None or stress < best_stress:
                 best_stress = stress
                 best_pos = pos.copy()
@@ -260,10 +403,18 @@ def smacof(dissimilarities, *, metric=True, n_components=2, init=None,
         seeds = random_state.randint(np.iinfo(np.int32).max, size=n_init)
         results = Parallel(n_jobs=n_jobs, verbose=max(verbose - 1, 0))(
             delayed(_smacof_single)(
-                dissimilarities, metric=metric, n_components=n_components,
-                init=init, max_iter=max_iter, verbose=verbose, eps=eps,
-                random_state=seed)
-            for seed in seeds)
+                dissimilarities,
+                metric=metric,
+                n_components=n_components,
+                init=init,
+                max_iter=max_iter,
+                verbose=verbose,
+                eps=eps,
+                random_state=seed,
+                normalized_stress=normalized_stress,
+            )
+            for seed in seeds
+        )
         positions, stress, n_iters = zip(*results)
         best = np.argmin(stress)
         best_stress = stress[best]
@@ -276,6 +427,7 @@ def smacof(dissimilarities, *, metric=True, n_components=2, init=None,
         return best_pos, best_stress
 
 
+# TODO(1.9): change default `n_init` to 1, see PR #31117
 class MDS(BaseEstimator):
     """Multidimensional scaling.
 
@@ -288,21 +440,30 @@ class MDS(BaseEstimator):
 
     metric : bool, default=True
         If ``True``, perform metric MDS; otherwise, perform nonmetric MDS.
+        When ``False`` (i.e. non-metric MDS), dissimilarities with 0 are considered as
+        missing values.
 
     n_init : int, default=4
         Number of times the SMACOF algorithm will be run with different
         initializations. The final results will be the best output of the runs,
         determined by the run with the smallest final stress.
 
+        .. versionchanged:: 1.9
+           The default value for `n_init` will change from 4 to 1 in version 1.9.
+
     max_iter : int, default=300
         Maximum number of iterations of the SMACOF algorithm for a single run.
 
     verbose : int, default=0
         Level of verbosity.
 
-    eps : float, default=1e-3
-        Relative tolerance with respect to stress at which to declare
-        convergence.
+    eps : float, default=1e-6
+        The tolerance with respect to stress (normalized by the sum of squared
+        embedding distances) at which to declare convergence.
+
+        .. versionchanged:: 1.7
+           The default value for `eps` has changed from 1e-3 to 1e-6, as a result
+           of a bugfix in the computation of the convergence criterion.
 
     n_jobs : int, default=None
         The number of jobs to use for the computation. If multiple
@@ -316,7 +477,7 @@ class MDS(BaseEstimator):
     random_state : int, RandomState instance or None, default=None
         Determines the random number generator used to initialize the centers.
         Pass an int for reproducible results across multiple function calls.
-        See :term: `Glossary `.
+        See :term:`Glossary `.
 
     dissimilarity : {'euclidean', 'precomputed'}, default='euclidean'
         Dissimilarity measure to use:
@@ -328,6 +489,19 @@ class MDS(BaseEstimator):
             Pre-computed dissimilarities are passed directly to ``fit`` and
             ``fit_transform``.
 
+    normalized_stress : bool or "auto" default="auto"
+        Whether to return normalized stress value (Stress-1) instead of raw
+        stress. By default, metric MDS returns raw stress while non-metric MDS
+        returns normalized stress.
+
+        .. versionadded:: 1.2
+
+        .. versionchanged:: 1.4
+           The default value changed from `False` to `"auto"` in version 1.4.
+
+        .. versionchanged:: 1.7
+           Normalized stress is now supported for metric MDS as well.
+
     Attributes
     ----------
     embedding_ : ndarray of shape (n_samples, n_components)
@@ -336,6 +510,9 @@ class MDS(BaseEstimator):
     stress_ : float
         The final value of the stress (sum of squared distance of the
         disparities and the distances for all constrained points).
+        If `normalized_stress=True`, returns Stress-1.
+        A value of 0 indicates "perfect" fit, 0.025 excellent, 0.05 good,
+        0.1 fair, and 0.2 poor [1]_.
 
     dissimilarity_matrix_ : ndarray of shape (n_samples, n_samples)
         Pairwise dissimilarities between the points. Symmetric matrix that:
@@ -345,9 +522,42 @@ class MDS(BaseEstimator):
         - or constructs a dissimilarity matrix from data using
           Euclidean distances.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     n_iter_ : int
         The number of iterations corresponding to the best stress.
 
+    See Also
+    --------
+    sklearn.decomposition.PCA : Principal component analysis that is a linear
+        dimensionality reduction method.
+    sklearn.decomposition.KernelPCA : Non-linear dimensionality reduction using
+        kernels and PCA.
+    TSNE : T-distributed Stochastic Neighbor Embedding.
+    Isomap : Manifold learning based on Isometric Mapping.
+    LocallyLinearEmbedding : Manifold learning using Locally Linear Embedding.
+    SpectralEmbedding : Spectral embedding for non-linear dimensionality.
+
+    References
+    ----------
+    .. [1] "Nonmetric multidimensional scaling: a numerical method" Kruskal, J.
+       Psychometrika, 29 (1964)
+
+    .. [2] "Multidimensional scaling by optimizing goodness of fit to a nonmetric
+       hypothesis" Kruskal, J. Psychometrika, 29, (1964)
+
+    .. [3] "Modern Multidimensional Scaling - Theory and Applications" Borg, I.;
+       Groenen P. Springer Series in Statistics (1997)
+
     Examples
     --------
     >>> from sklearn.datasets import load_digits
@@ -355,27 +565,45 @@ class MDS(BaseEstimator):
     >>> X, _ = load_digits(return_X_y=True)
     >>> X.shape
     (1797, 64)
-    >>> embedding = MDS(n_components=2)
+    >>> embedding = MDS(n_components=2, n_init=1)
     >>> X_transformed = embedding.fit_transform(X[:100])
     >>> X_transformed.shape
     (100, 2)
 
-    References
-    ----------
-    "Modern Multidimensional Scaling - Theory and Applications" Borg, I.;
-    Groenen P. Springer Series in Statistics (1997)
-
-    "Nonmetric multidimensional scaling: a numerical method" Kruskal, J.
-    Psychometrika, 29 (1964)
-
-    "Multidimensional scaling by optimizing goodness of fit to a nonmetric
-    hypothesis" Kruskal, J. Psychometrika, 29, (1964)
+    For a more detailed example of usage, see
+    :ref:`sphx_glr_auto_examples_manifold_plot_mds.py`.
 
+    For a comparison of manifold learning techniques, see
+    :ref:`sphx_glr_auto_examples_manifold_plot_compare_methods.py`.
     """
-    @_deprecate_positional_args
-    def __init__(self, n_components=2, *, metric=True, n_init=4,
-                 max_iter=300, verbose=0, eps=1e-3, n_jobs=None,
-                 random_state=None, dissimilarity="euclidean"):
+
+    _parameter_constraints: dict = {
+        "n_components": [Interval(Integral, 1, None, closed="left")],
+        "metric": ["boolean"],
+        "n_init": [Interval(Integral, 1, None, closed="left"), StrOptions({"warn"})],
+        "max_iter": [Interval(Integral, 1, None, closed="left")],
+        "verbose": ["verbose"],
+        "eps": [Interval(Real, 0.0, None, closed="left")],
+        "n_jobs": [None, Integral],
+        "random_state": ["random_state"],
+        "dissimilarity": [StrOptions({"euclidean", "precomputed"})],
+        "normalized_stress": ["boolean", StrOptions({"auto"})],
+    }
+
+    def __init__(
+        self,
+        n_components=2,
+        *,
+        metric=True,
+        n_init="warn",
+        max_iter=300,
+        verbose=0,
+        eps=1e-6,
+        n_jobs=None,
+        random_state=None,
+        dissimilarity="euclidean",
+        normalized_stress="auto",
+    ):
         self.n_components = n_components
         self.dissimilarity = dissimilarity
         self.metric = metric
@@ -385,21 +613,16 @@ def __init__(self, n_components=2, *, metric=True, n_init=4,
         self.verbose = verbose
         self.n_jobs = n_jobs
         self.random_state = random_state
+        self.normalized_stress = normalized_stress
 
-    def _more_tags(self):
-        return {'pairwise': self.dissimilarity == 'precomputed'}
-
-    # TODO: Remove in 1.1
-    # mypy error: Decorated property not supported
-    @deprecated("Attribute _pairwise was deprecated in "  # type: ignore
-                "version 0.24 and will be removed in 1.1 (renaming of 0.26).")
-    @property
-    def _pairwise(self):
-        return self.dissimilarity == "precomputed"
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.pairwise = self.dissimilarity == "precomputed"
+        return tags
 
     def fit(self, X, y=None, init=None):
         """
-        Computes the position of the points in the embedding space.
+        Compute the position of the points in the embedding space.
 
         Parameters
         ----------
@@ -409,18 +632,25 @@ def fit(self, X, y=None, init=None):
             be the dissimilarity matrix.
 
         y : Ignored
+            Not used, present for API consistency by convention.
 
-        init : ndarray of shape (n_samples,), default=None
+        init : ndarray of shape (n_samples, n_components), default=None
             Starting configuration of the embedding to initialize the SMACOF
             algorithm. By default, the algorithm is initialized with a randomly
             chosen array.
+
+        Returns
+        -------
+        self : object
+            Fitted estimator.
         """
         self.fit_transform(X, init=init)
         return self
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit_transform(self, X, y=None, init=None):
         """
-        Fit the data from X, and returns the embedded coordinates.
+        Fit the data from `X`, and returns the embedded coordinates.
 
         Parameters
         ----------
@@ -430,32 +660,55 @@ def fit_transform(self, X, y=None, init=None):
             be the dissimilarity matrix.
 
         y : Ignored
+            Not used, present for API consistency by convention.
 
-        init : ndarray of shape (n_samples,), default=None
+        init : ndarray of shape (n_samples, n_components), default=None
             Starting configuration of the embedding to initialize the SMACOF
             algorithm. By default, the algorithm is initialized with a randomly
             chosen array.
+
+        Returns
+        -------
+        X_new : ndarray of shape (n_samples, n_components)
+            X transformed in the new space.
         """
-        X = self._validate_data(X)
+
+        if self.n_init == "warn":
+            warnings.warn(
+                "The default value of `n_init` will change from 4 to 1 in 1.9.",
+                FutureWarning,
+            )
+            self._n_init = 4
+        else:
+            self._n_init = self.n_init
+
+        X = validate_data(self, X)
         if X.shape[0] == X.shape[1] and self.dissimilarity != "precomputed":
-            warnings.warn("The MDS API has changed. ``fit`` now constructs an"
-                          " dissimilarity matrix from data. To use a custom "
-                          "dissimilarity matrix, set "
-                          "``dissimilarity='precomputed'``.")
+            warnings.warn(
+                "The MDS API has changed. ``fit`` now constructs a"
+                " dissimilarity matrix from data. To use a custom "
+                "dissimilarity matrix, set "
+                "``dissimilarity='precomputed'``."
+            )
 
         if self.dissimilarity == "precomputed":
             self.dissimilarity_matrix_ = X
         elif self.dissimilarity == "euclidean":
             self.dissimilarity_matrix_ = euclidean_distances(X)
-        else:
-            raise ValueError("Proximity must be 'precomputed' or 'euclidean'."
-                             " Got %s instead" % str(self.dissimilarity))
 
         self.embedding_, self.stress_, self.n_iter_ = smacof(
-            self.dissimilarity_matrix_, metric=self.metric,
-            n_components=self.n_components, init=init, n_init=self.n_init,
-            n_jobs=self.n_jobs, max_iter=self.max_iter, verbose=self.verbose,
-            eps=self.eps, random_state=self.random_state,
-            return_n_iter=True)
+            self.dissimilarity_matrix_,
+            metric=self.metric,
+            n_components=self.n_components,
+            init=init,
+            n_init=self._n_init,
+            n_jobs=self.n_jobs,
+            max_iter=self.max_iter,
+            verbose=self.verbose,
+            eps=self.eps,
+            random_state=self.random_state,
+            return_n_iter=True,
+            normalized_stress=self.normalized_stress,
+        )
 
         return self.embedding_
diff --git a/sklearn/manifold/_spectral_embedding.py b/sklearn/manifold/_spectral_embedding.py
index 76f52946e8e87..1a3b95e023897 100644
--- a/sklearn/manifold/_spectral_embedding.py
+++ b/sklearn/manifold/_spectral_embedding.py
@@ -1,32 +1,31 @@
 """Spectral Embedding."""
 
-# Author: Gael Varoquaux 
-#         Wei LI 
-# License: BSD 3 clause
-
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import warnings
+from numbers import Integral, Real
 
 import numpy as np
 from scipy import sparse
 from scipy.linalg import eigh
-from scipy.sparse.linalg import eigsh
 from scipy.sparse.csgraph import connected_components
-from scipy.sparse.csgraph import laplacian as csgraph_laplacian
+from scipy.sparse.linalg import eigsh, lobpcg
 
-from ..base import BaseEstimator
+from ..base import BaseEstimator, _fit_context
+from ..metrics.pairwise import rbf_kernel
+from ..neighbors import NearestNeighbors, kneighbors_graph
 from ..utils import (
     check_array,
     check_random_state,
     check_symmetric,
 )
 from ..utils._arpack import _init_arpack_v0
+from ..utils._param_validation import Interval, StrOptions, validate_params
 from ..utils.extmath import _deterministic_vector_sign_flip
-from ..utils.fixes import lobpcg
-from ..metrics.pairwise import rbf_kernel
-from ..neighbors import kneighbors_graph, NearestNeighbors
-from ..utils.validation import _deprecate_positional_args
-from ..utils.deprecation import deprecated
+from ..utils.fixes import laplacian as csgraph_laplacian
+from ..utils.fixes import parse_version, sp_version
+from ..utils.validation import validate_data
 
 
 def _graph_connected_component(graph, node_id):
@@ -65,7 +64,9 @@ def _graph_connected_component(graph, node_id):
         nodes_to_explore.fill(False)
         for i in indices:
             if sparse.issparse(graph):
-                neighbors = graph[i].toarray().ravel()
+                # scipy not yet implemented 1D sparse slices; can be changed back to
+                # `neighbors = graph[i].toarray().ravel()` once implemented
+                neighbors = graph[[i], :].toarray().ravel()
             else:
                 neighbors = graph[i]
             np.logical_or(nodes_to_explore, neighbors, out=nodes_to_explore)
@@ -73,7 +74,7 @@ def _graph_connected_component(graph, node_id):
 
 
 def _graph_is_connected(graph):
-    """ Return whether the graph is connected (True) or Not (False).
+    """Return whether the graph is connected (True) or Not (False).
 
     Parameters
     ----------
@@ -86,7 +87,16 @@ def _graph_is_connected(graph):
     is_connected : bool
         True means the graph is fully connected and False means not.
     """
-    if sparse.isspmatrix(graph):
+    if sparse.issparse(graph):
+        # Before Scipy 1.11.3, `connected_components` only supports 32-bit indices.
+        # PR: https://github.com/scipy/scipy/pull/18913
+        # First integration in 1.11.3: https://github.com/scipy/scipy/pull/19279
+        # TODO(jjerphan): Once SciPy 1.11.3 is the minimum supported version, use
+        # `accept_large_sparse=True`.
+        accept_large_sparse = sp_version >= parse_version("1.11.3")
+        graph = check_array(
+            graph, accept_sparse=True, accept_large_sparse=accept_large_sparse
+        )
         # sparse graph, find all the connected components
         n_connected_components, _ = connected_components(graph)
         return n_connected_components == 1
@@ -119,13 +129,13 @@ def _set_diag(laplacian, value, norm_laplacian):
     """
     n_nodes = laplacian.shape[0]
     # We need all entries in the diagonal to values
-    if not sparse.isspmatrix(laplacian):
+    if not sparse.issparse(laplacian):
         if norm_laplacian:
-            laplacian.flat[::n_nodes + 1] = value
+            laplacian.flat[:: n_nodes + 1] = value
     else:
         laplacian = laplacian.tocoo()
         if norm_laplacian:
-            diag_idx = (laplacian.row == laplacian.col)
+            diag_idx = laplacian.row == laplacian.col
             laplacian.data[diag_idx] = value
         # If the matrix has a small number of diagonals (as in the
         # case of structured matrices coming from images), the
@@ -141,10 +151,28 @@ def _set_diag(laplacian, value, norm_laplacian):
     return laplacian
 
 
-@_deprecate_positional_args
-def spectral_embedding(adjacency, *, n_components=8, eigen_solver=None,
-                       random_state=None, eigen_tol=0.0,
-                       norm_laplacian=True, drop_first=True):
+@validate_params(
+    {
+        "adjacency": ["array-like", "sparse matrix"],
+        "n_components": [Interval(Integral, 1, None, closed="left")],
+        "eigen_solver": [StrOptions({"arpack", "lobpcg", "amg"}), None],
+        "random_state": ["random_state"],
+        "eigen_tol": [Interval(Real, 0, None, closed="left"), StrOptions({"auto"})],
+        "norm_laplacian": ["boolean"],
+        "drop_first": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def spectral_embedding(
+    adjacency,
+    *,
+    n_components=8,
+    eigen_solver=None,
+    random_state=None,
+    eigen_tol="auto",
+    norm_laplacian=True,
+    drop_first=True,
+):
     """Project the sample on the first eigenvectors of the graph Laplacian.
 
     The adjacency matrix is used to compute a normalized graph Laplacian
@@ -180,17 +208,38 @@ def spectral_embedding(adjacency, *, n_components=8, eigen_solver=None,
         used.
 
     random_state : int, RandomState instance or None, default=None
-        Determines the random number generator used for the initialization of
-        the lobpcg eigenvectors decomposition when ``solver`` == 'amg'. Pass
-        an int for reproducible results across multiple function calls.
-        See :term: `Glossary `.
-
-    eigen_tol : float, default=0.0
-        Stopping criterion for eigendecomposition of the Laplacian matrix
-        when using arpack eigen_solver.
+        A pseudo random number generator used for the initialization
+        of the lobpcg eigen vectors decomposition when `eigen_solver ==
+        'amg'`, and for the K-Means initialization. Use an int to make
+        the results deterministic across calls (See
+        :term:`Glossary `).
+
+        .. note::
+            When using `eigen_solver == 'amg'`,
+            it is necessary to also fix the global numpy seed with
+            `np.random.seed(int)` to get deterministic results. See
+            https://github.com/pyamg/pyamg/issues/139 for further
+            information.
+
+    eigen_tol : float, default="auto"
+        Stopping criterion for eigendecomposition of the Laplacian matrix.
+        If `eigen_tol="auto"` then the passed tolerance will depend on the
+        `eigen_solver`:
+
+        - If `eigen_solver="arpack"`, then `eigen_tol=0.0`;
+        - If `eigen_solver="lobpcg"` or `eigen_solver="amg"`, then
+          `eigen_tol=None` which configures the underlying `lobpcg` solver to
+          automatically resolve the value according to their heuristics. See,
+          :func:`scipy.sparse.linalg.lobpcg` for details.
+
+        Note that when using `eigen_solver="amg"` values of `tol<1e-5` may lead
+        to convergence issues and should be avoided.
+
+        .. versionadded:: 1.2
+           Added 'auto' option.
 
     norm_laplacian : bool, default=True
-        If True, then compute normalized Laplacian.
+        If True, then compute symmetric normalized Laplacian.
 
     drop_first : bool, default=True
         Whether to drop the first eigenvector. For spectral embedding, this
@@ -213,28 +262,62 @@ def spectral_embedding(adjacency, *, n_components=8, eigen_solver=None,
     ----------
     * https://en.wikipedia.org/wiki/LOBPCG
 
-    * Toward the Optimal Preconditioned Eigensolver: Locally Optimal
-      Block Preconditioned Conjugate Gradient Method
+    * :doi:`"Toward the Optimal Preconditioned Eigensolver: Locally Optimal
+      Block Preconditioned Conjugate Gradient Method",
       Andrew V. Knyazev
-      https://doi.org/10.1137%2FS1064827500366124
+      <10.1137/S1064827500366124>`
+
+    Examples
+    --------
+    >>> from sklearn.datasets import load_digits
+    >>> from sklearn.neighbors import kneighbors_graph
+    >>> from sklearn.manifold import spectral_embedding
+    >>> X, _ = load_digits(return_X_y=True)
+    >>> X = X[:100]
+    >>> affinity_matrix = kneighbors_graph(
+    ...     X, n_neighbors=int(X.shape[0] / 10), include_self=True
+    ... )
+    >>> # make the matrix symmetric
+    >>> affinity_matrix = 0.5 * (affinity_matrix + affinity_matrix.T)
+    >>> embedding = spectral_embedding(affinity_matrix, n_components=2, random_state=42)
+    >>> embedding.shape
+    (100, 2)
     """
+    random_state = check_random_state(random_state)
+
+    return _spectral_embedding(
+        adjacency,
+        n_components=n_components,
+        eigen_solver=eigen_solver,
+        random_state=random_state,
+        eigen_tol=eigen_tol,
+        norm_laplacian=norm_laplacian,
+        drop_first=drop_first,
+    )
+
+
+def _spectral_embedding(
+    adjacency,
+    *,
+    n_components=8,
+    eigen_solver=None,
+    random_state=None,
+    eigen_tol="auto",
+    norm_laplacian=True,
+    drop_first=True,
+):
     adjacency = check_symmetric(adjacency)
 
-    try:
-        from pyamg import smoothed_aggregation_solver
-    except ImportError as e:
-        if eigen_solver == "amg":
-            raise ValueError("The eigen_solver was set to 'amg', but pyamg is "
-                             "not available.") from e
+    if eigen_solver == "amg":
+        try:
+            from pyamg import smoothed_aggregation_solver
+        except ImportError as e:
+            raise ValueError(
+                "The eigen_solver was set to 'amg', but pyamg is not available."
+            ) from e
 
     if eigen_solver is None:
-        eigen_solver = 'arpack'
-    elif eigen_solver not in ('arpack', 'lobpcg', 'amg'):
-        raise ValueError("Unknown value for eigen_solver: '%s'."
-                         "Should be 'amg', 'arpack', or 'lobpcg'"
-                         % eigen_solver)
-
-    random_state = check_random_state(random_state)
+        eigen_solver = "arpack"
 
     n_nodes = adjacency.shape[0]
     # Whether to drop the first eigenvector
@@ -242,13 +325,17 @@ def spectral_embedding(adjacency, *, n_components=8, eigen_solver=None,
         n_components = n_components + 1
 
     if not _graph_is_connected(adjacency):
-        warnings.warn("Graph is not fully connected, spectral embedding"
-                      " may not work as expected.")
-
-    laplacian, dd = csgraph_laplacian(adjacency, normed=norm_laplacian,
-                                      return_diag=True)
-    if (eigen_solver == 'arpack' or eigen_solver != 'lobpcg' and
-       (not sparse.isspmatrix(laplacian) or n_nodes < 5 * n_components)):
+        warnings.warn(
+            "Graph is not fully connected, spectral embedding may not work as expected."
+        )
+
+    laplacian, dd = csgraph_laplacian(
+        adjacency, normed=norm_laplacian, return_diag=True
+    )
+    if eigen_solver == "arpack" or (
+        eigen_solver != "lobpcg"
+        and (not sparse.issparse(laplacian) or n_nodes < 5 * n_components)
+    ):
         # lobpcg used with eigen_solver='amg' has bugs for low number of nodes
         # for details see the source code in scipy:
         # https://github.com/scipy/scipy/blob/v0.11.0/scipy/sparse/linalg/eigen
@@ -274,13 +361,18 @@ def spectral_embedding(adjacency, *, n_components=8, eigen_solver=None,
         try:
             # We are computing the opposite of the laplacian inplace so as
             # to spare a memory allocation of a possibly very large array
+            tol = 0 if eigen_tol == "auto" else eigen_tol
             laplacian *= -1
             v0 = _init_arpack_v0(laplacian.shape[0], random_state)
+            laplacian = check_array(
+                laplacian, accept_sparse="csr", accept_large_sparse=False
+            )
             _, diffusion_map = eigsh(
-                laplacian, k=n_components, sigma=1.0, which='LM',
-                tol=eigen_tol, v0=v0)
+                laplacian, k=n_components, sigma=1.0, which="LM", tol=tol, v0=v0
+            )
             embedding = diffusion_map.T[n_components::-1]
             if norm_laplacian:
+                # recover u = D^-1/2 x from the eigenvector output x
                 embedding = embedding / dd
         except RuntimeError:
             # When submatrices are exactly singular, an LU decomposition
@@ -289,14 +381,14 @@ def spectral_embedding(adjacency, *, n_components=8, eigen_solver=None,
             # Revert the laplacian to its opposite to have lobpcg work
             laplacian *= -1
 
-    elif eigen_solver == 'amg':
+    elif eigen_solver == "amg":
         # Use AMG to get a preconditioner and speed up the eigenvalue
         # problem.
         if not sparse.issparse(laplacian):
             warnings.warn("AMG works better for sparse matrices")
-        # lobpcg needs double precision floats
-        laplacian = check_array(laplacian, dtype=np.float64,
-                                accept_sparse=True)
+        laplacian = check_array(
+            laplacian, dtype=[np.float64, np.float32], accept_sparse=True
+        )
         laplacian = _set_diag(laplacian, 1, norm_laplacian)
 
         # The Laplacian matrix is always singular, having at least one zero
@@ -310,45 +402,60 @@ def spectral_embedding(adjacency, *, n_components=8, eigen_solver=None,
         # matrix to the solver and afterward set it back to the original.
         diag_shift = 1e-5 * sparse.eye(laplacian.shape[0])
         laplacian += diag_shift
-        ml = smoothed_aggregation_solver(check_array(laplacian,
-                                                     accept_sparse='csr'))
+        if hasattr(sparse, "csr_array") and isinstance(laplacian, sparse.csr_array):
+            # `pyamg` does not work with `csr_array` and we need to convert it to a
+            # `csr_matrix` object.
+            laplacian = sparse.csr_matrix(laplacian)
+        ml = smoothed_aggregation_solver(check_array(laplacian, accept_sparse="csr"))
         laplacian -= diag_shift
 
         M = ml.aspreconditioner()
-        X = random_state.rand(laplacian.shape[0], n_components + 1)
+        # Create initial approximation X to eigenvectors
+        X = random_state.standard_normal(size=(laplacian.shape[0], n_components + 1))
         X[:, 0] = dd.ravel()
-        _, diffusion_map = lobpcg(laplacian, X, M=M, tol=1.e-5,
-                                  largest=False)
+        X = X.astype(laplacian.dtype)
+
+        tol = None if eigen_tol == "auto" else eigen_tol
+        _, diffusion_map = lobpcg(laplacian, X, M=M, tol=tol, largest=False)
         embedding = diffusion_map.T
         if norm_laplacian:
+            # recover u = D^-1/2 x from the eigenvector output x
             embedding = embedding / dd
         if embedding.shape[0] == 1:
             raise ValueError
 
     if eigen_solver == "lobpcg":
-        # lobpcg needs double precision floats
-        laplacian = check_array(laplacian, dtype=np.float64,
-                                accept_sparse=True)
+        laplacian = check_array(
+            laplacian, dtype=[np.float64, np.float32], accept_sparse=True
+        )
         if n_nodes < 5 * n_components + 1:
             # see note above under arpack why lobpcg has problems with small
             # number of nodes
             # lobpcg will fallback to eigh, so we short circuit it
-            if sparse.isspmatrix(laplacian):
+            if sparse.issparse(laplacian):
                 laplacian = laplacian.toarray()
             _, diffusion_map = eigh(laplacian, check_finite=False)
             embedding = diffusion_map.T[:n_components]
             if norm_laplacian:
+                # recover u = D^-1/2 x from the eigenvector output x
                 embedding = embedding / dd
         else:
             laplacian = _set_diag(laplacian, 1, norm_laplacian)
             # We increase the number of eigenvectors requested, as lobpcg
-            # doesn't behave well in low dimension
-            X = random_state.rand(laplacian.shape[0], n_components + 1)
+            # doesn't behave well in low dimension and create initial
+            # approximation X to eigenvectors
+            X = random_state.standard_normal(
+                size=(laplacian.shape[0], n_components + 1)
+            )
             X[:, 0] = dd.ravel()
-            _, diffusion_map = lobpcg(laplacian, X, tol=1e-15,
-                                      largest=False, maxiter=2000)
+            X = X.astype(laplacian.dtype)
+            tol = None if eigen_tol == "auto" else eigen_tol
+            _, diffusion_map = lobpcg(
+                laplacian, X, tol=tol, largest=False, maxiter=2000
+            )
             embedding = diffusion_map.T[:n_components]
             if norm_laplacian:
+                # recover u = D^-1/2 x from the eigenvector output x
                 embedding = embedding / dd
             if embedding.shape[0] == 1:
                 raise ValueError
@@ -398,16 +505,41 @@ class SpectralEmbedding(BaseEstimator):
         1/n_features.
 
     random_state : int, RandomState instance or None, default=None
-        Determines the random number generator used for the initialization of
-        the lobpcg eigenvectors when ``solver`` == 'amg'.  Pass an int for
-        reproducible results across multiple function calls.
-        See :term: `Glossary `.
+        A pseudo random number generator used for the initialization
+        of the lobpcg eigen vectors decomposition when `eigen_solver ==
+        'amg'`, and for the K-Means initialization. Use an int to make
+        the results deterministic across calls (See
+        :term:`Glossary `).
+
+        .. note::
+            When using `eigen_solver == 'amg'`,
+            it is necessary to also fix the global numpy seed with
+            `np.random.seed(int)` to get deterministic results. See
+            https://github.com/pyamg/pyamg/issues/139 for further
+            information.
 
     eigen_solver : {'arpack', 'lobpcg', 'amg'}, default=None
         The eigenvalue decomposition strategy to use. AMG requires pyamg
         to be installed. It can be faster on very large, sparse problems.
         If None, then ``'arpack'`` is used.
 
+    eigen_tol : float, default="auto"
+        Stopping criterion for eigendecomposition of the Laplacian matrix.
+        If `eigen_tol="auto"` then the passed tolerance will depend on the
+        `eigen_solver`:
+
+        - If `eigen_solver="arpack"`, then `eigen_tol=0.0`;
+        - If `eigen_solver="lobpcg"` or `eigen_solver="amg"`, then
+          `eigen_tol=None` which configures the underlying `lobpcg` solver to
+          automatically resolve the value according to their heuristics. See,
+          :func:`scipy.sparse.linalg.lobpcg` for details.
+
+        Note that when using `eigen_solver="lobpcg"` or `eigen_solver="amg"`
+        values of `tol<1e-5` may lead to convergence issues and should be
+        avoided.
+
+        .. versionadded:: 1.2
+
     n_neighbors : int, default=None
         Number of nearest neighbors for nearest_neighbors graph building.
         If None, n_neighbors will be set to max(n_samples/10, 1).
@@ -426,9 +558,39 @@ class SpectralEmbedding(BaseEstimator):
     affinity_matrix_ : ndarray of shape (n_samples, n_samples)
         Affinity_matrix constructed from samples or precomputed.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     n_neighbors_ : int
         Number of nearest neighbors effectively used.
 
+    See Also
+    --------
+    Isomap : Non-linear dimensionality reduction through Isometric Mapping.
+
+    References
+    ----------
+
+    - :doi:`A Tutorial on Spectral Clustering, 2007
+      Ulrike von Luxburg
+      <10.1007/s11222-007-9033-z>`
+
+    - `On Spectral Clustering: Analysis and an algorithm, 2001
+      Andrew Y. Ng, Michael I. Jordan, Yair Weiss
+      `_
+
+    - :doi:`Normalized cuts and image segmentation, 2000
+      Jianbo Shi, Jitendra Malik
+      <10.1109/34.868688>`
+
     Examples
     --------
     >>> from sklearn.datasets import load_digits
@@ -440,54 +602,66 @@ class SpectralEmbedding(BaseEstimator):
     >>> X_transformed = embedding.fit_transform(X[:100])
     >>> X_transformed.shape
     (100, 2)
-
-    References
-    ----------
-
-    - A Tutorial on Spectral Clustering, 2007
-      Ulrike von Luxburg
-      http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.165.9323
-
-    - On Spectral Clustering: Analysis and an algorithm, 2001
-      Andrew Y. Ng, Michael I. Jordan, Yair Weiss
-      http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.19.8100
-
-    - Normalized cuts and image segmentation, 2000
-      Jianbo Shi, Jitendra Malik
-      http://citeseer.ist.psu.edu/viewdoc/summary?doi=10.1.1.160.2324
     """
-    @_deprecate_positional_args
-    def __init__(self, n_components=2, *, affinity="nearest_neighbors",
-                 gamma=None, random_state=None, eigen_solver=None,
-                 n_neighbors=None, n_jobs=None):
+
+    _parameter_constraints: dict = {
+        "n_components": [Interval(Integral, 1, None, closed="left")],
+        "affinity": [
+            StrOptions(
+                {
+                    "nearest_neighbors",
+                    "rbf",
+                    "precomputed",
+                    "precomputed_nearest_neighbors",
+                },
+            ),
+            callable,
+        ],
+        "gamma": [Interval(Real, 0, None, closed="left"), None],
+        "random_state": ["random_state"],
+        "eigen_solver": [StrOptions({"arpack", "lobpcg", "amg"}), None],
+        "eigen_tol": [Interval(Real, 0, None, closed="left"), StrOptions({"auto"})],
+        "n_neighbors": [Interval(Integral, 1, None, closed="left"), None],
+        "n_jobs": [None, Integral],
+    }
+
+    def __init__(
+        self,
+        n_components=2,
+        *,
+        affinity="nearest_neighbors",
+        gamma=None,
+        random_state=None,
+        eigen_solver=None,
+        eigen_tol="auto",
+        n_neighbors=None,
+        n_jobs=None,
+    ):
         self.n_components = n_components
         self.affinity = affinity
         self.gamma = gamma
         self.random_state = random_state
         self.eigen_solver = eigen_solver
+        self.eigen_tol = eigen_tol
         self.n_neighbors = n_neighbors
         self.n_jobs = n_jobs
 
-    def _more_tags(self):
-        return {'pairwise': self.affinity in ["precomputed",
-                                              "precomputed_nearest_neighbors"]}
-
-    # TODO: Remove in 1.1
-    # mypy error: Decorated property not supported
-    @deprecated("Attribute _pairwise was deprecated in "  # type: ignore
-                "version 0.24 and will be removed in 1.1 (renaming of 0.26).")
-    @property
-    def _pairwise(self):
-        return self.affinity in ["precomputed",
-                                 "precomputed_nearest_neighbors"]
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
+        tags.input_tags.pairwise = self.affinity in [
+            "precomputed",
+            "precomputed_nearest_neighbors",
+        ]
+        return tags
 
     def _get_affinity_matrix(self, X, Y=None):
         """Calculate the affinity matrix from data
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
-            Training vector, where n_samples is the number of samples
-            and n_features is the number of features.
+            Training vector, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
 
             If affinity is "precomputed"
             X : array-like of shape (n_samples, n_samples),
@@ -500,49 +674,54 @@ def _get_affinity_matrix(self, X, Y=None):
         -------
         affinity_matrix of shape (n_samples, n_samples)
         """
-        if self.affinity == 'precomputed':
+        if self.affinity == "precomputed":
             self.affinity_matrix_ = X
             return self.affinity_matrix_
-        if self.affinity == 'precomputed_nearest_neighbors':
-            estimator = NearestNeighbors(n_neighbors=self.n_neighbors,
-                                         n_jobs=self.n_jobs,
-                                         metric="precomputed").fit(X)
-            connectivity = estimator.kneighbors_graph(X=X, mode='connectivity')
+        if self.affinity == "precomputed_nearest_neighbors":
+            estimator = NearestNeighbors(
+                n_neighbors=self.n_neighbors, n_jobs=self.n_jobs, metric="precomputed"
+            ).fit(X)
+            connectivity = estimator.kneighbors_graph(X=X, mode="connectivity")
             self.affinity_matrix_ = 0.5 * (connectivity + connectivity.T)
             return self.affinity_matrix_
-        if self.affinity == 'nearest_neighbors':
+        if self.affinity == "nearest_neighbors":
             if sparse.issparse(X):
-                warnings.warn("Nearest neighbors affinity currently does "
-                              "not support sparse input, falling back to "
-                              "rbf affinity")
+                warnings.warn(
+                    "Nearest neighbors affinity currently does "
+                    "not support sparse input, falling back to "
+                    "rbf affinity"
+                )
                 self.affinity = "rbf"
             else:
-                self.n_neighbors_ = (self.n_neighbors
-                                     if self.n_neighbors is not None
-                                     else max(int(X.shape[0] / 10), 1))
-                self.affinity_matrix_ = kneighbors_graph(X, self.n_neighbors_,
-                                                         include_self=True,
-                                                         n_jobs=self.n_jobs)
+                self.n_neighbors_ = (
+                    self.n_neighbors
+                    if self.n_neighbors is not None
+                    else max(int(X.shape[0] / 10), 1)
+                )
+                self.affinity_matrix_ = kneighbors_graph(
+                    X, self.n_neighbors_, include_self=True, n_jobs=self.n_jobs
+                )
                 # currently only symmetric affinity_matrix supported
-                self.affinity_matrix_ = 0.5 * (self.affinity_matrix_ +
-                                               self.affinity_matrix_.T)
+                self.affinity_matrix_ = 0.5 * (
+                    self.affinity_matrix_ + self.affinity_matrix_.T
+                )
                 return self.affinity_matrix_
-        if self.affinity == 'rbf':
-            self.gamma_ = (self.gamma
-                           if self.gamma is not None else 1.0 / X.shape[1])
+        if self.affinity == "rbf":
+            self.gamma_ = self.gamma if self.gamma is not None else 1.0 / X.shape[1]
             self.affinity_matrix_ = rbf_kernel(X, gamma=self.gamma_)
             return self.affinity_matrix_
         self.affinity_matrix_ = self.affinity(X)
         return self.affinity_matrix_
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y=None):
         """Fit the model from data in X.
 
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
-            Training vector, where n_samples is the number of samples
-            and n_features is the number of features.
+            Training vector, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
 
             If affinity is "precomputed"
             X : {array-like, sparse matrix}, shape (n_samples, n_samples),
@@ -550,32 +729,25 @@ def fit(self, X, y=None):
             samples.
 
         y : Ignored
+            Not used, present for API consistency by convention.
 
         Returns
         -------
         self : object
             Returns the instance itself.
         """
-
-        X = self._validate_data(X, accept_sparse='csr', ensure_min_samples=2,
-                                estimator=self)
+        X = validate_data(self, X, accept_sparse="csr", ensure_min_samples=2)
 
         random_state = check_random_state(self.random_state)
-        if isinstance(self.affinity, str):
-            if self.affinity not in {"nearest_neighbors", "rbf", "precomputed",
-                                     "precomputed_nearest_neighbors"}:
-                raise ValueError(("%s is not a valid affinity. Expected "
-                                  "'precomputed', 'rbf', 'nearest_neighbors' "
-                                  "or a callable.") % self.affinity)
-        elif not callable(self.affinity):
-            raise ValueError(("'affinity' is expected to be an affinity "
-                              "name or a callable. Got: %s") % self.affinity)
 
         affinity_matrix = self._get_affinity_matrix(X)
-        self.embedding_ = spectral_embedding(affinity_matrix,
-                                             n_components=self.n_components,
-                                             eigen_solver=self.eigen_solver,
-                                             random_state=random_state)
+        self.embedding_ = _spectral_embedding(
+            affinity_matrix,
+            n_components=self.n_components,
+            eigen_solver=self.eigen_solver,
+            eigen_tol=self.eigen_tol,
+            random_state=random_state,
+        )
         return self
 
     def fit_transform(self, X, y=None):
@@ -584,8 +756,8 @@ def fit_transform(self, X, y=None):
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
-            Training vector, where n_samples is the number of samples
-            and n_features is the number of features.
+            Training vector, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
 
             If affinity is "precomputed"
             X : {array-like, sparse matrix} of shape (n_samples, n_samples),
@@ -593,10 +765,12 @@ def fit_transform(self, X, y=None):
             samples.
 
         y : Ignored
+            Not used, present for API consistency by convention.
 
         Returns
         -------
         X_new : array-like of shape (n_samples, n_components)
+            Spectral embedding of the training matrix.
         """
         self.fit(X)
         return self.embedding_
diff --git a/sklearn/manifold/_t_sne.py b/sklearn/manifold/_t_sne.py
index b6072a6e198c4..51882a5b38abd 100644
--- a/sklearn/manifold/_t_sne.py
+++ b/sklearn/manifold/_t_sne.py
@@ -1,33 +1,36 @@
-# Author: Alexander Fabisch  -- 
-# Author: Christopher Moody 
-# Author: Nick Travers 
-# License: BSD 3 clause (C) 2014
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 # This is the exact and Barnes-Hut t-SNE implementation. There are other
 # modifications of the algorithm:
 # * Fast Optimization for t-SNE:
 #   https://cseweb.ucsd.edu/~lvdmaaten/workshops/nips2010/papers/vandermaaten.pdf
 
-import warnings
+from numbers import Integral, Real
 from time import time
+
 import numpy as np
 from scipy import linalg
-from scipy.spatial.distance import pdist
-from scipy.spatial.distance import squareform
 from scipy.sparse import csr_matrix, issparse
+from scipy.spatial.distance import pdist, squareform
+
+from ..base import (
+    BaseEstimator,
+    ClassNamePrefixFeaturesOutMixin,
+    TransformerMixin,
+    _fit_context,
+)
+from ..decomposition import PCA
+from ..metrics.pairwise import _VALID_METRICS, pairwise_distances
 from ..neighbors import NearestNeighbors
-from ..base import BaseEstimator
 from ..utils import check_random_state
 from ..utils._openmp_helpers import _openmp_effective_n_threads
-from ..utils.validation import check_non_negative
-from ..utils.validation import _deprecate_positional_args
-from ..decomposition import PCA
-from ..metrics.pairwise import pairwise_distances
+from ..utils._param_validation import Interval, StrOptions, validate_params
+from ..utils.validation import _num_samples, check_non_negative, validate_data
+
 # mypy error: Module 'sklearn.manifold' has no attribute '_utils'
-from . import _utils  # type: ignore
 # mypy error: Module 'sklearn.manifold' has no attribute '_barnes_hut_tsne'
-from . import _barnes_hut_tsne  # type: ignore
-
+from . import _barnes_hut_tsne, _utils  # type: ignore[attr-defined]
 
 MACHINE_EPSILON = np.finfo(np.double).eps
 
@@ -57,7 +60,8 @@ def _joint_probabilities(distances, desired_perplexity, verbose):
     # the desired perplexity
     distances = distances.astype(np.float32, copy=False)
     conditional_P = _utils._binary_search_perplexity(
-        distances, desired_perplexity, verbose)
+        distances, desired_perplexity, verbose
+    )
     P = conditional_P + conditional_P.T
     sum_P = np.maximum(np.sum(P), MACHINE_EPSILON)
     P = np.maximum(squareform(P) / sum_P, MACHINE_EPSILON)
@@ -99,14 +103,15 @@ def _joint_probabilities_nn(distances, desired_perplexity, verbose):
     distances_data = distances.data.reshape(n_samples, -1)
     distances_data = distances_data.astype(np.float32, copy=False)
     conditional_P = _utils._binary_search_perplexity(
-        distances_data, desired_perplexity, verbose)
-    assert np.all(np.isfinite(conditional_P)), \
-        "All probabilities should be finite"
+        distances_data, desired_perplexity, verbose
+    )
+    assert np.all(np.isfinite(conditional_P)), "All probabilities should be finite"
 
     # Symmetrize the joint probability distribution using sparse operations
-    P = csr_matrix((conditional_P.ravel(), distances.indices,
-                    distances.indptr),
-                   shape=(n_samples, n_samples))
+    P = csr_matrix(
+        (conditional_P.ravel(), distances.indices, distances.indptr),
+        shape=(n_samples, n_samples),
+    )
     P = P + P.T
 
     # Normalize the joint probability distribution
@@ -116,13 +121,19 @@ def _joint_probabilities_nn(distances, desired_perplexity, verbose):
     assert np.all(np.abs(P.data) <= 1.0)
     if verbose >= 2:
         duration = time() - t0
-        print("[t-SNE] Computed conditional probabilities in {:.3f}s"
-              .format(duration))
+        print("[t-SNE] Computed conditional probabilities in {:.3f}s".format(duration))
     return P
 
 
-def _kl_divergence(params, P, degrees_of_freedom, n_samples, n_components,
-                   skip_num_points=0, compute_error=True):
+def _kl_divergence(
+    params,
+    P,
+    degrees_of_freedom,
+    n_samples,
+    n_components,
+    skip_num_points=0,
+    compute_error=True,
+):
     """t-SNE objective function: gradient of the KL divergence
     of p_ijs and q_ijs and the absolute error.
 
@@ -165,7 +176,7 @@ def _kl_divergence(params, P, degrees_of_freedom, n_samples, n_components,
     # Q is a heavy-tailed distribution: Student's t-distribution
     dist = pdist(X_embedded, "sqeuclidean")
     dist /= degrees_of_freedom
-    dist += 1.
+    dist += 1.0
     dist **= (degrees_of_freedom + 1.0) / -2.0
     Q = np.maximum(dist / (2.0 * np.sum(dist)), MACHINE_EPSILON)
 
@@ -174,8 +185,7 @@ def _kl_divergence(params, P, degrees_of_freedom, n_samples, n_components,
 
     # Objective: C (Kullback-Leibler divergence of P and Q)
     if compute_error:
-        kl_divergence = 2.0 * np.dot(
-            P, np.log(np.maximum(P, MACHINE_EPSILON) / Q))
+        kl_divergence = 2.0 * np.dot(P, np.log(np.maximum(P, MACHINE_EPSILON) / Q))
     else:
         kl_divergence = np.nan
 
@@ -184,8 +194,7 @@ def _kl_divergence(params, P, degrees_of_freedom, n_samples, n_components,
     grad = np.ndarray((n_samples, n_components), dtype=params.dtype)
     PQd = squareform((P - Q) * dist)
     for i in range(skip_num_points, n_samples):
-        grad[i] = np.dot(np.ravel(PQd[i], order='K'),
-                         X_embedded[i] - X_embedded)
+        grad[i] = np.dot(np.ravel(PQd[i], order="K"), X_embedded[i] - X_embedded)
     grad = grad.ravel()
     c = 2.0 * (degrees_of_freedom + 1.0) / degrees_of_freedom
     grad *= c
@@ -193,9 +202,18 @@ def _kl_divergence(params, P, degrees_of_freedom, n_samples, n_components,
     return kl_divergence, grad
 
 
-def _kl_divergence_bh(params, P, degrees_of_freedom, n_samples, n_components,
-                      angle=0.5, skip_num_points=0, verbose=False,
-                      compute_error=True, num_threads=1):
+def _kl_divergence_bh(
+    params,
+    P,
+    degrees_of_freedom,
+    n_samples,
+    n_components,
+    angle=0.5,
+    skip_num_points=0,
+    verbose=False,
+    compute_error=True,
+    num_threads=1,
+):
     """t-SNE objective function: KL divergence of p_ijs and q_ijs.
 
     Uses Barnes-Hut tree methods to calculate the gradient that
@@ -260,11 +278,19 @@ def _kl_divergence_bh(params, P, degrees_of_freedom, n_samples, n_components,
     indptr = P.indptr.astype(np.int64, copy=False)
 
     grad = np.zeros(X_embedded.shape, dtype=np.float32)
-    error = _barnes_hut_tsne.gradient(val_P, X_embedded, neighbors, indptr,
-                                      grad, angle, n_components, verbose,
-                                      dof=degrees_of_freedom,
-                                      compute_error=compute_error,
-                                      num_threads=num_threads)
+    error = _barnes_hut_tsne.gradient(
+        val_P,
+        X_embedded,
+        neighbors,
+        indptr,
+        grad,
+        angle,
+        n_components,
+        verbose,
+        dof=degrees_of_freedom,
+        compute_error=compute_error,
+        num_threads=num_threads,
+    )
     c = 2.0 * (degrees_of_freedom + 1.0) / degrees_of_freedom
     grad = grad.ravel()
     grad *= c
@@ -272,10 +298,21 @@ def _kl_divergence_bh(params, P, degrees_of_freedom, n_samples, n_components,
     return error, grad
 
 
-def _gradient_descent(objective, p0, it, n_iter,
-                      n_iter_check=1, n_iter_without_progress=300,
-                      momentum=0.8, learning_rate=200.0, min_gain=0.01,
-                      min_grad_norm=1e-7, verbose=0, args=None, kwargs=None):
+def _gradient_descent(
+    objective,
+    p0,
+    it,
+    max_iter,
+    n_iter_check=1,
+    n_iter_without_progress=300,
+    momentum=0.8,
+    learning_rate=200.0,
+    min_gain=0.01,
+    min_grad_norm=1e-7,
+    verbose=0,
+    args=None,
+    kwargs=None,
+):
     """Batch gradient descent with momentum and individual gains.
 
     Parameters
@@ -293,7 +330,7 @@ def _gradient_descent(objective, p0, it, n_iter,
         Current number of iterations (this function will be called more than
         once during the optimization).
 
-    n_iter : int
+    max_iter : int
         Maximum number of gradient descent iterations.
 
     n_iter_check : int, default=1
@@ -355,13 +392,12 @@ def _gradient_descent(objective, p0, it, n_iter,
     best_iter = i = it
 
     tic = time()
-    for i in range(it, n_iter):
+    for i in range(it, max_iter):
         check_convergence = (i + 1) % n_iter_check == 0
         # only compute the error when needed
-        kwargs['compute_error'] = check_convergence or i == n_iter - 1
+        kwargs["compute_error"] = check_convergence or i == max_iter - 1
 
         error, grad = objective(p, *args, **kwargs)
-        grad_norm = linalg.norm(grad)
 
         inc = update * grad < 0.0
         dec = np.invert(inc)
@@ -376,34 +412,49 @@ def _gradient_descent(objective, p0, it, n_iter,
             toc = time()
             duration = toc - tic
             tic = toc
+            grad_norm = linalg.norm(grad)
 
             if verbose >= 2:
-                print("[t-SNE] Iteration %d: error = %.7f,"
-                      " gradient norm = %.7f"
-                      " (%s iterations in %0.3fs)"
-                      % (i + 1, error, grad_norm, n_iter_check, duration))
+                print(
+                    "[t-SNE] Iteration %d: error = %.7f,"
+                    " gradient norm = %.7f"
+                    " (%s iterations in %0.3fs)"
+                    % (i + 1, error, grad_norm, n_iter_check, duration)
+                )
 
             if error < best_error:
                 best_error = error
                 best_iter = i
             elif i - best_iter > n_iter_without_progress:
                 if verbose >= 2:
-                    print("[t-SNE] Iteration %d: did not make any progress "
-                          "during the last %d episodes. Finished."
-                          % (i + 1, n_iter_without_progress))
+                    print(
+                        "[t-SNE] Iteration %d: did not make any progress "
+                        "during the last %d episodes. Finished."
+                        % (i + 1, n_iter_without_progress)
+                    )
                 break
             if grad_norm <= min_grad_norm:
                 if verbose >= 2:
-                    print("[t-SNE] Iteration %d: gradient norm %f. Finished."
-                          % (i + 1, grad_norm))
+                    print(
+                        "[t-SNE] Iteration %d: gradient norm %f. Finished."
+                        % (i + 1, grad_norm)
+                    )
                 break
 
     return p, error, i
 
 
-@_deprecate_positional_args
-def trustworthiness(X, X_embedded, *, n_neighbors=5, metric='euclidean'):
-    r"""Expresses to what extent the local structure is retained.
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "X_embedded": ["array-like", "sparse matrix"],
+        "n_neighbors": [Interval(Integral, 1, None, closed="left")],
+        "metric": [StrOptions(set(_VALID_METRICS) | {"precomputed"}), callable],
+    },
+    prefer_skip_nested_validation=True,
+)
+def trustworthiness(X, X_embedded, *, n_neighbors=5, metric="euclidean"):
+    r"""Indicate to what extent the local structure is retained.
 
     The trustworthiness is within [0, 1]. It is defined as
 
@@ -418,30 +469,29 @@ def trustworthiness(X, X_embedded, *, n_neighbors=5, metric='euclidean'):
     neighbors in the output space are penalised in proportion to their rank in
     the input space.
 
-    * "Neighborhood Preservation in Nonlinear Projection Methods: An
-      Experimental Study"
-      J. Venna, S. Kaski
-    * "Learning a Parametric Embedding by Preserving Local Structure"
-      L.J.P. van der Maaten
-
     Parameters
     ----------
-    X : ndarray of shape (n_samples, n_features) or (n_samples, n_samples)
+    X : {array-like, sparse matrix} of shape (n_samples, n_features) or \
+        (n_samples, n_samples)
         If the metric is 'precomputed' X must be a square distance
         matrix. Otherwise it contains a sample per row.
 
-    X_embedded : ndarray of shape (n_samples, n_components)
+    X_embedded : {array-like, sparse matrix} of shape (n_samples, n_components)
         Embedding of the training data in low-dimensional space.
 
     n_neighbors : int, default=5
-        Number of neighbors k that will be considered.
+        The number of neighbors that will be considered. Should be fewer than
+        `n_samples / 2` to ensure the trustworthiness to lies within [0, 1], as
+        mentioned in [1]_. An error will be raised otherwise.
 
     metric : str or callable, default='euclidean'
         Which metric to use for computing pairwise distances between samples
         from the original input space. If metric is 'precomputed', X must be a
-        matrix of pairwise distances or squared distances. Otherwise, see the
-        documentation of argument metric in sklearn.pairwise.pairwise_distances
-        for a list of available metrics.
+        matrix of pairwise distances or squared distances. Otherwise, for a list
+        of available metrics, see the documentation of argument metric in
+        `sklearn.pairwise.pairwise_distances` and metrics listed in
+        `sklearn.metrics.pairwise.PAIRWISE_DISTANCE_FUNCTIONS`. Note that the
+        "cosine" metric uses :func:`~sklearn.metrics.pairwise.cosine_distances`.
 
         .. versionadded:: 0.20
 
@@ -449,36 +499,66 @@ def trustworthiness(X, X_embedded, *, n_neighbors=5, metric='euclidean'):
     -------
     trustworthiness : float
         Trustworthiness of the low-dimensional embedding.
+
+    References
+    ----------
+    .. [1] Jarkko Venna and Samuel Kaski. 2001. Neighborhood
+           Preservation in Nonlinear Projection Methods: An Experimental Study.
+           In Proceedings of the International Conference on Artificial Neural Networks
+           (ICANN '01). Springer-Verlag, Berlin, Heidelberg, 485-491.
+
+    .. [2] Laurens van der Maaten. Learning a Parametric Embedding by Preserving
+           Local Structure. Proceedings of the Twelfth International Conference on
+           Artificial Intelligence and Statistics, PMLR 5:384-391, 2009.
+
+    Examples
+    --------
+    >>> from sklearn.datasets import make_blobs
+    >>> from sklearn.decomposition import PCA
+    >>> from sklearn.manifold import trustworthiness
+    >>> X, _ = make_blobs(n_samples=100, n_features=10, centers=3, random_state=42)
+    >>> X_embedded = PCA(n_components=2).fit_transform(X)
+    >>> print(f"{trustworthiness(X, X_embedded, n_neighbors=5):.2f}")
+    0.92
     """
+    n_samples = _num_samples(X)
+    if n_neighbors >= n_samples / 2:
+        raise ValueError(
+            f"n_neighbors ({n_neighbors}) should be less than n_samples / 2"
+            f" ({n_samples / 2})"
+        )
     dist_X = pairwise_distances(X, metric=metric)
-    if metric == 'precomputed':
+    if metric == "precomputed":
         dist_X = dist_X.copy()
     # we set the diagonal to np.inf to exclude the points themselves from
     # their own neighborhood
     np.fill_diagonal(dist_X, np.inf)
     ind_X = np.argsort(dist_X, axis=1)
     # `ind_X[i]` is the index of sorted distances between i and other samples
-    ind_X_embedded = NearestNeighbors(n_neighbors=n_neighbors).fit(
-            X_embedded).kneighbors(return_distance=False)
+    ind_X_embedded = (
+        NearestNeighbors(n_neighbors=n_neighbors)
+        .fit(X_embedded)
+        .kneighbors(return_distance=False)
+    )
 
     # We build an inverted index of neighbors in the input space: For sample i,
     # we define `inverted_index[i]` as the inverted index of sorted distances:
     # inverted_index[i][ind_X[i]] = np.arange(1, n_sample + 1)
-    n_samples = X.shape[0]
     inverted_index = np.zeros((n_samples, n_samples), dtype=int)
     ordered_indices = np.arange(n_samples + 1)
-    inverted_index[ordered_indices[:-1, np.newaxis],
-                   ind_X] = ordered_indices[1:]
-    ranks = inverted_index[ordered_indices[:-1, np.newaxis],
-                           ind_X_embedded] - n_neighbors
+    inverted_index[ordered_indices[:-1, np.newaxis], ind_X] = ordered_indices[1:]
+    ranks = (
+        inverted_index[ordered_indices[:-1, np.newaxis], ind_X_embedded] - n_neighbors
+    )
     t = np.sum(ranks[ranks > 0])
-    t = 1.0 - t * (2.0 / (n_samples * n_neighbors *
-                          (2.0 * n_samples - 3.0 * n_neighbors - 1.0)))
+    t = 1.0 - t * (
+        2.0 / (n_samples * n_neighbors * (2.0 * n_samples - 3.0 * n_neighbors - 1.0))
+    )
     return t
 
 
-class TSNE(BaseEstimator):
-    """t-distributed Stochastic Neighbor Embedding.
+class TSNE(ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator):
+    """T-distributed Stochastic Neighbor Embedding.
 
     t-SNE [1] is a tool to visualize high-dimensional data. It converts
     similarities between data points to joint probabilities and tries
@@ -506,7 +586,8 @@ class TSNE(BaseEstimator):
         is used in other manifold learning algorithms. Larger datasets
         usually require a larger perplexity. Consider selecting a value
         between 5 and 50. Different values can result in significantly
-        different results.
+        different results. The perplexity must be less than the number
+        of samples.
 
     early_exaggeration : float, default=12.0
         Controls how tight natural clusters in the original space are in
@@ -517,18 +598,30 @@ class TSNE(BaseEstimator):
         optimization, the early exaggeration factor or the learning rate
         might be too high.
 
-    learning_rate : float, default=200.0
+    learning_rate : float or "auto", default="auto"
         The learning rate for t-SNE is usually in the range [10.0, 1000.0]. If
         the learning rate is too high, the data may look like a 'ball' with any
         point approximately equidistant from its nearest neighbours. If the
         learning rate is too low, most points may look compressed in a dense
         cloud with few outliers. If the cost function gets stuck in a bad local
         minimum increasing the learning rate may help.
+        Note that many other t-SNE implementations (bhtsne, FIt-SNE, openTSNE,
+        etc.) use a definition of learning_rate that is 4 times smaller than
+        ours. So our learning_rate=200 corresponds to learning_rate=800 in
+        those other implementations. The 'auto' option sets the learning_rate
+        to `max(N / early_exaggeration / 4, 50)` where N is the sample size,
+        following [4] and [5].
 
-    n_iter : int, default=1000
+        .. versionchanged:: 1.2
+           The default value changed to `"auto"`.
+
+    max_iter : int, default=1000
         Maximum number of iterations for the optimization. Should be at
         least 250.
 
+        .. versionchanged:: 1.5
+            Parameter name changed from `n_iter` to `max_iter`.
+
     n_iter_without_progress : int, default=300
         Maximum number of iterations without progress before we abort the
         optimization, used after 250 initial iterations with early
@@ -554,13 +647,20 @@ class TSNE(BaseEstimator):
         the distance between them. The default is "euclidean" which is
         interpreted as squared euclidean distance.
 
-    init : {'random', 'pca'} or ndarray of shape (n_samples, n_components), \
-            default='random'
-        Initialization of embedding. Possible options are 'random', 'pca',
-        and a numpy array of shape (n_samples, n_components).
+    metric_params : dict, default=None
+        Additional keyword arguments for the metric function.
+
+        .. versionadded:: 1.1
+
+    init : {"random", "pca"} or ndarray of shape (n_samples, n_components), \
+            default="pca"
+        Initialization of embedding.
         PCA initialization cannot be used with precomputed distances and is
         usually more globally stable than random initialization.
 
+        .. versionchanged:: 1.2
+           The default value changed to `"pca"`.
+
     verbose : int, default=0
         Verbosity level.
 
@@ -568,9 +668,9 @@ class TSNE(BaseEstimator):
         Determines the random number generator. Pass an int for reproducible
         results across multiple function calls. Note that different
         initializations might result in different local minima of the cost
-        function. See :term: `Glossary `.
+        function. See :term:`Glossary `.
 
-    method : str, default='barnes_hut'
+    method : {'barnes_hut', 'exact'}, default='barnes_hut'
         By default the gradient calculation algorithm uses Barnes-Hut
         approximation running in O(NlogN) time. method='exact'
         will run on the slower, but exact, algorithm in O(N^2) time. The
@@ -601,19 +701,6 @@ class TSNE(BaseEstimator):
 
         .. versionadded:: 0.22
 
-    square_distances : True or 'legacy', default='legacy'
-        Whether TSNE should square the distance values. ``'legacy'`` means
-        that distance values are squared only when ``metric="euclidean"``.
-        ``True`` means that distance values are squared for all metrics.
-
-        .. versionadded:: 0.24
-           Added to provide backward compatibility during deprecation of
-           legacy squaring behavior.
-        .. deprecated:: 0.24
-           Legacy squaring behavior was deprecated in 0.24. The ``'legacy'``
-           value will be removed in 1.1 (renaming of 0.26), at which point the
-           default value will change to ``True``.
-
     Attributes
     ----------
     embedding_ : array-like of shape (n_samples, n_components)
@@ -622,18 +709,41 @@ class TSNE(BaseEstimator):
     kl_divergence_ : float
         Kullback-Leibler divergence after optimization.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    learning_rate_ : float
+        Effective learning rate.
+
+        .. versionadded:: 1.2
+
     n_iter_ : int
         Number of iterations run.
 
-    Examples
+    See Also
     --------
-
-    >>> import numpy as np
-    >>> from sklearn.manifold import TSNE
-    >>> X = np.array([[0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 1]])
-    >>> X_embedded = TSNE(n_components=2).fit_transform(X)
-    >>> X_embedded.shape
-    (4, 2)
+    sklearn.decomposition.PCA : Principal component analysis that is a linear
+        dimensionality reduction method.
+    sklearn.decomposition.KernelPCA : Non-linear dimensionality reduction using
+        kernels and PCA.
+    MDS : Manifold learning using multidimensional scaling.
+    Isomap : Manifold learning based on Isometric Mapping.
+    LocallyLinearEmbedding : Manifold learning using Locally Linear Embedding.
+    SpectralEmbedding : Spectral embedding for non-linear dimensionality.
+
+    Notes
+    -----
+    For an example of using :class:`~sklearn.manifold.TSNE` in combination with
+    :class:`~sklearn.neighbors.KNeighborsTransformer` see
+    :ref:`sphx_glr_auto_examples_neighbors_approximate_nearest_neighbors.py`.
 
     References
     ----------
@@ -647,93 +757,161 @@ class TSNE(BaseEstimator):
     [3] L.J.P. van der Maaten. Accelerating t-SNE using Tree-Based Algorithms.
         Journal of Machine Learning Research 15(Oct):3221-3245, 2014.
         https://lvdmaaten.github.io/publications/papers/JMLR_2014.pdf
+
+    [4] Belkina, A. C., Ciccolella, C. O., Anno, R., Halpert, R., Spidlen, J.,
+        & Snyder-Cappione, J. E. (2019). Automated optimized parameters for
+        T-distributed stochastic neighbor embedding improve visualization
+        and analysis of large datasets. Nature Communications, 10(1), 1-12.
+
+    [5] Kobak, D., & Berens, P. (2019). The art of using t-SNE for single-cell
+        transcriptomics. Nature Communications, 10(1), 1-14.
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from sklearn.manifold import TSNE
+    >>> X = np.array([[0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 1]])
+    >>> X_embedded = TSNE(n_components=2, learning_rate='auto',
+    ...                   init='random', perplexity=3).fit_transform(X)
+    >>> X_embedded.shape
+    (4, 2)
     """
+
+    _parameter_constraints: dict = {
+        "n_components": [Interval(Integral, 1, None, closed="left")],
+        "perplexity": [Interval(Real, 0, None, closed="neither")],
+        "early_exaggeration": [Interval(Real, 1, None, closed="left")],
+        "learning_rate": [
+            StrOptions({"auto"}),
+            Interval(Real, 0, None, closed="neither"),
+        ],
+        "max_iter": [Interval(Integral, 250, None, closed="left")],
+        "n_iter_without_progress": [Interval(Integral, -1, None, closed="left")],
+        "min_grad_norm": [Interval(Real, 0, None, closed="left")],
+        "metric": [StrOptions(set(_VALID_METRICS) | {"precomputed"}), callable],
+        "metric_params": [dict, None],
+        "init": [
+            StrOptions({"pca", "random"}),
+            np.ndarray,
+        ],
+        "verbose": ["verbose"],
+        "random_state": ["random_state"],
+        "method": [StrOptions({"barnes_hut", "exact"})],
+        "angle": [Interval(Real, 0, 1, closed="both")],
+        "n_jobs": [None, Integral],
+    }
+
     # Control the number of exploration iterations with early_exaggeration on
-    _EXPLORATION_N_ITER = 250
+    _EXPLORATION_MAX_ITER = 250
 
     # Control the number of iterations between progress checks
     _N_ITER_CHECK = 50
 
-    @_deprecate_positional_args
-    def __init__(self, n_components=2, *, perplexity=30.0,
-                 early_exaggeration=12.0, learning_rate=200.0, n_iter=1000,
-                 n_iter_without_progress=300, min_grad_norm=1e-7,
-                 metric="euclidean", init="random", verbose=0,
-                 random_state=None, method='barnes_hut', angle=0.5,
-                 n_jobs=None, square_distances='legacy'):
+    def __init__(
+        self,
+        n_components=2,
+        *,
+        perplexity=30.0,
+        early_exaggeration=12.0,
+        learning_rate="auto",
+        max_iter=1000,
+        n_iter_without_progress=300,
+        min_grad_norm=1e-7,
+        metric="euclidean",
+        metric_params=None,
+        init="pca",
+        verbose=0,
+        random_state=None,
+        method="barnes_hut",
+        angle=0.5,
+        n_jobs=None,
+    ):
         self.n_components = n_components
         self.perplexity = perplexity
         self.early_exaggeration = early_exaggeration
         self.learning_rate = learning_rate
-        self.n_iter = n_iter
+        self.max_iter = max_iter
         self.n_iter_without_progress = n_iter_without_progress
         self.min_grad_norm = min_grad_norm
         self.metric = metric
+        self.metric_params = metric_params
         self.init = init
         self.verbose = verbose
         self.random_state = random_state
         self.method = method
         self.angle = angle
         self.n_jobs = n_jobs
-        # TODO Revisit deprecation of square_distances for 1.1-1.3 (#12401)
-        self.square_distances = square_distances
+
+    def _check_params_vs_input(self, X):
+        if self.perplexity >= X.shape[0]:
+            raise ValueError(
+                f"perplexity ({self.perplexity}) must be less "
+                f"than n_samples ({X.shape[0]})"
+            )
 
     def _fit(self, X, skip_num_points=0):
         """Private function to fit the model using X as training data."""
 
-        if self.method not in ['barnes_hut', 'exact']:
-            raise ValueError("'method' must be 'barnes_hut' or 'exact'")
-        if self.angle < 0.0 or self.angle > 1.0:
-            raise ValueError("'angle' must be between 0.0 - 1.0")
-        if self.square_distances not in [True, 'legacy']:
-            raise ValueError("'square_distances' must be True or 'legacy'.")
-        if self.metric != "euclidean" and self.square_distances is not True:
-            warnings.warn(
-                "'square_distances' has been introduced in 0.24 to help phase "
-                "out legacy squaring behavior. The 'legacy' setting will be "
-                "removed in 1.1 (renaming of 0.26), and the default setting "
-                "will be changed to True. In 1.3, 'square_distances' will be "
-                "removed altogether, and distances will be squared by "
-                "default. Set 'square_distances'=True to silence this "
-                "warning.",
-                FutureWarning
+        if isinstance(self.init, str) and self.init == "pca" and issparse(X):
+            raise TypeError(
+                "PCA initialization is currently not supported "
+                "with the sparse input matrix. Use "
+                'init="random" instead.'
+            )
+
+        if self.learning_rate == "auto":
+            # See issue #18018
+            self.learning_rate_ = X.shape[0] / self.early_exaggeration / 4
+            self.learning_rate_ = np.maximum(self.learning_rate_, 50)
+        else:
+            self.learning_rate_ = self.learning_rate
+
+        if self.method == "barnes_hut":
+            X = validate_data(
+                self,
+                X,
+                accept_sparse=["csr"],
+                ensure_min_samples=2,
+                dtype=[np.float32, np.float64],
             )
-        if self.method == 'barnes_hut':
-            X = self._validate_data(X, accept_sparse=['csr'],
-                                    ensure_min_samples=2,
-                                    dtype=[np.float32, np.float64])
         else:
-            X = self._validate_data(X, accept_sparse=['csr', 'csc', 'coo'],
-                                    dtype=[np.float32, np.float64])
+            X = validate_data(
+                self,
+                X,
+                accept_sparse=["csr", "csc", "coo"],
+                dtype=[np.float32, np.float64],
+            )
         if self.metric == "precomputed":
-            if isinstance(self.init, str) and self.init == 'pca':
-                raise ValueError("The parameter init=\"pca\" cannot be "
-                                 "used with metric=\"precomputed\".")
+            if isinstance(self.init, str) and self.init == "pca":
+                raise ValueError(
+                    'The parameter init="pca" cannot be used with metric="precomputed".'
+                )
             if X.shape[0] != X.shape[1]:
                 raise ValueError("X should be a square distance matrix")
 
-            check_non_negative(X, "TSNE.fit(). With metric='precomputed', X "
-                                  "should contain positive distances.")
+            check_non_negative(
+                X,
+                (
+                    "TSNE.fit(). With metric='precomputed', X "
+                    "should contain positive distances."
+                ),
+            )
 
             if self.method == "exact" and issparse(X):
                 raise TypeError(
                     'TSNE with method="exact" does not accept sparse '
                     'precomputed distance matrix. Use method="barnes_hut" '
-                    'or provide the dense distance matrix.')
-
-        if self.method == 'barnes_hut' and self.n_components > 3:
-            raise ValueError("'n_components' should be inferior to 4 for the "
-                             "barnes_hut algorithm as it relies on "
-                             "quad-tree or oct-tree.")
+                    "or provide the dense distance matrix."
+                )
+
+        if self.method == "barnes_hut" and self.n_components > 3:
+            raise ValueError(
+                "'n_components' should be inferior to 4 for the "
+                "barnes_hut algorithm as it relies on "
+                "quad-tree or oct-tree."
+            )
         random_state = check_random_state(self.random_state)
 
-        if self.early_exaggeration < 1.0:
-            raise ValueError("early_exaggeration must be at least 1, but is {}"
-                             .format(self.early_exaggeration))
-
-        if self.n_iter < 250:
-            raise ValueError("n_iter should be at least 250")
-
         n_samples = X.shape[0]
 
         neighbors_nn = None
@@ -752,85 +930,100 @@ def _fit(self, X, skip_num_points=0):
                     # squared distances, and returns np.sqrt(dist) for
                     # squared=False.
                     # Also, Euclidean is slower for n_jobs>1, so don't set here
-                    distances = pairwise_distances(X, metric=self.metric,
-                                                   squared=True)
+                    distances = pairwise_distances(X, metric=self.metric, squared=True)
                 else:
-                    distances = pairwise_distances(X, metric=self.metric,
-                                                   n_jobs=self.n_jobs)
+                    metric_params_ = self.metric_params or {}
+                    distances = pairwise_distances(
+                        X, metric=self.metric, n_jobs=self.n_jobs, **metric_params_
+                    )
 
             if np.any(distances < 0):
-                raise ValueError("All distances should be positive, the "
-                                 "metric given is not correct")
+                raise ValueError(
+                    "All distances should be positive, the metric given is not correct"
+                )
 
-            if self.metric != "euclidean" and self.square_distances is True:
+            if self.metric != "euclidean":
                 distances **= 2
 
             # compute the joint probability distribution for the input space
             P = _joint_probabilities(distances, self.perplexity, self.verbose)
             assert np.all(np.isfinite(P)), "All probabilities should be finite"
             assert np.all(P >= 0), "All probabilities should be non-negative"
-            assert np.all(P <= 1), ("All probabilities should be less "
-                                    "or then equal to one")
+            assert np.all(P <= 1), (
+                "All probabilities should be less or then equal to one"
+            )
 
         else:
             # Compute the number of nearest neighbors to find.
             # LvdM uses 3 * perplexity as the number of neighbors.
             # In the event that we have very small # of points
             # set the neighbors to n - 1.
-            n_neighbors = min(n_samples - 1, int(3. * self.perplexity + 1))
+            n_neighbors = min(n_samples - 1, int(3.0 * self.perplexity + 1))
 
             if self.verbose:
-                print("[t-SNE] Computing {} nearest neighbors..."
-                      .format(n_neighbors))
+                print("[t-SNE] Computing {} nearest neighbors...".format(n_neighbors))
 
             # Find the nearest neighbors for every point
-            knn = NearestNeighbors(algorithm='auto',
-                                   n_jobs=self.n_jobs,
-                                   n_neighbors=n_neighbors,
-                                   metric=self.metric)
+            knn = NearestNeighbors(
+                algorithm="auto",
+                n_jobs=self.n_jobs,
+                n_neighbors=n_neighbors,
+                metric=self.metric,
+                metric_params=self.metric_params,
+            )
             t0 = time()
             knn.fit(X)
             duration = time() - t0
             if self.verbose:
-                print("[t-SNE] Indexed {} samples in {:.3f}s...".format(
-                    n_samples, duration))
+                print(
+                    "[t-SNE] Indexed {} samples in {:.3f}s...".format(
+                        n_samples, duration
+                    )
+                )
 
             t0 = time()
-            distances_nn = knn.kneighbors_graph(mode='distance')
+            distances_nn = knn.kneighbors_graph(mode="distance")
             duration = time() - t0
             if self.verbose:
-                print("[t-SNE] Computed neighbors for {} samples "
-                      "in {:.3f}s...".format(n_samples, duration))
+                print(
+                    "[t-SNE] Computed neighbors for {} samples in {:.3f}s...".format(
+                        n_samples, duration
+                    )
+                )
 
             # Free the memory used by the ball_tree
             del knn
 
-            if self.square_distances is True or self.metric == "euclidean":
-                # knn return the euclidean distance but we need it squared
-                # to be consistent with the 'exact' method. Note that the
-                # the method was derived using the euclidean method as in the
-                # input space. Not sure of the implication of using a different
-                # metric.
-                distances_nn.data **= 2
+            # knn return the euclidean distance but we need it squared
+            # to be consistent with the 'exact' method. Note that the
+            # the method was derived using the euclidean method as in the
+            # input space. Not sure of the implication of using a different
+            # metric.
+            distances_nn.data **= 2
 
             # compute the joint probability distribution for the input space
-            P = _joint_probabilities_nn(distances_nn, self.perplexity,
-                                        self.verbose)
+            P = _joint_probabilities_nn(distances_nn, self.perplexity, self.verbose)
 
         if isinstance(self.init, np.ndarray):
             X_embedded = self.init
-        elif self.init == 'pca':
-            pca = PCA(n_components=self.n_components, svd_solver='randomized',
-                      random_state=random_state)
+        elif self.init == "pca":
+            pca = PCA(
+                n_components=self.n_components,
+                svd_solver="randomized",
+                random_state=random_state,
+            )
+            # Always output a numpy array, no matter what is configured globally
+            pca.set_output(transform="default")
             X_embedded = pca.fit_transform(X).astype(np.float32, copy=False)
-        elif self.init == 'random':
+            # PCA is rescaled so that PC1 has standard deviation 1e-4 which is
+            # the default value for random initialization. See issue #18018.
+            X_embedded = X_embedded / np.std(X_embedded[:, 0]) * 1e-4
+        elif self.init == "random":
             # The embedding is initialized with iid samples from Gaussians with
             # standard deviation 1e-4.
-            X_embedded = 1e-4 * random_state.randn(
-                n_samples, self.n_components).astype(np.float32)
-        else:
-            raise ValueError("'init' must be 'pca', 'random', or "
-                             "a numpy array")
+            X_embedded = 1e-4 * random_state.standard_normal(
+                size=(n_samples, self.n_components)
+            ).astype(np.float32)
 
         # Degrees of freedom of the Student's t-distribution. The suggestion
         # degrees_of_freedom = n_components - 1 comes from
@@ -838,13 +1031,24 @@ def _fit(self, X, skip_num_points=0):
         # Laurens van der Maaten, 2009.
         degrees_of_freedom = max(self.n_components - 1, 1)
 
-        return self._tsne(P, degrees_of_freedom, n_samples,
-                          X_embedded=X_embedded,
-                          neighbors=neighbors_nn,
-                          skip_num_points=skip_num_points)
-
-    def _tsne(self, P, degrees_of_freedom, n_samples, X_embedded,
-              neighbors=None, skip_num_points=0):
+        return self._tsne(
+            P,
+            degrees_of_freedom,
+            n_samples,
+            X_embedded=X_embedded,
+            neighbors=neighbors_nn,
+            skip_num_points=skip_num_points,
+        )
+
+    def _tsne(
+        self,
+        P,
+        degrees_of_freedom,
+        n_samples,
+        X_embedded,
+        neighbors=None,
+        skip_num_points=0,
+    ):
         """Runs t-SNE."""
         # t-SNE minimizes the Kullback-Leiber divergence of the Gaussians P
         # and the Student's t-distributions Q. The optimization algorithm that
@@ -857,95 +1061,124 @@ def _tsne(self, P, degrees_of_freedom, n_samples, X_embedded,
             "it": 0,
             "n_iter_check": self._N_ITER_CHECK,
             "min_grad_norm": self.min_grad_norm,
-            "learning_rate": self.learning_rate,
+            "learning_rate": self.learning_rate_,
             "verbose": self.verbose,
             "kwargs": dict(skip_num_points=skip_num_points),
             "args": [P, degrees_of_freedom, n_samples, self.n_components],
-            "n_iter_without_progress": self._EXPLORATION_N_ITER,
-            "n_iter": self._EXPLORATION_N_ITER,
+            "n_iter_without_progress": self._EXPLORATION_MAX_ITER,
+            "max_iter": self._EXPLORATION_MAX_ITER,
             "momentum": 0.5,
         }
-        if self.method == 'barnes_hut':
+        if self.method == "barnes_hut":
             obj_func = _kl_divergence_bh
-            opt_args['kwargs']['angle'] = self.angle
+            opt_args["kwargs"]["angle"] = self.angle
             # Repeat verbose argument for _kl_divergence_bh
-            opt_args['kwargs']['verbose'] = self.verbose
+            opt_args["kwargs"]["verbose"] = self.verbose
             # Get the number of threads for gradient computation here to
             # avoid recomputing it at each iteration.
-            opt_args['kwargs']['num_threads'] = _openmp_effective_n_threads()
+            opt_args["kwargs"]["num_threads"] = _openmp_effective_n_threads()
         else:
             obj_func = _kl_divergence
 
         # Learning schedule (part 1): do 250 iteration with lower momentum but
         # higher learning rate controlled via the early exaggeration parameter
         P *= self.early_exaggeration
-        params, kl_divergence, it = _gradient_descent(obj_func, params,
-                                                      **opt_args)
+        params, kl_divergence, it = _gradient_descent(obj_func, params, **opt_args)
         if self.verbose:
-            print("[t-SNE] KL divergence after %d iterations with early "
-                  "exaggeration: %f" % (it + 1, kl_divergence))
+            print(
+                "[t-SNE] KL divergence after %d iterations with early exaggeration: %f"
+                % (it + 1, kl_divergence)
+            )
 
         # Learning schedule (part 2): disable early exaggeration and finish
         # optimization with a higher momentum at 0.8
         P /= self.early_exaggeration
-        remaining = self.n_iter - self._EXPLORATION_N_ITER
-        if it < self._EXPLORATION_N_ITER or remaining > 0:
-            opt_args['n_iter'] = self.n_iter
-            opt_args['it'] = it + 1
-            opt_args['momentum'] = 0.8
-            opt_args['n_iter_without_progress'] = self.n_iter_without_progress
-            params, kl_divergence, it = _gradient_descent(obj_func, params,
-                                                          **opt_args)
+        remaining = self.max_iter - self._EXPLORATION_MAX_ITER
+        if it < self._EXPLORATION_MAX_ITER or remaining > 0:
+            opt_args["max_iter"] = self.max_iter
+            opt_args["it"] = it + 1
+            opt_args["momentum"] = 0.8
+            opt_args["n_iter_without_progress"] = self.n_iter_without_progress
+            params, kl_divergence, it = _gradient_descent(obj_func, params, **opt_args)
 
         # Save the final number of iterations
         self.n_iter_ = it
 
         if self.verbose:
-            print("[t-SNE] KL divergence after %d iterations: %f"
-                  % (it + 1, kl_divergence))
+            print(
+                "[t-SNE] KL divergence after %d iterations: %f"
+                % (it + 1, kl_divergence)
+            )
 
         X_embedded = params.reshape(n_samples, self.n_components)
         self.kl_divergence_ = kl_divergence
 
         return X_embedded
 
+    @_fit_context(
+        # TSNE.metric is not validated yet
+        prefer_skip_nested_validation=False
+    )
     def fit_transform(self, X, y=None):
-        """Fit X into an embedded space and return that transformed
-        output.
+        """Fit X into an embedded space and return that transformed output.
 
         Parameters
         ----------
-        X : ndarray of shape (n_samples, n_features) or (n_samples, n_samples)
+        X : {array-like, sparse matrix} of shape (n_samples, n_features) or \
+            (n_samples, n_samples)
             If the metric is 'precomputed' X must be a square distance
             matrix. Otherwise it contains a sample per row. If the method
             is 'exact', X may be a sparse matrix of type 'csr', 'csc'
             or 'coo'. If the method is 'barnes_hut' and the metric is
             'precomputed', X may be a precomputed sparse graph.
 
-        y : Ignored
+        y : None
+            Ignored.
 
         Returns
         -------
         X_new : ndarray of shape (n_samples, n_components)
             Embedding of the training data in low-dimensional space.
         """
+        self._check_params_vs_input(X)
         embedding = self._fit(X)
         self.embedding_ = embedding
         return self.embedding_
 
+    @_fit_context(
+        # TSNE.metric is not validated yet
+        prefer_skip_nested_validation=False
+    )
     def fit(self, X, y=None):
         """Fit X into an embedded space.
 
         Parameters
         ----------
-        X : ndarray of shape (n_samples, n_features) or (n_samples, n_samples)
+        X : {array-like, sparse matrix} of shape (n_samples, n_features) or \
+            (n_samples, n_samples)
             If the metric is 'precomputed' X must be a square distance
             matrix. Otherwise it contains a sample per row. If the method
             is 'exact', X may be a sparse matrix of type 'csr', 'csc'
             or 'coo'. If the method is 'barnes_hut' and the metric is
             'precomputed', X may be a precomputed sparse graph.
 
-        y : Ignored
+        y : None
+            Ignored.
+
+        Returns
+        -------
+        self : object
+            Fitted estimator.
         """
         self.fit_transform(X)
         return self
+
+    @property
+    def _n_features_out(self):
+        """Number of transformed output features."""
+        return self.embedding_.shape[1]
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.pairwise = self.metric == "precomputed"
+        return tags
diff --git a/sklearn/manifold/_utils.pyx b/sklearn/manifold/_utils.pyx
index cd6ade795ae91..be3a1d2f91f66 100644
--- a/sklearn/manifold/_utils.pyx
+++ b/sklearn/manifold/_utils.pyx
@@ -1,23 +1,18 @@
-# cython: boundscheck=False
-
-from libc cimport math
-cimport cython
 import numpy as np
-cimport numpy as np
-from libc.stdio cimport printf
-
-np.import_array()
 
+from libc cimport math
+from libc.math cimport INFINITY
 
-cdef extern from "numpy/npy_math.h":
-    float NPY_INFINITY
+from ..utils._typedefs cimport float32_t, float64_t
 
 
 cdef float EPSILON_DBL = 1e-8
 cdef float PERPLEXITY_TOLERANCE = 1e-5
 
-cpdef np.ndarray[np.float32_t, ndim=2] _binary_search_perplexity(
-        np.ndarray[np.float32_t, ndim=2] sqdistances,
+
+# TODO: have this function support float32 and float64 and preserve inputs' dtypes.
+def _binary_search_perplexity(
+        const float32_t[:, :] sqdistances,
         float desired_perplexity,
         int verbose):
     """Binary search for sigmas of conditional Gaussians.
@@ -27,7 +22,7 @@ cpdef np.ndarray[np.float32_t, ndim=2] _binary_search_perplexity(
 
     Parameters
     ----------
-    sqdistances : array-like, shape (n_samples, n_neighbors)
+    sqdistances : ndarray of shape (n_samples, n_neighbors), dtype=np.float32
         Distances between training samples and their k nearest neighbors.
         When using the exact method, this is a square (n_samples, n_samples)
         distance matrix. The TSNE default metric is "euclidean" which is
@@ -41,7 +36,7 @@ cpdef np.ndarray[np.float32_t, ndim=2] _binary_search_perplexity(
 
     Returns
     -------
-    P : array, shape (n_samples, n_samples)
+    P : ndarray of shape (n_samples, n_samples), dtype=np.float64
         Probabilities of conditional Gaussian distributions p_i|j.
     """
     # Maximum number of binary search steps
@@ -63,16 +58,16 @@ cpdef np.ndarray[np.float32_t, ndim=2] _binary_search_perplexity(
     cdef double entropy
     cdef double sum_Pi
     cdef double sum_disti_Pi
-    cdef long i, j, k, l
+    cdef long i, j, l
 
     # This array is later used as a 32bit array. It has multiple intermediate
     # floating point additions that benefit from the extra precision
-    cdef np.ndarray[np.float64_t, ndim=2] P = np.zeros(
+    cdef float64_t[:, :] P = np.zeros(
         (n_samples, n_neighbors), dtype=np.float64)
 
     for i in range(n_samples):
-        beta_min = -NPY_INFINITY
-        beta_max = NPY_INFINITY
+        beta_min = -INFINITY
+        beta_max = INFINITY
         beta = 1.0
 
         # Binary search of precision for i-th conditional distribution
@@ -102,13 +97,13 @@ cpdef np.ndarray[np.float32_t, ndim=2] _binary_search_perplexity(
 
             if entropy_diff > 0.0:
                 beta_min = beta
-                if beta_max == NPY_INFINITY:
+                if beta_max == INFINITY:
                     beta *= 2.0
                 else:
                     beta = (beta + beta_max) / 2.0
             else:
                 beta_max = beta
-                if beta_min == -NPY_INFINITY:
+                if beta_min == -INFINITY:
                     beta /= 2.0
                 else:
                     beta = (beta + beta_min) / 2.0
@@ -122,4 +117,4 @@ cpdef np.ndarray[np.float32_t, ndim=2] _binary_search_perplexity(
     if verbose:
         print("[t-SNE] Mean sigma: %f"
               % np.mean(math.sqrt(n_samples / beta_sum)))
-    return P
+    return np.asarray(P)
diff --git a/sklearn/manifold/meson.build b/sklearn/manifold/meson.build
new file mode 100644
index 0000000000000..c060590410d63
--- /dev/null
+++ b/sklearn/manifold/meson.build
@@ -0,0 +1,14 @@
+py.extension_module(
+  '_utils',
+  [cython_gen.process('_utils.pyx'), utils_cython_tree],
+  subdir: 'sklearn/manifold',
+  install: true
+)
+
+py.extension_module(
+  '_barnes_hut_tsne',
+  cython_gen.process('_barnes_hut_tsne.pyx'),
+  dependencies: [np_dep, openmp_dep],
+  subdir: 'sklearn/manifold',
+  install: true
+)
diff --git a/sklearn/manifold/setup.py b/sklearn/manifold/setup.py
deleted file mode 100644
index 0db2d5d04683a..0000000000000
--- a/sklearn/manifold/setup.py
+++ /dev/null
@@ -1,34 +0,0 @@
-import os
-
-import numpy
-
-
-def configuration(parent_package="", top_path=None):
-    from numpy.distutils.misc_util import Configuration
-
-    config = Configuration("manifold", parent_package, top_path)
-
-    libraries = []
-    if os.name == 'posix':
-        libraries.append('m')
-
-    config.add_extension("_utils",
-                         sources=["_utils.pyx"],
-                         include_dirs=[numpy.get_include()],
-                         libraries=libraries,
-                         extra_compile_args=["-O3"])
-
-    config.add_extension("_barnes_hut_tsne",
-                         sources=["_barnes_hut_tsne.pyx"],
-                         include_dirs=[numpy.get_include()],
-                         libraries=libraries,
-                         extra_compile_args=['-O3'])
-
-    config.add_subpackage('tests')
-
-    return config
-
-
-if __name__ == "__main__":
-    from numpy.distutils.core import setup
-    setup(**configuration().todict())
diff --git a/sklearn/manifold/tests/test_isomap.py b/sklearn/manifold/tests/test_isomap.py
index 9007772674a99..e38b92442e58d 100644
--- a/sklearn/manifold/tests/test_isomap.py
+++ b/sklearn/manifold/tests/test_isomap.py
@@ -1,88 +1,122 @@
+import math
 from itertools import product
+
 import numpy as np
-from numpy.testing import assert_almost_equal, assert_array_almost_equal
 import pytest
-
-from sklearn import datasets
-from sklearn import manifold
-from sklearn import neighbors
-from sklearn import pipeline
-from sklearn import preprocessing
-
 from scipy.sparse import rand as sparse_rand
 
-eigen_solvers = ['auto', 'dense', 'arpack']
-path_methods = ['auto', 'FW', 'D']
+from sklearn import clone, datasets, manifold, neighbors, pipeline, preprocessing
+from sklearn.datasets import make_blobs
+from sklearn.metrics.pairwise import pairwise_distances
+from sklearn.utils._testing import (
+    assert_allclose,
+    assert_allclose_dense_sparse,
+    assert_array_equal,
+)
+from sklearn.utils.fixes import CSR_CONTAINERS
 
+eigen_solvers = ["auto", "dense", "arpack"]
+path_methods = ["auto", "FW", "D"]
 
-def test_isomap_simple_grid():
-    # Isomap should preserve distances when all neighbors are used
-    N_per_side = 5
-    Npts = N_per_side ** 2
-    n_neighbors = Npts - 1
 
+def create_sample_data(dtype, n_pts=25, add_noise=False):
     # grid of equidistant points in 2D, n_components = n_dim
-    X = np.array(list(product(range(N_per_side), repeat=2)))
+    n_per_side = int(math.sqrt(n_pts))
+    X = np.array(list(product(range(n_per_side), repeat=2))).astype(dtype, copy=False)
+    if add_noise:
+        # add noise in a third dimension
+        rng = np.random.RandomState(0)
+        noise = 0.1 * rng.randn(n_pts, 1).astype(dtype, copy=False)
+        X = np.concatenate((X, noise), 1)
+    return X
+
+
+@pytest.mark.parametrize("n_neighbors, radius", [(24, None), (None, np.inf)])
+@pytest.mark.parametrize("eigen_solver", eigen_solvers)
+@pytest.mark.parametrize("path_method", path_methods)
+def test_isomap_simple_grid(
+    global_dtype, n_neighbors, radius, eigen_solver, path_method
+):
+    # Isomap should preserve distances when all neighbors are used
+    n_pts = 25
+    X = create_sample_data(global_dtype, n_pts=n_pts, add_noise=False)
 
     # distances from each point to all others
-    G = neighbors.kneighbors_graph(X, n_neighbors,
-                                   mode='distance').toarray()
-
-    for eigen_solver in eigen_solvers:
-        for path_method in path_methods:
-            clf = manifold.Isomap(n_neighbors=n_neighbors, n_components=2,
-                                  eigen_solver=eigen_solver,
-                                  path_method=path_method)
-            clf.fit(X)
-
-            G_iso = neighbors.kneighbors_graph(clf.embedding_,
-                                               n_neighbors,
-                                               mode='distance').toarray()
-            assert_array_almost_equal(G, G_iso)
+    if n_neighbors is not None:
+        G = neighbors.kneighbors_graph(X, n_neighbors, mode="distance")
+    else:
+        G = neighbors.radius_neighbors_graph(X, radius, mode="distance")
+
+    clf = manifold.Isomap(
+        n_neighbors=n_neighbors,
+        radius=radius,
+        n_components=2,
+        eigen_solver=eigen_solver,
+        path_method=path_method,
+    )
+    clf.fit(X)
+
+    if n_neighbors is not None:
+        G_iso = neighbors.kneighbors_graph(clf.embedding_, n_neighbors, mode="distance")
+    else:
+        G_iso = neighbors.radius_neighbors_graph(
+            clf.embedding_, radius, mode="distance"
+        )
+    atol = 1e-5 if global_dtype == np.float32 else 0
+    assert_allclose_dense_sparse(G, G_iso, atol=atol)
+
+
+@pytest.mark.parametrize("n_neighbors, radius", [(24, None), (None, np.inf)])
+@pytest.mark.parametrize("eigen_solver", eigen_solvers)
+@pytest.mark.parametrize("path_method", path_methods)
+def test_isomap_reconstruction_error(
+    global_dtype, n_neighbors, radius, eigen_solver, path_method
+):
+    if global_dtype is np.float32:
+        pytest.skip(
+            "Skipping test due to numerical instabilities on float32 data"
+            "from KernelCenterer used in the reconstruction_error method"
+        )
 
-
-def test_isomap_reconstruction_error():
     # Same setup as in test_isomap_simple_grid, with an added dimension
-    N_per_side = 5
-    Npts = N_per_side ** 2
-    n_neighbors = Npts - 1
-
-    # grid of equidistant points in 2D, n_components = n_dim
-    X = np.array(list(product(range(N_per_side), repeat=2)))
-
-    # add noise in a third dimension
-    rng = np.random.RandomState(0)
-    noise = 0.1 * rng.randn(Npts, 1)
-    X = np.concatenate((X, noise), 1)
+    n_pts = 25
+    X = create_sample_data(global_dtype, n_pts=n_pts, add_noise=True)
 
     # compute input kernel
-    G = neighbors.kneighbors_graph(X, n_neighbors,
-                                   mode='distance').toarray()
-
+    if n_neighbors is not None:
+        G = neighbors.kneighbors_graph(X, n_neighbors, mode="distance").toarray()
+    else:
+        G = neighbors.radius_neighbors_graph(X, radius, mode="distance").toarray()
     centerer = preprocessing.KernelCenterer()
-    K = centerer.fit_transform(-0.5 * G ** 2)
-
-    for eigen_solver in eigen_solvers:
-        for path_method in path_methods:
-            clf = manifold.Isomap(n_neighbors=n_neighbors, n_components=2,
-                                  eigen_solver=eigen_solver,
-                                  path_method=path_method)
-            clf.fit(X)
-
-            # compute output kernel
-            G_iso = neighbors.kneighbors_graph(clf.embedding_,
-                                               n_neighbors,
-                                               mode='distance').toarray()
-
-            K_iso = centerer.fit_transform(-0.5 * G_iso ** 2)
-
-            # make sure error agrees
-            reconstruction_error = np.linalg.norm(K - K_iso) / Npts
-            assert_almost_equal(reconstruction_error,
-                                clf.reconstruction_error())
-
-
-def test_transform():
+    K = centerer.fit_transform(-0.5 * G**2)
+
+    clf = manifold.Isomap(
+        n_neighbors=n_neighbors,
+        radius=radius,
+        n_components=2,
+        eigen_solver=eigen_solver,
+        path_method=path_method,
+    )
+    clf.fit(X)
+
+    # compute output kernel
+    if n_neighbors is not None:
+        G_iso = neighbors.kneighbors_graph(clf.embedding_, n_neighbors, mode="distance")
+    else:
+        G_iso = neighbors.radius_neighbors_graph(
+            clf.embedding_, radius, mode="distance"
+        )
+    G_iso = G_iso.toarray()
+    K_iso = centerer.fit_transform(-0.5 * G_iso**2)
+
+    # make sure error agrees
+    reconstruction_error = np.linalg.norm(K - K_iso) / n_pts
+    atol = 1e-5 if global_dtype == np.float32 else 0
+    assert_allclose(reconstruction_error, clf.reconstruction_error(), atol=atol)
+
+
+@pytest.mark.parametrize("n_neighbors, radius", [(2, None), (None, 0.5)])
+def test_transform(global_dtype, n_neighbors, radius):
     n_samples = 200
     n_components = 10
     noise_scale = 0.01
@@ -90,8 +124,12 @@ def test_transform():
     # Create S-curve dataset
     X, y = datasets.make_s_curve(n_samples, random_state=0)
 
+    X = X.astype(global_dtype, copy=False)
+
     # Compute isomap embedding
-    iso = manifold.Isomap(n_components=n_components, n_neighbors=2)
+    iso = manifold.Isomap(
+        n_components=n_components, n_neighbors=n_neighbors, radius=radius
+    )
     X_iso = iso.fit_transform(X)
 
     # Re-embed a noisy version of the points
@@ -103,67 +141,79 @@ def test_transform():
     assert np.sqrt(np.mean((X_iso - X_iso2) ** 2)) < 2 * noise_scale
 
 
-def test_pipeline():
+@pytest.mark.parametrize("n_neighbors, radius", [(2, None), (None, 10.0)])
+def test_pipeline(n_neighbors, radius, global_dtype):
     # check that Isomap works fine as a transformer in a Pipeline
     # only checks that no error is raised.
     # TODO check that it actually does something useful
     X, y = datasets.make_blobs(random_state=0)
+    X = X.astype(global_dtype, copy=False)
     clf = pipeline.Pipeline(
-        [('isomap', manifold.Isomap()),
-         ('clf', neighbors.KNeighborsClassifier())])
+        [
+            ("isomap", manifold.Isomap(n_neighbors=n_neighbors, radius=radius)),
+            ("clf", neighbors.KNeighborsClassifier()),
+        ]
+    )
     clf.fit(X, y)
-    assert .9 < clf.score(X, y)
+    assert 0.9 < clf.score(X, y)
 
 
-def test_pipeline_with_nearest_neighbors_transformer():
+def test_pipeline_with_nearest_neighbors_transformer(global_dtype):
     # Test chaining NearestNeighborsTransformer and Isomap with
     # neighbors_algorithm='precomputed'
-    algorithm = 'auto'
+    algorithm = "auto"
     n_neighbors = 10
 
     X, _ = datasets.make_blobs(random_state=0)
     X2, _ = datasets.make_blobs(random_state=1)
 
+    X = X.astype(global_dtype, copy=False)
+    X2 = X2.astype(global_dtype, copy=False)
+
     # compare the chained version and the compact version
     est_chain = pipeline.make_pipeline(
         neighbors.KNeighborsTransformer(
-            n_neighbors=n_neighbors, algorithm=algorithm, mode='distance'),
-        manifold.Isomap(n_neighbors=n_neighbors, metric='precomputed'))
-    est_compact = manifold.Isomap(n_neighbors=n_neighbors,
-                                  neighbors_algorithm=algorithm)
+            n_neighbors=n_neighbors, algorithm=algorithm, mode="distance"
+        ),
+        manifold.Isomap(n_neighbors=n_neighbors, metric="precomputed"),
+    )
+    est_compact = manifold.Isomap(
+        n_neighbors=n_neighbors, neighbors_algorithm=algorithm
+    )
 
     Xt_chain = est_chain.fit_transform(X)
     Xt_compact = est_compact.fit_transform(X)
-    assert_array_almost_equal(Xt_chain, Xt_compact)
+    assert_allclose(Xt_chain, Xt_compact)
 
     Xt_chain = est_chain.transform(X2)
     Xt_compact = est_compact.transform(X2)
-    assert_array_almost_equal(Xt_chain, Xt_compact)
-
-
-def test_different_metric():
-    # Test that the metric parameters work correctly, and default to euclidean
-    def custom_metric(x1, x2):
-        return np.sqrt(np.sum(x1 ** 2 + x2 ** 2))
-
-    # metric, p, is_euclidean
-    metrics = [('euclidean', 2, True),
-               ('manhattan', 1, False),
-               ('minkowski', 1, False),
-               ('minkowski', 2, True),
-               (custom_metric, 2, False)]
-
+    assert_allclose(Xt_chain, Xt_compact)
+
+
+@pytest.mark.parametrize(
+    "metric, p, is_euclidean",
+    [
+        ("euclidean", 2, True),
+        ("manhattan", 1, False),
+        ("minkowski", 1, False),
+        ("minkowski", 2, True),
+        (lambda x1, x2: np.sqrt(np.sum(x1**2 + x2**2)), 2, False),
+    ],
+)
+def test_different_metric(global_dtype, metric, p, is_euclidean):
+    # Isomap must work on various metric parameters work correctly
+    # and must default to euclidean.
     X, _ = datasets.make_blobs(random_state=0)
-    reference = manifold.Isomap().fit_transform(X)
+    X = X.astype(global_dtype, copy=False)
 
-    for metric, p, is_euclidean in metrics:
-        embedding = manifold.Isomap(metric=metric, p=p).fit_transform(X)
+    reference = manifold.Isomap().fit_transform(X)
+    embedding = manifold.Isomap(metric=metric, p=p).fit_transform(X)
 
-        if is_euclidean:
-            assert_array_almost_equal(embedding, reference)
-        else:
-            with pytest.raises(AssertionError, match='not almost equal'):
-                assert_array_almost_equal(embedding, reference)
+    if is_euclidean:
+        assert_allclose(embedding, reference)
+    else:
+        with pytest.raises(AssertionError, match="Not equal to tolerance"):
+            assert_allclose(embedding, reference)
 
 
 def test_isomap_clone_bug():
@@ -172,17 +222,127 @@ def test_isomap_clone_bug():
     for n_neighbors in [10, 15, 20]:
         model.set_params(n_neighbors=n_neighbors)
         model.fit(np.random.rand(50, 2))
-        assert (model.nbrs_.n_neighbors ==
-                     n_neighbors)
+        assert model.nbrs_.n_neighbors == n_neighbors
+
+
+@pytest.mark.parametrize("eigen_solver", eigen_solvers)
+@pytest.mark.parametrize("path_method", path_methods)
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sparse_input(
+    global_dtype, eigen_solver, path_method, global_random_seed, csr_container
+):
+    # TODO: compare results on dense and sparse data as proposed in:
+    # https://github.com/scikit-learn/scikit-learn/pull/23585#discussion_r968388186
+    X = csr_container(
+        sparse_rand(
+            100,
+            3,
+            density=0.1,
+            format="csr",
+            dtype=global_dtype,
+            random_state=global_random_seed,
+        )
+    )
+
+    iso_dense = manifold.Isomap(
+        n_components=2,
+        eigen_solver=eigen_solver,
+        path_method=path_method,
+        n_neighbors=8,
+    )
+    iso_sparse = clone(iso_dense)
+
+    X_trans_dense = iso_dense.fit_transform(X.toarray())
+    X_trans_sparse = iso_sparse.fit_transform(X)
+
+    assert_allclose(X_trans_sparse, X_trans_dense, rtol=1e-4, atol=1e-4)
+
+
+def test_isomap_fit_precomputed_radius_graph(global_dtype):
+    # Isomap.fit_transform must yield similar result when using
+    # a precomputed distance matrix.
+
+    X, y = datasets.make_s_curve(200, random_state=0)
+    X = X.astype(global_dtype, copy=False)
+    radius = 10
+
+    g = neighbors.radius_neighbors_graph(X, radius=radius, mode="distance")
+    isomap = manifold.Isomap(n_neighbors=None, radius=radius, metric="precomputed")
+    isomap.fit(g)
+    precomputed_result = isomap.embedding_
+
+    isomap = manifold.Isomap(n_neighbors=None, radius=radius, metric="minkowski")
+    result = isomap.fit_transform(X)
+    atol = 1e-5 if global_dtype == np.float32 else 0
+    assert_allclose(precomputed_result, result, atol=atol)
+
+
+def test_isomap_fitted_attributes_dtype(global_dtype):
+    """Check that the fitted attributes are stored accordingly to the
+    data type of X."""
+    iso = manifold.Isomap(n_neighbors=2)
+
+    X = np.array([[1, 2], [3, 4], [5, 6]], dtype=global_dtype)
+
+    iso.fit(X)
+
+    assert iso.dist_matrix_.dtype == global_dtype
+    assert iso.embedding_.dtype == global_dtype
+
+
+def test_isomap_dtype_equivalence():
+    """Check the equivalence of the results with 32 and 64 bits input."""
+    iso_32 = manifold.Isomap(n_neighbors=2)
+    X_32 = np.array([[1, 2], [3, 4], [5, 6]], dtype=np.float32)
+    iso_32.fit(X_32)
+
+    iso_64 = manifold.Isomap(n_neighbors=2)
+    X_64 = np.array([[1, 2], [3, 4], [5, 6]], dtype=np.float64)
+    iso_64.fit(X_64)
+
+    assert_allclose(iso_32.dist_matrix_, iso_64.dist_matrix_)
+
+
+def test_isomap_raise_error_when_neighbor_and_radius_both_set():
+    # Isomap.fit_transform must raise a ValueError if
+    # radius and n_neighbors are provided.
+
+    X, _ = datasets.load_digits(return_X_y=True)
+    isomap = manifold.Isomap(n_neighbors=3, radius=5.5)
+    msg = "Both n_neighbors and radius are provided"
+    with pytest.raises(ValueError, match=msg):
+        isomap.fit_transform(X)
+
+
+def test_multiple_connected_components():
+    # Test that a warning is raised when the graph has multiple components
+    X = np.array([0, 1, 2, 5, 6, 7])[:, None]
+    with pytest.warns(UserWarning, match="number of connected components"):
+        manifold.Isomap(n_neighbors=2).fit(X)
+
+
+def test_multiple_connected_components_metric_precomputed(global_dtype):
+    # Test that an error is raised when the graph has multiple components
+    # and when X is a precomputed neighbors graph.
+    X = np.array([0, 1, 2, 5, 6, 7])[:, None].astype(global_dtype, copy=False)
+
+    # works with a precomputed distance matrix (dense)
+    X_distances = pairwise_distances(X)
+    with pytest.warns(UserWarning, match="number of connected components"):
+        manifold.Isomap(n_neighbors=1, metric="precomputed").fit(X_distances)
+
+    # does not work with a precomputed neighbors graph (sparse)
+    X_graph = neighbors.kneighbors_graph(X, n_neighbors=2, mode="distance")
+    with pytest.raises(RuntimeError, match="number of connected components"):
+        manifold.Isomap(n_neighbors=1, metric="precomputed").fit(X_graph)
 
 
-def test_sparse_input():
-    X = sparse_rand(100, 3, density=0.1, format='csr')
+def test_get_feature_names_out():
+    """Check get_feature_names_out for Isomap."""
+    X, y = make_blobs(random_state=0, n_features=4)
+    n_components = 2
 
-    # Should not error
-    for eigen_solver in eigen_solvers:
-        for path_method in path_methods:
-            clf = manifold.Isomap(n_components=2,
-                                  eigen_solver=eigen_solver,
-                                  path_method=path_method)
-            clf.fit(X)
+    iso = manifold.Isomap(n_components=n_components)
+    iso.fit_transform(X)
+    names = iso.get_feature_names_out()
+    assert_array_equal([f"isomap{i}" for i in range(n_components)], names)
diff --git a/sklearn/manifold/tests/test_locally_linear.py b/sklearn/manifold/tests/test_locally_linear.py
index 952da3ef41163..835aa20fd1d32 100644
--- a/sklearn/manifold/tests/test_locally_linear.py
+++ b/sklearn/manifold/tests/test_locally_linear.py
@@ -1,139 +1,151 @@
 from itertools import product
 
 import numpy as np
-from numpy.testing import assert_almost_equal, assert_array_almost_equal
-from scipy import linalg
 import pytest
+from scipy import linalg
 
-from sklearn import neighbors, manifold
+from sklearn import manifold, neighbors
+from sklearn.datasets import make_blobs
 from sklearn.manifold._locally_linear import barycenter_kneighbors_graph
-from sklearn.utils._testing import ignore_warnings
-from sklearn.utils._testing import assert_raise_message
+from sklearn.utils._testing import (
+    assert_allclose,
+    assert_array_equal,
+    ignore_warnings,
+)
 
-eigen_solvers = ['dense', 'arpack']
+eigen_solvers = ["dense", "arpack"]
 
 
 # ----------------------------------------------------------------------
 # Test utility routines
-def test_barycenter_kneighbors_graph():
-    X = np.array([[0, 1], [1.01, 1.], [2, 0]])
+def test_barycenter_kneighbors_graph(global_dtype):
+    X = np.array([[0, 1], [1.01, 1.0], [2, 0]], dtype=global_dtype)
+
+    graph = barycenter_kneighbors_graph(X, 1)
+    expected_graph = np.array(
+        [[0.0, 1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0]], dtype=global_dtype
+    )
 
-    A = barycenter_kneighbors_graph(X, 1)
-    assert_array_almost_equal(
-        A.toarray(),
-        [[0.,  1.,  0.],
-         [1.,  0.,  0.],
-         [0.,  1.,  0.]])
+    assert graph.dtype == global_dtype
 
-    A = barycenter_kneighbors_graph(X, 2)
+    assert_allclose(graph.toarray(), expected_graph)
+
+    graph = barycenter_kneighbors_graph(X, 2)
     # check that columns sum to one
-    assert_array_almost_equal(np.sum(A.toarray(), 1), np.ones(3))
-    pred = np.dot(A.toarray(), X)
+    assert_allclose(np.sum(graph.toarray(), axis=1), np.ones(3))
+    pred = np.dot(graph.toarray(), X)
     assert linalg.norm(pred - X) / X.shape[0] < 1
 
 
 # ----------------------------------------------------------------------
 # Test LLE by computing the reconstruction error on some manifolds.
 
-def test_lle_simple_grid():
+
+def test_lle_simple_grid(global_dtype):
     # note: ARPACK is numerically unstable, so this test will fail for
-    #       some random seeds.  We choose 2 because the tests pass.
-    rng = np.random.RandomState(2)
+    #       some random seeds.  We choose 42 because the tests pass.
+    #       for arm64 platforms 2 makes the test fail.
+    # TODO: rewrite this test to make less sensitive to the random seed,
+    # irrespective of the platform.
+    rng = np.random.RandomState(42)
 
     # grid of equidistant points in 2D, n_components = n_dim
     X = np.array(list(product(range(5), repeat=2)))
     X = X + 1e-10 * rng.uniform(size=X.shape)
+    X = X.astype(global_dtype, copy=False)
+
     n_components = 2
-    clf = manifold.LocallyLinearEmbedding(n_neighbors=5,
-                                          n_components=n_components,
-                                          random_state=rng)
+    clf = manifold.LocallyLinearEmbedding(
+        n_neighbors=5, n_components=n_components, random_state=rng
+    )
     tol = 0.1
 
     N = barycenter_kneighbors_graph(X, clf.n_neighbors).toarray()
-    reconstruction_error = linalg.norm(np.dot(N, X) - X, 'fro')
+    reconstruction_error = linalg.norm(np.dot(N, X) - X, "fro")
     assert reconstruction_error < tol
 
     for solver in eigen_solvers:
         clf.set_params(eigen_solver=solver)
         clf.fit(X)
         assert clf.embedding_.shape[1] == n_components
-        reconstruction_error = linalg.norm(
-            np.dot(N, clf.embedding_) - clf.embedding_, 'fro') ** 2
+        reconstruction_error = (
+            linalg.norm(np.dot(N, clf.embedding_) - clf.embedding_, "fro") ** 2
+        )
 
         assert reconstruction_error < tol
-        assert_almost_equal(clf.reconstruction_error_,
-                            reconstruction_error, decimal=1)
+        assert_allclose(clf.reconstruction_error_, reconstruction_error, atol=1e-1)
 
     # re-embed a noisy version of X using the transform method
-    noise = rng.randn(*X.shape) / 100
+    noise = rng.randn(*X.shape).astype(global_dtype, copy=False) / 100
     X_reembedded = clf.transform(X + noise)
     assert linalg.norm(X_reembedded - clf.embedding_) < tol
 
 
-def test_lle_manifold():
+@pytest.mark.parametrize("method", ["standard", "hessian", "modified", "ltsa"])
+@pytest.mark.parametrize("solver", eigen_solvers)
+def test_lle_manifold(global_dtype, method, solver):
     rng = np.random.RandomState(0)
     # similar test on a slightly more complex manifold
     X = np.array(list(product(np.arange(18), repeat=2)))
     X = np.c_[X, X[:, 0] ** 2 / 18]
     X = X + 1e-10 * rng.uniform(size=X.shape)
+    X = X.astype(global_dtype, copy=False)
     n_components = 2
-    for method in ["standard", "hessian", "modified", "ltsa"]:
-        clf = manifold.LocallyLinearEmbedding(n_neighbors=6,
-                                              n_components=n_components,
-                                              method=method, random_state=0)
-        tol = 1.5 if method == "standard" else 3
-
-        N = barycenter_kneighbors_graph(X, clf.n_neighbors).toarray()
-        reconstruction_error = linalg.norm(np.dot(N, X) - X)
-        assert reconstruction_error < tol
 
-        for solver in eigen_solvers:
-            clf.set_params(eigen_solver=solver)
-            clf.fit(X)
-            assert clf.embedding_.shape[1] == n_components
-            reconstruction_error = linalg.norm(
-                np.dot(N, clf.embedding_) - clf.embedding_, 'fro') ** 2
-            details = ("solver: %s, method: %s" % (solver, method))
-            assert reconstruction_error < tol, details
-            assert (np.abs(clf.reconstruction_error_ -
-                           reconstruction_error) <
-                    tol * reconstruction_error), details
+    clf = manifold.LocallyLinearEmbedding(
+        n_neighbors=6, n_components=n_components, method=method, random_state=0
+    )
+    tol = 1.5 if method == "standard" else 3
 
+    N = barycenter_kneighbors_graph(X, clf.n_neighbors).toarray()
+    reconstruction_error = linalg.norm(np.dot(N, X) - X)
+    assert reconstruction_error < tol
 
-# Test the error raised when parameter passed to lle is invalid
-def test_lle_init_parameters():
-    X = np.random.rand(5, 3)
-
-    clf = manifold.LocallyLinearEmbedding(eigen_solver="error")
-    msg = "unrecognized eigen_solver 'error'"
-    assert_raise_message(ValueError, msg, clf.fit, X)
-
-    clf = manifold.LocallyLinearEmbedding(method="error")
-    msg = "unrecognized method 'error'"
-    assert_raise_message(ValueError, msg, clf.fit, X)
+    clf.set_params(eigen_solver=solver)
+    clf.fit(X)
+    assert clf.embedding_.shape[1] == n_components
+    reconstruction_error = (
+        linalg.norm(np.dot(N, clf.embedding_) - clf.embedding_, "fro") ** 2
+    )
+    details = "solver: %s, method: %s" % (solver, method)
+    assert reconstruction_error < tol, details
+    assert (
+        np.abs(clf.reconstruction_error_ - reconstruction_error)
+        < tol * reconstruction_error
+    ), details
 
 
 def test_pipeline():
     # check that LocallyLinearEmbedding works fine as a Pipeline
     # only checks that no error is raised.
     # TODO check that it actually does something useful
-    from sklearn import pipeline, datasets
+    from sklearn import datasets, pipeline
+
     X, y = datasets.make_blobs(random_state=0)
     clf = pipeline.Pipeline(
-        [('filter', manifold.LocallyLinearEmbedding(random_state=0)),
-         ('clf', neighbors.KNeighborsClassifier())])
+        [
+            ("filter", manifold.LocallyLinearEmbedding(random_state=0)),
+            ("clf", neighbors.KNeighborsClassifier()),
+        ]
+    )
     clf.fit(X, y)
-    assert .9 < clf.score(X, y)
+    assert 0.9 < clf.score(X, y)
 
 
 # Test the error raised when the weight matrix is singular
 def test_singular_matrix():
-    M = np.ones((10, 3))
+    M = np.ones((200, 3))
     f = ignore_warnings
-    with pytest.raises(ValueError):
-        f(manifold.locally_linear_embedding(M, n_neighbors=2, n_components=1,
-                                            method='standard',
-                                            eigen_solver='arpack'))
+    with pytest.raises(ValueError, match="Error in determining null-space with ARPACK"):
+        f(
+            manifold.locally_linear_embedding(
+                M,
+                n_neighbors=2,
+                n_components=1,
+                method="standard",
+                eigen_solver="arpack",
+            )
+        )
 
 
 # regression test for #6033
@@ -144,3 +156,16 @@ def test_integer_input():
     for method in ["standard", "hessian", "modified", "ltsa"]:
         clf = manifold.LocallyLinearEmbedding(method=method, n_neighbors=10)
         clf.fit(X)  # this previously raised a TypeError
+
+
+def test_get_feature_names_out():
+    """Check get_feature_names_out for LocallyLinearEmbedding."""
+    X, y = make_blobs(random_state=0, n_features=4)
+    n_components = 2
+
+    iso = manifold.LocallyLinearEmbedding(n_components=n_components)
+    iso.fit(X)
+    names = iso.get_feature_names_out()
+    assert_array_equal(
+        [f"locallylinearembedding{i}" for i in range(n_components)], names
+    )
diff --git a/sklearn/manifold/tests/test_mds.py b/sklearn/manifold/tests/test_mds.py
index 6e2016c798772..88dc842a1d5fc 100644
--- a/sklearn/manifold/tests/test_mds.py
+++ b/sklearn/manifold/tests/test_mds.py
@@ -1,85 +1,234 @@
+from unittest.mock import Mock
+
 import numpy as np
-from numpy.testing import assert_array_almost_equal
 import pytest
+from numpy.testing import assert_allclose, assert_array_almost_equal, assert_equal
 
+from sklearn.datasets import load_digits
 from sklearn.manifold import _mds as mds
-from sklearn.utils._testing import ignore_warnings
+from sklearn.metrics import euclidean_distances
 
 
 def test_smacof():
     # test metric smacof using the data of "Modern Multidimensional Scaling",
     # Borg & Groenen, p 154
-    sim = np.array([[0, 5, 3, 4],
-                    [5, 0, 2, 2],
-                    [3, 2, 0, 1],
-                    [4, 2, 1, 0]])
-    Z = np.array([[-.266, -.539],
-                  [.451, .252],
-                  [.016, -.238],
-                  [-.200, .524]])
+    sim = np.array([[0, 5, 3, 4], [5, 0, 2, 2], [3, 2, 0, 1], [4, 2, 1, 0]])
+    Z = np.array([[-0.266, -0.539], [0.451, 0.252], [0.016, -0.238], [-0.200, 0.524]])
     X, _ = mds.smacof(sim, init=Z, n_components=2, max_iter=1, n_init=1)
-    X_true = np.array([[-1.415, -2.471],
-                       [1.633, 1.107],
-                       [.249, -.067],
-                       [-.468, 1.431]])
+    X_true = np.array(
+        [[-1.415, -2.471], [1.633, 1.107], [0.249, -0.067], [-0.468, 1.431]]
+    )
     assert_array_almost_equal(X, X_true, decimal=3)
 
 
+def test_nonmetric_lower_normalized_stress():
+    # Testing that nonmetric MDS results in lower normalized stress compared
+    # compared to metric MDS (non-regression test for issue 27028)
+    sim = np.array([[0, 5, 3, 4], [5, 0, 2, 2], [3, 2, 0, 1], [4, 2, 1, 0]])
+    Z = np.array([[-0.266, -0.539], [0.451, 0.252], [0.016, -0.238], [-0.200, 0.524]])
+
+    _, stress1 = mds.smacof(
+        sim, init=Z, n_components=2, max_iter=1000, n_init=1, normalized_stress=True
+    )
+
+    _, stress2 = mds.smacof(
+        sim,
+        init=Z,
+        n_components=2,
+        max_iter=1000,
+        n_init=1,
+        normalized_stress=True,
+        metric=False,
+    )
+    assert stress1 > stress2
+
+
+def test_nonmetric_mds_optimization():
+    # Test that stress is decreasing during nonmetric MDS optimization
+    # (non-regression test for issue 27028)
+    X, _ = load_digits(return_X_y=True)
+    rng = np.random.default_rng(seed=42)
+    ind_subset = rng.choice(len(X), size=200, replace=False)
+    X = X[ind_subset]
+
+    mds_est = mds.MDS(
+        n_components=2,
+        n_init=1,
+        max_iter=2,
+        metric=False,
+        random_state=42,
+    ).fit(X)
+    stress_after_2_iter = mds_est.stress_
+
+    mds_est = mds.MDS(
+        n_components=2,
+        n_init=1,
+        max_iter=3,
+        metric=False,
+        random_state=42,
+    ).fit(X)
+    stress_after_3_iter = mds_est.stress_
+
+    assert stress_after_2_iter > stress_after_3_iter
+
+
+@pytest.mark.parametrize("metric", [True, False])
+def test_mds_recovers_true_data(metric):
+    X = np.array([[1, 1], [1, 4], [1, 5], [3, 3]])
+    mds_est = mds.MDS(
+        n_components=2,
+        n_init=1,
+        eps=1e-15,
+        max_iter=1000,
+        metric=metric,
+        random_state=42,
+    ).fit(X)
+    stress = mds_est.stress_
+    assert_allclose(stress, 0, atol=1e-6)
+
+
 def test_smacof_error():
     # Not symmetric similarity matrix:
-    sim = np.array([[0, 5, 9, 4],
-                    [5, 0, 2, 2],
-                    [3, 2, 0, 1],
-                    [4, 2, 1, 0]])
+    sim = np.array([[0, 5, 9, 4], [5, 0, 2, 2], [3, 2, 0, 1], [4, 2, 1, 0]])
 
     with pytest.raises(ValueError):
-        mds.smacof(sim)
+        mds.smacof(sim, n_init=1)
 
     # Not squared similarity matrix:
-    sim = np.array([[0, 5, 9, 4],
-                    [5, 0, 2, 2],
-                    [4, 2, 1, 0]])
+    sim = np.array([[0, 5, 9, 4], [5, 0, 2, 2], [4, 2, 1, 0]])
 
     with pytest.raises(ValueError):
-        mds.smacof(sim)
+        mds.smacof(sim, n_init=1)
 
     # init not None and not correct format:
-    sim = np.array([[0, 5, 3, 4],
-                    [5, 0, 2, 2],
-                    [3, 2, 0, 1],
-                    [4, 2, 1, 0]])
-
-    Z = np.array([[-.266, -.539],
-                  [.016, -.238],
-                  [-.200, .524]])
+    sim = np.array([[0, 5, 3, 4], [5, 0, 2, 2], [3, 2, 0, 1], [4, 2, 1, 0]])
+
+    Z = np.array([[-0.266, -0.539], [0.016, -0.238], [-0.200, 0.524]])
     with pytest.raises(ValueError):
         mds.smacof(sim, init=Z, n_init=1)
 
 
 def test_MDS():
-    sim = np.array([[0, 5, 3, 4],
-                    [5, 0, 2, 2],
-                    [3, 2, 0, 1],
-                    [4, 2, 1, 0]])
-    mds_clf = mds.MDS(metric=False, n_jobs=3, dissimilarity="precomputed")
+    sim = np.array([[0, 5, 3, 4], [5, 0, 2, 2], [3, 2, 0, 1], [4, 2, 1, 0]])
+    mds_clf = mds.MDS(
+        metric=False,
+        n_jobs=3,
+        n_init=3,
+        dissimilarity="precomputed",
+    )
     mds_clf.fit(sim)
 
 
-# TODO: Remove in 1.1
-def test_MDS_pairwise_deprecated():
-    mds_clf = mds.MDS(metric='precomputed')
-    msg = r"Attribute _pairwise was deprecated in version 0\.24"
-    with pytest.warns(FutureWarning, match=msg):
-        mds_clf._pairwise
-
-
-# TODO: Remove in 1.1
-@ignore_warnings(category=FutureWarning)
-@pytest.mark.parametrize("dissimilarity, expected_pairwise", [
-   ("precomputed", True),
-   ("euclidean", False),
-])
-def test_MDS_pairwise(dissimilarity, expected_pairwise):
-    # _pairwise attribute is set correctly
-    mds_clf = mds.MDS(dissimilarity=dissimilarity)
-    assert mds_clf._pairwise == expected_pairwise
+# TODO(1.9): remove warning filter
+@pytest.mark.filterwarnings("ignore::FutureWarning")
+@pytest.mark.parametrize("k", [0.5, 1.5, 2])
+def test_normed_stress(k):
+    """Test that non-metric MDS normalized stress is scale-invariant."""
+    sim = np.array([[0, 5, 3, 4], [5, 0, 2, 2], [3, 2, 0, 1], [4, 2, 1, 0]])
+
+    X1, stress1 = mds.smacof(sim, metric=False, max_iter=5, random_state=0)
+    X2, stress2 = mds.smacof(k * sim, metric=False, max_iter=5, random_state=0)
+
+    assert_allclose(stress1, stress2, rtol=1e-5)
+    assert_allclose(X1, X2, rtol=1e-5)
+
+
+# TODO(1.9): remove warning filter
+@pytest.mark.filterwarnings("ignore::FutureWarning")
+@pytest.mark.parametrize("metric", [True, False])
+def test_normalized_stress_auto(metric, monkeypatch):
+    rng = np.random.RandomState(0)
+    X = rng.randn(4, 3)
+    dist = euclidean_distances(X)
+
+    mock = Mock(side_effect=mds._smacof_single)
+    monkeypatch.setattr("sklearn.manifold._mds._smacof_single", mock)
+
+    est = mds.MDS(metric=metric, normalized_stress="auto", random_state=rng)
+    est.fit_transform(X)
+    assert mock.call_args[1]["normalized_stress"] != metric
+
+    mds.smacof(dist, metric=metric, normalized_stress="auto", random_state=rng)
+    assert mock.call_args[1]["normalized_stress"] != metric
+
+
+def test_isotonic_outofbounds():
+    # This particular configuration can trigger out of bounds error
+    # in the isotonic regression (non-regression test for issue 26999)
+    dis = np.array(
+        [
+            [0.0, 1.732050807568877, 1.7320508075688772],
+            [1.732050807568877, 0.0, 6.661338147750939e-16],
+            [1.7320508075688772, 6.661338147750939e-16, 0.0],
+        ]
+    )
+    init = np.array(
+        [
+            [0.08665881585055124, 0.7939114643387546],
+            [0.9959834154297658, 0.7555546025640025],
+            [0.8766008278401566, 0.4227358815811242],
+        ]
+    )
+    mds.smacof(dis, init=init, metric=False, n_init=1)
+
+
+# TODO(1.9): remove warning filter
+@pytest.mark.filterwarnings("ignore::FutureWarning")
+@pytest.mark.parametrize("normalized_stress", [True, False])
+def test_returned_stress(normalized_stress):
+    # Test that the final stress corresponds to the final embedding
+    # (non-regression test for issue 16846)
+    X = np.array([[1, 1], [1, 4], [1, 5], [3, 3]])
+    D = euclidean_distances(X)
+
+    mds_est = mds.MDS(
+        n_components=2,
+        random_state=42,
+        normalized_stress=normalized_stress,
+    ).fit(X)
+
+    Z = mds_est.embedding_
+    stress = mds_est.stress_
+
+    D_mds = euclidean_distances(Z)
+    stress_Z = ((D_mds.ravel() - D.ravel()) ** 2).sum() / 2
+
+    if normalized_stress:
+        stress_Z = np.sqrt(stress_Z / ((D_mds.ravel() ** 2).sum() / 2))
+
+    assert_allclose(stress, stress_Z)
+
+
+# TODO(1.9): remove warning filter
+@pytest.mark.filterwarnings("ignore::FutureWarning")
+@pytest.mark.parametrize("metric", [True, False])
+def test_convergence_does_not_depend_on_scale(metric):
+    # Test that the number of iterations until convergence does not depend on
+    # the scale of the input data
+    X = np.array([[1, 1], [1, 4], [1, 5], [3, 3]])
+
+    mds_est = mds.MDS(
+        n_components=2,
+        random_state=42,
+        metric=metric,
+    )
+
+    mds_est.fit(X * 100)
+    n_iter1 = mds_est.n_iter_
+
+    mds_est.fit(X / 100)
+    n_iter2 = mds_est.n_iter_
+
+    assert_equal(n_iter1, n_iter2)
+
+
+# TODO(1.9): delete this test
+def test_future_warning_n_init():
+    X = np.array([[1, 1], [1, 4], [1, 5], [3, 3]])
+    sim = np.array([[0, 5, 3, 4], [5, 0, 2, 2], [3, 2, 0, 1], [4, 2, 1, 0]])
+
+    with pytest.warns(FutureWarning):
+        mds.smacof(sim)
+
+    with pytest.warns(FutureWarning):
+        mds.MDS().fit(X)
diff --git a/sklearn/manifold/tests/test_spectral_embedding.py b/sklearn/manifold/tests/test_spectral_embedding.py
index 3d196fed45978..4c4115734a404 100644
--- a/sklearn/manifold/tests/test_spectral_embedding.py
+++ b/sklearn/manifold/tests/test_spectral_embedding.py
@@ -1,54 +1,78 @@
-import pytest
+import itertools
+from unittest.mock import Mock
 
 import numpy as np
-
+import pytest
 from scipy import sparse
-from scipy.sparse import csgraph
 from scipy.linalg import eigh
+from scipy.sparse.linalg import eigsh, lobpcg
 
-from sklearn.manifold import SpectralEmbedding
-from sklearn.manifold._spectral_embedding import _graph_is_connected
-from sklearn.manifold._spectral_embedding import _graph_connected_component
-from sklearn.manifold import spectral_embedding
-from sklearn.metrics.pairwise import rbf_kernel
-from sklearn.metrics import normalized_mutual_info_score
-from sklearn.neighbors import NearestNeighbors
 from sklearn.cluster import KMeans
 from sklearn.datasets import make_blobs
+from sklearn.manifold import SpectralEmbedding, _spectral_embedding, spectral_embedding
+from sklearn.manifold._spectral_embedding import (
+    _graph_connected_component,
+    _graph_is_connected,
+)
+from sklearn.metrics import normalized_mutual_info_score, pairwise_distances
+from sklearn.metrics.pairwise import rbf_kernel
+from sklearn.neighbors import NearestNeighbors
+from sklearn.utils._testing import assert_array_almost_equal, assert_array_equal
 from sklearn.utils.extmath import _deterministic_vector_sign_flip
-from sklearn.utils._testing import assert_array_almost_equal
-from sklearn.utils._testing import assert_array_equal
-
+from sklearn.utils.fixes import (
+    COO_CONTAINERS,
+    CSC_CONTAINERS,
+    CSR_CONTAINERS,
+    parse_version,
+    sp_version,
+)
+from sklearn.utils.fixes import laplacian as csgraph_laplacian
+
+try:
+    from pyamg import smoothed_aggregation_solver  # noqa: F401
+
+    pyamg_available = True
+except ImportError:
+    pyamg_available = False
+skip_if_no_pyamg = pytest.mark.skipif(
+    not pyamg_available, reason="PyAMG is required for the tests in this function."
+)
 
 # non centered, sparse centers to check the
-centers = np.array([
-    [0.0, 5.0, 0.0, 0.0, 0.0],
-    [0.0, 0.0, 4.0, 0.0, 0.0],
-    [1.0, 0.0, 0.0, 5.0, 1.0],
-])
+centers = np.array(
+    [
+        [0.0, 5.0, 0.0, 0.0, 0.0],
+        [0.0, 0.0, 4.0, 0.0, 0.0],
+        [1.0, 0.0, 0.0, 5.0, 1.0],
+    ]
+)
 n_samples = 1000
 n_clusters, n_features = centers.shape
-S, true_labels = make_blobs(n_samples=n_samples, centers=centers,
-                            cluster_std=1., random_state=42)
+S, true_labels = make_blobs(
+    n_samples=n_samples, centers=centers, cluster_std=1.0, random_state=42
+)
 
 
 def _assert_equal_with_sign_flipping(A, B, tol=0.0):
-    """ Check array A and B are equal with possible sign flipping on
-    each columns"""
-    tol_squared = tol ** 2
+    """Check array A and B are equal with possible sign flipping on
+    each column"""
+    tol_squared = tol**2
     for A_col, B_col in zip(A.T, B.T):
-        assert (np.max((A_col - B_col) ** 2) <= tol_squared or
-                np.max((A_col + B_col) ** 2) <= tol_squared)
+        assert (
+            np.max((A_col - B_col) ** 2) <= tol_squared
+            or np.max((A_col + B_col) ** 2) <= tol_squared
+        )
 
 
-def test_sparse_graph_connected_component():
+@pytest.mark.parametrize("coo_container", COO_CONTAINERS)
+def test_sparse_graph_connected_component(coo_container):
     rng = np.random.RandomState(42)
     n_samples = 300
     boundaries = [0, 42, 121, 200, n_samples]
     p = rng.permutation(n_samples)
     connections = []
 
-    for start, stop in zip(boundaries[:-1], boundaries[1:]):
+    for start, stop in itertools.pairwise(boundaries):
         group = p[start:stop]
         # Connect all elements within the group at least once via an
         # arbitrary path that spans the group.
@@ -64,11 +88,11 @@ def test_sparse_graph_connected_component():
 
     # Build a symmetric affinity matrix
     row_idx, column_idx = tuple(np.array(connections).T)
-    data = rng.uniform(.1, 42, size=len(connections))
-    affinity = sparse.coo_matrix((data, (row_idx, column_idx)))
+    data = rng.uniform(0.1, 42, size=len(connections))
+    affinity = coo_container((data, (row_idx, column_idx)))
     affinity = 0.5 * (affinity + affinity.T)
 
-    for start, stop in zip(boundaries[:-1], boundaries[1:]):
+    for start, stop in itertools.pairwise(boundaries):
         component_1 = _graph_connected_component(affinity, p[start])
         component_size = stop - start
         assert component_1.sum() == component_size
@@ -80,17 +104,32 @@ def test_sparse_graph_connected_component():
         assert_array_equal(component_1, component_2)
 
 
-def test_spectral_embedding_two_components(seed=36):
+# TODO: investigate why this test is seed-sensitive on 32-bit Python
+# runtimes. Is this revealing a numerical stability problem ? Or is it
+# expected from the test numerical design ? In the latter case the test
+# should be made less seed-sensitive instead.
+@pytest.mark.parametrize(
+    "eigen_solver",
+    [
+        "arpack",
+        "lobpcg",
+        pytest.param("amg", marks=skip_if_no_pyamg),
+    ],
+)
+@pytest.mark.parametrize("dtype", [np.float32, np.float64])
+def test_spectral_embedding_two_components(eigen_solver, dtype, seed=0):
     # Test spectral embedding with two components
     random_state = np.random.RandomState(seed)
     n_sample = 100
     affinity = np.zeros(shape=[n_sample * 2, n_sample * 2])
     # first component
-    affinity[0:n_sample,
-             0:n_sample] = np.abs(random_state.randn(n_sample, n_sample)) + 2
+    affinity[0:n_sample, 0:n_sample] = (
+        np.abs(random_state.randn(n_sample, n_sample)) + 2
+    )
     # second component
-    affinity[n_sample::,
-             n_sample::] = np.abs(random_state.randn(n_sample, n_sample)) + 2
+    affinity[n_sample::, n_sample::] = (
+        np.abs(random_state.randn(n_sample, n_sample)) + 2
+    )
 
     # Test of internal _graph_connected_component before connection
     component = _graph_connected_component(affinity, 0)
@@ -103,38 +142,58 @@ def test_spectral_embedding_two_components(seed=36):
     # connection
     affinity[0, n_sample + 1] = 1
     affinity[n_sample + 1, 0] = 1
-    affinity.flat[::2 * n_sample + 1] = 0
+    affinity.flat[:: 2 * n_sample + 1] = 0
     affinity = 0.5 * (affinity + affinity.T)
 
     true_label = np.zeros(shape=2 * n_sample)
     true_label[0:n_sample] = 1
 
-    se_precomp = SpectralEmbedding(n_components=1, affinity="precomputed",
-                                   random_state=np.random.RandomState(seed))
-    embedded_coordinate = se_precomp.fit_transform(affinity)
-    # Some numpy versions are touchy with types
-    embedded_coordinate = \
-        se_precomp.fit_transform(affinity.astype(np.float32))
-    # thresholding on the first components using 0.
-    label_ = np.array(embedded_coordinate.ravel() < 0, dtype="float")
-    assert normalized_mutual_info_score(
-        true_label, label_) == pytest.approx(1.0)
-
+    se_precomp = SpectralEmbedding(
+        n_components=1,
+        affinity="precomputed",
+        random_state=np.random.RandomState(seed),
+        eigen_solver=eigen_solver,
+    )
 
-@pytest.mark.parametrize("X", [S, sparse.csr_matrix(S)],
-                         ids=["dense", "sparse"])
-def test_spectral_embedding_precomputed_affinity(X, seed=36):
+    embedded_coordinate = se_precomp.fit_transform(affinity.astype(dtype))
+    # thresholding on the first components using 0.
+    label_ = np.array(embedded_coordinate.ravel() < 0, dtype=np.int64)
+    assert normalized_mutual_info_score(true_label, label_) == pytest.approx(1.0)
+
+
+@pytest.mark.parametrize("sparse_container", [None, *CSR_CONTAINERS])
+@pytest.mark.parametrize(
+    "eigen_solver",
+    [
+        "arpack",
+        "lobpcg",
+        pytest.param("amg", marks=skip_if_no_pyamg),
+    ],
+)
+@pytest.mark.parametrize("dtype", (np.float32, np.float64))
+def test_spectral_embedding_precomputed_affinity(
+    sparse_container, eigen_solver, dtype, seed=36
+):
     # Test spectral embedding with precomputed kernel
     gamma = 1.0
-    se_precomp = SpectralEmbedding(n_components=2, affinity="precomputed",
-                                   random_state=np.random.RandomState(seed))
-    se_rbf = SpectralEmbedding(n_components=2, affinity="rbf",
-                               gamma=gamma,
-                               random_state=np.random.RandomState(seed))
-    embed_precomp = se_precomp.fit_transform(rbf_kernel(X, gamma=gamma))
-    embed_rbf = se_rbf.fit_transform(X)
-    assert_array_almost_equal(
-        se_precomp.affinity_matrix_, se_rbf.affinity_matrix_)
+    X = S if sparse_container is None else sparse_container(S)
+
+    se_precomp = SpectralEmbedding(
+        n_components=2,
+        affinity="precomputed",
+        random_state=np.random.RandomState(seed),
+        eigen_solver=eigen_solver,
+    )
+    se_rbf = SpectralEmbedding(
+        n_components=2,
+        affinity="rbf",
+        gamma=gamma,
+        random_state=np.random.RandomState(seed),
+        eigen_solver=eigen_solver,
+    )
+    embed_precomp = se_precomp.fit_transform(rbf_kernel(X.astype(dtype), gamma=gamma))
+    embed_rbf = se_rbf.fit_transform(X.astype(dtype))
+    assert_array_almost_equal(se_precomp.affinity_matrix_, se_rbf.affinity_matrix_)
     _assert_equal_with_sign_flipping(embed_precomp, embed_rbf, 0.05)
 
 
@@ -143,163 +202,182 @@ def test_precomputed_nearest_neighbors_filtering():
     n_neighbors = 2
     results = []
     for additional_neighbors in [0, 10]:
-        nn = NearestNeighbors(
-            n_neighbors=n_neighbors + additional_neighbors).fit(S)
-        graph = nn.kneighbors_graph(S, mode='connectivity')
-        embedding = SpectralEmbedding(random_state=0, n_components=2,
-                                      affinity='precomputed_nearest_neighbors',
-                                      n_neighbors=n_neighbors
-                                      ).fit(graph).embedding_
+        nn = NearestNeighbors(n_neighbors=n_neighbors + additional_neighbors).fit(S)
+        graph = nn.kneighbors_graph(S, mode="connectivity")
+        embedding = (
+            SpectralEmbedding(
+                random_state=0,
+                n_components=2,
+                affinity="precomputed_nearest_neighbors",
+                n_neighbors=n_neighbors,
+            )
+            .fit(graph)
+            .embedding_
+        )
         results.append(embedding)
 
     assert_array_equal(results[0], results[1])
 
 
-@pytest.mark.parametrize("X", [S, sparse.csr_matrix(S)],
-                         ids=["dense", "sparse"])
-def test_spectral_embedding_callable_affinity(X, seed=36):
+@pytest.mark.parametrize("sparse_container", [None, *CSR_CONTAINERS])
+def test_spectral_embedding_callable_affinity(sparse_container, seed=36):
     # Test spectral embedding with callable affinity
     gamma = 0.9
     kern = rbf_kernel(S, gamma=gamma)
-    se_callable = SpectralEmbedding(n_components=2,
-                                    affinity=(
-                                        lambda x: rbf_kernel(x, gamma=gamma)),
-                                    gamma=gamma,
-                                    random_state=np.random.RandomState(seed))
-    se_rbf = SpectralEmbedding(n_components=2, affinity="rbf",
-                               gamma=gamma,
-                               random_state=np.random.RandomState(seed))
+    X = S if sparse_container is None else sparse_container(S)
+
+    se_callable = SpectralEmbedding(
+        n_components=2,
+        affinity=(lambda x: rbf_kernel(x, gamma=gamma)),
+        gamma=gamma,
+        random_state=np.random.RandomState(seed),
+    )
+    se_rbf = SpectralEmbedding(
+        n_components=2,
+        affinity="rbf",
+        gamma=gamma,
+        random_state=np.random.RandomState(seed),
+    )
     embed_rbf = se_rbf.fit_transform(X)
     embed_callable = se_callable.fit_transform(X)
-    assert_array_almost_equal(
-        se_callable.affinity_matrix_, se_rbf.affinity_matrix_)
+    assert_array_almost_equal(se_callable.affinity_matrix_, se_rbf.affinity_matrix_)
     assert_array_almost_equal(kern, se_rbf.affinity_matrix_)
     _assert_equal_with_sign_flipping(embed_rbf, embed_callable, 0.05)
 
 
-# TODO: Remove when pyamg does replaces sp.rand call with np.random.rand
-# https://github.com/scikit-learn/scikit-learn/issues/15913
-@pytest.mark.filterwarnings(
-    "ignore:scipy.rand is deprecated:DeprecationWarning:pyamg.*")
-# TODO: Remove when pyamg removes the use of np.float
-@pytest.mark.filterwarnings(
-    "ignore:`np.float` is a deprecated alias:DeprecationWarning:pyamg.*")
-def test_spectral_embedding_amg_solver(seed=36):
-    # Test spectral embedding with amg solver
-    pytest.importorskip('pyamg')
-
-    se_amg = SpectralEmbedding(n_components=2, affinity="nearest_neighbors",
-                               eigen_solver="amg", n_neighbors=5,
-                               random_state=np.random.RandomState(seed))
-    se_arpack = SpectralEmbedding(n_components=2, affinity="nearest_neighbors",
-                                  eigen_solver="arpack", n_neighbors=5,
-                                  random_state=np.random.RandomState(seed))
-    embed_amg = se_amg.fit_transform(S)
-    embed_arpack = se_arpack.fit_transform(S)
+@pytest.mark.skipif(
+    not pyamg_available, reason="PyAMG is required for the tests in this function."
+)
+@pytest.mark.parametrize("dtype", (np.float32, np.float64))
+@pytest.mark.parametrize("coo_container", COO_CONTAINERS)
+def test_spectral_embedding_amg_solver(dtype, coo_container, seed=36):
+    se_amg = SpectralEmbedding(
+        n_components=2,
+        affinity="nearest_neighbors",
+        eigen_solver="amg",
+        n_neighbors=5,
+        random_state=np.random.RandomState(seed),
+    )
+    se_arpack = SpectralEmbedding(
+        n_components=2,
+        affinity="nearest_neighbors",
+        eigen_solver="arpack",
+        n_neighbors=5,
+        random_state=np.random.RandomState(seed),
+    )
+    embed_amg = se_amg.fit_transform(S.astype(dtype))
+    embed_arpack = se_arpack.fit_transform(S.astype(dtype))
     _assert_equal_with_sign_flipping(embed_amg, embed_arpack, 1e-5)
 
     # same with special case in which amg is not actually used
     # regression test for #10715
     # affinity between nodes
-    row = [0, 0, 1, 2, 3, 3, 4]
-    col = [1, 2, 2, 3, 4, 5, 5]
-    val = [100, 100, 100, 1, 100, 100, 100]
-
-    affinity = sparse.coo_matrix((val + val, (row + col, col + row)),
-                                 shape=(6, 6)).toarray()
+    row = np.array([0, 0, 1, 2, 3, 3, 4], dtype=np.int32)
+    col = np.array([1, 2, 2, 3, 4, 5, 5], dtype=np.int32)
+    val = np.array([100, 100, 100, 1, 100, 100, 100], dtype=np.int64)
+
+    affinity = coo_container(
+        (np.hstack([val, val]), (np.hstack([row, col]), np.hstack([col, row]))),
+        shape=(6, 6),
+    )
     se_amg.affinity = "precomputed"
     se_arpack.affinity = "precomputed"
-    embed_amg = se_amg.fit_transform(affinity)
-    embed_arpack = se_arpack.fit_transform(affinity)
+    embed_amg = se_amg.fit_transform(affinity.astype(dtype))
+    embed_arpack = se_arpack.fit_transform(affinity.astype(dtype))
     _assert_equal_with_sign_flipping(embed_amg, embed_arpack, 1e-5)
 
-
-# TODO: Remove filterwarnings when pyamg does replaces sp.rand call with
-# np.random.rand:
-# https://github.com/scikit-learn/scikit-learn/issues/15913
-@pytest.mark.filterwarnings(
-    "ignore:scipy.rand is deprecated:DeprecationWarning:pyamg.*")
-# TODO: Remove when pyamg removes the use of np.float
-@pytest.mark.filterwarnings(
-    "ignore:`np.float` is a deprecated alias:DeprecationWarning:pyamg.*")
-def test_spectral_embedding_amg_solver_failure():
+    # Check that passing a sparse matrix with `np.int64` indices dtype raises an error
+    # or is successful based on the version of SciPy which is installed.
+    # Use a CSR matrix to avoid any conversion during the validation
+    affinity = affinity.tocsr()
+    affinity.indptr = affinity.indptr.astype(np.int64)
+    affinity.indices = affinity.indices.astype(np.int64)
+
+    # PR: https://github.com/scipy/scipy/pull/18913
+    # First integration in 1.11.3: https://github.com/scipy/scipy/pull/19279
+    scipy_graph_traversal_supports_int64_index = sp_version >= parse_version("1.11.3")
+    if scipy_graph_traversal_supports_int64_index:
+        se_amg.fit_transform(affinity)
+    else:
+        err_msg = "Only sparse matrices with 32-bit integer indices are accepted"
+        with pytest.raises(ValueError, match=err_msg):
+            se_amg.fit_transform(affinity)
+
+
+@pytest.mark.skipif(
+    not pyamg_available, reason="PyAMG is required for the tests in this function."
+)
+@pytest.mark.parametrize("dtype", (np.float32, np.float64))
+def test_spectral_embedding_amg_solver_failure(dtype, seed=36):
     # Non-regression test for amg solver failure (issue #13393 on github)
-    pytest.importorskip('pyamg')
-    seed = 36
     num_nodes = 100
     X = sparse.rand(num_nodes, num_nodes, density=0.1, random_state=seed)
+    X = X.astype(dtype)
     upper = sparse.triu(X) - sparse.diags(X.diagonal())
     sym_matrix = upper + upper.T
-    embedding = spectral_embedding(sym_matrix,
-                                   n_components=10,
-                                   eigen_solver='amg',
-                                   random_state=0)
+    embedding = spectral_embedding(
+        sym_matrix, n_components=10, eigen_solver="amg", random_state=0
+    )
 
     # Check that the learned embedding is stable w.r.t. random solver init:
     for i in range(3):
-        new_embedding = spectral_embedding(sym_matrix,
-                                           n_components=10,
-                                           eigen_solver='amg',
-                                           random_state=i + 1)
+        new_embedding = spectral_embedding(
+            sym_matrix, n_components=10, eigen_solver="amg", random_state=i + 1
+        )
         _assert_equal_with_sign_flipping(embedding, new_embedding, tol=0.05)
 
 
-@pytest.mark.filterwarnings("ignore:the behavior of nmi will "
-                            "change in version 0.22")
 def test_pipeline_spectral_clustering(seed=36):
     # Test using pipeline to do spectral clustering
     random_state = np.random.RandomState(seed)
-    se_rbf = SpectralEmbedding(n_components=n_clusters,
-                               affinity="rbf",
-                               random_state=random_state)
-    se_knn = SpectralEmbedding(n_components=n_clusters,
-                               affinity="nearest_neighbors",
-                               n_neighbors=5,
-                               random_state=random_state)
+    se_rbf = SpectralEmbedding(
+        n_components=n_clusters, affinity="rbf", random_state=random_state
+    )
+    se_knn = SpectralEmbedding(
+        n_components=n_clusters,
+        affinity="nearest_neighbors",
+        n_neighbors=5,
+        random_state=random_state,
+    )
     for se in [se_rbf, se_knn]:
-        km = KMeans(n_clusters=n_clusters, random_state=random_state)
+        km = KMeans(n_clusters=n_clusters, random_state=random_state, n_init=10)
         km.fit(se.fit_transform(S))
         assert_array_almost_equal(
-            normalized_mutual_info_score(
-                km.labels_,
-                true_labels), 1.0, 2)
-
-
-def test_spectral_embedding_unknown_eigensolver(seed=36):
-    # Test that SpectralClustering fails with an unknown eigensolver
-    se = SpectralEmbedding(n_components=1, affinity="precomputed",
-                           random_state=np.random.RandomState(seed),
-                           eigen_solver="")
-    with pytest.raises(ValueError):
-        se.fit(S)
-
-
-def test_spectral_embedding_unknown_affinity(seed=36):
-    # Test that SpectralClustering fails with an unknown affinity type
-    se = SpectralEmbedding(n_components=1, affinity="",
-                           random_state=np.random.RandomState(seed))
-    with pytest.raises(ValueError):
-        se.fit(S)
+            normalized_mutual_info_score(km.labels_, true_labels), 1.0, 2
+        )
 
 
 def test_connectivity(seed=36):
     # Test that graph connectivity test works as expected
-    graph = np.array([[1, 0, 0, 0, 0],
-                      [0, 1, 1, 0, 0],
-                      [0, 1, 1, 1, 0],
-                      [0, 0, 1, 1, 1],
-                      [0, 0, 0, 1, 1]])
+    graph = np.array(
+        [
+            [1, 0, 0, 0, 0],
+            [0, 1, 1, 0, 0],
+            [0, 1, 1, 1, 0],
+            [0, 0, 1, 1, 1],
+            [0, 0, 0, 1, 1],
+        ]
+    )
     assert not _graph_is_connected(graph)
-    assert not _graph_is_connected(sparse.csr_matrix(graph))
-    assert not _graph_is_connected(sparse.csc_matrix(graph))
-    graph = np.array([[1, 1, 0, 0, 0],
-                      [1, 1, 1, 0, 0],
-                      [0, 1, 1, 1, 0],
-                      [0, 0, 1, 1, 1],
-                      [0, 0, 0, 1, 1]])
+    for csr_container in CSR_CONTAINERS:
+        assert not _graph_is_connected(csr_container(graph))
+    for csc_container in CSC_CONTAINERS:
+        assert not _graph_is_connected(csc_container(graph))
+
+    graph = np.array(
+        [
+            [1, 1, 0, 0, 0],
+            [1, 1, 1, 0, 0],
+            [0, 1, 1, 1, 0],
+            [0, 0, 1, 1, 1],
+            [0, 0, 0, 1, 1],
+        ]
+    )
     assert _graph_is_connected(graph)
-    assert _graph_is_connected(sparse.csr_matrix(graph))
-    assert _graph_is_connected(sparse.csc_matrix(graph))
+    for csr_container in CSR_CONTAINERS:
+        assert _graph_is_connected(csr_container(graph))
+    for csc_container in CSC_CONTAINERS:
+        assert _graph_is_connected(csc_container(graph))
 
 
 def test_spectral_embedding_deterministic():
@@ -319,14 +397,12 @@ def test_spectral_embedding_unnormalized():
     data = random_state.randn(10, 30)
     sims = rbf_kernel(data)
     n_components = 8
-    embedding_1 = spectral_embedding(sims,
-                                     norm_laplacian=False,
-                                     n_components=n_components,
-                                     drop_first=False)
+    embedding_1 = spectral_embedding(
+        sims, norm_laplacian=False, n_components=n_components, drop_first=False
+    )
 
     # Verify using manual computation with dense eigh
-    laplacian, dd = csgraph.laplacian(sims, normed=False,
-                                      return_diag=True)
+    laplacian, dd = csgraph_laplacian(sims, normed=False, return_diag=True)
     _, diffusion_map = eigh(laplacian)
     embedding_2 = diffusion_map.T[:n_components]
     embedding_2 = _deterministic_vector_sign_flip(embedding_2).T
@@ -343,21 +419,85 @@ def test_spectral_embedding_first_eigen_vector():
     n_components = 2
 
     for seed in range(10):
-        embedding = spectral_embedding(sims,
-                                       norm_laplacian=False,
-                                       n_components=n_components,
-                                       drop_first=False,
-                                       random_state=seed)
+        embedding = spectral_embedding(
+            sims,
+            norm_laplacian=False,
+            n_components=n_components,
+            drop_first=False,
+            random_state=seed,
+        )
 
         assert np.std(embedding[:, 0]) == pytest.approx(0)
         assert np.std(embedding[:, 1]) > 1e-3
 
 
-# TODO: Remove in 1.1
-@pytest.mark.parametrize("affinity", ["precomputed",
-                                      "precomputed_nearest_neighbors"])
-def test_spectral_embedding_pairwise_deprecated(affinity):
-    se = SpectralEmbedding(affinity=affinity)
-    msg = r"Attribute _pairwise was deprecated in version 0\.24"
-    with pytest.warns(FutureWarning, match=msg):
-        se._pairwise
+@pytest.mark.parametrize(
+    "eigen_solver",
+    [
+        "arpack",
+        "lobpcg",
+        pytest.param("amg", marks=skip_if_no_pyamg),
+    ],
+)
+@pytest.mark.parametrize("dtype", [np.float32, np.float64])
+def test_spectral_embedding_preserves_dtype(eigen_solver, dtype):
+    """Check that `SpectralEmbedding is preserving the dtype of the fitted
+    attribute and transformed data.
+
+    Ideally, this test should be covered by the common test
+    `check_transformer_preserve_dtypes`. However, this test only run
+    with transformers implementing `transform` while `SpectralEmbedding`
+    implements only `fit_transform`.
+    """
+    X = S.astype(dtype)
+    se = SpectralEmbedding(
+        n_components=2, affinity="rbf", eigen_solver=eigen_solver, random_state=0
+    )
+    X_trans = se.fit_transform(X)
+
+    assert X_trans.dtype == dtype
+    assert se.embedding_.dtype == dtype
+    assert se.affinity_matrix_.dtype == dtype
+
+
+@pytest.mark.skipif(
+    pyamg_available,
+    reason="PyAMG is installed and we should not test for an error.",
+)
+def test_error_pyamg_not_available():
+    se_precomp = SpectralEmbedding(
+        n_components=2,
+        affinity="rbf",
+        eigen_solver="amg",
+    )
+    err_msg = "The eigen_solver was set to 'amg', but pyamg is not available."
+    with pytest.raises(ValueError, match=err_msg):
+        se_precomp.fit_transform(S)
+
+
+@pytest.mark.parametrize("solver", ["arpack", "amg", "lobpcg"])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_spectral_eigen_tol_auto(monkeypatch, solver, csr_container):
+    """Test that `eigen_tol="auto"` is resolved correctly"""
+    if solver == "amg" and not pyamg_available:
+        pytest.skip("PyAMG is not available.")
+    X, _ = make_blobs(
+        n_samples=200, random_state=0, centers=[[1, 1], [-1, -1]], cluster_std=0.01
+    )
+    D = pairwise_distances(X)  # Distance matrix
+    S = np.max(D) - D  # Similarity matrix
+
+    solver_func = eigsh if solver == "arpack" else lobpcg
+    default_value = 0 if solver == "arpack" else None
+    if solver == "amg":
+        S = csr_container(S)
+
+    mocked_solver = Mock(side_effect=solver_func)
+
+    monkeypatch.setattr(_spectral_embedding, solver_func.__qualname__, mocked_solver)
+
+    spectral_embedding(S, random_state=42, eigen_solver=solver, eigen_tol="auto")
+    mocked_solver.assert_called()
+
+    _, kwargs = mocked_solver.call_args
+    assert kwargs["tol"] == default_value
diff --git a/sklearn/manifold/tests/test_t_sne.py b/sklearn/manifold/tests/test_t_sne.py
index bd0cc3df339bf..4f32b889d5b1f 100644
--- a/sklearn/manifold/tests/test_t_sne.py
+++ b/sklearn/manifold/tests/test_t_sne.py
@@ -1,44 +1,54 @@
+import re
 import sys
 from io import StringIO
+
 import numpy as np
-from numpy.testing import assert_allclose
-import scipy.sparse as sp
 import pytest
+import scipy.sparse as sp
+from numpy.testing import assert_allclose
+from scipy.optimize import check_grad
+from scipy.spatial.distance import pdist, squareform
 
-from sklearn.neighbors import NearestNeighbors
-from sklearn.neighbors import kneighbors_graph
-from sklearn.exceptions import EfficiencyWarning
-from sklearn.utils._testing import ignore_warnings
-from sklearn.utils._testing import assert_almost_equal
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils._testing import assert_array_almost_equal
-from sklearn.utils._testing import skip_if_32bit
-from sklearn.utils import check_random_state
-from sklearn.manifold._t_sne import _joint_probabilities
-from sklearn.manifold._t_sne import _joint_probabilities_nn
-from sklearn.manifold._t_sne import _kl_divergence
-from sklearn.manifold._t_sne import _kl_divergence_bh
-from sklearn.manifold._t_sne import _gradient_descent
-from sklearn.manifold._t_sne import trustworthiness
-from sklearn.manifold import TSNE
-# mypy error: Module 'sklearn.manifold' has no attribute '_barnes_hut_tsne'
-from sklearn.manifold import _barnes_hut_tsne  # type: ignore
-from sklearn.manifold._utils import _binary_search_perplexity
+from sklearn import config_context
 from sklearn.datasets import make_blobs
-from scipy.optimize import check_grad
-from scipy.spatial.distance import pdist
-from scipy.spatial.distance import squareform
-from sklearn.metrics.pairwise import pairwise_distances
-from sklearn.metrics.pairwise import manhattan_distances
-from sklearn.metrics.pairwise import cosine_distances
 
+# mypy error: Module 'sklearn.manifold' has no attribute '_barnes_hut_tsne'
+from sklearn.manifold import (  # type: ignore[attr-defined]
+    TSNE,
+    _barnes_hut_tsne,
+)
+from sklearn.manifold._t_sne import (
+    _gradient_descent,
+    _joint_probabilities,
+    _joint_probabilities_nn,
+    _kl_divergence,
+    _kl_divergence_bh,
+    trustworthiness,
+)
+from sklearn.manifold._utils import _binary_search_perplexity
+from sklearn.metrics.pairwise import (
+    cosine_distances,
+    manhattan_distances,
+    pairwise_distances,
+)
+from sklearn.neighbors import NearestNeighbors, kneighbors_graph
+from sklearn.utils import check_random_state
+from sklearn.utils._testing import (
+    assert_almost_equal,
+    assert_array_almost_equal,
+    assert_array_equal,
+    skip_if_32bit,
+)
+from sklearn.utils.fixes import CSR_CONTAINERS, LIL_CONTAINERS
 
 x = np.linspace(0, 1, 10)
 xx, yy = np.meshgrid(x, x)
-X_2d_grid = np.hstack([
-    xx.ravel().reshape(-1, 1),
-    yy.ravel().reshape(-1, 1),
-])
+X_2d_grid = np.hstack(
+    [
+        xx.ravel().reshape(-1, 1),
+        yy.ravel().reshape(-1, 1),
+    ]
+)
 
 
 def test_gradient_descent_stops():
@@ -59,48 +69,72 @@ def flat_function(_, compute_error=True):
     sys.stdout = StringIO()
     try:
         _, error, it = _gradient_descent(
-            ObjectiveSmallGradient(), np.zeros(1), 0, n_iter=100,
-            n_iter_without_progress=100, momentum=0.0, learning_rate=0.0,
-            min_gain=0.0, min_grad_norm=1e-5, verbose=2)
+            ObjectiveSmallGradient(),
+            np.zeros(1),
+            0,
+            max_iter=100,
+            n_iter_without_progress=100,
+            momentum=0.0,
+            learning_rate=0.0,
+            min_gain=0.0,
+            min_grad_norm=1e-5,
+            verbose=2,
+        )
     finally:
         out = sys.stdout.getvalue()
         sys.stdout.close()
         sys.stdout = old_stdout
     assert error == 1.0
     assert it == 0
-    assert("gradient norm" in out)
+    assert "gradient norm" in out
 
     # Maximum number of iterations without improvement
     old_stdout = sys.stdout
     sys.stdout = StringIO()
     try:
         _, error, it = _gradient_descent(
-            flat_function, np.zeros(1), 0, n_iter=100,
-            n_iter_without_progress=10, momentum=0.0, learning_rate=0.0,
-            min_gain=0.0, min_grad_norm=0.0, verbose=2)
+            flat_function,
+            np.zeros(1),
+            0,
+            max_iter=100,
+            n_iter_without_progress=10,
+            momentum=0.0,
+            learning_rate=0.0,
+            min_gain=0.0,
+            min_grad_norm=0.0,
+            verbose=2,
+        )
     finally:
         out = sys.stdout.getvalue()
         sys.stdout.close()
         sys.stdout = old_stdout
     assert error == 0.0
     assert it == 11
-    assert("did not make any progress" in out)
+    assert "did not make any progress" in out
 
     # Maximum number of iterations
     old_stdout = sys.stdout
     sys.stdout = StringIO()
     try:
         _, error, it = _gradient_descent(
-            ObjectiveSmallGradient(), np.zeros(1), 0, n_iter=11,
-            n_iter_without_progress=100, momentum=0.0, learning_rate=0.0,
-            min_gain=0.0, min_grad_norm=0.0, verbose=2)
+            ObjectiveSmallGradient(),
+            np.zeros(1),
+            0,
+            max_iter=11,
+            n_iter_without_progress=100,
+            momentum=0.0,
+            learning_rate=0.0,
+            min_gain=0.0,
+            min_grad_norm=0.0,
+            verbose=2,
+        )
     finally:
         out = sys.stdout.getvalue()
         sys.stdout.close()
         sys.stdout = old_stdout
     assert error == 0.0
     assert it == 10
-    assert("Iteration 10" in out)
+    assert "Iteration 10" in out
 
 
 def test_binary_search():
@@ -111,8 +145,9 @@ def test_binary_search():
     desired_perplexity = 25.0
     P = _binary_search_perplexity(distances, desired_perplexity, verbose=0)
     P = np.maximum(P, np.finfo(np.double).eps)
-    mean_perplexity = np.mean([np.exp(-np.sum(P[i] * np.log(P[i])))
-                               for i in range(P.shape[0])])
+    mean_perplexity = np.mean(
+        [np.exp(-np.sum(P[i] * np.log(P[i]))) for i in range(P.shape[0])]
+    )
     assert_almost_equal(mean_perplexity, desired_perplexity, decimal=3)
 
 
@@ -142,26 +177,28 @@ def test_binary_search_neighbors():
     # Test that when we use all the neighbors the results are identical
     n_neighbors = n_samples - 1
     nn = NearestNeighbors().fit(data)
-    distance_graph = nn.kneighbors_graph(n_neighbors=n_neighbors,
-                                         mode='distance')
+    distance_graph = nn.kneighbors_graph(n_neighbors=n_neighbors, mode="distance")
     distances_nn = distance_graph.data.astype(np.float32, copy=False)
     distances_nn = distances_nn.reshape(n_samples, n_neighbors)
     P2 = _binary_search_perplexity(distances_nn, desired_perplexity, verbose=0)
 
     indptr = distance_graph.indptr
-    P1_nn = np.array([P1[k, distance_graph.indices[indptr[k]:indptr[k + 1]]]
-                     for k in range(n_samples)])
+    P1_nn = np.array(
+        [
+            P1[k, distance_graph.indices[indptr[k] : indptr[k + 1]]]
+            for k in range(n_samples)
+        ]
+    )
     assert_array_almost_equal(P1_nn, P2, decimal=4)
 
     # Test that the highest P_ij are the same when fewer neighbors are used
     for k in np.linspace(150, n_samples - 1, 5):
         k = int(k)
         topn = k * 10  # check the top 10 * k entries out of k * k entries
-        distance_graph = nn.kneighbors_graph(n_neighbors=k, mode='distance')
+        distance_graph = nn.kneighbors_graph(n_neighbors=k, mode="distance")
         distances_nn = distance_graph.data.astype(np.float32, copy=False)
         distances_nn = distances_nn.reshape(n_samples, k)
-        P2k = _binary_search_perplexity(distances_nn, desired_perplexity,
-                                        verbose=0)
+        P2k = _binary_search_perplexity(distances_nn, desired_perplexity, verbose=0)
         assert_array_almost_equal(P1_nn, P2, decimal=2)
         idx = np.argsort(P1.ravel())[::-1]
         P1top = P1.ravel()[idx][:topn]
@@ -179,17 +216,14 @@ def test_binary_perplexity_stability():
     random_state = check_random_state(0)
     data = random_state.randn(n_samples, 5)
     nn = NearestNeighbors().fit(data)
-    distance_graph = nn.kneighbors_graph(n_neighbors=n_neighbors,
-                                         mode='distance')
+    distance_graph = nn.kneighbors_graph(n_neighbors=n_neighbors, mode="distance")
     distances = distance_graph.data.astype(np.float32, copy=False)
     distances = distances.reshape(n_samples, n_neighbors)
     last_P = None
     desired_perplexity = 3
     for _ in range(100):
-        P = _binary_search_perplexity(distances.copy(), desired_perplexity,
-                                      verbose=0)
-        P1 = _joint_probabilities_nn(distance_graph, desired_perplexity,
-                                     verbose=0)
+        P = _binary_search_perplexity(distances.copy(), desired_perplexity, verbose=0)
+        P1 = _joint_probabilities_nn(distance_graph, desired_perplexity, verbose=0)
         # Convert the sparse matrix to a dense one for testing
         P1 = P1.toarray()
         if last_P is None:
@@ -214,8 +248,7 @@ def test_gradient():
     np.fill_diagonal(distances, 0.0)
     X_embedded = random_state.randn(n_samples, n_components).astype(np.float32)
 
-    P = _joint_probabilities(distances, desired_perplexity=25.0,
-                             verbose=0)
+    P = _joint_probabilities(distances, desired_perplexity=25.0, verbose=0)
 
     def fun(params):
         return _kl_divergence(params, P, alpha, n_samples, n_components)[0]
@@ -223,8 +256,7 @@ def fun(params):
     def grad(params):
         return _kl_divergence(params, P, alpha, n_samples, n_components)[1]
 
-    assert_almost_equal(check_grad(fun, grad, X_embedded.ravel()), 0.0,
-                        decimal=5)
+    assert_almost_equal(check_grad(fun, grad, X_embedded.ravel()), 0.0, decimal=5)
 
 
 def test_trustworthiness():
@@ -247,15 +279,37 @@ def test_trustworthiness():
     assert_almost_equal(trustworthiness(X, X_embedded, n_neighbors=1), 0.2)
 
 
-@pytest.mark.parametrize("method", ['exact', 'barnes_hut'])
-@pytest.mark.parametrize("init", ('random', 'pca'))
+def test_trustworthiness_n_neighbors_error():
+    """Raise an error when n_neighbors >= n_samples / 2.
+
+    Non-regression test for #18567.
+    """
+    regex = "n_neighbors .+ should be less than .+"
+    rng = np.random.RandomState(42)
+    X = rng.rand(7, 4)
+    X_embedded = rng.rand(7, 2)
+    with pytest.raises(ValueError, match=regex):
+        trustworthiness(X, X_embedded, n_neighbors=5)
+
+    trust = trustworthiness(X, X_embedded, n_neighbors=3)
+    assert 0 <= trust <= 1
+
+
+@pytest.mark.parametrize("method", ["exact", "barnes_hut"])
+@pytest.mark.parametrize("init", ("random", "pca"))
 def test_preserve_trustworthiness_approximately(method, init):
     # Nearest neighbors should be preserved approximately.
     random_state = check_random_state(0)
     n_components = 2
     X = random_state.randn(50, n_components).astype(np.float32)
-    tsne = TSNE(n_components=n_components, init=init, random_state=0,
-                method=method, n_iter=700)
+    tsne = TSNE(
+        n_components=n_components,
+        init=init,
+        random_state=0,
+        method=method,
+        max_iter=700,
+        learning_rate="auto",
+    )
     X_embedded = tsne.fit_transform(X)
     t = trustworthiness(X, X_embedded, n_neighbors=1)
     assert t > 0.85
@@ -266,27 +320,42 @@ def test_optimization_minimizes_kl_divergence():
     random_state = check_random_state(0)
     X, _ = make_blobs(n_features=3, random_state=random_state)
     kl_divergences = []
-    for n_iter in [250, 300, 350]:
-        tsne = TSNE(n_components=2, perplexity=10, learning_rate=100.0,
-                    n_iter=n_iter, random_state=0)
+    for max_iter in [250, 300, 350]:
+        tsne = TSNE(
+            n_components=2,
+            init="random",
+            perplexity=10,
+            learning_rate=100.0,
+            max_iter=max_iter,
+            random_state=0,
+        )
         tsne.fit_transform(X)
         kl_divergences.append(tsne.kl_divergence_)
     assert kl_divergences[1] <= kl_divergences[0]
     assert kl_divergences[2] <= kl_divergences[1]
 
 
-@pytest.mark.parametrize('method', ['exact', 'barnes_hut'])
-def test_fit_csr_matrix(method):
+@pytest.mark.parametrize("method", ["exact", "barnes_hut"])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_fit_transform_csr_matrix(method, csr_container):
+    # TODO: compare results on dense and sparse data as proposed in:
+    # https://github.com/scikit-learn/scikit-learn/pull/23585#discussion_r968388186
     # X can be a sparse matrix.
     rng = check_random_state(0)
     X = rng.randn(50, 2)
     X[(rng.randint(0, 50, 25), rng.randint(0, 2, 25))] = 0.0
-    X_csr = sp.csr_matrix(X)
-    tsne = TSNE(n_components=2, perplexity=10, learning_rate=100.0,
-                random_state=0, method=method, n_iter=750)
+    X_csr = csr_container(X)
+    tsne = TSNE(
+        n_components=2,
+        init="random",
+        perplexity=10,
+        learning_rate=100.0,
+        random_state=0,
+        method=method,
+        max_iter=750,
+    )
     X_embedded = tsne.fit_transform(X_csr)
-    assert_allclose(trustworthiness(X_csr, X_embedded, n_neighbors=1),
-                    1.0, rtol=1.1e-1)
+    assert_allclose(trustworthiness(X_csr, X_embedded, n_neighbors=1), 1.0, rtol=1.1e-1)
 
 
 def test_preserve_trustworthiness_approximately_with_precomputed_distances():
@@ -295,13 +364,20 @@ def test_preserve_trustworthiness_approximately_with_precomputed_distances():
     for i in range(3):
         X = random_state.randn(80, 2)
         D = squareform(pdist(X), "sqeuclidean")
-        tsne = TSNE(n_components=2, perplexity=2, learning_rate=100.0,
-                    early_exaggeration=2.0, metric="precomputed",
-                    random_state=i, verbose=0, n_iter=500,
-                    square_distances=True)
+        tsne = TSNE(
+            n_components=2,
+            perplexity=2,
+            learning_rate=100.0,
+            early_exaggeration=2.0,
+            metric="precomputed",
+            random_state=i,
+            verbose=0,
+            max_iter=500,
+            init="random",
+        )
         X_embedded = tsne.fit_transform(D)
         t = trustworthiness(D, X_embedded, n_neighbors=1, metric="precomputed")
-        assert t > .95
+        assert t > 0.95
 
 
 def test_trustworthiness_not_euclidean_metric():
@@ -309,75 +385,84 @@ def test_trustworthiness_not_euclidean_metric():
     # 'precomputed'
     random_state = check_random_state(0)
     X = random_state.randn(100, 2)
-    assert (trustworthiness(X, X, metric='cosine') ==
-            trustworthiness(pairwise_distances(X, metric='cosine'), X,
-                            metric='precomputed'))
-
-
-def test_early_exaggeration_too_small():
-    # Early exaggeration factor must be >= 1.
-    tsne = TSNE(early_exaggeration=0.99)
-    with pytest.raises(ValueError, match="early_exaggeration .*"):
-        tsne.fit_transform(np.array([[0.0], [0.0]]))
-
-
-def test_too_few_iterations():
-    # Number of gradient descent iterations must be at least 200.
-    tsne = TSNE(n_iter=199)
-    with pytest.raises(ValueError, match="n_iter .*"):
-        tsne.fit_transform(np.array([[0.0], [0.0]]))
-
-
-@pytest.mark.parametrize('method, retype', [
-    ('exact', np.asarray),
-    ('barnes_hut', np.asarray),
-    ('barnes_hut', sp.csr_matrix),
-])
-@pytest.mark.parametrize('D, message_regex', [
-    ([[0.0], [1.0]], ".* square distance matrix"),
-    ([[0., -1.], [1., 0.]], ".* positive.*"),
-])
+    assert trustworthiness(X, X, metric="cosine") == trustworthiness(
+        pairwise_distances(X, metric="cosine"), X, metric="precomputed"
+    )
+
+
+@pytest.mark.parametrize(
+    "method, retype",
+    [
+        ("exact", np.asarray),
+        ("barnes_hut", np.asarray),
+        *[("barnes_hut", csr_container) for csr_container in CSR_CONTAINERS],
+    ],
+)
+@pytest.mark.parametrize(
+    "D, message_regex",
+    [
+        ([[0.0], [1.0]], ".* square distance matrix"),
+        ([[0.0, -1.0], [1.0, 0.0]], ".* positive.*"),
+    ],
+)
 def test_bad_precomputed_distances(method, D, retype, message_regex):
-    tsne = TSNE(metric="precomputed", method=method,
-                square_distances=True)
+    tsne = TSNE(
+        metric="precomputed",
+        method=method,
+        init="random",
+        random_state=42,
+        perplexity=1,
+    )
     with pytest.raises(ValueError, match=message_regex):
         tsne.fit_transform(retype(D))
 
 
-def test_exact_no_precomputed_sparse():
-    tsne = TSNE(metric='precomputed', method='exact', square_distances=True)
-    with pytest.raises(TypeError, match='sparse'):
-        tsne.fit_transform(sp.csr_matrix([[0, 5], [5, 0]]))
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_exact_no_precomputed_sparse(csr_container):
+    tsne = TSNE(
+        metric="precomputed",
+        method="exact",
+        init="random",
+        random_state=42,
+        perplexity=1,
+    )
+    with pytest.raises(TypeError, match="sparse"):
+        tsne.fit_transform(csr_container([[0, 5], [5, 0]]))
 
 
-def test_high_perplexity_precomputed_sparse_distances():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_high_perplexity_precomputed_sparse_distances(csr_container):
     # Perplexity should be less than 50
-    dist = np.array([[1., 0., 0.], [0., 1., 0.], [1., 0., 0.]])
-    bad_dist = sp.csr_matrix(dist)
-    tsne = TSNE(metric="precomputed", square_distances=True)
+    dist = np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [1.0, 0.0, 0.0]])
+    bad_dist = csr_container(dist)
+    tsne = TSNE(metric="precomputed", init="random", random_state=42, perplexity=1)
     msg = "3 neighbors per samples are required, but some samples have only 1"
     with pytest.raises(ValueError, match=msg):
         tsne.fit_transform(bad_dist)
 
 
-@ignore_warnings(category=EfficiencyWarning)
-def test_sparse_precomputed_distance():
+@pytest.mark.filterwarnings(
+    "ignore:Precomputed sparse input was not sorted by "
+    "row values:sklearn.exceptions.EfficiencyWarning"
+)
+@pytest.mark.parametrize("sparse_container", CSR_CONTAINERS + LIL_CONTAINERS)
+def test_sparse_precomputed_distance(sparse_container):
     """Make sure that TSNE works identically for sparse and dense matrix"""
     random_state = check_random_state(0)
     X = random_state.randn(100, 2)
 
-    D_sparse = kneighbors_graph(X, n_neighbors=100, mode='distance',
-                                include_self=True)
+    D_sparse = kneighbors_graph(X, n_neighbors=100, mode="distance", include_self=True)
     D = pairwise_distances(X)
     assert sp.issparse(D_sparse)
-    assert_almost_equal(D_sparse.A, D)
+    assert_almost_equal(D_sparse.toarray(), D)
 
-    tsne = TSNE(metric="precomputed", random_state=0, square_distances=True)
+    tsne = TSNE(
+        metric="precomputed", random_state=0, init="random", learning_rate="auto"
+    )
     Xt_dense = tsne.fit_transform(D)
 
-    for fmt in ['csr', 'lil']:
-        Xt_sparse = tsne.fit_transform(D_sparse.asformat(fmt))
-        assert_almost_equal(Xt_dense, Xt_sparse)
+    Xt_sparse = tsne.fit_transform(sparse_container(D_sparse))
+    assert_almost_equal(Xt_dense, Xt_sparse)
 
 
 def test_non_positive_computed_distances():
@@ -386,23 +471,15 @@ def metric(x, y):
         return -1
 
     # Negative computed distances should be caught even if result is squared
-    tsne = TSNE(metric=metric, method='exact', square_distances=True)
+    tsne = TSNE(metric=metric, method="exact", perplexity=1)
     X = np.array([[0.0, 0.0], [1.0, 1.0]])
     with pytest.raises(ValueError, match="All distances .*metric given.*"):
         tsne.fit_transform(X)
 
 
-def test_init_not_available():
-    # 'init' must be 'pca', 'random', or numpy array.
-    tsne = TSNE(init="not available")
-    m = "'init' must be 'pca', 'random', or a numpy array"
-    with pytest.raises(ValueError, match=m):
-        tsne.fit_transform(np.array([[0.0], [1.0]]))
-
-
 def test_init_ndarray():
     # Initialize TSNE with ndarray and test fit
-    tsne = TSNE(init=np.zeros((100, 2)))
+    tsne = TSNE(init=np.zeros((100, 2)), learning_rate="auto")
     X_embedded = tsne.fit_transform(np.ones((100, 5)))
     assert_array_equal(np.zeros((100, 2)), X_embedded)
 
@@ -410,58 +487,35 @@ def test_init_ndarray():
 def test_init_ndarray_precomputed():
     # Initialize TSNE with ndarray and metric 'precomputed'
     # Make sure no FutureWarning is thrown from _fit
-    tsne = TSNE(init=np.zeros((100, 2)), metric="precomputed",
-                square_distances=True)
+    tsne = TSNE(
+        init=np.zeros((100, 2)),
+        metric="precomputed",
+        learning_rate=50.0,
+    )
     tsne.fit(np.zeros((100, 100)))
 
 
-def test_distance_not_available():
-    # 'metric' must be valid.
-    tsne = TSNE(metric="not available", method='exact', square_distances=True)
-    with pytest.raises(ValueError, match="Unknown metric not available.*"):
-        tsne.fit_transform(np.array([[0.0], [1.0]]))
-
-    tsne = TSNE(metric="not available", method='barnes_hut',
-                square_distances=True)
-    with pytest.raises(ValueError, match="Metric 'not available' not valid.*"):
-        tsne.fit_transform(np.array([[0.0], [1.0]]))
-
-
-def test_method_not_available():
-    # 'nethod' must be 'barnes_hut' or 'exact'
-    tsne = TSNE(method='not available')
-    with pytest.raises(ValueError, match="'method' must be 'barnes_hut' or "):
-        tsne.fit_transform(np.array([[0.0], [1.0]]))
-
-
-def test_square_distances_not_available():
-    # square_distances must be True or 'legacy'.
-    tsne = TSNE(square_distances="not_available")
-    with pytest.raises(ValueError, match="'square_distances' must be True or"):
+def test_pca_initialization_not_compatible_with_precomputed_kernel():
+    # Precomputed distance matrices cannot use PCA initialization.
+    tsne = TSNE(metric="precomputed", init="pca", perplexity=1)
+    with pytest.raises(
+        ValueError,
+        match='The parameter init="pca" cannot be used with metric="precomputed".',
+    ):
         tsne.fit_transform(np.array([[0.0], [1.0]]))
 
 
-def test_angle_out_of_range_checks():
-    # check the angle parameter range
-    for angle in [-1, -1e-6, 1 + 1e-6, 2]:
-        tsne = TSNE(angle=angle)
-        with pytest.raises(ValueError, match="'angle' must be between "
-                                             "0.0 - 1.0"):
-            tsne.fit_transform(np.array([[0.0], [1.0]]))
-
-
-def test_pca_initialization_not_compatible_with_precomputed_kernel():
-    # Precomputed distance matrices must be square matrices.
-    tsne = TSNE(metric="precomputed", init="pca", square_distances=True)
-    with pytest.raises(ValueError, match="The parameter init=\"pca\" cannot"
-                                         " be used with"
-                                         " metric=\"precomputed\"."):
-        tsne.fit_transform(np.array([[0.0], [1.0]]))
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_pca_initialization_not_compatible_with_sparse_input(csr_container):
+    # Sparse input matrices cannot use PCA initialization.
+    tsne = TSNE(init="pca", learning_rate=100.0, perplexity=1)
+    with pytest.raises(TypeError, match="PCA initialization.*"):
+        tsne.fit_transform(csr_container([[0, 5], [5, 0]]))
 
 
 def test_n_components_range():
     # barnes_hut method should only be used with n_components <= 3
-    tsne = TSNE(n_components=4, method="barnes_hut")
+    tsne = TSNE(n_components=4, method="barnes_hut", perplexity=1)
     with pytest.raises(ValueError, match="'n_components' should be .*"):
         tsne.fit_transform(np.array([[0.0], [1.0]]))
 
@@ -470,75 +524,104 @@ def test_early_exaggeration_used():
     # check that the ``early_exaggeration`` parameter has an effect
     random_state = check_random_state(0)
     n_components = 2
-    methods = ['exact', 'barnes_hut']
+    methods = ["exact", "barnes_hut"]
     X = random_state.randn(25, n_components).astype(np.float32)
     for method in methods:
-        tsne = TSNE(n_components=n_components, perplexity=1,
-                    learning_rate=100.0, init="pca", random_state=0,
-                    method=method, early_exaggeration=1.0, n_iter=250)
+        tsne = TSNE(
+            n_components=n_components,
+            perplexity=1,
+            learning_rate=100.0,
+            init="pca",
+            random_state=0,
+            method=method,
+            early_exaggeration=1.0,
+            max_iter=250,
+        )
         X_embedded1 = tsne.fit_transform(X)
-        tsne = TSNE(n_components=n_components, perplexity=1,
-                    learning_rate=100.0, init="pca", random_state=0,
-                    method=method, early_exaggeration=10.0, n_iter=250)
+        tsne = TSNE(
+            n_components=n_components,
+            perplexity=1,
+            learning_rate=100.0,
+            init="pca",
+            random_state=0,
+            method=method,
+            early_exaggeration=10.0,
+            max_iter=250,
+        )
         X_embedded2 = tsne.fit_transform(X)
 
         assert not np.allclose(X_embedded1, X_embedded2)
 
 
-def test_n_iter_used():
-    # check that the ``n_iter`` parameter has an effect
+def test_max_iter_used():
+    # check that the ``max_iter`` parameter has an effect
     random_state = check_random_state(0)
     n_components = 2
-    methods = ['exact', 'barnes_hut']
+    methods = ["exact", "barnes_hut"]
     X = random_state.randn(25, n_components).astype(np.float32)
     for method in methods:
-        for n_iter in [251, 500]:
-            tsne = TSNE(n_components=n_components, perplexity=1,
-                        learning_rate=0.5, init="random", random_state=0,
-                        method=method, early_exaggeration=1.0, n_iter=n_iter)
+        for max_iter in [251, 500]:
+            tsne = TSNE(
+                n_components=n_components,
+                perplexity=1,
+                learning_rate=0.5,
+                init="random",
+                random_state=0,
+                method=method,
+                early_exaggeration=1.0,
+                max_iter=max_iter,
+            )
             tsne.fit_transform(X)
 
-            assert tsne.n_iter_ == n_iter - 1
+            assert tsne.n_iter_ == max_iter - 1
 
 
-def test_answer_gradient_two_points():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_answer_gradient_two_points(csr_container):
     # Test the tree with only a single set of children.
     #
     # These tests & answers have been checked against the reference
     # implementation by LvdM.
     pos_input = np.array([[1.0, 0.0], [0.0, 1.0]])
-    pos_output = np.array([[-4.961291e-05, -1.072243e-04],
-                           [9.259460e-05, 2.702024e-04]])
-    neighbors = np.array([[1],
-                          [0]])
-    grad_output = np.array([[-2.37012478e-05, -6.29044398e-05],
-                            [2.37012478e-05, 6.29044398e-05]])
-    _run_answer_test(pos_input, pos_output, neighbors, grad_output)
-
-
-def test_answer_gradient_four_points():
+    pos_output = np.array(
+        [[-4.961291e-05, -1.072243e-04], [9.259460e-05, 2.702024e-04]]
+    )
+    neighbors = np.array([[1], [0]])
+    grad_output = np.array(
+        [[-2.37012478e-05, -6.29044398e-05], [2.37012478e-05, 6.29044398e-05]]
+    )
+    _run_answer_test(pos_input, pos_output, neighbors, grad_output, csr_container)
+
+
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_answer_gradient_four_points(csr_container):
     # Four points tests the tree with multiple levels of children.
     #
     # These tests & answers have been checked against the reference
     # implementation by LvdM.
-    pos_input = np.array([[1.0, 0.0], [0.0, 1.0],
-                          [5.0, 2.0], [7.3, 2.2]])
-    pos_output = np.array([[6.080564e-05, -7.120823e-05],
-                           [-1.718945e-04, -4.000536e-05],
-                           [-2.271720e-04, 8.663310e-05],
-                           [-1.032577e-04, -3.582033e-05]])
-    neighbors = np.array([[1, 2, 3],
-                          [0, 2, 3],
-                          [1, 0, 3],
-                          [1, 2, 0]])
-    grad_output = np.array([[5.81128448e-05, -7.78033454e-06],
-                            [-5.81526851e-05, 7.80976444e-06],
-                            [4.24275173e-08, -3.69569698e-08],
-                            [-2.58720939e-09, 7.52706374e-09]])
-    _run_answer_test(pos_input, pos_output, neighbors, grad_output)
-
-
-def test_skip_num_points_gradient():
+    pos_input = np.array([[1.0, 0.0], [0.0, 1.0], [5.0, 2.0], [7.3, 2.2]])
+    pos_output = np.array(
+        [
+            [6.080564e-05, -7.120823e-05],
+            [-1.718945e-04, -4.000536e-05],
+            [-2.271720e-04, 8.663310e-05],
+            [-1.032577e-04, -3.582033e-05],
+        ]
+    )
+    neighbors = np.array([[1, 2, 3], [0, 2, 3], [1, 0, 3], [1, 2, 0]])
+    grad_output = np.array(
+        [
+            [5.81128448e-05, -7.78033454e-06],
+            [-5.81526851e-05, 7.80976444e-06],
+            [4.24275173e-08, -3.69569698e-08],
+            [-2.58720939e-09, 7.52706374e-09],
+        ]
+    )
+    _run_answer_test(pos_input, pos_output, neighbors, grad_output, csr_container)
+
+
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_skip_num_points_gradient(csr_container):
     # Test the kwargs option skip_num_points.
     #
     # Skip num points should make it such that the Barnes_hut gradient
@@ -546,26 +629,39 @@ def test_skip_num_points_gradient():
     # Aside from skip_num_points=2 and the first two gradient rows
     # being set to zero, these data points are the same as in
     # test_answer_gradient_four_points()
-    pos_input = np.array([[1.0, 0.0], [0.0, 1.0],
-                          [5.0, 2.0], [7.3, 2.2]])
-    pos_output = np.array([[6.080564e-05, -7.120823e-05],
-                           [-1.718945e-04, -4.000536e-05],
-                           [-2.271720e-04, 8.663310e-05],
-                           [-1.032577e-04, -3.582033e-05]])
-    neighbors = np.array([[1, 2, 3],
-                          [0, 2, 3],
-                          [1, 0, 3],
-                          [1, 2, 0]])
-    grad_output = np.array([[0.0, 0.0],
-                            [0.0, 0.0],
-                            [4.24275173e-08, -3.69569698e-08],
-                            [-2.58720939e-09, 7.52706374e-09]])
-    _run_answer_test(pos_input, pos_output, neighbors, grad_output,
-                     False, 0.1, 2)
-
-
-def _run_answer_test(pos_input, pos_output, neighbors, grad_output,
-                     verbose=False, perplexity=0.1, skip_num_points=0):
+    pos_input = np.array([[1.0, 0.0], [0.0, 1.0], [5.0, 2.0], [7.3, 2.2]])
+    pos_output = np.array(
+        [
+            [6.080564e-05, -7.120823e-05],
+            [-1.718945e-04, -4.000536e-05],
+            [-2.271720e-04, 8.663310e-05],
+            [-1.032577e-04, -3.582033e-05],
+        ]
+    )
+    neighbors = np.array([[1, 2, 3], [0, 2, 3], [1, 0, 3], [1, 2, 0]])
+    grad_output = np.array(
+        [
+            [0.0, 0.0],
+            [0.0, 0.0],
+            [4.24275173e-08, -3.69569698e-08],
+            [-2.58720939e-09, 7.52706374e-09],
+        ]
+    )
+    _run_answer_test(
+        pos_input, pos_output, neighbors, grad_output, csr_container, False, 0.1, 2
+    )
+
+
+def _run_answer_test(
+    pos_input,
+    pos_output,
+    neighbors,
+    grad_output,
+    csr_container,
+    verbose=False,
+    perplexity=0.1,
+    skip_num_points=0,
+):
     distances = pairwise_distances(pos_input).astype(np.float32)
     args = distances, perplexity, verbose
     pos_output = pos_output.astype(np.float32)
@@ -574,21 +670,21 @@ def _run_answer_test(pos_input, pos_output, neighbors, grad_output,
     pij_input = squareform(pij_input).astype(np.float32)
     grad_bh = np.zeros(pos_output.shape, dtype=np.float32)
 
-    from scipy.sparse import csr_matrix
-    P = csr_matrix(pij_input)
+    P = csr_container(pij_input)
 
     neighbors = P.indices.astype(np.int64)
     indptr = P.indptr.astype(np.int64)
 
-    _barnes_hut_tsne.gradient(P.data, pos_output, neighbors, indptr,
-                              grad_bh, 0.5, 2, 1, skip_num_points=0)
+    _barnes_hut_tsne.gradient(
+        P.data, pos_output, neighbors, indptr, grad_bh, 0.5, 2, 1, skip_num_points=0
+    )
     assert_array_almost_equal(grad_bh, grad_output, decimal=4)
 
 
 def test_verbose():
     # Verbose options write to stdout.
     random_state = check_random_state(0)
-    tsne = TSNE(verbose=2)
+    tsne = TSNE(verbose=2, perplexity=4)
     X = random_state.randn(5, 2)
 
     old_stdout = sys.stdout
@@ -600,17 +696,17 @@ def test_verbose():
         sys.stdout.close()
         sys.stdout = old_stdout
 
-    assert("[t-SNE]" in out)
-    assert("nearest neighbors..." in out)
-    assert("Computed conditional probabilities" in out)
-    assert("Mean sigma" in out)
-    assert("early exaggeration" in out)
+    assert "[t-SNE]" in out
+    assert "nearest neighbors..." in out
+    assert "Computed conditional probabilities" in out
+    assert "Mean sigma" in out
+    assert "early exaggeration" in out
 
 
 def test_chebyshev_metric():
     # t-SNE should allow metrics that cannot be squared (issue #3526).
     random_state = check_random_state(0)
-    tsne = TSNE(metric="chebyshev", square_distances=True)
+    tsne = TSNE(metric="chebyshev", perplexity=4)
     X = random_state.randn(5, 2)
     tsne.fit_transform(X)
 
@@ -618,22 +714,29 @@ def test_chebyshev_metric():
 def test_reduction_to_one_component():
     # t-SNE should allow reduction to one component (issue #4154).
     random_state = check_random_state(0)
-    tsne = TSNE(n_components=1)
+    tsne = TSNE(n_components=1, perplexity=4)
     X = random_state.randn(5, 2)
     X_embedded = tsne.fit(X).embedding_
-    assert(np.all(np.isfinite(X_embedded)))
+    assert np.all(np.isfinite(X_embedded))
 
 
-@pytest.mark.parametrize('method', ['barnes_hut', 'exact'])
-@pytest.mark.parametrize('dt', [np.float32, np.float64])
+@pytest.mark.parametrize("method", ["barnes_hut", "exact"])
+@pytest.mark.parametrize("dt", [np.float32, np.float64])
 def test_64bit(method, dt):
     # Ensure 64bit arrays are handled correctly.
     random_state = check_random_state(0)
 
     X = random_state.randn(10, 2).astype(dt, copy=False)
-    tsne = TSNE(n_components=2, perplexity=2, learning_rate=100.0,
-                random_state=0, method=method, verbose=0,
-                n_iter=300)
+    tsne = TSNE(
+        n_components=2,
+        perplexity=2,
+        learning_rate=100.0,
+        random_state=0,
+        method=method,
+        verbose=0,
+        max_iter=300,
+        init="random",
+    )
     X_embedded = tsne.fit_transform(X)
     effective_type = X_embedded.dtype
 
@@ -642,15 +745,23 @@ def test_64bit(method, dt):
     assert effective_type == np.float32
 
 
-@pytest.mark.parametrize('method', ['barnes_hut', 'exact'])
+@pytest.mark.parametrize("method", ["barnes_hut", "exact"])
 def test_kl_divergence_not_nan(method):
     # Ensure kl_divergence_ is computed at last iteration
-    # even though n_iter % n_iter_check != 0, i.e. 1003 % 50 != 0
+    # even though max_iter % n_iter_check != 0, i.e. 1003 % 50 != 0
     random_state = check_random_state(0)
 
     X = random_state.randn(50, 2)
-    tsne = TSNE(n_components=2, perplexity=2, learning_rate=100.0,
-                random_state=0, method=method, verbose=0, n_iter=503)
+    tsne = TSNE(
+        n_components=2,
+        perplexity=2,
+        learning_rate=100.0,
+        random_state=0,
+        method=method,
+        verbose=0,
+        max_iter=503,
+        init="random",
+    )
     tsne.fit_transform(X)
 
     assert not np.isnan(tsne.kl_divergence_)
@@ -670,17 +781,27 @@ def test_barnes_hut_angle():
         distances = pairwise_distances(data)
         params = random_state.randn(n_samples, n_components)
         P = _joint_probabilities(distances, perplexity, verbose=0)
-        kl_exact, grad_exact = _kl_divergence(params, P, degrees_of_freedom,
-                                              n_samples, n_components)
+        kl_exact, grad_exact = _kl_divergence(
+            params, P, degrees_of_freedom, n_samples, n_components
+        )
 
         n_neighbors = n_samples - 1
-        distances_csr = NearestNeighbors().fit(data).kneighbors_graph(
-            n_neighbors=n_neighbors, mode='distance')
+        distances_csr = (
+            NearestNeighbors()
+            .fit(data)
+            .kneighbors_graph(n_neighbors=n_neighbors, mode="distance")
+        )
         P_bh = _joint_probabilities_nn(distances_csr, perplexity, verbose=0)
-        kl_bh, grad_bh = _kl_divergence_bh(params, P_bh, degrees_of_freedom,
-                                           n_samples, n_components,
-                                           angle=angle, skip_num_points=0,
-                                           verbose=0)
+        kl_bh, grad_bh = _kl_divergence_bh(
+            params,
+            P_bh,
+            degrees_of_freedom,
+            n_samples,
+            n_components,
+            angle=angle,
+            skip_num_points=0,
+            verbose=0,
+        )
 
         P = squareform(P)
         P_bh = P_bh.toarray()
@@ -694,10 +815,17 @@ def test_n_iter_without_progress():
     random_state = check_random_state(0)
     X = random_state.randn(100, 10)
     for method in ["barnes_hut", "exact"]:
-        tsne = TSNE(n_iter_without_progress=-1, verbose=2, learning_rate=1e8,
-                    random_state=0, method=method, n_iter=351, init="random")
+        tsne = TSNE(
+            n_iter_without_progress=-1,
+            verbose=2,
+            learning_rate=1e8,
+            random_state=0,
+            method=method,
+            max_iter=351,
+            init="random",
+        )
         tsne._N_ITER_CHECK = 1
-        tsne._EXPLORATION_N_ITER = 0
+        tsne._EXPLORATION_MAX_ITER = 0
 
         old_stdout = sys.stdout
         sys.stdout = StringIO()
@@ -709,8 +837,7 @@ def test_n_iter_without_progress():
             sys.stdout = old_stdout
 
         # The output needs to contain the value of n_iter_without_progress
-        assert ("did not make any progress during the "
-                "last -1 episodes. Finished." in out)
+        assert "did not make any progress during the last -1 episodes. Finished." in out
 
 
 def test_min_grad_norm():
@@ -718,8 +845,7 @@ def test_min_grad_norm():
     random_state = check_random_state(0)
     X = random_state.randn(100, 2)
     min_grad_norm = 0.002
-    tsne = TSNE(min_grad_norm=min_grad_norm, verbose=2,
-                random_state=0, method='exact')
+    tsne = TSNE(min_grad_norm=min_grad_norm, verbose=2, random_state=0, method="exact")
 
     old_stdout = sys.stdout
     sys.stdout = StringIO()
@@ -730,26 +856,27 @@ def test_min_grad_norm():
         sys.stdout.close()
         sys.stdout = old_stdout
 
-    lines_out = out.split('\n')
+    lines_out = out.split("\n")
 
     # extract the gradient norm from the verbose output
     gradient_norm_values = []
     for line in lines_out:
         # When the computation is Finished just an old gradient norm value
         # is repeated that we do not need to store
-        if 'Finished' in line:
+        if "Finished" in line:
             break
 
-        start_grad_norm = line.find('gradient norm')
+        start_grad_norm = line.find("gradient norm")
         if start_grad_norm >= 0:
             line = line[start_grad_norm:]
-            line = line.replace('gradient norm = ', '').split(' ')[0]
+            line = line.replace("gradient norm = ", "").split(" ")[0]
             gradient_norm_values.append(float(line))
 
     # Compute how often the gradient norm is smaller than min_grad_norm
     gradient_norm_values = np.array(gradient_norm_values)
-    n_smaller_gradient_norms = \
-        len(gradient_norm_values[gradient_norm_values <= min_grad_norm])
+    n_smaller_gradient_norms = len(
+        gradient_norm_values[gradient_norm_values <= min_grad_norm]
+    )
 
     # The gradient norm can be smaller than min_grad_norm at most once,
     # because in the moment it becomes smaller the optimization stops
@@ -760,9 +887,13 @@ def test_accessible_kl_divergence():
     # Ensures that the accessible kl_divergence matches the computed value
     random_state = check_random_state(0)
     X = random_state.randn(50, 2)
-    tsne = TSNE(n_iter_without_progress=2, verbose=2,
-                random_state=0, method='exact',
-                n_iter=500)
+    tsne = TSNE(
+        n_iter_without_progress=2,
+        verbose=2,
+        random_state=0,
+        method="exact",
+        max_iter=500,
+    )
 
     old_stdout = sys.stdout
     sys.stdout = StringIO()
@@ -775,16 +906,16 @@ def test_accessible_kl_divergence():
 
     # The output needs to contain the accessible kl_divergence as the error at
     # the last iteration
-    for line in out.split('\n')[::-1]:
-        if 'Iteration' in line:
-            _, _, error = line.partition('error = ')
+    for line in out.split("\n")[::-1]:
+        if "Iteration" in line:
+            _, _, error = line.partition("error = ")
             if error:
-                error, _, _ = error.partition(',')
+                error, _, _ = error.partition(",")
                 break
     assert_almost_equal(tsne.kl_divergence_, float(error), decimal=5)
 
 
-@pytest.mark.parametrize('method', ['barnes_hut', 'exact'])
+@pytest.mark.parametrize("method", ["barnes_hut", "exact"])
 def test_uniform_grid(method):
     """Make sure that TSNE can approximately recover a uniform 2D grid
 
@@ -798,10 +929,17 @@ def test_uniform_grid(method):
     enough.
     """
     seeds = range(3)
-    n_iter = 500
+    max_iter = 500
     for seed in seeds:
-        tsne = TSNE(n_components=2, init='random', random_state=seed,
-                    perplexity=50, n_iter=n_iter, method=method)
+        tsne = TSNE(
+            n_components=2,
+            init="random",
+            random_state=seed,
+            perplexity=50,
+            max_iter=max_iter,
+            method=method,
+            learning_rate="auto",
+        )
         Y = tsne.fit_transform(X_2d_grid)
 
         try_name = "{}_{}".format(method, seed)
@@ -828,7 +966,7 @@ def assert_uniform_grid(Y, try_name=None):
     smallest_to_mean = dist_to_nn.min() / np.mean(dist_to_nn)
     largest_to_mean = dist_to_nn.max() / np.mean(dist_to_nn)
 
-    assert smallest_to_mean > .5, try_name
+    assert smallest_to_mean > 0.5, try_name
     assert largest_to_mean < 2, try_name
 
 
@@ -839,18 +977,25 @@ def test_bh_match_exact():
     n_features = 10
     X = random_state.randn(30, n_features).astype(np.float32)
     X_embeddeds = {}
-    n_iter = {}
-    for method in ['exact', 'barnes_hut']:
-        tsne = TSNE(n_components=2, method=method, learning_rate=1.0,
-                    init="random", random_state=0, n_iter=251,
-                    perplexity=30.0, angle=0)
+    max_iter = {}
+    for method in ["exact", "barnes_hut"]:
+        tsne = TSNE(
+            n_components=2,
+            method=method,
+            learning_rate=1.0,
+            init="random",
+            random_state=0,
+            max_iter=251,
+            perplexity=29.5,
+            angle=0,
+        )
         # Kill the early_exaggeration
-        tsne._EXPLORATION_N_ITER = 0
+        tsne._EXPLORATION_MAX_ITER = 0
         X_embeddeds[method] = tsne.fit_transform(X)
-        n_iter[method] = tsne.n_iter_
+        max_iter[method] = tsne.n_iter_
 
-    assert n_iter['exact'] == n_iter['barnes_hut']
-    assert_allclose(X_embeddeds['exact'], X_embeddeds['barnes_hut'], rtol=1e-4)
+    assert max_iter["exact"] == max_iter["barnes_hut"]
+    assert_allclose(X_embeddeds["exact"], X_embeddeds["barnes_hut"], rtol=1e-4)
 
 
 def test_gradient_bh_multithread_match_sequential():
@@ -870,99 +1015,173 @@ def test_gradient_bh_multithread_match_sequential():
     params = random_state.randn(n_samples, n_components)
 
     n_neighbors = n_samples - 1
-    distances_csr = NearestNeighbors().fit(data).kneighbors_graph(
-        n_neighbors=n_neighbors, mode='distance')
+    distances_csr = (
+        NearestNeighbors()
+        .fit(data)
+        .kneighbors_graph(n_neighbors=n_neighbors, mode="distance")
+    )
     P_bh = _joint_probabilities_nn(distances_csr, perplexity, verbose=0)
     kl_sequential, grad_sequential = _kl_divergence_bh(
-        params, P_bh, degrees_of_freedom, n_samples, n_components,
-        angle=angle, skip_num_points=0, verbose=0, num_threads=1)
+        params,
+        P_bh,
+        degrees_of_freedom,
+        n_samples,
+        n_components,
+        angle=angle,
+        skip_num_points=0,
+        verbose=0,
+        num_threads=1,
+    )
     for num_threads in [2, 4]:
         kl_multithread, grad_multithread = _kl_divergence_bh(
-            params, P_bh, degrees_of_freedom, n_samples, n_components,
-            angle=angle, skip_num_points=0, verbose=0, num_threads=num_threads)
+            params,
+            P_bh,
+            degrees_of_freedom,
+            n_samples,
+            n_components,
+            angle=angle,
+            skip_num_points=0,
+            verbose=0,
+            num_threads=num_threads,
+        )
 
         assert_allclose(kl_multithread, kl_sequential, rtol=1e-6)
         assert_allclose(grad_multithread, grad_multithread)
 
 
-def test_tsne_with_different_distance_metrics():
+@pytest.mark.parametrize(
+    "metric, dist_func",
+    [("manhattan", manhattan_distances), ("cosine", cosine_distances)],
+)
+@pytest.mark.parametrize("method", ["barnes_hut", "exact"])
+def test_tsne_with_different_distance_metrics(metric, dist_func, method):
     """Make sure that TSNE works for different distance metrics"""
+
+    if method == "barnes_hut" and metric == "manhattan":
+        # The distances computed by `manhattan_distances` differ slightly from those
+        # computed internally by NearestNeighbors via the PairwiseDistancesReduction
+        # Cython code-based. This in turns causes T-SNE to converge to a different
+        # solution but this should not impact the qualitative results as both
+        # methods.
+        # NOTE: it's probably not valid from a mathematical point of view to use the
+        # Manhattan distance for T-SNE...
+        # TODO: re-enable this test if/when `manhattan_distances` is refactored to
+        # reuse the same underlying Cython code NearestNeighbors.
+        # For reference, see:
+        # https://github.com/scikit-learn/scikit-learn/pull/23865/files#r925721573
+        pytest.xfail(
+            "Distance computations are different for method == 'barnes_hut' and metric"
+            " == 'manhattan', but this is expected."
+        )
+
     random_state = check_random_state(0)
     n_components_original = 3
     n_components_embedding = 2
     X = random_state.randn(50, n_components_original).astype(np.float32)
-    metrics = ['manhattan', 'cosine']
-    dist_funcs = [manhattan_distances, cosine_distances]
-    for metric, dist_func in zip(metrics, dist_funcs):
-        X_transformed_tsne = TSNE(
-            metric=metric, n_components=n_components_embedding,
-            random_state=0, n_iter=300, square_distances=True).fit_transform(X)
-        X_transformed_tsne_precomputed = TSNE(
-            metric='precomputed', n_components=n_components_embedding,
-            random_state=0, n_iter=300,
-            square_distances=True).fit_transform(dist_func(X))
-        assert_array_equal(X_transformed_tsne, X_transformed_tsne_precomputed)
-
-
-@pytest.mark.parametrize('method', ['exact', 'barnes_hut'])
-@pytest.mark.parametrize('metric', ['euclidean', 'manhattan'])
-@pytest.mark.parametrize('square_distances', [True, 'legacy'])
-@ignore_warnings(category=FutureWarning)
-def test_tsne_different_square_distances(method, metric, square_distances):
-    # Make sure that TSNE works for different square_distances settings
-    # FIXME remove test when square_distances=True becomes the default in 1.1
-    random_state = check_random_state(0)
-    n_components_original = 3
-    n_components_embedding = 2
-
-    # Used to create data with structure; this avoids unstable behavior in TSNE
-    X, _ = make_blobs(n_features=n_components_original,
-                      random_state=random_state)
-    X_precomputed = pairwise_distances(X, metric=metric)
-
-    if metric == 'euclidean' and square_distances == 'legacy':
-        X_precomputed **= 2
-
     X_transformed_tsne = TSNE(
-        metric=metric, n_components=n_components_embedding,
-        square_distances=square_distances, method=method,
-        random_state=0).fit_transform(X)
+        metric=metric,
+        method=method,
+        n_components=n_components_embedding,
+        random_state=0,
+        max_iter=300,
+        init="random",
+        learning_rate="auto",
+    ).fit_transform(X)
     X_transformed_tsne_precomputed = TSNE(
-        metric='precomputed', n_components=n_components_embedding,
-        square_distances=square_distances, method=method,
-        random_state=0).fit_transform(X_precomputed)
+        metric="precomputed",
+        method=method,
+        n_components=n_components_embedding,
+        random_state=0,
+        max_iter=300,
+        init="random",
+        learning_rate="auto",
+    ).fit_transform(dist_func(X))
+    assert_array_equal(X_transformed_tsne, X_transformed_tsne_precomputed)
+
+
+@pytest.mark.parametrize("method", ["exact", "barnes_hut"])
+def test_tsne_n_jobs(method):
+    """Make sure that the n_jobs parameter doesn't impact the output"""
+    random_state = check_random_state(0)
+    n_features = 10
+    X = random_state.randn(30, n_features)
+    X_tr_ref = TSNE(
+        n_components=2,
+        method=method,
+        perplexity=25.0,
+        angle=0,
+        n_jobs=1,
+        random_state=0,
+        init="random",
+        learning_rate="auto",
+    ).fit_transform(X)
+    X_tr = TSNE(
+        n_components=2,
+        method=method,
+        perplexity=25.0,
+        angle=0,
+        n_jobs=2,
+        random_state=0,
+        init="random",
+        learning_rate="auto",
+    ).fit_transform(X)
 
-    assert_allclose(X_transformed_tsne, X_transformed_tsne_precomputed)
+    assert_allclose(X_tr_ref, X_tr)
 
 
-@pytest.mark.parametrize('metric', ['euclidean', 'manhattan'])
-@pytest.mark.parametrize('square_distances', [True, 'legacy'])
-def test_tsne_square_distances_futurewarning(metric, square_distances):
-    # Make sure that a FutureWarning is only raised when a non-Euclidean
-    # metric is specified and square_distances is not set to True.
+def test_tsne_with_mahalanobis_distance():
+    """Make sure that method_parameters works with mahalanobis distance."""
     random_state = check_random_state(0)
+    n_samples, n_features = 300, 10
+    X = random_state.randn(n_samples, n_features)
+    default_params = {
+        "perplexity": 40,
+        "max_iter": 250,
+        "learning_rate": "auto",
+        "init": "random",
+        "n_components": 3,
+        "random_state": 0,
+    }
+
+    tsne = TSNE(metric="mahalanobis", **default_params)
+    msg = "Must provide either V or VI for Mahalanobis distance"
+    with pytest.raises(ValueError, match=msg):
+        tsne.fit_transform(X)
 
-    X = random_state.randn(5, 2)
-    tsne = TSNE(metric=metric, square_distances=square_distances)
+    precomputed_X = squareform(pdist(X, metric="mahalanobis"), checks=True)
+    X_trans_expected = TSNE(metric="precomputed", **default_params).fit_transform(
+        precomputed_X
+    )
 
-    if metric != 'euclidean' and square_distances is not True:
-        with pytest.warns(FutureWarning, match="'square_distances'.*"):
-            tsne.fit_transform(X)
-    else:
-        with pytest.warns(None) as record:
-            tsne.fit_transform(X)
-        assert not record
+    X_trans = TSNE(
+        metric="mahalanobis", metric_params={"V": np.cov(X.T)}, **default_params
+    ).fit_transform(X)
+    assert_allclose(X_trans, X_trans_expected)
 
 
-@pytest.mark.parametrize('method', ['exact', 'barnes_hut'])
-def test_tsne_n_jobs(method):
-    """Make sure that the n_jobs parameter doesn't impact the output"""
+@pytest.mark.parametrize("perplexity", (20, 30))
+def test_tsne_perplexity_validation(perplexity):
+    """Make sure that perplexity > n_samples results in a ValueError"""
+
     random_state = check_random_state(0)
-    n_features = 10
-    X = random_state.randn(30, n_features)
-    X_tr_ref = TSNE(n_components=2, method=method, perplexity=30.0,
-                    angle=0, n_jobs=1, random_state=0).fit_transform(X)
-    X_tr = TSNE(n_components=2, method=method, perplexity=30.0,
-                angle=0, n_jobs=2, random_state=0).fit_transform(X)
+    X = random_state.randn(20, 2)
+    est = TSNE(
+        learning_rate="auto",
+        init="pca",
+        perplexity=perplexity,
+        random_state=random_state,
+    )
+    msg = re.escape(f"perplexity ({perplexity}) must be less than n_samples (20)")
+    with pytest.raises(ValueError, match=msg):
+        est.fit_transform(X)
 
-    assert_allclose(X_tr_ref, X_tr)
+
+def test_tsne_works_with_pandas_output():
+    """Make sure that TSNE works when the output is set to "pandas".
+
+    Non-regression test for gh-25365.
+    """
+    pytest.importorskip("pandas")
+    with config_context(transform_output="pandas"):
+        arr = np.arange(35 * 4).reshape(35, 4)
+        TSNE(n_components=2).fit_transform(arr)
diff --git a/sklearn/meson.build b/sklearn/meson.build
new file mode 100644
index 0000000000000..30feb944029d3
--- /dev/null
+++ b/sklearn/meson.build
@@ -0,0 +1,254 @@
+fs = import('fs')
+
+cython_args = []
+
+# Platform detection
+is_windows = host_machine.system() == 'windows'
+is_mingw = is_windows and cc.get_id() == 'gcc'
+
+# Adapted from Scipy. mingw is untested and not officially supported. If you
+# ever bump into issues when trying to compile for mingw, please open an issue
+# in the scikit-learn issue tracker
+if is_mingw
+  # For mingw-w64, link statically against the UCRT.
+  gcc_link_args = ['-lucrt', '-static']
+  add_project_link_arguments(gcc_link_args, language: ['c', 'cpp'])
+  # Force gcc to float64 long doubles for compatibility with MSVC
+  # builds, for C only.
+  add_project_arguments('-mlong-double-64', language: 'c')
+endif
+
+# Only check build dependencies version when not cross-compiling, as running
+# Python interpreter can be tricky in cross-compilation settings. For more
+# details, see https://docs.scipy.org/doc/scipy/building/cross_compilation.html
+if not meson.is_cross_build()
+  if not py.version().version_compare('>=3.10')
+    error('scikit-learn requires Python>=3.10, got ' + py.version() + ' instead')
+  endif
+
+  cython_min_version = run_command(py, ['_min_dependencies.py', 'cython'], check: true).stdout().strip()
+  if not cython.version().version_compare('>=' + cython_min_version)
+    error('scikit-learn requires Cython>=' + cython_min_version + ', got ' + cython.version() + ' instead')
+  endif
+
+  numpy_version = run_command(py,
+    ['-c', 'import numpy; print(numpy.__version__)'], check: true).stdout().strip()
+  numpy_min_version = run_command(py, ['_min_dependencies.py', 'numpy'], check: true).stdout().strip()
+  if not numpy_version.version_compare('>=' + numpy_min_version)
+    error('scikit-learn requires numpy>=' + numpy_min_version + ', got ' + numpy_version + ' instead')
+  endif
+
+  scipy_version = run_command(py,
+    ['-c', 'import scipy; print(scipy.__version__)'], check: true).stdout().strip()
+  scipy_min_version = run_command(py, ['_min_dependencies.py', 'scipy'], check: true).stdout().strip()
+  if not scipy_version.version_compare('>=' + scipy_min_version)
+    error('scikit-learn requires scipy>=' + scipy_min_version + ', got ' + scipy_version + ' instead')
+  endif
+
+  # meson-python is required only when going through pip. Using meson directly
+  # should not check meson-python version.
+  meson_python_version_command_result = run_command(py,
+    ['-c', 'import importlib.metadata; print(importlib.metadata.version("meson-python"))'], check: false)
+  meson_python_installed = meson_python_version_command_result.returncode() == 0
+  if meson_python_installed
+    meson_python_version = meson_python_version_command_result.stdout().strip()
+    meson_python_min_version = run_command(py, ['_min_dependencies.py', 'meson-python'], check: true).stdout().strip()
+    if not meson_python_version.version_compare('>=' + meson_python_min_version)
+      error('scikit-learn requires meson-python>=' + meson_python_min_version + ', got ' + meson_python_version + ' instead')
+    endif
+  endif
+
+endif
+
+# Adapted from scipy, each project seems to have its own tweaks for this. One
+# day using dependency('numpy') will be a thing, see
+# https://github.com/mesonbuild/meson/issues/9598.
+# NumPy include directory - needed in all submodules
+# Relative paths are needed when for example a virtualenv is
+# placed inside the source tree; Meson rejects absolute paths to places inside
+# the source tree. The try-except is needed because when things are split
+# across drives on Windows, there is no relative path and an exception gets
+# raised. There may be other such cases, so add a catch-all and switch to
+# an absolute path.
+# For cross-compilation it is often not possible to run the Python interpreter
+# in order to retrieve numpy's include directory. It can be specified in the
+# cross file instead:
+#   [properties]
+#   numpy-include-dir = /abspath/to/host-pythons/site-packages/numpy/core/include
+#
+# This uses the path as is, and avoids running the interpreter.
+incdir_numpy = meson.get_external_property('numpy-include-dir', 'not-given')
+if incdir_numpy == 'not-given'
+  incdir_numpy = run_command(py,
+    [
+      '-c',
+      '''
+import os
+import numpy as np
+try:
+  incdir = os.path.relpath(np.get_include())
+except Exception:
+  incdir = np.get_include()
+print(incdir)
+'''
+    ],
+    check: true
+  ).stdout().strip()
+endif
+
+inc_np = include_directories(incdir_numpy)
+# Don't use the deprecated NumPy C API. Define this to a fixed version instead of
+# NPY_API_VERSION in order not to break compilation for released SciPy versions
+# when NumPy introduces a new deprecation.
+numpy_no_deprecated_api = ['-DNPY_NO_DEPRECATED_API=NPY_1_9_API_VERSION']
+np_dep = declare_dependency(include_directories: inc_np, compile_args: numpy_no_deprecated_api)
+
+openmp_dep = dependency('OpenMP', language: 'c', required: false)
+
+if not openmp_dep.found()
+  warn_about_missing_openmp = true
+  # On Apple Clang avoid a misleading warning if compiler variables are set.
+  # See https://github.com/scikit-learn/scikit-learn/issues/28710 for more
+  # details. This may be removed if the OpenMP detection on Apple Clang improves,
+  # see https://github.com/mesonbuild/meson/issues/7435#issuecomment-2047585466.
+  if host_machine.system() == 'darwin' and cc.get_id() == 'clang'
+    compiler_env_vars_with_openmp = run_command(py,
+      [
+        '-c',
+        '''
+import os
+
+compiler_env_vars_to_check = ["CPPFLAGS", "CFLAGS", "CXXFLAGS"]
+
+compiler_env_vars_with_openmp = [
+    var for var in compiler_env_vars_to_check if "-fopenmp" in os.getenv(var, "")]
+print(compiler_env_vars_with_openmp)
+'''], check: true).stdout().strip()
+      warn_about_missing_openmp = compiler_env_vars_with_openmp == '[]'
+  endif
+  if warn_about_missing_openmp
+    warning(
+'''
+                ***********
+                * WARNING *
+                ***********
+
+It seems that scikit-learn cannot be built with OpenMP.
+
+- Make sure you have followed the installation instructions:
+
+    https://scikit-learn.org/dev/developers/advanced_installation.html
+
+- If your compiler supports OpenMP but you still see this
+  message, please submit a bug report at:
+
+    https://github.com/scikit-learn/scikit-learn/issues
+
+- The build will continue with OpenMP-based parallelism
+  disabled. Note however that some estimators will run in
+  sequential mode instead of leveraging thread-based
+  parallelism.
+
+                    ***
+''')
+  else
+    warning(
+'''It looks like compiler environment variables were set to enable OpenMP support.
+Check the output of "import sklearn; sklearn.show_versions()" after the build
+to make sure that scikit-learn was actually built with OpenMP support.
+''')
+  endif
+endif
+
+# For now, we keep supporting SKLEARN_ENABLE_DEBUG_CYTHON_DIRECTIVES variable
+# (see how it is done in sklearn/_build_utils/__init__.py when building with
+# setuptools). Accessing environment variables in meson.build is discouraged,
+# so once we drop setuptools this functionality should be behind a meson option
+# or buildtype
+boundscheck = run_command(py,
+    [
+      '-c',
+      '''
+import os
+
+if os.environ.get("SKLEARN_ENABLE_DEBUG_CYTHON_DIRECTIVES", "0") != "0":
+    print(True)
+else:
+    print(False)
+      '''
+    ],
+    check: true
+    ).stdout().strip()
+
+scikit_learn_cython_args = [
+  '-X language_level=3', '-X boundscheck=' + boundscheck, '-X wraparound=False',
+  '-X initializedcheck=False', '-X nonecheck=False', '-X cdivision=True',
+  '-X profile=False',
+  # Needed for cython imports across subpackages, e.g. cluster pyx that
+  # cimports metrics pxd
+  '--include-dir', meson.global_build_root(),
+]
+cython_args += scikit_learn_cython_args
+
+cython_program = find_program(cython.cmd_array()[0])
+
+cython_gen = generator(cython_program,
+  arguments : cython_args + ['@INPUT@', '--output-file', '@OUTPUT@'],
+  output : '@BASENAME@.c',
+)
+
+cython_gen_cpp = generator(cython_program,
+  arguments : cython_args + ['--cplus', '@INPUT@', '--output-file', '@OUTPUT@'],
+  output : '@BASENAME@.cpp',
+)
+
+
+# Write file in Meson build dir to be able to figure out from Python code
+# whether scikit-learn was built with Meson. Adapted from pandas
+# _version_meson.py.
+custom_target('write_built_with_meson_file',
+    output: '_built_with_meson.py',
+    command: [
+        py, '-c', 'with open("sklearn/_built_with_meson.py", "w") as f: f.write("")'
+    ],
+    install: true,
+    install_dir: py.get_install_dir() / 'sklearn'
+)
+
+extensions = ['_isotonic']
+
+py.extension_module(
+  '_isotonic',
+  cython_gen.process('_isotonic.pyx'),
+  cython_args: cython_args,
+  install: true,
+  subdir: 'sklearn',
+)
+
+# Need for Cython cimports across subpackages to work, i.e. avoid errors like
+# relative cimport from non-package directory is not allowed
+sklearn_root_cython_tree = [
+  fs.copyfile('__init__.py')
+]
+
+sklearn_dir = py.get_install_dir() / 'sklearn'
+
+# Subpackages are mostly in alphabetical order except to handle Cython
+# dependencies across subpackages
+subdir('__check_build')
+subdir('_loss')
+# utils needs to be early since plenty of other modules cimports utils .pxd
+subdir('utils')
+# metrics needs to be to be before cluster since cluster cimports metrics .pxd
+subdir('metrics')
+subdir('cluster')
+subdir('datasets')
+subdir('decomposition')
+subdir('ensemble')
+subdir('feature_extraction')
+subdir('linear_model')
+subdir('manifold')
+subdir('neighbors')
+subdir('preprocessing')
+subdir('svm')
+subdir('tree')
diff --git a/sklearn/metrics/__init__.py b/sklearn/metrics/__init__.py
index bca22e3916c61..ce86525acc368 100644
--- a/sklearn/metrics/__init__.py
+++ b/sklearn/metrics/__init__.py
@@ -1,174 +1,181 @@
-"""
-The :mod:`sklearn.metrics` module includes score functions, performance metrics
-and pairwise metrics and distance computations.
-"""
+"""Score functions, performance metrics, pairwise metrics and distance computations."""
 
-
-from ._ranking import auc
-from ._ranking import average_precision_score
-from ._ranking import coverage_error
-from ._ranking import det_curve
-from ._ranking import dcg_score
-from ._ranking import label_ranking_average_precision_score
-from ._ranking import label_ranking_loss
-from ._ranking import ndcg_score
-from ._ranking import precision_recall_curve
-from ._ranking import roc_auc_score
-from ._ranking import roc_curve
-from ._ranking import top_k_accuracy_score
-
-from ._classification import accuracy_score
-from ._classification import balanced_accuracy_score
-from ._classification import classification_report
-from ._classification import cohen_kappa_score
-from ._classification import confusion_matrix
-from ._classification import f1_score
-from ._classification import fbeta_score
-from ._classification import hamming_loss
-from ._classification import hinge_loss
-from ._classification import jaccard_score
-from ._classification import log_loss
-from ._classification import matthews_corrcoef
-from ._classification import precision_recall_fscore_support
-from ._classification import precision_score
-from ._classification import recall_score
-from ._classification import zero_one_loss
-from ._classification import brier_score_loss
-from ._classification import multilabel_confusion_matrix
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 from . import cluster
-from .cluster import adjusted_mutual_info_score
-from .cluster import adjusted_rand_score
-from .cluster import rand_score
-from .cluster import pair_confusion_matrix
-from .cluster import completeness_score
-from .cluster import consensus_score
-from .cluster import homogeneity_completeness_v_measure
-from .cluster import homogeneity_score
-from .cluster import mutual_info_score
-from .cluster import normalized_mutual_info_score
-from .cluster import fowlkes_mallows_score
-from .cluster import silhouette_samples
-from .cluster import silhouette_score
-from .cluster import calinski_harabasz_score
-from .cluster import v_measure_score
-from .cluster import davies_bouldin_score
-
-from .pairwise import euclidean_distances
-from .pairwise import nan_euclidean_distances
-from .pairwise import pairwise_distances
-from .pairwise import pairwise_distances_argmin
-from .pairwise import pairwise_distances_argmin_min
-from .pairwise import pairwise_kernels
-from .pairwise import pairwise_distances_chunked
-
-from ._regression import explained_variance_score
-from ._regression import max_error
-from ._regression import mean_absolute_error
-from ._regression import mean_squared_error
-from ._regression import mean_squared_log_error
-from ._regression import median_absolute_error
-from ._regression import mean_absolute_percentage_error
-from ._regression import mean_pinball_loss
-from ._regression import r2_score
-from ._regression import mean_tweedie_deviance
-from ._regression import mean_poisson_deviance
-from ._regression import mean_gamma_deviance
-
-
-from ._scorer import check_scoring
-from ._scorer import make_scorer
-from ._scorer import SCORERS
-from ._scorer import get_scorer
-
-from ._plot.det_curve import plot_det_curve
+from ._classification import (
+    accuracy_score,
+    balanced_accuracy_score,
+    brier_score_loss,
+    class_likelihood_ratios,
+    classification_report,
+    cohen_kappa_score,
+    confusion_matrix,
+    d2_log_loss_score,
+    f1_score,
+    fbeta_score,
+    hamming_loss,
+    hinge_loss,
+    jaccard_score,
+    log_loss,
+    matthews_corrcoef,
+    multilabel_confusion_matrix,
+    precision_recall_fscore_support,
+    precision_score,
+    recall_score,
+    zero_one_loss,
+)
+from ._dist_metrics import DistanceMetric
+from ._plot.confusion_matrix import ConfusionMatrixDisplay
 from ._plot.det_curve import DetCurveDisplay
-from ._plot.roc_curve import plot_roc_curve
-from ._plot.roc_curve import RocCurveDisplay
-from ._plot.precision_recall_curve import plot_precision_recall_curve
 from ._plot.precision_recall_curve import PrecisionRecallDisplay
-
-from ._plot.confusion_matrix import plot_confusion_matrix
-from ._plot.confusion_matrix import ConfusionMatrixDisplay
-
+from ._plot.regression import PredictionErrorDisplay
+from ._plot.roc_curve import RocCurveDisplay
+from ._ranking import (
+    auc,
+    average_precision_score,
+    coverage_error,
+    dcg_score,
+    det_curve,
+    label_ranking_average_precision_score,
+    label_ranking_loss,
+    ndcg_score,
+    precision_recall_curve,
+    roc_auc_score,
+    roc_curve,
+    top_k_accuracy_score,
+)
+from ._regression import (
+    d2_absolute_error_score,
+    d2_pinball_score,
+    d2_tweedie_score,
+    explained_variance_score,
+    max_error,
+    mean_absolute_error,
+    mean_absolute_percentage_error,
+    mean_gamma_deviance,
+    mean_pinball_loss,
+    mean_poisson_deviance,
+    mean_squared_error,
+    mean_squared_log_error,
+    mean_tweedie_deviance,
+    median_absolute_error,
+    r2_score,
+    root_mean_squared_error,
+    root_mean_squared_log_error,
+)
+from ._scorer import check_scoring, get_scorer, get_scorer_names, make_scorer
+from .cluster import (
+    adjusted_mutual_info_score,
+    adjusted_rand_score,
+    calinski_harabasz_score,
+    completeness_score,
+    consensus_score,
+    davies_bouldin_score,
+    fowlkes_mallows_score,
+    homogeneity_completeness_v_measure,
+    homogeneity_score,
+    mutual_info_score,
+    normalized_mutual_info_score,
+    pair_confusion_matrix,
+    rand_score,
+    silhouette_samples,
+    silhouette_score,
+    v_measure_score,
+)
+from .pairwise import (
+    euclidean_distances,
+    nan_euclidean_distances,
+    pairwise_distances,
+    pairwise_distances_argmin,
+    pairwise_distances_argmin_min,
+    pairwise_distances_chunked,
+    pairwise_kernels,
+)
 
 __all__ = [
-    'accuracy_score',
-    'adjusted_mutual_info_score',
-    'adjusted_rand_score',
-    'auc',
-    'average_precision_score',
-    'balanced_accuracy_score',
-    'calinski_harabasz_score',
-    'check_scoring',
-    'classification_report',
-    'cluster',
-    'cohen_kappa_score',
-    'completeness_score',
-    'ConfusionMatrixDisplay',
-    'confusion_matrix',
-    'consensus_score',
-    'coverage_error',
-    'dcg_score',
-    'davies_bouldin_score',
-    'DetCurveDisplay',
-    'det_curve',
-    'euclidean_distances',
-    'explained_variance_score',
-    'f1_score',
-    'fbeta_score',
-    'fowlkes_mallows_score',
-    'get_scorer',
-    'hamming_loss',
-    'hinge_loss',
-    'homogeneity_completeness_v_measure',
-    'homogeneity_score',
-    'jaccard_score',
-    'label_ranking_average_precision_score',
-    'label_ranking_loss',
-    'log_loss',
-    'make_scorer',
-    'nan_euclidean_distances',
-    'matthews_corrcoef',
-    'max_error',
-    'mean_absolute_error',
-    'mean_squared_error',
-    'mean_squared_log_error',
-    'mean_pinball_loss',
-    'mean_poisson_deviance',
-    'mean_gamma_deviance',
-    'mean_tweedie_deviance',
-    'median_absolute_error',
-    'mean_absolute_percentage_error',
-    'multilabel_confusion_matrix',
-    'mutual_info_score',
-    'ndcg_score',
-    'normalized_mutual_info_score',
-    'pair_confusion_matrix',
-    'pairwise_distances',
-    'pairwise_distances_argmin',
-    'pairwise_distances_argmin_min',
-    'pairwise_distances_chunked',
-    'pairwise_kernels',
-    'plot_confusion_matrix',
-    'plot_det_curve',
-    'plot_precision_recall_curve',
-    'plot_roc_curve',
-    'PrecisionRecallDisplay',
-    'precision_recall_curve',
-    'precision_recall_fscore_support',
-    'precision_score',
-    'r2_score',
-    'rand_score',
-    'recall_score',
-    'RocCurveDisplay',
-    'roc_auc_score',
-    'roc_curve',
-    'SCORERS',
-    'silhouette_samples',
-    'silhouette_score',
-    'top_k_accuracy_score',
-    'v_measure_score',
-    'zero_one_loss',
-    'brier_score_loss',
+    "ConfusionMatrixDisplay",
+    "DetCurveDisplay",
+    "DistanceMetric",
+    "PrecisionRecallDisplay",
+    "PredictionErrorDisplay",
+    "RocCurveDisplay",
+    "accuracy_score",
+    "adjusted_mutual_info_score",
+    "adjusted_rand_score",
+    "auc",
+    "average_precision_score",
+    "balanced_accuracy_score",
+    "brier_score_loss",
+    "calinski_harabasz_score",
+    "check_scoring",
+    "class_likelihood_ratios",
+    "classification_report",
+    "cluster",
+    "cohen_kappa_score",
+    "completeness_score",
+    "confusion_matrix",
+    "consensus_score",
+    "coverage_error",
+    "d2_absolute_error_score",
+    "d2_log_loss_score",
+    "d2_pinball_score",
+    "d2_tweedie_score",
+    "davies_bouldin_score",
+    "dcg_score",
+    "det_curve",
+    "euclidean_distances",
+    "explained_variance_score",
+    "f1_score",
+    "fbeta_score",
+    "fowlkes_mallows_score",
+    "get_scorer",
+    "get_scorer_names",
+    "hamming_loss",
+    "hinge_loss",
+    "homogeneity_completeness_v_measure",
+    "homogeneity_score",
+    "jaccard_score",
+    "label_ranking_average_precision_score",
+    "label_ranking_loss",
+    "log_loss",
+    "make_scorer",
+    "matthews_corrcoef",
+    "max_error",
+    "mean_absolute_error",
+    "mean_absolute_percentage_error",
+    "mean_gamma_deviance",
+    "mean_pinball_loss",
+    "mean_poisson_deviance",
+    "mean_squared_error",
+    "mean_squared_log_error",
+    "mean_tweedie_deviance",
+    "median_absolute_error",
+    "multilabel_confusion_matrix",
+    "mutual_info_score",
+    "nan_euclidean_distances",
+    "ndcg_score",
+    "normalized_mutual_info_score",
+    "pair_confusion_matrix",
+    "pairwise_distances",
+    "pairwise_distances_argmin",
+    "pairwise_distances_argmin_min",
+    "pairwise_distances_chunked",
+    "pairwise_kernels",
+    "precision_recall_curve",
+    "precision_recall_fscore_support",
+    "precision_score",
+    "r2_score",
+    "rand_score",
+    "recall_score",
+    "roc_auc_score",
+    "roc_curve",
+    "root_mean_squared_error",
+    "root_mean_squared_log_error",
+    "silhouette_samples",
+    "silhouette_score",
+    "top_k_accuracy_score",
+    "v_measure_score",
+    "zero_one_loss",
 ]
diff --git a/sklearn/metrics/_base.py b/sklearn/metrics/_base.py
index bacf7519390f3..aa4150c88a978 100644
--- a/sklearn/metrics/_base.py
+++ b/sklearn/metrics/_base.py
@@ -2,15 +2,9 @@
 Common code for all metrics.
 
 """
-# Authors: Alexandre Gramfort 
-#          Mathieu Blondel 
-#          Olivier Grisel 
-#          Arnaud Joly 
-#          Jochen Wersdorfer 
-#          Lars Buitinck
-#          Joel Nothman 
-#          Noel Dawe 
-# License: BSD 3 clause
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 from itertools import combinations
 
@@ -20,8 +14,7 @@
 from ..utils.multiclass import type_of_target
 
 
-def _average_binary_score(binary_metric, y_true, y_score, average,
-                          sample_weight=None):
+def _average_binary_score(binary_metric, y_true, y_score, average, sample_weight=None):
     """Average a binary metric for multilabel classification.
 
     Parameters
@@ -33,7 +26,7 @@ def _average_binary_score(binary_metric, y_true, y_score, average,
         Target scores, can either be probability estimates of the positive
         class, confidence values, or binary decisions.
 
-    average : string, [None, 'micro', 'macro' (default), 'samples', 'weighted']
+    average : {None, 'micro', 'macro', 'samples', 'weighted'}, default='macro'
         If ``None``, the scores for each class are returned. Otherwise,
         this determines the type of averaging performed on the data:
 
@@ -64,10 +57,9 @@ def _average_binary_score(binary_metric, y_true, y_score, average,
         classes.
 
     """
-    average_options = (None, 'micro', 'macro', 'weighted', 'samples')
+    average_options = (None, "micro", "macro", "weighted", "samples")
     if average not in average_options:
-        raise ValueError('average has to be one of {0}'
-                         ''.format(average_options))
+        raise ValueError("average has to be one of {0}".format(average_options))
 
     y_type = type_of_target(y_true)
     if y_type not in ("binary", "multilabel-indicator"):
@@ -90,16 +82,17 @@ def _average_binary_score(binary_metric, y_true, y_score, average,
         y_true = y_true.ravel()
         y_score = y_score.ravel()
 
-    elif average == 'weighted':
+    elif average == "weighted":
         if score_weight is not None:
-            average_weight = np.sum(np.multiply(
-                y_true, np.reshape(score_weight, (-1, 1))), axis=0)
+            average_weight = np.sum(
+                np.multiply(y_true, np.reshape(score_weight, (-1, 1))), axis=0
+            )
         else:
             average_weight = np.sum(y_true, axis=0)
         if np.isclose(average_weight.sum(), 0.0):
             return 0
 
-    elif average == 'samples':
+    elif average == "samples":
         # swap average_weight <-> score_weight
         average_weight = score_weight
         score_weight = None
@@ -116,8 +109,7 @@ def _average_binary_score(binary_metric, y_true, y_score, average,
     for c in range(n_classes):
         y_true_c = y_true.take([c], axis=not_average_axis).ravel()
         y_score_c = y_score.take([c], axis=not_average_axis).ravel()
-        score[c] = binary_metric(y_true_c, y_score_c,
-                                 sample_weight=score_weight)
+        score[c] = binary_metric(y_true_c, y_score_c, sample_weight=score_weight)
 
     # Average the results
     if average is not None:
@@ -126,13 +118,12 @@ def _average_binary_score(binary_metric, y_true, y_score, average,
             # score from being affected by 0-weighted NaN elements.
             average_weight = np.asarray(average_weight)
             score[average_weight == 0] = 0
-        return np.average(score, weights=average_weight)
+        return float(np.average(score, weights=average_weight))
     else:
         return score
 
 
-def _average_multiclass_ovo_score(binary_metric, y_true, y_score,
-                                  average='macro'):
+def _average_multiclass_ovo_score(binary_metric, y_true, y_score, average="macro"):
     """Average one-versus-one scores for multiclass classification.
 
     Uses the binary metric for one-vs-one multiclass classification,
@@ -200,52 +191,3 @@ def _average_multiclass_ovo_score(binary_metric, y_true, y_score,
         pair_scores[ix] = (a_true_score + b_true_score) / 2
 
     return np.average(pair_scores, weights=prevalence)
-
-
-def _check_pos_label_consistency(pos_label, y_true):
-    """Check if `pos_label` need to be specified or not.
-
-    In binary classification, we fix `pos_label=1` if the labels are in the set
-    {-1, 1} or {0, 1}. Otherwise, we raise an error asking to specify the
-    `pos_label` parameters.
-
-    Parameters
-    ----------
-    pos_label : int, str or None
-        The positive label.
-    y_true : ndarray of shape (n_samples,)
-        The target vector.
-
-    Returns
-    -------
-    pos_label : int
-        If `pos_label` can be inferred, it will be returned.
-
-    Raises
-    ------
-    ValueError
-        In the case that `y_true` does not have label in {-1, 1} or {0, 1},
-        it will raise a `ValueError`.
-    """
-    # ensure binary classification if pos_label is not specified
-    # classes.dtype.kind in ('O', 'U', 'S') is required to avoid
-    # triggering a FutureWarning by calling np.array_equal(a, b)
-    # when elements in the two arrays are not comparable.
-    classes = np.unique(y_true)
-    if (pos_label is None and (
-            classes.dtype.kind in 'OUS' or
-            not (np.array_equal(classes, [0, 1]) or
-                 np.array_equal(classes, [-1, 1]) or
-                 np.array_equal(classes, [0]) or
-                 np.array_equal(classes, [-1]) or
-                 np.array_equal(classes, [1])))):
-        classes_repr = ", ".join(repr(c) for c in classes)
-        raise ValueError(
-            f"y_true takes value in {{{classes_repr}}} and pos_label is not "
-            f"specified: either make y_true take value in {{0, 1}} or "
-            f"{{-1, 1}} or pass pos_label explicitly."
-        )
-    elif pos_label is None:
-        pos_label = 1.0
-
-    return pos_label
diff --git a/sklearn/metrics/_classification.py b/sklearn/metrics/_classification.py
index a68e17656a73b..2e31320ddb1f4 100644
--- a/sklearn/metrics/_classification.py
+++ b/sklearn/metrics/_classification.py
@@ -7,50 +7,63 @@
 the lower the better.
 """
 
-# Authors: Alexandre Gramfort 
-#          Mathieu Blondel 
-#          Olivier Grisel 
-#          Arnaud Joly 
-#          Jochen Wersdorfer 
-#          Lars Buitinck
-#          Joel Nothman 
-#          Noel Dawe 
-#          Jatin Shah 
-#          Saurabh Jha 
-#          Bernardo Stein 
-#          Shangwu Yao 
-# License: BSD 3 clause
-
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import warnings
+from numbers import Integral, Real
+
 import numpy as np
+from scipy.sparse import coo_matrix, csr_matrix, issparse
+from scipy.special import xlogy
 
-from scipy.sparse import coo_matrix
-from scipy.sparse import csr_matrix
-
-from ..preprocessing import LabelBinarizer
-from ..preprocessing import LabelEncoder
-from ..utils import assert_all_finite
-from ..utils import check_array
-from ..utils import check_consistent_length
-from ..utils import column_or_1d
-from ..utils.multiclass import unique_labels
-from ..utils.multiclass import type_of_target
-from ..utils.validation import _num_samples
-from ..utils.validation import _deprecate_positional_args
-from ..utils.sparsefuncs import count_nonzero
 from ..exceptions import UndefinedMetricWarning
-
-from ._base import _check_pos_label_consistency
+from ..preprocessing import LabelBinarizer, LabelEncoder
+from ..utils import (
+    assert_all_finite,
+    check_array,
+    check_consistent_length,
+    check_scalar,
+    column_or_1d,
+)
+from ..utils._array_api import (
+    _average,
+    _bincount,
+    _count_nonzero,
+    _find_matching_floating_dtype,
+    _is_numpy_namespace,
+    _max_precision_float_dtype,
+    _searchsorted,
+    _tolist,
+    _union1d,
+    get_namespace,
+    get_namespace_and_device,
+    xpx,
+)
+from ..utils._param_validation import (
+    Hidden,
+    Interval,
+    Options,
+    StrOptions,
+    validate_params,
+)
+from ..utils._unique import attach_unique
+from ..utils.extmath import _nanaverage
+from ..utils.multiclass import type_of_target, unique_labels
+from ..utils.validation import (
+    _check_pos_label_consistency,
+    _check_sample_weight,
+    _num_samples,
+)
 
 
 def _check_zero_division(zero_division):
     if isinstance(zero_division, str) and zero_division == "warn":
-        return
+        return np.float64(0.0)
     elif isinstance(zero_division, (int, float)) and zero_division in [0, 1]:
-        return
-    raise ValueError('Got zero_division={0}.'
-                     ' Must be one of ["warn", 0, 1]'.format(zero_division))
+        return np.float64(zero_division)
+    else:  # np.isnan(zero_division)
+        return np.nan
 
 
 def _check_targets(y_true, y_pred):
@@ -80,64 +93,206 @@ def _check_targets(y_true, y_pred):
 
     y_pred : array or indicator matrix
     """
+    xp, _ = get_namespace(y_true, y_pred)
     check_consistent_length(y_true, y_pred)
-    type_true = type_of_target(y_true)
-    type_pred = type_of_target(y_pred)
+    type_true = type_of_target(y_true, input_name="y_true")
+    type_pred = type_of_target(y_pred, input_name="y_pred")
 
     y_type = {type_true, type_pred}
     if y_type == {"binary", "multiclass"}:
         y_type = {"multiclass"}
 
     if len(y_type) > 1:
-        raise ValueError("Classification metrics can't handle a mix of {0} "
-                         "and {1} targets".format(type_true, type_pred))
+        raise ValueError(
+            "Classification metrics can't handle a mix of {0} and {1} targets".format(
+                type_true, type_pred
+            )
+        )
 
     # We can't have more than one value on y_type => The set is no more needed
     y_type = y_type.pop()
 
     # No metrics support "multiclass-multioutput" format
-    if (y_type not in ["binary", "multiclass", "multilabel-indicator"]):
+    if y_type not in ["binary", "multiclass", "multilabel-indicator"]:
         raise ValueError("{0} is not supported".format(y_type))
 
     if y_type in ["binary", "multiclass"]:
+        xp, _ = get_namespace(y_true, y_pred)
         y_true = column_or_1d(y_true)
         y_pred = column_or_1d(y_pred)
         if y_type == "binary":
             try:
-                unique_values = np.union1d(y_true, y_pred)
+                unique_values = _union1d(y_true, y_pred, xp)
             except TypeError as e:
                 # We expect y_true and y_pred to be of the same data type.
                 # If `y_true` was provided to the classifier as strings,
                 # `y_pred` given by the classifier will also be encoded with
                 # strings. So we raise a meaningful error
                 raise TypeError(
-                    f"Labels in y_true and y_pred should be of the same type. "
-                    f"Got y_true={np.unique(y_true)} and "
-                    f"y_pred={np.unique(y_pred)}. Make sure that the "
-                    f"predictions provided by the classifier coincides with "
-                    f"the true labels."
+                    "Labels in y_true and y_pred should be of the same type. "
+                    f"Got y_true={xp.unique(y_true)} and "
+                    f"y_pred={xp.unique(y_pred)}. Make sure that the "
+                    "predictions provided by the classifier coincides with "
+                    "the true labels."
                 ) from e
-            if len(unique_values) > 2:
+            if unique_values.shape[0] > 2:
                 y_type = "multiclass"
 
-    if y_type.startswith('multilabel'):
-        y_true = csr_matrix(y_true)
-        y_pred = csr_matrix(y_pred)
-        y_type = 'multilabel-indicator'
+    if y_type.startswith("multilabel"):
+        if _is_numpy_namespace(xp):
+            # XXX: do we really want to sparse-encode multilabel indicators when
+            # they are passed as a dense arrays? This is not possible for array
+            # API inputs in general hence we only do it for NumPy inputs. But even
+            # for NumPy the usefulness is questionable.
+            y_true = csr_matrix(y_true)
+            y_pred = csr_matrix(y_pred)
+        y_type = "multilabel-indicator"
 
     return y_type, y_true, y_pred
 
 
-def _weighted_sum(sample_score, sample_weight, normalize=False):
-    if normalize:
-        return np.average(sample_score, weights=sample_weight)
-    elif sample_weight is not None:
-        return np.dot(sample_score, sample_weight)
+def _validate_multiclass_probabilistic_prediction(
+    y_true, y_prob, sample_weight, labels
+):
+    r"""Convert y_true and y_prob to shape (n_samples, n_classes)
+
+    1. Verify that y_true, y_prob, and sample_weights have the same first dim
+    2. Ensure 2 or more classes in y_true i.e. valid classification task. The
+       classes are provided by the labels argument, or inferred using y_true.
+       When inferring y_true is assumed binary if it has shape (n_samples, ).
+    3. Validate y_true, and y_prob have the same number of classes. Convert to
+       shape (n_samples, n_classes)
+
+    Parameters
+    ----------
+    y_true : array-like or label indicator matrix
+        Ground truth (correct) labels for n_samples samples.
+
+    y_prob : array-like of float, shape=(n_samples, n_classes) or (n_samples,)
+        Predicted probabilities, as returned by a classifier's
+        predict_proba method. If `y_prob.shape = (n_samples,)`
+        the probabilities provided are assumed to be that of the
+        positive class. The labels in `y_prob` are assumed to be
+        ordered lexicographically, as done by
+        :class:`preprocessing.LabelBinarizer`.
+
+    sample_weight : array-like of shape (n_samples,), default=None
+        Sample weights.
+
+    labels : array-like, default=None
+        If not provided, labels will be inferred from y_true. If `labels`
+        is `None` and `y_prob` has shape `(n_samples,)` the labels are
+        assumed to be binary and are inferred from `y_true`.
+
+    Returns
+    -------
+    transformed_labels : array of shape (n_samples, n_classes)
+
+    y_prob : array of shape (n_samples, n_classes)
+    """
+    y_prob = check_array(
+        y_prob, ensure_2d=False, dtype=[np.float64, np.float32, np.float16]
+    )
+
+    if y_prob.max() > 1:
+        raise ValueError(f"y_prob contains values greater than 1: {y_prob.max()}")
+    if y_prob.min() < 0:
+        raise ValueError(f"y_prob contains values lower than 0: {y_prob.min()}")
+
+    check_consistent_length(y_prob, y_true, sample_weight)
+    lb = LabelBinarizer()
+
+    if labels is not None:
+        lb = lb.fit(labels)
+        # LabelBinarizer does not respect the order implied by labels, which
+        # can be misleading.
+        if not np.all(lb.classes_ == labels):
+            warnings.warn(
+                f"Labels passed were {labels}. But this function "
+                "assumes labels are ordered lexicographically. "
+                f"Pass the ordered labels={lb.classes_.tolist()} and ensure that "
+                "the columns of y_prob correspond to this ordering.",
+                UserWarning,
+            )
+        if not np.isin(y_true, labels).all():
+            undeclared_labels = set(y_true) - set(labels)
+            raise ValueError(
+                f"y_true contains values {undeclared_labels} not belonging "
+                f"to the passed labels {labels}."
+            )
+
     else:
-        return sample_score.sum()
+        lb = lb.fit(y_true)
+
+    if len(lb.classes_) == 1:
+        if labels is None:
+            raise ValueError(
+                "y_true contains only one label ({0}). Please "
+                "provide the list of all expected class labels explicitly through the "
+                "labels argument.".format(lb.classes_[0])
+            )
+        else:
+            raise ValueError(
+                "The labels array needs to contain at least two "
+                "labels, got {0}.".format(lb.classes_)
+            )
 
+    transformed_labels = lb.transform(y_true)
+
+    if transformed_labels.shape[1] == 1:
+        transformed_labels = np.append(
+            1 - transformed_labels, transformed_labels, axis=1
+        )
+
+    # If y_prob is of single dimension, assume y_true to be binary
+    # and then check.
+    if y_prob.ndim == 1:
+        y_prob = y_prob[:, np.newaxis]
+    if y_prob.shape[1] == 1:
+        y_prob = np.append(1 - y_prob, y_prob, axis=1)
+
+    eps = np.finfo(y_prob.dtype).eps
+
+    # Make sure y_prob is normalized
+    y_prob_sum = y_prob.sum(axis=1)
+    if not np.allclose(y_prob_sum, 1, rtol=np.sqrt(eps)):
+        warnings.warn(
+            "The y_prob values do not sum to one. Make sure to pass probabilities.",
+            UserWarning,
+        )
 
-@_deprecate_positional_args
+    # Check if dimensions are consistent.
+    transformed_labels = check_array(transformed_labels)
+    if len(lb.classes_) != y_prob.shape[1]:
+        if labels is None:
+            raise ValueError(
+                "y_true and y_prob contain different number of "
+                "classes: {0} vs {1}. Please provide the true "
+                "labels explicitly through the labels argument. "
+                "Classes found in "
+                "y_true: {2}".format(
+                    transformed_labels.shape[1], y_prob.shape[1], lb.classes_
+                )
+            )
+        else:
+            raise ValueError(
+                "The number of classes in labels is different "
+                "from that in y_prob. Classes found in "
+                "labels: {0}".format(lb.classes_)
+            )
+
+    return transformed_labels, y_prob
+
+
+@validate_params(
+    {
+        "y_true": ["array-like", "sparse matrix"],
+        "y_pred": ["array-like", "sparse matrix"],
+        "normalize": ["boolean"],
+        "sample_weight": ["array-like", None],
+    },
+    prefer_skip_nested_validation=True,
+)
 def accuracy_score(y_true, y_pred, *, normalize=True, sample_weight=None):
     """Accuracy classification score.
 
@@ -164,7 +319,7 @@ def accuracy_score(y_true, y_pred, *, normalize=True, sample_weight=None):
 
     Returns
     -------
-    score : float
+    score : float or int
         If ``normalize == True``, return the fraction of correctly
         classified samples (float), else returns the number of correctly
         classified samples (int).
@@ -174,12 +329,13 @@ def accuracy_score(y_true, y_pred, *, normalize=True, sample_weight=None):
 
     See Also
     --------
-    jaccard_score, hamming_loss, zero_one_loss
-
-    Notes
-    -----
-    In binary and multiclass classification, this function is equal
-    to the ``jaccard_score`` function.
+    balanced_accuracy_score : Compute the balanced accuracy to deal with
+        imbalanced datasets.
+    jaccard_score : Compute the Jaccard similarity coefficient score.
+    hamming_loss : Compute the average Hamming loss or Hamming distance between
+        two sets of samples.
+    zero_one_loss : Compute the Zero-one classification loss. By default, the
+        function will return the percentage of imperfectly predicted subsets.
 
     Examples
     --------
@@ -189,7 +345,7 @@ def accuracy_score(y_true, y_pred, *, normalize=True, sample_weight=None):
     >>> accuracy_score(y_true, y_pred)
     0.5
     >>> accuracy_score(y_true, y_pred, normalize=False)
-    2
+    2.0
 
     In the multilabel case with binary label indicators:
 
@@ -197,22 +353,34 @@ def accuracy_score(y_true, y_pred, *, normalize=True, sample_weight=None):
     >>> accuracy_score(np.array([[0, 1], [1, 1]]), np.ones((2, 2)))
     0.5
     """
-
+    xp, _, device = get_namespace_and_device(y_true, y_pred, sample_weight)
     # Compute accuracy for each possible representation
+    y_true, y_pred = attach_unique(y_true, y_pred)
     y_type, y_true, y_pred = _check_targets(y_true, y_pred)
     check_consistent_length(y_true, y_pred, sample_weight)
-    if y_type.startswith('multilabel'):
-        differing_labels = count_nonzero(y_true - y_pred, axis=1)
-        score = differing_labels == 0
+
+    if y_type.startswith("multilabel"):
+        differing_labels = _count_nonzero(y_true - y_pred, xp=xp, device=device, axis=1)
+        score = xp.asarray(differing_labels == 0, device=device)
     else:
         score = y_true == y_pred
 
-    return _weighted_sum(score, sample_weight, normalize)
-
-
-@_deprecate_positional_args
-def confusion_matrix(y_true, y_pred, *, labels=None, sample_weight=None,
-                     normalize=None):
+    return float(_average(score, weights=sample_weight, normalize=normalize))
+
+
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_pred": ["array-like"],
+        "labels": ["array-like", None],
+        "sample_weight": ["array-like", None],
+        "normalize": [StrOptions({"true", "pred", "all"}), None],
+    },
+    prefer_skip_nested_validation=True,
+)
+def confusion_matrix(
+    y_true, y_pred, *, labels=None, sample_weight=None, normalize=None
+):
     """Compute confusion matrix to evaluate the accuracy of a classification.
 
     By definition a confusion matrix :math:`C` is such that :math:`C_{i, j}`
@@ -289,13 +457,13 @@ def confusion_matrix(y_true, y_pred, *, labels=None, sample_weight=None,
            [0, 0, 1],
            [1, 0, 2]])
 
-    In the binary case, we can extract true positives, etc as follows:
+    In the binary case, we can extract true positives, etc. as follows:
 
-    >>> tn, fp, fn, tp = confusion_matrix([0, 1, 0, 1], [1, 1, 1, 0]).ravel()
+    >>> tn, fp, fn, tp = confusion_matrix([0, 1, 0, 1], [1, 1, 1, 0]).ravel().tolist()
     >>> (tn, fp, fn, tp)
     (0, 2, 1, 1)
-
     """
+    y_true, y_pred = attach_unique(y_true, y_pred)
     y_type, y_true, y_pred = _check_targets(y_true, y_pred)
     if y_type not in ("binary", "multiclass"):
         raise ValueError("%s is not supported" % y_type)
@@ -319,17 +487,14 @@ def confusion_matrix(y_true, y_pred, *, labels=None, sample_weight=None,
 
     check_consistent_length(y_true, y_pred, sample_weight)
 
-    if normalize not in ['true', 'pred', 'all', None]:
-        raise ValueError("normalize must be one of {'true', 'pred', "
-                         "'all', None}")
-
     n_labels = labels.size
     # If labels are not consecutive integers starting from zero, then
     # y_true and y_pred must be converted into index form
     need_index_conversion = not (
-        labels.dtype.kind in {'i', 'u', 'b'} and
-        np.all(labels == np.arange(n_labels)) and
-        y_true.min() >= 0 and y_pred.min() >= 0
+        labels.dtype.kind in {"i", "u", "b"}
+        and np.all(labels == np.arange(n_labels))
+        and y_true.min() >= 0
+        and y_pred.min() >= 0
     )
     if need_index_conversion:
         label_to_ind = {y: x for x, y in enumerate(labels)}
@@ -345,30 +510,52 @@ def confusion_matrix(y_true, y_pred, *, labels=None, sample_weight=None,
         sample_weight = sample_weight[ind]
 
     # Choose the accumulator dtype to always have high precision
-    if sample_weight.dtype.kind in {'i', 'u', 'b'}:
+    if sample_weight.dtype.kind in {"i", "u", "b"}:
         dtype = np.int64
     else:
         dtype = np.float64
 
-    cm = coo_matrix((sample_weight, (y_true, y_pred)),
-                    shape=(n_labels, n_labels), dtype=dtype,
-                    ).toarray()
+    cm = coo_matrix(
+        (sample_weight, (y_true, y_pred)),
+        shape=(n_labels, n_labels),
+        dtype=dtype,
+    ).toarray()
 
-    with np.errstate(all='ignore'):
-        if normalize == 'true':
+    with np.errstate(all="ignore"):
+        if normalize == "true":
             cm = cm / cm.sum(axis=1, keepdims=True)
-        elif normalize == 'pred':
+        elif normalize == "pred":
             cm = cm / cm.sum(axis=0, keepdims=True)
-        elif normalize == 'all':
+        elif normalize == "all":
             cm = cm / cm.sum()
         cm = np.nan_to_num(cm)
 
+    if cm.shape == (1, 1):
+        warnings.warn(
+            (
+                "A single label was found in 'y_true' and 'y_pred'. For the confusion "
+                "matrix to have the correct shape, use the 'labels' parameter to pass "
+                "all known labels."
+            ),
+            UserWarning,
+        )
+
     return cm
 
 
-@_deprecate_positional_args
-def multilabel_confusion_matrix(y_true, y_pred, *, sample_weight=None,
-                                labels=None, samplewise=False):
+@validate_params(
+    {
+        "y_true": ["array-like", "sparse matrix"],
+        "y_pred": ["array-like", "sparse matrix"],
+        "sample_weight": ["array-like", None],
+        "labels": ["array-like", None],
+        "samplewise": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def multilabel_confusion_matrix(
+    y_true, y_pred, *, sample_weight=None, labels=None, samplewise=False
+):
     """Compute a confusion matrix for each class or sample.
 
     .. versionadded:: 0.21
@@ -420,14 +607,16 @@ def multilabel_confusion_matrix(y_true, y_pred, *, sample_weight=None,
 
     See Also
     --------
-    confusion_matrix
+    confusion_matrix : Compute confusion matrix to evaluate the accuracy of a
+        classifier.
 
     Notes
     -----
-    The multilabel_confusion_matrix calculates class-wise or sample-wise
+    The `multilabel_confusion_matrix` calculates class-wise or sample-wise
     multilabel confusion matrices, and in multiclass tasks, labels are
-    binarized under a one-vs-rest way; while confusion_matrix calculates
-    one confusion matrix for confusion between every two classes.
+    binarized under a one-vs-rest way; while
+    :func:`~sklearn.metrics.confusion_matrix` calculates one confusion matrix
+    for confusion between every two classes.
 
     Examples
     --------
@@ -464,9 +653,11 @@ def multilabel_confusion_matrix(y_true, y_pred, *, sample_weight=None,
            [[2, 1],
             [1, 2]]])
     """
+    y_true, y_pred = attach_unique(y_true, y_pred)
+    xp, _, device_ = get_namespace_and_device(y_true, y_pred)
     y_type, y_true, y_pred = _check_targets(y_true, y_pred)
     if sample_weight is not None:
-        sample_weight = column_or_1d(sample_weight)
+        sample_weight = column_or_1d(sample_weight, device=device_)
     check_consistent_length(y_true, y_pred, sample_weight)
 
     if y_type not in ("binary", "multiclass", "multilabel-indicator"):
@@ -477,14 +668,19 @@ def multilabel_confusion_matrix(y_true, y_pred, *, sample_weight=None,
         labels = present_labels
         n_labels = None
     else:
-        n_labels = len(labels)
-        labels = np.hstack([labels, np.setdiff1d(present_labels, labels,
-                                                 assume_unique=True)])
+        labels = xp.asarray(labels, device=device_)
+        n_labels = labels.shape[0]
+        labels = xp.concat(
+            [labels, xpx.setdiff1d(present_labels, labels, assume_unique=True, xp=xp)],
+            axis=-1,
+        )
 
     if y_true.ndim == 1:
         if samplewise:
-            raise ValueError("Samplewise metrics are not available outside of "
-                             "multilabel classification.")
+            raise ValueError(
+                "Samplewise metrics are not available outside of "
+                "multilabel classification."
+            )
 
         le = LabelEncoder()
         le.fit(labels)
@@ -496,82 +692,116 @@ def multilabel_confusion_matrix(y_true, y_pred, *, sample_weight=None,
         tp = y_true == y_pred
         tp_bins = y_true[tp]
         if sample_weight is not None:
-            tp_bins_weights = np.asarray(sample_weight)[tp]
+            tp_bins_weights = sample_weight[tp]
         else:
             tp_bins_weights = None
 
-        if len(tp_bins):
-            tp_sum = np.bincount(tp_bins, weights=tp_bins_weights,
-                                 minlength=len(labels))
+        if tp_bins.shape[0]:
+            tp_sum = _bincount(
+                tp_bins, weights=tp_bins_weights, minlength=labels.shape[0], xp=xp
+            )
         else:
             # Pathological case
-            true_sum = pred_sum = tp_sum = np.zeros(len(labels))
-        if len(y_pred):
-            pred_sum = np.bincount(y_pred, weights=sample_weight,
-                                   minlength=len(labels))
-        if len(y_true):
-            true_sum = np.bincount(y_true, weights=sample_weight,
-                                   minlength=len(labels))
+            true_sum = pred_sum = tp_sum = xp.zeros(labels.shape[0])
+        if y_pred.shape[0]:
+            pred_sum = _bincount(
+                y_pred, weights=sample_weight, minlength=labels.shape[0], xp=xp
+            )
+        if y_true.shape[0]:
+            true_sum = _bincount(
+                y_true, weights=sample_weight, minlength=labels.shape[0], xp=xp
+            )
 
         # Retain only selected labels
-        indices = np.searchsorted(sorted_labels, labels[:n_labels])
-        tp_sum = tp_sum[indices]
-        true_sum = true_sum[indices]
-        pred_sum = pred_sum[indices]
+        indices = _searchsorted(sorted_labels, labels[:n_labels], xp=xp)
+        tp_sum = xp.take(tp_sum, indices, axis=0)
+        true_sum = xp.take(true_sum, indices, axis=0)
+        pred_sum = xp.take(pred_sum, indices, axis=0)
 
     else:
         sum_axis = 1 if samplewise else 0
 
         # All labels are index integers for multilabel.
         # Select labels:
-        if not np.array_equal(labels, present_labels):
-            if np.max(labels) > np.max(present_labels):
-                raise ValueError('All labels must be in [0, n labels) for '
-                                 'multilabel targets. '
-                                 'Got %d > %d' %
-                                 (np.max(labels), np.max(present_labels)))
-            if np.min(labels) < 0:
-                raise ValueError('All labels must be in [0, n labels) for '
-                                 'multilabel targets. '
-                                 'Got %d < 0' % np.min(labels))
+        if labels.shape != present_labels.shape or xp.any(
+            xp.not_equal(labels, present_labels)
+        ):
+            if xp.max(labels) > xp.max(present_labels):
+                raise ValueError(
+                    "All labels must be in [0, n labels) for "
+                    "multilabel targets. "
+                    "Got %d > %d" % (xp.max(labels), xp.max(present_labels))
+                )
+            if xp.min(labels) < 0:
+                raise ValueError(
+                    "All labels must be in [0, n labels) for "
+                    "multilabel targets. "
+                    "Got %d < 0" % xp.min(labels)
+                )
 
         if n_labels is not None:
             y_true = y_true[:, labels[:n_labels]]
             y_pred = y_pred[:, labels[:n_labels]]
 
+        if issparse(y_true) or issparse(y_pred):
+            true_and_pred = y_true.multiply(y_pred)
+        else:
+            true_and_pred = xp.multiply(y_true, y_pred)
+
         # calculate weighted counts
-        true_and_pred = y_true.multiply(y_pred)
-        tp_sum = count_nonzero(true_and_pred, axis=sum_axis,
-                               sample_weight=sample_weight)
-        pred_sum = count_nonzero(y_pred, axis=sum_axis,
-                                 sample_weight=sample_weight)
-        true_sum = count_nonzero(y_true, axis=sum_axis,
-                                 sample_weight=sample_weight)
+        tp_sum = _count_nonzero(
+            true_and_pred,
+            axis=sum_axis,
+            sample_weight=sample_weight,
+            xp=xp,
+            device=device_,
+        )
+        pred_sum = _count_nonzero(
+            y_pred,
+            axis=sum_axis,
+            sample_weight=sample_weight,
+            xp=xp,
+            device=device_,
+        )
+        true_sum = _count_nonzero(
+            y_true,
+            axis=sum_axis,
+            sample_weight=sample_weight,
+            xp=xp,
+            device=device_,
+        )
 
     fp = pred_sum - tp_sum
     fn = true_sum - tp_sum
     tp = tp_sum
 
     if sample_weight is not None and samplewise:
-        sample_weight = np.array(sample_weight)
-        tp = np.array(tp)
-        fp = np.array(fp)
-        fn = np.array(fn)
+        tp = xp.asarray(tp)
+        fp = xp.asarray(fp)
+        fn = xp.asarray(fn)
         tn = sample_weight * y_true.shape[1] - tp - fp - fn
     elif sample_weight is not None:
-        tn = sum(sample_weight) - tp - fp - fn
+        tn = xp.sum(sample_weight) - tp - fp - fn
     elif samplewise:
         tn = y_true.shape[1] - tp - fp - fn
     else:
         tn = y_true.shape[0] - tp - fp - fn
 
-    return np.array([tn, fp, fn, tp]).T.reshape(-1, 2, 2)
+    return xp.reshape(xp.stack([tn, fp, fn, tp]).T, (-1, 2, 2))
 
 
-@_deprecate_positional_args
-def cohen_kappa_score(y1, y2, *, labels=None, weights=None,
-                      sample_weight=None):
-    r"""Cohen's kappa: a statistic that measures inter-annotator agreement.
+@validate_params(
+    {
+        "y1": ["array-like"],
+        "y2": ["array-like"],
+        "labels": ["array-like", None],
+        "weights": [StrOptions({"linear", "quadratic"}), None],
+        "sample_weight": ["array-like", None],
+    },
+    prefer_skip_nested_validation=True,
+)
+def cohen_kappa_score(y1, y2, *, labels=None, weights=None, sample_weight=None):
+    r"""Compute Cohen's kappa: a statistic that measures inter-annotator agreement.
 
     This function computes Cohen's kappa [1]_, a score that expresses the level
     of agreement between two annotators on a classification problem. It is
@@ -590,21 +820,23 @@ class labels [2]_.
 
     Parameters
     ----------
-    y1 : array of shape (n_samples,)
+    y1 : array-like of shape (n_samples,)
         Labels assigned by the first annotator.
 
-    y2 : array of shape (n_samples,)
+    y2 : array-like of shape (n_samples,)
         Labels assigned by the second annotator. The kappa statistic is
         symmetric, so swapping ``y1`` and ``y2`` doesn't change the value.
 
     labels : array-like of shape (n_classes,), default=None
         List of labels to index the matrix. This may be used to select a
-        subset of labels. If None, all labels that appear at least once in
-        ``y1`` or ``y2`` are used.
+        subset of labels. If `None`, all labels that appear at least once in
+        ``y1`` or ``y2`` are used. Note that at least one label in `labels` must be
+         present in `y1`, even though this function is otherwise agnostic to the order
+         of `y1` and `y2`.
 
     weights : {'linear', 'quadratic'}, default=None
-        Weighting type to calculate the score. None means no weighted;
-        "linear" means linear weighted; "quadratic" means quadratic weighted.
+        Weighting type to calculate the score. `None` means not weighted;
+        "linear" means linear weighting; "quadratic" means quadratic weighting.
 
     sample_weight : array-like of shape (n_samples,), default=None
         Sample weights.
@@ -617,17 +849,35 @@ class labels [2]_.
 
     References
     ----------
-    .. [1] J. Cohen (1960). "A coefficient of agreement for nominal scales".
+    .. [1] :doi:`J. Cohen (1960). "A coefficient of agreement for nominal scales".
            Educational and Psychological Measurement 20(1):37-46.
-           doi:10.1177/001316446002000104.
+           <10.1177/001316446002000104>`
     .. [2] `R. Artstein and M. Poesio (2008). "Inter-coder agreement for
            computational linguistics". Computational Linguistics 34(4):555-596
            `_.
     .. [3] `Wikipedia entry for the Cohen's kappa
             `_.
+
+    Examples
+    --------
+    >>> from sklearn.metrics import cohen_kappa_score
+    >>> y1 = ["negative", "positive", "negative", "neutral", "positive"]
+    >>> y2 = ["negative", "positive", "negative", "neutral", "negative"]
+    >>> cohen_kappa_score(y1, y2)
+    0.6875
     """
-    confusion = confusion_matrix(y1, y2, labels=labels,
-                                 sample_weight=sample_weight)
+    try:
+        confusion = confusion_matrix(y1, y2, labels=labels, sample_weight=sample_weight)
+    except ValueError as e:
+        if "At least one label specified must be in y_true" in str(e):
+            msg = (
+                "At least one label in `labels` must be present in `y1` (even though "
+                "`cohen_kappa_score` is otherwise agnostic to the order of `y1` and "
+                "`y2`)."
+            )
+            raise ValueError(msg) from e
+        raise
+
     n_classes = confusion.shape[0]
     sum0 = np.sum(confusion, axis=0)
     sum1 = np.sum(confusion, axis=1)
@@ -636,23 +886,46 @@ class labels [2]_.
     if weights is None:
         w_mat = np.ones([n_classes, n_classes], dtype=int)
         w_mat.flat[:: n_classes + 1] = 0
-    elif weights == "linear" or weights == "quadratic":
+    else:  # "linear" or "quadratic"
         w_mat = np.zeros([n_classes, n_classes], dtype=int)
         w_mat += np.arange(n_classes)
         if weights == "linear":
             w_mat = np.abs(w_mat - w_mat.T)
         else:
             w_mat = (w_mat - w_mat.T) ** 2
-    else:
-        raise ValueError("Unknown kappa weighting type.")
 
     k = np.sum(w_mat * confusion) / np.sum(w_mat * expected)
-    return 1 - k
-
-
-@_deprecate_positional_args
-def jaccard_score(y_true, y_pred, *, labels=None, pos_label=1,
-                  average='binary', sample_weight=None, zero_division="warn"):
+    return float(1 - k)
+
+
+@validate_params(
+    {
+        "y_true": ["array-like", "sparse matrix"],
+        "y_pred": ["array-like", "sparse matrix"],
+        "labels": ["array-like", None],
+        "pos_label": [Real, str, "boolean", None],
+        "average": [
+            StrOptions({"micro", "macro", "samples", "weighted", "binary"}),
+            None,
+        ],
+        "sample_weight": ["array-like", None],
+        "zero_division": [
+            Options(Real, {0, 1}),
+            StrOptions({"warn"}),
+        ],
+    },
+    prefer_skip_nested_validation=True,
+)
+def jaccard_score(
+    y_true,
+    y_pred,
+    *,
+    labels=None,
+    pos_label=1,
+    average="binary",
+    sample_weight=None,
+    zero_division="warn",
+):
     """Jaccard similarity coefficient score.
 
     The Jaccard index [1], or Jaccard similarity coefficient, defined as
@@ -660,6 +933,16 @@ def jaccard_score(y_true, y_pred, *, labels=None, pos_label=1,
     sets, is used to compare set of predicted labels for a sample to the
     corresponding set of labels in ``y_true``.
 
+    Support beyond term:`binary` targets is achieved by treating :term:`multiclass`
+    and :term:`multilabel` data as a collection of binary problems, one for each
+    label. For the :term:`binary` case, setting `average='binary'` will return the
+    Jaccard similarity coefficient for `pos_label`. If `average` is not `'binary'`,
+    `pos_label` is ignored and scores for both classes are computed, then averaged or
+    both returned (when `average=None`). Similarly, for :term:`multiclass` and
+    :term:`multilabel` targets, scores for all `labels` are either returned or
+    averaged depending on the `average` parameter. Use `labels` specify the set of
+    labels to calculate the score for.
+
     Read more in the :ref:`User Guide `.
 
     Parameters
@@ -671,22 +954,21 @@ def jaccard_score(y_true, y_pred, *, labels=None, pos_label=1,
         Predicted labels, as returned by a classifier.
 
     labels : array-like of shape (n_classes,), default=None
-        The set of labels to include when ``average != 'binary'``, and their
-        order if ``average is None``. Labels present in the data can be
-        excluded, for example to calculate a multiclass average ignoring a
-        majority negative class, while labels not present in the data will
-        result in 0 components in a macro average. For multilabel targets,
-        labels are column indices. By default, all labels in ``y_true`` and
-        ``y_pred`` are used in sorted order.
-
-    pos_label : str or int, default=1
-        The class to report if ``average='binary'`` and the data is binary.
-        If the data are multiclass or multilabel, this will be ignored;
-        setting ``labels=[pos_label]`` and ``average != 'binary'`` will report
-        scores for that label only.
-
-    average : {None, 'micro', 'macro', 'samples', 'weighted', \
-            'binary'}, default='binary'
+        The set of labels to include when `average != 'binary'`, and their
+        order if `average is None`. Labels present in the data can be
+        excluded, for example in multiclass classification to exclude a "negative
+        class". Labels not present in the data can be included and will be
+        "assigned" 0 samples. For multilabel targets, labels are column indices.
+        By default, all labels in `y_true` and `y_pred` are used in sorted order.
+
+    pos_label : int, float, bool or str, default=1
+        The class to report if `average='binary'` and the data is binary,
+        otherwise this parameter is ignored.
+        For multiclass or multilabel targets, set `labels=[pos_label]` and
+        `average != 'binary'` to report metrics for one label only.
+
+    average : {'micro', 'macro', 'samples', 'weighted', \
+            'binary'} or None, default='binary'
         If ``None``, the scores for each class are returned. Otherwise, this
         determines the type of averaging performed on the data:
 
@@ -715,14 +997,20 @@ def jaccard_score(y_true, y_pred, *, labels=None, pos_label=1,
         there are no negative values in predictions and labels. If set to
         "warn", this acts like 0, but a warning is also raised.
 
+        .. versionadded:: 0.24
+
     Returns
     -------
-    score : float (if average is not None) or array of floats, shape =\
-            [n_unique_labels]
+    score : float or ndarray of shape (n_unique_labels,), dtype=np.float64
+        The Jaccard score. When `average` is not `None`, a single scalar is
+        returned.
 
     See Also
     --------
-    accuracy_score, f_score, multilabel_confusion_matrix
+    accuracy_score : Function for calculating the accuracy score.
+    f1_score : Function for calculating the F1 score.
+    multilabel_confusion_matrix : Function for computing a confusion matrix\
+                                  for each class or sample.
 
     Notes
     -----
@@ -748,14 +1036,19 @@ def jaccard_score(y_true, y_pred, *, labels=None, pos_label=1,
     In the binary case:
 
     >>> jaccard_score(y_true[0], y_pred[0])
-    0.6666...
+    0.6666
+
+    In the 2D comparison case (e.g. image similarity):
+
+    >>> jaccard_score(y_true, y_pred, average="micro")
+    0.6
 
     In the multilabel case:
 
     >>> jaccard_score(y_true, y_pred, average='samples')
-    0.5833...
+    0.5833
     >>> jaccard_score(y_true, y_pred, average='macro')
-    0.6666...
+    0.6666
     >>> jaccard_score(y_true, y_pred, average=None)
     array([0.5, 0.5, 1. ])
 
@@ -764,39 +1057,56 @@ def jaccard_score(y_true, y_pred, *, labels=None, pos_label=1,
     >>> y_pred = [0, 2, 1, 2]
     >>> y_true = [0, 1, 2, 2]
     >>> jaccard_score(y_true, y_pred, average=None)
-    array([1. , 0. , 0.33...])
+    array([1. , 0. , 0.33])
     """
-    labels = _check_set_wise_labels(y_true, y_pred, average, labels,
-                                    pos_label)
-    samplewise = average == 'samples'
-    MCM = multilabel_confusion_matrix(y_true, y_pred,
-                                      sample_weight=sample_weight,
-                                      labels=labels, samplewise=samplewise)
+    labels = _check_set_wise_labels(y_true, y_pred, average, labels, pos_label)
+    samplewise = average == "samples"
+    MCM = multilabel_confusion_matrix(
+        y_true,
+        y_pred,
+        sample_weight=sample_weight,
+        labels=labels,
+        samplewise=samplewise,
+    )
     numerator = MCM[:, 1, 1]
     denominator = MCM[:, 1, 1] + MCM[:, 0, 1] + MCM[:, 1, 0]
 
-    if average == 'micro':
-        numerator = np.array([numerator.sum()])
-        denominator = np.array([denominator.sum()])
-
-    jaccard = _prf_divide(numerator, denominator, 'jaccard',
-                          'true or predicted', average, ('jaccard',),
-                          zero_division=zero_division)
+    xp, _, device_ = get_namespace_and_device(y_true, y_pred)
+    if average == "micro":
+        numerator = xp.asarray(xp.sum(numerator, keepdims=True), device=device_)
+        denominator = xp.asarray(xp.sum(denominator, keepdims=True), device=device_)
+
+    jaccard = _prf_divide(
+        numerator,
+        denominator,
+        "jaccard",
+        "true or predicted",
+        average,
+        ("jaccard",),
+        zero_division=zero_division,
+    )
     if average is None:
         return jaccard
-    if average == 'weighted':
+    if average == "weighted":
         weights = MCM[:, 1, 0] + MCM[:, 1, 1]
-        if not np.any(weights):
+        if not xp.any(weights):
             # numerator is 0, and warning should have already been issued
             weights = None
-    elif average == 'samples' and sample_weight is not None:
+    elif average == "samples" and sample_weight is not None:
         weights = sample_weight
     else:
         weights = None
-    return np.average(jaccard, weights=weights)
+    return float(_average(jaccard, weights=weights, xp=xp))
 
 
-@_deprecate_positional_args
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_pred": ["array-like"],
+        "sample_weight": ["array-like", None],
+    },
+    prefer_skip_nested_validation=True,
+)
 def matthews_corrcoef(y_true, y_pred, *, sample_weight=None):
     """Compute the Matthews correlation coefficient (MCC).
 
@@ -817,10 +1127,10 @@ def matthews_corrcoef(y_true, y_pred, *, sample_weight=None):
 
     Parameters
     ----------
-    y_true : array, shape = [n_samples]
+    y_true : array-like of shape (n_samples,)
         Ground truth (correct) target values.
 
-    y_pred : array, shape = [n_samples]
+    y_pred : array-like of shape (n_samples,)
         Estimated targets as returned by a classifier.
 
     sample_weight : array-like of shape (n_samples,), default=None
@@ -837,12 +1147,12 @@ def matthews_corrcoef(y_true, y_pred, *, sample_weight=None):
 
     References
     ----------
-    .. [1] `Baldi, Brunak, Chauvin, Andersen and Nielsen, (2000). Assessing the
-       accuracy of prediction algorithms for classification: an overview
-       `_.
+    .. [1] :doi:`Baldi, Brunak, Chauvin, Andersen and Nielsen, (2000). Assessing the
+       accuracy of prediction algorithms for classification: an overview.
+       <10.1093/bioinformatics/16.5.412>`
 
-    .. [2] `Wikipedia entry for the Matthews Correlation Coefficient
-       `_.
+    .. [2] `Wikipedia entry for the Matthews Correlation Coefficient (phi coefficient)
+       `_.
 
     .. [3] `Gorodkin, (2004). Comparing two K-category assignments by a
         K-category correlation coefficient
@@ -858,8 +1168,9 @@ def matthews_corrcoef(y_true, y_pred, *, sample_weight=None):
     >>> y_true = [+1, +1, +1, -1]
     >>> y_pred = [+1, -1, +1, +1]
     >>> matthews_corrcoef(y_true, y_pred)
-    -0.33...
+    -0.33
     """
+    y_true, y_pred = attach_unique(y_true, y_pred)
     y_type, y_true, y_pred = _check_targets(y_true, y_pred)
     check_consistent_length(y_true, y_pred, sample_weight)
     if y_type not in {"binary", "multiclass"}:
@@ -876,17 +1187,25 @@ def matthews_corrcoef(y_true, y_pred, *, sample_weight=None):
     n_correct = np.trace(C, dtype=np.float64)
     n_samples = p_sum.sum()
     cov_ytyp = n_correct * n_samples - np.dot(t_sum, p_sum)
-    cov_ypyp = n_samples ** 2 - np.dot(p_sum, p_sum)
-    cov_ytyt = n_samples ** 2 - np.dot(t_sum, t_sum)
-    mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)
+    cov_ypyp = n_samples**2 - np.dot(p_sum, p_sum)
+    cov_ytyt = n_samples**2 - np.dot(t_sum, t_sum)
 
-    if np.isnan(mcc):
-        return 0.
+    cov_ypyp_ytyt = cov_ypyp * cov_ytyt
+    if cov_ypyp_ytyt == 0:
+        return 0.0
     else:
-        return mcc
-
-
-@_deprecate_positional_args
+        return float(cov_ytyp / np.sqrt(cov_ypyp_ytyt))
+
+
+@validate_params(
+    {
+        "y_true": ["array-like", "sparse matrix"],
+        "y_pred": ["array-like", "sparse matrix"],
+        "normalize": ["boolean"],
+        "sample_weight": ["array-like", None],
+    },
+    prefer_skip_nested_validation=True,
+)
 def zero_one_loss(y_true, y_pred, *, normalize=True, sample_weight=None):
     """Zero-one classification loss.
 
@@ -917,16 +1236,21 @@ def zero_one_loss(y_true, y_pred, *, normalize=True, sample_weight=None):
         If ``normalize == True``, return the fraction of misclassifications
         (float), else it returns the number of misclassifications (int).
 
+    See Also
+    --------
+    accuracy_score : Compute the accuracy score. By default, the function will
+        return the fraction of correct predictions divided by the total number
+        of predictions.
+    hamming_loss : Compute the average Hamming loss or Hamming distance between
+        two sets of samples.
+    jaccard_score : Compute the Jaccard similarity coefficient score.
+
     Notes
     -----
     In multilabel classification, the zero_one_loss function corresponds to
     the subset zero-one loss: for each sample, the entire set of labels must be
     correctly predicted, otherwise the loss for that sample is equal to one.
 
-    See Also
-    --------
-    accuracy_score, hamming_loss, jaccard_score
-
     Examples
     --------
     >>> from sklearn.metrics import zero_one_loss
@@ -935,7 +1259,7 @@ def zero_one_loss(y_true, y_pred, *, normalize=True, sample_weight=None):
     >>> zero_one_loss(y_true, y_pred)
     0.25
     >>> zero_one_loss(y_true, y_pred, normalize=False)
-    1
+    1.0
 
     In the multilabel case with binary label indicators:
 
@@ -943,35 +1267,75 @@ def zero_one_loss(y_true, y_pred, *, normalize=True, sample_weight=None):
     >>> zero_one_loss(np.array([[0, 1], [1, 1]]), np.ones((2, 2)))
     0.5
     """
-    score = accuracy_score(y_true, y_pred,
-                           normalize=normalize,
-                           sample_weight=sample_weight)
+    xp, _ = get_namespace(y_true, y_pred)
+    score = accuracy_score(
+        y_true, y_pred, normalize=normalize, sample_weight=sample_weight
+    )
 
     if normalize:
         return 1 - score
     else:
         if sample_weight is not None:
-            n_samples = np.sum(sample_weight)
+            n_samples = xp.sum(sample_weight)
         else:
             n_samples = _num_samples(y_true)
         return n_samples - score
 
 
-@_deprecate_positional_args
-def f1_score(y_true, y_pred, *, labels=None, pos_label=1, average='binary',
-             sample_weight=None, zero_division="warn"):
+@validate_params(
+    {
+        "y_true": ["array-like", "sparse matrix"],
+        "y_pred": ["array-like", "sparse matrix"],
+        "labels": ["array-like", None],
+        "pos_label": [Real, str, "boolean", None],
+        "average": [
+            StrOptions({"micro", "macro", "samples", "weighted", "binary"}),
+            None,
+        ],
+        "sample_weight": ["array-like", None],
+        "zero_division": [
+            Options(Real, {0.0, 1.0}),
+            "nan",
+            StrOptions({"warn"}),
+        ],
+    },
+    prefer_skip_nested_validation=True,
+)
+def f1_score(
+    y_true,
+    y_pred,
+    *,
+    labels=None,
+    pos_label=1,
+    average="binary",
+    sample_weight=None,
+    zero_division="warn",
+):
     """Compute the F1 score, also known as balanced F-score or F-measure.
 
-    The F1 score can be interpreted as a weighted average of the precision and
+    The F1 score can be interpreted as a harmonic mean of the precision and
     recall, where an F1 score reaches its best value at 1 and worst score at 0.
     The relative contribution of precision and recall to the F1 score are
-    equal. The formula for the F1 score is::
-
-        F1 = 2 * (precision * recall) / (precision + recall)
+    equal. The formula for the F1 score is:
 
-    In the multi-class and multi-label case, this is the average of
-    the F1 score of each class with weighting depending on the ``average``
-    parameter.
+    .. math::
+        \\text{F1} = \\frac{2 * \\text{TP}}{2 * \\text{TP} + \\text{FP} + \\text{FN}}
+
+    Where :math:`\\text{TP}` is the number of true positives, :math:`\\text{FN}` is the
+    number of false negatives, and :math:`\\text{FP}` is the number of false positives.
+    F1 is by default
+    calculated as 0.0 when there are no true positives, false negatives, or
+    false positives.
+
+    Support beyond :term:`binary` targets is achieved by treating :term:`multiclass`
+    and :term:`multilabel` data as a collection of binary problems, one for each
+    label. For the :term:`binary` case, setting `average='binary'` will return
+    F1 score for `pos_label`. If `average` is not `'binary'`, `pos_label` is ignored
+    and F1 score for both classes are computed, then averaged or both returned (when
+    `average=None`). Similarly, for :term:`multiclass` and :term:`multilabel` targets,
+    F1 score for all `labels` are either returned or averaged depending on the
+    `average` parameter. Use `labels` specify the set of labels to calculate F1 score
+    for.
 
     Read more in the :ref:`User Guide `.
 
@@ -984,27 +1348,26 @@ def f1_score(y_true, y_pred, *, labels=None, pos_label=1, average='binary',
         Estimated targets as returned by a classifier.
 
     labels : array-like, default=None
-        The set of labels to include when ``average != 'binary'``, and their
-        order if ``average is None``. Labels present in the data can be
-        excluded, for example to calculate a multiclass average ignoring a
-        majority negative class, while labels not present in the data will
-        result in 0 components in a macro average. For multilabel targets,
-        labels are column indices. By default, all labels in ``y_true`` and
-        ``y_pred`` are used in sorted order.
+        The set of labels to include when `average != 'binary'`, and their
+        order if `average is None`. Labels present in the data can be
+        excluded, for example in multiclass classification to exclude a "negative
+        class". Labels not present in the data can be included and will be
+        "assigned" 0 samples. For multilabel targets, labels are column indices.
+        By default, all labels in `y_true` and `y_pred` are used in sorted order.
 
         .. versionchanged:: 0.17
            Parameter `labels` improved for multiclass problem.
 
-    pos_label : str or int, default=1
-        The class to report if ``average='binary'`` and the data is binary.
-        If the data are multiclass or multilabel, this will be ignored;
-        setting ``labels=[pos_label]`` and ``average != 'binary'`` will report
-        scores for that label only.
+    pos_label : int, float, bool or str, default=1
+        The class to report if `average='binary'` and the data is binary,
+        otherwise this parameter is ignored.
+        For multiclass or multilabel targets, set `labels=[pos_label]` and
+        `average != 'binary'` to report metrics for one label only.
 
-    average : {'micro', 'macro', 'samples','weighted', 'binary'} or None, \
+    average : {'micro', 'macro', 'samples', 'weighted', 'binary'} or None, \
             default='binary'
         This parameter is required for multiclass/multilabel targets.
-        If ``None``, the scores for each class are returned. Otherwise, this
+        If ``None``, the metrics for each class are returned. Otherwise, this
         determines the type of averaging performed on the data:
 
         ``'binary'``:
@@ -1029,10 +1392,16 @@ def f1_score(y_true, y_pred, *, labels=None, pos_label=1, average='binary',
     sample_weight : array-like of shape (n_samples,), default=None
         Sample weights.
 
-    zero_division : "warn", 0 or 1, default="warn"
+    zero_division : {"warn", 0.0, 1.0, np.nan}, default="warn"
         Sets the value to return when there is a zero division, i.e. when all
-        predictions and labels are negative. If set to "warn", this acts as 0,
-        but warnings are also raised.
+        predictions and labels are negative.
+
+        Notes:
+        - If set to "warn", this acts like 0, but a warning is also raised.
+        - If set to `np.nan`, such values will be excluded from the average.
+
+        .. versionadded:: 1.3
+           `np.nan` option was added.
 
     Returns
     -------
@@ -1042,8 +1411,20 @@ def f1_score(y_true, y_pred, *, labels=None, pos_label=1, average='binary',
 
     See Also
     --------
-    fbeta_score, precision_recall_fscore_support, jaccard_score,
-    multilabel_confusion_matrix
+    fbeta_score : Compute the F-beta score.
+    precision_recall_fscore_support : Compute the precision, recall, F-score,
+        and support.
+    jaccard_score : Compute the Jaccard similarity coefficient score.
+    multilabel_confusion_matrix : Compute a confusion matrix for each class or
+        sample.
+
+    Notes
+    -----
+    When ``true positive + false positive + false negative == 0`` (i.e. a class
+    is completely absent from both ``y_true`` or ``y_pred``), f-score is
+    undefined. In such cases, by default f-score will be set to 0.0, and
+    ``UndefinedMetricWarning`` will be raised. This behavior can be modified by
+    setting the ``zero_division`` parameter.
 
     References
     ----------
@@ -1052,48 +1433,109 @@ def f1_score(y_true, y_pred, *, labels=None, pos_label=1, average='binary',
 
     Examples
     --------
+    >>> import numpy as np
     >>> from sklearn.metrics import f1_score
     >>> y_true = [0, 1, 2, 0, 1, 2]
     >>> y_pred = [0, 2, 1, 0, 0, 1]
     >>> f1_score(y_true, y_pred, average='macro')
-    0.26...
+    0.267
     >>> f1_score(y_true, y_pred, average='micro')
-    0.33...
+    0.33
     >>> f1_score(y_true, y_pred, average='weighted')
-    0.26...
+    0.267
     >>> f1_score(y_true, y_pred, average=None)
     array([0.8, 0. , 0. ])
-    >>> y_true = [0, 0, 0, 0, 0, 0]
-    >>> y_pred = [0, 0, 0, 0, 0, 0]
-    >>> f1_score(y_true, y_pred, zero_division=1)
+
+    >>> # binary classification
+    >>> y_true_empty = [0, 0, 0, 0, 0, 0]
+    >>> y_pred_empty = [0, 0, 0, 0, 0, 0]
+    >>> f1_score(y_true_empty, y_pred_empty)
+    0.0...
+    >>> f1_score(y_true_empty, y_pred_empty, zero_division=1.0)
     1.0...
+    >>> f1_score(y_true_empty, y_pred_empty, zero_division=np.nan)
+    nan...
 
-    Notes
-    -----
-    When ``true positive + false positive == 0``, precision is undefined.
-    When ``true positive + false negative == 0``, recall is undefined.
-    In such cases, by default the metric will be set to 0, as will f-score,
-    and ``UndefinedMetricWarning`` will be raised. This behavior can be
-    modified with ``zero_division``.
+    >>> # multilabel classification
+    >>> y_true = [[0, 0, 0], [1, 1, 1], [0, 1, 1]]
+    >>> y_pred = [[0, 0, 0], [1, 1, 1], [1, 1, 0]]
+    >>> f1_score(y_true, y_pred, average=None)
+    array([0.66666667, 1.        , 0.66666667])
     """
-    return fbeta_score(y_true, y_pred, beta=1, labels=labels,
-                       pos_label=pos_label, average=average,
-                       sample_weight=sample_weight,
-                       zero_division=zero_division)
+    return fbeta_score(
+        y_true,
+        y_pred,
+        beta=1,
+        labels=labels,
+        pos_label=pos_label,
+        average=average,
+        sample_weight=sample_weight,
+        zero_division=zero_division,
+    )
 
 
-@_deprecate_positional_args
-def fbeta_score(y_true, y_pred, *, beta, labels=None, pos_label=1,
-                average='binary', sample_weight=None, zero_division="warn"):
+@validate_params(
+    {
+        "y_true": ["array-like", "sparse matrix"],
+        "y_pred": ["array-like", "sparse matrix"],
+        "beta": [Interval(Real, 0.0, None, closed="both")],
+        "labels": ["array-like", None],
+        "pos_label": [Real, str, "boolean", None],
+        "average": [
+            StrOptions({"micro", "macro", "samples", "weighted", "binary"}),
+            None,
+        ],
+        "sample_weight": ["array-like", None],
+        "zero_division": [
+            Options(Real, {0.0, 1.0}),
+            "nan",
+            StrOptions({"warn"}),
+        ],
+    },
+    prefer_skip_nested_validation=True,
+)
+def fbeta_score(
+    y_true,
+    y_pred,
+    *,
+    beta,
+    labels=None,
+    pos_label=1,
+    average="binary",
+    sample_weight=None,
+    zero_division="warn",
+):
     """Compute the F-beta score.
 
     The F-beta score is the weighted harmonic mean of precision and recall,
     reaching its optimal value at 1 and its worst value at 0.
 
-    The `beta` parameter determines the weight of recall in the combined
-    score. ``beta < 1`` lends more weight to precision, while ``beta > 1``
-    favors recall (``beta -> 0`` considers only precision, ``beta -> +inf``
-    only recall).
+    The `beta` parameter represents the ratio of recall importance to
+    precision importance. `beta > 1` gives more weight to recall, while
+    `beta < 1` favors precision. For example, `beta = 2` makes recall twice
+    as important as precision, while `beta = 0.5` does the opposite.
+    Asymptotically, `beta -> +inf` considers only recall, and `beta -> 0`
+    only precision.
+
+    The formula for F-beta score is:
+
+    .. math::
+
+       F_\\beta = \\frac{(1 + \\beta^2) \\text{tp}}
+                        {(1 + \\beta^2) \\text{tp} + \\text{fp} + \\beta^2 \\text{fn}}
+
+    Where :math:`\\text{tp}` is the number of true positives, :math:`\\text{fp}` is the
+    number of false positives, and :math:`\\text{fn}` is the number of false negatives.
+
+    Support beyond term:`binary` targets is achieved by treating :term:`multiclass`
+    and :term:`multilabel` data as a collection of binary problems, one for each
+    label. For the :term:`binary` case, setting `average='binary'` will return
+    F-beta score for `pos_label`. If `average` is not `'binary'`, `pos_label` is
+    ignored and F-beta score for both classes are computed, then averaged or both
+    returned (when `average=None`). Similarly, for :term:`multiclass` and
+    :term:`multilabel` targets, F-beta score for all `labels` are either returned or
+    averaged depending on the `average` parameter. Use `labels` specify the set of
+    labels to calculate F-beta score for.
 
     Read more in the :ref:`User Guide `.
 
@@ -1109,27 +1551,26 @@ def fbeta_score(y_true, y_pred, *, beta, labels=None, pos_label=1,
         Determines the weight of recall in the combined score.
 
     labels : array-like, default=None
-        The set of labels to include when ``average != 'binary'``, and their
-        order if ``average is None``. Labels present in the data can be
-        excluded, for example to calculate a multiclass average ignoring a
-        majority negative class, while labels not present in the data will
-        result in 0 components in a macro average. For multilabel targets,
-        labels are column indices. By default, all labels in ``y_true`` and
-        ``y_pred`` are used in sorted order.
+        The set of labels to include when `average != 'binary'`, and their
+        order if `average is None`. Labels present in the data can be
+        excluded, for example in multiclass classification to exclude a "negative
+        class". Labels not present in the data can be included and will be
+        "assigned" 0 samples. For multilabel targets, labels are column indices.
+        By default, all labels in `y_true` and `y_pred` are used in sorted order.
 
         .. versionchanged:: 0.17
            Parameter `labels` improved for multiclass problem.
 
-    pos_label : str or int, default=1
-        The class to report if ``average='binary'`` and the data is binary.
-        If the data are multiclass or multilabel, this will be ignored;
-        setting ``labels=[pos_label]`` and ``average != 'binary'`` will report
-        scores for that label only.
+    pos_label : int, float, bool or str, default=1
+        The class to report if `average='binary'` and the data is binary,
+        otherwise this parameter is ignored.
+        For multiclass or multilabel targets, set `labels=[pos_label]` and
+        `average != 'binary'` to report metrics for one label only.
 
-    average : {'micro', 'macro', 'samples', 'weighted', 'binary'} or None \
+    average : {'micro', 'macro', 'samples', 'weighted', 'binary'} or None, \
             default='binary'
         This parameter is required for multiclass/multilabel targets.
-        If ``None``, the scores for each class are returned. Otherwise, this
+        If ``None``, the metrics for each class are returned. Otherwise, this
         determines the type of averaging performed on the data:
 
         ``'binary'``:
@@ -1154,10 +1595,17 @@ def fbeta_score(y_true, y_pred, *, beta, labels=None, pos_label=1,
     sample_weight : array-like of shape (n_samples,), default=None
         Sample weights.
 
-    zero_division : "warn", 0 or 1, default="warn"
+    zero_division : {"warn", 0.0, 1.0, np.nan}, default="warn"
         Sets the value to return when there is a zero division, i.e. when all
-        predictions and labels are negative. If set to "warn", this acts as 0,
-        but warnings are also raised.
+        predictions and labels are negative.
+
+        Notes:
+
+        - If set to "warn", this acts like 0, but a warning is also raised.
+        - If set to `np.nan`, such values will be excluded from the average.
+
+        .. versionadded:: 1.3
+           `np.nan` option was added.
 
     Returns
     -------
@@ -1168,14 +1616,16 @@ def fbeta_score(y_true, y_pred, *, beta, labels=None, pos_label=1,
 
     See Also
     --------
-    precision_recall_fscore_support, multilabel_confusion_matrix
+    precision_recall_fscore_support : Compute the precision, recall, F-score,
+        and support.
+    multilabel_confusion_matrix : Compute a confusion matrix for each class or
+        sample.
 
     Notes
     -----
-    When ``true positive + false positive == 0`` or
-    ``true positive + false negative == 0``, f-score returns 0 and raises
-    ``UndefinedMetricWarning``. This behavior can be
-    modified with ``zero_division``.
+    When ``true positive + false positive + false negative == 0``, f-score
+    returns 0.0 and raises ``UndefinedMetricWarning``. This behavior can be
+    modified by setting ``zero_division``.
 
     References
     ----------
@@ -1187,88 +1637,90 @@ def fbeta_score(y_true, y_pred, *, beta, labels=None, pos_label=1,
 
     Examples
     --------
+    >>> import numpy as np
     >>> from sklearn.metrics import fbeta_score
     >>> y_true = [0, 1, 2, 0, 1, 2]
     >>> y_pred = [0, 2, 1, 0, 0, 1]
     >>> fbeta_score(y_true, y_pred, average='macro', beta=0.5)
-    0.23...
+    0.238
     >>> fbeta_score(y_true, y_pred, average='micro', beta=0.5)
-    0.33...
+    0.33
     >>> fbeta_score(y_true, y_pred, average='weighted', beta=0.5)
-    0.23...
+    0.238
     >>> fbeta_score(y_true, y_pred, average=None, beta=0.5)
-    array([0.71..., 0.        , 0.        ])
+    array([0.71, 0.        , 0.        ])
+    >>> y_pred_empty = [0, 0, 0, 0, 0, 0]
+    >>> fbeta_score(y_true, y_pred_empty,
+    ...             average="macro", zero_division=np.nan, beta=0.5)
+    0.128
     """
 
-    _, _, f, _ = precision_recall_fscore_support(y_true, y_pred,
-                                                 beta=beta,
-                                                 labels=labels,
-                                                 pos_label=pos_label,
-                                                 average=average,
-                                                 warn_for=('f-score',),
-                                                 sample_weight=sample_weight,
-                                                 zero_division=zero_division)
+    _, _, f, _ = precision_recall_fscore_support(
+        y_true,
+        y_pred,
+        beta=beta,
+        labels=labels,
+        pos_label=pos_label,
+        average=average,
+        warn_for=("f-score",),
+        sample_weight=sample_weight,
+        zero_division=zero_division,
+    )
     return f
 
 
-def _prf_divide(numerator, denominator, metric,
-                modifier, average, warn_for, zero_division="warn"):
+def _prf_divide(
+    numerator, denominator, metric, modifier, average, warn_for, zero_division="warn"
+):
     """Performs division and handles divide-by-zero.
 
     On zero-division, sets the corresponding result elements equal to
-    0 or 1 (according to ``zero_division``). Plus, if
+    0, 1 or np.nan (according to ``zero_division``). Plus, if
     ``zero_division != "warn"`` raises a warning.
 
     The metric, modifier and average arguments are used only for determining
     an appropriate warning.
     """
-    mask = denominator == 0.0
-    denominator = denominator.copy()
+    xp, _ = get_namespace(numerator, denominator)
+    dtype_float = _find_matching_floating_dtype(numerator, denominator, xp=xp)
+    mask = denominator == 0
+    denominator = xp.asarray(denominator, copy=True, dtype=dtype_float)
     denominator[mask] = 1  # avoid infs/nans
-    result = numerator / denominator
+    result = xp.asarray(numerator, dtype=dtype_float) / denominator
 
-    if not np.any(mask):
+    if not xp.any(mask):
         return result
 
-    # if ``zero_division=1``, set those with denominator == 0 equal to 1
-    result[mask] = 0.0 if zero_division in ["warn", 0] else 1.0
+    # set those with 0 denominator to `zero_division`, and 0 when "warn"
+    zero_division_value = _check_zero_division(zero_division)
+    result[mask] = zero_division_value
 
-    # the user will be removing warnings if zero_division is set to something
-    # different than its default value. If we are computing only f-score
+    # we assume the user will be removing warnings if zero_division is set
+    # to something different than "warn". If we are computing only f-score
     # the warning will be raised only if precision and recall are ill-defined
     if zero_division != "warn" or metric not in warn_for:
         return result
 
     # build appropriate warning
-    # E.g. "Precision and F-score are ill-defined and being set to 0.0 in
-    # labels with no predicted samples. Use ``zero_division`` parameter to
-    # control this behavior."
-
-    if metric in warn_for and 'f-score' in warn_for:
-        msg_start = '{0} and F-score are'.format(metric.title())
-    elif metric in warn_for:
-        msg_start = '{0} is'.format(metric.title())
-    elif 'f-score' in warn_for:
-        msg_start = 'F-score is'
-    else:
-        return result
-
-    _warn_prf(average, modifier, msg_start, len(result))
+    if metric in warn_for:
+        _warn_prf(average, modifier, f"{metric.capitalize()} is", result.shape[0])
 
     return result
 
 
 def _warn_prf(average, modifier, msg_start, result_size):
-    axis0, axis1 = 'sample', 'label'
-    if average == 'samples':
+    axis0, axis1 = "sample", "label"
+    if average == "samples":
         axis0, axis1 = axis1, axis0
-    msg = ('{0} ill-defined and being set to 0.0 {{0}} '
-           'no {1} {2}s. Use `zero_division` parameter to control'
-           ' this behavior.'.format(msg_start, modifier, axis0))
+    msg = (
+        "{0} ill-defined and being set to 0.0 {{0}} "
+        "no {1} {2}s. Use `zero_division` parameter to control"
+        " this behavior.".format(msg_start, modifier, axis0)
+    )
     if result_size == 1:
-        msg = msg.format('due to')
+        msg = msg.format("due to")
     else:
-        msg = msg.format('in {0}s with'.format(axis1))
+        msg = msg.format("in {0}s with".format(axis1))
     warnings.warn(msg, UndefinedMetricWarning, stacklevel=2)
 
 
@@ -1277,17 +1729,17 @@ def _check_set_wise_labels(y_true, y_pred, average, labels, pos_label):
 
     Returns identified labels.
     """
-    average_options = (None, 'micro', 'macro', 'weighted', 'samples')
-    if average not in average_options and average != 'binary':
-        raise ValueError('average has to be one of ' +
-                         str(average_options))
+    average_options = (None, "micro", "macro", "weighted", "samples")
+    if average not in average_options and average != "binary":
+        raise ValueError("average has to be one of " + str(average_options))
 
+    y_true, y_pred = attach_unique(y_true, y_pred)
     y_type, y_true, y_pred = _check_targets(y_true, y_pred)
     # Convert to Python primitive type to avoid NumPy type / Python str
     # comparison. See https://github.com/numpy/numpy/issues/6784
-    present_labels = unique_labels(y_true, y_pred).tolist()
-    if average == 'binary':
-        if y_type == 'binary':
+    present_labels = _tolist(unique_labels(y_true, y_pred))
+    if average == "binary":
+        if y_type == "binary":
             if pos_label not in present_labels:
                 if len(present_labels) >= 2:
                     raise ValueError(
@@ -1297,32 +1749,62 @@ def _check_set_wise_labels(y_true, y_pred, average, labels, pos_label):
             labels = [pos_label]
         else:
             average_options = list(average_options)
-            if y_type == 'multiclass':
-                average_options.remove('samples')
-            raise ValueError("Target is %s but average='binary'. Please "
-                             "choose another average setting, one of %r."
-                             % (y_type, average_options))
+            if y_type == "multiclass":
+                average_options.remove("samples")
+            raise ValueError(
+                "Target is %s but average='binary'. Please "
+                "choose another average setting, one of %r." % (y_type, average_options)
+            )
     elif pos_label not in (None, 1):
-        warnings.warn("Note that pos_label (set to %r) is ignored when "
-                      "average != 'binary' (got %r). You may use "
-                      "labels=[pos_label] to specify a single positive class."
-                      % (pos_label, average), UserWarning)
+        warnings.warn(
+            "Note that pos_label (set to %r) is ignored when "
+            "average != 'binary' (got %r). You may use "
+            "labels=[pos_label] to specify a single positive class."
+            % (pos_label, average),
+            UserWarning,
+        )
     return labels
 
 
-@_deprecate_positional_args
-def precision_recall_fscore_support(y_true, y_pred, *, beta=1.0, labels=None,
-                                    pos_label=1, average=None,
-                                    warn_for=('precision', 'recall',
-                                              'f-score'),
-                                    sample_weight=None,
-                                    zero_division="warn"):
+@validate_params(
+    {
+        "y_true": ["array-like", "sparse matrix"],
+        "y_pred": ["array-like", "sparse matrix"],
+        "beta": [Interval(Real, 0.0, None, closed="both")],
+        "labels": ["array-like", None],
+        "pos_label": [Real, str, "boolean", None],
+        "average": [
+            StrOptions({"micro", "macro", "samples", "weighted", "binary"}),
+            None,
+        ],
+        "warn_for": [list, tuple, set],
+        "sample_weight": ["array-like", None],
+        "zero_division": [
+            Options(Real, {0.0, 1.0}),
+            "nan",
+            StrOptions({"warn"}),
+        ],
+    },
+    prefer_skip_nested_validation=True,
+)
+def precision_recall_fscore_support(
+    y_true,
+    y_pred,
+    *,
+    beta=1.0,
+    labels=None,
+    pos_label=1,
+    average=None,
+    warn_for=("precision", "recall", "f-score"),
+    sample_weight=None,
+    zero_division="warn",
+):
     """Compute precision, recall, F-measure and support for each class.
 
     The precision is the ratio ``tp / (tp + fp)`` where ``tp`` is the number of
     true positives and ``fp`` the number of false positives. The precision is
-    intuitively the ability of the classifier not to label as positive a sample
-    that is negative.
+    intuitively the ability of the classifier not to label a negative sample as
+    positive.
 
     The recall is the ratio ``tp / (tp + fn)`` where ``tp`` is the number of
     true positives and ``fn`` the number of false negatives. The recall is
@@ -1337,9 +1819,14 @@ def precision_recall_fscore_support(y_true, y_pred, *, beta=1.0, labels=None,
 
     The support is the number of occurrences of each class in ``y_true``.
 
-    If ``pos_label is None`` and in binary classification, this function
-    returns the average precision, recall and F-measure if ``average``
-    is one of ``'micro'``, ``'macro'``, ``'weighted'`` or ``'samples'``.
+    Support beyond term:`binary` targets is achieved by treating :term:`multiclass`
+    and :term:`multilabel` data as a collection of binary problems, one for each
+    label. For the :term:`binary` case, setting `average='binary'` will return
+    metrics for `pos_label`. If `average` is not `'binary'`, `pos_label` is ignored
+    and metrics for both classes are computed, then averaged or both returned (when
+    `average=None`). Similarly, for :term:`multiclass` and :term:`multilabel` targets,
+    metrics for all `labels` are either returned or averaged depending on the `average`
+    parameter. Use `labels` specify the set of labels to calculate metrics for.
 
     Read more in the :ref:`User Guide `.
 
@@ -1355,23 +1842,26 @@ def precision_recall_fscore_support(y_true, y_pred, *, beta=1.0, labels=None,
         The strength of recall versus precision in the F-score.
 
     labels : array-like, default=None
-        The set of labels to include when ``average != 'binary'``, and their
-        order if ``average is None``. Labels present in the data can be
-        excluded, for example to calculate a multiclass average ignoring a
-        majority negative class, while labels not present in the data will
-        result in 0 components in a macro average. For multilabel targets,
-        labels are column indices. By default, all labels in ``y_true`` and
-        ``y_pred`` are used in sorted order.
-
-    pos_label : str or int, default=1
-        The class to report if ``average='binary'`` and the data is binary.
-        If the data are multiclass or multilabel, this will be ignored;
-        setting ``labels=[pos_label]`` and ``average != 'binary'`` will report
-        scores for that label only.
-
-    average : {'binary', 'micro', 'macro', 'samples','weighted'}, \
-            default=None
-        If ``None``, the scores for each class are returned. Otherwise, this
+        The set of labels to include when `average != 'binary'`, and their
+        order if `average is None`. Labels present in the data can be
+        excluded, for example in multiclass classification to exclude a "negative
+        class". Labels not present in the data can be included and will be
+        "assigned" 0 samples. For multilabel targets, labels are column indices.
+        By default, all labels in `y_true` and `y_pred` are used in sorted order.
+
+        .. versionchanged:: 0.17
+           Parameter `labels` improved for multiclass problem.
+
+    pos_label : int, float, bool or str, default=1
+        The class to report if `average='binary'` and the data is binary,
+        otherwise this parameter is ignored.
+        For multiclass or multilabel targets, set `labels=[pos_label]` and
+        `average != 'binary'` to report metrics for one label only.
+
+    average : {'micro', 'macro', 'samples', 'weighted', 'binary'} or None, \
+            default='binary'
+        This parameter is required for multiclass/multilabel targets.
+        If ``None``, the metrics for each class are returned. Otherwise, this
         determines the type of averaging performed on the data:
 
         ``'binary'``:
@@ -1393,31 +1883,41 @@ def precision_recall_fscore_support(y_true, y_pred, *, beta=1.0, labels=None,
             meaningful for multilabel classification where this differs from
             :func:`accuracy_score`).
 
-    warn_for : tuple or set, for internal use
+    warn_for : list, tuple or set, for internal use
         This determines which warnings will be made in the case that this
         function is being used to return only one of its metrics.
 
     sample_weight : array-like of shape (n_samples,), default=None
         Sample weights.
 
-    zero_division : "warn", 0 or 1, default="warn"
+    zero_division : {"warn", 0.0, 1.0, np.nan}, default="warn"
         Sets the value to return when there is a zero division:
-           - recall: when there are no positive labels
-           - precision: when there are no positive predictions
-           - f-score: both
 
-        If set to "warn", this acts as 0, but warnings are also raised.
+        - recall: when there are no positive labels
+        - precision: when there are no positive predictions
+        - f-score: both
+
+        Notes:
+
+        - If set to "warn", this acts like 0, but a warning is also raised.
+        - If set to `np.nan`, such values will be excluded from the average.
+
+        .. versionadded:: 1.3
+           `np.nan` option was added.
 
     Returns
     -------
     precision : float (if average is not None) or array of float, shape =\
         [n_unique_labels]
+        Precision score.
 
-    recall : float (if average is not None) or array of float, , shape =\
+    recall : float (if average is not None) or array of float, shape =\
         [n_unique_labels]
+        Recall score.
 
     fbeta_score : float (if average is not None) or array of float, shape =\
         [n_unique_labels]
+        F-beta score.
 
     support : None (if average is not None) or array of int, shape =\
         [n_unique_labels]
@@ -1426,10 +1926,11 @@ def precision_recall_fscore_support(y_true, y_pred, *, beta=1.0, labels=None,
     Notes
     -----
     When ``true positive + false positive == 0``, precision is undefined.
-    When ``true positive + false negative == 0``, recall is undefined.
-    In such cases, by default the metric will be set to 0, as will f-score,
-    and ``UndefinedMetricWarning`` will be raised. This behavior can be
-    modified with ``zero_division``.
+    When ``true positive + false negative == 0``, recall is undefined. When
+    ``true positive + false negative + false positive == 0``, f-score is
+    undefined. In such cases, by default the metric will be set to 0, and
+    ``UndefinedMetricWarning`` will be raised. This behavior can be modified
+    with ``zero_division``.
 
     References
     ----------
@@ -1451,110 +1952,411 @@ def precision_recall_fscore_support(y_true, y_pred, *, beta=1.0, labels=None,
     >>> y_true = np.array(['cat', 'dog', 'pig', 'cat', 'dog', 'pig'])
     >>> y_pred = np.array(['cat', 'pig', 'dog', 'cat', 'cat', 'dog'])
     >>> precision_recall_fscore_support(y_true, y_pred, average='macro')
-    (0.22..., 0.33..., 0.26..., None)
+    (0.222, 0.333, 0.267, None)
     >>> precision_recall_fscore_support(y_true, y_pred, average='micro')
-    (0.33..., 0.33..., 0.33..., None)
+    (0.33, 0.33, 0.33, None)
     >>> precision_recall_fscore_support(y_true, y_pred, average='weighted')
-    (0.22..., 0.33..., 0.26..., None)
+    (0.222, 0.333, 0.267, None)
 
     It is possible to compute per-label precisions, recalls, F1-scores and
     supports instead of averaging:
 
     >>> precision_recall_fscore_support(y_true, y_pred, average=None,
     ... labels=['pig', 'dog', 'cat'])
-    (array([0.        , 0.        , 0.66...]),
+    (array([0.        , 0.        , 0.66]),
      array([0., 0., 1.]), array([0. , 0. , 0.8]),
      array([2, 2, 2]))
     """
     _check_zero_division(zero_division)
-    if beta < 0:
-        raise ValueError("beta should be >=0 in the F-beta score")
-    labels = _check_set_wise_labels(y_true, y_pred, average, labels,
-                                    pos_label)
+    labels = _check_set_wise_labels(y_true, y_pred, average, labels, pos_label)
 
     # Calculate tp_sum, pred_sum, true_sum ###
-    samplewise = average == 'samples'
-    MCM = multilabel_confusion_matrix(y_true, y_pred,
-                                      sample_weight=sample_weight,
-                                      labels=labels, samplewise=samplewise)
+    samplewise = average == "samples"
+    MCM = multilabel_confusion_matrix(
+        y_true,
+        y_pred,
+        sample_weight=sample_weight,
+        labels=labels,
+        samplewise=samplewise,
+    )
     tp_sum = MCM[:, 1, 1]
     pred_sum = tp_sum + MCM[:, 0, 1]
     true_sum = tp_sum + MCM[:, 1, 0]
 
-    if average == 'micro':
-        tp_sum = np.array([tp_sum.sum()])
-        pred_sum = np.array([pred_sum.sum()])
-        true_sum = np.array([true_sum.sum()])
+    xp, _, device_ = get_namespace_and_device(y_true, y_pred)
+    if average == "micro":
+        tp_sum = xp.reshape(xp.sum(tp_sum), (1,))
+        pred_sum = xp.reshape(xp.sum(pred_sum), (1,))
+        true_sum = xp.reshape(xp.sum(true_sum), (1,))
 
     # Finally, we have all our sufficient statistics. Divide! #
-    beta2 = beta ** 2
+    beta2 = beta**2
 
     # Divide, and on zero-division, set scores and/or warn according to
     # zero_division:
-    precision = _prf_divide(tp_sum, pred_sum, 'precision',
-                            'predicted', average, warn_for, zero_division)
-    recall = _prf_divide(tp_sum, true_sum, 'recall',
-                         'true', average, warn_for, zero_division)
-
-    # warn for f-score only if zero_division is warn, it is in warn_for
-    # and BOTH prec and rec are ill-defined
-    if zero_division == "warn" and ("f-score",) == warn_for:
-        if (pred_sum[true_sum == 0] == 0).any():
-            _warn_prf(
-                average, "true nor predicted", 'F-score is', len(true_sum)
-            )
+    precision = _prf_divide(
+        tp_sum, pred_sum, "precision", "predicted", average, warn_for, zero_division
+    )
+    recall = _prf_divide(
+        tp_sum, true_sum, "recall", "true", average, warn_for, zero_division
+    )
 
-    # if tp == 0 F will be 1 only if all predictions are zero, all labels are
-    # zero, and zero_division=1. In all other case, 0
     if np.isposinf(beta):
         f_score = recall
+    elif beta == 0:
+        f_score = precision
     else:
-        denom = beta2 * precision + recall
-
-        denom[denom == 0.] = 1  # avoid division by 0
-        f_score = (1 + beta2) * precision * recall / denom
+        # The score is defined as:
+        # score = (1 + beta**2) * precision * recall / (beta**2 * precision + recall)
+        # Therefore, we can express the score in terms of confusion matrix entries as:
+        # score = (1 + beta**2) * tp / ((1 + beta**2) * tp + beta**2 * fn + fp)
+
+        # Array api strict requires all arrays to be of the same type so we
+        # need to convert true_sum, pred_sum and tp_sum to the max supported
+        # float dtype because beta2 is a float
+        max_float_type = _max_precision_float_dtype(xp=xp, device=device_)
+        denom = beta2 * xp.astype(true_sum, max_float_type) + xp.astype(
+            pred_sum, max_float_type
+        )
+        f_score = _prf_divide(
+            (1 + beta2) * xp.astype(tp_sum, max_float_type),
+            denom,
+            "f-score",
+            "true nor predicted",
+            average,
+            warn_for,
+            zero_division,
+        )
 
     # Average the results
-    if average == 'weighted':
+    if average == "weighted":
         weights = true_sum
-        if weights.sum() == 0:
-            zero_division_value = np.float64(1.0)
-            if zero_division in ["warn", 0]:
-                zero_division_value = np.float64(0.0)
-            # precision is zero_division if there are no positive predictions
-            # recall is zero_division if there are no positive labels
-            # fscore is zero_division if all labels AND predictions are
-            # negative
-            if pred_sum.sum() == 0:
-                return (zero_division_value,
-                        zero_division_value,
-                        zero_division_value,
-                        None)
-            else:
-                return (np.float64(0.0),
-                        zero_division_value,
-                        np.float64(0.0),
-                        None)
-
-    elif average == 'samples':
+    elif average == "samples":
         weights = sample_weight
     else:
         weights = None
 
     if average is not None:
-        assert average != 'binary' or len(precision) == 1
-        precision = np.average(precision, weights=weights)
-        recall = np.average(recall, weights=weights)
-        f_score = np.average(f_score, weights=weights)
+        precision = float(_nanaverage(precision, weights=weights))
+        recall = float(_nanaverage(recall, weights=weights))
+        f_score = float(_nanaverage(f_score, weights=weights))
         true_sum = None  # return no support
 
     return precision, recall, f_score, true_sum
 
 
-@_deprecate_positional_args
-def precision_score(y_true, y_pred, *, labels=None, pos_label=1,
-                    average='binary', sample_weight=None,
-                    zero_division="warn"):
+@validate_params(
+    {
+        "y_true": ["array-like", "sparse matrix"],
+        "y_pred": ["array-like", "sparse matrix"],
+        "labels": ["array-like", None],
+        "sample_weight": ["array-like", None],
+        "raise_warning": ["boolean", Hidden(StrOptions({"deprecated"}))],
+        "replace_undefined_by": [
+            Options(Real, {1.0, np.nan}),
+            dict,
+        ],
+    },
+    prefer_skip_nested_validation=True,
+)
+def class_likelihood_ratios(
+    y_true,
+    y_pred,
+    *,
+    labels=None,
+    sample_weight=None,
+    raise_warning="deprecated",
+    replace_undefined_by=np.nan,
+):
+    """Compute binary classification positive and negative likelihood ratios.
+
+    The positive likelihood ratio is `LR+ = sensitivity / (1 - specificity)`
+    where the sensitivity or recall is the ratio `tp / (tp + fn)` and the
+    specificity is `tn / (tn + fp)`. The negative likelihood ratio is `LR- = (1
+    - sensitivity) / specificity`. Here `tp` is the number of true positives,
+    `fp` the number of false positives, `tn` is the number of true negatives and
+    `fn` the number of false negatives. Both class likelihood ratios can be used
+    to obtain post-test probabilities given a pre-test probability.
+
+    `LR+` ranges from 1.0 to infinity. A `LR+` of 1.0 indicates that the probability
+    of predicting the positive class is the same for samples belonging to either
+    class; therefore, the test is useless. The greater `LR+` is, the more a
+    positive prediction is likely to be a true positive when compared with the
+    pre-test probability. A value of `LR+` lower than 1.0 is invalid as it would
+    indicate that the odds of a sample being a true positive decrease with
+    respect to the pre-test odds.
+
+    `LR-` ranges from 0.0 to 1.0. The closer it is to 0.0, the lower the probability
+    of a given sample to be a false negative. A `LR-` of 1.0 means the test is
+    useless because the odds of having the condition did not change after the
+    test. A value of `LR-` greater than 1.0 invalidates the classifier as it
+    indicates an increase in the odds of a sample belonging to the positive
+    class after being classified as negative. This is the case when the
+    classifier systematically predicts the opposite of the true label.
+
+    A typical application in medicine is to identify the positive/negative class
+    to the presence/absence of a disease, respectively; the classifier being a
+    diagnostic test; the pre-test probability of an individual having the
+    disease can be the prevalence of such disease (proportion of a particular
+    population found to be affected by a medical condition); and the post-test
+    probabilities would be the probability that the condition is truly present
+    given a positive test result.
+
+    Read more in the :ref:`User Guide `.
+
+    Parameters
+    ----------
+    y_true : 1d array-like, or label indicator array / sparse matrix
+        Ground truth (correct) target values.
+
+    y_pred : 1d array-like, or label indicator array / sparse matrix
+        Estimated targets as returned by a classifier.
+
+    labels : array-like, default=None
+        List of labels to index the matrix. This may be used to select the
+        positive and negative classes with the ordering `labels=[negative_class,
+        positive_class]`. If `None` is given, those that appear at least once in
+        `y_true` or `y_pred` are used in sorted order.
+
+    sample_weight : array-like of shape (n_samples,), default=None
+        Sample weights.
+
+    raise_warning : bool, default=True
+        Whether or not a case-specific warning message is raised when there is division
+        by zero.
+
+        .. deprecated:: 1.7
+            `raise_warning` was deprecated in version 1.7 and will be removed in 1.9,
+            when an :class:`~sklearn.exceptions.UndefinedMetricWarning` will always
+            raise in case of a division by zero.
+
+    replace_undefined_by : np.nan, 1.0, or dict, default=np.nan
+        Sets the return values for LR+ and LR- when there is a division by zero. Can
+        take the following values:
+
+        - `np.nan` to return `np.nan` for both `LR+` and `LR-`
+        - `1.0` to return the worst possible scores: `{"LR+": 1.0, "LR-": 1.0}`
+        - a dict in the format `{"LR+": value_1, "LR-": value_2}` where the values can
+          be non-negative floats, `np.inf` or `np.nan` in the range of the
+          likelihood ratios. For example, `{"LR+": 1.0, "LR-": 1.0}` can be used for
+          returning the worst scores, indicating a useless model, and `{"LR+": np.inf,
+          "LR-": 0.0}` can be used for returning the best scores, indicating a useful
+          model.
+
+        If a division by zero occurs, only the affected metric is replaced with the set
+        value; the other metric is calculated as usual.
+
+        .. versionadded:: 1.7
+
+    Returns
+    -------
+    (positive_likelihood_ratio, negative_likelihood_ratio) : tuple
+        A tuple of two floats, the first containing the positive likelihood ratio (LR+)
+        and the second the negative likelihood ratio (LR-).
+
+    Warns
+    -----
+    Raises :class:`~sklearn.exceptions.UndefinedMetricWarning` when `y_true` and
+    `y_pred` lead to the following conditions:
+
+        - The number of false positives is 0 and `raise_warning` is set to `True`
+          (default): positive likelihood ratio is undefined.
+        - The number of true negatives is 0 and `raise_warning` is set to `True`
+          (default): negative likelihood ratio is undefined.
+        - The sum of true positives and false negatives is 0 (no samples of the positive
+          class are present in `y_true`): both likelihood ratios are undefined.
+
+        For the first two cases, an undefined metric can be defined by setting the
+        `replace_undefined_by` param.
+
+    References
+    ----------
+    .. [1] `Wikipedia entry for the Likelihood ratios in diagnostic testing
+           `_.
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from sklearn.metrics import class_likelihood_ratios
+    >>> class_likelihood_ratios([0, 1, 0, 1, 0], [1, 1, 0, 0, 0])
+    (1.5, 0.75)
+    >>> y_true = np.array(["non-cat", "cat", "non-cat", "cat", "non-cat"])
+    >>> y_pred = np.array(["cat", "cat", "non-cat", "non-cat", "non-cat"])
+    >>> class_likelihood_ratios(y_true, y_pred)
+    (1.33, 0.66)
+    >>> y_true = np.array(["non-zebra", "zebra", "non-zebra", "zebra", "non-zebra"])
+    >>> y_pred = np.array(["zebra", "zebra", "non-zebra", "non-zebra", "non-zebra"])
+    >>> class_likelihood_ratios(y_true, y_pred)
+    (1.5, 0.75)
+
+    To avoid ambiguities, use the notation `labels=[negative_class,
+    positive_class]`
+
+    >>> y_true = np.array(["non-cat", "cat", "non-cat", "cat", "non-cat"])
+    >>> y_pred = np.array(["cat", "cat", "non-cat", "non-cat", "non-cat"])
+    >>> class_likelihood_ratios(y_true, y_pred, labels=["non-cat", "cat"])
+    (1.5, 0.75)
+    """
+    # TODO(1.9): When `raise_warning` is removed, the following changes need to be made:
+    # The checks for `raise_warning==True` need to be removed and we will always warn,
+    # remove `FutureWarning`, and the Warns section in the docstring should not mention
+    # `raise_warning` anymore.
+    y_true, y_pred = attach_unique(y_true, y_pred)
+    y_type, y_true, y_pred = _check_targets(y_true, y_pred)
+    if y_type != "binary":
+        raise ValueError(
+            "class_likelihood_ratios only supports binary classification "
+            f"problems, got targets of type: {y_type}"
+        )
+
+    msg_deprecated_param = (
+        "`raise_warning` was deprecated in version 1.7 and will be removed in 1.9. An "
+        "`UndefinedMetricWarning` will always be raised in case of a division by zero "
+        "and the value set with the `replace_undefined_by` param will be returned."
+    )
+    if raise_warning != "deprecated":
+        warnings.warn(msg_deprecated_param, FutureWarning)
+    else:
+        raise_warning = True
+
+    if replace_undefined_by == 1.0:
+        replace_undefined_by = {"LR+": 1.0, "LR-": 1.0}
+
+    if isinstance(replace_undefined_by, dict):
+        msg = (
+            "The dictionary passed as `replace_undefined_by` needs to be in the form "
+            "`{'LR+': `value_1`, 'LR-': `value_2`}` where the value for `LR+` ranges "
+            "from `1.0` to `np.inf` or is `np.nan` and the value for `LR-` ranges from "
+            f"`0.0` to `1.0` or is `np.nan`; got `{replace_undefined_by}`."
+        )
+        if ("LR+" in replace_undefined_by) and ("LR-" in replace_undefined_by):
+            try:
+                desired_lr_pos = replace_undefined_by.get("LR+", None)
+                check_scalar(
+                    desired_lr_pos,
+                    "positive_likelihood_ratio",
+                    target_type=(Real),
+                    min_val=1.0,
+                    include_boundaries="left",
+                )
+                desired_lr_neg = replace_undefined_by.get("LR-", None)
+                check_scalar(
+                    desired_lr_neg,
+                    "negative_likelihood_ratio",
+                    target_type=(Real),
+                    min_val=0.0,
+                    max_val=1.0,
+                    include_boundaries="both",
+                )
+            except Exception as e:
+                raise ValueError(msg) from e
+        else:
+            raise ValueError(msg)
+
+    cm = confusion_matrix(
+        y_true,
+        y_pred,
+        sample_weight=sample_weight,
+        labels=labels,
+    )
+
+    tn, fp, fn, tp = cm.ravel()
+    support_pos = tp + fn
+    support_neg = tn + fp
+    pos_num = tp * support_neg
+    pos_denom = fp * support_pos
+    neg_num = fn * support_neg
+    neg_denom = tn * support_pos
+
+    # if `support_pos == 0`a division by zero will occur
+    if support_pos == 0:
+        msg = (
+            "No samples of the positive class are present in `y_true`. "
+            "`positive_likelihood_ratio` and `negative_likelihood_ratio` are both set "
+            "to `np.nan`. Use the `replace_undefined_by` param to control this "
+            "behavior. To suppress this warning or turn it into an error, see Python's "
+            "`warnings` module and `warnings.catch_warnings()`."
+        )
+        warnings.warn(msg, UndefinedMetricWarning, stacklevel=2)
+        positive_likelihood_ratio = np.nan
+        negative_likelihood_ratio = np.nan
+
+    # if `fp == 0`a division by zero will occur
+    if fp == 0:
+        if raise_warning:
+            if tp == 0:
+                msg_beginning = (
+                    "No samples were predicted for the positive class and "
+                    "`positive_likelihood_ratio` is "
+                )
+            else:
+                msg_beginning = "`positive_likelihood_ratio` is ill-defined and "
+            msg_end = "set to `np.nan`. Use the `replace_undefined_by` param to "
+            "control this behavior. To suppress this warning or turn it into an error, "
+            "see Python's `warnings` module and `warnings.catch_warnings()`."
+            warnings.warn(msg_beginning + msg_end, UndefinedMetricWarning, stacklevel=2)
+        if isinstance(replace_undefined_by, float) and np.isnan(replace_undefined_by):
+            positive_likelihood_ratio = replace_undefined_by
+        else:
+            # replace_undefined_by is a dict and
+            # isinstance(replace_undefined_by.get("LR+", None), Real); this includes
+            # `np.inf` and `np.nan`
+            positive_likelihood_ratio = desired_lr_pos
+    else:
+        positive_likelihood_ratio = pos_num / pos_denom
+
+    # if `tn == 0`a division by zero will occur
+    if tn == 0:
+        if raise_warning:
+            msg = (
+                "`negative_likelihood_ratio` is ill-defined and set to `np.nan`. "
+                "Use the `replace_undefined_by` param to control this behavior. To "
+                "suppress this warning or turn it into an error, see Python's "
+                "`warnings` module and `warnings.catch_warnings()`."
+            )
+            warnings.warn(msg, UndefinedMetricWarning, stacklevel=2)
+        if isinstance(replace_undefined_by, float) and np.isnan(replace_undefined_by):
+            negative_likelihood_ratio = replace_undefined_by
+        else:
+            # replace_undefined_by is a dict and
+            # isinstance(replace_undefined_by.get("LR-", None), Real); this includes
+            # `np.nan`
+            negative_likelihood_ratio = desired_lr_neg
+    else:
+        negative_likelihood_ratio = neg_num / neg_denom
+
+    return float(positive_likelihood_ratio), float(negative_likelihood_ratio)
+
+
+@validate_params(
+    {
+        "y_true": ["array-like", "sparse matrix"],
+        "y_pred": ["array-like", "sparse matrix"],
+        "labels": ["array-like", None],
+        "pos_label": [Real, str, "boolean", None],
+        "average": [
+            StrOptions({"micro", "macro", "samples", "weighted", "binary"}),
+            None,
+        ],
+        "sample_weight": ["array-like", None],
+        "zero_division": [
+            Options(Real, {0.0, 1.0}),
+            "nan",
+            StrOptions({"warn"}),
+        ],
+    },
+    prefer_skip_nested_validation=True,
+)
+def precision_score(
+    y_true,
+    y_pred,
+    *,
+    labels=None,
+    pos_label=1,
+    average="binary",
+    sample_weight=None,
+    zero_division="warn",
+):
     """Compute the precision.
 
     The precision is the ratio ``tp / (tp + fp)`` where ``tp`` is the number of
@@ -1564,6 +2366,16 @@ def precision_score(y_true, y_pred, *, labels=None, pos_label=1,
 
     The best value is 1 and the worst value is 0.
 
+    Support beyond term:`binary` targets is achieved by treating :term:`multiclass`
+    and :term:`multilabel` data as a collection of binary problems, one for each
+    label. For the :term:`binary` case, setting `average='binary'` will return
+    precision for `pos_label`. If `average` is not `'binary'`, `pos_label` is ignored
+    and precision for both classes are computed, then averaged or both returned (when
+    `average=None`). Similarly, for :term:`multiclass` and :term:`multilabel` targets,
+    precision for all `labels` are either returned or averaged depending on the
+    `average` parameter. Use `labels` specify the set of labels to calculate precision
+    for.
+
     Read more in the :ref:`User Guide `.
 
     Parameters
@@ -1575,27 +2387,26 @@ def precision_score(y_true, y_pred, *, labels=None, pos_label=1,
         Estimated targets as returned by a classifier.
 
     labels : array-like, default=None
-        The set of labels to include when ``average != 'binary'``, and their
-        order if ``average is None``. Labels present in the data can be
-        excluded, for example to calculate a multiclass average ignoring a
-        majority negative class, while labels not present in the data will
-        result in 0 components in a macro average. For multilabel targets,
-        labels are column indices. By default, all labels in ``y_true`` and
-        ``y_pred`` are used in sorted order.
+        The set of labels to include when `average != 'binary'`, and their
+        order if `average is None`. Labels present in the data can be
+        excluded, for example in multiclass classification to exclude a "negative
+        class". Labels not present in the data can be included and will be
+        "assigned" 0 samples. For multilabel targets, labels are column indices.
+        By default, all labels in `y_true` and `y_pred` are used in sorted order.
 
         .. versionchanged:: 0.17
            Parameter `labels` improved for multiclass problem.
 
-    pos_label : str or int, default=1
-        The class to report if ``average='binary'`` and the data is binary.
-        If the data are multiclass or multilabel, this will be ignored;
-        setting ``labels=[pos_label]`` and ``average != 'binary'`` will report
-        scores for that label only.
+    pos_label : int, float, bool or str, default=1
+        The class to report if `average='binary'` and the data is binary,
+        otherwise this parameter is ignored.
+        For multiclass or multilabel targets, set `labels=[pos_label]` and
+        `average != 'binary'` to report metrics for one label only.
 
-    average : {'micro', 'macro', 'samples', 'weighted', 'binary'} \
+    average : {'micro', 'macro', 'samples', 'weighted', 'binary'} or None, \
             default='binary'
         This parameter is required for multiclass/multilabel targets.
-        If ``None``, the scores for each class are returned. Otherwise, this
+        If ``None``, the metrics for each class are returned. Otherwise, this
         determines the type of averaging performed on the data:
 
         ``'binary'``:
@@ -1620,20 +2431,36 @@ def precision_score(y_true, y_pred, *, labels=None, pos_label=1,
     sample_weight : array-like of shape (n_samples,), default=None
         Sample weights.
 
-    zero_division : "warn", 0 or 1, default="warn"
-        Sets the value to return when there is a zero division. If set to
-        "warn", this acts as 0, but warnings are also raised.
+    zero_division : {"warn", 0.0, 1.0, np.nan}, default="warn"
+        Sets the value to return when there is a zero division.
+
+        Notes:
+
+        - If set to "warn", this acts like 0, but a warning is also raised.
+        - If set to `np.nan`, such values will be excluded from the average.
+
+        .. versionadded:: 1.3
+           `np.nan` option was added.
 
     Returns
     -------
-    precision : float (if average is not None) or array of float of shape
-        (n_unique_labels,)
+    precision : float (if average is not None) or array of float of shape \
+                (n_unique_labels,)
         Precision of the positive class in binary classification or weighted
         average of the precision of each class for the multiclass task.
 
     See Also
     --------
-    precision_recall_fscore_support, multilabel_confusion_matrix
+    precision_recall_fscore_support : Compute precision, recall, F-measure and
+        support for each class.
+    recall_score :  Compute the ratio ``tp / (tp + fn)`` where ``tp`` is the
+        number of true positives and ``fn`` the number of false negatives.
+    PrecisionRecallDisplay.from_estimator : Plot precision-recall curve given
+        an estimator and some data.
+    PrecisionRecallDisplay.from_predictions : Plot precision-recall curve given
+        binary class predictions.
+    multilabel_confusion_matrix : Compute a confusion matrix for each class or
+        sample.
 
     Notes
     -----
@@ -1643,37 +2470,74 @@ def precision_score(y_true, y_pred, *, labels=None, pos_label=1,
 
     Examples
     --------
+    >>> import numpy as np
     >>> from sklearn.metrics import precision_score
     >>> y_true = [0, 1, 2, 0, 1, 2]
     >>> y_pred = [0, 2, 1, 0, 0, 1]
     >>> precision_score(y_true, y_pred, average='macro')
-    0.22...
+    0.22
     >>> precision_score(y_true, y_pred, average='micro')
-    0.33...
+    0.33
     >>> precision_score(y_true, y_pred, average='weighted')
-    0.22...
+    0.22
     >>> precision_score(y_true, y_pred, average=None)
-    array([0.66..., 0.        , 0.        ])
+    array([0.66, 0.        , 0.        ])
     >>> y_pred = [0, 0, 0, 0, 0, 0]
     >>> precision_score(y_true, y_pred, average=None)
-    array([0.33..., 0.        , 0.        ])
+    array([0.33, 0.        , 0.        ])
     >>> precision_score(y_true, y_pred, average=None, zero_division=1)
-    array([0.33..., 1.        , 1.        ])
+    array([0.33, 1.        , 1.        ])
+    >>> precision_score(y_true, y_pred, average=None, zero_division=np.nan)
+    array([0.33,        nan,        nan])
 
+    >>> # multilabel classification
+    >>> y_true = [[0, 0, 0], [1, 1, 1], [0, 1, 1]]
+    >>> y_pred = [[0, 0, 0], [1, 1, 1], [1, 1, 0]]
+    >>> precision_score(y_true, y_pred, average=None)
+    array([0.5, 1. , 1. ])
     """
-    p, _, _, _ = precision_recall_fscore_support(y_true, y_pred,
-                                                 labels=labels,
-                                                 pos_label=pos_label,
-                                                 average=average,
-                                                 warn_for=('precision',),
-                                                 sample_weight=sample_weight,
-                                                 zero_division=zero_division)
+    p, _, _, _ = precision_recall_fscore_support(
+        y_true,
+        y_pred,
+        labels=labels,
+        pos_label=pos_label,
+        average=average,
+        warn_for=("precision",),
+        sample_weight=sample_weight,
+        zero_division=zero_division,
+    )
     return p
 
 
-@_deprecate_positional_args
-def recall_score(y_true, y_pred, *, labels=None, pos_label=1, average='binary',
-                 sample_weight=None, zero_division="warn"):
+@validate_params(
+    {
+        "y_true": ["array-like", "sparse matrix"],
+        "y_pred": ["array-like", "sparse matrix"],
+        "labels": ["array-like", None],
+        "pos_label": [Real, str, "boolean", None],
+        "average": [
+            StrOptions({"micro", "macro", "samples", "weighted", "binary"}),
+            None,
+        ],
+        "sample_weight": ["array-like", None],
+        "zero_division": [
+            Options(Real, {0.0, 1.0}),
+            "nan",
+            StrOptions({"warn"}),
+        ],
+    },
+    prefer_skip_nested_validation=True,
+)
+def recall_score(
+    y_true,
+    y_pred,
+    *,
+    labels=None,
+    pos_label=1,
+    average="binary",
+    sample_weight=None,
+    zero_division="warn",
+):
     """Compute the recall.
 
     The recall is the ratio ``tp / (tp + fn)`` where ``tp`` is the number of
@@ -1682,6 +2546,15 @@ def recall_score(y_true, y_pred, *, labels=None, pos_label=1, average='binary',
 
     The best value is 1 and the worst value is 0.
 
+    Support beyond term:`binary` targets is achieved by treating :term:`multiclass`
+    and :term:`multilabel` data as a collection of binary problems, one for each
+    label. For the :term:`binary` case, setting `average='binary'` will return
+    recall for `pos_label`. If `average` is not `'binary'`, `pos_label` is ignored
+    and recall for both classes are computed then averaged or both returned (when
+    `average=None`). Similarly, for :term:`multiclass` and :term:`multilabel` targets,
+    recall for all `labels` are either returned or averaged depending on the `average`
+    parameter. Use `labels` specify the set of labels to calculate recall for.
+
     Read more in the :ref:`User Guide `.
 
     Parameters
@@ -1693,27 +2566,26 @@ def recall_score(y_true, y_pred, *, labels=None, pos_label=1, average='binary',
         Estimated targets as returned by a classifier.
 
     labels : array-like, default=None
-        The set of labels to include when ``average != 'binary'``, and their
-        order if ``average is None``. Labels present in the data can be
-        excluded, for example to calculate a multiclass average ignoring a
-        majority negative class, while labels not present in the data will
-        result in 0 components in a macro average. For multilabel targets,
-        labels are column indices. By default, all labels in ``y_true`` and
-        ``y_pred`` are used in sorted order.
+        The set of labels to include when `average != 'binary'`, and their
+        order if `average is None`. Labels present in the data can be
+        excluded, for example in multiclass classification to exclude a "negative
+        class". Labels not present in the data can be included and will be
+        "assigned" 0 samples. For multilabel targets, labels are column indices.
+        By default, all labels in `y_true` and `y_pred` are used in sorted order.
 
         .. versionchanged:: 0.17
            Parameter `labels` improved for multiclass problem.
 
-    pos_label : str or int, default=1
-        The class to report if ``average='binary'`` and the data is binary.
-        If the data are multiclass or multilabel, this will be ignored;
-        setting ``labels=[pos_label]`` and ``average != 'binary'`` will report
-        scores for that label only.
+    pos_label : int, float, bool or str, default=1
+        The class to report if `average='binary'` and the data is binary,
+        otherwise this parameter is ignored.
+        For multiclass or multilabel targets, set `labels=[pos_label]` and
+        `average != 'binary'` to report metrics for one label only.
 
-    average : {'micro', 'macro', 'samples', 'weighted', 'binary'} \
+    average : {'micro', 'macro', 'samples', 'weighted', 'binary'} or None, \
             default='binary'
         This parameter is required for multiclass/multilabel targets.
-        If ``None``, the scores for each class are returned. Otherwise, this
+        If ``None``, the metrics for each class are returned. Otherwise, this
         determines the type of averaging performed on the data:
 
         ``'binary'``:
@@ -1729,7 +2601,8 @@ def recall_score(y_true, y_pred, *, labels=None, pos_label=1, average='binary',
             Calculate metrics for each label, and find their average weighted
             by support (the number of true instances for each label). This
             alters 'macro' to account for label imbalance; it can result in an
-            F-score that is not between precision and recall.
+            F-score that is not between precision and recall. Weighted recall
+            is equal to accuracy.
         ``'samples'``:
             Calculate metrics for each instance, and find their average (only
             meaningful for multilabel classification where this differs from
@@ -1738,21 +2611,38 @@ def recall_score(y_true, y_pred, *, labels=None, pos_label=1, average='binary',
     sample_weight : array-like of shape (n_samples,), default=None
         Sample weights.
 
-    zero_division : "warn", 0 or 1, default="warn"
-        Sets the value to return when there is a zero division. If set to
-        "warn", this acts as 0, but warnings are also raised.
+    zero_division : {"warn", 0.0, 1.0, np.nan}, default="warn"
+        Sets the value to return when there is a zero division.
+
+        Notes:
+
+        - If set to "warn", this acts like 0, but a warning is also raised.
+        - If set to `np.nan`, such values will be excluded from the average.
+
+        .. versionadded:: 1.3
+           `np.nan` option was added.
 
     Returns
     -------
-    recall : float (if average is not None) or array of float of shape
-        (n_unique_labels,)
+    recall : float (if average is not None) or array of float of shape \
+             (n_unique_labels,)
         Recall of the positive class in binary classification or weighted
         average of the recall of each class for the multiclass task.
 
     See Also
     --------
-    precision_recall_fscore_support, balanced_accuracy_score,
-    multilabel_confusion_matrix
+    precision_recall_fscore_support : Compute precision, recall, F-measure and
+        support for each class.
+    precision_score : Compute the ratio ``tp / (tp + fp)`` where ``tp`` is the
+        number of true positives and ``fp`` the number of false positives.
+    balanced_accuracy_score : Compute balanced accuracy to deal with imbalanced
+        datasets.
+    multilabel_confusion_matrix : Compute a confusion matrix for each class or
+        sample.
+    PrecisionRecallDisplay.from_estimator : Plot precision-recall curve given
+        an estimator and some data.
+    PrecisionRecallDisplay.from_predictions : Plot precision-recall curve given
+        binary class predictions.
 
     Notes
     -----
@@ -1762,15 +2652,16 @@ def recall_score(y_true, y_pred, *, labels=None, pos_label=1, average='binary',
 
     Examples
     --------
+    >>> import numpy as np
     >>> from sklearn.metrics import recall_score
     >>> y_true = [0, 1, 2, 0, 1, 2]
     >>> y_pred = [0, 2, 1, 0, 0, 1]
     >>> recall_score(y_true, y_pred, average='macro')
-    0.33...
+    0.33
     >>> recall_score(y_true, y_pred, average='micro')
-    0.33...
+    0.33
     >>> recall_score(y_true, y_pred, average='weighted')
-    0.33...
+    0.33
     >>> recall_score(y_true, y_pred, average=None)
     array([1., 0., 0.])
     >>> y_true = [0, 0, 0, 0, 0, 0]
@@ -1778,20 +2669,38 @@ def recall_score(y_true, y_pred, *, labels=None, pos_label=1, average='binary',
     array([0.5, 0. , 0. ])
     >>> recall_score(y_true, y_pred, average=None, zero_division=1)
     array([0.5, 1. , 1. ])
+    >>> recall_score(y_true, y_pred, average=None, zero_division=np.nan)
+    array([0.5, nan, nan])
+
+    >>> # multilabel classification
+    >>> y_true = [[0, 0, 0], [1, 1, 1], [0, 1, 1]]
+    >>> y_pred = [[0, 0, 0], [1, 1, 1], [1, 1, 0]]
+    >>> recall_score(y_true, y_pred, average=None)
+    array([1. , 1. , 0.5])
     """
-    _, r, _, _ = precision_recall_fscore_support(y_true, y_pred,
-                                                 labels=labels,
-                                                 pos_label=pos_label,
-                                                 average=average,
-                                                 warn_for=('recall',),
-                                                 sample_weight=sample_weight,
-                                                 zero_division=zero_division)
+    _, r, _, _ = precision_recall_fscore_support(
+        y_true,
+        y_pred,
+        labels=labels,
+        pos_label=pos_label,
+        average=average,
+        warn_for=("recall",),
+        sample_weight=sample_weight,
+        zero_division=zero_division,
+    )
     return r
 
 
-@_deprecate_positional_args
-def balanced_accuracy_score(y_true, y_pred, *, sample_weight=None,
-                            adjusted=False):
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_pred": ["array-like"],
+        "sample_weight": ["array-like", None],
+        "adjusted": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def balanced_accuracy_score(y_true, y_pred, *, sample_weight=None, adjusted=False):
     """Compute the balanced accuracy.
 
     The balanced accuracy in binary and multiclass classification problems to
@@ -1806,10 +2715,10 @@ def balanced_accuracy_score(y_true, y_pred, *, sample_weight=None,
 
     Parameters
     ----------
-    y_true : 1d array-like
+    y_true : array-like of shape (n_samples,)
         Ground truth (correct) target values.
 
-    y_pred : 1d array-like
+    y_pred : array-like of shape (n_samples,)
         Estimated targets as returned by a classifier.
 
     sample_weight : array-like of shape (n_samples,), default=None
@@ -1823,10 +2732,16 @@ def balanced_accuracy_score(y_true, y_pred, *, sample_weight=None,
     Returns
     -------
     balanced_accuracy : float
+        Balanced accuracy score.
 
     See Also
     --------
-    recall_score, roc_auc_score
+    average_precision_score : Compute average precision (AP) from prediction
+        scores.
+    precision_score : Compute the precision score.
+    recall_score : Compute the recall score.
+    roc_auc_score : Compute Area Under the Receiver Operating Characteristic
+        Curve (ROC AUC) from prediction scores.
 
     Notes
     -----
@@ -1853,13 +2768,12 @@ def balanced_accuracy_score(y_true, y_pred, *, sample_weight=None,
     >>> y_pred = [0, 1, 0, 0, 0, 1]
     >>> balanced_accuracy_score(y_true, y_pred)
     0.625
-
     """
     C = confusion_matrix(y_true, y_pred, sample_weight=sample_weight)
-    with np.errstate(divide='ignore', invalid='ignore'):
+    with np.errstate(divide="ignore", invalid="ignore"):
         per_class = np.diag(C) / C.sum(axis=1)
     if np.any(np.isnan(per_class)):
-        warnings.warn('y_pred contains classes not in y_true')
+        warnings.warn("y_pred contains classes not in y_true")
         per_class = per_class[~np.isnan(per_class)]
     score = np.mean(per_class)
     if adjusted:
@@ -1867,13 +2781,37 @@ def balanced_accuracy_score(y_true, y_pred, *, sample_weight=None,
         chance = 1 / n_classes
         score -= chance
         score /= 1 - chance
-    return score
-
-
-@_deprecate_positional_args
-def classification_report(y_true, y_pred, *, labels=None, target_names=None,
-                          sample_weight=None, digits=2, output_dict=False,
-                          zero_division="warn"):
+    return float(score)
+
+
+@validate_params(
+    {
+        "y_true": ["array-like", "sparse matrix"],
+        "y_pred": ["array-like", "sparse matrix"],
+        "labels": ["array-like", None],
+        "target_names": ["array-like", None],
+        "sample_weight": ["array-like", None],
+        "digits": [Interval(Integral, 0, None, closed="left")],
+        "output_dict": ["boolean"],
+        "zero_division": [
+            Options(Real, {0.0, 1.0}),
+            "nan",
+            StrOptions({"warn"}),
+        ],
+    },
+    prefer_skip_nested_validation=True,
+)
+def classification_report(
+    y_true,
+    y_pred,
+    *,
+    labels=None,
+    target_names=None,
+    sample_weight=None,
+    digits=2,
+    output_dict=False,
+    zero_division="warn",
+):
     """Build a text report showing the main classification metrics.
 
     Read more in the :ref:`User Guide `.
@@ -1889,7 +2827,7 @@ def classification_report(y_true, y_pred, *, labels=None, target_names=None,
     labels : array-like of shape (n_labels,), default=None
         Optional list of label indices to include in the report.
 
-    target_names : list of str of shape (n_labels,), default=None
+    target_names : array-like of shape (n_labels,), default=None
         Optional display names matching the labels (same order).
 
     sample_weight : array-like of shape (n_samples,), default=None
@@ -1905,13 +2843,16 @@ def classification_report(y_true, y_pred, *, labels=None, target_names=None,
 
         .. versionadded:: 0.20
 
-    zero_division : "warn", 0 or 1, default="warn"
+    zero_division : {"warn", 0.0, 1.0, np.nan}, default="warn"
         Sets the value to return when there is a zero division. If set to
         "warn", this acts as 0, but warnings are also raised.
 
+        .. versionadded:: 1.3
+           `np.nan` option was added.
+
     Returns
     -------
-    report : string / dict
+    report : str or dict
         Text summary of the precision, recall, F1 score for each class.
         Dictionary returned if output_dict is True. Dictionary has the
         following structure::
@@ -1940,8 +2881,11 @@ def classification_report(y_true, y_pred, *, labels=None, target_names=None,
 
     See Also
     --------
-    precision_recall_fscore_support, confusion_matrix,
-    multilabel_confusion_matrix
+    precision_recall_fscore_support: Compute precision, recall, F-measure and
+        support for each class.
+    confusion_matrix: Compute confusion matrix to evaluate the accuracy of a
+        classification.
+    multilabel_confusion_matrix: Compute a confusion matrix for each class or sample.
 
     Examples
     --------
@@ -1975,6 +2919,7 @@ class 2       1.00      0.67      0.80         3
     
     """
 
+    y_true, y_pred = attach_unique(y_true, y_pred)
     y_type, y_true, y_pred = _check_targets(y_true, y_pred)
 
     if labels is None:
@@ -1985,15 +2930,16 @@ class 2       1.00      0.67      0.80         3
         labels_given = True
 
     # labelled micro average
-    micro_is_accuracy = ((y_type == 'multiclass' or y_type == 'binary') and
-                         (not labels_given or
-                          (set(labels) == set(unique_labels(y_true, y_pred)))))
+    micro_is_accuracy = (y_type == "multiclass" or y_type == "binary") and (
+        not labels_given or (set(labels) >= set(unique_labels(y_true, y_pred)))
+    )
 
     if target_names is not None and len(labels) != len(target_names):
         if labels_given:
             warnings.warn(
-                "labels size, {0}, does not match size of target_names, {1}"
-                .format(len(labels), len(target_names))
+                "labels size, {0}, does not match size of target_names, {1}".format(
+                    len(labels), len(target_names)
+                )
             )
         else:
             raise ValueError(
@@ -2002,77 +2948,91 @@ class 2       1.00      0.67      0.80         3
                 "parameter".format(len(labels), len(target_names))
             )
     if target_names is None:
-        target_names = ['%s' % l for l in labels]
+        target_names = ["%s" % l for l in labels]
 
     headers = ["precision", "recall", "f1-score", "support"]
     # compute per-class results without averaging
-    p, r, f1, s = precision_recall_fscore_support(y_true, y_pred,
-                                                  labels=labels,
-                                                  average=None,
-                                                  sample_weight=sample_weight,
-                                                  zero_division=zero_division)
+    p, r, f1, s = precision_recall_fscore_support(
+        y_true,
+        y_pred,
+        labels=labels,
+        average=None,
+        sample_weight=sample_weight,
+        zero_division=zero_division,
+    )
     rows = zip(target_names, p, r, f1, s)
 
-    if y_type.startswith('multilabel'):
-        average_options = ('micro', 'macro', 'weighted', 'samples')
+    if y_type.startswith("multilabel"):
+        average_options = ("micro", "macro", "weighted", "samples")
     else:
-        average_options = ('micro', 'macro', 'weighted')
+        average_options = ("micro", "macro", "weighted")
 
     if output_dict:
         report_dict = {label[0]: label[1:] for label in rows}
         for label, scores in report_dict.items():
-            report_dict[label] = dict(zip(headers,
-                                          [i.item() for i in scores]))
+            report_dict[label] = dict(zip(headers, [float(i) for i in scores]))
     else:
-        longest_last_line_heading = 'weighted avg'
+        longest_last_line_heading = "weighted avg"
         name_width = max(len(cn) for cn in target_names)
         width = max(name_width, len(longest_last_line_heading), digits)
-        head_fmt = '{:>{width}s} ' + ' {:>9}' * len(headers)
-        report = head_fmt.format('', *headers, width=width)
-        report += '\n\n'
-        row_fmt = '{:>{width}s} ' + ' {:>9.{digits}f}' * 3 + ' {:>9}\n'
+        head_fmt = "{:>{width}s} " + " {:>9}" * len(headers)
+        report = head_fmt.format("", *headers, width=width)
+        report += "\n\n"
+        row_fmt = "{:>{width}s} " + " {:>9.{digits}f}" * 3 + " {:>9}\n"
         for row in rows:
             report += row_fmt.format(*row, width=width, digits=digits)
-        report += '\n'
+        report += "\n"
 
     # compute all applicable averages
     for average in average_options:
-        if average.startswith('micro') and micro_is_accuracy:
-            line_heading = 'accuracy'
+        if average.startswith("micro") and micro_is_accuracy:
+            line_heading = "accuracy"
         else:
-            line_heading = average + ' avg'
+            line_heading = average + " avg"
 
         # compute averages with specified averaging method
         avg_p, avg_r, avg_f1, _ = precision_recall_fscore_support(
-            y_true, y_pred, labels=labels,
-            average=average, sample_weight=sample_weight,
-            zero_division=zero_division)
+            y_true,
+            y_pred,
+            labels=labels,
+            average=average,
+            sample_weight=sample_weight,
+            zero_division=zero_division,
+        )
         avg = [avg_p, avg_r, avg_f1, np.sum(s)]
 
         if output_dict:
-            report_dict[line_heading] = dict(
-                zip(headers, [i.item() for i in avg]))
+            report_dict[line_heading] = dict(zip(headers, [float(i) for i in avg]))
         else:
-            if line_heading == 'accuracy':
-                row_fmt_accuracy = '{:>{width}s} ' + \
-                        ' {:>9.{digits}}' * 2 + ' {:>9.{digits}f}' + \
-                        ' {:>9}\n'
-                report += row_fmt_accuracy.format(line_heading, '', '',
-                                                  *avg[2:], width=width,
-                                                  digits=digits)
+            if line_heading == "accuracy":
+                row_fmt_accuracy = (
+                    "{:>{width}s} "
+                    + " {:>9.{digits}}" * 2
+                    + " {:>9.{digits}f}"
+                    + " {:>9}\n"
+                )
+                report += row_fmt_accuracy.format(
+                    line_heading, "", "", *avg[2:], width=width, digits=digits
+                )
             else:
-                report += row_fmt.format(line_heading, *avg,
-                                         width=width, digits=digits)
+                report += row_fmt.format(line_heading, *avg, width=width, digits=digits)
 
     if output_dict:
-        if 'accuracy' in report_dict.keys():
-            report_dict['accuracy'] = report_dict['accuracy']['precision']
+        if "accuracy" in report_dict.keys():
+            report_dict["accuracy"] = report_dict["accuracy"]["precision"]
         return report_dict
     else:
         return report
 
 
-@_deprecate_positional_args
+@validate_params(
+    {
+        "y_true": ["array-like", "sparse matrix"],
+        "y_pred": ["array-like", "sparse matrix"],
+        "sample_weight": ["array-like", None],
+    },
+    prefer_skip_nested_validation=True,
+)
 def hamming_loss(y_true, y_pred, *, sample_weight=None):
     """Compute the average Hamming loss.
 
@@ -2101,7 +3061,12 @@ def hamming_loss(y_true, y_pred, *, sample_weight=None):
 
     See Also
     --------
-    accuracy_score, jaccard_score, zero_one_loss
+    accuracy_score : Compute the accuracy score. By default, the function will
+        return the fraction of correct predictions divided by the total number
+        of predictions.
+    jaccard_score : Compute the Jaccard similarity coefficient score.
+    zero_one_loss : Compute the Zero-one classification loss. By default, the
+        function will return the percentage of imperfectly predicted subsets.
 
     Notes
     -----
@@ -2143,30 +3108,43 @@ def hamming_loss(y_true, y_pred, *, sample_weight=None):
     >>> hamming_loss(np.array([[0, 1], [1, 1]]), np.zeros((2, 2)))
     0.75
     """
-
+    y_true, y_pred = attach_unique(y_true, y_pred)
     y_type, y_true, y_pred = _check_targets(y_true, y_pred)
     check_consistent_length(y_true, y_pred, sample_weight)
 
+    xp, _, device = get_namespace_and_device(y_true, y_pred, sample_weight)
+
     if sample_weight is None:
-        weight_average = 1.
+        weight_average = 1.0
     else:
-        weight_average = np.mean(sample_weight)
+        sample_weight = xp.asarray(sample_weight, device=device)
+        weight_average = _average(sample_weight, xp=xp)
 
-    if y_type.startswith('multilabel'):
-        n_differences = count_nonzero(y_true - y_pred,
-                                      sample_weight=sample_weight)
-        return (n_differences /
-                (y_true.shape[0] * y_true.shape[1] * weight_average))
+    if y_type.startswith("multilabel"):
+        n_differences = _count_nonzero(
+            y_true - y_pred, xp=xp, device=device, sample_weight=sample_weight
+        )
+        return float(n_differences) / (
+            y_true.shape[0] * y_true.shape[1] * weight_average
+        )
 
     elif y_type in ["binary", "multiclass"]:
-        return _weighted_sum(y_true != y_pred, sample_weight, normalize=True)
+        return float(_average(y_true != y_pred, weights=sample_weight, normalize=True))
     else:
         raise ValueError("{0} is not supported".format(y_type))
 
 
-@_deprecate_positional_args
-def log_loss(y_true, y_pred, *, eps=1e-15, normalize=True, sample_weight=None,
-             labels=None):
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_pred": ["array-like"],
+        "normalize": ["boolean"],
+        "sample_weight": ["array-like", None],
+        "labels": ["array-like", None],
+    },
+    prefer_skip_nested_validation=True,
+)
+def log_loss(y_true, y_pred, *, normalize=True, sample_weight=None, labels=None):
     r"""Log loss, aka logistic loss or cross-entropy loss.
 
     This is the loss function used in (multinomial) logistic regression
@@ -2175,7 +3153,7 @@ def log_loss(y_true, y_pred, *, eps=1e-15, normalize=True, sample_weight=None,
     for its training data ``y_true``.
     The log loss is only defined for two or more labels.
     For a single sample with true label :math:`y \in \{0,1\}` and
-    and a probability estimate :math:`p = \operatorname{Pr}(y = 1)`, the log
+    a probability estimate :math:`p = \operatorname{Pr}(y = 1)`, the log
     loss is:
 
     .. math::
@@ -2194,11 +3172,10 @@ def log_loss(y_true, y_pred, *, eps=1e-15, normalize=True, sample_weight=None,
         the probabilities provided are assumed to be that of the
         positive class. The labels in ``y_pred`` are assumed to be
         ordered alphabetically, as done by
-        :class:`preprocessing.LabelBinarizer`.
+        :class:`~sklearn.preprocessing.LabelBinarizer`.
 
-    eps : float, default=1e-15
-        Log loss is undefined for p=0 or p=1, so probabilities are
-        clipped to max(eps, min(1 - eps, p)).
+        `y_pred` values are clipped to `[eps, 1-eps]` where `eps` is the machine
+        precision for `y_pred`'s dtype.
 
     normalize : bool, default=True
         If true, return the mean loss per sample.
@@ -2217,83 +3194,46 @@ def log_loss(y_true, y_pred, *, eps=1e-15, normalize=True, sample_weight=None,
     Returns
     -------
     loss : float
+        Log loss, aka logistic loss or cross-entropy loss.
 
     Notes
     -----
     The logarithm used is the natural logarithm (base-e).
 
+    References
+    ----------
+    C.M. Bishop (2006). Pattern Recognition and Machine Learning. Springer,
+    p. 209.
+
     Examples
     --------
     >>> from sklearn.metrics import log_loss
     >>> log_loss(["spam", "ham", "ham", "spam"],
     ...          [[.1, .9], [.9, .1], [.8, .2], [.35, .65]])
-    0.21616...
-
-    References
-    ----------
-    C.M. Bishop (2006). Pattern Recognition and Machine Learning. Springer,
-    p. 209.
+    0.21616
     """
-    y_pred = check_array(y_pred, ensure_2d=False)
-    check_consistent_length(y_pred, y_true, sample_weight)
-
-    lb = LabelBinarizer()
-
-    if labels is not None:
-        lb.fit(labels)
-    else:
-        lb.fit(y_true)
-
-    if len(lb.classes_) == 1:
-        if labels is None:
-            raise ValueError('y_true contains only one label ({0}). Please '
-                             'provide the true labels explicitly through the '
-                             'labels argument.'.format(lb.classes_[0]))
-        else:
-            raise ValueError('The labels array needs to contain at least two '
-                             'labels for log_loss, '
-                             'got {0}.'.format(lb.classes_))
-
-    transformed_labels = lb.transform(y_true)
-
-    if transformed_labels.shape[1] == 1:
-        transformed_labels = np.append(1 - transformed_labels,
-                                       transformed_labels, axis=1)
+    transformed_labels, y_pred = _validate_multiclass_probabilistic_prediction(
+        y_true, y_pred, sample_weight, labels
+    )
 
     # Clipping
+    eps = np.finfo(y_pred.dtype).eps
     y_pred = np.clip(y_pred, eps, 1 - eps)
 
-    # If y_pred is of single dimension, assume y_true to be binary
-    # and then check.
-    if y_pred.ndim == 1:
-        y_pred = y_pred[:, np.newaxis]
-    if y_pred.shape[1] == 1:
-        y_pred = np.append(1 - y_pred, y_pred, axis=1)
-
-    # Check if dimensions are consistent.
-    transformed_labels = check_array(transformed_labels)
-    if len(lb.classes_) != y_pred.shape[1]:
-        if labels is None:
-            raise ValueError("y_true and y_pred contain different number of "
-                             "classes {0}, {1}. Please provide the true "
-                             "labels explicitly through the labels argument. "
-                             "Classes found in "
-                             "y_true: {2}".format(transformed_labels.shape[1],
-                                                  y_pred.shape[1],
-                                                  lb.classes_))
-        else:
-            raise ValueError('The number of classes in labels is different '
-                             'from that in y_pred. Classes found in '
-                             'labels: {0}'.format(lb.classes_))
+    loss = -xlogy(transformed_labels, y_pred).sum(axis=1)
 
-    # Renormalize
-    y_pred /= y_pred.sum(axis=1)[:, np.newaxis]
-    loss = -(transformed_labels * np.log(y_pred)).sum(axis=1)
+    return float(_average(loss, weights=sample_weight, normalize=normalize))
 
-    return _weighted_sum(loss, sample_weight, normalize)
 
-
-@_deprecate_positional_args
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "pred_decision": ["array-like"],
+        "labels": ["array-like", None],
+        "sample_weight": ["array-like", None],
+    },
+    prefer_skip_nested_validation=True,
+)
 def hinge_loss(y_true, pred_decision, *, labels=None, sample_weight=None):
     """Average hinge loss (non-regularized).
 
@@ -2313,11 +3253,11 @@ def hinge_loss(y_true, pred_decision, *, labels=None, sample_weight=None):
 
     Parameters
     ----------
-    y_true : array of shape (n_samples,)
+    y_true : array-like of shape (n_samples,)
         True target, consisting of integers of two values. The positive label
         must be greater than the negative label.
 
-    pred_decision : array of shape (n_samples,) or (n_samples, n_classes)
+    pred_decision : array-like of shape (n_samples,) or (n_samples, n_classes)
         Predicted decisions, as output by decision_function (floats).
 
     labels : array-like, default=None
@@ -2329,6 +3269,7 @@ def hinge_loss(y_true, pred_decision, *, labels=None, sample_weight=None):
     Returns
     -------
     loss : float
+        Average hinge loss.
 
     References
     ----------
@@ -2342,8 +3283,7 @@ def hinge_loss(y_true, pred_decision, *, labels=None, sample_weight=None):
 
     .. [3] `L1 AND L2 Regularization for Multiclass Hinge Loss Models
            by Robert C. Moore, John DeNero
-           `_.
+           `_.
 
     Examples
     --------
@@ -2356,9 +3296,9 @@ def hinge_loss(y_true, pred_decision, *, labels=None, sample_weight=None):
     LinearSVC(random_state=0)
     >>> pred_decision = est.decision_function([[-2], [3], [0.5]])
     >>> pred_decision
-    array([-2.18...,  2.36...,  0.09...])
+    array([-2.18,  2.36,  0.09])
     >>> hinge_loss([-1, 1, 1], pred_decision)
-    0.30...
+    0.30
 
     In the multiclass case:
 
@@ -2372,7 +3312,7 @@ def hinge_loss(y_true, pred_decision, *, labels=None, sample_weight=None):
     >>> pred_decision = est.decision_function([[-1], [2], [3]])
     >>> y_true = [0, 2, 3]
     >>> hinge_loss(y_true, pred_decision, labels=labels)
-    0.56...
+    0.56
     """
     check_consistent_length(y_true, pred_decision, sample_weight)
     pred_decision = check_array(pred_decision, ensure_2d=False)
@@ -2380,27 +3320,32 @@ def hinge_loss(y_true, pred_decision, *, labels=None, sample_weight=None):
     y_true_unique = np.unique(labels if labels is not None else y_true)
 
     if y_true_unique.size > 2:
-
         if pred_decision.ndim <= 1:
-            raise ValueError("The shape of pred_decision cannot be 1d array"
-                             "with a multiclass target. pred_decision shape "
-                             "must be (n_samples, n_classes), that is "
-                             f"({y_true.shape[0]}, {y_true_unique.size})."
-                             f" Got: {pred_decision.shape}")
+            raise ValueError(
+                "The shape of pred_decision cannot be 1d array"
+                "with a multiclass target. pred_decision shape "
+                "must be (n_samples, n_classes), that is "
+                f"({y_true.shape[0]}, {y_true_unique.size})."
+                f" Got: {pred_decision.shape}"
+            )
 
         # pred_decision.ndim > 1 is true
         if y_true_unique.size != pred_decision.shape[1]:
             if labels is None:
-                raise ValueError("Please include all labels in y_true "
-                                 "or pass labels as third argument")
+                raise ValueError(
+                    "Please include all labels in y_true "
+                    "or pass labels as third argument"
+                )
             else:
-                raise ValueError("The shape of pred_decision is not "
-                                 "consistent with the number of classes. "
-                                 "With a multiclass target, pred_decision "
-                                 "shape must be "
-                                 "(n_samples, n_classes), that is "
-                                 f"({y_true.shape[0]}, {y_true_unique.size}). "
-                                 f"Got: {pred_decision.shape}")
+                raise ValueError(
+                    "The shape of pred_decision is not "
+                    "consistent with the number of classes. "
+                    "With a multiclass target, pred_decision "
+                    "shape must be "
+                    "(n_samples, n_classes), that is "
+                    f"({y_true.shape[0]}, {y_true_unique.size}). "
+                    f"Got: {pred_decision.shape}"
+                )
         if labels is None:
             labels = y_true_unique
         le = LabelEncoder()
@@ -2409,8 +3354,7 @@ def hinge_loss(y_true, pred_decision, *, labels=None, sample_weight=None):
         mask = np.ones_like(pred_decision, dtype=bool)
         mask[np.arange(y_true.shape[0]), y_true] = False
         margin = pred_decision[~mask]
-        margin -= np.max(pred_decision[mask].reshape(y_true.shape[0], -1),
-                         axis=1)
+        margin -= np.max(pred_decision[mask].reshape(y_true.shape[0], -1), axis=1)
 
     else:
         # Handles binary class case
@@ -2430,59 +3374,184 @@ def hinge_loss(y_true, pred_decision, *, labels=None, sample_weight=None):
     losses = 1 - margin
     # The hinge_loss doesn't penalize good enough predictions.
     np.clip(losses, 0, None, out=losses)
-    return np.average(losses, weights=sample_weight)
+    return float(np.average(losses, weights=sample_weight))
+
+
+def _validate_binary_probabilistic_prediction(y_true, y_prob, sample_weight, pos_label):
+    r"""Convert y_true and y_prob in binary classification to shape (n_samples, 2)
+
+    Parameters
+    ----------
+    y_true : array-like of shape (n_samples,)
+        True labels.
 
+    y_prob : array-like of shape (n_samples,)
+        Probabilities of the positive class.
 
-@_deprecate_positional_args
-def brier_score_loss(y_true, y_prob, *, sample_weight=None, pos_label=None):
-    """Compute the Brier score loss.
+    sample_weight : array-like of shape (n_samples,), default=None
+        Sample weights.
+
+    pos_label : int, float, bool or str, default=None
+        Label of the positive class. If None, `pos_label` will be inferred
+        in the following manner:
+
+        * if `y_true` in {-1, 1} or {0, 1}, `pos_label` defaults to 1;
+        * else if `y_true` contains string, an error will be raised and
+          `pos_label` should be explicitly specified;
+        * otherwise, `pos_label` defaults to the greater label,
+          i.e. `np.unique(y_true)[-1]`.
+
+    Returns
+    -------
+    transformed_labels : array of shape (n_samples, 2)
+
+    y_prob : array of shape (n_samples, 2)
+    """
+    # sanity checks on y_true and y_prob
+    y_true = column_or_1d(y_true)
+    y_prob = column_or_1d(y_prob)
+
+    assert_all_finite(y_true)
+    assert_all_finite(y_prob)
+
+    check_consistent_length(y_prob, y_true, sample_weight)
+
+    y_type = type_of_target(y_true, input_name="y_true")
+    if y_type != "binary":
+        raise ValueError(
+            f"The type of the target inferred from y_true is {y_type} but should be "
+            "binary according to the shape of y_prob."
+        )
+
+    if y_prob.max() > 1:
+        raise ValueError(f"y_prob contains values greater than 1: {y_prob.max()}")
+    if y_prob.min() < 0:
+        raise ValueError(f"y_prob contains values less than 0: {y_prob.min()}")
+
+    # check that pos_label is consistent with y_true
+    try:
+        pos_label = _check_pos_label_consistency(pos_label, y_true)
+    except ValueError:
+        classes = np.unique(y_true)
+        if classes.dtype.kind not in ("O", "U", "S"):
+            # for backward compatibility, if classes are not string then
+            # `pos_label` will correspond to the greater label
+            pos_label = classes[-1]
+        else:
+            raise
+
+    # convert (n_samples,) to (n_samples, 2) shape
+    y_true = np.array(y_true == pos_label, int)
+    transformed_labels = np.column_stack((1 - y_true, y_true))
+    y_prob = np.column_stack((1 - y_prob, y_prob))
+
+    return transformed_labels, y_prob
+
+
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_proba": ["array-like"],
+        "sample_weight": ["array-like", None],
+        "pos_label": [Real, str, "boolean", None],
+        "labels": ["array-like", None],
+        "scale_by_half": ["boolean", StrOptions({"auto"})],
+    },
+    prefer_skip_nested_validation=True,
+)
+def brier_score_loss(
+    y_true,
+    y_proba,
+    *,
+    sample_weight=None,
+    pos_label=None,
+    labels=None,
+    scale_by_half="auto",
+):
+    r"""Compute the Brier score loss.
 
     The smaller the Brier score loss, the better, hence the naming with "loss".
     The Brier score measures the mean squared difference between the predicted
-    probability and the actual outcome. The Brier score always
-    takes on a value between zero and one, since this is the largest
-    possible difference between a predicted probability (which must be
-    between zero and one) and the actual outcome (which can take on values
-    of only 0 and 1). It can be decomposed is the sum of refinement loss and
-    calibration loss.
-
-    The Brier score is appropriate for binary and categorical outcomes that
-    can be structured as true or false, but is inappropriate for ordinal
-    variables which can take on three or more values (this is because the
-    Brier score assumes that all possible outcomes are equivalently
-    "distant" from one another). Which label is considered to be the positive
-    label is controlled via the parameter `pos_label`, which defaults to
-    the greater label unless `y_true` is all 0 or all -1, in which case
-    `pos_label` defaults to 1.
+    probability and the actual outcome. The Brier score is a strictly proper scoring
+    rule.
 
     Read more in the :ref:`User Guide `.
 
     Parameters
     ----------
-    y_true : array of shape (n_samples,)
+    y_true : array-like of shape (n_samples,)
         True targets.
 
-    y_prob : array of shape (n_samples,)
-        Probabilities of the positive class.
+    y_proba : array-like of shape (n_samples,) or (n_samples, n_classes)
+        Predicted probabilities. If `y_proba.shape = (n_samples,)`
+        the probabilities provided are assumed to be that of the
+        positive class. If `y_proba.shape = (n_samples, n_classes)`
+        the columns in `y_proba` are assumed to correspond to the
+        labels in alphabetical order, as done by
+        :class:`~sklearn.preprocessing.LabelBinarizer`.
 
     sample_weight : array-like of shape (n_samples,), default=None
         Sample weights.
 
-    pos_label : int or str, default=None
-        Label of the positive class. `pos_label` will be infered in the
+    pos_label : int, float, bool or str, default=None
+        Label of the positive class when `y_proba.shape = (n_samples,)`.
+        If not provided, `pos_label` will be inferred in the
         following manner:
 
         * if `y_true` in {-1, 1} or {0, 1}, `pos_label` defaults to 1;
         * else if `y_true` contains string, an error will be raised and
-          `pos_label` should be explicitely specified;
+          `pos_label` should be explicitly specified;
         * otherwise, `pos_label` defaults to the greater label,
           i.e. `np.unique(y_true)[-1]`.
 
+    labels : array-like of shape (n_classes,), default=None
+        Class labels when `y_proba.shape = (n_samples, n_classes)`.
+        If not provided, labels will be inferred from `y_true`.
+
+        .. versionadded:: 1.7
+
+    scale_by_half : bool or "auto", default="auto"
+        When True, scale the Brier score by 1/2 to lie in the [0, 1] range instead
+        of the [0, 2] range. The default "auto" option implements the rescaling to
+        [0, 1] only for binary classification (as customary) but keeps the
+        original [0, 2] range for multiclasss classification.
+
+        .. versionadded:: 1.7
+
     Returns
     -------
     score : float
         Brier score loss.
 
+    Notes
+    -----
+
+    For :math:`N` observations labeled from :math:`C` possible classes, the Brier
+    score is defined as:
+
+    .. math::
+        \frac{1}{N}\sum_{i=1}^{N}\sum_{c=1}^{C}(y_{ic} - \hat{p}_{ic})^{2}
+
+    where :math:`y_{ic}` is 1 if observation `i` belongs to class `c`,
+    otherwise 0 and :math:`\hat{p}_{ic}` is the predicted probability for
+    observation `i` to belong to class `c`.
+    The Brier score then ranges between :math:`[0, 2]`.
+
+    In binary classification tasks the Brier score is usually divided by
+    two and then ranges between :math:`[0, 1]`. It can be alternatively
+    written as:
+
+    .. math::
+        \frac{1}{N}\sum_{i=1}^{N}(y_{i} - \hat{p}_{i})^{2}
+
+    where :math:`y_{i}` is the binary target and :math:`\hat{p}_{i}`
+    is the predicted probability of the positive class.
+
+    References
+    ----------
+    .. [1] `Wikipedia entry for the Brier score
+            `_.
+
     Examples
     --------
     >>> import numpy as np
@@ -2491,46 +3560,146 @@ def brier_score_loss(y_true, y_prob, *, sample_weight=None, pos_label=None):
     >>> y_true_categorical = np.array(["spam", "ham", "ham", "spam"])
     >>> y_prob = np.array([0.1, 0.9, 0.8, 0.3])
     >>> brier_score_loss(y_true, y_prob)
-    0.037...
+    0.0375
     >>> brier_score_loss(y_true, 1-y_prob, pos_label=0)
-    0.037...
+    0.0375
     >>> brier_score_loss(y_true_categorical, y_prob, pos_label="ham")
-    0.037...
+    0.0375
     >>> brier_score_loss(y_true, np.array(y_prob) > 0.5)
     0.0
+    >>> brier_score_loss(y_true, y_prob, scale_by_half=False)
+    0.075
+    >>> brier_score_loss(
+    ...    ["eggs", "ham", "spam"],
+    ...    [[0.8, 0.1, 0.1], [0.2, 0.7, 0.1], [0.2, 0.2, 0.6]],
+    ...    labels=["eggs", "ham", "spam"]
+    ... )
+    0.146
+    """
+    y_proba = check_array(
+        y_proba, ensure_2d=False, dtype=[np.float64, np.float32, np.float16]
+    )
 
-    References
+    if y_proba.ndim == 1 or y_proba.shape[1] == 1:
+        transformed_labels, y_proba = _validate_binary_probabilistic_prediction(
+            y_true, y_proba, sample_weight, pos_label
+        )
+    else:
+        transformed_labels, y_proba = _validate_multiclass_probabilistic_prediction(
+            y_true, y_proba, sample_weight, labels
+        )
+
+    brier_score = np.average(
+        np.sum((transformed_labels - y_proba) ** 2, axis=1), weights=sample_weight
+    )
+
+    if scale_by_half == "auto":
+        scale_by_half = y_proba.ndim == 1 or y_proba.shape[1] < 3
+    if scale_by_half:
+        brier_score *= 0.5
+
+    return float(brier_score)
+
+
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_pred": ["array-like"],
+        "sample_weight": ["array-like", None],
+        "labels": ["array-like", None],
+    },
+    prefer_skip_nested_validation=True,
+)
+def d2_log_loss_score(y_true, y_pred, *, sample_weight=None, labels=None):
+    """
+    :math:`D^2` score function, fraction of log loss explained.
+
+    Best possible score is 1.0 and it can be negative (because the model can be
+    arbitrarily worse). A model that always predicts the per-class proportions
+    of `y_true`, disregarding the input features, gets a D^2 score of 0.0.
+
+    Read more in the :ref:`User Guide `.
+
+    .. versionadded:: 1.5
+
+    Parameters
     ----------
-    .. [1] `Wikipedia entry for the Brier score
-            `_.
+    y_true : array-like or label indicator matrix
+        The actuals labels for the n_samples samples.
+
+    y_pred : array-like of shape (n_samples, n_classes) or (n_samples,)
+        Predicted probabilities, as returned by a classifier's
+        predict_proba method. If ``y_pred.shape = (n_samples,)``
+        the probabilities provided are assumed to be that of the
+        positive class. The labels in ``y_pred`` are assumed to be
+        ordered alphabetically, as done by
+        :class:`~sklearn.preprocessing.LabelBinarizer`.
+
+    sample_weight : array-like of shape (n_samples,), default=None
+        Sample weights.
+
+    labels : array-like, default=None
+        If not provided, labels will be inferred from y_true. If ``labels``
+        is ``None`` and ``y_pred`` has shape (n_samples,) the labels are
+        assumed to be binary and are inferred from ``y_true``.
+
+    Returns
+    -------
+    d2 : float or ndarray of floats
+        The D^2 score.
+
+    Notes
+    -----
+    This is not a symmetric function.
+
+    Like R^2, D^2 score may be negative (it need not actually be the square of
+    a quantity D).
+
+    This metric is not well-defined for a single sample and will return a NaN
+    value if n_samples is less than two.
     """
-    y_true = column_or_1d(y_true)
-    y_prob = column_or_1d(y_prob)
-    assert_all_finite(y_true)
-    assert_all_finite(y_prob)
-    check_consistent_length(y_true, y_prob, sample_weight)
+    y_pred = check_array(y_pred, ensure_2d=False, dtype="numeric")
+    check_consistent_length(y_pred, y_true, sample_weight)
+    if _num_samples(y_pred) < 2:
+        msg = "D^2 score is not well-defined with less than two samples."
+        warnings.warn(msg, UndefinedMetricWarning)
+        return float("nan")
+
+    # log loss of the fitted model
+    numerator = log_loss(
+        y_true=y_true,
+        y_pred=y_pred,
+        normalize=False,
+        sample_weight=sample_weight,
+        labels=labels,
+    )
 
-    y_type = type_of_target(y_true)
-    if y_type != "binary":
-        raise ValueError(
-            f"Only binary classification is supported. The type of the target "
-            f"is {y_type}."
+    # Proportion of labels in the dataset
+    weights = _check_sample_weight(sample_weight, y_true)
+
+    # If labels is passed, augment y_true to ensure that all labels are represented
+    # Use 0 weight for the new samples to not affect the counts
+    y_true_, weights_ = (
+        (
+            np.concatenate([y_true, labels]),
+            np.concatenate([weights, np.zeros_like(weights, shape=len(labels))]),
         )
+        if labels is not None
+        else (y_true, weights)
+    )
 
-    if y_prob.max() > 1:
-        raise ValueError("y_prob contains values greater than 1.")
-    if y_prob.min() < 0:
-        raise ValueError("y_prob contains values less than 0.")
+    _, y_value_indices = np.unique(y_true_, return_inverse=True)
+    counts = np.bincount(y_value_indices, weights=weights_)
+    y_prob = counts / weights.sum()
+    y_pred_null = np.tile(y_prob, (len(y_true), 1))
+
+    # log loss of the null model
+    denominator = log_loss(
+        y_true=y_true,
+        y_pred=y_pred_null,
+        normalize=False,
+        sample_weight=sample_weight,
+        labels=labels,
+    )
 
-    try:
-        pos_label = _check_pos_label_consistency(pos_label, y_true)
-    except ValueError:
-        classes = np.unique(y_true)
-        if classes.dtype.kind not in ('O', 'U', 'S'):
-            # for backward compatibility, if classes are not string then
-            # `pos_label` will correspond to the greater label
-            pos_label = classes[-1]
-        else:
-            raise
-    y_true = np.array(y_true == pos_label, int)
-    return np.average((y_true - y_prob) ** 2, weights=sample_weight)
+    return float(1 - (numerator / denominator))
diff --git a/sklearn/metrics/_dist_metrics.pxd.tp b/sklearn/metrics/_dist_metrics.pxd.tp
new file mode 100644
index 0000000000000..313225088c776
--- /dev/null
+++ b/sklearn/metrics/_dist_metrics.pxd.tp
@@ -0,0 +1,152 @@
+{{py:
+
+implementation_specific_values = [
+    # Values are the following ones:
+    #
+    # name_suffix, INPUT_DTYPE_t, INPUT_DTYPE
+    ('64', 'float64_t', 'np.float64'),
+    ('32', 'float32_t', 'np.float32')
+]
+
+}}
+from libc.math cimport sqrt, exp
+
+from ..utils._typedefs cimport float64_t, float32_t, int32_t, intp_t
+
+cdef class DistanceMetric:
+    pass
+
+{{for name_suffix, INPUT_DTYPE_t, INPUT_DTYPE in implementation_specific_values}}
+
+######################################################################
+# Inline distance functions
+#
+#  We use these for the default (euclidean) case so that they can be
+#  inlined.  This leads to faster computation for the most common case
+cdef inline float64_t euclidean_dist{{name_suffix}}(
+    const {{INPUT_DTYPE_t}}* x1,
+    const {{INPUT_DTYPE_t}}* x2,
+    intp_t size,
+) except -1 nogil:
+    cdef float64_t tmp, d=0
+    cdef intp_t j
+    for j in range(size):
+        tmp =  (x1[j] - x2[j])
+        d += tmp * tmp
+    return sqrt(d)
+
+
+cdef inline float64_t euclidean_rdist{{name_suffix}}(
+    const {{INPUT_DTYPE_t}}* x1,
+    const {{INPUT_DTYPE_t}}* x2,
+    intp_t size,
+) except -1 nogil:
+    cdef float64_t tmp, d=0
+    cdef intp_t j
+    for j in range(size):
+        tmp = (x1[j] - x2[j])
+        d += tmp * tmp
+    return d
+
+
+cdef inline float64_t euclidean_dist_to_rdist{{name_suffix}}(const {{INPUT_DTYPE_t}} dist) except -1 nogil:
+    return dist * dist
+
+
+cdef inline float64_t euclidean_rdist_to_dist{{name_suffix}}(const {{INPUT_DTYPE_t}} dist) except -1 nogil:
+    return sqrt(dist)
+
+
+######################################################################
+# DistanceMetric{{name_suffix}} base class
+cdef class DistanceMetric{{name_suffix}}(DistanceMetric):
+    # The following attributes are required for a few of the subclasses.
+    # we must define them here so that cython's limited polymorphism will work.
+    # Because we don't expect to instantiate a lot of these objects, the
+    # extra memory overhead of this setup should not be an issue.
+    cdef float64_t p
+    cdef const float64_t[::1] vec
+    cdef const float64_t[:, ::1] mat
+    cdef intp_t size
+    cdef object func
+    cdef object kwargs
+
+    cdef {{INPUT_DTYPE_t}} dist(
+        self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil
+
+    cdef {{INPUT_DTYPE_t}} rdist(
+        self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil
+
+    cdef {{INPUT_DTYPE_t}} dist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil
+
+    cdef {{INPUT_DTYPE_t}} rdist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil
+
+    cdef int pdist(
+        self,
+        const {{INPUT_DTYPE_t}}[:, ::1] X,
+        {{INPUT_DTYPE_t}}[:, ::1] D,
+    ) except -1
+
+    cdef int cdist(
+        self,
+        const {{INPUT_DTYPE_t}}[:, ::1] X,
+        const {{INPUT_DTYPE_t}}[:, ::1] Y,
+        {{INPUT_DTYPE_t}}[:, ::1] D,
+    ) except -1
+
+    cdef int pdist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t[::1] x1_indices,
+        const int32_t[::1] x1_indptr,
+        const intp_t size,
+        {{INPUT_DTYPE_t}}[:, ::1] D,
+    ) except -1 nogil
+
+    cdef int cdist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t[::1] x1_indices,
+        const int32_t[::1] x1_indptr,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t[::1] x2_indices,
+        const int32_t[::1] x2_indptr,
+        const intp_t size,
+        {{INPUT_DTYPE_t}}[:, ::1] D,
+    ) except -1 nogil
+
+    cdef {{INPUT_DTYPE_t}} _rdist_to_dist(self, {{INPUT_DTYPE_t}} rdist) except -1 nogil
+
+    cdef {{INPUT_DTYPE_t}} _dist_to_rdist(self, {{INPUT_DTYPE_t}} dist) except -1 nogil
+
+{{endfor}}
diff --git a/sklearn/metrics/_dist_metrics.pyx.tp b/sklearn/metrics/_dist_metrics.pyx.tp
new file mode 100644
index 0000000000000..b7d3d1f4d86a6
--- /dev/null
+++ b/sklearn/metrics/_dist_metrics.pyx.tp
@@ -0,0 +1,2811 @@
+{{py:
+
+implementation_specific_values = [
+    # Values are the following ones:
+    #
+    # name_suffix, INPUT_DTYPE_t, INPUT_DTYPE
+    ('64', 'float64_t', 'np.float64'),
+    ('32', 'float32_t', 'np.float32')
+]
+
+}}
+# By Jake Vanderplas (2013) 
+# written for the scikit-learn project
+# SPDX-License-Identifier: BSD-3-Clause
+
+import numpy as np
+cimport numpy as cnp
+
+cnp.import_array()  # required in order to use C-API
+
+from libc.math cimport fabs, sqrt, exp, pow, cos, sin, asin
+
+from scipy.sparse import csr_matrix, issparse
+from ..utils._typedefs cimport float64_t, float32_t, int32_t, intp_t
+from ..utils import check_array
+from ..utils.fixes import parse_version, sp_base_version
+
+cdef inline double fmax(double a, double b) noexcept nogil:
+    return max(a, b)
+
+
+######################################################################
+# newObj function
+#  this is a helper function for pickling
+def newObj(obj):
+    return obj.__new__(obj)
+
+
+BOOL_METRICS = [
+    "hamming",
+    "jaccard",
+    "dice",
+    "rogerstanimoto",
+    "russellrao",
+    "sokalsneath",
+]
+DEPRECATED_METRICS = []
+if sp_base_version < parse_version("1.17"):
+    # Deprecated in SciPy 1.15 and removed in SciPy 1.17
+    BOOL_METRICS += ["sokalmichener"]
+if sp_base_version >= parse_version("1.15"):
+    DEPRECATED_METRICS.append("sokalmichener")
+if sp_base_version < parse_version("1.11"):
+    # Deprecated in SciPy 1.9 and removed in SciPy 1.11
+    BOOL_METRICS += ["kulsinski"]
+if sp_base_version >= parse_version("1.9"):
+    DEPRECATED_METRICS.append("kulsinski")
+if sp_base_version < parse_version("1.9"):
+    # Deprecated in SciPy 1.0 and removed in SciPy 1.9
+    BOOL_METRICS += ["matching"]
+if sp_base_version >= parse_version("1.0"):
+    DEPRECATED_METRICS.append("matching")
+
+def get_valid_metric_ids(L):
+    """Given an iterable of metric class names or class identifiers,
+    return a list of metric IDs which map to those classes.
+
+    Example:
+    >>> L = get_valid_metric_ids([EuclideanDistance, 'ManhattanDistance'])
+    >>> sorted(L)
+    ['cityblock', 'euclidean', 'l1', 'l2', 'manhattan']
+    """
+    return [key for (key, val) in METRIC_MAPPING64.items()
+            if (val.__name__ in L) or (val in L)]
+
+cdef class DistanceMetric:
+    """Uniform interface for fast distance metric functions.
+
+    The `DistanceMetric` class provides a convenient way to compute pairwise distances
+    between samples. It supports various distance metrics, such as Euclidean distance,
+    Manhattan distance, and more.
+
+    The `pairwise` method can be used to compute pairwise distances between samples in
+    the input arrays. It returns a distance matrix representing the distances between
+    all pairs of samples.
+
+    The :meth:`get_metric` method allows you to retrieve a specific metric using its
+    string identifier.
+
+    Examples
+    --------
+    >>> from sklearn.metrics import DistanceMetric
+    >>> dist = DistanceMetric.get_metric('euclidean')
+    >>> X = [[1, 2], [3, 4], [5, 6]]
+    >>> Y = [[7, 8], [9, 10]]
+    >>> dist.pairwise(X,Y)
+    array([[7.81..., 10.63...]
+           [5.65...,  8.48...]
+           [1.41...,  4.24...]])
+
+    .. rubric:: Available Metrics
+
+    The following lists the string metric identifiers and the associated
+    distance metric classes:
+
+    **Metrics intended for real-valued vector spaces:**
+
+    ==============  ====================  ========  ===============================
+    identifier      class name            args      distance function
+    --------------  --------------------  --------  -------------------------------
+    "euclidean"     EuclideanDistance     -         ``sqrt(sum((x - y)^2))``
+    "manhattan"     ManhattanDistance     -         ``sum(|x - y|)``
+    "chebyshev"     ChebyshevDistance     -         ``max(|x - y|)``
+    "minkowski"     MinkowskiDistance     p, w      ``sum(w * |x - y|^p)^(1/p)``
+    "seuclidean"    SEuclideanDistance    V         ``sqrt(sum((x - y)^2 / V))``
+    "mahalanobis"   MahalanobisDistance   V or VI   ``sqrt((x - y)' V^-1 (x - y))``
+    ==============  ====================  ========  ===============================
+
+    **Metrics intended for two-dimensional vector spaces:**  Note that the haversine
+    distance metric requires data in the form of [latitude, longitude] and both
+    inputs and outputs are in units of radians.
+
+    ============  ==================  ===============================================================
+    identifier    class name          distance function
+    ------------  ------------------  ---------------------------------------------------------------
+    "haversine"   HaversineDistance   ``2 arcsin(sqrt(sin^2(0.5*dx) + cos(x1)cos(x2)sin^2(0.5*dy)))``
+    ============  ==================  ===============================================================
+
+
+    **Metrics intended for integer-valued vector spaces:**  Though intended
+    for integer-valued vectors, these are also valid metrics in the case of
+    real-valued vectors.
+
+    =============  ====================  ========================================
+    identifier     class name            distance function
+    -------------  --------------------  ----------------------------------------
+    "hamming"      HammingDistance       ``N_unequal(x, y) / N_tot``
+    "canberra"     CanberraDistance      ``sum(|x - y| / (|x| + |y|))``
+    "braycurtis"   BrayCurtisDistance    ``sum(|x - y|) / (sum(|x|) + sum(|y|))``
+    =============  ====================  ========================================
+
+    **Metrics intended for boolean-valued vector spaces:**  Any nonzero entry
+    is evaluated to "True".  In the listings below, the following
+    abbreviations are used:
+
+    - N: number of dimensions
+    - NTT: number of dims in which both values are True
+    - NTF: number of dims in which the first value is True, second is False
+    - NFT: number of dims in which the first value is False, second is True
+    - NFF: number of dims in which both values are False
+    - NNEQ: number of non-equal dimensions, NNEQ = NTF + NFT
+    - NNZ: number of nonzero dimensions, NNZ = NTF + NFT + NTT
+
+    =================  =======================  ===============================
+    identifier         class name               distance function
+    -----------------  -----------------------  -------------------------------
+    "jaccard"          JaccardDistance          NNEQ / NNZ
+    "matching"         MatchingDistance         NNEQ / N
+    "dice"             DiceDistance             NNEQ / (NTT + NNZ)
+    "kulsinski"        KulsinskiDistance        (NNEQ + N - NTT) / (NNEQ + N)
+    "rogerstanimoto"   RogersTanimotoDistance   2 * NNEQ / (N + NNEQ)
+    "russellrao"       RussellRaoDistance       (N - NTT) / N
+    "sokalmichener"    SokalMichenerDistance    2 * NNEQ / (N + NNEQ)
+    "sokalsneath"      SokalSneathDistance      NNEQ / (NNEQ + 0.5 * NTT)
+    =================  =======================  ===============================
+
+    **User-defined distance:**
+
+    ===========    ===============    =======
+    identifier     class name         args
+    -----------    ---------------    -------
+    "pyfunc"       PyFuncDistance     func
+    ===========    ===============    =======
+
+    Here ``func`` is a function which takes two one-dimensional numpy
+    arrays, and returns a distance.  Note that in order to be used within
+    the BallTree, the distance must be a true metric:
+    i.e. it must satisfy the following properties
+
+    1) Non-negativity: d(x, y) >= 0
+    2) Identity: d(x, y) = 0 if and only if x == y
+    3) Symmetry: d(x, y) = d(y, x)
+    4) Triangle Inequality: d(x, y) + d(y, z) >= d(x, z)
+
+    Because of the Python object overhead involved in calling the python
+    function, this will be fairly slow, but it will have the same
+    scaling as other distances.
+    """
+    @classmethod
+    def get_metric(cls, metric, dtype=np.float64, **kwargs):
+        """Get the given distance metric from the string identifier.
+
+        See the docstring of DistanceMetric for a list of available metrics.
+
+        Parameters
+        ----------
+        metric : str or class name
+            The string identifier or class name of the desired distance metric.
+            See the documentation of the `DistanceMetric` class for a list of
+            available metrics.
+
+        dtype : {np.float32, np.float64}, default=np.float64
+            The data type of the input on which the metric will be applied.
+            This affects the precision of the computed distances.
+            By default, it is set to `np.float64`.
+
+        **kwargs
+            Additional keyword arguments that will be passed to the requested metric.
+            These arguments can be used to customize the behavior of the specific
+            metric.
+
+        Returns
+        -------
+        metric_obj : instance of the requested metric
+            An instance of the requested distance metric class.
+        """
+        if dtype == np.float32:
+            specialized_class = DistanceMetric32
+        elif dtype == np.float64:
+            specialized_class = DistanceMetric64
+        else:
+            raise ValueError(
+                f"Unexpected dtype {dtype} provided. Please select a dtype from"
+                " {np.float32, np.float64}"
+            )
+
+        return specialized_class.get_metric(metric, **kwargs)
+
+{{for name_suffix, INPUT_DTYPE_t, INPUT_DTYPE in implementation_specific_values}}
+
+######################################################################
+# metric mappings
+#  These map from metric id strings to class names
+METRIC_MAPPING{{name_suffix}} = {
+    'euclidean': EuclideanDistance{{name_suffix}},
+    'l2': EuclideanDistance{{name_suffix}},
+    'minkowski': MinkowskiDistance{{name_suffix}},
+    'p': MinkowskiDistance{{name_suffix}},
+    'manhattan': ManhattanDistance{{name_suffix}},
+    'cityblock': ManhattanDistance{{name_suffix}},
+    'l1': ManhattanDistance{{name_suffix}},
+    'chebyshev': ChebyshevDistance{{name_suffix}},
+    'infinity': ChebyshevDistance{{name_suffix}},
+    'seuclidean': SEuclideanDistance{{name_suffix}},
+    'mahalanobis': MahalanobisDistance{{name_suffix}},
+    'hamming': HammingDistance{{name_suffix}},
+    'canberra': CanberraDistance{{name_suffix}},
+    'braycurtis': BrayCurtisDistance{{name_suffix}},
+    'matching': MatchingDistance{{name_suffix}},
+    'jaccard': JaccardDistance{{name_suffix}},
+    'dice': DiceDistance{{name_suffix}},
+    'kulsinski': KulsinskiDistance{{name_suffix}},
+    'rogerstanimoto': RogersTanimotoDistance{{name_suffix}},
+    'russellrao': RussellRaoDistance{{name_suffix}},
+    'sokalmichener': SokalMichenerDistance{{name_suffix}},
+    'sokalsneath': SokalSneathDistance{{name_suffix}},
+    'haversine': HaversineDistance{{name_suffix}},
+    'pyfunc': PyFuncDistance{{name_suffix}},
+}
+
+cdef inline object _buffer_to_ndarray{{name_suffix}}(const {{INPUT_DTYPE_t}}* x, intp_t n):
+    # Wrap a memory buffer with an ndarray. Warning: this is not robust.
+    # In particular, if x is deallocated before the returned array goes
+    # out of scope, this could cause memory errors.  Since there is not
+    # a possibility of this for our use-case, this should be safe.
+
+    # Note: this Segfaults unless np.import_array() is called above
+    # TODO: remove the explicit cast to cnp.intp_t* when cython min version >= 3.0
+    return cnp.PyArray_SimpleNewFromData(1, &n, cnp.NPY_FLOAT64, x)
+
+
+cdef {{INPUT_DTYPE_t}} INF{{name_suffix}} = np.inf
+
+
+######################################################################
+# Distance Metric Classes
+cdef class DistanceMetric{{name_suffix}}(DistanceMetric):
+    """DistanceMetric class
+
+    This class provides a uniform interface to fast distance metric
+    functions.  The various metrics can be accessed via the :meth:`get_metric`
+    class method and the metric string identifier (see below).
+
+    Examples
+    --------
+    >>> from sklearn.metrics import DistanceMetric
+    >>> dist = DistanceMetric.get_metric('euclidean')
+    >>> X = [[0, 1, 2],
+             [3, 4, 5]]
+    >>> dist.pairwise(X)
+    array([[ 0.        ,  5.19615242],
+           [ 5.19615242,  0.        ]])
+
+    Available Metrics
+
+    The following lists the string metric identifiers and the associated
+    distance metric classes:
+
+    **Metrics intended for real-valued vector spaces:**
+
+    ==============  ====================  ========  ===============================
+    identifier      class name            args      distance function
+    --------------  --------------------  --------  -------------------------------
+    "euclidean"     EuclideanDistance     -         ``sqrt(sum((x - y)^2))``
+    "manhattan"     ManhattanDistance     -         ``sum(|x - y|)``
+    "chebyshev"     ChebyshevDistance     -         ``max(|x - y|)``
+    "minkowski"     MinkowskiDistance     p, w      ``sum(w * |x - y|^p)^(1/p)``
+    "seuclidean"    SEuclideanDistance    V         ``sqrt(sum((x - y)^2 / V))``
+    "mahalanobis"   MahalanobisDistance   V or VI   ``sqrt((x - y)' V^-1 (x - y))``
+    ==============  ====================  ========  ===============================
+
+    **Metrics intended for two-dimensional vector spaces:**  Note that the haversine
+    distance metric requires data in the form of [latitude, longitude] and both
+    inputs and outputs are in units of radians.
+
+    ============  ==================  ===============================================================
+    identifier    class name          distance function
+    ------------  ------------------  ---------------------------------------------------------------
+    "haversine"   HaversineDistance   ``2 arcsin(sqrt(sin^2(0.5*dx) + cos(x1)cos(x2)sin^2(0.5*dy)))``
+    ============  ==================  ===============================================================
+
+
+    **Metrics intended for integer-valued vector spaces:**  Though intended
+    for integer-valued vectors, these are also valid metrics in the case of
+    real-valued vectors.
+
+    =============  ====================  ========================================
+    identifier     class name            distance function
+    -------------  --------------------  ----------------------------------------
+    "hamming"      HammingDistance       ``N_unequal(x, y) / N_tot``
+    "canberra"     CanberraDistance      ``sum(|x - y| / (|x| + |y|))``
+    "braycurtis"   BrayCurtisDistance    ``sum(|x - y|) / (sum(|x|) + sum(|y|))``
+    =============  ====================  ========================================
+
+    **Metrics intended for boolean-valued vector spaces:**  Any nonzero entry
+    is evaluated to "True".  In the listings below, the following
+    abbreviations are used:
+
+    - N: number of dimensions
+    - NTT: number of dims in which both values are True
+    - NTF: number of dims in which the first value is True, second is False
+    - NFT: number of dims in which the first value is False, second is True
+    - NFF: number of dims in which both values are False
+    - NNEQ: number of non-equal dimensions, NNEQ = NTF + NFT
+    - NNZ: number of nonzero dimensions, NNZ = NTF + NFT + NTT
+
+    =================  =======================  ===============================
+    identifier         class name               distance function
+    -----------------  -----------------------  -------------------------------
+    "jaccard"          JaccardDistance          NNEQ / NNZ
+    "matching"         MatchingDistance         NNEQ / N
+    "dice"             DiceDistance             NNEQ / (NTT + NNZ)
+    "kulsinski"        KulsinskiDistance        (NNEQ + N - NTT) / (NNEQ + N)
+    "rogerstanimoto"   RogersTanimotoDistance   2 * NNEQ / (N + NNEQ)
+    "russellrao"       RussellRaoDistance       (N - NTT) / N
+    "sokalmichener"    SokalMichenerDistance    2 * NNEQ / (N + NNEQ)
+    "sokalsneath"      SokalSneathDistance      NNEQ / (NNEQ + 0.5 * NTT)
+    =================  =======================  ===============================
+
+    **User-defined distance:**
+
+    ===========    ===============    =======
+    identifier     class name         args
+    -----------    ---------------    -------
+    "pyfunc"       PyFuncDistance     func
+    ===========    ===============    =======
+
+    Here ``func`` is a function which takes two one-dimensional numpy
+    arrays, and returns a distance.  Note that in order to be used within
+    the BallTree, the distance must be a true metric:
+    i.e. it must satisfy the following properties
+
+    1) Non-negativity: d(x, y) >= 0
+    2) Identity: d(x, y) = 0 if and only if x == y
+    3) Symmetry: d(x, y) = d(y, x)
+    4) Triangle Inequality: d(x, y) + d(y, z) >= d(x, z)
+
+    Because of the Python object overhead involved in calling the python
+    function, this will be fairly slow, but it will have the same
+    scaling as other distances.
+    """
+    def __cinit__(self):
+        self.p = 2
+        self.vec = np.zeros(1, dtype=np.float64, order='C')
+        self.mat = np.zeros((1, 1), dtype=np.float64, order='C')
+        self.size = 1
+
+    def __reduce__(self):
+        """
+        reduce method used for pickling
+        """
+        return (newObj, (self.__class__,), self.__getstate__())
+
+    def __getstate__(self):
+        """
+        get state for pickling
+        """
+        if self.__class__.__name__ == "PyFuncDistance{{name_suffix}}":
+            return (float(self.p), np.asarray(self.vec), np.asarray(self.mat), self.func, self.kwargs)
+        return (float(self.p), np.asarray(self.vec), np.asarray(self.mat))
+
+    def __setstate__(self, state):
+        """
+        set state for pickling
+        """
+        self.p = state[0]
+        self.vec = state[1]
+        self.mat = state[2]
+        if self.__class__.__name__ == "PyFuncDistance{{name_suffix}}":
+            self.func = state[3]
+            self.kwargs = state[4]
+        self.size = self.vec.shape[0]
+
+    @classmethod
+    def get_metric(cls, metric, **kwargs):
+        """Get the given distance metric from the string identifier.
+
+        See the docstring of DistanceMetric for a list of available metrics.
+
+        Parameters
+        ----------
+        metric : str or class name
+            The distance metric to use
+        **kwargs
+            additional arguments will be passed to the requested metric
+        """
+        if isinstance(metric, DistanceMetric{{name_suffix}}):
+            return metric
+
+        if callable(metric):
+            return PyFuncDistance{{name_suffix}}(metric, **kwargs)
+
+        # Map the metric string ID to the metric class
+        if isinstance(metric, type) and issubclass(metric, DistanceMetric{{name_suffix}}):
+            pass
+        else:
+            try:
+                metric = METRIC_MAPPING{{name_suffix}}[metric]
+            except:
+                raise ValueError("Unrecognized metric '%s'" % metric)
+
+        # In Minkowski special cases, return more efficient methods
+        if metric is MinkowskiDistance{{name_suffix}}:
+            p = kwargs.pop('p', 2)
+            w = kwargs.pop('w', None)
+            if p == 1 and w is None:
+                return ManhattanDistance{{name_suffix}}(**kwargs)
+            elif p == 2 and w is None:
+                return EuclideanDistance{{name_suffix}}(**kwargs)
+            elif np.isinf(p) and w is None:
+                return ChebyshevDistance{{name_suffix}}(**kwargs)
+            else:
+                return MinkowskiDistance{{name_suffix}}(p, w, **kwargs)
+        else:
+            return metric(**kwargs)
+
+    def __init__(self):
+        if self.__class__ is DistanceMetric{{name_suffix}}:
+            raise NotImplementedError("DistanceMetric{{name_suffix}} is an abstract class")
+
+    def _validate_data(self, X):
+        """Validate the input data.
+
+        This should be overridden in a base class if a specific input format
+        is required.
+        """
+        return
+
+    cdef {{INPUT_DTYPE_t}} dist(
+        self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil:
+        """Compute the distance between vectors x1 and x2
+
+        This should be overridden in a base class.
+        """
+        return -999
+
+    cdef {{INPUT_DTYPE_t}} rdist(
+        self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil:
+        """Compute the rank-preserving surrogate distance between vectors x1 and x2.
+
+        This can optionally be overridden in a base class.
+
+        The rank-preserving surrogate distance is any measure that yields the same
+        rank as the distance, but is more efficient to compute. For example, the
+        rank-preserving surrogate distance of the Euclidean metric is the
+        squared-euclidean distance.
+        """
+        return self.dist(x1, x2, size)
+
+    cdef int pdist(
+        self,
+        const {{INPUT_DTYPE_t}}[:, ::1] X,
+        {{INPUT_DTYPE_t}}[:, ::1] D,
+    ) except -1:
+        """Compute the pairwise distances between points in X"""
+        cdef intp_t i1, i2
+        for i1 in range(X.shape[0]):
+            for i2 in range(i1, X.shape[0]):
+                D[i1, i2] = self.dist(&X[i1, 0], &X[i2, 0], X.shape[1])
+                D[i2, i1] = D[i1, i2]
+        return 0
+
+
+    cdef int cdist(
+        self,
+        const {{INPUT_DTYPE_t}}[:, ::1] X,
+        const {{INPUT_DTYPE_t}}[:, ::1] Y,
+        {{INPUT_DTYPE_t}}[:, ::1] D,
+    ) except -1:
+        """Compute the cross-pairwise distances between arrays X and Y"""
+        cdef intp_t i1, i2
+        if X.shape[1] != Y.shape[1]:
+            raise ValueError('X and Y must have the same second dimension')
+        for i1 in range(X.shape[0]):
+            for i2 in range(Y.shape[0]):
+                D[i1, i2] = self.dist(&X[i1, 0], &Y[i2, 0], X.shape[1])
+        return 0
+
+    cdef {{INPUT_DTYPE_t}} dist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil:
+        """Compute the distance between vectors x1 and x2 represented
+        under the CSR format.
+
+        This must be overridden in a subclass.
+
+        Notes
+        -----
+        0. The implementation of this method in subclasses must be robust to the
+        presence of explicit zeros in the CSR representation.
+
+        1. The `data` arrays are passed using pointers to be able to support an
+        alternative representation of the CSR data structure for supporting
+        fused sparse-dense datasets pairs with minimum overhead.
+
+        See the explanations in `SparseDenseDatasetsPair.__init__`.
+
+        2. An alternative signature would be:
+
+            cdef {{INPUT_DTYPE_t}} dist_csr(
+                self,
+                const {{INPUT_DTYPE_t}}* x1_data,
+                const int32_t* x1_indices,
+                const {{INPUT_DTYPE_t}}* x2_data,
+                const int32_t* x2_indices,
+            ) except -1 nogil:
+
+        Where callers would use slicing on the original CSR data and indices
+        memoryviews:
+
+            x1_start = X1_csr.indices_ptr[i]
+            x1_end   = X1_csr.indices_ptr[i+1]
+            x2_start = X2_csr.indices_ptr[j]
+            x2_end   = X2_csr.indices_ptr[j+1]
+
+            self.dist_csr(
+                &x1_data[x1_start],
+                x1_indices[x1_start:x1_end],
+                &x2_data[x2_start],
+                x2_indices[x2_start:x2_end],
+            )
+
+        Yet, slicing on memoryview slows down execution as it takes the GIL.
+        See: https://github.com/scikit-learn/scikit-learn/issues/17299
+
+        Hence, to avoid slicing the data and indices arrays of the sparse
+        matrices containing respectively x1 and x2 (namely x{1,2}_{data,indices})
+        are passed as well as their indices pointers (namely x{1,2}_{start,end}).
+
+        3. For reference about the CSR format, see section 3.4 of
+        Saad, Y. (2003), Iterative Methods for Sparse Linear Systems, SIAM.
+        https://www-users.cse.umn.edu/~saad/IterMethBook_2ndEd.pdf
+        """
+        return -999
+
+    cdef {{INPUT_DTYPE_t}} rdist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil:
+        """Distance between rows of CSR matrices x1 and x2.
+
+        This can optionally be overridden in a subclass.
+
+        The rank-preserving surrogate distance is any measure that yields the same
+        rank as the distance, but is more efficient to compute. For example, the
+        rank-preserving surrogate distance of the Euclidean metric is the
+        squared-euclidean distance.
+
+        Notes
+        -----
+        The implementation of this method in subclasses must be robust to the
+        presence of explicit zeros in the CSR representation.
+
+        More information about the motives for this method signature is given
+        in the docstring of dist_csr.
+        """
+        return self.dist_csr(
+            x1_data,
+            x1_indices,
+            x2_data,
+            x2_indices,
+            x1_start,
+            x1_end,
+            x2_start,
+            x2_end,
+            size,
+        )
+
+    cdef int pdist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t[::1] x1_indices,
+        const int32_t[::1] x1_indptr,
+        const intp_t size,
+        {{INPUT_DTYPE_t}}[:, ::1] D,
+    ) except -1 nogil:
+        """Pairwise distances between rows in CSR matrix X.
+
+        Note that this implementation is twice faster than cdist_csr(X, X)
+        because it leverages the symmetry of the problem.
+        """
+        cdef:
+            intp_t i1, i2
+            intp_t n_x1 = x1_indptr.shape[0] - 1
+            intp_t x1_start, x1_end, x2_start, x2_end
+
+        for i1 in range(n_x1):
+            x1_start = x1_indptr[i1]
+            x1_end = x1_indptr[i1 + 1]
+            for i2 in range(i1, n_x1):
+                x2_start = x1_indptr[i2]
+                x2_end = x1_indptr[i2 + 1]
+                D[i1, i2] = D[i2, i1] = self.dist_csr(
+                    x1_data,
+                    &x1_indices[0],
+                    x1_data,
+                    &x1_indices[0],
+                    x1_start,
+                    x1_end,
+                    x2_start,
+                    x2_end,
+                    size,
+                )
+        return 0
+
+    cdef int cdist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t[::1] x1_indices,
+        const int32_t[::1] x1_indptr,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t[::1] x2_indices,
+        const int32_t[::1] x2_indptr,
+        const intp_t size,
+        {{INPUT_DTYPE_t}}[:, ::1] D,
+    ) except -1 nogil:
+        """Compute the cross-pairwise distances between arrays X and Y
+        represented in the CSR format."""
+        cdef:
+            intp_t i1, i2
+            intp_t n_x1 = x1_indptr.shape[0] - 1
+            intp_t n_x2 = x2_indptr.shape[0] - 1
+            intp_t x1_start, x1_end, x2_start, x2_end
+
+        for i1 in range(n_x1):
+            x1_start = x1_indptr[i1]
+            x1_end = x1_indptr[i1 + 1]
+            for i2 in range(n_x2):
+                x2_start = x2_indptr[i2]
+                x2_end = x2_indptr[i2 + 1]
+
+                D[i1, i2] = self.dist_csr(
+                    x1_data,
+                    &x1_indices[0],
+                    x2_data,
+                    &x2_indices[0],
+                    x1_start,
+                    x1_end,
+                    x2_start,
+                    x2_end,
+                    size,
+                )
+        return 0
+
+    cdef {{INPUT_DTYPE_t}} _rdist_to_dist(self, {{INPUT_DTYPE_t}} rdist) except -1 nogil:
+        """Convert the rank-preserving surrogate distance to the distance"""
+        return rdist
+
+    cdef {{INPUT_DTYPE_t}} _dist_to_rdist(self, {{INPUT_DTYPE_t}} dist) except -1 nogil:
+        """Convert the distance to the rank-preserving surrogate distance"""
+        return dist
+
+    def rdist_to_dist(self, rdist):
+        """Convert the rank-preserving surrogate distance to the distance.
+
+        The surrogate distance is any measure that yields the same rank as the
+        distance, but is more efficient to compute. For example, the
+        rank-preserving surrogate distance of the Euclidean metric is the
+        squared-euclidean distance.
+
+        Parameters
+        ----------
+        rdist : double
+            Surrogate distance.
+
+        Returns
+        -------
+        double
+            True distance.
+        """
+        return rdist
+
+    def dist_to_rdist(self, dist):
+        """Convert the true distance to the rank-preserving surrogate distance.
+
+        The surrogate distance is any measure that yields the same rank as the
+        distance, but is more efficient to compute. For example, the
+        rank-preserving surrogate distance of the Euclidean metric is the
+        squared-euclidean distance.
+
+        Parameters
+        ----------
+        dist : double
+            True distance.
+
+        Returns
+        -------
+        double
+            Surrogate distance.
+        """
+        return dist
+
+    def _pairwise_dense_dense(self, X, Y):
+        cdef const {{INPUT_DTYPE_t}}[:, ::1] Xarr
+        cdef const {{INPUT_DTYPE_t}}[:, ::1] Yarr
+        cdef {{INPUT_DTYPE_t}}[:, ::1] Darr
+
+        Xarr = np.asarray(X, dtype={{INPUT_DTYPE}}, order='C')
+        self._validate_data(Xarr)
+        if X is Y:
+            Darr = np.empty((Xarr.shape[0], Xarr.shape[0]), dtype={{INPUT_DTYPE}}, order='C')
+            self.pdist(Xarr, Darr)
+        else:
+            Yarr = np.asarray(Y, dtype={{INPUT_DTYPE}}, order='C')
+            self._validate_data(Yarr)
+            Darr = np.empty((Xarr.shape[0], Yarr.shape[0]), dtype={{INPUT_DTYPE}}, order='C')
+            self.cdist(Xarr, Yarr, Darr)
+        return np.asarray(Darr)
+
+    def _pairwise_sparse_sparse(self, X: csr_matrix , Y: csr_matrix):
+        cdef:
+            intp_t n_X, n_features
+            const {{INPUT_DTYPE_t}}[::1] X_data
+            const int32_t[::1] X_indices
+            const int32_t[::1] X_indptr
+
+            intp_t n_Y
+            const {{INPUT_DTYPE_t}}[::1] Y_data
+            const int32_t[::1] Y_indices
+            const int32_t[::1] Y_indptr
+
+            {{INPUT_DTYPE_t}}[:, ::1] Darr
+
+        X_csr = X.tocsr()
+        n_X, n_features = X_csr.shape
+        X_data = np.asarray(X_csr.data, dtype={{INPUT_DTYPE}})
+        X_indices = np.asarray(X_csr.indices, dtype=np.int32)
+        X_indptr = np.asarray(X_csr.indptr, dtype=np.int32)
+        if X is Y:
+            Darr = np.empty((n_X, n_X), dtype={{INPUT_DTYPE}}, order='C')
+            self.pdist_csr(
+                x1_data=&X_data[0],
+                x1_indices=X_indices,
+                x1_indptr=X_indptr,
+                size=n_features,
+                D=Darr,
+            )
+        else:
+            Y_csr = Y.tocsr()
+            n_Y, _ = Y_csr.shape
+            Y_data = np.asarray(Y_csr.data, dtype={{INPUT_DTYPE}})
+            Y_indices = np.asarray(Y_csr.indices, dtype=np.int32)
+            Y_indptr = np.asarray(Y_csr.indptr, dtype=np.int32)
+
+            Darr = np.empty((n_X, n_Y), dtype={{INPUT_DTYPE}}, order='C')
+            self.cdist_csr(
+                x1_data=&X_data[0],
+                x1_indices=X_indices,
+                x1_indptr=X_indptr,
+                x2_data=&Y_data[0],
+                x2_indices=Y_indices,
+                x2_indptr=Y_indptr,
+                size=n_features,
+                D=Darr,
+            )
+        return np.asarray(Darr)
+
+    def _pairwise_sparse_dense(self, X: csr_matrix, Y):
+        cdef:
+            intp_t n_X = X.shape[0]
+            intp_t n_features = X.shape[1]
+            const {{INPUT_DTYPE_t}}[::1] X_data = np.asarray(
+                X.data, dtype={{INPUT_DTYPE}},
+            )
+            const int32_t[::1] X_indices = np.asarray(
+                X.indices, dtype=np.int32,
+            )
+            const int32_t[::1] X_indptr = np.asarray(
+                X.indptr, dtype=np.int32,
+            )
+
+            const {{INPUT_DTYPE_t}}[:, ::1] Y_data = np.asarray(
+                Y, dtype={{INPUT_DTYPE}}, order="C",
+            )
+            intp_t n_Y = Y_data.shape[0]
+            const int32_t[::1] Y_indices = (
+                np.arange(n_features, dtype=np.int32)
+            )
+
+            {{INPUT_DTYPE_t}}[:, ::1] Darr = np.empty((n_X, n_Y), dtype={{INPUT_DTYPE}}, order='C')
+
+            intp_t i1, i2
+            intp_t x1_start, x1_end
+            {{INPUT_DTYPE_t}} * x2_data
+
+        with nogil:
+            # Use the exact same adaptation for CSR than in SparseDenseDatasetsPair
+            # for supporting the sparse-dense case with minimal overhead.
+            # Note: at this point this method is only a convenience method
+            # used in the tests via the DistanceMetric.pairwise method.
+            # Therefore, there is no need to attempt parallelization of those
+            # nested for-loops.
+            # Efficient parallel computation of pairwise distances can be
+            # achieved via the PairwiseDistances class instead. The latter
+            # internally calls into vector-wise distance computation from
+            # the DistanceMetric subclass while benefiting from the generic
+            # Cython/OpenMP parallelization template for the generic pairwise
+            # distance + reduction computational pattern.
+            for i1 in range(n_X):
+                x1_start = X_indptr[i1]
+                x1_end = X_indptr[i1 + 1]
+                for i2 in range(n_Y):
+                    x2_data = &Y_data[0, 0] + i2 * n_features
+
+                    Darr[i1, i2] = self.dist_csr(
+                        x1_data=&X_data[0],
+                        x1_indices=&X_indices[0],
+                        x2_data=x2_data,
+                        x2_indices=&Y_indices[0],
+                        x1_start=x1_start,
+                        x1_end=x1_end,
+                        x2_start=0,
+                        x2_end=n_features,
+                        size=n_features,
+                    )
+
+        return np.asarray(Darr)
+
+    def _pairwise_dense_sparse(self, X, Y: csr_matrix):
+        # We could have implemented this method using _pairwise_dense_sparse by
+        # swapping argument and by transposing the results, but this would
+        # have come with an extra copy to ensure C-contiguity of the result.
+        cdef:
+            intp_t n_X = X.shape[0]
+            intp_t n_features = X.shape[1]
+
+            const {{INPUT_DTYPE_t}}[:, ::1] X_data = np.asarray(
+                X, dtype={{INPUT_DTYPE}}, order="C",
+            )
+            const int32_t[::1] X_indices = np.arange(
+                n_features, dtype=np.int32,
+            )
+
+            intp_t n_Y = Y.shape[0]
+            const {{INPUT_DTYPE_t}}[::1] Y_data = np.asarray(
+                Y.data, dtype={{INPUT_DTYPE}},
+            )
+            const int32_t[::1] Y_indices = np.asarray(
+                Y.indices, dtype=np.int32,
+            )
+            const int32_t[::1] Y_indptr = np.asarray(
+                Y.indptr, dtype=np.int32,
+            )
+
+            {{INPUT_DTYPE_t}}[:, ::1] Darr = np.empty((n_X, n_Y), dtype={{INPUT_DTYPE}}, order='C')
+
+            intp_t i1, i2
+            {{INPUT_DTYPE_t}} * x1_data
+
+            intp_t x2_start, x2_end
+
+        with nogil:
+            # Use the exact same adaptation for CSR than in SparseDenseDatasetsPair
+            # for supporting the dense-sparse case with minimal overhead.
+            # Note: at this point this method is only a convenience method
+            # used in the tests via the DistanceMetric.pairwise method.
+            # Therefore, there is no need to attempt parallelization of those
+            # nested for-loops.
+            # Efficient parallel computation of pairwise distances can be
+            # achieved via the PairwiseDistances class instead. The latter
+            # internally calls into vector-wise distance computation from
+            # the DistanceMetric subclass while benefiting from the generic
+            # Cython/OpenMP parallelization template for the generic pairwise
+            # distance + reduction computational pattern.
+            for i1 in range(n_X):
+                x1_data = &X_data[0, 0] + i1 * n_features
+                for i2 in range(n_Y):
+                    x2_start = Y_indptr[i2]
+                    x2_end = Y_indptr[i2 + 1]
+
+                    Darr[i1, i2] = self.dist_csr(
+                        x1_data=x1_data,
+                        x1_indices=&X_indices[0],
+                        x2_data=&Y_data[0],
+                        x2_indices=&Y_indices[0],
+                        x1_start=0,
+                        x1_end=n_features,
+                        x2_start=x2_start,
+                        x2_end=x2_end,
+                        size=n_features,
+                    )
+
+        return np.asarray(Darr)
+
+
+    def pairwise(self, X, Y=None):
+        """Compute the pairwise distances between X and Y
+
+        This is a convenience routine for the sake of testing.  For many
+        metrics, the utilities in scipy.spatial.distance.cdist and
+        scipy.spatial.distance.pdist will be faster.
+
+        Parameters
+        ----------
+        X : ndarray or CSR matrix of shape (n_samples_X, n_features)
+            Input data.
+        Y : ndarray or CSR matrix of shape (n_samples_Y, n_features)
+            Input data.
+            If not specified, then Y=X.
+
+        Returns
+        -------
+        dist : ndarray of shape  (n_samples_X, n_samples_Y)
+            The distance matrix of pairwise distances between points in X and Y.
+        """
+        X = check_array(X, accept_sparse=['csr'])
+
+        if Y is None:
+            Y = X
+        else:
+            Y = check_array(Y, accept_sparse=['csr'])
+
+        X_is_sparse = issparse(X)
+        Y_is_sparse = issparse(Y)
+
+        if not X_is_sparse and not Y_is_sparse:
+            return self._pairwise_dense_dense(X, Y)
+
+        if X_is_sparse and Y_is_sparse:
+            return self._pairwise_sparse_sparse(X, Y)
+
+        if X_is_sparse and not Y_is_sparse:
+            return self._pairwise_sparse_dense(X, Y)
+
+        return self._pairwise_dense_sparse(X, Y)
+
+#------------------------------------------------------------
+# Euclidean Distance
+#  d = sqrt(sum(x_i^2 - y_i^2))
+cdef class EuclideanDistance{{name_suffix}}(DistanceMetric{{name_suffix}}):
+    r"""Euclidean Distance metric
+
+    .. math::
+       D(x, y) = \sqrt{ \sum_i (x_i - y_i) ^ 2 }
+    """
+    def __init__(self):
+        self.p = 2
+
+    cdef inline {{INPUT_DTYPE_t}} dist(self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil:
+        return euclidean_dist{{name_suffix}}(x1, x2, size)
+
+    cdef inline {{INPUT_DTYPE_t}} rdist(self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil:
+        return euclidean_rdist{{name_suffix}}(x1, x2, size)
+
+    cdef inline {{INPUT_DTYPE_t}} _rdist_to_dist(self, {{INPUT_DTYPE_t}} rdist) except -1 nogil:
+        return sqrt(rdist)
+
+    cdef inline {{INPUT_DTYPE_t}} _dist_to_rdist(self, {{INPUT_DTYPE_t}} dist) except -1 nogil:
+        return dist * dist
+
+    def rdist_to_dist(self, rdist):
+        return np.sqrt(rdist)
+
+    def dist_to_rdist(self, dist):
+        return dist ** 2
+
+    cdef inline {{INPUT_DTYPE_t}} rdist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil:
+
+        cdef:
+            intp_t ix1, ix2
+            intp_t i1 = x1_start
+            intp_t i2 = x2_start
+
+            float64_t d = 0.0
+            float64_t unsquared = 0
+
+        while i1 < x1_end and i2 < x2_end:
+            ix1 = x1_indices[i1]
+            ix2 = x2_indices[i2]
+
+            if ix1 == ix2:
+                unsquared = x1_data[i1] - x2_data[i2]
+                d = d + (unsquared * unsquared)
+                i1 = i1 + 1
+                i2 = i2 + 1
+            elif ix1 < ix2:
+                unsquared = x1_data[i1]
+                d = d + (unsquared * unsquared)
+                i1 = i1 + 1
+            else:
+                unsquared = x2_data[i2]
+                d = d + (unsquared * unsquared)
+                i2 = i2 + 1
+
+        if i1 == x1_end:
+            while i2 < x2_end:
+                unsquared = x2_data[i2]
+                d = d + (unsquared * unsquared)
+                i2 = i2 + 1
+        else:
+            while i1 < x1_end:
+                unsquared = x1_data[i1]
+                d = d + (unsquared * unsquared)
+                i1 = i1 + 1
+
+        return d
+
+    cdef inline {{INPUT_DTYPE_t}} dist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil:
+        return sqrt(
+            self.rdist_csr(
+            x1_data,
+            x1_indices,
+            x2_data,
+            x2_indices,
+            x1_start,
+            x1_end,
+            x2_start,
+            x2_end,
+            size,
+        ))
+
+#------------------------------------------------------------
+# SEuclidean Distance
+#  d = sqrt(sum((x_i - y_i2)^2 / v_i))
+cdef class SEuclideanDistance{{name_suffix}}(DistanceMetric{{name_suffix}}):
+    r"""Standardized Euclidean Distance metric
+
+    .. math::
+       D(x, y) = \sqrt{ \sum_i \frac{ (x_i - y_i) ^ 2}{V_i} }
+    """
+    def __init__(self, V):
+        self.vec = np.asarray(V, dtype=np.float64)
+        self.size = self.vec.shape[0]
+        self.p = 2
+
+    def _validate_data(self, X):
+        if X.shape[1] != self.size:
+            raise ValueError('SEuclidean dist: size of V does not match')
+
+    cdef inline {{INPUT_DTYPE_t}} rdist(
+        self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil:
+        cdef float64_t tmp, d=0
+        cdef intp_t j
+        for j in range(size):
+            tmp = x1[j] - x2[j]
+            d += (tmp * tmp / self.vec[j])
+        return d
+
+    cdef inline {{INPUT_DTYPE_t}} dist(
+        self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil:
+        return sqrt(self.rdist(x1, x2, size))
+
+    cdef inline {{INPUT_DTYPE_t}} _rdist_to_dist(self, {{INPUT_DTYPE_t}} rdist) except -1 nogil:
+        return sqrt(rdist)
+
+    cdef inline {{INPUT_DTYPE_t}} _dist_to_rdist(self, {{INPUT_DTYPE_t}} dist) except -1 nogil:
+        return dist * dist
+
+    def rdist_to_dist(self, rdist):
+        return np.sqrt(rdist)
+
+    def dist_to_rdist(self, dist):
+        return dist ** 2
+
+    cdef inline {{INPUT_DTYPE_t}} rdist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil:
+
+        cdef:
+            intp_t ix1, ix2
+            intp_t i1 = x1_start
+            intp_t i2 = x2_start
+
+            float64_t d = 0.0
+            float64_t unsquared = 0
+
+        while i1 < x1_end and i2 < x2_end:
+            ix1 = x1_indices[i1]
+            ix2 = x2_indices[i2]
+
+            if ix1 == ix2:
+                unsquared = x1_data[i1] - x2_data[i2]
+                d = d + (unsquared * unsquared) / self.vec[ix1]
+                i1 = i1 + 1
+                i2 = i2 + 1
+            elif ix1 < ix2:
+                unsquared = x1_data[i1]
+                d = d + (unsquared * unsquared) / self.vec[ix1]
+                i1 = i1 + 1
+            else:
+                unsquared = x2_data[i2]
+                d = d + (unsquared * unsquared) / self.vec[ix2]
+                i2 = i2 + 1
+
+        if i1 == x1_end:
+            while i2 < x2_end:
+                ix2 = x2_indices[i2]
+                unsquared = x2_data[i2]
+                d = d + (unsquared * unsquared) / self.vec[ix2]
+                i2 = i2 + 1
+        else:
+            while i1 < x1_end:
+                ix1 = x1_indices[i1]
+                unsquared = x1_data[i1]
+                d = d + (unsquared * unsquared) / self.vec[ix1]
+                i1 = i1 + 1
+        return d
+
+    cdef inline {{INPUT_DTYPE_t}} dist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil:
+        return sqrt(
+            self.rdist_csr(
+            x1_data,
+            x1_indices,
+            x2_data,
+            x2_indices,
+            x1_start,
+            x1_end,
+            x2_start,
+            x2_end,
+            size,
+        ))
+
+#------------------------------------------------------------
+# Manhattan Distance
+#  d = sum(abs(x_i - y_i))
+cdef class ManhattanDistance{{name_suffix}}(DistanceMetric{{name_suffix}}):
+    r"""Manhattan/City-block Distance metric
+
+    .. math::
+       D(x, y) = \sum_i |x_i - y_i|
+    """
+    def __init__(self):
+        self.p = 1
+
+    cdef inline {{INPUT_DTYPE_t}} dist(
+        self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil:
+        cdef float64_t d = 0
+        cdef intp_t j
+        for j in range(size):
+            d += fabs(x1[j] - x2[j])
+        return d
+
+    cdef inline {{INPUT_DTYPE_t}} dist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil:
+
+        cdef:
+            intp_t ix1, ix2
+            intp_t i1 = x1_start
+            intp_t i2 = x2_start
+
+            {{INPUT_DTYPE_t}} d = 0.0
+
+        while i1 < x1_end and i2 < x2_end:
+            ix1 = x1_indices[i1]
+            ix2 = x2_indices[i2]
+
+            if ix1 == ix2:
+                d = d + fabs(x1_data[i1] - x2_data[i2])
+                i1 = i1 + 1
+                i2 = i2 + 1
+            elif ix1 < ix2:
+                d = d + fabs(x1_data[i1])
+                i1 = i1 + 1
+            else:
+                d = d + fabs(x2_data[i2])
+                i2 = i2 + 1
+
+        if i1 == x1_end:
+            while i2 < x2_end:
+                d = d + fabs(x2_data[i2])
+                i2 = i2 + 1
+        else:
+            while i1 < x1_end:
+                d = d + fabs(x1_data[i1])
+                i1 = i1 + 1
+
+        return d
+
+
+#------------------------------------------------------------
+# Chebyshev Distance
+#  d = max_i(abs(x_i - y_i))
+cdef class ChebyshevDistance{{name_suffix}}(DistanceMetric{{name_suffix}}):
+    """Chebyshev/Infinity Distance
+
+    .. math::
+       D(x, y) = max_i (|x_i - y_i|)
+
+    Examples
+    --------
+    >>> from sklearn.metrics.dist_metrics import DistanceMetric
+    >>> dist = DistanceMetric.get_metric('chebyshev')
+    >>> X = [[0, 1, 2],
+    ...      [3, 4, 5]]
+    >>> Y = [[-1, 0, 1],
+    ...      [3, 4, 5]]
+    >>> dist.pairwise(X, Y)
+    array([[1.732..., 5.196...],
+           [6.928..., 0....   ]])
+    """
+    def __init__(self):
+        self.p = INF{{name_suffix}}
+
+    cdef inline {{INPUT_DTYPE_t}} dist(
+        self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil:
+        cdef float64_t d = 0
+        cdef intp_t j
+        for j in range(size):
+            d = fmax(d, fabs(x1[j] - x2[j]))
+        return d
+
+
+    cdef inline {{INPUT_DTYPE_t}} dist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil:
+
+        cdef:
+            intp_t ix1, ix2
+            intp_t i1 = x1_start
+            intp_t i2 = x2_start
+
+            float64_t d = 0.0
+
+        while i1 < x1_end and i2 < x2_end:
+            ix1 = x1_indices[i1]
+            ix2 = x2_indices[i2]
+
+            if ix1 == ix2:
+                d = fmax(d, fabs(x1_data[i1] - x2_data[i2]))
+                i1 = i1 + 1
+                i2 = i2 + 1
+            elif ix1 < ix2:
+                d = fmax(d, fabs(x1_data[i1]))
+                i1 = i1 + 1
+            else:
+                d = fmax(d, fabs(x2_data[i2]))
+                i2 = i2 + 1
+
+        if i1 == x1_end:
+            while i2 < x2_end:
+                d = fmax(d, fabs(x2_data[i2]))
+                i2 = i2 + 1
+        else:
+            while i1 < x1_end:
+                d = fmax(d, fabs(x1_data[i1]))
+                i1 = i1 + 1
+
+        return d
+
+
+#------------------------------------------------------------
+# Minkowski Distance
+cdef class MinkowskiDistance{{name_suffix}}(DistanceMetric{{name_suffix}}):
+    r"""Minkowski Distance
+
+    .. math::
+        D(x, y) = {||u-v||}_p
+
+    when w is None.
+
+    Here is the more general expanded expression for the weighted case:
+
+    .. math::
+        D(x, y) = [\sum_i w_i *|x_i - y_i|^p] ^ (1/p)
+
+    Parameters
+    ----------
+    p : float
+        The order of the p-norm of the difference (see above).
+
+        .. versionchanged:: 1.4.0
+            Minkowski distance allows `p` to be `0 0 and finite.
+    When :math:`p \in (0,1)`, it isn't a true metric but is permissible when
+    the triangular inequality isn't necessary.
+    For p = infinity, use ChebyshevDistance.
+    Note that for p=1, ManhattanDistance is more efficient, and for
+    p=2, EuclideanDistance is more efficient.
+
+    """
+    def __init__(self, p, w=None):
+        if p <= 0:
+            raise ValueError("p must be greater than 0")
+        elif np.isinf(p):
+            raise ValueError("MinkowskiDistance requires finite p. "
+                             "For p=inf, use ChebyshevDistance.")
+
+        self.p = p
+        if w is not None:
+            w_array = check_array(
+                w, ensure_2d=False, dtype=np.float64, input_name="w"
+            )
+            if (w_array < 0).any():
+                raise ValueError("w cannot contain negative weights")
+            self.vec = w_array
+            self.size = self.vec.shape[0]
+        else:
+            self.vec = np.asarray([], dtype=np.float64)
+            self.size = 0
+
+    def _validate_data(self, X):
+        if self.size > 0 and X.shape[1] != self.size:
+            raise ValueError("MinkowskiDistance: the size of w must match "
+                             f"the number of features ({X.shape[1]}). "
+                             f"Currently len(w)={self.size}.")
+
+    cdef inline {{INPUT_DTYPE_t}} rdist(
+        self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil:
+        cdef float64_t d=0
+        cdef intp_t j
+        cdef bint has_w = self.size > 0
+        if has_w:
+            for j in range(size):
+                d += (self.vec[j] * pow(fabs(x1[j] - x2[j]), self.p))
+        else:
+            for j in range(size):
+                d += (pow(fabs(x1[j] - x2[j]), self.p))
+        return d
+
+    cdef inline {{INPUT_DTYPE_t}} dist(
+        self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil:
+        return pow(self.rdist(x1, x2, size), 1. / self.p)
+
+    cdef inline {{INPUT_DTYPE_t}} _rdist_to_dist(self, {{INPUT_DTYPE_t}} rdist) except -1 nogil:
+        return pow(rdist, 1. / self.p)
+
+    cdef inline {{INPUT_DTYPE_t}} _dist_to_rdist(self, {{INPUT_DTYPE_t}} dist) except -1 nogil:
+        return pow(dist, self.p)
+
+    def rdist_to_dist(self, rdist):
+        return rdist ** (1. / self.p)
+
+    def dist_to_rdist(self, dist):
+        return dist ** self.p
+
+    cdef inline {{INPUT_DTYPE_t}} rdist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil:
+
+        cdef:
+            intp_t ix1, ix2
+            intp_t i1 = x1_start
+            intp_t i2 = x2_start
+
+            float64_t d = 0.0
+            bint has_w = self.size > 0
+
+        if has_w:
+            while i1 < x1_end and i2 < x2_end:
+                ix1 = x1_indices[i1]
+                ix2 = x2_indices[i2]
+
+                if ix1 == ix2:
+                    d = d + (self.vec[ix1] * pow(fabs(
+                        x1_data[i1] - x2_data[i2]
+                    ), self.p))
+                    i1 = i1 + 1
+                    i2 = i2 + 1
+                elif ix1 < ix2:
+                    d = d + (self.vec[ix1] * pow(fabs(x1_data[i1]), self.p))
+                    i1 = i1 + 1
+                else:
+                    d = d + (self.vec[ix2] * pow(fabs(x2_data[i2]), self.p))
+                    i2 = i2 + 1
+
+            if i1 == x1_end:
+                while i2 < x2_end:
+                    ix2 = x2_indices[i2]
+                    d = d + (self.vec[ix2] * pow(fabs(x2_data[i2]), self.p))
+                    i2 = i2 + 1
+            else:
+                while i1 < x1_end:
+                    ix1 = x1_indices[i1]
+                    d = d + (self.vec[ix1] * pow(fabs(x1_data[i1]), self.p))
+                    i1 = i1 + 1
+
+            return d
+        else:
+            while i1 < x1_end and i2 < x2_end:
+                ix1 = x1_indices[i1]
+                ix2 = x2_indices[i2]
+
+                if ix1 == ix2:
+                    d = d + (pow(fabs(
+                        x1_data[i1] - x2_data[i2]
+                    ), self.p))
+                    i1 = i1 + 1
+                    i2 = i2 + 1
+                elif ix1 < ix2:
+                    d = d + (pow(fabs(x1_data[i1]), self.p))
+                    i1 = i1 + 1
+                else:
+                    d = d + (pow(fabs(x2_data[i2]), self.p))
+                    i2 = i2 + 1
+
+            if i1 == x1_end:
+                while i2 < x2_end:
+                    d = d + (pow(fabs(x2_data[i2]), self.p))
+                    i2 = i2 + 1
+            else:
+                while i1 < x1_end:
+                    d = d + (pow(fabs(x1_data[i1]), self.p))
+                    i1 = i1 + 1
+
+            return d
+
+    cdef inline {{INPUT_DTYPE_t}} dist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil:
+        return pow(
+            self.rdist_csr(
+                x1_data,
+                x1_indices,
+                x2_data,
+                x2_indices,
+                x1_start,
+                x1_end,
+                x2_start,
+                x2_end,
+                size,
+            ),
+            1 / self.p
+        )
+
+#------------------------------------------------------------
+# Mahalanobis Distance
+#  d = sqrt( (x - y)^T V^-1 (x - y) )
+cdef class MahalanobisDistance{{name_suffix}}(DistanceMetric{{name_suffix}}):
+    """Mahalanobis Distance
+
+    .. math::
+       D(x, y) = \sqrt{ (x - y)^T V^{-1} (x - y) }
+
+    Parameters
+    ----------
+    V : array-like
+        Symmetric positive-definite covariance matrix.
+        The inverse of this matrix will be explicitly computed.
+    VI : array-like
+        optionally specify the inverse directly.  If VI is passed,
+        then V is not referenced.
+    """
+    cdef float64_t[::1] buffer
+
+    def __init__(self, V=None, VI=None):
+        if VI is None:
+            if V is None:
+                raise ValueError("Must provide either V or VI "
+                                 "for Mahalanobis distance")
+            VI = np.linalg.inv(V)
+        if VI.ndim != 2 or VI.shape[0] != VI.shape[1]:
+            raise ValueError("V/VI must be square")
+
+        self.mat = np.asarray(VI, dtype=np.float64, order='C')
+
+        self.size = self.mat.shape[0]
+
+        # We need to create a buffer to store the vectors' coordinates' differences
+        self.buffer = np.zeros(self.size, dtype=np.float64)
+
+    def __setstate__(self, state):
+        super().__setstate__(state)
+        self.size = self.mat.shape[0]
+        self.buffer = np.zeros(self.size, dtype=np.float64)
+
+    def _validate_data(self, X):
+        if X.shape[1] != self.size:
+            raise ValueError('Mahalanobis dist: size of V does not match')
+
+    cdef inline {{INPUT_DTYPE_t}} rdist(
+        self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil:
+        cdef float64_t tmp, d = 0
+        cdef intp_t i, j
+
+        # compute (x1 - x2).T * VI * (x1 - x2)
+        for i in range(size):
+            self.buffer[i] = x1[i] - x2[i]
+
+        for i in range(size):
+            tmp = 0
+            for j in range(size):
+                tmp += self.mat[i, j] * self.buffer[j]
+            d += tmp * self.buffer[i]
+        return d
+
+    cdef inline {{INPUT_DTYPE_t}} dist(
+        self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil:
+        return sqrt(self.rdist(x1, x2, size))
+
+    cdef inline {{INPUT_DTYPE_t}} _rdist_to_dist(self, {{INPUT_DTYPE_t}} rdist) except -1 nogil:
+        return sqrt(rdist)
+
+    cdef inline {{INPUT_DTYPE_t}} _dist_to_rdist(self, {{INPUT_DTYPE_t}} dist) except -1 nogil:
+        return dist * dist
+
+    def rdist_to_dist(self, rdist):
+        return np.sqrt(rdist)
+
+    def dist_to_rdist(self, dist):
+        return dist ** 2
+
+    cdef inline {{INPUT_DTYPE_t}} rdist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil:
+
+        cdef:
+            intp_t ix1, ix2
+            intp_t i1 = x1_start
+            intp_t i2 = x2_start
+
+            float64_t tmp, d = 0.0
+
+        while i1 < x1_end and i2 < x2_end:
+            ix1 = x1_indices[i1]
+            ix2 = x2_indices[i2]
+
+            if ix1 == ix2:
+                self.buffer[ix1] = x1_data[i1] - x2_data[i2]
+                i1 = i1 + 1
+                i2 = i2 + 1
+            elif ix1 < ix2:
+                self.buffer[ix1] = x1_data[i1]
+                i1 = i1 + 1
+            else:
+                self.buffer[ix2] = - x2_data[i2]
+                i2 = i2 + 1
+
+        if i1 == x1_end:
+            while i2 < x2_end:
+                ix2 = x2_indices[i2]
+                self.buffer[ix2] = - x2_data[i2]
+                i2 = i2 + 1
+        else:
+            while i1 < x1_end:
+                ix1 = x1_indices[i1]
+                self.buffer[ix1] = x1_data[i1]
+                i1 = i1 + 1
+
+        for i in range(size):
+            tmp = 0
+            for j in range(size):
+                tmp += self.mat[i, j] * self.buffer[j]
+            d += tmp * self.buffer[i]
+
+        return d
+
+    cdef inline {{INPUT_DTYPE_t}} dist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil:
+        return sqrt(
+            self.rdist_csr(
+            x1_data,
+            x1_indices,
+            x2_data,
+            x2_indices,
+            x1_start,
+            x1_end,
+            x2_start,
+            x2_end,
+            size,
+        ))
+
+#------------------------------------------------------------
+# Hamming Distance
+#  d = N_unequal(x, y) / N_tot
+cdef class HammingDistance{{name_suffix}}(DistanceMetric{{name_suffix}}):
+    r"""Hamming Distance
+
+    Hamming distance is meant for discrete-valued vectors, though it is
+    a valid metric for real-valued vectors.
+
+    .. math::
+       D(x, y) = \frac{1}{N} \sum_i \delta_{x_i, y_i}
+    """
+    cdef inline {{INPUT_DTYPE_t}} dist(
+        self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil:
+        cdef int n_unequal = 0
+        cdef intp_t j
+        for j in range(size):
+            if x1[j] != x2[j]:
+                n_unequal += 1
+        return float(n_unequal) / size
+
+
+    cdef inline {{INPUT_DTYPE_t}} dist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil:
+
+        cdef:
+            intp_t ix1, ix2
+            intp_t i1 = x1_start
+            intp_t i2 = x2_start
+
+            float64_t d = 0.0
+
+        while i1 < x1_end and i2 < x2_end:
+            ix1 = x1_indices[i1]
+            ix2 = x2_indices[i2]
+
+            if ix1 == ix2:
+                d += (x1_data[i1] != x2_data[i2])
+                i1 = i1 + 1
+                i2 = i2 + 1
+            elif ix1 < ix2:
+                d += (x1_data[i1] != 0)
+                i1 = i1 + 1
+            else:
+                d += (x2_data[i2] != 0)
+                i2 = i2 + 1
+
+        if i1 == x1_end:
+            while i2 < x2_end:
+                d += (x2_data[i2] != 0)
+                i2 = i2 + 1
+        else:
+            while i1 < x1_end:
+                d += (x1_data[i1] != 0)
+                i1 = i1 + 1
+
+        d /= size
+
+        return d
+
+
+#------------------------------------------------------------
+# Canberra Distance
+#  D(x, y) = sum[ abs(x_i - y_i) / (abs(x_i) + abs(y_i)) ]
+cdef class CanberraDistance{{name_suffix}}(DistanceMetric{{name_suffix}}):
+    r"""Canberra Distance
+
+    Canberra distance is meant for discrete-valued vectors, though it is
+    a valid metric for real-valued vectors.
+
+    .. math::
+       D(x, y) = \sum_i \frac{|x_i - y_i|}{|x_i| + |y_i|}
+    """
+    cdef inline {{INPUT_DTYPE_t}} dist(
+        self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil:
+        cdef float64_t denom, d = 0
+        cdef intp_t j
+        for j in range(size):
+            denom = fabs(x1[j]) + fabs(x2[j])
+            if denom > 0:
+                d += fabs(x1[j] - x2[j]) / denom
+        return d
+
+    cdef inline {{INPUT_DTYPE_t}} dist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil:
+
+        cdef:
+            intp_t ix1, ix2
+            intp_t i1 = x1_start
+            intp_t i2 = x2_start
+
+            float64_t d = 0.0
+
+        while i1 < x1_end and i2 < x2_end:
+            ix1 = x1_indices[i1]
+            ix2 = x2_indices[i2]
+
+            if ix1 == ix2:
+                d += (
+                        fabs(x1_data[i1] - x2_data[i2]) /
+                        (fabs(x1_data[i1]) + fabs(x2_data[i2]))
+                )
+                i1 = i1 + 1
+                i2 = i2 + 1
+            elif ix1 < ix2:
+                d += 1.
+                i1 = i1 + 1
+            else:
+                d += 1.
+                i2 = i2 + 1
+
+        if i1 == x1_end:
+            while i2 < x2_end:
+                d += 1.
+                i2 = i2 + 1
+        else:
+            while i1 < x1_end:
+                d += 1.
+                i1 = i1 + 1
+
+        return d
+
+#------------------------------------------------------------
+# Bray-Curtis Distance
+#  D(x, y) = sum[abs(x_i - y_i)] / sum[abs(x_i) + abs(y_i)]
+cdef class BrayCurtisDistance{{name_suffix}}(DistanceMetric{{name_suffix}}):
+    r"""Bray-Curtis Distance
+
+    Bray-Curtis distance is meant for discrete-valued vectors, though it is
+    a valid metric for real-valued vectors.
+
+    .. math::
+       D(x, y) = \frac{\sum_i |x_i - y_i|}{\sum_i(|x_i| + |y_i|)}
+    """
+    cdef inline {{INPUT_DTYPE_t}} dist(
+        self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil:
+        cdef float64_t num = 0, denom = 0
+        cdef intp_t j
+        for j in range(size):
+            num += fabs(x1[j] - x2[j])
+            denom += fabs(x1[j]) + fabs(x2[j])
+        if denom > 0:
+            return num / denom
+        else:
+            return 0.0
+
+    cdef inline {{INPUT_DTYPE_t}} dist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil:
+
+        cdef:
+            intp_t ix1, ix2
+            intp_t i1 = x1_start
+            intp_t i2 = x2_start
+
+            float64_t num = 0.0
+            float64_t denom = 0.0
+
+        while i1 < x1_end and i2 < x2_end:
+            ix1 = x1_indices[i1]
+            ix2 = x2_indices[i2]
+
+            if ix1 == ix2:
+                num += fabs(x1_data[i1] - x2_data[i2])
+                denom += fabs(x1_data[i1]) + fabs(x2_data[i2])
+                i1 = i1 + 1
+                i2 = i2 + 1
+            elif ix1 < ix2:
+                num += fabs(x1_data[i1])
+                denom += fabs(x1_data[i1])
+                i1 = i1 + 1
+            else:
+                num += fabs(x2_data[i2])
+                denom += fabs(x2_data[i2])
+                i2 = i2 + 1
+
+        if i1 == x1_end:
+            while i2 < x2_end:
+                num += fabs(x1_data[i1])
+                denom += fabs(x1_data[i1])
+                i2 = i2 + 1
+        else:
+            while i1 < x1_end:
+                num += fabs(x2_data[i2])
+                denom += fabs(x2_data[i2])
+                i1 = i1 + 1
+
+        return num / denom
+
+#------------------------------------------------------------
+# Jaccard Distance (boolean)
+#  D(x, y) = N_unequal(x, y) / N_nonzero(x, y)
+cdef class JaccardDistance{{name_suffix}}(DistanceMetric{{name_suffix}}):
+    r"""Jaccard Distance
+
+    Jaccard Distance is a dissimilarity measure for boolean-valued
+    vectors. All nonzero entries will be treated as True, zero entries will
+    be treated as False.
+
+        D(x, y) = (N_TF + N_FT) / (N_TT + N_TF + N_FT)
+    """
+    cdef inline {{INPUT_DTYPE_t}} dist(
+        self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil:
+        cdef int tf1, tf2, n_eq = 0, nnz = 0
+        cdef intp_t j
+        for j in range(size):
+            tf1 = x1[j] != 0
+            tf2 = x2[j] != 0
+            nnz += (tf1 or tf2)
+            n_eq += (tf1 and tf2)
+        # Based on https://github.com/scipy/scipy/pull/7373
+        # When comparing two all-zero vectors, scipy>=1.2.0 jaccard metric
+        # was changed to return 0, instead of nan.
+        if nnz == 0:
+            return 0
+        return (nnz - n_eq) * 1.0 / nnz
+
+    cdef inline {{INPUT_DTYPE_t}} dist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil:
+
+        cdef:
+            intp_t ix1, ix2
+            intp_t i1 = x1_start
+            intp_t i2 = x2_start
+
+            intp_t tf1, tf2, n_tt = 0, nnz = 0
+
+        while i1 < x1_end and i2 < x2_end:
+            ix1 = x1_indices[i1]
+            ix2 = x2_indices[i2]
+
+            tf1 = x1_data[i1] != 0
+            tf2 = x2_data[i2] != 0
+
+            if ix1 == ix2:
+                nnz += (tf1 or tf2)
+                n_tt += (tf1 and tf2)
+                i1 = i1 + 1
+                i2 = i2 + 1
+            elif ix1 < ix2:
+                nnz += tf1
+                i1 = i1 + 1
+            else:
+                nnz += tf2
+                i2 = i2 + 1
+
+        if i1 == x1_end:
+            while i2 < x2_end:
+                tf2 = x2_data[i2] != 0
+                nnz += tf2
+                i2 = i2 + 1
+        else:
+            while i1 < x1_end:
+                tf1 = x1_data[i1] != 0
+                nnz += tf1
+                i1 = i1 + 1
+
+        # Based on https://github.com/scipy/scipy/pull/7373
+        # When comparing two all-zero vectors, scipy>=1.2.0 jaccard metric
+        # was changed to return 0, instead of nan.
+        if nnz == 0:
+            return 0
+        return (nnz - n_tt) * 1.0 / nnz
+
+#------------------------------------------------------------
+# Matching Distance (boolean)
+#  D(x, y) = n_neq / n
+cdef class MatchingDistance{{name_suffix}}(DistanceMetric{{name_suffix}}):
+    r"""Matching Distance
+
+    Matching Distance is a dissimilarity measure for boolean-valued
+    vectors. All nonzero entries will be treated as True, zero entries will
+    be treated as False.
+
+        D(x, y) = (N_TF + N_FT) / N
+    """
+    cdef inline {{INPUT_DTYPE_t}} dist(
+        self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil:
+        cdef int tf1, tf2, n_neq = 0
+        cdef intp_t j
+        for j in range(size):
+            tf1 = x1[j] != 0
+            tf2 = x2[j] != 0
+            n_neq += (tf1 != tf2)
+        return n_neq * 1. / size
+
+    cdef inline {{INPUT_DTYPE_t}} dist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil:
+
+        cdef:
+            intp_t ix1, ix2
+            intp_t i1 = x1_start
+            intp_t i2 = x2_start
+
+            intp_t tf1, tf2, n_neq = 0
+
+        while i1 < x1_end and i2 < x2_end:
+            ix1 = x1_indices[i1]
+            ix2 = x2_indices[i2]
+
+            if ix1 == ix2:
+                tf1 = x1_data[i1] != 0
+                tf2 = x2_data[i2] != 0
+                n_neq += (tf1 != tf2)
+                i1 = i1 + 1
+                i2 = i2 + 1
+            elif ix1 < ix2:
+                n_neq += (x1_data[i1] != 0)
+                i1 = i1 + 1
+            else:
+                n_neq += (x2_data[i2] != 0)
+                i2 = i2 + 1
+
+        if i1 == x1_end:
+            while i2 < x2_end:
+                n_neq += (x2_data[i2] != 0)
+                i2 = i2 + 1
+        else:
+            while i1 < x1_end:
+                n_neq += (x1_data[i1] != 0)
+                i1 = i1 + 1
+
+        return n_neq * 1.0 / size
+
+#------------------------------------------------------------
+# Dice Distance (boolean)
+#  D(x, y) = n_neq / (2 * ntt + n_neq)
+cdef class DiceDistance{{name_suffix}}(DistanceMetric{{name_suffix}}):
+    r"""Dice Distance
+
+    Dice Distance is a dissimilarity measure for boolean-valued
+    vectors. All nonzero entries will be treated as True, zero entries will
+    be treated as False.
+
+        D(x, y) = (N_TF + N_FT) / (2 * N_TT + N_TF + N_FT)
+
+    """
+    cdef inline {{INPUT_DTYPE_t}} dist(
+        self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil:
+        cdef int tf1, tf2, n_neq = 0, n_tt = 0
+        cdef intp_t j
+        for j in range(size):
+            tf1 = x1[j] != 0
+            tf2 = x2[j] != 0
+            n_tt += (tf1 and tf2)
+            n_neq += (tf1 != tf2)
+        return n_neq / (2.0 * n_tt + n_neq)
+
+    cdef inline {{INPUT_DTYPE_t}} dist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil:
+
+        cdef:
+            intp_t ix1, ix2
+            intp_t i1 = x1_start
+            intp_t i2 = x2_start
+
+            intp_t tf1, tf2, n_tt = 0, n_neq = 0
+
+        while i1 < x1_end and i2 < x2_end:
+            ix1 = x1_indices[i1]
+            ix2 = x2_indices[i2]
+
+            tf1 = x1_data[i1] != 0
+            tf2 = x2_data[i2] != 0
+
+            if ix1 == ix2:
+                n_tt += (tf1 and tf2)
+                n_neq += (tf1 != tf2)
+                i1 = i1 + 1
+                i2 = i2 + 1
+            elif ix1 < ix2:
+                n_neq += tf1
+                i1 = i1 + 1
+            else:
+                n_neq += tf2
+                i2 = i2 + 1
+
+        if i1 == x1_end:
+            while i2 < x2_end:
+                tf2 = x2_data[i2] != 0
+                n_neq += tf2
+                i2 = i2 + 1
+        else:
+            while i1 < x1_end:
+                tf1 = x1_data[i1] != 0
+                n_neq += tf1
+                i1 = i1 + 1
+
+        return n_neq / (2.0 * n_tt + n_neq)
+
+
+#------------------------------------------------------------
+# Kulsinski Distance (boolean)
+#  D(x, y) = (ntf + nft - ntt + n) / (n_neq + n)
+cdef class KulsinskiDistance{{name_suffix}}(DistanceMetric{{name_suffix}}):
+    r"""Kulsinski Distance
+
+    Kulsinski Distance is a dissimilarity measure for boolean-valued
+    vectors. All nonzero entries will be treated as True, zero entries will
+    be treated as False.
+
+        D(x, y) = 1 - N_TT / (N + N_TF + N_FT)
+
+    """
+    cdef inline {{INPUT_DTYPE_t}} dist(
+        self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil:
+        cdef int tf1, tf2, n_tt = 0, n_neq = 0
+        cdef intp_t j
+        for j in range(size):
+            tf1 = x1[j] != 0
+            tf2 = x2[j] != 0
+            n_neq += (tf1 != tf2)
+            n_tt += (tf1 and tf2)
+        return (n_neq - n_tt + size) * 1.0 / (n_neq + size)
+
+    cdef inline {{INPUT_DTYPE_t}} dist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil:
+
+        cdef:
+            intp_t ix1, ix2
+            intp_t i1 = x1_start
+            intp_t i2 = x2_start
+
+            intp_t tf1, tf2, n_tt = 0, n_neq = 0
+
+        while i1 < x1_end and i2 < x2_end:
+            ix1 = x1_indices[i1]
+            ix2 = x2_indices[i2]
+
+            tf1 = x1_data[i1] != 0
+            tf2 = x2_data[i2] != 0
+
+            if ix1 == ix2:
+                n_tt += (tf1 and tf2)
+                n_neq += (tf1 != tf2)
+                i1 = i1 + 1
+                i2 = i2 + 1
+            elif ix1 < ix2:
+                n_neq += tf1
+                i1 = i1 + 1
+            else:
+                n_neq += tf2
+                i2 = i2 + 1
+
+        if i1 == x1_end:
+            while i2 < x2_end:
+                tf2 = x2_data[i2] != 0
+                n_neq += tf2
+                i2 = i2 + 1
+        else:
+            while i1 < x1_end:
+                tf1 = x1_data[i1] != 0
+                n_neq += tf1
+                i1 = i1 + 1
+
+        return (n_neq - n_tt + size) * 1.0 / (n_neq + size)
+
+#------------------------------------------------------------
+# Rogers-Tanimoto Distance (boolean)
+#  D(x, y) = 2 * n_neq / (n + n_neq)
+cdef class RogersTanimotoDistance{{name_suffix}}(DistanceMetric{{name_suffix}}):
+    r"""Rogers-Tanimoto Distance
+
+    Rogers-Tanimoto Distance is a dissimilarity measure for boolean-valued
+    vectors. All nonzero entries will be treated as True, zero entries will
+    be treated as False.
+
+        D(x, y) = 2 (N_TF + N_FT) / (N + N_TF + N_FT)
+    """
+    cdef inline {{INPUT_DTYPE_t}} dist(
+        self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil:
+        cdef int tf1, tf2, n_neq = 0
+        cdef intp_t j
+        for j in range(size):
+            tf1 = x1[j] != 0
+            tf2 = x2[j] != 0
+            n_neq += (tf1 != tf2)
+        return (2.0 * n_neq) / (size + n_neq)
+
+    cdef inline {{INPUT_DTYPE_t}} dist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil:
+
+        cdef:
+            intp_t ix1, ix2
+            intp_t i1 = x1_start
+            intp_t i2 = x2_start
+
+            intp_t tf1, tf2, n_neq = 0
+
+        while i1 < x1_end and i2 < x2_end:
+            ix1 = x1_indices[i1]
+            ix2 = x2_indices[i2]
+
+            tf1 = x1_data[i1] != 0
+            tf2 = x2_data[i2] != 0
+
+            if ix1 == ix2:
+                n_neq += (tf1 != tf2)
+                i1 = i1 + 1
+                i2 = i2 + 1
+            elif ix1 < ix2:
+                n_neq += tf1
+                i1 = i1 + 1
+            else:
+                n_neq += tf2
+                i2 = i2 + 1
+
+        if i1 == x1_end:
+            while i2 < x2_end:
+                tf2 = x2_data[i2] != 0
+                n_neq += tf2
+                i2 = i2 + 1
+        else:
+            while i1 < x1_end:
+                tf1 = x1_data[i1] != 0
+                n_neq += tf1
+                i1 = i1 + 1
+
+        return (2.0 * n_neq) / (size + n_neq)
+
+#------------------------------------------------------------
+# Russell-Rao Distance (boolean)
+#  D(x, y) = (n - ntt) / n
+cdef class RussellRaoDistance{{name_suffix}}(DistanceMetric{{name_suffix}}):
+    r"""Russell-Rao Distance
+
+    Russell-Rao Distance is a dissimilarity measure for boolean-valued
+    vectors. All nonzero entries will be treated as True, zero entries will
+    be treated as False.
+
+        D(x, y) = (N - N_TT) / N
+    """
+    cdef inline {{INPUT_DTYPE_t}} dist(
+        self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil:
+        cdef int tf1, tf2, n_tt = 0
+        cdef intp_t j
+        for j in range(size):
+            tf1 = x1[j] != 0
+            tf2 = x2[j] != 0
+            n_tt += (tf1 and tf2)
+        return (size - n_tt) * 1. / size
+
+    cdef inline {{INPUT_DTYPE_t}} dist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil:
+
+        cdef:
+            intp_t ix1, ix2
+            intp_t i1 = x1_start
+            intp_t i2 = x2_start
+
+            intp_t tf1, tf2, n_tt = 0
+
+        while i1 < x1_end and i2 < x2_end:
+            ix1 = x1_indices[i1]
+            ix2 = x2_indices[i2]
+
+            tf1 = x1_data[i1] != 0
+            tf2 = x2_data[i2] != 0
+
+            if ix1 == ix2:
+                n_tt += (tf1 and tf2)
+                i1 = i1 + 1
+                i2 = i2 + 1
+            elif ix1 < ix2:
+                i1 = i1 + 1
+            else:
+                i2 = i2 + 1
+
+        # We don't need to go through all the longest
+        # vector because tf1 or tf2 will be false
+        # and thus n_tt won't be increased.
+
+        return (size - n_tt) * 1. / size
+
+
+
+#------------------------------------------------------------
+# Sokal-Michener Distance (boolean)
+#  D(x, y) = 2 * n_neq / (n + n_neq)
+cdef class SokalMichenerDistance{{name_suffix}}(DistanceMetric{{name_suffix}}):
+    r"""Sokal-Michener Distance
+
+    Sokal-Michener Distance is a dissimilarity measure for boolean-valued
+    vectors. All nonzero entries will be treated as True, zero entries will
+    be treated as False.
+
+        D(x, y) = 2 (N_TF + N_FT) / (N + N_TF + N_FT)
+    """
+    cdef inline {{INPUT_DTYPE_t}} dist(
+        self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil:
+        cdef int tf1, tf2, n_neq = 0
+        cdef intp_t j
+        for j in range(size):
+            tf1 = x1[j] != 0
+            tf2 = x2[j] != 0
+            n_neq += (tf1 != tf2)
+        return (2.0 * n_neq) / (size + n_neq)
+
+    cdef inline {{INPUT_DTYPE_t}} dist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil:
+
+        cdef:
+            intp_t ix1, ix2
+            intp_t i1 = x1_start
+            intp_t i2 = x2_start
+
+            intp_t tf1, tf2, n_neq = 0
+
+        while i1 < x1_end and i2 < x2_end:
+            ix1 = x1_indices[i1]
+            ix2 = x2_indices[i2]
+
+            tf1 = x1_data[i1] != 0
+            tf2 = x2_data[i2] != 0
+
+            if ix1 == ix2:
+                n_neq += (tf1 != tf2)
+                i1 = i1 + 1
+                i2 = i2 + 1
+            elif ix1 < ix2:
+                n_neq += tf1
+                i1 = i1 + 1
+            else:
+                n_neq += tf2
+                i2 = i2 + 1
+
+        if i1 == x1_end:
+            while i2 < x2_end:
+                tf2 = x2_data[i2] != 0
+                n_neq += tf2
+                i2 = i2 + 1
+        else:
+            while i1 < x1_end:
+                tf1 = x1_data[i1] != 0
+                n_neq += tf1
+                i1 = i1 + 1
+
+        return (2.0 * n_neq) / (size + n_neq)
+
+#------------------------------------------------------------
+# Sokal-Sneath Distance (boolean)
+#  D(x, y) = n_neq / (0.5 * n_tt + n_neq)
+cdef class SokalSneathDistance{{name_suffix}}(DistanceMetric{{name_suffix}}):
+    r"""Sokal-Sneath Distance
+
+    Sokal-Sneath Distance is a dissimilarity measure for boolean-valued
+    vectors. All nonzero entries will be treated as True, zero entries will
+    be treated as False.
+
+        D(x, y) = (N_TF + N_FT) / (N_TT / 2 + N_FT + N_TF)
+    """
+    cdef inline {{INPUT_DTYPE_t}} dist(
+        self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil:
+        cdef int tf1, tf2, n_tt = 0, n_neq = 0
+        cdef intp_t j
+        for j in range(size):
+            tf1 = x1[j] != 0
+            tf2 = x2[j] != 0
+            n_neq += (tf1 != tf2)
+            n_tt += (tf1 and tf2)
+        return n_neq / (0.5 * n_tt + n_neq)
+
+    cdef inline {{INPUT_DTYPE_t}} dist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil:
+
+        cdef:
+            intp_t ix1, ix2
+            intp_t i1 = x1_start
+            intp_t i2 = x2_start
+
+            intp_t tf1, tf2, n_tt = 0, n_neq = 0
+
+        while i1 < x1_end and i2 < x2_end:
+            ix1 = x1_indices[i1]
+            ix2 = x2_indices[i2]
+
+            tf1 = x1_data[i1] != 0
+            tf2 = x2_data[i2] != 0
+
+            if ix1 == ix2:
+                n_tt += (tf1 and tf2)
+                n_neq += (tf1 != tf2)
+                i1 = i1 + 1
+                i2 = i2 + 1
+            elif ix1 < ix2:
+                n_neq += tf1
+                i1 = i1 + 1
+            else:
+                n_neq += tf2
+                i2 = i2 + 1
+
+        if i1 == x1_end:
+            while i2 < x2_end:
+                tf2 = x2_data[i2] != 0
+                n_neq += tf2
+                i2 = i2 + 1
+        else:
+            while i1 < x1_end:
+                tf1 = x1_data[i1] != 0
+                n_neq += tf1
+                i1 = i1 + 1
+
+        return n_neq / (0.5 * n_tt + n_neq)
+
+
+#------------------------------------------------------------
+# Haversine Distance (2 dimensional)
+#  D(x, y) = 2 arcsin{sqrt[sin^2 ((x1 - y1) / 2)
+#                          + cos(x1) cos(y1) sin^2 ((x2 - y2) / 2)]}
+cdef class HaversineDistance{{name_suffix}}(DistanceMetric{{name_suffix}}):
+    """Haversine (Spherical) Distance
+
+    The Haversine distance is the angular distance between two points on
+    the surface of a sphere.  The first distance of each point is assumed
+    to be the latitude, the second is the longitude, given in radians.
+    The dimension of the points must be 2:
+
+    D(x, y) = 2 arcsin[sqrt{sin^2((x1 - y1) / 2) + cos(x1)cos(y1)sin^2((x2 - y2) / 2)}]
+
+    """
+
+    def _validate_data(self, X):
+        if X.shape[1] != 2:
+            raise ValueError("Haversine distance only valid "
+                             "in 2 dimensions")
+
+    cdef inline {{INPUT_DTYPE_t}} rdist(self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil:
+        cdef float64_t sin_0 = sin(0.5 * ((x1[0]) - (x2[0])))
+        cdef float64_t sin_1 = sin(0.5 * ((x1[1]) - (x2[1])))
+        return (sin_0 * sin_0 + cos(x1[0]) * cos(x2[0]) * sin_1 * sin_1)
+
+    cdef inline {{INPUT_DTYPE_t}} dist(self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil:
+        return 2 * asin(sqrt(self.rdist(x1, x2, size)))
+
+    cdef inline {{INPUT_DTYPE_t}} _rdist_to_dist(self, {{INPUT_DTYPE_t}} rdist) except -1 nogil:
+        return 2 * asin(sqrt(rdist))
+
+    cdef inline {{INPUT_DTYPE_t}} _dist_to_rdist(self, {{INPUT_DTYPE_t}} dist) except -1 nogil:
+        cdef float64_t tmp = sin(0.5 *  dist)
+        return tmp * tmp
+
+    def rdist_to_dist(self, rdist):
+        return 2 * np.arcsin(np.sqrt(rdist))
+
+    def dist_to_rdist(self, dist):
+        tmp = np.sin(0.5 * dist)
+        return tmp * tmp
+
+    cdef inline {{INPUT_DTYPE_t}} dist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil:
+        return 2 * asin(sqrt(self.rdist_csr(
+            x1_data,
+            x1_indices,
+            x2_data,
+            x2_indices,
+            x1_start,
+            x1_end,
+            x2_start,
+            x2_end,
+            size,
+        )))
+
+    cdef inline {{INPUT_DTYPE_t}} rdist_csr(
+        self,
+        const {{INPUT_DTYPE_t}}* x1_data,
+        const int32_t* x1_indices,
+        const {{INPUT_DTYPE_t}}* x2_data,
+        const int32_t* x2_indices,
+        const int32_t x1_start,
+        const int32_t x1_end,
+        const int32_t x2_start,
+        const int32_t x2_end,
+        const intp_t size,
+    ) except -1 nogil:
+
+        cdef:
+            intp_t ix1, ix2
+            intp_t i1 = x1_start
+            intp_t i2 = x2_start
+
+            float64_t x1_0 = 0
+            float64_t x1_1 = 0
+            float64_t x2_0 = 0
+            float64_t x2_1 = 0
+            float64_t sin_0
+            float64_t sin_1
+
+        while i1 < x1_end and i2 < x2_end:
+            ix1 = x1_indices[i1]
+            ix2 = x2_indices[i2]
+
+            # Find the components in the 2D vectors to work with
+            x1_component = ix1 if (x1_start == 0) else ix1 % x1_start
+            x2_component = ix2 if (x2_start == 0) else ix2 % x2_start
+
+            if x1_component == 0:
+                x1_0 = x1_data[i1]
+            else:
+                x1_1 = x1_data[i1]
+
+            if x2_component == 0:
+                x2_0 = x2_data[i2]
+            else:
+                x2_1 = x2_data[i2]
+
+            i1 = i1 + 1
+            i2 = i2 + 1
+
+        if i1 == x1_end:
+            while i2 < x2_end:
+                ix2 = x2_indices[i2]
+                x2_component = ix2 if (x2_start == 0) else ix2 % x2_start
+                if x2_component == 0:
+                    x2_0 = x2_data[i2]
+                else:
+                    x2_1 = x2_data[i2]
+                i2 = i2 + 1
+        else:
+            while i1 < x1_end:
+                ix1 = x1_indices[i1]
+                x1_component = ix1 if (x1_start == 0) else ix1 % x1_start
+                if x1_component == 0:
+                    x1_0 = x1_data[i1]
+                else:
+                    x1_1 = x1_data[i1]
+                i1 = i1 + 1
+
+        sin_0 = sin(0.5 * (x1_0 - x2_0))
+        sin_1 = sin(0.5 * (x1_1 - x2_1))
+
+        return (sin_0 * sin_0 + cos(x1_0) * cos(x2_0) * sin_1 * sin_1)
+
+#------------------------------------------------------------
+# User-defined distance
+#
+cdef class PyFuncDistance{{name_suffix}}(DistanceMetric{{name_suffix}}):
+    """PyFunc Distance
+
+    A user-defined distance
+
+    Parameters
+    ----------
+    func : function
+        func should take two numpy arrays as input, and return a distance.
+    """
+    def __init__(self, func, **kwargs):
+        self.func = func
+        self.kwargs = kwargs
+
+    # in cython < 0.26, GIL was required to be acquired during definition of
+    # the function and inside the body of the function. This behaviour is not
+    # allowed in cython >= 0.26 since it is a redundant GIL acquisition. The
+    # only way to be back compatible is to inherit `dist` from the base class
+    # without GIL and called an inline `_dist` which acquire GIL.
+    cdef inline {{INPUT_DTYPE_t}} dist(
+        self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 nogil:
+        return self._dist(x1, x2, size)
+
+    cdef inline {{INPUT_DTYPE_t}} _dist(
+        self,
+        const {{INPUT_DTYPE_t}}* x1,
+        const {{INPUT_DTYPE_t}}* x2,
+        intp_t size,
+    ) except -1 with gil:
+        cdef:
+            object x1arr = _buffer_to_ndarray{{name_suffix}}(x1, size)
+            object x2arr = _buffer_to_ndarray{{name_suffix}}(x2, size)
+        d = self.func(x1arr, x2arr, **self.kwargs)
+        try:
+            # Cython generates code here that results in a TypeError
+            # if d is the wrong type.
+            return d
+        except TypeError:
+            raise TypeError("Custom distance function must accept two "
+                            "vectors and return a float.")
+
+{{endfor}}
diff --git a/sklearn/metrics/_pairwise_distances_reduction/__init__.py b/sklearn/metrics/_pairwise_distances_reduction/__init__.py
new file mode 100644
index 0000000000000..6b532e0fa8ff0
--- /dev/null
+++ b/sklearn/metrics/_pairwise_distances_reduction/__init__.py
@@ -0,0 +1,112 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+#
+# Pairwise Distances Reductions
+# =============================
+#
+# Overview
+# --------
+#
+#    This module provides routines to compute pairwise distances between a set
+#    of row vectors of X and another set of row vectors of Y and apply a
+#    reduction on top. The canonical example is the brute-force computation
+#    of the top k nearest neighbors by leveraging the arg-k-min reduction.
+#
+#    The reduction takes a matrix of pairwise distances between rows of X and Y
+#    as input and outputs an aggregate data-structure for each row of X. The
+#    aggregate values are typically smaller than the number of rows in Y, hence
+#    the term reduction.
+#
+#    For computational reasons, the reduction are performed on the fly on chunks
+#    of rows of X and Y so as to keep intermediate data-structures in CPU cache
+#    and avoid unnecessary round trips of large distance arrays with the RAM
+#    that would otherwise severely degrade the speed by making the overall
+#    processing memory-bound.
+#
+#    Finally, the routines follow a generic parallelization template to process
+#    chunks of data with OpenMP loops (via Cython prange), either on rows of X
+#    or rows of Y depending on their respective sizes.
+#
+#
+# Dispatching to specialized implementations
+# ------------------------------------------
+#
+#    Dispatchers are meant to be used in the Python code. Under the hood, a
+#    dispatcher must only define the logic to choose at runtime to the correct
+#    dtype-specialized :class:`BaseDistancesReductionDispatcher` implementation based
+#    on the dtype of X and of Y.
+#
+#
+# High-level diagram
+# ------------------
+#
+#    Legend:
+#
+#      A ---âŠŗ B: A inherits from B
+#      A ---x B: A dispatches to B
+#
+#
+#                                      (base dispatcher)
+#                               BaseDistancesReductionDispatcher
+#                                              ∆
+#                                              |
+#                                              |
+#           +------------------+---------------+---------------+------------------+
+#           |                  |                               |                  |
+#           |             (dispatcher)                    (dispatcher)            |
+#           |               ArgKmin                      RadiusNeighbors          |
+#           |                  |                               |                  |
+#           |                  |                               |                  |
+#           |                  |     (float{32,64} implem.)    |                  |
+#           |                  | BaseDistancesReduction{32,64} |                  |
+#           |                  |               ∆               |                  |
+#      (dispatcher)            |               |               |             (dispatcher)
+#    ArgKminClassMode          |               |               |        RadiusNeighborsClassMode
+#           |                  |    +----------+----------+    |                  |
+#           |                  |    |                     |    |                  |
+#           |                  |    |                     |    |                  |
+#           |                  x    |                     |    x                  |
+#           |     +-------âŠŗ ArgKmin{32,64}         RadiusNeighbors{32,64} ⊲---+   |
+#           x     |            |    ∆                     ∆    |              |   x
+#   ArgKminClassMode{32,64}    |    |                     |    |   RadiusNeighborsClassMode{32,64}
+# ===================================== Specializations ============================================
+#                              |    |                     |    |
+#                              |    |                     |    |
+#                              x    |                     |    x
+#                      EuclideanArgKmin{32,64}    EuclideanRadiusNeighbors{32,64}
+#
+#
+#    For instance :class:`ArgKmin` dispatches to:
+#      - :class:`ArgKmin64` if X and Y are two `float64` array-likes
+#      - :class:`ArgKmin32` if X and Y are two `float32` array-likes
+#
+#    In addition, if the metric parameter is set to "euclidean" or "sqeuclidean",
+#    then some direct subclass of `BaseDistancesReduction{32,64}` further dispatches
+#    to one of their subclass for euclidean-specialized implementation. For instance,
+#    :class:`ArgKmin64` dispatches to :class:`EuclideanArgKmin64`.
+#
+#    Those Euclidean-specialized implementations relies on optimal implementations of
+#    a decomposition of the squared euclidean distance matrix into a sum of three terms
+#    (see :class:`MiddleTermComputer{32,64}`).
+#
+
+from ._dispatcher import (
+    ArgKmin,
+    ArgKminClassMode,
+    BaseDistancesReductionDispatcher,
+    RadiusNeighbors,
+    RadiusNeighborsClassMode,
+    sqeuclidean_row_norms,
+)
+
+__all__ = [
+    "ArgKmin",
+    "ArgKminClassMode",
+    "BaseDistancesReductionDispatcher",
+    "RadiusNeighbors",
+    "RadiusNeighborsClassMode",
+    "sqeuclidean_row_norms",
+]
+
+# ruff: noqa: E501
diff --git a/sklearn/metrics/_pairwise_distances_reduction/_argkmin.pxd.tp b/sklearn/metrics/_pairwise_distances_reduction/_argkmin.pxd.tp
new file mode 100644
index 0000000000000..f3a9ce96e64c0
--- /dev/null
+++ b/sklearn/metrics/_pairwise_distances_reduction/_argkmin.pxd.tp
@@ -0,0 +1,31 @@
+from ...utils._typedefs cimport intp_t, float64_t
+
+{{for name_suffix in ['64', '32']}}
+
+from ._base cimport BaseDistancesReduction{{name_suffix}}
+from ._middle_term_computer cimport MiddleTermComputer{{name_suffix}}
+
+cdef class ArgKmin{{name_suffix}}(BaseDistancesReduction{{name_suffix}}):
+    """float{{name_suffix}} implementation of the ArgKmin."""
+
+    cdef:
+        intp_t k
+
+        intp_t[:, ::1] argkmin_indices
+        float64_t[:, ::1] argkmin_distances
+
+        # Used as array of pointers to private datastructures used in threads.
+        float64_t ** heaps_r_distances_chunks
+        intp_t ** heaps_indices_chunks
+
+
+cdef class EuclideanArgKmin{{name_suffix}}(ArgKmin{{name_suffix}}):
+    """EuclideanDistance-specialisation of ArgKmin{{name_suffix}}."""
+    cdef:
+        MiddleTermComputer{{name_suffix}} middle_term_computer
+        const float64_t[::1] X_norm_squared
+        const float64_t[::1] Y_norm_squared
+
+        bint use_squared_distances
+
+{{endfor}}
diff --git a/sklearn/metrics/_pairwise_distances_reduction/_argkmin.pyx.tp b/sklearn/metrics/_pairwise_distances_reduction/_argkmin.pyx.tp
new file mode 100644
index 0000000000000..c21717554e94b
--- /dev/null
+++ b/sklearn/metrics/_pairwise_distances_reduction/_argkmin.pyx.tp
@@ -0,0 +1,512 @@
+from libc.stdlib cimport free, malloc
+from libc.float cimport DBL_MAX
+from cython cimport final
+from cython.parallel cimport parallel, prange
+
+from ...utils._heap cimport heap_push
+from ...utils._sorting cimport simultaneous_sort
+from ...utils._typedefs cimport intp_t, float64_t
+
+import numpy as np
+import warnings
+
+from numbers import Integral
+from scipy.sparse import issparse
+from ...utils import check_array, check_scalar
+from ...utils.fixes import _in_unstable_openblas_configuration
+from ...utils.parallel import _get_threadpool_controller
+
+{{for name_suffix in ['64', '32']}}
+
+from ._base cimport (
+    BaseDistancesReduction{{name_suffix}},
+    _sqeuclidean_row_norms{{name_suffix}},
+)
+
+from ._datasets_pair cimport DatasetsPair{{name_suffix}}
+
+from ._middle_term_computer cimport MiddleTermComputer{{name_suffix}}
+
+
+cdef class ArgKmin{{name_suffix}}(BaseDistancesReduction{{name_suffix}}):
+    """float{{name_suffix}} implementation of the ArgKmin."""
+
+    @classmethod
+    def compute(
+        cls,
+        X,
+        Y,
+        intp_t k,
+        metric="euclidean",
+        chunk_size=None,
+        dict metric_kwargs=None,
+        str strategy=None,
+        bint return_distance=False,
+    ):
+        """Compute the argkmin reduction.
+
+        This classmethod is responsible for introspecting the arguments
+        values to dispatch to the most appropriate implementation of
+        :class:`ArgKmin{{name_suffix}}`.
+
+        This allows decoupling the API entirely from the implementation details
+        whilst maintaining RAII: all temporarily allocated datastructures necessary
+        for the concrete implementation are therefore freed when this classmethod
+        returns.
+
+        No instance should directly be created outside of this class method.
+        """
+        # Limit the number of threads in second level of nested parallelism for BLAS
+        # to avoid threads over-subscription (in DOT or GEMM for instance).
+        with _get_threadpool_controller().limit(limits=1, user_api='blas'):
+          if metric in ("euclidean", "sqeuclidean"):
+              # Specialized implementation of ArgKmin for the Euclidean distance
+              # for the dense-dense and sparse-sparse cases.
+              # This implementation computes the distances by chunk using
+              # a decomposition of the Squared Euclidean distance.
+              # This specialisation has an improved arithmetic intensity for both
+              # the dense and sparse settings, allowing in most case speed-ups of
+              # several orders of magnitude compared to the generic ArgKmin
+              # implementation.
+              # Note that squared norms of X and Y are precomputed in the
+              # constructor of this class by issuing BLAS calls that may use
+              # multithreading (depending on the BLAS implementation), hence calling
+              # the constructor needs to be protected under the threadpool_limits
+              # context, along with the main calls to _parallel_on_Y and
+              # _parallel_on_X.
+              # For more information see MiddleTermComputer.
+              use_squared_distances = metric == "sqeuclidean"
+              pda = EuclideanArgKmin{{name_suffix}}(
+                  X=X, Y=Y, k=k,
+                  use_squared_distances=use_squared_distances,
+                  chunk_size=chunk_size,
+                  strategy=strategy,
+                  metric_kwargs=metric_kwargs,
+              )
+          else:
+              # Fall back on a generic implementation that handles most scipy
+              # metrics by computing the distances between 2 vectors at a time.
+              pda = ArgKmin{{name_suffix}}(
+                  datasets_pair=DatasetsPair{{name_suffix}}.get_for(X, Y, metric, metric_kwargs),
+                  k=k,
+                  chunk_size=chunk_size,
+                  strategy=strategy,
+              )
+
+          if pda.execute_in_parallel_on_Y:
+              pda._parallel_on_Y()
+          else:
+              pda._parallel_on_X()
+
+        return pda._finalize_results(return_distance)
+
+    def __init__(
+        self,
+        DatasetsPair{{name_suffix}} datasets_pair,
+        chunk_size=None,
+        strategy=None,
+        intp_t k=1,
+    ):
+        super().__init__(
+            datasets_pair=datasets_pair,
+            chunk_size=chunk_size,
+            strategy=strategy,
+        )
+        self.k = check_scalar(k, "k", Integral, min_val=1)
+
+        # Allocating pointers to datastructures but not the datastructures themselves.
+        # There are as many pointers as effective threads.
+        #
+        # For the sake of explicitness:
+        #   - when parallelizing on X, the pointers of those heaps are referencing
+        #   (with proper offsets) addresses of the two main heaps (see below)
+        #   - when parallelizing on Y, the pointers of those heaps are referencing
+        #   small heaps which are thread-wise-allocated and whose content will be
+        #   merged with the main heaps'.
+        self.heaps_r_distances_chunks =  malloc(
+            sizeof(float64_t *) * self.chunks_n_threads
+        )
+        self.heaps_indices_chunks =  malloc(
+            sizeof(intp_t *) * self.chunks_n_threads
+        )
+
+        # Main heaps which will be returned as results by `ArgKmin{{name_suffix}}.compute`.
+        self.argkmin_indices = np.full((self.n_samples_X, self.k), 0, dtype=np.intp)
+        self.argkmin_distances = np.full((self.n_samples_X, self.k), DBL_MAX, dtype=np.float64)
+
+    def __dealloc__(self):
+        if self.heaps_indices_chunks is not NULL:
+            free(self.heaps_indices_chunks)
+
+        if self.heaps_r_distances_chunks is not NULL:
+            free(self.heaps_r_distances_chunks)
+
+    cdef void _compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil:
+        cdef:
+            intp_t i, j
+            intp_t n_samples_X = X_end - X_start
+            intp_t n_samples_Y = Y_end - Y_start
+            float64_t *heaps_r_distances = self.heaps_r_distances_chunks[thread_num]
+            intp_t *heaps_indices = self.heaps_indices_chunks[thread_num]
+
+        # Pushing the distances and their associated indices on a heap
+        # which by construction will keep track of the argkmin.
+        for i in range(n_samples_X):
+            for j in range(n_samples_Y):
+                heap_push(
+                    values=heaps_r_distances + i * self.k,
+                    indices=heaps_indices + i * self.k,
+                    size=self.k,
+                    val=self.datasets_pair.surrogate_dist(X_start + i, Y_start + j),
+                    val_idx=Y_start + j,
+                )
+
+    cdef void _parallel_on_X_init_chunk(
+        self,
+        intp_t thread_num,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil:
+        # As this strategy is embarrassingly parallel, we can set each
+        # thread's heaps pointer to the proper position on the main heaps.
+        self.heaps_r_distances_chunks[thread_num] = &self.argkmin_distances[X_start, 0]
+        self.heaps_indices_chunks[thread_num] = &self.argkmin_indices[X_start, 0]
+
+    cdef void _parallel_on_X_prange_iter_finalize(
+        self,
+        intp_t thread_num,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil:
+        cdef:
+            intp_t idx
+
+        # Sorting the main heaps portion associated to `X[X_start:X_end]`
+        # in ascending order w.r.t the distances.
+        for idx in range(X_end - X_start):
+            simultaneous_sort(
+                self.heaps_r_distances_chunks[thread_num] + idx * self.k,
+                self.heaps_indices_chunks[thread_num] + idx * self.k,
+                self.k
+            )
+
+    cdef void _parallel_on_Y_init(
+        self,
+    ) noexcept nogil:
+        cdef:
+            # Maximum number of scalar elements (the last chunks can be smaller)
+            intp_t heaps_size = self.X_n_samples_chunk * self.k
+            intp_t thread_num
+
+        # The allocation is done in parallel for data locality purposes: this way
+        # the heaps used in each threads are allocated in pages which are closer
+        # to the CPU core used by the thread.
+        # See comments about First Touch Placement Policy:
+        # https://www.openmp.org/wp-content/uploads/openmp-webinar-vanderPas-20210318.pdf #noqa
+        for thread_num in prange(self.chunks_n_threads, schedule='static', nogil=True,
+                                 num_threads=self.chunks_n_threads):
+            # As chunks of X are shared across threads, so must their
+            # heaps. To solve this, each thread has its own heaps
+            # which are then synchronised back in the main ones.
+            self.heaps_r_distances_chunks[thread_num] =  malloc(
+                heaps_size * sizeof(float64_t)
+            )
+            self.heaps_indices_chunks[thread_num] =  malloc(
+                heaps_size * sizeof(intp_t)
+            )
+
+    cdef void _parallel_on_Y_parallel_init(
+        self,
+        intp_t thread_num,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil:
+        # Initialising heaps (memset can't be used here)
+        for idx in range(self.X_n_samples_chunk * self.k):
+            self.heaps_r_distances_chunks[thread_num][idx] = DBL_MAX
+            self.heaps_indices_chunks[thread_num][idx] = -1
+
+    @final
+    cdef void _parallel_on_Y_synchronize(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil:
+        cdef:
+            intp_t idx, jdx, thread_num
+        with nogil, parallel(num_threads=self.effective_n_threads):
+            # Synchronising the thread heaps with the main heaps.
+            # This is done in parallel sample-wise (no need for locks).
+            #
+            # This might break each thread's data locality as each heap which
+            # was allocated in a thread is being now being used in several threads.
+            #
+            # Still, this parallel pattern has shown to be efficient in practice.
+            for idx in prange(X_end - X_start, schedule="static"):
+                for thread_num in range(self.chunks_n_threads):
+                    for jdx in range(self.k):
+                        heap_push(
+                            values=&self.argkmin_distances[X_start + idx, 0],
+                            indices=&self.argkmin_indices[X_start + idx, 0],
+                            size=self.k,
+                            val=self.heaps_r_distances_chunks[thread_num][idx * self.k + jdx],
+                            val_idx=self.heaps_indices_chunks[thread_num][idx * self.k + jdx],
+                        )
+
+    cdef void _parallel_on_Y_finalize(
+        self,
+    ) noexcept nogil:
+        cdef:
+            intp_t idx, thread_num
+
+        with nogil, parallel(num_threads=self.chunks_n_threads):
+            # Deallocating temporary datastructures
+            for thread_num in prange(self.chunks_n_threads, schedule='static'):
+                free(self.heaps_r_distances_chunks[thread_num])
+                free(self.heaps_indices_chunks[thread_num])
+
+            # Sorting the main in ascending order w.r.t the distances.
+            # This is done in parallel sample-wise (no need for locks).
+            for idx in prange(self.n_samples_X, schedule='static'):
+                simultaneous_sort(
+                    &self.argkmin_distances[idx, 0],
+                    &self.argkmin_indices[idx, 0],
+                    self.k,
+                )
+        return
+
+    cdef void compute_exact_distances(self) noexcept nogil:
+        cdef:
+            intp_t i, j
+            float64_t[:, ::1] distances = self.argkmin_distances
+        for i in prange(self.n_samples_X, schedule='static', nogil=True,
+                        num_threads=self.effective_n_threads):
+            for j in range(self.k):
+                distances[i, j] = self.datasets_pair.distance_metric._rdist_to_dist(
+                    # Guard against potential -0., causing nan production.
+                    max(distances[i, j], 0.)
+                )
+
+    def _finalize_results(self, bint return_distance=False):
+        if return_distance:
+            # We need to recompute distances because we relied on
+            # surrogate distances for the reduction.
+            self.compute_exact_distances()
+
+            # Values are returned identically to the way `KNeighborsMixin.kneighbors`
+            # returns values. This is counter-intuitive but this allows not using
+            # complex adaptations where `ArgKmin.compute` is called.
+            return np.asarray(self.argkmin_distances), np.asarray(self.argkmin_indices)
+
+        return np.asarray(self.argkmin_indices)
+
+
+cdef class EuclideanArgKmin{{name_suffix}}(ArgKmin{{name_suffix}}):
+    """EuclideanDistance-specialisation of ArgKmin{{name_suffix}}."""
+
+    @classmethod
+    def is_usable_for(cls, X, Y, metric) -> bool:
+        return (ArgKmin{{name_suffix}}.is_usable_for(X, Y, metric) and
+                not _in_unstable_openblas_configuration())
+
+    def __init__(
+        self,
+        X,
+        Y,
+        intp_t k,
+        bint use_squared_distances=False,
+        chunk_size=None,
+        strategy=None,
+        metric_kwargs=None,
+    ):
+        if (
+            isinstance(metric_kwargs, dict) and
+            (metric_kwargs.keys() - {"X_norm_squared", "Y_norm_squared"})
+        ):
+            warnings.warn(
+                f"Some metric_kwargs have been passed ({metric_kwargs}) but aren't "
+                f"usable for this case (EuclideanArgKmin64) and will be ignored.",
+                UserWarning,
+                stacklevel=3,
+            )
+
+        super().__init__(
+            # The datasets pair here is used for exact distances computations
+            datasets_pair=DatasetsPair{{name_suffix}}.get_for(X, Y, metric="euclidean"),
+            chunk_size=chunk_size,
+            strategy=strategy,
+            k=k,
+        )
+        cdef:
+            intp_t dist_middle_terms_chunks_size = self.Y_n_samples_chunk * self.X_n_samples_chunk
+
+        self.middle_term_computer = MiddleTermComputer{{name_suffix}}.get_for(
+            X,
+            Y,
+            self.effective_n_threads,
+            self.chunks_n_threads,
+            dist_middle_terms_chunks_size,
+            n_features=X.shape[1],
+            chunk_size=self.chunk_size,
+        )
+
+        if metric_kwargs is not None and "Y_norm_squared" in metric_kwargs:
+            self.Y_norm_squared = check_array(
+                metric_kwargs.pop("Y_norm_squared"),
+                ensure_2d=False,
+                input_name="Y_norm_squared",
+                dtype=np.float64,
+            )
+        else:
+            self.Y_norm_squared = _sqeuclidean_row_norms{{name_suffix}}(
+                Y,
+                self.effective_n_threads,
+            )
+
+        if metric_kwargs is not None and "X_norm_squared" in metric_kwargs:
+            self.X_norm_squared = check_array(
+                metric_kwargs.pop("X_norm_squared"),
+                ensure_2d=False,
+                input_name="X_norm_squared",
+                dtype=np.float64,
+            )
+        else:
+            # Do not recompute norms if datasets are identical.
+            self.X_norm_squared = (
+                self.Y_norm_squared if X is Y else
+                _sqeuclidean_row_norms{{name_suffix}}(
+                    X,
+                    self.effective_n_threads,
+                )
+            )
+
+        self.use_squared_distances = use_squared_distances
+
+    @final
+    cdef void compute_exact_distances(self) noexcept nogil:
+        if not self.use_squared_distances:
+            ArgKmin{{name_suffix}}.compute_exact_distances(self)
+
+    @final
+    cdef void _parallel_on_X_parallel_init(
+        self,
+        intp_t thread_num,
+    ) noexcept nogil:
+        ArgKmin{{name_suffix}}._parallel_on_X_parallel_init(self, thread_num)
+        self.middle_term_computer._parallel_on_X_parallel_init(thread_num)
+
+    @final
+    cdef void _parallel_on_X_init_chunk(
+        self,
+        intp_t thread_num,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil:
+        ArgKmin{{name_suffix}}._parallel_on_X_init_chunk(self, thread_num, X_start, X_end)
+        self.middle_term_computer._parallel_on_X_init_chunk(thread_num, X_start, X_end)
+
+    @final
+    cdef void _parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil:
+        ArgKmin{{name_suffix}}._parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
+            self,
+            X_start, X_end,
+            Y_start, Y_end,
+            thread_num,
+        )
+        self.middle_term_computer._parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
+            X_start, X_end, Y_start, Y_end, thread_num,
+        )
+
+    @final
+    cdef void _parallel_on_Y_init(
+        self,
+    ) noexcept nogil:
+        ArgKmin{{name_suffix}}._parallel_on_Y_init(self)
+        self.middle_term_computer._parallel_on_Y_init()
+
+    @final
+    cdef void _parallel_on_Y_parallel_init(
+        self,
+        intp_t thread_num,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil:
+        ArgKmin{{name_suffix}}._parallel_on_Y_parallel_init(self, thread_num, X_start, X_end)
+        self.middle_term_computer._parallel_on_Y_parallel_init(thread_num, X_start, X_end)
+
+    @final
+    cdef void _parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil:
+        ArgKmin{{name_suffix}}._parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
+            self,
+            X_start, X_end,
+            Y_start, Y_end,
+            thread_num,
+        )
+        self.middle_term_computer._parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
+            X_start, X_end, Y_start, Y_end, thread_num
+        )
+
+    @final
+    cdef void _compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil:
+        cdef:
+            intp_t i, j
+            float64_t sqeuclidean_dist_i_j
+            intp_t n_X = X_end - X_start
+            intp_t n_Y = Y_end - Y_start
+            float64_t * dist_middle_terms = self.middle_term_computer._compute_dist_middle_terms(
+                X_start, X_end, Y_start, Y_end, thread_num
+            )
+            float64_t * heaps_r_distances = self.heaps_r_distances_chunks[thread_num]
+            intp_t * heaps_indices = self.heaps_indices_chunks[thread_num]
+
+        # Pushing the distance and their associated indices on heaps
+        # which keep tracks of the argkmin.
+        for i in range(n_X):
+            for j in range(n_Y):
+                sqeuclidean_dist_i_j = (
+                    self.X_norm_squared[i + X_start] +
+                    dist_middle_terms[i * n_Y + j] +
+                    self.Y_norm_squared[j + Y_start]
+                )
+
+                # Catastrophic cancellation might cause -0. to be present,
+                # e.g. when computing d(x_i, y_i) when X is Y.
+                sqeuclidean_dist_i_j = max(0., sqeuclidean_dist_i_j)
+
+                heap_push(
+                    values=heaps_r_distances + i * self.k,
+                    indices=heaps_indices + i * self.k,
+                    size=self.k,
+                    val=sqeuclidean_dist_i_j,
+                    val_idx=j + Y_start,
+                )
+
+{{endfor}}
diff --git a/sklearn/metrics/_pairwise_distances_reduction/_argkmin_classmode.pyx.tp b/sklearn/metrics/_pairwise_distances_reduction/_argkmin_classmode.pyx.tp
new file mode 100644
index 0000000000000..51fb745dca784
--- /dev/null
+++ b/sklearn/metrics/_pairwise_distances_reduction/_argkmin_classmode.pyx.tp
@@ -0,0 +1,182 @@
+from cython cimport floating, integral
+from cython.parallel cimport parallel, prange
+from libcpp.map cimport map as cpp_map, pair as cpp_pair
+from libc.stdlib cimport free
+
+from ...utils._typedefs cimport intp_t, float64_t
+from ...utils.parallel import _get_threadpool_controller
+
+import numpy as np
+from scipy.sparse import issparse
+from ._classmode cimport WeightingStrategy
+
+{{for name_suffix in ["32", "64"]}}
+from ._argkmin cimport ArgKmin{{name_suffix}}
+from ._datasets_pair cimport DatasetsPair{{name_suffix}}
+
+cdef class ArgKminClassMode{{name_suffix}}(ArgKmin{{name_suffix}}):
+    """
+    {{name_suffix}}bit implementation of ArgKminClassMode.
+    """
+    cdef:
+        const intp_t[:] Y_labels,
+        const intp_t[:] unique_Y_labels
+        float64_t[:, :] class_scores
+        cpp_map[intp_t, intp_t] labels_to_index
+        WeightingStrategy weight_type
+
+    @classmethod
+    def compute(
+        cls,
+        X,
+        Y,
+        intp_t k,
+        weights,
+        Y_labels,
+        unique_Y_labels,
+        str metric="euclidean",
+        chunk_size=None,
+        dict metric_kwargs=None,
+        str strategy=None,
+    ):
+        """Compute the argkmin reduction with Y_labels.
+
+        This classmethod is responsible for introspecting the arguments
+        values to dispatch to the most appropriate implementation of
+        :class:`ArgKminClassMode{{name_suffix}}`.
+
+        This allows decoupling the API entirely from the implementation details
+        whilst maintaining RAII: all temporarily allocated datastructures necessary
+        for the concrete implementation are therefore freed when this classmethod
+        returns.
+
+        No instance _must_ directly be created outside of this class method.
+        """
+        # Use a generic implementation that handles most scipy
+        # metrics by computing the distances between 2 vectors at a time.
+        pda = ArgKminClassMode{{name_suffix}}(
+            datasets_pair=DatasetsPair{{name_suffix}}.get_for(X, Y, metric, metric_kwargs),
+            k=k,
+            chunk_size=chunk_size,
+            strategy=strategy,
+            weights=weights,
+            Y_labels=Y_labels,
+            unique_Y_labels=unique_Y_labels,
+        )
+
+        # Limit the number of threads in second level of nested parallelism for BLAS
+        # to avoid threads over-subscription (in GEMM for instance).
+        with _get_threadpool_controller().limit(limits=1, user_api="blas"):
+            if pda.execute_in_parallel_on_Y:
+                pda._parallel_on_Y()
+            else:
+                pda._parallel_on_X()
+
+        return pda._finalize_results()
+
+    def __init__(
+        self,
+        DatasetsPair{{name_suffix}} datasets_pair,
+        const intp_t[:] Y_labels,
+        const intp_t[:] unique_Y_labels,
+        chunk_size=None,
+        strategy=None,
+        intp_t k=1,
+        weights=None,
+    ):
+        super().__init__(
+            datasets_pair=datasets_pair,
+            chunk_size=chunk_size,
+            strategy=strategy,
+            k=k,
+        )
+
+        if weights == "uniform":
+            self.weight_type = WeightingStrategy.uniform
+        elif weights == "distance":
+            self.weight_type = WeightingStrategy.distance
+        else:
+            self.weight_type = WeightingStrategy.callable
+        self.Y_labels = Y_labels
+
+        self.unique_Y_labels = unique_Y_labels
+
+        cdef intp_t idx, neighbor_class_idx
+        # Map from set of unique labels to their indices in `class_scores`
+        # Buffer used in building a histogram for one-pass weighted mode
+        self.class_scores = np.zeros(
+            (self.n_samples_X, unique_Y_labels.shape[0]), dtype=np.float64,
+        )
+
+    def _finalize_results(self):
+        probabilities = np.asarray(self.class_scores)
+        probabilities /= probabilities.sum(axis=1, keepdims=True)
+        return probabilities
+
+    cdef inline void weighted_histogram_mode(
+        self,
+        intp_t sample_index,
+        intp_t* indices,
+        float64_t* distances,
+   ) noexcept nogil:
+        cdef:
+            intp_t neighbor_idx, neighbor_class_idx, label_index, multi_output_index
+            float64_t score_incr = 1
+            # TODO: Implement other WeightingStrategy values
+            bint use_distance_weighting = (
+                self.weight_type == WeightingStrategy.distance
+            )
+
+        # Iterate through the sample k-nearest neighbours
+        for neighbor_rank in range(self.k):
+            # Absolute indice of the neighbor_rank-th Nearest Neighbors
+            # in range [0, n_samples_Y)
+            # TODO: inspect if it worth permuting this condition
+            # and the for-loop above for improved branching.
+            if use_distance_weighting:
+                score_incr = 1 / distances[neighbor_rank]
+            neighbor_idx = indices[neighbor_rank]
+            neighbor_class_idx = self.Y_labels[neighbor_idx]
+            self.class_scores[sample_index][neighbor_class_idx] += score_incr
+        return
+
+    cdef void _parallel_on_X_prange_iter_finalize(
+        self,
+        intp_t thread_num,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil:
+        cdef:
+            intp_t idx, sample_index
+        for idx in range(X_end - X_start):
+            # One-pass top-one weighted mode
+            # Compute the absolute index in [0, n_samples_X)
+            sample_index = X_start + idx
+            self.weighted_histogram_mode(
+                sample_index,
+                &self.heaps_indices_chunks[thread_num][idx * self.k],
+                &self.heaps_r_distances_chunks[thread_num][idx * self.k],
+            )
+        return
+
+    cdef void _parallel_on_Y_finalize(
+        self,
+    ) noexcept nogil:
+        cdef:
+            intp_t sample_index, thread_num
+
+        with nogil, parallel(num_threads=self.chunks_n_threads):
+            # Deallocating temporary datastructures
+            for thread_num in prange(self.chunks_n_threads, schedule='static'):
+                free(self.heaps_r_distances_chunks[thread_num])
+                free(self.heaps_indices_chunks[thread_num])
+
+            for sample_index in prange(self.n_samples_X, schedule='static'):
+                self.weighted_histogram_mode(
+                    sample_index,
+                    &self.argkmin_indices[sample_index][0],
+                    &self.argkmin_distances[sample_index][0],
+                )
+        return
+
+{{endfor}}
diff --git a/sklearn/metrics/_pairwise_distances_reduction/_base.pxd.tp b/sklearn/metrics/_pairwise_distances_reduction/_base.pxd.tp
new file mode 100644
index 0000000000000..9578129993c37
--- /dev/null
+++ b/sklearn/metrics/_pairwise_distances_reduction/_base.pxd.tp
@@ -0,0 +1,135 @@
+from cython cimport final
+
+from ...utils._typedefs cimport intp_t, float64_t
+
+{{for name_suffix in ['64', '32']}}
+
+from ._datasets_pair cimport DatasetsPair{{name_suffix}}
+
+
+cpdef float64_t[::1] _sqeuclidean_row_norms{{name_suffix}}(
+    X,
+    intp_t num_threads,
+)
+
+cdef class BaseDistancesReduction{{name_suffix}}:
+    """
+    Base float{{name_suffix}} implementation template of the pairwise-distances
+    reduction backends.
+
+    Implementations inherit from this template and may override the several
+    defined hooks as needed in order to easily extend functionality with
+    minimal redundant code.
+    """
+
+    cdef:
+        readonly DatasetsPair{{name_suffix}} datasets_pair
+
+        # The number of threads that can be used is stored in effective_n_threads.
+        #
+        # The number of threads to use in the parallelization strategy
+        # (i.e. parallel_on_X or parallel_on_Y) can be smaller than effective_n_threads:
+        # for small datasets, fewer threads might be needed to loop over pair of chunks.
+        #
+        # Hence, the number of threads that _will_ be used for looping over chunks
+        # is stored in chunks_n_threads, allowing solely using what we need.
+        #
+        # Thus, an invariant is:
+        #
+        #                 chunks_n_threads <= effective_n_threads
+        #
+        intp_t effective_n_threads
+        intp_t chunks_n_threads
+
+        intp_t n_samples_chunk, chunk_size
+
+        intp_t n_samples_X, X_n_samples_chunk, X_n_chunks, X_n_samples_last_chunk
+        intp_t n_samples_Y, Y_n_samples_chunk, Y_n_chunks, Y_n_samples_last_chunk
+
+        bint execute_in_parallel_on_Y
+
+    @final
+    cdef void _parallel_on_X(self) noexcept nogil
+
+    @final
+    cdef void _parallel_on_Y(self) noexcept nogil
+
+    # Placeholder methods which have to be implemented
+
+    cdef void _compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil
+
+
+    # Placeholder methods which can be implemented
+
+    cdef void compute_exact_distances(self) noexcept nogil
+
+    cdef void _parallel_on_X_parallel_init(
+        self,
+        intp_t thread_num,
+    ) noexcept nogil
+
+    cdef void _parallel_on_X_init_chunk(
+        self,
+        intp_t thread_num,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil
+
+    cdef void _parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil
+
+    cdef void _parallel_on_X_prange_iter_finalize(
+        self,
+        intp_t thread_num,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil
+
+    cdef void _parallel_on_X_parallel_finalize(
+        self,
+        intp_t thread_num
+    ) noexcept nogil
+
+    cdef void _parallel_on_Y_init(
+        self,
+    ) noexcept nogil
+
+    cdef void _parallel_on_Y_parallel_init(
+        self,
+        intp_t thread_num,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil
+
+    cdef void _parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil
+
+    cdef void _parallel_on_Y_synchronize(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil
+
+    cdef void _parallel_on_Y_finalize(
+        self,
+    ) noexcept nogil
+{{endfor}}
diff --git a/sklearn/metrics/_pairwise_distances_reduction/_base.pyx.tp b/sklearn/metrics/_pairwise_distances_reduction/_base.pyx.tp
new file mode 100644
index 0000000000000..2bbfd74e2c2c3
--- /dev/null
+++ b/sklearn/metrics/_pairwise_distances_reduction/_base.pyx.tp
@@ -0,0 +1,504 @@
+from cython cimport final
+from cython.operator cimport dereference as deref
+from cython.parallel cimport parallel, prange
+from libcpp.vector cimport vector
+
+from ...utils._cython_blas cimport _dot
+from ...utils._openmp_helpers cimport omp_get_thread_num
+from ...utils._typedefs cimport intp_t, float32_t, float64_t, int32_t
+
+import numpy as np
+
+from scipy.sparse import issparse
+from numbers import Integral
+from sklearn import get_config
+from sklearn.utils import check_scalar
+from ...utils._openmp_helpers import _openmp_effective_n_threads
+
+#####################
+
+cdef float64_t[::1] _sqeuclidean_row_norms64_dense(
+    const float64_t[:, ::1] X,
+    intp_t num_threads,
+):
+    """Compute the squared euclidean norm of the rows of X in parallel.
+
+    This is faster than using np.einsum("ij, ij->i") even when using a single thread.
+    """
+    cdef:
+        # Casting for X to remove the const qualifier is needed because APIs
+        # exposed via scipy.linalg.cython_blas aren't reflecting the arguments'
+        # const qualifier.
+        # See: https://github.com/scipy/scipy/issues/14262
+        float64_t * X_ptr =  &X[0, 0]
+        intp_t idx = 0
+        intp_t n = X.shape[0]
+        intp_t d = X.shape[1]
+        float64_t[::1] squared_row_norms = np.empty(n, dtype=np.float64)
+
+    for idx in prange(n, schedule='static', nogil=True, num_threads=num_threads):
+        squared_row_norms[idx] = _dot(d, X_ptr + idx * d, 1, X_ptr + idx * d, 1)
+
+    return squared_row_norms
+
+
+cdef float64_t[::1] _sqeuclidean_row_norms32_dense(
+    const float32_t[:, ::1] X,
+    intp_t num_threads,
+):
+    """Compute the squared euclidean norm of the rows of X in parallel.
+
+    This is faster than using np.einsum("ij, ij->i") even when using a single thread.
+    """
+    cdef:
+        # Casting for X to remove the const qualifier is needed because APIs
+        # exposed via scipy.linalg.cython_blas aren't reflecting the arguments'
+        # const qualifier.
+        # See: https://github.com/scipy/scipy/issues/14262
+        float32_t * X_ptr =  &X[0, 0]
+        intp_t i = 0, j = 0
+        intp_t thread_num
+        intp_t n = X.shape[0]
+        intp_t d = X.shape[1]
+        float64_t[::1] squared_row_norms = np.empty(n, dtype=np.float64)
+
+        # To upcast the i-th row of X from float32 to float64
+        vector[vector[float64_t]] X_i_upcast = vector[vector[float64_t]](
+            num_threads, vector[float64_t](d)
+        )
+
+    with nogil, parallel(num_threads=num_threads):
+        thread_num = omp_get_thread_num()
+
+        for i in prange(n, schedule='static'):
+            # Upcasting the i-th row of X from float32 to float64
+            for j in range(d):
+                X_i_upcast[thread_num][j] =  deref(X_ptr + i * d + j)
+
+            squared_row_norms[i] = _dot(
+                d, X_i_upcast[thread_num].data(), 1,
+                X_i_upcast[thread_num].data(), 1,
+            )
+
+    return squared_row_norms
+
+
+cdef float64_t[::1] _sqeuclidean_row_norms64_sparse(
+    const float64_t[:] X_data,
+    const int32_t[:] X_indptr,
+    intp_t num_threads,
+):
+    cdef:
+        intp_t n = X_indptr.shape[0] - 1
+        int32_t X_i_ptr, idx = 0
+        float64_t[::1] squared_row_norms = np.zeros(n, dtype=np.float64)
+
+    for idx in prange(n, schedule='static', nogil=True, num_threads=num_threads):
+        for X_i_ptr in range(X_indptr[idx], X_indptr[idx+1]):
+            squared_row_norms[idx] += X_data[X_i_ptr] * X_data[X_i_ptr]
+
+    return squared_row_norms
+
+
+{{for name_suffix in ["64", "32"]}}
+
+from ._datasets_pair cimport DatasetsPair{{name_suffix}}
+
+
+cpdef float64_t[::1] _sqeuclidean_row_norms{{name_suffix}}(
+    X,
+    intp_t num_threads,
+):
+    if issparse(X):
+        # TODO: remove this instruction which is a cast in the float32 case
+        # by moving squared row norms computations in MiddleTermComputer.
+        X_data = np.asarray(X.data, dtype=np.float64)
+        X_indptr = np.asarray(X.indptr, dtype=np.int32)
+        return _sqeuclidean_row_norms64_sparse(X_data, X_indptr, num_threads)
+    else:
+        return _sqeuclidean_row_norms{{name_suffix}}_dense(X, num_threads)
+
+
+cdef class BaseDistancesReduction{{name_suffix}}:
+    """
+    Base float{{name_suffix}} implementation template of the pairwise-distances
+    reduction backends.
+
+    Implementations inherit from this template and may override the several
+    defined hooks as needed in order to easily extend functionality with
+    minimal redundant code.
+    """
+
+    def __init__(
+        self,
+        DatasetsPair{{name_suffix}} datasets_pair,
+        chunk_size=None,
+        strategy=None,
+     ):
+        cdef:
+            intp_t X_n_full_chunks, Y_n_full_chunks
+
+        if chunk_size is None:
+            chunk_size = get_config().get("pairwise_dist_chunk_size", 256)
+
+        self.chunk_size = check_scalar(chunk_size, "chunk_size", Integral, min_val=20)
+
+        self.effective_n_threads = _openmp_effective_n_threads()
+
+        self.datasets_pair = datasets_pair
+
+        self.n_samples_X = datasets_pair.n_samples_X()
+        self.X_n_samples_chunk = min(self.n_samples_X, self.chunk_size)
+        X_n_full_chunks = self.n_samples_X // self.X_n_samples_chunk
+        X_n_samples_remainder = self.n_samples_X % self.X_n_samples_chunk
+        self.X_n_chunks = X_n_full_chunks + (X_n_samples_remainder != 0)
+
+        if X_n_samples_remainder != 0:
+            self.X_n_samples_last_chunk = X_n_samples_remainder
+        else:
+            self.X_n_samples_last_chunk = self.X_n_samples_chunk
+
+        self.n_samples_Y = datasets_pair.n_samples_Y()
+        self.Y_n_samples_chunk = min(self.n_samples_Y, self.chunk_size)
+        Y_n_full_chunks = self.n_samples_Y // self.Y_n_samples_chunk
+        Y_n_samples_remainder = self.n_samples_Y % self.Y_n_samples_chunk
+        self.Y_n_chunks = Y_n_full_chunks + (Y_n_samples_remainder != 0)
+
+        if Y_n_samples_remainder != 0:
+            self.Y_n_samples_last_chunk = Y_n_samples_remainder
+        else:
+            self.Y_n_samples_last_chunk = self.Y_n_samples_chunk
+
+        if strategy is None:
+            strategy = get_config().get("pairwise_dist_parallel_strategy", 'auto')
+
+        if strategy not in ('parallel_on_X', 'parallel_on_Y', 'auto'):
+            raise RuntimeError(f"strategy must be 'parallel_on_X, 'parallel_on_Y', "
+                               f"or 'auto', but currently strategy='{self.strategy}'.")
+
+        if strategy == 'auto':
+            # This is a simple heuristic whose constant for the
+            # comparison has been chosen based on experiments.
+            # parallel_on_X has less synchronization overhead than
+            # parallel_on_Y and should therefore be used whenever
+            # n_samples_X is large enough to not starve any of the
+            # available hardware threads.
+            if self.n_samples_Y < self.n_samples_X:
+                # No point to even consider parallelizing on Y in this case. This
+                # is in particular important to do this on machines with a large
+                # number of hardware threads.
+                strategy = 'parallel_on_X'
+            elif 4 * self.chunk_size * self.effective_n_threads < self.n_samples_X:
+                # If Y is larger than X, but X is still large enough to allow for
+                # parallelism, we might still want to favor parallelizing on X.
+                strategy = 'parallel_on_X'
+            else:
+                strategy = 'parallel_on_Y'
+
+        self.execute_in_parallel_on_Y = strategy == "parallel_on_Y"
+
+        # Not using less, not using more.
+        self.chunks_n_threads = min(
+            self.Y_n_chunks if self.execute_in_parallel_on_Y else self.X_n_chunks,
+            self.effective_n_threads,
+        )
+
+    @final
+    cdef void _parallel_on_X(self) noexcept nogil:
+        """Perform computation and reduction in parallel on chunks of X.
+
+        This strategy dispatches tasks statically on threads. Each task
+        processes exactly only one chunk of X, computing and reducing
+        distances matrices between vectors of this chunk and vectors of all
+        chunks of Y, one chunk of Y at a time.
+
+        This strategy is embarrassingly parallel with no intermediate data
+        structures synchronization at all.
+
+        Private datastructures are modified internally by threads.
+
+        Private template methods can be implemented on subclasses to
+        interact with those datastructures at various stages.
+        """
+        cdef:
+            intp_t Y_start, Y_end, X_start, X_end, X_chunk_idx, Y_chunk_idx
+            intp_t thread_num
+
+        with nogil, parallel(num_threads=self.chunks_n_threads):
+            thread_num = omp_get_thread_num()
+
+            # Allocating thread datastructures
+            self._parallel_on_X_parallel_init(thread_num)
+
+            for X_chunk_idx in prange(self.X_n_chunks, schedule='static'):
+                X_start = X_chunk_idx * self.X_n_samples_chunk
+                if X_chunk_idx == self.X_n_chunks - 1:
+                    X_end = X_start + self.X_n_samples_last_chunk
+                else:
+                    X_end = X_start + self.X_n_samples_chunk
+
+                # Reinitializing thread datastructures for the new X chunk
+                self._parallel_on_X_init_chunk(thread_num, X_start, X_end)
+
+                for Y_chunk_idx in range(self.Y_n_chunks):
+                    Y_start = Y_chunk_idx * self.Y_n_samples_chunk
+                    if Y_chunk_idx == self.Y_n_chunks - 1:
+                        Y_end = Y_start + self.Y_n_samples_last_chunk
+                    else:
+                        Y_end = Y_start + self.Y_n_samples_chunk
+
+                    self._parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
+                        X_start, X_end,
+                        Y_start, Y_end,
+                        thread_num,
+                    )
+
+                    self._compute_and_reduce_distances_on_chunks(
+                        X_start, X_end,
+                        Y_start, Y_end,
+                        thread_num,
+                    )
+
+                # Adjusting thread datastructures on the full pass on Y
+                self._parallel_on_X_prange_iter_finalize(thread_num, X_start, X_end)
+
+            # end: for X_chunk_idx
+
+            # Deallocating thread datastructures
+            self._parallel_on_X_parallel_finalize(thread_num)
+
+        # end: with nogil, parallel
+        return
+
+    @final
+    cdef void _parallel_on_Y(self) noexcept nogil:
+        """Perform computation and reduction in parallel on chunks of Y.
+
+        This strategy is a sequence of embarrassingly parallel subtasks:
+        chunks of X are iterated over sequentially, and for each chunk of X,
+        tasks are dispatched statically on threads. Each task processes one
+        and only one chunk of Y, computing and reducing distances matrices
+        between vectors of the chunk of X and vectors of the Y.
+
+        It comes with lock-free and parallelized intermediate data structures
+        that synchronize at each iteration of the sequential outer loop on X
+        chunks.
+
+        Private datastructures are modified internally by threads.
+
+        Private template methods can be implemented on subclasses to
+        interact with those datastructures at various stages.
+        """
+        cdef:
+            intp_t Y_start, Y_end, X_start, X_end, X_chunk_idx, Y_chunk_idx
+            intp_t thread_num
+
+        # Allocating datastructures shared by all threads
+        self._parallel_on_Y_init()
+
+        for X_chunk_idx in range(self.X_n_chunks):
+            X_start = X_chunk_idx * self.X_n_samples_chunk
+            if X_chunk_idx == self.X_n_chunks - 1:
+                X_end = X_start + self.X_n_samples_last_chunk
+            else:
+                X_end = X_start + self.X_n_samples_chunk
+
+            with nogil, parallel(num_threads=self.chunks_n_threads):
+                thread_num = omp_get_thread_num()
+
+                # Initializing datastructures used in this thread
+                self._parallel_on_Y_parallel_init(thread_num, X_start, X_end)
+
+                for Y_chunk_idx in prange(self.Y_n_chunks, schedule='static'):
+                    Y_start = Y_chunk_idx * self.Y_n_samples_chunk
+                    if Y_chunk_idx == self.Y_n_chunks - 1:
+                        Y_end = Y_start + self.Y_n_samples_last_chunk
+                    else:
+                        Y_end = Y_start + self.Y_n_samples_chunk
+
+                    self._parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
+                        X_start, X_end,
+                        Y_start, Y_end,
+                        thread_num,
+                    )
+
+                    self._compute_and_reduce_distances_on_chunks(
+                        X_start, X_end,
+                        Y_start, Y_end,
+                        thread_num,
+                    )
+                # end: prange
+
+            # end: with nogil, parallel
+
+            # Synchronizing the thread datastructures with the main ones
+            self._parallel_on_Y_synchronize(X_start, X_end)
+
+        # end: for X_chunk_idx
+        # Deallocating temporary datastructures and adjusting main datastructures
+        self._parallel_on_Y_finalize()
+        return
+
+    # Placeholder methods which have to be implemented
+
+    cdef void _compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil:
+        """Compute the pairwise distances on two chunks of X and Y and reduce them.
+
+        This is THE core computational method of BaseDistancesReduction{{name_suffix}}.
+        This must be implemented in subclasses agnostically from the parallelization
+        strategies.
+        """
+        return
+
+    def _finalize_results(self, bint return_distance):
+        """Callback adapting datastructures before returning results.
+
+        This must be implemented in subclasses.
+        """
+        return None
+
+    # Placeholder methods which can be implemented
+
+    cdef void compute_exact_distances(self) noexcept nogil:
+        """Convert rank-preserving distances to exact distances or recompute them."""
+        return
+
+    cdef void _parallel_on_X_parallel_init(
+        self,
+        intp_t thread_num,
+    ) noexcept nogil:
+        """Allocate datastructures used in a thread given its number."""
+        return
+
+    cdef void _parallel_on_X_init_chunk(
+        self,
+        intp_t thread_num,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil:
+        """Initialize datastructures used in a thread given its number.
+
+        In this method, EuclideanDistance specialisations of subclass of
+        BaseDistancesReduction _must_ call:
+
+        self.middle_term_computer._parallel_on_X_init_chunk(
+            thread_num, X_start, X_end,
+        )
+
+        to ensure the proper upcast of X[X_start:X_end] to float64 prior
+        to the reduction with float64 accumulator buffers when X.dtype is
+        float32.
+        """
+        return
+
+    cdef void _parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil:
+        """Initialize datastructures just before the _compute_and_reduce_distances_on_chunks.
+
+        In this method, EuclideanDistance specialisations of subclass of
+        BaseDistancesReduction _must_ call:
+
+        self.middle_term_computer._parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
+            X_start, X_end, Y_start, Y_end, thread_num,
+        )
+
+        to ensure the proper upcast of Y[Y_start:Y_end] to float64 prior
+        to the reduction with float64 accumulator buffers when Y.dtype is
+        float32.
+        """
+        return
+
+    cdef void _parallel_on_X_prange_iter_finalize(
+        self,
+        intp_t thread_num,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil:
+        """Interact with datastructures after a reduction on chunks."""
+        return
+
+    cdef void _parallel_on_X_parallel_finalize(
+        self,
+        intp_t thread_num
+    ) noexcept nogil:
+        """Interact with datastructures after executing all the reductions."""
+        return
+
+    cdef void _parallel_on_Y_init(
+        self,
+    ) noexcept nogil:
+        """Allocate datastructures used in all threads."""
+        return
+
+    cdef void _parallel_on_Y_parallel_init(
+        self,
+        intp_t thread_num,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil:
+        """Initialize datastructures used in a thread given its number.
+
+        In this method, EuclideanDistance specialisations of subclass of
+        BaseDistancesReduction _must_ call:
+
+        self.middle_term_computer._parallel_on_Y_parallel_init(
+            thread_num, X_start, X_end,
+        )
+
+        to ensure the proper upcast of X[X_start:X_end] to float64 prior
+        to the reduction with float64 accumulator buffers when X.dtype is
+        float32.
+        """
+        return
+
+    cdef void _parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil:
+        """Initialize datastructures just before the _compute_and_reduce_distances_on_chunks.
+
+        In this method, EuclideanDistance specialisations of subclass of
+        BaseDistancesReduction _must_ call:
+
+        self.middle_term_computer._parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
+            X_start, X_end, Y_start, Y_end, thread_num,
+        )
+
+        to ensure the proper upcast of Y[Y_start:Y_end] to float64 prior
+        to the reduction with float64 accumulator buffers when Y.dtype is
+        float32.
+        """
+        return
+
+    cdef void _parallel_on_Y_synchronize(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil:
+        """Update thread datastructures before leaving a parallel region."""
+        return
+
+    cdef void _parallel_on_Y_finalize(
+        self,
+    ) noexcept nogil:
+        """Update datastructures after executing all the reductions."""
+        return
+
+{{endfor}}
diff --git a/sklearn/metrics/_pairwise_distances_reduction/_classmode.pxd b/sklearn/metrics/_pairwise_distances_reduction/_classmode.pxd
new file mode 100644
index 0000000000000..65db044d668e8
--- /dev/null
+++ b/sklearn/metrics/_pairwise_distances_reduction/_classmode.pxd
@@ -0,0 +1,5 @@
+cpdef enum WeightingStrategy:
+    uniform = 0
+    # TODO: Implement the following options in weighted_histogram_mode
+    distance = 1
+    callable = 2
diff --git a/sklearn/metrics/_pairwise_distances_reduction/_datasets_pair.pxd.tp b/sklearn/metrics/_pairwise_distances_reduction/_datasets_pair.pxd.tp
new file mode 100644
index 0000000000000..1e57b3291a8f4
--- /dev/null
+++ b/sklearn/metrics/_pairwise_distances_reduction/_datasets_pair.pxd.tp
@@ -0,0 +1,67 @@
+{{py:
+
+implementation_specific_values = [
+    # Values are the following ones:
+    #
+    # name_suffix, INPUT_DTYPE_t, INPUT_DTYPE
+    ('64', 'DistanceMetric64', 'float64_t'),
+    ('32', 'DistanceMetric32', 'float32_t')
+]
+
+}}
+from ...utils._typedefs cimport float64_t, float32_t, int32_t, intp_t
+from ...metrics._dist_metrics cimport DistanceMetric64, DistanceMetric32, DistanceMetric
+
+{{for name_suffix, DistanceMetric, INPUT_DTYPE_t in implementation_specific_values}}
+
+
+cdef class DatasetsPair{{name_suffix}}:
+    cdef:
+        {{DistanceMetric}} distance_metric
+        intp_t n_features
+
+    cdef intp_t n_samples_X(self) noexcept nogil
+
+    cdef intp_t n_samples_Y(self) noexcept nogil
+
+    cdef float64_t dist(self, intp_t i, intp_t j) noexcept nogil
+
+    cdef float64_t surrogate_dist(self, intp_t i, intp_t j) noexcept nogil
+
+
+cdef class DenseDenseDatasetsPair{{name_suffix}}(DatasetsPair{{name_suffix}}):
+    cdef:
+        const {{INPUT_DTYPE_t}}[:, ::1] X
+        const {{INPUT_DTYPE_t}}[:, ::1] Y
+
+
+cdef class SparseSparseDatasetsPair{{name_suffix}}(DatasetsPair{{name_suffix}}):
+    cdef:
+        const {{INPUT_DTYPE_t}}[:] X_data
+        const int32_t[::1] X_indices
+        const int32_t[::1] X_indptr
+
+        const {{INPUT_DTYPE_t}}[:] Y_data
+        const int32_t[::1] Y_indices
+        const int32_t[::1] Y_indptr
+
+
+cdef class SparseDenseDatasetsPair{{name_suffix}}(DatasetsPair{{name_suffix}}):
+    cdef:
+        const {{INPUT_DTYPE_t}}[:] X_data
+        const int32_t[::1] X_indices
+        const int32_t[::1] X_indptr
+
+        const {{INPUT_DTYPE_t}}[:] Y_data
+        const int32_t[::1] Y_indices
+        intp_t n_Y
+
+
+cdef class DenseSparseDatasetsPair{{name_suffix}}(DatasetsPair{{name_suffix}}):
+    cdef:
+        # As distance metrics are commutative, we can simply rely
+        # on the implementation of SparseDenseDatasetsPair and
+        # swap arguments.
+        DatasetsPair{{name_suffix}} datasets_pair
+
+{{endfor}}
diff --git a/sklearn/metrics/_pairwise_distances_reduction/_datasets_pair.pyx.tp b/sklearn/metrics/_pairwise_distances_reduction/_datasets_pair.pyx.tp
new file mode 100644
index 0000000000000..2c3ca44047145
--- /dev/null
+++ b/sklearn/metrics/_pairwise_distances_reduction/_datasets_pair.pyx.tp
@@ -0,0 +1,406 @@
+import copy
+
+{{py:
+
+implementation_specific_values = [
+    # Values are the following ones:
+    #
+    # name_suffix, INPUT_DTYPE_t, INPUT_DTYPE
+    ('64', 'DistanceMetric64', 'float64_t', 'np.float64'),
+    ('32', 'DistanceMetric32', 'float32_t', 'np.float32')
+]
+
+}}
+import numpy as np
+
+from cython cimport final
+
+from ...utils._typedefs cimport float64_t, float32_t, intp_t
+
+from scipy.sparse import issparse, csr_matrix
+
+{{for name_suffix, DistanceMetric, INPUT_DTYPE_t, INPUT_DTYPE in implementation_specific_values}}
+
+cdef class DatasetsPair{{name_suffix}}:
+    """Abstract class which wraps a pair of datasets (X, Y).
+
+    This class allows computing distances between a single pair of rows of
+    of X and Y at a time given the pair of their indices (i, j). This class is
+    specialized for each metric thanks to the :func:`get_for` factory classmethod.
+
+    The handling of parallelization over chunks to compute the distances
+    and aggregation for several rows at a time is done in dedicated
+    subclasses of :class:`BaseDistancesReductionDispatcher` that in-turn rely on
+    subclasses of :class:`DatasetsPair` for each pair of rows in the data. The
+    goal is to make it possible to decouple the generic parallelization and
+    aggregation logic from metric-specific computation as much as possible.
+
+    X and Y can be stored as C-contiguous np.ndarrays or CSR matrices
+    in subclasses.
+
+    This class avoids the overhead of dispatching distance computations
+    to :class:`sklearn.metrics.DistanceMetric` based on the physical
+    representation of the vectors (sparse vs. dense). It makes use of
+    cython.final to remove the overhead of dispatching method calls.
+
+    Parameters
+    ----------
+    distance_metric: {{DistanceMetric}}
+        The distance metric responsible for computing distances
+        between two vectors of (X, Y).
+    """
+
+    @classmethod
+    def get_for(
+        cls,
+        X,
+        Y,
+        metric="euclidean",
+        dict metric_kwargs=None,
+    ) -> DatasetsPair{{name_suffix}}:
+        """Return the DatasetsPair implementation for the given arguments.
+
+        Parameters
+        ----------
+        X : {ndarray, sparse matrix} of shape (n_samples_X, n_features)
+            Input data.
+            If provided as a ndarray, it must be C-contiguous.
+            If provided as a sparse matrix, it must be in CSR format.
+
+        Y : {ndarray, sparse matrix} of shape (n_samples_Y, n_features)
+            Input data.
+            If provided as a ndarray, it must be C-contiguous.
+            If provided as a sparse matrix, it must be in CSR format.
+
+        metric : str or DistanceMetric object, default='euclidean'
+            The distance metric to compute between rows of X and Y.
+            The default metric is a fast implementation of the Euclidean
+            metric. For a list of available metrics, see the documentation
+            of :class:`~sklearn.metrics.DistanceMetric`.
+
+        metric_kwargs : dict, default=None
+            Keyword arguments to pass to specified metric function.
+
+        Returns
+        -------
+        datasets_pair: DatasetsPair{{name_suffix}}
+            The suited DatasetsPair{{name_suffix}} implementation.
+        """
+        # X_norm_squared and Y_norm_squared might be propagated
+        # down to DatasetsPairs via metrics_kwargs when the Euclidean
+        # specialisations can't be used.
+        # To prevent X_norm_squared and Y_norm_squared to be passed
+        # down to DistanceMetrics (whose constructors would raise
+        # a RuntimeError), we pop them here.
+        if metric_kwargs is not None:
+            # Copying metric_kwargs not to pop "X_norm_squared"
+            # and "Y_norm_squared" where they are used
+            metric_kwargs = copy.copy(metric_kwargs)
+            metric_kwargs.pop("X_norm_squared", None)
+            metric_kwargs.pop("Y_norm_squared", None)
+        cdef:
+            {{DistanceMetric}} distance_metric = DistanceMetric.get_metric(
+                metric,
+                {{INPUT_DTYPE}},
+                **(metric_kwargs or {})
+            )
+
+        # Metric-specific checks that do not replace nor duplicate `check_array`.
+        distance_metric._validate_data(X)
+        distance_metric._validate_data(Y)
+
+        X_is_sparse = issparse(X)
+        Y_is_sparse = issparse(Y)
+
+        if not X_is_sparse and not Y_is_sparse:
+            return DenseDenseDatasetsPair{{name_suffix}}(X, Y, distance_metric)
+
+        if X_is_sparse and Y_is_sparse:
+            return SparseSparseDatasetsPair{{name_suffix}}(X, Y, distance_metric)
+
+        if X_is_sparse and not Y_is_sparse:
+            return SparseDenseDatasetsPair{{name_suffix}}(X, Y, distance_metric)
+
+        return DenseSparseDatasetsPair{{name_suffix}}(X, Y, distance_metric)
+
+    @classmethod
+    def unpack_csr_matrix(cls, X: csr_matrix):
+        """Ensure that the CSR matrix is indexed with np.int32."""
+        X_data = np.asarray(X.data, dtype={{INPUT_DTYPE}})
+        X_indices = np.asarray(X.indices, dtype=np.int32)
+        X_indptr = np.asarray(X.indptr, dtype=np.int32)
+        return X_data, X_indices, X_indptr
+
+    def __init__(self, {{DistanceMetric}} distance_metric, intp_t n_features):
+        self.distance_metric = distance_metric
+        self.n_features = n_features
+
+    cdef intp_t n_samples_X(self) noexcept nogil:
+        """Number of samples in X."""
+        # This is a abstract method.
+        # This _must_ always be overwritten in subclasses.
+        # TODO: add "with gil: raise" here when supporting Cython 3.0
+        return -999
+
+    cdef intp_t n_samples_Y(self) noexcept nogil:
+        """Number of samples in Y."""
+        # This is a abstract method.
+        # This _must_ always be overwritten in subclasses.
+        # TODO: add "with gil: raise" here when supporting Cython 3.0
+        return -999
+
+    cdef float64_t surrogate_dist(self, intp_t i, intp_t j) noexcept nogil:
+        return self.dist(i, j)
+
+    cdef float64_t dist(self, intp_t i, intp_t j) noexcept nogil:
+        # This is a abstract method.
+        # This _must_ always be overwritten in subclasses.
+        # TODO: add "with gil: raise" here when supporting Cython 3.0
+        return -1
+
+@final
+cdef class DenseDenseDatasetsPair{{name_suffix}}(DatasetsPair{{name_suffix}}):
+    """Compute distances between row vectors of two arrays.
+
+    Parameters
+    ----------
+    X: ndarray of shape (n_samples_X, n_features)
+        Rows represent vectors. Must be C-contiguous.
+
+    Y: ndarray of shape (n_samples_Y, n_features)
+        Rows represent vectors. Must be C-contiguous.
+
+    distance_metric: DistanceMetric
+        The distance metric responsible for computing distances
+        between two row vectors of (X, Y).
+    """
+
+    def __init__(
+        self,
+        const {{INPUT_DTYPE_t}}[:, ::1] X,
+        const {{INPUT_DTYPE_t}}[:, ::1] Y,
+        {{DistanceMetric}} distance_metric,
+    ):
+        super().__init__(distance_metric, n_features=X.shape[1])
+        # Arrays have already been checked
+        self.X = X
+        self.Y = Y
+
+    @final
+    cdef intp_t n_samples_X(self) noexcept nogil:
+        return self.X.shape[0]
+
+    @final
+    cdef intp_t n_samples_Y(self) noexcept nogil:
+        return self.Y.shape[0]
+
+    @final
+    cdef float64_t surrogate_dist(self, intp_t i, intp_t j) noexcept nogil:
+        return self.distance_metric.rdist(&self.X[i, 0], &self.Y[j, 0], self.n_features)
+
+    @final
+    cdef float64_t dist(self, intp_t i, intp_t j) noexcept nogil:
+        return self.distance_metric.dist(&self.X[i, 0], &self.Y[j, 0], self.n_features)
+
+
+@final
+cdef class SparseSparseDatasetsPair{{name_suffix}}(DatasetsPair{{name_suffix}}):
+    """Compute distances between vectors of two CSR matrices.
+
+    Parameters
+    ----------
+    X: sparse matrix of shape (n_samples_X, n_features)
+        Rows represent vectors. Must be in CSR format.
+
+    Y: sparse matrix of shape (n_samples_Y, n_features)
+        Rows represent vectors. Must be in CSR format.
+
+    distance_metric: DistanceMetric
+        The distance metric responsible for computing distances
+        between two vectors of (X, Y).
+    """
+
+    def __init__(self, X, Y, {{DistanceMetric}} distance_metric):
+        super().__init__(distance_metric, n_features=X.shape[1])
+
+        self.X_data, self.X_indices, self.X_indptr = self.unpack_csr_matrix(X)
+        self.Y_data, self.Y_indices, self.Y_indptr = self.unpack_csr_matrix(Y)
+
+    @final
+    cdef intp_t n_samples_X(self) noexcept nogil:
+        return self.X_indptr.shape[0] - 1
+
+    @final
+    cdef intp_t n_samples_Y(self) noexcept nogil:
+        return self.Y_indptr.shape[0] - 1
+
+    @final
+    cdef float64_t surrogate_dist(self, intp_t i, intp_t j) noexcept nogil:
+        return self.distance_metric.rdist_csr(
+            x1_data=&self.X_data[0],
+            x1_indices=&self.X_indices[0],
+            x2_data=&self.Y_data[0],
+            x2_indices=&self.Y_indices[0],
+            x1_start=self.X_indptr[i],
+            x1_end=self.X_indptr[i + 1],
+            x2_start=self.Y_indptr[j],
+            x2_end=self.Y_indptr[j + 1],
+            size=self.n_features,
+        )
+
+    @final
+    cdef float64_t dist(self, intp_t i, intp_t j) noexcept nogil:
+        return self.distance_metric.dist_csr(
+            x1_data=&self.X_data[0],
+            x1_indices=&self.X_indices[0],
+            x2_data=&self.Y_data[0],
+            x2_indices=&self.Y_indices[0],
+            x1_start=self.X_indptr[i],
+            x1_end=self.X_indptr[i + 1],
+            x2_start=self.Y_indptr[j],
+            x2_end=self.Y_indptr[j + 1],
+            size=self.n_features,
+        )
+
+
+@final
+cdef class SparseDenseDatasetsPair{{name_suffix}}(DatasetsPair{{name_suffix}}):
+    """Compute distances between vectors of a CSR matrix and a dense array.
+
+    Parameters
+    ----------
+    X: sparse matrix of shape (n_samples_X, n_features)
+        Rows represent vectors. Must be in CSR format.
+
+    Y: ndarray of shape (n_samples_Y, n_features)
+        Rows represent vectors. Must be C-contiguous.
+
+    distance_metric: DistanceMetric
+        The distance metric responsible for computing distances
+        between two vectors of (X, Y).
+    """
+
+    def __init__(self, X, Y, {{DistanceMetric}} distance_metric):
+        super().__init__(distance_metric, n_features=X.shape[1])
+
+        self.X_data, self.X_indices, self.X_indptr = self.unpack_csr_matrix(X)
+
+        # We support the sparse-dense case by using the sparse-sparse interfaces
+        # of `DistanceMetric` (namely `DistanceMetric.{dist_csr,rdist_csr}`) to
+        # avoid introducing a new complex set of interfaces. In this case, we
+        # need to convert `Y` (the dense array) into a CSR matrix.
+        #
+        # Here we motive using another simpler CSR representation to use for `Y`.
+        #
+        # If we were to use the usual CSR representation for `Y`, storing all
+        # the columns indices in `indices` would have required allocating an
+        # array of n_samples × n_features elements with repeated contiguous
+        # integers from 0 to n_features - 1. This would have been very wasteful
+        # from a memory point of view. This alternative representation just uses
+        # the necessary amount of information needed and only necessitates
+        # shifting the address of `data` before calling the CSR × CSR routines.
+        #
+        # In this representation:
+        #
+        #  - the `data` array is the original dense array, `Y`, whose first
+        #  element's address is shifted before calling the CSR × CSR routine
+        #
+        #  - the `indices` array is a single row of `n_features` elements:
+        #
+        #                      [0, 1, ..., n_features-1]
+        #
+        #  - the `indptr` array is not materialised as the indices pointers'
+        #  offset is constant (the offset equals `n_features`). Moreover, as
+        #  `data` is shifted, constant `start` and `end` indices pointers
+        #  respectively equalling 0 and n_features are used.
+
+        # Y array already has been checked here
+        self.n_Y = Y.shape[0]
+        self.Y_data = np.ravel(Y)
+        self.Y_indices = np.arange(self.n_features, dtype=np.int32)
+
+    @final
+    cdef intp_t n_samples_X(self) noexcept nogil:
+        return self.X_indptr.shape[0] - 1
+
+    @final
+    cdef intp_t n_samples_Y(self) noexcept nogil:
+        return self.n_Y
+
+    @final
+    cdef float64_t surrogate_dist(self, intp_t i, intp_t j) noexcept nogil:
+        return self.distance_metric.rdist_csr(
+            x1_data=&self.X_data[0],
+            x1_indices=&self.X_indices[0],
+            # Increment the data pointer such that x2_start=0 is aligned with the
+            # j-th row
+            x2_data=&self.Y_data[0] + j * self.n_features,
+            x2_indices=&self.Y_indices[0],
+            x1_start=self.X_indptr[i],
+            x1_end=self.X_indptr[i + 1],
+            x2_start=0,
+            x2_end=self.n_features,
+            size=self.n_features,
+        )
+
+    @final
+    cdef float64_t dist(self, intp_t i, intp_t j) noexcept nogil:
+
+        return self.distance_metric.dist_csr(
+            x1_data=&self.X_data[0],
+            x1_indices=&self.X_indices[0],
+            # Increment the data pointer such that x2_start=0 is aligned with the
+            # j-th row
+            x2_data=&self.Y_data[0] + j * self.n_features,
+            x2_indices=&self.Y_indices[0],
+            x1_start=self.X_indptr[i],
+            x1_end=self.X_indptr[i + 1],
+            x2_start=0,
+            x2_end=self.n_features,
+            size=self.n_features,
+        )
+
+
+@final
+cdef class DenseSparseDatasetsPair{{name_suffix}}(DatasetsPair{{name_suffix}}):
+    """Compute distances between vectors of a dense array and a CSR matrix.
+
+    Parameters
+    ----------
+    X: ndarray of shape (n_samples_X, n_features)
+        Rows represent vectors. Must be C-contiguous.
+
+    Y: sparse matrix of shape (n_samples_Y, n_features)
+        Rows represent vectors. Must be in CSR format.
+
+    distance_metric: DistanceMetric
+        The distance metric responsible for computing distances
+        between two vectors of (X, Y).
+    """
+
+    def __init__(self, X, Y, {{DistanceMetric}} distance_metric):
+        super().__init__(distance_metric, n_features=X.shape[1])
+        # Swapping arguments on the constructor
+        self.datasets_pair = SparseDenseDatasetsPair{{name_suffix}}(Y, X, distance_metric)
+
+    @final
+    cdef intp_t n_samples_X(self) noexcept nogil:
+        # Swapping interface
+        return self.datasets_pair.n_samples_Y()
+
+    @final
+    cdef intp_t n_samples_Y(self) noexcept nogil:
+        # Swapping interface
+        return self.datasets_pair.n_samples_X()
+
+    @final
+    cdef float64_t surrogate_dist(self, intp_t i, intp_t j) noexcept nogil:
+        # Swapping arguments on the same interface
+        return self.datasets_pair.surrogate_dist(j, i)
+
+    @final
+    cdef float64_t dist(self, intp_t i, intp_t j) noexcept nogil:
+        # Swapping arguments on the same interface
+        return self.datasets_pair.dist(j, i)
+
+{{endfor}}
diff --git a/sklearn/metrics/_pairwise_distances_reduction/_dispatcher.py b/sklearn/metrics/_pairwise_distances_reduction/_dispatcher.py
new file mode 100644
index 0000000000000..d8307cbe84eaa
--- /dev/null
+++ b/sklearn/metrics/_pairwise_distances_reduction/_dispatcher.py
@@ -0,0 +1,767 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+from abc import abstractmethod
+from typing import List
+
+import numpy as np
+from scipy.sparse import issparse
+
+from ... import get_config
+from .._dist_metrics import (
+    BOOL_METRICS,
+    METRIC_MAPPING64,
+    DistanceMetric,
+)
+from ._argkmin import (
+    ArgKmin32,
+    ArgKmin64,
+)
+from ._argkmin_classmode import (
+    ArgKminClassMode32,
+    ArgKminClassMode64,
+)
+from ._base import _sqeuclidean_row_norms32, _sqeuclidean_row_norms64
+from ._radius_neighbors import (
+    RadiusNeighbors32,
+    RadiusNeighbors64,
+)
+from ._radius_neighbors_classmode import (
+    RadiusNeighborsClassMode32,
+    RadiusNeighborsClassMode64,
+)
+
+
+def sqeuclidean_row_norms(X, num_threads):
+    """Compute the squared euclidean norm of the rows of X in parallel.
+
+    Parameters
+    ----------
+    X : ndarray or CSR matrix of shape (n_samples, n_features)
+        Input data. Must be c-contiguous.
+
+    num_threads : int
+        The number of OpenMP threads to use.
+
+    Returns
+    -------
+    sqeuclidean_row_norms : ndarray of shape (n_samples,)
+        Arrays containing the squared euclidean norm of each row of X.
+    """
+    if X.dtype == np.float64:
+        return np.asarray(_sqeuclidean_row_norms64(X, num_threads))
+    if X.dtype == np.float32:
+        return np.asarray(_sqeuclidean_row_norms32(X, num_threads))
+
+    raise ValueError(
+        "Only float64 or float32 datasets are supported at this time, "
+        f"got: X.dtype={X.dtype}."
+    )
+
+
+class BaseDistancesReductionDispatcher:
+    """Abstract base dispatcher for pairwise distance computation & reduction.
+
+    Each dispatcher extending the base :class:`BaseDistancesReductionDispatcher`
+    dispatcher must implement the :meth:`compute` classmethod.
+    """
+
+    @classmethod
+    def valid_metrics(cls) -> List[str]:
+        excluded = {
+            # PyFunc cannot be supported because it necessitates interacting with
+            # the CPython interpreter to call user defined functions.
+            "pyfunc",
+            "mahalanobis",  # is numerically unstable
+            # In order to support discrete distance metrics, we need to have a
+            # stable simultaneous sort which preserves the order of the indices
+            # because there generally is a lot of occurrences for a given values
+            # of distances in this case.
+            # TODO: implement a stable simultaneous_sort.
+            "hamming",
+            *BOOL_METRICS,
+        }
+        return sorted(({"sqeuclidean"} | set(METRIC_MAPPING64.keys())) - excluded)
+
+    @classmethod
+    def is_usable_for(cls, X, Y, metric) -> bool:
+        """Return True if the dispatcher can be used for the
+        given parameters.
+
+        Parameters
+        ----------
+        X : {ndarray, sparse matrix} of shape (n_samples_X, n_features)
+            Input data.
+
+        Y : {ndarray, sparse matrix} of shape (n_samples_Y, n_features)
+            Input data.
+
+        metric : str, default='euclidean'
+            The distance metric to use.
+            For a list of available metrics, see the documentation of
+            :class:`~sklearn.metrics.DistanceMetric`.
+
+        Returns
+        -------
+        True if the dispatcher can be used, else False.
+        """
+
+        # FIXME: the current Cython implementation is too slow for a large number of
+        # features. We temporarily disable it to fallback on SciPy's implementation.
+        # See: https://github.com/scikit-learn/scikit-learn/issues/28191
+        if (
+            issparse(X)
+            and issparse(Y)
+            and isinstance(metric, str)
+            and "euclidean" in metric
+        ):
+            return False
+
+        def is_numpy_c_ordered(X):
+            return hasattr(X, "flags") and getattr(X.flags, "c_contiguous", False)
+
+        def is_valid_sparse_matrix(X):
+            return (
+                issparse(X)
+                and X.format == "csr"
+                and
+                # TODO: support CSR matrices without non-zeros elements
+                X.nnz > 0
+                and
+                # TODO: support CSR matrices with int64 indices and indptr
+                # See: https://github.com/scikit-learn/scikit-learn/issues/23653
+                X.indices.dtype == X.indptr.dtype == np.int32
+            )
+
+        is_usable = (
+            get_config().get("enable_cython_pairwise_dist", True)
+            and (is_numpy_c_ordered(X) or is_valid_sparse_matrix(X))
+            and (is_numpy_c_ordered(Y) or is_valid_sparse_matrix(Y))
+            and X.dtype == Y.dtype
+            and X.dtype in (np.float32, np.float64)
+            and (metric in cls.valid_metrics() or isinstance(metric, DistanceMetric))
+        )
+
+        return is_usable
+
+    @classmethod
+    @abstractmethod
+    def compute(
+        cls,
+        X,
+        Y,
+        **kwargs,
+    ):
+        """Compute the reduction.
+
+        Parameters
+        ----------
+        X : ndarray or CSR matrix of shape (n_samples_X, n_features)
+            Input data.
+
+        Y : ndarray or CSR matrix of shape (n_samples_Y, n_features)
+            Input data.
+
+        **kwargs : additional parameters for the reduction
+
+        Notes
+        -----
+        This method is an abstract class method: it has to be implemented
+        for all subclasses.
+        """
+
+
+class ArgKmin(BaseDistancesReductionDispatcher):
+    """Compute the argkmin of row vectors of X on the ones of Y.
+
+    For each row vector of X, computes the indices of k first the rows
+    vectors of Y with the smallest distances.
+
+    ArgKmin is typically used to perform
+    bruteforce k-nearest neighbors queries.
+
+    This class is not meant to be instantiated, one should only use
+    its :meth:`compute` classmethod which handles allocation and
+    deallocation consistently.
+    """
+
+    @classmethod
+    def compute(
+        cls,
+        X,
+        Y,
+        k,
+        metric="euclidean",
+        chunk_size=None,
+        metric_kwargs=None,
+        strategy=None,
+        return_distance=False,
+    ):
+        """Compute the argkmin reduction.
+
+        Parameters
+        ----------
+        X : ndarray or CSR matrix of shape (n_samples_X, n_features)
+            Input data.
+
+        Y : ndarray or CSR matrix of shape (n_samples_Y, n_features)
+            Input data.
+
+        k : int
+            The k for the argkmin reduction.
+
+        metric : str, default='euclidean'
+            The distance metric to use for argkmin.
+            For a list of available metrics, see the documentation of
+            :class:`~sklearn.metrics.DistanceMetric`.
+
+        chunk_size : int, default=None,
+            The number of vectors per chunk. If None (default) looks-up in
+            scikit-learn configuration for `pairwise_dist_chunk_size`,
+            and use 256 if it is not set.
+
+        metric_kwargs : dict, default=None
+            Keyword arguments to pass to specified metric function.
+
+        strategy : str, {'auto', 'parallel_on_X', 'parallel_on_Y'}, default=None
+            The chunking strategy defining which dataset parallelization are made on.
+
+            For both strategies the computations happens with two nested loops,
+            respectively on chunks of X and chunks of Y.
+            Strategies differs on which loop (outer or inner) is made to run
+            in parallel with the Cython `prange` construct:
+
+              - 'parallel_on_X' dispatches chunks of X uniformly on threads.
+                Each thread then iterates on all the chunks of Y. This strategy is
+                embarrassingly parallel and comes with no datastructures
+                synchronisation.
+
+              - 'parallel_on_Y' dispatches chunks of Y uniformly on threads.
+                Each thread processes all the chunks of X in turn. This strategy is
+                a sequence of embarrassingly parallel subtasks (the inner loop on Y
+                chunks) with intermediate datastructures synchronisation at each
+                iteration of the sequential outer loop on X chunks.
+
+              - 'auto' relies on a simple heuristic to choose between
+                'parallel_on_X' and 'parallel_on_Y': when `X.shape[0]` is large enough,
+                'parallel_on_X' is usually the most efficient strategy.
+                When `X.shape[0]` is small but `Y.shape[0]` is large, 'parallel_on_Y'
+                brings more opportunity for parallelism and is therefore more efficient
+
+              - None (default) looks-up in scikit-learn configuration for
+                `pairwise_dist_parallel_strategy`, and use 'auto' if it is not set.
+
+        return_distance : boolean, default=False
+            Return distances between each X vector and its
+            argkmin if set to True.
+
+        Returns
+        -------
+        If return_distance=False:
+          - argkmin_indices : ndarray of shape (n_samples_X, k)
+            Indices of the argkmin for each vector in X.
+
+        If return_distance=True:
+          - argkmin_distances : ndarray of shape (n_samples_X, k)
+            Distances to the argkmin for each vector in X.
+          - argkmin_indices : ndarray of shape (n_samples_X, k)
+            Indices of the argkmin for each vector in X.
+
+        Notes
+        -----
+        This classmethod inspects the arguments values to dispatch to the
+        dtype-specialized implementation of :class:`ArgKmin`.
+
+        This allows decoupling the API entirely from the implementation details
+        whilst maintaining RAII: all temporarily allocated datastructures necessary
+        for the concrete implementation are therefore freed when this classmethod
+        returns.
+        """
+        if X.dtype == Y.dtype == np.float64:
+            return ArgKmin64.compute(
+                X=X,
+                Y=Y,
+                k=k,
+                metric=metric,
+                chunk_size=chunk_size,
+                metric_kwargs=metric_kwargs,
+                strategy=strategy,
+                return_distance=return_distance,
+            )
+
+        if X.dtype == Y.dtype == np.float32:
+            return ArgKmin32.compute(
+                X=X,
+                Y=Y,
+                k=k,
+                metric=metric,
+                chunk_size=chunk_size,
+                metric_kwargs=metric_kwargs,
+                strategy=strategy,
+                return_distance=return_distance,
+            )
+
+        raise ValueError(
+            "Only float64 or float32 datasets pairs are supported at this time, "
+            f"got: X.dtype={X.dtype} and Y.dtype={Y.dtype}."
+        )
+
+
+class RadiusNeighbors(BaseDistancesReductionDispatcher):
+    """Compute radius-based neighbors for two sets of vectors.
+
+    For each row-vector X[i] of the queries X, find all the indices j of
+    row-vectors in Y such that:
+
+                        dist(X[i], Y[j]) <= radius
+
+    The distance function `dist` depends on the values of the `metric`
+    and `metric_kwargs` parameters.
+
+    This class is not meant to be instantiated, one should only use
+    its :meth:`compute` classmethod which handles allocation and
+    deallocation consistently.
+    """
+
+    @classmethod
+    def compute(
+        cls,
+        X,
+        Y,
+        radius,
+        metric="euclidean",
+        chunk_size=None,
+        metric_kwargs=None,
+        strategy=None,
+        return_distance=False,
+        sort_results=False,
+    ):
+        """Return the results of the reduction for the given arguments.
+
+        Parameters
+        ----------
+        X : ndarray or CSR matrix of shape (n_samples_X, n_features)
+            Input data.
+
+        Y : ndarray or CSR matrix of shape (n_samples_Y, n_features)
+            Input data.
+
+        radius : float
+            The radius defining the neighborhood.
+
+        metric : str, default='euclidean'
+            The distance metric to use.
+            For a list of available metrics, see the documentation of
+            :class:`~sklearn.metrics.DistanceMetric`.
+
+        chunk_size : int, default=None,
+            The number of vectors per chunk. If None (default) looks-up in
+            scikit-learn configuration for `pairwise_dist_chunk_size`,
+            and use 256 if it is not set.
+
+        metric_kwargs : dict, default=None
+            Keyword arguments to pass to specified metric function.
+
+        strategy : str, {'auto', 'parallel_on_X', 'parallel_on_Y'}, default=None
+            The chunking strategy defining which dataset parallelization are made on.
+
+            For both strategies the computations happens with two nested loops,
+            respectively on chunks of X and chunks of Y.
+            Strategies differs on which loop (outer or inner) is made to run
+            in parallel with the Cython `prange` construct:
+
+              - 'parallel_on_X' dispatches chunks of X uniformly on threads.
+                Each thread then iterates on all the chunks of Y. This strategy is
+                embarrassingly parallel and comes with no datastructures
+                synchronisation.
+
+              - 'parallel_on_Y' dispatches chunks of Y uniformly on threads.
+                Each thread processes all the chunks of X in turn. This strategy is
+                a sequence of embarrassingly parallel subtasks (the inner loop on Y
+                chunks) with intermediate datastructures synchronisation at each
+                iteration of the sequential outer loop on X chunks.
+
+              - 'auto' relies on a simple heuristic to choose between
+                'parallel_on_X' and 'parallel_on_Y': when `X.shape[0]` is large enough,
+                'parallel_on_X' is usually the most efficient strategy.
+                When `X.shape[0]` is small but `Y.shape[0]` is large, 'parallel_on_Y'
+                brings more opportunity for parallelism and is therefore more efficient
+                despite the synchronization step at each iteration of the outer loop
+                on chunks of `X`.
+
+              - None (default) looks-up in scikit-learn configuration for
+                `pairwise_dist_parallel_strategy`, and use 'auto' if it is not set.
+
+        return_distance : boolean, default=False
+            Return distances between each X vector and its neighbors if set to True.
+
+        sort_results : boolean, default=False
+            Sort results with respect to distances between each X vector and its
+            neighbors if set to True.
+
+        Returns
+        -------
+        If return_distance=False:
+          - neighbors_indices : ndarray of n_samples_X ndarray
+            Indices of the neighbors for each vector in X.
+
+        If return_distance=True:
+          - neighbors_indices : ndarray of n_samples_X ndarray
+            Indices of the neighbors for each vector in X.
+          - neighbors_distances : ndarray of n_samples_X ndarray
+            Distances to the neighbors for each vector in X.
+
+        Notes
+        -----
+        This classmethod inspects the arguments values to dispatch to the
+        dtype-specialized implementation of :class:`RadiusNeighbors`.
+
+        This allows decoupling the API entirely from the implementation details
+        whilst maintaining RAII: all temporarily allocated datastructures necessary
+        for the concrete implementation are therefore freed when this classmethod
+        returns.
+        """
+        if X.dtype == Y.dtype == np.float64:
+            return RadiusNeighbors64.compute(
+                X=X,
+                Y=Y,
+                radius=radius,
+                metric=metric,
+                chunk_size=chunk_size,
+                metric_kwargs=metric_kwargs,
+                strategy=strategy,
+                sort_results=sort_results,
+                return_distance=return_distance,
+            )
+
+        if X.dtype == Y.dtype == np.float32:
+            return RadiusNeighbors32.compute(
+                X=X,
+                Y=Y,
+                radius=radius,
+                metric=metric,
+                chunk_size=chunk_size,
+                metric_kwargs=metric_kwargs,
+                strategy=strategy,
+                sort_results=sort_results,
+                return_distance=return_distance,
+            )
+
+        raise ValueError(
+            "Only float64 or float32 datasets pairs are supported at this time, "
+            f"got: X.dtype={X.dtype} and Y.dtype={Y.dtype}."
+        )
+
+
+class ArgKminClassMode(BaseDistancesReductionDispatcher):
+    """Compute the argkmin of row vectors of X on the ones of Y with labels.
+
+    For each row vector of X, computes the indices of k first the rows
+    vectors of Y with the smallest distances. Computes weighted mode of labels.
+
+    ArgKminClassMode is typically used to perform bruteforce k-nearest neighbors
+    queries when the weighted mode of the labels for the k-nearest neighbors
+    are required, such as in `predict` methods.
+
+    This class is not meant to be instantiated, one should only use
+    its :meth:`compute` classmethod which handles allocation and
+    deallocation consistently.
+    """
+
+    @classmethod
+    def valid_metrics(cls) -> List[str]:
+        excluded = {
+            # Euclidean is technically usable for ArgKminClassMode
+            # but its current implementation would not be competitive.
+            # TODO: implement Euclidean specialization using GEMM.
+            "euclidean",
+            "sqeuclidean",
+        }
+        return list(set(BaseDistancesReductionDispatcher.valid_metrics()) - excluded)
+
+    @classmethod
+    def compute(
+        cls,
+        X,
+        Y,
+        k,
+        weights,
+        Y_labels,
+        unique_Y_labels,
+        metric="euclidean",
+        chunk_size=None,
+        metric_kwargs=None,
+        strategy=None,
+    ):
+        """Compute the argkmin reduction.
+
+        Parameters
+        ----------
+        X : ndarray of shape (n_samples_X, n_features)
+            The input array to be labelled.
+
+        Y : ndarray of shape (n_samples_Y, n_features)
+            The input array whose class membership are provided through the
+            `Y_labels` parameter.
+
+        k : int
+            The number of nearest neighbors to consider.
+
+        weights : ndarray
+            The weights applied over the `Y_labels` of `Y` when computing the
+            weighted mode of the labels.
+
+        Y_labels : ndarray
+            An array containing the index of the class membership of the
+            associated samples in `Y`. This is used in labeling `X`.
+
+        unique_Y_labels : ndarray
+            An array containing all unique indices contained in the
+            corresponding `Y_labels` array.
+
+        metric : str, default='euclidean'
+            The distance metric to use. For a list of available metrics, see
+            the documentation of :class:`~sklearn.metrics.DistanceMetric`.
+            Currently does not support `'precomputed'`.
+
+        chunk_size : int, default=None,
+            The number of vectors per chunk. If None (default) looks-up in
+            scikit-learn configuration for `pairwise_dist_chunk_size`,
+            and use 256 if it is not set.
+
+        metric_kwargs : dict, default=None
+            Keyword arguments to pass to specified metric function.
+
+        strategy : str, {'auto', 'parallel_on_X', 'parallel_on_Y'}, default=None
+            The chunking strategy defining which dataset parallelization are made on.
+
+            For both strategies the computations happens with two nested loops,
+            respectively on chunks of X and chunks of Y.
+            Strategies differs on which loop (outer or inner) is made to run
+            in parallel with the Cython `prange` construct:
+
+              - 'parallel_on_X' dispatches chunks of X uniformly on threads.
+                Each thread then iterates on all the chunks of Y. This strategy is
+                embarrassingly parallel and comes with no datastructures
+                synchronisation.
+
+              - 'parallel_on_Y' dispatches chunks of Y uniformly on threads.
+                Each thread processes all the chunks of X in turn. This strategy is
+                a sequence of embarrassingly parallel subtasks (the inner loop on Y
+                chunks) with intermediate datastructures synchronisation at each
+                iteration of the sequential outer loop on X chunks.
+
+              - 'auto' relies on a simple heuristic to choose between
+                'parallel_on_X' and 'parallel_on_Y': when `X.shape[0]` is large enough,
+                'parallel_on_X' is usually the most efficient strategy.
+                When `X.shape[0]` is small but `Y.shape[0]` is large, 'parallel_on_Y'
+                brings more opportunity for parallelism and is therefore more efficient
+                despite the synchronization step at each iteration of the outer loop
+                on chunks of `X`.
+
+              - None (default) looks-up in scikit-learn configuration for
+                `pairwise_dist_parallel_strategy`, and use 'auto' if it is not set.
+
+        Returns
+        -------
+        probabilities : ndarray of shape (n_samples_X, n_classes)
+            An array containing the class probabilities for each sample.
+
+        Notes
+        -----
+        This classmethod is responsible for introspecting the arguments
+        values to dispatch to the most appropriate implementation of
+        :class:`PairwiseDistancesArgKmin`.
+
+        This allows decoupling the API entirely from the implementation details
+        whilst maintaining RAII: all temporarily allocated datastructures necessary
+        for the concrete implementation are therefore freed when this classmethod
+        returns.
+        """
+        if weights not in {"uniform", "distance"}:
+            raise ValueError(
+                "Only the 'uniform' or 'distance' weights options are supported"
+                f" at this time. Got: {weights=}."
+            )
+        if X.dtype == Y.dtype == np.float64:
+            return ArgKminClassMode64.compute(
+                X=X,
+                Y=Y,
+                k=k,
+                weights=weights,
+                Y_labels=np.array(Y_labels, dtype=np.intp),
+                unique_Y_labels=np.array(unique_Y_labels, dtype=np.intp),
+                metric=metric,
+                chunk_size=chunk_size,
+                metric_kwargs=metric_kwargs,
+                strategy=strategy,
+            )
+
+        if X.dtype == Y.dtype == np.float32:
+            return ArgKminClassMode32.compute(
+                X=X,
+                Y=Y,
+                k=k,
+                weights=weights,
+                Y_labels=np.array(Y_labels, dtype=np.intp),
+                unique_Y_labels=np.array(unique_Y_labels, dtype=np.intp),
+                metric=metric,
+                chunk_size=chunk_size,
+                metric_kwargs=metric_kwargs,
+                strategy=strategy,
+            )
+
+        raise ValueError(
+            "Only float64 or float32 datasets pairs are supported at this time, "
+            f"got: X.dtype={X.dtype} and Y.dtype={Y.dtype}."
+        )
+
+
+class RadiusNeighborsClassMode(BaseDistancesReductionDispatcher):
+    """Compute radius-based class modes of row vectors of X using the
+    those of Y.
+
+    For each row-vector X[i] of the queries X, find all the indices j of
+    row-vectors in Y such that:
+
+                        dist(X[i], Y[j]) <= radius
+
+    RadiusNeighborsClassMode is typically used to perform bruteforce
+    radius neighbors queries when the weighted mode of the labels for
+    the nearest neighbors within the specified radius are required,
+    such as in `predict` methods.
+
+    This class is not meant to be instantiated, one should only use
+    its :meth:`compute` classmethod which handles allocation and
+    deallocation consistently.
+    """
+
+    @classmethod
+    def valid_metrics(cls) -> List[str]:
+        excluded = {
+            # Euclidean is technically usable for RadiusNeighborsClassMode
+            # but it would not be competitive.
+            # TODO: implement Euclidean specialization using GEMM.
+            "euclidean",
+            "sqeuclidean",
+        }
+        return sorted(set(BaseDistancesReductionDispatcher.valid_metrics()) - excluded)
+
+    @classmethod
+    def compute(
+        cls,
+        X,
+        Y,
+        radius,
+        weights,
+        Y_labels,
+        unique_Y_labels,
+        outlier_label,
+        metric="euclidean",
+        chunk_size=None,
+        metric_kwargs=None,
+        strategy=None,
+    ):
+        """Return the results of the reduction for the given arguments.
+        Parameters
+        ----------
+        X : ndarray of shape (n_samples_X, n_features)
+            The input array to be labelled.
+        Y : ndarray of shape (n_samples_Y, n_features)
+            The input array whose class membership is provided through
+            the `Y_labels` parameter.
+        radius : float
+            The radius defining the neighborhood.
+        weights : ndarray
+            The weights applied to the `Y_labels` when computing the
+            weighted mode of the labels.
+        Y_labels : ndarray
+            An array containing the index of the class membership of the
+            associated samples in `Y`. This is used in labeling `X`.
+        unique_Y_labels : ndarray
+            An array containing all unique class labels.
+        outlier_label : int, default=None
+            Label for outlier samples (samples with no neighbors in given
+            radius). In the default case when the value is None if any
+            outlier is detected, a ValueError will be raised. The outlier
+            label should be selected from among the unique 'Y' labels. If
+            it is specified with a different value a warning will be raised
+            and all class probabilities of outliers will be assigned to be 0.
+        metric : str, default='euclidean'
+            The distance metric to use. For a list of available metrics, see
+            the documentation of :class:`~sklearn.metrics.DistanceMetric`.
+            Currently does not support `'precomputed'`.
+        chunk_size : int, default=None,
+            The number of vectors per chunk. If None (default) looks-up in
+            scikit-learn configuration for `pairwise_dist_chunk_size`,
+            and use 256 if it is not set.
+        metric_kwargs : dict, default=None
+            Keyword arguments to pass to specified metric function.
+        strategy : str, {'auto', 'parallel_on_X', 'parallel_on_Y'}, default=None
+            The chunking strategy defining which dataset parallelization are made on.
+            For both strategies the computations happens with two nested loops,
+            respectively on chunks of X and chunks of Y.
+            Strategies differs on which loop (outer or inner) is made to run
+            in parallel with the Cython `prange` construct:
+              - 'parallel_on_X' dispatches chunks of X uniformly on threads.
+                Each thread then iterates on all the chunks of Y. This strategy is
+                embarrassingly parallel and comes with no datastructures
+                synchronisation.
+              - 'parallel_on_Y' dispatches chunks of Y uniformly on threads.
+                Each thread processes all the chunks of X in turn. This strategy is
+                a sequence of embarrassingly parallel subtasks (the inner loop on Y
+                chunks) with intermediate datastructures synchronisation at each
+                iteration of the sequential outer loop on X chunks.
+              - 'auto' relies on a simple heuristic to choose between
+                'parallel_on_X' and 'parallel_on_Y': when `X.shape[0]` is large enough,
+                'parallel_on_X' is usually the most efficient strategy.
+                When `X.shape[0]` is small but `Y.shape[0]` is large, 'parallel_on_Y'
+                brings more opportunity for parallelism and is therefore more efficient
+                despite the synchronization step at each iteration of the outer loop
+                on chunks of `X`.
+              - None (default) looks-up in scikit-learn configuration for
+                `pairwise_dist_parallel_strategy`, and use 'auto' if it is not set.
+        Returns
+        -------
+        probabilities : ndarray of shape (n_samples_X, n_classes)
+            An array containing the class probabilities for each sample.
+        """
+        if weights not in {"uniform", "distance"}:
+            raise ValueError(
+                "Only the 'uniform' or 'distance' weights options are supported"
+                f" at this time. Got: {weights=}."
+            )
+        if X.dtype == Y.dtype == np.float64:
+            return RadiusNeighborsClassMode64.compute(
+                X=X,
+                Y=Y,
+                radius=radius,
+                weights=weights,
+                Y_labels=np.array(Y_labels, dtype=np.intp),
+                unique_Y_labels=np.array(unique_Y_labels, dtype=np.intp),
+                outlier_label=outlier_label,
+                metric=metric,
+                chunk_size=chunk_size,
+                metric_kwargs=metric_kwargs,
+                strategy=strategy,
+            )
+
+        if X.dtype == Y.dtype == np.float32:
+            return RadiusNeighborsClassMode32.compute(
+                X=X,
+                Y=Y,
+                radius=radius,
+                weights=weights,
+                Y_labels=np.array(Y_labels, dtype=np.intp),
+                unique_Y_labels=np.array(unique_Y_labels, dtype=np.intp),
+                outlier_label=outlier_label,
+                metric=metric,
+                chunk_size=chunk_size,
+                metric_kwargs=metric_kwargs,
+                strategy=strategy,
+            )
+
+        raise ValueError(
+            "Only float64 or float32 datasets pairs are supported at this time, "
+            f"got: X.dtype={X.dtype} and Y.dtype={Y.dtype}."
+        )
diff --git a/sklearn/metrics/_pairwise_distances_reduction/_middle_term_computer.pxd.tp b/sklearn/metrics/_pairwise_distances_reduction/_middle_term_computer.pxd.tp
new file mode 100644
index 0000000000000..bdf007bd0514a
--- /dev/null
+++ b/sklearn/metrics/_pairwise_distances_reduction/_middle_term_computer.pxd.tp
@@ -0,0 +1,228 @@
+{{py:
+
+implementation_specific_values = [
+    # Values are the following ones:
+    #
+    #       name_suffix, upcast_to_float64, INPUT_DTYPE_t, INPUT_DTYPE
+    #
+    # We also use the float64 dtype and C-type names as defined in
+    # `sklearn.utils._typedefs` to maintain consistency.
+    #
+    ('64', False, 'float64_t', 'np.float64'),
+    ('32', True, 'float32_t', 'np.float32')
+]
+
+}}
+from libcpp.vector cimport vector
+
+from ...utils._typedefs cimport float64_t, float32_t, int32_t, intp_t
+
+
+cdef void _middle_term_sparse_sparse_64(
+    const float64_t[:] X_data,
+    const int32_t[:] X_indices,
+    const int32_t[:] X_indptr,
+    intp_t X_start,
+    intp_t X_end,
+    const float64_t[:] Y_data,
+    const int32_t[:] Y_indices,
+    const int32_t[:] Y_indptr,
+    intp_t Y_start,
+    intp_t Y_end,
+    float64_t * D,
+) noexcept nogil
+
+
+{{for name_suffix, upcast_to_float64, INPUT_DTYPE_t, INPUT_DTYPE in implementation_specific_values}}
+
+
+cdef class MiddleTermComputer{{name_suffix}}:
+    cdef:
+        intp_t effective_n_threads
+        intp_t chunks_n_threads
+        intp_t dist_middle_terms_chunks_size
+        intp_t n_features
+        intp_t chunk_size
+
+        # Buffers for the `-2 * X_c @ Y_c.T` term computed via GEMM
+        vector[vector[float64_t]] dist_middle_terms_chunks
+
+    cdef void _parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil
+
+    cdef void _parallel_on_X_parallel_init(self, intp_t thread_num) noexcept nogil
+
+    cdef void _parallel_on_X_init_chunk(
+        self,
+        intp_t thread_num,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil
+
+    cdef void _parallel_on_Y_init(self) noexcept nogil
+
+    cdef void _parallel_on_Y_parallel_init(
+        self,
+        intp_t thread_num,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil
+
+    cdef void _parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num
+    ) noexcept nogil
+
+    cdef float64_t * _compute_dist_middle_terms(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil
+
+
+cdef class DenseDenseMiddleTermComputer{{name_suffix}}(MiddleTermComputer{{name_suffix}}):
+    cdef:
+        const {{INPUT_DTYPE_t}}[:, ::1] X
+        const {{INPUT_DTYPE_t}}[:, ::1] Y
+
+    {{if upcast_to_float64}}
+        # Buffers for upcasting chunks of X and Y from 32bit to 64bit
+        vector[vector[float64_t]] X_c_upcast
+        vector[vector[float64_t]] Y_c_upcast
+    {{endif}}
+
+    cdef void _parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil
+
+    cdef void _parallel_on_X_init_chunk(
+        self,
+        intp_t thread_num,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil
+
+    cdef void _parallel_on_Y_parallel_init(
+        self,
+        intp_t thread_num,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil
+
+    cdef void _parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num
+    ) noexcept nogil
+
+    cdef float64_t * _compute_dist_middle_terms(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil
+
+
+cdef class SparseSparseMiddleTermComputer{{name_suffix}}(MiddleTermComputer{{name_suffix}}):
+    cdef:
+        const float64_t[:] X_data
+        const int32_t[:] X_indices
+        const int32_t[:] X_indptr
+
+        const float64_t[:] Y_data
+        const int32_t[:] Y_indices
+        const int32_t[:] Y_indptr
+
+    cdef void _parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num
+    ) noexcept nogil
+
+    cdef void _parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num
+    ) noexcept nogil
+
+    cdef float64_t * _compute_dist_middle_terms(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil
+
+
+cdef class SparseDenseMiddleTermComputer{{name_suffix}}(MiddleTermComputer{{name_suffix}}):
+    cdef:
+        const float64_t[:] X_data
+        const int32_t[:] X_indices
+        const int32_t[:] X_indptr
+
+        const {{INPUT_DTYPE_t}}[:, ::1] Y
+
+        # We treat the dense-sparse case with the sparse-dense case by simply
+        # treating the dist_middle_terms as F-ordered and by swapping arguments.
+        # This attribute is meant to encode the case and adapt the logic
+        # accordingly.
+        bint c_ordered_middle_term
+
+    cdef void _parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num
+    ) noexcept nogil
+
+    cdef void _parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num
+    ) noexcept nogil
+
+    cdef float64_t * _compute_dist_middle_terms(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil
+
+{{endfor}}
diff --git a/sklearn/metrics/_pairwise_distances_reduction/_middle_term_computer.pyx.tp b/sklearn/metrics/_pairwise_distances_reduction/_middle_term_computer.pyx.tp
new file mode 100644
index 0000000000000..1fca2d674720c
--- /dev/null
+++ b/sklearn/metrics/_pairwise_distances_reduction/_middle_term_computer.pyx.tp
@@ -0,0 +1,633 @@
+{{py:
+
+implementation_specific_values = [
+    # Values are the following ones:
+    #
+    #       name_suffix, upcast_to_float64, INPUT_DTYPE_t, INPUT_DTYPE
+    #
+    # We also use the float64 dtype and C-type names as defined in
+    # `sklearn.utils._typedefs` to maintain consistency.
+    #
+    ('64', False, 'float64_t', 'np.float64'),
+    ('32', True, 'float32_t', 'np.float32')
+]
+
+}}
+from libcpp.vector cimport vector
+from libcpp.algorithm cimport fill
+
+from ...utils._cython_blas cimport (
+  BLAS_Order,
+  BLAS_Trans,
+  NoTrans,
+  RowMajor,
+  Trans,
+  _gemm,
+)
+from ...utils._typedefs cimport float64_t, float32_t, int32_t, intp_t
+
+import numpy as np
+from scipy.sparse import issparse, csr_matrix
+
+
+cdef void _middle_term_sparse_sparse_64(
+    const float64_t[:] X_data,
+    const int32_t[:] X_indices,
+    const int32_t[:] X_indptr,
+    intp_t X_start,
+    intp_t X_end,
+    const float64_t[:] Y_data,
+    const int32_t[:] Y_indices,
+    const int32_t[:] Y_indptr,
+    intp_t Y_start,
+    intp_t Y_end,
+    float64_t * D,
+) noexcept nogil:
+    # This routine assumes that D points to the first element of a
+    # zeroed buffer of length at least equal to n_X × n_Y, conceptually
+    # representing a 2-d C-ordered array.
+    cdef:
+        intp_t i, j, k
+        intp_t n_X = X_end - X_start
+        intp_t n_Y = Y_end - Y_start
+        intp_t x_col, x_ptr, y_col, y_ptr
+
+    for i in range(n_X):
+        for x_ptr in range(X_indptr[X_start+i], X_indptr[X_start+i+1]):
+            x_col = X_indices[x_ptr]
+            for j in range(n_Y):
+                k = i * n_Y + j
+                for y_ptr in range(Y_indptr[Y_start+j], Y_indptr[Y_start+j+1]):
+                    y_col = Y_indices[y_ptr]
+                    if x_col == y_col:
+                        D[k] += -2 * X_data[x_ptr] * Y_data[y_ptr]
+
+
+{{for name_suffix, upcast_to_float64, INPUT_DTYPE_t, INPUT_DTYPE in implementation_specific_values}}
+
+cdef void _middle_term_sparse_dense_{{name_suffix}}(
+    const float64_t[:] X_data,
+    const int32_t[:] X_indices,
+    const int32_t[:] X_indptr,
+    intp_t X_start,
+    intp_t X_end,
+    const {{INPUT_DTYPE_t}}[:, ::1] Y,
+    intp_t Y_start,
+    intp_t Y_end,
+    bint c_ordered_middle_term,
+    float64_t * dist_middle_terms,
+) noexcept nogil:
+    # This routine assumes that dist_middle_terms is a pointer to the first element
+    # of a buffer filled with zeros of length at least equal to n_X × n_Y, conceptually
+    # representing a 2-d C-ordered of F-ordered array.
+    cdef:
+        intp_t i, j, k
+        intp_t n_X = X_end - X_start
+        intp_t n_Y = Y_end - Y_start
+        intp_t X_i_col_idx, X_i_ptr, Y_j_col_idx, Y_j_ptr
+
+    for i in range(n_X):
+        for j in range(n_Y):
+            k = i * n_Y + j if c_ordered_middle_term else j * n_X + i
+            for X_i_ptr in range(X_indptr[X_start+i], X_indptr[X_start+i+1]):
+                X_i_col_idx = X_indices[X_i_ptr]
+                dist_middle_terms[k] += -2 * X_data[X_i_ptr] * Y[Y_start + j, X_i_col_idx]
+
+
+cdef class MiddleTermComputer{{name_suffix}}:
+    """Helper class to compute a Euclidean distance matrix in chunks.
+
+    This is an abstract base class that is further specialized depending
+    on the type of data (dense or sparse).
+
+    `EuclideanDistance` subclasses relies on the squared Euclidean
+    distances between chunks of vectors X_c and Y_c using the
+    following decomposition for the (i,j) pair :
+
+
+         ||X_c_i - Y_c_j||² = ||X_c_i||² - 2 X_c_i.Y_c_j^T + ||Y_c_j||²
+
+
+    This helper class is in charge of wrapping the common logic to compute
+    the middle term, i.e. `- 2 X_c_i.Y_c_j^T`.
+    """
+
+    @classmethod
+    def get_for(
+        cls,
+        X,
+        Y,
+        effective_n_threads,
+        chunks_n_threads,
+        dist_middle_terms_chunks_size,
+        n_features,
+        chunk_size,
+    ) -> MiddleTermComputer{{name_suffix}}:
+        """Return the MiddleTermComputer implementation for the given arguments.
+
+        Parameters
+        ----------
+        X : ndarray or CSR sparse matrix of shape (n_samples_X, n_features)
+            Input data.
+            If provided as a ndarray, it must be C-contiguous.
+
+        Y : ndarray or CSR sparse matrix of shape (n_samples_Y, n_features)
+            Input data.
+            If provided as a ndarray, it must be C-contiguous.
+
+        Returns
+        -------
+        middle_term_computer: MiddleTermComputer{{name_suffix}}
+            The suited MiddleTermComputer{{name_suffix}} implementation.
+        """
+        X_is_sparse = issparse(X)
+        Y_is_sparse = issparse(Y)
+
+        if not X_is_sparse and not Y_is_sparse:
+            return DenseDenseMiddleTermComputer{{name_suffix}}(
+                X,
+                Y,
+                effective_n_threads,
+                chunks_n_threads,
+                dist_middle_terms_chunks_size,
+                n_features,
+                chunk_size,
+            )
+        if X_is_sparse and Y_is_sparse:
+            return SparseSparseMiddleTermComputer{{name_suffix}}(
+                X,
+                Y,
+                effective_n_threads,
+                chunks_n_threads,
+                dist_middle_terms_chunks_size,
+                n_features,
+                chunk_size,
+            )
+        if X_is_sparse and not Y_is_sparse:
+            return SparseDenseMiddleTermComputer{{name_suffix}}(
+                X,
+                Y,
+                effective_n_threads,
+                chunks_n_threads,
+                dist_middle_terms_chunks_size,
+                n_features,
+                chunk_size,
+                c_ordered_middle_term=True
+            )
+        if not X_is_sparse and Y_is_sparse:
+            # NOTE: The Dense-Sparse case is implement via the Sparse-Dense case.
+            #
+            # To do so:
+            #    - X (dense) and Y (sparse) are swapped
+            #    - the distance middle term is seen as F-ordered for consistency
+            #      (c_ordered_middle_term = False)
+            return SparseDenseMiddleTermComputer{{name_suffix}}(
+                # Mind that X and Y are swapped here.
+                Y,
+                X,
+                effective_n_threads,
+                chunks_n_threads,
+                dist_middle_terms_chunks_size,
+                n_features,
+                chunk_size,
+                c_ordered_middle_term=False,
+            )
+        raise NotImplementedError(
+            "X and Y must be CSR sparse matrices or numpy arrays."
+        )
+
+    @classmethod
+    def unpack_csr_matrix(cls, X: csr_matrix):
+        """Ensure that the CSR matrix is indexed with np.int32."""
+        X_data = np.asarray(X.data, dtype=np.float64)
+        X_indices = np.asarray(X.indices, dtype=np.int32)
+        X_indptr = np.asarray(X.indptr, dtype=np.int32)
+        return X_data, X_indices, X_indptr
+
+    def __init__(
+        self,
+        intp_t effective_n_threads,
+        intp_t chunks_n_threads,
+        intp_t dist_middle_terms_chunks_size,
+        intp_t n_features,
+        intp_t chunk_size,
+    ):
+        self.effective_n_threads = effective_n_threads
+        self.chunks_n_threads = chunks_n_threads
+        self.dist_middle_terms_chunks_size = dist_middle_terms_chunks_size
+        self.n_features = n_features
+        self.chunk_size = chunk_size
+
+        self.dist_middle_terms_chunks = vector[vector[float64_t]](self.effective_n_threads)
+
+    cdef void _parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil:
+        return
+
+    cdef void _parallel_on_X_parallel_init(self, intp_t thread_num) noexcept nogil:
+        self.dist_middle_terms_chunks[thread_num].resize(self.dist_middle_terms_chunks_size)
+
+    cdef void _parallel_on_X_init_chunk(
+        self,
+        intp_t thread_num,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil:
+        return
+
+    cdef void _parallel_on_Y_init(self) noexcept nogil:
+        for thread_num in range(self.chunks_n_threads):
+            self.dist_middle_terms_chunks[thread_num].resize(
+                self.dist_middle_terms_chunks_size
+            )
+
+    cdef void _parallel_on_Y_parallel_init(
+        self,
+        intp_t thread_num,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil:
+        return
+
+    cdef void _parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num
+    ) noexcept nogil:
+        return
+
+    cdef float64_t * _compute_dist_middle_terms(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil:
+        return NULL
+
+
+cdef class DenseDenseMiddleTermComputer{{name_suffix}}(MiddleTermComputer{{name_suffix}}):
+    """Computes the middle term of the Euclidean distance between two chunked dense matrices
+    X_c and Y_c.
+
+                        dist_middle_terms = - 2 X_c_i.Y_c_j^T
+
+    This class use the BLAS gemm routine to perform the dot product of each chunks
+    of the distance matrix with improved arithmetic intensity and vector instruction (SIMD).
+    """
+
+    def __init__(
+        self,
+        const {{INPUT_DTYPE_t}}[:, ::1] X,
+        const {{INPUT_DTYPE_t}}[:, ::1] Y,
+        intp_t effective_n_threads,
+        intp_t chunks_n_threads,
+        intp_t dist_middle_terms_chunks_size,
+        intp_t n_features,
+        intp_t chunk_size,
+    ):
+        super().__init__(
+            effective_n_threads,
+            chunks_n_threads,
+            dist_middle_terms_chunks_size,
+            n_features,
+            chunk_size,
+        )
+        self.X = X
+        self.Y = Y
+
+{{if upcast_to_float64}}
+        # We populate the buffer for upcasting chunks of X and Y from float32 to float64.
+        self.X_c_upcast = vector[vector[float64_t]](self.effective_n_threads)
+        self.Y_c_upcast = vector[vector[float64_t]](self.effective_n_threads)
+
+        upcast_buffer_n_elements = self.chunk_size * n_features
+
+        for thread_num in range(self.effective_n_threads):
+            self.X_c_upcast[thread_num].resize(upcast_buffer_n_elements)
+            self.Y_c_upcast[thread_num].resize(upcast_buffer_n_elements)
+{{endif}}
+
+    cdef void _parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil:
+{{if upcast_to_float64}}
+        cdef:
+            intp_t i, j
+            intp_t n_chunk_samples = Y_end - Y_start
+
+        # Upcasting Y_c=Y[Y_start:Y_end, :] from float32 to float64
+        for i in range(n_chunk_samples):
+            for j in range(self.n_features):
+                self.Y_c_upcast[thread_num][i * self.n_features + j] =  self.Y[Y_start + i, j]
+{{else}}
+        return
+{{endif}}
+
+    cdef void _parallel_on_X_init_chunk(
+        self,
+        intp_t thread_num,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil:
+{{if upcast_to_float64}}
+        cdef:
+            intp_t i, j
+            intp_t n_chunk_samples = X_end - X_start
+
+        # Upcasting X_c=X[X_start:X_end, :] from float32 to float64
+        for i in range(n_chunk_samples):
+            for j in range(self.n_features):
+                self.X_c_upcast[thread_num][i * self.n_features + j] =  self.X[X_start + i, j]
+{{else}}
+        return
+{{endif}}
+
+    cdef void _parallel_on_Y_parallel_init(
+        self,
+        intp_t thread_num,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil:
+{{if upcast_to_float64}}
+        cdef:
+            intp_t i, j
+            intp_t n_chunk_samples = X_end - X_start
+
+        # Upcasting X_c=X[X_start:X_end, :] from float32 to float64
+        for i in range(n_chunk_samples):
+            for j in range(self.n_features):
+                self.X_c_upcast[thread_num][i * self.n_features + j] =  self.X[X_start + i, j]
+{{else}}
+        return
+{{endif}}
+
+    cdef void _parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num
+    ) noexcept nogil:
+{{if upcast_to_float64}}
+        cdef:
+            intp_t i, j
+            intp_t n_chunk_samples = Y_end - Y_start
+
+        # Upcasting Y_c=Y[Y_start:Y_end, :] from float32 to float64
+        for i in range(n_chunk_samples):
+            for j in range(self.n_features):
+                self.Y_c_upcast[thread_num][i * self.n_features + j] =  self.Y[Y_start + i, j]
+{{else}}
+        return
+{{endif}}
+
+    cdef float64_t * _compute_dist_middle_terms(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil:
+        cdef:
+            float64_t *dist_middle_terms = self.dist_middle_terms_chunks[thread_num].data()
+
+            # Careful: LDA, LDB and LDC are given for F-ordered arrays
+            # in BLAS documentations, for instance:
+            # https://www.netlib.org/lapack/explore-html/db/dc9/group__single__blas__level3_gafe51bacb54592ff5de056acabd83c260.html #noqa
+            #
+            # Here, we use their counterpart values to work with C-ordered arrays.
+            BLAS_Order order = RowMajor
+            BLAS_Trans ta = NoTrans
+            BLAS_Trans tb = Trans
+            intp_t m = X_end - X_start
+            intp_t n = Y_end - Y_start
+            intp_t K = self.n_features
+            float64_t alpha = - 2.
+{{if upcast_to_float64}}
+            float64_t * A = self.X_c_upcast[thread_num].data()
+            float64_t * B = self.Y_c_upcast[thread_num].data()
+{{else}}
+            # Casting for A and B to remove the const is needed because APIs exposed via
+            # scipy.linalg.cython_blas aren't reflecting the arguments' const qualifier.
+            # See: https://github.com/scipy/scipy/issues/14262
+            float64_t * A =  &self.X[X_start, 0]
+            float64_t * B =  &self.Y[Y_start, 0]
+{{endif}}
+            intp_t lda = self.n_features
+            intp_t ldb = self.n_features
+            float64_t beta = 0.
+            intp_t ldc = Y_end - Y_start
+
+        # dist_middle_terms = `-2 * X[X_start:X_end] @ Y[Y_start:Y_end].T`
+        _gemm(order, ta, tb, m, n, K, alpha, A, lda, B, ldb, beta, dist_middle_terms, ldc)
+
+        return dist_middle_terms
+
+
+cdef class SparseSparseMiddleTermComputer{{name_suffix}}(MiddleTermComputer{{name_suffix}}):
+    """Middle term of the Euclidean distance between two chunked CSR matrices.
+
+    The result is return as a contiguous array.
+
+            dist_middle_terms = - 2 X_c_i.Y_c_j^T
+
+    The logic of the computation is wrapped in the routine _middle_term_sparse_sparse_64.
+    This routine iterates over the data, indices and indptr arrays of the sparse matrices without
+    densifying them.
+    """
+
+    def __init__(
+        self,
+        X,
+        Y,
+        intp_t effective_n_threads,
+        intp_t chunks_n_threads,
+        intp_t dist_middle_terms_chunks_size,
+        intp_t n_features,
+        intp_t chunk_size,
+    ):
+        super().__init__(
+            effective_n_threads,
+            chunks_n_threads,
+            dist_middle_terms_chunks_size,
+            n_features,
+            chunk_size,
+        )
+        self.X_data, self.X_indices, self.X_indptr = self.unpack_csr_matrix(X)
+        self.Y_data, self.Y_indices, self.Y_indptr = self.unpack_csr_matrix(Y)
+
+    cdef void _parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil:
+        # Flush the thread dist_middle_terms_chunks to 0.0
+        fill(
+            self.dist_middle_terms_chunks[thread_num].begin(),
+            self.dist_middle_terms_chunks[thread_num].end(),
+            0.0,
+        )
+
+    cdef void _parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil:
+        # Flush the thread dist_middle_terms_chunks to 0.0
+        fill(
+            self.dist_middle_terms_chunks[thread_num].begin(),
+            self.dist_middle_terms_chunks[thread_num].end(),
+            0.0,
+        )
+
+    cdef float64_t * _compute_dist_middle_terms(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil:
+        cdef:
+            float64_t *dist_middle_terms = (
+                self.dist_middle_terms_chunks[thread_num].data()
+            )
+
+        _middle_term_sparse_sparse_64(
+            self.X_data,
+            self.X_indices,
+            self.X_indptr,
+            X_start,
+            X_end,
+            self.Y_data,
+            self.Y_indices,
+            self.Y_indptr,
+            Y_start,
+            Y_end,
+            dist_middle_terms,
+        )
+
+        return dist_middle_terms
+
+cdef class SparseDenseMiddleTermComputer{{name_suffix}}(MiddleTermComputer{{name_suffix}}):
+    """Middle term of the Euclidean distance between chunks of a CSR matrix and a np.ndarray.
+
+    The logic of the computation is wrapped in the routine _middle_term_sparse_dense_{{name_suffix}}.
+    This routine iterates over the data, indices and indptr arrays of the sparse matrices
+    without densifying them.
+    """
+
+    def __init__(
+        self,
+        X,
+        Y,
+        intp_t effective_n_threads,
+        intp_t chunks_n_threads,
+        intp_t dist_middle_terms_chunks_size,
+        intp_t n_features,
+        intp_t chunk_size,
+        bint c_ordered_middle_term,
+    ):
+        super().__init__(
+            effective_n_threads,
+            chunks_n_threads,
+            dist_middle_terms_chunks_size,
+            n_features,
+            chunk_size,
+        )
+        self.X_data, self.X_indices, self.X_indptr = self.unpack_csr_matrix(X)
+        self.Y = Y
+        self.c_ordered_middle_term = c_ordered_middle_term
+
+    cdef void _parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil:
+        # Fill the thread's dist_middle_terms_chunks with 0.0 before
+        # computing its elements in _compute_dist_middle_terms.
+        fill(
+            self.dist_middle_terms_chunks[thread_num].begin(),
+            self.dist_middle_terms_chunks[thread_num].end(),
+            0.0,
+        )
+
+    cdef void _parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil:
+        # Fill the thread's dist_middle_terms_chunks with 0.0 before
+        # computing its elements in _compute_dist_middle_terms.
+        fill(
+            self.dist_middle_terms_chunks[thread_num].begin(),
+            self.dist_middle_terms_chunks[thread_num].end(),
+            0.0,
+        )
+
+    cdef float64_t * _compute_dist_middle_terms(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil:
+        cdef:
+            float64_t *dist_middle_terms = (
+                self.dist_middle_terms_chunks[thread_num].data()
+            )
+
+        # For the dense-sparse case, we use the sparse-dense case
+        # with dist_middle_terms seen as F-ordered.
+        # Hence we swap indices pointers here.
+        if not self.c_ordered_middle_term:
+            X_start, Y_start = Y_start, X_start
+            X_end, Y_end = Y_end, X_end
+
+        _middle_term_sparse_dense_{{name_suffix}}(
+            self.X_data,
+            self.X_indices,
+            self.X_indptr,
+            X_start,
+            X_end,
+            self.Y,
+            Y_start,
+            Y_end,
+            self.c_ordered_middle_term,
+            dist_middle_terms,
+        )
+
+        return dist_middle_terms
+
+{{endfor}}
diff --git a/sklearn/metrics/_pairwise_distances_reduction/_radius_neighbors.pxd.tp b/sklearn/metrics/_pairwise_distances_reduction/_radius_neighbors.pxd.tp
new file mode 100644
index 0000000000000..809a80a68c5b0
--- /dev/null
+++ b/sklearn/metrics/_pairwise_distances_reduction/_radius_neighbors.pxd.tp
@@ -0,0 +1,90 @@
+cimport numpy as cnp
+
+from libcpp.memory cimport shared_ptr
+from libcpp.vector cimport vector
+from cython cimport final
+
+from ...utils._typedefs cimport intp_t, float64_t
+
+cnp.import_array()
+
+######################
+## std::vector to np.ndarray coercion
+# As type covariance is not supported for C++ containers via Cython,
+# we need to redefine fused types.
+ctypedef fused vector_double_intp_t:
+    vector[intp_t]
+    vector[float64_t]
+
+
+ctypedef fused vector_vector_double_intp_t:
+    vector[vector[intp_t]]
+    vector[vector[float64_t]]
+
+cdef cnp.ndarray[object, ndim=1] coerce_vectors_to_nd_arrays(
+    shared_ptr[vector_vector_double_intp_t] vecs
+)
+
+#####################
+{{for name_suffix in ['64', '32']}}
+
+from ._base cimport BaseDistancesReduction{{name_suffix}}
+from ._middle_term_computer cimport MiddleTermComputer{{name_suffix}}
+
+cdef class RadiusNeighbors{{name_suffix}}(BaseDistancesReduction{{name_suffix}}):
+    """float{{name_suffix}} implementation of the RadiusNeighbors."""
+
+    cdef:
+        float64_t radius
+
+        # DistanceMetric{{name_suffix}} compute rank-preserving surrogate distance via rdist
+        # which are proxies necessitating less computations.
+        # We get the equivalent for the radius to be able to compare it against
+        # vectors' rank-preserving surrogate distances.
+        float64_t r_radius
+
+        # Neighbors indices and distances are returned as np.ndarrays of np.ndarrays.
+        #
+        # For this implementation, we want resizable buffers which we will wrap
+        # into numpy arrays at the end. std::vector comes as a handy container
+        # for interacting efficiently with resizable buffers.
+        #
+        # Though it is possible to access their buffer address with
+        # std::vector::data, they can't be stolen: buffers lifetime
+        # is tied to their std::vector and are deallocated when
+        # std::vectors are.
+        #
+        # To solve this, we dynamically allocate std::vectors and then
+        # encapsulate them in a StdVectorSentinel responsible for
+        # freeing them when the associated np.ndarray is freed.
+        #
+        # Shared pointers (defined via shared_ptr) are use for safer memory management.
+        # Unique pointers (defined via unique_ptr) can't be used as datastructures
+        # are shared across threads for parallel_on_X; see _parallel_on_X_init_chunk.
+        shared_ptr[vector[vector[intp_t]]] neigh_indices
+        shared_ptr[vector[vector[float64_t]]] neigh_distances
+
+        # Used as array of pointers to private datastructures used in threads.
+        vector[shared_ptr[vector[vector[intp_t]]]] neigh_indices_chunks
+        vector[shared_ptr[vector[vector[float64_t]]]] neigh_distances_chunks
+
+        bint sort_results
+
+    @final
+    cdef void _merge_vectors(
+        self,
+        intp_t idx,
+        intp_t num_threads,
+    ) noexcept nogil
+
+
+cdef class EuclideanRadiusNeighbors{{name_suffix}}(RadiusNeighbors{{name_suffix}}):
+    """EuclideanDistance-specialisation of RadiusNeighbors{{name_suffix}}."""
+    cdef:
+        MiddleTermComputer{{name_suffix}} middle_term_computer
+        const float64_t[::1] X_norm_squared
+        const float64_t[::1] Y_norm_squared
+
+        bint use_squared_distances
+
+{{endfor}}
diff --git a/sklearn/metrics/_pairwise_distances_reduction/_radius_neighbors.pyx.tp b/sklearn/metrics/_pairwise_distances_reduction/_radius_neighbors.pyx.tp
new file mode 100644
index 0000000000000..d0567f2ead804
--- /dev/null
+++ b/sklearn/metrics/_pairwise_distances_reduction/_radius_neighbors.pyx.tp
@@ -0,0 +1,514 @@
+cimport numpy as cnp
+import numpy as np
+import warnings
+
+from libcpp.memory cimport shared_ptr, make_shared
+from libcpp.vector cimport vector
+from libcpp.algorithm cimport move
+from cython cimport final
+from cython.operator cimport dereference as deref
+from cython.parallel cimport parallel, prange
+
+from ...utils._sorting cimport simultaneous_sort
+from ...utils._typedefs cimport intp_t, float64_t
+from ...utils._vector_sentinel cimport vector_to_nd_array
+
+from numbers import Real
+from scipy.sparse import issparse
+from ...utils import check_array, check_scalar
+from ...utils.fixes import _in_unstable_openblas_configuration
+from ...utils.parallel import _get_threadpool_controller
+
+cnp.import_array()
+
+######################
+
+cdef cnp.ndarray[object, ndim=1] coerce_vectors_to_nd_arrays(
+    shared_ptr[vector_vector_double_intp_t] vecs
+):
+    """Coerce a std::vector of std::vector to a ndarray of ndarray."""
+    cdef:
+        intp_t n = deref(vecs).size()
+        cnp.ndarray[object, ndim=1] nd_arrays_of_nd_arrays = np.empty(n, dtype=np.ndarray)
+
+    for i in range(n):
+        nd_arrays_of_nd_arrays[i] = vector_to_nd_array(&(deref(vecs)[i]))
+
+    return nd_arrays_of_nd_arrays
+
+#####################
+{{for name_suffix in ['64', '32']}}
+
+from ._base cimport (
+    BaseDistancesReduction{{name_suffix}},
+    _sqeuclidean_row_norms{{name_suffix}}
+)
+
+from ._datasets_pair cimport DatasetsPair{{name_suffix}}
+
+from ._middle_term_computer cimport MiddleTermComputer{{name_suffix}}
+
+
+cdef class RadiusNeighbors{{name_suffix}}(BaseDistancesReduction{{name_suffix}}):
+    """float{{name_suffix}} implementation of the RadiusNeighbors."""
+
+    @classmethod
+    def compute(
+        cls,
+        X,
+        Y,
+        float64_t radius,
+        str metric="euclidean",
+        chunk_size=None,
+        dict metric_kwargs=None,
+        str strategy=None,
+        bint return_distance=False,
+        bint sort_results=False,
+    ):
+        """Compute the radius-neighbors reduction.
+
+        This classmethod is responsible for introspecting the arguments
+        values to dispatch to the most appropriate implementation of
+        :class:`RadiusNeighbors{{name_suffix}}`.
+
+        This allows decoupling the API entirely from the implementation details
+        whilst maintaining RAII: all temporarily allocated datastructures necessary
+        for the concrete implementation are therefore freed when this classmethod
+        returns.
+
+        No instance should directly be created outside of this class method.
+        """
+        if metric in ("euclidean", "sqeuclidean"):
+            # Specialized implementation of RadiusNeighbors for the Euclidean
+            # distance for the dense-dense and sparse-sparse cases.
+            # This implementation computes the distances by chunk using
+            # a decomposition of the Squared Euclidean distance.
+            # This specialisation has an improved arithmetic intensity for both
+            # the dense and sparse settings, allowing in most case speed-ups of
+            # several orders of magnitude compared to the generic RadiusNeighbors
+            # implementation.
+            # For more information see MiddleTermComputer.
+            use_squared_distances = metric == "sqeuclidean"
+            pda = EuclideanRadiusNeighbors{{name_suffix}}(
+                X=X, Y=Y, radius=radius,
+                use_squared_distances=use_squared_distances,
+                chunk_size=chunk_size,
+                strategy=strategy,
+                sort_results=sort_results,
+                metric_kwargs=metric_kwargs,
+            )
+        else:
+             # Fall back on a generic implementation that handles most scipy
+             # metrics by computing the distances between 2 vectors at a time.
+            pda = RadiusNeighbors{{name_suffix}}(
+                datasets_pair=DatasetsPair{{name_suffix}}.get_for(X, Y, metric, metric_kwargs),
+                radius=radius,
+                chunk_size=chunk_size,
+                strategy=strategy,
+                sort_results=sort_results,
+            )
+
+        # Limit the number of threads in second level of nested parallelism for BLAS
+        # to avoid threads over-subscription (in GEMM for instance).
+        with _get_threadpool_controller().limit(limits=1, user_api="blas"):
+            if pda.execute_in_parallel_on_Y:
+                pda._parallel_on_Y()
+            else:
+                pda._parallel_on_X()
+
+        return pda._finalize_results(return_distance)
+
+
+    def __init__(
+        self,
+        DatasetsPair{{name_suffix}} datasets_pair,
+        float64_t radius,
+        chunk_size=None,
+        strategy=None,
+        sort_results=False,
+    ):
+        super().__init__(
+            datasets_pair=datasets_pair,
+            chunk_size=chunk_size,
+            strategy=strategy,
+        )
+
+        self.radius = check_scalar(radius, "radius", Real, min_val=0)
+        self.r_radius = self.datasets_pair.distance_metric._dist_to_rdist(radius)
+        self.sort_results = sort_results
+
+        # Allocating pointers to datastructures but not the datastructures themselves.
+        # There are as many pointers as effective threads.
+        #
+        # For the sake of explicitness:
+        #   - when parallelizing on X, the pointers of those heaps are referencing
+        #   self.neigh_distances and self.neigh_indices
+        #   - when parallelizing on Y, the pointers of those heaps are referencing
+        #   std::vectors of std::vectors which are thread-wise-allocated and whose
+        #   content will be merged into self.neigh_distances and self.neigh_indices.
+        self.neigh_distances_chunks = vector[shared_ptr[vector[vector[float64_t]]]](
+            self.chunks_n_threads
+        )
+        self.neigh_indices_chunks = vector[shared_ptr[vector[vector[intp_t]]]](
+            self.chunks_n_threads
+        )
+
+        # Temporary datastructures which will be coerced to numpy arrays on before
+        # RadiusNeighbors.compute "return" and will be then freed.
+        self.neigh_distances = make_shared[vector[vector[float64_t]]](self.n_samples_X)
+        self.neigh_indices = make_shared[vector[vector[intp_t]]](self.n_samples_X)
+
+    cdef void _compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil:
+        cdef:
+            intp_t i, j
+            float64_t r_dist_i_j
+
+        for i in range(X_start, X_end):
+            for j in range(Y_start, Y_end):
+                r_dist_i_j = self.datasets_pair.surrogate_dist(i, j)
+                if r_dist_i_j <= self.r_radius:
+                    deref(self.neigh_distances_chunks[thread_num])[i].push_back(r_dist_i_j)
+                    deref(self.neigh_indices_chunks[thread_num])[i].push_back(j)
+
+    def _finalize_results(self, bint return_distance=False):
+        if return_distance:
+            # We need to recompute distances because we relied on
+            # surrogate distances for the reduction.
+            self.compute_exact_distances()
+            return (
+                coerce_vectors_to_nd_arrays(self.neigh_distances),
+                coerce_vectors_to_nd_arrays(self.neigh_indices),
+            )
+
+        return coerce_vectors_to_nd_arrays(self.neigh_indices)
+
+    cdef void _parallel_on_X_init_chunk(
+        self,
+        intp_t thread_num,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil:
+
+        # As this strategy is embarrassingly parallel, we can set the
+        # thread vectors' pointers to the main vectors'.
+        self.neigh_distances_chunks[thread_num] = self.neigh_distances
+        self.neigh_indices_chunks[thread_num] = self.neigh_indices
+
+    @final
+    cdef void _parallel_on_X_prange_iter_finalize(
+        self,
+        intp_t thread_num,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil:
+        cdef:
+            intp_t idx
+
+        # Sorting neighbors for each query vector of X
+        if self.sort_results:
+            for idx in range(X_start, X_end):
+                simultaneous_sort(
+                    deref(self.neigh_distances)[idx].data(),
+                    deref(self.neigh_indices)[idx].data(),
+                    deref(self.neigh_indices)[idx].size()
+                )
+
+    cdef void _parallel_on_Y_init(
+        self,
+    ) noexcept nogil:
+        cdef:
+            intp_t thread_num
+        # As chunks of X are shared across threads, so must datastructures to avoid race
+        # conditions: each thread has its own vectors of n_samples_X vectors which are
+        # then merged back in the main n_samples_X vectors.
+        for thread_num in range(self.chunks_n_threads):
+            self.neigh_distances_chunks[thread_num] = make_shared[vector[vector[float64_t]]](self.n_samples_X)
+            self.neigh_indices_chunks[thread_num] = make_shared[vector[vector[intp_t]]](self.n_samples_X)
+
+    @final
+    cdef void _merge_vectors(
+        self,
+        intp_t idx,
+        intp_t num_threads,
+    ) noexcept nogil:
+        cdef:
+            intp_t thread_num
+            intp_t idx_n_elements = 0
+            intp_t last_element_idx = deref(self.neigh_indices)[idx].size()
+
+        # Resizing buffers only once for the given number of elements.
+        for thread_num in range(num_threads):
+            idx_n_elements += deref(self.neigh_distances_chunks[thread_num])[idx].size()
+
+        deref(self.neigh_distances)[idx].resize(last_element_idx + idx_n_elements)
+        deref(self.neigh_indices)[idx].resize(last_element_idx + idx_n_elements)
+
+        # Moving the elements by range using the range first element
+        # as the reference for the insertion.
+        for thread_num in range(num_threads):
+            move(
+                deref(self.neigh_distances_chunks[thread_num])[idx].begin(),
+                deref(self.neigh_distances_chunks[thread_num])[idx].end(),
+                deref(self.neigh_distances)[idx].begin() + last_element_idx
+            )
+            move(
+                deref(self.neigh_indices_chunks[thread_num])[idx].begin(),
+                deref(self.neigh_indices_chunks[thread_num])[idx].end(),
+                deref(self.neigh_indices)[idx].begin() + last_element_idx
+            )
+            last_element_idx += deref(self.neigh_distances_chunks[thread_num])[idx].size()
+
+    cdef void _parallel_on_Y_finalize(
+        self,
+    ) noexcept nogil:
+        cdef:
+            intp_t idx
+
+        with nogil, parallel(num_threads=self.effective_n_threads):
+            # Merge vectors used in threads into the main ones.
+            # This is done in parallel sample-wise (no need for locks).
+            for idx in prange(self.n_samples_X, schedule='static'):
+                self._merge_vectors(idx, self.chunks_n_threads)
+
+            # The content of the vector have been std::moved.
+            # Hence they can't be used anymore and can be deleted.
+            # Their deletion is carried out automatically as the
+            # implementation relies on shared pointers.
+
+            # Sort in parallel in ascending order w.r.t the distances if requested.
+            if self.sort_results:
+                for idx in prange(self.n_samples_X, schedule='static'):
+                    simultaneous_sort(
+                        deref(self.neigh_distances)[idx].data(),
+                        deref(self.neigh_indices)[idx].data(),
+                        deref(self.neigh_indices)[idx].size()
+                    )
+
+        return
+
+    cdef void compute_exact_distances(self) noexcept nogil:
+        """Convert rank-preserving distances to pairwise distances in parallel."""
+        cdef:
+            intp_t i
+            vector[intp_t].size_type j
+
+        for i in prange(self.n_samples_X, nogil=True, schedule='static',
+                        num_threads=self.effective_n_threads):
+            for j in range(deref(self.neigh_indices)[i].size()):
+                deref(self.neigh_distances)[i][j] = (
+                        self.datasets_pair.distance_metric._rdist_to_dist(
+                            # Guard against potential -0., causing nan production.
+                            max(deref(self.neigh_distances)[i][j], 0.)
+                        )
+                )
+
+
+cdef class EuclideanRadiusNeighbors{{name_suffix}}(RadiusNeighbors{{name_suffix}}):
+    """EuclideanDistance-specialisation of RadiusNeighbors{{name_suffix}}."""
+
+    @classmethod
+    def is_usable_for(cls, X, Y, metric) -> bool:
+        return (RadiusNeighbors{{name_suffix}}.is_usable_for(X, Y, metric)
+                and not _in_unstable_openblas_configuration())
+
+    def __init__(
+        self,
+        X,
+        Y,
+        float64_t radius,
+        bint use_squared_distances=False,
+        chunk_size=None,
+        strategy=None,
+        sort_results=False,
+        metric_kwargs=None,
+    ):
+        if (
+            isinstance(metric_kwargs, dict) and
+            (metric_kwargs.keys() - {"X_norm_squared", "Y_norm_squared"})
+        ):
+            warnings.warn(
+                f"Some metric_kwargs have been passed ({metric_kwargs}) but aren't "
+                f"usable for this case (EuclideanRadiusNeighbors64) and will be ignored.",
+                UserWarning,
+                stacklevel=3,
+            )
+
+        super().__init__(
+            # The datasets pair here is used for exact distances computations
+            datasets_pair=DatasetsPair{{name_suffix}}.get_for(X, Y, metric="euclidean"),
+            radius=radius,
+            chunk_size=chunk_size,
+            strategy=strategy,
+            sort_results=sort_results,
+        )
+        cdef:
+            intp_t dist_middle_terms_chunks_size = self.Y_n_samples_chunk * self.X_n_samples_chunk
+
+        self.middle_term_computer = MiddleTermComputer{{name_suffix}}.get_for(
+            X,
+            Y,
+            self.effective_n_threads,
+            self.chunks_n_threads,
+            dist_middle_terms_chunks_size,
+            n_features=X.shape[1],
+            chunk_size=self.chunk_size,
+        )
+
+        if metric_kwargs is not None and "Y_norm_squared" in metric_kwargs:
+            self.Y_norm_squared = check_array(
+                metric_kwargs.pop("Y_norm_squared"),
+                ensure_2d=False,
+                input_name="Y_norm_squared",
+                dtype=np.float64,
+            )
+        else:
+            self.Y_norm_squared = _sqeuclidean_row_norms{{name_suffix}}(
+                Y,
+                self.effective_n_threads,
+            )
+
+        if metric_kwargs is not None and "X_norm_squared" in metric_kwargs:
+            self.X_norm_squared = check_array(
+                metric_kwargs.pop("X_norm_squared"),
+                ensure_2d=False,
+                input_name="X_norm_squared",
+                dtype=np.float64,
+            )
+        else:
+            # Do not recompute norms if datasets are identical.
+            self.X_norm_squared = (
+                self.Y_norm_squared if X is Y else
+                _sqeuclidean_row_norms{{name_suffix}}(
+                    X,
+                    self.effective_n_threads,
+                )
+            )
+
+        self.use_squared_distances = use_squared_distances
+
+        if use_squared_distances:
+            # In this specialisation and this setup, the value passed to the radius is
+            # already considered to be the adapted radius, so we overwrite it.
+            self.r_radius = radius
+
+    @final
+    cdef void _parallel_on_X_parallel_init(
+        self,
+        intp_t thread_num,
+    ) noexcept nogil:
+        RadiusNeighbors{{name_suffix}}._parallel_on_X_parallel_init(self, thread_num)
+        self.middle_term_computer._parallel_on_X_parallel_init(thread_num)
+
+    @final
+    cdef void _parallel_on_X_init_chunk(
+        self,
+        intp_t thread_num,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil:
+        RadiusNeighbors{{name_suffix}}._parallel_on_X_init_chunk(self, thread_num, X_start, X_end)
+        self.middle_term_computer._parallel_on_X_init_chunk(thread_num, X_start, X_end)
+
+    @final
+    cdef void _parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil:
+        RadiusNeighbors{{name_suffix}}._parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
+            self,
+            X_start, X_end,
+            Y_start, Y_end,
+            thread_num,
+        )
+        self.middle_term_computer._parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
+            X_start, X_end, Y_start, Y_end, thread_num,
+        )
+
+    @final
+    cdef void _parallel_on_Y_init(
+        self,
+    ) noexcept nogil:
+        RadiusNeighbors{{name_suffix}}._parallel_on_Y_init(self)
+        self.middle_term_computer._parallel_on_Y_init()
+
+    @final
+    cdef void _parallel_on_Y_parallel_init(
+        self,
+        intp_t thread_num,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil:
+        RadiusNeighbors{{name_suffix}}._parallel_on_Y_parallel_init(self, thread_num, X_start, X_end)
+        self.middle_term_computer._parallel_on_Y_parallel_init(thread_num, X_start, X_end)
+
+    @final
+    cdef void _parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil:
+        RadiusNeighbors{{name_suffix}}._parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
+            self,
+            X_start, X_end,
+            Y_start, Y_end,
+            thread_num,
+        )
+        self.middle_term_computer._parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
+            X_start, X_end, Y_start, Y_end, thread_num
+        )
+
+    @final
+    cdef void compute_exact_distances(self) noexcept nogil:
+        if not self.use_squared_distances:
+            RadiusNeighbors{{name_suffix}}.compute_exact_distances(self)
+
+    @final
+    cdef void _compute_and_reduce_distances_on_chunks(
+        self,
+        intp_t X_start,
+        intp_t X_end,
+        intp_t Y_start,
+        intp_t Y_end,
+        intp_t thread_num,
+    ) noexcept nogil:
+        cdef:
+            intp_t i, j
+            float64_t sqeuclidean_dist_i_j
+            intp_t n_X = X_end - X_start
+            intp_t n_Y = Y_end - Y_start
+            float64_t *dist_middle_terms = self.middle_term_computer._compute_dist_middle_terms(
+                X_start, X_end, Y_start, Y_end, thread_num
+            )
+
+        # Pushing the distance and their associated indices in vectors.
+        for i in range(n_X):
+            for j in range(n_Y):
+                sqeuclidean_dist_i_j = (
+                    self.X_norm_squared[i + X_start]
+                    + dist_middle_terms[i * n_Y + j]
+                    + self.Y_norm_squared[j + Y_start]
+                )
+
+                # Catastrophic cancellation might cause -0. to be present,
+                # e.g. when computing d(x_i, y_i) when X is Y.
+                sqeuclidean_dist_i_j = max(0., sqeuclidean_dist_i_j)
+
+                if sqeuclidean_dist_i_j <= self.r_radius:
+                    deref(self.neigh_distances_chunks[thread_num])[i + X_start].push_back(sqeuclidean_dist_i_j)
+                    deref(self.neigh_indices_chunks[thread_num])[i + X_start].push_back(j + Y_start)
+
+{{endfor}}
diff --git a/sklearn/metrics/_pairwise_distances_reduction/_radius_neighbors_classmode.pyx.tp b/sklearn/metrics/_pairwise_distances_reduction/_radius_neighbors_classmode.pyx.tp
new file mode 100644
index 0000000000000..0a9b22251843e
--- /dev/null
+++ b/sklearn/metrics/_pairwise_distances_reduction/_radius_neighbors_classmode.pyx.tp
@@ -0,0 +1,217 @@
+import warnings
+
+from cython cimport floating, final, integral
+from cython.operator cimport dereference as deref
+from cython.parallel cimport parallel, prange
+from ._classmode cimport WeightingStrategy
+from ...utils._typedefs cimport intp_t, float64_t, uint8_t
+
+import numpy as np
+from scipy.sparse import issparse
+from ...utils.parallel import _get_threadpool_controller
+
+
+{{for name_suffix in ["32", "64"]}}
+from ._radius_neighbors cimport RadiusNeighbors{{name_suffix}}
+from ._datasets_pair cimport DatasetsPair{{name_suffix}}
+
+cdef class RadiusNeighborsClassMode{{name_suffix}}(RadiusNeighbors{{name_suffix}}):
+    """
+    {{name_suffix}}bit implementation of RadiusNeighborsClassMode.
+    """
+    cdef:
+        const intp_t[::1] Y_labels
+        const intp_t[::1] unique_Y_labels
+        intp_t outlier_label_index
+        bint outlier_label_exists
+        bint outliers_exist
+        uint8_t[::1] outliers
+        object outlier_label
+        float64_t[:, ::1] class_scores
+        WeightingStrategy weight_type
+
+    @classmethod
+    def compute(
+        cls,
+        X,
+        Y,
+        float64_t radius,
+        weights,
+        Y_labels,
+        unique_Y_labels,
+        outlier_label=None,
+        str metric="euclidean",
+        chunk_size=None,
+        dict metric_kwargs=None,
+        str strategy=None,
+    ):
+        # Use a generic implementation that handles most scipy
+        # metrics by computing the distances between 2 vectors at a time.
+        pda = RadiusNeighborsClassMode{{name_suffix}}(
+            datasets_pair=DatasetsPair{{name_suffix}}.get_for(X, Y, metric, metric_kwargs),
+            radius=radius,
+            chunk_size=chunk_size,
+            strategy=strategy,
+            weights=weights,
+            Y_labels=Y_labels,
+            unique_Y_labels=unique_Y_labels,
+            outlier_label=outlier_label,
+        )
+
+        # Limit the number of threads in second level of nested parallelism for BLAS
+        # to avoid threads over-subscription (in GEMM for instance).
+        with _get_threadpool_controller().limit(limits=1, user_api="blas"):
+            if pda.execute_in_parallel_on_Y:
+                pda._parallel_on_Y()
+            else:
+                pda._parallel_on_X()
+
+        return pda._finalize_results()
+
+    def __init__(
+        self,
+        DatasetsPair{{name_suffix}} datasets_pair,
+        const intp_t[::1] Y_labels,
+        const intp_t[::1] unique_Y_labels,
+        float64_t radius,
+        chunk_size=None,
+        strategy=None,
+        weights=None,
+        outlier_label=None,
+    ):
+        super().__init__(
+            datasets_pair=datasets_pair,
+            chunk_size=chunk_size,
+            strategy=strategy,
+            radius=radius,
+        )
+
+        if weights == "uniform":
+            self.weight_type = WeightingStrategy.uniform
+        elif weights == "distance":
+            self.weight_type = WeightingStrategy.distance
+        else:
+            self.weight_type = WeightingStrategy.callable
+
+        self.Y_labels = Y_labels
+        self.unique_Y_labels = unique_Y_labels
+        self.outlier_label_index = -1
+        self.outliers_exist = False
+        self.outlier_label = outlier_label
+        self.outliers = np.zeros(self.n_samples_X, dtype=np.bool_)
+
+        cdef intp_t idx
+        if self.outlier_label is not None:
+            for idx in range(self.unique_Y_labels.shape[0]):
+                if self.unique_Y_labels[idx] == outlier_label:
+                    self.outlier_label_index = idx
+
+        # Map from set of unique labels to their indices in `class_scores`
+        # Buffer used in building a histogram for one-pass weighted mode
+        self.class_scores = np.zeros(
+            (self.n_samples_X, unique_Y_labels.shape[0]), dtype=np.float64,
+        )
+
+
+    cdef inline void weighted_histogram_mode(
+        self,
+        intp_t sample_index,
+        intp_t sample_n_neighbors,
+        intp_t* indices,
+        float64_t* distances,
+    ) noexcept nogil:
+        cdef:
+            intp_t neighbor_idx, neighbor_class_idx, label_index
+            float64_t score_incr = 1
+            bint use_distance_weighting = (
+                self.weight_type == WeightingStrategy.distance
+            )
+
+        if sample_n_neighbors == 0:
+            self.outliers_exist = True
+            self.outliers[sample_index] = True
+            if self.outlier_label_index >= 0:
+                self.class_scores[sample_index][self.outlier_label_index] = score_incr
+
+            return
+
+        # Iterate over the neighbors. This can be different for
+        # each of the samples as they are based on the radius.
+        for neighbor_rank in range(sample_n_neighbors):
+            if use_distance_weighting:
+                score_incr = 1 / distances[neighbor_rank]
+
+            neighbor_idx = indices[neighbor_rank]
+            neighbor_class_idx = self.Y_labels[neighbor_idx]
+            self.class_scores[sample_index][neighbor_class_idx] += score_incr
+
+        return
+
+    @final
+    cdef void _parallel_on_X_prange_iter_finalize(
+        self,
+        intp_t thread_num,
+        intp_t X_start,
+        intp_t X_end,
+    ) noexcept nogil:
+        cdef:
+            intp_t idx
+
+        for idx in range(X_start, X_end):
+            self.weighted_histogram_mode(
+                sample_index=idx,
+                sample_n_neighbors=deref(self.neigh_indices)[idx].size(),
+                indices=deref(self.neigh_indices)[idx].data(),
+                distances=deref(self.neigh_distances)[idx].data(),
+            )
+
+        return
+
+    @final
+    cdef void _parallel_on_Y_finalize(
+        self,
+    ) noexcept nogil:
+        cdef:
+            intp_t idx
+
+        with nogil, parallel(num_threads=self.effective_n_threads):
+            # Merge vectors used in threads into the main ones.
+            # This is done in parallel sample-wise (no need for locks).
+            for idx in prange(self.n_samples_X, schedule='static'):
+                self._merge_vectors(idx, self.chunks_n_threads)
+
+            for idx in prange(self.n_samples_X, schedule='static'):
+                self.weighted_histogram_mode(
+                    sample_index=idx,
+                    sample_n_neighbors=deref(self.neigh_indices)[idx].size(),
+                    indices=deref(self.neigh_indices)[idx].data(),
+                    distances=deref(self.neigh_distances)[idx].data(),
+                )
+
+        return
+
+    def _finalize_results(self):
+        if self.outliers_exist and self.outlier_label is None:
+            raise ValueError(
+                "No neighbors found for test samples %r, "
+                "you can try using larger radius, "
+                "giving a label for outliers, "
+                "or considering removing them from your dataset."
+                % np.where(self.outliers)[0]
+            )
+
+        if self.outliers_exist and self.outlier_label_index < 0:
+            warnings.warn(
+                "Outlier label %s is not in training "
+                "classes. All class probabilities of "
+                "outliers will be assigned with 0."
+                % self.outlier_label
+            )
+
+        probabilities = np.asarray(self.class_scores)
+        normalizer = probabilities.sum(axis=1, keepdims=True)
+        normalizer[normalizer == 0.0] = 1.0
+        probabilities /= normalizer
+        return probabilities
+
+{{endfor}}
diff --git a/sklearn/metrics/_pairwise_distances_reduction/meson.build b/sklearn/metrics/_pairwise_distances_reduction/meson.build
new file mode 100644
index 0000000000000..0f7eaa286399c
--- /dev/null
+++ b/sklearn/metrics/_pairwise_distances_reduction/meson.build
@@ -0,0 +1,193 @@
+# Note: the dependencies between different Cython files in
+# _pairwise_distances_reduction is probably one of the most involved in
+# scikit-learn. If you change this file make sure you build from scratch:
+# rm -rf build; make dev-meson
+# run a command like this:
+# ninja -C build/cp312 -t missingdeps
+# and make sure that the output is something like:
+# No missing dependencies on generated files found.
+
+# _pairwise_distances_reduction is cimported from other subpackages so this is
+# needed for the cimport to work
+_pairwise_distances_reduction_cython_tree = [
+  fs.copyfile('__init__.py'),
+  # We are in a sub-module of metrics, so we always need to have
+  # sklearn/metrics/__init__.py copied to the build directory to avoid the
+  # error:
+  # relative cimport beyond main package is not allowed
+  metrics_cython_tree
+]
+
+_classmode_pxd = fs.copyfile('_classmode.pxd')
+
+_datasets_pair_pxd = custom_target(
+  '_datasets_pair_pxd',
+  output: '_datasets_pair.pxd',
+  input: '_datasets_pair.pxd.tp',
+  command: [tempita, '@INPUT@', '-o', '@OUTDIR@']
+)
+_datasets_pair_pyx = custom_target(
+  '_datasets_pair_pyx',
+  output: '_datasets_pair.pyx',
+  input: '_datasets_pair.pyx.tp',
+  command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
+  # TODO in principle this should go in py.exension_module below. This is
+  # temporary work-around for dependency issue with .pyx.tp files. For more
+  # details, see https://github.com/mesonbuild/meson/issues/13212
+  depends: [_datasets_pair_pxd, _pairwise_distances_reduction_cython_tree, utils_cython_tree],
+)
+_datasets_pair = py.extension_module(
+  '_datasets_pair',
+  cython_gen_cpp.process(_datasets_pair_pyx),
+  dependencies: [np_dep],
+  subdir: 'sklearn/metrics/_pairwise_distances_reduction',
+  install: true
+)
+
+_base_pxd = custom_target(
+  '_base_pxd',
+  output: '_base.pxd',
+  input: '_base.pxd.tp',
+  command: [tempita, '@INPUT@', '-o', '@OUTDIR@']
+)
+_base_pyx = custom_target(
+  '_base_pyx',
+  output: '_base.pyx',
+  input: '_base.pyx.tp',
+  command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
+  # TODO in principle this should go in py.exension_module below. This is
+  # temporary work-around for dependency issue with .pyx.tp files. For more
+  # details, see https://github.com/mesonbuild/meson/issues/13212
+  depends: [_base_pxd, _pairwise_distances_reduction_cython_tree,
+            _datasets_pair_pxd, utils_cython_tree],
+)
+_base = py.extension_module(
+  '_base',
+  cython_gen_cpp.process(_base_pyx),
+  dependencies: [np_dep, openmp_dep],
+  subdir: 'sklearn/metrics/_pairwise_distances_reduction',
+  install: true
+)
+
+_middle_term_computer_pxd = custom_target(
+  '_middle_term_computer_pxd',
+  output: '_middle_term_computer.pxd',
+  input: '_middle_term_computer.pxd.tp',
+  command: [tempita, '@INPUT@', '-o', '@OUTDIR@']
+)
+_middle_term_computer_pyx = custom_target(
+  '_middle_term_computer_pyx',
+  output: '_middle_term_computer.pyx',
+  input: '_middle_term_computer.pyx.tp',
+  command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
+  # TODO in principle this should go in py.exension_module below. This is
+  # temporary work-around for dependency issue with .pyx.tp files. For more
+  # details, see https://github.com/mesonbuild/meson/issues/13212
+  depends: [_middle_term_computer_pxd,
+            _pairwise_distances_reduction_cython_tree,
+            utils_cython_tree],
+)
+_middle_term_computer = py.extension_module(
+  '_middle_term_computer',
+  cython_gen_cpp.process(_middle_term_computer_pyx),
+  dependencies: [np_dep],
+  subdir: 'sklearn/metrics/_pairwise_distances_reduction',
+  install: true
+)
+
+_argkmin_pxd = custom_target(
+    '_argkmin_pxd',
+    output: '_argkmin.pxd',
+    input: '_argkmin.pxd.tp',
+    command: [tempita, '@INPUT@', '-o', '@OUTDIR@']
+  )
+_argkmin_pyx = custom_target(
+    '_argkmin_pyx',
+    output: '_argkmin.pyx',
+    input: '_argkmin.pyx.tp',
+    command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
+    # TODO in principle this should go in py.exension_module below. This is
+    # temporary work-around for dependency issue with .pyx.tp files. For more
+    # details, see https://github.com/mesonbuild/meson/issues/13212
+    depends: [_argkmin_pxd,
+              _pairwise_distances_reduction_cython_tree,
+              _datasets_pair_pxd, _base_pxd, _middle_term_computer_pxd],
+      )
+_argkmin = py.extension_module(
+    '_argkmin',
+    cython_gen_cpp.process(_argkmin_pyx),
+    dependencies: [np_dep, openmp_dep],
+    subdir: 'sklearn/metrics/_pairwise_distances_reduction',
+    install: true
+)
+
+_radius_neighbors_pxd = custom_target(
+    '_radius_neighbors_pxd',
+    output: '_radius_neighbors.pxd',
+    input: '_radius_neighbors.pxd.tp',
+    command: [tempita, '@INPUT@', '-o', '@OUTDIR@']
+  )
+_radius_neighbors_pyx = custom_target(
+    '_radius_neighbors_pyx',
+    output: '_radius_neighbors.pyx',
+    input: '_radius_neighbors.pyx.tp',
+    command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
+    # TODO in principle this should go in py.exension_module below. This is
+    # temporary work-around for dependency issue with .pyx.tp files. For more
+    # details, see https://github.com/mesonbuild/meson/issues/13212
+    depends: [_radius_neighbors_pxd,
+              _datasets_pair_pxd, _base_pxd, _middle_term_computer_pxd,
+              _pairwise_distances_reduction_cython_tree, utils_cython_tree],
+)
+_radius_neighbors = py.extension_module(
+    '_radius_neighbors',
+    cython_gen_cpp.process(_radius_neighbors_pyx),
+    dependencies: [np_dep, openmp_dep],
+    subdir: 'sklearn/metrics/_pairwise_distances_reduction',
+    install: true
+)
+
+_argkmin_classmode_pyx = custom_target(
+  '_argkmin_classmode_pyx',
+  output: '_argkmin_classmode.pyx',
+  input: '_argkmin_classmode.pyx.tp',
+  command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
+  # TODO in principle this should go in py.exension_module below. This is
+  # temporary work-around for dependency issue with .pyx.tp files. For more
+  # details, see https://github.com/mesonbuild/meson/issues/13212
+  depends: [_classmode_pxd,
+            _argkmin_pxd, _pairwise_distances_reduction_cython_tree,
+            _datasets_pair_pxd, _base_pxd, _middle_term_computer_pxd, utils_cython_tree],
+)
+_argkmin_classmode = py.extension_module(
+  '_argkmin_classmode',
+  cython_gen_cpp.process(_argkmin_classmode_pyx),
+  dependencies: [np_dep, openmp_dep],
+  # XXX: for some reason -fno-sized-deallocation is needed otherwise there is
+  # an error with undefined symbol _ZdlPv at import time in manylinux wheels.
+  # See https://github.com/scikit-learn/scikit-learn/issues/28596 for more details.
+  cpp_args: ['-fno-sized-deallocation'],
+  subdir: 'sklearn/metrics/_pairwise_distances_reduction',
+  install: true
+)
+
+_radius_neighbors_classmode_pyx = custom_target(
+  '_radius_neighbors_classmode_pyx',
+  output: '_radius_neighbors_classmode.pyx',
+  input: '_radius_neighbors_classmode.pyx.tp',
+  command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
+  # TODO in principle this should go in py.exension_module below. This is
+  # temporary work-around for dependency issue with .pyx.tp files. For more
+  # details, see https://github.com/mesonbuild/meson/issues/13212
+  depends: [_classmode_pxd,
+            _middle_term_computer_pxd, _radius_neighbors_pxd,
+            _pairwise_distances_reduction_cython_tree,
+            _datasets_pair_pxd, _base_pxd, utils_cython_tree],
+)
+_radius_neighbors_classmode = py.extension_module(
+  '_radius_neighbors_classmode',
+  cython_gen_cpp.process(_radius_neighbors_classmode_pyx),
+  dependencies: [np_dep, openmp_dep],
+  subdir: 'sklearn/metrics/_pairwise_distances_reduction',
+  install: true
+)
diff --git a/sklearn/metrics/_pairwise_fast.pyx b/sklearn/metrics/_pairwise_fast.pyx
index f122972a15f89..bf4ded09b2610 100644
--- a/sklearn/metrics/_pairwise_fast.pyx
+++ b/sklearn/metrics/_pairwise_fast.pyx
@@ -1,31 +1,22 @@
-#cython: boundscheck=False
-#cython: cdivision=True
-#cython: wraparound=False
-#
-# Author: Andreas Mueller 
-#         Lars Buitinck
-#         Paolo Toccaceli
-#
-# License: BSD 3 clause
-
-import numpy as np
-cimport numpy as np
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
 from cython cimport floating
 from cython.parallel cimport prange
 from libc.math cimport fabs
 
-from ..utils._openmp_helpers import _openmp_effective_n_threads
+from ..utils._typedefs cimport intp_t
 
-np.import_array()
+from ..utils._openmp_helpers import _openmp_effective_n_threads
 
 
 def _chi2_kernel_fast(floating[:, :] X,
                       floating[:, :] Y,
                       floating[:, :] result):
-    cdef np.npy_intp i, j, k
-    cdef np.npy_intp n_samples_X = X.shape[0]
-    cdef np.npy_intp n_samples_Y = Y.shape[0]
-    cdef np.npy_intp n_features = X.shape[1]
+    cdef intp_t i, j, k
+    cdef intp_t n_samples_X = X.shape[0]
+    cdef intp_t n_samples_Y = Y.shape[0]
+    cdef intp_t n_features = X.shape[1]
     cdef double res, nom, denom
 
     with nogil:
@@ -36,13 +27,19 @@ def _chi2_kernel_fast(floating[:, :] X,
                     denom = (X[i, k] - Y[j, k])
                     nom = (X[i, k] + Y[j, k])
                     if nom != 0:
-                        res  += denom * denom / nom
+                        res += denom * denom / nom
                 result[i, j] = -res
 
 
-def _sparse_manhattan(floating[::1] X_data, int[:] X_indices, int[:] X_indptr,
-                      floating[::1] Y_data, int[:] Y_indices, int[:] Y_indptr,
-                      double[:, ::1] D):
+def _sparse_manhattan(
+    const floating[::1] X_data,
+    const int[:] X_indices,
+    const int[:] X_indptr,
+    const floating[::1] Y_data,
+    const int[:] Y_indices,
+    const int[:] Y_indptr,
+    double[:, ::1] D,
+):
     """Pairwise L1 distances for CSR matrices.
 
     Usage:
@@ -51,7 +48,7 @@ def _sparse_manhattan(floating[::1] X_data, int[:] X_indices, int[:] X_indptr,
     ...                   Y.data, Y.indices, Y.indptr,
     ...                   D)
     """
-    cdef np.npy_intp px, py, i, j, ix, iy
+    cdef intp_t px, py, i, j, ix, iy
     cdef double d = 0.0
 
     cdef int m = D.shape[0]
diff --git a/sklearn/metrics/_plot/__init__.py b/sklearn/metrics/_plot/__init__.py
index e69de29bb2d1d..67dd18fb94b59 100644
--- a/sklearn/metrics/_plot/__init__.py
+++ b/sklearn/metrics/_plot/__init__.py
@@ -0,0 +1,2 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
diff --git a/sklearn/metrics/_plot/base.py b/sklearn/metrics/_plot/base.py
deleted file mode 100644
index 4ac561f6d3dfa..0000000000000
--- a/sklearn/metrics/_plot/base.py
+++ /dev/null
@@ -1,114 +0,0 @@
-import numpy as np
-
-from ...base import is_classifier
-
-
-def _check_classifier_response_method(estimator, response_method):
-    """Return prediction method from the response_method
-
-    Parameters
-    ----------
-    estimator: object
-        Classifier to check
-
-    response_method: {'auto', 'predict_proba', 'decision_function'}
-        Specifies whether to use :term:`predict_proba` or
-        :term:`decision_function` as the target response. If set to 'auto',
-        :term:`predict_proba` is tried first and if it does not exist
-        :term:`decision_function` is tried next.
-
-    Returns
-    -------
-    prediction_method: callable
-        prediction method of estimator
-    """
-
-    if response_method not in ("predict_proba", "decision_function", "auto"):
-        raise ValueError("response_method must be 'predict_proba', "
-                         "'decision_function' or 'auto'")
-
-    error_msg = "response method {} is not defined in {}"
-    if response_method != "auto":
-        prediction_method = getattr(estimator, response_method, None)
-        if prediction_method is None:
-            raise ValueError(error_msg.format(response_method,
-                                              estimator.__class__.__name__))
-    else:
-        predict_proba = getattr(estimator, 'predict_proba', None)
-        decision_function = getattr(estimator, 'decision_function', None)
-        prediction_method = predict_proba or decision_function
-        if prediction_method is None:
-            raise ValueError(error_msg.format(
-                "decision_function or predict_proba",
-                estimator.__class__.__name__))
-
-    return prediction_method
-
-
-def _get_response(X, estimator, response_method, pos_label=None):
-    """Return response and positive label.
-
-    Parameters
-    ----------
-    X : {array-like, sparse matrix} of shape (n_samples, n_features)
-        Input values.
-
-    estimator : estimator instance
-        Fitted classifier or a fitted :class:`~sklearn.pipeline.Pipeline`
-        in which the last estimator is a classifier.
-
-    response_method: {'auto', 'predict_proba', 'decision_function'}
-        Specifies whether to use :term:`predict_proba` or
-        :term:`decision_function` as the target response. If set to 'auto',
-        :term:`predict_proba` is tried first and if it does not exist
-        :term:`decision_function` is tried next.
-
-    pos_label : str or int, default=None
-        The class considered as the positive class when computing
-        the metrics. By default, `estimators.classes_[1]` is
-        considered as the positive class.
-
-    Returns
-    -------
-    y_pred: ndarray of shape (n_samples,)
-        Target scores calculated from the provided response_method
-        and pos_label.
-
-    pos_label: str or int
-        The class considered as the positive class when computing
-        the metrics.
-    """
-    classification_error = (
-        "{} should be a binary classifier".format(estimator.__class__.__name__)
-    )
-
-    if not is_classifier(estimator):
-        raise ValueError(classification_error)
-
-    prediction_method = _check_classifier_response_method(
-        estimator, response_method)
-
-    y_pred = prediction_method(X)
-
-    if pos_label is not None and pos_label not in estimator.classes_:
-        raise ValueError(
-            f"The class provided by 'pos_label' is unknown. Got "
-            f"{pos_label} instead of one of {estimator.classes_}"
-        )
-
-    if y_pred.ndim != 1:  # `predict_proba`
-        if y_pred.shape[1] != 2:
-            raise ValueError(classification_error)
-        if pos_label is None:
-            pos_label = estimator.classes_[1]
-            y_pred = y_pred[:, 1]
-        else:
-            class_idx = np.flatnonzero(estimator.classes_ == pos_label)
-            y_pred = y_pred[:, class_idx]
-    else:
-        if pos_label is None:
-            pos_label = estimator.classes_[1]
-        elif pos_label == estimator.classes_[0]:
-            y_pred *= -1
-
-    return y_pred, pos_label
diff --git a/sklearn/metrics/_plot/confusion_matrix.py b/sklearn/metrics/_plot/confusion_matrix.py
index 9fcecec775e6e..cee515bebe08e 100644
--- a/sklearn/metrics/_plot/confusion_matrix.py
+++ b/sklearn/metrics/_plot/confusion_matrix.py
@@ -1,25 +1,30 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
 from itertools import product
 
 import numpy as np
 
-from .. import confusion_matrix
-from ...utils import check_matplotlib_support
-from ...utils import deprecated
-from ...utils.multiclass import unique_labels
-from ...utils.validation import _deprecate_positional_args
 from ...base import is_classifier
+from ...utils._optional_dependencies import check_matplotlib_support
+from ...utils._plotting import _validate_style_kwargs
+from ...utils.multiclass import unique_labels
+from .. import confusion_matrix
 
 
 class ConfusionMatrixDisplay:
     """Confusion Matrix visualization.
 
-    It is recommend to use
+    It is recommended to use
     :func:`~sklearn.metrics.ConfusionMatrixDisplay.from_estimator` or
     :func:`~sklearn.metrics.ConfusionMatrixDisplay.from_predictions` to
     create a :class:`ConfusionMatrixDisplay`. All parameters are stored as
     attributes.
 
-    Read more in the :ref:`User Guide `.
+    For general information regarding `scikit-learn` visualization tools, see
+    the :ref:`Visualization Guide `.
+    For guidance on interpreting these plots, refer to the
+    :ref:`Model Evaluation Guide `.
 
     Parameters
     ----------
@@ -56,6 +61,7 @@ class ConfusionMatrixDisplay:
 
     Examples
     --------
+    >>> import matplotlib.pyplot as plt
     >>> from sklearn.datasets import make_classification
     >>> from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
     >>> from sklearn.model_selection import train_test_split
@@ -70,17 +76,27 @@ class ConfusionMatrixDisplay:
     >>> cm = confusion_matrix(y_test, predictions, labels=clf.classes_)
     >>> disp = ConfusionMatrixDisplay(confusion_matrix=cm,
     ...                               display_labels=clf.classes_)
-    >>> disp.plot() # doctest: +SKIP
+    >>> disp.plot()
+    <...>
+    >>> plt.show()
     """
-    @_deprecate_positional_args
+
     def __init__(self, confusion_matrix, *, display_labels=None):
         self.confusion_matrix = confusion_matrix
         self.display_labels = display_labels
 
-    @_deprecate_positional_args
-    def plot(self, *, include_values=True, cmap='viridis',
-             xticks_rotation='horizontal', values_format=None,
-             ax=None, colorbar=True):
+    def plot(
+        self,
+        *,
+        include_values=True,
+        cmap="viridis",
+        xticks_rotation="horizontal",
+        values_format=None,
+        ax=None,
+        colorbar=True,
+        im_kw=None,
+        text_kw=None,
+    ):
         """Plot visualization.
 
         Parameters
@@ -106,9 +122,19 @@ def plot(self, *, include_values=True, cmap='viridis',
         colorbar : bool, default=True
             Whether or not to add a colorbar to the plot.
 
+        im_kw : dict, default=None
+            Dict with keywords passed to `matplotlib.pyplot.imshow` call.
+
+        text_kw : dict, default=None
+            Dict with keywords passed to `matplotlib.pyplot.text` call.
+
+            .. versionadded:: 1.2
+
         Returns
         -------
         display : :class:`~sklearn.metrics.ConfusionMatrixDisplay`
+            Returns a :class:`~sklearn.metrics.ConfusionMatrixDisplay` instance
+            that contains all the information to plot the confusion matrix.
         """
         check_matplotlib_support("ConfusionMatrixDisplay.plot")
         import matplotlib.pyplot as plt
@@ -120,9 +146,15 @@ def plot(self, *, include_values=True, cmap='viridis',
 
         cm = self.confusion_matrix
         n_classes = cm.shape[0]
-        self.im_ = ax.imshow(cm, interpolation='nearest', cmap=cmap)
+
+        default_im_kw = dict(interpolation="nearest", cmap=cmap)
+        im_kw = im_kw or {}
+        im_kw = _validate_style_kwargs(default_im_kw, im_kw)
+        text_kw = text_kw or {}
+
+        self.im_ = ax.imshow(cm, **im_kw)
         self.text_ = None
-        cmap_min, cmap_max = self.im_.cmap(0), self.im_.cmap(256)
+        cmap_min, cmap_max = self.im_.cmap(0), self.im_.cmap(1.0)
 
         if include_values:
             self.text_ = np.empty_like(cm, dtype=object)
@@ -134,18 +166,18 @@ def plot(self, *, include_values=True, cmap='viridis',
                 color = cmap_max if cm[i, j] < thresh else cmap_min
 
                 if values_format is None:
-                    text_cm = format(cm[i, j], '.2g')
-                    if cm.dtype.kind != 'f':
-                        text_d = format(cm[i, j], 'd')
+                    text_cm = format(cm[i, j], ".2g")
+                    if cm.dtype.kind != "f":
+                        text_d = format(cm[i, j], "d")
                         if len(text_d) < len(text_cm):
                             text_cm = text_d
                 else:
                     text_cm = format(cm[i, j], values_format)
 
-                self.text_[i, j] = ax.text(
-                    j, i, text_cm,
-                    ha="center", va="center",
-                    color=color)
+                default_text_kwargs = dict(ha="center", va="center", color=color)
+                text_kwargs = _validate_style_kwargs(default_text_kwargs, text_kw)
+
+                self.text_[i, j] = ax.text(j, i, text_cm, **text_kwargs)
 
         if self.display_labels is None:
             display_labels = np.arange(n_classes)
@@ -153,12 +185,14 @@ def plot(self, *, include_values=True, cmap='viridis',
             display_labels = self.display_labels
         if colorbar:
             fig.colorbar(self.im_, ax=ax)
-        ax.set(xticks=np.arange(n_classes),
-               yticks=np.arange(n_classes),
-               xticklabels=display_labels,
-               yticklabels=display_labels,
-               ylabel="True label",
-               xlabel="Predicted label")
+        ax.set(
+            xticks=np.arange(n_classes),
+            yticks=np.arange(n_classes),
+            xticklabels=display_labels,
+            yticklabels=display_labels,
+            ylabel="True label",
+            xlabel="Predicted label",
+        )
 
         ax.set_ylim((n_classes - 0.5, -0.5))
         plt.setp(ax.get_xticklabels(), rotation=xticks_rotation)
@@ -184,10 +218,15 @@ def from_estimator(
         cmap="viridis",
         ax=None,
         colorbar=True,
+        im_kw=None,
+        text_kw=None,
     ):
         """Plot Confusion Matrix given an estimator and some data.
 
-        Read more in the :ref:`User Guide `.
+        For general information regarding `scikit-learn` visualization tools, see
+        the :ref:`Visualization Guide `.
+        For guidance on interpreting these plots, refer to the
+        :ref:`Model Evaluation Guide `.
 
         .. versionadded:: 1.0
 
@@ -249,6 +288,14 @@ def from_estimator(
         colorbar : bool, default=True
             Whether or not to add a colorbar to the plot.
 
+        im_kw : dict, default=None
+            Dict with keywords passed to `matplotlib.pyplot.imshow` call.
+
+        text_kw : dict, default=None
+            Dict with keywords passed to `matplotlib.pyplot.text` call.
+
+            .. versionadded:: 1.2
+
         Returns
         -------
         display : :class:`~sklearn.metrics.ConfusionMatrixDisplay`
@@ -260,7 +307,7 @@ def from_estimator(
 
         Examples
         --------
-        >>> import matplotlib.pyplot as plt  # doctest: +SKIP
+        >>> import matplotlib.pyplot as plt
         >>> from sklearn.datasets import make_classification
         >>> from sklearn.metrics import ConfusionMatrixDisplay
         >>> from sklearn.model_selection import train_test_split
@@ -272,8 +319,13 @@ def from_estimator(
         >>> clf.fit(X_train, y_train)
         SVC(random_state=0)
         >>> ConfusionMatrixDisplay.from_estimator(
-        ...     clf, X_test, y_test)  # doctest: +SKIP
-        >>> plt.show()  # doctest: +SKIP
+        ...     clf, X_test, y_test)
+        <...>
+        >>> plt.show()
+
+        For a detailed example of using a confusion matrix to evaluate a
+        Support Vector Classifier, please see
+        :ref:`sphx_glr_auto_examples_model_selection_plot_confusion_matrix.py`
         """
         method_name = f"{cls.__name__}.from_estimator"
         check_matplotlib_support(method_name)
@@ -294,6 +346,8 @@ def from_estimator(
             xticks_rotation=xticks_rotation,
             values_format=values_format,
             colorbar=colorbar,
+            im_kw=im_kw,
+            text_kw=text_kw,
         )
 
     @classmethod
@@ -312,12 +366,17 @@ def from_predictions(
         cmap="viridis",
         ax=None,
         colorbar=True,
+        im_kw=None,
+        text_kw=None,
     ):
         """Plot Confusion Matrix given true and predicted labels.
 
-        Read more in the :ref:`User Guide `.
+        For general information regarding `scikit-learn` visualization tools, see
+        the :ref:`Visualization Guide `.
+        For guidance on interpreting these plots, refer to the
+        :ref:`Model Evaluation Guide `.
 
-        .. versionadded:: 0.24
+        .. versionadded:: 1.0
 
         Parameters
         ----------
@@ -374,6 +433,14 @@ def from_predictions(
         colorbar : bool, default=True
             Whether or not to add a colorbar to the plot.
 
+        im_kw : dict, default=None
+            Dict with keywords passed to `matplotlib.pyplot.imshow` call.
+
+        text_kw : dict, default=None
+            Dict with keywords passed to `matplotlib.pyplot.text` call.
+
+            .. versionadded:: 1.2
+
         Returns
         -------
         display : :class:`~sklearn.metrics.ConfusionMatrixDisplay`
@@ -385,7 +452,7 @@ def from_predictions(
 
         Examples
         --------
-        >>> import matplotlib.pyplot as plt  # doctest: +SKIP
+        >>> import matplotlib.pyplot as plt
         >>> from sklearn.datasets import make_classification
         >>> from sklearn.metrics import ConfusionMatrixDisplay
         >>> from sklearn.model_selection import train_test_split
@@ -398,8 +465,9 @@ def from_predictions(
         SVC(random_state=0)
         >>> y_pred = clf.predict(X_test)
         >>> ConfusionMatrixDisplay.from_predictions(
-        ...    y_test, y_pred)  # doctest: +SKIP
-        >>> plt.show()  # doctest: +SKIP
+        ...    y_test, y_pred)
+        <...>
+        >>> plt.show()
         """
         check_matplotlib_support(f"{cls.__name__}.from_predictions")
 
@@ -426,134 +494,6 @@ def from_predictions(
             xticks_rotation=xticks_rotation,
             values_format=values_format,
             colorbar=colorbar,
+            im_kw=im_kw,
+            text_kw=text_kw,
         )
-
-
-@deprecated(
-    "Function plot_confusion_matrix is deprecated in 1.0 and will be "
-    "removed in 1.2. Use one of the class methods: "
-    "ConfusionMatrixDisplay.from_predictions or "
-    "ConfusionMatrixDisplay.from_estimator."
-)
-@_deprecate_positional_args
-def plot_confusion_matrix(estimator, X, y_true, *, labels=None,
-                          sample_weight=None, normalize=None,
-                          display_labels=None, include_values=True,
-                          xticks_rotation='horizontal',
-                          values_format=None,
-                          cmap='viridis', ax=None, colorbar=True):
-    """Plot Confusion Matrix.
-
-    Read more in the :ref:`User Guide `.
-
-    .. deprecated:: 1.0
-       `plot_confusion_matrix` is deprecated in 1.0 and will be removed in
-       1.2. Use one of the following class methods:
-       :func:`~sklearn.metrics.ConfusionMatrixDisplay.from_predictions` or
-       :func:`~sklearn.metrics.ConfusionMatrixDisplay.from_estimator`.
-
-    Parameters
-    ----------
-    estimator : estimator instance
-        Fitted classifier or a fitted :class:`~sklearn.pipeline.Pipeline`
-        in which the last estimator is a classifier.
-
-    X : {array-like, sparse matrix} of shape (n_samples, n_features)
-        Input values.
-
-    y_true : array-like of shape (n_samples,)
-        Target values.
-
-    labels : array-like of shape (n_classes,), default=None
-        List of labels to index the matrix. This may be used to reorder or
-        select a subset of labels. If `None` is given, those that appear at
-        least once in `y_true` or `y_pred` are used in sorted order.
-
-    sample_weight : array-like of shape (n_samples,), default=None
-        Sample weights.
-
-    normalize : {'true', 'pred', 'all'}, default=None
-        Either to normalize the counts display in the matrix:
-
-            - if `'true'`, the confusion matrix is normalized over the true
-              conditions (e.g. rows);
-            - if `'pred'`, the confusion matrix is normalized over the
-              predicted conditions (e.g. columns);
-            - if `'all'`, the confusion matrix is normalized by the total
-              number of samples;
-            - if `None` (default), the confusion matrix will not be normalized.
-
-    display_labels : array-like of shape (n_classes,), default=None
-        Target names used for plotting. By default, `labels` will be used if
-        it is defined, otherwise the unique labels of `y_true` and `y_pred`
-        will be used.
-
-    include_values : bool, default=True
-        Includes values in confusion matrix.
-
-    xticks_rotation : {'vertical', 'horizontal'} or float, \
-                        default='horizontal'
-        Rotation of xtick labels.
-
-    values_format : str, default=None
-        Format specification for values in confusion matrix. If `None`,
-        the format specification is 'd' or '.2g' whichever is shorter.
-
-    cmap : str or matplotlib Colormap, default='viridis'
-        Colormap recognized by matplotlib.
-
-    ax : matplotlib Axes, default=None
-        Axes object to plot on. If `None`, a new figure and axes is
-        created.
-
-    colorbar : bool, default=True
-        Whether or not to add a colorbar to the plot.
-
-        .. versionadded:: 0.24
-
-    Returns
-    -------
-    display : :class:`~sklearn.metrics.ConfusionMatrixDisplay`
-
-    See Also
-    --------
-    confusion_matrix : Compute Confusion Matrix to evaluate the accuracy of a
-        classification.
-    ConfusionMatrixDisplay : Confusion Matrix visualization.
-
-    Examples
-    --------
-    >>> import matplotlib.pyplot as plt  # doctest: +SKIP
-    >>> from sklearn.datasets import make_classification
-    >>> from sklearn.metrics import plot_confusion_matrix
-    >>> from sklearn.model_selection import train_test_split
-    >>> from sklearn.svm import SVC
-    >>> X, y = make_classification(random_state=0)
-    >>> X_train, X_test, y_train, y_test = train_test_split(
-    ...         X, y, random_state=0)
-    >>> clf = SVC(random_state=0)
-    >>> clf.fit(X_train, y_train)
-    SVC(random_state=0)
-    >>> plot_confusion_matrix(clf, X_test, y_test)  # doctest: +SKIP
-    >>> plt.show()  # doctest: +SKIP
-    """
-    check_matplotlib_support("plot_confusion_matrix")
-
-    if not is_classifier(estimator):
-        raise ValueError("plot_confusion_matrix only supports classifiers")
-
-    y_pred = estimator.predict(X)
-    cm = confusion_matrix(y_true, y_pred, sample_weight=sample_weight,
-                          labels=labels, normalize=normalize)
-
-    if display_labels is None:
-        if labels is None:
-            display_labels = unique_labels(y_true, y_pred)
-        else:
-            display_labels = labels
-
-    disp = ConfusionMatrixDisplay(confusion_matrix=cm,
-                                  display_labels=display_labels)
-    return disp.plot(include_values=include_values,
-                     cmap=cmap, ax=ax, xticks_rotation=xticks_rotation,
-                     values_format=values_format, colorbar=colorbar)
diff --git a/sklearn/metrics/_plot/det_curve.py b/sklearn/metrics/_plot/det_curve.py
index e512b2d972ce6..590b908d91723 100644
--- a/sklearn/metrics/_plot/det_curve.py
+++ b/sklearn/metrics/_plot/det_curve.py
@@ -1,19 +1,24 @@
-import scipy as sp
-
-from .base import _get_response
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-from .. import det_curve
+import numpy as np
+import scipy as sp
 
-from ...utils import check_matplotlib_support
+from ...utils._plotting import _BinaryClassifierCurveDisplayMixin
+from .._ranking import det_curve
 
 
-class DetCurveDisplay:
-    """DET curve visualization.
+class DetCurveDisplay(_BinaryClassifierCurveDisplayMixin):
+    """Detection Error Tradeoff (DET) curve visualization.
 
-    It is recommend to use :func:`~sklearn.metrics.plot_det_curve` to create a
+    It is recommended to use :func:`~sklearn.metrics.DetCurveDisplay.from_estimator`
+    or :func:`~sklearn.metrics.DetCurveDisplay.from_predictions` to create a
     visualizer. All parameters are stored as attributes.
 
-    Read more in the :ref:`User Guide `.
+    For general information regarding `scikit-learn` visualization tools, see
+    the :ref:`Visualization Guide `.
+    For guidance on interpreting these plots, refer to the
+    :ref:`Model Evaluation Guide `.
 
     .. versionadded:: 0.24
 
@@ -28,7 +33,7 @@ class DetCurveDisplay:
     estimator_name : str, default=None
         Name of estimator. If None, the estimator name is not shown.
 
-    pos_label : str or int, default=None
+    pos_label : int, float, bool or str, default=None
         The label of the positive class.
 
     Attributes
@@ -45,185 +50,322 @@ class DetCurveDisplay:
     See Also
     --------
     det_curve : Compute error rates for different probability thresholds.
-    plot_det_curve : Plot detection error tradeoff (DET) curve.
+    DetCurveDisplay.from_estimator : Plot DET curve given an estimator and
+        some data.
+    DetCurveDisplay.from_predictions : Plot DET curve given the true and
+        predicted labels.
 
     Examples
     --------
-    >>> import matplotlib.pyplot as plt  # doctest: +SKIP
-    >>> import numpy as np
-    >>> from sklearn import metrics
-    >>> y = np.array([0, 0, 1, 1])
-    >>> pred = np.array([0.1, 0.4, 0.35, 0.8])
-    >>> fpr, fnr, thresholds = metrics.det_curve(y, pred)
-    >>> display = metrics.DetCurveDisplay(
-    ...     fpr=fpr, fnr=fnr, estimator_name='example estimator'
+    >>> import matplotlib.pyplot as plt
+    >>> from sklearn.datasets import make_classification
+    >>> from sklearn.metrics import det_curve, DetCurveDisplay
+    >>> from sklearn.model_selection import train_test_split
+    >>> from sklearn.svm import SVC
+    >>> X, y = make_classification(n_samples=1000, random_state=0)
+    >>> X_train, X_test, y_train, y_test = train_test_split(
+    ...     X, y, test_size=0.4, random_state=0)
+    >>> clf = SVC(random_state=0).fit(X_train, y_train)
+    >>> y_pred = clf.decision_function(X_test)
+    >>> fpr, fnr, _ = det_curve(y_test, y_pred)
+    >>> display = DetCurveDisplay(
+    ...     fpr=fpr, fnr=fnr, estimator_name="SVC"
     ... )
-    >>> display.plot()  # doctest: +SKIP
-    >>> plt.show()      # doctest: +SKIP
+    >>> display.plot()
+    <...>
+    >>> plt.show()
     """
+
     def __init__(self, *, fpr, fnr, estimator_name=None, pos_label=None):
         self.fpr = fpr
         self.fnr = fnr
         self.estimator_name = estimator_name
         self.pos_label = pos_label
 
-    def plot(self, ax=None, *, name=None, **kwargs):
-        """Plot visualization.
+    @classmethod
+    def from_estimator(
+        cls,
+        estimator,
+        X,
+        y,
+        *,
+        sample_weight=None,
+        drop_intermediate=True,
+        response_method="auto",
+        pos_label=None,
+        name=None,
+        ax=None,
+        **kwargs,
+    ):
+        """Plot DET curve given an estimator and data.
+
+        For general information regarding `scikit-learn` visualization tools, see
+        the :ref:`Visualization Guide `.
+        For guidance on interpreting these plots, refer to the
+        :ref:`Model Evaluation Guide `.
+
+        .. versionadded:: 1.0
 
         Parameters
         ----------
-        ax : matplotlib axes, default=None
-            Axes object to plot on. If `None`, a new figure and axes is
-            created.
+        estimator : estimator instance
+            Fitted classifier or a fitted :class:`~sklearn.pipeline.Pipeline`
+            in which the last estimator is a classifier.
+
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Input values.
+
+        y : array-like of shape (n_samples,)
+            Target values.
+
+        sample_weight : array-like of shape (n_samples,), default=None
+            Sample weights.
+
+        drop_intermediate : bool, default=True
+            Whether to drop thresholds where true positives (tp) do not change
+            from the previous or subsequent threshold. All points with the same
+            tp value have the same `fnr` and thus same y coordinate.
+
+            .. versionadded:: 1.7
+
+        response_method : {'predict_proba', 'decision_function', 'auto'} \
+                default='auto'
+            Specifies whether to use :term:`predict_proba` or
+            :term:`decision_function` as the predicted target response. If set
+            to 'auto', :term:`predict_proba` is tried first and if it does not
+            exist :term:`decision_function` is tried next.
+
+        pos_label : int, float, bool or str, default=None
+            The label of the positive class. When `pos_label=None`, if `y_true`
+            is in {-1, 1} or {0, 1}, `pos_label` is set to 1, otherwise an
+            error will be raised.
 
         name : str, default=None
             Name of DET curve for labeling. If `None`, use the name of the
             estimator.
 
+        ax : matplotlib axes, default=None
+            Axes object to plot on. If `None`, a new figure and axes is
+            created.
+
+        **kwargs : dict
+            Additional keywords arguments passed to matplotlib `plot` function.
+
         Returns
         -------
-        display : :class:`~sklearn.metrics.plot.DetCurveDisplay`
+        display : :class:`~sklearn.metrics.DetCurveDisplay`
             Object that stores computed values.
+
+        See Also
+        --------
+        det_curve : Compute error rates for different probability thresholds.
+        DetCurveDisplay.from_predictions : Plot DET curve given the true and
+            predicted labels.
+
+        Examples
+        --------
+        >>> import matplotlib.pyplot as plt
+        >>> from sklearn.datasets import make_classification
+        >>> from sklearn.metrics import DetCurveDisplay
+        >>> from sklearn.model_selection import train_test_split
+        >>> from sklearn.svm import SVC
+        >>> X, y = make_classification(n_samples=1000, random_state=0)
+        >>> X_train, X_test, y_train, y_test = train_test_split(
+        ...     X, y, test_size=0.4, random_state=0)
+        >>> clf = SVC(random_state=0).fit(X_train, y_train)
+        >>> DetCurveDisplay.from_estimator(
+        ...    clf, X_test, y_test)
+        <...>
+        >>> plt.show()
         """
-        check_matplotlib_support('DetCurveDisplay.plot')
+        y_pred, pos_label, name = cls._validate_and_get_response_values(
+            estimator,
+            X,
+            y,
+            response_method=response_method,
+            pos_label=pos_label,
+            name=name,
+        )
 
-        name = self.estimator_name if name is None else name
-        line_kwargs = {} if name is None else {"label": name}
-        line_kwargs.update(**kwargs)
+        return cls.from_predictions(
+            y_true=y,
+            y_pred=y_pred,
+            sample_weight=sample_weight,
+            drop_intermediate=drop_intermediate,
+            name=name,
+            ax=ax,
+            pos_label=pos_label,
+            **kwargs,
+        )
 
-        import matplotlib.pyplot as plt
+    @classmethod
+    def from_predictions(
+        cls,
+        y_true,
+        y_pred,
+        *,
+        sample_weight=None,
+        drop_intermediate=True,
+        pos_label=None,
+        name=None,
+        ax=None,
+        **kwargs,
+    ):
+        """Plot the DET curve given the true and predicted labels.
+
+        For general information regarding `scikit-learn` visualization tools, see
+        the :ref:`Visualization Guide `.
+        For guidance on interpreting these plots, refer to the
+        :ref:`Model Evaluation Guide `.
+
+        .. versionadded:: 1.0
 
-        if ax is None:
-            _, ax = plt.subplots()
+        Parameters
+        ----------
+        y_true : array-like of shape (n_samples,)
+            True labels.
 
-        self.line_, = ax.plot(
-            sp.stats.norm.ppf(self.fpr),
-            sp.stats.norm.ppf(self.fnr),
-            **line_kwargs,
-        )
-        info_pos_label = (f" (Positive label: {self.pos_label})"
-                          if self.pos_label is not None else "")
+        y_pred : array-like of shape (n_samples,)
+            Target scores, can either be probability estimates of the positive
+            class, confidence values, or non-thresholded measure of decisions
+            (as returned by `decision_function` on some classifiers).
 
-        xlabel = "False Positive Rate" + info_pos_label
-        ylabel = "False Negative Rate" + info_pos_label
-        ax.set(xlabel=xlabel, ylabel=ylabel)
+        sample_weight : array-like of shape (n_samples,), default=None
+            Sample weights.
 
-        if "label" in line_kwargs:
-            ax.legend(loc="lower right")
+        drop_intermediate : bool, default=True
+            Whether to drop thresholds where true positives (tp) do not change
+            from the previous or subsequent threshold. All points with the same
+            tp value have the same `fnr` and thus same y coordinate.
 
-        ticks = [0.001, 0.01, 0.05, 0.20, 0.5, 0.80, 0.95, 0.99, 0.999]
-        tick_locations = sp.stats.norm.ppf(ticks)
-        tick_labels = [
-            '{:.0%}'.format(s) if (100*s).is_integer() else '{:.1%}'.format(s)
-            for s in ticks
-        ]
-        ax.set_xticks(tick_locations)
-        ax.set_xticklabels(tick_labels)
-        ax.set_xlim(-3, 3)
-        ax.set_yticks(tick_locations)
-        ax.set_yticklabels(tick_labels)
-        ax.set_ylim(-3, 3)
-
-        self.ax_ = ax
-        self.figure_ = ax.figure
-        return self
+            .. versionadded:: 1.7
 
+        pos_label : int, float, bool or str, default=None
+            The label of the positive class. When `pos_label=None`, if `y_true`
+            is in {-1, 1} or {0, 1}, `pos_label` is set to 1, otherwise an
+            error will be raised.
 
-def plot_det_curve(
-    estimator,
-    X,
-    y,
-    *,
-    sample_weight=None,
-    response_method="auto",
-    name=None,
-    ax=None,
-    pos_label=None,
-    **kwargs
-):
-    """Plot detection error tradeoff (DET) curve.
+        name : str, default=None
+            Name of DET curve for labeling. If `None`, name will be set to
+            `"Classifier"`.
 
-    Extra keyword arguments will be passed to matplotlib's `plot`.
+        ax : matplotlib axes, default=None
+            Axes object to plot on. If `None`, a new figure and axes is
+            created.
 
-    Read more in the :ref:`User Guide `.
+        **kwargs : dict
+            Additional keywords arguments passed to matplotlib `plot` function.
 
-    .. versionadded:: 0.24
+        Returns
+        -------
+        display : :class:`~sklearn.metrics.DetCurveDisplay`
+            Object that stores computed values.
 
-    Parameters
-    ----------
-    estimator : estimator instance
-        Fitted classifier or a fitted :class:`~sklearn.pipeline.Pipeline`
-        in which the last estimator is a classifier.
+        See Also
+        --------
+        det_curve : Compute error rates for different probability thresholds.
+        DetCurveDisplay.from_estimator : Plot DET curve given an estimator and
+            some data.
+
+        Examples
+        --------
+        >>> import matplotlib.pyplot as plt
+        >>> from sklearn.datasets import make_classification
+        >>> from sklearn.metrics import DetCurveDisplay
+        >>> from sklearn.model_selection import train_test_split
+        >>> from sklearn.svm import SVC
+        >>> X, y = make_classification(n_samples=1000, random_state=0)
+        >>> X_train, X_test, y_train, y_test = train_test_split(
+        ...     X, y, test_size=0.4, random_state=0)
+        >>> clf = SVC(random_state=0).fit(X_train, y_train)
+        >>> y_pred = clf.decision_function(X_test)
+        >>> DetCurveDisplay.from_predictions(
+        ...    y_test, y_pred)
+        <...>
+        >>> plt.show()
+        """
+        pos_label_validated, name = cls._validate_from_predictions_params(
+            y_true, y_pred, sample_weight=sample_weight, pos_label=pos_label, name=name
+        )
 
-    X : {array-like, sparse matrix} of shape (n_samples, n_features)
-        Input values.
+        fpr, fnr, _ = det_curve(
+            y_true,
+            y_pred,
+            pos_label=pos_label,
+            sample_weight=sample_weight,
+            drop_intermediate=drop_intermediate,
+        )
 
-    y : array-like of shape (n_samples,)
-        Target values.
+        viz = cls(
+            fpr=fpr,
+            fnr=fnr,
+            estimator_name=name,
+            pos_label=pos_label_validated,
+        )
 
-    sample_weight : array-like of shape (n_samples,), default=None
-        Sample weights.
+        return viz.plot(ax=ax, name=name, **kwargs)
 
-    response_method : {'predict_proba', 'decision_function', 'auto'} \
-            default='auto'
-        Specifies whether to use :term:`predict_proba` or
-        :term:`decision_function` as the predicted target response. If set to
-        'auto', :term:`predict_proba` is tried first and if it does not exist
-        :term:`decision_function` is tried next.
+    def plot(self, ax=None, *, name=None, **kwargs):
+        """Plot visualization.
 
-    name : str, default=None
-        Name of DET curve for labeling. If `None`, use the name of the
-        estimator.
+        Parameters
+        ----------
+        ax : matplotlib axes, default=None
+            Axes object to plot on. If `None`, a new figure and axes is
+            created.
 
-    ax : matplotlib axes, default=None
-        Axes object to plot on. If `None`, a new figure and axes is created.
+        name : str, default=None
+            Name of DET curve for labeling. If `None`, use `estimator_name` if
+            it is not `None`, otherwise no labeling is shown.
 
-    pos_label : str or int, default=None
-        The label of the positive class.
-        When `pos_label=None`, if `y_true` is in {-1, 1} or {0, 1},
-        `pos_label` is set to 1, otherwise an error will be raised.
+        **kwargs : dict
+            Additional keywords arguments passed to matplotlib `plot` function.
 
-    Returns
-    -------
-    display : :class:`~sklearn.metrics.DetCurveDisplay`
-        Object that stores computed values.
+        Returns
+        -------
+        display : :class:`~sklearn.metrics.DetCurveDisplay`
+            Object that stores computed values.
+        """
+        self.ax_, self.figure_, name = self._validate_plot_params(ax=ax, name=name)
 
-    See Also
-    --------
-    det_curve : Compute error rates for different probability thresholds.
-    DetCurveDisplay : DET curve visualization.
-    plot_roc_curve : Plot Receiver operating characteristic (ROC) curve.
+        line_kwargs = {} if name is None else {"label": name}
+        line_kwargs.update(**kwargs)
 
-    Examples
-    --------
-    >>> import matplotlib.pyplot as plt  # doctest: +SKIP
-    >>> from sklearn import datasets, metrics, model_selection, svm
-    >>> X, y = datasets.make_classification(random_state=0)
-    >>> X_train, X_test, y_train, y_test = model_selection.train_test_split(
-    ...     X, y, random_state=0)
-    >>> clf = svm.SVC(random_state=0)
-    >>> clf.fit(X_train, y_train)
-    SVC(random_state=0)
-    >>> metrics.plot_det_curve(clf, X_test, y_test)  # doctest: +SKIP
-    >>> plt.show()                                   # doctest: +SKIP
-    """
-    check_matplotlib_support('plot_det_curve')
+        # We have the following bounds:
+        # sp.stats.norm.ppf(0.0) = -np.inf
+        # sp.stats.norm.ppf(1.0) = np.inf
+        # We therefore clip to eps and 1 - eps to not provide infinity to matplotlib.
+        eps = np.finfo(self.fpr.dtype).eps
+        self.fpr = self.fpr.clip(eps, 1 - eps)
+        self.fnr = self.fnr.clip(eps, 1 - eps)
 
-    y_pred, pos_label = _get_response(
-        X, estimator, response_method, pos_label=pos_label
-    )
+        (self.line_,) = self.ax_.plot(
+            sp.stats.norm.ppf(self.fpr),
+            sp.stats.norm.ppf(self.fnr),
+            **line_kwargs,
+        )
+        info_pos_label = (
+            f" (Positive label: {self.pos_label})" if self.pos_label is not None else ""
+        )
 
-    fpr, fnr, _ = det_curve(
-        y, y_pred, pos_label=pos_label, sample_weight=sample_weight,
-    )
+        xlabel = "False Positive Rate" + info_pos_label
+        ylabel = "False Negative Rate" + info_pos_label
+        self.ax_.set(xlabel=xlabel, ylabel=ylabel)
 
-    name = estimator.__class__.__name__ if name is None else name
+        if "label" in line_kwargs:
+            self.ax_.legend(loc="lower right")
 
-    viz = DetCurveDisplay(
-        fpr=fpr,
-        fnr=fnr,
-        estimator_name=name,
-        pos_label=pos_label
-    )
+        ticks = [0.001, 0.01, 0.05, 0.20, 0.5, 0.80, 0.95, 0.99, 0.999]
+        tick_locations = sp.stats.norm.ppf(ticks)
+        tick_labels = [
+            "{:.0%}".format(s) if (100 * s).is_integer() else "{:.1%}".format(s)
+            for s in ticks
+        ]
+        self.ax_.set_xticks(tick_locations)
+        self.ax_.set_xticklabels(tick_labels)
+        self.ax_.set_xlim(-3, 3)
+        self.ax_.set_yticks(tick_locations)
+        self.ax_.set_yticklabels(tick_labels)
+        self.ax_.set_ylim(-3, 3)
 
-    return viz.plot(ax=ax, name=name, **kwargs)
+        return self
diff --git a/sklearn/metrics/_plot/precision_recall_curve.py b/sklearn/metrics/_plot/precision_recall_curve.py
index dcc20bbce25a7..30dd1fba08761 100644
--- a/sklearn/metrics/_plot/precision_recall_curve.py
+++ b/sklearn/metrics/_plot/precision_recall_curve.py
@@ -1,22 +1,32 @@
-from .base import _get_response
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-from .. import average_precision_score
-from .. import precision_recall_curve
+from collections import Counter
 
-from ...utils import check_matplotlib_support
-from ...utils.validation import _deprecate_positional_args
+from ...utils._plotting import (
+    _BinaryClassifierCurveDisplayMixin,
+    _despine,
+    _validate_style_kwargs,
+)
+from .._ranking import average_precision_score, precision_recall_curve
 
 
-class PrecisionRecallDisplay:
+class PrecisionRecallDisplay(_BinaryClassifierCurveDisplayMixin):
     """Precision Recall visualization.
 
-    It is recommend to use :func:`~sklearn.metrics.plot_precision_recall_curve`
-    to create a visualizer. All parameters are stored as attributes.
+    It is recommended to use
+    :func:`~sklearn.metrics.PrecisionRecallDisplay.from_estimator` or
+    :func:`~sklearn.metrics.PrecisionRecallDisplay.from_predictions` to create
+    a :class:`~sklearn.metrics.PrecisionRecallDisplay`. All parameters are
+    stored as attributes.
 
-    Read more in the :ref:`User Guide `.
+    For general information regarding `scikit-learn` visualization tools, see
+    the :ref:`Visualization Guide `.
+    For guidance on interpreting these plots, refer to the :ref:`Model
+    Evaluation Guide `.
 
     Parameters
-    -----------
+    ----------
     precision : ndarray
         Precision values.
 
@@ -29,17 +39,29 @@ class PrecisionRecallDisplay:
     estimator_name : str, default=None
         Name of estimator. If None, then the estimator name is not shown.
 
-    pos_label : str or int, default=None
+    pos_label : int, float, bool or str, default=None
         The class considered as the positive class. If None, the class will not
         be shown in the legend.
 
         .. versionadded:: 0.24
 
+    prevalence_pos_label : float, default=None
+        The prevalence of the positive label. It is used for plotting the
+        chance level line. If None, the chance level line will not be plotted
+        even if `plot_chance_level` is set to True when plotting.
+
+        .. versionadded:: 1.3
+
     Attributes
     ----------
     line_ : matplotlib Artist
         Precision recall curve.
 
+    chance_level_ : matplotlib Artist or None
+        The chance level line. It is `None` if the chance level is not plotted.
+
+        .. versionadded:: 1.3
+
     ax_ : matplotlib Axes
         Axes with precision recall curve.
 
@@ -50,11 +72,26 @@ class PrecisionRecallDisplay:
     --------
     precision_recall_curve : Compute precision-recall pairs for different
         probability thresholds.
-    plot_precision_recall_curve : Plot Precision Recall Curve for binary
-        classifiers.
+    PrecisionRecallDisplay.from_estimator : Plot Precision Recall Curve given
+        a binary classifier.
+    PrecisionRecallDisplay.from_predictions : Plot Precision Recall Curve
+        using predictions from a binary classifier.
+
+    Notes
+    -----
+    The average precision (cf. :func:`~sklearn.metrics.average_precision_score`) in
+    scikit-learn is computed without any interpolation. To be consistent with
+    this metric, the precision-recall curve is plotted without any
+    interpolation as well (step-wise style).
+
+    You can change this style by passing the keyword argument
+    `drawstyle="default"` in :meth:`plot`, :meth:`from_estimator`, or
+    :meth:`from_predictions`. However, the curve will not be strictly
+    consistent with the reported average precision.
 
     Examples
     --------
+    >>> import matplotlib.pyplot as plt
     >>> from sklearn.datasets import make_classification
     >>> from sklearn.metrics import (precision_recall_curve,
     ...                              PrecisionRecallDisplay)
@@ -69,19 +106,38 @@ class PrecisionRecallDisplay:
     >>> predictions = clf.predict(X_test)
     >>> precision, recall, _ = precision_recall_curve(y_test, predictions)
     >>> disp = PrecisionRecallDisplay(precision=precision, recall=recall)
-    >>> disp.plot() # doctest: +SKIP
+    >>> disp.plot()
+    <...>
+    >>> plt.show()
     """
-    @_deprecate_positional_args
-    def __init__(self, precision, recall, *,
-                 average_precision=None, estimator_name=None, pos_label=None):
+
+    def __init__(
+        self,
+        precision,
+        recall,
+        *,
+        average_precision=None,
+        estimator_name=None,
+        pos_label=None,
+        prevalence_pos_label=None,
+    ):
         self.estimator_name = estimator_name
         self.precision = precision
         self.recall = recall
         self.average_precision = average_precision
         self.pos_label = pos_label
-
-    @_deprecate_positional_args
-    def plot(self, ax=None, *, name=None, **kwargs):
+        self.prevalence_pos_label = prevalence_pos_label
+
+    def plot(
+        self,
+        ax=None,
+        *,
+        name=None,
+        plot_chance_level=False,
+        chance_level_kw=None,
+        despine=False,
+        **kwargs,
+    ):
         """Plot visualization.
 
         Extra keyword arguments will be passed to matplotlib's `plot`.
@@ -93,8 +149,26 @@ def plot(self, ax=None, *, name=None, **kwargs):
             created.
 
         name : str, default=None
-            Name of precision recall curve for labeling. If `None`, use the
-            name of the estimator.
+            Name of precision recall curve for labeling. If `None`, use
+            `estimator_name` if not `None`, otherwise no labeling is shown.
+
+        plot_chance_level : bool, default=False
+            Whether to plot the chance level. The chance level is the prevalence
+            of the positive label computed from the data passed during
+            :meth:`from_estimator` or :meth:`from_predictions` call.
+
+            .. versionadded:: 1.3
+
+        chance_level_kw : dict, default=None
+            Keyword arguments to be passed to matplotlib's `plot` for rendering
+            the chance level line.
+
+            .. versionadded:: 1.3
+
+        despine : bool, default=False
+            Whether to remove the top and right spines from the plot.
+
+            .. versionadded:: 1.6
 
         **kwargs : dict
             Keyword arguments to be passed to matplotlib's `plot`.
@@ -103,123 +177,379 @@ def plot(self, ax=None, *, name=None, **kwargs):
         -------
         display : :class:`~sklearn.metrics.PrecisionRecallDisplay`
             Object that stores computed values.
-        """
-        check_matplotlib_support("PrecisionRecallDisplay.plot")
 
-        name = self.estimator_name if name is None else name
+        Notes
+        -----
+        The average precision (cf. :func:`~sklearn.metrics.average_precision_score`)
+        in scikit-learn is computed without any interpolation. To be consistent
+        with this metric, the precision-recall curve is plotted without any
+        interpolation as well (step-wise style).
+
+        You can change this style by passing the keyword argument
+        `drawstyle="default"`. However, the curve will not be strictly
+        consistent with the reported average precision.
+        """
+        self.ax_, self.figure_, name = self._validate_plot_params(ax=ax, name=name)
 
-        line_kwargs = {"drawstyle": "steps-post"}
+        default_line_kwargs = {"drawstyle": "steps-post"}
         if self.average_precision is not None and name is not None:
-            line_kwargs["label"] = (f"{name} (AP = "
-                                    f"{self.average_precision:0.2f})")
+            default_line_kwargs["label"] = (
+                f"{name} (AP = {self.average_precision:0.2f})"
+            )
         elif self.average_precision is not None:
-            line_kwargs["label"] = (f"AP = "
-                                    f"{self.average_precision:0.2f}")
+            default_line_kwargs["label"] = f"AP = {self.average_precision:0.2f}"
         elif name is not None:
-            line_kwargs["label"] = name
-        line_kwargs.update(**kwargs)
+            default_line_kwargs["label"] = name
 
-        import matplotlib.pyplot as plt
+        line_kwargs = _validate_style_kwargs(default_line_kwargs, kwargs)
 
-        if ax is None:
-            fig, ax = plt.subplots()
+        (self.line_,) = self.ax_.plot(self.recall, self.precision, **line_kwargs)
 
-        self.line_, = ax.plot(self.recall, self.precision, **line_kwargs)
-        info_pos_label = (f" (Positive label: {self.pos_label})"
-                          if self.pos_label is not None else "")
+        info_pos_label = (
+            f" (Positive label: {self.pos_label})" if self.pos_label is not None else ""
+        )
 
         xlabel = "Recall" + info_pos_label
         ylabel = "Precision" + info_pos_label
-        ax.set(xlabel=xlabel, ylabel=ylabel)
-
-        if "label" in line_kwargs:
-            ax.legend(loc="lower left")
+        self.ax_.set(
+            xlabel=xlabel,
+            xlim=(-0.01, 1.01),
+            ylabel=ylabel,
+            ylim=(-0.01, 1.01),
+            aspect="equal",
+        )
+
+        if plot_chance_level:
+            if self.prevalence_pos_label is None:
+                raise ValueError(
+                    "You must provide prevalence_pos_label when constructing the "
+                    "PrecisionRecallDisplay object in order to plot the chance "
+                    "level line. Alternatively, you may use "
+                    "PrecisionRecallDisplay.from_estimator or "
+                    "PrecisionRecallDisplay.from_predictions "
+                    "to automatically set prevalence_pos_label"
+                )
+
+            default_chance_level_line_kw = {
+                "label": f"Chance level (AP = {self.prevalence_pos_label:0.2f})",
+                "color": "k",
+                "linestyle": "--",
+            }
+
+            if chance_level_kw is None:
+                chance_level_kw = {}
+
+            chance_level_line_kw = _validate_style_kwargs(
+                default_chance_level_line_kw, chance_level_kw
+            )
+
+            (self.chance_level_,) = self.ax_.plot(
+                (0, 1),
+                (self.prevalence_pos_label, self.prevalence_pos_label),
+                **chance_level_line_kw,
+            )
+        else:
+            self.chance_level_ = None
+
+        if despine:
+            _despine(self.ax_)
+
+        if "label" in line_kwargs or plot_chance_level:
+            self.ax_.legend(loc="lower left")
 
-        self.ax_ = ax
-        self.figure_ = ax.figure
         return self
 
+    @classmethod
+    def from_estimator(
+        cls,
+        estimator,
+        X,
+        y,
+        *,
+        sample_weight=None,
+        drop_intermediate=False,
+        response_method="auto",
+        pos_label=None,
+        name=None,
+        ax=None,
+        plot_chance_level=False,
+        chance_level_kw=None,
+        despine=False,
+        **kwargs,
+    ):
+        """Plot precision-recall curve given an estimator and some data.
+
+        For general information regarding `scikit-learn` visualization tools, see
+        the :ref:`Visualization Guide `.
+        For guidance on interpreting these plots, refer to the :ref:`Model
+        Evaluation Guide `.
 
-@_deprecate_positional_args
-def plot_precision_recall_curve(estimator, X, y, *,
-                                sample_weight=None, response_method="auto",
-                                name=None, ax=None, pos_label=None, **kwargs):
-    """Plot Precision Recall Curve for binary classifiers.
+        Parameters
+        ----------
+        estimator : estimator instance
+            Fitted classifier or a fitted :class:`~sklearn.pipeline.Pipeline`
+            in which the last estimator is a classifier.
 
-    Extra keyword arguments will be passed to matplotlib's `plot`.
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Input values.
 
-    Read more in the :ref:`User Guide `.
+        y : array-like of shape (n_samples,)
+            Target values.
 
-    Parameters
-    ----------
-    estimator : estimator instance
-        Fitted classifier or a fitted :class:`~sklearn.pipeline.Pipeline`
-        in which the last estimator is a classifier.
+        sample_weight : array-like of shape (n_samples,), default=None
+            Sample weights.
 
-    X : {array-like, sparse matrix} of shape (n_samples, n_features)
-        Input values.
+        drop_intermediate : bool, default=False
+            Whether to drop some suboptimal thresholds which would not appear
+            on a plotted precision-recall curve. This is useful in order to
+            create lighter precision-recall curves.
 
-    y : array-like of shape (n_samples,)
-        Binary target values.
+            .. versionadded:: 1.3
 
-    sample_weight : array-like of shape (n_samples,), default=None
-        Sample weights.
+        response_method : {'predict_proba', 'decision_function', 'auto'}, \
+            default='auto'
+            Specifies whether to use :term:`predict_proba` or
+            :term:`decision_function` as the target response. If set to 'auto',
+            :term:`predict_proba` is tried first and if it does not exist
+            :term:`decision_function` is tried next.
 
-    response_method : {'predict_proba', 'decision_function', 'auto'}, \
-                      default='auto'
-        Specifies whether to use :term:`predict_proba` or
-        :term:`decision_function` as the target response. If set to 'auto',
-        :term:`predict_proba` is tried first and if it does not exist
-        :term:`decision_function` is tried next.
+        pos_label : int, float, bool or str, default=None
+            The class considered as the positive class when computing the
+            precision and recall metrics. By default, `estimators.classes_[1]`
+            is considered as the positive class.
 
-    name : str, default=None
-        Name for labeling curve. If `None`, the name of the
-        estimator is used.
+        name : str, default=None
+            Name for labeling curve. If `None`, no name is used.
 
-    ax : matplotlib axes, default=None
-        Axes object to plot on. If `None`, a new figure and axes is created.
+        ax : matplotlib axes, default=None
+            Axes object to plot on. If `None`, a new figure and axes is created.
 
-    pos_label : str or int, default=None
-        The class considered as the positive class when computing the precision
-        and recall metrics. By default, `estimators.classes_[1]` is considered
-        as the positive class.
+        plot_chance_level : bool, default=False
+            Whether to plot the chance level. The chance level is the prevalence
+            of the positive label computed from the data passed during
+            :meth:`from_estimator` or :meth:`from_predictions` call.
 
-        .. versionadded:: 0.24
+            .. versionadded:: 1.3
 
-    **kwargs : dict
-        Keyword arguments to be passed to matplotlib's `plot`.
+        chance_level_kw : dict, default=None
+            Keyword arguments to be passed to matplotlib's `plot` for rendering
+            the chance level line.
 
-    Returns
-    -------
-    display : :class:`~sklearn.metrics.PrecisionRecallDisplay`
-        Object that stores computed values.
+            .. versionadded:: 1.3
 
-    See Also
-    --------
-    precision_recall_curve : Compute precision-recall pairs for different
-        probability thresholds.
-    PrecisionRecallDisplay : Precision Recall visualization.
-    """
-    check_matplotlib_support("plot_precision_recall_curve")
+        despine : bool, default=False
+            Whether to remove the top and right spines from the plot.
 
-    y_pred, pos_label = _get_response(
-        X, estimator, response_method, pos_label=pos_label)
+            .. versionadded:: 1.6
 
-    precision, recall, _ = precision_recall_curve(y, y_pred,
-                                                  pos_label=pos_label,
-                                                  sample_weight=sample_weight)
-    average_precision = average_precision_score(y, y_pred,
-                                                pos_label=pos_label,
-                                                sample_weight=sample_weight)
+        **kwargs : dict
+            Keyword arguments to be passed to matplotlib's `plot`.
 
-    name = name if name is not None else estimator.__class__.__name__
+        Returns
+        -------
+        display : :class:`~sklearn.metrics.PrecisionRecallDisplay`
+
+        See Also
+        --------
+        PrecisionRecallDisplay.from_predictions : Plot precision-recall curve
+            using estimated probabilities or output of decision function.
+
+        Notes
+        -----
+        The average precision (cf. :func:`~sklearn.metrics.average_precision_score`)
+        in scikit-learn is computed without any interpolation. To be consistent
+        with this metric, the precision-recall curve is plotted without any
+        interpolation as well (step-wise style).
+
+        You can change this style by passing the keyword argument
+        `drawstyle="default"`. However, the curve will not be strictly
+        consistent with the reported average precision.
+
+        Examples
+        --------
+        >>> import matplotlib.pyplot as plt
+        >>> from sklearn.datasets import make_classification
+        >>> from sklearn.metrics import PrecisionRecallDisplay
+        >>> from sklearn.model_selection import train_test_split
+        >>> from sklearn.linear_model import LogisticRegression
+        >>> X, y = make_classification(random_state=0)
+        >>> X_train, X_test, y_train, y_test = train_test_split(
+        ...         X, y, random_state=0)
+        >>> clf = LogisticRegression()
+        >>> clf.fit(X_train, y_train)
+        LogisticRegression()
+        >>> PrecisionRecallDisplay.from_estimator(
+        ...    clf, X_test, y_test)
+        <...>
+        >>> plt.show()
+        """
+        y_pred, pos_label, name = cls._validate_and_get_response_values(
+            estimator,
+            X,
+            y,
+            response_method=response_method,
+            pos_label=pos_label,
+            name=name,
+        )
+
+        return cls.from_predictions(
+            y,
+            y_pred,
+            sample_weight=sample_weight,
+            name=name,
+            pos_label=pos_label,
+            drop_intermediate=drop_intermediate,
+            ax=ax,
+            plot_chance_level=plot_chance_level,
+            chance_level_kw=chance_level_kw,
+            despine=despine,
+            **kwargs,
+        )
+
+    @classmethod
+    def from_predictions(
+        cls,
+        y_true,
+        y_pred,
+        *,
+        sample_weight=None,
+        drop_intermediate=False,
+        pos_label=None,
+        name=None,
+        ax=None,
+        plot_chance_level=False,
+        chance_level_kw=None,
+        despine=False,
+        **kwargs,
+    ):
+        """Plot precision-recall curve given binary class predictions.
+
+        For general information regarding `scikit-learn` visualization tools, see
+        the :ref:`Visualization Guide `.
+        For guidance on interpreting these plots, refer to the :ref:`Model
+        Evaluation Guide `.
+
+        Parameters
+        ----------
+        y_true : array-like of shape (n_samples,)
+            True binary labels.
+
+        y_pred : array-like of shape (n_samples,)
+            Estimated probabilities or output of decision function.
+
+        sample_weight : array-like of shape (n_samples,), default=None
+            Sample weights.
+
+        drop_intermediate : bool, default=False
+            Whether to drop some suboptimal thresholds which would not appear
+            on a plotted precision-recall curve. This is useful in order to
+            create lighter precision-recall curves.
+
+            .. versionadded:: 1.3
+
+        pos_label : int, float, bool or str, default=None
+            The class considered as the positive class when computing the
+            precision and recall metrics.
+
+        name : str, default=None
+            Name for labeling curve. If `None`, name will be set to
+            `"Classifier"`.
 
-    viz = PrecisionRecallDisplay(
-        precision=precision,
-        recall=recall,
-        average_precision=average_precision,
-        estimator_name=name,
-        pos_label=pos_label,
-    )
+        ax : matplotlib axes, default=None
+            Axes object to plot on. If `None`, a new figure and axes is created.
 
-    return viz.plot(ax=ax, name=name, **kwargs)
+        plot_chance_level : bool, default=False
+            Whether to plot the chance level. The chance level is the prevalence
+            of the positive label computed from the data passed during
+            :meth:`from_estimator` or :meth:`from_predictions` call.
+
+            .. versionadded:: 1.3
+
+        chance_level_kw : dict, default=None
+            Keyword arguments to be passed to matplotlib's `plot` for rendering
+            the chance level line.
+
+            .. versionadded:: 1.3
+
+        despine : bool, default=False
+            Whether to remove the top and right spines from the plot.
+
+            .. versionadded:: 1.6
+
+        **kwargs : dict
+            Keyword arguments to be passed to matplotlib's `plot`.
+
+        Returns
+        -------
+        display : :class:`~sklearn.metrics.PrecisionRecallDisplay`
+
+        See Also
+        --------
+        PrecisionRecallDisplay.from_estimator : Plot precision-recall curve
+            using an estimator.
+
+        Notes
+        -----
+        The average precision (cf. :func:`~sklearn.metrics.average_precision_score`)
+        in scikit-learn is computed without any interpolation. To be consistent
+        with this metric, the precision-recall curve is plotted without any
+        interpolation as well (step-wise style).
+
+        You can change this style by passing the keyword argument
+        `drawstyle="default"`. However, the curve will not be strictly
+        consistent with the reported average precision.
+
+        Examples
+        --------
+        >>> import matplotlib.pyplot as plt
+        >>> from sklearn.datasets import make_classification
+        >>> from sklearn.metrics import PrecisionRecallDisplay
+        >>> from sklearn.model_selection import train_test_split
+        >>> from sklearn.linear_model import LogisticRegression
+        >>> X, y = make_classification(random_state=0)
+        >>> X_train, X_test, y_train, y_test = train_test_split(
+        ...         X, y, random_state=0)
+        >>> clf = LogisticRegression()
+        >>> clf.fit(X_train, y_train)
+        LogisticRegression()
+        >>> y_pred = clf.predict_proba(X_test)[:, 1]
+        >>> PrecisionRecallDisplay.from_predictions(
+        ...    y_test, y_pred)
+        <...>
+        >>> plt.show()
+        """
+        pos_label, name = cls._validate_from_predictions_params(
+            y_true, y_pred, sample_weight=sample_weight, pos_label=pos_label, name=name
+        )
+
+        precision, recall, _ = precision_recall_curve(
+            y_true,
+            y_pred,
+            pos_label=pos_label,
+            sample_weight=sample_weight,
+            drop_intermediate=drop_intermediate,
+        )
+        average_precision = average_precision_score(
+            y_true, y_pred, pos_label=pos_label, sample_weight=sample_weight
+        )
+
+        class_count = Counter(y_true)
+        prevalence_pos_label = class_count[pos_label] / sum(class_count.values())
+
+        viz = cls(
+            precision=precision,
+            recall=recall,
+            average_precision=average_precision,
+            estimator_name=name,
+            pos_label=pos_label,
+            prevalence_pos_label=prevalence_pos_label,
+        )
+
+        return viz.plot(
+            ax=ax,
+            name=name,
+            plot_chance_level=plot_chance_level,
+            chance_level_kw=chance_level_kw,
+            despine=despine,
+            **kwargs,
+        )
diff --git a/sklearn/metrics/_plot/regression.py b/sklearn/metrics/_plot/regression.py
new file mode 100644
index 0000000000000..1b56859cabefd
--- /dev/null
+++ b/sklearn/metrics/_plot/regression.py
@@ -0,0 +1,413 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+import numbers
+
+import numpy as np
+
+from ...utils import _safe_indexing, check_random_state
+from ...utils._optional_dependencies import check_matplotlib_support
+from ...utils._plotting import _validate_style_kwargs
+
+
+class PredictionErrorDisplay:
+    """Visualization of the prediction error of a regression model.
+
+    This tool can display "residuals vs predicted" or "actual vs predicted"
+    using scatter plots to qualitatively assess the behavior of a regressor,
+    preferably on held-out data points.
+
+    See the details in the docstrings of
+    :func:`~sklearn.metrics.PredictionErrorDisplay.from_estimator` or
+    :func:`~sklearn.metrics.PredictionErrorDisplay.from_predictions` to
+    create a visualizer. All parameters are stored as attributes.
+
+    For general information regarding `scikit-learn` visualization tools, read
+    more in the :ref:`Visualization Guide `.
+    For details regarding interpreting these plots, refer to the
+    :ref:`Model Evaluation Guide `.
+
+    .. versionadded:: 1.2
+
+    Parameters
+    ----------
+    y_true : ndarray of shape (n_samples,)
+        True values.
+
+    y_pred : ndarray of shape (n_samples,)
+        Prediction values.
+
+    Attributes
+    ----------
+    line_ : matplotlib Artist
+        Optimal line representing `y_true == y_pred`. Therefore, it is a
+        diagonal line for `kind="predictions"` and a horizontal line for
+        `kind="residuals"`.
+
+    errors_lines_ : matplotlib Artist or None
+        Residual lines. If `with_errors=False`, then it is set to `None`.
+
+    scatter_ : matplotlib Artist
+        Scatter data points.
+
+    ax_ : matplotlib Axes
+        Axes with the different matplotlib axis.
+
+    figure_ : matplotlib Figure
+        Figure containing the scatter and lines.
+
+    See Also
+    --------
+    PredictionErrorDisplay.from_estimator : Prediction error visualization
+        given an estimator and some data.
+    PredictionErrorDisplay.from_predictions : Prediction error visualization
+        given the true and predicted targets.
+
+    Examples
+    --------
+    >>> import matplotlib.pyplot as plt
+    >>> from sklearn.datasets import load_diabetes
+    >>> from sklearn.linear_model import Ridge
+    >>> from sklearn.metrics import PredictionErrorDisplay
+    >>> X, y = load_diabetes(return_X_y=True)
+    >>> ridge = Ridge().fit(X, y)
+    >>> y_pred = ridge.predict(X)
+    >>> display = PredictionErrorDisplay(y_true=y, y_pred=y_pred)
+    >>> display.plot()
+    <...>
+    >>> plt.show()
+    """
+
+    def __init__(self, *, y_true, y_pred):
+        self.y_true = y_true
+        self.y_pred = y_pred
+
+    def plot(
+        self,
+        ax=None,
+        *,
+        kind="residual_vs_predicted",
+        scatter_kwargs=None,
+        line_kwargs=None,
+    ):
+        """Plot visualization.
+
+        Extra keyword arguments will be passed to matplotlib's ``plot``.
+
+        Parameters
+        ----------
+        ax : matplotlib axes, default=None
+            Axes object to plot on. If `None`, a new figure and axes is
+            created.
+
+        kind : {"actual_vs_predicted", "residual_vs_predicted"}, \
+                default="residual_vs_predicted"
+            The type of plot to draw:
+
+            - "actual_vs_predicted" draws the observed values (y-axis) vs.
+              the predicted values (x-axis).
+            - "residual_vs_predicted" draws the residuals, i.e. difference
+              between observed and predicted values, (y-axis) vs. the predicted
+              values (x-axis).
+
+        scatter_kwargs : dict, default=None
+            Dictionary with keywords passed to the `matplotlib.pyplot.scatter`
+            call.
+
+        line_kwargs : dict, default=None
+            Dictionary with keyword passed to the `matplotlib.pyplot.plot`
+            call to draw the optimal line.
+
+        Returns
+        -------
+        display : :class:`~sklearn.metrics.PredictionErrorDisplay`
+
+            Object that stores computed values.
+        """
+        check_matplotlib_support(f"{self.__class__.__name__}.plot")
+
+        expected_kind = ("actual_vs_predicted", "residual_vs_predicted")
+        if kind not in expected_kind:
+            raise ValueError(
+                f"`kind` must be one of {', '.join(expected_kind)}. "
+                f"Got {kind!r} instead."
+            )
+
+        import matplotlib.pyplot as plt
+
+        if scatter_kwargs is None:
+            scatter_kwargs = {}
+        if line_kwargs is None:
+            line_kwargs = {}
+
+        default_scatter_kwargs = {"color": "tab:blue", "alpha": 0.8}
+        default_line_kwargs = {"color": "black", "alpha": 0.7, "linestyle": "--"}
+
+        scatter_kwargs = _validate_style_kwargs(default_scatter_kwargs, scatter_kwargs)
+        line_kwargs = _validate_style_kwargs(default_line_kwargs, line_kwargs)
+
+        scatter_kwargs = {**default_scatter_kwargs, **scatter_kwargs}
+        line_kwargs = {**default_line_kwargs, **line_kwargs}
+
+        if ax is None:
+            _, ax = plt.subplots()
+
+        if kind == "actual_vs_predicted":
+            max_value = max(np.max(self.y_true), np.max(self.y_pred))
+            min_value = min(np.min(self.y_true), np.min(self.y_pred))
+            self.line_ = ax.plot(
+                [min_value, max_value], [min_value, max_value], **line_kwargs
+            )[0]
+
+            x_data, y_data = self.y_pred, self.y_true
+            xlabel, ylabel = "Predicted values", "Actual values"
+
+            self.scatter_ = ax.scatter(x_data, y_data, **scatter_kwargs)
+
+            # force to have a squared axis
+            ax.set_aspect("equal", adjustable="datalim")
+            ax.set_xticks(np.linspace(min_value, max_value, num=5))
+            ax.set_yticks(np.linspace(min_value, max_value, num=5))
+        else:  # kind == "residual_vs_predicted"
+            self.line_ = ax.plot(
+                [np.min(self.y_pred), np.max(self.y_pred)],
+                [0, 0],
+                **line_kwargs,
+            )[0]
+            self.scatter_ = ax.scatter(
+                self.y_pred, self.y_true - self.y_pred, **scatter_kwargs
+            )
+            xlabel, ylabel = "Predicted values", "Residuals (actual - predicted)"
+
+        ax.set(xlabel=xlabel, ylabel=ylabel)
+
+        self.ax_ = ax
+        self.figure_ = ax.figure
+
+        return self
+
+    @classmethod
+    def from_estimator(
+        cls,
+        estimator,
+        X,
+        y,
+        *,
+        kind="residual_vs_predicted",
+        subsample=1_000,
+        random_state=None,
+        ax=None,
+        scatter_kwargs=None,
+        line_kwargs=None,
+    ):
+        """Plot the prediction error given a regressor and some data.
+
+        For general information regarding `scikit-learn` visualization tools,
+        read more in the :ref:`Visualization Guide `.
+        For details regarding interpreting these plots, refer to the
+        :ref:`Model Evaluation Guide `.
+
+        .. versionadded:: 1.2
+
+        Parameters
+        ----------
+        estimator : estimator instance
+            Fitted regressor or a fitted :class:`~sklearn.pipeline.Pipeline`
+            in which the last estimator is a regressor.
+
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Input values.
+
+        y : array-like of shape (n_samples,)
+            Target values.
+
+        kind : {"actual_vs_predicted", "residual_vs_predicted"}, \
+                default="residual_vs_predicted"
+            The type of plot to draw:
+
+            - "actual_vs_predicted" draws the observed values (y-axis) vs.
+              the predicted values (x-axis).
+            - "residual_vs_predicted" draws the residuals, i.e. difference
+              between observed and predicted values, (y-axis) vs. the predicted
+              values (x-axis).
+
+        subsample : float, int or None, default=1_000
+            Sampling the samples to be shown on the scatter plot. If `float`,
+            it should be between 0 and 1 and represents the proportion of the
+            original dataset. If `int`, it represents the number of samples
+            display on the scatter plot. If `None`, no subsampling will be
+            applied. by default, 1000 samples or less will be displayed.
+
+        random_state : int or RandomState, default=None
+            Controls the randomness when `subsample` is not `None`.
+            See :term:`Glossary ` for details.
+
+        ax : matplotlib axes, default=None
+            Axes object to plot on. If `None`, a new figure and axes is
+            created.
+
+        scatter_kwargs : dict, default=None
+            Dictionary with keywords passed to the `matplotlib.pyplot.scatter`
+            call.
+
+        line_kwargs : dict, default=None
+            Dictionary with keyword passed to the `matplotlib.pyplot.plot`
+            call to draw the optimal line.
+
+        Returns
+        -------
+        display : :class:`~sklearn.metrics.PredictionErrorDisplay`
+            Object that stores the computed values.
+
+        See Also
+        --------
+        PredictionErrorDisplay : Prediction error visualization for regression.
+        PredictionErrorDisplay.from_predictions : Prediction error visualization
+            given the true and predicted targets.
+
+        Examples
+        --------
+        >>> import matplotlib.pyplot as plt
+        >>> from sklearn.datasets import load_diabetes
+        >>> from sklearn.linear_model import Ridge
+        >>> from sklearn.metrics import PredictionErrorDisplay
+        >>> X, y = load_diabetes(return_X_y=True)
+        >>> ridge = Ridge().fit(X, y)
+        >>> disp = PredictionErrorDisplay.from_estimator(ridge, X, y)
+        >>> plt.show()
+        """
+        check_matplotlib_support(f"{cls.__name__}.from_estimator")
+
+        y_pred = estimator.predict(X)
+
+        return cls.from_predictions(
+            y_true=y,
+            y_pred=y_pred,
+            kind=kind,
+            subsample=subsample,
+            random_state=random_state,
+            ax=ax,
+            scatter_kwargs=scatter_kwargs,
+            line_kwargs=line_kwargs,
+        )
+
+    @classmethod
+    def from_predictions(
+        cls,
+        y_true,
+        y_pred,
+        *,
+        kind="residual_vs_predicted",
+        subsample=1_000,
+        random_state=None,
+        ax=None,
+        scatter_kwargs=None,
+        line_kwargs=None,
+    ):
+        """Plot the prediction error given the true and predicted targets.
+
+        For general information regarding `scikit-learn` visualization tools,
+        read more in the :ref:`Visualization Guide `.
+        For details regarding interpreting these plots, refer to the
+        :ref:`Model Evaluation Guide `.
+
+        .. versionadded:: 1.2
+
+        Parameters
+        ----------
+        y_true : array-like of shape (n_samples,)
+            True target values.
+
+        y_pred : array-like of shape (n_samples,)
+            Predicted target values.
+
+        kind : {"actual_vs_predicted", "residual_vs_predicted"}, \
+                default="residual_vs_predicted"
+            The type of plot to draw:
+
+            - "actual_vs_predicted" draws the observed values (y-axis) vs.
+              the predicted values (x-axis).
+            - "residual_vs_predicted" draws the residuals, i.e. difference
+              between observed and predicted values, (y-axis) vs. the predicted
+              values (x-axis).
+
+        subsample : float, int or None, default=1_000
+            Sampling the samples to be shown on the scatter plot. If `float`,
+            it should be between 0 and 1 and represents the proportion of the
+            original dataset. If `int`, it represents the number of samples
+            display on the scatter plot. If `None`, no subsampling will be
+            applied. by default, 1000 samples or less will be displayed.
+
+        random_state : int or RandomState, default=None
+            Controls the randomness when `subsample` is not `None`.
+            See :term:`Glossary ` for details.
+
+        ax : matplotlib axes, default=None
+            Axes object to plot on. If `None`, a new figure and axes is
+            created.
+
+        scatter_kwargs : dict, default=None
+            Dictionary with keywords passed to the `matplotlib.pyplot.scatter`
+            call.
+
+        line_kwargs : dict, default=None
+            Dictionary with keyword passed to the `matplotlib.pyplot.plot`
+            call to draw the optimal line.
+
+        Returns
+        -------
+        display : :class:`~sklearn.metrics.PredictionErrorDisplay`
+            Object that stores the computed values.
+
+        See Also
+        --------
+        PredictionErrorDisplay : Prediction error visualization for regression.
+        PredictionErrorDisplay.from_estimator : Prediction error visualization
+            given an estimator and some data.
+
+        Examples
+        --------
+        >>> import matplotlib.pyplot as plt
+        >>> from sklearn.datasets import load_diabetes
+        >>> from sklearn.linear_model import Ridge
+        >>> from sklearn.metrics import PredictionErrorDisplay
+        >>> X, y = load_diabetes(return_X_y=True)
+        >>> ridge = Ridge().fit(X, y)
+        >>> y_pred = ridge.predict(X)
+        >>> disp = PredictionErrorDisplay.from_predictions(y_true=y, y_pred=y_pred)
+        >>> plt.show()
+        """
+        check_matplotlib_support(f"{cls.__name__}.from_predictions")
+
+        random_state = check_random_state(random_state)
+
+        n_samples = len(y_true)
+        if isinstance(subsample, numbers.Integral):
+            if subsample <= 0:
+                raise ValueError(
+                    f"When an integer, subsample={subsample} should be positive."
+                )
+        elif isinstance(subsample, numbers.Real):
+            if subsample <= 0 or subsample >= 1:
+                raise ValueError(
+                    f"When a floating-point, subsample={subsample} should"
+                    " be in the (0, 1) range."
+                )
+            subsample = int(n_samples * subsample)
+
+        if subsample is not None and subsample < n_samples:
+            indices = random_state.choice(np.arange(n_samples), size=subsample)
+            y_true = _safe_indexing(y_true, indices, axis=0)
+            y_pred = _safe_indexing(y_pred, indices, axis=0)
+
+        viz = cls(
+            y_true=y_true,
+            y_pred=y_pred,
+        )
+
+        return viz.plot(
+            ax=ax,
+            kind=kind,
+            scatter_kwargs=scatter_kwargs,
+            line_kwargs=line_kwargs,
+        )
diff --git a/sklearn/metrics/_plot/roc_curve.py b/sklearn/metrics/_plot/roc_curve.py
index 308ae4f4bf85d..439c6cfcdd996 100644
--- a/sklearn/metrics/_plot/roc_curve.py
+++ b/sklearn/metrics/_plot/roc_curve.py
@@ -1,45 +1,103 @@
-from .base import _get_response
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-from .. import auc
-from .. import roc_curve
 
-from ...utils import check_matplotlib_support
-from ...utils.validation import _deprecate_positional_args
+import warnings
 
+import numpy as np
 
-class RocCurveDisplay:
+from ...utils import _safe_indexing
+from ...utils._plotting import (
+    _BinaryClassifierCurveDisplayMixin,
+    _check_param_lengths,
+    _convert_to_list_leaving_none,
+    _deprecate_estimator_name,
+    _despine,
+    _validate_style_kwargs,
+)
+from ...utils._response import _get_response_values_binary
+from .._ranking import auc, roc_curve
+
+
+class RocCurveDisplay(_BinaryClassifierCurveDisplayMixin):
     """ROC Curve visualization.
 
-    It is recommend to use :func:`~sklearn.metrics.plot_roc_curve` to create a
-    visualizer. All parameters are stored as attributes.
+    It is recommended to use
+    :func:`~sklearn.metrics.RocCurveDisplay.from_estimator` or
+    :func:`~sklearn.metrics.RocCurveDisplay.from_predictions` or
+    :func:`~sklearn.metrics.RocCurveDisplay.from_cv_results` to create
+    a :class:`~sklearn.metrics.RocCurveDisplay`. All parameters are
+    stored as attributes.
 
-    Read more in the :ref:`User Guide `.
+    For general information regarding `scikit-learn` visualization tools, see
+    the :ref:`Visualization Guide `.
+    For guidance on interpreting these plots, refer to the :ref:`Model
+    Evaluation Guide `.
 
     Parameters
     ----------
-    fpr : ndarray
-        False positive rate.
-
-    tpr : ndarray
-        True positive rate.
-
-    roc_auc : float, default=None
-        Area under ROC curve. If None, the roc_auc score is not shown.
-
-    estimator_name : str, default=None
-        Name of estimator. If None, the estimator name is not shown.
-
-    pos_label : str or int, default=None
+    fpr : ndarray or list of ndarrays
+        False positive rates. Each ndarray should contain values for a single curve.
+        If plotting multiple curves, list should be of same length as `tpr`.
+
+        .. versionchanged:: 1.7
+            Now accepts a list for plotting multiple curves.
+
+    tpr : ndarray or list of ndarrays
+        True positive rates. Each ndarray should contain values for a single curve.
+        If plotting multiple curves, list should be of same length as `fpr`.
+
+        .. versionchanged:: 1.7
+            Now accepts a list for plotting multiple curves.
+
+    roc_auc : float or list of floats, default=None
+        Area under ROC curve, used for labeling each curve in the legend.
+        If plotting multiple curves, should be a list of the same length as `fpr`
+        and `tpr`. If `None`, ROC AUC scores are not shown in the legend.
+
+        .. versionchanged:: 1.7
+            Now accepts a list for plotting multiple curves.
+
+    name : str or list of str, default=None
+        Name for labeling legend entries. The number of legend entries
+        is determined by the `curve_kwargs` passed to `plot`.
+        To label each curve, provide a list of strings. To avoid labeling
+        individual curves that have the same appearance, this cannot be used in
+        conjunction with `curve_kwargs` being a dictionary or None. If a
+        string is provided, it will be used to either label the single legend entry
+        or if there are multiple legend entries, label each individual curve with
+        the same name. If `None`, set to `name` provided at `RocCurveDisplay`
+        initialization. If still `None`, no name is shown in the legend.
+
+        .. versionadded:: 1.7
+
+    pos_label : int, float, bool or str, default=None
         The class considered as the positive class when computing the roc auc
         metrics. By default, `estimators.classes_[1]` is considered
         as the positive class.
 
         .. versionadded:: 0.24
 
+    estimator_name : str, default=None
+        Name of estimator. If None, the estimator name is not shown.
+
+        .. deprecated:: 1.7
+            `estimator_name` is deprecated and will be removed in 1.9. Use `name`
+            instead.
+
     Attributes
     ----------
-    line_ : matplotlib Artist
-        ROC Curve.
+    line_ : matplotlib Artist or list of matplotlib Artists
+        ROC Curves.
+
+        .. versionchanged:: 1.7
+            This attribute can now be a list of Artists, for when multiple curves
+            are plotted.
+
+    chance_level_ : matplotlib Artist or None
+        The chance level line. It is `None` if the chance level is not plotted.
+
+        .. versionadded:: 1.3
 
     ax_ : matplotlib Axes
         Axes with ROC Curve.
@@ -50,37 +108,74 @@ class RocCurveDisplay:
     See Also
     --------
     roc_curve : Compute Receiver operating characteristic (ROC) curve.
-    plot_roc_curve : Plot Receiver operating characteristic (ROC) curve.
+    RocCurveDisplay.from_estimator : Plot Receiver Operating Characteristic
+        (ROC) curve given an estimator and some data.
+    RocCurveDisplay.from_predictions : Plot Receiver Operating Characteristic
+        (ROC) curve given the true and predicted values.
     roc_auc_score : Compute the area under the ROC curve.
 
     Examples
     --------
-    >>> import matplotlib.pyplot as plt  # doctest: +SKIP
+    >>> import matplotlib.pyplot as plt
     >>> import numpy as np
     >>> from sklearn import metrics
-    >>> y = np.array([0, 0, 1, 1])
-    >>> pred = np.array([0.1, 0.4, 0.35, 0.8])
-    >>> fpr, tpr, thresholds = metrics.roc_curve(y, pred)
+    >>> y_true = np.array([0, 0, 1, 1])
+    >>> y_score = np.array([0.1, 0.4, 0.35, 0.8])
+    >>> fpr, tpr, thresholds = metrics.roc_curve(y_true, y_score)
     >>> roc_auc = metrics.auc(fpr, tpr)
-    >>> display = metrics.RocCurveDisplay(fpr=fpr, tpr=tpr, roc_auc=roc_auc,\
-                                          estimator_name='example estimator')
-    >>> display.plot()  # doctest: +SKIP
-    >>> plt.show()      # doctest: +SKIP
+    >>> display = metrics.RocCurveDisplay(fpr=fpr, tpr=tpr, roc_auc=roc_auc,
+    ...                                   name='example estimator')
+    >>> display.plot()
+    <...>
+    >>> plt.show()
     """
-    @_deprecate_positional_args
-    def __init__(self, *, fpr, tpr,
-                 roc_auc=None, estimator_name=None, pos_label=None):
-        self.estimator_name = estimator_name
+
+    def __init__(
+        self,
+        *,
+        fpr,
+        tpr,
+        roc_auc=None,
+        name=None,
+        pos_label=None,
+        estimator_name="deprecated",
+    ):
         self.fpr = fpr
         self.tpr = tpr
         self.roc_auc = roc_auc
+        self.name = _deprecate_estimator_name(estimator_name, name, "1.7")
         self.pos_label = pos_label
 
-    @_deprecate_positional_args
-    def plot(self, ax=None, *, name=None, **kwargs):
-        """Plot visualization
-
-        Extra keyword arguments will be passed to matplotlib's ``plot``.
+    def _validate_plot_params(self, *, ax, name):
+        self.ax_, self.figure_, name = super()._validate_plot_params(ax=ax, name=name)
+
+        fpr = _convert_to_list_leaving_none(self.fpr)
+        tpr = _convert_to_list_leaving_none(self.tpr)
+        roc_auc = _convert_to_list_leaving_none(self.roc_auc)
+        name = _convert_to_list_leaving_none(name)
+
+        optional = {"self.roc_auc": roc_auc}
+        if isinstance(name, list) and len(name) != 1:
+            optional.update({"'name' (or self.name)": name})
+        _check_param_lengths(
+            required={"self.fpr": fpr, "self.tpr": tpr},
+            optional=optional,
+            class_name="RocCurveDisplay",
+        )
+        return fpr, tpr, roc_auc, name
+
+    def plot(
+        self,
+        ax=None,
+        *,
+        name=None,
+        curve_kwargs=None,
+        plot_chance_level=False,
+        chance_level_kw=None,
+        despine=False,
+        **kwargs,
+    ):
+        """Plot visualization.
 
         Parameters
         ----------
@@ -88,143 +183,614 @@ def plot(self, ax=None, *, name=None, **kwargs):
             Axes object to plot on. If `None`, a new figure and axes is
             created.
 
+        name : str or list of str, default=None
+            Name for labeling legend entries. The number of legend entries
+            is determined by `curve_kwargs`.
+            To label each curve, provide a list of strings. To avoid labeling
+            individual curves that have the same appearance, this cannot be used in
+            conjunction with `curve_kwargs` being a dictionary or None. If a
+            string is provided, it will be used to either label the single legend entry
+            or if there are multiple legend entries, label each individual curve with
+            the same name. If `None`, set to `name` provided at `RocCurveDisplay`
+            initialization. If still `None`, no name is shown in the legend.
+
+            .. versionadded:: 1.7
+
+        curve_kwargs : dict or list of dict, default=None
+            Keywords arguments to be passed to matplotlib's `plot` function
+            to draw individual ROC curves. For single curve plotting, should be
+            a dictionary. For multi-curve plotting, if a list is provided the
+            parameters are applied to the ROC curves of each CV fold
+            sequentially and a legend entry is added for each curve.
+            If a single dictionary is provided, the same parameters are applied
+            to all ROC curves and a single legend entry for all curves is added,
+            labeled with the mean ROC AUC score.
+
+            .. versionadded:: 1.7
+
+        plot_chance_level : bool, default=False
+            Whether to plot the chance level.
+
+            .. versionadded:: 1.3
+
+        chance_level_kw : dict, default=None
+            Keyword arguments to be passed to matplotlib's `plot` for rendering
+            the chance level line.
+
+            .. versionadded:: 1.3
+
+        despine : bool, default=False
+            Whether to remove the top and right spines from the plot.
+
+            .. versionadded:: 1.6
+
+        **kwargs : dict
+            Keyword arguments to be passed to matplotlib's `plot`.
+
+            .. deprecated:: 1.7
+                kwargs is deprecated and will be removed in 1.9. Pass matplotlib
+                arguments to `curve_kwargs` as a dictionary instead.
+
+        Returns
+        -------
+        display : :class:`~sklearn.metrics.RocCurveDisplay`
+            Object that stores computed values.
+        """
+        fpr, tpr, roc_auc, name = self._validate_plot_params(ax=ax, name=name)
+        n_curves = len(fpr)
+        if not isinstance(curve_kwargs, list) and n_curves > 1:
+            if roc_auc:
+                legend_metric = {"mean": np.mean(roc_auc), "std": np.std(roc_auc)}
+            else:
+                legend_metric = {"mean": None, "std": None}
+        else:
+            roc_auc = roc_auc if roc_auc is not None else [None] * n_curves
+            legend_metric = {"metric": roc_auc}
+
+        curve_kwargs = self._validate_curve_kwargs(
+            n_curves,
+            name,
+            legend_metric,
+            "AUC",
+            curve_kwargs=curve_kwargs,
+            **kwargs,
+        )
+
+        default_chance_level_line_kw = {
+            "label": "Chance level (AUC = 0.5)",
+            "color": "k",
+            "linestyle": "--",
+        }
+
+        if chance_level_kw is None:
+            chance_level_kw = {}
+
+        chance_level_kw = _validate_style_kwargs(
+            default_chance_level_line_kw, chance_level_kw
+        )
+
+        self.line_ = []
+        for fpr, tpr, line_kw in zip(fpr, tpr, curve_kwargs):
+            self.line_.extend(self.ax_.plot(fpr, tpr, **line_kw))
+        # Return single artist if only one curve is plotted
+        if len(self.line_) == 1:
+            self.line_ = self.line_[0]
+
+        info_pos_label = (
+            f" (Positive label: {self.pos_label})" if self.pos_label is not None else ""
+        )
+
+        xlabel = "False Positive Rate" + info_pos_label
+        ylabel = "True Positive Rate" + info_pos_label
+        self.ax_.set(
+            xlabel=xlabel,
+            xlim=(-0.01, 1.01),
+            ylabel=ylabel,
+            ylim=(-0.01, 1.01),
+            aspect="equal",
+        )
+
+        if plot_chance_level:
+            (self.chance_level_,) = self.ax_.plot((0, 1), (0, 1), **chance_level_kw)
+        else:
+            self.chance_level_ = None
+
+        if despine:
+            _despine(self.ax_)
+
+        if curve_kwargs[0].get("label") is not None or (
+            plot_chance_level and chance_level_kw.get("label") is not None
+        ):
+            self.ax_.legend(loc="lower right")
+
+        return self
+
+    @classmethod
+    def from_estimator(
+        cls,
+        estimator,
+        X,
+        y,
+        *,
+        sample_weight=None,
+        drop_intermediate=True,
+        response_method="auto",
+        pos_label=None,
+        name=None,
+        ax=None,
+        curve_kwargs=None,
+        plot_chance_level=False,
+        chance_level_kw=None,
+        despine=False,
+        **kwargs,
+    ):
+        """Create a ROC Curve display from an estimator.
+
+        For general information regarding `scikit-learn` visualization tools,
+        see the :ref:`Visualization Guide `.
+        For guidance on interpreting these plots, refer to the :ref:`Model
+        Evaluation Guide `.
+
+        Parameters
+        ----------
+        estimator : estimator instance
+            Fitted classifier or a fitted :class:`~sklearn.pipeline.Pipeline`
+            in which the last estimator is a classifier.
+
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Input values.
+
+        y : array-like of shape (n_samples,)
+            Target values.
+
+        sample_weight : array-like of shape (n_samples,), default=None
+            Sample weights.
+
+        drop_intermediate : bool, default=True
+            Whether to drop thresholds where the resulting point is collinear
+            with its neighbors in ROC space. This has no effect on the ROC AUC
+            or visual shape of the curve, but reduces the number of plotted
+            points.
+
+        response_method : {'predict_proba', 'decision_function', 'auto'} \
+                default='auto'
+            Specifies whether to use :term:`predict_proba` or
+            :term:`decision_function` as the target response. If set to 'auto',
+            :term:`predict_proba` is tried first and if it does not exist
+            :term:`decision_function` is tried next.
+
+        pos_label : int, float, bool or str, default=None
+            The class considered as the positive class when computing the ROC AUC.
+            By default, `estimators.classes_[1]` is considered
+            as the positive class.
+
         name : str, default=None
             Name of ROC Curve for labeling. If `None`, use the name of the
             estimator.
 
+        ax : matplotlib axes, default=None
+            Axes object to plot on. If `None`, a new figure and axes is created.
+
+        curve_kwargs : dict, default=None
+            Keywords arguments to be passed to matplotlib's `plot` function.
+
+            .. versionadded:: 1.7
+
+        plot_chance_level : bool, default=False
+            Whether to plot the chance level.
+
+            .. versionadded:: 1.3
+
+        chance_level_kw : dict, default=None
+            Keyword arguments to be passed to matplotlib's `plot` for rendering
+            the chance level line.
+
+            .. versionadded:: 1.3
+
+        despine : bool, default=False
+            Whether to remove the top and right spines from the plot.
+
+            .. versionadded:: 1.6
+
+        **kwargs : dict
+            Keyword arguments to be passed to matplotlib's `plot`.
+
+            .. deprecated:: 1.7
+                kwargs is deprecated and will be removed in 1.9. Pass matplotlib
+                arguments to `curve_kwargs` as a dictionary instead.
+
         Returns
         -------
-        display : :class:`~sklearn.metrics.plot.RocCurveDisplay`
-            Object that stores computed values.
+        display : :class:`~sklearn.metrics.RocCurveDisplay`
+            The ROC Curve display.
+
+        See Also
+        --------
+        roc_curve : Compute Receiver operating characteristic (ROC) curve.
+        RocCurveDisplay.from_predictions : ROC Curve visualization given the
+            probabilities of scores of a classifier.
+        roc_auc_score : Compute the area under the ROC curve.
+
+        Examples
+        --------
+        >>> import matplotlib.pyplot as plt
+        >>> from sklearn.datasets import make_classification
+        >>> from sklearn.metrics import RocCurveDisplay
+        >>> from sklearn.model_selection import train_test_split
+        >>> from sklearn.svm import SVC
+        >>> X, y = make_classification(random_state=0)
+        >>> X_train, X_test, y_train, y_test = train_test_split(
+        ...     X, y, random_state=0)
+        >>> clf = SVC(random_state=0).fit(X_train, y_train)
+        >>> RocCurveDisplay.from_estimator(
+        ...    clf, X_test, y_test)
+        <...>
+        >>> plt.show()
         """
-        check_matplotlib_support('RocCurveDisplay.plot')
+        y_score, pos_label, name = cls._validate_and_get_response_values(
+            estimator,
+            X,
+            y,
+            response_method=response_method,
+            pos_label=pos_label,
+            name=name,
+        )
+
+        return cls.from_predictions(
+            y_true=y,
+            y_score=y_score,
+            sample_weight=sample_weight,
+            drop_intermediate=drop_intermediate,
+            name=name,
+            ax=ax,
+            pos_label=pos_label,
+            curve_kwargs=curve_kwargs,
+            plot_chance_level=plot_chance_level,
+            chance_level_kw=chance_level_kw,
+            despine=despine,
+            **kwargs,
+        )
+
+    @classmethod
+    def from_predictions(
+        cls,
+        y_true,
+        y_score=None,
+        *,
+        sample_weight=None,
+        drop_intermediate=True,
+        pos_label=None,
+        name=None,
+        ax=None,
+        curve_kwargs=None,
+        plot_chance_level=False,
+        chance_level_kw=None,
+        despine=False,
+        y_pred="deprecated",
+        **kwargs,
+    ):
+        """Plot ROC curve given the true and predicted values.
+
+        For general information regarding `scikit-learn` visualization tools,
+        see the :ref:`Visualization Guide `.
+        For guidance on interpreting these plots, refer to the :ref:`Model
+        Evaluation Guide `.
+
+        .. versionadded:: 1.0
 
-        name = self.estimator_name if name is None else name
+        Parameters
+        ----------
+        y_true : array-like of shape (n_samples,)
+            True labels.
 
-        line_kwargs = {}
-        if self.roc_auc is not None and name is not None:
-            line_kwargs["label"] = f"{name} (AUC = {self.roc_auc:0.2f})"
-        elif self.roc_auc is not None:
-            line_kwargs["label"] = f"AUC = {self.roc_auc:0.2f}"
-        elif name is not None:
-            line_kwargs["label"] = name
+        y_score : array-like of shape (n_samples,)
+            Target scores, can either be probability estimates of the positive
+            class, confidence values, or non-thresholded measure of decisions
+            (as returned by “decision_function” on some classifiers).
 
-        line_kwargs.update(**kwargs)
+            .. versionadded:: 1.7
+                `y_pred` has been renamed to `y_score`.
 
-        import matplotlib.pyplot as plt
+        sample_weight : array-like of shape (n_samples,), default=None
+            Sample weights.
 
-        if ax is None:
-            fig, ax = plt.subplots()
+        drop_intermediate : bool, default=True
+            Whether to drop thresholds where the resulting point is collinear
+            with its neighbors in ROC space. This has no effect on the ROC AUC
+            or visual shape of the curve, but reduces the number of plotted
+            points.
 
-        self.line_, = ax.plot(self.fpr, self.tpr, **line_kwargs)
-        info_pos_label = (f" (Positive label: {self.pos_label})"
-                          if self.pos_label is not None else "")
+        pos_label : int, float, bool or str, default=None
+            The label of the positive class when computing the ROC AUC.
+            When `pos_label=None`, if `y_true` is in {-1, 1} or {0, 1}, `pos_label`
+            is set to 1, otherwise an error will be raised.
 
-        xlabel = "False Positive Rate" + info_pos_label
-        ylabel = "True Positive Rate" + info_pos_label
-        ax.set(xlabel=xlabel, ylabel=ylabel)
+        name : str, default=None
+            Name of ROC curve for legend labeling. If `None`, name will be set to
+            `"Classifier"`.
 
-        if "label" in line_kwargs:
-            ax.legend(loc="lower right")
+        ax : matplotlib axes, default=None
+            Axes object to plot on. If `None`, a new figure and axes is
+            created.
 
-        self.ax_ = ax
-        self.figure_ = ax.figure
-        return self
+        curve_kwargs : dict, default=None
+            Keywords arguments to be passed to matplotlib's `plot` function.
 
+            .. versionadded:: 1.7
 
-@_deprecate_positional_args
-def plot_roc_curve(estimator, X, y, *, sample_weight=None,
-                   drop_intermediate=True, response_method="auto",
-                   name=None, ax=None, pos_label=None, **kwargs):
-    """Plot Receiver operating characteristic (ROC) curve.
+        plot_chance_level : bool, default=False
+            Whether to plot the chance level.
 
-    Extra keyword arguments will be passed to matplotlib's `plot`.
+            .. versionadded:: 1.3
 
-    Read more in the :ref:`User Guide `.
+        chance_level_kw : dict, default=None
+            Keyword arguments to be passed to matplotlib's `plot` for rendering
+            the chance level line.
 
-    Parameters
-    ----------
-    estimator : estimator instance
-        Fitted classifier or a fitted :class:`~sklearn.pipeline.Pipeline`
-        in which the last estimator is a classifier.
+            .. versionadded:: 1.3
 
-    X : {array-like, sparse matrix} of shape (n_samples, n_features)
-        Input values.
+        despine : bool, default=False
+            Whether to remove the top and right spines from the plot.
 
-    y : array-like of shape (n_samples,)
-        Target values.
+            .. versionadded:: 1.6
 
-    sample_weight : array-like of shape (n_samples,), default=None
-        Sample weights.
+        y_pred : array-like of shape (n_samples,)
+            Target scores, can either be probability estimates of the positive
+            class, confidence values, or non-thresholded measure of decisions
+            (as returned by “decision_function” on some classifiers).
 
-    drop_intermediate : boolean, default=True
-        Whether to drop some suboptimal thresholds which would not appear
-        on a plotted ROC curve. This is useful in order to create lighter
-        ROC curves.
+            .. deprecated:: 1.7
+                `y_pred` is deprecated and will be removed in 1.9. Use
+                `y_score` instead.
 
-    response_method : {'predict_proba', 'decision_function', 'auto'} \
-    default='auto'
-        Specifies whether to use :term:`predict_proba` or
-        :term:`decision_function` as the target response. If set to 'auto',
-        :term:`predict_proba` is tried first and if it does not exist
-        :term:`decision_function` is tried next.
+        **kwargs : dict
+            Additional keywords arguments passed to matplotlib `plot` function.
 
-    name : str, default=None
-        Name of ROC Curve for labeling. If `None`, use the name of the
-        estimator.
+            .. deprecated:: 1.7
+                kwargs is deprecated and will be removed in 1.9. Pass matplotlib
+                arguments to `curve_kwargs` as a dictionary instead.
 
-    ax : matplotlib axes, default=None
-        Axes object to plot on. If `None`, a new figure and axes is created.
+        Returns
+        -------
+        display : :class:`~sklearn.metrics.RocCurveDisplay`
+            Object that stores computed values.
 
-    pos_label : str or int, default=None
-        The class considered as the positive class when computing the roc auc
-        metrics. By default, `estimators.classes_[1]` is considered
-        as the positive class.
+        See Also
+        --------
+        roc_curve : Compute Receiver operating characteristic (ROC) curve.
+        RocCurveDisplay.from_estimator : ROC Curve visualization given an
+            estimator and some data.
+        roc_auc_score : Compute the area under the ROC curve.
+
+        Examples
+        --------
+        >>> import matplotlib.pyplot as plt
+        >>> from sklearn.datasets import make_classification
+        >>> from sklearn.metrics import RocCurveDisplay
+        >>> from sklearn.model_selection import train_test_split
+        >>> from sklearn.svm import SVC
+        >>> X, y = make_classification(random_state=0)
+        >>> X_train, X_test, y_train, y_test = train_test_split(
+        ...     X, y, random_state=0)
+        >>> clf = SVC(random_state=0).fit(X_train, y_train)
+        >>> y_score = clf.decision_function(X_test)
+        >>> RocCurveDisplay.from_predictions(y_test, y_score)
+        <...>
+        >>> plt.show()
+        """
+        # TODO(1.9): remove after the end of the deprecation period of `y_pred`
+        if y_score is not None and not (
+            isinstance(y_pred, str) and y_pred == "deprecated"
+        ):
+            raise ValueError(
+                "`y_pred` and `y_score` cannot be both specified. Please use `y_score`"
+                " only as `y_pred` is deprecated in 1.7 and will be removed in 1.9."
+            )
+        if not (isinstance(y_pred, str) and y_pred == "deprecated"):
+            warnings.warn(
+                (
+                    "y_pred is deprecated in 1.7 and will be removed in 1.9. "
+                    "Please use `y_score` instead."
+                ),
+                FutureWarning,
+            )
+            y_score = y_pred
+
+        pos_label_validated, name = cls._validate_from_predictions_params(
+            y_true, y_score, sample_weight=sample_weight, pos_label=pos_label, name=name
+        )
+
+        fpr, tpr, _ = roc_curve(
+            y_true,
+            y_score,
+            pos_label=pos_label,
+            sample_weight=sample_weight,
+            drop_intermediate=drop_intermediate,
+        )
+        roc_auc = auc(fpr, tpr)
+
+        viz = cls(
+            fpr=fpr,
+            tpr=tpr,
+            roc_auc=roc_auc,
+            name=name,
+            pos_label=pos_label_validated,
+        )
+
+        return viz.plot(
+            ax=ax,
+            curve_kwargs=curve_kwargs,
+            plot_chance_level=plot_chance_level,
+            chance_level_kw=chance_level_kw,
+            despine=despine,
+            **kwargs,
+        )
+
+    @classmethod
+    def from_cv_results(
+        cls,
+        cv_results,
+        X,
+        y,
+        *,
+        sample_weight=None,
+        drop_intermediate=True,
+        response_method="auto",
+        pos_label=None,
+        ax=None,
+        name=None,
+        curve_kwargs=None,
+        plot_chance_level=False,
+        chance_level_kwargs=None,
+        despine=False,
+    ):
+        """Create a multi-fold ROC curve display given cross-validation results.
+
+        .. versionadded:: 1.7
 
-        .. versionadded:: 0.24
+        Parameters
+        ----------
+        cv_results : dict
+            Dictionary as returned by :func:`~sklearn.model_selection.cross_validate`
+            using `return_estimator=True` and `return_indices=True` (i.e., dictionary
+            should contain the keys "estimator" and "indices").
 
-    Returns
-    -------
-    display : :class:`~sklearn.metrics.RocCurveDisplay`
-        Object that stores computed values.
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Input values.
 
-    See Also
-    --------
-    roc_curve : Compute Receiver operating characteristic (ROC) curve.
-    RocCurveDisplay : ROC Curve visualization.
-    roc_auc_score : Compute the area under the ROC curve.
+        y : array-like of shape (n_samples,)
+            Target values.
 
-    Examples
-    --------
-    >>> import matplotlib.pyplot as plt  # doctest: +SKIP
-    >>> from sklearn import datasets, metrics, model_selection, svm
-    >>> X, y = datasets.make_classification(random_state=0)
-    >>> X_train, X_test, y_train, y_test = model_selection.train_test_split(
-    ...     X, y, random_state=0)
-    >>> clf = svm.SVC(random_state=0)
-    >>> clf.fit(X_train, y_train)
-    SVC(random_state=0)
-    >>> metrics.plot_roc_curve(clf, X_test, y_test)  # doctest: +SKIP
-    >>> plt.show()                                   # doctest: +SKIP
-    """
-    check_matplotlib_support('plot_roc_curve')
+        sample_weight : array-like of shape (n_samples,), default=None
+            Sample weights.
+
+        drop_intermediate : bool, default=True
+            Whether to drop some suboptimal thresholds which would not appear
+            on a plotted ROC curve. This is useful in order to create lighter
+            ROC curves.
 
-    y_pred, pos_label = _get_response(
-        X, estimator, response_method, pos_label=pos_label)
+        response_method : {'predict_proba', 'decision_function', 'auto'} \
+                default='auto'
+            Specifies whether to use :term:`predict_proba` or
+            :term:`decision_function` as the target response. If set to 'auto',
+            :term:`predict_proba` is tried first and if it does not exist
+            :term:`decision_function` is tried next.
 
-    fpr, tpr, _ = roc_curve(y, y_pred, pos_label=pos_label,
-                            sample_weight=sample_weight,
-                            drop_intermediate=drop_intermediate)
-    roc_auc = auc(fpr, tpr)
+        pos_label : int, float, bool or str, default=None
+            The class considered as the positive class when computing the ROC AUC
+            metrics. By default, `estimators.classes_[1]` is considered
+            as the positive class.
 
-    name = estimator.__class__.__name__ if name is None else name
+        ax : matplotlib axes, default=None
+            Axes object to plot on. If `None`, a new figure and axes is
+            created.
 
-    viz = RocCurveDisplay(
-        fpr=fpr,
-        tpr=tpr,
-        roc_auc=roc_auc,
-        estimator_name=name,
-        pos_label=pos_label
-    )
+        name : str or list of str, default=None
+            Name for labeling legend entries. The number of legend entries
+            is determined by `curve_kwargs`.
+            To label each curve, provide a list of strings. To avoid labeling
+            individual curves that have the same appearance, this cannot be used in
+            conjunction with `curve_kwargs` being a dictionary or None. If a
+            string is provided, it will be used to either label the single legend entry
+            or if there are multiple legend entries, label each individual curve with
+            the same name. If `None`, no name is shown in the legend.
+
+        curve_kwargs : dict or list of dict, default=None
+            Keywords arguments to be passed to matplotlib's `plot` function
+            to draw individual ROC curves. If a list is provided the
+            parameters are applied to the ROC curves of each CV fold
+            sequentially and a legend entry is added for each curve.
+            If a single dictionary is provided, the same parameters are applied
+            to all ROC curves and a single legend entry for all curves is added,
+            labeled with the mean ROC AUC score.
+
+        plot_chance_level : bool, default=False
+            Whether to plot the chance level.
+
+        chance_level_kwargs : dict, default=None
+            Keyword arguments to be passed to matplotlib's `plot` for rendering
+            the chance level line.
+
+        despine : bool, default=False
+            Whether to remove the top and right spines from the plot.
 
-    return viz.plot(ax=ax, name=name, **kwargs)
+        Returns
+        -------
+        display : :class:`~sklearn.metrics.RocCurveDisplay`
+            The multi-fold ROC curve display.
+
+        See Also
+        --------
+        roc_curve : Compute Receiver operating characteristic (ROC) curve.
+            RocCurveDisplay.from_estimator : ROC Curve visualization given an
+            estimator and some data.
+        RocCurveDisplay.from_predictions : ROC Curve visualization given the
+            probabilities of scores of a classifier.
+        roc_auc_score : Compute the area under the ROC curve.
+
+        Examples
+        --------
+        >>> import matplotlib.pyplot as plt
+        >>> from sklearn.datasets import make_classification
+        >>> from sklearn.metrics import RocCurveDisplay
+        >>> from sklearn.model_selection import cross_validate
+        >>> from sklearn.svm import SVC
+        >>> X, y = make_classification(random_state=0)
+        >>> clf = SVC(random_state=0)
+        >>> cv_results = cross_validate(
+        ...     clf, X, y, cv=3, return_estimator=True, return_indices=True)
+        >>> RocCurveDisplay.from_cv_results(cv_results, X, y)
+        <...>
+        >>> plt.show()
+        """
+        pos_label_ = cls._validate_from_cv_results_params(
+            cv_results,
+            X,
+            y,
+            sample_weight=sample_weight,
+            pos_label=pos_label,
+        )
+
+        fpr_folds, tpr_folds, auc_folds = [], [], []
+        for estimator, test_indices in zip(
+            cv_results["estimator"], cv_results["indices"]["test"]
+        ):
+            y_true = _safe_indexing(y, test_indices)
+            y_pred, _ = _get_response_values_binary(
+                estimator,
+                _safe_indexing(X, test_indices),
+                response_method=response_method,
+                pos_label=pos_label_,
+            )
+            sample_weight_fold = (
+                None
+                if sample_weight is None
+                else _safe_indexing(sample_weight, test_indices)
+            )
+            fpr, tpr, _ = roc_curve(
+                y_true,
+                y_pred,
+                pos_label=pos_label_,
+                sample_weight=sample_weight_fold,
+                drop_intermediate=drop_intermediate,
+            )
+            roc_auc = auc(fpr, tpr)
+
+            fpr_folds.append(fpr)
+            tpr_folds.append(tpr)
+            auc_folds.append(roc_auc)
+
+        viz = cls(
+            fpr=fpr_folds,
+            tpr=tpr_folds,
+            name=name,
+            roc_auc=auc_folds,
+            pos_label=pos_label_,
+        )
+        return viz.plot(
+            ax=ax,
+            curve_kwargs=curve_kwargs,
+            plot_chance_level=plot_chance_level,
+            chance_level_kw=chance_level_kwargs,
+            despine=despine,
+        )
diff --git a/sklearn/metrics/_plot/tests/test_common_curve_display.py b/sklearn/metrics/_plot/tests/test_common_curve_display.py
new file mode 100644
index 0000000000000..753f2a1e7319d
--- /dev/null
+++ b/sklearn/metrics/_plot/tests/test_common_curve_display.py
@@ -0,0 +1,292 @@
+import numpy as np
+import pytest
+
+from sklearn.base import BaseEstimator, ClassifierMixin, clone
+from sklearn.calibration import CalibrationDisplay
+from sklearn.compose import make_column_transformer
+from sklearn.datasets import load_iris
+from sklearn.exceptions import NotFittedError
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import (
+    ConfusionMatrixDisplay,
+    DetCurveDisplay,
+    PrecisionRecallDisplay,
+    PredictionErrorDisplay,
+    RocCurveDisplay,
+)
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import StandardScaler
+from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
+
+
+@pytest.fixture(scope="module")
+def data():
+    return load_iris(return_X_y=True)
+
+
+@pytest.fixture(scope="module")
+def data_binary(data):
+    X, y = data
+    return X[y < 2], y[y < 2]
+
+
+@pytest.mark.parametrize(
+    "Display",
+    [CalibrationDisplay, DetCurveDisplay, PrecisionRecallDisplay, RocCurveDisplay],
+)
+def test_display_curve_error_classifier(pyplot, data, data_binary, Display):
+    """Check that a proper error is raised when only binary classification is
+    supported."""
+    X, y = data
+    X_binary, y_binary = data_binary
+    clf = DecisionTreeClassifier().fit(X, y)
+
+    # Case 1: multiclass classifier with multiclass target
+    msg = "Expected 'estimator' to be a binary classifier. Got 3 classes instead."
+    with pytest.raises(ValueError, match=msg):
+        Display.from_estimator(clf, X, y)
+
+    # Case 2: multiclass classifier with binary target
+    with pytest.raises(ValueError, match=msg):
+        Display.from_estimator(clf, X_binary, y_binary)
+
+    # Case 3: binary classifier with multiclass target
+    clf = DecisionTreeClassifier().fit(X_binary, y_binary)
+    msg = "The target y is not binary. Got multiclass type of target."
+    with pytest.raises(ValueError, match=msg):
+        Display.from_estimator(clf, X, y)
+
+
+@pytest.mark.parametrize(
+    "Display",
+    [CalibrationDisplay, DetCurveDisplay, PrecisionRecallDisplay, RocCurveDisplay],
+)
+def test_display_curve_error_regression(pyplot, data_binary, Display):
+    """Check that we raise an error with regressor."""
+
+    # Case 1: regressor
+    X, y = data_binary
+    regressor = DecisionTreeRegressor().fit(X, y)
+
+    msg = "Expected 'estimator' to be a binary classifier. Got DecisionTreeRegressor"
+    with pytest.raises(ValueError, match=msg):
+        Display.from_estimator(regressor, X, y)
+
+    # Case 2: regression target
+    classifier = DecisionTreeClassifier().fit(X, y)
+    # Force `y_true` to be seen as a regression problem
+    y = y + 0.5
+    msg = "The target y is not binary. Got continuous type of target."
+    with pytest.raises(ValueError, match=msg):
+        Display.from_estimator(classifier, X, y)
+    with pytest.raises(ValueError, match=msg):
+        Display.from_predictions(y, regressor.fit(X, y).predict(X))
+
+
+@pytest.mark.parametrize(
+    "response_method, msg",
+    [
+        (
+            "predict_proba",
+            "MyClassifier has none of the following attributes: predict_proba.",
+        ),
+        (
+            "decision_function",
+            "MyClassifier has none of the following attributes: decision_function.",
+        ),
+        (
+            "auto",
+            (
+                "MyClassifier has none of the following attributes: predict_proba,"
+                " decision_function."
+            ),
+        ),
+        (
+            "bad_method",
+            "MyClassifier has none of the following attributes: bad_method.",
+        ),
+    ],
+)
+@pytest.mark.parametrize(
+    "Display", [DetCurveDisplay, PrecisionRecallDisplay, RocCurveDisplay]
+)
+def test_display_curve_error_no_response(
+    pyplot,
+    data_binary,
+    response_method,
+    msg,
+    Display,
+):
+    """Check that a proper error is raised when the response method requested
+    is not defined for the given trained classifier."""
+    X, y = data_binary
+
+    class MyClassifier(ClassifierMixin, BaseEstimator):
+        def fit(self, X, y):
+            self.classes_ = [0, 1]
+            return self
+
+    clf = MyClassifier().fit(X, y)
+
+    with pytest.raises(AttributeError, match=msg):
+        Display.from_estimator(clf, X, y, response_method=response_method)
+
+
+@pytest.mark.parametrize("Display", [DetCurveDisplay, PrecisionRecallDisplay])
+@pytest.mark.parametrize("constructor_name", ["from_estimator", "from_predictions"])
+def test_display_curve_estimator_name_multiple_calls(
+    pyplot,
+    data_binary,
+    Display,
+    constructor_name,
+):
+    """Check that passing `name` when calling `plot` will overwrite the original name
+    in the legend."""
+    X, y = data_binary
+    clf_name = "my hand-crafted name"
+    clf = LogisticRegression().fit(X, y)
+    y_pred = clf.predict_proba(X)[:, 1]
+
+    # safe guard for the binary if/else construction
+    assert constructor_name in ("from_estimator", "from_predictions")
+
+    if constructor_name == "from_estimator":
+        disp = Display.from_estimator(clf, X, y, name=clf_name)
+    else:
+        disp = Display.from_predictions(y, y_pred, name=clf_name)
+    assert disp.estimator_name == clf_name
+    pyplot.close("all")
+    disp.plot()
+    assert clf_name in disp.line_.get_label()
+    pyplot.close("all")
+    clf_name = "another_name"
+    disp.plot(name=clf_name)
+    assert clf_name in disp.line_.get_label()
+
+
+# TODO: remove this test once classes moved to using `name` instead of
+# `estimator_name`
+@pytest.mark.parametrize(
+    "clf",
+    [
+        LogisticRegression(),
+        make_pipeline(StandardScaler(), LogisticRegression()),
+        make_pipeline(
+            make_column_transformer((StandardScaler(), [0, 1])), LogisticRegression()
+        ),
+    ],
+)
+@pytest.mark.parametrize("Display", [DetCurveDisplay, PrecisionRecallDisplay])
+def test_display_curve_not_fitted_errors_old_name(pyplot, data_binary, clf, Display):
+    """Check that a proper error is raised when the classifier is not
+    fitted."""
+    X, y = data_binary
+    # clone since we parametrize the test and the classifier will be fitted
+    # when testing the second and subsequent plotting function
+    model = clone(clf)
+    with pytest.raises(NotFittedError):
+        Display.from_estimator(model, X, y)
+    model.fit(X, y)
+    disp = Display.from_estimator(model, X, y)
+    assert model.__class__.__name__ in disp.line_.get_label()
+    assert disp.estimator_name == model.__class__.__name__
+
+
+@pytest.mark.parametrize(
+    "clf",
+    [
+        LogisticRegression(),
+        make_pipeline(StandardScaler(), LogisticRegression()),
+        make_pipeline(
+            make_column_transformer((StandardScaler(), [0, 1])), LogisticRegression()
+        ),
+    ],
+)
+@pytest.mark.parametrize("Display", [RocCurveDisplay])
+def test_display_curve_not_fitted_errors(pyplot, data_binary, clf, Display):
+    """Check that a proper error is raised when the classifier is not fitted."""
+    X, y = data_binary
+    # clone since we parametrize the test and the classifier will be fitted
+    # when testing the second and subsequent plotting function
+    model = clone(clf)
+    with pytest.raises(NotFittedError):
+        Display.from_estimator(model, X, y)
+    model.fit(X, y)
+    disp = Display.from_estimator(model, X, y)
+    assert model.__class__.__name__ in disp.line_.get_label()
+    assert disp.name == model.__class__.__name__
+
+
+@pytest.mark.parametrize(
+    "Display", [DetCurveDisplay, PrecisionRecallDisplay, RocCurveDisplay]
+)
+def test_display_curve_n_samples_consistency(pyplot, data_binary, Display):
+    """Check the error raised when `y_pred` or `sample_weight` have inconsistent
+    length."""
+    X, y = data_binary
+    classifier = DecisionTreeClassifier().fit(X, y)
+
+    msg = "Found input variables with inconsistent numbers of samples"
+    with pytest.raises(ValueError, match=msg):
+        Display.from_estimator(classifier, X[:-2], y)
+    with pytest.raises(ValueError, match=msg):
+        Display.from_estimator(classifier, X, y[:-2])
+    with pytest.raises(ValueError, match=msg):
+        Display.from_estimator(classifier, X, y, sample_weight=np.ones(X.shape[0] - 2))
+
+
+@pytest.mark.parametrize(
+    "Display", [DetCurveDisplay, PrecisionRecallDisplay, RocCurveDisplay]
+)
+def test_display_curve_error_pos_label(pyplot, data_binary, Display):
+    """Check consistence of error message when `pos_label` should be specified."""
+    X, y = data_binary
+    y = y + 10
+
+    classifier = DecisionTreeClassifier().fit(X, y)
+    y_pred = classifier.predict_proba(X)[:, -1]
+    msg = r"y_true takes value in {10, 11} and pos_label is not specified"
+    with pytest.raises(ValueError, match=msg):
+        Display.from_predictions(y, y_pred)
+
+
+@pytest.mark.parametrize(
+    "Display",
+    [
+        CalibrationDisplay,
+        DetCurveDisplay,
+        PrecisionRecallDisplay,
+        RocCurveDisplay,
+        PredictionErrorDisplay,
+        ConfusionMatrixDisplay,
+    ],
+)
+@pytest.mark.parametrize(
+    "constructor",
+    ["from_predictions", "from_estimator"],
+)
+def test_classifier_display_curve_named_constructor_return_type(
+    pyplot, data_binary, Display, constructor
+):
+    """Check that named constructors return the correct type when subclassed.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/pull/27675
+    """
+    X, y = data_binary
+
+    # This can be anything - we just need to check the named constructor return
+    # type so the only requirement here is instantiating the class without error
+    y_pred = y
+
+    classifier = LogisticRegression().fit(X, y)
+
+    class SubclassOfDisplay(Display):
+        pass
+
+    if constructor == "from_predictions":
+        curve = SubclassOfDisplay.from_predictions(y, y_pred)
+    else:  # constructor == "from_estimator"
+        curve = SubclassOfDisplay.from_estimator(classifier, X, y)
+
+    assert isinstance(curve, SubclassOfDisplay)
diff --git a/sklearn/metrics/_plot/tests/test_confusion_matrix_display.py b/sklearn/metrics/_plot/tests/test_confusion_matrix_display.py
index ed0bc04117396..6e93bf4993a93 100644
--- a/sklearn/metrics/_plot/tests/test_confusion_matrix_display.py
+++ b/sklearn/metrics/_plot/tests/test_confusion_matrix_display.py
@@ -1,28 +1,18 @@
+import numpy as np
+import pytest
 from numpy.testing import (
     assert_allclose,
     assert_array_equal,
 )
-import numpy as np
-import pytest
 
-from sklearn.datasets import make_classification
 from sklearn.compose import make_column_transformer
+from sklearn.datasets import make_classification
 from sklearn.exceptions import NotFittedError
 from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import ConfusionMatrixDisplay, confusion_matrix
 from sklearn.pipeline import make_pipeline
 from sklearn.preprocessing import StandardScaler
-from sklearn.svm import SVC
-from sklearn.svm import SVR
-
-from sklearn.metrics import ConfusionMatrixDisplay
-from sklearn.metrics import confusion_matrix
-
-
-# TODO: Remove when https://github.com/numpy/numpy/issues/14397 is resolved
-pytestmark = pytest.mark.filterwarnings(
-    "ignore:In future, it will be an error for 'np.bool_':DeprecationWarning:"
-    "matplotlib.*"
-)
+from sklearn.svm import SVC, SVR
 
 
 def test_confusion_matrix_display_validation(pyplot):
@@ -31,6 +21,9 @@ def test_confusion_matrix_display_validation(pyplot):
         n_samples=100, n_informative=5, n_classes=5, random_state=0
     )
 
+    with pytest.raises(NotFittedError):
+        ConfusionMatrixDisplay.from_estimator(SVC(), X, y)
+
     regressor = SVR().fit(X, y)
     y_pred_regressor = regressor.predict(X)
     y_pred_classifier = SVC().fit(X, y).predict(X)
@@ -51,36 +44,7 @@ def test_confusion_matrix_display_validation(pyplot):
         ConfusionMatrixDisplay.from_predictions(y, y_pred_classifier[::2])
 
 
-@pytest.mark.parametrize(
-    "constructor_name", ["from_estimator", "from_predictions"]
-)
-def test_confusion_matrix_display_invalid_option(pyplot, constructor_name):
-    """Check the error raise if an invalid parameter value is passed."""
-    X, y = make_classification(
-        n_samples=100, n_informative=5, n_classes=5, random_state=0
-    )
-    classifier = SVC().fit(X, y)
-    y_pred = classifier.predict(X)
-
-    # safe guard for the binary if/else construction
-    assert constructor_name in ("from_estimator", "from_predictions")
-    extra_params = {"normalize": "invalid"}
-
-    err_msg = r"normalize must be one of \{'true', 'pred', 'all', None\}"
-    with pytest.raises(ValueError, match=err_msg):
-        if constructor_name == "from_estimator":
-            ConfusionMatrixDisplay.from_estimator(
-                classifier, X, y, **extra_params
-            )
-        else:
-            ConfusionMatrixDisplay.from_predictions(
-                y, y_pred, **extra_params
-            )
-
-
-@pytest.mark.parametrize(
-    "constructor_name", ["from_estimator", "from_predictions"]
-)
+@pytest.mark.parametrize("constructor_name", ["from_estimator", "from_predictions"])
 @pytest.mark.parametrize("with_labels", [True, False])
 @pytest.mark.parametrize("with_display_labels", [True, False])
 def test_confusion_matrix_display_custom_labels(
@@ -108,13 +72,9 @@ def test_confusion_matrix_display_custom_labels(
         "labels": labels,
     }
     if constructor_name == "from_estimator":
-        disp = ConfusionMatrixDisplay.from_estimator(
-            classifier, X, y, **common_kwargs
-        )
+        disp = ConfusionMatrixDisplay.from_estimator(classifier, X, y, **common_kwargs)
     else:
-        disp = ConfusionMatrixDisplay.from_predictions(
-            y, y_pred, **common_kwargs
-        )
+        disp = ConfusionMatrixDisplay.from_predictions(y, y_pred, **common_kwargs)
     assert_allclose(disp.confusion_matrix, cm)
 
     if with_display_labels:
@@ -124,8 +84,7 @@ def test_confusion_matrix_display_custom_labels(
     else:
         expected_display_labels = list(range(n_classes))
 
-    expected_display_labels_str = [str(name)
-                                   for name in expected_display_labels]
+    expected_display_labels_str = [str(name) for name in expected_display_labels]
 
     x_ticks = [tick.get_text() for tick in disp.ax_.get_xticklabels()]
     y_ticks = [tick.get_text() for tick in disp.ax_.get_yticklabels()]
@@ -135,13 +94,14 @@ def test_confusion_matrix_display_custom_labels(
     assert_array_equal(y_ticks, expected_display_labels_str)
 
 
-@pytest.mark.parametrize(
-    "constructor_name", ["from_estimator", "from_predictions"]
-)
+@pytest.mark.parametrize("constructor_name", ["from_estimator", "from_predictions"])
 @pytest.mark.parametrize("normalize", ["true", "pred", "all", None])
 @pytest.mark.parametrize("include_values", [True, False])
 def test_confusion_matrix_display_plotting(
-    pyplot, constructor_name, normalize, include_values,
+    pyplot,
+    constructor_name,
+    normalize,
+    include_values,
 ):
     """Check the overall plotting rendering."""
     n_classes = 5
@@ -165,13 +125,9 @@ def test_confusion_matrix_display_plotting(
         "include_values": include_values,
     }
     if constructor_name == "from_estimator":
-        disp = ConfusionMatrixDisplay.from_estimator(
-            classifier, X, y, **common_kwargs
-        )
+        disp = ConfusionMatrixDisplay.from_estimator(classifier, X, y, **common_kwargs)
     else:
-        disp = ConfusionMatrixDisplay.from_predictions(
-            y, y_pred, **common_kwargs
-        )
+        disp = ConfusionMatrixDisplay.from_predictions(y, y_pred, **common_kwargs)
 
     assert disp.ax_ == ax
 
@@ -198,9 +154,7 @@ def test_confusion_matrix_display_plotting(
 
     expected_display_labels = list(range(n_classes))
 
-    expected_display_labels_str = [
-        str(name) for name in expected_display_labels
-    ]
+    expected_display_labels_str = [str(name) for name in expected_display_labels]
 
     assert_array_equal(disp.display_labels, expected_display_labels)
     assert_array_equal(x_ticks, expected_display_labels_str)
@@ -213,17 +167,13 @@ def test_confusion_matrix_display_plotting(
         assert disp.text_.shape == (n_classes, n_classes)
         fmt = ".2g"
         expected_text = np.array([format(v, fmt) for v in cm.ravel(order="C")])
-        text_text = np.array(
-            [t.get_text() for t in disp.text_.ravel(order="C")]
-        )
+        text_text = np.array([t.get_text() for t in disp.text_.ravel(order="C")])
         assert_array_equal(expected_text, text_text)
     else:
         assert disp.text_ is None
 
 
-@pytest.mark.parametrize(
-    "constructor_name", ["from_estimator", "from_predictions"]
-)
+@pytest.mark.parametrize("constructor_name", ["from_estimator", "from_predictions"])
 def test_confusion_matrix_display(pyplot, constructor_name):
     """Check the behaviour of the default constructor without using the class
     methods."""
@@ -245,13 +195,9 @@ def test_confusion_matrix_display(pyplot, constructor_name):
         "xticks_rotation": 45.0,
     }
     if constructor_name == "from_estimator":
-        disp = ConfusionMatrixDisplay.from_estimator(
-            classifier, X, y, **common_kwargs
-        )
+        disp = ConfusionMatrixDisplay.from_estimator(classifier, X, y, **common_kwargs)
     else:
-        disp = ConfusionMatrixDisplay.from_predictions(
-            y, y_pred, **common_kwargs
-        )
+        disp = ConfusionMatrixDisplay.from_predictions(y, y_pred, **common_kwargs)
 
     assert_allclose(disp.confusion_matrix, cm)
     assert disp.text_.shape == (n_classes, n_classes)
@@ -325,7 +271,7 @@ def test_confusion_matrix_contrast(pyplot):
             LogisticRegression(),
         ),
     ],
-    ids=["clf", "pipeline-clf", "pipeline-column_transformer-clf"]
+    ids=["clf", "pipeline-clf", "pipeline-column_transformer-clf"],
 )
 def test_confusion_matrix_pipeline(pyplot, clf):
     """Check the behaviour of the plotting with more complex pipeline."""
@@ -345,9 +291,7 @@ def test_confusion_matrix_pipeline(pyplot, clf):
     assert disp.text_.shape == (n_classes, n_classes)
 
 
-@pytest.mark.parametrize(
-    "constructor_name", ["from_estimator", "from_predictions"]
-)
+@pytest.mark.parametrize("constructor_name", ["from_estimator", "from_predictions"])
 def test_confusion_matrix_with_unknown_labels(pyplot, constructor_name):
     """Check that when labels=None, the unique values in `y_pred` and `y_true`
     will be used.
@@ -369,14 +313,62 @@ def test_confusion_matrix_with_unknown_labels(pyplot, constructor_name):
 
     common_kwargs = {"labels": None}
     if constructor_name == "from_estimator":
-        disp = ConfusionMatrixDisplay.from_estimator(
-            classifier, X, y, **common_kwargs
-        )
+        disp = ConfusionMatrixDisplay.from_estimator(classifier, X, y, **common_kwargs)
     else:
-        disp = ConfusionMatrixDisplay.from_predictions(
-            y, y_pred, **common_kwargs
-        )
+        disp = ConfusionMatrixDisplay.from_predictions(y, y_pred, **common_kwargs)
 
     display_labels = [tick.get_text() for tick in disp.ax_.get_xticklabels()]
     expected_labels = [str(i) for i in range(n_classes + 1)]
     assert_array_equal(expected_labels, display_labels)
+
+
+def test_colormap_max(pyplot):
+    """Check that the max color is used for the color of the text."""
+    gray = pyplot.get_cmap("gray", 1024)
+    confusion_matrix = np.array([[1.0, 0.0], [0.0, 1.0]])
+
+    disp = ConfusionMatrixDisplay(confusion_matrix)
+    disp.plot(cmap=gray)
+
+    color = disp.text_[1, 0].get_color()
+    assert_allclose(color, [1.0, 1.0, 1.0, 1.0])
+
+
+def test_im_kw_adjust_vmin_vmax(pyplot):
+    """Check that im_kw passes kwargs to imshow"""
+
+    confusion_matrix = np.array([[0.48, 0.04], [0.08, 0.4]])
+    disp = ConfusionMatrixDisplay(confusion_matrix)
+    disp.plot(im_kw=dict(vmin=0.0, vmax=0.8))
+
+    clim = disp.im_.get_clim()
+    assert clim[0] == pytest.approx(0.0)
+    assert clim[1] == pytest.approx(0.8)
+
+
+def test_confusion_matrix_text_kw(pyplot):
+    """Check that text_kw is passed to the text call."""
+    font_size = 15.0
+    X, y = make_classification(random_state=0)
+    classifier = SVC().fit(X, y)
+
+    # from_estimator passes the font size
+    disp = ConfusionMatrixDisplay.from_estimator(
+        classifier, X, y, text_kw={"fontsize": font_size}
+    )
+    for text in disp.text_.reshape(-1):
+        assert text.get_fontsize() == font_size
+
+    # plot adjusts plot to new font size
+    new_font_size = 20.0
+    disp.plot(text_kw={"fontsize": new_font_size})
+    for text in disp.text_.reshape(-1):
+        assert text.get_fontsize() == new_font_size
+
+    # from_predictions passes the font size
+    y_pred = classifier.predict(X)
+    disp = ConfusionMatrixDisplay.from_predictions(
+        y, y_pred, text_kw={"fontsize": font_size}
+    )
+    for text in disp.text_.reshape(-1):
+        assert text.get_fontsize() == font_size
diff --git a/sklearn/metrics/_plot/tests/test_det_curve_display.py b/sklearn/metrics/_plot/tests/test_det_curve_display.py
new file mode 100644
index 0000000000000..105778c631030
--- /dev/null
+++ b/sklearn/metrics/_plot/tests/test_det_curve_display.py
@@ -0,0 +1,114 @@
+import numpy as np
+import pytest
+from numpy.testing import assert_allclose
+
+from sklearn.datasets import load_iris
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import DetCurveDisplay, det_curve
+
+
+@pytest.mark.parametrize("constructor_name", ["from_estimator", "from_predictions"])
+@pytest.mark.parametrize("response_method", ["predict_proba", "decision_function"])
+@pytest.mark.parametrize("with_sample_weight", [True, False])
+@pytest.mark.parametrize("drop_intermediate", [True, False])
+@pytest.mark.parametrize("with_strings", [True, False])
+def test_det_curve_display(
+    pyplot,
+    constructor_name,
+    response_method,
+    with_sample_weight,
+    drop_intermediate,
+    with_strings,
+):
+    X, y = load_iris(return_X_y=True)
+    # Binarize the data with only the two first classes
+    X, y = X[y < 2], y[y < 2]
+
+    pos_label = None
+    if with_strings:
+        y = np.array(["c", "b"])[y]
+        pos_label = "c"
+
+    if with_sample_weight:
+        rng = np.random.RandomState(42)
+        sample_weight = rng.randint(1, 4, size=(X.shape[0]))
+    else:
+        sample_weight = None
+
+    lr = LogisticRegression()
+    lr.fit(X, y)
+    y_pred = getattr(lr, response_method)(X)
+    if y_pred.ndim == 2:
+        y_pred = y_pred[:, 1]
+
+    # safe guard for the binary if/else construction
+    assert constructor_name in ("from_estimator", "from_predictions")
+
+    common_kwargs = {
+        "name": lr.__class__.__name__,
+        "alpha": 0.8,
+        "sample_weight": sample_weight,
+        "drop_intermediate": drop_intermediate,
+        "pos_label": pos_label,
+    }
+    if constructor_name == "from_estimator":
+        disp = DetCurveDisplay.from_estimator(lr, X, y, **common_kwargs)
+    else:
+        disp = DetCurveDisplay.from_predictions(y, y_pred, **common_kwargs)
+
+    fpr, fnr, _ = det_curve(
+        y,
+        y_pred,
+        sample_weight=sample_weight,
+        drop_intermediate=drop_intermediate,
+        pos_label=pos_label,
+    )
+
+    assert_allclose(disp.fpr, fpr, atol=1e-7)
+    assert_allclose(disp.fnr, fnr, atol=1e-7)
+
+    assert disp.estimator_name == "LogisticRegression"
+
+    # cannot fail thanks to pyplot fixture
+    import matplotlib as mpl
+
+    assert isinstance(disp.line_, mpl.lines.Line2D)
+    assert disp.line_.get_alpha() == 0.8
+    assert isinstance(disp.ax_, mpl.axes.Axes)
+    assert isinstance(disp.figure_, mpl.figure.Figure)
+    assert disp.line_.get_label() == "LogisticRegression"
+
+    expected_pos_label = 1 if pos_label is None else pos_label
+    expected_ylabel = f"False Negative Rate (Positive label: {expected_pos_label})"
+    expected_xlabel = f"False Positive Rate (Positive label: {expected_pos_label})"
+    assert disp.ax_.get_ylabel() == expected_ylabel
+    assert disp.ax_.get_xlabel() == expected_xlabel
+
+
+@pytest.mark.parametrize(
+    "constructor_name, expected_clf_name",
+    [
+        ("from_estimator", "LogisticRegression"),
+        ("from_predictions", "Classifier"),
+    ],
+)
+def test_det_curve_display_default_name(
+    pyplot,
+    constructor_name,
+    expected_clf_name,
+):
+    # Check the default name display in the figure when `name` is not provided
+    X, y = load_iris(return_X_y=True)
+    # Binarize the data with only the two first classes
+    X, y = X[y < 2], y[y < 2]
+
+    lr = LogisticRegression().fit(X, y)
+    y_pred = lr.predict_proba(X)[:, 1]
+
+    if constructor_name == "from_estimator":
+        disp = DetCurveDisplay.from_estimator(lr, X, y)
+    else:
+        disp = DetCurveDisplay.from_predictions(y, y_pred)
+
+    assert disp.estimator_name == expected_clf_name
+    assert disp.line_.get_label() == expected_clf_name
diff --git a/sklearn/metrics/_plot/tests/test_plot_confusion_matrix.py b/sklearn/metrics/_plot/tests/test_plot_confusion_matrix.py
deleted file mode 100644
index 6fba7ec4d1a0d..0000000000000
--- a/sklearn/metrics/_plot/tests/test_plot_confusion_matrix.py
+++ /dev/null
@@ -1,369 +0,0 @@
-# TODO: remove this file when plot_confusion_matrix will be deprecated in 1.2
-import pytest
-import numpy as np
-from numpy.testing import assert_allclose
-from numpy.testing import assert_array_equal
-
-from sklearn.compose import make_column_transformer
-from sklearn.datasets import make_classification
-from sklearn.exceptions import NotFittedError
-from sklearn.linear_model import LogisticRegression
-from sklearn.pipeline import make_pipeline
-from sklearn.preprocessing import StandardScaler
-from sklearn.svm import SVC, SVR
-
-from sklearn.metrics import confusion_matrix
-from sklearn.metrics import plot_confusion_matrix
-from sklearn.metrics import ConfusionMatrixDisplay
-
-
-# TODO: Remove when https://github.com/numpy/numpy/issues/14397 is resolved
-pytestmark = pytest.mark.filterwarnings(
-    "ignore:In future, it will be an error for 'np.bool_':DeprecationWarning:"
-    "matplotlib.*")
-
-
-@pytest.fixture(scope="module")
-def n_classes():
-    return 5
-
-
-@pytest.fixture(scope="module")
-def data(n_classes):
-    X, y = make_classification(n_samples=100, n_informative=5,
-                               n_classes=n_classes, random_state=0)
-    return X, y
-
-
-@pytest.fixture(scope="module")
-def fitted_clf(data):
-    return SVC(kernel='linear', C=0.01).fit(*data)
-
-
-@pytest.fixture(scope="module")
-def y_pred(data, fitted_clf):
-    X, _ = data
-    return fitted_clf.predict(X)
-
-
-@pytest.mark.filterwarnings(
-    "ignore: Function plot_confusion_matrix is deprecated"
-)
-def test_error_on_regressor(pyplot, data):
-    X, y = data
-    est = SVR().fit(X, y)
-
-    msg = "plot_confusion_matrix only supports classifiers"
-    with pytest.raises(ValueError, match=msg):
-        plot_confusion_matrix(est, X, y)
-
-
-@pytest.mark.filterwarnings(
-    "ignore: Function plot_confusion_matrix is deprecated"
-)
-def test_error_on_invalid_option(pyplot, fitted_clf, data):
-    X, y = data
-    msg = (r"normalize must be one of \{'true', 'pred', 'all', "
-           r"None\}")
-
-    with pytest.raises(ValueError, match=msg):
-        plot_confusion_matrix(fitted_clf, X, y, normalize='invalid')
-
-
-@pytest.mark.filterwarnings(
-    "ignore: Function plot_confusion_matrix is deprecated"
-)
-@pytest.mark.parametrize("with_labels", [True, False])
-@pytest.mark.parametrize("with_display_labels", [True, False])
-def test_plot_confusion_matrix_custom_labels(pyplot, data, y_pred, fitted_clf,
-                                             n_classes, with_labels,
-                                             with_display_labels):
-    X, y = data
-    ax = pyplot.gca()
-    labels = [2, 1, 0, 3, 4] if with_labels else None
-    display_labels = ['b', 'd', 'a', 'e', 'f'] if with_display_labels else None
-
-    cm = confusion_matrix(y, y_pred, labels=labels)
-    disp = plot_confusion_matrix(fitted_clf, X, y,
-                                 ax=ax, display_labels=display_labels,
-                                 labels=labels)
-
-    assert_allclose(disp.confusion_matrix, cm)
-
-    if with_display_labels:
-        expected_display_labels = display_labels
-    elif with_labels:
-        expected_display_labels = labels
-    else:
-        expected_display_labels = list(range(n_classes))
-
-    expected_display_labels_str = [str(name)
-                                   for name in expected_display_labels]
-
-    x_ticks = [tick.get_text() for tick in disp.ax_.get_xticklabels()]
-    y_ticks = [tick.get_text() for tick in disp.ax_.get_yticklabels()]
-
-    assert_array_equal(disp.display_labels, expected_display_labels)
-    assert_array_equal(x_ticks, expected_display_labels_str)
-    assert_array_equal(y_ticks, expected_display_labels_str)
-
-
-@pytest.mark.filterwarnings(
-    "ignore: Function plot_confusion_matrix is deprecated"
-)
-@pytest.mark.parametrize("normalize", ['true', 'pred', 'all', None])
-@pytest.mark.parametrize("include_values", [True, False])
-def test_plot_confusion_matrix(pyplot, data, y_pred, n_classes, fitted_clf,
-                               normalize, include_values):
-    X, y = data
-    ax = pyplot.gca()
-    cmap = 'plasma'
-    cm = confusion_matrix(y, y_pred)
-    disp = plot_confusion_matrix(fitted_clf, X, y,
-                                 normalize=normalize,
-                                 cmap=cmap, ax=ax,
-                                 include_values=include_values)
-
-    assert disp.ax_ == ax
-
-    if normalize == 'true':
-        cm = cm / cm.sum(axis=1, keepdims=True)
-    elif normalize == 'pred':
-        cm = cm / cm.sum(axis=0, keepdims=True)
-    elif normalize == 'all':
-        cm = cm / cm.sum()
-
-    assert_allclose(disp.confusion_matrix, cm)
-    import matplotlib as mpl
-    assert isinstance(disp.im_, mpl.image.AxesImage)
-    assert disp.im_.get_cmap().name == cmap
-    assert isinstance(disp.ax_, pyplot.Axes)
-    assert isinstance(disp.figure_, pyplot.Figure)
-
-    assert disp.ax_.get_ylabel() == "True label"
-    assert disp.ax_.get_xlabel() == "Predicted label"
-
-    x_ticks = [tick.get_text() for tick in disp.ax_.get_xticklabels()]
-    y_ticks = [tick.get_text() for tick in disp.ax_.get_yticklabels()]
-
-    expected_display_labels = list(range(n_classes))
-
-    expected_display_labels_str = [str(name)
-                                   for name in expected_display_labels]
-
-    assert_array_equal(disp.display_labels, expected_display_labels)
-    assert_array_equal(x_ticks, expected_display_labels_str)
-    assert_array_equal(y_ticks, expected_display_labels_str)
-
-    image_data = disp.im_.get_array().data
-    assert_allclose(image_data, cm)
-
-    if include_values:
-        assert disp.text_.shape == (n_classes, n_classes)
-        fmt = '.2g'
-        expected_text = np.array([format(v, fmt) for v in cm.ravel(order="C")])
-        text_text = np.array([
-            t.get_text() for t in disp.text_.ravel(order="C")])
-        assert_array_equal(expected_text, text_text)
-    else:
-        assert disp.text_ is None
-
-
-@pytest.mark.filterwarnings(
-    "ignore: Function plot_confusion_matrix is deprecated"
-)
-def test_confusion_matrix_display(pyplot, data, fitted_clf, y_pred, n_classes):
-    X, y = data
-
-    cm = confusion_matrix(y, y_pred)
-    disp = plot_confusion_matrix(fitted_clf, X, y, normalize=None,
-                                 include_values=True, cmap='viridis',
-                                 xticks_rotation=45.0)
-
-    assert_allclose(disp.confusion_matrix, cm)
-    assert disp.text_.shape == (n_classes, n_classes)
-
-    rotations = [tick.get_rotation() for tick in disp.ax_.get_xticklabels()]
-    assert_allclose(rotations, 45.0)
-
-    image_data = disp.im_.get_array().data
-    assert_allclose(image_data, cm)
-
-    disp.plot(cmap='plasma')
-    assert disp.im_.get_cmap().name == 'plasma'
-
-    disp.plot(include_values=False)
-    assert disp.text_ is None
-
-    disp.plot(xticks_rotation=90.0)
-    rotations = [tick.get_rotation() for tick in disp.ax_.get_xticklabels()]
-    assert_allclose(rotations, 90.0)
-
-    disp.plot(values_format='e')
-    expected_text = np.array([format(v, 'e') for v in cm.ravel(order="C")])
-    text_text = np.array([
-        t.get_text() for t in disp.text_.ravel(order="C")])
-    assert_array_equal(expected_text, text_text)
-
-
-def test_confusion_matrix_contrast(pyplot):
-    # make sure text color is appropriate depending on background
-
-    cm = np.eye(2) / 2
-    disp = ConfusionMatrixDisplay(cm, display_labels=[0, 1])
-
-    disp.plot(cmap=pyplot.cm.gray)
-    # diagonal text is black
-    assert_allclose(disp.text_[0, 0].get_color(), [0.0, 0.0, 0.0, 1.0])
-    assert_allclose(disp.text_[1, 1].get_color(), [0.0, 0.0, 0.0, 1.0])
-
-    # off-diagonal text is white
-    assert_allclose(disp.text_[0, 1].get_color(), [1.0, 1.0, 1.0, 1.0])
-    assert_allclose(disp.text_[1, 0].get_color(), [1.0, 1.0, 1.0, 1.0])
-
-    disp.plot(cmap=pyplot.cm.gray_r)
-    # diagonal text is white
-    assert_allclose(disp.text_[0, 1].get_color(), [0.0, 0.0, 0.0, 1.0])
-    assert_allclose(disp.text_[1, 0].get_color(), [0.0, 0.0, 0.0, 1.0])
-
-    # off-diagonal text is black
-    assert_allclose(disp.text_[0, 0].get_color(), [1.0, 1.0, 1.0, 1.0])
-    assert_allclose(disp.text_[1, 1].get_color(), [1.0, 1.0, 1.0, 1.0])
-
-    # Regression test for #15920
-    cm = np.array([[19, 34], [32, 58]])
-    disp = ConfusionMatrixDisplay(cm, display_labels=[0, 1])
-
-    disp.plot(cmap=pyplot.cm.Blues)
-    min_color = pyplot.cm.Blues(0)
-    max_color = pyplot.cm.Blues(255)
-    assert_allclose(disp.text_[0, 0].get_color(), max_color)
-    assert_allclose(disp.text_[0, 1].get_color(), max_color)
-    assert_allclose(disp.text_[1, 0].get_color(), max_color)
-    assert_allclose(disp.text_[1, 1].get_color(), min_color)
-
-
-@pytest.mark.filterwarnings(
-    "ignore: Function plot_confusion_matrix is deprecated"
-)
-@pytest.mark.parametrize(
-    "clf", [LogisticRegression(),
-            make_pipeline(StandardScaler(), LogisticRegression()),
-            make_pipeline(make_column_transformer((StandardScaler(), [0, 1])),
-                          LogisticRegression())])
-def test_confusion_matrix_pipeline(pyplot, clf, data, n_classes):
-    X, y = data
-    with pytest.raises(NotFittedError):
-        plot_confusion_matrix(clf, X, y)
-    clf.fit(X, y)
-    y_pred = clf.predict(X)
-
-    disp = plot_confusion_matrix(clf, X, y)
-    cm = confusion_matrix(y, y_pred)
-
-    assert_allclose(disp.confusion_matrix, cm)
-    assert disp.text_.shape == (n_classes, n_classes)
-
-
-@pytest.mark.filterwarnings(
-    "ignore: Function plot_confusion_matrix is deprecated"
-)
-@pytest.mark.parametrize("colorbar", [True, False])
-def test_plot_confusion_matrix_colorbar(pyplot, data, fitted_clf, colorbar):
-    X, y = data
-
-    def _check_colorbar(disp, has_colorbar):
-        if has_colorbar:
-            assert disp.im_.colorbar is not None
-            assert disp.im_.colorbar.__class__.__name__ == "Colorbar"
-        else:
-            assert disp.im_.colorbar is None
-    disp = plot_confusion_matrix(fitted_clf, X, y, colorbar=colorbar)
-    _check_colorbar(disp, colorbar)
-    # attempt a plot with the opposite effect of colorbar
-    disp.plot(colorbar=not colorbar)
-    _check_colorbar(disp, not colorbar)
-
-
-@pytest.mark.filterwarnings(
-    "ignore: Function plot_confusion_matrix is deprecated"
-)
-@pytest.mark.parametrize("values_format", ['e', 'n'])
-def test_confusion_matrix_text_format(pyplot, data, y_pred, n_classes,
-                                      fitted_clf, values_format):
-    # Make sure plot text is formatted with 'values_format'.
-    X, y = data
-    cm = confusion_matrix(y, y_pred)
-    disp = plot_confusion_matrix(fitted_clf, X, y,
-                                 include_values=True,
-                                 values_format=values_format)
-
-    assert disp.text_.shape == (n_classes, n_classes)
-
-    expected_text = np.array([format(v, values_format)
-                              for v in cm.ravel()])
-    text_text = np.array([
-        t.get_text() for t in disp.text_.ravel()])
-    assert_array_equal(expected_text, text_text)
-
-
-def test_confusion_matrix_standard_format(pyplot):
-    cm = np.array([[10000000, 0], [123456, 12345678]])
-    plotted_text = ConfusionMatrixDisplay(
-        cm, display_labels=[False, True]).plot().text_
-    # Values should be shown as whole numbers 'd',
-    # except the first number which should be shown as 1e+07 (longer length)
-    # and the last number will be shown as 1.2e+07 (longer length)
-    test = [t.get_text() for t in plotted_text.ravel()]
-    assert test == ['1e+07', '0', '123456', '1.2e+07']
-
-    cm = np.array([[0.1, 10], [100, 0.525]])
-    plotted_text = ConfusionMatrixDisplay(
-        cm, display_labels=[False, True]).plot().text_
-    # Values should now formatted as '.2g', since there's a float in
-    # Values are have two dec places max, (e.g 100 becomes 1e+02)
-    test = [t.get_text() for t in plotted_text.ravel()]
-    assert test == ['0.1', '10', '1e+02', '0.53']
-
-
-@pytest.mark.parametrize("display_labels, expected_labels", [
-    (None, ["0", "1"]),
-    (["cat", "dog"], ["cat", "dog"]),
-])
-def test_default_labels(pyplot, display_labels, expected_labels):
-    cm = np.array([[10, 0], [12, 120]])
-    disp = ConfusionMatrixDisplay(cm, display_labels=display_labels).plot()
-
-    x_ticks = [tick.get_text() for tick in disp.ax_.get_xticklabels()]
-    y_ticks = [tick.get_text() for tick in disp.ax_.get_yticklabels()]
-
-    assert_array_equal(x_ticks, expected_labels)
-    assert_array_equal(y_ticks, expected_labels)
-
-
-@pytest.mark.filterwarnings(
-    "ignore: Function plot_confusion_matrix is deprecated"
-)
-def test_error_on_a_dataset_with_unseen_labels(
-    pyplot, fitted_clf, data, n_classes
-):
-    """Check that when labels=None, the unique values in `y_pred` and `y_true`
-    will be used.
-    Non-regression test for:
-    https://github.com/scikit-learn/scikit-learn/pull/18405
-    """
-    X, y = data
-
-    # create unseen labels in `y_true` not seen during fitting and not present
-    # in 'fitted_clf.classes_'
-    y = y + 1
-    disp = plot_confusion_matrix(fitted_clf, X, y)
-
-    display_labels = [tick.get_text() for tick in disp.ax_.get_xticklabels()]
-    expected_labels = [str(i) for i in range(n_classes + 1)]
-    assert_array_equal(expected_labels, display_labels)
-
-
-def test_plot_confusion_matrix_deprecation_warning(pyplot, fitted_clf, data):
-    with pytest.warns(FutureWarning):
-        plot_confusion_matrix(fitted_clf, *data)
diff --git a/sklearn/metrics/_plot/tests/test_plot_curve_common.py b/sklearn/metrics/_plot/tests/test_plot_curve_common.py
deleted file mode 100644
index c3b56f1724372..0000000000000
--- a/sklearn/metrics/_plot/tests/test_plot_curve_common.py
+++ /dev/null
@@ -1,103 +0,0 @@
-import pytest
-
-from sklearn.base import ClassifierMixin
-from sklearn.base import clone
-from sklearn.compose import make_column_transformer
-from sklearn.datasets import load_iris
-from sklearn.exceptions import NotFittedError
-from sklearn.linear_model import LogisticRegression
-from sklearn.pipeline import make_pipeline
-from sklearn.preprocessing import StandardScaler
-from sklearn.tree import DecisionTreeClassifier
-
-from sklearn.metrics import plot_det_curve
-from sklearn.metrics import plot_roc_curve
-
-
-@pytest.fixture(scope="module")
-def data():
-    return load_iris(return_X_y=True)
-
-
-@pytest.fixture(scope="module")
-def data_binary(data):
-    X, y = data
-    return X[y < 2], y[y < 2]
-
-
-@pytest.mark.parametrize("plot_func", [plot_det_curve, plot_roc_curve])
-def test_plot_curve_error_non_binary(pyplot, data, plot_func):
-    X, y = data
-    clf = DecisionTreeClassifier()
-    clf.fit(X, y)
-
-    msg = "DecisionTreeClassifier should be a binary classifier"
-    with pytest.raises(ValueError, match=msg):
-        plot_func(clf, X, y)
-
-
-@pytest.mark.parametrize(
-    "response_method, msg",
-    [("predict_proba", "response method predict_proba is not defined in "
-                       "MyClassifier"),
-     ("decision_function", "response method decision_function is not defined "
-                           "in MyClassifier"),
-     ("auto", "response method decision_function or predict_proba is not "
-              "defined in MyClassifier"),
-     ("bad_method", "response_method must be 'predict_proba', "
-                    "'decision_function' or 'auto'")]
-)
-@pytest.mark.parametrize("plot_func", [plot_det_curve, plot_roc_curve])
-def test_plot_curve_error_no_response(
-    pyplot, data_binary, response_method, msg, plot_func,
-):
-    X, y = data_binary
-
-    class MyClassifier(ClassifierMixin):
-        def fit(self, X, y):
-            self.classes_ = [0, 1]
-            return self
-
-    clf = MyClassifier().fit(X, y)
-
-    with pytest.raises(ValueError, match=msg):
-        plot_func(clf, X, y, response_method=response_method)
-
-
-@pytest.mark.parametrize("plot_func", [plot_det_curve, plot_roc_curve])
-def test_plot_curve_estimator_name_multiple_calls(
-    pyplot, data_binary, plot_func
-):
-    # non-regression test checking that the `name` used when calling
-    # `plot_func` is used as well when calling `disp.plot()`
-    X, y = data_binary
-    clf_name = "my hand-crafted name"
-    clf = LogisticRegression().fit(X, y)
-    disp = plot_func(clf, X, y, name=clf_name)
-    assert disp.estimator_name == clf_name
-    pyplot.close("all")
-    disp.plot()
-    assert clf_name in disp.line_.get_label()
-    pyplot.close("all")
-    clf_name = "another_name"
-    disp.plot(name=clf_name)
-    assert clf_name in disp.line_.get_label()
-
-
-@pytest.mark.parametrize(
-    "clf", [LogisticRegression(),
-            make_pipeline(StandardScaler(), LogisticRegression()),
-            make_pipeline(make_column_transformer((StandardScaler(), [0, 1])),
-                          LogisticRegression())])
-@pytest.mark.parametrize("plot_func", [plot_det_curve, plot_roc_curve])
-def test_plot_det_curve_not_fitted_errors(pyplot, data_binary, clf, plot_func):
-    X, y = data_binary
-    # clone since we parametrize the test and the classifier will be fitted
-    # when testing the second and subsequent plotting function
-    model = clone(clf)
-    with pytest.raises(NotFittedError):
-        plot_func(model, X, y)
-    model.fit(X, y)
-    disp = plot_func(model, X, y)
-    assert model.__class__.__name__ in disp.line_.get_label()
-    assert disp.estimator_name == model.__class__.__name__
diff --git a/sklearn/metrics/_plot/tests/test_plot_det_curve.py b/sklearn/metrics/_plot/tests/test_plot_det_curve.py
deleted file mode 100644
index 9ef10237af879..0000000000000
--- a/sklearn/metrics/_plot/tests/test_plot_det_curve.py
+++ /dev/null
@@ -1,84 +0,0 @@
-import pytest
-import numpy as np
-from numpy.testing import assert_allclose
-
-from sklearn.datasets import load_iris
-from sklearn.linear_model import LogisticRegression
-
-from sklearn.metrics import det_curve
-from sklearn.metrics import plot_det_curve
-
-
-@pytest.fixture(scope="module")
-def data():
-    return load_iris(return_X_y=True)
-
-
-@pytest.fixture(scope="module")
-def data_binary(data):
-    X, y = data
-    return X[y < 2], y[y < 2]
-
-
-@pytest.mark.parametrize(
-    "response_method", ["predict_proba", "decision_function"]
-)
-@pytest.mark.parametrize("with_sample_weight", [True, False])
-@pytest.mark.parametrize("with_strings", [True, False])
-def test_plot_det_curve(
-    pyplot,
-    response_method,
-    data_binary,
-    with_sample_weight,
-    with_strings
-):
-    X, y = data_binary
-
-    pos_label = None
-    if with_strings:
-        y = np.array(["c", "b"])[y]
-        pos_label = "c"
-
-    if with_sample_weight:
-        rng = np.random.RandomState(42)
-        sample_weight = rng.randint(1, 4, size=(X.shape[0]))
-    else:
-        sample_weight = None
-
-    lr = LogisticRegression()
-    lr.fit(X, y)
-
-    viz = plot_det_curve(
-        lr, X, y, alpha=0.8, sample_weight=sample_weight,
-    )
-
-    y_pred = getattr(lr, response_method)(X)
-    if y_pred.ndim == 2:
-        y_pred = y_pred[:, 1]
-
-    fpr, fnr, _ = det_curve(
-        y, y_pred, sample_weight=sample_weight, pos_label=pos_label,
-    )
-
-    assert_allclose(viz.fpr, fpr)
-    assert_allclose(viz.fnr, fnr)
-
-    assert viz.estimator_name == "LogisticRegression"
-
-    # cannot fail thanks to pyplot fixture
-    import matplotlib as mpl  # noqal
-    assert isinstance(viz.line_, mpl.lines.Line2D)
-    assert viz.line_.get_alpha() == 0.8
-    assert isinstance(viz.ax_, mpl.axes.Axes)
-    assert isinstance(viz.figure_, mpl.figure.Figure)
-    assert viz.line_.get_label() == "LogisticRegression"
-
-    expected_pos_label = 1 if pos_label is None else pos_label
-    expected_ylabel = (
-        f"False Negative Rate (Positive label: {expected_pos_label})"
-    )
-    expected_xlabel = (
-        f"False Positive Rate (Positive label: {expected_pos_label})"
-    )
-    assert viz.ax_.get_ylabel() == expected_ylabel
-    assert viz.ax_.get_xlabel() == expected_xlabel
diff --git a/sklearn/metrics/_plot/tests/test_plot_precision_recall.py b/sklearn/metrics/_plot/tests/test_plot_precision_recall.py
deleted file mode 100644
index 48db806df87bf..0000000000000
--- a/sklearn/metrics/_plot/tests/test_plot_precision_recall.py
+++ /dev/null
@@ -1,240 +0,0 @@
-import pytest
-import numpy as np
-from numpy.testing import assert_allclose
-
-from sklearn.base import BaseEstimator, ClassifierMixin
-from sklearn.metrics import plot_precision_recall_curve
-from sklearn.metrics import PrecisionRecallDisplay
-from sklearn.metrics import average_precision_score
-from sklearn.metrics import precision_recall_curve
-from sklearn.datasets import make_classification
-from sklearn.datasets import load_breast_cancer
-from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
-from sklearn.linear_model import LogisticRegression
-from sklearn.model_selection import train_test_split
-from sklearn.exceptions import NotFittedError
-from sklearn.pipeline import make_pipeline
-from sklearn.preprocessing import StandardScaler
-from sklearn.utils import shuffle
-from sklearn.compose import make_column_transformer
-
-# TODO: Remove when https://github.com/numpy/numpy/issues/14397 is resolved
-pytestmark = pytest.mark.filterwarnings(
-    "ignore:In future, it will be an error for 'np.bool_':DeprecationWarning:"
-    "matplotlib.*")
-
-
-def test_errors(pyplot):
-    X, y_multiclass = make_classification(n_classes=3, n_samples=50,
-                                          n_informative=3,
-                                          random_state=0)
-    y_binary = y_multiclass == 0
-
-    # Unfitted classifer
-    binary_clf = DecisionTreeClassifier()
-    with pytest.raises(NotFittedError):
-        plot_precision_recall_curve(binary_clf, X, y_binary)
-    binary_clf.fit(X, y_binary)
-
-    multi_clf = DecisionTreeClassifier().fit(X, y_multiclass)
-
-    # Fitted multiclass classifier with binary data
-    msg = "DecisionTreeClassifier should be a binary classifier"
-    with pytest.raises(ValueError, match=msg):
-        plot_precision_recall_curve(multi_clf, X, y_binary)
-
-    reg = DecisionTreeRegressor().fit(X, y_multiclass)
-    msg = "DecisionTreeRegressor should be a binary classifier"
-    with pytest.raises(ValueError, match=msg):
-        plot_precision_recall_curve(reg, X, y_binary)
-
-
-@pytest.mark.parametrize(
-    "response_method, msg",
-    [("predict_proba", "response method predict_proba is not defined in "
-                       "MyClassifier"),
-     ("decision_function", "response method decision_function is not defined "
-                           "in MyClassifier"),
-     ("auto", "response method decision_function or predict_proba is not "
-              "defined in MyClassifier"),
-     ("bad_method", "response_method must be 'predict_proba', "
-                    "'decision_function' or 'auto'")])
-def test_error_bad_response(pyplot, response_method, msg):
-    X, y = make_classification(n_classes=2, n_samples=50, random_state=0)
-
-    class MyClassifier(ClassifierMixin, BaseEstimator):
-        def fit(self, X, y):
-            self.fitted_ = True
-            self.classes_ = [0, 1]
-            return self
-
-    clf = MyClassifier().fit(X, y)
-
-    with pytest.raises(ValueError, match=msg):
-        plot_precision_recall_curve(clf, X, y, response_method=response_method)
-
-
-@pytest.mark.parametrize("response_method",
-                         ["predict_proba", "decision_function"])
-@pytest.mark.parametrize("with_sample_weight", [True, False])
-def test_plot_precision_recall(pyplot, response_method, with_sample_weight):
-    X, y = make_classification(n_classes=2, n_samples=50, random_state=0)
-
-    lr = LogisticRegression().fit(X, y)
-
-    if with_sample_weight:
-        rng = np.random.RandomState(42)
-        sample_weight = rng.randint(0, 4, size=X.shape[0])
-    else:
-        sample_weight = None
-
-    disp = plot_precision_recall_curve(lr, X, y, alpha=0.8,
-                                       response_method=response_method,
-                                       sample_weight=sample_weight)
-
-    y_score = getattr(lr, response_method)(X)
-    if response_method == 'predict_proba':
-        y_score = y_score[:, 1]
-
-    prec, recall, _ = precision_recall_curve(y, y_score,
-                                             sample_weight=sample_weight)
-    avg_prec = average_precision_score(y, y_score, sample_weight=sample_weight)
-
-    assert_allclose(disp.precision, prec)
-    assert_allclose(disp.recall, recall)
-    assert disp.average_precision == pytest.approx(avg_prec)
-
-    assert disp.estimator_name == "LogisticRegression"
-
-    # cannot fail thanks to pyplot fixture
-    import matplotlib as mpl  # noqa
-    assert isinstance(disp.line_, mpl.lines.Line2D)
-    assert disp.line_.get_alpha() == 0.8
-    assert isinstance(disp.ax_, mpl.axes.Axes)
-    assert isinstance(disp.figure_, mpl.figure.Figure)
-
-    expected_label = "LogisticRegression (AP = {:0.2f})".format(avg_prec)
-    assert disp.line_.get_label() == expected_label
-    assert disp.ax_.get_xlabel() == "Recall (Positive label: 1)"
-    assert disp.ax_.get_ylabel() == "Precision (Positive label: 1)"
-
-    # draw again with another label
-    disp.plot(name="MySpecialEstimator")
-    expected_label = "MySpecialEstimator (AP = {:0.2f})".format(avg_prec)
-    assert disp.line_.get_label() == expected_label
-
-
-@pytest.mark.parametrize(
-    "clf", [make_pipeline(StandardScaler(), LogisticRegression()),
-            make_pipeline(make_column_transformer((StandardScaler(), [0, 1])),
-                          LogisticRegression())])
-def test_precision_recall_curve_pipeline(pyplot, clf):
-    X, y = make_classification(n_classes=2, n_samples=50, random_state=0)
-    with pytest.raises(NotFittedError):
-        plot_precision_recall_curve(clf, X, y)
-    clf.fit(X, y)
-    disp = plot_precision_recall_curve(clf, X, y)
-    assert disp.estimator_name == clf.__class__.__name__
-
-
-def test_precision_recall_curve_string_labels(pyplot):
-    # regression test #15738
-    cancer = load_breast_cancer()
-    X = cancer.data
-    y = cancer.target_names[cancer.target]
-
-    lr = make_pipeline(StandardScaler(), LogisticRegression())
-    lr.fit(X, y)
-    for klass in cancer.target_names:
-        assert klass in lr.classes_
-    disp = plot_precision_recall_curve(lr, X, y)
-
-    y_pred = lr.predict_proba(X)[:, 1]
-    avg_prec = average_precision_score(y, y_pred,
-                                       pos_label=lr.classes_[1])
-
-    assert disp.average_precision == pytest.approx(avg_prec)
-    assert disp.estimator_name == lr.__class__.__name__
-
-
-def test_plot_precision_recall_curve_estimator_name_multiple_calls(pyplot):
-    # non-regression test checking that the `name` used when calling
-    # `plot_roc_curve` is used as well when calling `disp.plot()`
-    X, y = make_classification(n_classes=2, n_samples=50, random_state=0)
-    clf_name = "my hand-crafted name"
-    clf = LogisticRegression().fit(X, y)
-    disp = plot_precision_recall_curve(clf, X, y, name=clf_name)
-    assert disp.estimator_name == clf_name
-    pyplot.close("all")
-    disp.plot()
-    assert clf_name in disp.line_.get_label()
-    pyplot.close("all")
-    clf_name = "another_name"
-    disp.plot(name=clf_name)
-    assert clf_name in disp.line_.get_label()
-
-
-@pytest.mark.parametrize(
-    "average_precision, estimator_name, expected_label",
-    [
-        (0.9, None, "AP = 0.90"),
-        (None, "my_est", "my_est"),
-        (0.8, "my_est2", "my_est2 (AP = 0.80)"),
-    ]
-)
-def test_default_labels(pyplot, average_precision, estimator_name,
-                        expected_label):
-    prec = np.array([1, 0.5, 0])
-    recall = np.array([0, 0.5, 1])
-    disp = PrecisionRecallDisplay(prec, recall,
-                                  average_precision=average_precision,
-                                  estimator_name=estimator_name)
-    disp.plot()
-    assert disp.line_.get_label() == expected_label
-
-
-@pytest.mark.parametrize(
-    "response_method", ["predict_proba", "decision_function"]
-)
-def test_plot_precision_recall_pos_label(pyplot, response_method):
-    # check that we can provide the positive label and display the proper
-    # statistics
-    X, y = load_breast_cancer(return_X_y=True)
-    # create an highly imbalanced version of the breast cancer dataset
-    idx_positive = np.flatnonzero(y == 1)
-    idx_negative = np.flatnonzero(y == 0)
-    idx_selected = np.hstack([idx_negative, idx_positive[:25]])
-    X, y = X[idx_selected], y[idx_selected]
-    X, y = shuffle(X, y, random_state=42)
-    # only use 2 features to make the problem even harder
-    X = X[:, :2]
-    y = np.array(
-        ["cancer" if c == 1 else "not cancer" for c in y], dtype=object
-    )
-    X_train, X_test, y_train, y_test = train_test_split(
-        X, y, stratify=y, random_state=0,
-    )
-
-    classifier = LogisticRegression()
-    classifier.fit(X_train, y_train)
-
-    # sanity check to be sure the positive class is classes_[0] and that we
-    # are betrayed by the class imbalance
-    assert classifier.classes_.tolist() == ["cancer", "not cancer"]
-
-    disp = plot_precision_recall_curve(
-        classifier, X_test, y_test, pos_label="cancer",
-        response_method=response_method
-    )
-    # we should obtain the statistics of the "cancer" class
-    avg_prec_limit = 0.65
-    assert disp.average_precision < avg_prec_limit
-    assert -np.trapz(disp.precision, disp.recall) < avg_prec_limit
-
-    # otherwise we should obtain the statistics of the "not cancer" class
-    disp = plot_precision_recall_curve(
-        classifier, X_test, y_test, response_method=response_method,
-    )
-    avg_prec_limit = 0.95
-    assert disp.average_precision > avg_prec_limit
-    assert -np.trapz(disp.precision, disp.recall) > avg_prec_limit
diff --git a/sklearn/metrics/_plot/tests/test_plot_roc_curve.py b/sklearn/metrics/_plot/tests/test_plot_roc_curve.py
deleted file mode 100644
index de5a23d81af19..0000000000000
--- a/sklearn/metrics/_plot/tests/test_plot_roc_curve.py
+++ /dev/null
@@ -1,173 +0,0 @@
-import pytest
-import numpy as np
-from numpy.testing import assert_allclose
-
-from sklearn.metrics import plot_roc_curve
-from sklearn.metrics import RocCurveDisplay
-from sklearn.metrics import roc_curve
-from sklearn.metrics import auc
-from sklearn.datasets import load_iris
-from sklearn.datasets import load_breast_cancer
-from sklearn.linear_model import LogisticRegression
-from sklearn.model_selection import train_test_split
-from sklearn.exceptions import NotFittedError
-from sklearn.pipeline import make_pipeline
-from sklearn.preprocessing import StandardScaler
-from sklearn.utils import shuffle
-from sklearn.compose import make_column_transformer
-
-# TODO: Remove when https://github.com/numpy/numpy/issues/14397 is resolved
-pytestmark = pytest.mark.filterwarnings(
-    "ignore:In future, it will be an error for 'np.bool_':DeprecationWarning:"
-    "matplotlib.*")
-
-
-@pytest.fixture(scope="module")
-def data():
-    return load_iris(return_X_y=True)
-
-
-@pytest.fixture(scope="module")
-def data_binary(data):
-    X, y = data
-    return X[y < 2], y[y < 2]
-
-@pytest.mark.parametrize("response_method",
-                         ["predict_proba", "decision_function"])
-@pytest.mark.parametrize("with_sample_weight", [True, False])
-@pytest.mark.parametrize("drop_intermediate", [True, False])
-@pytest.mark.parametrize("with_strings", [True, False])
-def test_plot_roc_curve(pyplot, response_method, data_binary,
-                        with_sample_weight, drop_intermediate,
-                        with_strings):
-    X, y = data_binary
-
-    pos_label = None
-    if with_strings:
-        y = np.array(["c", "b"])[y]
-        pos_label = "c"
-
-    if with_sample_weight:
-        rng = np.random.RandomState(42)
-        sample_weight = rng.randint(1, 4, size=(X.shape[0]))
-    else:
-        sample_weight = None
-
-    lr = LogisticRegression()
-    lr.fit(X, y)
-
-    viz = plot_roc_curve(lr, X, y, alpha=0.8, sample_weight=sample_weight,
-                         drop_intermediate=drop_intermediate)
-
-    y_pred = getattr(lr, response_method)(X)
-    if y_pred.ndim == 2:
-        y_pred = y_pred[:, 1]
-
-    fpr, tpr, _ = roc_curve(y, y_pred, sample_weight=sample_weight,
-                            drop_intermediate=drop_intermediate,
-                            pos_label=pos_label)
-
-    assert_allclose(viz.roc_auc, auc(fpr, tpr))
-    assert_allclose(viz.fpr, fpr)
-    assert_allclose(viz.tpr, tpr)
-
-    assert viz.estimator_name == "LogisticRegression"
-
-    # cannot fail thanks to pyplot fixture
-    import matplotlib as mpl  # noqal
-    assert isinstance(viz.line_, mpl.lines.Line2D)
-    assert viz.line_.get_alpha() == 0.8
-    assert isinstance(viz.ax_, mpl.axes.Axes)
-    assert isinstance(viz.figure_, mpl.figure.Figure)
-
-    expected_label = "LogisticRegression (AUC = {:0.2f})".format(viz.roc_auc)
-    assert viz.line_.get_label() == expected_label
-
-    expected_pos_label = 1 if pos_label is None else pos_label
-    expected_ylabel = f"True Positive Rate (Positive label: " \
-                      f"{expected_pos_label})"
-    expected_xlabel = f"False Positive Rate (Positive label: " \
-                      f"{expected_pos_label})"
-
-    assert viz.ax_.get_ylabel() == expected_ylabel
-    assert viz.ax_.get_xlabel() == expected_xlabel
-
-
-@pytest.mark.parametrize(
-    "clf", [LogisticRegression(),
-            make_pipeline(StandardScaler(), LogisticRegression()),
-            make_pipeline(make_column_transformer((StandardScaler(), [0, 1])),
-                          LogisticRegression())])
-def test_roc_curve_not_fitted_errors(pyplot, data_binary, clf):
-    X, y = data_binary
-    with pytest.raises(NotFittedError):
-        plot_roc_curve(clf, X, y)
-    clf.fit(X, y)
-    disp = plot_roc_curve(clf, X, y)
-    assert clf.__class__.__name__ in disp.line_.get_label()
-    assert disp.estimator_name == clf.__class__.__name__
-
-
-@pytest.mark.parametrize(
-    "roc_auc, estimator_name, expected_label",
-    [
-        (0.9, None, "AUC = 0.90"),
-        (None, "my_est", "my_est"),
-        (0.8, "my_est2", "my_est2 (AUC = 0.80)")
-    ]
-)
-def test_default_labels(pyplot, roc_auc, estimator_name,
-                        expected_label):
-    fpr = np.array([0, 0.5, 1])
-    tpr = np.array([0, 0.5, 1])
-    disp = RocCurveDisplay(fpr=fpr, tpr=tpr, roc_auc=roc_auc,
-                           estimator_name=estimator_name).plot()
-    assert disp.line_.get_label() == expected_label
-
-
-@pytest.mark.parametrize(
-    "response_method", ["predict_proba", "decision_function"]
-)
-def test_plot_roc_curve_pos_label(pyplot, response_method):
-    # check that we can provide the positive label and display the proper
-    # statistics
-    X, y = load_breast_cancer(return_X_y=True)
-    # create an highly imbalanced
-    idx_positive = np.flatnonzero(y == 1)
-    idx_negative = np.flatnonzero(y == 0)
-    idx_selected = np.hstack([idx_negative, idx_positive[:25]])
-    X, y = X[idx_selected], y[idx_selected]
-    X, y = shuffle(X, y, random_state=42)
-    # only use 2 features to make the problem even harder
-    X = X[:, :2]
-    y = np.array(
-        ["cancer" if c == 1 else "not cancer" for c in y], dtype=object
-    )
-    X_train, X_test, y_train, y_test = train_test_split(
-        X, y, stratify=y, random_state=0,
-    )
-
-    classifier = LogisticRegression()
-    classifier.fit(X_train, y_train)
-
-    # sanity check to be sure the positive class is classes_[0] and that we
-    # are betrayed by the class imbalance
-    assert classifier.classes_.tolist() == ["cancer", "not cancer"]
-
-    disp = plot_roc_curve(
-        classifier, X_test, y_test, pos_label="cancer",
-        response_method=response_method
-    )
-
-    roc_auc_limit = 0.95679
-
-    assert disp.roc_auc == pytest.approx(roc_auc_limit)
-    assert np.trapz(disp.tpr, disp.fpr) == pytest.approx(roc_auc_limit)
-
-    disp = plot_roc_curve(
-        classifier, X_test, y_test,
-        response_method=response_method,
-    )
-
-    assert disp.roc_auc == pytest.approx(roc_auc_limit)
-    assert np.trapz(disp.tpr, disp.fpr) == pytest.approx(roc_auc_limit)
diff --git a/sklearn/metrics/_plot/tests/test_precision_recall_display.py b/sklearn/metrics/_plot/tests/test_precision_recall_display.py
new file mode 100644
index 0000000000000..022a5fbf28a91
--- /dev/null
+++ b/sklearn/metrics/_plot/tests/test_precision_recall_display.py
@@ -0,0 +1,382 @@
+from collections import Counter
+
+import numpy as np
+import pytest
+from scipy.integrate import trapezoid
+
+from sklearn.compose import make_column_transformer
+from sklearn.datasets import load_breast_cancer, make_classification
+from sklearn.exceptions import NotFittedError
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import (
+    PrecisionRecallDisplay,
+    average_precision_score,
+    precision_recall_curve,
+)
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import StandardScaler
+from sklearn.utils import shuffle
+
+
+@pytest.mark.parametrize("constructor_name", ["from_estimator", "from_predictions"])
+@pytest.mark.parametrize("response_method", ["predict_proba", "decision_function"])
+@pytest.mark.parametrize("drop_intermediate", [True, False])
+def test_precision_recall_display_plotting(
+    pyplot, constructor_name, response_method, drop_intermediate
+):
+    """Check the overall plotting rendering."""
+    X, y = make_classification(n_classes=2, n_samples=50, random_state=0)
+    pos_label = 1
+
+    classifier = LogisticRegression().fit(X, y)
+    classifier.fit(X, y)
+
+    y_pred = getattr(classifier, response_method)(X)
+    y_pred = y_pred if y_pred.ndim == 1 else y_pred[:, pos_label]
+
+    # safe guard for the binary if/else construction
+    assert constructor_name in ("from_estimator", "from_predictions")
+
+    if constructor_name == "from_estimator":
+        display = PrecisionRecallDisplay.from_estimator(
+            classifier,
+            X,
+            y,
+            response_method=response_method,
+            drop_intermediate=drop_intermediate,
+        )
+    else:
+        display = PrecisionRecallDisplay.from_predictions(
+            y, y_pred, pos_label=pos_label, drop_intermediate=drop_intermediate
+        )
+
+    precision, recall, _ = precision_recall_curve(
+        y, y_pred, pos_label=pos_label, drop_intermediate=drop_intermediate
+    )
+    average_precision = average_precision_score(y, y_pred, pos_label=pos_label)
+
+    np.testing.assert_allclose(display.precision, precision)
+    np.testing.assert_allclose(display.recall, recall)
+    assert display.average_precision == pytest.approx(average_precision)
+
+    import matplotlib as mpl
+
+    assert isinstance(display.line_, mpl.lines.Line2D)
+    assert isinstance(display.ax_, mpl.axes.Axes)
+    assert isinstance(display.figure_, mpl.figure.Figure)
+
+    assert display.ax_.get_xlabel() == "Recall (Positive label: 1)"
+    assert display.ax_.get_ylabel() == "Precision (Positive label: 1)"
+    assert display.ax_.get_adjustable() == "box"
+    assert display.ax_.get_aspect() in ("equal", 1.0)
+    assert display.ax_.get_xlim() == display.ax_.get_ylim() == (-0.01, 1.01)
+
+    # plotting passing some new parameters
+    display.plot(alpha=0.8, name="MySpecialEstimator")
+    expected_label = f"MySpecialEstimator (AP = {average_precision:0.2f})"
+    assert display.line_.get_label() == expected_label
+    assert display.line_.get_alpha() == pytest.approx(0.8)
+
+    # Check that the chance level line is not plotted by default
+    assert display.chance_level_ is None
+
+
+@pytest.mark.parametrize("chance_level_kw", [None, {"color": "r"}, {"c": "r"}])
+@pytest.mark.parametrize("constructor_name", ["from_estimator", "from_predictions"])
+def test_precision_recall_chance_level_line(
+    pyplot,
+    chance_level_kw,
+    constructor_name,
+):
+    """Check the chance level line plotting behavior."""
+    X, y = make_classification(n_classes=2, n_samples=50, random_state=0)
+    pos_prevalence = Counter(y)[1] / len(y)
+
+    lr = LogisticRegression()
+    y_pred = lr.fit(X, y).predict_proba(X)[:, 1]
+
+    if constructor_name == "from_estimator":
+        display = PrecisionRecallDisplay.from_estimator(
+            lr,
+            X,
+            y,
+            plot_chance_level=True,
+            chance_level_kw=chance_level_kw,
+        )
+    else:
+        display = PrecisionRecallDisplay.from_predictions(
+            y,
+            y_pred,
+            plot_chance_level=True,
+            chance_level_kw=chance_level_kw,
+        )
+
+    import matplotlib as mpl
+
+    assert isinstance(display.chance_level_, mpl.lines.Line2D)
+    assert tuple(display.chance_level_.get_xdata()) == (0, 1)
+    assert tuple(display.chance_level_.get_ydata()) == (pos_prevalence, pos_prevalence)
+
+    # Checking for chance level line styles
+    if chance_level_kw is None:
+        assert display.chance_level_.get_color() == "k"
+    else:
+        assert display.chance_level_.get_color() == "r"
+
+
+@pytest.mark.parametrize(
+    "constructor_name, default_label",
+    [
+        ("from_estimator", "LogisticRegression (AP = {:.2f})"),
+        ("from_predictions", "Classifier (AP = {:.2f})"),
+    ],
+)
+def test_precision_recall_display_name(pyplot, constructor_name, default_label):
+    """Check the behaviour of the name parameters"""
+    X, y = make_classification(n_classes=2, n_samples=100, random_state=0)
+    pos_label = 1
+
+    classifier = LogisticRegression().fit(X, y)
+    classifier.fit(X, y)
+
+    y_pred = classifier.predict_proba(X)[:, pos_label]
+
+    # safe guard for the binary if/else construction
+    assert constructor_name in ("from_estimator", "from_predictions")
+
+    if constructor_name == "from_estimator":
+        display = PrecisionRecallDisplay.from_estimator(classifier, X, y)
+    else:
+        display = PrecisionRecallDisplay.from_predictions(
+            y, y_pred, pos_label=pos_label
+        )
+
+    average_precision = average_precision_score(y, y_pred, pos_label=pos_label)
+
+    # check that the default name is used
+    assert display.line_.get_label() == default_label.format(average_precision)
+
+    # check that the name can be set
+    display.plot(name="MySpecialEstimator")
+    assert (
+        display.line_.get_label()
+        == f"MySpecialEstimator (AP = {average_precision:.2f})"
+    )
+
+
+@pytest.mark.parametrize(
+    "clf",
+    [
+        make_pipeline(StandardScaler(), LogisticRegression()),
+        make_pipeline(
+            make_column_transformer((StandardScaler(), [0, 1])), LogisticRegression()
+        ),
+    ],
+)
+def test_precision_recall_display_pipeline(pyplot, clf):
+    X, y = make_classification(n_classes=2, n_samples=50, random_state=0)
+    with pytest.raises(NotFittedError):
+        PrecisionRecallDisplay.from_estimator(clf, X, y)
+    clf.fit(X, y)
+    display = PrecisionRecallDisplay.from_estimator(clf, X, y)
+    assert display.estimator_name == clf.__class__.__name__
+
+
+def test_precision_recall_display_string_labels(pyplot):
+    # regression test #15738
+    cancer = load_breast_cancer()
+    X, y = cancer.data, cancer.target_names[cancer.target]
+
+    lr = make_pipeline(StandardScaler(), LogisticRegression())
+    lr.fit(X, y)
+    for klass in cancer.target_names:
+        assert klass in lr.classes_
+    display = PrecisionRecallDisplay.from_estimator(lr, X, y)
+
+    y_pred = lr.predict_proba(X)[:, 1]
+    avg_prec = average_precision_score(y, y_pred, pos_label=lr.classes_[1])
+
+    assert display.average_precision == pytest.approx(avg_prec)
+    assert display.estimator_name == lr.__class__.__name__
+
+    err_msg = r"y_true takes value in {'benign', 'malignant'}"
+    with pytest.raises(ValueError, match=err_msg):
+        PrecisionRecallDisplay.from_predictions(y, y_pred)
+
+    display = PrecisionRecallDisplay.from_predictions(
+        y, y_pred, pos_label=lr.classes_[1]
+    )
+    assert display.average_precision == pytest.approx(avg_prec)
+
+
+@pytest.mark.parametrize(
+    "average_precision, estimator_name, expected_label",
+    [
+        (0.9, None, "AP = 0.90"),
+        (None, "my_est", "my_est"),
+        (0.8, "my_est2", "my_est2 (AP = 0.80)"),
+    ],
+)
+def test_default_labels(pyplot, average_precision, estimator_name, expected_label):
+    """Check the default labels used in the display."""
+    precision = np.array([1, 0.5, 0])
+    recall = np.array([0, 0.5, 1])
+    display = PrecisionRecallDisplay(
+        precision,
+        recall,
+        average_precision=average_precision,
+        estimator_name=estimator_name,
+    )
+    display.plot()
+    assert display.line_.get_label() == expected_label
+
+
+@pytest.mark.parametrize("constructor_name", ["from_estimator", "from_predictions"])
+@pytest.mark.parametrize("response_method", ["predict_proba", "decision_function"])
+def test_plot_precision_recall_pos_label(pyplot, constructor_name, response_method):
+    # check that we can provide the positive label and display the proper
+    # statistics
+    X, y = load_breast_cancer(return_X_y=True)
+    # create an highly imbalanced version of the breast cancer dataset
+    idx_positive = np.flatnonzero(y == 1)
+    idx_negative = np.flatnonzero(y == 0)
+    idx_selected = np.hstack([idx_negative, idx_positive[:25]])
+    X, y = X[idx_selected], y[idx_selected]
+    X, y = shuffle(X, y, random_state=42)
+    # only use 2 features to make the problem even harder
+    X = X[:, :2]
+    y = np.array(["cancer" if c == 1 else "not cancer" for c in y], dtype=object)
+    X_train, X_test, y_train, y_test = train_test_split(
+        X,
+        y,
+        stratify=y,
+        random_state=0,
+    )
+
+    classifier = LogisticRegression()
+    classifier.fit(X_train, y_train)
+
+    # sanity check to be sure the positive class is classes_[0] and that we
+    # are betrayed by the class imbalance
+    assert classifier.classes_.tolist() == ["cancer", "not cancer"]
+
+    y_pred = getattr(classifier, response_method)(X_test)
+    # we select the corresponding probability columns or reverse the decision
+    #  function otherwise
+    y_pred_cancer = -1 * y_pred if y_pred.ndim == 1 else y_pred[:, 0]
+    y_pred_not_cancer = y_pred if y_pred.ndim == 1 else y_pred[:, 1]
+
+    if constructor_name == "from_estimator":
+        display = PrecisionRecallDisplay.from_estimator(
+            classifier,
+            X_test,
+            y_test,
+            pos_label="cancer",
+            response_method=response_method,
+        )
+    else:
+        display = PrecisionRecallDisplay.from_predictions(
+            y_test,
+            y_pred_cancer,
+            pos_label="cancer",
+        )
+    # we should obtain the statistics of the "cancer" class
+    avg_prec_limit = 0.65
+    assert display.average_precision < avg_prec_limit
+    assert -trapezoid(display.precision, display.recall) < avg_prec_limit
+
+    # otherwise we should obtain the statistics of the "not cancer" class
+    if constructor_name == "from_estimator":
+        display = PrecisionRecallDisplay.from_estimator(
+            classifier,
+            X_test,
+            y_test,
+            response_method=response_method,
+            pos_label="not cancer",
+        )
+    else:
+        display = PrecisionRecallDisplay.from_predictions(
+            y_test,
+            y_pred_not_cancer,
+            pos_label="not cancer",
+        )
+    avg_prec_limit = 0.95
+    assert display.average_precision > avg_prec_limit
+    assert -trapezoid(display.precision, display.recall) > avg_prec_limit
+
+
+@pytest.mark.parametrize("constructor_name", ["from_estimator", "from_predictions"])
+def test_precision_recall_prevalence_pos_label_reusable(pyplot, constructor_name):
+    # Check that even if one passes plot_chance_level=False the first time
+    # one can still call disp.plot with plot_chance_level=True and get the
+    # chance level line
+    X, y = make_classification(n_classes=2, n_samples=50, random_state=0)
+
+    lr = LogisticRegression()
+    y_pred = lr.fit(X, y).predict_proba(X)[:, 1]
+
+    if constructor_name == "from_estimator":
+        display = PrecisionRecallDisplay.from_estimator(
+            lr, X, y, plot_chance_level=False
+        )
+    else:
+        display = PrecisionRecallDisplay.from_predictions(
+            y, y_pred, plot_chance_level=False
+        )
+    assert display.chance_level_ is None
+
+    import matplotlib as mpl
+
+    # When calling from_estimator or from_predictions,
+    # prevalence_pos_label should have been set, so that directly
+    # calling plot_chance_level=True should plot the chance level line
+    display.plot(plot_chance_level=True)
+    assert isinstance(display.chance_level_, mpl.lines.Line2D)
+
+
+def test_precision_recall_raise_no_prevalence(pyplot):
+    # Check that raises correctly when plotting chance level with
+    # no prvelance_pos_label is provided
+    precision = np.array([1, 0.5, 0])
+    recall = np.array([0, 0.5, 1])
+    display = PrecisionRecallDisplay(precision, recall)
+
+    msg = (
+        "You must provide prevalence_pos_label when constructing the "
+        "PrecisionRecallDisplay object in order to plot the chance "
+        "level line. Alternatively, you may use "
+        "PrecisionRecallDisplay.from_estimator or "
+        "PrecisionRecallDisplay.from_predictions "
+        "to automatically set prevalence_pos_label"
+    )
+
+    with pytest.raises(ValueError, match=msg):
+        display.plot(plot_chance_level=True)
+
+
+@pytest.mark.parametrize("despine", [True, False])
+@pytest.mark.parametrize("constructor_name", ["from_estimator", "from_predictions"])
+def test_plot_precision_recall_despine(pyplot, despine, constructor_name):
+    # Check that the despine keyword is working correctly
+    X, y = make_classification(n_classes=2, n_samples=50, random_state=0)
+
+    clf = LogisticRegression().fit(X, y)
+    clf.fit(X, y)
+
+    y_pred = clf.decision_function(X)
+
+    # safe guard for the binary if/else construction
+    assert constructor_name in ("from_estimator", "from_predictions")
+
+    if constructor_name == "from_estimator":
+        display = PrecisionRecallDisplay.from_estimator(clf, X, y, despine=despine)
+    else:
+        display = PrecisionRecallDisplay.from_predictions(y, y_pred, despine=despine)
+
+    for s in ["top", "right"]:
+        assert display.ax_.spines[s].get_visible() is not despine
+
+    if despine:
+        for s in ["bottom", "left"]:
+            assert display.ax_.spines[s].get_bounds() == (0, 1)
diff --git a/sklearn/metrics/_plot/tests/test_predict_error_display.py b/sklearn/metrics/_plot/tests/test_predict_error_display.py
new file mode 100644
index 0000000000000..b2cb888e88849
--- /dev/null
+++ b/sklearn/metrics/_plot/tests/test_predict_error_display.py
@@ -0,0 +1,169 @@
+import pytest
+from numpy.testing import assert_allclose
+
+from sklearn.datasets import load_diabetes
+from sklearn.exceptions import NotFittedError
+from sklearn.linear_model import Ridge
+from sklearn.metrics import PredictionErrorDisplay
+
+X, y = load_diabetes(return_X_y=True)
+
+
+@pytest.fixture
+def regressor_fitted():
+    return Ridge().fit(X, y)
+
+
+@pytest.mark.parametrize(
+    "regressor, params, err_type, err_msg",
+    [
+        (
+            Ridge().fit(X, y),
+            {"subsample": -1},
+            ValueError,
+            "When an integer, subsample=-1 should be",
+        ),
+        (
+            Ridge().fit(X, y),
+            {"subsample": 20.0},
+            ValueError,
+            "When a floating-point, subsample=20.0 should be",
+        ),
+        (
+            Ridge().fit(X, y),
+            {"subsample": -20.0},
+            ValueError,
+            "When a floating-point, subsample=-20.0 should be",
+        ),
+        (
+            Ridge().fit(X, y),
+            {"kind": "xxx"},
+            ValueError,
+            "`kind` must be one of",
+        ),
+    ],
+)
+@pytest.mark.parametrize("class_method", ["from_estimator", "from_predictions"])
+def test_prediction_error_display_raise_error(
+    pyplot, class_method, regressor, params, err_type, err_msg
+):
+    """Check that we raise the proper error when making the parameters
+    # validation."""
+    with pytest.raises(err_type, match=err_msg):
+        if class_method == "from_estimator":
+            PredictionErrorDisplay.from_estimator(regressor, X, y, **params)
+        else:
+            y_pred = regressor.predict(X)
+            PredictionErrorDisplay.from_predictions(y_true=y, y_pred=y_pred, **params)
+
+
+def test_from_estimator_not_fitted(pyplot):
+    """Check that we raise a `NotFittedError` when the passed regressor is not
+    fit."""
+    regressor = Ridge()
+    with pytest.raises(NotFittedError, match="is not fitted yet."):
+        PredictionErrorDisplay.from_estimator(regressor, X, y)
+
+
+@pytest.mark.parametrize("class_method", ["from_estimator", "from_predictions"])
+@pytest.mark.parametrize("kind", ["actual_vs_predicted", "residual_vs_predicted"])
+def test_prediction_error_display(pyplot, regressor_fitted, class_method, kind):
+    """Check the default behaviour of the display."""
+    if class_method == "from_estimator":
+        display = PredictionErrorDisplay.from_estimator(
+            regressor_fitted, X, y, kind=kind
+        )
+    else:
+        y_pred = regressor_fitted.predict(X)
+        display = PredictionErrorDisplay.from_predictions(
+            y_true=y, y_pred=y_pred, kind=kind
+        )
+
+    if kind == "actual_vs_predicted":
+        assert_allclose(display.line_.get_xdata(), display.line_.get_ydata())
+        assert display.ax_.get_xlabel() == "Predicted values"
+        assert display.ax_.get_ylabel() == "Actual values"
+        assert display.line_ is not None
+    else:
+        assert display.ax_.get_xlabel() == "Predicted values"
+        assert display.ax_.get_ylabel() == "Residuals (actual - predicted)"
+        assert display.line_ is not None
+
+    assert display.ax_.get_legend() is None
+
+
+@pytest.mark.parametrize("class_method", ["from_estimator", "from_predictions"])
+@pytest.mark.parametrize(
+    "subsample, expected_size",
+    [(5, 5), (0.1, int(X.shape[0] * 0.1)), (None, X.shape[0])],
+)
+def test_plot_prediction_error_subsample(
+    pyplot, regressor_fitted, class_method, subsample, expected_size
+):
+    """Check the behaviour of `subsample`."""
+    if class_method == "from_estimator":
+        display = PredictionErrorDisplay.from_estimator(
+            regressor_fitted, X, y, subsample=subsample
+        )
+    else:
+        y_pred = regressor_fitted.predict(X)
+        display = PredictionErrorDisplay.from_predictions(
+            y_true=y, y_pred=y_pred, subsample=subsample
+        )
+    assert len(display.scatter_.get_offsets()) == expected_size
+
+
+@pytest.mark.parametrize("class_method", ["from_estimator", "from_predictions"])
+def test_plot_prediction_error_ax(pyplot, regressor_fitted, class_method):
+    """Check that we can pass an axis to the display."""
+    _, ax = pyplot.subplots()
+    if class_method == "from_estimator":
+        display = PredictionErrorDisplay.from_estimator(regressor_fitted, X, y, ax=ax)
+    else:
+        y_pred = regressor_fitted.predict(X)
+        display = PredictionErrorDisplay.from_predictions(
+            y_true=y, y_pred=y_pred, ax=ax
+        )
+    assert display.ax_ is ax
+
+
+@pytest.mark.parametrize("class_method", ["from_estimator", "from_predictions"])
+@pytest.mark.parametrize(
+    "scatter_kwargs",
+    [None, {"color": "blue", "alpha": 0.9}, {"c": "blue", "alpha": 0.9}],
+)
+@pytest.mark.parametrize(
+    "line_kwargs", [None, {"color": "red", "linestyle": "-"}, {"c": "red", "ls": "-"}]
+)
+def test_prediction_error_custom_artist(
+    pyplot, regressor_fitted, class_method, scatter_kwargs, line_kwargs
+):
+    """Check that we can tune the style of the line and the scatter."""
+    extra_params = {
+        "kind": "actual_vs_predicted",
+        "scatter_kwargs": scatter_kwargs,
+        "line_kwargs": line_kwargs,
+    }
+    if class_method == "from_estimator":
+        display = PredictionErrorDisplay.from_estimator(
+            regressor_fitted, X, y, **extra_params
+        )
+    else:
+        y_pred = regressor_fitted.predict(X)
+        display = PredictionErrorDisplay.from_predictions(
+            y_true=y, y_pred=y_pred, **extra_params
+        )
+
+    if line_kwargs is not None:
+        assert display.line_.get_linestyle() == "-"
+        assert display.line_.get_color() == "red"
+    else:
+        assert display.line_.get_linestyle() == "--"
+        assert display.line_.get_color() == "black"
+        assert display.line_.get_alpha() == 0.7
+
+    if scatter_kwargs is not None:
+        assert_allclose(display.scatter_.get_facecolor(), [[0.0, 0.0, 1.0, 0.9]])
+        assert_allclose(display.scatter_.get_edgecolor(), [[0.0, 0.0, 1.0, 0.9]])
+    else:
+        assert display.scatter_.get_alpha() == 0.8
diff --git a/sklearn/metrics/_plot/tests/test_roc_curve_display.py b/sklearn/metrics/_plot/tests/test_roc_curve_display.py
new file mode 100644
index 0000000000000..23fa2f2e3a5e6
--- /dev/null
+++ b/sklearn/metrics/_plot/tests/test_roc_curve_display.py
@@ -0,0 +1,987 @@
+from collections.abc import Mapping
+
+import numpy as np
+import pytest
+from numpy.testing import assert_allclose
+from scipy.integrate import trapezoid
+
+from sklearn import clone
+from sklearn.compose import make_column_transformer
+from sklearn.datasets import load_breast_cancer, make_classification
+from sklearn.exceptions import NotFittedError
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import RocCurveDisplay, auc, roc_curve
+from sklearn.model_selection import cross_validate, train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import StandardScaler
+from sklearn.utils import _safe_indexing, shuffle
+from sklearn.utils._response import _get_response_values_binary
+
+
+@pytest.fixture(scope="module")
+def data_binary():
+    X, y = make_classification(
+        n_samples=200,
+        n_features=20,
+        n_informative=5,
+        n_redundant=2,
+        flip_y=0.1,
+        class_sep=0.8,
+        random_state=42,
+    )
+    return X, y
+
+
+def _check_figure_axes_and_labels(display, pos_label):
+    """Check mpl axes and figure defaults are correct."""
+    import matplotlib as mpl
+
+    assert isinstance(display.ax_, mpl.axes.Axes)
+    assert isinstance(display.figure_, mpl.figure.Figure)
+    assert display.ax_.get_adjustable() == "box"
+    assert display.ax_.get_aspect() in ("equal", 1.0)
+    assert display.ax_.get_xlim() == display.ax_.get_ylim() == (-0.01, 1.01)
+
+    expected_pos_label = 1 if pos_label is None else pos_label
+    expected_ylabel = f"True Positive Rate (Positive label: {expected_pos_label})"
+    expected_xlabel = f"False Positive Rate (Positive label: {expected_pos_label})"
+
+    assert display.ax_.get_ylabel() == expected_ylabel
+    assert display.ax_.get_xlabel() == expected_xlabel
+
+
+@pytest.mark.parametrize("response_method", ["predict_proba", "decision_function"])
+@pytest.mark.parametrize("with_sample_weight", [True, False])
+@pytest.mark.parametrize("drop_intermediate", [True, False])
+@pytest.mark.parametrize("with_strings", [True, False])
+@pytest.mark.parametrize(
+    "constructor_name, default_name",
+    [
+        ("from_estimator", "LogisticRegression"),
+        ("from_predictions", "Classifier"),
+    ],
+)
+def test_roc_curve_display_plotting(
+    pyplot,
+    response_method,
+    data_binary,
+    with_sample_weight,
+    drop_intermediate,
+    with_strings,
+    constructor_name,
+    default_name,
+):
+    """Check the overall plotting behaviour for single curve."""
+    X, y = data_binary
+
+    pos_label = None
+    if with_strings:
+        y = np.array(["c", "b"])[y]
+        pos_label = "c"
+
+    if with_sample_weight:
+        rng = np.random.RandomState(42)
+        sample_weight = rng.randint(1, 4, size=(X.shape[0]))
+    else:
+        sample_weight = None
+
+    lr = LogisticRegression()
+    lr.fit(X, y)
+
+    y_score = getattr(lr, response_method)(X)
+    y_score = y_score if y_score.ndim == 1 else y_score[:, 1]
+
+    if constructor_name == "from_estimator":
+        display = RocCurveDisplay.from_estimator(
+            lr,
+            X,
+            y,
+            sample_weight=sample_weight,
+            drop_intermediate=drop_intermediate,
+            pos_label=pos_label,
+            curve_kwargs={"alpha": 0.8},
+        )
+    else:
+        display = RocCurveDisplay.from_predictions(
+            y,
+            y_score,
+            sample_weight=sample_weight,
+            drop_intermediate=drop_intermediate,
+            pos_label=pos_label,
+            curve_kwargs={"alpha": 0.8},
+        )
+
+    fpr, tpr, _ = roc_curve(
+        y,
+        y_score,
+        sample_weight=sample_weight,
+        drop_intermediate=drop_intermediate,
+        pos_label=pos_label,
+    )
+
+    assert_allclose(display.roc_auc, auc(fpr, tpr))
+    assert_allclose(display.fpr, fpr)
+    assert_allclose(display.tpr, tpr)
+
+    assert display.name == default_name
+
+    import matplotlib as mpl
+
+    _check_figure_axes_and_labels(display, pos_label)
+    assert isinstance(display.line_, mpl.lines.Line2D)
+    assert display.line_.get_alpha() == 0.8
+
+    expected_label = f"{default_name} (AUC = {display.roc_auc:.2f})"
+    assert display.line_.get_label() == expected_label
+
+
+@pytest.mark.parametrize(
+    "params, err_msg",
+    [
+        (
+            {
+                "fpr": [np.array([0, 0.5, 1]), np.array([0, 0.5, 1])],
+                "tpr": [np.array([0, 0.5, 1])],
+                "roc_auc": None,
+                "name": None,
+            },
+            "self.fpr and self.tpr from `RocCurveDisplay` initialization,",
+        ),
+        (
+            {
+                "fpr": [np.array([0, 0.5, 1])],
+                "tpr": [np.array([0, 0.5, 1]), np.array([0, 0.5, 1])],
+                "roc_auc": [0.8, 0.9],
+                "name": None,
+            },
+            "self.fpr, self.tpr and self.roc_auc from `RocCurveDisplay`",
+        ),
+        (
+            {
+                "fpr": [np.array([0, 0.5, 1]), np.array([0, 0.5, 1])],
+                "tpr": [np.array([0, 0.5, 1]), np.array([0, 0.5, 1])],
+                "roc_auc": [0.8],
+                "name": None,
+            },
+            "Got: self.fpr: 2, self.tpr: 2, self.roc_auc: 1",
+        ),
+        (
+            {
+                "fpr": [np.array([0, 0.5, 1]), np.array([0, 0.5, 1])],
+                "tpr": [np.array([0, 0.5, 1]), np.array([0, 0.5, 1])],
+                "roc_auc": [0.8, 0.9],
+                "name": ["curve1", "curve2", "curve3"],
+            },
+            r"self.fpr, self.tpr, self.roc_auc and 'name' \(or self.name\)",
+        ),
+        (
+            {
+                "fpr": [np.array([0, 0.5, 1]), np.array([0, 0.5, 1])],
+                "tpr": [np.array([0, 0.5, 1]), np.array([0, 0.5, 1])],
+                "roc_auc": [0.8, 0.9],
+                # List of length 1 is always allowed
+                "name": ["curve1"],
+            },
+            None,
+        ),
+    ],
+)
+def test_roc_curve_plot_parameter_length_validation(pyplot, params, err_msg):
+    """Check `plot` parameter length validation performed correctly."""
+    display = RocCurveDisplay(**params)
+    if err_msg:
+        with pytest.raises(ValueError, match=err_msg):
+            display.plot()
+    else:
+        # No error should be raised
+        display.plot()
+
+
+def test_validate_plot_params(pyplot):
+    """Check `_validate_plot_params` returns the correct variables."""
+    fpr = np.array([0, 0.5, 1])
+    tpr = [np.array([0, 0.5, 1])]
+    roc_auc = None
+    name = "test_curve"
+
+    # Initialize display with test inputs
+    display = RocCurveDisplay(
+        fpr=fpr,
+        tpr=tpr,
+        roc_auc=roc_auc,
+        name=name,
+        pos_label=None,
+    )
+    fpr_out, tpr_out, roc_auc_out, name_out = display._validate_plot_params(
+        ax=None, name=None
+    )
+
+    assert isinstance(fpr_out, list)
+    assert isinstance(tpr_out, list)
+    assert len(fpr_out) == 1
+    assert len(tpr_out) == 1
+    assert roc_auc_out is None
+    assert name_out == ["test_curve"]
+
+
+def test_roc_curve_from_cv_results_param_validation(pyplot, data_binary):
+    """Check parameter validation is correct."""
+    X, y = data_binary
+
+    # `cv_results` missing key
+    cv_results_no_est = cross_validate(
+        LogisticRegression(), X, y, cv=3, return_estimator=True, return_indices=False
+    )
+    cv_results_no_indices = cross_validate(
+        LogisticRegression(), X, y, cv=3, return_estimator=True, return_indices=False
+    )
+    for cv_results in (cv_results_no_est, cv_results_no_indices):
+        with pytest.raises(
+            ValueError,
+            match="`cv_results` does not contain one of the following required",
+        ):
+            RocCurveDisplay.from_cv_results(cv_results, X, y)
+
+    cv_results = cross_validate(
+        LogisticRegression(), X, y, cv=3, return_estimator=True, return_indices=True
+    )
+
+    # `X` wrong length
+    with pytest.raises(ValueError, match="`X` does not contain the correct"):
+        RocCurveDisplay.from_cv_results(cv_results, X[:10, :], y)
+
+    # `y` not binary
+    y_multi = y.copy()
+    y_multi[0] = 2
+    with pytest.raises(ValueError, match="The target `y` is not binary."):
+        RocCurveDisplay.from_cv_results(cv_results, X, y_multi)
+
+    # input inconsistent length
+    with pytest.raises(ValueError, match="Found input variables with inconsistent"):
+        RocCurveDisplay.from_cv_results(cv_results, X, y[:10])
+    with pytest.raises(ValueError, match="Found input variables with inconsistent"):
+        RocCurveDisplay.from_cv_results(cv_results, X, y, sample_weight=[1, 2])
+
+    # `pos_label` inconsistency
+    y_multi[y_multi == 1] = 2
+    with pytest.raises(ValueError, match=r"y takes value in \{0, 2\}"):
+        RocCurveDisplay.from_cv_results(cv_results, X, y_multi)
+
+    # `name` is list while `curve_kwargs` is None or dict
+    for curve_kwargs in (None, {"alpha": 0.2}):
+        with pytest.raises(ValueError, match="To avoid labeling individual curves"):
+            RocCurveDisplay.from_cv_results(
+                cv_results,
+                X,
+                y,
+                name=["one", "two", "three"],
+                curve_kwargs=curve_kwargs,
+            )
+
+    # `curve_kwargs` incorrect length
+    with pytest.raises(ValueError, match="`curve_kwargs` must be None, a dictionary"):
+        RocCurveDisplay.from_cv_results(cv_results, X, y, curve_kwargs=[{"alpha": 1}])
+
+    # `curve_kwargs` both alias provided
+    with pytest.raises(TypeError, match="Got both c and"):
+        RocCurveDisplay.from_cv_results(
+            cv_results, X, y, curve_kwargs={"c": "blue", "color": "red"}
+        )
+
+
+@pytest.mark.parametrize(
+    "curve_kwargs",
+    [None, {"alpha": 0.2}, [{"alpha": 0.2}, {"alpha": 0.3}, {"alpha": 0.4}]],
+)
+def test_roc_curve_display_from_cv_results_curve_kwargs(
+    pyplot, data_binary, curve_kwargs
+):
+    """Check `curve_kwargs` correctly passed."""
+    X, y = data_binary
+    n_cv = 3
+    cv_results = cross_validate(
+        LogisticRegression(), X, y, cv=n_cv, return_estimator=True, return_indices=True
+    )
+    display = RocCurveDisplay.from_cv_results(
+        cv_results,
+        X,
+        y,
+        curve_kwargs=curve_kwargs,
+    )
+    if curve_kwargs is None:
+        # Default `alpha` used
+        assert all(line.get_alpha() == 0.5 for line in display.line_)
+    elif isinstance(curve_kwargs, Mapping):
+        # `alpha` from dict used for all curves
+        assert all(line.get_alpha() == 0.2 for line in display.line_)
+    else:
+        # Different `alpha` used for each curve
+        assert all(
+            line.get_alpha() == curve_kwargs[i]["alpha"]
+            for i, line in enumerate(display.line_)
+        )
+
+
+# TODO(1.9): Remove in 1.9
+def test_roc_curve_display_estimator_name_deprecation(pyplot):
+    """Check deprecation of `estimator_name`."""
+    fpr = np.array([0, 0.5, 1])
+    tpr = np.array([0, 0.5, 1])
+    with pytest.warns(FutureWarning, match="`estimator_name` is deprecated in"):
+        RocCurveDisplay(fpr=fpr, tpr=tpr, estimator_name="test")
+
+
+# TODO(1.9): Remove in 1.9
+@pytest.mark.parametrize(
+    "constructor_name", ["from_estimator", "from_predictions", "plot"]
+)
+def test_roc_curve_display_kwargs_deprecation(pyplot, data_binary, constructor_name):
+    """Check **kwargs deprecated correctly in favour of `curve_kwargs`."""
+    X, y = data_binary
+    lr = LogisticRegression()
+    lr.fit(X, y)
+    fpr = np.array([0, 0.5, 1])
+    tpr = np.array([0, 0.5, 1])
+
+    # Error when both `curve_kwargs` and `**kwargs` provided
+    with pytest.raises(ValueError, match="Cannot provide both `curve_kwargs`"):
+        if constructor_name == "from_estimator":
+            RocCurveDisplay.from_estimator(
+                lr, X, y, curve_kwargs={"alpha": 1}, label="test"
+            )
+        elif constructor_name == "from_predictions":
+            RocCurveDisplay.from_predictions(
+                y, y, curve_kwargs={"alpha": 1}, label="test"
+            )
+        else:
+            RocCurveDisplay(fpr=fpr, tpr=tpr).plot(
+                curve_kwargs={"alpha": 1}, label="test"
+            )
+
+    # Warning when `**kwargs`` provided
+    with pytest.warns(FutureWarning, match=r"`\*\*kwargs` is deprecated and will be"):
+        if constructor_name == "from_estimator":
+            RocCurveDisplay.from_estimator(lr, X, y, label="test")
+        elif constructor_name == "from_predictions":
+            RocCurveDisplay.from_predictions(y, y, label="test")
+        else:
+            RocCurveDisplay(fpr=fpr, tpr=tpr).plot(label="test")
+
+
+@pytest.mark.parametrize(
+    "curve_kwargs",
+    [
+        None,
+        {"color": "blue"},
+        [{"color": "blue"}, {"color": "green"}, {"color": "red"}],
+    ],
+)
+@pytest.mark.parametrize("drop_intermediate", [True, False])
+@pytest.mark.parametrize("response_method", ["predict_proba", "decision_function"])
+@pytest.mark.parametrize("with_sample_weight", [True, False])
+@pytest.mark.parametrize("with_strings", [True, False])
+def test_roc_curve_display_plotting_from_cv_results(
+    pyplot,
+    data_binary,
+    with_strings,
+    with_sample_weight,
+    response_method,
+    drop_intermediate,
+    curve_kwargs,
+):
+    """Check overall plotting of `from_cv_results`."""
+    X, y = data_binary
+
+    pos_label = None
+    if with_strings:
+        y = np.array(["c", "b"])[y]
+        pos_label = "c"
+
+    if with_sample_weight:
+        rng = np.random.RandomState(42)
+        sample_weight = rng.randint(1, 4, size=(X.shape[0]))
+    else:
+        sample_weight = None
+
+    cv_results = cross_validate(
+        LogisticRegression(), X, y, cv=3, return_estimator=True, return_indices=True
+    )
+    display = RocCurveDisplay.from_cv_results(
+        cv_results,
+        X,
+        y,
+        sample_weight=sample_weight,
+        drop_intermediate=drop_intermediate,
+        response_method=response_method,
+        pos_label=pos_label,
+        curve_kwargs=curve_kwargs,
+    )
+
+    for idx, (estimator, test_indices) in enumerate(
+        zip(cv_results["estimator"], cv_results["indices"]["test"])
+    ):
+        y_true = _safe_indexing(y, test_indices)
+        y_pred = _get_response_values_binary(
+            estimator,
+            _safe_indexing(X, test_indices),
+            response_method=response_method,
+            pos_label=pos_label,
+        )[0]
+        sample_weight_fold = (
+            None
+            if sample_weight is None
+            else _safe_indexing(sample_weight, test_indices)
+        )
+        fpr, tpr, _ = roc_curve(
+            y_true,
+            y_pred,
+            sample_weight=sample_weight_fold,
+            drop_intermediate=drop_intermediate,
+            pos_label=pos_label,
+        )
+        assert_allclose(display.roc_auc[idx], auc(fpr, tpr))
+        assert_allclose(display.fpr[idx], fpr)
+        assert_allclose(display.tpr[idx], tpr)
+
+    assert display.name is None
+
+    import matplotlib as mpl
+
+    _check_figure_axes_and_labels(display, pos_label)
+    if with_sample_weight:
+        aggregate_expected_labels = ["AUC = 0.64 +/- 0.04", "_child1", "_child2"]
+    else:
+        aggregate_expected_labels = ["AUC = 0.61 +/- 0.05", "_child1", "_child2"]
+    for idx, line in enumerate(display.line_):
+        assert isinstance(line, mpl.lines.Line2D)
+        # Default alpha for `from_cv_results`
+        line.get_alpha() == 0.5
+        if isinstance(curve_kwargs, list):
+            # Each individual curve labelled
+            assert line.get_label() == f"AUC = {display.roc_auc[idx]:.2f}"
+        else:
+            # Single aggregate label
+            assert line.get_label() == aggregate_expected_labels[idx]
+
+
+@pytest.mark.parametrize("roc_auc", [[1.0, 1.0, 1.0], None])
+@pytest.mark.parametrize(
+    "curve_kwargs",
+    [None, {"color": "red"}, [{"c": "red"}, {"c": "green"}, {"c": "yellow"}]],
+)
+@pytest.mark.parametrize("name", [None, "single", ["one", "two", "three"]])
+def test_roc_curve_plot_legend_label(pyplot, data_binary, name, curve_kwargs, roc_auc):
+    """Check legend label correct with all `curve_kwargs`, `name` combinations."""
+    fpr = [np.array([0, 0.5, 1]), np.array([0, 0.5, 1]), np.array([0, 0.5, 1])]
+    tpr = [np.array([0, 0.5, 1]), np.array([0, 0.5, 1]), np.array([0, 0.5, 1])]
+    if not isinstance(curve_kwargs, list) and isinstance(name, list):
+        with pytest.raises(ValueError, match="To avoid labeling individual curves"):
+            RocCurveDisplay(fpr=fpr, tpr=tpr, roc_auc=roc_auc).plot(
+                name=name, curve_kwargs=curve_kwargs
+            )
+
+    else:
+        display = RocCurveDisplay(fpr=fpr, tpr=tpr, roc_auc=roc_auc).plot(
+            name=name, curve_kwargs=curve_kwargs
+        )
+        legend = display.ax_.get_legend()
+        if legend is None:
+            # No legend is created, exit test early
+            assert name is None
+            assert roc_auc is None
+            return
+        else:
+            legend_labels = [text.get_text() for text in legend.get_texts()]
+
+        if isinstance(curve_kwargs, list):
+            # Multiple labels in legend
+            assert len(legend_labels) == 3
+            for idx, label in enumerate(legend_labels):
+                if name is None:
+                    expected_label = "AUC = 1.00" if roc_auc else None
+                    assert label == expected_label
+                elif isinstance(name, str):
+                    expected_label = "single (AUC = 1.00)" if roc_auc else "single"
+                    assert label == expected_label
+                else:
+                    # `name` is a list of different strings
+                    expected_label = (
+                        f"{name[idx]} (AUC = 1.00)" if roc_auc else f"{name[idx]}"
+                    )
+                    assert label == expected_label
+        else:
+            # Single label in legend
+            assert len(legend_labels) == 1
+            if name is None:
+                expected_label = "AUC = 1.00 +/- 0.00" if roc_auc else None
+                assert legend_labels[0] == expected_label
+            else:
+                # name is single string
+                expected_label = "single (AUC = 1.00 +/- 0.00)" if roc_auc else "single"
+                assert legend_labels[0] == expected_label
+
+
+@pytest.mark.parametrize(
+    "curve_kwargs",
+    [None, {"color": "red"}, [{"c": "red"}, {"c": "green"}, {"c": "yellow"}]],
+)
+@pytest.mark.parametrize("name", [None, "single", ["one", "two", "three"]])
+def test_roc_curve_from_cv_results_legend_label(
+    pyplot, data_binary, name, curve_kwargs
+):
+    """Check legend label correct with all `curve_kwargs`, `name` combinations."""
+    X, y = data_binary
+    n_cv = 3
+    cv_results = cross_validate(
+        LogisticRegression(), X, y, cv=n_cv, return_estimator=True, return_indices=True
+    )
+
+    if not isinstance(curve_kwargs, list) and isinstance(name, list):
+        with pytest.raises(ValueError, match="To avoid labeling individual curves"):
+            RocCurveDisplay.from_cv_results(
+                cv_results, X, y, name=name, curve_kwargs=curve_kwargs
+            )
+    else:
+        display = RocCurveDisplay.from_cv_results(
+            cv_results, X, y, name=name, curve_kwargs=curve_kwargs
+        )
+
+        legend = display.ax_.get_legend()
+        legend_labels = [text.get_text() for text in legend.get_texts()]
+        if isinstance(curve_kwargs, list):
+            # Multiple labels in legend
+            assert len(legend_labels) == 3
+            auc = ["0.62", "0.66", "0.55"]
+            for idx, label in enumerate(legend_labels):
+                if name is None:
+                    assert label == f"AUC = {auc[idx]}"
+                elif isinstance(name, str):
+                    assert label == f"single (AUC = {auc[idx]})"
+                else:
+                    # `name` is a list of different strings
+                    assert label == f"{name[idx]} (AUC = {auc[idx]})"
+        else:
+            # Single label in legend
+            assert len(legend_labels) == 1
+            if name is None:
+                assert legend_labels[0] == "AUC = 0.61 +/- 0.05"
+            else:
+                # name is single string
+                assert legend_labels[0] == "single (AUC = 0.61 +/- 0.05)"
+
+
+@pytest.mark.parametrize(
+    "curve_kwargs",
+    [None, {"color": "red"}, [{"c": "red"}, {"c": "green"}, {"c": "yellow"}]],
+)
+def test_roc_curve_from_cv_results_curve_kwargs(pyplot, data_binary, curve_kwargs):
+    """Check line kwargs passed correctly in `from_cv_results`."""
+
+    X, y = data_binary
+    cv_results = cross_validate(
+        LogisticRegression(), X, y, cv=3, return_estimator=True, return_indices=True
+    )
+    display = RocCurveDisplay.from_cv_results(
+        cv_results, X, y, curve_kwargs=curve_kwargs
+    )
+
+    for idx, line in enumerate(display.line_):
+        color = line.get_color()
+        if curve_kwargs is None:
+            # Default color
+            assert color == "blue"
+        elif isinstance(curve_kwargs, Mapping):
+            # All curves "red"
+            assert color == "red"
+        else:
+            assert color == curve_kwargs[idx]["c"]
+
+
+def _check_chance_level(plot_chance_level, chance_level_kw, display):
+    """Check chance level line and line styles correct."""
+    import matplotlib as mpl
+
+    if plot_chance_level:
+        assert isinstance(display.chance_level_, mpl.lines.Line2D)
+        assert tuple(display.chance_level_.get_xdata()) == (0, 1)
+        assert tuple(display.chance_level_.get_ydata()) == (0, 1)
+    else:
+        assert display.chance_level_ is None
+
+    # Checking for chance level line styles
+    if plot_chance_level and chance_level_kw is None:
+        assert display.chance_level_.get_color() == "k"
+        assert display.chance_level_.get_linestyle() == "--"
+        assert display.chance_level_.get_label() == "Chance level (AUC = 0.5)"
+    elif plot_chance_level:
+        if "c" in chance_level_kw:
+            assert display.chance_level_.get_color() == chance_level_kw["c"]
+        else:
+            assert display.chance_level_.get_color() == chance_level_kw["color"]
+        if "lw" in chance_level_kw:
+            assert display.chance_level_.get_linewidth() == chance_level_kw["lw"]
+        else:
+            assert display.chance_level_.get_linewidth() == chance_level_kw["linewidth"]
+        if "ls" in chance_level_kw:
+            assert display.chance_level_.get_linestyle() == chance_level_kw["ls"]
+        else:
+            assert display.chance_level_.get_linestyle() == chance_level_kw["linestyle"]
+
+
+@pytest.mark.parametrize("plot_chance_level", [True, False])
+@pytest.mark.parametrize("label", [None, "Test Label"])
+@pytest.mark.parametrize(
+    "chance_level_kw",
+    [
+        None,
+        {"linewidth": 1, "color": "red", "linestyle": "-", "label": "DummyEstimator"},
+        {"lw": 1, "c": "red", "ls": "-", "label": "DummyEstimator"},
+        {"lw": 1, "color": "blue", "ls": "-", "label": None},
+    ],
+)
+@pytest.mark.parametrize("constructor_name", ["from_estimator", "from_predictions"])
+def test_roc_curve_chance_level_line(
+    pyplot,
+    data_binary,
+    plot_chance_level,
+    chance_level_kw,
+    label,
+    constructor_name,
+):
+    """Check chance level plotting behavior of `from_predictions`, `from_estimator`."""
+    X, y = data_binary
+
+    lr = LogisticRegression()
+    lr.fit(X, y)
+
+    y_score = getattr(lr, "predict_proba")(X)
+    y_score = y_score if y_score.ndim == 1 else y_score[:, 1]
+
+    if constructor_name == "from_estimator":
+        display = RocCurveDisplay.from_estimator(
+            lr,
+            X,
+            y,
+            curve_kwargs={"alpha": 0.8, "label": label},
+            plot_chance_level=plot_chance_level,
+            chance_level_kw=chance_level_kw,
+        )
+    else:
+        display = RocCurveDisplay.from_predictions(
+            y,
+            y_score,
+            curve_kwargs={"alpha": 0.8, "label": label},
+            plot_chance_level=plot_chance_level,
+            chance_level_kw=chance_level_kw,
+        )
+
+    import matplotlib as mpl
+
+    assert isinstance(display.line_, mpl.lines.Line2D)
+    assert display.line_.get_alpha() == 0.8
+    assert isinstance(display.ax_, mpl.axes.Axes)
+    assert isinstance(display.figure_, mpl.figure.Figure)
+
+    _check_chance_level(plot_chance_level, chance_level_kw, display)
+
+    # Checking for legend behaviour
+    if plot_chance_level and chance_level_kw is not None:
+        if label is not None or chance_level_kw.get("label") is not None:
+            legend = display.ax_.get_legend()
+            assert legend is not None  #  Legend should be present if any label is set
+            legend_labels = [text.get_text() for text in legend.get_texts()]
+            if label is not None:
+                assert label in legend_labels
+            if chance_level_kw.get("label") is not None:
+                assert chance_level_kw["label"] in legend_labels
+        else:
+            assert display.ax_.get_legend() is None
+
+
+@pytest.mark.parametrize("plot_chance_level", [True, False])
+@pytest.mark.parametrize(
+    "chance_level_kw",
+    [
+        None,
+        {"linewidth": 1, "color": "red", "linestyle": "-", "label": "DummyEstimator"},
+        {"lw": 1, "c": "red", "ls": "-", "label": "DummyEstimator"},
+        {"lw": 1, "color": "blue", "ls": "-", "label": None},
+    ],
+)
+@pytest.mark.parametrize("curve_kwargs", [None, {"alpha": 0.8}])
+def test_roc_curve_chance_level_line_from_cv_results(
+    pyplot,
+    data_binary,
+    plot_chance_level,
+    chance_level_kw,
+    curve_kwargs,
+):
+    """Check chance level plotting behavior with `from_cv_results`."""
+    X, y = data_binary
+    n_cv = 3
+    cv_results = cross_validate(
+        LogisticRegression(), X, y, cv=n_cv, return_estimator=True, return_indices=True
+    )
+
+    display = RocCurveDisplay.from_cv_results(
+        cv_results,
+        X,
+        y,
+        plot_chance_level=plot_chance_level,
+        chance_level_kwargs=chance_level_kw,
+        curve_kwargs=curve_kwargs,
+    )
+
+    import matplotlib as mpl
+
+    assert all(isinstance(line, mpl.lines.Line2D) for line in display.line_)
+    # Ensure both curve line kwargs passed correctly as well
+    if curve_kwargs:
+        assert all(line.get_alpha() == 0.8 for line in display.line_)
+    assert isinstance(display.ax_, mpl.axes.Axes)
+    assert isinstance(display.figure_, mpl.figure.Figure)
+
+    _check_chance_level(plot_chance_level, chance_level_kw, display)
+
+    legend = display.ax_.get_legend()
+    # There is always a legend, to indicate each 'Fold' curve
+    assert legend is not None
+    legend_labels = [text.get_text() for text in legend.get_texts()]
+    if plot_chance_level and chance_level_kw is not None:
+        if chance_level_kw.get("label") is not None:
+            assert chance_level_kw["label"] in legend_labels
+        else:
+            assert len(legend_labels) == 1
+
+
+@pytest.mark.parametrize(
+    "clf",
+    [
+        LogisticRegression(),
+        make_pipeline(StandardScaler(), LogisticRegression()),
+        make_pipeline(
+            make_column_transformer((StandardScaler(), [0, 1])), LogisticRegression()
+        ),
+    ],
+)
+@pytest.mark.parametrize("constructor_name", ["from_estimator", "from_predictions"])
+def test_roc_curve_display_complex_pipeline(pyplot, data_binary, clf, constructor_name):
+    """Check the behaviour with complex pipeline."""
+    X, y = data_binary
+
+    clf = clone(clf)
+
+    if constructor_name == "from_estimator":
+        with pytest.raises(NotFittedError):
+            RocCurveDisplay.from_estimator(clf, X, y)
+
+    clf.fit(X, y)
+
+    if constructor_name == "from_estimator":
+        display = RocCurveDisplay.from_estimator(clf, X, y)
+        name = clf.__class__.__name__
+    else:
+        display = RocCurveDisplay.from_predictions(y, y)
+        name = "Classifier"
+
+    assert name in display.line_.get_label()
+    assert display.name == name
+
+
+@pytest.mark.parametrize(
+    "roc_auc, name, curve_kwargs, expected_labels",
+    [
+        ([0.9, 0.8], None, None, ["AUC = 0.85 +/- 0.05", "_child1"]),
+        ([0.9, 0.8], "Est name", None, ["Est name (AUC = 0.85 +/- 0.05)", "_child1"]),
+        (
+            [0.8, 0.7],
+            ["fold1", "fold2"],
+            [{"c": "blue"}, {"c": "red"}],
+            ["fold1 (AUC = 0.80)", "fold2 (AUC = 0.70)"],
+        ),
+        (None, ["fold1", "fold2"], [{"c": "blue"}, {"c": "red"}], ["fold1", "fold2"]),
+    ],
+)
+def test_roc_curve_display_default_labels(
+    pyplot, roc_auc, name, curve_kwargs, expected_labels
+):
+    """Check the default labels used in the display."""
+    fpr = [np.array([0, 0.5, 1]), np.array([0, 0.3, 1])]
+    tpr = [np.array([0, 0.5, 1]), np.array([0, 0.3, 1])]
+    disp = RocCurveDisplay(fpr=fpr, tpr=tpr, roc_auc=roc_auc, name=name).plot(
+        curve_kwargs=curve_kwargs
+    )
+    for idx, expected_label in enumerate(expected_labels):
+        assert disp.line_[idx].get_label() == expected_label
+
+
+def _check_auc(display, constructor_name):
+    roc_auc_limit = 0.95679
+    roc_auc_limit_multi = [0.97007, 0.985915, 0.980952]
+
+    if constructor_name == "from_cv_results":
+        for idx, roc_auc in enumerate(display.roc_auc):
+            assert roc_auc == pytest.approx(roc_auc_limit_multi[idx])
+    else:
+        assert display.roc_auc == pytest.approx(roc_auc_limit)
+        assert trapezoid(display.tpr, display.fpr) == pytest.approx(roc_auc_limit)
+
+
+@pytest.mark.parametrize("response_method", ["predict_proba", "decision_function"])
+@pytest.mark.parametrize(
+    "constructor_name", ["from_estimator", "from_predictions", "from_cv_results"]
+)
+def test_plot_roc_curve_pos_label(pyplot, response_method, constructor_name):
+    # check that we can provide the positive label and display the proper
+    # statistics
+    X, y = load_breast_cancer(return_X_y=True)
+    # create an highly imbalanced
+    idx_positive = np.flatnonzero(y == 1)
+    idx_negative = np.flatnonzero(y == 0)
+    idx_selected = np.hstack([idx_negative, idx_positive[:25]])
+    X, y = X[idx_selected], y[idx_selected]
+    X, y = shuffle(X, y, random_state=42)
+    # only use 2 features to make the problem even harder
+    X = X[:, :2]
+    y = np.array(["cancer" if c == 1 else "not cancer" for c in y], dtype=object)
+    X_train, X_test, y_train, y_test = train_test_split(
+        X,
+        y,
+        stratify=y,
+        random_state=0,
+    )
+
+    classifier = LogisticRegression()
+    classifier.fit(X_train, y_train)
+    cv_results = cross_validate(
+        LogisticRegression(), X, y, cv=3, return_estimator=True, return_indices=True
+    )
+
+    # Sanity check to be sure the positive class is `classes_[0]`
+    # Class imbalance ensures a large difference in prediction values between classes,
+    # allowing us to catch errors when we switch `pos_label`
+    assert classifier.classes_.tolist() == ["cancer", "not cancer"]
+
+    y_score = getattr(classifier, response_method)(X_test)
+    # we select the corresponding probability columns or reverse the decision
+    # function otherwise
+    y_score_cancer = -1 * y_score if y_score.ndim == 1 else y_score[:, 0]
+    y_score_not_cancer = y_score if y_score.ndim == 1 else y_score[:, 1]
+
+    pos_label = "cancer"
+    y_score = y_score_cancer
+    if constructor_name == "from_estimator":
+        display = RocCurveDisplay.from_estimator(
+            classifier,
+            X_test,
+            y_test,
+            pos_label=pos_label,
+            response_method=response_method,
+        )
+    elif constructor_name == "from_predictions":
+        display = RocCurveDisplay.from_predictions(
+            y_test,
+            y_score,
+            pos_label=pos_label,
+        )
+    else:
+        display = RocCurveDisplay.from_cv_results(
+            cv_results,
+            X,
+            y,
+            response_method=response_method,
+            pos_label=pos_label,
+        )
+
+    _check_auc(display, constructor_name)
+
+    pos_label = "not cancer"
+    y_score = y_score_not_cancer
+    if constructor_name == "from_estimator":
+        display = RocCurveDisplay.from_estimator(
+            classifier,
+            X_test,
+            y_test,
+            response_method=response_method,
+            pos_label=pos_label,
+        )
+    elif constructor_name == "from_predictions":
+        display = RocCurveDisplay.from_predictions(
+            y_test,
+            y_score,
+            pos_label=pos_label,
+        )
+    else:
+        display = RocCurveDisplay.from_cv_results(
+            cv_results,
+            X,
+            y,
+            response_method=response_method,
+            pos_label=pos_label,
+        )
+
+    _check_auc(display, constructor_name)
+
+
+# TODO(1.9): remove
+def test_y_score_and_y_pred_specified_error():
+    """Check that an error is raised when both y_score and y_pred are specified."""
+    y_true = np.array([0, 1, 1, 0])
+    y_score = np.array([0.1, 0.4, 0.35, 0.8])
+    y_pred = np.array([0.2, 0.3, 0.5, 0.1])
+
+    with pytest.raises(
+        ValueError, match="`y_pred` and `y_score` cannot be both specified"
+    ):
+        RocCurveDisplay.from_predictions(y_true, y_score=y_score, y_pred=y_pred)
+
+
+# TODO(1.9): remove
+def test_y_pred_deprecation_warning(pyplot):
+    """Check that a warning is raised when y_pred is specified."""
+    y_true = np.array([0, 1, 1, 0])
+    y_score = np.array([0.1, 0.4, 0.35, 0.8])
+
+    with pytest.warns(FutureWarning, match="y_pred is deprecated in 1.7"):
+        display_y_pred = RocCurveDisplay.from_predictions(y_true, y_pred=y_score)
+
+    assert_allclose(display_y_pred.fpr, [0, 0.5, 0.5, 1])
+    assert_allclose(display_y_pred.tpr, [0, 0, 1, 1])
+
+    display_y_score = RocCurveDisplay.from_predictions(y_true, y_score)
+    assert_allclose(display_y_score.fpr, [0, 0.5, 0.5, 1])
+    assert_allclose(display_y_score.tpr, [0, 0, 1, 1])
+
+
+@pytest.mark.parametrize("despine", [True, False])
+@pytest.mark.parametrize(
+    "constructor_name", ["from_estimator", "from_predictions", "from_cv_results"]
+)
+def test_plot_roc_curve_despine(pyplot, data_binary, despine, constructor_name):
+    # Check that the despine keyword is working correctly
+    X, y = data_binary
+
+    lr = LogisticRegression().fit(X, y)
+    lr.fit(X, y)
+    cv_results = cross_validate(
+        LogisticRegression(), X, y, cv=3, return_estimator=True, return_indices=True
+    )
+
+    y_pred = lr.decision_function(X)
+
+    # safe guard for the if/else construction
+    assert constructor_name in ("from_estimator", "from_predictions", "from_cv_results")
+
+    if constructor_name == "from_estimator":
+        display = RocCurveDisplay.from_estimator(lr, X, y, despine=despine)
+    elif constructor_name == "from_predictions":
+        display = RocCurveDisplay.from_predictions(y, y_pred, despine=despine)
+    else:
+        display = RocCurveDisplay.from_cv_results(cv_results, X, y, despine=despine)
+
+    for s in ["top", "right"]:
+        assert display.ax_.spines[s].get_visible() is not despine
+
+    if despine:
+        for s in ["bottom", "left"]:
+            assert display.ax_.spines[s].get_bounds() == (0, 1)
diff --git a/sklearn/metrics/_ranking.py b/sklearn/metrics/_ranking.py
index 0364fbba52f63..2d0e5211c236c 100644
--- a/sklearn/metrics/_ranking.py
+++ b/sklearn/metrics/_ranking.py
@@ -7,42 +7,39 @@
 the lower the better.
 """
 
-# Authors: Alexandre Gramfort 
-#          Mathieu Blondel 
-#          Olivier Grisel 
-#          Arnaud Joly 
-#          Jochen Wersdorfer 
-#          Lars Buitinck
-#          Joel Nothman 
-#          Noel Dawe 
-# License: BSD 3 clause
-
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import warnings
 from functools import partial
+from numbers import Integral, Real
 
 import numpy as np
-from scipy.sparse import csr_matrix
+from scipy.integrate import trapezoid
+from scipy.sparse import csr_matrix, issparse
 from scipy.stats import rankdata
 
-from ..utils import assert_all_finite
-from ..utils import check_consistent_length
-from ..utils import column_or_1d, check_array
-from ..utils.multiclass import type_of_target
-from ..utils.extmath import stable_cumsum
-from ..utils.sparsefuncs import count_nonzero
-from ..utils.validation import _deprecate_positional_args
 from ..exceptions import UndefinedMetricWarning
 from ..preprocessing import label_binarize
-from ..utils._encode import _encode, _unique
-
-from ._base import (
-    _average_binary_score,
-    _average_multiclass_ovo_score,
-    _check_pos_label_consistency,
+from ..utils import (
+    assert_all_finite,
+    check_array,
+    check_consistent_length,
+    column_or_1d,
 )
+from ..utils._encode import _encode, _unique
+from ..utils._param_validation import Interval, StrOptions, validate_params
+from ..utils.extmath import stable_cumsum
+from ..utils.multiclass import type_of_target
+from ..utils.sparsefuncs import count_nonzero
+from ..utils.validation import _check_pos_label_consistency, _check_sample_weight
+from ._base import _average_binary_score, _average_multiclass_ovo_score
 
 
+@validate_params(
+    {"x": ["array-like"], "y": ["array-like"]},
+    prefer_skip_nested_validation=True,
+)
 def auc(x, y):
     """Compute Area Under the Curve (AUC) using the trapezoidal rule.
 
@@ -53,15 +50,16 @@ def auc(x, y):
 
     Parameters
     ----------
-    x : ndarray of shape (n,)
-        x coordinates. These must be either monotonic increasing or monotonic
+    x : array-like of shape (n,)
+        X coordinates. These must be either monotonic increasing or monotonic
         decreasing.
-    y : ndarray of shape, (n,)
-        y coordinates.
+    y : array-like of shape (n,)
+        Y coordinates.
 
     Returns
     -------
     auc : float
+        Area Under the Curve.
 
     See Also
     --------
@@ -74,9 +72,9 @@ def auc(x, y):
     --------
     >>> import numpy as np
     >>> from sklearn import metrics
-    >>> y = np.array([1, 1, 2, 2])
-    >>> pred = np.array([0.1, 0.4, 0.35, 0.8])
-    >>> fpr, tpr, thresholds = metrics.roc_curve(y, pred, pos_label=2)
+    >>> y_true = np.array([1, 1, 2, 2])
+    >>> y_score = np.array([0.1, 0.4, 0.35, 0.8])
+    >>> fpr, tpr, thresholds = metrics.roc_curve(y_true, y_score, pos_label=2)
     >>> metrics.auc(fpr, tpr)
     0.75
     """
@@ -85,8 +83,10 @@ def auc(x, y):
     y = column_or_1d(y)
 
     if x.shape[0] < 2:
-        raise ValueError('At least 2 points are needed to compute'
-                         ' area under curve, but x.shape = %s' % x.shape)
+        raise ValueError(
+            "At least 2 points are needed to compute area under curve, but x.shape = %s"
+            % x.shape
+        )
 
     direction = 1
     dx = np.diff(x)
@@ -94,21 +94,30 @@ def auc(x, y):
         if np.all(dx <= 0):
             direction = -1
         else:
-            raise ValueError("x is neither increasing nor decreasing "
-                             ": {}.".format(x))
+            raise ValueError("x is neither increasing nor decreasing : {}.".format(x))
 
-    area = direction * np.trapz(y, x)
+    area = direction * trapezoid(y, x)
     if isinstance(area, np.memmap):
-        # Reductions such as .sum used internally in np.trapz do not return a
+        # Reductions such as .sum used internally in trapezoid do not return a
         # scalar by default for numpy.memmap instances contrary to
         # regular numpy.ndarray instances.
         area = area.dtype.type(area)
-    return area
-
-
-@_deprecate_positional_args
-def average_precision_score(y_true, y_score, *, average="macro", pos_label=1,
-                            sample_weight=None):
+    return float(area)
+
+
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_score": ["array-like"],
+        "average": [StrOptions({"micro", "samples", "weighted", "macro"}), None],
+        "pos_label": [Real, str, "boolean"],
+        "sample_weight": ["array-like", None],
+    },
+    prefer_skip_nested_validation=True,
+)
+def average_precision_score(
+    y_true, y_score, *, average="macro", pos_label=1, sample_weight=None
+):
     """Compute average precision (AP) from prediction scores.
 
     AP summarizes a precision-recall curve as the weighted mean of precisions
@@ -124,20 +133,19 @@ def average_precision_score(y_true, y_score, *, average="macro", pos_label=1,
     trapezoidal rule, which uses linear interpolation and can be too
     optimistic.
 
-    Note: this implementation is restricted to the binary classification task
-    or multilabel classification task.
-
     Read more in the :ref:`User Guide `.
 
     Parameters
     ----------
-    y_true : ndarray of shape (n_samples,) or (n_samples, n_classes)
+    y_true : array-like of shape (n_samples,) or (n_samples, n_classes)
         True binary labels or binary label indicators.
 
-    y_score : ndarray of shape (n_samples,) or (n_samples, n_classes)
+    y_score : array-like of shape (n_samples,) or (n_samples, n_classes)
         Target scores, can either be probability estimates of the positive
         class, confidence values, or non-thresholded measure of decisions
         (as returned by :term:`decision_function` on some classifiers).
+        For :term:`decision_function` scores, values greater than or equal to
+        zero should indicate the positive class.
 
     average : {'micro', 'samples', 'weighted', 'macro'} or None, \
             default='macro'
@@ -158,7 +166,7 @@ def average_precision_score(y_true, y_score, *, average="macro", pos_label=1,
 
         Will be ignored when ``y_true`` is binary.
 
-    pos_label : int or str, default=1
+    pos_label : int, float, bool or str, default=1
         The label of the positive class. Only applied to binary ``y_true``.
         For multilabel-indicator ``y_true``, ``pos_label`` is fixed to 1.
 
@@ -168,12 +176,17 @@ def average_precision_score(y_true, y_score, *, average="macro", pos_label=1,
     Returns
     -------
     average_precision : float
+        Average precision score.
 
     See Also
     --------
     roc_auc_score : Compute the area under the ROC curve.
     precision_recall_curve : Compute precision-recall pairs for different
         probability thresholds.
+    PrecisionRecallDisplay.from_estimator : Plot the precision recall curve
+        using an estimator and data.
+    PrecisionRecallDisplay.from_predictions : Plot the precision recall curve
+        using true and predicted labels.
 
     Notes
     -----
@@ -194,39 +207,81 @@ def average_precision_score(y_true, y_score, *, average="macro", pos_label=1,
     >>> y_true = np.array([0, 0, 1, 1])
     >>> y_scores = np.array([0.1, 0.4, 0.35, 0.8])
     >>> average_precision_score(y_true, y_scores)
-    0.83...
+    0.83
+    >>> y_true = np.array([0, 0, 1, 1, 2, 2])
+    >>> y_scores = np.array([
+    ...     [0.7, 0.2, 0.1],
+    ...     [0.4, 0.3, 0.3],
+    ...     [0.1, 0.8, 0.1],
+    ...     [0.2, 0.3, 0.5],
+    ...     [0.4, 0.4, 0.2],
+    ...     [0.1, 0.2, 0.7],
+    ... ])
+    >>> average_precision_score(y_true, y_scores)
+    0.77
     """
+
     def _binary_uninterpolated_average_precision(
-            y_true, y_score, pos_label=1, sample_weight=None):
+        y_true, y_score, pos_label=1, sample_weight=None
+    ):
         precision, recall, _ = precision_recall_curve(
-            y_true, y_score, pos_label=pos_label, sample_weight=sample_weight)
+            y_true, y_score, pos_label=pos_label, sample_weight=sample_weight
+        )
         # Return the step function integral
         # The following works because the last entry of precision is
-        # guaranteed to be 1, as returned by precision_recall_curve
-        return -np.sum(np.diff(recall) * np.array(precision)[:-1])
-
-    y_type = type_of_target(y_true)
-    if y_type == "multilabel-indicator" and pos_label != 1:
-        raise ValueError("Parameter pos_label is fixed to 1 for "
-                         "multilabel-indicator y_true. Do not set "
-                         "pos_label or set pos_label to 1.")
-    elif y_type == "binary":
-        # Convert to Python primitive type to avoid NumPy type / Python str
-        # comparison. See https://github.com/numpy/numpy/issues/6784
-        present_labels = np.unique(y_true).tolist()
+        # guaranteed to be 1, as returned by precision_recall_curve.
+        # Due to numerical error, we can get `-0.0` and we therefore clip it.
+        return float(max(0.0, -np.sum(np.diff(recall) * np.array(precision)[:-1])))
+
+    y_type = type_of_target(y_true, input_name="y_true")
+
+    # Convert to Python primitive type to avoid NumPy type / Python str
+    # comparison. See https://github.com/numpy/numpy/issues/6784
+    present_labels = np.unique(y_true).tolist()
+
+    if y_type == "binary":
         if len(present_labels) == 2 and pos_label not in present_labels:
             raise ValueError(
                 f"pos_label={pos_label} is not a valid label. It should be "
                 f"one of {present_labels}"
             )
-    average_precision = partial(_binary_uninterpolated_average_precision,
-                                pos_label=pos_label)
-    return _average_binary_score(average_precision, y_true, y_score,
-                                 average, sample_weight=sample_weight)
+
+    elif y_type == "multilabel-indicator" and pos_label != 1:
+        raise ValueError(
+            "Parameter pos_label is fixed to 1 for multilabel-indicator y_true. "
+            "Do not set pos_label or set pos_label to 1."
+        )
+
+    elif y_type == "multiclass":
+        if pos_label != 1:
+            raise ValueError(
+                "Parameter pos_label is fixed to 1 for multiclass y_true. "
+                "Do not set pos_label or set pos_label to 1."
+            )
+        y_true = label_binarize(y_true, classes=present_labels)
+
+    average_precision = partial(
+        _binary_uninterpolated_average_precision, pos_label=pos_label
+    )
+    return _average_binary_score(
+        average_precision, y_true, y_score, average, sample_weight=sample_weight
+    )
 
 
-def det_curve(y_true, y_score, pos_label=None, sample_weight=None):
-    """Compute error rates for different probability thresholds.
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_score": ["array-like"],
+        "pos_label": [Real, str, "boolean", None],
+        "sample_weight": ["array-like", None],
+        "drop_intermediate": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def det_curve(
+    y_true, y_score, pos_label=None, sample_weight=None, drop_intermediate=False
+):
+    """Compute Detection Error Tradeoff (DET) for different probability thresholds.
 
     .. note::
        This metric is used for evaluation of ranking and error tradeoffs of
@@ -236,6 +291,11 @@ def det_curve(y_true, y_score, pos_label=None, sample_weight=None):
 
     .. versionadded:: 0.24
 
+    .. versionchanged:: 1.7
+       An arbitrary threshold at infinity is added to represent a classifier
+       that always predicts the negative class, i.e. `fpr=0` and `fnr=1`, unless
+       `fpr=0` is already reached at a finite threshold.
+
     Parameters
     ----------
     y_true : ndarray of shape (n_samples,)
@@ -246,8 +306,10 @@ def det_curve(y_true, y_score, pos_label=None, sample_weight=None):
         Target scores, can either be probability estimates of the positive
         class, confidence values, or non-thresholded measure of decisions
         (as returned by "decision_function" on some classifiers).
+        For :term:`decision_function` scores, values greater than or equal to
+        zero should indicate the positive class.
 
-    pos_label : int or str, default=None
+    pos_label : int, float, bool or str, default=None
         The label of the positive class.
         When ``pos_label=None``, if `y_true` is in {-1, 1} or {0, 1},
         ``pos_label`` is set to 1, otherwise an error will be raised.
@@ -255,12 +317,19 @@ def det_curve(y_true, y_score, pos_label=None, sample_weight=None):
     sample_weight : array-like of shape (n_samples,), default=None
         Sample weights.
 
+    drop_intermediate : bool, default=False
+        Whether to drop thresholds where true positives (tp) do not change from
+        the previous or subsequent threshold. All points with the same tp value
+        have the same `fnr` and thus same y coordinate.
+
+        .. versionadded:: 1.7
+
     Returns
     -------
     fpr : ndarray of shape (n_thresholds,)
         False positive rate (FPR) such that element i is the false positive
         rate of predictions with score >= thresholds[i]. This is occasionally
-        referred to as false acceptance propability or fall-out.
+        referred to as false acceptance probability or fall-out.
 
     fnr : ndarray of shape (n_thresholds,)
         False negative rate (FNR) such that element i is the false negative
@@ -268,11 +337,19 @@ def det_curve(y_true, y_score, pos_label=None, sample_weight=None):
         referred to as false rejection or miss rate.
 
     thresholds : ndarray of shape (n_thresholds,)
-        Decreasing score values.
+        Decreasing thresholds on the decision function (either `predict_proba`
+        or `decision_function`) used to compute FPR and FNR.
+
+        .. versionchanged:: 1.7
+           An arbitrary threshold at infinity is added for the case `fpr=0`
+           and `fnr=1`.
 
     See Also
     --------
-    plot_det_curve : Plot detection error tradeoff (DET) curve.
+    DetCurveDisplay.from_estimator : Plot DET curve given an estimator and
+        some data.
+    DetCurveDisplay.from_predictions : Plot DET curve given the true and
+        predicted labels.
     DetCurveDisplay : DET curve visualization.
     roc_curve : Compute Receiver operating characteristic (ROC) curve.
     precision_recall_curve : Compute precision-recall curve.
@@ -291,22 +368,46 @@ def det_curve(y_true, y_score, pos_label=None, sample_weight=None):
     >>> thresholds
     array([0.35, 0.4 , 0.8 ])
     """
-    if len(np.unique(y_true)) != 2:
-        raise ValueError("Only one class present in y_true. Detection error "
-                         "tradeoff curve is not defined in that case.")
-
     fps, tps, thresholds = _binary_clf_curve(
         y_true, y_score, pos_label=pos_label, sample_weight=sample_weight
     )
 
+    # add a threshold at inf where the clf always predicts the negative class
+    # i.e. tps = fps = 0
+    tps = np.concatenate(([0], tps))
+    fps = np.concatenate(([0], fps))
+    thresholds = np.concatenate(([np.inf], thresholds))
+
+    if drop_intermediate and len(fps) > 2:
+        # Drop thresholds where true positives (tp) do not change from the
+        # previous or subsequent threshold. As tp + fn, is fixed for a dataset,
+        # this means the false negative rate (fnr) remains constant while the
+        # false positive rate (fpr) changes, producing horizontal line segments
+        # in the transformed (normal deviate) scale. These intermediate points
+        # can be dropped to create lighter DET curve plots.
+        optimal_idxs = np.where(
+            np.concatenate(
+                [[True], np.logical_or(np.diff(tps[:-1]), np.diff(tps[1:])), [True]]
+            )
+        )[0]
+        fps = fps[optimal_idxs]
+        tps = tps[optimal_idxs]
+        thresholds = thresholds[optimal_idxs]
+
+    if len(np.unique(y_true)) != 2:
+        raise ValueError(
+            "Only one class is present in y_true. Detection error "
+            "tradeoff curve is not defined in that case."
+        )
+
     fns = tps[-1] - tps
     p_count = tps[-1]
     n_count = fps[-1]
 
-    # start with false positives zero
+    # start with false positives zero, which may be at a finite threshold
     first_ind = (
-        fps.searchsorted(fps[0], side='right') - 1
-        if fps.searchsorted(fps[0], side='right') > 0
+        fps.searchsorted(fps[0], side="right") - 1
+        if fps.searchsorted(fps[0], side="right") > 0
         else None
     )
     # stop with false negatives zero
@@ -314,28 +415,29 @@ def det_curve(y_true, y_score, pos_label=None, sample_weight=None):
     sl = slice(first_ind, last_ind)
 
     # reverse the output such that list of false positives is decreasing
-    return (
-        fps[sl][::-1] / n_count,
-        fns[sl][::-1] / p_count,
-        thresholds[sl][::-1]
-    )
+    return (fps[sl][::-1] / n_count, fns[sl][::-1] / p_count, thresholds[sl][::-1])
 
 
 def _binary_roc_auc_score(y_true, y_score, sample_weight=None, max_fpr=None):
     """Binary roc auc score."""
     if len(np.unique(y_true)) != 2:
-        raise ValueError("Only one class present in y_true. ROC AUC score "
-                         "is not defined in that case.")
+        warnings.warn(
+            (
+                "Only one class is present in y_true. ROC AUC score "
+                "is not defined in that case."
+            ),
+            UndefinedMetricWarning,
+        )
+        return np.nan
 
-    fpr, tpr, _ = roc_curve(y_true, y_score,
-                            sample_weight=sample_weight)
+    fpr, tpr, _ = roc_curve(y_true, y_score, sample_weight=sample_weight)
     if max_fpr is None or max_fpr == 1:
         return auc(fpr, tpr)
     if max_fpr <= 0 or max_fpr > 1:
         raise ValueError("Expected max_fpr in range (0, 1], got: %r" % max_fpr)
 
     # Add a single point at max_fpr by linear interpolation
-    stop = np.searchsorted(fpr, max_fpr, 'right')
+    stop = np.searchsorted(fpr, max_fpr, "right")
     x_interp = [fpr[stop - 1], fpr[stop]]
     y_interp = [tpr[stop - 1], tpr[stop]]
     tpr = np.append(tpr[:stop], np.interp(max_fpr, x_interp, y_interp))
@@ -349,10 +451,29 @@ def _binary_roc_auc_score(y_true, y_score, sample_weight=None, max_fpr=None):
     return 0.5 * (1 + (partial_auc - min_area) / (max_area - min_area))
 
 
-@_deprecate_positional_args
-def roc_auc_score(y_true, y_score, *, average="macro", sample_weight=None,
-                  max_fpr=None, multi_class="raise", labels=None):
-    """Compute Area Under the Receiver Operating Characteristic Curve (ROC AUC)
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_score": ["array-like"],
+        "average": [StrOptions({"micro", "macro", "samples", "weighted"}), None],
+        "sample_weight": ["array-like", None],
+        "max_fpr": [Interval(Real, 0.0, 1, closed="right"), None],
+        "multi_class": [StrOptions({"raise", "ovr", "ovo"})],
+        "labels": ["array-like", None],
+    },
+    prefer_skip_nested_validation=True,
+)
+def roc_auc_score(
+    y_true,
+    y_score,
+    *,
+    average="macro",
+    sample_weight=None,
+    max_fpr=None,
+    multi_class="raise",
+    labels=None,
+):
+    """Compute Area Under the Receiver Operating Characteristic Curve (ROC AUC) \
     from prediction scores.
 
     Note: this implementation can be used with binary, multiclass and
@@ -396,10 +517,12 @@ class scores must correspond to the order of ``labels``,
 
     average : {'micro', 'macro', 'samples', 'weighted'} or None, \
             default='macro'
-        If ``None``, the scores for each class are returned. Otherwise,
-        this determines the type of averaging performed on the data:
+        If ``None``, the scores for each class are returned.
+        Otherwise, this determines the type of averaging performed on the data.
         Note: multiclass ROC AUC currently only handles the 'macro' and
-        'weighted' averages.
+        'weighted' averages. For multiclass targets, `average=None` is only
+        implemented for `multi_class='ovr'` and `average='micro'` is only
+        implemented for `multi_class='ovr'`.
 
         ``'micro'``:
             Calculate metrics globally by considering each element of the label
@@ -450,6 +573,27 @@ class scores must correspond to the order of ``labels``,
     Returns
     -------
     auc : float
+        Area Under the Curve score.
+
+    See Also
+    --------
+    average_precision_score : Area under the precision-recall curve.
+    roc_curve : Compute Receiver operating characteristic (ROC) curve.
+    RocCurveDisplay.from_estimator : Plot Receiver Operating Characteristic
+        (ROC) curve given an estimator and some data.
+    RocCurveDisplay.from_predictions : Plot Receiver Operating Characteristic
+        (ROC) curve given the true and predicted values.
+
+    Notes
+    -----
+    The Gini Coefficient is a summary measure of the ranking ability of binary
+    classifiers. It is expressed using the area under of the ROC as follows:
+
+    G = 2 * AUC - 1
+
+    Where G is the Gini coefficient and AUC is the ROC-AUC score. This normalisation
+    will ensure that random guessing will yield a score of 0 in expectation, and it is
+    upper bounded by 1.
 
     References
     ----------
@@ -471,12 +615,8 @@ class scores must correspond to the order of ``labels``,
             Under the ROC Curve for Multiple Class Classification Problems.
             Machine Learning, 45(2), 171-186.
             `_
-
-    See Also
-    --------
-    average_precision_score : Area under the precision-recall curve.
-    roc_curve : Compute Receiver operating characteristic (ROC) curve.
-    plot_roc_curve : Plot Receiver operating characteristic (ROC) curve.
+    .. [6] `Wikipedia entry for the Gini coefficient
+            `_
 
     Examples
     --------
@@ -486,72 +626,83 @@ class scores must correspond to the order of ``labels``,
     >>> from sklearn.linear_model import LogisticRegression
     >>> from sklearn.metrics import roc_auc_score
     >>> X, y = load_breast_cancer(return_X_y=True)
-    >>> clf = LogisticRegression(solver="liblinear", random_state=0).fit(X, y)
+    >>> clf = LogisticRegression(solver="newton-cholesky", random_state=0).fit(X, y)
     >>> roc_auc_score(y, clf.predict_proba(X)[:, 1])
-    0.99...
+    0.99
     >>> roc_auc_score(y, clf.decision_function(X))
-    0.99...
+    0.99
 
     Multiclass case:
 
     >>> from sklearn.datasets import load_iris
     >>> X, y = load_iris(return_X_y=True)
-    >>> clf = LogisticRegression(solver="liblinear").fit(X, y)
+    >>> clf = LogisticRegression(solver="newton-cholesky").fit(X, y)
     >>> roc_auc_score(y, clf.predict_proba(X), multi_class='ovr')
-    0.99...
+    0.99
 
     Multilabel case:
 
+    >>> import numpy as np
     >>> from sklearn.datasets import make_multilabel_classification
     >>> from sklearn.multioutput import MultiOutputClassifier
     >>> X, y = make_multilabel_classification(random_state=0)
     >>> clf = MultiOutputClassifier(clf).fit(X, y)
     >>> # get a list of n_output containing probability arrays of shape
     >>> # (n_samples, n_classes)
-    >>> y_pred = clf.predict_proba(X)
+    >>> y_score = clf.predict_proba(X)
     >>> # extract the positive columns for each output
-    >>> y_pred = np.transpose([pred[:, 1] for pred in y_pred])
-    >>> roc_auc_score(y, y_pred, average=None)
-    array([0.82..., 0.86..., 0.94..., 0.85... , 0.94...])
+    >>> y_score = np.transpose([score[:, 1] for score in y_score])
+    >>> roc_auc_score(y, y_score, average=None)
+    array([0.828, 0.852, 0.94, 0.869, 0.95])
     >>> from sklearn.linear_model import RidgeClassifierCV
     >>> clf = RidgeClassifierCV().fit(X, y)
     >>> roc_auc_score(y, clf.decision_function(X), average=None)
-    array([0.81..., 0.84... , 0.93..., 0.87..., 0.94...])
+    array([0.82, 0.847, 0.93, 0.872, 0.944])
     """
 
-    y_type = type_of_target(y_true)
+    y_type = type_of_target(y_true, input_name="y_true")
     y_true = check_array(y_true, ensure_2d=False, dtype=None)
     y_score = check_array(y_score, ensure_2d=False)
 
-    if y_type == "multiclass" or (y_type == "binary" and
-                                  y_score.ndim == 2 and
-                                  y_score.shape[1] > 2):
+    if y_type == "multiclass" or (
+        y_type == "binary" and y_score.ndim == 2 and y_score.shape[1] > 2
+    ):
         # do not support partial ROC computation for multiclass
-        if max_fpr is not None and max_fpr != 1.:
-            raise ValueError("Partial AUC computation not available in "
-                             "multiclass setting, 'max_fpr' must be"
-                             " set to `None`, received `max_fpr={0}` "
-                             "instead".format(max_fpr))
-        if multi_class == 'raise':
+        if max_fpr is not None and max_fpr != 1.0:
+            raise ValueError(
+                "Partial AUC computation not available in "
+                "multiclass setting, 'max_fpr' must be"
+                " set to `None`, received `max_fpr={0}` "
+                "instead".format(max_fpr)
+            )
+        if multi_class == "raise":
             raise ValueError("multi_class must be in ('ovo', 'ovr')")
-        return _multiclass_roc_auc_score(y_true, y_score, labels,
-                                         multi_class, average, sample_weight)
+        return _multiclass_roc_auc_score(
+            y_true, y_score, labels, multi_class, average, sample_weight
+        )
     elif y_type == "binary":
         labels = np.unique(y_true)
         y_true = label_binarize(y_true, classes=labels)[:, 0]
-        return _average_binary_score(partial(_binary_roc_auc_score,
-                                             max_fpr=max_fpr),
-                                     y_true, y_score, average,
-                                     sample_weight=sample_weight)
+        return _average_binary_score(
+            partial(_binary_roc_auc_score, max_fpr=max_fpr),
+            y_true,
+            y_score,
+            average,
+            sample_weight=sample_weight,
+        )
     else:  # multilabel-indicator
-        return _average_binary_score(partial(_binary_roc_auc_score,
-                                             max_fpr=max_fpr),
-                                     y_true, y_score, average,
-                                     sample_weight=sample_weight)
+        return _average_binary_score(
+            partial(_binary_roc_auc_score, max_fpr=max_fpr),
+            y_true,
+            y_score,
+            average,
+            sample_weight=sample_weight,
+        )
 
 
-def _multiclass_roc_auc_score(y_true, y_score, labels,
-                              multi_class, average, sample_weight):
+def _multiclass_roc_auc_score(
+    y_true, y_score, labels, multi_class, average, sample_weight
+):
     """Multiclass roc auc score.
 
     Parameters
@@ -576,9 +727,15 @@ def _multiclass_roc_auc_score(y_true, y_score, labels,
             Calculate metrics for the multiclass case using the one-vs-one
             approach.
 
-    average : {'macro', 'weighted'}
+    average : {'micro', 'macro', 'weighted'}
         Determines the type of averaging performed on the pairwise binary
         metric scores
+        ``'micro'``:
+            Calculate metrics for the binarized-raveled classes. Only supported
+            for `multi_class='ovr'`.
+
+        .. versionadded:: 1.2
+
         ``'macro'``:
             Calculate metrics for each label, and find their unweighted
             mean. This does not take label imbalance into account. Classes
@@ -595,20 +752,30 @@ def _multiclass_roc_auc_score(y_true, y_score, labels,
     if not np.allclose(1, y_score.sum(axis=1)):
         raise ValueError(
             "Target scores need to be probabilities for multiclass "
-            "roc_auc, i.e. they should sum up to 1.0 over classes")
+            "roc_auc, i.e. they should sum up to 1.0 over classes"
+        )
 
     # validation for multiclass parameter specifications
-    average_options = ("macro", "weighted")
+    average_options = ("macro", "weighted", None)
+    if multi_class == "ovr":
+        average_options = ("micro",) + average_options
     if average not in average_options:
-        raise ValueError("average must be one of {0} for "
-                         "multiclass problems".format(average_options))
+        raise ValueError(
+            "average must be one of {0} for multiclass problems".format(average_options)
+        )
 
     multiclass_options = ("ovo", "ovr")
     if multi_class not in multiclass_options:
-        raise ValueError("multi_class='{0}' is not supported "
-                         "for multiclass ROC AUC, multi_class must be "
-                         "in {1}".format(
-                                multi_class, multiclass_options))
+        raise ValueError(
+            "multi_class='{0}' is not supported "
+            "for multiclass ROC AUC, multi_class must be "
+            "in {1}".format(multi_class, multiclass_options)
+        )
+
+    if average is None and multi_class == "ovo":
+        raise NotImplementedError(
+            "average=None is not implemented for multi_class='ovo'."
+        )
 
     if labels is not None:
         labels = column_or_1d(labels)
@@ -620,34 +787,40 @@ def _multiclass_roc_auc_score(y_true, y_score, labels,
         if len(classes) != y_score.shape[1]:
             raise ValueError(
                 "Number of given labels, {0}, not equal to the number "
-                "of columns in 'y_score', {1}".format(
-                    len(classes), y_score.shape[1]))
+                "of columns in 'y_score', {1}".format(len(classes), y_score.shape[1])
+            )
         if len(np.setdiff1d(y_true, classes)):
-            raise ValueError(
-                "'y_true' contains labels not in parameter 'labels'")
+            raise ValueError("'y_true' contains labels not in parameter 'labels'")
     else:
         classes = _unique(y_true)
         if len(classes) != y_score.shape[1]:
             raise ValueError(
                 "Number of classes in y_true not equal to the number of "
-                "columns in 'y_score'")
+                "columns in 'y_score'"
+            )
 
     if multi_class == "ovo":
         if sample_weight is not None:
-            raise ValueError("sample_weight is not supported "
-                             "for multiclass one-vs-one ROC AUC, "
-                             "'sample_weight' must be None in this case.")
+            raise ValueError(
+                "sample_weight is not supported "
+                "for multiclass one-vs-one ROC AUC, "
+                "'sample_weight' must be None in this case."
+            )
         y_true_encoded = _encode(y_true, uniques=classes)
         # Hand & Till (2001) implementation (ovo)
-        return _average_multiclass_ovo_score(_binary_roc_auc_score,
-                                             y_true_encoded,
-                                             y_score, average=average)
+        return _average_multiclass_ovo_score(
+            _binary_roc_auc_score, y_true_encoded, y_score, average=average
+        )
     else:
         # ovr is same as multi-label
         y_true_multilabel = label_binarize(y_true, classes=classes)
-        return _average_binary_score(_binary_roc_auc_score, y_true_multilabel,
-                                     y_score, average,
-                                     sample_weight=sample_weight)
+        return _average_binary_score(
+            _binary_roc_auc_score,
+            y_true_multilabel,
+            y_score,
+            average,
+            sample_weight=sample_weight,
+        )
 
 
 def _binary_clf_curve(y_true, y_score, pos_label=None, sample_weight=None):
@@ -661,7 +834,7 @@ def _binary_clf_curve(y_true, y_score, pos_label=None, sample_weight=None):
     y_score : ndarray of shape (n_samples,)
         Estimated probabilities or output of a decision function.
 
-    pos_label : int or str, default=None
+    pos_label : int, float, bool or str, default=None
         The label of the positive class.
 
     sample_weight : array-like of shape (n_samples,), default=None
@@ -685,9 +858,8 @@ def _binary_clf_curve(y_true, y_score, pos_label=None, sample_weight=None):
         Decreasing score values.
     """
     # Check to make sure y_true is valid
-    y_type = type_of_target(y_true)
-    if not (y_type == "binary" or
-            (y_type == "multiclass" and pos_label is not None)):
+    y_type = type_of_target(y_true, input_name="y_true")
+    if not (y_type == "binary" or (y_type == "multiclass" and pos_label is not None)):
         raise ValueError("{0} format is not supported".format(y_type))
 
     check_consistent_length(y_true, y_score, sample_weight)
@@ -696,13 +868,19 @@ def _binary_clf_curve(y_true, y_score, pos_label=None, sample_weight=None):
     assert_all_finite(y_true)
     assert_all_finite(y_score)
 
+    # Filter out zero-weighted samples, as they should not impact the result
     if sample_weight is not None:
         sample_weight = column_or_1d(sample_weight)
+        sample_weight = _check_sample_weight(sample_weight, y_true)
+        nonzero_weight_mask = sample_weight != 0
+        y_true = y_true[nonzero_weight_mask]
+        y_score = y_score[nonzero_weight_mask]
+        sample_weight = sample_weight[nonzero_weight_mask]
 
     pos_label = _check_pos_label_consistency(pos_label, y_true)
 
     # make y_true a boolean vector
-    y_true = (y_true == pos_label)
+    y_true = y_true == pos_label
 
     # sort scores and corresponding truth values
     desc_score_indices = np.argsort(y_score, kind="mergesort")[::-1]
@@ -711,7 +889,7 @@ def _binary_clf_curve(y_true, y_score, pos_label=None, sample_weight=None):
     if sample_weight is not None:
         weight = sample_weight[desc_score_indices]
     else:
-        weight = 1.
+        weight = 1.0
 
     # y_score typically has many tied values. Here we extract
     # the indices associated with the distinct values. We also
@@ -730,9 +908,24 @@ def _binary_clf_curve(y_true, y_score, pos_label=None, sample_weight=None):
     return fps, tps, y_score[threshold_idxs]
 
 
-@_deprecate_positional_args
-def precision_recall_curve(y_true, probas_pred, *, pos_label=None,
-                           sample_weight=None):
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_score": ["array-like"],
+        "pos_label": [Real, str, "boolean", None],
+        "sample_weight": ["array-like", None],
+        "drop_intermediate": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def precision_recall_curve(
+    y_true,
+    y_score,
+    *,
+    pos_label=None,
+    sample_weight=None,
+    drop_intermediate=False,
+):
     """Compute precision-recall pairs for different probability thresholds.
 
     Note: this implementation is restricted to the binary classification task.
@@ -750,18 +943,25 @@ def precision_recall_curve(y_true, probas_pred, *, pos_label=None,
     have a corresponding threshold. This ensures that the graph starts on the
     y axis.
 
+    The first precision and recall values are precision=class balance and recall=1.0
+    which corresponds to a classifier that always predicts the positive class.
+
     Read more in the :ref:`User Guide `.
 
     Parameters
     ----------
-    y_true : ndarray of shape (n_samples,)
+    y_true : array-like of shape (n_samples,)
         True binary labels. If labels are not either {-1, 1} or {0, 1}, then
         pos_label should be explicitly given.
 
-    probas_pred : ndarray of shape (n_samples,)
-        Estimated probabilities or output of a decision function.
+    y_score : array-like of shape (n_samples,)
+        Target scores, can either be probability estimates of the positive
+        class, or non-thresholded measure of decisions (as returned by
+        `decision_function` on some classifiers).
+        For :term:`decision_function` scores, values greater than or equal to
+        zero should indicate the positive class.
 
-    pos_label : int or str, default=None
+    pos_label : int, float, bool or str, default=None
         The label of the positive class.
         When ``pos_label=None``, if y_true is in {-1, 1} or {0, 1},
         ``pos_label`` is set to 1, otherwise an error will be raised.
@@ -769,6 +969,13 @@ def precision_recall_curve(y_true, probas_pred, *, pos_label=None,
     sample_weight : array-like of shape (n_samples,), default=None
         Sample weights.
 
+    drop_intermediate : bool, default=False
+        Whether to drop some suboptimal thresholds which would not appear
+        on a plotted precision-recall curve. This is useful in order to create
+        lighter precision-recall curves.
+
+        .. versionadded:: 1.3
+
     Returns
     -------
     precision : ndarray of shape (n_thresholds + 1,)
@@ -781,13 +988,14 @@ def precision_recall_curve(y_true, probas_pred, *, pos_label=None,
 
     thresholds : ndarray of shape (n_thresholds,)
         Increasing thresholds on the decision function used to compute
-        precision and recall. n_thresholds <= len(np.unique(probas_pred)).
+        precision and recall where `n_thresholds = len(np.unique(y_score))`.
 
     See Also
     --------
-    plot_precision_recall_curve : Plot Precision Recall Curve for binary
-        classifiers.
-    PrecisionRecallDisplay : Precision Recall visualization.
+    PrecisionRecallDisplay.from_estimator : Plot Precision Recall Curve given
+        a binary classifier.
+    PrecisionRecallDisplay.from_predictions : Plot Precision Recall Curve
+        using predictions from a binary classifier.
     average_precision_score : Compute average precision from prediction scores.
     det_curve: Compute error rates for different probability thresholds.
     roc_curve : Compute Receiver operating characteristic (ROC) curve.
@@ -801,31 +1009,66 @@ def precision_recall_curve(y_true, probas_pred, *, pos_label=None,
     >>> precision, recall, thresholds = precision_recall_curve(
     ...     y_true, y_scores)
     >>> precision
-    array([0.66666667, 0.5       , 1.        , 1.        ])
+    array([0.5       , 0.66666667, 0.5       , 1.        , 1.        ])
     >>> recall
-    array([1. , 0.5, 0.5, 0. ])
+    array([1. , 1. , 0.5, 0.5, 0. ])
     >>> thresholds
-    array([0.35, 0.4 , 0.8 ])
-
+    array([0.1 , 0.35, 0.4 , 0.8 ])
     """
-    fps, tps, thresholds = _binary_clf_curve(y_true, probas_pred,
-                                             pos_label=pos_label,
-                                             sample_weight=sample_weight)
-
-    precision = tps / (tps + fps)
-    precision[np.isnan(precision)] = 0
-    recall = tps / tps[-1]
+    fps, tps, thresholds = _binary_clf_curve(
+        y_true, y_score, pos_label=pos_label, sample_weight=sample_weight
+    )
 
-    # stop when full recall attained
-    # and reverse the outputs so recall is decreasing
-    last_ind = tps.searchsorted(tps[-1])
-    sl = slice(last_ind, None, -1)
-    return np.r_[precision[sl], 1], np.r_[recall[sl], 0], thresholds[sl]
+    if drop_intermediate and len(fps) > 2:
+        # Drop thresholds corresponding to points where true positives (tps)
+        # do not change from the previous or subsequent point. This will keep
+        # only the first and last point for each tps value. All points
+        # with the same tps value have the same recall and thus x coordinate.
+        # They appear as a vertical line on the plot.
+        optimal_idxs = np.where(
+            np.concatenate(
+                [[True], np.logical_or(np.diff(tps[:-1]), np.diff(tps[1:])), [True]]
+            )
+        )[0]
+        fps = fps[optimal_idxs]
+        tps = tps[optimal_idxs]
+        thresholds = thresholds[optimal_idxs]
 
+    ps = tps + fps
+    # Initialize the result array with zeros to make sure that precision[ps == 0]
+    # does not contain uninitialized values.
+    precision = np.zeros_like(tps)
+    np.divide(tps, ps, out=precision, where=(ps != 0))
 
-@_deprecate_positional_args
-def roc_curve(y_true, y_score, *, pos_label=None, sample_weight=None,
-              drop_intermediate=True):
+    # When no positive label in y_true, recall is set to 1 for all thresholds
+    # tps[-1] == 0 <=> y_true == all negative labels
+    if tps[-1] == 0:
+        warnings.warn(
+            "No positive class found in y_true, "
+            "recall is set to one for all thresholds."
+        )
+        recall = np.ones_like(tps)
+    else:
+        recall = tps / tps[-1]
+
+    # reverse the outputs so recall is decreasing
+    sl = slice(None, None, -1)
+    return np.hstack((precision[sl], 1)), np.hstack((recall[sl], 0)), thresholds[sl]
+
+
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_score": ["array-like"],
+        "pos_label": [Real, str, "boolean", None],
+        "sample_weight": ["array-like", None],
+        "drop_intermediate": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def roc_curve(
+    y_true, y_score, *, pos_label=None, sample_weight=None, drop_intermediate=True
+):
     """Compute Receiver operating characteristic (ROC).
 
     Note: this implementation is restricted to the binary classification task.
@@ -834,16 +1077,18 @@ def roc_curve(y_true, y_score, *, pos_label=None, sample_weight=None,
 
     Parameters
     ----------
-    y_true : ndarray of shape (n_samples,)
+    y_true : array-like of shape (n_samples,)
         True binary labels. If labels are not either {-1, 1} or {0, 1}, then
         pos_label should be explicitly given.
 
-    y_score : ndarray of shape (n_samples,)
+    y_score : array-like of shape (n_samples,)
         Target scores, can either be probability estimates of the positive
         class, confidence values, or non-thresholded measure of decisions
         (as returned by "decision_function" on some classifiers).
+        For :term:`decision_function` scores, values greater than or equal to
+        zero should indicate the positive class.
 
-    pos_label : int or str, default=None
+    pos_label : int, float, bool or str, default=None
         The label of the positive class.
         When ``pos_label=None``, if `y_true` is in {-1, 1} or {0, 1},
         ``pos_label`` is set to 1, otherwise an error will be raised.
@@ -852,9 +1097,9 @@ def roc_curve(y_true, y_score, *, pos_label=None, sample_weight=None,
         Sample weights.
 
     drop_intermediate : bool, default=True
-        Whether to drop some suboptimal thresholds which would not appear
-        on a plotted ROC curve. This is useful in order to create lighter
-        ROC curves.
+        Whether to drop thresholds where the resulting point is collinear with
+        its neighbors in ROC space. This has no effect on the ROC AUC or visual
+        shape of the curve, but reduces the number of plotted points.
 
         .. versionadded:: 0.17
            parameter *drop_intermediate*.
@@ -869,15 +1114,21 @@ def roc_curve(y_true, y_score, *, pos_label=None, sample_weight=None,
         Increasing true positive rates such that element `i` is the true
         positive rate of predictions with score >= `thresholds[i]`.
 
-    thresholds : ndarray of shape = (n_thresholds,)
+    thresholds : ndarray of shape (n_thresholds,)
         Decreasing thresholds on the decision function used to compute
-        fpr and tpr. `thresholds[0]` represents no instances being predicted
-        and is arbitrarily set to `max(y_score) + 1`.
+        fpr and tpr. The first threshold is set to `np.inf`.
+
+        .. versionchanged:: 1.3
+           An arbitrary threshold at infinity (stored in `thresholds[0]`) is
+           added to represent a classifier that always predicts the negative
+           class, i.e. `fpr=0` and `tpr=0`.
 
     See Also
     --------
-    plot_roc_curve : Plot Receiver operating characteristic (ROC) curve.
-    RocCurveDisplay : ROC Curve visualization.
+    RocCurveDisplay.from_estimator : Plot Receiver Operating Characteristic
+        (ROC) curve given an estimator and some data.
+    RocCurveDisplay.from_predictions : Plot Receiver Operating Characteristic
+        (ROC) curve given the true and predicted values.
     det_curve: Compute error rates for different probability thresholds.
     roc_auc_score : Compute the area under the ROC curve.
 
@@ -907,11 +1158,11 @@ def roc_curve(y_true, y_score, *, pos_label=None, sample_weight=None,
     >>> tpr
     array([0. , 0.5, 0.5, 1. , 1. ])
     >>> thresholds
-    array([1.8 , 0.8 , 0.4 , 0.35, 0.1 ])
-
+    array([ inf, 0.8 , 0.4 , 0.35, 0.1 ])
     """
     fps, tps, thresholds = _binary_clf_curve(
-        y_true, y_score, pos_label=pos_label, sample_weight=sample_weight)
+        y_true, y_score, pos_label=pos_label, sample_weight=sample_weight
+    )
 
     # Attempt to drop thresholds corresponding to points in between and
     # collinear with other points. These are always suboptimal and do not
@@ -923,10 +1174,9 @@ def roc_curve(y_true, y_score, *, pos_label=None, sample_weight=None,
     # but does not drop more complicated cases like fps = [1, 3, 7],
     # tps = [1, 2, 4]; there is no harm in keeping too many thresholds.
     if drop_intermediate and len(fps) > 2:
-        optimal_idxs = np.where(np.r_[True,
-                                      np.logical_or(np.diff(fps, 2),
-                                                    np.diff(tps, 2)),
-                                      True])[0]
+        optimal_idxs = np.where(
+            np.r_[True, np.logical_or(np.diff(fps, 2), np.diff(tps, 2)), True]
+        )[0]
         fps = fps[optimal_idxs]
         tps = tps[optimal_idxs]
         thresholds = thresholds[optimal_idxs]
@@ -935,20 +1185,23 @@ def roc_curve(y_true, y_score, *, pos_label=None, sample_weight=None,
     # to make sure that the curve starts at (0, 0)
     tps = np.r_[0, tps]
     fps = np.r_[0, fps]
-    thresholds = np.r_[thresholds[0] + 1, thresholds]
+    # get dtype of `y_score` even if it is an array-like
+    thresholds = np.r_[np.inf, thresholds]
 
     if fps[-1] <= 0:
-        warnings.warn("No negative samples in y_true, "
-                      "false positive value should be meaningless",
-                      UndefinedMetricWarning)
+        warnings.warn(
+            "No negative samples in y_true, false positive value should be meaningless",
+            UndefinedMetricWarning,
+        )
         fpr = np.repeat(np.nan, fps.shape)
     else:
         fpr = fps / fps[-1]
 
     if tps[-1] <= 0:
-        warnings.warn("No positive samples in y_true, "
-                      "true positive value should be meaningless",
-                      UndefinedMetricWarning)
+        warnings.warn(
+            "No positive samples in y_true, true positive value should be meaningless",
+            UndefinedMetricWarning,
+        )
         tpr = np.repeat(np.nan, tps.shape)
     else:
         tpr = tps / tps[-1]
@@ -956,9 +1209,15 @@ def roc_curve(y_true, y_score, *, pos_label=None, sample_weight=None,
     return fpr, tpr, thresholds
 
 
-@_deprecate_positional_args
-def label_ranking_average_precision_score(y_true, y_score, *,
-                                          sample_weight=None):
+@validate_params(
+    {
+        "y_true": ["array-like", "sparse matrix"],
+        "y_score": ["array-like"],
+        "sample_weight": ["array-like", None],
+    },
+    prefer_skip_nested_validation=True,
+)
+def label_ranking_average_precision_score(y_true, y_score, *, sample_weight=None):
     """Compute ranking-based average precision.
 
     Label ranking average precision (LRAP) is the average over each ground
@@ -975,13 +1234,15 @@ def label_ranking_average_precision_score(y_true, y_score, *,
 
     Parameters
     ----------
-    y_true : {ndarray, sparse matrix} of shape (n_samples, n_labels)
+    y_true : {array-like, sparse matrix} of shape (n_samples, n_labels)
         True binary labels in binary indicator format.
 
-    y_score : ndarray of shape (n_samples, n_labels)
+    y_score : array-like of shape (n_samples, n_labels)
         Target scores, can either be probability estimates of the positive
         class, confidence values, or non-thresholded measure of decisions
         (as returned by "decision_function" on some classifiers).
+        For :term:`decision_function` scores, values greater than or equal to
+        zero should indicate the positive class.
 
     sample_weight : array-like of shape (n_samples,), default=None
         Sample weights.
@@ -991,6 +1252,7 @@ def label_ranking_average_precision_score(y_true, y_score, *,
     Returns
     -------
     score : float
+        Ranking-based average precision score.
 
     Examples
     --------
@@ -999,39 +1261,41 @@ def label_ranking_average_precision_score(y_true, y_score, *,
     >>> y_true = np.array([[1, 0, 0], [0, 0, 1]])
     >>> y_score = np.array([[0.75, 0.5, 1], [1, 0.2, 0.1]])
     >>> label_ranking_average_precision_score(y_true, y_score)
-    0.416...
-
+    0.416
     """
     check_consistent_length(y_true, y_score, sample_weight)
-    y_true = check_array(y_true, ensure_2d=False)
+    y_true = check_array(y_true, ensure_2d=False, accept_sparse="csr")
     y_score = check_array(y_score, ensure_2d=False)
 
     if y_true.shape != y_score.shape:
         raise ValueError("y_true and y_score have different shape")
 
     # Handle badly formatted array and the degenerate case with one label
-    y_type = type_of_target(y_true)
-    if (y_type != "multilabel-indicator" and
-            not (y_type == "binary" and y_true.ndim == 2)):
+    y_type = type_of_target(y_true, input_name="y_true")
+    if y_type != "multilabel-indicator" and not (
+        y_type == "binary" and y_true.ndim == 2
+    ):
         raise ValueError("{0} format is not supported".format(y_type))
 
-    y_true = csr_matrix(y_true)
+    if not issparse(y_true):
+        y_true = csr_matrix(y_true)
+
     y_score = -y_score
 
     n_samples, n_labels = y_true.shape
 
-    out = 0.
+    out = 0.0
     for i, (start, stop) in enumerate(zip(y_true.indptr, y_true.indptr[1:])):
         relevant = y_true.indices[start:stop]
 
-        if (relevant.size == 0 or relevant.size == n_labels):
+        if relevant.size == 0 or relevant.size == n_labels:
             # If all labels are relevant or unrelevant, the score is also
             # equal to 1. The label ranking has no meaning.
-            aux = 1.
+            aux = 1.0
         else:
             scores_i = y_score[i]
-            rank = rankdata(scores_i, 'max')[relevant]
-            L = rankdata(scores_i[relevant], 'max')
+            rank = rankdata(scores_i, "max")[relevant]
+            L = rankdata(scores_i[relevant], "max")
             aux = (L / rank).mean()
 
         if sample_weight is not None:
@@ -1043,10 +1307,17 @@ def label_ranking_average_precision_score(y_true, y_score, *,
     else:
         out /= np.sum(sample_weight)
 
-    return out
+    return float(out)
 
 
-@_deprecate_positional_args
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_score": ["array-like"],
+        "sample_weight": ["array-like", None],
+    },
+    prefer_skip_nested_validation=True,
+)
 def coverage_error(y_true, y_score, *, sample_weight=None):
     """Coverage error measure.
 
@@ -1065,13 +1336,15 @@ def coverage_error(y_true, y_score, *, sample_weight=None):
 
     Parameters
     ----------
-    y_true : ndarray of shape (n_samples, n_labels)
+    y_true : array-like of shape (n_samples, n_labels)
         True binary labels in binary indicator format.
 
-    y_score : ndarray of shape (n_samples, n_labels)
+    y_score : array-like of shape (n_samples, n_labels)
         Target scores, can either be probability estimates of the positive
         class, confidence values, or non-thresholded measure of decisions
         (as returned by "decision_function" on some classifiers).
+        For :term:`decision_function` scores, values greater than or equal to
+        zero should indicate the positive class.
 
     sample_weight : array-like of shape (n_samples,), default=None
         Sample weights.
@@ -1079,6 +1352,7 @@ def coverage_error(y_true, y_score, *, sample_weight=None):
     Returns
     -------
     coverage_error : float
+        The coverage error.
 
     References
     ----------
@@ -1086,12 +1360,19 @@ def coverage_error(y_true, y_score, *, sample_weight=None):
            Mining multi-label data. In Data mining and knowledge discovery
            handbook (pp. 667-685). Springer US.
 
+    Examples
+    --------
+    >>> from sklearn.metrics import coverage_error
+    >>> y_true = [[1, 0, 0], [0, 1, 1]]
+    >>> y_score = [[1, 0, 0], [0, 1, 1]]
+    >>> coverage_error(y_true, y_score)
+    1.5
     """
-    y_true = check_array(y_true, ensure_2d=False)
-    y_score = check_array(y_score, ensure_2d=False)
+    y_true = check_array(y_true, ensure_2d=True)
+    y_score = check_array(y_score, ensure_2d=True)
     check_consistent_length(y_true, y_score, sample_weight)
 
-    y_type = type_of_target(y_true)
+    y_type = type_of_target(y_true, input_name="y_true")
     if y_type != "multilabel-indicator":
         raise ValueError("{0} format is not supported".format(y_type))
 
@@ -1103,10 +1384,17 @@ def coverage_error(y_true, y_score, *, sample_weight=None):
     coverage = (y_score >= y_min_relevant).sum(axis=1)
     coverage = coverage.filled(0)
 
-    return np.average(coverage, weights=sample_weight)
+    return float(np.average(coverage, weights=sample_weight))
 
 
-@_deprecate_positional_args
+@validate_params(
+    {
+        "y_true": ["array-like", "sparse matrix"],
+        "y_score": ["array-like"],
+        "sample_weight": ["array-like", None],
+    },
+    prefer_skip_nested_validation=True,
+)
 def label_ranking_loss(y_true, y_score, *, sample_weight=None):
     """Compute Ranking loss measure.
 
@@ -1125,13 +1413,15 @@ def label_ranking_loss(y_true, y_score, *, sample_weight=None):
 
     Parameters
     ----------
-    y_true : {ndarray, sparse matrix} of shape (n_samples, n_labels)
+    y_true : {array-like, sparse matrix} of shape (n_samples, n_labels)
         True binary labels in binary indicator format.
 
-    y_score : ndarray of shape (n_samples, n_labels)
+    y_score : array-like of shape (n_samples, n_labels)
         Target scores, can either be probability estimates of the positive
         class, confidence values, or non-thresholded measure of decisions
         (as returned by "decision_function" on some classifiers).
+        For :term:`decision_function` scores, values greater than or equal to
+        zero should indicate the positive class.
 
     sample_weight : array-like of shape (n_samples,), default=None
         Sample weights.
@@ -1139,18 +1429,29 @@ def label_ranking_loss(y_true, y_score, *, sample_weight=None):
     Returns
     -------
     loss : float
+        Average number of label pairs that are incorrectly ordered given
+        y_score weighted by the size of the label set and the number of labels not
+        in the label set.
 
     References
     ----------
     .. [1] Tsoumakas, G., Katakis, I., & Vlahavas, I. (2010).
            Mining multi-label data. In Data mining and knowledge discovery
            handbook (pp. 667-685). Springer US.
+
+    Examples
+    --------
+    >>> from sklearn.metrics import label_ranking_loss
+    >>> y_true = [[1, 0, 0], [0, 0, 1]]
+    >>> y_score = [[0.75, 0.5, 1], [1, 0.2, 0.1]]
+    >>> label_ranking_loss(y_true, y_score)
+    0.75
     """
-    y_true = check_array(y_true, ensure_2d=False, accept_sparse='csr')
+    y_true = check_array(y_true, ensure_2d=False, accept_sparse="csr")
     y_score = check_array(y_score, ensure_2d=False)
     check_consistent_length(y_true, y_score, sample_weight)
 
-    y_type = type_of_target(y_true)
+    y_type = type_of_target(y_true, input_name="y_true")
     if y_type not in ("multilabel-indicator",):
         raise ValueError("{0} format is not supported".format(y_type))
 
@@ -1164,35 +1465,31 @@ def label_ranking_loss(y_true, y_score, *, sample_weight=None):
     loss = np.zeros(n_samples)
     for i, (start, stop) in enumerate(zip(y_true.indptr, y_true.indptr[1:])):
         # Sort and bin the label scores
-        unique_scores, unique_inverse = np.unique(y_score[i],
-                                                  return_inverse=True)
+        unique_scores, unique_inverse = np.unique(y_score[i], return_inverse=True)
         true_at_reversed_rank = np.bincount(
-            unique_inverse[y_true.indices[start:stop]],
-            minlength=len(unique_scores))
-        all_at_reversed_rank = np.bincount(unique_inverse,
-                                           minlength=len(unique_scores))
+            unique_inverse[y_true.indices[start:stop]], minlength=len(unique_scores)
+        )
+        all_at_reversed_rank = np.bincount(unique_inverse, minlength=len(unique_scores))
         false_at_reversed_rank = all_at_reversed_rank - true_at_reversed_rank
 
         # if the scores are ordered, it's possible to count the number of
         # incorrectly ordered paires in linear time by cumulatively counting
         # how many false labels of a given score have a score higher than the
         # accumulated true labels with lower score.
-        loss[i] = np.dot(true_at_reversed_rank.cumsum(),
-                         false_at_reversed_rank)
+        loss[i] = np.dot(true_at_reversed_rank.cumsum(), false_at_reversed_rank)
 
     n_positives = count_nonzero(y_true, axis=1)
     with np.errstate(divide="ignore", invalid="ignore"):
-        loss /= ((n_labels - n_positives) * n_positives)
+        loss /= (n_labels - n_positives) * n_positives
 
     # When there is no positive or no negative labels, those values should
     # be consider as correct, i.e. the ranking doesn't matter.
-    loss[np.logical_or(n_positives == 0, n_positives == n_labels)] = 0.
+    loss[np.logical_or(n_positives == 0, n_positives == n_labels)] = 0.0
 
-    return np.average(loss, weights=sample_weight)
+    return float(np.average(loss, weights=sample_weight))
 
 
-def _dcg_sample_scores(y_true, y_score, k=None,
-                       log_base=2, ignore_ties=False):
+def _dcg_sample_scores(y_true, y_score, k=None, log_base=2, ignore_ties=False):
     """Compute Discounted Cumulative Gain.
 
     Sum the true scores ranked in the order induced by the predicted scores,
@@ -1213,7 +1510,7 @@ def _dcg_sample_scores(y_true, y_score, k=None,
         "decision_function" on some classifiers).
 
     k : int, default=None
-        Only consider the highest k scores in the ranking. If None, use all
+        Only consider the highest k scores in the ranking. If `None`, use all
         outputs.
 
     log_base : float, default=2
@@ -1244,8 +1541,10 @@ def _dcg_sample_scores(y_true, y_score, k=None,
         cumulative_gains = discount.dot(ranked.T)
     else:
         discount_cumsum = np.cumsum(discount)
-        cumulative_gains = [_tie_averaged_dcg(y_t, y_s, discount_cumsum)
-                            for y_t, y_s in zip(y_true, y_score)]
+        cumulative_gains = [
+            _tie_averaged_dcg(y_t, y_s, discount_cumsum)
+            for y_t, y_s in zip(y_true, y_score)
+        ]
         cumulative_gains = np.asarray(cumulative_gains)
     return cumulative_gains
 
@@ -1287,8 +1586,7 @@ def _tie_averaged_dcg(y_true, y_score, discount_cumsum):
     European conference on information retrieval (pp. 414-421). Springer,
     Berlin, Heidelberg.
     """
-    _, inv, counts = np.unique(
-        - y_score, return_inverse=True, return_counts=True)
+    _, inv, counts = np.unique(-y_score, return_inverse=True, return_counts=True)
     ranked = np.zeros(len(counts))
     np.add.at(ranked, inv, y_true)
     ranked /= counts
@@ -1300,18 +1598,34 @@ def _tie_averaged_dcg(y_true, y_score, discount_cumsum):
 
 
 def _check_dcg_target_type(y_true):
-    y_type = type_of_target(y_true)
-    supported_fmt = ("multilabel-indicator", "continuous-multioutput",
-                     "multiclass-multioutput")
+    y_type = type_of_target(y_true, input_name="y_true")
+    supported_fmt = (
+        "multilabel-indicator",
+        "continuous-multioutput",
+        "multiclass-multioutput",
+    )
     if y_type not in supported_fmt:
         raise ValueError(
             "Only {} formats are supported. Got {} instead".format(
-                supported_fmt, y_type))
+                supported_fmt, y_type
+            )
+        )
 
 
-@_deprecate_positional_args
-def dcg_score(y_true, y_score, *, k=None,
-              log_base=2, sample_weight=None, ignore_ties=False):
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_score": ["array-like"],
+        "k": [Interval(Integral, 1, None, closed="left"), None],
+        "log_base": [Interval(Real, 0.0, None, closed="neither")],
+        "sample_weight": ["array-like", None],
+        "ignore_ties": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def dcg_score(
+    y_true, y_score, *, k=None, log_base=2, sample_weight=None, ignore_ties=False
+):
     """Compute Discounted Cumulative Gain.
 
     Sum the true scores ranked in the order induced by the predicted scores,
@@ -1325,11 +1639,11 @@ def dcg_score(y_true, y_score, *, k=None,
 
     Parameters
     ----------
-    y_true : ndarray of shape (n_samples, n_labels)
+    y_true : array-like of shape (n_samples, n_labels)
         True targets of multilabel classification, or true scores of entities
         to be ranked.
 
-    y_score : ndarray of shape (n_samples, n_labels)
+    y_score : array-like of shape (n_samples, n_labels)
         Target scores, can either be probability estimates, confidence values,
         or non-thresholded measure of decisions (as returned by
         "decision_function" on some classifiers).
@@ -1342,8 +1656,8 @@ def dcg_score(y_true, y_score, *, k=None,
         Base of the logarithm used for the discount. A low value means a
         sharper discount (top results are more important).
 
-    sample_weight : ndarray of shape (n_samples,), default=None
-        Sample weights. If None, all samples are given the same weight.
+    sample_weight : array-like of shape (n_samples,), default=None
+        Sample weights. If `None`, all samples are given the same weight.
 
     ignore_ties : bool, default=False
         Assume that there are no ties in y_score (which is likely to be the
@@ -1380,16 +1694,17 @@ def dcg_score(y_true, y_score, *, k=None,
 
     Examples
     --------
+    >>> import numpy as np
     >>> from sklearn.metrics import dcg_score
-    >>> # we have groud-truth relevance of some answers to a query:
+    >>> # we have ground-truth relevance of some answers to a query:
     >>> true_relevance = np.asarray([[10, 0, 0, 1, 5]])
     >>> # we predict scores for the answers
     >>> scores = np.asarray([[.1, .2, .3, 4, 70]])
     >>> dcg_score(true_relevance, scores)
-    9.49...
+    9.49
     >>> # we can set k to truncate the sum; only top k answers contribute
     >>> dcg_score(true_relevance, scores, k=2)
-    5.63...
+    5.63
     >>> # now we have some ties in our prediction
     >>> scores = np.asarray([[1, 0, 0, 0, 1]])
     >>> # by default ties are averaged, so here we get the average true
@@ -1402,17 +1717,19 @@ def dcg_score(y_true, y_score, *, k=None,
     >>> dcg_score(true_relevance,
     ...           scores, k=1, ignore_ties=True)
     5.0
-
     """
     y_true = check_array(y_true, ensure_2d=False)
     y_score = check_array(y_score, ensure_2d=False)
     check_consistent_length(y_true, y_score, sample_weight)
     _check_dcg_target_type(y_true)
-    return np.average(
-        _dcg_sample_scores(
-            y_true, y_score, k=k, log_base=log_base,
-            ignore_ties=ignore_ties),
-        weights=sample_weight)
+    return float(
+        np.average(
+            _dcg_sample_scores(
+                y_true, y_score, k=k, log_base=log_base, ignore_ties=ignore_ties
+            ),
+            weights=sample_weight,
+        )
+    )
 
 
 def _ndcg_sample_scores(y_true, y_score, k=None, ignore_ties=False):
@@ -1466,9 +1783,17 @@ def _ndcg_sample_scores(y_true, y_score, k=None, ignore_ties=False):
     return gain
 
 
-@_deprecate_positional_args
-def ndcg_score(y_true, y_score, *, k=None, sample_weight=None,
-               ignore_ties=False):
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_score": ["array-like"],
+        "k": [Interval(Integral, 1, None, closed="left"), None],
+        "sample_weight": ["array-like", None],
+        "ignore_ties": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def ndcg_score(y_true, y_score, *, k=None, sample_weight=None, ignore_ties=False):
     """Compute Normalized Discounted Cumulative Gain.
 
     Sum the true scores ranked in the order induced by the predicted scores,
@@ -1476,26 +1801,27 @@ def ndcg_score(y_true, y_score, *, k=None, sample_weight=None,
     score (Ideal DCG, obtained for a perfect ranking) to obtain a score between
     0 and 1.
 
-    This ranking metric yields a high value if true labels are ranked high by
+    This ranking metric returns a high value if true labels are ranked high by
     ``y_score``.
 
     Parameters
     ----------
-    y_true : ndarray of shape (n_samples, n_labels)
+    y_true : array-like of shape (n_samples, n_labels)
         True targets of multilabel classification, or true scores of entities
-        to be ranked.
+        to be ranked. Negative values in `y_true` may result in an output
+        that is not between 0 and 1.
 
-    y_score : ndarray of shape (n_samples, n_labels)
+    y_score : array-like of shape (n_samples, n_labels)
         Target scores, can either be probability estimates, confidence values,
         or non-thresholded measure of decisions (as returned by
         "decision_function" on some classifiers).
 
     k : int, default=None
-        Only consider the highest k scores in the ranking. If None, use all
+        Only consider the highest k scores in the ranking. If `None`, use all
         outputs.
 
-    sample_weight : ndarray of shape (n_samples,), default=None
-        Sample weights. If None, all samples are given the same weight.
+    sample_weight : array-like of shape (n_samples,), default=None
+        Sample weights. If `None`, all samples are given the same weight.
 
     ignore_ties : bool, default=False
         Assume that there are no ties in y_score (which is likely to be the
@@ -1530,23 +1856,24 @@ def ndcg_score(y_true, y_score, *, k=None, sample_weight=None,
 
     Examples
     --------
+    >>> import numpy as np
     >>> from sklearn.metrics import ndcg_score
-    >>> # we have groud-truth relevance of some answers to a query:
+    >>> # we have ground-truth relevance of some answers to a query:
     >>> true_relevance = np.asarray([[10, 0, 0, 1, 5]])
     >>> # we predict some scores (relevance) for the answers
     >>> scores = np.asarray([[.1, .2, .3, 4, 70]])
     >>> ndcg_score(true_relevance, scores)
-    0.69...
+    0.69
     >>> scores = np.asarray([[.05, 1.1, 1., .5, .0]])
     >>> ndcg_score(true_relevance, scores)
-    0.49...
+    0.49
     >>> # we can set k to truncate the sum; only top k answers contribute.
     >>> ndcg_score(true_relevance, scores, k=4)
-    0.35...
+    0.35
     >>> # the normalization takes k into account so a perfect answer
     >>> # would still get 1.0
     >>> ndcg_score(true_relevance, true_relevance, k=4)
-    1.0
+    1.0...
     >>> # now we have some ties in our prediction
     >>> scores = np.asarray([[1, 0, 0, 0, 1]])
     >>> # by default ties are averaged, so here we get the average (normalized)
@@ -1558,19 +1885,38 @@ def ndcg_score(y_true, y_score, *, k=None, sample_weight=None,
     >>> # wrong results:
     >>> ndcg_score(true_relevance,
     ...           scores, k=1, ignore_ties=True)
-    0.5
-
+    0.5...
     """
     y_true = check_array(y_true, ensure_2d=False)
     y_score = check_array(y_score, ensure_2d=False)
     check_consistent_length(y_true, y_score, sample_weight)
+
+    if y_true.min() < 0:
+        raise ValueError("ndcg_score should not be used on negative y_true values.")
+    if y_true.ndim > 1 and y_true.shape[1] <= 1:
+        raise ValueError(
+            "Computing NDCG is only meaningful when there is more than 1 document. "
+            f"Got {y_true.shape[1]} instead."
+        )
     _check_dcg_target_type(y_true)
     gain = _ndcg_sample_scores(y_true, y_score, k=k, ignore_ties=ignore_ties)
-    return np.average(gain, weights=sample_weight)
-
-
-def top_k_accuracy_score(y_true, y_score, *, k=2, normalize=True,
-                         sample_weight=None, labels=None):
+    return float(np.average(gain, weights=sample_weight))
+
+
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_score": ["array-like"],
+        "k": [Interval(Integral, 1, None, closed="left")],
+        "normalize": ["boolean"],
+        "sample_weight": ["array-like", None],
+        "labels": ["array-like", None],
+    },
+    prefer_skip_nested_validation=True,
+)
+def top_k_accuracy_score(
+    y_true, y_score, *, k=2, normalize=True, sample_weight=None, labels=None
+):
     """Top-k Accuracy classification score.
 
     This metric computes the number of times where the correct label is among
@@ -1587,12 +1933,14 @@ def top_k_accuracy_score(y_true, y_score, *, k=2, normalize=True,
     y_score : array-like of shape (n_samples,) or (n_samples, n_classes)
         Target scores. These can be either probability estimates or
         non-thresholded decision values (as returned by
-        :term:`decision_function` on some classifiers). The binary case expects
-        scores with shape (n_samples,) while the multiclass case expects scores
-        with shape (n_samples, n_classes). In the nulticlass case, the order of
-        the class scores must correspond to the order of ``labels``, if
-        provided, or else to the numerical or lexicographical order of the
-        labels in ``y_true``.
+        :term:`decision_function` on some classifiers).
+        The binary case expects scores with shape (n_samples,) while the
+        multiclass case expects scores with shape (n_samples, n_classes).
+        In the multiclass case, the order of the class scores must
+        correspond to the order of ``labels``, if provided, or else to
+        the numerical or lexicographical order of the labels in ``y_true``.
+        If ``y_true`` does not contain all the labels, ``labels`` must be
+        provided.
 
     k : int, default=2
         Number of most likely outcomes considered to find the correct label.
@@ -1607,7 +1955,8 @@ def top_k_accuracy_score(y_true, y_score, *, k=2, normalize=True,
     labels : array-like of shape (n_classes,), default=None
         Multiclass only. List of labels that index the classes in ``y_score``.
         If ``None``, the numerical or lexicographical order of the labels in
-        ``y_true`` is used.
+        ``y_true`` is used. If ``y_true`` does not contain all the labels,
+        ``labels`` must be provided.
 
     Returns
     -------
@@ -1616,9 +1965,11 @@ def top_k_accuracy_score(y_true, y_score, *, k=2, normalize=True,
         `normalize == True` and the number of samples with
         `normalize == False`.
 
-    See also
+    See Also
     --------
-    accuracy_score
+    accuracy_score : Compute the accuracy score. By default, the function will
+        return the fraction of correct predictions divided by the total number
+        of predictions.
 
     Notes
     -----
@@ -1640,21 +1991,28 @@ def top_k_accuracy_score(y_true, y_score, *, k=2, normalize=True,
     0.75
     >>> # Not normalizing gives the number of "correctly" classified samples
     >>> top_k_accuracy_score(y_true, y_score, k=2, normalize=False)
-    3
-
+    3.0
     """
     y_true = check_array(y_true, ensure_2d=False, dtype=None)
     y_true = column_or_1d(y_true)
-    y_type = type_of_target(y_true)
-    y_score = check_array(y_score, ensure_2d=False)
-    y_score = column_or_1d(y_score) if y_type == 'binary' else y_score
-    check_consistent_length(y_true, y_score, sample_weight)
-
-    if y_type not in {'binary', 'multiclass'}:
+    y_type = type_of_target(y_true, input_name="y_true")
+    if y_type == "binary" and labels is not None and len(labels) > 2:
+        y_type = "multiclass"
+    if y_type not in {"binary", "multiclass"}:
         raise ValueError(
             f"y type must be 'binary' or 'multiclass', got '{y_type}' instead."
         )
+    y_score = check_array(y_score, ensure_2d=False)
+    if y_type == "binary":
+        if y_score.ndim == 2 and y_score.shape[1] != 1:
+            raise ValueError(
+                "`y_true` is binary while y_score is 2d with"
+                f" {y_score.shape[1]} classes. If `y_true` does not contain all the"
+                " labels, `labels` must be provided."
+            )
+        y_score = column_or_1d(y_score)
 
+    check_consistent_length(y_true, y_score, sample_weight)
     y_score_n_classes = y_score.shape[1] if y_score.ndim == 2 else 2
 
     if labels is None:
@@ -1665,6 +2023,8 @@ def top_k_accuracy_score(y_true, y_score, *, k=2, normalize=True,
             raise ValueError(
                 f"Number of classes in 'y_true' ({n_classes}) not equal "
                 f"to the number of classes in 'y_score' ({y_score_n_classes})."
+                "You can provide a list of all known classes by assigning it "
+                "to the `labels` parameter."
             )
     else:
         labels = column_or_1d(labels)
@@ -1685,33 +2045,33 @@ def top_k_accuracy_score(y_true, y_score, *, k=2, normalize=True,
             )
 
         if len(np.setdiff1d(y_true, classes)):
-            raise ValueError(
-                "'y_true' contains labels not in parameter 'labels'."
-            )
+            raise ValueError("'y_true' contains labels not in parameter 'labels'.")
 
     if k >= n_classes:
         warnings.warn(
-            f"'k' ({k}) greater than or equal to 'n_classes' ({n_classes}) "
-            "will result in a perfect score and is therefore meaningless.",
-            UndefinedMetricWarning
+            (
+                f"'k' ({k}) greater than or equal to 'n_classes' ({n_classes}) "
+                "will result in a perfect score and is therefore meaningless."
+            ),
+            UndefinedMetricWarning,
         )
 
     y_true_encoded = _encode(y_true, uniques=classes)
 
-    if y_type == 'binary':
+    if y_type == "binary":
         if k == 1:
-            threshold = .5 if y_score.min() >= 0 and y_score.max() <= 1 else 0
+            threshold = 0.5 if y_score.min() >= 0 and y_score.max() <= 1 else 0
             y_pred = (y_score > threshold).astype(np.int64)
             hits = y_pred == y_true_encoded
         else:
             hits = np.ones_like(y_score, dtype=np.bool_)
-    elif y_type == 'multiclass':
-        sorted_pred = np.argsort(y_score, axis=1, kind='mergesort')[:, ::-1]
+    elif y_type == "multiclass":
+        sorted_pred = np.argsort(y_score, axis=1, kind="mergesort")[:, ::-1]
         hits = (y_true_encoded == sorted_pred[:, :k].T).any(axis=0)
 
     if normalize:
-        return np.average(hits, weights=sample_weight)
+        return float(np.average(hits, weights=sample_weight))
     elif sample_weight is None:
-        return np.sum(hits)
+        return float(np.sum(hits))
     else:
-        return np.dot(hits, sample_weight)
+        return float(np.dot(hits, sample_weight))
diff --git a/sklearn/metrics/_regression.py b/sklearn/metrics/_regression.py
index 7edf7924e50e1..e7435756c52b2 100644
--- a/sklearn/metrics/_regression.py
+++ b/sklearn/metrics/_regression.py
@@ -7,34 +7,35 @@
 the lower the better.
 """
 
-# Authors: Alexandre Gramfort 
-#          Mathieu Blondel 
-#          Olivier Grisel 
-#          Arnaud Joly 
-#          Jochen Wersdorfer 
-#          Lars Buitinck
-#          Joel Nothman 
-#          Karan Desai 
-#          Noel Dawe 
-#          Manoj Kumar 
-#          Michael Eickenberg 
-#          Konstantin Shmelkov 
-#          Christian Lorentzen 
-#          Ashutosh Hathidara 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-import numpy as np
 import warnings
+from numbers import Real
 
-from .._loss.glm_distribution import TweedieDistribution
-from ..utils.validation import (check_array, check_consistent_length,
-                                _num_samples)
-from ..utils.validation import column_or_1d
-from ..utils.validation import _deprecate_positional_args
-from ..utils.validation import _check_sample_weight
-from ..utils.stats import _weighted_percentile
-from ..exceptions import UndefinedMetricWarning
+import numpy as np
 
+from ..exceptions import UndefinedMetricWarning
+from ..utils._array_api import (
+    _average,
+    _find_matching_floating_dtype,
+    _median,
+    get_namespace,
+    get_namespace_and_device,
+    size,
+)
+from ..utils._array_api import (
+    _xlogy as xlogy,
+)
+from ..utils._param_validation import Interval, StrOptions, validate_params
+from ..utils.stats import _weighted_percentile
+from ..utils.validation import (
+    _check_sample_weight,
+    _num_samples,
+    check_array,
+    check_consistent_length,
+    column_or_1d,
+)
 
 __ALL__ = [
     "max_error",
@@ -45,26 +46,49 @@
     "mean_absolute_percentage_error",
     "mean_pinball_loss",
     "r2_score",
+    "root_mean_squared_log_error",
+    "root_mean_squared_error",
     "explained_variance_score",
     "mean_tweedie_deviance",
     "mean_poisson_deviance",
     "mean_gamma_deviance",
+    "d2_tweedie_score",
+    "d2_pinball_score",
+    "d2_absolute_error_score",
 ]
 
 
-def _check_reg_targets(y_true, y_pred, multioutput, dtype="numeric"):
-    """Check that y_true and y_pred belong to the same regression task.
+def _check_reg_targets(
+    y_true, y_pred, sample_weight, multioutput, dtype="numeric", xp=None
+):
+    """Check that y_true, y_pred and sample_weight belong to the same regression task.
+
+    To reduce redundancy when calling `_find_matching_floating_dtype`,
+    please use `_check_reg_targets_with_floating_dtype` instead.
 
     Parameters
     ----------
-    y_true : array-like
+    y_true : array-like of shape (n_samples,) or (n_samples, n_outputs)
+        Ground truth (correct) target values.
 
-    y_pred : array-like
+    y_pred : array-like of shape (n_samples,) or (n_samples, n_outputs)
+        Estimated target values.
+
+    sample_weight : array-like of shape (n_samples,) or None
+        Sample weights.
 
     multioutput : array-like or string in ['raw_values', uniform_average',
         'variance_weighted'] or None
         None is accepted due to backward compatibility of r2_score().
 
+    dtype : str or list, default="numeric"
+        the dtype argument passed to check_array.
+
+    xp : module, default=None
+        Precomputed array namespace module. When passed, typically from a caller
+        that has already performed inspection of its own inputs, skips array
+        namespace inspection.
+
     Returns
     -------
     type_true : one of {'continuous', continuous-multioutput'}
@@ -77,59 +101,135 @@ def _check_reg_targets(y_true, y_pred, multioutput, dtype="numeric"):
     y_pred : array-like of shape (n_samples, n_outputs)
         Estimated target values.
 
+    sample_weight : array-like of shape (n_samples,) or None
+        Sample weights.
+
     multioutput : array-like of shape (n_outputs) or string in ['raw_values',
         uniform_average', 'variance_weighted'] or None
         Custom output weights if ``multioutput`` is array-like or
         just the corresponding argument if ``multioutput`` is a
         correct keyword.
-
-    dtype : str or list, default="numeric"
-        the dtype argument passed to check_array.
     """
-    check_consistent_length(y_true, y_pred)
+    xp, _ = get_namespace(y_true, y_pred, multioutput, xp=xp)
+
+    check_consistent_length(y_true, y_pred, sample_weight)
     y_true = check_array(y_true, ensure_2d=False, dtype=dtype)
     y_pred = check_array(y_pred, ensure_2d=False, dtype=dtype)
+    if sample_weight is not None:
+        sample_weight = _check_sample_weight(sample_weight, y_true, dtype=dtype)
 
     if y_true.ndim == 1:
-        y_true = y_true.reshape((-1, 1))
+        y_true = xp.reshape(y_true, (-1, 1))
 
     if y_pred.ndim == 1:
-        y_pred = y_pred.reshape((-1, 1))
+        y_pred = xp.reshape(y_pred, (-1, 1))
 
     if y_true.shape[1] != y_pred.shape[1]:
-        raise ValueError("y_true and y_pred have different number of output "
-                         "({0}!={1})".format(y_true.shape[1], y_pred.shape[1]))
+        raise ValueError(
+            "y_true and y_pred have different number of output ({0}!={1})".format(
+                y_true.shape[1], y_pred.shape[1]
+            )
+        )
 
     n_outputs = y_true.shape[1]
-    allowed_multioutput_str = ('raw_values', 'uniform_average',
-                               'variance_weighted')
+    allowed_multioutput_str = ("raw_values", "uniform_average", "variance_weighted")
     if isinstance(multioutput, str):
         if multioutput not in allowed_multioutput_str:
-            raise ValueError("Allowed 'multioutput' string values are {}. "
-                             "You provided multioutput={!r}".format(
-                                 allowed_multioutput_str,
-                                 multioutput))
+            raise ValueError(
+                "Allowed 'multioutput' string values are {}. "
+                "You provided multioutput={!r}".format(
+                    allowed_multioutput_str, multioutput
+                )
+            )
     elif multioutput is not None:
         multioutput = check_array(multioutput, ensure_2d=False)
         if n_outputs == 1:
-            raise ValueError("Custom weights are useful only in "
-                             "multi-output cases.")
-        elif n_outputs != len(multioutput):
-            raise ValueError(("There must be equally many custom weights "
-                              "(%d) as outputs (%d).") %
-                             (len(multioutput), n_outputs))
-    y_type = 'continuous' if n_outputs == 1 else 'continuous-multioutput'
+            raise ValueError("Custom weights are useful only in multi-output cases.")
+        elif n_outputs != multioutput.shape[0]:
+            raise ValueError(
+                "There must be equally many custom weights "
+                f"({multioutput.shape[0]}) as outputs ({n_outputs})."
+            )
+    y_type = "continuous" if n_outputs == 1 else "continuous-multioutput"
+
+    return y_type, y_true, y_pred, sample_weight, multioutput
+
+
+def _check_reg_targets_with_floating_dtype(
+    y_true, y_pred, sample_weight, multioutput, xp=None
+):
+    """Ensures y_true, y_pred, and sample_weight correspond to same regression task.
+
+    Extends `_check_reg_targets` by automatically selecting a suitable floating-point
+    data type for inputs using `_find_matching_floating_dtype`.
+
+    Use this private method only when converting inputs to array API-compatibles.
+
+    Parameters
+    ----------
+    y_true : array-like of shape (n_samples,) or (n_samples, n_outputs)
+        Ground truth (correct) target values.
+
+    y_pred : array-like of shape (n_samples,) or (n_samples, n_outputs)
+        Estimated target values.
+
+    sample_weight : array-like of shape (n_samples,)
+
+    multioutput : array-like or string in ['raw_values', 'uniform_average', \
+        'variance_weighted'] or None
+        None is accepted due to backward compatibility of r2_score().
+
+    xp : module, default=None
+        Precomputed array namespace module. When passed, typically from a caller
+        that has already performed inspection of its own inputs, skips array
+        namespace inspection.
+
+    Returns
+    -------
+    type_true : one of {'continuous', 'continuous-multioutput'}
+        The type of the true target data, as output by
+        'utils.multiclass.type_of_target'.
+
+    y_true : array-like of shape (n_samples, n_outputs)
+        Ground truth (correct) target values.
+
+    y_pred : array-like of shape (n_samples, n_outputs)
+        Estimated target values.
+
+    sample_weight : array-like of shape (n_samples,), default=None
+        Sample weights.
 
-    return y_type, y_true, y_pred, multioutput
+    multioutput : array-like of shape (n_outputs) or string in ['raw_values', \
+        'uniform_average', 'variance_weighted'] or None
+        Custom output weights if ``multioutput`` is array-like or
+        just the corresponding argument if ``multioutput`` is a
+        correct keyword.
+    """
+    dtype_name = _find_matching_floating_dtype(y_true, y_pred, sample_weight, xp=xp)
 
+    y_type, y_true, y_pred, sample_weight, multioutput = _check_reg_targets(
+        y_true, y_pred, sample_weight, multioutput, dtype=dtype_name, xp=xp
+    )
 
-@_deprecate_positional_args
-def mean_absolute_error(y_true, y_pred, *,
-                        sample_weight=None,
-                        multioutput='uniform_average'):
+    return y_type, y_true, y_pred, sample_weight, multioutput
+
+
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_pred": ["array-like"],
+        "sample_weight": ["array-like", None],
+        "multioutput": [StrOptions({"raw_values", "uniform_average"}), "array-like"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def mean_absolute_error(
+    y_true, y_pred, *, sample_weight=None, multioutput="uniform_average"
+):
     """Mean absolute error regression loss.
 
-    Read more in the :ref:`User Guide `.
+    The mean absolute error is a non-negative floating point value, where best value
+    is 0.0. Read more in the :ref:`User Guide `.
 
     Parameters
     ----------
@@ -153,10 +253,9 @@ def mean_absolute_error(y_true, y_pred, *,
         'uniform_average' :
             Errors of all outputs are averaged with uniform weight.
 
-
     Returns
     -------
-    loss : float or ndarray of floats
+    loss : float or array of floats
         If multioutput is 'raw_values', then mean absolute error is returned
         for each output separately.
         If multioutput is 'uniform_average' or an ndarray of weights, then the
@@ -180,25 +279,47 @@ def mean_absolute_error(y_true, y_pred, *,
     >>> mean_absolute_error(y_true, y_pred, multioutput=[0.3, 0.7])
     0.85...
     """
-    y_type, y_true, y_pred, multioutput = _check_reg_targets(
-        y_true, y_pred, multioutput)
-    check_consistent_length(y_true, y_pred, sample_weight)
-    output_errors = np.average(np.abs(y_pred - y_true),
-                               weights=sample_weight, axis=0)
+    xp, _ = get_namespace(y_true, y_pred, sample_weight, multioutput)
+
+    _, y_true, y_pred, sample_weight, multioutput = (
+        _check_reg_targets_with_floating_dtype(
+            y_true, y_pred, sample_weight, multioutput, xp=xp
+        )
+    )
+
+    output_errors = _average(
+        xp.abs(y_pred - y_true), weights=sample_weight, axis=0, xp=xp
+    )
     if isinstance(multioutput, str):
-        if multioutput == 'raw_values':
+        if multioutput == "raw_values":
             return output_errors
-        elif multioutput == 'uniform_average':
-            # pass None as weights to np.average: uniform mean
+        elif multioutput == "uniform_average":
+            # pass None as weights to _average: uniform mean
             multioutput = None
 
-    return np.average(output_errors, weights=multioutput)
-
-
-def mean_pinball_loss(y_true, y_pred, *,
-                      sample_weight=None,
-                      alpha=0.5,
-                      multioutput='uniform_average'):
+    # Average across the outputs (if needed).
+    # The second call to `_average` should always return
+    # a scalar array that we convert to a Python float to
+    # consistently return the same eager evaluated value.
+    # Therefore, `axis=None`.
+    mean_absolute_error = _average(output_errors, weights=multioutput)
+
+    return float(mean_absolute_error)
+
+
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_pred": ["array-like"],
+        "sample_weight": ["array-like", None],
+        "alpha": [Interval(Real, 0, 1, closed="both")],
+        "multioutput": [StrOptions({"raw_values", "uniform_average"}), "array-like"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def mean_pinball_loss(
+    y_true, y_pred, *, sample_weight=None, alpha=0.5, multioutput="uniform_average"
+):
     """Pinball loss for quantile regression.
 
     Read more in the :ref:`User Guide `.
@@ -214,8 +335,8 @@ def mean_pinball_loss(y_true, y_pred, *,
     sample_weight : array-like of shape (n_samples,), default=None
         Sample weights.
 
-    alpha: double, slope of the pinball loss, default=0.5,
-        this loss is equivalent to :ref:`mean_absolute_error` when `alpha=0.5`,
+    alpha : float, slope of the pinball loss, default=0.5,
+        This loss is equivalent to :ref:`mean_absolute_error` when `alpha=0.5`,
         `alpha=0.95` is minimized by estimators of the 95th percentile.
 
     multioutput : {'raw_values', 'uniform_average'}  or array-like of shape \
@@ -228,6 +349,7 @@ def mean_pinball_loss(y_true, y_pred, *,
 
         'uniform_average' :
             Errors of all outputs are averaged with uniform weight.
+
     Returns
     -------
     loss : float or ndarray of floats
@@ -256,35 +378,53 @@ def mean_pinball_loss(y_true, y_pred, *,
     >>> mean_pinball_loss(y_true, y_true, alpha=0.9)
     0.0
     """
-    y_type, y_true, y_pred, multioutput = _check_reg_targets(
-        y_true, y_pred, multioutput)
-    check_consistent_length(y_true, y_pred, sample_weight)
-    diff = y_true - y_pred
-    sign = (diff >= 0).astype(diff.dtype)
-    loss = alpha * sign * diff - (1 - alpha) * (1 - sign) * diff
-    output_errors = np.average(loss, weights=sample_weight, axis=0)
-    if isinstance(multioutput, str):
-        if multioutput == 'raw_values':
-            return output_errors
-        elif multioutput == 'uniform_average':
-            # pass None as weights to np.average: uniform mean
-            multioutput = None
-        else:
-            raise ValueError("multioutput is expected to be 'raw_values' "
-                             "or 'uniform_average' but we got %r"
-                             " instead." % multioutput)
-
-    return np.average(output_errors, weights=multioutput)
+    xp, _ = get_namespace(y_true, y_pred, sample_weight, multioutput)
 
+    _, y_true, y_pred, sample_weight, multioutput = (
+        _check_reg_targets_with_floating_dtype(
+            y_true, y_pred, sample_weight, multioutput, xp=xp
+        )
+    )
 
-def mean_absolute_percentage_error(y_true, y_pred,
-                                   sample_weight=None,
-                                   multioutput='uniform_average'):
-    """Mean absolute percentage error regression loss.
-
-    Note here that we do not represent the output as a percentage in range
-    [0, 100]. Instead, we represent it in range [0, 1/eps]. Read more in the
-    :ref:`User Guide `.
+    diff = y_true - y_pred
+    sign = xp.astype(diff >= 0, diff.dtype)
+    loss = alpha * sign * diff - (1 - alpha) * (1 - sign) * diff
+    output_errors = _average(loss, weights=sample_weight, axis=0)
+
+    if isinstance(multioutput, str) and multioutput == "raw_values":
+        return output_errors
+
+    if isinstance(multioutput, str) and multioutput == "uniform_average":
+        # pass None as weights to _average: uniform mean
+        multioutput = None
+
+    # Average across the outputs (if needed).
+    # The second call to `_average` should always return
+    # a scalar array that we convert to a Python float to
+    # consistently return the same eager evaluated value.
+    # Therefore, `axis=None`.
+    return float(_average(output_errors, weights=multioutput))
+
+
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_pred": ["array-like"],
+        "sample_weight": ["array-like", None],
+        "multioutput": [StrOptions({"raw_values", "uniform_average"}), "array-like"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def mean_absolute_percentage_error(
+    y_true, y_pred, *, sample_weight=None, multioutput="uniform_average"
+):
+    """Mean absolute percentage error (MAPE) regression loss.
+
+    Note that we are not using the common "percentage" definition: the percentage
+    in the range [0, 100] is converted to a relative value in the range [0, 1]
+    by dividing by 100. Thus, an error of 200% corresponds to a relative error of 2.
+
+    Read more in the :ref:`User Guide `.
 
     .. versionadded:: 0.24
 
@@ -312,16 +452,16 @@ def mean_absolute_percentage_error(y_true, y_pred,
 
     Returns
     -------
-    loss : float or ndarray of floats in the range [0, 1/eps]
+    loss : float or ndarray of floats
         If multioutput is 'raw_values', then mean absolute percentage error
         is returned for each output separately.
         If multioutput is 'uniform_average' or an ndarray of weights, then the
         weighted average of all output errors is returned.
 
         MAPE output is non-negative floating point. The best value is 0.0.
-        But note the fact that bad predictions can lead to arbitarily large
-        MAPE values, especially if some y_true values are very close to zero.
-        Note that we return a large value instead of `inf` when y_true is zero.
+        But note that bad predictions can lead to arbitrarily large
+        MAPE values, especially if some `y_true` values are very close to zero.
+        Note that we return a large value instead of `inf` when `y_true` is zero.
 
     Examples
     --------
@@ -336,28 +476,58 @@ def mean_absolute_percentage_error(y_true, y_pred,
     0.5515...
     >>> mean_absolute_percentage_error(y_true, y_pred, multioutput=[0.3, 0.7])
     0.6198...
+    >>> # the value when some element of the y_true is zero is arbitrarily high because
+    >>> # of the division by epsilon
+    >>> y_true = [1., 0., 2.4, 7.]
+    >>> y_pred = [1.2, 0.1, 2.4, 8.]
+    >>> mean_absolute_percentage_error(y_true, y_pred)
+    112589990684262.48
     """
-    y_type, y_true, y_pred, multioutput = _check_reg_targets(
-        y_true, y_pred, multioutput)
-    check_consistent_length(y_true, y_pred, sample_weight)
-    epsilon = np.finfo(np.float64).eps
-    mape = np.abs(y_pred - y_true) / np.maximum(np.abs(y_true), epsilon)
-    output_errors = np.average(mape,
-                               weights=sample_weight, axis=0)
+    xp, _, device_ = get_namespace_and_device(
+        y_true, y_pred, sample_weight, multioutput
+    )
+    _, y_true, y_pred, sample_weight, multioutput = (
+        _check_reg_targets_with_floating_dtype(
+            y_true, y_pred, sample_weight, multioutput, xp=xp
+        )
+    )
+    epsilon = xp.asarray(xp.finfo(xp.float64).eps, dtype=y_true.dtype, device=device_)
+    y_true_abs = xp.abs(y_true)
+    mape = xp.abs(y_pred - y_true) / xp.maximum(y_true_abs, epsilon)
+    output_errors = _average(mape, weights=sample_weight, axis=0)
     if isinstance(multioutput, str):
-        if multioutput == 'raw_values':
+        if multioutput == "raw_values":
             return output_errors
-        elif multioutput == 'uniform_average':
-            # pass None as weights to np.average: uniform mean
+        elif multioutput == "uniform_average":
+            # pass None as weights to _average: uniform mean
             multioutput = None
 
-    return np.average(output_errors, weights=multioutput)
-
-
-@_deprecate_positional_args
-def mean_squared_error(y_true, y_pred, *,
-                       sample_weight=None,
-                       multioutput='uniform_average', squared=True):
+    # Average across the outputs (if needed).
+    # The second call to `_average` should always return
+    # a scalar array that we convert to a Python float to
+    # consistently return the same eager evaluated value.
+    # Therefore, `axis=None`.
+    mean_absolute_percentage_error = _average(output_errors, weights=multioutput)
+
+    return float(mean_absolute_percentage_error)
+
+
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_pred": ["array-like"],
+        "sample_weight": ["array-like", None],
+        "multioutput": [StrOptions({"raw_values", "uniform_average"}), "array-like"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def mean_squared_error(
+    y_true,
+    y_pred,
+    *,
+    sample_weight=None,
+    multioutput="uniform_average",
+):
     """Mean squared error regression loss.
 
     Read more in the :ref:`User Guide `.
@@ -384,12 +554,9 @@ def mean_squared_error(y_true, y_pred, *,
         'uniform_average' :
             Errors of all outputs are averaged with uniform weight.
 
-    squared : bool, default=True
-        If True returns MSE value, if False returns RMSE value.
-
     Returns
     -------
-    loss : float or ndarray of floats
+    loss : float or array of floats
         A non-negative floating point value (the best value is 0.0), or an
         array of floating point values, one for each individual target.
 
@@ -400,44 +567,140 @@ def mean_squared_error(y_true, y_pred, *,
     >>> y_pred = [2.5, 0.0, 2, 8]
     >>> mean_squared_error(y_true, y_pred)
     0.375
-    >>> y_true = [3, -0.5, 2, 7]
-    >>> y_pred = [2.5, 0.0, 2, 8]
-    >>> mean_squared_error(y_true, y_pred, squared=False)
-    0.612...
     >>> y_true = [[0.5, 1],[-1, 1],[7, -6]]
     >>> y_pred = [[0, 2],[-1, 2],[8, -5]]
     >>> mean_squared_error(y_true, y_pred)
     0.708...
-    >>> mean_squared_error(y_true, y_pred, squared=False)
-    0.822...
     >>> mean_squared_error(y_true, y_pred, multioutput='raw_values')
     array([0.41666667, 1.        ])
     >>> mean_squared_error(y_true, y_pred, multioutput=[0.3, 0.7])
     0.825...
     """
-    y_type, y_true, y_pred, multioutput = _check_reg_targets(
-        y_true, y_pred, multioutput)
-    check_consistent_length(y_true, y_pred, sample_weight)
-    output_errors = np.average((y_true - y_pred) ** 2, axis=0,
-                               weights=sample_weight)
-
-    if not squared:
-        output_errors = np.sqrt(output_errors)
+    xp, _ = get_namespace(y_true, y_pred, sample_weight, multioutput)
+    _, y_true, y_pred, sample_weight, multioutput = (
+        _check_reg_targets_with_floating_dtype(
+            y_true, y_pred, sample_weight, multioutput, xp=xp
+        )
+    )
+    output_errors = _average((y_true - y_pred) ** 2, axis=0, weights=sample_weight)
 
     if isinstance(multioutput, str):
-        if multioutput == 'raw_values':
+        if multioutput == "raw_values":
             return output_errors
-        elif multioutput == 'uniform_average':
-            # pass None as weights to np.average: uniform mean
+        elif multioutput == "uniform_average":
+            # pass None as weights to _average: uniform mean
             multioutput = None
 
-    return np.average(output_errors, weights=multioutput)
+    # Average across the outputs (if needed).
+    # The second call to `_average` should always return
+    # a scalar array that we convert to a Python float to
+    # consistently return the same eager evaluated value.
+    # Therefore, `axis=None`.
+    mean_squared_error = _average(output_errors, weights=multioutput)
+
+    return float(mean_squared_error)
+
+
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_pred": ["array-like"],
+        "sample_weight": ["array-like", None],
+        "multioutput": [StrOptions({"raw_values", "uniform_average"}), "array-like"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def root_mean_squared_error(
+    y_true, y_pred, *, sample_weight=None, multioutput="uniform_average"
+):
+    """Root mean squared error regression loss.
+
+    Read more in the :ref:`User Guide `.
+
+    .. versionadded:: 1.4
+
+    Parameters
+    ----------
+    y_true : array-like of shape (n_samples,) or (n_samples, n_outputs)
+        Ground truth (correct) target values.
+
+    y_pred : array-like of shape (n_samples,) or (n_samples, n_outputs)
+        Estimated target values.
+
+    sample_weight : array-like of shape (n_samples,), default=None
+        Sample weights.
+
+    multioutput : {'raw_values', 'uniform_average'} or array-like of shape \
+            (n_outputs,), default='uniform_average'
+        Defines aggregating of multiple output values.
+        Array-like value defines weights used to average errors.
 
+        'raw_values' :
+            Returns a full set of errors in case of multioutput input.
+
+        'uniform_average' :
+            Errors of all outputs are averaged with uniform weight.
+
+    Returns
+    -------
+    loss : float or ndarray of floats
+        A non-negative floating point value (the best value is 0.0), or an
+        array of floating point values, one for each individual target.
+
+    Examples
+    --------
+    >>> from sklearn.metrics import root_mean_squared_error
+    >>> y_true = [3, -0.5, 2, 7]
+    >>> y_pred = [2.5, 0.0, 2, 8]
+    >>> root_mean_squared_error(y_true, y_pred)
+    0.612...
+    >>> y_true = [[0.5, 1],[-1, 1],[7, -6]]
+    >>> y_pred = [[0, 2],[-1, 2],[8, -5]]
+    >>> root_mean_squared_error(y_true, y_pred)
+    0.822...
+    """
 
-@_deprecate_positional_args
-def mean_squared_log_error(y_true, y_pred, *,
-                           sample_weight=None,
-                           multioutput='uniform_average'):
+    xp, _ = get_namespace(y_true, y_pred, sample_weight, multioutput)
+
+    output_errors = xp.sqrt(
+        mean_squared_error(
+            y_true, y_pred, sample_weight=sample_weight, multioutput="raw_values"
+        )
+    )
+
+    if isinstance(multioutput, str):
+        if multioutput == "raw_values":
+            return output_errors
+        elif multioutput == "uniform_average":
+            # pass None as weights to _average: uniform mean
+            multioutput = None
+
+    # Average across the outputs (if needed).
+    # The second call to `_average` should always return
+    # a scalar array that we convert to a Python float to
+    # consistently return the same eager evaluated value.
+    # Therefore, `axis=None`.
+    root_mean_squared_error = _average(output_errors, weights=multioutput)
+
+    return float(root_mean_squared_error)
+
+
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_pred": ["array-like"],
+        "sample_weight": ["array-like", None],
+        "multioutput": [StrOptions({"raw_values", "uniform_average"}), "array-like"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def mean_squared_log_error(
+    y_true,
+    y_pred,
+    *,
+    sample_weight=None,
+    multioutput="uniform_average",
+):
     """Mean squared logarithmic error regression loss.
 
     Read more in the :ref:`User Guide `.
@@ -488,22 +751,118 @@ def mean_squared_log_error(y_true, y_pred, *,
     >>> mean_squared_log_error(y_true, y_pred, multioutput=[0.3, 0.7])
     0.060...
     """
-    y_type, y_true, y_pred, multioutput = _check_reg_targets(
-        y_true, y_pred, multioutput)
-    check_consistent_length(y_true, y_pred, sample_weight)
+    xp, _ = get_namespace(y_true, y_pred)
+
+    _, y_true, y_pred, sample_weight, multioutput = (
+        _check_reg_targets_with_floating_dtype(
+            y_true, y_pred, sample_weight, multioutput, xp=xp
+        )
+    )
+
+    if xp.any(y_true <= -1) or xp.any(y_pred <= -1):
+        raise ValueError(
+            "Mean Squared Logarithmic Error cannot be used when "
+            "targets contain values less than or equal to -1."
+        )
+
+    return mean_squared_error(
+        xp.log1p(y_true),
+        xp.log1p(y_pred),
+        sample_weight=sample_weight,
+        multioutput=multioutput,
+    )
+
+
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_pred": ["array-like"],
+        "sample_weight": ["array-like", None],
+        "multioutput": [StrOptions({"raw_values", "uniform_average"}), "array-like"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def root_mean_squared_log_error(
+    y_true, y_pred, *, sample_weight=None, multioutput="uniform_average"
+):
+    """Root mean squared logarithmic error regression loss.
+
+    Read more in the :ref:`User Guide `.
+
+    .. versionadded:: 1.4
+
+    Parameters
+    ----------
+    y_true : array-like of shape (n_samples,) or (n_samples, n_outputs)
+        Ground truth (correct) target values.
+
+    y_pred : array-like of shape (n_samples,) or (n_samples, n_outputs)
+        Estimated target values.
+
+    sample_weight : array-like of shape (n_samples,), default=None
+        Sample weights.
 
-    if (y_true < 0).any() or (y_pred < 0).any():
-        raise ValueError("Mean Squared Logarithmic Error cannot be used when "
-                         "targets contain negative values.")
+    multioutput : {'raw_values', 'uniform_average'} or array-like of shape \
+            (n_outputs,), default='uniform_average'
 
-    return mean_squared_error(np.log1p(y_true), np.log1p(y_pred),
-                              sample_weight=sample_weight,
-                              multioutput=multioutput)
+        Defines aggregating of multiple output values.
+        Array-like value defines weights used to average errors.
 
+        'raw_values' :
+            Returns a full set of errors when the input is of multioutput
+            format.
 
-@_deprecate_positional_args
-def median_absolute_error(y_true, y_pred, *, multioutput='uniform_average',
-                          sample_weight=None):
+        'uniform_average' :
+            Errors of all outputs are averaged with uniform weight.
+
+    Returns
+    -------
+    loss : float or ndarray of floats
+        A non-negative floating point value (the best value is 0.0), or an
+        array of floating point values, one for each individual target.
+
+    Examples
+    --------
+    >>> from sklearn.metrics import root_mean_squared_log_error
+    >>> y_true = [3, 5, 2.5, 7]
+    >>> y_pred = [2.5, 5, 4, 8]
+    >>> root_mean_squared_log_error(y_true, y_pred)
+    0.199...
+    """
+    xp, _ = get_namespace(y_true, y_pred)
+
+    _, y_true, y_pred, sample_weight, multioutput = (
+        _check_reg_targets_with_floating_dtype(
+            y_true, y_pred, sample_weight, multioutput, xp=xp
+        )
+    )
+
+    if xp.any(y_true <= -1) or xp.any(y_pred <= -1):
+        raise ValueError(
+            "Root Mean Squared Logarithmic Error cannot be used when "
+            "targets contain values less than or equal to -1."
+        )
+
+    return root_mean_squared_error(
+        xp.log1p(y_true),
+        xp.log1p(y_pred),
+        sample_weight=sample_weight,
+        multioutput=multioutput,
+    )
+
+
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_pred": ["array-like"],
+        "multioutput": [StrOptions({"raw_values", "uniform_average"}), "array-like"],
+        "sample_weight": ["array-like", None],
+    },
+    prefer_skip_nested_validation=True,
+)
+def median_absolute_error(
+    y_true, y_pred, *, multioutput="uniform_average", sample_weight=None
+):
     """Median absolute error regression loss.
 
     Median absolute error output is non-negative floating point. The best value
@@ -511,10 +870,10 @@ def median_absolute_error(y_true, y_pred, *, multioutput='uniform_average',
 
     Parameters
     ----------
-    y_true : array-like of shape = (n_samples) or (n_samples, n_outputs)
+    y_true : array-like of shape (n_samples,) or (n_samples, n_outputs)
         Ground truth (correct) target values.
 
-    y_pred : array-like of shape = (n_samples) or (n_samples, n_outputs)
+    y_pred : array-like of shape (n_samples,) or (n_samples, n_outputs)
         Estimated target values.
 
     multioutput : {'raw_values', 'uniform_average'} or array-like of shape \
@@ -557,32 +916,116 @@ def median_absolute_error(y_true, y_pred, *, multioutput='uniform_average',
     >>> median_absolute_error(y_true, y_pred, multioutput=[0.3, 0.7])
     0.85
     """
-    y_type, y_true, y_pred, multioutput = _check_reg_targets(
-        y_true, y_pred, multioutput)
+    xp, _ = get_namespace(y_true, y_pred, multioutput, sample_weight)
+    _, y_true, y_pred, sample_weight, multioutput = _check_reg_targets(
+        y_true, y_pred, sample_weight, multioutput
+    )
     if sample_weight is None:
-        output_errors = np.median(np.abs(y_pred - y_true), axis=0)
+        output_errors = _median(xp.abs(y_pred - y_true), axis=0)
     else:
-        sample_weight = _check_sample_weight(sample_weight, y_pred)
-        output_errors = _weighted_percentile(np.abs(y_pred - y_true),
-                                             sample_weight=sample_weight)
+        output_errors = _weighted_percentile(
+            xp.abs(y_pred - y_true), sample_weight=sample_weight
+        )
     if isinstance(multioutput, str):
-        if multioutput == 'raw_values':
+        if multioutput == "raw_values":
             return output_errors
-        elif multioutput == 'uniform_average':
+        elif multioutput == "uniform_average":
             # pass None as weights to np.average: uniform mean
             multioutput = None
 
-    return np.average(output_errors, weights=multioutput)
+    return float(_average(output_errors, weights=multioutput))
+
+
+def _assemble_r2_explained_variance(
+    numerator, denominator, n_outputs, multioutput, force_finite, xp, device
+):
+    """Common part used by explained variance score and :math:`R^2` score."""
+    dtype = numerator.dtype
+
+    nonzero_denominator = denominator != 0
+
+    if not force_finite:
+        # Standard formula, that may lead to NaN or -Inf
+        output_scores = 1 - (numerator / denominator)
+    else:
+        nonzero_numerator = numerator != 0
+        # Default = Zero Numerator = perfect predictions. Set to 1.0
+        # (note: even if denominator is zero, thus avoiding NaN scores)
+        output_scores = xp.ones([n_outputs], device=device, dtype=dtype)
+        # Non-zero Numerator and Non-zero Denominator: use the formula
+        valid_score = nonzero_denominator & nonzero_numerator
+
+        output_scores[valid_score] = 1 - (
+            numerator[valid_score] / denominator[valid_score]
+        )
+
+        # Non-zero Numerator and Zero Denominator:
+        # arbitrary set to 0.0 to avoid -inf scores
+        output_scores[nonzero_numerator & ~nonzero_denominator] = 0.0
 
+    if isinstance(multioutput, str):
+        if multioutput == "raw_values":
+            # return scores individually
+            return output_scores
+        elif multioutput == "uniform_average":
+            # pass None as weights to _average: uniform mean
+            avg_weights = None
+        elif multioutput == "variance_weighted":
+            avg_weights = denominator
+            if not xp.any(nonzero_denominator):
+                # All weights are zero, _average would raise a ZeroDiv error.
+                # This only happens when all y are constant (or 1-element long)
+                # Since weights are all equal, fall back to uniform weights.
+                avg_weights = None
+    else:
+        avg_weights = multioutput
 
-@_deprecate_positional_args
-def explained_variance_score(y_true, y_pred, *,
-                             sample_weight=None,
-                             multioutput='uniform_average'):
+    result = _average(output_scores, weights=avg_weights)
+    if size(result) == 1:
+        return float(result)
+    return result
+
+
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_pred": ["array-like"],
+        "sample_weight": ["array-like", None],
+        "multioutput": [
+            StrOptions({"raw_values", "uniform_average", "variance_weighted"}),
+            "array-like",
+        ],
+        "force_finite": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def explained_variance_score(
+    y_true,
+    y_pred,
+    *,
+    sample_weight=None,
+    multioutput="uniform_average",
+    force_finite=True,
+):
     """Explained variance regression score function.
 
     Best possible score is 1.0, lower values are worse.
 
+    In the particular case when ``y_true`` is constant, the explained variance
+    score is not finite: it is either ``NaN`` (perfect predictions) or
+    ``-Inf`` (imperfect predictions). To prevent such non-finite numbers to
+    pollute higher-level experiments such as a grid search cross-validation,
+    by default these cases are replaced with 1.0 (perfect predictions) or 0.0
+    (imperfect predictions) respectively. If ``force_finite``
+    is set to ``False``, this score falls back on the original :math:`R^2`
+    definition.
+
+    .. note::
+       The Explained Variance score is similar to the
+       :func:`R^2 score `, with the notable difference that it
+       does not account for systematic offsets in the prediction. Most often
+       the :func:`R^2 score ` should be preferred.
+
     Read more in the :ref:`User Guide `.
 
     Parameters
@@ -611,11 +1054,26 @@ def explained_variance_score(y_true, y_pred, *,
             Scores of all outputs are averaged, weighted by the variances
             of each individual output.
 
+    force_finite : bool, default=True
+        Flag indicating if ``NaN`` and ``-Inf`` scores resulting from constant
+        data should be replaced with real numbers (``1.0`` if prediction is
+        perfect, ``0.0`` otherwise). Default is ``True``, a convenient setting
+        for hyperparameters' search procedures (e.g. grid search
+        cross-validation).
+
+        .. versionadded:: 1.1
+
     Returns
     -------
     score : float or ndarray of floats
         The explained variance or ndarray if 'multioutput' is 'raw_values'.
 
+    See Also
+    --------
+    r2_score :
+        Similar metric, but accounting for systematic offsets in
+        prediction.
+
     Notes
     -----
     This is not a symmetric function.
@@ -631,51 +1089,86 @@ def explained_variance_score(y_true, y_pred, *,
     >>> y_pred = [[0, 2], [-1, 2], [8, -5]]
     >>> explained_variance_score(y_true, y_pred, multioutput='uniform_average')
     0.983...
+    >>> y_true = [-2, -2, -2]
+    >>> y_pred = [-2, -2, -2]
+    >>> explained_variance_score(y_true, y_pred)
+    1.0
+    >>> explained_variance_score(y_true, y_pred, force_finite=False)
+    nan
+    >>> y_true = [-2, -2, -2]
+    >>> y_pred = [-2, -2, -2 + 1e-8]
+    >>> explained_variance_score(y_true, y_pred)
+    0.0
+    >>> explained_variance_score(y_true, y_pred, force_finite=False)
+    -inf
     """
-    y_type, y_true, y_pred, multioutput = _check_reg_targets(
-        y_true, y_pred, multioutput)
-    check_consistent_length(y_true, y_pred, sample_weight)
-
-    y_diff_avg = np.average(y_true - y_pred, weights=sample_weight, axis=0)
-    numerator = np.average((y_true - y_pred - y_diff_avg) ** 2,
-                           weights=sample_weight, axis=0)
-
-    y_true_avg = np.average(y_true, weights=sample_weight, axis=0)
-    denominator = np.average((y_true - y_true_avg) ** 2,
-                             weights=sample_weight, axis=0)
+    xp, _, device = get_namespace_and_device(y_true, y_pred, sample_weight, multioutput)
 
-    nonzero_numerator = numerator != 0
-    nonzero_denominator = denominator != 0
-    valid_score = nonzero_numerator & nonzero_denominator
-    output_scores = np.ones(y_true.shape[1])
+    _, y_true, y_pred, sample_weight, multioutput = (
+        _check_reg_targets_with_floating_dtype(
+            y_true, y_pred, sample_weight, multioutput, xp=xp
+        )
+    )
 
-    output_scores[valid_score] = 1 - (numerator[valid_score] /
-                                      denominator[valid_score])
-    output_scores[nonzero_numerator & ~nonzero_denominator] = 0.
-    if isinstance(multioutput, str):
-        if multioutput == 'raw_values':
-            # return scores individually
-            return output_scores
-        elif multioutput == 'uniform_average':
-            # passing to np.average() None as weights results is uniform mean
-            avg_weights = None
-        elif multioutput == 'variance_weighted':
-            avg_weights = denominator
-    else:
-        avg_weights = multioutput
+    y_diff_avg = _average(y_true - y_pred, weights=sample_weight, axis=0)
+    numerator = _average(
+        (y_true - y_pred - y_diff_avg) ** 2, weights=sample_weight, axis=0
+    )
 
-    return np.average(output_scores, weights=avg_weights)
+    y_true_avg = _average(y_true, weights=sample_weight, axis=0)
+    denominator = _average((y_true - y_true_avg) ** 2, weights=sample_weight, axis=0)
+
+    return _assemble_r2_explained_variance(
+        numerator=numerator,
+        denominator=denominator,
+        n_outputs=y_true.shape[1],
+        multioutput=multioutput,
+        force_finite=force_finite,
+        xp=xp,
+        device=device,
+    )
 
 
-@_deprecate_positional_args
-def r2_score(y_true, y_pred, *, sample_weight=None,
-             multioutput="uniform_average"):
-    """R^2 (coefficient of determination) regression score function.
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_pred": ["array-like"],
+        "sample_weight": ["array-like", None],
+        "multioutput": [
+            StrOptions({"raw_values", "uniform_average", "variance_weighted"}),
+            "array-like",
+            None,
+        ],
+        "force_finite": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def r2_score(
+    y_true,
+    y_pred,
+    *,
+    sample_weight=None,
+    multioutput="uniform_average",
+    force_finite=True,
+):
+    """:math:`R^2` (coefficient of determination) regression score function.
 
     Best possible score is 1.0 and it can be negative (because the
-    model can be arbitrarily worse). A constant model that always
-    predicts the expected value of y, disregarding the input features,
-    would get a R^2 score of 0.0.
+    model can be arbitrarily worse). In the general case when the true y is
+    non-constant, a constant model that always predicts the average y
+    disregarding the input features would get a :math:`R^2` score of 0.0.
+
+    In the particular case when ``y_true`` is constant, the :math:`R^2` score
+    is not finite: it is either ``NaN`` (perfect predictions) or ``-Inf``
+    (imperfect predictions). To prevent such non-finite numbers to pollute
+    higher-level experiments such as a grid search cross-validation, by default
+    these cases are replaced with 1.0 (perfect predictions) or 0.0 (imperfect
+    predictions) respectively. You can set ``force_finite`` to ``False`` to
+    prevent this fix from happening.
+
+    Note: when the prediction residuals have zero mean, the :math:`R^2` score
+    is identical to the
+    :func:`Explained Variance score `.
 
     Read more in the :ref:`User Guide `.
 
@@ -710,18 +1203,27 @@ def r2_score(y_true, y_pred, *, sample_weight=None,
         .. versionchanged:: 0.19
             Default value of multioutput is 'uniform_average'.
 
+    force_finite : bool, default=True
+        Flag indicating if ``NaN`` and ``-Inf`` scores resulting from constant
+        data should be replaced with real numbers (``1.0`` if prediction is
+        perfect, ``0.0`` otherwise). Default is ``True``, a convenient setting
+        for hyperparameters' search procedures (e.g. grid search
+        cross-validation).
+
+        .. versionadded:: 1.1
+
     Returns
     -------
     z : float or ndarray of floats
-        The R^2 score or ndarray of scores if 'multioutput' is
+        The :math:`R^2` score or ndarray of scores if 'multioutput' is
         'raw_values'.
 
     Notes
     -----
     This is not a symmetric function.
 
-    Unlike most other scores, R^2 score may be negative (it need not actually
-    be the square of a quantity R).
+    Unlike most other scores, :math:`R^2` score may be negative (it need not
+    actually be the square of a quantity R).
 
     This metric is not well-defined for single samples and will return a NaN
     value if n_samples is less than two.
@@ -755,60 +1257,67 @@ def r2_score(y_true, y_pred, *, sample_weight=None,
     >>> y_pred = [3, 2, 1]
     >>> r2_score(y_true, y_pred)
     -3.0
+    >>> y_true = [-2, -2, -2]
+    >>> y_pred = [-2, -2, -2]
+    >>> r2_score(y_true, y_pred)
+    1.0
+    >>> r2_score(y_true, y_pred, force_finite=False)
+    nan
+    >>> y_true = [-2, -2, -2]
+    >>> y_pred = [-2, -2, -2 + 1e-8]
+    >>> r2_score(y_true, y_pred)
+    0.0
+    >>> r2_score(y_true, y_pred, force_finite=False)
+    -inf
     """
-    y_type, y_true, y_pred, multioutput = _check_reg_targets(
-        y_true, y_pred, multioutput)
-    check_consistent_length(y_true, y_pred, sample_weight)
+    xp, _, device_ = get_namespace_and_device(
+        y_true, y_pred, sample_weight, multioutput
+    )
+
+    _, y_true, y_pred, sample_weight, multioutput = (
+        _check_reg_targets_with_floating_dtype(
+            y_true, y_pred, sample_weight, multioutput, xp=xp
+        )
+    )
 
     if _num_samples(y_pred) < 2:
         msg = "R^2 score is not well-defined with less than two samples."
         warnings.warn(msg, UndefinedMetricWarning)
-        return float('nan')
+        return float("nan")
 
     if sample_weight is not None:
         sample_weight = column_or_1d(sample_weight)
-        weight = sample_weight[:, np.newaxis]
+        weight = sample_weight[:, None]
     else:
-        weight = 1.
+        weight = 1.0
 
-    numerator = (weight * (y_true - y_pred) ** 2).sum(axis=0,
-                                                      dtype=np.float64)
-    denominator = (weight * (y_true - np.average(
-        y_true, axis=0, weights=sample_weight)) ** 2).sum(axis=0,
-                                                          dtype=np.float64)
-    nonzero_denominator = denominator != 0
-    nonzero_numerator = numerator != 0
-    valid_score = nonzero_denominator & nonzero_numerator
-    output_scores = np.ones([y_true.shape[1]])
-    output_scores[valid_score] = 1 - (numerator[valid_score] /
-                                      denominator[valid_score])
-    # arbitrary set to zero to avoid -inf scores, having a constant
-    # y_true is not interesting for scoring a regression anyway
-    output_scores[nonzero_numerator & ~nonzero_denominator] = 0.
-    if isinstance(multioutput, str):
-        if multioutput == 'raw_values':
-            # return scores individually
-            return output_scores
-        elif multioutput == 'uniform_average':
-            # passing None as weights results is uniform mean
-            avg_weights = None
-        elif multioutput == 'variance_weighted':
-            avg_weights = denominator
-            # avoid fail on constant y or one-element arrays
-            if not np.any(nonzero_denominator):
-                if not np.any(nonzero_numerator):
-                    return 1.0
-                else:
-                    return 0.0
-    else:
-        avg_weights = multioutput
+    numerator = xp.sum(weight * (y_true - y_pred) ** 2, axis=0)
+    denominator = xp.sum(
+        weight * (y_true - _average(y_true, axis=0, weights=sample_weight, xp=xp)) ** 2,
+        axis=0,
+    )
 
-    return np.average(output_scores, weights=avg_weights)
+    return _assemble_r2_explained_variance(
+        numerator=numerator,
+        denominator=denominator,
+        n_outputs=y_true.shape[1],
+        multioutput=multioutput,
+        force_finite=force_finite,
+        xp=xp,
+        device=device_,
+    )
 
 
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_pred": ["array-like"],
+    },
+    prefer_skip_nested_validation=True,
+)
 def max_error(y_true, y_pred):
     """
-    max_error metric calculates the maximum residual error.
+    The max_error metric calculates the maximum residual error.
 
     Read more in the :ref:`User Guide `.
 
@@ -831,15 +1340,62 @@ def max_error(y_true, y_pred):
     >>> y_true = [3, 2, 7, 1]
     >>> y_pred = [4, 2, 7, 1]
     >>> max_error(y_true, y_pred)
-    1
+    1.0
     """
-    y_type, y_true, y_pred, _ = _check_reg_targets(y_true, y_pred, None)
-    if y_type == 'continuous-multioutput':
+    xp, _ = get_namespace(y_true, y_pred)
+    y_type, y_true, y_pred, _, _ = _check_reg_targets(
+        y_true, y_pred, sample_weight=None, multioutput=None, xp=xp
+    )
+    if y_type == "continuous-multioutput":
         raise ValueError("Multioutput not supported in max_error")
-    return np.max(np.abs(y_true - y_pred))
-
-
-@_deprecate_positional_args
+    return float(xp.max(xp.abs(y_true - y_pred)))
+
+
+def _mean_tweedie_deviance(y_true, y_pred, sample_weight, power):
+    """Mean Tweedie deviance regression loss."""
+    xp, _, device_ = get_namespace_and_device(y_true, y_pred)
+    p = power
+    if p < 0:
+        # 'Extreme stable', y any real number, y_pred > 0
+        dev = 2 * (
+            xp.pow(
+                xp.where(y_true > 0, y_true, 0.0),
+                2 - p,
+            )
+            / ((1 - p) * (2 - p))
+            - y_true * xp.pow(y_pred, 1 - p) / (1 - p)
+            + xp.pow(y_pred, 2 - p) / (2 - p)
+        )
+    elif p == 0:
+        # Normal distribution, y and y_pred any real number
+        dev = (y_true - y_pred) ** 2
+    elif p == 1:
+        # Poisson distribution
+        dev = 2 * (xlogy(y_true, y_true / y_pred) - y_true + y_pred)
+    elif p == 2:
+        # Gamma distribution
+        dev = 2 * (xp.log(y_pred / y_true) + y_true / y_pred - 1)
+    else:
+        dev = 2 * (
+            xp.pow(y_true, 2 - p) / ((1 - p) * (2 - p))
+            - y_true * xp.pow(y_pred, 1 - p) / (1 - p)
+            + xp.pow(y_pred, 2 - p) / (2 - p)
+        )
+    return float(_average(dev, weights=sample_weight))
+
+
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_pred": ["array-like"],
+        "sample_weight": ["array-like", None],
+        "power": [
+            Interval(Real, None, 0, closed="right"),
+            Interval(Real, 1, None, closed="left"),
+        ],
+    },
+    prefer_skip_nested_validation=True,
+)
 def mean_tweedie_deviance(y_true, y_pred, *, sample_weight=None, power=0):
     """Mean Tweedie deviance regression loss.
 
@@ -888,23 +1444,50 @@ def mean_tweedie_deviance(y_true, y_pred, *, sample_weight=None, power=0):
     >>> mean_tweedie_deviance(y_true, y_pred, power=1)
     1.4260...
     """
-    y_type, y_true, y_pred, _ = _check_reg_targets(
-        y_true, y_pred, None, dtype=[np.float64, np.float32])
-    if y_type == 'continuous-multioutput':
+    xp, _ = get_namespace(y_true, y_pred)
+    y_type, y_true, y_pred, sample_weight, _ = _check_reg_targets_with_floating_dtype(
+        y_true, y_pred, sample_weight, multioutput=None, xp=xp
+    )
+    if y_type == "continuous-multioutput":
         raise ValueError("Multioutput not supported in mean_tweedie_deviance")
-    check_consistent_length(y_true, y_pred, sample_weight)
 
     if sample_weight is not None:
         sample_weight = column_or_1d(sample_weight)
         sample_weight = sample_weight[:, np.newaxis]
 
-    dist = TweedieDistribution(power=power)
-    dev = dist.unit_deviance(y_true, y_pred, check_input=True)
-
-    return np.average(dev, weights=sample_weight)
+    message = f"Mean Tweedie deviance error with power={power} can only be used on "
+    if power < 0:
+        # 'Extreme stable', y any real number, y_pred > 0
+        if xp.any(y_pred <= 0):
+            raise ValueError(message + "strictly positive y_pred.")
+    elif power == 0:
+        # Normal, y and y_pred can be any real number
+        pass
+    elif 1 <= power < 2:
+        # Poisson and compound Poisson distribution, y >= 0, y_pred > 0
+        if xp.any(y_true < 0) or xp.any(y_pred <= 0):
+            raise ValueError(message + "non-negative y and strictly positive y_pred.")
+    elif power >= 2:
+        # Gamma and Extreme stable distribution, y and y_pred > 0
+        if xp.any(y_true <= 0) or xp.any(y_pred <= 0):
+            raise ValueError(message + "strictly positive y and y_pred.")
+    else:  # pragma: nocover
+        # Unreachable statement
+        raise ValueError
+
+    return _mean_tweedie_deviance(
+        y_true, y_pred, sample_weight=sample_weight, power=power
+    )
 
 
-@_deprecate_positional_args
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_pred": ["array-like"],
+        "sample_weight": ["array-like", None],
+    },
+    prefer_skip_nested_validation=True,
+)
 def mean_poisson_deviance(y_true, y_pred, *, sample_weight=None):
     """Mean Poisson deviance regression loss.
 
@@ -937,12 +1520,17 @@ def mean_poisson_deviance(y_true, y_pred, *, sample_weight=None):
     >>> mean_poisson_deviance(y_true, y_pred)
     1.4260...
     """
-    return mean_tweedie_deviance(
-        y_true, y_pred, sample_weight=sample_weight, power=1
-    )
+    return mean_tweedie_deviance(y_true, y_pred, sample_weight=sample_weight, power=1)
 
 
-@_deprecate_positional_args
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_pred": ["array-like"],
+        "sample_weight": ["array-like", None],
+    },
+    prefer_skip_nested_validation=True,
+)
 def mean_gamma_deviance(y_true, y_pred, *, sample_weight=None):
     """Mean Gamma deviance regression loss.
 
@@ -976,6 +1564,369 @@ def mean_gamma_deviance(y_true, y_pred, *, sample_weight=None):
     >>> mean_gamma_deviance(y_true, y_pred)
     1.0568...
     """
-    return mean_tweedie_deviance(
-        y_true, y_pred, sample_weight=sample_weight, power=2
+    return mean_tweedie_deviance(y_true, y_pred, sample_weight=sample_weight, power=2)
+
+
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_pred": ["array-like"],
+        "sample_weight": ["array-like", None],
+        "power": [
+            Interval(Real, None, 0, closed="right"),
+            Interval(Real, 1, None, closed="left"),
+        ],
+    },
+    prefer_skip_nested_validation=True,
+)
+def d2_tweedie_score(y_true, y_pred, *, sample_weight=None, power=0):
+    """
+    :math:`D^2` regression score function, fraction of Tweedie deviance explained.
+
+    Best possible score is 1.0 and it can be negative (because the model can be
+    arbitrarily worse). A model that always uses the empirical mean of `y_true` as
+    constant prediction, disregarding the input features, gets a D^2 score of 0.0.
+
+    Read more in the :ref:`User Guide `.
+
+    .. versionadded:: 1.0
+
+    Parameters
+    ----------
+    y_true : array-like of shape (n_samples,)
+        Ground truth (correct) target values.
+
+    y_pred : array-like of shape (n_samples,)
+        Estimated target values.
+
+    sample_weight : array-like of shape (n_samples,), default=None
+        Sample weights.
+
+    power : float, default=0
+        Tweedie power parameter. Either power <= 0 or power >= 1.
+
+        The higher `p` the less weight is given to extreme
+        deviations between true and predicted targets.
+
+        - power < 0: Extreme stable distribution. Requires: y_pred > 0.
+        - power = 0 : Normal distribution, output corresponds to r2_score.
+          y_true and y_pred can be any real numbers.
+        - power = 1 : Poisson distribution. Requires: y_true >= 0 and
+          y_pred > 0.
+        - 1 < p < 2 : Compound Poisson distribution. Requires: y_true >= 0
+          and y_pred > 0.
+        - power = 2 : Gamma distribution. Requires: y_true > 0 and y_pred > 0.
+        - power = 3 : Inverse Gaussian distribution. Requires: y_true > 0
+          and y_pred > 0.
+        - otherwise : Positive stable distribution. Requires: y_true > 0
+          and y_pred > 0.
+
+    Returns
+    -------
+    z : float
+        The D^2 score.
+
+    Notes
+    -----
+    This is not a symmetric function.
+
+    Like R^2, D^2 score may be negative (it need not actually be the square of
+    a quantity D).
+
+    This metric is not well-defined for single samples and will return a NaN
+    value if n_samples is less than two.
+
+    References
+    ----------
+    .. [1] Eq. (3.11) of Hastie, Trevor J., Robert Tibshirani and Martin J.
+           Wainwright. "Statistical Learning with Sparsity: The Lasso and
+           Generalizations." (2015). https://hastie.su.domains/StatLearnSparsity/
+
+    Examples
+    --------
+    >>> from sklearn.metrics import d2_tweedie_score
+    >>> y_true = [0.5, 1, 2.5, 7]
+    >>> y_pred = [1, 1, 5, 3.5]
+    >>> d2_tweedie_score(y_true, y_pred)
+    0.285...
+    >>> d2_tweedie_score(y_true, y_pred, power=1)
+    0.487...
+    >>> d2_tweedie_score(y_true, y_pred, power=2)
+    0.630...
+    >>> d2_tweedie_score(y_true, y_true, power=2)
+    1.0
+    """
+    xp, _ = get_namespace(y_true, y_pred)
+
+    y_type, y_true, y_pred, sample_weight, _ = _check_reg_targets_with_floating_dtype(
+        y_true, y_pred, sample_weight, multioutput=None, xp=xp
+    )
+    if y_type == "continuous-multioutput":
+        raise ValueError("Multioutput not supported in d2_tweedie_score")
+
+    if _num_samples(y_pred) < 2:
+        msg = "D^2 score is not well-defined with less than two samples."
+        warnings.warn(msg, UndefinedMetricWarning)
+        return float("nan")
+
+    y_true, y_pred = xp.squeeze(y_true, axis=1), xp.squeeze(y_pred, axis=1)
+    numerator = mean_tweedie_deviance(
+        y_true, y_pred, sample_weight=sample_weight, power=power
+    )
+
+    y_avg = _average(y_true, weights=sample_weight, xp=xp)
+    denominator = _mean_tweedie_deviance(
+        y_true, y_avg, sample_weight=sample_weight, power=power
+    )
+
+    return 1 - numerator / denominator
+
+
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_pred": ["array-like"],
+        "sample_weight": ["array-like", None],
+        "alpha": [Interval(Real, 0, 1, closed="both")],
+        "multioutput": [
+            StrOptions({"raw_values", "uniform_average"}),
+            "array-like",
+        ],
+    },
+    prefer_skip_nested_validation=True,
+)
+def d2_pinball_score(
+    y_true, y_pred, *, sample_weight=None, alpha=0.5, multioutput="uniform_average"
+):
+    """
+    :math:`D^2` regression score function, fraction of pinball loss explained.
+
+    Best possible score is 1.0 and it can be negative (because the model can be
+    arbitrarily worse). A model that always uses the empirical alpha-quantile of
+    `y_true` as constant prediction, disregarding the input features,
+    gets a :math:`D^2` score of 0.0.
+
+    Read more in the :ref:`User Guide `.
+
+    .. versionadded:: 1.1
+
+    Parameters
+    ----------
+    y_true : array-like of shape (n_samples,) or (n_samples, n_outputs)
+        Ground truth (correct) target values.
+
+    y_pred : array-like of shape (n_samples,) or (n_samples, n_outputs)
+        Estimated target values.
+
+    sample_weight : array-like of shape (n_samples,), default=None
+        Sample weights.
+
+    alpha : float, default=0.5
+        Slope of the pinball deviance. It determines the quantile level alpha
+        for which the pinball deviance and also D2 are optimal.
+        The default `alpha=0.5` is equivalent to `d2_absolute_error_score`.
+
+    multioutput : {'raw_values', 'uniform_average'} or array-like of shape \
+            (n_outputs,), default='uniform_average'
+        Defines aggregating of multiple output values.
+        Array-like value defines weights used to average scores.
+
+        'raw_values' :
+            Returns a full set of errors in case of multioutput input.
+
+        'uniform_average' :
+            Scores of all outputs are averaged with uniform weight.
+
+    Returns
+    -------
+    score : float or ndarray of floats
+        The :math:`D^2` score with a pinball deviance
+        or ndarray of scores if `multioutput='raw_values'`.
+
+    Notes
+    -----
+    Like :math:`R^2`, :math:`D^2` score may be negative
+    (it need not actually be the square of a quantity D).
+
+    This metric is not well-defined for a single point and will return a NaN
+    value if n_samples is less than two.
+
+     References
+    ----------
+    .. [1] Eq. (7) of `Koenker, Roger; Machado, JosÊ A. F. (1999).
+           "Goodness of Fit and Related Inference Processes for Quantile Regression"
+           `_
+    .. [2] Eq. (3.11) of Hastie, Trevor J., Robert Tibshirani and Martin J.
+           Wainwright. "Statistical Learning with Sparsity: The Lasso and
+           Generalizations." (2015). https://hastie.su.domains/StatLearnSparsity/
+
+    Examples
+    --------
+    >>> from sklearn.metrics import d2_pinball_score
+    >>> y_true = [1, 2, 3]
+    >>> y_pred = [1, 3, 3]
+    >>> d2_pinball_score(y_true, y_pred)
+    0.5
+    >>> d2_pinball_score(y_true, y_pred, alpha=0.9)
+    0.772...
+    >>> d2_pinball_score(y_true, y_pred, alpha=0.1)
+    -1.045...
+    >>> d2_pinball_score(y_true, y_true, alpha=0.1)
+    1.0
+    """
+    _, y_true, y_pred, sample_weight, multioutput = _check_reg_targets(
+        y_true, y_pred, sample_weight, multioutput
+    )
+
+    if _num_samples(y_pred) < 2:
+        msg = "D^2 score is not well-defined with less than two samples."
+        warnings.warn(msg, UndefinedMetricWarning)
+        return float("nan")
+
+    numerator = mean_pinball_loss(
+        y_true,
+        y_pred,
+        sample_weight=sample_weight,
+        alpha=alpha,
+        multioutput="raw_values",
+    )
+
+    if sample_weight is None:
+        y_quantile = np.tile(
+            np.percentile(y_true, q=alpha * 100, axis=0), (len(y_true), 1)
+        )
+    else:
+        y_quantile = np.tile(
+            _weighted_percentile(
+                y_true, sample_weight=sample_weight, percentile_rank=alpha * 100
+            ),
+            (len(y_true), 1),
+        )
+
+    denominator = mean_pinball_loss(
+        y_true,
+        y_quantile,
+        sample_weight=sample_weight,
+        alpha=alpha,
+        multioutput="raw_values",
+    )
+
+    nonzero_numerator = numerator != 0
+    nonzero_denominator = denominator != 0
+    valid_score = nonzero_numerator & nonzero_denominator
+    output_scores = np.ones(y_true.shape[1])
+
+    output_scores[valid_score] = 1 - (numerator[valid_score] / denominator[valid_score])
+    output_scores[nonzero_numerator & ~nonzero_denominator] = 0.0
+
+    if isinstance(multioutput, str):
+        if multioutput == "raw_values":
+            # return scores individually
+            return output_scores
+        else:  # multioutput == "uniform_average"
+            # passing None as weights to np.average results in uniform mean
+            avg_weights = None
+    else:
+        avg_weights = multioutput
+
+    return float(np.average(output_scores, weights=avg_weights))
+
+
+@validate_params(
+    {
+        "y_true": ["array-like"],
+        "y_pred": ["array-like"],
+        "sample_weight": ["array-like", None],
+        "multioutput": [
+            StrOptions({"raw_values", "uniform_average"}),
+            "array-like",
+        ],
+    },
+    prefer_skip_nested_validation=True,
+)
+def d2_absolute_error_score(
+    y_true, y_pred, *, sample_weight=None, multioutput="uniform_average"
+):
+    """
+    :math:`D^2` regression score function, fraction of absolute error explained.
+
+    Best possible score is 1.0 and it can be negative (because the model can be
+    arbitrarily worse). A model that always uses the empirical median of `y_true`
+    as constant prediction, disregarding the input features,
+    gets a :math:`D^2` score of 0.0.
+
+    Read more in the :ref:`User Guide `.
+
+    .. versionadded:: 1.1
+
+    Parameters
+    ----------
+    y_true : array-like of shape (n_samples,) or (n_samples, n_outputs)
+        Ground truth (correct) target values.
+
+    y_pred : array-like of shape (n_samples,) or (n_samples, n_outputs)
+        Estimated target values.
+
+    sample_weight : array-like of shape (n_samples,), default=None
+        Sample weights.
+
+    multioutput : {'raw_values', 'uniform_average'} or array-like of shape \
+            (n_outputs,), default='uniform_average'
+        Defines aggregating of multiple output values.
+        Array-like value defines weights used to average scores.
+
+        'raw_values' :
+            Returns a full set of errors in case of multioutput input.
+
+        'uniform_average' :
+            Scores of all outputs are averaged with uniform weight.
+
+    Returns
+    -------
+    score : float or ndarray of floats
+        The :math:`D^2` score with an absolute error deviance
+        or ndarray of scores if 'multioutput' is 'raw_values'.
+
+    Notes
+    -----
+    Like :math:`R^2`, :math:`D^2` score may be negative
+    (it need not actually be the square of a quantity D).
+
+    This metric is not well-defined for single samples and will return a NaN
+    value if n_samples is less than two.
+
+     References
+    ----------
+    .. [1] Eq. (3.11) of Hastie, Trevor J., Robert Tibshirani and Martin J.
+           Wainwright. "Statistical Learning with Sparsity: The Lasso and
+           Generalizations." (2015). https://hastie.su.domains/StatLearnSparsity/
+
+    Examples
+    --------
+    >>> from sklearn.metrics import d2_absolute_error_score
+    >>> y_true = [3, -0.5, 2, 7]
+    >>> y_pred = [2.5, 0.0, 2, 8]
+    >>> d2_absolute_error_score(y_true, y_pred)
+    0.764...
+    >>> y_true = [[0.5, 1], [-1, 1], [7, -6]]
+    >>> y_pred = [[0, 2], [-1, 2], [8, -5]]
+    >>> d2_absolute_error_score(y_true, y_pred, multioutput='uniform_average')
+    0.691...
+    >>> d2_absolute_error_score(y_true, y_pred, multioutput='raw_values')
+    array([0.8125    , 0.57142857])
+    >>> y_true = [1, 2, 3]
+    >>> y_pred = [1, 2, 3]
+    >>> d2_absolute_error_score(y_true, y_pred)
+    1.0
+    >>> y_true = [1, 2, 3]
+    >>> y_pred = [2, 2, 2]
+    >>> d2_absolute_error_score(y_true, y_pred)
+    0.0
+    >>> y_true = [1, 2, 3]
+    >>> y_pred = [3, 2, 1]
+    >>> d2_absolute_error_score(y_true, y_pred)
+    -1.0
+    """
+    return d2_pinball_score(
+        y_true, y_pred, sample_weight=sample_weight, alpha=0.5, multioutput=multioutput
     )
diff --git a/sklearn/metrics/_scorer.py b/sklearn/metrics/_scorer.py
index c686d3b7c0b34..08e5a20187de7 100644
--- a/sklearn/metrics/_scorer.py
+++ b/sklearn/metrics/_scorer.py
@@ -13,51 +13,88 @@
 ground truth labeling (or ``None`` in the case of unsupervised models).
 """
 
-# Authors: Andreas Mueller 
-#          Lars Buitinck
-#          Arnaud Joly 
-# License: Simplified BSD
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-from collections.abc import Iterable
-from functools import partial
+import copy
+import warnings
 from collections import Counter
+from functools import partial
+from inspect import signature
+from numbers import Integral
+from traceback import format_exc
 
 import numpy as np
 
-from . import (r2_score, median_absolute_error, max_error, mean_absolute_error,
-               mean_squared_error, mean_squared_log_error,
-               mean_poisson_deviance, mean_gamma_deviance, accuracy_score,
-               top_k_accuracy_score, f1_score, roc_auc_score,
-               average_precision_score, precision_score, recall_score,
-               log_loss, balanced_accuracy_score, explained_variance_score,
-               brier_score_loss, jaccard_score, mean_absolute_percentage_error)
-
-from .cluster import adjusted_rand_score
-from .cluster import rand_score
-from .cluster import homogeneity_score
-from .cluster import completeness_score
-from .cluster import v_measure_score
-from .cluster import mutual_info_score
-from .cluster import adjusted_mutual_info_score
-from .cluster import normalized_mutual_info_score
-from .cluster import fowlkes_mallows_score
-
-from ..utils.multiclass import type_of_target
-from ..utils.validation import _deprecate_positional_args
 from ..base import is_regressor
+from ..utils import Bunch
+from ..utils._param_validation import HasMethods, Hidden, StrOptions, validate_params
+from ..utils._response import _get_response_values
+from ..utils.metadata_routing import (
+    MetadataRequest,
+    MetadataRouter,
+    MethodMapping,
+    _MetadataRequester,
+    _raise_for_params,
+    _routing_enabled,
+    get_routing_for_object,
+    process_routing,
+)
+from ..utils.validation import _check_response_method
+from . import (
+    accuracy_score,
+    average_precision_score,
+    balanced_accuracy_score,
+    brier_score_loss,
+    class_likelihood_ratios,
+    d2_absolute_error_score,
+    explained_variance_score,
+    f1_score,
+    jaccard_score,
+    log_loss,
+    matthews_corrcoef,
+    max_error,
+    mean_absolute_error,
+    mean_absolute_percentage_error,
+    mean_gamma_deviance,
+    mean_poisson_deviance,
+    mean_squared_error,
+    mean_squared_log_error,
+    median_absolute_error,
+    precision_score,
+    r2_score,
+    recall_score,
+    roc_auc_score,
+    root_mean_squared_error,
+    root_mean_squared_log_error,
+    top_k_accuracy_score,
+)
+from .cluster import (
+    adjusted_mutual_info_score,
+    adjusted_rand_score,
+    completeness_score,
+    fowlkes_mallows_score,
+    homogeneity_score,
+    mutual_info_score,
+    normalized_mutual_info_score,
+    rand_score,
+    v_measure_score,
+)
 
 
-def _cached_call(cache, estimator, method, *args, **kwargs):
+def _cached_call(cache, estimator, response_method, *args, **kwargs):
     """Call estimator with method and args and kwargs."""
-    if cache is None:
-        return getattr(estimator, method)(*args, **kwargs)
+    if cache is not None and response_method in cache:
+        return cache[response_method]
 
-    try:
-        return cache[method]
-    except KeyError:
-        result = getattr(estimator, method)(*args, **kwargs)
-        cache[method] = result
-        return result
+    result, _ = _get_response_values(
+        estimator, *args, response_method=response_method, **kwargs
+    )
+
+    if cache is not None:
+        cache[response_method] = result
+
+    return result
 
 
 class _MultimetricScorer:
@@ -72,9 +109,16 @@ class _MultimetricScorer:
     ----------
     scorers : dict
         Dictionary mapping names to callable scorers.
+
+    raise_exc : bool, default=True
+        Whether to raise the exception in `__call__` or not. If set to `False`
+        a formatted string of the exception details is passed as result of
+        the failing scorer.
     """
-    def __init__(self, **scorers):
+
+    def __init__(self, *, scorers, raise_exc=True):
         self._scorers = scorers
+        self._raise_exc = raise_exc
 
     def __call__(self, estimator, *args, **kwargs):
         """Evaluate predicted target values."""
@@ -82,138 +126,146 @@ def __call__(self, estimator, *args, **kwargs):
         cache = {} if self._use_cache(estimator) else None
         cached_call = partial(_cached_call, cache)
 
+        if _routing_enabled():
+            routed_params = process_routing(self, "score", **kwargs)
+        else:
+            # Scorers all get the same args, and get all of them except sample_weight.
+            # Only the ones having `sample_weight` in their signature will receive it.
+            # This does not work for metadata other than sample_weight, and for those
+            # users have to enable metadata routing.
+            common_kwargs = {
+                arg: value for arg, value in kwargs.items() if arg != "sample_weight"
+            }
+            routed_params = Bunch(
+                **{name: Bunch(score=common_kwargs.copy()) for name in self._scorers}
+            )
+            if "sample_weight" in kwargs:
+                for name, scorer in self._scorers.items():
+                    if scorer._accept_sample_weight():
+                        routed_params[name].score["sample_weight"] = kwargs[
+                            "sample_weight"
+                        ]
+
         for name, scorer in self._scorers.items():
-            if isinstance(scorer, _BaseScorer):
-                score = scorer._score(cached_call, estimator,
-                                      *args, **kwargs)
-            else:
-                score = scorer(estimator, *args, **kwargs)
-            scores[name] = score
+            try:
+                if isinstance(scorer, _BaseScorer):
+                    score = scorer._score(
+                        cached_call, estimator, *args, **routed_params.get(name).score
+                    )
+                else:
+                    score = scorer(estimator, *args, **routed_params.get(name).score)
+                scores[name] = score
+            except Exception as e:
+                if self._raise_exc:
+                    raise e
+                else:
+                    scores[name] = format_exc()
         return scores
 
-    def _use_cache(self, estimator):
-        """Return True if using a cache is beneficial.
+    def __repr__(self):
+        scorers = ", ".join([f'"{s}"' for s in self._scorers])
+        return f"MultiMetricScorer({scorers})"
 
-        Caching may be beneficial when one of these conditions holds:
-          - `_ProbaScorer` will be called twice.
-          - `_PredictScorer` will be called twice.
-          - `_ThresholdScorer` will be called twice.
-          - `_ThresholdScorer` and `_PredictScorer` are called and
-             estimator is a regressor.
-          - `_ThresholdScorer` and `_ProbaScorer` are called and
-             estimator does not have a `decision_function` attribute.
+    def _accept_sample_weight(self):
+        # TODO(slep006): remove when metadata routing is the only way
+        return any(scorer._accept_sample_weight() for scorer in self._scorers.values())
 
+    def _use_cache(self, estimator):
+        """Return True if using a cache is beneficial, thus when a response method will
+        be called several time.
         """
         if len(self._scorers) == 1:  # Only one scorer
             return False
 
-        counter = Counter([type(v) for v in self._scorers.values()])
-
-        if any(counter[known_type] > 1 for known_type in
-               [_PredictScorer, _ProbaScorer, _ThresholdScorer]):
+        counter = Counter(
+            [
+                _check_response_method(estimator, scorer._response_method).__name__
+                for scorer in self._scorers.values()
+                if isinstance(scorer, _BaseScorer)
+            ]
+        )
+        if any(val > 1 for val in counter.values()):
+            # The exact same response method or iterable of response methods
+            # will be called more than once.
             return True
 
-        if counter[_ThresholdScorer]:
-            if is_regressor(estimator) and counter[_PredictScorer]:
-                return True
-            elif (counter[_ProbaScorer] and
-                  not hasattr(estimator, "decision_function")):
-                return True
         return False
 
+    def get_metadata_routing(self):
+        """Get metadata routing of this object.
 
-class _BaseScorer:
-    def __init__(self, score_func, sign, kwargs):
-        self._kwargs = kwargs
-        self._score_func = score_func
-        self._sign = sign
-
-    @staticmethod
-    def _check_pos_label(pos_label, classes):
-        if pos_label not in list(classes):
-            raise ValueError(
-                f"pos_label={pos_label} is not a valid label: {classes}"
-            )
-
-    def _select_proba_binary(self, y_pred, classes):
-        """Select the column of the positive label in `y_pred` when
-        probabilities are provided.
-
-        Parameters
-        ----------
-        y_pred : ndarray of shape (n_samples, n_classes)
-            The prediction given by `predict_proba`.
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
 
-        classes : ndarray of shape (n_classes,)
-            The class labels for the estimator.
+        .. versionadded:: 1.3
 
         Returns
         -------
-        y_pred : ndarray of shape (n_samples,)
-            Probability predictions of the positive class.
+        routing : MetadataRouter
+            A :class:`~utils.metadata_routing.MetadataRouter` encapsulating
+            routing information.
         """
-        if y_pred.shape[1] == 2:
-            pos_label = self._kwargs.get("pos_label", classes[1])
-            self._check_pos_label(pos_label, classes)
-            col_idx = np.flatnonzero(classes == pos_label)[0]
-            return y_pred[:, col_idx]
-
-        err_msg = (
-            f"Got predict_proba of shape {y_pred.shape}, but need "
-            f"classifier with two classes for {self._score_func.__name__} "
-            f"scoring"
+        return MetadataRouter(owner=self.__class__.__name__).add(
+            **self._scorers,
+            method_mapping=MethodMapping().add(caller="score", callee="score"),
         )
-        raise ValueError(err_msg)
 
-    def __repr__(self):
-        kwargs_string = "".join([", %s=%s" % (str(k), str(v))
-                                 for k, v in self._kwargs.items()])
-        return ("make_scorer(%s%s%s%s)"
-                % (self._score_func.__name__,
-                   "" if self._sign > 0 else ", greater_is_better=False",
-                   self._factory_args(), kwargs_string))
-
-    def __call__(self, estimator, X, y_true, sample_weight=None):
-        """Evaluate predicted target values for X relative to y_true.
 
-        Parameters
-        ----------
-        estimator : object
-            Trained estimator to use for scoring. Must have a predict_proba
-            method; the output of that is used to compute the score.
+class _BaseScorer(_MetadataRequester):
+    """Base scorer that is used as `scorer(estimator, X, y_true)`.
 
-        X : {array-like, sparse matrix}
-            Test data that will be fed to estimator.predict.
+    Parameters
+    ----------
+    score_func : callable
+        The score function to use. It will be called as
+        `score_func(y_true, y_pred, **kwargs)`.
 
-        y_true : array-like
-            Gold standard target values for X.
+    sign : int
+        Either 1 or -1 to returns the score with `sign * score_func(estimator, X, y)`.
+        Thus, `sign` defined if higher scores are better or worse.
 
-        sample_weight : array-like of shape (n_samples,), default=None
-            Sample weights.
+    kwargs : dict
+        Additional parameters to pass to the score function.
 
-        Returns
-        -------
-        score : float
-            Score function applied to prediction of estimator on X.
-        """
-        return self._score(partial(_cached_call, None), estimator, X, y_true,
-                           sample_weight=sample_weight)
+    response_method : str
+        The method to call on the estimator to get the response values.
+    """
 
-    def _factory_args(self):
-        """Return non-default make_scorer arguments for repr."""
-        return ""
+    def __init__(self, score_func, sign, kwargs, response_method="predict"):
+        self._score_func = score_func
+        self._sign = sign
+        self._kwargs = kwargs
+        self._response_method = response_method
+        # TODO (1.8): remove in 1.8 (scoring="max_error" has been deprecated in 1.6)
+        self._deprecation_msg = None
+
+    def _get_pos_label(self):
+        if "pos_label" in self._kwargs:
+            return self._kwargs["pos_label"]
+        score_func_params = signature(self._score_func).parameters
+        if "pos_label" in score_func_params:
+            return score_func_params["pos_label"].default
+        return None
+
+    def _accept_sample_weight(self):
+        # TODO(slep006): remove when metadata routing is the only way
+        return "sample_weight" in signature(self._score_func).parameters
 
+    def __repr__(self):
+        sign_string = "" if self._sign > 0 else ", greater_is_better=False"
+        response_method_string = f", response_method={self._response_method!r}"
+        kwargs_string = "".join([f", {k}={v}" for k, v in self._kwargs.items()])
 
-class _PredictScorer(_BaseScorer):
-    def _score(self, method_caller, estimator, X, y_true, sample_weight=None):
+        return (
+            f"make_scorer({self._score_func.__name__}{sign_string}"
+            f"{response_method_string}{kwargs_string})"
+        )
+
+    def __call__(self, estimator, X, y_true, sample_weight=None, **kwargs):
         """Evaluate predicted target values for X relative to y_true.
 
         Parameters
         ----------
-        method_caller : callable
-            Returns predictions given an estimator, method name, and other
-            arguments, potentially caching results.
-
         estimator : object
             Trained estimator to use for scoring. Must have a predict_proba
             method; the output of that is used to compute the score.
@@ -227,73 +279,86 @@ def _score(self, method_caller, estimator, X, y_true, sample_weight=None):
         sample_weight : array-like of shape (n_samples,), default=None
             Sample weights.
 
+        **kwargs : dict
+            Other parameters passed to the scorer. Refer to
+            :func:`set_score_request` for more details.
+
+            Only available if `enable_metadata_routing=True`. See the
+            :ref:`User Guide `.
+
+            .. versionadded:: 1.3
+
         Returns
         -------
         score : float
             Score function applied to prediction of estimator on X.
         """
+        # TODO (1.8): remove in 1.8 (scoring="max_error" has been deprecated in 1.6)
+        if self._deprecation_msg is not None:
+            warnings.warn(
+                self._deprecation_msg, category=DeprecationWarning, stacklevel=2
+            )
 
-        y_pred = method_caller(estimator, "predict", X)
-        if sample_weight is not None:
-            return self._sign * self._score_func(y_true, y_pred,
-                                                 sample_weight=sample_weight,
-                                                 **self._kwargs)
-        else:
-            return self._sign * self._score_func(y_true, y_pred,
-                                                 **self._kwargs)
+        _raise_for_params(kwargs, self, None)
 
+        _kwargs = copy.deepcopy(kwargs)
+        if sample_weight is not None:
+            _kwargs["sample_weight"] = sample_weight
 
-class _ProbaScorer(_BaseScorer):
-    def _score(self, method_caller, clf, X, y, sample_weight=None):
-        """Evaluate predicted probabilities for X relative to y_true.
+        return self._score(partial(_cached_call, None), estimator, X, y_true, **_kwargs)
 
-        Parameters
-        ----------
-        method_caller : callable
-            Returns predictions given an estimator, method name, and other
-            arguments, potentially caching results.
+    def _warn_overlap(self, message, kwargs):
+        """Warn if there is any overlap between ``self._kwargs`` and ``kwargs``.
 
-        clf : object
-            Trained classifier to use for scoring. Must have a predict_proba
-            method; the output of that is used to compute the score.
+        This method is intended to be used to check for overlap between
+        ``self._kwargs`` and ``kwargs`` passed as metadata.
+        """
+        _kwargs = set() if self._kwargs is None else set(self._kwargs.keys())
+        overlap = _kwargs.intersection(kwargs.keys())
+        if overlap:
+            warnings.warn(
+                f"{message} Overlapping parameters are: {overlap}", UserWarning
+            )
 
-        X : {array-like, sparse matrix}
-            Test data that will be fed to clf.predict_proba.
+    def set_score_request(self, **kwargs):
+        """Set requested parameters by the scorer.
 
-        y : array-like
-            Gold standard target values for X. These must be class labels,
-            not probabilities.
+        Please see :ref:`User Guide ` on how the routing
+        mechanism works.
 
-        sample_weight : array-like, default=None
-            Sample weights.
+        .. versionadded:: 1.3
 
-        Returns
-        -------
-        score : float
-            Score function applied to prediction of estimator on X.
+        Parameters
+        ----------
+        kwargs : dict
+            Arguments should be of the form ``param_name=alias``, and `alias`
+            can be one of ``{True, False, None, str}``.
         """
+        if not _routing_enabled():
+            raise RuntimeError(
+                "This method is only available when metadata routing is enabled."
+                " You can enable it using"
+                " sklearn.set_config(enable_metadata_routing=True)."
+            )
 
-        y_type = type_of_target(y)
-        y_pred = method_caller(clf, "predict_proba", X)
-        if y_type == "binary" and y_pred.shape[1] <= 2:
-            # `y_type` could be equal to "binary" even in a multi-class
-            # problem: (when only 2 class are given to `y_true` during scoring)
-            # Thus, we need to check for the shape of `y_pred`.
-            y_pred = self._select_proba_binary(y_pred, clf.classes_)
-        if sample_weight is not None:
-            return self._sign * self._score_func(y, y_pred,
-                                                 sample_weight=sample_weight,
-                                                 **self._kwargs)
-        else:
-            return self._sign * self._score_func(y, y_pred, **self._kwargs)
-
-    def _factory_args(self):
-        return ", needs_proba=True"
+        self._warn_overlap(
+            message=(
+                "You are setting metadata request for parameters which are "
+                "already set as kwargs for this metric. These set values will be "
+                "overridden by passed metadata if provided. Please pass them either "
+                "as metadata or kwargs to `make_scorer`."
+            ),
+            kwargs=kwargs,
+        )
+        self._metadata_request = MetadataRequest(owner=self.__class__.__name__)
+        for param, alias in kwargs.items():
+            self._metadata_request.score.add_request(param=param, alias=alias)
+        return self
 
 
-class _ThresholdScorer(_BaseScorer):
-    def _score(self, method_caller, clf, X, y, sample_weight=None):
-        """Evaluate decision function output for X relative to y_true.
+class _Scorer(_BaseScorer):
+    def _score(self, method_caller, estimator, X, y_true, **kwargs):
+        """Evaluate the response method of `estimator` on `X` and `y_true`.
 
         Parameters
         ----------
@@ -301,168 +366,187 @@ def _score(self, method_caller, clf, X, y, sample_weight=None):
             Returns predictions given an estimator, method name, and other
             arguments, potentially caching results.
 
-        clf : object
-            Trained classifier to use for scoring. Must have either a
-            decision_function method or a predict_proba method; the output of
-            that is used to compute the score.
+        estimator : object
+            Trained estimator to use for scoring.
 
         X : {array-like, sparse matrix}
             Test data that will be fed to clf.decision_function or
             clf.predict_proba.
 
-        y : array-like
+        y_true : array-like
             Gold standard target values for X. These must be class labels,
             not decision function values.
 
-        sample_weight : array-like, default=None
-            Sample weights.
+        **kwargs : dict
+            Other parameters passed to the scorer. Refer to
+            :func:`set_score_request` for more details.
 
         Returns
         -------
         score : float
             Score function applied to prediction of estimator on X.
         """
+        self._warn_overlap(
+            message=(
+                "There is an overlap between set kwargs of this scorer instance and"
+                " passed metadata. Please pass them either as kwargs to `make_scorer`"
+                " or metadata, but not both."
+            ),
+            kwargs=kwargs,
+        )
 
-        y_type = type_of_target(y)
-        if y_type not in ("binary", "multilabel-indicator"):
-            raise ValueError("{0} format is not supported".format(y_type))
-
-        if is_regressor(clf):
-            y_pred = method_caller(clf, "predict", X)
-        else:
-            try:
-                y_pred = method_caller(clf, "decision_function", X)
-
-                if isinstance(y_pred, list):
-                    # For multi-output multi-class estimator
-                    y_pred = np.vstack([p for p in y_pred]).T
-                elif y_type == "binary" and "pos_label" in self._kwargs:
-                    self._check_pos_label(
-                        self._kwargs["pos_label"], clf.classes_
-                    )
-                    if self._kwargs["pos_label"] == clf.classes_[0]:
-                        # The implicit positive class of the binary classifier
-                        # does not match `pos_label`: we need to invert the
-                        # predictions
-                        y_pred *= -1
-
-            except (NotImplementedError, AttributeError):
-                y_pred = method_caller(clf, "predict_proba", X)
-
-                if y_type == "binary":
-                    y_pred = self._select_proba_binary(y_pred, clf.classes_)
-                elif isinstance(y_pred, list):
-                    y_pred = np.vstack([p[:, -1] for p in y_pred]).T
-
-        if sample_weight is not None:
-            return self._sign * self._score_func(y, y_pred,
-                                                 sample_weight=sample_weight,
-                                                 **self._kwargs)
-        else:
-            return self._sign * self._score_func(y, y_pred, **self._kwargs)
+        pos_label = None if is_regressor(estimator) else self._get_pos_label()
+        response_method = _check_response_method(estimator, self._response_method)
+        y_pred = method_caller(
+            estimator,
+            _get_response_method_name(response_method),
+            X,
+            pos_label=pos_label,
+        )
 
-    def _factory_args(self):
-        return ", needs_threshold=True"
+        scoring_kwargs = {**self._kwargs, **kwargs}
+        return self._sign * self._score_func(y_true, y_pred, **scoring_kwargs)
 
 
+@validate_params(
+    {
+        "scoring": [str, callable, None],
+    },
+    prefer_skip_nested_validation=True,
+)
 def get_scorer(scoring):
     """Get a scorer from string.
 
     Read more in the :ref:`User Guide `.
+    :func:`~sklearn.metrics.get_scorer_names` can be used to retrieve the names
+    of all available scorers.
 
     Parameters
     ----------
-    scoring : str or callable
+    scoring : str, callable or None
         Scoring method as string. If callable it is returned as is.
+        If None, returns None.
 
     Returns
     -------
     scorer : callable
         The scorer.
+
+    Notes
+    -----
+    When passed a string, this function always returns a copy of the scorer
+    object. Calling `get_scorer` twice for the same scorer results in two
+    separate scorer objects.
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from sklearn.dummy import DummyClassifier
+    >>> from sklearn.metrics import get_scorer
+    >>> X = np.reshape([0, 1, -1, -0.5, 2], (-1, 1))
+    >>> y = np.array([0, 1, 1, 0, 1])
+    >>> classifier = DummyClassifier(strategy="constant", constant=0).fit(X, y)
+    >>> accuracy = get_scorer("accuracy")
+    >>> accuracy(classifier, X, y)
+    0.4
     """
     if isinstance(scoring, str):
         try:
-            scorer = SCORERS[scoring]
+            if scoring == "max_error":
+                # TODO (1.8): scoring="max_error" has been deprecated in 1.6,
+                # remove in 1.8
+                scorer = max_error_scorer
+            else:
+                scorer = copy.deepcopy(_SCORERS[scoring])
         except KeyError:
-            raise ValueError('%r is not a valid scoring value. '
-                             'Use sorted(sklearn.metrics.SCORERS.keys()) '
-                             'to get valid options.' % scoring)
+            raise ValueError(
+                "%r is not a valid scoring value. "
+                "Use sklearn.metrics.get_scorer_names() "
+                "to get valid options." % scoring
+            )
     else:
         scorer = scoring
     return scorer
 
 
-def _passthrough_scorer(estimator, *args, **kwargs):
-    """Function that wraps estimator.score"""
-    return estimator.score(*args, **kwargs)
+class _PassthroughScorer(_MetadataRequester):
+    # Passes scoring of estimator's `score` method back to estimator if scoring
+    # is `None`.
 
+    def __init__(self, estimator):
+        self._estimator = estimator
 
-@_deprecate_positional_args
-def check_scoring(estimator, scoring=None, *, allow_none=False):
-    """Determine scorer from user options.
+        requests = MetadataRequest(owner=self.__class__.__name__)
+        try:
+            requests.score = copy.deepcopy(estimator._metadata_request.score)
+        except AttributeError:
+            try:
+                requests.score = copy.deepcopy(estimator._get_default_requests().score)
+            except AttributeError:
+                pass
 
-    A TypeError will be thrown if the estimator cannot be scored.
+        self._metadata_request = requests
 
-    Parameters
-    ----------
-    estimator : estimator object implementing 'fit'
-        The object to use to fit the data.
+    def __call__(self, estimator, *args, **kwargs):
+        """Method that wraps estimator.score"""
+        return estimator.score(*args, **kwargs)
 
-    scoring : str or callable, default=None
-        A string (see model evaluation documentation) or
-        a scorer callable object / function with signature
-        ``scorer(estimator, X, y)``.
+    def __repr__(self):
+        return f"{self._estimator.__class__}.score"
 
-    allow_none : bool, default=False
-        If no scoring is specified and the estimator has no score function, we
-        can either return None or raise an exception.
+    def _accept_sample_weight(self):
+        # TODO(slep006): remove when metadata routing is the only way
+        return "sample_weight" in signature(self._estimator.score).parameters
 
-    Returns
-    -------
-    scoring : callable
-        A scorer callable object / function with signature
-        ``scorer(estimator, X, y)``.
-    """
-    if not hasattr(estimator, 'fit'):
-        raise TypeError("estimator should be an estimator implementing "
-                        "'fit' method, %r was passed" % estimator)
-    if isinstance(scoring, str):
-        return get_scorer(scoring)
-    elif callable(scoring):
-        # Heuristic to ensure user has not passed a metric
-        module = getattr(scoring, '__module__', None)
-        if hasattr(module, 'startswith') and \
-           module.startswith('sklearn.metrics.') and \
-           not module.startswith('sklearn.metrics._scorer') and \
-           not module.startswith('sklearn.metrics.tests.'):
-            raise ValueError('scoring value %r looks like it is a metric '
-                             'function rather than a scorer. A scorer should '
-                             'require an estimator as its first parameter. '
-                             'Please use `make_scorer` to convert a metric '
-                             'to a scorer.' % scoring)
-        return get_scorer(scoring)
-    elif scoring is None:
-        if hasattr(estimator, 'score'):
-            return _passthrough_scorer
-        elif allow_none:
-            return None
-        else:
-            raise TypeError(
-                "If no scoring is specified, the estimator passed should "
-                "have a 'score' method. The estimator %r does not."
-                % estimator)
-    elif isinstance(scoring, Iterable):
-        raise ValueError("For evaluating multiple scores, use "
-                         "sklearn.model_selection.cross_validate instead. "
-                         "{0} was passed.".format(scoring))
-    else:
-        raise ValueError("scoring value should either be a callable, string or"
-                         " None. %r was passed" % scoring)
+    def get_metadata_routing(self):
+        """Get requested data properties.
+
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
+
+        .. versionadded:: 1.3
+
+        Returns
+        -------
+        routing : MetadataRouter
+            A :class:`~utils.metadata_routing.MetadataRouter` encapsulating
+            routing information.
+        """
+        return get_routing_for_object(self._metadata_request)
+
+    def set_score_request(self, **kwargs):
+        """Set requested parameters by the scorer.
+
+        Please see :ref:`User Guide ` on how the routing
+        mechanism works.
+
+        .. versionadded:: 1.5
+
+        Parameters
+        ----------
+        kwargs : dict
+            Arguments should be of the form ``param_name=alias``, and `alias`
+            can be one of ``{True, False, None, str}``.
+        """
+        if not _routing_enabled():
+            raise RuntimeError(
+                "This method is only available when metadata routing is enabled."
+                " You can enable it using"
+                " sklearn.set_config(enable_metadata_routing=True)."
+            )
+
+        for param, alias in kwargs.items():
+            self._metadata_request.score.add_request(param=param, alias=alias)
+        return self
 
 
 def _check_multimetric_scoring(estimator, scoring):
     """Check the scoring parameter in cases when multiple metrics are allowed.
 
+    In addition, multimetric scoring leverages a caching mechanism to not call the same
+    estimator response method multiple times. Hence, the scorer is modified to only use
+    a single response method given a list of response methods and the estimator.
+
     Parameters
     ----------
     estimator : sklearn estimator instance
@@ -489,69 +573,98 @@ def _check_multimetric_scoring(estimator, scoring):
     err_msg_generic = (
         f"scoring is invalid (got {scoring!r}). Refer to the "
         "scoring glossary for details: "
-        "https://scikit-learn.org/stable/glossary.html#term-scoring")
+        "https://scikit-learn.org/stable/glossary.html#term-scoring"
+    )
 
     if isinstance(scoring, (list, tuple, set)):
-        err_msg = ("The list/tuple elements must be unique "
-                   "strings of predefined scorers. ")
-        invalid = False
+        err_msg = (
+            "The list/tuple elements must be unique strings of predefined scorers. "
+        )
         try:
             keys = set(scoring)
-        except TypeError:
-            invalid = True
-        if invalid:
-            raise ValueError(err_msg)
+        except TypeError as e:
+            raise ValueError(err_msg) from e
 
         if len(keys) != len(scoring):
-            raise ValueError(f"{err_msg} Duplicate elements were found in"
-                             f" the given list. {scoring!r}")
+            raise ValueError(
+                f"{err_msg} Duplicate elements were found in"
+                f" the given list. {scoring!r}"
+            )
         elif len(keys) > 0:
             if not all(isinstance(k, str) for k in keys):
                 if any(callable(k) for k in keys):
-                    raise ValueError(f"{err_msg} One or more of the elements "
-                                     "were callables. Use a dict of score "
-                                     "name mapped to the scorer callable. "
-                                     f"Got {scoring!r}")
+                    raise ValueError(
+                        f"{err_msg} One or more of the elements "
+                        "were callables. Use a dict of score "
+                        "name mapped to the scorer callable. "
+                        f"Got {scoring!r}"
+                    )
                 else:
-                    raise ValueError(f"{err_msg} Non-string types were found "
-                                     f"in the given list. Got {scoring!r}")
-            scorers = {scorer: check_scoring(estimator, scoring=scorer)
-                       for scorer in scoring}
+                    raise ValueError(
+                        f"{err_msg} Non-string types were found "
+                        f"in the given list. Got {scoring!r}"
+                    )
+            scorers = {
+                scorer: check_scoring(estimator, scoring=scorer) for scorer in scoring
+            }
         else:
             raise ValueError(f"{err_msg} Empty list was given. {scoring!r}")
 
     elif isinstance(scoring, dict):
         keys = set(scoring)
         if not all(isinstance(k, str) for k in keys):
-            raise ValueError("Non-string types were found in the keys of "
-                             f"the given dict. scoring={scoring!r}")
+            raise ValueError(
+                "Non-string types were found in the keys of "
+                f"the given dict. scoring={scoring!r}"
+            )
         if len(keys) == 0:
             raise ValueError(f"An empty dict was passed. {scoring!r}")
-        scorers = {key: check_scoring(estimator, scoring=scorer)
-                   for key, scorer in scoring.items()}
+        scorers = {
+            key: check_scoring(estimator, scoring=scorer)
+            for key, scorer in scoring.items()
+        }
     else:
         raise ValueError(err_msg_generic)
+
     return scorers
 
 
-@_deprecate_positional_args
-def make_scorer(score_func, *, greater_is_better=True, needs_proba=False,
-                needs_threshold=False, **kwargs):
+def _get_response_method_name(response_method):
+    try:
+        return response_method.__name__
+    except AttributeError:
+        return _get_response_method_name(response_method.func)
+
+
+@validate_params(
+    {
+        "score_func": [callable],
+        "response_method": [
+            None,
+            list,
+            tuple,
+            StrOptions({"predict", "predict_proba", "decision_function"}),
+            Hidden(StrOptions({"default"})),
+        ],
+        "greater_is_better": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def make_scorer(
+    score_func, *, response_method="default", greater_is_better=True, **kwargs
+):
     """Make a scorer from a performance metric or loss function.
 
-    This factory function wraps scoring functions for use in
-    :class:`~sklearn.model_selection.GridSearchCV` and
-    :func:`~sklearn.model_selection.cross_val_score`.
-    It takes a score function, such as :func:`~sklearn.metrics.accuracy_score`,
-    :func:`~sklearn.metrics.mean_squared_error`,
-    :func:`~sklearn.metrics.adjusted_rand_index` or
-    :func:`~sklearn.metrics.average_precision`
-    and returns a callable that scores an estimator's output.
-    The signature of the call is `(estimator, X, y)` where `estimator`
-    is the model to be evaluated, `X` is the data and `y` is the
-    ground truth labeling (or `None` in the case of unsupervised models).
+    A scorer is a wrapper around an arbitrary metric or loss function that is called
+    with the signature `scorer(estimator, X, y_true, **kwargs)`.
+
+    It is accepted in all scikit-learn estimators or functions allowing a `scoring`
+    parameter.
+
+    The parameter `response_method` allows to specify which method of the estimator
+    should be used to feed the scoring/loss function.
 
-    Read more in the :ref:`User Guide `.
+    Read more in the :ref:`User Guide `.
 
     Parameters
     ----------
@@ -559,33 +672,32 @@ def make_scorer(score_func, *, greater_is_better=True, needs_proba=False,
         Score function (or loss function) with signature
         ``score_func(y, y_pred, **kwargs)``.
 
-    greater_is_better : bool, default=True
-        Whether score_func is a score function (default), meaning high is good,
-        or a loss function, meaning low is good. In the latter case, the
-        scorer object will sign-flip the outcome of the score_func.
+    response_method : {"predict_proba", "decision_function", "predict"} or \
+            list/tuple of such str, default=None
 
-    needs_proba : bool, default=False
-        Whether score_func requires predict_proba to get probability estimates
-        out of a classifier.
+        Specifies the response method to use get prediction from an estimator
+        (i.e. :term:`predict_proba`, :term:`decision_function` or
+        :term:`predict`). Possible choices are:
 
-        If True, for binary `y_true`, the score function is supposed to accept
-        a 1D `y_pred` (i.e., probability of the positive class, shape
-        `(n_samples,)`).
+        - if `str`, it corresponds to the name to the method to return;
+        - if a list or tuple of `str`, it provides the method names in order of
+          preference. The method returned corresponds to the first method in
+          the list and which is implemented by `estimator`.
+        - if `None`, it is equivalent to `"predict"`.
 
-    needs_threshold : bool, default=False
-        Whether score_func takes a continuous decision certainty.
-        This only works for binary classification using estimators that
-        have either a decision_function or predict_proba method.
+        .. versionadded:: 1.4
 
-        If True, for binary `y_true`, the score function is supposed to accept
-        a 1D `y_pred` (i.e., probability of the positive class or the decision
-        function, shape `(n_samples,)`).
+        .. deprecated:: 1.6
+            None is equivalent to 'predict' and is deprecated. It will be removed in
+            version 1.8.
 
-        For example ``average_precision`` or the area under the roc curve
-        can not be computed using discrete predictions alone.
+    greater_is_better : bool, default=True
+        Whether `score_func` is a score function (default), meaning high is
+        good, or a loss function, meaning low is good. In the latter case, the
+        scorer object will sign-flip the outcome of the `score_func`.
 
     **kwargs : additional arguments
-        Additional parameters to be passed to score_func.
+        Additional parameters to be passed to `score_func`.
 
     Returns
     -------
@@ -597,54 +709,58 @@ def make_scorer(score_func, *, greater_is_better=True, needs_proba=False,
     >>> from sklearn.metrics import fbeta_score, make_scorer
     >>> ftwo_scorer = make_scorer(fbeta_score, beta=2)
     >>> ftwo_scorer
-    make_scorer(fbeta_score, beta=2)
+    make_scorer(fbeta_score, response_method='predict', beta=2)
     >>> from sklearn.model_selection import GridSearchCV
     >>> from sklearn.svm import LinearSVC
     >>> grid = GridSearchCV(LinearSVC(), param_grid={'C': [1, 10]},
     ...                     scoring=ftwo_scorer)
-
-    Notes
-    -----
-    If `needs_proba=False` and `needs_threshold=False`, the score
-    function is supposed to accept the output of :term:`predict`. If
-    `needs_proba=True`, the score function is supposed to accept the
-    output of :term:`predict_proba` (For binary `y_true`, the score function is
-    supposed to accept probability of the positive class). If
-    `needs_threshold=True`, the score function is supposed to accept the
-    output of :term:`decision_function`.
     """
     sign = 1 if greater_is_better else -1
-    if needs_proba and needs_threshold:
-        raise ValueError("Set either needs_proba or needs_threshold to True,"
-                         " but not both.")
-    if needs_proba:
-        cls = _ProbaScorer
-    elif needs_threshold:
-        cls = _ThresholdScorer
-    else:
-        cls = _PredictScorer
-    return cls(score_func, sign, kwargs)
+
+    if response_method is None:
+        warnings.warn(
+            "response_method=None is deprecated in version 1.6 and will be removed "
+            "in version 1.8. Leave it to its default value to avoid this warning.",
+            FutureWarning,
+        )
+        response_method = "predict"
+    elif response_method == "default":
+        response_method = "predict"
+
+    return _Scorer(score_func, sign, kwargs, response_method)
 
 
 # Standard regression scores
 explained_variance_scorer = make_scorer(explained_variance_score)
 r2_scorer = make_scorer(r2_score)
-max_error_scorer = make_scorer(max_error,
-                               greater_is_better=False)
-neg_mean_squared_error_scorer = make_scorer(mean_squared_error,
-                                            greater_is_better=False)
-neg_mean_squared_log_error_scorer = make_scorer(mean_squared_log_error,
-                                                greater_is_better=False)
-neg_mean_absolute_error_scorer = make_scorer(mean_absolute_error,
-                                             greater_is_better=False)
+neg_max_error_scorer = make_scorer(max_error, greater_is_better=False)
+max_error_scorer = make_scorer(max_error, greater_is_better=False)
+# TODO (1.8): remove in 1.8 (scoring="max_error" has been deprecated in 1.6)
+deprecation_msg = (
+    "Scoring method max_error was renamed to "
+    "neg_max_error in version 1.6 and will "
+    "be removed in 1.8."
+)
+max_error_scorer._deprecation_msg = deprecation_msg
+neg_mean_squared_error_scorer = make_scorer(mean_squared_error, greater_is_better=False)
+neg_mean_squared_log_error_scorer = make_scorer(
+    mean_squared_log_error, greater_is_better=False
+)
+neg_mean_absolute_error_scorer = make_scorer(
+    mean_absolute_error, greater_is_better=False
+)
 neg_mean_absolute_percentage_error_scorer = make_scorer(
     mean_absolute_percentage_error, greater_is_better=False
 )
-neg_median_absolute_error_scorer = make_scorer(median_absolute_error,
-                                               greater_is_better=False)
-neg_root_mean_squared_error_scorer = make_scorer(mean_squared_error,
-                                                 greater_is_better=False,
-                                                 squared=False)
+neg_median_absolute_error_scorer = make_scorer(
+    median_absolute_error, greater_is_better=False
+)
+neg_root_mean_squared_error_scorer = make_scorer(
+    root_mean_squared_error, greater_is_better=False
+)
+neg_root_mean_squared_log_error_scorer = make_scorer(
+    root_mean_squared_log_error, greater_is_better=False
+)
 neg_mean_poisson_deviance_scorer = make_scorer(
     mean_poisson_deviance, greater_is_better=False
 )
@@ -652,39 +768,71 @@ def make_scorer(score_func, *, greater_is_better=True, needs_proba=False,
 neg_mean_gamma_deviance_scorer = make_scorer(
     mean_gamma_deviance, greater_is_better=False
 )
+d2_absolute_error_scorer = make_scorer(d2_absolute_error_score)
 
 # Standard Classification Scores
 accuracy_scorer = make_scorer(accuracy_score)
 balanced_accuracy_scorer = make_scorer(balanced_accuracy_score)
+matthews_corrcoef_scorer = make_scorer(matthews_corrcoef)
+
+
+def positive_likelihood_ratio(y_true, y_pred):
+    return class_likelihood_ratios(y_true, y_pred, replace_undefined_by=1.0)[0]
+
+
+def negative_likelihood_ratio(y_true, y_pred):
+    return class_likelihood_ratios(y_true, y_pred, replace_undefined_by=1.0)[1]
+
+
+positive_likelihood_ratio_scorer = make_scorer(positive_likelihood_ratio)
+neg_negative_likelihood_ratio_scorer = make_scorer(
+    negative_likelihood_ratio, greater_is_better=False
+)
 
 # Score functions that need decision values
-top_k_accuracy_scorer = make_scorer(top_k_accuracy_score,
-                                    greater_is_better=True,
-                                    needs_threshold=True)
-roc_auc_scorer = make_scorer(roc_auc_score, greater_is_better=True,
-                             needs_threshold=True)
-average_precision_scorer = make_scorer(average_precision_score,
-                                       needs_threshold=True)
-roc_auc_ovo_scorer = make_scorer(roc_auc_score, needs_proba=True,
-                                 multi_class='ovo')
-roc_auc_ovo_weighted_scorer = make_scorer(roc_auc_score, needs_proba=True,
-                                          multi_class='ovo',
-                                          average='weighted')
-roc_auc_ovr_scorer = make_scorer(roc_auc_score, needs_proba=True,
-                                 multi_class='ovr')
-roc_auc_ovr_weighted_scorer = make_scorer(roc_auc_score, needs_proba=True,
-                                          multi_class='ovr',
-                                          average='weighted')
+top_k_accuracy_scorer = make_scorer(
+    top_k_accuracy_score,
+    greater_is_better=True,
+    response_method=("decision_function", "predict_proba"),
+)
+roc_auc_scorer = make_scorer(
+    roc_auc_score,
+    greater_is_better=True,
+    response_method=("decision_function", "predict_proba"),
+)
+average_precision_scorer = make_scorer(
+    average_precision_score,
+    response_method=("decision_function", "predict_proba"),
+)
+roc_auc_ovo_scorer = make_scorer(
+    roc_auc_score, response_method="predict_proba", multi_class="ovo"
+)
+roc_auc_ovo_weighted_scorer = make_scorer(
+    roc_auc_score,
+    response_method="predict_proba",
+    multi_class="ovo",
+    average="weighted",
+)
+roc_auc_ovr_scorer = make_scorer(
+    roc_auc_score, response_method="predict_proba", multi_class="ovr"
+)
+roc_auc_ovr_weighted_scorer = make_scorer(
+    roc_auc_score,
+    response_method="predict_proba",
+    multi_class="ovr",
+    average="weighted",
+)
 
 # Score function for probabilistic classification
-neg_log_loss_scorer = make_scorer(log_loss, greater_is_better=False,
-                                  needs_proba=True)
-neg_brier_score_scorer = make_scorer(brier_score_loss,
-                                     greater_is_better=False,
-                                     needs_proba=True)
-brier_score_loss_scorer = make_scorer(brier_score_loss,
-                                      greater_is_better=False,
-                                      needs_proba=True)
+neg_log_loss_scorer = make_scorer(
+    log_loss, greater_is_better=False, response_method="predict_proba"
+)
+neg_brier_score_scorer = make_scorer(
+    brier_score_loss, greater_is_better=False, response_method="predict_proba"
+)
+brier_score_loss_scorer = make_scorer(
+    brier_score_loss, greater_is_better=False, response_method="predict_proba"
+)
 
 
 # Clustering scores
@@ -699,45 +847,320 @@ def make_scorer(score_func, *, greater_is_better=True, needs_proba=False,
 fowlkes_mallows_scorer = make_scorer(fowlkes_mallows_score)
 
 
-SCORERS = dict(explained_variance=explained_variance_scorer,
-               r2=r2_scorer,
-               max_error=max_error_scorer,
-               neg_median_absolute_error=neg_median_absolute_error_scorer,
-               neg_mean_absolute_error=neg_mean_absolute_error_scorer,
-               neg_mean_absolute_percentage_error=neg_mean_absolute_percentage_error_scorer,  # noqa
-               neg_mean_squared_error=neg_mean_squared_error_scorer,
-               neg_mean_squared_log_error=neg_mean_squared_log_error_scorer,
-               neg_root_mean_squared_error=neg_root_mean_squared_error_scorer,
-               neg_mean_poisson_deviance=neg_mean_poisson_deviance_scorer,
-               neg_mean_gamma_deviance=neg_mean_gamma_deviance_scorer,
-               accuracy=accuracy_scorer,
-               top_k_accuracy=top_k_accuracy_scorer,
-               roc_auc=roc_auc_scorer,
-               roc_auc_ovr=roc_auc_ovr_scorer,
-               roc_auc_ovo=roc_auc_ovo_scorer,
-               roc_auc_ovr_weighted=roc_auc_ovr_weighted_scorer,
-               roc_auc_ovo_weighted=roc_auc_ovo_weighted_scorer,
-               balanced_accuracy=balanced_accuracy_scorer,
-               average_precision=average_precision_scorer,
-               neg_log_loss=neg_log_loss_scorer,
-               neg_brier_score=neg_brier_score_scorer,
-               # Cluster metrics that use supervised evaluation
-               adjusted_rand_score=adjusted_rand_scorer,
-               rand_score=rand_scorer,
-               homogeneity_score=homogeneity_scorer,
-               completeness_score=completeness_scorer,
-               v_measure_score=v_measure_scorer,
-               mutual_info_score=mutual_info_scorer,
-               adjusted_mutual_info_score=adjusted_mutual_info_scorer,
-               normalized_mutual_info_score=normalized_mutual_info_scorer,
-               fowlkes_mallows_score=fowlkes_mallows_scorer)
-
-
-for name, metric in [('precision', precision_score),
-                     ('recall', recall_score), ('f1', f1_score),
-                     ('jaccard', jaccard_score)]:
-    SCORERS[name] = make_scorer(metric, average='binary')
-    for average in ['macro', 'micro', 'samples', 'weighted']:
-        qualified_name = '{0}_{1}'.format(name, average)
-        SCORERS[qualified_name] = make_scorer(metric, pos_label=None,
-                                              average=average)
+_SCORERS = dict(
+    explained_variance=explained_variance_scorer,
+    r2=r2_scorer,
+    neg_max_error=neg_max_error_scorer,
+    matthews_corrcoef=matthews_corrcoef_scorer,
+    neg_median_absolute_error=neg_median_absolute_error_scorer,
+    neg_mean_absolute_error=neg_mean_absolute_error_scorer,
+    neg_mean_absolute_percentage_error=neg_mean_absolute_percentage_error_scorer,
+    neg_mean_squared_error=neg_mean_squared_error_scorer,
+    neg_mean_squared_log_error=neg_mean_squared_log_error_scorer,
+    neg_root_mean_squared_error=neg_root_mean_squared_error_scorer,
+    neg_root_mean_squared_log_error=neg_root_mean_squared_log_error_scorer,
+    neg_mean_poisson_deviance=neg_mean_poisson_deviance_scorer,
+    neg_mean_gamma_deviance=neg_mean_gamma_deviance_scorer,
+    d2_absolute_error_score=d2_absolute_error_scorer,
+    accuracy=accuracy_scorer,
+    top_k_accuracy=top_k_accuracy_scorer,
+    roc_auc=roc_auc_scorer,
+    roc_auc_ovr=roc_auc_ovr_scorer,
+    roc_auc_ovo=roc_auc_ovo_scorer,
+    roc_auc_ovr_weighted=roc_auc_ovr_weighted_scorer,
+    roc_auc_ovo_weighted=roc_auc_ovo_weighted_scorer,
+    balanced_accuracy=balanced_accuracy_scorer,
+    average_precision=average_precision_scorer,
+    neg_log_loss=neg_log_loss_scorer,
+    neg_brier_score=neg_brier_score_scorer,
+    positive_likelihood_ratio=positive_likelihood_ratio_scorer,
+    neg_negative_likelihood_ratio=neg_negative_likelihood_ratio_scorer,
+    # Cluster metrics that use supervised evaluation
+    adjusted_rand_score=adjusted_rand_scorer,
+    rand_score=rand_scorer,
+    homogeneity_score=homogeneity_scorer,
+    completeness_score=completeness_scorer,
+    v_measure_score=v_measure_scorer,
+    mutual_info_score=mutual_info_scorer,
+    adjusted_mutual_info_score=adjusted_mutual_info_scorer,
+    normalized_mutual_info_score=normalized_mutual_info_scorer,
+    fowlkes_mallows_score=fowlkes_mallows_scorer,
+)
+
+
+def get_scorer_names():
+    """Get the names of all available scorers.
+
+    These names can be passed to :func:`~sklearn.metrics.get_scorer` to
+    retrieve the scorer object.
+
+    Returns
+    -------
+    list of str
+        Names of all available scorers.
+
+    Examples
+    --------
+    >>> from sklearn.metrics import get_scorer_names
+    >>> all_scorers = get_scorer_names()
+    >>> type(all_scorers)
+    
+    >>> all_scorers[:3]
+    ['accuracy', 'adjusted_mutual_info_score', 'adjusted_rand_score']
+    >>> "roc_auc" in all_scorers
+    True
+    """
+    return sorted(_SCORERS.keys())
+
+
+for name, metric in [
+    ("precision", precision_score),
+    ("recall", recall_score),
+    ("f1", f1_score),
+    ("jaccard", jaccard_score),
+]:
+    _SCORERS[name] = make_scorer(metric, average="binary")
+    for average in ["macro", "micro", "samples", "weighted"]:
+        qualified_name = "{0}_{1}".format(name, average)
+        _SCORERS[qualified_name] = make_scorer(metric, pos_label=None, average=average)
+
+
+@validate_params(
+    {
+        "estimator": [HasMethods("fit"), None],
+        "scoring": [
+            StrOptions(set(get_scorer_names())),
+            callable,
+            list,
+            set,
+            tuple,
+            dict,
+            None,
+        ],
+        "allow_none": ["boolean"],
+        "raise_exc": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def check_scoring(estimator=None, scoring=None, *, allow_none=False, raise_exc=True):
+    """Determine scorer from user options.
+
+    A TypeError will be thrown if the estimator cannot be scored.
+
+    Parameters
+    ----------
+    estimator : estimator object implementing 'fit' or None, default=None
+        The object to use to fit the data. If `None`, then this function may error
+        depending on `allow_none`.
+
+    scoring : str, callable, list, tuple, set, or dict, default=None
+        Scorer to use. If `scoring` represents a single score, one can use:
+
+        - a single string (see :ref:`scoring_string_names`);
+        - a callable (see :ref:`scoring_callable`) that returns a single value;
+        - `None`, the `estimator`'s
+          :ref:`default evaluation criterion ` is used.
+
+        If `scoring` represents multiple scores, one can use:
+
+        - a list, tuple or set of unique strings;
+        - a callable returning a dictionary where the keys are the metric names and the
+          values are the metric scorers;
+        - a dictionary with metric names as keys and callables a values. The callables
+          need to have the signature `callable(estimator, X, y)`.
+
+    allow_none : bool, default=False
+        Whether to return None or raise an error if no `scoring` is specified and the
+        estimator has no `score` method.
+
+    raise_exc : bool, default=True
+        Whether to raise an exception (if a subset of the scorers in multimetric scoring
+        fails) or to return an error code.
+
+        - If set to `True`, raises the failing scorer's exception.
+        - If set to `False`, a formatted string of the exception details is passed as
+          result of the failing scorer(s).
+
+        This applies if `scoring` is list, tuple, set, or dict. Ignored if `scoring` is
+        a str or a callable.
+
+        .. versionadded:: 1.6
+
+    Returns
+    -------
+    scoring : callable
+        A scorer callable object / function with signature ``scorer(estimator, X, y)``.
+
+    Examples
+    --------
+    >>> from sklearn.datasets import load_iris
+    >>> from sklearn.metrics import check_scoring
+    >>> from sklearn.tree import DecisionTreeClassifier
+    >>> X, y = load_iris(return_X_y=True)
+    >>> classifier = DecisionTreeClassifier(max_depth=2).fit(X, y)
+    >>> scorer = check_scoring(classifier, scoring='accuracy')
+    >>> scorer(classifier, X, y)
+    0.96...
+
+    >>> from sklearn.metrics import make_scorer, accuracy_score, mean_squared_log_error
+    >>> X, y = load_iris(return_X_y=True)
+    >>> y *= -1
+    >>> clf = DecisionTreeClassifier().fit(X, y)
+    >>> scoring = {
+    ...     "accuracy": make_scorer(accuracy_score),
+    ...     "mean_squared_log_error": make_scorer(mean_squared_log_error),
+    ... }
+    >>> scoring_call = check_scoring(estimator=clf, scoring=scoring, raise_exc=False)
+    >>> scores = scoring_call(clf, X, y)
+    >>> scores
+    {'accuracy': 1.0, 'mean_squared_log_error': 'Traceback ...'}
+    """
+    if isinstance(scoring, str):
+        return get_scorer(scoring)
+    if callable(scoring):
+        # Heuristic to ensure user has not passed a metric
+        module = getattr(scoring, "__module__", None)
+        if (
+            hasattr(module, "startswith")
+            and module.startswith("sklearn.metrics.")
+            and not module.startswith("sklearn.metrics._scorer")
+            and not module.startswith("sklearn.metrics.tests.")
+        ):
+            raise ValueError(
+                "scoring value %r looks like it is a metric "
+                "function rather than a scorer. A scorer should "
+                "require an estimator as its first parameter. "
+                "Please use `make_scorer` to convert a metric "
+                "to a scorer." % scoring
+            )
+        return get_scorer(scoring)
+    if isinstance(scoring, (list, tuple, set, dict)):
+        scorers = _check_multimetric_scoring(estimator, scoring=scoring)
+        return _MultimetricScorer(scorers=scorers, raise_exc=raise_exc)
+    if scoring is None:
+        if hasattr(estimator, "score"):
+            return _PassthroughScorer(estimator)
+        elif allow_none:
+            return None
+        else:
+            raise TypeError(
+                "If no scoring is specified, the estimator passed should "
+                "have a 'score' method. The estimator %r does not." % estimator
+            )
+
+
+def _threshold_scores_to_class_labels(y_score, threshold, classes, pos_label):
+    """Threshold `y_score` and return the associated class labels."""
+    if pos_label is None:
+        map_thresholded_score_to_label = np.array([0, 1])
+    else:
+        pos_label_idx = np.flatnonzero(classes == pos_label)[0]
+        neg_label_idx = np.flatnonzero(classes != pos_label)[0]
+        map_thresholded_score_to_label = np.array([neg_label_idx, pos_label_idx])
+
+    return classes[map_thresholded_score_to_label[(y_score >= threshold).astype(int)]]
+
+
+class _CurveScorer(_BaseScorer):
+    """Scorer taking a continuous response and output a score for each threshold.
+
+    Parameters
+    ----------
+    score_func : callable
+        The score function to use. It will be called as
+        `score_func(y_true, y_pred, **kwargs)`.
+
+    sign : int
+        Either 1 or -1 to returns the score with `sign * score_func(estimator, X, y)`.
+        Thus, `sign` defined if higher scores are better or worse.
+
+    kwargs : dict
+        Additional parameters to pass to the score function.
+
+    thresholds : int or array-like
+        Related to the number of decision thresholds for which we want to compute the
+        score. If an integer, it will be used to generate `thresholds` thresholds
+        uniformly distributed between the minimum and maximum predicted scores. If an
+        array-like, it will be used as the thresholds.
+
+    response_method : str
+        The method to call on the estimator to get the response values.
+    """
+
+    def __init__(self, score_func, sign, kwargs, thresholds, response_method):
+        super().__init__(
+            score_func=score_func,
+            sign=sign,
+            kwargs=kwargs,
+            response_method=response_method,
+        )
+        self._thresholds = thresholds
+
+    @classmethod
+    def from_scorer(cls, scorer, response_method, thresholds):
+        """Create a continuous scorer from a normal scorer."""
+        instance = cls(
+            score_func=scorer._score_func,
+            sign=scorer._sign,
+            response_method=response_method,
+            thresholds=thresholds,
+            kwargs=scorer._kwargs,
+        )
+        # transfer the metadata request
+        instance._metadata_request = scorer._get_metadata_request()
+        return instance
+
+    def _score(self, method_caller, estimator, X, y_true, **kwargs):
+        """Evaluate predicted target values for X relative to y_true.
+
+        Parameters
+        ----------
+        method_caller : callable
+            Returns predictions given an estimator, method name, and other
+            arguments, potentially caching results.
+
+        estimator : object
+            Trained estimator to use for scoring.
+
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Test data that will be fed to estimator.predict.
+
+        y_true : array-like of shape (n_samples,)
+            Gold standard target values for X.
+
+        **kwargs : dict
+            Other parameters passed to the scorer. Refer to
+            :func:`set_score_request` for more details.
+
+        Returns
+        -------
+        scores : ndarray of shape (thresholds,)
+            The scores associated to each threshold.
+
+        potential_thresholds : ndarray of shape (thresholds,)
+            The potential thresholds used to compute the scores.
+        """
+        pos_label = self._get_pos_label()
+        y_score = method_caller(
+            estimator, self._response_method, X, pos_label=pos_label
+        )
+
+        scoring_kwargs = {**self._kwargs, **kwargs}
+        if isinstance(self._thresholds, Integral):
+            potential_thresholds = np.linspace(
+                np.min(y_score), np.max(y_score), self._thresholds
+            )
+        else:
+            potential_thresholds = np.asarray(self._thresholds)
+        score_thresholds = [
+            self._sign
+            * self._score_func(
+                y_true,
+                _threshold_scores_to_class_labels(
+                    y_score, th, estimator.classes_, pos_label
+                ),
+                **scoring_kwargs,
+            )
+            for th in potential_thresholds
+        ]
+        return np.array(score_thresholds), potential_thresholds
diff --git a/sklearn/metrics/cluster/__init__.py b/sklearn/metrics/cluster/__init__.py
index 9e116b40e31da..76020d80f8eb0 100644
--- a/sklearn/metrics/cluster/__init__.py
+++ b/sklearn/metrics/cluster/__init__.py
@@ -1,35 +1,55 @@
-"""
-The :mod:`sklearn.metrics.cluster` submodule contains evaluation metrics for
-cluster analysis results. There are two forms of evaluation:
+"""Evaluation metrics for cluster analysis results.
 
-- supervised, which uses a ground truth class values for each sample.
-- unsupervised, which does not and measures the 'quality' of the model itself.
+- Supervised evaluation uses a ground truth class values for each sample.
+- Unsupervised evaluation does not use ground truths and measures the "quality" of the
+  model itself.
 """
-from ._supervised import adjusted_mutual_info_score
-from ._supervised import normalized_mutual_info_score
-from ._supervised import adjusted_rand_score
-from ._supervised import rand_score
-from ._supervised import completeness_score
-from ._supervised import contingency_matrix
-from ._supervised import pair_confusion_matrix
-from ._supervised import expected_mutual_information
-from ._supervised import homogeneity_completeness_v_measure
-from ._supervised import homogeneity_score
-from ._supervised import mutual_info_score
-from ._supervised import v_measure_score
-from ._supervised import fowlkes_mallows_score
-from ._supervised import entropy
-from ._unsupervised import silhouette_samples
-from ._unsupervised import silhouette_score
-from ._unsupervised import calinski_harabasz_score
-from ._unsupervised import davies_bouldin_score
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
 from ._bicluster import consensus_score
+from ._supervised import (
+    adjusted_mutual_info_score,
+    adjusted_rand_score,
+    completeness_score,
+    contingency_matrix,
+    entropy,
+    expected_mutual_information,
+    fowlkes_mallows_score,
+    homogeneity_completeness_v_measure,
+    homogeneity_score,
+    mutual_info_score,
+    normalized_mutual_info_score,
+    pair_confusion_matrix,
+    rand_score,
+    v_measure_score,
+)
+from ._unsupervised import (
+    calinski_harabasz_score,
+    davies_bouldin_score,
+    silhouette_samples,
+    silhouette_score,
+)
 
-__all__ = ["adjusted_mutual_info_score", "normalized_mutual_info_score",
-           "adjusted_rand_score", "rand_score", "completeness_score",
-           "pair_confusion_matrix", "contingency_matrix",
-           "expected_mutual_information", "homogeneity_completeness_v_measure",
-           "homogeneity_score", "mutual_info_score", "v_measure_score",
-           "fowlkes_mallows_score", "entropy", "silhouette_samples",
-           "silhouette_score", "calinski_harabasz_score",
-           "davies_bouldin_score", "consensus_score"]
+__all__ = [
+    "adjusted_mutual_info_score",
+    "adjusted_rand_score",
+    "calinski_harabasz_score",
+    "completeness_score",
+    "consensus_score",
+    "contingency_matrix",
+    "davies_bouldin_score",
+    "entropy",
+    "expected_mutual_information",
+    "fowlkes_mallows_score",
+    "homogeneity_completeness_v_measure",
+    "homogeneity_score",
+    "mutual_info_score",
+    "normalized_mutual_info_score",
+    "pair_confusion_matrix",
+    "rand_score",
+    "silhouette_samples",
+    "silhouette_score",
+    "v_measure_score",
+]
diff --git a/sklearn/metrics/cluster/_bicluster.py b/sklearn/metrics/cluster/_bicluster.py
index e267b44cee229..bb306c025b694 100644
--- a/sklearn/metrics/cluster/_bicluster.py
+++ b/sklearn/metrics/cluster/_bicluster.py
@@ -1,8 +1,11 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
 import numpy as np
 from scipy.optimize import linear_sum_assignment
 
-from ...utils.validation import check_consistent_length, check_array
-from ...utils.validation import _deprecate_positional_args
+from ...utils._param_validation import StrOptions, validate_params
+from ...utils.validation import check_array, check_consistent_length
 
 __all__ = ["consensus_score"]
 
@@ -19,8 +22,7 @@ def _check_rows_and_columns(a, b):
 
 def _jaccard(a_rows, a_cols, b_rows, b_cols):
     """Jaccard coefficient on the elements of the two biclusters."""
-    intersection = ((a_rows * b_rows).sum() *
-                    (a_cols * b_cols).sum())
+    intersection = (a_rows * b_rows).sum() * (a_cols * b_cols).sum()
 
     a_size = a_rows.sum() * a_cols.sum()
     b_size = b_rows.sum() * b_cols.sum()
@@ -38,19 +40,29 @@ def _pairwise_similarity(a, b, similarity):
     a_rows, a_cols, b_rows, b_cols = _check_rows_and_columns(a, b)
     n_a = a_rows.shape[0]
     n_b = b_rows.shape[0]
-    result = np.array(list(list(similarity(a_rows[i], a_cols[i],
-                                           b_rows[j], b_cols[j])
-                                for j in range(n_b))
-                           for i in range(n_a)))
+    result = np.array(
+        [
+            [similarity(a_rows[i], a_cols[i], b_rows[j], b_cols[j]) for j in range(n_b)]
+            for i in range(n_a)
+        ]
+    )
     return result
 
 
-@_deprecate_positional_args
+@validate_params(
+    {
+        "a": [tuple],
+        "b": [tuple],
+        "similarity": [callable, StrOptions({"jaccard"})],
+    },
+    prefer_skip_nested_validation=True,
+)
 def consensus_score(a, b, *, similarity="jaccard"):
     """The similarity of two sets of biclusters.
 
-    Similarity between individual biclusters is computed. Then the
-    best matching between sets is found using the Hungarian algorithm.
+    Similarity between individual biclusters is computed. Then the best
+    matching between sets is found by solving a linear sum assignment problem,
+    using a modified Jonker-Volgenant algorithm.
     The final score is the sum of similarities divided by the size of
     the larger set.
 
@@ -58,10 +70,10 @@ def consensus_score(a, b, *, similarity="jaccard"):
 
     Parameters
     ----------
-    a : (rows, columns)
+    a : tuple (rows, columns)
         Tuple of row and column indicators for a set of biclusters.
 
-    b : (rows, columns)
+    b : tuple (rows, columns)
         Another set of biclusters like ``a``.
 
     similarity : 'jaccard' or callable, default='jaccard'
@@ -69,18 +81,34 @@ def consensus_score(a, b, *, similarity="jaccard"):
         any function that takes four arguments, each of which is a 1d
         indicator vector: (a_rows, a_columns, b_rows, b_columns).
 
+    Returns
+    -------
+    consensus_score : float
+       Consensus score, a non-negative value, sum of similarities
+       divided by size of larger set.
+
+    See Also
+    --------
+    scipy.optimize.linear_sum_assignment : Solve the linear sum assignment problem.
+
     References
     ----------
-
     * Hochreiter, Bodenhofer, et. al., 2010. `FABIA: factor analysis
       for bicluster acquisition
       `__.
 
+    Examples
+    --------
+    >>> from sklearn.metrics import consensus_score
+    >>> a = ([[True, False], [False, True]], [[False, True], [True, False]])
+    >>> b = ([[False, True], [True, False]], [[True, False], [False, True]])
+    >>> consensus_score(a, b, similarity='jaccard')
+    1.0
     """
     if similarity == "jaccard":
         similarity = _jaccard
     matrix = _pairwise_similarity(a, b, similarity)
-    row_indices, col_indices = linear_sum_assignment(1. - matrix)
+    row_indices, col_indices = linear_sum_assignment(1.0 - matrix)
     n_a = len(a[0])
     n_b = len(b[0])
-    return matrix[row_indices, col_indices].sum() / max(n_a, n_b)
+    return float(matrix[row_indices, col_indices].sum() / max(n_a, n_b))
diff --git a/sklearn/metrics/cluster/_expected_mutual_info_fast.pyx b/sklearn/metrics/cluster/_expected_mutual_info_fast.pyx
index d2f9cd8578b12..3d51def36c255 100644
--- a/sklearn/metrics/cluster/_expected_mutual_info_fast.pyx
+++ b/sklearn/metrics/cluster/_expected_mutual_info_fast.pyx
@@ -1,70 +1,69 @@
-# cython: cdivision=True
-# cython: boundscheck=False
-# cython: wraparound=False
-#
-# Authors: Robert Layton 
-#           Corey Lynch 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 from libc.math cimport exp, lgamma
-from scipy.special import gammaln
-import numpy as np
-cimport numpy as np
-cimport cython
 
-np.import_array()
-ctypedef np.float64_t DOUBLE
+from ...utils._typedefs cimport float64_t, int64_t
+
+import numpy as np
+from scipy.special import gammaln
 
 
-def expected_mutual_information(contingency, int n_samples):
+def expected_mutual_information(contingency, int64_t n_samples):
     """Calculate the expected mutual information for two labelings."""
-    cdef int R, C
-    cdef DOUBLE N, gln_N, emi, term2, term3, gln
-    cdef np.ndarray[DOUBLE] gln_a, gln_b, gln_Na, gln_Nb, gln_nij, log_Nnij
-    cdef np.ndarray[DOUBLE] nijs, term1
-    cdef np.ndarray[DOUBLE] log_a, log_b
-    cdef np.ndarray[np.int32_t] a, b
-    #cdef np.ndarray[int, ndim=2] start, end
-    R, C = contingency.shape
-    N = n_samples
-    a = np.ravel(contingency.sum(axis=1).astype(np.int32, copy=False))
-    b = np.ravel(contingency.sum(axis=0).astype(np.int32, copy=False))
+    cdef:
+        float64_t emi = 0
+        int64_t n_rows, n_cols
+        float64_t term2, term3, gln
+        int64_t[::1] a_view, b_view
+        float64_t[::1] term1
+        float64_t[::1] gln_a, gln_b, gln_Na, gln_Nb, gln_Nnij, log_Nnij
+        float64_t[::1] log_a, log_b
+        Py_ssize_t i, j, nij
+        int64_t start, end
+
+    n_rows, n_cols = contingency.shape
+    a = np.ravel(contingency.sum(axis=1).astype(np.int64, copy=False))
+    b = np.ravel(contingency.sum(axis=0).astype(np.int64, copy=False))
+    a_view = a
+    b_view = b
+
+    # any labelling with zero entropy implies EMI = 0
+    if a.size == 1 or b.size == 1:
+        return 0.0
+
     # There are three major terms to the EMI equation, which are multiplied to
     # and then summed over varying nij values.
     # While nijs[0] will never be used, having it simplifies the indexing.
     nijs = np.arange(0, max(np.max(a), np.max(b)) + 1, dtype='float')
     nijs[0] = 1  # Stops divide by zero warnings. As its not used, no issue.
     # term1 is nij / N
-    term1 = nijs / N
+    term1 = nijs / n_samples
     # term2 is log((N*nij) / (a * b)) == log(N * nij) - log(a * b)
     log_a = np.log(a)
     log_b = np.log(b)
     # term2 uses log(N * nij) = log(N) + log(nij)
-    log_Nnij = np.log(N) + np.log(nijs)
+    log_Nnij = np.log(n_samples) + np.log(nijs)
     # term3 is large, and involved many factorials. Calculate these in log
     # space to stop overflows.
     gln_a = gammaln(a + 1)
     gln_b = gammaln(b + 1)
-    gln_Na = gammaln(N - a + 1)
-    gln_Nb = gammaln(N - b + 1)
-    gln_N = gammaln(N + 1)
-    gln_nij = gammaln(nijs + 1)
-    # start and end values for nij terms for each summation.
-    start = np.array([[v - N + w for w in b] for v in a], dtype='int')
-    start = np.maximum(start, 1)
-    end = np.minimum(np.resize(a, (C, R)).T, np.resize(b, (R, C))) + 1
+    gln_Na = gammaln(n_samples - a + 1)
+    gln_Nb = gammaln(n_samples - b + 1)
+    gln_Nnij = gammaln(nijs + 1) + gammaln(n_samples + 1)
+
     # emi itself is a summation over the various values.
-    emi = 0.0
-    cdef Py_ssize_t i, j, nij
-    for i in range(R):
-        for j in range(C):
-            for nij in range(start[i,j], end[i,j]):
+    for i in range(n_rows):
+        for j in range(n_cols):
+            start = max(1, a_view[i] - n_samples + b_view[j])
+            end = min(a_view[i], b_view[j]) + 1
+            for nij in range(start, end):
                 term2 = log_Nnij[nij] - log_a[i] - log_b[j]
                 # Numerators are positive, denominators are negative.
                 gln = (gln_a[i] + gln_b[j] + gln_Na[i] + gln_Nb[j]
-                     - gln_N - gln_nij[nij] - lgamma(a[i] - nij + 1)
-                     - lgamma(b[j] - nij + 1)
-                     - lgamma(N - a[i] - b[j] + nij + 1))
+                       - gln_Nnij[nij] - lgamma(a_view[i] - nij + 1)
+                       - lgamma(b_view[j] - nij + 1)
+                       - lgamma(n_samples - a_view[i] - b_view[j] + nij + 1))
                 term3 = exp(gln)
                 emi += (term1[nij] * term2 * term3)
     return emi
diff --git a/sklearn/metrics/cluster/_supervised.py b/sklearn/metrics/cluster/_supervised.py
index 19d1552518db4..ccc11d752adba 100644
--- a/sklearn/metrics/cluster/_supervised.py
+++ b/sklearn/metrics/cluster/_supervised.py
@@ -4,29 +4,21 @@
 better.
 """
 
-# Authors: Olivier Grisel 
-#          Wei LI 
-#          Diego Molla 
-#          Arnaud Fouchet 
-#          Thierry Guillemot 
-#          Gregory Stupp 
-#          Joel Nothman 
-#          Arya McCarthy 
-#          Uwe F Mayer 
-# License: BSD 3 clause
-
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import warnings
 from math import log
+from numbers import Real
 
 import numpy as np
 from scipy import sparse as sp
 
-from ._expected_mutual_info_fast import expected_mutual_information
-from ...utils.fixes import _astype_copy_false
+from ...utils._array_api import _max_precision_float_dtype, get_namespace_and_device
+from ...utils._param_validation import Hidden, Interval, StrOptions, validate_params
 from ...utils.multiclass import type_of_target
-from ...utils.validation import _deprecate_positional_args
 from ...utils.validation import check_array, check_consistent_length
+from ._expected_mutual_info_fast import expected_mutual_information
 
 
 def check_clusterings(labels_true, labels_pred):
@@ -41,29 +33,35 @@ def check_clusterings(labels_true, labels_pred):
         The predicted labels.
     """
     labels_true = check_array(
-        labels_true, ensure_2d=False, ensure_min_samples=0, dtype=None,
+        labels_true,
+        ensure_2d=False,
+        ensure_min_samples=0,
+        dtype=None,
     )
 
     labels_pred = check_array(
-        labels_pred, ensure_2d=False, ensure_min_samples=0, dtype=None,
+        labels_pred,
+        ensure_2d=False,
+        ensure_min_samples=0,
+        dtype=None,
     )
 
     type_label = type_of_target(labels_true)
     type_pred = type_of_target(labels_pred)
 
-    if 'continuous' in (type_pred, type_label):
-        msg = f'Clustering metrics expects discrete values but received' \
-              f' {type_label} values for label, and {type_pred} values ' \
-              f'for target'
+    if "continuous" in (type_pred, type_label):
+        msg = (
+            "Clustering metrics expects discrete values but received"
+            f" {type_label} values for label, and {type_pred} values "
+            "for target"
+        )
         warnings.warn(msg, UserWarning)
 
     # input checks
     if labels_true.ndim != 1:
-        raise ValueError(
-            "labels_true must be 1D: shape is %r" % (labels_true.shape,))
+        raise ValueError("labels_true must be 1D: shape is %r" % (labels_true.shape,))
     if labels_pred.ndim != 1:
-        raise ValueError(
-            "labels_pred must be 1D: shape is %r" % (labels_pred.shape,))
+        raise ValueError("labels_pred must be 1D: shape is %r" % (labels_pred.shape,))
     check_consistent_length(labels_true, labels_pred)
 
     return labels_true, labels_pred
@@ -80,18 +78,31 @@ def _generalized_average(U, V, average_method):
     elif average_method == "max":
         return max(U, V)
     else:
-        raise ValueError("'average_method' must be 'min', 'geometric', "
-                         "'arithmetic', or 'max'")
-
-
-@_deprecate_positional_args
-def contingency_matrix(labels_true, labels_pred, *, eps=None, sparse=False,
-                       dtype=np.int64):
+        raise ValueError(
+            "'average_method' must be 'min', 'geometric', 'arithmetic', or 'max'"
+        )
+
+
+@validate_params(
+    {
+        "labels_true": ["array-like", None],
+        "labels_pred": ["array-like", None],
+        "eps": [Interval(Real, 0, None, closed="left"), None],
+        "sparse": ["boolean"],
+        "dtype": "no_validation",  # delegate the validation to SciPy
+    },
+    prefer_skip_nested_validation=True,
+)
+def contingency_matrix(
+    labels_true, labels_pred, *, eps=None, sparse=False, dtype=np.int64
+):
     """Build a contingency matrix describing the relationship between labels.
 
+    Read more in the :ref:`User Guide `.
+
     Parameters
     ----------
-    labels_true : int array, shape = [n_samples]
+    labels_true : array-like of shape (n_samples,)
         Ground truth class labels to be used as a reference.
 
     labels_pred : array-like of shape (n_samples,)
@@ -103,7 +114,7 @@ def contingency_matrix(labels_true, labels_pred, *, eps=None, sparse=False,
         If ``None``, nothing is adjusted.
 
     sparse : bool, default=False
-        If `True`, return a sparse CSR continency matrix. If `eps` is not
+        If `True`, return a sparse CSR contingency matrix. If `eps` is not
         `None` and `sparse` is `True` will raise ValueError.
 
         .. versionadded:: 0.18
@@ -122,6 +133,16 @@ def contingency_matrix(labels_true, labels_pred, *, eps=None, sparse=False,
         otherwise with the ``dtype`` argument. If ``eps`` is given, the dtype
         will be float.
         Will be a ``sklearn.sparse.csr_matrix`` if ``sparse=True``.
+
+    Examples
+    --------
+    >>> from sklearn.metrics.cluster import contingency_matrix
+    >>> labels_true = [0, 0, 1, 1, 2, 2]
+    >>> labels_pred = [1, 0, 2, 1, 0, 2]
+    >>> contingency_matrix(labels_true, labels_pred)
+    array([[1, 1, 0],
+           [0, 1, 1],
+           [1, 0, 1]])
     """
 
     if eps is not None and sparse:
@@ -134,10 +155,11 @@ def contingency_matrix(labels_true, labels_pred, *, eps=None, sparse=False,
     # Using coo_matrix to accelerate simple histogram calculation,
     # i.e. bins are consecutive integers
     # Currently, coo_matrix is faster than histogram2d for simple cases
-    contingency = sp.coo_matrix((np.ones(class_idx.shape[0]),
-                                 (class_idx, cluster_idx)),
-                                shape=(n_classes, n_clusters),
-                                dtype=dtype)
+    contingency = sp.coo_matrix(
+        (np.ones(class_idx.shape[0]), (class_idx, cluster_idx)),
+        shape=(n_classes, n_clusters),
+        dtype=dtype,
+    )
     if sparse:
         contingency = contingency.tocsr()
         contingency.sum_duplicates()
@@ -151,13 +173,21 @@ def contingency_matrix(labels_true, labels_pred, *, eps=None, sparse=False,
 
 # clustering measures
 
+
+@validate_params(
+    {
+        "labels_true": ["array-like"],
+        "labels_pred": ["array-like"],
+    },
+    prefer_skip_nested_validation=True,
+)
 def pair_confusion_matrix(labels_true, labels_pred):
     """Pair confusion matrix arising from two clusterings.
 
     The pair confusion matrix :math:`C` computes a 2 by 2 similarity matrix
     between two clusterings by considering all pairs of samples and counting
     pairs that are assigned into the same or into different clusters under
-    the true and predicted clusterings.
+    the true and predicted clusterings [1]_.
 
     Considering a pair of samples that is clustered together a positive pair,
     then as in binary classification the count of true negatives is
@@ -181,9 +211,15 @@ def pair_confusion_matrix(labels_true, labels_pred):
 
     See Also
     --------
-    rand_score: Rand Score
-    adjusted_rand_score: Adjusted Rand Score
-    adjusted_mutual_info_score: Adjusted Mutual Information
+    sklearn.metrics.rand_score : Rand Score.
+    sklearn.metrics.adjusted_rand_score : Adjusted Rand Score.
+    sklearn.metrics.adjusted_mutual_info_score : Adjusted Mutual Information.
+
+    References
+    ----------
+    .. [1] :doi:`Hubert, L., Arabie, P. "Comparing partitions."
+           Journal of Classification 2, 193–218 (1985).
+           <10.1007/BF01908075>`
 
     Examples
     --------
@@ -204,12 +240,6 @@ def pair_confusion_matrix(labels_true, labels_pred):
              [0, 2]]...
 
     Note that the matrix is not symmetric.
-
-    References
-    ----------
-    .. L. Hubert and P. Arabie, Comparing Partitions, Journal of
-      Classification 1985
-      https://link.springer.com/article/10.1007%2FBF01908075
     """
     labels_true, labels_pred = check_clusterings(labels_true, labels_pred)
     n_samples = np.int64(labels_true.shape[0])
@@ -220,24 +250,33 @@ def pair_confusion_matrix(labels_true, labels_pred):
     )
     n_c = np.ravel(contingency.sum(axis=1))
     n_k = np.ravel(contingency.sum(axis=0))
-    sum_squares = (contingency.data ** 2).sum()
+    sum_squares = (contingency.data**2).sum()
     C = np.empty((2, 2), dtype=np.int64)
     C[1, 1] = sum_squares - n_samples
     C[0, 1] = contingency.dot(n_k).sum() - sum_squares
     C[1, 0] = contingency.transpose().dot(n_c).sum() - sum_squares
-    C[0, 0] = n_samples ** 2 - C[0, 1] - C[1, 0] - sum_squares
+    C[0, 0] = n_samples**2 - C[0, 1] - C[1, 0] - sum_squares
     return C
 
 
+@validate_params(
+    {
+        "labels_true": ["array-like"],
+        "labels_pred": ["array-like"],
+    },
+    prefer_skip_nested_validation=True,
+)
 def rand_score(labels_true, labels_pred):
     """Rand index.
 
     The Rand Index computes a similarity measure between two clusterings
     by considering all pairs of samples and counting pairs that are
     assigned in the same or different clusters in the predicted and
-    true clusterings.
+    true clusterings [1]_ [2]_.
 
-    The raw RI score is:
+    The raw RI score [3]_ is:
+
+    .. code-block:: text
 
         RI = (number of agreeing pairs) / (number of pairs)
 
@@ -259,8 +298,19 @@ def rand_score(labels_true, labels_pred):
 
     See Also
     --------
-    adjusted_rand_score: Adjusted Rand Score
-    adjusted_mutual_info_score: Adjusted Mutual Information
+    adjusted_rand_score: Adjusted Rand Score.
+    adjusted_mutual_info_score: Adjusted Mutual Information.
+
+    References
+    ----------
+    .. [1] :doi:`Hubert, L., Arabie, P. "Comparing partitions."
+       Journal of Classification 2, 193–218 (1985).
+       <10.1007/BF01908075>`.
+
+    .. [2] `Wikipedia: Simple Matching Coefficient
+        `_
+
+    .. [3] `Wikipedia: Rand Index `_
 
     Examples
     --------
@@ -274,17 +324,7 @@ def rand_score(labels_true, labels_pred):
     are complete but may not always be pure, hence penalized:
 
       >>> rand_score([0, 0, 1, 2], [0, 0, 1, 1])
-      0.83...
-
-    References
-    ----------
-    .. L. Hubert and P. Arabie, Comparing Partitions, Journal of
-      Classification 1985
-      https://link.springer.com/article/10.1007%2FBF01908075
-
-    .. https://en.wikipedia.org/wiki/Simple_matching_coefficient
-
-    .. https://en.wikipedia.org/wiki/Rand_index
+      0.83
     """
     contingency = pair_confusion_matrix(labels_true, labels_pred)
     numerator = contingency.diagonal().sum()
@@ -296,9 +336,16 @@ def rand_score(labels_true, labels_pred):
         # cluster. These are perfect matches hence return 1.0.
         return 1.0
 
-    return numerator / denominator
+    return float(numerator / denominator)
 
 
+@validate_params(
+    {
+        "labels_true": ["array-like"],
+        "labels_pred": ["array-like"],
+    },
+    prefer_skip_nested_validation=True,
+)
 def adjusted_rand_score(labels_true, labels_pred):
     """Rand index adjusted for chance.
 
@@ -315,7 +362,8 @@ def adjusted_rand_score(labels_true, labels_pred):
     The adjusted Rand index is thus ensured to have a value close to
     0.0 for random labeling independently of the number of clusters and
     samples and exactly 1.0 when the clusterings are identical (up to
-    a permutation).
+    a permutation). The adjusted Rand index is bounded below by -0.5 for
+    especially discordant clusterings.
 
     ARI is a symmetric measure::
 
@@ -325,18 +373,36 @@ def adjusted_rand_score(labels_true, labels_pred):
 
     Parameters
     ----------
-    labels_true : int array, shape = [n_samples]
-        Ground truth class labels to be used as a reference
+    labels_true : array-like of shape (n_samples,), dtype=int
+        Ground truth class labels to be used as a reference.
 
-    labels_pred : array-like of shape (n_samples,)
-        Cluster labels to evaluate
+    labels_pred : array-like of shape (n_samples,), dtype=int
+        Cluster labels to evaluate.
 
     Returns
     -------
     ARI : float
-       Similarity score between -1.0 and 1.0. Random labelings have an ARI
+       Similarity score between -0.5 and 1.0. Random labelings have an ARI
        close to 0.0. 1.0 stands for perfect match.
 
+    See Also
+    --------
+    adjusted_mutual_info_score : Adjusted Mutual Information.
+
+    References
+    ----------
+    .. [Hubert1985] L. Hubert and P. Arabie, Comparing Partitions,
+      Journal of Classification 1985
+      https://link.springer.com/article/10.1007%2FBF01908075
+
+    .. [Steinley2004] D. Steinley, Properties of the Hubert-Arabie
+      adjusted Rand index, Psychological Methods 2004
+
+    .. [wk] https://en.wikipedia.org/wiki/Rand_index#Adjusted_Rand_index
+
+    .. [Chacon] :doi:`Minimum adjusted Rand index for two clusterings of a given size,
+      2022, J. E. ChacÃŗn and A. I. Rastrojo <10.1007/s11634-022-00491-w>`
+
     Examples
     --------
     Perfectly matching labelings have a score of 1 even
@@ -351,13 +417,13 @@ def adjusted_rand_score(labels_true, labels_pred):
     are complete but may not always be pure, hence penalized::
 
       >>> adjusted_rand_score([0, 0, 1, 2], [0, 0, 1, 1])
-      0.57...
+      0.57
 
     ARI is symmetric, so labelings that have pure clusters with members
     coming from the same classes but unnecessary splits are penalized::
 
       >>> adjusted_rand_score([0, 0, 1, 1], [0, 0, 1, 2])
-      0.57...
+      0.57
 
     If classes members are completely split across different clusters, the
     assignment is totally incomplete, hence the ARI is very low::
@@ -365,32 +431,34 @@ def adjusted_rand_score(labels_true, labels_pred):
       >>> adjusted_rand_score([0, 0, 0, 0], [0, 1, 2, 3])
       0.0
 
-    References
-    ----------
-    .. [Hubert1985] L. Hubert and P. Arabie, Comparing Partitions,
-      Journal of Classification 1985
-      https://link.springer.com/article/10.1007%2FBF01908075
-
-    .. [Steinley2004] D. Steinley, Properties of the Hubert-Arabie
-      adjusted Rand index, Psychological Methods 2004
+    ARI may take a negative value for especially discordant labelings that
+    are a worse choice than the expected value of random labels::
 
-    .. [wk] https://en.wikipedia.org/wiki/Rand_index#Adjusted_Rand_index
+      >>> adjusted_rand_score([0, 0, 1, 1], [0, 1, 0, 1])
+      -0.5
 
-    See Also
-    --------
-    adjusted_mutual_info_score : Adjusted Mutual Information.
+    See :ref:`sphx_glr_auto_examples_cluster_plot_adjusted_for_chance_measures.py`
+    for a more detailed example.
     """
     (tn, fp), (fn, tp) = pair_confusion_matrix(labels_true, labels_pred)
+    # convert to Python integer types, to avoid overflow or underflow
+    tn, fp, fn, tp = int(tn), int(fp), int(fn), int(tp)
 
     # Special cases: empty data or full agreement
     if fn == 0 and fp == 0:
         return 1.0
 
-    return 2. * (tp * tn - fn * fp) / ((tp + fn) * (fn + tn) +
-                                       (tp + fp) * (fp + tn))
+    return 2.0 * (tp * tn - fn * fp) / ((tp + fn) * (fn + tn) + (tp + fp) * (fp + tn))
 
 
-@_deprecate_positional_args
+@validate_params(
+    {
+        "labels_true": ["array-like"],
+        "labels_pred": ["array-like"],
+        "beta": [Interval(Real, 0, None, closed="left")],
+    },
+    prefer_skip_nested_validation=True,
+)
 def homogeneity_completeness_v_measure(labels_true, labels_pred, *, beta=1.0):
     """Compute the homogeneity and completeness and V-Measure scores at once.
 
@@ -421,11 +489,11 @@ def homogeneity_completeness_v_measure(labels_true, labels_pred, *, beta=1.0):
 
     Parameters
     ----------
-    labels_true : int array, shape = [n_samples]
-        ground truth class labels to be used as a reference
+    labels_true : array-like of shape (n_samples,)
+        Ground truth class labels to be used as a reference.
 
     labels_pred : array-like of shape (n_samples,)
-        cluster labels to evaluate
+        Cluster labels to evaluate.
 
     beta : float, default=1.0
         Ratio of weight attributed to ``homogeneity`` vs ``completeness``.
@@ -436,19 +504,26 @@ def homogeneity_completeness_v_measure(labels_true, labels_pred, *, beta=1.0):
     Returns
     -------
     homogeneity : float
-       score between 0.0 and 1.0. 1.0 stands for perfectly homogeneous labeling
+        Score between 0.0 and 1.0. 1.0 stands for perfectly homogeneous labeling.
 
     completeness : float
-       score between 0.0 and 1.0. 1.0 stands for perfectly complete labeling
+        Score between 0.0 and 1.0. 1.0 stands for perfectly complete labeling.
 
     v_measure : float
-        harmonic mean of the first two
+        Harmonic mean of the first two.
 
     See Also
     --------
-    homogeneity_score
-    completeness_score
-    v_measure_score
+    homogeneity_score : Homogeneity metric of cluster labeling.
+    completeness_score : Completeness metric of cluster labeling.
+    v_measure_score : V-Measure (NMI with arithmetic mean option).
+
+    Examples
+    --------
+    >>> from sklearn.metrics import homogeneity_completeness_v_measure
+    >>> y_true, y_pred = [0, 0, 1, 1, 2, 2], [0, 0, 1, 2, 2, 2]
+    >>> homogeneity_completeness_v_measure(y_true, y_pred)
+    (0.71, 0.771, 0.74)
     """
     labels_true, labels_pred = check_clusterings(labels_true, labels_pred)
 
@@ -467,12 +542,23 @@ def homogeneity_completeness_v_measure(labels_true, labels_pred, *, beta=1.0):
     if homogeneity + completeness == 0.0:
         v_measure_score = 0.0
     else:
-        v_measure_score = ((1 + beta) * homogeneity * completeness
-                           / (beta * homogeneity + completeness))
-
-    return homogeneity, completeness, v_measure_score
-
-
+        v_measure_score = (
+            (1 + beta)
+            * homogeneity
+            * completeness
+            / (beta * homogeneity + completeness)
+        )
+
+    return float(homogeneity), float(completeness), float(v_measure_score)
+
+
+@validate_params(
+    {
+        "labels_true": ["array-like"],
+        "labels_pred": ["array-like"],
+    },
+    prefer_skip_nested_validation=True,
+)
 def homogeneity_score(labels_true, labels_pred):
     """Homogeneity metric of a cluster labeling given a ground truth.
 
@@ -491,16 +577,21 @@ def homogeneity_score(labels_true, labels_pred):
 
     Parameters
     ----------
-    labels_true : int array, shape = [n_samples]
-        ground truth class labels to be used as a reference
+    labels_true : array-like of shape (n_samples,)
+        Ground truth class labels to be used as a reference.
 
     labels_pred : array-like of shape (n_samples,)
-        cluster labels to evaluate
+        Cluster labels to evaluate.
 
     Returns
     -------
     homogeneity : float
-       score between 0.0 and 1.0. 1.0 stands for perfectly homogeneous labeling
+       Score between 0.0 and 1.0. 1.0 stands for perfectly homogeneous labeling.
+
+    See Also
+    --------
+    completeness_score : Completeness metric of cluster labeling.
+    v_measure_score : V-Measure (NMI with arithmetic mean option).
 
     References
     ----------
@@ -509,11 +600,6 @@ def homogeneity_score(labels_true, labels_pred):
        conditional entropy-based external cluster evaluation measure
        `_
 
-    See Also
-    --------
-    completeness_score
-    v_measure_score
-
     Examples
     --------
 
@@ -542,8 +628,15 @@ def homogeneity_score(labels_true, labels_pred):
     return homogeneity_completeness_v_measure(labels_true, labels_pred)[0]
 
 
+@validate_params(
+    {
+        "labels_true": ["array-like"],
+        "labels_pred": ["array-like"],
+    },
+    prefer_skip_nested_validation=True,
+)
 def completeness_score(labels_true, labels_pred):
-    """Completeness metric of a cluster labeling given a ground truth.
+    """Compute completeness metric of a cluster labeling given a ground truth.
 
     A clustering result satisfies completeness if all the data points
     that are members of a given class are elements of the same cluster.
@@ -560,16 +653,21 @@ def completeness_score(labels_true, labels_pred):
 
     Parameters
     ----------
-    labels_true : int array, shape = [n_samples]
-        ground truth class labels to be used as a reference
+    labels_true : array-like of shape (n_samples,)
+        Ground truth class labels to be used as a reference.
 
     labels_pred : array-like of shape (n_samples,)
-        cluster labels to evaluate
+        Cluster labels to evaluate.
 
     Returns
     -------
     completeness : float
-       score between 0.0 and 1.0. 1.0 stands for perfectly complete labeling
+       Score between 0.0 and 1.0. 1.0 stands for perfectly complete labeling.
+
+    See Also
+    --------
+    homogeneity_score : Homogeneity metric of cluster labeling.
+    v_measure_score : V-Measure (NMI with arithmetic mean option).
 
     References
     ----------
@@ -578,11 +676,6 @@ def completeness_score(labels_true, labels_pred):
        conditional entropy-based external cluster evaluation measure
        `_
 
-    See Also
-    --------
-    homogeneity_score
-    v_measure_score
-
     Examples
     --------
 
@@ -598,7 +691,7 @@ def completeness_score(labels_true, labels_pred):
       >>> print(completeness_score([0, 0, 1, 1], [0, 0, 0, 0]))
       1.0
       >>> print(completeness_score([0, 1, 2, 3], [0, 0, 1, 1]))
-      0.999...
+      0.999
 
     If classes members are split across different clusters, the
     assignment cannot be complete::
@@ -611,7 +704,14 @@ def completeness_score(labels_true, labels_pred):
     return homogeneity_completeness_v_measure(labels_true, labels_pred)[1]
 
 
-@_deprecate_positional_args
+@validate_params(
+    {
+        "labels_true": ["array-like"],
+        "labels_pred": ["array-like"],
+        "beta": [Interval(Real, 0, None, closed="left")],
+    },
+    prefer_skip_nested_validation=True,
+)
 def v_measure_score(labels_true, labels_pred, *, beta=1.0):
     """V-measure cluster labeling given a ground truth.
 
@@ -632,16 +732,15 @@ def v_measure_score(labels_true, labels_pred, *, beta=1.0):
     measure the agreement of two independent label assignments strategies
     on the same dataset when the real ground truth is not known.
 
-
     Read more in the :ref:`User Guide `.
 
     Parameters
     ----------
-    labels_true : int array, shape = [n_samples]
-        ground truth class labels to be used as a reference
+    labels_true : array-like of shape (n_samples,)
+        Ground truth class labels to be used as a reference.
 
     labels_pred : array-like of shape (n_samples,)
-        cluster labels to evaluate
+        Cluster labels to evaluate.
 
     beta : float, default=1.0
         Ratio of weight attributed to ``homogeneity`` vs ``completeness``.
@@ -652,7 +751,13 @@ def v_measure_score(labels_true, labels_pred, *, beta=1.0):
     Returns
     -------
     v_measure : float
-       score between 0.0 and 1.0. 1.0 stands for perfectly complete labeling
+       Score between 0.0 and 1.0. 1.0 stands for perfectly complete labeling.
+
+    See Also
+    --------
+    homogeneity_score : Homogeneity metric of cluster labeling.
+    completeness_score : Completeness metric of cluster labeling.
+    normalized_mutual_info_score : Normalized Mutual Information.
 
     References
     ----------
@@ -661,15 +766,8 @@ def v_measure_score(labels_true, labels_pred, *, beta=1.0):
        conditional entropy-based external cluster evaluation measure
        `_
 
-    See Also
-    --------
-    homogeneity_score
-    completeness_score
-    normalized_mutual_info_score
-
     Examples
     --------
-
     Perfect labelings are both homogeneous and complete, hence have score 1.0::
 
       >>> from sklearn.metrics.cluster import v_measure_score
@@ -679,44 +777,50 @@ def v_measure_score(labels_true, labels_pred, *, beta=1.0):
       1.0
 
     Labelings that assign all classes members to the same clusters
-    are complete be not homogeneous, hence penalized::
+    are complete but not homogeneous, hence penalized::
 
       >>> print("%.6f" % v_measure_score([0, 0, 1, 2], [0, 0, 1, 1]))
-      0.8...
+      0.8
       >>> print("%.6f" % v_measure_score([0, 1, 2, 3], [0, 0, 1, 1]))
-      0.66...
+      0.67
 
     Labelings that have pure clusters with members coming from the same
-    classes are homogeneous but un-necessary splits harms completeness
+    classes are homogeneous but un-necessary splits harm completeness
     and thus penalize V-measure as well::
 
       >>> print("%.6f" % v_measure_score([0, 0, 1, 1], [0, 0, 1, 2]))
-      0.8...
+      0.8
       >>> print("%.6f" % v_measure_score([0, 0, 1, 1], [0, 1, 2, 3]))
-      0.66...
+      0.67
 
     If classes members are completely split across different clusters,
     the assignment is totally incomplete, hence the V-Measure is null::
 
       >>> print("%.6f" % v_measure_score([0, 0, 0, 0], [0, 1, 2, 3]))
-      0.0...
+      0.0
 
     Clusters that include samples from totally different classes totally
     destroy the homogeneity of the labeling, hence::
 
       >>> print("%.6f" % v_measure_score([0, 0, 1, 1], [0, 0, 0, 0]))
-      0.0...
+      0.0
     """
-    return homogeneity_completeness_v_measure(labels_true, labels_pred,
-                                              beta=beta)[2]
+    return homogeneity_completeness_v_measure(labels_true, labels_pred, beta=beta)[2]
 
 
-@_deprecate_positional_args
+@validate_params(
+    {
+        "labels_true": ["array-like", None],
+        "labels_pred": ["array-like", None],
+        "contingency": ["array-like", "sparse matrix", None],
+    },
+    prefer_skip_nested_validation=True,
+)
 def mutual_info_score(labels_true, labels_pred, *, contingency=None):
     """Mutual Information between two clusterings.
 
-    The Mutual Information is a measure of the similarity between two labels of
-    the same data. Where :math:`|U_i|` is the number of the samples
+    The Mutual Information is a measure of the similarity between two labels
+    of the same data. Where :math:`|U_i|` is the number of the samples
     in cluster :math:`U_i` and :math:`|V_j|` is the number of the
     samples in cluster :math:`V_j`, the Mutual Information
     between clusterings :math:`U` and :math:`V` is given as:
@@ -730,78 +834,107 @@ def mutual_info_score(labels_true, labels_pred, *, contingency=None):
     a permutation of the class or cluster label values won't change the
     score value in any way.
 
-    This metric is furthermore symmetric: switching ``label_true`` with
-    ``label_pred`` will return the same score value. This can be useful to
-    measure the agreement of two independent label assignments strategies
-    on the same dataset when the real ground truth is not known.
+    This metric is furthermore symmetric: switching :math:`U` (i.e
+    ``label_true``) with :math:`V` (i.e. ``label_pred``) will return the
+    same score value. This can be useful to measure the agreement of two
+    independent label assignments strategies on the same dataset when the
+    real ground truth is not known.
 
     Read more in the :ref:`User Guide `.
 
     Parameters
     ----------
-    labels_true : int array, shape = [n_samples]
-        A clustering of the data into disjoint subsets.
+    labels_true : array-like of shape (n_samples,), dtype=integral
+        A clustering of the data into disjoint subsets, called :math:`U` in
+        the above formula.
 
-    labels_pred : int array-like of shape (n_samples,)
-        A clustering of the data into disjoint subsets.
+    labels_pred : array-like of shape (n_samples,), dtype=integral
+        A clustering of the data into disjoint subsets, called :math:`V` in
+        the above formula.
 
-    contingency : {ndarray, sparse matrix} of shape \
+    contingency : {array-like, sparse matrix} of shape \
             (n_classes_true, n_classes_pred), default=None
-        A contingency matrix given by the :func:`contingency_matrix` function.
-        If value is ``None``, it will be computed, otherwise the given value is
-        used, with ``labels_true`` and ``labels_pred`` ignored.
+        A contingency matrix given by the
+        :func:`~sklearn.metrics.cluster.contingency_matrix` function. If value
+        is ``None``, it will be computed, otherwise the given value is used,
+        with ``labels_true`` and ``labels_pred`` ignored.
 
     Returns
     -------
     mi : float
-       Mutual information, a non-negative value
+       Mutual information, a non-negative value, measured in nats using the
+       natural logarithm.
+
+    See Also
+    --------
+    adjusted_mutual_info_score : Adjusted against chance Mutual Information.
+    normalized_mutual_info_score : Normalized Mutual Information.
 
     Notes
     -----
     The logarithm used is the natural logarithm (base-e).
 
-    See Also
+    Examples
     --------
-    adjusted_mutual_info_score : Adjusted against chance Mutual Information.
-    normalized_mutual_info_score : Normalized Mutual Information.
+    >>> from sklearn.metrics import mutual_info_score
+    >>> labels_true = [0, 1, 1, 0, 1, 0]
+    >>> labels_pred = [0, 1, 0, 0, 1, 1]
+    >>> mutual_info_score(labels_true, labels_pred)
+    0.0566
     """
     if contingency is None:
         labels_true, labels_pred = check_clusterings(labels_true, labels_pred)
         contingency = contingency_matrix(labels_true, labels_pred, sparse=True)
     else:
-        contingency = check_array(contingency,
-                                  accept_sparse=['csr', 'csc', 'coo'],
-                                  dtype=[int, np.int32, np.int64])
+        contingency = check_array(
+            contingency,
+            accept_sparse=["csr", "csc", "coo"],
+            dtype=[int, np.int32, np.int64],
+        )
 
     if isinstance(contingency, np.ndarray):
         # For an array
         nzx, nzy = np.nonzero(contingency)
         nz_val = contingency[nzx, nzy]
-    elif sp.issparse(contingency):
+    else:
         # For a sparse matrix
         nzx, nzy, nz_val = sp.find(contingency)
-    else:
-        raise ValueError("Unsupported type for 'contingency': %s" %
-                         type(contingency))
 
     contingency_sum = contingency.sum()
     pi = np.ravel(contingency.sum(axis=1))
     pj = np.ravel(contingency.sum(axis=0))
+
+    # Since MI <= min(H(X), H(Y)), any labelling with zero entropy, i.e. containing a
+    # single cluster, implies MI = 0
+    if pi.size == 1 or pj.size == 1:
+        return 0.0
+
     log_contingency_nm = np.log(nz_val)
     contingency_nm = nz_val / contingency_sum
     # Don't need to calculate the full outer product, just for non-zeroes
-    outer = (pi.take(nzx).astype(np.int64, copy=False)
-             * pj.take(nzy).astype(np.int64, copy=False))
+    outer = pi.take(nzx).astype(np.int64, copy=False) * pj.take(nzy).astype(
+        np.int64, copy=False
+    )
     log_outer = -np.log(outer) + log(pi.sum()) + log(pj.sum())
-    mi = (contingency_nm * (log_contingency_nm - log(contingency_sum)) +
-          contingency_nm * log_outer)
+    mi = (
+        contingency_nm * (log_contingency_nm - log(contingency_sum))
+        + contingency_nm * log_outer
+    )
     mi = np.where(np.abs(mi) < np.finfo(mi.dtype).eps, 0.0, mi)
-    return np.clip(mi.sum(), 0.0, None)
-
-
-@_deprecate_positional_args
-def adjusted_mutual_info_score(labels_true, labels_pred, *,
-                               average_method='arithmetic'):
+    return float(np.clip(mi.sum(), 0.0, None))
+
+
+@validate_params(
+    {
+        "labels_true": ["array-like"],
+        "labels_pred": ["array-like"],
+        "average_method": [StrOptions({"arithmetic", "max", "min", "geometric"})],
+    },
+    prefer_skip_nested_validation=True,
+)
+def adjusted_mutual_info_score(
+    labels_true, labels_pred, *, average_method="arithmetic"
+):
     """Adjusted Mutual Information between two clusterings.
 
     Adjusted Mutual Information (AMI) is an adjustment of the Mutual
@@ -816,10 +949,10 @@ def adjusted_mutual_info_score(labels_true, labels_pred, *,
     a permutation of the class or cluster label values won't change the
     score value in any way.
 
-    This metric is furthermore symmetric: switching ``label_true`` with
-    ``label_pred`` will return the same score value. This can be useful to
-    measure the agreement of two independent label assignments strategies
-    on the same dataset when the real ground truth is not known.
+    This metric is furthermore symmetric: switching :math:`U` (``label_true``)
+    with :math:`V` (``labels_pred``) will return the same score value. This can
+    be useful to measure the agreement of two independent label assignments
+    strategies on the same dataset when the real ground truth is not known.
 
     Be mindful that this function is an order of magnitude slower than other
     metrics, such as the Adjusted Rand Index.
@@ -828,15 +961,16 @@ def adjusted_mutual_info_score(labels_true, labels_pred, *,
 
     Parameters
     ----------
-    labels_true : int array, shape = [n_samples]
-        A clustering of the data into disjoint subsets.
+    labels_true : int array-like of shape (n_samples,)
+        A clustering of the data into disjoint subsets, called :math:`U` in
+        the above formula.
 
     labels_pred : int array-like of shape (n_samples,)
-        A clustering of the data into disjoint subsets.
+        A clustering of the data into disjoint subsets, called :math:`V` in
+        the above formula.
 
-    average_method : str, default='arithmetic'
-        How to compute the normalizer in the denominator. Possible options
-        are 'min', 'geometric', 'arithmetic', and 'max'.
+    average_method : {'min', 'geometric', 'arithmetic', 'max'}, default='arithmetic'
+        How to compute the normalizer in the denominator.
 
         .. versionadded:: 0.20
 
@@ -849,13 +983,24 @@ def adjusted_mutual_info_score(labels_true, labels_pred, *,
     ami: float (upperlimited by 1.0)
        The AMI returns a value of 1 when the two partitions are identical
        (ie perfectly matched). Random partitions (independent labellings) have
-       an expected AMI around 0 on average hence can be negative.
+       an expected AMI around 0 on average hence can be negative. The value is
+       in adjusted nats (based on the natural logarithm).
 
     See Also
     --------
     adjusted_rand_score : Adjusted Rand Index.
     mutual_info_score : Mutual Information (not adjusted for chance).
 
+    References
+    ----------
+    .. [1] `Vinh, Epps, and Bailey, (2010). Information Theoretic Measures for
+       Clusterings Comparison: Variants, Properties, Normalization and
+       Correction for Chance, JMLR
+       `_
+
+    .. [2] `Wikipedia entry for the Adjusted Mutual Information
+       `_
+
     Examples
     --------
 
@@ -864,65 +1009,70 @@ def adjusted_mutual_info_score(labels_true, labels_pred, *,
 
       >>> from sklearn.metrics.cluster import adjusted_mutual_info_score
       >>> adjusted_mutual_info_score([0, 0, 1, 1], [0, 0, 1, 1])
-      ... # doctest: +SKIP
       1.0
       >>> adjusted_mutual_info_score([0, 0, 1, 1], [1, 1, 0, 0])
-      ... # doctest: +SKIP
       1.0
 
     If classes members are completely split across different clusters,
     the assignment is totally in-complete, hence the AMI is null::
 
       >>> adjusted_mutual_info_score([0, 0, 0, 0], [0, 1, 2, 3])
-      ... # doctest: +SKIP
       0.0
-
-    References
-    ----------
-    .. [1] `Vinh, Epps, and Bailey, (2010). Information Theoretic Measures for
-       Clusterings Comparison: Variants, Properties, Normalization and
-       Correction for Chance, JMLR
-       `_
-
-    .. [2] `Wikipedia entry for the Adjusted Mutual Information
-       `_
     """
     labels_true, labels_pred = check_clusterings(labels_true, labels_pred)
     n_samples = labels_true.shape[0]
     classes = np.unique(labels_true)
     clusters = np.unique(labels_pred)
+
     # Special limit cases: no clustering since the data is not split.
+    # It corresponds to both labellings having zero entropy.
     # This is a perfect match hence return 1.0.
-    if (classes.shape[0] == clusters.shape[0] == 1 or
-            classes.shape[0] == clusters.shape[0] == 0):
+    if (
+        classes.shape[0] == clusters.shape[0] == 1
+        or classes.shape[0] == clusters.shape[0] == 0
+    ):
         return 1.0
+    # if there is only one class or one cluster return 0.0.
+    elif classes.shape[0] == 1 or clusters.shape[0] == 1:
+        return 0.0
+
     contingency = contingency_matrix(labels_true, labels_pred, sparse=True)
-    contingency = contingency.astype(np.float64,
-                                     **_astype_copy_false(contingency))
     # Calculate the MI for the two clusterings
-    mi = mutual_info_score(labels_true, labels_pred,
-                           contingency=contingency)
+    mi = mutual_info_score(labels_true, labels_pred, contingency=contingency)
     # Calculate the expected value for the mutual information
     emi = expected_mutual_information(contingency, n_samples)
     # Calculate entropy for each labeling
     h_true, h_pred = entropy(labels_true), entropy(labels_pred)
     normalizer = _generalized_average(h_true, h_pred, average_method)
     denominator = normalizer - emi
-    # Avoid 0.0 / 0.0 when expectation equals maximum, i.e a perfect match.
+    # Avoid 0.0 / 0.0 when expectation equals maximum, i.e. a perfect match.
     # normalizer should always be >= emi, but because of floating-point
     # representation, sometimes emi is slightly larger. Correct this
     # by preserving the sign.
     if denominator < 0:
-        denominator = min(denominator, -np.finfo('float64').eps)
+        denominator = min(denominator, -np.finfo("float64").eps)
     else:
-        denominator = max(denominator, np.finfo('float64').eps)
-    ami = (mi - emi) / denominator
-    return ami
-
-
-@_deprecate_positional_args
-def normalized_mutual_info_score(labels_true, labels_pred, *,
-                                 average_method='arithmetic'):
+        denominator = max(denominator, np.finfo("float64").eps)
+    # The same applies analogously to mi and emi.
+    numerator = mi - emi
+    if numerator < 0:
+        numerator = min(numerator, -np.finfo("float64").eps)
+    else:
+        numerator = max(numerator, np.finfo("float64").eps)
+    return float(numerator / denominator)
+
+
+@validate_params(
+    {
+        "labels_true": ["array-like"],
+        "labels_pred": ["array-like"],
+        "average_method": [StrOptions({"arithmetic", "max", "min", "geometric"})],
+    },
+    prefer_skip_nested_validation=True,
+)
+def normalized_mutual_info_score(
+    labels_true, labels_pred, *, average_method="arithmetic"
+):
     """Normalized Mutual Information between two clusterings.
 
     Normalized Mutual Information (NMI) is a normalization of the Mutual
@@ -947,15 +1097,14 @@ def normalized_mutual_info_score(labels_true, labels_pred, *,
 
     Parameters
     ----------
-    labels_true : int array, shape = [n_samples]
+    labels_true : int array-like of shape (n_samples,)
         A clustering of the data into disjoint subsets.
 
     labels_pred : int array-like of shape (n_samples,)
         A clustering of the data into disjoint subsets.
 
-    average_method : str, default='arithmetic'
-        How to compute the normalizer in the denominator. Possible options
-        are 'min', 'geometric', 'arithmetic', and 'max'.
+    average_method : {'min', 'geometric', 'arithmetic', 'max'}, default='arithmetic'
+        How to compute the normalizer in the denominator.
 
         .. versionadded:: 0.20
 
@@ -966,7 +1115,8 @@ def normalized_mutual_info_score(labels_true, labels_pred, *,
     Returns
     -------
     nmi : float
-       score between 0.0 and 1.0. 1.0 stands for perfectly complete labeling
+       Score between 0.0 and 1.0 in normalized nats (based on the natural
+       logarithm). 1.0 stands for perfectly complete labeling.
 
     See Also
     --------
@@ -983,17 +1133,14 @@ def normalized_mutual_info_score(labels_true, labels_pred, *,
 
       >>> from sklearn.metrics.cluster import normalized_mutual_info_score
       >>> normalized_mutual_info_score([0, 0, 1, 1], [0, 0, 1, 1])
-      ... # doctest: +SKIP
       1.0
       >>> normalized_mutual_info_score([0, 0, 1, 1], [1, 1, 0, 0])
-      ... # doctest: +SKIP
       1.0
 
     If classes members are completely split across different clusters,
     the assignment is totally in-complete, hence the NMI is null::
 
       >>> normalized_mutual_info_score([0, 0, 0, 0], [0, 1, 2, 3])
-      ... # doctest: +SKIP
       0.0
     """
     labels_true, labels_pred = check_clusterings(labels_true, labels_pred)
@@ -1001,44 +1148,57 @@ def normalized_mutual_info_score(labels_true, labels_pred, *,
     clusters = np.unique(labels_pred)
 
     # Special limit cases: no clustering since the data is not split.
+    # It corresponds to both labellings having zero entropy.
     # This is a perfect match hence return 1.0.
-    if (classes.shape[0] == clusters.shape[0] == 1 or
-            classes.shape[0] == clusters.shape[0] == 0):
+    if (
+        classes.shape[0] == clusters.shape[0] == 1
+        or classes.shape[0] == clusters.shape[0] == 0
+    ):
         return 1.0
+
     contingency = contingency_matrix(labels_true, labels_pred, sparse=True)
-    contingency = contingency.astype(np.float64,
-                                     **_astype_copy_false(contingency))
+    contingency = contingency.astype(np.float64, copy=False)
     # Calculate the MI for the two clusterings
-    mi = mutual_info_score(labels_true, labels_pred,
-                           contingency=contingency)
-    # Calculate the expected value for the mutual information
+    mi = mutual_info_score(labels_true, labels_pred, contingency=contingency)
+
+    # At this point mi = 0 can't be a perfect match (the special case of a single
+    # cluster has been dealt with before). Hence, if mi = 0, the nmi must be 0 whatever
+    # the normalization.
+    if mi == 0:
+        return 0.0
+
     # Calculate entropy for each labeling
     h_true, h_pred = entropy(labels_true), entropy(labels_pred)
-    normalizer = _generalized_average(h_true, h_pred, average_method)
-    # Avoid 0.0 / 0.0 when either entropy is zero.
-    normalizer = max(normalizer, np.finfo('float64').eps)
-    nmi = mi / normalizer
-    return nmi
 
-
-@_deprecate_positional_args
-def fowlkes_mallows_score(labels_true, labels_pred, *, sparse=False):
+    normalizer = _generalized_average(h_true, h_pred, average_method)
+    return float(mi / normalizer)
+
+
+@validate_params(
+    {
+        "labels_true": ["array-like"],
+        "labels_pred": ["array-like"],
+        "sparse": ["boolean", Hidden(StrOptions({"deprecated"}))],
+    },
+    prefer_skip_nested_validation=True,
+)
+def fowlkes_mallows_score(labels_true, labels_pred, *, sparse="deprecated"):
     """Measure the similarity of two clusterings of a set of points.
 
     .. versionadded:: 0.18
 
-    The Fowlkes-Mallows index (FMI) is defined as the geometric mean between of
+    The Fowlkes-Mallows index (FMI) is defined as the geometric mean of
     the precision and recall::
 
         FMI = TP / sqrt((TP + FP) * (TP + FN))
 
-    Where ``TP`` is the number of **True Positive** (i.e. the number of pair of
-    points that belongs in the same clusters in both ``labels_true`` and
+    Where ``TP`` is the number of **True Positive** (i.e. the number of pairs of
+    points that belong to the same cluster in both ``labels_true`` and
     ``labels_pred``), ``FP`` is the number of **False Positive** (i.e. the
-    number of pair of points that belongs in the same clusters in
-    ``labels_true`` and not in ``labels_pred``) and ``FN`` is the number of
-    **False Negative** (i.e the number of pair of points that belongs in the
-    same clusters in ``labels_pred`` and not in ``labels_True``).
+    number of pairs of points that belong to the same cluster in
+    ``labels_pred`` but not in ``labels_true``) and ``FN`` is the number of
+    **False Negative** (i.e. the number of pairs of points that belong to the
+    same cluster in ``labels_true`` but not in ``labels_pred``).
 
     The score ranges from 0 to 1. A high value indicates a good similarity
     between two clusters.
@@ -1047,20 +1207,34 @@ def fowlkes_mallows_score(labels_true, labels_pred, *, sparse=False):
 
     Parameters
     ----------
-    labels_true : int array, shape = (``n_samples``,)
+    labels_true : array-like of shape (n_samples,), dtype=int
         A clustering of the data into disjoint subsets.
 
-    labels_pred : array, shape = (``n_samples``, )
+    labels_pred : array-like of shape (n_samples,), dtype=int
         A clustering of the data into disjoint subsets.
 
     sparse : bool, default=False
         Compute contingency matrix internally with sparse matrix.
 
+        .. deprecated:: 1.7
+            The ``sparse`` parameter is deprecated and will be removed in 1.9. It has
+            no effect.
+
     Returns
     -------
     score : float
        The resulting Fowlkes-Mallows score.
 
+    References
+    ----------
+    .. [1] `E. B. Fowkles and C. L. Mallows, 1983. "A method for comparing two
+       hierarchical clusterings". Journal of the American Statistical
+       Association
+       `_
+
+    .. [2] `Wikipedia entry for the Fowlkes-Mallows Index
+           `_
+
     Examples
     --------
 
@@ -1078,47 +1252,63 @@ def fowlkes_mallows_score(labels_true, labels_pred, *, sparse=False):
 
       >>> fowlkes_mallows_score([0, 0, 0, 0], [0, 1, 2, 3])
       0.0
-
-    References
-    ----------
-    .. [1] `E. B. Fowkles and C. L. Mallows, 1983. "A method for comparing two
-       hierarchical clusterings". Journal of the American Statistical
-       Association
-       `_
-
-    .. [2] `Wikipedia entry for the Fowlkes-Mallows Index
-           `_
     """
+    # TODO(1.9): remove the sparse parameter
+    if sparse != "deprecated":
+        warnings.warn(
+            "The 'sparse' parameter was deprecated in 1.7 and will be removed in 1.9. "
+            "It has no effect. Leave it to its default value to silence this warning.",
+            FutureWarning,
+        )
+
     labels_true, labels_pred = check_clusterings(labels_true, labels_pred)
-    n_samples, = labels_true.shape
+    (n_samples,) = labels_true.shape
 
-    c = contingency_matrix(labels_true, labels_pred,
-                           sparse=True)
-    c = c.astype(np.int64, **_astype_copy_false(c))
+    c = contingency_matrix(labels_true, labels_pred, sparse=True)
+    c = c.astype(np.int64, copy=False)
     tk = np.dot(c.data, c.data) - n_samples
     pk = np.sum(np.asarray(c.sum(axis=0)).ravel() ** 2) - n_samples
     qk = np.sum(np.asarray(c.sum(axis=1)).ravel() ** 2) - n_samples
-    return np.sqrt(tk / pk) * np.sqrt(tk / qk) if tk != 0. else 0.
+    return float(np.sqrt(tk / pk) * np.sqrt(tk / qk)) if tk != 0.0 else 0.0
 
 
+@validate_params(
+    {
+        "labels": ["array-like"],
+    },
+    prefer_skip_nested_validation=True,
+)
 def entropy(labels):
-    """Calculates the entropy for a labeling.
+    """Calculate the entropy for a labeling.
 
     Parameters
     ----------
-    labels : int array, shape = [n_samples]
-        The labels
+    labels : array-like of shape (n_samples,), dtype=int
+        The labels.
+
+    Returns
+    -------
+    entropy : float
+       The entropy for a labeling.
 
     Notes
     -----
     The logarithm used is the natural logarithm (base-e).
     """
-    if len(labels) == 0:
+    xp, is_array_api_compliant, device_ = get_namespace_and_device(labels)
+    labels_len = labels.shape[0] if is_array_api_compliant else len(labels)
+    if labels_len == 0:
         return 1.0
-    label_idx = np.unique(labels, return_inverse=True)[1]
-    pi = np.bincount(label_idx).astype(np.float64)
-    pi = pi[pi > 0]
-    pi_sum = np.sum(pi)
+
+    pi = xp.astype(xp.unique_counts(labels)[1], _max_precision_float_dtype(xp, device_))
+
+    # single cluster => zero entropy
+    if pi.size == 1:
+        return 0.0
+
+    pi_sum = xp.sum(pi)
     # log(a / b) should be calculated as log(a) - log(b) for
     # possible loss of precision
-    return -np.sum((pi / pi_sum) * (np.log(pi) - log(pi_sum)))
+    # Always convert the result as a Python scalar (on CPU) instead of a device
+    # specific scalar array.
+    return float(-xp.sum((pi / pi_sum) * (xp.log(pi) - log(pi_sum))))
diff --git a/sklearn/metrics/cluster/_unsupervised.py b/sklearn/metrics/cluster/_unsupervised.py
index c597277a55b31..38cec419e73f7 100644
--- a/sklearn/metrics/cluster/_unsupervised.py
+++ b/sklearn/metrics/cluster/_unsupervised.py
@@ -1,22 +1,23 @@
 """Unsupervised evaluation metrics."""
 
-# Authors: Robert Layton 
-#          Arnaud Fouchet 
-#          Thierry Guillemot 
-# License: BSD 3 clause
-
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import functools
+from numbers import Integral
 
 import numpy as np
+from scipy.sparse import issparse
 
-from ...utils import check_random_state
-from ...utils import check_X_y
-from ...utils import _safe_indexing
-from ..pairwise import pairwise_distances_chunked
-from ..pairwise import pairwise_distances
 from ...preprocessing import LabelEncoder
-from ...utils.validation import _deprecate_positional_args
+from ...utils import _safe_indexing, check_random_state, check_X_y
+from ...utils._array_api import _atol_for_type
+from ...utils._param_validation import (
+    Interval,
+    StrOptions,
+    validate_params,
+)
+from ..pairwise import _VALID_METRICS, pairwise_distances, pairwise_distances_chunked
 
 
 def check_number_of_labels(n_labels, n_samples):
@@ -31,13 +32,25 @@ def check_number_of_labels(n_labels, n_samples):
         Number of samples.
     """
     if not 1 < n_labels < n_samples:
-        raise ValueError("Number of labels is %d. Valid values are 2 "
-                         "to n_samples - 1 (inclusive)" % n_labels)
-
-
-@_deprecate_positional_args
-def silhouette_score(X, labels, *, metric='euclidean', sample_size=None,
-                     random_state=None, **kwds):
+        raise ValueError(
+            "Number of labels is %d. Valid values are 2 to n_samples - 1 (inclusive)"
+            % n_labels
+        )
+
+
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "labels": ["array-like"],
+        "metric": [StrOptions(set(_VALID_METRICS) | {"precomputed"}), callable],
+        "sample_size": [Interval(Integral, 1, None, closed="left"), None],
+        "random_state": ["random_state"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def silhouette_score(
+    X, labels, *, metric="euclidean", sample_size=None, random_state=None, **kwds
+):
     """Compute the mean Silhouette Coefficient of all samples.
 
     The Silhouette Coefficient is calculated using the mean intra-cluster
@@ -59,7 +72,7 @@ def silhouette_score(X, labels, *, metric='euclidean', sample_size=None,
 
     Parameters
     ----------
-    X : array-like of shape (n_samples_a, n_samples_a) if metric == \
+    X : {array-like, sparse matrix} of shape (n_samples_a, n_samples_a) if metric == \
             "precomputed" or (n_samples_a, n_features) otherwise
         An array of pairwise distances between samples, or a feature array.
 
@@ -69,8 +82,7 @@ def silhouette_score(X, labels, *, metric='euclidean', sample_size=None,
     metric : str or callable, default='euclidean'
         The metric to use when calculating distance between instances in a
         feature array. If metric is a string, it must be one of the options
-        allowed by :func:`metrics.pairwise.pairwise_distances
-        `. If ``X`` is
+        allowed by :func:`~sklearn.metrics.pairwise_distances`. If ``X`` is
         the distance array itself, use ``metric="precomputed"``.
 
     sample_size : int, default=None
@@ -105,16 +117,25 @@ def silhouette_score(X, labels, *, metric='euclidean', sample_size=None,
     .. [2] `Wikipedia entry on the Silhouette Coefficient
            `_
 
+    Examples
+    --------
+    >>> from sklearn.datasets import make_blobs
+    >>> from sklearn.cluster import KMeans
+    >>> from sklearn.metrics import silhouette_score
+    >>> X, y = make_blobs(random_state=42)
+    >>> kmeans = KMeans(n_clusters=2, random_state=42)
+    >>> silhouette_score(X, kmeans.fit_predict(X))
+    0.49...
     """
     if sample_size is not None:
-        X, labels = check_X_y(X, labels, accept_sparse=['csc', 'csr'])
+        X, labels = check_X_y(X, labels, accept_sparse=["csc", "csr"])
         random_state = check_random_state(random_state)
         indices = random_state.permutation(X.shape[0])[:sample_size]
         if metric == "precomputed":
             X, labels = X[indices].T[indices].T, labels[indices]
         else:
             X, labels = X[indices], labels[indices]
-    return np.mean(silhouette_samples(X, labels, metric=metric, **kwds))
+    return float(np.mean(silhouette_samples(X, labels, metric=metric, **kwds)))
 
 
 def _silhouette_reduce(D_chunk, start, labels, label_freqs):
@@ -122,8 +143,9 @@ def _silhouette_reduce(D_chunk, start, labels, label_freqs):
 
     Parameters
     ----------
-    D_chunk : array-like of shape (n_chunk_samples, n_samples)
-        Precomputed distances for a chunk.
+    D_chunk : {array-like, sparse matrix} of shape (n_chunk_samples, n_samples)
+        Precomputed distances for a chunk. If a sparse matrix is provided,
+        only CSR format is accepted.
     start : int
         First index in the chunk.
     labels : array-like of shape (n_samples,)
@@ -131,26 +153,54 @@ def _silhouette_reduce(D_chunk, start, labels, label_freqs):
     label_freqs : array-like
         Distribution of cluster labels in ``labels``.
     """
+    n_chunk_samples = D_chunk.shape[0]
     # accumulate distances from each sample to each cluster
-    clust_dists = np.zeros((len(D_chunk), len(label_freqs)),
-                           dtype=D_chunk.dtype)
-    for i in range(len(D_chunk)):
-        clust_dists[i] += np.bincount(labels, weights=D_chunk[i],
-                                      minlength=len(label_freqs))
-
-    # intra_index selects intra-cluster distances within clust_dists
-    intra_index = (np.arange(len(D_chunk)), labels[start:start + len(D_chunk)])
-    # intra_clust_dists are averaged over cluster size outside this function
-    intra_clust_dists = clust_dists[intra_index]
-    # of the remaining distances we normalise and extract the minimum
-    clust_dists[intra_index] = np.inf
-    clust_dists /= label_freqs
-    inter_clust_dists = clust_dists.min(axis=1)
-    return intra_clust_dists, inter_clust_dists
-
+    cluster_distances = np.zeros(
+        (n_chunk_samples, len(label_freqs)), dtype=D_chunk.dtype
+    )
+
+    if issparse(D_chunk):
+        if D_chunk.format != "csr":
+            raise TypeError(
+                "Expected CSR matrix. Please pass sparse matrix in CSR format."
+            )
+        for i in range(n_chunk_samples):
+            indptr = D_chunk.indptr
+            indices = D_chunk.indices[indptr[i] : indptr[i + 1]]
+            sample_weights = D_chunk.data[indptr[i] : indptr[i + 1]]
+            sample_labels = np.take(labels, indices)
+            cluster_distances[i] += np.bincount(
+                sample_labels, weights=sample_weights, minlength=len(label_freqs)
+            )
+    else:
+        for i in range(n_chunk_samples):
+            sample_weights = D_chunk[i]
+            sample_labels = labels
+            cluster_distances[i] += np.bincount(
+                sample_labels, weights=sample_weights, minlength=len(label_freqs)
+            )
 
-@_deprecate_positional_args
-def silhouette_samples(X, labels, *, metric='euclidean', **kwds):
+    # intra_index selects intra-cluster distances within cluster_distances
+    end = start + n_chunk_samples
+    intra_index = (np.arange(n_chunk_samples), labels[start:end])
+    # intra_cluster_distances are averaged over cluster size outside this function
+    intra_cluster_distances = cluster_distances[intra_index]
+    # of the remaining distances we normalise and extract the minimum
+    cluster_distances[intra_index] = np.inf
+    cluster_distances /= label_freqs
+    inter_cluster_distances = cluster_distances.min(axis=1)
+    return intra_cluster_distances, inter_cluster_distances
+
+
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "labels": ["array-like"],
+        "metric": [StrOptions(set(_VALID_METRICS) | {"precomputed"}), callable],
+    },
+    prefer_skip_nested_validation=True,
+)
+def silhouette_samples(X, labels, *, metric="euclidean", **kwds):
     """Compute the Silhouette Coefficient for each sample.
 
     The Silhouette Coefficient is a measure of how well samples are clustered
@@ -175,9 +225,11 @@ def silhouette_samples(X, labels, *, metric='euclidean', **kwds):
 
     Parameters
     ----------
-    X : array-like of shape (n_samples_a, n_samples_a) if metric == \
+    X : {array-like, sparse matrix} of shape (n_samples_a, n_samples_a) if metric == \
             "precomputed" or (n_samples_a, n_features) otherwise
-        An array of pairwise distances between samples, or a feature array.
+        An array of pairwise distances between samples, or a feature array. If
+        a sparse matrix is provided, CSR format should be favoured avoiding
+        an additional copy.
 
     labels : array-like of shape (n_samples,)
         Label values for each sample.
@@ -185,11 +237,11 @@ def silhouette_samples(X, labels, *, metric='euclidean', **kwds):
     metric : str or callable, default='euclidean'
         The metric to use when calculating distance between instances in a
         feature array. If metric is a string, it must be one of the options
-        allowed by :func:`sklearn.metrics.pairwise.pairwise_distances`.
+        allowed by :func:`~sklearn.metrics.pairwise_distances`.
         If ``X`` is the distance array itself, use "precomputed" as the metric.
         Precomputed distance matrices must have 0 along the diagonal.
 
-    `**kwds` : optional keyword parameters
+    **kwds : optional keyword parameters
         Any further parameters are passed directly to the distance function.
         If using a ``scipy.spatial.distance`` metric, the parameters are still
         metric dependent. See the scipy docs for usage examples.
@@ -210,17 +262,32 @@ def silhouette_samples(X, labels, *, metric='euclidean', **kwds):
     .. [2] `Wikipedia entry on the Silhouette Coefficient
        `_
 
+    Examples
+    --------
+    >>> from sklearn.metrics import silhouette_samples
+    >>> from sklearn.datasets import make_blobs
+    >>> from sklearn.cluster import KMeans
+    >>> X, y = make_blobs(n_samples=50, random_state=42)
+    >>> kmeans = KMeans(n_clusters=3, random_state=42)
+    >>> labels = kmeans.fit_predict(X)
+    >>> silhouette_samples(X, labels)
+    array([...])
     """
-    X, labels = check_X_y(X, labels, accept_sparse=['csc', 'csr'])
+    X, labels = check_X_y(X, labels, accept_sparse=["csr"])
 
     # Check for non-zero diagonal entries in precomputed distance matrix
-    if metric == 'precomputed':
-        atol = np.finfo(X.dtype).eps * 100
-        if np.any(np.abs(np.diagonal(X)) > atol):
-            raise ValueError(
-                'The precomputed distance matrix contains non-zero '
-                'elements on the diagonal. Use np.fill_diagonal(X, 0).'
-            )
+    if metric == "precomputed":
+        error_msg = ValueError(
+            "The precomputed distance matrix contains non-zero "
+            "elements on the diagonal. Use np.fill_diagonal(X, 0)."
+        )
+        if X.dtype.kind == "f":
+            atol = _atol_for_type(X.dtype)
+
+            if np.any(np.abs(X.diagonal()) > atol):
+                raise error_msg
+        elif np.any(X.diagonal() != 0):  # integral dtype
+            raise error_msg
 
     le = LabelEncoder()
     labels = le.fit_transform(labels)
@@ -228,16 +295,16 @@ def silhouette_samples(X, labels, *, metric='euclidean', **kwds):
     label_freqs = np.bincount(labels)
     check_number_of_labels(len(le.classes_), n_samples)
 
-    kwds['metric'] = metric
-    reduce_func = functools.partial(_silhouette_reduce,
-                                    labels=labels, label_freqs=label_freqs)
-    results = zip(*pairwise_distances_chunked(X, reduce_func=reduce_func,
-                                              **kwds))
+    kwds["metric"] = metric
+    reduce_func = functools.partial(
+        _silhouette_reduce, labels=labels, label_freqs=label_freqs
+    )
+    results = zip(*pairwise_distances_chunked(X, reduce_func=reduce_func, **kwds))
     intra_clust_dists, inter_clust_dists = results
     intra_clust_dists = np.concatenate(intra_clust_dists)
     inter_clust_dists = np.concatenate(inter_clust_dists)
 
-    denom = (label_freqs - 1).take(labels, mode='clip')
+    denom = (label_freqs - 1).take(labels, mode="clip")
     with np.errstate(divide="ignore", invalid="ignore"):
         intra_clust_dists /= denom
 
@@ -248,13 +315,20 @@ def silhouette_samples(X, labels, *, metric='euclidean', **kwds):
     return np.nan_to_num(sil_samples)
 
 
+@validate_params(
+    {
+        "X": ["array-like"],
+        "labels": ["array-like"],
+    },
+    prefer_skip_nested_validation=True,
+)
 def calinski_harabasz_score(X, labels):
     """Compute the Calinski and Harabasz score.
 
     It is also known as the Variance Ratio Criterion.
 
-    The score is defined as ratio between the within-cluster dispersion and
-    the between-cluster dispersion.
+    The score is defined as ratio of the sum of between-cluster dispersion and
+    of within-cluster dispersion.
 
     Read more in the :ref:`User Guide `.
 
@@ -277,6 +351,16 @@ def calinski_harabasz_score(X, labels):
     .. [1] `T. Calinski and J. Harabasz, 1974. "A dendrite method for cluster
        analysis". Communications in Statistics
        `_
+
+    Examples
+    --------
+    >>> from sklearn.datasets import make_blobs
+    >>> from sklearn.cluster import KMeans
+    >>> from sklearn.metrics import calinski_harabasz_score
+    >>> X, _ = make_blobs(random_state=0)
+    >>> kmeans = KMeans(n_clusters=3, random_state=0,).fit(X)
+    >>> calinski_harabasz_score(X, kmeans.labels_)
+    114.8...
     """
     X, labels = check_X_y(X, labels)
     le = LabelEncoder()
@@ -287,7 +371,7 @@ def calinski_harabasz_score(X, labels):
 
     check_number_of_labels(n_labels, n_samples)
 
-    extra_disp, intra_disp = 0., 0.
+    extra_disp, intra_disp = 0.0, 0.0
     mean = np.mean(X, axis=0)
     for k in range(n_labels):
         cluster_k = X[labels == k]
@@ -295,13 +379,22 @@ def calinski_harabasz_score(X, labels):
         extra_disp += len(cluster_k) * np.sum((mean_k - mean) ** 2)
         intra_disp += np.sum((cluster_k - mean_k) ** 2)
 
-    return (1. if intra_disp == 0. else
-            extra_disp * (n_samples - n_labels) /
-            (intra_disp * (n_labels - 1.)))
+    return float(
+        1.0
+        if intra_disp == 0.0
+        else extra_disp * (n_samples - n_labels) / (intra_disp * (n_labels - 1.0))
+    )
 
 
+@validate_params(
+    {
+        "X": ["array-like"],
+        "labels": ["array-like"],
+    },
+    prefer_skip_nested_validation=True,
+)
 def davies_bouldin_score(X, labels):
-    """Computes the Davies-Bouldin score.
+    """Compute the Davies-Bouldin score.
 
     The score is defined as the average similarity measure of each cluster with
     its most similar cluster, where similarity is the ratio of within-cluster
@@ -335,6 +428,14 @@ def davies_bouldin_score(X, labels):
        `__.
        IEEE Transactions on Pattern Analysis and Machine Intelligence.
        PAMI-1 (2): 224-227
+
+    Examples
+    --------
+    >>> from sklearn.metrics import davies_bouldin_score
+    >>> X = [[0, 1], [1, 1], [3, 4]]
+    >>> labels = [0, 0, 1]
+    >>> davies_bouldin_score(X, labels)
+    0.12...
     """
     X, labels = check_X_y(X, labels)
     le = LabelEncoder()
@@ -349,8 +450,7 @@ def davies_bouldin_score(X, labels):
         cluster_k = _safe_indexing(X, labels == k)
         centroid = cluster_k.mean(axis=0)
         centroids[k] = centroid
-        intra_dists[k] = np.average(pairwise_distances(
-            cluster_k, [centroid]))
+        intra_dists[k] = np.average(pairwise_distances(cluster_k, [centroid]))
 
     centroid_distances = pairwise_distances(centroids)
 
@@ -360,4 +460,4 @@ def davies_bouldin_score(X, labels):
     centroid_distances[centroid_distances == 0] = np.inf
     combined_intra_dists = intra_dists[:, None] + intra_dists
     scores = np.max(combined_intra_dists / centroid_distances, axis=1)
-    return np.mean(scores)
+    return float(np.mean(scores))
diff --git a/sklearn/metrics/cluster/meson.build b/sklearn/metrics/cluster/meson.build
new file mode 100644
index 0000000000000..5f25296c7540f
--- /dev/null
+++ b/sklearn/metrics/cluster/meson.build
@@ -0,0 +1,6 @@
+py.extension_module(
+  '_expected_mutual_info_fast',
+  cython_gen.process('_expected_mutual_info_fast.pyx'),
+  subdir: 'sklearn/metrics/cluster',
+  install: true
+)
diff --git a/sklearn/metrics/cluster/setup.py b/sklearn/metrics/cluster/setup.py
deleted file mode 100644
index c39e414d9f3b0..0000000000000
--- a/sklearn/metrics/cluster/setup.py
+++ /dev/null
@@ -1,24 +0,0 @@
-import os
-
-import numpy
-from numpy.distutils.misc_util import Configuration
-
-
-def configuration(parent_package="", top_path=None):
-    config = Configuration("cluster", parent_package, top_path)
-    libraries = []
-    if os.name == 'posix':
-        libraries.append('m')
-    config.add_extension("_expected_mutual_info_fast",
-                         sources=["_expected_mutual_info_fast.pyx"],
-                         include_dirs=[numpy.get_include()],
-                         libraries=libraries)
-
-    config.add_subpackage("tests")
-
-    return config
-
-
-if __name__ == "__main__":
-    from numpy.distutils.core import setup
-    setup(**configuration().todict())
diff --git a/sklearn/metrics/cluster/tests/test_bicluster.py b/sklearn/metrics/cluster/tests/test_bicluster.py
index dcc55e311eaee..53f7805100a13 100644
--- a/sklearn/metrics/cluster/tests/test_bicluster.py
+++ b/sklearn/metrics/cluster/tests/test_bicluster.py
@@ -2,10 +2,9 @@
 
 import numpy as np
 
-from sklearn.utils._testing import assert_almost_equal
-
-from sklearn.metrics.cluster._bicluster import _jaccard
 from sklearn.metrics import consensus_score
+from sklearn.metrics.cluster._bicluster import _jaccard
+from sklearn.utils._testing import assert_almost_equal
 
 
 def test_jaccard():
@@ -21,8 +20,7 @@ def test_jaccard():
 
 
 def test_consensus_score():
-    a = [[True, True, False, False],
-         [False, False, True, True]]
+    a = [[True, True, False, False], [False, False, True, True]]
     b = a[::-1]
 
     assert consensus_score((a, a), (a, a)) == 1
@@ -37,14 +35,22 @@ def test_consensus_score():
 
 
 def test_consensus_score_issue2445():
-    ''' Different number of biclusters in A and B'''
-    a_rows = np.array([[True, True, False, False],
-                       [False, False, True, True],
-                       [False, False, False, True]])
-    a_cols = np.array([[True, True, False, False],
-                       [False, False, True, True],
-                       [False, False, False, True]])
+    """Different number of biclusters in A and B"""
+    a_rows = np.array(
+        [
+            [True, True, False, False],
+            [False, False, True, True],
+            [False, False, False, True],
+        ]
+    )
+    a_cols = np.array(
+        [
+            [True, True, False, False],
+            [False, False, True, True],
+            [False, False, False, True],
+        ]
+    )
     idx = [0, 2]
     s = consensus_score((a_rows, a_cols), (a_rows[idx], a_cols[idx]))
     # B contains 2 of the 3 biclusters in A, so score should be 2/3
-    assert_almost_equal(s, 2.0/3.0)
+    assert_almost_equal(s, 2.0 / 3.0)
diff --git a/sklearn/metrics/cluster/tests/test_common.py b/sklearn/metrics/cluster/tests/test_common.py
index a0d87ad4baa61..a73670fbffce4 100644
--- a/sklearn/metrics/cluster/tests/test_common.py
+++ b/sklearn/metrics/cluster/tests/test_common.py
@@ -1,24 +1,25 @@
 from functools import partial
+from itertools import chain
 
-import pytest
 import numpy as np
+import pytest
 
-from sklearn.metrics.cluster import adjusted_mutual_info_score
-from sklearn.metrics.cluster import adjusted_rand_score
-from sklearn.metrics.cluster import rand_score
-from sklearn.metrics.cluster import completeness_score
-from sklearn.metrics.cluster import fowlkes_mallows_score
-from sklearn.metrics.cluster import homogeneity_score
-from sklearn.metrics.cluster import mutual_info_score
-from sklearn.metrics.cluster import normalized_mutual_info_score
-from sklearn.metrics.cluster import v_measure_score
-from sklearn.metrics.cluster import silhouette_score
-from sklearn.metrics.cluster import calinski_harabasz_score
-from sklearn.metrics.cluster import davies_bouldin_score
-
+from sklearn.metrics.cluster import (
+    adjusted_mutual_info_score,
+    adjusted_rand_score,
+    calinski_harabasz_score,
+    completeness_score,
+    davies_bouldin_score,
+    fowlkes_mallows_score,
+    homogeneity_score,
+    mutual_info_score,
+    normalized_mutual_info_score,
+    rand_score,
+    silhouette_score,
+    v_measure_score,
+)
 from sklearn.utils._testing import assert_allclose
 
-
 # Dictionaries of metrics
 # ------------------------
 # The goal of having those dictionaries is to have an easy way to call a
@@ -40,14 +41,14 @@
     "mutual_info_score": mutual_info_score,
     "normalized_mutual_info_score": normalized_mutual_info_score,
     "v_measure_score": v_measure_score,
-    "fowlkes_mallows_score": fowlkes_mallows_score
+    "fowlkes_mallows_score": fowlkes_mallows_score,
 }
 
 UNSUPERVISED_METRICS = {
     "silhouette_score": silhouette_score,
-    "silhouette_manhattan": partial(silhouette_score, metric='manhattan'),
+    "silhouette_manhattan": partial(silhouette_score, metric="manhattan"),
     "calinski_harabasz_score": calinski_harabasz_score,
-    "davies_bouldin_score": davies_bouldin_score
+    "davies_bouldin_score": davies_bouldin_score,
 }
 
 # Lists of metrics with common properties
@@ -60,18 +61,27 @@
 # Symmetric with respect to their input arguments y_true and y_pred.
 # Symmetric metrics only apply to supervised clusters.
 SYMMETRIC_METRICS = [
-    "adjusted_rand_score", "rand_score", "v_measure_score",
-    "mutual_info_score", "adjusted_mutual_info_score",
-    "normalized_mutual_info_score", "fowlkes_mallows_score"
+    "adjusted_rand_score",
+    "rand_score",
+    "v_measure_score",
+    "mutual_info_score",
+    "adjusted_mutual_info_score",
+    "normalized_mutual_info_score",
+    "fowlkes_mallows_score",
 ]
 
 NON_SYMMETRIC_METRICS = ["homogeneity_score", "completeness_score"]
 
 # Metrics whose upper bound is 1
 NORMALIZED_METRICS = [
-    "adjusted_rand_score", "rand_score", "homogeneity_score",
-    "completeness_score", "v_measure_score", "adjusted_mutual_info_score",
-    "fowlkes_mallows_score", "normalized_mutual_info_score"
+    "adjusted_rand_score",
+    "rand_score",
+    "homogeneity_score",
+    "completeness_score",
+    "v_measure_score",
+    "adjusted_mutual_info_score",
+    "fowlkes_mallows_score",
+    "normalized_mutual_info_score",
 ]
 
 
@@ -81,15 +91,13 @@
 
 
 def test_symmetric_non_symmetric_union():
-    assert (sorted(SYMMETRIC_METRICS + NON_SYMMETRIC_METRICS) ==
-            sorted(SUPERVISED_METRICS))
+    assert sorted(SYMMETRIC_METRICS + NON_SYMMETRIC_METRICS) == sorted(
+        SUPERVISED_METRICS
+    )
 
 
-# 0.22 AMI and NMI changes
-@pytest.mark.filterwarnings('ignore::FutureWarning')
 @pytest.mark.parametrize(
-    'metric_name, y1, y2',
-    [(name, y1, y2) for name in SYMMETRIC_METRICS]
+    "metric_name, y1, y2", [(name, y1, y2) for name in SYMMETRIC_METRICS]
 )
 def test_symmetry(metric_name, y1, y2):
     metric = SUPERVISED_METRICS[metric_name]
@@ -97,16 +105,13 @@ def test_symmetry(metric_name, y1, y2):
 
 
 @pytest.mark.parametrize(
-    'metric_name, y1, y2',
-    [(name, y1, y2) for name in NON_SYMMETRIC_METRICS]
+    "metric_name, y1, y2", [(name, y1, y2) for name in NON_SYMMETRIC_METRICS]
 )
 def test_non_symmetry(metric_name, y1, y2):
     metric = SUPERVISED_METRICS[metric_name]
     assert metric(y1, y2) != pytest.approx(metric(y2, y1))
 
 
-# 0.22 AMI and NMI changes
-@pytest.mark.filterwarnings('ignore::FutureWarning')
 @pytest.mark.parametrize("metric_name", NORMALIZED_METRICS)
 def test_normalized_output(metric_name):
     upper_bound_1 = [0, 0, 0, 1, 1, 1]
@@ -120,16 +125,13 @@ def test_normalized_output(metric_name):
 
     lower_bound_1 = [0, 0, 0, 0, 0, 0]
     lower_bound_2 = [0, 1, 2, 3, 4, 5]
-    score = np.array([metric(lower_bound_1, lower_bound_2),
-                      metric(lower_bound_2, lower_bound_1)])
+    score = np.array(
+        [metric(lower_bound_1, lower_bound_2), metric(lower_bound_2, lower_bound_1)]
+    )
     assert not (score < 0).any()
 
 
-# 0.22 AMI and NMI changes
-@pytest.mark.filterwarnings('ignore::FutureWarning')
-@pytest.mark.parametrize(
-    "metric_name", dict(SUPERVISED_METRICS, **UNSUPERVISED_METRICS)
-)
+@pytest.mark.parametrize("metric_name", chain(SUPERVISED_METRICS, UNSUPERVISED_METRICS))
 def test_permute_labels(metric_name):
     # All clustering metrics do not change score due to permutations of labels
     # that is when 0 and 1 exchanged.
@@ -148,11 +150,7 @@ def test_permute_labels(metric_name):
         assert_allclose(score_1, metric(X, 1 - y_pred))
 
 
-# 0.22 AMI and NMI changes
-@pytest.mark.filterwarnings('ignore::FutureWarning')
-@pytest.mark.parametrize(
-    "metric_name", dict(SUPERVISED_METRICS, **UNSUPERVISED_METRICS)
-)
+@pytest.mark.parametrize("metric_name", chain(SUPERVISED_METRICS, UNSUPERVISED_METRICS))
 # For all clustering metrics Input parameters can be both
 # in the form of arrays lists, positive, negative or string
 def test_format_invariance(metric_name):
@@ -161,21 +159,22 @@ def test_format_invariance(metric_name):
 
     def generate_formats(y):
         y = np.array(y)
-        yield y, 'array of ints'
-        yield y.tolist(), 'list of ints'
-        yield [str(x) + "-a" for x in y.tolist()], 'list of strs'
-        yield (np.array([str(x) + "-a" for x in y.tolist()], dtype=object),
-               'array of strs')
-        yield y - 1, 'including negative ints'
-        yield y + 1, 'strictly positive ints'
+        yield y, "array of ints"
+        yield y.tolist(), "list of ints"
+        yield [str(x) + "-a" for x in y.tolist()], "list of strs"
+        yield (
+            np.array([str(x) + "-a" for x in y.tolist()], dtype=object),
+            "array of strs",
+        )
+        yield y - 1, "including negative ints"
+        yield y + 1, "strictly positive ints"
 
     if metric_name in SUPERVISED_METRICS:
         metric = SUPERVISED_METRICS[metric_name]
         score_1 = metric(y_true, y_pred)
         y_true_gen = generate_formats(y_true)
         y_pred_gen = generate_formats(y_pred)
-        for (y_true_fmt, fmt_name), (y_pred_fmt, _) in zip(y_true_gen,
-                                                           y_pred_gen):
+        for (y_true_fmt, fmt_name), (y_pred_fmt, _) in zip(y_true_gen, y_pred_gen):
             assert score_1 == metric(y_true_fmt, y_pred_fmt)
     else:
         metric = UNSUPERVISED_METRICS[metric_name]
@@ -183,7 +182,7 @@ def generate_formats(y):
         score_1 = metric(X, y_true)
         assert score_1 == metric(X.astype(float), y_true)
         y_true_gen = generate_formats(y_true)
-        for (y_true_fmt, fmt_name) in y_true_gen:
+        for y_true_fmt, fmt_name in y_true_gen:
             assert score_1 == metric(X, y_true_fmt)
 
 
@@ -195,19 +194,41 @@ def test_single_sample(metric):
 
 
 @pytest.mark.parametrize(
-    "metric_name, metric_func",
-    dict(SUPERVISED_METRICS, **UNSUPERVISED_METRICS).items()
+    "metric_name, metric_func", dict(SUPERVISED_METRICS, **UNSUPERVISED_METRICS).items()
 )
 def test_inf_nan_input(metric_name, metric_func):
     if metric_name in SUPERVISED_METRICS:
-        invalids = [([0, 1], [np.inf, np.inf]),
-                    ([0, 1], [np.nan, np.nan]),
-                    ([0, 1], [np.nan, np.inf])]
+        invalids = [
+            ([0, 1], [np.inf, np.inf]),
+            ([0, 1], [np.nan, np.nan]),
+            ([0, 1], [np.nan, np.inf]),
+        ]
     else:
         X = np.random.randint(10, size=(2, 10))
-        invalids = [(X, [np.inf, np.inf]),
-                    (X, [np.nan, np.nan]),
-                    (X, [np.nan, np.inf])]
-    with pytest.raises(ValueError, match='contains NaN, infinity'):
+        invalids = [(X, [np.inf, np.inf]), (X, [np.nan, np.nan]), (X, [np.nan, np.inf])]
+    with pytest.raises(ValueError, match=r"contains (NaN|infinity)"):
         for args in invalids:
             metric_func(*args)
+
+
+@pytest.mark.parametrize("name", chain(SUPERVISED_METRICS, UNSUPERVISED_METRICS))
+def test_returned_value_consistency(name):
+    """Ensure that the returned values of all metrics are consistent.
+
+    It can only be a float. It should not be a numpy float64 or float32.
+    """
+
+    rng = np.random.RandomState(0)
+    X = rng.randint(10, size=(20, 10))
+    labels_true = rng.randint(0, 3, size=(20,))
+    labels_pred = rng.randint(0, 3, size=(20,))
+
+    if name in SUPERVISED_METRICS:
+        metric = SUPERVISED_METRICS[name]
+        score = metric(labels_true, labels_pred)
+    else:
+        metric = UNSUPERVISED_METRICS[name]
+        score = metric(X, labels_pred)
+
+    assert isinstance(score, float)
+    assert not isinstance(score, (np.float64, np.float32))
diff --git a/sklearn/metrics/cluster/tests/test_supervised.py b/sklearn/metrics/cluster/tests/test_supervised.py
index c6d3d2f808843..7421b726ebe67 100644
--- a/sklearn/metrics/cluster/tests/test_supervised.py
+++ b/sklearn/metrics/cluster/tests/test_supervised.py
@@ -1,29 +1,33 @@
+import warnings
+
 import numpy as np
 import pytest
+from numpy.testing import assert_allclose, assert_array_almost_equal, assert_array_equal
 
-from sklearn.metrics.cluster import adjusted_mutual_info_score
-from sklearn.metrics.cluster import adjusted_rand_score
-from sklearn.metrics.cluster import rand_score
-from sklearn.metrics.cluster import completeness_score
-from sklearn.metrics.cluster import contingency_matrix
-from sklearn.metrics.cluster import pair_confusion_matrix
-from sklearn.metrics.cluster import entropy
-from sklearn.metrics.cluster import expected_mutual_information
-from sklearn.metrics.cluster import fowlkes_mallows_score
-from sklearn.metrics.cluster import homogeneity_completeness_v_measure
-from sklearn.metrics.cluster import homogeneity_score
-from sklearn.metrics.cluster import mutual_info_score
-from sklearn.metrics.cluster import normalized_mutual_info_score
-from sklearn.metrics.cluster import v_measure_score
-from sklearn.metrics.cluster._supervised import _generalized_average
-from sklearn.metrics.cluster._supervised import check_clusterings
-
+from sklearn.base import config_context
+from sklearn.metrics.cluster import (
+    adjusted_mutual_info_score,
+    adjusted_rand_score,
+    completeness_score,
+    contingency_matrix,
+    entropy,
+    expected_mutual_information,
+    fowlkes_mallows_score,
+    homogeneity_completeness_v_measure,
+    homogeneity_score,
+    mutual_info_score,
+    normalized_mutual_info_score,
+    pair_confusion_matrix,
+    rand_score,
+    v_measure_score,
+)
+from sklearn.metrics.cluster._supervised import _generalized_average, check_clusterings
 from sklearn.utils import assert_all_finite
-from sklearn.utils._testing import (
-    assert_almost_equal, ignore_warnings)
-from numpy.testing import (
-    assert_array_equal, assert_array_almost_equal, assert_allclose)
-
+from sklearn.utils._array_api import (
+    _get_namespace_device_dtype_ids,
+    yield_namespace_device_dtype_combinations,
+)
+from sklearn.utils._testing import _array_api_for_tests, assert_almost_equal
 
 score_funcs = [
     adjusted_rand_score,
@@ -36,21 +40,19 @@
 ]
 
 
-@ignore_warnings(category=FutureWarning)
-def test_error_messages_on_wrong_input():
-    for score_func in score_funcs:
-        expected = (r'Found input variables with inconsistent numbers '
-                    r'of samples: \[2, 3\]')
-        with pytest.raises(ValueError, match=expected):
-            score_func([0, 1], [1, 1, 1])
+@pytest.mark.parametrize("score_func", score_funcs)
+def test_error_messages_on_wrong_input(score_func):
+    expected = r"Found input variables with inconsistent numbers of samples: \[2, 3\]"
+    with pytest.raises(ValueError, match=expected):
+        score_func([0, 1], [1, 1, 1])
 
-        expected = r"labels_true must be 1D: shape is \(2"
-        with pytest.raises(ValueError, match=expected):
-            score_func([[0, 1], [1, 0]], [1, 1, 1])
+    expected = r"labels_true must be 1D: shape is \(2"
+    with pytest.raises(ValueError, match=expected):
+        score_func([[0, 1], [1, 0]], [1, 1, 1])
 
-        expected = r"labels_pred must be 1D: shape is \(2"
-        with pytest.raises(ValueError, match=expected):
-            score_func([0, 1, 0], [[1, 1], [0, 0]])
+    expected = r"labels_pred must be 1D: shape is \(2"
+    with pytest.raises(ValueError, match=expected):
+        score_func([0, 1, 0], [[1, 1], [0, 0]])
 
 
 def test_generalized_average():
@@ -63,41 +65,55 @@ def test_generalized_average():
     assert means[0] == means[1] == means[2] == means[3]
 
 
-@ignore_warnings(category=FutureWarning)
-def test_perfect_matches():
-    for score_func in score_funcs:
-        assert score_func([], []) == pytest.approx(1.0)
-        assert score_func([0], [1]) == pytest.approx(1.0)
-        assert score_func([0, 0, 0], [0, 0, 0]) == pytest.approx(1.0)
-        assert score_func([0, 1, 0], [42, 7, 42]) == pytest.approx(1.0)
-        assert score_func([0., 1., 0.], [42., 7., 42.]) == pytest.approx(1.0)
-        assert score_func([0., 1., 2.], [42., 7., 2.]) == pytest.approx(1.0)
-        assert score_func([0, 1, 2], [42, 7, 2]) == pytest.approx(1.0)
-    score_funcs_with_changing_means = [
+@pytest.mark.parametrize("score_func", score_funcs)
+def test_perfect_matches(score_func):
+    assert score_func([], []) == pytest.approx(1.0)
+    assert score_func([0], [1]) == pytest.approx(1.0)
+    assert score_func([0, 0, 0], [0, 0, 0]) == pytest.approx(1.0)
+    assert score_func([0, 1, 0], [42, 7, 42]) == pytest.approx(1.0)
+    assert score_func([0.0, 1.0, 0.0], [42.0, 7.0, 42.0]) == pytest.approx(1.0)
+    assert score_func([0.0, 1.0, 2.0], [42.0, 7.0, 2.0]) == pytest.approx(1.0)
+    assert score_func([0, 1, 2], [42, 7, 2]) == pytest.approx(1.0)
+
+
+@pytest.mark.parametrize(
+    "score_func",
+    [
         normalized_mutual_info_score,
         adjusted_mutual_info_score,
-    ]
-    means = {"min", "geometric", "arithmetic", "max"}
-    for score_func in score_funcs_with_changing_means:
-        for mean in means:
-            assert score_func([], [], mean) == pytest.approx(1.0)
-            assert score_func([0], [1], mean) == pytest.approx(1.0)
-            assert score_func([0, 0, 0], [0, 0, 0], mean) == pytest.approx(1.0)
-            assert score_func(
-                [0, 1, 0], [42, 7, 42], mean) == pytest.approx(1.0)
-            assert score_func(
-                [0., 1., 0.], [42., 7., 42.], mean) == pytest.approx(1.0)
-            assert score_func(
-                [0., 1., 2.], [42., 7., 2.], mean) == pytest.approx(1.0)
-            assert score_func(
-                [0, 1, 2], [42, 7, 2], mean) == pytest.approx(1.0)
+    ],
+)
+@pytest.mark.parametrize("average_method", ["min", "geometric", "arithmetic", "max"])
+def test_perfect_matches_with_changing_means(score_func, average_method):
+    assert score_func([], [], average_method=average_method) == pytest.approx(1.0)
+    assert score_func([0], [1], average_method=average_method) == pytest.approx(1.0)
+    assert score_func(
+        [0, 0, 0], [0, 0, 0], average_method=average_method
+    ) == pytest.approx(1.0)
+    assert score_func(
+        [0, 1, 0], [42, 7, 42], average_method=average_method
+    ) == pytest.approx(1.0)
+    assert score_func(
+        [0.0, 1.0, 0.0], [42.0, 7.0, 42.0], average_method=average_method
+    ) == pytest.approx(1.0)
+    assert score_func(
+        [0.0, 1.0, 2.0], [42.0, 7.0, 2.0], average_method=average_method
+    ) == pytest.approx(1.0)
+    assert score_func(
+        [0, 1, 2], [42, 7, 2], average_method=average_method
+    ) == pytest.approx(1.0)
+    # Non-regression tests for: https://github.com/scikit-learn/scikit-learn/issues/30950
+    assert score_func([0, 1], [0, 1], average_method=average_method) == pytest.approx(
+        1.0
+    )
+    assert score_func(
+        [0, 1, 2, 3], [0, 1, 2, 3], average_method=average_method
+    ) == pytest.approx(1.0)
 
 
 def test_homogeneous_but_not_complete_labeling():
     # homogeneous but not complete clustering
-    h, c, v = homogeneity_completeness_v_measure(
-        [0, 0, 0, 1, 1, 1],
-        [0, 0, 0, 1, 2, 2])
+    h, c, v = homogeneity_completeness_v_measure([0, 0, 0, 1, 1, 1], [0, 0, 0, 1, 2, 2])
     assert_almost_equal(h, 1.00, 2)
     assert_almost_equal(c, 0.69, 2)
     assert_almost_equal(v, 0.81, 2)
@@ -105,9 +121,7 @@ def test_homogeneous_but_not_complete_labeling():
 
 def test_complete_but_not_homogeneous_labeling():
     # complete but not homogeneous clustering
-    h, c, v = homogeneity_completeness_v_measure(
-        [0, 0, 1, 1, 2, 2],
-        [0, 0, 1, 1, 1, 1])
+    h, c, v = homogeneity_completeness_v_measure([0, 0, 1, 1, 2, 2], [0, 0, 1, 1, 1, 1])
     assert_almost_equal(h, 0.58, 2)
     assert_almost_equal(c, 1.00, 2)
     assert_almost_equal(v, 0.73, 2)
@@ -115,9 +129,7 @@ def test_complete_but_not_homogeneous_labeling():
 
 def test_not_complete_and_not_homogeneous_labeling():
     # neither complete nor homogeneous but not so bad either
-    h, c, v = homogeneity_completeness_v_measure(
-        [0, 0, 0, 1, 1, 1],
-        [0, 1, 0, 1, 2, 2])
+    h, c, v = homogeneity_completeness_v_measure([0, 0, 0, 1, 1, 1], [0, 1, 0, 1, 2, 2])
     assert_almost_equal(h, 0.67, 2)
     assert_almost_equal(c, 0.42, 2)
     assert_almost_equal(v, 0.52, 2)
@@ -130,36 +142,27 @@ def test_beta_parameter():
     beta_test = 0.2
     h_test = 0.67
     c_test = 0.42
-    v_test = ((1 + beta_test) * h_test * c_test
-              / (beta_test * h_test + c_test))
+    v_test = (1 + beta_test) * h_test * c_test / (beta_test * h_test + c_test)
 
     h, c, v = homogeneity_completeness_v_measure(
-        [0, 0, 0, 1, 1, 1],
-        [0, 1, 0, 1, 2, 2],
-        beta=beta_test)
+        [0, 0, 0, 1, 1, 1], [0, 1, 0, 1, 2, 2], beta=beta_test
+    )
     assert_almost_equal(h, h_test, 2)
     assert_almost_equal(c, c_test, 2)
     assert_almost_equal(v, v_test, 2)
 
-    v = v_measure_score(
-        [0, 0, 0, 1, 1, 1],
-        [0, 1, 0, 1, 2, 2],
-        beta=beta_test)
+    v = v_measure_score([0, 0, 0, 1, 1, 1], [0, 1, 0, 1, 2, 2], beta=beta_test)
     assert_almost_equal(v, v_test, 2)
 
 
 def test_non_consecutive_labels():
     # regression tests for labels with gaps
-    h, c, v = homogeneity_completeness_v_measure(
-        [0, 0, 0, 2, 2, 2],
-        [0, 1, 0, 1, 2, 2])
+    h, c, v = homogeneity_completeness_v_measure([0, 0, 0, 2, 2, 2], [0, 1, 0, 1, 2, 2])
     assert_almost_equal(h, 0.67, 2)
     assert_almost_equal(c, 0.42, 2)
     assert_almost_equal(v, 0.52, 2)
 
-    h, c, v = homogeneity_completeness_v_measure(
-        [0, 0, 0, 1, 1, 1],
-        [0, 4, 0, 4, 2, 2])
+    h, c, v = homogeneity_completeness_v_measure([0, 0, 0, 1, 1, 1], [0, 4, 0, 4, 2, 2])
     assert_almost_equal(h, 0.67, 2)
     assert_almost_equal(c, 0.42, 2)
     assert_almost_equal(v, 0.52, 2)
@@ -175,9 +178,7 @@ def test_non_consecutive_labels():
     assert_almost_equal(ri_2, 0.66, 2)
 
 
-@ignore_warnings(category=FutureWarning)
-def uniform_labelings_scores(score_func, n_samples, k_range, n_runs=10,
-                             seed=42):
+def uniform_labelings_scores(score_func, n_samples, k_range, n_runs=10, seed=42):
     # Compute score for random uniform cluster labelings
     random_labels = np.random.RandomState(seed).randint
     scores = np.zeros((len(k_range), n_runs))
@@ -189,7 +190,6 @@ def uniform_labelings_scores(score_func, n_samples, k_range, n_runs=10,
     return scores
 
 
-@ignore_warnings(category=FutureWarning)
 def test_adjustment_for_chance():
     # Check that adjusted scores are almost zero on random labels
     n_clusters_range = [2, 10, 50, 90]
@@ -197,7 +197,8 @@ def test_adjustment_for_chance():
     n_runs = 10
 
     scores = uniform_labelings_scores(
-        adjusted_rand_score, n_samples, n_clusters_range, n_runs)
+        adjusted_rand_score, n_samples, n_clusters_range, n_runs
+    )
 
     max_abs_scores = np.abs(scores).max(axis=1)
     assert_array_almost_equal(max_abs_scores, [0.02, 0.03, 0.03, 0.02], 2)
@@ -242,32 +243,60 @@ def test_expected_mutual_info_overflow():
 
 def test_int_overflow_mutual_info_fowlkes_mallows_score():
     # Test overflow in mutual_info_classif and fowlkes_mallows_score
-    x = np.array([1] * (52632 + 2529) + [2] * (14660 + 793) + [3] * (3271 +
-                 204) + [4] * (814 + 39) + [5] * (316 + 20))
-    y = np.array([0] * 52632 + [1] * 2529 + [0] * 14660 + [1] * 793 +
-                 [0] * 3271 + [1] * 204 + [0] * 814 + [1] * 39 + [0] * 316 +
-                 [1] * 20)
+    x = np.array(
+        [1] * (52632 + 2529)
+        + [2] * (14660 + 793)
+        + [3] * (3271 + 204)
+        + [4] * (814 + 39)
+        + [5] * (316 + 20)
+    )
+    y = np.array(
+        [0] * 52632
+        + [1] * 2529
+        + [0] * 14660
+        + [1] * 793
+        + [0] * 3271
+        + [1] * 204
+        + [0] * 814
+        + [1] * 39
+        + [0] * 316
+        + [1] * 20
+    )
 
     assert_all_finite(mutual_info_score(x, y))
     assert_all_finite(fowlkes_mallows_score(x, y))
 
 
 def test_entropy():
-    ent = entropy([0, 0, 42.])
-    assert_almost_equal(ent, 0.6365141, 5)
+    assert_almost_equal(entropy([0, 0, 42.0]), 0.6365141, 5)
     assert_almost_equal(entropy([]), 1)
+    assert entropy([1, 1, 1, 1]) == 0
+
+
+@pytest.mark.parametrize(
+    "array_namespace, device, dtype_name",
+    yield_namespace_device_dtype_combinations(),
+    ids=_get_namespace_device_dtype_ids,
+)
+def test_entropy_array_api(array_namespace, device, dtype_name):
+    xp = _array_api_for_tests(array_namespace, device)
+    float_labels = xp.asarray(np.asarray([0, 0, 42.0], dtype=dtype_name), device=device)
+    empty_int32_labels = xp.asarray([], dtype=xp.int32, device=device)
+    int_labels = xp.asarray([1, 1, 1, 1], device=device)
+    with config_context(array_api_dispatch=True):
+        assert entropy(float_labels) == pytest.approx(0.6365141, abs=1e-5)
+        assert entropy(empty_int32_labels) == 1
+        assert entropy(int_labels) == 0
 
 
 def test_contingency_matrix():
     labels_a = np.array([1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3])
     labels_b = np.array([1, 1, 1, 1, 2, 1, 2, 2, 2, 2, 3, 1, 3, 3, 3, 2, 2])
     C = contingency_matrix(labels_a, labels_b)
-    C2 = np.histogram2d(labels_a, labels_b,
-                        bins=(np.arange(1, 5),
-                              np.arange(1, 5)))[0]
+    C2 = np.histogram2d(labels_a, labels_b, bins=(np.arange(1, 5), np.arange(1, 5)))[0]
     assert_array_almost_equal(C, C2)
-    C = contingency_matrix(labels_a, labels_b, eps=.1)
-    assert_array_almost_equal(C, C2 + .1)
+    C = contingency_matrix(labels_a, labels_b, eps=0.1)
+    assert_array_almost_equal(C, C2 + 0.1)
 
 
 def test_contingency_matrix_sparse():
@@ -280,65 +309,65 @@ def test_contingency_matrix_sparse():
         contingency_matrix(labels_a, labels_b, eps=1e-10, sparse=True)
 
 
-@ignore_warnings(category=FutureWarning)
 def test_exactly_zero_info_score():
     # Check numerical stability when information is exactly zero
     for i in np.logspace(1, 4, 4).astype(int):
-        labels_a, labels_b = (np.ones(i, dtype=int),
-                              np.arange(i, dtype=int))
-        assert normalized_mutual_info_score(
-            labels_a, labels_b) == pytest.approx(0.0)
-        assert v_measure_score(
-            labels_a, labels_b) == pytest.approx(0.0)
-        assert adjusted_mutual_info_score(
-            labels_a, labels_b) == pytest.approx(0.0)
-        assert normalized_mutual_info_score(
-            labels_a, labels_b) == pytest.approx(0.0)
+        labels_a, labels_b = (np.ones(i, dtype=int), np.arange(i, dtype=int))
+        assert normalized_mutual_info_score(labels_a, labels_b) == pytest.approx(0.0)
+        assert v_measure_score(labels_a, labels_b) == pytest.approx(0.0)
+        assert adjusted_mutual_info_score(labels_a, labels_b) == 0.0
+        assert normalized_mutual_info_score(labels_a, labels_b) == pytest.approx(0.0)
         for method in ["min", "geometric", "arithmetic", "max"]:
-            assert adjusted_mutual_info_score(
-                labels_a, labels_b,  method) == pytest.approx(0.0)
+            assert (
+                adjusted_mutual_info_score(labels_a, labels_b, average_method=method)
+                == 0.0
+            )
             assert normalized_mutual_info_score(
-                labels_a, labels_b, method) == pytest.approx(0.0)
+                labels_a, labels_b, average_method=method
+            ) == pytest.approx(0.0)
 
 
 def test_v_measure_and_mutual_information(seed=36):
     # Check relation between v_measure, entropy and mutual information
     for i in np.logspace(1, 4, 4).astype(int):
         random_state = np.random.RandomState(seed)
-        labels_a, labels_b = (random_state.randint(0, 10, i),
-                              random_state.randint(0, 10, i))
-        assert_almost_equal(v_measure_score(labels_a, labels_b),
-                            2.0 * mutual_info_score(labels_a, labels_b) /
-                            (entropy(labels_a) + entropy(labels_b)), 0)
-        avg = 'arithmetic'
-        assert_almost_equal(v_measure_score(labels_a, labels_b),
-                            normalized_mutual_info_score(labels_a, labels_b,
-                                                         average_method=avg)
-                            )
+        labels_a, labels_b = (
+            random_state.randint(0, 10, i),
+            random_state.randint(0, 10, i),
+        )
+        assert_almost_equal(
+            v_measure_score(labels_a, labels_b),
+            2.0
+            * mutual_info_score(labels_a, labels_b)
+            / (entropy(labels_a) + entropy(labels_b)),
+            0,
+        )
+        avg = "arithmetic"
+        assert_almost_equal(
+            v_measure_score(labels_a, labels_b),
+            normalized_mutual_info_score(labels_a, labels_b, average_method=avg),
+        )
 
 
 def test_fowlkes_mallows_score():
     # General case
-    score = fowlkes_mallows_score([0, 0, 0, 1, 1, 1],
-                                  [0, 0, 1, 1, 2, 2])
-    assert_almost_equal(score, 4. / np.sqrt(12. * 6.))
+    score = fowlkes_mallows_score([0, 0, 0, 1, 1, 1], [0, 0, 1, 1, 2, 2])
+    assert_almost_equal(score, 4.0 / np.sqrt(12.0 * 6.0))
 
     # Perfect match but where the label names changed
-    perfect_score = fowlkes_mallows_score([0, 0, 0, 1, 1, 1],
-                                          [1, 1, 1, 0, 0, 0])
-    assert_almost_equal(perfect_score, 1.)
+    perfect_score = fowlkes_mallows_score([0, 0, 0, 1, 1, 1], [1, 1, 1, 0, 0, 0])
+    assert_almost_equal(perfect_score, 1.0)
 
     # Worst case
-    worst_score = fowlkes_mallows_score([0, 0, 0, 0, 0, 0],
-                                        [0, 1, 2, 3, 4, 5])
-    assert_almost_equal(worst_score, 0.)
+    worst_score = fowlkes_mallows_score([0, 0, 0, 0, 0, 0], [0, 1, 2, 3, 4, 5])
+    assert_almost_equal(worst_score, 0.0)
 
 
 def test_fowlkes_mallows_score_properties():
     # handcrafted example
     labels_a = np.array([0, 0, 0, 1, 1, 2])
     labels_b = np.array([1, 1, 2, 2, 0, 0])
-    expected = 1. / np.sqrt((1. + 3.) * (1. + 2.))
+    expected = 1.0 / np.sqrt((1.0 + 3.0) * (1.0 + 2.0))
     # FMI = TP / sqrt((TP + FP) * (TP + FN))
 
     score_original = fowlkes_mallows_score(labels_a, labels_b)
@@ -357,15 +386,20 @@ def test_fowlkes_mallows_score_properties():
     assert_almost_equal(score_both, expected)
 
 
-@pytest.mark.parametrize('labels_true, labels_pred', [
-    (['a'] * 6, [1, 1, 0, 0, 1, 1]),
-    ([1] * 6, [1, 1, 0, 0, 1, 1]),
-    ([1, 1, 0, 0, 1, 1], ['a'] * 6),
-    ([1, 1, 0, 0, 1, 1], [1] * 6),
-])
+@pytest.mark.parametrize(
+    "labels_true, labels_pred",
+    [
+        (["a"] * 6, [1, 1, 0, 0, 1, 1]),
+        ([1] * 6, [1, 1, 0, 0, 1, 1]),
+        ([1, 1, 0, 0, 1, 1], ["a"] * 6),
+        ([1, 1, 0, 0, 1, 1], [1] * 6),
+        (["a"] * 6, ["a"] * 6),
+    ],
+)
 def test_mutual_info_score_positive_constant_label(labels_true, labels_pred):
+    # Check that MI = 0 when one or both labelling are constant
     # non-regression test for #16355
-    assert mutual_info_score(labels_true, labels_pred) >= 0
+    assert mutual_info_score(labels_true, labels_pred) == 0
 
 
 def test_check_clustering_error():
@@ -373,9 +407,11 @@ def test_check_clustering_error():
     rng = np.random.RandomState(42)
     noise = rng.rand(500)
     wavelength = np.linspace(0.01, 1, 500) * 1e-6
-    msg = 'Clustering metrics expects discrete values but received ' \
-          'continuous values for label, and continuous values for ' \
-          'target'
+    msg = (
+        "Clustering metrics expects discrete values but received "
+        "continuous values for label, and continuous values for "
+        "target"
+    )
 
     with pytest.warns(UserWarning, match=msg):
         check_clusterings(wavelength, noise)
@@ -387,9 +423,7 @@ def test_pair_confusion_matrix_fully_dispersed():
     clustering1 = list(range(N))
     clustering2 = clustering1
     expected = np.array([[N * (N - 1), 0], [0, 0]])
-    assert_array_equal(
-        pair_confusion_matrix(clustering1, clustering2), expected
-    )
+    assert_array_equal(pair_confusion_matrix(clustering1, clustering2), expected)
 
 
 def test_pair_confusion_matrix_single_cluster():
@@ -398,15 +432,13 @@ def test_pair_confusion_matrix_single_cluster():
     clustering1 = np.zeros((N,))
     clustering2 = clustering1
     expected = np.array([[0, 0], [0, N * (N - 1)]])
-    assert_array_equal(
-        pair_confusion_matrix(clustering1, clustering2), expected
-    )
+    assert_array_equal(pair_confusion_matrix(clustering1, clustering2), expected)
 
 
 def test_pair_confusion_matrix():
     # regular case: different non-trivial clusterings
     n = 10
-    N = n ** 2
+    N = n**2
     clustering1 = np.hstack([[i + 1] * n for i in range(n)])
     clustering2 = np.hstack([[i + 1] * (n + 1) for i in range(n)])[:N]
     # basic quadratic implementation
@@ -417,20 +449,17 @@ def test_pair_confusion_matrix():
                 same_cluster_1 = int(clustering1[i] == clustering1[j])
                 same_cluster_2 = int(clustering2[i] == clustering2[j])
                 expected[same_cluster_1, same_cluster_2] += 1
-    assert_array_equal(
-        pair_confusion_matrix(clustering1, clustering2), expected
-    )
+    assert_array_equal(pair_confusion_matrix(clustering1, clustering2), expected)
 
 
 @pytest.mark.parametrize(
     "clustering1, clustering2",
-    [(list(range(100)), list(range(100))),
-     (np.zeros((100,)), np.zeros((100,)))]
+    [(list(range(100)), list(range(100))), (np.zeros((100,)), np.zeros((100,)))],
 )
 def test_rand_score_edge_cases(clustering1, clustering2):
     # edge case 1: every element is its own cluster
     # edge case 2: only one cluster
-    assert_allclose(rand_score(clustering1, clustering2), 1.)
+    assert_allclose(rand_score(clustering1, clustering2), 1.0)
 
 
 def test_rand_score():
@@ -447,3 +476,47 @@ def test_rand_score():
     expected_denominator = D00 + D01 + D10 + D11
     expected = expected_numerator / expected_denominator
     assert_allclose(rand_score(clustering1, clustering2), expected)
+
+
+def test_adjusted_rand_score_overflow():
+    """Check that large amount of data will not lead to overflow in
+    `adjusted_rand_score`.
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/20305
+    """
+    rng = np.random.RandomState(0)
+    y_true = rng.randint(0, 2, 100_000, dtype=np.int8)
+    y_pred = rng.randint(0, 2, 100_000, dtype=np.int8)
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", RuntimeWarning)
+        adjusted_rand_score(y_true, y_pred)
+
+
+@pytest.mark.parametrize("average_method", ["min", "arithmetic", "geometric", "max"])
+def test_normalized_mutual_info_score_bounded(average_method):
+    """Check that nmi returns a score between 0 (included) and 1 (excluded
+    for non-perfect match)
+
+    Non-regression test for issue #13836
+    """
+    labels1 = [0] * 469
+    labels2 = [1] + labels1[1:]
+    labels3 = [0, 1] + labels1[2:]
+
+    # labels1 is constant. The mutual info between labels1 and any other labelling is 0.
+    nmi = normalized_mutual_info_score(labels1, labels2, average_method=average_method)
+    assert nmi == 0
+
+    # non constant, non perfect matching labels
+    nmi = normalized_mutual_info_score(labels2, labels3, average_method=average_method)
+    assert 0 <= nmi < 1
+
+
+# TODO(1.9): remove
+@pytest.mark.parametrize("sparse", [True, False])
+def test_fowlkes_mallows_sparse_deprecated(sparse):
+    """Check deprecation warning for 'sparse' parameter of fowlkes_mallows_score."""
+    with pytest.warns(
+        FutureWarning, match="The 'sparse' parameter was deprecated in 1.7"
+    ):
+        fowlkes_mallows_score([0, 1], [1, 1], sparse=sparse)
diff --git a/sklearn/metrics/cluster/tests/test_unsupervised.py b/sklearn/metrics/cluster/tests/test_unsupervised.py
index 354b6c94a7548..a0420bbd406ec 100644
--- a/sklearn/metrics/cluster/tests/test_unsupervised.py
+++ b/sklearn/metrics/cluster/tests/test_unsupervised.py
@@ -1,57 +1,52 @@
+import warnings
+
 import numpy as np
-import scipy.sparse as sp
 import pytest
-from scipy.sparse import csr_matrix
+from numpy.testing import assert_allclose
+from scipy.sparse import issparse
 
 from sklearn import datasets
-from sklearn.utils._testing import assert_array_equal
-from sklearn.metrics.cluster import silhouette_score
-from sklearn.metrics.cluster import silhouette_samples
 from sklearn.metrics import pairwise_distances
-from sklearn.metrics.cluster import calinski_harabasz_score
-from sklearn.metrics.cluster import davies_bouldin_score
-
-
-def test_silhouette():
+from sklearn.metrics.cluster import (
+    calinski_harabasz_score,
+    davies_bouldin_score,
+    silhouette_samples,
+    silhouette_score,
+)
+from sklearn.metrics.cluster._unsupervised import _silhouette_reduce
+from sklearn.utils._testing import assert_array_equal
+from sklearn.utils.fixes import (
+    CSC_CONTAINERS,
+    CSR_CONTAINERS,
+    DOK_CONTAINERS,
+    LIL_CONTAINERS,
+)
+
+
+@pytest.mark.parametrize(
+    "sparse_container",
+    [None] + CSR_CONTAINERS + CSC_CONTAINERS + DOK_CONTAINERS + LIL_CONTAINERS,
+)
+@pytest.mark.parametrize("sample_size", [None, "half"])
+def test_silhouette(sparse_container, sample_size):
     # Tests the Silhouette Coefficient.
     dataset = datasets.load_iris()
-    X_dense = dataset.data
-    X_csr = csr_matrix(X_dense)
-    X_dok = sp.dok_matrix(X_dense)
-    X_lil = sp.lil_matrix(X_dense)
-    y = dataset.target
-
-    for X in [X_dense, X_csr, X_dok, X_lil]:
-        D = pairwise_distances(X, metric='euclidean')
-        # Given that the actual labels are used, we can assume that S would be
-        # positive.
-        score_precomputed = silhouette_score(D, y, metric='precomputed')
-        assert score_precomputed > 0
-        # Test without calculating D
-        score_euclidean = silhouette_score(X, y, metric='euclidean')
-        pytest.approx(score_precomputed, score_euclidean)
-
-        if X is X_dense:
-            score_dense_without_sampling = score_precomputed
-        else:
-            pytest.approx(score_euclidean,
-                          score_dense_without_sampling)
-
-        # Test with sampling
-        score_precomputed = silhouette_score(D, y, metric='precomputed',
-                                             sample_size=int(X.shape[0] / 2),
-                                             random_state=0)
-        score_euclidean = silhouette_score(X, y, metric='euclidean',
-                                           sample_size=int(X.shape[0] / 2),
-                                           random_state=0)
-        assert score_precomputed > 0
-        assert score_euclidean > 0
-        pytest.approx(score_euclidean, score_precomputed)
-
-        if X is X_dense:
-            score_dense_with_sampling = score_precomputed
-        else:
-            pytest.approx(score_euclidean, score_dense_with_sampling)
+    X, y = dataset.data, dataset.target
+    if sparse_container is not None:
+        X = sparse_container(X)
+    sample_size = int(X.shape[0] / 2) if sample_size == "half" else sample_size
+
+    D = pairwise_distances(X, metric="euclidean")
+    # Given that the actual labels are used, we can assume that S would be positive.
+    score_precomputed = silhouette_score(
+        D, y, metric="precomputed", sample_size=sample_size, random_state=0
+    )
+    score_euclidean = silhouette_score(
+        X, y, metric="euclidean", sample_size=sample_size, random_state=0
+    )
+    assert score_precomputed > 0
+    assert score_euclidean > 0
+    assert score_precomputed == pytest.approx(score_euclidean)
 
 
 def test_cluster_size_1():
@@ -60,7 +55,7 @@ def test_cluster_size_1():
     # as the only members of a cluster (cluster 2). To our knowledge, this case
     # is not discussed in reference material, and we choose for it a sample
     # score of 1.
-    X = [[0.], [1.], [1.], [2.], [3.], [3.]]
+    X = [[0.0], [1.0], [1.0], [2.0], [3.0], [3.0]]
     labels = np.array([0, 1, 1, 1, 2, 2])
 
     # Cluster 0: 1 sample -> score of 0 by Rousseeuw's convention
@@ -74,56 +69,149 @@ def test_cluster_size_1():
     silhouette = silhouette_score(X, labels)
     assert not np.isnan(silhouette)
     ss = silhouette_samples(X, labels)
-    assert_array_equal(ss, [0, .5, .5, 0, 1, 1])
+    assert_array_equal(ss, [0, 0.5, 0.5, 0, 1, 1])
 
 
 def test_silhouette_paper_example():
     # Explicitly check per-sample results against Rousseeuw (1987)
     # Data from Table 1
-    lower = [5.58,
-             7.00, 6.50,
-             7.08, 7.00, 3.83,
-             4.83, 5.08, 8.17, 5.83,
-             2.17, 5.75, 6.67, 6.92, 4.92,
-             6.42, 5.00, 5.58, 6.00, 4.67, 6.42,
-             3.42, 5.50, 6.42, 6.42, 5.00, 3.92, 6.17,
-             2.50, 4.92, 6.25, 7.33, 4.50, 2.25, 6.33, 2.75,
-             6.08, 6.67, 4.25, 2.67, 6.00, 6.17, 6.17, 6.92, 6.17,
-             5.25, 6.83, 4.50, 3.75, 5.75, 5.42, 6.08, 5.83, 6.67, 3.67,
-             4.75, 3.00, 6.08, 6.67, 5.00, 5.58, 4.83, 6.17, 5.67, 6.50, 6.92]
+    lower = [
+        5.58,
+        7.00,
+        6.50,
+        7.08,
+        7.00,
+        3.83,
+        4.83,
+        5.08,
+        8.17,
+        5.83,
+        2.17,
+        5.75,
+        6.67,
+        6.92,
+        4.92,
+        6.42,
+        5.00,
+        5.58,
+        6.00,
+        4.67,
+        6.42,
+        3.42,
+        5.50,
+        6.42,
+        6.42,
+        5.00,
+        3.92,
+        6.17,
+        2.50,
+        4.92,
+        6.25,
+        7.33,
+        4.50,
+        2.25,
+        6.33,
+        2.75,
+        6.08,
+        6.67,
+        4.25,
+        2.67,
+        6.00,
+        6.17,
+        6.17,
+        6.92,
+        6.17,
+        5.25,
+        6.83,
+        4.50,
+        3.75,
+        5.75,
+        5.42,
+        6.08,
+        5.83,
+        6.67,
+        3.67,
+        4.75,
+        3.00,
+        6.08,
+        6.67,
+        5.00,
+        5.58,
+        4.83,
+        6.17,
+        5.67,
+        6.50,
+        6.92,
+    ]
     D = np.zeros((12, 12))
     D[np.tril_indices(12, -1)] = lower
     D += D.T
 
-    names = ['BEL', 'BRA', 'CHI', 'CUB', 'EGY', 'FRA', 'IND', 'ISR', 'USA',
-             'USS', 'YUG', 'ZAI']
+    names = [
+        "BEL",
+        "BRA",
+        "CHI",
+        "CUB",
+        "EGY",
+        "FRA",
+        "IND",
+        "ISR",
+        "USA",
+        "USS",
+        "YUG",
+        "ZAI",
+    ]
 
     # Data from Figure 2
     labels1 = [1, 1, 2, 2, 1, 1, 2, 1, 1, 2, 2, 1]
-    expected1 = {'USA': .43, 'BEL': .39, 'FRA': .35, 'ISR': .30, 'BRA': .22,
-                 'EGY': .20, 'ZAI': .19, 'CUB': .40, 'USS': .34, 'CHI': .33,
-                 'YUG': .26, 'IND': -.04}
-    score1 = .28
+    expected1 = {
+        "USA": 0.43,
+        "BEL": 0.39,
+        "FRA": 0.35,
+        "ISR": 0.30,
+        "BRA": 0.22,
+        "EGY": 0.20,
+        "ZAI": 0.19,
+        "CUB": 0.40,
+        "USS": 0.34,
+        "CHI": 0.33,
+        "YUG": 0.26,
+        "IND": -0.04,
+    }
+    score1 = 0.28
 
     # Data from Figure 3
     labels2 = [1, 2, 3, 3, 1, 1, 2, 1, 1, 3, 3, 2]
-    expected2 = {'USA': .47, 'FRA': .44, 'BEL': .42, 'ISR': .37, 'EGY': .02,
-                 'ZAI': .28, 'BRA': .25, 'IND': .17, 'CUB': .48, 'USS': .44,
-                 'YUG': .31, 'CHI': .31}
-    score2 = .33
-
-    for labels, expected, score in [(labels1, expected1, score1),
-                                    (labels2, expected2, score2)]:
+    expected2 = {
+        "USA": 0.47,
+        "FRA": 0.44,
+        "BEL": 0.42,
+        "ISR": 0.37,
+        "EGY": 0.02,
+        "ZAI": 0.28,
+        "BRA": 0.25,
+        "IND": 0.17,
+        "CUB": 0.48,
+        "USS": 0.44,
+        "YUG": 0.31,
+        "CHI": 0.31,
+    }
+    score2 = 0.33
+
+    for labels, expected, score in [
+        (labels1, expected1, score1),
+        (labels2, expected2, score2),
+    ]:
         expected = [expected[name] for name in names]
         # we check to 2dp because that's what's in the paper
-        pytest.approx(expected,
-                      silhouette_samples(D, np.array(labels),
-                                         metric='precomputed'),
-                      abs=1e-2)
-        pytest.approx(score,
-                      silhouette_score(D, np.array(labels),
-                                       metric='precomputed'),
-                      abs=1e-2)
+        pytest.approx(
+            expected,
+            silhouette_samples(D, np.array(labels), metric="precomputed"),
+            abs=1e-2,
+        )
+        pytest.approx(
+            score, silhouette_score(D, np.array(labels), metric="precomputed"), abs=1e-2
+        )
 
 
 def test_correct_labelsize():
@@ -133,15 +221,19 @@ def test_correct_labelsize():
 
     # n_labels = n_samples
     y = np.arange(X.shape[0])
-    err_msg = (r'Number of labels is %d\. Valid values are 2 '
-               r'to n_samples - 1 \(inclusive\)' % len(np.unique(y)))
+    err_msg = (
+        r"Number of labels is %d\. Valid values are 2 "
+        r"to n_samples - 1 \(inclusive\)" % len(np.unique(y))
+    )
     with pytest.raises(ValueError, match=err_msg):
         silhouette_score(X, y)
 
     # n_labels = 1
     y = np.zeros(X.shape[0])
-    err_msg = (r'Number of labels is %d\. Valid values are 2 '
-               r'to n_samples - 1 \(inclusive\)' % len(np.unique(y)))
+    err_msg = (
+        r"Number of labels is %d\. Valid values are 2 "
+        r"to n_samples - 1 \(inclusive\)" % len(np.unique(y))
+    )
     with pytest.raises(ValueError, match=err_msg):
         silhouette_score(X, y)
 
@@ -150,38 +242,87 @@ def test_non_encoded_labels():
     dataset = datasets.load_iris()
     X = dataset.data
     labels = dataset.target
-    assert (
-        silhouette_score(X, labels * 2 + 10) == silhouette_score(X, labels))
+    assert silhouette_score(X, labels * 2 + 10) == silhouette_score(X, labels)
     assert_array_equal(
-        silhouette_samples(X, labels * 2 + 10), silhouette_samples(X, labels))
+        silhouette_samples(X, labels * 2 + 10), silhouette_samples(X, labels)
+    )
 
 
 def test_non_numpy_labels():
     dataset = datasets.load_iris()
     X = dataset.data
     y = dataset.target
-    assert (
-        silhouette_score(list(X), list(y)) == silhouette_score(X, y))
+    assert silhouette_score(list(X), list(y)) == silhouette_score(X, y)
 
 
-@pytest.mark.parametrize('dtype', (np.float32, np.float64))
+@pytest.mark.parametrize("dtype", (np.float32, np.float64))
 def test_silhouette_nonzero_diag(dtype):
     # Make sure silhouette_samples requires diagonal to be zero.
     # Non-regression test for #12178
 
     # Construct a zero-diagonal matrix
     dists = pairwise_distances(
-        np.array([[0.2, 0.1, 0.12, 1.34, 1.11, 1.6]], dtype=dtype).T)
+        np.array([[0.2, 0.1, 0.12, 1.34, 1.11, 1.6]], dtype=dtype).T
+    )
     labels = [0, 0, 0, 1, 1, 1]
 
     # small values on the diagonal are OK
     dists[2][2] = np.finfo(dists.dtype).eps * 10
-    silhouette_samples(dists, labels, metric='precomputed')
+    silhouette_samples(dists, labels, metric="precomputed")
 
     # values bigger than eps * 100 are not
     dists[2][2] = np.finfo(dists.dtype).eps * 1000
-    with pytest.raises(ValueError, match='contains non-zero'):
-        silhouette_samples(dists, labels, metric='precomputed')
+    with pytest.raises(ValueError, match="contains non-zero"):
+        silhouette_samples(dists, labels, metric="precomputed")
+
+
+@pytest.mark.parametrize(
+    "sparse_container",
+    CSC_CONTAINERS + CSR_CONTAINERS + DOK_CONTAINERS + LIL_CONTAINERS,
+)
+def test_silhouette_samples_precomputed_sparse(sparse_container):
+    """Check that silhouette_samples works for sparse matrices correctly."""
+    X = np.array([[0.2, 0.1, 0.1, 0.2, 0.1, 1.6, 0.2, 0.1]], dtype=np.float32).T
+    y = [0, 0, 0, 0, 1, 1, 1, 1]
+    pdist_dense = pairwise_distances(X)
+    pdist_sparse = sparse_container(pdist_dense)
+    assert issparse(pdist_sparse)
+    output_with_sparse_input = silhouette_samples(pdist_sparse, y, metric="precomputed")
+    output_with_dense_input = silhouette_samples(pdist_dense, y, metric="precomputed")
+    assert_allclose(output_with_sparse_input, output_with_dense_input)
+
+
+@pytest.mark.parametrize(
+    "sparse_container",
+    CSC_CONTAINERS + CSR_CONTAINERS + DOK_CONTAINERS + LIL_CONTAINERS,
+)
+def test_silhouette_samples_euclidean_sparse(sparse_container):
+    """Check that silhouette_samples works for sparse matrices correctly."""
+    X = np.array([[0.2, 0.1, 0.1, 0.2, 0.1, 1.6, 0.2, 0.1]], dtype=np.float32).T
+    y = [0, 0, 0, 0, 1, 1, 1, 1]
+    pdist_dense = pairwise_distances(X)
+    pdist_sparse = sparse_container(pdist_dense)
+    assert issparse(pdist_sparse)
+    output_with_sparse_input = silhouette_samples(pdist_sparse, y)
+    output_with_dense_input = silhouette_samples(pdist_dense, y)
+    assert_allclose(output_with_sparse_input, output_with_dense_input)
+
+
+@pytest.mark.parametrize(
+    "sparse_container", CSC_CONTAINERS + DOK_CONTAINERS + LIL_CONTAINERS
+)
+def test_silhouette_reduce(sparse_container):
+    """Check for non-CSR input to private method `_silhouette_reduce`."""
+    X = np.array([[0.2, 0.1, 0.1, 0.2, 0.1, 1.6, 0.2, 0.1]], dtype=np.float32).T
+    pdist_dense = pairwise_distances(X)
+    pdist_sparse = sparse_container(pdist_dense)
+    y = [0, 0, 0, 0, 1, 1, 1, 1]
+    label_freqs = np.bincount(y)
+    with pytest.raises(
+        TypeError,
+        match="Expected CSR matrix. Please pass sparse matrix in CSR format.",
+    ):
+        _silhouette_reduce(pdist_sparse, start=0, labels=y, label_freqs=label_freqs)
 
 
 def assert_raises_on_only_one_label(func):
@@ -204,19 +345,20 @@ def test_calinski_harabasz_score():
     assert_raises_on_all_points_same_cluster(calinski_harabasz_score)
 
     # Assert the value is 1. when all samples are equals
-    assert 1. == calinski_harabasz_score(np.ones((10, 2)),
-                                         [0] * 5 + [1] * 5)
+    assert 1.0 == calinski_harabasz_score(np.ones((10, 2)), [0] * 5 + [1] * 5)
 
     # Assert the value is 0. when all the mean cluster are equal
-    assert 0. == calinski_harabasz_score([[-1, -1], [1, 1]] * 10,
-                                         [0] * 10 + [1] * 10)
+    assert 0.0 == calinski_harabasz_score([[-1, -1], [1, 1]] * 10, [0] * 10 + [1] * 10)
 
     # General case (with non numpy arrays)
-    X = ([[0, 0], [1, 1]] * 5 + [[3, 3], [4, 4]] * 5 +
-         [[0, 4], [1, 3]] * 5 + [[3, 1], [4, 0]] * 5)
+    X = (
+        [[0, 0], [1, 1]] * 5
+        + [[3, 3], [4, 4]] * 5
+        + [[0, 4], [1, 3]] * 5
+        + [[3, 1], [4, 0]] * 5
+    )
     labels = [0] * 10 + [1] * 10 + [2] * 10 + [3] * 10
-    pytest.approx(calinski_harabasz_score(X, labels),
-                  45 * (40 - 4) / (5 * (4 - 1)))
+    pytest.approx(calinski_harabasz_score(X, labels), 45 * (40 - 4) / (5 * (4 - 1)))
 
 
 def test_davies_bouldin_score():
@@ -224,29 +366,48 @@ def test_davies_bouldin_score():
     assert_raises_on_all_points_same_cluster(davies_bouldin_score)
 
     # Assert the value is 0. when all samples are equals
-    assert davies_bouldin_score(np.ones((10, 2)),
-                                [0] * 5 + [1] * 5) == pytest.approx(0.0)
+    assert davies_bouldin_score(np.ones((10, 2)), [0] * 5 + [1] * 5) == pytest.approx(
+        0.0
+    )
 
     # Assert the value is 0. when all the mean cluster are equal
-    assert davies_bouldin_score([[-1, -1], [1, 1]] * 10,
-                                [0] * 10 + [1] * 10) == pytest.approx(0.0)
+    assert davies_bouldin_score(
+        [[-1, -1], [1, 1]] * 10, [0] * 10 + [1] * 10
+    ) == pytest.approx(0.0)
 
     # General case (with non numpy arrays)
-    X = ([[0, 0], [1, 1]] * 5 + [[3, 3], [4, 4]] * 5 +
-         [[0, 4], [1, 3]] * 5 + [[3, 1], [4, 0]] * 5)
+    X = (
+        [[0, 0], [1, 1]] * 5
+        + [[3, 3], [4, 4]] * 5
+        + [[0, 4], [1, 3]] * 5
+        + [[3, 1], [4, 0]] * 5
+    )
     labels = [0] * 10 + [1] * 10 + [2] * 10 + [3] * 10
     pytest.approx(davies_bouldin_score(X, labels), 2 * np.sqrt(0.5) / 3)
 
     # Ensure divide by zero warning is not raised in general case
-    with pytest.warns(None) as record:
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", RuntimeWarning)
         davies_bouldin_score(X, labels)
-    div_zero_warnings = [
-        warning for warning in record
-        if "divide by zero encountered" in warning.message.args[0]
-    ]
-    assert len(div_zero_warnings) == 0
 
     # General case - cluster have one sample
-    X = ([[0, 0], [2, 2], [3, 3], [5, 5]])
+    X = [[0, 0], [2, 2], [3, 3], [5, 5]]
     labels = [0, 0, 1, 2]
-    pytest.approx(davies_bouldin_score(X, labels), (5. / 4) / 3)
+    pytest.approx(davies_bouldin_score(X, labels), (5.0 / 4) / 3)
+
+
+def test_silhouette_score_integer_precomputed():
+    """Check that silhouette_score works for precomputed metrics that are integers.
+
+    Non-regression test for #22107.
+    """
+    result = silhouette_score(
+        [[0, 1, 2], [1, 0, 1], [2, 1, 0]], [0, 0, 1], metric="precomputed"
+    )
+    assert result == pytest.approx(1 / 6)
+
+    # non-zero on diagonal for ints raises an error
+    with pytest.raises(ValueError, match="contains non-zero"):
+        silhouette_score(
+            [[1, 1, 2], [1, 0, 1], [2, 1, 0]], [0, 0, 1], metric="precomputed"
+        )
diff --git a/sklearn/metrics/meson.build b/sklearn/metrics/meson.build
new file mode 100644
index 0000000000000..f0f9894cc6f59
--- /dev/null
+++ b/sklearn/metrics/meson.build
@@ -0,0 +1,49 @@
+# Metrics is cimported from other subpackages so this is needed for the cimport
+# to work
+metrics_cython_tree = [
+  fs.copyfile('__init__.py')
+]
+# Some metrics code cimports code from utils, we may as well copy all the necessary files
+metrics_cython_tree += utils_cython_tree
+
+_dist_metrics_pxd = custom_target(
+  '_dist_metrics_pxd',
+  output: '_dist_metrics.pxd',
+  input: '_dist_metrics.pxd.tp',
+  command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
+  # Need to install the generated pxd because it is needed in other subpackages
+  # Cython code, e.g. sklearn.cluster
+  install_dir: sklearn_dir / 'metrics',
+  install: true,
+)
+metrics_cython_tree += [_dist_metrics_pxd]
+
+_dist_metrics_pyx = custom_target(
+  '_dist_metrics_pyx',
+  output: '_dist_metrics.pyx',
+  input: '_dist_metrics.pyx.tp',
+  command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
+  # TODO in principle this should go in py.exension_module below. This is
+  # temporary work-around for dependency issue with .pyx.tp files. For more
+  # details, see https://github.com/mesonbuild/meson/issues/13212
+  depends: metrics_cython_tree,
+)
+
+_dist_metrics = py.extension_module(
+  '_dist_metrics',
+  cython_gen.process(_dist_metrics_pyx),
+  dependencies: [np_dep],
+  subdir: 'sklearn/metrics',
+  install: true
+)
+
+py.extension_module(
+  '_pairwise_fast',
+  [cython_gen.process('_pairwise_fast.pyx'), metrics_cython_tree],
+  dependencies: [openmp_dep],
+  subdir: 'sklearn/metrics',
+  install: true
+)
+
+subdir('_pairwise_distances_reduction')
+subdir('cluster')
diff --git a/sklearn/metrics/pairwise.py b/sklearn/metrics/pairwise.py
index 45eb256d59f67..f0e6cee65bc28 100644
--- a/sklearn/metrics/pairwise.py
+++ b/sklearn/metrics/pairwise.py
@@ -1,39 +1,50 @@
-# -*- coding: utf-8 -*-
+"""Metrics for pairwise distances and affinity of sets of samples."""
 
-# Authors: Alexandre Gramfort 
-#          Mathieu Blondel 
-#          Robert Layton 
-#          Andreas Mueller 
-#          Philippe Gervais 
-#          Lars Buitinck
-#          Joel Nothman 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import itertools
-from functools import partial
+import math
 import warnings
+from functools import partial
+from numbers import Integral, Real
 
 import numpy as np
+from joblib import effective_n_jobs
+from scipy.sparse import csr_matrix, issparse
 from scipy.spatial import distance
-from scipy.sparse import csr_matrix
-from scipy.sparse import issparse
-from joblib import Parallel, effective_n_jobs
-
-from ..utils.validation import _num_samples
-from ..utils.validation import check_non_negative
-from ..utils import check_array
-from ..utils import gen_even_slices
-from ..utils import gen_batches, get_chunk_n_rows
-from ..utils import is_scalar_nan
-from ..utils.extmath import row_norms, safe_sparse_dot
+
+from .. import config_context
+from ..exceptions import DataConversionWarning
 from ..preprocessing import normalize
+from ..utils import check_array, gen_batches, gen_even_slices
+from ..utils._array_api import (
+    _fill_or_add_to_diagonal,
+    _find_matching_floating_dtype,
+    _is_numpy_namespace,
+    _max_precision_float_dtype,
+    _modify_in_place_if_numpy,
+    get_namespace,
+    get_namespace_and_device,
+)
+from ..utils._chunking import get_chunk_n_rows
 from ..utils._mask import _get_mask
-from ..utils.validation import _deprecate_positional_args
-from ..utils.fixes import delayed
-from ..utils.fixes import sp_version, parse_version
-
+from ..utils._missing import is_scalar_nan
+from ..utils._param_validation import (
+    Hidden,
+    Interval,
+    MissingValues,
+    Options,
+    StrOptions,
+    validate_params,
+)
+from ..utils.deprecation import _deprecate_force_all_finite
+from ..utils.extmath import row_norms, safe_sparse_dot
+from ..utils.fixes import parse_version, sp_base_version
+from ..utils.parallel import Parallel, delayed
+from ..utils.validation import _num_samples, check_non_negative
+from ._pairwise_distances_reduction import ArgKmin
 from ._pairwise_fast import _chi2_kernel_fast, _sparse_manhattan
-from ..exceptions import DataConversionWarning
 
 
 # Utility Functions
@@ -61,10 +72,18 @@ def _return_float_dtype(X, Y):
     return X, Y, dtype
 
 
-@_deprecate_positional_args
-def check_pairwise_arrays(X, Y, *, precomputed=False, dtype=None,
-                          accept_sparse='csr', force_all_finite=True,
-                          copy=False):
+def check_pairwise_arrays(
+    X,
+    Y,
+    *,
+    precomputed=False,
+    dtype="infer_float",
+    accept_sparse="csr",
+    force_all_finite="deprecated",
+    ensure_all_finite=None,
+    ensure_2d=True,
+    copy=False,
+):
     """Set X and Y appropriately and checks inputs.
 
     If Y is None, it is set as a pointer to X (i.e. not a copy).
@@ -88,9 +107,10 @@ def check_pairwise_arrays(X, Y, *, precomputed=False, dtype=None,
         True if X is to be treated as precomputed distances to the samples in
         Y.
 
-    dtype : str, type, list of type, default=None
-        Data type required for X and Y. If None, the dtype will be an
-        appropriate float type selected by _return_float_dtype.
+    dtype : str, type, list of type or None default="infer_float"
+        Data type required for X and Y. If "infer_float", the dtype will be an
+        appropriate float type selected by _return_float_dtype. If None, the
+        dtype of the input is preserved.
 
         .. versionadded:: 0.18
 
@@ -116,6 +136,29 @@ def check_pairwise_arrays(X, Y, *, precomputed=False, dtype=None,
         .. versionchanged:: 0.23
            Accepts `pd.NA` and converts it into `np.nan`.
 
+        .. deprecated:: 1.6
+           `force_all_finite` was renamed to `ensure_all_finite` and will be removed
+           in 1.8.
+
+    ensure_all_finite : bool or 'allow-nan', default=True
+        Whether to raise an error on np.inf, np.nan, pd.NA in array. The
+        possibilities are:
+
+        - True: Force all values of array to be finite.
+        - False: accepts np.inf, np.nan, pd.NA in array.
+        - 'allow-nan': accepts only np.nan and pd.NA values in array. Values
+          cannot be infinite.
+
+        .. versionadded:: 1.6
+           `force_all_finite` was renamed to `ensure_all_finite`.
+
+    ensure_2d : bool, default=True
+        Whether to raise an error when the input arrays are not 2-dimensional. Setting
+        this to `False` is necessary when using a custom metric with certain
+        non-numerical inputs (e.g. a list of strings).
+
+        .. versionadded:: 1.5
+
     copy : bool, default=False
         Whether a forced copy will be triggered. If copy=False, a copy might
         be triggered by a conversion.
@@ -130,36 +173,63 @@ def check_pairwise_arrays(X, Y, *, precomputed=False, dtype=None,
     safe_Y : {array-like, sparse matrix} of shape (n_samples_Y, n_features)
         An array equal to Y if Y was not None, guaranteed to be a numpy array.
         If Y was None, safe_Y will be a pointer to X.
-
     """
-    X, Y, dtype_float = _return_float_dtype(X, Y)
+    ensure_all_finite = _deprecate_force_all_finite(force_all_finite, ensure_all_finite)
+
+    xp, _ = get_namespace(X, Y)
+    if any([issparse(X), issparse(Y)]) or _is_numpy_namespace(xp):
+        X, Y, dtype_float = _return_float_dtype(X, Y)
+    else:
+        dtype_float = _find_matching_floating_dtype(X, Y, xp=xp)
 
-    estimator = 'check_pairwise_arrays'
-    if dtype is None:
+    estimator = "check_pairwise_arrays"
+    if dtype == "infer_float":
         dtype = dtype_float
 
     if Y is X or Y is None:
-        X = Y = check_array(X, accept_sparse=accept_sparse, dtype=dtype,
-                            copy=copy, force_all_finite=force_all_finite,
-                            estimator=estimator)
+        X = Y = check_array(
+            X,
+            accept_sparse=accept_sparse,
+            dtype=dtype,
+            copy=copy,
+            ensure_all_finite=ensure_all_finite,
+            estimator=estimator,
+            ensure_2d=ensure_2d,
+        )
     else:
-        X = check_array(X, accept_sparse=accept_sparse, dtype=dtype,
-                        copy=copy, force_all_finite=force_all_finite,
-                        estimator=estimator)
-        Y = check_array(Y, accept_sparse=accept_sparse, dtype=dtype,
-                        copy=copy, force_all_finite=force_all_finite,
-                        estimator=estimator)
+        X = check_array(
+            X,
+            accept_sparse=accept_sparse,
+            dtype=dtype,
+            copy=copy,
+            ensure_all_finite=ensure_all_finite,
+            estimator=estimator,
+            ensure_2d=ensure_2d,
+        )
+        Y = check_array(
+            Y,
+            accept_sparse=accept_sparse,
+            dtype=dtype,
+            copy=copy,
+            ensure_all_finite=ensure_all_finite,
+            estimator=estimator,
+            ensure_2d=ensure_2d,
+        )
 
     if precomputed:
         if X.shape[1] != Y.shape[0]:
-            raise ValueError("Precomputed metric requires shape "
-                             "(n_queries, n_indexed). Got (%d, %d) "
-                             "for %d indexed." %
-                             (X.shape[0], X.shape[1], Y.shape[0]))
-    elif X.shape[1] != Y.shape[1]:
-        raise ValueError("Incompatible dimension for X and Y matrices: "
-                         "X.shape[1] == %d while Y.shape[1] == %d" % (
-                             X.shape[1], Y.shape[1]))
+            raise ValueError(
+                "Precomputed metric requires shape "
+                "(n_queries, n_indexed). Got (%d, %d) "
+                "for %d indexed." % (X.shape[0], X.shape[1], Y.shape[0])
+            )
+    elif ensure_2d and X.shape[1] != Y.shape[1]:
+        # Only check the number of features if 2d arrays are enforced. Otherwise,
+        # validation is left to the user for custom metrics.
+        raise ValueError(
+            "Incompatible dimension for X and Y matrices: "
+            "X.shape[1] == %d while Y.shape[1] == %d" % (X.shape[1], Y.shape[1])
+        )
 
     return X, Y
 
@@ -189,22 +259,32 @@ def check_paired_arrays(X, Y):
     safe_Y : {array-like, sparse matrix} of shape (n_samples_Y, n_features)
         An array equal to Y if Y was not None, guaranteed to be a numpy array.
         If Y was None, safe_Y will be a pointer to X.
-
     """
     X, Y = check_pairwise_arrays(X, Y)
     if X.shape != Y.shape:
-        raise ValueError("X and Y should be of same shape. They were "
-                         "respectively %r and %r long." % (X.shape, Y.shape))
+        raise ValueError(
+            "X and Y should be of same shape. They were respectively %r and %r long."
+            % (X.shape, Y.shape)
+        )
     return X, Y
 
 
 # Pairwise distances
-@_deprecate_positional_args
-def euclidean_distances(X, Y=None, *, Y_norm_squared=None, squared=False,
-                        X_norm_squared=None):
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "Y": ["array-like", "sparse matrix", None],
+        "Y_norm_squared": ["array-like", None],
+        "squared": ["boolean"],
+        "X_norm_squared": ["array-like", None],
+    },
+    prefer_skip_nested_validation=True,
+)
+def euclidean_distances(
+    X, Y=None, *, Y_norm_squared=None, squared=False, X_norm_squared=None
+):
     """
-    Considering the rows of X (and Y=X) as vectors, compute the
-    distance matrix between each pair of vectors.
+    Compute the distance matrix between each pair from a feature array X and Y.
 
     For efficiency reasons, the euclidean distance between a pair of row
     vector x and y is computed as::
@@ -226,9 +306,12 @@ def euclidean_distances(X, Y=None, *, Y_norm_squared=None, squared=False,
     Parameters
     ----------
     X : {array-like, sparse matrix} of shape (n_samples_X, n_features)
+        An array where each row is a sample and each column is a feature.
 
     Y : {array-like, sparse matrix} of shape (n_samples_Y, n_features), \
             default=None
+        An array where each row is a sample and each column is a feature.
+        If `None`, method uses `Y=X`.
 
     Y_norm_squared : array-like of shape (n_samples_Y,) or (n_samples_Y, 1) \
             or (1, n_samples_Y), default=None
@@ -245,18 +328,20 @@ def euclidean_distances(X, Y=None, *, Y_norm_squared=None, squared=False,
         ``(X**2).sum(axis=1)``)
         May be ignored in some cases, see the note below.
 
-    Notes
-    -----
-    To achieve better accuracy, `X_norm_squared` and `Y_norm_squared` may be
-    unused if they are passed as ``float32``.
-
     Returns
     -------
     distances : ndarray of shape (n_samples_X, n_samples_Y)
+        Returns the distances between the row vectors of `X`
+        and the row vectors of `Y`.
 
     See Also
     --------
-    paired_distances : Distances betweens pairs of elements of X and Y.
+    paired_distances : Distances between pairs of elements of X and Y.
+
+    Notes
+    -----
+    To achieve a better accuracy, `X_norm_squared` and `Y_norm_squared` may be
+    unused if they are passed as `np.float32`.
 
     Examples
     --------
@@ -271,37 +356,39 @@ def euclidean_distances(X, Y=None, *, Y_norm_squared=None, squared=False,
     array([[1.        ],
            [1.41421356]])
     """
+    xp, _ = get_namespace(X, Y)
     X, Y = check_pairwise_arrays(X, Y)
 
     if X_norm_squared is not None:
         X_norm_squared = check_array(X_norm_squared, ensure_2d=False)
         original_shape = X_norm_squared.shape
         if X_norm_squared.shape == (X.shape[0],):
-            X_norm_squared = X_norm_squared.reshape(-1, 1)
+            X_norm_squared = xp.reshape(X_norm_squared, (-1, 1))
         if X_norm_squared.shape == (1, X.shape[0]):
             X_norm_squared = X_norm_squared.T
         if X_norm_squared.shape != (X.shape[0], 1):
             raise ValueError(
                 f"Incompatible dimensions for X of shape {X.shape} and "
-                f"X_norm_squared of shape {original_shape}.")
+                f"X_norm_squared of shape {original_shape}."
+            )
 
     if Y_norm_squared is not None:
         Y_norm_squared = check_array(Y_norm_squared, ensure_2d=False)
         original_shape = Y_norm_squared.shape
         if Y_norm_squared.shape == (Y.shape[0],):
-            Y_norm_squared = Y_norm_squared.reshape(1, -1)
+            Y_norm_squared = xp.reshape(Y_norm_squared, (1, -1))
         if Y_norm_squared.shape == (Y.shape[0], 1):
             Y_norm_squared = Y_norm_squared.T
         if Y_norm_squared.shape != (1, Y.shape[0]):
             raise ValueError(
                 f"Incompatible dimensions for Y of shape {Y.shape} and "
-                f"Y_norm_squared of shape {original_shape}.")
+                f"Y_norm_squared of shape {original_shape}."
+            )
 
     return _euclidean_distances(X, Y, X_norm_squared, Y_norm_squared, squared)
 
 
-def _euclidean_distances(X, Y, X_norm_squared=None, Y_norm_squared=None,
-                         squared=False):
+def _euclidean_distances(X, Y, X_norm_squared=None, Y_norm_squared=None, squared=False):
     """Computational part of euclidean_distances
 
     Assumes inputs are already checked.
@@ -310,51 +397,64 @@ def _euclidean_distances(X, Y, X_norm_squared=None, Y_norm_squared=None,
     float32, norms needs to be recomputed on upcast chunks.
     TODO: use a float64 accumulator in row_norms to avoid the latter.
     """
-    if X_norm_squared is not None:
-        if X_norm_squared.dtype == np.float32:
-            XX = None
-        else:
-            XX = X_norm_squared.reshape(-1, 1)
-    elif X.dtype == np.float32:
-        XX = None
+    xp, _, device_ = get_namespace_and_device(X, Y)
+    if X_norm_squared is not None and X_norm_squared.dtype != xp.float32:
+        XX = xp.reshape(X_norm_squared, (-1, 1))
+    elif X.dtype != xp.float32:
+        XX = row_norms(X, squared=True)[:, None]
     else:
-        XX = row_norms(X, squared=True)[:, np.newaxis]
+        XX = None
 
     if Y is X:
         YY = None if XX is None else XX.T
     else:
-        if Y_norm_squared is not None:
-            if Y_norm_squared.dtype == np.float32:
-                YY = None
-            else:
-                YY = Y_norm_squared.reshape(1, -1)
-        elif Y.dtype == np.float32:
-            YY = None
+        if Y_norm_squared is not None and Y_norm_squared.dtype != xp.float32:
+            YY = xp.reshape(Y_norm_squared, (1, -1))
+        elif Y.dtype != xp.float32:
+            YY = row_norms(Y, squared=True)[None, :]
         else:
-            YY = row_norms(Y, squared=True)[np.newaxis, :]
+            YY = None
 
-    if X.dtype == np.float32:
+    if X.dtype == xp.float32 or Y.dtype == xp.float32:
         # To minimize precision issues with float32, we compute the distance
         # matrix on chunks of X and Y upcast to float64
         distances = _euclidean_distances_upcast(X, XX, Y, YY)
     else:
         # if dtype is already float64, no need to chunk and upcast
-        distances = - 2 * safe_sparse_dot(X, Y.T, dense_output=True)
+        distances = -2 * safe_sparse_dot(X, Y.T, dense_output=True)
         distances += XX
         distances += YY
-    np.maximum(distances, 0, out=distances)
+
+    xp_zero = xp.asarray(0, device=device_, dtype=distances.dtype)
+    distances = _modify_in_place_if_numpy(
+        xp, xp.maximum, distances, xp_zero, out=distances
+    )
 
     # Ensure that distances between vectors and themselves are set to 0.0.
     # This may not be the case due to floating point rounding errors.
     if X is Y:
-        np.fill_diagonal(distances, 0)
+        _fill_or_add_to_diagonal(distances, 0, xp=xp, add_value=False)
+
+    if squared:
+        return distances
 
-    return distances if squared else np.sqrt(distances, out=distances)
+    distances = _modify_in_place_if_numpy(xp, xp.sqrt, distances, out=distances)
+    return distances
 
 
-@_deprecate_positional_args
-def nan_euclidean_distances(X, Y=None, *, squared=False,
-                            missing_values=np.nan, copy=True):
+@validate_params(
+    {
+        "X": ["array-like"],
+        "Y": ["array-like", None],
+        "squared": ["boolean"],
+        "missing_values": [MissingValues(numeric_only=True)],
+        "copy": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def nan_euclidean_distances(
+    X, Y=None, *, squared=False, missing_values=np.nan, copy=True
+):
     """Calculate the euclidean distances in the presence of missing values.
 
     Compute the euclidean distance between each pair of samples in X and Y,
@@ -363,15 +463,20 @@ def nan_euclidean_distances(X, Y=None, *, squared=False,
     missing value in either sample and scales up the weight of the remaining
     coordinates:
 
+    .. code-block:: text
+
         dist(x,y) = sqrt(weight * sq. distance from present coordinates)
-        where,
+
+    where:
+
+    .. code-block:: text
+
         weight = Total # of coordinates / # of present coordinates
 
-    For example, the distance between ``[3, na, na, 6]`` and ``[1, na, 4, 5]``
-    is:
+    For example, the distance between ``[3, na, na, 6]`` and ``[1, na, 4, 5]`` is:
 
-        .. math::
-            \\sqrt{\\frac{4}{2}((3-1)^2 + (6-5)^2)}
+    .. math::
+        \\sqrt{\\frac{4}{2}((3-1)^2 + (6-5)^2)}
 
     If all the coordinates are missing or if there are no common present
     coordinates then NaN is returned for that pair.
@@ -382,14 +487,17 @@ def nan_euclidean_distances(X, Y=None, *, squared=False,
 
     Parameters
     ----------
-    X : array-like of shape=(n_samples_X, n_features)
+    X : array-like of shape (n_samples_X, n_features)
+        An array where each row is a sample and each column is a feature.
 
-    Y : array-like of shape=(n_samples_Y, n_features), default=None
+    Y : array-like of shape (n_samples_Y, n_features), default=None
+        An array where each row is a sample and each column is a feature.
+        If `None`, method uses `Y=X`.
 
     squared : bool, default=False
         Return squared Euclidean distances.
 
-    missing_values : np.nan or int, default=np.nan
+    missing_values : np.nan, float or int, default=np.nan
         Representation of missing value.
 
     copy : bool, default=True
@@ -398,11 +506,20 @@ def nan_euclidean_distances(X, Y=None, *, squared=False,
     Returns
     -------
     distances : ndarray of shape (n_samples_X, n_samples_Y)
+        Returns the distances between the row vectors of `X`
+        and the row vectors of `Y`.
 
     See Also
     --------
     paired_distances : Distances between pairs of elements of X and Y.
 
+    References
+    ----------
+    * John K. Dixon, "Pattern Recognition with Partly Missing Data",
+      IEEE Transactions on Systems, Man, and Cybernetics, Volume: 9, Issue:
+      10, pp. 617 - 621, Oct. 1979.
+      http://ieeexplore.ieee.org/abstract/document/4310090/
+
     Examples
     --------
     >>> from sklearn.metrics.pairwise import nan_euclidean_distances
@@ -416,18 +533,12 @@ def nan_euclidean_distances(X, Y=None, *, squared=False,
     >>> nan_euclidean_distances(X, [[0, 0]])
     array([[1.        ],
            [1.41421356]])
-
-    References
-    ----------
-    * John K. Dixon, "Pattern Recognition with Partly Missing Data",
-      IEEE Transactions on Systems, Man, and Cybernetics, Volume: 9, Issue:
-      10, pp. 617 - 621, Oct. 1979.
-      http://ieeexplore.ieee.org/abstract/document/4310090/
     """
 
-    force_all_finite = 'allow-nan' if is_scalar_nan(missing_values) else True
-    X, Y = check_pairwise_arrays(X, Y, accept_sparse=False,
-                                 force_all_finite=force_all_finite, copy=copy)
+    ensure_all_finite = "allow-nan" if is_scalar_nan(missing_values) else True
+    X, Y = check_pairwise_arrays(
+        X, Y, accept_sparse=False, ensure_all_finite=ensure_all_finite, copy=copy
+    )
     # Get missing mask for X
     missing_X = _get_mask(X, missing_values)
 
@@ -477,11 +588,12 @@ def _euclidean_distances_upcast(X, XX=None, Y=None, YY=None, batch_size=None):
     X and Y are upcast to float64 by chunks, which size is chosen to limit
     memory increase by approximately 10% (at least 10MiB).
     """
+    xp, _, device_ = get_namespace_and_device(X, Y)
     n_samples_X = X.shape[0]
     n_samples_Y = Y.shape[0]
     n_features = X.shape[1]
 
-    distances = np.empty((n_samples_X, n_samples_Y), dtype=np.float32)
+    distances = xp.empty((n_samples_X, n_samples_Y), dtype=xp.float32, device=device_)
 
     if batch_size is None:
         x_density = X.nnz / np.prod(X.shape) if issparse(X) else 1
@@ -490,9 +602,13 @@ def _euclidean_distances_upcast(X, XX=None, Y=None, YY=None, batch_size=None):
         # Allow 10% more memory than X, Y and the distance matrix take (at
         # least 10MiB)
         maxmem = max(
-            ((x_density * n_samples_X + y_density * n_samples_Y) * n_features
-             + (x_density * n_samples_X * y_density * n_samples_Y)) / 10,
-            10 * 2 ** 17)
+            (
+                (x_density * n_samples_X + y_density * n_samples_Y) * n_features
+                + (x_density * n_samples_X * y_density * n_samples_Y)
+            )
+            / 10,
+            10 * 2**17,
+        )
 
         # The increase amount of memory in 8-byte blocks is:
         # - x_density * batch_size * n_features (copy of chunk of X)
@@ -501,15 +617,15 @@ def _euclidean_distances_upcast(X, XX=None, Y=None, YY=None, batch_size=None):
         # Hence x² + (xd+yd)kx = M, where x=batch_size, k=n_features, M=maxmem
         #                                 xd=x_density and yd=y_density
         tmp = (x_density + y_density) * n_features
-        batch_size = (-tmp + np.sqrt(tmp ** 2 + 4 * maxmem)) / 2
+        batch_size = (-tmp + math.sqrt(tmp**2 + 4 * maxmem)) / 2
         batch_size = max(int(batch_size), 1)
 
     x_batches = gen_batches(n_samples_X, batch_size)
-
+    xp_max_float = _max_precision_float_dtype(xp=xp, device=device_)
     for i, x_slice in enumerate(x_batches):
-        X_chunk = X[x_slice].astype(np.float64)
+        X_chunk = xp.astype(X[x_slice, :], xp_max_float)
         if XX is None:
-            XX_chunk = row_norms(X_chunk, squared=True)[:, np.newaxis]
+            XX_chunk = row_norms(X_chunk, squared=True)[:, None]
         else:
             XX_chunk = XX[x_slice]
 
@@ -522,9 +638,9 @@ def _euclidean_distances_upcast(X, XX=None, Y=None, YY=None, batch_size=None):
                 d = distances[y_slice, x_slice].T
 
             else:
-                Y_chunk = Y[y_slice].astype(np.float64)
+                Y_chunk = xp.astype(Y[y_slice, :], xp_max_float)
                 if YY is None:
-                    YY_chunk = row_norms(Y_chunk, squared=True)[np.newaxis, :]
+                    YY_chunk = row_norms(Y_chunk, squared=True)[None, :]
                 else:
                     YY_chunk = YY[:, y_slice]
 
@@ -532,27 +648,90 @@ def _euclidean_distances_upcast(X, XX=None, Y=None, YY=None, batch_size=None):
                 d += XX_chunk
                 d += YY_chunk
 
-            distances[x_slice, y_slice] = d.astype(np.float32, copy=False)
+            distances[x_slice, y_slice] = xp.astype(d, xp.float32, copy=False)
 
     return distances
 
 
 def _argmin_min_reduce(dist, start):
+    # `start` is specified in the signature but not used. This is because the higher
+    # order `pairwise_distances_chunked` function needs reduction functions that are
+    # passed as argument to have a two arguments signature.
     indices = dist.argmin(axis=1)
     values = dist[np.arange(dist.shape[0]), indices]
     return indices, values
 
 
-@_deprecate_positional_args
-def pairwise_distances_argmin_min(X, Y, *, axis=1, metric="euclidean",
-                                  metric_kwargs=None):
+def _argmin_reduce(dist, start):
+    # `start` is specified in the signature but not used. This is because the higher
+    # order `pairwise_distances_chunked` function needs reduction functions that are
+    # passed as argument to have a two arguments signature.
+    return dist.argmin(axis=1)
+
+
+_VALID_METRICS = [
+    "euclidean",
+    "l2",
+    "l1",
+    "manhattan",
+    "cityblock",
+    "braycurtis",
+    "canberra",
+    "chebyshev",
+    "correlation",
+    "cosine",
+    "dice",
+    "hamming",
+    "jaccard",
+    "mahalanobis",
+    "matching",
+    "minkowski",
+    "rogerstanimoto",
+    "russellrao",
+    "seuclidean",
+    "sokalsneath",
+    "sqeuclidean",
+    "yule",
+    "wminkowski",
+    "nan_euclidean",
+    "haversine",
+]
+if sp_base_version < parse_version("1.17"):  # pragma: no cover
+    # Deprecated in SciPy 1.15 and removed in SciPy 1.17
+    _VALID_METRICS += ["sokalmichener"]
+if sp_base_version < parse_version("1.11"):  # pragma: no cover
+    # Deprecated in SciPy 1.9 and removed in SciPy 1.11
+    _VALID_METRICS += ["kulsinski"]
+if sp_base_version < parse_version("1.9"):
+    # Deprecated in SciPy 1.0 and removed in SciPy 1.9
+    _VALID_METRICS += ["matching"]
+
+_NAN_METRICS = ["nan_euclidean"]
+
+
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "Y": ["array-like", "sparse matrix"],
+        "axis": [Options(Integral, {0, 1})],
+        "metric": [
+            StrOptions(set(_VALID_METRICS).union(ArgKmin.valid_metrics())),
+            callable,
+        ],
+        "metric_kwargs": [dict, None],
+    },
+    prefer_skip_nested_validation=False,  # metric is not validated yet
+)
+def pairwise_distances_argmin_min(
+    X, Y, *, axis=1, metric="euclidean", metric_kwargs=None
+):
     """Compute minimum distances between one point and a set of points.
 
     This function computes for each row in X, the index of the row of Y which
     is closest (according to the specified distance). The minimal distances are
     also returned.
 
-    This is mostly equivalent to calling:
+    This is mostly equivalent to calling::
 
         (pairwise_distances(X, Y=Y, metric=metric).argmin(axis=axis),
          pairwise_distances(X, Y=Y, metric=metric).min(axis=axis))
@@ -585,7 +764,7 @@ def pairwise_distances_argmin_min(X, Y, *, axis=1, metric="euclidean",
         Valid values for metric are:
 
         - from scikit-learn: ['cityblock', 'cosine', 'euclidean', 'l1', 'l2',
-          'manhattan']
+          'manhattan', 'nan_euclidean']
 
         - from scipy.spatial.distance: ['braycurtis', 'canberra', 'chebyshev',
           'correlation', 'dice', 'hamming', 'jaccard', 'kulsinski',
@@ -596,6 +775,12 @@ def pairwise_distances_argmin_min(X, Y, *, axis=1, metric="euclidean",
         See the documentation for scipy.spatial.distance for details on these
         metrics.
 
+        .. note::
+           `'kulsinski'` is deprecated from SciPy 1.9 and will be removed in SciPy 1.11.
+
+        .. note::
+           `'matching'` has been removed in SciPy 1.9 (use `'hamming'` instead).
+
     metric_kwargs : dict, default=None
         Keyword arguments to pass to specified metric function.
 
@@ -605,40 +790,95 @@ def pairwise_distances_argmin_min(X, Y, *, axis=1, metric="euclidean",
         Y[argmin[i], :] is the row in Y that is closest to X[i, :].
 
     distances : ndarray
-        distances[i] is the distance between the i-th row in X and the
-        argmin[i]-th row in Y.
+        The array of minimum distances. `distances[i]` is the distance between
+        the i-th row in X and the argmin[i]-th row in Y.
 
     See Also
     --------
-    sklearn.metrics.pairwise_distances
-    sklearn.metrics.pairwise_distances_argmin
-    """
-    X, Y = check_pairwise_arrays(X, Y)
+    pairwise_distances : Distances between every pair of samples of X and Y.
+    pairwise_distances_argmin : Same as `pairwise_distances_argmin_min` but only
+        returns the argmins.
 
-    if metric_kwargs is None:
-        metric_kwargs = {}
+    Examples
+    --------
+    >>> from sklearn.metrics.pairwise import pairwise_distances_argmin_min
+    >>> X = [[0, 0, 0], [1, 1, 1]]
+    >>> Y = [[1, 0, 0], [1, 1, 0]]
+    >>> argmin, distances = pairwise_distances_argmin_min(X, Y)
+    >>> argmin
+    array([0, 1])
+    >>> distances
+    array([1., 1.])
+    """
+    ensure_all_finite = "allow-nan" if metric == "nan_euclidean" else True
+    X, Y = check_pairwise_arrays(X, Y, ensure_all_finite=ensure_all_finite)
 
     if axis == 0:
         X, Y = Y, X
 
-    indices, values = zip(*pairwise_distances_chunked(
-        X, Y, reduce_func=_argmin_min_reduce, metric=metric,
-        **metric_kwargs))
-    indices = np.concatenate(indices)
-    values = np.concatenate(values)
+    if metric_kwargs is None:
+        metric_kwargs = {}
+
+    if ArgKmin.is_usable_for(X, Y, metric):
+        # This is an adaptor for one "sqeuclidean" specification.
+        # For this backend, we can directly use "sqeuclidean".
+        if metric_kwargs.get("squared", False) and metric == "euclidean":
+            metric = "sqeuclidean"
+            metric_kwargs = {}
+
+        values, indices = ArgKmin.compute(
+            X=X,
+            Y=Y,
+            k=1,
+            metric=metric,
+            metric_kwargs=metric_kwargs,
+            strategy="auto",
+            return_distance=True,
+        )
+        values = values.flatten()
+        indices = indices.flatten()
+    else:
+        # Joblib-based backend, which is used when user-defined callable
+        # are passed for metric.
+
+        # This won't be used in the future once PairwiseDistancesReductions support:
+        #   - DistanceMetrics which work on supposedly binary data
+        #   - CSR-dense and dense-CSR case if 'euclidean' in metric.
+
+        # Turn off check for finiteness because this is costly and because arrays
+        # have already been validated.
+        with config_context(assume_finite=True):
+            indices, values = zip(
+                *pairwise_distances_chunked(
+                    X, Y, reduce_func=_argmin_min_reduce, metric=metric, **metric_kwargs
+                )
+            )
+        indices = np.concatenate(indices)
+        values = np.concatenate(values)
 
     return indices, values
 
 
-@_deprecate_positional_args
-def pairwise_distances_argmin(X, Y, *, axis=1, metric="euclidean",
-                              metric_kwargs=None):
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "Y": ["array-like", "sparse matrix"],
+        "axis": [Options(Integral, {0, 1})],
+        "metric": [
+            StrOptions(set(_VALID_METRICS).union(ArgKmin.valid_metrics())),
+            callable,
+        ],
+        "metric_kwargs": [dict, None],
+    },
+    prefer_skip_nested_validation=False,  # metric is not validated yet
+)
+def pairwise_distances_argmin(X, Y, *, axis=1, metric="euclidean", metric_kwargs=None):
     """Compute minimum distances between one point and a set of points.
 
     This function computes for each row in X, the index of the row of Y which
     is closest (according to the specified distance).
 
-    This is mostly equivalent to calling:
+    This is mostly equivalent to calling::
 
         pairwise_distances(X, Y=Y, metric=metric).argmin(axis=axis)
 
@@ -648,10 +888,10 @@ def pairwise_distances_argmin(X, Y, *, axis=1, metric="euclidean",
 
     Parameters
     ----------
-    X : array-like of shape (n_samples_X, n_features)
+    X : {array-like, sparse matrix} of shape (n_samples_X, n_features)
         Array containing points.
 
-    Y : array-like of shape (n_samples_Y, n_features)
+    Y : {array-like, sparse matrix} of shape (n_samples_Y, n_features)
         Arrays containing points.
 
     axis : int, default=1
@@ -672,7 +912,7 @@ def pairwise_distances_argmin(X, Y, *, axis=1, metric="euclidean",
         Valid values for metric are:
 
         - from scikit-learn: ['cityblock', 'cosine', 'euclidean', 'l1', 'l2',
-          'manhattan']
+          'manhattan', 'nan_euclidean']
 
         - from scipy.spatial.distance: ['braycurtis', 'canberra', 'chebyshev',
           'correlation', 'dice', 'hamming', 'jaccard', 'kulsinski',
@@ -683,6 +923,12 @@ def pairwise_distances_argmin(X, Y, *, axis=1, metric="euclidean",
         See the documentation for scipy.spatial.distance for details on these
         metrics.
 
+        .. note::
+           `'kulsinski'` is deprecated from SciPy 1.9 and will be removed in SciPy 1.11.
+
+        .. note::
+           `'matching'` has been removed in SciPy 1.9 (use `'hamming'` instead).
+
     metric_kwargs : dict, default=None
         Keyword arguments to pass to specified metric function.
 
@@ -693,16 +939,72 @@ def pairwise_distances_argmin(X, Y, *, axis=1, metric="euclidean",
 
     See Also
     --------
-    sklearn.metrics.pairwise_distances
-    sklearn.metrics.pairwise_distances_argmin_min
+    pairwise_distances : Distances between every pair of samples of X and Y.
+    pairwise_distances_argmin_min : Same as `pairwise_distances_argmin` but also
+        returns the distances.
+
+    Examples
+    --------
+    >>> from sklearn.metrics.pairwise import pairwise_distances_argmin
+    >>> X = [[0, 0, 0], [1, 1, 1]]
+    >>> Y = [[1, 0, 0], [1, 1, 0]]
+    >>> pairwise_distances_argmin(X, Y)
+    array([0, 1])
     """
+    ensure_all_finite = "allow-nan" if metric == "nan_euclidean" else True
+    X, Y = check_pairwise_arrays(X, Y, ensure_all_finite=ensure_all_finite)
+
+    if axis == 0:
+        X, Y = Y, X
+
     if metric_kwargs is None:
         metric_kwargs = {}
 
-    return pairwise_distances_argmin_min(X, Y, axis=axis, metric=metric,
-                                         metric_kwargs=metric_kwargs)[0]
+    if ArgKmin.is_usable_for(X, Y, metric):
+        # This is an adaptor for one "sqeuclidean" specification.
+        # For this backend, we can directly use "sqeuclidean".
+        if metric_kwargs.get("squared", False) and metric == "euclidean":
+            metric = "sqeuclidean"
+            metric_kwargs = {}
+
+        indices = ArgKmin.compute(
+            X=X,
+            Y=Y,
+            k=1,
+            metric=metric,
+            metric_kwargs=metric_kwargs,
+            strategy="auto",
+            return_distance=False,
+        )
+        indices = indices.flatten()
+    else:
+        # Joblib-based backend, which is used when user-defined callable
+        # are passed for metric.
+
+        # This won't be used in the future once PairwiseDistancesReductions support:
+        #   - DistanceMetrics which work on supposedly binary data
+        #   - CSR-dense and dense-CSR case if 'euclidean' in metric.
+
+        # Turn off check for finiteness because this is costly and because arrays
+        # have already been validated.
+        with config_context(assume_finite=True):
+            indices = np.concatenate(
+                list(
+                    # This returns a np.ndarray generator whose arrays we need
+                    # to flatten into one.
+                    pairwise_distances_chunked(
+                        X, Y, reduce_func=_argmin_reduce, metric=metric, **metric_kwargs
+                    )
+                )
+            )
+
+    return indices
 
 
+@validate_params(
+    {"X": ["array-like", "sparse matrix"], "Y": ["array-like", "sparse matrix", None]},
+    prefer_skip_nested_validation=True,
+)
 def haversine_distances(X, Y=None):
     """Compute the Haversine distance between samples in X and Y.
 
@@ -712,18 +1014,22 @@ def haversine_distances(X, Y=None):
     in radians. The dimension of the data must be 2.
 
     .. math::
-       D(x, y) = 2\\arcsin[\\sqrt{\\sin^2((x1 - y1) / 2)
-                                + \\cos(x1)\\cos(y1)\\sin^2((x2 - y2) / 2)}]
+       D(x, y) = 2\\arcsin[\\sqrt{\\sin^2((x_{lat} - y_{lat}) / 2)
+                                + \\cos(x_{lat})\\cos(y_{lat})\\
+                                sin^2((x_{lon} - y_{lon}) / 2)}]
 
     Parameters
     ----------
-    X : array-like of shape (n_samples_X, 2)
+    X : {array-like, sparse matrix} of shape (n_samples_X, 2)
+        A feature array.
 
-    Y : array-like of shape (n_samples_Y, 2), default=None
+    Y : {array-like, sparse matrix} of shape (n_samples_Y, 2), default=None
+        An optional second feature array. If `None`, uses `Y=X`.
 
     Returns
     -------
-    distance : ndarray of shape (n_samples_X, n_samples_Y)
+    distances : ndarray of shape (n_samples_X, n_samples_Y)
+        The distance matrix.
 
     Notes
     -----
@@ -748,42 +1054,39 @@ def haversine_distances(X, Y=None):
     array([[    0.        , 11099.54035582],
            [11099.54035582,     0.        ]])
     """
-    from ..neighbors import DistanceMetric
-    return DistanceMetric.get_metric('haversine').pairwise(X, Y)
+    from ..metrics import DistanceMetric
 
+    return DistanceMetric.get_metric("haversine").pairwise(X, Y)
 
-@_deprecate_positional_args
-def manhattan_distances(X, Y=None, *, sum_over_features=True):
-    """Compute the L1 distances between the vectors in X and Y.
 
-    With sum_over_features equal to False it returns the componentwise
-    distances.
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "Y": ["array-like", "sparse matrix", None],
+    },
+    prefer_skip_nested_validation=True,
+)
+def manhattan_distances(X, Y=None):
+    """Compute the L1 distances between the vectors in X and Y.
 
     Read more in the :ref:`User Guide `.
 
     Parameters
     ----------
-    X : array-like of shape (n_samples_X, n_features)
-
-    Y : array-like of shape (n_samples_Y, n_features), default=None
+    X : {array-like, sparse matrix} of shape (n_samples_X, n_features)
+        An array where each row is a sample and each column is a feature.
 
-    sum_over_features : bool, default=True
-        If True the function returns the pairwise distance matrix
-        else it returns the componentwise L1 pairwise-distances.
-        Not supported for sparse matrix inputs.
+    Y : {array-like, sparse matrix} of shape (n_samples_Y, n_features), default=None
+        An array where each row is a sample and each column is a feature.
+        If `None`, method uses `Y=X`.
 
     Returns
     -------
-    D : ndarray of shape (n_samples_X * n_samples_Y, n_features) or \
-            (n_samples_X, n_samples_Y)
-        If sum_over_features is False shape is
-        (n_samples_X * n_samples_Y, n_features) and D contains the
-        componentwise L1 pairwise-distances (ie. absolute difference),
-        else shape is (n_samples_X, n_samples_Y) and D contains
-        the pairwise L1 distances.
+    distances : ndarray of shape (n_samples_X, n_samples_Y)
+        Pairwise L1 distances.
 
     Notes
-    --------
+    -----
     When X and/or Y are CSR sparse matrices and they are not already
     in canonical format, this function modifies them in-place to
     make them canonical.
@@ -801,38 +1104,28 @@ def manhattan_distances(X, Y=None, *, sum_over_features=True):
          [[1, 2], [0, 3]])
     array([[0., 2.],
            [4., 4.]])
-    >>> import numpy as np
-    >>> X = np.ones((1, 2))
-    >>> y = np.full((2, 2), 2.)
-    >>> manhattan_distances(X, y, sum_over_features=False)
-    array([[1., 1.],
-           [1., 1.]])
     """
     X, Y = check_pairwise_arrays(X, Y)
 
     if issparse(X) or issparse(Y):
-        if not sum_over_features:
-            raise TypeError("sum_over_features=%r not supported"
-                            " for sparse matrices" % sum_over_features)
-
         X = csr_matrix(X, copy=False)
         Y = csr_matrix(Y, copy=False)
-        X.sum_duplicates()   # this also sorts indices in-place
+        X.sum_duplicates()  # this also sorts indices in-place
         Y.sum_duplicates()
         D = np.zeros((X.shape[0], Y.shape[0]))
-        _sparse_manhattan(X.data, X.indices, X.indptr,
-                          Y.data, Y.indices, Y.indptr,
-                          D)
+        _sparse_manhattan(X.data, X.indices, X.indptr, Y.data, Y.indices, Y.indptr, D)
         return D
 
-    if sum_over_features:
-        return distance.cdist(X, Y, 'cityblock')
-
-    D = X[:, np.newaxis, :] - Y[np.newaxis, :, :]
-    D = np.abs(D, D)
-    return D.reshape((-1, X.shape[1]))
+    return distance.cdist(X, Y, "cityblock")
 
 
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "Y": ["array-like", "sparse matrix", None],
+    },
+    prefer_skip_nested_validation=True,
+)
 def cosine_distances(X, Y=None):
     """Compute cosine distance between samples in X and Y.
 
@@ -851,60 +1144,107 @@ def cosine_distances(X, Y=None):
 
     Returns
     -------
-    distance matrix : ndarray of shape (n_samples_X, n_samples_Y)
+    distances : ndarray of shape (n_samples_X, n_samples_Y)
+        Returns the cosine distance between samples in X and Y.
 
     See Also
     --------
-    cosine_similarity
+    cosine_similarity : Compute cosine similarity between samples in X and Y.
     scipy.spatial.distance.cosine : Dense matrices only.
+
+    Examples
+    --------
+    >>> from sklearn.metrics.pairwise import cosine_distances
+    >>> X = [[0, 0, 0], [1, 1, 1]]
+    >>> Y = [[1, 0, 0], [1, 1, 0]]
+    >>> cosine_distances(X, Y)
+    array([[1.   , 1.   ],
+           [0.422, 0.183]])
     """
+    xp, _ = get_namespace(X, Y)
+
     # 1.0 - cosine_similarity(X, Y) without copy
     S = cosine_similarity(X, Y)
     S *= -1
     S += 1
-    np.clip(S, 0, 2, out=S)
+    S = xp.clip(S, 0.0, 2.0)
     if X is Y or Y is None:
         # Ensure that distances between vectors and themselves are set to 0.0.
         # This may not be the case due to floating point rounding errors.
-        S[np.diag_indices_from(S)] = 0.0
+        _fill_or_add_to_diagonal(S, 0.0, xp, add_value=False)
     return S
 
 
 # Paired distances
+@validate_params(
+    {"X": ["array-like", "sparse matrix"], "Y": ["array-like", "sparse matrix"]},
+    prefer_skip_nested_validation=True,
+)
 def paired_euclidean_distances(X, Y):
-    """
-    Computes the paired euclidean distances between X and Y.
+    """Compute the paired euclidean distances between X and Y.
 
     Read more in the :ref:`User Guide `.
 
     Parameters
     ----------
-    X : array-like of shape (n_samples, n_features)
+    X : {array-like, sparse matrix} of shape (n_samples, n_features)
+        Input array/matrix X.
 
-    Y : array-like of shape (n_samples, n_features)
+    Y : {array-like, sparse matrix} of shape (n_samples, n_features)
+        Input array/matrix Y.
 
     Returns
     -------
     distances : ndarray of shape (n_samples,)
+        Output array/matrix containing the calculated paired euclidean
+        distances.
+
+    Examples
+    --------
+    >>> from sklearn.metrics.pairwise import paired_euclidean_distances
+    >>> X = [[0, 0, 0], [1, 1, 1]]
+    >>> Y = [[1, 0, 0], [1, 1, 0]]
+    >>> paired_euclidean_distances(X, Y)
+    array([1., 1.])
     """
     X, Y = check_paired_arrays(X, Y)
     return row_norms(X - Y)
 
 
+@validate_params(
+    {"X": ["array-like", "sparse matrix"], "Y": ["array-like", "sparse matrix"]},
+    prefer_skip_nested_validation=True,
+)
 def paired_manhattan_distances(X, Y):
-    """Compute the L1 distances between the vectors in X and Y.
+    """Compute the paired L1 distances between X and Y.
+
+    Distances are calculated between (X[0], Y[0]), (X[1], Y[1]), ...,
+    (X[n_samples], Y[n_samples]).
 
     Read more in the :ref:`User Guide `.
 
     Parameters
     ----------
-    X : array-like of shape (n_samples, n_features)
+    X : {array-like, sparse matrix} of shape (n_samples, n_features)
+        An array-like where each row is a sample and each column is a feature.
 
-    Y : array-like of shape (n_samples, n_features)
+    Y : {array-like, sparse matrix} of shape (n_samples, n_features)
+        An array-like where each row is a sample and each column is a feature.
 
     Returns
     -------
     distances : ndarray of shape (n_samples,)
+        L1 paired distances between the row vectors of `X`
+        and the row vectors of `Y`.
+
+    Examples
+    --------
+    >>> from sklearn.metrics.pairwise import paired_manhattan_distances
+    >>> import numpy as np
+    >>> X = np.array([[1, 1, 0], [0, 1, 0], [0, 0, 1]])
+    >>> Y = np.array([[0, 1, 0], [0, 0, 1], [0, 0, 0]])
+    >>> paired_manhattan_distances(X, Y)
+    array([1., 2., 1.])
     """
     X, Y = check_paired_arrays(X, Y)
     diff = X - Y
@@ -915,46 +1255,71 @@ def paired_manhattan_distances(X, Y):
         return np.abs(diff).sum(axis=-1)
 
 
+@validate_params(
+    {"X": ["array-like", "sparse matrix"], "Y": ["array-like", "sparse matrix"]},
+    prefer_skip_nested_validation=True,
+)
 def paired_cosine_distances(X, Y):
     """
-    Computes the paired cosine distances between X and Y.
+    Compute the paired cosine distances between X and Y.
 
     Read more in the :ref:`User Guide `.
 
     Parameters
     ----------
-    X : array-like of shape (n_samples, n_features)
+    X : {array-like, sparse matrix} of shape (n_samples, n_features)
+        An array where each row is a sample and each column is a feature.
 
-    Y : array-like of shape (n_samples, n_features)
+    Y : {array-like, sparse matrix} of shape (n_samples, n_features)
+        An array where each row is a sample and each column is a feature.
 
     Returns
     -------
     distances : ndarray of shape (n_samples,)
+        Returns the distances between the row vectors of `X`
+        and the row vectors of `Y`, where `distances[i]` is the
+        distance between `X[i]` and `Y[i]`.
 
     Notes
     -----
     The cosine distance is equivalent to the half the squared
     euclidean distance if each sample is normalized to unit norm.
+
+    Examples
+    --------
+    >>> from sklearn.metrics.pairwise import paired_cosine_distances
+    >>> X = [[0, 0, 0], [1, 1, 1]]
+    >>> Y = [[1, 0, 0], [1, 1, 0]]
+    >>> paired_cosine_distances(X, Y)
+    array([0.5       , 0.184])
     """
     X, Y = check_paired_arrays(X, Y)
-    return .5 * row_norms(normalize(X) - normalize(Y), squared=True)
+    return 0.5 * row_norms(normalize(X) - normalize(Y), squared=True)
 
 
 PAIRED_DISTANCES = {
-    'cosine': paired_cosine_distances,
-    'euclidean': paired_euclidean_distances,
-    'l2': paired_euclidean_distances,
-    'l1': paired_manhattan_distances,
-    'manhattan': paired_manhattan_distances,
-    'cityblock': paired_manhattan_distances}
+    "cosine": paired_cosine_distances,
+    "euclidean": paired_euclidean_distances,
+    "l2": paired_euclidean_distances,
+    "l1": paired_manhattan_distances,
+    "manhattan": paired_manhattan_distances,
+    "cityblock": paired_manhattan_distances,
+}
 
 
-@_deprecate_positional_args
+@validate_params(
+    {
+        "X": ["array-like"],
+        "Y": ["array-like"],
+        "metric": [StrOptions(set(PAIRED_DISTANCES)), callable],
+    },
+    prefer_skip_nested_validation=True,
+)
 def paired_distances(X, Y, *, metric="euclidean", **kwds):
     """
-    Computes the paired distances between X and Y.
+    Compute the paired distances between X and Y.
 
-    Computes the distances between (X[0], Y[0]), (X[1], Y[1]), etc...
+    Compute the distances between (X[0], Y[0]), (X[1], Y[1]), etc...
 
     Read more in the :ref:`User Guide `.
 
@@ -973,16 +1338,22 @@ def paired_distances(X, Y, *, metric="euclidean", **kwds):
         "manhattan", or "cosine".
         Alternatively, if metric is a callable function, it is called on each
         pair of instances (rows) and the resulting value recorded. The callable
-        should take two arrays from X as input and return a value indicating
+        should take two arrays from `X` as input and return a value indicating
         the distance between them.
 
+    **kwds : dict
+        Unused parameters.
+
     Returns
     -------
     distances : ndarray of shape (n_samples,)
+        Returns the distances between the row vectors of `X`
+        and the row vectors of `Y`.
 
     See Also
     --------
-    pairwise_distances : Computes the distance between every pair of samples.
+    sklearn.metrics.pairwise_distances : Computes the distance between every pair of
+        samples.
 
     Examples
     --------
@@ -1003,11 +1374,17 @@ def paired_distances(X, Y, *, metric="euclidean", **kwds):
         for i in range(len(X)):
             distances[i] = metric(X[i], Y[i])
         return distances
-    else:
-        raise ValueError('Unknown distance %s' % metric)
 
 
 # Kernels
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "Y": ["array-like", "sparse matrix", None],
+        "dense_output": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
 def linear_kernel(X, Y=None, dense_output=True):
     """
     Compute the linear kernel between X and Y.
@@ -1016,9 +1393,11 @@ def linear_kernel(X, Y=None, dense_output=True):
 
     Parameters
     ----------
-    X : ndarray of shape (n_samples_X, n_features)
+    X : {array-like, sparse matrix} of shape (n_samples_X, n_features)
+        A feature array.
 
-    Y : ndarray of shape (n_samples_Y, n_features), default=None
+    Y : {array-like, sparse matrix} of shape (n_samples_Y, n_features), default=None
+        An optional second feature array. If `None`, uses `Y=X`.
 
     dense_output : bool, default=True
         Whether to return dense output even when the input is sparse. If
@@ -1028,36 +1407,76 @@ def linear_kernel(X, Y=None, dense_output=True):
 
     Returns
     -------
-    Gram matrix : ndarray of shape (n_samples_X, n_samples_Y)
+    kernel : ndarray of shape (n_samples_X, n_samples_Y)
+        The Gram matrix of the linear kernel, i.e. `X @ Y.T`.
+
+    Examples
+    --------
+    >>> from sklearn.metrics.pairwise import linear_kernel
+    >>> X = [[0, 0, 0], [1, 1, 1]]
+    >>> Y = [[1, 0, 0], [1, 1, 0]]
+    >>> linear_kernel(X, Y)
+    array([[0., 0.],
+           [1., 2.]])
     """
     X, Y = check_pairwise_arrays(X, Y)
     return safe_sparse_dot(X, Y.T, dense_output=dense_output)
 
 
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "Y": ["array-like", "sparse matrix", None],
+        "degree": [Interval(Real, 1, None, closed="left")],
+        "gamma": [
+            Interval(Real, 0, None, closed="left"),
+            None,
+            Hidden(np.ndarray),
+        ],
+        "coef0": [Interval(Real, None, None, closed="neither")],
+    },
+    prefer_skip_nested_validation=True,
+)
 def polynomial_kernel(X, Y=None, degree=3, gamma=None, coef0=1):
     """
-    Compute the polynomial kernel between X and Y::
+    Compute the polynomial kernel between X and Y.
+
+    .. code-block:: text
 
-        K(X, Y) = (gamma  + coef0)^degree
+        K(X, Y) = (gamma  + coef0) ^ degree
 
     Read more in the :ref:`User Guide `.
 
     Parameters
     ----------
-    X : ndarray of shape (n_samples_X, n_features)
+    X : {array-like, sparse matrix} of shape (n_samples_X, n_features)
+        A feature array.
 
-    Y : ndarray of shape (n_samples_Y, n_features), default=None
+    Y : {array-like, sparse matrix} of shape (n_samples_Y, n_features), default=None
+        An optional second feature array. If `None`, uses `Y=X`.
 
-    degree : int, default=3
+    degree : float, default=3
+        Kernel degree.
 
     gamma : float, default=None
-        If None, defaults to 1.0 / n_features.
+        Coefficient of the vector inner product. If None, defaults to 1.0 / n_features.
 
     coef0 : float, default=1
+        Constant offset added to scaled inner product.
 
     Returns
     -------
-    Gram matrix : ndarray of shape (n_samples_X, n_samples_Y)
+    kernel : ndarray of shape (n_samples_X, n_samples_Y)
+        The polynomial kernel.
+
+    Examples
+    --------
+    >>> from sklearn.metrics.pairwise import polynomial_kernel
+    >>> X = [[0, 0, 0], [1, 1, 1]]
+    >>> Y = [[1, 0, 0], [1, 1, 0]]
+    >>> polynomial_kernel(X, Y, degree=2)
+    array([[1.     , 1.     ],
+           [1.77, 2.77]])
     """
     X, Y = check_pairwise_arrays(X, Y)
     if gamma is None:
@@ -1070,9 +1489,23 @@ def polynomial_kernel(X, Y=None, degree=3, gamma=None, coef0=1):
     return K
 
 
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "Y": ["array-like", "sparse matrix", None],
+        "gamma": [
+            Interval(Real, 0, None, closed="left"),
+            None,
+            Hidden(np.ndarray),
+        ],
+        "coef0": [Interval(Real, None, None, closed="neither")],
+    },
+    prefer_skip_nested_validation=True,
+)
 def sigmoid_kernel(X, Y=None, gamma=None, coef0=1):
-    """
-    Compute the sigmoid kernel between X and Y::
+    """Compute the sigmoid kernel between X and Y.
+
+    .. code-block:: text
 
         K(X, Y) = tanh(gamma  + coef0)
 
@@ -1080,19 +1513,33 @@ def sigmoid_kernel(X, Y=None, gamma=None, coef0=1):
 
     Parameters
     ----------
-    X : ndarray of shape (n_samples_X, n_features)
+    X : {array-like, sparse matrix} of shape (n_samples_X, n_features)
+        A feature array.
 
-    Y : ndarray of shape (n_samples_Y, n_features), default=None
+    Y : {array-like, sparse matrix} of shape (n_samples_Y, n_features), default=None
+        An optional second feature array. If `None`, uses `Y=X`.
 
     gamma : float, default=None
-        If None, defaults to 1.0 / n_features.
+        Coefficient of the vector inner product. If None, defaults to 1.0 / n_features.
 
     coef0 : float, default=1
+        Constant offset added to scaled inner product.
 
     Returns
     -------
-    Gram matrix : ndarray of shape (n_samples_X, n_samples_Y)
+    kernel : ndarray of shape (n_samples_X, n_samples_Y)
+        Sigmoid kernel between two arrays.
+
+    Examples
+    --------
+    >>> from sklearn.metrics.pairwise import sigmoid_kernel
+    >>> X = [[0, 0, 0], [1, 1, 1]]
+    >>> Y = [[1, 0, 0], [1, 1, 0]]
+    >>> sigmoid_kernel(X, Y)
+    array([[0.76, 0.76],
+           [0.87, 0.93]])
     """
+    xp, _ = get_namespace(X, Y)
     X, Y = check_pairwise_arrays(X, Y)
     if gamma is None:
         gamma = 1.0 / X.shape[1]
@@ -1100,13 +1547,27 @@ def sigmoid_kernel(X, Y=None, gamma=None, coef0=1):
     K = safe_sparse_dot(X, Y.T, dense_output=True)
     K *= gamma
     K += coef0
-    np.tanh(K, K)  # compute tanh in-place
+    # compute tanh in-place for numpy
+    K = _modify_in_place_if_numpy(xp, xp.tanh, K, out=K)
     return K
 
 
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "Y": ["array-like", "sparse matrix", None],
+        "gamma": [
+            Interval(Real, 0, None, closed="left"),
+            None,
+            Hidden(np.ndarray),
+        ],
+    },
+    prefer_skip_nested_validation=True,
+)
 def rbf_kernel(X, Y=None, gamma=None):
-    """
-    Compute the rbf (gaussian) kernel between X and Y::
+    """Compute the rbf (gaussian) kernel between X and Y.
+
+    .. code-block:: text
 
         K(x, y) = exp(-gamma ||x-y||^2)
 
@@ -1116,31 +1577,59 @@ def rbf_kernel(X, Y=None, gamma=None):
 
     Parameters
     ----------
-    X : ndarray of shape (n_samples_X, n_features)
+    X : {array-like, sparse matrix} of shape (n_samples_X, n_features)
+        A feature array.
 
-    Y : ndarray of shape (n_samples_Y, n_features), default=None
+    Y : {array-like, sparse matrix} of shape (n_samples_Y, n_features), default=None
+        An optional second feature array. If `None`, uses `Y=X`.
 
     gamma : float, default=None
         If None, defaults to 1.0 / n_features.
 
     Returns
     -------
-    kernel_matrix : ndarray of shape (n_samples_X, n_samples_Y)
+    kernel : ndarray of shape (n_samples_X, n_samples_Y)
+        The RBF kernel.
+
+    Examples
+    --------
+    >>> from sklearn.metrics.pairwise import rbf_kernel
+    >>> X = [[0, 0, 0], [1, 1, 1]]
+    >>> Y = [[1, 0, 0], [1, 1, 0]]
+    >>> rbf_kernel(X, Y)
+    array([[0.71, 0.51],
+           [0.51, 0.71]])
     """
+    xp, _ = get_namespace(X, Y)
     X, Y = check_pairwise_arrays(X, Y)
     if gamma is None:
         gamma = 1.0 / X.shape[1]
 
     K = euclidean_distances(X, Y, squared=True)
     K *= -gamma
-    np.exp(K, K)  # exponentiate K in-place
+    # exponentiate K in-place when using numpy
+    K = _modify_in_place_if_numpy(xp, xp.exp, K, out=K)
     return K
 
 
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "Y": ["array-like", "sparse matrix", None],
+        "gamma": [
+            Interval(Real, 0, None, closed="neither"),
+            Hidden(np.ndarray),
+            None,
+        ],
+    },
+    prefer_skip_nested_validation=True,
+)
 def laplacian_kernel(X, Y=None, gamma=None):
     """Compute the laplacian kernel between X and Y.
 
-    The laplacian kernel is defined as::
+    The laplacian kernel is defined as:
+
+    .. code-block:: text
 
         K(x, y) = exp(-gamma ||x-y||_1)
 
@@ -1151,16 +1640,28 @@ def laplacian_kernel(X, Y=None, gamma=None):
 
     Parameters
     ----------
-    X : ndarray of shape (n_samples_X, n_features)
+    X : {array-like, sparse matrix} of shape (n_samples_X, n_features)
+        A feature array.
 
-    Y : ndarray of shape (n_samples_Y, n_features), default=None
+    Y : {array-like, sparse matrix} of shape (n_samples_Y, n_features), default=None
+        An optional second feature array. If `None`, uses `Y=X`.
 
     gamma : float, default=None
-        If None, defaults to 1.0 / n_features.
+        If None, defaults to 1.0 / n_features. Otherwise it should be strictly positive.
 
     Returns
     -------
-    kernel_matrix : ndarray of shape (n_samples_X, n_samples_Y)
+    kernel : ndarray of shape (n_samples_X, n_samples_Y)
+        The kernel matrix.
+
+    Examples
+    --------
+    >>> from sklearn.metrics.pairwise import laplacian_kernel
+    >>> X = [[0, 0, 0], [1, 1, 1]]
+    >>> Y = [[1, 0, 0], [1, 1, 0]]
+    >>> laplacian_kernel(X, Y)
+    array([[0.71, 0.51],
+           [0.51, 0.71]])
     """
     X, Y = check_pairwise_arrays(X, Y)
     if gamma is None:
@@ -1171,12 +1672,22 @@ def laplacian_kernel(X, Y=None, gamma=None):
     return K
 
 
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "Y": ["array-like", "sparse matrix", None],
+        "dense_output": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
 def cosine_similarity(X, Y=None, dense_output=True):
     """Compute cosine similarity between samples in X and Y.
 
     Cosine similarity, or the cosine kernel, computes similarity as the
     normalized dot product of X and Y:
 
+    .. code-block:: text
+
         K(X, Y) =  / (||X||*||Y||)
 
     On L2-normalized data, this function is equivalent to linear_kernel.
@@ -1185,10 +1696,10 @@ def cosine_similarity(X, Y=None, dense_output=True):
 
     Parameters
     ----------
-    X : {ndarray, sparse matrix} of shape (n_samples_X, n_features)
+    X : {array-like, sparse matrix} of shape (n_samples_X, n_features)
         Input data.
 
-    Y : {ndarray, sparse matrix} of shape (n_samples_Y, n_features), \
+    Y : {array-like, sparse matrix} of shape (n_samples_Y, n_features), \
             default=None
         Input data. If ``None``, the output will be the pairwise
         similarities between all samples in ``X``.
@@ -1202,10 +1713,18 @@ def cosine_similarity(X, Y=None, dense_output=True):
 
     Returns
     -------
-    kernel matrix : ndarray of shape (n_samples_X, n_samples_Y)
-    """
-    # to avoid recursive import
+    similarities : ndarray or sparse matrix of shape (n_samples_X, n_samples_Y)
+        Returns the cosine similarity between samples in X and Y.
 
+    Examples
+    --------
+    >>> from sklearn.metrics.pairwise import cosine_similarity
+    >>> X = [[0, 0, 0], [1, 1, 1]]
+    >>> Y = [[1, 0, 0], [1, 1, 0]]
+    >>> cosine_similarity(X, Y)
+    array([[0.   , 0.   ],
+           [0.577, 0.816]])
+    """
     X, Y = check_pairwise_arrays(X, Y)
 
     X_normalized = normalize(X, copy=True)
@@ -1214,21 +1733,25 @@ def cosine_similarity(X, Y=None, dense_output=True):
     else:
         Y_normalized = normalize(Y, copy=True)
 
-    K = safe_sparse_dot(X_normalized, Y_normalized.T,
-                        dense_output=dense_output)
+    K = safe_sparse_dot(X_normalized, Y_normalized.T, dense_output=dense_output)
 
     return K
 
 
+@validate_params(
+    {"X": ["array-like"], "Y": ["array-like", None]},
+    prefer_skip_nested_validation=True,
+)
 def additive_chi2_kernel(X, Y=None):
-    """Computes the additive chi-squared kernel between observations in X and
-    Y.
+    """Compute the additive chi-squared kernel between observations in X and Y.
 
     The chi-squared kernel is computed between each pair of rows in X and Y.  X
     and Y have to be non-negative. This kernel is most commonly applied to
     histograms.
 
-    The chi-squared kernel is given by::
+    The chi-squared kernel is given by:
+
+    .. code-block:: text
 
         k(x, y) = -Sum [(x - y)^2 / (x + y)]
 
@@ -1236,21 +1759,18 @@ def additive_chi2_kernel(X, Y=None):
 
     Read more in the :ref:`User Guide `.
 
-    Notes
-    -----
-    As the negative of a distance, this kernel is only conditionally positive
-    definite.
-
-
     Parameters
     ----------
     X : array-like of shape (n_samples_X, n_features)
+        A feature array.
 
-    Y : ndarray of shape (n_samples_Y, n_features), default=None
+    Y : array-like of shape (n_samples_Y, n_features), default=None
+        An optional second feature array. If `None`, uses `Y=X`.
 
     Returns
     -------
-    kernel_matrix : ndarray of shape (n_samples_X, n_samples_Y)
+    kernel : array-like of shape (n_samples_X, n_samples_Y)
+        The kernel matrix.
 
     See Also
     --------
@@ -1259,35 +1779,68 @@ def additive_chi2_kernel(X, Y=None):
     sklearn.kernel_approximation.AdditiveChi2Sampler : A Fourier approximation
         to this kernel.
 
+    Notes
+    -----
+    As the negative of a distance, this kernel is only conditionally positive
+    definite.
+
     References
     ----------
     * Zhang, J. and Marszalek, M. and Lazebnik, S. and Schmid, C.
       Local features and kernels for classification of texture and object
       categories: A comprehensive study
       International Journal of Computer Vision 2007
-      https://research.microsoft.com/en-us/um/people/manik/projects/trade-off/papers/ZhangIJCV06.pdf
+      https://hal.archives-ouvertes.fr/hal-00171412/document
+
+    Examples
+    --------
+    >>> from sklearn.metrics.pairwise import additive_chi2_kernel
+    >>> X = [[0, 0, 0], [1, 1, 1]]
+    >>> Y = [[1, 0, 0], [1, 1, 0]]
+    >>> additive_chi2_kernel(X, Y)
+    array([[-1., -2.],
+           [-2., -1.]])
     """
-    if issparse(X) or issparse(Y):
-        raise ValueError("additive_chi2 does not support sparse matrices.")
-    X, Y = check_pairwise_arrays(X, Y)
-    if (X < 0).any():
+    xp, _, device_ = get_namespace_and_device(X, Y)
+    X, Y = check_pairwise_arrays(X, Y, accept_sparse=False)
+    if xp.any(X < 0):
         raise ValueError("X contains negative values.")
-    if Y is not X and (Y < 0).any():
+    if Y is not X and xp.any(Y < 0):
         raise ValueError("Y contains negative values.")
 
-    result = np.zeros((X.shape[0], Y.shape[0]), dtype=X.dtype)
-    _chi2_kernel_fast(X, Y, result)
-    return result
-
-
-def chi2_kernel(X, Y=None, gamma=1.):
-    """Computes the exponential chi-squared kernel X and Y.
+    if _is_numpy_namespace(xp):
+        result = np.zeros((X.shape[0], Y.shape[0]), dtype=X.dtype)
+        _chi2_kernel_fast(X, Y, result)
+        return result
+    else:
+        dtype = _find_matching_floating_dtype(X, Y, xp=xp)
+        xb = X[:, None, :]
+        yb = Y[None, :, :]
+        nom = -((xb - yb) ** 2)
+        denom = xb + yb
+        nom = xp.where(denom == 0, xp.asarray(0, dtype=dtype, device=device_), nom)
+        denom = xp.where(denom == 0, xp.asarray(1, dtype=dtype, device=device_), denom)
+        return xp.sum(nom / denom, axis=2)
+
+
+@validate_params(
+    {
+        "X": ["array-like"],
+        "Y": ["array-like", None],
+        "gamma": [Interval(Real, 0, None, closed="neither"), Hidden(np.ndarray)],
+    },
+    prefer_skip_nested_validation=True,
+)
+def chi2_kernel(X, Y=None, gamma=1.0):
+    """Compute the exponential chi-squared kernel between X and Y.
 
     The chi-squared kernel is computed between each pair of rows in X and Y.  X
     and Y have to be non-negative. This kernel is most commonly applied to
     histograms.
 
-    The chi-squared kernel is given by::
+    The chi-squared kernel is given by:
+
+    .. code-block:: text
 
         k(x, y) = exp(-gamma Sum [(x - y)^2 / (x + y)])
 
@@ -1298,15 +1851,18 @@ def chi2_kernel(X, Y=None, gamma=1.):
     Parameters
     ----------
     X : array-like of shape (n_samples_X, n_features)
+        A feature array.
 
-    Y : ndarray of shape (n_samples_Y, n_features), default=None
+    Y : array-like of shape (n_samples_Y, n_features), default=None
+        An optional second feature array. If `None`, uses `Y=X`.
 
-    gamma : float, default=1.
+    gamma : float, default=1
         Scaling parameter of the chi2 kernel.
 
     Returns
     -------
-    kernel_matrix : ndarray of shape (n_samples_X, n_samples_Y)
+    kernel : ndarray of shape (n_samples_X, n_samples_Y)
+        The kernel matrix.
 
     See Also
     --------
@@ -1320,26 +1876,38 @@ def chi2_kernel(X, Y=None, gamma=1.):
       Local features and kernels for classification of texture and object
       categories: A comprehensive study
       International Journal of Computer Vision 2007
-      https://research.microsoft.com/en-us/um/people/manik/projects/trade-off/papers/ZhangIJCV06.pdf
+      https://hal.archives-ouvertes.fr/hal-00171412/document
+
+    Examples
+    --------
+    >>> from sklearn.metrics.pairwise import chi2_kernel
+    >>> X = [[0, 0, 0], [1, 1, 1]]
+    >>> Y = [[1, 0, 0], [1, 1, 0]]
+    >>> chi2_kernel(X, Y)
+    array([[0.368, 0.135],
+           [0.135, 0.368]])
     """
+    xp, _ = get_namespace(X, Y)
     K = additive_chi2_kernel(X, Y)
     K *= gamma
-    return np.exp(K, K)
+    if _is_numpy_namespace(xp):
+        return np.exp(K, out=K)
+    return xp.exp(K)
 
 
 # Helper functions - distance
 PAIRWISE_DISTANCE_FUNCTIONS = {
     # If updating this dictionary, update the doc in both distance_metrics()
     # and also in pairwise_distances()!
-    'cityblock': manhattan_distances,
-    'cosine': cosine_distances,
-    'euclidean': euclidean_distances,
-    'haversine': haversine_distances,
-    'l2': euclidean_distances,
-    'l1': manhattan_distances,
-    'manhattan': manhattan_distances,
-    'precomputed': None,  # HACK: precomputed is always allowed, never called
-    'nan_euclidean': nan_euclidean_distances,
+    "cityblock": manhattan_distances,
+    "cosine": cosine_distances,
+    "euclidean": euclidean_distances,
+    "haversine": haversine_distances,
+    "l2": euclidean_distances,
+    "l1": manhattan_distances,
+    "manhattan": manhattan_distances,
+    "precomputed": None,  # HACK: precomputed is always allowed, never called
+    "nan_euclidean": nan_euclidean_distances,
 }
 
 
@@ -1367,6 +1935,10 @@ def distance_metrics():
 
     Read more in the :ref:`User Guide `.
 
+    Returns
+    -------
+    distance_metrics : dict
+        Returns valid metrics for pairwise_distances.
     """
     return PAIRWISE_DISTANCE_FUNCTIONS
 
@@ -1378,7 +1950,7 @@ def _dist_wrapper(dist_func, dist_matrix, slice_, *args, **kwargs):
 
 def _parallel_pairwise(X, Y, func, n_jobs, **kwds):
     """Break the pairwise matrix in n_jobs even slices
-    and compute them in parallel."""
+    and compute them using multithreading."""
 
     if Y is None:
         Y = X
@@ -1389,10 +1961,11 @@ def _parallel_pairwise(X, Y, func, n_jobs, **kwds):
 
     # enforce a threading backend to prevent data communication overhead
     fd = delayed(_dist_wrapper)
-    ret = np.empty((X.shape[0], Y.shape[0]), dtype=dtype, order='F')
+    ret = np.empty((X.shape[0], Y.shape[0]), dtype=dtype, order="F")
     Parallel(backend="threading", n_jobs=n_jobs)(
         fd(func, ret, s, X, Y[s], **kwds)
-        for s in gen_even_slices(_num_samples(Y), effective_n_jobs(n_jobs)))
+        for s in gen_even_slices(_num_samples(Y), effective_n_jobs(n_jobs))
+    )
 
     if (X is Y or Y is None) and func is euclidean_distances:
         # zeroing diagonal for euclidean norm.
@@ -1402,17 +1975,27 @@ def _parallel_pairwise(X, Y, func, n_jobs, **kwds):
     return ret
 
 
-def _pairwise_callable(X, Y, metric, force_all_finite=True, **kwds):
-    """Handle the callable case for pairwise_{distances,kernels}.
-    """
-    X, Y = check_pairwise_arrays(X, Y, force_all_finite=force_all_finite)
+def _pairwise_callable(X, Y, metric, ensure_all_finite=True, **kwds):
+    """Handle the callable case for pairwise_{distances,kernels}."""
+    X, Y = check_pairwise_arrays(
+        X,
+        Y,
+        dtype=None,
+        ensure_all_finite=ensure_all_finite,
+        # No input dimension checking done for custom metrics (left to user)
+        ensure_2d=False,
+    )
 
     if X is Y:
         # Only calculate metric for upper triangle
-        out = np.zeros((X.shape[0], Y.shape[0]), dtype='float')
+        out = np.zeros((X.shape[0], Y.shape[0]), dtype="float")
         iterator = itertools.combinations(range(X.shape[0]), 2)
         for i, j in iterator:
-            out[i, j] = metric(X[i], Y[j], **kwds)
+            # scipy has not yet implemented 1D sparse slices; once implemented this can
+            # be removed and `arr[ind]` can be simply used.
+            x = X[[i], :] if issparse(X) else X[i]
+            y = Y[[j], :] if issparse(Y) else Y[j]
+            out[i, j] = metric(x, y, **kwds)
 
         # Make symmetric
         # NB: out += out.T will produce incorrect results
@@ -1421,105 +2004,107 @@ def _pairwise_callable(X, Y, metric, force_all_finite=True, **kwds):
         # Calculate diagonal
         # NB: nonzero diagonals are allowed for both metrics and kernels
         for i in range(X.shape[0]):
-            x = X[i]
+            # scipy has not yet implemented 1D sparse slices; once implemented this can
+            # be removed and `arr[ind]` can be simply used.
+            x = X[[i], :] if issparse(X) else X[i]
             out[i, i] = metric(x, x, **kwds)
 
     else:
         # Calculate all cells
-        out = np.empty((X.shape[0], Y.shape[0]), dtype='float')
+        out = np.empty((X.shape[0], Y.shape[0]), dtype="float")
         iterator = itertools.product(range(X.shape[0]), range(Y.shape[0]))
         for i, j in iterator:
-            out[i, j] = metric(X[i], Y[j], **kwds)
+            # scipy has not yet implemented 1D sparse slices; once implemented this can
+            # be removed and `arr[ind]` can be simply used.
+            x = X[[i], :] if issparse(X) else X[i]
+            y = Y[[j], :] if issparse(Y) else Y[j]
+            out[i, j] = metric(x, y, **kwds)
 
     return out
 
 
-_VALID_METRICS = ['euclidean', 'l2', 'l1', 'manhattan', 'cityblock',
-                  'braycurtis', 'canberra', 'chebyshev', 'correlation',
-                  'cosine', 'dice', 'hamming', 'jaccard', 'kulsinski',
-                  'mahalanobis', 'matching', 'minkowski', 'rogerstanimoto',
-                  'russellrao', 'seuclidean', 'sokalmichener',
-                  'sokalsneath', 'sqeuclidean', 'yule', "wminkowski",
-                  'nan_euclidean', 'haversine']
-
-_NAN_METRICS = ['nan_euclidean']
-
-
 def _check_chunk_size(reduced, chunk_size):
-    """Checks chunk is a sequence of expected size or a tuple of same.
-    """
+    """Checks chunk is a sequence of expected size or a tuple of same."""
     if reduced is None:
         return
     is_tuple = isinstance(reduced, tuple)
     if not is_tuple:
         reduced = (reduced,)
-    if any(isinstance(r, tuple) or not hasattr(r, '__iter__')
-           for r in reduced):
-        raise TypeError('reduce_func returned %r. '
-                        'Expected sequence(s) of length %d.' %
-                        (reduced if is_tuple else reduced[0], chunk_size))
+    if any(isinstance(r, tuple) or not hasattr(r, "__iter__") for r in reduced):
+        raise TypeError(
+            "reduce_func returned %r. Expected sequence(s) of length %d."
+            % (reduced if is_tuple else reduced[0], chunk_size)
+        )
     if any(_num_samples(r) != chunk_size for r in reduced):
         actual_size = tuple(_num_samples(r) for r in reduced)
-        raise ValueError('reduce_func returned object of length %s. '
-                         'Expected same length as input: %d.' %
-                         (actual_size if is_tuple else actual_size[0],
-                          chunk_size))
+        raise ValueError(
+            "reduce_func returned object of length %s. "
+            "Expected same length as input: %d."
+            % (actual_size if is_tuple else actual_size[0], chunk_size)
+        )
 
 
 def _precompute_metric_params(X, Y, metric=None, **kwds):
-    """Precompute data-derived metric parameters if not provided.
-    """
-    if metric == "seuclidean" and 'V' not in kwds:
-        # There is a bug in scipy < 1.5 that will cause a crash if
-        # X.dtype != np.double (float64). See PR #15730
-        dtype = np.float64 if sp_version < parse_version('1.5') else None
+    """Precompute data-derived metric parameters if not provided."""
+    if metric == "seuclidean" and "V" not in kwds:
         if X is Y:
-            V = np.var(X, axis=0, ddof=1, dtype=dtype)
+            V = np.var(X, axis=0, ddof=1)
         else:
-            warnings.warn(
-                "from version 1.0 (renaming of 0.25), pairwise_distances for "
-                "metric='seuclidean' will require V to be specified if Y is "
-                "passed.",
-                FutureWarning
+            raise ValueError(
+                "The 'V' parameter is required for the seuclidean metric "
+                "when Y is passed."
             )
-            V = np.var(np.vstack([X, Y]), axis=0, ddof=1, dtype=dtype)
-        return {'V': V}
-    if metric == "mahalanobis" and 'VI' not in kwds:
+        return {"V": V}
+    if metric == "mahalanobis" and "VI" not in kwds:
         if X is Y:
             VI = np.linalg.inv(np.cov(X.T)).T
         else:
-            warnings.warn(
-                "from version 1.0 (renaming of 0.25), pairwise_distances for "
-                "metric='mahalanobis' will require VI to be specified if Y "
-                "is passed.",
-                FutureWarning
+            raise ValueError(
+                "The 'VI' parameter is required for the mahalanobis metric "
+                "when Y is passed."
             )
-            VI = np.linalg.inv(np.cov(np.vstack([X, Y]).T)).T
-        return {'VI': VI}
+        return {"VI": VI}
     return {}
 
 
-@_deprecate_positional_args
-def pairwise_distances_chunked(X, Y=None, *, reduce_func=None,
-                               metric='euclidean', n_jobs=None,
-                               working_memory=None, **kwds):
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "Y": ["array-like", "sparse matrix", None],
+        "reduce_func": [callable, None],
+        "metric": [StrOptions({"precomputed"}.union(_VALID_METRICS)), callable],
+        "n_jobs": [Integral, None],
+        "working_memory": [Interval(Real, 0, None, closed="left"), None],
+    },
+    prefer_skip_nested_validation=False,  # metric is not validated yet
+)
+def pairwise_distances_chunked(
+    X,
+    Y=None,
+    *,
+    reduce_func=None,
+    metric="euclidean",
+    n_jobs=None,
+    working_memory=None,
+    **kwds,
+):
     """Generate a distance matrix chunk by chunk with optional reduction.
 
-    In cases where not all of a pairwise distance matrix needs to be stored at
-    once, this is used to calculate pairwise distances in
-    ``working_memory``-sized chunks.  If ``reduce_func`` is given, it is run
-    on each chunk and its return values are concatenated into lists, arrays
-    or sparse matrices.
+    In cases where not all of a pairwise distance matrix needs to be
+    stored at once, this is used to calculate pairwise distances in
+    ``working_memory``-sized chunks.  If ``reduce_func`` is given, it is
+    run on each chunk and its return values are concatenated into lists,
+    arrays or sparse matrices.
 
     Parameters
     ----------
-    X : ndarray of shape (n_samples_X, n_samples_X) or \
+    X : {array-like, sparse matrix} of shape (n_samples_X, n_samples_X) or \
             (n_samples_X, n_features)
         Array of pairwise distances between samples, or a feature array.
         The shape the array should be (n_samples_X, n_samples_X) if
         metric='precomputed' and (n_samples_X, n_features) otherwise.
 
-    Y : ndarray of shape (n_samples_Y, n_features), default=None
+    Y : {array-like, sparse matrix} of shape (n_samples_Y, n_features), default=None
         An optional second feature array. Only allowed if
         metric != "precomputed".
 
@@ -1529,8 +2114,9 @@ def pairwise_distances_chunked(X, Y=None, *, reduce_func=None,
         is called repeatedly, where ``D_chunk`` is a contiguous vertical
         slice of the pairwise distance matrix, starting at row ``start``.
         It should return one of: None; an array, a list, or a sparse matrix
-        of length ``D_chunk.shape[0]``; or a tuple of such objects. Returning
-        None is useful for in-place operations, rather than reductions.
+        of length ``D_chunk.shape[0]``; or a tuple of such objects.
+        Returning None is useful for in-place operations, rather than
+        reductions.
 
         If None, pairwise_distances_chunked returns a generator of vertical
         chunks of the distance matrix.
@@ -1538,29 +2124,29 @@ def pairwise_distances_chunked(X, Y=None, *, reduce_func=None,
     metric : str or callable, default='euclidean'
         The metric to use when calculating distance between instances in a
         feature array. If metric is a string, it must be one of the options
-        allowed by scipy.spatial.distance.pdist for its metric parameter, or
-        a metric listed in pairwise.PAIRWISE_DISTANCE_FUNCTIONS.
+        allowed by scipy.spatial.distance.pdist for its metric parameter,
+        or a metric listed in pairwise.PAIRWISE_DISTANCE_FUNCTIONS.
         If metric is "precomputed", X is assumed to be a distance matrix.
-        Alternatively, if metric is a callable function, it is called on each
-        pair of instances (rows) and the resulting value recorded. The callable
-        should take two arrays from X as input and return a value indicating
-        the distance between them.
+        Alternatively, if metric is a callable function, it is called on
+        each pair of instances (rows) and the resulting value recorded.
+        The callable should take two arrays from X as input and return a
+        value indicating the distance between them.
 
     n_jobs : int, default=None
-        The number of jobs to use for the computation. This works by breaking
-        down the pairwise matrix into n_jobs even slices and computing them in
-        parallel.
+        The number of jobs to use for the computation. This works by
+        breaking down the pairwise matrix into n_jobs even slices and
+        computing them in parallel.
 
         ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
         ``-1`` means using all processors. See :term:`Glossary `
         for more details.
 
-    working_memory : int, default=None
+    working_memory : float, default=None
         The sought maximum memory for temporary distance matrix chunks.
         When None (default), the value of
         ``sklearn.get_config()['working_memory']`` is used.
 
-    `**kwds` : optional keyword parameters
+    **kwds : optional keyword parameters
         Any further parameters are passed directly to the distance function.
         If using a scipy.spatial.distance metric, the parameters are still
         metric dependent. See the scipy docs for usage examples.
@@ -1580,11 +2166,11 @@ def pairwise_distances_chunked(X, Y=None, *, reduce_func=None,
     >>> X = np.random.RandomState(0).rand(5, 3)
     >>> D_chunk = next(pairwise_distances_chunked(X))
     >>> D_chunk
-    array([[0.  ..., 0.29..., 0.41..., 0.19..., 0.57...],
-           [0.29..., 0.  ..., 0.57..., 0.41..., 0.76...],
-           [0.41..., 0.57..., 0.  ..., 0.44..., 0.90...],
-           [0.19..., 0.41..., 0.44..., 0.  ..., 0.51...],
-           [0.57..., 0.76..., 0.90..., 0.51..., 0.  ...]])
+    array([[0.   , 0.295, 0.417, 0.197, 0.572],
+           [0.295, 0.   , 0.576, 0.419, 0.764],
+           [0.417, 0.576, 0.   , 0.449, 0.903],
+           [0.197, 0.419, 0.449, 0.   , 0.512],
+           [0.572, 0.764, 0.903, 0.512, 0.   ]])
 
     Retrieve all neighbors and average distance within radius r:
 
@@ -1598,7 +2184,7 @@ def pairwise_distances_chunked(X, Y=None, *, reduce_func=None,
     >>> neigh
     [array([0, 3]), array([1]), array([2]), array([0, 3]), array([4])]
     >>> avg_dist
-    array([0.039..., 0.        , 0.        , 0.039..., 0.        ])
+    array([0.039, 0.        , 0.        , 0.039, 0.        ])
 
     Where r is defined per sample, we need to make use of ``start``:
 
@@ -1621,7 +2207,7 @@ def pairwise_distances_chunked(X, Y=None, *, reduce_func=None,
     [array([0, 1])]
     """
     n_samples_X = _num_samples(X)
-    if metric == 'precomputed':
+    if metric == "precomputed":
         slices = (slice(0, n_samples_X),)
     else:
         if Y is None:
@@ -1635,9 +2221,11 @@ def pairwise_distances_chunked(X, Y=None, *, reduce_func=None,
         #  - this does not account for any temporary memory usage while
         #    calculating distances (e.g. difference of vectors in manhattan
         #    distance.
-        chunk_n_rows = get_chunk_n_rows(row_bytes=8 * _num_samples(Y),
-                                        max_n_rows=n_samples_X,
-                                        working_memory=working_memory)
+        chunk_n_rows = get_chunk_n_rows(
+            row_bytes=8 * _num_samples(Y),
+            max_n_rows=n_samples_X,
+            working_memory=working_memory,
+        )
         slices = gen_batches(n_samples_X, chunk_n_rows)
 
     # precompute data-derived metric params
@@ -1649,14 +2237,13 @@ def pairwise_distances_chunked(X, Y=None, *, reduce_func=None,
             X_chunk = X  # enable optimised paths for X is Y
         else:
             X_chunk = X[sl]
-        D_chunk = pairwise_distances(X_chunk, Y, metric=metric,
-                                     n_jobs=n_jobs, **kwds)
-        if ((X is Y or Y is None)
-                and PAIRWISE_DISTANCE_FUNCTIONS.get(metric, None)
-                is euclidean_distances):
+        D_chunk = pairwise_distances(X_chunk, Y, metric=metric, n_jobs=n_jobs, **kwds)
+        if (X is Y or Y is None) and PAIRWISE_DISTANCE_FUNCTIONS.get(
+            metric, None
+        ) is euclidean_distances:
             # zeroing diagonal, taking care of aliases of "euclidean",
             # i.e. "l2"
-            D_chunk.flat[sl.start::_num_samples(X) + 1] = 0
+            D_chunk.flat[sl.start :: _num_samples(X) + 1] = 0
         if reduce_func is not None:
             chunk_size = D_chunk.shape[0]
             D_chunk = reduce_func(D_chunk, sl.start)
@@ -1664,28 +2251,58 @@ def pairwise_distances_chunked(X, Y=None, *, reduce_func=None,
         yield D_chunk
 
 
-@_deprecate_positional_args
-def pairwise_distances(X, Y=None, metric="euclidean", *, n_jobs=None,
-                       force_all_finite=True, **kwds):
-    """Compute the distance matrix from a vector array X and optional Y.
-
-    This method takes either a vector array or a distance matrix, and returns
-    a distance matrix. If the input is a vector array, the distances are
-    computed. If the input is a distances matrix, it is returned instead.
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "Y": ["array-like", "sparse matrix", None],
+        "metric": [StrOptions(set(_VALID_METRICS) | {"precomputed"}), callable],
+        "n_jobs": [Integral, None],
+        "force_all_finite": [
+            "boolean",
+            StrOptions({"allow-nan"}),
+            Hidden(StrOptions({"deprecated"})),
+        ],
+        "ensure_all_finite": ["boolean", StrOptions({"allow-nan"}), Hidden(None)],
+    },
+    prefer_skip_nested_validation=True,
+)
+def pairwise_distances(
+    X,
+    Y=None,
+    metric="euclidean",
+    *,
+    n_jobs=None,
+    force_all_finite="deprecated",
+    ensure_all_finite=None,
+    **kwds,
+):
+    """Compute the distance matrix from a feature array X and optional Y.
+
+    This function takes one or two feature arrays or a distance matrix, and returns
+    a distance matrix.
+
+    - If `X` is a feature array, of shape (n_samples_X, n_features), and:
+
+      - `Y` is `None` and `metric` is not 'precomputed', the pairwise distances
+        between `X` and itself are returned.
+      - `Y` is a feature array of shape (n_samples_Y, n_features), the pairwise
+        distances between `X` and `Y` is returned.
+
+    - If `X` is a distance matrix, of shape (n_samples_X, n_samples_X), `metric`
+      should be 'precomputed'. `Y` is thus ignored and `X` is returned as is.
+
+    If the input is a collection of non-numeric data (e.g. a list of strings or a
+    boolean array), a custom metric must be passed.
 
     This method provides a safe way to take a distance matrix as input, while
     preserving compatibility with many other algorithms that take a vector
     array.
 
-    If Y is given (default is None), then the returned matrix is the pairwise
-    distance between the arrays from both X and Y.
-
     Valid values for metric are:
 
     - From scikit-learn: ['cityblock', 'cosine', 'euclidean', 'l1', 'l2',
-      'manhattan']. These metrics support sparse matrix
-      inputs.
-      ['nan_euclidean'] but it does not yet support sparse matrices.
+      'manhattan', 'nan_euclidean']. All metrics support sparse matrix
+      inputs except 'nan_euclidean'.
 
     - From scipy.spatial.distance: ['braycurtis', 'canberra', 'chebyshev',
       'correlation', 'dice', 'hamming', 'jaccard', 'kulsinski', 'mahalanobis',
@@ -1694,24 +2311,30 @@ def pairwise_distances(X, Y=None, metric="euclidean", *, n_jobs=None,
       See the documentation for scipy.spatial.distance for details on these
       metrics. These metrics do not support sparse matrix inputs.
 
+    .. note::
+        `'kulsinski'` is deprecated from SciPy 1.9 and will be removed in SciPy 1.11.
+
+    .. note::
+        `'matching'` has been removed in SciPy 1.9 (use `'hamming'` instead).
+
     Note that in the case of 'cityblock', 'cosine' and 'euclidean' (which are
     valid scipy.spatial.distance metrics), the scikit-learn implementation
     will be used, which is faster and has support for sparse matrices (except
     for 'cityblock'). For a verbose description of the metrics from
-    scikit-learn, see the __doc__ of the sklearn.pairwise.distance_metrics
+    scikit-learn, see :func:`sklearn.metrics.pairwise.distance_metrics`
     function.
 
     Read more in the :ref:`User Guide `.
 
     Parameters
     ----------
-    X : ndarray of shape (n_samples_X, n_samples_X) or \
+    X : {array-like, sparse matrix} of shape (n_samples_X, n_samples_X) or \
             (n_samples_X, n_features)
         Array of pairwise distances between samples, or a feature array.
         The shape of the array should be (n_samples_X, n_samples_X) if
         metric == "precomputed" and (n_samples_X, n_features) otherwise.
 
-    Y : ndarray of shape (n_samples_Y, n_features), default=None
+    Y : {array-like, sparse matrix} of shape (n_samples_Y, n_features), default=None
         An optional second feature array. Only allowed if
         metric != "precomputed".
 
@@ -1728,13 +2351,17 @@ def pairwise_distances(X, Y=None, metric="euclidean", *, n_jobs=None,
 
     n_jobs : int, default=None
         The number of jobs to use for the computation. This works by breaking
-        down the pairwise matrix into n_jobs even slices and computing them in
-        parallel.
+        down the pairwise matrix into n_jobs even slices and computing them
+        using multithreading.
 
         ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
         ``-1`` means using all processors. See :term:`Glossary `
         for more details.
 
+        The "euclidean" and "cosine" metrics rely heavily on BLAS which is already
+        multithreaded. So, increasing `n_jobs` would likely cause oversubscription
+        and quickly degrade performance.
+
     force_all_finite : bool or 'allow-nan', default=True
         Whether to raise an error on np.inf, np.nan, pd.NA in array. Ignored
         for a metric listed in ``pairwise.PAIRWISE_DISTANCE_FUNCTIONS``. The
@@ -1751,6 +2378,23 @@ def pairwise_distances(X, Y=None, metric="euclidean", *, n_jobs=None,
         .. versionchanged:: 0.23
            Accepts `pd.NA` and converts it into `np.nan`.
 
+        .. deprecated:: 1.6
+           `force_all_finite` was renamed to `ensure_all_finite` and will be removed
+           in 1.8.
+
+    ensure_all_finite : bool or 'allow-nan', default=True
+        Whether to raise an error on np.inf, np.nan, pd.NA in array. Ignored
+        for a metric listed in ``pairwise.PAIRWISE_DISTANCE_FUNCTIONS``. The
+        possibilities are:
+
+        - True: Force all values of array to be finite.
+        - False: accepts np.inf, np.nan, pd.NA in array.
+        - 'allow-nan': accepts only np.nan and pd.NA values in array. Values
+          cannot be infinite.
+
+        .. versionadded:: 1.6
+           `force_all_finite` was renamed to `ensure_all_finite`.
+
     **kwds : optional keyword parameters
         Any further parameters are passed directly to the distance function.
         If using a scipy.spatial.distance metric, the parameters are still
@@ -1770,50 +2414,64 @@ def pairwise_distances(X, Y=None, metric="euclidean", *, n_jobs=None,
     pairwise_distances_chunked : Performs the same calculation as this
         function, but returns a generator of chunks of the distance matrix, in
         order to limit memory usage.
-    paired_distances : Computes the distances between corresponding elements
-        of two arrays.
+    sklearn.metrics.pairwise.paired_distances : Computes the distances between
+        corresponding elements of two arrays.
+
+    Notes
+    -----
+    If metric is a callable, no restrictions are placed on `X` and `Y` dimensions.
+
+    Examples
+    --------
+    >>> from sklearn.metrics.pairwise import pairwise_distances
+    >>> X = [[0, 0, 0], [1, 1, 1]]
+    >>> Y = [[1, 0, 0], [1, 1, 0]]
+    >>> pairwise_distances(X, Y, metric='sqeuclidean')
+    array([[1., 2.],
+           [2., 1.]])
     """
-    if (metric not in _VALID_METRICS and
-            not callable(metric) and metric != "precomputed"):
-        raise ValueError("Unknown metric %s. "
-                         "Valid metrics are %s, or 'precomputed', or a "
-                         "callable" % (metric, _VALID_METRICS))
+    ensure_all_finite = _deprecate_force_all_finite(force_all_finite, ensure_all_finite)
 
     if metric == "precomputed":
-        X, _ = check_pairwise_arrays(X, Y, precomputed=True,
-                                     force_all_finite=force_all_finite)
-
-        whom = ("`pairwise_distances`. Precomputed distance "
-                " need to have non-negative values.")
+        X, _ = check_pairwise_arrays(
+            X, Y, precomputed=True, ensure_all_finite=ensure_all_finite
+        )
+
+        whom = (
+            "`pairwise_distances`. Precomputed distance "
+            " need to have non-negative values."
+        )
         check_non_negative(X, whom=whom)
         return X
     elif metric in PAIRWISE_DISTANCE_FUNCTIONS:
         func = PAIRWISE_DISTANCE_FUNCTIONS[metric]
     elif callable(metric):
-        func = partial(_pairwise_callable, metric=metric,
-                       force_all_finite=force_all_finite, **kwds)
+        func = partial(
+            _pairwise_callable,
+            metric=metric,
+            ensure_all_finite=ensure_all_finite,
+            **kwds,
+        )
     else:
         if issparse(X) or issparse(Y):
-            raise TypeError("scipy distance metrics do not"
-                            " support sparse matrices.")
+            raise TypeError("scipy distance metrics do not support sparse matrices.")
 
-        dtype = bool if metric in PAIRWISE_BOOLEAN_FUNCTIONS else None
+        dtype = bool if metric in PAIRWISE_BOOLEAN_FUNCTIONS else "infer_float"
 
-        if (dtype == bool and
-                (X.dtype != bool or (Y is not None and Y.dtype != bool))):
+        if dtype is bool and (X.dtype != bool or (Y is not None and Y.dtype != bool)):
             msg = "Data was converted to boolean for metric %s" % metric
             warnings.warn(msg, DataConversionWarning)
 
-        X, Y = check_pairwise_arrays(X, Y, dtype=dtype,
-                                     force_all_finite=force_all_finite)
+        X, Y = check_pairwise_arrays(
+            X, Y, dtype=dtype, ensure_all_finite=ensure_all_finite
+        )
 
         # precompute data-derived metric params
         params = _precompute_metric_params(X, Y, metric=metric, **kwds)
         kwds.update(**params)
 
         if effective_n_jobs(n_jobs) == 1 and X is Y:
-            return distance.squareform(distance.pdist(X, metric=metric,
-                                                      **kwds))
+            return distance.squareform(distance.pdist(X, metric=metric, **kwds))
         func = partial(distance.cdist, metric=metric, **kwds)
 
     return _parallel_pairwise(X, Y, func, n_jobs, **kwds)
@@ -1821,30 +2479,37 @@ def pairwise_distances(X, Y=None, metric="euclidean", *, n_jobs=None,
 
 # These distances require boolean arrays, when using scipy.spatial.distance
 PAIRWISE_BOOLEAN_FUNCTIONS = [
-    'dice',
-    'jaccard',
-    'kulsinski',
-    'matching',
-    'rogerstanimoto',
-    'russellrao',
-    'sokalmichener',
-    'sokalsneath',
-    'yule',
+    "dice",
+    "jaccard",
+    "rogerstanimoto",
+    "russellrao",
+    "sokalsneath",
+    "yule",
 ]
+if sp_base_version < parse_version("1.17"):
+    # Deprecated in SciPy 1.15 and removed in SciPy 1.17
+    PAIRWISE_BOOLEAN_FUNCTIONS += ["sokalmichener"]
+if sp_base_version < parse_version("1.11"):
+    # Deprecated in SciPy 1.9 and removed in SciPy 1.11
+    PAIRWISE_BOOLEAN_FUNCTIONS += ["kulsinski"]
+if sp_base_version < parse_version("1.9"):
+    # Deprecated in SciPy 1.0 and removed in SciPy 1.9
+    PAIRWISE_BOOLEAN_FUNCTIONS += ["matching"]
 
 # Helper functions - distance
 PAIRWISE_KERNEL_FUNCTIONS = {
     # If updating this dictionary, update the doc in both distance_metrics()
     # and also in pairwise_distances()!
-    'additive_chi2': additive_chi2_kernel,
-    'chi2': chi2_kernel,
-    'linear': linear_kernel,
-    'polynomial': polynomial_kernel,
-    'poly': polynomial_kernel,
-    'rbf': rbf_kernel,
-    'laplacian': laplacian_kernel,
-    'sigmoid': sigmoid_kernel,
-    'cosine': cosine_similarity, }
+    "additive_chi2": additive_chi2_kernel,
+    "chi2": chi2_kernel,
+    "linear": linear_kernel,
+    "polynomial": polynomial_kernel,
+    "poly": polynomial_kernel,
+    "rbf": rbf_kernel,
+    "laplacian": laplacian_kernel,
+    "sigmoid": sigmoid_kernel,
+    "cosine": cosine_similarity,
+}
 
 
 def kernel_metrics():
@@ -1870,6 +2535,11 @@ def kernel_metrics():
       ===============   ========================================
 
     Read more in the :ref:`User Guide `.
+
+    Returns
+    -------
+    kernel_metrics : dict
+        Returns valid metrics for pairwise_kernels.
     """
     return PAIRWISE_KERNEL_FUNCTIONS
 
@@ -1887,22 +2557,41 @@ def kernel_metrics():
 }
 
 
-@_deprecate_positional_args
-def pairwise_kernels(X, Y=None, metric="linear", *, filter_params=False,
-                     n_jobs=None, **kwds):
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "Y": ["array-like", "sparse matrix", None],
+        "metric": [
+            StrOptions(set(PAIRWISE_KERNEL_FUNCTIONS) | {"precomputed"}),
+            callable,
+        ],
+        "filter_params": ["boolean"],
+        "n_jobs": [Integral, None],
+    },
+    prefer_skip_nested_validation=True,
+)
+def pairwise_kernels(
+    X, Y=None, metric="linear", *, filter_params=False, n_jobs=None, **kwds
+):
     """Compute the kernel between arrays X and optional array Y.
 
-    This method takes either a vector array or a kernel matrix, and returns
-    a kernel matrix. If the input is a vector array, the kernels are
-    computed. If the input is a kernel matrix, it is returned instead.
+    This function takes one or two feature arrays or a kernel matrix, and returns
+    a kernel matrix.
+
+    - If `X` is a feature array, of shape (n_samples_X, n_features), and:
+
+      - `Y` is `None` and `metric` is not 'precomputed', the pairwise kernels
+        between `X` and itself are returned.
+      - `Y` is a feature array of shape (n_samples_Y, n_features), the pairwise
+        kernels between `X` and `Y` is returned.
+
+    - If `X` is a kernel matrix, of shape (n_samples_X, n_samples_X), `metric`
+      should be 'precomputed'. `Y` is thus ignored and `X` is returned as is.
 
     This method provides a safe way to take a kernel matrix as input, while
     preserving compatibility with many other algorithms that take a vector
     array.
 
-    If Y is given (default is None), then the returned matrix is the pairwise
-    kernel between the arrays from both X and Y.
-
     Valid values for metric are:
         ['additive_chi2', 'chi2', 'linear', 'poly', 'polynomial', 'rbf',
         'laplacian', 'sigmoid', 'cosine']
@@ -1911,19 +2600,19 @@ def pairwise_kernels(X, Y=None, metric="linear", *, filter_params=False,
 
     Parameters
     ----------
-    X : ndarray of shape (n_samples_X, n_samples_X) or \
+    X : {array-like, sparse matrix}  of shape (n_samples_X, n_samples_X) or \
             (n_samples_X, n_features)
         Array of pairwise kernels between samples, or a feature array.
         The shape of the array should be (n_samples_X, n_samples_X) if
         metric == "precomputed" and (n_samples_X, n_features) otherwise.
 
-    Y : ndarray of shape (n_samples_Y, n_features), default=None
+    Y : {array-like, sparse matrix} of shape (n_samples_Y, n_features), default=None
         A second feature array only if X has shape (n_samples_X, n_features).
 
     metric : str or callable, default="linear"
         The metric to use when calculating kernel between instances in a
         feature array. If metric is a string, it must be one of the metrics
-        in pairwise.PAIRWISE_KERNEL_FUNCTIONS.
+        in ``pairwise.PAIRWISE_KERNEL_FUNCTIONS``.
         If metric is "precomputed", X is assumed to be a kernel matrix.
         Alternatively, if metric is a callable function, it is called on each
         pair of instances (rows) and the resulting value recorded. The callable
@@ -1938,8 +2627,8 @@ def pairwise_kernels(X, Y=None, metric="linear", *, filter_params=False,
 
     n_jobs : int, default=None
         The number of jobs to use for the computation. This works by breaking
-        down the pairwise matrix into n_jobs even slices and computing them in
-        parallel.
+        down the pairwise matrix into n_jobs even slices and computing them
+        using multithreading.
 
         ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
         ``-1`` means using all processors. See :term:`Glossary `
@@ -1950,8 +2639,7 @@ def pairwise_kernels(X, Y=None, metric="linear", *, filter_params=False,
 
     Returns
     -------
-    K : ndarray of shape (n_samples_X, n_samples_X) or \
-            (n_samples_X, n_samples_Y)
+    K : ndarray of shape (n_samples_X, n_samples_X) or (n_samples_X, n_samples_Y)
         A kernel matrix K such that K_{i, j} is the kernel between the
         ith and jth vectors of the given matrix X, if Y is None.
         If Y is not None, then K_{i, j} is the kernel between the ith array
@@ -1959,8 +2647,16 @@ def pairwise_kernels(X, Y=None, metric="linear", *, filter_params=False,
 
     Notes
     -----
-    If metric is 'precomputed', Y is ignored and X is returned.
+    If metric is a callable, no restrictions are placed on `X` and `Y` dimensions.
 
+    Examples
+    --------
+    >>> from sklearn.metrics.pairwise import pairwise_kernels
+    >>> X = [[0, 0, 0], [1, 1, 1]]
+    >>> Y = [[1, 0, 0], [1, 1, 0]]
+    >>> pairwise_kernels(X, Y, metric='linear')
+    array([[0., 0.],
+           [1., 2.]])
     """
     # import GPKernel locally to prevent circular imports
     from ..gaussian_process.kernels import Kernel as GPKernel
@@ -1972,12 +2668,9 @@ def pairwise_kernels(X, Y=None, metric="linear", *, filter_params=False,
         func = metric.__call__
     elif metric in PAIRWISE_KERNEL_FUNCTIONS:
         if filter_params:
-            kwds = {k: kwds[k] for k in kwds
-                    if k in KERNEL_PARAMS[metric]}
+            kwds = {k: kwds[k] for k in kwds if k in KERNEL_PARAMS[metric]}
         func = PAIRWISE_KERNEL_FUNCTIONS[metric]
     elif callable(metric):
         func = partial(_pairwise_callable, metric=metric, **kwds)
-    else:
-        raise ValueError("Unknown kernel %r" % metric)
 
     return _parallel_pairwise(X, Y, func, n_jobs, **kwds)
diff --git a/sklearn/metrics/setup.py b/sklearn/metrics/setup.py
deleted file mode 100644
index 07aa01da308b8..0000000000000
--- a/sklearn/metrics/setup.py
+++ /dev/null
@@ -1,28 +0,0 @@
-import os
-
-from numpy.distutils.misc_util import Configuration
-
-
-def configuration(parent_package="", top_path=None):
-    config = Configuration("metrics", parent_package, top_path)
-
-    libraries = []
-    if os.name == 'posix':
-        libraries.append('m')
-
-    config.add_subpackage('_plot')
-    config.add_subpackage('_plot.tests')
-    config.add_subpackage('cluster')
-
-    config.add_extension("_pairwise_fast",
-                         sources=["_pairwise_fast.pyx"],
-                         libraries=libraries)
-
-    config.add_subpackage('tests')
-
-    return config
-
-
-if __name__ == "__main__":
-    from numpy.distutils.core import setup
-    setup(**configuration().todict())
diff --git a/sklearn/metrics/tests/test_classification.py b/sklearn/metrics/tests/test_classification.py
index 7b634e88f2275..b66353e5ecfab 100644
--- a/sklearn/metrics/tests/test_classification.py
+++ b/sklearn/metrics/tests/test_classification.py
@@ -1,55 +1,55 @@
-
-from functools import partial
-from itertools import product
-from itertools import chain
-from itertools import permutations
-import warnings
 import re
+import warnings
+from functools import partial
+from itertools import chain, permutations, product
 
 import numpy as np
-from scipy import linalg
 import pytest
+from scipy import linalg
+from scipy.spatial.distance import hamming as sp_hamming
+from scipy.stats import bernoulli
 
-from sklearn import datasets
-from sklearn import svm
-
+from sklearn import datasets, svm
 from sklearn.datasets import make_multilabel_classification
-from sklearn.preprocessing import label_binarize, LabelBinarizer
-from sklearn.utils.validation import check_random_state
-from sklearn.utils._testing import assert_almost_equal
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils._testing import assert_array_almost_equal
-from sklearn.utils._testing import assert_allclose
-from sklearn.utils._testing import assert_warns_div0
-from sklearn.utils._testing import assert_no_warnings
-from sklearn.utils._testing import assert_warns_message
-from sklearn.utils._testing import ignore_warnings
-from sklearn.utils._mocking import MockDataFrame
-
-from sklearn.metrics import accuracy_score
-from sklearn.metrics import average_precision_score
-from sklearn.metrics import balanced_accuracy_score
-from sklearn.metrics import classification_report
-from sklearn.metrics import cohen_kappa_score
-from sklearn.metrics import confusion_matrix
-from sklearn.metrics import f1_score
-from sklearn.metrics import fbeta_score
-from sklearn.metrics import hamming_loss
-from sklearn.metrics import hinge_loss
-from sklearn.metrics import jaccard_score
-from sklearn.metrics import log_loss
-from sklearn.metrics import matthews_corrcoef
-from sklearn.metrics import precision_recall_fscore_support
-from sklearn.metrics import precision_score
-from sklearn.metrics import recall_score
-from sklearn.metrics import zero_one_loss
-from sklearn.metrics import brier_score_loss
-from sklearn.metrics import multilabel_confusion_matrix
-
-from sklearn.metrics._classification import _check_targets
 from sklearn.exceptions import UndefinedMetricWarning
-
-from scipy.spatial.distance import hamming as sp_hamming
+from sklearn.metrics import (
+    accuracy_score,
+    average_precision_score,
+    balanced_accuracy_score,
+    brier_score_loss,
+    class_likelihood_ratios,
+    classification_report,
+    cohen_kappa_score,
+    confusion_matrix,
+    f1_score,
+    fbeta_score,
+    hamming_loss,
+    hinge_loss,
+    jaccard_score,
+    log_loss,
+    make_scorer,
+    matthews_corrcoef,
+    multilabel_confusion_matrix,
+    precision_recall_fscore_support,
+    precision_score,
+    recall_score,
+    zero_one_loss,
+)
+from sklearn.metrics._classification import _check_targets, d2_log_loss_score
+from sklearn.model_selection import cross_val_score
+from sklearn.preprocessing import LabelBinarizer, label_binarize
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.utils._mocking import MockDataFrame
+from sklearn.utils._testing import (
+    assert_allclose,
+    assert_almost_equal,
+    assert_array_almost_equal,
+    assert_array_equal,
+    ignore_warnings,
+)
+from sklearn.utils.extmath import _nanaverage
+from sklearn.utils.fixes import CSC_CONTAINERS, CSR_CONTAINERS
+from sklearn.utils.validation import check_random_state
 
 ###############################################################################
 # Utilities for testing
@@ -86,114 +86,157 @@ def make_prediction(dataset=None, binary=False):
     X = np.c_[X, rng.randn(n_samples, 200 * n_features)]
 
     # run classifier, get class probabilities and label predictions
-    clf = svm.SVC(kernel='linear', probability=True, random_state=0)
-    probas_pred = clf.fit(X[:half], y[:half]).predict_proba(X[half:])
+    clf = svm.SVC(kernel="linear", probability=True, random_state=0)
+    y_pred_proba = clf.fit(X[:half], y[:half]).predict_proba(X[half:])
 
     if binary:
         # only interested in probabilities of the positive case
         # XXX: do we really want a special API for the binary case?
-        probas_pred = probas_pred[:, 1]
+        y_pred_proba = y_pred_proba[:, 1]
 
     y_pred = clf.predict(X[half:])
     y_true = y[half:]
-    return y_true, y_pred, probas_pred
+    return y_true, y_pred, y_pred_proba
 
 
 ###############################################################################
 # Tests
 
-def test_classification_report_dictionary_output():
 
+def test_classification_report_dictionary_output():
     # Test performance report with dictionary output
     iris = datasets.load_iris()
     y_true, y_pred, _ = make_prediction(dataset=iris, binary=False)
 
     # print classification report with class names
-    expected_report = {'setosa': {'precision': 0.82608695652173914,
-                                  'recall': 0.79166666666666663,
-                                  'f1-score': 0.8085106382978724,
-                                  'support': 24},
-                       'versicolor': {'precision': 0.33333333333333331,
-                                      'recall': 0.096774193548387094,
-                                      'f1-score': 0.15000000000000002,
-                                      'support': 31},
-                       'virginica': {'precision': 0.41860465116279072,
-                                     'recall': 0.90000000000000002,
-                                     'f1-score': 0.57142857142857151,
-                                     'support': 20},
-                       'macro avg': {'f1-score': 0.5099797365754813,
-                                     'precision': 0.5260083136726211,
-                                     'recall': 0.596146953405018,
-                                     'support': 75},
-                       'accuracy': 0.5333333333333333,
-                       'weighted avg': {'f1-score': 0.47310435663627154,
-                                        'precision': 0.5137535108414785,
-                                        'recall': 0.5333333333333333,
-                                        'support': 75}}
+    expected_report = {
+        "setosa": {
+            "precision": 0.82608695652173914,
+            "recall": 0.79166666666666663,
+            "f1-score": 0.8085106382978724,
+            "support": 24,
+        },
+        "versicolor": {
+            "precision": 0.33333333333333331,
+            "recall": 0.096774193548387094,
+            "f1-score": 0.15000000000000002,
+            "support": 31,
+        },
+        "virginica": {
+            "precision": 0.41860465116279072,
+            "recall": 0.90000000000000002,
+            "f1-score": 0.57142857142857151,
+            "support": 20,
+        },
+        "macro avg": {
+            "f1-score": 0.5099797365754813,
+            "precision": 0.5260083136726211,
+            "recall": 0.596146953405018,
+            "support": 75,
+        },
+        "accuracy": 0.5333333333333333,
+        "weighted avg": {
+            "f1-score": 0.47310435663627154,
+            "precision": 0.5137535108414785,
+            "recall": 0.5333333333333333,
+            "support": 75,
+        },
+    }
 
     report = classification_report(
-        y_true, y_pred, labels=np.arange(len(iris.target_names)),
-        target_names=iris.target_names, output_dict=True)
+        y_true,
+        y_pred,
+        labels=np.arange(len(iris.target_names)),
+        target_names=iris.target_names,
+        output_dict=True,
+    )
 
     # assert the 2 dicts are equal.
-    assert(report.keys() == expected_report.keys())
+    assert report.keys() == expected_report.keys()
     for key in expected_report:
-        if key == 'accuracy':
+        if key == "accuracy":
             assert isinstance(report[key], float)
             assert report[key] == expected_report[key]
         else:
             assert report[key].keys() == expected_report[key].keys()
             for metric in expected_report[key]:
-                assert_almost_equal(expected_report[key][metric],
-                                    report[key][metric])
+                assert_almost_equal(expected_report[key][metric], report[key][metric])
 
-    assert type(expected_report['setosa']['precision']) == float
-    assert type(expected_report['macro avg']['precision']) == float
-    assert type(expected_report['setosa']['support']) == int
-    assert type(expected_report['macro avg']['support']) == int
+    assert isinstance(expected_report["setosa"]["precision"], float)
+    assert isinstance(expected_report["macro avg"]["precision"], float)
+    assert isinstance(expected_report["setosa"]["support"], int)
+    assert isinstance(expected_report["macro avg"]["support"], int)
 
 
 def test_classification_report_output_dict_empty_input():
     report = classification_report(y_true=[], y_pred=[], output_dict=True)
-    expected_report = {'accuracy': 0.0,
-                       'macro avg': {'f1-score': np.nan,
-                                     'precision': np.nan,
-                                     'recall': np.nan,
-                                     'support': 0},
-                       'weighted avg': {'f1-score': 0.0,
-                                        'precision': 0.0,
-                                        'recall': 0.0,
-                                        'support': 0}}
+    expected_report = {
+        "accuracy": 0.0,
+        "macro avg": {
+            "f1-score": np.nan,
+            "precision": np.nan,
+            "recall": np.nan,
+            "support": 0,
+        },
+        "weighted avg": {
+            "f1-score": np.nan,
+            "precision": np.nan,
+            "recall": np.nan,
+            "support": 0,
+        },
+    }
     assert isinstance(report, dict)
     # assert the 2 dicts are equal.
     assert report.keys() == expected_report.keys()
     for key in expected_report:
-        if key == 'accuracy':
+        if key == "accuracy":
             assert isinstance(report[key], float)
             assert report[key] == expected_report[key]
         else:
             assert report[key].keys() == expected_report[key].keys()
             for metric in expected_report[key]:
-                assert_almost_equal(expected_report[key][metric],
-                                    report[key][metric])
+                assert_almost_equal(expected_report[key][metric], report[key][metric])
 
 
-@pytest.mark.parametrize('zero_division', ["warn", 0, 1])
+@pytest.mark.parametrize("zero_division", ["warn", 0, 1, np.nan])
 def test_classification_report_zero_division_warning(zero_division):
     y_true, y_pred = ["a", "b", "c"], ["a", "b", "d"]
     with warnings.catch_warnings(record=True) as record:
         classification_report(
-            y_true, y_pred, zero_division=zero_division, output_dict=True)
+            y_true, y_pred, zero_division=zero_division, output_dict=True
+        )
         if zero_division == "warn":
             assert len(record) > 1
             for item in record:
-                msg = ("Use `zero_division` parameter to control this "
-                       "behavior.")
+                msg = "Use `zero_division` parameter to control this behavior."
                 assert msg in str(item.message)
         else:
             assert not record
 
 
+@pytest.mark.parametrize(
+    "labels, show_micro_avg", [([0], True), ([0, 1], False), ([0, 1, 2], False)]
+)
+def test_classification_report_labels_subset_superset(labels, show_micro_avg):
+    """Check the behaviour of passing `labels` as a superset or subset of the labels.
+    WHen a superset, we expect to show the "accuracy" in the report while it should be
+    the micro-averaging if this is a subset.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/27927
+    """
+
+    y_true, y_pred = [0, 1], [0, 1]
+
+    report = classification_report(y_true, y_pred, labels=labels, output_dict=True)
+    if show_micro_avg:
+        assert "micro avg" in report
+        assert "accuracy" not in report
+    else:  # accuracy should be shown
+        assert "accuracy" in report
+        assert "micro avg" not in report
+
+
 def test_multilabel_accuracy_score_subset_accuracy():
     # Dense label indicator matrix format
     y1 = np.array([[0, 1, 1], [1, 0, 1]])
@@ -222,182 +265,218 @@ def test_precision_recall_f1_score_binary():
     # individual scoring function that can be used for grid search: in the
     # binary class case the score is the value of the measure for the positive
     # class (e.g. label == 1). This is deprecated for average != 'binary'.
-    for kwargs, my_assert in [({}, assert_no_warnings),
-                              ({'average': 'binary'}, assert_no_warnings)]:
-        ps = my_assert(precision_score, y_true, y_pred, **kwargs)
-        assert_array_almost_equal(ps, 0.85, 2)
+    for kwargs in [{}, {"average": "binary"}]:
+        with warnings.catch_warnings():
+            warnings.simplefilter("error")
+
+            ps = precision_score(y_true, y_pred, **kwargs)
+            assert_array_almost_equal(ps, 0.85, 2)
 
-        rs = my_assert(recall_score, y_true, y_pred, **kwargs)
-        assert_array_almost_equal(rs, 0.68, 2)
+            rs = recall_score(y_true, y_pred, **kwargs)
+            assert_array_almost_equal(rs, 0.68, 2)
 
-        fs = my_assert(f1_score, y_true, y_pred, **kwargs)
-        assert_array_almost_equal(fs, 0.76, 2)
+            fs = f1_score(y_true, y_pred, **kwargs)
+            assert_array_almost_equal(fs, 0.76, 2)
 
-        assert_almost_equal(my_assert(fbeta_score, y_true, y_pred, beta=2,
-                                      **kwargs),
-                            (1 + 2 ** 2) * ps * rs / (2 ** 2 * ps + rs), 2)
+            assert_almost_equal(
+                fbeta_score(y_true, y_pred, beta=2, **kwargs),
+                (1 + 2**2) * ps * rs / (2**2 * ps + rs),
+                2,
+            )
 
 
-@ignore_warnings
+@pytest.mark.filterwarnings(r"ignore::sklearn.exceptions.UndefinedMetricWarning")
 def test_precision_recall_f_binary_single_class():
     # Test precision, recall and F-scores behave with a single positive or
     # negative class
     # Such a case may occur with non-stratified cross-validation
-    assert 1. == precision_score([1, 1], [1, 1])
-    assert 1. == recall_score([1, 1], [1, 1])
-    assert 1. == f1_score([1, 1], [1, 1])
-    assert 1. == fbeta_score([1, 1], [1, 1], 0)
-
-    assert 0. == precision_score([-1, -1], [-1, -1])
-    assert 0. == recall_score([-1, -1], [-1, -1])
-    assert 0. == f1_score([-1, -1], [-1, -1])
-    assert 0. == fbeta_score([-1, -1], [-1, -1], float('inf'))
-    assert fbeta_score([-1, -1], [-1, -1], float('inf')) == pytest.approx(
-        fbeta_score([-1, -1], [-1, -1], beta=1e5))
+    assert 1.0 == precision_score([1, 1], [1, 1])
+    assert 1.0 == recall_score([1, 1], [1, 1])
+    assert 1.0 == f1_score([1, 1], [1, 1])
+    assert 1.0 == fbeta_score([1, 1], [1, 1], beta=0)
+
+    assert 0.0 == precision_score([-1, -1], [-1, -1])
+    assert 0.0 == recall_score([-1, -1], [-1, -1])
+    assert 0.0 == f1_score([-1, -1], [-1, -1])
+    assert 0.0 == fbeta_score([-1, -1], [-1, -1], beta=float("inf"))
+    assert fbeta_score([-1, -1], [-1, -1], beta=float("inf")) == pytest.approx(
+        fbeta_score([-1, -1], [-1, -1], beta=1e5)
+    )
 
 
-@ignore_warnings
+@pytest.mark.filterwarnings(r"ignore::sklearn.exceptions.UndefinedMetricWarning")
 def test_precision_recall_f_extra_labels():
     # Test handling of explicit additional (not in input) labels to PRF
     y_true = [1, 3, 3, 2]
     y_pred = [1, 1, 3, 2]
     y_true_bin = label_binarize(y_true, classes=np.arange(5))
     y_pred_bin = label_binarize(y_pred, classes=np.arange(5))
-    data = [(y_true, y_pred),
-            (y_true_bin, y_pred_bin)]
+    data = [(y_true, y_pred), (y_true_bin, y_pred_bin)]
 
     for i, (y_true, y_pred) in enumerate(data):
         # No average: zeros in array
-        actual = recall_score(y_true, y_pred, labels=[0, 1, 2, 3, 4],
-                              average=None)
-        assert_array_almost_equal([0., 1., 1., .5, 0.], actual)
+        actual = recall_score(y_true, y_pred, labels=[0, 1, 2, 3, 4], average=None)
+        assert_array_almost_equal([0.0, 1.0, 1.0, 0.5, 0.0], actual)
 
         # Macro average is changed
-        actual = recall_score(y_true, y_pred, labels=[0, 1, 2, 3, 4],
-                              average='macro')
-        assert_array_almost_equal(np.mean([0., 1., 1., .5, 0.]), actual)
+        actual = recall_score(y_true, y_pred, labels=[0, 1, 2, 3, 4], average="macro")
+        assert_array_almost_equal(np.mean([0.0, 1.0, 1.0, 0.5, 0.0]), actual)
 
-        # No effect otheriwse
-        for average in ['micro', 'weighted', 'samples']:
-            if average == 'samples' and i == 0:
+        # No effect otherwise
+        for average in ["micro", "weighted", "samples"]:
+            if average == "samples" and i == 0:
                 continue
-            assert_almost_equal(recall_score(y_true, y_pred,
-                                             labels=[0, 1, 2, 3, 4],
-                                             average=average),
-                                recall_score(y_true, y_pred, labels=None,
-                                             average=average))
+            assert_almost_equal(
+                recall_score(y_true, y_pred, labels=[0, 1, 2, 3, 4], average=average),
+                recall_score(y_true, y_pred, labels=None, average=average),
+            )
 
     # Error when introducing invalid label in multilabel case
     # (although it would only affect performance if average='macro'/None)
-    for average in [None, 'macro', 'micro', 'samples']:
+    for average in [None, "macro", "micro", "samples"]:
         with pytest.raises(ValueError):
-            recall_score(y_true_bin, y_pred_bin, labels=np.arange(6),
-                         average=average)
+            recall_score(y_true_bin, y_pred_bin, labels=np.arange(6), average=average)
         with pytest.raises(ValueError):
-            recall_score(y_true_bin, y_pred_bin, labels=np.arange(-1, 4),
-                         average=average)
+            recall_score(
+                y_true_bin, y_pred_bin, labels=np.arange(-1, 4), average=average
+            )
 
     # tests non-regression on issue #10307
     y_true = np.array([[0, 1, 1], [1, 0, 0]])
     y_pred = np.array([[1, 1, 1], [1, 0, 1]])
-    p, r, f, _ = precision_recall_fscore_support(y_true, y_pred,
-                                                 average='samples',
-                                                 labels=[0, 1])
+    p, r, f, _ = precision_recall_fscore_support(
+        y_true, y_pred, average="samples", labels=[0, 1]
+    )
     assert_almost_equal(np.array([p, r, f]), np.array([3 / 4, 1, 5 / 6]))
 
 
-@ignore_warnings
+@pytest.mark.filterwarnings(r"ignore::sklearn.exceptions.UndefinedMetricWarning")
 def test_precision_recall_f_ignored_labels():
     # Test a subset of labels may be requested for PRF
     y_true = [1, 1, 2, 3]
     y_pred = [1, 3, 3, 3]
     y_true_bin = label_binarize(y_true, classes=np.arange(5))
     y_pred_bin = label_binarize(y_pred, classes=np.arange(5))
-    data = [(y_true, y_pred),
-            (y_true_bin, y_pred_bin)]
+    data = [(y_true, y_pred), (y_true_bin, y_pred_bin)]
 
     for i, (y_true, y_pred) in enumerate(data):
         recall_13 = partial(recall_score, y_true, y_pred, labels=[1, 3])
         recall_all = partial(recall_score, y_true, y_pred, labels=None)
 
-        assert_array_almost_equal([.5, 1.], recall_13(average=None))
-        assert_almost_equal((.5 + 1.) / 2, recall_13(average='macro'))
-        assert_almost_equal((.5 * 2 + 1. * 1) / 3,
-                            recall_13(average='weighted'))
-        assert_almost_equal(2. / 3, recall_13(average='micro'))
+        assert_array_almost_equal([0.5, 1.0], recall_13(average=None))
+        assert_almost_equal((0.5 + 1.0) / 2, recall_13(average="macro"))
+        assert_almost_equal((0.5 * 2 + 1.0 * 1) / 3, recall_13(average="weighted"))
+        assert_almost_equal(2.0 / 3, recall_13(average="micro"))
 
         # ensure the above were meaningful tests:
-        for average in ['macro', 'weighted', 'micro']:
-            assert (recall_13(average=average) !=
-                    recall_all(average=average))
-
-
-def test_average_precision_score_score_non_binary_class():
-    # Test that average_precision_score function returns an error when trying
-    # to compute average_precision_score for multiclass task.
-    rng = check_random_state(404)
-    y_pred = rng.rand(10)
-
-    # y_true contains three different class values
-    y_true = rng.randint(0, 3, size=10)
-    err_msg = "multiclass format is not supported"
+        for average in ["macro", "weighted", "micro"]:
+            assert recall_13(average=average) != recall_all(average=average)
+
+
+def test_average_precision_score_non_binary_class():
+    """Test multiclass-multiouptut for `average_precision_score`."""
+    y_true = np.array(
+        [
+            [2, 2, 1],
+            [1, 2, 0],
+            [0, 1, 2],
+            [1, 2, 1],
+            [2, 0, 1],
+            [1, 2, 1],
+        ]
+    )
+    y_score = np.array(
+        [
+            [0.7, 0.2, 0.1],
+            [0.4, 0.3, 0.3],
+            [0.1, 0.8, 0.1],
+            [0.2, 0.3, 0.5],
+            [0.4, 0.4, 0.2],
+            [0.1, 0.2, 0.7],
+        ]
+    )
+    err_msg = "multiclass-multioutput format is not supported"
     with pytest.raises(ValueError, match=err_msg):
-        average_precision_score(y_true, y_pred)
+        average_precision_score(y_true, y_score, pos_label=2)
 
 
-def test_average_precision_score_duplicate_values():
-    # Duplicate values with precision-recall require a different
-    # processing than when computing the AUC of a ROC, because the
-    # precision-recall curve is a decreasing curve
-    # The following situation corresponds to a perfect
-    # test statistic, the average_precision_score should be 1
-    y_true = [0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1]
-    y_score = [0, .1, .1, .4, .5, .6, .6, .9, .9, 1, 1]
+@pytest.mark.parametrize(
+    "y_true, y_score",
+    [
+        (
+            [0, 0, 1, 2],
+            np.array(
+                [
+                    [0.7, 0.2, 0.1],
+                    [0.4, 0.3, 0.3],
+                    [0.1, 0.8, 0.1],
+                    [0.2, 0.3, 0.5],
+                ]
+            ),
+        ),
+        (
+            [0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1],
+            [0, 0.1, 0.1, 0.4, 0.5, 0.6, 0.6, 0.9, 0.9, 1, 1],
+        ),
+    ],
+)
+def test_average_precision_score_duplicate_values(y_true, y_score):
+    """
+    Duplicate values with precision-recall require a different
+    processing than when computing the AUC of a ROC, because the
+    precision-recall curve is a decreasing curve
+    The following situation corresponds to a perfect
+    test statistic, the average_precision_score should be 1.
+    """
     assert average_precision_score(y_true, y_score) == 1
 
 
-def test_average_precision_score_tied_values():
+@pytest.mark.parametrize(
+    "y_true, y_score",
+    [
+        (
+            [2, 2, 1, 1, 0],
+            np.array(
+                [
+                    [0.2, 0.3, 0.5],
+                    [0.2, 0.3, 0.5],
+                    [0.4, 0.5, 0.3],
+                    [0.4, 0.5, 0.3],
+                    [0.8, 0.5, 0.3],
+                ]
+            ),
+        ),
+        (
+            [0, 1, 1],
+            [0.5, 0.5, 0.6],
+        ),
+    ],
+)
+def test_average_precision_score_tied_values(y_true, y_score):
     # Here if we go from left to right in y_true, the 0 values are
-    # are separated from the 1 values, so it appears that we've
-    # Correctly sorted our classifications. But in fact the first two
+    # separated from the 1 values, so it appears that we've
+    # correctly sorted our classifications. But in fact the first two
     # values have the same score (0.5) and so the first two values
     # could be swapped around, creating an imperfect sorting. This
     # imperfection should come through in the end score, making it less
     # than one.
-    y_true = [0, 1, 1]
-    y_score = [.5, .5, .6]
-    assert average_precision_score(y_true, y_score) != 1.
-
-
-@ignore_warnings
-def test_precision_recall_fscore_support_errors():
-    y_true, y_pred, _ = make_prediction(binary=True)
-
-    # Bad beta
-    with pytest.raises(ValueError):
-        precision_recall_fscore_support(y_true, y_pred, beta=-0.1)
-
-    # Bad pos_label
-    with pytest.raises(ValueError):
-        precision_recall_fscore_support(y_true, y_pred,
-                                        pos_label=2,
-                                        average='binary')
-
-    # Bad average option
-    with pytest.raises(ValueError):
-        precision_recall_fscore_support([0, 1, 2], [1, 2, 0],
-                                        average='mega')
+    assert average_precision_score(y_true, y_score) != 1.0
 
 
 def test_precision_recall_f_unused_pos_label():
     # Check warning that pos_label unused when set to non-default value
     # but average != 'binary'; even if data is binary.
-    assert_warns_message(UserWarning,
-                         "Note that pos_label (set to 2) is "
-                         "ignored when average != 'binary' (got 'macro'). You "
-                         "may use labels=[pos_label] to specify a single "
-                         "positive class.", precision_recall_fscore_support,
-                         [1, 2, 1], [1, 2, 2], pos_label=2, average='macro')
+
+    msg = (
+        r"Note that pos_label \(set to 2\) is "
+        r"ignored when average != 'binary' \(got 'macro'\). You "
+        r"may use labels=\[pos_label\] to specify a single "
+        "positive class."
+    )
+    with pytest.warns(UserWarning, match=msg):
+        precision_recall_fscore_support(
+            [1, 2, 1], [1, 2, 2], pos_label=2, average="macro"
+        )
 
 
 def test_confusion_matrix_binary():
@@ -409,7 +488,7 @@ def test(y_true, y_pred):
         assert_array_equal(cm, [[22, 3], [8, 17]])
 
         tp, fp, fn, tn = cm.flatten()
-        num = (tp * tn - fp * fn)
+        num = tp * tn - fp * fn
         den = np.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))
 
         true_mcc = 0 if den == 0 else num / den
@@ -418,8 +497,7 @@ def test(y_true, y_pred):
         assert_array_almost_equal(mcc, 0.57, decimal=2)
 
     test(y_true, y_pred)
-    test([str(y) for y in y_true],
-         [str(y) for y in y_pred])
+    test([str(y) for y in y_true], [str(y) for y in y_pred])
 
 
 def test_multilabel_confusion_matrix_binary():
@@ -428,12 +506,10 @@ def test_multilabel_confusion_matrix_binary():
 
     def test(y_true, y_pred):
         cm = multilabel_confusion_matrix(y_true, y_pred)
-        assert_array_equal(cm, [[[17, 8], [3, 22]],
-                                [[22, 3], [8, 17]]])
+        assert_array_equal(cm, [[[17, 8], [3, 22]], [[22, 3], [8, 17]]])
 
     test(y_true, y_pred)
-    test([str(y) for y in y_true],
-         [str(y) for y in y_pred])
+    test([str(y) for y in y_true], [str(y) for y in y_pred])
 
 
 def test_multilabel_confusion_matrix_multiclass():
@@ -443,47 +519,49 @@ def test_multilabel_confusion_matrix_multiclass():
     def test(y_true, y_pred, string_type=False):
         # compute confusion matrix with default labels introspection
         cm = multilabel_confusion_matrix(y_true, y_pred)
-        assert_array_equal(cm, [[[47, 4], [5, 19]],
-                                [[38, 6], [28, 3]],
-                                [[30, 25], [2, 18]]])
+        assert_array_equal(
+            cm, [[[47, 4], [5, 19]], [[38, 6], [28, 3]], [[30, 25], [2, 18]]]
+        )
 
         # compute confusion matrix with explicit label ordering
-        labels = ['0', '2', '1'] if string_type else [0, 2, 1]
+        labels = ["0", "2", "1"] if string_type else [0, 2, 1]
         cm = multilabel_confusion_matrix(y_true, y_pred, labels=labels)
-        assert_array_equal(cm, [[[47, 4], [5, 19]],
-                                [[30, 25], [2, 18]],
-                                [[38, 6], [28, 3]]])
+        assert_array_equal(
+            cm, [[[47, 4], [5, 19]], [[30, 25], [2, 18]], [[38, 6], [28, 3]]]
+        )
 
         # compute confusion matrix with super set of present labels
-        labels = ['0', '2', '1', '3'] if string_type else [0, 2, 1, 3]
+        labels = ["0", "2", "1", "3"] if string_type else [0, 2, 1, 3]
         cm = multilabel_confusion_matrix(y_true, y_pred, labels=labels)
-        assert_array_equal(cm, [[[47, 4], [5, 19]],
-                                [[30, 25], [2, 18]],
-                                [[38, 6], [28, 3]],
-                                [[75, 0], [0, 0]]])
+        assert_array_equal(
+            cm,
+            [
+                [[47, 4], [5, 19]],
+                [[30, 25], [2, 18]],
+                [[38, 6], [28, 3]],
+                [[75, 0], [0, 0]],
+            ],
+        )
 
     test(y_true, y_pred)
-    test(list(str(y) for y in y_true),
-         list(str(y) for y in y_pred),
-         string_type=True)
+    test([str(y) for y in y_true], [str(y) for y in y_pred], string_type=True)
 
 
-def test_multilabel_confusion_matrix_multilabel():
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_multilabel_confusion_matrix_multilabel(csc_container, csr_container):
     # Test multilabel confusion matrix - multilabel-indicator case
-    from scipy.sparse import csc_matrix, csr_matrix
 
     y_true = np.array([[1, 0, 1], [0, 1, 0], [1, 1, 0]])
     y_pred = np.array([[1, 0, 0], [0, 1, 1], [0, 0, 1]])
-    y_true_csr = csr_matrix(y_true)
-    y_pred_csr = csr_matrix(y_pred)
-    y_true_csc = csc_matrix(y_true)
-    y_pred_csc = csc_matrix(y_pred)
+    y_true_csr = csr_container(y_true)
+    y_pred_csr = csr_container(y_pred)
+    y_true_csc = csc_container(y_true)
+    y_pred_csc = csc_container(y_pred)
 
     # cross test different types
     sample_weight = np.array([2, 1, 3])
-    real_cm = [[[1, 0], [1, 1]],
-               [[1, 0], [1, 1]],
-               [[0, 2], [1, 0]]]
+    real_cm = [[[1, 0], [1, 1]], [[1, 0], [1, 1]], [[0, 2], [1, 0]]]
     trues = [y_true, y_true_csr, y_true_csc]
     preds = [y_pred, y_pred_csr, y_pred_csc]
 
@@ -494,29 +572,21 @@ def test_multilabel_confusion_matrix_multilabel():
 
     # test support for samplewise
     cm = multilabel_confusion_matrix(y_true, y_pred, samplewise=True)
-    assert_array_equal(cm, [[[1, 0], [1, 1]],
-                            [[1, 1], [0, 1]],
-                            [[0, 1], [2, 0]]])
+    assert_array_equal(cm, [[[1, 0], [1, 1]], [[1, 1], [0, 1]], [[0, 1], [2, 0]]])
 
     # test support for labels
     cm = multilabel_confusion_matrix(y_true, y_pred, labels=[2, 0])
-    assert_array_equal(cm, [[[0, 2], [1, 0]],
-                            [[1, 0], [1, 1]]])
+    assert_array_equal(cm, [[[0, 2], [1, 0]], [[1, 0], [1, 1]]])
 
     # test support for labels with samplewise
-    cm = multilabel_confusion_matrix(y_true, y_pred, labels=[2, 0],
-                                     samplewise=True)
-    assert_array_equal(cm, [[[0, 0], [1, 1]],
-                            [[1, 1], [0, 0]],
-                            [[0, 1], [1, 0]]])
+    cm = multilabel_confusion_matrix(y_true, y_pred, labels=[2, 0], samplewise=True)
+    assert_array_equal(cm, [[[0, 0], [1, 1]], [[1, 1], [0, 0]], [[0, 1], [1, 0]]])
 
     # test support for sample_weight with sample_wise
-    cm = multilabel_confusion_matrix(y_true, y_pred,
-                                     sample_weight=sample_weight,
-                                     samplewise=True)
-    assert_array_equal(cm, [[[2, 0], [2, 2]],
-                            [[1, 1], [0, 1]],
-                            [[0, 3], [6, 0]]])
+    cm = multilabel_confusion_matrix(
+        y_true, y_pred, sample_weight=sample_weight, samplewise=True
+    )
+    assert_array_equal(cm, [[[2, 0], [2, 2]], [[1, 1], [0, 1]], [[0, 3], [6, 0]]])
 
 
 def test_multilabel_confusion_matrix_errors():
@@ -527,10 +597,9 @@ def test_multilabel_confusion_matrix_errors():
     with pytest.raises(ValueError, match="inconsistent numbers of samples"):
         multilabel_confusion_matrix(y_true, y_pred, sample_weight=[1, 2])
     with pytest.raises(ValueError, match="should be a 1d array"):
-        multilabel_confusion_matrix(y_true, y_pred,
-                                    sample_weight=[[1, 2, 3],
-                                                   [2, 3, 4],
-                                                   [3, 4, 5]])
+        multilabel_confusion_matrix(
+            y_true, y_pred, sample_weight=[[1, 2, 3], [2, 3, 4], [3, 4, 5]]
+        )
 
     # Bad labels
     err_msg = r"All labels must be in \[0, n labels\)"
@@ -547,16 +616,17 @@ def test_multilabel_confusion_matrix_errors():
     # Bad y_type
     err_msg = "multiclass-multioutput is not supported"
     with pytest.raises(ValueError, match=err_msg):
-        multilabel_confusion_matrix([[0, 1, 2], [2, 1, 0]],
-                                    [[1, 2, 0], [1, 0, 2]])
+        multilabel_confusion_matrix([[0, 1, 2], [2, 1, 0]], [[1, 2, 0], [1, 0, 2]])
 
 
 @pytest.mark.parametrize(
     "normalize, cm_dtype, expected_results",
-    [('true', 'f', 0.333333333),
-     ('pred', 'f', 0.333333333),
-     ('all', 'f', 0.1111111111),
-     (None, 'i', 2)]
+    [
+        ("true", "f", 0.333333333),
+        ("pred", "f", 0.333333333),
+        ("all", "f", 0.1111111111),
+        (None, "i", 2),
+    ],
 )
 def test_confusion_matrix_normalize(normalize, cm_dtype, expected_results):
     y_test = [0, 1, 2] * 6
@@ -566,29 +636,244 @@ def test_confusion_matrix_normalize(normalize, cm_dtype, expected_results):
     assert cm.dtype.kind == cm_dtype
 
 
-def test_confusion_matrix_normalize_wrong_option():
-    y_test = [0, 0, 0, 0, 1, 1, 1, 1]
-    y_pred = [0, 0, 0, 0, 0, 0, 0, 0]
-    with pytest.raises(ValueError, match='normalize must be one of'):
-        confusion_matrix(y_test, y_pred, normalize=True)
-
-
 def test_confusion_matrix_normalize_single_class():
     y_test = [0, 0, 0, 0, 1, 1, 1, 1]
     y_pred = [0, 0, 0, 0, 0, 0, 0, 0]
 
-    cm_true = confusion_matrix(y_test, y_pred, normalize='true')
+    cm_true = confusion_matrix(y_test, y_pred, normalize="true")
     assert cm_true.sum() == pytest.approx(2.0)
 
     # additionally check that no warnings are raised due to a division by zero
-    with pytest.warns(None) as rec:
-        cm_pred = confusion_matrix(y_test, y_pred, normalize='pred')
-    assert not rec
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", RuntimeWarning)
+        cm_pred = confusion_matrix(y_test, y_pred, normalize="pred")
+
     assert cm_pred.sum() == pytest.approx(1.0)
 
-    with pytest.warns(None) as rec:
-        cm_pred = confusion_matrix(y_pred, y_test, normalize='true')
-    assert not rec
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", RuntimeWarning)
+        confusion_matrix(y_pred, y_test, normalize="true")
+
+
+def test_confusion_matrix_single_label():
+    """Test `confusion_matrix` warns when only one label found."""
+    y_test = [0, 0, 0, 0]
+    y_pred = [0, 0, 0, 0]
+
+    with pytest.warns(UserWarning, match="A single label was found in"):
+        confusion_matrix(y_pred, y_test)
+
+
+@pytest.mark.parametrize(
+    "params, warn_msg",
+    [
+        # When `fp == 0` and `tp != 0`, LR+ is undefined
+        (
+            {
+                "y_true": np.array([1, 1, 1, 0, 0, 0]),
+                "y_pred": np.array([1, 1, 1, 0, 0, 0]),
+            },
+            "`positive_likelihood_ratio` is ill-defined and set to `np.nan`.",
+        ),
+        # When `fp == 0` and `tp == 0`, LR+ is undefined
+        (
+            {
+                "y_true": np.array([1, 1, 1, 0, 0, 0]),
+                "y_pred": np.array([0, 0, 0, 0, 0, 0]),
+            },
+            (
+                "No samples were predicted for the positive class and "
+                "`positive_likelihood_ratio` is set to `np.nan`."
+            ),
+        ),
+        # When `tn == 0`, LR- is undefined
+        (
+            {
+                "y_true": np.array([1, 1, 1, 0, 0, 0]),
+                "y_pred": np.array([0, 0, 0, 1, 1, 1]),
+            },
+            "`negative_likelihood_ratio` is ill-defined and set to `np.nan`.",
+        ),
+        # When `tp + fn == 0` both ratios are undefined
+        (
+            {
+                "y_true": np.array([0, 0, 0, 0, 0, 0]),
+                "y_pred": np.array([1, 1, 1, 0, 0, 0]),
+            },
+            "No samples of the positive class are present in `y_true`.",
+        ),
+    ],
+)
+def test_likelihood_ratios_warnings(params, warn_msg):
+    # likelihood_ratios must raise warnings when at
+    # least one of the ratios is ill-defined.
+
+    with pytest.warns(UserWarning, match=warn_msg):
+        class_likelihood_ratios(**params)
+
+
+@pytest.mark.parametrize(
+    "params, err_msg",
+    [
+        (
+            {
+                "y_true": np.array([0, 1, 0, 1, 0]),
+                "y_pred": np.array([1, 1, 0, 0, 2]),
+            },
+            (
+                "class_likelihood_ratios only supports binary classification "
+                "problems, got targets of type: multiclass"
+            ),
+        ),
+    ],
+)
+def test_likelihood_ratios_errors(params, err_msg):
+    # likelihood_ratios must raise error when attempting
+    # non-binary classes to avoid Simpson's paradox
+    with pytest.raises(ValueError, match=err_msg):
+        class_likelihood_ratios(**params)
+
+
+def test_likelihood_ratios():
+    # Build confusion matrix with tn=9, fp=8, fn=1, tp=2,
+    # sensitivity=2/3, specificity=9/17, prevalence=3/20,
+    # LR+=34/24, LR-=17/27
+    y_true = np.array([1] * 3 + [0] * 17)
+    y_pred = np.array([1] * 2 + [0] * 10 + [1] * 8)
+
+    pos, neg = class_likelihood_ratios(y_true, y_pred)
+    assert_allclose(pos, 34 / 24)
+    assert_allclose(neg, 17 / 27)
+
+    # Build limit case with y_pred = y_true
+    pos, neg = class_likelihood_ratios(y_true, y_true)
+    assert_array_equal(pos, np.nan * 2)
+    assert_allclose(neg, np.zeros(2), rtol=1e-12)
+
+    # Ignore last 5 samples to get tn=9, fp=3, fn=1, tp=2,
+    # sensitivity=2/3, specificity=9/12, prevalence=3/20,
+    # LR+=24/9, LR-=12/27
+    sample_weight = np.array([1.0] * 15 + [0.0] * 5)
+    pos, neg = class_likelihood_ratios(y_true, y_pred, sample_weight=sample_weight)
+    assert_allclose(pos, 24 / 9)
+    assert_allclose(neg, 12 / 27)
+
+
+# TODO(1.9): remove test
+@pytest.mark.parametrize("raise_warning", [True, False])
+def test_likelihood_ratios_raise_warning_deprecation(raise_warning):
+    """Test that class_likelihood_ratios raises a `FutureWarning` when `raise_warning`
+    param is set."""
+    y_true = np.array([1, 0])
+    y_pred = np.array([1, 0])
+
+    msg = "`raise_warning` was deprecated in version 1.7 and will be removed in 1.9."
+    with pytest.warns(FutureWarning, match=msg):
+        class_likelihood_ratios(y_true, y_pred, raise_warning=raise_warning)
+
+
+def test_likelihood_ratios_replace_undefined_by_worst():
+    """Test that class_likelihood_ratios returns the worst scores `1.0` for both LR+ and
+    LR- when `replace_undefined_by=1` is set."""
+    # This data causes fp=0 (0 false positives) in the confusion_matrix and a division
+    # by zero that affects the positive_likelihood_ratio:
+    y_true = np.array([1, 1, 0])
+    y_pred = np.array([1, 0, 0])
+
+    positive_likelihood_ratio, _ = class_likelihood_ratios(
+        y_true, y_pred, replace_undefined_by=1
+    )
+    assert positive_likelihood_ratio == pytest.approx(1.0)
+
+    # This data causes tn=0 (0 true negatives) in the confusion_matrix and a division
+    # by zero that affects the negative_likelihood_ratio:
+    y_true = np.array([1, 0, 0])
+    y_pred = np.array([1, 1, 1])
+
+    _, negative_likelihood_ratio = class_likelihood_ratios(
+        y_true, y_pred, replace_undefined_by=1
+    )
+    assert negative_likelihood_ratio == pytest.approx(1.0)
+
+
+@pytest.mark.parametrize(
+    "replace_undefined_by",
+    [
+        {"LR+": 0.0},
+        {"LR-": 0.0},
+        {"LR+": -5.0, "LR-": 0.0},
+        {"LR+": 1.0, "LR-": "nan"},
+        {"LR+": 0.0, "LR-": 0.0},
+        {"LR+": 1.0, "LR-": 2.0},
+    ],
+)
+def test_likelihood_ratios_wrong_dict_replace_undefined_by(replace_undefined_by):
+    """Test that class_likelihood_ratios raises a `ValueError` if the input dict for
+    `replace_undefined_by` is in the wrong format or contains impossible values."""
+    y_true = np.array([1, 0])
+    y_pred = np.array([1, 0])
+
+    msg = "The dictionary passed as `replace_undefined_by` needs to be in the form"
+    with pytest.raises(ValueError, match=msg):
+        class_likelihood_ratios(
+            y_true, y_pred, replace_undefined_by=replace_undefined_by
+        )
+
+
+@pytest.mark.parametrize(
+    "replace_undefined_by, expected",
+    [
+        ({"LR+": 1.0, "LR-": 1.0}, 1.0),
+        ({"LR+": np.inf, "LR-": 0.0}, np.inf),
+        ({"LR+": 2.0, "LR-": 0.0}, 2.0),
+        ({"LR+": np.nan, "LR-": np.nan}, np.nan),
+        (np.nan, np.nan),
+    ],
+)
+def test_likelihood_ratios_replace_undefined_by_0_fp(replace_undefined_by, expected):
+    """Test that the `replace_undefined_by` param returns the right value for the
+    positive_likelihood_ratio as defined by the user."""
+    # This data causes fp=0 (0 false positives) in the confusion_matrix and a division
+    # by zero that affects the positive_likelihood_ratio:
+    y_true = np.array([1, 1, 0])
+    y_pred = np.array([1, 0, 0])
+
+    positive_likelihood_ratio, _ = class_likelihood_ratios(
+        y_true, y_pred, replace_undefined_by=replace_undefined_by
+    )
+
+    if np.isnan(expected):
+        assert np.isnan(positive_likelihood_ratio)
+    else:
+        assert positive_likelihood_ratio == pytest.approx(expected)
+
+
+@pytest.mark.parametrize(
+    "replace_undefined_by, expected",
+    [
+        ({"LR+": 1.0, "LR-": 1.0}, 1.0),
+        ({"LR+": np.inf, "LR-": 0.0}, 0.0),
+        ({"LR+": np.inf, "LR-": 0.5}, 0.5),
+        ({"LR+": np.nan, "LR-": np.nan}, np.nan),
+        (np.nan, np.nan),
+    ],
+)
+def test_likelihood_ratios_replace_undefined_by_0_tn(replace_undefined_by, expected):
+    """Test that the `replace_undefined_by` param returns the right value for the
+    negative_likelihood_ratio as defined by the user."""
+    # This data causes tn=0 (0 true negatives) in the confusion_matrix and a division
+    # by zero that affects the negative_likelihood_ratio:
+    y_true = np.array([1, 0, 0])
+    y_pred = np.array([1, 1, 1])
+
+    _, negative_likelihood_ratio = class_likelihood_ratios(
+        y_true, y_pred, replace_undefined_by=replace_undefined_by
+    )
+
+    if np.isnan(expected):
+        assert np.isnan(negative_likelihood_ratio)
+    else:
+        assert negative_likelihood_ratio == pytest.approx(expected)
 
 
 def test_cohen_kappa():
@@ -597,7 +882,7 @@ def test_cohen_kappa():
     y1 = np.array([0] * 40 + [1] * 60)
     y2 = np.array([0] * 20 + [1] * 20 + [0] * 10 + [1] * 50)
     kappa = cohen_kappa_score(y1, y2)
-    assert_almost_equal(kappa, .348, decimal=3)
+    assert_almost_equal(kappa, 0.348, decimal=3)
     assert kappa == cohen_kappa_score(y2, y1)
 
     # Add spurious labels and ignore them.
@@ -605,71 +890,129 @@ def test_cohen_kappa():
     y2 = np.append(y2, [2] * 4)
     assert cohen_kappa_score(y1, y2, labels=[0, 1]) == kappa
 
-    assert_almost_equal(cohen_kappa_score(y1, y1), 1.)
+    assert_almost_equal(cohen_kappa_score(y1, y1), 1.0)
 
     # Multiclass example: Artstein and Poesio, Table 4.
     y1 = np.array([0] * 46 + [1] * 44 + [2] * 10)
     y2 = np.array([0] * 52 + [1] * 32 + [2] * 16)
-    assert_almost_equal(cohen_kappa_score(y1, y2), .8013, decimal=4)
+    assert_almost_equal(cohen_kappa_score(y1, y2), 0.8013, decimal=4)
 
     # Weighting example: none, linear, quadratic.
     y1 = np.array([0] * 46 + [1] * 44 + [2] * 10)
     y2 = np.array([0] * 50 + [1] * 40 + [2] * 10)
-    assert_almost_equal(cohen_kappa_score(y1, y2), .9315, decimal=4)
-    assert_almost_equal(cohen_kappa_score(y1, y2,
-                        weights="linear"), 0.9412, decimal=4)
-    assert_almost_equal(cohen_kappa_score(y1, y2,
-                        weights="quadratic"), 0.9541, decimal=4)
+    assert_almost_equal(cohen_kappa_score(y1, y2), 0.9315, decimal=4)
+    assert_almost_equal(cohen_kappa_score(y1, y2, weights="linear"), 0.9412, decimal=4)
+    assert_almost_equal(
+        cohen_kappa_score(y1, y2, weights="quadratic"), 0.9541, decimal=4
+    )
 
 
-@ignore_warnings
-def test_matthews_corrcoef_nan():
-    assert matthews_corrcoef([0], [1]) == 0.0
-    assert matthews_corrcoef([0, 0], [0, 1]) == 0.0
+def test_cohen_kappa_score_error_wrong_label():
+    """Test that correct error is raised when users pass labels that are not in y1."""
+    labels = [1, 2]
+    y1 = np.array(["a"] * 5 + ["b"] * 5)
+    y2 = np.array(["b"] * 10)
+    with pytest.raises(
+        ValueError, match="At least one label in `labels` must be present in `y1`"
+    ):
+        cohen_kappa_score(y1, y2, labels=labels)
 
 
-def test_matthews_corrcoef_against_numpy_corrcoef():
-    rng = np.random.RandomState(0)
+@pytest.mark.parametrize("zero_division", [0, 1, np.nan])
+@pytest.mark.parametrize("y_true, y_pred", [([0], [0])])
+@pytest.mark.parametrize(
+    "metric",
+    [
+        f1_score,
+        partial(fbeta_score, beta=1),
+        precision_score,
+        recall_score,
+    ],
+)
+def test_zero_division_nan_no_warning(metric, y_true, y_pred, zero_division):
+    """Check the behaviour of `zero_division` when setting to 0, 1 or np.nan.
+    No warnings should be raised.
+    """
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        result = metric(y_true, y_pred, zero_division=zero_division)
+
+    if np.isnan(zero_division):
+        assert np.isnan(result)
+    else:
+        assert result == zero_division
+
+
+@pytest.mark.parametrize("y_true, y_pred", [([0], [0])])
+@pytest.mark.parametrize(
+    "metric",
+    [
+        f1_score,
+        partial(fbeta_score, beta=1),
+        precision_score,
+        recall_score,
+    ],
+)
+def test_zero_division_nan_warning(metric, y_true, y_pred):
+    """Check the behaviour of `zero_division` when setting to "warn".
+    A `UndefinedMetricWarning` should be raised.
+    """
+    with pytest.warns(UndefinedMetricWarning):
+        result = metric(y_true, y_pred, zero_division="warn")
+    assert result == 0.0
+
+
+def test_matthews_corrcoef_against_numpy_corrcoef(global_random_seed):
+    rng = np.random.RandomState(global_random_seed)
     y_true = rng.randint(0, 2, size=20)
     y_pred = rng.randint(0, 2, size=20)
 
-    assert_almost_equal(matthews_corrcoef(y_true, y_pred),
-                        np.corrcoef(y_true, y_pred)[0, 1], 10)
+    assert_almost_equal(
+        matthews_corrcoef(y_true, y_pred), np.corrcoef(y_true, y_pred)[0, 1], 10
+    )
 
 
-def test_matthews_corrcoef_against_jurman():
+def test_matthews_corrcoef_against_jurman(global_random_seed):
     # Check that the multiclass matthews_corrcoef agrees with the definition
     # presented in Jurman, Riccadonna, Furlanello, (2012). A Comparison of MCC
     # and CEN Error Measures in MultiClass Prediction
-    rng = np.random.RandomState(0)
+    rng = np.random.RandomState(global_random_seed)
     y_true = rng.randint(0, 2, size=20)
     y_pred = rng.randint(0, 2, size=20)
     sample_weight = rng.rand(20)
 
     C = confusion_matrix(y_true, y_pred, sample_weight=sample_weight)
     N = len(C)
-    cov_ytyp = sum([
-        C[k, k] * C[m, l] - C[l, k] * C[k, m]
-        for k in range(N) for m in range(N) for l in range(N)
-    ])
-    cov_ytyt = sum([
-        C[:, k].sum() *
-        np.sum([C[g, f] for f in range(N) for g in range(N) if f != k])
-        for k in range(N)
-    ])
-    cov_ypyp = np.sum([
-        C[k, :].sum() *
-        np.sum([C[f, g] for f in range(N) for g in range(N) if f != k])
-        for k in range(N)
-    ])
+    cov_ytyp = sum(
+        [
+            C[k, k] * C[m, l] - C[l, k] * C[k, m]
+            for k in range(N)
+            for m in range(N)
+            for l in range(N)
+        ]
+    )
+    cov_ytyt = sum(
+        [
+            C[:, k].sum()
+            * np.sum([C[g, f] for f in range(N) for g in range(N) if f != k])
+            for k in range(N)
+        ]
+    )
+    cov_ypyp = np.sum(
+        [
+            C[k, :].sum()
+            * np.sum([C[f, g] for f in range(N) for g in range(N) if f != k])
+            for k in range(N)
+        ]
+    )
     mcc_jurman = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)
     mcc_ours = matthews_corrcoef(y_true, y_pred, sample_weight=sample_weight)
 
     assert_almost_equal(mcc_ours, mcc_jurman, 10)
 
 
-def test_matthews_corrcoef():
-    rng = np.random.RandomState(0)
+def test_matthews_corrcoef(global_random_seed):
+    rng = np.random.RandomState(global_random_seed)
     y_true = ["a" if i == 0 else "b" for i in rng.randint(0, 2, size=20)]
 
     # corrcoef of same vectors must be 1
@@ -680,39 +1023,32 @@ def test_matthews_corrcoef():
     assert_almost_equal(matthews_corrcoef(y_true, y_true_inv), -1)
 
     y_true_inv2 = label_binarize(y_true, classes=["a", "b"])
-    y_true_inv2 = np.where(y_true_inv2, 'a', 'b')
+    y_true_inv2 = np.where(y_true_inv2, "a", "b")
     assert_almost_equal(matthews_corrcoef(y_true, y_true_inv2), -1)
 
     # For the zero vector case, the corrcoef cannot be calculated and should
-    # result in a RuntimeWarning
-    mcc = assert_warns_div0(matthews_corrcoef, [0, 0, 0, 0], [0, 0, 0, 0])
-
-    # But will output 0
-    assert_almost_equal(mcc, 0.)
+    # output 0
+    assert_almost_equal(matthews_corrcoef([0, 0, 0, 0], [0, 0, 0, 0]), 0.0)
 
     # And also for any other vector with 0 variance
-    mcc = assert_warns_div0(matthews_corrcoef, y_true, ['a'] * len(y_true))
-
-    # But will output 0
-    assert_almost_equal(mcc, 0.)
+    assert_almost_equal(matthews_corrcoef(y_true, ["a"] * len(y_true)), 0.0)
 
     # These two vectors have 0 correlation and hence mcc should be 0
     y_1 = [1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1]
     y_2 = [1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1]
-    assert_almost_equal(matthews_corrcoef(y_1, y_2), 0.)
+    assert_almost_equal(matthews_corrcoef(y_1, y_2), 0.0)
 
     # Check that sample weight is able to selectively exclude
     mask = [1] * 10 + [0] * 10
     # Now the first half of the vector elements are alone given a weight of 1
     # and hence the mcc will not be a perfect 0 as in the previous case
     with pytest.raises(AssertionError):
-        assert_almost_equal(matthews_corrcoef(y_1, y_2,
-                                              sample_weight=mask), 0.)
+        assert_almost_equal(matthews_corrcoef(y_1, y_2, sample_weight=mask), 0.0)
 
 
-def test_matthews_corrcoef_multiclass():
-    rng = np.random.RandomState(0)
-    ord_a = ord('a')
+def test_matthews_corrcoef_multiclass(global_random_seed):
+    rng = np.random.RandomState(global_random_seed)
+    ord_a = ord("a")
     n_classes = 4
     y_true = [chr(ord_a + i) for i in rng.randint(0, n_classes, size=20)]
 
@@ -722,26 +1058,28 @@ def test_matthews_corrcoef_multiclass():
     # with multiclass > 2 it is not possible to achieve -1
     y_true = [0, 0, 1, 1, 2, 2]
     y_pred_bad = [2, 2, 0, 0, 1, 1]
-    assert_almost_equal(matthews_corrcoef(y_true, y_pred_bad), -.5)
+    assert_almost_equal(matthews_corrcoef(y_true, y_pred_bad), -0.5)
 
     # Maximizing false positives and negatives minimizes the MCC
     # The minimum will be different for depending on the input
     y_true = [0, 0, 1, 1, 2, 2]
     y_pred_min = [1, 1, 0, 0, 0, 0]
-    assert_almost_equal(matthews_corrcoef(y_true, y_pred_min),
-                        -12 / np.sqrt(24 * 16))
+    assert_almost_equal(matthews_corrcoef(y_true, y_pred_min), -12 / np.sqrt(24 * 16))
 
-    # Zero variance will result in an mcc of zero and a Runtime Warning
+    # Zero variance will result in an mcc of zero
     y_true = [0, 1, 2]
     y_pred = [3, 3, 3]
-    mcc = assert_warns_message(RuntimeWarning, 'invalid value encountered',
-                               matthews_corrcoef, y_true, y_pred)
-    assert_almost_equal(mcc, 0.0)
+    assert_almost_equal(matthews_corrcoef(y_true, y_pred), 0.0)
+
+    # Also for ground truth with zero variance
+    y_true = [3, 3, 3]
+    y_pred = [0, 1, 2]
+    assert_almost_equal(matthews_corrcoef(y_true, y_pred), 0.0)
 
     # These two vectors have 0 correlation and hence mcc should be 0
     y_1 = [0, 1, 2, 0, 1, 2, 0, 1, 2]
     y_2 = [1, 1, 1, 2, 2, 2, 0, 0, 0]
-    assert_almost_equal(matthews_corrcoef(y_1, y_2), 0.)
+    assert_almost_equal(matthews_corrcoef(y_1, y_2), 0.0)
 
     # We can test that binary assumptions hold using the multiclass computation
     # by masking the weight of samples not in the first two classes
@@ -750,26 +1088,24 @@ def test_matthews_corrcoef_multiclass():
     y_true = [0, 0, 1, 1, 2]
     y_pred = [1, 1, 0, 0, 2]
     sample_weight = [1, 1, 1, 1, 0]
-    assert_almost_equal(matthews_corrcoef(y_true, y_pred,
-                                          sample_weight=sample_weight), -1)
+    assert_almost_equal(
+        matthews_corrcoef(y_true, y_pred, sample_weight=sample_weight), -1
+    )
 
     # For the zero vector case, the corrcoef cannot be calculated and should
-    # result in a RuntimeWarning
+    # output 0
     y_true = [0, 0, 1, 2]
     y_pred = [0, 0, 1, 2]
     sample_weight = [1, 1, 0, 0]
-    mcc = assert_warns_message(RuntimeWarning, 'invalid value encountered',
-                               matthews_corrcoef, y_true, y_pred,
-                               sample_weight=sample_weight)
-
-    # But will output 0
-    assert_almost_equal(mcc, 0.)
+    assert_almost_equal(
+        matthews_corrcoef(y_true, y_pred, sample_weight=sample_weight), 0.0
+    )
 
 
-@pytest.mark.parametrize('n_points', [100, 10000])
-def test_matthews_corrcoef_overflow(n_points):
+@pytest.mark.parametrize("n_points", [100, 10000])
+def test_matthews_corrcoef_overflow(n_points, global_random_seed):
     # https://github.com/scikit-learn/scikit-learn/issues/9622
-    rng = np.random.RandomState(20170906)
+    rng = np.random.RandomState(global_random_seed)
 
     def mcc_safe(y_true, y_pred):
         conf_matrix = confusion_matrix(y_true, y_pred)
@@ -783,22 +1119,21 @@ def mcc_safe(y_true, y_pred):
         mcc_denominator = activity * pos_rate * (1 - activity) * (1 - pos_rate)
         return mcc_numerator / np.sqrt(mcc_denominator)
 
-    def random_ys(n_points):    # binary
+    def random_ys(n_points):  # binary
         x_true = rng.random_sample(n_points)
         x_pred = x_true + 0.2 * (rng.random_sample(n_points) - 0.5)
-        y_true = (x_true > 0.5)
-        y_pred = (x_pred > 0.5)
+        y_true = x_true > 0.5
+        y_pred = x_pred > 0.5
         return y_true, y_pred
 
-    arr = np.repeat([0., 1.], n_points)  # binary
+    arr = np.repeat([0.0, 1.0], n_points)  # binary
     assert_almost_equal(matthews_corrcoef(arr, arr), 1.0)
-    arr = np.repeat([0., 1., 2.], n_points)  # multiclass
+    arr = np.repeat([0.0, 1.0, 2.0], n_points)  # multiclass
     assert_almost_equal(matthews_corrcoef(arr, arr), 1.0)
 
     y_true, y_pred = random_ys(n_points)
     assert_almost_equal(matthews_corrcoef(y_true, y_true), 1.0)
-    assert_almost_equal(matthews_corrcoef(y_true, y_pred),
-                        mcc_safe(y_true, y_pred))
+    assert_almost_equal(matthews_corrcoef(y_true, y_pred), mcc_safe(y_true, y_pred))
 
 
 def test_precision_recall_f1_score_multiclass():
@@ -813,31 +1148,31 @@ def test_precision_recall_f1_score_multiclass():
     assert_array_equal(s, [24, 31, 20])
 
     # averaging tests
-    ps = precision_score(y_true, y_pred, pos_label=1, average='micro')
+    ps = precision_score(y_true, y_pred, pos_label=1, average="micro")
     assert_array_almost_equal(ps, 0.53, 2)
 
-    rs = recall_score(y_true, y_pred, average='micro')
+    rs = recall_score(y_true, y_pred, average="micro")
     assert_array_almost_equal(rs, 0.53, 2)
 
-    fs = f1_score(y_true, y_pred, average='micro')
+    fs = f1_score(y_true, y_pred, average="micro")
     assert_array_almost_equal(fs, 0.53, 2)
 
-    ps = precision_score(y_true, y_pred, average='macro')
+    ps = precision_score(y_true, y_pred, average="macro")
     assert_array_almost_equal(ps, 0.53, 2)
 
-    rs = recall_score(y_true, y_pred, average='macro')
+    rs = recall_score(y_true, y_pred, average="macro")
     assert_array_almost_equal(rs, 0.60, 2)
 
-    fs = f1_score(y_true, y_pred, average='macro')
+    fs = f1_score(y_true, y_pred, average="macro")
     assert_array_almost_equal(fs, 0.51, 2)
 
-    ps = precision_score(y_true, y_pred, average='weighted')
+    ps = precision_score(y_true, y_pred, average="weighted")
     assert_array_almost_equal(ps, 0.51, 2)
 
-    rs = recall_score(y_true, y_pred, average='weighted')
+    rs = recall_score(y_true, y_pred, average="weighted")
     assert_array_almost_equal(rs, 0.53, 2)
 
-    fs = f1_score(y_true, y_pred, average='weighted')
+    fs = f1_score(y_true, y_pred, average="weighted")
     assert_array_almost_equal(fs, 0.47, 2)
 
     with pytest.raises(ValueError):
@@ -851,21 +1186,22 @@ def test_precision_recall_f1_score_multiclass():
 
     # same prediction but with and explicit label ordering
     p, r, f, s = precision_recall_fscore_support(
-        y_true, y_pred, labels=[0, 2, 1], average=None)
+        y_true, y_pred, labels=[0, 2, 1], average=None
+    )
     assert_array_almost_equal(p, [0.83, 0.41, 0.33], 2)
     assert_array_almost_equal(r, [0.79, 0.90, 0.10], 2)
     assert_array_almost_equal(f, [0.81, 0.57, 0.15], 2)
     assert_array_equal(s, [24, 20, 31])
 
 
-@pytest.mark.parametrize('average',
-                         ['samples', 'micro', 'macro', 'weighted', None])
+@pytest.mark.parametrize("average", ["samples", "micro", "macro", "weighted", None])
 def test_precision_refcall_f1_score_multilabel_unordered_labels(average):
     # test that labels need not be sorted in the multilabel case
     y_true = np.array([[1, 1, 0, 0]])
     y_pred = np.array([[0, 0, 1, 1]])
     p, r, f, s = precision_recall_fscore_support(
-        y_true, y_pred, labels=[3, 0, 1, 2], warn_for=[], average=average)
+        y_true, y_pred, labels=[3, 0, 1, 2], warn_for=[], average=average
+    )
     assert_array_equal(p, 0)
     assert_array_equal(r, 0)
     assert_array_equal(f, 0)
@@ -878,15 +1214,12 @@ def test_precision_recall_f1_score_binary_averaged():
     y_pred = np.array([1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1])
 
     # compute scores with default labels introspection
-    ps, rs, fs, _ = precision_recall_fscore_support(y_true, y_pred,
-                                                    average=None)
-    p, r, f, _ = precision_recall_fscore_support(y_true, y_pred,
-                                                 average='macro')
+    ps, rs, fs, _ = precision_recall_fscore_support(y_true, y_pred, average=None)
+    p, r, f, _ = precision_recall_fscore_support(y_true, y_pred, average="macro")
     assert p == np.mean(ps)
     assert r == np.mean(rs)
     assert f == np.mean(fs)
-    p, r, f, _ = precision_recall_fscore_support(y_true, y_pred,
-                                                 average='weighted')
+    p, r, f, _ = precision_recall_fscore_support(y_true, y_pred, average="weighted")
     support = np.bincount(y_true)
     assert p == np.average(ps, weights=support)
     assert r == np.average(rs, weights=support)
@@ -896,18 +1229,15 @@ def test_precision_recall_f1_score_binary_averaged():
 def test_zero_precision_recall():
     # Check that pathological cases do not bring NaNs
 
-    old_error_settings = np.seterr(all='raise')
+    old_error_settings = np.seterr(all="raise")
 
     try:
         y_true = np.array([0, 1, 2, 0, 1, 2])
         y_pred = np.array([2, 0, 1, 1, 2, 0])
 
-        assert_almost_equal(precision_score(y_true, y_pred,
-                                            average='macro'), 0.0, 2)
-        assert_almost_equal(recall_score(y_true, y_pred, average='macro'),
-                            0.0, 2)
-        assert_almost_equal(f1_score(y_true, y_pred, average='macro'),
-                            0.0, 2)
+        assert_almost_equal(precision_score(y_true, y_pred, average="macro"), 0.0, 2)
+        assert_almost_equal(recall_score(y_true, y_pred, average="macro"), 0.0, 2)
+        assert_almost_equal(f1_score(y_true, y_pred, average="macro"), 0.0, 2)
 
     finally:
         np.seterr(**old_error_settings)
@@ -919,27 +1249,26 @@ def test_confusion_matrix_multiclass_subset_labels():
 
     # compute confusion matrix with only first two labels considered
     cm = confusion_matrix(y_true, y_pred, labels=[0, 1])
-    assert_array_equal(cm, [[19, 4],
-                            [4, 3]])
+    assert_array_equal(cm, [[19, 4], [4, 3]])
 
     # compute confusion matrix with explicit label ordering for only subset
     # of labels
     cm = confusion_matrix(y_true, y_pred, labels=[2, 1])
-    assert_array_equal(cm, [[18, 2],
-                            [24, 3]])
+    assert_array_equal(cm, [[18, 2], [24, 3]])
 
     # a label not in y_true should result in zeros for that row/column
     extra_label = np.max(y_true) + 1
     cm = confusion_matrix(y_true, y_pred, labels=[2, extra_label])
-    assert_array_equal(cm, [[18, 0],
-                            [0, 0]])
+    assert_array_equal(cm, [[18, 0], [0, 0]])
 
 
 @pytest.mark.parametrize(
     "labels, err_msg",
-    [([], "'labels' should contains at least one label."),
-     ([3, 4], "At least one label specified must be in y_true")],
-    ids=["empty list", "unknown labels"]
+    [
+        ([], "'labels' should contains at least one label."),
+        ([3, 4], "At least one label specified must be in y_true"),
+    ],
+    ids=["empty list", "unknown labels"],
 )
 def test_confusion_matrix_error(labels, err_msg):
     y_true, y_pred, _ = make_prediction(binary=False)
@@ -948,8 +1277,7 @@ def test_confusion_matrix_error(labels, err_msg):
 
 
 @pytest.mark.parametrize(
-    'labels', (None, [0, 1], [0, 1, 2]),
-    ids=['None', 'binary', 'multiclass']
+    "labels", (None, [0, 1], [0, 1, 2]), ids=["None", "binary", "multiclass"]
 )
 def test_confusion_matrix_on_zero_length_input(labels):
     expected_n_classes = len(labels) if labels else 0
@@ -966,12 +1294,10 @@ def test_confusion_matrix_dtype():
     assert cm.dtype == np.int64
     # The dtype of confusion_matrix is always 64 bit
     for dtype in [np.bool_, np.int32, np.uint64]:
-        cm = confusion_matrix(y, y,
-                              sample_weight=weight.astype(dtype, copy=False))
+        cm = confusion_matrix(y, y, sample_weight=weight.astype(dtype, copy=False))
         assert cm.dtype == np.int64
     for dtype in [np.float32, np.float64, None, object]:
-        cm = confusion_matrix(y, y,
-                              sample_weight=weight.astype(dtype, copy=False))
+        cm = confusion_matrix(y, y, sample_weight=weight.astype(dtype, copy=False))
         assert cm.dtype == np.float64
 
     # np.iinfo(np.uint32).max should be accumulated correctly
@@ -987,6 +1313,24 @@ def test_confusion_matrix_dtype():
     assert cm[1, 1] == -2
 
 
+@pytest.mark.parametrize("dtype", ["Int64", "Float64", "boolean"])
+def test_confusion_matrix_pandas_nullable(dtype):
+    """Checks that confusion_matrix works with pandas nullable dtypes.
+
+    Non-regression test for gh-25635.
+    """
+    pd = pytest.importorskip("pandas")
+
+    y_ndarray = np.array([1, 0, 0, 1, 0, 1, 1, 0, 1])
+    y_true = pd.Series(y_ndarray, dtype=dtype)
+    y_predicted = pd.Series([0, 0, 1, 1, 0, 1, 1, 1, 1], dtype="int64")
+
+    output = confusion_matrix(y_true, y_predicted)
+    expected_output = confusion_matrix(y_ndarray, y_predicted)
+
+    assert_array_equal(output, expected_output)
+
+
 def test_classification_report_multiclass():
     # Test performance report
     iris = datasets.load_iris()
@@ -1005,8 +1349,11 @@ def test_classification_report_multiclass():
 weighted avg       0.51      0.53      0.47        75
 """
     report = classification_report(
-        y_true, y_pred, labels=np.arange(len(iris.target_names)),
-        target_names=iris.target_names)
+        y_true,
+        y_pred,
+        labels=np.arange(len(iris.target_names)),
+        target_names=iris.target_names,
+    )
     assert report == expected_report
 
 
@@ -1066,8 +1413,12 @@ def test_classification_report_multiclass_with_digits():
 weighted avg    0.51375   0.53333   0.47310        75
 """
     report = classification_report(
-        y_true, y_pred, labels=np.arange(len(iris.target_names)),
-        target_names=iris.target_names, digits=5)
+        y_true,
+        y_pred,
+        labels=np.arange(len(iris.target_names)),
+        target_names=iris.target_names,
+        digits=5,
+    )
     assert report == expected_report
 
 
@@ -1102,8 +1453,7 @@ def test_classification_report_multiclass_with_string_label():
    macro avg       0.53      0.60      0.51        75
 weighted avg       0.51      0.53      0.47        75
 """
-    report = classification_report(y_true, y_pred,
-                                   target_names=["a", "b", "c"])
+    report = classification_report(y_true, y_pred, target_names=["a", "b", "c"])
     assert report == expected_report
 
 
@@ -1155,42 +1505,39 @@ def test_classification_report_multiclass_with_long_string_label():
 def test_classification_report_labels_target_names_unequal_length():
     y_true = [0, 0, 2, 0, 0]
     y_pred = [0, 2, 2, 0, 0]
-    target_names = ['class 0', 'class 1', 'class 2']
+    target_names = ["class 0", "class 1", "class 2"]
 
-    assert_warns_message(UserWarning,
-                         "labels size, 2, does not "
-                         "match size of target_names, 3",
-                         classification_report,
-                         y_true, y_pred, labels=[0, 2],
-                         target_names=target_names)
+    msg = "labels size, 2, does not match size of target_names, 3"
+    with pytest.warns(UserWarning, match=msg):
+        classification_report(y_true, y_pred, labels=[0, 2], target_names=target_names)
 
 
 def test_classification_report_no_labels_target_names_unequal_length():
     y_true = [0, 0, 2, 0, 0]
     y_pred = [0, 2, 2, 0, 0]
-    target_names = ['class 0', 'class 1', 'class 2']
+    target_names = ["class 0", "class 1", "class 2"]
 
-    err_msg = ("Number of classes, 2, does not "
-               "match size of target_names, 3. "
-               "Try specifying the labels parameter")
+    err_msg = (
+        "Number of classes, 2, does not "
+        "match size of target_names, 3. "
+        "Try specifying the labels parameter"
+    )
     with pytest.raises(ValueError, match=err_msg):
         classification_report(y_true, y_pred, target_names=target_names)
 
 
-@ignore_warnings
+@pytest.mark.filterwarnings(r"ignore::sklearn.exceptions.UndefinedMetricWarning")
 def test_multilabel_classification_report():
     n_classes = 4
     n_samples = 50
 
-    _, y_true = make_multilabel_classification(n_features=1,
-                                               n_samples=n_samples,
-                                               n_classes=n_classes,
-                                               random_state=0)
+    _, y_true = make_multilabel_classification(
+        n_features=1, n_samples=n_samples, n_classes=n_classes, random_state=0
+    )
 
-    _, y_pred = make_multilabel_classification(n_features=1,
-                                               n_samples=n_samples,
-                                               n_classes=n_classes,
-                                               random_state=1)
+    _, y_pred = make_multilabel_classification(
+        n_features=1, n_samples=n_samples, n_classes=n_classes, random_state=1
+    )
 
     expected_report = """\
               precision    recall  f1-score   support
@@ -1237,9 +1584,9 @@ def test_multilabel_hamming_loss():
     assert hamming_loss(y1, 1 - y1) == 1
     assert hamming_loss(y1, np.zeros(y1.shape)) == 4 / 6
     assert hamming_loss(y2, np.zeros(y1.shape)) == 0.5
-    assert hamming_loss(y1, y2, sample_weight=w) == 1. / 12
-    assert hamming_loss(y1, 1-y2, sample_weight=w) == 11. / 12
-    assert hamming_loss(y1, np.zeros_like(y1), sample_weight=w) == 2. / 3
+    assert hamming_loss(y1, y2, sample_weight=w) == 1.0 / 12
+    assert hamming_loss(y1, 1 - y2, sample_weight=w) == 11.0 / 12
+    assert hamming_loss(y1, np.zeros_like(y1), sample_weight=w) == 2.0 / 3
     # sp_hamming only works with 1-D arrays
     assert hamming_loss(y1[0], y2[0]) == sp_hamming(y1[0], y2[0])
 
@@ -1249,34 +1596,39 @@ def test_jaccard_score_validation():
     y_pred = np.array([0, 1, 0, 1, 1])
     err_msg = r"pos_label=2 is not a valid label. It should be one of \[0, 1\]"
     with pytest.raises(ValueError, match=err_msg):
-        jaccard_score(y_true, y_pred, average='binary', pos_label=2)
+        jaccard_score(y_true, y_pred, average="binary", pos_label=2)
 
     y_true = np.array([[0, 1, 1], [1, 0, 0]])
     y_pred = np.array([[1, 1, 1], [1, 0, 1]])
-    msg1 = (r"Target is multilabel-indicator but average='binary'. "
-            r"Please choose another average setting, one of \[None, "
-            r"'micro', 'macro', 'weighted', 'samples'\].")
+    msg1 = (
+        r"Target is multilabel-indicator but average='binary'. "
+        r"Please choose another average setting, one of \[None, "
+        r"'micro', 'macro', 'weighted', 'samples'\]."
+    )
     with pytest.raises(ValueError, match=msg1):
-        jaccard_score(y_true, y_pred, average='binary', pos_label=-1)
+        jaccard_score(y_true, y_pred, average="binary", pos_label=-1)
 
     y_true = np.array([0, 1, 1, 0, 2])
     y_pred = np.array([1, 1, 1, 1, 0])
-    msg2 = (r"Target is multiclass but average='binary'. Please choose "
-            r"another average setting, one of \[None, 'micro', 'macro', "
-            r"'weighted'\].")
+    msg2 = (
+        r"Target is multiclass but average='binary'. Please choose "
+        r"another average setting, one of \[None, 'micro', 'macro', "
+        r"'weighted'\]."
+    )
     with pytest.raises(ValueError, match=msg2):
-        jaccard_score(y_true, y_pred, average='binary')
-    msg3 = ("Samplewise metrics are not available outside of multilabel "
-            "classification.")
+        jaccard_score(y_true, y_pred, average="binary")
+    msg3 = "Samplewise metrics are not available outside of multilabel classification."
     with pytest.raises(ValueError, match=msg3):
-        jaccard_score(y_true, y_pred, average='samples')
+        jaccard_score(y_true, y_pred, average="samples")
 
-    assert_warns_message(UserWarning,
-                         "Note that pos_label (set to 3) is ignored when "
-                         "average != 'binary' (got 'micro'). You may use "
-                         "labels=[pos_label] to specify a single positive "
-                         "class.", jaccard_score, y_true, y_pred,
-                         average='micro', pos_label=3)
+    msg = (
+        r"Note that pos_label \(set to 3\) is ignored when "
+        r"average != 'binary' \(got 'micro'\). You may use "
+        r"labels=\[pos_label\] to specify a single positive "
+        "class."
+    )
+    with pytest.warns(UserWarning, match=msg):
+        jaccard_score(y_true, y_pred, average="micro", pos_label=3)
 
 
 def test_multilabel_jaccard_score(recwarn):
@@ -1287,123 +1639,131 @@ def test_multilabel_jaccard_score(recwarn):
     # size(y1 \inter y2) = [1, 2]
     # size(y1 \union y2) = [2, 2]
 
-    assert jaccard_score(y1, y2, average='samples') == 0.75
-    assert jaccard_score(y1, y1, average='samples') == 1
-    assert jaccard_score(y2, y2, average='samples') == 1
-    assert jaccard_score(y2, np.logical_not(y2), average='samples') == 0
-    assert jaccard_score(y1, np.logical_not(y1), average='samples') == 0
-    assert jaccard_score(y1, np.zeros(y1.shape), average='samples') == 0
-    assert jaccard_score(y2, np.zeros(y1.shape), average='samples') == 0
+    assert jaccard_score(y1, y2, average="samples") == 0.75
+    assert jaccard_score(y1, y1, average="samples") == 1
+    assert jaccard_score(y2, y2, average="samples") == 1
+    assert jaccard_score(y2, np.logical_not(y2), average="samples") == 0
+    assert jaccard_score(y1, np.logical_not(y1), average="samples") == 0
+    assert jaccard_score(y1, np.zeros(y1.shape), average="samples") == 0
+    assert jaccard_score(y2, np.zeros(y1.shape), average="samples") == 0
 
     y_true = np.array([[0, 1, 1], [1, 0, 0]])
     y_pred = np.array([[1, 1, 1], [1, 0, 1]])
     # average='macro'
-    assert_almost_equal(jaccard_score(y_true, y_pred,
-                                      average='macro'), 2. / 3)
+    assert_almost_equal(jaccard_score(y_true, y_pred, average="macro"), 2.0 / 3)
     # average='micro'
-    assert_almost_equal(jaccard_score(y_true, y_pred,
-                                      average='micro'), 3. / 5)
+    assert_almost_equal(jaccard_score(y_true, y_pred, average="micro"), 3.0 / 5)
     # average='samples'
-    assert_almost_equal(jaccard_score(y_true, y_pred, average='samples'),
-                        7. / 12)
-    assert_almost_equal(jaccard_score(y_true, y_pred,
-                                      average='samples',
-                                      labels=[0, 2]), 1. / 2)
-    assert_almost_equal(jaccard_score(y_true, y_pred,
-                                      average='samples',
-                                      labels=[1, 2]), 1. / 2)
+    assert_almost_equal(jaccard_score(y_true, y_pred, average="samples"), 7.0 / 12)
+    assert_almost_equal(
+        jaccard_score(y_true, y_pred, average="samples", labels=[0, 2]), 1.0 / 2
+    )
+    assert_almost_equal(
+        jaccard_score(y_true, y_pred, average="samples", labels=[1, 2]), 1.0 / 2
+    )
     # average=None
-    assert_array_equal(jaccard_score(y_true, y_pred, average=None),
-                       np.array([1. / 2, 1., 1. / 2]))
+    assert_array_equal(
+        jaccard_score(y_true, y_pred, average=None), np.array([1.0 / 2, 1.0, 1.0 / 2])
+    )
 
     y_true = np.array([[0, 1, 1], [1, 0, 1]])
     y_pred = np.array([[1, 1, 1], [1, 0, 1]])
-    assert_almost_equal(jaccard_score(y_true, y_pred,
-                                      average='macro'), 5. / 6)
+    assert_almost_equal(jaccard_score(y_true, y_pred, average="macro"), 5.0 / 6)
     # average='weighted'
-    assert_almost_equal(jaccard_score(y_true, y_pred,
-                                      average='weighted'), 7. / 8)
+    assert_almost_equal(jaccard_score(y_true, y_pred, average="weighted"), 7.0 / 8)
 
-    msg2 = 'Got 4 > 2'
+    msg2 = "Got 4 > 2"
     with pytest.raises(ValueError, match=msg2):
-        jaccard_score(y_true, y_pred, labels=[4], average='macro')
-    msg3 = 'Got -1 < 0'
+        jaccard_score(y_true, y_pred, labels=[4], average="macro")
+    msg3 = "Got -1 < 0"
     with pytest.raises(ValueError, match=msg3):
-        jaccard_score(y_true, y_pred, labels=[-1], average='macro')
-
-    msg = ('Jaccard is ill-defined and being set to 0.0 in labels '
-           'with no true or predicted samples.')
-    assert assert_warns_message(UndefinedMetricWarning, msg,
-                                jaccard_score,
-                                np.array([[0, 1]]),
-                                np.array([[0, 1]]),
-                                average='macro') == 0.5
-
-    msg = ('Jaccard is ill-defined and being set to 0.0 in samples '
-           'with no true or predicted labels.')
-    assert assert_warns_message(UndefinedMetricWarning, msg,
-                                jaccard_score,
-                                np.array([[0, 0], [1, 1]]),
-                                np.array([[0, 0], [1, 1]]),
-                                average='samples') == 0.5
+        jaccard_score(y_true, y_pred, labels=[-1], average="macro")
+
+    msg = (
+        "Jaccard is ill-defined and being set to 0.0 in labels "
+        "with no true or predicted samples."
+    )
+
+    with pytest.warns(UndefinedMetricWarning, match=msg):
+        assert (
+            jaccard_score(np.array([[0, 1]]), np.array([[0, 1]]), average="macro")
+            == 0.5
+        )
+
+    msg = (
+        "Jaccard is ill-defined and being set to 0.0 in samples "
+        "with no true or predicted labels."
+    )
+
+    with pytest.warns(UndefinedMetricWarning, match=msg):
+        assert (
+            jaccard_score(
+                np.array([[0, 0], [1, 1]]),
+                np.array([[0, 0], [1, 1]]),
+                average="samples",
+            )
+            == 0.5
+        )
 
     assert not list(recwarn)
 
 
 def test_multiclass_jaccard_score(recwarn):
-    y_true = ['ant', 'ant', 'cat', 'cat', 'ant', 'cat', 'bird', 'bird']
-    y_pred = ['cat', 'ant', 'cat', 'cat', 'ant', 'bird', 'bird', 'cat']
-    labels = ['ant', 'bird', 'cat']
+    y_true = ["ant", "ant", "cat", "cat", "ant", "cat", "bird", "bird"]
+    y_pred = ["cat", "ant", "cat", "cat", "ant", "bird", "bird", "cat"]
+    labels = ["ant", "bird", "cat"]
     lb = LabelBinarizer()
     lb.fit(labels)
     y_true_bin = lb.transform(y_true)
     y_pred_bin = lb.transform(y_pred)
-    multi_jaccard_score = partial(jaccard_score, y_true,
-                                  y_pred)
-    bin_jaccard_score = partial(jaccard_score,
-                                y_true_bin, y_pred_bin)
-    multi_labels_list = [['ant', 'bird'], ['ant', 'cat'], ['cat', 'bird'],
-                         ['ant'], ['bird'], ['cat'], None]
+    multi_jaccard_score = partial(jaccard_score, y_true, y_pred)
+    bin_jaccard_score = partial(jaccard_score, y_true_bin, y_pred_bin)
+    multi_labels_list = [
+        ["ant", "bird"],
+        ["ant", "cat"],
+        ["cat", "bird"],
+        ["ant"],
+        ["bird"],
+        ["cat"],
+        None,
+    ]
     bin_labels_list = [[0, 1], [0, 2], [2, 1], [0], [1], [2], None]
 
     # other than average='samples'/'none-samples', test everything else here
-    for average in ('macro', 'weighted', 'micro', None):
+    for average in ("macro", "weighted", "micro", None):
         for m_label, b_label in zip(multi_labels_list, bin_labels_list):
-            assert_almost_equal(multi_jaccard_score(average=average,
-                                                    labels=m_label),
-                                bin_jaccard_score(average=average,
-                                                  labels=b_label))
+            assert_almost_equal(
+                multi_jaccard_score(average=average, labels=m_label),
+                bin_jaccard_score(average=average, labels=b_label),
+            )
 
     y_true = np.array([[0, 0], [0, 0], [0, 0]])
     y_pred = np.array([[0, 0], [0, 0], [0, 0]])
     with ignore_warnings():
-        assert (jaccard_score(y_true, y_pred, average='weighted')
-                == 0)
+        assert jaccard_score(y_true, y_pred, average="weighted") == 0
 
     assert not list(recwarn)
 
 
 def test_average_binary_jaccard_score(recwarn):
     # tp=0, fp=0, fn=1, tn=0
-    assert jaccard_score([1], [0], average='binary') == 0.
+    assert jaccard_score([1], [0], average="binary") == 0.0
     # tp=0, fp=0, fn=0, tn=1
-    msg = ('Jaccard is ill-defined and being set to 0.0 due to '
-           'no true or predicted samples')
-    assert assert_warns_message(UndefinedMetricWarning,
-                                msg,
-                                jaccard_score,
-                                [0, 0], [0, 0],
-                                average='binary') == 0.
+    msg = (
+        "Jaccard is ill-defined and being set to 0.0 due to "
+        "no true or predicted samples"
+    )
+    with pytest.warns(UndefinedMetricWarning, match=msg):
+        assert jaccard_score([0, 0], [0, 0], average="binary") == 0.0
+
     # tp=1, fp=0, fn=0, tn=0 (pos_label=0)
-    assert jaccard_score([0], [0], pos_label=0,
-                         average='binary') == 1.
+    assert jaccard_score([0], [0], pos_label=0, average="binary") == 1.0
     y_true = np.array([1, 0, 1, 1, 0])
     y_pred = np.array([1, 0, 1, 1, 1])
-    assert_almost_equal(jaccard_score(y_true, y_pred,
-                                      average='binary'), 3. / 4)
-    assert_almost_equal(jaccard_score(y_true, y_pred,
-                                      average='binary',
-                                      pos_label=0), 1. / 2)
+    assert_almost_equal(jaccard_score(y_true, y_pred, average="binary"), 3.0 / 4)
+    assert_almost_equal(
+        jaccard_score(y_true, y_pred, average="binary", pos_label=0), 1.0 / 2
+    )
 
     assert not list(recwarn)
 
@@ -1413,31 +1773,30 @@ def test_jaccard_score_zero_division_warning():
     # happens
     y_true = np.array([[1, 0, 1], [0, 0, 0]])
     y_pred = np.array([[0, 0, 0], [0, 0, 0]])
-    msg = ('Jaccard is ill-defined and being set to 0.0 in '
-           'samples with no true or predicted labels.'
-           ' Use `zero_division` parameter to control this behavior.')
+    msg = (
+        "Jaccard is ill-defined and being set to 0.0 in "
+        "samples with no true or predicted labels."
+        " Use `zero_division` parameter to control this behavior."
+    )
     with pytest.warns(UndefinedMetricWarning, match=msg):
-        score = jaccard_score(
-            y_true, y_pred, average='samples', zero_division='warn'
-        )
+        score = jaccard_score(y_true, y_pred, average="samples", zero_division="warn")
         assert score == pytest.approx(0.0)
 
 
-@pytest.mark.parametrize(
-    "zero_division, expected_score", [(0, 0), (1, 0.5)]
-)
+@pytest.mark.parametrize("zero_division, expected_score", [(0, 0), (1, 0.5)])
 def test_jaccard_score_zero_division_set_value(zero_division, expected_score):
     # check that we don't issue warning by passing the zero_division parameter
     y_true = np.array([[1, 0, 1], [0, 0, 0]])
     y_pred = np.array([[0, 0, 0], [0, 0, 0]])
-    with pytest.warns(None) as record:
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", UndefinedMetricWarning)
         score = jaccard_score(
             y_true, y_pred, average="samples", zero_division=zero_division
         )
     assert score == pytest.approx(expected_score)
-    assert len(record) == 0
 
-@ignore_warnings
+
+@pytest.mark.filterwarnings(r"ignore::sklearn.exceptions.UndefinedMetricWarning")
 def test_precision_recall_f1_score_multilabel_1():
     # Test precision_recall_f1_score on a crafted multilabel example
     # First crafted example
@@ -1462,51 +1821,49 @@ def test_precision_recall_f1_score_multilabel_1():
     assert_array_almost_equal(f2, [0, 0.83, 1, 0], 2)
 
     # Check macro
-    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
-                                                 average="macro")
+    p, r, f, s = precision_recall_fscore_support(y_true, y_pred, average="macro")
     assert_almost_equal(p, 1.5 / 4)
     assert_almost_equal(r, 0.5)
     assert_almost_equal(f, 2.5 / 1.5 * 0.25)
     assert s is None
-    assert_almost_equal(fbeta_score(y_true, y_pred, beta=2, average="macro"),
-                        np.mean(f2))
+    assert_almost_equal(
+        fbeta_score(y_true, y_pred, beta=2, average="macro"), np.mean(f2)
+    )
 
     # Check micro
-    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
-                                                 average="micro")
+    p, r, f, s = precision_recall_fscore_support(y_true, y_pred, average="micro")
     assert_almost_equal(p, 0.5)
     assert_almost_equal(r, 0.5)
     assert_almost_equal(f, 0.5)
     assert s is None
-    assert_almost_equal(fbeta_score(y_true, y_pred, beta=2,
-                                    average="micro"),
-                        (1 + 4) * p * r / (4 * p + r))
+    assert_almost_equal(
+        fbeta_score(y_true, y_pred, beta=2, average="micro"),
+        (1 + 4) * p * r / (4 * p + r),
+    )
 
     # Check weighted
-    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
-                                                 average="weighted")
+    p, r, f, s = precision_recall_fscore_support(y_true, y_pred, average="weighted")
     assert_almost_equal(p, 1.5 / 4)
     assert_almost_equal(r, 0.5)
     assert_almost_equal(f, 2.5 / 1.5 * 0.25)
     assert s is None
-    assert_almost_equal(fbeta_score(y_true, y_pred, beta=2,
-                                    average="weighted"),
-                        np.average(f2, weights=support))
+    assert_almost_equal(
+        fbeta_score(y_true, y_pred, beta=2, average="weighted"),
+        np.average(f2, weights=support),
+    )
     # Check samples
     # |h(x_i) inter y_i | = [0, 1, 1]
     # |y_i| = [1, 1, 2]
     # |h(x_i)| = [1, 1, 2]
-    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
-                                                 average="samples")
+    p, r, f, s = precision_recall_fscore_support(y_true, y_pred, average="samples")
     assert_almost_equal(p, 0.5)
     assert_almost_equal(r, 0.5)
     assert_almost_equal(f, 0.5)
     assert s is None
-    assert_almost_equal(fbeta_score(y_true, y_pred, beta=2, average="samples"),
-                        0.5)
+    assert_almost_equal(fbeta_score(y_true, y_pred, beta=2, average="samples"), 0.5)
 
 
-@ignore_warnings
+@pytest.mark.filterwarnings(r"ignore::sklearn.exceptions.UndefinedMetricWarning")
 def test_precision_recall_f1_score_multilabel_2():
     # Test precision_recall_f1_score on a crafted multilabel example 2
     # Second crafted example
@@ -1517,8 +1874,7 @@ def test_precision_recall_f1_score_multilabel_2():
     # fp = [ 1.  0.  0.  2.]
     # fn = [ 1.  1.  1.  0.]
 
-    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
-                                                 average=None)
+    p, r, f, s = precision_recall_fscore_support(y_true, y_pred, average=None)
     assert_array_almost_equal(p, [0.0, 1.0, 0.0, 0.0], 2)
     assert_array_almost_equal(r, [0.0, 0.5, 0.0, 0.0], 2)
     assert_array_almost_equal(f, [0.0, 0.66, 0.0, 0.0], 2)
@@ -1528,38 +1884,36 @@ def test_precision_recall_f1_score_multilabel_2():
     support = s
     assert_array_almost_equal(f2, [0, 0.55, 0, 0], 2)
 
-    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
-                                                 average="micro")
+    p, r, f, s = precision_recall_fscore_support(y_true, y_pred, average="micro")
     assert_almost_equal(p, 0.25)
     assert_almost_equal(r, 0.25)
     assert_almost_equal(f, 2 * 0.25 * 0.25 / 0.5)
     assert s is None
-    assert_almost_equal(fbeta_score(y_true, y_pred, beta=2,
-                                    average="micro"),
-                        (1 + 4) * p * r / (4 * p + r))
+    assert_almost_equal(
+        fbeta_score(y_true, y_pred, beta=2, average="micro"),
+        (1 + 4) * p * r / (4 * p + r),
+    )
 
-    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
-                                                 average="macro")
+    p, r, f, s = precision_recall_fscore_support(y_true, y_pred, average="macro")
     assert_almost_equal(p, 0.25)
     assert_almost_equal(r, 0.125)
     assert_almost_equal(f, 2 / 12)
     assert s is None
-    assert_almost_equal(fbeta_score(y_true, y_pred, beta=2,
-                                    average="macro"),
-                        np.mean(f2))
+    assert_almost_equal(
+        fbeta_score(y_true, y_pred, beta=2, average="macro"), np.mean(f2)
+    )
 
-    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
-                                                 average="weighted")
+    p, r, f, s = precision_recall_fscore_support(y_true, y_pred, average="weighted")
     assert_almost_equal(p, 2 / 4)
     assert_almost_equal(r, 1 / 4)
     assert_almost_equal(f, 2 / 3 * 2 / 4)
     assert s is None
-    assert_almost_equal(fbeta_score(y_true, y_pred, beta=2,
-                                    average="weighted"),
-                        np.average(f2, weights=support))
+    assert_almost_equal(
+        fbeta_score(y_true, y_pred, beta=2, average="weighted"),
+        np.average(f2, weights=support),
+    )
 
-    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
-                                                 average="samples")
+    p, r, f, s = precision_recall_fscore_support(y_true, y_pred, average="samples")
     # Check samples
     # |h(x_i) inter y_i | = [0, 0, 1]
     # |y_i| = [1, 1, 2]
@@ -1569,72 +1923,93 @@ def test_precision_recall_f1_score_multilabel_2():
     assert_almost_equal(r, 1 / 6)
     assert_almost_equal(f, 2 / 4 * 1 / 3)
     assert s is None
-    assert_almost_equal(fbeta_score(y_true, y_pred, beta=2,
-                                    average="samples"),
-                        0.1666, 2)
+    assert_almost_equal(
+        fbeta_score(y_true, y_pred, beta=2, average="samples"), 0.1666, 2
+    )
 
 
-@ignore_warnings
-@pytest.mark.parametrize('zero_division', ["warn", 0, 1])
-def test_precision_recall_f1_score_with_an_empty_prediction(zero_division):
+@pytest.mark.filterwarnings(r"ignore::sklearn.exceptions.UndefinedMetricWarning")
+@pytest.mark.parametrize(
+    "zero_division, zero_division_expected",
+    [("warn", 0), (0, 0), (1, 1), (np.nan, np.nan)],
+)
+def test_precision_recall_f1_score_with_an_empty_prediction(
+    zero_division, zero_division_expected
+):
     y_true = np.array([[0, 1, 0, 0], [1, 0, 0, 0], [0, 1, 1, 0]])
     y_pred = np.array([[0, 0, 0, 0], [0, 0, 0, 1], [0, 1, 1, 0]])
 
     # true_pos = [ 0.  1.  1.  0.]
     # false_pos = [ 0.  0.  0.  1.]
     # false_neg = [ 1.  1.  0.  0.]
-    zero_division = 1.0 if zero_division == 1.0 else 0.0
-    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
-                                                 average=None,
-                                                 zero_division=zero_division)
-    assert_array_almost_equal(p, [zero_division, 1.0, 1.0, 0.0], 2)
-    assert_array_almost_equal(r, [0.0, 0.5, 1.0, zero_division], 2)
-    assert_array_almost_equal(f, [0.0, 1 / 1.5, 1, 0.0], 2)
+
+    p, r, f, s = precision_recall_fscore_support(
+        y_true, y_pred, average=None, zero_division=zero_division
+    )
+
+    assert_array_almost_equal(p, [zero_division_expected, 1.0, 1.0, 0.0], 2)
+    assert_array_almost_equal(r, [0.0, 0.5, 1.0, zero_division_expected], 2)
+    expected_f = 0
+    assert_array_almost_equal(f, [expected_f, 1 / 1.5, 1, expected_f], 2)
     assert_array_almost_equal(s, [1, 2, 1, 0], 2)
 
-    f2 = fbeta_score(y_true, y_pred, beta=2, average=None,
-                     zero_division=zero_division)
+    f2 = fbeta_score(y_true, y_pred, beta=2, average=None, zero_division=zero_division)
     support = s
-    assert_array_almost_equal(f2, [0, 0.55, 1, 0], 2)
-
-    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
-                                                 average="macro",
-                                                 zero_division=zero_division)
-    assert_almost_equal(p, (2 + zero_division) / 4)
-    assert_almost_equal(r, (1.5 + zero_division) / 4)
-    assert_almost_equal(f, 2.5 / (4 * 1.5))
+    assert_array_almost_equal(f2, [expected_f, 0.55, 1, expected_f], 2)
+
+    p, r, f, s = precision_recall_fscore_support(
+        y_true, y_pred, average="macro", zero_division=zero_division
+    )
+
+    value_to_sum = 0 if np.isnan(zero_division_expected) else zero_division_expected
+    values_to_average = 3 + (not np.isnan(zero_division_expected))
+
+    assert_almost_equal(p, (2 + value_to_sum) / values_to_average)
+    assert_almost_equal(r, (1.5 + value_to_sum) / values_to_average)
+    expected_f = (2 / 3 + 1) / 4
+    assert_almost_equal(f, expected_f)
     assert s is None
-    assert_almost_equal(fbeta_score(y_true, y_pred, beta=2,
-                                    average="macro"),
-                        np.mean(f2))
+    assert_almost_equal(
+        fbeta_score(
+            y_true,
+            y_pred,
+            beta=2,
+            average="macro",
+            zero_division=zero_division,
+        ),
+        _nanaverage(f2, weights=None),
+    )
 
-    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
-                                                 average="micro",
-                                                 zero_division=zero_division)
+    p, r, f, s = precision_recall_fscore_support(
+        y_true, y_pred, average="micro", zero_division=zero_division
+    )
     assert_almost_equal(p, 2 / 3)
     assert_almost_equal(r, 0.5)
     assert_almost_equal(f, 2 / 3 / (2 / 3 + 0.5))
     assert s is None
-    assert_almost_equal(fbeta_score(y_true, y_pred, beta=2,
-                                    average="micro",
-                                    zero_division=zero_division),
-                        (1 + 4) * p * r / (4 * p + r))
-
-    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
-                                                 average="weighted",
-                                                 zero_division=zero_division)
-    assert_almost_equal(p, 3 / 4 if zero_division == 0 else 1.0)
+    assert_almost_equal(
+        fbeta_score(
+            y_true, y_pred, beta=2, average="micro", zero_division=zero_division
+        ),
+        (1 + 4) * p * r / (4 * p + r),
+    )
+
+    p, r, f, s = precision_recall_fscore_support(
+        y_true, y_pred, average="weighted", zero_division=zero_division
+    )
+    assert_almost_equal(p, 3 / 4 if zero_division_expected == 0 else 1.0)
     assert_almost_equal(r, 0.5)
-    assert_almost_equal(f, (2 / 1.5 + 1) / 4)
+    values_to_average = 4
+    assert_almost_equal(f, (2 * 2 / 3 + 1) / values_to_average)
     assert s is None
-    assert_almost_equal(fbeta_score(y_true, y_pred, beta=2,
-                                    average="weighted",
-                                    zero_division=zero_division),
-                        np.average(f2, weights=support),
-                        )
-
-    p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
-                                                 average="samples")
+    assert_almost_equal(
+        fbeta_score(
+            y_true, y_pred, beta=2, average="weighted", zero_division=zero_division
+        ),
+        _nanaverage(f2, weights=support),
+    )
+
+    p, r, f, s = precision_recall_fscore_support(y_true, y_pred, average="samples")
     # |h(x_i) inter y_i | = [0, 0, 2]
     # |y_i| = [1, 1, 2]
     # |h(x_i)| = [0, 1, 2]
@@ -1642,35 +2017,57 @@ def test_precision_recall_f1_score_with_an_empty_prediction(zero_division):
     assert_almost_equal(r, 1 / 3)
     assert_almost_equal(f, 1 / 3)
     assert s is None
-    assert_almost_equal(fbeta_score(y_true, y_pred, beta=2,
-                                    average="samples",
-                                    zero_division=zero_division),
-                        0.333, 2)
+    expected_result = 0.333
+    assert_almost_equal(
+        fbeta_score(
+            y_true, y_pred, beta=2, average="samples", zero_division=zero_division
+        ),
+        expected_result,
+        2,
+    )
 
 
-@pytest.mark.parametrize('beta', [1])
-@pytest.mark.parametrize('average', ["macro", "micro", "weighted", "samples"])
-@pytest.mark.parametrize('zero_division', [0, 1])
+@pytest.mark.parametrize("beta", [1])
+@pytest.mark.parametrize("average", ["macro", "micro", "weighted", "samples"])
+@pytest.mark.parametrize("zero_division", [0, 1, np.nan])
 def test_precision_recall_f1_no_labels(beta, average, zero_division):
     y_true = np.zeros((20, 3))
     y_pred = np.zeros_like(y_true)
 
-    p, r, f, s = assert_no_warnings(precision_recall_fscore_support, y_true,
-                                    y_pred, average=average, beta=beta,
-                                    zero_division=zero_division)
-    fbeta = assert_no_warnings(fbeta_score, y_true, y_pred, beta=beta,
-                               average=average, zero_division=zero_division)
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+
+        p, r, f, s = precision_recall_fscore_support(
+            y_true,
+            y_pred,
+            average=average,
+            beta=beta,
+            zero_division=zero_division,
+        )
+        fbeta = fbeta_score(
+            y_true,
+            y_pred,
+            beta=beta,
+            average=average,
+            zero_division=zero_division,
+        )
+    assert s is None
+
+    # if zero_division = nan, check that all metrics are nan and exit
+    if np.isnan(zero_division):
+        for metric in [p, r, f, fbeta]:
+            assert np.isnan(metric)
+        return
 
     zero_division = float(zero_division)
     assert_almost_equal(p, zero_division)
     assert_almost_equal(r, zero_division)
     assert_almost_equal(f, zero_division)
-    assert s is None
 
     assert_almost_equal(fbeta, float(zero_division))
 
 
-@pytest.mark.parametrize('average', ["macro", "micro", "weighted", "samples"])
+@pytest.mark.parametrize("average", ["macro", "micro", "weighted", "samples"])
 def test_precision_recall_f1_no_labels_check_warnings(average):
     y_true = np.zeros((20, 3))
     y_pred = np.zeros_like(y_true)
@@ -1690,7 +2087,7 @@ def test_precision_recall_f1_no_labels_check_warnings(average):
     assert_almost_equal(fbeta, 0)
 
 
-@pytest.mark.parametrize('zero_division', [0, 1])
+@pytest.mark.parametrize("zero_division", [0, 1, np.nan])
 def test_precision_recall_f1_no_labels_average_none(zero_division):
     y_true = np.zeros((20, 3))
     y_pred = np.zeros_like(y_true)
@@ -1703,28 +2100,27 @@ def test_precision_recall_f1_no_labels_average_none(zero_division):
     # |y_i| = [0, 0, 0]
     # |y_hat_i| = [0, 0, 0]
 
-    p, r, f, s = assert_no_warnings(precision_recall_fscore_support,
-                                    y_true, y_pred,
-                                    average=None, beta=1.0,
-                                    zero_division=zero_division)
-    fbeta = assert_no_warnings(fbeta_score, y_true, y_pred, beta=1.0,
-                               average=None, zero_division=zero_division)
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
 
-    zero_division = float(zero_division)
-    assert_array_almost_equal(
-        p, [zero_division, zero_division, zero_division], 2
-    )
-    assert_array_almost_equal(
-        r, [zero_division, zero_division, zero_division], 2
-    )
-    assert_array_almost_equal(
-        f, [zero_division, zero_division, zero_division], 2
-    )
+        p, r, f, s = precision_recall_fscore_support(
+            y_true,
+            y_pred,
+            average=None,
+            beta=1.0,
+            zero_division=zero_division,
+        )
+        fbeta = fbeta_score(
+            y_true, y_pred, beta=1.0, average=None, zero_division=zero_division
+        )
+
+    zero_division = np.float64(zero_division)
+    assert_array_almost_equal(p, [zero_division, zero_division, zero_division], 2)
+    assert_array_almost_equal(r, [zero_division, zero_division, zero_division], 2)
+    assert_array_almost_equal(f, [zero_division, zero_division, zero_division], 2)
     assert_array_almost_equal(s, [0, 0, 0], 2)
 
-    assert_array_almost_equal(
-        fbeta, [zero_division, zero_division, zero_division], 2
-    )
+    assert_array_almost_equal(fbeta, [zero_division, zero_division, zero_division], 2)
 
 
 def test_precision_recall_f1_no_labels_average_none_warn():
@@ -1758,206 +2154,284 @@ def test_precision_recall_f1_no_labels_average_none_warn():
 def test_prf_warnings():
     # average of per-label scores
     f, w = precision_recall_fscore_support, UndefinedMetricWarning
-    for average in [None, 'weighted', 'macro']:
-
-        msg = ('Precision and F-score are ill-defined and '
-               'being set to 0.0 in labels with no predicted samples.'
-               ' Use `zero_division` parameter to control'
-               ' this behavior.')
-        assert_warns_message(w, msg, f, [0, 1, 2], [1, 1, 2], average=average)
-
-        msg = ('Recall and F-score are ill-defined and '
-               'being set to 0.0 in labels with no true samples.'
-               ' Use `zero_division` parameter to control'
-               ' this behavior.')
-        assert_warns_message(w, msg, f, [1, 1, 2], [0, 1, 2], average=average)
+    for average in [None, "weighted", "macro"]:
+        msg = (
+            "Precision is ill-defined and "
+            "being set to 0.0 in labels with no predicted samples."
+            " Use `zero_division` parameter to control"
+            " this behavior."
+        )
+        with pytest.warns(w, match=msg):
+            f([0, 1, 2], [1, 1, 2], average=average)
+
+        msg = (
+            "Recall is ill-defined and "
+            "being set to 0.0 in labels with no true samples."
+            " Use `zero_division` parameter to control"
+            " this behavior."
+        )
+        with pytest.warns(w, match=msg):
+            f([1, 1, 2], [0, 1, 2], average=average)
 
     # average of per-sample scores
-    msg = ('Precision and F-score are ill-defined and '
-           'being set to 0.0 in samples with no predicted labels.'
-           ' Use `zero_division` parameter to control'
-           ' this behavior.')
-    assert_warns_message(w, msg, f, np.array([[1, 0], [1, 0]]),
-                         np.array([[1, 0], [0, 0]]), average='samples')
-
-    msg = ('Recall and F-score are ill-defined and '
-           'being set to 0.0 in samples with no true labels.'
-           ' Use `zero_division` parameter to control'
-           ' this behavior.')
-    assert_warns_message(w, msg, f, np.array([[1, 0], [0, 0]]),
-                         np.array([[1, 0], [1, 0]]), average='samples')
+    msg = (
+        "Precision is ill-defined and "
+        "being set to 0.0 in samples with no predicted labels."
+        " Use `zero_division` parameter to control"
+        " this behavior."
+    )
+    with pytest.warns(w, match=msg):
+        f(np.array([[1, 0], [1, 0]]), np.array([[1, 0], [0, 0]]), average="samples")
+
+    msg = (
+        "Recall is ill-defined and "
+        "being set to 0.0 in samples with no true labels."
+        " Use `zero_division` parameter to control"
+        " this behavior."
+    )
+    with pytest.warns(w, match=msg):
+        f(np.array([[1, 0], [0, 0]]), np.array([[1, 0], [1, 0]]), average="samples")
 
     # single score: micro-average
-    msg = ('Precision and F-score are ill-defined and '
-           'being set to 0.0 due to no predicted samples.'
-           ' Use `zero_division` parameter to control'
-           ' this behavior.')
-    assert_warns_message(w, msg, f, np.array([[1, 1], [1, 1]]),
-                         np.array([[0, 0], [0, 0]]), average='micro')
-
-    msg = ('Recall and F-score are ill-defined and '
-           'being set to 0.0 due to no true samples.'
-           ' Use `zero_division` parameter to control'
-           ' this behavior.')
-    assert_warns_message(w, msg, f, np.array([[0, 0], [0, 0]]),
-                         np.array([[1, 1], [1, 1]]), average='micro')
+    msg = (
+        "Precision is ill-defined and "
+        "being set to 0.0 due to no predicted samples."
+        " Use `zero_division` parameter to control"
+        " this behavior."
+    )
+    with pytest.warns(w, match=msg):
+        f(np.array([[1, 1], [1, 1]]), np.array([[0, 0], [0, 0]]), average="micro")
+
+    msg = (
+        "Recall is ill-defined and "
+        "being set to 0.0 due to no true samples."
+        " Use `zero_division` parameter to control"
+        " this behavior."
+    )
+    with pytest.warns(w, match=msg):
+        f(np.array([[0, 0], [0, 0]]), np.array([[1, 1], [1, 1]]), average="micro")
 
     # single positive label
-    msg = ('Precision and F-score are ill-defined and '
-           'being set to 0.0 due to no predicted samples.'
-           ' Use `zero_division` parameter to control'
-           ' this behavior.')
-    assert_warns_message(w, msg, f, [1, 1], [-1, -1], average='binary')
-
-    msg = ('Recall and F-score are ill-defined and '
-           'being set to 0.0 due to no true samples.'
-           ' Use `zero_division` parameter to control'
-           ' this behavior.')
-    assert_warns_message(w, msg, f, [-1, -1], [1, 1], average='binary')
+    msg = (
+        "Precision is ill-defined and "
+        "being set to 0.0 due to no predicted samples."
+        " Use `zero_division` parameter to control"
+        " this behavior."
+    )
+    with pytest.warns(w, match=msg):
+        f([1, 1], [-1, -1], average="binary")
+
+    msg = (
+        "Recall is ill-defined and "
+        "being set to 0.0 due to no true samples."
+        " Use `zero_division` parameter to control"
+        " this behavior."
+    )
+    with pytest.warns(w, match=msg):
+        f([-1, -1], [1, 1], average="binary")
 
     with warnings.catch_warnings(record=True) as record:
-        warnings.simplefilter('always')
+        warnings.simplefilter("always")
         precision_recall_fscore_support([0, 0], [0, 0], average="binary")
-        msg = ('Recall and F-score are ill-defined and '
-               'being set to 0.0 due to no true samples.'
-               ' Use `zero_division` parameter to control'
-               ' this behavior.')
+        msg = (
+            "F-score is ill-defined and being set to 0.0 due to no true nor "
+            "predicted samples. Use `zero_division` parameter to control this"
+            " behavior."
+        )
         assert str(record.pop().message) == msg
-        msg = ('Precision and F-score are ill-defined and '
-               'being set to 0.0 due to no predicted samples.'
-               ' Use `zero_division` parameter to control'
-               ' this behavior.')
+        msg = (
+            "Recall is ill-defined and "
+            "being set to 0.0 due to no true samples."
+            " Use `zero_division` parameter to control"
+            " this behavior."
+        )
+        assert str(record.pop().message) == msg
+        msg = (
+            "Precision is ill-defined and "
+            "being set to 0.0 due to no predicted samples."
+            " Use `zero_division` parameter to control"
+            " this behavior."
+        )
         assert str(record.pop().message) == msg
 
 
-@pytest.mark.parametrize('zero_division', [0, 1])
+@pytest.mark.parametrize("zero_division", [0, 1, np.nan])
 def test_prf_no_warnings_if_zero_division_set(zero_division):
-    # average of per-label scores
-    f = precision_recall_fscore_support
-    for average in [None, 'weighted', 'macro']:
-
-        assert_no_warnings(f, [0, 1, 2], [1, 1, 2], average=average,
-                           zero_division=zero_division)
-
-        assert_no_warnings(f, [1, 1, 2], [0, 1, 2], average=average,
-                           zero_division=zero_division)
-
-    # average of per-sample scores
-    assert_no_warnings(f, np.array([[1, 0], [1, 0]]),
-                       np.array([[1, 0], [0, 0]]), average='samples',
-                       zero_division=zero_division)
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+
+        # average of per-label scores
+        for average in [None, "weighted", "macro"]:
+            precision_recall_fscore_support(
+                [0, 1, 2], [1, 1, 2], average=average, zero_division=zero_division
+            )
+
+            precision_recall_fscore_support(
+                [1, 1, 2], [0, 1, 2], average=average, zero_division=zero_division
+            )
+
+        # average of per-sample scores
+        precision_recall_fscore_support(
+            np.array([[1, 0], [1, 0]]),
+            np.array([[1, 0], [0, 0]]),
+            average="samples",
+            zero_division=zero_division,
+        )
 
-    assert_no_warnings(f, np.array([[1, 0], [0, 0]]),
-                       np.array([[1, 0], [1, 0]]),
-                       average='samples', zero_division=zero_division)
+        precision_recall_fscore_support(
+            np.array([[1, 0], [0, 0]]),
+            np.array([[1, 0], [1, 0]]),
+            average="samples",
+            zero_division=zero_division,
+        )
 
-    # single score: micro-average
-    assert_no_warnings(f, np.array([[1, 1], [1, 1]]),
-                       np.array([[0, 0], [0, 0]]), average='micro',
-                       zero_division=zero_division)
+        # single score: micro-average
+        precision_recall_fscore_support(
+            np.array([[1, 1], [1, 1]]),
+            np.array([[0, 0], [0, 0]]),
+            average="micro",
+            zero_division=zero_division,
+        )
 
-    assert_no_warnings(f, np.array([[0, 0], [0, 0]]),
-                       np.array([[1, 1], [1, 1]]), average='micro',
-                       zero_division=zero_division)
+        precision_recall_fscore_support(
+            np.array([[0, 0], [0, 0]]),
+            np.array([[1, 1], [1, 1]]),
+            average="micro",
+            zero_division=zero_division,
+        )
 
-    # single positive label
-    assert_no_warnings(f, [1, 1], [-1, -1], average='binary',
-                       zero_division=zero_division)
+        # single positive label
+        precision_recall_fscore_support(
+            [1, 1], [-1, -1], average="binary", zero_division=zero_division
+        )
 
-    assert_no_warnings(f, [-1, -1], [1, 1], average='binary',
-                       zero_division=zero_division)
+        precision_recall_fscore_support(
+            [-1, -1], [1, 1], average="binary", zero_division=zero_division
+        )
 
     with warnings.catch_warnings(record=True) as record:
-        warnings.simplefilter('always')
-        precision_recall_fscore_support([0, 0], [0, 0], average="binary",
-                                        zero_division=zero_division)
+        warnings.simplefilter("always")
+        precision_recall_fscore_support(
+            [0, 0], [0, 0], average="binary", zero_division=zero_division
+        )
         assert len(record) == 0
 
 
-@pytest.mark.parametrize('zero_division', ["warn", 0, 1])
+@pytest.mark.parametrize("zero_division", ["warn", 0, 1, np.nan])
 def test_recall_warnings(zero_division):
-    assert_no_warnings(recall_score,
-                       np.array([[1, 1], [1, 1]]),
-                       np.array([[0, 0], [0, 0]]),
-                       average='micro', zero_division=zero_division)
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+
+        recall_score(
+            np.array([[1, 1], [1, 1]]),
+            np.array([[0, 0], [0, 0]]),
+            average="micro",
+            zero_division=zero_division,
+        )
+
     with warnings.catch_warnings(record=True) as record:
-        warnings.simplefilter('always')
-        recall_score(np.array([[0, 0], [0, 0]]),
-                     np.array([[1, 1], [1, 1]]),
-                     average='micro', zero_division=zero_division)
+        warnings.simplefilter("always")
+        recall_score(
+            np.array([[0, 0], [0, 0]]),
+            np.array([[1, 1], [1, 1]]),
+            average="micro",
+            zero_division=zero_division,
+        )
         if zero_division == "warn":
-            assert (str(record.pop().message) ==
-                    'Recall is ill-defined and '
-                    'being set to 0.0 due to no true samples.'
-                    ' Use `zero_division` parameter to control'
-                    ' this behavior.')
+            assert (
+                str(record.pop().message) == "Recall is ill-defined and "
+                "being set to 0.0 due to no true samples."
+                " Use `zero_division` parameter to control"
+                " this behavior."
+            )
         else:
             assert len(record) == 0
 
         recall_score([0, 0], [0, 0])
         if zero_division == "warn":
-            assert (str(record.pop().message) ==
-                    'Recall is ill-defined and '
-                    'being set to 0.0 due to no true samples.'
-                    ' Use `zero_division` parameter to control'
-                    ' this behavior.')
+            assert (
+                str(record.pop().message) == "Recall is ill-defined and "
+                "being set to 0.0 due to no true samples."
+                " Use `zero_division` parameter to control"
+                " this behavior."
+            )
 
 
-@pytest.mark.parametrize('zero_division', ["warn", 0, 1])
+@pytest.mark.parametrize("zero_division", ["warn", 0, 1, np.nan])
 def test_precision_warnings(zero_division):
     with warnings.catch_warnings(record=True) as record:
-        warnings.simplefilter('always')
-        precision_score(np.array([[1, 1], [1, 1]]),
-                        np.array([[0, 0], [0, 0]]),
-                        average='micro', zero_division=zero_division)
+        warnings.simplefilter("always")
+        precision_score(
+            np.array([[1, 1], [1, 1]]),
+            np.array([[0, 0], [0, 0]]),
+            average="micro",
+            zero_division=zero_division,
+        )
         if zero_division == "warn":
-            assert (str(record.pop().message) ==
-                    'Precision is ill-defined and '
-                    'being set to 0.0 due to no predicted samples.'
-                    ' Use `zero_division` parameter to control'
-                    ' this behavior.')
+            assert (
+                str(record.pop().message) == "Precision is ill-defined and "
+                "being set to 0.0 due to no predicted samples."
+                " Use `zero_division` parameter to control"
+                " this behavior."
+            )
         else:
             assert len(record) == 0
 
         precision_score([0, 0], [0, 0])
         if zero_division == "warn":
-            assert (str(record.pop().message) ==
-                    'Precision is ill-defined and '
-                    'being set to 0.0 due to no predicted samples.'
-                    ' Use `zero_division` parameter to control'
-                    ' this behavior.')
-
-    assert_no_warnings(precision_score,
-                       np.array([[0, 0], [0, 0]]),
-                       np.array([[1, 1], [1, 1]]),
-                       average='micro', zero_division=zero_division)
+            assert (
+                str(record.pop().message) == "Precision is ill-defined and "
+                "being set to 0.0 due to no predicted samples."
+                " Use `zero_division` parameter to control"
+                " this behavior."
+            )
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+
+        precision_score(
+            np.array([[0, 0], [0, 0]]),
+            np.array([[1, 1], [1, 1]]),
+            average="micro",
+            zero_division=zero_division,
+        )
 
 
-@pytest.mark.parametrize('zero_division', ["warn", 0, 1])
+@pytest.mark.parametrize("zero_division", ["warn", 0, 1, np.nan])
 def test_fscore_warnings(zero_division):
     with warnings.catch_warnings(record=True) as record:
-        warnings.simplefilter('always')
+        warnings.simplefilter("always")
 
         for score in [f1_score, partial(fbeta_score, beta=2)]:
-            score(np.array([[1, 1], [1, 1]]),
-                  np.array([[0, 0], [0, 0]]),
-                  average='micro', zero_division=zero_division)
+            score(
+                np.array([[1, 1], [1, 1]]),
+                np.array([[0, 0], [0, 0]]),
+                average="micro",
+                zero_division=zero_division,
+            )
             assert len(record) == 0
 
-            score(np.array([[0, 0], [0, 0]]),
-                  np.array([[1, 1], [1, 1]]),
-                  average='micro', zero_division=zero_division)
+            score(
+                np.array([[0, 0], [0, 0]]),
+                np.array([[1, 1], [1, 1]]),
+                average="micro",
+                zero_division=zero_division,
+            )
             assert len(record) == 0
 
-            score(np.array([[0, 0], [0, 0]]),
-                  np.array([[0, 0], [0, 0]]),
-                  average='micro', zero_division=zero_division)
+            score(
+                np.array([[0, 0], [0, 0]]),
+                np.array([[0, 0], [0, 0]]),
+                average="micro",
+                zero_division=zero_division,
+            )
             if zero_division == "warn":
-                assert (str(record.pop().message) ==
-                        'F-score is ill-defined and '
-                        'being set to 0.0 due to no true nor predicted '
-                        'samples. Use `zero_division` parameter to '
-                        'control this behavior.')
+                assert (
+                    str(record.pop().message) == "F-score is ill-defined and "
+                    "being set to 0.0 due to no true nor predicted "
+                    "samples. Use `zero_division` parameter to "
+                    "control this behavior."
+                )
             else:
                 assert len(record) == 0
 
@@ -1966,21 +2440,29 @@ def test_prf_average_binary_data_non_binary():
     # Error if user does not explicitly set non-binary average mode
     y_true_mc = [1, 2, 3, 3]
     y_pred_mc = [1, 2, 3, 1]
-    msg_mc = (r"Target is multiclass but average='binary'. Please "
-              r"choose another average setting, one of \["
-              r"None, 'micro', 'macro', 'weighted'\].")
+    msg_mc = (
+        r"Target is multiclass but average='binary'. Please "
+        r"choose another average setting, one of \["
+        r"None, 'micro', 'macro', 'weighted'\]."
+    )
     y_true_ind = np.array([[0, 1, 1], [1, 0, 0], [0, 0, 1]])
     y_pred_ind = np.array([[0, 1, 0], [1, 0, 0], [0, 0, 1]])
-    msg_ind = (r"Target is multilabel-indicator but average='binary'. Please "
-               r"choose another average setting, one of \["
-               r"None, 'micro', 'macro', 'weighted', 'samples'\].")
+    msg_ind = (
+        r"Target is multilabel-indicator but average='binary'. Please "
+        r"choose another average setting, one of \["
+        r"None, 'micro', 'macro', 'weighted', 'samples'\]."
+    )
 
     for y_true, y_pred, msg in [
         (y_true_mc, y_pred_mc, msg_mc),
         (y_true_ind, y_pred_ind, msg_ind),
     ]:
-        for metric in [precision_score, recall_score, f1_score,
-                       partial(fbeta_score, beta=2)]:
+        for metric in [
+            precision_score,
+            recall_score,
+            f1_score,
+            partial(fbeta_score, beta=2),
+        ]:
             with pytest.raises(ValueError, match=msg):
                 metric(y_true, y_pred)
 
@@ -1988,12 +2470,12 @@ def test_prf_average_binary_data_non_binary():
 def test__check_targets():
     # Check that _check_targets correctly merges target types, squeezes
     # output and fails if input lengths differ.
-    IND = 'multilabel-indicator'
-    MC = 'multiclass'
-    BIN = 'binary'
-    CNT = 'continuous'
-    MMC = 'multiclass-multioutput'
-    MCN = 'continuous-multioutput'
+    IND = "multilabel-indicator"
+    MC = "multiclass"
+    BIN = "binary"
+    CNT = "continuous"
+    MMC = "multiclass-multioutput"
+    MCN = "continuous-multioutput"
     # all of length 3
     EXAMPLES = [
         (IND, np.array([[0, 1, 1], [1, 0, 0], [0, 0, 1]])),
@@ -2001,12 +2483,12 @@ def test__check_targets():
         (IND, np.array([[0, 1], [1, 0], [1, 1]])),
         (MC, [2, 3, 1]),
         (BIN, [0, 1, 1]),
-        (CNT, [0., 1.5, 1.]),
+        (CNT, [0.0, 1.5, 1.0]),
         (MC, np.array([[2], [3], [1]])),
         (BIN, np.array([[0], [1], [1]])),
-        (CNT, np.array([[0.], [1.5], [1.]])),
+        (CNT, np.array([[0.0], [1.5], [1.0]])),
         (MMC, np.array([[0, 2], [1, 3], [2, 3]])),
-        (MCN, np.array([[0.5, 2.], [1.1, 3.], [2., 3.]])),
+        (MCN, np.array([[0.5, 2.0], [1.1, 3.0], [2.0, 3.0]])),
     ]
     # expected type given input types, or None for error
     # (types will be tried in either order)
@@ -2014,11 +2496,9 @@ def test__check_targets():
         (IND, IND): IND,
         (MC, MC): MC,
         (BIN, BIN): BIN,
-
         (MC, IND): None,
         (BIN, IND): None,
         (BIN, MC): MC,
-
         # Disallowed types
         (CNT, CNT): None,
         (MMC, MMC): None,
@@ -2047,8 +2527,10 @@ def test__check_targets():
                 _check_targets(y1, y2)
 
             if type1 != type2:
-                err_msg = ("Classification metrics can't handle a mix "
-                           "of {0} and {1} targets".format(type1, type2))
+                err_msg = (
+                    "Classification metrics can't handle a mix "
+                    "of {0} and {1} targets".format(type1, type2)
+                )
                 with pytest.raises(ValueError, match=err_msg):
                     _check_targets(y1, y2)
 
@@ -2061,9 +2543,9 @@ def test__check_targets():
         else:
             merged_type, y1out, y2out = _check_targets(y1, y2)
             assert merged_type == expected
-            if merged_type.startswith('multilabel'):
-                assert y1out.format == 'csr'
-                assert y2out.format == 'csr'
+            if merged_type.startswith("multilabel"):
+                assert y1out.format == "csr"
+                assert y2out.format == "csr"
             else:
                 assert_array_equal(y1out, np.squeeze(y1))
                 assert_array_equal(y2out, np.squeeze(y2))
@@ -2071,12 +2553,14 @@ def test__check_targets():
                 _check_targets(y1[:-1], y2)
 
     # Make sure seq of seq is not supported
-    y1 = [(1, 2,), (0, 2, 3)]
-    y2 = [(2,), (0, 2,)]
-    msg = ('You appear to be using a legacy multi-label data representation. '
-           'Sequence of sequences are no longer supported; use a binary array'
-           ' or sparse matrix instead - the MultiLabelBinarizer'
-           ' transformer can convert to this format.')
+    y1 = [(1, 2), (0, 2, 3)]
+    y2 = [(2,), (0, 2)]
+    msg = (
+        "You appear to be using a legacy multi-label data representation. "
+        "Sequence of sequences are no longer supported; use a binary array"
+        " or sparse matrix instead - the MultiLabelBinarizer"
+        " transformer can convert to this format."
+    )
     with pytest.raises(ValueError, match=msg):
         _check_targets(y1, y2)
 
@@ -2085,7 +2569,7 @@ def test__check_targets_multiclass_with_both_y_true_and_y_pred_binary():
     # https://github.com/scikit-learn/scikit-learn/issues/8098
     y_true = [0, 1]
     y_pred = [0, -1]
-    assert _check_targets(y_true, y_pred)[0] == 'multiclass'
+    assert _check_targets(y_true, y_pred)[0] == "multiclass"
 
 
 def test_hinge_loss_binary():
@@ -2099,39 +2583,45 @@ def test_hinge_loss_binary():
 
 
 def test_hinge_loss_multiclass():
-    pred_decision = np.array([
-        [+0.36, -0.17, -0.58, -0.99],
-        [-0.54, -0.37, -0.48, -0.58],
-        [-1.45, -0.58, -0.38, -0.17],
-        [-0.54, -0.38, -0.48, -0.58],
-        [-2.36, -0.79, -0.27, +0.24],
-        [-1.45, -0.58, -0.38, -0.17]
-    ])
+    pred_decision = np.array(
+        [
+            [+0.36, -0.17, -0.58, -0.99],
+            [-0.54, -0.37, -0.48, -0.58],
+            [-1.45, -0.58, -0.38, -0.17],
+            [-0.54, -0.38, -0.48, -0.58],
+            [-2.36, -0.79, -0.27, +0.24],
+            [-1.45, -0.58, -0.38, -0.17],
+        ]
+    )
     y_true = np.array([0, 1, 2, 1, 3, 2])
-    dummy_losses = np.array([
-        1 - pred_decision[0][0] + pred_decision[0][1],
-        1 - pred_decision[1][1] + pred_decision[1][2],
-        1 - pred_decision[2][2] + pred_decision[2][3],
-        1 - pred_decision[3][1] + pred_decision[3][2],
-        1 - pred_decision[4][3] + pred_decision[4][2],
-        1 - pred_decision[5][2] + pred_decision[5][3]
-    ])
+    dummy_losses = np.array(
+        [
+            1 - pred_decision[0][0] + pred_decision[0][1],
+            1 - pred_decision[1][1] + pred_decision[1][2],
+            1 - pred_decision[2][2] + pred_decision[2][3],
+            1 - pred_decision[3][1] + pred_decision[3][2],
+            1 - pred_decision[4][3] + pred_decision[4][2],
+            1 - pred_decision[5][2] + pred_decision[5][3],
+        ]
+    )
     np.clip(dummy_losses, 0, None, out=dummy_losses)
     dummy_hinge_loss = np.mean(dummy_losses)
-    assert (hinge_loss(y_true, pred_decision) ==
-            dummy_hinge_loss)
+    assert hinge_loss(y_true, pred_decision) == dummy_hinge_loss
 
 
 def test_hinge_loss_multiclass_missing_labels_with_labels_none():
     y_true = np.array([0, 1, 2, 2])
-    pred_decision = np.array([
-        [+1.27, 0.034, -0.68, -1.40],
-        [-1.45, -0.58, -0.38, -0.17],
-        [-2.36, -0.79, -0.27, +0.24],
-        [-2.36, -0.79, -0.27, +0.24]
-    ])
-    error_message = ("Please include all labels in y_true "
-                     "or pass labels as third argument")
+    pred_decision = np.array(
+        [
+            [+1.27, 0.034, -0.68, -1.40],
+            [-1.45, -0.58, -0.38, -0.17],
+            [-2.36, -0.79, -0.27, +0.24],
+            [-2.36, -0.79, -0.27, +0.24],
+        ]
+    )
+    error_message = (
+        "Please include all labels in y_true or pass labels as third argument"
+    )
     with pytest.raises(ValueError, match=error_message):
         hinge_loss(y_true, pred_decision)
 
@@ -2141,47 +2631,53 @@ def test_hinge_loss_multiclass_no_consistent_pred_decision_shape():
     # argument
     y_true = np.array([2, 1, 0, 1, 0, 1, 1])
     pred_decision = np.array([0, 1, 2, 1, 0, 2, 1])
-    error_message = ("The shape of pred_decision cannot be 1d array"
-                     "with a multiclass target. pred_decision shape "
-                     "must be (n_samples, n_classes), that is "
-                     "(7, 3). Got: (7,)")
+    error_message = (
+        "The shape of pred_decision cannot be 1d array"
+        "with a multiclass target. pred_decision shape "
+        "must be (n_samples, n_classes), that is "
+        "(7, 3). Got: (7,)"
+    )
     with pytest.raises(ValueError, match=re.escape(error_message)):
         hinge_loss(y_true=y_true, pred_decision=pred_decision)
 
     # test for inconsistency between pred_decision shape and labels number
-    pred_decision = np.array([[0, 1], [0, 1], [0, 1], [0, 1],
-                              [2, 0], [0, 1], [1, 0]])
+    pred_decision = np.array([[0, 1], [0, 1], [0, 1], [0, 1], [2, 0], [0, 1], [1, 0]])
     labels = [0, 1, 2]
-    error_message = ("The shape of pred_decision is not "
-                     "consistent with the number of classes. "
-                     "With a multiclass target, pred_decision "
-                     "shape must be (n_samples, n_classes), that is "
-                     "(7, 3). Got: (7, 2)")
+    error_message = (
+        "The shape of pred_decision is not "
+        "consistent with the number of classes. "
+        "With a multiclass target, pred_decision "
+        "shape must be (n_samples, n_classes), that is "
+        "(7, 3). Got: (7, 2)"
+    )
     with pytest.raises(ValueError, match=re.escape(error_message)):
         hinge_loss(y_true=y_true, pred_decision=pred_decision, labels=labels)
 
 
 def test_hinge_loss_multiclass_with_missing_labels():
-    pred_decision = np.array([
-        [+0.36, -0.17, -0.58, -0.99],
-        [-0.55, -0.38, -0.48, -0.58],
-        [-1.45, -0.58, -0.38, -0.17],
-        [-0.55, -0.38, -0.48, -0.58],
-        [-1.45, -0.58, -0.38, -0.17]
-    ])
+    pred_decision = np.array(
+        [
+            [+0.36, -0.17, -0.58, -0.99],
+            [-0.55, -0.38, -0.48, -0.58],
+            [-1.45, -0.58, -0.38, -0.17],
+            [-0.55, -0.38, -0.48, -0.58],
+            [-1.45, -0.58, -0.38, -0.17],
+        ]
+    )
     y_true = np.array([0, 1, 2, 1, 2])
     labels = np.array([0, 1, 2, 3])
-    dummy_losses = np.array([
-        1 - pred_decision[0][0] + pred_decision[0][1],
-        1 - pred_decision[1][1] + pred_decision[1][2],
-        1 - pred_decision[2][2] + pred_decision[2][3],
-        1 - pred_decision[3][1] + pred_decision[3][2],
-        1 - pred_decision[4][2] + pred_decision[4][3]
-    ])
+    dummy_losses = np.array(
+        [
+            1 - pred_decision[0][0] + pred_decision[0][1],
+            1 - pred_decision[1][1] + pred_decision[1][2],
+            1 - pred_decision[2][2] + pred_decision[2][3],
+            1 - pred_decision[3][1] + pred_decision[3][2],
+            1 - pred_decision[4][2] + pred_decision[4][3],
+        ]
+    )
     np.clip(dummy_losses, 0, None, out=dummy_losses)
     dummy_hinge_loss = np.mean(dummy_losses)
-    assert (hinge_loss(y_true, pred_decision, labels=labels) ==
-            dummy_hinge_loss)
+    assert hinge_loss(y_true, pred_decision, labels=labels) == dummy_hinge_loss
 
 
 def test_hinge_loss_multiclass_missing_labels_only_two_unq_in_y_true():
@@ -2189,27 +2685,30 @@ def test_hinge_loss_multiclass_missing_labels_only_two_unq_in_y_true():
     # https://github.com/scikit-learn/scikit-learn/issues/17630
     # check that we can compute the hinge loss when providing an array
     # with labels allowing to not have all labels in y_true
-    pred_decision = np.array([
-        [+0.36, -0.17, -0.58],
-        [-0.15, -0.58, -0.48],
-        [-1.45, -0.58, -0.38],
-        [-0.55, -0.78, -0.42],
-        [-1.45, -0.58, -0.38]
-    ])
+    pred_decision = np.array(
+        [
+            [+0.36, -0.17, -0.58],
+            [-0.15, -0.58, -0.48],
+            [-1.45, -0.58, -0.38],
+            [-0.55, -0.78, -0.42],
+            [-1.45, -0.58, -0.38],
+        ]
+    )
     y_true = np.array([0, 2, 2, 0, 2])
     labels = np.array([0, 1, 2])
-    dummy_losses = np.array([
-        1 - pred_decision[0][0] + pred_decision[0][1],
-        1 - pred_decision[1][2] + pred_decision[1][0],
-        1 - pred_decision[2][2] + pred_decision[2][1],
-        1 - pred_decision[3][0] + pred_decision[3][2],
-        1 - pred_decision[4][2] + pred_decision[4][1]
-    ])
+    dummy_losses = np.array(
+        [
+            1 - pred_decision[0][0] + pred_decision[0][1],
+            1 - pred_decision[1][2] + pred_decision[1][0],
+            1 - pred_decision[2][2] + pred_decision[2][1],
+            1 - pred_decision[3][0] + pred_decision[3][2],
+            1 - pred_decision[4][2] + pred_decision[4][1],
+        ]
+    )
     np.clip(dummy_losses, 0, None, out=dummy_losses)
     dummy_hinge_loss = np.mean(dummy_losses)
     assert_almost_equal(
-        hinge_loss(y_true, pred_decision, labels=labels),
-        dummy_hinge_loss
+        hinge_loss(y_true, pred_decision, labels=labels), dummy_hinge_loss
     )
 
 
@@ -2217,102 +2716,152 @@ def test_hinge_loss_multiclass_invariance_lists():
     # Currently, invariance of string and integer labels cannot be tested
     # in common invariance tests because invariance tests for multiclass
     # decision functions is not implemented yet.
-    y_true = ['blue', 'green', 'red',
-              'green', 'white', 'red']
+    y_true = ["blue", "green", "red", "green", "white", "red"]
     pred_decision = [
         [+0.36, -0.17, -0.58, -0.99],
         [-0.55, -0.38, -0.48, -0.58],
         [-1.45, -0.58, -0.38, -0.17],
         [-0.55, -0.38, -0.48, -0.58],
         [-2.36, -0.79, -0.27, +0.24],
-        [-1.45, -0.58, -0.38, -0.17]]
-    dummy_losses = np.array([
-        1 - pred_decision[0][0] + pred_decision[0][1],
-        1 - pred_decision[1][1] + pred_decision[1][2],
-        1 - pred_decision[2][2] + pred_decision[2][3],
-        1 - pred_decision[3][1] + pred_decision[3][2],
-        1 - pred_decision[4][3] + pred_decision[4][2],
-        1 - pred_decision[5][2] + pred_decision[5][3]
-    ])
+        [-1.45, -0.58, -0.38, -0.17],
+    ]
+    dummy_losses = np.array(
+        [
+            1 - pred_decision[0][0] + pred_decision[0][1],
+            1 - pred_decision[1][1] + pred_decision[1][2],
+            1 - pred_decision[2][2] + pred_decision[2][3],
+            1 - pred_decision[3][1] + pred_decision[3][2],
+            1 - pred_decision[4][3] + pred_decision[4][2],
+            1 - pred_decision[5][2] + pred_decision[5][3],
+        ]
+    )
     np.clip(dummy_losses, 0, None, out=dummy_losses)
     dummy_hinge_loss = np.mean(dummy_losses)
-    assert (hinge_loss(y_true, pred_decision) ==
-            dummy_hinge_loss)
+    assert hinge_loss(y_true, pred_decision) == dummy_hinge_loss
 
 
 def test_log_loss():
     # binary case with symbolic labels ("no" < "yes")
     y_true = ["no", "no", "no", "yes", "yes", "yes"]
-    y_pred = np.array([[0.5, 0.5], [0.1, 0.9], [0.01, 0.99],
-                       [0.9, 0.1], [0.75, 0.25], [0.001, 0.999]])
+    y_pred = np.array(
+        [[0.5, 0.5], [0.1, 0.9], [0.01, 0.99], [0.9, 0.1], [0.75, 0.25], [0.001, 0.999]]
+    )
     loss = log_loss(y_true, y_pred)
-    assert_almost_equal(loss, 1.8817971)
+    loss_true = -np.mean(bernoulli.logpmf(np.array(y_true) == "yes", y_pred[:, 1]))
+    assert_allclose(loss, loss_true)
 
     # multiclass case; adapted from http://bit.ly/RJJHWA
     y_true = [1, 0, 2]
     y_pred = [[0.2, 0.7, 0.1], [0.6, 0.2, 0.2], [0.6, 0.1, 0.3]]
     loss = log_loss(y_true, y_pred, normalize=True)
-    assert_almost_equal(loss, 0.6904911)
+    assert_allclose(loss, 0.6904911)
 
     # check that we got all the shapes and axes right
     # by doubling the length of y_true and y_pred
     y_true *= 2
     y_pred *= 2
     loss = log_loss(y_true, y_pred, normalize=False)
-    assert_almost_equal(loss, 0.6904911 * 6, decimal=6)
-
-    # check eps and handling of absolute zero and one probabilities
-    y_pred = np.asarray(y_pred) > .5
-    loss = log_loss(y_true, y_pred, normalize=True, eps=.1)
-    assert_almost_equal(loss, log_loss(y_true, np.clip(y_pred, .1, .9)))
+    assert_allclose(loss, 0.6904911 * 6)
 
     # raise error if number of classes are not equal.
     y_true = [1, 0, 2]
-    y_pred = [[0.2, 0.7], [0.6, 0.5], [0.4, 0.1]]
+    y_pred = [[0.3, 0.7], [0.6, 0.4], [0.4, 0.6]]
     with pytest.raises(ValueError):
         log_loss(y_true, y_pred)
 
+    # raise error if labels do not contain all values of y_true
+    y_true = ["a", "b", "c"]
+    y_pred = [[0.9, 0.1, 0.0], [0.1, 0.9, 0.0], [0.1, 0.1, 0.8]]
+    labels = ["a", "c", "d"]
+    error_str = (
+        "y_true contains values {'b'} not belonging to the passed "
+        "labels ['a', 'c', 'd']."
+    )
+    with pytest.raises(ValueError, match=re.escape(error_str)):
+        log_loss(y_true, y_pred, labels=labels)
+
     # case when y_true is a string array object
     y_true = ["ham", "spam", "spam", "ham"]
-    y_pred = [[0.2, 0.7], [0.6, 0.5], [0.4, 0.1], [0.7, 0.2]]
+    y_pred = [[0.3, 0.7], [0.6, 0.4], [0.4, 0.6], [0.7, 0.3]]
     loss = log_loss(y_true, y_pred)
-    assert_almost_equal(loss, 1.0383217, decimal=6)
+    assert_allclose(loss, 0.7469410)
 
     # test labels option
 
     y_true = [2, 2]
-    y_pred = [[0.2, 0.7], [0.6, 0.5]]
+    y_pred = [[0.2, 0.8], [0.6, 0.4]]
     y_score = np.array([[0.1, 0.9], [0.1, 0.9]])
-    error_str = (r'y_true contains only one label \(2\). Please provide '
-                 r'the true labels explicitly through the labels argument.')
-    with pytest.raises(ValueError, match=error_str):
+    error_str = (
+        "y_true contains only one label (2). Please provide the list of all "
+        "expected class labels explicitly through the labels argument."
+    )
+    with pytest.raises(ValueError, match=re.escape(error_str)):
         log_loss(y_true, y_pred)
 
-    y_pred = [[0.2, 0.7], [0.6, 0.5], [0.2, 0.3]]
-    error_str = ('Found input variables with inconsistent numbers of samples: '
-                 '[3, 2]')
-    (ValueError, error_str, log_loss, y_true, y_pred)
+    y_pred = [[0.2, 0.8], [0.6, 0.4], [0.7, 0.3]]
+    error_str = "Found input variables with inconsistent numbers of samples: [3, 2]"
+    with pytest.raises(ValueError, match=re.escape(error_str)):
+        log_loss(y_true, y_pred)
 
     # works when the labels argument is used
 
     true_log_loss = -np.mean(np.log(y_score[:, 1]))
     calculated_log_loss = log_loss(y_true, y_score, labels=[1, 2])
-    assert_almost_equal(calculated_log_loss, true_log_loss)
+    assert_allclose(calculated_log_loss, true_log_loss)
 
     # ensure labels work when len(np.unique(y_true)) != y_pred.shape[1]
     y_true = [1, 2, 2]
-    y_score2 = [[0.2, 0.7, 0.3], [0.6, 0.5, 0.3], [0.3, 0.9, 0.1]]
+    y_score2 = [[0.7, 0.1, 0.2], [0.2, 0.7, 0.1], [0.1, 0.7, 0.2]]
     loss = log_loss(y_true, y_score2, labels=[1, 2, 3])
-    assert_almost_equal(loss, 1.0630345, decimal=6)
+    assert_allclose(loss, -np.log(0.7))
+
+
+@pytest.mark.parametrize("dtype", [np.float64, np.float32, np.float16])
+def test_log_loss_eps(dtype):
+    """Check the behaviour internal eps that changes depending on the input dtype.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/24315
+    """
+    y_true = np.array([0, 1], dtype=dtype)
+    y_pred = np.array([1, 0], dtype=dtype)
+
+    loss = log_loss(y_true, y_pred)
+    assert np.isfinite(loss)
+
+
+@pytest.mark.parametrize("dtype", [np.float64, np.float32, np.float16])
+def test_log_loss_not_probabilities_warning(dtype):
+    """Check that log_loss raises a warning when y_pred values don't sum to 1."""
+    y_true = np.array([0, 1, 1, 0])
+    y_pred = np.array([[0.2, 0.7], [0.6, 0.3], [0.4, 0.7], [0.8, 0.3]], dtype=dtype)
+
+    with pytest.warns(UserWarning, match="The y_prob values do not sum to one."):
+        log_loss(y_true, y_pred)
+
+
+@pytest.mark.parametrize(
+    "y_true, y_pred",
+    [
+        ([0, 1, 0], [0, 1, 0]),
+        ([0, 1, 0], [[1, 0], [0, 1], [1, 0]]),
+        ([0, 1, 2], [[1, 0, 0], [0, 1, 0], [0, 0, 1]]),
+    ],
+)
+def test_log_loss_perfect_predictions(y_true, y_pred):
+    """Check that log_loss returns 0 for perfect predictions."""
+    # Because of the clipping, the result is not exactly 0
+    assert log_loss(y_true, y_pred) == pytest.approx(0)
 
 
 def test_log_loss_pandas_input():
     # case when input is a pandas series and dataframe gh-5715
     y_tr = np.array(["ham", "spam", "spam", "ham"])
-    y_pr = np.array([[0.2, 0.7], [0.6, 0.5], [0.4, 0.1], [0.7, 0.2]])
+    y_pr = np.array([[0.3, 0.7], [0.6, 0.4], [0.4, 0.6], [0.7, 0.3]])
     types = [(MockDataFrame, MockDataFrame)]
     try:
-        from pandas import Series, DataFrame
+        from pandas import DataFrame, Series
+
         types.append((Series, DataFrame))
     except ImportError:
         pass
@@ -2320,63 +2869,211 @@ def test_log_loss_pandas_input():
         # y_pred dataframe, y_true series
         y_true, y_pred = TrueInputType(y_tr), PredInputType(y_pr)
         loss = log_loss(y_true, y_pred)
-        assert_almost_equal(loss, 1.0383217, decimal=6)
+        assert_allclose(loss, 0.7469410)
 
 
-def test_brier_score_loss():
+def test_log_loss_warnings():
+    expected_message = re.escape(
+        "Labels passed were ['spam', 'eggs', 'ham']. But this function "
+        "assumes labels are ordered lexicographically. "
+        "Pass the ordered labels=['eggs', 'ham', 'spam'] and ensure that "
+        "the columns of y_prob correspond to this ordering."
+    )
+    with pytest.warns(UserWarning, match=expected_message):
+        log_loss(
+            ["eggs", "spam", "ham"],
+            [[1, 0, 0], [0, 1, 0], [0, 0, 1]],
+            labels=["spam", "eggs", "ham"],
+        )
+
+
+def test_brier_score_loss_binary():
     # Check brier_score_loss function
     y_true = np.array([0, 1, 1, 0, 1, 1])
-    y_pred = np.array([0.1, 0.8, 0.9, 0.3, 1., 0.95])
-    true_score = linalg.norm(y_true - y_pred) ** 2 / len(y_true)
+    y_prob = np.array([0.1, 0.8, 0.9, 0.3, 1.0, 0.95])
+    true_score = linalg.norm(y_true - y_prob) ** 2 / len(y_true)
 
     assert_almost_equal(brier_score_loss(y_true, y_true), 0.0)
-    assert_almost_equal(brier_score_loss(y_true, y_pred), true_score)
-    assert_almost_equal(brier_score_loss(1. + y_true, y_pred),
-                        true_score)
-    assert_almost_equal(brier_score_loss(2 * y_true - 1, y_pred),
-                        true_score)
+    assert_almost_equal(brier_score_loss(y_true, y_prob), true_score)
+    assert_almost_equal(brier_score_loss(1.0 + y_true, y_prob), true_score)
+    assert_almost_equal(brier_score_loss(2 * y_true - 1, y_prob), true_score)
+
+    # check that using (n_samples, 2) y_prob or y_true gives the same score
+    y_prob_reshaped = np.column_stack((1 - y_prob, y_prob))
+    y_true_reshaped = np.column_stack((1 - y_true, y_true))
+    assert_almost_equal(brier_score_loss(y_true, y_prob_reshaped), true_score)
+    assert_almost_equal(brier_score_loss(y_true_reshaped, y_prob_reshaped), true_score)
+
+    # check scale_by_half argument
+    assert_almost_equal(
+        brier_score_loss(y_true, y_prob, scale_by_half="auto"), true_score
+    )
+    assert_almost_equal(
+        brier_score_loss(y_true, y_prob, scale_by_half=True), true_score
+    )
+    assert_almost_equal(
+        brier_score_loss(y_true, y_prob, scale_by_half=False), 2 * true_score
+    )
+
+    # calculate correctly when there's only one class in y_true
+    assert_almost_equal(brier_score_loss([-1], [0.4]), 0.4**2)
+    assert_almost_equal(brier_score_loss([0], [0.4]), 0.4**2)
+    assert_almost_equal(brier_score_loss([1], [0.4]), (1 - 0.4) ** 2)
+    assert_almost_equal(brier_score_loss(["foo"], [0.4], pos_label="bar"), 0.4**2)
+    assert_almost_equal(
+        brier_score_loss(["foo"], [0.4], pos_label="foo"),
+        (1 - 0.4) ** 2,
+    )
+
+
+def test_brier_score_loss_multiclass():
+    # test cases for multi-class
+    assert_almost_equal(
+        brier_score_loss(
+            ["eggs", "spam", "ham"],
+            [[1, 0, 0, 0], [0, 1, 0, 0], [0, 1, 0, 0]],
+            labels=["eggs", "ham", "spam", "yams"],
+        ),
+        2 / 3,
+    )
+
+    assert_almost_equal(
+        brier_score_loss(
+            [1, 0, 2], [[0.2, 0.7, 0.1], [0.6, 0.2, 0.2], [0.6, 0.1, 0.3]]
+        ),
+        0.41333333,
+    )
+
+    # check perfect predictions for 3 classes
+    assert_almost_equal(
+        brier_score_loss(
+            [0, 1, 2], [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]
+        ),
+        0,
+    )
+
+    # check perfectly incorrect predictions for 3 classes
+    assert_almost_equal(
+        brier_score_loss(
+            [0, 1, 2], [[0.0, 1.0, 0.0], [1.0, 0.0, 0.0], [1.0, 0.0, 0.0]]
+        ),
+        2,
+    )
+
+
+def test_brier_score_loss_invalid_inputs():
+    # binary case
+    y_true = np.array([0, 1, 1, 0, 1, 1])
+    y_prob = np.array([0.1, 0.8, 0.9, 0.3, 1.0, 0.95])
     with pytest.raises(ValueError):
-        brier_score_loss(y_true, y_pred[1:])
+        # bad length of y_prob
+        brier_score_loss(y_true, y_prob[1:])
     with pytest.raises(ValueError):
-        brier_score_loss(y_true, y_pred + 1.)
+        # y_pred has value greater than 1
+        brier_score_loss(y_true, y_prob + 1.0)
     with pytest.raises(ValueError):
-        brier_score_loss(y_true, y_pred - 1.)
+        # y_pred has value less than 0
+        brier_score_loss(y_true, y_prob - 1.0)
 
-    # ensure to raise an error for multiclass y_true
+    # multiclass case
+    y_true = np.array([1, 0, 2])
+    y_prob = np.array([[0.2, 0.7, 0.1], [0.6, 0.2, 0.2], [0.6, 0.1, 0.3]])
+    with pytest.raises(ValueError):
+        # bad length of y_pred
+        brier_score_loss(y_true, y_prob[1:])
+    with pytest.raises(ValueError):
+        # y_pred has value greater than 1
+        brier_score_loss(y_true, y_prob + 1.0)
+    with pytest.raises(ValueError):
+        # y_pred has value less than 0
+        brier_score_loss(y_true, y_prob - 1.0)
+
+    # raise an error for multiclass y_true and binary y_prob
     y_true = np.array([0, 1, 2, 0])
-    y_pred = np.array([0.8, 0.6, 0.4, 0.2])
+    y_prob = np.array([0.8, 0.6, 0.4, 0.2])
+    error_message = re.escape(
+        "The type of the target inferred from y_true is multiclass "
+        "but should be binary according to the shape of y_prob."
+    )
+    with pytest.raises(ValueError, match=error_message):
+        brier_score_loss(y_true, y_prob)
+
+    # raise an error for wrong number of classes
+    y_true = [0, 1, 2]
+    y_prob = [[1, 0], [0, 1], [0, 1]]
     error_message = (
-        "Only binary classification is supported. The type of the target is "
-        "multiclass"
+        "y_true and y_prob contain different number of "
+        "classes: 3 vs 2. Please provide the true "
+        "labels explicitly through the labels argument. "
+        "Classes found in "
+        "y_true: [0 1 2]"
     )
+    with pytest.raises(ValueError, match=re.escape(error_message)):
+        brier_score_loss(y_true, y_prob)
 
-    with pytest.raises(ValueError, match=error_message):
-        brier_score_loss(y_true, y_pred)
+    y_true = ["eggs", "spam", "ham"]
+    y_prob = [[1, 0, 0], [0, 1, 0], [0, 1, 0]]
+    labels = ["eggs", "spam", "ham", "yams"]
+    error_message = (
+        "The number of classes in labels is different "
+        "from that in y_prob. Classes found in "
+        "labels: ['eggs' 'ham' 'spam' 'yams']"
+    )
+    with pytest.raises(ValueError, match=re.escape(error_message)):
+        brier_score_loss(y_true, y_prob, labels=labels)
 
-    # calculate correctly when there's only one class in y_true
-    assert_almost_equal(brier_score_loss([-1], [0.4]), 0.16)
-    assert_almost_equal(brier_score_loss([0], [0.4]), 0.16)
-    assert_almost_equal(brier_score_loss([1], [0.4]), 0.36)
-    assert_almost_equal(
-        brier_score_loss(['foo'], [0.4], pos_label='bar'), 0.16)
-    assert_almost_equal(
-        brier_score_loss(['foo'], [0.4], pos_label='foo'), 0.36)
+    # raise error message when there's only one class in y_true
+    y_true = ["eggs"]
+    y_prob = [[0.9, 0.1]]
+    error_message = (
+        "y_true contains only one label (eggs). Please "
+        "provide the list of all expected class labels explicitly through the "
+        "labels argument."
+    )
+    with pytest.raises(ValueError, match=re.escape(error_message)):
+        brier_score_loss(y_true, y_prob)
+
+    # error is fixed when labels is specified
+    assert_almost_equal(brier_score_loss(y_true, y_prob, labels=["eggs", "ham"]), 0.01)
+
+
+def test_brier_score_loss_warnings():
+    expected_message = re.escape(
+        "Labels passed were ['spam', 'eggs', 'ham']. But this function "
+        "assumes labels are ordered lexicographically. "
+        "Pass the ordered labels=['eggs', 'ham', 'spam'] and ensure that "
+        "the columns of y_prob correspond to this ordering."
+    )
+    with pytest.warns(UserWarning, match=expected_message):
+        brier_score_loss(
+            ["eggs", "spam", "ham"],
+            [
+                [1, 0, 0],
+                [0, 1, 0],
+                [0, 0, 1],
+            ],
+            labels=["spam", "eggs", "ham"],
+        )
 
 
 def test_balanced_accuracy_score_unseen():
-    assert_warns_message(UserWarning, 'y_pred contains classes not in y_true',
-                         balanced_accuracy_score, [0, 0, 0], [0, 0, 1])
+    msg = "y_pred contains classes not in y_true"
+    with pytest.warns(UserWarning, match=msg):
+        balanced_accuracy_score([0, 0, 0], [0, 0, 1])
 
 
-@pytest.mark.parametrize('y_true,y_pred',
-                         [
-                             (['a', 'b', 'a', 'b'], ['a', 'a', 'a', 'b']),
-                             (['a', 'b', 'c', 'b'], ['a', 'a', 'a', 'b']),
-                             (['a', 'a', 'a', 'b'], ['a', 'b', 'c', 'b']),
-                         ])
+@pytest.mark.parametrize(
+    "y_true,y_pred",
+    [
+        (["a", "b", "a", "b"], ["a", "a", "a", "b"]),
+        (["a", "b", "c", "b"], ["a", "a", "a", "b"]),
+        (["a", "a", "a", "b"], ["a", "b", "c", "b"]),
+    ],
+)
 def test_balanced_accuracy_score(y_true, y_pred):
-    macro_recall = recall_score(y_true, y_pred, average='macro',
-                                labels=np.unique(y_true))
+    macro_recall = recall_score(
+        y_true, y_pred, average="macro", labels=np.unique(y_true)
+    )
     with ignore_warnings():
         # Warnings are tested in test_balanced_accuracy_score_unseen
         balanced = balanced_accuracy_score(y_true, y_pred)
@@ -2384,3 +3081,317 @@ def test_balanced_accuracy_score(y_true, y_pred):
     adjusted = balanced_accuracy_score(y_true, y_pred, adjusted=True)
     chance = balanced_accuracy_score(y_true, np.full_like(y_true, y_true[0]))
     assert adjusted == (balanced - chance) / (1 - chance)
+
+
+@pytest.mark.parametrize(
+    "metric",
+    [
+        jaccard_score,
+        f1_score,
+        partial(fbeta_score, beta=0.5),
+        precision_recall_fscore_support,
+        precision_score,
+        recall_score,
+        brier_score_loss,
+    ],
+)
+@pytest.mark.parametrize(
+    "classes", [(False, True), (0, 1), (0.0, 1.0), ("zero", "one")]
+)
+def test_classification_metric_pos_label_types(metric, classes):
+    """Check that the metric works with different types of `pos_label`.
+
+    We can expect `pos_label` to be a bool, an integer, a float, a string.
+    No error should be raised for those types.
+    """
+    rng = np.random.RandomState(42)
+    n_samples, pos_label = 10, classes[-1]
+    y_true = rng.choice(classes, size=n_samples, replace=True)
+    if metric is brier_score_loss:
+        # brier score loss requires probabilities
+        y_pred = rng.uniform(size=n_samples)
+    else:
+        y_pred = y_true.copy()
+    result = metric(y_true, y_pred, pos_label=pos_label)
+    assert not np.any(np.isnan(result))
+
+
+@pytest.mark.parametrize(
+    "y_true, y_pred, expected_score",
+    [
+        (np.array([0, 1]), np.array([1, 0]), 0.0),
+        (np.array([0, 1]), np.array([0, 1]), 1.0),
+        (np.array([0, 1]), np.array([0, 0]), 0.0),
+        (np.array([0, 0]), np.array([0, 0]), 1.0),
+    ],
+)
+def test_f1_for_small_binary_inputs_with_zero_division(y_true, y_pred, expected_score):
+    """Check the behaviour of `zero_division` for f1-score.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/26965
+    """
+    assert f1_score(y_true, y_pred, zero_division=1.0) == pytest.approx(expected_score)
+
+
+@pytest.mark.parametrize(
+    "scoring",
+    [
+        make_scorer(f1_score, zero_division=np.nan),
+        make_scorer(fbeta_score, beta=2, zero_division=np.nan),
+        make_scorer(precision_score, zero_division=np.nan),
+        make_scorer(recall_score, zero_division=np.nan),
+    ],
+)
+def test_classification_metric_division_by_zero_nan_validaton(scoring):
+    """Check that we validate `np.nan` properly for classification metrics.
+
+    With `n_jobs=2` in cross-validation, the `np.nan` used for the singleton will be
+    different in the sub-process and we should not use the `is` operator but
+    `math.isnan`.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/27563
+    """
+    X, y = datasets.make_classification(random_state=0)
+    classifier = DecisionTreeClassifier(max_depth=3, random_state=0).fit(X, y)
+    cross_val_score(classifier, X, y, scoring=scoring, n_jobs=2, error_score="raise")
+
+
+def test_d2_log_loss_score():
+    y_true = [0, 0, 0, 1, 1, 1]
+    y_true_string = ["no", "no", "no", "yes", "yes", "yes"]
+    y_pred = np.array(
+        [
+            [0.5, 0.5],
+            [0.9, 0.1],
+            [0.4, 0.6],
+            [0.6, 0.4],
+            [0.35, 0.65],
+            [0.01, 0.99],
+        ]
+    )
+    y_pred_null = np.array(
+        [
+            [0.5, 0.5],
+            [0.5, 0.5],
+            [0.5, 0.5],
+            [0.5, 0.5],
+            [0.5, 0.5],
+            [0.5, 0.5],
+        ]
+    )
+    d2_score = d2_log_loss_score(y_true=y_true, y_pred=y_pred)
+    log_likelihood = log_loss(y_true=y_true, y_pred=y_pred, normalize=False)
+    log_likelihood_null = log_loss(y_true=y_true, y_pred=y_pred_null, normalize=False)
+    d2_score_true = 1 - log_likelihood / log_likelihood_null
+    assert d2_score == pytest.approx(d2_score_true)
+
+    # check that using sample weight also gives the correct d2 score
+    sample_weight = np.array([2, 1, 3, 4, 3, 1])
+    y_pred_null[:, 0] = sample_weight[:3].sum() / sample_weight.sum()
+    y_pred_null[:, 1] = sample_weight[3:].sum() / sample_weight.sum()
+    d2_score = d2_log_loss_score(
+        y_true=y_true, y_pred=y_pred, sample_weight=sample_weight
+    )
+    log_likelihood = log_loss(
+        y_true=y_true,
+        y_pred=y_pred,
+        sample_weight=sample_weight,
+        normalize=False,
+    )
+    log_likelihood_null = log_loss(
+        y_true=y_true,
+        y_pred=y_pred_null,
+        sample_weight=sample_weight,
+        normalize=False,
+    )
+    d2_score_true = 1 - log_likelihood / log_likelihood_null
+    assert d2_score == pytest.approx(d2_score_true)
+
+    # check if good predictions give a relatively higher value for the d2 score
+    y_pred = np.array(
+        [
+            [0.9, 0.1],
+            [0.8, 0.2],
+            [0.9, 0.1],
+            [0.1, 0.9],
+            [0.2, 0.8],
+            [0.1, 0.9],
+        ]
+    )
+    d2_score = d2_log_loss_score(y_true, y_pred)
+    assert 0.5 < d2_score < 1.0
+    # check that a similar value is obtained for string labels
+    d2_score_string = d2_log_loss_score(y_true_string, y_pred)
+    assert d2_score_string == pytest.approx(d2_score)
+
+    # check if poor predictions gives a relatively low value for the d2 score
+    y_pred = np.array(
+        [
+            [0.5, 0.5],
+            [0.1, 0.9],
+            [0.1, 0.9],
+            [0.9, 0.1],
+            [0.75, 0.25],
+            [0.1, 0.9],
+        ]
+    )
+    d2_score = d2_log_loss_score(y_true, y_pred)
+    assert d2_score < 0
+    # check that a similar value is obtained for string labels
+    d2_score_string = d2_log_loss_score(y_true_string, y_pred)
+    assert d2_score_string == pytest.approx(d2_score)
+
+    # check if simply using the average of the classes as the predictions
+    # gives a d2 score of 0
+    y_true = [0, 0, 0, 1, 1, 1]
+    y_pred = np.array(
+        [
+            [0.5, 0.5],
+            [0.5, 0.5],
+            [0.5, 0.5],
+            [0.5, 0.5],
+            [0.5, 0.5],
+            [0.5, 0.5],
+        ]
+    )
+    d2_score = d2_log_loss_score(y_true, y_pred)
+    assert d2_score == 0
+    d2_score_string = d2_log_loss_score(y_true_string, y_pred)
+    assert d2_score_string == 0
+
+    # check if simply using the average of the classes as the predictions
+    # gives a d2 score of 0 when the positive class has a higher proportion
+    y_true = [0, 1, 1, 1]
+    y_true_string = ["no", "yes", "yes", "yes"]
+    y_pred = np.array([[0.25, 0.75], [0.25, 0.75], [0.25, 0.75], [0.25, 0.75]])
+    d2_score = d2_log_loss_score(y_true, y_pred)
+    assert d2_score == 0
+    d2_score_string = d2_log_loss_score(y_true_string, y_pred)
+    assert d2_score_string == 0
+    sample_weight = [2, 2, 2, 2]
+    d2_score_with_sample_weight = d2_log_loss_score(
+        y_true, y_pred, sample_weight=sample_weight
+    )
+    assert d2_score_with_sample_weight == 0
+
+    # check that the d2 scores seem correct when more than 2
+    # labels are specified
+    y_true = ["high", "high", "low", "neutral"]
+    sample_weight = [1.4, 0.6, 0.8, 0.2]
+
+    y_pred = np.array(
+        [
+            [0.8, 0.1, 0.1],
+            [0.8, 0.1, 0.1],
+            [0.1, 0.8, 0.1],
+            [0.1, 0.1, 0.8],
+        ]
+    )
+    d2_score = d2_log_loss_score(y_true, y_pred)
+    assert 0.5 < d2_score < 1.0
+    d2_score = d2_log_loss_score(y_true, y_pred, sample_weight=sample_weight)
+    assert 0.5 < d2_score < 1.0
+
+    y_pred = np.array(
+        [
+            [0.2, 0.5, 0.3],
+            [0.1, 0.7, 0.2],
+            [0.1, 0.1, 0.8],
+            [0.2, 0.7, 0.1],
+        ]
+    )
+    d2_score = d2_log_loss_score(y_true, y_pred)
+    assert d2_score < 0
+    d2_score = d2_log_loss_score(y_true, y_pred, sample_weight=sample_weight)
+    assert d2_score < 0
+
+
+def test_d2_log_loss_score_missing_labels():
+    """Check that d2_log_loss_score works when not all labels are present in y_true
+
+    non-regression test for https://github.com/scikit-learn/scikit-learn/issues/30713
+    """
+    y_true = [2, 0, 2, 0]
+    labels = [0, 1, 2]
+    sample_weight = [1.4, 0.6, 0.7, 0.3]
+    y_pred = np.tile([1, 0, 0], (4, 1))
+
+    log_loss_obs = log_loss(y_true, y_pred, sample_weight=sample_weight, labels=labels)
+
+    # Null model consists of weighted average of the classes.
+    # Given that the sum of the weights is 3,
+    # - weighted average of 0s is (0.6 + 0.3) / 3 = 0.3
+    # - weighted average of 1s is 0
+    # - weighted average of 2s is (1.4 + 0.7) / 3 = 0.7
+    y_pred_null = np.tile([0.3, 0, 0.7], (4, 1))
+    log_loss_null = log_loss(
+        y_true, y_pred_null, sample_weight=sample_weight, labels=labels
+    )
+
+    expected_d2_score = 1 - log_loss_obs / log_loss_null
+    d2_score = d2_log_loss_score(
+        y_true, y_pred, sample_weight=sample_weight, labels=labels
+    )
+    assert_allclose(d2_score, expected_d2_score)
+
+
+def test_d2_log_loss_score_label_order():
+    """Check that d2_log_loss_score doesn't depend on the order of the labels."""
+    y_true = [2, 0, 2, 0]
+    y_pred = np.tile([1, 0, 0], (4, 1))
+
+    d2_score = d2_log_loss_score(y_true, y_pred, labels=[0, 1, 2])
+    d2_score_other = d2_log_loss_score(y_true, y_pred, labels=[0, 2, 1])
+
+    assert_allclose(d2_score, d2_score_other)
+
+
+def test_d2_log_loss_score_raises():
+    """Test that d2_log_loss_score raises the appropriate errors on
+    invalid inputs."""
+    y_true = [0, 1, 2]
+    y_pred = [[0.2, 0.8], [0.5, 0.5], [0.4, 0.6]]
+    err = "contain different number of classes"
+    with pytest.raises(ValueError, match=err):
+        d2_log_loss_score(y_true, y_pred)
+
+    # check error if the number of classes in labels do not match the number
+    # of classes in y_pred.
+    y_true = [0, 1, 2]
+    y_pred = [[0.5, 0.5], [0.5, 0.5], [0.5, 0.5]]
+    labels = [0, 1, 2]
+    err = "number of classes in labels is different"
+    with pytest.raises(ValueError, match=err):
+        d2_log_loss_score(y_true, y_pred, labels=labels)
+
+    # check error if y_true and y_pred do not have equal lengths
+    y_true = [0, 1, 2]
+    y_pred = [[0.5, 0.5, 0.5], [0.6, 0.3, 0.1]]
+    err = "inconsistent numbers of samples"
+    with pytest.raises(ValueError, match=err):
+        d2_log_loss_score(y_true, y_pred)
+
+    # check warning for samples < 2
+    y_true = [1]
+    y_pred = [[0.5, 0.5]]
+    err = "score is not well-defined"
+    with pytest.warns(UndefinedMetricWarning, match=err):
+        d2_log_loss_score(y_true, y_pred)
+
+    # check error when y_true only has 1 label
+    y_true = [1, 1, 1]
+    y_pred = [[0.5, 0.5], [0.5, 0.5], [0.5, 0.5]]
+    err = "y_true contains only one label"
+    with pytest.raises(ValueError, match=err):
+        d2_log_loss_score(y_true, y_pred)
+
+    # check error when y_true only has 1 label and labels also has
+    # only 1 label
+    y_true = [1, 1, 1]
+    labels = [1]
+    y_pred = [[0.5, 0.5], [0.5, 0.5], [0.5, 0.5]]
+    err = "The labels array needs to contain at least two"
+    with pytest.raises(ValueError, match=err):
+        d2_log_loss_score(y_true, y_pred, labels=labels)
diff --git a/sklearn/metrics/tests/test_common.py b/sklearn/metrics/tests/test_common.py
index dbf1bdd458f1a..238ea821d8340 100644
--- a/sklearn/metrics/tests/test_common.py
+++ b/sklearn/metrics/tests/test_common.py
@@ -1,69 +1,93 @@
-
+import math
 from functools import partial
 from inspect import signature
-from itertools import product
-from itertools import chain
-from itertools import permutations
+from itertools import chain, permutations, product
 
 import numpy as np
-import scipy.sparse as sp
-
 import pytest
 
+from sklearn._config import config_context
 from sklearn.datasets import make_multilabel_classification
+from sklearn.exceptions import UndefinedMetricWarning
+from sklearn.metrics import (
+    accuracy_score,
+    average_precision_score,
+    balanced_accuracy_score,
+    brier_score_loss,
+    cohen_kappa_score,
+    confusion_matrix,
+    coverage_error,
+    d2_absolute_error_score,
+    d2_pinball_score,
+    d2_tweedie_score,
+    dcg_score,
+    det_curve,
+    explained_variance_score,
+    f1_score,
+    fbeta_score,
+    hamming_loss,
+    hinge_loss,
+    jaccard_score,
+    label_ranking_average_precision_score,
+    label_ranking_loss,
+    log_loss,
+    matthews_corrcoef,
+    max_error,
+    mean_absolute_error,
+    mean_absolute_percentage_error,
+    mean_gamma_deviance,
+    mean_pinball_loss,
+    mean_poisson_deviance,
+    mean_squared_error,
+    mean_squared_log_error,
+    mean_tweedie_deviance,
+    median_absolute_error,
+    multilabel_confusion_matrix,
+    ndcg_score,
+    precision_recall_curve,
+    precision_score,
+    r2_score,
+    recall_score,
+    roc_auc_score,
+    roc_curve,
+    root_mean_squared_error,
+    root_mean_squared_log_error,
+    top_k_accuracy_score,
+    zero_one_loss,
+)
+from sklearn.metrics._base import _average_binary_score
+from sklearn.metrics.pairwise import (
+    additive_chi2_kernel,
+    chi2_kernel,
+    cosine_distances,
+    cosine_similarity,
+    euclidean_distances,
+    linear_kernel,
+    paired_cosine_distances,
+    paired_euclidean_distances,
+    polynomial_kernel,
+    rbf_kernel,
+    sigmoid_kernel,
+)
 from sklearn.preprocessing import LabelBinarizer
-from sklearn.utils.multiclass import type_of_target
-from sklearn.utils.validation import _num_samples
-from sklearn.utils.validation import check_random_state
 from sklearn.utils import shuffle
-
-from sklearn.utils._testing import assert_allclose
-from sklearn.utils._testing import assert_almost_equal
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils._testing import assert_array_less
-from sklearn.utils._testing import ignore_warnings
-
-from sklearn.metrics import accuracy_score
-from sklearn.metrics import average_precision_score
-from sklearn.metrics import balanced_accuracy_score
-from sklearn.metrics import brier_score_loss
-from sklearn.metrics import cohen_kappa_score
-from sklearn.metrics import confusion_matrix
-from sklearn.metrics import coverage_error
-from sklearn.metrics import det_curve
-from sklearn.metrics import explained_variance_score
-from sklearn.metrics import f1_score
-from sklearn.metrics import fbeta_score
-from sklearn.metrics import hamming_loss
-from sklearn.metrics import hinge_loss
-from sklearn.metrics import jaccard_score
-from sklearn.metrics import label_ranking_average_precision_score
-from sklearn.metrics import label_ranking_loss
-from sklearn.metrics import log_loss
-from sklearn.metrics import max_error
-from sklearn.metrics import matthews_corrcoef
-from sklearn.metrics import mean_absolute_error
-from sklearn.metrics import mean_absolute_percentage_error
-from sklearn.metrics import mean_squared_error
-from sklearn.metrics import mean_tweedie_deviance
-from sklearn.metrics import mean_poisson_deviance
-from sklearn.metrics import mean_gamma_deviance
-from sklearn.metrics import median_absolute_error
-from sklearn.metrics import multilabel_confusion_matrix
-from sklearn.metrics import mean_pinball_loss
-from sklearn.metrics import precision_recall_curve
-from sklearn.metrics import precision_score
-from sklearn.metrics import r2_score
-from sklearn.metrics import recall_score
-from sklearn.metrics import roc_auc_score
-from sklearn.metrics import roc_curve
-from sklearn.metrics import zero_one_loss
-from sklearn.metrics import ndcg_score
-from sklearn.metrics import dcg_score
-from sklearn.metrics import top_k_accuracy_score
-
-from sklearn.metrics._base import _average_binary_score
-
+from sklearn.utils._array_api import (
+    _atol_for_type,
+    _convert_to_numpy,
+    _get_namespace_device_dtype_ids,
+    yield_namespace_device_dtype_combinations,
+)
+from sklearn.utils._testing import (
+    _array_api_for_tests,
+    assert_allclose,
+    assert_almost_equal,
+    assert_array_equal,
+    assert_array_less,
+    ignore_warnings,
+)
+from sklearn.utils.fixes import COO_CONTAINERS, parse_version, sp_version
+from sklearn.utils.multiclass import type_of_target
+from sklearn.utils.validation import _num_samples, check_random_state
 
 # Note toward developers about metric testing
 # -------------------------------------------
@@ -79,7 +103,7 @@
 # all metrics that have the same behavior.
 #
 # Two types of datastructures are used in order to implement this system:
-# dictionaries of metrics and lists of metrics wit common properties.
+# dictionaries of metrics and lists of metrics with common properties.
 #
 # Dictionaries of metrics
 # ------------------------
@@ -102,43 +126,44 @@
     "max_error": max_error,
     "mean_absolute_error": mean_absolute_error,
     "mean_squared_error": mean_squared_error,
+    "mean_squared_log_error": mean_squared_log_error,
     "mean_pinball_loss": mean_pinball_loss,
     "median_absolute_error": median_absolute_error,
     "mean_absolute_percentage_error": mean_absolute_percentage_error,
     "explained_variance_score": explained_variance_score,
-    "r2_score": partial(r2_score, multioutput='variance_weighted'),
+    "r2_score": partial(r2_score, multioutput="variance_weighted"),
+    "root_mean_squared_error": root_mean_squared_error,
+    "root_mean_squared_log_error": root_mean_squared_log_error,
     "mean_normal_deviance": partial(mean_tweedie_deviance, power=0),
     "mean_poisson_deviance": mean_poisson_deviance,
     "mean_gamma_deviance": mean_gamma_deviance,
-    "mean_compound_poisson_deviance":
-    partial(mean_tweedie_deviance, power=1.4),
+    "mean_compound_poisson_deviance": partial(mean_tweedie_deviance, power=1.4),
+    "d2_tweedie_score": partial(d2_tweedie_score, power=1.4),
+    "d2_pinball_score": d2_pinball_score,
+    "d2_absolute_error_score": d2_absolute_error_score,
 }
 
 CLASSIFICATION_METRICS = {
     "accuracy_score": accuracy_score,
     "balanced_accuracy_score": balanced_accuracy_score,
-    "adjusted_balanced_accuracy_score": partial(balanced_accuracy_score,
-                                                adjusted=True),
+    "adjusted_balanced_accuracy_score": partial(balanced_accuracy_score, adjusted=True),
     "unnormalized_accuracy_score": partial(accuracy_score, normalize=False),
-
     # `confusion_matrix` returns absolute values and hence behaves unnormalized
     # . Naming it with an unnormalized_ prefix is necessary for this module to
     # skip sample_weight scaling checks which will fail for unnormalized
     # metrics.
     "unnormalized_confusion_matrix": confusion_matrix,
     "normalized_confusion_matrix": lambda *args, **kwargs: (
-        confusion_matrix(*args, **kwargs).astype('float') / confusion_matrix(
-            *args, **kwargs).sum(axis=1)[:, np.newaxis]
+        confusion_matrix(*args, **kwargs).astype("float")
+        / confusion_matrix(*args, **kwargs).sum(axis=1)[:, np.newaxis]
     ),
-
     "unnormalized_multilabel_confusion_matrix": multilabel_confusion_matrix,
-    "unnormalized_multilabel_confusion_matrix_sample":
-        partial(multilabel_confusion_matrix, samplewise=True),
+    "unnormalized_multilabel_confusion_matrix_sample": partial(
+        multilabel_confusion_matrix, samplewise=True
+    ),
     "hamming_loss": hamming_loss,
-
     "zero_one_loss": zero_one_loss,
     "unnormalized_zero_one_loss": partial(zero_one_loss, normalize=False),
-
     # These are needed to test averaging
     "jaccard_score": jaccard_score,
     "precision_score": precision_score,
@@ -147,35 +172,30 @@
     "f2_score": partial(fbeta_score, beta=2),
     "f0.5_score": partial(fbeta_score, beta=0.5),
     "matthews_corrcoef_score": matthews_corrcoef,
-
     "weighted_f0.5_score": partial(fbeta_score, average="weighted", beta=0.5),
     "weighted_f1_score": partial(f1_score, average="weighted"),
     "weighted_f2_score": partial(fbeta_score, average="weighted", beta=2),
     "weighted_precision_score": partial(precision_score, average="weighted"),
     "weighted_recall_score": partial(recall_score, average="weighted"),
     "weighted_jaccard_score": partial(jaccard_score, average="weighted"),
-
     "micro_f0.5_score": partial(fbeta_score, average="micro", beta=0.5),
     "micro_f1_score": partial(f1_score, average="micro"),
     "micro_f2_score": partial(fbeta_score, average="micro", beta=2),
     "micro_precision_score": partial(precision_score, average="micro"),
     "micro_recall_score": partial(recall_score, average="micro"),
     "micro_jaccard_score": partial(jaccard_score, average="micro"),
-
     "macro_f0.5_score": partial(fbeta_score, average="macro", beta=0.5),
     "macro_f1_score": partial(f1_score, average="macro"),
     "macro_f2_score": partial(fbeta_score, average="macro", beta=2),
     "macro_precision_score": partial(precision_score, average="macro"),
     "macro_recall_score": partial(recall_score, average="macro"),
     "macro_jaccard_score": partial(jaccard_score, average="macro"),
-
     "samples_f0.5_score": partial(fbeta_score, average="samples", beta=0.5),
     "samples_f1_score": partial(f1_score, average="samples"),
     "samples_f2_score": partial(fbeta_score, average="samples", beta=2),
     "samples_precision_score": partial(precision_score, average="samples"),
     "samples_recall_score": partial(recall_score, average="samples"),
     "samples_jaccard_score": partial(jaccard_score, average="samples"),
-
     "cohen_kappa_score": cohen_kappa_score,
 }
 
@@ -197,14 +217,18 @@ def precision_recall_curve_padded_thresholds(*args, **kwargs):
 
     pad_threshholds = len(precision) - len(thresholds)
 
-    return np.array([
-        precision,
-        recall,
-        np.pad(thresholds.astype(np.float64),
-               pad_width=(0, pad_threshholds),
-               mode='constant',
-               constant_values=[np.nan])
-    ])
+    return np.array(
+        [
+            precision,
+            recall,
+            np.pad(
+                thresholds.astype(np.float64),
+                pad_width=(0, pad_threshholds),
+                mode="constant",
+                constant_values=[np.nan],
+            ),
+        ]
+    )
 
 
 CURVE_METRICS = {
@@ -218,37 +242,33 @@ def precision_recall_curve_padded_thresholds(*args, **kwargs):
     "label_ranking_loss": label_ranking_loss,
     "log_loss": log_loss,
     "unnormalized_log_loss": partial(log_loss, normalize=False),
-
     "hinge_loss": hinge_loss,
-
     "brier_score_loss": brier_score_loss,
-
     "roc_auc_score": roc_auc_score,  # default: average="macro"
     "weighted_roc_auc": partial(roc_auc_score, average="weighted"),
     "samples_roc_auc": partial(roc_auc_score, average="samples"),
     "micro_roc_auc": partial(roc_auc_score, average="micro"),
-    "ovr_roc_auc": partial(roc_auc_score, average="macro", multi_class='ovr'),
-    "weighted_ovr_roc_auc": partial(roc_auc_score, average="weighted",
-                                    multi_class='ovr'),
-    "ovo_roc_auc": partial(roc_auc_score, average="macro", multi_class='ovo'),
-    "weighted_ovo_roc_auc": partial(roc_auc_score, average="weighted",
-                                    multi_class='ovo'),
+    "ovr_roc_auc": partial(roc_auc_score, average="macro", multi_class="ovr"),
+    "weighted_ovr_roc_auc": partial(
+        roc_auc_score, average="weighted", multi_class="ovr"
+    ),
+    "ovo_roc_auc": partial(roc_auc_score, average="macro", multi_class="ovo"),
+    "weighted_ovo_roc_auc": partial(
+        roc_auc_score, average="weighted", multi_class="ovo"
+    ),
     "partial_roc_auc": partial(roc_auc_score, max_fpr=0.5),
-
-    "average_precision_score":
-    average_precision_score,  # default: average="macro"
-    "weighted_average_precision_score":
-    partial(average_precision_score, average="weighted"),
-    "samples_average_precision_score":
-    partial(average_precision_score, average="samples"),
-    "micro_average_precision_score":
-    partial(average_precision_score, average="micro"),
-    "label_ranking_average_precision_score":
-    label_ranking_average_precision_score,
+    "average_precision_score": average_precision_score,  # default: average="macro"
+    "weighted_average_precision_score": partial(
+        average_precision_score, average="weighted"
+    ),
+    "samples_average_precision_score": partial(
+        average_precision_score, average="samples"
+    ),
+    "micro_average_precision_score": partial(average_precision_score, average="micro"),
+    "label_ranking_average_precision_score": label_ranking_average_precision_score,
     "ndcg_score": ndcg_score,
     "dcg_score": dcg_score,
-
-    "top_k_accuracy_score": top_k_accuracy_score
+    "top_k_accuracy_score": top_k_accuracy_score,
 }
 
 ALL_METRICS = dict()
@@ -279,33 +299,23 @@ def precision_recall_curve_padded_thresholds(*args, **kwargs):
     "label_ranking_loss",
     "label_ranking_average_precision_score",
     "dcg_score",
-    "ndcg_score"
+    "ndcg_score",
 }
 
 # Those metrics don't support multiclass inputs
 METRIC_UNDEFINED_MULTICLASS = {
-    "brier_score_loss",
-
     "micro_roc_auc",
     "samples_roc_auc",
     "partial_roc_auc",
     "roc_auc_score",
     "weighted_roc_auc",
-
-    "average_precision_score",
-    "weighted_average_precision_score",
-    "micro_average_precision_score",
-    "samples_average_precision_score",
-
     "jaccard_score",
-
     # with default average='binary', multiclass is prohibited
     "precision_score",
     "recall_score",
     "f1_score",
     "f2_score",
     "f0.5_score",
-
     # curves
     "roc_curve",
     "precision_recall_curve",
@@ -314,17 +324,24 @@ def precision_recall_curve_padded_thresholds(*args, **kwargs):
 
 # Metric undefined with "binary" or "multiclass" input
 METRIC_UNDEFINED_BINARY_MULTICLASS = METRIC_UNDEFINED_BINARY.union(
-    METRIC_UNDEFINED_MULTICLASS)
+    METRIC_UNDEFINED_MULTICLASS
+)
 
 # Metrics with an "average" argument
 METRICS_WITH_AVERAGING = {
-    "precision_score", "recall_score", "f1_score", "f2_score", "f0.5_score",
-    "jaccard_score"
+    "precision_score",
+    "recall_score",
+    "f1_score",
+    "f2_score",
+    "f0.5_score",
+    "jaccard_score",
 }
 
 # Threshold-based metrics with an "average" argument
 THRESHOLDED_METRICS_WITH_AVERAGING = {
-    "roc_auc_score", "average_precision_score", "partial_roc_auc",
+    "roc_auc_score",
+    "average_precision_score",
+    "partial_roc_auc",
 }
 
 # Metrics with a "pos_label" argument
@@ -332,26 +349,17 @@ def precision_recall_curve_padded_thresholds(*args, **kwargs):
     "roc_curve",
     "precision_recall_curve",
     "det_curve",
-
     "brier_score_loss",
-
-    "precision_score", "recall_score", "f1_score", "f2_score", "f0.5_score",
+    "precision_score",
+    "recall_score",
+    "f1_score",
+    "f2_score",
+    "f0.5_score",
     "jaccard_score",
-
     "average_precision_score",
     "weighted_average_precision_score",
     "micro_average_precision_score",
     "samples_average_precision_score",
-
-    # pos_label support deprecated; to be removed in 0.18:
-    "weighted_f0.5_score", "weighted_f1_score", "weighted_f2_score",
-    "weighted_precision_score", "weighted_recall_score",
-
-    "micro_f0.5_score", "micro_f1_score", "micro_f2_score",
-    "micro_precision_score", "micro_recall_score",
-
-    "macro_f0.5_score", "macro_f1_score", "macro_f2_score",
-    "macro_precision_score", "macro_recall_score",
 }
 
 # Metrics with a "labels" argument
@@ -363,26 +371,35 @@ def precision_recall_curve_padded_thresholds(*args, **kwargs):
     "roc_curve",
     "precision_recall_curve",
     "det_curve",
-
-    "precision_score", "recall_score", "f1_score", "f2_score", "f0.5_score",
+    "precision_score",
+    "recall_score",
+    "f1_score",
+    "f2_score",
+    "f0.5_score",
     "jaccard_score",
-
-    "weighted_f0.5_score", "weighted_f1_score", "weighted_f2_score",
-    "weighted_precision_score", "weighted_recall_score",
+    "weighted_f0.5_score",
+    "weighted_f1_score",
+    "weighted_f2_score",
+    "weighted_precision_score",
+    "weighted_recall_score",
     "weighted_jaccard_score",
-
-    "micro_f0.5_score", "micro_f1_score", "micro_f2_score",
-    "micro_precision_score", "micro_recall_score",
+    "micro_f0.5_score",
+    "micro_f1_score",
+    "micro_f2_score",
+    "micro_precision_score",
+    "micro_recall_score",
     "micro_jaccard_score",
-
-    "macro_f0.5_score", "macro_f1_score", "macro_f2_score",
-    "macro_precision_score", "macro_recall_score",
+    "macro_f0.5_score",
+    "macro_f1_score",
+    "macro_f2_score",
+    "macro_precision_score",
+    "macro_recall_score",
     "macro_jaccard_score",
-
     "unnormalized_multilabel_confusion_matrix",
     "unnormalized_multilabel_confusion_matrix_sample",
-
     "cohen_kappa_score",
+    "log_loss",
+    "brier_score_loss",
 }
 
 # Metrics with a "normalize" option
@@ -396,77 +413,107 @@ def precision_recall_curve_padded_thresholds(*args, **kwargs):
 THRESHOLDED_MULTILABEL_METRICS = {
     "log_loss",
     "unnormalized_log_loss",
-
-    "roc_auc_score", "weighted_roc_auc", "samples_roc_auc",
-    "micro_roc_auc", "partial_roc_auc",
-
-    "average_precision_score", "weighted_average_precision_score",
-    "samples_average_precision_score", "micro_average_precision_score",
-
-    "coverage_error", "label_ranking_loss",
-
+    "brier_score_loss",
+    "roc_auc_score",
+    "weighted_roc_auc",
+    "samples_roc_auc",
+    "micro_roc_auc",
+    "partial_roc_auc",
+    "average_precision_score",
+    "weighted_average_precision_score",
+    "samples_average_precision_score",
+    "micro_average_precision_score",
+    "coverage_error",
+    "label_ranking_loss",
     "ndcg_score",
     "dcg_score",
-
     "label_ranking_average_precision_score",
 }
 
 # Classification metrics with  "multilabel-indicator" format
 MULTILABELS_METRICS = {
-    "accuracy_score", "unnormalized_accuracy_score",
+    "accuracy_score",
+    "unnormalized_accuracy_score",
     "hamming_loss",
-    "zero_one_loss", "unnormalized_zero_one_loss",
-
-    "weighted_f0.5_score", "weighted_f1_score", "weighted_f2_score",
-    "weighted_precision_score", "weighted_recall_score",
+    "zero_one_loss",
+    "unnormalized_zero_one_loss",
+    "weighted_f0.5_score",
+    "weighted_f1_score",
+    "weighted_f2_score",
+    "weighted_precision_score",
+    "weighted_recall_score",
     "weighted_jaccard_score",
-
-    "macro_f0.5_score", "macro_f1_score", "macro_f2_score",
-    "macro_precision_score", "macro_recall_score",
+    "macro_f0.5_score",
+    "macro_f1_score",
+    "macro_f2_score",
+    "macro_precision_score",
+    "macro_recall_score",
     "macro_jaccard_score",
-
-    "micro_f0.5_score", "micro_f1_score", "micro_f2_score",
-    "micro_precision_score", "micro_recall_score",
+    "micro_f0.5_score",
+    "micro_f1_score",
+    "micro_f2_score",
+    "micro_precision_score",
+    "micro_recall_score",
     "micro_jaccard_score",
-
     "unnormalized_multilabel_confusion_matrix",
-
-    "samples_f0.5_score", "samples_f1_score", "samples_f2_score",
-    "samples_precision_score", "samples_recall_score",
+    "samples_f0.5_score",
+    "samples_f1_score",
+    "samples_f2_score",
+    "samples_precision_score",
+    "samples_recall_score",
     "samples_jaccard_score",
 }
 
 # Regression metrics with "multioutput-continuous" format support
 MULTIOUTPUT_METRICS = {
-    "mean_absolute_error", "median_absolute_error", "mean_squared_error",
-    "r2_score", "explained_variance_score", "mean_absolute_percentage_error",
-    "mean_pinball_loss"
+    "mean_absolute_error",
+    "median_absolute_error",
+    "mean_squared_error",
+    "mean_squared_log_error",
+    "r2_score",
+    "root_mean_squared_error",
+    "root_mean_squared_log_error",
+    "explained_variance_score",
+    "mean_absolute_percentage_error",
+    "mean_pinball_loss",
+    "d2_pinball_score",
+    "d2_absolute_error_score",
 }
 
 # Symmetric with respect to their input arguments y_true and y_pred
 # metric(y_true, y_pred) == metric(y_pred, y_true).
 SYMMETRIC_METRICS = {
-    "accuracy_score", "unnormalized_accuracy_score",
+    "accuracy_score",
+    "unnormalized_accuracy_score",
     "hamming_loss",
-    "zero_one_loss", "unnormalized_zero_one_loss",
-
-    "micro_jaccard_score", "macro_jaccard_score",
+    "zero_one_loss",
+    "unnormalized_zero_one_loss",
+    "micro_jaccard_score",
+    "macro_jaccard_score",
     "jaccard_score",
     "samples_jaccard_score",
-
-    "f1_score", "micro_f1_score", "macro_f1_score",
+    "f1_score",
+    "micro_f1_score",
+    "macro_f1_score",
     "weighted_recall_score",
+    "mean_squared_log_error",
+    "root_mean_squared_error",
+    "root_mean_squared_log_error",
     # P = R = F = accuracy in multiclass case
-    "micro_f0.5_score", "micro_f1_score", "micro_f2_score",
-    "micro_precision_score", "micro_recall_score",
-
-    "matthews_corrcoef_score", "mean_absolute_error", "mean_squared_error",
-    "median_absolute_error", "max_error",
-
+    "micro_f0.5_score",
+    "micro_f1_score",
+    "micro_f2_score",
+    "micro_precision_score",
+    "micro_recall_score",
+    "matthews_corrcoef_score",
+    "mean_absolute_error",
+    "mean_squared_error",
+    "median_absolute_error",
+    "max_error",
     # Pinball loss is only symmetric for alpha=0.5 which is the default.
     "mean_pinball_loss",
-
-    "cohen_kappa_score", "mean_normal_deviance"
+    "cohen_kappa_score",
+    "mean_normal_deviance",
 }
 
 # Asymmetric with respect to their input arguments y_true and y_pred
@@ -481,17 +528,28 @@ def precision_recall_curve_padded_thresholds(*args, **kwargs):
     "roc_curve",
     "precision_recall_curve",
     "det_curve",
-
-    "precision_score", "recall_score", "f2_score", "f0.5_score",
-
-    "weighted_f0.5_score", "weighted_f1_score", "weighted_f2_score",
-    "weighted_precision_score", "weighted_jaccard_score",
+    "precision_score",
+    "recall_score",
+    "f2_score",
+    "f0.5_score",
+    "weighted_f0.5_score",
+    "weighted_f1_score",
+    "weighted_f2_score",
+    "weighted_precision_score",
+    "weighted_jaccard_score",
     "unnormalized_multilabel_confusion_matrix",
-
-    "macro_f0.5_score", "macro_f2_score", "macro_precision_score",
-    "macro_recall_score", "log_loss", "hinge_loss",
-    "mean_gamma_deviance", "mean_poisson_deviance",
-    "mean_compound_poisson_deviance", "mean_absolute_percentage_error"
+    "macro_f0.5_score",
+    "macro_f2_score",
+    "macro_precision_score",
+    "macro_recall_score",
+    "hinge_loss",
+    "mean_gamma_deviance",
+    "mean_poisson_deviance",
+    "mean_compound_poisson_deviance",
+    "d2_tweedie_score",
+    "d2_pinball_score",
+    "d2_absolute_error_score",
+    "mean_absolute_percentage_error",
 }
 
 
@@ -500,13 +558,20 @@ def precision_recall_curve_padded_thresholds(*args, **kwargs):
     "median_absolute_error",
     "max_error",
     "ovo_roc_auc",
-    "weighted_ovo_roc_auc"
+    "weighted_ovo_roc_auc",
 }
 
 METRICS_REQUIRE_POSITIVE_Y = {
     "mean_poisson_deviance",
     "mean_gamma_deviance",
     "mean_compound_poisson_deviance",
+    "d2_tweedie_score",
+}
+
+# Metrics involving y = log(1+x)
+METRICS_WITH_LOG1P_Y = {
+    "mean_squared_log_error",
+    "root_mean_squared_log_error",
 }
 
 
@@ -518,130 +583,186 @@ def _require_positive_targets(y1, y2):
     return y1, y2
 
 
-def test_symmetry_consistency():
+def _require_log1p_targets(y1, y2):
+    """Make targets strictly larger than -1"""
+    offset = abs(min(y1.min(), y2.min())) - 0.99
+    y1 = y1.astype(np.float64)
+    y2 = y2.astype(np.float64)
+    y1 += offset
+    y2 += offset
+    return y1, y2
 
-    # We shouldn't forget any metrics
-    assert (SYMMETRIC_METRICS.union(
-        NOT_SYMMETRIC_METRICS, set(THRESHOLDED_METRICS),
-        METRIC_UNDEFINED_BINARY_MULTICLASS) ==
-        set(ALL_METRICS))
 
+def test_symmetry_consistency():
+    # We shouldn't forget any metrics
     assert (
-        SYMMETRIC_METRICS.intersection(NOT_SYMMETRIC_METRICS) ==
-        set())
+        SYMMETRIC_METRICS
+        | NOT_SYMMETRIC_METRICS
+        | set(THRESHOLDED_METRICS)
+        | METRIC_UNDEFINED_BINARY_MULTICLASS
+    ) == set(ALL_METRICS)
+
+    assert (SYMMETRIC_METRICS & NOT_SYMMETRIC_METRICS) == set()
 
 
 @pytest.mark.parametrize("name", sorted(SYMMETRIC_METRICS))
 def test_symmetric_metric(name):
     # Test the symmetry of score and loss functions
     random_state = check_random_state(0)
-    y_true = random_state.randint(0, 2, size=(20, ))
-    y_pred = random_state.randint(0, 2, size=(20, ))
+    y_true = random_state.randint(0, 2, size=(20,))
+    y_pred = random_state.randint(0, 2, size=(20,))
 
     if name in METRICS_REQUIRE_POSITIVE_Y:
         y_true, y_pred = _require_positive_targets(y_true, y_pred)
 
+    elif name in METRICS_WITH_LOG1P_Y:
+        y_true, y_pred = _require_log1p_targets(y_true, y_pred)
+
     y_true_bin = random_state.randint(0, 2, size=(20, 25))
     y_pred_bin = random_state.randint(0, 2, size=(20, 25))
 
     metric = ALL_METRICS[name]
     if name in METRIC_UNDEFINED_BINARY:
         if name in MULTILABELS_METRICS:
-            assert_allclose(metric(y_true_bin, y_pred_bin),
-                            metric(y_pred_bin, y_true_bin),
-                            err_msg="%s is not symmetric" % name)
+            assert_allclose(
+                metric(y_true_bin, y_pred_bin),
+                metric(y_pred_bin, y_true_bin),
+                err_msg="%s is not symmetric" % name,
+            )
         else:
             assert False, "This case is currently unhandled"
     else:
-        assert_allclose(metric(y_true, y_pred),
-                        metric(y_pred, y_true),
-                        err_msg="%s is not symmetric" % name)
+        assert_allclose(
+            metric(y_true, y_pred),
+            metric(y_pred, y_true),
+            err_msg="%s is not symmetric" % name,
+        )
 
 
 @pytest.mark.parametrize("name", sorted(NOT_SYMMETRIC_METRICS))
 def test_not_symmetric_metric(name):
     # Test the symmetry of score and loss functions
     random_state = check_random_state(0)
-    y_true = random_state.randint(0, 2, size=(20, ))
-    y_pred = random_state.randint(0, 2, size=(20, ))
-
-    if name in METRICS_REQUIRE_POSITIVE_Y:
-        y_true, y_pred = _require_positive_targets(y_true, y_pred)
-
     metric = ALL_METRICS[name]
 
-    # use context manager to supply custom error message
-    with pytest.raises(AssertionError):
-        assert_array_equal(metric(y_true, y_pred), metric(y_pred, y_true))
-        raise ValueError("%s seems to be symmetric" % name)
+    # The metric can be accidentally symmetric on a random draw.
+    # We run several random draws to check that at least of them
+    # gives an asymmetric result.
+    always_symmetric = True
+    for _ in range(5):
+        y_true = random_state.randint(0, 2, size=(20,))
+        y_pred = random_state.randint(0, 2, size=(20,))
+
+        if name in METRICS_REQUIRE_POSITIVE_Y:
+            y_true, y_pred = _require_positive_targets(y_true, y_pred)
+
+        nominal = metric(y_true, y_pred)
+        swapped = metric(y_pred, y_true)
+        if not np.allclose(nominal, swapped):
+            always_symmetric = False
+            break
+
+    if always_symmetric:
+        raise ValueError(f"{name} seems to be symmetric")
+
+
+def test_symmetry_tests():
+    # check test_symmetric_metric and test_not_symmetric_metric
+    sym = "accuracy_score"
+    not_sym = "recall_score"
+    # test_symmetric_metric passes on a symmetric metric
+    # but fails on a not symmetric metric
+    test_symmetric_metric(sym)
+    with pytest.raises(AssertionError, match=f"{not_sym} is not symmetric"):
+        test_symmetric_metric(not_sym)
+    # test_not_symmetric_metric passes on a not symmetric metric
+    # but fails on a symmetric metric
+    test_not_symmetric_metric(not_sym)
+    with pytest.raises(ValueError, match=f"{sym} seems to be symmetric"):
+        test_not_symmetric_metric(sym)
 
 
 @pytest.mark.parametrize(
-        'name',
-        sorted(set(ALL_METRICS) - METRIC_UNDEFINED_BINARY_MULTICLASS))
+    "name", sorted(set(ALL_METRICS) - METRIC_UNDEFINED_BINARY_MULTICLASS)
+)
 def test_sample_order_invariance(name):
     random_state = check_random_state(0)
-    y_true = random_state.randint(0, 2, size=(20, ))
-    y_pred = random_state.randint(0, 2, size=(20, ))
+    y_true = random_state.randint(0, 2, size=(20,))
+    y_pred = random_state.randint(0, 2, size=(20,))
 
     if name in METRICS_REQUIRE_POSITIVE_Y:
         y_true, y_pred = _require_positive_targets(y_true, y_pred)
+    elif name in METRICS_WITH_LOG1P_Y:
+        y_true, y_pred = _require_log1p_targets(y_true, y_pred)
 
     y_true_shuffle, y_pred_shuffle = shuffle(y_true, y_pred, random_state=0)
 
     with ignore_warnings():
         metric = ALL_METRICS[name]
-        assert_allclose(metric(y_true, y_pred),
-                        metric(y_true_shuffle, y_pred_shuffle),
-                        err_msg="%s is not sample order invariant" % name)
+        assert_allclose(
+            metric(y_true, y_pred),
+            metric(y_true_shuffle, y_pred_shuffle),
+            err_msg="%s is not sample order invariant" % name,
+        )
 
 
-@ignore_warnings
 def test_sample_order_invariance_multilabel_and_multioutput():
     random_state = check_random_state(0)
 
     # Generate some data
     y_true = random_state.randint(0, 2, size=(20, 25))
     y_pred = random_state.randint(0, 2, size=(20, 25))
-    y_score = random_state.normal(size=y_true.shape)
+    y_score = random_state.uniform(size=y_true.shape)
 
-    y_true_shuffle, y_pred_shuffle, y_score_shuffle = shuffle(y_true,
-                                                              y_pred,
-                                                              y_score,
-                                                              random_state=0)
+    # Some metrics (e.g. log_loss) require y_score to be probabilities (sum to 1)
+    y_score /= y_score.sum(axis=1, keepdims=True)
+
+    y_true_shuffle, y_pred_shuffle, y_score_shuffle = shuffle(
+        y_true, y_pred, y_score, random_state=0
+    )
 
     for name in MULTILABELS_METRICS:
         metric = ALL_METRICS[name]
-        assert_allclose(metric(y_true, y_pred),
-                        metric(y_true_shuffle, y_pred_shuffle),
-                        err_msg="%s is not sample order invariant" % name)
+        assert_allclose(
+            metric(y_true, y_pred),
+            metric(y_true_shuffle, y_pred_shuffle),
+            err_msg="%s is not sample order invariant" % name,
+        )
 
     for name in THRESHOLDED_MULTILABEL_METRICS:
         metric = ALL_METRICS[name]
-        assert_allclose(metric(y_true, y_score),
-                        metric(y_true_shuffle, y_score_shuffle),
-                        err_msg="%s is not sample order invariant" % name)
+        assert_allclose(
+            metric(y_true, y_score),
+            metric(y_true_shuffle, y_score_shuffle),
+            err_msg="%s is not sample order invariant" % name,
+        )
 
     for name in MULTIOUTPUT_METRICS:
         metric = ALL_METRICS[name]
-        assert_allclose(metric(y_true, y_score),
-                        metric(y_true_shuffle, y_score_shuffle),
-                        err_msg="%s is not sample order invariant" % name)
-        assert_allclose(metric(y_true, y_pred),
-                        metric(y_true_shuffle, y_pred_shuffle),
-                        err_msg="%s is not sample order invariant" % name)
+        assert_allclose(
+            metric(y_true, y_score),
+            metric(y_true_shuffle, y_score_shuffle),
+            err_msg="%s is not sample order invariant" % name,
+        )
+        assert_allclose(
+            metric(y_true, y_pred),
+            metric(y_true_shuffle, y_pred_shuffle),
+            err_msg="%s is not sample order invariant" % name,
+        )
 
 
 @pytest.mark.parametrize(
-        'name',
-        sorted(set(ALL_METRICS) - METRIC_UNDEFINED_BINARY_MULTICLASS))
+    "name", sorted(set(ALL_METRICS) - METRIC_UNDEFINED_BINARY_MULTICLASS)
+)
 def test_format_invariance_with_1d_vectors(name):
     random_state = check_random_state(0)
-    y1 = random_state.randint(0, 2, size=(20, ))
-    y2 = random_state.randint(0, 2, size=(20, ))
+    y1 = random_state.randint(0, 2, size=(20,))
+    y2 = random_state.randint(0, 2, size=(20,))
 
     if name in METRICS_REQUIRE_POSITIVE_Y:
         y1, y2 = _require_positive_targets(y1, y2)
+    elif name in METRICS_WITH_LOG1P_Y:
+        y1, y2 = _require_log1p_targets(y1, y2)
 
     y1_list = list(y1)
     y2_list = list(y2)
@@ -659,42 +780,76 @@ def test_format_invariance_with_1d_vectors(name):
 
         measure = metric(y1, y2)
 
-        assert_allclose(metric(y1_list, y2_list), measure,
-                        err_msg="%s is not representation invariant with list"
-                                "" % name)
+        assert_allclose(
+            metric(y1_list, y2_list),
+            measure,
+            err_msg="%s is not representation invariant with list" % name,
+        )
 
-        assert_allclose(metric(y1_1d, y2_1d), measure,
-                        err_msg="%s is not representation invariant with "
-                                "np-array-1d" % name)
+        assert_allclose(
+            metric(y1_1d, y2_1d),
+            measure,
+            err_msg="%s is not representation invariant with np-array-1d" % name,
+        )
 
-        assert_allclose(metric(y1_column, y2_column), measure,
-                        err_msg="%s is not representation invariant with "
-                                "np-array-column" % name)
+        assert_allclose(
+            metric(y1_column, y2_column),
+            measure,
+            err_msg="%s is not representation invariant with np-array-column" % name,
+        )
 
         # Mix format support
-        assert_allclose(metric(y1_1d, y2_list), measure,
-                        err_msg="%s is not representation invariant with mix "
-                                "np-array-1d and list" % name)
-
-        assert_allclose(metric(y1_list, y2_1d), measure,
-                        err_msg="%s is not representation invariant with mix "
-                                "np-array-1d and list" % name)
-
-        assert_allclose(metric(y1_1d, y2_column), measure,
-                        err_msg="%s is not representation invariant with mix "
-                                "np-array-1d and np-array-column" % name)
-
-        assert_allclose(metric(y1_column, y2_1d), measure,
-                        err_msg="%s is not representation invariant with mix "
-                                "np-array-1d and np-array-column" % name)
-
-        assert_allclose(metric(y1_list, y2_column), measure,
-                        err_msg="%s is not representation invariant with mix "
-                                "list and np-array-column" % name)
-
-        assert_allclose(metric(y1_column, y2_list), measure,
-                        err_msg="%s is not representation invariant with mix "
-                                "list and np-array-column" % name)
+        assert_allclose(
+            metric(y1_1d, y2_list),
+            measure,
+            err_msg="%s is not representation invariant with mix np-array-1d and list"
+            % name,
+        )
+
+        assert_allclose(
+            metric(y1_list, y2_1d),
+            measure,
+            err_msg="%s is not representation invariant with mix np-array-1d and list"
+            % name,
+        )
+
+        assert_allclose(
+            metric(y1_1d, y2_column),
+            measure,
+            err_msg=(
+                "%s is not representation invariant with mix "
+                "np-array-1d and np-array-column"
+            )
+            % name,
+        )
+
+        assert_allclose(
+            metric(y1_column, y2_1d),
+            measure,
+            err_msg=(
+                "%s is not representation invariant with mix "
+                "np-array-1d and np-array-column"
+            )
+            % name,
+        )
+
+        assert_allclose(
+            metric(y1_list, y2_column),
+            measure,
+            err_msg=(
+                "%s is not representation invariant with mix list and np-array-column"
+            )
+            % name,
+        )
+
+        assert_allclose(
+            metric(y1_column, y2_list),
+            measure,
+            err_msg=(
+                "%s is not representation invariant with mix list and np-array-column"
+            )
+            % name,
+        )
 
         # These mix representations aren't allowed
         with pytest.raises(ValueError):
@@ -712,20 +867,27 @@ def test_format_invariance_with_1d_vectors(name):
 
         # NB: We do not test for y1_row, y2_row as these may be
         # interpreted as multilabel or multioutput data.
-        if (name not in (MULTIOUTPUT_METRICS | THRESHOLDED_MULTILABEL_METRICS |
-                         MULTILABELS_METRICS)):
-            with pytest.raises(ValueError):
-                metric(y1_row, y2_row)
+        if name not in (
+            MULTIOUTPUT_METRICS | THRESHOLDED_MULTILABEL_METRICS | MULTILABELS_METRICS
+        ):
+            if "roc_auc" in name:
+                # for consistency between the `roc_cuve` and `roc_auc_score`
+                # np.nan is returned and an `UndefinedMetricWarning` is raised
+                with pytest.warns(UndefinedMetricWarning):
+                    assert math.isnan(metric(y1_row, y2_row))
+            else:
+                with pytest.raises(ValueError):
+                    metric(y1_row, y2_row)
 
 
 @pytest.mark.parametrize(
-    'name',
-    sorted(set(CLASSIFICATION_METRICS) - METRIC_UNDEFINED_BINARY_MULTICLASS))
+    "name", sorted(set(CLASSIFICATION_METRICS) - METRIC_UNDEFINED_BINARY_MULTICLASS)
+)
 def test_classification_invariance_string_vs_numbers_labels(name):
     # Ensure that classification metrics with string labels are invariant
     random_state = check_random_state(0)
-    y1 = random_state.randint(0, 2, size=(20, ))
-    y2 = random_state.randint(0, 2, size=(20, ))
+    y1 = random_state.randint(0, 2, size=(20,))
+    y2 = random_state.randint(0, 2, size=(20,))
 
     y1_str = np.array(["eggs", "spam"])[y1]
     y2_str = np.array(["eggs", "spam"])[y2]
@@ -744,36 +906,42 @@ def test_classification_invariance_string_vs_numbers_labels(name):
 
         measure_with_str = metric_str(y1_str, y2_str)
 
-        assert_array_equal(measure_with_number, measure_with_str,
-                           err_msg="{0} failed string vs number invariance "
-                                   "test".format(name))
+        assert_array_equal(
+            measure_with_number,
+            measure_with_str,
+            err_msg="{0} failed string vs number invariance test".format(name),
+        )
 
-        measure_with_strobj = metric_str(y1_str.astype('O'),
-                                         y2_str.astype('O'))
-        assert_array_equal(measure_with_number, measure_with_strobj,
-                           err_msg="{0} failed string object vs number "
-                                   "invariance test".format(name))
+        measure_with_strobj = metric_str(y1_str.astype("O"), y2_str.astype("O"))
+        assert_array_equal(
+            measure_with_number,
+            measure_with_strobj,
+            err_msg="{0} failed string object vs number invariance test".format(name),
+        )
 
         if name in METRICS_WITH_LABELS:
             metric_str = partial(metric_str, labels=labels_str)
             measure_with_str = metric_str(y1_str, y2_str)
-            assert_array_equal(measure_with_number, measure_with_str,
-                               err_msg="{0} failed string vs number  "
-                                       "invariance test".format(name))
+            assert_array_equal(
+                measure_with_number,
+                measure_with_str,
+                err_msg="{0} failed string vs number  invariance test".format(name),
+            )
 
-            measure_with_strobj = metric_str(y1_str.astype('O'),
-                                             y2_str.astype('O'))
-            assert_array_equal(measure_with_number, measure_with_strobj,
-                               err_msg="{0} failed string vs number  "
-                                       "invariance test".format(name))
+            measure_with_strobj = metric_str(y1_str.astype("O"), y2_str.astype("O"))
+            assert_array_equal(
+                measure_with_number,
+                measure_with_strobj,
+                err_msg="{0} failed string vs number  invariance test".format(name),
+            )
 
 
-@pytest.mark.parametrize('name', THRESHOLDED_METRICS)
+@pytest.mark.parametrize("name", THRESHOLDED_METRICS)
 def test_thresholded_invariance_string_vs_numbers_labels(name):
     # Ensure that thresholded metrics with string labels are invariant
     random_state = check_random_state(0)
-    y1 = random_state.randint(0, 2, size=(20, ))
-    y2 = random_state.randint(0, 2, size=(20, ))
+    y1 = random_state.randint(0, 2, size=(20,))
+    y2 = random_state.randint(0, 2, size=(20,))
 
     y1_str = np.array(["eggs", "spam"])[y1]
 
@@ -789,20 +957,26 @@ def test_thresholded_invariance_string_vs_numbers_labels(name):
 
             measure_with_number = metric(y1, y2)
             measure_with_str = metric_str(y1_str, y2)
-            assert_array_equal(measure_with_number, measure_with_str,
-                               err_msg="{0} failed string vs number "
-                                       "invariance test".format(name))
-
-            measure_with_strobj = metric_str(y1_str.astype('O'), y2)
-            assert_array_equal(measure_with_number, measure_with_strobj,
-                               err_msg="{0} failed string object vs number "
-                                       "invariance test".format(name))
+            assert_array_equal(
+                measure_with_number,
+                measure_with_str,
+                err_msg="{0} failed string vs number invariance test".format(name),
+            )
+
+            measure_with_strobj = metric_str(y1_str.astype("O"), y2)
+            assert_array_equal(
+                measure_with_number,
+                measure_with_strobj,
+                err_msg="{0} failed string object vs number invariance test".format(
+                    name
+                ),
+            )
         else:
             # TODO those metrics doesn't support string label yet
             with pytest.raises(ValueError):
                 metric(y1_str, y2)
             with pytest.raises(ValueError):
-                metric(y1_str.astype('O'), y2)
+                metric(y1_str.astype("O"), y2)
 
 
 invalids_nan_inf = [
@@ -815,46 +989,65 @@ def test_thresholded_invariance_string_vs_numbers_labels(name):
 
 
 @pytest.mark.parametrize(
-    'metric',
-    chain(THRESHOLDED_METRICS.values(), REGRESSION_METRICS.values())
+    "metric", chain(THRESHOLDED_METRICS.values(), REGRESSION_METRICS.values())
 )
 @pytest.mark.parametrize("y_true, y_score", invalids_nan_inf)
 def test_regression_thresholded_inf_nan_input(metric, y_true, y_score):
-    with pytest.raises(ValueError, match="contains NaN, infinity"):
+    # Reshape since coverage_error only accepts 2D arrays.
+    if metric == coverage_error:
+        y_true = [y_true]
+        y_score = [y_score]
+    with pytest.raises(ValueError, match=r"contains (NaN|infinity)"):
         metric(y_true, y_score)
 
 
-@pytest.mark.parametrize('metric', CLASSIFICATION_METRICS.values())
+@pytest.mark.parametrize("metric", CLASSIFICATION_METRICS.values())
 @pytest.mark.parametrize(
     "y_true, y_score",
-    invalids_nan_inf +
+    invalids_nan_inf
+    +
     # Add an additional case for classification only
     # non-regression test for:
     # https://github.com/scikit-learn/scikit-learn/issues/6809
-    [([np.nan, 1, 2], [1, 2, 3])]
+    [
+        ([np.nan, 1, 2], [1, 2, 3]),
+        ([np.inf, 1, 2], [1, 2, 3]),
+    ],
 )
 def test_classification_inf_nan_input(metric, y_true, y_score):
     """check that classification metrics raise a message mentioning the
     occurrence of non-finite values in the target vectors."""
-    err_msg = "Input contains NaN, infinity or a value too large"
+    if not np.isfinite(y_true).all():
+        input_name = "y_true"
+        if np.isnan(y_true).any():
+            unexpected_value = "NaN"
+        else:
+            unexpected_value = "infinity or a value too large"
+    else:
+        input_name = "y_pred"
+        if np.isnan(y_score).any():
+            unexpected_value = "NaN"
+        else:
+            unexpected_value = "infinity or a value too large"
+
+    err_msg = f"Input {input_name} contains {unexpected_value}"
+
     with pytest.raises(ValueError, match=err_msg):
         metric(y_true, y_score)
 
 
-@pytest.mark.parametrize('metric', CLASSIFICATION_METRICS.values())
+@pytest.mark.parametrize("metric", CLASSIFICATION_METRICS.values())
 def test_classification_binary_continuous_input(metric):
     """check that classification metrics raise a message of mixed type data
     with continuous/binary target vectors."""
-    y_true, y_score = ['a', 'b', 'a'], [0.1, 0.2, 0.3]
+    y_true, y_score = ["a", "b", "a"], [0.1, 0.2, 0.3]
     err_msg = (
-        "Classification metrics can't handle a mix of binary and continuous "
-        "targets"
+        "Classification metrics can't handle a mix of binary and continuous targets"
     )
     with pytest.raises(ValueError, match=err_msg):
         metric(y_true, y_score)
 
 
-@ignore_warnings
 def check_single_sample(name):
     # Non-regression test: scores should work with a single sample.
     # This is important for leave-one-out cross validation.
@@ -865,37 +1058,44 @@ def check_single_sample(name):
     # assert that no exception is thrown
     if name in METRICS_REQUIRE_POSITIVE_Y:
         values = [1, 2]
+    elif name in METRICS_WITH_LOG1P_Y:
+        values = [-0.7, 1]
     else:
         values = [0, 1]
     for i, j in product(values, repeat=2):
         metric([i], [j])
 
 
-@ignore_warnings
 def check_single_sample_multioutput(name):
     metric = ALL_METRICS[name]
     for i, j, k, l in product([0, 1], repeat=4):
         metric(np.array([[i, j]]), np.array([[k, l]]))
 
 
+# filter many metric specific warnings
+@pytest.mark.filterwarnings("ignore")
 @pytest.mark.parametrize(
-    'name',
+    "name",
     sorted(
         set(ALL_METRICS)
         # Those metrics are not always defined with one sample
         # or in multiclass classification
-        - METRIC_UNDEFINED_BINARY_MULTICLASS - set(THRESHOLDED_METRICS)))
+        - METRIC_UNDEFINED_BINARY_MULTICLASS
+        - set(THRESHOLDED_METRICS)
+    ),
+)
 def test_single_sample(name):
     check_single_sample(name)
 
 
-@pytest.mark.parametrize('name',
-                         sorted(MULTIOUTPUT_METRICS | MULTILABELS_METRICS))
+# filter many metric specific warnings
+@pytest.mark.filterwarnings("ignore")
+@pytest.mark.parametrize("name", sorted(MULTIOUTPUT_METRICS | MULTILABELS_METRICS))
 def test_single_sample_multioutput(name):
     check_single_sample_multioutput(name)
 
 
-@pytest.mark.parametrize('name', sorted(MULTIOUTPUT_METRICS))
+@pytest.mark.parametrize("name", sorted(MULTIOUTPUT_METRICS))
 def test_multioutput_number_of_output_differ(name):
     y_true = np.array([[1, 0, 0, 1], [0, 1, 1, 1], [1, 1, 0, 1]])
     y_pred = np.array([[0, 0], [1, 0], [0, 0]])
@@ -905,7 +1105,7 @@ def test_multioutput_number_of_output_differ(name):
         metric(y_true, y_pred)
 
 
-@pytest.mark.parametrize('name', sorted(MULTIOUTPUT_METRICS))
+@pytest.mark.parametrize("name", sorted(MULTIOUTPUT_METRICS))
 def test_multioutput_regression_invariance_to_dimension_shuffling(name):
     # test invariance to dimension shuffling
     random_state = check_random_state(0)
@@ -917,31 +1117,41 @@ def test_multioutput_regression_invariance_to_dimension_shuffling(name):
 
     for _ in range(3):
         perm = random_state.permutation(y_true.shape[1])
-        assert_allclose(metric(y_true[:, perm], y_pred[:, perm]),
-                        error,
-                        err_msg="%s is not dimension shuffling invariant" % (
-                            name))
+        assert_allclose(
+            metric(y_true[:, perm], y_pred[:, perm]),
+            error,
+            err_msg="%s is not dimension shuffling invariant" % (name),
+        )
 
 
-@ignore_warnings
-def test_multilabel_representation_invariance():
+@pytest.mark.filterwarnings("ignore::sklearn.exceptions.UndefinedMetricWarning")
+@pytest.mark.parametrize("coo_container", COO_CONTAINERS)
+def test_multilabel_representation_invariance(coo_container):
     # Generate some data
     n_classes = 4
     n_samples = 50
 
-    _, y1 = make_multilabel_classification(n_features=1, n_classes=n_classes,
-                                           random_state=0, n_samples=n_samples,
-                                           allow_unlabeled=True)
-    _, y2 = make_multilabel_classification(n_features=1, n_classes=n_classes,
-                                           random_state=1, n_samples=n_samples,
-                                           allow_unlabeled=True)
+    _, y1 = make_multilabel_classification(
+        n_features=1,
+        n_classes=n_classes,
+        random_state=0,
+        n_samples=n_samples,
+        allow_unlabeled=True,
+    )
+    _, y2 = make_multilabel_classification(
+        n_features=1,
+        n_classes=n_classes,
+        random_state=1,
+        n_samples=n_samples,
+        allow_unlabeled=True,
+    )
 
     # To make sure at least one empty label is present
     y1 = np.vstack([y1, [[0] * n_classes]])
     y2 = np.vstack([y2, [[0] * n_classes]])
 
-    y1_sparse_indicator = sp.coo_matrix(y1)
-    y2_sparse_indicator = sp.coo_matrix(y2)
+    y1_sparse_indicator = coo_container(y1)
+    y2_sparse_indicator = coo_container(y2)
 
     y1_list_array_indicator = list(y1)
     y2_list_array_indicator = list(y2)
@@ -954,31 +1164,44 @@ def test_multilabel_representation_invariance():
 
         # XXX cruel hack to work with partial functions
         if isinstance(metric, partial):
-            metric.__module__ = 'tmp'
+            metric.__module__ = "tmp"
             metric.__name__ = name
 
         measure = metric(y1, y2)
 
         # Check representation invariance
-        assert_allclose(metric(y1_sparse_indicator, y2_sparse_indicator),
-                        measure,
-                        err_msg="%s failed representation invariance between "
-                                "dense and sparse indicator formats." % name)
-        assert_almost_equal(metric(y1_list_list_indicator,
-                                   y2_list_list_indicator),
-                            measure,
-                            err_msg="%s failed representation invariance  "
-                                    "between dense array and list of list "
-                                    "indicator formats." % name)
-        assert_almost_equal(metric(y1_list_array_indicator,
-                                   y2_list_array_indicator),
-                            measure,
-                            err_msg="%s failed representation invariance  "
-                                    "between dense and list of array "
-                                    "indicator formats." % name)
-
-
-@pytest.mark.parametrize('name', sorted(MULTILABELS_METRICS))
+        assert_allclose(
+            metric(y1_sparse_indicator, y2_sparse_indicator),
+            measure,
+            err_msg=(
+                "%s failed representation invariance between "
+                "dense and sparse indicator formats."
+            )
+            % name,
+        )
+        assert_almost_equal(
+            metric(y1_list_list_indicator, y2_list_list_indicator),
+            measure,
+            err_msg=(
+                "%s failed representation invariance  "
+                "between dense array and list of list "
+                "indicator formats."
+            )
+            % name,
+        )
+        assert_almost_equal(
+            metric(y1_list_array_indicator, y2_list_array_indicator),
+            measure,
+            err_msg=(
+                "%s failed representation invariance  "
+                "between dense and list of array "
+                "indicator formats."
+            )
+            % name,
+        )
+
+
+@pytest.mark.parametrize("name", sorted(MULTILABELS_METRICS))
 def test_raise_value_error_multilabel_sequences(name):
     # make sure the multilabel-sequence format raises ValueError
     multilabel_sequences = [
@@ -986,7 +1209,8 @@ def test_raise_value_error_multilabel_sequences(name):
         [(), (2), (0, 1)],
         [[]],
         [()],
-        np.array([[], [1, 2]], dtype='object')]
+        np.array([[], [1, 2]], dtype="object"),
+    ]
 
     metric = ALL_METRICS[name]
     for seq in multilabel_sequences:
@@ -994,15 +1218,15 @@ def test_raise_value_error_multilabel_sequences(name):
             metric(seq, seq)
 
 
-@pytest.mark.parametrize('name', sorted(METRICS_WITH_NORMALIZE_OPTION))
+@pytest.mark.parametrize("name", sorted(METRICS_WITH_NORMALIZE_OPTION))
 def test_normalize_option_binary_classification(name):
     # Test in the binary case
     n_classes = 2
     n_samples = 20
     random_state = check_random_state(0)
 
-    y_true = random_state.randint(0, n_classes, size=(n_samples, ))
-    y_pred = random_state.randint(0, n_classes, size=(n_samples, ))
+    y_true = random_state.randint(0, n_classes, size=(n_samples,))
+    y_pred = random_state.randint(0, n_classes, size=(n_samples,))
     y_score = random_state.normal(size=y_true.shape)
 
     metrics = ALL_METRICS[name]
@@ -1010,23 +1234,28 @@ def test_normalize_option_binary_classification(name):
     measure_normalized = metrics(y_true, pred, normalize=True)
     measure_not_normalized = metrics(y_true, pred, normalize=False)
 
-    assert_array_less(-1.0 * measure_normalized, 0,
-                      err_msg="We failed to test correctly the normalize "
-                              "option")
+    assert_array_less(
+        -1.0 * measure_normalized,
+        0,
+        err_msg="We failed to test correctly the normalize option",
+    )
 
-    assert_allclose(measure_normalized, measure_not_normalized / n_samples,
-                    err_msg=f"Failed with {name}")
+    assert_allclose(
+        measure_normalized,
+        measure_not_normalized / n_samples,
+        err_msg=f"Failed with {name}",
+    )
 
 
-@pytest.mark.parametrize('name', sorted(METRICS_WITH_NORMALIZE_OPTION))
+@pytest.mark.parametrize("name", sorted(METRICS_WITH_NORMALIZE_OPTION))
 def test_normalize_option_multiclass_classification(name):
     # Test in the multiclass case
     n_classes = 4
     n_samples = 20
     random_state = check_random_state(0)
 
-    y_true = random_state.randint(0, n_classes, size=(n_samples, ))
-    y_pred = random_state.randint(0, n_classes, size=(n_samples, ))
+    y_true = random_state.randint(0, n_classes, size=(n_samples,))
+    y_pred = random_state.randint(0, n_classes, size=(n_samples,))
     y_score = random_state.uniform(size=(n_samples, n_classes))
 
     metrics = ALL_METRICS[name]
@@ -1034,17 +1263,22 @@ def test_normalize_option_multiclass_classification(name):
     measure_normalized = metrics(y_true, pred, normalize=True)
     measure_not_normalized = metrics(y_true, pred, normalize=False)
 
-    assert_array_less(-1.0 * measure_normalized, 0,
-                      err_msg="We failed to test correctly the normalize "
-                              "option")
+    assert_array_less(
+        -1.0 * measure_normalized,
+        0,
+        err_msg="We failed to test correctly the normalize option",
+    )
 
-    assert_allclose(measure_normalized, measure_not_normalized / n_samples,
-                    err_msg=f"Failed with {name}")
+    assert_allclose(
+        measure_normalized,
+        measure_not_normalized / n_samples,
+        err_msg=f"Failed with {name}",
+    )
 
 
-@pytest.mark.parametrize('name', sorted(
-    METRICS_WITH_NORMALIZE_OPTION.intersection(MULTILABELS_METRICS)
-))
+@pytest.mark.parametrize(
+    "name", sorted(METRICS_WITH_NORMALIZE_OPTION.intersection(MULTILABELS_METRICS))
+)
 def test_normalize_option_multilabel_classification(name):
     # Test in the multilabel case
     n_classes = 4
@@ -1053,51 +1287,65 @@ def test_normalize_option_multilabel_classification(name):
 
     # for both random_state 0 and 1, y_true and y_pred has at least one
     # unlabelled entry
-    _, y_true = make_multilabel_classification(n_features=1,
-                                               n_classes=n_classes,
-                                               random_state=0,
-                                               allow_unlabeled=True,
-                                               n_samples=n_samples)
-    _, y_pred = make_multilabel_classification(n_features=1,
-                                               n_classes=n_classes,
-                                               random_state=1,
-                                               allow_unlabeled=True,
-                                               n_samples=n_samples)
+    _, y_true = make_multilabel_classification(
+        n_features=1,
+        n_classes=n_classes,
+        random_state=0,
+        allow_unlabeled=True,
+        n_samples=n_samples,
+    )
+    _, y_pred = make_multilabel_classification(
+        n_features=1,
+        n_classes=n_classes,
+        random_state=1,
+        allow_unlabeled=True,
+        n_samples=n_samples,
+    )
 
     y_score = random_state.uniform(size=y_true.shape)
 
     # To make sure at least one empty label is present
-    y_true += [0]*n_classes
-    y_pred += [0]*n_classes
+    y_true += [0] * n_classes
+    y_pred += [0] * n_classes
 
     metrics = ALL_METRICS[name]
     pred = y_score if name in THRESHOLDED_METRICS else y_pred
     measure_normalized = metrics(y_true, pred, normalize=True)
     measure_not_normalized = metrics(y_true, pred, normalize=False)
 
-    assert_array_less(-1.0 * measure_normalized, 0,
-                      err_msg="We failed to test correctly the normalize "
-                              "option")
+    assert_array_less(
+        -1.0 * measure_normalized,
+        0,
+        err_msg="We failed to test correctly the normalize option",
+    )
 
-    assert_allclose(measure_normalized, measure_not_normalized / n_samples,
-                    err_msg=f"Failed with {name}")
+    assert_allclose(
+        measure_normalized,
+        measure_not_normalized / n_samples,
+        err_msg=f"Failed with {name}",
+    )
 
 
-@ignore_warnings
-def _check_averaging(metric, y_true, y_pred, y_true_binarize, y_pred_binarize,
-                     is_multilabel):
+def _check_averaging(
+    metric, y_true, y_pred, y_true_binarize, y_pred_binarize, is_multilabel
+):
     n_samples, n_classes = y_true_binarize.shape
 
     # No averaging
     label_measure = metric(y_true, y_pred, average=None)
-    assert_allclose(label_measure,
-                    [metric(y_true_binarize[:, i], y_pred_binarize[:, i])
-                     for i in range(n_classes)])
+    assert_allclose(
+        label_measure,
+        [
+            metric(y_true_binarize[:, i], y_pred_binarize[:, i])
+            for i in range(n_classes)
+        ],
+    )
 
     # Micro measure
     micro_measure = metric(y_true, y_pred, average="micro")
-    assert_allclose(micro_measure,
-                    metric(y_true_binarize.ravel(), y_pred_binarize.ravel()))
+    assert_allclose(
+        micro_measure, metric(y_true_binarize.ravel(), y_pred_binarize.ravel())
+    )
 
     # Macro measure
     macro_measure = metric(y_true, y_pred, average="macro")
@@ -1108,8 +1356,7 @@ def _check_averaging(metric, y_true, y_pred, y_true_binarize, y_pred_binarize,
 
     if np.sum(weights) != 0:
         weighted_measure = metric(y_true, y_pred, average="weighted")
-        assert_allclose(weighted_measure,
-                        np.average(label_measure, weights=weights))
+        assert_allclose(weighted_measure, np.average(label_measure, weights=weights))
     else:
         weighted_measure = metric(y_true, y_pred, average="weighted")
         assert_allclose(weighted_measure, 0)
@@ -1117,9 +1364,15 @@ def _check_averaging(metric, y_true, y_pred, y_true_binarize, y_pred_binarize,
     # Sample measure
     if is_multilabel:
         sample_measure = metric(y_true, y_pred, average="samples")
-        assert_allclose(sample_measure,
-                        np.mean([metric(y_true_binarize[i], y_pred_binarize[i])
-                                 for i in range(n_samples)]))
+        assert_allclose(
+            sample_measure,
+            np.mean(
+                [
+                    metric(y_true_binarize[i], y_pred_binarize[i])
+                    for i in range(n_samples)
+                ]
+            ),
+        )
 
     with pytest.raises(ValueError):
         metric(y_true, y_pred, average="unknown")
@@ -1127,57 +1380,60 @@ def _check_averaging(metric, y_true, y_pred, y_true_binarize, y_pred_binarize,
         metric(y_true, y_pred, average="garbage")
 
 
-def check_averaging(name, y_true, y_true_binarize, y_pred, y_pred_binarize,
-                    y_score):
+def check_averaging(name, y_true, y_true_binarize, y_pred, y_pred_binarize, y_score):
     is_multilabel = type_of_target(y_true).startswith("multilabel")
 
     metric = ALL_METRICS[name]
 
     if name in METRICS_WITH_AVERAGING:
-        _check_averaging(metric, y_true, y_pred, y_true_binarize,
-                         y_pred_binarize, is_multilabel)
+        _check_averaging(
+            metric, y_true, y_pred, y_true_binarize, y_pred_binarize, is_multilabel
+        )
     elif name in THRESHOLDED_METRICS_WITH_AVERAGING:
-        _check_averaging(metric, y_true, y_score, y_true_binarize,
-                         y_score, is_multilabel)
+        _check_averaging(
+            metric, y_true, y_score, y_true_binarize, y_score, is_multilabel
+        )
     else:
         raise ValueError("Metric is not recorded as having an average option")
 
 
-@pytest.mark.parametrize('name', sorted(METRICS_WITH_AVERAGING))
+@pytest.mark.parametrize("name", sorted(METRICS_WITH_AVERAGING))
 def test_averaging_multiclass(name):
     n_samples, n_classes = 50, 3
     random_state = check_random_state(0)
-    y_true = random_state.randint(0, n_classes, size=(n_samples, ))
-    y_pred = random_state.randint(0, n_classes, size=(n_samples, ))
+    y_true = random_state.randint(0, n_classes, size=(n_samples,))
+    y_pred = random_state.randint(0, n_classes, size=(n_samples,))
     y_score = random_state.uniform(size=(n_samples, n_classes))
 
     lb = LabelBinarizer().fit(y_true)
     y_true_binarize = lb.transform(y_true)
     y_pred_binarize = lb.transform(y_pred)
 
-    check_averaging(name, y_true, y_true_binarize,
-                    y_pred, y_pred_binarize, y_score)
+    check_averaging(name, y_true, y_true_binarize, y_pred, y_pred_binarize, y_score)
 
 
 @pytest.mark.parametrize(
-    'name',
-    sorted(METRICS_WITH_AVERAGING | THRESHOLDED_METRICS_WITH_AVERAGING))
+    "name", sorted(METRICS_WITH_AVERAGING | THRESHOLDED_METRICS_WITH_AVERAGING)
+)
 def test_averaging_multilabel(name):
     n_samples, n_classes = 40, 5
-    _, y = make_multilabel_classification(n_features=1, n_classes=n_classes,
-                                          random_state=5, n_samples=n_samples,
-                                          allow_unlabeled=False)
+    _, y = make_multilabel_classification(
+        n_features=1,
+        n_classes=n_classes,
+        random_state=5,
+        n_samples=n_samples,
+        allow_unlabeled=False,
+    )
     y_true = y[:20]
     y_pred = y[20:]
     y_score = check_random_state(0).normal(size=(20, n_classes))
     y_true_binarize = y_true
     y_pred_binarize = y_pred
 
-    check_averaging(name, y_true, y_true_binarize,
-                    y_pred, y_pred_binarize, y_score)
+    check_averaging(name, y_true, y_true_binarize, y_pred, y_pred_binarize, y_score)
 
 
-@pytest.mark.parametrize('name', sorted(METRICS_WITH_AVERAGING))
+@pytest.mark.parametrize("name", sorted(METRICS_WITH_AVERAGING))
 def test_averaging_multilabel_all_zeroes(name):
     y_true = np.zeros((20, 3))
     y_pred = np.zeros((20, 3))
@@ -1185,8 +1441,7 @@ def test_averaging_multilabel_all_zeroes(name):
     y_true_binarize = y_true
     y_pred_binarize = y_pred
 
-    check_averaging(name, y_true, y_true_binarize,
-                    y_pred, y_pred_binarize, y_score)
+    check_averaging(name, y_true, y_true_binarize, y_pred, y_pred_binarize, y_score)
 
 
 def test_averaging_binary_multilabel_all_zeroes():
@@ -1195,14 +1450,20 @@ def test_averaging_binary_multilabel_all_zeroes():
     y_true_binarize = y_true
     y_pred_binarize = y_pred
     # Test _average_binary_score for weight.sum() == 0
-    binary_metric = (lambda y_true, y_score, average="macro":
-                     _average_binary_score(
-                         precision_score, y_true, y_score, average))
-    _check_averaging(binary_metric, y_true, y_pred, y_true_binarize,
-                     y_pred_binarize, is_multilabel=True)
+    binary_metric = lambda y_true, y_score, average="macro": _average_binary_score(
+        precision_score, y_true, y_score, average
+    )
+    _check_averaging(
+        binary_metric,
+        y_true,
+        y_pred,
+        y_true_binarize,
+        y_pred_binarize,
+        is_multilabel=True,
+    )
 
 
-@pytest.mark.parametrize('name', sorted(METRICS_WITH_AVERAGING))
+@pytest.mark.parametrize("name", sorted(METRICS_WITH_AVERAGING))
 def test_averaging_multilabel_all_ones(name):
     y_true = np.ones((20, 3))
     y_pred = np.ones((20, 3))
@@ -1210,11 +1471,9 @@ def test_averaging_multilabel_all_ones(name):
     y_true_binarize = y_true
     y_pred_binarize = y_pred
 
-    check_averaging(name, y_true, y_true_binarize,
-                    y_pred, y_pred_binarize, y_score)
+    check_averaging(name, y_true, y_true_binarize, y_pred, y_pred_binarize, y_score)
 
 
-@ignore_warnings
 def check_sample_weight_invariance(name, metric, y1, y2):
     rng = np.random.RandomState(0)
     sample_weight = rng.randint(1, 10, size=len(y1))
@@ -1229,8 +1488,9 @@ def check_sample_weight_invariance(name, metric, y1, y2):
     assert_allclose(
         unweighted_score,
         metric(y1, y2, sample_weight=np.ones(shape=len(y1))),
-        err_msg="For %s sample_weight=None is not equivalent to "
-                "sample_weight=ones" % name)
+        err_msg="For %s sample_weight=None is not equivalent to sample_weight=ones"
+        % name,
+    )
 
     # check that the weighted and unweighted scores are unequal
     weighted_score = metric(y1, y2, sample_weight=sample_weight)
@@ -1238,26 +1498,35 @@ def check_sample_weight_invariance(name, metric, y1, y2):
     # use context manager to supply custom error message
     with pytest.raises(AssertionError):
         assert_allclose(unweighted_score, weighted_score)
-        raise ValueError("Unweighted and weighted scores are unexpectedly "
-                         "almost equal (%s) and (%s) "
-                         "for %s" % (unweighted_score, weighted_score, name))
+        raise ValueError(
+            "Unweighted and weighted scores are unexpectedly "
+            "almost equal (%s) and (%s) "
+            "for %s" % (unweighted_score, weighted_score, name)
+        )
 
     # check that sample_weight can be a list
-    weighted_score_list = metric(y1, y2,
-                                 sample_weight=sample_weight.tolist())
+    weighted_score_list = metric(y1, y2, sample_weight=sample_weight.tolist())
     assert_allclose(
-        weighted_score, weighted_score_list,
-        err_msg=("Weighted scores for array and list "
-                 "sample_weight input are not equal (%s != %s) for %s") % (
-                     weighted_score, weighted_score_list, name))
+        weighted_score,
+        weighted_score_list,
+        err_msg=(
+            "Weighted scores for array and list "
+            "sample_weight input are not equal (%s != %s) for %s"
+        )
+        % (weighted_score, weighted_score_list, name),
+    )
 
     # check that integer weights is the same as repeated samples
     repeat_weighted_score = metric(
         np.repeat(y1, sample_weight, axis=0),
-        np.repeat(y2, sample_weight, axis=0), sample_weight=None)
+        np.repeat(y2, sample_weight, axis=0),
+        sample_weight=None,
+    )
     assert_allclose(
-        weighted_score, repeat_weighted_score,
-        err_msg="Weighting %s is not equal to repeating samples" % name)
+        weighted_score,
+        repeat_weighted_score,
+        err_msg="Weighting %s is not equal to repeating samples" % name,
+    )
 
     # check that ignoring a fraction of the samples is equivalent to setting
     # the corresponding weights to zero
@@ -1266,42 +1535,49 @@ def check_sample_weight_invariance(name, metric, y1, y2):
     sample_weight_zeroed[::2] = 0
     y1_subset = y1[1::2]
     y2_subset = y2[1::2]
-    weighted_score_subset = metric(y1_subset, y2_subset,
-                                   sample_weight=sample_weight_subset)
-    weighted_score_zeroed = metric(y1, y2,
-                                   sample_weight=sample_weight_zeroed)
+    weighted_score_subset = metric(
+        y1_subset, y2_subset, sample_weight=sample_weight_subset
+    )
+    weighted_score_zeroed = metric(y1, y2, sample_weight=sample_weight_zeroed)
     assert_allclose(
-        weighted_score_subset, weighted_score_zeroed,
-        err_msg=("Zeroing weights does not give the same result as "
-                 "removing the corresponding samples (%s != %s) for %s" %
-                 (weighted_score_zeroed, weighted_score_subset, name)))
+        weighted_score_subset,
+        weighted_score_zeroed,
+        err_msg=(
+            "Zeroing weights does not give the same result as "
+            "removing the corresponding samples (%s != %s) for %s"
+        )
+        % (weighted_score_zeroed, weighted_score_subset, name),
+    )
 
-    if not name.startswith('unnormalized'):
+    if not name.startswith("unnormalized"):
         # check that the score is invariant under scaling of the weights by a
         # common factor
         for scaling in [2, 0.3]:
             assert_allclose(
                 weighted_score,
                 metric(y1, y2, sample_weight=sample_weight * scaling),
-                err_msg="%s sample_weight is not invariant "
-                        "under scaling" % name)
+                err_msg="%s sample_weight is not invariant under scaling" % name,
+            )
 
     # Check that if number of samples in y_true and sample_weight are not
     # equal, meaningful error is raised.
-    error_message = (r"Found input variables with inconsistent numbers of "
-                     r"samples: \[{}, {}, {}\]".format(
-                         _num_samples(y1), _num_samples(y2),
-                         _num_samples(sample_weight) * 2))
+    error_message = (
+        r"Found input variables with inconsistent numbers of "
+        r"samples: \[{}, {}, {}\]".format(
+            _num_samples(y1), _num_samples(y2), _num_samples(sample_weight) * 2
+        )
+    )
     with pytest.raises(ValueError, match=error_message):
-        metric(y1, y2, sample_weight=np.hstack([sample_weight,
-                                                sample_weight]))
+        metric(y1, y2, sample_weight=np.hstack([sample_weight, sample_weight]))
 
 
 @pytest.mark.parametrize(
-    'name',
+    "name",
     sorted(
-        set(ALL_METRICS).intersection(set(REGRESSION_METRICS)) -
-        METRICS_WITHOUT_SAMPLE_WEIGHT))
+        set(ALL_METRICS).intersection(set(REGRESSION_METRICS))
+        - METRICS_WITHOUT_SAMPLE_WEIGHT
+    ),
+)
 def test_regression_sample_weight_invariance(name):
     n_samples = 50
     random_state = check_random_state(0)
@@ -1313,16 +1589,46 @@ def test_regression_sample_weight_invariance(name):
 
 
 @pytest.mark.parametrize(
-    'name',
+    "name",
     sorted(
-        set(ALL_METRICS) - set(REGRESSION_METRICS) -
-        METRICS_WITHOUT_SAMPLE_WEIGHT - METRIC_UNDEFINED_BINARY))
+        set(ALL_METRICS).intersection(set(REGRESSION_METRICS))
+        - METRICS_WITHOUT_SAMPLE_WEIGHT
+    ),
+)
+def test_regression_with_invalid_sample_weight(name):
+    # Check that `sample_weight` with incorrect length raises error
+    n_samples = 50
+    random_state = check_random_state(0)
+    y_true = random_state.random_sample(size=(n_samples,))
+    y_pred = random_state.random_sample(size=(n_samples,))
+    metric = ALL_METRICS[name]
+
+    sample_weight = random_state.random_sample(size=(n_samples - 1,))
+    with pytest.raises(ValueError, match="Found input variables with inconsistent"):
+        metric(y_true, y_pred, sample_weight=sample_weight)
+
+    sample_weight = random_state.random_sample(size=(n_samples * 2,)).reshape(
+        (n_samples, 2)
+    )
+    with pytest.raises(ValueError, match="Sample weights must be 1D array or scalar"):
+        metric(y_true, y_pred, sample_weight=sample_weight)
+
+
+@pytest.mark.parametrize(
+    "name",
+    sorted(
+        set(ALL_METRICS)
+        - set(REGRESSION_METRICS)
+        - METRICS_WITHOUT_SAMPLE_WEIGHT
+        - METRIC_UNDEFINED_BINARY
+    ),
+)
 def test_binary_sample_weight_invariance(name):
     # binary
     n_samples = 50
     random_state = check_random_state(0)
-    y_true = random_state.randint(0, 2, size=(n_samples, ))
-    y_pred = random_state.randint(0, 2, size=(n_samples, ))
+    y_true = random_state.randint(0, 2, size=(n_samples,))
+    y_pred = random_state.randint(0, 2, size=(n_samples,))
     y_score = random_state.random_sample(size=(n_samples,))
     metric = ALL_METRICS[name]
     if name in THRESHOLDED_METRICS:
@@ -1332,16 +1638,20 @@ def test_binary_sample_weight_invariance(name):
 
 
 @pytest.mark.parametrize(
-    'name',
+    "name",
     sorted(
-        set(ALL_METRICS) - set(REGRESSION_METRICS) -
-        METRICS_WITHOUT_SAMPLE_WEIGHT - METRIC_UNDEFINED_BINARY_MULTICLASS))
+        set(ALL_METRICS)
+        - set(REGRESSION_METRICS)
+        - METRICS_WITHOUT_SAMPLE_WEIGHT
+        - METRIC_UNDEFINED_BINARY_MULTICLASS
+    ),
+)
 def test_multiclass_sample_weight_invariance(name):
     # multiclass
     n_samples = 50
     random_state = check_random_state(0)
-    y_true = random_state.randint(0, 5, size=(n_samples, ))
-    y_pred = random_state.randint(0, 5, size=(n_samples, ))
+    y_true = random_state.randint(0, 5, size=(n_samples,))
+    y_pred = random_state.randint(0, 5, size=(n_samples,))
     y_score = random_state.random_sample(size=(n_samples, 5))
     metric = ALL_METRICS[name]
     if name in THRESHOLDED_METRICS:
@@ -1354,21 +1664,27 @@ def test_multiclass_sample_weight_invariance(name):
 
 
 @pytest.mark.parametrize(
-    'name',
-    sorted((MULTILABELS_METRICS | THRESHOLDED_MULTILABEL_METRICS
-            | MULTIOUTPUT_METRICS) - METRICS_WITHOUT_SAMPLE_WEIGHT))
+    "name",
+    sorted(
+        (MULTILABELS_METRICS | THRESHOLDED_MULTILABEL_METRICS)
+        - METRICS_WITHOUT_SAMPLE_WEIGHT
+    ),
+)
 def test_multilabel_sample_weight_invariance(name):
     # multilabel indicator
     random_state = check_random_state(0)
-    _, ya = make_multilabel_classification(n_features=1, n_classes=10,
-                                           random_state=0, n_samples=50,
-                                           allow_unlabeled=False)
-    _, yb = make_multilabel_classification(n_features=1, n_classes=10,
-                                           random_state=1, n_samples=50,
-                                           allow_unlabeled=False)
+    _, ya = make_multilabel_classification(
+        n_features=1, n_classes=10, random_state=0, n_samples=50, allow_unlabeled=False
+    )
+    _, yb = make_multilabel_classification(
+        n_features=1, n_classes=10, random_state=1, n_samples=50, allow_unlabeled=False
+    )
     y_true = np.vstack([ya, yb])
     y_pred = np.vstack([ya, ya])
-    y_score = random_state.randint(1, 4, size=y_true.shape)
+    y_score = random_state.uniform(size=y_true.shape)
+
+    # Some metrics (e.g. log_loss) require y_score to be probabilities (sum to 1)
+    y_score /= y_score.sum(axis=1, keepdims=True)
 
     metric = ALL_METRICS[name]
     if name in THRESHOLDED_METRICS:
@@ -1377,7 +1693,19 @@ def test_multilabel_sample_weight_invariance(name):
         check_sample_weight_invariance(name, metric, y_true, y_pred)
 
 
-@ignore_warnings
+@pytest.mark.parametrize(
+    "name",
+    sorted(MULTIOUTPUT_METRICS - METRICS_WITHOUT_SAMPLE_WEIGHT),
+)
+def test_multioutput_sample_weight_invariance(name):
+    random_state = check_random_state(0)
+    y_true = random_state.uniform(0, 2, size=(20, 5))
+    y_pred = random_state.uniform(0, 2, size=(20, 5))
+
+    metric = ALL_METRICS[name]
+    check_sample_weight_invariance(name, metric, y_true, y_pred)
+
+
 def test_no_averaging_labels():
     # test labels argument when not using averaging
     # in multi-class and multi-label cases
@@ -1389,8 +1717,10 @@ def test_no_averaging_labels():
     _, inverse_labels = np.unique(labels, return_inverse=True)
 
     for name in METRICS_WITH_AVERAGING:
-        for y_true, y_pred in [[y_true_multiclass, y_pred_multiclass],
-                               [y_true_multilabel, y_pred_multilabel]]:
+        for y_true, y_pred in [
+            [y_true_multiclass, y_pred_multiclass],
+            [y_true_multilabel, y_pred_multilabel],
+        ]:
             if name not in MULTILABELS_METRICS and y_pred.ndim > 1:
                 continue
 
@@ -1402,8 +1732,8 @@ def test_no_averaging_labels():
 
 
 @pytest.mark.parametrize(
-    'name',
-    sorted(MULTILABELS_METRICS - {"unnormalized_multilabel_confusion_matrix"}))
+    "name", sorted(MULTILABELS_METRICS - {"unnormalized_multilabel_confusion_matrix"})
+)
 def test_multilabel_label_permutations_invariance(name):
     random_state = check_random_state(0)
     n_samples, n_classes = 20, 4
@@ -1423,12 +1753,16 @@ def test_multilabel_label_permutations_invariance(name):
 
 
 @pytest.mark.parametrize(
-    'name', sorted(THRESHOLDED_MULTILABEL_METRICS | MULTIOUTPUT_METRICS))
+    "name", sorted(THRESHOLDED_MULTILABEL_METRICS | MULTIOUTPUT_METRICS)
+)
 def test_thresholded_multilabel_multioutput_permutations_invariance(name):
     random_state = check_random_state(0)
     n_samples, n_classes = 20, 4
     y_true = random_state.randint(0, 2, size=(n_samples, n_classes))
-    y_score = random_state.normal(size=y_true.shape)
+    y_score = random_state.uniform(size=y_true.shape)
+
+    # Some metrics (e.g. log_loss) require y_score to be probabilities (sum to 1)
+    y_score /= y_score.sum(axis=1, keepdims=True)
 
     # Makes sure all samples have at least one label. This works around errors
     # when running metrics where average="sample"
@@ -1455,8 +1789,8 @@ def test_thresholded_multilabel_multioutput_permutations_invariance(name):
 
 
 @pytest.mark.parametrize(
-    'name',
-    sorted(set(THRESHOLDED_METRICS) - METRIC_UNDEFINED_BINARY_MULTICLASS))
+    "name", sorted(set(THRESHOLDED_METRICS) - METRIC_UNDEFINED_BINARY_MULTICLASS)
+)
 def test_thresholded_metric_permutation_invariance(name):
     n_samples, n_classes = 100, 3
     random_state = check_random_state(0)
@@ -1522,16 +1856,513 @@ def test_metrics_pos_label_error_str(metric, y_pred_threshold, dtype_y_str):
         "pass pos_label explicit"
     )
     err_msg_pos_label_1 = (
-        r"pos_label=1 is not a valid label. It should be one of "
-        r"\['eggs', 'spam'\]"
+        r"pos_label=1 is not a valid label. It should be one of \['eggs', 'spam'\]"
     )
 
     pos_label_default = signature(metric).parameters["pos_label"].default
 
-    err_msg = (
-        err_msg_pos_label_1
-        if pos_label_default == 1
-        else err_msg_pos_label_None
-    )
+    err_msg = err_msg_pos_label_1 if pos_label_default == 1 else err_msg_pos_label_None
     with pytest.raises(ValueError, match=err_msg):
         metric(y1, y2)
+
+
+def check_array_api_metric(
+    metric, array_namespace, device, dtype_name, a_np, b_np, **metric_kwargs
+):
+    xp = _array_api_for_tests(array_namespace, device)
+
+    a_xp = xp.asarray(a_np, device=device)
+    b_xp = xp.asarray(b_np, device=device)
+
+    metric_np = metric(a_np, b_np, **metric_kwargs)
+
+    if metric_kwargs.get("sample_weight") is not None:
+        metric_kwargs["sample_weight"] = xp.asarray(
+            metric_kwargs["sample_weight"], device=device
+        )
+
+    multioutput = metric_kwargs.get("multioutput")
+    if isinstance(multioutput, np.ndarray):
+        metric_kwargs["multioutput"] = xp.asarray(multioutput, device=device)
+
+    # When array API dispatch is disabled, and np.asarray works (for example PyTorch
+    # with CPU device), calling the metric function with such numpy compatible inputs
+    # should work (albeit by implicitly converting to numpy arrays instead of
+    # dispatching to the array library).
+    try:
+        np.asarray(a_xp)
+        np.asarray(b_xp)
+        numpy_as_array_works = True
+    except (TypeError, RuntimeError):
+        # PyTorch with CUDA device and CuPy raise TypeError consistently.
+        # array-api-strict chose to raise RuntimeError instead. Exception type
+        # may need to be updated in the future for other libraries.
+        numpy_as_array_works = False
+
+    if numpy_as_array_works:
+        metric_xp = metric(a_xp, b_xp, **metric_kwargs)
+        assert_allclose(
+            metric_xp,
+            metric_np,
+            atol=_atol_for_type(dtype_name),
+        )
+        metric_xp_mixed_1 = metric(a_np, b_xp, **metric_kwargs)
+        assert_allclose(
+            metric_xp_mixed_1,
+            metric_np,
+            atol=_atol_for_type(dtype_name),
+        )
+        metric_xp_mixed_2 = metric(a_xp, b_np, **metric_kwargs)
+        assert_allclose(
+            metric_xp_mixed_2,
+            metric_np,
+            atol=_atol_for_type(dtype_name),
+        )
+
+    with config_context(array_api_dispatch=True):
+        metric_xp = metric(a_xp, b_xp, **metric_kwargs)
+
+        assert_allclose(
+            _convert_to_numpy(xp.asarray(metric_xp), xp),
+            metric_np,
+            atol=_atol_for_type(dtype_name),
+        )
+
+
+def check_array_api_binary_classification_metric(
+    metric, array_namespace, device, dtype_name
+):
+    y_true_np = np.array([0, 0, 1, 1])
+    y_pred_np = np.array([0, 1, 0, 1])
+
+    check_array_api_metric(
+        metric,
+        array_namespace,
+        device,
+        dtype_name,
+        a_np=y_true_np,
+        b_np=y_pred_np,
+        sample_weight=None,
+    )
+
+    sample_weight = np.array([0.0, 0.1, 2.0, 1.0], dtype=dtype_name)
+
+    check_array_api_metric(
+        metric,
+        array_namespace,
+        device,
+        dtype_name,
+        a_np=y_true_np,
+        b_np=y_pred_np,
+        sample_weight=sample_weight,
+    )
+
+
+def check_array_api_multiclass_classification_metric(
+    metric, array_namespace, device, dtype_name
+):
+    y_true_np = np.array([0, 1, 2, 3])
+    y_pred_np = np.array([0, 1, 0, 2])
+
+    additional_params = {
+        "average": ("micro", "macro", "weighted"),
+        "beta": (0.2, 0.5, 0.8),
+    }
+    metric_kwargs_combinations = _get_metric_kwargs_for_array_api_testing(
+        metric=metric,
+        params=additional_params,
+    )
+    for metric_kwargs in metric_kwargs_combinations:
+        check_array_api_metric(
+            metric,
+            array_namespace,
+            device,
+            dtype_name,
+            a_np=y_true_np,
+            b_np=y_pred_np,
+            sample_weight=None,
+            **metric_kwargs,
+        )
+
+        sample_weight = np.array([0.0, 0.1, 2.0, 1.0], dtype=dtype_name)
+
+        check_array_api_metric(
+            metric,
+            array_namespace,
+            device,
+            dtype_name,
+            a_np=y_true_np,
+            b_np=y_pred_np,
+            sample_weight=sample_weight,
+            **metric_kwargs,
+        )
+
+
+def check_array_api_multilabel_classification_metric(
+    metric, array_namespace, device, dtype_name
+):
+    y_true_np = np.array([[1, 1], [0, 1], [0, 0]], dtype=dtype_name)
+    y_pred_np = np.array([[1, 1], [1, 1], [1, 1]], dtype=dtype_name)
+
+    additional_params = {
+        "average": ("micro", "macro", "weighted"),
+        "beta": (0.2, 0.5, 0.8),
+    }
+    metric_kwargs_combinations = _get_metric_kwargs_for_array_api_testing(
+        metric=metric,
+        params=additional_params,
+    )
+    for metric_kwargs in metric_kwargs_combinations:
+        check_array_api_metric(
+            metric,
+            array_namespace,
+            device,
+            dtype_name,
+            a_np=y_true_np,
+            b_np=y_pred_np,
+            sample_weight=None,
+            **metric_kwargs,
+        )
+
+        sample_weight = np.array([0.0, 0.1, 2.0], dtype=dtype_name)
+
+        check_array_api_metric(
+            metric,
+            array_namespace,
+            device,
+            dtype_name,
+            a_np=y_true_np,
+            b_np=y_pred_np,
+            sample_weight=sample_weight,
+            **metric_kwargs,
+        )
+
+
+def check_array_api_regression_metric(metric, array_namespace, device, dtype_name):
+    func_name = metric.func.__name__ if isinstance(metric, partial) else metric.__name__
+    if func_name == "mean_poisson_deviance" and sp_version < parse_version("1.14.0"):
+        pytest.skip(
+            "mean_poisson_deviance's dependency `xlogy` is available as of scipy 1.14.0"
+        )
+
+    y_true_np = np.array([2.0, 0.1, 1.0, 4.0], dtype=dtype_name)
+    y_pred_np = np.array([0.5, 0.5, 2, 2], dtype=dtype_name)
+
+    metric_kwargs = {}
+    metric_params = signature(metric).parameters
+
+    if "sample_weight" in metric_params:
+        metric_kwargs["sample_weight"] = None
+
+    check_array_api_metric(
+        metric,
+        array_namespace,
+        device,
+        dtype_name,
+        a_np=y_true_np,
+        b_np=y_pred_np,
+        **metric_kwargs,
+    )
+
+    if "sample_weight" in metric_params:
+        metric_kwargs["sample_weight"] = np.array(
+            [0.1, 2.0, 1.5, 0.5], dtype=dtype_name
+        )
+
+        check_array_api_metric(
+            metric,
+            array_namespace,
+            device,
+            dtype_name,
+            a_np=y_true_np,
+            b_np=y_pred_np,
+            **metric_kwargs,
+        )
+
+
+def check_array_api_regression_metric_multioutput(
+    metric, array_namespace, device, dtype_name
+):
+    y_true_np = np.array([[1, 3, 2], [1, 2, 2]], dtype=dtype_name)
+    y_pred_np = np.array([[1, 4, 4], [1, 1, 1]], dtype=dtype_name)
+
+    check_array_api_metric(
+        metric,
+        array_namespace,
+        device,
+        dtype_name,
+        a_np=y_true_np,
+        b_np=y_pred_np,
+        sample_weight=None,
+    )
+
+    sample_weight = np.array([0.1, 2.0], dtype=dtype_name)
+
+    check_array_api_metric(
+        metric,
+        array_namespace,
+        device,
+        dtype_name,
+        a_np=y_true_np,
+        b_np=y_pred_np,
+        sample_weight=sample_weight,
+    )
+
+    check_array_api_metric(
+        metric,
+        array_namespace,
+        device,
+        dtype_name,
+        a_np=y_true_np,
+        b_np=y_pred_np,
+        multioutput=np.array([0.1, 0.3, 0.7], dtype=dtype_name),
+    )
+
+    check_array_api_metric(
+        metric,
+        array_namespace,
+        device,
+        dtype_name,
+        a_np=y_true_np,
+        b_np=y_pred_np,
+        multioutput="raw_values",
+    )
+
+
+def check_array_api_metric_pairwise(metric, array_namespace, device, dtype_name):
+    X_np = np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]], dtype=dtype_name)
+    Y_np = np.array([[0.2, 0.3, 0.4], [0.5, 0.6, 0.7]], dtype=dtype_name)
+
+    metric_kwargs = {}
+    if "dense_output" in signature(metric).parameters:
+        metric_kwargs["dense_output"] = False
+        check_array_api_metric(
+            metric,
+            array_namespace,
+            device,
+            dtype_name,
+            a_np=X_np,
+            b_np=Y_np,
+            **metric_kwargs,
+        )
+        metric_kwargs["dense_output"] = True
+
+    check_array_api_metric(
+        metric,
+        array_namespace,
+        device,
+        dtype_name,
+        a_np=X_np,
+        b_np=Y_np,
+        **metric_kwargs,
+    )
+
+
+array_api_metric_checkers = {
+    accuracy_score: [
+        check_array_api_binary_classification_metric,
+        check_array_api_multiclass_classification_metric,
+        check_array_api_multilabel_classification_metric,
+    ],
+    f1_score: [
+        check_array_api_binary_classification_metric,
+        check_array_api_multiclass_classification_metric,
+        check_array_api_multilabel_classification_metric,
+    ],
+    fbeta_score: [
+        check_array_api_multiclass_classification_metric,
+        check_array_api_multilabel_classification_metric,
+    ],
+    jaccard_score: [
+        check_array_api_binary_classification_metric,
+        check_array_api_multiclass_classification_metric,
+        check_array_api_multilabel_classification_metric,
+    ],
+    multilabel_confusion_matrix: [
+        check_array_api_binary_classification_metric,
+        check_array_api_multiclass_classification_metric,
+        check_array_api_multilabel_classification_metric,
+    ],
+    precision_score: [
+        check_array_api_binary_classification_metric,
+        check_array_api_multiclass_classification_metric,
+        check_array_api_multilabel_classification_metric,
+    ],
+    recall_score: [
+        check_array_api_binary_classification_metric,
+        check_array_api_multiclass_classification_metric,
+        check_array_api_multilabel_classification_metric,
+    ],
+    zero_one_loss: [
+        check_array_api_binary_classification_metric,
+        check_array_api_multiclass_classification_metric,
+        check_array_api_multilabel_classification_metric,
+    ],
+    hamming_loss: [
+        check_array_api_binary_classification_metric,
+        check_array_api_multiclass_classification_metric,
+        check_array_api_multilabel_classification_metric,
+    ],
+    mean_tweedie_deviance: [check_array_api_regression_metric],
+    partial(mean_tweedie_deviance, power=-0.5): [check_array_api_regression_metric],
+    partial(mean_tweedie_deviance, power=1.5): [check_array_api_regression_metric],
+    r2_score: [
+        check_array_api_regression_metric,
+        check_array_api_regression_metric_multioutput,
+    ],
+    cosine_similarity: [check_array_api_metric_pairwise],
+    explained_variance_score: [
+        check_array_api_regression_metric,
+        check_array_api_regression_metric_multioutput,
+    ],
+    mean_absolute_error: [
+        check_array_api_regression_metric,
+        check_array_api_regression_metric_multioutput,
+    ],
+    mean_pinball_loss: [
+        check_array_api_regression_metric,
+        check_array_api_regression_metric_multioutput,
+    ],
+    mean_squared_error: [
+        check_array_api_regression_metric,
+        check_array_api_regression_metric_multioutput,
+    ],
+    mean_squared_log_error: [
+        check_array_api_regression_metric,
+        check_array_api_regression_metric_multioutput,
+    ],
+    median_absolute_error: [
+        check_array_api_regression_metric,
+        check_array_api_regression_metric_multioutput,
+    ],
+    d2_tweedie_score: [
+        check_array_api_regression_metric,
+    ],
+    paired_cosine_distances: [check_array_api_metric_pairwise],
+    mean_poisson_deviance: [check_array_api_regression_metric],
+    additive_chi2_kernel: [check_array_api_metric_pairwise],
+    mean_gamma_deviance: [check_array_api_regression_metric],
+    max_error: [check_array_api_regression_metric],
+    mean_absolute_percentage_error: [
+        check_array_api_regression_metric,
+        check_array_api_regression_metric_multioutput,
+    ],
+    chi2_kernel: [check_array_api_metric_pairwise],
+    paired_euclidean_distances: [check_array_api_metric_pairwise],
+    cosine_distances: [check_array_api_metric_pairwise],
+    euclidean_distances: [check_array_api_metric_pairwise],
+    linear_kernel: [check_array_api_metric_pairwise],
+    polynomial_kernel: [check_array_api_metric_pairwise],
+    rbf_kernel: [check_array_api_metric_pairwise],
+    root_mean_squared_error: [
+        check_array_api_regression_metric,
+        check_array_api_regression_metric_multioutput,
+    ],
+    root_mean_squared_log_error: [
+        check_array_api_regression_metric,
+        check_array_api_regression_metric_multioutput,
+    ],
+    sigmoid_kernel: [check_array_api_metric_pairwise],
+}
+
+
+def yield_metric_checker_combinations(metric_checkers=array_api_metric_checkers):
+    for metric, checkers in metric_checkers.items():
+        for checker in checkers:
+            yield metric, checker
+
+
+@pytest.mark.parametrize(
+    "array_namespace, device, dtype_name",
+    yield_namespace_device_dtype_combinations(),
+    ids=_get_namespace_device_dtype_ids,
+)
+@pytest.mark.parametrize("metric, check_func", yield_metric_checker_combinations())
+def test_array_api_compliance(metric, array_namespace, device, dtype_name, check_func):
+    # TODO: Remove once array-api-strict > 2.3.1
+    # https://github.com/data-apis/array-api-strict/issues/134 has been fixed but
+    # not released yet.
+    if (
+        getattr(metric, "__name__", None) == "median_absolute_error"
+        and array_namespace == "array_api_strict"
+    ):
+        try:
+            import array_api_strict
+        except ImportError:
+            pass
+        else:
+            if device == array_api_strict.Device("device1"):
+                pytest.xfail(
+                    "`_weighted_percentile` is affected by array_api_strict bug when "
+                    "indexing with tuple of arrays on non-'CPU_DEVICE' devices."
+                )
+    check_func(metric, array_namespace, device, dtype_name)
+
+
+@pytest.mark.parametrize("df_lib_name", ["pandas", "polars"])
+@pytest.mark.parametrize("metric_name", sorted(ALL_METRICS))
+def test_metrics_dataframe_series(metric_name, df_lib_name):
+    df_lib = pytest.importorskip(df_lib_name)
+
+    y_pred = df_lib.Series([0.0, 1.0, 0, 1.0])
+    y_true = df_lib.Series([1.0, 0.0, 0.0, 0.0])
+
+    metric = ALL_METRICS[metric_name]
+    try:
+        expected_metric = metric(y_pred.to_numpy(), y_true.to_numpy())
+    except ValueError:
+        pytest.skip(f"{metric_name} can not deal with 1d inputs")
+
+    assert_allclose(metric(y_pred, y_true), expected_metric)
+
+
+def _get_metric_kwargs_for_array_api_testing(metric, params):
+    """Helper function to enable specifying a variety of additional params and
+    their corresponding values, so that they can be passed to a metric function
+    when testing for array api compliance."""
+    metric_kwargs_combinations = [{}]
+    for param, values in params.items():
+        if param not in signature(metric).parameters:
+            continue
+
+        new_combinations = []
+        for kwargs in metric_kwargs_combinations:
+            for value in values:
+                new_kwargs = kwargs.copy()
+                new_kwargs[param] = value
+                new_combinations.append(new_kwargs)
+
+        metric_kwargs_combinations = new_combinations
+
+    return metric_kwargs_combinations
+
+
+@pytest.mark.parametrize("name", sorted(ALL_METRICS))
+def test_returned_value_consistency(name):
+    """Ensure that the returned values of all metrics are consistent.
+
+    It can either be a float, a numpy array, or a tuple of floats or numpy arrays.
+    It should not be a numpy float64 or float32.
+    """
+
+    rng = np.random.RandomState(0)
+    y_true = rng.randint(0, 2, size=(20,))
+    y_pred = rng.randint(0, 2, size=(20,))
+
+    if name in METRICS_REQUIRE_POSITIVE_Y:
+        y_true, y_pred = _require_positive_targets(y_true, y_pred)
+
+    if name in METRIC_UNDEFINED_BINARY:
+        y_true = rng.randint(0, 2, size=(20, 3))
+        y_pred = rng.randint(0, 2, size=(20, 3))
+
+    metric = ALL_METRICS[name]
+    score = metric(y_true, y_pred)
+
+    assert isinstance(score, (float, np.ndarray, tuple))
+    assert not isinstance(score, (np.float64, np.float32))
+
+    if isinstance(score, tuple):
+        assert all(isinstance(v, float) for v in score) or all(
+            isinstance(v, np.ndarray) for v in score
+        )
diff --git a/sklearn/metrics/tests/test_dist_metrics.py b/sklearn/metrics/tests/test_dist_metrics.py
new file mode 100644
index 0000000000000..f93d3b984bdb7
--- /dev/null
+++ b/sklearn/metrics/tests/test_dist_metrics.py
@@ -0,0 +1,431 @@
+import copy
+import itertools
+import pickle
+
+import numpy as np
+import pytest
+from scipy.spatial.distance import cdist
+
+from sklearn.metrics import DistanceMetric
+from sklearn.metrics._dist_metrics import (
+    BOOL_METRICS,
+    DEPRECATED_METRICS,
+    DistanceMetric32,
+    DistanceMetric64,
+)
+from sklearn.utils import check_random_state
+from sklearn.utils._testing import (
+    assert_allclose,
+    create_memmap_backed_data,
+    ignore_warnings,
+)
+from sklearn.utils.fixes import CSR_CONTAINERS
+
+
+def dist_func(x1, x2, p):
+    return np.sum((x1 - x2) ** p) ** (1.0 / p)
+
+
+rng = check_random_state(0)
+d = 4
+n1 = 20
+n2 = 25
+X64 = rng.random_sample((n1, d))
+Y64 = rng.random_sample((n2, d))
+X32 = X64.astype("float32")
+Y32 = Y64.astype("float32")
+
+[X_mmap, Y_mmap] = create_memmap_backed_data([X64, Y64])
+
+# make boolean arrays: ones and zeros
+X_bool = (X64 < 0.3).astype(np.float64)  # quite sparse
+Y_bool = (Y64 < 0.7).astype(np.float64)  # not too sparse
+
+[X_bool_mmap, Y_bool_mmap] = create_memmap_backed_data([X_bool, Y_bool])
+
+
+V = rng.random_sample((d, d))
+VI = np.dot(V, V.T)
+
+METRICS_DEFAULT_PARAMS = [
+    ("euclidean", {}),
+    ("cityblock", {}),
+    ("minkowski", dict(p=(0.5, 1, 1.5, 2, 3))),
+    ("chebyshev", {}),
+    ("seuclidean", dict(V=(rng.random_sample(d),))),
+    ("mahalanobis", dict(VI=(VI,))),
+    ("hamming", {}),
+    ("canberra", {}),
+    ("braycurtis", {}),
+    ("minkowski", dict(p=(0.5, 1, 1.5, 3), w=(rng.random_sample(d),))),
+]
+
+
+@pytest.mark.parametrize(
+    "metric_param_grid", METRICS_DEFAULT_PARAMS, ids=lambda params: params[0]
+)
+@pytest.mark.parametrize("X, Y", [(X64, Y64), (X32, Y32), (X_mmap, Y_mmap)])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_cdist(metric_param_grid, X, Y, csr_container):
+    metric, param_grid = metric_param_grid
+    keys = param_grid.keys()
+    X_csr, Y_csr = csr_container(X), csr_container(Y)
+    for vals in itertools.product(*param_grid.values()):
+        kwargs = dict(zip(keys, vals))
+        rtol_dict = {}
+        if metric == "mahalanobis" and X.dtype == np.float32:
+            # Computation of mahalanobis differs between
+            # the scipy and scikit-learn implementation.
+            # Hence, we increase the relative tolerance.
+            # TODO: Inspect slight numerical discrepancy
+            # with scipy
+            rtol_dict = {"rtol": 1e-6}
+
+        D_scipy_cdist = cdist(X, Y, metric, **kwargs)
+
+        dm = DistanceMetric.get_metric(metric, X.dtype, **kwargs)
+
+        # DistanceMetric.pairwise must be consistent for all
+        # combinations of formats in {sparse, dense}.
+        D_sklearn = dm.pairwise(X, Y)
+        assert D_sklearn.flags.c_contiguous
+        assert_allclose(D_sklearn, D_scipy_cdist, **rtol_dict)
+
+        D_sklearn = dm.pairwise(X_csr, Y_csr)
+        assert D_sklearn.flags.c_contiguous
+        assert_allclose(D_sklearn, D_scipy_cdist, **rtol_dict)
+
+        D_sklearn = dm.pairwise(X_csr, Y)
+        assert D_sklearn.flags.c_contiguous
+        assert_allclose(D_sklearn, D_scipy_cdist, **rtol_dict)
+
+        D_sklearn = dm.pairwise(X, Y_csr)
+        assert D_sklearn.flags.c_contiguous
+        assert_allclose(D_sklearn, D_scipy_cdist, **rtol_dict)
+
+
+@pytest.mark.parametrize("metric", BOOL_METRICS)
+@pytest.mark.parametrize(
+    "X_bool, Y_bool", [(X_bool, Y_bool), (X_bool_mmap, Y_bool_mmap)]
+)
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_cdist_bool_metric(metric, X_bool, Y_bool, csr_container):
+    if metric in DEPRECATED_METRICS:
+        with ignore_warnings(category=DeprecationWarning):
+            # Some metrics can be deprecated depending on the scipy version.
+            # But if they are present, we still want to test whether
+            # scikit-learn gives the same result, whether or not they are
+            # deprecated.
+            D_scipy_cdist = cdist(X_bool, Y_bool, metric)
+    else:
+        D_scipy_cdist = cdist(X_bool, Y_bool, metric)
+
+    dm = DistanceMetric.get_metric(metric)
+    D_sklearn = dm.pairwise(X_bool, Y_bool)
+    assert_allclose(D_sklearn, D_scipy_cdist)
+
+    # DistanceMetric.pairwise must be consistent
+    # on all combinations of format in {sparse, dense}².
+    X_bool_csr, Y_bool_csr = csr_container(X_bool), csr_container(Y_bool)
+
+    D_sklearn = dm.pairwise(X_bool, Y_bool)
+    assert D_sklearn.flags.c_contiguous
+    assert_allclose(D_sklearn, D_scipy_cdist)
+
+    D_sklearn = dm.pairwise(X_bool_csr, Y_bool_csr)
+    assert D_sklearn.flags.c_contiguous
+    assert_allclose(D_sklearn, D_scipy_cdist)
+
+    D_sklearn = dm.pairwise(X_bool, Y_bool_csr)
+    assert D_sklearn.flags.c_contiguous
+    assert_allclose(D_sklearn, D_scipy_cdist)
+
+    D_sklearn = dm.pairwise(X_bool_csr, Y_bool)
+    assert D_sklearn.flags.c_contiguous
+    assert_allclose(D_sklearn, D_scipy_cdist)
+
+
+@pytest.mark.parametrize(
+    "metric_param_grid", METRICS_DEFAULT_PARAMS, ids=lambda params: params[0]
+)
+@pytest.mark.parametrize("X", [X64, X32, X_mmap])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_pdist(metric_param_grid, X, csr_container):
+    metric, param_grid = metric_param_grid
+    keys = param_grid.keys()
+    X_csr = csr_container(X)
+    for vals in itertools.product(*param_grid.values()):
+        kwargs = dict(zip(keys, vals))
+        rtol_dict = {}
+        if metric == "mahalanobis" and X.dtype == np.float32:
+            # Computation of mahalanobis differs between
+            # the scipy and scikit-learn implementation.
+            # Hence, we increase the relative tolerance.
+            # TODO: Inspect slight numerical discrepancy
+            # with scipy
+            rtol_dict = {"rtol": 1e-6}
+
+        D_scipy_pdist = cdist(X, X, metric, **kwargs)
+
+        dm = DistanceMetric.get_metric(metric, X.dtype, **kwargs)
+        D_sklearn = dm.pairwise(X)
+        assert D_sklearn.flags.c_contiguous
+        assert_allclose(D_sklearn, D_scipy_pdist, **rtol_dict)
+
+        D_sklearn_csr = dm.pairwise(X_csr)
+        assert D_sklearn.flags.c_contiguous
+        assert_allclose(D_sklearn_csr, D_scipy_pdist, **rtol_dict)
+
+        D_sklearn_csr = dm.pairwise(X_csr, X_csr)
+        assert D_sklearn.flags.c_contiguous
+        assert_allclose(D_sklearn_csr, D_scipy_pdist, **rtol_dict)
+
+
+@pytest.mark.parametrize(
+    "metric_param_grid", METRICS_DEFAULT_PARAMS, ids=lambda params: params[0]
+)
+def test_distance_metrics_dtype_consistency(metric_param_grid):
+    # DistanceMetric must return similar distances for both float32 and float64
+    # input data.
+    metric, param_grid = metric_param_grid
+    keys = param_grid.keys()
+
+    # Choose rtol to make sure that this test is robust to changes in the random
+    # seed in the module-level test data generation code.
+    rtol = 1e-5
+
+    for vals in itertools.product(*param_grid.values()):
+        kwargs = dict(zip(keys, vals))
+        dm64 = DistanceMetric.get_metric(metric, np.float64, **kwargs)
+        dm32 = DistanceMetric.get_metric(metric, np.float32, **kwargs)
+
+        D64 = dm64.pairwise(X64)
+        D32 = dm32.pairwise(X32)
+
+        assert D64.dtype == np.float64
+        assert D32.dtype == np.float32
+
+        # assert_allclose introspects the dtype of the input arrays to decide
+        # which rtol value to use by default but in this case we know that D32
+        # is not computed with the same precision so we set rtol manually.
+        assert_allclose(D64, D32, rtol=rtol)
+
+        D64 = dm64.pairwise(X64, Y64)
+        D32 = dm32.pairwise(X32, Y32)
+        assert_allclose(D64, D32, rtol=rtol)
+
+
+@pytest.mark.parametrize("metric", BOOL_METRICS)
+@pytest.mark.parametrize("X_bool", [X_bool, X_bool_mmap])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_pdist_bool_metrics(metric, X_bool, csr_container):
+    if metric in DEPRECATED_METRICS:
+        with ignore_warnings(category=DeprecationWarning):
+            # Some metrics can be deprecated depending on the scipy version.
+            # But if they are present, we still want to test whether
+            # scikit-learn gives the same result, whether or not they are
+            # deprecated.
+            D_scipy_pdist = cdist(X_bool, X_bool, metric)
+    else:
+        D_scipy_pdist = cdist(X_bool, X_bool, metric)
+
+    dm = DistanceMetric.get_metric(metric)
+    D_sklearn = dm.pairwise(X_bool)
+    assert_allclose(D_sklearn, D_scipy_pdist)
+
+    X_bool_csr = csr_container(X_bool)
+    D_sklearn = dm.pairwise(X_bool_csr)
+    assert_allclose(D_sklearn, D_scipy_pdist)
+
+
+@pytest.mark.parametrize("writable_kwargs", [True, False])
+@pytest.mark.parametrize(
+    "metric_param_grid", METRICS_DEFAULT_PARAMS, ids=lambda params: params[0]
+)
+@pytest.mark.parametrize("X", [X64, X32])
+def test_pickle(writable_kwargs, metric_param_grid, X):
+    metric, param_grid = metric_param_grid
+    keys = param_grid.keys()
+    for vals in itertools.product(*param_grid.values()):
+        if any(isinstance(val, np.ndarray) for val in vals):
+            vals = copy.deepcopy(vals)
+            for val in vals:
+                if isinstance(val, np.ndarray):
+                    val.setflags(write=writable_kwargs)
+        kwargs = dict(zip(keys, vals))
+        dm = DistanceMetric.get_metric(metric, X.dtype, **kwargs)
+        D1 = dm.pairwise(X)
+        dm2 = pickle.loads(pickle.dumps(dm))
+        D2 = dm2.pairwise(X)
+        assert_allclose(D1, D2)
+
+
+@pytest.mark.parametrize("metric", BOOL_METRICS)
+@pytest.mark.parametrize("X_bool", [X_bool, X_bool_mmap])
+def test_pickle_bool_metrics(metric, X_bool):
+    dm = DistanceMetric.get_metric(metric)
+    D1 = dm.pairwise(X_bool)
+    dm2 = pickle.loads(pickle.dumps(dm))
+    D2 = dm2.pairwise(X_bool)
+    assert_allclose(D1, D2)
+
+
+@pytest.mark.parametrize("X, Y", [(X64, Y64), (X32, Y32), (X_mmap, Y_mmap)])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_haversine_metric(X, Y, csr_container):
+    # The Haversine DistanceMetric only works on 2 features.
+    X = np.asarray(X[:, :2])
+    Y = np.asarray(Y[:, :2])
+
+    X_csr, Y_csr = csr_container(X), csr_container(Y)
+
+    # Haversine is not supported by scipy.special.distance.{cdist,pdist}
+    # So we reimplement it to have a reference.
+    def haversine_slow(x1, x2):
+        return 2 * np.arcsin(
+            np.sqrt(
+                np.sin(0.5 * (x1[0] - x2[0])) ** 2
+                + np.cos(x1[0]) * np.cos(x2[0]) * np.sin(0.5 * (x1[1] - x2[1])) ** 2
+            )
+        )
+
+    D_reference = np.zeros((X_csr.shape[0], Y_csr.shape[0]))
+    for i, xi in enumerate(X):
+        for j, yj in enumerate(Y):
+            D_reference[i, j] = haversine_slow(xi, yj)
+
+    haversine = DistanceMetric.get_metric("haversine", X.dtype)
+
+    D_sklearn = haversine.pairwise(X, Y)
+    assert_allclose(
+        haversine.dist_to_rdist(D_sklearn), np.sin(0.5 * D_reference) ** 2, rtol=1e-6
+    )
+
+    assert_allclose(D_sklearn, D_reference)
+
+    D_sklearn = haversine.pairwise(X_csr, Y_csr)
+    assert D_sklearn.flags.c_contiguous
+    assert_allclose(D_sklearn, D_reference)
+
+    D_sklearn = haversine.pairwise(X_csr, Y)
+    assert D_sklearn.flags.c_contiguous
+    assert_allclose(D_sklearn, D_reference)
+
+    D_sklearn = haversine.pairwise(X, Y_csr)
+    assert D_sklearn.flags.c_contiguous
+    assert_allclose(D_sklearn, D_reference)
+
+
+def test_pyfunc_metric():
+    X = np.random.random((10, 3))
+
+    euclidean = DistanceMetric.get_metric("euclidean")
+    pyfunc = DistanceMetric.get_metric("pyfunc", func=dist_func, p=2)
+
+    # Check if both callable metric and predefined metric initialized
+    # DistanceMetric object is picklable
+    euclidean_pkl = pickle.loads(pickle.dumps(euclidean))
+    pyfunc_pkl = pickle.loads(pickle.dumps(pyfunc))
+
+    D1 = euclidean.pairwise(X)
+    D2 = pyfunc.pairwise(X)
+
+    D1_pkl = euclidean_pkl.pairwise(X)
+    D2_pkl = pyfunc_pkl.pairwise(X)
+
+    assert_allclose(D1, D2)
+    assert_allclose(D1_pkl, D2_pkl)
+
+
+def test_input_data_size():
+    # Regression test for #6288
+    # Previously, a metric requiring a particular input dimension would fail
+    def custom_metric(x, y):
+        assert x.shape[0] == 3
+        return np.sum((x - y) ** 2)
+
+    rng = check_random_state(0)
+    X = rng.rand(10, 3)
+
+    pyfunc = DistanceMetric.get_metric("pyfunc", func=custom_metric)
+    eucl = DistanceMetric.get_metric("euclidean")
+    assert_allclose(pyfunc.pairwise(X), eucl.pairwise(X) ** 2)
+
+
+def test_readonly_kwargs():
+    # Non-regression test for:
+    # https://github.com/scikit-learn/scikit-learn/issues/21685
+
+    rng = check_random_state(0)
+
+    weights = rng.rand(100)
+    VI = rng.rand(10, 10)
+    weights.setflags(write=False)
+    VI.setflags(write=False)
+
+    # Those distances metrics have to support readonly buffers.
+    DistanceMetric.get_metric("seuclidean", V=weights)
+    DistanceMetric.get_metric("mahalanobis", VI=VI)
+
+
+@pytest.mark.parametrize(
+    "w, err_type, err_msg",
+    [
+        (np.array([1, 1.5, -13]), ValueError, "w cannot contain negative weights"),
+        (np.array([1, 1.5, np.nan]), ValueError, "w contains NaN"),
+        *[
+            (
+                csr_container([[1, 1.5, 1]]),
+                TypeError,
+                "Sparse data was passed for w, but dense data is required",
+            )
+            for csr_container in CSR_CONTAINERS
+        ],
+        (np.array(["a", "b", "c"]), ValueError, "could not convert string to float"),
+        (np.array([]), ValueError, "a minimum of 1 is required"),
+    ],
+)
+def test_minkowski_metric_validate_weights_values(w, err_type, err_msg):
+    with pytest.raises(err_type, match=err_msg):
+        DistanceMetric.get_metric("minkowski", p=3, w=w)
+
+
+def test_minkowski_metric_validate_weights_size():
+    w2 = rng.random_sample(d + 1)
+    dm = DistanceMetric.get_metric("minkowski", p=3, w=w2)
+    msg = (
+        "MinkowskiDistance: the size of w must match "
+        f"the number of features \\({X64.shape[1]}\\). "
+        f"Currently len\\(w\\)={w2.shape[0]}."
+    )
+    with pytest.raises(ValueError, match=msg):
+        dm.pairwise(X64, Y64)
+
+
+@pytest.mark.parametrize("metric, metric_kwargs", METRICS_DEFAULT_PARAMS)
+@pytest.mark.parametrize("dtype", (np.float32, np.float64))
+def test_get_metric_dtype(metric, metric_kwargs, dtype):
+    specialized_cls = {
+        np.float32: DistanceMetric32,
+        np.float64: DistanceMetric64,
+    }[dtype]
+
+    # We don't need the entire grid, just one for a sanity check
+    metric_kwargs = {k: v[0] for k, v in metric_kwargs.items()}
+    generic_type = type(DistanceMetric.get_metric(metric, dtype, **metric_kwargs))
+    specialized_type = type(specialized_cls.get_metric(metric, **metric_kwargs))
+
+    assert generic_type is specialized_type
+
+
+def test_get_metric_bad_dtype():
+    dtype = np.int32
+    msg = r"Unexpected dtype .* provided. Please select a dtype from"
+    with pytest.raises(ValueError, match=msg):
+        DistanceMetric.get_metric("manhattan", dtype)
+
+
+def test_minkowski_metric_validate_bad_p_parameter():
+    msg = "p must be greater than 0"
+    with pytest.raises(ValueError, match=msg):
+        DistanceMetric.get_metric("minkowski", p=0)
diff --git a/sklearn/metrics/tests/test_pairwise.py b/sklearn/metrics/tests/test_pairwise.py
index 1ff62af04c05f..4c1ba4b2f7d52 100644
--- a/sklearn/metrics/tests/test_pairwise.py
+++ b/sklearn/metrics/tests/test_pairwise.py
@@ -1,121 +1,137 @@
+import warnings
 from types import GeneratorType
 
 import numpy as np
-from numpy import linalg
-
-from scipy.sparse import dok_matrix, csr_matrix, issparse
-from scipy.spatial.distance import cosine, cityblock, minkowski
-from scipy.spatial.distance import cdist, pdist, squareform
-try:
-    from scipy.spatial.distance import wminkowski
-except ImportError:
-    # In scipy 1.6.0, wminkowski is deprecated and minkowski
-    # should be used instead.
-    from scipy.spatial.distance import minkowski as wminkowski
-
-from sklearn.utils.fixes import sp_version, parse_version
-
 import pytest
+from numpy import linalg
+from scipy.sparse import issparse
+from scipy.spatial.distance import (
+    cdist,
+    cityblock,
+    cosine,
+    minkowski,
+    pdist,
+    squareform,
+)
 
 from sklearn import config_context
-
-from sklearn.utils._testing import assert_array_almost_equal
-from sklearn.utils._testing import assert_allclose
-from sklearn.utils._testing import assert_almost_equal
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils._testing import ignore_warnings
-
-from sklearn.metrics.pairwise import euclidean_distances
-from sklearn.metrics.pairwise import nan_euclidean_distances
-from sklearn.metrics.pairwise import manhattan_distances
-from sklearn.metrics.pairwise import haversine_distances
-from sklearn.metrics.pairwise import linear_kernel
-from sklearn.metrics.pairwise import chi2_kernel, additive_chi2_kernel
-from sklearn.metrics.pairwise import polynomial_kernel
-from sklearn.metrics.pairwise import rbf_kernel
-from sklearn.metrics.pairwise import laplacian_kernel
-from sklearn.metrics.pairwise import sigmoid_kernel
-from sklearn.metrics.pairwise import cosine_similarity
-from sklearn.metrics.pairwise import cosine_distances
-from sklearn.metrics.pairwise import pairwise_distances
-from sklearn.metrics.pairwise import pairwise_distances_chunked
-from sklearn.metrics.pairwise import pairwise_distances_argmin_min
-from sklearn.metrics.pairwise import pairwise_distances_argmin
-from sklearn.metrics.pairwise import pairwise_kernels
-from sklearn.metrics.pairwise import PAIRWISE_KERNEL_FUNCTIONS
-from sklearn.metrics.pairwise import PAIRWISE_DISTANCE_FUNCTIONS
-from sklearn.metrics.pairwise import PAIRWISE_BOOLEAN_FUNCTIONS
-from sklearn.metrics.pairwise import PAIRED_DISTANCES
-from sklearn.metrics.pairwise import check_pairwise_arrays
-from sklearn.metrics.pairwise import check_paired_arrays
-from sklearn.metrics.pairwise import paired_distances
-from sklearn.metrics.pairwise import paired_euclidean_distances
-from sklearn.metrics.pairwise import paired_manhattan_distances
-from sklearn.metrics.pairwise import _euclidean_distances_upcast
-from sklearn.preprocessing import normalize
 from sklearn.exceptions import DataConversionWarning
-
-
-def test_pairwise_distances():
+from sklearn.metrics.pairwise import (
+    PAIRED_DISTANCES,
+    PAIRWISE_BOOLEAN_FUNCTIONS,
+    PAIRWISE_DISTANCE_FUNCTIONS,
+    PAIRWISE_KERNEL_FUNCTIONS,
+    _euclidean_distances_upcast,
+    additive_chi2_kernel,
+    check_paired_arrays,
+    check_pairwise_arrays,
+    chi2_kernel,
+    cosine_distances,
+    cosine_similarity,
+    euclidean_distances,
+    haversine_distances,
+    laplacian_kernel,
+    linear_kernel,
+    manhattan_distances,
+    nan_euclidean_distances,
+    paired_cosine_distances,
+    paired_distances,
+    paired_euclidean_distances,
+    paired_manhattan_distances,
+    pairwise_distances,
+    pairwise_distances_argmin,
+    pairwise_distances_argmin_min,
+    pairwise_distances_chunked,
+    pairwise_kernels,
+    polynomial_kernel,
+    rbf_kernel,
+    sigmoid_kernel,
+)
+from sklearn.preprocessing import normalize
+from sklearn.utils._testing import (
+    assert_allclose,
+    assert_almost_equal,
+    assert_array_equal,
+    ignore_warnings,
+)
+from sklearn.utils.fixes import (
+    BSR_CONTAINERS,
+    COO_CONTAINERS,
+    CSC_CONTAINERS,
+    CSR_CONTAINERS,
+    DOK_CONTAINERS,
+)
+from sklearn.utils.parallel import Parallel, delayed
+
+
+def test_pairwise_distances_for_dense_data(global_dtype):
     # Test the pairwise_distance helper function.
     rng = np.random.RandomState(0)
 
     # Euclidean distance should be equivalent to calling the function.
-    X = rng.random_sample((5, 4))
+    X = rng.random_sample((5, 4)).astype(global_dtype, copy=False)
     S = pairwise_distances(X, metric="euclidean")
     S2 = euclidean_distances(X)
-    assert_array_almost_equal(S, S2)
+    assert_allclose(S, S2)
+    assert S.dtype == S2.dtype == global_dtype
 
     # Euclidean distance, with Y != X.
-    Y = rng.random_sample((2, 4))
+    Y = rng.random_sample((2, 4)).astype(global_dtype, copy=False)
     S = pairwise_distances(X, Y, metric="euclidean")
     S2 = euclidean_distances(X, Y)
-    assert_array_almost_equal(S, S2)
+    assert_allclose(S, S2)
+    assert S.dtype == S2.dtype == global_dtype
+
     # Check to ensure NaNs work with pairwise_distances.
-    X_masked = rng.random_sample((5, 4))
-    Y_masked = rng.random_sample((2, 4))
+    X_masked = rng.random_sample((5, 4)).astype(global_dtype, copy=False)
+    Y_masked = rng.random_sample((2, 4)).astype(global_dtype, copy=False)
     X_masked[0, 0] = np.nan
     Y_masked[0, 0] = np.nan
     S_masked = pairwise_distances(X_masked, Y_masked, metric="nan_euclidean")
     S2_masked = nan_euclidean_distances(X_masked, Y_masked)
-    assert_array_almost_equal(S_masked, S2_masked)
+    assert_allclose(S_masked, S2_masked)
+    assert S_masked.dtype == S2_masked.dtype == global_dtype
+
     # Test with tuples as X and Y
     X_tuples = tuple([tuple([v for v in row]) for row in X])
     Y_tuples = tuple([tuple([v for v in row]) for row in Y])
     S2 = pairwise_distances(X_tuples, Y_tuples, metric="euclidean")
-    assert_array_almost_equal(S, S2)
+    assert_allclose(S, S2)
+    assert S.dtype == S2.dtype == global_dtype
 
     # Test haversine distance
     # The data should be valid latitude and longitude
-    X = rng.random_sample((5, 2))
-    X[:, 0] = (X[:, 0] - 0.5) * 2 * np.pi/2
+    # haversine converts to float64 currently so we don't check dtypes.
+    X = rng.random_sample((5, 2)).astype(global_dtype, copy=False)
+    X[:, 0] = (X[:, 0] - 0.5) * 2 * np.pi / 2
     X[:, 1] = (X[:, 1] - 0.5) * 2 * np.pi
     S = pairwise_distances(X, metric="haversine")
     S2 = haversine_distances(X)
-    assert_array_almost_equal(S, S2)
+    assert_allclose(S, S2)
 
     # Test haversine distance, with Y != X
-    Y = rng.random_sample((2, 2))
-    Y[:, 0] = (Y[:, 0] - 0.5)*2*np.pi/2
-    Y[:, 1] = (Y[:, 1] - 0.5)*2*np.pi
+    Y = rng.random_sample((2, 2)).astype(global_dtype, copy=False)
+    Y[:, 0] = (Y[:, 0] - 0.5) * 2 * np.pi / 2
+    Y[:, 1] = (Y[:, 1] - 0.5) * 2 * np.pi
     S = pairwise_distances(X, Y, metric="haversine")
     S2 = haversine_distances(X, Y)
-    assert_array_almost_equal(S, S2)
+    assert_allclose(S, S2)
 
     # "cityblock" uses scikit-learn metric, cityblock (function) is
     # scipy.spatial.
+    # The metric functions from scipy converts to float64 so we don't check the dtypes.
     S = pairwise_distances(X, metric="cityblock")
     S2 = pairwise_distances(X, metric=cityblock)
     assert S.shape[0] == S.shape[1]
     assert S.shape[0] == X.shape[0]
-    assert_array_almost_equal(S, S2)
+    assert_allclose(S, S2)
 
     # The manhattan metric should be equivalent to cityblock.
     S = pairwise_distances(X, Y, metric="manhattan")
     S2 = pairwise_distances(X, Y, metric=cityblock)
     assert S.shape[0] == X.shape[0]
     assert S.shape[1] == Y.shape[0]
-    assert_array_almost_equal(S, S2)
+    assert_allclose(S, S2)
 
     # Test cosine as a string metric versus cosine callable
     # The string "cosine" uses sklearn.metric,
@@ -124,35 +140,70 @@ def test_pairwise_distances():
     S2 = pairwise_distances(X, Y, metric=cosine)
     assert S.shape[0] == X.shape[0]
     assert S.shape[1] == Y.shape[0]
-    assert_array_almost_equal(S, S2)
+    assert_allclose(S, S2)
+
+
+@pytest.mark.parametrize("coo_container", COO_CONTAINERS)
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+@pytest.mark.parametrize("bsr_container", BSR_CONTAINERS)
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_pairwise_distances_for_sparse_data(
+    coo_container, csc_container, bsr_container, csr_container, global_dtype
+):
+    # Test the pairwise_distance helper function.
+    rng = np.random.RandomState(0)
+    X = rng.random_sample((5, 4)).astype(global_dtype, copy=False)
+    Y = rng.random_sample((2, 4)).astype(global_dtype, copy=False)
 
     # Test with sparse X and Y,
     # currently only supported for Euclidean, L1 and cosine.
-    X_sparse = csr_matrix(X)
-    Y_sparse = csr_matrix(Y)
+    X_sparse = csr_container(X)
+    Y_sparse = csr_container(Y)
+
     S = pairwise_distances(X_sparse, Y_sparse, metric="euclidean")
     S2 = euclidean_distances(X_sparse, Y_sparse)
-    assert_array_almost_equal(S, S2)
+    assert_allclose(S, S2)
+    assert S.dtype == S2.dtype == global_dtype
+
     S = pairwise_distances(X_sparse, Y_sparse, metric="cosine")
     S2 = cosine_distances(X_sparse, Y_sparse)
-    assert_array_almost_equal(S, S2)
-    S = pairwise_distances(X_sparse, Y_sparse.tocsc(), metric="manhattan")
-    S2 = manhattan_distances(X_sparse.tobsr(), Y_sparse.tocoo())
-    assert_array_almost_equal(S, S2)
+    assert_allclose(S, S2)
+    assert S.dtype == S2.dtype == global_dtype
+
+    S = pairwise_distances(X_sparse, csc_container(Y), metric="manhattan")
+    S2 = manhattan_distances(bsr_container(X), coo_container(Y))
+    assert_allclose(S, S2)
+    if global_dtype == np.float64:
+        assert S.dtype == S2.dtype == global_dtype
+    else:
+        # TODO Fix manhattan_distances to preserve dtype.
+        # currently pairwise_distances uses manhattan_distances but converts the result
+        # back to the input dtype
+        with pytest.raises(AssertionError):
+            assert S.dtype == S2.dtype == global_dtype
+
     S2 = manhattan_distances(X, Y)
-    assert_array_almost_equal(S, S2)
+    assert_allclose(S, S2)
+    if global_dtype == np.float64:
+        assert S.dtype == S2.dtype == global_dtype
+    else:
+        # TODO Fix manhattan_distances to preserve dtype.
+        # currently pairwise_distances uses manhattan_distances but converts the result
+        # back to the input dtype
+        with pytest.raises(AssertionError):
+            assert S.dtype == S2.dtype == global_dtype
 
     # Test with scipy.spatial.distance metric, with a kwd
     kwds = {"p": 2.0}
     S = pairwise_distances(X, Y, metric="minkowski", **kwds)
     S2 = pairwise_distances(X, Y, metric=minkowski, **kwds)
-    assert_array_almost_equal(S, S2)
+    assert_allclose(S, S2)
 
     # same with Y = None
     kwds = {"p": 2.0}
     S = pairwise_distances(X, metric="minkowski", **kwds)
     S2 = pairwise_distances(X, metric=minkowski, **kwds)
-    assert_array_almost_equal(S, S2)
+    assert_allclose(S, S2)
 
     # Test that scipy distance metrics throw an error if sparse matrix given
     with pytest.raises(TypeError):
@@ -160,12 +211,11 @@ def test_pairwise_distances():
     with pytest.raises(TypeError):
         pairwise_distances(X, Y_sparse, metric="minkowski")
 
-    # Test that a value error is raised if the metric is unknown
-    with pytest.raises(ValueError):
-        pairwise_distances(X, Y, metric="blah")
-
 
-@pytest.mark.parametrize('metric', PAIRWISE_BOOLEAN_FUNCTIONS)
+# Some scipy metrics are deprecated (depending on the scipy version) but we
+# still want to test them.
+@ignore_warnings(category=DeprecationWarning)
+@pytest.mark.parametrize("metric", PAIRWISE_BOOLEAN_FUNCTIONS)
 def test_pairwise_boolean_distance(metric):
     # test that we convert to boolean arrays for boolean distances
     rng = np.random.RandomState(0)
@@ -177,7 +227,7 @@ def test_pairwise_boolean_distance(metric):
     with ignore_warnings(category=DataConversionWarning):
         for Z in [Y, None]:
             res = pairwise_distances(X, Z, metric=metric)
-            res[np.isnan(res)] = 0
+            np.nan_to_num(res, nan=0, posinf=0, neginf=0, copy=False)
             assert np.sum(res != 0) == 0
 
     # non-boolean arrays are converted to boolean for boolean
@@ -191,31 +241,31 @@ def test_pairwise_boolean_distance(metric):
         pairwise_distances(X.astype(bool), Y=Y, metric=metric)
 
     # Check that no warning is raised if X is already boolean and Y is None:
-    with pytest.warns(None) as records:
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", DataConversionWarning)
         pairwise_distances(X.astype(bool), metric=metric)
-    assert len(records) == 0
 
 
 def test_no_data_conversion_warning():
     # No warnings issued if metric is not a boolean distance function
     rng = np.random.RandomState(0)
     X = rng.randn(5, 4)
-    with pytest.warns(None) as records:
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", DataConversionWarning)
         pairwise_distances(X, metric="minkowski")
-    assert len(records) == 0
 
 
-@pytest.mark.parametrize('func', [pairwise_distances, pairwise_kernels])
+@pytest.mark.parametrize("func", [pairwise_distances, pairwise_kernels])
 def test_pairwise_precomputed(func):
     # Test correct shape
-    with pytest.raises(ValueError, match='.* shape .*'):
-        func(np.zeros((5, 3)), metric='precomputed')
+    with pytest.raises(ValueError, match=".* shape .*"):
+        func(np.zeros((5, 3)), metric="precomputed")
     # with two args
-    with pytest.raises(ValueError, match='.* shape .*'):
-        func(np.zeros((5, 3)), np.zeros((4, 4)), metric='precomputed')
+    with pytest.raises(ValueError, match=".* shape .*"):
+        func(np.zeros((5, 3)), np.zeros((4, 4)), metric="precomputed")
     # even if shape[1] agrees (although thus second arg is spurious)
-    with pytest.raises(ValueError, match='.* shape .*'):
-        func(np.zeros((5, 3)), np.zeros((4, 3)), metric='precomputed')
+    with pytest.raises(ValueError, match=".* shape .*"):
+        func(np.zeros((5, 3)), np.zeros((4, 3)), metric="precomputed")
 
     # Test not copied (if appropriate dtype)
     S = np.zeros((5, 5))
@@ -227,85 +277,55 @@ def test_pairwise_precomputed(func):
     assert S is S2
 
     # Test always returns float dtype
-    S = func(np.array([[1]], dtype='int'), metric='precomputed')
-    assert 'f' == S.dtype.kind
+    S = func(np.array([[1]], dtype="int"), metric="precomputed")
+    assert "f" == S.dtype.kind
 
     # Test converts list to array-like
-    S = func([[1.]], metric='precomputed')
+    S = func([[1.0]], metric="precomputed")
     assert isinstance(S, np.ndarray)
 
 
 def test_pairwise_precomputed_non_negative():
     # Test non-negative values
-    with pytest.raises(ValueError, match='.* non-negative values.*'):
-        pairwise_distances(np.full((5, 5), -1), metric='precomputed')
+    with pytest.raises(ValueError, match=".* non-negative values.*"):
+        pairwise_distances(np.full((5, 5), -1), metric="precomputed")
 
 
-_minkowski_kwds = {'w': np.arange(1, 5).astype('double', copy=False), 'p': 1}
-_wminkowski_kwds = {'w': np.arange(1, 5).astype('double', copy=False), 'p': 1}
+_minkowski_kwds = {"w": np.arange(1, 5).astype("double", copy=False), "p": 1}
 
 
 def callable_rbf_kernel(x, y, **kwds):
     # Callable version of pairwise.rbf_kernel.
     K = rbf_kernel(np.atleast_2d(x), np.atleast_2d(y), **kwds)
-    return K
+    # unpack the output since this is a scalar packed in a 0-dim array
+    return K.item()
 
 
 @pytest.mark.parametrize(
-        'func, metric, kwds',
-        [(pairwise_distances, 'euclidean', {}),
-         pytest.param(
-             pairwise_distances, minkowski, _minkowski_kwds,
-             marks=pytest.mark.skipif(
-                 sp_version < parse_version("1.0"),
-                 reason="minkowski does not accept the w "
-                        "parameter prior to scipy 1.0."
-             )
-         ),
-         pytest.param(
-             pairwise_distances, 'minkowski', _minkowski_kwds,
-             marks=pytest.mark.skipif(
-                 sp_version < parse_version("1.0"),
-                 reason="minkowski does not accept the w "
-                        "parameter prior to scipy 1.0."
-             )
-         ),
-         pytest.param(
-             pairwise_distances, wminkowski, _wminkowski_kwds,
-             marks=pytest.mark.skipif(
-                 sp_version >= parse_version("1.6.0"),
-                 reason="wminkowski is now minkowski "
-                        "and it has been already tested."
-             )
-         ),
-         pytest.param(
-             pairwise_distances, 'wminkowski', _wminkowski_kwds,
-             marks=pytest.mark.skipif(
-                 sp_version >= parse_version("1.6.0"),
-                 reason="wminkowski is now minkowski "
-                        "and it has been already tested."
-             )
-         ),
-         (pairwise_kernels, 'polynomial', {'degree': 1}),
-         (pairwise_kernels, callable_rbf_kernel, {'gamma': .1})])
-@pytest.mark.parametrize('array_constr', [np.array, csr_matrix])
-@pytest.mark.parametrize('dtype', [np.float64, int])
-def test_pairwise_parallel(func, metric, kwds, array_constr, dtype):
+    "func, metric, kwds",
+    [
+        (pairwise_distances, "euclidean", {}),
+        (
+            pairwise_distances,
+            minkowski,
+            _minkowski_kwds,
+        ),
+        (
+            pairwise_distances,
+            "minkowski",
+            _minkowski_kwds,
+        ),
+        (pairwise_kernels, "polynomial", {"degree": 1}),
+        (pairwise_kernels, callable_rbf_kernel, {"gamma": 0.1}),
+    ],
+)
+@pytest.mark.parametrize("dtype", [np.float64, np.float32, int])
+def test_pairwise_parallel(func, metric, kwds, dtype):
     rng = np.random.RandomState(0)
-    X = array_constr(5 * rng.random_sample((5, 4)), dtype=dtype)
-    Y = array_constr(5 * rng.random_sample((3, 4)), dtype=dtype)
-
-    try:
-        S = func(X, metric=metric, n_jobs=1, **kwds)
-    except (TypeError, ValueError) as exc:
-        # Not all metrics support sparse input
-        # ValueError may be triggered by bad callable
-        if array_constr is csr_matrix:
-            with pytest.raises(type(exc)):
-                func(X, metric=metric, n_jobs=2, **kwds)
-            return
-        else:
-            raise
+    X = np.array(5 * rng.random_sample((5, 4)), dtype=dtype)
+    Y = np.array(5 * rng.random_sample((3, 4)), dtype=dtype)
+
+    S = func(X, metric=metric, n_jobs=1, **kwds)
     S2 = func(X, metric=metric, n_jobs=2, **kwds)
     assert_allclose(S, S2)
 
@@ -318,15 +338,16 @@ def test_pairwise_callable_nonstrict_metric():
     # paired_distances should allow callable metric where metric(x, x) != 0
     # Knowing that the callable is a strict metric would allow the diagonal to
     # be left uncalculated and set to 0.
-    assert pairwise_distances([[1.]], metric=lambda x, y: 5)[0, 0] == 5
+    assert pairwise_distances([[1.0]], metric=lambda x, y: 5)[0, 0] == 5
 
 
 # Test with all metrics that should be in PAIRWISE_KERNEL_FUNCTIONS.
 @pytest.mark.parametrize(
-        'metric',
-        ["rbf", "laplacian", "sigmoid", "polynomial", "linear",
-         "chi2", "additive_chi2"])
-def test_pairwise_kernels(metric):
+    "metric",
+    ["rbf", "laplacian", "sigmoid", "polynomial", "linear", "chi2", "additive_chi2"],
+)
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_pairwise_kernels(metric, csr_container):
     # Test the pairwise_kernels helper function.
 
     rng = np.random.RandomState(0)
@@ -336,28 +357,25 @@ def test_pairwise_kernels(metric):
     # Test with Y=None
     K1 = pairwise_kernels(X, metric=metric)
     K2 = function(X)
-    assert_array_almost_equal(K1, K2)
+    assert_allclose(K1, K2)
     # Test with Y=Y
     K1 = pairwise_kernels(X, Y=Y, metric=metric)
     K2 = function(X, Y=Y)
-    assert_array_almost_equal(K1, K2)
+    assert_allclose(K1, K2)
     # Test with tuples as X and Y
     X_tuples = tuple([tuple([v for v in row]) for row in X])
     Y_tuples = tuple([tuple([v for v in row]) for row in Y])
     K2 = pairwise_kernels(X_tuples, Y_tuples, metric=metric)
-    assert_array_almost_equal(K1, K2)
+    assert_allclose(K1, K2)
 
     # Test with sparse X and Y
-    X_sparse = csr_matrix(X)
-    Y_sparse = csr_matrix(Y)
+    X_sparse = csr_container(X)
+    Y_sparse = csr_container(Y)
     if metric in ["chi2", "additive_chi2"]:
         # these don't support sparse matrices yet
-        with pytest.raises(ValueError):
-            pairwise_kernels(X_sparse, Y=Y_sparse,
-                             metric=metric)
         return
     K1 = pairwise_kernels(X_sparse, Y=Y_sparse, metric=metric)
-    assert_array_almost_equal(K1, K2)
+    assert_allclose(K1, K2)
 
 
 def test_pairwise_kernels_callable():
@@ -368,15 +386,15 @@ def test_pairwise_kernels_callable():
     Y = rng.random_sample((2, 4))
 
     metric = callable_rbf_kernel
-    kwds = {'gamma': 0.1}
+    kwds = {"gamma": 0.1}
     K1 = pairwise_kernels(X, Y=Y, metric=metric, **kwds)
     K2 = rbf_kernel(X, Y=Y, **kwds)
-    assert_array_almost_equal(K1, K2)
+    assert_allclose(K1, K2)
 
     # callable function, X=Y
     K1 = pairwise_kernels(X, Y=X, metric=metric, **kwds)
     K2 = rbf_kernel(X, Y=X, **kwds)
-    assert_array_almost_equal(K1, K2)
+    assert_allclose(K1, K2)
 
 
 def test_pairwise_kernels_filter_param():
@@ -386,14 +404,15 @@ def test_pairwise_kernels_filter_param():
     K = rbf_kernel(X, Y, gamma=0.1)
     params = {"gamma": 0.1, "blabla": ":)"}
     K2 = pairwise_kernels(X, Y, metric="rbf", filter_params=True, **params)
-    assert_array_almost_equal(K, K2)
+    assert_allclose(K, K2)
 
     with pytest.raises(TypeError):
         pairwise_kernels(X, Y, metric="rbf", **params)
 
 
-@pytest.mark.parametrize('metric, func', PAIRED_DISTANCES.items())
-def test_paired_distances(metric, func):
+@pytest.mark.parametrize("metric, func", PAIRED_DISTANCES.items())
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_paired_distances(metric, func, csr_container):
     # Test the pairwise_distance helper function.
     rng = np.random.RandomState(0)
     # Euclidean distance should be equivalent to calling the function.
@@ -403,29 +422,29 @@ def test_paired_distances(metric, func):
 
     S = paired_distances(X, Y, metric=metric)
     S2 = func(X, Y)
-    assert_array_almost_equal(S, S2)
-    S3 = func(csr_matrix(X), csr_matrix(Y))
-    assert_array_almost_equal(S, S3)
+    assert_allclose(S, S2)
+    S3 = func(csr_container(X), csr_container(Y))
+    assert_allclose(S, S3)
     if metric in PAIRWISE_DISTANCE_FUNCTIONS:
         # Check the pairwise_distances implementation
         # gives the same value
         distances = PAIRWISE_DISTANCE_FUNCTIONS[metric](X, Y)
         distances = np.diag(distances)
-        assert_array_almost_equal(distances, S)
+        assert_allclose(distances, S)
 
 
-def test_paired_distances_callable():
-    # Test the pairwise_distance helper function
+def test_paired_distances_callable(global_dtype):
+    # Test the paired_distance helper function
     # with the callable implementation
     rng = np.random.RandomState(0)
     # Euclidean distance should be equivalent to calling the function.
-    X = rng.random_sample((5, 4))
+    X = rng.random_sample((5, 4)).astype(global_dtype, copy=False)
     # Euclidean distance, with Y != X.
-    Y = rng.random_sample((5, 4))
+    Y = rng.random_sample((5, 4)).astype(global_dtype, copy=False)
 
-    S = paired_distances(X, Y, metric='manhattan')
+    S = paired_distances(X, Y, metric="manhattan")
     S2 = paired_distances(X, Y, metric=lambda x, y: np.abs(x - y).sum(axis=0))
-    assert_array_almost_equal(S, S2)
+    assert_allclose(S, S2)
 
     # Test that a value error is raised when the lengths of X and Y should not
     # differ
@@ -434,13 +453,15 @@ def test_paired_distances_callable():
         paired_distances(X, Y)
 
 
-def test_pairwise_distances_argmin_min():
+@pytest.mark.parametrize("dok_container", DOK_CONTAINERS)
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_pairwise_distances_argmin_min(dok_container, csr_container, global_dtype):
     # Check pairwise minimum distances computation for any metric
-    X = [[0], [1]]
-    Y = [[-2], [3]]
+    X = np.asarray([[0], [1]], dtype=global_dtype)
+    Y = np.asarray([[-2], [3]], dtype=global_dtype)
 
-    Xsp = dok_matrix(X)
-    Ysp = csr_matrix(Y, dtype=np.float32)
+    Xsp = dok_container(X)
+    Ysp = csr_container(Y, dtype=global_dtype)
 
     expected_idx = [0, 1]
     expected_vals = [2, 2]
@@ -449,45 +470,63 @@ def test_pairwise_distances_argmin_min():
     # euclidean metric
     idx, vals = pairwise_distances_argmin_min(X, Y, metric="euclidean")
     idx2 = pairwise_distances_argmin(X, Y, metric="euclidean")
-    assert_array_almost_equal(idx, expected_idx)
-    assert_array_almost_equal(idx2, expected_idx)
-    assert_array_almost_equal(vals, expected_vals)
+    assert_allclose(idx, expected_idx)
+    assert_allclose(idx2, expected_idx)
+    assert_allclose(vals, expected_vals)
     # sparse matrix case
     idxsp, valssp = pairwise_distances_argmin_min(Xsp, Ysp, metric="euclidean")
-    assert_array_almost_equal(idxsp, expected_idx)
-    assert_array_almost_equal(valssp, expected_vals)
+    idxsp2 = pairwise_distances_argmin(Xsp, Ysp, metric="euclidean")
+    assert_allclose(idxsp, expected_idx)
+    assert_allclose(idxsp2, expected_idx)
+    assert_allclose(valssp, expected_vals)
     # We don't want np.matrix here
     assert type(idxsp) == np.ndarray
     assert type(valssp) == np.ndarray
 
-    # euclidean metric squared
-    idx, vals = pairwise_distances_argmin_min(X, Y, metric="euclidean",
-                                              metric_kwargs={"squared": True})
-    assert_array_almost_equal(idx, expected_idx)
-    assert_array_almost_equal(vals, expected_vals_sq)
+    # Squared Euclidean metric
+    idx, vals = pairwise_distances_argmin_min(X, Y, metric="sqeuclidean")
+    idx2, vals2 = pairwise_distances_argmin_min(
+        X, Y, metric="euclidean", metric_kwargs={"squared": True}
+    )
+    idx3 = pairwise_distances_argmin(X, Y, metric="sqeuclidean")
+    idx4 = pairwise_distances_argmin(
+        X, Y, metric="euclidean", metric_kwargs={"squared": True}
+    )
+
+    assert_allclose(vals, expected_vals_sq)
+    assert_allclose(vals2, expected_vals_sq)
+
+    assert_allclose(idx, expected_idx)
+    assert_allclose(idx2, expected_idx)
+    assert_allclose(idx3, expected_idx)
+    assert_allclose(idx4, expected_idx)
 
     # Non-euclidean scikit-learn metric
     idx, vals = pairwise_distances_argmin_min(X, Y, metric="manhattan")
     idx2 = pairwise_distances_argmin(X, Y, metric="manhattan")
-    assert_array_almost_equal(idx, expected_idx)
-    assert_array_almost_equal(idx2, expected_idx)
-    assert_array_almost_equal(vals, expected_vals)
+    assert_allclose(idx, expected_idx)
+    assert_allclose(idx2, expected_idx)
+    assert_allclose(vals, expected_vals)
     # sparse matrix case
     idxsp, valssp = pairwise_distances_argmin_min(Xsp, Ysp, metric="manhattan")
-    assert_array_almost_equal(idxsp, expected_idx)
-    assert_array_almost_equal(valssp, expected_vals)
+    idxsp2 = pairwise_distances_argmin(Xsp, Ysp, metric="manhattan")
+    assert_allclose(idxsp, expected_idx)
+    assert_allclose(idxsp2, expected_idx)
+    assert_allclose(valssp, expected_vals)
 
     # Non-euclidean Scipy distance (callable)
-    idx, vals = pairwise_distances_argmin_min(X, Y, metric=minkowski,
-                                              metric_kwargs={"p": 2})
-    assert_array_almost_equal(idx, expected_idx)
-    assert_array_almost_equal(vals, expected_vals)
+    idx, vals = pairwise_distances_argmin_min(
+        X, Y, metric=minkowski, metric_kwargs={"p": 2}
+    )
+    assert_allclose(idx, expected_idx)
+    assert_allclose(vals, expected_vals)
 
     # Non-euclidean Scipy distance (string)
-    idx, vals = pairwise_distances_argmin_min(X, Y, metric="minkowski",
-                                              metric_kwargs={"p": 2})
-    assert_array_almost_equal(idx, expected_idx)
-    assert_array_almost_equal(vals, expected_vals)
+    idx, vals = pairwise_distances_argmin_min(
+        X, Y, metric="minkowski", metric_kwargs={"p": 2}
+    )
+    assert_allclose(idx, expected_idx)
+    assert_allclose(vals, expected_vals)
 
     # Compare with naive implementation
     rng = np.random.RandomState(0)
@@ -499,182 +538,247 @@ def test_pairwise_distances_argmin_min():
     dist_orig_val = dist[dist_orig_ind, range(len(dist_orig_ind))]
 
     dist_chunked_ind, dist_chunked_val = pairwise_distances_argmin_min(
-        X, Y, axis=0, metric="manhattan")
-    np.testing.assert_almost_equal(dist_orig_ind, dist_chunked_ind, decimal=7)
-    np.testing.assert_almost_equal(dist_orig_val, dist_chunked_val, decimal=7)
+        X, Y, axis=0, metric="manhattan"
+    )
+    assert_allclose(dist_orig_ind, dist_chunked_ind, rtol=1e-7)
+    assert_allclose(dist_orig_val, dist_chunked_val, rtol=1e-7)
+
+    # Changing the axis and permuting datasets must give the same results
+    argmin_0, dist_0 = pairwise_distances_argmin_min(X, Y, axis=0)
+    argmin_1, dist_1 = pairwise_distances_argmin_min(Y, X, axis=1)
+
+    assert_allclose(dist_0, dist_1)
+    assert_array_equal(argmin_0, argmin_1)
+
+    argmin_0, dist_0 = pairwise_distances_argmin_min(X, X, axis=0)
+    argmin_1, dist_1 = pairwise_distances_argmin_min(X, X, axis=1)
+
+    assert_allclose(dist_0, dist_1)
+    assert_array_equal(argmin_0, argmin_1)
+
+    # Changing the axis and permuting datasets must give the same results
+    argmin_0 = pairwise_distances_argmin(X, Y, axis=0)
+    argmin_1 = pairwise_distances_argmin(Y, X, axis=1)
+
+    assert_array_equal(argmin_0, argmin_1)
+
+    argmin_0 = pairwise_distances_argmin(X, X, axis=0)
+    argmin_1 = pairwise_distances_argmin(X, X, axis=1)
+
+    assert_array_equal(argmin_0, argmin_1)
+
+    # F-contiguous arrays must be supported and must return identical results.
+    argmin_C_contiguous = pairwise_distances_argmin(X, Y)
+    argmin_F_contiguous = pairwise_distances_argmin(
+        np.asfortranarray(X), np.asfortranarray(Y)
+    )
+
+    assert_array_equal(argmin_C_contiguous, argmin_F_contiguous)
 
 
 def _reduce_func(dist, start):
     return dist[:, :100]
 
 
-def test_pairwise_distances_chunked_reduce():
+def test_pairwise_distances_chunked_reduce(global_dtype):
     rng = np.random.RandomState(0)
-    X = rng.random_sample((400, 4))
+    X = rng.random_sample((400, 4)).astype(global_dtype, copy=False)
     # Reduced Euclidean distance
     S = pairwise_distances(X)[:, :100]
-    S_chunks = pairwise_distances_chunked(X, None, reduce_func=_reduce_func,
-                                          working_memory=2 ** -16)
+    S_chunks = pairwise_distances_chunked(
+        X, None, reduce_func=_reduce_func, working_memory=2**-16
+    )
     assert isinstance(S_chunks, GeneratorType)
     S_chunks = list(S_chunks)
     assert len(S_chunks) > 1
+    assert S_chunks[0].dtype == X.dtype
+
     # atol is for diagonal where S is explicitly zeroed on the diagonal
     assert_allclose(np.vstack(S_chunks), S, atol=1e-7)
 
 
-def test_pairwise_distances_chunked_reduce_none():
+def test_pairwise_distances_chunked_reduce_none(global_dtype):
     # check that the reduce func is allowed to return None
     rng = np.random.RandomState(0)
-    X = rng.random_sample((10, 4))
-    S_chunks = pairwise_distances_chunked(X, None,
-                                          reduce_func=lambda dist, start: None,
-                                          working_memory=2 ** -16)
+    X = rng.random_sample((10, 4)).astype(global_dtype, copy=False)
+    S_chunks = pairwise_distances_chunked(
+        X, None, reduce_func=lambda dist, start: None, working_memory=2**-16
+    )
     assert isinstance(S_chunks, GeneratorType)
     S_chunks = list(S_chunks)
     assert len(S_chunks) > 1
     assert all(chunk is None for chunk in S_chunks)
 
 
-@pytest.mark.parametrize('good_reduce', [
-    lambda D, start: list(D),
-    lambda D, start: np.array(D),
-    lambda D, start: csr_matrix(D),
-    lambda D, start: (list(D), list(D)),
-    lambda D, start: (dok_matrix(D), np.array(D), list(D)),
-    ])
+@pytest.mark.parametrize(
+    "good_reduce",
+    [
+        lambda D, start: list(D),
+        lambda D, start: np.array(D),
+        lambda D, start: (list(D), list(D)),
+    ]
+    + [
+        lambda D, start, scipy_csr_type=scipy_csr_type: scipy_csr_type(D)
+        for scipy_csr_type in CSR_CONTAINERS
+    ]
+    + [
+        lambda D, start, scipy_dok_type=scipy_dok_type: (
+            scipy_dok_type(D),
+            np.array(D),
+            list(D),
+        )
+        for scipy_dok_type in DOK_CONTAINERS
+    ],
+)
 def test_pairwise_distances_chunked_reduce_valid(good_reduce):
     X = np.arange(10).reshape(-1, 1)
-    S_chunks = pairwise_distances_chunked(X, None, reduce_func=good_reduce,
-                                          working_memory=64)
+    S_chunks = pairwise_distances_chunked(
+        X, None, reduce_func=good_reduce, working_memory=64
+    )
     next(S_chunks)
 
 
-@pytest.mark.parametrize(('bad_reduce', 'err_type', 'message'), [
-    (lambda D, s: np.concatenate([D, D[-1:]]), ValueError,
-     r'length 11\..* input: 10\.'),
-    (lambda D, s: (D, np.concatenate([D, D[-1:]])), ValueError,
-     r'length \(10, 11\)\..* input: 10\.'),
-    (lambda D, s: (D[:9], D), ValueError,
-     r'length \(9, 10\)\..* input: 10\.'),
-    (lambda D, s: 7, TypeError,
-     r'returned 7\. Expected sequence\(s\) of length 10\.'),
-    (lambda D, s: (7, 8), TypeError,
-     r'returned \(7, 8\)\. Expected sequence\(s\) of length 10\.'),
-    (lambda D, s: (np.arange(10), 9), TypeError,
-     r', 9\)\. Expected sequence\(s\) of length 10\.'),
-])
-def test_pairwise_distances_chunked_reduce_invalid(bad_reduce, err_type,
-                                                   message):
-    X = np.arange(10).reshape(-1, 1)
-    S_chunks = pairwise_distances_chunked(X, None, reduce_func=bad_reduce,
-                                          working_memory=64)
+@pytest.mark.parametrize(
+    ("bad_reduce", "err_type", "message"),
+    [
+        (
+            lambda D, s: np.concatenate([D, D[-1:]]),
+            ValueError,
+            r"length 11\..* input: 10\.",
+        ),
+        (
+            lambda D, s: (D, np.concatenate([D, D[-1:]])),
+            ValueError,
+            r"length \(10, 11\)\..* input: 10\.",
+        ),
+        (lambda D, s: (D[:9], D), ValueError, r"length \(9, 10\)\..* input: 10\."),
+        (
+            lambda D, s: 7,
+            TypeError,
+            r"returned 7\. Expected sequence\(s\) of length 10\.",
+        ),
+        (
+            lambda D, s: (7, 8),
+            TypeError,
+            r"returned \(7, 8\)\. Expected sequence\(s\) of length 10\.",
+        ),
+        (
+            lambda D, s: (np.arange(10), 9),
+            TypeError,
+            r", 9\)\. Expected sequence\(s\) of length 10\.",
+        ),
+    ],
+)
+def test_pairwise_distances_chunked_reduce_invalid(
+    global_dtype, bad_reduce, err_type, message
+):
+    X = np.arange(10).reshape(-1, 1).astype(global_dtype, copy=False)
+    S_chunks = pairwise_distances_chunked(
+        X, None, reduce_func=bad_reduce, working_memory=64
+    )
     with pytest.raises(err_type, match=message):
         next(S_chunks)
 
 
-def check_pairwise_distances_chunked(X, Y, working_memory, metric='euclidean'):
-    gen = pairwise_distances_chunked(X, Y, working_memory=working_memory,
-                                     metric=metric)
+def check_pairwise_distances_chunked(X, Y, working_memory, metric="euclidean"):
+    gen = pairwise_distances_chunked(X, Y, working_memory=working_memory, metric=metric)
     assert isinstance(gen, GeneratorType)
     blockwise_distances = list(gen)
     Y = X if Y is None else Y
-    min_block_mib = len(Y) * 8 * 2 ** -20
+    min_block_mib = len(Y) * 8 * 2**-20
 
     for block in blockwise_distances:
         memory_used = block.nbytes
-        assert memory_used <= max(working_memory, min_block_mib) * 2 ** 20
+        assert memory_used <= max(working_memory, min_block_mib) * 2**20
 
     blockwise_distances = np.vstack(blockwise_distances)
     S = pairwise_distances(X, Y, metric=metric)
-    assert_array_almost_equal(blockwise_distances, S)
+    assert_allclose(blockwise_distances, S, atol=1e-7)
 
 
-@pytest.mark.parametrize(
-        'metric',
-        ('euclidean', 'l2', 'sqeuclidean'))
-def test_pairwise_distances_chunked_diagonal(metric):
+@pytest.mark.parametrize("metric", ("euclidean", "l2", "sqeuclidean"))
+def test_pairwise_distances_chunked_diagonal(metric, global_dtype):
     rng = np.random.RandomState(0)
-    X = rng.normal(size=(1000, 10), scale=1e10)
-    chunks = list(pairwise_distances_chunked(X, working_memory=1,
-                                             metric=metric))
+    X = rng.normal(size=(1000, 10), scale=1e10).astype(global_dtype, copy=False)
+    chunks = list(pairwise_distances_chunked(X, working_memory=1, metric=metric))
     assert len(chunks) > 1
-    assert_array_almost_equal(np.diag(np.vstack(chunks)), 0, decimal=10)
+    assert_allclose(np.diag(np.vstack(chunks)), 0, rtol=1e-10)
 
 
-@pytest.mark.parametrize(
-        'metric',
-        ('euclidean', 'l2', 'sqeuclidean'))
-def test_parallel_pairwise_distances_diagonal(metric):
+@pytest.mark.parametrize("metric", ("euclidean", "l2", "sqeuclidean"))
+def test_parallel_pairwise_distances_diagonal(metric, global_dtype):
     rng = np.random.RandomState(0)
-    X = rng.normal(size=(1000, 10), scale=1e10)
+    X = rng.normal(size=(1000, 10), scale=1e10).astype(global_dtype, copy=False)
     distances = pairwise_distances(X, metric=metric, n_jobs=2)
     assert_allclose(np.diag(distances), 0, atol=1e-10)
 
 
-@ignore_warnings
-def test_pairwise_distances_chunked():
+@pytest.mark.filterwarnings("ignore:Could not adhere to working_memory config")
+def test_pairwise_distances_chunked(global_dtype):
     # Test the pairwise_distance helper function.
     rng = np.random.RandomState(0)
     # Euclidean distance should be equivalent to calling the function.
-    X = rng.random_sample((200, 4))
-    check_pairwise_distances_chunked(X, None, working_memory=1,
-                                     metric='euclidean')
+    X = rng.random_sample((200, 4)).astype(global_dtype, copy=False)
+    check_pairwise_distances_chunked(X, None, working_memory=1, metric="euclidean")
     # Test small amounts of memory
     for power in range(-16, 0):
-        check_pairwise_distances_chunked(X, None, working_memory=2 ** power,
-                                         metric='euclidean')
+        check_pairwise_distances_chunked(
+            X, None, working_memory=2**power, metric="euclidean"
+        )
     # X as list
-    check_pairwise_distances_chunked(X.tolist(), None, working_memory=1,
-                                     metric='euclidean')
+    check_pairwise_distances_chunked(
+        X.tolist(), None, working_memory=1, metric="euclidean"
+    )
     # Euclidean distance, with Y != X.
-    Y = rng.random_sample((100, 4))
-    check_pairwise_distances_chunked(X, Y, working_memory=1,
-                                     metric='euclidean')
-    check_pairwise_distances_chunked(X.tolist(), Y.tolist(), working_memory=1,
-                                     metric='euclidean')
+    Y = rng.random_sample((100, 4)).astype(global_dtype, copy=False)
+    check_pairwise_distances_chunked(X, Y, working_memory=1, metric="euclidean")
+    check_pairwise_distances_chunked(
+        X.tolist(), Y.tolist(), working_memory=1, metric="euclidean"
+    )
     # absurdly large working_memory
-    check_pairwise_distances_chunked(X, Y, working_memory=10000,
-                                     metric='euclidean')
+    check_pairwise_distances_chunked(X, Y, working_memory=10000, metric="euclidean")
     # "cityblock" uses scikit-learn metric, cityblock (function) is
     # scipy.spatial.
-    check_pairwise_distances_chunked(X, Y, working_memory=1,
-                                     metric='cityblock')
-    # Test that a value error is raised if the metric is unknown
-    with pytest.raises(ValueError):
-        next(pairwise_distances_chunked(X, Y, metric="blah"))
+    check_pairwise_distances_chunked(X, Y, working_memory=1, metric="cityblock")
 
     # Test precomputed returns all at once
     D = pairwise_distances(X)
-    gen = pairwise_distances_chunked(D,
-                                     working_memory=2 ** -16,
-                                     metric='precomputed')
+    gen = pairwise_distances_chunked(D, working_memory=2**-16, metric="precomputed")
     assert isinstance(gen, GeneratorType)
     assert next(gen) is D
     with pytest.raises(StopIteration):
         next(gen)
 
 
-@pytest.mark.parametrize("x_array_constr", [np.array, csr_matrix],
-                         ids=["dense", "sparse"])
-@pytest.mark.parametrize("y_array_constr", [np.array, csr_matrix],
-                         ids=["dense", "sparse"])
+@pytest.mark.parametrize(
+    "x_array_constr",
+    [np.array] + CSR_CONTAINERS,
+    ids=["dense"] + [container.__name__ for container in CSR_CONTAINERS],
+)
+@pytest.mark.parametrize(
+    "y_array_constr",
+    [np.array] + CSR_CONTAINERS,
+    ids=["dense"] + [container.__name__ for container in CSR_CONTAINERS],
+)
 def test_euclidean_distances_known_result(x_array_constr, y_array_constr):
     # Check the pairwise Euclidean distances computation on known result
     X = x_array_constr([[0]])
     Y = y_array_constr([[1], [2]])
     D = euclidean_distances(X, Y)
-    assert_allclose(D, [[1., 2.]])
+    assert_allclose(D, [[1.0, 2.0]])
 
 
-@pytest.mark.parametrize("dtype", [np.float32, np.float64])
-@pytest.mark.parametrize("y_array_constr", [np.array, csr_matrix],
-                         ids=["dense", "sparse"])
-def test_euclidean_distances_with_norms(dtype, y_array_constr):
+@pytest.mark.parametrize(
+    "y_array_constr",
+    [np.array] + CSR_CONTAINERS,
+    ids=["dense"] + [container.__name__ for container in CSR_CONTAINERS],
+)
+def test_euclidean_distances_with_norms(global_dtype, y_array_constr):
     # check that we still get the right answers with {X,Y}_norm_squared
     # and that we get a wrong answer with wrong {X,Y}_norm_squared
     rng = np.random.RandomState(0)
-    X = rng.random_sample((10, 10)).astype(dtype, copy=False)
-    Y = rng.random_sample((20, 10)).astype(dtype, copy=False)
+    X = rng.random_sample((10, 10)).astype(global_dtype, copy=False)
+    Y = rng.random_sample((20, 10)).astype(global_dtype, copy=False)
 
     # norms will only be used if their dtype is float64
     X_norm_sq = (X.astype(np.float64) ** 2).sum(axis=1).reshape(1, -1)
@@ -685,38 +789,63 @@ def test_euclidean_distances_with_norms(dtype, y_array_constr):
     D1 = euclidean_distances(X, Y)
     D2 = euclidean_distances(X, Y, X_norm_squared=X_norm_sq)
     D3 = euclidean_distances(X, Y, Y_norm_squared=Y_norm_sq)
-    D4 = euclidean_distances(X, Y, X_norm_squared=X_norm_sq,
-                             Y_norm_squared=Y_norm_sq)
+    D4 = euclidean_distances(X, Y, X_norm_squared=X_norm_sq, Y_norm_squared=Y_norm_sq)
     assert_allclose(D2, D1)
     assert_allclose(D3, D1)
     assert_allclose(D4, D1)
 
     # check we get the wrong answer with wrong {X,Y}_norm_squared
-    wrong_D = euclidean_distances(X, Y,
-                                  X_norm_squared=np.zeros_like(X_norm_sq),
-                                  Y_norm_squared=np.zeros_like(Y_norm_sq))
+    wrong_D = euclidean_distances(
+        X,
+        Y,
+        X_norm_squared=np.zeros_like(X_norm_sq),
+        Y_norm_squared=np.zeros_like(Y_norm_sq),
+    )
     with pytest.raises(AssertionError):
         assert_allclose(wrong_D, D1)
 
 
+@pytest.mark.parametrize("symmetric", [True, False])
+def test_euclidean_distances_float32_norms(global_random_seed, symmetric):
+    # Non-regression test for #27621
+    rng = np.random.RandomState(global_random_seed)
+    X = rng.random_sample((10, 10))
+    Y = X if symmetric else rng.random_sample((20, 10))
+    X_norm_sq = (X.astype(np.float32) ** 2).sum(axis=1).reshape(1, -1)
+    Y_norm_sq = (Y.astype(np.float32) ** 2).sum(axis=1).reshape(1, -1)
+    D1 = euclidean_distances(X, Y)
+    D2 = euclidean_distances(X, Y, X_norm_squared=X_norm_sq)
+    D3 = euclidean_distances(X, Y, Y_norm_squared=Y_norm_sq)
+    D4 = euclidean_distances(X, Y, X_norm_squared=X_norm_sq, Y_norm_squared=Y_norm_sq)
+    assert_allclose(D2, D1)
+    assert_allclose(D3, D1)
+    assert_allclose(D4, D1)
+
+
 def test_euclidean_distances_norm_shapes():
     # Check all accepted shapes for the norms or appropriate error messages.
     rng = np.random.RandomState(0)
     X = rng.random_sample((10, 10))
     Y = rng.random_sample((20, 10))
 
-    X_norm_squared = (X ** 2).sum(axis=1)
-    Y_norm_squared = (Y ** 2).sum(axis=1)
-
-    D1 = euclidean_distances(X, Y,
-                             X_norm_squared=X_norm_squared,
-                             Y_norm_squared=Y_norm_squared)
-    D2 = euclidean_distances(X, Y,
-                             X_norm_squared=X_norm_squared.reshape(-1, 1),
-                             Y_norm_squared=Y_norm_squared.reshape(-1, 1))
-    D3 = euclidean_distances(X, Y,
-                             X_norm_squared=X_norm_squared.reshape(1, -1),
-                             Y_norm_squared=Y_norm_squared.reshape(1, -1))
+    X_norm_squared = (X**2).sum(axis=1)
+    Y_norm_squared = (Y**2).sum(axis=1)
+
+    D1 = euclidean_distances(
+        X, Y, X_norm_squared=X_norm_squared, Y_norm_squared=Y_norm_squared
+    )
+    D2 = euclidean_distances(
+        X,
+        Y,
+        X_norm_squared=X_norm_squared.reshape(-1, 1),
+        Y_norm_squared=Y_norm_squared.reshape(-1, 1),
+    )
+    D3 = euclidean_distances(
+        X,
+        Y,
+        X_norm_squared=X_norm_squared.reshape(1, -1),
+        Y_norm_squared=Y_norm_squared.reshape(1, -1),
+    )
 
     assert_allclose(D2, D1)
     assert_allclose(D3, D1)
@@ -727,18 +856,23 @@ def test_euclidean_distances_norm_shapes():
         euclidean_distances(X, Y, Y_norm_squared=Y_norm_squared[:5])
 
 
-@pytest.mark.parametrize("dtype", [np.float32, np.float64])
-@pytest.mark.parametrize("x_array_constr", [np.array, csr_matrix],
-                         ids=["dense", "sparse"])
-@pytest.mark.parametrize("y_array_constr", [np.array, csr_matrix],
-                         ids=["dense", "sparse"])
-def test_euclidean_distances(dtype, x_array_constr, y_array_constr):
+@pytest.mark.parametrize(
+    "x_array_constr",
+    [np.array] + CSR_CONTAINERS,
+    ids=["dense"] + [container.__name__ for container in CSR_CONTAINERS],
+)
+@pytest.mark.parametrize(
+    "y_array_constr",
+    [np.array] + CSR_CONTAINERS,
+    ids=["dense"] + [container.__name__ for container in CSR_CONTAINERS],
+)
+def test_euclidean_distances(global_dtype, x_array_constr, y_array_constr):
     # check that euclidean distances gives same result as scipy cdist
     # when X and Y != X are provided
     rng = np.random.RandomState(0)
-    X = rng.random_sample((100, 10)).astype(dtype, copy=False)
+    X = rng.random_sample((100, 10)).astype(global_dtype, copy=False)
     X[X < 0.8] = 0
-    Y = rng.random_sample((10, 10)).astype(dtype, copy=False)
+    Y = rng.random_sample((10, 10)).astype(global_dtype, copy=False)
     Y[Y < 0.8] = 0
 
     expected = cdist(X, Y)
@@ -750,17 +884,19 @@ def test_euclidean_distances(dtype, x_array_constr, y_array_constr):
     # the default rtol=1e-7 is too close to the float32 precision
     # and fails due to rounding errors.
     assert_allclose(distances, expected, rtol=1e-6)
-    assert distances.dtype == dtype
+    assert distances.dtype == global_dtype
 
 
-@pytest.mark.parametrize("dtype", [np.float32, np.float64])
-@pytest.mark.parametrize("x_array_constr", [np.array, csr_matrix],
-                         ids=["dense", "sparse"])
-def test_euclidean_distances_sym(dtype, x_array_constr):
+@pytest.mark.parametrize(
+    "x_array_constr",
+    [np.array] + CSR_CONTAINERS,
+    ids=["dense"] + [container.__name__ for container in CSR_CONTAINERS],
+)
+def test_euclidean_distances_sym(global_dtype, x_array_constr):
     # check that euclidean distances gives same result as scipy pdist
     # when only X is provided
     rng = np.random.RandomState(0)
-    X = rng.random_sample((100, 10)).astype(dtype, copy=False)
+    X = rng.random_sample((100, 10)).astype(global_dtype, copy=False)
     X[X < 0.8] = 0
 
     expected = squareform(pdist(X))
@@ -771,16 +907,21 @@ def test_euclidean_distances_sym(dtype, x_array_constr):
     # the default rtol=1e-7 is too close to the float32 precision
     # and fails due to rounding errors.
     assert_allclose(distances, expected, rtol=1e-6)
-    assert distances.dtype == dtype
+    assert distances.dtype == global_dtype
 
 
 @pytest.mark.parametrize("batch_size", [None, 5, 7, 101])
-@pytest.mark.parametrize("x_array_constr", [np.array, csr_matrix],
-                         ids=["dense", "sparse"])
-@pytest.mark.parametrize("y_array_constr", [np.array, csr_matrix],
-                         ids=["dense", "sparse"])
-def test_euclidean_distances_upcast(batch_size, x_array_constr,
-                                    y_array_constr):
+@pytest.mark.parametrize(
+    "x_array_constr",
+    [np.array] + CSR_CONTAINERS,
+    ids=["dense"] + [container.__name__ for container in CSR_CONTAINERS],
+)
+@pytest.mark.parametrize(
+    "y_array_constr",
+    [np.array] + CSR_CONTAINERS,
+    ids=["dense"] + [container.__name__ for container in CSR_CONTAINERS],
+)
+def test_euclidean_distances_upcast(batch_size, x_array_constr, y_array_constr):
     # check batches handling when Y != X (#13910)
     rng = np.random.RandomState(0)
     X = rng.random_sample((100, 10)).astype(np.float32)
@@ -801,8 +942,11 @@ def test_euclidean_distances_upcast(batch_size, x_array_constr,
 
 
 @pytest.mark.parametrize("batch_size", [None, 5, 7, 101])
-@pytest.mark.parametrize("x_array_constr", [np.array, csr_matrix],
-                         ids=["dense", "sparse"])
+@pytest.mark.parametrize(
+    "x_array_constr",
+    [np.array] + CSR_CONTAINERS,
+    ids=["dense"] + [container.__name__ for container in CSR_CONTAINERS],
+)
 def test_euclidean_distances_upcast_sym(batch_size, x_array_constr):
     # check batches handling when X is Y (#13910)
     rng = np.random.RandomState(0)
@@ -822,16 +966,22 @@ def test_euclidean_distances_upcast_sym(batch_size, x_array_constr):
 
 @pytest.mark.parametrize(
     "dtype, eps, rtol",
-    [(np.float32, 1e-4, 1e-5),
-     pytest.param(
-         np.float64, 1e-8, 0.99,
-         marks=pytest.mark.xfail(reason='failing due to lack of precision'))])
+    [
+        (np.float32, 1e-4, 1e-5),
+        pytest.param(
+            np.float64,
+            1e-8,
+            0.99,
+            marks=pytest.mark.xfail(reason="failing due to lack of precision"),
+        ),
+    ],
+)
 @pytest.mark.parametrize("dim", [1, 1000000])
 def test_euclidean_distances_extreme_values(dtype, eps, rtol, dim):
     # check that euclidean distances is correct with float32 input thanks to
     # upcasting. On float64 there are still precision issues.
-    X = np.array([[1.] * dim], dtype=dtype)
-    Y = np.array([[1. + eps] * dim], dtype=dtype)
+    X = np.array([[1.0] * dim], dtype=dtype)
+    Y = np.array([[1.0 + eps] * dim], dtype=dtype)
 
     distances = euclidean_distances(X, Y)
     expected = cdist(X, Y)
@@ -851,48 +1001,44 @@ def test_nan_euclidean_distances_equal_to_euclidean_distance(squared):
     assert_allclose(normal_distance, nan_distance)
 
 
-@pytest.mark.parametrize(
-    "X", [np.array([[np.inf, 0]]), np.array([[0, -np.inf]])])
-@pytest.mark.parametrize(
-    "Y", [np.array([[np.inf, 0]]), np.array([[0, -np.inf]]), None])
+@pytest.mark.parametrize("X", [np.array([[np.inf, 0]]), np.array([[0, -np.inf]])])
+@pytest.mark.parametrize("Y", [np.array([[np.inf, 0]]), np.array([[0, -np.inf]]), None])
 def test_nan_euclidean_distances_infinite_values(X, Y):
-
     with pytest.raises(ValueError) as excinfo:
         nan_euclidean_distances(X, Y=Y)
 
-    exp_msg = ("Input contains infinity or a value too large for "
-               "dtype('float64').")
+    exp_msg = "Input contains infinity or a value too large for dtype('float64')."
     assert exp_msg == str(excinfo.value)
 
 
-@pytest.mark.parametrize("X, X_diag, missing_value", [
-    (np.array([[0, 1], [1, 0]]), np.sqrt(2), np.nan),
-    (np.array([[0, 1], [1, np.nan]]), np.sqrt(2), np.nan),
-    (np.array([[np.nan, 1], [1, np.nan]]), np.nan, np.nan),
-    (np.array([[np.nan, 1], [np.nan, 0]]), np.sqrt(2), np.nan),
-    (np.array([[0, np.nan], [1, np.nan]]), np.sqrt(2), np.nan),
-    (np.array([[0, 1], [1, 0]]), np.sqrt(2), -1),
-    (np.array([[0, 1], [1, -1]]), np.sqrt(2), -1),
-    (np.array([[-1, 1], [1, -1]]), np.nan, -1),
-    (np.array([[-1, 1], [-1, 0]]), np.sqrt(2), -1),
-    (np.array([[0, -1], [1, -1]]), np.sqrt(2), -1)
-])
+@pytest.mark.parametrize(
+    "X, X_diag, missing_value",
+    [
+        (np.array([[0, 1], [1, 0]]), np.sqrt(2), np.nan),
+        (np.array([[0, 1], [1, np.nan]]), np.sqrt(2), np.nan),
+        (np.array([[np.nan, 1], [1, np.nan]]), np.nan, np.nan),
+        (np.array([[np.nan, 1], [np.nan, 0]]), np.sqrt(2), np.nan),
+        (np.array([[0, np.nan], [1, np.nan]]), np.sqrt(2), np.nan),
+        (np.array([[0, 1], [1, 0]]), np.sqrt(2), -1),
+        (np.array([[0, 1], [1, -1]]), np.sqrt(2), -1),
+        (np.array([[-1, 1], [1, -1]]), np.nan, -1),
+        (np.array([[-1, 1], [-1, 0]]), np.sqrt(2), -1),
+        (np.array([[0, -1], [1, -1]]), np.sqrt(2), -1),
+    ],
+)
 def test_nan_euclidean_distances_2x2(X, X_diag, missing_value):
-
-    exp_dist = np.array([[0., X_diag], [X_diag, 0]])
+    exp_dist = np.array([[0.0, X_diag], [X_diag, 0]])
 
     dist = nan_euclidean_distances(X, missing_values=missing_value)
     assert_allclose(exp_dist, dist)
 
-    dist_sq = nan_euclidean_distances(
-        X, squared=True, missing_values=missing_value)
+    dist_sq = nan_euclidean_distances(X, squared=True, missing_values=missing_value)
     assert_allclose(exp_dist**2, dist_sq)
 
     dist_two = nan_euclidean_distances(X, X, missing_values=missing_value)
     assert_allclose(exp_dist, dist_two)
 
-    dist_two_copy = nan_euclidean_distances(
-        X, X.copy(), missing_values=missing_value)
+    dist_two_copy = nan_euclidean_distances(X, X.copy(), missing_values=missing_value)
     assert_allclose(exp_dist, dist_two_copy)
 
 
@@ -905,23 +1051,30 @@ def test_nan_euclidean_distances_complete_nan(missing_value):
     dist = nan_euclidean_distances(X, missing_values=missing_value)
     assert_allclose(exp_dist, dist)
 
-    dist = nan_euclidean_distances(
-            X, X.copy(), missing_values=missing_value)
+    dist = nan_euclidean_distances(X, X.copy(), missing_values=missing_value)
     assert_allclose(exp_dist, dist)
 
 
 @pytest.mark.parametrize("missing_value", [np.nan, -1])
 def test_nan_euclidean_distances_not_trival(missing_value):
-    X = np.array([[1., missing_value, 3., 4., 2.],
-                  [missing_value, 4., 6., 1., missing_value],
-                  [3., missing_value, missing_value, missing_value, 1.]])
-
-    Y = np.array([[missing_value, 7., 7., missing_value, 2.],
-                  [missing_value, missing_value, 5., 4., 7.],
-                  [missing_value, missing_value, missing_value, 4., 5.]])
+    X = np.array(
+        [
+            [1.0, missing_value, 3.0, 4.0, 2.0],
+            [missing_value, 4.0, 6.0, 1.0, missing_value],
+            [3.0, missing_value, missing_value, missing_value, 1.0],
+        ]
+    )
+
+    Y = np.array(
+        [
+            [missing_value, 7.0, 7.0, missing_value, 2.0],
+            [missing_value, missing_value, 5.0, 4.0, 7.0],
+            [missing_value, missing_value, missing_value, 4.0, 5.0],
+        ]
+    )
 
     # Check for symmetry
-    D1 = nan_euclidean_distances(X, Y,  missing_values=missing_value)
+    D1 = nan_euclidean_distances(X, Y, missing_values=missing_value)
     D2 = nan_euclidean_distances(Y, X, missing_values=missing_value)
 
     assert_almost_equal(D1, D2.T)
@@ -929,14 +1082,18 @@ def test_nan_euclidean_distances_not_trival(missing_value):
     # Check with explicit formula and squared=True
     assert_allclose(
         nan_euclidean_distances(
-            X[:1], Y[:1], squared=True, missing_values=missing_value),
-        [[5.0 / 2.0 * ((7 - 3)**2 + (2 - 2)**2)]])
+            X[:1], Y[:1], squared=True, missing_values=missing_value
+        ),
+        [[5.0 / 2.0 * ((7 - 3) ** 2 + (2 - 2) ** 2)]],
+    )
 
     # Check with explicit formula and squared=False
     assert_allclose(
         nan_euclidean_distances(
-            X[1:2], Y[1:2], squared=False, missing_values=missing_value),
-        [[np.sqrt(5.0 / 2.0 * ((6 - 5)**2 + (1 - 4)**2))]])
+            X[1:2], Y[1:2], squared=False, missing_values=missing_value
+        ),
+        [[np.sqrt(5.0 / 2.0 * ((6 - 5) ** 2 + (1 - 4) ** 2))]],
+    )
 
     # Check when Y = X is explicitly passed
     D3 = nan_euclidean_distances(X, missing_values=missing_value)
@@ -956,15 +1113,19 @@ def test_nan_euclidean_distances_one_feature_match_positive(missing_value):
     # First feature is the only feature that is non-nan and in both
     # samples. The result of `nan_euclidean_distances` with squared=True
     # should be non-negative. The non-squared version should all be close to 0.
-    X = np.array([[-122.27, 648., missing_value, 37.85],
-                  [-122.27, missing_value, 2.34701493, missing_value]])
-
-    dist_squared = nan_euclidean_distances(X, missing_values=missing_value,
-                                           squared=True)
+    X = np.array(
+        [
+            [-122.27, 648.0, missing_value, 37.85],
+            [-122.27, missing_value, 2.34701493, missing_value],
+        ]
+    )
+
+    dist_squared = nan_euclidean_distances(
+        X, missing_values=missing_value, squared=True
+    )
     assert np.all(dist_squared >= 0)
 
-    dist = nan_euclidean_distances(X, missing_values=missing_value,
-                                   squared=False)
+    dist = nan_euclidean_distances(X, missing_values=missing_value, squared=False)
     assert_allclose(dist, 0.0)
 
 
@@ -974,28 +1135,28 @@ def test_cosine_distances():
     x = np.abs(rng.rand(910))
     XA = np.vstack([x, x])
     D = cosine_distances(XA)
-    assert_array_almost_equal(D, [[0., 0.], [0., 0.]])
+    assert_allclose(D, [[0.0, 0.0], [0.0, 0.0]], atol=1e-10)
     # check that all elements are in [0, 2]
-    assert np.all(D >= 0.)
-    assert np.all(D <= 2.)
+    assert np.all(D >= 0.0)
+    assert np.all(D <= 2.0)
     # check that diagonal elements are equal to 0
-    assert_array_almost_equal(D[np.diag_indices_from(D)], [0., 0.])
+    assert_allclose(D[np.diag_indices_from(D)], [0.0, 0.0])
 
     XB = np.vstack([x, -x])
     D2 = cosine_distances(XB)
     # check that all elements are in [0, 2]
-    assert np.all(D2 >= 0.)
-    assert np.all(D2 <= 2.)
+    assert np.all(D2 >= 0.0)
+    assert np.all(D2 <= 2.0)
     # check that diagonal elements are equal to 0 and non diagonal to 2
-    assert_array_almost_equal(D2, [[0., 2.], [2., 0.]])
+    assert_allclose(D2, [[0.0, 2.0], [2.0, 0.0]])
 
     # check large random matrix
     X = np.abs(rng.rand(1000, 5000))
     D = cosine_distances(X)
     # check that diagonal elements are equal to 0
-    assert_array_almost_equal(D[np.diag_indices_from(D)], [0.] * D.shape[0])
-    assert np.all(D >= 0.)
-    assert np.all(D <= 2.)
+    assert_allclose(D[np.diag_indices_from(D)], [0.0] * D.shape[0])
+    assert np.all(D >= 0.0)
+    assert np.all(D <= 2.0)
 
 
 def test_haversine_distances():
@@ -1004,16 +1165,17 @@ def slow_haversine_distances(x, y):
         diff_lat = y[0] - x[0]
         diff_lon = y[1] - x[1]
         a = np.sin(diff_lat / 2) ** 2 + (
-            np.cos(x[0]) * np.cos(y[0]) * np.sin(diff_lon/2) ** 2
+            np.cos(x[0]) * np.cos(y[0]) * np.sin(diff_lon / 2) ** 2
         )
         c = 2 * np.arcsin(np.sqrt(a))
         return c
+
     rng = np.random.RandomState(0)
     X = rng.random_sample((5, 2))
     Y = rng.random_sample((10, 2))
     D1 = np.array([[slow_haversine_distances(x, y) for y in Y] for x in X])
     D2 = haversine_distances(X, Y)
-    assert_array_almost_equal(D1, D2)
+    assert_allclose(D1, D2)
     # Test haversine distance does not accept X where n_feature != 2
     X = rng.random_sample((10, 3))
     err_msg = "Haversine distance only valid in 2 dimensions"
@@ -1023,12 +1185,13 @@ def slow_haversine_distances(x, y):
 
 # Paired distances
 
+
 def test_paired_euclidean_distances():
     # Check the paired Euclidean distances computation
     X = [[0], [0]]
     Y = [[1], [2]]
     D = paired_euclidean_distances(X, Y)
-    assert_array_almost_equal(D, [1., 2.])
+    assert_allclose(D, [1.0, 2.0])
 
 
 def test_paired_manhattan_distances():
@@ -1036,7 +1199,15 @@ def test_paired_manhattan_distances():
     X = [[0], [0]]
     Y = [[1], [2]]
     D = paired_manhattan_distances(X, Y)
-    assert_array_almost_equal(D, [1., 2.])
+    assert_allclose(D, [1.0, 2.0])
+
+
+def test_paired_cosine_distances():
+    # Check the paired manhattan distances computation
+    X = [[0], [0]]
+    Y = [[1], [2]]
+    D = paired_cosine_distances(X, Y)
+    assert_allclose(D, [0.5, 0.5])
 
 
 def test_chi_square_kernel():
@@ -1074,8 +1245,8 @@ def test_chi_square_kernel():
     assert K.dtype == float
 
     # check that kernel of similar things is greater than dissimilar ones
-    X = [[.3, .7], [1., 0]]
-    Y = [[0, 1], [.9, .1]]
+    X = [[0.3, 0.7], [1.0, 0]]
+    Y = [[0, 1], [0.9, 0.1]]
     K = chi2_kernel(X, Y)
     assert K[0, 0] > K[0, 1]
     assert K[1, 1] > K[1, 0]
@@ -1090,38 +1261,47 @@ def test_chi_square_kernel():
 
     # different n_features in X and Y
     with pytest.raises(ValueError):
-        chi2_kernel([[0, 1]], [[.2, .2, .6]])
-
-    # sparse matrices
-    with pytest.raises(ValueError):
-        chi2_kernel(csr_matrix(X), csr_matrix(Y))
-    with pytest.raises(ValueError):
-        additive_chi2_kernel(csr_matrix(X), csr_matrix(Y))
+        chi2_kernel([[0, 1]], [[0.2, 0.2, 0.6]])
 
 
 @pytest.mark.parametrize(
-        'kernel',
-        (linear_kernel, polynomial_kernel, rbf_kernel,
-         laplacian_kernel, sigmoid_kernel, cosine_similarity))
+    "kernel",
+    (
+        linear_kernel,
+        polynomial_kernel,
+        rbf_kernel,
+        laplacian_kernel,
+        sigmoid_kernel,
+        cosine_similarity,
+    ),
+)
 def test_kernel_symmetry(kernel):
     # Valid kernels should be symmetric
     rng = np.random.RandomState(0)
     X = rng.random_sample((5, 4))
     K = kernel(X, X)
-    assert_array_almost_equal(K, K.T, 15)
+    assert_allclose(K, K.T, 15)
 
 
 @pytest.mark.parametrize(
-        'kernel',
-        (linear_kernel, polynomial_kernel, rbf_kernel,
-         laplacian_kernel, sigmoid_kernel, cosine_similarity))
-def test_kernel_sparse(kernel):
+    "kernel",
+    (
+        linear_kernel,
+        polynomial_kernel,
+        rbf_kernel,
+        laplacian_kernel,
+        sigmoid_kernel,
+        cosine_similarity,
+    ),
+)
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_kernel_sparse(kernel, csr_container):
     rng = np.random.RandomState(0)
     X = rng.random_sample((5, 4))
-    X_sparse = csr_matrix(X)
+    X_sparse = csr_container(X)
     K = kernel(X, X)
     K2 = kernel(X_sparse, X_sparse)
-    assert_array_almost_equal(K, K2)
+    assert_allclose(K, K2)
 
 
 def test_linear_kernel():
@@ -1129,7 +1309,7 @@ def test_linear_kernel():
     X = rng.random_sample((5, 4))
     K = linear_kernel(X, X)
     # the diagonal elements of a linear kernel are their squared norm
-    assert_array_almost_equal(K.flat[::6], [linalg.norm(x) ** 2 for x in X])
+    assert_allclose(K.flat[::6], [linalg.norm(x) ** 2 for x in X])
 
 
 def test_rbf_kernel():
@@ -1137,7 +1317,7 @@ def test_rbf_kernel():
     X = rng.random_sample((5, 4))
     K = rbf_kernel(X, X)
     # the diagonal elements of a rbf kernel are 1
-    assert_array_almost_equal(K.flat[::6], np.ones(5))
+    assert_allclose(K.flat[::6], np.ones(5))
 
 
 def test_laplacian_kernel():
@@ -1145,22 +1325,24 @@ def test_laplacian_kernel():
     X = rng.random_sample((5, 4))
     K = laplacian_kernel(X, X)
     # the diagonal elements of a laplacian kernel are 1
-    assert_array_almost_equal(np.diag(K), np.ones(5))
+    assert_allclose(np.diag(K), np.ones(5))
 
     # off-diagonal elements are < 1 but > 0:
     assert np.all(K > 0)
     assert np.all(K - np.diag(np.diag(K)) < 1)
 
 
-@pytest.mark.parametrize('metric, pairwise_func',
-                         [('linear', linear_kernel),
-                          ('cosine', cosine_similarity)])
-def test_pairwise_similarity_sparse_output(metric, pairwise_func):
+@pytest.mark.parametrize(
+    "metric, pairwise_func",
+    [("linear", linear_kernel), ("cosine", cosine_similarity)],
+)
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_pairwise_similarity_sparse_output(metric, pairwise_func, csr_container):
     rng = np.random.RandomState(0)
     X = rng.random_sample((5, 4))
     Y = rng.random_sample((3, 4))
-    Xcsr = csr_matrix(X)
-    Ycsr = csr_matrix(Y)
+    Xcsr = csr_container(X)
+    Ycsr = csr_container(Y)
 
     # should be sparse
     K1 = pairwise_func(Xcsr, Ycsr, dense_output=False)
@@ -1169,24 +1351,24 @@ def test_pairwise_similarity_sparse_output(metric, pairwise_func):
     # should be dense, and equal to K1
     K2 = pairwise_func(X, Y, dense_output=True)
     assert not issparse(K2)
-    assert_array_almost_equal(K1.todense(), K2)
+    assert_allclose(K1.toarray(), K2)
 
-    # show the kernel output equal to the sparse.todense()
+    # show the kernel output equal to the sparse.toarray()
     K3 = pairwise_kernels(X, Y=Y, metric=metric)
-    assert_array_almost_equal(K1.todense(), K3)
+    assert_allclose(K1.toarray(), K3)
 
 
-def test_cosine_similarity():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_cosine_similarity(csr_container):
     # Test the cosine_similarity.
 
     rng = np.random.RandomState(0)
     X = rng.random_sample((5, 4))
     Y = rng.random_sample((3, 4))
-    Xcsr = csr_matrix(X)
-    Ycsr = csr_matrix(Y)
+    Xcsr = csr_container(X)
+    Ycsr = csr_container(Y)
 
-    for X_, Y_ in ((X, None), (X, Y),
-                   (Xcsr, None), (Xcsr, Ycsr)):
+    for X_, Y_ in ((X, None), (X, Y), (Xcsr, None), (Xcsr, Ycsr)):
         # Test that the cosine is kernel is equal to a linear kernel when data
         # has been previously normalized by L2-norm.
         K1 = pairwise_kernels(X_, Y=Y_, metric="cosine")
@@ -1194,7 +1376,7 @@ def test_cosine_similarity():
         if Y_ is not None:
             Y_ = normalize(Y_)
         K2 = pairwise_kernels(X_, Y=Y_, metric="linear")
-        assert_array_almost_equal(K1, K2)
+        assert_allclose(K1, K2)
 
 
 def test_check_dense_matrices():
@@ -1248,13 +1430,14 @@ def test_check_invalid_dimensions():
         check_pairwise_arrays(XA, XB)
 
 
-def test_check_sparse_arrays():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_check_sparse_arrays(csr_container):
     # Ensures that checks return valid sparse matrices.
     rng = np.random.RandomState(0)
     XA = rng.random_sample((5, 4))
-    XA_sparse = csr_matrix(XA)
+    XA_sparse = csr_container(XA)
     XB = rng.random_sample((5, 4))
-    XB_sparse = csr_matrix(XB)
+    XB_sparse = csr_container(XB)
     XA_checked, XB_checked = check_pairwise_arrays(XA_sparse, XB_sparse)
     # compare their difference because testing csr matrices for
     # equality with '==' does not work as expected.
@@ -1307,72 +1490,68 @@ def test_check_preserve_type():
     assert XB_checked.dtype == np.float32
 
     # mismatched A
-    XA_checked, XB_checked = check_pairwise_arrays(XA.astype(float),
-                                                   XB)
+    XA_checked, XB_checked = check_pairwise_arrays(XA.astype(float), XB)
     assert XA_checked.dtype == float
     assert XB_checked.dtype == float
 
     # mismatched B
-    XA_checked, XB_checked = check_pairwise_arrays(XA,
-                                                   XB.astype(float))
+    XA_checked, XB_checked = check_pairwise_arrays(XA, XB.astype(float))
     assert XA_checked.dtype == float
     assert XB_checked.dtype == float
 
 
 @pytest.mark.parametrize("n_jobs", [1, 2])
 @pytest.mark.parametrize("metric", ["seuclidean", "mahalanobis"])
-@pytest.mark.parametrize("dist_function",
-                         [pairwise_distances, pairwise_distances_chunked])
-@pytest.mark.parametrize("y_is_x", [True, False], ids=["Y is X", "Y is not X"])
-def test_pairwise_distances_data_derived_params(n_jobs, metric, dist_function,
-                                                y_is_x):
+@pytest.mark.parametrize(
+    "dist_function", [pairwise_distances, pairwise_distances_chunked]
+)
+def test_pairwise_distances_data_derived_params(n_jobs, metric, dist_function):
     # check that pairwise_distances give the same result in sequential and
     # parallel, when metric has data-derived parameters.
     with config_context(working_memory=0.1):  # to have more than 1 chunk
         rng = np.random.RandomState(0)
         X = rng.random_sample((100, 10))
 
-        if y_is_x:
-            Y = X
-            expected_dist_default_params = squareform(pdist(X, metric=metric))
-            if metric == "seuclidean":
-                params = {'V': np.var(X, axis=0, ddof=1)}
-            else:
-                params = {'VI': np.linalg.inv(np.cov(X.T)).T}
-        else:
-            Y = rng.random_sample((100, 10))
-            expected_dist_default_params = cdist(X, Y, metric=metric)
-            if metric == "seuclidean":
-                params = {'V': np.var(np.vstack([X, Y]), axis=0, ddof=1)}
-            else:
-                params = {'VI': np.linalg.inv(np.cov(np.vstack([X, Y]).T)).T}
-
-        expected_dist_explicit_params = cdist(X, Y, metric=metric, **params)
-        # TODO: Remove warn_checker in 1.0
-        if y_is_x:
-            warn_checker = pytest.warns(None)
-        else:
-            warn_checker = pytest.warns(FutureWarning,
-                                        match="to be specified if Y is passed")
-        with warn_checker:
-            dist = np.vstack(tuple(dist_function(X, Y,
-                                                 metric=metric,
-                                                 n_jobs=n_jobs)))
-
-        assert_allclose(dist, expected_dist_explicit_params)
-        assert_allclose(dist, expected_dist_default_params)
+        expected_dist = squareform(pdist(X, metric=metric))
+        dist = np.vstack(tuple(dist_function(X, metric=metric, n_jobs=n_jobs)))
+
+        assert_allclose(dist, expected_dist)
+
+
+@pytest.mark.parametrize("metric", ["seuclidean", "mahalanobis"])
+def test_pairwise_distances_data_derived_params_error(metric):
+    # check that pairwise_distances raises an error when Y is passed but
+    # metric has data-derived params that are not provided by the user.
+    rng = np.random.RandomState(0)
+    X = rng.random_sample((100, 10))
+    Y = rng.random_sample((100, 10))
+
+    with pytest.raises(
+        ValueError,
+        match=rf"The '(V|VI)' parameter is required for the {metric} metric",
+    ):
+        pairwise_distances(X, Y, metric=metric)
 
 
 @pytest.mark.parametrize(
-        'metric', [
-            'braycurtis', 'canberra', 'chebyshev',
-            'correlation', 'hamming', 'mahalanobis', 'minkowski', 'seuclidean',
-            'sqeuclidean', 'cityblock', 'cosine', 'euclidean'])
-@pytest.mark.parametrize(
-        "dtype",
-        [np.float32, np.float64])
+    "metric",
+    [
+        "braycurtis",
+        "canberra",
+        "chebyshev",
+        "correlation",
+        "hamming",
+        "mahalanobis",
+        "minkowski",
+        "seuclidean",
+        "sqeuclidean",
+        "cityblock",
+        "cosine",
+        "euclidean",
+    ],
+)
 @pytest.mark.parametrize("y_is_x", [True, False], ids=["Y is X", "Y is not X"])
-def test_numeric_pairwise_distances_datatypes(metric, dtype, y_is_x):
+def test_numeric_pairwise_distances_datatypes(metric, global_dtype, y_is_x):
     # Check that pairwise distances gives the same result as pdist and cdist
     # regardless of input datatype when using any scipy metric for comparing
     # numeric vectors
@@ -1383,25 +1562,122 @@ def test_numeric_pairwise_distances_datatypes(metric, dtype, y_is_x):
 
     rng = np.random.RandomState(0)
 
-    X = rng.random_sample((5, 4)).astype(dtype)
+    X = rng.random_sample((5, 4)).astype(global_dtype, copy=False)
 
     params = {}
     if y_is_x:
         Y = X
         expected_dist = squareform(pdist(X, metric=metric))
     else:
-        Y = rng.random_sample((5, 4)).astype(dtype)
+        Y = rng.random_sample((5, 4)).astype(global_dtype, copy=False)
         expected_dist = cdist(X, Y, metric=metric)
         # precompute parameters for seuclidean & mahalanobis when x is not y
-        if metric == 'seuclidean':
-            params = {'V': np.var(np.vstack([X, Y]),
-                                  axis=0, ddof=1, dtype=np.float64)}
-        elif metric == 'mahalanobis':
-            params = {'VI': np.linalg.inv(np.cov(np.vstack([X, Y]).T)).T}
+        if metric == "seuclidean":
+            params = {"V": np.var(np.vstack([X, Y]), axis=0, ddof=1, dtype=np.float64)}
+        elif metric == "mahalanobis":
+            params = {"VI": np.linalg.inv(np.cov(np.vstack([X, Y]).T)).T}
 
     dist = pairwise_distances(X, Y, metric=metric, **params)
 
-    # the default rtol=1e-7 is too close to the float32 precision
-    # and fails due to rounding errors
-    rtol = 1e-5 if dtype is np.float32 else 1e-7
-    assert_allclose(dist, expected_dist, rtol=rtol)
+    assert_allclose(dist, expected_dist)
+
+
+@pytest.mark.parametrize(
+    "pairwise_distances_func",
+    [pairwise_distances, pairwise_distances_argmin, pairwise_distances_argmin_min],
+)
+def test_nan_euclidean_support(pairwise_distances_func):
+    """Check that `nan_euclidean` is lenient with `nan` values."""
+
+    X = [[0, 1], [1, np.nan], [2, 3], [3, 5]]
+    output = pairwise_distances_func(X, X, metric="nan_euclidean")
+
+    assert not np.isnan(output).any()
+
+
+def test_nan_euclidean_constant_input_argmin():
+    """Check that the behavior of constant input is the same in the case of
+    full of nan vector and full of zero vector.
+    """
+
+    X_nan = [[np.nan, np.nan], [np.nan, np.nan], [np.nan, np.nan]]
+    argmin_nan = pairwise_distances_argmin(X_nan, X_nan, metric="nan_euclidean")
+
+    X_const = [[0, 0], [0, 0], [0, 0]]
+    argmin_const = pairwise_distances_argmin(X_const, X_const, metric="nan_euclidean")
+
+    assert_allclose(argmin_nan, argmin_const)
+
+
+@pytest.mark.parametrize(
+    "X,Y,expected_distance",
+    [
+        (
+            ["a", "ab", "abc"],
+            None,
+            [[0.0, 1.0, 2.0], [1.0, 0.0, 1.0], [2.0, 1.0, 0.0]],
+        ),
+        (
+            ["a", "ab", "abc"],
+            ["a", "ab"],
+            [[0.0, 1.0], [1.0, 0.0], [2.0, 1.0]],
+        ),
+    ],
+)
+def test_pairwise_dist_custom_metric_for_string(X, Y, expected_distance):
+    """Check pairwise_distances with lists of strings as input."""
+
+    def dummy_string_similarity(x, y):
+        return np.abs(len(x) - len(y))
+
+    actual_distance = pairwise_distances(X=X, Y=Y, metric=dummy_string_similarity)
+    assert_allclose(actual_distance, expected_distance)
+
+
+def test_pairwise_dist_custom_metric_for_bool():
+    """Check that pairwise_distances does not convert boolean input to float
+    when using a custom metric.
+    """
+
+    def dummy_bool_dist(v1, v2):
+        # dummy distance func using `&` and thus relying on the input data being boolean
+        return 1 - (v1 & v2).sum() / (v1 | v2).sum()
+
+    X = np.array([[1, 0, 0, 0], [1, 0, 1, 0], [1, 1, 1, 1]], dtype=bool)
+
+    expected_distance = np.array(
+        [
+            [0.0, 0.5, 0.75],
+            [0.5, 0.0, 0.5],
+            [0.75, 0.5, 0.0],
+        ]
+    )
+
+    actual_distance = pairwise_distances(X=X, metric=dummy_bool_dist)
+    assert_allclose(actual_distance, expected_distance)
+
+
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sparse_manhattan_readonly_dataset(csr_container):
+    # Non-regression test for: https://github.com/scikit-learn/scikit-learn/issues/7981
+    matrices1 = [csr_container(np.ones((5, 5)))]
+    matrices2 = [csr_container(np.ones((5, 5)))]
+    # Joblib memory maps datasets which makes them read-only.
+    # The following call was reporting as failing in #7981, but this must pass.
+    Parallel(n_jobs=2, max_nbytes=0)(
+        delayed(manhattan_distances)(m1, m2) for m1, m2 in zip(matrices1, matrices2)
+    )
+
+
+# TODO(1.8): remove
+def test_force_all_finite_rename_warning():
+    X = np.random.uniform(size=(10, 10))
+    Y = np.random.uniform(size=(10, 10))
+
+    msg = "'force_all_finite' was renamed to 'ensure_all_finite'"
+
+    with pytest.warns(FutureWarning, match=msg):
+        check_pairwise_arrays(X, Y, force_all_finite=True)
+
+    with pytest.warns(FutureWarning, match=msg):
+        pairwise_distances(X, Y, force_all_finite=True)
diff --git a/sklearn/metrics/tests/test_pairwise_distances_reduction.py b/sklearn/metrics/tests/test_pairwise_distances_reduction.py
new file mode 100644
index 0000000000000..0ea6d5d094d56
--- /dev/null
+++ b/sklearn/metrics/tests/test_pairwise_distances_reduction.py
@@ -0,0 +1,1643 @@
+import itertools
+import re
+import warnings
+from functools import partial
+
+import numpy as np
+import pytest
+from scipy.spatial.distance import cdist
+
+from sklearn.metrics import euclidean_distances, pairwise_distances
+from sklearn.metrics._pairwise_distances_reduction import (
+    ArgKmin,
+    ArgKminClassMode,
+    BaseDistancesReductionDispatcher,
+    RadiusNeighbors,
+    RadiusNeighborsClassMode,
+    sqeuclidean_row_norms,
+)
+from sklearn.utils._testing import (
+    assert_allclose,
+    assert_array_equal,
+    create_memmap_backed_data,
+)
+from sklearn.utils.fixes import CSR_CONTAINERS
+from sklearn.utils.parallel import _get_threadpool_controller
+
+# Common supported metric between scipy.spatial.distance.cdist
+# and BaseDistanceReductionDispatcher.
+# This allows constructing tests to check consistency of results
+# of concrete BaseDistanceReductionDispatcher on some metrics using APIs
+# from scipy and numpy.
+CDIST_PAIRWISE_DISTANCES_REDUCTION_COMMON_METRICS = [
+    "braycurtis",
+    "canberra",
+    "chebyshev",
+    "cityblock",
+    "euclidean",
+    "minkowski",
+    "seuclidean",
+]
+
+
+def _get_metric_params_list(metric: str, n_features: int, seed: int = 1):
+    """Return list of dummy DistanceMetric kwargs for tests."""
+
+    # Distinguishing on cases not to compute unneeded datastructures.
+    rng = np.random.RandomState(seed)
+
+    if metric == "minkowski":
+        minkowski_kwargs = [
+            dict(p=1.5),
+            dict(p=2),
+            dict(p=3),
+            dict(p=np.inf),
+            dict(p=3, w=rng.rand(n_features)),
+        ]
+
+        return minkowski_kwargs
+
+    if metric == "seuclidean":
+        return [dict(V=rng.rand(n_features))]
+
+    # Case of: "euclidean", "manhattan", "chebyshev", "haversine" or any other metric.
+    # In those cases, no kwargs is needed.
+    return [{}]
+
+
+def assert_same_distances_for_common_neighbors(
+    query_idx,
+    dist_row_a,
+    dist_row_b,
+    indices_row_a,
+    indices_row_b,
+    rtol,
+    atol,
+):
+    """Check that the distances of common neighbors are equal up to tolerance.
+
+    This does not check if there are missing neighbors in either result set.
+    Missingness is handled by assert_no_missing_neighbors.
+    """
+    # Compute a mapping from indices to distances for each result set and
+    # check that the computed neighbors with matching indices are within
+    # the expected distance tolerance.
+    indices_to_dist_a = dict(zip(indices_row_a, dist_row_a))
+    indices_to_dist_b = dict(zip(indices_row_b, dist_row_b))
+
+    common_indices = set(indices_row_a).intersection(set(indices_row_b))
+    for idx in common_indices:
+        dist_a = indices_to_dist_a[idx]
+        dist_b = indices_to_dist_b[idx]
+        try:
+            assert_allclose(dist_a, dist_b, rtol=rtol, atol=atol)
+        except AssertionError as e:
+            # Wrap exception to provide more context while also including
+            # the original exception with the computed absolute and
+            # relative differences.
+            raise AssertionError(
+                f"Query vector with index {query_idx} lead to different distances"
+                f" for common neighbor with index {idx}:"
+                f" dist_a={dist_a} vs dist_b={dist_b} (with atol={atol} and"
+                f" rtol={rtol})"
+            ) from e
+
+
+def assert_no_missing_neighbors(
+    query_idx,
+    dist_row_a,
+    dist_row_b,
+    indices_row_a,
+    indices_row_b,
+    threshold,
+):
+    """Compare the indices of neighbors in two results sets.
+
+    Any neighbor index with a distance below the precision threshold should
+    match one in the other result set. We ignore the last few neighbors beyond
+    the threshold as those can typically be missing due to rounding errors.
+
+    For radius queries, the threshold is just the radius minus the expected
+    precision level.
+
+    For k-NN queries, it is the maximum distance to the k-th neighbor minus the
+    expected precision level.
+    """
+    mask_a = dist_row_a < threshold
+    mask_b = dist_row_b < threshold
+    missing_from_b = np.setdiff1d(indices_row_a[mask_a], indices_row_b)
+    missing_from_a = np.setdiff1d(indices_row_b[mask_b], indices_row_a)
+    if len(missing_from_a) > 0 or len(missing_from_b) > 0:
+        raise AssertionError(
+            f"Query vector with index {query_idx} lead to mismatched result indices:\n"
+            f"neighbors in b missing from a: {missing_from_a}\n"
+            f"neighbors in a missing from b: {missing_from_b}\n"
+            f"dist_row_a={dist_row_a}\n"
+            f"dist_row_b={dist_row_b}\n"
+            f"indices_row_a={indices_row_a}\n"
+            f"indices_row_b={indices_row_b}\n"
+        )
+
+
+def assert_compatible_argkmin_results(
+    neighbors_dists_a,
+    neighbors_dists_b,
+    neighbors_indices_a,
+    neighbors_indices_b,
+    rtol=1e-5,
+    atol=1e-6,
+):
+    """Assert that argkmin results are valid up to rounding errors.
+
+    This function asserts that the results of argkmin queries are valid up to:
+    - rounding error tolerance on distance values;
+    - permutations of indices for distances values that differ up to the
+      expected precision level.
+
+    Furthermore, the distances must be sorted.
+
+    To be used for testing neighbors queries on float32 datasets: we accept
+    neighbors rank swaps only if they are caused by small rounding errors on
+    the distance computations.
+    """
+    is_sorted = lambda a: np.all(a[:-1] <= a[1:])
+
+    assert (
+        neighbors_dists_a.shape
+        == neighbors_dists_b.shape
+        == neighbors_indices_a.shape
+        == neighbors_indices_b.shape
+    ), "Arrays of results have incompatible shapes."
+
+    n_queries, _ = neighbors_dists_a.shape
+
+    # Asserting equality results one row at a time
+    for query_idx in range(n_queries):
+        dist_row_a = neighbors_dists_a[query_idx]
+        dist_row_b = neighbors_dists_b[query_idx]
+        indices_row_a = neighbors_indices_a[query_idx]
+        indices_row_b = neighbors_indices_b[query_idx]
+
+        assert is_sorted(dist_row_a), f"Distances aren't sorted on row {query_idx}"
+        assert is_sorted(dist_row_b), f"Distances aren't sorted on row {query_idx}"
+
+        assert_same_distances_for_common_neighbors(
+            query_idx,
+            dist_row_a,
+            dist_row_b,
+            indices_row_a,
+            indices_row_b,
+            rtol,
+            atol,
+        )
+
+        # Check that any neighbor with distances below the rounding error
+        # threshold have matching indices. The threshold is the distance to the
+        # k-th neighbors minus the expected precision level:
+        #
+        # (1 - rtol) * dist_k - atol
+        #
+        # Where dist_k is defined as the maximum distance to the kth-neighbor
+        # among the two result sets. This way of defining the threshold is
+        # stricter than taking the minimum of the two.
+        threshold = (1 - rtol) * np.maximum(
+            np.max(dist_row_a), np.max(dist_row_b)
+        ) - atol
+        assert_no_missing_neighbors(
+            query_idx,
+            dist_row_a,
+            dist_row_b,
+            indices_row_a,
+            indices_row_b,
+            threshold,
+        )
+
+
+def _non_trivial_radius(
+    *,
+    X=None,
+    Y=None,
+    metric=None,
+    precomputed_dists=None,
+    expected_n_neighbors=10,
+    n_subsampled_queries=10,
+    **metric_kwargs,
+):
+    # Find a non-trivial radius using a small subsample of the pairwise
+    # distances between X and Y: we want to return around expected_n_neighbors
+    # on average. Yielding too many results would make the test slow (because
+    # checking the results is expensive for large result sets), yielding 0 most
+    # of the time would make the test useless.
+    assert precomputed_dists is not None or metric is not None, (
+        "Either metric or precomputed_dists must be provided."
+    )
+
+    if precomputed_dists is None:
+        assert X is not None
+        assert Y is not None
+        sampled_dists = pairwise_distances(X, Y, metric=metric, **metric_kwargs)
+    else:
+        sampled_dists = precomputed_dists[:n_subsampled_queries].copy()
+    sampled_dists.sort(axis=1)
+    return sampled_dists[:, expected_n_neighbors].mean()
+
+
+def assert_compatible_radius_results(
+    neighbors_dists_a,
+    neighbors_dists_b,
+    neighbors_indices_a,
+    neighbors_indices_b,
+    radius,
+    check_sorted=True,
+    rtol=1e-5,
+    atol=1e-6,
+):
+    """Assert that radius neighborhood results are valid up to:
+
+      - relative and absolute tolerance on computed distance values
+      - permutations of indices for distances values that differ up to
+        a precision level
+      - missing or extra last elements if their distance is
+        close to the radius
+
+    To be used for testing neighbors queries on float32 datasets: we
+    accept neighbors rank swaps only if they are caused by small
+    rounding errors on the distance computations.
+
+    Input arrays must be sorted w.r.t distances.
+    """
+    is_sorted = lambda a: np.all(a[:-1] <= a[1:])
+
+    assert (
+        len(neighbors_dists_a)
+        == len(neighbors_dists_b)
+        == len(neighbors_indices_a)
+        == len(neighbors_indices_b)
+    )
+
+    n_queries = len(neighbors_dists_a)
+
+    # Asserting equality of results one vector at a time
+    for query_idx in range(n_queries):
+        dist_row_a = neighbors_dists_a[query_idx]
+        dist_row_b = neighbors_dists_b[query_idx]
+        indices_row_a = neighbors_indices_a[query_idx]
+        indices_row_b = neighbors_indices_b[query_idx]
+
+        if check_sorted:
+            assert is_sorted(dist_row_a), f"Distances aren't sorted on row {query_idx}"
+            assert is_sorted(dist_row_b), f"Distances aren't sorted on row {query_idx}"
+
+        assert len(dist_row_a) == len(indices_row_a)
+        assert len(dist_row_b) == len(indices_row_b)
+
+        # Check that all distances are within the requested radius
+        if len(dist_row_a) > 0:
+            max_dist_a = np.max(dist_row_a)
+            assert max_dist_a <= radius, (
+                f"Largest returned distance {max_dist_a} not within requested"
+                f" radius {radius} on row {query_idx}"
+            )
+        if len(dist_row_b) > 0:
+            max_dist_b = np.max(dist_row_b)
+            assert max_dist_b <= radius, (
+                f"Largest returned distance {max_dist_b} not within requested"
+                f" radius {radius} on row {query_idx}"
+            )
+
+        assert_same_distances_for_common_neighbors(
+            query_idx,
+            dist_row_a,
+            dist_row_b,
+            indices_row_a,
+            indices_row_b,
+            rtol,
+            atol,
+        )
+
+        threshold = (1 - rtol) * radius - atol
+        assert_no_missing_neighbors(
+            query_idx,
+            dist_row_a,
+            dist_row_b,
+            indices_row_a,
+            indices_row_b,
+            threshold,
+        )
+
+
+FLOAT32_TOLS = {
+    "atol": 1e-7,
+    "rtol": 1e-5,
+}
+FLOAT64_TOLS = {
+    "atol": 1e-9,
+    "rtol": 1e-7,
+}
+ASSERT_RESULT = {
+    (ArgKmin, np.float64): partial(assert_compatible_argkmin_results, **FLOAT64_TOLS),
+    (ArgKmin, np.float32): partial(assert_compatible_argkmin_results, **FLOAT32_TOLS),
+    (
+        RadiusNeighbors,
+        np.float64,
+    ): partial(assert_compatible_radius_results, **FLOAT64_TOLS),
+    (
+        RadiusNeighbors,
+        np.float32,
+    ): partial(assert_compatible_radius_results, **FLOAT32_TOLS),
+}
+
+
+def test_assert_compatible_argkmin_results():
+    atol = 1e-7
+    rtol = 0.0
+    tols = dict(atol=atol, rtol=rtol)
+
+    eps = atol / 3
+    _1m = 1.0 - eps
+    _1p = 1.0 + eps
+
+    _6_1m = 6.1 - eps
+    _6_1p = 6.1 + eps
+
+    ref_dist = np.array(
+        [
+            [1.2, 2.5, _6_1m, 6.1, _6_1p],
+            [_1m, _1m, 1, _1p, _1p],
+        ]
+    )
+    ref_indices = np.array(
+        [
+            [1, 2, 3, 4, 5],
+            [6, 7, 8, 9, 10],
+        ]
+    )
+
+    # Sanity check: compare the reference results to themselves.
+    assert_compatible_argkmin_results(
+        ref_dist, ref_dist, ref_indices, ref_indices, rtol
+    )
+
+    # Apply valid permutation on indices: the last 3 points are all very close
+    # to one another so we accept any permutation on their rankings.
+    assert_compatible_argkmin_results(
+        np.array([[1.2, 2.5, _6_1m, 6.1, _6_1p]]),
+        np.array([[1.2, 2.5, _6_1m, 6.1, _6_1p]]),
+        np.array([[1, 2, 3, 4, 5]]),
+        np.array([[1, 2, 5, 4, 3]]),
+        **tols,
+    )
+
+    # The last few indices do not necessarily have to match because of the rounding
+    # errors on the distances: there could be tied results at the boundary.
+    assert_compatible_argkmin_results(
+        np.array([[1.2, 2.5, 3.0, 6.1, _6_1p]]),
+        np.array([[1.2, 2.5, 3.0, _6_1m, 6.1]]),
+        np.array([[1, 2, 3, 4, 5]]),
+        np.array([[1, 2, 3, 6, 7]]),
+        **tols,
+    )
+
+    # All points have close distances so any ranking permutation
+    # is valid for this query result.
+    assert_compatible_argkmin_results(
+        np.array([[_1m, 1, _1p, _1p, _1p]]),
+        np.array([[1, 1, 1, 1, _1p]]),
+        np.array([[7, 6, 8, 10, 9]]),
+        np.array([[6, 9, 7, 8, 10]]),
+        **tols,
+    )
+
+    # They could also be nearly truncation of very large nearly tied result
+    # sets hence all indices can also be distinct in this case:
+    assert_compatible_argkmin_results(
+        np.array([[_1m, 1, _1p, _1p, _1p]]),
+        np.array([[_1m, 1, 1, 1, _1p]]),
+        np.array([[34, 30, 8, 12, 24]]),
+        np.array([[42, 1, 21, 13, 3]]),
+        **tols,
+    )
+
+    # Apply invalid permutation on indices: permuting the ranks of the 2
+    # nearest neighbors is invalid because the distance values are too
+    # different.
+    msg = re.escape(
+        "Query vector with index 0 lead to different distances for common neighbor with"
+        " index 1: dist_a=1.2 vs dist_b=2.5"
+    )
+    with pytest.raises(AssertionError, match=msg):
+        assert_compatible_argkmin_results(
+            np.array([[1.2, 2.5, _6_1m, 6.1, _6_1p]]),
+            np.array([[1.2, 2.5, _6_1m, 6.1, _6_1p]]),
+            np.array([[1, 2, 3, 4, 5]]),
+            np.array([[2, 1, 3, 4, 5]]),
+            **tols,
+        )
+
+    # Detect missing indices within the expected precision level, even when the
+    # distances match exactly.
+    msg = re.escape(
+        "neighbors in b missing from a: [12]\nneighbors in a missing from b: [1]"
+    )
+    with pytest.raises(AssertionError, match=msg):
+        assert_compatible_argkmin_results(
+            np.array([[1.2, 2.5, _6_1m, 6.1, _6_1p]]),
+            np.array([[1.2, 2.5, _6_1m, 6.1, _6_1p]]),
+            np.array([[1, 2, 3, 4, 5]]),
+            np.array([[12, 2, 4, 11, 3]]),
+            **tols,
+        )
+
+    # Detect missing indices outside the expected precision level.
+    msg = re.escape(
+        "neighbors in b missing from a: []\nneighbors in a missing from b: [3]"
+    )
+    with pytest.raises(AssertionError, match=msg):
+        assert_compatible_argkmin_results(
+            np.array([[_1m, 1.0, _6_1m, 6.1, _6_1p]]),
+            np.array([[1.0, 1.0, _6_1m, 6.1, 7]]),
+            np.array([[1, 2, 3, 4, 5]]),
+            np.array([[2, 1, 4, 5, 12]]),
+            **tols,
+        )
+
+    # Detect missing indices outside the expected precision level, in the other
+    # direction:
+    msg = re.escape(
+        "neighbors in b missing from a: [5]\nneighbors in a missing from b: []"
+    )
+    with pytest.raises(AssertionError, match=msg):
+        assert_compatible_argkmin_results(
+            np.array([[_1m, 1.0, _6_1m, 6.1, 7]]),
+            np.array([[1.0, 1.0, _6_1m, 6.1, _6_1p]]),
+            np.array([[1, 2, 3, 4, 12]]),
+            np.array([[2, 1, 5, 3, 4]]),
+            **tols,
+        )
+
+    # Distances aren't properly sorted
+    msg = "Distances aren't sorted on row 0"
+    with pytest.raises(AssertionError, match=msg):
+        assert_compatible_argkmin_results(
+            np.array([[1.2, 2.5, _6_1m, 6.1, _6_1p]]),
+            np.array([[2.5, 1.2, _6_1m, 6.1, _6_1p]]),
+            np.array([[1, 2, 3, 4, 5]]),
+            np.array([[2, 1, 4, 5, 3]]),
+            **tols,
+        )
+
+
+@pytest.mark.parametrize("check_sorted", [True, False])
+def test_assert_compatible_radius_results(check_sorted):
+    atol = 1e-7
+    rtol = 0.0
+    tols = dict(atol=atol, rtol=rtol)
+
+    eps = atol / 3
+    _1m = 1.0 - eps
+    _1p = 1.0 + eps
+    _6_1m = 6.1 - eps
+    _6_1p = 6.1 + eps
+
+    ref_dist = [
+        np.array([1.2, 2.5, _6_1m, 6.1, _6_1p]),
+        np.array([_1m, 1, _1p, _1p]),
+    ]
+
+    ref_indices = [
+        np.array([1, 2, 3, 4, 5]),
+        np.array([6, 7, 8, 9]),
+    ]
+
+    # Sanity check: compare the reference results to themselves.
+    assert_compatible_radius_results(
+        ref_dist,
+        ref_dist,
+        ref_indices,
+        ref_indices,
+        radius=7.0,
+        check_sorted=check_sorted,
+        **tols,
+    )
+
+    # Apply valid permutation on indices
+    assert_compatible_radius_results(
+        np.array([np.array([1.2, 2.5, _6_1m, 6.1, _6_1p])]),
+        np.array([np.array([1.2, 2.5, _6_1m, 6.1, _6_1p])]),
+        np.array([np.array([1, 2, 3, 4, 5])]),
+        np.array([np.array([1, 2, 4, 5, 3])]),
+        radius=7.0,
+        check_sorted=check_sorted,
+        **tols,
+    )
+    assert_compatible_radius_results(
+        np.array([np.array([_1m, _1m, 1, _1p, _1p])]),
+        np.array([np.array([_1m, _1m, 1, _1p, _1p])]),
+        np.array([np.array([6, 7, 8, 9, 10])]),
+        np.array([np.array([6, 9, 7, 8, 10])]),
+        radius=7.0,
+        check_sorted=check_sorted,
+        **tols,
+    )
+
+    # Apply invalid permutation on indices
+    msg = re.escape(
+        "Query vector with index 0 lead to different distances for common neighbor with"
+        " index 1: dist_a=1.2 vs dist_b=2.5"
+    )
+    with pytest.raises(AssertionError, match=msg):
+        assert_compatible_radius_results(
+            np.array([np.array([1.2, 2.5, _6_1m, 6.1, _6_1p])]),
+            np.array([np.array([1.2, 2.5, _6_1m, 6.1, _6_1p])]),
+            np.array([np.array([1, 2, 3, 4, 5])]),
+            np.array([np.array([2, 1, 3, 4, 5])]),
+            radius=7.0,
+            check_sorted=check_sorted,
+            **tols,
+        )
+
+    # Having extra last or missing elements is valid if they are in the
+    # tolerated rounding error range: [(1 - rtol) * radius - atol, radius]
+    assert_compatible_radius_results(
+        np.array([np.array([1.2, 2.5, _6_1m, 6.1, _6_1p, _6_1p])]),
+        np.array([np.array([1.2, 2.5, _6_1m, 6.1])]),
+        np.array([np.array([1, 2, 3, 4, 5, 7])]),
+        np.array([np.array([1, 2, 3, 6])]),
+        radius=_6_1p,
+        check_sorted=check_sorted,
+        **tols,
+    )
+
+    # Any discrepancy outside the tolerated rounding error range is invalid and
+    # indicates a missing neighbor in one of the result sets.
+    msg = re.escape(
+        "Query vector with index 0 lead to mismatched result indices:\nneighbors in b"
+        " missing from a: []\nneighbors in a missing from b: [3]"
+    )
+    with pytest.raises(AssertionError, match=msg):
+        assert_compatible_radius_results(
+            np.array([np.array([1.2, 2.5, 6])]),
+            np.array([np.array([1.2, 2.5])]),
+            np.array([np.array([1, 2, 3])]),
+            np.array([np.array([1, 2])]),
+            radius=6.1,
+            check_sorted=check_sorted,
+            **tols,
+        )
+    msg = re.escape(
+        "Query vector with index 0 lead to mismatched result indices:\nneighbors in b"
+        " missing from a: [4]\nneighbors in a missing from b: [2]"
+    )
+    with pytest.raises(AssertionError, match=msg):
+        assert_compatible_radius_results(
+            np.array([np.array([1.2, 2.1, 2.5])]),
+            np.array([np.array([1.2, 2, 2.5])]),
+            np.array([np.array([1, 2, 3])]),
+            np.array([np.array([1, 4, 3])]),
+            radius=6.1,
+            check_sorted=check_sorted,
+            **tols,
+        )
+
+    # Radius upper bound is strictly checked
+    msg = re.escape(
+        "Largest returned distance 6.100000033333333 not within requested radius 6.1 on"
+        " row 0"
+    )
+    with pytest.raises(AssertionError, match=msg):
+        assert_compatible_radius_results(
+            np.array([np.array([1.2, 2.5, _6_1m, 6.1, _6_1p])]),
+            np.array([np.array([1.2, 2.5, _6_1m, 6.1, 6.1])]),
+            np.array([np.array([1, 2, 3, 4, 5])]),
+            np.array([np.array([2, 1, 4, 5, 3])]),
+            radius=6.1,
+            check_sorted=check_sorted,
+            **tols,
+        )
+    with pytest.raises(AssertionError, match=msg):
+        assert_compatible_radius_results(
+            np.array([np.array([1.2, 2.5, _6_1m, 6.1, 6.1])]),
+            np.array([np.array([1.2, 2.5, _6_1m, 6.1, _6_1p])]),
+            np.array([np.array([1, 2, 3, 4, 5])]),
+            np.array([np.array([2, 1, 4, 5, 3])]),
+            radius=6.1,
+            check_sorted=check_sorted,
+            **tols,
+        )
+
+    if check_sorted:
+        # Distances aren't properly sorted
+        msg = "Distances aren't sorted on row 0"
+        with pytest.raises(AssertionError, match=msg):
+            assert_compatible_radius_results(
+                np.array([np.array([1.2, 2.5, _6_1m, 6.1, _6_1p])]),
+                np.array([np.array([2.5, 1.2, _6_1m, 6.1, _6_1p])]),
+                np.array([np.array([1, 2, 3, 4, 5])]),
+                np.array([np.array([2, 1, 4, 5, 3])]),
+                radius=_6_1p,
+                check_sorted=True,
+                **tols,
+            )
+    else:
+        assert_compatible_radius_results(
+            np.array([np.array([1.2, 2.5, _6_1m, 6.1, _6_1p])]),
+            np.array([np.array([2.5, 1.2, _6_1m, 6.1, _6_1p])]),
+            np.array([np.array([1, 2, 3, 4, 5])]),
+            np.array([np.array([2, 1, 4, 5, 3])]),
+            radius=_6_1p,
+            check_sorted=False,
+            **tols,
+        )
+
+
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_pairwise_distances_reduction_is_usable_for(csr_container):
+    rng = np.random.RandomState(0)
+    X = rng.rand(100, 10)
+    Y = rng.rand(100, 10)
+    X_csr = csr_container(X)
+    Y_csr = csr_container(Y)
+    metric = "manhattan"
+
+    # Must be usable for all possible pair of {dense, sparse} datasets
+    assert BaseDistancesReductionDispatcher.is_usable_for(X, Y, metric)
+    assert BaseDistancesReductionDispatcher.is_usable_for(X_csr, Y_csr, metric)
+    assert BaseDistancesReductionDispatcher.is_usable_for(X_csr, Y, metric)
+    assert BaseDistancesReductionDispatcher.is_usable_for(X, Y_csr, metric)
+
+    assert BaseDistancesReductionDispatcher.is_usable_for(
+        X.astype(np.float64), Y.astype(np.float64), metric
+    )
+
+    assert BaseDistancesReductionDispatcher.is_usable_for(
+        X.astype(np.float32), Y.astype(np.float32), metric
+    )
+
+    assert not BaseDistancesReductionDispatcher.is_usable_for(
+        X.astype(np.int64), Y.astype(np.int64), metric
+    )
+
+    assert not BaseDistancesReductionDispatcher.is_usable_for(X, Y, metric="pyfunc")
+    assert not BaseDistancesReductionDispatcher.is_usable_for(
+        X.astype(np.float32), Y, metric
+    )
+    assert not BaseDistancesReductionDispatcher.is_usable_for(
+        X, Y.astype(np.int32), metric
+    )
+
+    # F-ordered arrays are not supported
+    assert not BaseDistancesReductionDispatcher.is_usable_for(
+        np.asfortranarray(X), Y, metric
+    )
+
+    assert BaseDistancesReductionDispatcher.is_usable_for(X_csr, Y, metric="euclidean")
+    assert BaseDistancesReductionDispatcher.is_usable_for(
+        X, Y_csr, metric="sqeuclidean"
+    )
+
+    # FIXME: the current Cython implementation is too slow for a large number of
+    # features. We temporarily disable it to fallback on SciPy's implementation.
+    # See: https://github.com/scikit-learn/scikit-learn/issues/28191
+    assert not BaseDistancesReductionDispatcher.is_usable_for(
+        X_csr, Y_csr, metric="sqeuclidean"
+    )
+    assert not BaseDistancesReductionDispatcher.is_usable_for(
+        X_csr, Y_csr, metric="euclidean"
+    )
+
+    # CSR matrices without non-zeros elements aren't currently supported
+    # TODO: support CSR matrices without non-zeros elements
+    X_csr_0_nnz = csr_container(X * 0)
+    assert not BaseDistancesReductionDispatcher.is_usable_for(X_csr_0_nnz, Y, metric)
+
+    # CSR matrices with int64 indices and indptr (e.g. large nnz, or large n_features)
+    # aren't supported as of now.
+    # See: https://github.com/scikit-learn/scikit-learn/issues/23653
+    # TODO: support CSR matrices with int64 indices and indptr
+    X_csr_int64 = csr_container(X)
+    X_csr_int64.indices = X_csr_int64.indices.astype(np.int64)
+    assert not BaseDistancesReductionDispatcher.is_usable_for(X_csr_int64, Y, metric)
+
+
+def test_argkmin_factory_method_wrong_usages():
+    rng = np.random.RandomState(1)
+    X = rng.rand(100, 10)
+    Y = rng.rand(100, 10)
+    k = 5
+    metric = "euclidean"
+
+    msg = (
+        "Only float64 or float32 datasets pairs are supported at this time, "
+        "got: X.dtype=float32 and Y.dtype=float64"
+    )
+    with pytest.raises(ValueError, match=msg):
+        ArgKmin.compute(X=X.astype(np.float32), Y=Y, k=k, metric=metric)
+
+    msg = (
+        "Only float64 or float32 datasets pairs are supported at this time, "
+        "got: X.dtype=float64 and Y.dtype=int32"
+    )
+    with pytest.raises(ValueError, match=msg):
+        ArgKmin.compute(X=X, Y=Y.astype(np.int32), k=k, metric=metric)
+
+    with pytest.raises(ValueError, match="k == -1, must be >= 1."):
+        ArgKmin.compute(X=X, Y=Y, k=-1, metric=metric)
+
+    with pytest.raises(ValueError, match="k == 0, must be >= 1."):
+        ArgKmin.compute(X=X, Y=Y, k=0, metric=metric)
+
+    with pytest.raises(ValueError, match="Unrecognized metric"):
+        ArgKmin.compute(X=X, Y=Y, k=k, metric="wrong metric")
+
+    with pytest.raises(
+        ValueError, match=r"Buffer has wrong number of dimensions \(expected 2, got 1\)"
+    ):
+        ArgKmin.compute(X=np.array([1.0, 2.0]), Y=Y, k=k, metric=metric)
+
+    with pytest.raises(ValueError, match="ndarray is not C-contiguous"):
+        ArgKmin.compute(X=np.asfortranarray(X), Y=Y, k=k, metric=metric)
+
+    # A UserWarning must be raised in this case.
+    unused_metric_kwargs = {"p": 3}
+
+    message = r"Some metric_kwargs have been passed \({'p': 3}\) but"
+
+    with pytest.warns(UserWarning, match=message):
+        ArgKmin.compute(
+            X=X, Y=Y, k=k, metric=metric, metric_kwargs=unused_metric_kwargs
+        )
+
+    # A UserWarning must be raised in this case.
+    metric_kwargs = {
+        "p": 3,  # unused
+        "Y_norm_squared": sqeuclidean_row_norms(Y, num_threads=2),
+    }
+
+    message = r"Some metric_kwargs have been passed \({'p': 3, 'Y_norm_squared'"
+
+    with pytest.warns(UserWarning, match=message):
+        ArgKmin.compute(X=X, Y=Y, k=k, metric=metric, metric_kwargs=metric_kwargs)
+
+    # No user warning must be raised in this case.
+    metric_kwargs = {
+        "X_norm_squared": sqeuclidean_row_norms(X, num_threads=2),
+    }
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", category=UserWarning)
+        ArgKmin.compute(X=X, Y=Y, k=k, metric=metric, metric_kwargs=metric_kwargs)
+
+    # No user warning must be raised in this case.
+    metric_kwargs = {
+        "X_norm_squared": sqeuclidean_row_norms(X, num_threads=2),
+        "Y_norm_squared": sqeuclidean_row_norms(Y, num_threads=2),
+    }
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", category=UserWarning)
+        ArgKmin.compute(X=X, Y=Y, k=k, metric=metric, metric_kwargs=metric_kwargs)
+
+
+def test_argkmin_classmode_factory_method_wrong_usages():
+    rng = np.random.RandomState(1)
+    X = rng.rand(100, 10)
+    Y = rng.rand(100, 10)
+    k = 5
+    metric = "manhattan"
+
+    weights = "uniform"
+    Y_labels = rng.randint(low=0, high=10, size=100)
+    unique_Y_labels = np.unique(Y_labels)
+
+    msg = (
+        "Only float64 or float32 datasets pairs are supported at this time, "
+        "got: X.dtype=float32 and Y.dtype=float64"
+    )
+    with pytest.raises(ValueError, match=msg):
+        ArgKminClassMode.compute(
+            X=X.astype(np.float32),
+            Y=Y,
+            k=k,
+            metric=metric,
+            weights=weights,
+            Y_labels=Y_labels,
+            unique_Y_labels=unique_Y_labels,
+        )
+
+    msg = (
+        "Only float64 or float32 datasets pairs are supported at this time, "
+        "got: X.dtype=float64 and Y.dtype=int32"
+    )
+    with pytest.raises(ValueError, match=msg):
+        ArgKminClassMode.compute(
+            X=X,
+            Y=Y.astype(np.int32),
+            k=k,
+            metric=metric,
+            weights=weights,
+            Y_labels=Y_labels,
+            unique_Y_labels=unique_Y_labels,
+        )
+
+    with pytest.raises(ValueError, match="k == -1, must be >= 1."):
+        ArgKminClassMode.compute(
+            X=X,
+            Y=Y,
+            k=-1,
+            metric=metric,
+            weights=weights,
+            Y_labels=Y_labels,
+            unique_Y_labels=unique_Y_labels,
+        )
+
+    with pytest.raises(ValueError, match="k == 0, must be >= 1."):
+        ArgKminClassMode.compute(
+            X=X,
+            Y=Y,
+            k=0,
+            metric=metric,
+            weights=weights,
+            Y_labels=Y_labels,
+            unique_Y_labels=unique_Y_labels,
+        )
+
+    with pytest.raises(ValueError, match="Unrecognized metric"):
+        ArgKminClassMode.compute(
+            X=X,
+            Y=Y,
+            k=k,
+            metric="wrong metric",
+            weights=weights,
+            Y_labels=Y_labels,
+            unique_Y_labels=unique_Y_labels,
+        )
+
+    with pytest.raises(
+        ValueError, match=r"Buffer has wrong number of dimensions \(expected 2, got 1\)"
+    ):
+        ArgKminClassMode.compute(
+            X=np.array([1.0, 2.0]),
+            Y=Y,
+            k=k,
+            metric=metric,
+            weights=weights,
+            Y_labels=Y_labels,
+            unique_Y_labels=unique_Y_labels,
+        )
+
+    with pytest.raises(ValueError, match="ndarray is not C-contiguous"):
+        ArgKminClassMode.compute(
+            X=np.asfortranarray(X),
+            Y=Y,
+            k=k,
+            metric=metric,
+            weights=weights,
+            Y_labels=Y_labels,
+            unique_Y_labels=unique_Y_labels,
+        )
+
+    non_existent_weights_strategy = "non_existent_weights_strategy"
+    message = (
+        "Only the 'uniform' or 'distance' weights options are supported at this time. "
+        f"Got: weights='{non_existent_weights_strategy}'."
+    )
+    with pytest.raises(ValueError, match=message):
+        ArgKminClassMode.compute(
+            X=X,
+            Y=Y,
+            k=k,
+            metric=metric,
+            weights=non_existent_weights_strategy,
+            Y_labels=Y_labels,
+            unique_Y_labels=unique_Y_labels,
+        )
+
+    # TODO: introduce assertions on UserWarnings once the Euclidean specialisation
+    # of ArgKminClassMode is supported.
+
+
+def test_radius_neighbors_factory_method_wrong_usages():
+    rng = np.random.RandomState(1)
+    X = rng.rand(100, 10)
+    Y = rng.rand(100, 10)
+    radius = 5
+    metric = "euclidean"
+
+    msg = (
+        "Only float64 or float32 datasets pairs are supported at this time, "
+        "got: X.dtype=float32 and Y.dtype=float64"
+    )
+    with pytest.raises(
+        ValueError,
+        match=msg,
+    ):
+        RadiusNeighbors.compute(
+            X=X.astype(np.float32), Y=Y, radius=radius, metric=metric
+        )
+
+    msg = (
+        "Only float64 or float32 datasets pairs are supported at this time, "
+        "got: X.dtype=float64 and Y.dtype=int32"
+    )
+    with pytest.raises(
+        ValueError,
+        match=msg,
+    ):
+        RadiusNeighbors.compute(X=X, Y=Y.astype(np.int32), radius=radius, metric=metric)
+
+    with pytest.raises(ValueError, match="radius == -1.0, must be >= 0."):
+        RadiusNeighbors.compute(X=X, Y=Y, radius=-1, metric=metric)
+
+    with pytest.raises(ValueError, match="Unrecognized metric"):
+        RadiusNeighbors.compute(X=X, Y=Y, radius=radius, metric="wrong metric")
+
+    with pytest.raises(
+        ValueError, match=r"Buffer has wrong number of dimensions \(expected 2, got 1\)"
+    ):
+        RadiusNeighbors.compute(
+            X=np.array([1.0, 2.0]), Y=Y, radius=radius, metric=metric
+        )
+
+    with pytest.raises(ValueError, match="ndarray is not C-contiguous"):
+        RadiusNeighbors.compute(
+            X=np.asfortranarray(X), Y=Y, radius=radius, metric=metric
+        )
+
+    unused_metric_kwargs = {"p": 3}
+
+    # A UserWarning must be raised in this case.
+    message = r"Some metric_kwargs have been passed \({'p': 3}\) but"
+
+    with pytest.warns(UserWarning, match=message):
+        RadiusNeighbors.compute(
+            X=X, Y=Y, radius=radius, metric=metric, metric_kwargs=unused_metric_kwargs
+        )
+
+    # A UserWarning must be raised in this case.
+    metric_kwargs = {
+        "p": 3,  # unused
+        "Y_norm_squared": sqeuclidean_row_norms(Y, num_threads=2),
+    }
+
+    message = r"Some metric_kwargs have been passed \({'p': 3, 'Y_norm_squared'"
+
+    with pytest.warns(UserWarning, match=message):
+        RadiusNeighbors.compute(
+            X=X, Y=Y, radius=radius, metric=metric, metric_kwargs=metric_kwargs
+        )
+
+    # No user warning must be raised in this case.
+    metric_kwargs = {
+        "X_norm_squared": sqeuclidean_row_norms(X, num_threads=2),
+        "Y_norm_squared": sqeuclidean_row_norms(Y, num_threads=2),
+    }
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", category=UserWarning)
+        RadiusNeighbors.compute(
+            X=X, Y=Y, radius=radius, metric=metric, metric_kwargs=metric_kwargs
+        )
+
+    # No user warning must be raised in this case.
+    metric_kwargs = {
+        "X_norm_squared": sqeuclidean_row_norms(X, num_threads=2),
+    }
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", category=UserWarning)
+        RadiusNeighbors.compute(
+            X=X, Y=Y, radius=radius, metric=metric, metric_kwargs=metric_kwargs
+        )
+
+
+def test_radius_neighbors_classmode_factory_method_wrong_usages():
+    rng = np.random.RandomState(1)
+    X = rng.rand(100, 10)
+    Y = rng.rand(100, 10)
+    radius = 5
+    metric = "manhattan"
+    weights = "uniform"
+    Y_labels = rng.randint(low=0, high=10, size=100)
+    unique_Y_labels = np.unique(Y_labels)
+
+    msg = (
+        "Only float64 or float32 datasets pairs are supported at this time, "
+        "got: X.dtype=float32 and Y.dtype=float64"
+    )
+    with pytest.raises(ValueError, match=msg):
+        RadiusNeighborsClassMode.compute(
+            X=X.astype(np.float32),
+            Y=Y,
+            radius=radius,
+            metric=metric,
+            weights=weights,
+            Y_labels=Y_labels,
+            unique_Y_labels=unique_Y_labels,
+            outlier_label=None,
+        )
+
+    msg = (
+        "Only float64 or float32 datasets pairs are supported at this time, "
+        "got: X.dtype=float64 and Y.dtype=int32"
+    )
+    with pytest.raises(ValueError, match=msg):
+        RadiusNeighborsClassMode.compute(
+            X=X,
+            Y=Y.astype(np.int32),
+            radius=radius,
+            metric=metric,
+            weights=weights,
+            Y_labels=Y_labels,
+            unique_Y_labels=unique_Y_labels,
+            outlier_label=None,
+        )
+
+    with pytest.raises(ValueError, match="radius == -1.0, must be >= 0."):
+        RadiusNeighborsClassMode.compute(
+            X=X,
+            Y=Y,
+            radius=-1,
+            metric=metric,
+            weights=weights,
+            Y_labels=Y_labels,
+            unique_Y_labels=unique_Y_labels,
+            outlier_label=None,
+        )
+
+    with pytest.raises(ValueError, match="Unrecognized metric"):
+        RadiusNeighborsClassMode.compute(
+            X=X,
+            Y=Y,
+            radius=-1,
+            metric="wrong_metric",
+            weights=weights,
+            Y_labels=Y_labels,
+            unique_Y_labels=unique_Y_labels,
+            outlier_label=None,
+        )
+
+    with pytest.raises(
+        ValueError, match=r"Buffer has wrong number of dimensions \(expected 2, got 1\)"
+    ):
+        RadiusNeighborsClassMode.compute(
+            X=np.array([1.0, 2.0]),
+            Y=Y,
+            radius=radius,
+            metric=metric,
+            weights=weights,
+            Y_labels=Y_labels,
+            unique_Y_labels=unique_Y_labels,
+            outlier_label=None,
+        )
+
+    with pytest.raises(ValueError, match="ndarray is not C-contiguous"):
+        RadiusNeighborsClassMode.compute(
+            X=np.asfortranarray(X),
+            Y=Y,
+            radius=radius,
+            metric=metric,
+            weights=weights,
+            Y_labels=Y_labels,
+            unique_Y_labels=unique_Y_labels,
+            outlier_label=None,
+        )
+
+    non_existent_weights_strategy = "non_existent_weights_strategy"
+    msg = (
+        "Only the 'uniform' or 'distance' weights options are supported at this time. "
+        f"Got: weights='{non_existent_weights_strategy}'."
+    )
+    with pytest.raises(ValueError, match=msg):
+        RadiusNeighborsClassMode.compute(
+            X=X,
+            Y=Y,
+            radius=radius,
+            metric="wrong_metric",
+            weights=non_existent_weights_strategy,
+            Y_labels=Y_labels,
+            unique_Y_labels=unique_Y_labels,
+            outlier_label=None,
+        )
+
+
+@pytest.mark.parametrize("Dispatcher", [ArgKmin, RadiusNeighbors])
+@pytest.mark.parametrize("dtype", [np.float64, np.float32])
+def test_chunk_size_agnosticism(
+    global_random_seed,
+    Dispatcher,
+    dtype,
+    n_features=100,
+):
+    """Check that results do not depend on the chunk size."""
+    rng = np.random.RandomState(global_random_seed)
+    spread = 100
+    n_samples_X, n_samples_Y = rng.choice([97, 100, 101, 500], size=2, replace=False)
+    X = rng.rand(n_samples_X, n_features).astype(dtype) * spread
+    Y = rng.rand(n_samples_Y, n_features).astype(dtype) * spread
+
+    if Dispatcher is ArgKmin:
+        parameter = 10
+        check_parameters = {}
+        compute_parameters = {}
+    else:
+        radius = _non_trivial_radius(X=X, Y=Y, metric="euclidean")
+        parameter = radius
+        check_parameters = {"radius": radius}
+        compute_parameters = {"sort_results": True}
+
+    ref_dist, ref_indices = Dispatcher.compute(
+        X,
+        Y,
+        parameter,
+        chunk_size=256,  # default
+        metric="manhattan",
+        return_distance=True,
+        **compute_parameters,
+    )
+
+    dist, indices = Dispatcher.compute(
+        X,
+        Y,
+        parameter,
+        chunk_size=41,
+        metric="manhattan",
+        return_distance=True,
+        **compute_parameters,
+    )
+
+    ASSERT_RESULT[(Dispatcher, dtype)](
+        ref_dist, dist, ref_indices, indices, **check_parameters
+    )
+
+
+@pytest.mark.parametrize("Dispatcher", [ArgKmin, RadiusNeighbors])
+@pytest.mark.parametrize("dtype", [np.float64, np.float32])
+def test_n_threads_agnosticism(
+    global_random_seed,
+    Dispatcher,
+    dtype,
+    n_features=100,
+):
+    """Check that results do not depend on the number of threads."""
+    rng = np.random.RandomState(global_random_seed)
+    n_samples_X, n_samples_Y = rng.choice([97, 100, 101, 500], size=2, replace=False)
+    spread = 100
+    X = rng.rand(n_samples_X, n_features).astype(dtype) * spread
+    Y = rng.rand(n_samples_Y, n_features).astype(dtype) * spread
+
+    if Dispatcher is ArgKmin:
+        parameter = 10
+        check_parameters = {}
+        compute_parameters = {}
+    else:
+        radius = _non_trivial_radius(X=X, Y=Y, metric="euclidean")
+        parameter = radius
+        check_parameters = {"radius": radius}
+        compute_parameters = {"sort_results": True}
+
+    ref_dist, ref_indices = Dispatcher.compute(
+        X,
+        Y,
+        parameter,
+        chunk_size=25,  # make sure we use multiple threads
+        return_distance=True,
+        **compute_parameters,
+    )
+
+    with _get_threadpool_controller().limit(limits=1, user_api="openmp"):
+        dist, indices = Dispatcher.compute(
+            X,
+            Y,
+            parameter,
+            chunk_size=25,
+            return_distance=True,
+            **compute_parameters,
+        )
+
+    ASSERT_RESULT[(Dispatcher, dtype)](
+        ref_dist, dist, ref_indices, indices, **check_parameters
+    )
+
+
+@pytest.mark.parametrize(
+    "Dispatcher, dtype",
+    [
+        (ArgKmin, np.float64),
+        (RadiusNeighbors, np.float32),
+        (ArgKmin, np.float32),
+        (RadiusNeighbors, np.float64),
+    ],
+)
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_format_agnosticism(
+    global_random_seed,
+    Dispatcher,
+    dtype,
+    csr_container,
+):
+    """Check that results do not depend on the format (dense, sparse) of the input."""
+    rng = np.random.RandomState(global_random_seed)
+    spread = 100
+    n_samples, n_features = 100, 100
+
+    X = rng.rand(n_samples, n_features).astype(dtype) * spread
+    Y = rng.rand(n_samples, n_features).astype(dtype) * spread
+
+    X_csr = csr_container(X)
+    Y_csr = csr_container(Y)
+
+    if Dispatcher is ArgKmin:
+        parameter = 10
+        check_parameters = {}
+        compute_parameters = {}
+    else:
+        # Adjusting the radius to ensure that the expected results is neither
+        # trivially empty nor too large.
+        radius = _non_trivial_radius(X=X, Y=Y, metric="euclidean")
+        parameter = radius
+        check_parameters = {"radius": radius}
+        compute_parameters = {"sort_results": True}
+
+    dist_dense, indices_dense = Dispatcher.compute(
+        X,
+        Y,
+        parameter,
+        chunk_size=50,
+        return_distance=True,
+        **compute_parameters,
+    )
+
+    for _X, _Y in itertools.product((X, X_csr), (Y, Y_csr)):
+        if _X is X and _Y is Y:
+            continue
+        dist, indices = Dispatcher.compute(
+            _X,
+            _Y,
+            parameter,
+            chunk_size=50,
+            return_distance=True,
+            **compute_parameters,
+        )
+        ASSERT_RESULT[(Dispatcher, dtype)](
+            dist_dense,
+            dist,
+            indices_dense,
+            indices,
+            **check_parameters,
+        )
+
+
+@pytest.mark.parametrize("Dispatcher", [ArgKmin, RadiusNeighbors])
+def test_strategies_consistency(
+    global_random_seed,
+    global_dtype,
+    Dispatcher,
+    n_features=10,
+):
+    """Check that the results do not depend on the strategy used."""
+    rng = np.random.RandomState(global_random_seed)
+    metric = rng.choice(
+        np.array(
+            [
+                "euclidean",
+                "minkowski",
+                "manhattan",
+                "haversine",
+            ],
+            dtype=object,
+        )
+    )
+    n_samples_X, n_samples_Y = rng.choice([97, 100, 101, 500], size=2, replace=False)
+    spread = 100
+    X = rng.rand(n_samples_X, n_features).astype(global_dtype) * spread
+    Y = rng.rand(n_samples_Y, n_features).astype(global_dtype) * spread
+
+    # Haversine distance only accepts 2D data
+    if metric == "haversine":
+        X = np.ascontiguousarray(X[:, :2])
+        Y = np.ascontiguousarray(Y[:, :2])
+
+    if Dispatcher is ArgKmin:
+        parameter = 10
+        check_parameters = {}
+        compute_parameters = {}
+    else:
+        radius = _non_trivial_radius(X=X, Y=Y, metric=metric)
+        parameter = radius
+        check_parameters = {"radius": radius}
+        compute_parameters = {"sort_results": True}
+
+    dist_par_X, indices_par_X = Dispatcher.compute(
+        X,
+        Y,
+        parameter,
+        metric=metric,
+        # Taking the first
+        metric_kwargs=_get_metric_params_list(
+            metric, n_features, seed=global_random_seed
+        )[0],
+        # To be sure to use parallelization
+        chunk_size=n_samples_X // 4,
+        strategy="parallel_on_X",
+        return_distance=True,
+        **compute_parameters,
+    )
+
+    dist_par_Y, indices_par_Y = Dispatcher.compute(
+        X,
+        Y,
+        parameter,
+        metric=metric,
+        # Taking the first
+        metric_kwargs=_get_metric_params_list(
+            metric, n_features, seed=global_random_seed
+        )[0],
+        # To be sure to use parallelization
+        chunk_size=n_samples_Y // 4,
+        strategy="parallel_on_Y",
+        return_distance=True,
+        **compute_parameters,
+    )
+
+    ASSERT_RESULT[(Dispatcher, global_dtype)](
+        dist_par_X, dist_par_Y, indices_par_X, indices_par_Y, **check_parameters
+    )
+
+
+# "Concrete Dispatchers"-specific tests
+
+
+@pytest.mark.parametrize("metric", CDIST_PAIRWISE_DISTANCES_REDUCTION_COMMON_METRICS)
+@pytest.mark.parametrize("strategy", ("parallel_on_X", "parallel_on_Y"))
+@pytest.mark.parametrize("dtype", [np.float64, np.float32])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_pairwise_distances_argkmin(
+    global_random_seed,
+    metric,
+    strategy,
+    dtype,
+    csr_container,
+    n_queries=5,
+    n_samples=100,
+    k=10,
+):
+    rng = np.random.RandomState(global_random_seed)
+    n_features = rng.choice([50, 500])
+    translation = rng.choice([0, 1e6])
+    spread = 1000
+    X = translation + rng.rand(n_queries, n_features).astype(dtype) * spread
+    Y = translation + rng.rand(n_samples, n_features).astype(dtype) * spread
+
+    X_csr = csr_container(X)
+    Y_csr = csr_container(Y)
+
+    # Haversine distance only accepts 2D data
+    if metric == "haversine":
+        X = np.ascontiguousarray(X[:, :2])
+        Y = np.ascontiguousarray(Y[:, :2])
+
+    metric_kwargs = _get_metric_params_list(metric, n_features)[0]
+
+    # Reference for argkmin results
+    if metric == "euclidean":
+        # Compare to scikit-learn GEMM optimized implementation
+        dist_matrix = euclidean_distances(X, Y)
+    else:
+        dist_matrix = cdist(X, Y, metric=metric, **metric_kwargs)
+    # Taking argkmin (indices of the k smallest values)
+    argkmin_indices_ref = np.argsort(dist_matrix, axis=1)[:, :k]
+    # Getting the associated distances
+    argkmin_distances_ref = np.zeros(argkmin_indices_ref.shape, dtype=np.float64)
+    for row_idx in range(argkmin_indices_ref.shape[0]):
+        argkmin_distances_ref[row_idx] = dist_matrix[
+            row_idx, argkmin_indices_ref[row_idx]
+        ]
+
+    for _X, _Y in itertools.product((X, X_csr), (Y, Y_csr)):
+        argkmin_distances, argkmin_indices = ArgKmin.compute(
+            _X,
+            _Y,
+            k,
+            metric=metric,
+            metric_kwargs=metric_kwargs,
+            return_distance=True,
+            # So as to have more than a chunk, forcing parallelism.
+            chunk_size=n_samples // 4,
+            strategy=strategy,
+        )
+
+        ASSERT_RESULT[(ArgKmin, dtype)](
+            argkmin_distances,
+            argkmin_distances_ref,
+            argkmin_indices,
+            argkmin_indices_ref,
+        )
+
+
+@pytest.mark.parametrize("metric", CDIST_PAIRWISE_DISTANCES_REDUCTION_COMMON_METRICS)
+@pytest.mark.parametrize("strategy", ("parallel_on_X", "parallel_on_Y"))
+@pytest.mark.parametrize("dtype", [np.float64, np.float32])
+def test_pairwise_distances_radius_neighbors(
+    global_random_seed,
+    metric,
+    strategy,
+    dtype,
+    n_queries=5,
+    n_samples=100,
+):
+    rng = np.random.RandomState(global_random_seed)
+    n_features = rng.choice([50, 500])
+    translation = rng.choice([0, 1e6])
+    spread = 1000
+    X = translation + rng.rand(n_queries, n_features).astype(dtype) * spread
+    Y = translation + rng.rand(n_samples, n_features).astype(dtype) * spread
+
+    metric_kwargs = _get_metric_params_list(
+        metric, n_features, seed=global_random_seed
+    )[0]
+
+    # Reference for argkmin results
+    if metric == "euclidean":
+        # Compare to scikit-learn GEMM optimized implementation
+        dist_matrix = euclidean_distances(X, Y)
+    else:
+        dist_matrix = cdist(X, Y, metric=metric, **metric_kwargs)
+
+    radius = _non_trivial_radius(precomputed_dists=dist_matrix)
+
+    # Getting the neighbors for a given radius
+    neigh_indices_ref = []
+    neigh_distances_ref = []
+
+    for row in dist_matrix:
+        ind = np.arange(row.shape[0])[row <= radius]
+        dist = row[ind]
+
+        sort = np.argsort(dist)
+        ind, dist = ind[sort], dist[sort]
+
+        neigh_indices_ref.append(ind)
+        neigh_distances_ref.append(dist)
+
+    neigh_distances, neigh_indices = RadiusNeighbors.compute(
+        X,
+        Y,
+        radius,
+        metric=metric,
+        metric_kwargs=metric_kwargs,
+        return_distance=True,
+        # So as to have more than a chunk, forcing parallelism.
+        chunk_size=n_samples // 4,
+        strategy=strategy,
+        sort_results=True,
+    )
+
+    ASSERT_RESULT[(RadiusNeighbors, dtype)](
+        neigh_distances, neigh_distances_ref, neigh_indices, neigh_indices_ref, radius
+    )
+
+
+@pytest.mark.parametrize("Dispatcher", [ArgKmin, RadiusNeighbors])
+@pytest.mark.parametrize("metric", ["manhattan", "euclidean"])
+@pytest.mark.parametrize("dtype", [np.float64, np.float32])
+def test_memmap_backed_data(
+    metric,
+    Dispatcher,
+    dtype,
+):
+    """Check that the results do not depend on the datasets writability."""
+    rng = np.random.RandomState(0)
+    spread = 100
+    n_samples, n_features = 128, 10
+    X = rng.rand(n_samples, n_features).astype(dtype) * spread
+    Y = rng.rand(n_samples, n_features).astype(dtype) * spread
+
+    # Create read only datasets
+    X_mm, Y_mm = create_memmap_backed_data([X, Y])
+
+    if Dispatcher is ArgKmin:
+        parameter = 10
+        check_parameters = {}
+        compute_parameters = {}
+    else:
+        # Scaling the radius slightly with the numbers of dimensions
+        radius = 10 ** np.log(n_features)
+        parameter = radius
+        check_parameters = {"radius": radius}
+        compute_parameters = {"sort_results": True}
+
+    ref_dist, ref_indices = Dispatcher.compute(
+        X,
+        Y,
+        parameter,
+        metric=metric,
+        return_distance=True,
+        **compute_parameters,
+    )
+
+    dist_mm, indices_mm = Dispatcher.compute(
+        X_mm,
+        Y_mm,
+        parameter,
+        metric=metric,
+        return_distance=True,
+        **compute_parameters,
+    )
+
+    ASSERT_RESULT[(Dispatcher, dtype)](
+        ref_dist, dist_mm, ref_indices, indices_mm, **check_parameters
+    )
+
+
+@pytest.mark.parametrize("dtype", [np.float64, np.float32])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sqeuclidean_row_norms(
+    global_random_seed,
+    dtype,
+    csr_container,
+):
+    rng = np.random.RandomState(global_random_seed)
+    spread = 100
+    n_samples = rng.choice([97, 100, 101, 1000])
+    n_features = rng.choice([5, 10, 100])
+    num_threads = rng.choice([1, 2, 8])
+    X = rng.rand(n_samples, n_features).astype(dtype) * spread
+
+    X_csr = csr_container(X)
+
+    sq_row_norm_reference = np.linalg.norm(X, axis=1) ** 2
+    sq_row_norm = sqeuclidean_row_norms(X, num_threads=num_threads)
+
+    sq_row_norm_csr = sqeuclidean_row_norms(X_csr, num_threads=num_threads)
+
+    assert_allclose(sq_row_norm_reference, sq_row_norm)
+    assert_allclose(sq_row_norm_reference, sq_row_norm_csr)
+
+    with pytest.raises(ValueError):
+        X = np.asfortranarray(X)
+        sqeuclidean_row_norms(X, num_threads=num_threads)
+
+
+def test_argkmin_classmode_strategy_consistent():
+    rng = np.random.RandomState(1)
+    X = rng.rand(100, 10)
+    Y = rng.rand(100, 10)
+    k = 5
+    metric = "manhattan"
+
+    weights = "uniform"
+    Y_labels = rng.randint(low=0, high=10, size=100)
+    unique_Y_labels = np.unique(Y_labels)
+    results_X = ArgKminClassMode.compute(
+        X=X,
+        Y=Y,
+        k=k,
+        metric=metric,
+        weights=weights,
+        Y_labels=Y_labels,
+        unique_Y_labels=unique_Y_labels,
+        strategy="parallel_on_X",
+    )
+    results_Y = ArgKminClassMode.compute(
+        X=X,
+        Y=Y,
+        k=k,
+        metric=metric,
+        weights=weights,
+        Y_labels=Y_labels,
+        unique_Y_labels=unique_Y_labels,
+        strategy="parallel_on_Y",
+    )
+    assert_array_equal(results_X, results_Y)
+
+
+@pytest.mark.parametrize("outlier_label", [None, 0, 3, 6, 9])
+def test_radius_neighbors_classmode_strategy_consistent(outlier_label):
+    rng = np.random.RandomState(1)
+    X = rng.rand(100, 10)
+    Y = rng.rand(100, 10)
+    radius = 5
+    metric = "manhattan"
+
+    weights = "uniform"
+    Y_labels = rng.randint(low=0, high=10, size=100)
+    unique_Y_labels = np.unique(Y_labels)
+    results_X = RadiusNeighborsClassMode.compute(
+        X=X,
+        Y=Y,
+        radius=radius,
+        metric=metric,
+        weights=weights,
+        Y_labels=Y_labels,
+        unique_Y_labels=unique_Y_labels,
+        outlier_label=outlier_label,
+        strategy="parallel_on_X",
+    )
+    results_Y = RadiusNeighborsClassMode.compute(
+        X=X,
+        Y=Y,
+        radius=radius,
+        metric=metric,
+        weights=weights,
+        Y_labels=Y_labels,
+        unique_Y_labels=unique_Y_labels,
+        outlier_label=outlier_label,
+        strategy="parallel_on_Y",
+    )
+    assert_allclose(results_X, results_Y)
diff --git a/sklearn/metrics/tests/test_ranking.py b/sklearn/metrics/tests/test_ranking.py
index fd32e2cc0b860..7d740249f8aba 100644
--- a/sklearn/metrics/tests/test_ranking.py
+++ b/sklearn/metrics/tests/test_ranking.py
@@ -1,46 +1,58 @@
+import math
 import re
-import pytest
-import numpy as np
-import warnings
-from scipy.sparse import csr_matrix
 
-from sklearn import datasets
-from sklearn import svm
+import numpy as np
+import pytest
+from scipy import stats
 
-from sklearn.utils.extmath import softmax
+from sklearn import datasets, svm
 from sklearn.datasets import make_multilabel_classification
-from sklearn.random_projection import _sparse_random_matrix
-from sklearn.utils.validation import check_array, check_consistent_length
-from sklearn.utils.validation import check_random_state
-
-from sklearn.utils._testing import assert_allclose
-from sklearn.utils._testing import assert_almost_equal
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils._testing import assert_array_almost_equal
-from sklearn.utils._testing import assert_warns
-
-from sklearn.metrics import accuracy_score
-from sklearn.metrics import auc
-from sklearn.metrics import average_precision_score
-from sklearn.metrics import coverage_error
-from sklearn.metrics import det_curve
-from sklearn.metrics import label_ranking_average_precision_score
-from sklearn.metrics import precision_recall_curve
-from sklearn.metrics import label_ranking_loss
-from sklearn.metrics import roc_auc_score
-from sklearn.metrics import roc_curve
-from sklearn.metrics._ranking import _ndcg_sample_scores, _dcg_sample_scores
-from sklearn.metrics import ndcg_score, dcg_score
-from sklearn.metrics import top_k_accuracy_score
-
 from sklearn.exceptions import UndefinedMetricWarning
-from sklearn.model_selection import train_test_split
 from sklearn.linear_model import LogisticRegression
-
+from sklearn.metrics import (
+    accuracy_score,
+    auc,
+    average_precision_score,
+    coverage_error,
+    dcg_score,
+    det_curve,
+    label_ranking_average_precision_score,
+    label_ranking_loss,
+    ndcg_score,
+    precision_recall_curve,
+    roc_auc_score,
+    roc_curve,
+    top_k_accuracy_score,
+)
+from sklearn.metrics._ranking import _dcg_sample_scores, _ndcg_sample_scores
+from sklearn.model_selection import train_test_split
+from sklearn.preprocessing import label_binarize
+from sklearn.random_projection import _sparse_random_matrix
+from sklearn.utils._testing import (
+    _convert_container,
+    assert_allclose,
+    assert_almost_equal,
+    assert_array_almost_equal,
+    assert_array_equal,
+)
+from sklearn.utils.extmath import softmax
+from sklearn.utils.fixes import CSR_CONTAINERS
+from sklearn.utils.validation import (
+    check_array,
+    check_consistent_length,
+    check_random_state,
+)
 
 ###############################################################################
 # Utilities for testing
 
+CURVE_FUNCS = [
+    det_curve,
+    precision_recall_curve,
+    roc_curve,
+]
+
+
 def make_prediction(dataset=None, binary=False):
     """Make some classification predictions on a toy dataset using a SVC
 
@@ -72,22 +84,23 @@ def make_prediction(dataset=None, binary=False):
     X = np.c_[X, rng.randn(n_samples, 200 * n_features)]
 
     # run classifier, get class probabilities and label predictions
-    clf = svm.SVC(kernel='linear', probability=True, random_state=0)
-    probas_pred = clf.fit(X[:half], y[:half]).predict_proba(X[half:])
+    clf = svm.SVC(kernel="linear", probability=True, random_state=0)
+    y_score = clf.fit(X[:half], y[:half]).predict_proba(X[half:])
 
     if binary:
         # only interested in probabilities of the positive case
         # XXX: do we really want a special API for the binary case?
-        probas_pred = probas_pred[:, 1]
+        y_score = y_score[:, 1]
 
     y_pred = clf.predict(X[half:])
     y_true = y[half:]
-    return y_true, y_pred, probas_pred
+    return y_true, y_pred, y_score
 
 
 ###############################################################################
 # Tests
 
+
 def _auc(y_true, y_score):
     """Alternative implementation to check for correctness of
     `roc_auc_score`."""
@@ -127,7 +140,7 @@ def _average_precision(y_true, y_score):
             for j in range(0, i + 1):
                 if y_true[j] == pos_label:
                     prec += 1.0
-            prec /= (i + 1.0)
+            prec /= i + 1.0
             score += prec
 
     return score / n_pos
@@ -180,17 +193,16 @@ def _partial_roc(y_true, y_predict, max_fpr):
     return 0.5 * (1 + (partial_auc - min_area) / (max_area - min_area))
 
 
-@pytest.mark.parametrize('drop', [True, False])
+@pytest.mark.parametrize("drop", [True, False])
 def test_roc_curve(drop):
     # Test Area under Receiver Operating Characteristic (ROC) curve
-    y_true, _, probas_pred = make_prediction(binary=True)
-    expected_auc = _auc(y_true, probas_pred)
+    y_true, _, y_score = make_prediction(binary=True)
+    expected_auc = _auc(y_true, y_score)
 
-    fpr, tpr, thresholds = roc_curve(y_true, probas_pred,
-                                     drop_intermediate=drop)
+    fpr, tpr, thresholds = roc_curve(y_true, y_score, drop_intermediate=drop)
     roc_auc = auc(fpr, tpr)
     assert_array_almost_equal(roc_auc, expected_auc, decimal=2)
-    assert_almost_equal(roc_auc, roc_auc_score(y_true, probas_pred))
+    assert_almost_equal(roc_auc, roc_auc_score(y_true, y_score))
     assert fpr.shape == tpr.shape
     assert fpr.shape == thresholds.shape
 
@@ -211,13 +223,13 @@ def test_roc_curve_end_points():
 def test_roc_returns_consistency():
     # Test whether the returned threshold matches up with tpr
     # make small toy dataset
-    y_true, _, probas_pred = make_prediction(binary=True)
-    fpr, tpr, thresholds = roc_curve(y_true, probas_pred)
+    y_true, _, y_score = make_prediction(binary=True)
+    fpr, tpr, thresholds = roc_curve(y_true, y_score)
 
     # use the given thresholds to determine the tpr
     tpr_correct = []
     for t in thresholds:
-        tp = np.sum((probas_pred >= t) & y_true)
+        tp = np.sum((y_score >= t) & y_true)
         p = np.sum(y_true)
         tpr_correct.append(1.0 * tp / p)
 
@@ -229,17 +241,17 @@ def test_roc_returns_consistency():
 
 def test_roc_curve_multi():
     # roc_curve not applicable for multi-class problems
-    y_true, _, probas_pred = make_prediction(binary=False)
+    y_true, _, y_score = make_prediction(binary=False)
 
     with pytest.raises(ValueError):
-        roc_curve(y_true, probas_pred)
+        roc_curve(y_true, y_score)
 
 
 def test_roc_curve_confidence():
     # roc_curve for confidence scores
-    y_true, _, probas_pred = make_prediction(binary=True)
+    y_true, _, y_score = make_prediction(binary=True)
 
-    fpr, tpr, thresholds = roc_curve(y_true, probas_pred - 0.5)
+    fpr, tpr, thresholds = roc_curve(y_true, y_score - 0.5)
     roc_auc = auc(fpr, tpr)
     assert_array_almost_equal(roc_auc, 0.90, decimal=2)
     assert fpr.shape == tpr.shape
@@ -248,7 +260,7 @@ def test_roc_curve_confidence():
 
 def test_roc_curve_hard():
     # roc_curve for hard decisions
-    y_true, pred, probas_pred = make_prediction(binary=True)
+    y_true, pred, y_score = make_prediction(binary=True)
 
     # always predict one
     trivial_pred = np.ones(y_true.shape)
@@ -278,17 +290,23 @@ def test_roc_curve_one_label():
     y_true = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
     y_pred = [0, 1, 0, 1, 0, 1, 0, 1, 0, 1]
     # assert there are warnings
-    w = UndefinedMetricWarning
-    fpr, tpr, thresholds = assert_warns(w, roc_curve, y_true, y_pred)
+    expected_message = (
+        "No negative samples in y_true, false positive value should be meaningless"
+    )
+    with pytest.warns(UndefinedMetricWarning, match=expected_message):
+        fpr, tpr, thresholds = roc_curve(y_true, y_pred)
+
     # all true labels, all fpr should be nan
     assert_array_equal(fpr, np.full(len(thresholds), np.nan))
     assert fpr.shape == tpr.shape
     assert fpr.shape == thresholds.shape
 
     # assert there are warnings
-    fpr, tpr, thresholds = assert_warns(w, roc_curve,
-                                        [1 - x for x in y_true],
-                                        y_pred)
+    expected_message = (
+        "No positive samples in y_true, true positive value should be meaningless"
+    )
+    with pytest.warns(UndefinedMetricWarning, match=expected_message):
+        fpr, tpr, thresholds = roc_curve([1 - x for x in y_true], y_pred)
     # all negative labels, all tpr should be nan
     assert_array_equal(tpr, np.full(len(thresholds), np.nan))
     assert fpr.shape == tpr.shape
@@ -303,7 +321,7 @@ def test_roc_curve_toydata():
     roc_auc = roc_auc_score(y_true, y_score)
     assert_array_almost_equal(tpr, [0, 0, 1])
     assert_array_almost_equal(fpr, [0, 1, 1])
-    assert_almost_equal(roc_auc, 1.)
+    assert_almost_equal(roc_auc, 1.0)
 
     y_true = [0, 1]
     y_score = [1, 0]
@@ -311,7 +329,7 @@ def test_roc_curve_toydata():
     roc_auc = roc_auc_score(y_true, y_score)
     assert_array_almost_equal(tpr, [0, 1, 1])
     assert_array_almost_equal(fpr, [0, 0, 1])
-    assert_almost_equal(roc_auc, 0.)
+    assert_almost_equal(roc_auc, 0.0)
 
     y_true = [1, 0]
     y_score = [1, 1]
@@ -327,7 +345,7 @@ def test_roc_curve_toydata():
     roc_auc = roc_auc_score(y_true, y_score)
     assert_array_almost_equal(tpr, [0, 0, 1])
     assert_array_almost_equal(fpr, [0, 1, 1])
-    assert_almost_equal(roc_auc, 1.)
+    assert_almost_equal(roc_auc, 1.0)
 
     y_true = [1, 0]
     y_score = [0.5, 0.5]
@@ -335,43 +353,61 @@ def test_roc_curve_toydata():
     roc_auc = roc_auc_score(y_true, y_score)
     assert_array_almost_equal(tpr, [0, 1])
     assert_array_almost_equal(fpr, [0, 1])
-    assert_almost_equal(roc_auc, .5)
+    assert_almost_equal(roc_auc, 0.5)
 
+    # case with no positive samples
     y_true = [0, 0]
     y_score = [0.25, 0.75]
     # assert UndefinedMetricWarning because of no positive sample in y_true
-    tpr, fpr, _ = assert_warns(UndefinedMetricWarning, roc_curve, y_true,
-                               y_score)
-    with pytest.raises(ValueError):
-        roc_auc_score(y_true, y_score)
-    assert_array_almost_equal(tpr, [0., 0.5, 1.])
+    expected_message = (
+        "No positive samples in y_true, true positive value should be meaningless"
+    )
+    with pytest.warns(UndefinedMetricWarning, match=expected_message):
+        tpr, fpr, _ = roc_curve(y_true, y_score)
+    assert_array_almost_equal(tpr, [0.0, 0.5, 1.0])
     assert_array_almost_equal(fpr, [np.nan, np.nan, np.nan])
+    expected_message = (
+        "Only one class is present in y_true. "
+        "ROC AUC score is not defined in that case."
+    )
+    with pytest.warns(UndefinedMetricWarning, match=expected_message):
+        auc = roc_auc_score(y_true, y_score)
+    assert math.isnan(auc)
 
+    # case with no negative samples
     y_true = [1, 1]
     y_score = [0.25, 0.75]
     # assert UndefinedMetricWarning because of no negative sample in y_true
-    tpr, fpr, _ = assert_warns(UndefinedMetricWarning, roc_curve, y_true,
-                               y_score)
-    with pytest.raises(ValueError):
-        roc_auc_score(y_true, y_score)
+    expected_message = (
+        "No negative samples in y_true, false positive value should be meaningless"
+    )
+    with pytest.warns(UndefinedMetricWarning, match=expected_message):
+        tpr, fpr, _ = roc_curve(y_true, y_score)
     assert_array_almost_equal(tpr, [np.nan, np.nan, np.nan])
-    assert_array_almost_equal(fpr, [0., 0.5, 1.])
+    assert_array_almost_equal(fpr, [0.0, 0.5, 1.0])
+    expected_message = (
+        "Only one class is present in y_true. "
+        "ROC AUC score is not defined in that case."
+    )
+    with pytest.warns(UndefinedMetricWarning, match=expected_message):
+        auc = roc_auc_score(y_true, y_score)
+    assert math.isnan(auc)
 
     # Multi-label classification task
     y_true = np.array([[0, 1], [0, 1]])
     y_score = np.array([[0, 1], [0, 1]])
-    with pytest.raises(ValueError):
+    with pytest.warns(UndefinedMetricWarning, match=expected_message):
         roc_auc_score(y_true, y_score, average="macro")
-    with pytest.raises(ValueError):
+    with pytest.warns(UndefinedMetricWarning, match=expected_message):
         roc_auc_score(y_true, y_score, average="weighted")
-    assert_almost_equal(roc_auc_score(y_true, y_score, average="samples"), 1.)
-    assert_almost_equal(roc_auc_score(y_true, y_score, average="micro"), 1.)
+    assert_almost_equal(roc_auc_score(y_true, y_score, average="samples"), 1.0)
+    assert_almost_equal(roc_auc_score(y_true, y_score, average="micro"), 1.0)
 
     y_true = np.array([[0, 1], [0, 1]])
     y_score = np.array([[0, 1], [1, 0]])
-    with pytest.raises(ValueError):
+    with pytest.warns(UndefinedMetricWarning, match=expected_message):
         roc_auc_score(y_true, y_score, average="macro")
-    with pytest.raises(ValueError):
+    with pytest.warns(UndefinedMetricWarning, match=expected_message):
         roc_auc_score(y_true, y_score, average="weighted")
     assert_almost_equal(roc_auc_score(y_true, y_score, average="samples"), 0.5)
     assert_almost_equal(roc_auc_score(y_true, y_score, average="micro"), 0.5)
@@ -385,27 +421,24 @@ def test_roc_curve_toydata():
 
     y_true = np.array([[1, 0], [0, 1]])
     y_score = np.array([[0.5, 0.5], [0.5, 0.5]])
-    assert_almost_equal(roc_auc_score(y_true, y_score, average="macro"), .5)
-    assert_almost_equal(roc_auc_score(y_true, y_score, average="weighted"), .5)
-    assert_almost_equal(roc_auc_score(y_true, y_score, average="samples"), .5)
-    assert_almost_equal(roc_auc_score(y_true, y_score, average="micro"), .5)
+    assert_almost_equal(roc_auc_score(y_true, y_score, average="macro"), 0.5)
+    assert_almost_equal(roc_auc_score(y_true, y_score, average="weighted"), 0.5)
+    assert_almost_equal(roc_auc_score(y_true, y_score, average="samples"), 0.5)
+    assert_almost_equal(roc_auc_score(y_true, y_score, average="micro"), 0.5)
 
 
 def test_roc_curve_drop_intermediate():
     # Test that drop_intermediate drops the correct thresholds
     y_true = [0, 0, 0, 0, 1, 1]
-    y_score = [0., 0.2, 0.5, 0.6, 0.7, 1.0]
+    y_score = [0.0, 0.2, 0.5, 0.6, 0.7, 1.0]
     tpr, fpr, thresholds = roc_curve(y_true, y_score, drop_intermediate=True)
-    assert_array_almost_equal(thresholds, [2., 1., 0.7, 0.])
+    assert_array_almost_equal(thresholds, [np.inf, 1.0, 0.7, 0.0])
 
     # Test dropping thresholds with repeating scores
-    y_true = [0, 0, 0, 0, 0, 0, 0,
-              1, 1, 1, 1, 1, 1]
-    y_score = [0., 0.1, 0.6, 0.6, 0.7, 0.8, 0.9,
-               0.6, 0.7, 0.8, 0.9, 0.9, 1.0]
+    y_true = [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
+    y_score = [0.0, 0.1, 0.6, 0.6, 0.7, 0.8, 0.9, 0.6, 0.7, 0.8, 0.9, 0.9, 1.0]
     tpr, fpr, thresholds = roc_curve(y_true, y_score, drop_intermediate=True)
-    assert_array_almost_equal(thresholds,
-                              [2.0, 1.0, 0.9, 0.7, 0.6, 0.])
+    assert_array_almost_equal(thresholds, [np.inf, 1.0, 0.9, 0.7, 0.6, 0.0])
 
 
 def test_roc_curve_fpr_tpr_increasing():
@@ -451,24 +484,26 @@ def test_auc_errors():
     # x is not in order
     x = [2, 1, 3, 4]
     y = [5, 6, 7, 8]
-    error_message = ("x is neither increasing nor decreasing : "
-                     "{}".format(np.array(x)))
+    error_message = "x is neither increasing nor decreasing : {}".format(np.array(x))
     with pytest.raises(ValueError, match=re.escape(error_message)):
         auc(x, y)
 
 
 @pytest.mark.parametrize(
     "y_true, labels",
-    [(np.array([0, 1, 0, 2]), [0, 1, 2]),
-     (np.array([0, 1, 0, 2]), None),
-     (["a", "b", "a", "c"], ["a", "b", "c"]),
-     (["a", "b", "a", "c"], None)]
+    [
+        (np.array([0, 1, 0, 2]), [0, 1, 2]),
+        (np.array([0, 1, 0, 2]), None),
+        (["a", "b", "a", "c"], ["a", "b", "c"]),
+        (["a", "b", "a", "c"], None),
+    ],
 )
 def test_multiclass_ovo_roc_auc_toydata(y_true, labels):
     # Tests the one-vs-one multiclass ROC AUC algorithm
     # on a small example, representative of an expected use case.
     y_scores = np.array(
-        [[0.1, 0.8, 0.1], [0.3, 0.4, 0.3], [0.35, 0.5, 0.15], [0, 0.2, 0.8]])
+        [[0.1, 0.8, 0.1], [0.3, 0.4, 0.3], [0.35, 0.5, 0.15], [0, 0.2, 0.8]]
+    )
 
     # Used to compute the expected output.
     # Consider labels 0 and 1:
@@ -489,11 +524,11 @@ def test_multiclass_ovo_roc_auc_toydata(y_true, labels):
     average_score_12 = (score_12 + score_21) / 2
 
     # Unweighted, one-vs-one multiclass ROC AUC algorithm
-    ovo_unweighted_score = (
-        average_score_01 + average_score_02 + average_score_12) / 3
+    ovo_unweighted_score = (average_score_01 + average_score_02 + average_score_12) / 3
     assert_almost_equal(
         roc_auc_score(y_true, y_scores, labels=labels, multi_class="ovo"),
-        ovo_unweighted_score)
+        ovo_unweighted_score,
+    )
 
     # Weighted, one-vs-one multiclass ROC AUC algorithm
     # Each term is weighted by the prevalence for the positive label.
@@ -502,22 +537,31 @@ def test_multiclass_ovo_roc_auc_toydata(y_true, labels):
     ovo_weighted_score = np.average(pair_scores, weights=prevalence)
     assert_almost_equal(
         roc_auc_score(
-            y_true,
-            y_scores,
-            labels=labels,
-            multi_class="ovo",
-            average="weighted"), ovo_weighted_score)
+            y_true, y_scores, labels=labels, multi_class="ovo", average="weighted"
+        ),
+        ovo_weighted_score,
+    )
 
+    # Check that average=None raises NotImplemented error
+    error_message = "average=None is not implemented for multi_class='ovo'."
+    with pytest.raises(NotImplementedError, match=error_message):
+        roc_auc_score(y_true, y_scores, labels=labels, multi_class="ovo", average=None)
 
-@pytest.mark.parametrize("y_true, labels",
-                         [(np.array([0, 2, 0, 2]), [0, 1, 2]),
-                          (np.array(['a', 'd', 'a', 'd']), ['a', 'b', 'd'])])
+
+@pytest.mark.parametrize(
+    "y_true, labels",
+    [
+        (np.array([0, 2, 0, 2]), [0, 1, 2]),
+        (np.array(["a", "d", "a", "d"]), ["a", "b", "d"]),
+    ],
+)
 def test_multiclass_ovo_roc_auc_toydata_binary(y_true, labels):
     # Tests the one-vs-one multiclass ROC AUC algorithm for binary y_true
     #
     # on a small example, representative of an expected use case.
     y_scores = np.array(
-        [[0.2, 0.0, 0.8], [0.6, 0.0, 0.4], [0.55, 0.0, 0.45], [0.4, 0.0, 0.6]])
+        [[0.2, 0.0, 0.8], [0.6, 0.0, 0.4], [0.55, 0.0, 0.45], [0.4, 0.0, 0.6]]
+    )
 
     # Used to compute the expected output.
     # Consider labels 0 and 1:
@@ -528,102 +572,239 @@ def test_multiclass_ovo_roc_auc_toydata_binary(y_true, labels):
     ovo_score = (score_01 + score_10) / 2
 
     assert_almost_equal(
-        roc_auc_score(y_true, y_scores, labels=labels, multi_class='ovo'),
-        ovo_score)
+        roc_auc_score(y_true, y_scores, labels=labels, multi_class="ovo"), ovo_score
+    )
 
     # Weighted, one-vs-one multiclass ROC AUC algorithm
     assert_almost_equal(
-        roc_auc_score(y_true, y_scores, labels=labels, multi_class='ovo',
-                      average="weighted"), ovo_score)
+        roc_auc_score(
+            y_true, y_scores, labels=labels, multi_class="ovo", average="weighted"
+        ),
+        ovo_score,
+    )
 
 
 @pytest.mark.parametrize(
     "y_true, labels",
-    [(np.array([0, 1, 2, 2]), None),
-     (["a", "b", "c", "c"], None),
-     ([0, 1, 2, 2], [0, 1, 2]),
-     (["a", "b", "c", "c"], ["a", "b", "c"])])
+    [
+        (np.array([0, 1, 2, 2]), None),
+        (["a", "b", "c", "c"], None),
+        ([0, 1, 2, 2], [0, 1, 2]),
+        (["a", "b", "c", "c"], ["a", "b", "c"]),
+    ],
+)
 def test_multiclass_ovr_roc_auc_toydata(y_true, labels):
     # Tests the unweighted, one-vs-rest multiclass ROC AUC algorithm
     # on a small example, representative of an expected use case.
     y_scores = np.array(
-        [[1.0, 0.0, 0.0], [0.1, 0.5, 0.4], [0.1, 0.1, 0.8], [0.3, 0.3, 0.4]])
+        [[1.0, 0.0, 0.0], [0.1, 0.5, 0.4], [0.1, 0.1, 0.8], [0.3, 0.3, 0.4]]
+    )
     # Compute the expected result by individually computing the 'one-vs-rest'
     # ROC AUC scores for classes 0, 1, and 2.
     out_0 = roc_auc_score([1, 0, 0, 0], y_scores[:, 0])
     out_1 = roc_auc_score([0, 1, 0, 0], y_scores[:, 1])
     out_2 = roc_auc_score([0, 0, 1, 1], y_scores[:, 2])
-    result_unweighted = (out_0 + out_1 + out_2) / 3.
+    assert_almost_equal(
+        roc_auc_score(y_true, y_scores, multi_class="ovr", labels=labels, average=None),
+        [out_0, out_1, out_2],
+    )
 
+    # Compute unweighted results (default behaviour is average="macro")
+    result_unweighted = (out_0 + out_1 + out_2) / 3.0
     assert_almost_equal(
         roc_auc_score(y_true, y_scores, multi_class="ovr", labels=labels),
-        result_unweighted)
+        result_unweighted,
+    )
 
     # Tests the weighted, one-vs-rest multiclass ROC AUC algorithm
     # on the same input (Provost & Domingos, 2000)
     result_weighted = out_0 * 0.25 + out_1 * 0.25 + out_2 * 0.5
     assert_almost_equal(
         roc_auc_score(
-            y_true,
-            y_scores,
-            multi_class="ovr",
-            labels=labels,
-            average="weighted"), result_weighted)
+            y_true, y_scores, multi_class="ovr", labels=labels, average="weighted"
+        ),
+        result_weighted,
+    )
+
+
+@pytest.mark.parametrize(
+    "multi_class, average",
+    [
+        ("ovr", "macro"),
+        ("ovr", "micro"),
+        ("ovo", "macro"),
+    ],
+)
+def test_perfect_imperfect_chance_multiclass_roc_auc(multi_class, average):
+    y_true = np.array([3, 1, 2, 0])
+
+    # Perfect classifier (from a ranking point of view) has roc_auc_score = 1.0
+    y_perfect = [
+        [0.0, 0.0, 0.0, 1.0],
+        [0.0, 1.0, 0.0, 0.0],
+        [0.0, 0.0, 1.0, 0.0],
+        [0.75, 0.05, 0.05, 0.15],
+    ]
+    assert_almost_equal(
+        roc_auc_score(y_true, y_perfect, multi_class=multi_class, average=average),
+        1.0,
+    )
+
+    # Imperfect classifier has roc_auc_score < 1.0
+    y_imperfect = [
+        [0.0, 0.0, 0.0, 1.0],
+        [0.0, 1.0, 0.0, 0.0],
+        [0.0, 0.0, 1.0, 0.0],
+        [0.0, 0.0, 0.0, 1.0],
+    ]
+    assert (
+        roc_auc_score(y_true, y_imperfect, multi_class=multi_class, average=average)
+        < 1.0
+    )
+
+    # Chance level classifier has roc_auc_score = 5.0
+    y_chance = 0.25 * np.ones((4, 4))
+    assert roc_auc_score(
+        y_true, y_chance, multi_class=multi_class, average=average
+    ) == pytest.approx(0.5)
+
+
+def test_micro_averaged_ovr_roc_auc(global_random_seed):
+    seed = global_random_seed
+    # Let's generate a set of random predictions and matching true labels such
+    # that the predictions are not perfect. To make the problem more interesting,
+    # we use an imbalanced class distribution (by using different parameters
+    # in the Dirichlet prior (conjugate prior of the multinomial distribution).
+    y_pred = stats.dirichlet.rvs([2.0, 1.0, 0.5], size=1000, random_state=seed)
+    y_true = np.asarray(
+        [
+            stats.multinomial.rvs(n=1, p=y_pred_i, random_state=seed).argmax()
+            for y_pred_i in y_pred
+        ]
+    )
+    y_onehot = label_binarize(y_true, classes=[0, 1, 2])
+    fpr, tpr, _ = roc_curve(y_onehot.ravel(), y_pred.ravel())
+    roc_auc_by_hand = auc(fpr, tpr)
+    roc_auc_auto = roc_auc_score(y_true, y_pred, multi_class="ovr", average="micro")
+    assert roc_auc_by_hand == pytest.approx(roc_auc_auto)
 
 
 @pytest.mark.parametrize(
     "msg, y_true, labels",
-    [("Parameter 'labels' must be unique", np.array([0, 1, 2, 2]), [0, 2, 0]),
-     ("Parameter 'labels' must be unique", np.array(["a", "b", "c", "c"]),
-      ["a", "a", "b"]),
-     ("Number of classes in y_true not equal to the number of columns "
-      "in 'y_score'", np.array([0, 2, 0, 2]), None),
-     ("Parameter 'labels' must be ordered", np.array(["a", "b", "c", "c"]),
-      ["a", "c", "b"]),
-     ("Number of given labels, 2, not equal to the number of columns in "
-      "'y_score', 3",
-      np.array([0, 1, 2, 2]), [0, 1]),
-     ("Number of given labels, 2, not equal to the number of columns in "
-      "'y_score', 3",
-      np.array(["a", "b", "c", "c"]), ["a", "b"]),
-     ("Number of given labels, 4, not equal to the number of columns in "
-      "'y_score', 3",
-      np.array([0, 1, 2, 2]), [0, 1, 2, 3]),
-     ("Number of given labels, 4, not equal to the number of columns in "
-      "'y_score', 3",
-      np.array(["a", "b", "c", "c"]), ["a", "b", "c", "d"]),
-     ("'y_true' contains labels not in parameter 'labels'",
-      np.array(["a", "b", "c", "e"]), ["a", "b", "c"]),
-     ("'y_true' contains labels not in parameter 'labels'",
-      np.array(["a", "b", "c", "d"]), ["a", "b", "c"]),
-     ("'y_true' contains labels not in parameter 'labels'",
-      np.array([0, 1, 2, 3]), [0, 1, 2])])
+    [
+        ("Parameter 'labels' must be unique", np.array([0, 1, 2, 2]), [0, 2, 0]),
+        (
+            "Parameter 'labels' must be unique",
+            np.array(["a", "b", "c", "c"]),
+            ["a", "a", "b"],
+        ),
+        (
+            (
+                "Number of classes in y_true not equal to the number of columns "
+                "in 'y_score'"
+            ),
+            np.array([0, 2, 0, 2]),
+            None,
+        ),
+        (
+            "Parameter 'labels' must be ordered",
+            np.array(["a", "b", "c", "c"]),
+            ["a", "c", "b"],
+        ),
+        (
+            (
+                "Number of given labels, 2, not equal to the number of columns in "
+                "'y_score', 3"
+            ),
+            np.array([0, 1, 2, 2]),
+            [0, 1],
+        ),
+        (
+            (
+                "Number of given labels, 2, not equal to the number of columns in "
+                "'y_score', 3"
+            ),
+            np.array(["a", "b", "c", "c"]),
+            ["a", "b"],
+        ),
+        (
+            (
+                "Number of given labels, 4, not equal to the number of columns in "
+                "'y_score', 3"
+            ),
+            np.array([0, 1, 2, 2]),
+            [0, 1, 2, 3],
+        ),
+        (
+            (
+                "Number of given labels, 4, not equal to the number of columns in "
+                "'y_score', 3"
+            ),
+            np.array(["a", "b", "c", "c"]),
+            ["a", "b", "c", "d"],
+        ),
+        (
+            "'y_true' contains labels not in parameter 'labels'",
+            np.array(["a", "b", "c", "e"]),
+            ["a", "b", "c"],
+        ),
+        (
+            "'y_true' contains labels not in parameter 'labels'",
+            np.array(["a", "b", "c", "d"]),
+            ["a", "b", "c"],
+        ),
+        (
+            "'y_true' contains labels not in parameter 'labels'",
+            np.array([0, 1, 2, 3]),
+            [0, 1, 2],
+        ),
+    ],
+)
 @pytest.mark.parametrize("multi_class", ["ovo", "ovr"])
-def test_roc_auc_score_multiclass_labels_error(
-        msg, y_true, labels, multi_class):
+def test_roc_auc_score_multiclass_labels_error(msg, y_true, labels, multi_class):
     y_scores = np.array(
-        [[0.1, 0.8, 0.1], [0.3, 0.4, 0.3], [0.35, 0.5, 0.15], [0, 0.2, 0.8]])
+        [[0.1, 0.8, 0.1], [0.3, 0.4, 0.3], [0.35, 0.5, 0.15], [0, 0.2, 0.8]]
+    )
 
     with pytest.raises(ValueError, match=msg):
         roc_auc_score(y_true, y_scores, labels=labels, multi_class=multi_class)
 
 
-@pytest.mark.parametrize("msg, kwargs", [
-    ((r"average must be one of \('macro', 'weighted'\) for "
-      r"multiclass problems"), {"average": "samples", "multi_class": "ovo"}),
-    ((r"average must be one of \('macro', 'weighted'\) for "
-      r"multiclass problems"), {"average": "micro", "multi_class": "ovr"}),
-    ((r"sample_weight is not supported for multiclass one-vs-one "
-      r"ROC AUC, 'sample_weight' must be None in this case"),
-     {"multi_class": "ovo", "sample_weight": []}),
-    ((r"Partial AUC computation not available in multiclass setting, "
-      r"'max_fpr' must be set to `None`, received `max_fpr=0.5` "
-      r"instead"), {"multi_class": "ovo", "max_fpr": 0.5}),
-    ((r"multi_class='ovp' is not supported for multiclass ROC AUC, "
-      r"multi_class must be in \('ovo', 'ovr'\)"),
-     {"multi_class": "ovp"}),
-    (r"multi_class must be in \('ovo', 'ovr'\)", {})
-])
+@pytest.mark.parametrize(
+    "msg, kwargs",
+    [
+        (
+            (
+                r"average must be one of \('macro', 'weighted', None\) for "
+                r"multiclass problems"
+            ),
+            {"average": "samples", "multi_class": "ovo"},
+        ),
+        (
+            (
+                r"average must be one of \('micro', 'macro', 'weighted', None\) for "
+                r"multiclass problems"
+            ),
+            {"average": "samples", "multi_class": "ovr"},
+        ),
+        (
+            (
+                r"sample_weight is not supported for multiclass one-vs-one "
+                r"ROC AUC, 'sample_weight' must be None in this case"
+            ),
+            {"multi_class": "ovo", "sample_weight": []},
+        ),
+        (
+            (
+                r"Partial AUC computation not available in multiclass setting, "
+                r"'max_fpr' must be set to `None`, received `max_fpr=0.5` "
+                r"instead"
+            ),
+            {"multi_class": "ovo", "max_fpr": 0.5},
+        ),
+        (r"multi_class must be in \('ovo', 'ovr'\)", {}),
+    ],
+)
 def test_roc_auc_score_multiclass_error(msg, kwargs):
     # Test that roc_auc_score function returns an error when trying
     # to compute multiclass AUC for parameters where an output
@@ -643,141 +824,152 @@ def test_auc_score_non_binary_class():
     y_pred = rng.rand(10)
     # y_true contains only one class value
     y_true = np.zeros(10, dtype="int")
-    err_msg = "ROC AUC score is not defined"
-    with pytest.raises(ValueError, match=err_msg):
+    warn_message = (
+        "Only one class is present in y_true. "
+        "ROC AUC score is not defined in that case."
+    )
+    with pytest.warns(UndefinedMetricWarning, match=warn_message):
         roc_auc_score(y_true, y_pred)
     y_true = np.ones(10, dtype="int")
-    with pytest.raises(ValueError, match=err_msg):
+    with pytest.warns(UndefinedMetricWarning, match=warn_message):
         roc_auc_score(y_true, y_pred)
     y_true = np.full(10, -1, dtype="int")
-    with pytest.raises(ValueError, match=err_msg):
+    with pytest.warns(UndefinedMetricWarning, match=warn_message):
         roc_auc_score(y_true, y_pred)
 
-    with warnings.catch_warnings(record=True):
-        rng = check_random_state(404)
-        y_pred = rng.rand(10)
-        # y_true contains only one class value
-        y_true = np.zeros(10, dtype="int")
-        with pytest.raises(ValueError, match=err_msg):
-            roc_auc_score(y_true, y_pred)
-        y_true = np.ones(10, dtype="int")
-        with pytest.raises(ValueError, match=err_msg):
-            roc_auc_score(y_true, y_pred)
-        y_true = np.full(10, -1, dtype="int")
-        with pytest.raises(ValueError, match=err_msg):
-            roc_auc_score(y_true, y_pred)
-
-
-def test_binary_clf_curve_multiclass_error():
+
+@pytest.mark.parametrize("curve_func", CURVE_FUNCS)
+def test_binary_clf_curve_multiclass_error(curve_func):
     rng = check_random_state(404)
     y_true = rng.randint(0, 3, size=10)
     y_pred = rng.rand(10)
     msg = "multiclass format is not supported"
-
-    with pytest.raises(ValueError, match=msg):
-        precision_recall_curve(y_true, y_pred)
-
     with pytest.raises(ValueError, match=msg):
-        roc_curve(y_true, y_pred)
+        curve_func(y_true, y_pred)
 
 
-@pytest.mark.parametrize("curve_func", [
-    precision_recall_curve,
-    roc_curve,
-])
+@pytest.mark.parametrize("curve_func", CURVE_FUNCS)
 def test_binary_clf_curve_implicit_pos_label(curve_func):
     # Check that using string class labels raises an informative
     # error for any supported string dtype:
-    msg = ("y_true takes value in {'a', 'b'} and pos_label is "
-           "not specified: either make y_true take "
-           "value in {0, 1} or {-1, 1} or pass pos_label "
-           "explicitly.")
+    msg = (
+        "y_true takes value in {'a', 'b'} and pos_label is "
+        "not specified: either make y_true take "
+        "value in {0, 1} or {-1, 1} or pass pos_label "
+        "explicitly."
+    )
     with pytest.raises(ValueError, match=msg):
-        roc_curve(np.array(["a", "b"], dtype='= 0 and y_score.max() <= 1 else 0
+    threshold = 0.5 if y_score.min() >= 0 and y_score.max() <= 1 else 0
     y_pred = (y_score > threshold).astype(np.int64) if k == 1 else y_true
 
     score = top_k_accuracy_score(y_true, y_score, k=k)
@@ -1650,10 +2037,40 @@ def test_top_k_accuracy_score_binary(y_score, k, true_score):
     assert score == score_acc == pytest.approx(true_score)
 
 
+@pytest.mark.parametrize(
+    "y_true, true_score, labels",
+    [
+        (np.array([0, 1, 1, 2]), 0.75, [0, 1, 2, 3]),
+        (np.array([0, 1, 1, 1]), 0.5, [0, 1, 2, 3]),
+        (np.array([1, 1, 1, 1]), 0.5, [0, 1, 2, 3]),
+        (np.array(["a", "e", "e", "a"]), 0.75, ["a", "b", "d", "e"]),
+    ],
+)
+@pytest.mark.parametrize("labels_as_ndarray", [True, False])
+def test_top_k_accuracy_score_multiclass_with_labels(
+    y_true, true_score, labels, labels_as_ndarray
+):
+    """Test when labels and y_score are multiclass."""
+    if labels_as_ndarray:
+        labels = np.asarray(labels)
+    y_score = np.array(
+        [
+            [0.4, 0.3, 0.2, 0.1],
+            [0.1, 0.3, 0.4, 0.2],
+            [0.4, 0.1, 0.2, 0.3],
+            [0.3, 0.2, 0.4, 0.1],
+        ]
+    )
+
+    score = top_k_accuracy_score(y_true, y_score, k=2, labels=labels)
+    assert score == pytest.approx(true_score)
+
+
 def test_top_k_accuracy_score_increasing():
     # Make sure increasing k leads to a higher score
-    X, y = datasets.make_classification(n_classes=10, n_samples=1000,
-                                        n_informative=10, random_state=0)
+    X, y = datasets.make_classification(
+        n_classes=10, n_samples=1000, n_informative=10, random_state=0
+    )
 
     X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
 
@@ -1662,86 +2079,192 @@ def test_top_k_accuracy_score_increasing():
 
     for X, y in zip((X_train, X_test), (y_train, y_test)):
         scores = [
-            top_k_accuracy_score(y, clf.predict_proba(X), k=k)
-            for k in range(2, 10)
+            top_k_accuracy_score(y, clf.predict_proba(X), k=k) for k in range(2, 10)
         ]
 
         assert np.all(np.diff(scores) > 0)
 
 
-@pytest.mark.parametrize('y_true, k, true_score', [
-    ([0, 1, 2, 3], 1, 0.25),
-    ([0, 1, 2, 3], 2, 0.5),
-    ([0, 1, 2, 3], 3, 1),
-])
+@pytest.mark.parametrize(
+    "y_true, k, true_score",
+    [
+        ([0, 1, 2, 3], 1, 0.25),
+        ([0, 1, 2, 3], 2, 0.5),
+        ([0, 1, 2, 3], 3, 1),
+    ],
+)
 def test_top_k_accuracy_score_ties(y_true, k, true_score):
     # Make sure highest indices labels are chosen first in case of ties
-    y_score = np.array([
-        [5, 5, 7, 0],
-        [1, 5, 5, 5],
-        [0, 0, 3, 3],
-        [1, 1, 1, 1],
-    ])
-    assert top_k_accuracy_score(y_true, y_score,
-                                k=k) == pytest.approx(true_score)
-
-
-@pytest.mark.parametrize('y_true, k', [
-    ([0, 1, 2, 3], 4),
-    ([0, 1, 2, 3], 5),
-])
+    y_score = np.array(
+        [
+            [5, 5, 7, 0],
+            [1, 5, 5, 5],
+            [0, 0, 3, 3],
+            [1, 1, 1, 1],
+        ]
+    )
+    assert top_k_accuracy_score(y_true, y_score, k=k) == pytest.approx(true_score)
+
+
+@pytest.mark.parametrize(
+    "y_true, k",
+    [
+        ([0, 1, 2, 3], 4),
+        ([0, 1, 2, 3], 5),
+    ],
+)
 def test_top_k_accuracy_score_warning(y_true, k):
-    y_score = np.array([
-        [0.4, 0.3, 0.2, 0.1],
-        [0.1, 0.4, 0.3, 0.2],
-        [0.2, 0.1, 0.4, 0.3],
-        [0.3, 0.2, 0.1, 0.4],
-    ])
-    w = UndefinedMetricWarning
-    score = assert_warns(w, top_k_accuracy_score, y_true, y_score, k=k)
+    y_score = np.array(
+        [
+            [0.4, 0.3, 0.2, 0.1],
+            [0.1, 0.4, 0.3, 0.2],
+            [0.2, 0.1, 0.4, 0.3],
+            [0.3, 0.2, 0.1, 0.4],
+        ]
+    )
+    expected_message = (
+        r"'k' \(\d+\) greater than or equal to 'n_classes' \(\d+\) will result in a "
+        "perfect score and is therefore meaningless."
+    )
+    with pytest.warns(UndefinedMetricWarning, match=expected_message):
+        score = top_k_accuracy_score(y_true, y_score, k=k)
     assert score == 1
 
 
-@pytest.mark.parametrize('y_true, labels, msg', [
-    (
-        [0, .57, 1, 2],
-        None,
-        "y type must be 'binary' or 'multiclass', got 'continuous'"
-    ),
-    (
-        [0, 1, 2, 3],
-        None,
-        r"Number of classes in 'y_true' \(4\) not equal to the number of "
-        r"classes in 'y_score' \(3\)."
-    ),
-    (
-        ['c', 'c', 'a', 'b'],
-        ['a', 'b', 'c', 'c'],
-        "Parameter 'labels' must be unique."
-    ),
-    (
-        ['c', 'c', 'a', 'b'],
-        ['a', 'c', 'b'],
-        "Parameter 'labels' must be ordered."
-    ),
-    (
-        [0, 0, 1, 2],
-        [0, 1, 2, 3],
-        r"Number of given labels \(4\) not equal to the number of classes in "
-        r"'y_score' \(3\)."
-    ),
-    (
-        [0, 0, 1, 2],
-        [0, 1, 3],
-        "'y_true' contains labels not in parameter 'labels'."
-    ),
-])
-def test_top_k_accuracy_score_error(y_true, labels, msg):
-    y_score = np.array([
-        [0.2, 0.1, 0.7],
-        [0.4, 0.3, 0.3],
-        [0.3, 0.4, 0.3],
-        [0.4, 0.5, 0.1],
-    ])
+@pytest.mark.parametrize(
+    "y_true, y_score, labels, msg",
+    [
+        (
+            [0, 0.57, 1, 2],
+            [
+                [0.2, 0.1, 0.7],
+                [0.4, 0.3, 0.3],
+                [0.3, 0.4, 0.3],
+                [0.4, 0.5, 0.1],
+            ],
+            None,
+            "y type must be 'binary' or 'multiclass', got 'continuous'",
+        ),
+        (
+            [0, 1, 2, 3],
+            [
+                [0.2, 0.1, 0.7],
+                [0.4, 0.3, 0.3],
+                [0.3, 0.4, 0.3],
+                [0.4, 0.5, 0.1],
+            ],
+            None,
+            r"Number of classes in 'y_true' \(4\) not equal to the number of "
+            r"classes in 'y_score' \(3\).",
+        ),
+        (
+            ["c", "c", "a", "b"],
+            [
+                [0.2, 0.1, 0.7],
+                [0.4, 0.3, 0.3],
+                [0.3, 0.4, 0.3],
+                [0.4, 0.5, 0.1],
+            ],
+            ["a", "b", "c", "c"],
+            "Parameter 'labels' must be unique.",
+        ),
+        (
+            ["c", "c", "a", "b"],
+            [
+                [0.2, 0.1, 0.7],
+                [0.4, 0.3, 0.3],
+                [0.3, 0.4, 0.3],
+                [0.4, 0.5, 0.1],
+            ],
+            ["a", "c", "b"],
+            "Parameter 'labels' must be ordered.",
+        ),
+        (
+            [0, 0, 1, 2],
+            [
+                [0.2, 0.1, 0.7],
+                [0.4, 0.3, 0.3],
+                [0.3, 0.4, 0.3],
+                [0.4, 0.5, 0.1],
+            ],
+            [0, 1, 2, 3],
+            r"Number of given labels \(4\) not equal to the number of classes in "
+            r"'y_score' \(3\).",
+        ),
+        (
+            [0, 0, 1, 2],
+            [
+                [0.2, 0.1, 0.7],
+                [0.4, 0.3, 0.3],
+                [0.3, 0.4, 0.3],
+                [0.4, 0.5, 0.1],
+            ],
+            [0, 1, 3],
+            "'y_true' contains labels not in parameter 'labels'.",
+        ),
+        (
+            [0, 1],
+            [[0.5, 0.2, 0.2], [0.3, 0.4, 0.2]],
+            None,
+            (
+                "`y_true` is binary while y_score is 2d with 3 classes. If"
+                " `y_true` does not contain all the labels, `labels` must be provided"
+            ),
+        ),
+    ],
+)
+def test_top_k_accuracy_score_error(y_true, y_score, labels, msg):
     with pytest.raises(ValueError, match=msg):
         top_k_accuracy_score(y_true, y_score, k=2, labels=labels)
+
+
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_label_ranking_avg_precision_score_should_allow_csr_matrix_for_y_true_input(
+    csr_container,
+):
+    # Test that label_ranking_avg_precision_score accept sparse y_true.
+    # Non-regression test for #22575
+    y_true = csr_container([[1, 0, 0], [0, 0, 1]])
+    y_score = np.array([[0.5, 0.9, 0.6], [0, 0, 1]])
+    result = label_ranking_average_precision_score(y_true, y_score)
+    assert result == pytest.approx(2 / 3)
+
+
+@pytest.mark.parametrize(
+    "metric", [average_precision_score, det_curve, precision_recall_curve, roc_curve]
+)
+@pytest.mark.parametrize(
+    "classes", [(False, True), (0, 1), (0.0, 1.0), ("zero", "one")]
+)
+def test_ranking_metric_pos_label_types(metric, classes):
+    """Check that the metric works with different types of `pos_label`.
+
+    We can expect `pos_label` to be a bool, an integer, a float, a string.
+    No error should be raised for those types.
+    """
+    rng = np.random.RandomState(42)
+    n_samples, pos_label = 10, classes[-1]
+    y_true = rng.choice(classes, size=n_samples, replace=True)
+    y_proba = rng.rand(n_samples)
+    result = metric(y_true, y_proba, pos_label=pos_label)
+    if isinstance(result, float):
+        assert not np.isnan(result)
+    else:
+        metric_1, metric_2, thresholds = result
+        assert not np.isnan(metric_1).any()
+        assert not np.isnan(metric_2).any()
+        assert not np.isnan(thresholds).any()
+
+
+def test_roc_curve_with_probablity_estimates(global_random_seed):
+    """Check that thresholds do not exceed 1.0 when `y_score` is a probability
+    estimate.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/26193
+    """
+    rng = np.random.RandomState(global_random_seed)
+    y_true = rng.randint(0, 2, size=10)
+    y_score = rng.rand(10)
+    _, _, thresholds = roc_curve(y_true, y_score)
+    assert np.isinf(thresholds[0])
diff --git a/sklearn/metrics/tests/test_regression.py b/sklearn/metrics/tests/test_regression.py
index 8e935173d3319..396ae5d0ffae1 100644
--- a/sklearn/metrics/tests/test_regression.py
+++ b/sklearn/metrics/tests/test_regression.py
@@ -1,31 +1,38 @@
+from itertools import product
 
 import numpy as np
-from scipy import optimize
-from numpy.testing import assert_allclose
-from itertools import product
 import pytest
+from numpy.testing import assert_allclose
+from scipy import optimize
+from scipy.special import factorial, xlogy
 
-from sklearn.utils._testing import assert_almost_equal
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils._testing import assert_array_almost_equal
 from sklearn.dummy import DummyRegressor
-from sklearn.model_selection import GridSearchCV
-
-from sklearn.metrics import explained_variance_score
-from sklearn.metrics import mean_absolute_error
-from sklearn.metrics import mean_squared_error
-from sklearn.metrics import mean_squared_log_error
-from sklearn.metrics import median_absolute_error
-from sklearn.metrics import mean_absolute_percentage_error
-from sklearn.metrics import max_error
-from sklearn.metrics import mean_pinball_loss
-from sklearn.metrics import r2_score
-from sklearn.metrics import mean_tweedie_deviance
-from sklearn.metrics import make_scorer
-
-from sklearn.metrics._regression import _check_reg_targets
-
 from sklearn.exceptions import UndefinedMetricWarning
+from sklearn.metrics import (
+    d2_absolute_error_score,
+    d2_pinball_score,
+    d2_tweedie_score,
+    explained_variance_score,
+    make_scorer,
+    max_error,
+    mean_absolute_error,
+    mean_absolute_percentage_error,
+    mean_pinball_loss,
+    mean_squared_error,
+    mean_squared_log_error,
+    mean_tweedie_deviance,
+    median_absolute_error,
+    r2_score,
+    root_mean_squared_error,
+    root_mean_squared_log_error,
+)
+from sklearn.metrics._regression import _check_reg_targets
+from sklearn.model_selection import GridSearchCV
+from sklearn.utils._testing import (
+    assert_almost_equal,
+    assert_array_almost_equal,
+    assert_array_equal,
+)
 
 
 def test_regression_metrics(n_samples=50):
@@ -33,24 +40,54 @@ def test_regression_metrics(n_samples=50):
     y_pred = y_true + 1
     y_pred_2 = y_true - 1
 
-    assert_almost_equal(mean_squared_error(y_true, y_pred), 1.)
-    assert_almost_equal(mean_squared_log_error(y_true, y_pred),
-                        mean_squared_error(np.log(1 + y_true),
-                                           np.log(1 + y_pred)))
-    assert_almost_equal(mean_absolute_error(y_true, y_pred), 1.)
+    assert_almost_equal(mean_squared_error(y_true, y_pred), 1.0)
+    assert_almost_equal(
+        mean_squared_log_error(y_true, y_pred),
+        mean_squared_error(np.log(1 + y_true), np.log(1 + y_pred)),
+    )
+    assert_almost_equal(mean_absolute_error(y_true, y_pred), 1.0)
     assert_almost_equal(mean_pinball_loss(y_true, y_pred), 0.5)
     assert_almost_equal(mean_pinball_loss(y_true, y_pred_2), 0.5)
     assert_almost_equal(mean_pinball_loss(y_true, y_pred, alpha=0.4), 0.6)
     assert_almost_equal(mean_pinball_loss(y_true, y_pred_2, alpha=0.4), 0.4)
-    assert_almost_equal(median_absolute_error(y_true, y_pred), 1.)
+    assert_almost_equal(median_absolute_error(y_true, y_pred), 1.0)
     mape = mean_absolute_percentage_error(y_true, y_pred)
     assert np.isfinite(mape)
     assert mape > 1e6
-    assert_almost_equal(max_error(y_true, y_pred), 1.)
-    assert_almost_equal(r2_score(y_true, y_pred),  0.995, 2)
-    assert_almost_equal(explained_variance_score(y_true, y_pred), 1.)
-    assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=0),
-                        mean_squared_error(y_true, y_pred))
+    assert_almost_equal(max_error(y_true, y_pred), 1.0)
+    assert_almost_equal(r2_score(y_true, y_pred), 0.995, 2)
+    assert_almost_equal(r2_score(y_true, y_pred, force_finite=False), 0.995, 2)
+    assert_almost_equal(explained_variance_score(y_true, y_pred), 1.0)
+    assert_almost_equal(
+        explained_variance_score(y_true, y_pred, force_finite=False), 1.0
+    )
+    assert_almost_equal(
+        mean_tweedie_deviance(y_true, y_pred, power=0),
+        mean_squared_error(y_true, y_pred),
+    )
+    assert_almost_equal(
+        d2_tweedie_score(y_true, y_pred, power=0), r2_score(y_true, y_pred)
+    )
+    dev_median = np.abs(y_true - np.median(y_true)).sum()
+    assert_array_almost_equal(
+        d2_absolute_error_score(y_true, y_pred),
+        1 - np.abs(y_true - y_pred).sum() / dev_median,
+    )
+    alpha = 0.2
+    pinball_loss = lambda y_true, y_pred, alpha: alpha * np.maximum(
+        y_true - y_pred, 0
+    ) + (1 - alpha) * np.maximum(y_pred - y_true, 0)
+    y_quantile = np.percentile(y_true, q=alpha * 100)
+    assert_almost_equal(
+        d2_pinball_score(y_true, y_pred, alpha=alpha),
+        1
+        - pinball_loss(y_true, y_pred, alpha).sum()
+        / pinball_loss(y_true, y_quantile, alpha).sum(),
+    )
+    assert_almost_equal(
+        d2_absolute_error_score(y_true, y_pred),
+        d2_pinball_score(y_true, y_pred, alpha=0.5),
+    )
 
     # Tweedie deviance needs positive y_pred, except for p=0,
     # p>=2 needs positive y_true
@@ -58,28 +95,42 @@ def test_regression_metrics(n_samples=50):
     y_true = np.arange(1, 1 + n_samples)
     y_pred = 2 * y_true
     n = n_samples
-    assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=-1),
-                        5/12 * n * (n**2 + 2 * n + 1))
-    assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=1),
-                        (n + 1) * (1 - np.log(2)))
-    assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=2),
-                        2 * np.log(2) - 1)
-    assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=3/2),
-                        ((6 * np.sqrt(2) - 8) / n) * np.sqrt(y_true).sum())
-    assert_almost_equal(mean_tweedie_deviance(y_true, y_pred, power=3),
-                        np.sum(1 / y_true) / (4 * n))
-
-
-def test_mean_squared_error_multioutput_raw_value_squared():
-    # non-regression test for
-    # https://github.com/scikit-learn/scikit-learn/pull/16323
-    mse1 = mean_squared_error(
-        [[1]], [[10]], multioutput="raw_values", squared=True
+    assert_almost_equal(
+        mean_tweedie_deviance(y_true, y_pred, power=-1),
+        5 / 12 * n * (n**2 + 2 * n + 1),
+    )
+    assert_almost_equal(
+        mean_tweedie_deviance(y_true, y_pred, power=1), (n + 1) * (1 - np.log(2))
+    )
+    assert_almost_equal(
+        mean_tweedie_deviance(y_true, y_pred, power=2), 2 * np.log(2) - 1
     )
-    mse2 = mean_squared_error(
-        [[1]], [[10]], multioutput="raw_values", squared=False
+    assert_almost_equal(
+        mean_tweedie_deviance(y_true, y_pred, power=3 / 2),
+        ((6 * np.sqrt(2) - 8) / n) * np.sqrt(y_true).sum(),
+    )
+    assert_almost_equal(
+        mean_tweedie_deviance(y_true, y_pred, power=3), np.sum(1 / y_true) / (4 * n)
+    )
+
+    dev_mean = 2 * np.mean(xlogy(y_true, 2 * y_true / (n + 1)))
+    assert_almost_equal(
+        d2_tweedie_score(y_true, y_pred, power=1),
+        1 - (n + 1) * (1 - np.log(2)) / dev_mean,
     )
-    assert np.sqrt(mse1) == pytest.approx(mse2)
+
+    dev_mean = 2 * np.log((n + 1) / 2) - 2 / n * np.log(factorial(n))
+    assert_almost_equal(
+        d2_tweedie_score(y_true, y_pred, power=2), 1 - (2 * np.log(2) - 1) / dev_mean
+    )
+
+
+def test_root_mean_squared_error_multioutput_raw_value():
+    # non-regression test for
+    # https://github.com/scikit-learn/scikit-learn/pull/16323
+    mse = mean_squared_error([[1]], [[10]], multioutput="raw_values")
+    rmse = root_mean_squared_error([[1]], [[10]], multioutput="raw_values")
+    assert np.sqrt(mse) == pytest.approx(rmse)
 
 
 def test_multioutput_regression():
@@ -87,95 +138,184 @@ def test_multioutput_regression():
     y_pred = np.array([[0, 0, 0, 1], [1, 0, 1, 1], [0, 0, 0, 1]])
 
     error = mean_squared_error(y_true, y_pred)
-    assert_almost_equal(error, (1. / 3 + 2. / 3 + 2. / 3) / 4.)
+    assert_almost_equal(error, (1.0 / 3 + 2.0 / 3 + 2.0 / 3) / 4.0)
 
-    error = mean_squared_error(y_true, y_pred, squared=False)
+    error = root_mean_squared_error(y_true, y_pred)
     assert_almost_equal(error, 0.454, decimal=2)
 
     error = mean_squared_log_error(y_true, y_pred)
     assert_almost_equal(error, 0.200, decimal=2)
 
+    error = root_mean_squared_log_error(y_true, y_pred)
+    assert_almost_equal(error, 0.315, decimal=2)
+
     # mean_absolute_error and mean_squared_error are equal because
     # it is a binary problem.
     error = mean_absolute_error(y_true, y_pred)
-    assert_almost_equal(error, (1. + 2. / 3) / 4.)
+    assert_almost_equal(error, (1.0 + 2.0 / 3) / 4.0)
 
     error = mean_pinball_loss(y_true, y_pred)
-    assert_almost_equal(error, (1. + 2. / 3) / 8.)
+    assert_almost_equal(error, (1.0 + 2.0 / 3) / 8.0)
 
-    error = np.around(mean_absolute_percentage_error(y_true, y_pred),
-                      decimals=2)
+    error = np.around(mean_absolute_percentage_error(y_true, y_pred), decimals=2)
     assert np.isfinite(error)
     assert error > 1e6
     error = median_absolute_error(y_true, y_pred)
-    assert_almost_equal(error, (1. + 1.) / 4.)
-
-    error = r2_score(y_true, y_pred, multioutput='variance_weighted')
-    assert_almost_equal(error, 1. - 5. / 2)
-    error = r2_score(y_true, y_pred, multioutput='uniform_average')
-    assert_almost_equal(error, -.875)
+    assert_almost_equal(error, (1.0 + 1.0) / 4.0)
+
+    error = r2_score(y_true, y_pred, multioutput="variance_weighted")
+    assert_almost_equal(error, 1.0 - 5.0 / 2)
+    error = r2_score(y_true, y_pred, multioutput="uniform_average")
+    assert_almost_equal(error, -0.875)
+
+    score = d2_pinball_score(y_true, y_pred, alpha=0.5, multioutput="raw_values")
+    raw_expected_score = [
+        1
+        - np.abs(y_true[:, i] - y_pred[:, i]).sum()
+        / np.abs(y_true[:, i] - np.median(y_true[:, i])).sum()
+        for i in range(y_true.shape[1])
+    ]
+    # in the last case, the denominator vanishes and hence we get nan,
+    # but since the numerator vanishes as well the expected score is 1.0
+    raw_expected_score = np.where(np.isnan(raw_expected_score), 1, raw_expected_score)
+    assert_array_almost_equal(score, raw_expected_score)
+
+    score = d2_pinball_score(y_true, y_pred, alpha=0.5, multioutput="uniform_average")
+    assert_almost_equal(score, raw_expected_score.mean())
+    # constant `y_true` with force_finite=True leads to 1. or 0.
+    yc = [5.0, 5.0]
+    error = r2_score(yc, [5.0, 5.0], multioutput="variance_weighted")
+    assert_almost_equal(error, 1.0)
+    error = r2_score(yc, [5.0, 5.1], multioutput="variance_weighted")
+    assert_almost_equal(error, 0.0)
+
+    # Setting force_finite=False results in the nan for 4th output propagating
+    error = r2_score(
+        y_true, y_pred, multioutput="variance_weighted", force_finite=False
+    )
+    assert_almost_equal(error, np.nan)
+    error = r2_score(y_true, y_pred, multioutput="uniform_average", force_finite=False)
+    assert_almost_equal(error, np.nan)
+
+    # Dropping the 4th output to check `force_finite=False` for nominal
+    y_true = y_true[:, :-1]
+    y_pred = y_pred[:, :-1]
+    error = r2_score(y_true, y_pred, multioutput="variance_weighted")
+    error2 = r2_score(
+        y_true, y_pred, multioutput="variance_weighted", force_finite=False
+    )
+    assert_almost_equal(error, error2)
+    error = r2_score(y_true, y_pred, multioutput="uniform_average")
+    error2 = r2_score(y_true, y_pred, multioutput="uniform_average", force_finite=False)
+    assert_almost_equal(error, error2)
+
+    # constant `y_true` with force_finite=False leads to NaN or -Inf.
+    error = r2_score(
+        yc, [5.0, 5.0], multioutput="variance_weighted", force_finite=False
+    )
+    assert_almost_equal(error, np.nan)
+    error = r2_score(
+        yc, [5.0, 6.0], multioutput="variance_weighted", force_finite=False
+    )
+    assert_almost_equal(error, -np.inf)
 
 
 def test_regression_metrics_at_limits():
-    assert_almost_equal(mean_squared_error([0.], [0.]), 0.0)
-    assert_almost_equal(mean_squared_error([0.], [0.], squared=False), 0.0)
-    assert_almost_equal(mean_squared_log_error([0.], [0.]), 0.0)
-    assert_almost_equal(mean_absolute_error([0.], [0.]), 0.0)
-    assert_almost_equal(mean_pinball_loss([0.], [0.]), 0.0)
-    assert_almost_equal(mean_absolute_percentage_error([0.], [0.]), 0.0)
-    assert_almost_equal(median_absolute_error([0.], [0.]), 0.0)
-    assert_almost_equal(max_error([0.], [0.]), 0.0)
-    assert_almost_equal(explained_variance_score([0.], [0.]), 1.0)
-    assert_almost_equal(r2_score([0., 1], [0., 1]), 1.0)
-    err_msg = ("Mean Squared Logarithmic Error cannot be used when targets "
-               "contain negative values.")
-    with pytest.raises(ValueError, match=err_msg):
-        mean_squared_log_error([-1.], [-1.])
-    err_msg = ("Mean Squared Logarithmic Error cannot be used when targets "
-               "contain negative values.")
-    with pytest.raises(ValueError, match=err_msg):
-        mean_squared_log_error([1., 2., 3.], [1., -2., 3.])
-    err_msg = ("Mean Squared Logarithmic Error cannot be used when targets "
-               "contain negative values.")
-    with pytest.raises(ValueError, match=err_msg):
-        mean_squared_log_error([1., -2., 3.], [1., 2., 3.])
+    # Single-sample case
+    # Note: for r2 and d2_tweedie see also test_regression_single_sample
+    assert_almost_equal(mean_squared_error([0.0], [0.0]), 0.0)
+    assert_almost_equal(root_mean_squared_error([0.0], [0.0]), 0.0)
+    assert_almost_equal(mean_squared_log_error([0.0], [0.0]), 0.0)
+    assert_almost_equal(mean_absolute_error([0.0], [0.0]), 0.0)
+    assert_almost_equal(mean_pinball_loss([0.0], [0.0]), 0.0)
+    assert_almost_equal(mean_absolute_percentage_error([0.0], [0.0]), 0.0)
+    assert_almost_equal(median_absolute_error([0.0], [0.0]), 0.0)
+    assert_almost_equal(max_error([0.0], [0.0]), 0.0)
+    assert_almost_equal(explained_variance_score([0.0], [0.0]), 1.0)
+
+    # Perfect cases
+    assert_almost_equal(r2_score([0.0, 1], [0.0, 1]), 1.0)
+    assert_almost_equal(d2_pinball_score([0.0, 1], [0.0, 1]), 1.0)
+
+    # Non-finite cases
+    # R² and explained variance have a fix by default for non-finite cases
+    for s in (r2_score, explained_variance_score):
+        assert_almost_equal(s([0, 0], [1, -1]), 0.0)
+        assert_almost_equal(s([0, 0], [1, -1], force_finite=False), -np.inf)
+        assert_almost_equal(s([1, 1], [1, 1]), 1.0)
+        assert_almost_equal(s([1, 1], [1, 1], force_finite=False), np.nan)
+    msg = (
+        "Mean Squared Logarithmic Error cannot be used when "
+        "targets contain values less than or equal to -1."
+    )
+    with pytest.raises(ValueError, match=msg):
+        mean_squared_log_error([-1.0], [-1.0])
+    msg = (
+        "Mean Squared Logarithmic Error cannot be used when "
+        "targets contain values less than or equal to -1."
+    )
+    with pytest.raises(ValueError, match=msg):
+        mean_squared_log_error([1.0, 2.0, 3.0], [1.0, -2.0, 3.0])
+    msg = (
+        "Mean Squared Logarithmic Error cannot be used when "
+        "targets contain values less than or equal to -1."
+    )
+    with pytest.raises(ValueError, match=msg):
+        mean_squared_log_error([1.0, -2.0, 3.0], [1.0, 2.0, 3.0])
+    msg = (
+        "Mean Squared Logarithmic Error cannot be used when "
+        "targets contain values less than or equal to -1."
+    )
+    with pytest.raises(ValueError, match=msg):
+        root_mean_squared_log_error([1.0, -2.0, 3.0], [1.0, 2.0, 3.0])
+    msg = (
+        "Root Mean Squared Logarithmic Error cannot be used when "
+        "targets contain values less than or equal to -1."
+    )
 
     # Tweedie deviance error
     power = -1.2
-    assert_allclose(mean_tweedie_deviance([0], [1.], power=power),
-                    2 / (2 - power), rtol=1e-3)
-    with pytest.raises(ValueError,
-                       match="can only be used on strictly positive y_pred."):
-        mean_tweedie_deviance([0.], [0.], power=power)
-    assert_almost_equal(mean_tweedie_deviance([0.], [0.], power=0), 0.00, 2)
+    assert_allclose(
+        mean_tweedie_deviance([0], [1.0], power=power), 2 / (2 - power), rtol=1e-3
+    )
+    msg = "can only be used on strictly positive y_pred."
+    with pytest.raises(ValueError, match=msg):
+        mean_tweedie_deviance([0.0], [0.0], power=power)
+    with pytest.raises(ValueError, match=msg):
+        d2_tweedie_score([0.0] * 2, [0.0] * 2, power=power)
+
+    assert_almost_equal(mean_tweedie_deviance([0.0], [0.0], power=0), 0.0, 2)
 
+    power = 1.0
     msg = "only be used on non-negative y and strictly positive y_pred."
     with pytest.raises(ValueError, match=msg):
-        mean_tweedie_deviance([0.], [0.], power=1.0)
+        mean_tweedie_deviance([0.0], [0.0], power=power)
+    with pytest.raises(ValueError, match=msg):
+        d2_tweedie_score([0.0] * 2, [0.0] * 2, power=power)
 
     power = 1.5
-    assert_allclose(mean_tweedie_deviance([0.], [1.], power=power),
-                    2 / (2 - power))
+    assert_allclose(mean_tweedie_deviance([0.0], [1.0], power=power), 2 / (2 - power))
     msg = "only be used on non-negative y and strictly positive y_pred."
     with pytest.raises(ValueError, match=msg):
-        mean_tweedie_deviance([0.], [0.], power=power)
-    power = 2.
-    assert_allclose(mean_tweedie_deviance([1.], [1.], power=power), 0.00,
-                    atol=1e-8)
-    msg = "can only be used on strictly positive y and y_pred."
+        mean_tweedie_deviance([0.0], [0.0], power=power)
     with pytest.raises(ValueError, match=msg):
-        mean_tweedie_deviance([0.], [0.], power=power)
-    power = 3.
-    assert_allclose(mean_tweedie_deviance([1.], [1.], power=power),
-                    0.00, atol=1e-8)
+        d2_tweedie_score([0.0] * 2, [0.0] * 2, power=power)
 
+    power = 2.0
+    assert_allclose(mean_tweedie_deviance([1.0], [1.0], power=power), 0.00, atol=1e-8)
     msg = "can only be used on strictly positive y and y_pred."
     with pytest.raises(ValueError, match=msg):
-        mean_tweedie_deviance([0.], [0.], power=power)
+        mean_tweedie_deviance([0.0], [0.0], power=power)
+    with pytest.raises(ValueError, match=msg):
+        d2_tweedie_score([0.0] * 2, [0.0] * 2, power=power)
 
-    with pytest.raises(ValueError,
-                       match="is only defined for power<=0 and power>=1"):
-        mean_tweedie_deviance([0.], [0.], power=0.5)
+    power = 3.0
+    assert_allclose(mean_tweedie_deviance([1.0], [1.0], power=power), 0.00, atol=1e-8)
+    msg = "can only be used on strictly positive y and y_pred."
+    with pytest.raises(ValueError, match=msg):
+        mean_tweedie_deviance([0.0], [0.0], power=power)
+    with pytest.raises(ValueError, match=msg):
+        d2_tweedie_score([0.0] * 2, [0.0] * 2, power=power)
 
 
 def test__check_reg_targets():
@@ -188,14 +328,13 @@ def test__check_reg_targets():
         ("continuous-multioutput", [[1, 3, 4], [2, 2, 2], [3, 1, 1]], 3),
     ]
 
-    for (type1, y1, n_out1), (type2, y2, n_out2) in product(EXAMPLES,
-                                                            repeat=2):
-
+    for (type1, y1, n_out1), (type2, y2, n_out2) in product(EXAMPLES, repeat=2):
         if type1 == type2 and n_out1 == n_out2:
-            y_type, y_check1, y_check2, multioutput = _check_reg_targets(
-                y1, y2, None)
+            y_type, y_check1, y_check2, _, _ = _check_reg_targets(
+                y1, y2, sample_weight=None, multioutput=None
+            )
             assert type1 == y_type
-            if type1 == 'continuous':
+            if type1 == "continuous":
                 assert_array_equal(y_check1, np.reshape(y1, (-1, 1)))
                 assert_array_equal(y_check2, np.reshape(y2, (-1, 1)))
             else:
@@ -203,80 +342,111 @@ def test__check_reg_targets():
                 assert_array_equal(y_check2, y2)
         else:
             with pytest.raises(ValueError):
-                _check_reg_targets(y1, y2, None)
+                _check_reg_targets(y1, y2, sample_weight=None, multioutput=None)
 
 
 def test__check_reg_targets_exception():
-    invalid_multioutput = 'this_value_is_not_valid'
-    expected_message = ("Allowed 'multioutput' string values are.+"
-                        "You provided multioutput={!r}".format(
-                            invalid_multioutput))
+    invalid_multioutput = "this_value_is_not_valid"
+    expected_message = (
+        "Allowed 'multioutput' string values are.+You provided multioutput={!r}".format(
+            invalid_multioutput
+        )
+    )
     with pytest.raises(ValueError, match=expected_message):
-        _check_reg_targets([1, 2, 3], [[1], [2], [3]], invalid_multioutput)
+        _check_reg_targets([1, 2, 3], [[1], [2], [3]], None, invalid_multioutput)
 
 
 def test_regression_multioutput_array():
     y_true = [[1, 2], [2.5, -1], [4.5, 3], [5, 7]]
     y_pred = [[1, 1], [2, -1], [5, 4], [5, 6.5]]
 
-    mse = mean_squared_error(y_true, y_pred, multioutput='raw_values')
-    mae = mean_absolute_error(y_true, y_pred, multioutput='raw_values')
-    err_msg = ("multioutput is expected to be 'raw_values' "
-               "or 'uniform_average' but we got 'variance_weighted' instead.")
-    with pytest.raises(ValueError, match=err_msg):
-        mean_pinball_loss(y_true, y_pred, multioutput='variance_weighted')
-    pbl = mean_pinball_loss(y_true, y_pred, multioutput='raw_values')
-    mape = mean_absolute_percentage_error(y_true, y_pred,
-                                          multioutput='raw_values')
-    r = r2_score(y_true, y_pred, multioutput='raw_values')
-    evs = explained_variance_score(y_true, y_pred, multioutput='raw_values')
+    mse = mean_squared_error(y_true, y_pred, multioutput="raw_values")
+    mae = mean_absolute_error(y_true, y_pred, multioutput="raw_values")
+
+    pbl = mean_pinball_loss(y_true, y_pred, multioutput="raw_values")
+    mape = mean_absolute_percentage_error(y_true, y_pred, multioutput="raw_values")
+    r = r2_score(y_true, y_pred, multioutput="raw_values")
+    evs = explained_variance_score(y_true, y_pred, multioutput="raw_values")
+    d2ps = d2_pinball_score(y_true, y_pred, alpha=0.5, multioutput="raw_values")
+    evs2 = explained_variance_score(
+        y_true, y_pred, multioutput="raw_values", force_finite=False
+    )
 
     assert_array_almost_equal(mse, [0.125, 0.5625], decimal=2)
     assert_array_almost_equal(mae, [0.25, 0.625], decimal=2)
-    assert_array_almost_equal(pbl, [0.25/2, 0.625/2], decimal=2)
+    assert_array_almost_equal(pbl, [0.25 / 2, 0.625 / 2], decimal=2)
     assert_array_almost_equal(mape, [0.0778, 0.2262], decimal=2)
     assert_array_almost_equal(r, [0.95, 0.93], decimal=2)
     assert_array_almost_equal(evs, [0.95, 0.93], decimal=2)
+    assert_array_almost_equal(d2ps, [0.833, 0.722], decimal=2)
+    assert_array_almost_equal(evs2, [0.95, 0.93], decimal=2)
 
     # mean_absolute_error and mean_squared_error are equal because
     # it is a binary problem.
-    y_true = [[0, 0]]*4
-    y_pred = [[1, 1]]*4
-    mse = mean_squared_error(y_true, y_pred, multioutput='raw_values')
-    mae = mean_absolute_error(y_true, y_pred, multioutput='raw_values')
-    pbl = mean_pinball_loss(y_true, y_pred, multioutput='raw_values')
-    r = r2_score(y_true, y_pred, multioutput='raw_values')
-    assert_array_almost_equal(mse, [1., 1.], decimal=2)
-    assert_array_almost_equal(mae, [1., 1.], decimal=2)
+    y_true = [[0, 0]] * 4
+    y_pred = [[1, 1]] * 4
+    mse = mean_squared_error(y_true, y_pred, multioutput="raw_values")
+    mae = mean_absolute_error(y_true, y_pred, multioutput="raw_values")
+    pbl = mean_pinball_loss(y_true, y_pred, multioutput="raw_values")
+    r = r2_score(y_true, y_pred, multioutput="raw_values")
+    d2ps = d2_pinball_score(y_true, y_pred, multioutput="raw_values")
+    assert_array_almost_equal(mse, [1.0, 1.0], decimal=2)
+    assert_array_almost_equal(mae, [1.0, 1.0], decimal=2)
     assert_array_almost_equal(pbl, [0.5, 0.5], decimal=2)
-    assert_array_almost_equal(r, [0., 0.], decimal=2)
+    assert_array_almost_equal(r, [0.0, 0.0], decimal=2)
+    assert_array_almost_equal(d2ps, [0.0, 0.0], decimal=2)
 
-    r = r2_score([[0, -1], [0, 1]], [[2, 2], [1, 1]], multioutput='raw_values')
+    r = r2_score([[0, -1], [0, 1]], [[2, 2], [1, 1]], multioutput="raw_values")
     assert_array_almost_equal(r, [0, -3.5], decimal=2)
-    assert np.mean(r) == r2_score([[0, -1], [0, 1]], [[2, 2], [1, 1]],
-                                  multioutput='uniform_average')
-    evs = explained_variance_score([[0, -1], [0, 1]], [[2, 2], [1, 1]],
-                                   multioutput='raw_values')
+    assert np.mean(r) == r2_score(
+        [[0, -1], [0, 1]], [[2, 2], [1, 1]], multioutput="uniform_average"
+    )
+    evs = explained_variance_score(
+        [[0, -1], [0, 1]], [[2, 2], [1, 1]], multioutput="raw_values"
+    )
     assert_array_almost_equal(evs, [0, -1.25], decimal=2)
+    evs2 = explained_variance_score(
+        [[0, -1], [0, 1]],
+        [[2, 2], [1, 1]],
+        multioutput="raw_values",
+        force_finite=False,
+    )
+    assert_array_almost_equal(evs2, [-np.inf, -1.25], decimal=2)
 
     # Checking for the condition in which both numerator and denominator is
     # zero.
-    y_true = [[1, 3], [-1, 2]]
-    y_pred = [[1, 4], [-1, 1]]
-    r2 = r2_score(y_true, y_pred, multioutput='raw_values')
-    assert_array_almost_equal(r2, [1., -3.], decimal=2)
-    assert np.mean(r2) == r2_score(y_true, y_pred,
-                                   multioutput='uniform_average')
-    evs = explained_variance_score(y_true, y_pred, multioutput='raw_values')
-    assert_array_almost_equal(evs, [1., -3.], decimal=2)
+    y_true = [[1, 3], [1, 2]]
+    y_pred = [[1, 4], [1, 1]]
+    r2 = r2_score(y_true, y_pred, multioutput="raw_values")
+    assert_array_almost_equal(r2, [1.0, -3.0], decimal=2)
+    assert np.mean(r2) == r2_score(y_true, y_pred, multioutput="uniform_average")
+    r22 = r2_score(y_true, y_pred, multioutput="raw_values", force_finite=False)
+    assert_array_almost_equal(r22, [np.nan, -3.0], decimal=2)
+    assert_almost_equal(
+        np.mean(r22),
+        r2_score(y_true, y_pred, multioutput="uniform_average", force_finite=False),
+    )
+
+    evs = explained_variance_score(y_true, y_pred, multioutput="raw_values")
+    assert_array_almost_equal(evs, [1.0, -3.0], decimal=2)
     assert np.mean(evs) == explained_variance_score(y_true, y_pred)
+    d2ps = d2_pinball_score(y_true, y_pred, alpha=0.5, multioutput="raw_values")
+    assert_array_almost_equal(d2ps, [1.0, -1.0], decimal=2)
+    evs2 = explained_variance_score(
+        y_true, y_pred, multioutput="raw_values", force_finite=False
+    )
+    assert_array_almost_equal(evs2, [np.nan, -3.0], decimal=2)
+    assert_almost_equal(
+        np.mean(evs2), explained_variance_score(y_true, y_pred, force_finite=False)
+    )
 
     # Handling msle separately as it does not accept negative inputs.
     y_true = np.array([[0.5, 1], [1, 2], [7, 6]])
     y_pred = np.array([[0.5, 2], [1, 2.5], [8, 8]])
-    msle = mean_squared_log_error(y_true, y_pred, multioutput='raw_values')
-    msle2 = mean_squared_error(np.log(1 + y_true), np.log(1 + y_pred),
-                               multioutput='raw_values')
+    msle = mean_squared_log_error(y_true, y_pred, multioutput="raw_values")
+    msle2 = mean_squared_error(
+        np.log(1 + y_true), np.log(1 + y_pred), multioutput="raw_values"
+    )
     assert_array_almost_equal(msle, msle2, decimal=2)
 
 
@@ -285,13 +455,15 @@ def test_regression_custom_weights():
     y_pred = [[1, 1], [2, -1], [5, 4], [5, 6.5]]
 
     msew = mean_squared_error(y_true, y_pred, multioutput=[0.4, 0.6])
-    rmsew = mean_squared_error(y_true, y_pred, multioutput=[0.4, 0.6],
-                               squared=False)
+    rmsew = root_mean_squared_error(y_true, y_pred, multioutput=[0.4, 0.6])
     maew = mean_absolute_error(y_true, y_pred, multioutput=[0.4, 0.6])
-    mapew = mean_absolute_percentage_error(y_true, y_pred,
-                                           multioutput=[0.4, 0.6])
+    mapew = mean_absolute_percentage_error(y_true, y_pred, multioutput=[0.4, 0.6])
     rw = r2_score(y_true, y_pred, multioutput=[0.4, 0.6])
     evsw = explained_variance_score(y_true, y_pred, multioutput=[0.4, 0.6])
+    d2psw = d2_pinball_score(y_true, y_pred, alpha=0.5, multioutput=[0.4, 0.6])
+    evsw2 = explained_variance_score(
+        y_true, y_pred, multioutput=[0.4, 0.6], force_finite=False
+    )
 
     assert_almost_equal(msew, 0.39, decimal=2)
     assert_almost_equal(rmsew, 0.59, decimal=2)
@@ -299,21 +471,24 @@ def test_regression_custom_weights():
     assert_almost_equal(mapew, 0.1668, decimal=2)
     assert_almost_equal(rw, 0.94, decimal=2)
     assert_almost_equal(evsw, 0.94, decimal=2)
+    assert_almost_equal(d2psw, 0.766, decimal=2)
+    assert_almost_equal(evsw2, 0.94, decimal=2)
 
     # Handling msle separately as it does not accept negative inputs.
     y_true = np.array([[0.5, 1], [1, 2], [7, 6]])
     y_pred = np.array([[0.5, 2], [1, 2.5], [8, 8]])
     msle = mean_squared_log_error(y_true, y_pred, multioutput=[0.3, 0.7])
-    msle2 = mean_squared_error(np.log(1 + y_true), np.log(1 + y_pred),
-                               multioutput=[0.3, 0.7])
+    msle2 = mean_squared_error(
+        np.log(1 + y_true), np.log(1 + y_pred), multioutput=[0.3, 0.7]
+    )
     assert_almost_equal(msle, msle2, decimal=2)
 
 
-@pytest.mark.parametrize('metric', [r2_score])
+@pytest.mark.parametrize("metric", [r2_score, d2_tweedie_score, d2_pinball_score])
 def test_regression_single_sample(metric):
     y_true = [0]
     y_pred = [1]
-    warning_msg = 'not well-defined with less than two samples.'
+    warning_msg = "not well-defined with less than two samples."
 
     # Trigger the warning
     with pytest.warns(UndefinedMetricWarning, match=warning_msg):
@@ -321,60 +496,71 @@ def test_regression_single_sample(metric):
         assert np.isnan(score)
 
 
-def test_tweedie_deviance_continuity():
+def test_tweedie_deviance_continuity(global_random_seed):
     n_samples = 100
 
-    y_true = np.random.RandomState(0).rand(n_samples) + 0.1
-    y_pred = np.random.RandomState(1).rand(n_samples) + 0.1
+    rng = np.random.RandomState(global_random_seed)
 
-    assert_allclose(mean_tweedie_deviance(y_true, y_pred, power=0 - 1e-10),
-                    mean_tweedie_deviance(y_true, y_pred, power=0))
+    y_true = rng.rand(n_samples) + 0.1
+    y_pred = rng.rand(n_samples) + 0.1
 
-    # Ws we get closer to the limit, with 1e-12 difference the absolute
+    assert_allclose(
+        mean_tweedie_deviance(y_true, y_pred, power=0 - 1e-10),
+        mean_tweedie_deviance(y_true, y_pred, power=0),
+    )
+
+    # Ws we get closer to the limit, with 1e-12 difference the
     # tolerance to pass the below check increases. There are likely
     # numerical precision issues on the edges of different definition
     # regions.
-    assert_allclose(mean_tweedie_deviance(y_true, y_pred, power=1 + 1e-10),
-                    mean_tweedie_deviance(y_true, y_pred, power=1),
-                    atol=1e-6)
+    assert_allclose(
+        mean_tweedie_deviance(y_true, y_pred, power=1 + 1e-10),
+        mean_tweedie_deviance(y_true, y_pred, power=1),
+        rtol=1e-5,
+    )
 
-    assert_allclose(mean_tweedie_deviance(y_true, y_pred, power=2 - 1e-10),
-                    mean_tweedie_deviance(y_true, y_pred, power=2),
-                    atol=1e-6)
+    assert_allclose(
+        mean_tweedie_deviance(y_true, y_pred, power=2 - 1e-10),
+        mean_tweedie_deviance(y_true, y_pred, power=2),
+        rtol=1e-5,
+    )
 
-    assert_allclose(mean_tweedie_deviance(y_true, y_pred, power=2 + 1e-10),
-                    mean_tweedie_deviance(y_true, y_pred, power=2),
-                    atol=1e-6)
+    assert_allclose(
+        mean_tweedie_deviance(y_true, y_pred, power=2 + 1e-10),
+        mean_tweedie_deviance(y_true, y_pred, power=2),
+        rtol=1e-5,
+    )
 
 
-def test_mean_absolute_percentage_error():
-    random_number_generator = np.random.RandomState(42)
+def test_mean_absolute_percentage_error(global_random_seed):
+    random_number_generator = np.random.RandomState(global_random_seed)
     y_true = random_number_generator.exponential(size=100)
     y_pred = 1.2 * y_true
     assert mean_absolute_percentage_error(y_true, y_pred) == pytest.approx(0.2)
 
 
-@pytest.mark.parametrize("distribution",
-                         ["normal", "lognormal", "exponential", "uniform"])
+@pytest.mark.parametrize(
+    "distribution", ["normal", "lognormal", "exponential", "uniform"]
+)
 @pytest.mark.parametrize("target_quantile", [0.05, 0.5, 0.75])
 def test_mean_pinball_loss_on_constant_predictions(
-    distribution,
-    target_quantile
+    distribution, target_quantile, global_random_seed
 ):
     if not hasattr(np, "quantile"):
-        pytest.skip("This test requires a more recent version of numpy "
-                    "with support for np.quantile.")
+        pytest.skip(
+            "This test requires a more recent version of numpy "
+            "with support for np.quantile."
+        )
 
     # Check that the pinball loss is minimized by the empirical quantile.
     n_samples = 3000
-    rng = np.random.RandomState(42)
+    rng = np.random.RandomState(global_random_seed)
     data = getattr(rng, distribution)(size=n_samples)
 
     # Compute the best possible pinball loss for any constant predictor:
     best_pred = np.quantile(data, target_quantile)
     best_constant_pred = np.full(n_samples, fill_value=best_pred)
-    best_pbl = mean_pinball_loss(data, best_constant_pred,
-                                 alpha=target_quantile)
+    best_pbl = mean_pinball_loss(data, best_constant_pred, alpha=target_quantile)
 
     # Evaluate the loss on a grid of quantiles
     candidate_predictions = np.quantile(data, np.linspace(0, 1, 100))
@@ -386,14 +572,13 @@ def test_mean_pinball_loss_on_constant_predictions(
         # Check that the loss of this constant predictor is greater or equal
         # than the loss of using the optimal quantile (up to machine
         # precision):
-        assert pbl >= best_pbl - np.finfo(best_pbl.dtype).eps
+        assert pbl >= best_pbl - np.finfo(np.float64).eps
 
         # Check that the value of the pinball loss matches the analytical
         # formula.
-        expected_pbl = (
-            (pred - data[data < pred]).sum() * (1 - target_quantile) +
-            (data[data >= pred] - pred).sum() * target_quantile
-        )
+        expected_pbl = (pred - data[data < pred]).sum() * (1 - target_quantile) + (
+            data[data >= pred] - pred
+        ).sum() * target_quantile
         expected_pbl /= n_samples
         assert_almost_equal(expected_pbl, pbl)
 
@@ -403,21 +588,22 @@ def objective_func(x):
         constant_pred = np.full(n_samples, fill_value=x)
         return mean_pinball_loss(data, constant_pred, alpha=target_quantile)
 
-    result = optimize.minimize(objective_func, data.mean(),
-                               method="Nelder-Mead")
+    result = optimize.minimize(objective_func, data.mean())
     assert result.success
     # The minimum is not unique with limited data, hence the large tolerance.
-    assert result.x == pytest.approx(best_pred, rel=1e-2)
+    # For the normal distribution and the 0.5 quantile, the expected result is close to
+    # 0, hence the additional use of absolute tolerance.
+    assert_allclose(result.x, best_pred, rtol=1e-1, atol=1e-3)
     assert result.fun == pytest.approx(best_pbl)
 
 
-def test_dummy_quantile_parameter_tuning():
+def test_dummy_quantile_parameter_tuning(global_random_seed):
     # Integration test to check that it is possible to use the pinball loss to
     # tune the hyperparameter of a quantile regressor. This is conceptually
     # similar to the previous test but using the scikit-learn estimator and
     # scoring API instead.
     n_samples = 1000
-    rng = np.random.RandomState(0)
+    rng = np.random.RandomState(global_random_seed)
     X = rng.normal(size=(n_samples, 5))  # Ignored
     y = rng.exponential(size=n_samples)
 
@@ -436,3 +622,15 @@ def test_dummy_quantile_parameter_tuning():
         ).fit(X, y)
 
         assert grid_search.best_params_["quantile"] == pytest.approx(alpha)
+
+
+def test_pinball_loss_relation_with_mae(global_random_seed):
+    # Test that mean_pinball loss with alpha=0.5 if half of mean absolute error
+    rng = np.random.RandomState(global_random_seed)
+    n = 100
+    y_true = rng.normal(size=n)
+    y_pred = y_true.copy() + rng.uniform(n)
+    assert (
+        mean_absolute_error(y_true, y_pred)
+        == mean_pinball_loss(y_true, y_pred, alpha=0.5) * 2
+    )
diff --git a/sklearn/metrics/tests/test_score_objects.py b/sklearn/metrics/tests/test_score_objects.py
index be214944e6ee4..672ed8ae7eecc 100644
--- a/sklearn/metrics/tests/test_score_objects.py
+++ b/sklearn/metrics/tests/test_score_objects.py
@@ -1,96 +1,143 @@
-from copy import deepcopy
-import pickle
-import tempfile
-import shutil
-import os
 import numbers
-from unittest.mock import Mock
+import pickle
+import warnings
+from copy import deepcopy
 from functools import partial
 
+import joblib
 import numpy as np
 import pytest
-import joblib
-
 from numpy.testing import assert_allclose
-from sklearn.utils._testing import assert_almost_equal
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils._testing import ignore_warnings
 
-from sklearn.base import BaseEstimator
+from sklearn import config_context
+from sklearn.base import BaseEstimator, ClassifierMixin
+from sklearn.cluster import KMeans
+from sklearn.datasets import (
+    load_diabetes,
+    make_blobs,
+    make_classification,
+    make_multilabel_classification,
+    make_regression,
+)
+from sklearn.linear_model import LogisticRegression, Perceptron, Ridge
 from sklearn.metrics import (
     accuracy_score,
-    balanced_accuracy_score,
     average_precision_score,
+    balanced_accuracy_score,
     brier_score_loss,
+    check_scoring,
     f1_score,
     fbeta_score,
+    get_scorer,
+    get_scorer_names,
     jaccard_score,
     log_loss,
+    make_scorer,
+    matthews_corrcoef,
     precision_score,
     r2_score,
     recall_score,
     roc_auc_score,
-    top_k_accuracy_score
+    top_k_accuracy_score,
 )
 from sklearn.metrics import cluster as cluster_module
-from sklearn.metrics import check_scoring
-from sklearn.metrics._scorer import (_PredictScorer, _passthrough_scorer,
-                                     _MultimetricScorer,
-                                     _check_multimetric_scoring)
-from sklearn.metrics import make_scorer, get_scorer, SCORERS
+from sklearn.metrics._scorer import (
+    _check_multimetric_scoring,
+    _CurveScorer,
+    _MultimetricScorer,
+    _PassthroughScorer,
+    _Scorer,
+)
+from sklearn.model_selection import GridSearchCV, cross_val_score, train_test_split
+from sklearn.multiclass import OneVsRestClassifier
 from sklearn.neighbors import KNeighborsClassifier
-from sklearn.svm import LinearSVC
 from sklearn.pipeline import make_pipeline
-from sklearn.cluster import KMeans
-from sklearn.linear_model import Ridge, LogisticRegression, Perceptron
+from sklearn.svm import LinearSVC
+from sklearn.tests.metadata_routing_common import (
+    assert_request_is_empty,
+)
 from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
-from sklearn.datasets import make_blobs
-from sklearn.datasets import make_classification, make_regression
-from sklearn.datasets import make_multilabel_classification
-from sklearn.datasets import load_diabetes
-from sklearn.model_selection import train_test_split, cross_val_score
-from sklearn.model_selection import GridSearchCV
-from sklearn.multiclass import OneVsRestClassifier
-
-
-REGRESSION_SCORERS = ['explained_variance', 'r2',
-                      'neg_mean_absolute_error', 'neg_mean_squared_error',
-                      'neg_mean_absolute_percentage_error',
-                      'neg_mean_squared_log_error',
-                      'neg_median_absolute_error',
-                      'neg_root_mean_squared_error',
-                      'mean_absolute_error',
-                      'mean_absolute_percentage_error',
-                      'mean_squared_error', 'median_absolute_error',
-                      'max_error', 'neg_mean_poisson_deviance',
-                      'neg_mean_gamma_deviance']
-
-CLF_SCORERS = ['accuracy', 'balanced_accuracy', 'top_k_accuracy',
-               'f1', 'f1_weighted', 'f1_macro', 'f1_micro',
-               'roc_auc', 'average_precision', 'precision',
-               'precision_weighted', 'precision_macro', 'precision_micro',
-               'recall', 'recall_weighted', 'recall_macro', 'recall_micro',
-               'neg_log_loss', 'neg_brier_score',
-               'jaccard', 'jaccard_weighted', 'jaccard_macro',
-               'jaccard_micro', 'roc_auc_ovr', 'roc_auc_ovo',
-               'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted']
+from sklearn.utils._testing import (
+    assert_almost_equal,
+    assert_array_equal,
+    ignore_warnings,
+)
+from sklearn.utils.metadata_routing import MetadataRouter, MethodMapping
+
+REGRESSION_SCORERS = [
+    "d2_absolute_error_score",
+    "explained_variance",
+    "r2",
+    "neg_mean_absolute_error",
+    "neg_mean_squared_error",
+    "neg_mean_absolute_percentage_error",
+    "neg_mean_squared_log_error",
+    "neg_median_absolute_error",
+    "neg_root_mean_squared_error",
+    "neg_root_mean_squared_log_error",
+    "mean_absolute_error",
+    "mean_absolute_percentage_error",
+    "mean_squared_error",
+    "median_absolute_error",
+    "neg_max_error",
+    "neg_mean_poisson_deviance",
+    "neg_mean_gamma_deviance",
+]
+
+CLF_SCORERS = [
+    "accuracy",
+    "balanced_accuracy",
+    "top_k_accuracy",
+    "f1",
+    "f1_weighted",
+    "f1_macro",
+    "f1_micro",
+    "roc_auc",
+    "average_precision",
+    "precision",
+    "precision_weighted",
+    "precision_macro",
+    "precision_micro",
+    "recall",
+    "recall_weighted",
+    "recall_macro",
+    "recall_micro",
+    "neg_log_loss",
+    "neg_brier_score",
+    "jaccard",
+    "jaccard_weighted",
+    "jaccard_macro",
+    "jaccard_micro",
+    "roc_auc_ovr",
+    "roc_auc_ovo",
+    "roc_auc_ovr_weighted",
+    "roc_auc_ovo_weighted",
+    "matthews_corrcoef",
+    "positive_likelihood_ratio",
+    "neg_negative_likelihood_ratio",
+]
 
 # All supervised cluster scorers (They behave like classification metric)
-CLUSTER_SCORERS = ["adjusted_rand_score",
-                   "rand_score",
-                   "homogeneity_score",
-                   "completeness_score",
-                   "v_measure_score",
-                   "mutual_info_score",
-                   "adjusted_mutual_info_score",
-                   "normalized_mutual_info_score",
-                   "fowlkes_mallows_score"]
-
-MULTILABEL_ONLY_SCORERS = ['precision_samples', 'recall_samples', 'f1_samples',
-                           'jaccard_samples']
-
-REQUIRE_POSITIVE_Y_SCORERS = ['neg_mean_poisson_deviance',
-                              'neg_mean_gamma_deviance']
+CLUSTER_SCORERS = [
+    "adjusted_rand_score",
+    "rand_score",
+    "homogeneity_score",
+    "completeness_score",
+    "v_measure_score",
+    "mutual_info_score",
+    "adjusted_mutual_info_score",
+    "normalized_mutual_info_score",
+    "fowlkes_mallows_score",
+]
+
+MULTILABEL_ONLY_SCORERS = [
+    "precision_samples",
+    "recall_samples",
+    "f1_samples",
+    "jaccard_samples",
+]
+
+REQUIRE_POSITIVE_Y_SCORERS = ["neg_mean_poisson_deviance", "neg_mean_gamma_deviance"]
 
 
 def _require_positive_y(y):
@@ -110,51 +157,36 @@ def _make_estimators(X_train, y_train, y_ml_train):
     sensible_ml_clf = DecisionTreeClassifier(random_state=0)
     sensible_ml_clf.fit(X_train, y_ml_train)
     return dict(
-        [(name, sensible_regr) for name in REGRESSION_SCORERS] +
-        [(name, sensible_clf) for name in CLF_SCORERS] +
-        [(name, sensible_clf) for name in CLUSTER_SCORERS] +
-        [(name, sensible_ml_clf) for name in MULTILABEL_ONLY_SCORERS]
+        [(name, sensible_regr) for name in REGRESSION_SCORERS]
+        + [(name, sensible_clf) for name in CLF_SCORERS]
+        + [(name, sensible_clf) for name in CLUSTER_SCORERS]
+        + [(name, sensible_ml_clf) for name in MULTILABEL_ONLY_SCORERS]
     )
 
 
-X_mm, y_mm, y_ml_mm = None, None, None
-ESTIMATORS = None
-TEMP_FOLDER = None
-
-
-def setup_module():
-    # Create some memory mapped data
-    global X_mm, y_mm, y_ml_mm, TEMP_FOLDER, ESTIMATORS
-    TEMP_FOLDER = tempfile.mkdtemp(prefix='sklearn_test_score_objects_')
+@pytest.fixture(scope="module")
+def memmap_data_and_estimators(tmp_path_factory):
+    temp_folder = tmp_path_factory.mktemp("sklearn_test_score_objects")
     X, y = make_classification(n_samples=30, n_features=5, random_state=0)
-    _, y_ml = make_multilabel_classification(n_samples=X.shape[0],
-                                             random_state=0)
-    filename = os.path.join(TEMP_FOLDER, 'test_data.pkl')
+    _, y_ml = make_multilabel_classification(n_samples=X.shape[0], random_state=0)
+    filename = temp_folder / "test_data.pkl"
     joblib.dump((X, y, y_ml), filename)
-    X_mm, y_mm, y_ml_mm = joblib.load(filename, mmap_mode='r')
-    ESTIMATORS = _make_estimators(X_mm, y_mm, y_ml_mm)
-
-
-def teardown_module():
-    global X_mm, y_mm, y_ml_mm, TEMP_FOLDER, ESTIMATORS
-    # GC closes the mmap file descriptors
-    X_mm, y_mm, y_ml_mm, ESTIMATORS = None, None, None, None
-    shutil.rmtree(TEMP_FOLDER)
-
+    X_mm, y_mm, y_ml_mm = joblib.load(filename, mmap_mode="r")
+    estimators = _make_estimators(X_mm, y_mm, y_ml_mm)
 
-class EstimatorWithoutFit:
-    """Dummy estimator to test scoring validators"""
-    pass
+    yield X_mm, y_mm, y_ml_mm, estimators
 
 
 class EstimatorWithFit(BaseEstimator):
     """Dummy estimator to test scoring validators"""
+
     def fit(self, X, y):
         return self
 
 
-class EstimatorWithFitAndScore:
+class EstimatorWithFitAndScore(BaseEstimator):
     """Dummy estimator to test scoring validators"""
+
     def fit(self, X, y):
         return self
 
@@ -162,8 +194,9 @@ def score(self, X, y):
         return 1.0
 
 
-class EstimatorWithFitAndPredict:
+class EstimatorWithFitAndPredict(BaseEstimator):
     """Dummy estimator to test scoring validators"""
+
     def fit(self, X, y):
         self.y = y
         return self
@@ -174,34 +207,31 @@ def predict(self, X):
 
 class DummyScorer:
     """Dummy scorer that always returns 1."""
+
     def __call__(self, est, X, y):
         return 1
 
 
 def test_all_scorers_repr():
     # Test that all scorers have a working repr
-    for name, scorer in SCORERS.items():
-        repr(scorer)
+    for name in get_scorer_names():
+        repr(get_scorer(name))
 
 
 def check_scoring_validator_for_single_metric_usecases(scoring_validator):
     # Test all branches of single metric usecases
-    estimator = EstimatorWithoutFit()
-    pattern = (r"estimator should be an estimator implementing 'fit' method,"
-               r" .* was passed")
-    with pytest.raises(TypeError, match=pattern):
-        scoring_validator(estimator)
-
     estimator = EstimatorWithFitAndScore()
     estimator.fit([[1]], [1])
     scorer = scoring_validator(estimator)
-    assert scorer is _passthrough_scorer
+    assert isinstance(scorer, _PassthroughScorer)
     assert_almost_equal(scorer(estimator, [[1]], [1]), 1.0)
 
     estimator = EstimatorWithFitAndPredict()
     estimator.fit([[1]], [1])
-    pattern = (r"If no scoring is specified, the estimator passed should have"
-               r" a 'score' method\. The estimator .* does not\.")
+    pattern = (
+        r"If no scoring is specified, the estimator passed should have"
+        r" a 'score' method\. The estimator .* does not\."
+    )
     with pytest.raises(TypeError, match=pattern):
         scoring_validator(estimator)
 
@@ -210,7 +240,8 @@ def check_scoring_validator_for_single_metric_usecases(scoring_validator):
 
     estimator = EstimatorWithFit()
     scorer = scoring_validator(estimator, scoring="accuracy")
-    assert isinstance(scorer, _PredictScorer)
+    assert isinstance(scorer, _Scorer)
+    assert scorer._response_method == "predict"
 
     # Test the allow_none parameter for check_scoring alone
     if scoring_validator is check_scoring:
@@ -222,14 +253,25 @@ def check_scoring_validator_for_single_metric_usecases(scoring_validator):
 @pytest.mark.parametrize(
     "scoring",
     (
-        ('accuracy', ), ['precision'],
-        {'acc': 'accuracy', 'precision': 'precision'},
-        ('accuracy', 'precision'),
-        ['precision', 'accuracy'],
-        {'accuracy': make_scorer(accuracy_score),
-         'precision': make_scorer(precision_score)}
-    ), ids=["single_tuple", "single_list", "dict_str",
-            "multi_tuple", "multi_list", "dict_callable"])
+        ("accuracy",),
+        ["precision"],
+        {"acc": "accuracy", "precision": "precision"},
+        ("accuracy", "precision"),
+        ["precision", "accuracy"],
+        {
+            "accuracy": make_scorer(accuracy_score),
+            "precision": make_scorer(precision_score),
+        },
+    ),
+    ids=[
+        "single_tuple",
+        "single_list",
+        "dict_str",
+        "multi_tuple",
+        "multi_list",
+        "dict_callable",
+    ],
+)
 def test_check_scoring_and_check_multimetric_scoring(scoring):
     check_scoring_validator_for_single_metric_usecases(check_scoring)
     # To make sure the check_scoring is correctly applied to the constituent
@@ -241,43 +283,54 @@ def test_check_scoring_and_check_multimetric_scoring(scoring):
     scorers = _check_multimetric_scoring(estimator, scoring)
     assert isinstance(scorers, dict)
     assert sorted(scorers.keys()) == sorted(list(scoring))
-    assert all([isinstance(scorer, _PredictScorer)
-                for scorer in list(scorers.values())])
-
-    if 'acc' in scoring:
-        assert_almost_equal(scorers['acc'](
-            estimator, [[1], [2], [3]], [1, 0, 0]), 2. / 3.)
-    if 'accuracy' in scoring:
-        assert_almost_equal(scorers['accuracy'](
-            estimator, [[1], [2], [3]], [1, 0, 0]), 2. / 3.)
-    if 'precision' in scoring:
-        assert_almost_equal(scorers['precision'](
-            estimator, [[1], [2], [3]], [1, 0, 0]), 0.5)
-
-
-@pytest.mark.parametrize("scoring", [
-    ((make_scorer(precision_score), make_scorer(accuracy_score)),
-     "One or more of the elements were callables"),
-    ([5], "Non-string types were found"),
-    ((make_scorer(precision_score), ),
-     "One of mor eof the elements were callables"),
-    ((), "Empty list was given"),
-    (('f1', 'f1'), "Duplicate elements were found"),
-    ({4: 'accuracy'}, "Non-string types were found in the keys"),
-    ({}, "An empty dict was passed"),
-], ids=[
-    "tuple of callables", "list of int",
-    "tuple of one callable", "empty tuple",
-    "non-unique str", "non-string key dict",
-    "empty dict"])
-def test_check_scoring_and_check_multimetric_scoring_errors(scoring):
+    assert all([isinstance(scorer, _Scorer) for scorer in list(scorers.values())])
+    assert all(scorer._response_method == "predict" for scorer in scorers.values())
+
+    if "acc" in scoring:
+        assert_almost_equal(
+            scorers["acc"](estimator, [[1], [2], [3]], [1, 0, 0]), 2.0 / 3.0
+        )
+    if "accuracy" in scoring:
+        assert_almost_equal(
+            scorers["accuracy"](estimator, [[1], [2], [3]], [1, 0, 0]), 2.0 / 3.0
+        )
+    if "precision" in scoring:
+        assert_almost_equal(
+            scorers["precision"](estimator, [[1], [2], [3]], [1, 0, 0]), 0.5
+        )
+
+
+@pytest.mark.parametrize(
+    "scoring, msg",
+    [
+        (
+            (make_scorer(precision_score), make_scorer(accuracy_score)),
+            "One or more of the elements were callables",
+        ),
+        ([5], "Non-string types were found"),
+        ((make_scorer(precision_score),), "One or more of the elements were callables"),
+        ((), "Empty list was given"),
+        (("f1", "f1"), "Duplicate elements were found"),
+        ({4: "accuracy"}, "Non-string types were found in the keys"),
+        ({}, "An empty dict was passed"),
+    ],
+    ids=[
+        "tuple of callables",
+        "list of int",
+        "tuple of one callable",
+        "empty tuple",
+        "non-unique str",
+        "non-string key dict",
+        "empty dict",
+    ],
+)
+def test_check_scoring_and_check_multimetric_scoring_errors(scoring, msg):
     # Make sure it raises errors when scoring parameter is not valid.
     # More weird corner cases are tested at test_validation.py
     estimator = EstimatorWithFitAndPredict()
     estimator.fit([[1]], [1])
 
-    error_message_regexp = ".*must be unique strings.*"
-    with pytest.raises(ValueError, match=error_message_regexp):
+    with pytest.raises(ValueError, match=msg):
         _check_multimetric_scoring(estimator, scoring=scoring)
 
 
@@ -285,48 +338,48 @@ def test_check_scoring_gridsearchcv():
     # test that check_scoring works on GridSearchCV and pipeline.
     # slightly redundant non-regression test.
 
-    grid = GridSearchCV(LinearSVC(), param_grid={'C': [.1, 1]}, cv=3)
+    grid = GridSearchCV(LinearSVC(), param_grid={"C": [0.1, 1]}, cv=3)
     scorer = check_scoring(grid, scoring="f1")
-    assert isinstance(scorer, _PredictScorer)
+    assert isinstance(scorer, _Scorer)
+    assert scorer._response_method == "predict"
 
     pipe = make_pipeline(LinearSVC())
     scorer = check_scoring(pipe, scoring="f1")
-    assert isinstance(scorer, _PredictScorer)
+    assert isinstance(scorer, _Scorer)
+    assert scorer._response_method == "predict"
 
     # check that cross_val_score definitely calls the scorer
     # and doesn't make any assumptions about the estimator apart from having a
     # fit.
-    scores = cross_val_score(EstimatorWithFit(), [[1], [2], [3]], [1, 0, 1],
-                             scoring=DummyScorer(), cv=3)
+    scores = cross_val_score(
+        EstimatorWithFit(), [[1], [2], [3]], [1, 0, 1], scoring=DummyScorer(), cv=3
+    )
     assert_array_equal(scores, 1)
 
 
-def test_make_scorer():
-    # Sanity check on the make_scorer factory function.
-    f = lambda *args: 0
-    with pytest.raises(ValueError):
-        make_scorer(f, needs_threshold=True, needs_proba=True)
-
-
-@pytest.mark.parametrize('scorer_name, metric', [
-    ('f1', f1_score),
-    ('f1_weighted', partial(f1_score, average='weighted')),
-    ('f1_macro', partial(f1_score, average='macro')),
-    ('f1_micro', partial(f1_score, average='micro')),
-    ('precision', precision_score),
-    ('precision_weighted', partial(precision_score, average='weighted')),
-    ('precision_macro', partial(precision_score, average='macro')),
-    ('precision_micro', partial(precision_score, average='micro')),
-    ('recall', recall_score),
-    ('recall_weighted', partial(recall_score, average='weighted')),
-    ('recall_macro', partial(recall_score, average='macro')),
-    ('recall_micro', partial(recall_score, average='micro')),
-    ('jaccard', jaccard_score),
-    ('jaccard_weighted', partial(jaccard_score, average='weighted')),
-    ('jaccard_macro', partial(jaccard_score, average='macro')),
-    ('jaccard_micro', partial(jaccard_score, average='micro')),
-    ('top_k_accuracy', top_k_accuracy_score),
-])
+@pytest.mark.parametrize(
+    "scorer_name, metric",
+    [
+        ("f1", f1_score),
+        ("f1_weighted", partial(f1_score, average="weighted")),
+        ("f1_macro", partial(f1_score, average="macro")),
+        ("f1_micro", partial(f1_score, average="micro")),
+        ("precision", precision_score),
+        ("precision_weighted", partial(precision_score, average="weighted")),
+        ("precision_macro", partial(precision_score, average="macro")),
+        ("precision_micro", partial(precision_score, average="micro")),
+        ("recall", recall_score),
+        ("recall_weighted", partial(recall_score, average="weighted")),
+        ("recall_macro", partial(recall_score, average="macro")),
+        ("recall_micro", partial(recall_score, average="micro")),
+        ("jaccard", jaccard_score),
+        ("jaccard_weighted", partial(jaccard_score, average="weighted")),
+        ("jaccard_macro", partial(jaccard_score, average="macro")),
+        ("jaccard_micro", partial(jaccard_score, average="micro")),
+        ("top_k_accuracy", top_k_accuracy_score),
+        ("matthews_corrcoef", matthews_corrcoef),
+    ],
+)
 def test_classification_binary_scores(scorer_name, metric):
     # check consistency between score and scorer for scores supporting
     # binary classification.
@@ -335,27 +388,30 @@ def test_classification_binary_scores(scorer_name, metric):
     clf = LinearSVC(random_state=0)
     clf.fit(X_train, y_train)
 
-    score = SCORERS[scorer_name](clf, X_test, y_test)
+    score = get_scorer(scorer_name)(clf, X_test, y_test)
     expected_score = metric(y_test, clf.predict(X_test))
     assert_almost_equal(score, expected_score)
 
 
-@pytest.mark.parametrize('scorer_name, metric', [
-    ('accuracy', accuracy_score),
-    ('balanced_accuracy', balanced_accuracy_score),
-    ('f1_weighted', partial(f1_score, average='weighted')),
-    ('f1_macro', partial(f1_score, average='macro')),
-    ('f1_micro', partial(f1_score, average='micro')),
-    ('precision_weighted', partial(precision_score, average='weighted')),
-    ('precision_macro', partial(precision_score, average='macro')),
-    ('precision_micro', partial(precision_score, average='micro')),
-    ('recall_weighted', partial(recall_score, average='weighted')),
-    ('recall_macro', partial(recall_score, average='macro')),
-    ('recall_micro', partial(recall_score, average='micro')),
-    ('jaccard_weighted', partial(jaccard_score, average='weighted')),
-    ('jaccard_macro', partial(jaccard_score, average='macro')),
-    ('jaccard_micro', partial(jaccard_score, average='micro')),
-])
+@pytest.mark.parametrize(
+    "scorer_name, metric",
+    [
+        ("accuracy", accuracy_score),
+        ("balanced_accuracy", balanced_accuracy_score),
+        ("f1_weighted", partial(f1_score, average="weighted")),
+        ("f1_macro", partial(f1_score, average="macro")),
+        ("f1_micro", partial(f1_score, average="micro")),
+        ("precision_weighted", partial(precision_score, average="weighted")),
+        ("precision_macro", partial(precision_score, average="macro")),
+        ("precision_micro", partial(precision_score, average="micro")),
+        ("recall_weighted", partial(recall_score, average="weighted")),
+        ("recall_macro", partial(recall_score, average="macro")),
+        ("recall_micro", partial(recall_score, average="micro")),
+        ("jaccard_weighted", partial(jaccard_score, average="weighted")),
+        ("jaccard_macro", partial(jaccard_score, average="macro")),
+        ("jaccard_micro", partial(jaccard_score, average="micro")),
+    ],
+)
 def test_classification_multiclass_scores(scorer_name, metric):
     # check consistency between score and scorer for scores supporting
     # multiclass classification.
@@ -370,7 +426,7 @@ def test_classification_multiclass_scores(scorer_name, metric):
 
     clf = DecisionTreeClassifier(random_state=0)
     clf.fit(X_train, y_train)
-    score = SCORERS[scorer_name](clf, X_test, y_test)
+    score = get_scorer(scorer_name)(clf, X_test, y_test)
     expected_score = metric(y_test, clf.predict(X_test))
     assert score == pytest.approx(expected_score)
 
@@ -399,7 +455,7 @@ def test_regression_scorers():
     X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
     clf = Ridge()
     clf.fit(X_train, y_train)
-    score1 = get_scorer('r2')(clf, X_test, y_test)
+    score1 = get_scorer("r2")(clf, X_test, y_test)
     score2 = r2_score(y_test, clf.predict(X_test))
     assert_almost_equal(score1, score2)
 
@@ -410,36 +466,36 @@ def test_thresholded_scorers():
     X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
     clf = LogisticRegression(random_state=0)
     clf.fit(X_train, y_train)
-    score1 = get_scorer('roc_auc')(clf, X_test, y_test)
+    score1 = get_scorer("roc_auc")(clf, X_test, y_test)
     score2 = roc_auc_score(y_test, clf.decision_function(X_test))
     score3 = roc_auc_score(y_test, clf.predict_proba(X_test)[:, 1])
     assert_almost_equal(score1, score2)
     assert_almost_equal(score1, score3)
 
-    logscore = get_scorer('neg_log_loss')(clf, X_test, y_test)
+    logscore = get_scorer("neg_log_loss")(clf, X_test, y_test)
     logloss = log_loss(y_test, clf.predict_proba(X_test))
     assert_almost_equal(-logscore, logloss)
 
     # same for an estimator without decision_function
     clf = DecisionTreeClassifier()
     clf.fit(X_train, y_train)
-    score1 = get_scorer('roc_auc')(clf, X_test, y_test)
+    score1 = get_scorer("roc_auc")(clf, X_test, y_test)
     score2 = roc_auc_score(y_test, clf.predict_proba(X_test)[:, 1])
     assert_almost_equal(score1, score2)
 
     # test with a regressor (no decision_function)
     reg = DecisionTreeRegressor()
     reg.fit(X_train, y_train)
-    score1 = get_scorer('roc_auc')(reg, X_test, y_test)
-    score2 = roc_auc_score(y_test, reg.predict(X_test))
-    assert_almost_equal(score1, score2)
+    err_msg = "DecisionTreeRegressor has none of the following attributes"
+    with pytest.raises(AttributeError, match=err_msg):
+        get_scorer("roc_auc")(reg, X_test, y_test)
 
     # Test that an exception is raised on more than two classes
     X, y = make_blobs(random_state=0, centers=3)
     X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
     clf.fit(X_train, y_train)
-    with pytest.raises(ValueError, match="multiclass format is not supported"):
-        get_scorer('roc_auc')(clf, X_test, y_test)
+    with pytest.raises(ValueError, match="multi_class must be in \\('ovo', 'ovr'\\)"):
+        get_scorer("roc_auc")(clf, X_test, y_test)
 
     # test error is raised with a single class present in model
     # (predict_proba shape is not suitable for binary auc)
@@ -448,52 +504,38 @@ def test_thresholded_scorers():
     clf = DecisionTreeClassifier()
     clf.fit(X_train, np.zeros_like(y_train))
     with pytest.raises(ValueError, match="need classifier with two classes"):
-        get_scorer('roc_auc')(clf, X_test, y_test)
+        get_scorer("roc_auc")(clf, X_test, y_test)
 
     # for proba scorers
     with pytest.raises(ValueError, match="need classifier with two classes"):
-        get_scorer('neg_log_loss')(clf, X_test, y_test)
+        get_scorer("neg_log_loss")(clf, X_test, y_test)
 
 
 def test_thresholded_scorers_multilabel_indicator_data():
     # Test that the scorer work with multilabel-indicator format
     # for multilabel and multi-output multi-class classifier
-    X, y = make_multilabel_classification(allow_unlabeled=False,
-                                          random_state=0)
+    X, y = make_multilabel_classification(allow_unlabeled=False, random_state=0)
     X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
 
     # Multi-output multi-class predict_proba
     clf = DecisionTreeClassifier()
     clf.fit(X_train, y_train)
     y_proba = clf.predict_proba(X_test)
-    score1 = get_scorer('roc_auc')(clf, X_test, y_test)
+    score1 = get_scorer("roc_auc")(clf, X_test, y_test)
     score2 = roc_auc_score(y_test, np.vstack([p[:, -1] for p in y_proba]).T)
     assert_almost_equal(score1, score2)
 
-    # Multi-output multi-class decision_function
-    # TODO Is there any yet?
-    clf = DecisionTreeClassifier()
-    clf.fit(X_train, y_train)
-    clf._predict_proba = clf.predict_proba
-    clf.predict_proba = None
-    clf.decision_function = lambda X: [p[:, 1] for p in clf._predict_proba(X)]
-
-    y_proba = clf.decision_function(X_test)
-    score1 = get_scorer('roc_auc')(clf, X_test, y_test)
-    score2 = roc_auc_score(y_test, np.vstack([p for p in y_proba]).T)
-    assert_almost_equal(score1, score2)
-
     # Multilabel predict_proba
     clf = OneVsRestClassifier(DecisionTreeClassifier())
     clf.fit(X_train, y_train)
-    score1 = get_scorer('roc_auc')(clf, X_test, y_test)
+    score1 = get_scorer("roc_auc")(clf, X_test, y_test)
     score2 = roc_auc_score(y_test, clf.predict_proba(X_test))
     assert_almost_equal(score1, score2)
 
     # Multilabel decision function
     clf = OneVsRestClassifier(LinearSVC(random_state=0))
     clf.fit(X_train, y_train)
-    score1 = get_scorer('roc_auc')(clf, X_test, y_test)
+    score1 = get_scorer("roc_auc")(clf, X_test, y_test)
     score2 = roc_auc_score(y_test, clf.decision_function(X_test))
     assert_almost_equal(score1, score2)
 
@@ -502,7 +544,7 @@ def test_supervised_cluster_scorers():
     # Test clustering scorers against gold standard labeling.
     X, y = make_blobs(random_state=0, centers=2)
     X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
-    km = KMeans(n_clusters=3)
+    km = KMeans(n_clusters=3, n_init="auto")
     km.fit(X_train)
     for name in CLUSTER_SCORERS:
         score1 = get_scorer(name)(km, X_test, y_test)
@@ -510,7 +552,6 @@ def test_supervised_cluster_scorers():
         assert_almost_equal(score1, score2)
 
 
-@ignore_warnings
 def test_raises_on_score_list():
     # Test that when a list of scores is returned, we raise proper errors.
     X, y = make_blobs(random_state=0)
@@ -518,13 +559,13 @@ def test_raises_on_score_list():
     clf = DecisionTreeClassifier()
     with pytest.raises(ValueError):
         cross_val_score(clf, X, y, scoring=f1_scorer_no_average)
-    grid_search = GridSearchCV(clf, scoring=f1_scorer_no_average,
-                               param_grid={'max_depth': [1, 2]})
+    grid_search = GridSearchCV(
+        clf, scoring=f1_scorer_no_average, param_grid={"max_depth": [1, 2]}
+    )
     with pytest.raises(ValueError):
         grid_search.fit(X, y)
 
 
-@ignore_warnings
 def test_classification_scorer_sample_weight():
     # Test that classification scorers support sample_weight or raise sensible
     # errors
@@ -533,8 +574,7 @@ def test_classification_scorer_sample_weight():
     # to ensure that, on the classifier output, weighted and unweighted
     # scores really should be unequal.
     X, y = make_classification(random_state=0)
-    _, y_ml = make_multilabel_classification(n_samples=X.shape[0],
-                                             random_state=0)
+    _, y_ml = make_multilabel_classification(n_samples=X.shape[0], random_state=0)
     split = train_test_split(X, y, y_ml, random_state=0)
     X_train, X_test, y_train, y_test, y_ml_train, y_ml_test = split
 
@@ -544,37 +584,47 @@ def test_classification_scorer_sample_weight():
     # get sensible estimators for each metric
     estimator = _make_estimators(X_train, y_train, y_ml_train)
 
-    for name, scorer in SCORERS.items():
+    for name in get_scorer_names():
+        scorer = get_scorer(name)
         if name in REGRESSION_SCORERS:
             # skip the regression scores
             continue
-        if name == 'top_k_accuracy':
+        if name == "top_k_accuracy":
             # in the binary case k > 1 will always lead to a perfect score
-            scorer._kwargs = {'k': 1}
+            scorer._kwargs = {"k": 1}
         if name in MULTILABEL_ONLY_SCORERS:
             target = y_ml_test
         else:
             target = y_test
         try:
-            weighted = scorer(estimator[name], X_test, target,
-                              sample_weight=sample_weight)
+            weighted = scorer(
+                estimator[name], X_test, target, sample_weight=sample_weight
+            )
             ignored = scorer(estimator[name], X_test[10:], target[10:])
             unweighted = scorer(estimator[name], X_test, target)
+            # this should not raise. sample_weight should be ignored if None.
+            _ = scorer(estimator[name], X_test[:10], target[:10], sample_weight=None)
             assert weighted != unweighted, (
                 f"scorer {name} behaves identically when called with "
-                f"sample weights: {weighted} vs {unweighted}")
-            assert_almost_equal(weighted, ignored,
-                                err_msg=f"scorer {name} behaves differently "
-                                f"when ignoring samples and setting "
-                                f"sample_weight to 0: {weighted} vs {ignored}")
+                f"sample weights: {weighted} vs {unweighted}"
+            )
+            assert_almost_equal(
+                weighted,
+                ignored,
+                err_msg=(
+                    f"scorer {name} behaves differently "
+                    "when ignoring samples and setting "
+                    f"sample_weight to 0: {weighted} vs {ignored}"
+                ),
+            )
 
         except TypeError as e:
             assert "sample_weight" in str(e), (
-                   f"scorer {name} raises unhelpful exception when called "
-                   f"with sample weights: {str(e)}")
+                f"scorer {name} raises unhelpful exception when called "
+                f"with sample weights: {e}"
+            )
 
 
-@ignore_warnings
 def test_regression_scorer_sample_weight():
     # Test that regression scorers support sample_weight or raise sensible
     # errors
@@ -591,34 +641,42 @@ def test_regression_scorer_sample_weight():
     reg = DecisionTreeRegressor(random_state=0)
     reg.fit(X_train, y_train)
 
-    for name, scorer in SCORERS.items():
+    for name in get_scorer_names():
+        scorer = get_scorer(name)
         if name not in REGRESSION_SCORERS:
             # skip classification scorers
             continue
         try:
-            weighted = scorer(reg, X_test, y_test,
-                              sample_weight=sample_weight)
+            weighted = scorer(reg, X_test, y_test, sample_weight=sample_weight)
             ignored = scorer(reg, X_test[11:], y_test[11:])
             unweighted = scorer(reg, X_test, y_test)
             assert weighted != unweighted, (
                 f"scorer {name} behaves identically when called with "
-                f"sample weights: {weighted} vs {unweighted}")
-            assert_almost_equal(weighted, ignored,
-                                err_msg=f"scorer {name} behaves differently "
-                                f"when ignoring samples and setting "
-                                f"sample_weight to 0: {weighted} vs {ignored}")
+                f"sample weights: {weighted} vs {unweighted}"
+            )
+            assert_almost_equal(
+                weighted,
+                ignored,
+                err_msg=(
+                    f"scorer {name} behaves differently "
+                    "when ignoring samples and setting "
+                    f"sample_weight to 0: {weighted} vs {ignored}"
+                ),
+            )
 
         except TypeError as e:
             assert "sample_weight" in str(e), (
-                   f"scorer {name} raises unhelpful exception when called "
-                   f"with sample weights: {str(e)}")
+                f"scorer {name} raises unhelpful exception when called "
+                f"with sample weights: {e}"
+            )
 
 
-@pytest.mark.parametrize('name', SCORERS)
-def test_scorer_memmap_input(name):
+@pytest.mark.parametrize("name", get_scorer_names())
+def test_scorer_memmap_input(name, memmap_data_and_estimators):
     # Non-regression test for #6147: some score functions would
     # return singleton memmap when computed on memmap data instead of scalar
     # float values.
+    X_mm, y_mm, y_ml_mm, estimators = memmap_data_and_estimators
 
     if name in REQUIRE_POSITIVE_Y_SCORERS:
         y_mm_1 = _require_positive_y(y_mm)
@@ -628,7 +686,7 @@ def test_scorer_memmap_input(name):
 
     # UndefinedMetricWarning for P / R scores
     with ignore_warnings():
-        scorer, estimator = SCORERS[name], ESTIMATORS[name]
+        scorer, estimator = get_scorer(name), estimators[name]
         if name in MULTILABEL_ONLY_SCORERS:
             score = scorer(estimator, X_mm, y_ml_mm_1)
         else:
@@ -637,59 +695,111 @@ def test_scorer_memmap_input(name):
 
 
 def test_scoring_is_not_metric():
-    with pytest.raises(ValueError, match='make_scorer'):
+    with pytest.raises(ValueError, match="make_scorer"):
         check_scoring(LogisticRegression(), scoring=f1_score)
-    with pytest.raises(ValueError, match='make_scorer'):
+    with pytest.raises(ValueError, match="make_scorer"):
         check_scoring(LogisticRegression(), scoring=roc_auc_score)
-    with pytest.raises(ValueError, match='make_scorer'):
+    with pytest.raises(ValueError, match="make_scorer"):
         check_scoring(Ridge(), scoring=r2_score)
-    with pytest.raises(ValueError, match='make_scorer'):
+    with pytest.raises(ValueError, match="make_scorer"):
         check_scoring(KMeans(), scoring=cluster_module.adjusted_rand_score)
-    with pytest.raises(ValueError, match='make_scorer'):
+    with pytest.raises(ValueError, match="make_scorer"):
         check_scoring(KMeans(), scoring=cluster_module.rand_score)
 
 
-@pytest.mark.parametrize(
-    ("scorers,expected_predict_count,"
-     "expected_predict_proba_count,expected_decision_func_count"),
-    [({'a1': 'accuracy', 'a2': 'accuracy',
-       'll1': 'neg_log_loss', 'll2': 'neg_log_loss',
-        'ra1': 'roc_auc', 'ra2': 'roc_auc'}, 1, 1, 1),
-     (['roc_auc', 'accuracy'], 1, 0, 1),
-     (['neg_log_loss', 'accuracy'], 1, 1, 0)])
-def test_multimetric_scorer_calls_method_once(scorers, expected_predict_count,
-                                              expected_predict_proba_count,
-                                              expected_decision_func_count):
-    X, y = np.array([[1], [1], [0], [0], [0]]), np.array([0, 1, 1, 1, 0])
+def test_deprecated_scorer():
+    X, y = make_regression(n_samples=10, n_features=1, random_state=0)
+    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
+    reg = DecisionTreeRegressor()
+    reg.fit(X_train, y_train)
+    deprecated_scorer = get_scorer("max_error")
+    with pytest.warns(DeprecationWarning):
+        deprecated_scorer(reg, X_test, y_test)
 
-    mock_est = Mock()
-    fit_func = Mock(return_value=mock_est)
-    predict_func = Mock(return_value=y)
 
+@pytest.mark.parametrize(
+    (
+        "scorers,expected_predict_count,"
+        "expected_predict_proba_count,expected_decision_func_count"
+    ),
+    [
+        (
+            {
+                "a1": "accuracy",
+                "a2": "accuracy",
+                "ll1": "neg_log_loss",
+                "ll2": "neg_log_loss",
+                "ra1": "roc_auc",
+                "ra2": "roc_auc",
+            },
+            1,
+            1,
+            1,
+        ),
+        (["roc_auc", "accuracy"], 1, 0, 1),
+        (["neg_log_loss", "accuracy"], 1, 1, 0),
+    ],
+)
+def test_multimetric_scorer_calls_method_once(
+    scorers,
+    expected_predict_count,
+    expected_predict_proba_count,
+    expected_decision_func_count,
+):
+    X, y = np.array([[1], [1], [0], [0], [0]]), np.array([0, 1, 1, 1, 0])
     pos_proba = np.random.rand(X.shape[0])
     proba = np.c_[1 - pos_proba, pos_proba]
-    predict_proba_func = Mock(return_value=proba)
-    decision_function_func = Mock(return_value=pos_proba)
 
-    mock_est.fit = fit_func
-    mock_est.predict = predict_func
-    mock_est.predict_proba = predict_proba_func
-    mock_est.decision_function = decision_function_func
-    # add the classes that would be found during fit
-    mock_est.classes_ = np.array([0, 1])
+    class MyClassifier(ClassifierMixin, BaseEstimator):
+        def __init__(self):
+            self._expected_predict_count = 0
+            self._expected_predict_proba_count = 0
+            self._expected_decision_function_count = 0
+
+        def fit(self, X, y):
+            self.classes_ = np.unique(y)
+            return self
+
+        def predict(self, X):
+            self._expected_predict_count += 1
+            return y
+
+        def predict_proba(self, X):
+            self._expected_predict_proba_count += 1
+            return proba
+
+        def decision_function(self, X):
+            self._expected_decision_function_count += 1
+            return pos_proba
 
+    mock_est = MyClassifier().fit(X, y)
     scorer_dict = _check_multimetric_scoring(LogisticRegression(), scorers)
-    multi_scorer = _MultimetricScorer(**scorer_dict)
+    multi_scorer = _MultimetricScorer(scorers=scorer_dict)
     results = multi_scorer(mock_est, X, y)
 
     assert set(scorers) == set(results)  # compare dict keys
 
-    assert predict_func.call_count == expected_predict_count
-    assert predict_proba_func.call_count == expected_predict_proba_count
-    assert decision_function_func.call_count == expected_decision_func_count
+    assert mock_est._expected_predict_count == expected_predict_count
+    assert mock_est._expected_predict_proba_count == expected_predict_proba_count
+    assert mock_est._expected_decision_function_count == expected_decision_func_count
 
 
-def test_multimetric_scorer_calls_method_once_classifier_no_decision():
+@pytest.mark.parametrize(
+    "scorers",
+    [
+        (["roc_auc", "neg_log_loss"]),
+        (
+            {
+                "roc_auc": make_scorer(
+                    roc_auc_score,
+                    response_method=["predict_proba", "decision_function"],
+                ),
+                "neg_log_loss": make_scorer(log_loss, response_method="predict_proba"),
+            }
+        ),
+    ],
+)
+def test_multimetric_scorer_calls_method_once_classifier_no_decision(scorers):
     predict_proba_call_cnt = 0
 
     class MockKNeighborsClassifier(KNeighborsClassifier):
@@ -704,9 +814,8 @@ def predict_proba(self, X):
     clf = MockKNeighborsClassifier(n_neighbors=1)
     clf.fit(X, y)
 
-    scorers = ['roc_auc', 'neg_log_loss']
     scorer_dict = _check_multimetric_scoring(clf, scorers)
-    scorer = _MultimetricScorer(**scorer_dict)
+    scorer = _MultimetricScorer(scorers=scorer_dict)
     scorer(clf, X, y)
 
     assert predict_proba_call_cnt == 1
@@ -727,9 +836,9 @@ def predict(self, X):
     clf = MockDecisionTreeRegressor()
     clf.fit(X, y)
 
-    scorers = {'neg_mse': 'neg_mean_squared_error', 'r2': 'roc_auc'}
+    scorers = {"neg_mse": "neg_mean_squared_error", "r2": "r2"}
     scorer_dict = _check_multimetric_scoring(clf, scorers)
-    scorer = _MultimetricScorer(**scorer_dict)
+    scorer = _MultimetricScorer(scorers=scorer_dict)
     scorer(clf, X, y)
 
     assert predict_called_cnt == 1
@@ -737,9 +846,14 @@ def predict(self, X):
 
 def test_multimetric_scorer_sanity_check():
     # scoring dictionary returned is the same as calling each scorer separately
-    scorers = {'a1': 'accuracy', 'a2': 'accuracy',
-               'll1': 'neg_log_loss', 'll2': 'neg_log_loss',
-               'ra1': 'roc_auc', 'ra2': 'roc_auc'}
+    scorers = {
+        "a1": "accuracy",
+        "a2": "accuracy",
+        "ll1": "neg_log_loss",
+        "ll2": "neg_log_loss",
+        "ra1": "roc_auc",
+        "ra2": "roc_auc",
+    }
 
     X, y = make_classification(random_state=0)
 
@@ -747,31 +861,85 @@ def test_multimetric_scorer_sanity_check():
     clf.fit(X, y)
 
     scorer_dict = _check_multimetric_scoring(clf, scorers)
-    multi_scorer = _MultimetricScorer(**scorer_dict)
+    multi_scorer = _MultimetricScorer(scorers=scorer_dict)
 
     result = multi_scorer(clf, X, y)
 
     separate_scores = {
         name: get_scorer(name)(clf, X, y)
-        for name in ['accuracy', 'neg_log_loss', 'roc_auc']}
+        for name in ["accuracy", "neg_log_loss", "roc_auc"]
+    }
 
     for key, value in result.items():
         score_name = scorers[key]
         assert_allclose(value, separate_scores[score_name])
 
 
-@pytest.mark.parametrize('scorer_name, metric', [
-    ('roc_auc_ovr', partial(roc_auc_score, multi_class='ovr')),
-    ('roc_auc_ovo', partial(roc_auc_score, multi_class='ovo')),
-    ('roc_auc_ovr_weighted', partial(roc_auc_score, multi_class='ovr',
-                                     average='weighted')),
-    ('roc_auc_ovo_weighted', partial(roc_auc_score, multi_class='ovo',
-                                     average='weighted'))])
+@pytest.mark.parametrize("raise_exc", [True, False])
+def test_multimetric_scorer_exception_handling(raise_exc):
+    """Check that the calling of the `_MultimetricScorer` returns
+    exception messages in the result dict for the failing scorers
+    in case of `raise_exc` is `False` and if `raise_exc` is `True`,
+    then the proper exception is raised.
+    """
+    scorers = {
+        "failing_1": "neg_mean_squared_log_error",
+        "non_failing": "neg_median_absolute_error",
+        "failing_2": "neg_mean_squared_log_error",
+    }
+
+    X, y = make_classification(
+        n_samples=50, n_features=2, n_redundant=0, random_state=0
+    )
+    # neg_mean_squared_log_error fails if y contains values less than or equal to -1
+    y *= -1
+
+    clf = DecisionTreeClassifier().fit(X, y)
+
+    scorer_dict = _check_multimetric_scoring(clf, scorers)
+    multi_scorer = _MultimetricScorer(scorers=scorer_dict, raise_exc=raise_exc)
+
+    error_msg = (
+        "Mean Squared Logarithmic Error cannot be used when "
+        "targets contain values less than or equal to -1."
+    )
+
+    if raise_exc:
+        with pytest.raises(ValueError, match=error_msg):
+            multi_scorer(clf, X, y)
+    else:
+        result = multi_scorer(clf, X, y)
+
+        exception_message_1 = result["failing_1"]
+        score = result["non_failing"]
+        exception_message_2 = result["failing_2"]
+
+        assert isinstance(exception_message_1, str) and error_msg in exception_message_1
+        assert isinstance(score, float)
+        assert isinstance(exception_message_2, str) and error_msg in exception_message_2
+
+
+@pytest.mark.parametrize(
+    "scorer_name, metric",
+    [
+        ("roc_auc_ovr", partial(roc_auc_score, multi_class="ovr")),
+        ("roc_auc_ovo", partial(roc_auc_score, multi_class="ovo")),
+        (
+            "roc_auc_ovr_weighted",
+            partial(roc_auc_score, multi_class="ovr", average="weighted"),
+        ),
+        (
+            "roc_auc_ovo_weighted",
+            partial(roc_auc_score, multi_class="ovo", average="weighted"),
+        ),
+    ],
+)
 def test_multiclass_roc_proba_scorer(scorer_name, metric):
     scorer = get_scorer(scorer_name)
-    X, y = make_classification(n_classes=3, n_informative=3, n_samples=20,
-                               random_state=0)
-    lr = LogisticRegression(multi_class="multinomial").fit(X, y)
+    X, y = make_classification(
+        n_classes=3, n_informative=3, n_samples=20, random_state=0
+    )
+    lr = LogisticRegression().fit(X, y)
     y_proba = lr.predict_proba(X)
     expected_score = metric(y, y_proba)
 
@@ -779,31 +947,38 @@ def test_multiclass_roc_proba_scorer(scorer_name, metric):
 
 
 def test_multiclass_roc_proba_scorer_label():
-    scorer = make_scorer(roc_auc_score, multi_class='ovo',
-                         labels=[0, 1, 2], needs_proba=True)
-    X, y = make_classification(n_classes=3, n_informative=3, n_samples=20,
-                               random_state=0)
-    lr = LogisticRegression(multi_class="multinomial").fit(X, y)
+    scorer = make_scorer(
+        roc_auc_score,
+        multi_class="ovo",
+        labels=[0, 1, 2],
+        response_method="predict_proba",
+    )
+    X, y = make_classification(
+        n_classes=3, n_informative=3, n_samples=20, random_state=0
+    )
+    lr = LogisticRegression().fit(X, y)
     y_proba = lr.predict_proba(X)
 
     y_binary = y == 0
-    expected_score = roc_auc_score(y_binary, y_proba,
-                                   multi_class='ovo',
-                                   labels=[0, 1, 2])
+    expected_score = roc_auc_score(
+        y_binary, y_proba, multi_class="ovo", labels=[0, 1, 2]
+    )
 
     assert scorer(lr, X, y_binary) == pytest.approx(expected_score)
 
 
-@pytest.mark.parametrize('scorer_name', [
-    'roc_auc_ovr', 'roc_auc_ovo',
-    'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted'])
+@pytest.mark.parametrize(
+    "scorer_name",
+    ["roc_auc_ovr", "roc_auc_ovo", "roc_auc_ovr_weighted", "roc_auc_ovo_weighted"],
+)
 def test_multiclass_roc_no_proba_scorer_errors(scorer_name):
     # Perceptron has no predict_proba
     scorer = get_scorer(scorer_name)
-    X, y = make_classification(n_classes=3, n_informative=3, n_samples=20,
-                               random_state=0)
+    X, y = make_classification(
+        n_classes=3, n_informative=3, n_samples=20, random_state=0
+    )
     lr = Perceptron().fit(X, y)
-    msg = "'Perceptron' object has no attribute 'predict_proba'"
+    msg = "Perceptron has none of the following attributes: predict_proba."
     with pytest.raises(AttributeError, match=msg):
         scorer(lr, X, y)
 
@@ -849,11 +1024,12 @@ def string_labeled_classification_problem():
     X, y = shuffle(X, y, random_state=42)
     # only use 2 features to make the problem even harder
     X = X[:, :2]
-    y = np.array(
-        ["cancer" if c == 1 else "not cancer" for c in y], dtype=object
-    )
+    y = np.array(["cancer" if c == 1 else "not cancer" for c in y], dtype=object)
     X_train, X_test, y_train, y_test = train_test_split(
-        X, y, stratify=y, random_state=0,
+        X,
+        y,
+        stratify=y,
+        random_state=0,
     )
     classifier = LogisticRegression().fit(X_train, y_train)
     y_pred = classifier.predict(X_test)
@@ -864,11 +1040,17 @@ def string_labeled_classification_problem():
 
 
 def test_average_precision_pos_label(string_labeled_classification_problem):
-    # check that _ThresholdScorer will lead to the right score when passing
+    # check that _Scorer will lead to the right score when passing
     # `pos_label`. Currently, only `average_precision_score` is defined to
     # be such a scorer.
-    clf, X_test, y_test, _, y_pred_proba, y_pred_decision = \
-        string_labeled_classification_problem
+    (
+        clf,
+        X_test,
+        y_test,
+        _,
+        y_pred_proba,
+        y_pred_decision,
+    ) = string_labeled_classification_problem
 
     pos_label = "cancer"
     # we need to select the positive column or reverse the decision values
@@ -878,9 +1060,7 @@ def test_average_precision_pos_label(string_labeled_classification_problem):
 
     # check that when calling the scoring function, probability estimates and
     # decision values lead to the same results
-    ap_proba = average_precision_score(
-        y_test, y_pred_proba, pos_label=pos_label
-    )
+    ap_proba = average_precision_score(y_test, y_pred_proba, pos_label=pos_label)
     ap_decision_function = average_precision_score(
         y_test, y_pred_decision, pos_label=pos_label
     )
@@ -889,7 +1069,8 @@ def test_average_precision_pos_label(string_labeled_classification_problem):
     # create a scorer which would require to pass a `pos_label`
     # check that it fails if `pos_label` is not provided
     average_precision_scorer = make_scorer(
-        average_precision_score, needs_threshold=True,
+        average_precision_score,
+        response_method=("decision_function", "predict_proba"),
     )
     err_msg = "pos_label=1 is not a valid label. It should be one of "
     with pytest.raises(ValueError, match=err_msg):
@@ -898,7 +1079,9 @@ def test_average_precision_pos_label(string_labeled_classification_problem):
     # otherwise, the scorer should give the same results than calling the
     # scoring function
     average_precision_scorer = make_scorer(
-        average_precision_score, needs_threshold=True, pos_label=pos_label
+        average_precision_score,
+        response_method=("decision_function", "predict_proba"),
+        pos_label=pos_label,
     )
     ap_scorer = average_precision_scorer(clf, X_test, y_test)
 
@@ -918,33 +1101,30 @@ def _predict_proba(self, X):
     with pytest.raises(NotImplementedError):
         clf_without_predict_proba.predict_proba(X_test)
 
-    ap_scorer = average_precision_scorer(
-        clf_without_predict_proba, X_test, y_test
-    )
+    ap_scorer = average_precision_scorer(clf_without_predict_proba, X_test, y_test)
     assert ap_scorer == pytest.approx(ap_proba)
 
 
 def test_brier_score_loss_pos_label(string_labeled_classification_problem):
-    # check that _ProbaScorer leads to the right score when `pos_label` is
+    # check that _Scorer leads to the right score when `pos_label` is
     # provided. Currently only the `brier_score_loss` is defined to be such
     # a scorer.
-    clf, X_test, y_test, _, y_pred_proba, _ = \
-        string_labeled_classification_problem
+    clf, X_test, y_test, _, y_pred_proba, _ = string_labeled_classification_problem
 
     pos_label = "cancer"
     assert clf.classes_[0] == pos_label
 
     # brier score loss is symmetric
-    brier_pos_cancer = brier_score_loss(
-        y_test, y_pred_proba[:, 0], pos_label="cancer"
-    )
+    brier_pos_cancer = brier_score_loss(y_test, y_pred_proba[:, 0], pos_label="cancer")
     brier_pos_not_cancer = brier_score_loss(
         y_test, y_pred_proba[:, 1], pos_label="not cancer"
     )
     assert brier_pos_cancer == pytest.approx(brier_pos_not_cancer)
 
     brier_scorer = make_scorer(
-        brier_score_loss, needs_proba=True, pos_label=pos_label,
+        brier_score_loss,
+        response_method="predict_proba",
+        pos_label=pos_label,
     )
     assert brier_scorer(clf, X_test, y_test) == pytest.approx(brier_pos_cancer)
 
@@ -955,7 +1135,7 @@ def test_brier_score_loss_pos_label(string_labeled_classification_problem):
 def test_non_symmetric_metric_pos_label(
     score_func, string_labeled_classification_problem
 ):
-    # check that _PredictScorer leads to the right score when `pos_label` is
+    # check that _Scorer leads to the right score when `pos_label` is
     # provided. We check for all possible metric supported.
     # Note: At some point we may end up having "scorer tags".
     clf, X_test, y_test, y_pred, _, _ = string_labeled_classification_problem
@@ -976,15 +1156,17 @@ def test_non_symmetric_metric_pos_label(
     "scorer",
     [
         make_scorer(
-            average_precision_score, needs_threshold=True, pos_label="xxx"
+            average_precision_score,
+            response_method=("decision_function", "predict_proba"),
+            pos_label="xxx",
         ),
-        make_scorer(brier_score_loss, needs_proba=True, pos_label="xxx"),
-        make_scorer(f1_score, pos_label="xxx")
+        make_scorer(brier_score_loss, response_method="predict_proba", pos_label="xxx"),
+        make_scorer(f1_score, pos_label="xxx"),
     ],
-    ids=["ThresholdScorer", "ProbaScorer", "PredictScorer"],
+    ids=["non-thresholded scorer", "probability scorer", "thresholded scorer"],
 )
 def test_scorer_select_proba_error(scorer):
-    # check that we raise the the proper error when passing an unknown
+    # check that we raise the proper error when passing an unknown
     # pos_label
     X, y = make_classification(
         n_classes=2, n_informative=3, n_samples=20, random_state=0
@@ -997,8 +1179,13 @@ def test_scorer_select_proba_error(scorer):
         scorer(lr, X, y)
 
 
+def test_get_scorer_return_copy():
+    # test that get_scorer returns a copy
+    assert get_scorer("roc_auc") is not get_scorer("roc_auc")
+
+
 def test_scorer_no_op_multiclass_select_proba():
-    # check that calling a ProbaScorer on a multiclass problem do not raise
+    # check that calling a _Scorer on a multiclass problem do not raise
     # even if `y_true` would be binary during the scoring.
     # `_select_proba_binary` should not be called in this case.
     X, y = make_classification(
@@ -1011,6 +1198,468 @@ def test_scorer_no_op_multiclass_select_proba():
     assert_array_equal(np.unique(y_test), lr.classes_[:-1])
 
     scorer = make_scorer(
-        roc_auc_score, needs_proba=True, multi_class="ovo", labels=lr.classes_,
+        roc_auc_score,
+        response_method="predict_proba",
+        multi_class="ovo",
+        labels=lr.classes_,
     )
     scorer(lr, X_test, y_test)
+
+
+@pytest.mark.parametrize("name", get_scorer_names())
+def test_scorer_set_score_request_raises(name):
+    """Test that set_score_request is only available when feature flag is on."""
+    # Make sure they expose the routing methods.
+    scorer = get_scorer(name)
+    with pytest.raises(RuntimeError, match="This method is only available"):
+        scorer.set_score_request()
+
+
+@pytest.mark.parametrize("name", get_scorer_names(), ids=get_scorer_names())
+@config_context(enable_metadata_routing=True)
+def test_scorer_metadata_request(name):
+    """Testing metadata requests for scorers.
+
+    This test checks many small things in a large test, to reduce the
+    boilerplate required for each section.
+    """
+    # Make sure they expose the routing methods.
+    scorer = get_scorer(name)
+    assert hasattr(scorer, "set_score_request")
+    assert hasattr(scorer, "get_metadata_routing")
+
+    # Check that by default no metadata is requested.
+    assert_request_is_empty(scorer.get_metadata_routing())
+
+    weighted_scorer = scorer.set_score_request(sample_weight=True)
+    # set_score_request should mutate the instance, rather than returning a
+    # new instance
+    assert weighted_scorer is scorer
+
+    # make sure the scorer doesn't request anything on methods other than
+    # `score`, and that the requested value on `score` is correct.
+    assert_request_is_empty(weighted_scorer.get_metadata_routing(), exclude="score")
+    assert (
+        weighted_scorer.get_metadata_routing().score.requests["sample_weight"] is True
+    )
+
+    # make sure putting the scorer in a router doesn't request anything by
+    # default
+    router = MetadataRouter(owner="test").add(
+        scorer=get_scorer(name),
+        method_mapping=MethodMapping().add(caller="score", callee="score"),
+    )
+    # make sure `sample_weight` is refused if passed.
+    with pytest.raises(TypeError, match="got unexpected argument"):
+        router.validate_metadata(params={"sample_weight": 1}, method="score")
+    # make sure `sample_weight` is not routed even if passed.
+    routed_params = router.route_params(params={"sample_weight": 1}, caller="score")
+    assert not routed_params.scorer.score
+
+    # make sure putting weighted_scorer in a router requests sample_weight
+    router = MetadataRouter(owner="test").add(
+        scorer=weighted_scorer,
+        method_mapping=MethodMapping().add(caller="score", callee="score"),
+    )
+    router.validate_metadata(params={"sample_weight": 1}, method="score")
+    routed_params = router.route_params(params={"sample_weight": 1}, caller="score")
+    assert list(routed_params.scorer.score.keys()) == ["sample_weight"]
+
+
+@config_context(enable_metadata_routing=True)
+def test_metadata_kwarg_conflict():
+    """This test makes sure the right warning is raised if the user passes
+    some metadata both as a constructor to make_scorer, and during __call__.
+    """
+    X, y = make_classification(
+        n_classes=3, n_informative=3, n_samples=20, random_state=0
+    )
+    lr = LogisticRegression().fit(X, y)
+
+    scorer = make_scorer(
+        roc_auc_score,
+        response_method="predict_proba",
+        multi_class="ovo",
+        labels=lr.classes_,
+    )
+    with pytest.warns(UserWarning, match="already set as kwargs"):
+        scorer.set_score_request(labels=True)
+
+    with pytest.warns(UserWarning, match="There is an overlap"):
+        scorer(lr, X, y, labels=lr.classes_)
+
+
+@config_context(enable_metadata_routing=True)
+def test_PassthroughScorer_set_score_request():
+    """Test that _PassthroughScorer.set_score_request adds the correct metadata request
+    on itself and doesn't change its estimator's routing."""
+    est = LogisticRegression().set_score_request(sample_weight="estimator_weights")
+    # make a `_PassthroughScorer` with `check_scoring`:
+    scorer = check_scoring(est, None)
+    assert (
+        scorer.get_metadata_routing().score.requests["sample_weight"]
+        == "estimator_weights"
+    )
+
+    scorer.set_score_request(sample_weight="scorer_weights")
+    assert (
+        scorer.get_metadata_routing().score.requests["sample_weight"]
+        == "scorer_weights"
+    )
+
+    # making sure changing the passthrough object doesn't affect the estimator.
+    assert (
+        est.get_metadata_routing().score.requests["sample_weight"]
+        == "estimator_weights"
+    )
+
+
+def test_PassthroughScorer_set_score_request_raises_without_routing_enabled():
+    """Test that _PassthroughScorer.set_score_request raises if metadata routing is
+    disabled."""
+    scorer = check_scoring(LogisticRegression(), None)
+    msg = "This method is only available when metadata routing is enabled."
+
+    with pytest.raises(RuntimeError, match=msg):
+        scorer.set_score_request(sample_weight="my_weights")
+
+
+@config_context(enable_metadata_routing=True)
+def test_multimetric_scoring_metadata_routing():
+    # Test that _MultimetricScorer properly routes metadata.
+    def score1(y_true, y_pred):
+        return 1
+
+    def score2(y_true, y_pred, sample_weight="test"):
+        # make sure sample_weight is not passed
+        assert sample_weight == "test"
+        return 1
+
+    def score3(y_true, y_pred, sample_weight=None):
+        # make sure sample_weight is passed
+        assert sample_weight is not None
+        return 1
+
+    scorers = {
+        "score1": make_scorer(score1),
+        "score2": make_scorer(score2).set_score_request(sample_weight=False),
+        "score3": make_scorer(score3).set_score_request(sample_weight=True),
+    }
+
+    X, y = make_classification(
+        n_samples=50, n_features=2, n_redundant=0, random_state=0
+    )
+
+    clf = DecisionTreeClassifier().fit(X, y)
+
+    scorer_dict = _check_multimetric_scoring(clf, scorers)
+    multi_scorer = _MultimetricScorer(scorers=scorer_dict)
+    # This passes since routing is done.
+    multi_scorer(clf, X, y, sample_weight=1)
+
+
+@config_context(enable_metadata_routing=False)
+def test_multimetric_scoring_kwargs():
+    # Test that _MultimetricScorer correctly forwards kwargs
+    # to the scorers when metadata routing is disabled.
+    # `sample_weight` is only forwarded to the scorers that accept it.
+    # Other arguments are forwarded to all scorers.
+    def score1(y_true, y_pred, common_arg=None):
+        # make sure common_arg is passed
+        assert common_arg is not None
+        return 1
+
+    def score2(y_true, y_pred, common_arg=None, sample_weight=None):
+        # make sure common_arg is passed
+        assert common_arg is not None
+        # make sure sample_weight is passed
+        assert sample_weight is not None
+        return 1
+
+    scorers = {
+        "score1": make_scorer(score1),
+        "score2": make_scorer(score2),
+    }
+
+    X, y = make_classification(
+        n_samples=50, n_features=2, n_redundant=0, random_state=0
+    )
+
+    clf = DecisionTreeClassifier().fit(X, y)
+
+    scorer_dict = _check_multimetric_scoring(clf, scorers)
+    multi_scorer = _MultimetricScorer(scorers=scorer_dict)
+    multi_scorer(clf, X, y, common_arg=1, sample_weight=1)
+
+
+def test_kwargs_without_metadata_routing_error():
+    # Test that kwargs are not supported in scorers if metadata routing is not
+    # enabled.
+    # TODO: remove when enable_metadata_routing is deprecated
+    def score(y_true, y_pred, param=None):
+        return 1  # pragma: no cover
+
+    X, y = make_classification(
+        n_samples=50, n_features=2, n_redundant=0, random_state=0
+    )
+
+    clf = DecisionTreeClassifier().fit(X, y)
+    scorer = make_scorer(score)
+    with config_context(enable_metadata_routing=False):
+        with pytest.raises(
+            ValueError, match="is only supported if enable_metadata_routing=True"
+        ):
+            scorer(clf, X, y, param="blah")
+
+
+def test_get_scorer_multilabel_indicator():
+    """Check that our scorer deal with multi-label indicator matrices.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/26817
+    """
+    X, Y = make_multilabel_classification(n_samples=72, n_classes=3, random_state=0)
+    X_train, X_test, Y_train, Y_test = train_test_split(X, Y, random_state=0)
+
+    estimator = KNeighborsClassifier().fit(X_train, Y_train)
+
+    score = get_scorer("average_precision")(estimator, X_test, Y_test)
+    assert score > 0.8
+
+
+@pytest.mark.parametrize(
+    "scorer, expected_repr",
+    [
+        (
+            get_scorer("accuracy"),
+            "make_scorer(accuracy_score, response_method='predict')",
+        ),
+        (
+            get_scorer("neg_log_loss"),
+            (
+                "make_scorer(log_loss, greater_is_better=False,"
+                " response_method='predict_proba')"
+            ),
+        ),
+        (
+            get_scorer("roc_auc"),
+            (
+                "make_scorer(roc_auc_score, response_method="
+                "('decision_function', 'predict_proba'))"
+            ),
+        ),
+        (
+            make_scorer(fbeta_score, beta=2),
+            "make_scorer(fbeta_score, response_method='predict', beta=2)",
+        ),
+    ],
+)
+def test_make_scorer_repr(scorer, expected_repr):
+    """Check the representation of the scorer."""
+    assert repr(scorer) == expected_repr
+
+
+@pytest.mark.parametrize("pass_estimator", [True, False])
+def test_get_scorer_multimetric(pass_estimator):
+    """Check that check_scoring is compatible with multi-metric configurations."""
+    X, y = make_classification(n_samples=150, n_features=10, random_state=0)
+    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
+    clf = LogisticRegression(random_state=0)
+
+    if pass_estimator:
+        check_scoring_ = check_scoring
+    else:
+        check_scoring_ = partial(check_scoring, clf)
+
+    clf.fit(X_train, y_train)
+
+    y_pred = clf.predict(X_test)
+    y_proba = clf.predict_proba(X_test)
+
+    expected_results = {
+        "r2": r2_score(y_test, y_pred),
+        "roc_auc": roc_auc_score(y_test, y_proba[:, 1]),
+        "accuracy": accuracy_score(y_test, y_pred),
+    }
+
+    for container in [set, list, tuple]:
+        scoring = check_scoring_(scoring=container(["r2", "roc_auc", "accuracy"]))
+        result = scoring(clf, X_test, y_test)
+
+        assert result.keys() == expected_results.keys()
+        for name in result:
+            assert result[name] == pytest.approx(expected_results[name])
+
+    def double_accuracy(y_true, y_pred):
+        return 2 * accuracy_score(y_true, y_pred)
+
+    custom_scorer = make_scorer(double_accuracy, response_method="predict")
+
+    # dict with different names
+    dict_scoring = check_scoring_(
+        scoring={
+            "my_r2": "r2",
+            "my_roc_auc": "roc_auc",
+            "double_accuracy": custom_scorer,
+        }
+    )
+    dict_result = dict_scoring(clf, X_test, y_test)
+    assert len(dict_result) == 3
+    assert dict_result["my_r2"] == pytest.approx(expected_results["r2"])
+    assert dict_result["my_roc_auc"] == pytest.approx(expected_results["roc_auc"])
+    assert dict_result["double_accuracy"] == pytest.approx(
+        2 * expected_results["accuracy"]
+    )
+
+
+def test_multimetric_scorer_repr():
+    """Check repr for multimetric scorer"""
+    multi_metric_scorer = check_scoring(scoring=["accuracy", "r2"])
+
+    assert str(multi_metric_scorer) == 'MultiMetricScorer("accuracy", "r2")'
+
+
+def test_check_scoring_multimetric_raise_exc():
+    """Test that check_scoring returns error code for a subset of scorers in
+    multimetric scoring if raise_exc=False and raises otherwise."""
+
+    def raising_scorer(estimator, X, y):
+        raise ValueError("That doesn't work.")
+
+    X, y = make_classification(n_samples=150, n_features=10, random_state=0)
+    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
+    clf = LogisticRegression().fit(X_train, y_train)
+
+    # "raising_scorer" is raising ValueError and should return an string representation
+    # of the error of the last scorer:
+    scoring = {
+        "accuracy": make_scorer(accuracy_score),
+        "raising_scorer": raising_scorer,
+    }
+    scoring_call = check_scoring(estimator=clf, scoring=scoring, raise_exc=False)
+    scores = scoring_call(clf, X_test, y_test)
+    assert "That doesn't work." in scores["raising_scorer"]
+
+    # should raise an error
+    scoring_call = check_scoring(estimator=clf, scoring=scoring, raise_exc=True)
+    err_msg = "That doesn't work."
+    with pytest.raises(ValueError, match=err_msg):
+        scores = scoring_call(clf, X_test, y_test)
+
+
+@pytest.mark.parametrize("enable_metadata_routing", [True, False])
+def test_metadata_routing_multimetric_metadata_routing(enable_metadata_routing):
+    """Test multimetric scorer works with and without metadata routing enabled when
+    there is no actual metadata to pass.
+
+    Non-regression test for https://github.com/scikit-learn/scikit-learn/issues/28256
+    """
+    X, y = make_classification(n_samples=50, n_features=10, random_state=0)
+    estimator = EstimatorWithFitAndPredict().fit(X, y)
+
+    multimetric_scorer = _MultimetricScorer(scorers={"acc": get_scorer("accuracy")})
+    with config_context(enable_metadata_routing=enable_metadata_routing):
+        multimetric_scorer(estimator, X, y)
+
+
+def test_curve_scorer():
+    """Check the behaviour of the `_CurveScorer` class."""
+    X, y = make_classification(random_state=0)
+    estimator = LogisticRegression().fit(X, y)
+    curve_scorer = _CurveScorer(
+        balanced_accuracy_score,
+        sign=1,
+        response_method="predict_proba",
+        thresholds=10,
+        kwargs={},
+    )
+    scores, thresholds = curve_scorer(estimator, X, y)
+
+    assert thresholds.shape == scores.shape
+    # check that the thresholds are probabilities with extreme values close to 0 and 1.
+    # they are not exactly 0 and 1 because they are the extremum of the
+    # `estimator.predict_proba(X)` values.
+    assert 0 <= thresholds.min() <= 0.01
+    assert 0.99 <= thresholds.max() <= 1
+    # balanced accuracy should be between 0.5 and 1 when it is not adjusted
+    assert 0.5 <= scores.min() <= 1
+
+    # check that passing kwargs to the scorer works
+    curve_scorer = _CurveScorer(
+        balanced_accuracy_score,
+        sign=1,
+        response_method="predict_proba",
+        thresholds=10,
+        kwargs={"adjusted": True},
+    )
+    scores, thresholds = curve_scorer(estimator, X, y)
+
+    # balanced accuracy should be between 0.5 and 1 when it is not adjusted
+    assert 0 <= scores.min() <= 0.5
+
+    # check that we can inverse the sign of the score when dealing with `neg_*` scorer
+    curve_scorer = _CurveScorer(
+        balanced_accuracy_score,
+        sign=-1,
+        response_method="predict_proba",
+        thresholds=10,
+        kwargs={"adjusted": True},
+    )
+    scores, thresholds = curve_scorer(estimator, X, y)
+
+    assert all(scores <= 0)
+
+
+def test_curve_scorer_pos_label(global_random_seed):
+    """Check that we propagate properly the `pos_label` parameter to the scorer."""
+    n_samples = 30
+    X, y = make_classification(
+        n_samples=n_samples, weights=[0.9, 0.1], random_state=global_random_seed
+    )
+    estimator = LogisticRegression().fit(X, y)
+
+    curve_scorer = _CurveScorer(
+        recall_score,
+        sign=1,
+        response_method="predict_proba",
+        thresholds=10,
+        kwargs={"pos_label": 1},
+    )
+    scores_pos_label_1, thresholds_pos_label_1 = curve_scorer(estimator, X, y)
+
+    curve_scorer = _CurveScorer(
+        recall_score,
+        sign=1,
+        response_method="predict_proba",
+        thresholds=10,
+        kwargs={"pos_label": 0},
+    )
+    scores_pos_label_0, thresholds_pos_label_0 = curve_scorer(estimator, X, y)
+
+    # Since `pos_label` is forwarded to the curve_scorer, the thresholds are not equal.
+    assert not (thresholds_pos_label_1 == thresholds_pos_label_0).all()
+    # The min-max range for the thresholds is defined by the probabilities of the
+    # `pos_label` class (the column of `predict_proba`).
+    y_pred = estimator.predict_proba(X)
+    assert thresholds_pos_label_0.min() == pytest.approx(y_pred.min(axis=0)[0])
+    assert thresholds_pos_label_0.max() == pytest.approx(y_pred.max(axis=0)[0])
+    assert thresholds_pos_label_1.min() == pytest.approx(y_pred.min(axis=0)[1])
+    assert thresholds_pos_label_1.max() == pytest.approx(y_pred.max(axis=0)[1])
+
+    # The recall cannot be negative and `pos_label=1` should have a higher recall
+    # since there is less samples to be considered.
+    assert 0.0 < scores_pos_label_0.min() < scores_pos_label_1.min()
+    assert scores_pos_label_0.max() == pytest.approx(1.0)
+    assert scores_pos_label_1.max() == pytest.approx(1.0)
+
+
+# TODO(1.8): remove
+def test_make_scorer_reponse_method_default_warning():
+    with pytest.warns(FutureWarning, match="response_method=None is deprecated"):
+        make_scorer(accuracy_score, response_method=None)
+
+    # No warning is raised if response_method is left to its default value
+    # because the future default value has the same effect as the current one.
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", FutureWarning)
+        make_scorer(accuracy_score)
diff --git a/sklearn/mixture/__init__.py b/sklearn/mixture/__init__.py
index 9c5a89dceaa5e..c27263a0ed743 100644
--- a/sklearn/mixture/__init__.py
+++ b/sklearn/mixture/__init__.py
@@ -1,10 +1,9 @@
-"""
-The :mod:`sklearn.mixture` module implements mixture modeling algorithms.
-"""
+"""Mixture modeling algorithms."""
 
-from ._gaussian_mixture import GaussianMixture
-from ._bayesian_mixture import BayesianGaussianMixture
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
+from ._bayesian_mixture import BayesianGaussianMixture
+from ._gaussian_mixture import GaussianMixture
 
-__all__ = ['GaussianMixture',
-           'BayesianGaussianMixture']
+__all__ = ["BayesianGaussianMixture", "GaussianMixture"]
diff --git a/sklearn/mixture/_base.py b/sklearn/mixture/_base.py
index a9aac7f4dac19..f66344a284753 100644
--- a/sklearn/mixture/_base.py
+++ b/sklearn/mixture/_base.py
@@ -1,22 +1,23 @@
 """Base class for mixture models."""
 
-# Author: Wei Xue 
-# Modified by Thierry Guillemot 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import warnings
 from abc import ABCMeta, abstractmethod
+from numbers import Integral, Real
 from time import time
 
 import numpy as np
 from scipy.special import logsumexp
 
 from .. import cluster
-from ..base import BaseEstimator
-from ..base import DensityMixin
+from ..base import BaseEstimator, DensityMixin, _fit_context
+from ..cluster import kmeans_plusplus
 from ..exceptions import ConvergenceWarning
-from ..utils import check_array, check_random_state
-from ..utils.validation import check_is_fitted
+from ..utils import check_random_state
+from ..utils._param_validation import Interval, StrOptions
+from ..utils.validation import check_is_fitted, validate_data
 
 
 def _check_shape(param, param_shape, name):
@@ -28,38 +29,14 @@ def _check_shape(param, param_shape, name):
 
     param_shape : tuple
 
-    name : string
+    name : str
     """
     param = np.array(param)
     if param.shape != param_shape:
-        raise ValueError("The parameter '%s' should have the shape of %s, "
-                         "but got %s" % (name, param_shape, param.shape))
-
-
-def _check_X(X, n_components=None, n_features=None, ensure_min_samples=1):
-    """Check the input data X.
-
-    Parameters
-    ----------
-    X : array-like of shape (n_samples, n_features)
-
-    n_components : int
-
-    Returns
-    -------
-    X : array, shape (n_samples, n_features)
-    """
-    X = check_array(X, dtype=[np.float64, np.float32],
-                    ensure_min_samples=ensure_min_samples)
-    if n_components is not None and X.shape[0] < n_components:
-        raise ValueError('Expected n_samples >= n_components '
-                         'but got n_components = %d, n_samples = %d'
-                         % (n_components, X.shape[0]))
-    if n_features is not None and X.shape[1] != n_features:
-        raise ValueError("Expected the input data X have %d features, "
-                         "but got %d features"
-                         % (n_features, X.shape[1]))
-    return X
+        raise ValueError(
+            "The parameter '%s' should have the shape of %s, but got %s"
+            % (name, param_shape, param.shape)
+        )
 
 
 class BaseMixture(DensityMixin, BaseEstimator, metaclass=ABCMeta):
@@ -69,9 +46,34 @@ class BaseMixture(DensityMixin, BaseEstimator, metaclass=ABCMeta):
     provides basic common methods for mixture models.
     """
 
-    def __init__(self, n_components, tol, reg_covar,
-                 max_iter, n_init, init_params, random_state, warm_start,
-                 verbose, verbose_interval):
+    _parameter_constraints: dict = {
+        "n_components": [Interval(Integral, 1, None, closed="left")],
+        "tol": [Interval(Real, 0.0, None, closed="left")],
+        "reg_covar": [Interval(Real, 0.0, None, closed="left")],
+        "max_iter": [Interval(Integral, 0, None, closed="left")],
+        "n_init": [Interval(Integral, 1, None, closed="left")],
+        "init_params": [
+            StrOptions({"kmeans", "random", "random_from_data", "k-means++"})
+        ],
+        "random_state": ["random_state"],
+        "warm_start": ["boolean"],
+        "verbose": ["verbose"],
+        "verbose_interval": [Interval(Integral, 1, None, closed="left")],
+    }
+
+    def __init__(
+        self,
+        n_components,
+        tol,
+        reg_covar,
+        max_iter,
+        n_init,
+        init_params,
+        random_state,
+        warm_start,
+        verbose,
+        verbose_interval,
+    ):
         self.n_components = n_components
         self.tol = tol
         self.reg_covar = reg_covar
@@ -83,42 +85,6 @@ def __init__(self, n_components, tol, reg_covar,
         self.verbose = verbose
         self.verbose_interval = verbose_interval
 
-    def _check_initial_parameters(self, X):
-        """Check values of the basic parameters.
-
-        Parameters
-        ----------
-        X : array-like of shape (n_samples, n_features)
-        """
-        if self.n_components < 1:
-            raise ValueError("Invalid value for 'n_components': %d "
-                             "Estimation requires at least one component"
-                             % self.n_components)
-
-        if self.tol < 0.:
-            raise ValueError("Invalid value for 'tol': %.5f "
-                             "Tolerance used by the EM must be non-negative"
-                             % self.tol)
-
-        if self.n_init < 1:
-            raise ValueError("Invalid value for 'n_init': %d "
-                             "Estimation requires at least one run"
-                             % self.n_init)
-
-        if self.max_iter < 1:
-            raise ValueError("Invalid value for 'max_iter': %d "
-                             "Estimation requires at least one iteration"
-                             % self.max_iter)
-
-        if self.reg_covar < 0.:
-            raise ValueError("Invalid value for 'reg_covar': %.5f "
-                             "regularization on covariance must be "
-                             "non-negative"
-                             % self.reg_covar)
-
-        # Check all the parameters values of the derived class
-        self._check_parameters(X)
-
     @abstractmethod
     def _check_parameters(self, X):
         """Check initial parameters of the derived class.
@@ -142,17 +108,35 @@ def _initialize_parameters(self, X, random_state):
         """
         n_samples, _ = X.shape
 
-        if self.init_params == 'kmeans':
-            resp = np.zeros((n_samples, self.n_components))
-            label = cluster.KMeans(n_clusters=self.n_components, n_init=1,
-                                   random_state=random_state).fit(X).labels_
+        if self.init_params == "kmeans":
+            resp = np.zeros((n_samples, self.n_components), dtype=X.dtype)
+            label = (
+                cluster.KMeans(
+                    n_clusters=self.n_components, n_init=1, random_state=random_state
+                )
+                .fit(X)
+                .labels_
+            )
             resp[np.arange(n_samples), label] = 1
-        elif self.init_params == 'random':
-            resp = random_state.rand(n_samples, self.n_components)
+        elif self.init_params == "random":
+            resp = np.asarray(
+                random_state.uniform(size=(n_samples, self.n_components)), dtype=X.dtype
+            )
             resp /= resp.sum(axis=1)[:, np.newaxis]
-        else:
-            raise ValueError("Unimplemented initialization method '%s'"
-                             % self.init_params)
+        elif self.init_params == "random_from_data":
+            resp = np.zeros((n_samples, self.n_components), dtype=X.dtype)
+            indices = random_state.choice(
+                n_samples, size=self.n_components, replace=False
+            )
+            resp[indices, np.arange(self.n_components)] = 1
+        elif self.init_params == "k-means++":
+            resp = np.zeros((n_samples, self.n_components), dtype=X.dtype)
+            _, indices = kmeans_plusplus(
+                X,
+                self.n_components,
+                random_state=random_state,
+            )
+            resp[indices, np.arange(self.n_components)] = 1
 
         self._initialize(X, resp)
 
@@ -186,13 +170,19 @@ def fit(self, X, y=None):
             List of n_features-dimensional data points. Each row
             corresponds to a single data point.
 
+        y : Ignored
+            Not used, present for API consistency by convention.
+
         Returns
         -------
-        self
+        self : object
+            The fitted mixture.
         """
+        # parameters are validated in fit_predict
         self.fit_predict(X, y)
         return self
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit_predict(self, X, y=None):
         """Estimate model parameters using X and predict the labels for X.
 
@@ -212,20 +202,29 @@ def fit_predict(self, X, y=None):
             List of n_features-dimensional data points. Each row
             corresponds to a single data point.
 
+        y : Ignored
+            Not used, present for API consistency by convention.
+
         Returns
         -------
         labels : array, shape (n_samples,)
             Component labels.
         """
-        X = _check_X(X, self.n_components, ensure_min_samples=2)
-        self._check_n_features(X, reset=True)
-        self._check_initial_parameters(X)
+        X = validate_data(self, X, dtype=[np.float64, np.float32], ensure_min_samples=2)
+        if X.shape[0] < self.n_components:
+            raise ValueError(
+                "Expected n_samples >= n_components "
+                f"but got n_components = {self.n_components}, "
+                f"n_samples = {X.shape[0]}"
+            )
+        self._check_parameters(X)
 
         # if we enable warm_start, we will have a unique initialisation
-        do_init = not(self.warm_start and hasattr(self, 'converged_'))
+        do_init = not (self.warm_start and hasattr(self, "converged_"))
         n_init = self.n_init if do_init else 1
 
-        max_lower_bound = -np.infty
+        max_lower_bound = -np.inf
+        best_lower_bounds = []
         self.converged_ = False
 
         random_state = check_random_state(self.random_state)
@@ -237,40 +236,55 @@ def fit_predict(self, X, y=None):
             if do_init:
                 self._initialize_parameters(X, random_state)
 
-            lower_bound = (-np.infty if do_init else self.lower_bound_)
-
-            for n_iter in range(1, self.max_iter + 1):
-                prev_lower_bound = lower_bound
-
-                log_prob_norm, log_resp = self._e_step(X)
-                self._m_step(X, log_resp)
-                lower_bound = self._compute_lower_bound(
-                    log_resp, log_prob_norm)
-
-                change = lower_bound - prev_lower_bound
-                self._print_verbose_msg_iter_end(n_iter, change)
+            lower_bound = -np.inf if do_init else self.lower_bound_
+            current_lower_bounds = []
 
-                if abs(change) < self.tol:
-                    self.converged_ = True
-                    break
-
-            self._print_verbose_msg_init_end(lower_bound)
-
-            if lower_bound > max_lower_bound:
-                max_lower_bound = lower_bound
+            if self.max_iter == 0:
                 best_params = self._get_parameters()
-                best_n_iter = n_iter
-
-        if not self.converged_:
-            warnings.warn('Initialization %d did not converge. '
-                          'Try different init parameters, '
-                          'or increase max_iter, tol '
-                          'or check for degenerate data.'
-                          % (init + 1), ConvergenceWarning)
+                best_n_iter = 0
+            else:
+                converged = False
+                for n_iter in range(1, self.max_iter + 1):
+                    prev_lower_bound = lower_bound
+
+                    log_prob_norm, log_resp = self._e_step(X)
+                    self._m_step(X, log_resp)
+                    lower_bound = self._compute_lower_bound(log_resp, log_prob_norm)
+                    current_lower_bounds.append(lower_bound)
+
+                    change = lower_bound - prev_lower_bound
+                    self._print_verbose_msg_iter_end(n_iter, change)
+
+                    if abs(change) < self.tol:
+                        converged = True
+                        break
+
+                self._print_verbose_msg_init_end(lower_bound, converged)
+
+                if lower_bound > max_lower_bound or max_lower_bound == -np.inf:
+                    max_lower_bound = lower_bound
+                    best_params = self._get_parameters()
+                    best_n_iter = n_iter
+                    best_lower_bounds = current_lower_bounds
+                    self.converged_ = converged
+
+        # Should only warn about convergence if max_iter > 0, otherwise
+        # the user is assumed to have used 0-iters initialization
+        # to get the initial means.
+        if not self.converged_ and self.max_iter > 0:
+            warnings.warn(
+                (
+                    "Best performing initialization did not converge. "
+                    "Try different init parameters, or increase max_iter, "
+                    "tol, or check for degenerate data."
+                ),
+                ConvergenceWarning,
+            )
 
         self._set_parameters(best_params)
         self.n_iter_ = best_n_iter
         self.lower_bound_ = max_lower_bound
+        self.lower_bounds_ = best_lower_bounds
 
         # Always do a final e-step to guarantee that the labels returned by
         # fit_predict(X) are always consistent with fit(X).predict(X)
@@ -321,7 +335,7 @@ def _set_parameters(self, params):
         pass
 
     def score_samples(self, X):
-        """Compute the weighted log probabilities for each sample.
+        """Compute the log-likelihood of each sample.
 
         Parameters
         ----------
@@ -332,10 +346,10 @@ def score_samples(self, X):
         Returns
         -------
         log_prob : array, shape (n_samples,)
-            Log probabilities of each data point in X.
+            Log-likelihood of each sample in `X` under the current model.
         """
         check_is_fitted(self)
-        X = _check_X(X, None, self.means_.shape[1])
+        X = validate_data(self, X, reset=False)
 
         return logsumexp(self._estimate_weighted_log_prob(X), axis=1)
 
@@ -348,10 +362,13 @@ def score(self, X, y=None):
             List of n_features-dimensional data points. Each row
             corresponds to a single data point.
 
+        y : Ignored
+            Not used, present for API consistency by convention.
+
         Returns
         -------
         log_likelihood : float
-            Log likelihood of the Gaussian mixture given X.
+            Log-likelihood of `X` under the Gaussian mixture model.
         """
         return self.score_samples(X).mean()
 
@@ -370,11 +387,11 @@ def predict(self, X):
             Component labels.
         """
         check_is_fitted(self)
-        X = _check_X(X, None, self.means_.shape[1])
+        X = validate_data(self, X, reset=False)
         return self._estimate_weighted_log_prob(X).argmax(axis=1)
 
     def predict_proba(self, X):
-        """Predict posterior probability of each component given the data.
+        """Evaluate the components' density for each sample.
 
         Parameters
         ----------
@@ -385,11 +402,10 @@ def predict_proba(self, X):
         Returns
         -------
         resp : array, shape (n_samples, n_components)
-            Returns the probability each Gaussian (state) in
-            the model given each sample.
+            Density of each Gaussian component for each sample in X.
         """
         check_is_fitted(self)
-        X = _check_X(X, None, self.means_.shape[1])
+        X = validate_data(self, X, reset=False)
         _, log_resp = self._estimate_log_prob_resp(X)
         return np.exp(log_resp)
 
@@ -404,41 +420,54 @@ def sample(self, n_samples=1):
         Returns
         -------
         X : array, shape (n_samples, n_features)
-            Randomly generated sample
+            Randomly generated sample.
 
         y : array, shape (nsamples,)
-            Component labels
-
+            Component labels.
         """
         check_is_fitted(self)
 
         if n_samples < 1:
             raise ValueError(
                 "Invalid value for 'n_samples': %d . The sampling requires at "
-                "least one sample." % (self.n_components))
+                "least one sample." % (self.n_components)
+            )
 
         _, n_features = self.means_.shape
         rng = check_random_state(self.random_state)
         n_samples_comp = rng.multinomial(n_samples, self.weights_)
 
-        if self.covariance_type == 'full':
-            X = np.vstack([
-                rng.multivariate_normal(mean, covariance, int(sample))
-                for (mean, covariance, sample) in zip(
-                    self.means_, self.covariances_, n_samples_comp)])
+        if self.covariance_type == "full":
+            X = np.vstack(
+                [
+                    rng.multivariate_normal(mean, covariance, int(sample))
+                    for (mean, covariance, sample) in zip(
+                        self.means_, self.covariances_, n_samples_comp
+                    )
+                ]
+            )
         elif self.covariance_type == "tied":
-            X = np.vstack([
-                rng.multivariate_normal(mean, self.covariances_, int(sample))
-                for (mean, sample) in zip(
-                    self.means_, n_samples_comp)])
+            X = np.vstack(
+                [
+                    rng.multivariate_normal(mean, self.covariances_, int(sample))
+                    for (mean, sample) in zip(self.means_, n_samples_comp)
+                ]
+            )
         else:
-            X = np.vstack([
-                mean + rng.randn(sample, n_features) * np.sqrt(covariance)
-                for (mean, covariance, sample) in zip(
-                    self.means_, self.covariances_, n_samples_comp)])
-
-        y = np.concatenate([np.full(sample, j, dtype=int)
-                           for j, sample in enumerate(n_samples_comp)])
+            X = np.vstack(
+                [
+                    mean
+                    + rng.standard_normal(size=(sample, n_features))
+                    * np.sqrt(covariance)
+                    for (mean, covariance, sample) in zip(
+                        self.means_, self.covariances_, n_samples_comp
+                    )
+                ]
+            )
+
+        y = np.concatenate(
+            [np.full(sample, j, dtype=int) for j, sample in enumerate(n_samples_comp)]
+        )
 
         return (X, y)
 
@@ -502,7 +531,7 @@ def _estimate_log_prob_resp(self, X):
         """
         weighted_log_prob = self._estimate_weighted_log_prob(X)
         log_prob_norm = logsumexp(weighted_log_prob, axis=1)
-        with np.errstate(under='ignore'):
+        with np.errstate(under="ignore"):
             # ignore underflow
             log_resp = weighted_log_prob - log_prob_norm[:, np.newaxis]
         return log_prob_norm, log_resp
@@ -523,14 +552,20 @@ def _print_verbose_msg_iter_end(self, n_iter, diff_ll):
                 print("  Iteration %d" % n_iter)
             elif self.verbose >= 2:
                 cur_time = time()
-                print("  Iteration %d\t time lapse %.5fs\t ll change %.5f" % (
-                    n_iter, cur_time - self._iter_prev_time, diff_ll))
+                print(
+                    "  Iteration %d\t time lapse %.5fs\t ll change %.5f"
+                    % (n_iter, cur_time - self._iter_prev_time, diff_ll)
+                )
                 self._iter_prev_time = cur_time
 
-    def _print_verbose_msg_init_end(self, ll):
+    def _print_verbose_msg_init_end(self, lb, init_has_converged):
         """Print verbose message on the end of iteration."""
+        converged_msg = "converged" if init_has_converged else "did not converge"
         if self.verbose == 1:
-            print("Initialization converged: %s" % self.converged_)
+            print(f"Initialization {converged_msg}.")
         elif self.verbose >= 2:
-            print("Initialization converged: %s\t time lapse %.5fs\t ll %.5f" %
-                  (self.converged_, time() - self._init_prev_time, ll))
+            t = time() - self._init_prev_time
+            print(
+                f"Initialization {converged_msg}. time lapse {t:.5f}s\t lower bound"
+                f" {lb:.5f}."
+            )
diff --git a/sklearn/mixture/_bayesian_mixture.py b/sklearn/mixture/_bayesian_mixture.py
index 34cef090be22b..57220186faf61 100644
--- a/sklearn/mixture/_bayesian_mixture.py
+++ b/sklearn/mixture/_bayesian_mixture.py
@@ -1,21 +1,25 @@
 """Bayesian Gaussian Mixture Model."""
-# Author: Wei Xue 
-#         Thierry Guillemot 
-# License: BSD 3 clause
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import math
+from numbers import Real
+
 import numpy as np
 from scipy.special import betaln, digamma, gammaln
 
-from ._base import BaseMixture, _check_shape
-from ._gaussian_mixture import _check_precision_matrix
-from ._gaussian_mixture import _check_precision_positivity
-from ._gaussian_mixture import _compute_log_det_cholesky
-from ._gaussian_mixture import _compute_precision_cholesky
-from ._gaussian_mixture import _estimate_gaussian_parameters
-from ._gaussian_mixture import _estimate_log_gaussian_prob
 from ..utils import check_array
-from ..utils.validation import _deprecate_positional_args
+from ..utils._param_validation import Interval, StrOptions
+from ._base import BaseMixture, _check_shape
+from ._gaussian_mixture import (
+    _check_precision_matrix,
+    _check_precision_positivity,
+    _compute_log_det_cholesky,
+    _compute_precision_cholesky,
+    _estimate_gaussian_parameters,
+    _estimate_log_gaussian_prob,
+)
 
 
 def _log_dirichlet_norm(dirichlet_concentration):
@@ -31,8 +35,9 @@ def _log_dirichlet_norm(dirichlet_concentration):
     log_dirichlet_norm : float
         The log normalization of the Dirichlet distribution.
     """
-    return (gammaln(np.sum(dirichlet_concentration)) -
-            np.sum(gammaln(dirichlet_concentration)))
+    return gammaln(np.sum(dirichlet_concentration)) - np.sum(
+        gammaln(dirichlet_concentration)
+    )
 
 
 def _log_wishart_norm(degrees_of_freedom, log_det_precisions_chol, n_features):
@@ -56,10 +61,14 @@ def _log_wishart_norm(degrees_of_freedom, log_det_precisions_chol, n_features):
         The log normalization of the Wishart distribution.
     """
     # To simplify the computation we have removed the np.log(np.pi) term
-    return -(degrees_of_freedom * log_det_precisions_chol +
-             degrees_of_freedom * n_features * .5 * math.log(2.) +
-             np.sum(gammaln(.5 * (degrees_of_freedom -
-                                  np.arange(n_features)[:, np.newaxis])), 0))
+    return -(
+        degrees_of_freedom * log_det_precisions_chol
+        + degrees_of_freedom * n_features * 0.5 * math.log(2.0)
+        + np.sum(
+            gammaln(0.5 * (degrees_of_freedom - np.arange(n_features)[:, np.newaxis])),
+            0,
+        )
+    )
 
 
 class BayesianGaussianMixture(BaseMixture):
@@ -91,12 +100,12 @@ class BayesianGaussianMixture(BaseMixture):
 
     covariance_type : {'full', 'tied', 'diag', 'spherical'}, default='full'
         String describing the type of covariance parameters to use.
-        Must be one of::
+        Must be one of:
 
-            'full' (each component has its own general covariance matrix),
-            'tied' (all components share the same general covariance matrix),
-            'diag' (each component has its own diagonal covariance matrix),
-            'spherical' (each component has its own single variance).
+        - 'full' (each component has its own general covariance matrix),
+        - 'tied' (all components share the same general covariance matrix),
+        - 'diag' (each component has its own diagonal covariance matrix),
+        - 'spherical' (each component has its own single variance).
 
     tol : float, default=1e-3
         The convergence threshold. EM iterations will stop when the
@@ -114,22 +123,25 @@ class BayesianGaussianMixture(BaseMixture):
         The number of initializations to perform. The result with the highest
         lower bound value on the likelihood is kept.
 
-    init_params : {'kmeans', 'random'}, default='kmeans'
+    init_params : {'kmeans', 'k-means++', 'random', 'random_from_data'}, \
+    default='kmeans'
         The method used to initialize the weights, the means and the
-        covariances.
-        Must be one of::
+        covariances. String must be one of:
 
-            'kmeans' : responsibilities are initialized using kmeans.
-            'random' : responsibilities are initialized randomly.
+        - 'kmeans': responsibilities are initialized using kmeans.
+        - 'k-means++': use the k-means++ method to initialize.
+        - 'random': responsibilities are initialized randomly.
+        - 'random_from_data': initial means are randomly selected data points.
 
-    weight_concentration_prior_type : str, default='dirichlet_process'
-        String describing the type of the weight concentration prior.
-        Must be one of::
+        .. versionchanged:: v1.1
+            `init_params` now accepts 'random_from_data' and 'k-means++' as
+            initialization methods.
 
-            'dirichlet_process' (using the Stick-breaking representation),
-            'dirichlet_distribution' (can favor more uniform weights).
+    weight_concentration_prior_type : {'dirichlet_process', 'dirichlet_distribution'}, \
+            default='dirichlet_process'
+        String describing the type of the weight concentration prior.
 
-    weight_concentration_prior : float | None, default=None.
+    weight_concentration_prior : float or None, default=None
         The dirichlet concentration of each component on the weight
         distribution (Dirichlet). This is commonly called gamma in the
         literature. The higher concentration puts more mass in
@@ -138,22 +150,22 @@ class BayesianGaussianMixture(BaseMixture):
         mixture weights simplex. The value of the parameter must be greater
         than 0. If it is None, it's set to ``1. / n_components``.
 
-    mean_precision_prior : float | None, default=None.
+    mean_precision_prior : float or None, default=None
         The precision prior on the mean distribution (Gaussian).
         Controls the extent of where means can be placed. Larger
         values concentrate the cluster means around `mean_prior`.
         The value of the parameter must be greater than 0.
         If it is None, it is set to 1.
 
-    mean_prior : array-like, shape (n_features,), default=None.
+    mean_prior : array-like, shape (n_features,), default=None
         The prior on the mean distribution (Gaussian).
         If it is None, it is set to the mean of X.
 
-    degrees_of_freedom_prior : float | None, default=None.
+    degrees_of_freedom_prior : float or None, default=None
         The prior of the number of degrees of freedom on the covariance
         distributions (Wishart). If it is None, it's set to `n_features`.
 
-    covariance_prior : float or array-like, default=None.
+    covariance_prior : float or array-like, default=None
         The prior on the covariance distribution (Wishart).
         If it is None, the emiprical covariance prior is initialized using the
         covariance of X. The shape depends on `covariance_type`::
@@ -232,15 +244,19 @@ class BayesianGaussianMixture(BaseMixture):
             (n_components, n_features, n_features) if 'full'
 
     converged_ : bool
-        True when convergence was reached in fit(), False otherwise.
+        True when convergence of the best fit of inference was reached, False otherwise.
 
     n_iter_ : int
         Number of step used by the best fit of inference to reach the
         convergence.
 
     lower_bound_ : float
-        Lower bound value on the likelihood (of the training data with
-        respect to the model) of the best fit of inference.
+        Lower bound value on the model evidence (of the training data) of the
+        best fit of inference.
+
+    lower_bounds_ : array-like of shape (`n_iter_`,)
+        The list of lower bound values on the model evidence from each iteration
+        of the best fit of inference.
 
     weight_concentration_prior_ : tuple or float
         The dirichlet concentration of each component on the weight
@@ -288,17 +304,16 @@ class BayesianGaussianMixture(BaseMixture):
             (n_features)             if 'diag',
             float                    if 'spherical'
 
-    Examples
-    --------
-    >>> import numpy as np
-    >>> from sklearn.mixture import BayesianGaussianMixture
-    >>> X = np.array([[1, 2], [1, 4], [1, 0], [4, 2], [12, 4], [10, 7]])
-    >>> bgm = BayesianGaussianMixture(n_components=2, random_state=42).fit(X)
-    >>> bgm.means_
-    array([[2.49... , 2.29...],
-           [8.45..., 4.52... ]])
-    >>> bgm.predict([[0, 0], [9, 3]])
-    array([0, 1])
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
 
     See Also
     --------
@@ -314,26 +329,78 @@ class BayesianGaussianMixture(BaseMixture):
     .. [2] `Hagai Attias. (2000). "A Variational Bayesian Framework for
        Graphical Models". In Advances in Neural Information Processing
        Systems 12.
-       `_
+       `_
 
     .. [3] `Blei, David M. and Michael I. Jordan. (2006). "Variational
        inference for Dirichlet process mixtures". Bayesian analysis 1.1
        `_
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from sklearn.mixture import BayesianGaussianMixture
+    >>> X = np.array([[1, 2], [1, 4], [1, 0], [4, 2], [12, 4], [10, 7]])
+    >>> bgm = BayesianGaussianMixture(n_components=2, random_state=42).fit(X)
+    >>> bgm.means_
+    array([[2.49 , 2.29],
+           [8.45, 4.52 ]])
+    >>> bgm.predict([[0, 0], [9, 3]])
+    array([0, 1])
     """
-    @_deprecate_positional_args
-    def __init__(self, *, n_components=1, covariance_type='full', tol=1e-3,
-                 reg_covar=1e-6, max_iter=100, n_init=1, init_params='kmeans',
-                 weight_concentration_prior_type='dirichlet_process',
-                 weight_concentration_prior=None,
-                 mean_precision_prior=None, mean_prior=None,
-                 degrees_of_freedom_prior=None, covariance_prior=None,
-                 random_state=None, warm_start=False, verbose=0,
-                 verbose_interval=10):
+
+    _parameter_constraints: dict = {
+        **BaseMixture._parameter_constraints,
+        "covariance_type": [StrOptions({"spherical", "tied", "diag", "full"})],
+        "weight_concentration_prior_type": [
+            StrOptions({"dirichlet_process", "dirichlet_distribution"})
+        ],
+        "weight_concentration_prior": [
+            None,
+            Interval(Real, 0.0, None, closed="neither"),
+        ],
+        "mean_precision_prior": [None, Interval(Real, 0.0, None, closed="neither")],
+        "mean_prior": [None, "array-like"],
+        "degrees_of_freedom_prior": [None, Interval(Real, 0.0, None, closed="neither")],
+        "covariance_prior": [
+            None,
+            "array-like",
+            Interval(Real, 0.0, None, closed="neither"),
+        ],
+    }
+
+    def __init__(
+        self,
+        *,
+        n_components=1,
+        covariance_type="full",
+        tol=1e-3,
+        reg_covar=1e-6,
+        max_iter=100,
+        n_init=1,
+        init_params="kmeans",
+        weight_concentration_prior_type="dirichlet_process",
+        weight_concentration_prior=None,
+        mean_precision_prior=None,
+        mean_prior=None,
+        degrees_of_freedom_prior=None,
+        covariance_prior=None,
+        random_state=None,
+        warm_start=False,
+        verbose=0,
+        verbose_interval=10,
+    ):
         super().__init__(
-            n_components=n_components, tol=tol, reg_covar=reg_covar,
-            max_iter=max_iter, n_init=n_init, init_params=init_params,
-            random_state=random_state, warm_start=warm_start,
-            verbose=verbose, verbose_interval=verbose_interval)
+            n_components=n_components,
+            tol=tol,
+            reg_covar=reg_covar,
+            max_iter=max_iter,
+            n_init=n_init,
+            init_params=init_params,
+            random_state=random_state,
+            warm_start=warm_start,
+            verbose=verbose,
+            verbose_interval=verbose_interval,
+        )
 
         self.covariance_type = covariance_type
         self.weight_concentration_prior_type = weight_concentration_prior_type
@@ -350,20 +417,6 @@ def _check_parameters(self, X):
         ----------
         X : array-like of shape (n_samples, n_features)
         """
-        if self.covariance_type not in ['spherical', 'tied', 'diag', 'full']:
-            raise ValueError("Invalid value for 'covariance_type': %s "
-                             "'covariance_type' should be in "
-                             "['spherical', 'tied', 'diag', 'full']"
-                             % self.covariance_type)
-
-        if (self.weight_concentration_prior_type not in
-                ['dirichlet_process', 'dirichlet_distribution']):
-            raise ValueError(
-                "Invalid value for 'weight_concentration_prior_type': %s "
-                "'weight_concentration_prior_type' should be in "
-                "['dirichlet_process', 'dirichlet_distribution']"
-                % self.weight_concentration_prior_type)
-
         self._check_weights_parameters()
         self._check_means_parameters(X)
         self._check_precision_parameters(X)
@@ -372,14 +425,9 @@ def _check_parameters(self, X):
     def _check_weights_parameters(self):
         """Check the parameter of the Dirichlet distribution."""
         if self.weight_concentration_prior is None:
-            self.weight_concentration_prior_ = 1. / self.n_components
-        elif self.weight_concentration_prior > 0.:
-            self.weight_concentration_prior_ = (
-                self.weight_concentration_prior)
+            self.weight_concentration_prior_ = 1.0 / self.n_components
         else:
-            raise ValueError("The parameter 'weight_concentration_prior' "
-                             "should be greater than 0., but got %.3f."
-                             % self.weight_concentration_prior)
+            self.weight_concentration_prior_ = self.weight_concentration_prior
 
     def _check_means_parameters(self, X):
         """Check the parameters of the Gaussian distribution.
@@ -391,21 +439,17 @@ def _check_means_parameters(self, X):
         _, n_features = X.shape
 
         if self.mean_precision_prior is None:
-            self.mean_precision_prior_ = 1.
-        elif self.mean_precision_prior > 0.:
-            self.mean_precision_prior_ = self.mean_precision_prior
+            self.mean_precision_prior_ = 1.0
         else:
-            raise ValueError("The parameter 'mean_precision_prior' should be "
-                             "greater than 0., but got %.3f."
-                             % self.mean_precision_prior)
+            self.mean_precision_prior_ = self.mean_precision_prior
 
         if self.mean_prior is None:
             self.mean_prior_ = X.mean(axis=0)
         else:
-            self.mean_prior_ = check_array(self.mean_prior,
-                                           dtype=[np.float64, np.float32],
-                                           ensure_2d=False)
-            _check_shape(self.mean_prior_, (n_features, ), 'means')
+            self.mean_prior_ = check_array(
+                self.mean_prior, dtype=[np.float64, np.float32], ensure_2d=False
+            )
+            _check_shape(self.mean_prior_, (n_features,), "means")
 
     def _check_precision_parameters(self, X):
         """Check the prior parameters of the precision distribution.
@@ -418,12 +462,14 @@ def _check_precision_parameters(self, X):
 
         if self.degrees_of_freedom_prior is None:
             self.degrees_of_freedom_prior_ = n_features
-        elif self.degrees_of_freedom_prior > n_features - 1.:
+        elif self.degrees_of_freedom_prior > n_features - 1.0:
             self.degrees_of_freedom_prior_ = self.degrees_of_freedom_prior
         else:
-            raise ValueError("The parameter 'degrees_of_freedom_prior' "
-                             "should be greater than %d, but got %.3f."
-                             % (n_features - 1, self.degrees_of_freedom_prior))
+            raise ValueError(
+                "The parameter 'degrees_of_freedom_prior' "
+                "should be greater than %d, but got %.3f."
+                % (n_features - 1, self.degrees_of_freedom_prior)
+            )
 
     def _checkcovariance_prior_parameter(self, X):
         """Check the `covariance_prior_`.
@@ -436,35 +482,35 @@ def _checkcovariance_prior_parameter(self, X):
 
         if self.covariance_prior is None:
             self.covariance_prior_ = {
-                'full': np.atleast_2d(np.cov(X.T)),
-                'tied': np.atleast_2d(np.cov(X.T)),
-                'diag': np.var(X, axis=0, ddof=1),
-                'spherical': np.var(X, axis=0, ddof=1).mean()
+                "full": np.atleast_2d(np.cov(X.T)),
+                "tied": np.atleast_2d(np.cov(X.T)),
+                "diag": np.var(X, axis=0, ddof=1),
+                "spherical": np.var(X, axis=0, ddof=1).mean(),
             }[self.covariance_type]
 
-        elif self.covariance_type in ['full', 'tied']:
+        elif self.covariance_type in ["full", "tied"]:
             self.covariance_prior_ = check_array(
-                self.covariance_prior, dtype=[np.float64, np.float32],
-                ensure_2d=False)
-            _check_shape(self.covariance_prior_, (n_features, n_features),
-                         '%s covariance_prior' % self.covariance_type)
-            _check_precision_matrix(self.covariance_prior_,
-                                    self.covariance_type)
-        elif self.covariance_type == 'diag':
+                self.covariance_prior, dtype=[np.float64, np.float32], ensure_2d=False
+            )
+            _check_shape(
+                self.covariance_prior_,
+                (n_features, n_features),
+                "%s covariance_prior" % self.covariance_type,
+            )
+            _check_precision_matrix(self.covariance_prior_, self.covariance_type)
+        elif self.covariance_type == "diag":
             self.covariance_prior_ = check_array(
-                self.covariance_prior, dtype=[np.float64, np.float32],
-                ensure_2d=False)
-            _check_shape(self.covariance_prior_, (n_features,),
-                         '%s covariance_prior' % self.covariance_type)
-            _check_precision_positivity(self.covariance_prior_,
-                                        self.covariance_type)
+                self.covariance_prior, dtype=[np.float64, np.float32], ensure_2d=False
+            )
+            _check_shape(
+                self.covariance_prior_,
+                (n_features,),
+                "%s covariance_prior" % self.covariance_type,
+            )
+            _check_precision_positivity(self.covariance_prior_, self.covariance_type)
         # spherical case
-        elif self.covariance_prior > 0.:
-            self.covariance_prior_ = self.covariance_prior
         else:
-            raise ValueError("The parameter 'spherical covariance_prior' "
-                             "should be greater than 0., but got %.3f."
-                             % self.covariance_prior)
+            self.covariance_prior_ = self.covariance_prior
 
     def _initialize(self, X, resp):
         """Initialization of the mixture parameters.
@@ -475,8 +521,9 @@ def _initialize(self, X, resp):
 
         resp : array-like of shape (n_samples, n_components)
         """
-        nk, xk, sk = _estimate_gaussian_parameters(X, resp, self.reg_covar,
-                                                   self.covariance_type)
+        nk, xk, sk = _estimate_gaussian_parameters(
+            X, resp, self.reg_covar, self.covariance_type
+        )
 
         self._estimate_weights(nk)
         self._estimate_means(nk, xk)
@@ -489,15 +536,18 @@ def _estimate_weights(self, nk):
         ----------
         nk : array-like of shape (n_components,)
         """
-        if self.weight_concentration_prior_type == 'dirichlet_process':
+        if self.weight_concentration_prior_type == "dirichlet_process":
             # For dirichlet process weight_concentration will be a tuple
             # containing the two parameters of the beta distribution
             self.weight_concentration_ = (
-                1. + nk,
-                (self.weight_concentration_prior_ +
-                 np.hstack((np.cumsum(nk[::-1])[-2::-1], 0))))
+                1.0 + nk,
+                (
+                    self.weight_concentration_prior_
+                    + np.hstack((np.cumsum(nk[::-1])[-2::-1], 0))
+                ),
+            )
         else:
-            # case Variationnal Gaussian mixture with dirichlet distribution
+            # case Variational Gaussian mixture with dirichlet distribution
             self.weight_concentration_ = self.weight_concentration_prior_ + nk
 
     def _estimate_means(self, nk, xk):
@@ -510,9 +560,9 @@ def _estimate_means(self, nk, xk):
         xk : array-like of shape (n_components, n_features)
         """
         self.mean_precision_ = self.mean_precision_prior_ + nk
-        self.means_ = ((self.mean_precision_prior_ * self.mean_prior_ +
-                        nk[:, np.newaxis] * xk) /
-                       self.mean_precision_[:, np.newaxis])
+        self.means_ = (
+            self.mean_precision_prior_ * self.mean_prior_ + nk[:, np.newaxis] * xk
+        ) / self.mean_precision_[:, np.newaxis]
 
     def _estimate_precisions(self, nk, xk, sk):
         """Estimate the precisions parameters of the precision distribution.
@@ -530,14 +580,16 @@ def _estimate_precisions(self, nk, xk, sk):
             'diag' : (n_components, n_features)
             'spherical' : (n_components,)
         """
-        {"full": self._estimate_wishart_full,
-         "tied": self._estimate_wishart_tied,
-         "diag": self._estimate_wishart_diag,
-         "spherical": self._estimate_wishart_spherical
-         }[self.covariance_type](nk, xk, sk)
+        {
+            "full": self._estimate_wishart_full,
+            "tied": self._estimate_wishart_tied,
+            "diag": self._estimate_wishart_diag,
+            "spherical": self._estimate_wishart_spherical,
+        }[self.covariance_type](nk, xk, sk)
 
         self.precisions_cholesky_ = _compute_precision_cholesky(
-            self.covariances_, self.covariance_type)
+            self.covariances_, self.covariance_type
+        )
 
     def _estimate_wishart_full(self, nk, xk, sk):
         """Estimate the full Wishart distribution parameters.
@@ -559,19 +611,21 @@ def _estimate_wishart_full(self, nk, xk, sk):
         # the correct formula
         self.degrees_of_freedom_ = self.degrees_of_freedom_prior_ + nk
 
-        self.covariances_ = np.empty((self.n_components, n_features,
-                                      n_features))
+        self.covariances_ = np.empty((self.n_components, n_features, n_features))
 
         for k in range(self.n_components):
             diff = xk[k] - self.mean_prior_
-            self.covariances_[k] = (self.covariance_prior_ + nk[k] * sk[k] +
-                                    nk[k] * self.mean_precision_prior_ /
-                                    self.mean_precision_[k] * np.outer(diff,
-                                                                       diff))
+            self.covariances_[k] = (
+                self.covariance_prior_
+                + nk[k] * sk[k]
+                + nk[k]
+                * self.mean_precision_prior_
+                / self.mean_precision_[k]
+                * np.outer(diff, diff)
+            )
 
         # Contrary to the original bishop book, we normalize the covariances
-        self.covariances_ /= (
-            self.degrees_of_freedom_[:, np.newaxis, np.newaxis])
+        self.covariances_ /= self.degrees_of_freedom_[:, np.newaxis, np.newaxis]
 
     def _estimate_wishart_tied(self, nk, xk, sk):
         """Estimate the tied Wishart distribution parameters.
@@ -592,13 +646,17 @@ def _estimate_wishart_tied(self, nk, xk, sk):
         # `degrees_of_freedom_k = degrees_of_freedom_0 + Nk`
         # is the correct formula
         self.degrees_of_freedom_ = (
-            self.degrees_of_freedom_prior_ + nk.sum() / self.n_components)
+            self.degrees_of_freedom_prior_ + nk.sum() / self.n_components
+        )
 
         diff = xk - self.mean_prior_
         self.covariances_ = (
-            self.covariance_prior_ + sk * nk.sum() / self.n_components +
-            self.mean_precision_prior_ / self.n_components * np.dot(
-                (nk / self.mean_precision_) * diff.T, diff))
+            self.covariance_prior_
+            + sk * nk.sum() / self.n_components
+            + self.mean_precision_prior_
+            / self.n_components
+            * np.dot((nk / self.mean_precision_) * diff.T, diff)
+        )
 
         # Contrary to the original bishop book, we normalize the covariances
         self.covariances_ /= self.degrees_of_freedom_
@@ -624,10 +682,11 @@ def _estimate_wishart_diag(self, nk, xk, sk):
         self.degrees_of_freedom_ = self.degrees_of_freedom_prior_ + nk
 
         diff = xk - self.mean_prior_
-        self.covariances_ = (
-            self.covariance_prior_ + nk[:, np.newaxis] * (
-                sk + (self.mean_precision_prior_ /
-                      self.mean_precision_)[:, np.newaxis] * np.square(diff)))
+        self.covariances_ = self.covariance_prior_ + nk[:, np.newaxis] * (
+            sk
+            + (self.mean_precision_prior_ / self.mean_precision_)[:, np.newaxis]
+            * np.square(diff)
+        )
 
         # Contrary to the original bishop book, we normalize the covariances
         self.covariances_ /= self.degrees_of_freedom_[:, np.newaxis]
@@ -653,10 +712,12 @@ def _estimate_wishart_spherical(self, nk, xk, sk):
         self.degrees_of_freedom_ = self.degrees_of_freedom_prior_ + nk
 
         diff = xk - self.mean_prior_
-        self.covariances_ = (
-            self.covariance_prior_ + nk * (
-                sk + self.mean_precision_prior_ / self.mean_precision_ *
-                np.mean(np.square(diff), 1)))
+        self.covariances_ = self.covariance_prior_ + nk * (
+            sk
+            + self.mean_precision_prior_
+            / self.mean_precision_
+            * np.mean(np.square(diff), 1)
+        )
 
         # Contrary to the original bishop book, we normalize the covariances
         self.covariances_ /= self.degrees_of_freedom_
@@ -675,38 +736,47 @@ def _m_step(self, X, log_resp):
         n_samples, _ = X.shape
 
         nk, xk, sk = _estimate_gaussian_parameters(
-            X, np.exp(log_resp), self.reg_covar, self.covariance_type)
+            X, np.exp(log_resp), self.reg_covar, self.covariance_type
+        )
         self._estimate_weights(nk)
         self._estimate_means(nk, xk)
         self._estimate_precisions(nk, xk, sk)
 
     def _estimate_log_weights(self):
-        if self.weight_concentration_prior_type == 'dirichlet_process':
-            digamma_sum = digamma(self.weight_concentration_[0] +
-                                  self.weight_concentration_[1])
+        if self.weight_concentration_prior_type == "dirichlet_process":
+            digamma_sum = digamma(
+                self.weight_concentration_[0] + self.weight_concentration_[1]
+            )
             digamma_a = digamma(self.weight_concentration_[0])
             digamma_b = digamma(self.weight_concentration_[1])
-            return (digamma_a - digamma_sum +
-                    np.hstack((0, np.cumsum(digamma_b - digamma_sum)[:-1])))
+            return (
+                digamma_a
+                - digamma_sum
+                + np.hstack((0, np.cumsum(digamma_b - digamma_sum)[:-1]))
+            )
         else:
-            # case Variationnal Gaussian mixture with dirichlet distribution
-            return (digamma(self.weight_concentration_) -
-                    digamma(np.sum(self.weight_concentration_)))
+            # case Variational Gaussian mixture with dirichlet distribution
+            return digamma(self.weight_concentration_) - digamma(
+                np.sum(self.weight_concentration_)
+            )
 
     def _estimate_log_prob(self, X):
         _, n_features = X.shape
         # We remove `n_features * np.log(self.degrees_of_freedom_)` because
         # the precision matrix is normalized
-        log_gauss = (_estimate_log_gaussian_prob(
-            X, self.means_, self.precisions_cholesky_, self.covariance_type) -
-            .5 * n_features * np.log(self.degrees_of_freedom_))
+        log_gauss = _estimate_log_gaussian_prob(
+            X, self.means_, self.precisions_cholesky_, self.covariance_type
+        ) - 0.5 * n_features * np.log(self.degrees_of_freedom_)
 
-        log_lambda = n_features * np.log(2.) + np.sum(digamma(
-            .5 * (self.degrees_of_freedom_ -
-                  np.arange(0, n_features)[:, np.newaxis])), 0)
+        log_lambda = n_features * np.log(2.0) + np.sum(
+            digamma(
+                0.5
+                * (self.degrees_of_freedom_ - np.arange(0, n_features)[:, np.newaxis])
+            ),
+            0,
+        )
 
-        return log_gauss + .5 * (log_lambda -
-                                 n_features / self.mean_precision_)
+        return log_gauss + 0.5 * (log_lambda - n_features / self.mean_precision_)
 
     def _compute_lower_bound(self, log_resp, log_prob_norm):
         """Estimate the lower bound of the model.
@@ -732,63 +802,90 @@ def _compute_lower_bound(self, log_resp, log_prob_norm):
         """
         # Contrary to the original formula, we have done some simplification
         # and removed all the constant terms.
-        n_features, = self.mean_prior_.shape
+        (n_features,) = self.mean_prior_.shape
 
         # We removed `.5 * n_features * np.log(self.degrees_of_freedom_)`
         # because the precision matrix is normalized.
-        log_det_precisions_chol = (_compute_log_det_cholesky(
-            self.precisions_cholesky_, self.covariance_type, n_features) -
-            .5 * n_features * np.log(self.degrees_of_freedom_))
-
-        if self.covariance_type == 'tied':
-            log_wishart = self.n_components * np.float64(_log_wishart_norm(
-                self.degrees_of_freedom_, log_det_precisions_chol, n_features))
+        log_det_precisions_chol = _compute_log_det_cholesky(
+            self.precisions_cholesky_, self.covariance_type, n_features
+        ) - 0.5 * n_features * np.log(self.degrees_of_freedom_)
+
+        if self.covariance_type == "tied":
+            log_wishart = self.n_components * np.float64(
+                _log_wishart_norm(
+                    self.degrees_of_freedom_, log_det_precisions_chol, n_features
+                )
+            )
         else:
-            log_wishart = np.sum(_log_wishart_norm(
-                self.degrees_of_freedom_, log_det_precisions_chol, n_features))
+            log_wishart = np.sum(
+                _log_wishart_norm(
+                    self.degrees_of_freedom_, log_det_precisions_chol, n_features
+                )
+            )
 
-        if self.weight_concentration_prior_type == 'dirichlet_process':
-            log_norm_weight = -np.sum(betaln(self.weight_concentration_[0],
-                                             self.weight_concentration_[1]))
+        if self.weight_concentration_prior_type == "dirichlet_process":
+            log_norm_weight = -np.sum(
+                betaln(self.weight_concentration_[0], self.weight_concentration_[1])
+            )
         else:
             log_norm_weight = _log_dirichlet_norm(self.weight_concentration_)
 
-        return (-np.sum(np.exp(log_resp) * log_resp) -
-                log_wishart - log_norm_weight -
-                0.5 * n_features * np.sum(np.log(self.mean_precision_)))
+        return (
+            -np.sum(np.exp(log_resp) * log_resp)
+            - log_wishart
+            - log_norm_weight
+            - 0.5 * n_features * np.sum(np.log(self.mean_precision_))
+        )
 
     def _get_parameters(self):
-        return (self.weight_concentration_,
-                self.mean_precision_, self.means_,
-                self.degrees_of_freedom_, self.covariances_,
-                self.precisions_cholesky_)
+        return (
+            self.weight_concentration_,
+            self.mean_precision_,
+            self.means_,
+            self.degrees_of_freedom_,
+            self.covariances_,
+            self.precisions_cholesky_,
+        )
 
     def _set_parameters(self, params):
-        (self.weight_concentration_, self.mean_precision_, self.means_,
-         self.degrees_of_freedom_, self.covariances_,
-         self.precisions_cholesky_) = params
+        (
+            self.weight_concentration_,
+            self.mean_precision_,
+            self.means_,
+            self.degrees_of_freedom_,
+            self.covariances_,
+            self.precisions_cholesky_,
+        ) = params
 
         # Weights computation
         if self.weight_concentration_prior_type == "dirichlet_process":
-            weight_dirichlet_sum = (self.weight_concentration_[0] +
-                                    self.weight_concentration_[1])
+            weight_dirichlet_sum = (
+                self.weight_concentration_[0] + self.weight_concentration_[1]
+            )
             tmp = self.weight_concentration_[1] / weight_dirichlet_sum
             self.weights_ = (
-                self.weight_concentration_[0] / weight_dirichlet_sum *
-                np.hstack((1, np.cumprod(tmp[:-1]))))
+                self.weight_concentration_[0]
+                / weight_dirichlet_sum
+                * np.hstack((1, np.cumprod(tmp[:-1])))
+            )
             self.weights_ /= np.sum(self.weights_)
         else:
-            self. weights_ = (self.weight_concentration_ /
-                              np.sum(self.weight_concentration_))
+            self.weights_ = self.weight_concentration_ / np.sum(
+                self.weight_concentration_
+            )
 
         # Precisions matrices computation
-        if self.covariance_type == 'full':
-            self.precisions_ = np.array([
-                np.dot(prec_chol, prec_chol.T)
-                for prec_chol in self.precisions_cholesky_])
-
-        elif self.covariance_type == 'tied':
-            self.precisions_ = np.dot(self.precisions_cholesky_,
-                                      self.precisions_cholesky_.T)
+        if self.covariance_type == "full":
+            self.precisions_ = np.array(
+                [
+                    np.dot(prec_chol, prec_chol.T)
+                    for prec_chol in self.precisions_cholesky_
+                ]
+            )
+
+        elif self.covariance_type == "tied":
+            self.precisions_ = np.dot(
+                self.precisions_cholesky_, self.precisions_cholesky_.T
+            )
         else:
-            self.precisions_ = self.precisions_cholesky_ ** 2
+            self.precisions_ = self.precisions_cholesky_**2
diff --git a/sklearn/mixture/_gaussian_mixture.py b/sklearn/mixture/_gaussian_mixture.py
index f510c81cec2dd..c4bdd3a0d68c8 100644
--- a/sklearn/mixture/_gaussian_mixture.py
+++ b/sklearn/mixture/_gaussian_mixture.py
@@ -1,22 +1,20 @@
 """Gaussian Mixture Model."""
 
-# Author: Wei Xue 
-# Modified by Thierry Guillemot 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import numpy as np
-
 from scipy import linalg
 
-from ._base import BaseMixture, _check_shape
 from ..utils import check_array
+from ..utils._param_validation import StrOptions
 from ..utils.extmath import row_norms
-from ..utils.validation import _deprecate_positional_args
-
+from ._base import BaseMixture, _check_shape
 
 ###############################################################################
 # Gaussian mixture shape checkers used by the GaussianMixture class
 
+
 def _check_weights(weights, n_components):
     """Check the user provided 'weights'.
 
@@ -32,21 +30,24 @@ def _check_weights(weights, n_components):
     -------
     weights : array, shape (n_components,)
     """
-    weights = check_array(weights, dtype=[np.float64, np.float32],
-                          ensure_2d=False)
-    _check_shape(weights, (n_components,), 'weights')
+    weights = check_array(weights, dtype=[np.float64, np.float32], ensure_2d=False)
+    _check_shape(weights, (n_components,), "weights")
 
     # check range
-    if (any(np.less(weights, 0.)) or
-            any(np.greater(weights, 1.))):
-        raise ValueError("The parameter 'weights' should be in the range "
-                         "[0, 1], but got max value %.5f, min value %.5f"
-                         % (np.min(weights), np.max(weights)))
+    if any(np.less(weights, 0.0)) or any(np.greater(weights, 1.0)):
+        raise ValueError(
+            "The parameter 'weights' should be in the range "
+            "[0, 1], but got max value %.5f, min value %.5f"
+            % (np.min(weights), np.max(weights))
+        )
 
     # check normalization
-    if not np.allclose(np.abs(1. - np.sum(weights)), 0.):
-        raise ValueError("The parameter 'weights' should be normalized, "
-                         "but got sum(weights) = %.5f" % np.sum(weights))
+    atol = 1e-6 if weights.dtype == np.float32 else 1e-8
+    if not np.allclose(np.abs(1.0 - np.sum(weights)), 0.0, atol=atol):
+        raise ValueError(
+            "The parameter 'weights' should be normalized, but got sum(weights) = %.5f"
+            % np.sum(weights)
+        )
     return weights
 
 
@@ -69,23 +70,24 @@ def _check_means(means, n_components, n_features):
     means : array, (n_components, n_features)
     """
     means = check_array(means, dtype=[np.float64, np.float32], ensure_2d=False)
-    _check_shape(means, (n_components, n_features), 'means')
+    _check_shape(means, (n_components, n_features), "means")
     return means
 
 
 def _check_precision_positivity(precision, covariance_type):
     """Check a precision vector is positive-definite."""
     if np.any(np.less_equal(precision, 0.0)):
-        raise ValueError("'%s precision' should be "
-                         "positive" % covariance_type)
+        raise ValueError("'%s precision' should be positive" % covariance_type)
 
 
 def _check_precision_matrix(precision, covariance_type):
     """Check a precision matrix is symmetric and positive-definite."""
-    if not (np.allclose(precision, precision.T) and
-            np.all(linalg.eigvalsh(precision) > 0.)):
-        raise ValueError("'%s precision' should be symmetric, "
-                         "positive-definite" % covariance_type)
+    if not (
+        np.allclose(precision, precision.T) and np.all(linalg.eigvalsh(precision) > 0.0)
+    ):
+        raise ValueError(
+            "'%s precision' should be symmetric, positive-definite" % covariance_type
+        )
 
 
 def _check_precisions_full(precisions, covariance_type):
@@ -105,7 +107,7 @@ def _check_precisions(precisions, covariance_type, n_components, n_features):
         'diag' : shape of (n_components, n_features)
         'spherical' : shape of (n_components,)
 
-    covariance_type : string
+    covariance_type : str
 
     n_components : int
         Number of components.
@@ -117,21 +119,29 @@ def _check_precisions(precisions, covariance_type, n_components, n_features):
     -------
     precisions : array
     """
-    precisions = check_array(precisions, dtype=[np.float64, np.float32],
-                             ensure_2d=False,
-                             allow_nd=covariance_type == 'full')
-
-    precisions_shape = {'full': (n_components, n_features, n_features),
-                        'tied': (n_features, n_features),
-                        'diag': (n_components, n_features),
-                        'spherical': (n_components,)}
-    _check_shape(precisions, precisions_shape[covariance_type],
-                 '%s precision' % covariance_type)
-
-    _check_precisions = {'full': _check_precisions_full,
-                         'tied': _check_precision_matrix,
-                         'diag': _check_precision_positivity,
-                         'spherical': _check_precision_positivity}
+    precisions = check_array(
+        precisions,
+        dtype=[np.float64, np.float32],
+        ensure_2d=False,
+        allow_nd=covariance_type == "full",
+    )
+
+    precisions_shape = {
+        "full": (n_components, n_features, n_features),
+        "tied": (n_features, n_features),
+        "diag": (n_components, n_features),
+        "spherical": (n_components,),
+    }
+    _check_shape(
+        precisions, precisions_shape[covariance_type], "%s precision" % covariance_type
+    )
+
+    _check_precisions = {
+        "full": _check_precisions_full,
+        "tied": _check_precision_matrix,
+        "diag": _check_precision_positivity,
+        "spherical": _check_precision_positivity,
+    }
     _check_precisions[covariance_type](precisions, covariance_type)
     return precisions
 
@@ -139,6 +149,7 @@ def _check_precisions(precisions, covariance_type, n_components, n_features):
 ###############################################################################
 # Gaussian mixture parameters estimators (used by the M-Step)
 
+
 def _estimate_gaussian_covariances_full(resp, X, nk, means, reg_covar):
     """Estimate the full covariance matrices.
 
@@ -160,11 +171,11 @@ def _estimate_gaussian_covariances_full(resp, X, nk, means, reg_covar):
         The covariance matrix of the current components.
     """
     n_components, n_features = means.shape
-    covariances = np.empty((n_components, n_features, n_features))
+    covariances = np.empty((n_components, n_features, n_features), dtype=X.dtype)
     for k in range(n_components):
         diff = X - means[k]
         covariances[k] = np.dot(resp[:, k] * diff.T, diff) / nk[k]
-        covariances[k].flat[::n_features + 1] += reg_covar
+        covariances[k].flat[:: n_features + 1] += reg_covar
     return covariances
 
 
@@ -192,7 +203,7 @@ def _estimate_gaussian_covariances_tied(resp, X, nk, means, reg_covar):
     avg_means2 = np.dot(nk * means.T, means)
     covariance = avg_X2 - avg_means2
     covariance /= nk.sum()
-    covariance.flat[::len(covariance) + 1] += reg_covar
+    covariance.flat[:: len(covariance) + 1] += reg_covar
     return covariance
 
 
@@ -217,9 +228,8 @@ def _estimate_gaussian_covariances_diag(resp, X, nk, means, reg_covar):
         The covariance vector of the current components.
     """
     avg_X2 = np.dot(resp.T, X * X) / nk[:, np.newaxis]
-    avg_means2 = means ** 2
-    avg_X_means = means * np.dot(resp.T, X) / nk[:, np.newaxis]
-    return avg_X2 - 2 * avg_X_means + avg_means2 + reg_covar
+    avg_means2 = means**2
+    return avg_X2 - avg_means2 + reg_covar
 
 
 def _estimate_gaussian_covariances_spherical(resp, X, nk, means, reg_covar):
@@ -242,8 +252,7 @@ def _estimate_gaussian_covariances_spherical(resp, X, nk, means, reg_covar):
     variances : array, shape (n_components,)
         The variance values of each components.
     """
-    return _estimate_gaussian_covariances_diag(resp, X, nk,
-                                               means, reg_covar).mean(1)
+    return _estimate_gaussian_covariances_diag(resp, X, nk, means, reg_covar).mean(1)
 
 
 def _estimate_gaussian_parameters(X, resp, reg_covar, covariance_type):
@@ -277,11 +286,12 @@ def _estimate_gaussian_parameters(X, resp, reg_covar, covariance_type):
     """
     nk = resp.sum(axis=0) + 10 * np.finfo(resp.dtype).eps
     means = np.dot(resp.T, X) / nk[:, np.newaxis]
-    covariances = {"full": _estimate_gaussian_covariances_full,
-                   "tied": _estimate_gaussian_covariances_tied,
-                   "diag": _estimate_gaussian_covariances_diag,
-                   "spherical": _estimate_gaussian_covariances_spherical
-                   }[covariance_type](resp, X, nk, means, reg_covar)
+    covariances = {
+        "full": _estimate_gaussian_covariances_full,
+        "tied": _estimate_gaussian_covariances_tied,
+        "diag": _estimate_gaussian_covariances_diag,
+        "spherical": _estimate_gaussian_covariances_spherical,
+    }[covariance_type](resp, X, nk, means, reg_covar)
     return nk, means, covariances
 
 
@@ -307,34 +317,97 @@ def _compute_precision_cholesky(covariances, covariance_type):
         "Fitting the mixture model failed because some components have "
         "ill-defined empirical covariance (for instance caused by singleton "
         "or collapsed samples). Try to decrease the number of components, "
-        "or increase reg_covar.")
-
-    if covariance_type == 'full':
+        "increase reg_covar, or scale the input data."
+    )
+    dtype = covariances.dtype
+    if dtype == np.float32:
+        estimate_precision_error_message += (
+            " The numerical accuracy can also be improved by passing float64"
+            " data instead of float32."
+        )
+
+    if covariance_type == "full":
         n_components, n_features, _ = covariances.shape
-        precisions_chol = np.empty((n_components, n_features, n_features))
+        precisions_chol = np.empty((n_components, n_features, n_features), dtype=dtype)
         for k, covariance in enumerate(covariances):
             try:
                 cov_chol = linalg.cholesky(covariance, lower=True)
             except linalg.LinAlgError:
                 raise ValueError(estimate_precision_error_message)
-            precisions_chol[k] = linalg.solve_triangular(cov_chol,
-                                                         np.eye(n_features),
-                                                         lower=True).T
-    elif covariance_type == 'tied':
+            precisions_chol[k] = linalg.solve_triangular(
+                cov_chol, np.eye(n_features, dtype=dtype), lower=True
+            ).T
+    elif covariance_type == "tied":
         _, n_features = covariances.shape
         try:
             cov_chol = linalg.cholesky(covariances, lower=True)
         except linalg.LinAlgError:
             raise ValueError(estimate_precision_error_message)
-        precisions_chol = linalg.solve_triangular(cov_chol, np.eye(n_features),
-                                                  lower=True).T
+        precisions_chol = linalg.solve_triangular(
+            cov_chol, np.eye(n_features, dtype=dtype), lower=True
+        ).T
     else:
         if np.any(np.less_equal(covariances, 0.0)):
             raise ValueError(estimate_precision_error_message)
-        precisions_chol = 1. / np.sqrt(covariances)
+        precisions_chol = 1.0 / np.sqrt(covariances)
     return precisions_chol
 
 
+def _flipudlr(array):
+    """Reverse the rows and columns of an array."""
+    return np.flipud(np.fliplr(array))
+
+
+def _compute_precision_cholesky_from_precisions(precisions, covariance_type):
+    r"""Compute the Cholesky decomposition of precisions using precisions themselves.
+
+    As implemented in :func:`_compute_precision_cholesky`, the `precisions_cholesky_` is
+    an upper-triangular matrix for each Gaussian component, which can be expressed as
+    the $UU^T$ factorization of the precision matrix for each Gaussian component, where
+    $U$ is an upper-triangular matrix.
+
+    In order to use the Cholesky decomposition to get $UU^T$, the precision matrix
+    $\Lambda$ needs to be permutated such that its rows and columns are reversed, which
+    can be done by applying a similarity transformation with an exchange matrix $J$,
+    where the 1 elements reside on the anti-diagonal and all other elements are 0. In
+    particular, the Cholesky decomposition of the transformed precision matrix is
+    $J\Lambda J=LL^T$, where $L$ is a lower-triangular matrix. Because $\Lambda=UU^T$
+    and $J=J^{-1}=J^T$, the `precisions_cholesky_` for each Gaussian component can be
+    expressed as $JLJ$.
+
+    Refer to #26415 for details.
+
+    Parameters
+    ----------
+    precisions : array-like
+        The precision matrix of the current components.
+        The shape depends on the covariance_type.
+
+    covariance_type : {'full', 'tied', 'diag', 'spherical'}
+        The type of precision matrices.
+
+    Returns
+    -------
+    precisions_cholesky : array-like
+        The cholesky decomposition of sample precisions of the current
+        components. The shape depends on the covariance_type.
+    """
+    if covariance_type == "full":
+        precisions_cholesky = np.array(
+            [
+                _flipudlr(linalg.cholesky(_flipudlr(precision), lower=True))
+                for precision in precisions
+            ]
+        )
+    elif covariance_type == "tied":
+        precisions_cholesky = _flipudlr(
+            linalg.cholesky(_flipudlr(precisions), lower=True)
+        )
+    else:
+        precisions_cholesky = np.sqrt(precisions)
+    return precisions_cholesky
+
+
 ###############################################################################
 # Gaussian mixture probability estimators
 def _compute_log_det_cholesky(matrix_chol, covariance_type, n_features):
@@ -359,20 +432,20 @@ def _compute_log_det_cholesky(matrix_chol, covariance_type, n_features):
     log_det_precision_chol : array-like of shape (n_components,)
         The determinant of the precision matrix for each component.
     """
-    if covariance_type == 'full':
+    if covariance_type == "full":
         n_components, _, _ = matrix_chol.shape
-        log_det_chol = (np.sum(np.log(
-            matrix_chol.reshape(
-                n_components, -1)[:, ::n_features + 1]), 1))
+        log_det_chol = np.sum(
+            np.log(matrix_chol.reshape(n_components, -1)[:, :: n_features + 1]), axis=1
+        )
 
-    elif covariance_type == 'tied':
-        log_det_chol = (np.sum(np.log(np.diag(matrix_chol))))
+    elif covariance_type == "tied":
+        log_det_chol = np.sum(np.log(np.diag(matrix_chol)))
 
-    elif covariance_type == 'diag':
-        log_det_chol = (np.sum(np.log(matrix_chol), axis=1))
+    elif covariance_type == "diag":
+        log_det_chol = np.sum(np.log(matrix_chol), axis=1)
 
     else:
-        log_det_chol = n_features * (np.log(matrix_chol))
+        log_det_chol = n_features * np.log(matrix_chol)
 
     return log_det_chol
 
@@ -401,34 +474,42 @@ def _estimate_log_gaussian_prob(X, means, precisions_chol, covariance_type):
     """
     n_samples, n_features = X.shape
     n_components, _ = means.shape
-    # det(precision_chol) is half of det(precision)
-    log_det = _compute_log_det_cholesky(
-        precisions_chol, covariance_type, n_features)
-
-    if covariance_type == 'full':
-        log_prob = np.empty((n_samples, n_components))
+    # The determinant of the precision matrix from the Cholesky decomposition
+    # corresponds to the negative half of the determinant of the full precision
+    # matrix.
+    # In short: det(precision_chol) = - det(precision) / 2
+    log_det = _compute_log_det_cholesky(precisions_chol, covariance_type, n_features)
+
+    if covariance_type == "full":
+        log_prob = np.empty((n_samples, n_components), dtype=X.dtype)
         for k, (mu, prec_chol) in enumerate(zip(means, precisions_chol)):
             y = np.dot(X, prec_chol) - np.dot(mu, prec_chol)
             log_prob[:, k] = np.sum(np.square(y), axis=1)
 
-    elif covariance_type == 'tied':
-        log_prob = np.empty((n_samples, n_components))
+    elif covariance_type == "tied":
+        log_prob = np.empty((n_samples, n_components), dtype=X.dtype)
         for k, mu in enumerate(means):
             y = np.dot(X, precisions_chol) - np.dot(mu, precisions_chol)
             log_prob[:, k] = np.sum(np.square(y), axis=1)
 
-    elif covariance_type == 'diag':
-        precisions = precisions_chol ** 2
-        log_prob = (np.sum((means ** 2 * precisions), 1) -
-                    2. * np.dot(X, (means * precisions).T) +
-                    np.dot(X ** 2, precisions.T))
-
-    elif covariance_type == 'spherical':
-        precisions = precisions_chol ** 2
-        log_prob = (np.sum(means ** 2, 1) * precisions -
-                    2 * np.dot(X, means.T * precisions) +
-                    np.outer(row_norms(X, squared=True), precisions))
-    return -.5 * (n_features * np.log(2 * np.pi) + log_prob) + log_det
+    elif covariance_type == "diag":
+        precisions = precisions_chol**2
+        log_prob = (
+            np.sum((means**2 * precisions), 1)
+            - 2.0 * np.dot(X, (means * precisions).T)
+            + np.dot(X**2, precisions.T)
+        )
+
+    elif covariance_type == "spherical":
+        precisions = precisions_chol**2
+        log_prob = (
+            np.sum(means**2, 1) * precisions
+            - 2 * np.dot(X, means.T * precisions)
+            + np.outer(row_norms(X, squared=True), precisions)
+        )
+    # Since we are using the precision of the Cholesky decomposition,
+    # `- 0.5 * log_det_precision` becomes `+ log_det_precision_chol`
+    return -0.5 * (n_features * np.log(2 * np.pi).astype(X.dtype) + log_prob) + log_det
 
 
 class GaussianMixture(BaseMixture):
@@ -451,14 +532,13 @@ class GaussianMixture(BaseMixture):
         String describing the type of covariance parameters to use.
         Must be one of:
 
-        'full'
-            each component has its own general covariance matrix
-        'tied'
-            all components share the same general covariance matrix
-        'diag'
-            each component has its own diagonal covariance matrix
-        'spherical'
-            each component has its own single variance
+        - 'full': each component has its own general covariance matrix.
+        - 'tied': all components share the same general covariance matrix.
+        - 'diag': each component has its own diagonal covariance matrix.
+        - 'spherical': each component has its own single variance.
+
+        For an example of using `covariance_type`, refer to
+        :ref:`sphx_glr_auto_examples_mixture_plot_gmm_selection.py`.
 
     tol : float, default=1e-3
         The convergence threshold. EM iterations will stop when the
@@ -474,13 +554,20 @@ class GaussianMixture(BaseMixture):
     n_init : int, default=1
         The number of initializations to perform. The best results are kept.
 
-    init_params : {'kmeans', 'random'}, default='kmeans'
+    init_params : {'kmeans', 'k-means++', 'random', 'random_from_data'}, \
+    default='kmeans'
         The method used to initialize the weights, the means and the
         precisions.
-        Must be one of::
+        String must be one of:
+
+        - 'kmeans' : responsibilities are initialized using kmeans.
+        - 'k-means++' : use the k-means++ method to initialize.
+        - 'random' : responsibilities are initialized randomly.
+        - 'random_from_data' : initial means are randomly selected data points.
 
-            'kmeans' : responsibilities are initialized using kmeans.
-            'random' : responsibilities are initialized randomly.
+        .. versionchanged:: v1.1
+            `init_params` now accepts 'random_from_data' and 'k-means++' as
+            initialization methods.
 
     weights_init : array-like of shape (n_components, ), default=None
         The user-provided initial weights.
@@ -544,6 +631,9 @@ class GaussianMixture(BaseMixture):
             (n_components, n_features)             if 'diag',
             (n_components, n_features, n_features) if 'full'
 
+        For an example of using covariances, refer to
+        :ref:`sphx_glr_auto_examples_mixture_plot_gmm_covariances.py`.
+
     precisions_ : array-like
         The precision matrices for each component in the mixture. A precision
         matrix is the inverse of a covariance matrix. A covariance matrix is
@@ -573,7 +663,7 @@ class GaussianMixture(BaseMixture):
             (n_components, n_features, n_features) if 'full'
 
     converged_ : bool
-        True when convergence was reached in fit(), False otherwise.
+        True when convergence of the best fit of EM was reached, False otherwise.
 
     n_iter_ : int
         Number of step used by the best fit of EM to reach the convergence.
@@ -582,6 +672,26 @@ class GaussianMixture(BaseMixture):
         Lower bound value on the log-likelihood (of the training data with
         respect to the model) of the best fit of EM.
 
+    lower_bounds_ : array-like of shape (`n_iter_`,)
+        The list of lower bound values on the log-likelihood from each
+        iteration of the best fit of EM.
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    BayesianGaussianMixture : Gaussian mixture model fit with a variational
+        inference.
+
     Examples
     --------
     >>> import numpy as np
@@ -594,22 +704,48 @@ class GaussianMixture(BaseMixture):
     >>> gm.predict([[0, 0], [12, 3]])
     array([1, 0])
 
-    See Also
-    --------
-    BayesianGaussianMixture : Gaussian mixture model fit with a variational
-        inference.
+    For a comparison of Gaussian Mixture with other clustering algorithms, see
+    :ref:`sphx_glr_auto_examples_cluster_plot_cluster_comparison.py`
     """
-    @_deprecate_positional_args
-    def __init__(self, n_components=1, *, covariance_type='full', tol=1e-3,
-                 reg_covar=1e-6, max_iter=100, n_init=1, init_params='kmeans',
-                 weights_init=None, means_init=None, precisions_init=None,
-                 random_state=None, warm_start=False,
-                 verbose=0, verbose_interval=10):
+
+    _parameter_constraints: dict = {
+        **BaseMixture._parameter_constraints,
+        "covariance_type": [StrOptions({"full", "tied", "diag", "spherical"})],
+        "weights_init": ["array-like", None],
+        "means_init": ["array-like", None],
+        "precisions_init": ["array-like", None],
+    }
+
+    def __init__(
+        self,
+        n_components=1,
+        *,
+        covariance_type="full",
+        tol=1e-3,
+        reg_covar=1e-6,
+        max_iter=100,
+        n_init=1,
+        init_params="kmeans",
+        weights_init=None,
+        means_init=None,
+        precisions_init=None,
+        random_state=None,
+        warm_start=False,
+        verbose=0,
+        verbose_interval=10,
+    ):
         super().__init__(
-            n_components=n_components, tol=tol, reg_covar=reg_covar,
-            max_iter=max_iter, n_init=n_init, init_params=init_params,
-            random_state=random_state, warm_start=warm_start,
-            verbose=verbose, verbose_interval=verbose_interval)
+            n_components=n_components,
+            tol=tol,
+            reg_covar=reg_covar,
+            max_iter=max_iter,
+            n_init=n_init,
+            init_params=init_params,
+            random_state=random_state,
+            warm_start=warm_start,
+            verbose=verbose,
+            verbose_interval=verbose_interval,
+        )
 
         self.covariance_type = covariance_type
         self.weights_init = weights_init
@@ -619,25 +755,35 @@ def __init__(self, n_components=1, *, covariance_type='full', tol=1e-3,
     def _check_parameters(self, X):
         """Check the Gaussian mixture parameters are well defined."""
         _, n_features = X.shape
-        if self.covariance_type not in ['spherical', 'tied', 'diag', 'full']:
-            raise ValueError("Invalid value for 'covariance_type': %s "
-                             "'covariance_type' should be in "
-                             "['spherical', 'tied', 'diag', 'full']"
-                             % self.covariance_type)
 
         if self.weights_init is not None:
-            self.weights_init = _check_weights(self.weights_init,
-                                               self.n_components)
+            self.weights_init = _check_weights(self.weights_init, self.n_components)
 
         if self.means_init is not None:
-            self.means_init = _check_means(self.means_init,
-                                           self.n_components, n_features)
+            self.means_init = _check_means(
+                self.means_init, self.n_components, n_features
+            )
 
         if self.precisions_init is not None:
-            self.precisions_init = _check_precisions(self.precisions_init,
-                                                     self.covariance_type,
-                                                     self.n_components,
-                                                     n_features)
+            self.precisions_init = _check_precisions(
+                self.precisions_init,
+                self.covariance_type,
+                self.n_components,
+                n_features,
+            )
+
+    def _initialize_parameters(self, X, random_state):
+        # If all the initial parameters are all provided, then there is no need to run
+        # the initialization.
+        compute_resp = (
+            self.weights_init is None
+            or self.means_init is None
+            or self.precisions_init is None
+        )
+        if compute_resp:
+            super()._initialize_parameters(X, random_state)
+        else:
+            self._initialize(X, None)
 
     def _initialize(self, X, resp):
         """Initialization of the Gaussian mixture parameters.
@@ -649,28 +795,26 @@ def _initialize(self, X, resp):
         resp : array-like of shape (n_samples, n_components)
         """
         n_samples, _ = X.shape
-
-        weights, means, covariances = _estimate_gaussian_parameters(
-            X, resp, self.reg_covar, self.covariance_type)
-        weights /= n_samples
-
-        self.weights_ = (weights if self.weights_init is None
-                         else self.weights_init)
+        weights, means, covariances = None, None, None
+        if resp is not None:
+            weights, means, covariances = _estimate_gaussian_parameters(
+                X, resp, self.reg_covar, self.covariance_type
+            )
+            if self.weights_init is None:
+                weights /= n_samples
+
+        self.weights_ = weights if self.weights_init is None else self.weights_init
         self.means_ = means if self.means_init is None else self.means_init
 
         if self.precisions_init is None:
             self.covariances_ = covariances
             self.precisions_cholesky_ = _compute_precision_cholesky(
-                covariances, self.covariance_type)
-        elif self.covariance_type == 'full':
-            self.precisions_cholesky_ = np.array(
-                [linalg.cholesky(prec_init, lower=True)
-                 for prec_init in self.precisions_init])
-        elif self.covariance_type == 'tied':
-            self.precisions_cholesky_ = linalg.cholesky(self.precisions_init,
-                                                        lower=True)
+                covariances, self.covariance_type
+            )
         else:
-            self.precisions_cholesky_ = self.precisions_init
+            self.precisions_cholesky_ = _compute_precision_cholesky_from_precisions(
+                self.precisions_init, self.covariance_type
+            )
 
     def _m_step(self, X, log_resp):
         """M step.
@@ -683,17 +827,18 @@ def _m_step(self, X, log_resp):
             Logarithm of the posterior probabilities (or responsibilities) of
             the point of each sample in X.
         """
-        n_samples, _ = X.shape
-        self.weights_, self.means_, self.covariances_ = (
-            _estimate_gaussian_parameters(X, np.exp(log_resp), self.reg_covar,
-                                          self.covariance_type))
-        self.weights_ /= n_samples
+        self.weights_, self.means_, self.covariances_ = _estimate_gaussian_parameters(
+            X, np.exp(log_resp), self.reg_covar, self.covariance_type
+        )
+        self.weights_ /= self.weights_.sum()
         self.precisions_cholesky_ = _compute_precision_cholesky(
-            self.covariances_, self.covariance_type)
+            self.covariances_, self.covariance_type
+        )
 
     def _estimate_log_prob(self, X):
         return _estimate_log_gaussian_prob(
-            X, self.means_, self.precisions_cholesky_, self.covariance_type)
+            X, self.means_, self.precisions_cholesky_, self.covariance_type
+        )
 
     def _estimate_log_weights(self):
         return np.log(self.weights_)
@@ -702,37 +847,47 @@ def _compute_lower_bound(self, _, log_prob_norm):
         return log_prob_norm
 
     def _get_parameters(self):
-        return (self.weights_, self.means_, self.covariances_,
-                self.precisions_cholesky_)
+        return (
+            self.weights_,
+            self.means_,
+            self.covariances_,
+            self.precisions_cholesky_,
+        )
 
     def _set_parameters(self, params):
-        (self.weights_, self.means_, self.covariances_,
-         self.precisions_cholesky_) = params
+        (
+            self.weights_,
+            self.means_,
+            self.covariances_,
+            self.precisions_cholesky_,
+        ) = params
 
         # Attributes computation
         _, n_features = self.means_.shape
 
-        if self.covariance_type == 'full':
-            self.precisions_ = np.empty(self.precisions_cholesky_.shape)
+        dtype = self.precisions_cholesky_.dtype
+        if self.covariance_type == "full":
+            self.precisions_ = np.empty_like(self.precisions_cholesky_)
             for k, prec_chol in enumerate(self.precisions_cholesky_):
                 self.precisions_[k] = np.dot(prec_chol, prec_chol.T)
 
-        elif self.covariance_type == 'tied':
-            self.precisions_ = np.dot(self.precisions_cholesky_,
-                                      self.precisions_cholesky_.T)
+        elif self.covariance_type == "tied":
+            self.precisions_ = np.dot(
+                self.precisions_cholesky_, self.precisions_cholesky_.T
+            )
         else:
-            self.precisions_ = self.precisions_cholesky_ ** 2
+            self.precisions_ = self.precisions_cholesky_**2
 
     def _n_parameters(self):
         """Return the number of free parameters in the model."""
         _, n_features = self.means_.shape
-        if self.covariance_type == 'full':
-            cov_params = self.n_components * n_features * (n_features + 1) / 2.
-        elif self.covariance_type == 'diag':
+        if self.covariance_type == "full":
+            cov_params = self.n_components * n_features * (n_features + 1) / 2.0
+        elif self.covariance_type == "diag":
             cov_params = self.n_components * n_features
-        elif self.covariance_type == 'tied':
-            cov_params = n_features * (n_features + 1) / 2.
-        elif self.covariance_type == 'spherical':
+        elif self.covariance_type == "tied":
+            cov_params = n_features * (n_features + 1) / 2.0
+        elif self.covariance_type == "spherical":
             cov_params = self.n_components
         mean_params = n_features * self.n_components
         return int(cov_params + mean_params + self.n_components - 1)
@@ -740,24 +895,36 @@ def _n_parameters(self):
     def bic(self, X):
         """Bayesian information criterion for the current model on the input X.
 
+        You can refer to this :ref:`mathematical section ` for more
+        details regarding the formulation of the BIC used.
+
+        For an example of GMM selection using `bic` information criterion,
+        refer to :ref:`sphx_glr_auto_examples_mixture_plot_gmm_selection.py`.
+
         Parameters
         ----------
         X : array of shape (n_samples, n_dimensions)
+            The input samples.
 
         Returns
         -------
         bic : float
             The lower the better.
         """
-        return (-2 * self.score(X) * X.shape[0] +
-                self._n_parameters() * np.log(X.shape[0]))
+        return -2 * self.score(X) * X.shape[0] + self._n_parameters() * np.log(
+            X.shape[0]
+        )
 
     def aic(self, X):
         """Akaike information criterion for the current model on the input X.
 
+        You can refer to this :ref:`mathematical section ` for more
+        details regarding the formulation of the AIC used.
+
         Parameters
         ----------
         X : array of shape (n_samples, n_dimensions)
+            The input samples.
 
         Returns
         -------
diff --git a/sklearn/mixture/tests/test_bayesian_mixture.py b/sklearn/mixture/tests/test_bayesian_mixture.py
index 1d061da908e3c..d36543903cb87 100644
--- a/sklearn/mixture/tests/test_bayesian_mixture.py
+++ b/sklearn/mixture/tests/test_bayesian_mixture.py
@@ -1,38 +1,33 @@
-# Author: Wei Xue 
-#         Thierry Guillemot 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
 import copy
 
 import numpy as np
-from scipy.special import gammaln
 import pytest
+from scipy.special import gammaln
 
-from sklearn.utils._testing import assert_raise_message
-from sklearn.utils._testing import assert_almost_equal
-from sklearn.utils._testing import assert_array_equal
-
+from sklearn.exceptions import NotFittedError
 from sklearn.metrics.cluster import adjusted_rand_score
-
-from sklearn.mixture._bayesian_mixture import _log_dirichlet_norm
-from sklearn.mixture._bayesian_mixture import _log_wishart_norm
-
 from sklearn.mixture import BayesianGaussianMixture
-
+from sklearn.mixture._bayesian_mixture import _log_dirichlet_norm, _log_wishart_norm
 from sklearn.mixture.tests.test_gaussian_mixture import RandomData
-from sklearn.exceptions import ConvergenceWarning, NotFittedError
-from sklearn.utils._testing import ignore_warnings
+from sklearn.utils._testing import (
+    assert_almost_equal,
+    assert_array_equal,
+)
 
-
-COVARIANCE_TYPE = ['full', 'tied', 'diag', 'spherical']
-PRIOR_TYPE = ['dirichlet_process', 'dirichlet_distribution']
+COVARIANCE_TYPE = ["full", "tied", "diag", "spherical"]
+PRIOR_TYPE = ["dirichlet_process", "dirichlet_distribution"]
 
 
 def test_log_dirichlet_norm():
     rng = np.random.RandomState(0)
 
     weight_concentration = rng.rand(2)
-    expected_norm = (gammaln(np.sum(weight_concentration)) -
-                     np.sum(gammaln(weight_concentration)))
+    expected_norm = gammaln(np.sum(weight_concentration)) - np.sum(
+        gammaln(weight_concentration)
+    )
     predected_norm = _log_dirichlet_norm(weight_concentration)
 
     assert_almost_equal(expected_norm, predected_norm)
@@ -42,80 +37,45 @@ def test_log_wishart_norm():
     rng = np.random.RandomState(0)
 
     n_components, n_features = 5, 2
-    degrees_of_freedom = np.abs(rng.rand(n_components)) + 1.
+    degrees_of_freedom = np.abs(rng.rand(n_components)) + 1.0
     log_det_precisions_chol = n_features * np.log(range(2, 2 + n_components))
 
     expected_norm = np.empty(5)
     for k, (degrees_of_freedom_k, log_det_k) in enumerate(
-            zip(degrees_of_freedom, log_det_precisions_chol)):
+        zip(degrees_of_freedom, log_det_precisions_chol)
+    ):
         expected_norm[k] = -(
-            degrees_of_freedom_k * (log_det_k + .5 * n_features * np.log(2.)) +
-            np.sum(gammaln(.5 * (degrees_of_freedom_k -
-                                 np.arange(0, n_features)[:, np.newaxis])), 0))
-    predected_norm = _log_wishart_norm(degrees_of_freedom,
-                                       log_det_precisions_chol, n_features)
+            degrees_of_freedom_k * (log_det_k + 0.5 * n_features * np.log(2.0))
+            + np.sum(
+                gammaln(
+                    0.5
+                    * (degrees_of_freedom_k - np.arange(0, n_features)[:, np.newaxis])
+                ),
+                0,
+            )
+        ).item()
+    predected_norm = _log_wishart_norm(
+        degrees_of_freedom, log_det_precisions_chol, n_features
+    )
 
     assert_almost_equal(expected_norm, predected_norm)
 
 
-def test_bayesian_mixture_covariance_type():
-    rng = np.random.RandomState(0)
-    n_samples, n_features = 10, 2
-    X = rng.rand(n_samples, n_features)
-
-    covariance_type = 'bad_covariance_type'
-    bgmm = BayesianGaussianMixture(covariance_type=covariance_type,
-                                   random_state=rng)
-    assert_raise_message(ValueError,
-                         "Invalid value for 'covariance_type': %s "
-                         "'covariance_type' should be in "
-                         "['spherical', 'tied', 'diag', 'full']"
-                         % covariance_type, bgmm.fit, X)
-
-
-def test_bayesian_mixture_weight_concentration_prior_type():
-    rng = np.random.RandomState(0)
-    n_samples, n_features = 10, 2
-    X = rng.rand(n_samples, n_features)
-
-    bad_prior_type = 'bad_prior_type'
-    bgmm = BayesianGaussianMixture(
-        weight_concentration_prior_type=bad_prior_type, random_state=rng)
-    assert_raise_message(ValueError,
-                         "Invalid value for 'weight_concentration_prior_type':"
-                         " %s 'weight_concentration_prior_type' should be in "
-                         "['dirichlet_process', 'dirichlet_distribution']"
-                         % bad_prior_type, bgmm.fit, X)
-
-
 def test_bayesian_mixture_weights_prior_initialisation():
     rng = np.random.RandomState(0)
     n_samples, n_components, n_features = 10, 5, 2
     X = rng.rand(n_samples, n_features)
 
-    # Check raise message for a bad value of weight_concentration_prior
-    bad_weight_concentration_prior_ = 0.
-    bgmm = BayesianGaussianMixture(
-        weight_concentration_prior=bad_weight_concentration_prior_,
-        random_state=0)
-    assert_raise_message(ValueError,
-                         "The parameter 'weight_concentration_prior' "
-                         "should be greater than 0., but got %.3f."
-                         % bad_weight_concentration_prior_,
-                         bgmm.fit, X)
-
     # Check correct init for a given value of weight_concentration_prior
     weight_concentration_prior = rng.rand()
     bgmm = BayesianGaussianMixture(
-        weight_concentration_prior=weight_concentration_prior,
-        random_state=rng).fit(X)
-    assert_almost_equal(weight_concentration_prior,
-                        bgmm.weight_concentration_prior_)
+        weight_concentration_prior=weight_concentration_prior, random_state=rng
+    ).fit(X)
+    assert_almost_equal(weight_concentration_prior, bgmm.weight_concentration_prior_)
 
     # Check correct init for the default value of weight_concentration_prior
-    bgmm = BayesianGaussianMixture(n_components=n_components,
-                                   random_state=rng).fit(X)
-    assert_almost_equal(1. / n_components, bgmm.weight_concentration_prior_)
+    bgmm = BayesianGaussianMixture(n_components=n_components, random_state=rng).fit(X)
+    assert_almost_equal(1.0 / n_components, bgmm.weight_concentration_prior_)
 
 
 def test_bayesian_mixture_mean_prior_initialisation():
@@ -123,47 +83,26 @@ def test_bayesian_mixture_mean_prior_initialisation():
     n_samples, n_components, n_features = 10, 3, 2
     X = rng.rand(n_samples, n_features)
 
-    # Check raise message for a bad value of mean_precision_prior
-    bad_mean_precision_prior_ = 0.
-    bgmm = BayesianGaussianMixture(
-        mean_precision_prior=bad_mean_precision_prior_,
-        random_state=rng)
-    assert_raise_message(ValueError,
-                         "The parameter 'mean_precision_prior' should be "
-                         "greater than 0., but got %.3f."
-                         % bad_mean_precision_prior_,
-                         bgmm.fit, X)
-
     # Check correct init for a given value of mean_precision_prior
     mean_precision_prior = rng.rand()
     bgmm = BayesianGaussianMixture(
-        mean_precision_prior=mean_precision_prior,
-        random_state=rng).fit(X)
+        mean_precision_prior=mean_precision_prior, random_state=rng
+    ).fit(X)
     assert_almost_equal(mean_precision_prior, bgmm.mean_precision_prior_)
 
     # Check correct init for the default value of mean_precision_prior
     bgmm = BayesianGaussianMixture(random_state=rng).fit(X)
-    assert_almost_equal(1., bgmm.mean_precision_prior_)
-
-    # Check raise message for a bad shape of mean_prior
-    mean_prior = rng.rand(n_features + 1)
-    bgmm = BayesianGaussianMixture(n_components=n_components,
-                                   mean_prior=mean_prior,
-                                   random_state=rng)
-    assert_raise_message(ValueError,
-                         "The parameter 'means' should have the shape of ",
-                         bgmm.fit, X)
+    assert_almost_equal(1.0, bgmm.mean_precision_prior_)
 
     # Check correct init for a given value of mean_prior
     mean_prior = rng.rand(n_features)
-    bgmm = BayesianGaussianMixture(n_components=n_components,
-                                   mean_prior=mean_prior,
-                                   random_state=rng).fit(X)
+    bgmm = BayesianGaussianMixture(
+        n_components=n_components, mean_prior=mean_prior, random_state=rng
+    ).fit(X)
     assert_almost_equal(mean_prior, bgmm.mean_prior_)
 
     # Check correct init for the default value of bemean_priorta
-    bgmm = BayesianGaussianMixture(n_components=n_components,
-                                   random_state=rng).fit(X)
+    bgmm = BayesianGaussianMixture(n_components=n_components, random_state=rng).fit(X)
     assert_almost_equal(X.mean(axis=0), bgmm.mean_prior_)
 
 
@@ -173,71 +112,61 @@ def test_bayesian_mixture_precisions_prior_initialisation():
     X = rng.rand(n_samples, n_features)
 
     # Check raise message for a bad value of degrees_of_freedom_prior
-    bad_degrees_of_freedom_prior_ = n_features - 1.
+    bad_degrees_of_freedom_prior_ = n_features - 1.0
     bgmm = BayesianGaussianMixture(
-        degrees_of_freedom_prior=bad_degrees_of_freedom_prior_,
-        random_state=rng)
-    assert_raise_message(ValueError,
-                         "The parameter 'degrees_of_freedom_prior' should be "
-                         "greater than %d, but got %.3f."
-                         % (n_features - 1, bad_degrees_of_freedom_prior_),
-                         bgmm.fit, X)
+        degrees_of_freedom_prior=bad_degrees_of_freedom_prior_, random_state=rng
+    )
+    msg = (
+        "The parameter 'degrees_of_freedom_prior' should be greater than"
+        f" {n_features - 1}, but got {bad_degrees_of_freedom_prior_:.3f}."
+    )
+    with pytest.raises(ValueError, match=msg):
+        bgmm.fit(X)
 
     # Check correct init for a given value of degrees_of_freedom_prior
-    degrees_of_freedom_prior = rng.rand() + n_features - 1.
+    degrees_of_freedom_prior = rng.rand() + n_features - 1.0
     bgmm = BayesianGaussianMixture(
-        degrees_of_freedom_prior=degrees_of_freedom_prior,
-        random_state=rng).fit(X)
-    assert_almost_equal(degrees_of_freedom_prior,
-                        bgmm.degrees_of_freedom_prior_)
+        degrees_of_freedom_prior=degrees_of_freedom_prior, random_state=rng
+    ).fit(X)
+    assert_almost_equal(degrees_of_freedom_prior, bgmm.degrees_of_freedom_prior_)
 
     # Check correct init for the default value of degrees_of_freedom_prior
     degrees_of_freedom_prior_default = n_features
     bgmm = BayesianGaussianMixture(
-        degrees_of_freedom_prior=degrees_of_freedom_prior_default,
-        random_state=rng).fit(X)
-    assert_almost_equal(degrees_of_freedom_prior_default,
-                        bgmm.degrees_of_freedom_prior_)
+        degrees_of_freedom_prior=degrees_of_freedom_prior_default, random_state=rng
+    ).fit(X)
+    assert_almost_equal(
+        degrees_of_freedom_prior_default, bgmm.degrees_of_freedom_prior_
+    )
 
     # Check correct init for a given value of covariance_prior
     covariance_prior = {
-        'full': np.cov(X.T, bias=1) + 10,
-        'tied': np.cov(X.T, bias=1) + 5,
-        'diag': np.diag(np.atleast_2d(np.cov(X.T, bias=1))) + 3,
-        'spherical': rng.rand()}
+        "full": np.cov(X.T, bias=1) + 10,
+        "tied": np.cov(X.T, bias=1) + 5,
+        "diag": np.diag(np.atleast_2d(np.cov(X.T, bias=1))) + 3,
+        "spherical": rng.rand(),
+    }
 
     bgmm = BayesianGaussianMixture(random_state=rng)
-    for cov_type in ['full', 'tied', 'diag', 'spherical']:
+    for cov_type in ["full", "tied", "diag", "spherical"]:
         bgmm.covariance_type = cov_type
         bgmm.covariance_prior = covariance_prior[cov_type]
         bgmm.fit(X)
-        assert_almost_equal(covariance_prior[cov_type],
-                            bgmm.covariance_prior_)
-
-    # Check raise message for a bad spherical value of covariance_prior
-    bad_covariance_prior_ = -1.
-    bgmm = BayesianGaussianMixture(covariance_type='spherical',
-                                   covariance_prior=bad_covariance_prior_,
-                                   random_state=rng)
-    assert_raise_message(ValueError,
-                         "The parameter 'spherical covariance_prior' "
-                         "should be greater than 0., but got %.3f."
-                         % bad_covariance_prior_,
-                         bgmm.fit, X)
+        assert_almost_equal(covariance_prior[cov_type], bgmm.covariance_prior_)
 
     # Check correct init for the default value of covariance_prior
     covariance_prior_default = {
-        'full': np.atleast_2d(np.cov(X.T)),
-        'tied': np.atleast_2d(np.cov(X.T)),
-        'diag': np.var(X, axis=0, ddof=1),
-        'spherical': np.var(X, axis=0, ddof=1).mean()}
+        "full": np.atleast_2d(np.cov(X.T)),
+        "tied": np.atleast_2d(np.cov(X.T)),
+        "diag": np.var(X, axis=0, ddof=1),
+        "spherical": np.var(X, axis=0, ddof=1).mean(),
+    }
 
     bgmm = BayesianGaussianMixture(random_state=0)
-    for cov_type in ['full', 'tied', 'diag', 'spherical']:
+    for cov_type in ["full", "tied", "diag", "spherical"]:
         bgmm.covariance_type = cov_type
         bgmm.fit(X)
-        assert_almost_equal(covariance_prior_default[cov_type],
-                            bgmm.covariance_prior_)
+        assert_almost_equal(covariance_prior_default[cov_type], bgmm.covariance_prior_)
 
 
 def test_bayesian_mixture_check_is_fitted():
@@ -247,9 +176,10 @@ def test_bayesian_mixture_check_is_fitted():
     # Check raise message
     bgmm = BayesianGaussianMixture(random_state=rng)
     X = rng.rand(n_samples, n_features)
-    assert_raise_message(ValueError,
-                         'This BayesianGaussianMixture instance is not '
-                         'fitted yet.', bgmm.score, X)
+
+    msg = "This BayesianGaussianMixture instance is not fitted yet."
+    with pytest.raises(ValueError, match=msg):
+        bgmm.score(X)
 
 
 def test_bayesian_mixture_weights():
@@ -261,28 +191,35 @@ def test_bayesian_mixture_weights():
     # Case Dirichlet distribution for the weight concentration prior type
     bgmm = BayesianGaussianMixture(
         weight_concentration_prior_type="dirichlet_distribution",
-        n_components=3, random_state=rng).fit(X)
+        n_components=3,
+        random_state=rng,
+    ).fit(X)
 
-    expected_weights = (bgmm.weight_concentration_ /
-                        np.sum(bgmm.weight_concentration_))
+    expected_weights = bgmm.weight_concentration_ / np.sum(bgmm.weight_concentration_)
     assert_almost_equal(expected_weights, bgmm.weights_)
     assert_almost_equal(np.sum(bgmm.weights_), 1.0)
 
     # Case Dirichlet process for the weight concentration prior type
     dpgmm = BayesianGaussianMixture(
         weight_concentration_prior_type="dirichlet_process",
-        n_components=3, random_state=rng).fit(X)
-    weight_dirichlet_sum = (dpgmm.weight_concentration_[0] +
-                            dpgmm.weight_concentration_[1])
+        n_components=3,
+        random_state=rng,
+    ).fit(X)
+    weight_dirichlet_sum = (
+        dpgmm.weight_concentration_[0] + dpgmm.weight_concentration_[1]
+    )
     tmp = dpgmm.weight_concentration_[1] / weight_dirichlet_sum
-    expected_weights = (dpgmm.weight_concentration_[0] / weight_dirichlet_sum *
-                        np.hstack((1, np.cumprod(tmp[:-1]))))
+    expected_weights = (
+        dpgmm.weight_concentration_[0]
+        / weight_dirichlet_sum
+        * np.hstack((1, np.cumprod(tmp[:-1])))
+    )
     expected_weights /= np.sum(expected_weights)
     assert_almost_equal(expected_weights, dpgmm.weights_)
     assert_almost_equal(np.sum(dpgmm.weights_), 1.0)
 
 
-@ignore_warnings(category=ConvergenceWarning)
+@pytest.mark.filterwarnings("ignore::sklearn.exceptions.ConvergenceWarning")
 def test_monotonic_likelihood():
     # We check that each step of the each step of variational inference without
     # regularization improve monotonically the training set of the bound
@@ -295,9 +232,14 @@ def test_monotonic_likelihood():
             X = rand_data.X[covar_type]
             bgmm = BayesianGaussianMixture(
                 weight_concentration_prior_type=prior_type,
-                n_components=2 * n_components, covariance_type=covar_type,
-                warm_start=True, max_iter=1, random_state=rng, tol=1e-3)
-            current_lower_bound = -np.infty
+                n_components=2 * n_components,
+                covariance_type=covar_type,
+                warm_start=True,
+                max_iter=1,
+                random_state=rng,
+                tol=1e-3,
+            )
+            current_lower_bound = -np.inf
             # Do one training iteration at a time so we can make sure that the
             # training log likelihood increases after each iteration.
             for _ in range(600):
@@ -307,7 +249,7 @@ def test_monotonic_likelihood():
 
                 if bgmm.converged_:
                     break
-            assert(bgmm.converged_)
+            assert bgmm.converged_
 
 
 def test_compare_covar_type():
@@ -315,27 +257,35 @@ def test_compare_covar_type():
     # 1 iter of the M-step (done during _initialize_parameters).
     rng = np.random.RandomState(0)
     rand_data = RandomData(rng, scale=7)
-    X = rand_data.X['full']
+    X = rand_data.X["full"]
     n_components = rand_data.n_components
 
     for prior_type in PRIOR_TYPE:
         # Computation of the full_covariance
         bgmm = BayesianGaussianMixture(
             weight_concentration_prior_type=prior_type,
-            n_components=2 * n_components, covariance_type='full',
-            max_iter=1, random_state=0, tol=1e-7)
-        bgmm._check_initial_parameters(X)
+            n_components=2 * n_components,
+            covariance_type="full",
+            max_iter=1,
+            random_state=0,
+            tol=1e-7,
+        )
+        bgmm._check_parameters(X)
         bgmm._initialize_parameters(X, np.random.RandomState(0))
         full_covariances = (
-            bgmm.covariances_ *
-            bgmm.degrees_of_freedom_[:, np.newaxis, np.newaxis])
+            bgmm.covariances_ * bgmm.degrees_of_freedom_[:, np.newaxis, np.newaxis]
+        )
 
         # Check tied_covariance = mean(full_covariances, 0)
         bgmm = BayesianGaussianMixture(
             weight_concentration_prior_type=prior_type,
-            n_components=2 * n_components, covariance_type='tied',
-            max_iter=1, random_state=0, tol=1e-7)
-        bgmm._check_initial_parameters(X)
+            n_components=2 * n_components,
+            covariance_type="tied",
+            max_iter=1,
+            random_state=0,
+            tol=1e-7,
+        )
+        bgmm._check_parameters(X)
         bgmm._initialize_parameters(X, np.random.RandomState(0))
 
         tied_covariance = bgmm.covariances_ * bgmm.degrees_of_freedom_
@@ -344,31 +294,37 @@ def test_compare_covar_type():
         # Check diag_covariance = diag(full_covariances)
         bgmm = BayesianGaussianMixture(
             weight_concentration_prior_type=prior_type,
-            n_components=2 * n_components, covariance_type='diag',
-            max_iter=1, random_state=0, tol=1e-7)
-        bgmm._check_initial_parameters(X)
+            n_components=2 * n_components,
+            covariance_type="diag",
+            max_iter=1,
+            random_state=0,
+            tol=1e-7,
+        )
+        bgmm._check_parameters(X)
         bgmm._initialize_parameters(X, np.random.RandomState(0))
 
-        diag_covariances = (bgmm.covariances_ *
-                            bgmm.degrees_of_freedom_[:, np.newaxis])
-        assert_almost_equal(diag_covariances,
-                            np.array([np.diag(cov)
-                                     for cov in full_covariances]))
+        diag_covariances = bgmm.covariances_ * bgmm.degrees_of_freedom_[:, np.newaxis]
+        assert_almost_equal(
+            diag_covariances, np.array([np.diag(cov) for cov in full_covariances])
+        )
 
         # Check spherical_covariance = np.mean(diag_covariances, 0)
         bgmm = BayesianGaussianMixture(
             weight_concentration_prior_type=prior_type,
-            n_components=2 * n_components, covariance_type='spherical',
-            max_iter=1, random_state=0, tol=1e-7)
-        bgmm._check_initial_parameters(X)
+            n_components=2 * n_components,
+            covariance_type="spherical",
+            max_iter=1,
+            random_state=0,
+            tol=1e-7,
+        )
+        bgmm._check_parameters(X)
         bgmm._initialize_parameters(X, np.random.RandomState(0))
 
         spherical_covariances = bgmm.covariances_ * bgmm.degrees_of_freedom_
-        assert_almost_equal(
-            spherical_covariances, np.mean(diag_covariances, 1))
+        assert_almost_equal(spherical_covariances, np.mean(diag_covariances, 1))
 
 
-@ignore_warnings(category=ConvergenceWarning)
+@pytest.mark.filterwarnings("ignore::sklearn.exceptions.ConvergenceWarning")
 def test_check_covariance_precision():
     # We check that the dot product of the covariance and the precision
     # matrices is identity.
@@ -377,31 +333,33 @@ def test_check_covariance_precision():
     n_components, n_features = 2 * rand_data.n_components, 2
 
     # Computation of the full_covariance
-    bgmm = BayesianGaussianMixture(n_components=n_components,
-                                   max_iter=100, random_state=rng, tol=1e-3,
-                                   reg_covar=0)
+    bgmm = BayesianGaussianMixture(
+        n_components=n_components, max_iter=100, random_state=rng, tol=1e-3, reg_covar=0
+    )
     for covar_type in COVARIANCE_TYPE:
         bgmm.covariance_type = covar_type
         bgmm.fit(rand_data.X[covar_type])
 
-        if covar_type == 'full':
+        if covar_type == "full":
             for covar, precision in zip(bgmm.covariances_, bgmm.precisions_):
-                assert_almost_equal(np.dot(covar, precision),
-                                    np.eye(n_features))
-        elif covar_type == 'tied':
-            assert_almost_equal(np.dot(bgmm.covariances_, bgmm.precisions_),
-                                np.eye(n_features))
-
-        elif covar_type == 'diag':
-            assert_almost_equal(bgmm.covariances_ * bgmm.precisions_,
-                                np.ones((n_components, n_features)))
+                assert_almost_equal(np.dot(covar, precision), np.eye(n_features))
+        elif covar_type == "tied":
+            assert_almost_equal(
+                np.dot(bgmm.covariances_, bgmm.precisions_), np.eye(n_features)
+            )
+
+        elif covar_type == "diag":
+            assert_almost_equal(
+                bgmm.covariances_ * bgmm.precisions_,
+                np.ones((n_components, n_features)),
+            )
 
         else:
-            assert_almost_equal(bgmm.covariances_ * bgmm.precisions_,
-                                np.ones(n_components))
+            assert_almost_equal(
+                bgmm.covariances_ * bgmm.precisions_, np.ones(n_components)
+            )
 
 
-@ignore_warnings(category=ConvergenceWarning)
 def test_invariant_translation():
     # We check here that adding a constant in the data change correctly the
     # parameters of the mixture
@@ -414,12 +372,20 @@ def test_invariant_translation():
             X = rand_data.X[covar_type]
             bgmm1 = BayesianGaussianMixture(
                 weight_concentration_prior_type=prior_type,
-                n_components=n_components, max_iter=100, random_state=0,
-                tol=1e-3, reg_covar=0).fit(X)
+                n_components=n_components,
+                max_iter=100,
+                random_state=0,
+                tol=1e-3,
+                reg_covar=0,
+            ).fit(X)
             bgmm2 = BayesianGaussianMixture(
                 weight_concentration_prior_type=prior_type,
-                n_components=n_components, max_iter=100, random_state=0,
-                tol=1e-3, reg_covar=0).fit(X + 100)
+                n_components=n_components,
+                max_iter=100,
+                random_state=0,
+                tol=1e-3,
+                reg_covar=0,
+            ).fit(X + 100)
 
             assert_almost_equal(bgmm1.means_, bgmm2.means_ - 100)
             assert_almost_equal(bgmm1.weights_, bgmm2.weights_)
@@ -427,21 +393,28 @@ def test_invariant_translation():
 
 
 @pytest.mark.filterwarnings("ignore:.*did not converge.*")
-@pytest.mark.parametrize('seed, max_iter, tol', [
-    (0, 2, 1e-7),    # strict non-convergence
-    (1, 2, 1e-1),    # loose non-convergence
-    (3, 300, 1e-7),  # strict convergence
-    (4, 300, 1e-1),  # loose convergence
-])
+@pytest.mark.parametrize(
+    "seed, max_iter, tol",
+    [
+        (0, 2, 1e-7),  # strict non-convergence
+        (1, 2, 1e-1),  # loose non-convergence
+        (3, 300, 1e-7),  # strict convergence
+        (4, 300, 1e-1),  # loose convergence
+    ],
+)
 def test_bayesian_mixture_fit_predict(seed, max_iter, tol):
     rng = np.random.RandomState(seed)
     rand_data = RandomData(rng, n_samples=50, scale=7)
     n_components = 2 * rand_data.n_components
 
     for covar_type in COVARIANCE_TYPE:
-        bgmm1 = BayesianGaussianMixture(n_components=n_components,
-                                        max_iter=max_iter, random_state=rng,
-                                        tol=tol, reg_covar=0)
+        bgmm1 = BayesianGaussianMixture(
+            n_components=n_components,
+            max_iter=max_iter,
+            random_state=rng,
+            tol=tol,
+            reg_covar=0,
+        )
         bgmm1.covariance_type = covar_type
         bgmm2 = copy.deepcopy(bgmm1)
         X = rand_data.X[covar_type]
@@ -472,17 +445,20 @@ def test_bayesian_mixture_predict_predict_proba():
                 n_components=rand_data.n_components,
                 random_state=rng,
                 weight_concentration_prior_type=prior_type,
-                covariance_type=covar_type)
+                covariance_type=covar_type,
+            )
 
             # Check a warning message arrive if we don't do fit
-            assert_raise_message(NotFittedError,
-                                 "This BayesianGaussianMixture instance"
-                                 " is not fitted yet. Call 'fit' with "
-                                 "appropriate arguments before using "
-                                 "this estimator.", bgmm.predict, X)
+            msg = (
+                "This BayesianGaussianMixture instance is not fitted yet. "
+                "Call 'fit' with appropriate arguments before using this "
+                "estimator."
+            )
+            with pytest.raises(NotFittedError, match=msg):
+                bgmm.predict(X)
 
             bgmm.fit(X)
             Y_pred = bgmm.predict(X)
             Y_pred_proba = bgmm.predict_proba(X).argmax(axis=1)
             assert_array_equal(Y_pred, Y_pred_proba)
-            assert adjusted_rand_score(Y, Y_pred) >= .95
+            assert adjusted_rand_score(Y, Y_pred) >= 0.95
diff --git a/sklearn/mixture/tests/test_gaussian_mixture.py b/sklearn/mixture/tests/test_gaussian_mixture.py
index 403fbb2208618..488a2ab147e83 100644
--- a/sklearn/mixture/tests/test_gaussian_mixture.py
+++ b/sklearn/mixture/tests/test_gaussian_mixture.py
@@ -1,101 +1,144 @@
-# Author: Wei Xue 
-#         Thierry Guillemot 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-import sys
 import copy
+import itertools
+import re
+import sys
 import warnings
-import pytest
+from io import StringIO
+from unittest.mock import Mock
 
 import numpy as np
-from scipy import stats, linalg
+import pytest
+from scipy import linalg, stats
 
+import sklearn
+from sklearn.cluster import KMeans
 from sklearn.covariance import EmpiricalCovariance
 from sklearn.datasets import make_spd_matrix
-from io import StringIO
+from sklearn.exceptions import ConvergenceWarning, NotFittedError
 from sklearn.metrics.cluster import adjusted_rand_score
 from sklearn.mixture import GaussianMixture
 from sklearn.mixture._gaussian_mixture import (
-    _estimate_gaussian_covariances_full,
-    _estimate_gaussian_covariances_tied,
+    _compute_log_det_cholesky,
+    _compute_precision_cholesky,
     _estimate_gaussian_covariances_diag,
+    _estimate_gaussian_covariances_full,
     _estimate_gaussian_covariances_spherical,
-    _compute_precision_cholesky,
-    _compute_log_det_cholesky,
-    )
-from sklearn.exceptions import ConvergenceWarning, NotFittedError
+    _estimate_gaussian_covariances_tied,
+    _estimate_gaussian_parameters,
+)
+from sklearn.utils._testing import (
+    assert_allclose,
+    assert_almost_equal,
+    assert_array_almost_equal,
+    assert_array_equal,
+)
 from sklearn.utils.extmath import fast_logdet
-from sklearn.utils._testing import assert_allclose
-from sklearn.utils._testing import assert_almost_equal
-from sklearn.utils._testing import assert_array_almost_equal
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils._testing import assert_raise_message
-from sklearn.utils._testing import assert_warns_message
-from sklearn.utils._testing import ignore_warnings
 
+COVARIANCE_TYPE = ["full", "tied", "diag", "spherical"]
 
-COVARIANCE_TYPE = ['full', 'tied', 'diag', 'spherical']
 
-
-def generate_data(n_samples, n_features, weights, means, precisions,
-                  covariance_type):
+def generate_data(
+    n_samples, n_features, weights, means, precisions, covariance_type, dtype=np.float64
+):
     rng = np.random.RandomState(0)
 
     X = []
-    if covariance_type == 'spherical':
-        for _, (w, m, c) in enumerate(zip(weights, means,
-                                          precisions['spherical'])):
-            X.append(rng.multivariate_normal(m, c * np.eye(n_features),
-                                             int(np.round(w * n_samples))))
-    if covariance_type == 'diag':
-        for _, (w, m, c) in enumerate(zip(weights, means,
-                                          precisions['diag'])):
-            X.append(rng.multivariate_normal(m, np.diag(c),
-                                             int(np.round(w * n_samples))))
-    if covariance_type == 'tied':
+    if covariance_type == "spherical":
+        for _, (w, m, c) in enumerate(zip(weights, means, precisions["spherical"])):
+            X.append(
+                rng.multivariate_normal(
+                    m, c * np.eye(n_features), int(np.round(w * n_samples))
+                ).astype(dtype)
+            )
+    if covariance_type == "diag":
+        for _, (w, m, c) in enumerate(zip(weights, means, precisions["diag"])):
+            X.append(
+                rng.multivariate_normal(
+                    m, np.diag(c), int(np.round(w * n_samples))
+                ).astype(dtype)
+            )
+    if covariance_type == "tied":
         for _, (w, m) in enumerate(zip(weights, means)):
-            X.append(rng.multivariate_normal(m, precisions['tied'],
-                                             int(np.round(w * n_samples))))
-    if covariance_type == 'full':
-        for _, (w, m, c) in enumerate(zip(weights, means,
-                                          precisions['full'])):
-            X.append(rng.multivariate_normal(m, c,
-                                             int(np.round(w * n_samples))))
+            X.append(
+                rng.multivariate_normal(
+                    m, precisions["tied"], int(np.round(w * n_samples))
+                ).astype(dtype)
+            )
+    if covariance_type == "full":
+        for _, (w, m, c) in enumerate(zip(weights, means, precisions["full"])):
+            X.append(
+                rng.multivariate_normal(m, c, int(np.round(w * n_samples))).astype(
+                    dtype
+                )
+            )
 
     X = np.vstack(X)
     return X
 
 
 class RandomData:
-    def __init__(self, rng, n_samples=200, n_components=2, n_features=2,
-                 scale=50):
+    def __init__(
+        self,
+        rng,
+        n_samples=200,
+        n_components=2,
+        n_features=2,
+        scale=50,
+        dtype=np.float64,
+    ):
         self.n_samples = n_samples
         self.n_components = n_components
         self.n_features = n_features
 
-        self.weights = rng.rand(n_components)
-        self.weights = self.weights / self.weights.sum()
-        self.means = rng.rand(n_components, n_features) * scale
+        self.weights = rng.rand(n_components).astype(dtype)
+        self.weights = self.weights.astype(dtype) / self.weights.sum()
+        self.means = rng.rand(n_components, n_features).astype(dtype) * scale
         self.covariances = {
-            'spherical': .5 + rng.rand(n_components),
-            'diag': (.5 + rng.rand(n_components, n_features)) ** 2,
-            'tied': make_spd_matrix(n_features, random_state=rng),
-            'full': np.array([
-                make_spd_matrix(n_features, random_state=rng) * .5
-                for _ in range(n_components)])}
+            "spherical": 0.5 + rng.rand(n_components).astype(dtype),
+            "diag": (0.5 + rng.rand(n_components, n_features).astype(dtype)) ** 2,
+            "tied": make_spd_matrix(n_features, random_state=rng).astype(dtype),
+            "full": np.array(
+                [
+                    make_spd_matrix(n_features, random_state=rng).astype(dtype) * 0.5
+                    for _ in range(n_components)
+                ]
+            ),
+        }
         self.precisions = {
-            'spherical': 1. / self.covariances['spherical'],
-            'diag': 1. / self.covariances['diag'],
-            'tied': linalg.inv(self.covariances['tied']),
-            'full': np.array([linalg.inv(covariance)
-                             for covariance in self.covariances['full']])}
-
-        self.X = dict(zip(COVARIANCE_TYPE, [generate_data(
-            n_samples, n_features, self.weights, self.means, self.covariances,
-            covar_type) for covar_type in COVARIANCE_TYPE]))
-        self.Y = np.hstack([np.full(int(np.round(w * n_samples)), k,
-                                    dtype=int)
-                            for k, w in enumerate(self.weights)])
+            "spherical": 1.0 / self.covariances["spherical"],
+            "diag": 1.0 / self.covariances["diag"],
+            "tied": linalg.inv(self.covariances["tied"]),
+            "full": np.array(
+                [linalg.inv(covariance) for covariance in self.covariances["full"]]
+            ),
+        }
+
+        self.X = dict(
+            zip(
+                COVARIANCE_TYPE,
+                [
+                    generate_data(
+                        n_samples,
+                        n_features,
+                        self.weights,
+                        self.means,
+                        self.covariances,
+                        covar_type,
+                        dtype=dtype,
+                    )
+                    for covar_type in COVARIANCE_TYPE
+                ],
+            )
+        )
+        self.Y = np.hstack(
+            [
+                np.full(int(np.round(w * n_samples)), k, dtype=int)
+                for k, w in enumerate(self.weights)
+            ]
+        )
 
 
 def test_gaussian_mixture_attributes():
@@ -103,65 +146,18 @@ def test_gaussian_mixture_attributes():
     rng = np.random.RandomState(0)
     X = rng.rand(10, 2)
 
-    n_components_bad = 0
-    gmm = GaussianMixture(n_components=n_components_bad)
-    assert_raise_message(ValueError,
-                         "Invalid value for 'n_components': %d "
-                         "Estimation requires at least one component"
-                         % n_components_bad, gmm.fit, X)
-
-    # covariance_type should be in [spherical, diag, tied, full]
-    covariance_type_bad = 'bad_covariance_type'
-    gmm = GaussianMixture(covariance_type=covariance_type_bad)
-    assert_raise_message(ValueError,
-                         "Invalid value for 'covariance_type': %s "
-                         "'covariance_type' should be in "
-                         "['spherical', 'tied', 'diag', 'full']"
-                         % covariance_type_bad,
-                         gmm.fit, X)
-
-    tol_bad = -1
-    gmm = GaussianMixture(tol=tol_bad)
-    assert_raise_message(ValueError,
-                         "Invalid value for 'tol': %.5f "
-                         "Tolerance used by the EM must be non-negative"
-                         % tol_bad, gmm.fit, X)
-
-    reg_covar_bad = -1
-    gmm = GaussianMixture(reg_covar=reg_covar_bad)
-    assert_raise_message(ValueError,
-                         "Invalid value for 'reg_covar': %.5f "
-                         "regularization on covariance must be "
-                         "non-negative" % reg_covar_bad, gmm.fit, X)
-
-    max_iter_bad = 0
-    gmm = GaussianMixture(max_iter=max_iter_bad)
-    assert_raise_message(ValueError,
-                         "Invalid value for 'max_iter': %d "
-                         "Estimation requires at least one iteration"
-                         % max_iter_bad, gmm.fit, X)
-
-    n_init_bad = 0
-    gmm = GaussianMixture(n_init=n_init_bad)
-    assert_raise_message(ValueError,
-                         "Invalid value for 'n_init': %d "
-                         "Estimation requires at least one run"
-                         % n_init_bad, gmm.fit, X)
-
-    init_params_bad = 'bad_method'
-    gmm = GaussianMixture(init_params=init_params_bad)
-    assert_raise_message(ValueError,
-                         "Unimplemented initialization method '%s'"
-                         % init_params_bad,
-                         gmm.fit, X)
-
     # test good parameters
     n_components, tol, n_init, max_iter, reg_covar = 2, 1e-4, 3, 30, 1e-1
-    covariance_type, init_params = 'full', 'random'
-    gmm = GaussianMixture(n_components=n_components, tol=tol, n_init=n_init,
-                          max_iter=max_iter, reg_covar=reg_covar,
-                          covariance_type=covariance_type,
-                          init_params=init_params).fit(X)
+    covariance_type, init_params = "full", "random"
+    gmm = GaussianMixture(
+        n_components=n_components,
+        tol=tol,
+        n_init=n_init,
+        max_iter=max_iter,
+        reg_covar=reg_covar,
+        covariance_type=covariance_type,
+        init_params=init_params,
+    ).fit(X)
 
     assert gmm.n_components == n_components
     assert gmm.covariance_type == covariance_type
@@ -172,67 +168,46 @@ def test_gaussian_mixture_attributes():
     assert gmm.init_params == init_params
 
 
-def test_check_X():
-    from sklearn.mixture._base import _check_X
-    rng = np.random.RandomState(0)
-
-    n_samples, n_components, n_features = 10, 2, 2
-
-    X_bad_dim = rng.rand(n_components - 1, n_features)
-    assert_raise_message(ValueError,
-                         'Expected n_samples >= n_components '
-                         'but got n_components = %d, n_samples = %d'
-                         % (n_components, X_bad_dim.shape[0]),
-                         _check_X, X_bad_dim, n_components)
-
-    X_bad_dim = rng.rand(n_components, n_features + 1)
-    assert_raise_message(ValueError,
-                         'Expected the input data X have %d features, '
-                         'but got %d features'
-                         % (n_features, X_bad_dim.shape[1]),
-                         _check_X, X_bad_dim, n_components, n_features)
-
-    X = rng.rand(n_samples, n_features)
-    assert_array_equal(X, _check_X(X, n_components, n_features))
-
-
 def test_check_weights():
     rng = np.random.RandomState(0)
     rand_data = RandomData(rng)
 
     n_components = rand_data.n_components
-    X = rand_data.X['full']
+    X = rand_data.X["full"]
 
     g = GaussianMixture(n_components=n_components)
 
     # Check bad shape
     weights_bad_shape = rng.rand(n_components, 1)
     g.weights_init = weights_bad_shape
-    assert_raise_message(ValueError,
-                         "The parameter 'weights' should have the shape of "
-                         "(%d,), but got %s" %
-                         (n_components, str(weights_bad_shape.shape)),
-                         g.fit, X)
+    msg = re.escape(
+        "The parameter 'weights' should have the shape of "
+        f"({n_components},), but got {weights_bad_shape.shape}"
+    )
+    with pytest.raises(ValueError, match=msg):
+        g.fit(X)
 
     # Check bad range
     weights_bad_range = rng.rand(n_components) + 1
     g.weights_init = weights_bad_range
-    assert_raise_message(ValueError,
-                         "The parameter 'weights' should be in the range "
-                         "[0, 1], but got max value %.5f, min value %.5f"
-                         % (np.min(weights_bad_range),
-                            np.max(weights_bad_range)),
-                         g.fit, X)
+    msg = re.escape(
+        "The parameter 'weights' should be in the range [0, 1], but got"
+        f" max value {np.min(weights_bad_range):.5f}, "
+        f"min value {np.max(weights_bad_range):.5f}"
+    )
+    with pytest.raises(ValueError, match=msg):
+        g.fit(X)
 
     # Check bad normalization
     weights_bad_norm = rng.rand(n_components)
     weights_bad_norm = weights_bad_norm / (weights_bad_norm.sum() + 1)
     g.weights_init = weights_bad_norm
-    assert_raise_message(ValueError,
-                         "The parameter 'weights' should be normalized, "
-                         "but got sum(weights) = %.5f"
-                         % np.sum(weights_bad_norm),
-                         g.fit, X)
+    msg = re.escape(
+        "The parameter 'weights' should be normalized, "
+        f"but got sum(weights) = {np.sum(weights_bad_norm):.5f}"
+    )
+    with pytest.raises(ValueError, match=msg):
+        g.fit(X)
 
     # Check good weights matrix
     weights = rand_data.weights
@@ -246,16 +221,16 @@ def test_check_means():
     rand_data = RandomData(rng)
 
     n_components, n_features = rand_data.n_components, rand_data.n_features
-    X = rand_data.X['full']
+    X = rand_data.X["full"]
 
     g = GaussianMixture(n_components=n_components)
 
     # Check means bad shape
     means_bad_shape = rng.rand(n_components + 1, n_features)
     g.means_init = means_bad_shape
-    assert_raise_message(ValueError,
-                         "The parameter 'means' should have the shape of ",
-                         g.fit, X)
+    msg = "The parameter 'means' should have the shape of "
+    with pytest.raises(ValueError, match=msg):
+        g.fit(X)
 
     # Check good means matrix
     means = rand_data.means
@@ -272,47 +247,48 @@ def test_check_precisions():
 
     # Define the bad precisions for each covariance_type
     precisions_bad_shape = {
-        'full': np.ones((n_components + 1, n_features, n_features)),
-        'tied': np.ones((n_features + 1, n_features + 1)),
-        'diag': np.ones((n_components + 1, n_features)),
-        'spherical': np.ones((n_components + 1))}
+        "full": np.ones((n_components + 1, n_features, n_features)),
+        "tied": np.ones((n_features + 1, n_features + 1)),
+        "diag": np.ones((n_components + 1, n_features)),
+        "spherical": np.ones((n_components + 1)),
+    }
 
     # Define not positive-definite precisions
     precisions_not_pos = np.ones((n_components, n_features, n_features))
     precisions_not_pos[0] = np.eye(n_features)
-    precisions_not_pos[0, 0, 0] = -1.
+    precisions_not_pos[0, 0, 0] = -1.0
 
     precisions_not_positive = {
-        'full': precisions_not_pos,
-        'tied': precisions_not_pos[0],
-        'diag': np.full((n_components, n_features), -1.),
-        'spherical': np.full(n_components, -1.)}
+        "full": precisions_not_pos,
+        "tied": precisions_not_pos[0],
+        "diag": np.full((n_components, n_features), -1.0),
+        "spherical": np.full(n_components, -1.0),
+    }
 
     not_positive_errors = {
-        'full': 'symmetric, positive-definite',
-        'tied': 'symmetric, positive-definite',
-        'diag': 'positive',
-        'spherical': 'positive'}
+        "full": "symmetric, positive-definite",
+        "tied": "symmetric, positive-definite",
+        "diag": "positive",
+        "spherical": "positive",
+    }
 
     for covar_type in COVARIANCE_TYPE:
         X = RandomData(rng).X[covar_type]
-        g = GaussianMixture(n_components=n_components,
-                            covariance_type=covar_type,
-                            random_state=rng)
+        g = GaussianMixture(
+            n_components=n_components, covariance_type=covar_type, random_state=rng
+        )
 
         # Check precisions with bad shapes
         g.precisions_init = precisions_bad_shape[covar_type]
-        assert_raise_message(ValueError,
-                             "The parameter '%s precision' should have "
-                             "the shape of" % covar_type,
-                             g.fit, X)
+        msg = f"The parameter '{covar_type} precision' should have the shape of"
+        with pytest.raises(ValueError, match=msg):
+            g.fit(X)
 
         # Check not positive precisions
         g.precisions_init = precisions_not_positive[covar_type]
-        assert_raise_message(ValueError,
-                             "'%s precision' should be %s"
-                             % (covar_type, not_positive_errors[covar_type]),
-                             g.fit, X)
+        msg = f"'{covar_type} precision' should be {not_positive_errors[covar_type]}"
+        with pytest.raises(ValueError, match=msg):
+            g.fit(X)
 
         # Check the correct init of precisions_init
         g.precisions_init = rand_data.precisions[covar_type]
@@ -336,11 +312,11 @@ def test_suffstat_sk_full():
     covars_pred = _estimate_gaussian_covariances_full(resp, X, nk, xk, 0)
     ecov = EmpiricalCovariance(assume_centered=True)
     ecov.fit(X_resp)
-    assert_almost_equal(ecov.error_norm(covars_pred[0], norm='frobenius'), 0)
-    assert_almost_equal(ecov.error_norm(covars_pred[0], norm='spectral'), 0)
+    assert_almost_equal(ecov.error_norm(covars_pred[0], norm="frobenius"), 0)
+    assert_almost_equal(ecov.error_norm(covars_pred[0], norm="spectral"), 0)
 
     # check the precision computation
-    precs_chol_pred = _compute_precision_cholesky(covars_pred, 'full')
+    precs_chol_pred = _compute_precision_cholesky(covars_pred, "full")
     precs_pred = np.array([np.dot(prec, prec.T) for prec in precs_chol_pred])
     precs_est = np.array([linalg.inv(cov) for cov in covars_pred])
     assert_array_almost_equal(precs_est, precs_pred)
@@ -352,11 +328,11 @@ def test_suffstat_sk_full():
     covars_pred = _estimate_gaussian_covariances_full(resp, X, nk, xk, 0)
     ecov = EmpiricalCovariance(assume_centered=False)
     ecov.fit(X)
-    assert_almost_equal(ecov.error_norm(covars_pred[0], norm='frobenius'), 0)
-    assert_almost_equal(ecov.error_norm(covars_pred[0], norm='spectral'), 0)
+    assert_almost_equal(ecov.error_norm(covars_pred[0], norm="frobenius"), 0)
+    assert_almost_equal(ecov.error_norm(covars_pred[0], norm="spectral"), 0)
 
     # check the precision computation
-    precs_chol_pred = _compute_precision_cholesky(covars_pred, 'full')
+    precs_chol_pred = _compute_precision_cholesky(covars_pred, "full")
     precs_pred = np.array([np.dot(prec, prec.T) for prec in precs_chol_pred])
     precs_est = np.array([linalg.inv(cov) for cov in covars_pred])
     assert_array_almost_equal(precs_est, precs_pred)
@@ -374,18 +350,19 @@ def test_suffstat_sk_tied():
     xk = np.dot(resp.T, X) / nk[:, np.newaxis]
 
     covars_pred_full = _estimate_gaussian_covariances_full(resp, X, nk, xk, 0)
-    covars_pred_full = np.sum(nk[:, np.newaxis, np.newaxis] * covars_pred_full,
-                              0) / n_samples
+    covars_pred_full = (
+        np.sum(nk[:, np.newaxis, np.newaxis] * covars_pred_full, 0) / n_samples
+    )
 
     covars_pred_tied = _estimate_gaussian_covariances_tied(resp, X, nk, xk, 0)
 
     ecov = EmpiricalCovariance()
     ecov.covariance_ = covars_pred_full
-    assert_almost_equal(ecov.error_norm(covars_pred_tied, norm='frobenius'), 0)
-    assert_almost_equal(ecov.error_norm(covars_pred_tied, norm='spectral'), 0)
+    assert_almost_equal(ecov.error_norm(covars_pred_tied, norm="frobenius"), 0)
+    assert_almost_equal(ecov.error_norm(covars_pred_tied, norm="spectral"), 0)
 
     # check the precision computation
-    precs_chol_pred = _compute_precision_cholesky(covars_pred_tied, 'tied')
+    precs_chol_pred = _compute_precision_cholesky(covars_pred_tied, "tied")
     precs_pred = np.dot(precs_chol_pred, precs_chol_pred.T)
     precs_est = linalg.inv(covars_pred_tied)
     assert_array_almost_equal(precs_est, precs_pred)
@@ -405,60 +382,66 @@ def test_suffstat_sk_diag():
     covars_pred_diag = _estimate_gaussian_covariances_diag(resp, X, nk, xk, 0)
 
     ecov = EmpiricalCovariance()
-    for (cov_full, cov_diag) in zip(covars_pred_full, covars_pred_diag):
+    for cov_full, cov_diag in zip(covars_pred_full, covars_pred_diag):
         ecov.covariance_ = np.diag(np.diag(cov_full))
         cov_diag = np.diag(cov_diag)
-        assert_almost_equal(ecov.error_norm(cov_diag, norm='frobenius'), 0)
-        assert_almost_equal(ecov.error_norm(cov_diag, norm='spectral'), 0)
+        assert_almost_equal(ecov.error_norm(cov_diag, norm="frobenius"), 0)
+        assert_almost_equal(ecov.error_norm(cov_diag, norm="spectral"), 0)
 
     # check the precision computation
-    precs_chol_pred = _compute_precision_cholesky(covars_pred_diag, 'diag')
-    assert_almost_equal(covars_pred_diag, 1. / precs_chol_pred ** 2)
+    precs_chol_pred = _compute_precision_cholesky(covars_pred_diag, "diag")
+    assert_almost_equal(covars_pred_diag, 1.0 / precs_chol_pred**2)
 
 
-def test_gaussian_suffstat_sk_spherical():
+def test_gaussian_suffstat_sk_spherical(global_dtype):
     # computing spherical covariance equals to the variance of one-dimension
     # data after flattening, n_components=1
     rng = np.random.RandomState(0)
     n_samples, n_features = 500, 2
 
-    X = rng.rand(n_samples, n_features)
+    X = rng.rand(n_samples, n_features).astype(global_dtype)
     X = X - X.mean()
-    resp = np.ones((n_samples, 1))
-    nk = np.array([n_samples])
+    resp = np.ones((n_samples, 1), dtype=global_dtype)
+    nk = np.array([n_samples], dtype=global_dtype)
     xk = X.mean()
-    covars_pred_spherical = _estimate_gaussian_covariances_spherical(resp, X,
-                                                                     nk, xk, 0)
-    covars_pred_spherical2 = (np.dot(X.flatten().T, X.flatten()) /
-                              (n_features * n_samples))
+    covars_pred_spherical = _estimate_gaussian_covariances_spherical(resp, X, nk, xk, 0)
+    covars_pred_spherical2 = np.dot(X.flatten().T, X.flatten()) / (
+        n_features * n_samples
+    )
     assert_almost_equal(covars_pred_spherical, covars_pred_spherical2)
+    assert covars_pred_spherical.dtype == global_dtype
 
     # check the precision computation
-    precs_chol_pred = _compute_precision_cholesky(covars_pred_spherical,
-                                                  'spherical')
-    assert_almost_equal(covars_pred_spherical, 1. / precs_chol_pred ** 2)
+    precs_chol_pred = _compute_precision_cholesky(covars_pred_spherical, "spherical")
+    assert_almost_equal(covars_pred_spherical, 1.0 / precs_chol_pred**2)
+    assert precs_chol_pred.dtype == global_dtype
 
 
-def test_compute_log_det_cholesky():
+def test_compute_log_det_cholesky(global_dtype):
     n_features = 2
-    rand_data = RandomData(np.random.RandomState(0))
+    rand_data = RandomData(np.random.RandomState(0), dtype=global_dtype)
 
     for covar_type in COVARIANCE_TYPE:
         covariance = rand_data.covariances[covar_type]
 
-        if covar_type == 'full':
+        if covar_type == "full":
             predected_det = np.array([linalg.det(cov) for cov in covariance])
-        elif covar_type == 'tied':
+        elif covar_type == "tied":
             predected_det = linalg.det(covariance)
-        elif covar_type == 'diag':
+        elif covar_type == "diag":
             predected_det = np.array([np.prod(cov) for cov in covariance])
-        elif covar_type == 'spherical':
-            predected_det = covariance ** n_features
+        elif covar_type == "spherical":
+            predected_det = covariance**n_features
 
         # We compute the cholesky decomposition of the covariance matrix
-        expected_det = _compute_log_det_cholesky(_compute_precision_cholesky(
-            covariance, covar_type), covar_type, n_features=n_features)
-        assert_array_almost_equal(expected_det, - .5 * np.log(predected_det))
+        assert covariance.dtype == global_dtype
+        expected_det = _compute_log_det_cholesky(
+            _compute_precision_cholesky(covariance, covar_type),
+            covar_type,
+            n_features=n_features,
+        )
+        assert_array_almost_equal(expected_det, -0.5 * np.log(predected_det))
+        assert expected_det.dtype == global_dtype
 
 
 def _naive_lmvnpdf_diag(X, means, covars):
@@ -485,36 +468,35 @@ def test_gaussian_mixture_log_probabilities():
     log_prob_naive = _naive_lmvnpdf_diag(X, means, covars_diag)
 
     # full covariances
-    precs_full = np.array([np.diag(1. / np.sqrt(x)) for x in covars_diag])
+    precs_full = np.array([np.diag(1.0 / np.sqrt(x)) for x in covars_diag])
 
-    log_prob = _estimate_log_gaussian_prob(X, means, precs_full, 'full')
+    log_prob = _estimate_log_gaussian_prob(X, means, precs_full, "full")
     assert_array_almost_equal(log_prob, log_prob_naive)
 
     # diag covariances
-    precs_chol_diag = 1. / np.sqrt(covars_diag)
-    log_prob = _estimate_log_gaussian_prob(X, means, precs_chol_diag, 'diag')
+    precs_chol_diag = 1.0 / np.sqrt(covars_diag)
+    log_prob = _estimate_log_gaussian_prob(X, means, precs_chol_diag, "diag")
     assert_array_almost_equal(log_prob, log_prob_naive)
 
     # tied
     covars_tied = np.array([x for x in covars_diag]).mean(axis=0)
-    precs_tied = np.diag(np.sqrt(1. / covars_tied))
+    precs_tied = np.diag(np.sqrt(1.0 / covars_tied))
 
-    log_prob_naive = _naive_lmvnpdf_diag(X, means,
-                                         [covars_tied] * n_components)
-    log_prob = _estimate_log_gaussian_prob(X, means, precs_tied, 'tied')
+    log_prob_naive = _naive_lmvnpdf_diag(X, means, [covars_tied] * n_components)
+    log_prob = _estimate_log_gaussian_prob(X, means, precs_tied, "tied")
 
     assert_array_almost_equal(log_prob, log_prob_naive)
 
     # spherical
     covars_spherical = covars_diag.mean(axis=1)
-    precs_spherical = 1. / np.sqrt(covars_diag.mean(axis=1))
-    log_prob_naive = _naive_lmvnpdf_diag(X, means,
-                                         [[k] * n_features for k in
-                                          covars_spherical])
-    log_prob = _estimate_log_gaussian_prob(X, means,
-                                           precs_spherical, 'spherical')
+    precs_spherical = 1.0 / np.sqrt(covars_diag.mean(axis=1))
+    log_prob_naive = _naive_lmvnpdf_diag(
+        X, means, [[k] * n_features for k in covars_spherical]
+    )
+    log_prob = _estimate_log_gaussian_prob(X, means, precs_spherical, "spherical")
     assert_array_almost_equal(log_prob, log_prob_naive)
 
+
 # skip tests on weighted_log_probabilities, log_weights
 
 
@@ -531,10 +513,14 @@ def test_gaussian_mixture_estimate_log_prob_resp():
         weights = rand_data.weights
         means = rand_data.means
         precisions = rand_data.precisions[covar_type]
-        g = GaussianMixture(n_components=n_components, random_state=rng,
-                            weights_init=weights, means_init=means,
-                            precisions_init=precisions,
-                            covariance_type=covar_type)
+        g = GaussianMixture(
+            n_components=n_components,
+            random_state=rng,
+            weights_init=weights,
+            means_init=means,
+            precisions_init=precisions,
+            covariance_type=covar_type,
+        )
         g.fit(X)
         resp = g.predict_proba(X)
         assert_array_almost_equal(resp.sum(axis=1), np.ones(n_samples))
@@ -549,51 +535,66 @@ def test_gaussian_mixture_predict_predict_proba():
     for covar_type in COVARIANCE_TYPE:
         X = rand_data.X[covar_type]
         Y = rand_data.Y
-        g = GaussianMixture(n_components=rand_data.n_components,
-                            random_state=rng, weights_init=rand_data.weights,
-                            means_init=rand_data.means,
-                            precisions_init=rand_data.precisions[covar_type],
-                            covariance_type=covar_type)
+        g = GaussianMixture(
+            n_components=rand_data.n_components,
+            random_state=rng,
+            weights_init=rand_data.weights,
+            means_init=rand_data.means,
+            precisions_init=rand_data.precisions[covar_type],
+            covariance_type=covar_type,
+        )
 
         # Check a warning message arrive if we don't do fit
-        assert_raise_message(NotFittedError,
-                             "This GaussianMixture instance is not fitted "
-                             "yet. Call 'fit' with appropriate arguments "
-                             "before using this estimator.", g.predict, X)
+        msg = (
+            "This GaussianMixture instance is not fitted yet. Call 'fit' "
+            "with appropriate arguments before using this estimator."
+        )
+        with pytest.raises(NotFittedError, match=msg):
+            g.predict(X)
 
         g.fit(X)
         Y_pred = g.predict(X)
         Y_pred_proba = g.predict_proba(X).argmax(axis=1)
         assert_array_equal(Y_pred, Y_pred_proba)
-        assert adjusted_rand_score(Y, Y_pred) > .95
+        assert adjusted_rand_score(Y, Y_pred) > 0.95
 
 
 @pytest.mark.filterwarnings("ignore:.*did not converge.*")
-@pytest.mark.parametrize('seed, max_iter, tol', [
-    (0, 2, 1e-7),    # strict non-convergence
-    (1, 2, 1e-1),    # loose non-convergence
-    (3, 300, 1e-7),  # strict convergence
-    (4, 300, 1e-1),  # loose convergence
-])
-def test_gaussian_mixture_fit_predict(seed, max_iter, tol):
+@pytest.mark.parametrize(
+    "seed, max_iter, tol",
+    [
+        (0, 2, 1e-7),  # strict non-convergence
+        (1, 2, 1e-1),  # loose non-convergence
+        (3, 300, 1e-7),  # strict convergence
+        (4, 300, 1e-1),  # loose convergence
+    ],
+)
+def test_gaussian_mixture_fit_predict(seed, max_iter, tol, global_dtype):
     rng = np.random.RandomState(seed)
-    rand_data = RandomData(rng)
+    rand_data = RandomData(rng, dtype=global_dtype)
     for covar_type in COVARIANCE_TYPE:
         X = rand_data.X[covar_type]
         Y = rand_data.Y
-        g = GaussianMixture(n_components=rand_data.n_components,
-                            random_state=rng, weights_init=rand_data.weights,
-                            means_init=rand_data.means,
-                            precisions_init=rand_data.precisions[covar_type],
-                            covariance_type=covar_type,
-                            max_iter=max_iter, tol=tol)
+        g = GaussianMixture(
+            n_components=rand_data.n_components,
+            random_state=rng,
+            weights_init=rand_data.weights,
+            means_init=rand_data.means,
+            precisions_init=rand_data.precisions[covar_type],
+            covariance_type=covar_type,
+            max_iter=max_iter,
+            tol=tol,
+        )
 
         # check if fit_predict(X) is equivalent to fit(X).predict(X)
         f = copy.deepcopy(g)
         Y_pred1 = f.fit(X).predict(X)
         Y_pred2 = g.fit_predict(X)
         assert_array_equal(Y_pred1, Y_pred2)
-        assert adjusted_rand_score(Y, Y_pred2) > .95
+        assert adjusted_rand_score(Y, Y_pred2) > 0.95
+        assert g.means_.dtype == global_dtype
+        assert g.weights_.dtype == global_dtype
+        assert g.precisions_.dtype == global_dtype
 
 
 def test_gaussian_mixture_fit_predict_n_init():
@@ -605,44 +606,49 @@ def test_gaussian_mixture_fit_predict_n_init():
     assert_array_equal(y_pred1, y_pred2)
 
 
-def test_gaussian_mixture_fit():
+def test_gaussian_mixture_fit(global_dtype):
     # recover the ground truth
     rng = np.random.RandomState(0)
-    rand_data = RandomData(rng)
+    rand_data = RandomData(rng, dtype=global_dtype)
     n_features = rand_data.n_features
     n_components = rand_data.n_components
 
     for covar_type in COVARIANCE_TYPE:
         X = rand_data.X[covar_type]
-        g = GaussianMixture(n_components=n_components, n_init=20,
-                            reg_covar=0, random_state=rng,
-                            covariance_type=covar_type)
+        g = GaussianMixture(
+            n_components=n_components,
+            n_init=20,
+            reg_covar=0,
+            random_state=rng,
+            covariance_type=covar_type,
+        )
         g.fit(X)
 
         # needs more data to pass the test with rtol=1e-7
-        assert_allclose(np.sort(g.weights_), np.sort(rand_data.weights),
-                        rtol=0.1, atol=1e-2)
+        assert_allclose(
+            np.sort(g.weights_), np.sort(rand_data.weights), rtol=0.1, atol=1e-2
+        )
 
         arg_idx1 = g.means_[:, 0].argsort()
         arg_idx2 = rand_data.means[:, 0].argsort()
-        assert_allclose(g.means_[arg_idx1], rand_data.means[arg_idx2],
-                        rtol=0.1, atol=1e-2)
+        assert_allclose(
+            g.means_[arg_idx1], rand_data.means[arg_idx2], rtol=0.1, atol=1e-2
+        )
 
-        if covar_type == 'full':
+        if covar_type == "full":
             prec_pred = g.precisions_
-            prec_test = rand_data.precisions['full']
-        elif covar_type == 'tied':
+            prec_test = rand_data.precisions["full"]
+        elif covar_type == "tied":
             prec_pred = np.array([g.precisions_] * n_components)
-            prec_test = np.array([rand_data.precisions['tied']] * n_components)
-        elif covar_type == 'spherical':
-            prec_pred = np.array([np.eye(n_features) * c
-                                 for c in g.precisions_])
-            prec_test = np.array([np.eye(n_features) * c for c in
-                                 rand_data.precisions['spherical']])
-        elif covar_type == 'diag':
+            prec_test = np.array([rand_data.precisions["tied"]] * n_components)
+        elif covar_type == "spherical":
+            prec_pred = np.array([np.eye(n_features) * c for c in g.precisions_])
+            prec_test = np.array(
+                [np.eye(n_features) * c for c in rand_data.precisions["spherical"]]
+            )
+        elif covar_type == "diag":
             prec_pred = np.array([np.diag(d) for d in g.precisions_])
-            prec_test = np.array([np.diag(d) for d in
-                                 rand_data.precisions['diag']])
+            prec_test = np.array([np.diag(d) for d in rand_data.precisions["diag"]])
 
         arg_idx1 = np.trace(prec_pred, axis1=1, axis2=2).argsort()
         arg_idx2 = np.trace(prec_test, axis1=1, axis2=2).argsort()
@@ -652,6 +658,10 @@ def test_gaussian_mixture_fit():
             # the accuracy depends on the number of data and randomness, rng
             assert_allclose(ecov.error_norm(prec_pred[k]), 0, atol=0.15)
 
+        assert g.means_.dtype == global_dtype
+        assert g.covariances_.dtype == global_dtype
+        assert g.precisions_.dtype == global_dtype
+
 
 def test_gaussian_mixture_fit_best_params():
     rng = np.random.RandomState(0)
@@ -660,16 +670,25 @@ def test_gaussian_mixture_fit_best_params():
     n_init = 10
     for covar_type in COVARIANCE_TYPE:
         X = rand_data.X[covar_type]
-        g = GaussianMixture(n_components=n_components, n_init=1, reg_covar=0,
-                            random_state=rng, covariance_type=covar_type)
+        g = GaussianMixture(
+            n_components=n_components,
+            n_init=1,
+            reg_covar=0,
+            random_state=rng,
+            covariance_type=covar_type,
+        )
         ll = []
         for _ in range(n_init):
             g.fit(X)
             ll.append(g.score(X))
         ll = np.array(ll)
-        g_best = GaussianMixture(n_components=n_components,
-                                 n_init=n_init, reg_covar=0, random_state=rng,
-                                 covariance_type=covar_type)
+        g_best = GaussianMixture(
+            n_components=n_components,
+            n_init=n_init,
+            reg_covar=0,
+            random_state=rng,
+            covariance_type=covar_type,
+        )
         g_best.fit(X)
         assert_almost_equal(ll.min(), g_best.score(X))
 
@@ -681,15 +700,21 @@ def test_gaussian_mixture_fit_convergence_warning():
     max_iter = 1
     for covar_type in COVARIANCE_TYPE:
         X = rand_data.X[covar_type]
-        g = GaussianMixture(n_components=n_components, n_init=1,
-                            max_iter=max_iter, reg_covar=0, random_state=rng,
-                            covariance_type=covar_type)
-        assert_warns_message(ConvergenceWarning,
-                             'Initialization %d did not converge. '
-                             'Try different init parameters, '
-                             'or increase max_iter, tol '
-                             'or check for degenerate data.'
-                             % max_iter, g.fit, X)
+        g = GaussianMixture(
+            n_components=n_components,
+            n_init=1,
+            max_iter=max_iter,
+            reg_covar=0,
+            random_state=rng,
+            covariance_type=covar_type,
+        )
+        msg = (
+            "Best performing initialization did not converge. "
+            "Try different init parameters, or increase max_iter, "
+            "tol, or check for degenerate data."
+        )
+        with pytest.warns(ConvergenceWarning, match=msg):
+            g.fit(X)
 
 
 def test_multiple_init():
@@ -698,12 +723,23 @@ def test_multiple_init():
     n_samples, n_features, n_components = 50, 5, 2
     X = rng.randn(n_samples, n_features)
     for cv_type in COVARIANCE_TYPE:
-        train1 = GaussianMixture(n_components=n_components,
-                                 covariance_type=cv_type,
-                                 random_state=0).fit(X).score(X)
-        train2 = GaussianMixture(n_components=n_components,
-                                 covariance_type=cv_type,
-                                 random_state=0, n_init=5).fit(X).score(X)
+        train1 = (
+            GaussianMixture(
+                n_components=n_components, covariance_type=cv_type, random_state=0
+            )
+            .fit(X)
+            .score(X)
+        )
+        train2 = (
+            GaussianMixture(
+                n_components=n_components,
+                covariance_type=cv_type,
+                random_state=0,
+                n_init=5,
+            )
+            .fit(X)
+            .score(X)
+        )
         assert train2 >= train1
 
 
@@ -712,11 +748,11 @@ def test_gaussian_mixture_n_parameters():
     rng = np.random.RandomState(0)
     n_samples, n_features, n_components = 50, 5, 2
     X = rng.randn(n_samples, n_features)
-    n_params = {'spherical': 13, 'diag': 21, 'tied': 26, 'full': 41}
+    n_params = {"spherical": 13, "diag": 21, "tied": 26, "full": 41}
     for cv_type in COVARIANCE_TYPE:
         g = GaussianMixture(
-            n_components=n_components, covariance_type=cv_type,
-            random_state=rng).fit(X)
+            n_components=n_components, covariance_type=cv_type, random_state=rng
+        ).fit(X)
         assert g._n_parameters() == n_params[cv_type]
 
 
@@ -726,13 +762,23 @@ def test_bic_1d_1component():
     rng = np.random.RandomState(0)
     n_samples, n_dim, n_components = 100, 1, 1
     X = rng.randn(n_samples, n_dim)
-    bic_full = GaussianMixture(n_components=n_components,
-                               covariance_type='full',
-                               random_state=rng).fit(X).bic(X)
-    for covariance_type in ['tied', 'diag', 'spherical']:
-        bic = GaussianMixture(n_components=n_components,
-                              covariance_type=covariance_type,
-                              random_state=rng).fit(X).bic(X)
+    bic_full = (
+        GaussianMixture(
+            n_components=n_components, covariance_type="full", random_state=rng
+        )
+        .fit(X)
+        .bic(X)
+    )
+    for covariance_type in ["tied", "diag", "spherical"]:
+        bic = (
+            GaussianMixture(
+                n_components=n_components,
+                covariance_type=covariance_type,
+                random_state=rng,
+            )
+            .fit(X)
+            .bic(X)
+        )
         assert_almost_equal(bic_full, bic)
 
 
@@ -742,16 +788,19 @@ def test_gaussian_mixture_aic_bic():
     n_samples, n_features, n_components = 50, 3, 2
     X = rng.randn(n_samples, n_features)
     # standard gaussian entropy
-    sgh = 0.5 * (fast_logdet(np.cov(X.T, bias=1)) +
-                 n_features * (1 + np.log(2 * np.pi)))
+    sgh = 0.5 * (
+        fast_logdet(np.cov(X.T, bias=1)) + n_features * (1 + np.log(2 * np.pi))
+    )
     for cv_type in COVARIANCE_TYPE:
         g = GaussianMixture(
-            n_components=n_components, covariance_type=cv_type,
-            random_state=rng, max_iter=200)
+            n_components=n_components,
+            covariance_type=cv_type,
+            random_state=rng,
+            max_iter=200,
+        )
         g.fit(X)
         aic = 2 * n_samples * sgh + 2 * g._n_parameters()
-        bic = (2 * n_samples * sgh +
-               np.log(n_samples) * g._n_parameters())
+        bic = 2 * n_samples * sgh + np.log(n_samples) * g._n_parameters()
         bound = n_features / np.sqrt(n_samples)
         assert (g.aic(X) - aic) / n_samples < bound
         assert (g.bic(X) - bic) / n_samples < bound
@@ -763,12 +812,22 @@ def test_gaussian_mixture_verbose():
     n_components = rand_data.n_components
     for covar_type in COVARIANCE_TYPE:
         X = rand_data.X[covar_type]
-        g = GaussianMixture(n_components=n_components, n_init=1, reg_covar=0,
-                            random_state=rng, covariance_type=covar_type,
-                            verbose=1)
-        h = GaussianMixture(n_components=n_components, n_init=1, reg_covar=0,
-                            random_state=rng, covariance_type=covar_type,
-                            verbose=2)
+        g = GaussianMixture(
+            n_components=n_components,
+            n_init=1,
+            reg_covar=0,
+            random_state=rng,
+            covariance_type=covar_type,
+            verbose=1,
+        )
+        h = GaussianMixture(
+            n_components=n_components,
+            n_init=1,
+            reg_covar=0,
+            random_state=rng,
+            covariance_type=covar_type,
+            verbose=2,
+        )
         old_stdout = sys.stdout
         sys.stdout = StringIO()
         try:
@@ -778,7 +837,7 @@ def test_gaussian_mixture_verbose():
             sys.stdout = old_stdout
 
 
-@pytest.mark.filterwarnings('ignore:.*did not converge.*')
+@pytest.mark.filterwarnings("ignore:.*did not converge.*")
 @pytest.mark.parametrize("seed", (0, 1, 2))
 def test_warm_start(seed):
     random_state = seed
@@ -787,12 +846,22 @@ def test_warm_start(seed):
     X = rng.rand(n_samples, n_features)
 
     # Assert the warm_start give the same result for the same number of iter
-    g = GaussianMixture(n_components=n_components, n_init=1, max_iter=2,
-                        reg_covar=0, random_state=random_state,
-                        warm_start=False)
-    h = GaussianMixture(n_components=n_components, n_init=1, max_iter=1,
-                        reg_covar=0, random_state=random_state,
-                        warm_start=True)
+    g = GaussianMixture(
+        n_components=n_components,
+        n_init=1,
+        max_iter=2,
+        reg_covar=0,
+        random_state=random_state,
+        warm_start=False,
+    )
+    h = GaussianMixture(
+        n_components=n_components,
+        n_init=1,
+        max_iter=1,
+        reg_covar=0,
+        random_state=random_state,
+        warm_start=True,
+    )
 
     g.fit(X)
     score1 = h.fit(X).score(X)
@@ -804,12 +873,24 @@ def test_warm_start(seed):
     assert score2 > score1
 
     # Assert that by using warm_start we can converge to a good solution
-    g = GaussianMixture(n_components=n_components, n_init=1,
-                        max_iter=5, reg_covar=0, random_state=random_state,
-                        warm_start=False, tol=1e-6)
-    h = GaussianMixture(n_components=n_components, n_init=1,
-                        max_iter=5, reg_covar=0, random_state=random_state,
-                        warm_start=True, tol=1e-6)
+    g = GaussianMixture(
+        n_components=n_components,
+        n_init=1,
+        max_iter=5,
+        reg_covar=0,
+        random_state=random_state,
+        warm_start=False,
+        tol=1e-6,
+    )
+    h = GaussianMixture(
+        n_components=n_components,
+        n_init=1,
+        max_iter=5,
+        reg_covar=0,
+        random_state=random_state,
+        warm_start=True,
+        tol=1e-6,
+    )
 
     g.fit(X)
     assert not g.converged_
@@ -825,17 +906,21 @@ def test_warm_start(seed):
     assert h.converged_
 
 
-@ignore_warnings(category=ConvergenceWarning)
+@pytest.mark.filterwarnings("ignore::sklearn.exceptions.ConvergenceWarning")
 def test_convergence_detected_with_warm_start():
     # We check that convergence is detected when warm_start=True
     rng = np.random.RandomState(0)
     rand_data = RandomData(rng)
     n_components = rand_data.n_components
-    X = rand_data.X['full']
+    X = rand_data.X["full"]
 
     for max_iter in (1, 2, 50):
-        gmm = GaussianMixture(n_components=n_components, warm_start=True,
-                              max_iter=max_iter, random_state=rng)
+        gmm = GaussianMixture(
+            n_components=n_components,
+            warm_start=True,
+            max_iter=max_iter,
+            random_state=rng,
+        )
         for _ in range(100):
             gmm.fit(X)
             if gmm.converged_:
@@ -844,51 +929,75 @@ def test_convergence_detected_with_warm_start():
         assert max_iter >= gmm.n_iter_
 
 
-def test_score():
-    covar_type = 'full'
+def test_score(global_dtype):
+    covar_type = "full"
     rng = np.random.RandomState(0)
-    rand_data = RandomData(rng, scale=7)
+    rand_data = RandomData(rng, scale=7, dtype=global_dtype)
     n_components = rand_data.n_components
     X = rand_data.X[covar_type]
+    assert X.dtype == global_dtype
 
     # Check the error message if we don't call fit
-    gmm1 = GaussianMixture(n_components=n_components, n_init=1,
-                           max_iter=1, reg_covar=0, random_state=rng,
-                           covariance_type=covar_type)
-    assert_raise_message(NotFittedError,
-                         "This GaussianMixture instance is not fitted "
-                         "yet. Call 'fit' with appropriate arguments "
-                         "before using this estimator.", gmm1.score, X)
+    gmm1 = GaussianMixture(
+        n_components=n_components,
+        n_init=1,
+        max_iter=1,
+        reg_covar=0,
+        random_state=rng,
+        covariance_type=covar_type,
+    )
+    msg = (
+        "This GaussianMixture instance is not fitted yet. Call 'fit' with "
+        "appropriate arguments before using this estimator."
+    )
+    with pytest.raises(NotFittedError, match=msg):
+        gmm1.score(X)
 
     # Check score value
     with warnings.catch_warnings():
         warnings.simplefilter("ignore", ConvergenceWarning)
         gmm1.fit(X)
+
+    assert gmm1.means_.dtype == global_dtype
+    assert gmm1.covariances_.dtype == global_dtype
+
     gmm_score = gmm1.score(X)
     gmm_score_proba = gmm1.score_samples(X).mean()
     assert_almost_equal(gmm_score, gmm_score_proba)
+    assert gmm_score_proba.dtype == global_dtype
 
     # Check if the score increase
-    gmm2 = GaussianMixture(n_components=n_components, n_init=1, reg_covar=0,
-                           random_state=rng,
-                           covariance_type=covar_type).fit(X)
+    gmm2 = GaussianMixture(
+        n_components=n_components,
+        n_init=1,
+        reg_covar=0,
+        random_state=rng,
+        covariance_type=covar_type,
+    ).fit(X)
     assert gmm2.score(X) > gmm1.score(X)
 
 
 def test_score_samples():
-    covar_type = 'full'
+    covar_type = "full"
     rng = np.random.RandomState(0)
     rand_data = RandomData(rng, scale=7)
     n_components = rand_data.n_components
     X = rand_data.X[covar_type]
 
     # Check the error message if we don't call fit
-    gmm = GaussianMixture(n_components=n_components, n_init=1, reg_covar=0,
-                          random_state=rng, covariance_type=covar_type)
-    assert_raise_message(NotFittedError,
-                         "This GaussianMixture instance is not fitted "
-                         "yet. Call 'fit' with appropriate arguments "
-                         "before using this estimator.", gmm.score_samples, X)
+    gmm = GaussianMixture(
+        n_components=n_components,
+        n_init=1,
+        reg_covar=0,
+        random_state=rng,
+        covariance_type=covar_type,
+    )
+    msg = (
+        "This GaussianMixture instance is not fitted yet. Call 'fit' with "
+        "appropriate arguments before using this estimator."
+    )
+    with pytest.raises(NotFittedError, match=msg):
+        gmm.score_samples(X)
 
     gmm_score_samples = gmm.fit(X).score_samples(X)
     assert gmm_score_samples.shape[0] == rand_data.n_samples
@@ -903,11 +1012,16 @@ def test_monotonic_likelihood():
 
     for covar_type in COVARIANCE_TYPE:
         X = rand_data.X[covar_type]
-        gmm = GaussianMixture(n_components=n_components,
-                              covariance_type=covar_type, reg_covar=0,
-                              warm_start=True, max_iter=1, random_state=rng,
-                              tol=1e-7)
-        current_log_likelihood = -np.infty
+        gmm = GaussianMixture(
+            n_components=n_components,
+            covariance_type=covar_type,
+            reg_covar=0,
+            warm_start=True,
+            max_iter=1,
+            random_state=rng,
+            tol=1e-7,
+        )
+        current_log_likelihood = -np.inf
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", ConvergenceWarning)
             # Do one training iteration at a time so we can make sure that the
@@ -929,46 +1043,55 @@ def test_regularisation():
     rng = np.random.RandomState(0)
     n_samples, n_features = 10, 5
 
-    X = np.vstack((np.ones((n_samples // 2, n_features)),
-                   np.zeros((n_samples // 2, n_features))))
+    X = np.vstack(
+        (np.ones((n_samples // 2, n_features)), np.zeros((n_samples // 2, n_features)))
+    )
 
     for covar_type in COVARIANCE_TYPE:
-        gmm = GaussianMixture(n_components=n_samples, reg_covar=0,
-                              covariance_type=covar_type, random_state=rng)
+        gmm = GaussianMixture(
+            n_components=n_samples,
+            reg_covar=0,
+            covariance_type=covar_type,
+            random_state=rng,
+        )
 
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", RuntimeWarning)
-            assert_raise_message(ValueError,
-                                 "Fitting the mixture model failed because "
-                                 "some components have ill-defined empirical "
-                                 "covariance (for instance caused by "
-                                 "singleton or collapsed samples). Try to "
-                                 "decrease the number of components, or "
-                                 "increase reg_covar.", gmm.fit, X)
+            msg = re.escape(
+                "Fitting the mixture model failed because some components have"
+                " ill-defined empirical covariance (for instance caused by "
+                "singleton or collapsed samples). Try to decrease the number "
+                "of components, increase reg_covar, or scale the input data."
+            )
+            with pytest.raises(ValueError, match=msg):
+                gmm.fit(X)
 
             gmm.set_params(reg_covar=1e-6).fit(X)
 
 
-def test_property():
+@pytest.mark.parametrize("covar_type", COVARIANCE_TYPE)
+def test_fitted_precision_covariance_concistency(covar_type, global_dtype):
     rng = np.random.RandomState(0)
-    rand_data = RandomData(rng, scale=7)
+    rand_data = RandomData(rng, scale=7, dtype=global_dtype)
     n_components = rand_data.n_components
 
-    for covar_type in COVARIANCE_TYPE:
-        X = rand_data.X[covar_type]
-        gmm = GaussianMixture(n_components=n_components,
-                              covariance_type=covar_type, random_state=rng,
-                              n_init=5)
-        gmm.fit(X)
-        if covar_type == 'full':
-            for prec, covar in zip(gmm.precisions_, gmm.covariances_):
-
-                assert_array_almost_equal(linalg.inv(prec), covar)
-        elif covar_type == 'tied':
-            assert_array_almost_equal(linalg.inv(gmm.precisions_),
-                                      gmm.covariances_)
-        else:
-            assert_array_almost_equal(gmm.precisions_, 1. / gmm.covariances_)
+    X = rand_data.X[covar_type]
+    gmm = GaussianMixture(
+        n_components=n_components,
+        covariance_type=covar_type,
+        random_state=rng,
+        n_init=5,
+    )
+    gmm.fit(X)
+    assert gmm.precisions_.dtype == global_dtype
+    assert gmm.covariances_.dtype == global_dtype
+    if covar_type == "full":
+        for prec, covar in zip(gmm.precisions_, gmm.covariances_):
+            assert_array_almost_equal(linalg.inv(prec), covar)
+    elif covar_type == "tied":
+        assert_array_almost_equal(linalg.inv(gmm.precisions_), gmm.covariances_)
+    else:
+        assert_array_almost_equal(gmm.precisions_, 1.0 / gmm.covariances_)
 
 
 def test_sample():
@@ -979,38 +1102,44 @@ def test_sample():
     for covar_type in COVARIANCE_TYPE:
         X = rand_data.X[covar_type]
 
-        gmm = GaussianMixture(n_components=n_components,
-                              covariance_type=covar_type, random_state=rng)
+        gmm = GaussianMixture(
+            n_components=n_components, covariance_type=covar_type, random_state=rng
+        )
         # To sample we need that GaussianMixture is fitted
-        assert_raise_message(NotFittedError, "This GaussianMixture instance "
-                             "is not fitted", gmm.sample, 0)
+        msg = "This GaussianMixture instance is not fitted"
+        with pytest.raises(NotFittedError, match=msg):
+            gmm.sample(0)
         gmm.fit(X)
 
-        assert_raise_message(ValueError, "Invalid value for 'n_samples",
-                             gmm.sample, 0)
+        msg = "Invalid value for 'n_samples'"
+        with pytest.raises(ValueError, match=msg):
+            gmm.sample(0)
 
         # Just to make sure the class samples correctly
         n_samples = 20000
         X_s, y_s = gmm.sample(n_samples)
 
         for k in range(n_components):
-            if covar_type == 'full':
-                assert_array_almost_equal(gmm.covariances_[k],
-                                          np.cov(X_s[y_s == k].T), decimal=1)
-            elif covar_type == 'tied':
-                assert_array_almost_equal(gmm.covariances_,
-                                          np.cov(X_s[y_s == k].T), decimal=1)
-            elif covar_type == 'diag':
-                assert_array_almost_equal(gmm.covariances_[k],
-                                          np.diag(np.cov(X_s[y_s == k].T)),
-                                          decimal=1)
+            if covar_type == "full":
+                assert_array_almost_equal(
+                    gmm.covariances_[k], np.cov(X_s[y_s == k].T), decimal=1
+                )
+            elif covar_type == "tied":
+                assert_array_almost_equal(
+                    gmm.covariances_, np.cov(X_s[y_s == k].T), decimal=1
+                )
+            elif covar_type == "diag":
+                assert_array_almost_equal(
+                    gmm.covariances_[k], np.diag(np.cov(X_s[y_s == k].T)), decimal=1
+                )
             else:
                 assert_array_almost_equal(
-                    gmm.covariances_[k], np.var(X_s[y_s == k] - gmm.means_[k]),
-                    decimal=1)
+                    gmm.covariances_[k],
+                    np.var(X_s[y_s == k] - gmm.means_[k]),
+                    decimal=1,
+                )
 
-        means_s = np.array([np.mean(X_s[y_s == k], 0)
-                           for k in range(n_components)])
+        means_s = np.array([np.mean(X_s[y_s == k], 0) for k in range(n_components)])
         assert_array_almost_equal(gmm.means_, means_s, decimal=1)
 
         # Check shapes of sampled data, see
@@ -1022,18 +1151,323 @@ def test_sample():
             assert X_s.shape == (sample_size, n_features)
 
 
-@ignore_warnings(category=ConvergenceWarning)
+@pytest.mark.filterwarnings("ignore::sklearn.exceptions.ConvergenceWarning")
 def test_init():
     # We check that by increasing the n_init number we have a better solution
     for random_state in range(15):
-        rand_data = RandomData(np.random.RandomState(random_state),
-                               n_samples=50, scale=1)
+        rand_data = RandomData(
+            np.random.RandomState(random_state), n_samples=50, scale=1
+        )
         n_components = rand_data.n_components
-        X = rand_data.X['full']
+        X = rand_data.X["full"]
 
-        gmm1 = GaussianMixture(n_components=n_components, n_init=1,
-                               max_iter=1, random_state=random_state).fit(X)
-        gmm2 = GaussianMixture(n_components=n_components, n_init=10,
-                               max_iter=1, random_state=random_state).fit(X)
+        gmm1 = GaussianMixture(
+            n_components=n_components, n_init=1, max_iter=1, random_state=random_state
+        ).fit(X)
+        gmm2 = GaussianMixture(
+            n_components=n_components, n_init=10, max_iter=1, random_state=random_state
+        ).fit(X)
 
         assert gmm2.lower_bound_ >= gmm1.lower_bound_
+
+
+def test_gaussian_mixture_setting_best_params():
+    """`GaussianMixture`'s best_parameters, `n_iter_` and `lower_bound_`
+    must be set appropriately in the case of divergence.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/18216
+    """
+    rnd = np.random.RandomState(0)
+    n_samples = 30
+    X = rnd.uniform(size=(n_samples, 3))
+
+    # following initialization parameters were found to lead to divergence
+    means_init = np.array(
+        [
+            [0.670637869618158, 0.21038256107384043, 0.12892629765485303],
+            [0.09394051075844147, 0.5759464955561779, 0.929296197576212],
+            [0.5033230372781258, 0.9569852381759425, 0.08654043447295741],
+            [0.18578301420435747, 0.5531158970919143, 0.19388943970532435],
+            [0.4548589928173794, 0.35182513658825276, 0.568146063202464],
+            [0.609279894978321, 0.7929063819678847, 0.9620097270828052],
+        ]
+    )
+    precisions_init = np.array(
+        [
+            999999.999604483,
+            999999.9990869573,
+            553.7603944542167,
+            204.78596008931834,
+            15.867423501783637,
+            85.4595728389735,
+        ]
+    )
+    weights_init = [
+        0.03333333333333341,
+        0.03333333333333341,
+        0.06666666666666674,
+        0.06666666666666674,
+        0.7000000000000001,
+        0.10000000000000007,
+    ]
+
+    gmm = GaussianMixture(
+        covariance_type="spherical",
+        reg_covar=0,
+        means_init=means_init,
+        weights_init=weights_init,
+        random_state=rnd,
+        n_components=len(weights_init),
+        precisions_init=precisions_init,
+        max_iter=1,
+    )
+    # ensure that no error is thrown during fit
+    gmm.fit(X)
+
+    # check that the fit did not converge
+    assert not gmm.converged_
+
+    # check that parameters are set for gmm
+    for attr in [
+        "weights_",
+        "means_",
+        "covariances_",
+        "precisions_cholesky_",
+        "n_iter_",
+        "lower_bound_",
+        "lower_bounds_",
+    ]:
+        assert hasattr(gmm, attr)
+
+
+@pytest.mark.parametrize(
+    "init_params", ["random", "random_from_data", "k-means++", "kmeans"]
+)
+def test_init_means_not_duplicated(init_params, global_random_seed):
+    # Check that all initialisations provide not duplicated starting means
+    rng = np.random.RandomState(global_random_seed)
+    rand_data = RandomData(rng, scale=5)
+    n_components = rand_data.n_components
+    X = rand_data.X["full"]
+
+    gmm = GaussianMixture(
+        n_components=n_components, init_params=init_params, random_state=rng, max_iter=0
+    )
+    gmm.fit(X)
+
+    means = gmm.means_
+    for i_mean, j_mean in itertools.combinations(means, r=2):
+        assert not np.allclose(i_mean, j_mean)
+
+
+@pytest.mark.parametrize(
+    "init_params", ["random", "random_from_data", "k-means++", "kmeans"]
+)
+def test_means_for_all_inits(init_params, global_random_seed, global_dtype):
+    # Check fitted means properties for all initializations
+    rng = np.random.RandomState(global_random_seed)
+    rand_data = RandomData(rng, scale=5, dtype=global_dtype)
+    n_components = rand_data.n_components
+    X = rand_data.X["full"]
+
+    gmm = GaussianMixture(
+        n_components=n_components, init_params=init_params, random_state=rng
+    )
+    gmm.fit(X)
+
+    assert gmm.means_.shape == (n_components, X.shape[1])
+    assert np.all(X.min(axis=0) <= gmm.means_)
+    assert np.all(gmm.means_ <= X.max(axis=0))
+    assert gmm.converged_
+    assert gmm.means_.dtype == global_dtype
+    assert gmm.covariances_.dtype == global_dtype
+    assert gmm.weights_.dtype == global_dtype
+
+
+def test_max_iter_zero():
+    # Check that max_iter=0 returns initialisation as expected
+    # Pick arbitrary initial means and check equal to max_iter=0
+    rng = np.random.RandomState(0)
+    rand_data = RandomData(rng, scale=5)
+    n_components = rand_data.n_components
+    X = rand_data.X["full"]
+    means_init = [[20, 30], [30, 25]]
+    gmm = GaussianMixture(
+        n_components=n_components,
+        random_state=rng,
+        means_init=means_init,
+        tol=1e-06,
+        max_iter=0,
+    )
+    gmm.fit(X)
+
+    assert_allclose(gmm.means_, means_init)
+
+
+def test_gaussian_mixture_precisions_init_diag(global_dtype):
+    """Check that we properly initialize `precision_cholesky_` when we manually
+    provide the precision matrix.
+
+    In this regard, we check the consistency between estimating the precision
+    matrix and providing the same precision matrix as initialization. It should
+    lead to the same results with the same number of iterations.
+
+    If the initialization is wrong then the number of iterations will increase.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/16944
+    """
+    # generate a toy dataset
+    n_samples = 300
+    rng = np.random.RandomState(0)
+    shifted_gaussian = rng.randn(n_samples, 2) + np.array([20, 20])
+    C = np.array([[0.0, -0.7], [3.5, 0.7]])
+    stretched_gaussian = np.dot(rng.randn(n_samples, 2), C)
+    X = np.vstack([shifted_gaussian, stretched_gaussian]).astype(global_dtype)
+
+    # common parameters to check the consistency of precision initialization
+    n_components, covariance_type, reg_covar, random_state = 2, "diag", 1e-6, 0
+
+    # execute the manual initialization to compute the precision matrix:
+    # - run KMeans to have an initial guess
+    # - estimate the covariance
+    # - compute the precision matrix from the estimated covariance
+    resp = np.zeros((X.shape[0], n_components)).astype(global_dtype)
+    label = (
+        KMeans(n_clusters=n_components, n_init=1, random_state=random_state)
+        .fit(X)
+        .labels_
+    )
+    resp[np.arange(X.shape[0]), label] = 1
+    _, _, covariance = _estimate_gaussian_parameters(
+        X, resp, reg_covar=reg_covar, covariance_type=covariance_type
+    )
+    assert covariance.dtype == global_dtype
+    precisions_init = 1 / covariance
+
+    gm_with_init = GaussianMixture(
+        n_components=n_components,
+        covariance_type=covariance_type,
+        reg_covar=reg_covar,
+        precisions_init=precisions_init,
+        random_state=random_state,
+    ).fit(X)
+    assert gm_with_init.means_.dtype == global_dtype
+    assert gm_with_init.covariances_.dtype == global_dtype
+    assert gm_with_init.precisions_cholesky_.dtype == global_dtype
+
+    gm_without_init = GaussianMixture(
+        n_components=n_components,
+        covariance_type=covariance_type,
+        reg_covar=reg_covar,
+        random_state=random_state,
+    ).fit(X)
+    assert gm_without_init.means_.dtype == global_dtype
+    assert gm_without_init.covariances_.dtype == global_dtype
+    assert gm_without_init.precisions_cholesky_.dtype == global_dtype
+
+    assert gm_without_init.n_iter_ == gm_with_init.n_iter_
+    assert_allclose(
+        gm_with_init.precisions_cholesky_, gm_without_init.precisions_cholesky_
+    )
+
+
+def _generate_data(seed, n_samples, n_features, n_components, dtype=np.float64):
+    """Randomly generate samples and responsibilities."""
+    rs = np.random.RandomState(seed)
+    X = rs.random_sample((n_samples, n_features)).astype(dtype)
+    resp = rs.random_sample((n_samples, n_components)).astype(dtype)
+    resp /= resp.sum(axis=1)[:, np.newaxis]
+    return X, resp
+
+
+def _calculate_precisions(X, resp, covariance_type):
+    """Calculate precision matrix of X and its Cholesky decomposition
+    for the given covariance type.
+    """
+    reg_covar = 1e-6
+    weights, means, covariances = _estimate_gaussian_parameters(
+        X, resp, reg_covar, covariance_type
+    )
+    precisions_cholesky = _compute_precision_cholesky(covariances, covariance_type)
+
+    _, n_components = resp.shape
+    # Instantiate a `GaussianMixture` model in order to use its
+    # `_set_parameters` method to return the `precisions_` and
+    #  `precisions_cholesky_` from matching the `covariance_type`
+    # provided.
+    gmm = GaussianMixture(n_components=n_components, covariance_type=covariance_type)
+    params = (weights, means, covariances, precisions_cholesky)
+    gmm._set_parameters(params)
+    return gmm.precisions_, gmm.precisions_cholesky_
+
+
+@pytest.mark.parametrize("covariance_type", COVARIANCE_TYPE)
+def test_gaussian_mixture_precisions_init(
+    covariance_type, global_random_seed, global_dtype
+):
+    """Non-regression test for #26415."""
+
+    X, resp = _generate_data(
+        seed=global_random_seed,
+        n_samples=100,
+        n_features=3,
+        n_components=4,
+        dtype=global_dtype,
+    )
+
+    precisions_init, desired_precisions_cholesky = _calculate_precisions(
+        X, resp, covariance_type
+    )
+    assert precisions_init.dtype == global_dtype
+    assert desired_precisions_cholesky.dtype == global_dtype
+
+    gmm = GaussianMixture(
+        covariance_type=covariance_type, precisions_init=precisions_init
+    )
+    gmm._initialize(X, resp)
+    actual_precisions_cholesky = gmm.precisions_cholesky_
+    assert_allclose(actual_precisions_cholesky, desired_precisions_cholesky)
+
+
+def test_gaussian_mixture_single_component_stable():
+    """
+    Non-regression test for #23032 ensuring 1-component GM works on only a
+    few samples.
+    """
+    rng = np.random.RandomState(0)
+    X = rng.multivariate_normal(np.zeros(2), np.identity(2), size=3)
+    gm = GaussianMixture(n_components=1)
+    gm.fit(X).sample()
+
+
+def test_gaussian_mixture_all_init_does_not_estimate_gaussian_parameters(
+    monkeypatch,
+    global_random_seed,
+):
+    """When all init parameters are provided, the Gaussian parameters
+    are not estimated.
+
+    Non-regression test for gh-26015.
+    """
+
+    mock = Mock(side_effect=_estimate_gaussian_parameters)
+    monkeypatch.setattr(
+        sklearn.mixture._gaussian_mixture, "_estimate_gaussian_parameters", mock
+    )
+
+    rng = np.random.RandomState(global_random_seed)
+    rand_data = RandomData(rng)
+
+    gm = GaussianMixture(
+        n_components=rand_data.n_components,
+        weights_init=rand_data.weights,
+        means_init=rand_data.means,
+        precisions_init=rand_data.precisions["full"],
+        random_state=rng,
+    )
+    gm.fit(rand_data.X["full"])
+    # The initial gaussian parameters are not estimated. They are estimated for every
+    # m_step.
+    assert mock.call_count == gm.n_iter_
diff --git a/sklearn/mixture/tests/test_mixture.py b/sklearn/mixture/tests/test_mixture.py
index a79cafe3bccec..9c98d150f06a8 100644
--- a/sklearn/mixture/tests/test_mixture.py
+++ b/sklearn/mixture/tests/test_mixture.py
@@ -1,18 +1,13 @@
-# Author: Guillaume Lemaitre 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-import pytest
 import numpy as np
+import pytest
 
-from sklearn.mixture import GaussianMixture
-from sklearn.mixture import BayesianGaussianMixture
+from sklearn.mixture import BayesianGaussianMixture, GaussianMixture
 
 
-@pytest.mark.parametrize(
-    "estimator",
-    [GaussianMixture(),
-     BayesianGaussianMixture()]
-)
+@pytest.mark.parametrize("estimator", [GaussianMixture(), BayesianGaussianMixture()])
 def test_gaussian_mixture_n_iter(estimator):
     # check that n_iter is the number of iteration performed.
     rng = np.random.RandomState(0)
@@ -21,3 +16,15 @@ def test_gaussian_mixture_n_iter(estimator):
     estimator.set_params(max_iter=max_iter)
     estimator.fit(X)
     assert estimator.n_iter_ == max_iter
+
+
+@pytest.mark.parametrize("estimator", [GaussianMixture(), BayesianGaussianMixture()])
+def test_mixture_n_components_greater_than_n_samples_error(estimator):
+    """Check error when n_components <= n_samples"""
+    rng = np.random.RandomState(0)
+    X = rng.rand(10, 5)
+    estimator.set_params(n_components=12)
+
+    msg = "Expected n_samples >= n_components"
+    with pytest.raises(ValueError, match=msg):
+        estimator.fit(X)
diff --git a/sklearn/model_selection/__init__.py b/sklearn/model_selection/__init__.py
index 897183414b5a6..8eb0ef772c552 100644
--- a/sklearn/model_selection/__init__.py
+++ b/sklearn/model_selection/__init__.py
@@ -1,69 +1,99 @@
-import typing
+"""Tools for model selection, such as cross validation and hyper-parameter tuning."""
 
-from ._split import BaseCrossValidator
-from ._split import KFold
-from ._split import GroupKFold
-from ._split import StratifiedKFold
-from ._split import TimeSeriesSplit
-from ._split import LeaveOneGroupOut
-from ._split import LeaveOneOut
-from ._split import LeavePGroupsOut
-from ._split import LeavePOut
-from ._split import RepeatedKFold
-from ._split import RepeatedStratifiedKFold
-from ._split import ShuffleSplit
-from ._split import GroupShuffleSplit
-from ._split import StratifiedShuffleSplit
-from ._split import PredefinedSplit
-from ._split import train_test_split
-from ._split import check_cv
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-from ._validation import cross_val_score
-from ._validation import cross_val_predict
-from ._validation import cross_validate
-from ._validation import learning_curve
-from ._validation import permutation_test_score
-from ._validation import validation_curve
+import typing
 
-from ._search import GridSearchCV
-from ._search import RandomizedSearchCV
-from ._search import ParameterGrid
-from ._search import ParameterSampler
-from ._search import fit_grid_point
+from ._classification_threshold import (
+    FixedThresholdClassifier,
+    TunedThresholdClassifierCV,
+)
+from ._plot import LearningCurveDisplay, ValidationCurveDisplay
+from ._search import GridSearchCV, ParameterGrid, ParameterSampler, RandomizedSearchCV
+from ._split import (
+    BaseCrossValidator,
+    BaseShuffleSplit,
+    GroupKFold,
+    GroupShuffleSplit,
+    KFold,
+    LeaveOneGroupOut,
+    LeaveOneOut,
+    LeavePGroupsOut,
+    LeavePOut,
+    PredefinedSplit,
+    RepeatedKFold,
+    RepeatedStratifiedKFold,
+    ShuffleSplit,
+    StratifiedGroupKFold,
+    StratifiedKFold,
+    StratifiedShuffleSplit,
+    TimeSeriesSplit,
+    check_cv,
+    train_test_split,
+)
+from ._validation import (
+    cross_val_predict,
+    cross_val_score,
+    cross_validate,
+    learning_curve,
+    permutation_test_score,
+    validation_curve,
+)
 
 if typing.TYPE_CHECKING:
     # Avoid errors in type checkers (e.g. mypy) for experimental estimators.
     # TODO: remove this check once the estimator is no longer experimental.
-    from ._search_successive_halving import (  # noqa
-        HalvingGridSearchCV, HalvingRandomSearchCV
+    from ._search_successive_halving import (  # noqa: F401
+        HalvingGridSearchCV,
+        HalvingRandomSearchCV,
     )
 
 
-__all__ = ['BaseCrossValidator',
-           'GridSearchCV',
-           'TimeSeriesSplit',
-           'KFold',
-           'GroupKFold',
-           'GroupShuffleSplit',
-           'LeaveOneGroupOut',
-           'LeaveOneOut',
-           'LeavePGroupsOut',
-           'LeavePOut',
-           'RepeatedKFold',
-           'RepeatedStratifiedKFold',
-           'ParameterGrid',
-           'ParameterSampler',
-           'PredefinedSplit',
-           'RandomizedSearchCV',
-           'ShuffleSplit',
-           'StratifiedKFold',
-           'StratifiedShuffleSplit',
-           'check_cv',
-           'cross_val_predict',
-           'cross_val_score',
-           'cross_validate',
-           'fit_grid_point',
-           'learning_curve',
-           'permutation_test_score',
-           'train_test_split',
-           'validation_curve']
+__all__ = [
+    "BaseCrossValidator",
+    "BaseShuffleSplit",
+    "FixedThresholdClassifier",
+    "GridSearchCV",
+    "GroupKFold",
+    "GroupShuffleSplit",
+    "KFold",
+    "LearningCurveDisplay",
+    "LeaveOneGroupOut",
+    "LeaveOneOut",
+    "LeavePGroupsOut",
+    "LeavePOut",
+    "ParameterGrid",
+    "ParameterSampler",
+    "PredefinedSplit",
+    "RandomizedSearchCV",
+    "RepeatedKFold",
+    "RepeatedStratifiedKFold",
+    "ShuffleSplit",
+    "StratifiedGroupKFold",
+    "StratifiedKFold",
+    "StratifiedShuffleSplit",
+    "TimeSeriesSplit",
+    "TunedThresholdClassifierCV",
+    "ValidationCurveDisplay",
+    "check_cv",
+    "cross_val_predict",
+    "cross_val_score",
+    "cross_validate",
+    "learning_curve",
+    "permutation_test_score",
+    "train_test_split",
+    "validation_curve",
+]
+
+
+# TODO: remove this check once the estimator is no longer experimental.
+def __getattr__(name):
+    if name in {"HalvingGridSearchCV", "HalvingRandomSearchCV"}:
+        raise ImportError(
+            f"{name} is experimental and the API might change without any "
+            "deprecation cycle. To use it, you need to explicitly import "
+            "enable_halving_search_cv:\n"
+            "from sklearn.experimental import enable_halving_search_cv"
+        )
+    raise AttributeError(f"module {__name__} has no attribute {name}")
diff --git a/sklearn/model_selection/_classification_threshold.py b/sklearn/model_selection/_classification_threshold.py
new file mode 100644
index 0000000000000..c68ed38b8819d
--- /dev/null
+++ b/sklearn/model_selection/_classification_threshold.py
@@ -0,0 +1,889 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+from collections.abc import MutableMapping
+from numbers import Integral, Real
+
+import numpy as np
+
+from ..base import (
+    BaseEstimator,
+    ClassifierMixin,
+    MetaEstimatorMixin,
+    _fit_context,
+    clone,
+)
+from ..exceptions import NotFittedError
+from ..metrics import (
+    check_scoring,
+    get_scorer_names,
+)
+from ..metrics._scorer import (
+    _CurveScorer,
+    _threshold_scores_to_class_labels,
+)
+from ..utils import _safe_indexing, get_tags
+from ..utils._param_validation import HasMethods, Interval, RealNotInt, StrOptions
+from ..utils._response import _get_response_values_binary
+from ..utils.metadata_routing import (
+    MetadataRouter,
+    MethodMapping,
+    _raise_for_params,
+    process_routing,
+)
+from ..utils.metaestimators import available_if
+from ..utils.multiclass import type_of_target
+from ..utils.parallel import Parallel, delayed
+from ..utils.validation import (
+    _check_method_params,
+    _estimator_has,
+    _num_samples,
+    check_is_fitted,
+    indexable,
+)
+from ._split import StratifiedShuffleSplit, check_cv
+
+
+def _check_is_fitted(estimator):
+    try:
+        check_is_fitted(estimator.estimator)
+    except NotFittedError:
+        check_is_fitted(estimator, "estimator_")
+
+
+class BaseThresholdClassifier(ClassifierMixin, MetaEstimatorMixin, BaseEstimator):
+    """Base class for binary classifiers that set a non-default decision threshold.
+
+    In this base class, we define the following interface:
+
+    - the validation of common parameters in `fit`;
+    - the different prediction methods that can be used with the classifier.
+
+    .. versionadded:: 1.5
+
+    Parameters
+    ----------
+    estimator : estimator instance
+        The binary classifier, fitted or not, for which we want to optimize
+        the decision threshold used during `predict`.
+
+    response_method : {"auto", "decision_function", "predict_proba"}, default="auto"
+        Methods by the classifier `estimator` corresponding to the
+        decision function for which we want to find a threshold. It can be:
+
+        * if `"auto"`, it will try to invoke, for each classifier,
+          `"predict_proba"` or `"decision_function"` in that order.
+        * otherwise, one of `"predict_proba"` or `"decision_function"`.
+          If the method is not implemented by the classifier, it will raise an
+          error.
+    """
+
+    _parameter_constraints: dict = {
+        "estimator": [
+            HasMethods(["fit", "predict_proba"]),
+            HasMethods(["fit", "decision_function"]),
+        ],
+        "response_method": [StrOptions({"auto", "predict_proba", "decision_function"})],
+    }
+
+    def __init__(self, estimator, *, response_method="auto"):
+        self.estimator = estimator
+        self.response_method = response_method
+
+    def _get_response_method(self):
+        """Define the response method."""
+        if self.response_method == "auto":
+            response_method = ["predict_proba", "decision_function"]
+        else:
+            response_method = self.response_method
+        return response_method
+
+    @_fit_context(
+        # *ThresholdClassifier*.estimator is not validated yet
+        prefer_skip_nested_validation=False
+    )
+    def fit(self, X, y, **params):
+        """Fit the classifier.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Training data.
+
+        y : array-like of shape (n_samples,)
+            Target values.
+
+        **params : dict
+            Parameters to pass to the `fit` method of the underlying
+            classifier.
+
+        Returns
+        -------
+        self : object
+            Returns an instance of self.
+        """
+        _raise_for_params(params, self, None)
+
+        X, y = indexable(X, y)
+
+        y_type = type_of_target(y, input_name="y")
+        if y_type != "binary":
+            raise ValueError(
+                f"Only binary classification is supported. Unknown label type: {y_type}"
+            )
+
+        self._fit(X, y, **params)
+
+        if hasattr(self.estimator_, "n_features_in_"):
+            self.n_features_in_ = self.estimator_.n_features_in_
+        if hasattr(self.estimator_, "feature_names_in_"):
+            self.feature_names_in_ = self.estimator_.feature_names_in_
+
+        return self
+
+    @property
+    def classes_(self):
+        """Classes labels."""
+        return self.estimator_.classes_
+
+    @available_if(_estimator_has("predict_proba"))
+    def predict_proba(self, X):
+        """Predict class probabilities for `X` using the fitted estimator.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Training vectors, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
+
+        Returns
+        -------
+        probabilities : ndarray of shape (n_samples, n_classes)
+            The class probabilities of the input samples.
+        """
+        _check_is_fitted(self)
+        estimator = getattr(self, "estimator_", self.estimator)
+        return estimator.predict_proba(X)
+
+    @available_if(_estimator_has("predict_log_proba"))
+    def predict_log_proba(self, X):
+        """Predict logarithm class probabilities for `X` using the fitted estimator.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Training vectors, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
+
+        Returns
+        -------
+        log_probabilities : ndarray of shape (n_samples, n_classes)
+            The logarithm class probabilities of the input samples.
+        """
+        _check_is_fitted(self)
+        estimator = getattr(self, "estimator_", self.estimator)
+        return estimator.predict_log_proba(X)
+
+    @available_if(_estimator_has("decision_function"))
+    def decision_function(self, X):
+        """Decision function for samples in `X` using the fitted estimator.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Training vectors, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
+
+        Returns
+        -------
+        decisions : ndarray of shape (n_samples,)
+            The decision function computed the fitted estimator.
+        """
+        _check_is_fitted(self)
+        estimator = getattr(self, "estimator_", self.estimator)
+        return estimator.decision_function(X)
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.classifier_tags.multi_class = False
+        tags.input_tags.sparse = get_tags(self.estimator).input_tags.sparse
+        return tags
+
+
+class FixedThresholdClassifier(BaseThresholdClassifier):
+    """Binary classifier that manually sets the decision threshold.
+
+    This classifier allows to change the default decision threshold used for
+    converting posterior probability estimates (i.e. output of `predict_proba`) or
+    decision scores (i.e. output of `decision_function`) into a class label.
+
+    Here, the threshold is not optimized and is set to a constant value.
+
+    Read more in the :ref:`User Guide `.
+
+    .. versionadded:: 1.5
+
+    Parameters
+    ----------
+    estimator : estimator instance
+        The binary classifier, fitted or not, for which we want to optimize
+        the decision threshold used during `predict`.
+
+    threshold : {"auto"} or float, default="auto"
+        The decision threshold to use when converting posterior probability estimates
+        (i.e. output of `predict_proba`) or decision scores (i.e. output of
+        `decision_function`) into a class label. When `"auto"`, the threshold is set
+        to 0.5 if `predict_proba` is used as `response_method`, otherwise it is set to
+        0 (i.e. the default threshold for `decision_function`).
+
+    pos_label : int, float, bool or str, default=None
+        The label of the positive class. Used to process the output of the
+        `response_method` method. When `pos_label=None`, if `y_true` is in `{-1, 1}` or
+        `{0, 1}`, `pos_label` is set to 1, otherwise an error will be raised.
+
+    response_method : {"auto", "decision_function", "predict_proba"}, default="auto"
+        Methods by the classifier `estimator` corresponding to the
+        decision function for which we want to find a threshold. It can be:
+
+        * if `"auto"`, it will try to invoke `"predict_proba"` or `"decision_function"`
+          in that order.
+        * otherwise, one of `"predict_proba"` or `"decision_function"`.
+          If the method is not implemented by the classifier, it will raise an
+          error.
+
+    Attributes
+    ----------
+    estimator_ : estimator instance
+        The fitted classifier used when predicting.
+
+    classes_ : ndarray of shape (n_classes,)
+        The class labels.
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`. Only defined if the
+        underlying estimator exposes such an attribute when fit.
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Only defined if the
+        underlying estimator exposes such an attribute when fit.
+
+    See Also
+    --------
+    sklearn.model_selection.TunedThresholdClassifierCV : Classifier that post-tunes
+        the decision threshold based on some metrics and using cross-validation.
+    sklearn.calibration.CalibratedClassifierCV : Estimator that calibrates
+        probabilities.
+
+    Examples
+    --------
+    >>> from sklearn.datasets import make_classification
+    >>> from sklearn.linear_model import LogisticRegression
+    >>> from sklearn.metrics import confusion_matrix
+    >>> from sklearn.model_selection import FixedThresholdClassifier, train_test_split
+    >>> X, y = make_classification(
+    ...     n_samples=1_000, weights=[0.9, 0.1], class_sep=0.8, random_state=42
+    ... )
+    >>> X_train, X_test, y_train, y_test = train_test_split(
+    ...     X, y, stratify=y, random_state=42
+    ... )
+    >>> classifier = LogisticRegression(random_state=0).fit(X_train, y_train)
+    >>> print(confusion_matrix(y_test, classifier.predict(X_test)))
+    [[217   7]
+     [ 19   7]]
+    >>> classifier_other_threshold = FixedThresholdClassifier(
+    ...     classifier, threshold=0.1, response_method="predict_proba"
+    ... ).fit(X_train, y_train)
+    >>> print(confusion_matrix(y_test, classifier_other_threshold.predict(X_test)))
+    [[184  40]
+     [  6  20]]
+    """
+
+    _parameter_constraints: dict = {
+        **BaseThresholdClassifier._parameter_constraints,
+        "threshold": [StrOptions({"auto"}), Real],
+        "pos_label": [Real, str, "boolean", None],
+    }
+
+    def __init__(
+        self,
+        estimator,
+        *,
+        threshold="auto",
+        pos_label=None,
+        response_method="auto",
+    ):
+        super().__init__(estimator=estimator, response_method=response_method)
+        self.pos_label = pos_label
+        self.threshold = threshold
+
+    @property
+    def classes_(self):
+        if estimator := getattr(self, "estimator_", None):
+            return estimator.classes_
+        try:
+            check_is_fitted(self.estimator)
+            return self.estimator.classes_
+        except NotFittedError:
+            raise AttributeError(
+                "The underlying estimator is not fitted yet."
+            ) from NotFittedError
+
+    def _fit(self, X, y, **params):
+        """Fit the classifier.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Training data.
+
+        y : array-like of shape (n_samples,)
+            Target values.
+
+        **params : dict
+            Parameters to pass to the `fit` method of the underlying
+            classifier.
+
+        Returns
+        -------
+        self : object
+            Returns an instance of self.
+        """
+        routed_params = process_routing(self, "fit", **params)
+        self.estimator_ = clone(self.estimator).fit(X, y, **routed_params.estimator.fit)
+        return self
+
+    def predict(self, X):
+        """Predict the target of new samples.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            The samples, as accepted by `estimator.predict`.
+
+        Returns
+        -------
+        class_labels : ndarray of shape (n_samples,)
+            The predicted class.
+        """
+        _check_is_fitted(self)
+
+        estimator = getattr(self, "estimator_", self.estimator)
+
+        y_score, _, response_method_used = _get_response_values_binary(
+            estimator,
+            X,
+            self._get_response_method(),
+            pos_label=self.pos_label,
+            return_response_method_used=True,
+        )
+
+        if self.threshold == "auto":
+            decision_threshold = 0.5 if response_method_used == "predict_proba" else 0.0
+        else:
+            decision_threshold = self.threshold
+
+        return _threshold_scores_to_class_labels(
+            y_score, decision_threshold, self.classes_, self.pos_label
+        )
+
+    def get_metadata_routing(self):
+        """Get metadata routing of this object.
+
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
+
+        Returns
+        -------
+        routing : MetadataRouter
+            A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
+            routing information.
+        """
+        router = MetadataRouter(owner=self.__class__.__name__).add(
+            estimator=self.estimator,
+            method_mapping=MethodMapping().add(callee="fit", caller="fit"),
+        )
+        return router
+
+
+def _fit_and_score_over_thresholds(
+    classifier,
+    X,
+    y,
+    *,
+    fit_params,
+    train_idx,
+    val_idx,
+    curve_scorer,
+    score_params,
+):
+    """Fit a classifier and compute the scores for different decision thresholds.
+
+    Parameters
+    ----------
+    classifier : estimator instance
+        The classifier to fit and use for scoring. If `classifier` is already fitted,
+        it will be used as is.
+
+    X : {array-like, sparse matrix} of shape (n_samples, n_features)
+        The entire dataset.
+
+    y : array-like of shape (n_samples,)
+        The entire target vector.
+
+    fit_params : dict
+        Parameters to pass to the `fit` method of the underlying classifier.
+
+    train_idx : ndarray of shape (n_train_samples,) or None
+        The indices of the training set. If `None`, `classifier` is expected to be
+        already fitted.
+
+    val_idx : ndarray of shape (n_val_samples,)
+        The indices of the validation set used to score `classifier`. If `train_idx`,
+        the entire set will be used.
+
+    curve_scorer : scorer instance
+        The scorer taking `classifier` and the validation set as input and outputting
+        decision thresholds and scores as a curve. Note that this is different from
+        the usual scorer that outputs a single score value as `curve_scorer`
+        outputs a single score value for each threshold.
+
+    score_params : dict
+        Parameters to pass to the `score` method of the underlying scorer.
+
+    Returns
+    -------
+    scores : ndarray of shape (thresholds,) or tuple of such arrays
+        The scores computed for each decision threshold. When TPR/TNR or precision/
+        recall are computed, `scores` is a tuple of two arrays.
+
+    potential_thresholds : ndarray of shape (thresholds,)
+        The decision thresholds used to compute the scores. They are returned in
+        ascending order.
+    """
+
+    if train_idx is not None:
+        X_train, X_val = _safe_indexing(X, train_idx), _safe_indexing(X, val_idx)
+        y_train, y_val = _safe_indexing(y, train_idx), _safe_indexing(y, val_idx)
+        fit_params_train = _check_method_params(X, fit_params, indices=train_idx)
+        score_params_val = _check_method_params(X, score_params, indices=val_idx)
+        classifier.fit(X_train, y_train, **fit_params_train)
+    else:  # prefit estimator, only a validation set is provided
+        X_val, y_val, score_params_val = X, y, score_params
+
+    return curve_scorer(classifier, X_val, y_val, **score_params_val)
+
+
+def _mean_interpolated_score(target_thresholds, cv_thresholds, cv_scores):
+    """Compute the mean interpolated score across folds by defining common thresholds.
+
+    Parameters
+    ----------
+    target_thresholds : ndarray of shape (thresholds,)
+        The thresholds to use to compute the mean score.
+
+    cv_thresholds : ndarray of shape (n_folds, thresholds_fold)
+        The thresholds used to compute the scores for each fold.
+
+    cv_scores : ndarray of shape (n_folds, thresholds_fold)
+        The scores computed for each threshold for each fold.
+
+    Returns
+    -------
+    mean_score : ndarray of shape (thresholds,)
+        The mean score across all folds for each target threshold.
+    """
+    return np.mean(
+        [
+            np.interp(target_thresholds, split_thresholds, split_score)
+            for split_thresholds, split_score in zip(cv_thresholds, cv_scores)
+        ],
+        axis=0,
+    )
+
+
+class TunedThresholdClassifierCV(BaseThresholdClassifier):
+    """Classifier that post-tunes the decision threshold using cross-validation.
+
+    This estimator post-tunes the decision threshold (cut-off point) that is
+    used for converting posterior probability estimates (i.e. output of
+    `predict_proba`) or decision scores (i.e. output of `decision_function`)
+    into a class label. The tuning is done by optimizing a binary metric,
+    potentially constrained by a another metric.
+
+    Read more in the :ref:`User Guide `.
+
+    .. versionadded:: 1.5
+
+    Parameters
+    ----------
+    estimator : estimator instance
+        The classifier, fitted or not, for which we want to optimize
+        the decision threshold used during `predict`.
+
+    scoring : str or callable, default="balanced_accuracy"
+        The objective metric to be optimized. Can be one of:
+
+        - str: string associated to a scoring function for binary classification,
+          see :ref:`scoring_string_names` for options.
+        - callable: a scorer callable object (e.g., function) with signature
+          ``scorer(estimator, X, y)``. See :ref:`scoring_callable` for details.
+
+    response_method : {"auto", "decision_function", "predict_proba"}, default="auto"
+        Methods by the classifier `estimator` corresponding to the
+        decision function for which we want to find a threshold. It can be:
+
+        * if `"auto"`, it will try to invoke, for each classifier,
+          `"predict_proba"` or `"decision_function"` in that order.
+        * otherwise, one of `"predict_proba"` or `"decision_function"`.
+          If the method is not implemented by the classifier, it will raise an
+          error.
+
+    thresholds : int or array-like, default=100
+        The number of decision threshold to use when discretizing the output of the
+        classifier `method`. Pass an array-like to manually specify the thresholds
+        to use.
+
+    cv : int, float, cross-validation generator, iterable or "prefit", default=None
+        Determines the cross-validation splitting strategy to train classifier.
+        Possible inputs for cv are:
+
+        * `None`, to use the default 5-fold stratified K-fold cross validation;
+        * An integer number, to specify the number of folds in a stratified k-fold;
+        * A float number, to specify a single shuffle split. The floating number should
+          be in (0, 1) and represent the size of the validation set;
+        * An object to be used as a cross-validation generator;
+        * An iterable yielding train, test splits;
+        * `"prefit"`, to bypass the cross-validation.
+
+        Refer :ref:`User Guide ` for the various
+        cross-validation strategies that can be used here.
+
+        .. warning::
+            Using `cv="prefit"` and passing the same dataset for fitting `estimator`
+            and tuning the cut-off point is subject to undesired overfitting. You can
+            refer to :ref:`TunedThresholdClassifierCV_no_cv` for an example.
+
+            This option should only be used when the set used to fit `estimator` is
+            different from the one used to tune the cut-off point (by calling
+            :meth:`TunedThresholdClassifierCV.fit`).
+
+    refit : bool, default=True
+        Whether or not to refit the classifier on the entire training set once
+        the decision threshold has been found.
+        Note that forcing `refit=False` on cross-validation having more
+        than a single split will raise an error. Similarly, `refit=True` in
+        conjunction with `cv="prefit"` will raise an error.
+
+    n_jobs : int, default=None
+        The number of jobs to run in parallel. When `cv` represents a
+        cross-validation strategy, the fitting and scoring on each data split
+        is done in parallel. ``None`` means 1 unless in a
+        :obj:`joblib.parallel_backend` context. ``-1`` means using all
+        processors. See :term:`Glossary ` for more details.
+
+    random_state : int, RandomState instance or None, default=None
+        Controls the randomness of cross-validation when `cv` is a float.
+        See :term:`Glossary `.
+
+    store_cv_results : bool, default=False
+        Whether to store all scores and thresholds computed during the cross-validation
+        process.
+
+    Attributes
+    ----------
+    estimator_ : estimator instance
+        The fitted classifier used when predicting.
+
+    best_threshold_ : float
+        The new decision threshold.
+
+    best_score_ : float or None
+        The optimal score of the objective metric, evaluated at `best_threshold_`.
+
+    cv_results_ : dict or None
+        A dictionary containing the scores and thresholds computed during the
+        cross-validation process. Only exist if `store_cv_results=True`. The
+        keys are `"thresholds"` and `"scores"`.
+
+    classes_ : ndarray of shape (n_classes,)
+        The class labels.
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`. Only defined if the
+        underlying estimator exposes such an attribute when fit.
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Only defined if the
+        underlying estimator exposes such an attribute when fit.
+
+    See Also
+    --------
+    sklearn.model_selection.FixedThresholdClassifier : Classifier that uses a
+        constant threshold.
+    sklearn.calibration.CalibratedClassifierCV : Estimator that calibrates
+        probabilities.
+
+    Examples
+    --------
+    >>> from sklearn.datasets import make_classification
+    >>> from sklearn.ensemble import RandomForestClassifier
+    >>> from sklearn.metrics import classification_report
+    >>> from sklearn.model_selection import TunedThresholdClassifierCV, train_test_split
+    >>> X, y = make_classification(
+    ...     n_samples=1_000, weights=[0.9, 0.1], class_sep=0.8, random_state=42
+    ... )
+    >>> X_train, X_test, y_train, y_test = train_test_split(
+    ...     X, y, stratify=y, random_state=42
+    ... )
+    >>> classifier = RandomForestClassifier(random_state=0).fit(X_train, y_train)
+    >>> print(classification_report(y_test, classifier.predict(X_test)))
+                  precision    recall  f1-score   support
+    
+               0       0.94      0.99      0.96       224
+               1       0.80      0.46      0.59        26
+    
+        accuracy                           0.93       250
+       macro avg       0.87      0.72      0.77       250
+    weighted avg       0.93      0.93      0.92       250
+    
+    >>> classifier_tuned = TunedThresholdClassifierCV(
+    ...     classifier, scoring="balanced_accuracy"
+    ... ).fit(X_train, y_train)
+    >>> print(
+    ...     f"Cut-off point found at {classifier_tuned.best_threshold_:.3f}"
+    ... )
+    Cut-off point found at 0.342
+    >>> print(classification_report(y_test, classifier_tuned.predict(X_test)))
+                  precision    recall  f1-score   support
+    
+               0       0.96      0.95      0.96       224
+               1       0.61      0.65      0.63        26
+    
+        accuracy                           0.92       250
+       macro avg       0.78      0.80      0.79       250
+    weighted avg       0.92      0.92      0.92       250
+    
+    """
+
+    _parameter_constraints: dict = {
+        **BaseThresholdClassifier._parameter_constraints,
+        "scoring": [
+            StrOptions(set(get_scorer_names())),
+            callable,
+            MutableMapping,
+        ],
+        "thresholds": [Interval(Integral, 1, None, closed="left"), "array-like"],
+        "cv": [
+            "cv_object",
+            StrOptions({"prefit"}),
+            Interval(RealNotInt, 0.0, 1.0, closed="neither"),
+        ],
+        "refit": ["boolean"],
+        "n_jobs": [Integral, None],
+        "random_state": ["random_state"],
+        "store_cv_results": ["boolean"],
+    }
+
+    def __init__(
+        self,
+        estimator,
+        *,
+        scoring="balanced_accuracy",
+        response_method="auto",
+        thresholds=100,
+        cv=None,
+        refit=True,
+        n_jobs=None,
+        random_state=None,
+        store_cv_results=False,
+    ):
+        super().__init__(estimator=estimator, response_method=response_method)
+        self.scoring = scoring
+        self.thresholds = thresholds
+        self.cv = cv
+        self.refit = refit
+        self.n_jobs = n_jobs
+        self.random_state = random_state
+        self.store_cv_results = store_cv_results
+
+    def _fit(self, X, y, **params):
+        """Fit the classifier and post-tune the decision threshold.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Training data.
+
+        y : array-like of shape (n_samples,)
+            Target values.
+
+        **params : dict
+            Parameters to pass to the `fit` method of the underlying
+            classifier and to the `scoring` scorer.
+
+        Returns
+        -------
+        self : object
+            Returns an instance of self.
+        """
+        if isinstance(self.cv, Real) and 0 < self.cv < 1:
+            cv = StratifiedShuffleSplit(
+                n_splits=1, test_size=self.cv, random_state=self.random_state
+            )
+        elif self.cv == "prefit":
+            if self.refit is True:
+                raise ValueError("When cv='prefit', refit cannot be True.")
+            try:
+                check_is_fitted(self.estimator, "classes_")
+            except NotFittedError as exc:
+                raise NotFittedError(
+                    """When cv='prefit', `estimator` must be fitted."""
+                ) from exc
+            cv = self.cv
+        else:
+            cv = check_cv(self.cv, y=y, classifier=True)
+            if self.refit is False and cv.get_n_splits() > 1:
+                raise ValueError("When cv has several folds, refit cannot be False.")
+
+        routed_params = process_routing(self, "fit", **params)
+        self._curve_scorer = self._get_curve_scorer()
+
+        # in the following block, we:
+        # - define the final classifier `self.estimator_` and train it if necessary
+        # - define `classifier` to be used to post-tune the decision threshold
+        # - define `split` to be used to fit/score `classifier`
+        if cv == "prefit":
+            self.estimator_ = self.estimator
+            classifier = self.estimator_
+            splits = [(None, range(_num_samples(X)))]
+        else:
+            self.estimator_ = clone(self.estimator)
+            classifier = clone(self.estimator)
+            splits = cv.split(X, y, **routed_params.splitter.split)
+
+            if self.refit:
+                # train on the whole dataset
+                X_train, y_train, fit_params_train = X, y, routed_params.estimator.fit
+            else:
+                # single split cross-validation
+                train_idx, _ = next(cv.split(X, y, **routed_params.splitter.split))
+                X_train = _safe_indexing(X, train_idx)
+                y_train = _safe_indexing(y, train_idx)
+                fit_params_train = _check_method_params(
+                    X, routed_params.estimator.fit, indices=train_idx
+                )
+
+            self.estimator_.fit(X_train, y_train, **fit_params_train)
+
+        cv_scores, cv_thresholds = zip(
+            *Parallel(n_jobs=self.n_jobs)(
+                delayed(_fit_and_score_over_thresholds)(
+                    clone(classifier) if cv != "prefit" else classifier,
+                    X,
+                    y,
+                    fit_params=routed_params.estimator.fit,
+                    train_idx=train_idx,
+                    val_idx=val_idx,
+                    curve_scorer=self._curve_scorer,
+                    score_params=routed_params.scorer.score,
+                )
+                for train_idx, val_idx in splits
+            )
+        )
+
+        if any(np.isclose(th[0], th[-1]) for th in cv_thresholds):
+            raise ValueError(
+                "The provided estimator makes constant predictions. Therefore, it is "
+                "impossible to optimize the decision threshold."
+            )
+
+        # find the global min and max thresholds across all folds
+        min_threshold = min(
+            split_thresholds.min() for split_thresholds in cv_thresholds
+        )
+        max_threshold = max(
+            split_thresholds.max() for split_thresholds in cv_thresholds
+        )
+        if isinstance(self.thresholds, Integral):
+            decision_thresholds = np.linspace(
+                min_threshold, max_threshold, num=self.thresholds
+            )
+        else:
+            decision_thresholds = np.asarray(self.thresholds)
+
+        objective_scores = _mean_interpolated_score(
+            decision_thresholds, cv_thresholds, cv_scores
+        )
+        best_idx = objective_scores.argmax()
+        self.best_score_ = objective_scores[best_idx]
+        self.best_threshold_ = decision_thresholds[best_idx]
+        if self.store_cv_results:
+            self.cv_results_ = {
+                "thresholds": decision_thresholds,
+                "scores": objective_scores,
+            }
+
+        return self
+
+    def predict(self, X):
+        """Predict the target of new samples.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            The samples, as accepted by `estimator.predict`.
+
+        Returns
+        -------
+        class_labels : ndarray of shape (n_samples,)
+            The predicted class.
+        """
+        check_is_fitted(self, "estimator_")
+        pos_label = self._curve_scorer._get_pos_label()
+        y_score, _ = _get_response_values_binary(
+            self.estimator_,
+            X,
+            self._get_response_method(),
+            pos_label=pos_label,
+        )
+
+        return _threshold_scores_to_class_labels(
+            y_score, self.best_threshold_, self.classes_, pos_label
+        )
+
+    def get_metadata_routing(self):
+        """Get metadata routing of this object.
+
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
+
+        Returns
+        -------
+        routing : MetadataRouter
+            A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
+            routing information.
+        """
+        router = (
+            MetadataRouter(owner=self.__class__.__name__)
+            .add(
+                estimator=self.estimator,
+                method_mapping=MethodMapping().add(callee="fit", caller="fit"),
+            )
+            .add(
+                splitter=self.cv,
+                method_mapping=MethodMapping().add(callee="split", caller="fit"),
+            )
+            .add(
+                scorer=self._get_curve_scorer(),
+                method_mapping=MethodMapping().add(callee="score", caller="fit"),
+            )
+        )
+        return router
+
+    def _get_curve_scorer(self):
+        """Get the curve scorer based on the objective metric used."""
+        scoring = check_scoring(self.estimator, scoring=self.scoring)
+        curve_scorer = _CurveScorer.from_scorer(
+            scoring, self._get_response_method(), self.thresholds
+        )
+        return curve_scorer
diff --git a/sklearn/model_selection/_plot.py b/sklearn/model_selection/_plot.py
new file mode 100644
index 0000000000000..a69c8f455bd41
--- /dev/null
+++ b/sklearn/model_selection/_plot.py
@@ -0,0 +1,885 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+import numpy as np
+
+from ..utils._optional_dependencies import check_matplotlib_support
+from ..utils._plotting import _interval_max_min_ratio, _validate_score_name
+from ._validation import learning_curve, validation_curve
+
+
+class _BaseCurveDisplay:
+    def _plot_curve(
+        self,
+        x_data,
+        *,
+        ax=None,
+        negate_score=False,
+        score_name=None,
+        score_type="test",
+        std_display_style="fill_between",
+        line_kw=None,
+        fill_between_kw=None,
+        errorbar_kw=None,
+    ):
+        check_matplotlib_support(f"{self.__class__.__name__}.plot")
+
+        import matplotlib.pyplot as plt
+
+        if ax is None:
+            _, ax = plt.subplots()
+
+        if negate_score:
+            train_scores, test_scores = -self.train_scores, -self.test_scores
+        else:
+            train_scores, test_scores = self.train_scores, self.test_scores
+
+        if std_display_style not in ("errorbar", "fill_between", None):
+            raise ValueError(
+                f"Unknown std_display_style: {std_display_style}. Should be one of"
+                " 'errorbar', 'fill_between', or None."
+            )
+
+        if score_type not in ("test", "train", "both"):
+            raise ValueError(
+                f"Unknown score_type: {score_type}. Should be one of 'test', "
+                "'train', or 'both'."
+            )
+
+        if score_type == "train":
+            scores = {"Train": train_scores}
+        elif score_type == "test":
+            scores = {"Test": test_scores}
+        else:  # score_type == "both"
+            scores = {"Train": train_scores, "Test": test_scores}
+
+        if std_display_style in ("fill_between", None):
+            # plot the mean score
+            if line_kw is None:
+                line_kw = {}
+
+            self.lines_ = []
+            for line_label, score in scores.items():
+                self.lines_.append(
+                    *ax.plot(
+                        x_data,
+                        score.mean(axis=1),
+                        label=line_label,
+                        **line_kw,
+                    )
+                )
+            self.errorbar_ = None
+            self.fill_between_ = None  # overwritten below by fill_between
+
+        if std_display_style == "errorbar":
+            if errorbar_kw is None:
+                errorbar_kw = {}
+
+            self.errorbar_ = []
+            for line_label, score in scores.items():
+                self.errorbar_.append(
+                    ax.errorbar(
+                        x_data,
+                        score.mean(axis=1),
+                        score.std(axis=1),
+                        label=line_label,
+                        **errorbar_kw,
+                    )
+                )
+            self.lines_, self.fill_between_ = None, None
+        elif std_display_style == "fill_between":
+            if fill_between_kw is None:
+                fill_between_kw = {}
+            default_fill_between_kw = {"alpha": 0.5}
+            fill_between_kw = {**default_fill_between_kw, **fill_between_kw}
+
+            self.fill_between_ = []
+            for line_label, score in scores.items():
+                self.fill_between_.append(
+                    ax.fill_between(
+                        x_data,
+                        score.mean(axis=1) - score.std(axis=1),
+                        score.mean(axis=1) + score.std(axis=1),
+                        **fill_between_kw,
+                    )
+                )
+
+        score_name = self.score_name if score_name is None else score_name
+
+        ax.legend()
+
+        # We found that a ratio, smaller or bigger than 5, between the largest and
+        # smallest gap of the x values is a good indicator to choose between linear
+        # and log scale.
+        if _interval_max_min_ratio(x_data) > 5:
+            xscale = "symlog" if x_data.min() <= 0 else "log"
+        else:
+            xscale = "linear"
+
+        ax.set_xscale(xscale)
+        ax.set_ylabel(f"{score_name}")
+
+        self.ax_ = ax
+        self.figure_ = ax.figure
+
+
+class LearningCurveDisplay(_BaseCurveDisplay):
+    """Learning Curve visualization.
+
+    It is recommended to use
+    :meth:`~sklearn.model_selection.LearningCurveDisplay.from_estimator` to
+    create a :class:`~sklearn.model_selection.LearningCurveDisplay` instance.
+    All parameters are stored as attributes.
+
+    Read more in the :ref:`User Guide ` for general information
+    about the visualization API and
+    :ref:`detailed documentation ` regarding the learning
+    curve visualization.
+
+    .. versionadded:: 1.2
+
+    Parameters
+    ----------
+    train_sizes : ndarray of shape (n_unique_ticks,)
+        Numbers of training examples that has been used to generate the
+        learning curve.
+
+    train_scores : ndarray of shape (n_ticks, n_cv_folds)
+        Scores on training sets.
+
+    test_scores : ndarray of shape (n_ticks, n_cv_folds)
+        Scores on test set.
+
+    score_name : str, default=None
+        The name of the score used in `learning_curve`. It will override the name
+        inferred from the `scoring` parameter. If `score` is `None`, we use `"Score"` if
+        `negate_score` is `False` and `"Negative score"` otherwise. If `scoring` is a
+        string or a callable, we infer the name. We replace `_` by spaces and capitalize
+        the first letter. We remove `neg_` and replace it by `"Negative"` if
+        `negate_score` is `False` or just remove it otherwise.
+
+    Attributes
+    ----------
+    ax_ : matplotlib Axes
+        Axes with the learning curve.
+
+    figure_ : matplotlib Figure
+        Figure containing the learning curve.
+
+    errorbar_ : list of matplotlib Artist or None
+        When the `std_display_style` is `"errorbar"`, this is a list of
+        `matplotlib.container.ErrorbarContainer` objects. If another style is
+        used, `errorbar_` is `None`.
+
+    lines_ : list of matplotlib Artist or None
+        When the `std_display_style` is `"fill_between"`, this is a list of
+        `matplotlib.lines.Line2D` objects corresponding to the mean train and
+        test scores. If another style is used, `line_` is `None`.
+
+    fill_between_ : list of matplotlib Artist or None
+        When the `std_display_style` is `"fill_between"`, this is a list of
+        `matplotlib.collections.PolyCollection` objects. If another style is
+        used, `fill_between_` is `None`.
+
+    See Also
+    --------
+    sklearn.model_selection.learning_curve : Compute the learning curve.
+
+    Examples
+    --------
+    >>> import matplotlib.pyplot as plt
+    >>> from sklearn.datasets import load_iris
+    >>> from sklearn.model_selection import LearningCurveDisplay, learning_curve
+    >>> from sklearn.tree import DecisionTreeClassifier
+    >>> X, y = load_iris(return_X_y=True)
+    >>> tree = DecisionTreeClassifier(random_state=0)
+    >>> train_sizes, train_scores, test_scores = learning_curve(
+    ...     tree, X, y)
+    >>> display = LearningCurveDisplay(train_sizes=train_sizes,
+    ...     train_scores=train_scores, test_scores=test_scores, score_name="Score")
+    >>> display.plot()
+    <...>
+    >>> plt.show()
+    """
+
+    def __init__(self, *, train_sizes, train_scores, test_scores, score_name=None):
+        self.train_sizes = train_sizes
+        self.train_scores = train_scores
+        self.test_scores = test_scores
+        self.score_name = score_name
+
+    def plot(
+        self,
+        ax=None,
+        *,
+        negate_score=False,
+        score_name=None,
+        score_type="both",
+        std_display_style="fill_between",
+        line_kw=None,
+        fill_between_kw=None,
+        errorbar_kw=None,
+    ):
+        """Plot visualization.
+
+        Parameters
+        ----------
+        ax : matplotlib Axes, default=None
+            Axes object to plot on. If `None`, a new figure and axes is
+            created.
+
+        negate_score : bool, default=False
+            Whether or not to negate the scores obtained through
+            :func:`~sklearn.model_selection.learning_curve`. This is
+            particularly useful when using the error denoted by `neg_*` in
+            `scikit-learn`.
+
+        score_name : str, default=None
+            The name of the score used to decorate the y-axis of the plot. It will
+            override the name inferred from the `scoring` parameter. If `score` is
+            `None`, we use `"Score"` if `negate_score` is `False` and `"Negative score"`
+            otherwise. If `scoring` is a string or a callable, we infer the name. We
+            replace `_` by spaces and capitalize the first letter. We remove `neg_` and
+            replace it by `"Negative"` if `negate_score` is
+            `False` or just remove it otherwise.
+
+        score_type : {"test", "train", "both"}, default="both"
+            The type of score to plot. Can be one of `"test"`, `"train"`, or
+            `"both"`.
+
+        std_display_style : {"errorbar", "fill_between"} or None, default="fill_between"
+            The style used to display the score standard deviation around the
+            mean score. If None, no standard deviation representation is
+            displayed.
+
+        line_kw : dict, default=None
+            Additional keyword arguments passed to the `plt.plot` used to draw
+            the mean score.
+
+        fill_between_kw : dict, default=None
+            Additional keyword arguments passed to the `plt.fill_between` used
+            to draw the score standard deviation.
+
+        errorbar_kw : dict, default=None
+            Additional keyword arguments passed to the `plt.errorbar` used to
+            draw mean score and standard deviation score.
+
+        Returns
+        -------
+        display : :class:`~sklearn.model_selection.LearningCurveDisplay`
+            Object that stores computed values.
+        """
+        self._plot_curve(
+            self.train_sizes,
+            ax=ax,
+            negate_score=negate_score,
+            score_name=score_name,
+            score_type=score_type,
+            std_display_style=std_display_style,
+            line_kw=line_kw,
+            fill_between_kw=fill_between_kw,
+            errorbar_kw=errorbar_kw,
+        )
+        self.ax_.set_xlabel("Number of samples in the training set")
+        return self
+
+    @classmethod
+    def from_estimator(
+        cls,
+        estimator,
+        X,
+        y,
+        *,
+        groups=None,
+        train_sizes=np.linspace(0.1, 1.0, 5),
+        cv=None,
+        scoring=None,
+        exploit_incremental_learning=False,
+        n_jobs=None,
+        pre_dispatch="all",
+        verbose=0,
+        shuffle=False,
+        random_state=None,
+        error_score=np.nan,
+        fit_params=None,
+        ax=None,
+        negate_score=False,
+        score_name=None,
+        score_type="both",
+        std_display_style="fill_between",
+        line_kw=None,
+        fill_between_kw=None,
+        errorbar_kw=None,
+    ):
+        """Create a learning curve display from an estimator.
+
+        Read more in the :ref:`User Guide ` for general
+        information about the visualization API and :ref:`detailed
+        documentation ` regarding the learning curve
+        visualization.
+
+        Parameters
+        ----------
+        estimator : object type that implements the "fit" and "predict" methods
+            An object of that type which is cloned for each validation.
+
+        X : array-like of shape (n_samples, n_features)
+            Training data, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
+
+        y : array-like of shape (n_samples,) or (n_samples, n_outputs) or None
+            Target relative to X for classification or regression;
+            None for unsupervised learning.
+
+        groups : array-like of shape (n_samples,), default=None
+            Group labels for the samples used while splitting the dataset into
+            train/test set. Only used in conjunction with a "Group" :term:`cv`
+            instance (e.g., :class:`GroupKFold`).
+
+        train_sizes : array-like of shape (n_ticks,), \
+                default=np.linspace(0.1, 1.0, 5)
+            Relative or absolute numbers of training examples that will be used
+            to generate the learning curve. If the dtype is float, it is
+            regarded as a fraction of the maximum size of the training set
+            (that is determined by the selected validation method), i.e. it has
+            to be within (0, 1]. Otherwise it is interpreted as absolute sizes
+            of the training sets. Note that for classification the number of
+            samples usually have to be big enough to contain at least one
+            sample from each class.
+
+        cv : int, cross-validation generator or an iterable, default=None
+            Determines the cross-validation splitting strategy.
+            Possible inputs for cv are:
+
+            - None, to use the default 5-fold cross validation,
+            - int, to specify the number of folds in a `(Stratified)KFold`,
+            - :term:`CV splitter`,
+            - An iterable yielding (train, test) splits as arrays of indices.
+
+            For int/None inputs, if the estimator is a classifier and `y` is
+            either binary or multiclass,
+            :class:`~sklearn.model_selection.StratifiedKFold` is used. In all
+            other cases, :class:`~sklearn.model_selection.KFold` is used. These
+            splitters are instantiated with `shuffle=False` so the splits will
+            be the same across calls.
+
+            Refer :ref:`User Guide ` for the various
+            cross-validation strategies that can be used here.
+
+        scoring : str or callable, default=None
+            The scoring method to use when calculating the learning curve. Options:
+
+            - str: see :ref:`scoring_string_names` for options.
+            - callable: a scorer callable object (e.g., function) with signature
+              ``scorer(estimator, X, y)``. See :ref:`scoring_callable` for details.
+            - `None`: the `estimator`'s
+              :ref:`default evaluation criterion ` is used.
+
+        exploit_incremental_learning : bool, default=False
+            If the estimator supports incremental learning, this will be
+            used to speed up fitting for different training set sizes.
+
+        n_jobs : int, default=None
+            Number of jobs to run in parallel. Training the estimator and
+            computing the score are parallelized over the different training
+            and test sets. `None` means 1 unless in a
+            :obj:`joblib.parallel_backend` context. `-1` means using all
+            processors. See :term:`Glossary ` for more details.
+
+        pre_dispatch : int or str, default='all'
+            Number of predispatched jobs for parallel execution (default is
+            all). The option can reduce the allocated memory. The str can
+            be an expression like '2*n_jobs'.
+
+        verbose : int, default=0
+            Controls the verbosity: the higher, the more messages.
+
+        shuffle : bool, default=False
+            Whether to shuffle training data before taking prefixes of it
+            based on`train_sizes`.
+
+        random_state : int, RandomState instance or None, default=None
+            Used when `shuffle` is True. Pass an int for reproducible
+            output across multiple function calls.
+            See :term:`Glossary `.
+
+        error_score : 'raise' or numeric, default=np.nan
+            Value to assign to the score if an error occurs in estimator
+            fitting. If set to 'raise', the error is raised. If a numeric value
+            is given, FitFailedWarning is raised.
+
+        fit_params : dict, default=None
+            Parameters to pass to the fit method of the estimator.
+
+        ax : matplotlib Axes, default=None
+            Axes object to plot on. If `None`, a new figure and axes is
+            created.
+
+        negate_score : bool, default=False
+            Whether or not to negate the scores obtained through
+            :func:`~sklearn.model_selection.learning_curve`. This is
+            particularly useful when using the error denoted by `neg_*` in
+            `scikit-learn`.
+
+        score_name : str, default=None
+            The name of the score used to decorate the y-axis of the plot. It will
+            override the name inferred from the `scoring` parameter. If `score` is
+            `None`, we use `"Score"` if `negate_score` is `False` and `"Negative score"`
+            otherwise. If `scoring` is a string or a callable, we infer the name. We
+            replace `_` by spaces and capitalize the first letter. We remove `neg_` and
+            replace it by `"Negative"` if `negate_score` is
+            `False` or just remove it otherwise.
+
+        score_type : {"test", "train", "both"}, default="both"
+            The type of score to plot. Can be one of `"test"`, `"train"`, or
+            `"both"`.
+
+        std_display_style : {"errorbar", "fill_between"} or None, default="fill_between"
+            The style used to display the score standard deviation around the
+            mean score. If `None`, no representation of the standard deviation
+            is displayed.
+
+        line_kw : dict, default=None
+            Additional keyword arguments passed to the `plt.plot` used to draw
+            the mean score.
+
+        fill_between_kw : dict, default=None
+            Additional keyword arguments passed to the `plt.fill_between` used
+            to draw the score standard deviation.
+
+        errorbar_kw : dict, default=None
+            Additional keyword arguments passed to the `plt.errorbar` used to
+            draw mean score and standard deviation score.
+
+        Returns
+        -------
+        display : :class:`~sklearn.model_selection.LearningCurveDisplay`
+            Object that stores computed values.
+
+        Examples
+        --------
+        >>> import matplotlib.pyplot as plt
+        >>> from sklearn.datasets import load_iris
+        >>> from sklearn.model_selection import LearningCurveDisplay
+        >>> from sklearn.tree import DecisionTreeClassifier
+        >>> X, y = load_iris(return_X_y=True)
+        >>> tree = DecisionTreeClassifier(random_state=0)
+        >>> LearningCurveDisplay.from_estimator(tree, X, y)
+        <...>
+        >>> plt.show()
+        """
+        check_matplotlib_support(f"{cls.__name__}.from_estimator")
+
+        score_name = _validate_score_name(score_name, scoring, negate_score)
+
+        train_sizes, train_scores, test_scores = learning_curve(
+            estimator,
+            X,
+            y,
+            groups=groups,
+            train_sizes=train_sizes,
+            cv=cv,
+            scoring=scoring,
+            exploit_incremental_learning=exploit_incremental_learning,
+            n_jobs=n_jobs,
+            pre_dispatch=pre_dispatch,
+            verbose=verbose,
+            shuffle=shuffle,
+            random_state=random_state,
+            error_score=error_score,
+            return_times=False,
+            fit_params=fit_params,
+        )
+
+        viz = cls(
+            train_sizes=train_sizes,
+            train_scores=train_scores,
+            test_scores=test_scores,
+            score_name=score_name,
+        )
+        return viz.plot(
+            ax=ax,
+            negate_score=negate_score,
+            score_type=score_type,
+            std_display_style=std_display_style,
+            line_kw=line_kw,
+            fill_between_kw=fill_between_kw,
+            errorbar_kw=errorbar_kw,
+        )
+
+
+class ValidationCurveDisplay(_BaseCurveDisplay):
+    """Validation Curve visualization.
+
+    It is recommended to use
+    :meth:`~sklearn.model_selection.ValidationCurveDisplay.from_estimator` to
+    create a :class:`~sklearn.model_selection.ValidationCurveDisplay` instance.
+    All parameters are stored as attributes.
+
+    Read more in the :ref:`User Guide ` for general information
+    about the visualization API and :ref:`detailed documentation
+    ` regarding the validation curve visualization.
+
+    .. versionadded:: 1.3
+
+    Parameters
+    ----------
+    param_name : str
+        Name of the parameter that has been varied.
+
+    param_range : array-like of shape (n_ticks,)
+        The values of the parameter that have been evaluated.
+
+    train_scores : ndarray of shape (n_ticks, n_cv_folds)
+        Scores on training sets.
+
+    test_scores : ndarray of shape (n_ticks, n_cv_folds)
+        Scores on test set.
+
+    score_name : str, default=None
+        The name of the score used in `validation_curve`. It will override the name
+        inferred from the `scoring` parameter. If `score` is `None`, we use `"Score"` if
+        `negate_score` is `False` and `"Negative score"` otherwise. If `scoring` is a
+        string or a callable, we infer the name. We replace `_` by spaces and capitalize
+        the first letter. We remove `neg_` and replace it by `"Negative"` if
+        `negate_score` is `False` or just remove it otherwise.
+
+    Attributes
+    ----------
+    ax_ : matplotlib Axes
+        Axes with the validation curve.
+
+    figure_ : matplotlib Figure
+        Figure containing the validation curve.
+
+    errorbar_ : list of matplotlib Artist or None
+        When the `std_display_style` is `"errorbar"`, this is a list of
+        `matplotlib.container.ErrorbarContainer` objects. If another style is
+        used, `errorbar_` is `None`.
+
+    lines_ : list of matplotlib Artist or None
+        When the `std_display_style` is `"fill_between"`, this is a list of
+        `matplotlib.lines.Line2D` objects corresponding to the mean train and
+        test scores. If another style is used, `line_` is `None`.
+
+    fill_between_ : list of matplotlib Artist or None
+        When the `std_display_style` is `"fill_between"`, this is a list of
+        `matplotlib.collections.PolyCollection` objects. If another style is
+        used, `fill_between_` is `None`.
+
+    See Also
+    --------
+    sklearn.model_selection.validation_curve : Compute the validation curve.
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> import matplotlib.pyplot as plt
+    >>> from sklearn.datasets import make_classification
+    >>> from sklearn.model_selection import ValidationCurveDisplay, validation_curve
+    >>> from sklearn.linear_model import LogisticRegression
+    >>> X, y = make_classification(n_samples=1_000, random_state=0)
+    >>> logistic_regression = LogisticRegression()
+    >>> param_name, param_range = "C", np.logspace(-8, 3, 10)
+    >>> train_scores, test_scores = validation_curve(
+    ...     logistic_regression, X, y, param_name=param_name, param_range=param_range
+    ... )
+    >>> display = ValidationCurveDisplay(
+    ...     param_name=param_name, param_range=param_range,
+    ...     train_scores=train_scores, test_scores=test_scores, score_name="Score"
+    ... )
+    >>> display.plot()
+    <...>
+    >>> plt.show()
+    """
+
+    def __init__(
+        self, *, param_name, param_range, train_scores, test_scores, score_name=None
+    ):
+        self.param_name = param_name
+        self.param_range = param_range
+        self.train_scores = train_scores
+        self.test_scores = test_scores
+        self.score_name = score_name
+
+    def plot(
+        self,
+        ax=None,
+        *,
+        negate_score=False,
+        score_name=None,
+        score_type="both",
+        std_display_style="fill_between",
+        line_kw=None,
+        fill_between_kw=None,
+        errorbar_kw=None,
+    ):
+        """Plot visualization.
+
+        Parameters
+        ----------
+        ax : matplotlib Axes, default=None
+            Axes object to plot on. If `None`, a new figure and axes is
+            created.
+
+        negate_score : bool, default=False
+            Whether or not to negate the scores obtained through
+            :func:`~sklearn.model_selection.validation_curve`. This is
+            particularly useful when using the error denoted by `neg_*` in
+            `scikit-learn`.
+
+        score_name : str, default=None
+            The name of the score used to decorate the y-axis of the plot. It will
+            override the name inferred from the `scoring` parameter. If `score` is
+            `None`, we use `"Score"` if `negate_score` is `False` and `"Negative score"`
+            otherwise. If `scoring` is a string or a callable, we infer the name. We
+            replace `_` by spaces and capitalize the first letter. We remove `neg_` and
+            replace it by `"Negative"` if `negate_score` is
+            `False` or just remove it otherwise.
+
+        score_type : {"test", "train", "both"}, default="both"
+            The type of score to plot. Can be one of `"test"`, `"train"`, or
+            `"both"`.
+
+        std_display_style : {"errorbar", "fill_between"} or None, default="fill_between"
+            The style used to display the score standard deviation around the
+            mean score. If None, no standard deviation representation is
+            displayed.
+
+        line_kw : dict, default=None
+            Additional keyword arguments passed to the `plt.plot` used to draw
+            the mean score.
+
+        fill_between_kw : dict, default=None
+            Additional keyword arguments passed to the `plt.fill_between` used
+            to draw the score standard deviation.
+
+        errorbar_kw : dict, default=None
+            Additional keyword arguments passed to the `plt.errorbar` used to
+            draw mean score and standard deviation score.
+
+        Returns
+        -------
+        display : :class:`~sklearn.model_selection.ValidationCurveDisplay`
+            Object that stores computed values.
+        """
+        self._plot_curve(
+            self.param_range,
+            ax=ax,
+            negate_score=negate_score,
+            score_name=score_name,
+            score_type=score_type,
+            std_display_style=std_display_style,
+            line_kw=line_kw,
+            fill_between_kw=fill_between_kw,
+            errorbar_kw=errorbar_kw,
+        )
+        self.ax_.set_xlabel(f"{self.param_name}")
+        return self
+
+    @classmethod
+    def from_estimator(
+        cls,
+        estimator,
+        X,
+        y,
+        *,
+        param_name,
+        param_range,
+        groups=None,
+        cv=None,
+        scoring=None,
+        n_jobs=None,
+        pre_dispatch="all",
+        verbose=0,
+        error_score=np.nan,
+        fit_params=None,
+        ax=None,
+        negate_score=False,
+        score_name=None,
+        score_type="both",
+        std_display_style="fill_between",
+        line_kw=None,
+        fill_between_kw=None,
+        errorbar_kw=None,
+    ):
+        """Create a validation curve display from an estimator.
+
+        Read more in the :ref:`User Guide ` for general
+        information about the visualization API and :ref:`detailed
+        documentation ` regarding the validation curve
+        visualization.
+
+        Parameters
+        ----------
+        estimator : object type that implements the "fit" and "predict" methods
+            An object of that type which is cloned for each validation.
+
+        X : array-like of shape (n_samples, n_features)
+            Training data, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
+
+        y : array-like of shape (n_samples,) or (n_samples, n_outputs) or None
+            Target relative to X for classification or regression;
+            None for unsupervised learning.
+
+        param_name : str
+            Name of the parameter that will be varied.
+
+        param_range : array-like of shape (n_values,)
+            The values of the parameter that will be evaluated.
+
+        groups : array-like of shape (n_samples,), default=None
+            Group labels for the samples used while splitting the dataset into
+            train/test set. Only used in conjunction with a "Group" :term:`cv`
+            instance (e.g., :class:`GroupKFold`).
+
+        cv : int, cross-validation generator or an iterable, default=None
+            Determines the cross-validation splitting strategy.
+            Possible inputs for cv are:
+
+            - None, to use the default 5-fold cross validation,
+            - int, to specify the number of folds in a `(Stratified)KFold`,
+            - :term:`CV splitter`,
+            - An iterable yielding (train, test) splits as arrays of indices.
+
+            For int/None inputs, if the estimator is a classifier and `y` is
+            either binary or multiclass,
+            :class:`~sklearn.model_selection.StratifiedKFold` is used. In all
+            other cases, :class:`~sklearn.model_selection.KFold` is used. These
+            splitters are instantiated with `shuffle=False` so the splits will
+            be the same across calls.
+
+            Refer :ref:`User Guide ` for the various
+            cross-validation strategies that can be used here.
+
+        scoring : str or callable, default=None
+            Scoring method to use when computing the validation curve. Options:
+
+            - str: see :ref:`scoring_string_names` for options.
+            - callable: a scorer callable object (e.g., function) with signature
+              ``scorer(estimator, X, y)``. See :ref:`scoring_callable` for details.
+            - `None`: the `estimator`'s
+              :ref:`default evaluation criterion ` is used.
+
+        n_jobs : int, default=None
+            Number of jobs to run in parallel. Training the estimator and
+            computing the score are parallelized over the different training
+            and test sets. `None` means 1 unless in a
+            :obj:`joblib.parallel_backend` context. `-1` means using all
+            processors. See :term:`Glossary ` for more details.
+
+        pre_dispatch : int or str, default='all'
+            Number of predispatched jobs for parallel execution (default is
+            all). The option can reduce the allocated memory. The str can
+            be an expression like '2*n_jobs'.
+
+        verbose : int, default=0
+            Controls the verbosity: the higher, the more messages.
+
+        error_score : 'raise' or numeric, default=np.nan
+            Value to assign to the score if an error occurs in estimator
+            fitting. If set to 'raise', the error is raised. If a numeric value
+            is given, FitFailedWarning is raised.
+
+        fit_params : dict, default=None
+            Parameters to pass to the fit method of the estimator.
+
+        ax : matplotlib Axes, default=None
+            Axes object to plot on. If `None`, a new figure and axes is
+            created.
+
+        negate_score : bool, default=False
+            Whether or not to negate the scores obtained through
+            :func:`~sklearn.model_selection.validation_curve`. This is
+            particularly useful when using the error denoted by `neg_*` in
+            `scikit-learn`.
+
+        score_name : str, default=None
+            The name of the score used to decorate the y-axis of the plot. It will
+            override the name inferred from the `scoring` parameter. If `score` is
+            `None`, we use `"Score"` if `negate_score` is `False` and `"Negative score"`
+            otherwise. If `scoring` is a string or a callable, we infer the name. We
+            replace `_` by spaces and capitalize the first letter. We remove `neg_` and
+            replace it by `"Negative"` if `negate_score` is
+            `False` or just remove it otherwise.
+
+        score_type : {"test", "train", "both"}, default="both"
+            The type of score to plot. Can be one of `"test"`, `"train"`, or
+            `"both"`.
+
+        std_display_style : {"errorbar", "fill_between"} or None, default="fill_between"
+            The style used to display the score standard deviation around the
+            mean score. If `None`, no representation of the standard deviation
+            is displayed.
+
+        line_kw : dict, default=None
+            Additional keyword arguments passed to the `plt.plot` used to draw
+            the mean score.
+
+        fill_between_kw : dict, default=None
+            Additional keyword arguments passed to the `plt.fill_between` used
+            to draw the score standard deviation.
+
+        errorbar_kw : dict, default=None
+            Additional keyword arguments passed to the `plt.errorbar` used to
+            draw mean score and standard deviation score.
+
+        Returns
+        -------
+        display : :class:`~sklearn.model_selection.ValidationCurveDisplay`
+            Object that stores computed values.
+
+        Examples
+        --------
+        >>> import numpy as np
+        >>> import matplotlib.pyplot as plt
+        >>> from sklearn.datasets import make_classification
+        >>> from sklearn.model_selection import ValidationCurveDisplay
+        >>> from sklearn.linear_model import LogisticRegression
+        >>> X, y = make_classification(n_samples=1_000, random_state=0)
+        >>> logistic_regression = LogisticRegression()
+        >>> param_name, param_range = "C", np.logspace(-8, 3, 10)
+        >>> ValidationCurveDisplay.from_estimator(
+        ...     logistic_regression, X, y, param_name=param_name,
+        ...     param_range=param_range,
+        ... )
+        <...>
+        >>> plt.show()
+        """
+        check_matplotlib_support(f"{cls.__name__}.from_estimator")
+
+        score_name = _validate_score_name(score_name, scoring, negate_score)
+
+        train_scores, test_scores = validation_curve(
+            estimator,
+            X,
+            y,
+            param_name=param_name,
+            param_range=param_range,
+            groups=groups,
+            cv=cv,
+            scoring=scoring,
+            n_jobs=n_jobs,
+            pre_dispatch=pre_dispatch,
+            verbose=verbose,
+            error_score=error_score,
+            fit_params=fit_params,
+        )
+
+        viz = cls(
+            param_name=param_name,
+            param_range=np.asarray(param_range),
+            train_scores=train_scores,
+            test_scores=test_scores,
+            score_name=score_name,
+        )
+        return viz.plot(
+            ax=ax,
+            negate_score=negate_score,
+            score_type=score_type,
+            std_display_style=std_display_style,
+            line_kw=line_kw,
+            fill_between_kw=fill_between_kw,
+            errorbar_kw=errorbar_kw,
+        )
diff --git a/sklearn/model_selection/_search.py b/sklearn/model_selection/_search.py
index abe3b87488d8c..5bd3f81195631 100644
--- a/sklearn/model_selection/_search.py
+++ b/sklearn/model_selection/_search.py
@@ -3,49 +3,58 @@
 parameters of an estimator.
 """
 
-# Author: Alexandre Gramfort ,
-#         Gael Varoquaux 
-#         Andreas Mueller 
-#         Olivier Grisel 
-#         Raghav RV 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-from abc import ABCMeta, abstractmethod
-from collections import defaultdict
-from collections.abc import Mapping, Sequence, Iterable
-from functools import partial, reduce
-from itertools import product
 import numbers
 import operator
 import time
 import warnings
+from abc import ABCMeta, abstractmethod
+from collections import defaultdict
+from collections.abc import Iterable, Mapping, Sequence
+from copy import deepcopy
+from functools import partial, reduce
+from inspect import signature
+from itertools import product
 
 import numpy as np
 from numpy.ma import MaskedArray
 from scipy.stats import rankdata
 
-from ..base import BaseEstimator, is_classifier, clone
-from ..base import MetaEstimatorMixin
-from ._split import check_cv
-from ._validation import _fit_and_score
-from ._validation import _aggregate_score_dicts
-from ._validation import _insert_error_scores
-from ._validation import _normalize_score_results
+from ..base import BaseEstimator, MetaEstimatorMixin, _fit_context, clone, is_classifier
 from ..exceptions import NotFittedError
-from joblib import Parallel
-from ..utils import check_random_state
-from ..utils.random import sample_without_replacement
-from ..utils._tags import _safe_tags
-from ..utils.validation import indexable, check_is_fitted, _check_fit_params
-from ..utils.validation import _deprecate_positional_args
-from ..utils.metaestimators import if_delegate_has_method
-from ..utils.fixes import delayed
-from ..metrics._scorer import _check_multimetric_scoring
 from ..metrics import check_scoring
-from ..utils import deprecated
+from ..metrics._scorer import (
+    _check_multimetric_scoring,
+    _MultimetricScorer,
+    get_scorer_names,
+)
+from ..utils import Bunch, check_random_state
+from ..utils._param_validation import HasMethods, Interval, StrOptions
+from ..utils._repr_html.estimator import _VisualBlock
+from ..utils._tags import get_tags
+from ..utils.metadata_routing import (
+    MetadataRouter,
+    MethodMapping,
+    _raise_for_params,
+    _routing_enabled,
+    process_routing,
+)
+from ..utils.metaestimators import available_if
+from ..utils.parallel import Parallel, delayed
+from ..utils.random import sample_without_replacement
+from ..utils.validation import _check_method_params, check_is_fitted, indexable
+from ._split import check_cv
+from ._validation import (
+    _aggregate_score_dicts,
+    _fit_and_score,
+    _insert_error_scores,
+    _normalize_score_results,
+    _warn_or_raise_about_fit_failures,
+)
 
-__all__ = ['GridSearchCV', 'ParameterGrid', 'fit_grid_point',
-           'ParameterSampler', 'RandomizedSearchCV']
+__all__ = ["GridSearchCV", "ParameterGrid", "ParameterSampler", "RandomizedSearchCV"]
 
 
 class ParameterGrid:
@@ -94,8 +103,10 @@ class ParameterGrid:
 
     def __init__(self, param_grid):
         if not isinstance(param_grid, (Mapping, Iterable)):
-            raise TypeError('Parameter grid is not a dict or '
-                            'a list ({!r})'.format(param_grid))
+            raise TypeError(
+                f"Parameter grid should be a dict or a list, got: {param_grid!r} of"
+                f" type {type(param_grid).__name__}"
+            )
 
         if isinstance(param_grid, Mapping):
             # wrap dictionary in a singleton list to support either dict
@@ -105,13 +116,27 @@ def __init__(self, param_grid):
         # check if all entries are dictionaries of lists
         for grid in param_grid:
             if not isinstance(grid, dict):
-                raise TypeError('Parameter grid is not a '
-                                'dict ({!r})'.format(grid))
-            for key in grid:
-                if not isinstance(grid[key], Iterable):
-                    raise TypeError('Parameter grid value is not iterable '
-                                    '(key={!r}, value={!r})'
-                                    .format(key, grid[key]))
+                raise TypeError(f"Parameter grid is not a dict ({grid!r})")
+            for key, value in grid.items():
+                if isinstance(value, np.ndarray) and value.ndim > 1:
+                    raise ValueError(
+                        f"Parameter array for {key!r} should be one-dimensional, got:"
+                        f" {value!r} with shape {value.shape}"
+                    )
+                if isinstance(value, str) or not isinstance(
+                    value, (np.ndarray, Sequence)
+                ):
+                    raise TypeError(
+                        f"Parameter grid for parameter {key!r} needs to be a list or a"
+                        f" numpy array, but got {value!r} (of type "
+                        f"{type(value).__name__}) instead. Single values "
+                        "need to be wrapped in a list with one element."
+                    )
+                if len(value) == 0:
+                    raise ValueError(
+                        f"Parameter grid for parameter {key!r} need "
+                        f"to be a non-empty sequence, got: {value!r}"
+                    )
 
         self.param_grid = param_grid
 
@@ -137,10 +162,11 @@ def __iter__(self):
 
     def __len__(self):
         """Number of points on the grid."""
-        # Product function that can handle iterables (np.product can't).
+        # Product function that can handle iterables (np.prod can't).
         product = partial(reduce, operator.mul)
-        return sum(product(len(v) for v in p.values()) if p else 1
-                   for p in self.param_grid)
+        return sum(
+            product(len(v) for v in p.values()) if p else 1 for p in self.param_grid
+        )
 
     def __getitem__(self, ind):
         """Get the parameters that would be ``ind``th in iteration
@@ -169,7 +195,7 @@ def __getitem__(self, ind):
             # Reverse so most frequent cycling parameter comes first
             keys, values_lists = zip(*sorted(sub_grid.items())[::-1])
             sizes = [len(v_list) for v_list in values_lists]
-            total = np.product(sizes)
+            total = np.prod(sizes)
 
             if ind >= total:
                 # Try the next grid
@@ -181,7 +207,7 @@ def __getitem__(self, ind):
                     out[key] = v_list[offset]
                 return out
 
-        raise IndexError('ParameterGrid index out of range')
+        raise IndexError("ParameterGrid index out of range")
 
 
 class ParameterSampler:
@@ -239,11 +265,14 @@ class ParameterSampler:
     ...                  {'b': 1.038159, 'a': 2}]
     True
     """
-    @_deprecate_positional_args
+
     def __init__(self, param_distributions, n_iter, *, random_state=None):
         if not isinstance(param_distributions, (Mapping, Iterable)):
-            raise TypeError('Parameter distribution is not a dict or '
-                            'a list ({!r})'.format(param_distributions))
+            raise TypeError(
+                "Parameter distribution is not a dict or a list,"
+                f" got: {param_distributions!r} of type "
+                f"{type(param_distributions).__name__}"
+            )
 
         if isinstance(param_distributions, Mapping):
             # wrap dictionary in a singleton list to support either dict
@@ -252,14 +281,17 @@ def __init__(self, param_distributions, n_iter, *, random_state=None):
 
         for dist in param_distributions:
             if not isinstance(dist, dict):
-                raise TypeError('Parameter distribution is not a '
-                                'dict ({!r})'.format(dist))
+                raise TypeError(
+                    "Parameter distribution is not a dict ({!r})".format(dist)
+                )
             for key in dist:
-                if (not isinstance(dist[key], Iterable)
-                        and not hasattr(dist[key], 'rvs')):
-                    raise TypeError('Parameter value is not iterable '
-                                    'or distribution (key={!r}, value={!r})'
-                                    .format(key, dist[key]))
+                if not isinstance(dist[key], Iterable) and not hasattr(
+                    dist[key], "rvs"
+                ):
+                    raise TypeError(
+                        f"Parameter grid for parameter {key!r} is not iterable "
+                        f"or a distribution (value={dist[key]})"
+                    )
         self.n_iter = n_iter
         self.random_state = random_state
         self.param_distributions = param_distributions
@@ -283,13 +315,13 @@ def __iter__(self):
 
             if grid_size < n_iter:
                 warnings.warn(
-                    'The total space of parameters %d is smaller '
-                    'than n_iter=%d. Running %d iterations. For exhaustive '
-                    'searches, use GridSearchCV.'
-                    % (grid_size, self.n_iter, grid_size), UserWarning)
+                    "The total space of parameters %d is smaller "
+                    "than n_iter=%d. Running %d iterations. For exhaustive "
+                    "searches, use GridSearchCV." % (grid_size, self.n_iter, grid_size),
+                    UserWarning,
+                )
                 n_iter = grid_size
-            for i in sample_without_replacement(grid_size, n_iter,
-                                                random_state=rng):
+            for i in sample_without_replacement(grid_size, n_iter, random_state=rng):
                 yield param_grid[i]
 
         else:
@@ -314,110 +346,126 @@ def __len__(self):
             return self.n_iter
 
 
-# FIXME Remove fit_grid_point in 1.0
-@deprecated(
-    "fit_grid_point is deprecated in version 0.23 "
-    "and will be removed in version 1.0 (renaming of 0.25)"
-)
-def fit_grid_point(X, y, estimator, parameters, train, test, scorer,
-                   verbose, error_score=np.nan, **fit_params):
-    """Run fit on one set of parameters.
+def _check_refit(search_cv, attr):
+    if not search_cv.refit:
+        raise AttributeError(
+            f"This {type(search_cv).__name__} instance was initialized with "
+            f"`refit=False`. {attr} is available only after refitting on the best "
+            "parameters. You can refit an estimator manually using the "
+            "`best_params_` attribute"
+        )
 
-    Parameters
-    ----------
-    X : array-like, sparse matrix or list
-        Input data.
 
-    y : array-like or None
-        Targets for input data.
+def _search_estimator_has(attr):
+    """Check if we can delegate a method to the underlying estimator.
 
-    estimator : estimator object
-        A object of that type is instantiated for each grid point.
-        This is assumed to implement the scikit-learn estimator interface.
-        Either estimator needs to provide a ``score`` function,
-        or ``scoring`` must be passed.
+    Calling a prediction method will only be available if `refit=True`. In
+    such case, we check first the fitted best estimator. If it is not
+    fitted, we check the unfitted estimator.
 
-    parameters : dict
-        Parameters to be set on estimator for this grid point.
+    Checking the unfitted estimator allows to use `hasattr` on the `SearchCV`
+    instance even before calling `fit`.
+    """
 
-    train : ndarray, dtype int or bool
-        Boolean mask or indices for training set.
+    def check(self):
+        _check_refit(self, attr)
+        if hasattr(self, "best_estimator_"):
+            # raise an AttributeError if `attr` does not exist
+            getattr(self.best_estimator_, attr)
+            return True
+        # raise an AttributeError if `attr` does not exist
+        getattr(self.estimator, attr)
+        return True
 
-    test : ndarray, dtype int or bool
-        Boolean mask or indices for test set.
+    return check
 
-    scorer : callable or None
-        The scorer callable object / function must have its signature as
-        ``scorer(estimator, X, y)``.
 
-        If ``None`` the estimator's score method is used.
+def _yield_masked_array_for_each_param(candidate_params):
+    """
+    Yield a masked array for each candidate param.
 
-    verbose : int
-        Verbosity level.
+    `candidate_params` is a sequence of params which were used in
+    a `GridSearchCV`. We use masked arrays for the results, as not
+    all params are necessarily present in each element of
+    `candidate_params`. For example, if using `GridSearchCV` with
+    a `SVC` model, then one might search over params like:
 
-    **fit_params : kwargs
-        Additional parameter passed to the fit function of the estimator.
+        - kernel=["rbf"], gamma=[0.1, 1]
+        - kernel=["poly"], degree=[1, 2]
 
-    error_score : 'raise' or numeric, default=np.nan
-        Value to assign to the score if an error occurs in estimator fitting.
-        If set to 'raise', the error is raised. If a numeric value is given,
-        FitFailedWarning is raised. This parameter does not affect the refit
-        step, which will always raise the error.
+    and then param `'gamma'` would not be present in entries of
+    `candidate_params` corresponding to `kernel='poly'`.
+    """
+    n_candidates = len(candidate_params)
+    param_results = defaultdict(dict)
 
-    Returns
-    -------
-    score : float
-         Score of this parameter setting on given test split.
+    for cand_idx, params in enumerate(candidate_params):
+        for name, value in params.items():
+            param_results["param_%s" % name][cand_idx] = value
 
-    parameters : dict
-        The parameters that have been evaluated.
+    for key, param_result in param_results.items():
+        param_list = list(param_result.values())
+        try:
+            arr = np.array(param_list)
+        except ValueError:
+            # This can happen when param_list contains lists of different
+            # lengths, for example:
+            # param_list=[[1], [2, 3]]
+            arr_dtype = np.dtype(object)
+        else:
+            # There are two cases when we don't use the automatically inferred
+            # dtype when creating the array and we use object instead:
+            # - string dtype
+            # - when array.ndim > 1, that means that param_list was something
+            #   like a list of same-size sequences, which gets turned into a
+            #   multi-dimensional array but we want a 1d array
+            arr_dtype = arr.dtype if arr.dtype.kind != "U" and arr.ndim == 1 else object
 
-    n_samples_test : int
-        Number of test samples in this split.
-    """
-    # NOTE we are not using the return value as the scorer by itself should be
-    # validated before. We use check_scoring only to reject multimetric scorer
-    check_scoring(estimator, scorer)
-    results = _fit_and_score(estimator, X, y, scorer, train,
-                             test, verbose, parameters,
-                             fit_params=fit_params,
-                             return_n_test_samples=True,
-                             error_score=error_score)
-    return results["test_scores"], parameters, results["n_test_samples"]
-
-
-def _check_param_grid(param_grid):
-    if hasattr(param_grid, 'items'):
-        param_grid = [param_grid]
-
-    for p in param_grid:
-        for name, v in p.items():
-            if isinstance(v, np.ndarray) and v.ndim > 1:
-                raise ValueError("Parameter array should be one-dimensional.")
-
-            if (isinstance(v, str) or
-                    not isinstance(v, (np.ndarray, Sequence))):
-                raise ValueError("Parameter grid for parameter ({0}) needs to"
-                                 " be a list or numpy array, but got ({1})."
-                                 " Single values need to be wrapped in a list"
-                                 " with one element.".format(name, type(v)))
-
-            if len(v) == 0:
-                raise ValueError("Parameter values for parameter ({0}) need "
-                                 "to be a non-empty sequence.".format(name))
+        # Use one MaskedArray and mask all the places where the param is not
+        # applicable for that candidate (which may not contain all the params).
+        ma = MaskedArray(np.empty(n_candidates, dtype=arr_dtype), mask=True)
+        for index, value in param_result.items():
+            # Setting the value at an index unmasks that index
+            ma[index] = value
+        yield (key, ma)
 
 
 class BaseSearchCV(MetaEstimatorMixin, BaseEstimator, metaclass=ABCMeta):
-    """Abstract base class for hyper parameter search with cross-validation.
-    """
+    """Abstract base class for hyper parameter search with cross-validation."""
+
+    _parameter_constraints: dict = {
+        "estimator": [HasMethods(["fit"])],
+        "scoring": [
+            StrOptions(set(get_scorer_names())),
+            callable,
+            list,
+            tuple,
+            dict,
+            None,
+        ],
+        "n_jobs": [numbers.Integral, None],
+        "refit": ["boolean", str, callable],
+        "cv": ["cv_object"],
+        "verbose": ["verbose"],
+        "pre_dispatch": [numbers.Integral, str],
+        "error_score": [StrOptions({"raise"}), numbers.Real],
+        "return_train_score": ["boolean"],
+    }
 
     @abstractmethod
-    @_deprecate_positional_args
-    def __init__(self, estimator, *, scoring=None, n_jobs=None,
-                 refit=True, cv=None, verbose=0,
-                 pre_dispatch='2*n_jobs', error_score=np.nan,
-                 return_train_score=True):
-
+    def __init__(
+        self,
+        estimator,
+        *,
+        scoring=None,
+        n_jobs=None,
+        refit=True,
+        cv=None,
+        verbose=0,
+        pre_dispatch="2*n_jobs",
+        error_score=np.nan,
+        return_train_score=True,
+    ):
         self.scoring = scoring
         self.estimator = estimator
         self.n_jobs = n_jobs
@@ -429,28 +477,24 @@ def __init__(self, estimator, *, scoring=None, n_jobs=None,
         self.return_train_score = return_train_score
 
     @property
+    # TODO(1.8) remove this property
     def _estimator_type(self):
         return self.estimator._estimator_type
 
-    def _more_tags(self):
-        # allows cross-validation to see 'precomputed' metrics
-        return {
-            'pairwise': _safe_tags(self.estimator, "pairwise"),
-            "_xfail_checks": {"check_supervised_y_2d":
-                              "DataConversionWarning not caught"},
-        }
-
-    # TODO: Remove in 1.1
-    # mypy error: Decorated property not supported
-    @deprecated("Attribute _pairwise was deprecated in "  # type: ignore
-                "version 0.24 and will be removed in 1.1 (renaming of 0.26).")
-    @property
-    def _pairwise(self):
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        sub_estimator_tags = get_tags(self.estimator)
+        tags.estimator_type = sub_estimator_tags.estimator_type
+        tags.classifier_tags = deepcopy(sub_estimator_tags.classifier_tags)
+        tags.regressor_tags = deepcopy(sub_estimator_tags.regressor_tags)
         # allows cross-validation to see 'precomputed' metrics
-        return getattr(self.estimator, '_pairwise', False)
+        tags.input_tags.pairwise = sub_estimator_tags.input_tags.pairwise
+        tags.input_tags.sparse = sub_estimator_tags.input_tags.sparse
+        tags.array_api_support = sub_estimator_tags.array_api_support
+        return tags
 
-    def score(self, X, y=None):
-        """Returns the score on the given data, if the estimator has been refit.
+    def score(self, X, y=None, **params):
+        """Return the score on the given data, if the estimator has been refit.
 
         This uses the score defined by ``scoring`` where provided, and the
         ``best_estimator_.score`` method otherwise.
@@ -458,37 +502,57 @@ def score(self, X, y=None):
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
-            Input data, where n_samples is the number of samples and
-            n_features is the number of features.
+            Input data, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
 
         y : array-like of shape (n_samples, n_output) \
             or (n_samples,), default=None
             Target relative to X for classification or regression;
             None for unsupervised learning.
 
+        **params : dict
+            Parameters to be passed to the underlying scorer(s).
+
+            .. versionadded:: 1.4
+                Only available if `enable_metadata_routing=True`. See
+                :ref:`Metadata Routing User Guide ` for more
+                details.
+
         Returns
         -------
         score : float
+            The score defined by ``scoring`` if provided, and the
+            ``best_estimator_.score`` method otherwise.
         """
-        self._check_is_fitted('score')
+        _check_refit(self, "score")
+        check_is_fitted(self)
+
+        _raise_for_params(params, self, "score")
+
+        if _routing_enabled():
+            score_params = process_routing(self, "score", **params).scorer["score"]
+        else:
+            score_params = dict()
+
         if self.scorer_ is None:
-            raise ValueError("No score function explicitly defined, "
-                             "and the estimator doesn't provide one %s"
-                             % self.best_estimator_)
+            raise ValueError(
+                "No score function explicitly defined, "
+                "and the estimator doesn't provide one %s" % self.best_estimator_
+            )
         if isinstance(self.scorer_, dict):
             if self.multimetric_:
                 scorer = self.scorer_[self.refit]
             else:
                 scorer = self.scorer_
-            return scorer(self.best_estimator_, X, y)
+            return scorer(self.best_estimator_, X, y, **score_params)
 
         # callable
-        score = self.scorer_(self.best_estimator_, X, y)
+        score = self.scorer_(self.best_estimator_, X, y, **score_params)
         if self.multimetric_:
             score = score[self.refit]
         return score
 
-    @if_delegate_has_method(delegate=('best_estimator_', 'estimator'))
+    @available_if(_search_estimator_has("score_samples"))
     def score_samples(self, X):
         """Call score_samples on the estimator with the best found parameters.
 
@@ -506,23 +570,12 @@ def score_samples(self, X):
         Returns
         -------
         y_score : ndarray of shape (n_samples,)
+            The ``best_estimator_.score_samples`` method.
         """
-        self._check_is_fitted('score_samples')
+        check_is_fitted(self)
         return self.best_estimator_.score_samples(X)
 
-    def _check_is_fitted(self, method_name):
-        if not self.refit:
-            raise NotFittedError('This %s instance was initialized '
-                                 'with refit=False. %s is '
-                                 'available only after refitting on the best '
-                                 'parameters. You can refit an estimator '
-                                 'manually using the ``best_params_`` '
-                                 'attribute'
-                                 % (type(self).__name__, method_name))
-        else:
-            check_is_fitted(self)
-
-    @if_delegate_has_method(delegate=('best_estimator_', 'estimator'))
+    @available_if(_search_estimator_has("predict"))
     def predict(self, X):
         """Call predict on the estimator with the best found parameters.
 
@@ -535,11 +588,16 @@ def predict(self, X):
             Must fulfill the input assumptions of the
             underlying estimator.
 
+        Returns
+        -------
+        y_pred : ndarray of shape (n_samples,)
+            The predicted labels or values for `X` based on the estimator with
+            the best found parameters.
         """
-        self._check_is_fitted('predict')
+        check_is_fitted(self)
         return self.best_estimator_.predict(X)
 
-    @if_delegate_has_method(delegate=('best_estimator_', 'estimator'))
+    @available_if(_search_estimator_has("predict_proba"))
     def predict_proba(self, X):
         """Call predict_proba on the estimator with the best found parameters.
 
@@ -552,11 +610,17 @@ def predict_proba(self, X):
             Must fulfill the input assumptions of the
             underlying estimator.
 
+        Returns
+        -------
+        y_pred : ndarray of shape (n_samples,) or (n_samples, n_classes)
+            Predicted class probabilities for `X` based on the estimator with
+            the best found parameters. The order of the classes corresponds
+            to that in the fitted attribute :term:`classes_`.
         """
-        self._check_is_fitted('predict_proba')
+        check_is_fitted(self)
         return self.best_estimator_.predict_proba(X)
 
-    @if_delegate_has_method(delegate=('best_estimator_', 'estimator'))
+    @available_if(_search_estimator_has("predict_log_proba"))
     def predict_log_proba(self, X):
         """Call predict_log_proba on the estimator with the best found parameters.
 
@@ -569,11 +633,17 @@ def predict_log_proba(self, X):
             Must fulfill the input assumptions of the
             underlying estimator.
 
+        Returns
+        -------
+        y_pred : ndarray of shape (n_samples,) or (n_samples, n_classes)
+            Predicted class log-probabilities for `X` based on the estimator
+            with the best found parameters. The order of the classes
+            corresponds to that in the fitted attribute :term:`classes_`.
         """
-        self._check_is_fitted('predict_log_proba')
+        check_is_fitted(self)
         return self.best_estimator_.predict_log_proba(X)
 
-    @if_delegate_has_method(delegate=('best_estimator_', 'estimator'))
+    @available_if(_search_estimator_has("decision_function"))
     def decision_function(self, X):
         """Call decision_function on the estimator with the best found parameters.
 
@@ -586,11 +656,17 @@ def decision_function(self, X):
             Must fulfill the input assumptions of the
             underlying estimator.
 
+        Returns
+        -------
+        y_score : ndarray of shape (n_samples,) or (n_samples, n_classes) \
+                or (n_samples, n_classes * (n_classes-1) / 2)
+            Result of the decision function for `X` based on the estimator with
+            the best found parameters.
         """
-        self._check_is_fitted('decision_function')
+        check_is_fitted(self)
         return self.best_estimator_.decision_function(X)
 
-    @if_delegate_has_method(delegate=('best_estimator_', 'estimator'))
+    @available_if(_search_estimator_has("transform"))
     def transform(self, X):
         """Call transform on the estimator with the best found parameters.
 
@@ -603,12 +679,17 @@ def transform(self, X):
             Must fulfill the input assumptions of the
             underlying estimator.
 
+        Returns
+        -------
+        Xt : {ndarray, sparse matrix} of shape (n_samples, n_features)
+            `X` transformed in the new space based on the estimator with
+            the best found parameters.
         """
-        self._check_is_fitted('transform')
+        check_is_fitted(self)
         return self.best_estimator_.transform(X)
 
-    @if_delegate_has_method(delegate=('best_estimator_', 'estimator'))
-    def inverse_transform(self, Xt):
+    @available_if(_search_estimator_has("inverse_transform"))
+    def inverse_transform(self, X):
         """Call inverse_transform on the estimator with the best found params.
 
         Only available if the underlying estimator implements
@@ -616,38 +697,52 @@ def inverse_transform(self, Xt):
 
         Parameters
         ----------
-        Xt : indexable, length n_samples
+        X : indexable, length n_samples
             Must fulfill the input assumptions of the
             underlying estimator.
 
+        Returns
+        -------
+        X_original : {ndarray, sparse matrix} of shape (n_samples, n_features)
+            Result of the `inverse_transform` function for `X` based on the
+            estimator with the best found parameters.
         """
-        self._check_is_fitted('inverse_transform')
-        return self.best_estimator_.inverse_transform(Xt)
+        check_is_fitted(self)
+        return self.best_estimator_.inverse_transform(X)
 
     @property
     def n_features_in_(self):
+        """Number of features seen during :term:`fit`.
+
+        Only available when `refit=True`.
+        """
         # For consistency with other estimators we raise a AttributeError so
         # that hasattr() fails if the search estimator isn't fitted.
         try:
             check_is_fitted(self)
         except NotFittedError as nfe:
             raise AttributeError(
-                "{} object has no n_features_in_ attribute."
-                .format(self.__class__.__name__)
+                "{} object has no n_features_in_ attribute.".format(
+                    self.__class__.__name__
+                )
             ) from nfe
 
         return self.best_estimator_.n_features_in_
 
     @property
     def classes_(self):
-        self._check_is_fitted("classes_")
+        """Class labels.
+
+        Only available when `refit=True` and the estimator is a classifier.
+        """
+        _search_estimator_has("classes_")(self)
         return self.best_estimator_.classes_
 
     def _run_search(self, evaluate_candidates):
         """Repeatedly calls `evaluate_candidates` to conduct a search.
 
         This method, implemented in sub-classes, makes it possible to
-        customize the the scheduling of evaluations: GridSearchCV and
+        customize the scheduling of evaluations: GridSearchCV and
         RandomizedSearchCV schedule evaluations for their whole parameter
         search space at once but other more sequential approaches are also
         possible: for instance is possible to iteratively schedule evaluations
@@ -669,8 +764,8 @@ def _run_search(self, evaluate_candidates):
                 - an optional `cv` parameter which can be used to e.g.
                   evaluate candidates on different dataset splits, or
                   evaluate candidates on subsampled data (as done in the
-                  SucessiveHaling estimators). By default, the original `cv`
-                  parameter is used, and it is available as a private
+                  Successive Halving estimators). By default, the original
+                  `cv` parameter is used, and it is available as a private
                   `_checked_cv_orig` attribute.
                 - an optional `more_results` dict. Each key will be added to
                   the `cv_results_` attribute. Values should be lists of
@@ -711,113 +806,226 @@ def _check_refit_for_multimetric(self, scores):
             "parameter setting on the whole data and make the best_* "
             "attributes available for that metric. If this is not needed, "
             f"refit should be set to False explicitly. {self.refit!r} was "
-            "passed.")
+            "passed."
+        )
 
-        valid_refit_dict = (isinstance(self.refit, str) and
-                            self.refit in scores)
+        valid_refit_dict = isinstance(self.refit, str) and self.refit in scores
 
-        if (self.refit is not False and not valid_refit_dict
-                and not callable(self.refit)):
+        if (
+            self.refit is not False
+            and not valid_refit_dict
+            and not callable(self.refit)
+        ):
             raise ValueError(multimetric_refit_msg)
 
-    @_deprecate_positional_args
-    def fit(self, X, y=None, *, groups=None, **fit_params):
+    @staticmethod
+    def _select_best_index(refit, refit_metric, results):
+        """Select index of the best combination of hyperparemeters."""
+        if callable(refit):
+            # If callable, refit is expected to return the index of the best
+            # parameter set.
+            best_index = refit(results)
+            if not isinstance(best_index, numbers.Integral):
+                raise TypeError("best_index_ returned is not an integer")
+            if best_index < 0 or best_index >= len(results["params"]):
+                raise IndexError("best_index_ index out of range")
+        else:
+            best_index = results[f"rank_test_{refit_metric}"].argmin()
+        return best_index
+
+    def _get_scorers(self):
+        """Get the scorer(s) to be used.
+
+        This is used in ``fit`` and ``get_metadata_routing``.
+
+        Returns
+        -------
+        scorers, refit_metric
+        """
+        refit_metric = "score"
+
+        if callable(self.scoring):
+            scorers = self.scoring
+        elif self.scoring is None or isinstance(self.scoring, str):
+            scorers = check_scoring(self.estimator, self.scoring)
+        else:
+            scorers = _check_multimetric_scoring(self.estimator, self.scoring)
+            self._check_refit_for_multimetric(scorers)
+            refit_metric = self.refit
+            scorers = _MultimetricScorer(
+                scorers=scorers, raise_exc=(self.error_score == "raise")
+            )
+
+        return scorers, refit_metric
+
+    def _check_scorers_accept_sample_weight(self):
+        # TODO(slep006): remove when metadata routing is the only way
+        scorers, _ = self._get_scorers()
+        # In the multimetric case, warn the user for each scorer separately
+        if isinstance(scorers, _MultimetricScorer):
+            for name, scorer in scorers._scorers.items():
+                if not scorer._accept_sample_weight():
+                    warnings.warn(
+                        f"The scoring {name}={scorer} does not support sample_weight, "
+                        "which may lead to statistically incorrect results when "
+                        f"fitting {self} with sample_weight. "
+                    )
+            return scorers._accept_sample_weight()
+        # In most cases, scorers is a Scorer object
+        # But it's a function when user passes scoring=function
+        if hasattr(scorers, "_accept_sample_weight"):
+            accept = scorers._accept_sample_weight()
+        else:
+            accept = "sample_weight" in signature(scorers).parameters
+        if not accept:
+            warnings.warn(
+                f"The scoring {scorers} does not support sample_weight, "
+                "which may lead to statistically incorrect results when "
+                f"fitting {self} with sample_weight. "
+            )
+        return accept
+
+    def _get_routed_params_for_fit(self, params):
+        """Get the parameters to be used for routing.
+
+        This is a method instead of a snippet in ``fit`` since it's used twice,
+        here in ``fit``, and in ``HalvingRandomSearchCV.fit``.
+        """
+        if _routing_enabled():
+            routed_params = process_routing(self, "fit", **params)
+        else:
+            params = params.copy()
+            groups = params.pop("groups", None)
+            routed_params = Bunch(
+                estimator=Bunch(fit=params),
+                splitter=Bunch(split={"groups": groups}),
+                scorer=Bunch(score={}),
+            )
+            # NOTE: sample_weight is forwarded to the scorer if sample_weight
+            # is not None and scorers accept sample_weight. For _MultimetricScorer,
+            # sample_weight is forwarded if any scorer accepts sample_weight
+            if (
+                params.get("sample_weight") is not None
+                and self._check_scorers_accept_sample_weight()
+            ):
+                routed_params.scorer.score["sample_weight"] = params["sample_weight"]
+        return routed_params
+
+    @_fit_context(
+        # *SearchCV.estimator is not validated yet
+        prefer_skip_nested_validation=False
+    )
+    def fit(self, X, y=None, **params):
         """Run fit with all sets of parameters.
 
         Parameters
         ----------
 
-        X : array-like of shape (n_samples, n_features)
-            Training vector, where n_samples is the number of samples and
-            n_features is the number of features.
+        X : array-like of shape (n_samples, n_features) or (n_samples, n_samples)
+            Training vectors, where `n_samples` is the number of samples and
+            `n_features` is the number of features. For precomputed kernel or
+            distance matrix, the expected shape of X is (n_samples, n_samples).
 
         y : array-like of shape (n_samples, n_output) \
             or (n_samples,), default=None
             Target relative to X for classification or regression;
             None for unsupervised learning.
 
-        groups : array-like of shape (n_samples,), default=None
-            Group labels for the samples used while splitting the dataset into
-            train/test set. Only used in conjunction with a "Group" :term:`cv`
-            instance (e.g., :class:`~sklearn.model_selection.GroupKFold`).
+        **params : dict of str -> object
+            Parameters passed to the ``fit`` method of the estimator, the scorer,
+            and the CV splitter.
+
+            If a fit parameter is an array-like whose length is equal to
+            `num_samples` then it will be split by cross-validation along with
+            `X` and `y`. For example, the :term:`sample_weight` parameter is
+            split because `len(sample_weights) = len(X)`. However, this behavior
+            does not apply to `groups` which is passed to the splitter configured
+            via the `cv` parameter of the constructor. Thus, `groups` is used
+            *to perform the split* and determines which samples are
+            assigned to the each side of the a split.
 
-        **fit_params : dict of str -> object
-            Parameters passed to the ``fit`` method of the estimator
+        Returns
+        -------
+        self : object
+            Instance of fitted estimator.
         """
         estimator = self.estimator
-        refit_metric = "score"
+        scorers, refit_metric = self._get_scorers()
 
-        if callable(self.scoring):
-            scorers = self.scoring
-        elif self.scoring is None or isinstance(self.scoring, str):
-            scorers = check_scoring(self.estimator, self.scoring)
-        else:
-            scorers = _check_multimetric_scoring(self.estimator, self.scoring)
-            self._check_refit_for_multimetric(scorers)
-            refit_metric = self.refit
+        X, y = indexable(X, y)
+        params = _check_method_params(X, params=params)
 
-        X, y, groups = indexable(X, y, groups)
-        fit_params = _check_fit_params(X, fit_params)
+        routed_params = self._get_routed_params_for_fit(params)
 
         cv_orig = check_cv(self.cv, y, classifier=is_classifier(estimator))
-        n_splits = cv_orig.get_n_splits(X, y, groups)
+        n_splits = cv_orig.get_n_splits(X, y, **routed_params.splitter.split)
 
         base_estimator = clone(self.estimator)
 
-        parallel = Parallel(n_jobs=self.n_jobs,
-                            pre_dispatch=self.pre_dispatch)
-
-        fit_and_score_kwargs = dict(scorer=scorers,
-                                    fit_params=fit_params,
-                                    return_train_score=self.return_train_score,
-                                    return_n_test_samples=True,
-                                    return_times=True,
-                                    return_parameters=False,
-                                    error_score=self.error_score,
-                                    verbose=self.verbose)
+        parallel = Parallel(n_jobs=self.n_jobs, pre_dispatch=self.pre_dispatch)
+
+        fit_and_score_kwargs = dict(
+            scorer=scorers,
+            fit_params=routed_params.estimator.fit,
+            score_params=routed_params.scorer.score,
+            return_train_score=self.return_train_score,
+            return_n_test_samples=True,
+            return_times=True,
+            return_parameters=False,
+            error_score=self.error_score,
+            verbose=self.verbose,
+        )
         results = {}
         with parallel:
             all_candidate_params = []
             all_out = []
             all_more_results = defaultdict(list)
 
-            def evaluate_candidates(candidate_params, cv=None,
-                                    more_results=None):
+            def evaluate_candidates(candidate_params, cv=None, more_results=None):
                 cv = cv or cv_orig
                 candidate_params = list(candidate_params)
                 n_candidates = len(candidate_params)
 
                 if self.verbose > 0:
-                    print("Fitting {0} folds for each of {1} candidates,"
-                          " totalling {2} fits".format(
-                              n_splits, n_candidates, n_candidates * n_splits))
-
-                out = parallel(delayed(_fit_and_score)(clone(base_estimator),
-                                                       X, y,
-                                                       train=train, test=test,
-                                                       parameters=parameters,
-                                                       split_progress=(
-                                                           split_idx,
-                                                           n_splits),
-                                                       candidate_progress=(
-                                                           cand_idx,
-                                                           n_candidates),
-                                                       **fit_and_score_kwargs)
-                               for (cand_idx, parameters),
-                                   (split_idx, (train, test)) in product(
-                                   enumerate(candidate_params),
-                                   enumerate(cv.split(X, y, groups))))
+                    print(
+                        "Fitting {0} folds for each of {1} candidates,"
+                        " totalling {2} fits".format(
+                            n_splits, n_candidates, n_candidates * n_splits
+                        )
+                    )
+
+                out = parallel(
+                    delayed(_fit_and_score)(
+                        clone(base_estimator),
+                        X,
+                        y,
+                        train=train,
+                        test=test,
+                        parameters=parameters,
+                        split_progress=(split_idx, n_splits),
+                        candidate_progress=(cand_idx, n_candidates),
+                        **fit_and_score_kwargs,
+                    )
+                    for (cand_idx, parameters), (split_idx, (train, test)) in product(
+                        enumerate(candidate_params),
+                        enumerate(cv.split(X, y, **routed_params.splitter.split)),
+                    )
+                )
 
                 if len(out) < 1:
-                    raise ValueError('No fits were performed. '
-                                     'Was the CV iterator empty? '
-                                     'Were there no candidates?')
+                    raise ValueError(
+                        "No fits were performed. "
+                        "Was the CV iterator empty? "
+                        "Were there no candidates?"
+                    )
                 elif len(out) != n_candidates * n_splits:
-                    raise ValueError('cv.split and cv.get_n_splits returned '
-                                     'inconsistent results. Expected {} '
-                                     'splits, got {}'
-                                     .format(n_splits,
-                                             len(out) // n_candidates))
+                    raise ValueError(
+                        "cv.split and cv.get_n_splits returned "
+                        "inconsistent results. Expected {} "
+                        "splits, got {}".format(n_splits, len(out) // n_candidates)
+                    )
+
+                _warn_or_raise_about_fit_failures(out, self.error_score)
 
                 # For callable self.scoring, the return type is only know after
                 # calling. If the return type is a dictionary, the error scores
@@ -825,16 +1033,18 @@ def evaluate_candidates(candidate_params, cv=None,
                 # of out will be done in `_insert_error_scores`.
                 if callable(self.scoring):
                     _insert_error_scores(out, self.error_score)
+
                 all_candidate_params.extend(candidate_params)
                 all_out.extend(out)
+
                 if more_results is not None:
                     for key, value in more_results.items():
                         all_more_results[key].extend(value)
 
                 nonlocal results
                 results = self._format_results(
-                    all_candidate_params, n_splits, all_out,
-                    all_more_results)
+                    all_candidate_params, n_splits, all_out, all_more_results
+                )
 
                 return results
 
@@ -842,10 +1052,10 @@ def evaluate_candidates(candidate_params, cv=None,
 
             # multimetric is determined here because in the case of a callable
             # self.scoring the return type is only known after calling
-            first_test_score = all_out[0]['test_scores']
+            first_test_score = all_out[0]["test_scores"]
             self.multimetric_ = isinstance(first_test_score, dict)
 
-            # check refit_metric now for a callabe scorer that is multimetric
+            # check refit_metric now for a callable scorer that is multimetric
             if callable(self.scoring) and self.multimetric_:
                 self._check_refit_for_multimetric(first_test_score)
                 refit_metric = self.refit
@@ -854,102 +1064,111 @@ def evaluate_candidates(candidate_params, cv=None,
         # best_score_ iff refit is one of the scorer names
         # In single metric evaluation, refit_metric is "score"
         if self.refit or not self.multimetric_:
-            # If callable, refit is expected to return the index of the best
-            # parameter set.
-            if callable(self.refit):
-                self.best_index_ = self.refit(results)
-                if not isinstance(self.best_index_, numbers.Integral):
-                    raise TypeError('best_index_ returned is not an integer')
-                if (self.best_index_ < 0 or
-                   self.best_index_ >= len(results["params"])):
-                    raise IndexError('best_index_ index out of range')
-            else:
-                self.best_index_ = results["rank_test_%s"
-                                           % refit_metric].argmin()
-                self.best_score_ = results["mean_test_%s" % refit_metric][
-                                           self.best_index_]
+            self.best_index_ = self._select_best_index(
+                self.refit, refit_metric, results
+            )
+            if not callable(self.refit):
+                # With a non-custom callable, we can select the best score
+                # based on the best index
+                self.best_score_ = results[f"mean_test_{refit_metric}"][
+                    self.best_index_
+                ]
             self.best_params_ = results["params"][self.best_index_]
 
         if self.refit:
-            # we clone again after setting params in case some
-            # of the params are estimators as well.
-            self.best_estimator_ = clone(clone(base_estimator).set_params(
-                **self.best_params_))
+            # here we clone the estimator as well as the parameters, since
+            # sometimes the parameters themselves might be estimators, e.g.
+            # when we search over different estimators in a pipeline.
+            # ref: https://github.com/scikit-learn/scikit-learn/pull/26786
+            self.best_estimator_ = clone(base_estimator).set_params(
+                **clone(self.best_params_, safe=False)
+            )
+
             refit_start_time = time.time()
             if y is not None:
-                self.best_estimator_.fit(X, y, **fit_params)
+                self.best_estimator_.fit(X, y, **routed_params.estimator.fit)
             else:
-                self.best_estimator_.fit(X, **fit_params)
+                self.best_estimator_.fit(X, **routed_params.estimator.fit)
             refit_end_time = time.time()
             self.refit_time_ = refit_end_time - refit_start_time
 
+            if hasattr(self.best_estimator_, "feature_names_in_"):
+                self.feature_names_in_ = self.best_estimator_.feature_names_in_
+
         # Store the only scorer not as a dict for single metric evaluation
-        self.scorer_ = scorers
+        if isinstance(scorers, _MultimetricScorer):
+            self.scorer_ = scorers._scorers
+        else:
+            self.scorer_ = scorers
 
         self.cv_results_ = results
         self.n_splits_ = n_splits
 
         return self
 
-    def _format_results(self, candidate_params, n_splits, out,
-                        more_results=None):
+    def _format_results(self, candidate_params, n_splits, out, more_results=None):
         n_candidates = len(candidate_params)
         out = _aggregate_score_dicts(out)
 
         results = dict(more_results or {})
+        for key, val in results.items():
+            # each value is a list (as per evaluate_candidate's convention)
+            # we convert it to an array for consistency with the other keys
+            results[key] = np.asarray(val)
 
         def _store(key_name, array, weights=None, splits=False, rank=False):
             """A small helper to store the scores/times to the cv_results_"""
             # When iterated first by splits, then by parameters
             # We want `array` to have `n_candidates` rows and `n_splits` cols.
-            array = np.array(array, dtype=np.float64).reshape(n_candidates,
-                                                              n_splits)
+            array = np.array(array, dtype=np.float64).reshape(n_candidates, n_splits)
             if splits:
                 for split_idx in range(n_splits):
                     # Uses closure to alter the results
-                    results["split%d_%s"
-                            % (split_idx, key_name)] = array[:, split_idx]
+                    results["split%d_%s" % (split_idx, key_name)] = array[:, split_idx]
 
             array_means = np.average(array, axis=1, weights=weights)
-            results['mean_%s' % key_name] = array_means
+            results["mean_%s" % key_name] = array_means
 
-            if (key_name.startswith(("train_", "test_")) and
-                    np.any(~np.isfinite(array_means))):
+            if key_name.startswith(("train_", "test_")) and np.any(
+                ~np.isfinite(array_means)
+            ):
                 warnings.warn(
-                    f"One or more of the {key_name.split('_')[0]} scores "
-                    f"are non-finite: {array_means}",
-                    category=UserWarning
+                    (
+                        f"One or more of the {key_name.split('_')[0]} scores "
+                        f"are non-finite: {array_means}"
+                    ),
+                    category=UserWarning,
                 )
 
             # Weighted std is not directly available in numpy
-            array_stds = np.sqrt(np.average((array -
-                                             array_means[:, np.newaxis]) ** 2,
-                                            axis=1, weights=weights))
-            results['std_%s' % key_name] = array_stds
+            array_stds = np.sqrt(
+                np.average(
+                    (array - array_means[:, np.newaxis]) ** 2, axis=1, weights=weights
+                )
+            )
+            results["std_%s" % key_name] = array_stds
 
             if rank:
-                results["rank_%s" % key_name] = np.asarray(
-                    rankdata(-array_means, method='min'), dtype=np.int32)
-
-        _store('fit_time', out["fit_time"])
-        _store('score_time', out["score_time"])
-        # Use one MaskedArray and mask all the places where the param is not
-        # applicable for that candidate. Use defaultdict as each candidate may
-        # not contain all the params
-        param_results = defaultdict(partial(MaskedArray,
-                                            np.empty(n_candidates,),
-                                            mask=True,
-                                            dtype=object))
-        for cand_idx, params in enumerate(candidate_params):
-            for name, value in params.items():
-                # An all masked empty array gets created for the key
-                # `"param_%s" % name` at the first occurrence of `name`.
-                # Setting the value at an index also unmasks that index
-                param_results["param_%s" % name][cand_idx] = value
-
-        results.update(param_results)
+                # When the fit/scoring fails `array_means` contains NaNs, we
+                # will exclude them from the ranking process and consider them
+                # as tied with the worst performers.
+                if np.isnan(array_means).all():
+                    # All fit/scoring routines failed.
+                    rank_result = np.ones_like(array_means, dtype=np.int32)
+                else:
+                    min_array_means = np.nanmin(array_means) - 1
+                    array_means = np.nan_to_num(array_means, nan=min_array_means)
+                    rank_result = rankdata(-array_means, method="min").astype(
+                        np.int32, copy=False
+                    )
+                results["rank_%s" % key_name] = rank_result
+
+        _store("fit_time", out["fit_time"])
+        _store("score_time", out["score_time"])
         # Store a list of param dicts at the key 'params'
-        results['params'] = candidate_params
+        for param, ma in _yield_masked_array_for_each_param(candidate_params):
+            results[param] = ma
+        results["params"] = candidate_params
 
         test_scores_dict = _normalize_score_results(out["test_scores"])
         if self.return_train_score:
@@ -957,16 +1176,68 @@ def _store(key_name, array, weights=None, splits=False, rank=False):
 
         for scorer_name in test_scores_dict:
             # Computed the (weighted) mean and std for test scores alone
-            _store('test_%s' % scorer_name, test_scores_dict[scorer_name],
-                   splits=True, rank=True,
-                   weights=None)
+            _store(
+                "test_%s" % scorer_name,
+                test_scores_dict[scorer_name],
+                splits=True,
+                rank=True,
+                weights=None,
+            )
             if self.return_train_score:
-                _store('train_%s' % scorer_name,
-                       train_scores_dict[scorer_name],
-                       splits=True)
+                _store(
+                    "train_%s" % scorer_name,
+                    train_scores_dict[scorer_name],
+                    splits=True,
+                )
 
         return results
 
+    def get_metadata_routing(self):
+        """Get metadata routing of this object.
+
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
+
+        .. versionadded:: 1.4
+
+        Returns
+        -------
+        routing : MetadataRouter
+            A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
+            routing information.
+        """
+        router = MetadataRouter(owner=self.__class__.__name__)
+        router.add(
+            estimator=self.estimator,
+            method_mapping=MethodMapping().add(caller="fit", callee="fit"),
+        )
+
+        scorer, _ = self._get_scorers()
+        router.add(
+            scorer=scorer,
+            method_mapping=MethodMapping()
+            .add(caller="score", callee="score")
+            .add(caller="fit", callee="score"),
+        )
+        router.add(
+            splitter=self.cv,
+            method_mapping=MethodMapping().add(caller="fit", callee="split"),
+        )
+        return router
+
+    def _sk_visual_block_(self):
+        if hasattr(self, "best_estimator_"):
+            key, estimator = "best_estimator_", self.best_estimator_
+        else:
+            key, estimator = "estimator", self.estimator
+
+        return _VisualBlock(
+            "parallel",
+            [estimator],
+            names=[f"{key}: {estimator.__class__.__name__}"],
+            name_details=[str(estimator)],
+        )
+
 
 class GridSearchCV(BaseSearchCV):
     """Exhaustive search over specified parameter values for an estimator.
@@ -985,7 +1256,7 @@ class GridSearchCV(BaseSearchCV):
 
     Parameters
     ----------
-    estimator : estimator object.
+    estimator : estimator object
         This is assumed to implement the scikit-learn estimator interface.
         Either estimator needs to provide a ``score`` function,
         or ``scoring`` must be passed.
@@ -1003,15 +1274,17 @@ class GridSearchCV(BaseSearchCV):
 
         If `scoring` represents a single score, one can use:
 
-        - a single string (see :ref:`scoring_parameter`);
-        - a callable (see :ref:`scoring`) that returns a single value.
+        - a single string (see :ref:`scoring_string_names`);
+        - a callable (see :ref:`scoring_callable`) that returns a single value;
+        - `None`, the `estimator`'s
+          :ref:`default evaluation criterion ` is used.
 
         If `scoring` represents multiple scores, one can use:
 
         - a list or tuple of unique strings;
         - a callable returning a dictionary where the keys are the metric
           names and the values are the metric scores;
-        - a dictionary with metric names as keys and callables a values.
+        - a dictionary with metric names as keys and callables as values.
 
         See :ref:`multimetric_grid_search` for an example.
 
@@ -1051,6 +1324,15 @@ class GridSearchCV(BaseSearchCV):
         See ``scoring`` parameter to know more about multiple metric
         evaluation.
 
+        See :ref:`sphx_glr_auto_examples_model_selection_plot_grid_search_digits.py`
+        to see how to design a custom selection strategy using a callable
+        via `refit`.
+
+        See :ref:`this example
+        `
+        for an example of how to use ``refit=callable`` to balance model
+        complexity and cross-validated score.
+
         .. versionchanged:: 0.20
             Support for callable added.
 
@@ -1065,7 +1347,8 @@ class GridSearchCV(BaseSearchCV):
 
         For integer/None inputs, if the estimator is a classifier and ``y`` is
         either binary or multiclass, :class:`StratifiedKFold` is used. In all
-        other cases, :class:`KFold` is used.
+        other cases, :class:`KFold` is used. These splitters are instantiated
+        with `shuffle=False` so the splits will be the same across calls.
 
         Refer :ref:`User Guide ` for the various
         cross-validation strategies that can be used here.
@@ -1082,22 +1365,17 @@ class GridSearchCV(BaseSearchCV):
         - >3 : the fold and candidate parameter indexes are also displayed
           together with the starting time of the computation.
 
-    pre_dispatch : int, or str, default=n_jobs
+    pre_dispatch : int, or str, default='2*n_jobs'
         Controls the number of jobs that get dispatched during parallel
         execution. Reducing this number can be useful to avoid an
         explosion of memory consumption when more jobs get dispatched
         than CPUs can process. This parameter can be:
 
-            - None, in which case all the jobs are immediately
-              created and spawned. Use this for lightweight and
-              fast-running jobs, to avoid delays due to on-demand
-              spawning of the jobs
-
-            - An int, giving the exact number of total jobs that are
-              spawned
-
-            - A str, giving an expression as a function of n_jobs,
-              as in '2*n_jobs'
+        - None, in which case all the jobs are immediately created and spawned. Use
+          this for lightweight and fast-running jobs, to avoid delays due to on-demand
+          spawning of the jobs
+        - An int, giving the exact number of total jobs that are spawned
+        - A str, giving an expression as a function of n_jobs, as in '2*n_jobs'
 
     error_score : 'raise' or numeric, default=np.nan
         Value to assign to the score if an error occurs in estimator fitting.
@@ -1119,25 +1397,6 @@ class GridSearchCV(BaseSearchCV):
         .. versionchanged:: 0.21
             Default value was changed from ``True`` to ``False``
 
-
-    Examples
-    --------
-    >>> from sklearn import svm, datasets
-    >>> from sklearn.model_selection import GridSearchCV
-    >>> iris = datasets.load_iris()
-    >>> parameters = {'kernel':('linear', 'rbf'), 'C':[1, 10]}
-    >>> svc = svm.SVC()
-    >>> clf = GridSearchCV(svc, parameters)
-    >>> clf.fit(iris.data, iris.target)
-    GridSearchCV(estimator=SVC(),
-                 param_grid={'C': [1, 10], 'kernel': ('linear', 'rbf')})
-    >>> sorted(clf.cv_results_.keys())
-    ['mean_fit_time', 'mean_score_time', 'mean_test_score',...
-     'param_C', 'param_kernel', 'params',...
-     'rank_test_score', 'split0_test_score',...
-     'split2_test_score', ...
-     'std_fit_time', 'std_score_time', 'std_test_score']
-
     Attributes
     ----------
     cv_results_ : dict of numpy (masked) ndarrays
@@ -1248,6 +1507,35 @@ class GridSearchCV(BaseSearchCV):
     multimetric_ : bool
         Whether or not the scorers compute several metrics.
 
+    classes_ : ndarray of shape (n_classes,)
+        The classes labels. This is present only if ``refit`` is specified and
+        the underlying estimator is a classifier.
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`. Only defined if
+        `best_estimator_` is defined (see the documentation for the `refit`
+        parameter for more details) and that `best_estimator_` exposes
+        `n_features_in_` when fit.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Only defined if
+        `best_estimator_` is defined (see the documentation for the `refit`
+        parameter for more details) and that `best_estimator_` exposes
+        `feature_names_in_` when fit.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    ParameterGrid : Generates all the combinations of a hyperparameter grid.
+    train_test_split : Utility function to split the data into a development
+        set usable for fitting a GridSearchCV instance and an evaluation set
+        for its final evaluation.
+    sklearn.metrics.make_scorer : Make a scorer from a performance metric or
+        loss function.
+
     Notes
     -----
     The parameters selected are those that maximize the score of the left out
@@ -1261,30 +1549,56 @@ class GridSearchCV(BaseSearchCV):
     `pre_dispatch` many times. A reasonable value for `pre_dispatch` is `2 *
     n_jobs`.
 
-    See Also
-    ---------
-    ParameterGrid : Generates all the combinations of a hyperparameter grid.
-    train_test_split : Utility function to split the data into a development
-        set usable for fitting a GridSearchCV instance and an evaluation set
-        for its final evaluation.
-    sklearn.metrics.make_scorer : Make a scorer from a performance metric or
-        loss function.
-
+    Examples
+    --------
+    >>> from sklearn import svm, datasets
+    >>> from sklearn.model_selection import GridSearchCV
+    >>> iris = datasets.load_iris()
+    >>> parameters = {'kernel':('linear', 'rbf'), 'C':[1, 10]}
+    >>> svc = svm.SVC()
+    >>> clf = GridSearchCV(svc, parameters)
+    >>> clf.fit(iris.data, iris.target)
+    GridSearchCV(estimator=SVC(),
+                 param_grid={'C': [1, 10], 'kernel': ('linear', 'rbf')})
+    >>> sorted(clf.cv_results_.keys())
+    ['mean_fit_time', 'mean_score_time', 'mean_test_score',...
+     'param_C', 'param_kernel', 'params',...
+     'rank_test_score', 'split0_test_score',...
+     'split2_test_score', ...
+     'std_fit_time', 'std_score_time', 'std_test_score']
     """
-    _required_parameters = ["estimator", "param_grid"]
 
-    @_deprecate_positional_args
-    def __init__(self, estimator, param_grid, *, scoring=None,
-                 n_jobs=None, refit=True, cv=None,
-                 verbose=0, pre_dispatch='2*n_jobs',
-                 error_score=np.nan, return_train_score=False):
+    _parameter_constraints: dict = {
+        **BaseSearchCV._parameter_constraints,
+        "param_grid": [dict, list],
+    }
+
+    def __init__(
+        self,
+        estimator,
+        param_grid,
+        *,
+        scoring=None,
+        n_jobs=None,
+        refit=True,
+        cv=None,
+        verbose=0,
+        pre_dispatch="2*n_jobs",
+        error_score=np.nan,
+        return_train_score=False,
+    ):
         super().__init__(
-            estimator=estimator, scoring=scoring,
-            n_jobs=n_jobs, refit=refit, cv=cv, verbose=verbose,
-            pre_dispatch=pre_dispatch, error_score=error_score,
-            return_train_score=return_train_score)
+            estimator=estimator,
+            scoring=scoring,
+            n_jobs=n_jobs,
+            refit=refit,
+            cv=cv,
+            verbose=verbose,
+            pre_dispatch=pre_dispatch,
+            error_score=error_score,
+            return_train_score=return_train_score,
+        )
         self.param_grid = param_grid
-        _check_param_grid(param_grid)
 
     def _run_search(self, evaluate_candidates):
         """Search all candidates in param_grid"""
@@ -1319,8 +1633,8 @@ class RandomizedSearchCV(BaseSearchCV):
 
     Parameters
     ----------
-    estimator : estimator object.
-        A object of that type is instantiated for each grid point.
+    estimator : estimator object
+        An object of that type is instantiated for each grid point.
         This is assumed to implement the scikit-learn estimator interface.
         Either estimator needs to provide a ``score`` function,
         or ``scoring`` must be passed.
@@ -1343,15 +1657,17 @@ class RandomizedSearchCV(BaseSearchCV):
 
         If `scoring` represents a single score, one can use:
 
-        - a single string (see :ref:`scoring_parameter`);
-        - a callable (see :ref:`scoring`) that returns a single value.
+        - a single string (see :ref:`scoring_string_names`);
+        - a callable (see :ref:`scoring_callable`) that returns a single value;
+        - `None`, the `estimator`'s
+          :ref:`default evaluation criterion ` is used.
 
         If `scoring` represents multiple scores, one can use:
 
         - a list or tuple of unique strings;
         - a callable returning a dictionary where the keys are the metric
           names and the values are the metric scores;
-        - a dictionary with metric names as keys and callables a values.
+        - a dictionary with metric names as keys and callables as values.
 
         See :ref:`multimetric_grid_search` for an example.
 
@@ -1376,7 +1692,7 @@ class RandomizedSearchCV(BaseSearchCV):
 
         Where there are considerations other than maximum score in
         choosing a best estimator, ``refit`` can be set to a function which
-        returns the selected ``best_index_`` given the ``cv_results``. In that
+        returns the selected ``best_index_`` given the ``cv_results_``. In that
         case, the ``best_estimator_`` and ``best_params_`` will be set
         according to the returned ``best_index_`` while the ``best_score_``
         attribute will not be available.
@@ -1393,6 +1709,11 @@ class RandomizedSearchCV(BaseSearchCV):
         See ``scoring`` parameter to know more about multiple metric
         evaluation.
 
+        See :ref:`this example
+        `
+        for an example of how to use ``refit=callable`` to balance model
+        complexity and cross-validated score.
+
         .. versionchanged:: 0.20
             Support for callable added.
 
@@ -1407,7 +1728,8 @@ class RandomizedSearchCV(BaseSearchCV):
 
         For integer/None inputs, if the estimator is a classifier and ``y`` is
         either binary or multiclass, :class:`StratifiedKFold` is used. In all
-        other cases, :class:`KFold` is used.
+        other cases, :class:`KFold` is used. These splitters are instantiated
+        with `shuffle=False` so the splits will be the same across calls.
 
         Refer :ref:`User Guide ` for the various
         cross-validation strategies that can be used here.
@@ -1418,22 +1740,23 @@ class RandomizedSearchCV(BaseSearchCV):
     verbose : int
         Controls the verbosity: the higher, the more messages.
 
-    pre_dispatch : int, or str, default=None
+        - >1 : the computation time for each fold and parameter candidate is
+          displayed;
+        - >2 : the score is also displayed;
+        - >3 : the fold and candidate parameter indexes are also displayed
+          together with the starting time of the computation.
+
+    pre_dispatch : int, or str, default='2*n_jobs'
         Controls the number of jobs that get dispatched during parallel
         execution. Reducing this number can be useful to avoid an
         explosion of memory consumption when more jobs get dispatched
         than CPUs can process. This parameter can be:
 
-            - None, in which case all the jobs are immediately
-              created and spawned. Use this for lightweight and
-              fast-running jobs, to avoid delays due to on-demand
-              spawning of the jobs
-
-            - An int, giving the exact number of total jobs that are
-              spawned
-
-            - A str, giving an expression as a function of n_jobs,
-              as in '2*n_jobs'
+        - None, in which case all the jobs are immediately created and spawned. Use
+          this for lightweight and fast-running jobs, to avoid delays due to on-demand
+          spawning of the jobs
+        - An int, giving the exact number of total jobs that are spawned
+        - A str, giving an expression as a function of n_jobs, as in '2*n_jobs'
 
     random_state : int, RandomState instance or None, default=None
         Pseudo random number generator state used for random uniform sampling
@@ -1570,6 +1893,32 @@ class RandomizedSearchCV(BaseSearchCV):
     multimetric_ : bool
         Whether or not the scorers compute several metrics.
 
+    classes_ : ndarray of shape (n_classes,)
+        The classes labels. This is present only if ``refit`` is specified and
+        the underlying estimator is a classifier.
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`. Only defined if
+        `best_estimator_` is defined (see the documentation for the `refit`
+        parameter for more details) and that `best_estimator_` exposes
+        `n_features_in_` when fit.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Only defined if
+        `best_estimator_` is defined (see the documentation for the `refit`
+        parameter for more details) and that `best_estimator_` exposes
+        `feature_names_in_` when fit.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    GridSearchCV : Does exhaustive search over a grid of parameters.
+    ParameterSampler : A generator over parameter settings, constructed from
+        param_distributions.
+
     Notes
     -----
     The parameters selected are those that maximize the score of the held-out
@@ -1583,12 +1932,6 @@ class RandomizedSearchCV(BaseSearchCV):
     `pre_dispatch` many times. A reasonable value for `pre_dispatch` is `2 *
     n_jobs`.
 
-    See Also
-    --------
-    GridSearchCV : Does exhaustive search over a grid of parameters.
-    ParameterSampler : A generator over parameter settings, constructed from
-        param_distributions.
-
     Examples
     --------
     >>> from sklearn.datasets import load_iris
@@ -1603,27 +1946,51 @@ class RandomizedSearchCV(BaseSearchCV):
     >>> clf = RandomizedSearchCV(logistic, distributions, random_state=0)
     >>> search = clf.fit(iris.data, iris.target)
     >>> search.best_params_
-    {'C': 2..., 'penalty': 'l1'}
+    {'C': np.float64(2.195...), 'penalty': 'l1'}
     """
-    _required_parameters = ["estimator", "param_distributions"]
-
-    @_deprecate_positional_args
-    def __init__(self, estimator, param_distributions, *, n_iter=10,
-                 scoring=None, n_jobs=None, refit=True,
-                 cv=None, verbose=0, pre_dispatch='2*n_jobs',
-                 random_state=None, error_score=np.nan,
-                 return_train_score=False):
+
+    _parameter_constraints: dict = {
+        **BaseSearchCV._parameter_constraints,
+        "param_distributions": [dict, list],
+        "n_iter": [Interval(numbers.Integral, 1, None, closed="left")],
+        "random_state": ["random_state"],
+    }
+
+    def __init__(
+        self,
+        estimator,
+        param_distributions,
+        *,
+        n_iter=10,
+        scoring=None,
+        n_jobs=None,
+        refit=True,
+        cv=None,
+        verbose=0,
+        pre_dispatch="2*n_jobs",
+        random_state=None,
+        error_score=np.nan,
+        return_train_score=False,
+    ):
         self.param_distributions = param_distributions
         self.n_iter = n_iter
         self.random_state = random_state
         super().__init__(
-            estimator=estimator, scoring=scoring,
-            n_jobs=n_jobs, refit=refit, cv=cv, verbose=verbose,
-            pre_dispatch=pre_dispatch, error_score=error_score,
-            return_train_score=return_train_score)
+            estimator=estimator,
+            scoring=scoring,
+            n_jobs=n_jobs,
+            refit=refit,
+            cv=cv,
+            verbose=verbose,
+            pre_dispatch=pre_dispatch,
+            error_score=error_score,
+            return_train_score=return_train_score,
+        )
 
     def _run_search(self, evaluate_candidates):
         """Search n_iter candidates from param_distributions"""
-        evaluate_candidates(ParameterSampler(
-            self.param_distributions, self.n_iter,
-            random_state=self.random_state))
+        evaluate_candidates(
+            ParameterSampler(
+                self.param_distributions, self.n_iter, random_state=self.random_state
+            )
+        )
diff --git a/sklearn/model_selection/_search_successive_halving.py b/sklearn/model_selection/_search_successive_halving.py
index b522ce7fbda41..bcd9a83e6dc43 100644
--- a/sklearn/model_selection/_search_successive_halving.py
+++ b/sklearn/model_selection/_search_successive_halving.py
@@ -1,62 +1,64 @@
-from math import ceil, floor, log
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
 from abc import abstractmethod
-from numbers import Integral
+from math import ceil, floor, log
+from numbers import Integral, Real
 
 import numpy as np
-from ._search import _check_param_grid
-from ._search import BaseSearchCV
-from . import ParameterGrid, ParameterSampler
-from ..utils.validation import _num_samples
-from ..base import is_classifier
-from ._split import check_cv, _yields_constant_splits
-from ..utils import resample
 
+from ..base import _fit_context, is_classifier
+from ..metrics._scorer import get_scorer_names
+from ..utils import resample
+from ..utils._param_validation import Interval, StrOptions
+from ..utils.multiclass import check_classification_targets
+from ..utils.validation import _num_samples, validate_data
+from . import ParameterGrid, ParameterSampler
+from ._search import BaseSearchCV
+from ._split import _yields_constant_splits, check_cv
 
-__all__ = ['HalvingGridSearchCV', 'HalvingRandomSearchCV']
+__all__ = ["HalvingGridSearchCV", "HalvingRandomSearchCV"]
 
 
 class _SubsampleMetaSplitter:
     """Splitter that subsamples a given fraction of the dataset"""
+
     def __init__(self, *, base_cv, fraction, subsample_test, random_state):
         self.base_cv = base_cv
         self.fraction = fraction
         self.subsample_test = subsample_test
         self.random_state = random_state
 
-    def split(self, X, y, groups=None):
-        for train_idx, test_idx in self.base_cv.split(X, y, groups):
+    def split(self, X, y, **kwargs):
+        for train_idx, test_idx in self.base_cv.split(X, y, **kwargs):
             train_idx = resample(
-                train_idx, replace=False, random_state=self.random_state,
-                n_samples=int(self.fraction * train_idx.shape[0])
+                train_idx,
+                replace=False,
+                random_state=self.random_state,
+                n_samples=int(self.fraction * len(train_idx)),
             )
             if self.subsample_test:
                 test_idx = resample(
-                    test_idx, replace=False, random_state=self.random_state,
-                    n_samples=int(self.fraction * test_idx.shape[0])
+                    test_idx,
+                    replace=False,
+                    random_state=self.random_state,
+                    n_samples=int(self.fraction * len(test_idx)),
                 )
             yield train_idx, test_idx
 
 
-def _refit_callable(results):
-    # Custom refit callable to return the index of the best candidate. We want
-    # the best candidate out of the last iteration. By default BaseSearchCV
-    # would return the best candidate out of all iterations.
-
-    last_iter = np.max(results['iter'])
-    last_iter_indices = np.flatnonzero(results['iter'] == last_iter)
-    best_idx = np.argmax(results['mean_test_score'][last_iter_indices])
-    return last_iter_indices[best_idx]
-
-
 def _top_k(results, k, itr):
     # Return the best candidates of a given iteration
     iteration, mean_test_score, params = (
-        np.asarray(a) for a in (results['iter'],
-                                results['mean_test_score'],
-                                results['params'])
+        np.asarray(a)
+        for a in (results["iter"], results["mean_test_score"], results["params"])
     )
     iter_indices = np.flatnonzero(iteration == itr)
-    sorted_indices = np.argsort(mean_test_score[iter_indices])
+    scores = mean_test_score[iter_indices]
+    # argsort() places NaNs at the end of the array so we move NaNs to the
+    # front of the array so the last `k` items are the those with the
+    # highest scores.
+    sorted_indices = np.roll(np.argsort(scores), np.count_nonzero(np.isnan(scores)))
     return np.array(params[iter_indices][sorted_indices[-k:]])
 
 
@@ -67,18 +69,54 @@ class BaseSuccessiveHalving(BaseSearchCV):
     Almost optimal exploration in multi-armed bandits, ICML 13
     Zohar Karnin, Tomer Koren, Oren Somekh
     """
-    def __init__(self, estimator, *, scoring=None,
-                 n_jobs=None, refit=True, cv=5, verbose=0, random_state=None,
-                 error_score=np.nan, return_train_score=True,
-                 max_resources='auto', min_resources='exhaust',
-                 resource='n_samples', factor=3, aggressive_elimination=False):
-
-        refit = _refit_callable if refit else False
-        super().__init__(estimator, scoring=scoring,
-                         n_jobs=n_jobs, refit=refit, cv=cv,
-                         verbose=verbose,
-                         error_score=error_score,
-                         return_train_score=return_train_score)
+
+    _parameter_constraints: dict = {
+        **BaseSearchCV._parameter_constraints,
+        # overwrite `scoring` since multi-metrics are not supported
+        "scoring": [StrOptions(set(get_scorer_names())), callable, None],
+        "random_state": ["random_state"],
+        "max_resources": [
+            Interval(Integral, 0, None, closed="neither"),
+            StrOptions({"auto"}),
+        ],
+        "min_resources": [
+            Interval(Integral, 0, None, closed="neither"),
+            StrOptions({"exhaust", "smallest"}),
+        ],
+        "resource": [str],
+        "factor": [Interval(Real, 0, None, closed="neither")],
+        "aggressive_elimination": ["boolean"],
+    }
+    _parameter_constraints.pop("pre_dispatch")  # not used in this class
+
+    def __init__(
+        self,
+        estimator,
+        *,
+        scoring=None,
+        n_jobs=None,
+        refit=True,
+        cv=5,
+        verbose=0,
+        random_state=None,
+        error_score=np.nan,
+        return_train_score=True,
+        max_resources="auto",
+        min_resources="exhaust",
+        resource="n_samples",
+        factor=3,
+        aggressive_elimination=False,
+    ):
+        super().__init__(
+            estimator,
+            scoring=scoring,
+            n_jobs=n_jobs,
+            refit=refit,
+            cv=cv,
+            verbose=verbose,
+            error_score=error_score,
+            return_train_score=return_train_score,
+        )
 
         self.random_state = random_state
         self.max_resources = max_resources
@@ -87,14 +125,7 @@ def __init__(self, estimator, *, scoring=None,
         self.min_resources = min_resources
         self.aggressive_elimination = aggressive_elimination
 
-    def _check_input_parameters(self, X, y, groups):
-
-        if self.scoring is not None and not (isinstance(self.scoring, str)
-                                             or callable(self.scoring)):
-            raise ValueError('scoring parameter must be a string, '
-                             'a callable or None. Multimetric scoring is not '
-                             'supported.')
-
+    def _check_input_parameters(self, X, y, split_params):
         # We need to enforce that successive calls to cv.split() yield the same
         # splits: see https://github.com/scikit-learn/scikit-learn/issues/15149
         if not _yields_constant_splits(self._checked_cv_orig):
@@ -104,59 +135,34 @@ def _check_input_parameters(self, X, y, groups):
                 "shuffle=False."
             )
 
-        if (self.resource != 'n_samples'
-                and self.resource not in self.estimator.get_params()):
-            raise ValueError(
-                f'Cannot use resource={self.resource} which is not supported '
-                f'by estimator {self.estimator.__class__.__name__}'
-            )
-
-        if (isinstance(self.max_resources, str) and
-                self.max_resources != 'auto'):
+        if (
+            self.resource != "n_samples"
+            and self.resource not in self.estimator.get_params()
+        ):
             raise ValueError(
-                "max_resources must be either 'auto' or a positive integer"
-            )
-        if self.max_resources != 'auto' and (
-                not isinstance(self.max_resources, Integral) or
-                self.max_resources <= 0):
-            raise ValueError(
-                "max_resources must be either 'auto' or a positive integer"
-            )
-
-        if self.min_resources not in ('smallest', 'exhaust') and (
-                not isinstance(self.min_resources, Integral) or
-                self.min_resources <= 0):
-            raise ValueError(
-                "min_resources must be either 'smallest', 'exhaust', "
-                "or a positive integer "
-                "no greater than max_resources."
+                f"Cannot use resource={self.resource} which is not supported "
+                f"by estimator {self.estimator.__class__.__name__}"
             )
 
         if isinstance(self, HalvingRandomSearchCV):
-            if self.min_resources == self.n_candidates == 'exhaust':
+            if self.min_resources == self.n_candidates == "exhaust":
                 # for n_candidates=exhaust to work, we need to know what
                 # min_resources is. Similarly min_resources=exhaust needs to
                 # know the actual number of candidates.
                 raise ValueError(
-                    "n_candidates and min_resources cannot be both set to "
-                    "'exhaust'."
-                )
-            if self.n_candidates != 'exhaust' and (
-                    not isinstance(self.n_candidates, Integral) or
-                    self.n_candidates <= 0):
-                raise ValueError(
-                    "n_candidates must be either 'exhaust' "
-                    "or a positive integer"
+                    "n_candidates and min_resources cannot be both set to 'exhaust'."
                 )
 
         self.min_resources_ = self.min_resources
-        if self.min_resources_ in ('smallest', 'exhaust'):
-            if self.resource == 'n_samples':
-                n_splits = self._checked_cv_orig.get_n_splits(X, y, groups)
+        if self.min_resources_ in ("smallest", "exhaust"):
+            if self.resource == "n_samples":
+                n_splits = self._checked_cv_orig.get_n_splits(X, y, **split_params)
                 # please see https://gph.is/1KjihQe for a justification
                 magic_factor = 2
                 self.min_resources_ = n_splits * magic_factor
                 if is_classifier(self.estimator):
+                    y = validate_data(self, X="no_validation", y=y)
+                    check_classification_targets(y)
                     n_classes = np.unique(y).shape[0]
                     self.min_resources_ *= n_classes
             else:
@@ -165,64 +171,98 @@ def _check_input_parameters(self, X, y, groups):
             # in _run_search
 
         self.max_resources_ = self.max_resources
-        if self.max_resources_ == 'auto':
-            if not self.resource == 'n_samples':
+        if self.max_resources_ == "auto":
+            if not self.resource == "n_samples":
                 raise ValueError(
-                    "max_resources can only be 'auto' if resource='n_samples'")
+                    "resource can only be 'n_samples' when max_resources='auto'"
+                )
             self.max_resources_ = _num_samples(X)
 
         if self.min_resources_ > self.max_resources_:
             raise ValueError(
-                f'min_resources_={self.min_resources_} is greater '
-                f'than max_resources_={self.max_resources_}.'
+                f"min_resources_={self.min_resources_} is greater "
+                f"than max_resources_={self.max_resources_}."
             )
 
-    def fit(self, X, y=None, groups=None, **fit_params):
+        if self.min_resources_ == 0:
+            raise ValueError(
+                f"min_resources_={self.min_resources_}: you might have passed "
+                "an empty dataset X."
+            )
+
+    @staticmethod
+    def _select_best_index(refit, refit_metric, results):
+        """Custom refit callable to return the index of the best candidate.
+
+        We want the best candidate out of the last iteration. By default
+        BaseSearchCV would return the best candidate out of all iterations.
+
+        Currently, we only support for a single metric thus `refit` and
+        `refit_metric` are not required.
+        """
+        last_iter = np.max(results["iter"])
+        last_iter_indices = np.flatnonzero(results["iter"] == last_iter)
+
+        test_scores = results["mean_test_score"][last_iter_indices]
+        # If all scores are NaNs there is no way to pick between them,
+        # so we (arbitrarily) declare the zero'th entry the best one
+        if np.isnan(test_scores).all():
+            best_idx = 0
+        else:
+            best_idx = np.nanargmax(test_scores)
+
+        return last_iter_indices[best_idx]
+
+    @_fit_context(
+        # Halving*SearchCV.estimator is not validated yet
+        prefer_skip_nested_validation=False
+    )
+    def fit(self, X, y=None, **params):
         """Run fit with all sets of parameters.
 
         Parameters
         ----------
 
         X : array-like, shape (n_samples, n_features)
-            Training vector, where n_samples is the number of samples and
-            n_features is the number of features.
+            Training vector, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
 
         y : array-like, shape (n_samples,) or (n_samples, n_output), optional
             Target relative to X for classification or regression;
             None for unsupervised learning.
 
-        groups : array-like of shape (n_samples,), default=None
-            Group labels for the samples used while splitting the dataset into
-            train/test set. Only used in conjunction with a "Group" :term:`cv`
-            instance (e.g., :class:`~sklearn.model_selection.GroupKFold`).
+        **params : dict of string -> object
+            Parameters passed to the ``fit`` method of the estimator.
 
-        **fit_params : dict of string -> object
-            Parameters passed to the ``fit`` method of the estimator
+        Returns
+        -------
+        self : object
+            Instance of fitted estimator.
         """
         self._checked_cv_orig = check_cv(
-            self.cv, y, classifier=is_classifier(self.estimator))
+            self.cv, y, classifier=is_classifier(self.estimator)
+        )
 
+        routed_params = self._get_routed_params_for_fit(params)
         self._check_input_parameters(
-            X=X,
-            y=y,
-            groups=groups,
+            X=X, y=y, split_params=routed_params.splitter.split
         )
 
         self._n_samples_orig = _num_samples(X)
 
-        super().fit(X, y=y, groups=None, **fit_params)
+        super().fit(X, y=y, **params)
 
         # Set best_score_: BaseSearchCV does not set it, as refit is a callable
-        self.best_score_ = (
-            self.cv_results_['mean_test_score'][self.best_index_])
+        self.best_score_ = self.cv_results_["mean_test_score"][self.best_index_]
 
         return self
 
     def _run_search(self, evaluate_candidates):
         candidate_params = self._generate_candidate_params()
 
-        if self.resource != 'n_samples' and any(
-                self.resource in candidate for candidate in candidate_params):
+        if self.resource != "n_samples" and any(
+            self.resource in candidate for candidate in candidate_params
+        ):
             # Can only check this now since we need the candidates list
             raise ValueError(
                 f"Cannot use parameter {self.resource} as the resource since "
@@ -231,17 +271,16 @@ def _run_search(self, evaluate_candidates):
 
         # n_required_iterations is the number of iterations needed so that the
         # last iterations evaluates less than `factor` candidates.
-        n_required_iterations = 1 + floor(log(len(candidate_params),
-                                              self.factor))
+        n_required_iterations = 1 + floor(log(len(candidate_params), self.factor))
 
-        if self.min_resources == 'exhaust':
+        if self.min_resources == "exhaust":
             # To exhaust the resources, we want to start with the biggest
             # min_resources possible so that the last (required) iteration
             # uses as many resources as possible
             last_iteration = n_required_iterations - 1
             self.min_resources_ = max(
                 self.min_resources_,
-                self.max_resources_ // self.factor**last_iteration
+                self.max_resources_ // self.factor**last_iteration,
             )
 
         # n_possible_iterations is the number of iterations that we can
@@ -249,8 +288,9 @@ def _run_search(self, evaluate_candidates):
         # max_resources. Depending on max_resources and the number of
         # candidates, this may be higher or smaller than
         # n_required_iterations.
-        n_possible_iterations = 1 + floor(log(
-            self.max_resources_ // self.min_resources_, self.factor))
+        n_possible_iterations = 1 + floor(
+            log(self.max_resources_ // self.min_resources_, self.factor)
+        )
 
         if self.aggressive_elimination:
             n_iterations = n_required_iterations
@@ -258,29 +298,25 @@ def _run_search(self, evaluate_candidates):
             n_iterations = min(n_possible_iterations, n_required_iterations)
 
         if self.verbose:
-            print(f'n_iterations: {n_iterations}')
-            print(f'n_required_iterations: {n_required_iterations}')
-            print(f'n_possible_iterations: {n_possible_iterations}')
-            print(f'min_resources_: {self.min_resources_}')
-            print(f'max_resources_: {self.max_resources_}')
-            print(f'aggressive_elimination: {self.aggressive_elimination}')
-            print(f'factor: {self.factor}')
+            print(f"n_iterations: {n_iterations}")
+            print(f"n_required_iterations: {n_required_iterations}")
+            print(f"n_possible_iterations: {n_possible_iterations}")
+            print(f"min_resources_: {self.min_resources_}")
+            print(f"max_resources_: {self.max_resources_}")
+            print(f"aggressive_elimination: {self.aggressive_elimination}")
+            print(f"factor: {self.factor}")
 
         self.n_resources_ = []
         self.n_candidates_ = []
 
         for itr in range(n_iterations):
-
             power = itr  # default
             if self.aggressive_elimination:
                 # this will set n_resources to the initial value (i.e. the
                 # value of n_resources at the first iteration) for as many
                 # iterations as needed (while candidates are being
                 # eliminated), and then go on as usual.
-                power = max(
-                    0,
-                    itr - n_required_iterations + n_possible_iterations
-                )
+                power = max(0, itr - n_required_iterations + n_possible_iterations)
 
             n_resources = int(self.factor**power * self.min_resources_)
             # guard, probably not needed
@@ -291,18 +327,18 @@ def _run_search(self, evaluate_candidates):
             self.n_candidates_.append(n_candidates)
 
             if self.verbose:
-                print('-' * 10)
-                print(f'iter: {itr}')
-                print(f'n_candidates: {n_candidates}')
-                print(f'n_resources: {n_resources}')
+                print("-" * 10)
+                print(f"iter: {itr}")
+                print(f"n_candidates: {n_candidates}")
+                print(f"n_resources: {n_resources}")
 
-            if self.resource == 'n_samples':
+            if self.resource == "n_samples":
                 # subsampling will be done in cv.split()
                 cv = _SubsampleMetaSplitter(
                     base_cv=self._checked_cv_orig,
                     fraction=n_resources / self._n_samples_orig,
                     subsample_test=True,
-                    random_state=self.random_state
+                    random_state=self.random_state,
                 )
 
             else:
@@ -313,11 +349,14 @@ def _run_search(self, evaluate_candidates):
                     candidate[self.resource] = n_resources
                 cv = self._checked_cv_orig
 
-            more_results = {'iter': [itr] * n_candidates,
-                            'n_resources': [n_resources] * n_candidates}
+            more_results = {
+                "iter": [itr] * n_candidates,
+                "n_resources": [n_resources] * n_candidates,
+            }
 
-            results = evaluate_candidates(candidate_params, cv,
-                                          more_results=more_results)
+            results = evaluate_candidates(
+                candidate_params, cv, more_results=more_results
+            )
 
             n_candidates_to_keep = ceil(n_candidates / self.factor)
             candidate_params = _top_k(results, n_candidates_to_keep, itr)
@@ -354,7 +393,7 @@ class HalvingGridSearchCV(BaseSuccessiveHalving):
 
     Parameters
     ----------
-    estimator : estimator object.
+    estimator : estimator object
         This is assumed to implement the scikit-learn estimator interface.
         Either estimator needs to provide a ``score`` function,
         or ``scoring`` must be passed.
@@ -391,11 +430,12 @@ class HalvingGridSearchCV(BaseSuccessiveHalving):
         iteration.
 
         - 'smallest' is a heuristic that sets `r0` to a small value:
-            - ``n_splits * 2`` when ``resource='n_samples'`` for a regression
-               problem
-            - ``n_classes * n_splits * 2`` when ``resource='n_samples'`` for a
-               classification problem
-            - ``1`` when ``resource != 'n_samples'``
+
+          - ``n_splits * 2`` when ``resource='n_samples'`` for a regression problem
+          - ``n_classes * n_splits * 2`` when ``resource='n_samples'`` for a
+            classification problem
+          - ``1`` when ``resource != 'n_samples'``
+
         - 'exhaust' will set `r0` such that the **last** iteration uses as
           much resources as possible. Namely, the last iteration will use the
           highest value smaller than ``max_resources`` that is a multiple of
@@ -425,7 +465,8 @@ class HalvingGridSearchCV(BaseSuccessiveHalving):
 
         For integer/None inputs, if the estimator is a classifier and ``y`` is
         either binary or multiclass, :class:`StratifiedKFold` is used. In all
-        other cases, :class:`KFold` is used.
+        other cases, :class:`KFold` is used. These splitters are instantiated
+        with `shuffle=False` so the splits will be the same across calls.
 
         Refer :ref:`User Guide ` for the various
         cross-validation strategies that can be used here.
@@ -437,24 +478,40 @@ class HalvingGridSearchCV(BaseSuccessiveHalving):
             deactivating shuffling (`shuffle=False`), or by setting the
             `cv`'s `random_state` parameter to an integer.
 
-    scoring : string, callable, or None, default=None
-        A single string (see :ref:`scoring_parameter`) or a callable
-        (see :ref:`scoring`) to evaluate the predictions on the test set.
-        If None, the estimator's score method is used.
+    scoring : str or callable, default=None
+        Scoring method to use to evaluate the predictions on the test set.
 
-    refit : bool, default=True
-        If True, refit an estimator using the best found parameters on the
-        whole dataset.
+        - str: see :ref:`scoring_string_names` for options.
+        - callable: a scorer callable object (e.g., function) with signature
+          ``scorer(estimator, X, y)``. See :ref:`scoring_callable` for details.
+        - `None`: the `estimator`'s
+          :ref:`default evaluation criterion ` is used.
+
+    refit : bool or callable, default=True
+        Refit an estimator using the best found parameters on the whole
+        dataset.
+
+        Where there are considerations other than maximum score in
+        choosing a best estimator, ``refit`` can be set to a function which
+        returns the selected ``best_index_`` given ``cv_results_``. In that
+        case, the ``best_estimator_`` and ``best_params_`` will be set
+        according to the returned ``best_index_`` while the ``best_score_``
+        attribute will not be available.
 
         The refitted estimator is made available at the ``best_estimator_``
         attribute and permits using ``predict`` directly on this
         ``HalvingGridSearchCV`` instance.
 
+        See :ref:`this example
+        `
+        for an example of how to use ``refit=callable`` to balance model
+        complexity and cross-validated score.
+
     error_score : 'raise' or numeric
         Value to assign to the score if an error occurs in estimator fitting.
         If set to 'raise', the error is raised. If a numeric value is given,
         FitFailedWarning is raised. This parameter does not affect the refit
-        step, which will always raise the error. Default is ``np.nan``
+        step, which will always raise the error. Default is ``np.nan``.
 
     return_train_score : bool, default=False
         If ``False``, the ``cv_results_`` attribute will not include training
@@ -523,8 +580,8 @@ class HalvingGridSearchCV(BaseSuccessiveHalving):
 
     cv_results_ : dict of numpy (masked) ndarrays
         A dict with keys as column headers and values as columns, that can be
-        imported into a pandas ``DataFrame``. It contains many informations for
-        analysing the results of a search.
+        imported into a pandas ``DataFrame``. It contains lots of information
+        for analysing the results of a search.
         Please refer to the :ref:`User guide`
         for details.
 
@@ -559,6 +616,29 @@ class HalvingGridSearchCV(BaseSuccessiveHalving):
 
         This is present only if ``refit`` is not False.
 
+    multimetric_ : bool
+        Whether or not the scorers compute several metrics.
+
+    classes_ : ndarray of shape (n_classes,)
+        The classes labels. This is present only if ``refit`` is specified and
+        the underlying estimator is a classifier.
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`. Only defined if
+        `best_estimator_` is defined (see the documentation for the `refit`
+        parameter for more details) and that `best_estimator_` exposes
+        `n_features_in_` when fit.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Only defined if
+        `best_estimator_` is defined (see the documentation for the `refit`
+        parameter for more details) and that `best_estimator_` exposes
+        `feature_names_in_` when fit.
+
+        .. versionadded:: 1.0
+
     See Also
     --------
     :class:`HalvingRandomSearchCV`:
@@ -569,6 +649,8 @@ class HalvingGridSearchCV(BaseSuccessiveHalving):
     The parameters selected are those that maximize the score of the held-out
     data, according to the scoring parameter.
 
+    All parameter combinations scored with a NaN will share the lowest rank.
+
     Examples
     --------
 
@@ -588,23 +670,48 @@ class HalvingGridSearchCV(BaseSuccessiveHalving):
     >>> search.best_params_  # doctest: +SKIP
     {'max_depth': None, 'min_samples_split': 10, 'n_estimators': 9}
     """
-    _required_parameters = ["estimator", "param_grid"]
-
-    def __init__(self, estimator, param_grid, *,
-                 factor=3, resource='n_samples', max_resources='auto',
-                 min_resources='exhaust', aggressive_elimination=False,
-                 cv=5, scoring=None, refit=True, error_score=np.nan,
-                 return_train_score=True, random_state=None, n_jobs=None,
-                 verbose=0):
-        super().__init__(estimator, scoring=scoring,
-                         n_jobs=n_jobs, refit=refit, verbose=verbose, cv=cv,
-                         random_state=random_state, error_score=error_score,
-                         return_train_score=return_train_score,
-                         max_resources=max_resources, resource=resource,
-                         factor=factor, min_resources=min_resources,
-                         aggressive_elimination=aggressive_elimination)
+
+    _parameter_constraints: dict = {
+        **BaseSuccessiveHalving._parameter_constraints,
+        "param_grid": [dict, list],
+    }
+
+    def __init__(
+        self,
+        estimator,
+        param_grid,
+        *,
+        factor=3,
+        resource="n_samples",
+        max_resources="auto",
+        min_resources="exhaust",
+        aggressive_elimination=False,
+        cv=5,
+        scoring=None,
+        refit=True,
+        error_score=np.nan,
+        return_train_score=True,
+        random_state=None,
+        n_jobs=None,
+        verbose=0,
+    ):
+        super().__init__(
+            estimator,
+            scoring=scoring,
+            n_jobs=n_jobs,
+            refit=refit,
+            verbose=verbose,
+            cv=cv,
+            random_state=random_state,
+            error_score=error_score,
+            return_train_score=return_train_score,
+            max_resources=max_resources,
+            resource=resource,
+            factor=factor,
+            min_resources=min_resources,
+            aggressive_elimination=aggressive_elimination,
+        )
         self.param_grid = param_grid
-        _check_param_grid(self.param_grid)
 
     def _generate_candidate_params(self):
         return ParameterGrid(self.param_grid)
@@ -635,18 +742,20 @@ class HalvingRandomSearchCV(BaseSuccessiveHalving):
 
     Parameters
     ----------
-    estimator : estimator object.
+    estimator : estimator object
         This is assumed to implement the scikit-learn estimator interface.
         Either estimator needs to provide a ``score`` function,
         or ``scoring`` must be passed.
 
-    param_distributions : dict
-        Dictionary with parameters names (string) as keys and distributions
+    param_distributions : dict or list of dicts
+        Dictionary with parameters names (`str`) as keys and distributions
         or lists of parameters to try. Distributions must provide a ``rvs``
         method for sampling (such as those from scipy.stats.distributions).
         If a list is given, it is sampled uniformly.
+        If a list of dicts is given, first a dict is sampled uniformly, and
+        then a parameter is sampled using that dict as above.
 
-    n_candidates : int, default='exhaust'
+    n_candidates : "exhaust" or int, default="exhaust"
         The number of candidate parameters to sample, at the first
         iteration. Using 'exhaust' will sample enough candidates so that the
         last iteration uses as many resources as possible, based on
@@ -678,11 +787,12 @@ class HalvingRandomSearchCV(BaseSuccessiveHalving):
         iteration.
 
         - 'smallest' is a heuristic that sets `r0` to a small value:
-            - ``n_splits * 2`` when ``resource='n_samples'`` for a regression
-               problem
-            - ``n_classes * n_splits * 2`` when ``resource='n_samples'`` for a
-               classification problem
-            - ``1`` when ``resource != 'n_samples'``
+
+          - ``n_splits * 2`` when ``resource='n_samples'`` for a regression problem
+          - ``n_classes * n_splits * 2`` when ``resource='n_samples'`` for a
+            classification problem
+          - ``1`` when ``resource != 'n_samples'``
+
         - 'exhaust' will set `r0` such that the **last** iteration uses as
           much resources as possible. Namely, the last iteration will use the
           highest value smaller than ``max_resources`` that is a multiple of
@@ -712,7 +822,8 @@ class HalvingRandomSearchCV(BaseSuccessiveHalving):
 
         For integer/None inputs, if the estimator is a classifier and ``y`` is
         either binary or multiclass, :class:`StratifiedKFold` is used. In all
-        other cases, :class:`KFold` is used.
+        other cases, :class:`KFold` is used. These splitters are instantiated
+        with `shuffle=False` so the splits will be the same across calls.
 
         Refer :ref:`User Guide ` for the various
         cross-validation strategies that can be used here.
@@ -724,24 +835,40 @@ class HalvingRandomSearchCV(BaseSuccessiveHalving):
             deactivating shuffling (`shuffle=False`), or by setting the
             `cv`'s `random_state` parameter to an integer.
 
-    scoring : string, callable, or None, default=None
-        A single string (see :ref:`scoring_parameter`) or a callable
-        (see :ref:`scoring`) to evaluate the predictions on the test set.
-        If None, the estimator's score method is used.
+    scoring : str or callable, default=None
+        Scoring method to use to evaluate the predictions on the test set.
 
-    refit : bool, default=True
-        If True, refit an estimator using the best found parameters on the
-        whole dataset.
+        - str: see :ref:`scoring_string_names` for options.
+        - callable: a scorer callable object (e.g., function) with signature
+          ``scorer(estimator, X, y)``. See :ref:`scoring_callable` for details.
+        - `None`: the `estimator`'s
+          :ref:`default evaluation criterion ` is used.
+
+    refit : bool or callable, default=True
+        Refit an estimator using the best found parameters on the whole
+        dataset.
+
+        Where there are considerations other than maximum score in
+        choosing a best estimator, ``refit`` can be set to a function which
+        returns the selected ``best_index_`` given ``cv_results_``. In that
+        case, the ``best_estimator_`` and ``best_params_`` will be set
+        according to the returned ``best_index_`` while the ``best_score_``
+        attribute will not be available.
 
         The refitted estimator is made available at the ``best_estimator_``
         attribute and permits using ``predict`` directly on this
         ``HalvingRandomSearchCV`` instance.
 
+        See :ref:`this example
+        `
+        for an example of how to use ``refit=callable`` to balance model
+        complexity and cross-validated score.
+
     error_score : 'raise' or numeric
         Value to assign to the score if an error occurs in estimator fitting.
         If set to 'raise', the error is raised. If a numeric value is given,
         FitFailedWarning is raised. This parameter does not affect the refit
-        step, which will always raise the error. Default is ``np.nan``
+        step, which will always raise the error. Default is ``np.nan``.
 
     return_train_score : bool, default=False
         If ``False``, the ``cv_results_`` attribute will not include training
@@ -812,8 +939,8 @@ class HalvingRandomSearchCV(BaseSuccessiveHalving):
 
     cv_results_ : dict of numpy (masked) ndarrays
         A dict with keys as column headers and values as columns, that can be
-        imported into a pandas ``DataFrame``. It contains many informations for
-        analysing the results of a search.
+        imported into a pandas ``DataFrame``. It contains lots of information
+        for analysing the results of a search.
         Please refer to the :ref:`User guide`
         for details.
 
@@ -848,6 +975,29 @@ class HalvingRandomSearchCV(BaseSuccessiveHalving):
 
         This is present only if ``refit`` is not False.
 
+    multimetric_ : bool
+        Whether or not the scorers compute several metrics.
+
+    classes_ : ndarray of shape (n_classes,)
+        The classes labels. This is present only if ``refit`` is specified and
+        the underlying estimator is a classifier.
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`. Only defined if
+        `best_estimator_` is defined (see the documentation for the `refit`
+        parameter for more details) and that `best_estimator_` exposes
+        `n_features_in_` when fit.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Only defined if
+        `best_estimator_` is defined (see the documentation for the `refit`
+        parameter for more details) and that `best_estimator_` exposes
+        `feature_names_in_` when fit.
+
+        .. versionadded:: 1.0
+
     See Also
     --------
     :class:`HalvingGridSearchCV`:
@@ -858,6 +1008,8 @@ class HalvingRandomSearchCV(BaseSuccessiveHalving):
     The parameters selected are those that maximize the score of the held-out
     data, according to the scoring parameter.
 
+    All parameter combinations scored with a NaN will share the lowest rank.
+
     Examples
     --------
 
@@ -866,6 +1018,7 @@ class HalvingRandomSearchCV(BaseSuccessiveHalving):
     >>> from sklearn.experimental import enable_halving_search_cv  # noqa
     >>> from sklearn.model_selection import HalvingRandomSearchCV
     >>> from scipy.stats import randint
+    >>> import numpy as np
     ...
     >>> X, y = load_iris(return_X_y=True)
     >>> clf = RandomForestClassifier(random_state=0)
@@ -880,31 +1033,63 @@ class HalvingRandomSearchCV(BaseSuccessiveHalving):
     >>> search.best_params_  # doctest: +SKIP
     {'max_depth': None, 'min_samples_split': 10, 'n_estimators': 9}
     """
-    _required_parameters = ["estimator", "param_distributions"]
-
-    def __init__(self, estimator, param_distributions, *,
-                 n_candidates='exhaust', factor=3, resource='n_samples',
-                 max_resources='auto', min_resources='smallest',
-                 aggressive_elimination=False, cv=5, scoring=None,
-                 refit=True, error_score=np.nan, return_train_score=True,
-                 random_state=None, n_jobs=None, verbose=0):
-        super().__init__(estimator, scoring=scoring,
-                         n_jobs=n_jobs, refit=refit, verbose=verbose, cv=cv,
-                         random_state=random_state, error_score=error_score,
-                         return_train_score=return_train_score,
-                         max_resources=max_resources, resource=resource,
-                         factor=factor, min_resources=min_resources,
-                         aggressive_elimination=aggressive_elimination)
+
+    _parameter_constraints: dict = {
+        **BaseSuccessiveHalving._parameter_constraints,
+        "param_distributions": [dict, list],
+        "n_candidates": [
+            Interval(Integral, 0, None, closed="neither"),
+            StrOptions({"exhaust"}),
+        ],
+    }
+
+    def __init__(
+        self,
+        estimator,
+        param_distributions,
+        *,
+        n_candidates="exhaust",
+        factor=3,
+        resource="n_samples",
+        max_resources="auto",
+        min_resources="smallest",
+        aggressive_elimination=False,
+        cv=5,
+        scoring=None,
+        refit=True,
+        error_score=np.nan,
+        return_train_score=True,
+        random_state=None,
+        n_jobs=None,
+        verbose=0,
+    ):
+        super().__init__(
+            estimator,
+            scoring=scoring,
+            n_jobs=n_jobs,
+            refit=refit,
+            verbose=verbose,
+            cv=cv,
+            random_state=random_state,
+            error_score=error_score,
+            return_train_score=return_train_score,
+            max_resources=max_resources,
+            resource=resource,
+            factor=factor,
+            min_resources=min_resources,
+            aggressive_elimination=aggressive_elimination,
+        )
         self.param_distributions = param_distributions
         self.n_candidates = n_candidates
 
     def _generate_candidate_params(self):
         n_candidates_first_iter = self.n_candidates
-        if n_candidates_first_iter == 'exhaust':
+        if n_candidates_first_iter == "exhaust":
             # This will generate enough candidate so that the last iteration
             # uses as much resources as possible
-            n_candidates_first_iter = (
-                self.max_resources_ // self.min_resources_)
-        return ParameterSampler(self.param_distributions,
-                                n_candidates_first_iter,
-                                random_state=self.random_state)
+            n_candidates_first_iter = self.max_resources_ // self.min_resources_
+        return ParameterSampler(
+            self.param_distributions,
+            n_candidates_first_iter,
+            random_state=self.random_state,
+        )
diff --git a/sklearn/model_selection/_split.py b/sklearn/model_selection/_split.py
index 244b2b63af449..ee85af7fe39e6 100644
--- a/sklearn/model_selection/_split.py
+++ b/sklearn/model_selection/_split.py
@@ -3,62 +3,124 @@
 functions to split the data based on a preset strategy.
 """
 
-# Author: Alexandre Gramfort ,
-#         Gael Varoquaux ,
-#         Olivier Grisel 
-#         Raghav RV 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-from collections.abc import Iterable
-import warnings
-from itertools import chain, combinations
-from math import ceil, floor
 import numbers
+import warnings
 from abc import ABCMeta, abstractmethod
+from collections import defaultdict
+from collections.abc import Iterable
 from inspect import signature
+from itertools import chain, combinations
+from math import ceil, floor
 
 import numpy as np
 from scipy.special import comb
 
-from ..utils import indexable, check_random_state, _safe_indexing
-from ..utils import _approximate_mode
-from ..utils.validation import _num_samples, column_or_1d
-from ..utils.validation import check_array
-from ..utils.validation import _deprecate_positional_args
+from ..utils import (
+    _safe_indexing,
+    check_random_state,
+    indexable,
+    metadata_routing,
+)
+from ..utils._array_api import (
+    _convert_to_numpy,
+    ensure_common_namespace_device,
+    get_namespace,
+)
+from ..utils._param_validation import Interval, RealNotInt, validate_params
+from ..utils.extmath import _approximate_mode
+from ..utils.metadata_routing import _MetadataRequester
 from ..utils.multiclass import type_of_target
-from ..base import _pprint
-
-__all__ = ['BaseCrossValidator',
-           'KFold',
-           'GroupKFold',
-           'LeaveOneGroupOut',
-           'LeaveOneOut',
-           'LeavePGroupsOut',
-           'LeavePOut',
-           'RepeatedStratifiedKFold',
-           'RepeatedKFold',
-           'ShuffleSplit',
-           'GroupShuffleSplit',
-           'StratifiedKFold',
-           'StratifiedShuffleSplit',
-           'PredefinedSplit',
-           'train_test_split',
-           'check_cv']
-
-
-class BaseCrossValidator(metaclass=ABCMeta):
-    """Base class for all cross-validators
+from ..utils.validation import _num_samples, check_array, column_or_1d
+
+__all__ = [
+    "BaseCrossValidator",
+    "GroupKFold",
+    "GroupShuffleSplit",
+    "KFold",
+    "LeaveOneGroupOut",
+    "LeaveOneOut",
+    "LeavePGroupsOut",
+    "LeavePOut",
+    "PredefinedSplit",
+    "RepeatedKFold",
+    "RepeatedStratifiedKFold",
+    "ShuffleSplit",
+    "StratifiedGroupKFold",
+    "StratifiedKFold",
+    "StratifiedShuffleSplit",
+    "check_cv",
+    "train_test_split",
+]
+
+
+class _UnsupportedGroupCVMixin:
+    """Mixin for splitters that do not support Groups."""
+
+    def split(self, X, y=None, groups=None):
+        """Generate indices to split data into training and test set.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            Training data, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
+
+        y : array-like of shape (n_samples,)
+            The target variable for supervised learning problems.
+
+        groups : object
+            Always ignored, exists for compatibility.
+
+        Yields
+        ------
+        train : ndarray
+            The training set indices for that split.
+
+        test : ndarray
+            The testing set indices for that split.
+        """
+        if groups is not None:
+            warnings.warn(
+                f"The groups parameter is ignored by {self.__class__.__name__}",
+                UserWarning,
+            )
+        return super().split(X, y, groups=groups)
+
+
+class GroupsConsumerMixin(_MetadataRequester):
+    """A Mixin to ``groups`` by default.
+
+    This Mixin makes the object to request ``groups`` by default as ``True``.
+
+    .. versionadded:: 1.3
+    """
+
+    __metadata_request__split = {"groups": True}
+
+
+class BaseCrossValidator(_MetadataRequester, metaclass=ABCMeta):
+    """Base class for all cross-validators.
 
     Implementations must define `_iter_test_masks` or `_iter_test_indices`.
     """
+
+    # This indicates that by default CV splitters don't have a "groups" kwarg,
+    # unless indicated by inheriting from ``GroupsConsumerMixin``.
+    # This also prevents ``set_split_request`` to be generated for splitters
+    # which don't support ``groups``.
+    __metadata_request__split = {"groups": metadata_routing.UNUSED}
+
     def split(self, X, y=None, groups=None):
         """Generate indices to split data into training and test set.
 
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
-            Training data, where n_samples is the number of samples
-            and n_features is the number of features.
+            Training data, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
 
         y : array-like of shape (n_samples,)
             The target variable for supervised learning problems.
@@ -100,14 +162,14 @@ def _iter_test_indices(self, X=None, y=None, groups=None):
 
     @abstractmethod
     def get_n_splits(self, X=None, y=None, groups=None):
-        """Returns the number of splitting iterations in the cross-validator"""
+        """Returns the number of splitting iterations in the cross-validator."""
 
     def __repr__(self):
         return _build_repr(self)
 
 
-class LeaveOneOut(BaseCrossValidator):
-    """Leave-One-Out cross-validator
+class LeaveOneOut(_UnsupportedGroupCVMixin, BaseCrossValidator):
+    """Leave-One-Out cross-validator.
 
     Provides train/test indices to split data in train/test sets. Each
     sample is used once as a test set (singleton) while the remaining
@@ -134,15 +196,16 @@ class LeaveOneOut(BaseCrossValidator):
     2
     >>> print(loo)
     LeaveOneOut()
-    >>> for train_index, test_index in loo.split(X):
-    ...     print("TRAIN:", train_index, "TEST:", test_index)
-    ...     X_train, X_test = X[train_index], X[test_index]
-    ...     y_train, y_test = y[train_index], y[test_index]
-    ...     print(X_train, X_test, y_train, y_test)
-    TRAIN: [1] TEST: [0]
-    [[3 4]] [[1 2]] [2] [1]
-    TRAIN: [0] TEST: [1]
-    [[1 2]] [[3 4]] [1] [2]
+    >>> for i, (train_index, test_index) in enumerate(loo.split(X)):
+    ...     print(f"Fold {i}:")
+    ...     print(f"  Train: index={train_index}")
+    ...     print(f"  Test:  index={test_index}")
+    Fold 0:
+      Train: index=[1]
+      Test:  index=[0]
+    Fold 1:
+      Train: index=[0]
+      Test:  index=[1]
 
     See Also
     --------
@@ -155,19 +218,18 @@ def _iter_test_indices(self, X, y=None, groups=None):
         n_samples = _num_samples(X)
         if n_samples <= 1:
             raise ValueError(
-                'Cannot perform LeaveOneOut with n_samples={}.'.format(
-                    n_samples)
+                "Cannot perform LeaveOneOut with n_samples={}.".format(n_samples)
             )
         return range(n_samples)
 
     def get_n_splits(self, X, y=None, groups=None):
-        """Returns the number of splitting iterations in the cross-validator
+        """Returns the number of splitting iterations in the cross-validator.
 
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
-            Training data, where n_samples is the number of samples
-            and n_features is the number of features.
+            Training data, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
 
         y : object
             Always ignored, exists for compatibility.
@@ -185,8 +247,8 @@ def get_n_splits(self, X, y=None, groups=None):
         return _num_samples(X)
 
 
-class LeavePOut(BaseCrossValidator):
-    """Leave-P-Out cross-validator
+class LeavePOut(_UnsupportedGroupCVMixin, BaseCrossValidator):
+    """Leave-P-Out cross-validator.
 
     Provides train/test indices to split data in train/test sets. This results
     in testing on all distinct samples of size p, while the remaining n - p
@@ -219,16 +281,28 @@ class LeavePOut(BaseCrossValidator):
     6
     >>> print(lpo)
     LeavePOut(p=2)
-    >>> for train_index, test_index in lpo.split(X):
-    ...     print("TRAIN:", train_index, "TEST:", test_index)
-    ...     X_train, X_test = X[train_index], X[test_index]
-    ...     y_train, y_test = y[train_index], y[test_index]
-    TRAIN: [2 3] TEST: [0 1]
-    TRAIN: [1 3] TEST: [0 2]
-    TRAIN: [1 2] TEST: [0 3]
-    TRAIN: [0 3] TEST: [1 2]
-    TRAIN: [0 2] TEST: [1 3]
-    TRAIN: [0 1] TEST: [2 3]
+    >>> for i, (train_index, test_index) in enumerate(lpo.split(X)):
+    ...     print(f"Fold {i}:")
+    ...     print(f"  Train: index={train_index}")
+    ...     print(f"  Test:  index={test_index}")
+    Fold 0:
+      Train: index=[2 3]
+      Test:  index=[0 1]
+    Fold 1:
+      Train: index=[1 3]
+      Test:  index=[0 2]
+    Fold 2:
+      Train: index=[1 2]
+      Test:  index=[0 3]
+    Fold 3:
+      Train: index=[0 3]
+      Test:  index=[1 2]
+    Fold 4:
+      Train: index=[0 2]
+      Test:  index=[1 3]
+    Fold 5:
+      Train: index=[0 1]
+      Test:  index=[2 3]
     """
 
     def __init__(self, p):
@@ -238,20 +312,21 @@ def _iter_test_indices(self, X, y=None, groups=None):
         n_samples = _num_samples(X)
         if n_samples <= self.p:
             raise ValueError(
-                'p={} must be strictly less than the number of '
-                'samples={}'.format(self.p, n_samples)
+                "p={} must be strictly less than the number of samples={}".format(
+                    self.p, n_samples
+                )
             )
         for combination in combinations(range(n_samples), self.p):
             yield np.array(combination)
 
     def get_n_splits(self, X, y=None, groups=None):
-        """Returns the number of splitting iterations in the cross-validator
+        """Returns the number of splitting iterations in the cross-validator.
 
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
-            Training data, where n_samples is the number of samples
-            and n_features is the number of features.
+            Training data, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
 
         y : object
             Always ignored, exists for compatibility.
@@ -265,32 +340,34 @@ def get_n_splits(self, X, y=None, groups=None):
 
 
 class _BaseKFold(BaseCrossValidator, metaclass=ABCMeta):
-    """Base class for KFold, GroupKFold, and StratifiedKFold"""
+    """Base class for K-Fold cross-validators and TimeSeriesSplit."""
 
     @abstractmethod
-    @_deprecate_positional_args
     def __init__(self, n_splits, *, shuffle, random_state):
         if not isinstance(n_splits, numbers.Integral):
-            raise ValueError('The number of folds must be of Integral type. '
-                             '%s of type %s was passed.'
-                             % (n_splits, type(n_splits)))
+            raise ValueError(
+                "The number of folds must be of Integral type. "
+                "%s of type %s was passed." % (n_splits, type(n_splits))
+            )
         n_splits = int(n_splits)
 
         if n_splits <= 1:
             raise ValueError(
                 "k-fold cross-validation requires at least one"
                 " train/test split by setting n_splits=2 or more,"
-                " got n_splits={0}.".format(n_splits))
+                " got n_splits={0}.".format(n_splits)
+            )
 
         if not isinstance(shuffle, bool):
-            raise TypeError("shuffle must be True or False;"
-                            " got {0}".format(shuffle))
+            raise TypeError("shuffle must be True or False; got {0}".format(shuffle))
 
         if not shuffle and random_state is not None:  # None is the default
             raise ValueError(
-                'Setting a random_state has no effect since shuffle is '
-                'False. You should leave '
-                'random_state to its default (None), or set shuffle=True.',
+                (
+                    "Setting a random_state has no effect since shuffle is "
+                    "False. You should leave "
+                    "random_state to its default (None), or set shuffle=True."
+                ),
             )
 
         self.n_splits = n_splits
@@ -303,8 +380,8 @@ def split(self, X, y=None, groups=None):
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
-            Training data, where n_samples is the number of samples
-            and n_features is the number of features.
+            Training data, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
 
         y : array-like of shape (n_samples,), default=None
             The target variable for supervised learning problems.
@@ -325,15 +402,17 @@ def split(self, X, y=None, groups=None):
         n_samples = _num_samples(X)
         if self.n_splits > n_samples:
             raise ValueError(
-                ("Cannot have number of splits n_splits={0} greater"
-                 " than the number of samples: n_samples={1}.")
-                .format(self.n_splits, n_samples))
+                (
+                    "Cannot have number of splits n_splits={0} greater"
+                    " than the number of samples: n_samples={1}."
+                ).format(self.n_splits, n_samples)
+            )
 
         for train, test in super().split(X, y, groups):
             yield train, test
 
     def get_n_splits(self, X=None, y=None, groups=None):
-        """Returns the number of splitting iterations in the cross-validator
+        """Returns the number of splitting iterations in the cross-validator.
 
         Parameters
         ----------
@@ -354,8 +433,8 @@ def get_n_splits(self, X=None, y=None, groups=None):
         return self.n_splits
 
 
-class KFold(_BaseKFold):
-    """K-Folds cross-validator
+class KFold(_UnsupportedGroupCVMixin, _BaseKFold):
+    """K-Fold cross-validator.
 
     Provides train/test indices to split data in train/test sets. Split
     dataset into k consecutive folds (without shuffling by default).
@@ -365,6 +444,10 @@ class KFold(_BaseKFold):
 
     Read more in the :ref:`User Guide `.
 
+    For visualisation of cross-validation behaviour and
+    comparison between common scikit-learn split methods
+    refer to :ref:`sphx_glr_auto_examples_model_selection_plot_cv_indices.py`
+
     Parameters
     ----------
     n_splits : int, default=5
@@ -395,12 +478,16 @@ class KFold(_BaseKFold):
     2
     >>> print(kf)
     KFold(n_splits=2, random_state=None, shuffle=False)
-    >>> for train_index, test_index in kf.split(X):
-    ...     print("TRAIN:", train_index, "TEST:", test_index)
-    ...     X_train, X_test = X[train_index], X[test_index]
-    ...     y_train, y_test = y[train_index], y[test_index]
-    TRAIN: [2 3] TEST: [0 1]
-    TRAIN: [0 1] TEST: [2 3]
+    >>> for i, (train_index, test_index) in enumerate(kf.split(X)):
+    ...     print(f"Fold {i}:")
+    ...     print(f"  Train: index={train_index}")
+    ...     print(f"  Test:  index={test_index}")
+    Fold 0:
+      Train: index=[2 3]
+      Test:  index=[0 1]
+    Fold 1:
+      Train: index=[0 1]
+      Test:  index=[2 3]
 
     Notes
     -----
@@ -414,7 +501,7 @@ class KFold(_BaseKFold):
 
     See Also
     --------
-    StratifiedKFold : Takes group information into account to avoid building
+    StratifiedKFold : Takes class information into account to avoid building
         folds with imbalanced class distributions (for binary or multiclass
         classification tasks).
 
@@ -422,11 +509,9 @@ class KFold(_BaseKFold):
 
     RepeatedKFold : Repeats K-Fold n times.
     """
-    @_deprecate_positional_args
-    def __init__(self, n_splits=5, *, shuffle=False,
-                 random_state=None):
-        super().__init__(n_splits=n_splits, shuffle=shuffle,
-                         random_state=random_state)
+
+    def __init__(self, n_splits=5, *, shuffle=False, random_state=None):
+        super().__init__(n_splits=n_splits, shuffle=shuffle, random_state=random_state)
 
     def _iter_test_indices(self, X, y=None, groups=None):
         n_samples = _num_samples(X)
@@ -436,7 +521,7 @@ def _iter_test_indices(self, X, y=None, groups=None):
 
         n_splits = self.n_splits
         fold_sizes = np.full(n_splits, n_samples // n_splits, dtype=int)
-        fold_sizes[:n_samples % n_splits] += 1
+        fold_sizes[: n_samples % n_splits] += 1
         current = 0
         for fold_size in fold_sizes:
             start, stop = current, current + fold_size
@@ -444,17 +529,21 @@ def _iter_test_indices(self, X, y=None, groups=None):
             current = stop
 
 
-class GroupKFold(_BaseKFold):
+class GroupKFold(GroupsConsumerMixin, _BaseKFold):
     """K-fold iterator variant with non-overlapping groups.
 
-    The same group will not appear in two different folds (the number of
-    distinct groups has to be at least equal to the number of folds).
+    Each group will appear exactly once in the test set across all folds (the
+    number of distinct groups has to be at least equal to the number of folds).
 
     The folds are approximately balanced in the sense that the number of
-    distinct groups is approximately the same in each fold.
+    samples is approximately the same in each test fold when `shuffle` is True.
 
     Read more in the :ref:`User Guide `.
 
+    For visualisation of cross-validation behaviour and
+    comparison between common scikit-learn split methods
+    refer to :ref:`sphx_glr_auto_examples_model_selection_plot_cv_indices.py`
+
     Parameters
     ----------
     n_splits : int, default=5
@@ -463,77 +552,109 @@ class GroupKFold(_BaseKFold):
         .. versionchanged:: 0.22
             ``n_splits`` default value changed from 3 to 5.
 
+    shuffle : bool, default=False
+        Whether to shuffle the groups before splitting into batches.
+        Note that the samples within each split will not be shuffled.
+
+        .. versionadded:: 1.6
+
+    random_state : int, RandomState instance or None, default=None
+        When `shuffle` is True, `random_state` affects the ordering of the
+        indices, which controls the randomness of each fold. Otherwise, this
+        parameter has no effect.
+        Pass an int for reproducible output across multiple function calls.
+        See :term:`Glossary `.
+
+        .. versionadded:: 1.6
+
+    Notes
+    -----
+    Groups appear in an arbitrary order throughout the folds.
+
     Examples
     --------
     >>> import numpy as np
     >>> from sklearn.model_selection import GroupKFold
-    >>> X = np.array([[1, 2], [3, 4], [5, 6], [7, 8]])
-    >>> y = np.array([1, 2, 3, 4])
-    >>> groups = np.array([0, 0, 2, 2])
+    >>> X = np.array([[1, 2], [3, 4], [5, 6], [7, 8], [9, 10], [11, 12]])
+    >>> y = np.array([1, 2, 3, 4, 5, 6])
+    >>> groups = np.array([0, 0, 2, 2, 3, 3])
     >>> group_kfold = GroupKFold(n_splits=2)
     >>> group_kfold.get_n_splits(X, y, groups)
     2
     >>> print(group_kfold)
-    GroupKFold(n_splits=2)
-    >>> for train_index, test_index in group_kfold.split(X, y, groups):
-    ...     print("TRAIN:", train_index, "TEST:", test_index)
-    ...     X_train, X_test = X[train_index], X[test_index]
-    ...     y_train, y_test = y[train_index], y[test_index]
-    ...     print(X_train, X_test, y_train, y_test)
-    ...
-    TRAIN: [0 1] TEST: [2 3]
-    [[1 2]
-     [3 4]] [[5 6]
-     [7 8]] [1 2] [3 4]
-    TRAIN: [2 3] TEST: [0 1]
-    [[5 6]
-     [7 8]] [[1 2]
-     [3 4]] [3 4] [1 2]
+    GroupKFold(n_splits=2, random_state=None, shuffle=False)
+    >>> for i, (train_index, test_index) in enumerate(group_kfold.split(X, y, groups)):
+    ...     print(f"Fold {i}:")
+    ...     print(f"  Train: index={train_index}, group={groups[train_index]}")
+    ...     print(f"  Test:  index={test_index}, group={groups[test_index]}")
+    Fold 0:
+      Train: index=[2 3], group=[2 2]
+      Test:  index=[0 1 4 5], group=[0 0 3 3]
+    Fold 1:
+      Train: index=[0 1 4 5], group=[0 0 3 3]
+      Test:  index=[2 3], group=[2 2]
 
     See Also
     --------
     LeaveOneGroupOut : For splitting the data according to explicit
         domain-specific stratification of the dataset.
+
+    StratifiedKFold : Takes class information into account to avoid building
+        folds with imbalanced class proportions (for binary or multiclass
+        classification tasks).
     """
-    def __init__(self, n_splits=5):
-        super().__init__(n_splits, shuffle=False, random_state=None)
+
+    def __init__(self, n_splits=5, *, shuffle=False, random_state=None):
+        super().__init__(n_splits, shuffle=shuffle, random_state=random_state)
 
     def _iter_test_indices(self, X, y, groups):
         if groups is None:
             raise ValueError("The 'groups' parameter should not be None.")
-        groups = check_array(groups, ensure_2d=False, dtype=None)
+        groups = check_array(groups, input_name="groups", ensure_2d=False, dtype=None)
 
-        unique_groups, groups = np.unique(groups, return_inverse=True)
+        unique_groups, group_idx = np.unique(groups, return_inverse=True)
         n_groups = len(unique_groups)
 
         if self.n_splits > n_groups:
-            raise ValueError("Cannot have number of splits n_splits=%d greater"
-                             " than the number of groups: %d."
-                             % (self.n_splits, n_groups))
+            raise ValueError(
+                "Cannot have number of splits n_splits=%d greater"
+                " than the number of groups: %d." % (self.n_splits, n_groups)
+            )
+
+        if self.shuffle:
+            # Split and shuffle unique groups across n_splits
+            rng = check_random_state(self.random_state)
+            unique_groups = rng.permutation(unique_groups)
+            split_groups = np.array_split(unique_groups, self.n_splits)
 
-        # Weight groups by their number of occurrences
-        n_samples_per_group = np.bincount(groups)
+            for test_group_ids in split_groups:
+                test_mask = np.isin(groups, test_group_ids)
+                yield np.where(test_mask)[0]
+
+        else:
+            # Weight groups by their number of occurrences
+            n_samples_per_group = np.bincount(group_idx)
 
-        # Distribute the most frequent groups first
-        indices = np.argsort(n_samples_per_group)[::-1]
-        n_samples_per_group = n_samples_per_group[indices]
+            # Distribute the most frequent groups first
+            indices = np.argsort(n_samples_per_group)[::-1]
+            n_samples_per_group = n_samples_per_group[indices]
 
-        # Total weight of each fold
-        n_samples_per_fold = np.zeros(self.n_splits)
+            # Total weight of each fold
+            n_samples_per_fold = np.zeros(self.n_splits)
 
-        # Mapping from group index to fold index
-        group_to_fold = np.zeros(len(unique_groups))
+            # Mapping from group index to fold index
+            group_to_fold = np.zeros(len(unique_groups))
 
-        # Distribute samples by adding the largest weight to the lightest fold
-        for group_index, weight in enumerate(n_samples_per_group):
-            lightest_fold = np.argmin(n_samples_per_fold)
-            n_samples_per_fold[lightest_fold] += weight
-            group_to_fold[indices[group_index]] = lightest_fold
+            # Distribute samples by adding the largest weight to the lightest fold
+            for group_index, weight in enumerate(n_samples_per_group):
+                lightest_fold = np.argmin(n_samples_per_fold)
+                n_samples_per_fold[lightest_fold] += weight
+                group_to_fold[indices[group_index]] = lightest_fold
 
-        indices = group_to_fold[groups]
+            indices = group_to_fold[group_idx]
 
-        for f in range(self.n_splits):
-            yield np.where(indices == f)[0]
+            for f in range(self.n_splits):
+                yield np.where(indices == f)[0]
 
     def split(self, X, y=None, groups=None):
         """Generate indices to split data into training and test set.
@@ -541,8 +662,8 @@ def split(self, X, y=None, groups=None):
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
-            Training data, where n_samples is the number of samples
-            and n_features is the number of features.
+            Training data, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
 
         y : array-like of shape (n_samples,), default=None
             The target variable for supervised learning problems.
@@ -563,16 +684,26 @@ def split(self, X, y=None, groups=None):
 
 
 class StratifiedKFold(_BaseKFold):
-    """Stratified K-Folds cross-validator.
+    """Class-wise stratified K-Fold cross-validator.
 
     Provides train/test indices to split data in train/test sets.
 
     This cross-validation object is a variation of KFold that returns
     stratified folds. The folds are made by preserving the percentage of
-    samples for each class.
+    samples for each class in `y` in a binary or multiclass classification
+    setting.
 
     Read more in the :ref:`User Guide `.
 
+    For visualisation of cross-validation behaviour and
+    comparison between common scikit-learn split methods
+    refer to :ref:`sphx_glr_auto_examples_model_selection_plot_cv_indices.py`
+
+    .. note::
+
+        Stratification on the class label solves an engineering problem rather
+        than a statistical one. See :ref:`stratification` for more details.
+
     Parameters
     ----------
     n_splits : int, default=5
@@ -603,12 +734,16 @@ class StratifiedKFold(_BaseKFold):
     2
     >>> print(skf)
     StratifiedKFold(n_splits=2, random_state=None, shuffle=False)
-    >>> for train_index, test_index in skf.split(X, y):
-    ...     print("TRAIN:", train_index, "TEST:", test_index)
-    ...     X_train, X_test = X[train_index], X[test_index]
-    ...     y_train, y_test = y[train_index], y[test_index]
-    TRAIN: [1 3] TEST: [0 2]
-    TRAIN: [0 2] TEST: [1 3]
+    >>> for i, (train_index, test_index) in enumerate(skf.split(X, y)):
+    ...     print(f"Fold {i}:")
+    ...     print(f"  Train: index={train_index}")
+    ...     print(f"  Test:  index={test_index}")
+    Fold 0:
+      Train: index=[1 3]
+      Test:  index=[0 2]
+    Fold 1:
+      Train: index=[0 2]
+      Test:  index=[1 3]
 
     Notes
     -----
@@ -631,20 +766,29 @@ class StratifiedKFold(_BaseKFold):
     --------
     RepeatedStratifiedKFold : Repeats Stratified K-Fold n times.
     """
-    @_deprecate_positional_args
+
     def __init__(self, n_splits=5, *, shuffle=False, random_state=None):
-        super().__init__(n_splits=n_splits, shuffle=shuffle,
-                         random_state=random_state)
+        super().__init__(n_splits=n_splits, shuffle=shuffle, random_state=random_state)
 
     def _make_test_folds(self, X, y=None):
         rng = check_random_state(self.random_state)
-        y = np.asarray(y)
+        # XXX: as of now, cross-validation splitters only operate in NumPy-land
+        # without attempting to leverage array API namespace features. However
+        # they might be fed by array API inputs, e.g. in CV-enabled estimators so
+        # we need the following explicit conversion:
+        xp, is_array_api = get_namespace(y)
+        if is_array_api:
+            y = _convert_to_numpy(y, xp)
+        else:
+            y = np.asarray(y)
         type_of_target_y = type_of_target(y)
-        allowed_target_types = ('binary', 'multiclass')
+        allowed_target_types = ("binary", "multiclass")
         if type_of_target_y not in allowed_target_types:
             raise ValueError(
-                'Supported target types are: {}. Got {!r} instead.'.format(
-                    allowed_target_types, type_of_target_y))
+                "Supported target types are: {}. Got {!r} instead.".format(
+                    allowed_target_types, type_of_target_y
+                )
+            )
 
         y = column_or_1d(y)
 
@@ -659,26 +803,33 @@ def _make_test_folds(self, X, y=None):
         y_counts = np.bincount(y_encoded)
         min_groups = np.min(y_counts)
         if np.all(self.n_splits > y_counts):
-            raise ValueError("n_splits=%d cannot be greater than the"
-                             " number of members in each class."
-                             % (self.n_splits))
+            raise ValueError(
+                "n_splits=%d cannot be greater than the"
+                " number of members in each class." % (self.n_splits)
+            )
         if self.n_splits > min_groups:
-            warnings.warn(("The least populated class in y has only %d"
-                           " members, which is less than n_splits=%d."
-                           % (min_groups, self.n_splits)), UserWarning)
+            warnings.warn(
+                "The least populated class in y has only %d"
+                " members, which is less than n_splits=%d."
+                % (min_groups, self.n_splits),
+                UserWarning,
+            )
 
         # Determine the optimal number of samples from each class in each fold,
         # using round robin over the sorted y. (This can be done direct from
         # counts, but that code is unreadable.)
         y_order = np.sort(y_encoded)
         allocation = np.asarray(
-            [np.bincount(y_order[i::self.n_splits], minlength=n_classes)
-             for i in range(self.n_splits)])
+            [
+                np.bincount(y_order[i :: self.n_splits], minlength=n_classes)
+                for i in range(self.n_splits)
+            ]
+        )
 
         # To maintain the data order dependencies as best as possible within
         # the stratification constraint, we assign samples from each class in
         # blocks (and then mess that up when shuffle=True).
-        test_folds = np.empty(len(y), dtype='i')
+        test_folds = np.empty(len(y), dtype="i")
         for k in range(n_classes):
             # since the kth column of allocation stores the number of samples
             # of class k in each test set, this generates blocks of fold
@@ -700,8 +851,8 @@ def split(self, X, y, groups=None):
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
-            Training data, where n_samples is the number of samples
-            and n_features is the number of features.
+            Training data, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
 
             Note that providing ``y`` is sufficient to generate the splits and
             hence ``np.zeros(n_samples)`` may be used as a placeholder for
@@ -728,17 +879,235 @@ def split(self, X, y, groups=None):
         split. You can make the results identical by setting `random_state`
         to an integer.
         """
-        y = check_array(y, ensure_2d=False, dtype=None)
+        if groups is not None:
+            warnings.warn(
+                f"The groups parameter is ignored by {self.__class__.__name__}",
+                UserWarning,
+            )
+        y = check_array(y, input_name="y", ensure_2d=False, dtype=None)
         return super().split(X, y, groups)
 
 
+class StratifiedGroupKFold(GroupsConsumerMixin, _BaseKFold):
+    """Class-wise stratified K-Fold iterator variant with non-overlapping groups.
+
+    This cross-validation object is a variation of StratifiedKFold attempts to
+    return stratified folds with non-overlapping groups. The folds are made by
+    preserving the percentage of samples for each class in `y` in a binary or
+    multiclass classification setting.
+
+    Each group will appear exactly once in the test set across all folds (the
+    number of distinct groups has to be at least equal to the number of folds).
+
+    The difference between :class:`GroupKFold`
+    and `StratifiedGroupKFold` is that
+    the former attempts to create balanced folds such that the number of
+    distinct groups is approximately the same in each fold, whereas
+    `StratifiedGroupKFold` attempts to create folds which preserve the
+    percentage of samples for each class as much as possible given the
+    constraint of non-overlapping groups between splits.
+
+    Read more in the :ref:`User Guide `.
+
+    For visualisation of cross-validation behaviour and
+    comparison between common scikit-learn split methods
+    refer to :ref:`sphx_glr_auto_examples_model_selection_plot_cv_indices.py`
+
+    .. note::
+
+        Stratification on the class label solves an engineering problem rather
+        than a statistical one. See :ref:`stratification` for more details.
+
+    Parameters
+    ----------
+    n_splits : int, default=5
+        Number of folds. Must be at least 2.
+
+    shuffle : bool, default=False
+        Whether to shuffle each class's samples before splitting into batches.
+        Note that the samples within each split will not be shuffled.
+        This implementation can only shuffle groups that have approximately the
+        same y distribution, no global shuffle will be performed.
+
+    random_state : int or RandomState instance, default=None
+        When `shuffle` is True, `random_state` affects the ordering of the
+        indices, which controls the randomness of each fold for each class.
+        Otherwise, leave `random_state` as `None`.
+        Pass an int for reproducible output across multiple function calls.
+        See :term:`Glossary `.
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from sklearn.model_selection import StratifiedGroupKFold
+    >>> X = np.ones((17, 2))
+    >>> y = np.array([0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
+    >>> groups = np.array([1, 1, 2, 2, 3, 3, 3, 4, 5, 5, 5, 5, 6, 6, 7, 8, 8])
+    >>> sgkf = StratifiedGroupKFold(n_splits=3)
+    >>> sgkf.get_n_splits(X, y)
+    3
+    >>> print(sgkf)
+    StratifiedGroupKFold(n_splits=3, random_state=None, shuffle=False)
+    >>> for i, (train_index, test_index) in enumerate(sgkf.split(X, y, groups)):
+    ...     print(f"Fold {i}:")
+    ...     print(f"  Train: index={train_index}")
+    ...     print(f"         group={groups[train_index]}")
+    ...     print(f"  Test:  index={test_index}")
+    ...     print(f"         group={groups[test_index]}")
+    Fold 0:
+      Train: index=[ 0  1  2  3  7  8  9 10 11 15 16]
+             group=[1 1 2 2 4 5 5 5 5 8 8]
+      Test:  index=[ 4  5  6 12 13 14]
+             group=[3 3 3 6 6 7]
+    Fold 1:
+      Train: index=[ 4  5  6  7  8  9 10 11 12 13 14]
+             group=[3 3 3 4 5 5 5 5 6 6 7]
+      Test:  index=[ 0  1  2  3 15 16]
+             group=[1 1 2 2 8 8]
+    Fold 2:
+      Train: index=[ 0  1  2  3  4  5  6 12 13 14 15 16]
+             group=[1 1 2 2 3 3 3 6 6 7 8 8]
+      Test:  index=[ 7  8  9 10 11]
+             group=[4 5 5 5 5]
+
+    Notes
+    -----
+    The implementation is designed to:
+
+    * Mimic the behavior of StratifiedKFold as much as possible for trivial
+      groups (e.g. when each group contains only one sample).
+    * Be invariant to class label: relabelling ``y = ["Happy", "Sad"]`` to
+      ``y = [1, 0]`` should not change the indices generated.
+    * Stratify based on samples as much as possible while keeping
+      non-overlapping groups constraint. That means that in some cases when
+      there is a small number of groups containing a large number of samples
+      the stratification will not be possible and the behavior will be close
+      to GroupKFold.
+
+    See also
+    --------
+    StratifiedKFold: Takes class information into account to build folds which
+        retain class distributions (for binary or multiclass classification
+        tasks).
+
+    GroupKFold: K-fold iterator variant with non-overlapping groups.
+    """
+
+    def __init__(self, n_splits=5, shuffle=False, random_state=None):
+        super().__init__(n_splits=n_splits, shuffle=shuffle, random_state=random_state)
+
+    def _iter_test_indices(self, X, y, groups):
+        # Implementation is based on this kaggle kernel:
+        # https://www.kaggle.com/jakubwasikowski/stratified-group-k-fold-cross-validation
+        # and is a subject to Apache 2.0 License. You may obtain a copy of the
+        # License at http://www.apache.org/licenses/LICENSE-2.0
+        # Changelist:
+        # - Refactored function to a class following scikit-learn KFold
+        #   interface.
+        # - Added heuristic for assigning group to the least populated fold in
+        #   cases when all other criteria are equal
+        # - Swtch from using python ``Counter`` to ``np.unique`` to get class
+        #   distribution
+        # - Added scikit-learn checks for input: checking that target is binary
+        #   or multiclass, checking passed random state, checking that number
+        #   of splits is less than number of members in each class, checking
+        #   that least populated class has more members than there are splits.
+        rng = check_random_state(self.random_state)
+        y = np.asarray(y)
+        type_of_target_y = type_of_target(y)
+        allowed_target_types = ("binary", "multiclass")
+        if type_of_target_y not in allowed_target_types:
+            raise ValueError(
+                "Supported target types are: {}. Got {!r} instead.".format(
+                    allowed_target_types, type_of_target_y
+                )
+            )
+
+        y = column_or_1d(y)
+        _, y_inv, y_cnt = np.unique(y, return_inverse=True, return_counts=True)
+        if np.all(self.n_splits > y_cnt):
+            raise ValueError(
+                "n_splits=%d cannot be greater than the"
+                " number of members in each class." % (self.n_splits)
+            )
+        n_smallest_class = np.min(y_cnt)
+        if self.n_splits > n_smallest_class:
+            warnings.warn(
+                "The least populated class in y has only %d"
+                " members, which is less than n_splits=%d."
+                % (n_smallest_class, self.n_splits),
+                UserWarning,
+            )
+        n_classes = len(y_cnt)
+
+        _, groups_inv, groups_cnt = np.unique(
+            groups, return_inverse=True, return_counts=True
+        )
+        y_counts_per_group = np.zeros((len(groups_cnt), n_classes))
+        for class_idx, group_idx in zip(y_inv, groups_inv):
+            y_counts_per_group[group_idx, class_idx] += 1
+
+        y_counts_per_fold = np.zeros((self.n_splits, n_classes))
+        groups_per_fold = defaultdict(set)
+
+        if self.shuffle:
+            rng.shuffle(y_counts_per_group)
+
+        # Stable sort to keep shuffled order for groups with the same
+        # class distribution variance
+        sorted_groups_idx = np.argsort(
+            -np.std(y_counts_per_group, axis=1), kind="mergesort"
+        )
+
+        for group_idx in sorted_groups_idx:
+            group_y_counts = y_counts_per_group[group_idx]
+            best_fold = self._find_best_fold(
+                y_counts_per_fold=y_counts_per_fold,
+                y_cnt=y_cnt,
+                group_y_counts=group_y_counts,
+            )
+            y_counts_per_fold[best_fold] += group_y_counts
+            groups_per_fold[best_fold].add(group_idx)
+
+        for i in range(self.n_splits):
+            test_indices = [
+                idx
+                for idx, group_idx in enumerate(groups_inv)
+                if group_idx in groups_per_fold[i]
+            ]
+            yield test_indices
+
+    def _find_best_fold(self, y_counts_per_fold, y_cnt, group_y_counts):
+        best_fold = None
+        min_eval = np.inf
+        min_samples_in_fold = np.inf
+        for i in range(self.n_splits):
+            y_counts_per_fold[i] += group_y_counts
+            # Summarise the distribution over classes in each proposed fold
+            std_per_class = np.std(y_counts_per_fold / y_cnt.reshape(1, -1), axis=0)
+            y_counts_per_fold[i] -= group_y_counts
+            fold_eval = np.mean(std_per_class)
+            samples_in_fold = np.sum(y_counts_per_fold[i])
+            is_current_fold_better = fold_eval < min_eval or (
+                np.isclose(fold_eval, min_eval)
+                and samples_in_fold < min_samples_in_fold
+            )
+            if is_current_fold_better:
+                min_eval = fold_eval
+                min_samples_in_fold = samples_in_fold
+                best_fold = i
+        return best_fold
+
+
 class TimeSeriesSplit(_BaseKFold):
-    """Time Series cross-validator
+    """Time Series cross-validator.
 
-    Provides train/test indices to split time series data samples
-    that are observed at fixed time intervals, in train/test sets.
-    In each split, test indices must be higher than before, and thus shuffling
-    in cross validator is inappropriate.
+    Provides train/test indices to split time-ordered data, where other
+    cross-validation methods are inappropriate, as they would lead to training
+    on future data and evaluating on past data.
+    To ensure comparable metrics across folds, samples must be equally spaced.
+    Once this condition is met, each test set covers the same time duration,
+    while the train set size accumulates data from previous splits.
 
     This cross-validation object is a variation of :class:`KFold`.
     In the kth split, it returns first k folds as train set and the
@@ -749,6 +1118,10 @@ class TimeSeriesSplit(_BaseKFold):
 
     Read more in the :ref:`User Guide `.
 
+    For visualisation of cross-validation behaviour and
+    comparison between common scikit-learn split methods
+    refer to :ref:`sphx_glr_auto_examples_model_selection_plot_cv_indices.py`
+
     .. versionadded:: 0.18
 
     Parameters
@@ -784,50 +1157,72 @@ class TimeSeriesSplit(_BaseKFold):
     >>> tscv = TimeSeriesSplit()
     >>> print(tscv)
     TimeSeriesSplit(gap=0, max_train_size=None, n_splits=5, test_size=None)
-    >>> for train_index, test_index in tscv.split(X):
-    ...     print("TRAIN:", train_index, "TEST:", test_index)
-    ...     X_train, X_test = X[train_index], X[test_index]
-    ...     y_train, y_test = y[train_index], y[test_index]
-    TRAIN: [0] TEST: [1]
-    TRAIN: [0 1] TEST: [2]
-    TRAIN: [0 1 2] TEST: [3]
-    TRAIN: [0 1 2 3] TEST: [4]
-    TRAIN: [0 1 2 3 4] TEST: [5]
+    >>> for i, (train_index, test_index) in enumerate(tscv.split(X)):
+    ...     print(f"Fold {i}:")
+    ...     print(f"  Train: index={train_index}")
+    ...     print(f"  Test:  index={test_index}")
+    Fold 0:
+      Train: index=[0]
+      Test:  index=[1]
+    Fold 1:
+      Train: index=[0 1]
+      Test:  index=[2]
+    Fold 2:
+      Train: index=[0 1 2]
+      Test:  index=[3]
+    Fold 3:
+      Train: index=[0 1 2 3]
+      Test:  index=[4]
+    Fold 4:
+      Train: index=[0 1 2 3 4]
+      Test:  index=[5]
     >>> # Fix test_size to 2 with 12 samples
     >>> X = np.random.randn(12, 2)
     >>> y = np.random.randint(0, 2, 12)
     >>> tscv = TimeSeriesSplit(n_splits=3, test_size=2)
-    >>> for train_index, test_index in tscv.split(X):
-    ...    print("TRAIN:", train_index, "TEST:", test_index)
-    ...    X_train, X_test = X[train_index], X[test_index]
-    ...    y_train, y_test = y[train_index], y[test_index]
-    TRAIN: [0 1 2 3 4 5] TEST: [6 7]
-    TRAIN: [0 1 2 3 4 5 6 7] TEST: [8 9]
-    TRAIN: [0 1 2 3 4 5 6 7 8 9] TEST: [10 11]
+    >>> for i, (train_index, test_index) in enumerate(tscv.split(X)):
+    ...     print(f"Fold {i}:")
+    ...     print(f"  Train: index={train_index}")
+    ...     print(f"  Test:  index={test_index}")
+    Fold 0:
+      Train: index=[0 1 2 3 4 5]
+      Test:  index=[6 7]
+    Fold 1:
+      Train: index=[0 1 2 3 4 5 6 7]
+      Test:  index=[8 9]
+    Fold 2:
+      Train: index=[0 1 2 3 4 5 6 7 8 9]
+      Test:  index=[10 11]
     >>> # Add in a 2 period gap
     >>> tscv = TimeSeriesSplit(n_splits=3, test_size=2, gap=2)
-    >>> for train_index, test_index in tscv.split(X):
-    ...    print("TRAIN:", train_index, "TEST:", test_index)
-    ...    X_train, X_test = X[train_index], X[test_index]
-    ...    y_train, y_test = y[train_index], y[test_index]
-    TRAIN: [0 1 2 3] TEST: [6 7]
-    TRAIN: [0 1 2 3 4 5] TEST: [8 9]
-    TRAIN: [0 1 2 3 4 5 6 7] TEST: [10 11]
+    >>> for i, (train_index, test_index) in enumerate(tscv.split(X)):
+    ...     print(f"Fold {i}:")
+    ...     print(f"  Train: index={train_index}")
+    ...     print(f"  Test:  index={test_index}")
+    Fold 0:
+      Train: index=[0 1 2 3]
+      Test:  index=[6 7]
+    Fold 1:
+      Train: index=[0 1 2 3 4 5]
+      Test:  index=[8 9]
+    Fold 2:
+      Train: index=[0 1 2 3 4 5 6 7]
+      Test:  index=[10 11]
+
+    For a more extended example see
+    :ref:`sphx_glr_auto_examples_applications_plot_cyclical_feature_engineering.py`.
 
     Notes
     -----
     The training set has size ``i * n_samples // (n_splits + 1)
     + n_samples % (n_splits + 1)`` in the ``i`` th split,
     with a test set of size ``n_samples//(n_splits + 1)`` by default,
-    where ``n_samples`` is the number of samples.
+    where ``n_samples`` is the number of samples. Note that this
+    formula is only valid when ``test_size`` and ``max_train_size`` are
+    left to their default values.
     """
-    @_deprecate_positional_args
-    def __init__(self,
-                 n_splits=5,
-                 *,
-                 max_train_size=None,
-                 test_size=None,
-                 gap=0):
+
+    def __init__(self, n_splits=5, *, max_train_size=None, test_size=None, gap=0):
         super().__init__(n_splits, shuffle=False, random_state=None)
         self.max_train_size = max_train_size
         self.test_size = test_size
@@ -839,8 +1234,8 @@ def split(self, X, y=None, groups=None):
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
-            Training data, where n_samples is the number of samples
-            and n_features is the number of features.
+            Training data, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
 
         y : array-like of shape (n_samples,)
             Always ignored, exists for compatibility.
@@ -856,50 +1251,87 @@ def split(self, X, y=None, groups=None):
         test : ndarray
             The testing set indices for that split.
         """
-        X, y, groups = indexable(X, y, groups)
+        if groups is not None:
+            warnings.warn(
+                f"The groups parameter is ignored by {self.__class__.__name__}",
+                UserWarning,
+            )
+        return self._split(X)
+
+    def _split(self, X):
+        """Generate indices to split data into training and test set.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            Training data, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
+
+        Yields
+        ------
+        train : ndarray
+            The training set indices for that split.
+
+        test : ndarray
+            The testing set indices for that split.
+        """
+        (X,) = indexable(X)
         n_samples = _num_samples(X)
         n_splits = self.n_splits
         n_folds = n_splits + 1
         gap = self.gap
-        test_size = self.test_size if self.test_size is not None \
-            else n_samples // n_folds
+        test_size = (
+            self.test_size if self.test_size is not None else n_samples // n_folds
+        )
 
         # Make sure we have enough samples for the given split parameters
         if n_folds > n_samples:
             raise ValueError(
-                (f"Cannot have number of folds={n_folds} greater"
-                 f" than the number of samples={n_samples}."))
+                f"Cannot have number of folds={n_folds} greater"
+                f" than the number of samples={n_samples}."
+            )
         if n_samples - gap - (test_size * n_splits) <= 0:
             raise ValueError(
-                (f"Too many splits={n_splits} for number of samples"
-                 f"={n_samples} with test_size={test_size} and gap={gap}."))
+                f"Too many splits={n_splits} for number of samples"
+                f"={n_samples} with test_size={test_size} and gap={gap}."
+            )
 
         indices = np.arange(n_samples)
-        test_starts = range(n_samples - n_splits * test_size,
-                            n_samples, test_size)
+        test_starts = range(n_samples - n_splits * test_size, n_samples, test_size)
 
         for test_start in test_starts:
             train_end = test_start - gap
             if self.max_train_size and self.max_train_size < train_end:
-                yield (indices[train_end - self.max_train_size:train_end],
-                       indices[test_start:test_start + test_size])
+                yield (
+                    indices[train_end - self.max_train_size : train_end],
+                    indices[test_start : test_start + test_size],
+                )
             else:
-                yield (indices[:train_end],
-                       indices[test_start:test_start + test_size])
+                yield (
+                    indices[:train_end],
+                    indices[test_start : test_start + test_size],
+                )
 
 
-class LeaveOneGroupOut(BaseCrossValidator):
-    """Leave One Group Out cross-validator
+class LeaveOneGroupOut(GroupsConsumerMixin, BaseCrossValidator):
+    """Leave One Group Out cross-validator.
 
-    Provides train/test indices to split data according to a third-party
-    provided group. This group information can be used to encode arbitrary
-    domain specific stratifications of the samples as integers.
+    Provides train/test indices to split data such that each training set is
+    comprised of all samples except ones belonging to one specific group.
+    Arbitrary domain specific group information is provided as an array of integers
+    that encodes the group of each sample.
 
     For instance the groups could be the year of collection of the samples
     and thus allow for cross-validation against time-based splits.
 
     Read more in the :ref:`User Guide `.
 
+    Notes
+    -----
+    Splits are ordered according to the index of the group left out. The first
+    split has testing set consisting of the group whose index in `groups` is
+    lowest, and so on.
+
     Examples
     --------
     >>> import numpy as np
@@ -914,37 +1346,40 @@ class LeaveOneGroupOut(BaseCrossValidator):
     2
     >>> print(logo)
     LeaveOneGroupOut()
-    >>> for train_index, test_index in logo.split(X, y, groups):
-    ...     print("TRAIN:", train_index, "TEST:", test_index)
-    ...     X_train, X_test = X[train_index], X[test_index]
-    ...     y_train, y_test = y[train_index], y[test_index]
-    ...     print(X_train, X_test, y_train, y_test)
-    TRAIN: [2 3] TEST: [0 1]
-    [[5 6]
-     [7 8]] [[1 2]
-     [3 4]] [1 2] [1 2]
-    TRAIN: [0 1] TEST: [2 3]
-    [[1 2]
-     [3 4]] [[5 6]
-     [7 8]] [1 2] [1 2]
-
+    >>> for i, (train_index, test_index) in enumerate(logo.split(X, y, groups)):
+    ...     print(f"Fold {i}:")
+    ...     print(f"  Train: index={train_index}, group={groups[train_index]}")
+    ...     print(f"  Test:  index={test_index}, group={groups[test_index]}")
+    Fold 0:
+      Train: index=[2 3], group=[2 2]
+      Test:  index=[0 1], group=[1 1]
+    Fold 1:
+      Train: index=[0 1], group=[1 1]
+      Test:  index=[2 3], group=[2 2]
+
+    See also
+    --------
+    GroupKFold: K-fold iterator variant with non-overlapping groups.
     """
 
     def _iter_test_masks(self, X, y, groups):
         if groups is None:
             raise ValueError("The 'groups' parameter should not be None.")
         # We make a copy of groups to avoid side-effects during iteration
-        groups = check_array(groups, copy=True, ensure_2d=False, dtype=None)
+        groups = check_array(
+            groups, input_name="groups", copy=True, ensure_2d=False, dtype=None
+        )
         unique_groups = np.unique(groups)
         if len(unique_groups) <= 1:
             raise ValueError(
                 "The groups parameter contains fewer than 2 unique groups "
-                "(%s). LeaveOneGroupOut expects at least 2." % unique_groups)
+                "(%s). LeaveOneGroupOut expects at least 2." % unique_groups
+            )
         for i in unique_groups:
             yield groups == i
 
     def get_n_splits(self, X=None, y=None, groups=None):
-        """Returns the number of splitting iterations in the cross-validator
+        """Returns the number of splitting iterations in the cross-validator.
 
         Parameters
         ----------
@@ -967,7 +1402,7 @@ def get_n_splits(self, X=None, y=None, groups=None):
         """
         if groups is None:
             raise ValueError("The 'groups' parameter should not be None.")
-        groups = check_array(groups, ensure_2d=False, dtype=None)
+        groups = check_array(groups, input_name="groups", ensure_2d=False, dtype=None)
         return len(np.unique(groups))
 
     def split(self, X, y=None, groups=None):
@@ -976,8 +1411,8 @@ def split(self, X, y=None, groups=None):
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
-            Training data, where n_samples is the number of samples
-            and n_features is the number of features.
+            Training data, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
 
         y : array-like of shape (n_samples,), default=None
             The target variable for supervised learning problems.
@@ -997,8 +1432,8 @@ def split(self, X, y=None, groups=None):
         return super().split(X, y, groups)
 
 
-class LeavePGroupsOut(BaseCrossValidator):
-    """Leave P Group(s) Out cross-validator
+class LeavePGroupsOut(GroupsConsumerMixin, BaseCrossValidator):
+    """Leave P Group(s) Out cross-validator.
 
     Provides train/test indices to split data according to a third-party
     provided group. This group information can be used to encode arbitrary
@@ -1033,20 +1468,19 @@ class LeavePGroupsOut(BaseCrossValidator):
     3
     >>> print(lpgo)
     LeavePGroupsOut(n_groups=2)
-    >>> for train_index, test_index in lpgo.split(X, y, groups):
-    ...     print("TRAIN:", train_index, "TEST:", test_index)
-    ...     X_train, X_test = X[train_index], X[test_index]
-    ...     y_train, y_test = y[train_index], y[test_index]
-    ...     print(X_train, X_test, y_train, y_test)
-    TRAIN: [2] TEST: [0 1]
-    [[5 6]] [[1 2]
-     [3 4]] [1] [1 2]
-    TRAIN: [1] TEST: [0 2]
-    [[3 4]] [[1 2]
-     [5 6]] [2] [1 1]
-    TRAIN: [0] TEST: [1 2]
-    [[1 2]] [[3 4]
-     [5 6]] [1] [2 1]
+    >>> for i, (train_index, test_index) in enumerate(lpgo.split(X, y, groups)):
+    ...     print(f"Fold {i}:")
+    ...     print(f"  Train: index={train_index}, group={groups[train_index]}")
+    ...     print(f"  Test:  index={test_index}, group={groups[test_index]}")
+    Fold 0:
+      Train: index=[2], group=[3]
+      Test:  index=[0 1], group=[1 2]
+    Fold 1:
+      Train: index=[1], group=[2]
+      Test:  index=[0 2], group=[1 3]
+    Fold 2:
+      Train: index=[0], group=[1]
+      Test:  index=[1 2], group=[2 3]
 
     See Also
     --------
@@ -1059,14 +1493,17 @@ def __init__(self, n_groups):
     def _iter_test_masks(self, X, y, groups):
         if groups is None:
             raise ValueError("The 'groups' parameter should not be None.")
-        groups = check_array(groups, copy=True, ensure_2d=False, dtype=None)
+        groups = check_array(
+            groups, input_name="groups", copy=True, ensure_2d=False, dtype=None
+        )
         unique_groups = np.unique(groups)
         if self.n_groups >= len(unique_groups):
             raise ValueError(
                 "The groups parameter contains fewer than (or equal to) "
                 "n_groups (%d) numbers of unique groups (%s). LeavePGroupsOut "
                 "expects that at least n_groups + 1 (%d) unique groups be "
-                "present" % (self.n_groups, unique_groups, self.n_groups + 1))
+                "present" % (self.n_groups, unique_groups, self.n_groups + 1)
+            )
         combi = combinations(range(len(unique_groups)), self.n_groups)
         for indices in combi:
             test_index = np.zeros(_num_samples(X), dtype=bool)
@@ -1075,7 +1512,7 @@ def _iter_test_masks(self, X, y, groups):
             yield test_index
 
     def get_n_splits(self, X=None, y=None, groups=None):
-        """Returns the number of splitting iterations in the cross-validator
+        """Returns the number of splitting iterations in the cross-validator.
 
         Parameters
         ----------
@@ -1098,7 +1535,7 @@ def get_n_splits(self, X=None, y=None, groups=None):
         """
         if groups is None:
             raise ValueError("The 'groups' parameter should not be None.")
-        groups = check_array(groups, ensure_2d=False, dtype=None)
+        groups = check_array(groups, input_name="groups", ensure_2d=False, dtype=None)
         return int(comb(len(np.unique(groups)), self.n_groups, exact=True))
 
     def split(self, X, y=None, groups=None):
@@ -1107,8 +1544,8 @@ def split(self, X, y=None, groups=None):
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
-            Training data, where n_samples is the number of samples
-            and n_features is the number of features.
+            Training data, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
 
         y : array-like of shape (n_samples,), default=None
             The target variable for supervised learning problems.
@@ -1128,7 +1565,7 @@ def split(self, X, y=None, groups=None):
         return super().split(X, y, groups)
 
 
-class _RepeatedSplits(metaclass=ABCMeta):
+class _RepeatedSplits(_MetadataRequester, metaclass=ABCMeta):
     """Repeated splits for an arbitrary randomized CV splitter.
 
     Repeats splits for cross-validators n times with different randomization
@@ -1151,7 +1588,13 @@ class _RepeatedSplits(metaclass=ABCMeta):
         Constructor parameters for cv. Must not contain random_state
         and shuffle.
     """
-    @_deprecate_positional_args
+
+    # This indicates that by default CV splitters don't have a "groups" kwarg,
+    # unless indicated by inheriting from ``GroupsConsumerMixin``.
+    # This also prevents ``set_split_request`` to be generated for splitters
+    # which don't support ``groups``.
+    __metadata_request__split = {"groups": metadata_routing.UNUSED}
+
     def __init__(self, cv, *, n_repeats=10, random_state=None, **cvargs):
         if not isinstance(n_repeats, numbers.Integral):
             raise ValueError("Number of repetitions must be of Integral type.")
@@ -1159,9 +1602,8 @@ def __init__(self, cv, *, n_repeats=10, random_state=None, **cvargs):
         if n_repeats <= 0:
             raise ValueError("Number of repetitions must be greater than 0.")
 
-        if any(key in cvargs for key in ('random_state', 'shuffle')):
-            raise ValueError(
-                "cvargs must not contain random_state or shuffle.")
+        if any(key in cvargs for key in ("random_state", "shuffle")):
+            raise ValueError("cvargs must not contain random_state or shuffle.")
 
         self.cv = cv
         self.n_repeats = n_repeats
@@ -1174,8 +1616,8 @@ def split(self, X, y=None, groups=None):
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
-            Training data, where n_samples is the number of samples
-            and n_features is the number of features.
+            Training data, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
 
         y : array-like of shape (n_samples,)
             The target variable for supervised learning problems.
@@ -1196,13 +1638,12 @@ def split(self, X, y=None, groups=None):
         rng = check_random_state(self.random_state)
 
         for idx in range(n_repeats):
-            cv = self.cv(random_state=rng, shuffle=True,
-                         **self.cvargs)
+            cv = self.cv(random_state=rng, shuffle=True, **self.cvargs)
             for train_index, test_index in cv.split(X, y, groups):
                 yield train_index, test_index
 
     def get_n_splits(self, X=None, y=None, groups=None):
-        """Returns the number of splitting iterations in the cross-validator
+        """Returns the number of splitting iterations in the cross-validator.
 
         Parameters
         ----------
@@ -1224,18 +1665,17 @@ def get_n_splits(self, X=None, y=None, groups=None):
             Returns the number of splitting iterations in the cross-validator.
         """
         rng = check_random_state(self.random_state)
-        cv = self.cv(random_state=rng, shuffle=True,
-                     **self.cvargs)
+        cv = self.cv(random_state=rng, shuffle=True, **self.cvargs)
         return cv.get_n_splits(X, y, groups) * self.n_repeats
 
     def __repr__(self):
         return _build_repr(self)
 
 
-class RepeatedKFold(_RepeatedSplits):
+class RepeatedKFold(_UnsupportedGroupCVMixin, _RepeatedSplits):
     """Repeated K-Fold cross validator.
 
-    Repeats K-Fold n times with different randomization in each repetition.
+    Repeats K-Fold `n_repeats` times with different randomization in each repetition.
 
     Read more in the :ref:`User Guide `.
 
@@ -1259,15 +1699,27 @@ class RepeatedKFold(_RepeatedSplits):
     >>> X = np.array([[1, 2], [3, 4], [1, 2], [3, 4]])
     >>> y = np.array([0, 0, 1, 1])
     >>> rkf = RepeatedKFold(n_splits=2, n_repeats=2, random_state=2652124)
-    >>> for train_index, test_index in rkf.split(X):
-    ...     print("TRAIN:", train_index, "TEST:", test_index)
-    ...     X_train, X_test = X[train_index], X[test_index]
-    ...     y_train, y_test = y[train_index], y[test_index]
+    >>> rkf.get_n_splits(X, y)
+    4
+    >>> print(rkf)
+    RepeatedKFold(n_repeats=2, n_splits=2, random_state=2652124)
+    >>> for i, (train_index, test_index) in enumerate(rkf.split(X)):
+    ...     print(f"Fold {i}:")
+    ...     print(f"  Train: index={train_index}")
+    ...     print(f"  Test:  index={test_index}")
     ...
-    TRAIN: [0 1] TEST: [2 3]
-    TRAIN: [2 3] TEST: [0 1]
-    TRAIN: [1 2] TEST: [0 3]
-    TRAIN: [0 3] TEST: [1 2]
+    Fold 0:
+      Train: index=[0 1]
+      Test:  index=[2 3]
+    Fold 1:
+      Train: index=[2 3]
+      Test:  index=[0 1]
+    Fold 2:
+      Train: index=[1 2]
+      Test:  index=[0 3]
+    Fold 3:
+      Train: index=[0 3]
+      Test:  index=[1 2]
 
     Notes
     -----
@@ -1279,21 +1731,26 @@ class RepeatedKFold(_RepeatedSplits):
     --------
     RepeatedStratifiedKFold : Repeats Stratified K-Fold n times.
     """
-    @_deprecate_positional_args
+
     def __init__(self, *, n_splits=5, n_repeats=10, random_state=None):
         super().__init__(
-            KFold, n_repeats=n_repeats,
-            random_state=random_state, n_splits=n_splits)
+            KFold, n_repeats=n_repeats, random_state=random_state, n_splits=n_splits
+        )
 
 
-class RepeatedStratifiedKFold(_RepeatedSplits):
-    """Repeated Stratified K-Fold cross validator.
+class RepeatedStratifiedKFold(_UnsupportedGroupCVMixin, _RepeatedSplits):
+    """Repeated class-wise stratified K-Fold cross validator.
 
     Repeats Stratified K-Fold n times with different randomization in each
     repetition.
 
     Read more in the :ref:`User Guide `.
 
+    .. note::
+
+        Stratification on the class label solves an engineering problem rather
+        than a statistical one. See :ref:`stratification` for more details.
+
     Parameters
     ----------
     n_splits : int, default=5
@@ -1315,15 +1772,27 @@ class RepeatedStratifiedKFold(_RepeatedSplits):
     >>> y = np.array([0, 0, 1, 1])
     >>> rskf = RepeatedStratifiedKFold(n_splits=2, n_repeats=2,
     ...     random_state=36851234)
-    >>> for train_index, test_index in rskf.split(X, y):
-    ...     print("TRAIN:", train_index, "TEST:", test_index)
-    ...     X_train, X_test = X[train_index], X[test_index]
-    ...     y_train, y_test = y[train_index], y[test_index]
+    >>> rskf.get_n_splits(X, y)
+    4
+    >>> print(rskf)
+    RepeatedStratifiedKFold(n_repeats=2, n_splits=2, random_state=36851234)
+    >>> for i, (train_index, test_index) in enumerate(rskf.split(X, y)):
+    ...     print(f"Fold {i}:")
+    ...     print(f"  Train: index={train_index}")
+    ...     print(f"  Test:  index={test_index}")
     ...
-    TRAIN: [1 2] TEST: [0 3]
-    TRAIN: [0 3] TEST: [1 2]
-    TRAIN: [1 3] TEST: [0 2]
-    TRAIN: [0 2] TEST: [1 3]
+    Fold 0:
+      Train: index=[1 2]
+      Test:  index=[0 3]
+    Fold 1:
+      Train: index=[0 3]
+      Test:  index=[1 2]
+    Fold 2:
+      Train: index=[1 3]
+      Test:  index=[0 2]
+    Fold 3:
+      Train: index=[0 2]
+      Test:  index=[1 3]
 
     Notes
     -----
@@ -1335,18 +1804,89 @@ class RepeatedStratifiedKFold(_RepeatedSplits):
     --------
     RepeatedKFold : Repeats K-Fold n times.
     """
-    @_deprecate_positional_args
+
     def __init__(self, *, n_splits=5, n_repeats=10, random_state=None):
         super().__init__(
-            StratifiedKFold, n_repeats=n_repeats, random_state=random_state,
-            n_splits=n_splits)
+            StratifiedKFold,
+            n_repeats=n_repeats,
+            random_state=random_state,
+            n_splits=n_splits,
+        )
+
+    def split(self, X, y, groups=None):
+        """Generate indices to split data into training and test set.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            Training data, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
+
+            Note that providing ``y`` is sufficient to generate the splits and
+            hence ``np.zeros(n_samples)`` may be used as a placeholder for
+            ``X`` instead of actual training data.
+
+        y : array-like of shape (n_samples,)
+            The target variable for supervised learning problems.
+            Stratification is done based on the y labels.
+
+        groups : object
+            Always ignored, exists for compatibility.
+
+        Yields
+        ------
+        train : ndarray
+            The training set indices for that split.
 
+        test : ndarray
+            The testing set indices for that split.
+
+        Notes
+        -----
+        Randomized CV splitters may return different results for each call of
+        split. You can make the results identical by setting `random_state`
+        to an integer.
+        """
+        y = check_array(y, input_name="y", ensure_2d=False, dtype=None)
+        return super().split(X, y, groups=groups)
+
+
+class BaseShuffleSplit(_MetadataRequester, metaclass=ABCMeta):
+    """Base class for *ShuffleSplit.
+
+    Parameters
+    ----------
+    n_splits : int, default=10
+        Number of re-shuffling & splitting iterations.
+
+    test_size : float or int, default=None
+        If float, should be between 0.0 and 1.0 and represent the proportion
+        of the dataset to include in the test split. If int, represents the
+        absolute number of test samples. If None, the value is set to the
+        complement of the train size. If ``train_size`` is also None, it will
+        be set to 0.1.
 
-class BaseShuffleSplit(metaclass=ABCMeta):
-    """Base class for ShuffleSplit and StratifiedShuffleSplit"""
-    @_deprecate_positional_args
-    def __init__(self, n_splits=10, *, test_size=None, train_size=None,
-                 random_state=None):
+    train_size : float or int, default=None
+        If float, should be between 0.0 and 1.0 and represent the
+        proportion of the dataset to include in the train split. If
+        int, represents the absolute number of train samples. If None,
+        the value is automatically set to the complement of the test size.
+
+    random_state : int, RandomState instance or None, default=None
+        Controls the randomness of the training and testing indices produced.
+        Pass an int for reproducible output across multiple function calls.
+        See :term:`Glossary `.
+    """
+
+    # This indicates that by default CV splitters don't have a "groups" kwarg,
+    # unless indicated by inheriting from ``GroupsConsumerMixin``.
+    # This also prevents ``set_split_request`` to be generated for splitters
+    # which don't support ``groups``.
+    __metadata_request__split = {"groups": metadata_routing.UNUSED}
+
+    def __init__(
+        self, n_splits=10, *, test_size=None, train_size=None, random_state=None
+    ):
         self.n_splits = n_splits
         self.test_size = test_size
         self.train_size = train_size
@@ -1359,8 +1899,8 @@ def split(self, X, y=None, groups=None):
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
-            Training data, where n_samples is the number of samples
-            and n_features is the number of features.
+            Training data, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
 
         y : array-like of shape (n_samples,)
             The target variable for supervised learning problems.
@@ -1387,12 +1927,26 @@ def split(self, X, y=None, groups=None):
         for train, test in self._iter_indices(X, y, groups):
             yield train, test
 
-    @abstractmethod
     def _iter_indices(self, X, y=None, groups=None):
         """Generate (train, test) indices"""
+        n_samples = _num_samples(X)
+        n_train, n_test = _validate_shuffle_split(
+            n_samples,
+            self.test_size,
+            self.train_size,
+            default_test_size=self._default_test_size,
+        )
+
+        rng = check_random_state(self.random_state)
+        for i in range(self.n_splits):
+            # random partition
+            permutation = rng.permutation(n_samples)
+            ind_test = permutation[:n_test]
+            ind_train = permutation[n_test : (n_test + n_train)]
+            yield ind_train, ind_test
 
     def get_n_splits(self, X=None, y=None, groups=None):
-        """Returns the number of splitting iterations in the cross-validator
+        """Returns the number of splitting iterations in the cross-validator.
 
         Parameters
         ----------
@@ -1416,17 +1970,22 @@ def __repr__(self):
         return _build_repr(self)
 
 
-class ShuffleSplit(BaseShuffleSplit):
-    """Random permutation cross-validator
+class ShuffleSplit(_UnsupportedGroupCVMixin, BaseShuffleSplit):
+    """Random permutation cross-validator.
 
     Yields indices to split data into training and test sets.
 
     Note: contrary to other cross-validation strategies, random splits
-    do not guarantee that all folds will be different, although this is
-    still very likely for sizeable datasets.
+    do not guarantee that test sets across all folds will be mutually exclusive,
+    and might include overlapping samples. However, this is still very likely for
+    sizeable datasets.
 
     Read more in the :ref:`User Guide `.
 
+    For visualisation of cross-validation behaviour and
+    comparison between common scikit-learn split methods
+    refer to :ref:`sphx_glr_auto_examples_model_selection_plot_cv_indices.py`
+
     Parameters
     ----------
     n_splits : int, default=10
@@ -1461,50 +2020,63 @@ class ShuffleSplit(BaseShuffleSplit):
     5
     >>> print(rs)
     ShuffleSplit(n_splits=5, random_state=0, test_size=0.25, train_size=None)
-    >>> for train_index, test_index in rs.split(X):
-    ...     print("TRAIN:", train_index, "TEST:", test_index)
-    TRAIN: [1 3 0 4] TEST: [5 2]
-    TRAIN: [4 0 2 5] TEST: [1 3]
-    TRAIN: [1 2 4 0] TEST: [3 5]
-    TRAIN: [3 4 1 0] TEST: [5 2]
-    TRAIN: [3 5 1 0] TEST: [2 4]
+    >>> for i, (train_index, test_index) in enumerate(rs.split(X)):
+    ...     print(f"Fold {i}:")
+    ...     print(f"  Train: index={train_index}")
+    ...     print(f"  Test:  index={test_index}")
+    Fold 0:
+      Train: index=[1 3 0 4]
+      Test:  index=[5 2]
+    Fold 1:
+      Train: index=[4 0 2 5]
+      Test:  index=[1 3]
+    Fold 2:
+      Train: index=[1 2 4 0]
+      Test:  index=[3 5]
+    Fold 3:
+      Train: index=[3 4 1 0]
+      Test:  index=[5 2]
+    Fold 4:
+      Train: index=[3 5 1 0]
+      Test:  index=[2 4]
+    >>> # Specify train and test size
     >>> rs = ShuffleSplit(n_splits=5, train_size=0.5, test_size=.25,
     ...                   random_state=0)
-    >>> for train_index, test_index in rs.split(X):
-    ...     print("TRAIN:", train_index, "TEST:", test_index)
-    TRAIN: [1 3 0] TEST: [5 2]
-    TRAIN: [4 0 2] TEST: [1 3]
-    TRAIN: [1 2 4] TEST: [3 5]
-    TRAIN: [3 4 1] TEST: [5 2]
-    TRAIN: [3 5 1] TEST: [2 4]
+    >>> for i, (train_index, test_index) in enumerate(rs.split(X)):
+    ...     print(f"Fold {i}:")
+    ...     print(f"  Train: index={train_index}")
+    ...     print(f"  Test:  index={test_index}")
+    Fold 0:
+      Train: index=[1 3 0]
+      Test:  index=[5 2]
+    Fold 1:
+      Train: index=[4 0 2]
+      Test:  index=[1 3]
+    Fold 2:
+      Train: index=[1 2 4]
+      Test:  index=[3 5]
+    Fold 3:
+      Train: index=[3 4 1]
+      Test:  index=[5 2]
+    Fold 4:
+      Train: index=[3 5 1]
+      Test:  index=[2 4]
     """
-    @_deprecate_positional_args
-    def __init__(self, n_splits=10, *, test_size=None, train_size=None,
-                 random_state=None):
+
+    def __init__(
+        self, n_splits=10, *, test_size=None, train_size=None, random_state=None
+    ):
         super().__init__(
             n_splits=n_splits,
             test_size=test_size,
             train_size=train_size,
-            random_state=random_state)
+            random_state=random_state,
+        )
         self._default_test_size = 0.1
 
-    def _iter_indices(self, X, y=None, groups=None):
-        n_samples = _num_samples(X)
-        n_train, n_test = _validate_shuffle_split(
-            n_samples, self.test_size, self.train_size,
-            default_test_size=self._default_test_size)
-
-        rng = check_random_state(self.random_state)
-        for i in range(self.n_splits):
-            # random partition
-            permutation = rng.permutation(n_samples)
-            ind_test = permutation[:n_test]
-            ind_train = permutation[n_test:(n_test + n_train)]
-            yield ind_train, ind_test
 
-
-class GroupShuffleSplit(ShuffleSplit):
-    '''Shuffle-Group(s)-Out cross-validation iterator
+class GroupShuffleSplit(GroupsConsumerMixin, BaseShuffleSplit):
+    """Shuffle-Group(s)-Out cross-validation iterator.
 
     Provides randomized train/test indices to split data according to a
     third-party provided group. This group information can be used to encode
@@ -1513,33 +2085,40 @@ class GroupShuffleSplit(ShuffleSplit):
     For instance the groups could be the year of collection of the samples
     and thus allow for cross-validation against time-based splits.
 
-    The difference between LeavePGroupsOut and GroupShuffleSplit is that
+    The difference between :class:`LeavePGroupsOut` and ``GroupShuffleSplit`` is that
     the former generates splits using all subsets of size ``p`` unique groups,
-    whereas GroupShuffleSplit generates a user-determined number of random
+    whereas ``GroupShuffleSplit`` generates a user-determined number of random
     test splits, each with a user-determined fraction of unique groups.
 
     For example, a less computationally intensive alternative to
     ``LeavePGroupsOut(p=10)`` would be
     ``GroupShuffleSplit(test_size=10, n_splits=100)``.
 
+    Contrary to other cross-validation strategies, the random splits
+    do not guarantee that test sets across all folds will be mutually exclusive,
+    and might include overlapping samples. However, this is still very likely for
+    sizeable datasets.
+
     Note: The parameters ``test_size`` and ``train_size`` refer to groups, and
-    not to samples, as in ShuffleSplit.
+    not to samples as in :class:`ShuffleSplit`.
 
     Read more in the :ref:`User Guide `.
 
+    For visualisation of cross-validation behaviour and
+    comparison between common scikit-learn split methods
+    refer to :ref:`sphx_glr_auto_examples_model_selection_plot_cv_indices.py`
+
     Parameters
     ----------
     n_splits : int, default=5
         Number of re-shuffling & splitting iterations.
 
-    test_size : float, int, default=0.2
+    test_size : float, int, default=None
         If float, should be between 0.0 and 1.0 and represent the proportion
         of groups to include in the test split (rounded up). If int,
         represents the absolute number of test groups. If None, the value is
-        set to the complement of the train size.
-        The default will change in version 0.21. It will remain 0.2 only
-        if ``train_size`` is unspecified, otherwise it will complement
-        the specified ``train_size``.
+        set to the complement of the train size. If ``train_size`` is also None,
+        it will be set to 0.2.
 
     train_size : float or int, default=None
         If float, should be between 0.0 and 1.0 and represent the
@@ -1564,32 +2143,48 @@ class GroupShuffleSplit(ShuffleSplit):
     >>> gss = GroupShuffleSplit(n_splits=2, train_size=.7, random_state=42)
     >>> gss.get_n_splits()
     2
-    >>> for train_idx, test_idx in gss.split(X, y, groups):
-    ...     print("TRAIN:", train_idx, "TEST:", test_idx)
-    TRAIN: [2 3 4 5 6 7] TEST: [0 1]
-    TRAIN: [0 1 5 6 7] TEST: [2 3 4]
-    '''
-    @_deprecate_positional_args
-    def __init__(self, n_splits=5, *, test_size=None, train_size=None,
-                 random_state=None):
+    >>> print(gss)
+    GroupShuffleSplit(n_splits=2, random_state=42, test_size=None, train_size=0.7)
+    >>> for i, (train_index, test_index) in enumerate(gss.split(X, y, groups)):
+    ...     print(f"Fold {i}:")
+    ...     print(f"  Train: index={train_index}, group={groups[train_index]}")
+    ...     print(f"  Test:  index={test_index}, group={groups[test_index]}")
+    Fold 0:
+      Train: index=[2 3 4 5 6 7], group=[2 2 2 3 3 3]
+      Test:  index=[0 1], group=[1 1]
+    Fold 1:
+      Train: index=[0 1 5 6 7], group=[1 1 3 3 3]
+      Test:  index=[2 3 4], group=[2 2 2]
+
+    See Also
+    --------
+    ShuffleSplit : Shuffles samples to create independent test/train sets.
+
+    LeavePGroupsOut : Train set leaves out all possible subsets of `p` groups.
+    """
+
+    def __init__(
+        self, n_splits=5, *, test_size=None, train_size=None, random_state=None
+    ):
         super().__init__(
             n_splits=n_splits,
             test_size=test_size,
             train_size=train_size,
-            random_state=random_state)
+            random_state=random_state,
+        )
         self._default_test_size = 0.2
 
     def _iter_indices(self, X, y, groups):
         if groups is None:
             raise ValueError("The 'groups' parameter should not be None.")
-        groups = check_array(groups, ensure_2d=False, dtype=None)
+        groups = check_array(groups, input_name="groups", ensure_2d=False, dtype=None)
         classes, group_indices = np.unique(groups, return_inverse=True)
         for group_train, group_test in super()._iter_indices(X=classes):
             # these are the indices of classes in the partition
             # invert them into data indices
 
-            train = np.flatnonzero(np.in1d(group_indices, group_train))
-            test = np.flatnonzero(np.in1d(group_indices, group_test))
+            train = np.flatnonzero(np.isin(group_indices, group_train))
+            test = np.flatnonzero(np.isin(group_indices, group_test))
 
             yield train, test
 
@@ -1599,8 +2194,8 @@ def split(self, X, y=None, groups=None):
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
-            Training data, where n_samples is the number of samples
-            and n_features is the number of features.
+            Training data, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
 
         y : array-like of shape (n_samples,), default=None
             The target variable for supervised learning problems.
@@ -1627,20 +2222,31 @@ def split(self, X, y=None, groups=None):
 
 
 class StratifiedShuffleSplit(BaseShuffleSplit):
-    """Stratified ShuffleSplit cross-validator
+    """Class-wise stratified ShuffleSplit cross-validator.
 
     Provides train/test indices to split data in train/test sets.
 
-    This cross-validation object is a merge of StratifiedKFold and
-    ShuffleSplit, which returns stratified randomized folds. The folds
-    are made by preserving the percentage of samples for each class.
+    This cross-validation object is a merge of :class:`StratifiedKFold` and
+    :class:`ShuffleSplit`, which returns stratified randomized folds. The folds
+    are made by preserving the percentage of samples for each class in `y` in a
+    binary or multiclass classification setting.
 
-    Note: like the ShuffleSplit strategy, stratified random splits
-    do not guarantee that all folds will be different, although this is
-    still very likely for sizeable datasets.
+    Note: like the :class:`ShuffleSplit` strategy, stratified random splits
+    do not guarantee that test sets across all folds will be mutually exclusive,
+    and might include overlapping samples. However, this is still very likely for
+    sizeable datasets.
 
     Read more in the :ref:`User Guide `.
 
+    For visualisation of cross-validation behaviour and
+    comparison between common scikit-learn split methods
+    refer to :ref:`sphx_glr_auto_examples_model_selection_plot_cv_indices.py`
+
+    .. note::
+
+        Stratification on the class label solves an engineering problem rather
+        than a statistical one. See :ref:`stratification` for more details.
+
     Parameters
     ----------
     n_splits : int, default=10
@@ -1675,61 +2281,87 @@ class StratifiedShuffleSplit(BaseShuffleSplit):
     5
     >>> print(sss)
     StratifiedShuffleSplit(n_splits=5, random_state=0, ...)
-    >>> for train_index, test_index in sss.split(X, y):
-    ...     print("TRAIN:", train_index, "TEST:", test_index)
-    ...     X_train, X_test = X[train_index], X[test_index]
-    ...     y_train, y_test = y[train_index], y[test_index]
-    TRAIN: [5 2 3] TEST: [4 1 0]
-    TRAIN: [5 1 4] TEST: [0 2 3]
-    TRAIN: [5 0 2] TEST: [4 3 1]
-    TRAIN: [4 1 0] TEST: [2 3 5]
-    TRAIN: [0 5 1] TEST: [3 4 2]
+    >>> for i, (train_index, test_index) in enumerate(sss.split(X, y)):
+    ...     print(f"Fold {i}:")
+    ...     print(f"  Train: index={train_index}")
+    ...     print(f"  Test:  index={test_index}")
+    Fold 0:
+      Train: index=[5 2 3]
+      Test:  index=[4 1 0]
+    Fold 1:
+      Train: index=[5 1 4]
+      Test:  index=[0 2 3]
+    Fold 2:
+      Train: index=[5 0 2]
+      Test:  index=[4 3 1]
+    Fold 3:
+      Train: index=[4 1 0]
+      Test:  index=[2 3 5]
+    Fold 4:
+      Train: index=[0 5 1]
+      Test:  index=[3 4 2]
     """
-    @_deprecate_positional_args
-    def __init__(self, n_splits=10, *, test_size=None, train_size=None,
-                 random_state=None):
+
+    def __init__(
+        self, n_splits=10, *, test_size=None, train_size=None, random_state=None
+    ):
         super().__init__(
             n_splits=n_splits,
             test_size=test_size,
             train_size=train_size,
-            random_state=random_state)
+            random_state=random_state,
+        )
         self._default_test_size = 0.1
 
     def _iter_indices(self, X, y, groups=None):
         n_samples = _num_samples(X)
-        y = check_array(y, ensure_2d=False, dtype=None)
+        y = check_array(y, input_name="y", ensure_2d=False, dtype=None)
         n_train, n_test = _validate_shuffle_split(
-            n_samples, self.test_size, self.train_size,
-            default_test_size=self._default_test_size)
+            n_samples,
+            self.test_size,
+            self.train_size,
+            default_test_size=self._default_test_size,
+        )
+
+        # Convert to numpy as not all operations are supported by the Array API.
+        # `y` is probably never a very large array, which means that converting it
+        # should be cheap
+        xp, _ = get_namespace(y)
+        y = _convert_to_numpy(y, xp=xp)
 
         if y.ndim == 2:
             # for multi-label y, map each distinct row to a string repr
             # using join because str(row) uses an ellipsis if len(row) > 1000
-            y = np.array([' '.join(row.astype('str')) for row in y])
+            y = np.array([" ".join(row.astype("str")) for row in y])
 
         classes, y_indices = np.unique(y, return_inverse=True)
         n_classes = classes.shape[0]
 
         class_counts = np.bincount(y_indices)
         if np.min(class_counts) < 2:
-            raise ValueError("The least populated class in y has only 1"
-                             " member, which is too few. The minimum"
-                             " number of groups for any class cannot"
-                             " be less than 2.")
+            raise ValueError(
+                "The least populated class in y has only 1"
+                " member, which is too few. The minimum"
+                " number of groups for any class cannot"
+                " be less than 2."
+            )
 
         if n_train < n_classes:
-            raise ValueError('The train_size = %d should be greater or '
-                             'equal to the number of classes = %d' %
-                             (n_train, n_classes))
+            raise ValueError(
+                "The train_size = %d should be greater or "
+                "equal to the number of classes = %d" % (n_train, n_classes)
+            )
         if n_test < n_classes:
-            raise ValueError('The test_size = %d should be greater or '
-                             'equal to the number of classes = %d' %
-                             (n_test, n_classes))
+            raise ValueError(
+                "The test_size = %d should be greater or "
+                "equal to the number of classes = %d" % (n_test, n_classes)
+            )
 
         # Find the sorted list of instances for each class:
         # (np.unique above performs a sort, so code is O(n logn) already)
-        class_indices = np.split(np.argsort(y_indices, kind='mergesort'),
-                                 np.cumsum(class_counts)[:-1])
+        class_indices = np.split(
+            np.argsort(y_indices, kind="mergesort"), np.cumsum(class_counts)[:-1]
+        )
 
         rng = check_random_state(self.random_state)
 
@@ -1745,11 +2377,10 @@ def _iter_indices(self, X, y, groups=None):
 
             for i in range(n_classes):
                 permutation = rng.permutation(class_counts[i])
-                perm_indices_class_i = class_indices[i].take(permutation,
-                                                             mode='clip')
+                perm_indices_class_i = class_indices[i].take(permutation, mode="clip")
 
-                train.extend(perm_indices_class_i[:n_i[i]])
-                test.extend(perm_indices_class_i[n_i[i]:n_i[i] + t_i[i]])
+                train.extend(perm_indices_class_i[: n_i[i]])
+                test.extend(perm_indices_class_i[n_i[i] : n_i[i] + t_i[i]])
 
             train = rng.permutation(train)
             test = rng.permutation(test)
@@ -1762,8 +2393,8 @@ def split(self, X, y, groups=None):
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
-            Training data, where n_samples is the number of samples
-            and n_features is the number of features.
+            Training data, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
 
             Note that providing ``y`` is sufficient to generate the splits and
             hence ``np.zeros(n_samples)`` may be used as a placeholder for
@@ -1790,15 +2421,19 @@ def split(self, X, y, groups=None):
         split. You can make the results identical by setting `random_state`
         to an integer.
         """
-        y = check_array(y, ensure_2d=False, dtype=None)
+        if groups is not None:
+            warnings.warn(
+                f"The groups parameter is ignored by {self.__class__.__name__}",
+                UserWarning,
+            )
+        y = check_array(y, input_name="y", ensure_2d=False, dtype=None)
         return super().split(X, y, groups)
 
 
-def _validate_shuffle_split(n_samples, test_size, train_size,
-                            default_test_size=None):
+def _validate_shuffle_split(n_samples, test_size, train_size, default_test_size=None):
     """
-    Validation helper to check if the test/test sizes are meaningful wrt to the
-    size of the data (n_samples)
+    Validation helper to check if the train/test sizes are meaningful w.r.t. the
+    size of the data (n_samples).
     """
     if test_size is None and train_size is None:
         test_size = default_test_size
@@ -1806,38 +2441,43 @@ def _validate_shuffle_split(n_samples, test_size, train_size,
     test_size_type = np.asarray(test_size).dtype.kind
     train_size_type = np.asarray(train_size).dtype.kind
 
-    if (test_size_type == 'i' and (test_size >= n_samples or test_size <= 0)
-       or test_size_type == 'f' and (test_size <= 0 or test_size >= 1)):
-        raise ValueError('test_size={0} should be either positive and smaller'
-                         ' than the number of samples {1} or a float in the '
-                         '(0, 1) range'.format(test_size, n_samples))
+    if (test_size_type == "i" and (test_size >= n_samples or test_size <= 0)) or (
+        test_size_type == "f" and (test_size <= 0 or test_size >= 1)
+    ):
+        raise ValueError(
+            "test_size={0} should be either positive and smaller"
+            " than the number of samples {1} or a float in the "
+            "(0, 1) range".format(test_size, n_samples)
+        )
 
-    if (train_size_type == 'i' and (train_size >= n_samples or train_size <= 0)
-       or train_size_type == 'f' and (train_size <= 0 or train_size >= 1)):
-        raise ValueError('train_size={0} should be either positive and smaller'
-                         ' than the number of samples {1} or a float in the '
-                         '(0, 1) range'.format(train_size, n_samples))
+    if (train_size_type == "i" and (train_size >= n_samples or train_size <= 0)) or (
+        train_size_type == "f" and (train_size <= 0 or train_size >= 1)
+    ):
+        raise ValueError(
+            "train_size={0} should be either positive and smaller"
+            " than the number of samples {1} or a float in the "
+            "(0, 1) range".format(train_size, n_samples)
+        )
 
-    if train_size is not None and train_size_type not in ('i', 'f'):
+    if train_size is not None and train_size_type not in ("i", "f"):
         raise ValueError("Invalid value for train_size: {}".format(train_size))
-    if test_size is not None and test_size_type not in ('i', 'f'):
+    if test_size is not None and test_size_type not in ("i", "f"):
         raise ValueError("Invalid value for test_size: {}".format(test_size))
 
-    if (train_size_type == 'f' and test_size_type == 'f' and
-            train_size + test_size > 1):
+    if train_size_type == "f" and test_size_type == "f" and train_size + test_size > 1:
         raise ValueError(
-            'The sum of test_size and train_size = {}, should be in the (0, 1)'
-            ' range. Reduce test_size and/or train_size.'
-            .format(train_size + test_size))
+            "The sum of test_size and train_size = {}, should be in the (0, 1)"
+            " range. Reduce test_size and/or train_size.".format(train_size + test_size)
+        )
 
-    if test_size_type == 'f':
+    if test_size_type == "f":
         n_test = ceil(test_size * n_samples)
-    elif test_size_type == 'i':
+    elif test_size_type == "i":
         n_test = float(test_size)
 
-    if train_size_type == 'f':
+    if train_size_type == "f":
         n_train = floor(train_size * n_samples)
-    elif train_size_type == 'i':
+    elif train_size_type == "i":
         n_train = float(train_size)
 
     if train_size is None:
@@ -1846,26 +2486,27 @@ def _validate_shuffle_split(n_samples, test_size, train_size,
         n_test = n_samples - n_train
 
     if n_train + n_test > n_samples:
-        raise ValueError('The sum of train_size and test_size = %d, '
-                         'should be smaller than the number of '
-                         'samples %d. Reduce test_size and/or '
-                         'train_size.' % (n_train + n_test, n_samples))
+        raise ValueError(
+            "The sum of train_size and test_size = %d, "
+            "should be smaller than the number of "
+            "samples %d. Reduce test_size and/or "
+            "train_size." % (n_train + n_test, n_samples)
+        )
 
     n_train, n_test = int(n_train), int(n_test)
 
     if n_train == 0:
         raise ValueError(
-            'With n_samples={}, test_size={} and train_size={}, the '
-            'resulting train set will be empty. Adjust any of the '
-            'aforementioned parameters.'.format(n_samples, test_size,
-                                                train_size)
+            "With n_samples={}, test_size={} and train_size={}, the "
+            "resulting train set will be empty. Adjust any of the "
+            "aforementioned parameters.".format(n_samples, test_size, train_size)
         )
 
     return n_train, n_test
 
 
 class PredefinedSplit(BaseCrossValidator):
-    """Predefined split cross-validator
+    """Predefined split cross-validator.
 
     Provides train/test indices to split data into train/test sets using a
     predefined scheme specified by the user with the ``test_fold`` parameter.
@@ -1894,12 +2535,16 @@ class PredefinedSplit(BaseCrossValidator):
     2
     >>> print(ps)
     PredefinedSplit(test_fold=array([ 0,  1, -1,  1]))
-    >>> for train_index, test_index in ps.split():
-    ...     print("TRAIN:", train_index, "TEST:", test_index)
-    ...     X_train, X_test = X[train_index], X[test_index]
-    ...     y_train, y_test = y[train_index], y[test_index]
-    TRAIN: [1 2 3] TEST: [0]
-    TRAIN: [0 2] TEST: [1 3]
+    >>> for i, (train_index, test_index) in enumerate(ps.split()):
+    ...     print(f"Fold {i}:")
+    ...     print(f"  Train: index={train_index}")
+    ...     print(f"  Test:  index={test_index}")
+    Fold 0:
+      Train: index=[1 2 3]
+      Test:  index=[0]
+    Fold 1:
+      Train: index=[0 2]
+      Test:  index=[1 3]
     """
 
     def __init__(self, test_fold):
@@ -1922,6 +2567,24 @@ def split(self, X=None, y=None, groups=None):
         groups : object
             Always ignored, exists for compatibility.
 
+        Yields
+        ------
+        train : ndarray
+            The training set indices for that split.
+
+        test : ndarray
+            The testing set indices for that split.
+        """
+        if groups is not None:
+            warnings.warn(
+                f"The groups parameter is ignored by {self.__class__.__name__}",
+                UserWarning,
+            )
+        return self._split()
+
+    def _split(self):
+        """Generate indices to split data into training and test set.
+
         Yields
         ------
         train : ndarray
@@ -1945,7 +2608,7 @@ def _iter_test_masks(self):
             yield test_mask
 
     def get_n_splits(self, X=None, y=None, groups=None):
-        """Returns the number of splitting iterations in the cross-validator
+        """Returns the number of splitting iterations in the cross-validator.
 
         Parameters
         ----------
@@ -1968,11 +2631,12 @@ def get_n_splits(self, X=None, y=None, groups=None):
 
 class _CVIterableWrapper(BaseCrossValidator):
     """Wrapper class for old style cv objects and iterables."""
+
     def __init__(self, cv):
         self.cv = list(cv)
 
     def get_n_splits(self, X=None, y=None, groups=None):
-        """Returns the number of splitting iterations in the cross-validator
+        """Returns the number of splitting iterations in the cross-validator.
 
         Parameters
         ----------
@@ -2018,19 +2682,18 @@ def split(self, X=None, y=None, groups=None):
             yield train, test
 
 
-@_deprecate_positional_args
 def check_cv(cv=5, y=None, *, classifier=False):
-    """Input checker utility for building a cross-validator
+    """Input checker utility for building a cross-validator.
 
     Parameters
     ----------
-    cv : int, cross-validation generator or an iterable, default=None
+    cv : int, cross-validation generator, iterable or None, default=5
         Determines the cross-validation splitting strategy.
         Possible inputs for cv are:
         - None, to use the default 5-fold cross validation,
         - integer, to specify the number of folds.
         - :term:`CV splitter`,
-        - An iterable yielding (train, test) splits as arrays of indices.
+        - An iterable that generates (train, test) splits as arrays of indices.
 
         For integer/None inputs, if classifier is True and ``y`` is either
         binary or multiclass, :class:`StratifiedKFold` is used. In all other
@@ -2054,37 +2717,70 @@ def check_cv(cv=5, y=None, *, classifier=False):
     checked_cv : a cross-validator instance.
         The return value is a cross-validator which generates the train/test
         splits via the ``split`` method.
+
+    Examples
+    --------
+    >>> from sklearn.model_selection import check_cv
+    >>> check_cv(cv=5, y=None, classifier=False)
+    KFold(...)
+    >>> check_cv(cv=5, y=[1, 1, 0, 0, 0, 0], classifier=True)
+    StratifiedKFold(...)
     """
     cv = 5 if cv is None else cv
     if isinstance(cv, numbers.Integral):
-        if (classifier and (y is not None) and
-                (type_of_target(y) in ('binary', 'multiclass'))):
+        if (
+            classifier
+            and (y is not None)
+            and (type_of_target(y, input_name="y") in ("binary", "multiclass"))
+        ):
             return StratifiedKFold(cv)
         else:
             return KFold(cv)
 
-    if not hasattr(cv, 'split') or isinstance(cv, str):
+    if not hasattr(cv, "split") or isinstance(cv, str):
         if not isinstance(cv, Iterable) or isinstance(cv, str):
-            raise ValueError("Expected cv as an integer, cross-validation "
-                             "object (from sklearn.model_selection) "
-                             "or an iterable. Got %s." % cv)
+            raise ValueError(
+                "Expected cv as an integer, cross-validation "
+                "object (from sklearn.model_selection) "
+                "or an iterable. Got %s." % cv
+            )
         return _CVIterableWrapper(cv)
 
     return cv  # New style cv objects are passed without any modification
 
 
-def train_test_split(*arrays,
-                     test_size=None,
-                     train_size=None,
-                     random_state=None,
-                     shuffle=True,
-                     stratify=None):
-    """Split arrays or matrices into random train and test subsets
-
-    Quick utility that wraps input validation and
-    ``next(ShuffleSplit().split(X, y))`` and application to input data
-    into a single call for splitting (and optionally subsampling) data in a
-    oneliner.
+@validate_params(
+    {
+        "test_size": [
+            Interval(RealNotInt, 0, 1, closed="neither"),
+            Interval(numbers.Integral, 1, None, closed="left"),
+            None,
+        ],
+        "train_size": [
+            Interval(RealNotInt, 0, 1, closed="neither"),
+            Interval(numbers.Integral, 1, None, closed="left"),
+            None,
+        ],
+        "random_state": ["random_state"],
+        "shuffle": ["boolean"],
+        "stratify": ["array-like", None],
+    },
+    prefer_skip_nested_validation=True,
+)
+def train_test_split(
+    *arrays,
+    test_size=None,
+    train_size=None,
+    random_state=None,
+    shuffle=True,
+    stratify=None,
+):
+    """Split arrays or matrices into random train and test subsets.
+
+    Quick utility that wraps input validation,
+    ``next(ShuffleSplit().split(X, y))``, and application to input data
+    into a single call for splitting (and optionally subsampling) data into a
+    one-liner.
 
     Read more in the :ref:`User Guide `.
 
@@ -2112,7 +2808,6 @@ def train_test_split(*arrays,
         Pass an int for reproducible output across multiple function calls.
         See :term:`Glossary `.
 
-
     shuffle : bool, default=True
         Whether or not to shuffle the data before splitting. If shuffle=False
         then stratify must be None.
@@ -2164,6 +2859,55 @@ def train_test_split(*arrays,
     >>> train_test_split(y, shuffle=False)
     [[0, 1, 2], [3, 4]]
 
+    >>> from sklearn import datasets
+    >>> iris = datasets.load_iris(as_frame=True)
+    >>> X, y = iris['data'], iris['target']
+    >>> X.head()
+        sepal length (cm)  sepal width (cm)  petal length (cm)  petal width (cm)
+    0                5.1               3.5                1.4               0.2
+    1                4.9               3.0                1.4               0.2
+    2                4.7               3.2                1.3               0.2
+    3                4.6               3.1                1.5               0.2
+    4                5.0               3.6                1.4               0.2
+    >>> y.head()
+    0    0
+    1    0
+    2    0
+    3    0
+    4    0
+    ...
+
+    >>> X_train, X_test, y_train, y_test = train_test_split(
+    ... X, y, test_size=0.33, random_state=42)
+    ...
+    >>> X_train.head()
+        sepal length (cm)  sepal width (cm)  petal length (cm)  petal width (cm)
+    96                 5.7               2.9                4.2               1.3
+    105                7.6               3.0                6.6               2.1
+    66                 5.6               3.0                4.5               1.5
+    0                  5.1               3.5                1.4               0.2
+    122                7.7               2.8                6.7               2.0
+    >>> y_train.head()
+    96     1
+    105    2
+    66     1
+    0      0
+    122    2
+    ...
+    >>> X_test.head()
+        sepal length (cm)  sepal width (cm)  petal length (cm)  petal width (cm)
+    73                 6.1               2.8                4.7               1.2
+    18                 5.7               3.8                1.7               0.3
+    118                7.7               2.6                6.9               2.3
+    78                 6.0               2.9                4.5               1.5
+    76                 6.8               2.8                4.8               1.4
+    >>> y_test.head()
+    73     1
+    18     0
+    118    2
+    78     1
+    76     1
+    ...
     """
     n_arrays = len(arrays)
     if n_arrays == 0:
@@ -2172,14 +2916,15 @@ def train_test_split(*arrays,
     arrays = indexable(*arrays)
 
     n_samples = _num_samples(arrays[0])
-    n_train, n_test = _validate_shuffle_split(n_samples, test_size, train_size,
-                                              default_test_size=0.25)
+    n_train, n_test = _validate_shuffle_split(
+        n_samples, test_size, train_size, default_test_size=0.25
+    )
 
     if shuffle is False:
         if stratify is not None:
             raise ValueError(
-                "Stratified train/test split is not implemented for "
-                "shuffle=False")
+                "Stratified train/test split is not implemented for shuffle=False"
+            )
 
         train = np.arange(n_train)
         test = np.arange(n_train, n_train + n_test)
@@ -2190,34 +2935,92 @@ def train_test_split(*arrays,
         else:
             CVClass = ShuffleSplit
 
-        cv = CVClass(test_size=n_test,
-                     train_size=n_train,
-                     random_state=random_state)
+        cv = CVClass(test_size=n_test, train_size=n_train, random_state=random_state)
 
         train, test = next(cv.split(X=arrays[0], y=stratify))
 
-    return list(chain.from_iterable((_safe_indexing(a, train),
-                                     _safe_indexing(a, test)) for a in arrays))
+    train, test = ensure_common_namespace_device(arrays[0], train, test)
+
+    return list(
+        chain.from_iterable(
+            (_safe_indexing(a, train), _safe_indexing(a, test)) for a in arrays
+        )
+    )
 
 
 # Tell nose that train_test_split is not a test.
 # (Needed for external libraries that may use nose.)
 # Use setattr to avoid mypy errors when monkeypatching.
-setattr(train_test_split, '__test__', False)
+setattr(train_test_split, "__test__", False)
+
+
+def _pprint(params, offset=0, printer=repr):
+    """Pretty print the dictionary 'params'
+
+    Parameters
+    ----------
+    params : dict
+        The dictionary to pretty print
+
+    offset : int, default=0
+        The offset in characters to add at the begin of each line.
+
+    printer : callable, default=repr
+        The function to convert entries to strings, typically
+        the builtin str or repr
+
+    """
+    # Do a multi-line justified repr:
+    options = np.get_printoptions()
+    np.set_printoptions(precision=5, threshold=64, edgeitems=2)
+    params_list = list()
+    this_line_length = offset
+    line_sep = ",\n" + (1 + offset // 2) * " "
+    for i, (k, v) in enumerate(sorted(params.items())):
+        if isinstance(v, float):
+            # use str for representing floating point numbers
+            # this way we get consistent representation across
+            # architectures and versions.
+            this_repr = "%s=%s" % (k, str(v))
+        else:
+            # use repr of the rest
+            this_repr = "%s=%s" % (k, printer(v))
+        if len(this_repr) > 500:
+            this_repr = this_repr[:300] + "..." + this_repr[-100:]
+        if i > 0:
+            if this_line_length + len(this_repr) >= 75 or "\n" in this_repr:
+                params_list.append(line_sep)
+                this_line_length = len(line_sep)
+            else:
+                params_list.append(", ")
+                this_line_length += 2
+        params_list.append(this_repr)
+        this_line_length += len(this_repr)
+
+    np.set_printoptions(**options)
+    lines = "".join(params_list)
+    # Strip trailing space to avoid nightmare in doctests
+    lines = "\n".join(l.rstrip(" ") for l in lines.split("\n"))
+    return lines
 
 
 def _build_repr(self):
     # XXX This is copied from BaseEstimator's get_params
     cls = self.__class__
-    init = getattr(cls.__init__, 'deprecated_original', cls.__init__)
+    init = getattr(cls.__init__, "deprecated_original", cls.__init__)
     # Ignore varargs, kw and default values and pop self
     init_signature = signature(init)
     # Consider the constructor parameters excluding 'self'
     if init is object.__init__:
         args = []
     else:
-        args = sorted([p.name for p in init_signature.parameters.values()
-                       if p.name != 'self' and p.kind != p.VAR_KEYWORD])
+        args = sorted(
+            [
+                p.name
+                for p in init_signature.parameters.values()
+                if p.name != "self" and p.kind != p.VAR_KEYWORD
+            ]
+        )
     class_name = self.__class__.__name__
     params = dict()
     for key in args:
@@ -2229,16 +3032,16 @@ def _build_repr(self):
         try:
             with warnings.catch_warnings(record=True) as w:
                 value = getattr(self, key, None)
-                if value is None and hasattr(self, 'cvargs'):
+                if value is None and hasattr(self, "cvargs"):
                     value = self.cvargs.get(key, None)
-            if len(w) and w[0].category == FutureWarning:
+            if len(w) and w[0].category is FutureWarning:
                 # if the parameter is deprecated, don't show it
                 continue
         finally:
             warnings.filters.pop(0)
         params[key] = value
 
-    return '%s(%s)' % (class_name, _pprint(params, offset=len(class_name)))
+    return "%s(%s)" % (class_name, _pprint(params, offset=len(class_name)))
 
 
 def _yields_constant_splits(cv):
@@ -2247,6 +3050,6 @@ def _yields_constant_splits(cv):
     # default (e.g. ShuffleSplit). If it actually doesn't shuffle (e.g.
     # LeaveOneOut), then it won't have a random_state parameter anyway, in
     # which case it will default to 0, leading to output=True
-    shuffle = getattr(cv, 'shuffle', True)
-    random_state = getattr(cv, 'random_state', 0)
+    shuffle = getattr(cv, "shuffle", True)
+    random_state = getattr(cv, "random_state", 0)
     return isinstance(random_state, numbers.Integral) or not shuffle
diff --git a/sklearn/model_selection/_validation.py b/sklearn/model_selection/_validation.py
index e61e693b2fa74..8b70bf42603ef 100644
--- a/sklearn/model_selection/_validation.py
+++ b/sklearn/model_selection/_validation.py
@@ -3,46 +3,141 @@
 functions to validate the model.
 """
 
-# Author: Alexandre Gramfort 
-#         Gael Varoquaux 
-#         Olivier Grisel 
-#         Raghav RV 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-
-import warnings
 import numbers
 import time
-from traceback import format_exc
+import warnings
+from collections import Counter
 from contextlib import suppress
+from functools import partial
+from numbers import Real
+from traceback import format_exc
 
 import numpy as np
 import scipy.sparse as sp
-from joblib import Parallel, logger
-
-from ..base import is_classifier, clone
-from ..utils import indexable, check_random_state, _safe_indexing
-from ..utils.validation import _check_fit_params
-from ..utils.validation import _num_samples
-from ..utils.validation import _deprecate_positional_args
-from ..utils.fixes import delayed
+from joblib import logger
+
+from ..base import clone, is_classifier
+from ..exceptions import FitFailedWarning, UnsetMetadataPassedError
+from ..metrics import check_scoring, get_scorer_names
+from ..metrics._scorer import _MultimetricScorer
+from ..preprocessing import LabelEncoder
+from ..utils import Bunch, _safe_indexing, check_random_state, indexable
+from ..utils._array_api import device, get_namespace
+from ..utils._param_validation import (
+    HasMethods,
+    Integral,
+    Interval,
+    StrOptions,
+    validate_params,
+)
+from ..utils.metadata_routing import (
+    MetadataRouter,
+    MethodMapping,
+    _routing_enabled,
+    process_routing,
+)
 from ..utils.metaestimators import _safe_split
-from ..metrics import check_scoring
-from ..metrics._scorer import _check_multimetric_scoring, _MultimetricScorer
-from ..exceptions import FitFailedWarning, NotFittedError
+from ..utils.parallel import Parallel, delayed
+from ..utils.validation import _check_method_params, _num_samples
 from ._split import check_cv
-from ..preprocessing import LabelEncoder
 
+__all__ = [
+    "cross_val_predict",
+    "cross_val_score",
+    "cross_validate",
+    "learning_curve",
+    "permutation_test_score",
+    "validation_curve",
+]
+
+
+def _check_params_groups_deprecation(fit_params, params, groups, version):
+    """A helper function to check deprecations on `groups` and `fit_params`.
+
+    # TODO(SLEP6): To be removed when set_config(enable_metadata_routing=False) is not
+    # possible.
+    """
+    if params is not None and fit_params is not None:
+        raise ValueError(
+            "`params` and `fit_params` cannot both be provided. Pass parameters "
+            "via `params`. `fit_params` is deprecated and will be removed in "
+            f"version {version}."
+        )
+    elif fit_params is not None:
+        warnings.warn(
+            (
+                "`fit_params` is deprecated and will be removed in version {version}. "
+                "Pass parameters via `params` instead."
+            ),
+            FutureWarning,
+        )
+        params = fit_params
+
+    params = {} if params is None else params
 
-__all__ = ['cross_validate', 'cross_val_score', 'cross_val_predict',
-           'permutation_test_score', 'learning_curve', 'validation_curve']
+    _check_groups_routing_disabled(groups)
 
+    return params
 
-@_deprecate_positional_args
-def cross_validate(estimator, X, y=None, *, groups=None, scoring=None, cv=None,
-                   n_jobs=None, verbose=0, fit_params=None,
-                   pre_dispatch='2*n_jobs', return_train_score=False,
-                   return_estimator=False, error_score=np.nan):
+
+# TODO(SLEP6): To be removed when set_config(enable_metadata_routing=False) is not
+# possible.
+def _check_groups_routing_disabled(groups):
+    if groups is not None and _routing_enabled():
+        raise ValueError(
+            "`groups` can only be passed if metadata routing is not enabled via"
+            " `sklearn.set_config(enable_metadata_routing=True)`. When routing is"
+            " enabled, pass `groups` alongside other metadata via the `params` argument"
+            " instead."
+        )
+
+
+@validate_params(
+    {
+        "estimator": [HasMethods("fit")],
+        "X": ["array-like", "sparse matrix"],
+        "y": ["array-like", None],
+        "groups": ["array-like", None],
+        "scoring": [
+            StrOptions(set(get_scorer_names())),
+            callable,
+            list,
+            tuple,
+            dict,
+            None,
+        ],
+        "cv": ["cv_object"],
+        "n_jobs": [Integral, None],
+        "verbose": ["verbose"],
+        "params": [dict, None],
+        "pre_dispatch": [Integral, str],
+        "return_train_score": ["boolean"],
+        "return_estimator": ["boolean"],
+        "return_indices": ["boolean"],
+        "error_score": [StrOptions({"raise"}), Real],
+    },
+    prefer_skip_nested_validation=False,  # estimator is not validated yet
+)
+def cross_validate(
+    estimator,
+    X,
+    y=None,
+    *,
+    groups=None,
+    scoring=None,
+    cv=None,
+    n_jobs=None,
+    verbose=0,
+    params=None,
+    pre_dispatch="2*n_jobs",
+    return_train_score=False,
+    return_estimator=False,
+    return_indices=False,
+    error_score=np.nan,
+):
     """Evaluate metric(s) by cross-validation and also record fit/score times.
 
     Read more in the :ref:`User Guide `.
@@ -52,11 +147,10 @@ def cross_validate(estimator, X, y=None, *, groups=None, scoring=None, cv=None,
     estimator : estimator object implementing 'fit'
         The object to use to fit the data.
 
-    X : array-like of shape (n_samples, n_features)
+    X : {array-like, sparse matrix} of shape (n_samples, n_features)
         The data to fit. Can be for example a list, or an array.
 
-    y : array-like of shape (n_samples,) or (n_samples, n_outputs), \
-            default=None
+    y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None
         The target variable to try to predict in the case of
         supervised learning.
 
@@ -65,14 +159,23 @@ def cross_validate(estimator, X, y=None, *, groups=None, scoring=None, cv=None,
         train/test set. Only used in conjunction with a "Group" :term:`cv`
         instance (e.g., :class:`GroupKFold`).
 
+        .. versionchanged:: 1.4
+            ``groups`` can only be passed if metadata routing is not enabled
+            via ``sklearn.set_config(enable_metadata_routing=True)``. When routing
+            is enabled, pass ``groups`` alongside other metadata via the ``params``
+            argument instead. E.g.:
+            ``cross_validate(..., params={'groups': groups})``.
+
     scoring : str, callable, list, tuple, or dict, default=None
-        Strategy to evaluate the performance of the cross-validated model on
-        the test set.
+        Strategy to evaluate the performance of the `estimator` across cross-validation
+        splits.
 
         If `scoring` represents a single score, one can use:
 
-        - a single string (see :ref:`scoring_parameter`);
-        - a callable (see :ref:`scoring`) that returns a single value.
+        - a single string (see :ref:`scoring_string_names`);
+        - a callable (see :ref:`scoring_callable`) that returns a single value.
+        - `None`, the `estimator`'s
+          :ref:`default evaluation criterion ` is used.
 
         If `scoring` represents multiple scores, one can use:
 
@@ -94,7 +197,8 @@ def cross_validate(estimator, X, y=None, *, groups=None, scoring=None, cv=None,
 
         For int/None inputs, if the estimator is a classifier and ``y`` is
         either binary or multiclass, :class:`StratifiedKFold` is used. In all
-        other cases, :class:`KFold` is used.
+        other cases, :class:`KFold` is used. These splitters are instantiated
+        with `shuffle=False` so the splits will be the same across calls.
 
         Refer :ref:`User Guide ` for the various
         cross-validation strategies that can be used here.
@@ -112,8 +216,11 @@ def cross_validate(estimator, X, y=None, *, groups=None, scoring=None, cv=None,
     verbose : int, default=0
         The verbosity level.
 
-    fit_params : dict, default=None
-        Parameters to pass to the fit method of the estimator.
+    params : dict, default=None
+        Parameters to pass to the underlying estimator's ``fit``, the scorer,
+        and the CV splitter.
+
+        .. versionadded:: 1.4
 
     pre_dispatch : int or str, default='2*n_jobs'
         Controls the number of jobs that get dispatched during parallel
@@ -121,16 +228,8 @@ def cross_validate(estimator, X, y=None, *, groups=None, scoring=None, cv=None,
         explosion of memory consumption when more jobs get dispatched
         than CPUs can process. This parameter can be:
 
-            - None, in which case all the jobs are immediately
-              created and spawned. Use this for lightweight and
-              fast-running jobs, to avoid delays due to on-demand
-              spawning of the jobs
-
-            - An int, giving the exact number of total jobs that are
-              spawned
-
-            - A str, giving an expression as a function of n_jobs,
-              as in '2*n_jobs'
+        - An int, giving the exact number of total jobs that are spawned
+        - A str, giving an expression as a function of n_jobs, as in '2*n_jobs'
 
     return_train_score : bool, default=False
         Whether to include train scores.
@@ -150,6 +249,11 @@ def cross_validate(estimator, X, y=None, *, groups=None, scoring=None, cv=None,
 
         .. versionadded:: 0.20
 
+    return_indices : bool, default=False
+        Whether to return the train-test indices selected for each split.
+
+        .. versionadded:: 1.3
+
     error_score : 'raise' or numeric, default=np.nan
         Value to assign to the score if an error occurs in estimator fitting.
         If set to 'raise', the error is raised.
@@ -165,29 +269,44 @@ def cross_validate(estimator, X, y=None, *, groups=None, scoring=None, cv=None,
         A dict of arrays containing the score/time arrays for each scorer is
         returned. The possible keys for this ``dict`` are:
 
-            ``test_score``
-                The score array for test scores on each cv split.
-                Suffix ``_score`` in ``test_score`` changes to a specific
-                metric like ``test_r2`` or ``test_auc`` if there are
-                multiple scoring metrics in the scoring parameter.
-            ``train_score``
-                The score array for train scores on each cv split.
-                Suffix ``_score`` in ``train_score`` changes to a specific
-                metric like ``train_r2`` or ``train_auc`` if there are
-                multiple scoring metrics in the scoring parameter.
-                This is available only if ``return_train_score`` parameter
-                is ``True``.
-            ``fit_time``
-                The time for fitting the estimator on the train
-                set for each cv split.
-            ``score_time``
-                The time for scoring the estimator on the test set for each
-                cv split. (Note time for scoring on the train set is not
-                included even if ``return_train_score`` is set to ``True``
-            ``estimator``
-                The estimator objects for each cv split.
-                This is available only if ``return_estimator`` parameter
-                is set to ``True``.
+        ``test_score``
+            The score array for test scores on each cv split.
+            Suffix ``_score`` in ``test_score`` changes to a specific
+            metric like ``test_r2`` or ``test_auc`` if there are
+            multiple scoring metrics in the scoring parameter.
+        ``train_score``
+            The score array for train scores on each cv split.
+            Suffix ``_score`` in ``train_score`` changes to a specific
+            metric like ``train_r2`` or ``train_auc`` if there are
+            multiple scoring metrics in the scoring parameter.
+            This is available only if ``return_train_score`` parameter
+            is ``True``.
+        ``fit_time``
+            The time for fitting the estimator on the train
+            set for each cv split.
+        ``score_time``
+            The time for scoring the estimator on the test set for each
+            cv split. (Note: time for scoring on the train set is not
+            included even if ``return_train_score`` is set to ``True``).
+        ``estimator``
+            The estimator objects for each cv split.
+            This is available only if ``return_estimator`` parameter
+            is set to ``True``.
+        ``indices``
+            The train/test positional indices for each cv split. A dictionary
+            is returned where the keys are either `"train"` or `"test"`
+            and the associated values are a list of integer-dtyped NumPy
+            arrays with the indices. Available only if `return_indices=True`.
+
+    See Also
+    --------
+    cross_val_score : Run cross-validation for single metric evaluation.
+
+    cross_val_predict : Get predictions from each split of cross-validation for
+        diagnostic purposes.
+
+    sklearn.metrics.make_scorer : Make a scorer from a performance metric or
+        loss function.
 
     Examples
     --------
@@ -207,7 +326,7 @@ def cross_validate(estimator, X, y=None, *, groups=None, scoring=None, cv=None,
     >>> sorted(cv_results.keys())
     ['fit_time', 'score_time', 'test_score']
     >>> cv_results['test_score']
-    array([0.33150734, 0.08022311, 0.03531764])
+    array([0.3315057 , 0.08022103, 0.03531816])
 
     Multiple metric evaluation using ``cross_validate``
     (please refer the ``scoring`` parameter doc for more information)
@@ -216,45 +335,90 @@ def cross_validate(estimator, X, y=None, *, groups=None, scoring=None, cv=None,
     ...                         scoring=('r2', 'neg_mean_squared_error'),
     ...                         return_train_score=True)
     >>> print(scores['test_neg_mean_squared_error'])
-    [-3635.5... -3573.3... -6114.7...]
+    [-3635.5 -3573.3 -6114.7]
     >>> print(scores['train_r2'])
-    [0.28010158 0.39088426 0.22784852]
-
-    See Also
-    ---------
-    cross_val_score : Run cross-validation for single metric evaluation.
-
-    cross_val_predict : Get predictions from each split of cross-validation for
-        diagnostic purposes.
-
-    sklearn.metrics.make_scorer : Make a scorer from a performance metric or
-        loss function.
-
+    [0.28009951 0.3908844  0.22784907]
     """
-    X, y, groups = indexable(X, y, groups)
+    _check_groups_routing_disabled(groups)
 
+    X, y = indexable(X, y)
+    params = {} if params is None else params
     cv = check_cv(cv, y, classifier=is_classifier(estimator))
 
-    if callable(scoring):
-        scorers = scoring
-    elif scoring is None or isinstance(scoring, str):
-        scorers = check_scoring(estimator, scoring)
+    scorers = check_scoring(
+        estimator, scoring=scoring, raise_exc=(error_score == "raise")
+    )
+
+    if _routing_enabled():
+        # For estimators, a MetadataRouter is created in get_metadata_routing
+        # methods. For these router methods, we create the router to use
+        # `process_routing` on it.
+        router = (
+            MetadataRouter(owner="cross_validate")
+            .add(
+                splitter=cv,
+                method_mapping=MethodMapping().add(caller="fit", callee="split"),
+            )
+            .add(
+                estimator=estimator,
+                # TODO(SLEP6): also pass metadata to the predict method for
+                # scoring?
+                method_mapping=MethodMapping().add(caller="fit", callee="fit"),
+            )
+            .add(
+                scorer=scorers,
+                method_mapping=MethodMapping().add(caller="fit", callee="score"),
+            )
+        )
+        try:
+            routed_params = process_routing(router, "fit", **params)
+        except UnsetMetadataPassedError as e:
+            # The default exception would mention `fit` since in the above
+            # `process_routing` code, we pass `fit` as the caller. However,
+            # the user is not calling `fit` directly, so we change the message
+            # to make it more suitable for this case.
+            raise UnsetMetadataPassedError(
+                message=str(e).replace("cross_validate.fit", "cross_validate"),
+                unrequested_params=e.unrequested_params,
+                routed_params=e.routed_params,
+            )
     else:
-        scorers = _check_multimetric_scoring(estimator, scoring)
+        routed_params = Bunch()
+        routed_params.splitter = Bunch(split={"groups": groups})
+        routed_params.estimator = Bunch(fit=params)
+        routed_params.scorer = Bunch(score={})
+
+    indices = cv.split(X, y, **routed_params.splitter.split)
+    if return_indices:
+        # materialize the indices since we need to store them in the returned dict
+        indices = list(indices)
 
     # We clone the estimator to make sure that all the folds are
     # independent, and that it is pickle-able.
-    parallel = Parallel(n_jobs=n_jobs, verbose=verbose,
-                        pre_dispatch=pre_dispatch)
+    parallel = Parallel(n_jobs=n_jobs, verbose=verbose, pre_dispatch=pre_dispatch)
     results = parallel(
         delayed(_fit_and_score)(
-            clone(estimator), X, y, scorers, train, test, verbose, None,
-            fit_params, return_train_score=return_train_score,
-            return_times=True, return_estimator=return_estimator,
-            error_score=error_score)
-        for train, test in cv.split(X, y, groups))
+            clone(estimator),
+            X,
+            y,
+            scorer=scorers,
+            train=train,
+            test=test,
+            verbose=verbose,
+            parameters=None,
+            fit_params=routed_params.estimator.fit,
+            score_params=routed_params.scorer.score,
+            return_train_score=return_train_score,
+            return_times=True,
+            return_estimator=return_estimator,
+            error_score=error_score,
+        )
+        for train, test in indices
+    )
 
-    # For callabe scoring, the return type is only know after calling. If the
+    _warn_or_raise_about_fit_failures(results, error_score)
+
+    # For callable scoring, the return type is only know after calling. If the
     # return type is a dictionary, the error scores can now be inserted with
     # the correct key.
     if callable(scoring):
@@ -263,20 +427,24 @@ def cross_validate(estimator, X, y=None, *, groups=None, scoring=None, cv=None,
     results = _aggregate_score_dicts(results)
 
     ret = {}
-    ret['fit_time'] = results["fit_time"]
-    ret['score_time'] = results["score_time"]
+    ret["fit_time"] = results["fit_time"]
+    ret["score_time"] = results["score_time"]
 
     if return_estimator:
-        ret['estimator'] = results["estimator"]
+        ret["estimator"] = results["estimator"]
+
+    if return_indices:
+        ret["indices"] = {}
+        ret["indices"]["train"], ret["indices"]["test"] = zip(*indices)
 
     test_scores_dict = _normalize_score_results(results["test_scores"])
     if return_train_score:
         train_scores_dict = _normalize_score_results(results["train_scores"])
 
     for name in test_scores_dict:
-        ret['test_%s' % name] = test_scores_dict[name]
+        ret["test_%s" % name] = test_scores_dict[name]
         if return_train_score:
-            key = 'train_%s' % name
+            key = "train_%s" % name
             ret[key] = train_scores_dict[name]
 
     return ret
@@ -291,14 +459,11 @@ def _insert_error_scores(results, error_score):
     successful_score = None
     failed_indices = []
     for i, result in enumerate(results):
-        if result["fit_failed"]:
+        if result["fit_error"] is not None:
             failed_indices.append(i)
         elif successful_score is None:
             successful_score = result["test_scores"]
 
-    if successful_score is None:
-        raise NotFittedError("All estimators failed to fit")
-
     if isinstance(successful_score, dict):
         formatted_error = {name: error_score for name in successful_score}
         for i in failed_indices:
@@ -307,7 +472,7 @@ def _insert_error_scores(results, error_score):
                 results[i]["train_scores"] = formatted_error.copy()
 
 
-def _normalize_score_results(scores, scaler_score_key='score'):
+def _normalize_score_results(scores, scaler_score_key="score"):
     """Creates a scoring dictionary based on the type of `scores`"""
     if isinstance(scores[0], dict):
         # multimetric scoring
@@ -316,11 +481,72 @@ def _normalize_score_results(scores, scaler_score_key='score'):
     return {scaler_score_key: scores}
 
 
-@_deprecate_positional_args
-def cross_val_score(estimator, X, y=None, *, groups=None, scoring=None,
-                    cv=None, n_jobs=None, verbose=0, fit_params=None,
-                    pre_dispatch='2*n_jobs', error_score=np.nan):
-    """Evaluate a score by cross-validation
+def _warn_or_raise_about_fit_failures(results, error_score):
+    fit_errors = [
+        result["fit_error"] for result in results if result["fit_error"] is not None
+    ]
+    if fit_errors:
+        num_failed_fits = len(fit_errors)
+        num_fits = len(results)
+        fit_errors_counter = Counter(fit_errors)
+        delimiter = "-" * 80 + "\n"
+        fit_errors_summary = "\n".join(
+            f"{delimiter}{n} fits failed with the following error:\n{error}"
+            for error, n in fit_errors_counter.items()
+        )
+
+        if num_failed_fits == num_fits:
+            all_fits_failed_message = (
+                f"\nAll the {num_fits} fits failed.\n"
+                "It is very likely that your model is misconfigured.\n"
+                "You can try to debug the error by setting error_score='raise'.\n\n"
+                f"Below are more details about the failures:\n{fit_errors_summary}"
+            )
+            raise ValueError(all_fits_failed_message)
+
+        else:
+            some_fits_failed_message = (
+                f"\n{num_failed_fits} fits failed out of a total of {num_fits}.\n"
+                "The score on these train-test partitions for these parameters"
+                f" will be set to {error_score}.\n"
+                "If these failures are not expected, you can try to debug them "
+                "by setting error_score='raise'.\n\n"
+                f"Below are more details about the failures:\n{fit_errors_summary}"
+            )
+            warnings.warn(some_fits_failed_message, FitFailedWarning)
+
+
+@validate_params(
+    {
+        "estimator": [HasMethods("fit")],
+        "X": ["array-like", "sparse matrix"],
+        "y": ["array-like", None],
+        "groups": ["array-like", None],
+        "scoring": [StrOptions(set(get_scorer_names())), callable, None],
+        "cv": ["cv_object"],
+        "n_jobs": [Integral, None],
+        "verbose": ["verbose"],
+        "params": [dict, None],
+        "pre_dispatch": [Integral, str, None],
+        "error_score": [StrOptions({"raise"}), Real],
+    },
+    prefer_skip_nested_validation=False,  # estimator is not validated yet
+)
+def cross_val_score(
+    estimator,
+    X,
+    y=None,
+    *,
+    groups=None,
+    scoring=None,
+    cv=None,
+    n_jobs=None,
+    verbose=0,
+    params=None,
+    pre_dispatch="2*n_jobs",
+    error_score=np.nan,
+):
+    """Evaluate a score by cross-validation.
 
     Read more in the :ref:`User Guide `.
 
@@ -329,7 +555,7 @@ def cross_val_score(estimator, X, y=None, *, groups=None, scoring=None,
     estimator : estimator object implementing 'fit'
         The object to use to fit the data.
 
-    X : array-like of shape (n_samples, n_features)
+    X : {array-like, sparse matrix} of shape (n_samples, n_features)
         The data to fit. Can be for example a list, or an array.
 
     y : array-like of shape (n_samples,) or (n_samples, n_outputs), \
@@ -342,35 +568,46 @@ def cross_val_score(estimator, X, y=None, *, groups=None, scoring=None,
         train/test set. Only used in conjunction with a "Group" :term:`cv`
         instance (e.g., :class:`GroupKFold`).
 
+        .. versionchanged:: 1.4
+            ``groups`` can only be passed if metadata routing is not enabled
+            via ``sklearn.set_config(enable_metadata_routing=True)``. When routing
+            is enabled, pass ``groups`` alongside other metadata via the ``params``
+            argument instead. E.g.:
+            ``cross_val_score(..., params={'groups': groups})``.
+
     scoring : str or callable, default=None
-        A str (see model evaluation documentation) or
-        a scorer callable object / function with signature
-        ``scorer(estimator, X, y)`` which should return only
-        a single value.
+        Strategy to evaluate the performance of the `estimator` across cross-validation
+        splits.
 
-        Similar to :func:`cross_validate`
-        but only a single metric is permitted.
+        - str: see :ref:`scoring_string_names` for options.
+        - callable: a scorer callable object (e.g., function) with signature
+          ``scorer(estimator, X, y)``, which should return only a single value.
+          See :ref:`scoring_callable` for details.
+        - `None`: the `estimator`'s
+          :ref:`default evaluation criterion ` is used.
 
-        If None, the estimator's default scorer (if available) is used.
+        Similar to the use of `scoring` in :func:`cross_validate` but only a
+        single metric is permitted.
 
     cv : int, cross-validation generator or an iterable, default=None
         Determines the cross-validation splitting strategy.
         Possible inputs for cv are:
 
-        - None, to use the default 5-fold cross validation,
+        - `None`, to use the default 5-fold cross validation,
         - int, to specify the number of folds in a `(Stratified)KFold`,
         - :term:`CV splitter`,
-        - An iterable yielding (train, test) splits as arrays of indices.
+        - An iterable that generates (train, test) splits as arrays of indices.
 
-        For int/None inputs, if the estimator is a classifier and ``y`` is
+        For `int`/`None` inputs, if the estimator is a classifier and `y` is
         either binary or multiclass, :class:`StratifiedKFold` is used. In all
-        other cases, :class:`KFold` is used.
+        other cases, :class:`KFold` is used. These splitters are instantiated
+        with `shuffle=False` so the splits will be the same across calls.
 
         Refer :ref:`User Guide ` for the various
         cross-validation strategies that can be used here.
 
         .. versionchanged:: 0.22
-            ``cv`` default value if None changed from 3-fold to 5-fold.
+            `cv` default value if `None` changed from 3-fold to 5-fold.
 
     n_jobs : int, default=None
         Number of jobs to run in parallel. Training the estimator and computing
@@ -382,8 +619,11 @@ def cross_val_score(estimator, X, y=None, *, groups=None, scoring=None,
     verbose : int, default=0
         The verbosity level.
 
-    fit_params : dict, default=None
-        Parameters to pass to the fit method of the estimator.
+    params : dict, default=None
+        Parameters to pass to the underlying estimator's ``fit``, the scorer,
+        and the CV splitter.
+
+        .. versionadded:: 1.4
 
     pre_dispatch : int or str, default='2*n_jobs'
         Controls the number of jobs that get dispatched during parallel
@@ -391,16 +631,11 @@ def cross_val_score(estimator, X, y=None, *, groups=None, scoring=None,
         explosion of memory consumption when more jobs get dispatched
         than CPUs can process. This parameter can be:
 
-            - None, in which case all the jobs are immediately
-              created and spawned. Use this for lightweight and
-              fast-running jobs, to avoid delays due to on-demand
-              spawning of the jobs
-
-            - An int, giving the exact number of total jobs that are
-              spawned
-
-            - A str, giving an expression as a function of n_jobs,
-              as in '2*n_jobs'
+        - ``None``, in which case all the jobs are immediately created and spawned. Use
+          this for lightweight and fast-running jobs, to avoid delays due to on-demand
+          spawning of the jobs
+        - An int, giving the exact number of total jobs that are spawned
+        - A str, giving an expression as a function of n_jobs, as in '2*n_jobs'
 
     error_score : 'raise' or numeric, default=np.nan
         Value to assign to the score if an error occurs in estimator fitting.
@@ -414,19 +649,8 @@ def cross_val_score(estimator, X, y=None, *, groups=None, scoring=None,
     scores : ndarray of float of shape=(len(list(cv)),)
         Array of scores of the estimator for each run of the cross validation.
 
-    Examples
-    --------
-    >>> from sklearn import datasets, linear_model
-    >>> from sklearn.model_selection import cross_val_score
-    >>> diabetes = datasets.load_diabetes()
-    >>> X = diabetes.data[:150]
-    >>> y = diabetes.target[:150]
-    >>> lasso = linear_model.Lasso()
-    >>> print(cross_val_score(lasso, X, y, cv=3))
-    [0.33150734 0.08022311 0.03531764]
-
     See Also
-    ---------
+    --------
     cross_validate : To run cross-validation on multiple metrics and also to
         return train scores, fit times and score times.
 
@@ -436,26 +660,57 @@ def cross_val_score(estimator, X, y=None, *, groups=None, scoring=None,
     sklearn.metrics.make_scorer : Make a scorer from a performance metric or
         loss function.
 
+    Examples
+    --------
+    >>> from sklearn import datasets, linear_model
+    >>> from sklearn.model_selection import cross_val_score
+    >>> diabetes = datasets.load_diabetes()
+    >>> X = diabetes.data[:150]
+    >>> y = diabetes.target[:150]
+    >>> lasso = linear_model.Lasso()
+    >>> print(cross_val_score(lasso, X, y, cv=3))
+    [0.3315057  0.08022103 0.03531816]
     """
     # To ensure multimetric format is not supported
     scorer = check_scoring(estimator, scoring=scoring)
 
-    cv_results = cross_validate(estimator=estimator, X=X, y=y, groups=groups,
-                                scoring={'score': scorer}, cv=cv,
-                                n_jobs=n_jobs, verbose=verbose,
-                                fit_params=fit_params,
-                                pre_dispatch=pre_dispatch,
-                                error_score=error_score)
-    return cv_results['test_score']
-
-
-def _fit_and_score(estimator, X, y, scorer, train, test, verbose,
-                   parameters, fit_params, return_train_score=False,
-                   return_parameters=False, return_n_test_samples=False,
-                   return_times=False, return_estimator=False,
-                   split_progress=None, candidate_progress=None,
-                   error_score=np.nan):
-
+    cv_results = cross_validate(
+        estimator=estimator,
+        X=X,
+        y=y,
+        groups=groups,
+        scoring={"score": scorer},
+        cv=cv,
+        n_jobs=n_jobs,
+        verbose=verbose,
+        params=params,
+        pre_dispatch=pre_dispatch,
+        error_score=error_score,
+    )
+    return cv_results["test_score"]
+
+
+def _fit_and_score(
+    estimator,
+    X,
+    y,
+    *,
+    scorer,
+    train,
+    test,
+    verbose,
+    parameters,
+    fit_params,
+    score_params,
+    return_train_score=False,
+    return_parameters=False,
+    return_n_test_samples=False,
+    return_times=False,
+    return_estimator=False,
+    split_progress=None,
+    candidate_progress=None,
+    error_score=np.nan,
+):
     """Fit estimator and compute scores for a given dataset split.
 
     Parameters
@@ -500,6 +755,9 @@ def _fit_and_score(estimator, X, y, scorer, train, test, verbose,
     fit_params : dict or None
         Parameters that will be passed to ``estimator.fit``.
 
+    score_params : dict or None
+        Parameters that will be passed to the scorer.
+
     return_train_score : bool, default=False
         Compute and return score on training set.
 
@@ -540,10 +798,17 @@ def _fit_and_score(estimator, X, y, scorer, train, test, verbose,
             The parameters that have been evaluated.
         estimator : estimator object
             The fitted estimator.
-        fit_failed : bool
-            The estimator failed to fit.
+        fit_error : str or None
+            Traceback str if the fit failed, None if the fit succeeded.
     """
-    if not isinstance(error_score, numbers.Number) and error_score != 'raise':
+    xp, _ = get_namespace(X)
+    X_device = device(X)
+
+    # Make sure that we can fancy index X even if train and test are provided
+    # as NumPy arrays by NumPy only cross-validation splitters.
+    train, test = xp.asarray(train, device=X_device), xp.asarray(test, device=X_device)
+
+    if not isinstance(error_score, numbers.Number) and error_score != "raise":
         raise ValueError(
             "error_score must be the string 'raise' or a numeric value. "
             "(Hint: if using 'raise', please make sure that it has been "
@@ -553,35 +818,33 @@ def _fit_and_score(estimator, X, y, scorer, train, test, verbose,
     progress_msg = ""
     if verbose > 2:
         if split_progress is not None:
-            progress_msg = f" {split_progress[0]+1}/{split_progress[1]}"
+            progress_msg = f" {split_progress[0] + 1}/{split_progress[1]}"
         if candidate_progress and verbose > 9:
-            progress_msg += (f"; {candidate_progress[0]+1}/"
-                             f"{candidate_progress[1]}")
+            progress_msg += f"; {candidate_progress[0] + 1}/{candidate_progress[1]}"
 
     if verbose > 1:
         if parameters is None:
-            params_msg = ''
+            params_msg = ""
         else:
             sorted_keys = sorted(parameters)  # Ensure deterministic o/p
-            params_msg = (', '.join(f'{k}={parameters[k]}'
-                                    for k in sorted_keys))
+            params_msg = ", ".join(f"{k}={parameters[k]}" for k in sorted_keys)
     if verbose > 9:
         start_msg = f"[CV{progress_msg}] START {params_msg}"
         print(f"{start_msg}{(80 - len(start_msg)) * '.'}")
 
     # Adjust length of sample weights
     fit_params = fit_params if fit_params is not None else {}
-    fit_params = _check_fit_params(X, fit_params, train)
+    fit_params = _check_method_params(X, params=fit_params, indices=train)
+    score_params = score_params if score_params is not None else {}
+    score_params_train = _check_method_params(X, params=score_params, indices=train)
+    score_params_test = _check_method_params(X, params=score_params, indices=test)
 
     if parameters is not None:
-        # clone after setting parameters in case any parameters
-        # are estimators (like pipeline steps)
-        # because pipeline doesn't clone steps in fit
-        cloned_parameters = {}
-        for k, v in parameters.items():
-            cloned_parameters[k] = clone(v, safe=False)
-
-        estimator = estimator.set_params(**cloned_parameters)
+        # here we clone the parameters, since sometimes the parameters
+        # themselves might be estimators, e.g. when we search over different
+        # estimators in a pipeline.
+        # ref: https://github.com/scikit-learn/scikit-learn/pull/26786
+        estimator = estimator.set_params(**clone(parameters, safe=False))
 
     start_time = time.time()
 
@@ -595,36 +858,33 @@ def _fit_and_score(estimator, X, y, scorer, train, test, verbose,
         else:
             estimator.fit(X_train, y_train, **fit_params)
 
-    except Exception as e:
+    except Exception:
         # Note fit time as time until error
         fit_time = time.time() - start_time
         score_time = 0.0
-        if error_score == 'raise':
+        if error_score == "raise":
             raise
         elif isinstance(error_score, numbers.Number):
-            if isinstance(scorer, dict):
-                test_scores = {name: error_score for name in scorer}
+            if isinstance(scorer, _MultimetricScorer):
+                test_scores = {name: error_score for name in scorer._scorers}
                 if return_train_score:
                     train_scores = test_scores.copy()
             else:
                 test_scores = error_score
                 if return_train_score:
                     train_scores = error_score
-            warnings.warn("Estimator fit failed. The score on this train-test"
-                          " partition for these parameters will be set to %f. "
-                          "Details: \n%s" %
-                          (error_score, format_exc()),
-                          FitFailedWarning)
-        result["fit_failed"] = True
+        result["fit_error"] = format_exc()
     else:
-        result["fit_failed"] = False
+        result["fit_error"] = None
 
         fit_time = time.time() - start_time
-        test_scores = _score(estimator, X_test, y_test, scorer, error_score)
+        test_scores = _score(
+            estimator, X_test, y_test, scorer, score_params_test, error_score
+        )
         score_time = time.time() - start_time - fit_time
         if return_train_score:
             train_scores = _score(
-                estimator, X_train, y_train, scorer, error_score
+                estimator, X_train, y_train, scorer, score_params_train, error_score
             )
 
     if verbose > 1:
@@ -642,8 +902,7 @@ def _fit_and_score(estimator, X, y, scorer, train, test, verbose,
             else:
                 result_msg += ", score="
                 if return_train_score:
-                    result_msg += (f"(train={train_scores:.3f}, "
-                                   f"test={test_scores:.3f})")
+                    result_msg += f"(train={train_scores:.3f}, test={test_scores:.3f})"
                 else:
                     result_msg += f"{test_scores:.3f}"
         result_msg += f" total time={logger.short_format_time(total_time)}"
@@ -668,42 +927,60 @@ def _fit_and_score(estimator, X, y, scorer, train, test, verbose,
     return result
 
 
-def _score(estimator, X_test, y_test, scorer, error_score="raise"):
+def _score(estimator, X_test, y_test, scorer, score_params, error_score="raise"):
     """Compute the score(s) of an estimator on a given test set.
 
-    Will return a dict of floats if `scorer` is a dict, otherwise a single
+    Will return a dict of floats if `scorer` is a _MultiMetricScorer, otherwise a single
     float is returned.
     """
-    if isinstance(scorer, dict):
-        # will cache method calls if needed. scorer() returns a dict
-        scorer = _MultimetricScorer(**scorer)
+    score_params = {} if score_params is None else score_params
 
     try:
         if y_test is None:
-            scores = scorer(estimator, X_test)
+            scores = scorer(estimator, X_test, **score_params)
         else:
-            scores = scorer(estimator, X_test, y_test)
+            scores = scorer(estimator, X_test, y_test, **score_params)
     except Exception:
-        if error_score == 'raise':
+        if isinstance(scorer, _MultimetricScorer):
+            # If `_MultimetricScorer` raises exception, the `error_score`
+            # parameter is equal to "raise".
             raise
         else:
-            if isinstance(scorer, _MultimetricScorer):
-                scores = {name: error_score for name in scorer._scorers}
+            if error_score == "raise":
+                raise
             else:
                 scores = error_score
-            warnings.warn(
-                f"Scoring failed. The score on this train-test partition for "
-                f"these parameters will be set to {error_score}. Details: \n"
-                f"{format_exc()}",
-                UserWarning,
-            )
-
-    error_msg = (
-        "scoring must return a number, got %s (%s) instead. (scorer=%s)"
-    )
+                warnings.warn(
+                    (
+                        "Scoring failed. The score on this train-test partition for "
+                        f"these parameters will be set to {error_score}. Details: \n"
+                        f"{format_exc()}"
+                    ),
+                    UserWarning,
+                )
+
+    # Check non-raised error messages in `_MultimetricScorer`
+    if isinstance(scorer, _MultimetricScorer):
+        exception_messages = [
+            (name, str_e) for name, str_e in scores.items() if isinstance(str_e, str)
+        ]
+        if exception_messages:
+            # error_score != "raise"
+            for name, str_e in exception_messages:
+                scores[name] = error_score
+                warnings.warn(
+                    (
+                        "Scoring failed. The score on this train-test partition for "
+                        f"these parameters will be set to {error_score}. Details: \n"
+                        f"{str_e}"
+                    ),
+                    UserWarning,
+                )
+
+    error_msg = "scoring must return a number, got %s (%s) instead. (scorer=%s)"
     if isinstance(scores, dict):
         for name, score in scores.items():
-            if hasattr(score, 'item'):
+            if hasattr(score, "item"):
                 with suppress(ValueError):
                     # e.g. unwrap memmapped scalars
                     score = score.item()
@@ -711,7 +988,7 @@ def _score(estimator, X_test, y_test, scorer, error_score="raise"):
                 raise ValueError(error_msg % (score, type(score), name))
             scores[name] = score
     else:  # scalar
-        if hasattr(scores, 'item'):
+        if hasattr(scores, "item"):
             with suppress(ValueError):
                 # e.g. unwrap memmapped scalars
                 scores = scores.item()
@@ -720,11 +997,44 @@ def _score(estimator, X_test, y_test, scorer, error_score="raise"):
     return scores
 
 
-@_deprecate_positional_args
-def cross_val_predict(estimator, X, y=None, *, groups=None, cv=None,
-                      n_jobs=None, verbose=0, fit_params=None,
-                      pre_dispatch='2*n_jobs', method='predict'):
-    """Generate cross-validated estimates for each input data point
+@validate_params(
+    {
+        "estimator": [HasMethods(["fit", "predict"])],
+        "X": ["array-like", "sparse matrix"],
+        "y": ["array-like", "sparse matrix", None],
+        "groups": ["array-like", None],
+        "cv": ["cv_object"],
+        "n_jobs": [Integral, None],
+        "verbose": ["verbose"],
+        "params": [dict, None],
+        "pre_dispatch": [Integral, str, None],
+        "method": [
+            StrOptions(
+                {
+                    "predict",
+                    "predict_proba",
+                    "predict_log_proba",
+                    "decision_function",
+                }
+            )
+        ],
+    },
+    prefer_skip_nested_validation=False,  # estimator is not validated yet
+)
+def cross_val_predict(
+    estimator,
+    X,
+    y=None,
+    *,
+    groups=None,
+    cv=None,
+    n_jobs=None,
+    verbose=0,
+    params=None,
+    pre_dispatch="2*n_jobs",
+    method="predict",
+):
+    """Generate cross-validated estimates for each input data point.
 
     The data is split according to the cv parameter. Each sample belongs
     to exactly one test set, and its prediction is computed with an
@@ -739,13 +1049,14 @@ def cross_val_predict(estimator, X, y=None, *, groups=None, cv=None,
 
     Parameters
     ----------
-    estimator : estimator object implementing 'fit' and 'predict'
-        The object to use to fit the data.
+    estimator : estimator
+        The estimator instance to use to fit the data. It must implement a `fit`
+        method and the method given by the `method` parameter.
 
-    X : array-like of shape (n_samples, n_features)
+    X : {array-like, sparse matrix} of shape (n_samples, n_features)
         The data to fit. Can be, for example a list, or an array at least 2d.
 
-    y : array-like of shape (n_samples,) or (n_samples, n_outputs), \
+    y : {array-like, sparse matrix} of shape (n_samples,) or (n_samples, n_outputs), \
             default=None
         The target variable to try to predict in the case of
         supervised learning.
@@ -755,6 +1066,13 @@ def cross_val_predict(estimator, X, y=None, *, groups=None, cv=None,
         train/test set. Only used in conjunction with a "Group" :term:`cv`
         instance (e.g., :class:`GroupKFold`).
 
+        .. versionchanged:: 1.4
+            ``groups`` can only be passed if metadata routing is not enabled
+            via ``sklearn.set_config(enable_metadata_routing=True)``. When routing
+            is enabled, pass ``groups`` alongside other metadata via the ``params``
+            argument instead. E.g.:
+            ``cross_val_predict(..., params={'groups': groups})``.
+
     cv : int, cross-validation generator or an iterable, default=None
         Determines the cross-validation splitting strategy.
         Possible inputs for cv are:
@@ -762,11 +1080,12 @@ def cross_val_predict(estimator, X, y=None, *, groups=None, cv=None,
         - None, to use the default 5-fold cross validation,
         - int, to specify the number of folds in a `(Stratified)KFold`,
         - :term:`CV splitter`,
-        - An iterable yielding (train, test) splits as arrays of indices.
+        - An iterable that generates (train, test) splits as arrays of indices.
 
         For int/None inputs, if the estimator is a classifier and ``y`` is
         either binary or multiclass, :class:`StratifiedKFold` is used. In all
-        other cases, :class:`KFold` is used.
+        other cases, :class:`KFold` is used. These splitters are instantiated
+        with `shuffle=False` so the splits will be the same across calls.
 
         Refer :ref:`User Guide ` for the various
         cross-validation strategies that can be used here.
@@ -784,8 +1103,11 @@ def cross_val_predict(estimator, X, y=None, *, groups=None, cv=None,
     verbose : int, default=0
         The verbosity level.
 
-    fit_params : dict, defualt=None
-        Parameters to pass to the fit method of the estimator.
+    params : dict, default=None
+        Parameters to pass to the underlying estimator's ``fit`` and the CV
+        splitter.
+
+        .. versionadded:: 1.4
 
     pre_dispatch : int or str, default='2*n_jobs'
         Controls the number of jobs that get dispatched during parallel
@@ -793,16 +1115,11 @@ def cross_val_predict(estimator, X, y=None, *, groups=None, cv=None,
         explosion of memory consumption when more jobs get dispatched
         than CPUs can process. This parameter can be:
 
-            - None, in which case all the jobs are immediately
-              created and spawned. Use this for lightweight and
-              fast-running jobs, to avoid delays due to on-demand
-              spawning of the jobs
-
-            - An int, giving the exact number of total jobs that are
-              spawned
-
-            - A str, giving an expression as a function of n_jobs,
-              as in '2*n_jobs'
+        - None, in which case all the jobs are immediately created and spawned. Use
+          this for lightweight and fast-running jobs, to avoid delays due to on-demand
+          spawning of the jobs
+        - An int, giving the exact number of total jobs that are spawned
+        - A str, giving an expression as a function of n_jobs, as in '2*n_jobs'
 
     method : {'predict', 'predict_proba', 'predict_log_proba', \
               'decision_function'}, default='predict'
@@ -813,13 +1130,13 @@ def cross_val_predict(estimator, X, y=None, *, groups=None, cv=None,
     predictions : ndarray
         This is the result of calling `method`. Shape:
 
-            - When `method` is 'predict' and in special case where `method` is
-              'decision_function' and the target is binary: (n_samples,)
-            - When `method` is one of {'predict_proba', 'predict_log_proba',
-              'decision_function'} (unless special case above):
-              (n_samples, n_classes)
-            - If `estimator` is :term:`multioutput`, an extra dimension
-              'n_outputs' is added to the end of each shape above.
+        - When `method` is 'predict' and in special case where `method` is
+          'decision_function' and the target is binary: (n_samples,)
+        - When `method` is one of {'predict_proba', 'predict_log_proba',
+          'decision_function'} (unless special case above):
+          (n_samples, n_classes)
+        - If `estimator` is :term:`multioutput`, an extra dimension
+          'n_outputs' is added to the end of each shape above.
 
     See Also
     --------
@@ -846,19 +1163,56 @@ def cross_val_predict(estimator, X, y=None, *, groups=None, cv=None,
     >>> lasso = linear_model.Lasso()
     >>> y_pred = cross_val_predict(lasso, X, y, cv=3)
     """
-    X, y, groups = indexable(X, y, groups)
+    _check_groups_routing_disabled(groups)
+    X, y = indexable(X, y)
+    params = {} if params is None else params
+
+    if _routing_enabled():
+        # For estimators, a MetadataRouter is created in get_metadata_routing
+        # methods. For these router methods, we create the router to use
+        # `process_routing` on it.
+        router = (
+            MetadataRouter(owner="cross_val_predict")
+            .add(
+                splitter=cv,
+                method_mapping=MethodMapping().add(caller="fit", callee="split"),
+            )
+            .add(
+                estimator=estimator,
+                # TODO(SLEP6): also pass metadata for the predict method.
+                method_mapping=MethodMapping().add(caller="fit", callee="fit"),
+            )
+        )
+        try:
+            routed_params = process_routing(router, "fit", **params)
+        except UnsetMetadataPassedError as e:
+            # The default exception would mention `fit` since in the above
+            # `process_routing` code, we pass `fit` as the caller. However,
+            # the user is not calling `fit` directly, so we change the message
+            # to make it more suitable for this case.
+            raise UnsetMetadataPassedError(
+                message=str(e).replace("cross_val_predict.fit", "cross_val_predict"),
+                unrequested_params=e.unrequested_params,
+                routed_params=e.routed_params,
+            )
+    else:
+        routed_params = Bunch()
+        routed_params.splitter = Bunch(split={"groups": groups})
+        routed_params.estimator = Bunch(fit=params)
 
     cv = check_cv(cv, y, classifier=is_classifier(estimator))
-    splits = list(cv.split(X, y, groups))
+    splits = list(cv.split(X, y, **routed_params.splitter.split))
 
     test_indices = np.concatenate([test for _, test in splits])
     if not _check_is_permutation(test_indices, _num_samples(X)):
-        raise ValueError('cross_val_predict only works for partitions')
+        raise ValueError("cross_val_predict only works for partitions")
 
     # If classification methods produce multiple columns of output,
     # we need to manually encode classes to ensure consistent column ordering.
-    encode = method in ['decision_function', 'predict_proba',
-                        'predict_log_proba'] and y is not None
+    encode = (
+        method in ["decision_function", "predict_proba", "predict_log_proba"]
+        and y is not None
+    )
     if encode:
         y = np.asarray(y)
         if y.ndim == 1:
@@ -872,11 +1226,19 @@ def cross_val_predict(estimator, X, y=None, *, groups=None, cv=None,
 
     # We clone the estimator to make sure that all the folds are
     # independent, and that it is pickle-able.
-    parallel = Parallel(n_jobs=n_jobs, verbose=verbose,
-                        pre_dispatch=pre_dispatch)
-    predictions = parallel(delayed(_fit_and_predict)(
-        clone(estimator), X, y, train, test, verbose, fit_params, method)
-        for train, test in splits)
+    parallel = Parallel(n_jobs=n_jobs, verbose=verbose, pre_dispatch=pre_dispatch)
+    predictions = parallel(
+        delayed(_fit_and_predict)(
+            clone(estimator),
+            X,
+            y,
+            train,
+            test,
+            routed_params.estimator.fit,
+            method,
+        )
+        for train, test in splits
+    )
 
     inv_test_indices = np.empty(len(test_indices), dtype=int)
     inv_test_indices[test_indices] = np.arange(len(test_indices))
@@ -903,8 +1265,7 @@ def cross_val_predict(estimator, X, y=None, *, groups=None, cv=None,
         return predictions[inv_test_indices]
 
 
-def _fit_and_predict(estimator, X, y, train, test, verbose, fit_params,
-                     method):
+def _fit_and_predict(estimator, X, y, train, test, fit_params, method):
     """Fit estimator and predict values for a given dataset split.
 
     Read more in the :ref:`User Guide `.
@@ -930,9 +1291,6 @@ def _fit_and_predict(estimator, X, y, train, test, verbose, fit_params,
     test : array-like of shape (n_test_samples,)
         Indices of test samples.
 
-    verbose : int
-        The verbosity level.
-
     fit_params : dict or None
         Parameters that will be passed to ``estimator.fit``.
 
@@ -946,7 +1304,7 @@ def _fit_and_predict(estimator, X, y, train, test, verbose, fit_params,
     """
     # Adjust length of sample weights
     fit_params = fit_params if fit_params is not None else {}
-    fit_params = _check_fit_params(X, fit_params, train)
+    fit_params = _check_method_params(X, params=fit_params, indices=train)
 
     X_train, y_train = _safe_split(estimator, X, y, train)
     X_test, _ = _safe_split(estimator, X, y, test, train)
@@ -958,20 +1316,28 @@ def _fit_and_predict(estimator, X, y, train, test, verbose, fit_params,
     func = getattr(estimator, method)
     predictions = func(X_test)
 
-    encode = method in ['decision_function', 'predict_proba',
-                        'predict_log_proba'] and y is not None
+    encode = (
+        method in ["decision_function", "predict_proba", "predict_log_proba"]
+        and y is not None
+    )
 
     if encode:
         if isinstance(predictions, list):
-            predictions = [_enforce_prediction_order(
-                estimator.classes_[i_label], predictions[i_label],
-                n_classes=len(set(y[:, i_label])), method=method)
-                for i_label in range(len(predictions))]
+            predictions = [
+                _enforce_prediction_order(
+                    estimator.classes_[i_label],
+                    predictions[i_label],
+                    n_classes=len(set(y[:, i_label])),
+                    method=method,
+                )
+                for i_label in range(len(predictions))
+            ]
         else:
             # A 2D y array should be a binary label indicator matrix
             n_classes = len(set(y)) if y.ndim == 1 else y.shape[1]
             predictions = _enforce_prediction_order(
-                estimator.classes_, predictions, n_classes, method)
+                estimator.classes_, predictions, n_classes, method
+            )
     return predictions
 
 
@@ -990,43 +1356,52 @@ def _enforce_prediction_order(classes, predictions, n_classes, method):
     """
     if n_classes != len(classes):
         recommendation = (
-            'To fix this, use a cross-validation '
-            'technique resulting in properly '
-            'stratified folds')
-        warnings.warn('Number of classes in training fold ({}) does '
-                      'not match total number of classes ({}). '
-                      'Results may not be appropriate for your use case. '
-                      '{}'.format(len(classes), n_classes, recommendation),
-                      RuntimeWarning)
-        if method == 'decision_function':
-            if (predictions.ndim == 2 and
-                    predictions.shape[1] != len(classes)):
+            "To fix this, use a cross-validation "
+            "technique resulting in properly "
+            "stratified folds"
+        )
+        warnings.warn(
+            "Number of classes in training fold ({}) does "
+            "not match total number of classes ({}). "
+            "Results may not be appropriate for your use case. "
+            "{}".format(len(classes), n_classes, recommendation),
+            RuntimeWarning,
+        )
+        if method == "decision_function":
+            if predictions.ndim == 2 and predictions.shape[1] != len(classes):
                 # This handles the case when the shape of predictions
                 # does not match the number of classes used to train
                 # it with. This case is found when sklearn.svm.SVC is
                 # set to `decision_function_shape='ovo'`.
-                raise ValueError('Output shape {} of {} does not match '
-                                 'number of classes ({}) in fold. '
-                                 'Irregular decision_function outputs '
-                                 'are not currently supported by '
-                                 'cross_val_predict'.format(
-                                    predictions.shape, method, len(classes)))
+                raise ValueError(
+                    "Output shape {} of {} does not match "
+                    "number of classes ({}) in fold. "
+                    "Irregular decision_function outputs "
+                    "are not currently supported by "
+                    "cross_val_predict".format(predictions.shape, method, len(classes))
+                )
             if len(classes) <= 2:
                 # In this special case, `predictions` contains a 1D array.
-                raise ValueError('Only {} class/es in training fold, but {} '
-                                 'in overall dataset. This '
-                                 'is not supported for decision_function '
-                                 'with imbalanced folds. {}'.format(
-                                    len(classes), n_classes, recommendation))
+                raise ValueError(
+                    "Only {} class/es in training fold, but {} "
+                    "in overall dataset. This "
+                    "is not supported for decision_function "
+                    "with imbalanced folds. {}".format(
+                        len(classes), n_classes, recommendation
+                    )
+                )
 
         float_min = np.finfo(predictions.dtype).min
-        default_values = {'decision_function': float_min,
-                          'predict_log_proba': float_min,
-                          'predict_proba': 0}
-        predictions_for_all_classes = np.full((_num_samples(predictions),
-                                               n_classes),
-                                              default_values[method],
-                                              dtype=predictions.dtype)
+        default_values = {
+            "decision_function": float_min,
+            "predict_log_proba": float_min,
+            "predict_proba": 0,
+        }
+        predictions_for_all_classes = np.full(
+            (_num_samples(predictions), n_classes),
+            default_values[method],
+            dtype=predictions.dtype,
+        )
         predictions_for_all_classes[:, classes] = predictions
         predictions = predictions_for_all_classes
     return predictions
@@ -1056,18 +1431,46 @@ def _check_is_permutation(indices, n_samples):
     return True
 
 
-@_deprecate_positional_args
-def permutation_test_score(estimator, X, y, *, groups=None, cv=None,
-                           n_permutations=100, n_jobs=None, random_state=0,
-                           verbose=0, scoring=None, fit_params=None):
-    """Evaluate the significance of a cross-validated score with permutations
+@validate_params(
+    {
+        "estimator": [HasMethods("fit")],
+        "X": ["array-like", "sparse matrix"],
+        "y": ["array-like", None],
+        "groups": ["array-like", None],
+        "cv": ["cv_object"],
+        "n_permutations": [Interval(Integral, 1, None, closed="left")],
+        "n_jobs": [Integral, None],
+        "random_state": ["random_state"],
+        "verbose": ["verbose"],
+        "scoring": [StrOptions(set(get_scorer_names())), callable, None],
+        "fit_params": [dict, None],
+        "params": [dict, None],
+    },
+    prefer_skip_nested_validation=False,  # estimator is not validated yet
+)
+def permutation_test_score(
+    estimator,
+    X,
+    y,
+    *,
+    groups=None,
+    cv=None,
+    n_permutations=100,
+    n_jobs=None,
+    random_state=0,
+    verbose=0,
+    scoring=None,
+    fit_params=None,
+    params=None,
+):
+    """Evaluate the significance of a cross-validated score with permutations.
 
     Permutes targets to generate 'randomized data' and compute the empirical
     p-value against the null hypothesis that features and targets are
     independent.
 
     The p-value represents the fraction of randomized data sets where the
-    estimator performed as well or better than in the original data. A small
+    estimator performed as well or better than on the original data. A small
     p-value suggests that there is a real dependency between features and
     targets which has been used by the estimator to give good predictions.
     A large p-value may be due to lack of real dependency between features
@@ -1098,30 +1501,32 @@ def permutation_test_score(estimator, X, y, *, groups=None, cv=None,
         cross-validator uses them for grouping the samples  while splitting
         the dataset into train/test set.
 
-    scoring : str or callable, default=None
-        A single str (see :ref:`scoring_parameter`) or a callable
-        (see :ref:`scoring`) to evaluate the predictions on the test set.
-
-        If None the estimator's score method is used.
+        .. versionchanged:: 1.6
+            ``groups`` can only be passed if metadata routing is not enabled
+            via ``sklearn.set_config(enable_metadata_routing=True)``. When routing
+            is enabled, pass ``groups`` alongside other metadata via the ``params``
+            argument instead. E.g.:
+            ``permutation_test_score(..., params={'groups': groups})``.
 
     cv : int, cross-validation generator or an iterable, default=None
         Determines the cross-validation splitting strategy.
         Possible inputs for cv are:
 
-        - None, to use the default 5-fold cross validation,
+        - `None`, to use the default 5-fold cross validation,
         - int, to specify the number of folds in a `(Stratified)KFold`,
         - :term:`CV splitter`,
         - An iterable yielding (train, test) splits as arrays of indices.
 
-        For int/None inputs, if the estimator is a classifier and ``y`` is
+        For `int`/`None` inputs, if the estimator is a classifier and `y` is
         either binary or multiclass, :class:`StratifiedKFold` is used. In all
-        other cases, :class:`KFold` is used.
+        other cases, :class:`KFold` is used. These splitters are instantiated
+        with `shuffle=False` so the splits will be the same across calls.
 
         Refer :ref:`User Guide ` for the various
         cross-validation strategies that can be used here.
 
         .. versionchanged:: 0.22
-            ``cv`` default value if None changed from 3-fold to 5-fold.
+            `cv` default value if `None` changed from 3-fold to 5-fold.
 
     n_permutations : int, default=100
         Number of times to permute ``y``.
@@ -1140,10 +1545,35 @@ def permutation_test_score(estimator, X, y, *, groups=None, cv=None,
     verbose : int, default=0
         The verbosity level.
 
+    scoring : str or callable, default=None
+        Scoring method to use to evaluate the predictions on the validation set.
+
+        - str: see :ref:`scoring_string_names` for options.
+        - callable: a scorer callable object (e.g., function) with signature
+          ``scorer(estimator, X, y)``, which should return only a single value.
+          See :ref:`scoring_callable` for details.
+        - `None`: the `estimator`'s
+          :ref:`default evaluation criterion ` is used.
+
     fit_params : dict, default=None
         Parameters to pass to the fit method of the estimator.
 
-        .. versionadded:: 0.24
+        .. deprecated:: 1.6
+            This parameter is deprecated and will be removed in version 1.6. Use
+            ``params`` instead.
+
+    params : dict, default=None
+        Parameters to pass to the `fit` method of the estimator, the scorer
+        and the cv splitter.
+
+        - If `enable_metadata_routing=False` (default): Parameters directly passed to
+          the `fit` method of the estimator.
+
+        - If `enable_metadata_routing=True`: Parameters safely routed to the `fit`
+          method of the estimator, `cv` object and `scorer`. See :ref:`Metadata Routing
+          User Guide ` for more details.
+
+        .. versionadded:: 1.6
 
     Returns
     -------
@@ -1167,44 +1597,124 @@ def permutation_test_score(estimator, X, y, *, groups=None, cv=None,
     -----
     This function implements Test 1 in:
 
-        Ojala and Garriga. `Permutation Tests for Studying Classifier
-        Performance
-        `_. The
-        Journal of Machine Learning Research (2010) vol. 11
+    Ojala and Garriga. `Permutation Tests for Studying Classifier Performance
+    `_. The
+    Journal of Machine Learning Research (2010) vol. 11
 
+    Examples
+    --------
+    >>> from sklearn.datasets import make_classification
+    >>> from sklearn.linear_model import LogisticRegression
+    >>> from sklearn.model_selection import permutation_test_score
+    >>> X, y = make_classification(random_state=0)
+    >>> estimator = LogisticRegression()
+    >>> score, permutation_scores, pvalue = permutation_test_score(
+    ...     estimator, X, y, random_state=0
+    ... )
+    >>> print(f"Original Score: {score:.3f}")
+    Original Score: 0.810
+    >>> print(
+    ...     f"Permutation Scores: {permutation_scores.mean():.3f} +/- "
+    ...     f"{permutation_scores.std():.3f}"
+    ... )
+    Permutation Scores: 0.505 +/- 0.057
+    >>> print(f"P-value: {pvalue:.3f}")
+    P-value: 0.010
     """
+    params = _check_params_groups_deprecation(fit_params, params, groups, "1.8")
+
     X, y, groups = indexable(X, y, groups)
 
     cv = check_cv(cv, y, classifier=is_classifier(estimator))
     scorer = check_scoring(estimator, scoring=scoring)
     random_state = check_random_state(random_state)
 
+    if _routing_enabled():
+        router = (
+            MetadataRouter(owner="permutation_test_score")
+            .add(
+                estimator=estimator,
+                # TODO(SLEP6): also pass metadata to the predict method for
+                # scoring?
+                method_mapping=MethodMapping().add(caller="fit", callee="fit"),
+            )
+            .add(
+                splitter=cv,
+                method_mapping=MethodMapping().add(caller="fit", callee="split"),
+            )
+            .add(
+                scorer=scorer,
+                method_mapping=MethodMapping().add(caller="fit", callee="score"),
+            )
+        )
+
+        try:
+            routed_params = process_routing(router, "fit", **params)
+        except UnsetMetadataPassedError as e:
+            # The default exception would mention `fit` since in the above
+            # `process_routing` code, we pass `fit` as the caller. However,
+            # the user is not calling `fit` directly, so we change the message
+            # to make it more suitable for this case.
+            raise UnsetMetadataPassedError(
+                message=str(e).replace(
+                    "permutation_test_score.fit", "permutation_test_score"
+                ),
+                unrequested_params=e.unrequested_params,
+                routed_params=e.routed_params,
+            )
+
+    else:
+        routed_params = Bunch()
+        routed_params.estimator = Bunch(fit=params)
+        routed_params.splitter = Bunch(split={"groups": groups})
+        routed_params.scorer = Bunch(score={})
+
     # We clone the estimator to make sure that all the folds are
     # independent, and that it is pickle-able.
-    score = _permutation_test_score(clone(estimator), X, y, groups, cv, scorer,
-                                    fit_params=fit_params)
+    score = _permutation_test_score(
+        clone(estimator),
+        X,
+        y,
+        cv,
+        scorer,
+        split_params=routed_params.splitter.split,
+        fit_params=routed_params.estimator.fit,
+        score_params=routed_params.scorer.score,
+    )
     permutation_scores = Parallel(n_jobs=n_jobs, verbose=verbose)(
         delayed(_permutation_test_score)(
-            clone(estimator), X, _shuffle(y, groups, random_state),
-            groups, cv, scorer, fit_params=fit_params)
-        for _ in range(n_permutations))
+            clone(estimator),
+            X,
+            _shuffle(y, groups, random_state),
+            cv,
+            scorer,
+            split_params=routed_params.splitter.split,
+            fit_params=routed_params.estimator.fit,
+            score_params=routed_params.scorer.score,
+        )
+        for _ in range(n_permutations)
+    )
     permutation_scores = np.array(permutation_scores)
     pvalue = (np.sum(permutation_scores >= score) + 1.0) / (n_permutations + 1)
     return score, permutation_scores, pvalue
 
 
-def _permutation_test_score(estimator, X, y, groups, cv, scorer,
-                            fit_params):
+def _permutation_test_score(
+    estimator, X, y, cv, scorer, split_params, fit_params, score_params
+):
     """Auxiliary function for permutation_test_score"""
     # Adjust length of sample weights
     fit_params = fit_params if fit_params is not None else {}
+    score_params = score_params if score_params is not None else {}
+
     avg_score = []
-    for train, test in cv.split(X, y, groups):
+    for train, test in cv.split(X, y, **split_params):
         X_train, y_train = _safe_split(estimator, X, y, train)
         X_test, y_test = _safe_split(estimator, X, y, test, train)
-        fit_params = _check_fit_params(X, fit_params, train)
-        estimator.fit(X_train, y_train, **fit_params)
-        avg_score.append(scorer(estimator, X_test, y_test))
+        fit_params_train = _check_method_params(X, params=fit_params, indices=train)
+        score_params_test = _check_method_params(X, params=score_params, indices=test)
+        estimator.fit(X_train, y_train, **fit_params_train)
+        avg_score.append(scorer(estimator, X_test, y_test, **score_params_test))
     return np.mean(avg_score)
 
 
@@ -1215,18 +1725,53 @@ def _shuffle(y, groups, random_state):
     else:
         indices = np.arange(len(groups))
         for group in np.unique(groups):
-            this_mask = (groups == group)
+            this_mask = groups == group
             indices[this_mask] = random_state.permutation(indices[this_mask])
     return _safe_indexing(y, indices)
 
 
-@_deprecate_positional_args
-def learning_curve(estimator, X, y, *, groups=None,
-                   train_sizes=np.linspace(0.1, 1.0, 5), cv=None,
-                   scoring=None, exploit_incremental_learning=False,
-                   n_jobs=None, pre_dispatch="all", verbose=0, shuffle=False,
-                   random_state=None, error_score=np.nan, return_times=False,
-                   fit_params=None):
+@validate_params(
+    {
+        "estimator": [HasMethods(["fit"])],
+        "X": ["array-like", "sparse matrix"],
+        "y": ["array-like", None],
+        "groups": ["array-like", None],
+        "train_sizes": ["array-like"],
+        "cv": ["cv_object"],
+        "scoring": [StrOptions(set(get_scorer_names())), callable, None],
+        "exploit_incremental_learning": ["boolean"],
+        "n_jobs": [Integral, None],
+        "pre_dispatch": [Integral, str],
+        "verbose": ["verbose"],
+        "shuffle": ["boolean"],
+        "random_state": ["random_state"],
+        "error_score": [StrOptions({"raise"}), Real],
+        "return_times": ["boolean"],
+        "fit_params": [dict, None],
+        "params": [dict, None],
+    },
+    prefer_skip_nested_validation=False,  # estimator is not validated yet
+)
+def learning_curve(
+    estimator,
+    X,
+    y,
+    *,
+    groups=None,
+    train_sizes=np.linspace(0.1, 1.0, 5),
+    cv=None,
+    scoring=None,
+    exploit_incremental_learning=False,
+    n_jobs=None,
+    pre_dispatch="all",
+    verbose=0,
+    shuffle=False,
+    random_state=None,
+    error_score=np.nan,
+    return_times=False,
+    fit_params=None,
+    params=None,
+):
     """Learning curve.
 
     Determines cross-validated training and test scores for different training
@@ -1242,22 +1787,31 @@ def learning_curve(estimator, X, y, *, groups=None,
 
     Parameters
     ----------
-    estimator : object type that implements the "fit" and "predict" methods
-        An object of that type which is cloned for each validation.
+    estimator : object type that implements the "fit" method
+        An object of that type which is cloned for each validation. It must
+        also implement "predict" unless `scoring` is a callable that doesn't
+        rely on "predict" to compute a score.
 
-    X : array-like of shape (n_samples, n_features)
-        Training vector, where n_samples is the number of samples and
-        n_features is the number of features.
+    X : {array-like, sparse matrix} of shape (n_samples, n_features)
+        Training vector, where `n_samples` is the number of samples and
+        `n_features` is the number of features.
 
-    y : array-like of shape (n_samples,) or (n_samples, n_outputs)
+    y : array-like of shape (n_samples,) or (n_samples, n_outputs) or None
         Target relative to X for classification or regression;
         None for unsupervised learning.
 
-    groups : array-like of  shape (n_samples,), default=None
+    groups : array-like of shape (n_samples,), default=None
         Group labels for the samples used while splitting the dataset into
         train/test set. Only used in conjunction with a "Group" :term:`cv`
         instance (e.g., :class:`GroupKFold`).
 
+        .. versionchanged:: 1.6
+            ``groups`` can only be passed if metadata routing is not enabled
+            via ``sklearn.set_config(enable_metadata_routing=True)``. When routing
+            is enabled, pass ``groups`` alongside other metadata via the ``params``
+            argument instead. E.g.:
+            ``learning_curve(..., params={'groups': groups})``.
+
     train_sizes : array-like of shape (n_ticks,), \
             default=np.linspace(0.1, 1.0, 5)
         Relative or absolute numbers of training examples that will be used to
@@ -1265,7 +1819,7 @@ def learning_curve(estimator, X, y, *, groups=None,
         fraction of the maximum size of the training set (that is determined
         by the selected validation method), i.e. it has to be within (0, 1].
         Otherwise it is interpreted as absolute sizes of the training sets.
-        Note that for classification the number of samples usually have to
+        Note that for classification the number of samples usually has to
         be big enough to contain at least one sample from each class.
 
     cv : int, cross-validation generator or an iterable, default=None
@@ -1279,7 +1833,8 @@ def learning_curve(estimator, X, y, *, groups=None,
 
         For int/None inputs, if the estimator is a classifier and ``y`` is
         either binary or multiclass, :class:`StratifiedKFold` is used. In all
-        other cases, :class:`KFold` is used.
+        other cases, :class:`KFold` is used. These splitters are instantiated
+        with `shuffle=False` so the splits will be the same across calls.
 
         Refer :ref:`User Guide ` for the various
         cross-validation strategies that can be used here.
@@ -1288,9 +1843,13 @@ def learning_curve(estimator, X, y, *, groups=None,
             ``cv`` default value if None changed from 3-fold to 5-fold.
 
     scoring : str or callable, default=None
-        A str (see model evaluation documentation) or
-        a scorer callable object / function with signature
-        ``scorer(estimator, X, y)``.
+        Scoring method to use to evaluate the training and test sets.
+
+        - str: see :ref:`scoring_string_names` for options.
+        - callable: a scorer callable object (e.g., function) with signature
+          ``scorer(estimator, X, y)``. See :ref:`scoring_callable` for details.
+        - `None`: the `estimator`'s
+          :ref:`default evaluation criterion ` is used.
 
     exploit_incremental_learning : bool, default=False
         If the estimator supports incremental learning, this will be
@@ -1333,7 +1892,21 @@ def learning_curve(estimator, X, y, *, groups=None,
     fit_params : dict, default=None
         Parameters to pass to the fit method of the estimator.
 
-        .. versionadded:: 0.24
+        .. deprecated:: 1.6
+            This parameter is deprecated and will be removed in version 1.8. Use
+            ``params`` instead.
+
+    params : dict, default=None
+        Parameters to pass to the `fit` method of the estimator and to the scorer.
+
+        - If `enable_metadata_routing=False` (default): Parameters directly passed to
+          the `fit` method of the estimator.
+
+        - If `enable_metadata_routing=True`: Parameters safely routed to the `fit`
+          method of the estimator. See :ref:`Metadata Routing User Guide
+          ` for more details.
+
+        .. versionadded:: 1.6
 
     Returns
     -------
@@ -1356,34 +1929,104 @@ def learning_curve(estimator, X, y, *, groups=None,
         Times spent for scoring in seconds. Only present if ``return_times``
         is True.
 
-    Notes
-    -----
-    See :ref:`examples/model_selection/plot_learning_curve.py
-    `
+    See Also
+    --------
+    LearningCurveDisplay.from_estimator : Plot a learning curve using an
+        estimator and data.
+
+    Examples
+    --------
+    >>> from sklearn.datasets import make_classification
+    >>> from sklearn.tree import DecisionTreeClassifier
+    >>> from sklearn.model_selection import learning_curve
+    >>> X, y = make_classification(n_samples=100, n_features=10, random_state=42)
+    >>> tree = DecisionTreeClassifier(max_depth=4, random_state=42)
+    >>> train_size_abs, train_scores, test_scores = learning_curve(
+    ...     tree, X, y, train_sizes=[0.3, 0.6, 0.9]
+    ... )
+    >>> for train_size, cv_train_scores, cv_test_scores in zip(
+    ...     train_size_abs, train_scores, test_scores
+    ... ):
+    ...     print(f"{train_size} samples were used to train the model")
+    ...     print(f"The average train accuracy is {cv_train_scores.mean():.2f}")
+    ...     print(f"The average test accuracy is {cv_test_scores.mean():.2f}")
+    24 samples were used to train the model
+    The average train accuracy is 1.00
+    The average test accuracy is 0.85
+    48 samples were used to train the model
+    The average train accuracy is 1.00
+    The average test accuracy is 0.90
+    72 samples were used to train the model
+    The average train accuracy is 1.00
+    The average test accuracy is 0.93
     """
     if exploit_incremental_learning and not hasattr(estimator, "partial_fit"):
-        raise ValueError("An estimator must support the partial_fit interface "
-                         "to exploit incremental learning")
+        raise ValueError(
+            "An estimator must support the partial_fit interface "
+            "to exploit incremental learning"
+        )
+
+    params = _check_params_groups_deprecation(fit_params, params, groups, "1.8")
+
     X, y, groups = indexable(X, y, groups)
 
     cv = check_cv(cv, y, classifier=is_classifier(estimator))
-    # Store it as list as we will be iterating over the list multiple times
-    cv_iter = list(cv.split(X, y, groups))
 
     scorer = check_scoring(estimator, scoring=scoring)
 
+    if _routing_enabled():
+        router = (
+            MetadataRouter(owner="learning_curve")
+            .add(
+                estimator=estimator,
+                # TODO(SLEP6): also pass metadata to the predict method for
+                # scoring?
+                method_mapping=MethodMapping()
+                .add(caller="fit", callee="fit")
+                .add(caller="fit", callee="partial_fit"),
+            )
+            .add(
+                splitter=cv,
+                method_mapping=MethodMapping().add(caller="fit", callee="split"),
+            )
+            .add(
+                scorer=scorer,
+                method_mapping=MethodMapping().add(caller="fit", callee="score"),
+            )
+        )
+
+        try:
+            routed_params = process_routing(router, "fit", **params)
+        except UnsetMetadataPassedError as e:
+            # The default exception would mention `fit` since in the above
+            # `process_routing` code, we pass `fit` as the caller. However,
+            # the user is not calling `fit` directly, so we change the message
+            # to make it more suitable for this case.
+            raise UnsetMetadataPassedError(
+                message=str(e).replace("learning_curve.fit", "learning_curve"),
+                unrequested_params=e.unrequested_params,
+                routed_params=e.routed_params,
+            )
+
+    else:
+        routed_params = Bunch()
+        routed_params.estimator = Bunch(fit=params, partial_fit=params)
+        routed_params.splitter = Bunch(split={"groups": groups})
+        routed_params.scorer = Bunch(score={})
+
+    # Store cv as list as we will be iterating over the list multiple times
+    cv_iter = list(cv.split(X, y, **routed_params.splitter.split))
+
     n_max_training_samples = len(cv_iter[0][0])
     # Because the lengths of folds can be significantly different, it is
     # not guaranteed that we use all of the available training data when we
     # use the first 'n_max_training_samples' samples.
-    train_sizes_abs = _translate_train_sizes(train_sizes,
-                                             n_max_training_samples)
+    train_sizes_abs = _translate_train_sizes(train_sizes, n_max_training_samples)
     n_unique_ticks = train_sizes_abs.shape[0]
     if verbose > 0:
         print("[learning_curve] Training set sizes: " + str(train_sizes_abs))
 
-    parallel = Parallel(n_jobs=n_jobs, pre_dispatch=pre_dispatch,
-                        verbose=verbose)
+    parallel = Parallel(n_jobs=n_jobs, pre_dispatch=pre_dispatch, verbose=verbose)
 
     if shuffle:
         rng = check_random_state(random_state)
@@ -1391,10 +2034,21 @@ def learning_curve(estimator, X, y, *, groups=None,
 
     if exploit_incremental_learning:
         classes = np.unique(y) if is_classifier(estimator) else None
-        out = parallel(delayed(_incremental_fit_estimator)(
-            clone(estimator), X, y, classes, train, test, train_sizes_abs,
-            scorer, verbose, return_times, error_score=error_score,
-            fit_params=fit_params)
+        out = parallel(
+            delayed(_incremental_fit_estimator)(
+                clone(estimator),
+                X,
+                y,
+                classes,
+                train,
+                test,
+                train_sizes_abs,
+                scorer,
+                return_times,
+                error_score=error_score,
+                fit_params=routed_params.estimator.partial_fit,
+                score_params=routed_params.scorer.score,
+            )
             for train, test in cv_iter
         )
         out = np.asarray(out).transpose((2, 1, 0))
@@ -1404,12 +2058,25 @@ def learning_curve(estimator, X, y, *, groups=None,
             for n_train_samples in train_sizes_abs:
                 train_test_proportions.append((train[:n_train_samples], test))
 
-        results = parallel(delayed(_fit_and_score)(
-            clone(estimator), X, y, scorer, train, test, verbose,
-            parameters=None, fit_params=fit_params, return_train_score=True,
-            error_score=error_score, return_times=return_times)
+        results = parallel(
+            delayed(_fit_and_score)(
+                clone(estimator),
+                X,
+                y,
+                scorer=scorer,
+                train=train,
+                test=test,
+                verbose=verbose,
+                parameters=None,
+                fit_params=routed_params.estimator.fit,
+                score_params=routed_params.scorer.score,
+                return_train_score=True,
+                error_score=error_score,
+                return_times=return_times,
+            )
             for train, test in train_test_proportions
         )
+        _warn_or_raise_about_fit_failures(results, error_score)
         results = _aggregate_score_dicts(results)
         train_scores = results["train_scores"].reshape(-1, n_unique_ticks).T
         test_scores = results["test_scores"].reshape(-1, n_unique_ticks).T
@@ -1458,81 +2125,154 @@ def _translate_train_sizes(train_sizes, n_max_training_samples):
     n_max_required_samples = np.max(train_sizes_abs)
     if np.issubdtype(train_sizes_abs.dtype, np.floating):
         if n_min_required_samples <= 0.0 or n_max_required_samples > 1.0:
-            raise ValueError("train_sizes has been interpreted as fractions "
-                             "of the maximum number of training samples and "
-                             "must be within (0, 1], but is within [%f, %f]."
-                             % (n_min_required_samples,
-                                n_max_required_samples))
+            raise ValueError(
+                "train_sizes has been interpreted as fractions "
+                "of the maximum number of training samples and "
+                "must be within (0, 1], but is within [%f, %f]."
+                % (n_min_required_samples, n_max_required_samples)
+            )
         train_sizes_abs = (train_sizes_abs * n_max_training_samples).astype(
-                             dtype=int, copy=False)
-        train_sizes_abs = np.clip(train_sizes_abs, 1,
-                                  n_max_training_samples)
+            dtype=int, copy=False
+        )
+        train_sizes_abs = np.clip(train_sizes_abs, 1, n_max_training_samples)
     else:
-        if (n_min_required_samples <= 0 or
-                n_max_required_samples > n_max_training_samples):
-            raise ValueError("train_sizes has been interpreted as absolute "
-                             "numbers of training samples and must be within "
-                             "(0, %d], but is within [%d, %d]."
-                             % (n_max_training_samples,
-                                n_min_required_samples,
-                                n_max_required_samples))
+        if (
+            n_min_required_samples <= 0
+            or n_max_required_samples > n_max_training_samples
+        ):
+            raise ValueError(
+                "train_sizes has been interpreted as absolute "
+                "numbers of training samples and must be within "
+                "(0, %d], but is within [%d, %d]."
+                % (
+                    n_max_training_samples,
+                    n_min_required_samples,
+                    n_max_required_samples,
+                )
+            )
 
     train_sizes_abs = np.unique(train_sizes_abs)
     if n_ticks > train_sizes_abs.shape[0]:
-        warnings.warn("Removed duplicate entries from 'train_sizes'. Number "
-                      "of ticks will be less than the size of "
-                      "'train_sizes': %d instead of %d."
-                      % (train_sizes_abs.shape[0], n_ticks), RuntimeWarning)
+        warnings.warn(
+            "Removed duplicate entries from 'train_sizes'. Number "
+            "of ticks will be less than the size of "
+            "'train_sizes': %d instead of %d." % (train_sizes_abs.shape[0], n_ticks),
+            RuntimeWarning,
+        )
 
     return train_sizes_abs
 
 
-def _incremental_fit_estimator(estimator, X, y, classes, train, test,
-                               train_sizes, scorer, verbose,
-                               return_times, error_score, fit_params):
+def _incremental_fit_estimator(
+    estimator,
+    X,
+    y,
+    classes,
+    train,
+    test,
+    train_sizes,
+    scorer,
+    return_times,
+    error_score,
+    fit_params,
+    score_params,
+):
     """Train estimator on training subsets incrementally and compute scores."""
     train_scores, test_scores, fit_times, score_times = [], [], [], []
     partitions = zip(train_sizes, np.split(train, train_sizes)[:-1])
     if fit_params is None:
         fit_params = {}
+    if classes is None:
+        partial_fit_func = partial(estimator.partial_fit, **fit_params)
+    else:
+        partial_fit_func = partial(estimator.partial_fit, classes=classes, **fit_params)
+    score_params = score_params if score_params is not None else {}
+    score_params_train = _check_method_params(X, params=score_params, indices=train)
+    score_params_test = _check_method_params(X, params=score_params, indices=test)
+
     for n_train_samples, partial_train in partitions:
         train_subset = train[:n_train_samples]
         X_train, y_train = _safe_split(estimator, X, y, train_subset)
-        X_partial_train, y_partial_train = _safe_split(estimator, X, y,
-                                                       partial_train)
+        X_partial_train, y_partial_train = _safe_split(estimator, X, y, partial_train)
         X_test, y_test = _safe_split(estimator, X, y, test, train_subset)
         start_fit = time.time()
         if y_partial_train is None:
-            estimator.partial_fit(X_partial_train, classes=classes,
-                                  **fit_params)
+            partial_fit_func(X_partial_train)
         else:
-            estimator.partial_fit(X_partial_train, y_partial_train,
-                                  classes=classes, **fit_params)
+            partial_fit_func(X_partial_train, y_partial_train)
         fit_time = time.time() - start_fit
         fit_times.append(fit_time)
 
         start_score = time.time()
 
         test_scores.append(
-            _score(estimator, X_test, y_test, scorer, error_score)
+            _score(
+                estimator,
+                X_test,
+                y_test,
+                scorer,
+                score_params=score_params_test,
+                error_score=error_score,
+            )
         )
         train_scores.append(
-            _score(estimator, X_train, y_train, scorer, error_score)
+            _score(
+                estimator,
+                X_train,
+                y_train,
+                scorer,
+                score_params=score_params_train,
+                error_score=error_score,
+            )
         )
-
         score_time = time.time() - start_score
         score_times.append(score_time)
 
-    ret = ((train_scores, test_scores, fit_times, score_times)
-           if return_times else (train_scores, test_scores))
+    ret = (
+        (train_scores, test_scores, fit_times, score_times)
+        if return_times
+        else (train_scores, test_scores)
+    )
 
     return np.array(ret).T
 
 
-@_deprecate_positional_args
-def validation_curve(estimator, X, y, *, param_name, param_range, groups=None,
-                     cv=None, scoring=None, n_jobs=None, pre_dispatch="all",
-                     verbose=0, error_score=np.nan, fit_params=None):
+@validate_params(
+    {
+        "estimator": [HasMethods(["fit"])],
+        "X": ["array-like", "sparse matrix"],
+        "y": ["array-like", None],
+        "param_name": [str],
+        "param_range": ["array-like"],
+        "groups": ["array-like", None],
+        "cv": ["cv_object"],
+        "scoring": [StrOptions(set(get_scorer_names())), callable, None],
+        "n_jobs": [Integral, None],
+        "pre_dispatch": [Integral, str],
+        "verbose": ["verbose"],
+        "error_score": [StrOptions({"raise"}), Real],
+        "fit_params": [dict, None],
+        "params": [dict, None],
+    },
+    prefer_skip_nested_validation=False,  # estimator is not validated yet
+)
+def validation_curve(
+    estimator,
+    X,
+    y,
+    *,
+    param_name,
+    param_range,
+    groups=None,
+    cv=None,
+    scoring=None,
+    n_jobs=None,
+    pre_dispatch="all",
+    verbose=0,
+    error_score=np.nan,
+    fit_params=None,
+    params=None,
+):
     """Validation curve.
 
     Determine training and test scores for varying parameter values.
@@ -1546,12 +2286,14 @@ def validation_curve(estimator, X, y, *, param_name, param_range, groups=None,
 
     Parameters
     ----------
-    estimator : object type that implements the "fit" and "predict" methods
-        An object of that type which is cloned for each validation.
+    estimator : object type that implements the "fit" method
+        An object of that type which is cloned for each validation. It must
+        also implement "predict" unless `scoring` is a callable that doesn't
+        rely on "predict" to compute a score.
 
-    X : array-like of shape (n_samples, n_features)
-        Training vector, where n_samples is the number of samples and
-        n_features is the number of features.
+    X : {array-like, sparse matrix} of shape (n_samples, n_features)
+        Training vector, where `n_samples` is the number of samples and
+        `n_features` is the number of features.
 
     y : array-like of shape (n_samples,) or (n_samples, n_outputs) or None
         Target relative to X for classification or regression;
@@ -1568,6 +2310,13 @@ def validation_curve(estimator, X, y, *, param_name, param_range, groups=None,
         train/test set. Only used in conjunction with a "Group" :term:`cv`
         instance (e.g., :class:`GroupKFold`).
 
+        .. versionchanged:: 1.6
+            ``groups`` can only be passed if metadata routing is not enabled
+            via ``sklearn.set_config(enable_metadata_routing=True)``. When routing
+            is enabled, pass ``groups`` alongside other metadata via the ``params``
+            argument instead. E.g.:
+            ``validation_curve(..., params={'groups': groups})``.
+
     cv : int, cross-validation generator or an iterable, default=None
         Determines the cross-validation splitting strategy.
         Possible inputs for cv are:
@@ -1579,7 +2328,8 @@ def validation_curve(estimator, X, y, *, param_name, param_range, groups=None,
 
         For int/None inputs, if the estimator is a classifier and ``y`` is
         either binary or multiclass, :class:`StratifiedKFold` is used. In all
-        other cases, :class:`KFold` is used.
+        other cases, :class:`KFold` is used. These splitters are instantiated
+        with `shuffle=False` so the splits will be the same across calls.
 
         Refer :ref:`User Guide ` for the various
         cross-validation strategies that can be used here.
@@ -1588,9 +2338,13 @@ def validation_curve(estimator, X, y, *, param_name, param_range, groups=None,
             ``cv`` default value if None changed from 3-fold to 5-fold.
 
     scoring : str or callable, default=None
-        A str (see model evaluation documentation) or
-        a scorer callable object / function with signature
-        ``scorer(estimator, X, y)``.
+        Scoring method to use to evaluate the training and test sets.
+
+        - str: see :ref:`scoring_string_names` for options.
+        - callable: a scorer callable object (e.g., function) with signature
+          ``scorer(estimator, X, y)``. See :ref:`scoring_callable` for details.
+        - `None`: the `estimator`'s
+          :ref:`default evaluation criterion ` is used.
 
     n_jobs : int, default=None
         Number of jobs to run in parallel. Training the estimator and computing
@@ -1608,11 +2362,6 @@ def validation_curve(estimator, X, y, *, param_name, param_range, groups=None,
     verbose : int, default=0
         Controls the verbosity: the higher, the more messages.
 
-    fit_params : dict, default=None
-        Parameters to pass to the fit method of the estimator.
-
-        .. versionadded:: 0.24
-
     error_score : 'raise' or numeric, default=np.nan
         Value to assign to the score if an error occurs in estimator fitting.
         If set to 'raise', the error is raised.
@@ -1620,6 +2369,25 @@ def validation_curve(estimator, X, y, *, param_name, param_range, groups=None,
 
         .. versionadded:: 0.20
 
+    fit_params : dict, default=None
+        Parameters to pass to the fit method of the estimator.
+
+        .. deprecated:: 1.6
+            This parameter is deprecated and will be removed in version 1.8. Use
+            ``params`` instead.
+
+    params : dict, default=None
+        Parameters to pass to the estimator, scorer and cross-validation object.
+
+        - If `enable_metadata_routing=False` (default): Parameters directly passed to
+          the `fit` method of the estimator.
+
+        - If `enable_metadata_routing=True`: Parameters safely routed to the `fit`
+          method of the estimator, to the scorer and to the cross-validation object.
+          See :ref:`Metadata Routing User Guide ` for more details.
+
+        .. versionadded:: 1.6
+
     Returns
     -------
     train_scores : array of shape (n_ticks, n_cv_folds)
@@ -1628,25 +2396,94 @@ def validation_curve(estimator, X, y, *, param_name, param_range, groups=None,
     test_scores : array of shape (n_ticks, n_cv_folds)
         Scores on test set.
 
+    See Also
+    --------
+    ValidationCurveDisplay.from_estimator : Plot the validation curve
+        given an estimator, the data, and the parameter to vary.
+
     Notes
     -----
-    See :ref:`sphx_glr_auto_examples_model_selection_plot_validation_curve.py`
+    See :ref:`sphx_glr_auto_examples_model_selection_plot_train_error_vs_test_error.py`
 
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from sklearn.datasets import make_classification
+    >>> from sklearn.model_selection import validation_curve
+    >>> from sklearn.linear_model import LogisticRegression
+    >>> X, y = make_classification(n_samples=1_000, random_state=0)
+    >>> logistic_regression = LogisticRegression()
+    >>> param_name, param_range = "C", np.logspace(-8, 3, 10)
+    >>> train_scores, test_scores = validation_curve(
+    ...     logistic_regression, X, y, param_name=param_name, param_range=param_range
+    ... )
+    >>> print(f"The average train accuracy is {train_scores.mean():.2f}")
+    The average train accuracy is 0.81
+    >>> print(f"The average test accuracy is {test_scores.mean():.2f}")
+    The average test accuracy is 0.81
     """
+    params = _check_params_groups_deprecation(fit_params, params, groups, "1.8")
     X, y, groups = indexable(X, y, groups)
 
     cv = check_cv(cv, y, classifier=is_classifier(estimator))
     scorer = check_scoring(estimator, scoring=scoring)
 
-    parallel = Parallel(n_jobs=n_jobs, pre_dispatch=pre_dispatch,
-                        verbose=verbose)
-    results = parallel(delayed(_fit_and_score)(
-        clone(estimator), X, y, scorer, train, test, verbose,
-        parameters={param_name: v}, fit_params=fit_params,
-        return_train_score=True, error_score=error_score)
+    if _routing_enabled():
+        router = (
+            MetadataRouter(owner="validation_curve")
+            .add(
+                estimator=estimator,
+                method_mapping=MethodMapping().add(caller="fit", callee="fit"),
+            )
+            .add(
+                splitter=cv,
+                method_mapping=MethodMapping().add(caller="fit", callee="split"),
+            )
+            .add(
+                scorer=scorer,
+                method_mapping=MethodMapping().add(caller="fit", callee="score"),
+            )
+        )
+
+        try:
+            routed_params = process_routing(router, "fit", **params)
+        except UnsetMetadataPassedError as e:
+            # The default exception would mention `fit` since in the above
+            # `process_routing` code, we pass `fit` as the caller. However,
+            # the user is not calling `fit` directly, so we change the message
+            # to make it more suitable for this case.
+            raise UnsetMetadataPassedError(
+                message=str(e).replace("validation_curve.fit", "validation_curve"),
+                unrequested_params=e.unrequested_params,
+                routed_params=e.routed_params,
+            )
 
+    else:
+        routed_params = Bunch()
+        routed_params.estimator = Bunch(fit=params)
+        routed_params.splitter = Bunch(split={"groups": groups})
+        routed_params.scorer = Bunch(score={})
+
+    parallel = Parallel(n_jobs=n_jobs, pre_dispatch=pre_dispatch, verbose=verbose)
+    results = parallel(
+        delayed(_fit_and_score)(
+            clone(estimator),
+            X,
+            y,
+            scorer=scorer,
+            train=train,
+            test=test,
+            verbose=verbose,
+            parameters={param_name: v},
+            fit_params=routed_params.estimator.fit,
+            score_params=routed_params.scorer.score,
+            return_train_score=True,
+            error_score=error_score,
+        )
         # NOTE do not change order of iteration to allow one time cv splitters
-        for train, test in cv.split(X, y, groups) for v in param_range)
+        for train, test in cv.split(X, y, **routed_params.splitter.split)
+        for v in param_range
+    )
     n_params = len(param_range)
 
     results = _aggregate_score_dicts(results)
@@ -1680,8 +2517,10 @@ def _aggregate_score_dicts(scores):
      'b': array([10, 2, 3, 10])}
     """
     return {
-        key: np.asarray([score[key] for score in scores])
-        if isinstance(scores[0][key], numbers.Number)
-        else [score[key] for score in scores]
+        key: (
+            np.asarray([score[key] for score in scores])
+            if isinstance(scores[0][key], numbers.Number)
+            else [score[key] for score in scores]
+        )
         for key in scores[0]
     }
diff --git a/sklearn/model_selection/tests/common.py b/sklearn/model_selection/tests/common.py
index 13549eef377b7..54a993db76933 100644
--- a/sklearn/model_selection/tests/common.py
+++ b/sklearn/model_selection/tests/common.py
@@ -9,6 +9,7 @@
 
 class OneTimeSplitter:
     """A wrapper to make KFold single entry cv iterator"""
+
     def __init__(self, n_splits=4, n_samples=99):
         self.n_splits = n_splits
         self.n_samples = n_samples
diff --git a/sklearn/model_selection/tests/test_classification_threshold.py b/sklearn/model_selection/tests/test_classification_threshold.py
new file mode 100644
index 0000000000000..1ba4dcea36974
--- /dev/null
+++ b/sklearn/model_selection/tests/test_classification_threshold.py
@@ -0,0 +1,618 @@
+import numpy as np
+import pytest
+
+from sklearn import config_context
+from sklearn.base import clone
+from sklearn.datasets import (
+    load_breast_cancer,
+    load_iris,
+    make_classification,
+    make_multilabel_classification,
+)
+from sklearn.dummy import DummyClassifier
+from sklearn.ensemble import GradientBoostingClassifier
+from sklearn.exceptions import NotFittedError
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import (
+    balanced_accuracy_score,
+    f1_score,
+    fbeta_score,
+    make_scorer,
+)
+from sklearn.metrics._scorer import _CurveScorer
+from sklearn.model_selection import (
+    FixedThresholdClassifier,
+    StratifiedShuffleSplit,
+    TunedThresholdClassifierCV,
+)
+from sklearn.model_selection._classification_threshold import (
+    _fit_and_score_over_thresholds,
+)
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import StandardScaler
+from sklearn.svm import SVC
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.utils._mocking import CheckingClassifier
+from sklearn.utils._testing import (
+    _convert_container,
+    assert_allclose,
+    assert_array_equal,
+)
+
+
+def test_fit_and_score_over_thresholds_curve_scorers():
+    """Check that `_fit_and_score_over_thresholds` returns thresholds in ascending order
+    for the different accepted curve scorers."""
+    X, y = make_classification(n_samples=100, random_state=0)
+    train_idx, val_idx = np.arange(50), np.arange(50, 100)
+    classifier = LogisticRegression()
+
+    curve_scorer = _CurveScorer(
+        score_func=balanced_accuracy_score,
+        sign=1,
+        response_method="predict_proba",
+        thresholds=10,
+        kwargs={},
+    )
+    scores, thresholds = _fit_and_score_over_thresholds(
+        classifier,
+        X,
+        y,
+        fit_params={},
+        train_idx=train_idx,
+        val_idx=val_idx,
+        curve_scorer=curve_scorer,
+        score_params={},
+    )
+
+    assert np.all(thresholds[:-1] <= thresholds[1:])
+    assert isinstance(scores, np.ndarray)
+    assert np.logical_and(scores >= 0, scores <= 1).all()
+
+
+def test_fit_and_score_over_thresholds_prefit():
+    """Check the behaviour with a prefit classifier."""
+    X, y = make_classification(n_samples=100, random_state=0)
+
+    # `train_idx is None` to indicate that the classifier is prefit
+    train_idx, val_idx = None, np.arange(50, 100)
+    classifier = DecisionTreeClassifier(random_state=0).fit(X, y)
+    # make sure that the classifier memorized the full dataset such that
+    # we get perfect predictions and thus match the expected score
+    assert classifier.score(X[val_idx], y[val_idx]) == pytest.approx(1.0)
+
+    curve_scorer = _CurveScorer(
+        score_func=balanced_accuracy_score,
+        sign=1,
+        response_method="predict_proba",
+        thresholds=2,
+        kwargs={},
+    )
+    scores, thresholds = _fit_and_score_over_thresholds(
+        classifier,
+        X,
+        y,
+        fit_params={},
+        train_idx=train_idx,
+        val_idx=val_idx,
+        curve_scorer=curve_scorer,
+        score_params={},
+    )
+    assert np.all(thresholds[:-1] <= thresholds[1:])
+    assert_allclose(scores, [0.5, 1.0])
+
+
+@config_context(enable_metadata_routing=True)
+def test_fit_and_score_over_thresholds_sample_weight():
+    """Check that we dispatch the sample-weight to fit and score the classifier."""
+    X, y = load_iris(return_X_y=True)
+    X, y = X[:100], y[:100]  # only 2 classes
+
+    # create a dataset and repeat twice the sample of class #0
+    X_repeated, y_repeated = np.vstack([X, X[y == 0]]), np.hstack([y, y[y == 0]])
+    # create a sample weight vector that is equivalent to the repeated dataset
+    sample_weight = np.ones_like(y)
+    sample_weight[:50] *= 2
+
+    classifier = LogisticRegression()
+    train_repeated_idx = np.arange(X_repeated.shape[0])
+    val_repeated_idx = np.arange(X_repeated.shape[0])
+    curve_scorer = _CurveScorer(
+        score_func=balanced_accuracy_score,
+        sign=1,
+        response_method="predict_proba",
+        thresholds=10,
+        kwargs={},
+    )
+    scores_repeated, thresholds_repeated = _fit_and_score_over_thresholds(
+        classifier,
+        X_repeated,
+        y_repeated,
+        fit_params={},
+        train_idx=train_repeated_idx,
+        val_idx=val_repeated_idx,
+        curve_scorer=curve_scorer,
+        score_params={},
+    )
+
+    train_idx, val_idx = np.arange(X.shape[0]), np.arange(X.shape[0])
+    scores, thresholds = _fit_and_score_over_thresholds(
+        classifier.set_fit_request(sample_weight=True),
+        X,
+        y,
+        fit_params={"sample_weight": sample_weight},
+        train_idx=train_idx,
+        val_idx=val_idx,
+        curve_scorer=curve_scorer.set_score_request(sample_weight=True),
+        score_params={"sample_weight": sample_weight},
+    )
+
+    assert_allclose(thresholds_repeated, thresholds)
+    assert_allclose(scores_repeated, scores)
+
+
+@pytest.mark.parametrize("fit_params_type", ["list", "array"])
+@config_context(enable_metadata_routing=True)
+def test_fit_and_score_over_thresholds_fit_params(fit_params_type):
+    """Check that we pass `fit_params` to the classifier when calling `fit`."""
+    X, y = make_classification(n_samples=100, random_state=0)
+    fit_params = {
+        "a": _convert_container(y, fit_params_type),
+        "b": _convert_container(y, fit_params_type),
+    }
+
+    classifier = CheckingClassifier(expected_fit_params=["a", "b"], random_state=0)
+    classifier.set_fit_request(a=True, b=True)
+    train_idx, val_idx = np.arange(50), np.arange(50, 100)
+
+    curve_scorer = _CurveScorer(
+        score_func=balanced_accuracy_score,
+        sign=1,
+        response_method="predict_proba",
+        thresholds=10,
+        kwargs={},
+    )
+    _fit_and_score_over_thresholds(
+        classifier,
+        X,
+        y,
+        fit_params=fit_params,
+        train_idx=train_idx,
+        val_idx=val_idx,
+        curve_scorer=curve_scorer,
+        score_params={},
+    )
+
+
+@pytest.mark.parametrize(
+    "data",
+    [
+        make_classification(n_classes=3, n_clusters_per_class=1, random_state=0),
+        make_multilabel_classification(random_state=0),
+    ],
+)
+def test_tuned_threshold_classifier_no_binary(data):
+    """Check that we raise an informative error message for non-binary problem."""
+    err_msg = "Only binary classification is supported."
+    with pytest.raises(ValueError, match=err_msg):
+        TunedThresholdClassifierCV(LogisticRegression()).fit(*data)
+
+
+@pytest.mark.parametrize(
+    "params, err_type, err_msg",
+    [
+        (
+            {"cv": "prefit", "refit": True},
+            ValueError,
+            "When cv='prefit', refit cannot be True.",
+        ),
+        (
+            {"cv": 10, "refit": False},
+            ValueError,
+            "When cv has several folds, refit cannot be False.",
+        ),
+        (
+            {"cv": "prefit", "refit": False},
+            NotFittedError,
+            "`estimator` must be fitted.",
+        ),
+    ],
+)
+def test_tuned_threshold_classifier_conflict_cv_refit(params, err_type, err_msg):
+    """Check that we raise an informative error message when `cv` and `refit`
+    cannot be used together.
+    """
+    X, y = make_classification(n_samples=100, random_state=0)
+    with pytest.raises(err_type, match=err_msg):
+        TunedThresholdClassifierCV(LogisticRegression(), **params).fit(X, y)
+
+
+@pytest.mark.parametrize(
+    "estimator",
+    [LogisticRegression(), SVC(), GradientBoostingClassifier(n_estimators=4)],
+)
+@pytest.mark.parametrize(
+    "response_method", ["predict_proba", "predict_log_proba", "decision_function"]
+)
+@pytest.mark.parametrize(
+    "ThresholdClassifier", [FixedThresholdClassifier, TunedThresholdClassifierCV]
+)
+def test_threshold_classifier_estimator_response_methods(
+    ThresholdClassifier, estimator, response_method
+):
+    """Check that `TunedThresholdClassifierCV` exposes the same response methods as the
+    underlying estimator.
+    """
+    X, y = make_classification(n_samples=100, random_state=0)
+
+    model = ThresholdClassifier(estimator=estimator)
+    assert hasattr(model, response_method) == hasattr(estimator, response_method)
+
+    model.fit(X, y)
+    assert hasattr(model, response_method) == hasattr(estimator, response_method)
+
+    if hasattr(model, response_method):
+        y_pred_cutoff = getattr(model, response_method)(X)
+        y_pred_underlying_estimator = getattr(model.estimator_, response_method)(X)
+
+        assert_allclose(y_pred_cutoff, y_pred_underlying_estimator)
+
+
+@pytest.mark.parametrize(
+    "response_method", ["auto", "decision_function", "predict_proba"]
+)
+def test_tuned_threshold_classifier_without_constraint_value(response_method):
+    """Check that `TunedThresholdClassifierCV` is optimizing a given objective
+    metric."""
+    X, y = load_breast_cancer(return_X_y=True)
+    # remove feature to degrade performances
+    X = X[:, :5]
+
+    # make the problem completely imbalanced such that the balanced accuracy is low
+    indices_pos = np.flatnonzero(y == 1)
+    indices_pos = indices_pos[: indices_pos.size // 50]
+    indices_neg = np.flatnonzero(y == 0)
+
+    X = np.vstack([X[indices_neg], X[indices_pos]])
+    y = np.hstack([y[indices_neg], y[indices_pos]])
+
+    lr = make_pipeline(StandardScaler(), LogisticRegression()).fit(X, y)
+    thresholds = 100
+    model = TunedThresholdClassifierCV(
+        estimator=lr,
+        scoring="balanced_accuracy",
+        response_method=response_method,
+        thresholds=thresholds,
+        store_cv_results=True,
+    )
+    score_optimized = balanced_accuracy_score(y, model.fit(X, y).predict(X))
+    score_baseline = balanced_accuracy_score(y, lr.predict(X))
+    assert score_optimized > score_baseline
+    assert model.cv_results_["thresholds"].shape == (thresholds,)
+    assert model.cv_results_["scores"].shape == (thresholds,)
+
+
+def test_tuned_threshold_classifier_metric_with_parameter():
+    """Check that we can pass a metric with a parameter in addition check that
+    `f_beta` with `beta=1` is equivalent to `f1` and different from `f_beta` with
+    `beta=2`.
+    """
+    X, y = load_breast_cancer(return_X_y=True)
+    lr = make_pipeline(StandardScaler(), LogisticRegression()).fit(X, y)
+    model_fbeta_1 = TunedThresholdClassifierCV(
+        estimator=lr, scoring=make_scorer(fbeta_score, beta=1)
+    ).fit(X, y)
+    model_fbeta_2 = TunedThresholdClassifierCV(
+        estimator=lr, scoring=make_scorer(fbeta_score, beta=2)
+    ).fit(X, y)
+    model_f1 = TunedThresholdClassifierCV(
+        estimator=lr, scoring=make_scorer(f1_score)
+    ).fit(X, y)
+
+    assert model_fbeta_1.best_threshold_ == pytest.approx(model_f1.best_threshold_)
+    assert model_fbeta_1.best_threshold_ != pytest.approx(model_fbeta_2.best_threshold_)
+
+
+@pytest.mark.parametrize(
+    "response_method", ["auto", "decision_function", "predict_proba"]
+)
+@pytest.mark.parametrize(
+    "metric",
+    [
+        make_scorer(balanced_accuracy_score),
+        make_scorer(f1_score, pos_label="cancer"),
+    ],
+)
+def test_tuned_threshold_classifier_with_string_targets(response_method, metric):
+    """Check that targets represented by str are properly managed.
+    Also, check with several metrics to be sure that `pos_label` is properly
+    dispatched.
+    """
+    X, y = load_breast_cancer(return_X_y=True)
+    # Encode numeric targets by meaningful strings. We purposely designed the class
+    # names such that the `pos_label` is the first alphabetically sorted class and thus
+    # encoded as 0.
+    classes = np.array(["cancer", "healthy"], dtype=object)
+    y = classes[y]
+    model = TunedThresholdClassifierCV(
+        estimator=make_pipeline(StandardScaler(), LogisticRegression()),
+        scoring=metric,
+        response_method=response_method,
+        thresholds=100,
+    ).fit(X, y)
+    assert_array_equal(model.classes_, np.sort(classes))
+    y_pred = model.predict(X)
+    assert_array_equal(np.unique(y_pred), np.sort(classes))
+
+
+@pytest.mark.parametrize("with_sample_weight", [True, False])
+@config_context(enable_metadata_routing=True)
+def test_tuned_threshold_classifier_refit(with_sample_weight, global_random_seed):
+    """Check the behaviour of the `refit` parameter."""
+    rng = np.random.RandomState(global_random_seed)
+    X, y = make_classification(n_samples=100, random_state=0)
+    if with_sample_weight:
+        sample_weight = rng.randn(X.shape[0])
+        sample_weight = np.abs(sample_weight, out=sample_weight)
+    else:
+        sample_weight = None
+
+    # check that `estimator_` if fitted on the full dataset when `refit=True`
+    estimator = LogisticRegression().set_fit_request(sample_weight=True)
+    model = TunedThresholdClassifierCV(estimator, refit=True).fit(
+        X, y, sample_weight=sample_weight
+    )
+
+    assert model.estimator_ is not estimator
+    estimator.fit(X, y, sample_weight=sample_weight)
+    assert_allclose(model.estimator_.coef_, estimator.coef_)
+    assert_allclose(model.estimator_.intercept_, estimator.intercept_)
+
+    # check that `estimator_` was not altered when `refit=False` and `cv="prefit"`
+    estimator = LogisticRegression().set_fit_request(sample_weight=True)
+    estimator.fit(X, y, sample_weight=sample_weight)
+    coef = estimator.coef_.copy()
+    model = TunedThresholdClassifierCV(estimator, cv="prefit", refit=False).fit(
+        X, y, sample_weight=sample_weight
+    )
+
+    assert model.estimator_ is estimator
+    assert_allclose(model.estimator_.coef_, coef)
+
+    # check that we train `estimator_` on the training split of a given cross-validation
+    estimator = LogisticRegression().set_fit_request(sample_weight=True)
+    cv = [
+        (np.arange(50), np.arange(50, 100)),
+    ]  # single split
+    model = TunedThresholdClassifierCV(estimator, cv=cv, refit=False).fit(
+        X, y, sample_weight=sample_weight
+    )
+
+    assert model.estimator_ is not estimator
+    if with_sample_weight:
+        sw_train = sample_weight[cv[0][0]]
+    else:
+        sw_train = None
+    estimator.fit(X[cv[0][0]], y[cv[0][0]], sample_weight=sw_train)
+    assert_allclose(model.estimator_.coef_, estimator.coef_)
+
+
+@pytest.mark.parametrize("fit_params_type", ["list", "array"])
+@config_context(enable_metadata_routing=True)
+def test_tuned_threshold_classifier_fit_params(fit_params_type):
+    """Check that we pass `fit_params` to the classifier when calling `fit`."""
+    X, y = make_classification(n_samples=100, random_state=0)
+    fit_params = {
+        "a": _convert_container(y, fit_params_type),
+        "b": _convert_container(y, fit_params_type),
+    }
+
+    classifier = CheckingClassifier(expected_fit_params=["a", "b"], random_state=0)
+    classifier.set_fit_request(a=True, b=True)
+    model = TunedThresholdClassifierCV(classifier)
+    model.fit(X, y, **fit_params)
+
+
+@config_context(enable_metadata_routing=True)
+def test_tuned_threshold_classifier_cv_zeros_sample_weights_equivalence():
+    """Check that passing removing some sample from the dataset `X` is
+    equivalent to passing a `sample_weight` with a factor 0."""
+    X, y = load_iris(return_X_y=True)
+    # Scale the data to avoid any convergence issue
+    X = StandardScaler().fit_transform(X)
+    # Only use 2 classes and select samples such that 2-fold cross-validation
+    # split will lead to an equivalence with a `sample_weight` of 0
+    X = np.vstack((X[:40], X[50:90]))
+    y = np.hstack((y[:40], y[50:90]))
+    sample_weight = np.zeros_like(y)
+    sample_weight[::2] = 1
+
+    estimator = LogisticRegression().set_fit_request(sample_weight=True)
+    model_without_weights = TunedThresholdClassifierCV(estimator, cv=2)
+    model_with_weights = clone(model_without_weights)
+
+    model_with_weights.fit(X, y, sample_weight=sample_weight)
+    model_without_weights.fit(X[::2], y[::2])
+
+    assert_allclose(
+        model_with_weights.estimator_.coef_, model_without_weights.estimator_.coef_
+    )
+
+    y_pred_with_weights = model_with_weights.predict_proba(X)
+    y_pred_without_weights = model_without_weights.predict_proba(X)
+    assert_allclose(y_pred_with_weights, y_pred_without_weights)
+
+
+def test_tuned_threshold_classifier_thresholds_array():
+    """Check that we can pass an array to `thresholds` and it is used as candidate
+    threshold internally."""
+    X, y = make_classification(random_state=0)
+    estimator = LogisticRegression()
+    thresholds = np.linspace(0, 1, 11)
+    tuned_model = TunedThresholdClassifierCV(
+        estimator,
+        thresholds=thresholds,
+        response_method="predict_proba",
+        store_cv_results=True,
+    ).fit(X, y)
+    assert_allclose(tuned_model.cv_results_["thresholds"], thresholds)
+
+
+@pytest.mark.parametrize("store_cv_results", [True, False])
+def test_tuned_threshold_classifier_store_cv_results(store_cv_results):
+    """Check that if `cv_results_` exists depending on `store_cv_results`."""
+    X, y = make_classification(random_state=0)
+    estimator = LogisticRegression()
+    tuned_model = TunedThresholdClassifierCV(
+        estimator, store_cv_results=store_cv_results
+    ).fit(X, y)
+    if store_cv_results:
+        assert hasattr(tuned_model, "cv_results_")
+    else:
+        assert not hasattr(tuned_model, "cv_results_")
+
+
+def test_tuned_threshold_classifier_cv_float():
+    """Check the behaviour when `cv` is set to a float."""
+    X, y = make_classification(random_state=0)
+
+    # case where `refit=False` and cv is a float: the underlying estimator will be fit
+    # on the training set given by a ShuffleSplit. We check that we get the same model
+    # coefficients.
+    test_size = 0.3
+    estimator = LogisticRegression()
+    tuned_model = TunedThresholdClassifierCV(
+        estimator, cv=test_size, refit=False, random_state=0
+    ).fit(X, y)
+    tuned_model.fit(X, y)
+
+    cv = StratifiedShuffleSplit(n_splits=1, test_size=test_size, random_state=0)
+    train_idx, val_idx = next(cv.split(X, y))
+    cloned_estimator = clone(estimator).fit(X[train_idx], y[train_idx])
+
+    assert_allclose(tuned_model.estimator_.coef_, cloned_estimator.coef_)
+
+    # case where `refit=True`, then the underlying estimator is fitted on the full
+    # dataset.
+    tuned_model.set_params(refit=True).fit(X, y)
+    cloned_estimator = clone(estimator).fit(X, y)
+
+    assert_allclose(tuned_model.estimator_.coef_, cloned_estimator.coef_)
+
+
+def test_tuned_threshold_classifier_error_constant_predictor():
+    """Check that we raise a ValueError if the underlying classifier returns constant
+    probabilities such that we cannot find any threshold.
+    """
+    X, y = make_classification(random_state=0)
+    estimator = DummyClassifier(strategy="constant", constant=1)
+    tuned_model = TunedThresholdClassifierCV(estimator, response_method="predict_proba")
+    err_msg = "The provided estimator makes constant predictions"
+    with pytest.raises(ValueError, match=err_msg):
+        tuned_model.fit(X, y)
+
+
+@pytest.mark.parametrize(
+    "response_method", ["auto", "predict_proba", "decision_function"]
+)
+def test_fixed_threshold_classifier_equivalence_default(response_method):
+    """Check that `FixedThresholdClassifier` has the same behaviour as the vanilla
+    classifier.
+    """
+    X, y = make_classification(random_state=0)
+    classifier = LogisticRegression().fit(X, y)
+    classifier_default_threshold = FixedThresholdClassifier(
+        estimator=clone(classifier), response_method=response_method
+    )
+    classifier_default_threshold.fit(X, y)
+
+    # emulate the response method that should take into account the `pos_label`
+    if response_method in ("auto", "predict_proba"):
+        y_score = classifier_default_threshold.predict_proba(X)[:, 1]
+        threshold = 0.5
+    else:  # response_method == "decision_function"
+        y_score = classifier_default_threshold.decision_function(X)
+        threshold = 0.0
+
+    y_pred_lr = (y_score >= threshold).astype(int)
+    assert_allclose(classifier_default_threshold.predict(X), y_pred_lr)
+
+
+@pytest.mark.parametrize(
+    "response_method, threshold", [("predict_proba", 0.7), ("decision_function", 2.0)]
+)
+@pytest.mark.parametrize("pos_label", [0, 1])
+def test_fixed_threshold_classifier(response_method, threshold, pos_label):
+    """Check that applying `predict` lead to the same prediction as applying the
+    threshold to the output of the response method.
+    """
+    X, y = make_classification(n_samples=50, random_state=0)
+    logistic_regression = LogisticRegression().fit(X, y)
+    model = FixedThresholdClassifier(
+        estimator=clone(logistic_regression),
+        threshold=threshold,
+        response_method=response_method,
+        pos_label=pos_label,
+    ).fit(X, y)
+
+    # check that the underlying estimator is the same
+    assert_allclose(model.estimator_.coef_, logistic_regression.coef_)
+
+    # emulate the response method that should take into account the `pos_label`
+    if response_method == "predict_proba":
+        y_score = model.predict_proba(X)[:, pos_label]
+    else:  # response_method == "decision_function"
+        y_score = model.decision_function(X)
+        y_score = y_score if pos_label == 1 else -y_score
+
+    # create a mapping from boolean values to class labels
+    map_to_label = np.array([0, 1]) if pos_label == 1 else np.array([1, 0])
+    y_pred_lr = map_to_label[(y_score >= threshold).astype(int)]
+    assert_allclose(model.predict(X), y_pred_lr)
+
+    for method in ("predict_proba", "predict_log_proba", "decision_function"):
+        assert_allclose(
+            getattr(model, method)(X), getattr(logistic_regression, method)(X)
+        )
+        assert_allclose(
+            getattr(model.estimator_, method)(X),
+            getattr(logistic_regression, method)(X),
+        )
+
+
+@config_context(enable_metadata_routing=True)
+def test_fixed_threshold_classifier_metadata_routing():
+    """Check that everything works with metadata routing."""
+    X, y = make_classification(random_state=0)
+    sample_weight = np.ones_like(y)
+    sample_weight[::2] = 2
+    classifier = LogisticRegression().set_fit_request(sample_weight=True)
+    classifier.fit(X, y, sample_weight=sample_weight)
+    classifier_default_threshold = FixedThresholdClassifier(estimator=clone(classifier))
+    classifier_default_threshold.fit(X, y, sample_weight=sample_weight)
+    assert_allclose(classifier_default_threshold.estimator_.coef_, classifier.coef_)
+
+
+@pytest.mark.parametrize(
+    "method", ["predict_proba", "decision_function", "predict", "predict_log_proba"]
+)
+def test_fixed_threshold_classifier_fitted_estimator(method):
+    """Check that if the underlying estimator is already fitted, no fit is required."""
+    X, y = make_classification(random_state=0)
+    classifier = LogisticRegression().fit(X, y)
+    fixed_threshold_classifier = FixedThresholdClassifier(estimator=classifier)
+    # This should not raise an error
+    getattr(fixed_threshold_classifier, method)(X)
+
+
+def test_fixed_threshold_classifier_classes_():
+    """Check that the classes_ attribute is properly set."""
+    X, y = make_classification(random_state=0)
+    with pytest.raises(
+        AttributeError, match="The underlying estimator is not fitted yet."
+    ):
+        FixedThresholdClassifier(estimator=LogisticRegression()).classes_
+
+    classifier = LogisticRegression().fit(X, y)
+    fixed_threshold_classifier = FixedThresholdClassifier(estimator=classifier)
+    assert_array_equal(fixed_threshold_classifier.classes_, classifier.classes_)
diff --git a/sklearn/model_selection/tests/test_plot.py b/sklearn/model_selection/tests/test_plot.py
new file mode 100644
index 0000000000000..4e88475517454
--- /dev/null
+++ b/sklearn/model_selection/tests/test_plot.py
@@ -0,0 +1,572 @@
+import numpy as np
+import pytest
+
+from sklearn.datasets import load_iris
+from sklearn.model_selection import (
+    LearningCurveDisplay,
+    ValidationCurveDisplay,
+    learning_curve,
+    validation_curve,
+)
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.utils import shuffle
+from sklearn.utils._testing import assert_allclose, assert_array_equal
+
+
+@pytest.fixture
+def data():
+    return shuffle(*load_iris(return_X_y=True), random_state=0)
+
+
+@pytest.mark.parametrize(
+    "params, err_type, err_msg",
+    [
+        ({"std_display_style": "invalid"}, ValueError, "Unknown std_display_style:"),
+        ({"score_type": "invalid"}, ValueError, "Unknown score_type:"),
+    ],
+)
+@pytest.mark.parametrize(
+    "CurveDisplay, specific_params",
+    [
+        (ValidationCurveDisplay, {"param_name": "max_depth", "param_range": [1, 3, 5]}),
+        (LearningCurveDisplay, {"train_sizes": [0.3, 0.6, 0.9]}),
+    ],
+)
+def test_curve_display_parameters_validation(
+    pyplot, data, params, err_type, err_msg, CurveDisplay, specific_params
+):
+    """Check that we raise a proper error when passing invalid parameters."""
+    X, y = data
+    estimator = DecisionTreeClassifier(random_state=0)
+
+    with pytest.raises(err_type, match=err_msg):
+        CurveDisplay.from_estimator(estimator, X, y, **specific_params, **params)
+
+
+def test_learning_curve_display_default_usage(pyplot, data):
+    """Check the default usage of the LearningCurveDisplay class."""
+    X, y = data
+    estimator = DecisionTreeClassifier(random_state=0)
+
+    train_sizes = [0.3, 0.6, 0.9]
+    display = LearningCurveDisplay.from_estimator(
+        estimator, X, y, train_sizes=train_sizes
+    )
+
+    import matplotlib as mpl
+
+    assert display.errorbar_ is None
+
+    assert isinstance(display.lines_, list)
+    for line in display.lines_:
+        assert isinstance(line, mpl.lines.Line2D)
+
+    assert isinstance(display.fill_between_, list)
+    for fill in display.fill_between_:
+        assert isinstance(fill, mpl.collections.PolyCollection)
+        assert fill.get_alpha() == 0.5
+
+    assert display.score_name == "Score"
+    assert display.ax_.get_xlabel() == "Number of samples in the training set"
+    assert display.ax_.get_ylabel() == "Score"
+
+    _, legend_labels = display.ax_.get_legend_handles_labels()
+    assert legend_labels == ["Train", "Test"]
+
+    train_sizes_abs, train_scores, test_scores = learning_curve(
+        estimator, X, y, train_sizes=train_sizes
+    )
+
+    assert_array_equal(display.train_sizes, train_sizes_abs)
+    assert_allclose(display.train_scores, train_scores)
+    assert_allclose(display.test_scores, test_scores)
+
+
+def test_validation_curve_display_default_usage(pyplot, data):
+    """Check the default usage of the ValidationCurveDisplay class."""
+    X, y = data
+    estimator = DecisionTreeClassifier(random_state=0)
+
+    param_name, param_range = "max_depth", [1, 3, 5]
+    display = ValidationCurveDisplay.from_estimator(
+        estimator, X, y, param_name=param_name, param_range=param_range
+    )
+
+    import matplotlib as mpl
+
+    assert display.errorbar_ is None
+
+    assert isinstance(display.lines_, list)
+    for line in display.lines_:
+        assert isinstance(line, mpl.lines.Line2D)
+
+    assert isinstance(display.fill_between_, list)
+    for fill in display.fill_between_:
+        assert isinstance(fill, mpl.collections.PolyCollection)
+        assert fill.get_alpha() == 0.5
+
+    assert display.score_name == "Score"
+    assert display.ax_.get_xlabel() == f"{param_name}"
+    assert display.ax_.get_ylabel() == "Score"
+
+    _, legend_labels = display.ax_.get_legend_handles_labels()
+    assert legend_labels == ["Train", "Test"]
+
+    train_scores, test_scores = validation_curve(
+        estimator, X, y, param_name=param_name, param_range=param_range
+    )
+
+    assert_array_equal(display.param_range, param_range)
+    assert_allclose(display.train_scores, train_scores)
+    assert_allclose(display.test_scores, test_scores)
+
+
+@pytest.mark.parametrize(
+    "CurveDisplay, specific_params",
+    [
+        (ValidationCurveDisplay, {"param_name": "max_depth", "param_range": [1, 3, 5]}),
+        (LearningCurveDisplay, {"train_sizes": [0.3, 0.6, 0.9]}),
+    ],
+)
+def test_curve_display_negate_score(pyplot, data, CurveDisplay, specific_params):
+    """Check the behaviour of the `negate_score` parameter calling `from_estimator` and
+    `plot`.
+    """
+    X, y = data
+    estimator = DecisionTreeClassifier(max_depth=1, random_state=0)
+
+    negate_score = False
+    display = CurveDisplay.from_estimator(
+        estimator, X, y, **specific_params, negate_score=negate_score
+    )
+
+    positive_scores = display.lines_[0].get_data()[1]
+    assert (positive_scores >= 0).all()
+    assert display.ax_.get_ylabel() == "Score"
+
+    negate_score = True
+    display = CurveDisplay.from_estimator(
+        estimator, X, y, **specific_params, negate_score=negate_score
+    )
+
+    negative_scores = display.lines_[0].get_data()[1]
+    assert (negative_scores <= 0).all()
+    assert_allclose(negative_scores, -positive_scores)
+    assert display.ax_.get_ylabel() == "Negative score"
+
+    negate_score = False
+    display = CurveDisplay.from_estimator(
+        estimator, X, y, **specific_params, negate_score=negate_score
+    )
+    assert display.ax_.get_ylabel() == "Score"
+    display.plot(negate_score=not negate_score)
+    assert display.ax_.get_ylabel() == "Score"
+    assert (display.lines_[0].get_data()[1] < 0).all()
+
+
+@pytest.mark.parametrize(
+    "score_name, ylabel", [(None, "Score"), ("Accuracy", "Accuracy")]
+)
+@pytest.mark.parametrize(
+    "CurveDisplay, specific_params",
+    [
+        (ValidationCurveDisplay, {"param_name": "max_depth", "param_range": [1, 3, 5]}),
+        (LearningCurveDisplay, {"train_sizes": [0.3, 0.6, 0.9]}),
+    ],
+)
+def test_curve_display_score_name(
+    pyplot, data, score_name, ylabel, CurveDisplay, specific_params
+):
+    """Check that we can overwrite the default score name shown on the y-axis."""
+    X, y = data
+    estimator = DecisionTreeClassifier(random_state=0)
+
+    display = CurveDisplay.from_estimator(
+        estimator, X, y, **specific_params, score_name=score_name
+    )
+
+    assert display.ax_.get_ylabel() == ylabel
+    X, y = data
+    estimator = DecisionTreeClassifier(max_depth=1, random_state=0)
+
+    display = CurveDisplay.from_estimator(
+        estimator, X, y, **specific_params, score_name=score_name
+    )
+
+    assert display.score_name == ylabel
+
+
+@pytest.mark.parametrize("std_display_style", (None, "errorbar"))
+def test_learning_curve_display_score_type(pyplot, data, std_display_style):
+    """Check the behaviour of setting the `score_type` parameter."""
+    X, y = data
+    estimator = DecisionTreeClassifier(random_state=0)
+
+    train_sizes = [0.3, 0.6, 0.9]
+    train_sizes_abs, train_scores, test_scores = learning_curve(
+        estimator, X, y, train_sizes=train_sizes
+    )
+
+    score_type = "train"
+    display = LearningCurveDisplay.from_estimator(
+        estimator,
+        X,
+        y,
+        train_sizes=train_sizes,
+        score_type=score_type,
+        std_display_style=std_display_style,
+    )
+
+    _, legend_label = display.ax_.get_legend_handles_labels()
+    assert legend_label == ["Train"]
+
+    if std_display_style is None:
+        assert len(display.lines_) == 1
+        assert display.errorbar_ is None
+        x_data, y_data = display.lines_[0].get_data()
+    else:
+        assert display.lines_ is None
+        assert len(display.errorbar_) == 1
+        x_data, y_data = display.errorbar_[0].lines[0].get_data()
+
+    assert_array_equal(x_data, train_sizes_abs)
+    assert_allclose(y_data, train_scores.mean(axis=1))
+
+    score_type = "test"
+    display = LearningCurveDisplay.from_estimator(
+        estimator,
+        X,
+        y,
+        train_sizes=train_sizes,
+        score_type=score_type,
+        std_display_style=std_display_style,
+    )
+
+    _, legend_label = display.ax_.get_legend_handles_labels()
+    assert legend_label == ["Test"]
+
+    if std_display_style is None:
+        assert len(display.lines_) == 1
+        assert display.errorbar_ is None
+        x_data, y_data = display.lines_[0].get_data()
+    else:
+        assert display.lines_ is None
+        assert len(display.errorbar_) == 1
+        x_data, y_data = display.errorbar_[0].lines[0].get_data()
+
+    assert_array_equal(x_data, train_sizes_abs)
+    assert_allclose(y_data, test_scores.mean(axis=1))
+
+    score_type = "both"
+    display = LearningCurveDisplay.from_estimator(
+        estimator,
+        X,
+        y,
+        train_sizes=train_sizes,
+        score_type=score_type,
+        std_display_style=std_display_style,
+    )
+
+    _, legend_label = display.ax_.get_legend_handles_labels()
+    assert legend_label == ["Train", "Test"]
+
+    if std_display_style is None:
+        assert len(display.lines_) == 2
+        assert display.errorbar_ is None
+        x_data_train, y_data_train = display.lines_[0].get_data()
+        x_data_test, y_data_test = display.lines_[1].get_data()
+    else:
+        assert display.lines_ is None
+        assert len(display.errorbar_) == 2
+        x_data_train, y_data_train = display.errorbar_[0].lines[0].get_data()
+        x_data_test, y_data_test = display.errorbar_[1].lines[0].get_data()
+
+    assert_array_equal(x_data_train, train_sizes_abs)
+    assert_allclose(y_data_train, train_scores.mean(axis=1))
+    assert_array_equal(x_data_test, train_sizes_abs)
+    assert_allclose(y_data_test, test_scores.mean(axis=1))
+
+
+@pytest.mark.parametrize("std_display_style", (None, "errorbar"))
+def test_validation_curve_display_score_type(pyplot, data, std_display_style):
+    """Check the behaviour of setting the `score_type` parameter."""
+    X, y = data
+    estimator = DecisionTreeClassifier(random_state=0)
+
+    param_name, param_range = "max_depth", [1, 3, 5]
+    train_scores, test_scores = validation_curve(
+        estimator, X, y, param_name=param_name, param_range=param_range
+    )
+
+    score_type = "train"
+    display = ValidationCurveDisplay.from_estimator(
+        estimator,
+        X,
+        y,
+        param_name=param_name,
+        param_range=param_range,
+        score_type=score_type,
+        std_display_style=std_display_style,
+    )
+
+    _, legend_label = display.ax_.get_legend_handles_labels()
+    assert legend_label == ["Train"]
+
+    if std_display_style is None:
+        assert len(display.lines_) == 1
+        assert display.errorbar_ is None
+        x_data, y_data = display.lines_[0].get_data()
+    else:
+        assert display.lines_ is None
+        assert len(display.errorbar_) == 1
+        x_data, y_data = display.errorbar_[0].lines[0].get_data()
+
+    assert_array_equal(x_data, param_range)
+    assert_allclose(y_data, train_scores.mean(axis=1))
+
+    score_type = "test"
+    display = ValidationCurveDisplay.from_estimator(
+        estimator,
+        X,
+        y,
+        param_name=param_name,
+        param_range=param_range,
+        score_type=score_type,
+        std_display_style=std_display_style,
+    )
+
+    _, legend_label = display.ax_.get_legend_handles_labels()
+    assert legend_label == ["Test"]
+
+    if std_display_style is None:
+        assert len(display.lines_) == 1
+        assert display.errorbar_ is None
+        x_data, y_data = display.lines_[0].get_data()
+    else:
+        assert display.lines_ is None
+        assert len(display.errorbar_) == 1
+        x_data, y_data = display.errorbar_[0].lines[0].get_data()
+
+    assert_array_equal(x_data, param_range)
+    assert_allclose(y_data, test_scores.mean(axis=1))
+
+    score_type = "both"
+    display = ValidationCurveDisplay.from_estimator(
+        estimator,
+        X,
+        y,
+        param_name=param_name,
+        param_range=param_range,
+        score_type=score_type,
+        std_display_style=std_display_style,
+    )
+
+    _, legend_label = display.ax_.get_legend_handles_labels()
+    assert legend_label == ["Train", "Test"]
+
+    if std_display_style is None:
+        assert len(display.lines_) == 2
+        assert display.errorbar_ is None
+        x_data_train, y_data_train = display.lines_[0].get_data()
+        x_data_test, y_data_test = display.lines_[1].get_data()
+    else:
+        assert display.lines_ is None
+        assert len(display.errorbar_) == 2
+        x_data_train, y_data_train = display.errorbar_[0].lines[0].get_data()
+        x_data_test, y_data_test = display.errorbar_[1].lines[0].get_data()
+
+    assert_array_equal(x_data_train, param_range)
+    assert_allclose(y_data_train, train_scores.mean(axis=1))
+    assert_array_equal(x_data_test, param_range)
+    assert_allclose(y_data_test, test_scores.mean(axis=1))
+
+
+@pytest.mark.parametrize(
+    "CurveDisplay, specific_params, expected_xscale",
+    [
+        (
+            ValidationCurveDisplay,
+            {"param_name": "max_depth", "param_range": np.arange(1, 5)},
+            "linear",
+        ),
+        (LearningCurveDisplay, {"train_sizes": np.linspace(0.1, 0.9, num=5)}, "linear"),
+        (
+            ValidationCurveDisplay,
+            {
+                "param_name": "max_depth",
+                "param_range": np.round(np.logspace(0, 2, num=5)).astype(np.int64),
+            },
+            "log",
+        ),
+        (LearningCurveDisplay, {"train_sizes": np.logspace(-1, 0, num=5)}, "log"),
+    ],
+)
+def test_curve_display_xscale_auto(
+    pyplot, data, CurveDisplay, specific_params, expected_xscale
+):
+    """Check the behaviour of the x-axis scaling depending on the data provided."""
+    X, y = data
+    estimator = DecisionTreeClassifier(random_state=0)
+
+    display = CurveDisplay.from_estimator(estimator, X, y, **specific_params)
+    assert display.ax_.get_xscale() == expected_xscale
+
+
+@pytest.mark.parametrize(
+    "CurveDisplay, specific_params",
+    [
+        (ValidationCurveDisplay, {"param_name": "max_depth", "param_range": [1, 3, 5]}),
+        (LearningCurveDisplay, {"train_sizes": [0.3, 0.6, 0.9]}),
+    ],
+)
+def test_curve_display_std_display_style(pyplot, data, CurveDisplay, specific_params):
+    """Check the behaviour of the parameter `std_display_style`."""
+    X, y = data
+    estimator = DecisionTreeClassifier(random_state=0)
+
+    import matplotlib as mpl
+
+    std_display_style = None
+    display = CurveDisplay.from_estimator(
+        estimator,
+        X,
+        y,
+        **specific_params,
+        std_display_style=std_display_style,
+    )
+
+    assert len(display.lines_) == 2
+    for line in display.lines_:
+        assert isinstance(line, mpl.lines.Line2D)
+    assert display.errorbar_ is None
+    assert display.fill_between_ is None
+    _, legend_label = display.ax_.get_legend_handles_labels()
+    assert len(legend_label) == 2
+
+    std_display_style = "fill_between"
+    display = CurveDisplay.from_estimator(
+        estimator,
+        X,
+        y,
+        **specific_params,
+        std_display_style=std_display_style,
+    )
+
+    assert len(display.lines_) == 2
+    for line in display.lines_:
+        assert isinstance(line, mpl.lines.Line2D)
+    assert display.errorbar_ is None
+    assert len(display.fill_between_) == 2
+    for fill_between in display.fill_between_:
+        assert isinstance(fill_between, mpl.collections.PolyCollection)
+    _, legend_label = display.ax_.get_legend_handles_labels()
+    assert len(legend_label) == 2
+
+    std_display_style = "errorbar"
+    display = CurveDisplay.from_estimator(
+        estimator,
+        X,
+        y,
+        **specific_params,
+        std_display_style=std_display_style,
+    )
+
+    assert display.lines_ is None
+    assert len(display.errorbar_) == 2
+    for errorbar in display.errorbar_:
+        assert isinstance(errorbar, mpl.container.ErrorbarContainer)
+    assert display.fill_between_ is None
+    _, legend_label = display.ax_.get_legend_handles_labels()
+    assert len(legend_label) == 2
+
+
+@pytest.mark.parametrize(
+    "CurveDisplay, specific_params",
+    [
+        (ValidationCurveDisplay, {"param_name": "max_depth", "param_range": [1, 3, 5]}),
+        (LearningCurveDisplay, {"train_sizes": [0.3, 0.6, 0.9]}),
+    ],
+)
+def test_curve_display_plot_kwargs(pyplot, data, CurveDisplay, specific_params):
+    """Check the behaviour of the different plotting keyword arguments: `line_kw`,
+    `fill_between_kw`, and `errorbar_kw`."""
+    X, y = data
+    estimator = DecisionTreeClassifier(random_state=0)
+
+    std_display_style = "fill_between"
+    line_kw = {"color": "red"}
+    fill_between_kw = {"color": "red", "alpha": 1.0}
+    display = CurveDisplay.from_estimator(
+        estimator,
+        X,
+        y,
+        **specific_params,
+        std_display_style=std_display_style,
+        line_kw=line_kw,
+        fill_between_kw=fill_between_kw,
+    )
+
+    assert display.lines_[0].get_color() == "red"
+    assert_allclose(
+        display.fill_between_[0].get_facecolor(),
+        [[1.0, 0.0, 0.0, 1.0]],  # trust me, it's red
+    )
+
+    std_display_style = "errorbar"
+    errorbar_kw = {"color": "red"}
+    display = CurveDisplay.from_estimator(
+        estimator,
+        X,
+        y,
+        **specific_params,
+        std_display_style=std_display_style,
+        errorbar_kw=errorbar_kw,
+    )
+
+    assert display.errorbar_[0].lines[0].get_color() == "red"
+
+
+@pytest.mark.parametrize(
+    "param_range, xscale",
+    [([5, 10, 15], "linear"), ([-50, 5, 50, 500], "symlog"), ([5, 50, 500], "log")],
+)
+def test_validation_curve_xscale_from_param_range_provided_as_a_list(
+    pyplot, data, param_range, xscale
+):
+    """Check the induced xscale from the provided param_range values."""
+    X, y = data
+    estimator = DecisionTreeClassifier(random_state=0)
+
+    param_name = "max_depth"
+    display = ValidationCurveDisplay.from_estimator(
+        estimator,
+        X,
+        y,
+        param_name=param_name,
+        param_range=param_range,
+    )
+
+    assert display.ax_.get_xscale() == xscale
+
+
+@pytest.mark.parametrize(
+    "Display, params",
+    [
+        (LearningCurveDisplay, {}),
+        (ValidationCurveDisplay, {"param_name": "max_depth", "param_range": [1, 3, 5]}),
+    ],
+)
+def test_subclassing_displays(pyplot, data, Display, params):
+    """Check that named constructors return the correct type when subclassed.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/pull/27675
+    """
+    X, y = data
+    estimator = DecisionTreeClassifier(random_state=0)
+
+    class SubclassOfDisplay(Display):
+        pass
+
+    display = SubclassOfDisplay.from_estimator(estimator, X, y, **params)
+    assert isinstance(display, SubclassOfDisplay)
diff --git a/sklearn/model_selection/tests/test_search.py b/sklearn/model_selection/tests/test_search.py
index 25c4ce8cc22f7..7888dd2d1766b 100644
--- a/sklearn/model_selection/tests/test_search.py
+++ b/sklearn/model_selection/tests/test_search.py
@@ -1,83 +1,112 @@
 """Test the search module"""
 
+import pickle
+import re
+import sys
+import warnings
 from collections.abc import Iterable, Sized
+from functools import partial
 from io import StringIO
 from itertools import chain, product
-from functools import partial
-import pickle
-import sys
 from types import GeneratorType
-import re
 
 import numpy as np
-import scipy.sparse as sp
 import pytest
+from scipy.stats import bernoulli, expon, uniform
 
+from sklearn import config_context
+from sklearn.base import BaseEstimator, ClassifierMixin, clone, is_classifier
+from sklearn.cluster import KMeans
+from sklearn.compose import ColumnTransformer
+from sklearn.datasets import (
+    make_blobs,
+    make_classification,
+    make_multilabel_classification,
+)
+from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
+from sklearn.dummy import DummyClassifier
+from sklearn.ensemble import HistGradientBoostingClassifier
+from sklearn.exceptions import FitFailedWarning
+from sklearn.experimental import enable_halving_search_cv  # noqa: F401
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.impute import SimpleImputer
+from sklearn.linear_model import (
+    LinearRegression,
+    LogisticRegression,
+    Ridge,
+    SGDClassifier,
+)
+from sklearn.metrics import (
+    accuracy_score,
+    confusion_matrix,
+    f1_score,
+    make_scorer,
+    r2_score,
+    recall_score,
+    roc_auc_score,
+)
+from sklearn.metrics.pairwise import euclidean_distances
+from sklearn.model_selection import (
+    GridSearchCV,
+    GroupKFold,
+    GroupShuffleSplit,
+    HalvingGridSearchCV,
+    KFold,
+    LeaveOneGroupOut,
+    LeavePGroupsOut,
+    ParameterGrid,
+    ParameterSampler,
+    RandomizedSearchCV,
+    StratifiedKFold,
+    StratifiedShuffleSplit,
+    train_test_split,
+)
+from sklearn.model_selection._search import (
+    BaseSearchCV,
+    _yield_masked_array_for_each_param,
+)
+from sklearn.model_selection.tests.common import OneTimeSplitter
+from sklearn.naive_bayes import ComplementNB
+from sklearn.neighbors import KernelDensity, KNeighborsClassifier, LocalOutlierFactor
+from sklearn.pipeline import Pipeline, make_pipeline
+from sklearn.preprocessing import (
+    OneHotEncoder,
+    OrdinalEncoder,
+    SplineTransformer,
+    StandardScaler,
+)
+from sklearn.svm import SVC, LinearSVC
+from sklearn.tests.metadata_routing_common import (
+    ConsumingScorer,
+    _Registry,
+    check_recorded_metadata,
+)
+from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
+from sklearn.utils._array_api import (
+    _get_namespace_device_dtype_ids,
+    yield_namespace_device_dtype_combinations,
+)
+from sklearn.utils._mocking import CheckingClassifier, MockDataFrame
 from sklearn.utils._testing import (
-    assert_array_equal,
-    assert_array_almost_equal,
-    assert_allclose,
-    assert_almost_equal,
-    ignore_warnings,
     MinimalClassifier,
     MinimalRegressor,
     MinimalTransformer,
+    _array_api_for_tests,
+    assert_allclose,
+    assert_allclose_dense_sparse,
+    assert_almost_equal,
+    assert_array_almost_equal,
+    assert_array_equal,
+    set_random_state,
 )
-from sklearn.utils._mocking import CheckingClassifier, MockDataFrame
-
-from scipy.stats import bernoulli, expon, uniform
-
-from sklearn.base import BaseEstimator, ClassifierMixin
-from sklearn.base import clone, is_classifier
-from sklearn.exceptions import NotFittedError
-from sklearn.datasets import make_classification
-from sklearn.datasets import make_blobs
-from sklearn.datasets import make_multilabel_classification
-
-from sklearn.model_selection import fit_grid_point
-from sklearn.model_selection import train_test_split
-from sklearn.model_selection import KFold
-from sklearn.model_selection import StratifiedKFold
-from sklearn.model_selection import StratifiedShuffleSplit
-from sklearn.model_selection import LeaveOneGroupOut
-from sklearn.model_selection import LeavePGroupsOut
-from sklearn.model_selection import GroupKFold
-from sklearn.model_selection import GroupShuffleSplit
-from sklearn.model_selection import GridSearchCV
-from sklearn.model_selection import RandomizedSearchCV
-from sklearn.model_selection import ParameterGrid
-from sklearn.model_selection import ParameterSampler
-from sklearn.model_selection._search import BaseSearchCV
-
-from sklearn.model_selection._validation import FitFailedWarning
-
-from sklearn.svm import LinearSVC, SVC
-from sklearn.tree import DecisionTreeRegressor
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.cluster import KMeans
-from sklearn.neighbors import KernelDensity
-from sklearn.neighbors import LocalOutlierFactor
-from sklearn.neighbors import KNeighborsClassifier
-from sklearn.metrics import f1_score
-from sklearn.metrics import recall_score
-from sklearn.metrics import accuracy_score
-from sklearn.metrics import make_scorer
-from sklearn.metrics import roc_auc_score
-from sklearn.metrics import confusion_matrix
-from sklearn.metrics import r2_score
-from sklearn.metrics.pairwise import euclidean_distances
-from sklearn.impute import SimpleImputer
-from sklearn.pipeline import Pipeline
-from sklearn.linear_model import Ridge, SGDClassifier, LinearRegression
-from sklearn.experimental import enable_hist_gradient_boosting  # noqa
-from sklearn.ensemble import HistGradientBoostingClassifier
-
-from sklearn.model_selection.tests.common import OneTimeSplitter
+from sklearn.utils.estimator_checks import _enforce_estimator_tags_y
+from sklearn.utils.fixes import CSR_CONTAINERS
+from sklearn.utils.validation import _num_samples
 
 
 # Neither of the following two estimators inherit from BaseEstimator,
 # to test hyperparameter search on user-defined classifiers.
-class MockClassifier:
+class MockClassifier(ClassifierMixin, BaseEstimator):
     """Dummy classifier to test the parameter search algorithms"""
 
     def __init__(self, foo_param=0):
@@ -103,21 +132,22 @@ def inverse_transform(self, X):
 
     def score(self, X=None, Y=None):
         if self.foo_param > 1:
-            score = 1.
+            score = 1.0
         else:
-            score = 0.
+            score = 0.0
         return score
 
     def get_params(self, deep=False):
-        return {'foo_param': self.foo_param}
+        return {"foo_param": self.foo_param}
 
     def set_params(self, **params):
-        self.foo_param = params['foo_param']
+        self.foo_param = params["foo_param"]
         return self
 
 
 class LinearSVCNoScore(LinearSVC):
-    """An LinearSVC classifier that has no score method."""
+    """A LinearSVC classifier that has no score method."""
+
     @property
     def score(self):
         raise AttributeError
@@ -130,14 +160,20 @@ def score(self):
 def assert_grid_iter_equals_getitem(grid):
     assert list(grid) == [grid[i] for i in range(len(grid))]
 
-@pytest.mark.parametrize("klass", [ParameterGrid,
-                                   partial(ParameterSampler, n_iter=10)])
+
+@pytest.mark.parametrize("klass", [ParameterGrid, partial(ParameterSampler, n_iter=10)])
 @pytest.mark.parametrize(
     "input, error_type, error_message",
-    [(0, TypeError, r'Parameter .* is not a dict or a list \(0\)'),
-     ([{'foo': [0]}, 0], TypeError, r'Parameter .* is not a dict \(0\)'),
-     ({'foo': 0}, TypeError, "Parameter.* value is not iterable .*"
-      r"\(key='foo', value=0\)")]
+    [
+        (0, TypeError, r"Parameter .* a dict or a list, got: 0 of type int"),
+        ([{"foo": [0]}, 0], TypeError, r"Parameter .* is not a dict \(0\)"),
+        (
+            {"foo": 0},
+            TypeError,
+            r"Parameter (grid|distribution) for parameter 'foo' (is not|needs to be) "
+            r"(a list or a numpy array|iterable or a distribution).*",
+        ),
+    ],
 )
 def test_validate_parameter_input(klass, input, error_type, error_message):
     with pytest.raises(error_type, match=error_message):
@@ -145,7 +181,6 @@ def test_validate_parameter_input(klass, input, error_type, error_message):
 
 
 def test_parameter_grid():
-
     # Test basic properties of ParameterGrid.
     params1 = {"foo": [1, 2, 3]}
     grid1 = ParameterGrid(params1)
@@ -154,8 +189,7 @@ def test_parameter_grid():
     assert len(grid1) == 3
     assert_grid_iter_equals_getitem(grid1)
 
-    params2 = {"foo": [4, 2],
-               "bar": ["ham", "spam", "eggs"]}
+    params2 = {"foo": [4, 2], "bar": ["ham", "spam", "eggs"]}
     grid2 = ParameterGrid(params2)
     assert len(grid2) == 6
 
@@ -163,9 +197,9 @@ def test_parameter_grid():
     for i in range(2):
         # tuple + chain transforms {"a": 1, "b": 2} to ("a", 1, "b", 2)
         points = set(tuple(chain(*(sorted(p.items())))) for p in grid2)
-        assert (points ==
-                set(("bar", x, "foo", y)
-                    for x, y in product(params2["bar"], params2["foo"])))
+        assert points == set(
+            ("bar", x, "foo", y) for x, y in product(params2["bar"], params2["foo"])
+        )
     assert_grid_iter_equals_getitem(grid2)
 
     # Special case: empty grid (useful to get default estimator settings)
@@ -176,16 +210,16 @@ def test_parameter_grid():
     with pytest.raises(IndexError):
         empty[1]
 
-    has_empty = ParameterGrid([{'C': [1, 10]}, {}, {'C': [.5]}])
+    has_empty = ParameterGrid([{"C": [1, 10]}, {}, {"C": [0.5]}])
     assert len(has_empty) == 4
-    assert list(has_empty) == [{'C': 1}, {'C': 10}, {}, {'C': .5}]
+    assert list(has_empty) == [{"C": 1}, {"C": 10}, {}, {"C": 0.5}]
     assert_grid_iter_equals_getitem(has_empty)
 
 
 def test_grid_search():
     # Test that the best estimator contains the right value for foo_param
     clf = MockClassifier()
-    grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]}, cv=3, verbose=3)
+    grid_search = GridSearchCV(clf, {"foo_param": [1, 2, 3]}, cv=2, verbose=3)
     # make sure it selects the smallest parameter in case of ties
     old_stdout = sys.stdout
     sys.stdout = StringIO()
@@ -193,8 +227,7 @@ def test_grid_search():
     sys.stdout = old_stdout
     assert grid_search.best_estimator_.foo_param == 2
 
-    assert_array_equal(grid_search.cv_results_["param_foo_param"].data,
-                       [1, 2, 3])
+    assert_array_equal(grid_search.cv_results_["param_foo_param"].data, [1, 2, 3])
 
     # Smoke test the score etc:
     grid_search.score(X, y)
@@ -203,37 +236,35 @@ def test_grid_search():
     grid_search.transform(X)
 
     # Test exception handling on scoring
-    grid_search.scoring = 'sklearn'
+    grid_search.scoring = "sklearn"
     with pytest.raises(ValueError):
         grid_search.fit(X, y)
 
 
 def test_grid_search_pipeline_steps():
     # check that parameters that are estimators are cloned before fitting
-    pipe = Pipeline([('regressor', LinearRegression())])
-    param_grid = {'regressor': [LinearRegression(), Ridge()]}
+    pipe = Pipeline([("regressor", LinearRegression())])
+    param_grid = {"regressor": [LinearRegression(), Ridge()]}
     grid_search = GridSearchCV(pipe, param_grid, cv=2)
     grid_search.fit(X, y)
-    regressor_results = grid_search.cv_results_['param_regressor']
+    regressor_results = grid_search.cv_results_["param_regressor"]
     assert isinstance(regressor_results[0], LinearRegression)
     assert isinstance(regressor_results[1], Ridge)
-    assert not hasattr(regressor_results[0], 'coef_')
-    assert not hasattr(regressor_results[1], 'coef_')
+    assert not hasattr(regressor_results[0], "coef_")
+    assert not hasattr(regressor_results[1], "coef_")
     assert regressor_results[0] is not grid_search.best_estimator_
     assert regressor_results[1] is not grid_search.best_estimator_
     # check that we didn't modify the parameter grid that was passed
-    assert not hasattr(param_grid['regressor'][0], 'coef_')
-    assert not hasattr(param_grid['regressor'][1], 'coef_')
+    assert not hasattr(param_grid["regressor"][0], "coef_")
+    assert not hasattr(param_grid["regressor"][1], "coef_")
 
 
 @pytest.mark.parametrize("SearchCV", [GridSearchCV, RandomizedSearchCV])
 def test_SearchCV_with_fit_params(SearchCV):
     X = np.arange(100).reshape(10, 10)
     y = np.array([0] * 5 + [1] * 5)
-    clf = CheckingClassifier(expected_fit_params=['spam', 'eggs'])
-    searcher = SearchCV(
-        clf, {'foo_param': [1, 2, 3]}, cv=2, error_score="raise"
-    )
+    clf = CheckingClassifier(expected_fit_params=["spam", "eggs"])
+    searcher = SearchCV(clf, {"foo_param": [1, 2, 3]}, cv=2, error_score="raise")
 
     # The CheckingClassifier generates an assertion error if
     # a parameter is missing or has length != len(X).
@@ -247,18 +278,16 @@ def test_SearchCV_with_fit_params(SearchCV):
     searcher.fit(X, y, spam=np.ones(10), eggs=np.zeros(10))
 
 
-@ignore_warnings
 def test_grid_search_no_score():
     # Test grid-search on classifier that has no score function.
     clf = LinearSVC(random_state=0)
     X, y = make_blobs(random_state=0, centers=2)
-    Cs = [.1, 1, 10]
+    Cs = [0.1, 1, 10]
     clf_no_score = LinearSVCNoScore(random_state=0)
-    grid_search = GridSearchCV(clf, {'C': Cs}, scoring='accuracy')
+    grid_search = GridSearchCV(clf, {"C": Cs}, scoring="accuracy")
     grid_search.fit(X, y)
 
-    grid_search_no_score = GridSearchCV(clf_no_score, {'C': Cs},
-                                        scoring='accuracy')
+    grid_search_no_score = GridSearchCV(clf_no_score, {"C": Cs}, scoring="accuracy")
     # smoketest grid search
     grid_search_no_score.fit(X, y)
 
@@ -268,23 +297,22 @@ def test_grid_search_no_score():
     assert grid_search.score(X, y) == grid_search_no_score.score(X, y)
 
     # giving no scoring function raises an error
-    grid_search_no_score = GridSearchCV(clf_no_score, {'C': Cs})
+    grid_search_no_score = GridSearchCV(clf_no_score, {"C": Cs})
     with pytest.raises(TypeError, match="no scoring"):
         grid_search_no_score.fit([[1]])
 
 
 def test_grid_search_score_method():
-    X, y = make_classification(n_samples=100, n_classes=2, flip_y=.2,
-                               random_state=0)
+    X, y = make_classification(n_samples=100, n_classes=2, flip_y=0.2, random_state=0)
     clf = LinearSVC(random_state=0)
-    grid = {'C': [.1]}
+    grid = {"C": [0.1]}
 
     search_no_scoring = GridSearchCV(clf, grid, scoring=None).fit(X, y)
-    search_accuracy = GridSearchCV(clf, grid, scoring='accuracy').fit(X, y)
-    search_no_score_method_auc = GridSearchCV(LinearSVCNoScore(), grid,
-                                              scoring='roc_auc'
-                                              ).fit(X, y)
-    search_auc = GridSearchCV(clf, grid, scoring='roc_auc').fit(X, y)
+    search_accuracy = GridSearchCV(clf, grid, scoring="accuracy").fit(X, y)
+    search_no_score_method_auc = GridSearchCV(
+        LinearSVCNoScore(), grid, scoring="roc_auc"
+    ).fit(X, y)
+    search_auc = GridSearchCV(clf, grid, scoring="roc_auc").fit(X, y)
 
     # Check warning only occurs in situation where behavior changed:
     # estimator requires score method to compete with scoring parameter
@@ -311,10 +339,14 @@ def test_grid_search_groups():
     groups = rng.randint(0, 3, 15)
 
     clf = LinearSVC(random_state=0)
-    grid = {'C': [1]}
-
-    group_cvs = [LeaveOneGroupOut(), LeavePGroupsOut(2),
-                 GroupKFold(n_splits=3), GroupShuffleSplit()]
+    grid = {"C": [1]}
+
+    group_cvs = [
+        LeaveOneGroupOut(),
+        LeavePGroupsOut(2),
+        GroupKFold(n_splits=3),
+        GroupShuffleSplit(),
+    ]
     error_msg = "The 'groups' parameter should not be None."
     for cv in group_cvs:
         gs = GridSearchCV(clf, grid, cv=cv)
@@ -333,37 +365,35 @@ def test_classes__property():
     # Test that classes_ property matches best_estimator_.classes_
     X = np.arange(100).reshape(10, 10)
     y = np.array([0] * 5 + [1] * 5)
-    Cs = [.1, 1, 10]
+    Cs = [0.1, 1, 10]
 
-    grid_search = GridSearchCV(LinearSVC(random_state=0), {'C': Cs})
+    grid_search = GridSearchCV(LinearSVC(random_state=0), {"C": Cs})
     grid_search.fit(X, y)
-    assert_array_equal(grid_search.best_estimator_.classes_,
-                       grid_search.classes_)
+    assert_array_equal(grid_search.best_estimator_.classes_, grid_search.classes_)
 
     # Test that regressors do not have a classes_ attribute
-    grid_search = GridSearchCV(Ridge(), {'alpha': [1.0, 2.0]})
+    grid_search = GridSearchCV(Ridge(), {"alpha": [1.0, 2.0]})
     grid_search.fit(X, y)
-    assert not hasattr(grid_search, 'classes_')
+    assert not hasattr(grid_search, "classes_")
 
     # Test that the grid searcher has no classes_ attribute before it's fit
-    grid_search = GridSearchCV(LinearSVC(random_state=0), {'C': Cs})
-    assert not hasattr(grid_search, 'classes_')
+    grid_search = GridSearchCV(LinearSVC(random_state=0), {"C": Cs})
+    assert not hasattr(grid_search, "classes_")
 
     # Test that the grid searcher has no classes_ attribute without a refit
-    grid_search = GridSearchCV(LinearSVC(random_state=0),
-                               {'C': Cs}, refit=False)
+    grid_search = GridSearchCV(LinearSVC(random_state=0), {"C": Cs}, refit=False)
     grid_search.fit(X, y)
-    assert not hasattr(grid_search, 'classes_')
+    assert not hasattr(grid_search, "classes_")
 
 
 def test_trivial_cv_results_attr():
     # Test search over a "grid" with only one point.
     clf = MockClassifier()
-    grid_search = GridSearchCV(clf, {'foo_param': [1]}, cv=3)
+    grid_search = GridSearchCV(clf, {"foo_param": [1]}, cv=2)
     grid_search.fit(X, y)
     assert hasattr(grid_search, "cv_results_")
 
-    random_search = RandomizedSearchCV(clf, {'foo_param': [0]}, n_iter=1, cv=3)
+    random_search = RandomizedSearchCV(clf, {"foo_param": [0]}, n_iter=1, cv=2)
     random_search.fit(X, y)
     assert hasattr(grid_search, "cv_results_")
 
@@ -371,33 +401,43 @@ def test_trivial_cv_results_attr():
 def test_no_refit():
     # Test that GSCV can be used for model selection alone without refitting
     clf = MockClassifier()
-    for scoring in [None, ['accuracy', 'precision']]:
-        grid_search = GridSearchCV(
-            clf, {'foo_param': [1, 2, 3]}, refit=False, cv=3
-        )
+    for scoring in [None, ["accuracy", "precision"]]:
+        grid_search = GridSearchCV(clf, {"foo_param": [1, 2, 3]}, refit=False, cv=2)
         grid_search.fit(X, y)
-        assert not hasattr(grid_search, "best_estimator_") and \
-            hasattr(grid_search, "best_index_") and \
-            hasattr(grid_search, "best_params_")
+        assert (
+            not hasattr(grid_search, "best_estimator_")
+            and hasattr(grid_search, "best_index_")
+            and hasattr(grid_search, "best_params_")
+        )
 
-        # Make sure the functions predict/transform etc raise meaningful
+        # Make sure the functions predict/transform etc. raise meaningful
         # error messages
-        for fn_name in ('predict', 'predict_proba', 'predict_log_proba',
-                        'transform', 'inverse_transform'):
-            error_msg = (f"refit=False. {fn_name} is available only after "
-                         f"refitting on the best parameters")
-            with pytest.raises(NotFittedError, match=error_msg):
+        for fn_name in (
+            "predict",
+            "predict_proba",
+            "predict_log_proba",
+            "transform",
+            "inverse_transform",
+        ):
+            outer_msg = f"has no attribute '{fn_name}'"
+            inner_msg = (
+                f"`refit=False`. {fn_name} is available only after "
+                "refitting on the best parameters"
+            )
+            with pytest.raises(AttributeError, match=outer_msg) as exec_info:
                 getattr(grid_search, fn_name)(X)
 
+            assert isinstance(exec_info.value.__cause__, AttributeError)
+            assert inner_msg in str(exec_info.value.__cause__)
+
     # Test that an invalid refit param raises appropriate error messages
-    error_msg = ("For multi-metric scoring, the parameter refit must be set to"
-                 " a scorer key")
-    for refit in ["", 5, True, 'recall', 'accuracy']:
+    error_msg = (
+        "For multi-metric scoring, the parameter refit must be set to a scorer key"
+    )
+    for refit in [True, "recall", "accuracy"]:
         with pytest.raises(ValueError, match=error_msg):
             GridSearchCV(
-                clf, {},
-                refit=refit,
-                scoring={'acc': 'accuracy', 'prec': 'precision'}
+                clf, {}, refit=refit, scoring={"acc": "accuracy", "prec": "precision"}
             ).fit(X, y)
 
 
@@ -406,7 +446,7 @@ def test_grid_search_error():
     X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)
 
     clf = LinearSVC()
-    cv = GridSearchCV(clf, {'C': [0.1, 1.0]})
+    cv = GridSearchCV(clf, {"C": [0.1, 1.0]})
     with pytest.raises(ValueError):
         cv.fit(X_[:180], y_)
 
@@ -415,7 +455,7 @@ def test_grid_search_one_grid_point():
     X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)
     param_dict = {"C": [1.0], "kernel": ["rbf"], "gamma": [0.1]}
 
-    clf = SVC(gamma='auto')
+    clf = SVC(gamma="auto")
     cv = GridSearchCV(clf, param_dict)
     cv.fit(X_, y_)
 
@@ -429,81 +469,86 @@ def test_grid_search_when_param_grid_includes_range():
     # Test that the best estimator contains the right value for foo_param
     clf = MockClassifier()
     grid_search = None
-    grid_search = GridSearchCV(clf, {'foo_param': range(1, 4)}, cv=3)
+    grid_search = GridSearchCV(clf, {"foo_param": range(1, 4)}, cv=2)
     grid_search.fit(X, y)
     assert grid_search.best_estimator_.foo_param == 2
 
 
 def test_grid_search_bad_param_grid():
+    X, y = make_classification(n_samples=10, n_features=5, random_state=0)
     param_dict = {"C": 1}
-    clf = SVC(gamma='auto')
+    clf = SVC(gamma="auto")
     error_msg = re.escape(
-        "Parameter grid for parameter (C) needs to"
-        " be a list or numpy array, but got ()."
-        " Single values need to be wrapped in a list"
-        " with one element."
+        "Parameter grid for parameter 'C' needs to be a list or "
+        "a numpy array, but got 1 (of type int) instead. Single "
+        "values need to be wrapped in a list with one element."
     )
-    with pytest.raises(ValueError, match=error_msg):
-        GridSearchCV(clf, param_dict)
+    search = GridSearchCV(clf, param_dict)
+    with pytest.raises(TypeError, match=error_msg):
+        search.fit(X, y)
 
     param_dict = {"C": []}
     clf = SVC()
     error_msg = re.escape(
-        "Parameter values for parameter (C) need to be a non-empty sequence."
+        "Parameter grid for parameter 'C' need to be a non-empty sequence, got: []"
     )
+    search = GridSearchCV(clf, param_dict)
     with pytest.raises(ValueError, match=error_msg):
-        GridSearchCV(clf, param_dict)
+        search.fit(X, y)
 
     param_dict = {"C": "1,2,3"}
-    clf = SVC(gamma='auto')
+    clf = SVC(gamma="auto")
     error_msg = re.escape(
-        "Parameter grid for parameter (C) needs to"
-        " be a list or numpy array, but got ()."
-        " Single values need to be wrapped in a list"
-        " with one element."
+        "Parameter grid for parameter 'C' needs to be a list or a numpy array, "
+        "but got '1,2,3' (of type str) instead. Single values need to be "
+        "wrapped in a list with one element."
     )
-    with pytest.raises(ValueError, match=error_msg):
-        GridSearchCV(clf, param_dict)
+    search = GridSearchCV(clf, param_dict)
+    with pytest.raises(TypeError, match=error_msg):
+        search.fit(X, y)
 
     param_dict = {"C": np.ones((3, 2))}
     clf = SVC()
+    search = GridSearchCV(clf, param_dict)
     with pytest.raises(ValueError):
-        GridSearchCV(clf, param_dict)
+        search.fit(X, y)
 
 
-def test_grid_search_sparse():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_grid_search_sparse(csr_container):
     # Test that grid search works with both dense and sparse matrices
     X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)
 
     clf = LinearSVC()
-    cv = GridSearchCV(clf, {'C': [0.1, 1.0]})
+    cv = GridSearchCV(clf, {"C": [0.1, 1.0]})
     cv.fit(X_[:180], y_[:180])
     y_pred = cv.predict(X_[180:])
     C = cv.best_estimator_.C
 
-    X_ = sp.csr_matrix(X_)
+    X_ = csr_container(X_)
     clf = LinearSVC()
-    cv = GridSearchCV(clf, {'C': [0.1, 1.0]})
+    cv = GridSearchCV(clf, {"C": [0.1, 1.0]})
     cv.fit(X_[:180].tocoo(), y_[:180])
     y_pred2 = cv.predict(X_[180:])
     C2 = cv.best_estimator_.C
 
-    assert np.mean(y_pred == y_pred2) >= .9
+    assert np.mean(y_pred == y_pred2) >= 0.9
     assert C == C2
 
 
-def test_grid_search_sparse_scoring():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_grid_search_sparse_scoring(csr_container):
     X_, y_ = make_classification(n_samples=200, n_features=100, random_state=0)
 
     clf = LinearSVC()
-    cv = GridSearchCV(clf, {'C': [0.1, 1.0]}, scoring="f1")
+    cv = GridSearchCV(clf, {"C": [0.1, 1.0]}, scoring="f1")
     cv.fit(X_[:180], y_[:180])
     y_pred = cv.predict(X_[180:])
     C = cv.best_estimator_.C
 
-    X_ = sp.csr_matrix(X_)
+    X_ = csr_container(X_)
     clf = LinearSVC()
-    cv = GridSearchCV(clf, {'C': [0.1, 1.0]}, scoring="f1")
+    cv = GridSearchCV(clf, {"C": [0.1, 1.0]}, scoring="f1")
     cv.fit(X_[:180], y_[:180])
     y_pred2 = cv.predict(X_[180:])
     C2 = cv.best_estimator_.C
@@ -517,8 +562,9 @@ def test_grid_search_sparse_scoring():
     # test loss where greater is worse
     def f1_loss(y_true_, y_pred_):
         return -f1_score(y_true_, y_pred_)
+
     F1Loss = make_scorer(f1_loss, greater_is_better=False)
-    cv = GridSearchCV(clf, {'C': [0.1, 1.0]}, scoring=F1Loss)
+    cv = GridSearchCV(clf, {"C": [0.1, 1.0]}, scoring=F1Loss)
     cv.fit(X_[:180], y_[:180])
     y_pred3 = cv.predict(X_[180:])
     C3 = cv.best_estimator_.C
@@ -536,8 +582,8 @@ def test_grid_search_precomputed_kernel():
     K_train = np.dot(X_[:180], X_[:180].T)
     y_train = y_[:180]
 
-    clf = SVC(kernel='precomputed')
-    cv = GridSearchCV(clf, {'C': [0.1, 1.0]})
+    clf = SVC(kernel="precomputed")
+    cv = GridSearchCV(clf, {"C": [0.1, 1.0]})
     cv.fit(K_train, y_train)
 
     assert cv.best_score_ >= 0
@@ -560,9 +606,9 @@ def test_grid_search_precomputed_kernel_error_nonsquare():
     # Test that grid search returns an error with a non-square precomputed
     # training kernel matrix
     K_train = np.zeros((10, 20))
-    y_train = np.ones((10, ))
-    clf = SVC(kernel='precomputed')
-    cv = GridSearchCV(clf, {'C': [0.1, 1.0]})
+    y_train = np.ones((10,))
+    clf = SVC(kernel="precomputed")
+    cv = GridSearchCV(clf, {"C": [0.1, 1.0]})
     with pytest.raises(ValueError):
         cv.fit(K_train, y_train)
 
@@ -574,14 +620,14 @@ def __init__(self, parameter=None):
         self.parameter = parameter
 
     def fit(self, X, y):
-        assert not hasattr(self, 'has_been_fit_')
+        assert not hasattr(self, "has_been_fit_")
         self.has_been_fit_ = True
 
     def predict(self, X):
         return np.zeros(X.shape[0])
 
 
-@ignore_warnings
+@pytest.mark.filterwarnings("ignore::sklearn.exceptions.UndefinedMetricWarning")
 def test_refit():
     # Regression test for bug in refitting
     # Simulates re-fitting a broken estimator; this used to break with
@@ -589,8 +635,9 @@ def test_refit():
     X = np.arange(100).reshape(10, 10)
     y = np.array([0] * 5 + [1] * 5)
 
-    clf = GridSearchCV(BrokenClassifier(), [{'parameter': [0, 1]}],
-                       scoring="precision", refit=True)
+    clf = GridSearchCV(
+        BrokenClassifier(), [{"parameter": [0, 1]}], scoring="precision", refit=True
+    )
     clf.fit(X, y)
 
 
@@ -599,6 +646,7 @@ def test_refit_callable():
     Test refit=callable, which adds flexibility in identifying the
     "best" estimator.
     """
+
     def refit_callable(cv_results):
         """
         A dummy function tests `refit=callable` interface.
@@ -607,10 +655,13 @@ def refit_callable(cv_results):
         """
         # Fit a dummy clf with `refit=True` to get a list of keys in
         # clf.cv_results_.
-        X, y = make_classification(n_samples=100, n_features=4,
-                                   random_state=42)
-        clf = GridSearchCV(LinearSVC(random_state=42), {'C': [0.01, 0.1, 1]},
-                           scoring='precision', refit=True)
+        X, y = make_classification(n_samples=100, n_features=4, random_state=42)
+        clf = GridSearchCV(
+            LinearSVC(random_state=42),
+            {"C": [0.01, 0.1, 1]},
+            scoring="precision",
+            refit=True,
+        )
         clf.fit(X, y)
         # Ensure that `best_index_ != 0` for this dummy clf
         assert clf.best_index_ != 0
@@ -619,17 +670,20 @@ def refit_callable(cv_results):
         for key in clf.cv_results_.keys():
             assert key in cv_results
 
-        return cv_results['mean_test_score'].argmin()
+        return cv_results["mean_test_score"].argmin()
 
-    X, y = make_classification(n_samples=100, n_features=4,
-                               random_state=42)
-    clf = GridSearchCV(LinearSVC(random_state=42), {'C': [0.01, 0.1, 1]},
-                       scoring='precision', refit=refit_callable)
+    X, y = make_classification(n_samples=100, n_features=4, random_state=42)
+    clf = GridSearchCV(
+        LinearSVC(random_state=42),
+        {"C": [0.01, 0.1, 1]},
+        scoring="precision",
+        refit=refit_callable,
+    )
     clf.fit(X, y)
 
     assert clf.best_index_ == 0
     # Ensure `best_score_` is disabled when using `refit=callable`
-    assert not hasattr(clf, 'best_score_')
+    assert not hasattr(clf, "best_score_")
 
 
 def test_refit_callable_invalid_type():
@@ -637,41 +691,48 @@ def test_refit_callable_invalid_type():
     Test implementation catches the errors when 'best_index_' returns an
     invalid result.
     """
+
     def refit_callable_invalid_type(cv_results):
         """
         A dummy function tests when returned 'best_index_' is not integer.
         """
         return None
 
-    X, y = make_classification(n_samples=100, n_features=4,
-                               random_state=42)
+    X, y = make_classification(n_samples=100, n_features=4, random_state=42)
 
-    clf = GridSearchCV(LinearSVC(random_state=42), {'C': [0.1, 1]},
-                       scoring='precision', refit=refit_callable_invalid_type)
-    with pytest.raises(TypeError,
-                       match='best_index_ returned is not an integer'):
+    clf = GridSearchCV(
+        LinearSVC(random_state=42),
+        {"C": [0.1, 1]},
+        scoring="precision",
+        refit=refit_callable_invalid_type,
+    )
+    with pytest.raises(TypeError, match="best_index_ returned is not an integer"):
         clf.fit(X, y)
 
 
-@pytest.mark.parametrize('out_bound_value', [-1, 2])
-@pytest.mark.parametrize('search_cv', [RandomizedSearchCV, GridSearchCV])
+@pytest.mark.parametrize("out_bound_value", [-1, 2])
+@pytest.mark.parametrize("search_cv", [RandomizedSearchCV, GridSearchCV])
 def test_refit_callable_out_bound(out_bound_value, search_cv):
     """
     Test implementation catches the errors when 'best_index_' returns an
     out of bound result.
     """
+
     def refit_callable_out_bound(cv_results):
         """
         A dummy function tests when returned 'best_index_' is out of bounds.
         """
         return out_bound_value
 
-    X, y = make_classification(n_samples=100, n_features=4,
-                               random_state=42)
+    X, y = make_classification(n_samples=100, n_features=4, random_state=42)
 
-    clf = search_cv(LinearSVC(random_state=42), {'C': [0.1, 1]},
-                    scoring='precision', refit=refit_callable_out_bound)
-    with pytest.raises(IndexError, match='best_index_ index out of range'):
+    clf = search_cv(
+        LinearSVC(random_state=42),
+        {"C": [0.1, 1]},
+        scoring="precision",
+        refit=refit_callable_out_bound,
+    )
+    with pytest.raises(IndexError, match="best_index_ index out of range"):
         clf.fit(X, y)
 
 
@@ -679,37 +740,48 @@ def test_refit_callable_multi_metric():
     """
     Test refit=callable in multiple metric evaluation setting
     """
+
     def refit_callable(cv_results):
         """
         A dummy function tests `refit=callable` interface.
         Return the index of a model that has the least
         `mean_test_prec`.
         """
-        assert 'mean_test_prec' in cv_results
-        return cv_results['mean_test_prec'].argmin()
-
-    X, y = make_classification(n_samples=100, n_features=4,
-                               random_state=42)
-    scoring = {'Accuracy': make_scorer(accuracy_score), 'prec': 'precision'}
-    clf = GridSearchCV(LinearSVC(random_state=42), {'C': [0.01, 0.1, 1]},
-                       scoring=scoring, refit=refit_callable)
+        assert "mean_test_prec" in cv_results
+        return cv_results["mean_test_prec"].argmin()
+
+    X, y = make_classification(n_samples=100, n_features=4, random_state=42)
+    scoring = {"Accuracy": make_scorer(accuracy_score), "prec": "precision"}
+    clf = GridSearchCV(
+        LinearSVC(random_state=42),
+        {"C": [0.01, 0.1, 1]},
+        scoring=scoring,
+        refit=refit_callable,
+    )
     clf.fit(X, y)
 
     assert clf.best_index_ == 0
     # Ensure `best_score_` is disabled when using `refit=callable`
-    assert not hasattr(clf, 'best_score_')
+    assert not hasattr(clf, "best_score_")
 
 
 def test_gridsearch_nd():
     # Pass X as list in GridSearchCV
     X_4d = np.arange(10 * 5 * 3 * 2).reshape(10, 5, 3, 2)
     y_3d = np.arange(10 * 7 * 11).reshape(10, 7, 11)
-    def check_X(x): return x.shape[1:] == (5, 3, 2)
-    def check_y(x): return x.shape[1:] == (7, 11)
+
+    def check_X(x):
+        return x.shape[1:] == (5, 3, 2)
+
+    def check_y(x):
+        return x.shape[1:] == (7, 11)
+
     clf = CheckingClassifier(
-        check_X=check_X, check_y=check_y, methods_to_check=["fit"],
+        check_X=check_X,
+        check_y=check_y,
+        methods_to_check=["fit"],
     )
-    grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]})
+    grid_search = GridSearchCV(clf, {"foo_param": [1, 2, 3]})
     grid_search.fit(X_4d, y_3d).score(X, y)
     assert hasattr(grid_search, "cv_results_")
 
@@ -720,10 +792,11 @@ def test_X_as_list():
     y = np.array([0] * 5 + [1] * 5)
 
     clf = CheckingClassifier(
-        check_X=lambda x: isinstance(x, list), methods_to_check=["fit"],
+        check_X=lambda x: isinstance(x, list),
+        methods_to_check=["fit"],
     )
     cv = KFold(n_splits=3)
-    grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]}, cv=cv)
+    grid_search = GridSearchCV(clf, {"foo_param": [1, 2, 3]}, cv=cv)
     grid_search.fit(X.tolist(), y).score(X, y)
     assert hasattr(grid_search, "cv_results_")
 
@@ -734,20 +807,21 @@ def test_y_as_list():
     y = np.array([0] * 5 + [1] * 5)
 
     clf = CheckingClassifier(
-        check_y=lambda x: isinstance(x, list), methods_to_check=["fit"],
+        check_y=lambda x: isinstance(x, list),
+        methods_to_check=["fit"],
     )
     cv = KFold(n_splits=3)
-    grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]}, cv=cv)
+    grid_search = GridSearchCV(clf, {"foo_param": [1, 2, 3]}, cv=cv)
     grid_search.fit(X, y.tolist()).score(X, y)
     assert hasattr(grid_search, "cv_results_")
 
 
-@ignore_warnings
 def test_pandas_input():
     # check cross_val_score doesn't destroy pandas dataframe
     types = [(MockDataFrame, MockDataFrame)]
     try:
-        from pandas import Series, DataFrame
+        from pandas import DataFrame, Series
+
         types.append((DataFrame, Series))
     except ImportError:
         pass
@@ -767,7 +841,7 @@ def check_series(x):
 
         clf = CheckingClassifier(check_X=check_df, check_y=check_series)
 
-        grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]})
+        grid_search = GridSearchCV(clf, {"foo_param": [1, 2, 3]})
         grid_search.fit(X_df, y_ser).score(X_df, y_ser)
         grid_search.predict(X_df)
         assert hasattr(grid_search, "cv_results_")
@@ -779,17 +853,19 @@ def test_unsupervised_grid_search():
     km = KMeans(random_state=0, init="random", n_init=1)
 
     # Multi-metric evaluation unsupervised
-    scoring = ['adjusted_rand_score', 'fowlkes_mallows_score']
-    for refit in ['adjusted_rand_score', 'fowlkes_mallows_score']:
-        grid_search = GridSearchCV(km, param_grid=dict(n_clusters=[2, 3, 4]),
-                                   scoring=scoring, refit=refit)
+    scoring = ["adjusted_rand_score", "fowlkes_mallows_score"]
+    for refit in ["adjusted_rand_score", "fowlkes_mallows_score"]:
+        grid_search = GridSearchCV(
+            km, param_grid=dict(n_clusters=[2, 3, 4]), scoring=scoring, refit=refit
+        )
         grid_search.fit(X, y)
         # Both ARI and FMS can find the right number :)
         assert grid_search.best_params_["n_clusters"] == 3
 
     # Single metric evaluation unsupervised
-    grid_search = GridSearchCV(km, param_grid=dict(n_clusters=[2, 3, 4]),
-                               scoring='fowlkes_mallows_score')
+    grid_search = GridSearchCV(
+        km, param_grid=dict(n_clusters=[2, 3, 4]), scoring="fowlkes_mallows_score"
+    )
     grid_search.fit(X, y)
     assert grid_search.best_params_["n_clusters"] == 3
 
@@ -803,23 +879,25 @@ def test_gridsearch_no_predict():
     # test grid-search with an estimator without predict.
     # slight duplication of a test from KDE
     def custom_scoring(estimator, X):
-        return 42 if estimator.bandwidth == .1 else 0
-    X, _ = make_blobs(cluster_std=.1, random_state=1,
-                      centers=[[0, 1], [1, 0], [0, 0]])
-    search = GridSearchCV(KernelDensity(),
-                          param_grid=dict(bandwidth=[.01, .1, 1]),
-                          scoring=custom_scoring)
+        return 42 if estimator.bandwidth == 0.1 else 0
+
+    X, _ = make_blobs(cluster_std=0.1, random_state=1, centers=[[0, 1], [1, 0], [0, 0]])
+    search = GridSearchCV(
+        KernelDensity(),
+        param_grid=dict(bandwidth=[0.01, 0.1, 1]),
+        scoring=custom_scoring,
+    )
     search.fit(X)
-    assert search.best_params_['bandwidth'] == .1
+    assert search.best_params_["bandwidth"] == 0.1
     assert search.best_score_ == 42
 
 
 def test_param_sampler():
     # test basic properties of param sampler
-    param_distributions = {"kernel": ["rbf", "linear"],
-                           "C": uniform(0, 1)}
-    sampler = ParameterSampler(param_distributions=param_distributions,
-                               n_iter=10, random_state=0)
+    param_distributions = {"kernel": ["rbf", "linear"], "C": uniform(0, 1)}
+    sampler = ParameterSampler(
+        param_distributions=param_distributions, n_iter=10, random_state=0
+    )
     samples = [x for x in sampler]
     assert len(samples) == 10
     for sample in samples:
@@ -828,145 +906,223 @@ def test_param_sampler():
 
     # test that repeated calls yield identical parameters
     param_distributions = {"C": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}
-    sampler = ParameterSampler(param_distributions=param_distributions,
-                               n_iter=3, random_state=0)
+    sampler = ParameterSampler(
+        param_distributions=param_distributions, n_iter=3, random_state=0
+    )
     assert [x for x in sampler] == [x for x in sampler]
 
     param_distributions = {"C": uniform(0, 1)}
-    sampler = ParameterSampler(param_distributions=param_distributions,
-                               n_iter=10, random_state=0)
+    sampler = ParameterSampler(
+        param_distributions=param_distributions, n_iter=10, random_state=0
+    )
     assert [x for x in sampler] == [x for x in sampler]
 
 
-def check_cv_results_array_types(search, param_keys, score_keys):
+def check_cv_results_array_types(
+    search, param_keys, score_keys, expected_cv_results_kinds
+):
     # Check if the search `cv_results`'s array are of correct types
     cv_results = search.cv_results_
-    assert all(isinstance(cv_results[param], np.ma.MaskedArray)
-               for param in param_keys)
-    assert all(cv_results[key].dtype == object for key in param_keys)
-    assert not any(isinstance(cv_results[key], np.ma.MaskedArray)
-                   for key in score_keys)
-    assert all(cv_results[key].dtype == np.float64
-               for key in score_keys if not key.startswith('rank'))
+    assert all(isinstance(cv_results[param], np.ma.MaskedArray) for param in param_keys)
+    assert {
+        key: cv_results[key].dtype.kind for key in param_keys
+    } == expected_cv_results_kinds
+    assert not any(isinstance(cv_results[key], np.ma.MaskedArray) for key in score_keys)
+    assert all(
+        cv_results[key].dtype == np.float64
+        for key in score_keys
+        if not key.startswith("rank")
+    )
 
-    scorer_keys = search.scorer_.keys() if search.multimetric_ else ['score']
+    scorer_keys = search.scorer_.keys() if search.multimetric_ else ["score"]
 
     for key in scorer_keys:
-        assert cv_results['rank_test_%s' % key].dtype == np.int32
+        assert cv_results["rank_test_%s" % key].dtype == np.int32
 
 
-def check_cv_results_keys(cv_results, param_keys, score_keys, n_cand):
+def check_cv_results_keys(cv_results, param_keys, score_keys, n_cand, extra_keys=()):
     # Test the search.cv_results_ contains all the required results
-    assert_array_equal(sorted(cv_results.keys()),
-                       sorted(param_keys + score_keys + ('params',)))
-    assert all(cv_results[key].shape == (n_cand,)
-               for key in param_keys + score_keys)
+    all_keys = param_keys + score_keys + extra_keys
+    assert_array_equal(sorted(cv_results.keys()), sorted(all_keys + ("params",)))
+    assert all(cv_results[key].shape == (n_cand,) for key in param_keys + score_keys)
 
 
 def test_grid_search_cv_results():
-    X, y = make_classification(n_samples=50, n_features=4,
-                               random_state=42)
+    X, y = make_classification(n_samples=50, n_features=4, random_state=42)
 
-    n_splits = 3
     n_grid_points = 6
-    params = [dict(kernel=['rbf', ], C=[1, 10], gamma=[0.1, 1]),
-              dict(kernel=['poly', ], degree=[1, 2])]
-
-    param_keys = ('param_C', 'param_degree', 'param_gamma', 'param_kernel')
-    score_keys = ('mean_test_score', 'mean_train_score',
-                  'rank_test_score',
-                  'split0_test_score', 'split1_test_score',
-                  'split2_test_score',
-                  'split0_train_score', 'split1_train_score',
-                  'split2_train_score',
-                  'std_test_score', 'std_train_score',
-                  'mean_fit_time', 'std_fit_time',
-                  'mean_score_time', 'std_score_time')
+    params = [
+        dict(
+            kernel=[
+                "rbf",
+            ],
+            C=[1, 10],
+            gamma=[0.1, 1],
+        ),
+        dict(
+            kernel=[
+                "poly",
+            ],
+            degree=[1, 2],
+        ),
+    ]
+
+    param_keys = ("param_C", "param_degree", "param_gamma", "param_kernel")
+    score_keys = (
+        "mean_test_score",
+        "mean_train_score",
+        "rank_test_score",
+        "split0_test_score",
+        "split1_test_score",
+        "split2_test_score",
+        "split0_train_score",
+        "split1_train_score",
+        "split2_train_score",
+        "std_test_score",
+        "std_train_score",
+        "mean_fit_time",
+        "std_fit_time",
+        "mean_score_time",
+        "std_score_time",
+    )
     n_candidates = n_grid_points
 
-    search = GridSearchCV(SVC(), cv=n_splits, param_grid=params,
-                          return_train_score=True)
+    search = GridSearchCV(SVC(), cv=3, param_grid=params, return_train_score=True)
     search.fit(X, y)
     cv_results = search.cv_results_
     # Check if score and timing are reasonable
-    assert all(cv_results['rank_test_score'] >= 1)
-    assert (all(cv_results[k] >= 0) for k in score_keys
-            if k != 'rank_test_score')
-    assert (all(cv_results[k] <= 1) for k in score_keys
-            if 'time' not in k and
-            k != 'rank_test_score')
+    assert all(cv_results["rank_test_score"] >= 1)
+    assert (all(cv_results[k] >= 0) for k in score_keys if k != "rank_test_score")
+    assert (
+        all(cv_results[k] <= 1)
+        for k in score_keys
+        if "time" not in k and k != "rank_test_score"
+    )
     # Check cv_results structure
-    check_cv_results_array_types(search, param_keys, score_keys)
+    expected_cv_results_kinds = {
+        "param_C": "i",
+        "param_degree": "i",
+        "param_gamma": "f",
+        "param_kernel": "O",
+    }
+    check_cv_results_array_types(
+        search, param_keys, score_keys, expected_cv_results_kinds
+    )
     check_cv_results_keys(cv_results, param_keys, score_keys, n_candidates)
     # Check masking
     cv_results = search.cv_results_
-    n_candidates = len(search.cv_results_['params'])
-    assert all((cv_results['param_C'].mask[i] and
-                cv_results['param_gamma'].mask[i] and
-                not cv_results['param_degree'].mask[i])
-               for i in range(n_candidates)
-               if cv_results['param_kernel'][i] == 'linear')
-    assert all((not cv_results['param_C'].mask[i] and
-                not cv_results['param_gamma'].mask[i] and
-                cv_results['param_degree'].mask[i])
-               for i in range(n_candidates)
-               if cv_results['param_kernel'][i] == 'rbf')
+
+    poly_results = [
+        (
+            cv_results["param_C"].mask[i]
+            and cv_results["param_gamma"].mask[i]
+            and not cv_results["param_degree"].mask[i]
+        )
+        for i in range(n_candidates)
+        if cv_results["param_kernel"][i] == "poly"
+    ]
+    assert all(poly_results)
+    assert len(poly_results) == 2
+
+    rbf_results = [
+        (
+            not cv_results["param_C"].mask[i]
+            and not cv_results["param_gamma"].mask[i]
+            and cv_results["param_degree"].mask[i]
+        )
+        for i in range(n_candidates)
+        if cv_results["param_kernel"][i] == "rbf"
+    ]
+    assert all(rbf_results)
+    assert len(rbf_results) == 4
 
 
 def test_random_search_cv_results():
     X, y = make_classification(n_samples=50, n_features=4, random_state=42)
 
-    n_splits = 3
     n_search_iter = 30
 
-    params = [{'kernel': ['rbf'], 'C': expon(scale=10),
-               'gamma': expon(scale=0.1)},
-              {'kernel': ['poly'], 'degree': [2, 3]}]
-    param_keys = ('param_C', 'param_degree', 'param_gamma', 'param_kernel')
-    score_keys = ('mean_test_score', 'mean_train_score',
-                  'rank_test_score',
-                  'split0_test_score', 'split1_test_score',
-                  'split2_test_score',
-                  'split0_train_score', 'split1_train_score',
-                  'split2_train_score',
-                  'std_test_score', 'std_train_score',
-                  'mean_fit_time', 'std_fit_time',
-                  'mean_score_time', 'std_score_time')
-    n_cand = n_search_iter
-
-    search = RandomizedSearchCV(SVC(), n_iter=n_search_iter,
-                                cv=n_splits,
-                                param_distributions=params,
-                                return_train_score=True)
+    params = [
+        {"kernel": ["rbf"], "C": expon(scale=10), "gamma": expon(scale=0.1)},
+        {"kernel": ["poly"], "degree": [2, 3]},
+    ]
+    param_keys = ("param_C", "param_degree", "param_gamma", "param_kernel")
+    score_keys = (
+        "mean_test_score",
+        "mean_train_score",
+        "rank_test_score",
+        "split0_test_score",
+        "split1_test_score",
+        "split2_test_score",
+        "split0_train_score",
+        "split1_train_score",
+        "split2_train_score",
+        "std_test_score",
+        "std_train_score",
+        "mean_fit_time",
+        "std_fit_time",
+        "mean_score_time",
+        "std_score_time",
+    )
+    n_candidates = n_search_iter
+
+    search = RandomizedSearchCV(
+        SVC(),
+        n_iter=n_search_iter,
+        cv=3,
+        param_distributions=params,
+        return_train_score=True,
+    )
     search.fit(X, y)
     cv_results = search.cv_results_
     # Check results structure
-    check_cv_results_array_types(search, param_keys, score_keys)
-    check_cv_results_keys(cv_results, param_keys, score_keys, n_cand)
-    n_candidates = len(search.cv_results_['params'])
-    assert all((cv_results['param_C'].mask[i] and
-                cv_results['param_gamma'].mask[i] and
-                not cv_results['param_degree'].mask[i])
-               for i in range(n_candidates)
-               if cv_results['param_kernel'][i] == 'linear')
-    assert all((not cv_results['param_C'].mask[i] and
-                not cv_results['param_gamma'].mask[i] and
-                cv_results['param_degree'].mask[i])
-               for i in range(n_candidates)
-               if cv_results['param_kernel'][i] == 'rbf')
+    expected_cv_results_kinds = {
+        "param_C": "f",
+        "param_degree": "i",
+        "param_gamma": "f",
+        "param_kernel": "O",
+    }
+    check_cv_results_array_types(
+        search, param_keys, score_keys, expected_cv_results_kinds
+    )
+    check_cv_results_keys(cv_results, param_keys, score_keys, n_candidates)
+    assert all(
+        (
+            cv_results["param_C"].mask[i]
+            and cv_results["param_gamma"].mask[i]
+            and not cv_results["param_degree"].mask[i]
+        )
+        for i in range(n_candidates)
+        if cv_results["param_kernel"][i] == "poly"
+    )
+    assert all(
+        (
+            not cv_results["param_C"].mask[i]
+            and not cv_results["param_gamma"].mask[i]
+            and cv_results["param_degree"].mask[i]
+        )
+        for i in range(n_candidates)
+        if cv_results["param_kernel"][i] == "rbf"
+    )
 
 
 @pytest.mark.parametrize(
     "SearchCV, specialized_params",
-    [(GridSearchCV, {'param_grid': {'C': [1, 10]}}),
-     (RandomizedSearchCV,
-      {'param_distributions': {'C': [1, 10]}, 'n_iter': 2})]
+    [
+        (GridSearchCV, {"param_grid": {"C": [1, 10]}}),
+        (RandomizedSearchCV, {"param_distributions": {"C": [1, 10]}, "n_iter": 2}),
+    ],
 )
 def test_search_default_iid(SearchCV, specialized_params):
     # Test the IID parameter  TODO: Clearly this test does something else???
     # noise-free simple 2d-data
-    X, y = make_blobs(centers=[[0, 0], [1, 0], [0, 1], [1, 1]], random_state=0,
-                      cluster_std=0.1, shuffle=False, n_samples=80)
+    X, y = make_blobs(
+        centers=[[0, 0], [1, 0], [0, 1], [1, 1]],
+        random_state=0,
+        cluster_std=0.1,
+        shuffle=False,
+        n_samples=80,
+    )
     # split dataset into two folds that are not iid
     # first one contains data of all 4 blobs, second only from two.
     mask = np.ones(X.shape[0], dtype=bool)
@@ -977,28 +1133,31 @@ def test_search_default_iid(SearchCV, specialized_params):
     # create "cv" for splits
     cv = [[mask, ~mask], [~mask, mask]]
 
-    common_params = {'estimator': SVC(), 'cv': cv,
-                     'return_train_score': True}
+    common_params = {"estimator": SVC(), "cv": cv, "return_train_score": True}
     search = SearchCV(**common_params, **specialized_params)
     search.fit(X, y)
 
     test_cv_scores = np.array(
-        [search.cv_results_['split%d_test_score' % s][0]
-         for s in range(search.n_splits_)]
+        [
+            search.cv_results_["split%d_test_score" % s][0]
+            for s in range(search.n_splits_)
+        ]
     )
-    test_mean = search.cv_results_['mean_test_score'][0]
-    test_std = search.cv_results_['std_test_score'][0]
+    test_mean = search.cv_results_["mean_test_score"][0]
+    test_std = search.cv_results_["std_test_score"][0]
 
     train_cv_scores = np.array(
-        [search.cv_results_['split%d_train_score' % s][0]
-         for s in range(search.n_splits_)]
+        [
+            search.cv_results_["split%d_train_score" % s][0]
+            for s in range(search.n_splits_)
+        ]
     )
-    train_mean = search.cv_results_['mean_train_score'][0]
-    train_std = search.cv_results_['std_train_score'][0]
+    train_mean = search.cv_results_["mean_train_score"][0]
+    train_std = search.cv_results_["std_train_score"][0]
 
-    assert search.cv_results_['param_C'][0] == 1
+    assert search.cv_results_["param_C"][0] == 1
     # scores are the same as above
-    assert_allclose(test_cv_scores, [1, 1. / 3.])
+    assert_allclose(test_cv_scores, [1, 1.0 / 3.0])
     assert_allclose(train_cv_scores, [1, 1])
     # Unweighted mean/std is used
     assert test_mean == pytest.approx(np.mean(test_cv_scores))
@@ -1014,16 +1173,31 @@ def test_grid_search_cv_results_multimetric():
     X, y = make_classification(n_samples=50, n_features=4, random_state=42)
 
     n_splits = 3
-    params = [dict(kernel=['rbf', ], C=[1, 10], gamma=[0.1, 1]),
-              dict(kernel=['poly', ], degree=[1, 2])]
+    params = [
+        dict(
+            kernel=[
+                "rbf",
+            ],
+            C=[1, 10],
+            gamma=[0.1, 1],
+        ),
+        dict(
+            kernel=[
+                "poly",
+            ],
+            degree=[1, 2],
+        ),
+    ]
 
     grid_searches = []
-    for scoring in ({'accuracy': make_scorer(accuracy_score),
-                     'recall': make_scorer(recall_score)},
-                    'accuracy', 'recall'):
-        grid_search = GridSearchCV(SVC(), cv=n_splits,
-                                   param_grid=params,
-                                   scoring=scoring, refit=False)
+    for scoring in (
+        {"accuracy": make_scorer(accuracy_score), "recall": make_scorer(recall_score)},
+        "accuracy",
+        "recall",
+    ):
+        grid_search = GridSearchCV(
+            SVC(), cv=n_splits, param_grid=params, scoring=scoring, refit=False
+        )
         grid_search.fit(X, y)
         grid_searches.append(grid_search)
 
@@ -1037,106 +1211,240 @@ def test_random_search_cv_results_multimetric():
     n_search_iter = 30
 
     # Scipy 0.12's stats dists do not accept seed, hence we use param grid
-    params = dict(C=np.logspace(-4, 1, 3),
-                  gamma=np.logspace(-5, 0, 3, base=0.1))
+    params = dict(C=np.logspace(-4, 1, 3), gamma=np.logspace(-5, 0, 3, base=0.1))
     for refit in (True, False):
         random_searches = []
-        for scoring in (('accuracy', 'recall'), 'accuracy', 'recall'):
+        for scoring in (("accuracy", "recall"), "accuracy", "recall"):
             # If True, for multi-metric pass refit='accuracy'
             if refit:
                 probability = True
-                refit = 'accuracy' if isinstance(scoring, tuple) else refit
+                refit = "accuracy" if isinstance(scoring, tuple) else refit
             else:
                 probability = False
             clf = SVC(probability=probability, random_state=42)
-            random_search = RandomizedSearchCV(clf, n_iter=n_search_iter,
-                                               cv=n_splits,
-                                               param_distributions=params,
-                                               scoring=scoring,
-                                               refit=refit, random_state=0)
+            random_search = RandomizedSearchCV(
+                clf,
+                n_iter=n_search_iter,
+                cv=n_splits,
+                param_distributions=params,
+                scoring=scoring,
+                refit=refit,
+                random_state=0,
+            )
             random_search.fit(X, y)
             random_searches.append(random_search)
 
         compare_cv_results_multimetric_with_single(*random_searches)
         compare_refit_methods_when_refit_with_acc(
-            random_searches[0], random_searches[1], refit)
+            random_searches[0], random_searches[1], refit
+        )
 
 
-def compare_cv_results_multimetric_with_single(
-        search_multi, search_acc, search_rec):
+def compare_cv_results_multimetric_with_single(search_multi, search_acc, search_rec):
     """Compare multi-metric cv_results with the ensemble of multiple
     single metric cv_results from single metric grid/random search"""
 
     assert search_multi.multimetric_
-    assert_array_equal(sorted(search_multi.scorer_),
-                       ('accuracy', 'recall'))
+    assert_array_equal(sorted(search_multi.scorer_), ("accuracy", "recall"))
 
     cv_results_multi = search_multi.cv_results_
-    cv_results_acc_rec = {re.sub('_score$', '_accuracy', k): v
-                          for k, v in search_acc.cv_results_.items()}
-    cv_results_acc_rec.update({re.sub('_score$', '_recall', k): v
-                               for k, v in search_rec.cv_results_.items()})
+    cv_results_acc_rec = {
+        re.sub("_score$", "_accuracy", k): v for k, v in search_acc.cv_results_.items()
+    }
+    cv_results_acc_rec.update(
+        {re.sub("_score$", "_recall", k): v for k, v in search_rec.cv_results_.items()}
+    )
 
     # Check if score and timing are reasonable, also checks if the keys
     # are present
-    assert all((np.all(cv_results_multi[k] <= 1) for k in (
-                    'mean_score_time', 'std_score_time', 'mean_fit_time',
-                    'std_fit_time')))
+    assert all(
+        (
+            np.all(cv_results_multi[k] <= 1)
+            for k in (
+                "mean_score_time",
+                "std_score_time",
+                "mean_fit_time",
+                "std_fit_time",
+            )
+        )
+    )
 
     # Compare the keys, other than time keys, among multi-metric and
     # single metric grid search results. np.testing.assert_equal performs a
     # deep nested comparison of the two cv_results dicts
-    np.testing.assert_equal({k: v for k, v in cv_results_multi.items()
-                             if not k.endswith('_time')},
-                            {k: v for k, v in cv_results_acc_rec.items()
-                             if not k.endswith('_time')})
+    np.testing.assert_equal(
+        {k: v for k, v in cv_results_multi.items() if not k.endswith("_time")},
+        {k: v for k, v in cv_results_acc_rec.items() if not k.endswith("_time")},
+    )
 
 
 def compare_refit_methods_when_refit_with_acc(search_multi, search_acc, refit):
     """Compare refit multi-metric search methods with single metric methods"""
     assert search_acc.refit == refit
     if refit:
-        assert search_multi.refit == 'accuracy'
+        assert search_multi.refit == "accuracy"
     else:
         assert not search_multi.refit
         return  # search cannot predict/score without refit
 
     X, y = make_blobs(n_samples=100, n_features=4, random_state=42)
-    for method in ('predict', 'predict_proba', 'predict_log_proba'):
-        assert_almost_equal(getattr(search_multi, method)(X),
-                            getattr(search_acc, method)(X))
+    for method in ("predict", "predict_proba", "predict_log_proba"):
+        assert_almost_equal(
+            getattr(search_multi, method)(X), getattr(search_acc, method)(X)
+        )
     assert_almost_equal(search_multi.score(X, y), search_acc.score(X, y))
-    for key in ('best_index_', 'best_score_', 'best_params_'):
+    for key in ("best_index_", "best_score_", "best_params_"):
         assert getattr(search_multi, key) == getattr(search_acc, key)
 
 
-@pytest.mark.parametrize('search_cv', [
-    RandomizedSearchCV(estimator=DecisionTreeClassifier(),
-                       param_distributions={'max_depth': [5, 10]}),
-    GridSearchCV(estimator=DecisionTreeClassifier(),
-                 param_grid={'max_depth': [5, 10]})
-])
+@pytest.mark.parametrize(
+    "search_cv",
+    [
+        RandomizedSearchCV(
+            estimator=DecisionTreeClassifier(),
+            param_distributions={"max_depth": [5, 10]},
+        ),
+        GridSearchCV(
+            estimator=DecisionTreeClassifier(), param_grid={"max_depth": [5, 10]}
+        ),
+    ],
+)
 def test_search_cv_score_samples_error(search_cv):
     X, y = make_blobs(n_samples=100, n_features=4, random_state=42)
     search_cv.fit(X, y)
 
     # Make sure to error out when underlying estimator does not implement
     # the method `score_samples`
-    err_msg = ("'DecisionTreeClassifier' object has no attribute "
-               "'score_samples'")
+    outer_msg = f"'{search_cv.__class__.__name__}' has no attribute 'score_samples'"
+    inner_msg = "'DecisionTreeClassifier' object has no attribute 'score_samples'"
 
-    with pytest.raises(AttributeError, match=err_msg):
+    with pytest.raises(AttributeError, match=outer_msg) as exec_info:
         search_cv.score_samples(X)
+    assert isinstance(exec_info.value.__cause__, AttributeError)
+    assert inner_msg == str(exec_info.value.__cause__)
+
+
+def test_unsupported_sample_weight_scorer():
+    """Checks that fitting with sample_weight raises a warning if the scorer does not
+    support sample_weight"""
+
+    def fake_score_func(y_true, y_pred):
+        "Fake scoring function that does not support sample_weight"
+        return 0.5
+
+    fake_scorer = make_scorer(fake_score_func)
+
+    X, y = make_classification(n_samples=10, n_features=4, random_state=42)
+    sw = np.ones_like(y)
+    search_cv = GridSearchCV(estimator=LogisticRegression(), param_grid={"C": [1, 10]})
+    # function
+    search_cv.set_params(scoring=fake_score_func)
+    with pytest.warns(UserWarning, match="does not support sample_weight"):
+        search_cv.fit(X, y, sample_weight=sw)
+    # scorer
+    search_cv.set_params(scoring=fake_scorer)
+    with pytest.warns(UserWarning, match="does not support sample_weight"):
+        search_cv.fit(X, y, sample_weight=sw)
+    # multi-metric evaluation
+    search_cv.set_params(
+        scoring=dict(fake=fake_scorer, accuracy="accuracy"), refit=False
+    )
+    # only fake scorer does not support sample_weight
+    with pytest.warns(
+        UserWarning, match=r"The scoring fake=.* does not support sample_weight"
+    ):
+        search_cv.fit(X, y, sample_weight=sw)
+
+
+@pytest.mark.parametrize(
+    "estimator",
+    [
+        GridSearchCV(estimator=LogisticRegression(), param_grid={"C": [1, 10, 100]}),
+        RandomizedSearchCV(
+            estimator=Ridge(), param_distributions={"alpha": [1, 0.1, 0.01]}
+        ),
+    ],
+)
+def test_search_cv_sample_weight_equivalence(estimator):
+    estimator_weighted = clone(estimator)
+    estimator_repeated = clone(estimator)
+    set_random_state(estimator_weighted, random_state=0)
+    set_random_state(estimator_repeated, random_state=0)
 
+    rng = np.random.RandomState(42)
+    n_classes = 3
+    n_samples_per_group = 30
+    n_groups = 4
+    n_samples = n_groups * n_samples_per_group
+    X = rng.rand(n_samples, n_samples * 2)
+    y = rng.randint(0, n_classes, size=n_samples)
+    sw = rng.randint(0, 5, size=n_samples)
+    # we use groups with LeaveOneGroupOut to ensure that
+    # the splits are the same in the repeated/weighted datasets
+    groups = np.tile(np.arange(n_groups), n_samples_per_group)
+
+    X_weighted = X
+    y_weighted = y
+    groups_weighted = groups
+    splits_weighted = list(LeaveOneGroupOut().split(X_weighted, groups=groups_weighted))
+    estimator_weighted.set_params(cv=splits_weighted)
+    # repeat samples according to weights
+    X_repeated = X_weighted.repeat(repeats=sw, axis=0)
+    y_repeated = y_weighted.repeat(repeats=sw)
+    groups_repeated = groups_weighted.repeat(repeats=sw)
+    splits_repeated = list(LeaveOneGroupOut().split(X_repeated, groups=groups_repeated))
+    estimator_repeated.set_params(cv=splits_repeated)
+
+    y_weighted = _enforce_estimator_tags_y(estimator_weighted, y_weighted)
+    y_repeated = _enforce_estimator_tags_y(estimator_repeated, y_repeated)
+
+    estimator_repeated.fit(X_repeated, y=y_repeated, sample_weight=None)
+    estimator_weighted.fit(X_weighted, y=y_weighted, sample_weight=sw)
+
+    # check that scores stored in cv_results_
+    # are equal for the weighted/repeated datasets
+    score_keys = [
+        key for key in estimator_repeated.cv_results_ if key.endswith("score")
+    ]
+    for key in score_keys:
+        s1 = estimator_repeated.cv_results_[key]
+        s2 = estimator_weighted.cv_results_[key]
+        err_msg = f"{key} values are not equal for weighted/repeated datasets"
+        assert_allclose(s1, s2, err_msg=err_msg)
+
+    for key in ["best_score_", "best_index_"]:
+        s1 = getattr(estimator_repeated, key)
+        s2 = getattr(estimator_weighted, key)
+        err_msg = f"{key} values are not equal for weighted/repeated datasets"
+        assert_almost_equal(s1, s2, err_msg=err_msg)
+
+    for method in ["predict_proba", "decision_function", "predict", "transform"]:
+        if hasattr(estimator, method):
+            s1 = getattr(estimator_repeated, method)(X)
+            s2 = getattr(estimator_weighted, method)(X)
+            err_msg = (
+                f"Comparing the output of {method} revealed that fitting "
+                "with `sample_weight` is not equivalent to fitting with removed "
+                "or repeated data points."
+            )
+            assert_allclose_dense_sparse(s1, s2, err_msg=err_msg)
 
-@pytest.mark.parametrize('search_cv', [
-    RandomizedSearchCV(estimator=LocalOutlierFactor(novelty=True),
-                       param_distributions={'n_neighbors': [5, 10]},
-                       scoring="precision"),
-    GridSearchCV(estimator=LocalOutlierFactor(novelty=True),
-                 param_grid={'n_neighbors': [5, 10]},
-                 scoring="precision")
-])
+
+@pytest.mark.parametrize(
+    "search_cv",
+    [
+        RandomizedSearchCV(
+            estimator=LocalOutlierFactor(novelty=True),
+            param_distributions={"n_neighbors": [5, 10]},
+            scoring="precision",
+        ),
+        GridSearchCV(
+            estimator=LocalOutlierFactor(novelty=True),
+            param_grid={"n_neighbors": [5, 10]},
+            scoring="precision",
+        ),
+    ],
+)
 def test_search_cv_score_samples_method(search_cv):
     # Set parameters
     rng = np.random.RandomState(42)
@@ -1146,11 +1454,15 @@ def test_search_cv_score_samples_method(search_cv):
     n_inliers = n_samples - n_outliers
 
     # Create dataset
-    X = make_blobs(n_samples=n_inliers, n_features=2, centers=[[0, 0], [0, 0]],
-                   cluster_std=0.5, random_state=0)[0]
+    X = make_blobs(
+        n_samples=n_inliers,
+        n_features=2,
+        centers=[[0, 0], [0, 0]],
+        cluster_std=0.5,
+        random_state=0,
+    )[0]
     # Add some noisy points
-    X = np.concatenate([X, rng.uniform(low=-6, high=6,
-                                       size=(n_outliers, 2))], axis=0)
+    X = np.concatenate([X, rng.uniform(low=-6, high=6, size=(n_outliers, 2))], axis=0)
 
     # Define labels to be able to score the estimator with `search_cv`
     y_true = np.array([1] * n_samples)
@@ -1161,8 +1473,9 @@ def test_search_cv_score_samples_method(search_cv):
 
     # Verify that the stand alone estimator yields the same results
     # as the ones obtained with *SearchCV
-    assert_allclose(search_cv.score_samples(X),
-                    search_cv.best_estimator_.score_samples(X))
+    assert_allclose(
+        search_cv.score_samples(X), search_cv.best_estimator_.score_samples(X)
+    )
 
 
 def test_search_cv_results_rank_tie_breaking():
@@ -1170,13 +1483,12 @@ def test_search_cv_results_rank_tie_breaking():
 
     # The two C values are close enough to give similar models
     # which would result in a tie of their mean cv-scores
-    param_grid = {'C': [1, 1.001, 0.001]}
+    param_grid = {"C": [1, 1.001, 0.001]}
 
-    grid_search = GridSearchCV(SVC(), param_grid=param_grid,
-                               return_train_score=True)
-    random_search = RandomizedSearchCV(SVC(), n_iter=3,
-                                       param_distributions=param_grid,
-                                       return_train_score=True)
+    grid_search = GridSearchCV(SVC(), param_grid=param_grid, return_train_score=True)
+    random_search = RandomizedSearchCV(
+        SVC(), n_iter=3, param_distributions=param_grid, return_train_score=True
+    )
 
     for search in (grid_search, random_search):
         search.fit(X, y)
@@ -1184,16 +1496,20 @@ def test_search_cv_results_rank_tie_breaking():
         # Check tie breaking strategy -
         # Check that there is a tie in the mean scores between
         # candidates 1 and 2 alone
-        assert_almost_equal(cv_results['mean_test_score'][0],
-                            cv_results['mean_test_score'][1])
-        assert_almost_equal(cv_results['mean_train_score'][0],
-                            cv_results['mean_train_score'][1])
-        assert not np.allclose(cv_results['mean_test_score'][1],
-                               cv_results['mean_test_score'][2])
-        assert not np.allclose(cv_results['mean_train_score'][1],
-                               cv_results['mean_train_score'][2])
+        assert_almost_equal(
+            cv_results["mean_test_score"][0], cv_results["mean_test_score"][1]
+        )
+        assert_almost_equal(
+            cv_results["mean_train_score"][0], cv_results["mean_train_score"][1]
+        )
+        assert not np.allclose(
+            cv_results["mean_test_score"][1], cv_results["mean_test_score"][2]
+        )
+        assert not np.allclose(
+            cv_results["mean_train_score"][1], cv_results["mean_train_score"][2]
+        )
         # 'min' rank should be assigned to the tied candidates
-        assert_almost_equal(search.cv_results_['rank_test_score'], [1, 1, 3])
+        assert_almost_equal(search.cv_results_["rank_test_score"], [1, 1, 3])
 
 
 def test_search_cv_results_none_param():
@@ -1203,31 +1519,46 @@ def test_search_cv_results_none_param():
     cv = KFold()
 
     for est in estimators:
-        grid_search = GridSearchCV(est, est_parameters, cv=cv,
-                                   ).fit(X, y)
-        assert_array_equal(grid_search.cv_results_['param_random_state'],
-                           [0, None])
+        grid_search = GridSearchCV(
+            est,
+            est_parameters,
+            cv=cv,
+        ).fit(X, y)
+        assert_array_equal(grid_search.cv_results_["param_random_state"], [0, None])
 
 
-@ignore_warnings()
+@pytest.mark.filterwarnings("ignore::sklearn.exceptions.FitFailedWarning")
 def test_search_cv_timing():
     svc = LinearSVC(random_state=0)
 
-    X = [[1, ], [2, ], [3, ], [4, ]]
+    X = [
+        [
+            1,
+        ],
+        [
+            2,
+        ],
+        [
+            3,
+        ],
+        [
+            4,
+        ],
+    ]
     y = [0, 1, 1, 0]
 
-    gs = GridSearchCV(svc, {'C': [0, 1]}, cv=2, error_score=0)
-    rs = RandomizedSearchCV(svc, {'C': [0, 1]}, cv=2, error_score=0, n_iter=2)
+    gs = GridSearchCV(svc, {"C": [0, 1]}, cv=2, error_score=0)
+    rs = RandomizedSearchCV(svc, {"C": [0, 1]}, cv=2, error_score=0, n_iter=2)
 
     for search in (gs, rs):
         search.fit(X, y)
-        for key in ['mean_fit_time', 'std_fit_time']:
+        for key in ["mean_fit_time", "std_fit_time"]:
             # NOTE The precision of time.time in windows is not high
             # enough for the fit/score times to be non-zero for trivial X and y
             assert np.all(search.cv_results_[key] >= 0)
             assert np.all(search.cv_results_[key] < 1)
 
-        for key in ['mean_score_time', 'std_score_time']:
+        for key in ["mean_score_time", "std_score_time"]:
             assert search.cv_results_[key][1] >= 0
             assert search.cv_results_[key][0] == 0.0
             assert np.all(search.cv_results_[key] < 1)
@@ -1242,26 +1573,28 @@ def test_grid_search_correct_score_results():
     n_splits = 3
     clf = LinearSVC(random_state=0)
     X, y = make_blobs(random_state=0, centers=2)
-    Cs = [.1, 1, 10]
-    for score in ['f1', 'roc_auc']:
-        grid_search = GridSearchCV(clf, {'C': Cs}, scoring=score, cv=n_splits)
+    Cs = [0.1, 1, 10]
+    for score in ["f1", "roc_auc"]:
+        grid_search = GridSearchCV(clf, {"C": Cs}, scoring=score, cv=n_splits)
         cv_results = grid_search.fit(X, y).cv_results_
 
         # Test scorer names
         result_keys = list(cv_results.keys())
-        expected_keys = (("mean_test_score", "rank_test_score") +
-                         tuple("split%d_test_score" % cv_i
-                               for cv_i in range(n_splits)))
-        assert all(np.in1d(expected_keys, result_keys))
+        expected_keys = ("mean_test_score", "rank_test_score") + tuple(
+            "split%d_test_score" % cv_i for cv_i in range(n_splits)
+        )
+        assert all(np.isin(expected_keys, result_keys))
 
         cv = StratifiedKFold(n_splits=n_splits)
         n_splits = grid_search.n_splits_
         for candidate_i, C in enumerate(Cs):
             clf.set_params(C=C)
             cv_scores = np.array(
-                list(grid_search.cv_results_['split%d_test_score'
-                                             % s][candidate_i]
-                     for s in range(n_splits)))
+                [
+                    grid_search.cv_results_["split%d_test_score" % s][candidate_i]
+                    for s in range(n_splits)
+                ]
+            )
             for i, (train, test) in enumerate(cv.split(X, y)):
                 clf.fit(X[train], y[train])
                 if score == "f1":
@@ -1272,88 +1605,42 @@ def test_grid_search_correct_score_results():
                 assert_almost_equal(correct_score, cv_scores[i])
 
 
-# FIXME remove test_fit_grid_point as the function will be removed on 1.0
-@ignore_warnings(category=FutureWarning)
-def test_fit_grid_point():
-    X, y = make_classification(random_state=0)
-    cv = StratifiedKFold()
-    svc = LinearSVC(random_state=0)
-    scorer = make_scorer(accuracy_score)
-
-    for params in ({'C': 0.1}, {'C': 0.01}, {'C': 0.001}):
-        for train, test in cv.split(X, y):
-            this_scores, this_params, n_test_samples = fit_grid_point(
-                X, y, clone(svc), params, train, test,
-                scorer, verbose=False)
-
-            est = clone(svc).set_params(**params)
-            est.fit(X[train], y[train])
-            expected_score = scorer(est, X[test], y[test])
-
-            # Test the return values of fit_grid_point
-            assert_almost_equal(this_scores, expected_score)
-            assert params == this_params
-            assert n_test_samples == test.size
-
-    # Should raise an error upon multimetric scorer
-    error_msg = ("For evaluating multiple scores, use "
-                 "sklearn.model_selection.cross_validate instead.")
-    with pytest.raises(ValueError, match=error_msg):
-        fit_grid_point(
-            X, y, svc, params, train, test, {'score': scorer},
-            verbose=True
-        )
-
-
-# FIXME remove test_fit_grid_point_deprecated as
-# fit_grid_point will be removed on 1.0
-def test_fit_grid_point_deprecated():
-    X, y = make_classification(random_state=0)
-    svc = LinearSVC(random_state=0)
-    scorer = make_scorer(accuracy_score)
-    msg = ("fit_grid_point is deprecated in version 0.23 "
-           "and will be removed in version 1.0")
-    params = {'C': 0.1}
-    train, test = next(StratifiedKFold().split(X, y))
-
-    with pytest.warns(FutureWarning, match=msg):
-        fit_grid_point(X, y, svc, params, train, test, scorer, verbose=False)
-
-
 def test_pickle():
     # Test that a fit search can be pickled
     clf = MockClassifier()
-    grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]}, refit=True, cv=3)
+    grid_search = GridSearchCV(clf, {"foo_param": [1, 2, 3]}, refit=True, cv=2)
     grid_search.fit(X, y)
     grid_search_pickled = pickle.loads(pickle.dumps(grid_search))
-    assert_array_almost_equal(grid_search.predict(X),
-                              grid_search_pickled.predict(X))
+    assert_array_almost_equal(grid_search.predict(X), grid_search_pickled.predict(X))
 
-    random_search = RandomizedSearchCV(clf, {'foo_param': [1, 2, 3]},
-                                       refit=True, n_iter=3, cv=3)
+    random_search = RandomizedSearchCV(
+        clf, {"foo_param": [1, 2, 3]}, refit=True, n_iter=3, cv=2
+    )
     random_search.fit(X, y)
     random_search_pickled = pickle.loads(pickle.dumps(random_search))
-    assert_array_almost_equal(random_search.predict(X),
-                              random_search_pickled.predict(X))
+    assert_array_almost_equal(
+        random_search.predict(X), random_search_pickled.predict(X)
+    )
 
 
 def test_grid_search_with_multioutput_data():
     # Test search with multi-output estimator
 
-    X, y = make_multilabel_classification(return_indicator=True,
-                                          random_state=0)
+    X, y = make_multilabel_classification(return_indicator=True, random_state=0)
 
     est_parameters = {"max_depth": [1, 2, 3, 4]}
     cv = KFold()
 
-    estimators = [DecisionTreeRegressor(random_state=0),
-                  DecisionTreeClassifier(random_state=0)]
+    estimators = [
+        DecisionTreeRegressor(random_state=0),
+        DecisionTreeClassifier(random_state=0),
+    ]
 
     # Test with grid search cv
     for est in estimators:
         grid_search = GridSearchCV(est, est_parameters, cv=cv)
         grid_search.fit(X, y)
-        res_params = grid_search.cv_results_['params']
+        res_params = grid_search.cv_results_["params"]
         for cand_i in range(len(res_params)):
             est.set_params(**res_params[cand_i])
 
@@ -1362,14 +1649,14 @@ def test_grid_search_with_multioutput_data():
                 correct_score = est.score(X[test], y[test])
                 assert_almost_equal(
                     correct_score,
-                    grid_search.cv_results_['split%d_test_score' % i][cand_i])
+                    grid_search.cv_results_["split%d_test_score" % i][cand_i],
+                )
 
     # Test with a randomized search
     for est in estimators:
-        random_search = RandomizedSearchCV(est, est_parameters,
-                                           cv=cv, n_iter=3)
+        random_search = RandomizedSearchCV(est, est_parameters, cv=cv, n_iter=3)
         random_search.fit(X, y)
-        res_params = random_search.cv_results_['params']
+        res_params = random_search.cv_results_["params"]
         for cand_i in range(len(res_params)):
             est.set_params(**res_params[cand_i])
 
@@ -1378,8 +1665,8 @@ def test_grid_search_with_multioutput_data():
                 correct_score = est.score(X[test], y[test])
                 assert_almost_equal(
                     correct_score,
-                    random_search.cv_results_['split%d_test_score'
-                                              % i][cand_i])
+                    random_search.cv_results_["split%d_test_score" % i][cand_i],
+                )
 
 
 def test_predict_proba_disabled():
@@ -1396,11 +1683,13 @@ def test_grid_search_allows_nans():
     X = np.arange(20, dtype=np.float64).reshape(5, -1)
     X[2, :] = np.nan
     y = [0, 0, 1, 1, 1]
-    p = Pipeline([
-        ('imputer', SimpleImputer(strategy='mean', missing_values=np.nan)),
-        ('classifier', MockClassifier()),
-    ])
-    GridSearchCV(p, {'classifier__foo_param': [1, 2, 3]}, cv=2).fit(X, y)
+    p = Pipeline(
+        [
+            ("imputer", SimpleImputer(strategy="mean", missing_values=np.nan)),
+            ("classifier", MockClassifier()),
+        ]
+    )
+    GridSearchCV(p, {"classifier__foo_param": [1, 2, 3]}, cv=2).fit(X, y)
 
 
 class FailingClassifier(BaseEstimator):
@@ -1419,7 +1708,7 @@ def predict(self, X):
         return np.zeros(X.shape[0])
 
     def score(self, X=None, Y=None):
-        return 0.
+        return 0.0
 
 
 def test_grid_search_failing_classifier():
@@ -1435,42 +1724,66 @@ def test_grid_search_failing_classifier():
     # refit was done, then an exception would be raised on refit and not
     # caught by grid_search (expected behavior), and this would cause an
     # error in this test.
-    gs = GridSearchCV(clf, [{'parameter': [0, 1, 2]}], scoring='accuracy',
-                      refit=False, error_score=0.0)
-    warning_message = (
-        "Estimator fit failed. The score on this train-test partition "
-        "for these parameters will be set to 0.0.*."
+    gs = GridSearchCV(
+        clf,
+        [{"parameter": [0, 1, 2]}],
+        scoring="accuracy",
+        refit=False,
+        error_score=0.0,
+    )
+
+    warning_message = re.compile(
+        "5 fits failed.+total of 15.+The score on these"
+        r" train-test partitions for these parameters will be set to 0\.0.+"
+        "5 fits failed with the following error.+ValueError.+Failing classifier failed"
+        " as required",
+        flags=re.DOTALL,
     )
     with pytest.warns(FitFailedWarning, match=warning_message):
         gs.fit(X, y)
-    n_candidates = len(gs.cv_results_['params'])
+    n_candidates = len(gs.cv_results_["params"])
 
     # Ensure that grid scores were set to zero as required for those fits
     # that are expected to fail.
     def get_cand_scores(i):
-        return np.array(list(gs.cv_results_['split%d_test_score' % s][i]
-                             for s in range(gs.n_splits_)))
+        return np.array(
+            [gs.cv_results_["split%d_test_score" % s][i] for s in range(gs.n_splits_)]
+        )
 
-    assert all((np.all(get_cand_scores(cand_i) == 0.0)
-                for cand_i in range(n_candidates)
-                if gs.cv_results_['param_parameter'][cand_i] ==
-                FailingClassifier.FAILING_PARAMETER))
+    assert all(
+        (
+            np.all(get_cand_scores(cand_i) == 0.0)
+            for cand_i in range(n_candidates)
+            if gs.cv_results_["param_parameter"][cand_i]
+            == FailingClassifier.FAILING_PARAMETER
+        )
+    )
 
-    gs = GridSearchCV(clf, [{'parameter': [0, 1, 2]}], scoring='accuracy',
-                      refit=False, error_score=float('nan'))
-    warning_message = (
-        "Estimator fit failed. The score on this train-test partition "
-        "for these parameters will be set to nan."
+    gs = GridSearchCV(
+        clf,
+        [{"parameter": [0, 1, 2]}],
+        scoring="accuracy",
+        refit=False,
+        error_score=float("nan"),
+    )
+    warning_message = re.compile(
+        "5 fits failed.+total of 15.+The score on these"
+        r" train-test partitions for these parameters will be set to nan.+"
+        "5 fits failed with the following error.+ValueError.+Failing classifier failed"
+        " as required",
+        flags=re.DOTALL,
     )
     with pytest.warns(FitFailedWarning, match=warning_message):
         gs.fit(X, y)
-    n_candidates = len(gs.cv_results_['params'])
-    assert all(np.all(np.isnan(get_cand_scores(cand_i)))
-               for cand_i in range(n_candidates)
-               if gs.cv_results_['param_parameter'][cand_i] ==
-               FailingClassifier.FAILING_PARAMETER)
+    n_candidates = len(gs.cv_results_["params"])
+    assert all(
+        np.all(np.isnan(get_cand_scores(cand_i)))
+        for cand_i in range(n_candidates)
+        if gs.cv_results_["param_parameter"][cand_i]
+        == FailingClassifier.FAILING_PARAMETER
+    )
 
-    ranks = gs.cv_results_['rank_test_score']
+    ranks = gs.cv_results_["rank_test_score"]
 
     # Check that succeeded estimators have lower ranks
     assert ranks[0] <= 2 and ranks[1] <= 2
@@ -1479,6 +1792,28 @@ def get_cand_scores(i):
     assert gs.best_index_ != clf.FAILING_PARAMETER
 
 
+def test_grid_search_classifier_all_fits_fail():
+    X, y = make_classification(n_samples=20, n_features=10, random_state=0)
+
+    clf = FailingClassifier()
+
+    gs = GridSearchCV(
+        clf,
+        [{"parameter": [FailingClassifier.FAILING_PARAMETER] * 3}],
+        error_score=0.0,
+    )
+
+    warning_message = re.compile(
+        (
+            "All the 15 fits failed.+15 fits failed with the following"
+            " error.+ValueError.+Failing classifier failed as required"
+        ),
+        flags=re.DOTALL,
+    )
+    with pytest.raises(ValueError, match=warning_message):
+        gs.fit(X, y)
+
+
 def test_grid_search_failing_classifier_raise():
     # GridSearchCV with on_error == 'raise' raises the error
 
@@ -1487,8 +1822,13 @@ def test_grid_search_failing_classifier_raise():
     clf = FailingClassifier()
 
     # refit=False because we want to test the behaviour of the grid search part
-    gs = GridSearchCV(clf, [{'parameter': [0, 1, 2]}], scoring='accuracy',
-                      refit=False, error_score='raise')
+    gs = GridSearchCV(
+        clf,
+        [{"parameter": [0, 1, 2]}],
+        scoring="accuracy",
+        refit=False,
+        error_score="raise",
+    )
 
     # FailingClassifier issues a ValueError so this is what we look for.
     with pytest.raises(ValueError):
@@ -1497,15 +1837,18 @@ def test_grid_search_failing_classifier_raise():
 
 def test_parameters_sampler_replacement():
     # raise warning if n_iter is bigger than total parameter space
-    params = [{'first': [0, 1], 'second': ['a', 'b', 'c']},
-              {'third': ['two', 'values']}]
+    params = [
+        {"first": [0, 1], "second": ["a", "b", "c"]},
+        {"third": ["two", "values"]},
+    ]
     sampler = ParameterSampler(params, n_iter=9)
     n_iter = 9
     grid_size = 8
-    expected_warning = ('The total space of parameters %d is smaller '
-                        'than n_iter=%d. Running %d iterations. For '
-                        'exhaustive searches, use GridSearchCV.'
-                        % (grid_size, n_iter, grid_size))
+    expected_warning = (
+        "The total space of parameters %d is smaller "
+        "than n_iter=%d. Running %d iterations. For "
+        "exhaustive searches, use GridSearchCV." % (grid_size, n_iter, grid_size)
+    )
     with pytest.warns(UserWarning, match=expected_warning):
         list(sampler)
 
@@ -1518,16 +1861,15 @@ def test_parameters_sampler_replacement():
     assert len(ParameterSampler(params, n_iter=1000)) == 8
 
     # test sampling without replacement in a large grid
-    params = {'a': range(10), 'b': range(10), 'c': range(10)}
+    params = {"a": range(10), "b": range(10), "c": range(10)}
     sampler = ParameterSampler(params, n_iter=99, random_state=42)
     samples = list(sampler)
     assert len(samples) == 99
-    hashable_samples = ["a%db%dc%d" % (p['a'], p['b'], p['c'])
-                        for p in samples]
+    hashable_samples = ["a%db%dc%d" % (p["a"], p["b"], p["c"]) for p in samples]
     assert len(set(hashable_samples)) == 99
 
     # doesn't go into infinite loops
-    params_distribution = {'first': bernoulli(.5), 'second': ['a', 'b', 'c']}
+    params_distribution = {"first": bernoulli(0.5), "second": ["a", "b", "c"]}
     sampler = ParameterSampler(params_distribution, n_iter=7)
     samples = list(sampler)
     assert len(samples) == 7
@@ -1537,12 +1879,13 @@ def test_stochastic_gradient_loss_param():
     # Make sure the predict_proba works when loss is specified
     # as one of the parameters in the param_grid.
     param_grid = {
-        'loss': ['log'],
+        "loss": ["log_loss"],
     }
     X = np.arange(24).reshape(6, -1)
     y = [0, 0, 0, 1, 1, 1]
-    clf = GridSearchCV(estimator=SGDClassifier(loss='hinge'),
-                       param_grid=param_grid, cv=3)
+    clf = GridSearchCV(
+        estimator=SGDClassifier(loss="hinge"), param_grid=param_grid, cv=3
+    )
 
     # When the estimator is not fitted, `predict_proba` is not available as the
     # loss is 'hinge'.
@@ -1554,10 +1897,11 @@ def test_stochastic_gradient_loss_param():
     # Make sure `predict_proba` is not available when setting loss=['hinge']
     # in param_grid
     param_grid = {
-        'loss': ['hinge'],
+        "loss": ["hinge"],
     }
-    clf = GridSearchCV(estimator=SGDClassifier(loss='hinge'),
-                       param_grid=param_grid, cv=3)
+    clf = GridSearchCV(
+        estimator=SGDClassifier(loss="hinge"), param_grid=param_grid, cv=3
+    )
     assert not hasattr(clf, "predict_proba")
     clf.fit(X, y)
     assert not hasattr(clf, "predict_proba")
@@ -1568,7 +1912,7 @@ def test_search_train_scores_set_to_false():
     y = [0, 0, 0, 1, 1, 1]
     clf = LinearSVC(random_state=0)
 
-    gs = GridSearchCV(clf, param_grid={'C': [0.1, 0.2]}, cv=3)
+    gs = GridSearchCV(clf, param_grid={"C": [0.1, 0.2]}, cv=3)
     gs.fit(X, y)
 
 
@@ -1578,45 +1922,58 @@ def test_grid_search_cv_splits_consistency():
     n_splits = 5
     X, y = make_classification(n_samples=n_samples, random_state=0)
 
-    gs = GridSearchCV(LinearSVC(random_state=0),
-                      param_grid={'C': [0.1, 0.2, 0.3]},
-                      cv=OneTimeSplitter(n_splits=n_splits,
-                                         n_samples=n_samples),
-                      return_train_score=True)
+    gs = GridSearchCV(
+        LinearSVC(random_state=0),
+        param_grid={"C": [0.1, 0.2, 0.3]},
+        cv=OneTimeSplitter(n_splits=n_splits, n_samples=n_samples),
+        return_train_score=True,
+    )
     gs.fit(X, y)
 
-    gs2 = GridSearchCV(LinearSVC(random_state=0),
-                       param_grid={'C': [0.1, 0.2, 0.3]},
-                       cv=KFold(n_splits=n_splits), return_train_score=True)
+    gs2 = GridSearchCV(
+        LinearSVC(random_state=0),
+        param_grid={"C": [0.1, 0.2, 0.3]},
+        cv=KFold(n_splits=n_splits),
+        return_train_score=True,
+    )
     gs2.fit(X, y)
 
     # Give generator as a cv parameter
-    assert isinstance(KFold(n_splits=n_splits,
-                            shuffle=True, random_state=0).split(X, y),
-                      GeneratorType)
-    gs3 = GridSearchCV(LinearSVC(random_state=0),
-                       param_grid={'C': [0.1, 0.2, 0.3]},
-                       cv=KFold(n_splits=n_splits, shuffle=True,
-                                random_state=0).split(X, y),
-                       return_train_score=True)
+    assert isinstance(
+        KFold(n_splits=n_splits, shuffle=True, random_state=0).split(X, y),
+        GeneratorType,
+    )
+    gs3 = GridSearchCV(
+        LinearSVC(random_state=0),
+        param_grid={"C": [0.1, 0.2, 0.3]},
+        cv=KFold(n_splits=n_splits, shuffle=True, random_state=0).split(X, y),
+        return_train_score=True,
+    )
     gs3.fit(X, y)
 
-    gs4 = GridSearchCV(LinearSVC(random_state=0),
-                       param_grid={'C': [0.1, 0.2, 0.3]},
-                       cv=KFold(n_splits=n_splits, shuffle=True,
-                                random_state=0), return_train_score=True)
+    gs4 = GridSearchCV(
+        LinearSVC(random_state=0),
+        param_grid={"C": [0.1, 0.2, 0.3]},
+        cv=KFold(n_splits=n_splits, shuffle=True, random_state=0),
+        return_train_score=True,
+    )
     gs4.fit(X, y)
 
     def _pop_time_keys(cv_results):
-        for key in ('mean_fit_time', 'std_fit_time',
-                    'mean_score_time', 'std_score_time'):
+        for key in (
+            "mean_fit_time",
+            "std_fit_time",
+            "mean_score_time",
+            "std_score_time",
+        ):
             cv_results.pop(key)
         return cv_results
 
     # Check if generators are supported as cv and
     # that the splits are consistent
-    np.testing.assert_equal(_pop_time_keys(gs3.cv_results_),
-                            _pop_time_keys(gs4.cv_results_))
+    np.testing.assert_equal(
+        _pop_time_keys(gs3.cv_results_), _pop_time_keys(gs4.cv_results_)
+    )
 
     # OneTimeSplitter is a non-re-entrant cv where split can be called only
     # once if ``cv.split`` is called once per param setting in GridSearchCV.fit
@@ -1624,38 +1981,39 @@ def _pop_time_keys(cv_results):
     # will be generated for the 2nd and subsequent cv.split calls.
     # This is a check to make sure cv.split is not called once per param
     # setting.
-    np.testing.assert_equal({k: v for k, v in gs.cv_results_.items()
-                             if not k.endswith('_time')},
-                            {k: v for k, v in gs2.cv_results_.items()
-                             if not k.endswith('_time')})
+    np.testing.assert_equal(
+        {k: v for k, v in gs.cv_results_.items() if not k.endswith("_time")},
+        {k: v for k, v in gs2.cv_results_.items() if not k.endswith("_time")},
+    )
 
     # Check consistency of folds across the parameters
-    gs = GridSearchCV(LinearSVC(random_state=0),
-                      param_grid={'C': [0.1, 0.1, 0.2, 0.2]},
-                      cv=KFold(n_splits=n_splits, shuffle=True),
-                      return_train_score=True)
+    gs = GridSearchCV(
+        LinearSVC(random_state=0),
+        param_grid={"C": [0.1, 0.1, 0.2, 0.2]},
+        cv=KFold(n_splits=n_splits, shuffle=True),
+        return_train_score=True,
+    )
     gs.fit(X, y)
 
     # As the first two param settings (C=0.1) and the next two param
     # settings (C=0.2) are same, the test and train scores must also be
     # same as long as the same train/test indices are generated for all
     # the cv splits, for both param setting
-    for score_type in ('train', 'test'):
+    for score_type in ("train", "test"):
         per_param_scores = {}
         for param_i in range(4):
-            per_param_scores[param_i] = list(
-                gs.cv_results_['split%d_%s_score' % (s, score_type)][param_i]
-                for s in range(5))
+            per_param_scores[param_i] = [
+                gs.cv_results_["split%d_%s_score" % (s, score_type)][param_i]
+                for s in range(5)
+            ]
 
-        assert_array_almost_equal(per_param_scores[0],
-                                  per_param_scores[1])
-        assert_array_almost_equal(per_param_scores[2],
-                                  per_param_scores[3])
+        assert_array_almost_equal(per_param_scores[0], per_param_scores[1])
+        assert_array_almost_equal(per_param_scores[2], per_param_scores[3])
 
 
 def test_transform_inverse_transform_round_trip():
     clf = MockClassifier()
-    grid_search = GridSearchCV(clf, {'foo_param': [1, 2, 3]}, cv=3, verbose=3)
+    grid_search = GridSearchCV(clf, {"foo_param": [1, 2, 3]}, cv=2, verbose=3)
 
     grid_search.fit(X, y)
     X_round_trip = grid_search.inverse_transform(grid_search.transform(X))
@@ -1667,48 +2025,56 @@ def check_results(results, gscv):
         exp_results = gscv.cv_results_
         assert sorted(results.keys()) == sorted(exp_results)
         for k in results:
-            if not k.endswith('_time'):
+            if not k.endswith("_time"):
                 # XXX: results['params'] is a list :|
                 results[k] = np.asanyarray(results[k])
-                if results[k].dtype.kind == 'O':
-                    assert_array_equal(exp_results[k], results[k],
-                                       err_msg='Checking ' + k)
+                if results[k].dtype.kind == "O":
+                    assert_array_equal(
+                        exp_results[k], results[k], err_msg="Checking " + k
+                    )
                 else:
-                    assert_allclose(exp_results[k], results[k],
-                                    err_msg='Checking ' + k)
+                    assert_allclose(exp_results[k], results[k], err_msg="Checking " + k)
 
     def fit_grid(param_grid):
-        return GridSearchCV(clf, param_grid,
-                            return_train_score=True).fit(X, y)
+        return GridSearchCV(clf, param_grid, return_train_score=True).fit(X, y)
 
     class CustomSearchCV(BaseSearchCV):
         def __init__(self, estimator, **kwargs):
             super().__init__(estimator, **kwargs)
 
         def _run_search(self, evaluate):
-            results = evaluate([{'max_depth': 1}, {'max_depth': 2}])
-            check_results(results, fit_grid({'max_depth': [1, 2]}))
-            results = evaluate([{'min_samples_split': 5},
-                                {'min_samples_split': 10}])
-            check_results(results, fit_grid([{'max_depth': [1, 2]},
-                                             {'min_samples_split': [5, 10]}]))
+            results = evaluate([{"max_depth": 1}, {"max_depth": 2}])
+            check_results(results, fit_grid({"max_depth": [1, 2]}))
+            results = evaluate([{"min_samples_split": 5}, {"min_samples_split": 10}])
+            check_results(
+                results,
+                fit_grid([{"max_depth": [1, 2]}, {"min_samples_split": [5, 10]}]),
+            )
 
     # Using regressor to make sure each score differs
     clf = DecisionTreeRegressor(random_state=0)
-    X, y = make_classification(n_samples=100, n_informative=4,
-                               random_state=0)
+    X, y = make_classification(n_samples=100, n_informative=4, random_state=0)
     mycv = CustomSearchCV(clf, return_train_score=True).fit(X, y)
-    gscv = fit_grid([{'max_depth': [1, 2]},
-                     {'min_samples_split': [5, 10]}])
+    gscv = fit_grid([{"max_depth": [1, 2]}, {"min_samples_split": [5, 10]}])
 
     results = mycv.cv_results_
     check_results(results, gscv)
     for attr in dir(gscv):
-        if (attr[0].islower() and attr[-1:] == '_' and
-                attr not in {'cv_results_', 'best_estimator_',
-                             'refit_time_', 'classes_'}):
-            assert getattr(gscv, attr) == getattr(mycv, attr), \
+        if (
+            attr[0].islower()
+            and attr[-1:] == "_"
+            and attr
+            not in {
+                "cv_results_",
+                "best_estimator_",
+                "refit_time_",
+                "classes_",
+                "scorer_",
+            }
+        ):
+            assert getattr(gscv, attr) == getattr(mycv, attr), (
                 "Attribute %s not equal" % attr
+            )
 
 
 def test__custom_fit_no_run_search():
@@ -1726,8 +2092,7 @@ class BadSearchCV(BaseSearchCV):
         def __init__(self, estimator, **kwargs):
             super().__init__(estimator, **kwargs)
 
-    with pytest.raises(NotImplementedError,
-                       match="_run_search not implemented."):
+    with pytest.raises(NotImplementedError, match="_run_search not implemented."):
         # this should raise a NotImplementedError
         BadSearchCV(SVC()).fit(X, y)
 
@@ -1743,14 +2108,17 @@ def test_empty_cv_iterator_error():
     # cv is empty now
 
     train_size = 100
-    ridge = RandomizedSearchCV(Ridge(), {'alpha': [1e-3, 1e-2, 1e-1]},
-                               cv=cv, n_jobs=4)
+    ridge = RandomizedSearchCV(Ridge(), {"alpha": [1e-3, 1e-2, 1e-1]}, cv=cv, n_jobs=4)
 
     # assert that this raises an error
-    with pytest.raises(ValueError,
-                       match='No fits were performed. '
-                             'Was the CV iterator empty\\? '
-                             'Were there no candidates\\?'):
+    with pytest.raises(
+        ValueError,
+        match=(
+            "No fits were performed. "
+            "Was the CV iterator empty\\? "
+            "Were there no candidates\\?"
+        ),
+    ):
         ridge.fit(X[:train_size], y[:train_size])
 
 
@@ -1765,25 +2133,34 @@ def get_n_splits(self, *args, **kw):
     cv = BrokenKFold(n_splits=3)
 
     train_size = 100
-    ridge = RandomizedSearchCV(Ridge(), {'alpha': [1e-3, 1e-2, 1e-1]},
-                               cv=cv, n_jobs=4)
+    ridge = RandomizedSearchCV(Ridge(), {"alpha": [1e-3, 1e-2, 1e-1]}, cv=cv, n_jobs=4)
 
     # assert that this raises an error
-    with pytest.raises(ValueError,
-                       match='cv.split and cv.get_n_splits returned '
-                             'inconsistent results. Expected \\d+ '
-                             'splits, got \\d+'):
+    with pytest.raises(
+        ValueError,
+        match=(
+            "cv.split and cv.get_n_splits returned "
+            "inconsistent results. Expected \\d+ "
+            "splits, got \\d+"
+        ),
+    ):
         ridge.fit(X[:train_size], y[:train_size])
 
 
 @pytest.mark.parametrize("return_train_score", [False, True])
 @pytest.mark.parametrize(
     "SearchCV, specialized_params",
-    [(GridSearchCV, {"param_grid": {"max_depth": [2, 3]}}),
-     (RandomizedSearchCV,
-      {"param_distributions": {"max_depth": [2, 3]}, "n_iter": 2})])
+    [
+        (GridSearchCV, {"param_grid": {"max_depth": [2, 3, 5, 8]}}),
+        (
+            RandomizedSearchCV,
+            {"param_distributions": {"max_depth": [2, 3, 5, 8]}, "n_iter": 4},
+        ),
+    ],
+)
 def test_searchcv_raise_warning_with_non_finite_score(
-        SearchCV, specialized_params, return_train_score):
+    SearchCV, specialized_params, return_train_score
+):
     # Non-regression test for:
     # https://github.com/scikit-learn/scikit-learn/issues/10529
     # Check that we raise a UserWarning when a non-finite score is
@@ -1807,7 +2184,7 @@ def __call__(self, estimator, X, y):
         scoring=FailingScorer(),
         cv=3,
         return_train_score=return_train_score,
-        **specialized_params
+        **specialized_params,
     )
 
     with pytest.warns(UserWarning) as warn_msg:
@@ -1816,8 +2193,14 @@ def __call__(self, estimator, X, y):
     set_with_warning = ["test", "train"] if return_train_score else ["test"]
     assert len(warn_msg) == len(set_with_warning)
     for msg, dataset in zip(warn_msg, set_with_warning):
-        assert (f"One or more of the {dataset} scores are non-finite" in
-                str(msg.message))
+        assert f"One or more of the {dataset} scores are non-finite" in str(msg.message)
+
+    # all non-finite scores should be equally ranked last
+    last_rank = grid.cv_results_["rank_test_score"].max()
+    non_finite_mask = np.isnan(grid.cv_results_["mean_test_score"])
+    assert_array_equal(grid.cv_results_["rank_test_score"][non_finite_mask], last_rank)
+    # all finite scores should be better ranked than the non-finite scores
+    assert np.all(grid.cv_results_["rank_test_score"][~non_finite_mask] < last_rank)
 
 
 def test_callable_multimetric_confusion_matrix():
@@ -1826,17 +2209,15 @@ def test_callable_multimetric_confusion_matrix():
     def custom_scorer(clf, X, y):
         y_pred = clf.predict(X)
         cm = confusion_matrix(y, y_pred)
-        return {'tn': cm[0, 0], 'fp': cm[0, 1], 'fn': cm[1, 0], 'tp': cm[1, 1]}
+        return {"tn": cm[0, 0], "fp": cm[0, 1], "fn": cm[1, 0], "tp": cm[1, 1]}
 
-    X, y = make_classification(n_samples=40, n_features=4,
-                               random_state=42)
+    X, y = make_classification(n_samples=40, n_features=4, random_state=42)
     est = LinearSVC(random_state=42)
-    search = GridSearchCV(est, {'C': [0.1, 1]}, scoring=custom_scorer,
-                          refit='fp')
+    search = GridSearchCV(est, {"C": [0.1, 1]}, scoring=custom_scorer, refit="fp")
 
     search.fit(X, y)
 
-    score_names = ['tn', 'fp', 'fn', 'tp']
+    score_names = ["tn", "fp", "fn", "tp"]
     for name in score_names:
         assert "mean_test_{}".format(name) in search.cv_results_
 
@@ -1849,16 +2230,19 @@ def test_callable_multimetric_same_as_list_of_strings():
     # Test callable multimetric is the same as a list of strings
     def custom_scorer(est, X, y):
         y_pred = est.predict(X)
-        return {'recall': recall_score(y, y_pred),
-                'accuracy': accuracy_score(y, y_pred)}
+        return {
+            "recall": recall_score(y, y_pred),
+            "accuracy": accuracy_score(y, y_pred),
+        }
 
-    X, y = make_classification(n_samples=40, n_features=4,
-                               random_state=42)
+    X, y = make_classification(n_samples=40, n_features=4, random_state=42)
     est = LinearSVC(random_state=42)
-    search_callable = GridSearchCV(est, {'C': [0.1, 1]},
-                                   scoring=custom_scorer, refit='recall')
-    search_str = GridSearchCV(est, {'C': [0.1, 1]},
-                              scoring=['recall', 'accuracy'], refit='recall')
+    search_callable = GridSearchCV(
+        est, {"C": [0.1, 1]}, scoring=custom_scorer, refit="recall"
+    )
+    search_str = GridSearchCV(
+        est, {"C": [0.1, 1]}, scoring=["recall", "accuracy"], refit="recall"
+    )
 
     search_callable.fit(X, y)
     search_str.fit(X, y)
@@ -1874,15 +2258,15 @@ def custom_scorer(est, X, y):
         y_pred = est.predict(X)
         return recall_score(y, y_pred)
 
-    X, y = make_classification(n_samples=40, n_features=4,
-                               random_state=42)
+    X, y = make_classification(n_samples=40, n_features=4, random_state=42)
     est = LinearSVC(random_state=42)
-    search_callable = GridSearchCV(est, {'C': [0.1, 1]},
-                                   scoring=custom_scorer, refit=True)
-    search_str = GridSearchCV(est, {'C': [0.1, 1]},
-                              scoring='recall', refit='recall')
-    search_list_str = GridSearchCV(est, {'C': [0.1, 1]},
-                                   scoring=['recall'], refit='recall')
+    search_callable = GridSearchCV(
+        est, {"C": [0.1, 1]}, scoring=custom_scorer, refit=True
+    )
+    search_str = GridSearchCV(est, {"C": [0.1, 1]}, scoring="recall", refit="recall")
+    search_list_str = GridSearchCV(
+        est, {"C": [0.1, 1]}, scoring=["recall"], refit="recall"
+    )
     search_callable.fit(X, y)
     search_str.fit(X, y)
     search_list_str.fit(X, y)
@@ -1899,15 +2283,20 @@ def custom_scorer(est, X, y):
 def test_callable_multimetric_error_on_invalid_key():
     # Raises when the callable scorer does not return a dict with `refit` key.
     def bad_scorer(est, X, y):
-        return {'bad_name': 1}
-
-    X, y = make_classification(n_samples=40, n_features=4,
-                               random_state=42)
-    clf = GridSearchCV(LinearSVC(random_state=42), {'C': [0.1, 1]},
-                       scoring=bad_scorer, refit='good_name')
+        return {"bad_name": 1}
+
+    X, y = make_classification(n_samples=40, n_features=4, random_state=42)
+    clf = GridSearchCV(
+        LinearSVC(random_state=42),
+        {"C": [0.1, 1]},
+        scoring=bad_scorer,
+        refit="good_name",
+    )
 
-    msg = ('For multi-metric scoring, the parameter refit must be set to a '
-           'scorer key or a callable to refit')
+    msg = (
+        "For multi-metric scoring, the parameter refit must be set to a "
+        "scorer key or a callable to refit"
+    )
     with pytest.raises(ValueError, match=msg):
         clf.fit(X, y)
 
@@ -1916,34 +2305,57 @@ def test_callable_multimetric_error_failing_clf():
     # Warns when there is an estimator the fails to fit with a float
     # error_score
     def custom_scorer(est, X, y):
-        return {'acc': 1}
+        return {"acc": 1}
 
     X, y = make_classification(n_samples=20, n_features=10, random_state=0)
 
     clf = FailingClassifier()
-    gs = GridSearchCV(clf, [{'parameter': [0, 1, 2]}], scoring=custom_scorer,
-                      refit=False, error_score=0.1)
+    gs = GridSearchCV(
+        clf,
+        [{"parameter": [0, 1, 2]}],
+        scoring=custom_scorer,
+        refit=False,
+        error_score=0.1,
+    )
 
-    with pytest.warns(FitFailedWarning, match='Estimator fit failed'):
+    warning_message = re.compile(
+        "5 fits failed.+total of 15.+The score on these"
+        r" train-test partitions for these parameters will be set to 0\.1",
+        flags=re.DOTALL,
+    )
+    with pytest.warns(FitFailedWarning, match=warning_message):
         gs.fit(X, y)
 
-    assert_allclose(gs.cv_results_['mean_test_acc'], [1, 1, 0.1])
+    assert_allclose(gs.cv_results_["mean_test_acc"], [1, 1, 0.1])
 
 
-def test_callable_multimetric_clf_all_fails():
+def test_callable_multimetric_clf_all_fits_fail():
     # Warns and raises when all estimator fails to fit.
     def custom_scorer(est, X, y):
-        return {'acc': 1}
+        return {"acc": 1}
+
     X, y = make_classification(n_samples=20, n_features=10, random_state=0)
 
     clf = FailingClassifier()
 
-    gs = GridSearchCV(clf, [{'parameter': [2, 2, 2]}], scoring=custom_scorer,
-                      refit=False, error_score=0.1)
+    gs = GridSearchCV(
+        clf,
+        [{"parameter": [FailingClassifier.FAILING_PARAMETER] * 3}],
+        scoring=custom_scorer,
+        refit=False,
+        error_score=0.1,
+    )
+
+    individual_fit_error_message = "ValueError: Failing classifier failed as required"
+    error_message = re.compile(
+        (
+            "All the 15 fits failed.+your model is misconfigured.+"
+            f"{individual_fit_error_message}"
+        ),
+        flags=re.DOTALL,
+    )
 
-    with pytest.warns(FitFailedWarning, match='Estimator fit failed'), \
-            pytest.raises(NotFittedError,
-                          match="All estimators failed to fit"):
+    with pytest.raises(ValueError, match=error_message):
         gs.fit(X, y)
 
 
@@ -1953,11 +2365,11 @@ def test_n_features_in():
     n_features = 4
     X, y = make_classification(n_features=n_features)
     gbdt = HistGradientBoostingClassifier()
-    param_grid = {'max_iter': [3, 4]}
+    param_grid = {"max_iter": [3, 4]}
     gs = GridSearchCV(gbdt, param_grid)
     rs = RandomizedSearchCV(gbdt, param_grid, n_iter=1)
-    assert not hasattr(gs, 'n_features_in_')
-    assert not hasattr(rs, 'n_features_in_')
+    assert not hasattr(gs, "n_features_in_")
+    assert not hasattr(rs, "n_features_in_")
     gs.fit(X, y)
     rs.fit(X, y)
     assert gs.n_features_in_ == n_features
@@ -1973,33 +2385,46 @@ def test_search_cv_pairwise_property_delegated_to_base_estimator(pairwise):
 
     Non-regression test for issue #13920.
     """
+
     class TestEstimator(BaseEstimator):
-        def _more_tags(self):
-            return {'pairwise': pairwise}
+        def __sklearn_tags__(self):
+            tags = super().__sklearn_tags__()
+            tags.input_tags.pairwise = pairwise
+            return tags
 
     est = TestEstimator()
     attr_message = "BaseSearchCV pairwise tag must match estimator"
-    cv = GridSearchCV(est, {'n_neighbors': [10]})
-    assert pairwise == cv._get_tags()['pairwise'], attr_message
+    cv = GridSearchCV(est, {"n_neighbors": [10]})
+    assert pairwise == cv.__sklearn_tags__().input_tags.pairwise, attr_message
 
 
-# TODO: Remove in 1.1
-@ignore_warnings(category=FutureWarning)
 def test_search_cv__pairwise_property_delegated_to_base_estimator():
     """
-    Test implementation of BaseSearchCV has the _pairwise property
-    which matches the _pairwise property of its estimator.
-    This test make sure _pairwise is delegated to the base estimator.
+    Test implementation of BaseSearchCV has the pairwise property
+    which matches the pairwise tag of its estimator.
+    This test make sure pairwise tag is delegated to the base estimator.
 
     Non-regression test for issue #13920.
     """
-    est = BaseEstimator()
+
+    class EstimatorPairwise(BaseEstimator):
+        def __init__(self, pairwise=True):
+            self.pairwise = pairwise
+
+        def __sklearn_tags__(self):
+            tags = super().__sklearn_tags__()
+            tags.input_tags.pairwise = self.pairwise
+            return tags
+
+    est = EstimatorPairwise()
     attr_message = "BaseSearchCV _pairwise property must match estimator"
 
     for _pairwise_setting in [True, False]:
-        setattr(est, '_pairwise', _pairwise_setting)
-        cv = GridSearchCV(est, {'n_neighbors': [10]})
-        assert _pairwise_setting == cv._pairwise, attr_message
+        est.set_params(pairwise=_pairwise_setting)
+        cv = GridSearchCV(est, {"n_neighbors": [10]})
+        assert _pairwise_setting == cv.__sklearn_tags__().input_tags.pairwise, (
+            attr_message
+        )
 
 
 def test_search_cv_pairwise_property_equivalence_of_precomputed():
@@ -2013,7 +2438,7 @@ def test_search_cv_pairwise_property_equivalence_of_precomputed():
     n_samples = 50
     n_splits = 2
     X, y = make_classification(n_samples=n_samples, random_state=0)
-    grid_params = {'n_neighbors': [10]}
+    grid_params = {"n_neighbors": [10]}
 
     # defaults to euclidean metric (minkowski p = 2)
     clf = KNeighborsClassifier()
@@ -2023,7 +2448,7 @@ def test_search_cv_pairwise_property_equivalence_of_precomputed():
 
     # precompute euclidean metric to validate pairwise is working
     X_precomputed = euclidean_distances(X)
-    clf = KNeighborsClassifier(metric='precomputed')
+    clf = KNeighborsClassifier(metric="precomputed")
     cv = GridSearchCV(clf, grid_params, cv=n_splits)
     cv.fit(X_precomputed, y)
     preds_precomputed = cv.predict(X_precomputed)
@@ -2034,8 +2459,7 @@ def test_search_cv_pairwise_property_equivalence_of_precomputed():
 
 @pytest.mark.parametrize(
     "SearchCV, param_search",
-    [(GridSearchCV, {'a': [0.1, 0.01]}),
-     (RandomizedSearchCV, {'a': uniform(1, 3)})]
+    [(GridSearchCV, {"a": [0.1, 0.01]}), (RandomizedSearchCV, {"a": uniform(1, 3)})],
 )
 def test_scalar_fit_param(SearchCV, param_search):
     # unofficially sanctioned tolerance for scalar values in fit_params
@@ -2059,8 +2483,10 @@ def predict(self, X):
 
 @pytest.mark.parametrize(
     "SearchCV, param_search",
-    [(GridSearchCV, {'alpha': [0.1, 0.01]}),
-     (RandomizedSearchCV, {'alpha': uniform(0.01, 0.1)})]
+    [
+        (GridSearchCV, {"alpha": [0.1, 0.01]}),
+        (RandomizedSearchCV, {"alpha": uniform(0.01, 0.1)}),
+    ],
 )
 def test_scalar_fit_param_compat(SearchCV, param_search):
     # check support for scalar values in fit_params, for instance in LightGBM
@@ -2074,9 +2500,15 @@ def test_scalar_fit_param_compat(SearchCV, param_search):
     )
 
     class _FitParamClassifier(SGDClassifier):
-
-        def fit(self, X, y, sample_weight=None, tuple_of_arrays=None,
-                scalar_param=None, callable_param=None):
+        def fit(
+            self,
+            X,
+            y,
+            sample_weight=None,
+            tuple_of_arrays=None,
+            scalar_param=None,
+            callable_param=None,
+        ):
             super().fit(X, y, sample_weight=sample_weight)
             assert scalar_param > 0
             assert callable(callable_param)
@@ -2090,9 +2522,7 @@ def fit(self, X, y, sample_weight=None, tuple_of_arrays=None,
     def _fit_param_callable():
         pass
 
-    model = SearchCV(
-        _FitParamClassifier(), param_search
-    )
+    model = SearchCV(_FitParamClassifier(), param_search)
 
     # NOTE: `fit_params` should be data dependent (e.g. `sample_weight`) which
     # is not the case for the following parameters. But this abuse is common in
@@ -2100,9 +2530,9 @@ def _fit_param_callable():
     # now and be careful not to break support for those without following
     # proper deprecation cycle.
     fit_params = {
-        'tuple_of_arrays': (X_valid, y_valid),
-        'callable_param': _fit_param_callable,
-        'scalar_param': 42,
+        "tuple_of_arrays": (X_valid, y_valid),
+        "callable_param": _fit_param_callable,
+        "scalar_param": 42,
     }
     model.fit(X_train, y_train, **fit_params)
 
@@ -2118,12 +2548,13 @@ def test_search_cv_using_minimal_compatible_estimator(SearchCV, Predictor):
     rng = np.random.RandomState(0)
     X, y = rng.randn(25, 2), np.array([0] * 5 + [1] * 20)
 
-    model = Pipeline([
-        ("transformer", MinimalTransformer()), ("predictor", Predictor())
-    ])
+    model = Pipeline(
+        [("transformer", MinimalTransformer()), ("predictor", Predictor())]
+    )
 
     params = {
-        "transformer__param": [1, 10], "predictor__parama": [1, 10],
+        "transformer__param": [1, 10],
+        "predictor__parama": [1, 10],
     }
     search = SearchCV(model, params, error_score="raise")
     search.fit(X, y)
@@ -2142,17 +2573,394 @@ def test_search_cv_using_minimal_compatible_estimator(SearchCV, Predictor):
 @pytest.mark.parametrize("return_train_score", [True, False])
 def test_search_cv_verbose_3(capsys, return_train_score):
     """Check that search cv with verbose>2 shows the score for single
-    metrics. non-regression test fo #19658."""
-    X, y = make_classification(n_samples=100, n_classes=2, flip_y=.2,
-                               random_state=0)
+    metrics. non-regression test for #19658."""
+    X, y = make_classification(n_samples=100, n_classes=2, flip_y=0.2, random_state=0)
     clf = LinearSVC(random_state=0)
-    grid = {'C': [.1]}
+    grid = {"C": [0.1]}
 
-    GridSearchCV(clf, grid, scoring='accuracy', verbose=3, cv=3,
-                 return_train_score=return_train_score).fit(X, y)
+    GridSearchCV(
+        clf,
+        grid,
+        scoring="accuracy",
+        verbose=3,
+        cv=3,
+        return_train_score=return_train_score,
+    ).fit(X, y)
     captured = capsys.readouterr().out
     if return_train_score:
         match = re.findall(r"score=\(train=[\d\.]+, test=[\d.]+\)", captured)
     else:
         match = re.findall(r"score=[\d\.]+", captured)
     assert len(match) == 3
+
+
+@pytest.mark.parametrize(
+    "SearchCV, param_search",
+    [
+        (GridSearchCV, "param_grid"),
+        (RandomizedSearchCV, "param_distributions"),
+        (HalvingGridSearchCV, "param_grid"),
+    ],
+)
+def test_search_estimator_param(SearchCV, param_search):
+    # test that SearchCV object doesn't change the object given in the parameter grid
+    X, y = make_classification(random_state=42)
+
+    params = {"clf": [LinearSVC()], "clf__C": [0.01]}
+    orig_C = params["clf"][0].C
+
+    pipe = Pipeline([("trs", MinimalTransformer()), ("clf", None)])
+
+    param_grid_search = {param_search: params}
+    gs = SearchCV(pipe, refit=True, cv=2, scoring="accuracy", **param_grid_search).fit(
+        X, y
+    )
+
+    # testing that the original object in params is not changed
+    assert params["clf"][0].C == orig_C
+    # testing that the GS is setting the parameter of the step correctly
+    assert gs.best_estimator_.named_steps["clf"].C == 0.01
+
+
+def test_search_with_2d_array():
+    parameter_grid = {
+        "vect__ngram_range": ((1, 1), (1, 2)),  # unigrams or bigrams
+        "vect__norm": ("l1", "l2"),
+    }
+    pipeline = Pipeline(
+        [
+            ("vect", TfidfVectorizer()),
+            ("clf", ComplementNB()),
+        ]
+    )
+    random_search = RandomizedSearchCV(
+        estimator=pipeline,
+        param_distributions=parameter_grid,
+        n_iter=3,
+        random_state=0,
+        n_jobs=2,
+        verbose=1,
+        cv=3,
+    )
+    data_train = ["one", "two", "three", "four", "five"]
+    data_target = [0, 0, 1, 0, 1]
+    random_search.fit(data_train, data_target)
+    result = random_search.cv_results_["param_vect__ngram_range"]
+    expected_data = np.empty(3, dtype=object)
+    expected_data[:] = [(1, 2), (1, 2), (1, 1)]
+    np.testing.assert_array_equal(result.data, expected_data)
+
+
+def test_search_html_repr():
+    """Test different HTML representations for GridSearchCV."""
+    X, y = make_classification(random_state=42)
+
+    pipeline = Pipeline([("scale", StandardScaler()), ("clf", DummyClassifier())])
+    param_grid = {"clf": [DummyClassifier(), LogisticRegression()]}
+
+    # Unfitted shows the original pipeline
+    search_cv = GridSearchCV(pipeline, param_grid=param_grid, refit=False)
+    with config_context(display="diagram"):
+        repr_html = search_cv._repr_html_()
+        assert "
DummyClassifier
" in repr_html
+
+    # Fitted with `refit=False` shows the original pipeline
+    search_cv.fit(X, y)
+    with config_context(display="diagram"):
+        repr_html = search_cv._repr_html_()
+        assert "
DummyClassifier
" in repr_html
+
+    # Fitted with `refit=True` shows the best estimator
+    search_cv = GridSearchCV(pipeline, param_grid=param_grid, refit=True)
+    search_cv.fit(X, y)
+    with config_context(display="diagram"):
+        repr_html = search_cv._repr_html_()
+        assert "
DummyClassifier
" not in repr_html
+        assert "
LogisticRegression
" in repr_html
+
+
+# Metadata Routing Tests
+# ======================
+
+
+@pytest.mark.parametrize(
+    "SearchCV, param_search",
+    [
+        (GridSearchCV, "param_grid"),
+        (RandomizedSearchCV, "param_distributions"),
+    ],
+)
+@config_context(enable_metadata_routing=True)
+def test_multi_metric_search_forwards_metadata(SearchCV, param_search):
+    """Test that *SearchCV forwards metadata correctly when passed multiple metrics."""
+    X, y = make_classification(random_state=42)
+    n_samples = _num_samples(X)
+    rng = np.random.RandomState(0)
+    score_weights = rng.rand(n_samples)
+    score_metadata = rng.rand(n_samples)
+
+    est = LinearSVC()
+    param_grid_search = {param_search: {"C": [1]}}
+
+    scorer_registry = _Registry()
+    scorer = ConsumingScorer(registry=scorer_registry).set_score_request(
+        sample_weight="score_weights", metadata="score_metadata"
+    )
+    scoring = dict(my_scorer=scorer, accuracy="accuracy")
+    SearchCV(est, refit="accuracy", cv=2, scoring=scoring, **param_grid_search).fit(
+        X, y, score_weights=score_weights, score_metadata=score_metadata
+    )
+    assert len(scorer_registry)
+    for _scorer in scorer_registry:
+        check_recorded_metadata(
+            obj=_scorer,
+            method="score",
+            parent="_score",
+            split_params=("sample_weight", "metadata"),
+            sample_weight=score_weights,
+            metadata=score_metadata,
+        )
+
+
+@pytest.mark.parametrize(
+    "SearchCV, param_search",
+    [
+        (GridSearchCV, "param_grid"),
+        (RandomizedSearchCV, "param_distributions"),
+        (HalvingGridSearchCV, "param_grid"),
+    ],
+)
+def test_score_rejects_params_with_no_routing_enabled(SearchCV, param_search):
+    """*SearchCV should reject **params when metadata routing is not enabled
+    since this is added only when routing is enabled."""
+    X, y = make_classification(random_state=42)
+    est = LinearSVC()
+    param_grid_search = {param_search: {"C": [1]}}
+
+    gs = SearchCV(est, cv=2, **param_grid_search).fit(X, y)
+
+    with pytest.raises(ValueError, match="is only supported if"):
+        gs.score(X, y, metadata=1)
+
+
+# End of Metadata Routing Tests
+# =============================
+
+
+def test_cv_results_dtype_issue_29074():
+    """Non-regression test for https://github.com/scikit-learn/scikit-learn/issues/29074"""
+
+    class MetaEstimator(BaseEstimator, ClassifierMixin):
+        def __init__(
+            self,
+            base_clf,
+            parameter1=None,
+            parameter2=None,
+            parameter3=None,
+            parameter4=None,
+        ):
+            self.base_clf = base_clf
+            self.parameter1 = parameter1
+            self.parameter2 = parameter2
+            self.parameter3 = parameter3
+            self.parameter4 = parameter4
+
+        def fit(self, X, y=None):
+            self.base_clf.fit(X, y)
+            return self
+
+        def score(self, X, y):
+            return self.base_clf.score(X, y)
+
+    # Values of param_grid are such that np.result_type gives slightly
+    # different errors, in particular ValueError and TypeError
+    param_grid = {
+        "parameter1": [None, {"option": "A"}, {"option": "B"}],
+        "parameter2": [None, [1, 2]],
+        "parameter3": [{"a": 1}],
+        "parameter4": ["str1", "str2"],
+    }
+    grid_search = GridSearchCV(
+        estimator=MetaEstimator(LogisticRegression()),
+        param_grid=param_grid,
+        cv=3,
+    )
+
+    X, y = make_blobs(random_state=0)
+    grid_search.fit(X, y)
+    for param in param_grid:
+        assert grid_search.cv_results_[f"param_{param}"].dtype == object
+
+
+def test_search_with_estimators_issue_29157():
+    """Check cv_results_ for estimators with a `dtype` parameter, e.g. OneHotEncoder."""
+    pd = pytest.importorskip("pandas")
+    df = pd.DataFrame(
+        {
+            "numeric_1": [1, 2, 3, 4, 5],
+            "object_1": ["a", "a", "a", "a", "a"],
+            "target": [1.0, 4.1, 2.0, 3.0, 1.0],
+        }
+    )
+    X = df.drop("target", axis=1)
+    y = df["target"]
+    enc = ColumnTransformer(
+        [("enc", OneHotEncoder(sparse_output=False), ["object_1"])],
+        remainder="passthrough",
+    )
+    pipe = Pipeline(
+        [
+            ("enc", enc),
+            ("regressor", LinearRegression()),
+        ]
+    )
+    grid_params = {
+        "enc__enc": [
+            OneHotEncoder(sparse_output=False),
+            OrdinalEncoder(),
+        ]
+    }
+    grid_search = GridSearchCV(pipe, grid_params, cv=2)
+    grid_search.fit(X, y)
+    assert grid_search.cv_results_["param_enc__enc"].dtype == object
+
+
+def test_cv_results_multi_size_array():
+    """Check that GridSearchCV works with params that are arrays of different sizes.
+
+    Non-regression test for #29277.
+    """
+    n_features = 10
+    X, y = make_classification(n_features=10)
+
+    spline_reg_pipe = make_pipeline(
+        SplineTransformer(extrapolation="periodic"),
+        LogisticRegression(),
+    )
+
+    n_knots_list = [n_features * i for i in [10, 11, 12]]
+    knots_list = [
+        np.linspace(0, np.pi * 2, n_knots).reshape((-1, n_features))
+        for n_knots in n_knots_list
+    ]
+    spline_reg_pipe_cv = GridSearchCV(
+        estimator=spline_reg_pipe,
+        param_grid={
+            "splinetransformer__knots": knots_list,
+        },
+    )
+
+    spline_reg_pipe_cv.fit(X, y)
+    assert (
+        spline_reg_pipe_cv.cv_results_["param_splinetransformer__knots"].dtype == object
+    )
+
+
+@pytest.mark.parametrize(
+    "array_namespace, device, dtype",
+    yield_namespace_device_dtype_combinations(),
+    ids=_get_namespace_device_dtype_ids,
+)
+@pytest.mark.parametrize("SearchCV", [GridSearchCV, RandomizedSearchCV])
+def test_array_api_search_cv_classifier(SearchCV, array_namespace, device, dtype):
+    xp = _array_api_for_tests(array_namespace, device)
+
+    X = np.arange(100).reshape((10, 10))
+    X_np = X.astype(dtype)
+    X_xp = xp.asarray(X_np, device=device)
+
+    # y should always be an integer, no matter what `dtype` is
+    y_np = np.array([0] * 5 + [1] * 5)
+    y_xp = xp.asarray(y_np, device=device)
+
+    with config_context(array_api_dispatch=True):
+        searcher = SearchCV(
+            LinearDiscriminantAnalysis(),
+            {"tol": [1e-2, 1e-3, 1e-4, 1e-5, 1e-6, 1e-7]},
+            cv=2,
+            error_score="raise",
+        )
+        searcher.fit(X_xp, y_xp)
+        searcher.score(X_xp, y_xp)
+
+
+# Construct these outside the tests so that the same object is used
+# for both input and `expected`
+one_hot_encoder = OneHotEncoder()
+ordinal_encoder = OrdinalEncoder()
+
+# If we construct this directly via `MaskedArray`, the list of tuples
+# gets auto-converted to a 2D array.
+ma_with_tuples = np.ma.MaskedArray(np.empty(2), mask=True, dtype=object)  # type: ignore[var-annotated]
+ma_with_tuples[0] = (1, 2)
+ma_with_tuples[1] = (3, 4)
+
+
+@pytest.mark.parametrize(
+    ("candidate_params", "expected"),
+    [
+        pytest.param(
+            [{"foo": 1}, {"foo": 2}],
+            [
+                ("param_foo", np.ma.MaskedArray(np.array([1, 2]))),
+            ],
+            id="simple numeric, single param",
+        ),
+        pytest.param(
+            [{"foo": 1, "bar": 3}, {"foo": 2, "bar": 4}, {"foo": 3}],
+            [
+                ("param_foo", np.ma.MaskedArray(np.array([1, 2, 3]))),
+                (
+                    "param_bar",
+                    np.ma.MaskedArray(np.array([3, 4, 0]), mask=[False, False, True]),
+                ),
+            ],
+            id="simple numeric, one param is missing in one round",
+        ),
+        pytest.param(
+            [{"foo": [[1], [2], [3]]}, {"foo": [[1], [2]]}],
+            [
+                (
+                    "param_foo",
+                    np.ma.MaskedArray([[[1], [2], [3]], [[1], [2]]], dtype=object),
+                ),
+            ],
+            id="lists of different lengths",
+        ),
+        pytest.param(
+            [{"foo": (1, 2)}, {"foo": (3, 4)}],
+            [
+                (
+                    "param_foo",
+                    ma_with_tuples,
+                ),
+            ],
+            id="lists tuples",
+        ),
+        pytest.param(
+            [{"foo": ordinal_encoder}, {"foo": one_hot_encoder}],
+            [
+                (
+                    "param_foo",
+                    np.ma.MaskedArray([ordinal_encoder, one_hot_encoder], dtype=object),
+                ),
+            ],
+            id="estimators",
+        ),
+    ],
+)
+def test_yield_masked_array_for_each_param(candidate_params, expected):
+    result = list(_yield_masked_array_for_each_param(candidate_params))
+    for (key, value), (expected_key, expected_value) in zip(result, expected):
+        assert key == expected_key
+        assert value.dtype == expected_value.dtype
+        np.testing.assert_array_equal(value, expected_value)
+        np.testing.assert_array_equal(value.mask, expected_value.mask)
+
+
+def test_yield_masked_array_no_runtime_warning():
+    # non-regression test for https://github.com/scikit-learn/scikit-learn/issues/29929
+    candidate_params = [{"param": i} for i in range(1000)]
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", RuntimeWarning)
+        list(_yield_masked_array_for_each_param(candidate_params))
diff --git a/sklearn/model_selection/tests/test_split.py b/sklearn/model_selection/tests/test_split.py
index 80c19c7f2e08c..0f31055d9b7f9 100644
--- a/sklearn/model_selection/tests/test_split.py
+++ b/sklearn/model_selection/tests/test_split.py
@@ -1,66 +1,122 @@
 """Test the split module"""
-import warnings
-import pytest
+
 import re
+import warnings
+from itertools import combinations, combinations_with_replacement, permutations
+
 import numpy as np
-from scipy.sparse import coo_matrix, csc_matrix, csr_matrix
+import pytest
 from scipy import stats
+from scipy.sparse import issparse
 from scipy.special import comb
-from itertools import combinations
-from itertools import combinations_with_replacement
-from itertools import permutations
-
-from sklearn.utils._testing import assert_allclose
-from sklearn.utils._testing import assert_array_almost_equal
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils._testing import ignore_warnings
-from sklearn.utils.validation import _num_samples
-from sklearn.utils._mocking import MockDataFrame
-
-from sklearn.model_selection import cross_val_score
-from sklearn.model_selection import KFold
-from sklearn.model_selection import StratifiedKFold
-from sklearn.model_selection import GroupKFold
-from sklearn.model_selection import TimeSeriesSplit
-from sklearn.model_selection import LeaveOneOut
-from sklearn.model_selection import LeaveOneGroupOut
-from sklearn.model_selection import LeavePOut
-from sklearn.model_selection import LeavePGroupsOut
-from sklearn.model_selection import ShuffleSplit
-from sklearn.model_selection import GroupShuffleSplit
-from sklearn.model_selection import StratifiedShuffleSplit
-from sklearn.model_selection import PredefinedSplit
-from sklearn.model_selection import check_cv
-from sklearn.model_selection import train_test_split
-from sklearn.model_selection import GridSearchCV
-from sklearn.model_selection import RepeatedKFold
-from sklearn.model_selection import RepeatedStratifiedKFold
-
-from sklearn.linear_model import Ridge
-
-from sklearn.model_selection._split import _validate_shuffle_split
-from sklearn.model_selection._split import _build_repr
-from sklearn.model_selection._split import _yields_constant_splits
-
-from sklearn.datasets import load_digits
-from sklearn.datasets import make_classification
 
+from sklearn import config_context
+from sklearn.datasets import load_digits, make_classification
+from sklearn.dummy import DummyClassifier
+from sklearn.model_selection import (
+    GridSearchCV,
+    GroupKFold,
+    GroupShuffleSplit,
+    KFold,
+    LeaveOneGroupOut,
+    LeaveOneOut,
+    LeavePGroupsOut,
+    LeavePOut,
+    PredefinedSplit,
+    RepeatedKFold,
+    RepeatedStratifiedKFold,
+    ShuffleSplit,
+    StratifiedGroupKFold,
+    StratifiedKFold,
+    StratifiedShuffleSplit,
+    TimeSeriesSplit,
+    check_cv,
+    cross_val_score,
+    train_test_split,
+)
+from sklearn.model_selection._split import (
+    _build_repr,
+    _validate_shuffle_split,
+    _yields_constant_splits,
+)
 from sklearn.svm import SVC
+from sklearn.tests.metadata_routing_common import assert_request_is_empty
+from sklearn.utils._array_api import (
+    _convert_to_numpy,
+    _get_namespace_device_dtype_ids,
+    get_namespace,
+    yield_namespace_device_dtype_combinations,
+)
+from sklearn.utils._array_api import (
+    device as array_api_device,
+)
+from sklearn.utils._mocking import MockDataFrame
+from sklearn.utils._testing import (
+    assert_allclose,
+    assert_array_almost_equal,
+    assert_array_equal,
+    ignore_warnings,
+)
+from sklearn.utils.estimator_checks import (
+    _array_api_for_tests,
+)
+from sklearn.utils.fixes import COO_CONTAINERS, CSC_CONTAINERS, CSR_CONTAINERS
+from sklearn.utils.validation import _num_samples
+
+NO_GROUP_SPLITTERS = [
+    KFold(),
+    StratifiedKFold(),
+    TimeSeriesSplit(),
+    LeaveOneOut(),
+    LeavePOut(p=2),
+    ShuffleSplit(),
+    StratifiedShuffleSplit(test_size=0.5),
+    PredefinedSplit([1, 1, 2, 2]),
+    RepeatedKFold(),
+    RepeatedStratifiedKFold(),
+]
+
+GROUP_SPLITTERS = [
+    GroupKFold(),
+    LeavePGroupsOut(n_groups=1),
+    StratifiedGroupKFold(),
+    LeaveOneGroupOut(),
+    GroupShuffleSplit(),
+]
+GROUP_SPLITTER_NAMES = set(splitter.__class__.__name__ for splitter in GROUP_SPLITTERS)
+
+ALL_SPLITTERS = NO_GROUP_SPLITTERS + GROUP_SPLITTERS  # type: ignore[list-item]
+
+SPLITTERS_REQUIRING_TARGET = [
+    StratifiedKFold(),
+    StratifiedShuffleSplit(),
+    RepeatedStratifiedKFold(),
+]
 
 X = np.ones(10)
 y = np.arange(10) // 2
-P_sparse = coo_matrix(np.eye(5))
 test_groups = (
     np.array([1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3]),
     np.array([0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3]),
     np.array([0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2]),
     np.array([1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4]),
     [1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3],
-    ['1', '1', '1', '1', '2', '2', '2', '3', '3', '3', '3', '3'])
+    ["1", "1", "1", "1", "2", "2", "2", "3", "3", "3", "3", "3"],
+)
 digits = load_digits()
 
+pytestmark = pytest.mark.filterwarnings(
+    "error:The groups parameter:UserWarning:sklearn.*"
+)
+
+
+def _split(splitter, X, y, groups):
+    if splitter.__class__.__name__ in GROUP_SPLITTER_NAMES:
+        return splitter.split(X, y, groups=groups)
+    else:
+        return splitter.split(X, y)
+
 
-@ignore_warnings
 def test_cross_validator_with_default_params():
     n_samples = 4
     n_unique_groups = 4
@@ -80,6 +136,7 @@ def test_cross_validator_with_default_params():
     lopo = LeavePGroupsOut(p)
     ss = ShuffleSplit(random_state=0)
     ps = PredefinedSplit([1, 1, 2, 2])  # n_splits = np of unique folds = 2
+    sgkf = StratifiedGroupKFold(n_splits)
 
     loo_repr = "LeaveOneOut()"
     lpo_repr = "LeavePOut(p=2)"
@@ -87,29 +144,52 @@ def test_cross_validator_with_default_params():
     skf_repr = "StratifiedKFold(n_splits=2, random_state=None, shuffle=False)"
     lolo_repr = "LeaveOneGroupOut()"
     lopo_repr = "LeavePGroupsOut(n_groups=2)"
-    ss_repr = ("ShuffleSplit(n_splits=10, random_state=0, "
-               "test_size=None, train_size=None)")
+    ss_repr = (
+        "ShuffleSplit(n_splits=10, random_state=0, test_size=None, train_size=None)"
+    )
     ps_repr = "PredefinedSplit(test_fold=array([1, 1, 2, 2]))"
-
-    n_splits_expected = [n_samples, comb(n_samples, p), n_splits, n_splits,
-                         n_unique_groups, comb(n_unique_groups, p),
-                         n_shuffle_splits, 2]
-
-    for i, (cv, cv_repr) in enumerate(zip(
-            [loo, lpo, kf, skf, lolo, lopo, ss, ps],
-            [loo_repr, lpo_repr, kf_repr, skf_repr, lolo_repr, lopo_repr,
-             ss_repr, ps_repr])):
+    sgkf_repr = "StratifiedGroupKFold(n_splits=2, random_state=None, shuffle=False)"
+
+    n_splits_expected = [
+        n_samples,
+        comb(n_samples, p),
+        n_splits,
+        n_splits,
+        n_unique_groups,
+        comb(n_unique_groups, p),
+        n_shuffle_splits,
+        2,
+        n_splits,
+    ]
+
+    for i, (cv, cv_repr) in enumerate(
+        zip(
+            [loo, lpo, kf, skf, lolo, lopo, ss, ps, sgkf],
+            [
+                loo_repr,
+                lpo_repr,
+                kf_repr,
+                skf_repr,
+                lolo_repr,
+                lopo_repr,
+                ss_repr,
+                ps_repr,
+                sgkf_repr,
+            ],
+        )
+    ):
         # Test if get_n_splits works correctly
         assert n_splits_expected[i] == cv.get_n_splits(X, y, groups)
 
         # Test if the cross-validator works as expected even if
         # the data is 1d
-        np.testing.assert_equal(list(cv.split(X, y, groups)),
-                                list(cv.split(X_1d, y, groups)))
+        np.testing.assert_equal(
+            list(_split(cv, X, y, groups)), list(_split(cv, X_1d, y, groups))
+        )
         # Test that train, test indices returned are integers
-        for train, test in cv.split(X, y, groups):
-            assert np.asarray(train).dtype.kind == 'i'
-            assert np.asarray(test).dtype.kind == 'i'
+        for train, test in _split(cv, X, y, groups):
+            assert np.asarray(train).dtype.kind == "i"
+            assert np.asarray(test).dtype.kind == "i"
 
         # Test if the repr works without any errors
         assert cv_repr == repr(cv)
@@ -131,21 +211,33 @@ def test_2d_y():
     y_2d = y.reshape(-1, 1)
     y_multilabel = rng.randint(0, 2, size=(n_samples, 3))
     groups = rng.randint(0, 3, size=(n_samples,))
-    splitters = [LeaveOneOut(), LeavePOut(p=2), KFold(), StratifiedKFold(),
-                 RepeatedKFold(), RepeatedStratifiedKFold(),
-                 ShuffleSplit(), StratifiedShuffleSplit(test_size=.5),
-                 GroupShuffleSplit(), LeaveOneGroupOut(),
-                 LeavePGroupsOut(n_groups=2), GroupKFold(n_splits=3),
-                 TimeSeriesSplit(), PredefinedSplit(test_fold=groups)]
+    splitters = [
+        LeaveOneOut(),
+        LeavePOut(p=2),
+        KFold(),
+        StratifiedKFold(),
+        RepeatedKFold(),
+        RepeatedStratifiedKFold(),
+        StratifiedGroupKFold(),
+        ShuffleSplit(),
+        StratifiedShuffleSplit(test_size=0.5),
+        GroupShuffleSplit(),
+        LeaveOneGroupOut(),
+        LeavePGroupsOut(n_groups=2),
+        GroupKFold(n_splits=3),
+        TimeSeriesSplit(),
+        PredefinedSplit(test_fold=groups),
+    ]
     for splitter in splitters:
-        list(splitter.split(X, y, groups))
-        list(splitter.split(X, y_2d, groups))
+        list(_split(splitter, X, y, groups=groups))
+        list(_split(splitter, X, y_2d, groups=groups))
         try:
-            list(splitter.split(X, y_multilabel, groups))
+            list(_split(splitter, X, y_multilabel, groups=groups))
         except ValueError as e:
-            allowed_target_types = ('binary', 'multiclass')
+            allowed_target_types = ("binary", "multiclass")
             msg = "Supported target types are: {}. Got 'multilabel".format(
-                allowed_target_types)
+                allowed_target_types
+            )
             assert msg in str(e)
 
 
@@ -193,6 +285,11 @@ def test_kfold_valueerrors():
     with pytest.warns(Warning, match="The least populated class"):
         next(skf_3.split(X2, y))
 
+    sgkf_3 = StratifiedGroupKFold(3)
+    naive_groups = np.arange(len(y))
+    with pytest.warns(Warning, match="The least populated class"):
+        next(sgkf_3.split(X2, y, naive_groups))
+
     # Check that despite the warning the folds are still computed even
     # though all the classes are not necessarily represented at on each
     # side of the split at each split
@@ -200,24 +297,33 @@ def test_kfold_valueerrors():
         warnings.simplefilter("ignore")
         check_cv_coverage(skf_3, X2, y, groups=None, expected_n_splits=3)
 
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        check_cv_coverage(sgkf_3, X2, y, groups=naive_groups, expected_n_splits=3)
+
     # Check that errors are raised if all n_groups for individual
     # classes are less than n_splits.
     y = np.array([3, 3, -1, -1, 2])
 
     with pytest.raises(ValueError):
         next(skf_3.split(X2, y))
+    with pytest.raises(ValueError):
+        next(sgkf_3.split(X2, y))
 
     # Error when number of folds is <= 1
     with pytest.raises(ValueError):
         KFold(0)
     with pytest.raises(ValueError):
         KFold(1)
-    error_string = ("k-fold cross-validation requires at least one"
-                    " train/test split")
+    error_string = "k-fold cross-validation requires at least one train/test split"
     with pytest.raises(ValueError, match=error_string):
         StratifiedKFold(0)
     with pytest.raises(ValueError, match=error_string):
         StratifiedKFold(1)
+    with pytest.raises(ValueError, match=error_string):
+        StratifiedGroupKFold(0)
+    with pytest.raises(ValueError, match=error_string):
+        StratifiedGroupKFold(1)
 
     # When n_splits is not integer:
     with pytest.raises(ValueError):
@@ -228,6 +334,10 @@ def test_kfold_valueerrors():
         StratifiedKFold(1.5)
     with pytest.raises(ValueError):
         StratifiedKFold(2.0)
+    with pytest.raises(ValueError):
+        StratifiedGroupKFold(1.5)
+    with pytest.raises(ValueError):
+        StratifiedGroupKFold(2.0)
 
     # When shuffle is not  a bool:
     with pytest.raises(TypeError):
@@ -302,58 +412,73 @@ def test_stratified_kfold_no_shuffle():
 
     # Make sure string labels are also supported
     X = np.ones(7)
-    y1 = ['1', '1', '1', '0', '0', '0', '0']
+    y1 = ["1", "1", "1", "0", "0", "0", "0"]
     y2 = [1, 1, 1, 0, 0, 0, 0]
     np.testing.assert_equal(
-        list(StratifiedKFold(2).split(X, y1)),
-        list(StratifiedKFold(2).split(X, y2)))
+        list(StratifiedKFold(2).split(X, y1)), list(StratifiedKFold(2).split(X, y2))
+    )
 
     # Check equivalence to KFold
     y = [0, 1, 0, 1, 0, 1, 0, 1]
     X = np.ones_like(y)
     np.testing.assert_equal(
-        list(StratifiedKFold(3).split(X, y)),
-        list(KFold(3).split(X, y)))
+        list(StratifiedKFold(3).split(X, y)), list(KFold(3).split(X, y))
+    )
 
 
-@pytest.mark.parametrize('shuffle', [False, True])
-@pytest.mark.parametrize('k', [4, 5, 6, 7, 8, 9, 10])
-def test_stratified_kfold_ratios(k, shuffle):
+@pytest.mark.parametrize("shuffle", [False, True])
+@pytest.mark.parametrize("k", [4, 5, 6, 7, 8, 9, 10])
+@pytest.mark.parametrize("kfold", [StratifiedKFold, StratifiedGroupKFold])
+def test_stratified_kfold_ratios(k, shuffle, kfold):
     # Check that stratified kfold preserves class ratios in individual splits
     # Repeat with shuffling turned off and on
     n_samples = 1000
     X = np.ones(n_samples)
-    y = np.array([4] * int(0.10 * n_samples) +
-                 [0] * int(0.89 * n_samples) +
-                 [1] * int(0.01 * n_samples))
+    y = np.array(
+        [4] * int(0.10 * n_samples)
+        + [0] * int(0.89 * n_samples)
+        + [1] * int(0.01 * n_samples)
+    )
+    # ensure perfect stratification with StratifiedGroupKFold
+    groups = np.arange(len(y))
     distr = np.bincount(y) / len(y)
 
     test_sizes = []
     random_state = None if not shuffle else 0
-    skf = StratifiedKFold(k, random_state=random_state, shuffle=shuffle)
-    for train, test in skf.split(X, y):
+    skf = kfold(k, random_state=random_state, shuffle=shuffle)
+    for train, test in _split(skf, X, y, groups=groups):
         assert_allclose(np.bincount(y[train]) / len(train), distr, atol=0.02)
         assert_allclose(np.bincount(y[test]) / len(test), distr, atol=0.02)
         test_sizes.append(len(test))
     assert np.ptp(test_sizes) <= 1
 
 
-@pytest.mark.parametrize('shuffle', [False, True])
-@pytest.mark.parametrize('k', [4, 6, 7])
-def test_stratified_kfold_label_invariance(k, shuffle):
+@pytest.mark.parametrize("shuffle", [False, True])
+@pytest.mark.parametrize("k", [4, 6, 7])
+@pytest.mark.parametrize("kfold", [StratifiedKFold, StratifiedGroupKFold])
+def test_stratified_kfold_label_invariance(k, shuffle, kfold):
     # Check that stratified kfold gives the same indices regardless of labels
     n_samples = 100
-    y = np.array([2] * int(0.10 * n_samples) +
-                 [0] * int(0.89 * n_samples) +
-                 [1] * int(0.01 * n_samples))
+    y = np.array(
+        [2] * int(0.10 * n_samples)
+        + [0] * int(0.89 * n_samples)
+        + [1] * int(0.01 * n_samples)
+    )
     X = np.ones(len(y))
+    # ensure perfect stratification with StratifiedGroupKFold
+    groups = np.arange(len(y))
 
     def get_splits(y):
         random_state = None if not shuffle else 0
-        return [(list(train), list(test))
-                for train, test
-                in StratifiedKFold(k, random_state=random_state,
-                                   shuffle=shuffle).split(X, y)]
+        return [
+            (list(train), list(test))
+            for train, test in _split(
+                kfold(k, random_state=random_state, shuffle=shuffle),
+                X,
+                y,
+                groups=groups,
+            )
+        ]
 
     splits_base = get_splits(y)
     for perm in permutations([0, 1, 2]):
@@ -372,17 +497,20 @@ def test_kfold_balance():
         assert np.sum(sizes) == i
 
 
-def test_stratifiedkfold_balance():
+@pytest.mark.parametrize("kfold", [StratifiedKFold, StratifiedGroupKFold])
+def test_stratifiedkfold_balance(kfold):
     # Check that KFold returns folds with balanced sizes (only when
     # stratification is possible)
     # Repeat with shuffling turned off and on
     X = np.ones(17)
     y = [0] * 3 + [1] * 14
+    # ensure perfect stratification with StratifiedGroupKFold
+    groups = np.arange(len(y))
 
     for shuffle in (True, False):
-        cv = StratifiedKFold(3, shuffle=shuffle)
+        cv = kfold(3, shuffle=shuffle)
         for i in range(11, 17):
-            skf = cv.split(X[:i], y[:i])
+            skf = _split(cv, X[:i], y[:i], groups[:i])
             sizes = [len(test) for _, test in skf]
 
             assert (np.max(sizes) - np.min(sizes)) <= 1
@@ -399,7 +527,8 @@ def test_shuffle_kfold():
 
     all_folds = np.zeros(300)
     for (tr1, te1), (tr2, te2), (tr3, te3) in zip(
-            kf.split(X), kf2.split(X), kf3.split(X)):
+        kf.split(X), kf2.split(X), kf3.split(X)
+    ):
         for tr_a, tr_b in combinations((tr1, tr2, tr3), 2):
             # Assert that there is no complete overlap
             assert len(np.intersect1d(tr_a, tr_b)) != len(tr1)
@@ -411,39 +540,36 @@ def test_shuffle_kfold():
     assert sum(all_folds) == 300
 
 
-def test_shuffle_kfold_stratifiedkfold_reproducibility():
+@pytest.mark.parametrize("kfold", [KFold, StratifiedKFold, StratifiedGroupKFold])
+def test_shuffle_kfold_stratifiedkfold_reproducibility(kfold):
     X = np.ones(15)  # Divisible by 3
     y = [0] * 7 + [1] * 8
+    groups_1 = np.arange(len(y))
     X2 = np.ones(16)  # Not divisible by 3
     y2 = [0] * 8 + [1] * 8
+    groups_2 = np.arange(len(y2))
 
     # Check that when the shuffle is True, multiple split calls produce the
     # same split when random_state is int
-    kf = KFold(3, shuffle=True, random_state=0)
-    skf = StratifiedKFold(3, shuffle=True, random_state=0)
+    kf = kfold(3, shuffle=True, random_state=0)
 
-    for cv in (kf, skf):
-        np.testing.assert_equal(list(cv.split(X, y)), list(cv.split(X, y)))
-        np.testing.assert_equal(list(cv.split(X2, y2)), list(cv.split(X2, y2)))
+    np.testing.assert_equal(
+        list(_split(kf, X, y, groups_1)), list(_split(kf, X, y, groups_1))
+    )
 
     # Check that when the shuffle is True, multiple split calls often
     # (not always) produce different splits when random_state is
     # RandomState instance or None
-    kf = KFold(3, shuffle=True, random_state=np.random.RandomState(0))
-    skf = StratifiedKFold(3, shuffle=True,
-                          random_state=np.random.RandomState(0))
-
-    for cv in (kf, skf):
-        for data in zip((X, X2), (y, y2)):
-            # Test if the two splits are different cv
-            for (_, test_a), (_, test_b) in zip(cv.split(*data),
-                                                cv.split(*data)):
-                # cv.split(...) returns an array of tuples, each tuple
-                # consisting of an array with train indices and test indices
-                # Ensure that the splits for data are not same
-                # when random state is not set
-                with pytest.raises(AssertionError):
-                    np.testing.assert_array_equal(test_a, test_b)
+    kf = kfold(3, shuffle=True, random_state=np.random.RandomState(0))
+    for data in zip((X, X2), (y, y2), (groups_1, groups_2)):
+        # Test if the two splits are different cv
+        for (_, test_a), (_, test_b) in zip(_split(kf, *data), _split(kf, *data)):
+            # cv.split(...) returns an array of tuples, each tuple
+            # consisting of an array with train indices and test indices
+            # Ensure that the splits for data are not same
+            # when random state is not set
+            with pytest.raises(AssertionError):
+                np.testing.assert_array_equal(test_a, test_b)
 
 
 def test_shuffle_stratifiedkfold():
@@ -453,8 +579,7 @@ def test_shuffle_stratifiedkfold():
     y = [0] * 20 + [1] * 20
     kf0 = StratifiedKFold(5, shuffle=True, random_state=0)
     kf1 = StratifiedKFold(5, shuffle=True, random_state=1)
-    for (_, test0), (_, test1) in zip(kf0.split(X_40, y),
-                                      kf1.split(X_40, y)):
+    for (_, test0), (_, test1) in zip(kf0.split(X_40, y), kf1.split(X_40, y)):
         assert set(test0) != set(test1)
     check_cv_coverage(kf0, X_40, y, groups=None, expected_n_splits=5)
 
@@ -470,6 +595,30 @@ def test_shuffle_stratifiedkfold():
     assert test_set1 != test_set2
 
 
+def test_shuffle_groupkfold():
+    # Check that shuffling is happening when requested, and for proper
+    # sample coverage
+    X = np.ones(40)
+    y = [0] * 20 + [1] * 20
+    groups = np.arange(40) // 3
+    gkf0 = GroupKFold(4, shuffle=True, random_state=0)
+    gkf1 = GroupKFold(4, shuffle=True, random_state=1)
+
+    # Check that the groups are shuffled differently
+    test_groups0 = [
+        set(groups[test_idx]) for _, test_idx in gkf0.split(X, None, groups)
+    ]
+    test_groups1 = [
+        set(groups[test_idx]) for _, test_idx in gkf1.split(X, None, groups)
+    ]
+    for g0, g1 in zip(test_groups0, test_groups1):
+        assert g0 != g1, "Test groups should differ with different random states"
+
+    # Check coverage and splits
+    check_cv_coverage(gkf0, X, y, groups, expected_n_splits=4)
+    check_cv_coverage(gkf1, X, y, groups, expected_n_splits=4)
+
+
 def test_kfold_can_detect_dependent_samples_on_digits():  # see #2372
     # The digits samples are dependent: they are apparently grouped by authors
     # although we don't have any information on the groups segment locations
@@ -514,11 +663,101 @@ def test_kfold_can_detect_dependent_samples_on_digits():  # see #2372
     assert mean_score > 0.80
 
 
+def test_stratified_group_kfold_trivial():
+    sgkf = StratifiedGroupKFold(n_splits=3)
+    # Trivial example - groups with the same distribution
+    y = np.array([1] * 6 + [0] * 12)
+    X = np.ones_like(y).reshape(-1, 1)
+    groups = np.asarray((1, 2, 3, 4, 5, 6, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6))
+    distr = np.bincount(y) / len(y)
+    test_sizes = []
+    for train, test in sgkf.split(X, y, groups):
+        # check group constraint
+        assert np.intersect1d(groups[train], groups[test]).size == 0
+        # check y distribution
+        assert_allclose(np.bincount(y[train]) / len(train), distr, atol=0.02)
+        assert_allclose(np.bincount(y[test]) / len(test), distr, atol=0.02)
+        test_sizes.append(len(test))
+    assert np.ptp(test_sizes) <= 1
+
+
+def test_stratified_group_kfold_approximate():
+    # Not perfect stratification (even though it is possible) because of
+    # iteration over groups
+    sgkf = StratifiedGroupKFold(n_splits=3)
+    y = np.array([1] * 6 + [0] * 12)
+    X = np.ones_like(y).reshape(-1, 1)
+    groups = np.array([1, 2, 3, 3, 4, 4, 1, 1, 2, 2, 3, 4, 5, 5, 5, 6, 6, 6])
+    expected = np.asarray([[0.833, 0.166], [0.666, 0.333], [0.5, 0.5]])
+    test_sizes = []
+    for (train, test), expect_dist in zip(sgkf.split(X, y, groups), expected):
+        # check group constraint
+        assert np.intersect1d(groups[train], groups[test]).size == 0
+        split_dist = np.bincount(y[test]) / len(test)
+        assert_allclose(split_dist, expect_dist, atol=0.001)
+        test_sizes.append(len(test))
+    assert np.ptp(test_sizes) <= 1
+
+
+@pytest.mark.parametrize(
+    "y, groups, expected",
+    [
+        (
+            np.array([0] * 6 + [1] * 6),
+            np.array([1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6]),
+            np.asarray([[0.5, 0.5], [0.5, 0.5], [0.5, 0.5]]),
+        ),
+        (
+            np.array([0] * 9 + [1] * 3),
+            np.array([1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 5, 6]),
+            np.asarray([[0.75, 0.25], [0.75, 0.25], [0.75, 0.25]]),
+        ),
+    ],
+)
+def test_stratified_group_kfold_homogeneous_groups(y, groups, expected):
+    sgkf = StratifiedGroupKFold(n_splits=3)
+    X = np.ones_like(y).reshape(-1, 1)
+    for (train, test), expect_dist in zip(sgkf.split(X, y, groups), expected):
+        # check group constraint
+        assert np.intersect1d(groups[train], groups[test]).size == 0
+        split_dist = np.bincount(y[test]) / len(test)
+        assert_allclose(split_dist, expect_dist, atol=0.001)
+
+
+@pytest.mark.parametrize("cls_distr", [(0.4, 0.6), (0.3, 0.7), (0.2, 0.8), (0.8, 0.2)])
+@pytest.mark.parametrize("n_groups", [5, 30, 70])
+def test_stratified_group_kfold_against_group_kfold(cls_distr, n_groups):
+    # Check that given sufficient amount of samples StratifiedGroupKFold
+    # produces better stratified folds than regular GroupKFold
+    n_splits = 5
+    sgkf = StratifiedGroupKFold(n_splits=n_splits)
+    gkf = GroupKFold(n_splits=n_splits)
+    rng = np.random.RandomState(0)
+    n_points = 1000
+    y = rng.choice(2, size=n_points, p=cls_distr)
+    X = np.ones_like(y).reshape(-1, 1)
+    g = rng.choice(n_groups, n_points)
+    sgkf_folds = sgkf.split(X, y, groups=g)
+    gkf_folds = gkf.split(X, y, groups=g)
+    sgkf_entr = 0
+    gkf_entr = 0
+    for (sgkf_train, sgkf_test), (_, gkf_test) in zip(sgkf_folds, gkf_folds):
+        # check group constraint
+        assert np.intersect1d(g[sgkf_train], g[sgkf_test]).size == 0
+        sgkf_distr = np.bincount(y[sgkf_test]) / len(sgkf_test)
+        gkf_distr = np.bincount(y[gkf_test]) / len(gkf_test)
+        sgkf_entr += stats.entropy(sgkf_distr, qk=cls_distr)
+        gkf_entr += stats.entropy(gkf_distr, qk=cls_distr)
+    sgkf_entr /= n_splits
+    gkf_entr /= n_splits
+    assert sgkf_entr <= gkf_entr
+
+
 def test_shuffle_split():
     ss1 = ShuffleSplit(test_size=0.2, random_state=0).split(X)
     ss2 = ShuffleSplit(test_size=2, random_state=0).split(X)
     ss3 = ShuffleSplit(test_size=np.int32(2), random_state=0).split(X)
-    ss4 = ShuffleSplit(test_size=int(2), random_state=0).split(X)
+    ss4 = ShuffleSplit(test_size=2, random_state=0).split(X)
     for t1, t2, t3, t4 in zip(ss1, ss2, ss3, ss4):
         assert_array_equal(t1[0], t2[0])
         assert_array_equal(t2[0], t3[0])
@@ -528,14 +767,11 @@ def test_shuffle_split():
         assert_array_equal(t3[1], t4[1])
 
 
-@pytest.mark.parametrize("split_class", [ShuffleSplit,
-                                         StratifiedShuffleSplit])
-@pytest.mark.parametrize("train_size, exp_train, exp_test",
-                         [(None, 9, 1),
-                          (8, 8, 2),
-                          (0.8, 8, 2)])
-def test_shuffle_split_default_test_size(split_class, train_size, exp_train,
-                                         exp_test):
+@pytest.mark.parametrize("split_class", [ShuffleSplit, StratifiedShuffleSplit])
+@pytest.mark.parametrize(
+    "train_size, exp_train, exp_test", [(None, 9, 1), (8, 8, 2), (0.8, 8, 2)]
+)
+def test_shuffle_split_default_test_size(split_class, train_size, exp_train, exp_test):
     # Check that the default value has the expected behavior, i.e. 0.1 if both
     # unspecified or complement train_size unless both are specified.
     X = np.ones(10)
@@ -547,40 +783,36 @@ def test_shuffle_split_default_test_size(split_class, train_size, exp_train,
     assert len(X_test) == exp_test
 
 
-@pytest.mark.parametrize("train_size, exp_train, exp_test",
-                         [(None, 8, 2),
-                          (7, 7, 3),
-                          (0.7, 7, 3)])
-def test_group_shuffle_split_default_test_size(train_size, exp_train,
-                                               exp_test):
+@pytest.mark.parametrize(
+    "train_size, exp_train, exp_test", [(None, 8, 2), (7, 7, 3), (0.7, 7, 3)]
+)
+def test_group_shuffle_split_default_test_size(train_size, exp_train, exp_test):
     # Check that the default value has the expected behavior, i.e. 0.2 if both
     # unspecified or complement train_size unless both are specified.
     X = np.ones(10)
     y = np.ones(10)
     groups = range(10)
 
-    X_train, X_test = next(GroupShuffleSplit(train_size=train_size)
-                           .split(X, y, groups))
+    X_train, X_test = next(GroupShuffleSplit(train_size=train_size).split(X, y, groups))
 
     assert len(X_train) == exp_train
     assert len(X_test) == exp_test
 
 
-@ignore_warnings
 def test_stratified_shuffle_split_init():
     X = np.arange(7)
     y = np.asarray([0, 1, 1, 1, 2, 2, 2])
     # Check that error is raised if there is a class with only one sample
     with pytest.raises(ValueError):
-        next(StratifiedShuffleSplit(3, 0.2).split(X, y))
+        next(StratifiedShuffleSplit(3, test_size=0.2).split(X, y))
 
     # Check that error is raised if the test set size is smaller than n_classes
     with pytest.raises(ValueError):
-        next(StratifiedShuffleSplit(3, 2).split(X, y))
+        next(StratifiedShuffleSplit(3, test_size=2).split(X, y))
     # Check that error is raised if the train set size is smaller than
     # n_classes
     with pytest.raises(ValueError):
-        next(StratifiedShuffleSplit(3, 3, 2).split(X, y))
+        next(StratifiedShuffleSplit(3, test_size=3, train_size=2).split(X, y))
 
     X = np.arange(9)
     y = np.asarray([0, 0, 0, 1, 1, 1, 2, 2, 2])
@@ -596,27 +828,30 @@ def test_stratified_shuffle_split_respects_test_size():
     y = np.array([0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2])
     test_size = 5
     train_size = 10
-    sss = StratifiedShuffleSplit(6, test_size=test_size, train_size=train_size,
-                                 random_state=0).split(np.ones(len(y)), y)
+    sss = StratifiedShuffleSplit(
+        6, test_size=test_size, train_size=train_size, random_state=0
+    ).split(np.ones(len(y)), y)
     for train, test in sss:
         assert len(train) == train_size
         assert len(test) == test_size
 
 
 def test_stratified_shuffle_split_iter():
-    ys = [np.array([1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3]),
-          np.array([0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3]),
-          np.array([0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2] * 2),
-          np.array([1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4]),
-          np.array([-1] * 800 + [1] * 50),
-          np.concatenate([[i] * (100 + i) for i in range(11)]),
-          [1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3],
-          ['1', '1', '1', '1', '2', '2', '2', '3', '3', '3', '3', '3'],
-          ]
+    ys = [
+        np.array([1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3]),
+        np.array([0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3]),
+        np.array([0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2] * 2),
+        np.array([1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4]),
+        np.array([-1] * 800 + [1] * 50),
+        np.concatenate([[i] * (100 + i) for i in range(11)]),
+        [1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3],
+        ["1", "1", "1", "1", "2", "2", "2", "3", "3", "3", "3", "3"],
+    ]
 
     for y in ys:
-        sss = StratifiedShuffleSplit(6, test_size=0.33,
-                                     random_state=0).split(np.ones(len(y)), y)
+        sss = StratifiedShuffleSplit(6, test_size=0.33, random_state=0).split(
+            np.ones(len(y)), y
+        )
         y = np.asanyarray(y)  # To make it indexable for y[train]
         # this is how test-size is computed internally
         # in _validate_shuffle_split
@@ -625,17 +860,17 @@ def test_stratified_shuffle_split_iter():
         for train, test in sss:
             assert_array_equal(np.unique(y[train]), np.unique(y[test]))
             # Checks if folds keep classes proportions
-            p_train = (np.bincount(np.unique(y[train],
-                                   return_inverse=True)[1]) /
-                       float(len(y[train])))
-            p_test = (np.bincount(np.unique(y[test],
-                                  return_inverse=True)[1]) /
-                      float(len(y[test])))
+            p_train = np.bincount(np.unique(y[train], return_inverse=True)[1]) / float(
+                len(y[train])
+            )
+            p_test = np.bincount(np.unique(y[test], return_inverse=True)[1]) / float(
+                len(y[test])
+            )
             assert_array_almost_equal(p_train, p_test, 1)
             assert len(train) + len(test) == y.size
             assert len(train) == train_size
             assert len(test) == test_size
-            assert_array_equal(np.lib.arraysetops.intersect1d(train, test), [])
+            assert_array_equal(np.intersect1d(train, test), [])
 
 
 def test_stratified_shuffle_split_even():
@@ -651,14 +886,15 @@ def assert_counts_are_ok(idx_counts, p):
         bf = stats.binom(n_splits, p)
         for count in idx_counts:
             prob = bf.pmf(count)
-            assert prob > threshold, \
+            assert prob > threshold, (
                 "An index is not drawn with chance corresponding to even draws"
+            )
 
     for n_samples in (6, 22):
         groups = np.array((n_samples // 2) * [0, 1])
-        splits = StratifiedShuffleSplit(n_splits=n_splits,
-                                        test_size=1. / n_folds,
-                                        random_state=0)
+        splits = StratifiedShuffleSplit(
+            n_splits=n_splits, test_size=1.0 / n_folds, random_state=0
+        )
 
         train_counts = [0] * n_samples
         test_counts = [0] * n_samples
@@ -671,7 +907,8 @@ def assert_counts_are_ok(idx_counts, p):
         assert n_splits_actual == n_splits
 
         n_train, n_test = _validate_shuffle_split(
-            n_samples, test_size=1. / n_folds, train_size=1. - (1. / n_folds))
+            n_samples, test_size=1.0 / n_folds, train_size=1.0 - (1.0 / n_folds)
+        )
 
         assert len(train) == n_train
         assert len(test) == n_test
@@ -694,8 +931,7 @@ def test_stratified_shuffle_split_overlap_train_test_bug():
     y = [0, 1, 2, 3] * 3 + [4, 5] * 5
     X = np.ones_like(y)
 
-    sss = StratifiedShuffleSplit(n_splits=1,
-                                 test_size=0.5, random_state=0)
+    sss = StratifiedShuffleSplit(n_splits=1, test_size=0.5, random_state=0)
 
     train, test = next(sss.split(X=X, y=y))
 
@@ -708,8 +944,10 @@ def test_stratified_shuffle_split_overlap_train_test_bug():
 
 def test_stratified_shuffle_split_multilabel():
     # fix for issue 9037
-    for y in [np.array([[0, 1], [1, 0], [1, 0], [0, 1]]),
-              np.array([[0, 1], [1, 1], [1, 1], [0, 1]])]:
+    for y in [
+        np.array([[0, 1], [1, 0], [1, 0], [0, 1]]),
+        np.array([[0, 1], [1, 1], [1, 1], [0, 1]]),
+    ]:
         X = np.ones_like(y)
         sss = StratifiedShuffleSplit(n_splits=1, test_size=0.5, random_state=0)
         train, test = next(sss.split(X=X, y=y))
@@ -754,7 +992,7 @@ def test_stratified_shuffle_split_multilabel_many_labels():
 
 def test_predefinedsplit_with_kfold_split():
     # Check that PredefinedSplit can reproduce a split generated by Kfold.
-    folds = np.full(10, -1.)
+    folds = np.full(10, -1.0)
     kf_train = []
     kf_test = []
     for i, (train_ind, test_ind) in enumerate(KFold(5, shuffle=True).split(X)):
@@ -773,7 +1011,7 @@ def test_group_shuffle_split():
     for groups_i in test_groups:
         X = y = np.ones(len(groups_i))
         n_splits = 6
-        test_size = 1. / 3
+        test_size = 1.0 / 3
         slo = GroupShuffleSplit(n_splits, test_size=test_size, random_state=0)
 
         # Make sure the repr works
@@ -789,8 +1027,8 @@ def test_group_shuffle_split():
             # First test: no train group is in the test set and vice versa
             l_train_unique = np.unique(l[train])
             l_test_unique = np.unique(l[test])
-            assert not np.any(np.in1d(l[train], l_test_unique))
-            assert not np.any(np.in1d(l[test], l_train_unique))
+            assert not np.any(np.isin(l[train], l_test_unique))
+            assert not np.any(np.isin(l[test], l_train_unique))
 
             # Second test: train and test add up to all the data
             assert l[train].size + l[test].size == l.size
@@ -800,10 +1038,10 @@ def test_group_shuffle_split():
 
             # Fourth test:
             # unique train and test groups are correct, +- 1 for rounding error
-            assert abs(len(l_test_unique) -
-                       round(test_size * len(l_unique))) <= 1
-            assert abs(len(l_train_unique) -
-                       round((1.0 - test_size) * len(l_unique))) <= 1
+            assert abs(len(l_test_unique) - round(test_size * len(l_unique))) <= 1
+            assert (
+                abs(len(l_train_unique) - round((1.0 - test_size) * len(l_unique))) <= 1
+            )
 
 
 def test_leave_one_p_group_out():
@@ -812,18 +1050,15 @@ def test_leave_one_p_group_out():
     lpgo_2 = LeavePGroupsOut(n_groups=2)
 
     # Make sure the repr works
-    assert repr(logo) == 'LeaveOneGroupOut()'
-    assert repr(lpgo_1) == 'LeavePGroupsOut(n_groups=1)'
-    assert repr(lpgo_2) == 'LeavePGroupsOut(n_groups=2)'
-    assert (repr(LeavePGroupsOut(n_groups=3)) ==
-                 'LeavePGroupsOut(n_groups=3)')
-
-    for j, (cv, p_groups_out) in enumerate(((logo, 1), (lpgo_1, 1),
-                                            (lpgo_2, 2))):
+    assert repr(logo) == "LeaveOneGroupOut()"
+    assert repr(lpgo_1) == "LeavePGroupsOut(n_groups=1)"
+    assert repr(lpgo_2) == "LeavePGroupsOut(n_groups=2)"
+    assert repr(LeavePGroupsOut(n_groups=3)) == "LeavePGroupsOut(n_groups=3)"
+
+    for j, (cv, p_groups_out) in enumerate(((logo, 1), (lpgo_1, 1), (lpgo_2, 2))):
         for i, groups_i in enumerate(test_groups):
             n_groups = len(np.unique(groups_i))
-            n_splits = (n_groups if p_groups_out == 1
-                        else n_groups * (n_groups - 1) / 2)
+            n_splits = n_groups if p_groups_out == 1 else n_groups * (n_groups - 1) / 2
             X = y = np.ones(len(groups_i))
 
             # Test that the length is correct
@@ -834,9 +1069,9 @@ def test_leave_one_p_group_out():
             # Split using the original list / array / list of string groups_i
             for train, test in cv.split(X, y, groups=groups_i):
                 # First test: no train group is in the test set and vice versa
-                assert_array_equal(np.intersect1d(groups_arr[train],
-                                                  groups_arr[test]).tolist(),
-                                   [])
+                assert_array_equal(
+                    np.intersect1d(groups_arr[train], groups_arr[test]).tolist(), []
+                )
 
                 # Second test: train and test add up to all the data
                 assert len(train) + len(test) == len(groups_i)
@@ -846,7 +1081,7 @@ def test_leave_one_p_group_out():
                 assert np.unique(groups_arr[test]).shape[0], p_groups_out
 
     # check get_n_splits() with dummy parameters
-    assert logo.get_n_splits(None, None, ['a', 'b', 'c', 'b', 'c']) == 3
+    assert logo.get_n_splits(None, None, ["a", "b", "c", "b", "c"]) == 3
     assert logo.get_n_splits(groups=[1.0, 1.1, 1.0, 1.2]) == 3
     assert lpgo_2.get_n_splits(None, None, np.arange(4)) == 6
     assert lpgo_1.get_n_splits(groups=np.arange(4)) == 4
@@ -881,12 +1116,29 @@ def test_leave_group_out_changing_groups():
             assert_array_equal(test, test_chan)
 
     # n_splits = no of 2 (p) group combinations of the unique groups = 3C2 = 3
-    assert (
-        3 == LeavePGroupsOut(n_groups=2).get_n_splits(X, y=X,
-                                                    groups=groups))
+    assert 3 == LeavePGroupsOut(n_groups=2).get_n_splits(X, y=X, groups=groups)
     # n_splits = no of unique groups (C(uniq_lbls, 1) = n_unique_groups)
-    assert 3 == LeaveOneGroupOut().get_n_splits(X, y=X,
-                                                groups=groups)
+    assert 3 == LeaveOneGroupOut().get_n_splits(X, y=X, groups=groups)
+
+
+def test_leave_group_out_order_dependence():
+    # Check that LeaveOneGroupOut orders the splits according to the index
+    # of the group left out.
+    groups = np.array([2, 2, 0, 0, 1, 1])
+    X = np.ones(len(groups))
+
+    splits = iter(LeaveOneGroupOut().split(X, groups=groups))
+
+    expected_indices = [
+        ([0, 1, 4, 5], [2, 3]),
+        ([0, 1, 2, 3], [4, 5]),
+        ([2, 3, 4, 5], [0, 1]),
+    ]
+
+    for expected_train, expected_test in expected_indices:
+        train, test = next(splits)
+        assert_array_equal(train, expected_train)
+        assert_array_equal(test, expected_test)
 
 
 def test_leave_one_p_group_out_error_on_fewer_number_of_groups():
@@ -898,33 +1150,32 @@ def test_leave_one_p_group_out_error_on_fewer_number_of_groups():
     X = y = groups = np.ones(1)
     msg = re.escape(
         f"The groups parameter contains fewer than 2 unique groups ({groups})."
-        f" LeaveOneGroupOut expects at least 2."
+        " LeaveOneGroupOut expects at least 2."
     )
     with pytest.raises(ValueError, match=msg):
         next(LeaveOneGroupOut().split(X, y, groups))
 
     X = y = groups = np.ones(1)
     msg = re.escape(
-        f"The groups parameter contains fewer than (or equal to) n_groups "
+        "The groups parameter contains fewer than (or equal to) n_groups "
         f"(3) numbers of unique groups ({groups}). LeavePGroupsOut expects "
-        f"that at least n_groups + 1 (4) unique groups "
-        f"be present"
+        "that at least n_groups + 1 (4) unique groups "
+        "be present"
     )
     with pytest.raises(ValueError, match=msg):
         next(LeavePGroupsOut(n_groups=3).split(X, y, groups))
 
     X = y = groups = np.arange(3)
     msg = re.escape(
-        f"The groups parameter contains fewer than (or equal to) n_groups "
+        "The groups parameter contains fewer than (or equal to) n_groups "
         f"(3) numbers of unique groups ({groups}). LeavePGroupsOut expects "
-        f"that at least n_groups + 1 (4) unique groups "
-        f"be present"
+        "that at least n_groups + 1 (4) unique groups "
+        "be present"
     )
     with pytest.raises(ValueError, match=msg):
         next(LeavePGroupsOut(n_groups=3).split(X, y, groups))
 
 
-@ignore_warnings
 def test_repeated_cv_value_errors():
     # n_repeats is not integer or <= 0
     for cv in (RepeatedKFold, RepeatedStratifiedKFold):
@@ -934,24 +1185,20 @@ def test_repeated_cv_value_errors():
             cv(n_repeats=1.5)
 
 
-@pytest.mark.parametrize(
-    "RepeatedCV", [RepeatedKFold, RepeatedStratifiedKFold]
-)
+@pytest.mark.parametrize("RepeatedCV", [RepeatedKFold, RepeatedStratifiedKFold])
 def test_repeated_cv_repr(RepeatedCV):
     n_splits, n_repeats = 2, 6
     repeated_cv = RepeatedCV(n_splits=n_splits, n_repeats=n_repeats)
-    repeated_cv_repr = ('{}(n_repeats=6, n_splits=2, random_state=None)'
-                        .format(repeated_cv.__class__.__name__))
+    repeated_cv_repr = "{}(n_repeats=6, n_splits=2, random_state=None)".format(
+        repeated_cv.__class__.__name__
+    )
     assert repeated_cv_repr == repr(repeated_cv)
 
 
 def test_repeated_kfold_determinstic_split():
     X = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]]
     random_state = 258173307
-    rkf = RepeatedKFold(
-        n_splits=2,
-        n_repeats=2,
-        random_state=random_state)
+    rkf = RepeatedKFold(n_splits=2, n_repeats=2, random_state=random_state)
 
     # split should produce same and deterministic splits on
     # each call
@@ -997,10 +1244,7 @@ def test_repeated_stratified_kfold_determinstic_split():
     X = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]]
     y = [1, 1, 1, 0, 0]
     random_state = 1944695409
-    rskf = RepeatedStratifiedKFold(
-        n_splits=2,
-        n_repeats=2,
-        random_state=random_state)
+    rskf = RepeatedStratifiedKFold(n_splits=2, n_repeats=2, random_state=random_state)
 
     # split should produce same and deterministic splits on
     # each call
@@ -1031,59 +1275,32 @@ def test_train_test_split_errors():
 
     pytest.raises(ValueError, train_test_split, range(3), train_size=1.1)
 
-    pytest.raises(ValueError, train_test_split, range(3), test_size=0.6,
-                  train_size=0.6)
-    pytest.raises(ValueError, train_test_split, range(3),
-                  test_size=np.float32(0.6), train_size=np.float32(0.6))
-    pytest.raises(ValueError, train_test_split, range(3),
-                  test_size="wrong_type")
-    pytest.raises(ValueError, train_test_split, range(3), test_size=2,
-                  train_size=4)
-    pytest.raises(TypeError, train_test_split, range(3),
-                  some_argument=1.1)
+    pytest.raises(ValueError, train_test_split, range(3), test_size=0.6, train_size=0.6)
+    pytest.raises(
+        ValueError,
+        train_test_split,
+        range(3),
+        test_size=np.float32(0.6),
+        train_size=np.float32(0.6),
+    )
+    pytest.raises(ValueError, train_test_split, range(3), test_size="wrong_type")
+    pytest.raises(ValueError, train_test_split, range(3), test_size=2, train_size=4)
+    pytest.raises(TypeError, train_test_split, range(3), some_argument=1.1)
     pytest.raises(ValueError, train_test_split, range(3), range(42))
-    pytest.raises(ValueError, train_test_split, range(10),
-                  shuffle=False, stratify=True)
+    pytest.raises(ValueError, train_test_split, range(10), shuffle=False, stratify=True)
 
-    with pytest.raises(ValueError,
-                       match=r'train_size=11 should be either positive and '
-                             r'smaller than the number of samples 10 or a '
-                             r'float in the \(0, 1\) range'):
+    with pytest.raises(
+        ValueError,
+        match=r"train_size=11 should be either positive and "
+        r"smaller than the number of samples 10 or a "
+        r"float in the \(0, 1\) range",
+    ):
         train_test_split(range(10), train_size=11, test_size=1)
 
 
-@pytest.mark.parametrize("train_size,test_size", [
-    (1.2, 0.8),
-    (1., 0.8),
-    (0.0, 0.8),
-    (-.2, 0.8),
-    (0.8, 1.2),
-    (0.8, 1.),
-    (0.8, 0.),
-    (0.8, -.2)])
-def test_train_test_split_invalid_sizes1(train_size, test_size):
-    with pytest.raises(ValueError,
-                       match=r'should be .* in the \(0, 1\) range'):
-        train_test_split(range(10), train_size=train_size, test_size=test_size)
-
-
-@pytest.mark.parametrize("train_size,test_size", [
-    (-10, 0.8),
-    (0, 0.8),
-    (11, 0.8),
-    (0.8, -10),
-    (0.8, 0),
-    (0.8, 11)])
-def test_train_test_split_invalid_sizes2(train_size, test_size):
-    with pytest.raises(ValueError,
-                       match=r'should be either positive and smaller'):
-        train_test_split(range(10), train_size=train_size, test_size=test_size)
-
-
-@pytest.mark.parametrize("train_size, exp_train, exp_test",
-                         [(None, 7, 3),
-                          (8, 8, 2),
-                          (0.8, 8, 2)])
+@pytest.mark.parametrize(
+    "train_size, exp_train, exp_test", [(None, 7, 3), (8, 8, 2), (0.8, 8, 2)]
+)
 def test_train_test_split_default_test_size(train_size, exp_train, exp_test):
     # Check that the default value has the expected behavior, i.e. complement
     # train_size unless both are specified.
@@ -1093,13 +1310,82 @@ def test_train_test_split_default_test_size(train_size, exp_train, exp_test):
     assert len(X_test) == exp_test
 
 
-def test_train_test_split():
+@pytest.mark.parametrize(
+    "array_namespace, device, dtype_name",
+    yield_namespace_device_dtype_combinations(),
+    ids=_get_namespace_device_dtype_ids,
+)
+@pytest.mark.parametrize(
+    "shuffle,stratify",
+    (
+        (True, None),
+        (True, np.hstack((np.ones(6), np.zeros(4)))),
+        # stratification only works with shuffling
+        (False, None),
+    ),
+)
+def test_array_api_train_test_split(
+    shuffle, stratify, array_namespace, device, dtype_name
+):
+    xp = _array_api_for_tests(array_namespace, device)
+
     X = np.arange(100).reshape((10, 10))
-    X_s = coo_matrix(X)
+    y = np.arange(10)
+
+    X_np = X.astype(dtype_name)
+    X_xp = xp.asarray(X_np, device=device)
+
+    y_np = y.astype(dtype_name)
+    y_xp = xp.asarray(y_np, device=device)
+
+    X_train_np, X_test_np, y_train_np, y_test_np = train_test_split(
+        X_np, y, random_state=0, shuffle=shuffle, stratify=stratify
+    )
+    with config_context(array_api_dispatch=True):
+        if stratify is not None:
+            stratify_xp = xp.asarray(stratify)
+        else:
+            stratify_xp = stratify
+        X_train_xp, X_test_xp, y_train_xp, y_test_xp = train_test_split(
+            X_xp, y_xp, shuffle=shuffle, stratify=stratify_xp, random_state=0
+        )
+
+        # Check that namespace is preserved, has to happen with
+        # array_api_dispatch enabled.
+        assert get_namespace(X_train_xp)[0] == get_namespace(X_xp)[0]
+        assert get_namespace(X_test_xp)[0] == get_namespace(X_xp)[0]
+        assert get_namespace(y_train_xp)[0] == get_namespace(y_xp)[0]
+        assert get_namespace(y_test_xp)[0] == get_namespace(y_xp)[0]
+
+    # Check device and dtype is preserved on output
+    assert array_api_device(X_train_xp) == array_api_device(X_xp)
+    assert array_api_device(y_train_xp) == array_api_device(y_xp)
+    assert array_api_device(X_test_xp) == array_api_device(X_xp)
+    assert array_api_device(y_test_xp) == array_api_device(y_xp)
+
+    assert X_train_xp.dtype == X_xp.dtype
+    assert y_train_xp.dtype == y_xp.dtype
+    assert X_test_xp.dtype == X_xp.dtype
+    assert y_test_xp.dtype == y_xp.dtype
+
+    assert_allclose(
+        _convert_to_numpy(X_train_xp, xp=xp),
+        X_train_np,
+    )
+    assert_allclose(
+        _convert_to_numpy(X_test_xp, xp=xp),
+        X_test_np,
+    )
+
+
+@pytest.mark.parametrize("coo_container", COO_CONTAINERS)
+def test_train_test_split(coo_container):
+    X = np.arange(100).reshape((10, 10))
+    X_s = coo_container(X)
     y = np.arange(10)
 
     # simple test
-    split = train_test_split(X, y, test_size=None, train_size=.5)
+    split = train_test_split(X, y, test_size=None, train_size=0.5)
     X_train, X_test, y_train, y_test = split
     assert len(y_test) == len(y_train)
     # test correspondence of X and y
@@ -1123,11 +1409,10 @@ def test_train_test_split():
 
     # test stratification option
     y = np.array([1, 1, 1, 1, 2, 2, 2, 2])
-    for test_size, exp_test_size in zip([2, 4, 0.25, 0.5, 0.75],
-                                        [2, 4, 2, 4, 6]):
-        train, test = train_test_split(y, test_size=test_size,
-                                       stratify=y,
-                                       random_state=0)
+    for test_size, exp_test_size in zip([2, 4, 0.25, 0.5, 0.75], [2, 4, 2, 4, 6]):
+        train, test = train_test_split(
+            y, test_size=test_size, stratify=y, random_state=0
+        )
         assert len(test) == exp_test_size
         assert len(test) + len(train) == len(y)
         # check the 1:1 ratio of ones and twos in the data is preserved
@@ -1141,12 +1426,35 @@ def test_train_test_split():
         assert_array_equal(train, [0, 1, 2, 3, 4, 5, 6, 7])
 
 
-@ignore_warnings
+def test_train_test_split_32bit_overflow():
+    """Check for integer overflow on 32-bit platforms.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/20774
+    """
+
+    # A number 'n' big enough for expression 'n * n * train_size' to cause
+    # an overflow for signed 32-bit integer
+    big_number = 100000
+
+    # Definition of 'y' is a part of reproduction - population for at least
+    # one class should be in the same order of magnitude as size of X
+    X = np.arange(big_number)
+    y = X > (0.99 * big_number)
+
+    split = train_test_split(X, y, stratify=y, train_size=0.25)
+    X_train, X_test, y_train, y_test = split
+
+    assert X_train.size + X_test.size == big_number
+    assert y_train.size + y_test.size == big_number
+
+
 def test_train_test_split_pandas():
     # check train_test_split doesn't destroy pandas dataframe
     types = [MockDataFrame]
     try:
         from pandas import DataFrame
+
         types.append(DataFrame)
     except ImportError:
         pass
@@ -1158,16 +1466,17 @@ def test_train_test_split_pandas():
         assert isinstance(X_test, InputFeatureType)
 
 
-def test_train_test_split_sparse():
+@pytest.mark.parametrize(
+    "sparse_container", COO_CONTAINERS + CSC_CONTAINERS + CSR_CONTAINERS
+)
+def test_train_test_split_sparse(sparse_container):
     # check that train_test_split converts scipy sparse matrices
     # to csr, as stated in the documentation
     X = np.arange(100).reshape((10, 10))
-    sparse_types = [csr_matrix, csc_matrix, coo_matrix]
-    for InputFeatureType in sparse_types:
-        X_s = InputFeatureType(X)
-        X_train, X_test = train_test_split(X_s)
-        assert isinstance(X_train, csr_matrix)
-        assert isinstance(X_test, csr_matrix)
+    X_s = sparse_container(X)
+    X_train, X_test = train_test_split(X_s)
+    assert issparse(X_train) and X_train.format == "csr"
+    assert issparse(X_test) and X_test.format == "csr"
 
 
 def test_train_test_split_mock_pandas():
@@ -1182,17 +1491,20 @@ def test_train_test_split_mock_pandas():
 def test_train_test_split_list_input():
     # Check that when y is a list / list of string labels, it works.
     X = np.ones(7)
-    y1 = ['1'] * 4 + ['0'] * 3
+    y1 = ["1"] * 4 + ["0"] * 3
     y2 = np.hstack((np.ones(4), np.zeros(3)))
     y3 = y2.tolist()
 
     for stratify in (True, False):
         X_train1, X_test1, y_train1, y_test1 = train_test_split(
-            X, y1, stratify=y1 if stratify else None, random_state=0)
+            X, y1, stratify=y1 if stratify else None, random_state=0
+        )
         X_train2, X_test2, y_train2, y_test2 = train_test_split(
-            X, y2, stratify=y2 if stratify else None, random_state=0)
+            X, y2, stratify=y2 if stratify else None, random_state=0
+        )
         X_train3, X_test3, y_train3, y_test3 = train_test_split(
-            X, y3, stratify=y3 if stratify else None, random_state=0)
+            X, y3, stratify=y3 if stratify else None, random_state=0
+        )
 
         np.testing.assert_equal(X_train1, X_train2)
         np.testing.assert_equal(y_train2, y_train3)
@@ -1200,14 +1512,10 @@ def test_train_test_split_list_input():
         np.testing.assert_equal(y_test3, y_test2)
 
 
-@pytest.mark.parametrize("test_size, train_size",
-                         [(2.0, None),
-                          (1.0, None),
-                          (0.1, 0.95),
-                          (None, 1j),
-                          (11, None),
-                          (10, None),
-                          (8, 3)])
+@pytest.mark.parametrize(
+    "test_size, train_size",
+    [(2.0, None), (1.0, None), (0.1, 0.95), (None, 1j), (11, None), (10, None), (8, 3)],
+)
 def test_shufflesplit_errors(test_size, train_size):
     with pytest.raises(ValueError):
         next(ShuffleSplit(test_size=test_size, train_size=train_size).split(X))
@@ -1217,22 +1525,19 @@ def test_shufflesplit_reproducible():
     # Check that iterating twice on the ShuffleSplit gives the same
     # sequence of train-test when the random_state is given
     ss = ShuffleSplit(random_state=21)
-    assert_array_equal(list(a for a, b in ss.split(X)),
-                       list(a for a, b in ss.split(X)))
+    assert_array_equal([a for a, b in ss.split(X)], [a for a, b in ss.split(X)])
 
 
 def test_stratifiedshufflesplit_list_input():
     # Check that when y is a list / list of string labels, it works.
     sss = StratifiedShuffleSplit(test_size=2, random_state=42)
     X = np.ones(7)
-    y1 = ['1'] * 4 + ['0'] * 3
+    y1 = ["1"] * 4 + ["0"] * 3
     y2 = np.hstack((np.ones(4), np.zeros(3)))
     y3 = y2.tolist()
 
-    np.testing.assert_equal(list(sss.split(X, y1)),
-                            list(sss.split(X, y2)))
-    np.testing.assert_equal(list(sss.split(X, y3)),
-                            list(sss.split(X, y2)))
+    np.testing.assert_equal(list(sss.split(X, y1)), list(sss.split(X, y2)))
+    np.testing.assert_equal(list(sss.split(X, y3)), list(sss.split(X, y2)))
 
 
 def test_train_test_split_allow_nans():
@@ -1252,26 +1557,32 @@ def test_check_cv():
 
     y_binary = np.array([0, 1, 0, 1, 0, 0, 1, 1, 1])
     cv = check_cv(3, y_binary, classifier=True)
-    np.testing.assert_equal(list(StratifiedKFold(3).split(X, y_binary)),
-                            list(cv.split(X, y_binary)))
+    np.testing.assert_equal(
+        list(StratifiedKFold(3).split(X, y_binary)), list(cv.split(X, y_binary))
+    )
 
     y_multiclass = np.array([0, 1, 0, 1, 2, 1, 2, 0, 2])
     cv = check_cv(3, y_multiclass, classifier=True)
-    np.testing.assert_equal(list(StratifiedKFold(3).split(X, y_multiclass)),
-                            list(cv.split(X, y_multiclass)))
+    np.testing.assert_equal(
+        list(StratifiedKFold(3).split(X, y_multiclass)), list(cv.split(X, y_multiclass))
+    )
     # also works with 2d multiclass
     y_multiclass_2d = y_multiclass.reshape(-1, 1)
     cv = check_cv(3, y_multiclass_2d, classifier=True)
-    np.testing.assert_equal(list(StratifiedKFold(3).split(X, y_multiclass_2d)),
-                            list(cv.split(X, y_multiclass_2d)))
+    np.testing.assert_equal(
+        list(StratifiedKFold(3).split(X, y_multiclass_2d)),
+        list(cv.split(X, y_multiclass_2d)),
+    )
 
     assert not np.all(
-        next(StratifiedKFold(3).split(X, y_multiclass_2d))[0] ==
-        next(KFold(3).split(X, y_multiclass_2d))[0])
+        next(StratifiedKFold(3).split(X, y_multiclass_2d))[0]
+        == next(KFold(3).split(X, y_multiclass_2d))[0]
+    )
 
     X = np.ones(5)
-    y_multilabel = np.array([[0, 0, 0, 0], [0, 1, 1, 0], [0, 0, 0, 1],
-                             [1, 1, 0, 1], [0, 0, 1, 0]])
+    y_multilabel = np.array(
+        [[0, 0, 0, 0], [0, 1, 1, 0], [0, 0, 0, 1], [1, 1, 0, 1], [0, 0, 1, 0]]
+    )
     cv = check_cv(3, y_multilabel, classifier=True)
     np.testing.assert_equal(list(KFold(3).split(X)), list(cv.split(X)))
 
@@ -1289,29 +1600,37 @@ def test_cv_iterable_wrapper():
     # Since the wrapped iterable is enlisted and stored,
     # split can be called any number of times to produce
     # consistent results.
-    np.testing.assert_equal(list(kf_iter_wrapped.split(X, y)),
-                            list(kf_iter_wrapped.split(X, y)))
+    np.testing.assert_equal(
+        list(kf_iter_wrapped.split(X, y)), list(kf_iter_wrapped.split(X, y))
+    )
     # If the splits are randomized, successive calls to split yields different
     # results
     kf_randomized_iter = KFold(shuffle=True, random_state=0).split(X, y)
     kf_randomized_iter_wrapped = check_cv(kf_randomized_iter)
     # numpy's assert_array_equal properly compares nested lists
-    np.testing.assert_equal(list(kf_randomized_iter_wrapped.split(X, y)),
-                            list(kf_randomized_iter_wrapped.split(X, y)))
+    np.testing.assert_equal(
+        list(kf_randomized_iter_wrapped.split(X, y)),
+        list(kf_randomized_iter_wrapped.split(X, y)),
+    )
 
     try:
         splits_are_equal = True
-        np.testing.assert_equal(list(kf_iter_wrapped.split(X, y)),
-                                list(kf_randomized_iter_wrapped.split(X, y)))
+        np.testing.assert_equal(
+            list(kf_iter_wrapped.split(X, y)),
+            list(kf_randomized_iter_wrapped.split(X, y)),
+        )
     except AssertionError:
         splits_are_equal = False
     assert not splits_are_equal, (
         "If the splits are randomized, "
-        "successive calls to split should yield different results")
+        "successive calls to split should yield different results"
+    )
 
 
-def test_group_kfold():
-    rng = np.random.RandomState(0)
+@pytest.mark.parametrize("kfold", [GroupKFold, StratifiedGroupKFold])
+@pytest.mark.parametrize("shuffle", [True, False])
+def test_group_kfold(kfold, shuffle, global_random_seed):
+    rng = np.random.RandomState(global_random_seed)
 
     # Parameters of the test
     n_groups = 15
@@ -1329,15 +1648,15 @@ def test_group_kfold():
     len(np.unique(groups))
     # Get the test fold indices from the test set indices of each fold
     folds = np.zeros(n_samples)
-    lkf = GroupKFold(n_splits=n_splits)
+    random_state = None if not shuffle else global_random_seed
+    lkf = kfold(n_splits=n_splits, shuffle=shuffle, random_state=random_state)
     for i, (_, test) in enumerate(lkf.split(X, y, groups)):
         folds[test] = i
 
     # Check that folds have approximately the same size
     assert len(folds) == len(groups)
     for i in np.unique(folds):
-        assert (tolerance >=
-                             abs(sum(folds == i) - ideal_n_groups_per_fold))
+        assert tolerance >= abs(sum(folds == i) - ideal_n_groups_per_fold)
 
     # Check that each group appears only in 1 fold
     for group in np.unique(groups):
@@ -1349,13 +1668,48 @@ def test_group_kfold():
         assert len(np.intersect1d(groups[train], groups[test])) == 0
 
     # Construct the test data
-    groups = np.array(['Albert', 'Jean', 'Bertrand', 'Michel', 'Jean',
-                       'Francis', 'Robert', 'Michel', 'Rachel', 'Lois',
-                       'Michelle', 'Bernard', 'Marion', 'Laura', 'Jean',
-                       'Rachel', 'Franck', 'John', 'Gael', 'Anna', 'Alix',
-                       'Robert', 'Marion', 'David', 'Tony', 'Abel', 'Becky',
-                       'Madmood', 'Cary', 'Mary', 'Alexandre', 'David',
-                       'Francis', 'Barack', 'Abdoul', 'Rasha', 'Xi', 'Silvia'])
+    groups = np.array(
+        [
+            "Albert",
+            "Jean",
+            "Bertrand",
+            "Michel",
+            "Jean",
+            "Francis",
+            "Robert",
+            "Michel",
+            "Rachel",
+            "Lois",
+            "Michelle",
+            "Bernard",
+            "Marion",
+            "Laura",
+            "Jean",
+            "Rachel",
+            "Franck",
+            "John",
+            "Gael",
+            "Anna",
+            "Alix",
+            "Robert",
+            "Marion",
+            "David",
+            "Tony",
+            "Abel",
+            "Becky",
+            "Madmood",
+            "Cary",
+            "Mary",
+            "Alexandre",
+            "David",
+            "Francis",
+            "Barack",
+            "Abdoul",
+            "Rasha",
+            "Xi",
+            "Silvia",
+        ]
+    )
 
     n_groups = len(np.unique(groups))
     n_samples = len(groups)
@@ -1372,9 +1726,9 @@ def test_group_kfold():
 
     # Check that folds have approximately the same size
     assert len(folds) == len(groups)
-    for i in np.unique(folds):
-        assert (tolerance >=
-                             abs(sum(folds == i) - ideal_n_groups_per_fold))
+    if not shuffle:
+        for i in np.unique(folds):
+            assert tolerance >= abs(sum(folds == i) - ideal_n_groups_per_fold)
 
     # Check that each group appears only in 1 fold
     with warnings.catch_warnings():
@@ -1388,19 +1742,17 @@ def test_group_kfold():
         assert len(np.intersect1d(groups[train], groups[test])) == 0
 
     # groups can also be a list
+    # use a new instance for reproducibility when shuffle=True
+    lkf_copy = kfold(n_splits=n_splits, shuffle=shuffle, random_state=random_state)
     cv_iter = list(lkf.split(X, y, groups.tolist()))
-    for (train1, test1), (train2, test2) in zip(lkf.split(X, y, groups),
-                                                cv_iter):
+    for (train1, test1), (train2, test2) in zip(lkf_copy.split(X, y, groups), cv_iter):
         assert_array_equal(train1, train2)
         assert_array_equal(test1, test2)
 
     # Should fail if there are more folds than groups
     groups = np.array([1, 1, 1, 2, 2])
     X = y = np.ones(len(groups))
-    with pytest.raises(
-        ValueError,
-        match="Cannot have number of splits.*greater"
-    ):
+    with pytest.raises(ValueError, match="Cannot have number of splits.*greater"):
         next(GroupKFold(n_splits=3).split(X, y, groups))
 
 
@@ -1408,10 +1760,7 @@ def test_time_series_cv():
     X = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10], [11, 12], [13, 14]]
 
     # Should fail if there are more folds than samples
-    with pytest.raises(
-        ValueError,
-        match="Cannot have number of folds.*greater"
-    ):
+    with pytest.raises(ValueError, match="Cannot have number of folds.*greater"):
         next(TimeSeriesSplit(n_splits=7).split(X))
 
     tscv = TimeSeriesSplit(2)
@@ -1486,8 +1835,7 @@ def test_time_series_test_size():
     assert_array_equal(test, [7, 8, 9])
 
     # Test with max_train_size
-    splits = TimeSeriesSplit(n_splits=2, test_size=2,
-                             max_train_size=4).split(X)
+    splits = TimeSeriesSplit(n_splits=2, test_size=2, max_train_size=4).split(X)
 
     train, test = next(splits)
     assert_array_equal(train, [2, 3, 4, 5])
@@ -1533,8 +1881,7 @@ def test_time_series_gap():
     assert_array_equal(test, [8, 9])
 
     # Test with test_size
-    splits = TimeSeriesSplit(n_splits=2, gap=2,
-                             max_train_size=4, test_size=2).split(X)
+    splits = TimeSeriesSplit(n_splits=2, gap=2, max_train_size=4, test_size=2).split(X)
 
     train, test = next(splits)
     assert_array_equal(train, [0, 1, 2, 3])
@@ -1561,6 +1908,7 @@ def test_time_series_gap():
         next(splits)
 
 
+@ignore_warnings
 def test_nested_cv():
     # Test if nested cross validation works with different combinations of cv
     rng = np.random.RandomState(0)
@@ -1568,15 +1916,26 @@ def test_nested_cv():
     X, y = make_classification(n_samples=15, n_classes=2, random_state=0)
     groups = rng.randint(0, 5, 15)
 
-    cvs = [LeaveOneGroupOut(), LeaveOneOut(), GroupKFold(n_splits=3),
-           StratifiedKFold(),
-           StratifiedShuffleSplit(n_splits=3, random_state=0)]
+    cvs = [
+        LeaveOneGroupOut(),
+        StratifiedKFold(n_splits=2),
+        LeaveOneOut(),
+        GroupKFold(n_splits=3),
+        StratifiedKFold(),
+        StratifiedGroupKFold(),
+        StratifiedShuffleSplit(n_splits=3, random_state=0),
+    ]
 
     for inner_cv, outer_cv in combinations_with_replacement(cvs, 2):
-        gs = GridSearchCV(Ridge(), param_grid={'alpha': [1, .1]},
-                          cv=inner_cv, error_score='raise')
-        cross_val_score(gs, X=X, y=y, groups=groups, cv=outer_cv,
-                        fit_params={'groups': groups})
+        gs = GridSearchCV(
+            DummyClassifier(),
+            param_grid={"strategy": ["stratified", "most_frequent"]},
+            cv=inner_cv,
+            error_score="raise",
+        )
+        cross_val_score(
+            gs, X=X, y=y, groups=groups, cv=outer_cv, params={"groups": groups}
+        )
 
 
 def test_build_repr():
@@ -1592,41 +1951,49 @@ def __repr__(self):
     assert repr(MockSplitter(5, 6)) == "MockSplitter(a=5, b=6, c=None)"
 
 
-@pytest.mark.parametrize('CVSplitter', (ShuffleSplit, GroupShuffleSplit,
-                                        StratifiedShuffleSplit))
+@pytest.mark.parametrize(
+    "CVSplitter", (ShuffleSplit, GroupShuffleSplit, StratifiedShuffleSplit)
+)
 def test_shuffle_split_empty_trainset(CVSplitter):
-    cv = CVSplitter(test_size=.99)
+    cv = CVSplitter(test_size=0.99)
     X, y = [[1]], [0]  # 1 sample
     with pytest.raises(
-            ValueError,
-            match='With n_samples=1, test_size=0.99 and train_size=None, '
-            'the resulting train set will be empty'):
-        next(cv.split(X, y, groups=[1]))
+        ValueError,
+        match=(
+            "With n_samples=1, test_size=0.99 and train_size=None, "
+            "the resulting train set will be empty"
+        ),
+    ):
+        next(_split(cv, X, y, groups=[1]))
 
 
 def test_train_test_split_empty_trainset():
-    X, = [[1]]  # 1 sample
+    (X,) = [[1]]  # 1 sample
     with pytest.raises(
-            ValueError,
-            match='With n_samples=1, test_size=0.99 and train_size=None, '
-            'the resulting train set will be empty'):
-        train_test_split(X, test_size=.99)
+        ValueError,
+        match=(
+            "With n_samples=1, test_size=0.99 and train_size=None, "
+            "the resulting train set will be empty"
+        ),
+    ):
+        train_test_split(X, test_size=0.99)
 
     X = [[1], [1], [1]]  # 3 samples, ask for more than 2 thirds
     with pytest.raises(
-            ValueError,
-            match='With n_samples=3, test_size=0.67 and train_size=None, '
-            'the resulting train set will be empty'):
-        train_test_split(X, test_size=.67)
+        ValueError,
+        match=(
+            "With n_samples=3, test_size=0.67 and train_size=None, "
+            "the resulting train set will be empty"
+        ),
+    ):
+        train_test_split(X, test_size=0.67)
 
 
 def test_leave_one_out_empty_trainset():
     # LeaveOneGroup out expect at least 2 groups so no need to check
     cv = LeaveOneOut()
     X, y = [[1]], [0]  # 1 sample
-    with pytest.raises(
-            ValueError,
-            match='Cannot perform LeaveOneOut with n_samples=1'):
+    with pytest.raises(ValueError, match="Cannot perform LeaveOneOut with n_samples=1"):
         next(cv.split(X, y))
 
 
@@ -1635,52 +2002,101 @@ def test_leave_p_out_empty_trainset():
     cv = LeavePOut(p=2)
     X, y = [[1], [2]], [0, 3]  # 2 samples
     with pytest.raises(
-            ValueError,
-            match='p=2 must be strictly less than the number of samples=2'):
-        next(cv.split(X, y, groups=[1, 2]))
+        ValueError, match="p=2 must be strictly less than the number of samples=2"
+    ):
+        next(cv.split(X, y))
 
 
-@pytest.mark.parametrize('Klass', (KFold, StratifiedKFold))
+@pytest.mark.parametrize(
+    "Klass", (KFold, StratifiedKFold, StratifiedGroupKFold, GroupKFold)
+)
 def test_random_state_shuffle_false(Klass):
     # passing a non-default random_state when shuffle=False makes no sense
-    with pytest.raises(ValueError,
-                       match='has no effect since shuffle is False'):
+    with pytest.raises(ValueError, match="has no effect since shuffle is False"):
         Klass(3, shuffle=False, random_state=0)
 
 
-@pytest.mark.parametrize('cv, expected', [
-    (KFold(), True),
-    (KFold(shuffle=True, random_state=123), True),
-    (StratifiedKFold(), True),
-    (StratifiedKFold(shuffle=True, random_state=123), True),
-    (RepeatedKFold(random_state=123), True),
-    (RepeatedStratifiedKFold(random_state=123), True),
-    (ShuffleSplit(random_state=123), True),
-    (GroupShuffleSplit(random_state=123), True),
-    (StratifiedShuffleSplit(random_state=123), True),
-    (GroupKFold(), True),
-    (TimeSeriesSplit(), True),
-    (LeaveOneOut(), True),
-    (LeaveOneGroupOut(), True),
-    (LeavePGroupsOut(n_groups=2), True),
-    (LeavePOut(p=2), True),
-
-    (KFold(shuffle=True, random_state=None), False),
-    (KFold(shuffle=True, random_state=None), False),
-    (StratifiedKFold(shuffle=True, random_state=np.random.RandomState(0)),
-     False),
-    (StratifiedKFold(shuffle=True, random_state=np.random.RandomState(0)),
-     False),
-    (RepeatedKFold(random_state=None), False),
-    (RepeatedKFold(random_state=np.random.RandomState(0)), False),
-    (RepeatedStratifiedKFold(random_state=None), False),
-    (RepeatedStratifiedKFold(random_state=np.random.RandomState(0)), False),
-    (ShuffleSplit(random_state=None), False),
-    (ShuffleSplit(random_state=np.random.RandomState(0)), False),
-    (GroupShuffleSplit(random_state=None), False),
-    (GroupShuffleSplit(random_state=np.random.RandomState(0)), False),
-    (StratifiedShuffleSplit(random_state=None), False),
-    (StratifiedShuffleSplit(random_state=np.random.RandomState(0)), False),
-])
+@pytest.mark.parametrize(
+    "cv, expected",
+    [
+        (KFold(), True),
+        (KFold(shuffle=True, random_state=123), True),
+        (StratifiedKFold(), True),
+        (StratifiedKFold(shuffle=True, random_state=123), True),
+        (StratifiedGroupKFold(shuffle=True, random_state=123), True),
+        (StratifiedGroupKFold(), True),
+        (RepeatedKFold(random_state=123), True),
+        (RepeatedStratifiedKFold(random_state=123), True),
+        (ShuffleSplit(random_state=123), True),
+        (GroupShuffleSplit(random_state=123), True),
+        (StratifiedShuffleSplit(random_state=123), True),
+        (GroupKFold(), True),
+        (GroupKFold(shuffle=True, random_state=123), True),
+        (TimeSeriesSplit(), True),
+        (LeaveOneOut(), True),
+        (LeaveOneGroupOut(), True),
+        (LeavePGroupsOut(n_groups=2), True),
+        (LeavePOut(p=2), True),
+        (KFold(shuffle=True, random_state=None), False),
+        (KFold(shuffle=True, random_state=None), False),
+        (StratifiedKFold(shuffle=True, random_state=np.random.RandomState(0)), False),
+        (StratifiedKFold(shuffle=True, random_state=np.random.RandomState(0)), False),
+        (RepeatedKFold(random_state=None), False),
+        (RepeatedKFold(random_state=np.random.RandomState(0)), False),
+        (RepeatedStratifiedKFold(random_state=None), False),
+        (RepeatedStratifiedKFold(random_state=np.random.RandomState(0)), False),
+        (ShuffleSplit(random_state=None), False),
+        (ShuffleSplit(random_state=np.random.RandomState(0)), False),
+        (GroupShuffleSplit(random_state=None), False),
+        (GroupShuffleSplit(random_state=np.random.RandomState(0)), False),
+        (StratifiedShuffleSplit(random_state=None), False),
+        (StratifiedShuffleSplit(random_state=np.random.RandomState(0)), False),
+    ],
+)
 def test_yields_constant_splits(cv, expected):
     assert _yields_constant_splits(cv) == expected
+
+
+@pytest.mark.parametrize("cv", ALL_SPLITTERS, ids=[str(cv) for cv in ALL_SPLITTERS])
+def test_splitter_get_metadata_routing(cv):
+    """Check get_metadata_routing returns the correct MetadataRouter."""
+    assert hasattr(cv, "get_metadata_routing")
+    metadata = cv.get_metadata_routing()
+    if cv in GROUP_SPLITTERS:
+        assert metadata.split.requests["groups"] is True
+    elif cv in NO_GROUP_SPLITTERS:
+        assert not metadata.split.requests
+
+    assert_request_is_empty(metadata, exclude=["split"])
+
+
+@pytest.mark.parametrize("cv", ALL_SPLITTERS, ids=[str(cv) for cv in ALL_SPLITTERS])
+def test_splitter_set_split_request(cv):
+    """Check set_split_request is defined for group splitters and not for others."""
+    if cv in GROUP_SPLITTERS:
+        assert hasattr(cv, "set_split_request")
+    elif cv in NO_GROUP_SPLITTERS:
+        assert not hasattr(cv, "set_split_request")
+
+
+@pytest.mark.parametrize("cv", NO_GROUP_SPLITTERS, ids=str)
+def test_no_group_splitters_warns_with_groups(cv):
+    msg = f"The groups parameter is ignored by {cv.__class__.__name__}"
+
+    n_samples = 30
+    rng = np.random.RandomState(1)
+    X = rng.randint(0, 3, size=(n_samples, 2))
+    y = rng.randint(0, 3, size=(n_samples,))
+    groups = rng.randint(0, 3, size=(n_samples,))
+
+    with pytest.warns(UserWarning, match=msg):
+        cv.split(X, y, groups=groups)
+
+
+@pytest.mark.parametrize(
+    "cv", SPLITTERS_REQUIRING_TARGET, ids=[str(cv) for cv in SPLITTERS_REQUIRING_TARGET]
+)
+def test_stratified_splitter_without_y(cv):
+    msg = "missing 1 required positional argument: 'y'"
+    with pytest.raises(TypeError, match=msg):
+        cv.split(X)
diff --git a/sklearn/model_selection/tests/test_successive_halving.py b/sklearn/model_selection/tests/test_successive_halving.py
index 2c55f6aa6cd85..bdfab45b4f7ca 100644
--- a/sklearn/model_selection/tests/test_successive_halving.py
+++ b/sklearn/model_selection/tests/test_successive_halving.py
@@ -1,17 +1,33 @@
 from math import ceil
 
-import pytest
-from scipy.stats import norm, randint
 import numpy as np
+import pytest
+from scipy.stats import expon, norm, randint
 
 from sklearn.datasets import make_classification
 from sklearn.dummy import DummyClassifier
-from sklearn.experimental import enable_halving_search_cv  # noqa
-from sklearn.model_selection import HalvingGridSearchCV
-from sklearn.model_selection import HalvingRandomSearchCV
-from sklearn.model_selection import KFold, ShuffleSplit
+from sklearn.experimental import enable_halving_search_cv  # noqa: F401
+from sklearn.model_selection import (
+    GroupKFold,
+    GroupShuffleSplit,
+    HalvingGridSearchCV,
+    HalvingRandomSearchCV,
+    KFold,
+    LeaveOneGroupOut,
+    LeavePGroupsOut,
+    ShuffleSplit,
+    StratifiedKFold,
+    StratifiedShuffleSplit,
+)
 from sklearn.model_selection._search_successive_halving import (
-    _SubsampleMetaSplitter, _top_k, _refit_callable)
+    _SubsampleMetaSplitter,
+    _top_k,
+)
+from sklearn.model_selection.tests.test_search import (
+    check_cv_results_array_types,
+    check_cv_results_keys,
+)
+from sklearn.svm import SVC, LinearSVC
 
 
 class FastClassifier(DummyClassifier):
@@ -20,68 +36,158 @@ class FastClassifier(DummyClassifier):
     These parameter don't affect the predictions and are useful for fast
     grid searching."""
 
-    def __init__(self, strategy='stratified', random_state=None,
-                 constant=None, **kwargs):
-        super().__init__(strategy=strategy, random_state=random_state,
-                         constant=constant)
+    # update the constraints such that we accept all parameters from a to z
+    _parameter_constraints: dict = {
+        **DummyClassifier._parameter_constraints,
+        **{chr(key): "no_validation" for key in range(ord("a"), ord("z") + 1)},
+    }
+
+    def __init__(
+        self, strategy="stratified", random_state=None, constant=None, **kwargs
+    ):
+        super().__init__(
+            strategy=strategy, random_state=random_state, constant=constant
+        )
 
     def get_params(self, deep=False):
         params = super().get_params(deep=deep)
-        for char in range(ord('a'), ord('z') + 1):
-            params[chr(char)] = 'whatever'
+        for char in range(ord("a"), ord("z") + 1):
+            params[chr(char)] = "whatever"
         return params
 
 
-@pytest.mark.parametrize('Est', (HalvingGridSearchCV, HalvingRandomSearchCV))
+class SometimesFailClassifier(DummyClassifier):
+    def __init__(
+        self,
+        strategy="stratified",
+        random_state=None,
+        constant=None,
+        n_estimators=10,
+        fail_fit=False,
+        fail_predict=False,
+        a=0,
+    ):
+        self.fail_fit = fail_fit
+        self.fail_predict = fail_predict
+        self.n_estimators = n_estimators
+        self.a = a
+
+        super().__init__(
+            strategy=strategy, random_state=random_state, constant=constant
+        )
+
+    def fit(self, X, y):
+        if self.fail_fit:
+            raise Exception("fitting failed")
+        return super().fit(X, y)
+
+    def predict(self, X):
+        if self.fail_predict:
+            raise Exception("predict failed")
+        return super().predict(X)
+
+
+@pytest.mark.filterwarnings("ignore::sklearn.exceptions.FitFailedWarning")
+@pytest.mark.filterwarnings("ignore:Scoring failed:UserWarning")
+@pytest.mark.filterwarnings("ignore:One or more of the:UserWarning")
+@pytest.mark.parametrize("HalvingSearch", (HalvingGridSearchCV, HalvingRandomSearchCV))
+@pytest.mark.parametrize("fail_at", ("fit", "predict"))
+def test_nan_handling(HalvingSearch, fail_at):
+    """Check the selection of the best scores in presence of failure represented by
+    NaN values."""
+    n_samples = 1_000
+    X, y = make_classification(n_samples=n_samples, random_state=0)
+
+    search = HalvingSearch(
+        SometimesFailClassifier(),
+        {f"fail_{fail_at}": [False, True], "a": range(3)},
+        resource="n_estimators",
+        max_resources=6,
+        min_resources=1,
+        factor=2,
+    )
+
+    search.fit(X, y)
+
+    # estimators that failed during fit/predict should always rank lower
+    # than ones where the fit/predict succeeded
+    assert not search.best_params_[f"fail_{fail_at}"]
+    scores = search.cv_results_["mean_test_score"]
+    ranks = search.cv_results_["rank_test_score"]
+
+    # some scores should be NaN
+    assert np.isnan(scores).any()
+
+    unique_nan_ranks = np.unique(ranks[np.isnan(scores)])
+    # all NaN scores should have the same rank
+    assert unique_nan_ranks.shape[0] == 1
+    # NaNs should have the lowest rank
+    assert (unique_nan_ranks[0] >= ranks).all()
+
+
+@pytest.mark.parametrize("Est", (HalvingGridSearchCV, HalvingRandomSearchCV))
 @pytest.mark.parametrize(
-    ('aggressive_elimination,'
-     'max_resources,'
-     'expected_n_iterations,'
-     'expected_n_required_iterations,'
-     'expected_n_possible_iterations,'
-     'expected_n_remaining_candidates,'
-     'expected_n_candidates,'
-     'expected_n_resources,'), [
-         # notice how it loops at the beginning
-         # also, the number of candidates evaluated at the last iteration is
-         # <= factor
-         (True, 'limited', 4, 4, 3, 1, [60, 20, 7, 3], [20, 20, 60, 180]),
-         # no aggressive elimination: we end up with less iterations, and
-         # the number of candidates at the last iter is > factor, which isn't
-         # ideal
-         (False, 'limited', 3, 4, 3, 3, [60, 20, 7], [20, 60, 180]),
+    (
+        "aggressive_elimination,"
+        "max_resources,"
+        "expected_n_iterations,"
+        "expected_n_required_iterations,"
+        "expected_n_possible_iterations,"
+        "expected_n_remaining_candidates,"
+        "expected_n_candidates,"
+        "expected_n_resources,"
+    ),
+    [
+        # notice how it loops at the beginning
+        # also, the number of candidates evaluated at the last iteration is
+        # <= factor
+        (True, "limited", 4, 4, 3, 1, [60, 20, 7, 3], [20, 20, 60, 180]),
+        # no aggressive elimination: we end up with less iterations, and
+        # the number of candidates at the last iter is > factor, which isn't
+        # ideal
+        (False, "limited", 3, 4, 3, 3, [60, 20, 7], [20, 60, 180]),
         #  # When the amount of resource isn't limited, aggressive_elimination
         #  # has no effect. Here the default min_resources='exhaust' will take
         #  # over.
-         (True, 'unlimited', 4, 4, 4, 1, [60, 20, 7, 3], [37, 111, 333, 999]),
-         (False, 'unlimited', 4, 4, 4, 1, [60, 20, 7, 3], [37, 111, 333, 999]),
-     ]
+        (True, "unlimited", 4, 4, 4, 1, [60, 20, 7, 3], [37, 111, 333, 999]),
+        (False, "unlimited", 4, 4, 4, 1, [60, 20, 7, 3], [37, 111, 333, 999]),
+    ],
 )
 def test_aggressive_elimination(
-        Est, aggressive_elimination, max_resources, expected_n_iterations,
-        expected_n_required_iterations, expected_n_possible_iterations,
-        expected_n_remaining_candidates, expected_n_candidates,
-        expected_n_resources):
+    Est,
+    aggressive_elimination,
+    max_resources,
+    expected_n_iterations,
+    expected_n_required_iterations,
+    expected_n_possible_iterations,
+    expected_n_remaining_candidates,
+    expected_n_candidates,
+    expected_n_resources,
+):
     # Test the aggressive_elimination parameter.
 
     n_samples = 1000
     X, y = make_classification(n_samples=n_samples, random_state=0)
-    param_grid = {'a': ('l1', 'l2'), 'b': list(range(30))}
+    param_grid = {"a": ("l1", "l2"), "b": list(range(30))}
     base_estimator = FastClassifier()
 
-    if max_resources == 'limited':
+    if max_resources == "limited":
         max_resources = 180
     else:
         max_resources = n_samples
 
-    sh = Est(base_estimator, param_grid,
-             aggressive_elimination=aggressive_elimination,
-             max_resources=max_resources, factor=3)
+    sh = Est(
+        base_estimator,
+        param_grid,
+        aggressive_elimination=aggressive_elimination,
+        max_resources=max_resources,
+        factor=3,
+    )
     sh.set_params(verbose=True)  # just for test coverage
 
     if Est is HalvingRandomSearchCV:
         # same number of candidates as with the grid
-        sh.set_params(n_candidates=2 * 30, min_resources='exhaust')
+        sh.set_params(n_candidates=2 * 30, min_resources="exhaust")
 
     sh.fit(X, y)
 
@@ -94,44 +200,56 @@ def test_aggressive_elimination(
     assert ceil(sh.n_candidates_[-1] / sh.factor) == sh.n_remaining_candidates_
 
 
-@pytest.mark.parametrize('Est', (HalvingGridSearchCV, HalvingRandomSearchCV))
+@pytest.mark.parametrize("Est", (HalvingGridSearchCV, HalvingRandomSearchCV))
 @pytest.mark.parametrize(
-    ('min_resources,'
-     'max_resources,'
-     'expected_n_iterations,'
-     'expected_n_possible_iterations,'
-     'expected_n_resources,'), [
-         # with enough resources
-         ('smallest', 'auto', 2, 4, [20, 60]),
-         # with enough resources but min_resources set manually
-         (50, 'auto', 2, 3, [50, 150]),
-         # without enough resources, only one iteration can be done
-         ('smallest', 30, 1, 1, [20]),
-         # with exhaust: use as much resources as possible at the last iter
-         ('exhaust', 'auto', 2, 2, [333, 999]),
-         ('exhaust', 1000, 2, 2, [333, 999]),
-         ('exhaust', 999, 2, 2, [333, 999]),
-         ('exhaust', 600, 2, 2, [200, 600]),
-         ('exhaust', 599, 2, 2, [199, 597]),
-         ('exhaust', 300, 2, 2, [100, 300]),
-         ('exhaust', 60, 2, 2, [20, 60]),
-         ('exhaust', 50, 1, 1, [20]),
-         ('exhaust', 20, 1, 1, [20]),
-     ]
+    (
+        "min_resources,"
+        "max_resources,"
+        "expected_n_iterations,"
+        "expected_n_possible_iterations,"
+        "expected_n_resources,"
+    ),
+    [
+        # with enough resources
+        ("smallest", "auto", 2, 4, [20, 60]),
+        # with enough resources but min_resources set manually
+        (50, "auto", 2, 3, [50, 150]),
+        # without enough resources, only one iteration can be done
+        ("smallest", 30, 1, 1, [20]),
+        # with exhaust: use as much resources as possible at the last iter
+        ("exhaust", "auto", 2, 2, [333, 999]),
+        ("exhaust", 1000, 2, 2, [333, 999]),
+        ("exhaust", 999, 2, 2, [333, 999]),
+        ("exhaust", 600, 2, 2, [200, 600]),
+        ("exhaust", 599, 2, 2, [199, 597]),
+        ("exhaust", 300, 2, 2, [100, 300]),
+        ("exhaust", 60, 2, 2, [20, 60]),
+        ("exhaust", 50, 1, 1, [20]),
+        ("exhaust", 20, 1, 1, [20]),
+    ],
 )
 def test_min_max_resources(
-        Est, min_resources, max_resources, expected_n_iterations,
-        expected_n_possible_iterations,
-        expected_n_resources):
+    Est,
+    min_resources,
+    max_resources,
+    expected_n_iterations,
+    expected_n_possible_iterations,
+    expected_n_resources,
+):
     # Test the min_resources and max_resources parameters, and how they affect
     # the number of resources used at each iteration
     n_samples = 1000
     X, y = make_classification(n_samples=n_samples, random_state=0)
-    param_grid = {'a': [1, 2], 'b': [1, 2, 3]}
+    param_grid = {"a": [1, 2], "b": [1, 2, 3]}
     base_estimator = FastClassifier()
 
-    sh = Est(base_estimator, param_grid, factor=3, min_resources=min_resources,
-             max_resources=max_resources)
+    sh = Est(
+        base_estimator,
+        param_grid,
+        factor=3,
+        min_resources=min_resources,
+        max_resources=max_resources,
+    )
     if Est is HalvingRandomSearchCV:
         sh.set_params(n_candidates=6)  # same number as with the grid
 
@@ -142,15 +260,15 @@ def test_min_max_resources(
     assert sh.n_required_iterations_ == expected_n_required_iterations
     assert sh.n_possible_iterations_ == expected_n_possible_iterations
     assert sh.n_resources_ == expected_n_resources
-    if min_resources == 'exhaust':
-        assert (sh.n_possible_iterations_ == sh.n_iterations_ ==
-                len(sh.n_resources_))
+    if min_resources == "exhaust":
+        assert sh.n_possible_iterations_ == sh.n_iterations_ == len(sh.n_resources_)
 
 
-@pytest.mark.parametrize('Est', (HalvingRandomSearchCV, HalvingGridSearchCV))
+@pytest.mark.parametrize("Est", (HalvingRandomSearchCV, HalvingGridSearchCV))
 @pytest.mark.parametrize(
-    'max_resources, n_iterations, n_possible_iterations', [
-        ('auto', 5, 9),  # all resources are used
+    "max_resources, n_iterations, n_possible_iterations",
+    [
+        ("auto", 5, 9),  # all resources are used
         (1024, 5, 9),
         (700, 5, 8),
         (512, 5, 8),
@@ -159,20 +277,27 @@ def test_min_max_resources(
         (31, 3, 3),
         (16, 3, 3),
         (4, 1, 1),  # max_resources == min_resources, only one iteration is
-                    # possible
-    ])
+        # possible
+    ],
+)
 def test_n_iterations(Est, max_resources, n_iterations, n_possible_iterations):
     # test the number of actual iterations that were run depending on
     # max_resources
 
     n_samples = 1024
     X, y = make_classification(n_samples=n_samples, random_state=1)
-    param_grid = {'a': [1, 2], 'b': list(range(10))}
+    param_grid = {"a": [1, 2], "b": list(range(10))}
     base_estimator = FastClassifier()
     factor = 2
 
-    sh = Est(base_estimator, param_grid, cv=2, factor=factor,
-             max_resources=max_resources, min_resources=4)
+    sh = Est(
+        base_estimator,
+        param_grid,
+        cv=2,
+        factor=factor,
+        max_resources=max_resources,
+        min_resources=4,
+    )
     if Est is HalvingRandomSearchCV:
         sh.set_params(n_candidates=20)  # same as for HalvingGridSearchCV
     sh.fit(X, y)
@@ -181,120 +306,133 @@ def test_n_iterations(Est, max_resources, n_iterations, n_possible_iterations):
     assert sh.n_possible_iterations_ == n_possible_iterations
 
 
-@pytest.mark.parametrize('Est', (HalvingRandomSearchCV, HalvingGridSearchCV))
+@pytest.mark.parametrize("Est", (HalvingRandomSearchCV, HalvingGridSearchCV))
 def test_resource_parameter(Est):
     # Test the resource parameter
 
     n_samples = 1000
     X, y = make_classification(n_samples=n_samples, random_state=0)
-    param_grid = {'a': [1, 2], 'b': list(range(10))}
+    param_grid = {"a": [1, 2], "b": list(range(10))}
     base_estimator = FastClassifier()
-    sh = Est(base_estimator, param_grid, cv=2, resource='c',
-             max_resources=10, factor=3)
+    sh = Est(base_estimator, param_grid, cv=2, resource="c", max_resources=10, factor=3)
     sh.fit(X, y)
     assert set(sh.n_resources_) == set([1, 3, 9])
-    for r_i, params, param_c in zip(sh.cv_results_['n_resources'],
-                                    sh.cv_results_['params'],
-                                    sh.cv_results_['param_c']):
-        assert r_i == params['c'] == param_c
+    for r_i, params, param_c in zip(
+        sh.cv_results_["n_resources"],
+        sh.cv_results_["params"],
+        sh.cv_results_["param_c"],
+    ):
+        assert r_i == params["c"] == param_c
 
     with pytest.raises(
-            ValueError,
-            match='Cannot use resource=1234 which is not supported '):
-        sh = HalvingGridSearchCV(base_estimator, param_grid, cv=2,
-                                 resource='1234', max_resources=10)
+        ValueError, match="Cannot use resource=1234 which is not supported "
+    ):
+        sh = HalvingGridSearchCV(
+            base_estimator, param_grid, cv=2, resource="1234", max_resources=10
+        )
         sh.fit(X, y)
 
     with pytest.raises(
-            ValueError,
-            match='Cannot use parameter c as the resource since it is part '
-                  'of the searched parameters.'):
-        param_grid = {'a': [1, 2], 'b': [1, 2], 'c': [1, 3]}
-        sh = HalvingGridSearchCV(base_estimator, param_grid, cv=2,
-                                 resource='c', max_resources=10)
+        ValueError,
+        match=(
+            "Cannot use parameter c as the resource since it is part "
+            "of the searched parameters."
+        ),
+    ):
+        param_grid = {"a": [1, 2], "b": [1, 2], "c": [1, 3]}
+        sh = HalvingGridSearchCV(
+            base_estimator, param_grid, cv=2, resource="c", max_resources=10
+        )
         sh.fit(X, y)
 
 
 @pytest.mark.parametrize(
-    'max_resources, n_candidates, expected_n_candidates', [
-        (512, 'exhaust', 128),  # generate exactly as much as needed
-        (32, 'exhaust', 8),
+    "max_resources, n_candidates, expected_n_candidates",
+    [
+        (512, "exhaust", 128),  # generate exactly as much as needed
+        (32, "exhaust", 8),
         (32, 8, 8),
         (32, 7, 7),  # ask for less than what we could
         (32, 9, 9),  # ask for more than 'reasonable'
-    ])
+    ],
+)
 def test_random_search(max_resources, n_candidates, expected_n_candidates):
     # Test random search and make sure the number of generated candidates is
     # as expected
 
     n_samples = 1024
     X, y = make_classification(n_samples=n_samples, random_state=0)
-    param_grid = {'a': norm, 'b': norm}
+    param_grid = {"a": norm, "b": norm}
     base_estimator = FastClassifier()
-    sh = HalvingRandomSearchCV(base_estimator, param_grid,
-                               n_candidates=n_candidates, cv=2,
-                               max_resources=max_resources, factor=2,
-                               min_resources=4)
+    sh = HalvingRandomSearchCV(
+        base_estimator,
+        param_grid,
+        n_candidates=n_candidates,
+        cv=2,
+        max_resources=max_resources,
+        factor=2,
+        min_resources=4,
+    )
     sh.fit(X, y)
     assert sh.n_candidates_[0] == expected_n_candidates
-    if n_candidates == 'exhaust':
+    if n_candidates == "exhaust":
         # Make sure 'exhaust' makes the last iteration use as much resources as
         # we can
         assert sh.n_resources_[-1] == max_resources
 
 
-@pytest.mark.parametrize('param_distributions, expected_n_candidates', [
-    ({'a': [1, 2]}, 2),  # all lists, sample less than n_candidates
-    ({'a': randint(1, 3)}, 10),  # not all list, respect n_candidates
-])
-def test_random_search_discrete_distributions(param_distributions,
-                                              expected_n_candidates):
+@pytest.mark.parametrize(
+    "param_distributions, expected_n_candidates",
+    [
+        ({"a": [1, 2]}, 2),  # all lists, sample less than n_candidates
+        ({"a": randint(1, 3)}, 10),  # not all list, respect n_candidates
+    ],
+)
+def test_random_search_discrete_distributions(
+    param_distributions, expected_n_candidates
+):
     # Make sure random search samples the appropriate number of candidates when
     # we ask for more than what's possible. How many parameters are sampled
     # depends whether the distributions are 'all lists' or not (see
     # ParameterSampler for details). This is somewhat redundant with the checks
     # in ParameterSampler but interaction bugs were discovered during
-    # developement of SH
+    # development of SH
 
     n_samples = 1024
     X, y = make_classification(n_samples=n_samples, random_state=0)
     base_estimator = FastClassifier()
-    sh = HalvingRandomSearchCV(base_estimator, param_distributions,
-                               n_candidates=10)
+    sh = HalvingRandomSearchCV(base_estimator, param_distributions, n_candidates=10)
     sh.fit(X, y)
     assert sh.n_candidates_[0] == expected_n_candidates
 
 
-@pytest.mark.parametrize('Est', (HalvingGridSearchCV, HalvingRandomSearchCV))
-@pytest.mark.parametrize('params, expected_error_message', [
-    ({'scoring': {'accuracy', 'accuracy'}},
-     'Multimetric scoring is not supported'),
-    ({'resource': 'not_a_parameter'},
-     'Cannot use resource=not_a_parameter which is not supported'),
-    ({'resource': 'a', 'max_resources': 100},
-     'Cannot use parameter a as the resource since it is part of'),
-    ({'max_resources': 'not_auto'},
-     'max_resources must be either'),
-    ({'max_resources': 100.5},
-     'max_resources must be either'),
-    ({'max_resources': -10},
-     'max_resources must be either'),
-    ({'min_resources': 'bad str'},
-     'min_resources must be either'),
-    ({'min_resources': 0.5},
-     'min_resources must be either'),
-    ({'min_resources': -10},
-     'min_resources must be either'),
-    ({'max_resources': 'auto', 'resource': 'b'},
-     "max_resources can only be 'auto' if resource='n_samples'"),
-    ({'min_resources': 15, 'max_resources': 14},
-     "min_resources_=15 is greater than max_resources_=14"),
-    ({'cv': KFold(shuffle=True)}, "must yield consistent folds"),
-    ({'cv': ShuffleSplit()}, "must yield consistent folds"),
-])
+@pytest.mark.parametrize("Est", (HalvingGridSearchCV, HalvingRandomSearchCV))
+@pytest.mark.parametrize(
+    "params, expected_error_message",
+    [
+        (
+            {"resource": "not_a_parameter"},
+            "Cannot use resource=not_a_parameter which is not supported",
+        ),
+        (
+            {"resource": "a", "max_resources": 100},
+            "Cannot use parameter a as the resource since it is part of",
+        ),
+        (
+            {"max_resources": "auto", "resource": "b"},
+            "resource can only be 'n_samples' when max_resources='auto'",
+        ),
+        (
+            {"min_resources": 15, "max_resources": 14},
+            "min_resources_=15 is greater than max_resources_=14",
+        ),
+        ({"cv": KFold(shuffle=True)}, "must yield consistent folds"),
+        ({"cv": ShuffleSplit()}, "must yield consistent folds"),
+    ],
+)
 def test_input_errors(Est, params, expected_error_message):
     base_estimator = FastClassifier()
-    param_grid = {'a': [1]}
+    param_grid = {"a": [1]}
     X, y = make_classification(100)
 
     sh = Est(base_estimator, param_grid, **params)
@@ -303,17 +441,20 @@ def test_input_errors(Est, params, expected_error_message):
         sh.fit(X, y)
 
 
-@pytest.mark.parametrize('params, expected_error_message', [
-    ({'n_candidates': 'exhaust', 'min_resources': 'exhaust'},
-     "cannot be both set to 'exhaust'"),
-    ({'n_candidates': 'bad'}, "either 'exhaust' or a positive integer"),
-    ({'n_candidates': 0}, "either 'exhaust' or a positive integer"),
-])
+@pytest.mark.parametrize(
+    "params, expected_error_message",
+    [
+        (
+            {"n_candidates": "exhaust", "min_resources": "exhaust"},
+            "cannot be both set to 'exhaust'",
+        ),
+    ],
+)
 def test_input_errors_randomized(params, expected_error_message):
     # tests specific to HalvingRandomSearchCV
 
     base_estimator = FastClassifier()
-    param_grid = {'a': [1]}
+    param_grid = {"a": [1]}
     X, y = make_classification(100)
 
     sh = HalvingRandomSearchCV(base_estimator, param_grid, **params)
@@ -323,21 +464,28 @@ def test_input_errors_randomized(params, expected_error_message):
 
 
 @pytest.mark.parametrize(
-    'fraction, subsample_test, expected_train_size, expected_test_size', [
-        (.5, True, 40, 10),
-        (.5, False, 40, 20),
-        (.2, True, 16, 4),
-        (.2, False, 16, 20)])
-def test_subsample_splitter_shapes(fraction, subsample_test,
-                                   expected_train_size, expected_test_size):
+    "fraction, subsample_test, expected_train_size, expected_test_size",
+    [
+        (0.5, True, 40, 10),
+        (0.5, False, 40, 20),
+        (0.2, True, 16, 4),
+        (0.2, False, 16, 20),
+    ],
+)
+def test_subsample_splitter_shapes(
+    fraction, subsample_test, expected_train_size, expected_test_size
+):
     # Make sure splits returned by SubsampleMetaSplitter are of appropriate
     # size
 
     n_samples = 100
     X, y = make_classification(n_samples)
-    cv = _SubsampleMetaSplitter(base_cv=KFold(5), fraction=fraction,
-                                subsample_test=subsample_test,
-                                random_state=None)
+    cv = _SubsampleMetaSplitter(
+        base_cv=KFold(5),
+        fraction=fraction,
+        subsample_test=subsample_test,
+        random_state=None,
+    )
 
     for train, test in cv.split(X, y):
         assert train.shape[0] == expected_train_size
@@ -348,7 +496,7 @@ def test_subsample_splitter_shapes(fraction, subsample_test,
             assert test.shape[0] == n_samples // cv.base_cv.get_n_splits()
 
 
-@pytest.mark.parametrize('subsample_test', (True, False))
+@pytest.mark.parametrize("subsample_test", (True, False))
 def test_subsample_splitter_determinism(subsample_test):
     # Make sure _SubsampleMetaSplitter is consistent across calls to split():
     # - we're OK having training sets differ (they're always sampled with a
@@ -362,9 +510,9 @@ def test_subsample_splitter_determinism(subsample_test):
 
     n_samples = 100
     X, y = make_classification(n_samples)
-    cv = _SubsampleMetaSplitter(base_cv=KFold(5), fraction=.5,
-                                subsample_test=subsample_test,
-                                random_state=None)
+    cv = _SubsampleMetaSplitter(
+        base_cv=KFold(5), fraction=0.5, subsample_test=subsample_test, random_state=None
+    )
 
     folds_a = list(cv.split(X, y, groups=None))
     folds_b = list(cv.split(X, y, groups=None))
@@ -379,52 +527,42 @@ def test_subsample_splitter_determinism(subsample_test):
             assert np.all(X[test_a] == X[test_b])
 
 
-@pytest.mark.parametrize('k, itr, expected', [
-    (1, 0, ['c']),
-    (2, 0, ['a', 'c']),
-    (4, 0, ['d', 'b', 'a', 'c']),
-    (10, 0, ['d', 'b', 'a', 'c']),
-
-    (1, 1, ['e']),
-    (2, 1, ['f', 'e']),
-    (10, 1, ['f', 'e']),
-
-    (1, 2, ['i']),
-    (10, 2, ['g', 'h', 'i']),
-])
+@pytest.mark.parametrize(
+    "k, itr, expected",
+    [
+        (1, 0, ["c"]),
+        (2, 0, ["a", "c"]),
+        (4, 0, ["d", "b", "a", "c"]),
+        (10, 0, ["d", "b", "a", "c"]),
+        (1, 1, ["e"]),
+        (2, 1, ["f", "e"]),
+        (10, 1, ["f", "e"]),
+        (1, 2, ["i"]),
+        (10, 2, ["g", "h", "i"]),
+    ],
+)
 def test_top_k(k, itr, expected):
-
     results = {  # this isn't a 'real world' result dict
-        'iter': [0, 0, 0, 0, 1, 1, 2, 2, 2],
-        'mean_test_score': [4, 3, 5, 1, 11, 10, 5, 6, 9],
-        'params': ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i'],
+        "iter": [0, 0, 0, 0, 1, 1, 2, 2, 2],
+        "mean_test_score": [4, 3, 5, 1, 11, 10, 5, 6, 9],
+        "params": ["a", "b", "c", "d", "e", "f", "g", "h", "i"],
     }
     got = _top_k(results, k=k, itr=itr)
     assert np.all(got == expected)
 
 
-def test_refit_callable():
-
-    results = {  # this isn't a 'real world' result dict
-        'iter': np.array([0, 0, 0, 0, 1, 1, 2, 2, 2]),
-        'mean_test_score': np.array([4, 3, 5, 1, 11, 10, 5, 6, 9]),
-        'params': np.array(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i']),
-    }
-    assert _refit_callable(results) == 8  # index of 'i'
-
-
-@pytest.mark.parametrize('Est', (HalvingRandomSearchCV, HalvingGridSearchCV))
+@pytest.mark.parametrize("Est", (HalvingRandomSearchCV, HalvingGridSearchCV))
 def test_cv_results(Est):
     # test that the cv_results_ matches correctly the logic of the
     # tournament: in particular that the candidates continued in each
     # successive iteration are those that were best in the previous iteration
-    pd = pytest.importorskip('pandas')
+    pd = pytest.importorskip("pandas")
 
     rng = np.random.RandomState(0)
 
     n_samples = 1000
     X, y = make_classification(n_samples=n_samples, random_state=0)
-    param_grid = {'a': ('l1', 'l2'), 'b': list(range(30))}
+    param_grid = {"a": ("l1", "l2"), "b": list(range(30))}
     base_estimator = FastClassifier()
 
     # generate random scores: we want to avoid ties, which would otherwise
@@ -435,17 +573,24 @@ def scorer(est, X, y):
     sh = Est(base_estimator, param_grid, factor=2, scoring=scorer)
     if Est is HalvingRandomSearchCV:
         # same number of candidates as with the grid
-        sh.set_params(n_candidates=2 * 30, min_resources='exhaust')
+        sh.set_params(n_candidates=2 * 30, min_resources="exhaust")
 
     sh.fit(X, y)
+
+    # non-regression check for
+    # https://github.com/scikit-learn/scikit-learn/issues/19203
+    assert isinstance(sh.cv_results_["iter"], np.ndarray)
+    assert isinstance(sh.cv_results_["n_resources"], np.ndarray)
+
     cv_results_df = pd.DataFrame(sh.cv_results_)
 
     # just make sure we don't have ties
-    assert len(cv_results_df['mean_test_score'].unique()) == len(cv_results_df)
+    assert len(cv_results_df["mean_test_score"].unique()) == len(cv_results_df)
 
-    cv_results_df['params_str'] = cv_results_df['params'].apply(str)
-    table = cv_results_df.pivot(index='params_str', columns='iter',
-                                values='mean_test_score')
+    cv_results_df["params_str"] = cv_results_df["params"].apply(str)
+    table = cv_results_df.pivot(
+        index="params_str", columns="iter", values="mean_test_score"
+    )
 
     # table looks like something like this:
     # iter                    0      1       2        3   4   5
@@ -469,8 +614,9 @@ def scorer(est, X, y):
 
         # make sure that if a candidate is already discarded, we don't evaluate
         # it later
-        assert (already_discarded_mask & nan_mask[it + 1] ==
-                already_discarded_mask).all()
+        assert (
+            already_discarded_mask & nan_mask[it + 1] == already_discarded_mask
+        ).all()
 
         # make sure that the number of discarded candidate is correct
         discarded_now_mask = ~already_discarded_mask & nan_mask[it + 1]
@@ -489,32 +635,34 @@ def scorer(est, X, y):
     # earlier rounds (this isn't generally the case, but worth ensuring it's
     # possible).
 
-    last_iter = cv_results_df['iter'].max()
-    idx_best_last_iter = (
-        cv_results_df[cv_results_df['iter'] == last_iter]
-        ['mean_test_score'].idxmax()
-    )
-    idx_best_all_iters = cv_results_df['mean_test_score'].idxmax()
+    last_iter = cv_results_df["iter"].max()
+    idx_best_last_iter = cv_results_df[cv_results_df["iter"] == last_iter][
+        "mean_test_score"
+    ].idxmax()
+    idx_best_all_iters = cv_results_df["mean_test_score"].idxmax()
 
-    assert sh.best_params_ == cv_results_df.iloc[idx_best_last_iter]['params']
-    assert (cv_results_df.iloc[idx_best_last_iter]['mean_test_score'] <
-            cv_results_df.iloc[idx_best_all_iters]['mean_test_score'])
-    assert (cv_results_df.iloc[idx_best_last_iter]['params'] !=
-            cv_results_df.iloc[idx_best_all_iters]['params'])
+    assert sh.best_params_ == cv_results_df.iloc[idx_best_last_iter]["params"]
+    assert (
+        cv_results_df.iloc[idx_best_last_iter]["mean_test_score"]
+        < cv_results_df.iloc[idx_best_all_iters]["mean_test_score"]
+    )
+    assert (
+        cv_results_df.iloc[idx_best_last_iter]["params"]
+        != cv_results_df.iloc[idx_best_all_iters]["params"]
+    )
 
 
-@pytest.mark.parametrize('Est', (HalvingGridSearchCV, HalvingRandomSearchCV))
+@pytest.mark.parametrize("Est", (HalvingGridSearchCV, HalvingRandomSearchCV))
 def test_base_estimator_inputs(Est):
     # make sure that the base estimators are passed the correct parameters and
     # number of samples at each iteration.
-    pd = pytest.importorskip('pandas')
+    pd = pytest.importorskip("pandas")
 
     passed_n_samples_fit = []
     passed_n_samples_predict = []
     passed_params = []
 
     class FastClassifierBookKeeping(FastClassifier):
-
         def fit(self, X, y):
             passed_n_samples_fit.append(X.shape[0])
             return super().fit(X, y)
@@ -530,20 +678,27 @@ def set_params(self, **params):
     n_samples = 1024
     n_splits = 2
     X, y = make_classification(n_samples=n_samples, random_state=0)
-    param_grid = {'a': ('l1', 'l2'), 'b': list(range(30))}
+    param_grid = {"a": ("l1", "l2"), "b": list(range(30))}
     base_estimator = FastClassifierBookKeeping()
 
-    sh = Est(base_estimator, param_grid, factor=2, cv=n_splits,
-             return_train_score=False, refit=False)
+    sh = Est(
+        base_estimator,
+        param_grid,
+        factor=2,
+        cv=n_splits,
+        return_train_score=False,
+        refit=False,
+    )
     if Est is HalvingRandomSearchCV:
         # same number of candidates as with the grid
-        sh.set_params(n_candidates=2 * 30, min_resources='exhaust')
+        sh.set_params(n_candidates=2 * 30, min_resources="exhaust")
 
     sh.fit(X, y)
 
     assert len(passed_n_samples_fit) == len(passed_n_samples_predict)
-    passed_n_samples = [x + y for (x, y) in zip(passed_n_samples_fit,
-                                                passed_n_samples_predict)]
+    passed_n_samples = [
+        x + y for (x, y) in zip(passed_n_samples_fit, passed_n_samples_predict)
+    ]
 
     # Lists are of length n_splits * n_iter * n_candidates_at_i.
     # Each chunk of size n_splits corresponds to the n_splits folds for the
@@ -560,5 +715,139 @@ def set_params(self, **params):
     assert (sh.n_resources_ == uniques).all()
     assert (sh.n_candidates_ == counts).all()
 
-    assert (cv_results_df['params'] == passed_params).all()
-    assert (cv_results_df['n_resources'] == passed_n_samples).all()
+    assert (cv_results_df["params"] == passed_params).all()
+    assert (cv_results_df["n_resources"] == passed_n_samples).all()
+
+
+@pytest.mark.parametrize("Est", (HalvingGridSearchCV, HalvingRandomSearchCV))
+def test_groups_support(Est):
+    # Check if ValueError (when groups is None) propagates to
+    # HalvingGridSearchCV and HalvingRandomSearchCV
+    # And also check if groups is correctly passed to the cv object
+    rng = np.random.RandomState(0)
+
+    X, y = make_classification(n_samples=50, n_classes=2, random_state=0)
+    groups = rng.randint(0, 3, 50)
+
+    clf = LinearSVC(random_state=0)
+    grid = {"C": [1]}
+
+    group_cvs = [
+        LeaveOneGroupOut(),
+        LeavePGroupsOut(2),
+        GroupKFold(n_splits=3),
+        GroupShuffleSplit(random_state=0),
+    ]
+    error_msg = "The 'groups' parameter should not be None."
+    for cv in group_cvs:
+        gs = Est(clf, grid, cv=cv, random_state=0)
+        with pytest.raises(ValueError, match=error_msg):
+            gs.fit(X, y)
+        gs.fit(X, y, groups=groups)
+
+    non_group_cvs = [StratifiedKFold(), StratifiedShuffleSplit(random_state=0)]
+    for cv in non_group_cvs:
+        gs = Est(clf, grid, cv=cv)
+        # Should not raise an error
+        gs.fit(X, y)
+
+
+@pytest.mark.parametrize("SearchCV", [HalvingRandomSearchCV, HalvingGridSearchCV])
+def test_min_resources_null(SearchCV):
+    """Check that we raise an error if the minimum resources is set to 0."""
+    base_estimator = FastClassifier()
+    param_grid = {"a": [1]}
+    X = np.empty(0).reshape(0, 3)
+
+    search = SearchCV(base_estimator, param_grid, min_resources="smallest")
+
+    err_msg = "min_resources_=0: you might have passed an empty dataset X."
+    with pytest.raises(ValueError, match=err_msg):
+        search.fit(X, [])
+
+
+@pytest.mark.parametrize("SearchCV", [HalvingGridSearchCV, HalvingRandomSearchCV])
+def test_select_best_index(SearchCV):
+    """Check the selection strategy of the halving search."""
+    results = {  # this isn't a 'real world' result dict
+        "iter": np.array([0, 0, 0, 0, 1, 1, 2, 2, 2]),
+        "mean_test_score": np.array([4, 3, 5, 1, 11, 10, 5, 6, 9]),
+        "params": np.array(["a", "b", "c", "d", "e", "f", "g", "h", "i"]),
+    }
+
+    # we expect the index of 'i'
+    best_index = SearchCV._select_best_index(None, None, results)
+    assert best_index == 8
+
+
+def test_halving_random_search_list_of_dicts():
+    """Check the behaviour of the `HalvingRandomSearchCV` with `param_distribution`
+    being a list of dictionary.
+    """
+    X, y = make_classification(n_samples=150, n_features=4, random_state=42)
+
+    params = [
+        {"kernel": ["rbf"], "C": expon(scale=10), "gamma": expon(scale=0.1)},
+        {"kernel": ["poly"], "degree": [2, 3]},
+    ]
+    param_keys = (
+        "param_C",
+        "param_degree",
+        "param_gamma",
+        "param_kernel",
+    )
+    score_keys = (
+        "mean_test_score",
+        "mean_train_score",
+        "rank_test_score",
+        "split0_test_score",
+        "split1_test_score",
+        "split2_test_score",
+        "split0_train_score",
+        "split1_train_score",
+        "split2_train_score",
+        "std_test_score",
+        "std_train_score",
+        "mean_fit_time",
+        "std_fit_time",
+        "mean_score_time",
+        "std_score_time",
+    )
+    extra_keys = ("n_resources", "iter")
+
+    search = HalvingRandomSearchCV(
+        SVC(), cv=3, param_distributions=params, return_train_score=True, random_state=0
+    )
+    search.fit(X, y)
+    n_candidates = sum(search.n_candidates_)
+    cv_results = search.cv_results_
+    # Check results structure
+    check_cv_results_keys(cv_results, param_keys, score_keys, n_candidates, extra_keys)
+    expected_cv_results_kinds = {
+        "param_C": "f",
+        "param_degree": "i",
+        "param_gamma": "f",
+        "param_kernel": "O",
+    }
+    check_cv_results_array_types(
+        search, param_keys, score_keys, expected_cv_results_kinds
+    )
+
+    assert all(
+        (
+            cv_results["param_C"].mask[i]
+            and cv_results["param_gamma"].mask[i]
+            and not cv_results["param_degree"].mask[i]
+        )
+        for i in range(n_candidates)
+        if cv_results["param_kernel"][i] == "poly"
+    )
+    assert all(
+        (
+            not cv_results["param_C"].mask[i]
+            and not cv_results["param_gamma"].mask[i]
+            and cv_results["param_degree"].mask[i]
+        )
+        for i in range(n_candidates)
+        if cv_results["param_kernel"][i] == "rbf"
+    )
diff --git a/sklearn/model_selection/tests/test_validation.py b/sklearn/model_selection/tests/test_validation.py
index c280d1e8ef140..c20131b8d3f38 100644
--- a/sklearn/model_selection/tests/test_validation.py
+++ b/sklearn/model_selection/tests/test_validation.py
@@ -1,90 +1,103 @@
 """Test the validation module"""
+
 import os
 import re
 import sys
 import tempfile
 import warnings
 from functools import partial
+from io import StringIO
 from time import sleep
 
-import pytest
 import numpy as np
-from scipy.sparse import coo_matrix, csr_matrix
-from sklearn.exceptions import FitFailedWarning
-
-from sklearn.model_selection.tests.test_search import FailingClassifier
-
-from sklearn.utils._testing import assert_almost_equal
-from sklearn.utils._testing import assert_array_almost_equal
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils._testing import assert_allclose
-from sklearn.utils._mocking import CheckingClassifier, MockDataFrame
-
-from sklearn.utils.validation import _num_samples
+import pytest
+from scipy.sparse import issparse
 
-from sklearn.model_selection import cross_val_score, ShuffleSplit
-from sklearn.model_selection import cross_val_predict
-from sklearn.model_selection import cross_validate
-from sklearn.model_selection import permutation_test_score
-from sklearn.model_selection import KFold
-from sklearn.model_selection import StratifiedKFold
-from sklearn.model_selection import LeaveOneOut
-from sklearn.model_selection import LeaveOneGroupOut
-from sklearn.model_selection import LeavePGroupsOut
-from sklearn.model_selection import GroupKFold
-from sklearn.model_selection import GroupShuffleSplit
-from sklearn.model_selection import learning_curve
-from sklearn.model_selection import validation_curve
-from sklearn.model_selection._validation import _check_is_permutation
-from sklearn.model_selection._validation import _fit_and_score
-from sklearn.model_selection._validation import _score
-
-from sklearn.datasets import make_regression
-from sklearn.datasets import load_diabetes
-from sklearn.datasets import load_iris
-from sklearn.datasets import load_digits
-from sklearn.metrics import explained_variance_score
-from sklearn.metrics import make_scorer
-from sklearn.metrics import accuracy_score
-from sklearn.metrics import confusion_matrix
-from sklearn.metrics import precision_recall_fscore_support
-from sklearn.metrics import precision_score
-from sklearn.metrics import r2_score
-from sklearn.metrics import mean_squared_error
-from sklearn.metrics import check_scoring
-
-from sklearn.linear_model import Ridge, LogisticRegression, SGDClassifier
-from sklearn.linear_model import PassiveAggressiveClassifier, RidgeClassifier
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.neighbors import KNeighborsClassifier
-from sklearn.svm import SVC, LinearSVC
+from sklearn import config_context
+from sklearn.base import BaseEstimator, ClassifierMixin, clone
 from sklearn.cluster import KMeans
-
+from sklearn.datasets import (
+    load_diabetes,
+    load_digits,
+    load_iris,
+    make_classification,
+    make_multilabel_classification,
+    make_regression,
+)
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.exceptions import FitFailedWarning, UnsetMetadataPassedError
 from sklearn.impute import SimpleImputer
-
-from sklearn.preprocessing import LabelEncoder
-from sklearn.pipeline import Pipeline
-
-from io import StringIO
-from sklearn.base import BaseEstimator
-from sklearn.base import clone
+from sklearn.linear_model import (
+    LogisticRegression,
+    PassiveAggressiveClassifier,
+    Ridge,
+    RidgeClassifier,
+    SGDClassifier,
+)
+from sklearn.metrics import (
+    accuracy_score,
+    check_scoring,
+    confusion_matrix,
+    explained_variance_score,
+    make_scorer,
+    mean_squared_error,
+    precision_recall_fscore_support,
+    precision_score,
+    r2_score,
+)
+from sklearn.metrics._scorer import _MultimetricScorer
+from sklearn.model_selection import (
+    GridSearchCV,
+    GroupKFold,
+    GroupShuffleSplit,
+    KFold,
+    LeaveOneGroupOut,
+    LeaveOneOut,
+    LeavePGroupsOut,
+    ShuffleSplit,
+    StratifiedKFold,
+    cross_val_predict,
+    cross_val_score,
+    cross_validate,
+    learning_curve,
+    permutation_test_score,
+    validation_curve,
+)
+from sklearn.model_selection._validation import (
+    _check_is_permutation,
+    _fit_and_score,
+    _score,
+)
+from sklearn.model_selection.tests.common import OneTimeSplitter
+from sklearn.model_selection.tests.test_search import FailingClassifier
 from sklearn.multiclass import OneVsRestClassifier
+from sklearn.neighbors import KNeighborsClassifier
+from sklearn.neural_network import MLPRegressor
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import LabelEncoder, scale
+from sklearn.svm import SVC, LinearSVC
+from sklearn.tests.metadata_routing_common import (
+    ConsumingClassifier,
+    ConsumingScorer,
+    ConsumingSplitter,
+    _Registry,
+    check_recorded_metadata,
+)
 from sklearn.utils import shuffle
-from sklearn.datasets import make_classification
-from sklearn.datasets import make_multilabel_classification
-
-from sklearn.model_selection.tests.common import OneTimeSplitter
-from sklearn.model_selection import GridSearchCV
-
-
-try:
-    WindowsError
-except NameError:
-    WindowsError = None
+from sklearn.utils._mocking import CheckingClassifier, MockDataFrame
+from sklearn.utils._testing import (
+    assert_allclose,
+    assert_almost_equal,
+    assert_array_almost_equal,
+    assert_array_equal,
+)
+from sklearn.utils.fixes import COO_CONTAINERS, CSR_CONTAINERS
+from sklearn.utils.validation import _num_samples
 
 
 class MockImprovingEstimator(BaseEstimator):
     """Dummy classifier to test the learning curve"""
+
     def __init__(self, n_max_train_sizes):
         self.n_max_train_sizes = n_max_train_sizes
         self.train_sizes = 0
@@ -101,7 +114,7 @@ def predict(self, X):
     def score(self, X=None, Y=None):
         # training score becomes worse (2 -> 1), test error better (0 -> 1)
         if self._is_training_data(X):
-            return 2. - float(self.train_sizes) / self.n_max_train_sizes
+            return 2.0 - float(self.train_sizes) / self.n_max_train_sizes
         else:
             return float(self.train_sizes) / self.n_max_train_sizes
 
@@ -111,6 +124,7 @@ def _is_training_data(self, X):
 
 class MockIncrementalImprovingEstimator(MockImprovingEstimator):
     """Dummy classifier that provides partial_fit"""
+
     def __init__(self, n_max_train_sizes, expected_fit_params=None):
         super().__init__(n_max_train_sizes)
         self.x = None
@@ -126,19 +140,21 @@ def partial_fit(self, X, y=None, **params):
             missing = set(self.expected_fit_params) - set(params)
             if missing:
                 raise AssertionError(
-                    f'Expected fit parameter(s) {list(missing)} not seen.'
+                    f"Expected fit parameter(s) {list(missing)} not seen."
                 )
             for key, value in params.items():
-                if key in self.expected_fit_params and \
-                   _num_samples(value) != _num_samples(X):
+                if key in self.expected_fit_params and _num_samples(
+                    value
+                ) != _num_samples(X):
                     raise AssertionError(
-                        f'Fit parameter {key} has length {_num_samples(value)}'
-                        f'; expected {_num_samples(X)}.'
+                        f"Fit parameter {key} has length {_num_samples(value)}"
+                        f"; expected {_num_samples(X)}."
                     )
 
 
 class MockEstimatorWithParameter(BaseEstimator):
     """Dummy classifier to test the validation curve"""
+
     def __init__(self, param=0.5):
         self.X_subset = None
         self.param = param
@@ -162,8 +178,7 @@ class MockEstimatorWithSingleFitCallAllowed(MockEstimatorWithParameter):
     """Dummy classifier that disallows repeated calls of fit method"""
 
     def fit(self, X_subset, y_subset):
-        assert not hasattr(self, 'fit_called_'), \
-                   'fit is called the second time'
+        assert not hasattr(self, "fit_called_"), "fit is called the second time"
         self.fit_called_ = True
         return super().fit(X_subset, y_subset)
 
@@ -171,16 +186,26 @@ def predict(self, X):
         raise NotImplementedError
 
 
-class MockClassifier:
+class MockClassifier(ClassifierMixin, BaseEstimator):
     """Dummy classifier to test the cross-validation"""
 
     def __init__(self, a=0, allow_nd=False):
         self.a = a
         self.allow_nd = allow_nd
 
-    def fit(self, X, Y=None, sample_weight=None, class_prior=None,
-            sparse_sample_weight=None, sparse_param=None, dummy_int=None,
-            dummy_str=None, dummy_obj=None, callback=None):
+    def fit(
+        self,
+        X,
+        Y=None,
+        sample_weight=None,
+        class_prior=None,
+        sparse_sample_weight=None,
+        sparse_param=None,
+        dummy_int=None,
+        dummy_str=None,
+        dummy_obj=None,
+        callback=None,
+    ):
         """The dummy arguments are to test that this fit function can
         accept non-array arguments through cross-validation, such as:
             - int
@@ -197,29 +222,39 @@ def fit(self, X, Y=None, sample_weight=None, class_prior=None,
         if self.allow_nd:
             X = X.reshape(len(X), -1)
         if X.ndim >= 3 and not self.allow_nd:
-            raise ValueError('X cannot be d')
+            raise ValueError("X cannot be d")
         if sample_weight is not None:
             assert sample_weight.shape[0] == X.shape[0], (
-                'MockClassifier extra fit_param '
-                'sample_weight.shape[0] is {0}, should be {1}'
-                .format(sample_weight.shape[0], X.shape[0]))
+                "MockClassifier extra fit_param "
+                "sample_weight.shape[0] is {0}, should be {1}".format(
+                    sample_weight.shape[0], X.shape[0]
+                )
+            )
         if class_prior is not None:
             assert class_prior.shape[0] == len(np.unique(y)), (
-                'MockClassifier extra fit_param class_prior.shape[0]'
-                ' is {0}, should be {1}'.format(class_prior.shape[0],
-                                                len(np.unique(y))))
+                "MockClassifier extra fit_param class_prior.shape[0]"
+                " is {0}, should be {1}".format(class_prior.shape[0], len(np.unique(y)))
+            )
         if sparse_sample_weight is not None:
-            fmt = ('MockClassifier extra fit_param sparse_sample_weight'
-                   '.shape[0] is {0}, should be {1}')
-            assert sparse_sample_weight.shape[0] == X.shape[0], \
-                fmt.format(sparse_sample_weight.shape[0], X.shape[0])
+            fmt = (
+                "MockClassifier extra fit_param sparse_sample_weight"
+                ".shape[0] is {0}, should be {1}"
+            )
+            assert sparse_sample_weight.shape[0] == X.shape[0], fmt.format(
+                sparse_sample_weight.shape[0], X.shape[0]
+            )
         if sparse_param is not None:
-            fmt = ('MockClassifier extra fit_param sparse_param.shape '
-                   'is ({0}, {1}), should be ({2}, {3})')
-            assert sparse_param.shape == P_sparse.shape, (
-                fmt.format(sparse_param.shape[0],
-                           sparse_param.shape[1],
-                           P_sparse.shape[0], P_sparse.shape[1]))
+            fmt = (
+                "MockClassifier extra fit_param sparse_param.shape "
+                "is ({0}, {1}), should be ({2}, {3})"
+            )
+            assert sparse_param.shape == P.shape, fmt.format(
+                sparse_param.shape[0],
+                sparse_param.shape[1],
+                P.shape[0],
+                P.shape[1],
+            )
+        self.classes_ = np.unique(y)
         return self
 
     def predict(self, T):
@@ -231,25 +266,26 @@ def predict_proba(self, T):
         return T
 
     def score(self, X=None, Y=None):
-        return 1. / (1 + np.abs(self.a))
+        return 1.0 / (1 + np.abs(self.a))
 
     def get_params(self, deep=False):
-        return {'a': self.a, 'allow_nd': self.allow_nd}
+        return {"a": self.a, "allow_nd": self.allow_nd}
 
 
 # XXX: use 2D array, since 1D X is being detected as a single sample in
 # check_consistent_length
-X = np.ones((10, 2))
-X_sparse = coo_matrix(X)
-y = np.array([0, 0, 1, 1, 2, 2, 3, 3, 4, 4])
+X = np.ones((15, 2))
+y = np.array([0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 6])
 # The number of samples per class needs to be > n_splits,
 # for StratifiedKFold(n_splits=3)
-y2 = np.array([1, 1, 1, 2, 2, 2, 3, 3, 3, 3])
-P_sparse = coo_matrix(np.eye(5))
+y2 = np.array([1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3])
+P = np.eye(5)
 
 
-def test_cross_val_score():
+@pytest.mark.parametrize("coo_container", COO_CONTAINERS)
+def test_cross_val_score(coo_container):
     clf = MockClassifier()
+    X_sparse = coo_container(X)
 
     for a in range(-10, 10):
         clf.a = a
@@ -277,9 +313,6 @@ def test_cross_val_score():
     clf = CheckingClassifier(check_y=list_check)
     scores = cross_val_score(clf, X, y2.tolist(), cv=3)
 
-    with pytest.raises(ValueError):
-        cross_val_score(clf, X, y2, scoring="sklearn")
-
     # test with 3d X and
     X_3d = X[:, :, np.newaxis]
     clf = MockClassifier(allow_nd=True)
@@ -287,7 +320,7 @@ def test_cross_val_score():
 
     clf = MockClassifier(allow_nd=False)
     with pytest.raises(ValueError):
-        cross_val_score(clf, X_3d, y2, error_score='raise')
+        cross_val_score(clf, X_3d, y2, error_score="raise")
 
 
 def test_cross_validate_many_jobs():
@@ -295,8 +328,8 @@ def test_cross_validate_many_jobs():
     # the parameters leading to a failure in check_cv due to cv is 'warn'
     # instead of cv == 'warn'.
     X, y = load_iris(return_X_y=True)
-    clf = SVC(gamma='auto')
-    grid = GridSearchCV(clf, param_grid={'C': [1, 10]})
+    clf = SVC(gamma="auto")
+    grid = GridSearchCV(clf, param_grid={"C": [1, 10]})
     cross_validate(grid, X, y, n_jobs=2)
 
 
@@ -310,46 +343,40 @@ def test_cross_validate_invalid_scoring_param():
     # List/tuple of callables should raise a message advising users to use
     # dict of names to callables mapping
     with pytest.raises(ValueError, match=error_message_regexp):
-        cross_validate(estimator, X, y, scoring=(make_scorer(precision_score),
-                                                 make_scorer(accuracy_score)))
+        cross_validate(
+            estimator,
+            X,
+            y,
+            scoring=(make_scorer(precision_score), make_scorer(accuracy_score)),
+        )
     with pytest.raises(ValueError, match=error_message_regexp):
-        cross_validate(estimator, X, y,
-                       scoring=(make_scorer(precision_score),))
+        cross_validate(estimator, X, y, scoring=(make_scorer(precision_score),))
 
     # So should empty lists/tuples
-    with pytest.raises(
-        ValueError,
-        match=error_message_regexp + "Empty list.*"
-    ):
+    with pytest.raises(ValueError, match=error_message_regexp + "Empty list.*"):
         cross_validate(estimator, X, y, scoring=())
 
     # So should duplicated entries
     with pytest.raises(ValueError, match=error_message_regexp + "Duplicate.*"):
-        cross_validate(estimator, X, y, scoring=('f1_micro', 'f1_micro'))
+        cross_validate(estimator, X, y, scoring=("f1_micro", "f1_micro"))
 
     # Nested Lists should raise a generic error message
     with pytest.raises(ValueError, match=error_message_regexp):
-        cross_validate(estimator, X, y,
-                       scoring=[[make_scorer(precision_score)]])
-
-    error_message_regexp = (".*scoring is invalid.*Refer to the scoring "
-                            "glossary for details:.*")
+        cross_validate(estimator, X, y, scoring=[[make_scorer(precision_score)]])
 
     # Empty dict should raise invalid scoring error
     with pytest.raises(ValueError, match="An empty dict"):
         cross_validate(estimator, X, y, scoring=(dict()))
 
-    # And so should any other invalid entry
-    with pytest.raises(ValueError, match=error_message_regexp):
-        cross_validate(estimator, X, y, scoring=5)
-
     multiclass_scorer = make_scorer(precision_recall_fscore_support)
 
     # Multiclass Scorers that return multiple values are not supported yet
     # the warning message we're expecting to see
-    warning_message = ("Scoring failed. The score on this train-test "
-                       "partition for these parameters will be set to %f. "
-                       "Details: \n" % np.nan)
+    warning_message = (
+        "Scoring failed. The score on this train-test "
+        f"partition for these parameters will be set to {np.nan}. "
+        "Details: \n"
+    )
 
     with pytest.warns(UserWarning, match=warning_message):
         cross_validate(estimator, X, y, scoring=multiclass_scorer)
@@ -357,22 +384,18 @@ def test_cross_validate_invalid_scoring_param():
     with pytest.warns(UserWarning, match=warning_message):
         cross_validate(estimator, X, y, scoring={"foo": multiclass_scorer})
 
-    with pytest.raises(
-        ValueError,
-        match="'mse' is not a valid scoring value."
-    ):
-        cross_validate(SVC(), X, y, scoring="mse")
-
 
 def test_cross_validate_nested_estimator():
     # Non-regression test to ensure that nested
     # estimators are properly returned in a list
     # https://github.com/scikit-learn/scikit-learn/pull/17745
     (X, y) = load_iris(return_X_y=True)
-    pipeline = Pipeline([
-        ("imputer", SimpleImputer()),
-        ("classifier", MockClassifier()),
-    ])
+    pipeline = Pipeline(
+        [
+            ("imputer", SimpleImputer()),
+            ("classifier", MockClassifier()),
+        ]
+    )
 
     results = cross_validate(pipeline, X, y, return_estimator=True)
     estimators = results["estimator"]
@@ -381,7 +404,9 @@ def test_cross_validate_nested_estimator():
     assert all(isinstance(estimator, Pipeline) for estimator in estimators)
 
 
-def test_cross_validate():
+@pytest.mark.parametrize("use_sparse", [False, True])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_cross_validate(use_sparse: bool, csr_container):
     # Compute train and test mse/r2 scores
     cv = KFold()
 
@@ -393,17 +418,22 @@ def test_cross_validate():
     X_clf, y_clf = make_classification(n_samples=30, random_state=0)
     clf = SVC(kernel="linear", random_state=0)
 
+    if use_sparse:
+        X_reg = csr_container(X_reg)
+        X_clf = csr_container(X_clf)
+
     for X, y, est in ((X_reg, y_reg, reg), (X_clf, y_clf, clf)):
         # It's okay to evaluate regression metrics on classification too
-        mse_scorer = check_scoring(est, scoring='neg_mean_squared_error')
-        r2_scorer = check_scoring(est, scoring='r2')
+        mse_scorer = check_scoring(est, scoring="neg_mean_squared_error")
+        r2_scorer = check_scoring(est, scoring="r2")
         train_mse_scores = []
         test_mse_scores = []
         train_r2_scores = []
         test_r2_scores = []
         fitted_estimators = []
+
         for train, test in cv.split(X, y):
-            est = clone(reg).fit(X[train], y[train])
+            est = clone(est).fit(X[train], y[train])
             train_mse_scores.append(mse_scorer(est, X[train], y[train]))
             train_r2_scores.append(r2_scorer(est, X[train], y[train]))
             test_mse_scores.append(mse_scorer(est, X[test], y[test]))
@@ -416,111 +446,158 @@ def test_cross_validate():
         test_r2_scores = np.array(test_r2_scores)
         fitted_estimators = np.array(fitted_estimators)
 
-        scores = (train_mse_scores, test_mse_scores, train_r2_scores,
-                  test_r2_scores, fitted_estimators)
-
-        check_cross_validate_single_metric(est, X, y, scores)
-        check_cross_validate_multi_metric(est, X, y, scores)
-
+        scores = (
+            train_mse_scores,
+            test_mse_scores,
+            train_r2_scores,
+            test_r2_scores,
+            fitted_estimators,
+        )
 
-def check_cross_validate_single_metric(clf, X, y, scores):
-    (train_mse_scores, test_mse_scores, train_r2_scores,
-     test_r2_scores, fitted_estimators) = scores
+        # To ensure that the test does not suffer from
+        # large statistical fluctuations due to slicing small datasets,
+        # we pass the cross-validation instance
+        check_cross_validate_single_metric(est, X, y, scores, cv)
+        check_cross_validate_multi_metric(est, X, y, scores, cv)
+
+
+def check_cross_validate_single_metric(clf, X, y, scores, cv):
+    (
+        train_mse_scores,
+        test_mse_scores,
+        train_r2_scores,
+        test_r2_scores,
+        fitted_estimators,
+    ) = scores
     # Test single metric evaluation when scoring is string or singleton list
-    for (return_train_score, dict_len) in ((True, 4), (False, 3)):
+    for return_train_score, dict_len in ((True, 4), (False, 3)):
         # Single metric passed as a string
         if return_train_score:
-            mse_scores_dict = cross_validate(clf, X, y,
-                                             scoring='neg_mean_squared_error',
-                                             return_train_score=True)
-            assert_array_almost_equal(mse_scores_dict['train_score'],
-                                      train_mse_scores)
+            mse_scores_dict = cross_validate(
+                clf,
+                X,
+                y,
+                scoring="neg_mean_squared_error",
+                return_train_score=True,
+                cv=cv,
+            )
+            assert_array_almost_equal(mse_scores_dict["train_score"], train_mse_scores)
         else:
-            mse_scores_dict = cross_validate(clf, X, y,
-                                             scoring='neg_mean_squared_error',
-                                             return_train_score=False)
+            mse_scores_dict = cross_validate(
+                clf,
+                X,
+                y,
+                scoring="neg_mean_squared_error",
+                return_train_score=False,
+                cv=cv,
+            )
         assert isinstance(mse_scores_dict, dict)
         assert len(mse_scores_dict) == dict_len
-        assert_array_almost_equal(mse_scores_dict['test_score'],
-                                  test_mse_scores)
+        assert_array_almost_equal(mse_scores_dict["test_score"], test_mse_scores)
 
         # Single metric passed as a list
         if return_train_score:
             # It must be True by default - deprecated
-            r2_scores_dict = cross_validate(clf, X, y, scoring=['r2'],
-                                            return_train_score=True)
-            assert_array_almost_equal(r2_scores_dict['train_r2'],
-                                      train_r2_scores, True)
+            r2_scores_dict = cross_validate(
+                clf, X, y, scoring=["r2"], return_train_score=True, cv=cv
+            )
+            assert_array_almost_equal(r2_scores_dict["train_r2"], train_r2_scores, True)
         else:
-            r2_scores_dict = cross_validate(clf, X, y, scoring=['r2'],
-                                            return_train_score=False)
+            r2_scores_dict = cross_validate(
+                clf, X, y, scoring=["r2"], return_train_score=False, cv=cv
+            )
         assert isinstance(r2_scores_dict, dict)
         assert len(r2_scores_dict) == dict_len
-        assert_array_almost_equal(r2_scores_dict['test_r2'], test_r2_scores)
+        assert_array_almost_equal(r2_scores_dict["test_r2"], test_r2_scores)
 
     # Test return_estimator option
-    mse_scores_dict = cross_validate(clf, X, y,
-                                     scoring='neg_mean_squared_error',
-                                     return_estimator=True)
-    for k, est in enumerate(mse_scores_dict['estimator']):
-        assert_almost_equal(est.coef_, fitted_estimators[k].coef_)
+    mse_scores_dict = cross_validate(
+        clf, X, y, scoring="neg_mean_squared_error", return_estimator=True, cv=cv
+    )
+    for k, est in enumerate(mse_scores_dict["estimator"]):
+        est_coef = est.coef_.copy()
+        if issparse(est_coef):
+            est_coef = est_coef.toarray()
+
+        fitted_est_coef = fitted_estimators[k].coef_.copy()
+        if issparse(fitted_est_coef):
+            fitted_est_coef = fitted_est_coef.toarray()
+
+        assert_almost_equal(est_coef, fitted_est_coef)
         assert_almost_equal(est.intercept_, fitted_estimators[k].intercept_)
 
 
-def check_cross_validate_multi_metric(clf, X, y, scores):
+def check_cross_validate_multi_metric(clf, X, y, scores, cv):
     # Test multimetric evaluation when scoring is a list / dict
-    (train_mse_scores, test_mse_scores, train_r2_scores,
-     test_r2_scores, fitted_estimators) = scores
+    (
+        train_mse_scores,
+        test_mse_scores,
+        train_r2_scores,
+        test_r2_scores,
+        fitted_estimators,
+    ) = scores
 
     def custom_scorer(clf, X, y):
         y_pred = clf.predict(X)
-        return {'r2': r2_score(y, y_pred),
-                'neg_mean_squared_error': -mean_squared_error(y, y_pred)}
-
-    all_scoring = (('r2', 'neg_mean_squared_error'),
-                   {'r2': make_scorer(r2_score),
-                    'neg_mean_squared_error': 'neg_mean_squared_error'},
-                   custom_scorer)
+        return {
+            "r2": r2_score(y, y_pred),
+            "neg_mean_squared_error": -mean_squared_error(y, y_pred),
+        }
+
+    all_scoring = (
+        ("r2", "neg_mean_squared_error"),
+        {
+            "r2": make_scorer(r2_score),
+            "neg_mean_squared_error": "neg_mean_squared_error",
+        },
+        custom_scorer,
+    )
 
-    keys_sans_train = {'test_r2', 'test_neg_mean_squared_error',
-                       'fit_time', 'score_time'}
+    keys_sans_train = {
+        "test_r2",
+        "test_neg_mean_squared_error",
+        "fit_time",
+        "score_time",
+    }
     keys_with_train = keys_sans_train.union(
-            {'train_r2', 'train_neg_mean_squared_error'})
+        {"train_r2", "train_neg_mean_squared_error"}
+    )
 
     for return_train_score in (True, False):
         for scoring in all_scoring:
             if return_train_score:
                 # return_train_score must be True by default - deprecated
-                cv_results = cross_validate(clf, X, y, scoring=scoring,
-                                            return_train_score=True)
-                assert_array_almost_equal(cv_results['train_r2'],
-                                          train_r2_scores)
+                cv_results = cross_validate(
+                    clf, X, y, scoring=scoring, return_train_score=True, cv=cv
+                )
+                assert_array_almost_equal(cv_results["train_r2"], train_r2_scores)
                 assert_array_almost_equal(
-                    cv_results['train_neg_mean_squared_error'],
-                    train_mse_scores)
+                    cv_results["train_neg_mean_squared_error"], train_mse_scores
+                )
             else:
-                cv_results = cross_validate(clf, X, y, scoring=scoring,
-                                            return_train_score=False)
+                cv_results = cross_validate(
+                    clf, X, y, scoring=scoring, return_train_score=False, cv=cv
+                )
             assert isinstance(cv_results, dict)
-            assert (set(cv_results.keys()) ==
-                    (keys_with_train if return_train_score
-                     else keys_sans_train))
-            assert_array_almost_equal(cv_results['test_r2'], test_r2_scores)
+            assert set(cv_results.keys()) == (
+                keys_with_train if return_train_score else keys_sans_train
+            )
+            assert_array_almost_equal(cv_results["test_r2"], test_r2_scores)
             assert_array_almost_equal(
-                cv_results['test_neg_mean_squared_error'], test_mse_scores)
+                cv_results["test_neg_mean_squared_error"], test_mse_scores
+            )
 
             # Make sure all the arrays are of np.ndarray type
-            assert type(cv_results['test_r2']) == np.ndarray
-            assert (type(cv_results['test_neg_mean_squared_error']) ==
-                    np.ndarray)
-            assert type(cv_results['fit_time']) == np.ndarray
-            assert type(cv_results['score_time']) == np.ndarray
+            assert isinstance(cv_results["test_r2"], np.ndarray)
+            assert isinstance(cv_results["test_neg_mean_squared_error"], np.ndarray)
+            assert isinstance(cv_results["fit_time"], np.ndarray)
+            assert isinstance(cv_results["score_time"], np.ndarray)
 
             # Ensure all the times are within sane limits
-            assert np.all(cv_results['fit_time'] >= 0)
-            assert np.all(cv_results['fit_time'] < 10)
-            assert np.all(cv_results['score_time'] >= 0)
-            assert np.all(cv_results['score_time'] < 10)
+            assert np.all(cv_results["fit_time"] >= 0)
+            assert np.all(cv_results["fit_time"] < 10)
+            assert np.all(cv_results["score_time"] >= 0)
+            assert np.all(cv_results["score_time"] < 10)
 
 
 def test_cross_val_score_predict_groups():
@@ -531,8 +608,12 @@ def test_cross_val_score_predict_groups():
 
     clf = SVC(kernel="linear")
 
-    group_cvs = [LeaveOneGroupOut(), LeavePGroupsOut(2), GroupKFold(),
-                 GroupShuffleSplit()]
+    group_cvs = [
+        LeaveOneGroupOut(),
+        LeavePGroupsOut(2),
+        GroupKFold(),
+        GroupShuffleSplit(),
+    ]
     error_message = "The 'groups' parameter should not be None."
     for cv in group_cvs:
         with pytest.raises(ValueError, match=error_message):
@@ -541,18 +622,18 @@ def test_cross_val_score_predict_groups():
             cross_val_predict(estimator=clf, X=X, y=y, cv=cv)
 
 
-@pytest.mark.filterwarnings('ignore: Using or importing the ABCs from')
 def test_cross_val_score_pandas():
     # check cross_val_score doesn't destroy pandas dataframe
     types = [(MockDataFrame, MockDataFrame)]
     try:
-        from pandas import Series, DataFrame
+        from pandas import DataFrame, Series
+
         types.append((Series, DataFrame))
     except ImportError:
         pass
     for TargetType, InputFeatureType in types:
         # X dataframe, y series
-        # 3 fold cross val is used so we need atleast 3 samples per class
+        # 3 fold cross val is used so we need at least 3 samples per class
         X_df, y_ser = InputFeatureType(X), TargetType(y2)
         check_df = lambda x: isinstance(x, InputFeatureType)
         check_series = lambda x: isinstance(x, TargetType)
@@ -606,17 +687,19 @@ def test_cross_val_score_precomputed():
         cross_val_score(svm, linear_kernel.tolist(), y)
 
 
-def test_cross_val_score_fit_params():
+@pytest.mark.parametrize("coo_container", COO_CONTAINERS)
+def test_cross_val_score_fit_params(coo_container):
     clf = MockClassifier()
     n_samples = X.shape[0]
     n_classes = len(np.unique(y))
 
-    W_sparse = coo_matrix((np.array([1]), (np.array([1]), np.array([0]))),
-                          shape=(10, 1))
-    P_sparse = coo_matrix(np.eye(5))
+    W_sparse = coo_container(
+        (np.array([1]), (np.array([1]), np.array([0]))), shape=(15, 1)
+    )
+    P_sparse = coo_container(np.eye(5))
 
     DUMMY_INT = 42
-    DUMMY_STR = '42'
+    DUMMY_STR = "42"
     DUMMY_OBJ = object()
 
     def assert_fit_params(clf):
@@ -627,15 +710,17 @@ def assert_fit_params(clf):
         assert clf.dummy_str == DUMMY_STR
         assert clf.dummy_obj == DUMMY_OBJ
 
-    fit_params = {'sample_weight': np.ones(n_samples),
-                  'class_prior': np.full(n_classes, 1. / n_classes),
-                  'sparse_sample_weight': W_sparse,
-                  'sparse_param': P_sparse,
-                  'dummy_int': DUMMY_INT,
-                  'dummy_str': DUMMY_STR,
-                  'dummy_obj': DUMMY_OBJ,
-                  'callback': assert_fit_params}
-    cross_val_score(clf, X, y, fit_params=fit_params)
+    fit_params = {
+        "sample_weight": np.ones(n_samples),
+        "class_prior": np.full(n_classes, 1.0 / n_classes),
+        "sparse_sample_weight": W_sparse,
+        "sparse_param": P_sparse,
+        "dummy_int": DUMMY_INT,
+        "dummy_str": DUMMY_STR,
+        "dummy_obj": DUMMY_OBJ,
+        "callback": assert_fit_params,
+    }
+    cross_val_score(clf, X, y2, params=fit_params)
 
 
 def test_cross_val_score_score_func():
@@ -654,38 +739,27 @@ def score_func(y_test, y_predict):
     assert len(_score_func_args) == 3
 
 
-def test_cross_val_score_errors():
-    class BrokenEstimator:
-        pass
-
-    with pytest.raises(TypeError):
-        cross_val_score(BrokenEstimator(), X)
-
-
 def test_cross_val_score_with_score_func_classification():
     iris = load_iris()
-    clf = SVC(kernel='linear')
+    clf = SVC(kernel="linear")
 
     # Default score (should be the accuracy score)
     scores = cross_val_score(clf, iris.data, iris.target)
-    assert_array_almost_equal(scores, [0.97, 1., 0.97, 0.97, 1.], 2)
+    assert_array_almost_equal(scores, [0.97, 1.0, 0.97, 0.97, 1.0], 2)
 
     # Correct classification score (aka. zero / one score) - should be the
     # same as the default estimator score
-    zo_scores = cross_val_score(clf, iris.data, iris.target,
-                                scoring="accuracy")
-    assert_array_almost_equal(zo_scores, [0.97, 1., 0.97, 0.97, 1.], 2)
+    zo_scores = cross_val_score(clf, iris.data, iris.target, scoring="accuracy")
+    assert_array_almost_equal(zo_scores, [0.97, 1.0, 0.97, 0.97, 1.0], 2)
 
     # F1 score (class are balanced so f1_score should be equal to zero/one
     # score
-    f1_scores = cross_val_score(clf, iris.data, iris.target,
-                                scoring="f1_weighted")
-    assert_array_almost_equal(f1_scores, [0.97, 1., 0.97, 0.97, 1.], 2)
+    f1_scores = cross_val_score(clf, iris.data, iris.target, scoring="f1_weighted")
+    assert_array_almost_equal(f1_scores, [0.97, 1.0, 0.97, 0.97, 1.0], 2)
 
 
 def test_cross_val_score_with_score_func_regression():
-    X, y = make_regression(n_samples=30, n_features=20, n_informative=5,
-                           random_state=0)
+    X, y = make_regression(n_samples=30, n_features=20, n_informative=5, random_state=0)
     reg = Ridge()
 
     # Default score of the Ridge regression estimator
@@ -698,8 +772,7 @@ def test_cross_val_score_with_score_func_regression():
     assert_array_almost_equal(r2_scores, [0.94, 0.97, 0.97, 0.99, 0.92], 2)
 
     # Mean squared error; this is a loss function, so "scores" are negative
-    neg_mse_scores = cross_val_score(reg, X, y,
-                                     scoring="neg_mean_squared_error")
+    neg_mse_scores = cross_val_score(reg, X, y, scoring="neg_mean_squared_error")
     expected_neg_mse = np.array([-763.07, -553.16, -274.38, -273.26, -1681.99])
     assert_array_almost_equal(neg_mse_scores, expected_neg_mse, 2)
 
@@ -709,51 +782,68 @@ def test_cross_val_score_with_score_func_regression():
     assert_array_almost_equal(ev_scores, [0.94, 0.97, 0.97, 0.99, 0.92], 2)
 
 
-def test_permutation_score():
+@pytest.mark.parametrize("coo_container", COO_CONTAINERS)
+def test_permutation_score(coo_container):
     iris = load_iris()
     X = iris.data
-    X_sparse = coo_matrix(X)
+    X_sparse = coo_container(X)
     y = iris.target
-    svm = SVC(kernel='linear')
+    svm = SVC(kernel="linear")
     cv = StratifiedKFold(2)
 
     score, scores, pvalue = permutation_test_score(
-        svm, X, y, n_permutations=30, cv=cv, scoring="accuracy")
+        svm, X, y, n_permutations=30, cv=cv, scoring="accuracy"
+    )
     assert score > 0.9
     assert_almost_equal(pvalue, 0.0, 1)
 
     score_group, _, pvalue_group = permutation_test_score(
-        svm, X, y, n_permutations=30, cv=cv, scoring="accuracy",
-        groups=np.ones(y.size), random_state=0)
+        svm,
+        X,
+        y,
+        n_permutations=30,
+        cv=cv,
+        scoring="accuracy",
+        groups=np.ones(y.size),
+        random_state=0,
+    )
     assert score_group == score
     assert pvalue_group == pvalue
 
     # check that we obtain the same results with a sparse representation
-    svm_sparse = SVC(kernel='linear')
+    svm_sparse = SVC(kernel="linear")
     cv_sparse = StratifiedKFold(2)
     score_group, _, pvalue_group = permutation_test_score(
-        svm_sparse, X_sparse, y, n_permutations=30, cv=cv_sparse,
-        scoring="accuracy", groups=np.ones(y.size), random_state=0)
+        svm_sparse,
+        X_sparse,
+        y,
+        n_permutations=30,
+        cv=cv_sparse,
+        scoring="accuracy",
+        groups=np.ones(y.size),
+        random_state=0,
+    )
 
     assert score_group == score
     assert pvalue_group == pvalue
 
     # test with custom scoring object
     def custom_score(y_true, y_pred):
-        return (((y_true == y_pred).sum() - (y_true != y_pred).sum()) /
-                y_true.shape[0])
+        return ((y_true == y_pred).sum() - (y_true != y_pred).sum()) / y_true.shape[0]
 
     scorer = make_scorer(custom_score)
     score, _, pvalue = permutation_test_score(
-        svm, X, y, n_permutations=100, scoring=scorer, cv=cv, random_state=0)
-    assert_almost_equal(score, .93, 2)
+        svm, X, y, n_permutations=100, scoring=scorer, cv=cv, random_state=0
+    )
+    assert_almost_equal(score, 0.93, 2)
     assert_almost_equal(pvalue, 0.01, 3)
 
     # set random y
     y = np.mod(np.arange(len(y)), 3)
 
     score, scores, pvalue = permutation_test_score(
-        svm, X, y, n_permutations=30, cv=cv, scoring="accuracy")
+        svm, X, y, n_permutations=30, cv=cv, scoring="accuracy"
+    )
 
     assert score < 0.5
     assert pvalue > 0.2
@@ -764,28 +854,28 @@ def test_permutation_test_score_allow_nans():
     X = np.arange(200, dtype=np.float64).reshape(10, -1)
     X[2, :] = np.nan
     y = np.repeat([0, 1], X.shape[0] / 2)
-    p = Pipeline([
-        ('imputer', SimpleImputer(strategy='mean', missing_values=np.nan)),
-        ('classifier', MockClassifier()),
-    ])
+    p = Pipeline(
+        [
+            ("imputer", SimpleImputer(strategy="mean", missing_values=np.nan)),
+            ("classifier", MockClassifier()),
+        ]
+    )
     permutation_test_score(p, X, y)
 
 
-def test_permutation_test_score_fit_params():
+def test_permutation_test_score_params():
     X = np.arange(100).reshape(10, 10)
     y = np.array([0] * 5 + [1] * 5)
-    clf = CheckingClassifier(expected_fit_params=['sample_weight'])
+    clf = CheckingClassifier(expected_sample_weight=True)
 
-    err_msg = r"Expected fit parameter\(s\) \['sample_weight'\] not seen."
+    err_msg = r"Expected sample_weight to be passed"
     with pytest.raises(AssertionError, match=err_msg):
         permutation_test_score(clf, X, y)
 
-    err_msg = "Fit parameter sample_weight has length 1; expected"
-    with pytest.raises(AssertionError, match=err_msg):
-        permutation_test_score(clf, X, y,
-                               fit_params={'sample_weight': np.ones(1)})
-    permutation_test_score(clf, X, y,
-                           fit_params={'sample_weight': np.ones(10)})
+    err_msg = r"sample_weight.shape == \(1,\), expected \(8,\)!"
+    with pytest.raises(ValueError, match=err_msg):
+        permutation_test_score(clf, X, y, params={"sample_weight": np.ones(1)})
+    permutation_test_score(clf, X, y, params={"sample_weight": np.ones(10)})
 
 
 def test_cross_val_score_allow_nans():
@@ -793,22 +883,37 @@ def test_cross_val_score_allow_nans():
     X = np.arange(200, dtype=np.float64).reshape(10, -1)
     X[2, :] = np.nan
     y = np.repeat([0, 1], X.shape[0] / 2)
-    p = Pipeline([
-        ('imputer', SimpleImputer(strategy='mean', missing_values=np.nan)),
-        ('classifier', MockClassifier()),
-    ])
+    p = Pipeline(
+        [
+            ("imputer", SimpleImputer(strategy="mean", missing_values=np.nan)),
+            ("classifier", MockClassifier()),
+        ]
+    )
     cross_val_score(p, X, y)
 
 
 def test_cross_val_score_multilabel():
-    X = np.array([[-3, 4], [2, 4], [3, 3], [0, 2], [-3, 1],
-                  [-2, 1], [0, 0], [-2, -1], [-1, -2], [1, -2]])
-    y = np.array([[1, 1], [0, 1], [0, 1], [0, 1], [1, 1],
-                  [0, 1], [1, 0], [1, 1], [1, 0], [0, 0]])
+    X = np.array(
+        [
+            [-3, 4],
+            [2, 4],
+            [3, 3],
+            [0, 2],
+            [-3, 1],
+            [-2, 1],
+            [0, 0],
+            [-2, -1],
+            [-1, -2],
+            [1, -2],
+        ]
+    )
+    y = np.array(
+        [[1, 1], [0, 1], [0, 1], [0, 1], [1, 1], [0, 1], [1, 0], [1, 1], [1, 0], [0, 0]]
+    )
     clf = KNeighborsClassifier(n_neighbors=1)
-    scoring_micro = make_scorer(precision_score, average='micro')
-    scoring_macro = make_scorer(precision_score, average='macro')
-    scoring_samples = make_scorer(precision_score, average='samples')
+    scoring_micro = make_scorer(precision_score, average="micro")
+    scoring_macro = make_scorer(precision_score, average="macro")
+    scoring_samples = make_scorer(precision_score, average="samples")
     score_micro = cross_val_score(clf, X, y, scoring=scoring_micro)
     score_macro = cross_val_score(clf, X, y, scoring=scoring_macro)
     score_samples = cross_val_score(clf, X, y, scoring=scoring_samples)
@@ -817,7 +922,8 @@ def test_cross_val_score_multilabel():
     assert_almost_equal(score_samples, [1, 1 / 2, 3 / 4, 1 / 2, 1 / 4])
 
 
-def test_cross_val_predict():
+@pytest.mark.parametrize("coo_container", COO_CONTAINERS)
+def test_cross_val_predict(coo_container):
     X, y = load_diabetes(return_X_y=True)
     cv = KFold()
 
@@ -840,15 +946,15 @@ def test_cross_val_predict():
     assert len(preds) == len(y)
 
     Xsp = X.copy()
-    Xsp *= (Xsp > np.median(Xsp))
-    Xsp = coo_matrix(Xsp)
+    Xsp *= Xsp > np.median(Xsp)
+    Xsp = coo_container(Xsp)
     preds = cross_val_predict(est, Xsp, y)
     assert_array_almost_equal(len(preds), len(y))
 
-    preds = cross_val_predict(KMeans(), X)
+    preds = cross_val_predict(KMeans(n_init="auto"), X)
     assert len(preds) == len(y)
 
-    class BadCV():
+    class BadCV:
         def split(self, X, y=None, groups=None):
             for i in range(4):
                 yield np.array([0, 1, 2, 3]), np.array([4, 5, 6, 7, 8])
@@ -858,25 +964,30 @@ def split(self, X, y=None, groups=None):
 
     X, y = load_iris(return_X_y=True)
 
-    warning_message = (r'Number of classes in training fold \(2\) does '
-                       r'not match total number of classes \(3\). '
-                       'Results may not be appropriate for your use case.')
+    warning_message = (
+        r"Number of classes in training fold \(2\) does "
+        r"not match total number of classes \(3\). "
+        "Results may not be appropriate for your use case."
+    )
     with pytest.warns(RuntimeWarning, match=warning_message):
-        cross_val_predict(LogisticRegression(solver="liblinear"),
-                          X, y, method='predict_proba', cv=KFold(2))
+        cross_val_predict(
+            LogisticRegression(solver="liblinear"),
+            X,
+            y,
+            method="predict_proba",
+            cv=KFold(2),
+        )
 
 
 def test_cross_val_predict_decision_function_shape():
     X, y = make_classification(n_classes=2, n_samples=50, random_state=0)
 
-    preds = cross_val_predict(LogisticRegression(solver="liblinear"), X, y,
-                              method='decision_function')
+    preds = cross_val_predict(LogisticRegression(), X, y, method="decision_function")
     assert preds.shape == (50,)
 
     X, y = load_iris(return_X_y=True)
 
-    preds = cross_val_predict(LogisticRegression(solver="liblinear"), X, y,
-                              method='decision_function')
+    preds = cross_val_predict(LogisticRegression(), X, y, method="decision_function")
     assert preds.shape == (150, 3)
 
     # This specifically tests imbalanced splits for binary
@@ -885,71 +996,70 @@ def test_cross_val_predict_decision_function_shape():
     # class.
     X = X[:100]
     y = y[:100]
-    error_message = 'Only 1 class/es in training fold,'\
-                    ' but 2 in overall dataset. This'\
-                    ' is not supported for decision_function'\
-                    ' with imbalanced folds. To fix '\
-                    'this, use a cross-validation technique '\
-                    'resulting in properly stratified folds'
+    error_message = (
+        "Only 1 class/es in training fold,"
+        " but 2 in overall dataset. This"
+        " is not supported for decision_function"
+        " with imbalanced folds. To fix "
+        "this, use a cross-validation technique "
+        "resulting in properly stratified folds"
+    )
     with pytest.raises(ValueError, match=error_message):
-        cross_val_predict(RidgeClassifier(), X, y, method='decision_function',
-                          cv=KFold(2))
+        cross_val_predict(
+            RidgeClassifier(), X, y, method="decision_function", cv=KFold(2)
+        )
 
     X, y = load_digits(return_X_y=True)
-    est = SVC(kernel='linear', decision_function_shape='ovo')
+    est = SVC(kernel="linear", decision_function_shape="ovo")
 
-    preds = cross_val_predict(est,
-                              X, y,
-                              method='decision_function')
+    preds = cross_val_predict(est, X, y, method="decision_function")
     assert preds.shape == (1797, 45)
 
     ind = np.argsort(y)
     X, y = X[ind], y[ind]
-    error_message_regexp = r'Output shape \(599L?, 21L?\) of ' \
-                           'decision_function does not match number of ' \
-                           r'classes \(7\) in fold. Irregular ' \
-                           'decision_function .*'
+    error_message_regexp = (
+        r"Output shape \(599L?, 21L?\) of "
+        "decision_function does not match number of "
+        r"classes \(7\) in fold. Irregular "
+        "decision_function .*"
+    )
     with pytest.raises(ValueError, match=error_message_regexp):
-        cross_val_predict(est, X, y, cv=KFold(n_splits=3),
-                          method='decision_function')
+        cross_val_predict(est, X, y, cv=KFold(n_splits=3), method="decision_function")
 
 
 def test_cross_val_predict_predict_proba_shape():
     X, y = make_classification(n_classes=2, n_samples=50, random_state=0)
 
-    preds = cross_val_predict(LogisticRegression(solver="liblinear"), X, y,
-                              method='predict_proba')
+    preds = cross_val_predict(LogisticRegression(), X, y, method="predict_proba")
     assert preds.shape == (50, 2)
 
     X, y = load_iris(return_X_y=True)
 
-    preds = cross_val_predict(LogisticRegression(solver="liblinear"), X, y,
-                              method='predict_proba')
+    preds = cross_val_predict(LogisticRegression(), X, y, method="predict_proba")
     assert preds.shape == (150, 3)
 
 
 def test_cross_val_predict_predict_log_proba_shape():
     X, y = make_classification(n_classes=2, n_samples=50, random_state=0)
 
-    preds = cross_val_predict(LogisticRegression(solver="liblinear"), X, y,
-                              method='predict_log_proba')
+    preds = cross_val_predict(LogisticRegression(), X, y, method="predict_log_proba")
     assert preds.shape == (50, 2)
 
     X, y = load_iris(return_X_y=True)
 
-    preds = cross_val_predict(LogisticRegression(solver="liblinear"), X, y,
-                              method='predict_log_proba')
+    preds = cross_val_predict(LogisticRegression(), X, y, method="predict_log_proba")
     assert preds.shape == (150, 3)
 
 
-def test_cross_val_predict_input_types():
+@pytest.mark.parametrize("coo_container", COO_CONTAINERS)
+def test_cross_val_predict_input_types(coo_container):
     iris = load_iris()
     X, y = iris.data, iris.target
-    X_sparse = coo_matrix(X)
+    X_sparse = coo_container(X)
     multioutput_y = np.column_stack([y, y[::-1]])
 
     clf = Ridge(fit_intercept=False, random_state=0)
-    # 3 fold cv is used --> atleast 3 samples per class
+    # 3 fold cv is used --> at least 3 samples per class
     # Smoke test
     predictions = cross_val_predict(clf, X, y)
     assert predictions.shape == (150,)
@@ -974,12 +1084,18 @@ def test_cross_val_predict_input_types():
     predictions = cross_val_predict(clf, X, y.tolist())
 
     # test with X and y as list and non empty method
-    predictions = cross_val_predict(LogisticRegression(solver="liblinear"),
-                                    X.tolist(),
-                                    y.tolist(), method='decision_function')
-    predictions = cross_val_predict(LogisticRegression(solver="liblinear"),
-                                    X,
-                                    y.tolist(), method='decision_function')
+    predictions = cross_val_predict(
+        LogisticRegression(),
+        X.tolist(),
+        y.tolist(),
+        method="decision_function",
+    )
+    predictions = cross_val_predict(
+        LogisticRegression(),
+        X,
+        y.tolist(),
+        method="decision_function",
+    )
 
     # test with 3d X and
     X_3d = X[:, :, np.newaxis]
@@ -989,13 +1105,12 @@ def test_cross_val_predict_input_types():
     assert_array_equal(predictions.shape, (150,))
 
 
-@pytest.mark.filterwarnings('ignore: Using or importing the ABCs from')
-# python3.7 deprecation warnings in pandas via matplotlib :-/
 def test_cross_val_predict_pandas():
     # check cross_val_score doesn't destroy pandas dataframe
     types = [(MockDataFrame, MockDataFrame)]
     try:
-        from pandas import Series, DataFrame
+        from pandas import DataFrame, Series
+
         types.append((Series, DataFrame))
     except ImportError:
         pass
@@ -1009,12 +1124,17 @@ def test_cross_val_predict_pandas():
 
 
 def test_cross_val_predict_unbalanced():
-    X, y = make_classification(n_samples=100, n_features=2, n_redundant=0,
-                               n_informative=2, n_clusters_per_class=1,
-                               random_state=1)
+    X, y = make_classification(
+        n_samples=100,
+        n_features=2,
+        n_redundant=0,
+        n_informative=2,
+        n_clusters_per_class=1,
+        random_state=1,
+    )
     # Change the first sample to a new class
     y[0] = 2
-    clf = LogisticRegression(random_state=1, solver="liblinear")
+    clf = LogisticRegression(random_state=1)
     cv = StratifiedKFold(n_splits=2)
     train, test = list(cv.split(X, y))
     yhat_proba = cross_val_predict(clf, X, y, cv=cv, method="predict_proba")
@@ -1022,8 +1142,7 @@ def test_cross_val_predict_unbalanced():
     assert np.all(yhat_proba[test[0]][:, 2] == 0)
     assert np.all(yhat_proba[test[0]][:, 0:1] > 0)
     assert np.all(yhat_proba[test[1]] > 0)
-    assert_array_almost_equal(yhat_proba.sum(axis=1), np.ones(y.shape),
-                              decimal=12)
+    assert_array_almost_equal(yhat_proba.sum(axis=1), np.ones(y.shape), decimal=12)
 
 
 def test_cross_val_predict_y_none():
@@ -1031,36 +1150,54 @@ def test_cross_val_predict_y_none():
     mock_classifier = MockClassifier()
     rng = np.random.RandomState(42)
     X = rng.rand(100, 10)
-    y_hat = cross_val_predict(mock_classifier, X, y=None, cv=5,
-                              method='predict')
+    y_hat = cross_val_predict(mock_classifier, X, y=None, cv=5, method="predict")
     assert_allclose(X[:, 0], y_hat)
-    y_hat_proba = cross_val_predict(mock_classifier, X, y=None, cv=5,
-                                    method='predict_proba')
+    y_hat_proba = cross_val_predict(
+        mock_classifier, X, y=None, cv=5, method="predict_proba"
+    )
     assert_allclose(X, y_hat_proba)
 
 
-def test_cross_val_score_sparse_fit_params():
+@pytest.mark.parametrize("coo_container", COO_CONTAINERS)
+def test_cross_val_score_sparse_fit_params(coo_container):
     iris = load_iris()
     X, y = iris.data, iris.target
     clf = MockClassifier()
-    fit_params = {'sparse_sample_weight': coo_matrix(np.eye(X.shape[0]))}
-    a = cross_val_score(clf, X, y, fit_params=fit_params, cv=3)
+    fit_params = {"sparse_sample_weight": coo_container(np.eye(X.shape[0]))}
+    a = cross_val_score(clf, X, y, params=fit_params, cv=3)
     assert_array_equal(a, np.ones(3))
 
 
 def test_learning_curve():
     n_samples = 30
     n_splits = 3
-    X, y = make_classification(n_samples=n_samples, n_features=1,
-                               n_informative=1, n_redundant=0, n_classes=2,
-                               n_clusters_per_class=1, random_state=0)
+    X, y = make_classification(
+        n_samples=n_samples,
+        n_features=1,
+        n_informative=1,
+        n_redundant=0,
+        n_classes=2,
+        n_clusters_per_class=1,
+        random_state=0,
+    )
     estimator = MockImprovingEstimator(n_samples * ((n_splits - 1) / n_splits))
     for shuffle_train in [False, True]:
         with warnings.catch_warnings(record=True) as w:
-            train_sizes, train_scores, test_scores, fit_times, score_times = \
-                learning_curve(estimator, X, y, cv=KFold(n_splits=n_splits),
-                               train_sizes=np.linspace(0.1, 1.0, 10),
-                               shuffle=shuffle_train, return_times=True)
+            (
+                train_sizes,
+                train_scores,
+                test_scores,
+                fit_times,
+                score_times,
+            ) = learning_curve(
+                estimator,
+                X,
+                y,
+                cv=KFold(n_splits=n_splits),
+                train_sizes=np.linspace(0.1, 1.0, 10),
+                shuffle=shuffle_train,
+                return_times=True,
+            )
         if len(w) > 0:
             raise RuntimeError("Unexpected warning: %r" % w[0].message)
         assert train_scores.shape == (10, 3)
@@ -1068,10 +1205,8 @@ def test_learning_curve():
         assert fit_times.shape == (10, 3)
         assert score_times.shape == (10, 3)
         assert_array_equal(train_sizes, np.linspace(2, 20, 10))
-        assert_array_almost_equal(train_scores.mean(axis=1),
-                                  np.linspace(1.9, 1.0, 10))
-        assert_array_almost_equal(test_scores.mean(axis=1),
-                                  np.linspace(0.1, 1.0, 10))
+        assert_array_almost_equal(train_scores.mean(axis=1), np.linspace(1.9, 1.0, 10))
+        assert_array_almost_equal(test_scores.mean(axis=1), np.linspace(0.1, 1.0, 10))
 
         # Cannot use assert_array_almost_equal for fit and score times because
         # the values are hardware-dependant
@@ -1081,10 +1216,13 @@ def test_learning_curve():
         # Test a custom cv splitter that can iterate only once
         with warnings.catch_warnings(record=True) as w:
             train_sizes2, train_scores2, test_scores2 = learning_curve(
-                estimator, X, y,
+                estimator,
+                X,
+                y,
                 cv=OneTimeSplitter(n_splits=n_splits, n_samples=n_samples),
                 train_sizes=np.linspace(0.1, 1.0, 10),
-                shuffle=shuffle_train)
+                shuffle=shuffle_train,
+            )
         if len(w) > 0:
             raise RuntimeError("Unexpected warning: %r" % w[0].message)
         assert_array_almost_equal(train_scores2, train_scores)
@@ -1092,42 +1230,60 @@ def test_learning_curve():
 
 
 def test_learning_curve_unsupervised():
-    X, _ = make_classification(n_samples=30, n_features=1, n_informative=1,
-                               n_redundant=0, n_classes=2,
-                               n_clusters_per_class=1, random_state=0)
+    X, _ = make_classification(
+        n_samples=30,
+        n_features=1,
+        n_informative=1,
+        n_redundant=0,
+        n_classes=2,
+        n_clusters_per_class=1,
+        random_state=0,
+    )
     estimator = MockImprovingEstimator(20)
     train_sizes, train_scores, test_scores = learning_curve(
-        estimator, X, y=None, cv=3, train_sizes=np.linspace(0.1, 1.0, 10))
+        estimator, X, y=None, cv=3, train_sizes=np.linspace(0.1, 1.0, 10)
+    )
     assert_array_equal(train_sizes, np.linspace(2, 20, 10))
-    assert_array_almost_equal(train_scores.mean(axis=1),
-                              np.linspace(1.9, 1.0, 10))
-    assert_array_almost_equal(test_scores.mean(axis=1),
-                              np.linspace(0.1, 1.0, 10))
+    assert_array_almost_equal(train_scores.mean(axis=1), np.linspace(1.9, 1.0, 10))
+    assert_array_almost_equal(test_scores.mean(axis=1), np.linspace(0.1, 1.0, 10))
 
 
 def test_learning_curve_verbose():
-    X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
-                               n_redundant=0, n_classes=2,
-                               n_clusters_per_class=1, random_state=0)
+    X, y = make_classification(
+        n_samples=30,
+        n_features=1,
+        n_informative=1,
+        n_redundant=0,
+        n_classes=2,
+        n_clusters_per_class=1,
+        random_state=0,
+    )
     estimator = MockImprovingEstimator(20)
 
     old_stdout = sys.stdout
     sys.stdout = StringIO()
     try:
-        train_sizes, train_scores, test_scores = \
-            learning_curve(estimator, X, y, cv=3, verbose=1)
+        train_sizes, train_scores, test_scores = learning_curve(
+            estimator, X, y, cv=3, verbose=1
+        )
     finally:
         out = sys.stdout.getvalue()
         sys.stdout.close()
         sys.stdout = old_stdout
 
-    assert("[learning_curve]" in out)
+    assert "[learning_curve]" in out
 
 
 def test_learning_curve_incremental_learning_not_possible():
-    X, y = make_classification(n_samples=2, n_features=1, n_informative=1,
-                               n_redundant=0, n_classes=2,
-                               n_clusters_per_class=1, random_state=0)
+    X, y = make_classification(
+        n_samples=2,
+        n_features=1,
+        n_informative=1,
+        n_redundant=0,
+        n_classes=2,
+        n_clusters_per_class=1,
+        random_state=0,
+    )
     # The mockup does not have partial_fit()
     estimator = MockImprovingEstimator(1)
     with pytest.raises(ValueError):
@@ -1135,64 +1291,104 @@ def test_learning_curve_incremental_learning_not_possible():
 
 
 def test_learning_curve_incremental_learning():
-    X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
-                               n_redundant=0, n_classes=2,
-                               n_clusters_per_class=1, random_state=0)
+    X, y = make_classification(
+        n_samples=30,
+        n_features=1,
+        n_informative=1,
+        n_redundant=0,
+        n_classes=2,
+        n_clusters_per_class=1,
+        random_state=0,
+    )
     estimator = MockIncrementalImprovingEstimator(20)
     for shuffle_train in [False, True]:
         train_sizes, train_scores, test_scores = learning_curve(
-            estimator, X, y, cv=3, exploit_incremental_learning=True,
-            train_sizes=np.linspace(0.1, 1.0, 10), shuffle=shuffle_train)
+            estimator,
+            X,
+            y,
+            cv=3,
+            exploit_incremental_learning=True,
+            train_sizes=np.linspace(0.1, 1.0, 10),
+            shuffle=shuffle_train,
+        )
         assert_array_equal(train_sizes, np.linspace(2, 20, 10))
-        assert_array_almost_equal(train_scores.mean(axis=1),
-                                  np.linspace(1.9, 1.0, 10))
-        assert_array_almost_equal(test_scores.mean(axis=1),
-                                  np.linspace(0.1, 1.0, 10))
+        assert_array_almost_equal(train_scores.mean(axis=1), np.linspace(1.9, 1.0, 10))
+        assert_array_almost_equal(test_scores.mean(axis=1), np.linspace(0.1, 1.0, 10))
 
 
 def test_learning_curve_incremental_learning_unsupervised():
-    X, _ = make_classification(n_samples=30, n_features=1, n_informative=1,
-                               n_redundant=0, n_classes=2,
-                               n_clusters_per_class=1, random_state=0)
+    X, _ = make_classification(
+        n_samples=30,
+        n_features=1,
+        n_informative=1,
+        n_redundant=0,
+        n_classes=2,
+        n_clusters_per_class=1,
+        random_state=0,
+    )
     estimator = MockIncrementalImprovingEstimator(20)
     train_sizes, train_scores, test_scores = learning_curve(
-        estimator, X, y=None, cv=3, exploit_incremental_learning=True,
-        train_sizes=np.linspace(0.1, 1.0, 10))
+        estimator,
+        X,
+        y=None,
+        cv=3,
+        exploit_incremental_learning=True,
+        train_sizes=np.linspace(0.1, 1.0, 10),
+    )
     assert_array_equal(train_sizes, np.linspace(2, 20, 10))
-    assert_array_almost_equal(train_scores.mean(axis=1),
-                              np.linspace(1.9, 1.0, 10))
-    assert_array_almost_equal(test_scores.mean(axis=1),
-                              np.linspace(0.1, 1.0, 10))
+    assert_array_almost_equal(train_scores.mean(axis=1), np.linspace(1.9, 1.0, 10))
+    assert_array_almost_equal(test_scores.mean(axis=1), np.linspace(0.1, 1.0, 10))
 
 
 def test_learning_curve_batch_and_incremental_learning_are_equal():
-    X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
-                               n_redundant=0, n_classes=2,
-                               n_clusters_per_class=1, random_state=0)
+    X, y = make_classification(
+        n_samples=30,
+        n_features=1,
+        n_informative=1,
+        n_redundant=0,
+        n_classes=2,
+        n_clusters_per_class=1,
+        random_state=0,
+    )
     train_sizes = np.linspace(0.2, 1.0, 5)
-    estimator = PassiveAggressiveClassifier(max_iter=1, tol=None,
-                                            shuffle=False)
+    estimator = PassiveAggressiveClassifier(max_iter=1, tol=None, shuffle=False)
 
-    train_sizes_inc, train_scores_inc, test_scores_inc = \
-        learning_curve(
-            estimator, X, y, train_sizes=train_sizes,
-            cv=3, exploit_incremental_learning=True)
-    train_sizes_batch, train_scores_batch, test_scores_batch = \
-        learning_curve(
-            estimator, X, y, cv=3, train_sizes=train_sizes,
-            exploit_incremental_learning=False)
+    train_sizes_inc, train_scores_inc, test_scores_inc = learning_curve(
+        estimator,
+        X,
+        y,
+        train_sizes=train_sizes,
+        cv=3,
+        exploit_incremental_learning=True,
+    )
+    train_sizes_batch, train_scores_batch, test_scores_batch = learning_curve(
+        estimator,
+        X,
+        y,
+        cv=3,
+        train_sizes=train_sizes,
+        exploit_incremental_learning=False,
+    )
 
     assert_array_equal(train_sizes_inc, train_sizes_batch)
-    assert_array_almost_equal(train_scores_inc.mean(axis=1),
-                              train_scores_batch.mean(axis=1))
-    assert_array_almost_equal(test_scores_inc.mean(axis=1),
-                              test_scores_batch.mean(axis=1))
+    assert_array_almost_equal(
+        train_scores_inc.mean(axis=1), train_scores_batch.mean(axis=1)
+    )
+    assert_array_almost_equal(
+        test_scores_inc.mean(axis=1), test_scores_batch.mean(axis=1)
+    )
 
 
 def test_learning_curve_n_sample_range_out_of_bounds():
-    X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
-                               n_redundant=0, n_classes=2,
-                               n_clusters_per_class=1, random_state=0)
+    X, y = make_classification(
+        n_samples=30,
+        n_features=1,
+        n_informative=1,
+        n_redundant=0,
+        n_classes=2,
+        n_clusters_per_class=1,
+        random_state=0,
+    )
     estimator = MockImprovingEstimator(20)
     with pytest.raises(ValueError):
         learning_curve(estimator, X, y, cv=3, train_sizes=[0, 1])
@@ -1207,9 +1403,15 @@ def test_learning_curve_n_sample_range_out_of_bounds():
 
 
 def test_learning_curve_remove_duplicate_sample_sizes():
-    X, y = make_classification(n_samples=3, n_features=1, n_informative=1,
-                               n_redundant=0, n_classes=2,
-                               n_clusters_per_class=1, random_state=0)
+    X, y = make_classification(
+        n_samples=3,
+        n_features=1,
+        n_informative=1,
+        n_redundant=0,
+        n_classes=2,
+        n_clusters_per_class=1,
+        random_state=0,
+    )
     estimator = MockImprovingEstimator(2)
     warning_message = (
         "Removed duplicate entries from 'train_sizes'. Number of ticks "
@@ -1217,112 +1419,195 @@ def test_learning_curve_remove_duplicate_sample_sizes():
     )
     with pytest.warns(RuntimeWarning, match=warning_message):
         train_sizes, _, _ = learning_curve(
-            estimator, X, y, cv=3, train_sizes=np.linspace(0.33, 1.0, 3))
+            estimator, X, y, cv=3, train_sizes=np.linspace(0.33, 1.0, 3)
+        )
     assert_array_equal(train_sizes, [1, 2])
 
 
 def test_learning_curve_with_boolean_indices():
-    X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
-                               n_redundant=0, n_classes=2,
-                               n_clusters_per_class=1, random_state=0)
+    X, y = make_classification(
+        n_samples=30,
+        n_features=1,
+        n_informative=1,
+        n_redundant=0,
+        n_classes=2,
+        n_clusters_per_class=1,
+        random_state=0,
+    )
     estimator = MockImprovingEstimator(20)
     cv = KFold(n_splits=3)
     train_sizes, train_scores, test_scores = learning_curve(
-        estimator, X, y, cv=cv, train_sizes=np.linspace(0.1, 1.0, 10))
+        estimator, X, y, cv=cv, train_sizes=np.linspace(0.1, 1.0, 10)
+    )
     assert_array_equal(train_sizes, np.linspace(2, 20, 10))
-    assert_array_almost_equal(train_scores.mean(axis=1),
-                              np.linspace(1.9, 1.0, 10))
-    assert_array_almost_equal(test_scores.mean(axis=1),
-                              np.linspace(0.1, 1.0, 10))
+    assert_array_almost_equal(train_scores.mean(axis=1), np.linspace(1.9, 1.0, 10))
+    assert_array_almost_equal(test_scores.mean(axis=1), np.linspace(0.1, 1.0, 10))
 
 
 def test_learning_curve_with_shuffle():
     # Following test case was designed this way to verify the code
     # changes made in pull request: #7506.
-    X = np.array([[1, 2], [3, 4], [5, 6], [7, 8], [11, 12], [13, 14], [15, 16],
-                 [17, 18], [19, 20], [7, 8], [9, 10], [11, 12], [13, 14],
-                 [15, 16], [17, 18]])
+    X = np.array(
+        [
+            [1, 2],
+            [3, 4],
+            [5, 6],
+            [7, 8],
+            [11, 12],
+            [13, 14],
+            [15, 16],
+            [17, 18],
+            [19, 20],
+            [7, 8],
+            [9, 10],
+            [11, 12],
+            [13, 14],
+            [15, 16],
+            [17, 18],
+        ]
+    )
     y = np.array([1, 1, 1, 2, 3, 4, 1, 1, 2, 3, 4, 1, 2, 3, 4])
     groups = np.array([1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 4, 4, 4, 4])
     # Splits on these groups fail without shuffle as the first iteration
     # of the learning curve doesn't contain label 4 in the training set.
-    estimator = PassiveAggressiveClassifier(max_iter=5, tol=None,
-                                            shuffle=False)
+    estimator = PassiveAggressiveClassifier(max_iter=5, tol=None, shuffle=False)
 
     cv = GroupKFold(n_splits=2)
     train_sizes_batch, train_scores_batch, test_scores_batch = learning_curve(
-        estimator, X, y, cv=cv, n_jobs=1, train_sizes=np.linspace(0.3, 1.0, 3),
-        groups=groups, shuffle=True, random_state=2)
-    assert_array_almost_equal(train_scores_batch.mean(axis=1),
-                              np.array([0.75, 0.3, 0.36111111]))
-    assert_array_almost_equal(test_scores_batch.mean(axis=1),
-                              np.array([0.36111111, 0.25, 0.25]))
+        estimator,
+        X,
+        y,
+        cv=cv,
+        n_jobs=1,
+        train_sizes=np.linspace(0.3, 1.0, 3),
+        groups=groups,
+        shuffle=True,
+        random_state=2,
+    )
+    assert_array_almost_equal(
+        train_scores_batch.mean(axis=1), np.array([0.75, 0.3, 0.36111111])
+    )
+    assert_array_almost_equal(
+        test_scores_batch.mean(axis=1), np.array([0.36111111, 0.25, 0.25])
+    )
     with pytest.raises(ValueError):
-        learning_curve(estimator, X, y, cv=cv, n_jobs=1,
-                       train_sizes=np.linspace(0.3, 1.0, 3), groups=groups,
-                       error_score='raise')
+        learning_curve(
+            estimator,
+            X,
+            y,
+            cv=cv,
+            n_jobs=1,
+            train_sizes=np.linspace(0.3, 1.0, 3),
+            groups=groups,
+            error_score="raise",
+        )
 
     train_sizes_inc, train_scores_inc, test_scores_inc = learning_curve(
-        estimator, X, y, cv=cv, n_jobs=1, train_sizes=np.linspace(0.3, 1.0, 3),
-        groups=groups, shuffle=True, random_state=2,
-        exploit_incremental_learning=True)
-    assert_array_almost_equal(train_scores_inc.mean(axis=1),
-                              train_scores_batch.mean(axis=1))
-    assert_array_almost_equal(test_scores_inc.mean(axis=1),
-                              test_scores_batch.mean(axis=1))
+        estimator,
+        X,
+        y,
+        cv=cv,
+        n_jobs=1,
+        train_sizes=np.linspace(0.3, 1.0, 3),
+        groups=groups,
+        shuffle=True,
+        random_state=2,
+        exploit_incremental_learning=True,
+    )
+    assert_array_almost_equal(
+        train_scores_inc.mean(axis=1), train_scores_batch.mean(axis=1)
+    )
+    assert_array_almost_equal(
+        test_scores_inc.mean(axis=1), test_scores_batch.mean(axis=1)
+    )
 
 
-def test_learning_curve_fit_params():
+def test_learning_curve_params():
     X = np.arange(100).reshape(10, 10)
     y = np.array([0] * 5 + [1] * 5)
-    clf = CheckingClassifier(expected_fit_params=['sample_weight'])
+    clf = CheckingClassifier(expected_sample_weight=True)
 
-    err_msg = r"Expected fit parameter\(s\) \['sample_weight'\] not seen."
+    err_msg = r"Expected sample_weight to be passed"
     with pytest.raises(AssertionError, match=err_msg):
-        learning_curve(clf, X, y, error_score='raise')
+        learning_curve(clf, X, y, error_score="raise")
 
-    err_msg = "Fit parameter sample_weight has length 1; expected"
-    with pytest.raises(AssertionError, match=err_msg):
-        learning_curve(clf, X, y, error_score='raise',
-                       fit_params={'sample_weight': np.ones(1)})
-    learning_curve(clf, X, y, error_score='raise',
-                   fit_params={'sample_weight': np.ones(10)})
+    err_msg = r"sample_weight.shape == \(1,\), expected \(2,\)!"
+    with pytest.raises(ValueError, match=err_msg):
+        learning_curve(
+            clf, X, y, error_score="raise", params={"sample_weight": np.ones(1)}
+        )
+    learning_curve(
+        clf, X, y, error_score="raise", params={"sample_weight": np.ones(10)}
+    )
 
 
-def test_learning_curve_incremental_learning_fit_params():
-    X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
-                               n_redundant=0, n_classes=2,
-                               n_clusters_per_class=1, random_state=0)
-    estimator = MockIncrementalImprovingEstimator(20, ['sample_weight'])
+def test_learning_curve_incremental_learning_params():
+    X, y = make_classification(
+        n_samples=30,
+        n_features=1,
+        n_informative=1,
+        n_redundant=0,
+        n_classes=2,
+        n_clusters_per_class=1,
+        random_state=0,
+    )
+    estimator = MockIncrementalImprovingEstimator(20, ["sample_weight"])
     err_msg = r"Expected fit parameter\(s\) \['sample_weight'\] not seen."
     with pytest.raises(AssertionError, match=err_msg):
-        learning_curve(estimator, X, y, cv=3,
-                       exploit_incremental_learning=True,
-                       train_sizes=np.linspace(0.1, 1.0, 10),
-                       error_score='raise')
+        learning_curve(
+            estimator,
+            X,
+            y,
+            cv=3,
+            exploit_incremental_learning=True,
+            train_sizes=np.linspace(0.1, 1.0, 10),
+            error_score="raise",
+        )
 
     err_msg = "Fit parameter sample_weight has length 3; expected"
     with pytest.raises(AssertionError, match=err_msg):
-        learning_curve(estimator, X, y, cv=3,
-                       exploit_incremental_learning=True,
-                       train_sizes=np.linspace(0.1, 1.0, 10),
-                       error_score='raise',
-                       fit_params={'sample_weight': np.ones(3)})
+        learning_curve(
+            estimator,
+            X,
+            y,
+            cv=3,
+            exploit_incremental_learning=True,
+            train_sizes=np.linspace(0.1, 1.0, 10),
+            error_score="raise",
+            params={"sample_weight": np.ones(3)},
+        )
 
-    learning_curve(estimator, X, y, cv=3, exploit_incremental_learning=True,
-                   train_sizes=np.linspace(0.1, 1.0, 10), error_score='raise',
-                   fit_params={'sample_weight': np.ones(2)})
+    learning_curve(
+        estimator,
+        X,
+        y,
+        cv=3,
+        exploit_incremental_learning=True,
+        train_sizes=np.linspace(0.1, 1.0, 10),
+        error_score="raise",
+        params={"sample_weight": np.ones(2)},
+    )
 
 
 def test_validation_curve():
-    X, y = make_classification(n_samples=2, n_features=1, n_informative=1,
-                               n_redundant=0, n_classes=2,
-                               n_clusters_per_class=1, random_state=0)
+    X, y = make_classification(
+        n_samples=2,
+        n_features=1,
+        n_informative=1,
+        n_redundant=0,
+        n_classes=2,
+        n_clusters_per_class=1,
+        random_state=0,
+    )
     param_range = np.linspace(0, 1, 10)
     with warnings.catch_warnings(record=True) as w:
         train_scores, test_scores = validation_curve(
-            MockEstimatorWithParameter(), X, y, param_name="param",
-            param_range=param_range, cv=2
+            MockEstimatorWithParameter(),
+            X,
+            y,
+            param_name="param",
+            param_range=param_range,
+            cv=2,
         )
     if len(w) > 0:
         raise RuntimeError("Unexpected warning: %r" % w[0].message)
@@ -1332,14 +1617,24 @@ def test_validation_curve():
 
 
 def test_validation_curve_clone_estimator():
-    X, y = make_classification(n_samples=2, n_features=1, n_informative=1,
-                               n_redundant=0, n_classes=2,
-                               n_clusters_per_class=1, random_state=0)
+    X, y = make_classification(
+        n_samples=2,
+        n_features=1,
+        n_informative=1,
+        n_redundant=0,
+        n_classes=2,
+        n_clusters_per_class=1,
+        random_state=0,
+    )
 
     param_range = np.linspace(1, 0, 10)
     _, _ = validation_curve(
-        MockEstimatorWithSingleFitCallAllowed(), X, y,
-        param_name="param", param_range=param_range, cv=2
+        MockEstimatorWithSingleFitCallAllowed(),
+        X,
+        y,
+        param_name="param",
+        param_range=param_range,
+        cv=2,
     )
 
 
@@ -1348,56 +1643,83 @@ def test_validation_curve_cv_splits_consistency():
     n_splits = 5
     X, y = make_classification(n_samples=100, random_state=0)
 
-    scores1 = validation_curve(SVC(kernel='linear', random_state=0), X, y,
-                               param_name='C',
-                               param_range=[0.1, 0.1, 0.2, 0.2],
-                               cv=OneTimeSplitter(n_splits=n_splits,
-                                                  n_samples=n_samples))
+    scores1 = validation_curve(
+        SVC(kernel="linear", random_state=0),
+        X,
+        y,
+        param_name="C",
+        param_range=[0.1, 0.1, 0.2, 0.2],
+        cv=OneTimeSplitter(n_splits=n_splits, n_samples=n_samples),
+    )
     # The OneTimeSplitter is a non-re-entrant cv splitter. Unless, the
     # `split` is called for each parameter, the following should produce
     # identical results for param setting 1 and param setting 2 as both have
     # the same C value.
-    assert_array_almost_equal(*np.vsplit(np.hstack(scores1)[(0, 2, 1, 3), :],
-                                         2))
-
-    scores2 = validation_curve(SVC(kernel='linear', random_state=0), X, y,
-                               param_name='C',
-                               param_range=[0.1, 0.1, 0.2, 0.2],
-                               cv=KFold(n_splits=n_splits, shuffle=True))
+    assert_array_almost_equal(*np.vsplit(np.hstack(scores1)[(0, 2, 1, 3), :], 2))
+
+    scores2 = validation_curve(
+        SVC(kernel="linear", random_state=0),
+        X,
+        y,
+        param_name="C",
+        param_range=[0.1, 0.1, 0.2, 0.2],
+        cv=KFold(n_splits=n_splits, shuffle=True),
+    )
 
     # For scores2, compare the 1st and 2nd parameter's scores
     # (Since the C value for 1st two param setting is 0.1, they must be
     # consistent unless the train test folds differ between the param settings)
-    assert_array_almost_equal(*np.vsplit(np.hstack(scores2)[(0, 2, 1, 3), :],
-                                         2))
-
-    scores3 = validation_curve(SVC(kernel='linear', random_state=0), X, y,
-                               param_name='C',
-                               param_range=[0.1, 0.1, 0.2, 0.2],
-                               cv=KFold(n_splits=n_splits))
+    assert_array_almost_equal(*np.vsplit(np.hstack(scores2)[(0, 2, 1, 3), :], 2))
+
+    scores3 = validation_curve(
+        SVC(kernel="linear", random_state=0),
+        X,
+        y,
+        param_name="C",
+        param_range=[0.1, 0.1, 0.2, 0.2],
+        cv=KFold(n_splits=n_splits),
+    )
 
     # OneTimeSplitter is basically unshuffled KFold(n_splits=5). Sanity check.
     assert_array_almost_equal(np.array(scores3), np.array(scores1))
 
 
-def test_validation_curve_fit_params():
+def test_validation_curve_params():
     X = np.arange(100).reshape(10, 10)
     y = np.array([0] * 5 + [1] * 5)
-    clf = CheckingClassifier(expected_fit_params=['sample_weight'])
+    clf = CheckingClassifier(expected_sample_weight=True)
 
-    err_msg = r"Expected fit parameter\(s\) \['sample_weight'\] not seen."
+    err_msg = r"Expected sample_weight to be passed"
     with pytest.raises(AssertionError, match=err_msg):
-        validation_curve(clf, X, y, param_name='foo_param',
-                         param_range=[1, 2, 3], error_score='raise')
+        validation_curve(
+            clf,
+            X,
+            y,
+            param_name="foo_param",
+            param_range=[1, 2, 3],
+            error_score="raise",
+        )
 
-    err_msg = "Fit parameter sample_weight has length 1; expected"
-    with pytest.raises(AssertionError, match=err_msg):
-        validation_curve(clf, X, y, param_name='foo_param',
-                         param_range=[1, 2, 3], error_score='raise',
-                         fit_params={'sample_weight': np.ones(1)})
-    validation_curve(clf, X, y, param_name='foo_param',
-                     param_range=[1, 2, 3], error_score='raise',
-                     fit_params={'sample_weight': np.ones(10)})
+    err_msg = r"sample_weight.shape == \(1,\), expected \(8,\)!"
+    with pytest.raises(ValueError, match=err_msg):
+        validation_curve(
+            clf,
+            X,
+            y,
+            param_name="foo_param",
+            param_range=[1, 2, 3],
+            error_score="raise",
+            params={"sample_weight": np.ones(1)},
+        )
+    validation_curve(
+        clf,
+        X,
+        y,
+        param_name="foo_param",
+        param_range=[1, 2, 3],
+        error_score="raise",
+        params={"sample_weight": np.ones(10)},
+    )
 
 
 def test_check_is_permutation():
@@ -1414,15 +1736,19 @@ def test_check_is_permutation():
     assert not _check_is_permutation(np.hstack((p, 0)), 100)
 
 
-def test_cross_val_predict_sparse_prediction():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_cross_val_predict_sparse_prediction(csr_container):
     # check that cross_val_predict gives same result for sparse and dense input
-    X, y = make_multilabel_classification(n_classes=2, n_labels=1,
-                                          allow_unlabeled=False,
-                                          return_indicator=True,
-                                          random_state=1)
-    X_sparse = csr_matrix(X)
-    y_sparse = csr_matrix(y)
-    classif = OneVsRestClassifier(SVC(kernel='linear'))
+    X, y = make_multilabel_classification(
+        n_classes=2,
+        n_labels=1,
+        allow_unlabeled=False,
+        return_indicator=True,
+        random_state=1,
+    )
+    X_sparse = csr_container(X)
+    y_sparse = csr_container(y)
+    classif = OneVsRestClassifier(SVC(kernel="linear"))
     preds = cross_val_predict(classif, X, y, cv=10)
     preds_sparse = cross_val_predict(classif, X_sparse, y_sparse, cv=10)
     preds_sparse = preds_sparse.toarray()
@@ -1435,7 +1761,7 @@ def check_cross_val_predict_binary(est, X, y, method):
 
     # Generate expected outputs
     if y.ndim == 1:
-        exp_shape = (len(X),) if method == 'decision_function' else (len(X), 2)
+        exp_shape = (len(X),) if method == "decision_function" else (len(X), 2)
     else:
         exp_shape = y.shape
     expected_predictions = np.zeros(exp_shape)
@@ -1444,9 +1770,10 @@ def check_cross_val_predict_binary(est, X, y, method):
         expected_predictions[test] = getattr(est, method)(X[test])
 
     # Check actual outputs for several representations of y
-    for tg in [y, y + 1, y - 2, y.astype('str')]:
-        assert_allclose(cross_val_predict(est, X, tg, method=method, cv=cv),
-                        expected_predictions)
+    for tg in [y, y + 1, y - 2, y.astype("str")]:
+        assert_allclose(
+            cross_val_predict(est, X, tg, method=method, cv=cv), expected_predictions
+        )
 
 
 def check_cross_val_predict_multiclass(est, X, y, method):
@@ -1455,12 +1782,14 @@ def check_cross_val_predict_multiclass(est, X, y, method):
 
     # Generate expected outputs
     float_min = np.finfo(np.float64).min
-    default_values = {'decision_function': float_min,
-                      'predict_log_proba': float_min,
-                      'predict_proba': 0}
-    expected_predictions = np.full((len(X), len(set(y))),
-                                   default_values[method],
-                                   dtype=np.float64)
+    default_values = {
+        "decision_function": float_min,
+        "predict_log_proba": float_min,
+        "predict_proba": 0,
+    }
+    expected_predictions = np.full(
+        (len(X), len(set(y))), default_values[method], dtype=np.float64
+    )
     _, y_enc = np.unique(y, return_inverse=True)
     for train, test in cv.split(X, y_enc):
         est = clone(est).fit(X[train], y_enc[train])
@@ -1469,9 +1798,10 @@ def check_cross_val_predict_multiclass(est, X, y, method):
         expected_predictions[np.ix_(test, i_cols_fit)] = fold_preds
 
     # Check actual outputs for several representations of y
-    for tg in [y, y + 1, y - 2, y.astype('str')]:
-        assert_allclose(cross_val_predict(est, X, tg, method=method, cv=cv),
-                        expected_predictions)
+    for tg in [y, y + 1, y - 2, y.astype("str")]:
+        assert_allclose(
+            cross_val_predict(est, X, tg, method=method, cv=cv), expected_predictions
+        )
 
 
 def check_cross_val_predict_multilabel(est, X, y, method):
@@ -1483,23 +1813,28 @@ def check_cross_val_predict_multilabel(est, X, y, method):
 
     # Create empty arrays of the correct size to hold outputs
     float_min = np.finfo(np.float64).min
-    default_values = {'decision_function': float_min,
-                      'predict_log_proba': float_min,
-                      'predict_proba': 0}
+    default_values = {
+        "decision_function": float_min,
+        "predict_log_proba": float_min,
+        "predict_proba": 0,
+    }
     n_targets = y.shape[1]
     expected_preds = []
     for i_col in range(n_targets):
         n_classes_in_label = len(set(y[:, i_col]))
-        if n_classes_in_label == 2 and method == 'decision_function':
+        if n_classes_in_label == 2 and method == "decision_function":
             exp_shape = (len(X),)
         else:
             exp_shape = (len(X), n_classes_in_label)
-        expected_preds.append(np.full(exp_shape, default_values[method],
-                                      dtype=np.float64))
+        expected_preds.append(
+            np.full(exp_shape, default_values[method], dtype=np.float64)
+        )
 
     # Generate expected outputs
-    y_enc_cols = [np.unique(y[:, i], return_inverse=True)[1][:, np.newaxis]
-                  for i in range(y.shape[1])]
+    y_enc_cols = [
+        np.unique(y[:, i], return_inverse=True)[1][:, np.newaxis]
+        for i in range(y.shape[1])
+    ]
     y_enc = np.concatenate(y_enc_cols, axis=1)
     for train, test in cv.split(X, y_enc):
         est = clone(est).fit(X[train], y_enc[train])
@@ -1514,7 +1849,7 @@ def check_cross_val_predict_multilabel(est, X, y, method):
                 expected_preds[i_col][idx] = fold_preds[i_col]
 
     # Check actual outputs for several representations of y
-    for tg in [y, y + 1, y - 2, y.astype('str')]:
+    for tg in [y, y + 1, y - 2, y.astype("str")]:
         cv_predict_output = cross_val_predict(est, X, tg, method=method, cv=cv)
         assert len(cv_predict_output) == len(expected_preds)
         for i in range(len(cv_predict_output)):
@@ -1524,8 +1859,8 @@ def check_cross_val_predict_multilabel(est, X, y, method):
 def check_cross_val_predict_with_method_binary(est):
     # This test includes the decision_function with two classes.
     # This is a special case: it has only one column of output.
-    X, y = make_classification(n_classes=2,  random_state=0)
-    for method in ['decision_function', 'predict_proba', 'predict_log_proba']:
+    X, y = make_classification(n_classes=2, random_state=0)
+    for method in ["decision_function", "predict_proba", "predict_log_proba"]:
         check_cross_val_predict_binary(est, X, y, method)
 
 
@@ -1533,15 +1868,13 @@ def check_cross_val_predict_with_method_multiclass(est):
     iris = load_iris()
     X, y = iris.data, iris.target
     X, y = shuffle(X, y, random_state=0)
-    for method in ['decision_function', 'predict_proba', 'predict_log_proba']:
+    for method in ["decision_function", "predict_proba", "predict_log_proba"]:
         check_cross_val_predict_multiclass(est, X, y, method)
 
 
 def test_cross_val_predict_with_method():
-    check_cross_val_predict_with_method_binary(
-            LogisticRegression(solver="liblinear"))
-    check_cross_val_predict_with_method_multiclass(
-            LogisticRegression(solver="liblinear"))
+    check_cross_val_predict_with_method_binary(LogisticRegression())
+    check_cross_val_predict_with_method_multiclass(LogisticRegression())
 
 
 def test_cross_val_predict_method_checking():
@@ -1550,8 +1883,8 @@ def test_cross_val_predict_method_checking():
     iris = load_iris()
     X, y = iris.data, iris.target
     X, y = shuffle(X, y, random_state=0)
-    for method in ['decision_function', 'predict_proba', 'predict_log_proba']:
-        est = SGDClassifier(loss='log', random_state=2)
+    for method in ["decision_function", "predict_proba", "predict_log_proba"]:
+        est = SGDClassifier(loss="log_loss", random_state=2)
         check_cross_val_predict_multiclass(est, X, y, method)
 
 
@@ -1559,10 +1892,8 @@ def test_gridsearchcv_cross_val_predict_with_method():
     iris = load_iris()
     X, y = iris.data, iris.target
     X, y = shuffle(X, y, random_state=0)
-    est = GridSearchCV(LogisticRegression(random_state=42, solver="liblinear"),
-                       {'C': [0.1, 1]},
-                       cv=2)
-    for method in ['decision_function', 'predict_proba', 'predict_log_proba']:
+    est = GridSearchCV(LogisticRegression(random_state=42), {"C": [0.1, 1]}, cv=2)
+    for method in ["decision_function", "predict_proba", "predict_log_proba"]:
         check_cross_val_predict_multiclass(est, X, y, method)
 
 
@@ -1572,12 +1903,11 @@ def test_cross_val_predict_with_method_multilabel_ovr():
     # is a 2D array with shape (n_samples, n_classes).
     n_samp = 100
     n_classes = 4
-    X, y = make_multilabel_classification(n_samples=n_samp, n_labels=3,
-                                          n_classes=n_classes, n_features=5,
-                                          random_state=42)
-    est = OneVsRestClassifier(LogisticRegression(solver="liblinear",
-                                                 random_state=0))
-    for method in ['predict_proba', 'decision_function']:
+    X, y = make_multilabel_classification(
+        n_samples=n_samp, n_labels=3, n_classes=n_classes, n_features=5, random_state=42
+    )
+    est = OneVsRestClassifier(LogisticRegression(solver="liblinear", random_state=0))
+    for method in ["predict_proba", "decision_function"]:
         check_cross_val_predict_binary(est, X, y, method=method)
 
 
@@ -1598,15 +1928,15 @@ def test_cross_val_predict_with_method_multilabel_rf():
     # Output of predict_proba is a list of outputs of predict_proba
     # for each individual label.
     n_classes = 4
-    X, y = make_multilabel_classification(n_samples=100, n_labels=3,
-                                          n_classes=n_classes, n_features=5,
-                                          random_state=42)
+    X, y = make_multilabel_classification(
+        n_samples=100, n_labels=3, n_classes=n_classes, n_features=5, random_state=42
+    )
     y[:, 0] += y[:, 1]  # Put three classes in the first column
-    for method in ['predict_proba', 'predict_log_proba', 'decision_function']:
+    for method in ["predict_proba", "predict_log_proba", "decision_function"]:
         est = RFWithDecisionFunction(n_estimators=5, random_state=0)
         with warnings.catch_warnings():
             # Suppress "RuntimeWarning: divide by zero encountered in log"
-            warnings.simplefilter('ignore')
+            warnings.simplefilter("ignore")
             check_cross_val_predict_multilabel(est, X, y, method=method)
 
 
@@ -1616,11 +1946,11 @@ def test_cross_val_predict_with_method_rare_class():
     rng = np.random.RandomState(0)
     X = rng.normal(0, 1, size=(14, 10))
     y = np.array([0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 3])
-    est = LogisticRegression(solver="liblinear")
-    for method in ['predict_proba', 'predict_log_proba', 'decision_function']:
+    est = LogisticRegression()
+    for method in ["predict_proba", "predict_log_proba", "decision_function"]:
         with warnings.catch_warnings():
             # Suppress warning about too few examples of a class
-            warnings.simplefilter('ignore')
+            warnings.simplefilter("ignore")
             check_cross_val_predict_multiclass(est, X, y, method)
 
 
@@ -1633,16 +1963,15 @@ def test_cross_val_predict_with_method_multilabel_rf_rare_class():
     rng = np.random.RandomState(0)
     X = rng.normal(0, 1, size=(5, 10))
     y = np.array([[0, 0], [1, 1], [2, 1], [0, 1], [1, 0]])
-    for method in ['predict_proba', 'predict_log_proba']:
+    for method in ["predict_proba", "predict_log_proba"]:
         est = RFWithDecisionFunction(n_estimators=5, random_state=0)
         with warnings.catch_warnings():
             # Suppress "RuntimeWarning: divide by zero encountered in log"
-            warnings.simplefilter('ignore')
+            warnings.simplefilter("ignore")
             check_cross_val_predict_multilabel(est, X, y, method=method)
 
 
 def get_expected_predictions(X, y, cv, classes, est, method):
-
     expected_predictions = np.zeros([len(y), classes])
     func = getattr(est, method)
 
@@ -1650,11 +1979,12 @@ def get_expected_predictions(X, y, cv, classes, est, method):
         est.fit(X[train], y[train])
         expected_predictions_ = func(X[test])
         # To avoid 2 dimensional indexing
-        if method == 'predict_proba':
+        if method == "predict_proba":
             exp_pred_test = np.zeros((len(test), classes))
         else:
-            exp_pred_test = np.full((len(test), classes),
-                                    np.finfo(expected_predictions.dtype).min)
+            exp_pred_test = np.full(
+                (len(test), classes), np.finfo(expected_predictions.dtype).min
+            )
         exp_pred_test[:, est.classes_] = expected_predictions_
         expected_predictions[test] = exp_pred_test
 
@@ -1662,7 +1992,6 @@ def get_expected_predictions(X, y, cv, classes, est, method):
 
 
 def test_cross_val_predict_class_subset():
-
     X = np.arange(200).reshape(100, 2)
     y = np.array([x // 10 for x in range(100)])
     classes = 10
@@ -1672,33 +2001,33 @@ def test_cross_val_predict_class_subset():
 
     le = LabelEncoder()
 
-    methods = ['decision_function', 'predict_proba', 'predict_log_proba']
+    methods = ["decision_function", "predict_proba", "predict_log_proba"]
     for method in methods:
-        est = LogisticRegression(solver="liblinear")
+        est = LogisticRegression()
 
         # Test with n_splits=3
-        predictions = cross_val_predict(est, X, y, method=method,
-                                        cv=kfold3)
+        predictions = cross_val_predict(est, X, y, method=method, cv=kfold3)
 
         # Runs a naive loop (should be same as cross_val_predict):
-        expected_predictions = get_expected_predictions(X, y, kfold3, classes,
-                                                        est, method)
+        expected_predictions = get_expected_predictions(
+            X, y, kfold3, classes, est, method
+        )
         assert_array_almost_equal(expected_predictions, predictions)
 
         # Test with n_splits=4
-        predictions = cross_val_predict(est, X, y, method=method,
-                                        cv=kfold4)
-        expected_predictions = get_expected_predictions(X, y, kfold4, classes,
-                                                        est, method)
+        predictions = cross_val_predict(est, X, y, method=method, cv=kfold4)
+        expected_predictions = get_expected_predictions(
+            X, y, kfold4, classes, est, method
+        )
         assert_array_almost_equal(expected_predictions, predictions)
 
         # Testing unordered labels
         y = shuffle(np.repeat(range(10), 10), random_state=0)
-        predictions = cross_val_predict(est, X, y, method=method,
-                                        cv=kfold3)
+        predictions = cross_val_predict(est, X, y, method=method, cv=kfold3)
         y = le.fit_transform(y)
-        expected_predictions = get_expected_predictions(X, y, kfold3, classes,
-                                                        est, method)
+        expected_predictions = get_expected_predictions(
+            X, y, kfold3, classes, est, method
+        )
         assert_array_almost_equal(expected_predictions, predictions)
 
 
@@ -1707,11 +2036,11 @@ def test_score_memmap():
     iris = load_iris()
     X, y = iris.data, iris.target
     clf = MockClassifier()
-    tf = tempfile.NamedTemporaryFile(mode='wb', delete=False)
-    tf.write(b'Hello world!!!!!')
+    tf = tempfile.NamedTemporaryFile(mode="wb", delete=False)
+    tf.write(b"Hello world!!!!!")
     tf.close()
     scores = np.memmap(tf.name, dtype=np.float64)
-    score = np.memmap(tf.name, shape=(), mode='r', dtype=np.float64)
+    score = np.memmap(tf.name, shape=(), mode="r", dtype=np.float64)
     try:
         cross_val_score(clf, X, y, scoring=lambda est, X, y: score)
         with pytest.raises(ValueError):
@@ -1724,16 +2053,16 @@ def test_score_memmap():
             try:
                 os.unlink(tf.name)
                 break
-            except WindowsError:
-                sleep(1.)
+            except OSError:
+                sleep(1.0)
 
 
-@pytest.mark.filterwarnings('ignore: Using or importing the ABCs from')
 def test_permutation_test_score_pandas():
     # check permutation_test_score doesn't destroy pandas dataframe
     types = [(MockDataFrame, MockDataFrame)]
     try:
-        from pandas import Series, DataFrame
+        from pandas import DataFrame, Series
+
         types.append((Series, DataFrame))
     except ImportError:
         pass
@@ -1753,83 +2082,115 @@ def test_fit_and_score_failing():
     failing_clf = FailingClassifier(FailingClassifier.FAILING_PARAMETER)
     # dummy X data
     X = np.arange(1, 10)
+    train, test = np.arange(0, 5), np.arange(5, 9)
+    fit_and_score_args = dict(
+        estimator=failing_clf,
+        X=X,
+        y=None,
+        scorer=dict(),
+        train=train,
+        test=test,
+        verbose=0,
+        parameters=None,
+        fit_params=None,
+        score_params=None,
+    )
+    # passing error score to trigger the warning message
+    fit_and_score_args["error_score"] = "raise"
+    # check if exception was raised, with default error_score='raise'
+    with pytest.raises(ValueError, match="Failing classifier failed as required"):
+        _fit_and_score(**fit_and_score_args)
+
+    assert failing_clf.score() == 0.0  # FailingClassifier coverage
+
+
+def test_fit_and_score_working():
+    X, y = make_classification(n_samples=30, random_state=0)
+    clf = SVC(kernel="linear", random_state=0)
+    train, test = next(ShuffleSplit().split(X))
+    # Test return_parameters option
+    fit_and_score_args = dict(
+        estimator=clf,
+        X=X,
+        y=y,
+        scorer=dict(),
+        train=train,
+        test=test,
+        verbose=0,
+        parameters={"max_iter": 100, "tol": 0.1},
+        fit_params=None,
+        score_params=None,
+        return_parameters=True,
+    )
+    result = _fit_and_score(**fit_and_score_args)
+    assert result["parameters"] == fit_and_score_args["parameters"]
+
+
+class DataDependentFailingClassifier(BaseEstimator):
+    def __init__(self, max_x_value=None):
+        self.max_x_value = max_x_value
+
+    def fit(self, X, y=None):
+        num_values_too_high = (X > self.max_x_value).sum()
+        if num_values_too_high:
+            raise ValueError(
+                f"Classifier fit failed with {num_values_too_high} values too high"
+            )
+
+    def score(self, X=None, Y=None):
+        return 0.0
+
+
+@pytest.mark.parametrize("error_score", [np.nan, 0])
+def test_cross_validate_some_failing_fits_warning(error_score):
+    # Create a failing classifier to deliberately fail
+    failing_clf = DataDependentFailingClassifier(max_x_value=8)
+    # dummy X data
+    X = np.arange(1, 10)
     y = np.ones(9)
-    fit_and_score_args = [failing_clf, X, None, dict(), None, None, 0,
-                          None, None]
     # passing error score to trigger the warning message
-    fit_and_score_kwargs = {'error_score': 0}
+    cross_validate_args = [failing_clf, X, y]
+    cross_validate_kwargs = {"cv": 3, "error_score": error_score}
     # check if the warning message type is as expected
-    warning_message = (
-        "Estimator fit failed. The score on this train-test partition for "
-        "these parameters will be set to %f."
-        % (fit_and_score_kwargs['error_score'])
+
+    individual_fit_error_message = (
+        "ValueError: Classifier fit failed with 1 values too high"
     )
-    with pytest.warns(FitFailedWarning, match=warning_message):
-        _fit_and_score(*fit_and_score_args, **fit_and_score_kwargs)
-    # since we're using FailingClassfier, our error will be the following
-    error_message = "ValueError: Failing classifier failed as required"
-    # the warning message we're expecting to see
-    warning_message = ("Estimator fit failed. The score on this train-test "
-                       "partition for these parameters will be set to %f. "
-                       "Details: \n%s" % (fit_and_score_kwargs['error_score'],
-                                          error_message))
-
-    def test_warn_trace(msg):
-        assert 'Traceback (most recent call last):\n' in msg
-        split = msg.splitlines()  # note: handles more than '\n'
-        mtb = split[0] + '\n' + split[-1]
-        return warning_message in mtb
-    # check traceback is included
-    warning_message = (
-        "Estimator fit failed. The score on this train-test partition for "
-        "these parameters will be set to %f."
-        % (fit_and_score_kwargs['error_score'])
+    warning_message = re.compile(
+        (
+            "2 fits failed.+total of 3.+The score on these"
+            " train-test partitions for these parameters will be set to"
+            f" {cross_validate_kwargs['error_score']}.+{individual_fit_error_message}"
+        ),
+        flags=re.DOTALL,
     )
+
     with pytest.warns(FitFailedWarning, match=warning_message):
-        _fit_and_score(*fit_and_score_args, **fit_and_score_kwargs)
+        cross_validate(*cross_validate_args, **cross_validate_kwargs)
 
-    fit_and_score_kwargs = {'error_score': 'raise'}
-    # check if exception was raised, with default error_score='raise'
-    with pytest.raises(
-        ValueError,
-        match="Failing classifier failed as required"
-    ):
-        _fit_and_score(*fit_and_score_args, **fit_and_score_kwargs)
 
-    # check that functions upstream pass error_score param to _fit_and_score
-    error_message = re.escape(
-        "error_score must be the string 'raise' or a numeric value. (Hint: if "
-        "using 'raise', please make sure that it has been spelled correctly.)"
-    )
-    with pytest.raises(ValueError, match=error_message):
-        cross_validate(failing_clf, X, cv=3, error_score='unvalid-string')
+@pytest.mark.parametrize("error_score", [np.nan, 0])
+def test_cross_validate_all_failing_fits_error(error_score):
+    # Create a failing classifier to deliberately fail
+    failing_clf = FailingClassifier(FailingClassifier.FAILING_PARAMETER)
+    # dummy X data
+    X = np.arange(1, 10)
+    y = np.ones(9)
 
-    with pytest.raises(ValueError, match=error_message):
-        cross_val_score(failing_clf, X, cv=3, error_score='unvalid-string')
+    cross_validate_args = [failing_clf, X, y]
+    cross_validate_kwargs = {"cv": 7, "error_score": error_score}
 
-    with pytest.raises(ValueError, match=error_message):
-        learning_curve(failing_clf, X, y, cv=3, error_score='unvalid-string')
+    individual_fit_error_message = "ValueError: Failing classifier failed as required"
+    error_message = re.compile(
+        (
+            "All the 7 fits failed.+your model is misconfigured.+"
+            f"{individual_fit_error_message}"
+        ),
+        flags=re.DOTALL,
+    )
 
     with pytest.raises(ValueError, match=error_message):
-        validation_curve(failing_clf, X, y, param_name='parameter',
-                         param_range=[FailingClassifier.FAILING_PARAMETER],
-                         cv=3, error_score='unvalid-string')
-
-    assert failing_clf.score() == 0.  # FailingClassifier coverage
-
-
-def test_fit_and_score_working():
-    X, y = make_classification(n_samples=30, random_state=0)
-    clf = SVC(kernel="linear", random_state=0)
-    train, test = next(ShuffleSplit().split(X))
-    # Test return_parameters option
-    fit_and_score_args = [clf, X, y, dict(), train, test, 0]
-    fit_and_score_kwargs = {'parameters': {'max_iter': 100, 'tol': 0.1},
-                            'fit_params': None,
-                            'return_parameters': True}
-    result = _fit_and_score(*fit_and_score_args,
-                            **fit_and_score_kwargs)
-    assert result['parameters'] == fit_and_score_kwargs['parameters']
+        cross_validate(*cross_validate_args, **cross_validate_kwargs)
 
 
 def _failing_scorer(estimator, X, y, error_msg):
@@ -1850,18 +2211,16 @@ def test_cross_val_score_failing_scorer(error_score):
     if error_score == "raise":
         with pytest.raises(ValueError, match=error_msg):
             cross_val_score(
-                clf, X, y, cv=3, scoring=failing_scorer,
-                error_score=error_score
+                clf, X, y, cv=3, scoring=failing_scorer, error_score=error_score
             )
     else:
         warning_msg = (
-            f"Scoring failed. The score on this train-test partition for "
+            "Scoring failed. The score on this train-test partition for "
             f"these parameters will be set to {error_score}"
         )
         with pytest.warns(UserWarning, match=warning_msg):
             scores = cross_val_score(
-                clf, X, y, cv=3, scoring=failing_scorer,
-                error_score=error_score
+                clf, X, y, cv=3, scoring=failing_scorer, error_score=error_score
             )
             assert_allclose(scores, error_score)
 
@@ -1873,45 +2232,62 @@ def test_cross_val_score_failing_scorer(error_score):
 def test_cross_validate_failing_scorer(
     error_score, return_train_score, with_multimetric
 ):
-    # check that an estimator can fail during scoring in `cross_validate` and
-    # that we can optionally replaced it with `error_score`
+    # Check that an estimator can fail during scoring in `cross_validate` and
+    # that we can optionally replace it with `error_score`. In the multimetric
+    # case also check the result of a non-failing scorer where the other scorers
+    # are failing.
     X, y = load_iris(return_X_y=True)
     clf = LogisticRegression(max_iter=5).fit(X, y)
 
     error_msg = "This scorer is supposed to fail!!!"
     failing_scorer = partial(_failing_scorer, error_msg=error_msg)
     if with_multimetric:
-        scoring = {"score_1": failing_scorer, "score_2": failing_scorer}
+        non_failing_scorer = make_scorer(mean_squared_error)
+        scoring = {
+            "score_1": failing_scorer,
+            "score_2": non_failing_scorer,
+            "score_3": failing_scorer,
+        }
     else:
         scoring = failing_scorer
 
     if error_score == "raise":
         with pytest.raises(ValueError, match=error_msg):
             cross_validate(
-                clf, X, y,
+                clf,
+                X,
+                y,
                 cv=3,
                 scoring=scoring,
                 return_train_score=return_train_score,
-                error_score=error_score
+                error_score=error_score,
             )
     else:
         warning_msg = (
-            f"Scoring failed. The score on this train-test partition for "
+            "Scoring failed. The score on this train-test partition for "
             f"these parameters will be set to {error_score}"
         )
         with pytest.warns(UserWarning, match=warning_msg):
             results = cross_validate(
-                clf, X, y,
+                clf,
+                X,
+                y,
                 cv=3,
                 scoring=scoring,
                 return_train_score=return_train_score,
-                error_score=error_score
+                error_score=error_score,
             )
             for key in results:
                 if "_score" in key:
-                    # check the test (and optionally train score) for all
-                    # scorers that should be assigned to `error_score`.
-                    assert_allclose(results[key], error_score)
+                    if "_score_2" in key:
+                        # check the test (and optionally train) score for the
+                        # scorer that should be non-failing
+                        for i in results[key]:
+                            assert isinstance(i, float)
+                    else:
+                        # check the test (and optionally train) score for all
+                        # scorers that should be assigned to `error_score`.
+                        assert_allclose(results[key], error_score)
 
 
 def three_params_scorer(i, j, k):
@@ -1919,33 +2295,67 @@ def three_params_scorer(i, j, k):
 
 
 @pytest.mark.parametrize(
-    "train_score, scorer, verbose, split_prg, cdt_prg, expected", [
-     (False, three_params_scorer, 2, (1, 3), (0, 1),
-      r"\[CV\] END ...................................................."
-      r" total time=   0.\ds"),
-     (True, {'sc1': three_params_scorer, 'sc2': three_params_scorer}, 3,
-      (1, 3), (0, 1),
-      r"\[CV 2/3\] END  sc1: \(train=3.421, test=3.421\) sc2: "
-      r"\(train=3.421, test=3.421\) total time=   0.\ds"),
-     (False, {'sc1': three_params_scorer, 'sc2': three_params_scorer}, 10,
-      (1, 3), (0, 1),
-      r"\[CV 2/3; 1/1\] END ....... sc1: \(test=3.421\) sc2: \(test=3.421\)"
-      r" total time=   0.\ds")
-    ])
-def test_fit_and_score_verbosity(capsys, train_score, scorer, verbose,
-                                 split_prg, cdt_prg, expected):
+    "train_score, scorer, verbose, split_prg, cdt_prg, expected",
+    [
+        (
+            False,
+            three_params_scorer,
+            2,
+            (1, 3),
+            (0, 1),
+            r"\[CV\] END ...................................................."
+            r" total time=   0.\ds",
+        ),
+        (
+            True,
+            _MultimetricScorer(
+                scorers={"sc1": three_params_scorer, "sc2": three_params_scorer}
+            ),
+            3,
+            (1, 3),
+            (0, 1),
+            r"\[CV 2/3\] END  sc1: \(train=3.421, test=3.421\) sc2: "
+            r"\(train=3.421, test=3.421\) total time=   0.\ds",
+        ),
+        (
+            False,
+            _MultimetricScorer(
+                scorers={"sc1": three_params_scorer, "sc2": three_params_scorer}
+            ),
+            10,
+            (1, 3),
+            (0, 1),
+            r"\[CV 2/3; 1/1\] END ....... sc1: \(test=3.421\) sc2: \(test=3.421\)"
+            r" total time=   0.\ds",
+        ),
+    ],
+)
+def test_fit_and_score_verbosity(
+    capsys, train_score, scorer, verbose, split_prg, cdt_prg, expected
+):
     X, y = make_classification(n_samples=30, random_state=0)
     clf = SVC(kernel="linear", random_state=0)
     train, test = next(ShuffleSplit().split(X))
 
     # test print without train score
-    fit_and_score_args = [clf, X, y, scorer, train, test, verbose, None, None]
-    fit_and_score_kwargs = {'return_train_score': train_score,
-                            'split_progress': split_prg,
-                            'candidate_progress': cdt_prg}
-    _fit_and_score(*fit_and_score_args, **fit_and_score_kwargs)
+    fit_and_score_args = dict(
+        estimator=clf,
+        X=X,
+        y=y,
+        scorer=scorer,
+        train=train,
+        test=test,
+        verbose=verbose,
+        parameters=None,
+        fit_params=None,
+        score_params=None,
+        return_train_score=train_score,
+        split_progress=split_prg,
+        candidate_progress=cdt_prg,
+    )
+    _fit_and_score(**fit_and_score_args)
     out, _ = capsys.readouterr()
-    outlines = out.split('\n')
+    outlines = out.split("\n")
     if len(outlines) > 2:
         assert re.match(expected, outlines[1])
     else:
@@ -1957,47 +2367,373 @@ def test_score():
 
     def two_params_scorer(estimator, X_test):
         return None
-    fit_and_score_args = [None, None, None, two_params_scorer]
+
     with pytest.raises(ValueError, match=error_message):
-        _score(*fit_and_score_args, error_score=np.nan)
+        _score(
+            estimator=None,
+            X_test=None,
+            y_test=None,
+            scorer=two_params_scorer,
+            score_params=None,
+            error_score=np.nan,
+        )
 
 
 def test_callable_multimetric_confusion_matrix_cross_validate():
     def custom_scorer(clf, X, y):
         y_pred = clf.predict(X)
         cm = confusion_matrix(y, y_pred)
-        return {'tn': cm[0, 0], 'fp': cm[0, 1], 'fn': cm[1, 0], 'tp': cm[1, 1]}
+        return {"tn": cm[0, 0], "fp": cm[0, 1], "fn": cm[1, 0], "tp": cm[1, 1]}
 
-    X, y = make_classification(n_samples=40, n_features=4,
-                               random_state=42)
+    X, y = make_classification(n_samples=40, n_features=4, random_state=42)
     est = LinearSVC(random_state=42)
     est.fit(X, y)
     cv_results = cross_validate(est, X, y, cv=5, scoring=custom_scorer)
 
-    score_names = ['tn', 'fp', 'fn', 'tp']
+    score_names = ["tn", "fp", "fn", "tp"]
     for name in score_names:
         assert "test_{}".format(name) in cv_results
 
 
-# TODO: Remove in 1.1 when the _pairwise attribute is removed
-def test_validation_pairwise():
-    # checks the interactions between the pairwise estimator tag
-    # and the _pairwise attribute
-    iris = load_iris()
-    X, y = iris.data, iris.target
-    linear_kernel = np.dot(X, X.T)
+def test_learning_curve_partial_fit_regressors():
+    """Check that regressors with partial_fit is supported.
 
-    svm = SVC(kernel="precomputed")
-    with pytest.warns(None) as record:
-        cross_validate(svm, linear_kernel, y, cv=2)
-    assert not record
-
-    # pairwise tag is not consistent with pairwise attribute
-    class IncorrectTagSVM(SVC):
-        def _more_tags(self):
-            return {'pairwise': False}
-
-    svm = IncorrectTagSVM(kernel='precomputed')
-    msg = "_pairwise was deprecated in 0.24 and will be removed in 1.1"
-    with pytest.warns(FutureWarning, match=msg):
-        cross_validate(svm, linear_kernel, y, cv=2)
+    Non-regression test for #22981.
+    """
+    X, y = make_regression(random_state=42)
+
+    # Does not error
+    learning_curve(MLPRegressor(), X, y, exploit_incremental_learning=True, cv=2)
+
+
+def test_learning_curve_some_failing_fits_warning(global_random_seed):
+    """Checks for fit failures in `learning_curve` and raises the required warning"""
+
+    X, y = make_classification(
+        n_samples=30,
+        n_classes=3,
+        n_informative=6,
+        shuffle=False,
+        random_state=global_random_seed,
+    )
+    # sorting the target to trigger SVC error on the 2 first splits because a single
+    # class is present
+    sorted_idx = np.argsort(y)
+    X, y = X[sorted_idx], y[sorted_idx]
+
+    svc = SVC()
+    warning_message = "10 fits failed out of a total of 25"
+
+    with pytest.warns(FitFailedWarning, match=warning_message):
+        _, train_score, test_score, *_ = learning_curve(
+            svc, X, y, cv=5, error_score=np.nan
+        )
+
+    # the first 2 splits should lead to warnings and thus np.nan scores
+    for idx in range(2):
+        assert np.isnan(train_score[idx]).all()
+        assert np.isnan(test_score[idx]).all()
+
+    for idx in range(2, train_score.shape[0]):
+        assert not np.isnan(train_score[idx]).any()
+        assert not np.isnan(test_score[idx]).any()
+
+
+def test_cross_validate_return_indices(global_random_seed):
+    """Check the behaviour of `return_indices` in `cross_validate`."""
+    X, y = load_iris(return_X_y=True)
+    X = scale(X)  # scale features for better convergence
+    estimator = LogisticRegression()
+
+    cv = KFold(n_splits=3, shuffle=True, random_state=global_random_seed)
+    cv_results = cross_validate(estimator, X, y, cv=cv, n_jobs=2, return_indices=False)
+    assert "indices" not in cv_results
+
+    cv_results = cross_validate(estimator, X, y, cv=cv, n_jobs=2, return_indices=True)
+    assert "indices" in cv_results
+    train_indices = cv_results["indices"]["train"]
+    test_indices = cv_results["indices"]["test"]
+    assert len(train_indices) == cv.n_splits
+    assert len(test_indices) == cv.n_splits
+
+    assert_array_equal([indices.size for indices in train_indices], 100)
+    assert_array_equal([indices.size for indices in test_indices], 50)
+
+    for split_idx, (expected_train_idx, expected_test_idx) in enumerate(cv.split(X, y)):
+        assert_array_equal(train_indices[split_idx], expected_train_idx)
+        assert_array_equal(test_indices[split_idx], expected_test_idx)
+
+
+# Tests for metadata routing in cross_val* and in *curve
+# ======================================================
+
+
+# TODO(1.8): remove `learning_curve`, `validation_curve` and `permutation_test_score`.
+@pytest.mark.parametrize(
+    "func, extra_args",
+    [
+        (learning_curve, {}),
+        (permutation_test_score, {}),
+        (validation_curve, {"param_name": "alpha", "param_range": np.array([1])}),
+    ],
+)
+def test_fit_param_deprecation(func, extra_args):
+    """Check that we warn about deprecating `fit_params`."""
+    with pytest.warns(FutureWarning, match="`fit_params` is deprecated"):
+        func(
+            estimator=ConsumingClassifier(), X=X, y=y, cv=2, fit_params={}, **extra_args
+        )
+
+    with pytest.raises(
+        ValueError, match="`params` and `fit_params` cannot both be provided"
+    ):
+        func(
+            estimator=ConsumingClassifier(),
+            X=X,
+            y=y,
+            fit_params={},
+            params={},
+            **extra_args,
+        )
+
+
+@pytest.mark.parametrize(
+    "func, extra_args",
+    [
+        (cross_validate, {}),
+        (cross_val_score, {}),
+        (cross_val_predict, {}),
+        (learning_curve, {}),
+        (permutation_test_score, {}),
+        (validation_curve, {"param_name": "alpha", "param_range": np.array([1])}),
+    ],
+)
+@config_context(enable_metadata_routing=True)
+def test_groups_with_routing_validation(func, extra_args):
+    """Check that we raise an error if `groups` are passed to the cv method instead
+    of `params` when metadata routing is enabled.
+    """
+    with pytest.raises(ValueError, match="`groups` can only be passed if"):
+        func(
+            estimator=ConsumingClassifier(),
+            X=X,
+            y=y,
+            groups=[],
+            **extra_args,
+        )
+
+
+@pytest.mark.parametrize(
+    "func, extra_args",
+    [
+        (cross_validate, {}),
+        (cross_val_score, {}),
+        (cross_val_predict, {}),
+        (learning_curve, {}),
+        (permutation_test_score, {}),
+        (validation_curve, {"param_name": "alpha", "param_range": np.array([1])}),
+    ],
+)
+@config_context(enable_metadata_routing=True)
+def test_cross_validate_params_none(func, extra_args):
+    """Test that no errors are raised when passing `params=None`, which is the
+    default value.
+    Non-regression test for: https://github.com/scikit-learn/scikit-learn/issues/30447
+    """
+    X, y = make_classification(n_samples=100, n_classes=2, random_state=0)
+    func(estimator=ConsumingClassifier(), X=X, y=y, **extra_args)
+
+
+@pytest.mark.parametrize(
+    "func, extra_args",
+    [
+        (cross_validate, {}),
+        (cross_val_score, {}),
+        (cross_val_predict, {}),
+        (learning_curve, {}),
+        (permutation_test_score, {}),
+        (validation_curve, {"param_name": "alpha", "param_range": np.array([1])}),
+    ],
+)
+@config_context(enable_metadata_routing=True)
+def test_passed_unrequested_metadata(func, extra_args):
+    """Check that we raise an error when passing metadata that is not
+    requested."""
+
+    err_msg = re.escape(
+        "[metadata] are passed but are not explicitly set as requested or not "
+        "requested for ConsumingClassifier.fit, which is used within"
+    )
+    with pytest.raises(UnsetMetadataPassedError, match=err_msg):
+        func(
+            estimator=ConsumingClassifier(),
+            X=X,
+            y=y2,
+            params=dict(metadata=[]),
+            **extra_args,
+        )
+
+    # cross_val_predict doesn't use scoring
+    if func == cross_val_predict:
+        return
+
+    err_msg = re.escape(
+        "[metadata] are passed but are not explicitly set as requested or not "
+        "requested for ConsumingClassifier.score, which is used within"
+    )
+    with pytest.raises(UnsetMetadataPassedError, match=err_msg):
+        func(
+            estimator=ConsumingClassifier()
+            .set_fit_request(metadata=True)
+            .set_partial_fit_request(metadata=True),
+            X=X,
+            y=y2,
+            params=dict(metadata=[]),
+            **extra_args,
+        )
+
+
+@pytest.mark.parametrize(
+    "func, extra_args",
+    [
+        (cross_validate, {}),
+        (cross_val_score, {}),
+        (cross_val_predict, {}),
+        (learning_curve, {}),
+        (permutation_test_score, {}),
+        (validation_curve, {"param_name": "alpha", "param_range": np.array([1])}),
+    ],
+)
+@config_context(enable_metadata_routing=True)
+def test_validation_functions_routing(func, extra_args):
+    """Check that the respective cv method is properly dispatching the metadata
+    to the consumer."""
+    scorer_registry = _Registry()
+    scorer = ConsumingScorer(registry=scorer_registry).set_score_request(
+        sample_weight="score_weights", metadata="score_metadata"
+    )
+    splitter_registry = _Registry()
+    splitter = ConsumingSplitter(registry=splitter_registry).set_split_request(
+        groups="split_groups", metadata="split_metadata"
+    )
+    estimator_registry = _Registry()
+    estimator = ConsumingClassifier(registry=estimator_registry).set_fit_request(
+        sample_weight="fit_sample_weight", metadata="fit_metadata"
+    )
+
+    n_samples = _num_samples(X)
+    rng = np.random.RandomState(0)
+    score_weights = rng.rand(n_samples)
+    score_metadata = rng.rand(n_samples)
+    split_groups = rng.randint(0, 3, n_samples)
+    split_metadata = rng.rand(n_samples)
+    fit_sample_weight = rng.rand(n_samples)
+    fit_metadata = rng.rand(n_samples)
+
+    scoring_args = {
+        cross_validate: dict(scoring=dict(my_scorer=scorer, accuracy="accuracy")),
+        cross_val_score: dict(scoring=scorer),
+        learning_curve: dict(scoring=scorer),
+        validation_curve: dict(scoring=scorer),
+        permutation_test_score: dict(scoring=scorer),
+        cross_val_predict: dict(),
+    }
+
+    params = dict(
+        split_groups=split_groups,
+        split_metadata=split_metadata,
+        fit_sample_weight=fit_sample_weight,
+        fit_metadata=fit_metadata,
+    )
+
+    if func is not cross_val_predict:
+        params.update(
+            score_weights=score_weights,
+            score_metadata=score_metadata,
+        )
+
+    func(
+        estimator,
+        X=X,
+        y=y,
+        cv=splitter,
+        **scoring_args[func],
+        **extra_args,
+        params=params,
+    )
+
+    if func is not cross_val_predict:
+        # cross_val_predict doesn't need a scorer
+        assert len(scorer_registry)
+    for _scorer in scorer_registry:
+        check_recorded_metadata(
+            obj=_scorer,
+            method="score",
+            parent=func.__name__,
+            split_params=("sample_weight", "metadata"),
+            sample_weight=score_weights,
+            metadata=score_metadata,
+        )
+
+    assert len(splitter_registry)
+    for _splitter in splitter_registry:
+        check_recorded_metadata(
+            obj=_splitter,
+            method="split",
+            parent=func.__name__,
+            groups=split_groups,
+            metadata=split_metadata,
+        )
+
+    assert len(estimator_registry)
+    for _estimator in estimator_registry:
+        check_recorded_metadata(
+            obj=_estimator,
+            method="fit",
+            parent=func.__name__,
+            split_params=("sample_weight", "metadata"),
+            sample_weight=fit_sample_weight,
+            metadata=fit_metadata,
+        )
+
+
+@config_context(enable_metadata_routing=True)
+def test_learning_curve_exploit_incremental_learning_routing():
+    """Test that learning_curve routes metadata to the estimator correctly while
+    partial_fitting it with `exploit_incremental_learning=True`."""
+
+    n_samples = _num_samples(X)
+    rng = np.random.RandomState(0)
+    fit_sample_weight = rng.rand(n_samples)
+    fit_metadata = rng.rand(n_samples)
+
+    estimator_registry = _Registry()
+    estimator = ConsumingClassifier(
+        registry=estimator_registry
+    ).set_partial_fit_request(
+        sample_weight="fit_sample_weight", metadata="fit_metadata"
+    )
+
+    learning_curve(
+        estimator,
+        X=X,
+        y=y,
+        cv=ConsumingSplitter(),
+        exploit_incremental_learning=True,
+        params=dict(fit_sample_weight=fit_sample_weight, fit_metadata=fit_metadata),
+    )
+
+    assert len(estimator_registry)
+    for _estimator in estimator_registry:
+        check_recorded_metadata(
+            obj=_estimator,
+            method="partial_fit",
+            parent="learning_curve",
+            split_params=("sample_weight", "metadata"),
+            sample_weight=fit_sample_weight,
+            metadata=fit_metadata,
+        )
+
+
+# End of metadata routing tests
+# =============================
diff --git a/sklearn/multiclass.py b/sklearn/multiclass.py
index da29fdd4daf11..d4208e0f542c7 100644
--- a/sklearn/multiclass.py
+++ b/sklearn/multiclass.py
@@ -1,11 +1,8 @@
-"""
-Multiclass classification strategies
-====================================
+"""Multiclass learning algorithms.
 
-This module implements multiclass learning algorithms:
-    - one-vs-the-rest / one-vs-all
-    - one-vs-one
-    - error correcting output codes
+- one-vs-the-rest / one-vs-all
+- one-vs-one
+- error correcting output codes
 
 The estimators provided in this module are meta-estimators: they require a base
 estimator to be provided in their constructor. For example, it is possible to
@@ -28,47 +25,60 @@
 case.
 """
 
-# Author: Mathieu Blondel 
-# Author: Hamzeh Alsalhi <93hamsal@gmail.com>
-#
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import array
-import numpy as np
+import itertools
 import warnings
+from numbers import Integral, Real
+
+import numpy as np
 import scipy.sparse as sp
-import itertools
 
-from .base import BaseEstimator, ClassifierMixin, clone, is_classifier
-from .base import MultiOutputMixin
-from .base import MetaEstimatorMixin, is_regressor
-from .base import _is_pairwise
+from .base import (
+    BaseEstimator,
+    ClassifierMixin,
+    MetaEstimatorMixin,
+    MultiOutputMixin,
+    _fit_context,
+    clone,
+    is_classifier,
+    is_regressor,
+)
+from .metrics.pairwise import pairwise_distances_argmin
 from .preprocessing import LabelBinarizer
-from .metrics.pairwise import euclidean_distances
 from .utils import check_random_state
-from .utils.deprecation import deprecated
-from .utils._tags import _safe_tags
-from .utils.validation import _num_samples
-from .utils.validation import check_is_fitted
-from .utils.validation import check_X_y, check_array
-from .utils.validation import _deprecate_positional_args
-from .utils.multiclass import (_check_partial_fit_first_call,
-                               check_classification_targets,
-                               _ovr_decision_function)
-from .utils.metaestimators import _safe_split, if_delegate_has_method
-from .utils.fixes import delayed
-from .exceptions import NotFittedError
-
-from joblib import Parallel
+from .utils._param_validation import HasMethods, Interval
+from .utils._tags import get_tags
+from .utils.metadata_routing import (
+    MetadataRouter,
+    MethodMapping,
+    _raise_for_params,
+    process_routing,
+)
+from .utils.metaestimators import _safe_split, available_if
+from .utils.multiclass import (
+    _check_partial_fit_first_call,
+    _ovr_decision_function,
+    check_classification_targets,
+)
+from .utils.parallel import Parallel, delayed
+from .utils.validation import (
+    _check_method_params,
+    _num_samples,
+    check_is_fitted,
+    validate_data,
+)
 
 __all__ = [
-    "OneVsRestClassifier",
     "OneVsOneClassifier",
+    "OneVsRestClassifier",
     "OutputCodeClassifier",
 ]
 
 
-def _fit_binary(estimator, X, y, classes=None):
+def _fit_binary(estimator, X, y, fit_params, classes=None):
     """Fit a single binary estimator."""
     unique_y = np.unique(y)
     if len(unique_y) == 1:
@@ -77,18 +87,19 @@ def _fit_binary(estimator, X, y, classes=None):
                 c = 0
             else:
                 c = y[0]
-            warnings.warn("Label %s is present in all training examples." %
-                          str(classes[c]))
+            warnings.warn(
+                "Label %s is present in all training examples." % str(classes[c])
+            )
         estimator = _ConstantPredictor().fit(X, unique_y)
     else:
         estimator = clone(estimator)
-        estimator.fit(X, y)
+        estimator.fit(X, y, **fit_params)
     return estimator
 
 
-def _partial_fit_binary(estimator, X, y):
+def _partial_fit_binary(estimator, X, y, partial_fit_params):
     """Partially fit a single binary estimator."""
-    estimator.partial_fit(X, y, np.array((0, 1)))
+    estimator.partial_fit(X, y, classes=np.array((0, 1)), **partial_fit_params)
     return estimator
 
 
@@ -104,39 +115,96 @@ def _predict_binary(estimator, X):
     return score
 
 
-def _check_estimator(estimator):
-    """Make sure that an estimator implements the necessary methods."""
-    if (not hasattr(estimator, "decision_function") and
-            not hasattr(estimator, "predict_proba")):
-        raise ValueError("The base estimator should implement "
-                         "decision_function or predict_proba!")
+def _threshold_for_binary_predict(estimator):
+    """Threshold for predictions from binary estimator."""
+    if hasattr(estimator, "decision_function") and is_classifier(estimator):
+        return 0.0
+    else:
+        # predict_proba threshold
+        return 0.5
 
 
 class _ConstantPredictor(BaseEstimator):
+    """Helper predictor to be used when only one class is present."""
 
     def fit(self, X, y):
+        check_params = dict(
+            ensure_all_finite=False, dtype=None, ensure_2d=False, accept_sparse=True
+        )
+        validate_data(
+            self, X, y, reset=True, validate_separately=(check_params, check_params)
+        )
         self.y_ = y
         return self
 
     def predict(self, X):
         check_is_fitted(self)
-
-        return np.repeat(self.y_, X.shape[0])
+        validate_data(
+            self,
+            X,
+            ensure_all_finite=False,
+            dtype=None,
+            accept_sparse=True,
+            ensure_2d=False,
+            reset=False,
+        )
+
+        return np.repeat(self.y_, _num_samples(X))
 
     def decision_function(self, X):
         check_is_fitted(self)
-
-        return np.repeat(self.y_, X.shape[0])
+        validate_data(
+            self,
+            X,
+            ensure_all_finite=False,
+            dtype=None,
+            accept_sparse=True,
+            ensure_2d=False,
+            reset=False,
+        )
+
+        return np.repeat(self.y_, _num_samples(X))
 
     def predict_proba(self, X):
         check_is_fitted(self)
+        validate_data(
+            self,
+            X,
+            ensure_all_finite=False,
+            dtype=None,
+            accept_sparse=True,
+            ensure_2d=False,
+            reset=False,
+        )
+        y_ = self.y_.astype(np.float64)
+        return np.repeat([np.hstack([1 - y_, y_])], _num_samples(X), axis=0)
+
+
+def _estimators_has(attr):
+    """Check if self.estimator or self.estimators_[0] has attr.
+
+    If `self.estimators_[0]` has the attr, then its safe to assume that other
+    estimators have it too. We raise the original `AttributeError` if `attr`
+    does not exist. This function is used together with `available_if`.
+    """
+
+    def check(self):
+        if hasattr(self, "estimators_"):
+            getattr(self.estimators_[0], attr)
+        else:
+            getattr(self.estimator, attr)
+
+        return True
 
-        return np.repeat([np.hstack([1 - self.y_, self.y_])],
-                         X.shape[0], axis=0)
+    return check
 
 
-class OneVsRestClassifier(MultiOutputMixin, ClassifierMixin,
-                          MetaEstimatorMixin, BaseEstimator):
+class OneVsRestClassifier(
+    MultiOutputMixin,
+    ClassifierMixin,
+    MetaEstimatorMixin,
+    BaseEstimator,
+):
     """One-vs-the-rest (OvR) multiclass strategy.
 
     Also known as one-vs-all, this strategy consists in fitting one classifier
@@ -161,8 +229,11 @@ class OneVsRestClassifier(MultiOutputMixin, ClassifierMixin,
     Parameters
     ----------
     estimator : estimator object
-        An estimator object implementing :term:`fit` and one of
-        :term:`decision_function` or :term:`predict_proba`.
+        A regressor or a classifier that implements :term:`fit`.
+        When a classifier is passed, :term:`decision_function` will be used
+        in priority and it will fallback to :term:`predict_proba` if it is not
+        available.
+        When a regressor is passed, :term:`predict` is used.
 
     n_jobs : int, default=None
         The number of jobs to use for the computation: the `n_classes`
@@ -172,41 +243,23 @@ class OneVsRestClassifier(MultiOutputMixin, ClassifierMixin,
         ``-1`` means using all processors. See :term:`Glossary `
         for more details.
 
-        .. versionchanged:: v0.20
+        .. versionchanged:: 0.20
            `n_jobs` default changed from 1 to None
 
+    verbose : int, default=0
+        The verbosity level, if non zero, progress messages are printed.
+        Below 50, the output is sent to stderr. Otherwise, the output is sent
+        to stdout. The frequency of the messages increases with the verbosity
+        level, reporting all iterations at 10. See :class:`joblib.Parallel` for
+        more details.
+
+        .. versionadded:: 1.1
+
     Attributes
     ----------
     estimators_ : list of `n_classes` estimators
         Estimators used for predictions.
 
-    coef_ : ndarray of shape (1, n_features) or (n_classes, n_features)
-        Coefficient of the features in the decision function. This attribute
-        exists only if the ``estimators_`` defines ``coef_``.
-
-        .. deprecated:: 0.24
-            This attribute is deprecated in 0.24 and will
-            be removed in 1.1 (renaming of 0.26). If you use this attribute
-            in :class:`~sklearn.feature_selection.RFE` or
-            :class:`~sklearn.feature_selection.SelectFromModel`,
-            you may pass a callable to the `importance_getter`
-            parameter that extracts feature the importances
-            from `estimators_`.
-
-    intercept_ : ndarray of shape (1, 1) or (n_classes, 1)
-        If ``y`` is binary, the shape is ``(1, 1)`` else ``(n_classes, 1)``
-        This attribute exists only if the ``estimators_`` defines
-        ``intercept_``.
-
-        .. deprecated:: 0.24
-            This attribute is deprecated in 0.24 and will
-            be removed in 1.1 (renaming of 0.26). If you use this attribute
-            in :class:`~sklearn.feature_selection.RFE` or
-            :class:`~sklearn.feature_selection.SelectFromModel`,
-            you may pass a callable to the `importance_getter`
-            parameter that extracts feature the importances
-            from `estimators_`.
-
     classes_ : array, shape = [`n_classes`]
         Class labels.
 
@@ -220,6 +273,27 @@ class OneVsRestClassifier(MultiOutputMixin, ClassifierMixin,
     multilabel_ : boolean
         Whether a OneVsRestClassifier is a multilabel classifier.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`. Only defined if the
+        underlying estimator exposes such an attribute when fit.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Only defined if the
+        underlying estimator exposes such an attribute when fit.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    OneVsOneClassifier : One-vs-one multiclass strategy.
+    OutputCodeClassifier : (Error-Correcting) Output-Code multiclass strategy.
+    sklearn.multioutput.MultiOutputClassifier : Alternate way of extending an
+        estimator for multilabel classification.
+    sklearn.preprocessing.MultiLabelBinarizer : Transform iterable of iterables
+        to binary indicator matrix.
+
     Examples
     --------
     >>> import numpy as np
@@ -237,35 +311,56 @@ class OneVsRestClassifier(MultiOutputMixin, ClassifierMixin,
     >>> clf = OneVsRestClassifier(SVC()).fit(X, y)
     >>> clf.predict([[-19, -20], [9, 9], [-5, 5]])
     array([2, 0, 1])
-
-    See Also
-    --------
-    sklearn.multioutput.MultiOutputClassifier : Alternate way of extending an
-        estimator for multilabel classification.
-    sklearn.preprocessing.MultiLabelBinarizer : Transform iterable of iterables
-        to binary indicator matrix.
     """
-    @_deprecate_positional_args
-    def __init__(self, estimator, *, n_jobs=None):
+
+    _parameter_constraints = {
+        "estimator": [HasMethods(["fit"])],
+        "n_jobs": [Integral, None],
+        "verbose": ["verbose"],
+    }
+
+    def __init__(self, estimator, *, n_jobs=None, verbose=0):
         self.estimator = estimator
         self.n_jobs = n_jobs
+        self.verbose = verbose
 
-    def fit(self, X, y):
+    @_fit_context(
+        # OneVsRestClassifier.estimator is not validated yet
+        prefer_skip_nested_validation=False
+    )
+    def fit(self, X, y, **fit_params):
         """Fit underlying estimators.
 
         Parameters
         ----------
-        X : (sparse) array-like of shape (n_samples, n_features)
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
             Data.
 
-        y : (sparse) array-like of shape (n_samples,) or (n_samples, n_classes)
+        y : {array-like, sparse matrix} of shape (n_samples,) or (n_samples, n_classes)
             Multi-class targets. An indicator matrix turns on multilabel
             classification.
 
+        **fit_params : dict
+            Parameters passed to the ``estimator.fit`` method of each
+            sub-estimator.
+
+            .. versionadded:: 1.4
+                Only available if `enable_metadata_routing=True`. See
+                :ref:`Metadata Routing User Guide ` for more
+                details.
+
         Returns
         -------
-        self
+        self : object
+            Instance of fitted estimator.
         """
+        _raise_for_params(fit_params, self, "fit")
+
+        routed_params = process_routing(
+            self,
+            "fit",
+            **fit_params,
+        )
         # A sparse LabelBinarizer, with sparse_output=True, has been shown to
         # outperform or match a dense label binarizer in all cases and has also
         # resulted in less or equal memory consumption in the fit_ovr function
@@ -278,27 +373,44 @@ def fit(self, X, y):
         # In cases where individual estimators are very fast to train setting
         # n_jobs > 1 in can results in slower performance due to the overhead
         # of spawning threads.  See joblib issue #112.
-        self.estimators_ = Parallel(n_jobs=self.n_jobs)(delayed(_fit_binary)(
-            self.estimator, X, column, classes=[
-                "not %s" % self.label_binarizer_.classes_[i],
-                self.label_binarizer_.classes_[i]])
-            for i, column in enumerate(columns))
+        self.estimators_ = Parallel(n_jobs=self.n_jobs, verbose=self.verbose)(
+            delayed(_fit_binary)(
+                self.estimator,
+                X,
+                column,
+                fit_params=routed_params.estimator.fit,
+                classes=[
+                    "not %s" % self.label_binarizer_.classes_[i],
+                    self.label_binarizer_.classes_[i],
+                ],
+            )
+            for i, column in enumerate(columns)
+        )
+
+        if hasattr(self.estimators_[0], "n_features_in_"):
+            self.n_features_in_ = self.estimators_[0].n_features_in_
+        if hasattr(self.estimators_[0], "feature_names_in_"):
+            self.feature_names_in_ = self.estimators_[0].feature_names_in_
 
         return self
 
-    @if_delegate_has_method('estimator')
-    def partial_fit(self, X, y, classes=None):
-        """Partially fit underlying estimators
+    @available_if(_estimators_has("partial_fit"))
+    @_fit_context(
+        # OneVsRestClassifier.estimator is not validated yet
+        prefer_skip_nested_validation=False
+    )
+    def partial_fit(self, X, y, classes=None, **partial_fit_params):
+        """Partially fit underlying estimators.
 
         Should be used when memory is inefficient to train all data.
-        Chunks of data can be passed in several iteration.
+        Chunks of data can be passed in several iterations.
 
         Parameters
         ----------
-        X : (sparse) array-like of shape (n_samples, n_features)
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
             Data.
 
-        y : (sparse) array-like of shape (n_samples,) or (n_samples, n_classes)
+        y : {array-like, sparse matrix} of shape (n_samples,) or (n_samples, n_classes)
             Multi-class targets. An indicator matrix turns on multilabel
             classification.
 
@@ -309,16 +421,30 @@ def partial_fit(self, X, y, classes=None):
             This argument is only required in the first call of partial_fit
             and can be omitted in the subsequent calls.
 
+        **partial_fit_params : dict
+            Parameters passed to the ``estimator.partial_fit`` method of each
+            sub-estimator.
+
+            .. versionadded:: 1.4
+                Only available if `enable_metadata_routing=True`. See
+                :ref:`Metadata Routing User Guide ` for more
+                details.
+
         Returns
         -------
-        self
+        self : object
+            Instance of partially fitted estimator.
         """
+        _raise_for_params(partial_fit_params, self, "partial_fit")
+
+        routed_params = process_routing(
+            self,
+            "partial_fit",
+            **partial_fit_params,
+        )
+
         if _check_partial_fit_first_call(self, classes):
-            if not hasattr(self.estimator, "partial_fit"):
-                raise ValueError(("Base estimator {0}, doesn't have "
-                                  "partial_fit method").format(self.estimator))
-            self.estimators_ = [clone(self.estimator) for _ in range
-                                (self.n_classes_)]
+            self.estimators_ = [clone(self.estimator) for _ in range(self.n_classes_)]
 
             # A sparse LabelBinarizer, with sparse_output=True, has been
             # shown to outperform or match a dense label binarizer in all
@@ -328,17 +454,28 @@ def partial_fit(self, X, y, classes=None):
             self.label_binarizer_.fit(self.classes_)
 
         if len(np.setdiff1d(y, self.classes_)):
-            raise ValueError(("Mini-batch contains {0} while classes " +
-                              "must be subset of {1}").format(np.unique(y),
-                                                              self.classes_))
+            raise ValueError(
+                (
+                    "Mini-batch contains {0} while classes " + "must be subset of {1}"
+                ).format(np.unique(y), self.classes_)
+            )
 
         Y = self.label_binarizer_.transform(y)
         Y = Y.tocsc()
         columns = (col.toarray().ravel() for col in Y.T)
 
         self.estimators_ = Parallel(n_jobs=self.n_jobs)(
-            delayed(_partial_fit_binary)(estimator, X, column)
-            for estimator, column in zip(self.estimators_, columns))
+            delayed(_partial_fit_binary)(
+                estimator,
+                X,
+                column,
+                partial_fit_params=routed_params.estimator.partial_fit,
+            )
+            for estimator, column in zip(self.estimators_, columns)
+        )
+
+        if hasattr(self.estimators_[0], "n_features_in_"):
+            self.n_features_in_ = self.estimators_[0].n_features_in_
 
         return self
 
@@ -347,12 +484,12 @@ def predict(self, X):
 
         Parameters
         ----------
-        X : (sparse) array-like of shape (n_samples, n_features)
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
             Data.
 
         Returns
         -------
-        y : (sparse) array-like of shape (n_samples,) or (n_samples, n_classes)
+        y : {array-like, sparse matrix} of shape (n_samples,) or (n_samples, n_classes)
             Predicted multi-class targets.
         """
         check_is_fitted(self)
@@ -368,22 +505,19 @@ def predict(self, X):
                 argmaxima[maxima == pred] = i
             return self.classes_[argmaxima]
         else:
-            if (hasattr(self.estimators_[0], "decision_function") and
-                    is_classifier(self.estimators_[0])):
-                thresh = 0
-            else:
-                thresh = .5
-            indices = array.array('i')
-            indptr = array.array('i', [0])
+            thresh = _threshold_for_binary_predict(self.estimators_[0])
+            indices = array.array("i")
+            indptr = array.array("i", [0])
             for e in self.estimators_:
                 indices.extend(np.where(_predict_binary(e, X) > thresh)[0])
                 indptr.append(len(indices))
             data = np.ones(len(indices), dtype=int)
-            indicator = sp.csc_matrix((data, indices, indptr),
-                                      shape=(n_samples, len(self.estimators_)))
+            indicator = sp.csc_matrix(
+                (data, indices, indptr), shape=(n_samples, len(self.estimators_))
+            )
             return self.label_binarizer_.inverse_transform(indicator)
 
-    @if_delegate_has_method(['_first_estimator', 'estimator'])
+    @available_if(_estimators_has("predict_proba"))
     def predict_proba(self, X):
         """Probability estimates.
 
@@ -399,11 +533,12 @@ def predict_proba(self, X):
 
         Parameters
         ----------
-        X : array-like of shape (n_samples, n_features)
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Input data.
 
         Returns
         -------
-        T : (sparse) array-like of shape (n_samples, n_classes)
+        T : array-like of shape (n_samples, n_classes)
             Returns the probability of the sample for each class in the model,
             where classes are ordered as they are in `self.classes_`.
         """
@@ -418,24 +553,30 @@ def predict_proba(self, X):
             Y = np.concatenate(((1 - Y), Y), axis=1)
 
         if not self.multilabel_:
-            # Then, probabilities should be normalized to 1.
-            Y /= np.sum(Y, axis=1)[:, np.newaxis]
+            # Then, (nonzero) sample probability distributions should be normalized.
+            row_sums = np.sum(Y, axis=1)[:, np.newaxis]
+            np.divide(Y, row_sums, out=Y, where=row_sums != 0)
+
         return Y
 
-    @if_delegate_has_method(['_first_estimator', 'estimator'])
+    @available_if(_estimators_has("decision_function"))
     def decision_function(self, X):
-        """Returns the distance of each sample from the decision boundary for
-        each class. This can only be used with estimators which implement the
-        decision_function method.
+        """Decision function for the OneVsRestClassifier.
+
+        Return the distance of each sample from the decision boundary for each
+        class. This can only be used with estimators which implement the
+        `decision_function` method.
 
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
+            Input data.
 
         Returns
         -------
         T : array-like of shape (n_samples, n_classes) or (n_samples,) for \
             binary classification.
+            Result of calling `decision_function` on the final estimator.
 
             .. versionchanged:: 0.19
                 output shape changed to ``(n_samples,)`` to conform to
@@ -444,96 +585,78 @@ def decision_function(self, X):
         check_is_fitted(self)
         if len(self.estimators_) == 1:
             return self.estimators_[0].decision_function(X)
-        return np.array([est.decision_function(X).ravel()
-                         for est in self.estimators_]).T
+        return np.array(
+            [est.decision_function(X).ravel() for est in self.estimators_]
+        ).T
 
     @property
     def multilabel_(self):
-        """Whether this is a multilabel classifier"""
-        return self.label_binarizer_.y_type_.startswith('multilabel')
+        """Whether this is a multilabel classifier."""
+        return self.label_binarizer_.y_type_.startswith("multilabel")
 
     @property
     def n_classes_(self):
+        """Number of classes."""
         return len(self.classes_)
 
-    # TODO: Remove coef_ attribute in 1.1
-    # mypy error: Decorated property not supported
-    @deprecated("Attribute coef_ was deprecated in "  # type: ignore
-                "version 0.24 and will be removed in 1.1 (renaming of 0.26). "
-                "If you observe this warning while using RFE "
-                "or SelectFromModel, use the importance_getter "
-                "parameter instead.")
-    @property
-    def coef_(self):
-        check_is_fitted(self)
-        if not hasattr(self.estimators_[0], "coef_"):
-            raise AttributeError(
-                "Base estimator doesn't have a coef_ attribute.")
-        coefs = [e.coef_ for e in self.estimators_]
-        if sp.issparse(coefs[0]):
-            return sp.vstack(coefs)
-        return np.vstack(coefs)
-
-    # TODO: Remove intercept_ attribute in 1.1
-    # mypy error: Decorated property not supported
-    @deprecated("Attribute intercept_ was deprecated in "  # type: ignore
-                "version 0.24 and will be removed in 1.1 (renaming of 0.26). "
-                "If you observe this warning while using RFE "
-                "or SelectFromModel, use the importance_getter "
-                "parameter instead.")
-    @property
-    def intercept_(self):
-        check_is_fitted(self)
-        if not hasattr(self.estimators_[0], "intercept_"):
-            raise AttributeError(
-                "Base estimator doesn't have an intercept_ attribute.")
-        return np.array([e.intercept_.ravel() for e in self.estimators_])
-
-    # TODO: Remove in 1.1
-    # mypy error: Decorated property not supported
-    @deprecated("Attribute _pairwise was deprecated in "  # type: ignore
-                "version 0.24 and will be removed in 1.1 (renaming of 0.26).")
-    @property
-    def _pairwise(self):
+    def __sklearn_tags__(self):
         """Indicate if wrapped estimator is using a precomputed Gram matrix"""
-        return getattr(self.estimator, "_pairwise", False)
+        tags = super().__sklearn_tags__()
+        tags.input_tags.pairwise = get_tags(self.estimator).input_tags.pairwise
+        tags.input_tags.sparse = get_tags(self.estimator).input_tags.sparse
+        return tags
 
-    def _more_tags(self):
-        """Indicate if wrapped estimator is using a precomputed Gram matrix"""
-        return {'pairwise': _safe_tags(self.estimator, key="pairwise")}
+    def get_metadata_routing(self):
+        """Get metadata routing of this object.
 
-    @property
-    def _first_estimator(self):
-        return self.estimators_[0]
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
 
-    @property
-    def n_features_in_(self):
-        # For consistency with other estimators we raise a AttributeError so
-        # that hasattr() fails if the OVR estimator isn't fitted.
-        try:
-            check_is_fitted(self)
-        except NotFittedError as nfe:
-            raise AttributeError(
-                "{} object has no n_features_in_ attribute."
-                .format(self.__class__.__name__)
-            ) from nfe
-        return self.estimators_[0].n_features_in_
-
-
-def _fit_ovo_binary(estimator, X, y, i, j):
+        .. versionadded:: 1.4
+
+        Returns
+        -------
+        routing : MetadataRouter
+            A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
+            routing information.
+        """
+
+        router = (
+            MetadataRouter(owner=self.__class__.__name__)
+            .add_self_request(self)
+            .add(
+                estimator=self.estimator,
+                method_mapping=MethodMapping()
+                .add(caller="fit", callee="fit")
+                .add(caller="partial_fit", callee="partial_fit"),
+            )
+        )
+        return router
+
+
+def _fit_ovo_binary(estimator, X, y, i, j, fit_params):
     """Fit a single binary estimator (one-vs-one)."""
     cond = np.logical_or(y == i, y == j)
     y = y[cond]
     y_binary = np.empty(y.shape, int)
     y_binary[y == i] = 0
     y_binary[y == j] = 1
-    indcond = np.arange(X.shape[0])[cond]
-    return _fit_binary(estimator,
-                       _safe_split(estimator, X, None, indices=indcond)[0],
-                       y_binary, classes=[i, j]), indcond
-
-
-def _partial_fit_ovo_binary(estimator, X, y, i, j):
+    indcond = np.arange(_num_samples(X))[cond]
+
+    fit_params_subset = _check_method_params(X, params=fit_params, indices=indcond)
+    return (
+        _fit_binary(
+            estimator,
+            _safe_split(estimator, X, None, indices=indcond)[0],
+            y_binary,
+            fit_params=fit_params_subset,
+            classes=[i, j],
+        ),
+        indcond,
+    )
+
+
+def _partial_fit_ovo_binary(estimator, X, y, i, j, partial_fit_params):
     """Partially fit a single binary estimator(one-vs-one)."""
 
     cond = np.logical_or(y == i, y == j)
@@ -541,12 +664,17 @@ def _partial_fit_ovo_binary(estimator, X, y, i, j):
     if len(y) != 0:
         y_binary = np.zeros_like(y)
         y_binary[y == j] = 1
-        return _partial_fit_binary(estimator, X[cond], y_binary)
+        partial_fit_params_subset = _check_method_params(
+            X, params=partial_fit_params, indices=cond
+        )
+        return _partial_fit_binary(
+            estimator, X[cond], y_binary, partial_fit_params=partial_fit_params_subset
+        )
     return estimator
 
 
 class OneVsOneClassifier(MetaEstimatorMixin, ClassifierMixin, BaseEstimator):
-    """One-vs-one multiclass strategy
+    """One-vs-one multiclass strategy.
 
     This strategy consists in fitting one classifier per class pair.
     At prediction time, the class which received the most votes is selected.
@@ -563,8 +691,11 @@ class OneVsOneClassifier(MetaEstimatorMixin, ClassifierMixin, BaseEstimator):
     Parameters
     ----------
     estimator : estimator object
-        An estimator object implementing :term:`fit` and one of
-        :term:`decision_function` or :term:`predict_proba`.
+        A regressor or a classifier that implements :term:`fit`.
+        When a classifier is passed, :term:`decision_function` will be used
+        in priority and it will fallback to :term:`predict_proba` if it is not
+        available.
+        When a regressor is passed, :term:`predict` is used.
 
     n_jobs : int, default=None
         The number of jobs to use for the computation: the `n_classes * (
@@ -583,17 +714,27 @@ class OneVsOneClassifier(MetaEstimatorMixin, ClassifierMixin, BaseEstimator):
         Array containing labels.
 
     n_classes_ : int
-        Number of classes
+        Number of classes.
 
     pairwise_indices_ : list, length = ``len(estimators_)``, or ``None``
         Indices of samples used when training the estimators.
         ``None`` when ``estimator``'s `pairwise` tag is False.
 
-        .. deprecated:: 0.24
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
 
-            The _pairwise attribute is deprecated in 0.24. From 1.1
-            (renaming of 0.25) and onward, `pairwise_indices_` will use the
-            pairwise estimator tag instead.
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    OneVsRestClassifier : One-vs-all multiclass strategy.
+    OutputCodeClassifier : (Error-Correcting) Output-Code multiclass strategy.
 
     Examples
     --------
@@ -609,60 +750,106 @@ class OneVsOneClassifier(MetaEstimatorMixin, ClassifierMixin, BaseEstimator):
     >>> clf.predict(X_test[:10])
     array([2, 1, 0, 2, 0, 2, 0, 1, 1, 1])
     """
-    @_deprecate_positional_args
+
+    _parameter_constraints: dict = {
+        "estimator": [HasMethods(["fit"])],
+        "n_jobs": [Integral, None],
+    }
+
     def __init__(self, estimator, *, n_jobs=None):
         self.estimator = estimator
         self.n_jobs = n_jobs
 
-    def fit(self, X, y):
+    @_fit_context(
+        # OneVsOneClassifier.estimator is not validated yet
+        prefer_skip_nested_validation=False
+    )
+    def fit(self, X, y, **fit_params):
         """Fit underlying estimators.
 
         Parameters
         ----------
-        X : (sparse) array-like of shape (n_samples, n_features)
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
             Data.
 
         y : array-like of shape (n_samples,)
             Multi-class targets.
 
+        **fit_params : dict
+            Parameters passed to the ``estimator.fit`` method of each
+            sub-estimator.
+
+            .. versionadded:: 1.4
+                Only available if `enable_metadata_routing=True`. See
+                :ref:`Metadata Routing User Guide ` for more
+                details.
+
         Returns
         -------
-        self
+        self : object
+            The fitted underlying estimator.
         """
-        X, y = self._validate_data(X, y, accept_sparse=['csr', 'csc'],
-                                   force_all_finite=False)
+        _raise_for_params(fit_params, self, "fit")
+
+        routed_params = process_routing(
+            self,
+            "fit",
+            **fit_params,
+        )
+
+        # We need to validate the data because we do a safe_indexing later.
+        X, y = validate_data(
+            self, X, y, accept_sparse=["csr", "csc"], ensure_all_finite=False
+        )
         check_classification_targets(y)
 
         self.classes_ = np.unique(y)
         if len(self.classes_) == 1:
-            raise ValueError("OneVsOneClassifier can not be fit when only one"
-                             " class is present.")
+            raise ValueError(
+                "OneVsOneClassifier can not be fit when only one class is present."
+            )
         n_classes = self.classes_.shape[0]
-        estimators_indices = list(zip(*(Parallel(n_jobs=self.n_jobs)(
-            delayed(_fit_ovo_binary)
-            (self.estimator, X, y, self.classes_[i], self.classes_[j])
-            for i in range(n_classes) for j in range(i + 1, n_classes)))))
+        estimators_indices = list(
+            zip(
+                *(
+                    Parallel(n_jobs=self.n_jobs)(
+                        delayed(_fit_ovo_binary)(
+                            self.estimator,
+                            X,
+                            y,
+                            self.classes_[i],
+                            self.classes_[j],
+                            fit_params=routed_params.estimator.fit,
+                        )
+                        for i in range(n_classes)
+                        for j in range(i + 1, n_classes)
+                    )
+                )
+            )
+        )
 
         self.estimators_ = estimators_indices[0]
 
-        pairwise = _is_pairwise(self)
-        self.pairwise_indices_ = (
-            estimators_indices[1] if pairwise else None)
+        pairwise = self.__sklearn_tags__().input_tags.pairwise
+        self.pairwise_indices_ = estimators_indices[1] if pairwise else None
 
         return self
 
-    @if_delegate_has_method(delegate='estimator')
-    def partial_fit(self, X, y, classes=None):
-        """Partially fit underlying estimators
+    @available_if(_estimators_has("partial_fit"))
+    @_fit_context(
+        # OneVsOneClassifier.estimator is not validated yet
+        prefer_skip_nested_validation=False
+    )
+    def partial_fit(self, X, y, classes=None, **partial_fit_params):
+        """Partially fit underlying estimators.
 
         Should be used when memory is inefficient to train all data. Chunks
         of data can be passed in several iteration, where the first call
         should have an array of all target variables.
 
-
         Parameters
         ----------
-        X : (sparse) array-like of shape (n_samples, n_features)
+        X : {array-like, sparse matrix) of shape (n_samples, n_features)
             Data.
 
         y : array-like of shape (n_samples,)
@@ -675,33 +862,69 @@ def partial_fit(self, X, y, classes=None):
             This argument is only required in the first call of partial_fit
             and can be omitted in the subsequent calls.
 
+        **partial_fit_params : dict
+            Parameters passed to the ``estimator.partial_fit`` method of each
+            sub-estimator.
+
+            .. versionadded:: 1.4
+                Only available if `enable_metadata_routing=True`. See
+                :ref:`Metadata Routing User Guide ` for more
+                details.
+
         Returns
         -------
-        self
+        self : object
+            The partially fitted underlying estimator.
         """
-        if _check_partial_fit_first_call(self, classes):
-            self.estimators_ = [clone(self.estimator) for _ in
-                                range(self.n_classes_ *
-                                      (self.n_classes_ - 1) // 2)]
+        _raise_for_params(partial_fit_params, self, "partial_fit")
 
-        if len(np.setdiff1d(y, self.classes_)):
-            raise ValueError("Mini-batch contains {0} while it "
-                             "must be subset of {1}".format(np.unique(y),
-                                                            self.classes_))
+        routed_params = process_routing(
+            self,
+            "partial_fit",
+            **partial_fit_params,
+        )
+
+        first_call = _check_partial_fit_first_call(self, classes)
+        if first_call:
+            self.estimators_ = [
+                clone(self.estimator)
+                for _ in range(self.n_classes_ * (self.n_classes_ - 1) // 2)
+            ]
 
-        X, y = check_X_y(X, y, accept_sparse=['csr', 'csc'],
-                         force_all_finite=False)
+        if len(np.setdiff1d(y, self.classes_)):
+            raise ValueError(
+                "Mini-batch contains {0} while it must be subset of {1}".format(
+                    np.unique(y), self.classes_
+                )
+            )
+
+        X, y = validate_data(
+            self,
+            X,
+            y,
+            accept_sparse=["csr", "csc"],
+            ensure_all_finite=False,
+            reset=first_call,
+        )
         check_classification_targets(y)
         combinations = itertools.combinations(range(self.n_classes_), 2)
-        self.estimators_ = Parallel(
-            n_jobs=self.n_jobs)(
-                delayed(_partial_fit_ovo_binary)(
-                    estimator, X, y, self.classes_[i], self.classes_[j])
-                for estimator, (i, j) in zip(self.estimators_,
-                                             (combinations)))
+        self.estimators_ = Parallel(n_jobs=self.n_jobs)(
+            delayed(_partial_fit_ovo_binary)(
+                estimator,
+                X,
+                y,
+                self.classes_[i],
+                self.classes_[j],
+                partial_fit_params=routed_params.estimator.partial_fit,
+            )
+            for estimator, (i, j) in zip(self.estimators_, (combinations))
+        )
 
         self.pairwise_indices_ = None
 
+        if hasattr(self.estimators_[0], "n_features_in_"):
+            self.n_features_in_ = self.estimators_[0].n_features_in_
+
         return self
 
     def predict(self, X):
@@ -713,7 +936,7 @@ def predict(self, X):
 
         Parameters
         ----------
-        X : (sparse) array-like of shape (n_samples, n_features)
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
             Data.
 
         Returns
@@ -723,7 +946,8 @@ def predict(self, X):
         """
         Y = self.decision_function(X)
         if self.n_classes_ == 2:
-            return self.classes_[(Y > 0).astype(int)]
+            thresh = _threshold_for_binary_predict(self.estimators_[0])
+            return self.classes_[(Y > thresh).astype(int)]
         return self.classes_[Y.argmax(axis=1)]
 
     def decision_function(self, X):
@@ -737,17 +961,25 @@ def decision_function(self, X):
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
+            Input data.
 
         Returns
         -------
-        Y : array-like of shape (n_samples, n_classes) or (n_samples,) for \
-            binary classification.
+        Y : array-like of shape (n_samples, n_classes) or (n_samples,)
+            Result of calling `decision_function` on the final estimator.
 
             .. versionchanged:: 0.19
                 output shape changed to ``(n_samples,)`` to conform to
                 scikit-learn conventions for binary classification.
         """
         check_is_fitted(self)
+        X = validate_data(
+            self,
+            X,
+            accept_sparse=True,
+            ensure_all_finite=False,
+            reset=False,
+        )
 
         indices = self.pairwise_indices_
         if indices is None:
@@ -755,38 +987,59 @@ def decision_function(self, X):
         else:
             Xs = [X[:, idx] for idx in indices]
 
-        predictions = np.vstack([est.predict(Xi)
-                                 for est, Xi in zip(self.estimators_, Xs)]).T
-        confidences = np.vstack([_predict_binary(est, Xi)
-                                 for est, Xi in zip(self.estimators_, Xs)]).T
-        Y = _ovr_decision_function(predictions,
-                                   confidences, len(self.classes_))
+        predictions = np.vstack(
+            [est.predict(Xi) for est, Xi in zip(self.estimators_, Xs)]
+        ).T
+        confidences = np.vstack(
+            [_predict_binary(est, Xi) for est, Xi in zip(self.estimators_, Xs)]
+        ).T
+        Y = _ovr_decision_function(predictions, confidences, len(self.classes_))
         if self.n_classes_ == 2:
             return Y[:, 1]
         return Y
 
     @property
     def n_classes_(self):
+        """Number of classes."""
         return len(self.classes_)
 
-    # TODO: Remove in 1.1
-    # mypy error: Decorated property not supported
-    @deprecated("Attribute _pairwise was deprecated in "  # type: ignore
-                "version 0.24 and will be removed in 1.1 (renaming of 0.26).")
-    @property
-    def _pairwise(self):
+    def __sklearn_tags__(self):
         """Indicate if wrapped estimator is using a precomputed Gram matrix"""
-        return getattr(self.estimator, "_pairwise", False)
+        tags = super().__sklearn_tags__()
+        tags.input_tags.pairwise = get_tags(self.estimator).input_tags.pairwise
+        tags.input_tags.sparse = get_tags(self.estimator).input_tags.sparse
+        return tags
 
-    def _more_tags(self):
-        """Indicate if wrapped estimator is using a precomputed Gram matrix"""
-        return {
-            'pairwise': _safe_tags(self.estimator, key="pairwise")
-        }
+    def get_metadata_routing(self):
+        """Get metadata routing of this object.
+
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
+
+        .. versionadded:: 1.4
+
+        Returns
+        -------
+        routing : MetadataRouter
+            A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
+            routing information.
+        """
+
+        router = (
+            MetadataRouter(owner=self.__class__.__name__)
+            .add_self_request(self)
+            .add(
+                estimator=self.estimator,
+                method_mapping=MethodMapping()
+                .add(caller="fit", callee="fit")
+                .add(caller="partial_fit", callee="partial_fit"),
+            )
+        )
+        return router
 
 
 class OutputCodeClassifier(MetaEstimatorMixin, ClassifierMixin, BaseEstimator):
-    """(Error-Correcting) Output-Code multiclass strategy
+    """(Error-Correcting) Output-Code multiclass strategy.
 
     Output-code based strategies consist in representing each class with a
     binary code (an array of 0s and 1s). At fitting time, one binary
@@ -794,8 +1047,8 @@ class OutputCodeClassifier(MetaEstimatorMixin, ClassifierMixin, BaseEstimator):
     classifiers are used to project new points in the class space and the class
     closest to the points is chosen. The main advantage of these strategies is
     that the number of classifiers used can be controlled by the user, either
-    for compressing the model (0 < code_size < 1) or for making the model more
-    robust to errors (code_size > 1). See the documentation for more details.
+    for compressing the model (0 < `code_size` < 1) or for making the model more
+    robust to errors (`code_size` > 1). See the documentation for more details.
 
     Read more in the :ref:`User Guide `.
 
@@ -805,7 +1058,7 @@ class OutputCodeClassifier(MetaEstimatorMixin, ClassifierMixin, BaseEstimator):
         An estimator object implementing :term:`fit` and one of
         :term:`decision_function` or :term:`predict_proba`.
 
-    code_size : float
+    code_size : float, default=1.5
         Percentage of the number of classes to be used to create the code book.
         A number between 0 and 1 will require fewer classifiers than
         one-vs-the-rest. A number greater than 1 will require more classifiers
@@ -829,25 +1082,28 @@ class OutputCodeClassifier(MetaEstimatorMixin, ClassifierMixin, BaseEstimator):
     estimators_ : list of `int(n_classes * code_size)` estimators
         Estimators used for predictions.
 
-    classes_ : numpy array of shape [n_classes]
+    classes_ : ndarray of shape (n_classes,)
         Array containing labels.
 
-    code_book_ : numpy array of shape [n_classes, code_size]
+    code_book_ : ndarray of shape (n_classes, `len(estimators_)`)
         Binary array containing the code of each class.
 
-    Examples
+    n_features_in_ : int
+        Number of features seen during :term:`fit`. Only defined if the
+        underlying estimator exposes such an attribute when fit.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Only defined if the
+        underlying estimator exposes such an attribute when fit.
+
+        .. versionadded:: 1.0
+
+    See Also
     --------
-    >>> from sklearn.multiclass import OutputCodeClassifier
-    >>> from sklearn.ensemble import RandomForestClassifier
-    >>> from sklearn.datasets import make_classification
-    >>> X, y = make_classification(n_samples=100, n_features=4,
-    ...                            n_informative=2, n_redundant=0,
-    ...                            random_state=0, shuffle=False)
-    >>> clf = OutputCodeClassifier(
-    ...     estimator=RandomForestClassifier(random_state=0),
-    ...     random_state=0).fit(X, y)
-    >>> clf.predict([[0, 0, 0, 0]])
-    array([1])
+    OneVsRestClassifier : One-vs-all multiclass strategy.
+    OneVsOneClassifier : One-vs-one multiclass strategy.
 
     References
     ----------
@@ -866,61 +1122,116 @@ class OutputCodeClassifier(MetaEstimatorMixin, ClassifierMixin, BaseEstimator):
     .. [3] "The Elements of Statistical Learning",
        Hastie T., Tibshirani R., Friedman J., page 606 (second-edition)
        2008.
+
+    Examples
+    --------
+    >>> from sklearn.multiclass import OutputCodeClassifier
+    >>> from sklearn.ensemble import RandomForestClassifier
+    >>> from sklearn.datasets import make_classification
+    >>> X, y = make_classification(n_samples=100, n_features=4,
+    ...                            n_informative=2, n_redundant=0,
+    ...                            random_state=0, shuffle=False)
+    >>> clf = OutputCodeClassifier(
+    ...     estimator=RandomForestClassifier(random_state=0),
+    ...     random_state=0).fit(X, y)
+    >>> clf.predict([[0, 0, 0, 0]])
+    array([1])
     """
-    @_deprecate_positional_args
-    def __init__(self, estimator, *, code_size=1.5, random_state=None,
-                 n_jobs=None):
+
+    _parameter_constraints: dict = {
+        "estimator": [
+            HasMethods(["fit", "decision_function"]),
+            HasMethods(["fit", "predict_proba"]),
+        ],
+        "code_size": [Interval(Real, 0.0, None, closed="neither")],
+        "random_state": ["random_state"],
+        "n_jobs": [Integral, None],
+    }
+
+    def __init__(self, estimator, *, code_size=1.5, random_state=None, n_jobs=None):
         self.estimator = estimator
         self.code_size = code_size
         self.random_state = random_state
         self.n_jobs = n_jobs
 
-    def fit(self, X, y):
+    @_fit_context(
+        # OutputCodeClassifier.estimator is not validated yet
+        prefer_skip_nested_validation=False
+    )
+    def fit(self, X, y, **fit_params):
         """Fit underlying estimators.
 
         Parameters
         ----------
-        X : (sparse) array-like of shape (n_samples, n_features)
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
             Data.
 
-        y : numpy array of shape [n_samples]
+        y : array-like of shape (n_samples,)
             Multi-class targets.
 
+        **fit_params : dict
+            Parameters passed to the ``estimator.fit`` method of each
+            sub-estimator.
+
+            .. versionadded:: 1.4
+                Only available if `enable_metadata_routing=True`. See
+                :ref:`Metadata Routing User Guide ` for more
+                details.
+
         Returns
         -------
-        self
+        self : object
+            Returns a fitted instance of self.
         """
-        X, y = self._validate_data(X, y, accept_sparse=True)
-        if self.code_size <= 0:
-            raise ValueError("code_size should be greater than 0, got {0}"
-                             "".format(self.code_size))
+        _raise_for_params(fit_params, self, "fit")
+
+        routed_params = process_routing(
+            self,
+            "fit",
+            **fit_params,
+        )
+
+        y = validate_data(self, X="no_validation", y=y)
 
-        _check_estimator(self.estimator)
         random_state = check_random_state(self.random_state)
         check_classification_targets(y)
 
         self.classes_ = np.unique(y)
         n_classes = self.classes_.shape[0]
-        code_size_ = int(n_classes * self.code_size)
+        if n_classes == 0:
+            raise ValueError(
+                "OutputCodeClassifier can not be fit when no class is present."
+            )
+        n_estimators = int(n_classes * self.code_size)
 
         # FIXME: there are more elaborate methods than generating the codebook
         # randomly.
-        self.code_book_ = random_state.random_sample((n_classes, code_size_))
-        self.code_book_[self.code_book_ > 0.5] = 1
+        self.code_book_ = random_state.uniform(size=(n_classes, n_estimators))
+        self.code_book_[self.code_book_ > 0.5] = 1.0
 
         if hasattr(self.estimator, "decision_function"):
-            self.code_book_[self.code_book_ != 1] = -1
+            self.code_book_[self.code_book_ != 1] = -1.0
         else:
-            self.code_book_[self.code_book_ != 1] = 0
+            self.code_book_[self.code_book_ != 1] = 0.0
 
         classes_index = {c: i for i, c in enumerate(self.classes_)}
 
-        Y = np.array([self.code_book_[classes_index[y[i]]]
-                      for i in range(X.shape[0])], dtype=int)
+        Y = np.array(
+            [self.code_book_[classes_index[y[i]]] for i in range(_num_samples(y))],
+            dtype=int,
+        )
 
         self.estimators_ = Parallel(n_jobs=self.n_jobs)(
-            delayed(_fit_binary)(self.estimator, X, Y[:, i])
-            for i in range(Y.shape[1]))
+            delayed(_fit_binary)(
+                self.estimator, X, Y[:, i], fit_params=routed_params.estimator.fit
+            )
+            for i in range(Y.shape[1])
+        )
+
+        if hasattr(self.estimators_[0], "n_features_in_"):
+            self.n_features_in_ = self.estimators_[0].n_features_in_
+        if hasattr(self.estimators_[0], "feature_names_in_"):
+            self.feature_names_in_ = self.estimators_[0].feature_names_in_
 
         return self
 
@@ -929,16 +1240,48 @@ def predict(self, X):
 
         Parameters
         ----------
-        X : (sparse) array-like of shape (n_samples, n_features)
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
             Data.
 
         Returns
         -------
-        y : numpy array of shape [n_samples]
+        y : ndarray of shape (n_samples,)
             Predicted multi-class targets.
         """
         check_is_fitted(self)
-        X = check_array(X, accept_sparse=True)
-        Y = np.array([_predict_binary(e, X) for e in self.estimators_]).T
-        pred = euclidean_distances(Y, self.code_book_).argmin(axis=1)
+        # ArgKmin only accepts C-contiguous array. The aggregated predictions need to be
+        # transposed. We therefore create a F-contiguous array to avoid a copy and have
+        # a C-contiguous array after the transpose operation.
+        Y = np.array(
+            [_predict_binary(e, X) for e in self.estimators_],
+            order="F",
+            dtype=np.float64,
+        ).T
+        pred = pairwise_distances_argmin(Y, self.code_book_, metric="euclidean")
         return self.classes_[pred]
+
+    def get_metadata_routing(self):
+        """Get metadata routing of this object.
+
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
+
+        .. versionadded:: 1.4
+
+        Returns
+        -------
+        routing : MetadataRouter
+            A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
+            routing information.
+        """
+
+        router = MetadataRouter(owner=self.__class__.__name__).add(
+            estimator=self.estimator,
+            method_mapping=MethodMapping().add(caller="fit", callee="fit"),
+        )
+        return router
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = get_tags(self.estimator).input_tags.sparse
+        return tags
diff --git a/sklearn/multioutput.py b/sklearn/multioutput.py
index 9987c01b13187..08b0c95c94558 100644
--- a/sklearn/multioutput.py
+++ b/sklearn/multioutput.py
@@ -1,36 +1,62 @@
-"""
-This module implements multioutput regression and classification.
+"""Multioutput regression and classification.
 
 The estimators provided in this module are meta-estimators: they require
 a base estimator to be provided in their constructor. The meta-estimator
 extends single output estimators to multioutput estimators.
 """
 
-# Author: Tim Head 
-# Author: Hugo Bowne-Anderson 
-# Author: Chris Rivera 
-# Author: Michael Williamson
-# Author: James Ashton Nichols 
-#
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+import warnings
+from abc import ABCMeta, abstractmethod
+from numbers import Integral
 
 import numpy as np
 import scipy.sparse as sp
-from joblib import Parallel
 
-from abc import ABCMeta, abstractmethod
-from .base import BaseEstimator, clone, MetaEstimatorMixin
-from .base import RegressorMixin, ClassifierMixin, is_classifier
+from .base import (
+    BaseEstimator,
+    ClassifierMixin,
+    MetaEstimatorMixin,
+    RegressorMixin,
+    _fit_context,
+    clone,
+    is_classifier,
+)
 from .model_selection import cross_val_predict
-from .utils import check_array, check_X_y, check_random_state
-from .utils.metaestimators import if_delegate_has_method
-from .utils.validation import (check_is_fitted, has_fit_parameter,
-                               _check_fit_params, _deprecate_positional_args)
+from .utils import Bunch, check_random_state, get_tags
+from .utils._param_validation import (
+    HasMethods,
+    Hidden,
+    StrOptions,
+)
+from .utils._response import _get_response_values
+from .utils._user_interface import _print_elapsed_time
+from .utils.metadata_routing import (
+    MetadataRouter,
+    MethodMapping,
+    _raise_for_params,
+    _routing_enabled,
+    process_routing,
+)
+from .utils.metaestimators import available_if
 from .utils.multiclass import check_classification_targets
-from .utils.fixes import delayed
-
-__all__ = ["MultiOutputRegressor", "MultiOutputClassifier",
-           "ClassifierChain", "RegressorChain"]
+from .utils.parallel import Parallel, delayed
+from .utils.validation import (
+    _check_method_params,
+    _check_response_method,
+    check_is_fitted,
+    has_fit_parameter,
+    validate_data,
+)
+
+__all__ = [
+    "ClassifierChain",
+    "MultiOutputClassifier",
+    "MultiOutputRegressor",
+    "RegressorChain",
+]
 
 
 def _fit_estimator(estimator, X, y, sample_weight=None, **fit_params):
@@ -42,104 +68,165 @@ def _fit_estimator(estimator, X, y, sample_weight=None, **fit_params):
     return estimator
 
 
-def _partial_fit_estimator(estimator, X, y, classes=None, sample_weight=None,
-                           first_time=True):
+def _partial_fit_estimator(
+    estimator, X, y, classes=None, partial_fit_params=None, first_time=True
+):
+    partial_fit_params = {} if partial_fit_params is None else partial_fit_params
     if first_time:
         estimator = clone(estimator)
 
-    if sample_weight is not None:
-        if classes is not None:
-            estimator.partial_fit(X, y, classes=classes,
-                                  sample_weight=sample_weight)
-        else:
-            estimator.partial_fit(X, y, sample_weight=sample_weight)
+    if classes is not None:
+        estimator.partial_fit(X, y, classes=classes, **partial_fit_params)
     else:
-        if classes is not None:
-            estimator.partial_fit(X, y, classes=classes)
-        else:
-            estimator.partial_fit(X, y)
+        estimator.partial_fit(X, y, **partial_fit_params)
     return estimator
 
 
-class _MultiOutputEstimator(MetaEstimatorMixin,
-                            BaseEstimator,
-                            metaclass=ABCMeta):
+def _available_if_estimator_has(attr):
+    """Return a function to check if the sub-estimator(s) has(have) `attr`.
+
+    Helper for Chain implementations.
+    """
+
+    def _check(self):
+        if hasattr(self, "estimators_"):
+            return all(hasattr(est, attr) for est in self.estimators_)
+
+        if hasattr(self.estimator, attr):
+            return True
+
+        return False
+
+    return available_if(_check)
+
+
+class _MultiOutputEstimator(MetaEstimatorMixin, BaseEstimator, metaclass=ABCMeta):
+    _parameter_constraints: dict = {
+        "estimator": [HasMethods(["fit", "predict"])],
+        "n_jobs": [Integral, None],
+    }
+
     @abstractmethod
-    @_deprecate_positional_args
     def __init__(self, estimator, *, n_jobs=None):
         self.estimator = estimator
         self.n_jobs = n_jobs
 
-    @if_delegate_has_method('estimator')
-    def partial_fit(self, X, y, classes=None, sample_weight=None):
-        """Incrementally fit the model to data.
-        Fit a separate model for each output variable.
+    @_available_if_estimator_has("partial_fit")
+    @_fit_context(
+        # MultiOutput*.estimator is not validated yet
+        prefer_skip_nested_validation=False
+    )
+    def partial_fit(self, X, y, classes=None, sample_weight=None, **partial_fit_params):
+        """Incrementally fit a separate model for each class output.
 
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
-            Data.
+            The input data.
 
         y : {array-like, sparse matrix} of shape (n_samples, n_outputs)
             Multi-output targets.
 
-        classes : list of ndarray of shape (n_outputs,)
-            Each array is unique classes for one output in str/int
-            Can be obtained by via
-            ``[np.unique(y[:, i]) for i in range(y.shape[1])]``, where y is the
-            target matrix of the entire dataset.
+        classes : list of ndarray of shape (n_outputs,), default=None
+            Each array is unique classes for one output in str/int.
+            Can be obtained via
+            ``[np.unique(y[:, i]) for i in range(y.shape[1])]``, where `y`
+            is the target matrix of the entire dataset.
             This argument is required for the first call to partial_fit
             and can be omitted in the subsequent calls.
-            Note that y doesn't need to contain all labels in `classes`.
+            Note that `y` doesn't need to contain all labels in `classes`.
 
         sample_weight : array-like of shape (n_samples,), default=None
-            Sample weights. If None, then samples are equally weighted.
+            Sample weights. If `None`, then samples are equally weighted.
             Only supported if the underlying regressor supports sample
             weights.
 
+        **partial_fit_params : dict of str -> object
+            Parameters passed to the ``estimator.partial_fit`` method of each
+            sub-estimator.
+
+            Only available if `enable_metadata_routing=True`. See the
+            :ref:`User Guide `.
+
+            .. versionadded:: 1.3
+
         Returns
         -------
         self : object
+            Returns a fitted instance.
         """
-        X, y = check_X_y(X, y,
-                         force_all_finite=False,
-                         multi_output=True,
-                         accept_sparse=True)
+        _raise_for_params(partial_fit_params, self, "partial_fit")
 
-        if y.ndim == 1:
-            raise ValueError("y must have at least two dimensions for "
-                             "multi-output regression but has only one.")
+        first_time = not hasattr(self, "estimators_")
 
-        if (sample_weight is not None and
-                not has_fit_parameter(self.estimator, 'sample_weight')):
-            raise ValueError("Underlying estimator does not support"
-                             " sample weights.")
+        y = validate_data(self, X="no_validation", y=y, multi_output=True)
 
-        first_time = not hasattr(self, 'estimators_')
+        if y.ndim == 1:
+            raise ValueError(
+                "y must have at least two dimensions for "
+                "multi-output regression but has only one."
+            )
+
+        if _routing_enabled():
+            if sample_weight is not None:
+                partial_fit_params["sample_weight"] = sample_weight
+            routed_params = process_routing(
+                self,
+                "partial_fit",
+                **partial_fit_params,
+            )
+        else:
+            if sample_weight is not None and not has_fit_parameter(
+                self.estimator, "sample_weight"
+            ):
+                raise ValueError(
+                    "Underlying estimator does not support sample weights."
+                )
+
+            if sample_weight is not None:
+                routed_params = Bunch(
+                    estimator=Bunch(partial_fit=Bunch(sample_weight=sample_weight))
+                )
+            else:
+                routed_params = Bunch(estimator=Bunch(partial_fit=Bunch()))
 
         self.estimators_ = Parallel(n_jobs=self.n_jobs)(
             delayed(_partial_fit_estimator)(
                 self.estimators_[i] if not first_time else self.estimator,
-                X, y[:, i],
+                X,
+                y[:, i],
                 classes[i] if classes is not None else None,
-                sample_weight, first_time) for i in range(y.shape[1]))
+                partial_fit_params=routed_params.estimator.partial_fit,
+                first_time=first_time,
+            )
+            for i in range(y.shape[1])
+        )
+
+        if first_time and hasattr(self.estimators_[0], "n_features_in_"):
+            self.n_features_in_ = self.estimators_[0].n_features_in_
+        if first_time and hasattr(self.estimators_[0], "feature_names_in_"):
+            self.feature_names_in_ = self.estimators_[0].feature_names_in_
+
         return self
 
+    @_fit_context(
+        # MultiOutput*.estimator is not validated yet
+        prefer_skip_nested_validation=False
+    )
     def fit(self, X, y, sample_weight=None, **fit_params):
-        """ Fit the model to data.
-        Fit a separate model for each output variable.
+        """Fit the model to data, separately for each output variable.
 
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
-            Data.
+            The input data.
 
         y : {array-like, sparse matrix} of shape (n_samples, n_outputs)
             Multi-output targets. An indicator matrix turns on multilabel
             estimation.
 
         sample_weight : array-like of shape (n_samples,), default=None
-            Sample weights. If None, then samples are equally weighted.
+            Sample weights. If `None`, then samples are equally weighted.
             Only supported if the underlying regressor supports sample
             weights.
 
@@ -151,45 +238,64 @@ def fit(self, X, y, sample_weight=None, **fit_params):
         Returns
         -------
         self : object
+            Returns a fitted instance.
         """
-
         if not hasattr(self.estimator, "fit"):
-            raise ValueError("The base estimator should implement"
-                             " a fit method")
+            raise ValueError("The base estimator should implement a fit method")
 
-        X, y = self._validate_data(X, y,
-                                   force_all_finite=False,
-                                   multi_output=True, accept_sparse=True)
+        y = validate_data(self, X="no_validation", y=y, multi_output=True)
 
         if is_classifier(self):
             check_classification_targets(y)
 
         if y.ndim == 1:
-            raise ValueError("y must have at least two dimensions for "
-                             "multi-output regression but has only one.")
-
-        if (sample_weight is not None and
-                not has_fit_parameter(self.estimator, 'sample_weight')):
-            raise ValueError("Underlying estimator does not support"
-                             " sample weights.")
-
-        fit_params_validated = _check_fit_params(X, fit_params)
+            raise ValueError(
+                "y must have at least two dimensions for "
+                "multi-output regression but has only one."
+            )
+
+        if _routing_enabled():
+            if sample_weight is not None:
+                fit_params["sample_weight"] = sample_weight
+            routed_params = process_routing(
+                self,
+                "fit",
+                **fit_params,
+            )
+        else:
+            if sample_weight is not None and not has_fit_parameter(
+                self.estimator, "sample_weight"
+            ):
+                raise ValueError(
+                    "Underlying estimator does not support sample weights."
+                )
+
+            fit_params_validated = _check_method_params(X, params=fit_params)
+            routed_params = Bunch(estimator=Bunch(fit=fit_params_validated))
+            if sample_weight is not None:
+                routed_params.estimator.fit["sample_weight"] = sample_weight
 
         self.estimators_ = Parallel(n_jobs=self.n_jobs)(
             delayed(_fit_estimator)(
-                self.estimator, X, y[:, i], sample_weight,
-                **fit_params_validated)
-            for i in range(y.shape[1]))
+                self.estimator, X, y[:, i], **routed_params.estimator.fit
+            )
+            for i in range(y.shape[1])
+        )
+
+        if hasattr(self.estimators_[0], "n_features_in_"):
+            self.n_features_in_ = self.estimators_[0].n_features_in_
+        if hasattr(self.estimators_[0], "feature_names_in_"):
+            self.feature_names_in_ = self.estimators_[0].feature_names_in_
+
         return self
 
     def predict(self, X):
-        """Predict multi-output variable using a model
-         trained for each target variable.
+        """Predict multi-output variable using model for each target variable.
 
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
-            Data.
+            The input data.
 
         Returns
         -------
@@ -198,24 +304,47 @@ def predict(self, X):
             Note: Separate models are generated for each predictor.
         """
         check_is_fitted(self)
-        if not hasattr(self.estimator, "predict"):
-            raise ValueError("The base estimator should implement"
-                             " a predict method")
-
-        X = check_array(X, force_all_finite=False, accept_sparse=True)
+        if not hasattr(self.estimators_[0], "predict"):
+            raise ValueError("The base estimator should implement a predict method")
 
         y = Parallel(n_jobs=self.n_jobs)(
-            delayed(e.predict)(X)
-            for e in self.estimators_)
+            delayed(e.predict)(X) for e in self.estimators_
+        )
 
         return np.asarray(y).T
 
-    def _more_tags(self):
-        return {'multioutput_only': True}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = get_tags(self.estimator).input_tags.sparse
+        tags.target_tags.single_output = False
+        tags.target_tags.multi_output = True
+        return tags
+
+    def get_metadata_routing(self):
+        """Get metadata routing of this object.
+
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
+
+        .. versionadded:: 1.3
+
+        Returns
+        -------
+        routing : MetadataRouter
+            A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
+            routing information.
+        """
+        router = MetadataRouter(owner=self.__class__.__name__).add(
+            estimator=self.estimator,
+            method_mapping=MethodMapping()
+            .add(caller="partial_fit", callee="partial_fit")
+            .add(caller="fit", callee="fit"),
+        )
+        return router
 
 
 class MultiOutputRegressor(RegressorMixin, _MultiOutputEstimator):
-    """Multi target regression
+    """Multi target regression.
 
     This strategy consists of fitting one regressor per target. This is a
     simple strategy for extending regressors that do not natively support
@@ -237,18 +366,37 @@ class MultiOutputRegressor(RegressorMixin, _MultiOutputEstimator):
         using ``n_jobs > 1`` can result in slower performance due
         to the parallelism overhead.
 
-        ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
+        ``None`` means `1` unless in a :obj:`joblib.parallel_backend` context.
         ``-1`` means using all available processes / threads.
         See :term:`Glossary ` for more details.
 
         .. versionchanged:: 0.20
-           `n_jobs` default changed from 1 to None
+            `n_jobs` default changed from `1` to `None`.
 
     Attributes
     ----------
     estimators_ : list of ``n_output`` estimators
         Estimators used for predictions.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`. Only defined if the
+        underlying `estimator` exposes such an attribute when fit.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Only defined if the
+        underlying estimators expose such an attribute when fit.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    RegressorChain : A multi-label model that arranges regressions into a
+        chain.
+    MultiOutputClassifier : Classifies each output independently rather than
+        chaining.
+
     Examples
     --------
     >>> import numpy as np
@@ -256,52 +404,61 @@ class MultiOutputRegressor(RegressorMixin, _MultiOutputEstimator):
     >>> from sklearn.multioutput import MultiOutputRegressor
     >>> from sklearn.linear_model import Ridge
     >>> X, y = load_linnerud(return_X_y=True)
-    >>> clf = MultiOutputRegressor(Ridge(random_state=123)).fit(X, y)
-    >>> clf.predict(X[[0]])
-    array([[176..., 35..., 57...]])
+    >>> regr = MultiOutputRegressor(Ridge(random_state=123)).fit(X, y)
+    >>> regr.predict(X[[0]])
+    array([[176, 35.1, 57.1]])
     """
-    @_deprecate_positional_args
+
     def __init__(self, estimator, *, n_jobs=None):
         super().__init__(estimator, n_jobs=n_jobs)
 
-    @if_delegate_has_method('estimator')
-    def partial_fit(self, X, y, sample_weight=None):
-        """Incrementally fit the model to data.
-        Fit a separate model for each output variable.
+    @_available_if_estimator_has("partial_fit")
+    def partial_fit(self, X, y, sample_weight=None, **partial_fit_params):
+        """Incrementally fit the model to data, for each output variable.
 
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
-            Data.
+            The input data.
 
         y : {array-like, sparse matrix} of shape (n_samples, n_outputs)
             Multi-output targets.
 
         sample_weight : array-like of shape (n_samples,), default=None
-            Sample weights. If None, then samples are equally weighted.
+            Sample weights. If `None`, then samples are equally weighted.
             Only supported if the underlying regressor supports sample
             weights.
 
+        **partial_fit_params : dict of str -> object
+            Parameters passed to the ``estimator.partial_fit`` method of each
+            sub-estimator.
+
+            Only available if `enable_metadata_routing=True`. See the
+            :ref:`User Guide `.
+
+            .. versionadded:: 1.3
+
         Returns
         -------
         self : object
+            Returns a fitted instance.
         """
-        super().partial_fit(
-            X, y, sample_weight=sample_weight)
+        super().partial_fit(X, y, sample_weight=sample_weight, **partial_fit_params)
 
 
 class MultiOutputClassifier(ClassifierMixin, _MultiOutputEstimator):
-    """Multi target classification
+    """Multi target classification.
 
     This strategy consists of fitting one classifier per target. This is a
     simple strategy for extending classifiers that do not natively support
-    multi-target classification
+    multi-target classification.
 
     Parameters
     ----------
     estimator : estimator object
-        An estimator object implementing :term:`fit`, :term:`score` and
-        :term:`predict_proba`.
+        An estimator object implementing :term:`fit` and :term:`predict`.
+        A :term:`predict_proba` method will be exposed only if `estimator` implements
+        it.
 
     n_jobs : int or None, optional (default=None)
         The number of jobs to run in parallel.
@@ -312,12 +469,12 @@ class MultiOutputClassifier(ClassifierMixin, _MultiOutputEstimator):
         using ``n_jobs > 1`` can result in slower performance due
         to the parallelism overhead.
 
-        ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
+        ``None`` means `1` unless in a :obj:`joblib.parallel_backend` context.
         ``-1`` means using all available processes / threads.
         See :term:`Glossary ` for more details.
 
         .. versionchanged:: 0.20
-           `n_jobs` default changed from 1 to None
+            `n_jobs` default changed from `1` to `None`.
 
     Attributes
     ----------
@@ -327,19 +484,37 @@ class MultiOutputClassifier(ClassifierMixin, _MultiOutputEstimator):
     estimators_ : list of ``n_output`` estimators
         Estimators used for predictions.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`. Only defined if the
+        underlying `estimator` exposes such an attribute when fit.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Only defined if the
+        underlying estimators expose such an attribute when fit.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    ClassifierChain : A multi-label model that arranges binary classifiers
+        into a chain.
+    MultiOutputRegressor : Fits one regressor per target variable.
+
     Examples
     --------
     >>> import numpy as np
     >>> from sklearn.datasets import make_multilabel_classification
     >>> from sklearn.multioutput import MultiOutputClassifier
-    >>> from sklearn.neighbors import KNeighborsClassifier
-
+    >>> from sklearn.linear_model import LogisticRegression
     >>> X, y = make_multilabel_classification(n_classes=3, random_state=0)
-    >>> clf = MultiOutputClassifier(KNeighborsClassifier()).fit(X, y)
+    >>> clf = MultiOutputClassifier(LogisticRegression()).fit(X, y)
     >>> clf.predict(X[-2:])
-    array([[1, 1, 0], [1, 1, 1]])
+    array([[1, 1, 1],
+           [1, 0, 1]])
     """
-    @_deprecate_positional_args
+
     def __init__(self, estimator, *, n_jobs=None):
         super().__init__(estimator, n_jobs=n_jobs)
 
@@ -350,12 +525,15 @@ def fit(self, X, Y, sample_weight=None, **fit_params):
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
             The input data.
+
         Y : array-like of shape (n_samples, n_classes)
             The target values.
+
         sample_weight : array-like of shape (n_samples,), default=None
-            Sample weights. If None, then samples are equally weighted.
+            Sample weights. If `None`, then samples are equally weighted.
             Only supported if the underlying classifier supports sample
             weights.
+
         **fit_params : dict of string -> object
             Parameters passed to the ``estimator.fit`` method of each step.
 
@@ -364,15 +542,26 @@ def fit(self, X, Y, sample_weight=None, **fit_params):
         Returns
         -------
         self : object
+            Returns a fitted instance.
         """
-        super().fit(X, Y, sample_weight, **fit_params)
+        super().fit(X, Y, sample_weight=sample_weight, **fit_params)
         self.classes_ = [estimator.classes_ for estimator in self.estimators_]
         return self
 
-    @property
-    def predict_proba(self):
-        """Probability estimates.
-        Returns prediction probabilities for each class of each output.
+    def _check_predict_proba(self):
+        if hasattr(self, "estimators_"):
+            # raise an AttributeError if `predict_proba` does not exist for
+            # each estimator
+            [getattr(est, "predict_proba") for est in self.estimators_]
+            return True
+        # raise an AttributeError if `predict_proba` does not exist for the
+        # unfitted estimator
+        getattr(self.estimator, "predict_proba")
+        return True
+
+    @available_if(_check_predict_proba)
+    def predict_proba(self, X):
+        """Return prediction probabilities for each class of each output.
 
         This method will raise a ``ValueError`` if any of the
         estimators do not have ``predict_proba``.
@@ -380,12 +569,12 @@ def predict_proba(self):
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
-            Data
+            The input data.
 
         Returns
         -------
         p : array of shape (n_samples, n_classes), or a list of n_outputs \
-            such arrays if n_outputs > 1.
+                such arrays if n_outputs > 1.
             The class probabilities of the input samples. The order of the
             classes corresponds to that in the attribute :term:`classes_`.
 
@@ -395,58 +584,161 @@ def predict_proba(self):
                 ``n_classes``) for that particular output.
         """
         check_is_fitted(self)
-        if not all([hasattr(estimator, "predict_proba")
-                    for estimator in self.estimators_]):
-            raise AttributeError("The base estimator should "
-                                 "implement predict_proba method")
-        return self._predict_proba
-
-    def _predict_proba(self, X):
-        results = [estimator.predict_proba(X) for estimator in
-                   self.estimators_]
+        results = [estimator.predict_proba(X) for estimator in self.estimators_]
         return results
 
     def score(self, X, y):
-        """Returns the mean accuracy on the given test data and labels.
+        """Return the mean accuracy on the given test data and labels.
 
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
-            Test samples
+            Test samples.
 
         y : array-like of shape (n_samples, n_outputs)
-            True values for X
+            True values for X.
 
         Returns
         -------
         scores : float
-            accuracy_score of self.predict(X) versus y
+            Mean accuracy of predicted target versus true target.
         """
         check_is_fitted(self)
         n_outputs_ = len(self.estimators_)
         if y.ndim == 1:
-            raise ValueError("y must have at least two dimensions for "
-                             "multi target classification but has only one")
+            raise ValueError(
+                "y must have at least two dimensions for "
+                "multi target classification but has only one"
+            )
         if y.shape[1] != n_outputs_:
-            raise ValueError("The number of outputs of Y for fit {0} and"
-                             " score {1} should be same".
-                             format(n_outputs_, y.shape[1]))
+            raise ValueError(
+                "The number of outputs of Y for fit {0} and"
+                " score {1} should be same".format(n_outputs_, y.shape[1])
+            )
         y_pred = self.predict(X)
         return np.mean(np.all(y == y_pred, axis=1))
 
-    def _more_tags(self):
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
         # FIXME
-        return {'_skip_test': True}
+        tags._skip_test = True
+        return tags
+
+
+def _available_if_base_estimator_has(attr):
+    """Return a function to check if `base_estimator` or `estimators_` has `attr`.
+
+    Helper for Chain implementations.
+    """
+
+    def _check(self):
+        return hasattr(self._get_estimator(), attr) or all(
+            hasattr(est, attr) for est in self.estimators_
+        )
+
+    return available_if(_check)
 
 
 class _BaseChain(BaseEstimator, metaclass=ABCMeta):
-    @_deprecate_positional_args
-    def __init__(self, base_estimator, *, order=None, cv=None,
-                 random_state=None):
+    _parameter_constraints: dict = {
+        "base_estimator": [
+            HasMethods(["fit", "predict"]),
+            StrOptions({"deprecated"}),
+        ],
+        "estimator": [
+            HasMethods(["fit", "predict"]),
+            Hidden(None),
+        ],
+        "order": ["array-like", StrOptions({"random"}), None],
+        "cv": ["cv_object", StrOptions({"prefit"})],
+        "random_state": ["random_state"],
+        "verbose": ["boolean"],
+    }
+
+    # TODO(1.9): Remove base_estimator
+    def __init__(
+        self,
+        estimator=None,
+        *,
+        order=None,
+        cv=None,
+        random_state=None,
+        verbose=False,
+        base_estimator="deprecated",
+    ):
+        self.estimator = estimator
         self.base_estimator = base_estimator
         self.order = order
         self.cv = cv
         self.random_state = random_state
+        self.verbose = verbose
+
+    # TODO(1.8): This is a temporary getter method to validate input wrt deprecation.
+    # It was only included to avoid relying on the presence of self.estimator_
+    def _get_estimator(self):
+        """Get and validate estimator."""
+
+        if self.estimator is not None and (self.base_estimator != "deprecated"):
+            raise ValueError(
+                "Both `estimator` and `base_estimator` are provided. You should only"
+                " pass `estimator`. `base_estimator` as a parameter is deprecated in"
+                " version 1.7, and will be removed in version 1.9."
+            )
+
+        if self.base_estimator != "deprecated":
+            warning_msg = (
+                "`base_estimator` as an argument was deprecated in 1.7 and will be"
+                " removed in 1.9. Use `estimator` instead."
+            )
+            warnings.warn(warning_msg, FutureWarning)
+            return self.base_estimator
+        else:
+            return self.estimator
+
+    def _log_message(self, *, estimator_idx, n_estimators, processing_msg):
+        if not self.verbose:
+            return None
+        return f"({estimator_idx} of {n_estimators}) {processing_msg}"
+
+    def _get_predictions(self, X, *, output_method):
+        """Get predictions for each model in the chain."""
+        check_is_fitted(self)
+        X = validate_data(self, X, accept_sparse=True, reset=False)
+        Y_output_chain = np.zeros((X.shape[0], len(self.estimators_)))
+        Y_feature_chain = np.zeros((X.shape[0], len(self.estimators_)))
+
+        # `RegressorChain` does not have a `chain_method_` parameter so we
+        # default to "predict"
+        chain_method = getattr(self, "chain_method_", "predict")
+        hstack = sp.hstack if sp.issparse(X) else np.hstack
+        for chain_idx, estimator in enumerate(self.estimators_):
+            previous_predictions = Y_feature_chain[:, :chain_idx]
+            # if `X` is a scipy sparse dok_array, we convert it to a sparse
+            # coo_array format before hstacking, it's faster; see
+            # https://github.com/scipy/scipy/issues/20060#issuecomment-1937007039:
+            if sp.issparse(X) and not sp.isspmatrix(X) and X.format == "dok":
+                X = sp.coo_array(X)
+            X_aug = hstack((X, previous_predictions))
+
+            feature_predictions, _ = _get_response_values(
+                estimator,
+                X_aug,
+                response_method=chain_method,
+            )
+            Y_feature_chain[:, chain_idx] = feature_predictions
+
+            output_predictions, _ = _get_response_values(
+                estimator,
+                X_aug,
+                response_method=output_method,
+            )
+            Y_output_chain[:, chain_idx] = output_predictions
+
+        inv_order = np.empty_like(self.order_)
+        inv_order[self.order_] = np.arange(len(self.order_))
+        Y_output = Y_output_chain[:, inv_order]
+
+        return Y_output
 
     @abstractmethod
     def fit(self, X, Y, **fit_params):
@@ -456,8 +748,10 @@ def fit(self, X, Y, **fit_params):
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
             The input data.
+
         Y : array-like of shape (n_samples, n_classes)
             The target values.
+
         **fit_params : dict of string -> object
             Parameters passed to the `fit` method of each step.
 
@@ -466,11 +760,11 @@ def fit(self, X, Y, **fit_params):
         Returns
         -------
         self : object
+            Returns a fitted instance.
         """
-        X, Y = self._validate_data(X, Y, multi_output=True, accept_sparse=True)
+        X, Y = validate_data(self, X, Y, multi_output=True, accept_sparse=True)
 
         random_state = check_random_state(self.random_state)
-        check_array(X, accept_sparse=True)
         self.order_ = self.order
         if isinstance(self.order_, tuple):
             self.order_ = np.array(self.order_)
@@ -478,25 +772,36 @@ def fit(self, X, Y, **fit_params):
         if self.order_ is None:
             self.order_ = np.array(range(Y.shape[1]))
         elif isinstance(self.order_, str):
-            if self.order_ == 'random':
+            if self.order_ == "random":
                 self.order_ = random_state.permutation(Y.shape[1])
         elif sorted(self.order_) != list(range(Y.shape[1])):
             raise ValueError("invalid order")
 
-        self.estimators_ = [clone(self.base_estimator)
-                            for _ in range(Y.shape[1])]
+        self.estimators_ = [clone(self._get_estimator()) for _ in range(Y.shape[1])]
 
         if self.cv is None:
             Y_pred_chain = Y[:, self.order_]
             if sp.issparse(X):
-                X_aug = sp.hstack((X, Y_pred_chain), format='lil')
+                X_aug = sp.hstack((X, Y_pred_chain), format="lil")
                 X_aug = X_aug.tocsr()
             else:
                 X_aug = np.hstack((X, Y_pred_chain))
 
         elif sp.issparse(X):
-            Y_pred_chain = sp.lil_matrix((X.shape[0], Y.shape[1]))
-            X_aug = sp.hstack((X, Y_pred_chain), format='lil')
+            # TODO: remove this condition check when the minimum supported scipy version
+            # doesn't support sparse matrices anymore
+            if not sp.isspmatrix(X):
+                # if `X` is a scipy sparse dok_array, we convert it to a sparse
+                # coo_array format before hstacking, it's faster; see
+                # https://github.com/scipy/scipy/issues/20060#issuecomment-1937007039:
+                if X.format == "dok":
+                    X = sp.coo_array(X)
+                # in case that `X` is a sparse array we create `Y_pred_chain` as a
+                # sparse array format:
+                Y_pred_chain = sp.coo_array((X.shape[0], Y.shape[1]))
+            else:
+                Y_pred_chain = sp.coo_matrix((X.shape[0], Y.shape[1]))
+            X_aug = sp.hstack((X, Y_pred_chain), format="lil")
 
         else:
             Y_pred_chain = np.zeros((X.shape[0], Y.shape[1]))
@@ -504,15 +809,47 @@ def fit(self, X, Y, **fit_params):
 
         del Y_pred_chain
 
+        if _routing_enabled():
+            routed_params = process_routing(self, "fit", **fit_params)
+        else:
+            routed_params = Bunch(estimator=Bunch(fit=fit_params))
+
+        if hasattr(self, "chain_method"):
+            chain_method = _check_response_method(
+                self._get_estimator(),
+                self.chain_method,
+            ).__name__
+            self.chain_method_ = chain_method
+        else:
+            # `RegressorChain` does not have a `chain_method` parameter
+            chain_method = "predict"
+
         for chain_idx, estimator in enumerate(self.estimators_):
+            message = self._log_message(
+                estimator_idx=chain_idx + 1,
+                n_estimators=len(self.estimators_),
+                processing_msg=f"Processing order {self.order_[chain_idx]}",
+            )
             y = Y[:, self.order_[chain_idx]]
-            estimator.fit(X_aug[:, :(X.shape[1] + chain_idx)], y,
-                          **fit_params)
+            with _print_elapsed_time("Chain", message):
+                estimator.fit(
+                    X_aug[:, : (X.shape[1] + chain_idx)],
+                    y,
+                    **routed_params.estimator.fit,
+                )
+
             if self.cv is not None and chain_idx < len(self.estimators_) - 1:
                 col_idx = X.shape[1] + chain_idx
                 cv_result = cross_val_predict(
-                    self.base_estimator, X_aug[:, :col_idx],
-                    y=y, cv=self.cv)
+                    self._get_estimator(),
+                    X_aug[:, :col_idx],
+                    y=y,
+                    cv=self.cv,
+                    method=chain_method,
+                )
+                # `predict_proba` output is 2D, we use only output for classes[-1]
+                if cv_result.ndim > 1:
+                    cv_result = cv_result[:, 1]
                 if sp.issparse(X_aug):
                     X_aug[:, col_idx] = np.expand_dims(cv_result, 1)
                 else:
@@ -532,27 +869,13 @@ def predict(self, X):
         -------
         Y_pred : array-like of shape (n_samples, n_classes)
             The predicted values.
-
         """
-        check_is_fitted(self)
-        X = check_array(X, accept_sparse=True)
-        Y_pred_chain = np.zeros((X.shape[0], len(self.estimators_)))
-        for chain_idx, estimator in enumerate(self.estimators_):
-            previous_predictions = Y_pred_chain[:, :chain_idx]
-            if sp.issparse(X):
-                if chain_idx == 0:
-                    X_aug = X
-                else:
-                    X_aug = sp.hstack((X, previous_predictions))
-            else:
-                X_aug = np.hstack((X, previous_predictions))
-            Y_pred_chain[:, chain_idx] = estimator.predict(X_aug)
+        return self._get_predictions(X, output_method="predict")
 
-        inv_order = np.empty_like(self.order_)
-        inv_order[self.order_] = np.arange(len(self.order_))
-        Y_pred = Y_pred_chain[:, inv_order]
-
-        return Y_pred
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = get_tags(self._get_estimator()).input_tags.sparse
+        return tags
 
 
 class ClassifierChain(MetaEstimatorMixin, ClassifierMixin, _BaseChain):
@@ -562,17 +885,22 @@ class ClassifierChain(MetaEstimatorMixin, ClassifierMixin, _BaseChain):
     all of the available features provided to the model plus the predictions
     of models that are earlier in the chain.
 
+    For an example of how to use ``ClassifierChain`` and benefit from its
+    ensemble, see
+    :ref:`ClassifierChain on a yeast dataset
+    ` example.
+
     Read more in the :ref:`User Guide `.
 
     .. versionadded:: 0.19
 
     Parameters
     ----------
-    base_estimator : estimator
+    estimator : estimator
         The base estimator from which the classifier chain is built.
 
     order : array-like of shape (n_outputs,) or 'random', default=None
-        If None, the order will be determined by the order of columns in
+        If `None`, the order will be determined by the order of columns in
         the label matrix Y.::
 
             order = [0, 1, 2, ..., Y.shape[1] - 1]
@@ -586,7 +914,7 @@ class ClassifierChain(MetaEstimatorMixin, ClassifierMixin, _BaseChain):
         column 1 in the Y matrix, the second model will make predictions
         for column 3, etc.
 
-        If order is 'random' a random ordering will be used.
+        If order is `random` a random ordering will be used.
 
     cv : int, cross-validation generator or an iterable, default=None
         Determines whether to use cross validated predictions or true
@@ -598,6 +926,19 @@ class ClassifierChain(MetaEstimatorMixin, ClassifierMixin, _BaseChain):
         - :term:`CV splitter`,
         - An iterable yielding (train, test) splits as arrays of indices.
 
+    chain_method : {'predict', 'predict_proba', 'predict_log_proba', \
+            'decision_function'} or list of such str's, default='predict'
+
+        Prediction method to be used by estimators in the chain for
+        the 'prediction' features of previous estimators in the chain.
+
+        - if `str`, name of the method;
+        - if a list of `str`, provides the method names in order of
+          preference. The method used corresponds to the first method in
+          the list that is implemented by `base_estimator`.
+
+        .. versionadded:: 1.5
+
     random_state : int, RandomState instance or None, optional (default=None)
         If ``order='random'``, determines random number generation for the
         chain order.
@@ -607,6 +948,18 @@ class ClassifierChain(MetaEstimatorMixin, ClassifierMixin, _BaseChain):
         Pass an int for reproducible output across multiple function calls.
         See :term:`Glossary `.
 
+    verbose : bool, default=False
+        If True, chain progress is output as each model is completed.
+
+        .. versionadded:: 1.2
+
+    base_estimator : estimator, default="deprecated"
+        Use `estimator` instead.
+
+        .. deprecated:: 1.7
+            `base_estimator` is deprecated and will be removed in 1.9.
+            Use `estimator` instead.
+
     Attributes
     ----------
     classes_ : list
@@ -619,6 +972,33 @@ class labels for each estimator in the chain.
     order_ : list
         The order of labels in the classifier chain.
 
+    chain_method_ : str
+        Prediction method used by estimators in the chain for the prediction
+        features.
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`. Only defined if the
+        underlying `base_estimator` exposes such an attribute when fit.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    RegressorChain : Equivalent for regression.
+    MultiOutputClassifier : Classifies each output independently rather than
+        chaining.
+
+    References
+    ----------
+    Jesse Read, Bernhard Pfahringer, Geoff Holmes, Eibe Frank, "Classifier
+    Chains for Multi-label Classification", 2009.
+
     Examples
     --------
     >>> from sklearn.datasets import make_multilabel_classification
@@ -638,78 +1018,117 @@ class labels for each estimator in the chain.
            [1., 0., 0.],
            [0., 1., 0.]])
     >>> chain.predict_proba(X_test)
-    array([[0.8387..., 0.9431..., 0.4576...],
-           [0.8878..., 0.3684..., 0.2640...],
-           [0.0321..., 0.9935..., 0.0625...]])
-
-    See Also
-    --------
-    RegressorChain : Equivalent for regression.
-    MultioutputClassifier : Classifies each output independently rather than
-        chaining.
-
-    References
-    ----------
-    Jesse Read, Bernhard Pfahringer, Geoff Holmes, Eibe Frank, "Classifier
-    Chains for Multi-label Classification", 2009.
+    array([[0.8387, 0.9431, 0.4576],
+           [0.8878, 0.3684, 0.2640],
+           [0.0321, 0.9935, 0.0626]])
     """
 
-    def fit(self, X, Y):
+    _parameter_constraints: dict = {
+        **_BaseChain._parameter_constraints,
+        "chain_method": [
+            list,
+            tuple,
+            StrOptions(
+                {"predict", "predict_proba", "predict_log_proba", "decision_function"}
+            ),
+        ],
+    }
+
+    # TODO(1.9): Remove base_estimator from __init__
+    def __init__(
+        self,
+        estimator=None,
+        *,
+        order=None,
+        cv=None,
+        chain_method="predict",
+        random_state=None,
+        verbose=False,
+        base_estimator="deprecated",
+    ):
+        super().__init__(
+            estimator,
+            order=order,
+            cv=cv,
+            random_state=random_state,
+            verbose=verbose,
+            base_estimator=base_estimator,
+        )
+        self.chain_method = chain_method
+
+    @_fit_context(
+        # ClassifierChain.base_estimator is not validated yet
+        prefer_skip_nested_validation=False
+    )
+    def fit(self, X, Y, **fit_params):
         """Fit the model to data matrix X and targets Y.
 
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
             The input data.
+
         Y : array-like of shape (n_samples, n_classes)
             The target values.
 
+        **fit_params : dict of string -> object
+            Parameters passed to the `fit` method of each step.
+
+            Only available if `enable_metadata_routing=True`. See the
+            :ref:`User Guide `.
+
+            .. versionadded:: 1.3
+
         Returns
         -------
         self : object
+            Class instance.
         """
-        super().fit(X, Y)
-        self.classes_ = [estimator.classes_
-                         for chain_idx, estimator
-                         in enumerate(self.estimators_)]
+        _raise_for_params(fit_params, self, "fit")
+
+        super().fit(X, Y, **fit_params)
+        self.classes_ = [estimator.classes_ for estimator in self.estimators_]
         return self
 
-    @if_delegate_has_method('base_estimator')
+    @_available_if_base_estimator_has("predict_proba")
     def predict_proba(self, X):
         """Predict probability estimates.
 
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            The input data.
 
         Returns
         -------
         Y_prob : array-like of shape (n_samples, n_classes)
+            The predicted probabilities.
         """
-        X = check_array(X, accept_sparse=True)
-        Y_prob_chain = np.zeros((X.shape[0], len(self.estimators_)))
-        Y_pred_chain = np.zeros((X.shape[0], len(self.estimators_)))
-        for chain_idx, estimator in enumerate(self.estimators_):
-            previous_predictions = Y_pred_chain[:, :chain_idx]
-            if sp.issparse(X):
-                X_aug = sp.hstack((X, previous_predictions))
-            else:
-                X_aug = np.hstack((X, previous_predictions))
-            Y_prob_chain[:, chain_idx] = estimator.predict_proba(X_aug)[:, 1]
-            Y_pred_chain[:, chain_idx] = estimator.predict(X_aug)
-        inv_order = np.empty_like(self.order_)
-        inv_order[self.order_] = np.arange(len(self.order_))
-        Y_prob = Y_prob_chain[:, inv_order]
+        return self._get_predictions(X, output_method="predict_proba")
 
-        return Y_prob
+    def predict_log_proba(self, X):
+        """Predict logarithm of probability estimates.
 
-    @if_delegate_has_method('base_estimator')
+        Parameters
+        ----------
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            The input data.
+
+        Returns
+        -------
+        Y_log_prob : array-like of shape (n_samples, n_classes)
+            The predicted logarithm of the probabilities.
+        """
+        return np.log(self.predict_proba(X))
+
+    @_available_if_base_estimator_has("decision_function")
     def decision_function(self, X):
         """Evaluate the decision_function of the models in the chain.
 
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
+            The input data.
 
         Returns
         -------
@@ -717,26 +1136,36 @@ def decision_function(self, X):
             Returns the decision function of the sample for each model
             in the chain.
         """
-        Y_decision_chain = np.zeros((X.shape[0], len(self.estimators_)))
-        Y_pred_chain = np.zeros((X.shape[0], len(self.estimators_)))
-        for chain_idx, estimator in enumerate(self.estimators_):
-            previous_predictions = Y_pred_chain[:, :chain_idx]
-            if sp.issparse(X):
-                X_aug = sp.hstack((X, previous_predictions))
-            else:
-                X_aug = np.hstack((X, previous_predictions))
-            Y_decision_chain[:, chain_idx] = estimator.decision_function(X_aug)
-            Y_pred_chain[:, chain_idx] = estimator.predict(X_aug)
+        return self._get_predictions(X, output_method="decision_function")
 
-        inv_order = np.empty_like(self.order_)
-        inv_order[self.order_] = np.arange(len(self.order_))
-        Y_decision = Y_decision_chain[:, inv_order]
+    def get_metadata_routing(self):
+        """Get metadata routing of this object.
+
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
 
-        return Y_decision
+        .. versionadded:: 1.3
 
-    def _more_tags(self):
-        return {'_skip_test': True,
-                'multioutput_only': True}
+        Returns
+        -------
+        routing : MetadataRouter
+            A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
+            routing information.
+        """
+
+        router = MetadataRouter(owner=self.__class__.__name__).add(
+            estimator=self._get_estimator(),
+            method_mapping=MethodMapping().add(caller="fit", callee="fit"),
+        )
+        return router
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        # FIXME
+        tags._skip_test = True
+        tags.target_tags.single_output = False
+        tags.target_tags.multi_output = True
+        return tags
 
 
 class RegressorChain(MetaEstimatorMixin, RegressorMixin, _BaseChain):
@@ -752,11 +1181,11 @@ class RegressorChain(MetaEstimatorMixin, RegressorMixin, _BaseChain):
 
     Parameters
     ----------
-    base_estimator : estimator
-        The base estimator from which the classifier chain is built.
+    estimator : estimator
+        The base estimator from which the regressor chain is built.
 
     order : array-like of shape (n_outputs,) or 'random', default=None
-        If None, the order will be determined by the order of columns in
+        If `None`, the order will be determined by the order of columns in
         the label matrix Y.::
 
             order = [0, 1, 2, ..., Y.shape[1] - 1]
@@ -791,6 +1220,18 @@ class RegressorChain(MetaEstimatorMixin, RegressorMixin, _BaseChain):
         Pass an int for reproducible output across multiple function calls.
         See :term:`Glossary `.
 
+    verbose : bool, default=False
+        If True, chain progress is output as each model is completed.
+
+        .. versionadded:: 1.2
+
+    base_estimator : estimator, default="deprecated"
+        Use `estimator` instead.
+
+        .. deprecated:: 1.7
+            `base_estimator` is deprecated and will be removed in 1.9.
+            Use `estimator` instead.
+
     Attributes
     ----------
     estimators_ : list
@@ -799,26 +1240,41 @@ class RegressorChain(MetaEstimatorMixin, RegressorMixin, _BaseChain):
     order_ : list
         The order of labels in the classifier chain.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`. Only defined if the
+        underlying `base_estimator` exposes such an attribute when fit.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    ClassifierChain : Equivalent for classification.
+    MultiOutputRegressor : Learns each output independently rather than
+        chaining.
+
     Examples
     --------
     >>> from sklearn.multioutput import RegressorChain
     >>> from sklearn.linear_model import LogisticRegression
-    >>> logreg = LogisticRegression(solver='lbfgs',multi_class='multinomial')
+    >>> logreg = LogisticRegression(solver='lbfgs')
     >>> X, Y = [[1, 0], [0, 1], [1, 1]], [[0, 2], [1, 1], [2, 0]]
-    >>> chain = RegressorChain(base_estimator=logreg, order=[0, 1]).fit(X, Y)
+    >>> chain = RegressorChain(logreg, order=[0, 1]).fit(X, Y)
     >>> chain.predict(X)
     array([[0., 2.],
            [1., 1.],
            [2., 0.]])
-
-    See Also
-    --------
-    ClassifierChain : Equivalent for classification.
-    MultioutputRegressor : Learns each output independently rather than
-        chaining.
-
     """
 
+    @_fit_context(
+        # RegressorChain.base_estimator is not validated yet
+        prefer_skip_nested_validation=False
+    )
     def fit(self, X, Y, **fit_params):
         """Fit the model to data matrix X and targets Y.
 
@@ -826,6 +1282,7 @@ def fit(self, X, Y, **fit_params):
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
             The input data.
+
         Y : array-like of shape (n_samples, n_classes)
             The target values.
 
@@ -838,9 +1295,34 @@ def fit(self, X, Y, **fit_params):
         Returns
         -------
         self : object
+            Returns a fitted instance.
         """
         super().fit(X, Y, **fit_params)
         return self
 
-    def _more_tags(self):
-        return {'multioutput_only': True}
+    def get_metadata_routing(self):
+        """Get metadata routing of this object.
+
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
+
+        .. versionadded:: 1.3
+
+        Returns
+        -------
+        routing : MetadataRouter
+            A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
+            routing information.
+        """
+
+        router = MetadataRouter(owner=self.__class__.__name__).add(
+            estimator=self._get_estimator(),
+            method_mapping=MethodMapping().add(caller="fit", callee="fit"),
+        )
+        return router
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.target_tags.single_output = False
+        tags.target_tags.multi_output = True
+        return tags
diff --git a/sklearn/naive_bayes.py b/sklearn/naive_bayes.py
index 70f5993f98b1a..e5b03abbb903a 100644
--- a/sklearn/naive_bayes.py
+++ b/sklearn/naive_bayes.py
@@ -1,42 +1,43 @@
-# -*- coding: utf-8 -*-
+"""Naive Bayes algorithms.
 
-"""
-The :mod:`sklearn.naive_bayes` module implements Naive Bayes algorithms. These
-are supervised learning methods based on applying Bayes' theorem with strong
+These are supervised learning methods based on applying Bayes' theorem with strong
 (naive) feature independence assumptions.
 """
 
-# Author: Vincent Michel 
-#         Minor fixes by Fabian Pedregosa
-#         Amit Aides 
-#         Yehuda Finkelstein 
-#         Lars Buitinck
-#         Jan Hendrik Metzen 
-#         (parts based on earlier work by Mathieu Blondel)
-#
-# License: BSD 3 clause
-import warnings
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
+import warnings
 from abc import ABCMeta, abstractmethod
-
+from numbers import Integral, Real
 
 import numpy as np
 from scipy.special import logsumexp
 
-from .base import BaseEstimator, ClassifierMixin
-from .preprocessing import binarize
-from .preprocessing import LabelBinarizer
-from .preprocessing import label_binarize
-from .utils import deprecated
+from .base import (
+    BaseEstimator,
+    ClassifierMixin,
+    _fit_context,
+)
+from .preprocessing import LabelBinarizer, binarize, label_binarize
+from .utils._param_validation import Interval
 from .utils.extmath import safe_sparse_dot
 from .utils.multiclass import _check_partial_fit_first_call
-from .utils.validation import check_is_fitted, check_non_negative
-from .utils.validation import _check_sample_weight
-from .utils.validation import _deprecate_positional_args
-
-
-__all__ = ['BernoulliNB', 'GaussianNB', 'MultinomialNB', 'ComplementNB',
-           'CategoricalNB']
+from .utils.validation import (
+    _check_n_features,
+    _check_sample_weight,
+    check_is_fitted,
+    check_non_negative,
+    validate_data,
+)
+
+__all__ = [
+    "BernoulliNB",
+    "CategoricalNB",
+    "ComplementNB",
+    "GaussianNB",
+    "MultinomialNB",
+]
 
 
 class _BaseNB(ClassifierMixin, BaseEstimator, metaclass=ABCMeta):
@@ -47,10 +48,12 @@ def _joint_log_likelihood(self, X):
         """Compute the unnormalized posterior log probability of X
 
         I.e. ``log P(c) + log P(x|c)`` for all rows x of X, as an array-like of
-        shape (n_classes, n_samples).
+        shape (n_samples, n_classes).
 
-        Input is passed to _joint_log_likelihood as-is by predict,
-        predict_proba and predict_log_proba.
+        Public methods predict, predict_proba, predict_log_proba, and
+        predict_joint_log_proba pass the input through _check_X before handing it
+        over to _joint_log_likelihood. The term "joint log likelihood" is used
+        interchangibly with "joint log probability".
         """
 
     @abstractmethod
@@ -60,6 +63,30 @@ def _check_X(self, X):
         Only used in predict* methods.
         """
 
+    def predict_joint_log_proba(self, X):
+        """Return joint log probability estimates for the test vector X.
+
+        For each row x of X and class y, the joint log probability is given by
+        ``log P(x, y) = log P(y) + log P(x|y),``
+        where ``log P(y)`` is the class prior probability and ``log P(x|y)`` is
+        the class-conditional probability.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            The input samples.
+
+        Returns
+        -------
+        C : ndarray of shape (n_samples, n_classes)
+            Returns the joint log-probability of the samples for each class in
+            the model. The columns correspond to the classes in sorted
+            order, as they appear in the attribute :term:`classes_`.
+        """
+        check_is_fitted(self)
+        X = self._check_X(X)
+        return self._joint_log_likelihood(X)
+
     def predict(self, X):
         """
         Perform classification on an array of test vectors X.
@@ -67,11 +94,12 @@ def predict(self, X):
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
+            The input samples.
 
         Returns
         -------
         C : ndarray of shape (n_samples,)
-            Predicted target values for X
+            Predicted target values for X.
         """
         check_is_fitted(self)
         X = self._check_X(X)
@@ -85,6 +113,7 @@ def predict_log_proba(self, X):
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
+            The input samples.
 
         Returns
         -------
@@ -107,6 +136,7 @@ def predict_proba(self, X):
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
+            The input samples.
 
         Returns
         -------
@@ -120,20 +150,19 @@ def predict_proba(self, X):
 
 class GaussianNB(_BaseNB):
     """
-    Gaussian Naive Bayes (GaussianNB)
+    Gaussian Naive Bayes (GaussianNB).
 
     Can perform online updates to model parameters via :meth:`partial_fit`.
     For details on algorithm used to update feature means and variance online,
-    see Stanford CS tech report STAN-CS-79-773 by Chan, Golub, and LeVeque:
-
-        http://i.stanford.edu/pub/cstr/reports/cs/tr/79/773/CS-TR-79-773.pdf
+    see `Stanford CS tech report STAN-CS-79-773 by Chan, Golub, and LeVeque
+    `_.
 
     Read more in the :ref:`User Guide `.
 
     Parameters
     ----------
-    priors : array-like of shape (n_classes,)
-        Prior probabilities of the classes. If specified the priors are not
+    priors : array-like of shape (n_classes,), default=None
+        Prior probabilities of the classes. If specified, the priors are not
         adjusted according to the data.
 
     var_smoothing : float, default=1e-9
@@ -151,17 +180,21 @@ class GaussianNB(_BaseNB):
         probability of each class.
 
     classes_ : ndarray of shape (n_classes,)
-        class labels known to the classifier
+        class labels known to the classifier.
 
     epsilon_ : float
-        absolute additive value to variances
+        absolute additive value to variances.
 
-    sigma_ : ndarray of shape (n_classes, n_features)
-        Variance of each feature per class.
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
 
-        .. deprecated:: 1.0
-           `sigma_` is deprecated in 1.0 and will be removed in 1.2.
-           Use `var_` instead.
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
 
     var_ : ndarray of shape (n_classes, n_features)
         Variance of each feature per class.
@@ -169,7 +202,14 @@ class labels known to the classifier
         .. versionadded:: 1.0
 
     theta_ : ndarray of shape (n_classes, n_features)
-        mean of each feature per class
+        mean of each feature per class.
+
+    See Also
+    --------
+    BernoulliNB : Naive Bayes classifier for multivariate Bernoulli models.
+    CategoricalNB : Naive Bayes classifier for categorical features.
+    ComplementNB : Complement Naive Bayes classifier.
+    MultinomialNB : Naive Bayes classifier for multinomial models.
 
     Examples
     --------
@@ -189,19 +229,24 @@ class labels known to the classifier
     [1]
     """
 
-    @_deprecate_positional_args
+    _parameter_constraints: dict = {
+        "priors": ["array-like", None],
+        "var_smoothing": [Interval(Real, 0, None, closed="left")],
+    }
+
     def __init__(self, *, priors=None, var_smoothing=1e-9):
         self.priors = priors
         self.var_smoothing = var_smoothing
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y, sample_weight=None):
-        """Fit Gaussian Naive Bayes according to X, y
+        """Fit Gaussian Naive Bayes according to X, y.
 
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
-            Training vectors, where n_samples is the number of samples
-            and n_features is the number of features.
+            Training vectors, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
 
         y : array-like of shape (n_samples,)
             Target values.
@@ -215,14 +260,16 @@ def fit(self, X, y, sample_weight=None):
         Returns
         -------
         self : object
+            Returns the instance itself.
         """
-        X, y = self._validate_data(X, y)
-        return self._partial_fit(X, y, np.unique(y), _refit=True,
-                                 sample_weight=sample_weight)
+        y = validate_data(self, y=y)
+        return self._partial_fit(
+            X, y, np.unique(y), _refit=True, sample_weight=sample_weight
+        )
 
     def _check_X(self, X):
         """Validate X, used only in predict* methods."""
-        return self._validate_data(X, reset=False)
+        return validate_data(self, X, reset=False)
 
     @staticmethod
     def _update_mean_variance(n_past, mu, var, X, sample_weight=None):
@@ -270,9 +317,10 @@ def _update_mean_variance(n_past, mu, var, X, sample_weight=None):
         # Compute (potentially weighted) mean and variance of new datapoints
         if sample_weight is not None:
             n_new = float(sample_weight.sum())
+            if np.isclose(n_new, 0.0):
+                return mu, var
             new_mu = np.average(X, axis=0, weights=sample_weight)
-            new_var = np.average((X - new_mu) ** 2, axis=0,
-                                 weights=sample_weight)
+            new_var = np.average((X - new_mu) ** 2, axis=0, weights=sample_weight)
         else:
             n_new = X.shape[0]
             new_var = np.var(X, axis=0)
@@ -292,12 +340,12 @@ def _update_mean_variance(n_past, mu, var, X, sample_weight=None):
         # the sum-of-squared-differences (ssd)
         old_ssd = n_past * var
         new_ssd = n_new * new_var
-        total_ssd = (old_ssd + new_ssd +
-                     (n_new * n_past / n_total) * (mu - new_mu) ** 2)
+        total_ssd = old_ssd + new_ssd + (n_new * n_past / n_total) * (mu - new_mu) ** 2
         total_var = total_ssd / n_total
 
         return total_mu, total_var
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def partial_fit(self, X, y, classes=None, sample_weight=None):
         """Incremental fit on a batch of samples.
 
@@ -316,8 +364,8 @@ def partial_fit(self, X, y, classes=None, sample_weight=None):
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
-            Training vectors, where n_samples is the number of samples and
-            n_features is the number of features.
+            Training vectors, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
 
         y : array-like of shape (n_samples,)
             Target values.
@@ -336,19 +384,20 @@ def partial_fit(self, X, y, classes=None, sample_weight=None):
         Returns
         -------
         self : object
+            Returns the instance itself.
         """
-        return self._partial_fit(X, y, classes, _refit=False,
-                                 sample_weight=sample_weight)
+        return self._partial_fit(
+            X, y, classes, _refit=False, sample_weight=sample_weight
+        )
 
-    def _partial_fit(self, X, y, classes=None, _refit=False,
-                     sample_weight=None):
+    def _partial_fit(self, X, y, classes=None, _refit=False, sample_weight=None):
         """Actual implementation of Gaussian NB fitting.
 
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
-            Training vectors, where n_samples is the number of samples and
-            n_features is the number of features.
+            Training vectors, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
 
         y : array-like of shape (n_samples,)
             Target values.
@@ -374,7 +423,7 @@ def _partial_fit(self, X, y, classes=None, _refit=False,
             self.classes_ = None
 
         first_call = _check_partial_fit_first_call(self, classes)
-        X, y = self._validate_data(X, y, reset=first_call)
+        X, y = validate_data(self, X, y, reset=first_call)
         if sample_weight is not None:
             sample_weight = _check_sample_weight(sample_weight, X)
 
@@ -398,21 +447,19 @@ def _partial_fit(self, X, y, classes=None, _refit=False,
             # Take into account the priors
             if self.priors is not None:
                 priors = np.asarray(self.priors)
-                # Check that the provide prior match the number of classes
+                # Check that the provided prior matches the number of classes
                 if len(priors) != n_classes:
-                    raise ValueError('Number of priors must match number of'
-                                     ' classes.')
+                    raise ValueError("Number of priors must match number of classes.")
                 # Check that the sum is 1
                 if not np.isclose(priors.sum(), 1.0):
-                    raise ValueError('The sum of the priors should be 1.')
-                # Check that the prior are non-negative
+                    raise ValueError("The sum of the priors should be 1.")
+                # Check that the priors are non-negative
                 if (priors < 0).any():
-                    raise ValueError('Priors must be non-negative.')
+                    raise ValueError("Priors must be non-negative.")
                 self.class_prior_ = priors
             else:
                 # Initialize the priors to zeros for each class
-                self.class_prior_ = np.zeros(len(self.classes_),
-                                             dtype=np.float64)
+                self.class_prior_ = np.zeros(len(self.classes_), dtype=np.float64)
         else:
             if X.shape[1] != self.theta_.shape[1]:
                 msg = "Number of features %d does not match previous data %d."
@@ -423,12 +470,13 @@ def _partial_fit(self, X, y, classes=None, _refit=False,
         classes = self.classes_
 
         unique_y = np.unique(y)
-        unique_y_in_classes = np.in1d(unique_y, classes)
+        unique_y_in_classes = np.isin(unique_y, classes)
 
         if not np.all(unique_y_in_classes):
-            raise ValueError("The target label(s) %s in y do not exist in the "
-                             "initial classes %s" %
-                             (unique_y[~unique_y_in_classes], classes))
+            raise ValueError(
+                "The target label(s) %s in y do not exist in the initial classes %s"
+                % (unique_y[~unique_y_in_classes], classes)
+            )
 
         for y_i in unique_y:
             i = classes.searchsorted(y_i)
@@ -442,8 +490,8 @@ def _partial_fit(self, X, y, classes=None, _refit=False,
                 N_i = X_i.shape[0]
 
             new_theta, new_sigma = self._update_mean_variance(
-                self.class_count_[i], self.theta_[i, :], self.var_[i, :],
-                X_i, sw_i)
+                self.class_count_[i], self.theta_[i, :], self.var_[i, :], X_i, sw_i
+            )
 
             self.theta_[i, :] = new_theta
             self.var_[i, :] = new_sigma
@@ -462,25 +510,13 @@ def _joint_log_likelihood(self, X):
         joint_log_likelihood = []
         for i in range(np.size(self.classes_)):
             jointi = np.log(self.class_prior_[i])
-            n_ij = - 0.5 * np.sum(np.log(2. * np.pi * self.var_[i, :]))
-            n_ij -= 0.5 * np.sum(((X - self.theta_[i, :]) ** 2) /
-                                 (self.var_[i, :]), 1)
+            n_ij = -0.5 * np.sum(np.log(2.0 * np.pi * self.var_[i, :]))
+            n_ij -= 0.5 * np.sum(((X - self.theta_[i, :]) ** 2) / (self.var_[i, :]), 1)
             joint_log_likelihood.append(jointi + n_ij)
 
         joint_log_likelihood = np.array(joint_log_likelihood).T
         return joint_log_likelihood
 
-    @deprecated(  # type: ignore
-        "Attribute sigma_ was deprecated in 1.0 and will be removed in"
-        "1.2. Use var_ instead."
-    )
-    @property
-    def sigma_(self):
-        return self.var_
-
-
-_ALPHA_MIN = 1e-10
-
 
 class _BaseDiscreteNB(_BaseNB):
     """Abstract base class for naive Bayes on discrete/categorical data
@@ -489,22 +525,72 @@ class _BaseDiscreteNB(_BaseNB):
 
     __init__
     _joint_log_likelihood(X) as per _BaseNB
+    _update_feature_log_prob(alpha)
+    _count(X, Y)
     """
 
+    _parameter_constraints: dict = {
+        "alpha": [Interval(Real, 0, None, closed="left"), "array-like"],
+        "fit_prior": ["boolean"],
+        "class_prior": ["array-like", None],
+        "force_alpha": ["boolean"],
+    }
+
+    def __init__(self, alpha=1.0, fit_prior=True, class_prior=None, force_alpha=True):
+        self.alpha = alpha
+        self.fit_prior = fit_prior
+        self.class_prior = class_prior
+        self.force_alpha = force_alpha
+
+    @abstractmethod
+    def _count(self, X, Y):
+        """Update counts that are used to calculate probabilities.
+
+        The counts make up a sufficient statistic extracted from the data.
+        Accordingly, this method is called each time `fit` or `partial_fit`
+        update the model. `class_count_` and `feature_count_` must be updated
+        here along with any model specific counts.
+
+        Parameters
+        ----------
+        X : {ndarray, sparse matrix} of shape (n_samples, n_features)
+            The input samples.
+        Y : ndarray of shape (n_samples, n_classes)
+            Binarized class labels.
+        """
+
+    @abstractmethod
+    def _update_feature_log_prob(self, alpha):
+        """Update feature log probabilities based on counts.
+
+        This method is called each time `fit` or `partial_fit` update the
+        model.
+
+        Parameters
+        ----------
+        alpha : float
+            smoothing parameter. See :meth:`_check_alpha`.
+        """
+
     def _check_X(self, X):
         """Validate X, used only in predict* methods."""
-        return self._validate_data(X, accept_sparse='csr', reset=False)
+        return validate_data(self, X, accept_sparse="csr", reset=False)
 
     def _check_X_y(self, X, y, reset=True):
         """Validate X and y in fit methods."""
-        return self._validate_data(X, y, accept_sparse='csr', reset=reset)
+        return validate_data(self, X, y, accept_sparse="csr", reset=reset)
 
     def _update_class_log_prior(self, class_prior=None):
+        """Update class log priors.
+
+        The class log priors are based on `class_prior`, class count or the
+        number of classes. This method is called each time `fit` or
+        `partial_fit` update the model.
+        """
         n_classes = len(self.classes_)
         if class_prior is not None:
             if len(class_prior) != n_classes:
-                raise ValueError("Number of priors must match number of"
-                                 " classes.")
+                raise ValueError("Number of priors must match number of classes.")
             self.class_log_prior_ = np.log(class_prior)
         elif self.fit_prior:
             with warnings.catch_warnings():
@@ -514,25 +600,35 @@ def _update_class_log_prior(self, class_prior=None):
                 log_class_count = np.log(self.class_count_)
 
             # empirical prior, with sample_weight taken into account
-            self.class_log_prior_ = (log_class_count -
-                                     np.log(self.class_count_.sum()))
+            self.class_log_prior_ = log_class_count - np.log(self.class_count_.sum())
         else:
             self.class_log_prior_ = np.full(n_classes, -np.log(n_classes))
 
     def _check_alpha(self):
-        if np.min(self.alpha) < 0:
-            raise ValueError('Smoothing parameter alpha = %.1e. '
-                             'alpha should be > 0.' % np.min(self.alpha))
-        if isinstance(self.alpha, np.ndarray):
-            if not self.alpha.shape[0] == self.n_features_in_:
-                raise ValueError("alpha should be a scalar or a numpy array "
-                                 "with shape [n_features]")
-        if np.min(self.alpha) < _ALPHA_MIN:
-            warnings.warn('alpha too small will result in numeric errors, '
-                          'setting alpha = %.1e' % _ALPHA_MIN)
-            return np.maximum(self.alpha, _ALPHA_MIN)
-        return self.alpha
-
+        alpha = (
+            np.asarray(self.alpha) if not isinstance(self.alpha, Real) else self.alpha
+        )
+        alpha_min = np.min(alpha)
+        if isinstance(alpha, np.ndarray):
+            if not alpha.shape[0] == self.n_features_in_:
+                raise ValueError(
+                    "When alpha is an array, it should contains `n_features`. "
+                    f"Got {alpha.shape[0]} elements instead of {self.n_features_in_}."
+                )
+            # check that all alpha are positive
+            if alpha_min < 0:
+                raise ValueError("All values in alpha must be greater than 0.")
+        alpha_lower_bound = 1e-10
+        if alpha_min < alpha_lower_bound and not self.force_alpha:
+            warnings.warn(
+                "alpha too small will result in numeric errors, setting alpha ="
+                f" {alpha_lower_bound:.1e}. Use `force_alpha=True` to keep alpha"
+                " unchanged."
+            )
+            return np.maximum(alpha, alpha_lower_bound)
+        return alpha
+
+    @_fit_context(prefer_skip_nested_validation=True)
     def partial_fit(self, X, y, classes=None, sample_weight=None):
         """Incremental fit on a batch of samples.
 
@@ -550,8 +646,8 @@ def partial_fit(self, X, y, classes=None, sample_weight=None):
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
-            Training vectors, where n_samples is the number of samples and
-            n_features is the number of features.
+            Training vectors, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
 
         y : array-like of shape (n_samples,)
             Target values.
@@ -568,8 +664,10 @@ def partial_fit(self, X, y, classes=None, sample_weight=None):
         Returns
         -------
         self : object
+            Returns the instance itself.
         """
         first_call = not hasattr(self, "classes_")
+
         X, y = self._check_X_y(X, y, reset=first_call)
         _, n_features = X.shape
 
@@ -583,7 +681,7 @@ def partial_fit(self, X, y, classes=None, sample_weight=None):
         if Y.shape[1] == 1:
             if len(self.classes_) == 2:
                 Y = np.concatenate((1 - Y, Y), axis=1)
-            else:    # degenerate case: just one class
+            else:  # degenerate case: just one class
                 Y = np.ones_like(Y)
 
         if X.shape[0] != Y.shape[0]:
@@ -613,14 +711,15 @@ def partial_fit(self, X, y, classes=None, sample_weight=None):
         self._update_class_log_prior(class_prior=class_prior)
         return self
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y, sample_weight=None):
-        """Fit Naive Bayes classifier according to X, y
+        """Fit Naive Bayes classifier according to X, y.
 
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
-            Training vectors, where n_samples is the number of samples and
-            n_features is the number of features.
+            Training vectors, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
 
         y : array-like of shape (n_samples,)
             Target values.
@@ -631,6 +730,7 @@ def fit(self, X, y, sample_weight=None):
         Returns
         -------
         self : object
+            Returns the instance itself.
         """
         X, y = self._check_X_y(X, y)
         _, n_features = X.shape
@@ -641,7 +741,7 @@ def fit(self, X, y, sample_weight=None):
         if Y.shape[1] == 1:
             if len(self.classes_) == 2:
                 Y = np.concatenate((1 - Y, Y), axis=1)
-            else:    # degenerate case: just one class
+            else:  # degenerate case: just one class
                 Y = np.ones_like(Y)
 
         # LabelBinarizer().fit_transform() returns arrays with dtype=np.int64.
@@ -667,42 +767,18 @@ def fit(self, X, y, sample_weight=None):
 
     def _init_counters(self, n_classes, n_features):
         self.class_count_ = np.zeros(n_classes, dtype=np.float64)
-        self.feature_count_ = np.zeros((n_classes, n_features),
-                                       dtype=np.float64)
-
-    # mypy error: Decorated property not supported
-    @deprecated("Attribute coef_ was deprecated in "  # type: ignore
-                "version 0.24 and will be removed in 1.1 (renaming of 0.26).")
-    @property
-    def coef_(self):
-        return (self.feature_log_prob_[1:]
-                if len(self.classes_) == 2 else self.feature_log_prob_)
-
-    # mypy error: Decorated property not supported
-    @deprecated("Attribute intercept_ was deprecated in "  # type: ignore
-                "version 0.24 and will be removed in 1.1 (renaming of 0.26).")
-    @property
-    def intercept_(self):
-        return (self.class_log_prior_[1:]
-                if len(self.classes_) == 2 else self.class_log_prior_)
-
-    def _more_tags(self):
-        return {'poor_score': True}
-
-    # TODO: Remove in 1.2
-    # mypy error: Decorated property not supported
-    @deprecated(  # type: ignore
-        "Attribute n_features_ was deprecated in version 1.0 and will be "
-        "removed in 1.2. Use 'n_features_in_' instead."
-    )
-    @property
-    def n_features_(self):
-        return self.n_features_in_
+        self.feature_count_ = np.zeros((n_classes, n_features), dtype=np.float64)
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
+        tags.classifier_tags.poor_score = True
+        return tags
 
 
 class MultinomialNB(_BaseDiscreteNB):
     """
-    Naive Bayes classifier for multinomial models
+    Naive Bayes classifier for multinomial models.
 
     The multinomial Naive Bayes classifier is suitable for classification with
     discrete features (e.g., word counts for text classification). The
@@ -713,16 +789,25 @@ class MultinomialNB(_BaseDiscreteNB):
 
     Parameters
     ----------
-    alpha : float, default=1.0
+    alpha : float or array-like of shape (n_features,), default=1.0
         Additive (Laplace/Lidstone) smoothing parameter
-        (0 for no smoothing).
+        (set alpha=0 and force_alpha=True, for no smoothing).
+
+    force_alpha : bool, default=True
+        If False and alpha is less than 1e-10, it will set alpha to
+        1e-10. If True, alpha will remain unchanged. This may cause
+        numerical errors if alpha is too close to 0.
+
+        .. versionadded:: 1.2
+        .. versionchanged:: 1.4
+           The default value of `force_alpha` changed to `True`.
 
     fit_prior : bool, default=True
         Whether to learn class prior probabilities or not.
         If false, a uniform prior will be used.
 
     class_prior : array-like of shape (n_classes,), default=None
-        Prior probabilities of the classes. If specified the priors are not
+        Prior probabilities of the classes. If specified, the priors are not
         adjusted according to the data.
 
     Attributes
@@ -737,14 +822,6 @@ class MultinomialNB(_BaseDiscreteNB):
     classes_ : ndarray of shape (n_classes,)
         Class labels known to the classifier
 
-    coef_ : ndarray of shape (n_classes, n_features)
-        Mirrors ``feature_log_prob_`` for interpreting `MultinomialNB`
-        as a linear model.
-
-        .. deprecated:: 0.24
-            ``coef_`` is deprecated in 0.24 and will be removed in 1.1
-            (renaming of 0.26).
-
     feature_count_ : ndarray of shape (n_classes, n_features)
         Number of samples encountered for each (class, feature)
         during fitting. This value is weighted by the sample weight when
@@ -754,20 +831,29 @@ class MultinomialNB(_BaseDiscreteNB):
         Empirical log probability of features
         given a class, ``P(x_i|y)``.
 
-    intercept_ : ndarray of shape (n_classes,)
-        Mirrors ``class_log_prior_`` for interpreting `MultinomialNB`
-        as a linear model.
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
 
-        .. deprecated:: 0.24
-            ``intercept_`` is deprecated in 0.24 and will be removed in 1.1
-            (renaming of 0.26).
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
 
-    n_features_ : int
-        Number of features of each sample.
+    See Also
+    --------
+    BernoulliNB : Naive Bayes classifier for multivariate Bernoulli models.
+    CategoricalNB : Naive Bayes classifier for categorical features.
+    ComplementNB : Complement Naive Bayes classifier.
+    GaussianNB : Gaussian Naive Bayes.
 
-        .. deprecated:: 1.0
-            Attribute `n_features_` was deprecated in version 1.0 and will be
-            removed in 1.2. Use `n_features_in_` instead.
+    References
+    ----------
+    C.D. Manning, P. Raghavan and H. Schuetze (2008). Introduction to
+    Information Retrieval. Cambridge University Press, pp. 234-265.
+    https://nlp.stanford.edu/IR-book/html/htmledition/naive-bayes-text-classification-1.html
 
     Examples
     --------
@@ -781,28 +867,22 @@ class MultinomialNB(_BaseDiscreteNB):
     MultinomialNB()
     >>> print(clf.predict(X[2:3]))
     [3]
-
-    Notes
-    -----
-    For the rationale behind the names `coef_` and `intercept_`, i.e.
-    naive Bayes as a linear classifier, see J. Rennie et al. (2003),
-    Tackling the poor assumptions of naive Bayes text classifiers, ICML.
-
-    References
-    ----------
-    C.D. Manning, P. Raghavan and H. Schuetze (2008). Introduction to
-    Information Retrieval. Cambridge University Press, pp. 234-265.
-    https://nlp.stanford.edu/IR-book/html/htmledition/naive-bayes-text-classification-1.html
     """
 
-    @_deprecate_positional_args
-    def __init__(self, *, alpha=1.0, fit_prior=True, class_prior=None):
-        self.alpha = alpha
-        self.fit_prior = fit_prior
-        self.class_prior = class_prior
-
-    def _more_tags(self):
-        return {'requires_positive_X': True}
+    def __init__(
+        self, *, alpha=1.0, force_alpha=True, fit_prior=True, class_prior=None
+    ):
+        super().__init__(
+            alpha=alpha,
+            fit_prior=fit_prior,
+            class_prior=class_prior,
+            force_alpha=force_alpha,
+        )
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.positive_only = True
+        return tags
 
     def _count(self, X, Y):
         """Count and smooth feature occurrences."""
@@ -815,13 +895,13 @@ def _update_feature_log_prob(self, alpha):
         smoothed_fc = self.feature_count_ + alpha
         smoothed_cc = smoothed_fc.sum(axis=1)
 
-        self.feature_log_prob_ = (np.log(smoothed_fc) -
-                                  np.log(smoothed_cc.reshape(-1, 1)))
+        self.feature_log_prob_ = np.log(smoothed_fc) - np.log(
+            smoothed_cc.reshape(-1, 1)
+        )
 
     def _joint_log_likelihood(self, X):
         """Calculate the posterior log probability of the samples X"""
-        return (safe_sparse_dot(X, self.feature_log_prob_.T) +
-                self.class_log_prior_)
+        return safe_sparse_dot(X, self.feature_log_prob_.T) + self.class_log_prior_
 
 
 class ComplementNB(_BaseDiscreteNB):
@@ -837,8 +917,18 @@ class ComplementNB(_BaseDiscreteNB):
 
     Parameters
     ----------
-    alpha : float, default=1.0
-        Additive (Laplace/Lidstone) smoothing parameter (0 for no smoothing).
+    alpha : float or array-like of shape (n_features,), default=1.0
+        Additive (Laplace/Lidstone) smoothing parameter
+        (set alpha=0 and force_alpha=True, for no smoothing).
+
+    force_alpha : bool, default=True
+        If False and alpha is less than 1e-10, it will set alpha to
+        1e-10. If True, alpha will remain unchanged. This may cause
+        numerical errors if alpha is too close to 0.
+
+        .. versionadded:: 1.2
+        .. versionchanged:: 1.4
+           The default value of `force_alpha` changed to `True`.
 
     fit_prior : bool, default=True
         Only used in edge case with a single class in the training set.
@@ -865,14 +955,6 @@ class ComplementNB(_BaseDiscreteNB):
     classes_ : ndarray of shape (n_classes,)
         Class labels known to the classifier
 
-    coef_ : ndarray of shape (n_classes, n_features)
-        Mirrors ``feature_log_prob_`` for interpreting `ComplementNB`
-        as a linear model.
-
-        .. deprecated:: 0.24
-            ``coef_`` is deprecated in 0.24 and will be removed in 1.1
-            (renaming of 0.26).
-
     feature_all_ : ndarray of shape (n_features,)
         Number of samples encountered for each feature during fitting. This
         value is weighted by the sample weight when provided.
@@ -884,20 +966,30 @@ class ComplementNB(_BaseDiscreteNB):
     feature_log_prob_ : ndarray of shape (n_classes, n_features)
         Empirical weights for class complements.
 
-    intercept_ : ndarray of shape (n_classes,)
-        Mirrors ``class_log_prior_`` for interpreting `ComplementNB`
-        as a linear model.
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
 
-        .. deprecated:: 0.24
-            ``coef_`` is deprecated in 0.24 and will be removed in 1.1
-            (renaming of 0.26).
+        .. versionadded:: 1.0
 
-    n_features_ : int
-        Number of features of each sample.
+    See Also
+    --------
+    BernoulliNB : Naive Bayes classifier for multivariate Bernoulli models.
+    CategoricalNB : Naive Bayes classifier for categorical features.
+    GaussianNB : Gaussian Naive Bayes.
+    MultinomialNB : Naive Bayes classifier for multinomial models.
 
-        .. deprecated:: 1.0
-            Attribute `n_features_` was deprecated in version 1.0 and will be
-            removed in 1.2. Use `n_features_in_` instead.
+    References
+    ----------
+    Rennie, J. D., Shih, L., Teevan, J., & Karger, D. R. (2003).
+    Tackling the poor assumptions of naive bayes text classifiers. In ICML
+    (Vol. 3, pp. 616-623).
+    https://people.csail.mit.edu/jrennie/papers/icml03-nb.pdf
 
     Examples
     --------
@@ -911,25 +1003,34 @@ class ComplementNB(_BaseDiscreteNB):
     ComplementNB()
     >>> print(clf.predict(X[2:3]))
     [3]
-
-    References
-    ----------
-    Rennie, J. D., Shih, L., Teevan, J., & Karger, D. R. (2003).
-    Tackling the poor assumptions of naive bayes text classifiers. In ICML
-    (Vol. 3, pp. 616-623).
-    https://people.csail.mit.edu/jrennie/papers/icml03-nb.pdf
     """
 
-    @_deprecate_positional_args
-    def __init__(self, *, alpha=1.0, fit_prior=True, class_prior=None,
-                 norm=False):
-        self.alpha = alpha
-        self.fit_prior = fit_prior
-        self.class_prior = class_prior
+    _parameter_constraints: dict = {
+        **_BaseDiscreteNB._parameter_constraints,
+        "norm": ["boolean"],
+    }
+
+    def __init__(
+        self,
+        *,
+        alpha=1.0,
+        force_alpha=True,
+        fit_prior=True,
+        class_prior=None,
+        norm=False,
+    ):
+        super().__init__(
+            alpha=alpha,
+            force_alpha=force_alpha,
+            fit_prior=fit_prior,
+            class_prior=class_prior,
+        )
         self.norm = norm
 
-    def _more_tags(self):
-        return {'requires_positive_X': True}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.positive_only = True
+        return tags
 
     def _count(self, X, Y):
         """Count feature occurrences."""
@@ -969,9 +1070,18 @@ class BernoulliNB(_BaseDiscreteNB):
 
     Parameters
     ----------
-    alpha : float, default=1.0
+    alpha : float or array-like of shape (n_features,), default=1.0
         Additive (Laplace/Lidstone) smoothing parameter
-        (0 for no smoothing).
+        (set alpha=0 and force_alpha=True, for no smoothing).
+
+    force_alpha : bool, default=True
+        If False and alpha is less than 1e-10, it will set alpha to
+        1e-10. If True, alpha will remain unchanged. This may cause
+        numerical errors if alpha is too close to 0.
+
+        .. versionadded:: 1.2
+        .. versionchanged:: 1.4
+           The default value of `force_alpha` changed to `True`.
 
     binarize : float or None, default=0.0
         Threshold for binarizing (mapping to booleans) of sample features.
@@ -982,7 +1092,7 @@ class BernoulliNB(_BaseDiscreteNB):
         If false, a uniform prior will be used.
 
     class_prior : array-like of shape (n_classes,), default=None
-        Prior probabilities of the classes. If specified the priors are not
+        Prior probabilities of the classes. If specified, the priors are not
         adjusted according to the data.
 
     Attributes
@@ -997,10 +1107,6 @@ class BernoulliNB(_BaseDiscreteNB):
     classes_ : ndarray of shape (n_classes,)
         Class labels known to the classifier
 
-    coef_ : ndarray of shape (n_classes, n_features)
-        Mirrors ``feature_log_prob_`` for interpreting `BernoulliNB`
-        as a linear model.
-
     feature_count_ : ndarray of shape (n_classes, n_features)
         Number of samples encountered for each (class, feature)
         during fitting. This value is weighted by the sample weight when
@@ -1009,29 +1115,24 @@ class BernoulliNB(_BaseDiscreteNB):
     feature_log_prob_ : ndarray of shape (n_classes, n_features)
         Empirical log probability of features given a class, P(x_i|y).
 
-    intercept_ : ndarray of shape (n_classes,)
-        Mirrors ``class_log_prior_`` for interpreting `BernoulliNB`
-        as a linear model.
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
 
-    n_features_ : int
-        Number of features of each sample.
+        .. versionadded:: 0.24
 
-        .. deprecated:: 1.0
-            Attribute `n_features_` was deprecated in version 1.0 and will be
-            removed in 1.2. Use `n_features_in_` instead.
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
 
-    Examples
+        .. versionadded:: 1.0
+
+    See Also
     --------
-    >>> import numpy as np
-    >>> rng = np.random.RandomState(1)
-    >>> X = rng.randint(5, size=(6, 100))
-    >>> Y = np.array([1, 2, 3, 4, 4, 5])
-    >>> from sklearn.naive_bayes import BernoulliNB
-    >>> clf = BernoulliNB()
-    >>> clf.fit(X, Y)
-    BernoulliNB()
-    >>> print(clf.predict(X[2:3]))
-    [3]
+    CategoricalNB : Naive Bayes classifier for categorical features.
+    ComplementNB : The Complement Naive Bayes classifier
+        described in Rennie et al. (2003).
+    GaussianNB : Gaussian Naive Bayes (GaussianNB).
+    MultinomialNB : Naive Bayes classifier for multinomial models.
 
     References
     ----------
@@ -1045,15 +1146,42 @@ class BernoulliNB(_BaseDiscreteNB):
 
     V. Metsis, I. Androutsopoulos and G. Paliouras (2006). Spam filtering with
     naive Bayes -- Which naive Bayes? 3rd Conf. on Email and Anti-Spam (CEAS).
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> rng = np.random.RandomState(1)
+    >>> X = rng.randint(5, size=(6, 100))
+    >>> Y = np.array([1, 2, 3, 4, 4, 5])
+    >>> from sklearn.naive_bayes import BernoulliNB
+    >>> clf = BernoulliNB()
+    >>> clf.fit(X, Y)
+    BernoulliNB()
+    >>> print(clf.predict(X[2:3]))
+    [3]
     """
 
-    @_deprecate_positional_args
-    def __init__(self, *, alpha=1.0, binarize=.0, fit_prior=True,
-                 class_prior=None):
-        self.alpha = alpha
+    _parameter_constraints: dict = {
+        **_BaseDiscreteNB._parameter_constraints,
+        "binarize": [None, Interval(Real, 0, None, closed="left")],
+    }
+
+    def __init__(
+        self,
+        *,
+        alpha=1.0,
+        force_alpha=True,
+        binarize=0.0,
+        fit_prior=True,
+        class_prior=None,
+    ):
+        super().__init__(
+            alpha=alpha,
+            fit_prior=fit_prior,
+            class_prior=class_prior,
+            force_alpha=force_alpha,
+        )
         self.binarize = binarize
-        self.fit_prior = fit_prior
-        self.class_prior = class_prior
 
     def _check_X(self, X):
         """Validate X, used only in predict* methods."""
@@ -1078,8 +1206,9 @@ def _update_feature_log_prob(self, alpha):
         smoothed_fc = self.feature_count_ + alpha
         smoothed_cc = self.class_count_ + alpha * 2
 
-        self.feature_log_prob_ = (np.log(smoothed_fc) -
-                                  np.log(smoothed_cc.reshape(-1, 1)))
+        self.feature_log_prob_ = np.log(smoothed_fc) - np.log(
+            smoothed_cc.reshape(-1, 1)
+        )
 
     def _joint_log_likelihood(self, X):
         """Calculate the posterior log probability of the samples X"""
@@ -1087,8 +1216,10 @@ def _joint_log_likelihood(self, X):
         n_features_X = X.shape[1]
 
         if n_features_X != n_features:
-            raise ValueError("Expected input with %d features, got %d instead"
-                             % (n_features, n_features_X))
+            raise ValueError(
+                "Expected input with %d features, got %d instead"
+                % (n_features, n_features_X)
+            )
 
         neg_prob = np.log(1 - np.exp(self.feature_log_prob_))
         # Compute  neg_prob · (1 - X).T  as  ∑neg_prob - X · neg_prob
@@ -1099,7 +1230,7 @@ def _joint_log_likelihood(self, X):
 
 
 class CategoricalNB(_BaseDiscreteNB):
-    """Naive Bayes classifier for categorical features
+    """Naive Bayes classifier for categorical features.
 
     The categorical Naive Bayes classifier is suitable for classification with
     discrete features that are categorically distributed. The categories of
@@ -1111,14 +1242,23 @@ class CategoricalNB(_BaseDiscreteNB):
     ----------
     alpha : float, default=1.0
         Additive (Laplace/Lidstone) smoothing parameter
-        (0 for no smoothing).
+        (set alpha=0 and force_alpha=True, for no smoothing).
+
+    force_alpha : bool, default=True
+        If False and alpha is less than 1e-10, it will set alpha to
+        1e-10. If True, alpha will remain unchanged. This may cause
+        numerical errors if alpha is too close to 0.
+
+        .. versionadded:: 1.2
+        .. versionchanged:: 1.4
+           The default value of `force_alpha` changed to `True`.
 
     fit_prior : bool, default=True
         Whether to learn class prior probabilities or not.
         If false, a uniform prior will be used.
 
     class_prior : array-like of shape (n_classes,), default=None
-        Prior probabilities of the classes. If specified the priors are not
+        Prior probabilities of the classes. If specified, the priors are not
         adjusted according to the data.
 
     min_categories : int or array-like of shape (n_features,), default=None
@@ -1155,12 +1295,16 @@ class CategoricalNB(_BaseDiscreteNB):
         for each feature. Each array provides the empirical log probability
         of categories given the respective feature and class, ``P(x_i|y)``.
 
-    n_features_ : int
-        Number of features of each sample.
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
 
-        .. deprecated:: 1.0
-            Attribute `n_features_` was deprecated in version 1.0 and will be
-            removed in 1.2. Use `n_features_in_` instead.
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
 
     n_categories_ : ndarray of shape (n_features,), dtype=np.int64
         Number of categories for each feature. This value is
@@ -1168,6 +1312,13 @@ class CategoricalNB(_BaseDiscreteNB):
 
         .. versionadded:: 0.24
 
+    See Also
+    --------
+    BernoulliNB : Naive Bayes classifier for multivariate Bernoulli models.
+    ComplementNB : Complement Naive Bayes classifier.
+    GaussianNB : Gaussian Naive Bayes.
+    MultinomialNB : Naive Bayes classifier for multinomial models.
+
     Examples
     --------
     >>> import numpy as np
@@ -1182,22 +1333,41 @@ class CategoricalNB(_BaseDiscreteNB):
     [3]
     """
 
-    @_deprecate_positional_args
-    def __init__(self, *, alpha=1.0, fit_prior=True, class_prior=None,
-                 min_categories=None):
-        self.alpha = alpha
-        self.fit_prior = fit_prior
-        self.class_prior = class_prior
+    _parameter_constraints: dict = {
+        **_BaseDiscreteNB._parameter_constraints,
+        "min_categories": [
+            None,
+            "array-like",
+            Interval(Integral, 1, None, closed="left"),
+        ],
+        "alpha": [Interval(Real, 0, None, closed="left")],
+    }
+
+    def __init__(
+        self,
+        *,
+        alpha=1.0,
+        force_alpha=True,
+        fit_prior=True,
+        class_prior=None,
+        min_categories=None,
+    ):
+        super().__init__(
+            alpha=alpha,
+            force_alpha=force_alpha,
+            fit_prior=fit_prior,
+            class_prior=class_prior,
+        )
         self.min_categories = min_categories
 
     def fit(self, X, y, sample_weight=None):
-        """Fit Naive Bayes classifier according to X, y
+        """Fit Naive Bayes classifier according to X, y.
 
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
-            Training vectors, where n_samples is the number of samples and
-            n_features is the number of features. Here, each feature of X is
+            Training vectors, where `n_samples` is the number of samples and
+            `n_features` is the number of features. Here, each feature of X is
             assumed to be from a different categorical distribution.
             It is further assumed that all categories of each feature are
             represented by the numbers 0, ..., n - 1, where n refers to the
@@ -1213,6 +1383,7 @@ def fit(self, X, y, sample_weight=None):
         Returns
         -------
         self : object
+            Returns the instance itself.
         """
         return super().fit(X, y, sample_weight=sample_weight)
 
@@ -1233,8 +1404,8 @@ def partial_fit(self, X, y, classes=None, sample_weight=None):
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
-            Training vectors, where n_samples is the number of samples and
-            n_features is the number of features. Here, each feature of X is
+            Training vectors, where `n_samples` is the number of samples and
+            `n_features` is the number of features. Here, each feature of X is
             assumed to be from a different categorical distribution.
             It is further assumed that all categories of each feature are
             represented by the numbers 0, ..., n - 1, where n refers to the
@@ -1256,30 +1427,45 @@ def partial_fit(self, X, y, classes=None, sample_weight=None):
         Returns
         -------
         self : object
+            Returns the instance itself.
         """
-        return super().partial_fit(X, y, classes,
-                                   sample_weight=sample_weight)
+        return super().partial_fit(X, y, classes, sample_weight=sample_weight)
 
-    def _more_tags(self):
-        return {'requires_positive_X': True}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = False
+        tags.input_tags.positive_only = True
+        return tags
 
     def _check_X(self, X):
         """Validate X, used only in predict* methods."""
-        X = self._validate_data(X, dtype='int', accept_sparse=False,
-                                force_all_finite=True, reset=False)
+        X = validate_data(
+            self,
+            X,
+            dtype="int",
+            accept_sparse=False,
+            ensure_all_finite=True,
+            reset=False,
+        )
         check_non_negative(X, "CategoricalNB (input X)")
         return X
 
     def _check_X_y(self, X, y, reset=True):
-        X, y = self._validate_data(X, y, dtype='int', accept_sparse=False,
-                                   force_all_finite=True, reset=reset)
+        X, y = validate_data(
+            self,
+            X,
+            y,
+            dtype="int",
+            accept_sparse=False,
+            ensure_all_finite=True,
+            reset=reset,
+        )
         check_non_negative(X, "CategoricalNB (input X)")
         return X, y
 
     def _init_counters(self, n_classes, n_features):
         self.class_count_ = np.zeros(n_classes, dtype=np.float64)
-        self.category_count_ = [np.zeros((n_classes, 0))
-                                for _ in range(n_features)]
+        self.category_count_ = [np.zeros((n_classes, 0)) for _ in range(n_features)]
 
     @staticmethod
     def _validate_n_categories(X, min_categories):
@@ -1289,16 +1475,14 @@ def _validate_n_categories(X, min_categories):
         if min_categories is not None:
             if not np.issubdtype(min_categories_.dtype, np.signedinteger):
                 raise ValueError(
-                    f"'min_categories' should have integral type. Got "
+                    "'min_categories' should have integral type. Got "
                     f"{min_categories_.dtype} instead."
                 )
-            n_categories_ = np.maximum(n_categories_X,
-                                       min_categories_,
-                                       dtype=np.int64)
+            n_categories_ = np.maximum(n_categories_X, min_categories_, dtype=np.int64)
             if n_categories_.shape != n_categories_X.shape:
                 raise ValueError(
                     f"'min_categories' should have shape ({X.shape[1]},"
-                    f") when an array-like is provided. Got"
+                    ") when an array-like is provided. Got"
                     f" {min_categories_.shape} instead."
                 )
             return n_categories_
@@ -1310,7 +1494,7 @@ def _update_cat_count_dims(cat_count, highest_feature):
             diff = highest_feature + 1 - cat_count.shape[1]
             if diff > 0:
                 # we append a column full of zeros for each new category
-                return np.pad(cat_count, [(0, 0), (0, diff)], 'constant')
+                return np.pad(cat_count, [(0, 0), (0, diff)], "constant")
             return cat_count
 
         def _update_cat_count(X_feature, Y, cat_count, n_classes):
@@ -1325,15 +1509,15 @@ def _update_cat_count(X_feature, Y, cat_count, n_classes):
                 cat_count[j, indices] += counts[indices]
 
         self.class_count_ += Y.sum(axis=0)
-        self.n_categories_ = self._validate_n_categories(
-            X, self.min_categories)
+        self.n_categories_ = self._validate_n_categories(X, self.min_categories)
         for i in range(self.n_features_in_):
             X_feature = X[:, i]
             self.category_count_[i] = _update_cat_count_dims(
-                self.category_count_[i], self.n_categories_[i] - 1)
-            _update_cat_count(X_feature, Y,
-                              self.category_count_[i],
-                              self.class_count_.shape[0])
+                self.category_count_[i], self.n_categories_[i] - 1
+            )
+            _update_cat_count(
+                X_feature, Y, self.category_count_[i], self.class_count_.shape[0]
+            )
 
     def _update_feature_log_prob(self, alpha):
         feature_log_prob = []
@@ -1341,12 +1525,12 @@ def _update_feature_log_prob(self, alpha):
             smoothed_cat_count = self.category_count_[i] + alpha
             smoothed_class_count = smoothed_cat_count.sum(axis=1)
             feature_log_prob.append(
-                np.log(smoothed_cat_count) -
-                np.log(smoothed_class_count.reshape(-1, 1)))
+                np.log(smoothed_cat_count) - np.log(smoothed_class_count.reshape(-1, 1))
+            )
         self.feature_log_prob_ = feature_log_prob
 
     def _joint_log_likelihood(self, X):
-        self._check_n_features(X, reset=False)
+        _check_n_features(self, X, reset=False)
         jll = np.zeros((X.shape[0], self.class_count_.shape[0]))
         for i in range(self.n_features_in_):
             indices = X[:, i]
diff --git a/sklearn/neighbors/__init__.py b/sklearn/neighbors/__init__.py
index 82f9993bec50c..4e0de99f5e7e3 100644
--- a/sklearn/neighbors/__init__.py
+++ b/sklearn/neighbors/__init__.py
@@ -1,37 +1,42 @@
-"""
-The :mod:`sklearn.neighbors` module implements the k-nearest neighbors
-algorithm.
-"""
+"""The k-nearest neighbors algorithms."""
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 from ._ball_tree import BallTree
-from ._kd_tree import KDTree
-from ._dist_metrics import DistanceMetric
-from ._graph import kneighbors_graph, radius_neighbors_graph
-from ._graph import KNeighborsTransformer, RadiusNeighborsTransformer
-from ._unsupervised import NearestNeighbors
+from ._base import VALID_METRICS, VALID_METRICS_SPARSE, sort_graph_by_row_values
 from ._classification import KNeighborsClassifier, RadiusNeighborsClassifier
-from ._regression import KNeighborsRegressor, RadiusNeighborsRegressor
-from ._nearest_centroid import NearestCentroid
+from ._graph import (
+    KNeighborsTransformer,
+    RadiusNeighborsTransformer,
+    kneighbors_graph,
+    radius_neighbors_graph,
+)
+from ._kd_tree import KDTree
 from ._kde import KernelDensity
 from ._lof import LocalOutlierFactor
 from ._nca import NeighborhoodComponentsAnalysis
-from ._base import VALID_METRICS, VALID_METRICS_SPARSE
+from ._nearest_centroid import NearestCentroid
+from ._regression import KNeighborsRegressor, RadiusNeighborsRegressor
+from ._unsupervised import NearestNeighbors
 
-__all__ = ['BallTree',
-           'DistanceMetric',
-           'KDTree',
-           'KNeighborsClassifier',
-           'KNeighborsRegressor',
-           'KNeighborsTransformer',
-           'NearestCentroid',
-           'NearestNeighbors',
-           'RadiusNeighborsClassifier',
-           'RadiusNeighborsRegressor',
-           'RadiusNeighborsTransformer',
-           'kneighbors_graph',
-           'radius_neighbors_graph',
-           'KernelDensity',
-           'LocalOutlierFactor',
-           'NeighborhoodComponentsAnalysis',
-           'VALID_METRICS',
-           'VALID_METRICS_SPARSE']
+__all__ = [
+    "VALID_METRICS",
+    "VALID_METRICS_SPARSE",
+    "BallTree",
+    "KDTree",
+    "KNeighborsClassifier",
+    "KNeighborsRegressor",
+    "KNeighborsTransformer",
+    "KernelDensity",
+    "LocalOutlierFactor",
+    "NearestCentroid",
+    "NearestNeighbors",
+    "NeighborhoodComponentsAnalysis",
+    "RadiusNeighborsClassifier",
+    "RadiusNeighborsRegressor",
+    "RadiusNeighborsTransformer",
+    "kneighbors_graph",
+    "radius_neighbors_graph",
+    "sort_graph_by_row_values",
+]
diff --git a/sklearn/neighbors/_ball_tree.pyx b/sklearn/neighbors/_ball_tree.pyx
deleted file mode 100644
index 81ce9606f7b80..0000000000000
--- a/sklearn/neighbors/_ball_tree.pyx
+++ /dev/null
@@ -1,192 +0,0 @@
-#!python
-#cython: boundscheck=False
-#cython: wraparound=False
-#cython: cdivision=True
-
-# Author: Jake Vanderplas 
-# License: BSD 3 clause
-
-__all__ = ['BallTree']
-
-DOC_DICT = {'BinaryTree': 'BallTree', 'binary_tree': 'ball_tree'}
-
-VALID_METRICS = ['EuclideanDistance', 'SEuclideanDistance',
-                 'ManhattanDistance', 'ChebyshevDistance',
-                 'MinkowskiDistance', 'WMinkowskiDistance',
-                 'MahalanobisDistance', 'HammingDistance',
-                 'CanberraDistance', 'BrayCurtisDistance',
-                 'JaccardDistance', 'MatchingDistance',
-                 'DiceDistance', 'KulsinskiDistance',
-                 'RogersTanimotoDistance', 'RussellRaoDistance',
-                 'SokalMichenerDistance', 'SokalSneathDistance',
-                 'PyFuncDistance', 'HaversineDistance']
-
-
-include "_binary_tree.pxi"
-
-# Inherit BallTree from BinaryTree
-cdef class BallTree(BinaryTree):
-    __doc__ = CLASS_DOC.format(**DOC_DICT)
-    pass
-
-
-#----------------------------------------------------------------------
-# The functions below specialized the Binary Tree as a Ball Tree
-#
-#   Note that these functions use the concept of "reduced distance".
-#   The reduced distance, defined for some metrics, is a quantity which
-#   is more efficient to compute than the distance, but preserves the
-#   relative rankings of the true distance.  For example, the reduced
-#   distance for the Euclidean metric is the squared-euclidean distance.
-#   For some metrics, the reduced distance is simply the distance.
-
-cdef int allocate_data(BinaryTree tree, ITYPE_t n_nodes,
-                       ITYPE_t n_features) except -1:
-    """Allocate arrays needed for the KD Tree"""
-    tree.node_bounds_arr = np.zeros((1, n_nodes, n_features), dtype=DTYPE)
-    tree.node_bounds = get_memview_DTYPE_3D(tree.node_bounds_arr)
-    return 0
-
-
-cdef int init_node(BinaryTree tree, ITYPE_t i_node,
-                   ITYPE_t idx_start, ITYPE_t idx_end) except -1:
-    """Initialize the node for the dataset stored in tree.data"""
-    cdef ITYPE_t n_features = tree.data.shape[1]
-    cdef ITYPE_t n_points = idx_end - idx_start
-
-    cdef ITYPE_t i, j
-    cdef DTYPE_t radius
-    cdef DTYPE_t *this_pt
-
-    cdef ITYPE_t* idx_array = &tree.idx_array[0]
-    cdef DTYPE_t* data = &tree.data[0, 0]
-    cdef DTYPE_t* centroid = &tree.node_bounds[0, i_node, 0]
-
-    cdef bint with_sample_weight = tree.sample_weight is not None
-    cdef DTYPE_t* sample_weight
-    cdef DTYPE_t sum_weight_node
-    if with_sample_weight:
-        sample_weight = &tree.sample_weight[0]
-
-    # determine Node centroid
-    for j in range(n_features):
-        centroid[j] = 0
-
-    if with_sample_weight:
-        sum_weight_node = 0
-        for i in range(idx_start, idx_end):
-            sum_weight_node += sample_weight[idx_array[i]]
-            this_pt = data + n_features * idx_array[i]
-            for j from 0 <= j < n_features:
-                centroid[j] += this_pt[j] * sample_weight[idx_array[i]]
-
-        for j in range(n_features):
-            centroid[j] /= sum_weight_node
-    else:
-        for i in range(idx_start, idx_end):
-            this_pt = data + n_features * idx_array[i]
-            for j from 0 <= j < n_features:
-                centroid[j] += this_pt[j]
-
-        for j in range(n_features):
-            centroid[j] /= n_points
-
-    # determine Node radius
-    radius = 0
-    for i in range(idx_start, idx_end):
-        radius = fmax(radius,
-                      tree.rdist(centroid,
-                                 data + n_features * idx_array[i],
-                                 n_features))
-
-    tree.node_data[i_node].radius = tree.dist_metric._rdist_to_dist(radius)
-    tree.node_data[i_node].idx_start = idx_start
-    tree.node_data[i_node].idx_end = idx_end
-    return 0
-
-
-cdef inline DTYPE_t min_dist(BinaryTree tree, ITYPE_t i_node,
-                             DTYPE_t* pt) nogil except -1:
-    """Compute the minimum distance between a point and a node"""
-    cdef DTYPE_t dist_pt = tree.dist(pt, &tree.node_bounds[0, i_node, 0],
-                                     tree.data.shape[1])
-    return fmax(0, dist_pt - tree.node_data[i_node].radius)
-
-
-cdef inline DTYPE_t max_dist(BinaryTree tree, ITYPE_t i_node,
-                             DTYPE_t* pt) except -1:
-    """Compute the maximum distance between a point and a node"""
-    cdef DTYPE_t dist_pt = tree.dist(pt, &tree.node_bounds[0, i_node, 0],
-                                     tree.data.shape[1])
-    return dist_pt + tree.node_data[i_node].radius
-
-
-cdef inline int min_max_dist(BinaryTree tree, ITYPE_t i_node, DTYPE_t* pt,
-                             DTYPE_t* min_dist, DTYPE_t* max_dist) nogil except -1:
-    """Compute the minimum and maximum distance between a point and a node"""
-    cdef DTYPE_t dist_pt = tree.dist(pt, &tree.node_bounds[0, i_node, 0],
-                                     tree.data.shape[1])
-    cdef DTYPE_t rad = tree.node_data[i_node].radius
-    min_dist[0] = fmax(0, dist_pt - rad)
-    max_dist[0] = dist_pt + rad
-    return 0
-
-
-cdef inline DTYPE_t min_rdist(BinaryTree tree, ITYPE_t i_node,
-                              DTYPE_t* pt) nogil except -1:
-    """Compute the minimum reduced-distance between a point and a node"""
-    if tree.euclidean:
-        return euclidean_dist_to_rdist(min_dist(tree, i_node, pt))
-    else:
-        return tree.dist_metric._dist_to_rdist(min_dist(tree, i_node, pt))
-
-
-cdef inline DTYPE_t max_rdist(BinaryTree tree, ITYPE_t i_node,
-                              DTYPE_t* pt) except -1:
-    """Compute the maximum reduced-distance between a point and a node"""
-    if tree.euclidean:
-        return euclidean_dist_to_rdist(max_dist(tree, i_node, pt))
-    else:
-        return tree.dist_metric._dist_to_rdist(max_dist(tree, i_node, pt))
-
-
-cdef inline DTYPE_t min_dist_dual(BinaryTree tree1, ITYPE_t i_node1,
-                                  BinaryTree tree2, ITYPE_t i_node2) except -1:
-    """compute the minimum distance between two nodes"""
-    cdef DTYPE_t dist_pt = tree1.dist(&tree2.node_bounds[0, i_node2, 0],
-                                      &tree1.node_bounds[0, i_node1, 0],
-                                      tree1.data.shape[1])
-    return fmax(0, (dist_pt - tree1.node_data[i_node1].radius
-                    - tree2.node_data[i_node2].radius))
-
-
-cdef inline DTYPE_t max_dist_dual(BinaryTree tree1, ITYPE_t i_node1,
-                                  BinaryTree tree2, ITYPE_t i_node2) except -1:
-    """compute the maximum distance between two nodes"""
-    cdef DTYPE_t dist_pt = tree1.dist(&tree2.node_bounds[0, i_node2, 0],
-                                      &tree1.node_bounds[0, i_node1, 0],
-                                      tree1.data.shape[1])
-    return (dist_pt + tree1.node_data[i_node1].radius
-            + tree2.node_data[i_node2].radius)
-
-
-cdef inline DTYPE_t min_rdist_dual(BinaryTree tree1, ITYPE_t i_node1,
-                                   BinaryTree tree2, ITYPE_t i_node2) except -1:
-    """compute the minimum reduced distance between two nodes"""
-    if tree1.euclidean:
-        return euclidean_dist_to_rdist(min_dist_dual(tree1, i_node1,
-                                                     tree2, i_node2))
-    else:
-        return tree1.dist_metric._dist_to_rdist(min_dist_dual(tree1, i_node1,
-                                                              tree2, i_node2))
-
-
-cdef inline DTYPE_t max_rdist_dual(BinaryTree tree1, ITYPE_t i_node1,
-                                   BinaryTree tree2, ITYPE_t i_node2) except -1:
-    """compute the maximum reduced distance between two nodes"""
-    if tree1.euclidean:
-        return euclidean_dist_to_rdist(max_dist_dual(tree1, i_node1,
-                                                     tree2, i_node2))
-    else:
-        return tree1.dist_metric._dist_to_rdist(max_dist_dual(tree1, i_node1,
-                                                              tree2, i_node2))
diff --git a/sklearn/neighbors/_ball_tree.pyx.tp b/sklearn/neighbors/_ball_tree.pyx.tp
new file mode 100644
index 0000000000000..44d876187c54f
--- /dev/null
+++ b/sklearn/neighbors/_ball_tree.pyx.tp
@@ -0,0 +1,284 @@
+{{py:
+
+# Generated file: _ball_tree.pyx
+
+implementation_specific_values = [
+    # The values are arranged as follows:
+    #
+    #       name_suffix, INPUT_DTYPE_t, INPUT_DTYPE
+    #
+    ('64', 'float64_t', 'np.float64'),
+    ('32', 'float32_t', 'np.float32')
+]
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+}}
+
+
+__all__ = ['BallTree', 'BallTree64', 'BallTree32']
+
+{{for name_suffix, INPUT_DTYPE_t, INPUT_DTYPE in implementation_specific_values}}
+
+DOC_DICT{{name_suffix}} = {
+    'BinaryTree': 'BallTree{{name_suffix}}',
+    'binary_tree': 'ball_tree{{name_suffix}}',
+}
+
+VALID_METRICS{{name_suffix}} = [
+    'BrayCurtisDistance{{name_suffix}}',
+    'CanberraDistance{{name_suffix}}',
+    'ChebyshevDistance{{name_suffix}}',
+    'DiceDistance{{name_suffix}}',
+    'EuclideanDistance{{name_suffix}}',
+    'HammingDistance{{name_suffix}}',
+    'HaversineDistance{{name_suffix}}',
+    'JaccardDistance{{name_suffix}}',
+    'MahalanobisDistance{{name_suffix}}',
+    'ManhattanDistance{{name_suffix}}',
+    'MinkowskiDistance{{name_suffix}}',
+    'PyFuncDistance{{name_suffix}}',
+    'RogersTanimotoDistance{{name_suffix}}',
+    'RussellRaoDistance{{name_suffix}}',
+    'SEuclideanDistance{{name_suffix}}',
+    'SokalMichenerDistance{{name_suffix}}',
+    'SokalSneathDistance{{name_suffix}}',
+    'WMinkowskiDistance{{name_suffix}}',
+]
+
+{{endfor}}
+
+include "_binary_tree.pxi"
+
+{{for name_suffix, INPUT_DTYPE_t, INPUT_DTYPE in implementation_specific_values}}
+
+# Inherit BallTree{{name_suffix}} from BinaryTree{{name_suffix}}
+cdef class BallTree{{name_suffix}}(BinaryTree{{name_suffix}}):
+    __doc__ = CLASS_DOC.format(**DOC_DICT{{name_suffix}})
+    pass
+
+{{endfor}}
+
+
+#----------------------------------------------------------------------
+# The functions below specialized the Binary Tree as a Ball Tree
+#
+#   Note that these functions use the concept of "reduced distance".
+#   The reduced distance, defined for some metrics, is a quantity which
+#   is more efficient to compute than the distance, but preserves the
+#   relative rankings of the true distance.  For example, the reduced
+#   distance for the Euclidean metric is the squared-euclidean distance.
+#   For some metrics, the reduced distance is simply the distance.
+
+{{for name_suffix, INPUT_DTYPE_t, INPUT_DTYPE in implementation_specific_values}}
+
+cdef int allocate_data{{name_suffix}}(
+    BinaryTree{{name_suffix}} tree,
+    intp_t n_nodes,
+    intp_t n_features,
+) except -1:
+    """Allocate arrays needed for the KD Tree"""
+    tree.node_bounds = np.zeros((1, n_nodes, n_features), dtype={{INPUT_DTYPE}})
+    return 0
+
+
+cdef int init_node{{name_suffix}}(
+    BinaryTree{{name_suffix}} tree,
+    NodeData_t[::1] node_data,
+    intp_t i_node,
+    intp_t idx_start,
+    intp_t idx_end,
+) except -1:
+    """Initialize the node for the dataset stored in tree.data"""
+    cdef intp_t n_features = tree.data.shape[1]
+    cdef intp_t n_points = idx_end - idx_start
+
+    cdef intp_t i, j
+    cdef float64_t radius
+    cdef const {{INPUT_DTYPE_t}} *this_pt
+
+    cdef intp_t* idx_array = &tree.idx_array[0]
+    cdef const {{INPUT_DTYPE_t}}* data = &tree.data[0, 0]
+    cdef {{INPUT_DTYPE_t}}* centroid = &tree.node_bounds[0, i_node, 0]
+
+    cdef bint with_sample_weight = tree.sample_weight is not None
+    cdef const {{INPUT_DTYPE_t}}* sample_weight
+    cdef float64_t sum_weight_node
+    if with_sample_weight:
+        sample_weight = &tree.sample_weight[0]
+
+    # determine Node centroid
+    for j in range(n_features):
+        centroid[j] = 0
+
+    if with_sample_weight:
+        sum_weight_node = 0
+        for i in range(idx_start, idx_end):
+            sum_weight_node += sample_weight[idx_array[i]]
+            this_pt = data + n_features * idx_array[i]
+            for j from 0 <= j < n_features:
+                centroid[j] += this_pt[j] * sample_weight[idx_array[i]]
+
+        for j in range(n_features):
+            centroid[j] /= sum_weight_node
+    else:
+        for i in range(idx_start, idx_end):
+            this_pt = data + n_features * idx_array[i]
+            for j from 0 <= j < n_features:
+                centroid[j] += this_pt[j]
+
+        for j in range(n_features):
+            centroid[j] /= n_points
+
+    # determine Node radius
+    radius = 0
+    for i in range(idx_start, idx_end):
+        radius = fmax(radius,
+                      tree.rdist(centroid,
+                                 data + n_features * idx_array[i],
+                                 n_features))
+
+    node_data[i_node].radius = tree.dist_metric._rdist_to_dist(radius)
+    node_data[i_node].idx_start = idx_start
+    node_data[i_node].idx_end = idx_end
+    return 0
+
+
+cdef inline float64_t min_dist{{name_suffix}}(
+    BinaryTree{{name_suffix}} tree,
+    intp_t i_node,
+    const {{INPUT_DTYPE_t}}* pt,
+) except -1 nogil:
+    """Compute the minimum distance between a point and a node"""
+    cdef float64_t dist_pt = tree.dist(pt, &tree.node_bounds[0, i_node, 0],
+                                     tree.data.shape[1])
+    return fmax(0, dist_pt - tree.node_data[i_node].radius)
+
+
+cdef inline float64_t max_dist{{name_suffix}}(
+    BinaryTree{{name_suffix}} tree,
+    intp_t i_node,
+    const {{INPUT_DTYPE_t}}* pt,
+) except -1:
+    """Compute the maximum distance between a point and a node"""
+    cdef float64_t dist_pt = tree.dist(pt, &tree.node_bounds[0, i_node, 0],
+                                     tree.data.shape[1])
+    return dist_pt + tree.node_data[i_node].radius
+
+
+cdef inline int min_max_dist{{name_suffix}}(
+    BinaryTree{{name_suffix}} tree,
+    intp_t i_node,
+    const {{INPUT_DTYPE_t}}* pt,
+    float64_t* min_dist,
+    float64_t* max_dist,
+) except -1 nogil:
+    """Compute the minimum and maximum distance between a point and a node"""
+    cdef float64_t dist_pt = tree.dist(pt, &tree.node_bounds[0, i_node, 0],
+                                     tree.data.shape[1])
+    cdef float64_t rad = tree.node_data[i_node].radius
+    min_dist[0] = fmax(0, dist_pt - rad)
+    max_dist[0] = dist_pt + rad
+    return 0
+
+
+cdef inline float64_t min_rdist{{name_suffix}}(
+    BinaryTree{{name_suffix}} tree,
+    intp_t i_node,
+    const {{INPUT_DTYPE_t}}* pt,
+) except -1 nogil:
+    """Compute the minimum reduced-distance between a point and a node"""
+    if tree.euclidean:
+        return euclidean_dist_to_rdist{{name_suffix}}(
+            min_dist{{name_suffix}}(tree, i_node, pt)
+        )
+    else:
+        return tree.dist_metric._dist_to_rdist(
+            min_dist{{name_suffix}}(tree, i_node, pt)
+        )
+
+
+cdef inline float64_t max_rdist{{name_suffix}}(
+    BinaryTree{{name_suffix}} tree,
+    intp_t i_node,
+    const {{INPUT_DTYPE_t}}* pt,
+) except -1:
+    """Compute the maximum reduced-distance between a point and a node"""
+    if tree.euclidean:
+        return euclidean_dist_to_rdist{{name_suffix}}(
+            max_dist{{name_suffix}}(tree, i_node, pt)
+        )
+    else:
+        return tree.dist_metric._dist_to_rdist(
+            max_dist{{name_suffix}}(tree, i_node, pt)
+        )
+
+
+cdef inline float64_t min_dist_dual{{name_suffix}}(
+    BinaryTree{{name_suffix}} tree1,
+    intp_t i_node1,
+    BinaryTree{{name_suffix}} tree2,
+    intp_t i_node2,
+) except -1:
+    """compute the minimum distance between two nodes"""
+    cdef float64_t dist_pt = tree1.dist(&tree2.node_bounds[0, i_node2, 0],
+                                      &tree1.node_bounds[0, i_node1, 0],
+                                      tree1.data.shape[1])
+    return fmax(0, (dist_pt - tree1.node_data[i_node1].radius
+                    - tree2.node_data[i_node2].radius))
+
+
+cdef inline float64_t max_dist_dual{{name_suffix}}(
+    BinaryTree{{name_suffix}} tree1,
+    intp_t i_node1,
+    BinaryTree{{name_suffix}} tree2,
+    intp_t i_node2,
+) except -1:
+    """compute the maximum distance between two nodes"""
+    cdef float64_t dist_pt = tree1.dist(&tree2.node_bounds[0, i_node2, 0],
+                                      &tree1.node_bounds[0, i_node1, 0],
+                                      tree1.data.shape[1])
+    return (dist_pt + tree1.node_data[i_node1].radius
+            + tree2.node_data[i_node2].radius)
+
+
+cdef inline float64_t min_rdist_dual{{name_suffix}}(
+    BinaryTree{{name_suffix}} tree1,
+    intp_t i_node1,
+    BinaryTree{{name_suffix}} tree2,
+    intp_t i_node2,
+) except -1:
+    """compute the minimum reduced distance between two nodes"""
+    if tree1.euclidean:
+        return euclidean_dist_to_rdist{{name_suffix}}(
+            min_dist_dual{{name_suffix}}(tree1, i_node1, tree2, i_node2)
+        )
+    else:
+        return tree1.dist_metric._dist_to_rdist(
+            min_dist_dual{{name_suffix}}(tree1, i_node1, tree2, i_node2)
+        )
+
+
+cdef inline float64_t max_rdist_dual{{name_suffix}}(
+    BinaryTree{{name_suffix}} tree1,
+    intp_t i_node1,
+    BinaryTree{{name_suffix}} tree2,
+    intp_t i_node2,
+) except -1:
+    """compute the maximum reduced distance between two nodes"""
+    if tree1.euclidean:
+        return euclidean_dist_to_rdist{{name_suffix}}(
+            max_dist_dual{{name_suffix}}(tree1, i_node1, tree2, i_node2)
+        )
+    else:
+        return tree1.dist_metric._dist_to_rdist(
+            max_dist_dual{{name_suffix}}(tree1, i_node1, tree2, i_node2)
+        )
+
+{{endfor}}
+
+
+class BallTree(BallTree64):
+    __doc__ = CLASS_DOC.format(BinaryTree="BallTree")
+    pass
diff --git a/sklearn/neighbors/_base.py b/sklearn/neighbors/_base.py
index 820b83eca1845..767eee1358aa8 100644
--- a/sklearn/neighbors/_base.py
+++ b/sklearn/neighbors/_base.py
@@ -1,81 +1,94 @@
-"""Base and mixin classes for nearest neighbors"""
-# Authors: Jake Vanderplas 
-#          Fabian Pedregosa 
-#          Alexandre Gramfort 
-#          Sparseness support by Lars Buitinck
-#          Multi-output support by Arnaud Joly 
-#
-# License: BSD 3 clause (C) INRIA, University of Amsterdam
-from functools import partial
+"""Base and mixin classes for nearest neighbors."""
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
+import itertools
+import numbers
 import warnings
 from abc import ABCMeta, abstractmethod
-import numbers
+from functools import partial
+from numbers import Integral, Real
 
 import numpy as np
+from joblib import effective_n_jobs
 from scipy.sparse import csr_matrix, issparse
-import joblib
-from joblib import Parallel, effective_n_jobs
 
-from ._ball_tree import BallTree
-from ._kd_tree import KDTree
-from ..base import BaseEstimator, MultiOutputMixin
-from ..base import is_classifier
-from ..metrics import pairwise_distances_chunked
+from ..base import BaseEstimator, MultiOutputMixin, is_classifier
+from ..exceptions import DataConversionWarning, EfficiencyWarning
+from ..metrics import DistanceMetric, pairwise_distances_chunked
+from ..metrics._pairwise_distances_reduction import (
+    ArgKmin,
+    RadiusNeighbors,
+)
 from ..metrics.pairwise import PAIRWISE_DISTANCE_FUNCTIONS
 from ..utils import (
     check_array,
     gen_even_slices,
-    _to_object_array,
+    get_tags,
 )
-from ..utils.deprecation import deprecated
+from ..utils._param_validation import Interval, StrOptions, validate_params
+from ..utils.fixes import parse_version, sp_base_version
 from ..utils.multiclass import check_classification_targets
-from ..utils.validation import check_is_fitted
-from ..utils.validation import check_non_negative
-from ..utils.fixes import delayed
-from ..utils.fixes import parse_version
-from ..exceptions import DataConversionWarning, EfficiencyWarning
+from ..utils.parallel import Parallel, delayed
+from ..utils.validation import _to_object_array, check_is_fitted, validate_data
+from ._ball_tree import BallTree
+from ._kd_tree import KDTree
 
-VALID_METRICS = dict(ball_tree=BallTree.valid_metrics,
-                     kd_tree=KDTree.valid_metrics,
-                     # The following list comes from the
-                     # sklearn.metrics.pairwise doc string
-                     brute=(list(PAIRWISE_DISTANCE_FUNCTIONS.keys()) +
-                            ['braycurtis', 'canberra', 'chebyshev',
-                             'correlation', 'cosine', 'dice', 'hamming',
-                             'jaccard', 'kulsinski', 'mahalanobis',
-                             'matching', 'minkowski', 'rogerstanimoto',
-                             'russellrao', 'seuclidean', 'sokalmichener',
-                             'sokalsneath', 'sqeuclidean',
-                             'yule', 'wminkowski']))
-
-
-VALID_METRICS_SPARSE = dict(ball_tree=[],
-                            kd_tree=[],
-                            brute=(PAIRWISE_DISTANCE_FUNCTIONS.keys() -
-                                   {'haversine', 'nan_euclidean'}))
-
-
-def _check_weights(weights):
-    """Check to make sure weights are valid"""
-    if weights in (None, 'uniform', 'distance'):
-        return weights
-    elif callable(weights):
-        return weights
-    else:
-        raise ValueError("weights not recognized: should be 'uniform', "
-                         "'distance', or a callable function")
+SCIPY_METRICS = [
+    "braycurtis",
+    "canberra",
+    "chebyshev",
+    "correlation",
+    "cosine",
+    "dice",
+    "hamming",
+    "jaccard",
+    "mahalanobis",
+    "minkowski",
+    "rogerstanimoto",
+    "russellrao",
+    "seuclidean",
+    "sokalsneath",
+    "sqeuclidean",
+    "yule",
+]
+if sp_base_version < parse_version("1.17"):
+    # Deprecated in SciPy 1.15 and removed in SciPy 1.17
+    SCIPY_METRICS += ["sokalmichener"]
+if sp_base_version < parse_version("1.11"):
+    # Deprecated in SciPy 1.9 and removed in SciPy 1.11
+    SCIPY_METRICS += ["kulsinski"]
+if sp_base_version < parse_version("1.9"):
+    # Deprecated in SciPy 1.0 and removed in SciPy 1.9
+    SCIPY_METRICS += ["matching"]
+
+VALID_METRICS = dict(
+    ball_tree=BallTree.valid_metrics,
+    kd_tree=KDTree.valid_metrics,
+    # The following list comes from the
+    # sklearn.metrics.pairwise doc string
+    brute=sorted(set(PAIRWISE_DISTANCE_FUNCTIONS).union(SCIPY_METRICS)),
+)
+
+VALID_METRICS_SPARSE = dict(
+    ball_tree=[],
+    kd_tree=[],
+    brute=(PAIRWISE_DISTANCE_FUNCTIONS.keys() - {"haversine", "nan_euclidean"}),
+)
 
 
 def _get_weights(dist, weights):
-    """Get the weights from an array of distances and a parameter ``weights``
+    """Get the weights from an array of distances and a parameter ``weights``.
+
+    Assume weights have already been validated.
 
     Parameters
     ----------
     dist : ndarray
         The input distances.
 
-    weights : {'uniform', 'distance' or a callable}
+    weights : {'uniform', 'distance'}, callable or None
         The kind of weighting used.
 
     Returns
@@ -83,9 +96,10 @@ def _get_weights(dist, weights):
     weights_arr : array of the same shape as ``dist``
         If ``weights == 'uniform'``, then returns None.
     """
-    if weights in (None, 'uniform'):
+    if weights in (None, "uniform"):
         return None
-    elif weights == 'distance':
+
+    if weights == "distance":
         # if user attempts to classify a point that was zero distance from one
         # or more training points, those training points are weighted as 1.0
         # and the other points as 0.0
@@ -94,26 +108,24 @@ def _get_weights(dist, weights):
                 # check if point_dist is iterable
                 # (ex: RadiusNeighborClassifier.predict may set an element of
                 # dist to 1e-6 to represent an 'outlier')
-                if hasattr(point_dist, '__contains__') and 0. in point_dist:
-                    dist[point_dist_i] = point_dist == 0.
+                if hasattr(point_dist, "__contains__") and 0.0 in point_dist:
+                    dist[point_dist_i] = point_dist == 0.0
                 else:
-                    dist[point_dist_i] = 1. / point_dist
+                    dist[point_dist_i] = 1.0 / point_dist
         else:
-            with np.errstate(divide='ignore'):
-                dist = 1. / dist
+            with np.errstate(divide="ignore"):
+                dist = 1.0 / dist
             inf_mask = np.isinf(dist)
             inf_row = np.any(inf_mask, axis=1)
             dist[inf_row] = inf_mask[inf_row]
         return dist
-    elif callable(weights):
+
+    if callable(weights):
         return weights(dist)
-    else:
-        raise ValueError("weights not recognized: should be 'uniform', "
-                         "'distance', or a callable function")
 
 
 def _is_sorted_by_data(graph):
-    """Returns whether the graph's non-zero entries are sorted by data
+    """Return whether the graph's non-zero entries are sorted by data.
 
     The non-zero entries are stored in graph.data and graph.indices.
     For each row (or sample), the non-zero entries can be either:
@@ -132,15 +144,15 @@ def _is_sorted_by_data(graph):
     res : bool
         Whether input graph is sorted by data.
     """
-    assert graph.format == 'csr'
+    assert graph.format == "csr"
     out_of_order = graph.data[:-1] > graph.data[1:]
     line_change = np.unique(graph.indptr[1:-1] - 1)
     line_change = line_change[line_change < out_of_order.shape[0]]
-    return (out_of_order.sum() == out_of_order[line_change].sum())
+    return out_of_order.sum() == out_of_order[line_change].sum()
 
 
 def _check_precomputed(X):
-    """Check precomputed distance matrix
+    """Check precomputed distance matrix.
 
     If the precomputed distance matrix is sparse, it checks that the non-zero
     entries are sorted by distances. If not, the matrix is copied and sorted.
@@ -158,47 +170,129 @@ def _check_precomputed(X):
         case only non-zero elements may be considered neighbors.
     """
     if not issparse(X):
-        X = check_array(X)
-        check_non_negative(X, whom="precomputed distance matrix.")
+        X = check_array(X, ensure_non_negative=True, input_name="X")
         return X
     else:
         graph = X
 
-    if graph.format not in ('csr', 'csc', 'coo', 'lil'):
-        raise TypeError('Sparse matrix in {!r} format is not supported due to '
-                        'its handling of explicit zeros'.format(graph.format))
-    copied = graph.format != 'csr'
-    graph = check_array(graph, accept_sparse='csr')
-    check_non_negative(graph, whom="precomputed distance matrix.")
+    if graph.format not in ("csr", "csc", "coo", "lil"):
+        raise TypeError(
+            "Sparse matrix in {!r} format is not supported due to "
+            "its handling of explicit zeros".format(graph.format)
+        )
+    copied = graph.format != "csr"
+    graph = check_array(
+        graph,
+        accept_sparse="csr",
+        ensure_non_negative=True,
+        input_name="precomputed distance matrix",
+    )
+    graph = sort_graph_by_row_values(graph, copy=not copied, warn_when_not_sorted=True)
+
+    return graph
+
+
+@validate_params(
+    {
+        "graph": ["sparse matrix"],
+        "copy": ["boolean"],
+        "warn_when_not_sorted": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def sort_graph_by_row_values(graph, copy=False, warn_when_not_sorted=True):
+    """Sort a sparse graph such that each row is stored with increasing values.
+
+    .. versionadded:: 1.2
 
-    if not _is_sorted_by_data(graph):
-        warnings.warn('Precomputed sparse input was not sorted by data.',
-                      EfficiencyWarning)
-        if not copied:
-            graph = graph.copy()
+    Parameters
+    ----------
+    graph : sparse matrix of shape (n_samples, n_samples)
+        Distance matrix to other samples, where only non-zero elements are
+        considered neighbors. Matrix is converted to CSR format if not already.
+
+    copy : bool, default=False
+        If True, the graph is copied before sorting. If False, the sorting is
+        performed inplace. If the graph is not of CSR format, `copy` must be
+        True to allow the conversion to CSR format, otherwise an error is
+        raised.
+
+    warn_when_not_sorted : bool, default=True
+        If True, a :class:`~sklearn.exceptions.EfficiencyWarning` is raised
+        when the input graph is not sorted by row values.
+
+    Returns
+    -------
+    graph : sparse matrix of shape (n_samples, n_samples)
+        Distance matrix to other samples, where only non-zero elements are
+        considered neighbors. Matrix is in CSR format.
+
+    Examples
+    --------
+    >>> from scipy.sparse import csr_matrix
+    >>> from sklearn.neighbors import sort_graph_by_row_values
+    >>> X = csr_matrix(
+    ...     [[0., 3., 1.],
+    ...      [3., 0., 2.],
+    ...      [1., 2., 0.]])
+    >>> X.data
+    array([3., 1., 3., 2., 1., 2.])
+    >>> X_ = sort_graph_by_row_values(X)
+    >>> X_.data
+    array([1., 3., 2., 3., 1., 2.])
+    """
+    if graph.format == "csr" and _is_sorted_by_data(graph):
+        return graph
+
+    if warn_when_not_sorted:
+        warnings.warn(
+            (
+                "Precomputed sparse input was not sorted by row values. Use the"
+                " function sklearn.neighbors.sort_graph_by_row_values to sort the input"
+                " by row values, with warn_when_not_sorted=False to remove this"
+                " warning."
+            ),
+            EfficiencyWarning,
+        )
+
+    if graph.format not in ("csr", "csc", "coo", "lil"):
+        raise TypeError(
+            f"Sparse matrix in {graph.format!r} format is not supported due to "
+            "its handling of explicit zeros"
+        )
+    elif graph.format != "csr":
+        if not copy:
+            raise ValueError(
+                "The input graph is not in CSR format. Use copy=True to allow "
+                "the conversion to CSR format."
+            )
+        graph = graph.asformat("csr")
+    elif copy:  # csr format with copy=True
+        graph = graph.copy()
 
+    row_nnz = np.diff(graph.indptr)
+    if row_nnz.max() == row_nnz.min():
         # if each sample has the same number of provided neighbors
-        row_nnz = np.diff(graph.indptr)
-        if row_nnz.max() == row_nnz.min():
-            n_samples = graph.shape[0]
-            distances = graph.data.reshape(n_samples, -1)
+        n_samples = graph.shape[0]
+        distances = graph.data.reshape(n_samples, -1)
 
-            order = np.argsort(distances, kind='mergesort')
-            order += np.arange(n_samples)[:, None] * row_nnz[0]
-            order = order.ravel()
-            graph.data = graph.data[order]
-            graph.indices = graph.indices[order]
+        order = np.argsort(distances, kind="mergesort")
+        order += np.arange(n_samples)[:, None] * row_nnz[0]
+        order = order.ravel()
+        graph.data = graph.data[order]
+        graph.indices = graph.indices[order]
+
+    else:
+        for start, stop in zip(graph.indptr, graph.indptr[1:]):
+            order = np.argsort(graph.data[start:stop], kind="mergesort")
+            graph.data[start:stop] = graph.data[start:stop][order]
+            graph.indices[start:stop] = graph.indices[start:stop][order]
 
-        else:
-            for start, stop in zip(graph.indptr, graph.indptr[1:]):
-                order = np.argsort(graph.data[start:stop], kind='mergesort')
-                graph.data[start:stop] = graph.data[start:stop][order]
-                graph.indices[start:stop] = graph.indices[start:stop][order]
     return graph
 
 
 def _kneighbors_from_graph(graph, n_neighbors, return_distance):
-    """Decompose a nearest neighbors sparse graph into distances and indices
+    """Decompose a nearest neighbors sparse graph into distances and indices.
 
     Parameters
     ----------
@@ -221,17 +315,18 @@ def _kneighbors_from_graph(graph, n_neighbors, return_distance):
         Indices of nearest neighbors.
     """
     n_samples = graph.shape[0]
-    assert graph.format == 'csr'
+    assert graph.format == "csr"
 
     # number of neighbors by samples
     row_nnz = np.diff(graph.indptr)
     row_nnz_min = row_nnz.min()
     if n_neighbors is not None and row_nnz_min < n_neighbors:
         raise ValueError(
-            '%d neighbors per samples are required, but some samples have only'
-            ' %d neighbors in precomputed graph matrix. Decrease number of '
-            'neighbors used or recompute the graph with more neighbors.'
-            % (n_neighbors, row_nnz_min))
+            "%d neighbors per samples are required, but some samples have only"
+            " %d neighbors in precomputed graph matrix. Decrease number of "
+            "neighbors used or recompute the graph with more neighbors."
+            % (n_neighbors, row_nnz_min)
+        )
 
     def extract(a):
         # if each sample has the same number of provided neighbors
@@ -240,7 +335,7 @@ def extract(a):
         else:
             idx = np.tile(np.arange(n_neighbors), (n_samples, 1))
             idx += graph.indptr[:-1, None]
-            return a.take(idx, mode='clip').reshape(n_samples, n_neighbors)
+            return a.take(idx, mode="clip").reshape(n_samples, n_neighbors)
 
     if return_distance:
         return extract(graph.data), extract(graph.indices)
@@ -249,7 +344,7 @@ def extract(a):
 
 
 def _radius_neighbors_from_graph(graph, radius, return_distance):
-    """Decompose a nearest neighbors sparse graph into distances and indices
+    """Decompose a nearest neighbors sparse graph into distances and indices.
 
     Parameters
     ----------
@@ -271,7 +366,7 @@ def _radius_neighbors_from_graph(graph, radius, return_distance):
     neigh_ind : ndarray of shape (n_samples,) of arrays
         Indices of nearest neighbors.
     """
-    assert graph.format == 'csr'
+    assert graph.format == "csr"
 
     no_filter_needed = bool(graph.data.max() <= radius)
 
@@ -299,11 +394,29 @@ def _radius_neighbors_from_graph(graph, radius, return_distance):
 class NeighborsBase(MultiOutputMixin, BaseEstimator, metaclass=ABCMeta):
     """Base class for nearest neighbors estimators."""
 
-    @abstractmethod
-    def __init__(self, n_neighbors=None, radius=None,
-                 algorithm='auto', leaf_size=30, metric='minkowski',
-                 p=2, metric_params=None, n_jobs=None):
+    _parameter_constraints: dict = {
+        "n_neighbors": [Interval(Integral, 1, None, closed="left"), None],
+        "radius": [Interval(Real, 0, None, closed="both"), None],
+        "algorithm": [StrOptions({"auto", "ball_tree", "kd_tree", "brute"})],
+        "leaf_size": [Interval(Integral, 1, None, closed="left")],
+        "p": [Interval(Real, 0, None, closed="right"), None],
+        "metric": [StrOptions(set(itertools.chain(*VALID_METRICS.values()))), callable],
+        "metric_params": [dict, None],
+        "n_jobs": [Integral, None],
+    }
 
+    @abstractmethod
+    def __init__(
+        self,
+        n_neighbors=None,
+        radius=None,
+        algorithm="auto",
+        leaf_size=30,
+        metric="minkowski",
+        p=2,
+        metric_params=None,
+        n_jobs=None,
+    ):
         self.n_neighbors = n_neighbors
         self.radius = radius
         self.algorithm = algorithm
@@ -312,66 +425,80 @@ def __init__(self, n_neighbors=None, radius=None,
         self.metric_params = metric_params
         self.p = p
         self.n_jobs = n_jobs
-        self._check_algorithm_metric()
 
     def _check_algorithm_metric(self):
-        if self.algorithm not in ['auto', 'brute',
-                                  'kd_tree', 'ball_tree']:
-            raise ValueError("unrecognized algorithm: '%s'" % self.algorithm)
-
-        if self.algorithm == 'auto':
-            if self.metric == 'precomputed':
-                alg_check = 'brute'
-            elif (callable(self.metric) or
-                  self.metric in VALID_METRICS['ball_tree']):
-                alg_check = 'ball_tree'
+        if self.algorithm == "auto":
+            if self.metric == "precomputed":
+                alg_check = "brute"
+            elif (
+                callable(self.metric)
+                or self.metric in VALID_METRICS["ball_tree"]
+                or isinstance(self.metric, DistanceMetric)
+            ):
+                alg_check = "ball_tree"
             else:
-                alg_check = 'brute'
+                alg_check = "brute"
         else:
             alg_check = self.algorithm
 
         if callable(self.metric):
-            if self.algorithm == 'kd_tree':
+            if self.algorithm == "kd_tree":
                 # callable metric is only valid for brute force and ball_tree
                 raise ValueError(
                     "kd_tree does not support callable metric '%s'"
                     "Function call overhead will result"
-                    "in very poor performance."
-                    % self.metric)
-        elif self.metric not in VALID_METRICS[alg_check]:
-            raise ValueError("Metric '%s' not valid. Use "
-                             "sorted(sklearn.neighbors.VALID_METRICS['%s']) "
-                             "to get valid options. "
-                             "Metric can also be a callable function."
-                             % (self.metric, alg_check))
-
-        if self.metric_params is not None and 'p' in self.metric_params:
-            if self.p is not None:
-                warnings.warn("Parameter p is found in metric_params. "
-                              "The corresponding parameter from __init__ "
-                              "is ignored.", SyntaxWarning, stacklevel=3)
-            effective_p = self.metric_params['p']
-        else:
-            effective_p = self.p
+                    "in very poor performance." % self.metric
+                )
+        elif self.metric not in VALID_METRICS[alg_check] and not isinstance(
+            self.metric, DistanceMetric
+        ):
+            raise ValueError(
+                "Metric '%s' not valid. Use "
+                "sorted(sklearn.neighbors.VALID_METRICS['%s']) "
+                "to get valid options. "
+                "Metric can also be a callable function." % (self.metric, alg_check)
+            )
 
-        if self.metric in ['wminkowski', 'minkowski'] and effective_p < 1:
-            raise ValueError("p must be greater than one for minkowski metric")
+        if self.metric_params is not None and "p" in self.metric_params:
+            if self.p is not None:
+                warnings.warn(
+                    (
+                        "Parameter p is found in metric_params. "
+                        "The corresponding parameter from __init__ "
+                        "is ignored."
+                    ),
+                    SyntaxWarning,
+                    stacklevel=3,
+                )
 
     def _fit(self, X, y=None):
-        if self._get_tags()["requires_y"]:
+        ensure_all_finite = "allow-nan" if get_tags(self).input_tags.allow_nan else True
+        if self.__sklearn_tags__().target_tags.required:
             if not isinstance(X, (KDTree, BallTree, NeighborsBase)):
-                X, y = self._validate_data(X, y, accept_sparse="csr",
-                                           multi_output=True)
+                X, y = validate_data(
+                    self,
+                    X,
+                    y,
+                    accept_sparse="csr",
+                    multi_output=True,
+                    order="C",
+                    ensure_all_finite=ensure_all_finite,
+                )
 
             if is_classifier(self):
                 # Classification targets require a specific format
-                if y.ndim == 1 or y.ndim == 2 and y.shape[1] == 1:
+                if y.ndim == 1 or (y.ndim == 2 and y.shape[1] == 1):
                     if y.ndim != 1:
-                        warnings.warn("A column-vector y was passed when a "
-                                      "1d array was expected. Please change "
-                                      "the shape of y to (n_samples,), for "
-                                      "example using ravel().",
-                                      DataConversionWarning, stacklevel=2)
+                        warnings.warn(
+                            (
+                                "A column-vector y was passed when a "
+                                "1d array was expected. Please change "
+                                "the shape of y to (n_samples,), for "
+                                "example using ravel()."
+                            ),
+                            DataConversionWarning,
+                            stacklevel=2,
+                        )
 
                     self.outputs_2d_ = False
                     y = y.reshape((-1, 1))
@@ -380,10 +507,12 @@ def _fit(self, X, y=None):
 
                 check_classification_targets(y)
                 self.classes_ = []
-                self._y = np.empty(y.shape, dtype=int)
+                # Using `dtype=np.intp` is necessary since `np.bincount`
+                # (called in _classification.py) fails when dealing
+                # with a float64 array on 32bit systems.
+                self._y = np.empty(y.shape, dtype=np.intp)
                 for k in range(self._y.shape[1]):
-                    classes, self._y[:, k] = np.unique(
-                        y[:, k], return_inverse=True)
+                    classes, self._y[:, k] = np.unique(y[:, k], return_inverse=True)
                     self.classes_.append(classes)
 
                 if not self.outputs_2d_:
@@ -394,7 +523,13 @@ def _fit(self, X, y=None):
 
         else:
             if not isinstance(X, (KDTree, BallTree, NeighborsBase)):
-                X = self._validate_data(X, accept_sparse='csr')
+                X = validate_data(
+                    self,
+                    X,
+                    ensure_all_finite=ensure_all_finite,
+                    accept_sparse="csr",
+                    order="C",
+                )
 
         self._check_algorithm_metric()
         if self.metric_params is None:
@@ -402,25 +537,26 @@ def _fit(self, X, y=None):
         else:
             self.effective_metric_params_ = self.metric_params.copy()
 
-        effective_p = self.effective_metric_params_.get('p', self.p)
-        if self.metric in ['wminkowski', 'minkowski']:
-            self.effective_metric_params_['p'] = effective_p
+        effective_p = self.effective_metric_params_.get("p", self.p)
+        if self.metric == "minkowski":
+            self.effective_metric_params_["p"] = effective_p
 
         self.effective_metric_ = self.metric
         # For minkowski distance, use more efficient methods where available
-        if self.metric == 'minkowski':
-            p = self.effective_metric_params_.pop('p', 2)
-            if p < 1:
-                raise ValueError("p must be greater than one "
-                                 "for minkowski metric")
-            elif p == 1:
-                self.effective_metric_ = 'manhattan'
-            elif p == 2:
-                self.effective_metric_ = 'euclidean'
-            elif p == np.inf:
-                self.effective_metric_ = 'chebyshev'
+        if self.metric == "minkowski":
+            p = self.effective_metric_params_.pop("p", 2)
+            w = self.effective_metric_params_.pop("w", None)
+
+            if p == 1 and w is None:
+                self.effective_metric_ = "manhattan"
+            elif p == 2 and w is None:
+                self.effective_metric_ = "euclidean"
+            elif p == np.inf and w is None:
+                self.effective_metric_ = "chebyshev"
             else:
-                self.effective_metric_params_['p'] = p
+                # Use the generic minkowski metric, possibly weighted.
+                self.effective_metric_params_["p"] = p
+                self.effective_metric_params_["w"] = w
 
         if isinstance(X, NeighborsBase):
             self._fit_X = X._fit_X
@@ -432,46 +568,50 @@ def _fit(self, X, y=None):
         elif isinstance(X, BallTree):
             self._fit_X = X.data
             self._tree = X
-            self._fit_method = 'ball_tree'
+            self._fit_method = "ball_tree"
             self.n_samples_fit_ = X.data.shape[0]
             return self
 
         elif isinstance(X, KDTree):
             self._fit_X = X.data
             self._tree = X
-            self._fit_method = 'kd_tree'
+            self._fit_method = "kd_tree"
             self.n_samples_fit_ = X.data.shape[0]
             return self
 
-        if self.effective_metric_ == 'precomputed':
+        if self.metric == "precomputed":
             X = _check_precomputed(X)
+            # Precomputed matrix X must be squared
+            if X.shape[0] != X.shape[1]:
+                raise ValueError(
+                    "Precomputed matrix must be square."
+                    " Input is a {}x{} matrix.".format(X.shape[0], X.shape[1])
+                )
             self.n_features_in_ = X.shape[1]
 
         n_samples = X.shape[0]
         if n_samples == 0:
             raise ValueError("n_samples must be greater than 0")
 
-        # Precomputed matrix X must be squared
-        if self.metric == 'precomputed' and X.shape[0] != X.shape[1]:
-            raise ValueError("Precomputed matrix must be a square matrix."
-                             " Input is a {}x{} matrix."
-                             .format(X.shape[0], X.shape[1]))
-
         if issparse(X):
-            if self.algorithm not in ('auto', 'brute'):
-                warnings.warn("cannot use tree with sparse input: "
-                              "using brute force")
-            if self.effective_metric_ not in VALID_METRICS_SPARSE['brute'] \
-                    and not callable(self.effective_metric_):
-                raise ValueError("Metric '%s' not valid for sparse input. "
-                                 "Use sorted(sklearn.neighbors."
-                                 "VALID_METRICS_SPARSE['brute']) "
-                                 "to get valid options. "
-                                 "Metric can also be a callable function."
-                                 % (self.effective_metric_))
+            if self.algorithm not in ("auto", "brute"):
+                warnings.warn("cannot use tree with sparse input: using brute force")
+
+            if (
+                self.effective_metric_ not in VALID_METRICS_SPARSE["brute"]
+                and not callable(self.effective_metric_)
+                and not isinstance(self.effective_metric_, DistanceMetric)
+            ):
+                raise ValueError(
+                    "Metric '%s' not valid for sparse input. "
+                    "Use sorted(sklearn.neighbors."
+                    "VALID_METRICS_SPARSE['brute']) "
+                    "to get valid options. "
+                    "Metric can also be a callable function." % (self.effective_metric_)
+                )
             self._fit_X = X.copy()
             self._tree = None
-            self._fit_method = 'brute'
+            self._fit_method = "brute"
             self.n_samples_fit_ = X.shape[0]
             return self
 
@@ -479,80 +619,108 @@ def _fit(self, X, y=None):
         self._fit_X = X
         self.n_samples_fit_ = X.shape[0]
 
-        if self._fit_method == 'auto':
+        if self._fit_method == "auto":
             # A tree approach is better for small number of neighbors or small
             # number of features, with KDTree generally faster when available
-            if (self.metric == 'precomputed' or self._fit_X.shape[1] > 15 or
-                    (self.n_neighbors is not None and
-                     self.n_neighbors >= self._fit_X.shape[0] // 2)):
-                self._fit_method = 'brute'
+            if (
+                self.metric == "precomputed"
+                or self._fit_X.shape[1] > 15
+                or (
+                    self.n_neighbors is not None
+                    and self.n_neighbors >= self._fit_X.shape[0] // 2
+                )
+            ):
+                self._fit_method = "brute"
             else:
-                if self.effective_metric_ in VALID_METRICS['kd_tree']:
-                    self._fit_method = 'kd_tree'
-                elif (callable(self.effective_metric_) or
-                        self.effective_metric_ in VALID_METRICS['ball_tree']):
-                    self._fit_method = 'ball_tree'
+                if (
+                    self.effective_metric_ == "minkowski"
+                    and self.effective_metric_params_["p"] < 1
+                ):
+                    self._fit_method = "brute"
+                elif (
+                    self.effective_metric_ == "minkowski"
+                    and self.effective_metric_params_.get("w") is not None
+                ):
+                    # 'minkowski' with weights is not supported by KDTree but is
+                    # supported byBallTree.
+                    self._fit_method = "ball_tree"
+                elif self.effective_metric_ in VALID_METRICS["kd_tree"]:
+                    self._fit_method = "kd_tree"
+                elif (
+                    callable(self.effective_metric_)
+                    or self.effective_metric_ in VALID_METRICS["ball_tree"]
+                ):
+                    self._fit_method = "ball_tree"
                 else:
-                    self._fit_method = 'brute'
-
-        if self._fit_method == 'ball_tree':
-            self._tree = BallTree(X, self.leaf_size,
-                                  metric=self.effective_metric_,
-                                  **self.effective_metric_params_)
-        elif self._fit_method == 'kd_tree':
-            self._tree = KDTree(X, self.leaf_size,
-                                metric=self.effective_metric_,
-                                **self.effective_metric_params_)
-        elif self._fit_method == 'brute':
-            self._tree = None
-        else:
-            raise ValueError("algorithm = '%s' not recognized"
-                             % self.algorithm)
+                    self._fit_method = "brute"
+
+        if (
+            self.effective_metric_ == "minkowski"
+            and self.effective_metric_params_["p"] < 1
+        ):
+            # For 0 < p < 1 Minkowski distances aren't valid distance
+            # metric as they do not satisfy triangular inequality:
+            # they are semi-metrics.
+            # algorithm="kd_tree" and algorithm="ball_tree" can't be used because
+            # KDTree and BallTree require a proper distance metric to work properly.
+            # However, the brute-force algorithm supports semi-metrics.
+            if self._fit_method == "brute":
+                warnings.warn(
+                    "Mind that for 0 < p < 1, Minkowski metrics are not distance"
+                    " metrics. Continuing the execution with `algorithm='brute'`."
+                )
+            else:  # self._fit_method in ("kd_tree", "ball_tree")
+                raise ValueError(
+                    f'algorithm="{self._fit_method}" does not support 0 < p < 1 for '
+                    "the Minkowski metric. To resolve this problem either "
+                    'set p >= 1 or algorithm="brute".'
+                )
 
-        if self.n_neighbors is not None:
-            if self.n_neighbors <= 0:
+        if self._fit_method == "ball_tree":
+            self._tree = BallTree(
+                X,
+                self.leaf_size,
+                metric=self.effective_metric_,
+                **self.effective_metric_params_,
+            )
+        elif self._fit_method == "kd_tree":
+            if (
+                self.effective_metric_ == "minkowski"
+                and self.effective_metric_params_.get("w") is not None
+            ):
                 raise ValueError(
-                    "Expected n_neighbors > 0. Got %d" %
-                    self.n_neighbors
+                    "algorithm='kd_tree' is not valid for "
+                    "metric='minkowski' with a weight parameter 'w': "
+                    "try algorithm='ball_tree' "
+                    "or algorithm='brute' instead."
                 )
-            else:
-                if not isinstance(self.n_neighbors, numbers.Integral):
-                    raise TypeError(
-                        "n_neighbors does not take %s value, "
-                        "enter integer value" %
-                        type(self.n_neighbors))
+            self._tree = KDTree(
+                X,
+                self.leaf_size,
+                metric=self.effective_metric_,
+                **self.effective_metric_params_,
+            )
+        elif self._fit_method == "brute":
+            self._tree = None
 
         return self
 
-    def _more_tags(self):
-        # For cross-validation routines to split data correctly
-        return {'pairwise': self.metric == 'precomputed'}
-
-    # TODO: Remove in 1.1
-    # mypy error: Decorated property not supported
-    @deprecated("Attribute _pairwise was deprecated in "  # type: ignore
-                "version 0.24 and will be removed in 1.1 (renaming of 0.26).")
-    @property
-    def _pairwise(self):
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
         # For cross-validation routines to split data correctly
-        return self.metric == 'precomputed'
-
-
-def _tree_query_parallel_helper(tree, *args, **kwargs):
-    """Helper for the Parallel calls in KNeighborsMixin.kneighbors
-
-    The Cython method tree.query is not directly picklable by cloudpickle
-    under PyPy.
-    """
-    return tree.query(*args, **kwargs)
+        tags.input_tags.pairwise = self.metric == "precomputed"
+        # when input is precomputed metric values, all those values need to be positive
+        tags.input_tags.positive_only = tags.input_tags.pairwise
+        tags.input_tags.allow_nan = self.metric == "nan_euclidean"
+        return tags
 
 
 class KNeighborsMixin:
-    """Mixin for k-neighbors searches"""
+    """Mixin for k-neighbors searches."""
 
-    def _kneighbors_reduce_func(self, dist, start,
-                                n_neighbors, return_distance):
-        """Reduce a chunk of distances to the nearest neighbors
+    def _kneighbors_reduce_func(self, dist, start, n_neighbors, return_distance):
+        """Reduce a chunk of distances to the nearest neighbors.
 
         Callback to :func:`sklearn.metrics.pairwise.pairwise_distances_chunked`
 
@@ -582,10 +750,9 @@ def _kneighbors_reduce_func(self, dist, start,
         neigh_ind = np.argpartition(dist, n_neighbors - 1, axis=1)
         neigh_ind = neigh_ind[:, :n_neighbors]
         # argpartition doesn't guarantee sorted order, so we sort again
-        neigh_ind = neigh_ind[
-            sample_range, np.argsort(dist[sample_range, neigh_ind])]
+        neigh_ind = neigh_ind[sample_range, np.argsort(dist[sample_range, neigh_ind])]
         if return_distance:
-            if self.effective_metric_ == 'euclidean':
+            if self.effective_metric_ == "euclidean":
                 result = np.sqrt(dist[sample_range, neigh_ind]), neigh_ind
             else:
                 result = dist[sample_range, neigh_ind], neigh_ind
@@ -594,15 +761,14 @@ def _kneighbors_reduce_func(self, dist, start,
         return result
 
     def kneighbors(self, X=None, n_neighbors=None, return_distance=True):
-        """Finds the K-neighbors of a point.
+        """Find the K-neighbors of a point.
 
         Returns indices of and distances to the neighbors of each point.
 
         Parameters
         ----------
-        X : array-like, shape (n_queries, n_features), \
-            or (n_queries, n_indexed) if metric == 'precomputed', \
-                default=None
+        X : {array-like, sparse matrix}, shape (n_queries, n_features), \
+            or (n_queries, n_indexed) if metric == 'precomputed', default=None
             The query point or points.
             If not provided, neighbors of each indexed point are returned.
             In this case, the query point is not considered its own neighbor.
@@ -618,7 +784,7 @@ def kneighbors(self, X=None, n_neighbors=None, return_distance=True):
         -------
         neigh_dist : ndarray of shape (n_queries, n_neighbors)
             Array representing the lengths to points, only present if
-            return_distance=True
+            return_distance=True.
 
         neigh_ind : ndarray of shape (n_queries, n_neighbors)
             Indices of the nearest points in the population matrix.
@@ -651,77 +817,111 @@ class from an array representing our data set and ask who's
         if n_neighbors is None:
             n_neighbors = self.n_neighbors
         elif n_neighbors <= 0:
-            raise ValueError(
-                "Expected n_neighbors > 0. Got %d" %
-                n_neighbors
+            raise ValueError("Expected n_neighbors > 0. Got %d" % n_neighbors)
+        elif not isinstance(n_neighbors, numbers.Integral):
+            raise TypeError(
+                "n_neighbors does not take %s value, enter integer value"
+                % type(n_neighbors)
             )
-        else:
-            if not isinstance(n_neighbors, numbers.Integral):
-                raise TypeError(
-                    "n_neighbors does not take %s value, "
-                    "enter integer value" %
-                    type(n_neighbors))
-
-        if X is not None:
-            query_is_train = False
-            if self.effective_metric_ == 'precomputed':
-                X = _check_precomputed(X)
-            else:
-                X = self._validate_data(X, accept_sparse='csr', reset=False)
-        else:
-            query_is_train = True
+
+        ensure_all_finite = "allow-nan" if get_tags(self).input_tags.allow_nan else True
+        query_is_train = X is None
+        if query_is_train:
             X = self._fit_X
             # Include an extra neighbor to account for the sample itself being
             # returned, which is removed later
             n_neighbors += 1
+        else:
+            if self.metric == "precomputed":
+                X = _check_precomputed(X)
+            else:
+                X = validate_data(
+                    self,
+                    X,
+                    ensure_all_finite=ensure_all_finite,
+                    accept_sparse="csr",
+                    reset=False,
+                    order="C",
+                )
 
         n_samples_fit = self.n_samples_fit_
         if n_neighbors > n_samples_fit:
+            if query_is_train:
+                n_neighbors -= 1  # ok to modify inplace because an error is raised
+                inequality_str = "n_neighbors < n_samples_fit"
+            else:
+                inequality_str = "n_neighbors <= n_samples_fit"
             raise ValueError(
-                "Expected n_neighbors <= n_samples, "
-                " but n_samples = %d, n_neighbors = %d" %
-                (n_samples_fit, n_neighbors)
+                f"Expected {inequality_str}, but "
+                f"n_neighbors = {n_neighbors}, n_samples_fit = {n_samples_fit}, "
+                f"n_samples = {X.shape[0]}"  # include n_samples for common tests
             )
 
         n_jobs = effective_n_jobs(self.n_jobs)
         chunked_results = None
-        if (self._fit_method == 'brute' and
-                self.effective_metric_ == 'precomputed' and issparse(X)):
+        use_pairwise_distances_reductions = (
+            self._fit_method == "brute"
+            and ArgKmin.is_usable_for(
+                X if X is not None else self._fit_X, self._fit_X, self.effective_metric_
+            )
+        )
+        if use_pairwise_distances_reductions:
+            results = ArgKmin.compute(
+                X=X,
+                Y=self._fit_X,
+                k=n_neighbors,
+                metric=self.effective_metric_,
+                metric_kwargs=self.effective_metric_params_,
+                strategy="auto",
+                return_distance=return_distance,
+            )
+
+        elif (
+            self._fit_method == "brute" and self.metric == "precomputed" and issparse(X)
+        ):
             results = _kneighbors_from_graph(
-                X, n_neighbors=n_neighbors,
-                return_distance=return_distance)
+                X, n_neighbors=n_neighbors, return_distance=return_distance
+            )
 
-        elif self._fit_method == 'brute':
-            reduce_func = partial(self._kneighbors_reduce_func,
-                                  n_neighbors=n_neighbors,
-                                  return_distance=return_distance)
+        elif self._fit_method == "brute":
+            # Joblib-based backend, which is used when user-defined callable
+            # are passed for metric.
+
+            # This won't be used in the future once PairwiseDistancesReductions
+            # support:
+            #   - DistanceMetrics which work on supposedly binary data
+            #   - CSR-dense and dense-CSR case if 'euclidean' in metric.
+            reduce_func = partial(
+                self._kneighbors_reduce_func,
+                n_neighbors=n_neighbors,
+                return_distance=return_distance,
+            )
 
             # for efficiency, use squared euclidean distances
-            if self.effective_metric_ == 'euclidean':
-                kwds = {'squared': True}
+            if self.effective_metric_ == "euclidean":
+                kwds = {"squared": True}
             else:
                 kwds = self.effective_metric_params_
 
-            chunked_results = list(pairwise_distances_chunked(
-                X, self._fit_X, reduce_func=reduce_func,
-                metric=self.effective_metric_, n_jobs=n_jobs,
-                **kwds))
+            chunked_results = list(
+                pairwise_distances_chunked(
+                    X,
+                    self._fit_X,
+                    reduce_func=reduce_func,
+                    metric=self.effective_metric_,
+                    n_jobs=n_jobs,
+                    **kwds,
+                )
+            )
 
-        elif self._fit_method in ['ball_tree', 'kd_tree']:
+        elif self._fit_method in ["ball_tree", "kd_tree"]:
             if issparse(X):
                 raise ValueError(
                     "%s does not work with sparse matrices. Densify the data, "
-                    "or set algorithm='brute'" % self._fit_method)
-            old_joblib = (
-                    parse_version(joblib.__version__) < parse_version('0.12'))
-            if old_joblib:
-                # Deal with change of API in joblib
-                parallel_kwargs = {"backend": "threading"}
-            else:
-                parallel_kwargs = {"prefer": "threads"}
-            chunked_results = Parallel(n_jobs, **parallel_kwargs)(
-                delayed(_tree_query_parallel_helper)(
-                    self._tree, X[s], n_neighbors, return_distance)
+                    "or set algorithm='brute'" % self._fit_method
+                )
+            chunked_results = Parallel(n_jobs, prefer="threads")(
+                delayed(self._tree.query)(X[s], n_neighbors, return_distance)
                 for s in gen_even_slices(X.shape[0], n_jobs)
             )
         else:
@@ -755,24 +955,22 @@ class from an array representing our data set and ask who's
             # In that case mask the first duplicate.
             dup_gr_nbrs = np.all(sample_mask, axis=1)
             sample_mask[:, 0][dup_gr_nbrs] = False
-            neigh_ind = np.reshape(
-                neigh_ind[sample_mask], (n_queries, n_neighbors - 1))
+            neigh_ind = np.reshape(neigh_ind[sample_mask], (n_queries, n_neighbors - 1))
 
             if return_distance:
                 neigh_dist = np.reshape(
-                    neigh_dist[sample_mask], (n_queries, n_neighbors - 1))
+                    neigh_dist[sample_mask], (n_queries, n_neighbors - 1)
+                )
                 return neigh_dist, neigh_ind
             return neigh_ind
 
-    def kneighbors_graph(self, X=None, n_neighbors=None,
-                         mode='connectivity'):
-        """Computes the (weighted) graph of k-Neighbors for points in X
+    def kneighbors_graph(self, X=None, n_neighbors=None, mode="connectivity"):
+        """Compute the (weighted) graph of k-Neighbors for points in X.
 
         Parameters
         ----------
-        X : array-like of shape (n_queries, n_features), \
-                or (n_queries, n_indexed) if metric == 'precomputed', \
-                default=None
+        X : {array-like, sparse matrix} of shape (n_queries, n_features), \
+            or (n_queries, n_indexed) if metric == 'precomputed', default=None
             The query point or points.
             If not provided, neighbors of each indexed point are returned.
             In this case, the query point is not considered its own neighbor.
@@ -787,15 +985,22 @@ def kneighbors_graph(self, X=None, n_neighbors=None,
         mode : {'connectivity', 'distance'}, default='connectivity'
             Type of returned matrix: 'connectivity' will return the
             connectivity matrix with ones and zeros, in 'distance' the
-            edges are Euclidean distance between points.
+            edges are distances between points, type of distance
+            depends on the selected metric parameter in
+            NearestNeighbors class.
 
         Returns
         -------
         A : sparse-matrix of shape (n_queries, n_samples_fit)
-            `n_samples_fit` is the number of samples in the fitted data
-            `A[i, j]` is assigned the weight of edge that connects `i` to `j`.
+            `n_samples_fit` is the number of samples in the fitted data.
+            `A[i, j]` gives the weight of the edge connecting `i` to `j`.
             The matrix is of CSR format.
 
+        See Also
+        --------
+        NearestNeighbors.radius_neighbors_graph : Compute the (weighted) graph
+            of Neighbors for points in X.
+
         Examples
         --------
         >>> X = [[0], [3], [1]]
@@ -808,10 +1013,6 @@ def kneighbors_graph(self, X=None, n_neighbors=None,
         array([[1., 0., 1.],
                [0., 1., 1.],
                [1., 0., 1.]])
-
-        See Also
-        --------
-        NearestNeighbors.radius_neighbors_graph
         """
         check_is_fitted(self)
         if n_neighbors is None:
@@ -820,47 +1021,38 @@ def kneighbors_graph(self, X=None, n_neighbors=None,
         # check the input only in self.kneighbors
 
         # construct CSR matrix representation of the k-NN graph
-        if mode == 'connectivity':
+        if mode == "connectivity":
             A_ind = self.kneighbors(X, n_neighbors, return_distance=False)
             n_queries = A_ind.shape[0]
             A_data = np.ones(n_queries * n_neighbors)
 
-        elif mode == 'distance':
-            A_data, A_ind = self.kneighbors(
-                X, n_neighbors, return_distance=True)
+        elif mode == "distance":
+            A_data, A_ind = self.kneighbors(X, n_neighbors, return_distance=True)
             A_data = np.ravel(A_data)
 
         else:
             raise ValueError(
-                'Unsupported mode, must be one of "connectivity" '
-                'or "distance" but got "%s" instead' % mode)
+                'Unsupported mode, must be one of "connectivity", '
+                f'or "distance" but got "{mode}" instead'
+            )
 
         n_queries = A_ind.shape[0]
         n_samples_fit = self.n_samples_fit_
         n_nonzero = n_queries * n_neighbors
         A_indptr = np.arange(0, n_nonzero + 1, n_neighbors)
 
-        kneighbors_graph = csr_matrix((A_data, A_ind.ravel(), A_indptr),
-                                      shape=(n_queries, n_samples_fit))
+        kneighbors_graph = csr_matrix(
+            (A_data, A_ind.ravel(), A_indptr), shape=(n_queries, n_samples_fit)
+        )
 
         return kneighbors_graph
 
 
-def _tree_query_radius_parallel_helper(tree, *args, **kwargs):
-    """Helper for the Parallel calls in RadiusNeighborsMixin.radius_neighbors
-
-    The Cython method tree.query_radius is not directly picklable by
-    cloudpickle under PyPy.
-    """
-    return tree.query_radius(*args, **kwargs)
-
-
 class RadiusNeighborsMixin:
-    """Mixin for radius-based neighbors searches"""
+    """Mixin for radius-based neighbors searches."""
 
-    def _radius_neighbors_reduce_func(self, dist, start,
-                                      radius, return_distance):
-        """Reduce a chunk of distances to the nearest neighbors
+    def _radius_neighbors_reduce_func(self, dist, start, radius, return_distance):
+        """Reduce a chunk of distances to the nearest neighbors.
 
         Callback to :func:`sklearn.metrics.pairwise.pairwise_distances_chunked`
 
@@ -889,20 +1081,19 @@ def _radius_neighbors_reduce_func(self, dist, start,
         neigh_ind = [np.where(d <= radius)[0] for d in dist]
 
         if return_distance:
-            if self.effective_metric_ == 'euclidean':
-                dist = [np.sqrt(d[neigh_ind[i]])
-                        for i, d in enumerate(dist)]
+            if self.effective_metric_ == "euclidean":
+                dist = [np.sqrt(d[neigh_ind[i]]) for i, d in enumerate(dist)]
             else:
-                dist = [d[neigh_ind[i]]
-                        for i, d in enumerate(dist)]
+                dist = [d[neigh_ind[i]] for i, d in enumerate(dist)]
             results = dist, neigh_ind
         else:
             results = neigh_ind
         return results
 
-    def radius_neighbors(self, X=None, radius=None, return_distance=True,
-                         sort_results=False):
-        """Finds the neighbors within a given radius of a point or points.
+    def radius_neighbors(
+        self, X=None, radius=None, return_distance=True, sort_results=False
+    ):
+        """Find the neighbors within a given radius of a point or points.
 
         Return the indices and distances of each point from the dataset
         lying in a ball with size ``radius`` around the points of the query
@@ -913,7 +1104,7 @@ def radius_neighbors(self, X=None, radius=None, return_distance=True,
 
         Parameters
         ----------
-        X : array-like of (n_samples, n_features), default=None
+        X : {array-like, sparse matrix} of (n_samples, n_features), default=None
             The query point or points.
             If not provided, neighbors of each indexed point are returned.
             In this case, the query point is not considered its own neighbor.
@@ -945,6 +1136,14 @@ def radius_neighbors(self, X=None, radius=None, return_distance=True,
             from the population matrix that lie within a ball of size
             ``radius`` around the query points.
 
+        Notes
+        -----
+        Because the number of neighbors of each point is not necessarily
+        equal, the results for multiple query points cannot be fit in a
+        standard data array.
+        For efficiency, `radius_neighbors` returns arrays of objects, where
+        each object is a 1D array of indices or distances.
+
         Examples
         --------
         In the following example, we construct a NeighborsClassifier
@@ -966,90 +1165,117 @@ class from an array representing our data set and ask who's
         The first array returned contains the distances to all points which
         are closer than 1.6, while the second array returned contains their
         indices.  In general, multiple points can be queried at the same time.
-
-        Notes
-        -----
-        Because the number of neighbors of each point is not necessarily
-        equal, the results for multiple query points cannot be fit in a
-        standard data array.
-        For efficiency, `radius_neighbors` returns arrays of objects, where
-        each object is a 1D array of indices or distances.
         """
         check_is_fitted(self)
 
-        if X is not None:
-            query_is_train = False
-            if self.effective_metric_ == 'precomputed':
+        if sort_results and not return_distance:
+            raise ValueError("return_distance must be True if sort_results is True.")
+
+        ensure_all_finite = "allow-nan" if get_tags(self).input_tags.allow_nan else True
+        query_is_train = X is None
+        if query_is_train:
+            X = self._fit_X
+        else:
+            if self.metric == "precomputed":
                 X = _check_precomputed(X)
             else:
-                X = self._validate_data(X, accept_sparse='csr', reset=False)
-        else:
-            query_is_train = True
-            X = self._fit_X
+                X = validate_data(
+                    self,
+                    X,
+                    ensure_all_finite=ensure_all_finite,
+                    accept_sparse="csr",
+                    reset=False,
+                    order="C",
+                )
 
         if radius is None:
             radius = self.radius
 
-        if (self._fit_method == 'brute' and
-                self.effective_metric_ == 'precomputed' and issparse(X)):
+        use_pairwise_distances_reductions = (
+            self._fit_method == "brute"
+            and RadiusNeighbors.is_usable_for(
+                X if X is not None else self._fit_X, self._fit_X, self.effective_metric_
+            )
+        )
+
+        if use_pairwise_distances_reductions:
+            results = RadiusNeighbors.compute(
+                X=X,
+                Y=self._fit_X,
+                radius=radius,
+                metric=self.effective_metric_,
+                metric_kwargs=self.effective_metric_params_,
+                strategy="auto",
+                return_distance=return_distance,
+                sort_results=sort_results,
+            )
+
+        elif (
+            self._fit_method == "brute" and self.metric == "precomputed" and issparse(X)
+        ):
             results = _radius_neighbors_from_graph(
-                X, radius=radius, return_distance=return_distance)
+                X, radius=radius, return_distance=return_distance
+            )
+
+        elif self._fit_method == "brute":
+            # Joblib-based backend, which is used when user-defined callable
+            # are passed for metric.
+
+            # This won't be used in the future once PairwiseDistancesReductions
+            # support:
+            #   - DistanceMetrics which work on supposedly binary data
+            #   - CSR-dense and dense-CSR case if 'euclidean' in metric.
 
-        elif self._fit_method == 'brute':
             # for efficiency, use squared euclidean distances
-            if self.effective_metric_ == 'euclidean':
+            if self.effective_metric_ == "euclidean":
                 radius *= radius
-                kwds = {'squared': True}
+                kwds = {"squared": True}
             else:
                 kwds = self.effective_metric_params_
 
-            reduce_func = partial(self._radius_neighbors_reduce_func,
-                                  radius=radius,
-                                  return_distance=return_distance)
+            reduce_func = partial(
+                self._radius_neighbors_reduce_func,
+                radius=radius,
+                return_distance=return_distance,
+            )
 
             chunked_results = pairwise_distances_chunked(
-                X, self._fit_X, reduce_func=reduce_func,
-                metric=self.effective_metric_, n_jobs=self.n_jobs,
-                **kwds)
+                X,
+                self._fit_X,
+                reduce_func=reduce_func,
+                metric=self.effective_metric_,
+                n_jobs=self.n_jobs,
+                **kwds,
+            )
             if return_distance:
                 neigh_dist_chunks, neigh_ind_chunks = zip(*chunked_results)
-                neigh_dist_list = sum(neigh_dist_chunks, [])
-                neigh_ind_list = sum(neigh_ind_chunks, [])
+                neigh_dist_list = list(itertools.chain.from_iterable(neigh_dist_chunks))
+                neigh_ind_list = list(itertools.chain.from_iterable(neigh_ind_chunks))
                 neigh_dist = _to_object_array(neigh_dist_list)
                 neigh_ind = _to_object_array(neigh_ind_list)
                 results = neigh_dist, neigh_ind
             else:
-                neigh_ind_list = sum(chunked_results, [])
+                neigh_ind_list = list(itertools.chain.from_iterable(chunked_results))
                 results = _to_object_array(neigh_ind_list)
 
             if sort_results:
-                if not return_distance:
-                    raise ValueError("return_distance must be True "
-                                     "if sort_results is True.")
                 for ii in range(len(neigh_dist)):
-                    order = np.argsort(neigh_dist[ii], kind='mergesort')
+                    order = np.argsort(neigh_dist[ii], kind="mergesort")
                     neigh_ind[ii] = neigh_ind[ii][order]
                     neigh_dist[ii] = neigh_dist[ii][order]
                 results = neigh_dist, neigh_ind
 
-        elif self._fit_method in ['ball_tree', 'kd_tree']:
+        elif self._fit_method in ["ball_tree", "kd_tree"]:
             if issparse(X):
                 raise ValueError(
                     "%s does not work with sparse matrices. Densify the data, "
-                    "or set algorithm='brute'" % self._fit_method)
+                    "or set algorithm='brute'" % self._fit_method
+                )
 
             n_jobs = effective_n_jobs(self.n_jobs)
-            delayed_query = delayed(_tree_query_radius_parallel_helper)
-            if parse_version(joblib.__version__) < parse_version('0.12'):
-                # Deal with change of API in joblib
-                parallel_kwargs = {"backend": "threading"}
-            else:
-                parallel_kwargs = {"prefer": "threads"}
-
-            chunked_results = Parallel(n_jobs, **parallel_kwargs)(
-                delayed_query(self._tree, X[s], radius, return_distance,
-                              sort_results=sort_results)
-
+            delayed_query = delayed(self._tree.query_radius)
+            chunked_results = Parallel(n_jobs, prefer="threads")(
+                delayed_query(X[s], radius, return_distance, sort_results=sort_results)
                 for s in gen_even_slices(X.shape[0], n_jobs)
             )
             if return_distance:
@@ -1082,16 +1308,17 @@ class from an array representing our data set and ask who's
                 return neigh_dist, neigh_ind
             return neigh_ind
 
-    def radius_neighbors_graph(self, X=None, radius=None, mode='connectivity',
-                               sort_results=False):
-        """Computes the (weighted) graph of Neighbors for points in X
+    def radius_neighbors_graph(
+        self, X=None, radius=None, mode="connectivity", sort_results=False
+    ):
+        """Compute the (weighted) graph of Neighbors for points in X.
 
         Neighborhoods are restricted the points at a distance lower than
         radius.
 
         Parameters
         ----------
-        X : array-like of shape (n_samples, n_features), default=None
+        X : {array-like, sparse matrix} of shape (n_samples, n_features), default=None
             The query point or points.
             If not provided, neighbors of each indexed point are returned.
             In this case, the query point is not considered its own neighbor.
@@ -1103,7 +1330,9 @@ def radius_neighbors_graph(self, X=None, radius=None, mode='connectivity',
         mode : {'connectivity', 'distance'}, default='connectivity'
             Type of returned matrix: 'connectivity' will return the
             connectivity matrix with ones and zeros, in 'distance' the
-            edges are Euclidean distance between points.
+            edges are distances between points, type of distance
+            depends on the selected metric parameter in
+            NearestNeighbors class.
 
         sort_results : bool, default=False
             If True, in each row of the result, the non-zero entries will be
@@ -1115,9 +1344,14 @@ def radius_neighbors_graph(self, X=None, radius=None, mode='connectivity',
         Returns
         -------
         A : sparse-matrix of shape (n_queries, n_samples_fit)
-            `n_samples_fit` is the number of samples in the fitted data
-            `A[i, j]` is assigned the weight of edge that connects `i` to `j`.
-            The matrix if of format CSR.
+            `n_samples_fit` is the number of samples in the fitted data.
+            `A[i, j]` gives the weight of the edge connecting `i` to `j`.
+            The matrix is of CSR format.
+
+        See Also
+        --------
+        kneighbors_graph : Compute the (weighted) graph of k-Neighbors for
+            points in X.
 
         Examples
         --------
@@ -1131,10 +1365,6 @@ def radius_neighbors_graph(self, X=None, radius=None, mode='connectivity',
         array([[1., 0., 1.],
                [0., 1., 0.],
                [1., 0., 1.]])
-
-        See Also
-        --------
-        kneighbors_graph
         """
         check_is_fitted(self)
 
@@ -1144,19 +1374,19 @@ def radius_neighbors_graph(self, X=None, radius=None, mode='connectivity',
             radius = self.radius
 
         # construct CSR matrix representation of the NN graph
-        if mode == 'connectivity':
-            A_ind = self.radius_neighbors(X, radius,
-                                          return_distance=False)
+        if mode == "connectivity":
+            A_ind = self.radius_neighbors(X, radius, return_distance=False)
             A_data = None
-        elif mode == 'distance':
-            dist, A_ind = self.radius_neighbors(X, radius,
-                                                return_distance=True,
-                                                sort_results=sort_results)
+        elif mode == "distance":
+            dist, A_ind = self.radius_neighbors(
+                X, radius, return_distance=True, sort_results=sort_results
+            )
             A_data = np.concatenate(list(dist))
         else:
             raise ValueError(
                 'Unsupported mode, must be one of "connectivity", '
-                'or "distance" but got %s instead' % mode)
+                f'or "distance" but got "{mode}" instead'
+            )
 
         n_queries = A_ind.shape[0]
         n_samples_fit = self.n_samples_fit_
@@ -1164,8 +1394,11 @@ def radius_neighbors_graph(self, X=None, radius=None, mode='connectivity',
         A_ind = np.concatenate(list(A_ind))
         if A_data is None:
             A_data = np.ones(len(A_ind))
-        A_indptr = np.concatenate((np.zeros(1, dtype=int),
-                                   np.cumsum(n_neighbors)))
+        A_indptr = np.concatenate((np.zeros(1, dtype=int), np.cumsum(n_neighbors)))
+
+        return csr_matrix((A_data, A_ind, A_indptr), shape=(n_queries, n_samples_fit))
 
-        return csr_matrix((A_data, A_ind, A_indptr),
-                          shape=(n_queries, n_samples_fit))
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.allow_nan = self.metric == "nan_euclidean"
+        return tags
diff --git a/sklearn/neighbors/_binary_tree.pxi b/sklearn/neighbors/_binary_tree.pxi.tp
old mode 100755
new mode 100644
similarity index 68%
rename from sklearn/neighbors/_binary_tree.pxi
rename to sklearn/neighbors/_binary_tree.pxi.tp
index 1acff082c7d76..de3bcb0e5d916
--- a/sklearn/neighbors/_binary_tree.pxi
+++ b/sklearn/neighbors/_binary_tree.pxi.tp
@@ -1,14 +1,29 @@
-#!python
+{{py:
 
+# Generated file: _binary_tree.pxi
+
+implementation_specific_values = [
+    # The values are arranged as follows:
+    #
+    #       name_suffix, INPUT_DTYPE_t, INPUT_DTYPE, NPY_TYPE
+    #
+    ('64', 'float64_t', 'np.float64', 'cnp.NPY_DOUBLE'),
+    ('32', 'float32_t', 'np.float32', 'cnp.NPY_FLOAT')
+]
 
 # KD Tree and Ball Tree
 # =====================
 #
-#    Author: Jake Vanderplas , 2012-2013
-#    License: BSD
-#
-# This file is meant to be a literal include in a pyx file.
-# See ball_tree.pyx and kd_tree.pyx
+# _binary_tree.pxi is generated and is then literally Cython included in
+# ball_tree.pyx and kd_tree.pyx. See ball_tree.pyx.tp and kd_tree.pyx.tp.
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+}}
+
+
+# KD Tree and Ball Tree
+# =====================
 #
 # The routines here are the core algorithms of the KDTree and BallTree
 # structures.  If Cython supported polymorphism, we would be able to
@@ -103,47 +118,47 @@
 # These are the names and descriptions of the "abstract" functions which are
 # defined in kd_tree.pyx and ball_tree.pyx:
 
-# cdef int allocate_data(BinaryTree tree, ITYPE_t n_nodes, ITYPE_t n_features):
+# cdef int allocate_data(BinaryTree tree, intp_t n_nodes, intp_t n_features):
 #     """Allocate arrays needed for the KD Tree"""
 
-# cdef int init_node(BinaryTree tree, ITYPE_t i_node,
-#                    ITYPE_t idx_start, ITYPE_t idx_end):
+# cdef int init_node(BinaryTree tree, intp_t i_node,
+#                    intp_t idx_start, intp_t idx_end):
 #    """Initialize the node for the dataset stored in tree.data"""
 
-# cdef DTYPE_t min_rdist(BinaryTree tree, ITYPE_t i_node, DTYPE_t* pt):
+# cdef float64_t min_rdist(BinaryTree tree, intp_t i_node, float64_t* pt):
 #     """Compute the minimum reduced-distance between a point and a node"""
 
-# cdef DTYPE_t min_dist(BinaryTree tree, ITYPE_t i_node, DTYPE_t* pt):
+# cdef float64_t min_dist(BinaryTree tree, intp_t i_node, float64_t* pt):
 #     """Compute the minimum distance between a point and a node"""
 
-# cdef DTYPE_t max_rdist(BinaryTree tree, ITYPE_t i_node, DTYPE_t* pt):
+# cdef float64_t max_rdist(BinaryTree tree, intp_t i_node, float64_t* pt):
 #     """Compute the maximum reduced-distance between a point and a node"""
 
-# cdef DTYPE_t max_dist(BinaryTree tree, ITYPE_t i_node, DTYPE_t* pt):
+# cdef float64_t max_dist(BinaryTree tree, intp_t i_node, float64_t* pt):
 #     """Compute the maximum distance between a point and a node"""
 
-# cdef inline int min_max_dist(BinaryTree tree, ITYPE_t i_node, DTYPE_t* pt,
-#                              DTYPE_t* min_dist, DTYPE_t* max_dist):
+# cdef inline int min_max_dist(BinaryTree tree, intp_t i_node, float64_t* pt,
+#                              float64_t* min_dist, float64_t* max_dist):
 #     """Compute the minimum and maximum distance between a point and a node"""
 
-# cdef inline DTYPE_t min_rdist_dual(BinaryTree tree1, ITYPE_t i_node1,
-#                                    BinaryTree tree2, ITYPE_t i_node2):
+# cdef inline float64_t min_rdist_dual(BinaryTree tree1, intp_t i_node1,
+#                                    BinaryTree tree2, intp_t i_node2):
 #     """Compute the minimum reduced distance between two nodes"""
 
-# cdef inline DTYPE_t min_dist_dual(BinaryTree tree1, ITYPE_t i_node1,
-#                                   BinaryTree tree2, ITYPE_t i_node2):
+# cdef inline float64_t min_dist_dual(BinaryTree tree1, intp_t i_node1,
+#                                   BinaryTree tree2, intp_t i_node2):
 #     """Compute the minimum distance between two nodes"""
 
-# cdef inline DTYPE_t max_rdist_dual(BinaryTree tree1, ITYPE_t i_node1,
-#                                    BinaryTree tree2, ITYPE_t i_node2):
+# cdef inline float64_t max_rdist_dual(BinaryTree tree1, intp_t i_node1,
+#                                    BinaryTree tree2, intp_t i_node2):
 #     """Compute the maximum reduced distance between two nodes"""
 
-# cdef inline DTYPE_t max_dist_dual(BinaryTree tree1, ITYPE_t i_node1,
-#                                   BinaryTree tree2, ITYPE_t i_node2):
+# cdef inline float64_t max_dist_dual(BinaryTree tree1, intp_t i_node1,
+#                                   BinaryTree tree2, intp_t i_node2):
 #     """Compute the maximum distance between two nodes"""
 
-cimport cython
-cimport numpy as np
+cimport numpy as cnp
+from cython cimport floating
 from libc.math cimport fabs, sqrt, exp, cos, pow, log, lgamma
 from libc.math cimport fmin, fmax
 from libc.stdlib cimport calloc, malloc, free
@@ -151,98 +166,68 @@ from libc.string cimport memcpy
 
 import numpy as np
 import warnings
+
+from ..metrics._dist_metrics cimport (
+    DistanceMetric,
+    DistanceMetric64,
+    DistanceMetric32,
+    euclidean_dist64,
+    euclidean_dist32,
+    euclidean_rdist64,
+    euclidean_rdist32,
+    euclidean_dist_to_rdist64,
+    euclidean_dist_to_rdist32,
+)
+
+from ._partition_nodes cimport partition_node_indices
+
 from ..utils import check_array
+from ..utils._typedefs cimport float32_t, float64_t, intp_t
+from ..utils._heap cimport heap_push
+from ..utils._sorting cimport simultaneous_sort as _simultaneous_sort
 
-from ._typedefs cimport DTYPE_t, ITYPE_t, DITYPE_t
-from ._typedefs import DTYPE, ITYPE
+cnp.import_array()
 
-from ._dist_metrics cimport (DistanceMetric, euclidean_dist, euclidean_rdist,
-                             euclidean_dist_to_rdist, euclidean_rdist_to_dist)
 
+# TODO: use cnp.PyArray_ENABLEFLAGS when Cython>=3.0 is used.
 cdef extern from "numpy/arrayobject.h":
-    void PyArray_ENABLEFLAGS(np.ndarray arr, int flags)
+    void PyArray_ENABLEFLAGS(cnp.ndarray arr, int flags)
 
-np.import_array()
 
 # some handy constants
-cdef DTYPE_t INF = np.inf
-cdef DTYPE_t NEG_INF = -np.inf
-cdef DTYPE_t PI = np.pi
-cdef DTYPE_t ROOT_2PI = sqrt(2 * PI)
-cdef DTYPE_t LOG_PI = log(PI)
-cdef DTYPE_t LOG_2PI = log(2 * PI)
+cdef float64_t INF = np.inf
+cdef float64_t NEG_INF = -np.inf
+cdef float64_t PI = np.pi
+cdef float64_t ROOT_2PI = sqrt(2 * PI)
+cdef float64_t LOG_PI = log(PI)
+cdef float64_t LOG_2PI = log(2 * PI)
 
 
 # Some compound datatypes used below:
 cdef struct NodeHeapData_t:
-    DTYPE_t val
-    ITYPE_t i1
-    ITYPE_t i2
+    float64_t val
+    intp_t i1
+    intp_t i2
 
 # build the corresponding numpy dtype for NodeHeapData
 cdef NodeHeapData_t nhd_tmp
 NodeHeapData = np.asarray((&nhd_tmp)).dtype
 
 cdef struct NodeData_t:
-    ITYPE_t idx_start
-    ITYPE_t idx_end
-    ITYPE_t is_leaf
-    DTYPE_t radius
+    intp_t idx_start
+    intp_t idx_end
+    intp_t is_leaf
+    float64_t radius
 
 # build the corresponding numpy dtype for NodeData
 cdef NodeData_t nd_tmp
 NodeData = np.asarray((&nd_tmp)).dtype
 
 
-######################################################################
-# Numpy 1.3-1.4 compatibility utilities
-cdef DTYPE_t[::1] get_memview_DTYPE_1D(
-                               np.ndarray[DTYPE_t, ndim=1, mode='c'] X):
-    return  ( X.data)
-
-
-cdef DTYPE_t[:, ::1] get_memview_DTYPE_2D(
-                               np.ndarray[DTYPE_t, ndim=2, mode='c'] X):
-    return  ( X.data)
-
-
-cdef DTYPE_t[:, :, ::1] get_memview_DTYPE_3D(
-                               np.ndarray[DTYPE_t, ndim=3, mode='c'] X):
-    return \
-                                                       ( X.data)
-
-
-cdef ITYPE_t[::1] get_memview_ITYPE_1D(
-                               np.ndarray[ITYPE_t, ndim=1, mode='c'] X):
-    return  ( X.data)
-
-
-cdef ITYPE_t[:, ::1] get_memview_ITYPE_2D(
-                               np.ndarray[ITYPE_t, ndim=2, mode='c'] X):
-    return  ( X.data)
-
-
-cdef NodeHeapData_t[::1] get_memview_NodeHeapData_1D(
-                    np.ndarray[NodeHeapData_t, ndim=1, mode='c'] X):
-    return  ( X.data)
-
-
-cdef NodeData_t[::1] get_memview_NodeData_1D(
-                    np.ndarray[NodeData_t, ndim=1, mode='c'] X):
-    return  ( X.data)
-
-######################################################################
-
-
-
 ######################################################################
 # Define doc strings, substituting the appropriate class name using
 # the DOC_DICT variable defined in the pyx files.
-CLASS_DOC = \
-"""
-{BinaryTree}(X, leaf_size=40, metric='minkowski', **kwargs)
-
-{BinaryTree} for fast generalized N-point problems
+CLASS_DOC = """{BinaryTree} for fast generalized N-point problems
 
 Read more in the :ref:`User Guide `.
 
@@ -264,12 +249,15 @@ leaf_size : positive int, default=40
     satisfy ``leaf_size <= n_points <= 2 * leaf_size``, except in
     the case that ``n_samples < leaf_size``.
 
-metric : str or DistanceMetric object
-    the distance metric to use for the tree.  Default='minkowski'
-    with p=2 (that is, a euclidean metric). See the documentation
-    of the DistanceMetric class for a list of available metrics.
-    {binary_tree}.valid_metrics gives a list of the metrics which
-    are valid for {BinaryTree}.
+metric : str or DistanceMetric64 object, default='minkowski'
+    Metric to use for distance computation. Default is "minkowski", which
+    results in the standard Euclidean distance when p = 2.
+    A list of valid metrics for {BinaryTree} is given by the attribute
+    `valid_metrics`.
+    See the documentation of `scipy.spatial.distance
+    `_ and
+    the metrics listed in :class:`~sklearn.metrics.pairwise.distance_metrics` for
+    more information on any distance metric.
 
 Additional keywords are passed to the distance metric class.
 Note: Callable functions in the metric parameter are NOT supported for KDTree
@@ -279,12 +267,15 @@ Attributes
 ----------
 data : memory view
     The training data
+valid_metrics: list of str
+    List of valid distance metrics.
 
 Examples
 --------
 Query for k-nearest neighbors
 
     >>> import numpy as np
+    >>> from sklearn.neighbors import {BinaryTree}
     >>> rng = np.random.RandomState(0)
     >>> X = rng.random_sample((10, 3))  # 10 points in 3 dimensions
     >>> tree = {BinaryTree}(X, leaf_size=2)              # doctest: +SKIP
@@ -347,15 +338,15 @@ Compute a two-point auto-correlation function
 
 ######################################################################
 # Utility functions
-cdef DTYPE_t logaddexp(DTYPE_t x1, DTYPE_t x2):
+cdef float64_t logaddexp(float64_t x1, float64_t x2):
     """logaddexp(x1, x2) -> log(exp(x1) + exp(x2))"""
-    cdef DTYPE_t a = fmax(x1, x2)
+    cdef float64_t a = fmax(x1, x2)
     if a == NEG_INF:
         return NEG_INF
     else:
         return a + log(exp(x1 - a) + exp(x2 - a))
 
-cdef DTYPE_t logsubexp(DTYPE_t x1, DTYPE_t x2):
+cdef float64_t logsubexp(float64_t x1, float64_t x2):
     """logsubexp(x1, x2) -> log(exp(x1) - exp(x2))"""
     if x1 <= x2:
         return NEG_INF
@@ -382,12 +373,12 @@ cdef enum KernelType:
     COSINE_KERNEL = 6
 
 
-cdef inline DTYPE_t log_gaussian_kernel(DTYPE_t dist, DTYPE_t h):
+cdef inline float64_t log_gaussian_kernel(float64_t dist, float64_t h):
     """log of the gaussian kernel for bandwidth h (unnormalized)"""
     return -0.5 * (dist * dist) / (h * h)
 
 
-cdef inline DTYPE_t log_tophat_kernel(DTYPE_t dist, DTYPE_t h):
+cdef inline float64_t log_tophat_kernel(float64_t dist, float64_t h):
     """log of the tophat kernel for bandwidth h (unnormalized)"""
     if dist < h:
         return 0.0
@@ -395,7 +386,7 @@ cdef inline DTYPE_t log_tophat_kernel(DTYPE_t dist, DTYPE_t h):
         return NEG_INF
 
 
-cdef inline DTYPE_t log_epanechnikov_kernel(DTYPE_t dist, DTYPE_t h):
+cdef inline float64_t log_epanechnikov_kernel(float64_t dist, float64_t h):
     """log of the epanechnikov kernel for bandwidth h (unnormalized)"""
     if dist < h:
         return log(1.0 - (dist * dist) / (h * h))
@@ -403,12 +394,12 @@ cdef inline DTYPE_t log_epanechnikov_kernel(DTYPE_t dist, DTYPE_t h):
         return NEG_INF
 
 
-cdef inline DTYPE_t log_exponential_kernel(DTYPE_t dist, DTYPE_t h):
+cdef inline float64_t log_exponential_kernel(float64_t dist, float64_t h):
     """log of the exponential kernel for bandwidth h (unnormalized)"""
     return -dist / h
 
 
-cdef inline DTYPE_t log_linear_kernel(DTYPE_t dist, DTYPE_t h):
+cdef inline float64_t log_linear_kernel(float64_t dist, float64_t h):
     """log of the linear kernel for bandwidth h (unnormalized)"""
     if dist < h:
         return log(1 - dist / h)
@@ -416,7 +407,7 @@ cdef inline DTYPE_t log_linear_kernel(DTYPE_t dist, DTYPE_t h):
         return NEG_INF
 
 
-cdef inline DTYPE_t log_cosine_kernel(DTYPE_t dist, DTYPE_t h):
+cdef inline float64_t log_cosine_kernel(float64_t dist, float64_t h):
     """log of the cosine kernel for bandwidth h (unnormalized)"""
     if dist < h:
         return log(cos(0.5 * PI * dist / h))
@@ -424,8 +415,8 @@ cdef inline DTYPE_t log_cosine_kernel(DTYPE_t dist, DTYPE_t h):
         return NEG_INF
 
 
-cdef inline DTYPE_t compute_log_kernel(DTYPE_t dist, DTYPE_t h,
-                                       KernelType kernel):
+cdef inline float64_t compute_log_kernel(float64_t dist, float64_t h,
+                                         KernelType kernel):
     """Given a KernelType enumeration, compute the appropriate log-kernel"""
     if kernel == GAUSSIAN_KERNEL:
         return log_gaussian_kernel(dist, h)
@@ -441,27 +432,27 @@ cdef inline DTYPE_t compute_log_kernel(DTYPE_t dist, DTYPE_t h,
         return log_cosine_kernel(dist, h)
 
 
-#------------------------------------------------------------
+# ------------------------------------------------------------
 # Kernel norms are defined via the volume element V_n
 # and surface element S_(n-1) of an n-sphere.
-cdef DTYPE_t logVn(ITYPE_t n):
+cdef float64_t logVn(intp_t n):
     """V_n = pi^(n/2) / gamma(n/2 - 1)"""
     return 0.5 * n * LOG_PI - lgamma(0.5 * n + 1)
 
 
-cdef DTYPE_t logSn(ITYPE_t n):
+cdef float64_t logSn(intp_t n):
     """V_(n+1) = int_0^1 S_n r^n dr"""
     return LOG_2PI + logVn(n - 1)
 
 
-cdef DTYPE_t _log_kernel_norm(DTYPE_t h, ITYPE_t d,
-                              KernelType kernel) except -1:
+cdef float64_t _log_kernel_norm(float64_t h, intp_t d,
+                                KernelType kernel) except -1:
     """Given a KernelType enumeration, compute the kernel normalization.
 
     h is the bandwidth, d is the dimension.
     """
-    cdef DTYPE_t tmp, factor = 0
-    cdef ITYPE_t k
+    cdef float64_t tmp, factor = 0
+    cdef intp_t k
     if kernel == GAUSSIAN_KERNEL:
         factor = 0.5 * d * LOG_2PI
     elif kernel == TOPHAT_KERNEL:
@@ -526,29 +517,9 @@ def kernel_norm(h, d, kernel, return_log=False):
     else:
         return np.exp(result)
 
+{{for name_suffix, INPUT_DTYPE_t, INPUT_DTYPE, NPY_TYPE in implementation_specific_values}}
 
-######################################################################
-# Tree Utility Routines
-cdef inline void swap(DITYPE_t* arr, ITYPE_t i1, ITYPE_t i2):
-    """swap the values at index i1 and i2 of arr"""
-    cdef DITYPE_t tmp = arr[i1]
-    arr[i1] = arr[i2]
-    arr[i2] = tmp
-
-
-cdef inline void dual_swap(DTYPE_t* darr, ITYPE_t* iarr,
-                           ITYPE_t i1, ITYPE_t i2) nogil:
-    """swap the values at inex i1 and i2 of both darr and iarr"""
-    cdef DTYPE_t dtmp = darr[i1]
-    darr[i1] = darr[i2]
-    darr[i2] = dtmp
-
-    cdef ITYPE_t itmp = iarr[i1]
-    iarr[i1] = iarr[i2]
-    iarr[i2] = itmp
-
-
-cdef class NeighborsHeap:
+cdef class NeighborsHeap{{name_suffix}}:
     """A max-heap structure to keep track of distances/indices of neighbors
 
     This implements an efficient pre-allocated set of fixed-size heaps
@@ -563,24 +534,21 @@ cdef class NeighborsHeap:
     n_nbrs : int
         the size of each heap.
     """
-    cdef np.ndarray distances_arr
-    cdef np.ndarray indices_arr
-
-    cdef DTYPE_t[:, ::1] distances
-    cdef ITYPE_t[:, ::1] indices
+    cdef {{INPUT_DTYPE_t}}[:, ::1] distances
+    cdef intp_t[:, ::1] indices
 
     def __cinit__(self):
-        self.distances_arr = np.zeros((1, 1), dtype=DTYPE, order='C')
-        self.indices_arr = np.zeros((1, 1), dtype=ITYPE, order='C')
-        self.distances = get_memview_DTYPE_2D(self.distances_arr)
-        self.indices = get_memview_ITYPE_2D(self.indices_arr)
+        # One-element arrays are used as placeholders to prevent
+        # any problem due to potential access to those attributes
+        # (e.g. assigning to NULL or a to value in another segment).
+        self.distances = np.zeros((1, 1), dtype={{INPUT_DTYPE}}, order='C')
+        self.indices = np.zeros((1, 1), dtype=np.intp, order='C')
 
     def __init__(self, n_pts, n_nbrs):
-        self.distances_arr = np.full((n_pts, n_nbrs), np.inf, dtype=DTYPE,
-                                     order='C')
-        self.indices_arr = np.zeros((n_pts, n_nbrs), dtype=ITYPE, order='C')
-        self.distances = get_memview_DTYPE_2D(self.distances_arr)
-        self.indices = get_memview_ITYPE_2D(self.indices_arr)
+        self.distances = np.full(
+            (n_pts, n_nbrs), np.inf, dtype={{INPUT_DTYPE}}, order='C'
+        )
+        self.indices = np.zeros((n_pts, n_nbrs), dtype=np.intp, order='C')
 
     def get_arrays(self, sort=True):
         """Get the arrays of distances and indices within the heap.
@@ -590,146 +558,47 @@ cdef class NeighborsHeap:
         """
         if sort:
             self._sort()
-        return self.distances_arr, self.indices_arr
+        return self.distances.base, self.indices.base
 
-    cdef inline DTYPE_t largest(self, ITYPE_t row) nogil except -1:
+    cdef inline float64_t largest(self, intp_t row) except -1 nogil:
         """Return the largest distance in the given row"""
         return self.distances[row, 0]
 
-    def push(self, ITYPE_t row, DTYPE_t val, ITYPE_t i_val):
+    def push(self, intp_t row, float64_t val, intp_t i_val):
         return self._push(row, val, i_val)
 
-    cdef int _push(self, ITYPE_t row, DTYPE_t val,
-                   ITYPE_t i_val) nogil except -1:
+    cdef int _push(self, intp_t row, float64_t val,
+                   intp_t i_val) except -1 nogil:
         """push (val, i_val) into the given row"""
-        cdef ITYPE_t i, ic1, ic2, i_swap
-        cdef ITYPE_t size = self.distances.shape[1]
-        cdef DTYPE_t* dist_arr = &self.distances[row, 0]
-        cdef ITYPE_t* ind_arr = &self.indices[row, 0]
-
-        # check if val should be in heap
-        if val > dist_arr[0]:
-            return 0
-
-        # insert val at position zero
-        dist_arr[0] = val
-        ind_arr[0] = i_val
-
-        # descend the heap, swapping values until the max heap criterion is met
-        i = 0
-        while True:
-            ic1 = 2 * i + 1
-            ic2 = ic1 + 1
-
-            if ic1 >= size:
-                break
-            elif ic2 >= size:
-                if dist_arr[ic1] > val:
-                    i_swap = ic1
-                else:
-                    break
-            elif dist_arr[ic1] >= dist_arr[ic2]:
-                if val < dist_arr[ic1]:
-                    i_swap = ic1
-                else:
-                    break
-            else:
-                if val < dist_arr[ic2]:
-                    i_swap = ic2
-                else:
-                    break
-
-            dist_arr[i] = dist_arr[i_swap]
-            ind_arr[i] = ind_arr[i_swap]
-
-            i = i_swap
-
-        dist_arr[i] = val
-        ind_arr[i] = i_val
-
-        return 0
+        return heap_push(
+            values=&self.distances[row, 0],
+            indices=&self.indices[row, 0],
+            size=self.distances.shape[1],
+            val=val,
+            val_idx=i_val,
+        )
 
     cdef int _sort(self) except -1:
         """simultaneously sort the distances and indices"""
-        cdef DTYPE_t[:, ::1] distances = self.distances
-        cdef ITYPE_t[:, ::1] indices = self.indices
-        cdef ITYPE_t row
-        for row in range(distances.shape[0]):
-            _simultaneous_sort(&distances[row, 0],
-                               &indices[row, 0],
-                               distances.shape[1])
+        cdef intp_t row
+        for row in range(self.distances.shape[0]):
+            _simultaneous_sort(
+                dist=&self.distances[row, 0],
+                idx=&self.indices[row, 0],
+                size=self.distances.shape[1],
+            )
         return 0
 
-
-cdef int _simultaneous_sort(DTYPE_t* dist, ITYPE_t* idx,
-                            ITYPE_t size) nogil except -1:
-    """
-    Perform a recursive quicksort on the dist array, simultaneously
-    performing the same swaps on the idx array.  The equivalent in
-    numpy (though quite a bit slower) is
-
-    def simultaneous_sort(dist, idx):
-        i = np.argsort(dist)
-        return dist[i], idx[i]
-    """
-    cdef ITYPE_t pivot_idx, i, store_idx
-    cdef DTYPE_t pivot_val
-
-    # in the small-array case, do things efficiently
-    if size <= 1:
-        pass
-    elif size == 2:
-        if dist[0] > dist[1]:
-            dual_swap(dist, idx, 0, 1)
-    elif size == 3:
-        if dist[0] > dist[1]:
-            dual_swap(dist, idx, 0, 1)
-        if dist[1] > dist[2]:
-            dual_swap(dist, idx, 1, 2)
-            if dist[0] > dist[1]:
-                dual_swap(dist, idx, 0, 1)
-    else:
-        # Determine the pivot using the median-of-three rule.
-        # The smallest of the three is moved to the beginning of the array,
-        # the middle (the pivot value) is moved to the end, and the largest
-        # is moved to the pivot index.
-        pivot_idx = size / 2
-        if dist[0] > dist[size - 1]:
-            dual_swap(dist, idx, 0, size - 1)
-        if dist[size - 1] > dist[pivot_idx]:
-            dual_swap(dist, idx, size - 1, pivot_idx)
-            if dist[0] > dist[size - 1]:
-                dual_swap(dist, idx, 0, size - 1)
-        pivot_val = dist[size - 1]
-
-        # partition indices about pivot.  At the end of this operation,
-        # pivot_idx will contain the pivot value, everything to the left
-        # will be smaller, and everything to the right will be larger.
-        store_idx = 0
-        for i in range(size - 1):
-            if dist[i] < pivot_val:
-                dual_swap(dist, idx, i, store_idx)
-                store_idx += 1
-        dual_swap(dist, idx, store_idx, size - 1)
-        pivot_idx = store_idx
-
-        # recursively sort each side of the pivot
-        if pivot_idx > 1:
-            _simultaneous_sort(dist, idx, pivot_idx)
-        if pivot_idx + 2 < size:
-            _simultaneous_sort(dist + pivot_idx + 1,
-                               idx + pivot_idx + 1,
-                               size - pivot_idx - 1)
-    return 0
+{{endfor}}
 
 #------------------------------------------------------------
 # find_node_split_dim:
 #  this computes the equivalent of
 #  j_max = np.argmax(np.max(data, 0) - np.min(data, 0))
-cdef ITYPE_t find_node_split_dim(DTYPE_t* data,
-                                 ITYPE_t* node_indices,
-                                 ITYPE_t n_features,
-                                 ITYPE_t n_points) except -1:
+cdef intp_t find_node_split_dim(const floating* data,
+                                 const intp_t* node_indices,
+                                 intp_t n_features,
+                                 intp_t n_points) except -1:
     """Find the dimension with the largest spread.
 
     Parameters
@@ -756,8 +625,8 @@ cdef ITYPE_t find_node_split_dim(DTYPE_t* data,
 
     The cython version is much more efficient in both computation and memory.
     """
-    cdef DTYPE_t min_val, max_val, val, spread, max_spread
-    cdef ITYPE_t i, j, j_max
+    cdef float64_t min_val, max_val, val, spread, max_spread
+    cdef intp_t i, j, j_max
 
     j_max = 0
     max_spread = 0
@@ -776,77 +645,10 @@ cdef ITYPE_t find_node_split_dim(DTYPE_t* data,
     return j_max
 
 
-cdef int partition_node_indices(DTYPE_t* data,
-                                ITYPE_t* node_indices,
-                                ITYPE_t split_dim,
-                                ITYPE_t split_index,
-                                ITYPE_t n_features,
-                                ITYPE_t n_points) except -1:
-    """Partition points in the node into two equal-sized groups.
-
-    Upon return, the values in node_indices will be rearranged such that
-    (assuming numpy-style indexing):
-
-        data[node_indices[0:split_index], split_dim]
-          <= data[node_indices[split_index], split_dim]
-
-    and
-
-        data[node_indices[split_index], split_dim]
-          <= data[node_indices[split_index:n_points], split_dim]
-
-    The algorithm is essentially a partial in-place quicksort around a
-    set pivot.
-
-    Parameters
-    ----------
-    data : double pointer
-        Pointer to a 2D array of the training data, of shape [N, n_features].
-        N must be greater than any of the values in node_indices.
-    node_indices : int pointer
-        Pointer to a 1D array of length n_points.  This lists the indices of
-        each of the points within the current node.  This will be modified
-        in-place.
-    split_dim : int
-        the dimension on which to split.  This will usually be computed via
-        the routine ``find_node_split_dim``
-    split_index : int
-        the index within node_indices around which to split the points.
-
-    Returns
-    -------
-    status : int
-        integer exit status.  On return, the contents of node_indices are
-        modified as noted above.
-    """
-    cdef ITYPE_t left, right, midindex, i
-    cdef DTYPE_t d1, d2
-    left = 0
-    right = n_points - 1
-
-    while True:
-        midindex = left
-        for i in range(left, right):
-            d1 = data[node_indices[i] * n_features + split_dim]
-            d2 = data[node_indices[right] * n_features + split_dim]
-            if d1 < d2:
-                swap(node_indices, i, midindex)
-                midindex += 1
-        swap(node_indices, midindex, right)
-        if midindex == split_index:
-            break
-        elif midindex < split_index:
-            left = midindex + 1
-        else:
-            right = midindex - 1
-
-    return 0
-
-
 ######################################################################
 # NodeHeap : min-heap used to keep track of nodes during
 #            breadth-first query
-cdef inline void swap_nodes(NodeHeapData_t* arr, ITYPE_t i1, ITYPE_t i2):
+cdef inline void swap_nodes(NodeHeapData_t* arr, intp_t i1, intp_t i2):
     cdef NodeHeapData_t tmp = arr[i1]
     arr[i1] = arr[i2]
     arr[i2] = tmp
@@ -865,31 +667,32 @@ cdef class NodeHeap:
 
         heap[i].val < min(heap[2 * i + 1].val, heap[2 * i + 2].val)
     """
-    cdef np.ndarray data_arr
-    cdef NodeHeapData_t[::1] data
-    cdef ITYPE_t n
+    cdef NodeHeapData_t[:] data
+    cdef intp_t n
 
     def __cinit__(self):
-        self.data_arr = np.zeros(1, dtype=NodeHeapData, order='C')
-        self.data = get_memview_NodeHeapData_1D(self.data_arr)
+        # A one-elements array is used as a placeholder to prevent
+        # any problem due to potential access to this attribute
+        # (e.g. assigning to NULL or a to value in another segment).
+        self.data = np.zeros(1, dtype=NodeHeapData, order='C')
 
     def __init__(self, size_guess=100):
         size_guess = max(size_guess, 1)  # need space for at least one item
-        self.data_arr = np.zeros(size_guess, dtype=NodeHeapData, order='C')
-        self.data = get_memview_NodeHeapData_1D(self.data_arr)
+        self.data = np.zeros(size_guess, dtype=NodeHeapData, order='C')
         self.n = size_guess
         self.clear()
 
-    cdef int resize(self, ITYPE_t new_size) except -1:
+    cdef int resize(self, intp_t new_size) except -1:
         """Resize the heap to be either larger or smaller"""
-        cdef NodeHeapData_t *data_ptr
-        cdef NodeHeapData_t *new_data_ptr
-        cdef ITYPE_t i
-        cdef ITYPE_t size = self.data.shape[0]
-        cdef np.ndarray new_data_arr = np.zeros(new_size,
-                                                dtype=NodeHeapData)
-        cdef NodeHeapData_t[::1] new_data =\
-                                    get_memview_NodeHeapData_1D(new_data_arr)
+        cdef:
+            NodeHeapData_t *data_ptr
+            NodeHeapData_t *new_data_ptr
+            intp_t i
+            intp_t size = self.data.shape[0]
+            NodeHeapData_t[:] new_data = np.zeros(
+                new_size,
+                dtype=NodeHeapData,
+            )
 
         if size > 0 and new_size > 0:
             data_ptr = &self.data[0]
@@ -901,12 +704,11 @@ cdef class NodeHeap:
             self.n = new_size
 
         self.data = new_data
-        self.data_arr = new_data_arr
         return 0
 
     cdef int push(self, NodeHeapData_t data) except -1:
         """Push a new item onto the heap"""
-        cdef ITYPE_t i, i_parent
+        cdef intp_t i, i_parent
         cdef NodeHeapData_t* data_arr
         self.n += 1
         if self.n > self.data.shape[0]:
@@ -936,7 +738,7 @@ cdef class NodeHeap:
         if self.n == 0:
             raise ValueError('cannot pop on empty heap')
 
-        cdef ITYPE_t i, i_child1, i_child2, i_swap
+        cdef intp_t i, i_child1, i_child2, i_swap
         cdef NodeHeapData_t* data_arr = &self.data[0]
         cdef NodeHeapData_t popped_element = data_arr[0]
 
@@ -982,34 +784,34 @@ def newObj(obj):
     return obj.__new__(obj)
 
 
+{{for name_suffix, INPUT_DTYPE_t, INPUT_DTYPE, NPY_TYPE in implementation_specific_values}}
+
 ######################################################################
-# define the reverse mapping of VALID_METRICS
-from ._dist_metrics import get_valid_metric_ids
-VALID_METRIC_IDS = get_valid_metric_ids(VALID_METRICS)
+# define the reverse mapping of VALID_METRICS{{name_suffix}}
+from sklearn.metrics._dist_metrics import get_valid_metric_ids
+VALID_METRIC_IDS{{name_suffix}} = get_valid_metric_ids(VALID_METRICS{{name_suffix}})
 
 
 ######################################################################
 # Binary Tree class
-cdef class BinaryTree:
-
-    cdef np.ndarray data_arr
-    cdef np.ndarray sample_weight_arr
-    cdef np.ndarray idx_array_arr
-    cdef np.ndarray node_data_arr
-    cdef np.ndarray node_bounds_arr
-
-    cdef readonly DTYPE_t[:, ::1] data
-    cdef readonly DTYPE_t[::1] sample_weight
-    cdef public DTYPE_t sum_weight
-    cdef public ITYPE_t[::1] idx_array
-    cdef public NodeData_t[::1] node_data
-    cdef public DTYPE_t[:, :, ::1] node_bounds
-
-    cdef ITYPE_t leaf_size
-    cdef ITYPE_t n_levels
-    cdef ITYPE_t n_nodes
-
-    cdef DistanceMetric dist_metric
+cdef class BinaryTree{{name_suffix}}:
+
+    cdef readonly const {{INPUT_DTYPE_t}}[:, ::1] data
+    cdef readonly const {{INPUT_DTYPE_t}}[::1] sample_weight
+    cdef public float64_t sum_weight
+
+    # TODO: idx_array and node_bounds must not be const, but this change needs
+    # to happen in a way which preserves pickling
+    # See also: https://github.com/cython/cython/issues/5639
+    cdef public const intp_t[::1] idx_array
+    cdef public const NodeData_t[::1] node_data
+    cdef public const {{INPUT_DTYPE_t}}[:, :, ::1] node_bounds
+
+    cdef intp_t leaf_size
+    cdef intp_t n_levels
+    cdef intp_t n_nodes
+
+    cdef DistanceMetric{{name_suffix}} dist_metric
     cdef int euclidean
 
     # variables to keep track of building & querying stats
@@ -1018,22 +820,19 @@ cdef class BinaryTree:
     cdef int n_splits
     cdef int n_calls
 
-    valid_metrics = VALID_METRIC_IDS
+    valid_metrics = VALID_METRIC_IDS{{name_suffix}}
 
     # Use cinit to initialize all arrays to empty: this will prevent memory
     # errors and seg-faults in rare cases where __init__ is not called
+    # A one-elements array is used as a placeholder to prevent
+    # any problem due to potential access to this attribute
+    # (e.g. assigning to NULL or a to value in another segment).
     def __cinit__(self):
-        self.data_arr = np.empty((1, 1), dtype=DTYPE, order='C')
-        self.sample_weight_arr = np.empty(1, dtype=DTYPE, order='C')
-        self.idx_array_arr = np.empty(1, dtype=ITYPE, order='C')
-        self.node_data_arr = np.empty(1, dtype=NodeData, order='C')
-        self.node_bounds_arr = np.empty((1, 1, 1), dtype=DTYPE)
-
-        self.data = get_memview_DTYPE_2D(self.data_arr)
-        self.sample_weight = get_memview_DTYPE_1D(self.sample_weight_arr)
-        self.idx_array = get_memview_ITYPE_1D(self.idx_array_arr)
-        self.node_data = get_memview_NodeData_1D(self.node_data_arr)
-        self.node_bounds = get_memview_DTYPE_3D(self.node_bounds_arr)
+        self.data = np.empty((1, 1), dtype={{INPUT_DTYPE}}, order='C')
+        self.sample_weight = np.empty(1, dtype={{INPUT_DTYPE}}, order='C')
+        self.idx_array = np.empty(1, dtype=np.intp, order='C')
+        self.node_data = np.empty(1, dtype=NodeData, order='C')
+        self.node_bounds = np.empty((1, 1, 1), dtype={{INPUT_DTYPE}})
 
         self.leaf_size = 0
         self.n_levels = 0
@@ -1049,64 +848,58 @@ cdef class BinaryTree:
     def __init__(self, data,
                  leaf_size=40, metric='minkowski', sample_weight=None, **kwargs):
         # validate data
-        self.data_arr = check_array(data, dtype=DTYPE, order='C')
-        if self.data_arr.size == 0:
+        self.data = check_array(data, dtype={{INPUT_DTYPE}}, order='C')
+        if self.data.size == 0:
             raise ValueError("X is an empty array")
 
-        n_samples = self.data_arr.shape[0]
-        n_features = self.data_arr.shape[1]
+        n_samples = self.data.shape[0]
+        n_features = self.data.shape[1]
 
         if leaf_size < 1:
             raise ValueError("leaf_size must be greater than or equal to 1")
         self.leaf_size = leaf_size
 
-        self.dist_metric = DistanceMetric.get_metric(metric, **kwargs)
+        self.dist_metric = DistanceMetric.get_metric(metric, dtype={{INPUT_DTYPE}}, **kwargs)
         self.euclidean = (self.dist_metric.__class__.__name__
-                          == 'EuclideanDistance')
+                          == 'EuclideanDistance{{name_suffix}}')
 
         metric = self.dist_metric.__class__.__name__
-        if metric not in VALID_METRICS:
+        if metric not in VALID_METRICS{{name_suffix}}:
             raise ValueError('metric {metric} is not valid for '
                              '{BinaryTree}'.format(metric=metric,
-                                                   **DOC_DICT))
-        self.dist_metric._validate_data(self.data_arr)
+                                                   **DOC_DICT{{name_suffix}}))
+        self.dist_metric._validate_data(self.data)
 
         # determine number of levels in the tree, and from this
         # the number of nodes in the tree.  This results in leaf nodes
         # with numbers of points between leaf_size and 2 * leaf_size
         self.n_levels = int(
             np.log2(fmax(1, (n_samples - 1) / self.leaf_size)) + 1)
-        self.n_nodes = (2 ** self.n_levels) - 1
+        self.n_nodes =  (2 ** self.n_levels) - 1
 
         # allocate arrays for storage
-        self.idx_array_arr = np.arange(n_samples, dtype=ITYPE)
-        self.node_data_arr = np.zeros(self.n_nodes, dtype=NodeData)
+        self.idx_array = np.arange(n_samples, dtype=np.intp)
+        self.node_data = np.zeros(self.n_nodes, dtype=NodeData)
 
         self._update_sample_weight(n_samples, sample_weight)
-        self._update_memviews()
 
         # Allocate tree-specific data
-        allocate_data(self, self.n_nodes, n_features)
-        self._recursive_build(0, 0, n_samples)
+        allocate_data{{name_suffix}}(self, self.n_nodes, n_features)
+        self._recursive_build(
+            node_data=self.node_data.base,
+            i_node=0,
+            idx_start=0,
+            idx_end=n_samples
+        )
 
     def _update_sample_weight(self, n_samples, sample_weight):
         if sample_weight is not None:
-            self.sample_weight_arr = np.asarray(
-                sample_weight, dtype=DTYPE, order='C')
-            self.sample_weight = get_memview_DTYPE_1D(
-                self.sample_weight_arr)
+            self.sample_weight = np.asarray(
+                sample_weight, dtype={{INPUT_DTYPE}}, order='C')
             self.sum_weight = np.sum(self.sample_weight)
         else:
             self.sample_weight = None
-            self.sample_weight_arr = np.empty(1, dtype=DTYPE, order='C')
-            self.sum_weight =  n_samples
-
-    def _update_memviews(self):
-        self.data = get_memview_DTYPE_2D(self.data_arr)
-        self.idx_array = get_memview_ITYPE_1D(self.idx_array_arr)
-        self.node_data = get_memview_NodeData_1D(self.node_data_arr)
-        self.node_bounds = get_memview_DTYPE_3D(self.node_bounds_arr)
-
+            self.sum_weight =  n_samples
 
     def __reduce__(self):
         """
@@ -1120,15 +913,15 @@ cdef class BinaryTree:
         """
         if self.sample_weight is not None:
             # pass the numpy array
-            sample_weight_arr = self.sample_weight_arr
+            sample_weight = self.sample_weight.base
         else:
             # pass None to avoid confusion with the empty place holder
             # of size 1 from __cinit__
-            sample_weight_arr = None
-        return (self.data_arr,
-                self.idx_array_arr,
-                self.node_data_arr,
-                self.node_bounds_arr,
+            sample_weight = None
+        return (self.data.base,
+                self.idx_array.base,
+                self.node_data.base,
+                self.node_bounds.base,
                 int(self.leaf_size),
                 int(self.n_levels),
                 int(self.n_nodes),
@@ -1137,16 +930,16 @@ cdef class BinaryTree:
                 int(self.n_splits),
                 int(self.n_calls),
                 self.dist_metric,
-                sample_weight_arr)
+                sample_weight)
 
     def __setstate__(self, state):
         """
         set state for pickling
         """
-        self.data_arr = state[0]
-        self.idx_array_arr = state[1]
-        self.node_data_arr = state[2]
-        self.node_bounds_arr = state[3]
+        self.data = state[0]
+        self.idx_array = state[1]
+        self.node_data = state[2]
+        self.node_bounds = state[3]
         self.leaf_size = state[4]
         self.n_levels = state[5]
         self.n_nodes = state[6]
@@ -1155,17 +948,16 @@ cdef class BinaryTree:
         self.n_splits = state[9]
         self.n_calls = state[10]
         self.dist_metric = state[11]
-        sample_weight_arr = state[12]
+        sample_weight = state[12]
 
         self.euclidean = (self.dist_metric.__class__.__name__
-                          == 'EuclideanDistance')
-        n_samples = self.data_arr.shape[0]
-        self._update_sample_weight(n_samples, sample_weight_arr)
-        self._update_memviews()
+                          == 'EuclideanDistance64')
+        n_samples = self.data.shape[0]
+        self._update_sample_weight(n_samples, sample_weight)
 
     def get_tree_stats(self):
         """
-        get_tree_stats(self)
+        get_tree_stats()
 
         Get tree status.
 
@@ -1178,7 +970,7 @@ cdef class BinaryTree:
 
     def reset_n_calls(self):
         """
-        reset_n_calls(self)
+        reset_n_calls()
 
         Reset number of calls to 0.
         """
@@ -1186,7 +978,7 @@ cdef class BinaryTree:
 
     def get_n_calls(self):
         """
-        get_n_calls(self)
+        get_n_calls()
 
         Get number of calls.
 
@@ -1199,7 +991,7 @@ cdef class BinaryTree:
 
     def get_arrays(self):
         """
-        get_arrays(self)
+        get_arrays()
 
         Get data and node arrays.
 
@@ -1208,20 +1000,24 @@ cdef class BinaryTree:
         arrays: tuple of array
             Arrays for storing tree data, index, node data and node bounds.
         """
-        return (self.data_arr, self.idx_array_arr,
-                self.node_data_arr, self.node_bounds_arr)
-
-    cdef inline DTYPE_t dist(self, DTYPE_t* x1, DTYPE_t* x2,
-                             ITYPE_t size) nogil except -1:
+        return (
+            self.data.base,
+            self.idx_array.base,
+            self.node_data.base,
+            self.node_bounds.base,
+        )
+
+    cdef inline float64_t dist(self, const {{INPUT_DTYPE_t}}* x1, const {{INPUT_DTYPE_t}}* x2,
+                             intp_t size) except -1 nogil:
         """Compute the distance between arrays x1 and x2"""
         self.n_calls += 1
         if self.euclidean:
-            return euclidean_dist(x1, x2, size)
+            return euclidean_dist{{name_suffix}}(x1, x2, size)
         else:
             return self.dist_metric.dist(x1, x2, size)
 
-    cdef inline DTYPE_t rdist(self, DTYPE_t* x1, DTYPE_t* x2,
-                              ITYPE_t size) nogil except -1:
+    cdef inline float64_t rdist(self, const {{INPUT_DTYPE_t}}* x1, const {{INPUT_DTYPE_t}}* x2,
+                              intp_t size) except -1 nogil:
         """Compute the reduced distance between arrays x1 and x2.
 
         The reduced distance, defined for some metrics, is a quantity which
@@ -1231,12 +1027,12 @@ cdef class BinaryTree:
         """
         self.n_calls += 1
         if self.euclidean:
-            return euclidean_rdist(x1, x2, size)
+            return euclidean_rdist{{name_suffix}}(x1, x2, size)
         else:
             return self.dist_metric.rdist(x1, x2, size)
 
-    cdef int _recursive_build(self, ITYPE_t i_node, ITYPE_t idx_start,
-                              ITYPE_t idx_end) except -1:
+    cdef int _recursive_build(self, NodeData_t[::1] node_data, intp_t i_node, intp_t idx_start,
+                              intp_t idx_end) except -1:
         """Recursively build the tree.
 
         Parameters
@@ -1247,18 +1043,18 @@ cdef class BinaryTree:
             the bounding indices in the idx_array which define the points that
             belong to this node.
         """
-        cdef ITYPE_t imax
-        cdef ITYPE_t n_features = self.data.shape[1]
-        cdef ITYPE_t n_points = idx_end - idx_start
-        cdef ITYPE_t n_mid = n_points / 2
-        cdef ITYPE_t* idx_array = &self.idx_array[idx_start]
-        cdef DTYPE_t* data = &self.data[0, 0]
+        cdef intp_t imax
+        cdef intp_t n_features = self.data.shape[1]
+        cdef intp_t n_points = idx_end - idx_start
+        cdef intp_t n_mid = n_points / 2
+        cdef intp_t* idx_array = &self.idx_array[idx_start]
+        cdef const {{INPUT_DTYPE_t}}* data = &self.data[0, 0]
 
         # initialize node data
-        init_node(self, i_node, idx_start, idx_end)
+        init_node{{name_suffix}}(self, node_data, i_node, idx_start, idx_end)
 
         if 2 * i_node + 1 >= self.n_nodes:
-            self.node_data[i_node].is_leaf = True
+            node_data[i_node].is_leaf = True
             if idx_end - idx_start > 2 * self.leaf_size:
                 # this shouldn't happen if our memory allocation is correct
                 # we'll proactively prevent memory errors, but raise a
@@ -1273,18 +1069,18 @@ cdef class BinaryTree:
             import warnings
             warnings.warn("Internal: memory layout is flawed: "
                           "too many nodes allocated")
-            self.node_data[i_node].is_leaf = True
+            node_data[i_node].is_leaf = True
 
         else:
             # split node and recursively construct child nodes.
-            self.node_data[i_node].is_leaf = False
+            node_data[i_node].is_leaf = False
             i_max = find_node_split_dim(data, idx_array,
                                         n_features, n_points)
             partition_node_indices(data, idx_array, i_max, n_mid,
                                    n_features, n_points)
-            self._recursive_build(2 * i_node + 1,
+            self._recursive_build(node_data, 2 * i_node + 1,
                                   idx_start, idx_start + n_mid)
-            self._recursive_build(2 * i_node + 2,
+            self._recursive_build(node_data, 2 * i_node + 2,
                                   idx_start + n_mid, idx_end)
 
     def query(self, X, k=1, return_distance=True,
@@ -1332,7 +1128,7 @@ cdef class BinaryTree:
             corresponding point.
         """
         # XXX: we should allow X to be a pre-built tree.
-        X = check_array(X, dtype=DTYPE, order='C')
+        X = check_array(X, dtype={{INPUT_DTYPE}}, order='C')
 
         if X.shape[X.ndim - 1] != self.data.shape[1]:
             raise ValueError("query data dimension must "
@@ -1344,13 +1140,13 @@ cdef class BinaryTree:
 
         # flatten X, and save original shape information
         np_Xarr = X.reshape((-1, self.data.shape[1]))
-        cdef DTYPE_t[:, ::1] Xarr = get_memview_DTYPE_2D(np_Xarr)
-        cdef DTYPE_t reduced_dist_LB
-        cdef ITYPE_t i
-        cdef DTYPE_t* pt
+        cdef const {{INPUT_DTYPE_t}}[:, ::1] Xarr = np_Xarr
+        cdef float64_t reduced_dist_LB
+        cdef intp_t i
+        cdef const {{INPUT_DTYPE_t}}* pt
 
         # initialize heap for neighbors
-        cdef NeighborsHeap heap = NeighborsHeap(Xarr.shape[0], k)
+        cdef NeighborsHeap{{name_suffix}} heap = NeighborsHeap{{name_suffix}}(Xarr.shape[0], k)
 
         # node heap for breadth-first queries
         cdef NodeHeap nodeheap
@@ -1358,7 +1154,7 @@ cdef class BinaryTree:
             nodeheap = NodeHeap(self.data.shape[0] // self.leaf_size)
 
         # bounds is needed for the dual tree algorithm
-        cdef DTYPE_t[::1] bounds
+        cdef float64_t[::1] bounds
 
         self.n_trims = 0
         self.n_leaves = 0
@@ -1370,7 +1166,7 @@ cdef class BinaryTree:
             if breadth_first:
                 self._query_dual_breadthfirst(other, heap, nodeheap)
             else:
-                reduced_dist_LB = min_rdist_dual(self, 0, other, 0)
+                reduced_dist_LB = min_rdist_dual{{name_suffix}}(self, 0, other, 0)
                 bounds = np.full(other.node_data.shape[0], np.inf)
                 self._query_dual_depthfirst(0, other, 0, bounds,
                                             heap, reduced_dist_LB)
@@ -1384,7 +1180,7 @@ cdef class BinaryTree:
             else:
                 with nogil:
                     for i in range(Xarr.shape[0]):
-                        reduced_dist_LB = min_rdist(self, 0, pt)
+                        reduced_dist_LB = min_rdist{{name_suffix}}(self, 0, pt)
                         self._query_single_depthfirst(0, pt, i, heap,
                                                       reduced_dist_LB)
                         pt += Xarr.shape[1]
@@ -1460,54 +1256,53 @@ cdef class BinaryTree:
             raise ValueError("return_distance must be True "
                              "if sort_results is True")
 
-        cdef ITYPE_t i, count_i = 0
-        cdef ITYPE_t n_features = self.data.shape[1]
-        cdef DTYPE_t[::1] dist_arr_i
-        cdef ITYPE_t[::1] idx_arr_i, counts
-        cdef DTYPE_t* pt
-        cdef ITYPE_t** indices = NULL
-        cdef DTYPE_t** distances = NULL
+        cdef intp_t i, count_i = 0
+        cdef intp_t n_features = self.data.shape[1]
+        cdef {{INPUT_DTYPE_t}}[::1] dist_arr_i
+        cdef intp_t[::1] idx_arr_i, counts
+        cdef const {{INPUT_DTYPE_t}}* pt
+        cdef intp_t** indices = NULL
+        cdef {{INPUT_DTYPE_t}}** distances = NULL
 
         # validate X and prepare for query
-        X = check_array(X, dtype=DTYPE, order='C')
+        X = check_array(X, dtype={{INPUT_DTYPE}}, order='C')
 
         if X.shape[X.ndim - 1] != self.data.shape[1]:
             raise ValueError("query data dimension must "
                              "match training data dimension")
 
-        cdef DTYPE_t[:, ::1] Xarr =\
-                get_memview_DTYPE_2D(X.reshape((-1, self.data.shape[1])))
+        cdef const {{INPUT_DTYPE_t}}[:, ::1] Xarr = X.reshape((-1, self.data.shape[1]))
 
         # prepare r for query
-        r = np.asarray(r, dtype=DTYPE, order='C')
+        r = np.asarray(r, dtype=np.float64, order='C')
         r = np.atleast_1d(r)
         if r.shape == (1,):
-            r = np.full(X.shape[:X.ndim - 1], r[0], dtype=DTYPE)
+            r = np.full(X.shape[:X.ndim - 1], r[0], dtype=np.float64)
         else:
             if r.shape != X.shape[:X.ndim - 1]:
                 raise ValueError("r must be broadcastable to X.shape")
 
         rarr_np = r.reshape(-1)  # store explicitly to keep in scope
-        cdef DTYPE_t[::1] rarr = get_memview_DTYPE_1D(rarr_np)
+        cdef float64_t[::1] rarr = rarr_np
 
         if not count_only:
-            indices = calloc(Xarr.shape[0], sizeof(ITYPE_t*))
+            indices = calloc(Xarr.shape[0], sizeof(intp_t*))
             if indices == NULL:
                 raise MemoryError()
             if return_distance:
-                distances = calloc(Xarr.shape[0], sizeof(DTYPE_t*))
+                distances = <{{INPUT_DTYPE_t}}**>calloc(Xarr.shape[0], sizeof({{INPUT_DTYPE_t}}*))
                 if distances == NULL:
                     free(indices)
                     raise MemoryError()
 
-        np_idx_arr = np.zeros(self.data.shape[0], dtype=ITYPE)
-        idx_arr_i = get_memview_ITYPE_1D(np_idx_arr)
+        np_idx_arr = np.zeros(self.data.shape[0], dtype=np.intp)
+        idx_arr_i = np_idx_arr
 
-        np_dist_arr = np.zeros(self.data.shape[0], dtype=DTYPE)
-        dist_arr_i = get_memview_DTYPE_1D(np_dist_arr)
+        np_dist_arr = np.zeros(self.data.shape[0], dtype={{INPUT_DTYPE}})
+        dist_arr_i = np_dist_arr
 
-        counts_arr = np.zeros(Xarr.shape[0], dtype=ITYPE)
-        counts = get_memview_ITYPE_1D(counts_arr)
+        counts_arr = np.zeros(Xarr.shape[0], dtype=np.intp)
+        counts = counts_arr
 
         pt = &Xarr[0, 0]
         memory_error = False
@@ -1528,19 +1323,19 @@ cdef class BinaryTree:
                                        counts[i])
 
                 # equivalent to: indices[i] = np_idx_arr[:counts[i]].copy()
-                indices[i] = malloc(counts[i] * sizeof(ITYPE_t))
+                indices[i] = malloc(counts[i] * sizeof(intp_t))
                 if indices[i] == NULL:
                     memory_error = True
                     break
-                memcpy(indices[i], &idx_arr_i[0], counts[i] * sizeof(ITYPE_t))
+                memcpy(indices[i], &idx_arr_i[0], counts[i] * sizeof(intp_t))
 
                 if return_distance:
                     # equivalent to: distances[i] = np_dist_arr[:counts[i]].copy()
-                    distances[i] = malloc(counts[i] * sizeof(DTYPE_t))
+                    distances[i] = <{{INPUT_DTYPE_t}}*>malloc(counts[i] * sizeof({{INPUT_DTYPE_t}}))
                     if distances[i] == NULL:
                         memory_error = True
                         break
-                    memcpy(distances[i], &dist_arr_i[0], counts[i] * sizeof(DTYPE_t))
+                    memcpy(distances[i], &dist_arr_i[0], counts[i] * sizeof({{INPUT_DTYPE_t}}))
 
         try:
             if memory_error:
@@ -1554,16 +1349,18 @@ cdef class BinaryTree:
                 distances_npy = np.zeros(Xarr.shape[0], dtype='object')
                 for i in range(Xarr.shape[0]):
                     # make a new numpy array that wraps the existing data
-                    indices_npy[i] = np.PyArray_SimpleNewFromData(1, &counts[i], np.NPY_INTP, indices[i])
+                    # TODO: remove the explicit cast to cnp.intp_t* when cython min version >= 3.0
+                    indices_npy[i] = cnp.PyArray_SimpleNewFromData(1, &counts[i], cnp.NPY_INTP, indices[i])
                     # make sure the data will be freed when the numpy array is garbage collected
-                    PyArray_ENABLEFLAGS(indices_npy[i], np.NPY_OWNDATA)
+                    PyArray_ENABLEFLAGS(indices_npy[i], cnp.NPY_ARRAY_OWNDATA)
                     # make sure the data is not freed twice
                     indices[i] = NULL
 
                     # make a new numpy array that wraps the existing data
-                    distances_npy[i] = np.PyArray_SimpleNewFromData(1, &counts[i], np.NPY_DOUBLE, distances[i])
+                    # TODO: remove the explicit cast to cnp.intp_t* when cython min version >= 3.0
+                    distances_npy[i] = cnp.PyArray_SimpleNewFromData(1, &counts[i], {{NPY_TYPE}}, distances[i])
                     # make sure the data will be freed when the numpy array is garbage collected
-                    PyArray_ENABLEFLAGS(distances_npy[i], np.NPY_OWNDATA)
+                    PyArray_ENABLEFLAGS(distances_npy[i], cnp.NPY_ARRAY_OWNDATA)
                     # make sure the data is not freed twice
                     distances[i] = NULL
 
@@ -1574,15 +1371,16 @@ cdef class BinaryTree:
                 indices_npy = np.zeros(Xarr.shape[0], dtype='object')
                 for i in range(Xarr.shape[0]):
                     # make a new numpy array that wraps the existing data
-                    indices_npy[i] = np.PyArray_SimpleNewFromData(1, &counts[i], np.NPY_INTP, indices[i])
+                    # TODO: remove the explicit cast to cnp.intp_t* when cython min version >= 3.0
+                    indices_npy[i] = cnp.PyArray_SimpleNewFromData(1, &counts[i], cnp.NPY_INTP, indices[i])
                     # make sure the data will be freed when the numpy array is garbage collected
-                    PyArray_ENABLEFLAGS(indices_npy[i], np.NPY_OWNDATA)
+                    PyArray_ENABLEFLAGS(indices_npy[i], cnp.NPY_ARRAY_OWNDATA)
                     # make sure the data is not freed twice
                     indices[i] = NULL
 
                 # deflatten results
                 return indices_npy.reshape(X.shape[:X.ndim - 1])
-        except:
+        except MemoryError:
             # free any buffer that is not owned by a numpy array
             for i in range(Xarr.shape[0]):
                 free(indices[i])
@@ -1593,12 +1391,11 @@ cdef class BinaryTree:
             free(indices)
             free(distances)
 
-
     def kernel_density(self, X, h, kernel='gaussian',
                        atol=0, rtol=1E-8,
                        breadth_first=True, return_log=False):
         """
-        kernel_density(self, X, h, kernel='gaussian', atol=0, rtol=1E-8,
+        kernel_density(X, h, kernel='gaussian', atol=0, rtol=1E-8,
                        breadth_first=True, return_log=False)
 
         Compute the kernel density estimate at points X with the given kernel,
@@ -1620,11 +1417,16 @@ cdef class BinaryTree:
             - 'linear'
             - 'cosine'
             Default is kernel = 'gaussian'
-        atol, rtol : float, default=0, 1e-8
-            Specify the desired relative and absolute tolerance of the result.
-            If the true result is K_true, then the returned result K_ret
+        atol : float, default=0
+            Specify the desired absolute tolerance of the result.
+            If the true result is `K_true`, then the returned result `K_ret`
+            satisfies ``abs(K_true - K_ret) < atol + rtol * K_ret``
+            The default is zero (i.e. machine precision).
+        rtol : float, default=1e-8
+            Specify the desired relative tolerance of the result.
+            If the true result is `K_true`, then the returned result `K_ret`
             satisfies ``abs(K_true - K_ret) < atol + rtol * K_ret``
-            The default is zero (i.e. machine precision) for both.
+            The default is `1e-8` (i.e. machine precision).
         breadth_first : bool, default=False
             If True, use a breadth-first search.  If False (default) use a
             depth-first search.  Breadth-first is generally faster for
@@ -1638,15 +1440,15 @@ cdef class BinaryTree:
         density : ndarray of shape X.shape[:-1]
             The array of (log)-density evaluations
         """
-        cdef DTYPE_t h_c = h
-        cdef DTYPE_t log_atol = log(atol)
-        cdef DTYPE_t log_rtol = log(rtol)
-        cdef DTYPE_t log_min_bound, log_max_bound, log_bound_spread
-        cdef DTYPE_t dist_LB = 0, dist_UB = 0
-
-        cdef ITYPE_t n_samples = self.data.shape[0]
-        cdef ITYPE_t n_features = self.data.shape[1]
-        cdef ITYPE_t i
+        cdef float64_t h_c = h
+        cdef float64_t log_atol = log(atol)
+        cdef float64_t log_rtol = log(rtol)
+        cdef float64_t log_min_bound, log_max_bound, log_bound_spread
+        cdef float64_t dist_LB = 0, dist_UB = 0
+
+        cdef intp_t n_samples = self.data.shape[0]
+        cdef intp_t n_features = self.data.shape[1]
+        cdef intp_t i
         cdef KernelType kernel_c
 
         # validate kernel
@@ -1665,35 +1467,35 @@ cdef class BinaryTree:
         else:
             raise ValueError("kernel = '%s' not recognized" % kernel)
 
-        cdef DTYPE_t log_knorm = _log_kernel_norm(h_c, n_features, kernel_c)
+        cdef float64_t log_knorm = _log_kernel_norm(h_c, n_features, kernel_c)
 
         # validate X and prepare for query
-        X = check_array(X, dtype=DTYPE, order='C')
+        X = check_array(X, dtype={{INPUT_DTYPE}}, order='C')
 
         if X.shape[X.ndim - 1] != n_features:
             raise ValueError("query data dimension must "
                              "match training data dimension")
         Xarr_np = X.reshape((-1, n_features))
-        cdef DTYPE_t[:, ::1] Xarr = get_memview_DTYPE_2D(Xarr_np)
+        cdef const {{INPUT_DTYPE_t}}[:, ::1] Xarr = Xarr_np
 
-        log_density_arr = np.zeros(Xarr.shape[0], dtype=DTYPE)
-        cdef DTYPE_t[::1] log_density = get_memview_DTYPE_1D(log_density_arr)
+        log_density_arr = np.zeros(Xarr.shape[0], dtype={{INPUT_DTYPE}})
+        cdef {{INPUT_DTYPE_t}}[::1] log_density = log_density_arr
 
-        cdef DTYPE_t* pt = &Xarr[0, 0]
+        cdef const {{INPUT_DTYPE_t}}* pt = &Xarr[0, 0]
 
         cdef NodeHeap nodeheap
         if breadth_first:
             nodeheap = NodeHeap(self.data.shape[0] // self.leaf_size)
-        cdef DTYPE_t[::1] node_log_min_bounds
-        cdef DTYPE_t[::1] node_bound_widths
+        cdef float64_t[::1] node_log_min_bounds
+        cdef float64_t[::1] node_bound_widths
         # TODO: implement dual tree approach.
         #       this is difficult because of the need to cache values
         #       computed between node pairs.
         if breadth_first:
             node_log_min_bounds_arr = np.full(self.n_nodes, -np.inf)
-            node_log_min_bounds = get_memview_DTYPE_1D(node_log_min_bounds_arr)
+            node_log_min_bounds = node_log_min_bounds_arr
             node_bound_widths_arr = np.zeros(self.n_nodes)
-            node_bound_widths = get_memview_DTYPE_1D(node_bound_widths_arr)
+            node_bound_widths = node_bound_widths_arr
             for i in range(Xarr.shape[0]):
                 log_density[i] = self._kde_single_breadthfirst(
                                             pt, kernel_c, h_c,
@@ -1704,7 +1506,7 @@ cdef class BinaryTree:
                 pt += n_features
         else:
             for i in range(Xarr.shape[0]):
-                min_max_dist(self, 0, pt, &dist_LB, &dist_UB)
+                min_max_dist{{name_suffix}}(self, 0, pt, &dist_LB, &dist_UB)
                 # compute max & min bounds on density within top node
                 log_min_bound = (log(self.sum_weight) +
                                  compute_log_kernel(dist_UB,
@@ -1758,33 +1560,33 @@ cdef class BinaryTree:
             counts[i] contains the number of pairs of points with distance
             less than or equal to r[i]
         """
-        cdef ITYPE_t n_features = self.data.shape[1]
-        cdef ITYPE_t i
+        cdef intp_t n_features = self.data.shape[1]
+        cdef intp_t i
 
         # validate X and prepare for query
-        X = check_array(X, dtype=DTYPE, order='C')
+        X = check_array(X, dtype={{INPUT_DTYPE}}, order='C')
 
         if X.shape[X.ndim - 1] != self.data.shape[1]:
             raise ValueError("query data dimension must "
                              "match training data dimension")
 
         np_Xarr = X.reshape((-1, self.data.shape[1]))
-        cdef DTYPE_t[:, ::1] Xarr = get_memview_DTYPE_2D(np_Xarr)
+        cdef {{INPUT_DTYPE_t}}[:, ::1] Xarr = np_Xarr
 
         # prepare r for query
-        r = np.asarray(r, dtype=DTYPE, order='C')
+        r = np.asarray(r, dtype=np.float64, order='C')
         r = np.atleast_1d(r)
         if r.ndim != 1:
             raise ValueError("r must be a 1-dimensional array")
         i_rsort = np.argsort(r)
         rarr_np = r[i_rsort]  # needed to keep memory in scope
-        cdef DTYPE_t[::1] rarr = get_memview_DTYPE_1D(rarr_np)
+        cdef float64_t[::1] rarr = rarr_np
 
         # create array to hold counts
-        count = np.zeros(r.shape[0], dtype=ITYPE)
-        cdef ITYPE_t[::1] carr = get_memview_ITYPE_1D(count)
+        count = np.zeros(r.shape[0], dtype=np.intp)
+        cdef intp_t[::1] carr = count
 
-        cdef DTYPE_t* pt = &Xarr[0, 0]
+        cdef const {{INPUT_DTYPE_t}}* pt = &Xarr[0, 0]
 
         if dualtree:
             other = self.__class__(Xarr, metric=self.dist_metric,
@@ -1799,25 +1601,29 @@ cdef class BinaryTree:
 
         return count
 
-    cdef int _query_single_depthfirst(self, ITYPE_t i_node,
-                                      DTYPE_t* pt, ITYPE_t i_pt,
-                                      NeighborsHeap heap,
-                                      DTYPE_t reduced_dist_LB) nogil except -1:
+    cdef int _query_single_depthfirst(
+        self,
+        intp_t i_node,
+        const {{INPUT_DTYPE_t}}* pt,
+        intp_t i_pt,
+        NeighborsHeap{{name_suffix}} heap,
+        float64_t reduced_dist_LB,
+    ) except -1 nogil:
         """Recursive Single-tree k-neighbors query, depth-first approach"""
         cdef NodeData_t node_info = self.node_data[i_node]
 
-        cdef DTYPE_t dist_pt, reduced_dist_LB_1, reduced_dist_LB_2
-        cdef ITYPE_t i, i1, i2
+        cdef float64_t dist_pt, reduced_dist_LB_1, reduced_dist_LB_2
+        cdef intp_t i, i1, i2
 
-        cdef DTYPE_t* data = &self.data[0, 0]
+        cdef const {{INPUT_DTYPE_t}}* data = &self.data[0, 0]
 
-        #------------------------------------------------------------
+        # ------------------------------------------------------------
         # Case 1: query point is outside node radius:
         #         trim it from the query
         if reduced_dist_LB > heap.largest(i_pt):
             self.n_trims += 1
 
-        #------------------------------------------------------------
+        # ------------------------------------------------------------
         # Case 2: this is a leaf node.  Update set of nearby points
         elif node_info.is_leaf:
             self.n_leaves += 1
@@ -1825,18 +1631,17 @@ cdef class BinaryTree:
                 dist_pt = self.rdist(pt,
                                      &self.data[self.idx_array[i], 0],
                                      self.data.shape[1])
-                if dist_pt < heap.largest(i_pt):
-                    heap._push(i_pt, dist_pt, self.idx_array[i])
+                heap._push(i_pt, dist_pt, self.idx_array[i])
 
-        #------------------------------------------------------------
+        # ------------------------------------------------------------
         # Case 3: Node is not a leaf.  Recursively query subnodes
         #         starting with the closest
         else:
             self.n_splits += 1
             i1 = 2 * i_node + 1
             i2 = i1 + 1
-            reduced_dist_LB_1 = min_rdist(self, i1, pt)
-            reduced_dist_LB_2 = min_rdist(self, i2, pt)
+            reduced_dist_LB_1 = min_rdist{{name_suffix}}(self, i1, pt)
+            reduced_dist_LB_2 = min_rdist{{name_suffix}}(self, i2, pt)
 
             # recursively query subnodes
             if reduced_dist_LB_1 <= reduced_dist_LB_2:
@@ -1851,19 +1656,22 @@ cdef class BinaryTree:
                                               reduced_dist_LB_1)
         return 0
 
-    cdef int _query_single_breadthfirst(self, DTYPE_t* pt,
-                                        ITYPE_t i_pt,
-                                        NeighborsHeap heap,
-                                        NodeHeap nodeheap) except -1:
+    cdef int _query_single_breadthfirst(
+        self,
+        const {{INPUT_DTYPE_t}}* pt,
+        intp_t i_pt,
+        NeighborsHeap{{name_suffix}} heap,
+        NodeHeap nodeheap,
+    ) except -1:
         """Non-recursive single-tree k-neighbors query, breadth-first search"""
-        cdef ITYPE_t i, i_node
-        cdef DTYPE_t dist_pt, reduced_dist_LB
-        cdef NodeData_t* node_data = &self.node_data[0]
-        cdef DTYPE_t* data = &self.data[0, 0]
+        cdef intp_t i, i_node
+        cdef float64_t dist_pt, reduced_dist_LB
+        cdef const NodeData_t* node_data = &self.node_data[0]
+        cdef const {{INPUT_DTYPE_t}}* data = &self.data[0, 0]
 
         # Set up the node heap and push the head node onto it
         cdef NodeHeapData_t nodeheap_item
-        nodeheap_item.val = min_rdist(self, 0, pt)
+        nodeheap_item.val = min_rdist{{name_suffix}}(self, 0, pt)
         nodeheap_item.i1 = 0
         nodeheap.push(nodeheap_item)
 
@@ -1873,13 +1681,13 @@ cdef class BinaryTree:
             i_node = nodeheap_item.i1
             node_info = node_data[i_node]
 
-            #------------------------------------------------------------
+            # ------------------------------------------------------------
             # Case 1: query point is outside node radius:
             #         trim it from the query
             if reduced_dist_LB > heap.largest(i_pt):
                 self.n_trims += 1
 
-            #------------------------------------------------------------
+            # ------------------------------------------------------------
             # Case 2: this is a leaf node.  Update set of nearby points
             elif node_data[i_node].is_leaf:
                 self.n_leaves += 1
@@ -1888,24 +1696,27 @@ cdef class BinaryTree:
                     dist_pt = self.rdist(pt,
                                          &self.data[self.idx_array[i], 0],
                                          self.data.shape[1])
-                    if dist_pt < heap.largest(i_pt):
-                        heap._push(i_pt, dist_pt, self.idx_array[i])
+                    heap._push(i_pt, dist_pt, self.idx_array[i])
 
-            #------------------------------------------------------------
+            # ------------------------------------------------------------
             # Case 3: Node is not a leaf.  Add subnodes to the node heap
             else:
                 self.n_splits += 1
                 for i in range(2 * i_node + 1, 2 * i_node + 3):
                     nodeheap_item.i1 = i
-                    nodeheap_item.val = min_rdist(self, i, pt)
+                    nodeheap_item.val = min_rdist{{name_suffix}}(self, i, pt)
                     nodeheap.push(nodeheap_item)
         return 0
 
-    cdef int _query_dual_depthfirst(self, ITYPE_t i_node1,
-                                    BinaryTree other, ITYPE_t i_node2,
-                                    DTYPE_t[::1] bounds,
-                                    NeighborsHeap heap,
-                                    DTYPE_t reduced_dist_LB) except -1:
+    cdef int _query_dual_depthfirst(
+        self,
+        intp_t i_node1,
+        BinaryTree{{name_suffix}} other,
+        intp_t i_node2,
+        float64_t[::1] bounds,
+        NeighborsHeap{{name_suffix}} heap,
+        float64_t reduced_dist_LB,
+    ) except -1:
         """Recursive dual-tree k-neighbors query, depth-first"""
         # note that the array `bounds` is maintained such that
         # bounds[i] is the largest distance among any of the
@@ -1913,20 +1724,20 @@ cdef class BinaryTree:
         cdef NodeData_t node_info1 = self.node_data[i_node1]
         cdef NodeData_t node_info2 = other.node_data[i_node2]
 
-        cdef DTYPE_t* data1 = &self.data[0, 0]
-        cdef DTYPE_t* data2 = &other.data[0, 0]
-        cdef ITYPE_t n_features = self.data.shape[1]
+        cdef const {{INPUT_DTYPE_t}}* data1 = &self.data[0, 0]
+        cdef const {{INPUT_DTYPE_t}}* data2 = &other.data[0, 0]
+        cdef intp_t n_features = self.data.shape[1]
 
-        cdef DTYPE_t bound_max, dist_pt, reduced_dist_LB1, reduced_dist_LB2
-        cdef ITYPE_t i1, i2, i_pt, i_parent
+        cdef float64_t bound_max, dist_pt, reduced_dist_LB1, reduced_dist_LB2
+        cdef intp_t i1, i2, i_pt, i_parent
 
-        #------------------------------------------------------------
+        # ------------------------------------------------------------
         # Case 1: nodes are further apart than the current bound:
         #         trim both from the query
         if reduced_dist_LB > bounds[i_node2]:
             pass
 
-        #------------------------------------------------------------
+        # ------------------------------------------------------------
         # Case 2: both nodes are leaves:
         #         do a brute-force search comparing all pairs
         elif node_info1.is_leaf and node_info2.is_leaf:
@@ -1943,8 +1754,7 @@ cdef class BinaryTree:
                         data1 + n_features * self.idx_array[i1],
                         data2 + n_features * i_pt,
                         n_features)
-                    if dist_pt < heap.largest(i_pt):
-                        heap._push(i_pt, dist_pt, self.idx_array[i1])
+                    heap._push(i_pt, dist_pt, self.idx_array[i1])
 
                 # keep track of node bound
                 bounds[i_node2] = fmax(bounds[i_node2],
@@ -1961,14 +1771,14 @@ cdef class BinaryTree:
                 else:
                     break
 
-        #------------------------------------------------------------
+        # ------------------------------------------------------------
         # Case 3a: node 1 is a leaf or is smaller: split node 2 and
         #          recursively query, starting with the nearest subnode
         elif node_info1.is_leaf or (not node_info2.is_leaf
                                     and node_info2.radius > node_info1.radius):
-            reduced_dist_LB1 = min_rdist_dual(self, i_node1,
+            reduced_dist_LB1 = min_rdist_dual{{name_suffix}}(self, i_node1,
                                               other, 2 * i_node2 + 1)
-            reduced_dist_LB2 = min_rdist_dual(self, i_node1,
+            reduced_dist_LB2 = min_rdist_dual{{name_suffix}}(self, i_node1,
                                               other, 2 * i_node2 + 2)
 
             if reduced_dist_LB1 < reduced_dist_LB2:
@@ -1982,13 +1792,13 @@ cdef class BinaryTree:
                 self._query_dual_depthfirst(i_node1, other, 2 * i_node2 + 1,
                                             bounds, heap, reduced_dist_LB1)
 
-        #------------------------------------------------------------
+        # ------------------------------------------------------------
         # Case 3b: node 2 is a leaf or is smaller: split node 1 and
         #          recursively query, starting with the nearest subnode
         else:
-            reduced_dist_LB1 = min_rdist_dual(self, 2 * i_node1 + 1,
+            reduced_dist_LB1 = min_rdist_dual{{name_suffix}}(self, 2 * i_node1 + 1,
                                               other, i_node2)
-            reduced_dist_LB2 = min_rdist_dual(self, 2 * i_node1 + 2,
+            reduced_dist_LB2 = min_rdist_dual{{name_suffix}}(self, 2 * i_node1 + 2,
                                               other, i_node2)
 
             if reduced_dist_LB1 < reduced_dist_LB2:
@@ -2003,23 +1813,26 @@ cdef class BinaryTree:
                                             bounds, heap, reduced_dist_LB1)
         return 0
 
-    cdef int _query_dual_breadthfirst(self, BinaryTree other,
-                                      NeighborsHeap heap,
-                                      NodeHeap nodeheap) except -1:
+    cdef int _query_dual_breadthfirst(
+        self,
+        BinaryTree{{name_suffix}} other,
+        NeighborsHeap{{name_suffix}} heap,
+        NodeHeap nodeheap,
+    ) except -1:
         """Non-recursive dual-tree k-neighbors query, breadth-first"""
-        cdef ITYPE_t i, i1, i2, i_node1, i_node2, i_pt
-        cdef DTYPE_t dist_pt, reduced_dist_LB
-        cdef DTYPE_t[::1] bounds = np.full(other.node_data.shape[0], np.inf)
-        cdef NodeData_t* node_data1 = &self.node_data[0]
-        cdef NodeData_t* node_data2 = &other.node_data[0]
+        cdef intp_t i, i1, i2, i_node1, i_node2, i_pt
+        cdef float64_t dist_pt, reduced_dist_LB
+        cdef float64_t[::1] bounds = np.full(other.node_data.shape[0], np.inf)
+        cdef const NodeData_t* node_data1 = &self.node_data[0]
+        cdef const NodeData_t* node_data2 = &other.node_data[0]
         cdef NodeData_t node_info1, node_info2
-        cdef DTYPE_t* data1 = &self.data[0, 0]
-        cdef DTYPE_t* data2 = &other.data[0, 0]
-        cdef ITYPE_t n_features = self.data.shape[1]
+        cdef const {{INPUT_DTYPE_t}}* data1 = &self.data[0, 0]
+        cdef const {{INPUT_DTYPE_t}}* data2 = &other.data[0, 0]
+        cdef intp_t n_features = self.data.shape[1]
 
         # Set up the node heap and push the head nodes onto it
         cdef NodeHeapData_t nodeheap_item
-        nodeheap_item.val = min_rdist_dual(self, 0, other, 0)
+        nodeheap_item.val = min_rdist_dual{{name_suffix}}(self, 0, other, 0)
         nodeheap_item.i1 = 0
         nodeheap_item.i2 = 0
         nodeheap.push(nodeheap_item)
@@ -2033,13 +1846,13 @@ cdef class BinaryTree:
             node_info1 = node_data1[i_node1]
             node_info2 = node_data2[i_node2]
 
-            #------------------------------------------------------------
+            # ------------------------------------------------------------
             # Case 1: nodes are further apart than the current bound:
             #         trim both from the query
             if reduced_dist_LB > bounds[i_node2]:
                 pass
 
-            #------------------------------------------------------------
+            # ------------------------------------------------------------
             # Case 2: both nodes are leaves:
             #         do a brute-force search comparing all pairs
             elif node_info1.is_leaf and node_info2.is_leaf:
@@ -2056,14 +1869,13 @@ cdef class BinaryTree:
                             data1 + n_features * self.idx_array[i1],
                             data2 + n_features * i_pt,
                             n_features)
-                        if dist_pt < heap.largest(i_pt):
-                            heap._push(i_pt, dist_pt, self.idx_array[i1])
+                        heap._push(i_pt, dist_pt, self.idx_array[i1])
 
                     # keep track of node bound
                     bounds[i_node2] = fmax(bounds[i_node2],
                                            heap.largest(i_pt))
 
-            #------------------------------------------------------------
+            # ------------------------------------------------------------
             # Case 3a: node 1 is a leaf or is smaller: split node 2 and
             #          recursively query, starting with the nearest subnode
             elif node_info1.is_leaf or (not node_info2.is_leaf
@@ -2072,49 +1884,52 @@ cdef class BinaryTree:
                 nodeheap_item.i1 = i_node1
                 for i2 in range(2 * i_node2 + 1, 2 * i_node2 + 3):
                     nodeheap_item.i2 = i2
-                    nodeheap_item.val = min_rdist_dual(self, i_node1,
+                    nodeheap_item.val = min_rdist_dual{{name_suffix}}(self, i_node1,
                                                        other, i2)
                     nodeheap.push(nodeheap_item)
 
-            #------------------------------------------------------------
+            # ------------------------------------------------------------
             # Case 3b: node 2 is a leaf or is smaller: split node 1 and
             #          recursively query, starting with the nearest subnode
             else:
                 nodeheap_item.i2 = i_node2
                 for i1 in range(2 * i_node1 + 1, 2 * i_node1 + 3):
                     nodeheap_item.i1 = i1
-                    nodeheap_item.val = min_rdist_dual(self, i1,
+                    nodeheap_item.val = min_rdist_dual{{name_suffix}}(self, i1,
                                                        other, i_node2)
                     nodeheap.push(nodeheap_item)
         return 0
 
-    cdef ITYPE_t _query_radius_single(self,
-                                      ITYPE_t i_node,
-                                      DTYPE_t* pt, DTYPE_t r,
-                                      ITYPE_t* indices,
-                                      DTYPE_t* distances,
-                                      ITYPE_t count,
-                                      int count_only,
-                                      int return_distance) nogil:
+    cdef intp_t _query_radius_single(
+        self,
+        intp_t i_node,
+        const {{INPUT_DTYPE_t}}* pt,
+        float64_t r,
+        intp_t* indices,
+        {{INPUT_DTYPE_t}}* distances,
+        intp_t count,
+        int count_only,
+        int return_distance,
+    ) noexcept nogil:
         """recursive single-tree radius query, depth-first"""
-        cdef DTYPE_t* data = &self.data[0, 0]
-        cdef ITYPE_t* idx_array = &self.idx_array[0]
-        cdef ITYPE_t n_features = self.data.shape[1]
+        cdef const {{INPUT_DTYPE_t}}* data = &self.data[0, 0]
+        cdef intp_t* idx_array = &self.idx_array[0]
+        cdef intp_t n_features = self.data.shape[1]
         cdef NodeData_t node_info = self.node_data[i_node]
 
-        cdef ITYPE_t i
-        cdef DTYPE_t reduced_r
+        cdef intp_t i
+        cdef float64_t reduced_r
 
-        cdef DTYPE_t dist_pt, dist_LB = 0, dist_UB = 0
-        min_max_dist(self, i_node, pt, &dist_LB, &dist_UB)
+        cdef float64_t dist_pt, dist_LB = 0, dist_UB = 0
+        min_max_dist{{name_suffix}}(self, i_node, pt, &dist_LB, &dist_UB)
 
-        #------------------------------------------------------------
+        # ------------------------------------------------------------
         # Case 1: all node points are outside distance r.
         #         prune this branch.
         if dist_LB > r:
             pass
 
-        #------------------------------------------------------------
+        # ------------------------------------------------------------
         # Case 2: all node points are within distance r
         #         add all points to neighbors
         elif dist_UB <= r:
@@ -2131,7 +1946,7 @@ cdef class BinaryTree:
                                                      n_features)
                     count += 1
 
-        #------------------------------------------------------------
+        # ------------------------------------------------------------
         # Case 3: this is a leaf node.  Go through all points to
         #         determine if they fall within radius
         elif node_info.is_leaf:
@@ -2152,7 +1967,7 @@ cdef class BinaryTree:
                                 self.dist_metric._rdist_to_dist(dist_pt)
                     count += 1
 
-        #------------------------------------------------------------
+        # ------------------------------------------------------------
         # Case 4: Node is not a leaf.  Recursively query subnodes
         else:
             count = self._query_radius_single(2 * i_node + 1, pt, r,
@@ -2164,13 +1979,17 @@ cdef class BinaryTree:
 
         return count
 
-    cdef DTYPE_t _kde_single_breadthfirst(self, DTYPE_t* pt,
-                                          KernelType kernel, DTYPE_t h,
-                                          DTYPE_t log_knorm,
-                                          DTYPE_t log_atol, DTYPE_t log_rtol,
-                                          NodeHeap nodeheap,
-                                          DTYPE_t* node_log_min_bounds,
-                                          DTYPE_t* node_log_bound_spreads):
+    cdef float64_t _kde_single_breadthfirst(
+        self, const {{INPUT_DTYPE_t}}* pt,
+        KernelType kernel,
+        float64_t h,
+        float64_t log_knorm,
+        float64_t log_atol,
+        float64_t log_rtol,
+        NodeHeap nodeheap,
+        float64_t* node_log_min_bounds,
+        float64_t* node_log_bound_spreads,
+    ):
         """non-recursive single-tree kernel density estimation"""
         # For the given point, node_log_min_bounds and node_log_bound_spreads
         # will encode the current bounds on the density between the point
@@ -2179,42 +1998,42 @@ cdef class BinaryTree:
         # keep track of the global bounds on density.  The procedure here is
         # to split nodes, updating these bounds, until the bounds are within
         # atol & rtol.
-        cdef ITYPE_t i, i1, i2, i_node
-        cdef DTYPE_t N1, N2
-        cdef DTYPE_t global_log_min_bound, global_log_bound_spread
-        cdef DTYPE_t global_log_max_bound
+        cdef intp_t i, i1, i2, i_node
+        cdef float64_t N1, N2
+        cdef float64_t global_log_min_bound, global_log_bound_spread
+        cdef float64_t global_log_max_bound
 
-        cdef DTYPE_t* data = &self.data[0, 0]
+        cdef const {{INPUT_DTYPE_t}}* data = &self.data[0, 0]
         cdef bint with_sample_weight = self.sample_weight is not None
-        cdef DTYPE_t* sample_weight
+        cdef const {{INPUT_DTYPE_t}}* sample_weight
         if with_sample_weight:
             sample_weight = &self.sample_weight[0]
-        cdef ITYPE_t* idx_array = &self.idx_array[0]
-        cdef NodeData_t* node_data = &self.node_data[0]
-        cdef DTYPE_t N
-        cdef DTYPE_t log_weight
+        cdef intp_t* idx_array = &self.idx_array[0]
+        cdef const NodeData_t* node_data = &self.node_data[0]
+        cdef float64_t N
+        cdef float64_t log_weight
         if with_sample_weight:
             N = self.sum_weight
         else:
-            N =  self.data.shape[0]
-        cdef ITYPE_t n_features = self.data.shape[1]
+            N =  self.data.shape[0]
+        cdef intp_t n_features = self.data.shape[1]
 
         cdef NodeData_t node_info
-        cdef DTYPE_t dist_pt, log_density
-        cdef DTYPE_t dist_LB_1 = 0, dist_LB_2 = 0
-        cdef DTYPE_t dist_UB_1 = 0, dist_UB_2 = 0
+        cdef float64_t dist_pt, log_density
+        cdef float64_t dist_LB_1 = 0, dist_LB_2 = 0
+        cdef float64_t dist_UB_1 = 0, dist_UB_2 = 0
 
-        cdef DTYPE_t dist_UB, dist_LB
+        cdef float64_t dist_UB, dist_LB
 
         # push the top node to the heap
         cdef NodeHeapData_t nodeheap_item
-        nodeheap_item.val = min_dist(self, 0, pt)
+        nodeheap_item.val = min_dist{{name_suffix}}(self, 0, pt)
         nodeheap_item.i1 = 0
         nodeheap.push(nodeheap_item)
 
-        global_log_min_bound = log(N) + compute_log_kernel(max_dist(self,
-                                                                    0, pt),
-                                                           h, kernel)
+        global_log_min_bound = log(N) + compute_log_kernel(
+            max_dist{{name_suffix}}(self, 0, pt), h, kernel
+        )
         global_log_max_bound = log(N) + compute_log_kernel(nodeheap_item.val,
                                                            h, kernel)
         global_log_bound_spread = logsubexp(global_log_max_bound,
@@ -2234,21 +2053,21 @@ cdef class BinaryTree:
             else:
                 N1 = node_info.idx_end - node_info.idx_start
 
-            #------------------------------------------------------------
+            # ------------------------------------------------------------
             # Case 1: local bounds are equal to within per-point tolerance.
             if (log_knorm + node_log_bound_spreads[i_node] - log(N1) + log(N)
                 <= logaddexp(log_atol, (log_rtol + log_knorm
                                         + node_log_min_bounds[i_node]))):
                 pass
 
-            #------------------------------------------------------------
+            # ------------------------------------------------------------
             # Case 2: global bounds are within rtol & atol.
             elif (log_knorm + global_log_bound_spread
                   <= logaddexp(log_atol,
                                log_rtol + log_knorm + global_log_min_bound)):
                 break
 
-            #------------------------------------------------------------
+            # ------------------------------------------------------------
             # Case 3: node is a leaf. Count contributions from all points
             elif node_info.is_leaf:
                 global_log_min_bound =\
@@ -2268,7 +2087,7 @@ cdef class BinaryTree:
                     global_log_min_bound = logaddexp(global_log_min_bound,
                                                      log_density + log_weight)
 
-            #------------------------------------------------------------
+            # ------------------------------------------------------------
             # Case 4: split node and query subnodes
             else:
                 i1 = 2 * i_node + 1
@@ -2283,8 +2102,8 @@ cdef class BinaryTree:
                     N1 = node_data[i1].idx_end - node_data[i1].idx_start
                     N2 = node_data[i2].idx_end - node_data[i2].idx_start
 
-                min_max_dist(self, i1, pt, &dist_LB_1, &dist_UB_1)
-                min_max_dist(self, i2, pt, &dist_LB_2, &dist_UB_2)
+                min_max_dist{{name_suffix}}(self, i1, pt, &dist_LB_1, &dist_UB_1)
+                min_max_dist{{name_suffix}}(self, i2, pt, &dist_LB_2, &dist_UB_2)
 
                 node_log_min_bounds[i1] = (log(N1) +
                                            compute_log_kernel(dist_UB_1,
@@ -2329,63 +2148,70 @@ cdef class BinaryTree:
                          global_log_bound_spread - log(2))
 
     cdef int _kde_single_depthfirst(
-                   self, ITYPE_t i_node, DTYPE_t* pt,
-                   KernelType kernel, DTYPE_t h,
-                   DTYPE_t log_knorm,
-                   DTYPE_t log_atol, DTYPE_t log_rtol,
-                   DTYPE_t local_log_min_bound,
-                   DTYPE_t local_log_bound_spread,
-                   DTYPE_t* global_log_min_bound,
-                   DTYPE_t* global_log_bound_spread) except -1:
+        self,
+        intp_t i_node,
+        const {{INPUT_DTYPE_t}}* pt,
+        KernelType kernel,
+        float64_t h,
+        float64_t log_knorm,
+        float64_t log_atol,
+        float64_t log_rtol,
+        float64_t local_log_min_bound,
+        float64_t local_log_bound_spread,
+        float64_t* global_log_min_bound,
+        float64_t* global_log_bound_spread,
+    ) except -1:
         """recursive single-tree kernel density estimate, depth-first"""
         # For the given point, local_min_bound and local_max_bound give the
         # minimum and maximum density for the current node, while
         # global_min_bound and global_max_bound give the minimum and maximum
         # density over the entire tree.  We recurse down until global_min_bound
         # and global_max_bound are within rtol and atol.
-        cdef ITYPE_t i, i1, i2, iw, start, end
-        cdef DTYPE_t N1, N2
+        cdef intp_t i, i1, i2, iw, start, end
+        cdef float64_t N1, N2
 
-        cdef DTYPE_t* data = &self.data[0, 0]
-        cdef NodeData_t* node_data = &self.node_data[0]
+        cdef const {{INPUT_DTYPE_t}}* data = &self.data[0, 0]
+        cdef const NodeData_t* node_data = &self.node_data[0]
         cdef bint with_sample_weight = self.sample_weight is not None
-        cdef DTYPE_t* sample_weight
-        cdef DTYPE_t log_weight
+        cdef const {{INPUT_DTYPE_t}}* sample_weight
+        cdef float64_t log_weight
         if with_sample_weight:
             sample_weight = &self.sample_weight[0]
-        cdef ITYPE_t* idx_array = &self.idx_array[0]
-        cdef ITYPE_t n_features = self.data.shape[1]
+        cdef intp_t* idx_array = &self.idx_array[0]
+        cdef intp_t n_features = self.data.shape[1]
 
         cdef NodeData_t node_info = self.node_data[i_node]
-        cdef DTYPE_t dist_pt, log_dens_contribution
+        cdef float64_t dist_pt, log_dens_contribution
 
-        cdef DTYPE_t child1_log_min_bound, child2_log_min_bound
-        cdef DTYPE_t child1_log_bound_spread, child2_log_bound_spread
-        cdef DTYPE_t dist_UB = 0, dist_LB = 0
+        cdef float64_t child1_log_min_bound, child2_log_min_bound
+        cdef float64_t child1_log_bound_spread, child2_log_bound_spread
+        cdef float64_t dist_UB = 0, dist_LB = 0
 
         if with_sample_weight:
-            N1  = _total_node_weight(node_data, sample_weight,
-                                     idx_array, i_node)
+            N1 = _total_node_weight(node_data, sample_weight,
+                                    idx_array, i_node)
             N2 = self.sum_weight
         else:
-            N1 = (node_info.idx_end - node_info.idx_start)
-            N2 = self.data.shape[0]
+            N1 = (node_info.idx_end - node_info.idx_start)
+            N2 = self.data.shape[0]
 
-        #------------------------------------------------------------
+        # ------------------------------------------------------------
         # Case 1: local bounds are equal to within errors.  Return
-        if (log_knorm + local_log_bound_spread - log(N1) + log(N2)
-            <= logaddexp(log_atol, (log_rtol + log_knorm
-                                    + local_log_min_bound))):
+        if (
+            log_knorm + local_log_bound_spread - log(N1) + log(N2)
+            <= logaddexp(log_atol, (log_rtol + log_knorm + local_log_min_bound))
+        ):
             pass
 
-        #------------------------------------------------------------
+        # ------------------------------------------------------------
         # Case 2: global bounds are within rtol & atol. Return
-        elif (log_knorm + global_log_bound_spread[0]
-            <= logaddexp(log_atol, (log_rtol + log_knorm
-                                    + global_log_min_bound[0]))):
+        elif (
+            log_knorm + global_log_bound_spread[0]
+            <= logaddexp(log_atol, (log_rtol + log_knorm + global_log_min_bound[0]))
+        ):
             pass
 
-        #------------------------------------------------------------
+        # ------------------------------------------------------------
         # Case 3: node is a leaf. Count contributions from all points
         elif node_info.is_leaf:
             global_log_min_bound[0] = logsubexp(global_log_min_bound[0],
@@ -2404,7 +2230,7 @@ cdef class BinaryTree:
                                                     (log_dens_contribution +
                                                      log_weight))
 
-        #------------------------------------------------------------
+        # ------------------------------------------------------------
         # Case 4: split node and query subnodes
         else:
             i1 = 2 * i_node + 1
@@ -2416,10 +2242,10 @@ cdef class BinaryTree:
                 N2 = _total_node_weight(node_data, sample_weight,
                                         idx_array, i2)
             else:
-                N1 = (self.node_data[i1].idx_end - self.node_data[i1].idx_start)
-                N2 = (self.node_data[i2].idx_end - self.node_data[i2].idx_start)
+                N1 = (self.node_data[i1].idx_end - self.node_data[i1].idx_start)
+                N2 = (self.node_data[i2].idx_end - self.node_data[i2].idx_start)
 
-            min_max_dist(self, i1, pt, &dist_LB, &dist_UB)
+            min_max_dist{{name_suffix}}(self, i1, pt, &dist_LB, &dist_UB)
             child1_log_min_bound = log(N1) + compute_log_kernel(dist_UB, h,
                                                                 kernel)
             child1_log_bound_spread = logsubexp(log(N1) +
@@ -2427,7 +2253,7 @@ cdef class BinaryTree:
                                                                    kernel),
                                                 child1_log_min_bound)
 
-            min_max_dist(self, i2, pt, &dist_LB, &dist_UB)
+            min_max_dist{{name_suffix}}(self, i2, pt, &dist_LB, &dist_UB)
             child2_log_min_bound = log(N2) + compute_log_kernel(dist_UB, h,
                                                                 kernel)
             child2_log_bound_spread = logsubexp(log(N2) +
@@ -2463,22 +2289,28 @@ cdef class BinaryTree:
                                         global_log_bound_spread)
         return 0
 
-    cdef int _two_point_single(self, ITYPE_t i_node, DTYPE_t* pt, DTYPE_t* r,
-                               ITYPE_t* count, ITYPE_t i_min,
-                               ITYPE_t i_max) except -1:
+    cdef int _two_point_single(
+        self,
+        intp_t i_node,
+        const {{INPUT_DTYPE_t}}* pt,
+        float64_t* r,
+        intp_t* count,
+        intp_t i_min,
+        intp_t i_max,
+    ) except -1:
         """recursive single-tree two-point correlation function query"""
-        cdef DTYPE_t* data = &self.data[0, 0]
-        cdef ITYPE_t* idx_array = &self.idx_array[0]
-        cdef ITYPE_t n_features = self.data.shape[1]
+        cdef const {{INPUT_DTYPE_t}}* data = &self.data[0, 0]
+        cdef intp_t* idx_array = &self.idx_array[0]
+        cdef intp_t n_features = self.data.shape[1]
         cdef NodeData_t node_info = self.node_data[i_node]
 
-        cdef ITYPE_t i, j, Npts
-        cdef DTYPE_t reduced_r
+        cdef intp_t i, j, Npts
+        cdef float64_t reduced_r
 
-        cdef DTYPE_t dist_pt, dist_LB = 0, dist_UB = 0
-        min_max_dist(self, i_node, pt, &dist_LB, &dist_UB)
+        cdef float64_t dist_pt, dist_LB = 0, dist_UB = 0
+        min_max_dist{{name_suffix}}(self, i_node, pt, &dist_LB, &dist_UB)
 
-        #------------------------------------------------------------
+        # ------------------------------------------------------------
         # Go through bounds and check for cuts
         while i_min < i_max:
             if dist_LB > r[i_min]:
@@ -2512,28 +2344,34 @@ cdef class BinaryTree:
                                        count, i_min, i_max)
         return 0
 
-    cdef int _two_point_dual(self, ITYPE_t i_node1,
-                             BinaryTree other, ITYPE_t i_node2,
-                             DTYPE_t* r, ITYPE_t* count,
-                             ITYPE_t i_min, ITYPE_t i_max) except -1:
+    cdef int _two_point_dual(
+        self,
+        intp_t i_node1,
+        BinaryTree{{name_suffix}} other,
+        intp_t i_node2,
+        float64_t* r,
+        intp_t* count,
+        intp_t i_min,
+        intp_t i_max,
+    ) except -1:
         """recursive dual-tree two-point correlation function query"""
-        cdef DTYPE_t* data1 = &self.data[0, 0]
-        cdef DTYPE_t* data2 = &other.data[0, 0]
-        cdef ITYPE_t* idx_array1 = &self.idx_array[0]
-        cdef ITYPE_t* idx_array2 = &other.idx_array[0]
+        cdef const {{INPUT_DTYPE_t}}* data1 = &self.data[0, 0]
+        cdef const {{INPUT_DTYPE_t}}* data2 = &other.data[0, 0]
+        cdef intp_t* idx_array1 = &self.idx_array[0]
+        cdef intp_t* idx_array2 = &other.idx_array[0]
         cdef NodeData_t node_info1 = self.node_data[i_node1]
         cdef NodeData_t node_info2 = other.node_data[i_node2]
 
-        cdef ITYPE_t n_features = self.data.shape[1]
+        cdef intp_t n_features = self.data.shape[1]
 
-        cdef ITYPE_t i1, i2, j, Npts
-        cdef DTYPE_t reduced_r
+        cdef intp_t i1, i2, j, Npts
+        cdef float64_t reduced_r
 
-        cdef DTYPE_t dist_pt, dist_LB = 0, dist_UB = 0
-        dist_LB = min_dist_dual(self, i_node1, other, i_node2)
-        dist_UB = max_dist_dual(self, i_node1, other, i_node2)
+        cdef float64_t dist_pt, dist_LB = 0, dist_UB = 0
+        dist_LB = min_dist_dual{{name_suffix}}(self, i_node1, other, i_node2)
+        dist_UB = max_dist_dual{{name_suffix}}(self, i_node1, other, i_node2)
 
-        #------------------------------------------------------------
+        # ------------------------------------------------------------
         # Go through bounds and check for cuts
         while i_min < i_max:
             if dist_LB > r[i_min]:
@@ -2577,29 +2415,19 @@ cdef class BinaryTree:
                                          r, count, i_min, i_max)
 
             else:
-                 # neither is a leaf: split & query both
+                # neither is a leaf: split & query both
                 for i1 in range(2 * i_node1 + 1, 2 * i_node1 + 3):
                     for i2 in range(2 * i_node2 + 1, 2 * i_node2 + 3):
                         self._two_point_dual(i1, other, i2,
                                              r, count, i_min, i_max)
         return 0
 
+{{endfor}}
 
 ######################################################################
 # Python functions for benchmarking and testing C implementations
 
-def load_heap(DTYPE_t[:, ::1] X, ITYPE_t k):
-    """test fully loading the heap"""
-    assert k <= X.shape[1]
-    cdef NeighborsHeap heap = NeighborsHeap(X.shape[0], k)
-    cdef ITYPE_t i, j
-    for i in range(X.shape[0]):
-        for j in range(X.shape[1]):
-            heap._push(i, X[i, j], j)
-    return heap.get_arrays()
-
-
-def simultaneous_sort(DTYPE_t[:, ::1] distances, ITYPE_t[:, ::1] indices):
+def simultaneous_sort(float64_t[:, ::1] distances, intp_t[:, ::1] indices):
     """In-place simultaneous sort the given row of the arrays
 
     This python wrapper exists primarily to enable unit testing
@@ -2607,22 +2435,22 @@ def simultaneous_sort(DTYPE_t[:, ::1] distances, ITYPE_t[:, ::1] indices):
     """
     assert distances.shape[0] == indices.shape[0]
     assert distances.shape[1] == indices.shape[1]
-    cdef ITYPE_t row
+    cdef intp_t row
     for row in range(distances.shape[0]):
         _simultaneous_sort(&distances[row, 0],
                            &indices[row, 0],
                            distances.shape[1])
 
 
-def nodeheap_sort(DTYPE_t[::1] vals):
+def nodeheap_sort(float64_t[::1] vals):
     """In-place reverse sort of vals using NodeHeap"""
-    cdef ITYPE_t[::1] indices = np.zeros(vals.shape[0], dtype=ITYPE)
-    cdef DTYPE_t[::1] vals_sorted = np.zeros_like(vals)
+    cdef intp_t[::1] indices = np.zeros(vals.shape[0], dtype=np.intp)
+    cdef float64_t[::1] vals_sorted = np.zeros_like(vals)
 
     # use initial size 0 to check corner case
     cdef NodeHeap heap = NodeHeap(0)
     cdef NodeHeapData_t data
-    cdef ITYPE_t i
+    cdef intp_t i
     for i in range(vals.shape[0]):
         data.val = vals[i]
         data.i1 = i
@@ -2637,12 +2465,14 @@ def nodeheap_sort(DTYPE_t[::1] vals):
     return np.asarray(vals_sorted), np.asarray(indices)
 
 
-cdef inline DTYPE_t _total_node_weight(NodeData_t* node_data,
-                                       DTYPE_t* sample_weight,
-                                       ITYPE_t* idx_array,
-                                       ITYPE_t i_node):
-    cdef ITYPE_t i
-    cdef DTYPE_t N = 0.0
+cdef inline float64_t _total_node_weight(
+    const NodeData_t* node_data,
+    const floating* sample_weight,
+    const intp_t* idx_array,
+    intp_t i_node,
+):
+    cdef intp_t i
+    cdef float64_t N = 0.0
     for i in range(node_data[i_node].idx_start, node_data[i_node].idx_end):
         N += sample_weight[idx_array[i]]
     return N
diff --git a/sklearn/neighbors/_classification.py b/sklearn/neighbors/_classification.py
index 71b869977f6aa..c70b83cb1d3bd 100644
--- a/sklearn/neighbors/_classification.py
+++ b/sklearn/neighbors/_classification.py
@@ -1,28 +1,43 @@
 """Nearest Neighbor Classification"""
 
-# Authors: Jake Vanderplas 
-#          Fabian Pedregosa 
-#          Alexandre Gramfort 
-#          Sparseness support by Lars Buitinck
-#          Multi-output support by Arnaud Joly 
-#
-# License: BSD 3 clause (C) INRIA, University of Amsterdam
-
-import numpy as np
-from scipy import stats
-from ..utils.extmath import weighted_mode
-from ..utils.validation import _is_arraylike, _num_samples
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import warnings
-from ._base import _check_weights, _get_weights
-from ._base import NeighborsBase, KNeighborsMixin, RadiusNeighborsMixin
-from ..base import ClassifierMixin
-from ..utils.validation import _deprecate_positional_args
+from numbers import Integral
 
+import numpy as np
 
-class KNeighborsClassifier(KNeighborsMixin,
-                           ClassifierMixin,
-                           NeighborsBase):
+from sklearn.neighbors._base import _check_precomputed
+
+from ..base import ClassifierMixin, _fit_context
+from ..metrics._pairwise_distances_reduction import (
+    ArgKminClassMode,
+    RadiusNeighborsClassMode,
+)
+from ..utils._param_validation import StrOptions
+from ..utils.arrayfuncs import _all_with_any_reduction_axis_1
+from ..utils.extmath import weighted_mode
+from ..utils.fixes import _mode
+from ..utils.validation import (
+    _is_arraylike,
+    _num_samples,
+    check_is_fitted,
+    validate_data,
+)
+from ._base import KNeighborsMixin, NeighborsBase, RadiusNeighborsMixin, _get_weights
+
+
+def _adjusted_metric(metric, metric_kwargs, p=None):
+    metric_kwargs = metric_kwargs or {}
+    if metric == "minkowski":
+        metric_kwargs["p"] = p
+        if p == 2:
+            metric = "euclidean"
+    return metric, metric_kwargs
+
+
+class KNeighborsClassifier(KNeighborsMixin, ClassifierMixin, NeighborsBase):
     """Classifier implementing the k-nearest neighbors vote.
 
     Read more in the :ref:`User Guide `.
@@ -32,8 +47,8 @@ class KNeighborsClassifier(KNeighborsMixin,
     n_neighbors : int, default=5
         Number of neighbors to use by default for :meth:`kneighbors` queries.
 
-    weights : {'uniform', 'distance'} or callable, default='uniform'
-        weight function used in prediction.  Possible values:
+    weights : {'uniform', 'distance'}, callable or None, default='uniform'
+        Weight function used in prediction.  Possible values:
 
         - 'uniform' : uniform weights.  All points in each neighborhood
           are weighted equally.
@@ -44,6 +59,11 @@ class KNeighborsClassifier(KNeighborsMixin,
           array of distances, and returns an array of the same shape
           containing the weights.
 
+        Refer to the example entitled
+        :ref:`sphx_glr_auto_examples_neighbors_plot_classification.py`
+        showing the impact of the `weights` parameter on the decision
+        boundary.
+
     algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, default='auto'
         Algorithm used to compute the nearest neighbors:
 
@@ -62,19 +82,29 @@ class KNeighborsClassifier(KNeighborsMixin,
         required to store the tree.  The optimal value depends on the
         nature of the problem.
 
-    p : int, default=2
-        Power parameter for the Minkowski metric. When p = 1, this is
-        equivalent to using manhattan_distance (l1), and euclidean_distance
-        (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used.
+    p : float, default=2
+        Power parameter for the Minkowski metric. When p = 1, this is equivalent
+        to using manhattan_distance (l1), and euclidean_distance (l2) for p = 2.
+        For arbitrary p, minkowski_distance (l_p) is used. This parameter is expected
+        to be positive.
 
     metric : str or callable, default='minkowski'
-        the distance metric to use for the tree.  The default metric is
-        minkowski, and with p=2 is equivalent to the standard Euclidean
-        metric. See the documentation of :class:`DistanceMetric` for a
-        list of available metrics.
+        Metric to use for distance computation. Default is "minkowski", which
+        results in the standard Euclidean distance when p = 2. See the
+        documentation of `scipy.spatial.distance
+        `_ and
+        the metrics listed in
+        :class:`~sklearn.metrics.pairwise.distance_metrics` for valid metric
+        values.
+
         If metric is "precomputed", X is assumed to be a distance matrix and
-        must be square during fit. X may be a :term:`sparse graph`,
-        in which case only "nonzero" elements may be considered neighbors.
+        must be square during fit. X may be a :term:`sparse graph`, in which
+        case only "nonzero" elements may be considered neighbors.
+
+        If metric is a callable function, it takes two arrays representing 1D
+        vectors as inputs and must return one value indicating the distance
+        between those vectors. This works for Scipy's metrics, but is less
+        efficient than passing the metric name as a string.
 
     metric_params : dict, default=None
         Additional keyword arguments for the metric function.
@@ -102,6 +132,17 @@ class KNeighborsClassifier(KNeighborsMixin,
         `p` parameter value if the `effective_metric_` attribute is set to
         'minkowski'.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     n_samples_fit_ : int
         Number of samples in the fitted data.
 
@@ -109,25 +150,12 @@ class KNeighborsClassifier(KNeighborsMixin,
         False when `y`'s shape is (n_samples, ) or (n_samples, 1) during fit
         otherwise True.
 
-    Examples
-    --------
-    >>> X = [[0], [1], [2], [3]]
-    >>> y = [0, 0, 1, 1]
-    >>> from sklearn.neighbors import KNeighborsClassifier
-    >>> neigh = KNeighborsClassifier(n_neighbors=3)
-    >>> neigh.fit(X, y)
-    KNeighborsClassifier(...)
-    >>> print(neigh.predict([[1.1]]))
-    [0]
-    >>> print(neigh.predict_proba([[0.9]]))
-    [[0.66666667 0.33333333]]
-
     See Also
     --------
-    RadiusNeighborsClassifier
-    KNeighborsRegressor
-    RadiusNeighborsRegressor
-    NearestNeighbors
+    RadiusNeighborsClassifier: Classifier based on neighbors within a fixed radius.
+    KNeighborsRegressor: Regression based on k-nearest neighbors.
+    RadiusNeighborsRegressor: Regression based on neighbors within a fixed radius.
+    NearestNeighbors: Unsupervised learner for implementing neighbor searches.
 
     Notes
     -----
@@ -142,21 +170,54 @@ class KNeighborsClassifier(KNeighborsMixin,
        training data.
 
     https://en.wikipedia.org/wiki/K-nearest_neighbor_algorithm
+
+    Examples
+    --------
+    >>> X = [[0], [1], [2], [3]]
+    >>> y = [0, 0, 1, 1]
+    >>> from sklearn.neighbors import KNeighborsClassifier
+    >>> neigh = KNeighborsClassifier(n_neighbors=3)
+    >>> neigh.fit(X, y)
+    KNeighborsClassifier(...)
+    >>> print(neigh.predict([[1.1]]))
+    [0]
+    >>> print(neigh.predict_proba([[0.9]]))
+    [[0.666 0.333]]
     """
 
-    @_deprecate_positional_args
-    def __init__(self, n_neighbors=5, *,
-                 weights='uniform', algorithm='auto', leaf_size=30,
-                 p=2, metric='minkowski', metric_params=None, n_jobs=None,
-                 **kwargs):
+    _parameter_constraints: dict = {**NeighborsBase._parameter_constraints}
+    _parameter_constraints.pop("radius")
+    _parameter_constraints.update(
+        {"weights": [StrOptions({"uniform", "distance"}), callable, None]}
+    )
+
+    def __init__(
+        self,
+        n_neighbors=5,
+        *,
+        weights="uniform",
+        algorithm="auto",
+        leaf_size=30,
+        p=2,
+        metric="minkowski",
+        metric_params=None,
+        n_jobs=None,
+    ):
         super().__init__(
             n_neighbors=n_neighbors,
             algorithm=algorithm,
-            leaf_size=leaf_size, metric=metric, p=p,
+            leaf_size=leaf_size,
+            metric=metric,
+            p=p,
             metric_params=metric_params,
-            n_jobs=n_jobs, **kwargs)
-        self.weights = _check_weights(weights)
-
+            n_jobs=n_jobs,
+        )
+        self.weights = weights
+
+    @_fit_context(
+        # KNeighborsClassifier.metric is not validated yet
+        prefer_skip_nested_validation=False
+    )
     def fit(self, X, y):
         """Fit the k-nearest neighbors classifier from the training dataset.
 
@@ -182,18 +243,39 @@ def predict(self, X):
 
         Parameters
         ----------
-        X : array-like of shape (n_queries, n_features), \
-                or (n_queries, n_indexed) if metric == 'precomputed'
-            Test samples.
+        X : {array-like, sparse matrix} of shape (n_queries, n_features), \
+                or (n_queries, n_indexed) if metric == 'precomputed', or None
+            Test samples. If `None`, predictions for all indexed points are
+            returned; in this case, points are not considered their own
+            neighbors.
 
         Returns
         -------
         y : ndarray of shape (n_queries,) or (n_queries, n_outputs)
             Class labels for each data sample.
         """
-        X = self._validate_data(X, accept_sparse='csr', reset=False)
+        check_is_fitted(self, "_fit_method")
+        if self.weights == "uniform":
+            if self._fit_method == "brute" and ArgKminClassMode.is_usable_for(
+                X, self._fit_X, self.metric
+            ):
+                probabilities = self.predict_proba(X)
+                if self.outputs_2d_:
+                    return np.stack(
+                        [
+                            self.classes_[idx][np.argmax(probas, axis=1)]
+                            for idx, probas in enumerate(probabilities)
+                        ],
+                        axis=1,
+                    )
+                return self.classes_[np.argmax(probabilities, axis=1)]
+            # In that case, we do not need the distances to perform
+            # the weighting so we do not compute them.
+            neigh_ind = self.kneighbors(X, return_distance=False)
+            neigh_dist = None
+        else:
+            neigh_dist, neigh_ind = self.kneighbors(X)
 
-        neigh_dist, neigh_ind = self.kneighbors(X)
         classes_ = self.classes_
         _y = self._y
         if not self.outputs_2d_:
@@ -201,13 +283,19 @@ def predict(self, X):
             classes_ = [self.classes_]
 
         n_outputs = len(classes_)
-        n_queries = _num_samples(X)
+        n_queries = _num_samples(self._fit_X if X is None else X)
         weights = _get_weights(neigh_dist, self.weights)
+        if weights is not None and _all_with_any_reduction_axis_1(weights, value=0):
+            raise ValueError(
+                "All neighbors of some sample is getting zero weights. "
+                "Please modify 'weights' to avoid this case if you are "
+                "using a user-defined function."
+            )
 
         y_pred = np.empty((n_queries, n_outputs), dtype=classes_[0].dtype)
         for k, classes_k in enumerate(classes_):
             if weights is None:
-                mode, _ = stats.mode(_y[neigh_ind, k], axis=1)
+                mode, _ = _mode(_y[neigh_ind, k], axis=1)
             else:
                 mode, _ = weighted_mode(_y[neigh_ind, k], weights, axis=1)
 
@@ -224,20 +312,66 @@ def predict_proba(self, X):
 
         Parameters
         ----------
-        X : array-like of shape (n_queries, n_features), \
-                or (n_queries, n_indexed) if metric == 'precomputed'
-            Test samples.
+        X : {array-like, sparse matrix} of shape (n_queries, n_features), \
+                or (n_queries, n_indexed) if metric == 'precomputed', or None
+            Test samples. If `None`, predictions for all indexed points are
+            returned; in this case, points are not considered their own
+            neighbors.
 
         Returns
         -------
-        p : ndarray of shape (n_queries, n_classes), or a list of n_outputs
-            of such arrays if n_outputs > 1.
+        p : ndarray of shape (n_queries, n_classes), or a list of n_outputs \
+                of such arrays if n_outputs > 1.
             The class probabilities of the input samples. Classes are ordered
             by lexicographic order.
         """
-        X = self._validate_data(X, accept_sparse='csr', reset=False)
-
-        neigh_dist, neigh_ind = self.kneighbors(X)
+        check_is_fitted(self, "_fit_method")
+        if self.weights == "uniform":
+            # TODO: systematize this mapping of metric for
+            # PairwiseDistancesReductions.
+            metric, metric_kwargs = _adjusted_metric(
+                metric=self.metric, metric_kwargs=self.metric_params, p=self.p
+            )
+            if (
+                self._fit_method == "brute"
+                and ArgKminClassMode.is_usable_for(X, self._fit_X, metric)
+                # TODO: Implement efficient multi-output solution
+                and not self.outputs_2d_
+            ):
+                if self.metric == "precomputed":
+                    X = _check_precomputed(X)
+                else:
+                    X = validate_data(
+                        self, X, accept_sparse="csr", reset=False, order="C"
+                    )
+
+                probabilities = ArgKminClassMode.compute(
+                    X,
+                    self._fit_X,
+                    k=self.n_neighbors,
+                    weights=self.weights,
+                    Y_labels=self._y,
+                    unique_Y_labels=self.classes_,
+                    metric=metric,
+                    metric_kwargs=metric_kwargs,
+                    # `strategy="parallel_on_X"` has in practice be shown
+                    # to be more efficient than `strategy="parallel_on_Y``
+                    # on many combination of datasets.
+                    # Hence, we choose to enforce it here.
+                    # For more information, see:
+                    # https://github.com/scikit-learn/scikit-learn/pull/24076#issuecomment-1445258342
+                    # TODO: adapt the heuristic for `strategy="auto"` for
+                    # `ArgKminClassMode` and use `strategy="auto"`.
+                    strategy="parallel_on_X",
+                )
+                return probabilities
+
+            # In that case, we do not need the distances to perform
+            # the weighting so we do not compute them.
+            neigh_ind = self.kneighbors(X, return_distance=False)
+            neigh_dist = None
+        else:
+            neigh_dist, neigh_ind = self.kneighbors(X)
 
         classes_ = self.classes_
         _y = self._y
@@ -245,13 +379,19 @@ def predict_proba(self, X):
             _y = self._y.reshape((-1, 1))
             classes_ = [self.classes_]
 
-        n_queries = _num_samples(X)
+        n_queries = _num_samples(self._fit_X if X is None else X)
 
         weights = _get_weights(neigh_dist, self.weights)
         if weights is None:
             weights = np.ones_like(neigh_ind)
-
-        all_rows = np.arange(X.shape[0])
+        elif _all_with_any_reduction_axis_1(weights, value=0):
+            raise ValueError(
+                "All neighbors of some sample is getting zero weights. "
+                "Please modify 'weights' to avoid this case if you are "
+                "using a user-defined function."
+            )
+
+        all_rows = np.arange(n_queries)
         probabilities = []
         for k, classes_k in enumerate(classes_):
             pred_labels = _y[:, k][neigh_ind]
@@ -263,7 +403,6 @@ def predict_proba(self, X):
 
             # normalize 'votes' into real [0,1] probabilities
             normalizer = proba_k.sum(axis=1)[:, np.newaxis]
-            normalizer[normalizer == 0.0] = 1.0
             proba_k /= normalizer
 
             probabilities.append(proba_k)
@@ -273,11 +412,48 @@ def predict_proba(self, X):
 
         return probabilities
 
+    # This function is defined here only to modify the parent docstring
+    # and add information about X=None
+    def score(self, X, y, sample_weight=None):
+        """
+        Return the mean accuracy on the given test data and labels.
+
+        In multi-label classification, this is the subset accuracy
+        which is a harsh metric since you require for each sample that
+        each label set be correctly predicted.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features), or None
+            Test samples. If `None`, predictions for all indexed points are
+            used; in this case, points are not considered their own
+            neighbors. This means that `knn.fit(X, y).score(None, y)`
+            implicitly performs a leave-one-out cross-validation procedure
+            and is equivalent to `cross_val_score(knn, X, y, cv=LeaveOneOut())`
+            but typically much faster.
+
+        y : array-like of shape (n_samples,) or (n_samples, n_outputs)
+            True labels for `X`.
+
+        sample_weight : array-like of shape (n_samples,), default=None
+            Sample weights.
+
+        Returns
+        -------
+        score : float
+            Mean accuracy of ``self.predict(X)`` w.r.t. `y`.
+        """
+        return super().score(X, y, sample_weight)
 
-class RadiusNeighborsClassifier(RadiusNeighborsMixin,
-                                ClassifierMixin,
-                                NeighborsBase):
-    """Classifier implementing a vote among neighbors within a given radius
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.classifier_tags.multi_label = True
+        tags.input_tags.pairwise = self.metric == "precomputed"
+        return tags
+
+
+class RadiusNeighborsClassifier(RadiusNeighborsMixin, ClassifierMixin, NeighborsBase):
+    """Classifier implementing a vote among neighbors within a given radius.
 
     Read more in the :ref:`User Guide `.
 
@@ -287,8 +463,8 @@ class RadiusNeighborsClassifier(RadiusNeighborsMixin,
         Range of parameter space to use by default for :meth:`radius_neighbors`
         queries.
 
-    weights : {'uniform', 'distance'} or callable, default='uniform'
-        weight function used in prediction.  Possible values:
+    weights : {'uniform', 'distance'}, callable or None, default='uniform'
+        Weight function used in prediction.  Possible values:
 
         - 'uniform' : uniform weights.  All points in each neighborhood
           are weighted equally.
@@ -319,28 +495,42 @@ class RadiusNeighborsClassifier(RadiusNeighborsMixin,
         required to store the tree.  The optimal value depends on the
         nature of the problem.
 
-    p : int, default=2
+    p : float, default=2
         Power parameter for the Minkowski metric. When p = 1, this is
         equivalent to using manhattan_distance (l1), and euclidean_distance
         (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used.
+        This parameter is expected to be positive.
 
     metric : str or callable, default='minkowski'
-        the distance metric to use for the tree.  The default metric is
-        minkowski, and with p=2 is equivalent to the standard Euclidean
-        metric. See the documentation of :class:`DistanceMetric` for a
-        list of available metrics.
+        Metric to use for distance computation. Default is "minkowski", which
+        results in the standard Euclidean distance when p = 2. See the
+        documentation of `scipy.spatial.distance
+        `_ and
+        the metrics listed in
+        :class:`~sklearn.metrics.pairwise.distance_metrics` for valid metric
+        values.
+
         If metric is "precomputed", X is assumed to be a distance matrix and
-        must be square during fit. X may be a :term:`sparse graph`,
-        in which case only "nonzero" elements may be considered neighbors.
+        must be square during fit. X may be a :term:`sparse graph`, in which
+        case only "nonzero" elements may be considered neighbors.
+
+        If metric is a callable function, it takes two arrays representing 1D
+        vectors as inputs and must return one value indicating the distance
+        between those vectors. This works for Scipy's metrics, but is less
+        efficient than passing the metric name as a string.
 
     outlier_label : {manual label, 'most_frequent'}, default=None
-        label for outlier samples (samples with no neighbors in given radius).
+        Label for outlier samples (samples with no neighbors in given radius).
 
         - manual label: str or int label (should be the same type as y)
           or list of manual labels if multi-output is used.
         - 'most_frequent' : assign the most frequent label of y to outliers.
         - None : when any outlier is detected, ValueError will be raised.
 
+        The outlier label should be selected from among the unique 'Y' labels.
+        If it is specified with a different value a warning will be raised and
+        all class probabilities of outliers will be assigned to be 0.
+
     metric_params : dict, default=None
         Additional keyword arguments for the metric function.
 
@@ -366,6 +556,17 @@ class RadiusNeighborsClassifier(RadiusNeighborsMixin,
         `p` parameter value if the `effective_metric_` attribute is set to
         'minkowski'.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     n_samples_fit_ : int
         Number of samples in the fitted data.
 
@@ -377,6 +578,23 @@ class RadiusNeighborsClassifier(RadiusNeighborsMixin,
         False when `y`'s shape is (n_samples, ) or (n_samples, 1) during fit
         otherwise True.
 
+    See Also
+    --------
+    KNeighborsClassifier : Classifier implementing the k-nearest neighbors
+        vote.
+    RadiusNeighborsRegressor : Regression based on neighbors within a
+        fixed radius.
+    KNeighborsRegressor : Regression based on k-nearest neighbors.
+    NearestNeighbors : Unsupervised learner for implementing neighbor
+        searches.
+
+    Notes
+    -----
+    See :ref:`Nearest Neighbors ` in the online documentation
+    for a discussion of the choice of ``algorithm`` and ``leaf_size``.
+
+    https://en.wikipedia.org/wiki/K-nearest_neighbor_algorithm
+
     Examples
     --------
     >>> X = [[0], [1], [2], [3]]
@@ -389,36 +607,44 @@ class RadiusNeighborsClassifier(RadiusNeighborsMixin,
     [0]
     >>> print(neigh.predict_proba([[1.0]]))
     [[0.66666667 0.33333333]]
-
-    See Also
-    --------
-    KNeighborsClassifier
-    RadiusNeighborsRegressor
-    KNeighborsRegressor
-    NearestNeighbors
-
-    Notes
-    -----
-    See :ref:`Nearest Neighbors ` in the online documentation
-    for a discussion of the choice of ``algorithm`` and ``leaf_size``.
-
-    https://en.wikipedia.org/wiki/K-nearest_neighbor_algorithm
     """
 
-    @_deprecate_positional_args
-    def __init__(self, radius=1.0, *, weights='uniform',
-                 algorithm='auto', leaf_size=30, p=2, metric='minkowski',
-                 outlier_label=None, metric_params=None, n_jobs=None,
-                 **kwargs):
+    _parameter_constraints: dict = {
+        **NeighborsBase._parameter_constraints,
+        "weights": [StrOptions({"uniform", "distance"}), callable, None],
+        "outlier_label": [Integral, str, "array-like", None],
+    }
+    _parameter_constraints.pop("n_neighbors")
+
+    def __init__(
+        self,
+        radius=1.0,
+        *,
+        weights="uniform",
+        algorithm="auto",
+        leaf_size=30,
+        p=2,
+        metric="minkowski",
+        outlier_label=None,
+        metric_params=None,
+        n_jobs=None,
+    ):
         super().__init__(
-              radius=radius,
-              algorithm=algorithm,
-              leaf_size=leaf_size,
-              metric=metric, p=p, metric_params=metric_params,
-              n_jobs=n_jobs, **kwargs)
-        self.weights = _check_weights(weights)
+            radius=radius,
+            algorithm=algorithm,
+            leaf_size=leaf_size,
+            metric=metric,
+            p=p,
+            metric_params=metric_params,
+            n_jobs=n_jobs,
+        )
+        self.weights = weights
         self.outlier_label = outlier_label
 
+    @_fit_context(
+        # RadiusNeighborsClassifier.metric is not validated yet
+        prefer_skip_nested_validation=False
+    )
     def fit(self, X, y):
         """Fit the radius neighbors classifier from the training dataset.
 
@@ -448,7 +674,7 @@ def fit(self, X, y):
         if self.outlier_label is None:
             outlier_label_ = None
 
-        elif self.outlier_label == 'most_frequent':
+        elif self.outlier_label == "most_frequent":
             outlier_label_ = []
             # iterate over multi-output, get the most frequent label for each
             # output.
@@ -457,29 +683,34 @@ def fit(self, X, y):
                 outlier_label_.append(classes_k[label_count.argmax()])
 
         else:
-            if (_is_arraylike(self.outlier_label) and
-               not isinstance(self.outlier_label, str)):
+            if _is_arraylike(self.outlier_label) and not isinstance(
+                self.outlier_label, str
+            ):
                 if len(self.outlier_label) != len(classes_):
-                    raise ValueError("The length of outlier_label: {} is "
-                                     "inconsistent with the output "
-                                     "length: {}".format(self.outlier_label,
-                                                         len(classes_)))
+                    raise ValueError(
+                        "The length of outlier_label: {} is "
+                        "inconsistent with the output "
+                        "length: {}".format(self.outlier_label, len(classes_))
+                    )
                 outlier_label_ = self.outlier_label
             else:
                 outlier_label_ = [self.outlier_label] * len(classes_)
 
             for classes, label in zip(classes_, outlier_label_):
-                if (_is_arraylike(label) and
-                   not isinstance(label, str)):
-                    # ensure the outlier lable for each output is a scalar.
-                    raise TypeError("The outlier_label of classes {} is "
-                                    "supposed to be a scalar, got "
-                                    "{}.".format(classes, label))
+                if _is_arraylike(label) and not isinstance(label, str):
+                    # ensure the outlier label for each output is a scalar.
+                    raise TypeError(
+                        "The outlier_label of classes {} is "
+                        "supposed to be a scalar, got "
+                        "{}.".format(classes, label)
+                    )
                 if np.append(classes, label).dtype != classes.dtype:
                     # ensure the dtype of outlier label is consistent with y.
-                    raise TypeError("The dtype of outlier_label {} is "
-                                    "inconsistent with classes {} in "
-                                    "y.".format(label, classes))
+                    raise TypeError(
+                        "The dtype of outlier_label {} is "
+                        "inconsistent with classes {} in "
+                        "y.".format(label, classes)
+                    )
 
         self.outlier_label_ = outlier_label_
 
@@ -490,9 +721,11 @@ def predict(self, X):
 
         Parameters
         ----------
-        X : array-like of shape (n_queries, n_features), \
-                or (n_queries, n_indexed) if metric == 'precomputed'
-            Test samples.
+        X : {array-like, sparse matrix} of shape (n_queries, n_features), \
+                or (n_queries, n_indexed) if metric == 'precomputed', or None
+            Test samples. If `None`, predictions for all indexed points are
+            returned; in this case, points are not considered their own
+            neighbors.
 
         Returns
         -------
@@ -532,20 +765,51 @@ def predict_proba(self, X):
 
         Parameters
         ----------
-        X : array-like of shape (n_queries, n_features), \
-                or (n_queries, n_indexed) if metric == 'precomputed'
-            Test samples.
+        X : {array-like, sparse matrix} of shape (n_queries, n_features), \
+                or (n_queries, n_indexed) if metric == 'precomputed', or None
+            Test samples. If `None`, predictions for all indexed points are
+            returned; in this case, points are not considered their own
+            neighbors.
 
         Returns
         -------
-        p : ndarray of shape (n_queries, n_classes), or a list of n_outputs
-            of such arrays if n_outputs > 1.
+        p : ndarray of shape (n_queries, n_classes), or a list of \
+                n_outputs of such arrays if n_outputs > 1.
             The class probabilities of the input samples. Classes are ordered
             by lexicographic order.
         """
-
-        X = self._validate_data(X, accept_sparse='csr', reset=False)
-        n_queries = _num_samples(X)
+        check_is_fitted(self, "_fit_method")
+        n_queries = _num_samples(self._fit_X if X is None else X)
+
+        metric, metric_kwargs = _adjusted_metric(
+            metric=self.metric, metric_kwargs=self.metric_params, p=self.p
+        )
+
+        if (
+            self.weights == "uniform"
+            and self._fit_method == "brute"
+            and not self.outputs_2d_
+            and RadiusNeighborsClassMode.is_usable_for(X, self._fit_X, metric)
+        ):
+            probabilities = RadiusNeighborsClassMode.compute(
+                X=X,
+                Y=self._fit_X,
+                radius=self.radius,
+                weights=self.weights,
+                Y_labels=self._y,
+                unique_Y_labels=self.classes_,
+                outlier_label=self.outlier_label,
+                metric=metric,
+                metric_kwargs=metric_kwargs,
+                strategy="parallel_on_X",
+                # `strategy="parallel_on_X"` has in practice be shown
+                # to be more efficient than `strategy="parallel_on_Y``
+                # on many combination of datasets.
+                # Hence, we choose to enforce it here.
+                # For more information, see:
+                # https://github.com/scikit-learn/scikit-learn/pull/26828/files#r1282398471
+            )
+            return probabilities
 
         neigh_dist, neigh_ind = self.radius_neighbors(X)
         outlier_mask = np.zeros(n_queries, dtype=bool)
@@ -560,11 +824,12 @@ def predict_proba(self, X):
             classes_ = [self.classes_]
 
         if self.outlier_label_ is None and outliers.size > 0:
-            raise ValueError('No neighbors found for test samples %r, '
-                             'you can try using larger radius, '
-                             'giving a label for outliers, '
-                             'or considering removing them from your dataset.'
-                             % outliers)
+            raise ValueError(
+                "No neighbors found for test samples %r, "
+                "you can try using larger radius, "
+                "giving a label for outliers, "
+                "or considering removing them from your dataset." % outliers
+            )
 
         weights = _get_weights(neigh_dist, self.weights)
         if weights is not None:
@@ -582,13 +847,12 @@ def predict_proba(self, X):
             # samples have different size of neighbors within the same radius
             if weights is None:
                 for i, idx in enumerate(pred_labels[inliers]):
-                    proba_inl[i, :] = np.bincount(idx,
-                                                  minlength=classes_k.size)
+                    proba_inl[i, :] = np.bincount(idx, minlength=classes_k.size)
             else:
                 for i, idx in enumerate(pred_labels[inliers]):
-                    proba_inl[i, :] = np.bincount(idx,
-                                                  weights[i],
-                                                  minlength=classes_k.size)
+                    proba_inl[i, :] = np.bincount(
+                        idx, weights[i], minlength=classes_k.size
+                    )
             proba_k[inliers, :] = proba_inl
 
             if outliers.size > 0:
@@ -597,10 +861,12 @@ def predict_proba(self, X):
                 if label_index.size == 1:
                     proba_k[outliers, label_index[0]] = 1.0
                 else:
-                    warnings.warn('Outlier label {} is not in training '
-                                  'classes. All class probabilities of '
-                                  'outliers will be assigned with 0.'
-                                  ''.format(self.outlier_label_[k]))
+                    warnings.warn(
+                        "Outlier label {} is not in training "
+                        "classes. All class probabilities of "
+                        "outliers will be assigned with 0."
+                        "".format(self.outlier_label_[k])
+                    )
 
             # normalize 'votes' into real [0,1] probabilities
             normalizer = proba_k.sum(axis=1)[:, np.newaxis]
@@ -613,3 +879,41 @@ def predict_proba(self, X):
             probabilities = probabilities[0]
 
         return probabilities
+
+    # This function is defined here only to modify the parent docstring
+    # and add information about X=None
+    def score(self, X, y, sample_weight=None):
+        """
+        Return the mean accuracy on the given test data and labels.
+
+        In multi-label classification, this is the subset accuracy
+        which is a harsh metric since you require for each sample that
+        each label set be correctly predicted.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features), or None
+            Test samples. If `None`, predictions for all indexed points are
+            used; in this case, points are not considered their own
+            neighbors. This means that `knn.fit(X, y).score(None, y)`
+            implicitly performs a leave-one-out cross-validation procedure
+            and is equivalent to `cross_val_score(knn, X, y, cv=LeaveOneOut())`
+            but typically much faster.
+
+        y : array-like of shape (n_samples,) or (n_samples, n_outputs)
+            True labels for `X`.
+
+        sample_weight : array-like of shape (n_samples,), default=None
+            Sample weights.
+
+        Returns
+        -------
+        score : float
+            Mean accuracy of ``self.predict(X)`` w.r.t. `y`.
+        """
+        return super().score(X, y, sample_weight)
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.classifier_tags.multi_label = True
+        return tags
diff --git a/sklearn/neighbors/_dist_metrics.pxd b/sklearn/neighbors/_dist_metrics.pxd
deleted file mode 100644
index 89c63cc46905f..0000000000000
--- a/sklearn/neighbors/_dist_metrics.pxd
+++ /dev/null
@@ -1,77 +0,0 @@
-#!python
-#cython: boundscheck=False
-#cython: wraparound=False
-#cython: cdivision=True
-
-cimport cython
-cimport numpy as np
-from libc.math cimport fabs, sqrt, exp, cos, pow
-
-from ._typedefs cimport DTYPE_t, ITYPE_t, DITYPE_t
-from ._typedefs import DTYPE, ITYPE
-
-######################################################################
-# Inline distance functions
-#
-#  We use these for the default (euclidean) case so that they can be
-#  inlined.  This leads to faster computation for the most common case
-cdef inline DTYPE_t euclidean_dist(DTYPE_t* x1, DTYPE_t* x2,
-                                   ITYPE_t size) nogil except -1:
-    cdef DTYPE_t tmp, d=0
-    cdef np.intp_t j
-    for j in range(size):
-        tmp = x1[j] - x2[j]
-        d += tmp * tmp
-    return sqrt(d)
-
-
-cdef inline DTYPE_t euclidean_rdist(DTYPE_t* x1, DTYPE_t* x2,
-                                    ITYPE_t size) nogil except -1:
-    cdef DTYPE_t tmp, d=0
-    cdef np.intp_t j
-    for j in range(size):
-        tmp = x1[j] - x2[j]
-        d += tmp * tmp
-    return d
-
-
-cdef inline DTYPE_t euclidean_dist_to_rdist(DTYPE_t dist) nogil except -1:
-    return dist * dist
-
-
-cdef inline DTYPE_t euclidean_rdist_to_dist(DTYPE_t dist) nogil except -1:
-    return sqrt(dist)
-
-
-######################################################################
-# DistanceMetric base class
-cdef class DistanceMetric:
-    # The following attributes are required for a few of the subclasses.
-    # we must define them here so that cython's limited polymorphism will work.
-    # Because we don't expect to instantiate a lot of these objects, the
-    # extra memory overhead of this setup should not be an issue.
-    cdef DTYPE_t p
-    #cdef DTYPE_t[::1] vec
-    #cdef DTYPE_t[:, ::1] mat
-    cdef np.ndarray vec
-    cdef np.ndarray mat
-    cdef DTYPE_t* vec_ptr
-    cdef DTYPE_t* mat_ptr
-    cdef ITYPE_t size
-    cdef object func
-    cdef object kwargs
-
-    cdef DTYPE_t dist(self, DTYPE_t* x1, DTYPE_t* x2,
-                      ITYPE_t size) nogil except -1
-
-    cdef DTYPE_t rdist(self, DTYPE_t* x1, DTYPE_t* x2,
-                       ITYPE_t size) nogil except -1
-
-    cdef int pdist(self, DTYPE_t[:, ::1] X, DTYPE_t[:, ::1] D) except -1
-
-    cdef int cdist(self, DTYPE_t[:, ::1] X, DTYPE_t[:, ::1] Y,
-                   DTYPE_t[:, ::1] D) except -1
-
-    cdef DTYPE_t _rdist_to_dist(self, DTYPE_t rdist) nogil except -1
-
-    cdef DTYPE_t _dist_to_rdist(self, DTYPE_t dist) nogil except -1
diff --git a/sklearn/neighbors/_dist_metrics.pyx b/sklearn/neighbors/_dist_metrics.pyx
deleted file mode 100755
index 8bee948eeaeba..0000000000000
--- a/sklearn/neighbors/_dist_metrics.pyx
+++ /dev/null
@@ -1,1149 +0,0 @@
-#!python
-#cython: boundscheck=False
-#cython: wraparound=False
-#cython: cdivision=True
-
-# By Jake Vanderplas (2013) 
-# written for the scikit-learn project
-# License: BSD
-
-import numpy as np
-cimport numpy as np
-np.import_array()  # required in order to use C-API
-
-
-######################################################################
-# Numpy 1.3-1.4 compatibility utilities
-cdef DTYPE_t[:, ::1] get_memview_DTYPE_2D(
-                               np.ndarray[DTYPE_t, ndim=2, mode='c'] X):
-    return  ( X.data)
-
-
-cdef DTYPE_t* get_vec_ptr(np.ndarray[DTYPE_t, ndim=1, mode='c'] vec):
-    return &vec[0]
-
-
-cdef DTYPE_t* get_mat_ptr(np.ndarray[DTYPE_t, ndim=2, mode='c'] mat):
-    return &mat[0, 0]
-######################################################################
-
-
-# First, define a function to get an ndarray from a memory bufffer
-cdef extern from "arrayobject.h":
-    object PyArray_SimpleNewFromData(int nd, np.npy_intp* dims,
-                                     int typenum, void* data)
-
-
-cdef inline np.ndarray _buffer_to_ndarray(DTYPE_t* x, np.npy_intp n):
-    # Wrap a memory buffer with an ndarray. Warning: this is not robust.
-    # In particular, if x is deallocated before the returned array goes
-    # out of scope, this could cause memory errors.  Since there is not
-    # a possibility of this for our use-case, this should be safe.
-
-    # Note: this Segfaults unless np.import_array() is called above
-    return PyArray_SimpleNewFromData(1, &n, DTYPECODE, x)
-
-
-# some handy constants
-from libc.math cimport fabs, sqrt, exp, pow, cos, sin, asin
-cdef DTYPE_t INF = np.inf
-
-from ._typedefs cimport DTYPE_t, ITYPE_t, DITYPE_t, DTYPECODE
-from ._typedefs import DTYPE, ITYPE
-
-
-######################################################################
-# newObj function
-#  this is a helper function for pickling
-def newObj(obj):
-    return obj.__new__(obj)
-
-
-######################################################################
-# metric mappings
-#  These map from metric id strings to class names
-METRIC_MAPPING = {'euclidean': EuclideanDistance,
-                  'l2': EuclideanDistance,
-                  'minkowski': MinkowskiDistance,
-                  'p': MinkowskiDistance,
-                  'manhattan': ManhattanDistance,
-                  'cityblock': ManhattanDistance,
-                  'l1': ManhattanDistance,
-                  'chebyshev': ChebyshevDistance,
-                  'infinity': ChebyshevDistance,
-                  'seuclidean': SEuclideanDistance,
-                  'mahalanobis': MahalanobisDistance,
-                  'wminkowski': WMinkowskiDistance,
-                  'hamming': HammingDistance,
-                  'canberra': CanberraDistance,
-                  'braycurtis': BrayCurtisDistance,
-                  'matching': MatchingDistance,
-                  'jaccard': JaccardDistance,
-                  'dice': DiceDistance,
-                  'kulsinski': KulsinskiDistance,
-                  'rogerstanimoto': RogersTanimotoDistance,
-                  'russellrao': RussellRaoDistance,
-                  'sokalmichener': SokalMichenerDistance,
-                  'sokalsneath': SokalSneathDistance,
-                  'haversine': HaversineDistance,
-                  'pyfunc': PyFuncDistance}
-
-
-def get_valid_metric_ids(L):
-    """Given an iterable of metric class names or class identifiers,
-    return a list of metric IDs which map to those classes.
-
-    Example:
-    >>> L = get_valid_metric_ids([EuclideanDistance, 'ManhattanDistance'])
-    >>> sorted(L)
-    ['cityblock', 'euclidean', 'l1', 'l2', 'manhattan']
-    """
-    return [key for (key, val) in METRIC_MAPPING.items()
-            if (val.__name__ in L) or (val in L)]
-
-
-######################################################################
-# Distance Metric Classes
-cdef class DistanceMetric:
-    """DistanceMetric class
-
-    This class provides a uniform interface to fast distance metric
-    functions.  The various metrics can be accessed via the :meth:`get_metric`
-    class method and the metric string identifier (see below).
-
-    Examples
-    --------
-    >>> from sklearn.neighbors import DistanceMetric
-    >>> dist = DistanceMetric.get_metric('euclidean')
-    >>> X = [[0, 1, 2],
-             [3, 4, 5]]
-    >>> dist.pairwise(X)
-    array([[ 0.        ,  5.19615242],
-           [ 5.19615242,  0.        ]])
-
-    Available Metrics
-
-    The following lists the string metric identifiers and the associated
-    distance metric classes:
-
-    **Metrics intended for real-valued vector spaces:**
-
-    ==============  ====================  ========  ===============================
-    identifier      class name            args      distance function
-    --------------  --------------------  --------  -------------------------------
-    "euclidean"     EuclideanDistance     -         ``sqrt(sum((x - y)^2))``
-    "manhattan"     ManhattanDistance     -         ``sum(|x - y|)``
-    "chebyshev"     ChebyshevDistance     -         ``max(|x - y|)``
-    "minkowski"     MinkowskiDistance     p         ``sum(|x - y|^p)^(1/p)``
-    "wminkowski"    WMinkowskiDistance    p, w      ``sum(|w * (x - y)|^p)^(1/p)``
-    "seuclidean"    SEuclideanDistance    V         ``sqrt(sum((x - y)^2 / V))``
-    "mahalanobis"   MahalanobisDistance   V or VI   ``sqrt((x - y)' V^-1 (x - y))``
-    ==============  ====================  ========  ===============================
-
-    **Metrics intended for two-dimensional vector spaces:**  Note that the haversine
-    distance metric requires data in the form of [latitude, longitude] and both
-    inputs and outputs are in units of radians.
-
-    ============  ==================  ===============================================================
-    identifier    class name          distance function
-    ------------  ------------------  ---------------------------------------------------------------
-    "haversine"   HaversineDistance   ``2 arcsin(sqrt(sin^2(0.5*dx) + cos(x1)cos(x2)sin^2(0.5*dy)))``
-    ============  ==================  ===============================================================
-
-
-    **Metrics intended for integer-valued vector spaces:**  Though intended
-    for integer-valued vectors, these are also valid metrics in the case of
-    real-valued vectors.
-
-    =============  ====================  ========================================
-    identifier     class name            distance function
-    -------------  --------------------  ----------------------------------------
-    "hamming"      HammingDistance       ``N_unequal(x, y) / N_tot``
-    "canberra"     CanberraDistance      ``sum(|x - y| / (|x| + |y|))``
-    "braycurtis"   BrayCurtisDistance    ``sum(|x - y|) / (sum(|x|) + sum(|y|))``
-    =============  ====================  ========================================
-
-    **Metrics intended for boolean-valued vector spaces:**  Any nonzero entry
-    is evaluated to "True".  In the listings below, the following
-    abbreviations are used:
-
-     - N  : number of dimensions
-     - NTT : number of dims in which both values are True
-     - NTF : number of dims in which the first value is True, second is False
-     - NFT : number of dims in which the first value is False, second is True
-     - NFF : number of dims in which both values are False
-     - NNEQ : number of non-equal dimensions, NNEQ = NTF + NFT
-     - NNZ : number of nonzero dimensions, NNZ = NTF + NFT + NTT
-
-    =================  =======================  ===============================
-    identifier         class name               distance function
-    -----------------  -----------------------  -------------------------------
-    "jaccard"          JaccardDistance          NNEQ / NNZ
-    "matching"         MatchingDistance         NNEQ / N
-    "dice"             DiceDistance             NNEQ / (NTT + NNZ)
-    "kulsinski"        KulsinskiDistance        (NNEQ + N - NTT) / (NNEQ + N)
-    "rogerstanimoto"   RogersTanimotoDistance   2 * NNEQ / (N + NNEQ)
-    "russellrao"       RussellRaoDistance       NNZ / N
-    "sokalmichener"    SokalMichenerDistance    2 * NNEQ / (N + NNEQ)
-    "sokalsneath"      SokalSneathDistance      NNEQ / (NNEQ + 0.5 * NTT)
-    =================  =======================  ===============================
-
-    **User-defined distance:**
-
-    ===========    ===============    =======
-    identifier     class name         args
-    -----------    ---------------    -------
-    "pyfunc"       PyFuncDistance     func
-    ===========    ===============    =======
-
-    Here ``func`` is a function which takes two one-dimensional numpy
-    arrays, and returns a distance.  Note that in order to be used within
-    the BallTree, the distance must be a true metric:
-    i.e. it must satisfy the following properties
-
-    1) Non-negativity: d(x, y) >= 0
-    2) Identity: d(x, y) = 0 if and only if x == y
-    3) Symmetry: d(x, y) = d(y, x)
-    4) Triangle Inequality: d(x, y) + d(y, z) >= d(x, z)
-
-    Because of the Python object overhead involved in calling the python
-    function, this will be fairly slow, but it will have the same
-    scaling as other distances.
-    """
-    def __cinit__(self):
-        self.p = 2
-        self.vec = np.zeros(1, dtype=DTYPE, order='c')
-        self.mat = np.zeros((1, 1), dtype=DTYPE, order='c')
-        self.vec_ptr = get_vec_ptr(self.vec)
-        self.mat_ptr = get_mat_ptr(self.mat)
-        self.size = 1
-
-    def __reduce__(self):
-        """
-        reduce method used for pickling
-        """
-        return (newObj, (self.__class__,), self.__getstate__())
-
-    def __getstate__(self):
-        """
-        get state for pickling
-        """
-        if self.__class__.__name__ == "PyFuncDistance":
-            return (float(self.p), self.vec, self.mat, self.func, self.kwargs)
-        return (float(self.p), self.vec, self.mat)
-
-    def __setstate__(self, state):
-        """
-        set state for pickling
-        """
-        self.p = state[0]
-        self.vec = state[1]
-        self.mat = state[2]
-        if self.__class__.__name__ == "PyFuncDistance":
-            self.func = state[3]
-            self.kwargs = state[4]
-        self.vec_ptr = get_vec_ptr(self.vec)
-        self.mat_ptr = get_mat_ptr(self.mat)
-        self.size = self.vec.shape[0]
-
-    @classmethod
-    def get_metric(cls, metric, **kwargs):
-        """Get the given distance metric from the string identifier.
-
-        See the docstring of DistanceMetric for a list of available metrics.
-
-        Parameters
-        ----------
-        metric : string or class name
-            The distance metric to use
-        **kwargs
-            additional arguments will be passed to the requested metric
-        """
-        if isinstance(metric, DistanceMetric):
-            return metric
-
-        if callable(metric):
-            return PyFuncDistance(metric, **kwargs)
-
-        # Map the metric string ID to the metric class
-        if isinstance(metric, type) and issubclass(metric, DistanceMetric):
-            pass
-        else:
-            try:
-                metric = METRIC_MAPPING[metric]
-            except:
-                raise ValueError("Unrecognized metric '%s'" % metric)
-
-        # In Minkowski special cases, return more efficient methods
-        if metric is MinkowskiDistance:
-            p = kwargs.pop('p', 2)
-            if p == 1:
-                return ManhattanDistance(**kwargs)
-            elif p == 2:
-                return EuclideanDistance(**kwargs)
-            elif np.isinf(p):
-                return ChebyshevDistance(**kwargs)
-            else:
-                return MinkowskiDistance(p, **kwargs)
-        else:
-            return metric(**kwargs)
-
-    def __init__(self):
-        if self.__class__ is DistanceMetric:
-            raise NotImplementedError("DistanceMetric is an abstract class")
-
-    def _validate_data(self, X):
-        """Validate the input data.
-
-        This should be overridden in a base class if a specific input format
-        is required.
-        """
-        return
-
-    cdef DTYPE_t dist(self, DTYPE_t* x1, DTYPE_t* x2,
-                      ITYPE_t size) nogil except -1:
-        """Compute the distance between vectors x1 and x2
-
-        This should be overridden in a base class.
-        """
-        return -999
-
-    cdef DTYPE_t rdist(self, DTYPE_t* x1, DTYPE_t* x2,
-                       ITYPE_t size) nogil except -1:
-        """Compute the reduced distance between vectors x1 and x2.
-
-        This can optionally be overridden in a base class.
-
-        The reduced distance is any measure that yields the same rank as the
-        distance, but is more efficient to compute.  For example, for the
-        Euclidean metric, the reduced distance is the squared-euclidean
-        distance.
-        """
-        return self.dist(x1, x2, size)
-
-    cdef int pdist(self, DTYPE_t[:, ::1] X, DTYPE_t[:, ::1] D) except -1:
-        """compute the pairwise distances between points in X"""
-        cdef ITYPE_t i1, i2
-        for i1 in range(X.shape[0]):
-            for i2 in range(i1, X.shape[0]):
-                D[i1, i2] = self.dist(&X[i1, 0], &X[i2, 0], X.shape[1])
-                D[i2, i1] = D[i1, i2]
-        return 0
-
-    cdef int cdist(self, DTYPE_t[:, ::1] X, DTYPE_t[:, ::1] Y,
-                   DTYPE_t[:, ::1] D) except -1:
-        """compute the cross-pairwise distances between arrays X and Y"""
-        cdef ITYPE_t i1, i2
-        if X.shape[1] != Y.shape[1]:
-            raise ValueError('X and Y must have the same second dimension')
-        for i1 in range(X.shape[0]):
-            for i2 in range(Y.shape[0]):
-                D[i1, i2] = self.dist(&X[i1, 0], &Y[i2, 0], X.shape[1])
-        return 0
-
-    cdef DTYPE_t _rdist_to_dist(self, DTYPE_t rdist) nogil except -1:
-        """Convert the reduced distance to the distance"""
-        return rdist
-
-    cdef DTYPE_t _dist_to_rdist(self, DTYPE_t dist) nogil except -1:
-        """Convert the distance to the reduced distance"""
-        return dist
-
-    def rdist_to_dist(self, rdist):
-        """Convert the Reduced distance to the true distance.
-
-        The reduced distance, defined for some metrics, is a computationally
-        more efficient measure which preserves the rank of the true distance.
-        For example, in the Euclidean distance metric, the reduced distance
-        is the squared-euclidean distance.
-        """
-        return rdist
-
-    def dist_to_rdist(self, dist):
-        """Convert the true distance to the reduced distance.
-
-        The reduced distance, defined for some metrics, is a computationally
-        more efficient measure which preserves the rank of the true distance.
-        For example, in the Euclidean distance metric, the reduced distance
-        is the squared-euclidean distance.
-        """
-        return dist
-
-    def pairwise(self, X, Y=None):
-        """Compute the pairwise distances between X and Y
-
-        This is a convenience routine for the sake of testing.  For many
-        metrics, the utilities in scipy.spatial.distance.cdist and
-        scipy.spatial.distance.pdist will be faster.
-
-        Parameters
-        ----------
-        X : array-like
-            Array of shape (Nx, D), representing Nx points in D dimensions.
-        Y : array-like (optional)
-            Array of shape (Ny, D), representing Ny points in D dimensions.
-            If not specified, then Y=X.
-        Returns
-        -------
-        dist : ndarray
-            The shape (Nx, Ny) array of pairwise distances between points in
-            X and Y.
-        """
-        cdef np.ndarray[DTYPE_t, ndim=2, mode='c'] Xarr
-        cdef np.ndarray[DTYPE_t, ndim=2, mode='c'] Yarr
-        cdef np.ndarray[DTYPE_t, ndim=2, mode='c'] Darr
-
-        Xarr = np.asarray(X, dtype=DTYPE, order='C')
-        self._validate_data(Xarr)
-        if Y is None:
-            Darr = np.zeros((Xarr.shape[0], Xarr.shape[0]),
-                         dtype=DTYPE, order='C')
-            self.pdist(get_memview_DTYPE_2D(Xarr),
-                       get_memview_DTYPE_2D(Darr))
-        else:
-            Yarr = np.asarray(Y, dtype=DTYPE, order='C')
-            self._validate_data(Yarr)
-            Darr = np.zeros((Xarr.shape[0], Yarr.shape[0]),
-                         dtype=DTYPE, order='C')
-            self.cdist(get_memview_DTYPE_2D(Xarr),
-                       get_memview_DTYPE_2D(Yarr),
-                       get_memview_DTYPE_2D(Darr))
-        return Darr
-
-
-#------------------------------------------------------------
-# Euclidean Distance
-#  d = sqrt(sum(x_i^2 - y_i^2))
-cdef class EuclideanDistance(DistanceMetric):
-    r"""Euclidean Distance metric
-
-    .. math::
-       D(x, y) = \sqrt{ \sum_i (x_i - y_i) ^ 2 }
-    """
-    def __init__(self):
-        self.p = 2
-
-    cdef inline DTYPE_t dist(self, DTYPE_t* x1, DTYPE_t* x2,
-                             ITYPE_t size) nogil except -1:
-        return euclidean_dist(x1, x2, size)
-
-    cdef inline DTYPE_t rdist(self, DTYPE_t* x1, DTYPE_t* x2,
-                              ITYPE_t size) nogil except -1:
-        return euclidean_rdist(x1, x2, size)
-
-    cdef inline DTYPE_t _rdist_to_dist(self, DTYPE_t rdist) nogil except -1:
-        return sqrt(rdist)
-
-    cdef inline DTYPE_t _dist_to_rdist(self, DTYPE_t dist) nogil except -1:
-        return dist * dist
-
-    def rdist_to_dist(self, rdist):
-        return np.sqrt(rdist)
-
-    def dist_to_rdist(self, dist):
-        return dist ** 2
-
-
-#------------------------------------------------------------
-# SEuclidean Distance
-#  d = sqrt(sum((x_i - y_i2)^2 / v_i))
-cdef class SEuclideanDistance(DistanceMetric):
-    r"""Standardized Euclidean Distance metric
-
-    .. math::
-       D(x, y) = \sqrt{ \sum_i \frac{ (x_i - y_i) ^ 2}{V_i} }
-    """
-    def __init__(self, V):
-        self.vec = np.asarray(V, dtype=DTYPE)
-        self.vec_ptr = get_vec_ptr(self.vec)
-        self.size = self.vec.shape[0]
-        self.p = 2
-
-    def _validate_data(self, X):
-        if X.shape[1] != self.size:
-            raise ValueError('SEuclidean dist: size of V does not match')
-
-    cdef inline DTYPE_t rdist(self, DTYPE_t* x1, DTYPE_t* x2,
-                              ITYPE_t size) nogil except -1:
-        cdef DTYPE_t tmp, d=0
-        cdef np.intp_t j
-        for j in range(size):
-            tmp = x1[j] - x2[j]
-            d += tmp * tmp / self.vec_ptr[j]
-        return d
-
-    cdef inline DTYPE_t dist(self, DTYPE_t* x1, DTYPE_t* x2,
-                             ITYPE_t size) nogil except -1:
-        return sqrt(self.rdist(x1, x2, size))
-
-    cdef inline DTYPE_t _rdist_to_dist(self, DTYPE_t rdist) nogil except -1:
-        return sqrt(rdist)
-
-    cdef inline DTYPE_t _dist_to_rdist(self, DTYPE_t dist) nogil except -1:
-        return dist * dist
-
-    def rdist_to_dist(self, rdist):
-        return np.sqrt(rdist)
-
-    def dist_to_rdist(self, dist):
-        return dist ** 2
-
-
-#------------------------------------------------------------
-# Manhattan Distance
-#  d = sum(abs(x_i - y_i))
-cdef class ManhattanDistance(DistanceMetric):
-    r"""Manhattan/City-block Distance metric
-
-    .. math::
-       D(x, y) = \sum_i |x_i - y_i|
-    """
-    def __init__(self):
-        self.p = 1
-
-    cdef inline DTYPE_t dist(self, DTYPE_t* x1, DTYPE_t* x2,
-                             ITYPE_t size) nogil except -1:
-        cdef DTYPE_t d = 0
-        cdef np.intp_t j
-        for j in range(size):
-            d += fabs(x1[j] - x2[j])
-        return d
-
-
-#------------------------------------------------------------
-# Chebyshev Distance
-#  d = max_i(abs(x_i - y_i))
-cdef class ChebyshevDistance(DistanceMetric):
-    """Chebyshev/Infinity Distance
-
-    .. math::
-       D(x, y) = max_i (|x_i - y_i|)
-
-    Examples
-    --------
-    >>> from sklearn.neighbors.dist_metrics import DistanceMetric
-    >>> dist = DistanceMetric.get_metric('chebyshev')
-    >>> X = [[0, 1, 2],
-    ...      [3, 4, 5]]
-    >>> Y = [[-1, 0, 1],
-    ...      [3, 4, 5]]
-    >>> dist.pairwise(X, Y)
-    array([[1.732..., 5.196...],
-           [6.928..., 0....   ]])
-    """
-    def __init__(self):
-        self.p = INF
-
-    cdef inline DTYPE_t dist(self, DTYPE_t* x1, DTYPE_t* x2,
-                             ITYPE_t size) nogil except -1:
-        cdef DTYPE_t d = 0
-        cdef np.intp_t j
-        for j in range(size):
-            d = fmax(d, fabs(x1[j] - x2[j]))
-        return d
-
-
-#------------------------------------------------------------
-# Minkowski Distance
-#  d = sum(x_i^p - y_i^p) ^ (1/p)
-cdef class MinkowskiDistance(DistanceMetric):
-    r"""Minkowski Distance
-
-    .. math::
-       D(x, y) = [\sum_i (x_i - y_i)^p] ^ (1/p)
-
-    Minkowski Distance requires p >= 1 and finite. For p = infinity,
-    use ChebyshevDistance.
-    Note that for p=1, ManhattanDistance is more efficient, and for
-    p=2, EuclideanDistance is more efficient.
-    """
-    def __init__(self, p):
-        if p < 1:
-            raise ValueError("p must be greater than 1")
-        elif np.isinf(p):
-            raise ValueError("MinkowskiDistance requires finite p. "
-                             "For p=inf, use ChebyshevDistance.")
-        self.p = p
-
-    cdef inline DTYPE_t rdist(self, DTYPE_t* x1, DTYPE_t* x2,
-                              ITYPE_t size) nogil except -1:
-        cdef DTYPE_t d=0
-        cdef np.intp_t j
-        for j in range(size):
-            d += pow(fabs(x1[j] - x2[j]), self.p)
-        return d
-
-    cdef inline DTYPE_t dist(self, DTYPE_t* x1, DTYPE_t* x2,
-                             ITYPE_t size) nogil except -1:
-        return pow(self.rdist(x1, x2, size), 1. / self.p)
-
-    cdef inline DTYPE_t _rdist_to_dist(self, DTYPE_t rdist) nogil except -1:
-        return pow(rdist, 1. / self.p)
-
-    cdef inline DTYPE_t _dist_to_rdist(self, DTYPE_t dist) nogil except -1:
-        return pow(dist, self.p)
-
-    def rdist_to_dist(self, rdist):
-        return rdist ** (1. / self.p)
-
-    def dist_to_rdist(self, dist):
-        return dist ** self.p
-
-
-#------------------------------------------------------------
-# W-Minkowski Distance
-#  d = sum(w_i^p * (x_i^p - y_i^p)) ^ (1/p)
-cdef class WMinkowskiDistance(DistanceMetric):
-    r"""Weighted Minkowski Distance
-
-    .. math::
-       D(x, y) = [\sum_i |w_i * (x_i - y_i)|^p] ^ (1/p)
-
-    Weighted Minkowski Distance requires p >= 1 and finite.
-
-    Parameters
-    ----------
-    p : int
-        The order of the norm of the difference :math:`{||u-v||}_p`.
-    w : (N,) array-like
-        The weight vector.
-
-    """
-    def __init__(self, p, w):
-        if p < 1:
-            raise ValueError("p must be greater than 1")
-        elif np.isinf(p):
-            raise ValueError("WMinkowskiDistance requires finite p. "
-                             "For p=inf, use ChebyshevDistance.")
-        self.p = p
-        self.vec = np.asarray(w, dtype=DTYPE)
-        self.vec_ptr = get_vec_ptr(self.vec)
-        self.size = self.vec.shape[0]
-
-    def _validate_data(self, X):
-        if X.shape[1] != self.size:
-            raise ValueError('WMinkowskiDistance dist: '
-                             'size of w does not match')
-
-    cdef inline DTYPE_t rdist(self, DTYPE_t* x1, DTYPE_t* x2,
-                              ITYPE_t size) nogil except -1:
-        cdef DTYPE_t d=0
-        cdef np.intp_t j
-        for j in range(size):
-            d += pow(self.vec_ptr[j] * fabs(x1[j] - x2[j]), self.p)
-        return d
-
-    cdef inline DTYPE_t dist(self, DTYPE_t* x1, DTYPE_t* x2,
-                             ITYPE_t size) nogil except -1:
-        return pow(self.rdist(x1, x2, size), 1. / self.p)
-
-    cdef inline DTYPE_t _rdist_to_dist(self, DTYPE_t rdist) nogil except -1:
-        return pow(rdist, 1. / self.p)
-
-    cdef inline DTYPE_t _dist_to_rdist(self, DTYPE_t dist) nogil except -1:
-        return pow(dist, self.p)
-
-    def rdist_to_dist(self, rdist):
-        return rdist ** (1. / self.p)
-
-    def dist_to_rdist(self, dist):
-        return dist ** self.p
-
-
-#------------------------------------------------------------
-# Mahalanobis Distance
-#  d = sqrt( (x - y)^T V^-1 (x - y) )
-cdef class MahalanobisDistance(DistanceMetric):
-    """Mahalanobis Distance
-
-    .. math::
-       D(x, y) = \sqrt{ (x - y)^T V^{-1} (x - y) }
-
-    Parameters
-    ----------
-    V : array-like
-        Symmetric positive-definite covariance matrix.
-        The inverse of this matrix will be explicitly computed.
-    VI : array-like
-        optionally specify the inverse directly.  If VI is passed,
-        then V is not referenced.
-    """
-    def __init__(self, V=None, VI=None):
-        if VI is None:
-            if V is None:
-                raise ValueError("Must provide either V or VI "
-                                 "for Mahalanobis distance")
-            VI = np.linalg.inv(V)
-        if VI.ndim != 2 or VI.shape[0] != VI.shape[1]:
-            raise ValueError("V/VI must be square")
-
-        self.mat = np.asarray(VI, dtype=float, order='C')
-        self.mat_ptr = get_mat_ptr(self.mat)
-
-        self.size = self.mat.shape[0]
-
-        # we need vec as a work buffer
-        self.vec = np.zeros(self.size, dtype=DTYPE)
-        self.vec_ptr = get_vec_ptr(self.vec)
-
-    def _validate_data(self, X):
-        if X.shape[1] != self.size:
-            raise ValueError('Mahalanobis dist: size of V does not match')
-
-    cdef inline DTYPE_t rdist(self, DTYPE_t* x1, DTYPE_t* x2,
-                              ITYPE_t size) nogil except -1:
-        cdef DTYPE_t tmp, d = 0
-        cdef np.intp_t i, j
-
-        # compute (x1 - x2).T * VI * (x1 - x2)
-        for i in range(size):
-            self.vec_ptr[i] = x1[i] - x2[i]
-
-        for i in range(size):
-            tmp = 0
-            for j in range(size):
-                tmp += self.mat_ptr[i * size + j] * self.vec_ptr[j]
-            d += tmp * self.vec_ptr[i]
-        return d
-
-    cdef inline DTYPE_t dist(self, DTYPE_t* x1, DTYPE_t* x2,
-                             ITYPE_t size) nogil except -1:
-        return sqrt(self.rdist(x1, x2, size))
-
-    cdef inline DTYPE_t _rdist_to_dist(self, DTYPE_t rdist) nogil except -1:
-        return sqrt(rdist)
-
-    cdef inline DTYPE_t _dist_to_rdist(self, DTYPE_t dist) nogil except -1:
-        return dist * dist
-
-    def rdist_to_dist(self, rdist):
-        return np.sqrt(rdist)
-
-    def dist_to_rdist(self, dist):
-        return dist ** 2
-
-
-#------------------------------------------------------------
-# Hamming Distance
-#  d = N_unequal(x, y) / N_tot
-cdef class HammingDistance(DistanceMetric):
-    r"""Hamming Distance
-
-    Hamming distance is meant for discrete-valued vectors, though it is
-    a valid metric for real-valued vectors.
-
-    .. math::
-       D(x, y) = \frac{1}{N} \sum_i \delta_{x_i, y_i}
-    """
-    cdef inline DTYPE_t dist(self, DTYPE_t* x1, DTYPE_t* x2,
-                             ITYPE_t size) nogil except -1:
-        cdef int n_unequal = 0
-        cdef np.intp_t j
-        for j in range(size):
-            if x1[j] != x2[j]:
-                n_unequal += 1
-        return float(n_unequal) / size
-
-
-#------------------------------------------------------------
-# Canberra Distance
-#  D(x, y) = sum[ abs(x_i - y_i) / (abs(x_i) + abs(y_i)) ]
-cdef class CanberraDistance(DistanceMetric):
-    r"""Canberra Distance
-
-    Canberra distance is meant for discrete-valued vectors, though it is
-    a valid metric for real-valued vectors.
-
-    .. math::
-       D(x, y) = \sum_i \frac{|x_i - y_i|}{|x_i| + |y_i|}
-    """
-    cdef inline DTYPE_t dist(self, DTYPE_t* x1, DTYPE_t* x2,
-                             ITYPE_t size) nogil except -1:
-        cdef DTYPE_t denom, d = 0
-        cdef np.intp_t j
-        for j in range(size):
-            denom = fabs(x1[j]) + fabs(x2[j])
-            if denom > 0:
-                d += fabs(x1[j] - x2[j]) / denom
-        return d
-
-
-#------------------------------------------------------------
-# Bray-Curtis Distance
-#  D(x, y) = sum[abs(x_i - y_i)] / sum[abs(x_i) + abs(y_i)]
-cdef class BrayCurtisDistance(DistanceMetric):
-    r"""Bray-Curtis Distance
-
-    Bray-Curtis distance is meant for discrete-valued vectors, though it is
-    a valid metric for real-valued vectors.
-
-    .. math::
-       D(x, y) = \frac{\sum_i |x_i - y_i|}{\sum_i(|x_i| + |y_i|)}
-    """
-    cdef inline DTYPE_t dist(self, DTYPE_t* x1, DTYPE_t* x2,
-                             ITYPE_t size) nogil except -1:
-        cdef DTYPE_t num = 0, denom = 0
-        cdef np.intp_t j
-        for j in range(size):
-            num += fabs(x1[j] - x2[j])
-            denom += fabs(x1[j]) + fabs(x2[j])
-        if denom > 0:
-            return num / denom
-        else:
-            return 0.0
-
-
-#------------------------------------------------------------
-# Jaccard Distance (boolean)
-#  D(x, y) = N_unequal(x, y) / N_nonzero(x, y)
-cdef class JaccardDistance(DistanceMetric):
-    r"""Jaccard Distance
-
-    Jaccard Distance is a dissimilarity measure for boolean-valued
-    vectors. All nonzero entries will be treated as True, zero entries will
-    be treated as False.
-
-    .. math::
-       D(x, y) = \frac{N_{TF} + N_{FT}}{N_{TT} + N_{TF} + N_{FT}}
-    """
-    cdef inline DTYPE_t dist(self, DTYPE_t* x1, DTYPE_t* x2,
-                             ITYPE_t size) nogil except -1:
-        cdef int tf1, tf2, n_eq = 0, nnz = 0
-        cdef np.intp_t j
-        for j in range(size):
-            tf1 = x1[j] != 0
-            tf2 = x2[j] != 0
-            nnz += (tf1 or tf2)
-            n_eq += (tf1 and tf2)
-        # Based on https://github.com/scipy/scipy/pull/7373
-        # When comparing two all-zero vectors, scipy>=1.2.0 jaccard metric
-        # was changed to return 0, instead of nan.
-        if nnz == 0:
-            return 0
-        return (nnz - n_eq) * 1.0 / nnz
-
-
-#------------------------------------------------------------
-# Matching Distance (boolean)
-#  D(x, y) = n_neq / n
-cdef class MatchingDistance(DistanceMetric):
-    r"""Matching Distance
-
-    Matching Distance is a dissimilarity measure for boolean-valued
-    vectors. All nonzero entries will be treated as True, zero entries will
-    be treated as False.
-
-    .. math::
-       D(x, y) = \frac{N_{TF} + N_{FT}}{N}
-    """
-    cdef inline DTYPE_t dist(self, DTYPE_t* x1, DTYPE_t* x2,
-                             ITYPE_t size) nogil except -1:
-        cdef int tf1, tf2, n_neq = 0
-        cdef np.intp_t j
-        for j in range(size):
-            tf1 = x1[j] != 0
-            tf2 = x2[j] != 0
-            n_neq += (tf1 != tf2)
-        return n_neq * 1. / size
-
-
-#------------------------------------------------------------
-# Dice Distance (boolean)
-#  D(x, y) = n_neq / (2 * ntt + n_neq)
-cdef class DiceDistance(DistanceMetric):
-    r"""Dice Distance
-
-    Dice Distance is a dissimilarity measure for boolean-valued
-    vectors. All nonzero entries will be treated as True, zero entries will
-    be treated as False.
-
-    .. math::
-       D(x, y) = \frac{N_{TF} + N_{FT}}{2 * N_{TT} + N_{TF} + N_{FT}}
-    """
-    cdef inline DTYPE_t dist(self, DTYPE_t* x1, DTYPE_t* x2,
-                             ITYPE_t size) nogil except -1:
-        cdef int tf1, tf2, n_neq = 0, ntt = 0
-        cdef np.intp_t j
-        for j in range(size):
-            tf1 = x1[j] != 0
-            tf2 = x2[j] != 0
-            ntt += (tf1 and tf2)
-            n_neq += (tf1 != tf2)
-        return n_neq / (2.0 * ntt + n_neq)
-
-
-#------------------------------------------------------------
-# Kulsinski Distance (boolean)
-#  D(x, y) = (ntf + nft - ntt + n) / (n_neq + n)
-cdef class KulsinskiDistance(DistanceMetric):
-    r"""Kulsinski Distance
-
-    Kulsinski Distance is a dissimilarity measure for boolean-valued
-    vectors. All nonzero entries will be treated as True, zero entries will
-    be treated as False.
-
-    .. math::
-       D(x, y) = 1 - \frac{N_{TT}}{N + N_{TF} + N_{FT}}
-    """
-    cdef inline DTYPE_t dist(self, DTYPE_t* x1, DTYPE_t* x2,
-                             ITYPE_t size) nogil except -1:
-        cdef int tf1, tf2, ntt = 0, n_neq = 0
-        cdef np.intp_t j
-        for j in range(size):
-            tf1 = x1[j] != 0
-            tf2 = x2[j] != 0
-            n_neq += (tf1 != tf2)
-            ntt += (tf1 and tf2)
-        return (n_neq - ntt + size) * 1.0 / (n_neq + size)
-
-
-#------------------------------------------------------------
-# Rogers-Tanimoto Distance (boolean)
-#  D(x, y) = 2 * n_neq / (n + n_neq)
-cdef class RogersTanimotoDistance(DistanceMetric):
-    r"""Rogers-Tanimoto Distance
-
-    Rogers-Tanimoto Distance is a dissimilarity measure for boolean-valued
-    vectors. All nonzero entries will be treated as True, zero entries will
-    be treated as False.
-
-    .. math::
-       D(x, y) = \frac{2 (N_{TF} + N_{FT})}{N + N_{TF} + N_{FT}}
-    """
-    cdef inline DTYPE_t dist(self, DTYPE_t* x1, DTYPE_t* x2,
-                             ITYPE_t size) nogil except -1:
-        cdef int tf1, tf2, n_neq = 0
-        cdef np.intp_t j
-        for j in range(size):
-            tf1 = x1[j] != 0
-            tf2 = x2[j] != 0
-            n_neq += (tf1 != tf2)
-        return (2.0 * n_neq) / (size + n_neq)
-
-
-#------------------------------------------------------------
-# Russell-Rao Distance (boolean)
-#  D(x, y) = (n - ntt) / n
-cdef class RussellRaoDistance(DistanceMetric):
-    r"""Russell-Rao Distance
-
-    Russell-Rao Distance is a dissimilarity measure for boolean-valued
-    vectors. All nonzero entries will be treated as True, zero entries will
-    be treated as False.
-
-    .. math::
-       D(x, y) = \frac{N - N_{TT}}{N}
-    """
-    cdef inline DTYPE_t dist(self, DTYPE_t* x1, DTYPE_t* x2,
-                             ITYPE_t size) nogil except -1:
-        cdef int tf1, tf2, ntt = 0
-        cdef np.intp_t j
-        for j in range(size):
-            tf1 = x1[j] != 0
-            tf2 = x2[j] != 0
-            ntt += (tf1 and tf2)
-        return (size - ntt) * 1. / size
-
-
-#------------------------------------------------------------
-# Sokal-Michener Distance (boolean)
-#  D(x, y) = 2 * n_neq / (n + n_neq)
-cdef class SokalMichenerDistance(DistanceMetric):
-    r"""Sokal-Michener Distance
-
-    Sokal-Michener Distance is a dissimilarity measure for boolean-valued
-    vectors. All nonzero entries will be treated as True, zero entries will
-    be treated as False.
-
-    .. math::
-       D(x, y) = \frac{2 (N_{TF} + N_{FT})}{N + N_{TF} + N_{FT}}
-    """
-    cdef inline DTYPE_t dist(self, DTYPE_t* x1, DTYPE_t* x2,
-                             ITYPE_t size) nogil except -1:
-        cdef int tf1, tf2, n_neq = 0
-        cdef np.intp_t j
-        for j in range(size):
-            tf1 = x1[j] != 0
-            tf2 = x2[j] != 0
-            n_neq += (tf1 != tf2)
-        return (2.0 * n_neq) / (size + n_neq)
-
-
-#------------------------------------------------------------
-# Sokal-Sneath Distance (boolean)
-#  D(x, y) = n_neq / (0.5 * n_tt + n_neq)
-cdef class SokalSneathDistance(DistanceMetric):
-    r"""Sokal-Sneath Distance
-
-    Sokal-Sneath Distance is a dissimilarity measure for boolean-valued
-    vectors. All nonzero entries will be treated as True, zero entries will
-    be treated as False.
-
-    .. math::
-       D(x, y) = \frac{N_{TF} + N_{FT}}{N_{TT} / 2 + N_{TF} + N_{FT}}
-    """
-    cdef inline DTYPE_t dist(self, DTYPE_t* x1, DTYPE_t* x2,
-                             ITYPE_t size) nogil except -1:
-        cdef int tf1, tf2, ntt = 0, n_neq = 0
-        cdef np.intp_t j
-        for j in range(size):
-            tf1 = x1[j] != 0
-            tf2 = x2[j] != 0
-            n_neq += (tf1 != tf2)
-            ntt += (tf1 and tf2)
-        return n_neq / (0.5 * ntt + n_neq)
-
-
-#------------------------------------------------------------
-# Haversine Distance (2 dimensional)
-#  D(x, y) = 2 arcsin{sqrt[sin^2 ((x1 - y1) / 2)
-#                          + cos(x1) cos(y1) sin^2 ((x2 - y2) / 2)]}
-cdef class HaversineDistance(DistanceMetric):
-    """Haversine (Spherical) Distance
-
-    The Haversine distance is the angular distance between two points on
-    the surface of a sphere.  The first distance of each point is assumed
-    to be the latitude, the second is the longitude, given in radians.
-    The dimension of the points must be 2:
-
-    .. math::
-       D(x, y) = 2\\arcsin[\\sqrt{\\sin^2((x1 - y1) / 2)
-                                + \\cos(x1)\\cos(y1)\\sin^2((x2 - y2) / 2)}]
-    """
-
-    def _validate_data(self, X):
-        if X.shape[1] != 2:
-            raise ValueError("Haversine distance only valid "
-                             "in 2 dimensions")
-
-    cdef inline DTYPE_t rdist(self, DTYPE_t* x1, DTYPE_t* x2,
-                              ITYPE_t size) nogil except -1:
-        cdef DTYPE_t sin_0 = sin(0.5 * (x1[0] - x2[0]))
-        cdef DTYPE_t sin_1 = sin(0.5 * (x1[1] - x2[1]))
-        return (sin_0 * sin_0 + cos(x1[0]) * cos(x2[0]) * sin_1 * sin_1)
-
-    cdef inline DTYPE_t dist(self, DTYPE_t* x1, DTYPE_t* x2,
-                             ITYPE_t size) nogil except -1:
-        return 2 * asin(sqrt(self.rdist(x1, x2, size)))
-
-    cdef inline DTYPE_t _rdist_to_dist(self, DTYPE_t rdist) nogil except -1:
-        return 2 * asin(sqrt(rdist))
-
-    cdef inline DTYPE_t _dist_to_rdist(self, DTYPE_t dist) nogil except -1:
-        cdef DTYPE_t tmp = sin(0.5 * dist)
-        return tmp * tmp
-
-    def rdist_to_dist(self, rdist):
-        return 2 * np.arcsin(np.sqrt(rdist))
-
-    def dist_to_rdist(self, dist):
-        tmp = np.sin(0.5 * dist)
-        return tmp * tmp
-
-
-#------------------------------------------------------------
-# Yule Distance (boolean)
-#  D(x, y) = 2 * ntf * nft / (ntt * nff + ntf * nft)
-# [This is not a true metric, so we will leave it out.]
-#
-#cdef class YuleDistance(DistanceMetric):
-#    cdef inline DTYPE_t dist(self, DTYPE_t* x1, DTYPE_t* x2, ITYPE_t size):
-#        cdef int tf1, tf2, ntf = 0, nft = 0, ntt = 0, nff = 0
-#        cdef np.intp_t j
-#        for j in range(size):
-#            tf1 = x1[j] != 0
-#            tf2 = x2[j] != 0
-#            ntt += tf1 and tf2
-#            ntf += tf1 and (tf2 == 0)
-#            nft += (tf1 == 0) and tf2
-#        nff = size - ntt - ntf - nft
-#        return (2.0 * ntf * nft) / (ntt * nff + ntf * nft)
-
-
-#------------------------------------------------------------
-# Cosine Distance
-#  D(x, y) = dot(x, y) / (|x| * |y|)
-# [This is not a true metric, so we will leave it out.]
-#
-#cdef class CosineDistance(DistanceMetric):
-#    cdef inline DTYPE_t dist(self, DTYPE_t* x1, DTYPE_t* x2, ITYPE_t size):
-#        cdef DTYPE_t d = 0, norm1 = 0, norm2 = 0
-#        cdef np.intp_t j
-#        for j in range(size):
-#            d += x1[j] * x2[j]
-#            norm1 += x1[j] * x1[j]
-#            norm2 += x2[j] * x2[j]
-#        return 1.0 - d / sqrt(norm1 * norm2)
-
-
-#------------------------------------------------------------
-# Correlation Distance
-#  D(x, y) = dot((x - mx), (y - my)) / (|x - mx| * |y - my|)
-# [This is not a true metric, so we will leave it out.]
-#
-#cdef class CorrelationDistance(DistanceMetric):
-#    cdef inline DTYPE_t dist(self, DTYPE_t* x1, DTYPE_t* x2, ITYPE_t size):
-#        cdef DTYPE_t mu1 = 0, mu2 = 0, x1nrm = 0, x2nrm = 0, x1Tx2 = 0
-#        cdef DTYPE_t tmp1, tmp2
-#
-#        cdef np.intp_t i
-#        for i in range(size):
-#            mu1 += x1[i]
-#            mu2 += x2[i]
-#        mu1 /= size
-#        mu2 /= size
-#
-#        for i in range(size):
-#            tmp1 = x1[i] - mu1
-#            tmp2 = x2[i] - mu2
-#            x1nrm += tmp1 * tmp1
-#            x2nrm += tmp2 * tmp2
-#            x1Tx2 += tmp1 * tmp2
-#
-#        return (1. - x1Tx2) / sqrt(x1nrm * x2nrm)
-
-
-#------------------------------------------------------------
-# User-defined distance
-#
-cdef class PyFuncDistance(DistanceMetric):
-    """PyFunc Distance
-
-    A user-defined distance
-
-    Parameters
-    ----------
-    func : function
-        func should take two numpy arrays as input, and return a distance.
-    """
-    def __init__(self, func, **kwargs):
-        self.func = func
-        self.kwargs = kwargs
-
-    # in cython < 0.26, GIL was required to be acquired during definition of
-    # the function and inside the body of the function. This behaviour is not
-    # allowed in cython >= 0.26 since it is a redundant GIL acquisition. The
-    # only way to be back compatible is to inherit `dist` from the base class
-    # without GIL and called an inline `_dist` which acquire GIL.
-    cdef inline DTYPE_t dist(self, DTYPE_t* x1, DTYPE_t* x2,
-                             ITYPE_t size) nogil except -1:
-        return self._dist(x1, x2, size)
-
-    cdef inline DTYPE_t _dist(self, DTYPE_t* x1, DTYPE_t* x2,
-                              ITYPE_t size) except -1 with gil:
-        cdef np.ndarray x1arr
-        cdef np.ndarray x2arr
-        x1arr = _buffer_to_ndarray(x1, size)
-        x2arr = _buffer_to_ndarray(x2, size)
-        d = self.func(x1arr, x2arr, **self.kwargs)
-        try:
-            # Cython generates code here that results in a TypeError
-            # if d is the wrong type.
-            return d
-        except TypeError:
-            raise TypeError("Custom distance function must accept two "
-                            "vectors and return a float.")
-
-
-cdef inline double fmax(double a, double b) nogil:
-    return max(a, b)
diff --git a/sklearn/neighbors/_graph.py b/sklearn/neighbors/_graph.py
index a23b45399f05c..3562fab1fcf01 100644
--- a/sklearn/neighbors/_graph.py
+++ b/sklearn/neighbors/_graph.py
@@ -1,33 +1,39 @@
 """Nearest Neighbors graph functions"""
 
-# Author: Jake Vanderplas 
-#         Tom Dupre la Tour
-#
-# License: BSD 3 clause (C) INRIA, University of Amsterdam
-from ._base import KNeighborsMixin, RadiusNeighborsMixin
-from ._base import NeighborsBase
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+import itertools
+
+from ..base import ClassNamePrefixFeaturesOutMixin, TransformerMixin, _fit_context
+from ..utils._param_validation import (
+    Integral,
+    Interval,
+    Real,
+    StrOptions,
+    validate_params,
+)
+from ..utils.validation import check_is_fitted
+from ._base import VALID_METRICS, KNeighborsMixin, NeighborsBase, RadiusNeighborsMixin
 from ._unsupervised import NearestNeighbors
-from ..base import TransformerMixin
-from ..utils.validation import check_is_fitted, _deprecate_positional_args
 
 
 def _check_params(X, metric, p, metric_params):
     """Check the validity of the input parameters"""
-    params = zip(['metric', 'p', 'metric_params'],
-                 [metric, p, metric_params])
+    params = zip(["metric", "p", "metric_params"], [metric, p, metric_params])
     est_params = X.get_params()
     for param_name, func_param in params:
         if func_param != est_params[param_name]:
             raise ValueError(
-                "Got %s for %s, while the estimator has %s for "
-                "the same parameter." % (
-                    func_param, param_name, est_params[param_name]))
+                "Got %s for %s, while the estimator has %s for the same parameter."
+                % (func_param, param_name, est_params[param_name])
+            )
 
 
 def _query_include_self(X, include_self, mode):
     """Return the query based on include_self param"""
-    if include_self == 'auto':
-        include_self = mode == 'connectivity'
+    if include_self == "auto":
+        include_self = mode == "connectivity"
 
     # it does not include each sample as its own neighbors
     if not include_self:
@@ -36,19 +42,38 @@ def _query_include_self(X, include_self, mode):
     return X
 
 
-@_deprecate_positional_args
-def kneighbors_graph(X, n_neighbors, *, mode='connectivity',
-                     metric='minkowski', p=2, metric_params=None,
-                     include_self=False, n_jobs=None):
-    """Computes the (weighted) graph of k-Neighbors for points in X
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix", KNeighborsMixin],
+        "n_neighbors": [Interval(Integral, 1, None, closed="left")],
+        "mode": [StrOptions({"connectivity", "distance"})],
+        "metric": [StrOptions(set(itertools.chain(*VALID_METRICS.values()))), callable],
+        "p": [Interval(Real, 0, None, closed="right"), None],
+        "metric_params": [dict, None],
+        "include_self": ["boolean", StrOptions({"auto"})],
+        "n_jobs": [Integral, None],
+    },
+    prefer_skip_nested_validation=False,  # metric is not validated yet
+)
+def kneighbors_graph(
+    X,
+    n_neighbors,
+    *,
+    mode="connectivity",
+    metric="minkowski",
+    p=2,
+    metric_params=None,
+    include_self=False,
+    n_jobs=None,
+):
+    """Compute the (weighted) graph of k-Neighbors for points in X.
 
     Read more in the :ref:`User Guide `.
 
     Parameters
     ----------
-    X : array-like of shape (n_samples, n_features) or BallTree
-        Sample data, in the form of a numpy array or a precomputed
-        :class:`BallTree`.
+    X : {array-like, sparse matrix} of shape (n_samples, n_features)
+        Sample data.
 
     n_neighbors : int
         Number of neighbors for each sample.
@@ -59,18 +84,22 @@ def kneighbors_graph(X, n_neighbors, *, mode='connectivity',
         between neighbors according to the given metric.
 
     metric : str, default='minkowski'
-        The distance metric used to calculate the k-Neighbors for each sample
-        point. The DistanceMetric class gives a list of available metrics.
-        The default distance is 'euclidean' ('minkowski' metric with the p
-        param equal to 2.)
-
-    p : int, default=2
-        Power parameter for the Minkowski metric. When p = 1, this is
-        equivalent to using manhattan_distance (l1), and euclidean_distance
-        (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used.
+        Metric to use for distance computation. Default is "minkowski", which
+        results in the standard Euclidean distance when p = 2. See the
+        documentation of `scipy.spatial.distance
+        `_ and
+        the metrics listed in
+        :class:`~sklearn.metrics.pairwise.distance_metrics` for valid metric
+        values.
+
+    p : float, default=2
+        Power parameter for the Minkowski metric. When p = 1, this is equivalent
+        to using manhattan_distance (l1), and euclidean_distance (l2) for p = 2.
+        For arbitrary p, minkowski_distance (l_p) is used. This parameter is expected
+        to be positive.
 
     metric_params : dict, default=None
-        additional keyword arguments for the metric function.
+        Additional keyword arguments for the metric function.
 
     include_self : bool or 'auto', default=False
         Whether or not to mark each sample as the first nearest neighbor to
@@ -89,6 +118,10 @@ def kneighbors_graph(X, n_neighbors, *, mode='connectivity',
         Graph where A[i, j] is assigned the weight of edge that
         connects i to j. The matrix is of CSR format.
 
+    See Also
+    --------
+    radius_neighbors_graph: Compute the (weighted) graph of Neighbors for points in X.
+
     Examples
     --------
     >>> X = [[0], [3], [1]]
@@ -98,14 +131,15 @@ def kneighbors_graph(X, n_neighbors, *, mode='connectivity',
     array([[1., 0., 1.],
            [0., 1., 1.],
            [1., 0., 1.]])
-
-    See Also
-    --------
-    radius_neighbors_graph
     """
     if not isinstance(X, KNeighborsMixin):
-        X = NearestNeighbors(n_neighbors=n_neighbors, metric=metric, p=p,
-                             metric_params=metric_params, n_jobs=n_jobs).fit(X)
+        X = NearestNeighbors(
+            n_neighbors=n_neighbors,
+            metric=metric,
+            p=p,
+            metric_params=metric_params,
+            n_jobs=n_jobs,
+        ).fit(X)
     else:
         _check_params(X, metric, p, metric_params)
 
@@ -113,11 +147,31 @@ def kneighbors_graph(X, n_neighbors, *, mode='connectivity',
     return X.kneighbors_graph(X=query, n_neighbors=n_neighbors, mode=mode)
 
 
-@_deprecate_positional_args
-def radius_neighbors_graph(X, radius, *, mode='connectivity',
-                           metric='minkowski', p=2, metric_params=None,
-                           include_self=False, n_jobs=None):
-    """Computes the (weighted) graph of Neighbors for points in X
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix", RadiusNeighborsMixin],
+        "radius": [Interval(Real, 0, None, closed="both")],
+        "mode": [StrOptions({"connectivity", "distance"})],
+        "metric": [StrOptions(set(itertools.chain(*VALID_METRICS.values()))), callable],
+        "p": [Interval(Real, 0, None, closed="right"), None],
+        "metric_params": [dict, None],
+        "include_self": ["boolean", StrOptions({"auto"})],
+        "n_jobs": [Integral, None],
+    },
+    prefer_skip_nested_validation=False,  # metric is not validated yet
+)
+def radius_neighbors_graph(
+    X,
+    radius,
+    *,
+    mode="connectivity",
+    metric="minkowski",
+    p=2,
+    metric_params=None,
+    include_self=False,
+    n_jobs=None,
+):
+    """Compute the (weighted) graph of Neighbors for points in X.
 
     Neighborhoods are restricted the points at a distance lower than
     radius.
@@ -126,9 +180,8 @@ def radius_neighbors_graph(X, radius, *, mode='connectivity',
 
     Parameters
     ----------
-    X : array-like of shape (n_samples, n_features) or BallTree
-        Sample data, in the form of a numpy array or a precomputed
-        :class:`BallTree`.
+    X : {array-like, sparse matrix} of shape (n_samples, n_features)
+        Sample data.
 
     radius : float
         Radius of neighborhoods.
@@ -139,18 +192,21 @@ def radius_neighbors_graph(X, radius, *, mode='connectivity',
         between neighbors according to the given metric.
 
     metric : str, default='minkowski'
-        The distance metric used to calculate the neighbors within a
-        given radius for each sample point. The DistanceMetric class
-        gives a list of available metrics. The default distance is
-        'euclidean' ('minkowski' metric with the param equal to 2.)
-
-    p : int, default=2
+        Metric to use for distance computation. Default is "minkowski", which
+        results in the standard Euclidean distance when p = 2. See the
+        documentation of `scipy.spatial.distance
+        `_ and
+        the metrics listed in
+        :class:`~sklearn.metrics.pairwise.distance_metrics` for valid metric
+        values.
+
+    p : float, default=2
         Power parameter for the Minkowski metric. When p = 1, this is
         equivalent to using manhattan_distance (l1), and euclidean_distance
         (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used.
 
     metric_params : dict, default=None
-        additional keyword arguments for the metric function.
+        Additional keyword arguments for the metric function.
 
     include_self : bool or 'auto', default=False
         Whether or not to mark each sample as the first nearest neighbor to
@@ -169,6 +225,10 @@ def radius_neighbors_graph(X, radius, *, mode='connectivity',
         Graph where A[i, j] is assigned the weight of edge that connects
         i to j. The matrix is of CSR format.
 
+    See Also
+    --------
+    kneighbors_graph: Compute the weighted graph of k-neighbors for points in X.
+
     Examples
     --------
     >>> X = [[0], [3], [1]]
@@ -179,14 +239,15 @@ def radius_neighbors_graph(X, radius, *, mode='connectivity',
     array([[1., 0., 1.],
            [0., 1., 0.],
            [1., 0., 1.]])
-
-    See Also
-    --------
-    kneighbors_graph
     """
     if not isinstance(X, RadiusNeighborsMixin):
-        X = NearestNeighbors(radius=radius, metric=metric, p=p,
-                             metric_params=metric_params, n_jobs=n_jobs).fit(X)
+        X = NearestNeighbors(
+            radius=radius,
+            metric=metric,
+            p=p,
+            metric_params=metric_params,
+            n_jobs=n_jobs,
+        ).fit(X)
     else:
         _check_params(X, metric, p, metric_params)
 
@@ -194,10 +255,10 @@ def radius_neighbors_graph(X, radius, *, mode='connectivity',
     return X.radius_neighbors_graph(query, radius, mode)
 
 
-class KNeighborsTransformer(KNeighborsMixin,
-                            TransformerMixin,
-                            NeighborsBase):
-    """Transform X into a (weighted) graph of k nearest neighbors
+class KNeighborsTransformer(
+    ClassNamePrefixFeaturesOutMixin, KNeighborsMixin, TransformerMixin, NeighborsBase
+):
+    """Transform X into a (weighted) graph of k nearest neighbors.
 
     The transformed data is a sparse graph as returned by kneighbors_graph.
 
@@ -237,41 +298,32 @@ class KNeighborsTransformer(KNeighborsMixin,
         nature of the problem.
 
     metric : str or callable, default='minkowski'
-        metric to use for distance computation. Any metric from scikit-learn
-        or scipy.spatial.distance can be used.
-
-        If metric is a callable function, it is called on each
-        pair of instances (rows) and the resulting value recorded. The callable
-        should take two arrays as input and return one value indicating the
-        distance between them. This works for Scipy's metrics, but is less
+        Metric to use for distance computation. Default is "minkowski", which
+        results in the standard Euclidean distance when p = 2. See the
+        documentation of `scipy.spatial.distance
+        `_ and
+        the metrics listed in
+        :class:`~sklearn.metrics.pairwise.distance_metrics` for valid metric
+        values.
+
+        If metric is a callable function, it takes two arrays representing 1D
+        vectors as inputs and must return one value indicating the distance
+        between those vectors. This works for Scipy's metrics, but is less
         efficient than passing the metric name as a string.
 
         Distance matrices are not supported.
 
-        Valid values for metric are:
-
-        - from scikit-learn: ['cityblock', 'cosine', 'euclidean', 'l1', 'l2',
-          'manhattan']
-
-        - from scipy.spatial.distance: ['braycurtis', 'canberra', 'chebyshev',
-          'correlation', 'dice', 'hamming', 'jaccard', 'kulsinski',
-          'mahalanobis', 'minkowski', 'rogerstanimoto', 'russellrao',
-          'seuclidean', 'sokalmichener', 'sokalsneath', 'sqeuclidean',
-          'yule']
-
-        See the documentation for scipy.spatial.distance for details on these
-        metrics.
-
-    p : int, default=2
+    p : float, default=2
         Parameter for the Minkowski metric from
         sklearn.metrics.pairwise.pairwise_distances. When p = 1, this is
         equivalent to using manhattan_distance (l1), and euclidean_distance
         (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used.
+        This parameter is expected to be positive.
 
     metric_params : dict, default=None
         Additional keyword arguments for the metric function.
 
-    n_jobs : int, default=1
+    n_jobs : int, default=None
         The number of parallel jobs to run for neighbors search.
         If ``-1``, then the number of jobs is set to the number of CPU cores.
 
@@ -288,28 +340,80 @@ class KNeighborsTransformer(KNeighborsMixin,
         `p` parameter value if the `effective_metric_` attribute is set to
         'minkowski'.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     n_samples_fit_ : int
         Number of samples in the fitted data.
 
+    See Also
+    --------
+    kneighbors_graph : Compute the weighted graph of k-neighbors for
+        points in X.
+    RadiusNeighborsTransformer : Transform X into a weighted graph of
+        neighbors nearer than a radius.
+
+    Notes
+    -----
+    For an example of using :class:`~sklearn.neighbors.KNeighborsTransformer`
+    in combination with :class:`~sklearn.manifold.TSNE` see
+    :ref:`sphx_glr_auto_examples_neighbors_approximate_nearest_neighbors.py`.
+
     Examples
     --------
-    >>> from sklearn.manifold import Isomap
+    >>> from sklearn.datasets import load_wine
     >>> from sklearn.neighbors import KNeighborsTransformer
-    >>> from sklearn.pipeline import make_pipeline
-    >>> estimator = make_pipeline(
-    ...     KNeighborsTransformer(n_neighbors=5, mode='distance'),
-    ...     Isomap(neighbors_algorithm='precomputed'))
+    >>> X, _ = load_wine(return_X_y=True)
+    >>> X.shape
+    (178, 13)
+    >>> transformer = KNeighborsTransformer(n_neighbors=5, mode='distance')
+    >>> X_dist_graph = transformer.fit_transform(X)
+    >>> X_dist_graph.shape
+    (178, 178)
     """
-    @_deprecate_positional_args
-    def __init__(self, *, mode='distance', n_neighbors=5, algorithm='auto',
-                 leaf_size=30, metric='minkowski', p=2, metric_params=None,
-                 n_jobs=1):
-        super(KNeighborsTransformer, self).__init__(
-            n_neighbors=n_neighbors, radius=None, algorithm=algorithm,
-            leaf_size=leaf_size, metric=metric, p=p,
-            metric_params=metric_params, n_jobs=n_jobs)
+
+    _parameter_constraints: dict = {
+        **NeighborsBase._parameter_constraints,
+        "mode": [StrOptions({"distance", "connectivity"})],
+    }
+    _parameter_constraints.pop("radius")
+
+    def __init__(
+        self,
+        *,
+        mode="distance",
+        n_neighbors=5,
+        algorithm="auto",
+        leaf_size=30,
+        metric="minkowski",
+        p=2,
+        metric_params=None,
+        n_jobs=None,
+    ):
+        super().__init__(
+            n_neighbors=n_neighbors,
+            radius=None,
+            algorithm=algorithm,
+            leaf_size=leaf_size,
+            metric=metric,
+            p=p,
+            metric_params=metric_params,
+            n_jobs=n_jobs,
+        )
         self.mode = mode
 
+    @_fit_context(
+        # KNeighborsTransformer.metric is not validated yet
+        prefer_skip_nested_validation=False
+    )
     def fit(self, X, y=None):
         """Fit the k-nearest neighbors transformer from the training dataset.
 
@@ -318,16 +422,20 @@ def fit(self, X, y=None):
         X : {array-like, sparse matrix} of shape (n_samples, n_features) or \
                 (n_samples, n_samples) if metric='precomputed'
             Training data.
+        y : Ignored
+            Not used, present for API consistency by convention.
 
         Returns
         -------
         self : KNeighborsTransformer
             The fitted k-nearest neighbors transformer.
         """
-        return self._fit(X)
+        self._fit(X)
+        self._n_features_out = self.n_samples_fit_
+        return self
 
     def transform(self, X):
-        """Computes the (weighted) graph of Neighbors for points in X
+        """Compute the (weighted) graph of Neighbors for points in X.
 
         Parameters
         ----------
@@ -343,9 +451,10 @@ def transform(self, X):
             The matrix is of CSR format.
         """
         check_is_fitted(self)
-        add_one = self.mode == 'distance'
-        return self.kneighbors_graph(X, mode=self.mode,
-                                     n_neighbors=self.n_neighbors + add_one)
+        add_one = self.mode == "distance"
+        return self.kneighbors_graph(
+            X, mode=self.mode, n_neighbors=self.n_neighbors + add_one
+        )
 
     def fit_transform(self, X, y=None):
         """Fit to data, then transform it.
@@ -358,7 +467,8 @@ def fit_transform(self, X, y=None):
         X : array-like of shape (n_samples, n_features)
             Training set.
 
-        y : ignored
+        y : Ignored
+            Not used, present for API consistency by convention.
 
         Returns
         -------
@@ -370,22 +480,17 @@ def fit_transform(self, X, y=None):
         """
         return self.fit(X).transform(X)
 
-    def _more_tags(self):
-        return {
-            '_xfail_checks': {
-                'check_methods_sample_order_invariance':
-                'check is not applicable.'
-            }
-        }
 
-
-class RadiusNeighborsTransformer(RadiusNeighborsMixin,
-                                 TransformerMixin,
-                                 NeighborsBase):
-    """Transform X into a (weighted) graph of neighbors nearer than a radius
+class RadiusNeighborsTransformer(
+    ClassNamePrefixFeaturesOutMixin,
+    RadiusNeighborsMixin,
+    TransformerMixin,
+    NeighborsBase,
+):
+    """Transform X into a (weighted) graph of neighbors nearer than a radius.
 
     The transformed data is a sparse graph as returned by
-    radius_neighbors_graph.
+    `radius_neighbors_graph`.
 
     Read more in the :ref:`User Guide `.
 
@@ -398,7 +503,7 @@ class RadiusNeighborsTransformer(RadiusNeighborsMixin,
         matrix with ones and zeros, and 'distance' will return the distances
         between neighbors according to the given metric.
 
-    radius : float, default=1.
+    radius : float, default=1.0
         Radius of neighborhood in the transformed sparse graph.
 
     algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, default='auto'
@@ -420,41 +525,32 @@ class RadiusNeighborsTransformer(RadiusNeighborsMixin,
         nature of the problem.
 
     metric : str or callable, default='minkowski'
-        metric to use for distance computation. Any metric from scikit-learn
-        or scipy.spatial.distance can be used.
-
-        If metric is a callable function, it is called on each
-        pair of instances (rows) and the resulting value recorded. The callable
-        should take two arrays as input and return one value indicating the
-        distance between them. This works for Scipy's metrics, but is less
+        Metric to use for distance computation. Default is "minkowski", which
+        results in the standard Euclidean distance when p = 2. See the
+        documentation of `scipy.spatial.distance
+        `_ and
+        the metrics listed in
+        :class:`~sklearn.metrics.pairwise.distance_metrics` for valid metric
+        values.
+
+        If metric is a callable function, it takes two arrays representing 1D
+        vectors as inputs and must return one value indicating the distance
+        between those vectors. This works for Scipy's metrics, but is less
         efficient than passing the metric name as a string.
 
         Distance matrices are not supported.
 
-        Valid values for metric are:
-
-        - from scikit-learn: ['cityblock', 'cosine', 'euclidean', 'l1', 'l2',
-          'manhattan']
-
-        - from scipy.spatial.distance: ['braycurtis', 'canberra', 'chebyshev',
-          'correlation', 'dice', 'hamming', 'jaccard', 'kulsinski',
-          'mahalanobis', 'minkowski', 'rogerstanimoto', 'russellrao',
-          'seuclidean', 'sokalmichener', 'sokalsneath', 'sqeuclidean',
-          'yule']
-
-        See the documentation for scipy.spatial.distance for details on these
-        metrics.
-
-    p : int, default=2
+    p : float, default=2
         Parameter for the Minkowski metric from
         sklearn.metrics.pairwise.pairwise_distances. When p = 1, this is
         equivalent to using manhattan_distance (l1), and euclidean_distance
         (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used.
+        This parameter is expected to be positive.
 
     metric_params : dict, default=None
         Additional keyword arguments for the metric function.
 
-    n_jobs : int, default=1
+    n_jobs : int, default=None
         The number of parallel jobs to run for neighbors search.
         If ``-1``, then the number of jobs is set to the number of CPU cores.
 
@@ -471,28 +567,78 @@ class RadiusNeighborsTransformer(RadiusNeighborsMixin,
         `p` parameter value if the `effective_metric_` attribute is set to
         'minkowski'.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     n_samples_fit_ : int
         Number of samples in the fitted data.
 
+    See Also
+    --------
+    kneighbors_graph : Compute the weighted graph of k-neighbors for
+        points in X.
+    KNeighborsTransformer : Transform X into a weighted graph of k
+        nearest neighbors.
+
     Examples
     --------
+    >>> import numpy as np
+    >>> from sklearn.datasets import load_wine
     >>> from sklearn.cluster import DBSCAN
     >>> from sklearn.neighbors import RadiusNeighborsTransformer
     >>> from sklearn.pipeline import make_pipeline
+    >>> X, _ = load_wine(return_X_y=True)
     >>> estimator = make_pipeline(
     ...     RadiusNeighborsTransformer(radius=42.0, mode='distance'),
-    ...     DBSCAN(min_samples=30, metric='precomputed'))
+    ...     DBSCAN(eps=25.0, metric='precomputed'))
+    >>> X_clustered = estimator.fit_predict(X)
+    >>> clusters, counts = np.unique(X_clustered, return_counts=True)
+    >>> print(counts)
+    [ 29  15 111  11  12]
     """
-    @_deprecate_positional_args
-    def __init__(self, *, mode='distance', radius=1., algorithm='auto',
-                 leaf_size=30, metric='minkowski', p=2, metric_params=None,
-                 n_jobs=1):
-        super(RadiusNeighborsTransformer, self).__init__(
-            n_neighbors=None, radius=radius, algorithm=algorithm,
-            leaf_size=leaf_size, metric=metric, p=p,
-            metric_params=metric_params, n_jobs=n_jobs)
+
+    _parameter_constraints: dict = {
+        **NeighborsBase._parameter_constraints,
+        "mode": [StrOptions({"distance", "connectivity"})],
+    }
+    _parameter_constraints.pop("n_neighbors")
+
+    def __init__(
+        self,
+        *,
+        mode="distance",
+        radius=1.0,
+        algorithm="auto",
+        leaf_size=30,
+        metric="minkowski",
+        p=2,
+        metric_params=None,
+        n_jobs=None,
+    ):
+        super().__init__(
+            n_neighbors=None,
+            radius=radius,
+            algorithm=algorithm,
+            leaf_size=leaf_size,
+            metric=metric,
+            p=p,
+            metric_params=metric_params,
+            n_jobs=n_jobs,
+        )
         self.mode = mode
 
+    @_fit_context(
+        # RadiusNeighborsTransformer.metric is not validated yet
+        prefer_skip_nested_validation=False
+    )
     def fit(self, X, y=None):
         """Fit the radius neighbors transformer from the training dataset.
 
@@ -502,20 +648,25 @@ def fit(self, X, y=None):
                 (n_samples, n_samples) if metric='precomputed'
             Training data.
 
+        y : Ignored
+            Not used, present for API consistency by convention.
+
         Returns
         -------
         self : RadiusNeighborsTransformer
             The fitted radius neighbors transformer.
         """
-        return self._fit(X)
+        self._fit(X)
+        self._n_features_out = self.n_samples_fit_
+        return self
 
     def transform(self, X):
-        """Computes the (weighted) graph of Neighbors for points in X
+        """Compute the (weighted) graph of Neighbors for points in X.
 
         Parameters
         ----------
         X : array-like of shape (n_samples_transform, n_features)
-            Sample data
+            Sample data.
 
         Returns
         -------
@@ -526,8 +677,7 @@ def transform(self, X):
             The matrix is of CSR format.
         """
         check_is_fitted(self)
-        return self.radius_neighbors_graph(X, mode=self.mode,
-                                           sort_results=True)
+        return self.radius_neighbors_graph(X, mode=self.mode, sort_results=True)
 
     def fit_transform(self, X, y=None):
         """Fit to data, then transform it.
@@ -540,7 +690,8 @@ def fit_transform(self, X, y=None):
         X : array-like of shape (n_samples, n_features)
             Training set.
 
-        y : ignored
+        y : Ignored
+            Not used, present for API consistency by convention.
 
         Returns
         -------
@@ -551,11 +702,3 @@ def fit_transform(self, X, y=None):
             The matrix is of CSR format.
         """
         return self.fit(X).transform(X)
-
-    def _more_tags(self):
-        return {
-            '_xfail_checks': {
-                'check_methods_sample_order_invariance':
-                'check is not applicable.'
-            }
-        }
diff --git a/sklearn/neighbors/_kd_tree.pyx b/sklearn/neighbors/_kd_tree.pyx.tp
similarity index 55%
rename from sklearn/neighbors/_kd_tree.pyx
rename to sklearn/neighbors/_kd_tree.pyx.tp
index bc1ab764a6fcf..d21af05270b9a 100644
--- a/sklearn/neighbors/_kd_tree.pyx
+++ b/sklearn/neighbors/_kd_tree.pyx.tp
@@ -1,29 +1,54 @@
-#!python
-#cython: boundscheck=False
-#cython: wraparound=False
-#cython: cdivision=True
+{{py:
+
+# Generated file: _kd_tree.pyx
+
+implementation_specific_values = [
+    # The values are arranged as follows:
+    #
+    #       name_suffix, INPUT_DTYPE_t, INPUT_DTYPE
+    #
+    ('64', 'float64_t', 'np.float64'),
+    ('32', 'float32_t', 'np.float32')
+]
 
 # By Jake Vanderplas (2013) 
 # written for the scikit-learn project
-# License: BSD
+# SPDX-License-Identifier: BSD-3-Clause
+
+}}
+
 
-__all__ = ['KDTree']
+__all__ = ['KDTree', 'KDTree64', 'KDTree32']
 
-DOC_DICT = {'BinaryTree': 'KDTree', 'binary_tree': 'kd_tree'}
+{{for name_suffix, INPUT_DTYPE_t, INPUT_DTYPE in implementation_specific_values}}
 
-VALID_METRICS = ['EuclideanDistance', 'ManhattanDistance',
-                 'ChebyshevDistance', 'MinkowskiDistance']
+DOC_DICT{{name_suffix}} = {
+    'BinaryTree': 'KDTree{{name_suffix}}',
+    'binary_tree': 'kd_tree{{name_suffix}}',
+}
 
+VALID_METRICS{{name_suffix}} = [
+    'EuclideanDistance{{name_suffix}}',
+    'ManhattanDistance{{name_suffix}}',
+    'ChebyshevDistance{{name_suffix}}',
+    'MinkowskiDistance{{name_suffix}}'
+]
+
+{{endfor}}
 
 include "_binary_tree.pxi"
 
-# Inherit KDTree from BinaryTree
-cdef class KDTree(BinaryTree):
-    __doc__ = CLASS_DOC.format(**DOC_DICT)
+{{for name_suffix, INPUT_DTYPE_t, INPUT_DTYPE in implementation_specific_values}}
+
+# Inherit KDTree{{name_suffix}} from BinaryTree{{name_suffix}}
+cdef class KDTree{{name_suffix}}(BinaryTree{{name_suffix}}):
+    __doc__ = CLASS_DOC.format(**DOC_DICT{{name_suffix}})
     pass
 
+{{endfor}}
+
 
-#----------------------------------------------------------------------
+# ----------------------------------------------------------------------
 # The functions below specialized the Binary Tree as a KD Tree
 #
 #   Note that these functions use the concept of "reduced distance".
@@ -33,28 +58,36 @@ cdef class KDTree(BinaryTree):
 #   distance for the Euclidean metric is the squared-euclidean distance.
 #   For some metrics, the reduced distance is simply the distance.
 
+{{for name_suffix, INPUT_DTYPE_t, INPUT_DTYPE in implementation_specific_values}}
 
-cdef int allocate_data(BinaryTree tree, ITYPE_t n_nodes,
-                       ITYPE_t n_features) except -1:
+cdef int allocate_data{{name_suffix}}(
+    BinaryTree{{name_suffix}} tree,
+    intp_t n_nodes,
+    intp_t n_features,
+) except -1:
     """Allocate arrays needed for the KD Tree"""
-    tree.node_bounds_arr = np.zeros((2, n_nodes, n_features), dtype=DTYPE)
-    tree.node_bounds = get_memview_DTYPE_3D(tree.node_bounds_arr)
+    tree.node_bounds = np.zeros((2, n_nodes, n_features), dtype={{INPUT_DTYPE}})
     return 0
 
 
-cdef int init_node(BinaryTree tree, ITYPE_t i_node,
-                   ITYPE_t idx_start, ITYPE_t idx_end) except -1:
+cdef int init_node{{name_suffix}}(
+    BinaryTree{{name_suffix}} tree,
+    NodeData_t[::1] node_data,
+    intp_t i_node,
+    intp_t idx_start,
+    intp_t idx_end,
+) except -1:
     """Initialize the node for the dataset stored in tree.data"""
-    cdef ITYPE_t n_features = tree.data.shape[1]
-    cdef ITYPE_t i, j
-    cdef DTYPE_t rad = 0
+    cdef intp_t n_features = tree.data.shape[1]
+    cdef intp_t i, j
+    cdef float64_t rad = 0
 
-    cdef DTYPE_t* lower_bounds = &tree.node_bounds[0, i_node, 0]
-    cdef DTYPE_t* upper_bounds = &tree.node_bounds[1, i_node, 0]
-    cdef DTYPE_t* data = &tree.data[0, 0]
-    cdef ITYPE_t* idx_array = &tree.idx_array[0]
+    cdef {{INPUT_DTYPE_t}}* lower_bounds = &tree.node_bounds[0, i_node, 0]
+    cdef {{INPUT_DTYPE_t}}* upper_bounds = &tree.node_bounds[1, i_node, 0]
+    cdef const {{INPUT_DTYPE_t}}* data = &tree.data[0, 0]
+    cdef const intp_t* idx_array = &tree.idx_array[0]
 
-    cdef DTYPE_t* data_row
+    cdef const {{INPUT_DTYPE_t}}* data_row
 
     # determine Node bounds
     for j in range(n_features):
@@ -77,22 +110,25 @@ cdef int init_node(BinaryTree tree, ITYPE_t i_node,
             rad += pow(0.5 * abs(upper_bounds[j] - lower_bounds[j]),
                        tree.dist_metric.p)
 
-    tree.node_data[i_node].idx_start = idx_start
-    tree.node_data[i_node].idx_end = idx_end
+    node_data[i_node].idx_start = idx_start
+    node_data[i_node].idx_end = idx_end
 
     # The radius will hold the size of the circumscribed hypersphere measured
     # with the specified metric: in querying, this is used as a measure of the
     # size of each node when deciding which nodes to split.
-    tree.node_data[i_node].radius = pow(rad, 1. / tree.dist_metric.p)
+    node_data[i_node].radius = pow(rad, 1. / tree.dist_metric.p)
     return 0
 
 
-cdef DTYPE_t min_rdist(BinaryTree tree, ITYPE_t i_node,
-                       DTYPE_t* pt) nogil except -1:
+cdef float64_t min_rdist{{name_suffix}}(
+    BinaryTree{{name_suffix}} tree,
+    intp_t i_node,
+    const {{INPUT_DTYPE_t}}* pt,
+) except -1 nogil:
     """Compute the minimum reduced-distance between a point and a node"""
-    cdef ITYPE_t n_features = tree.data.shape[1]
-    cdef DTYPE_t d, d_lo, d_hi, rdist=0.0
-    cdef ITYPE_t j
+    cdef intp_t n_features = tree.data.shape[1]
+    cdef float64_t d, d_lo, d_hi, rdist=0.0
+    cdef intp_t j
 
     if tree.dist_metric.p == INF:
         for j in range(n_features):
@@ -111,21 +147,31 @@ cdef DTYPE_t min_rdist(BinaryTree tree, ITYPE_t i_node,
     return rdist
 
 
-cdef DTYPE_t min_dist(BinaryTree tree, ITYPE_t i_node, DTYPE_t* pt) except -1:
+cdef float64_t min_dist{{name_suffix}}(
+    BinaryTree{{name_suffix}} tree,
+    intp_t i_node,
+    const {{INPUT_DTYPE_t}}* pt,
+) except -1:
     """Compute the minimum distance between a point and a node"""
     if tree.dist_metric.p == INF:
-        return min_rdist(tree, i_node, pt)
+        return min_rdist{{name_suffix}}(tree, i_node, pt)
     else:
-        return pow(min_rdist(tree, i_node, pt), 1. / tree.dist_metric.p)
+        return pow(
+            min_rdist{{name_suffix}}(tree, i_node, pt),
+            1. / tree.dist_metric.p
+        )
 
 
-cdef DTYPE_t max_rdist(BinaryTree tree,
-                       ITYPE_t i_node, DTYPE_t* pt) except -1:
+cdef float64_t max_rdist{{name_suffix}}(
+    BinaryTree{{name_suffix}} tree,
+    intp_t i_node,
+    const {{INPUT_DTYPE_t}}* pt,
+) except -1:
     """Compute the maximum reduced-distance between a point and a node"""
-    cdef ITYPE_t n_features = tree.data.shape[1]
+    cdef intp_t n_features = tree.data.shape[1]
 
-    cdef DTYPE_t d, d_lo, d_hi, rdist=0.0
-    cdef ITYPE_t j
+    cdef float64_t d_lo, d_hi, rdist=0.0
+    cdef intp_t j
 
     if tree.dist_metric.p == INF:
         for j in range(n_features):
@@ -140,21 +186,33 @@ cdef DTYPE_t max_rdist(BinaryTree tree,
     return rdist
 
 
-cdef DTYPE_t max_dist(BinaryTree tree, ITYPE_t i_node, DTYPE_t* pt) except -1:
+cdef float64_t max_dist{{name_suffix}}(
+    BinaryTree{{name_suffix}} tree,
+    intp_t i_node,
+    const {{INPUT_DTYPE_t}}* pt,
+) except -1:
     """Compute the maximum distance between a point and a node"""
     if tree.dist_metric.p == INF:
-        return max_rdist(tree, i_node, pt)
+        return max_rdist{{name_suffix}}(tree, i_node, pt)
     else:
-        return pow(max_rdist(tree, i_node, pt), 1. / tree.dist_metric.p)
-
-
-cdef inline int min_max_dist(BinaryTree tree, ITYPE_t i_node, DTYPE_t* pt,
-                             DTYPE_t* min_dist, DTYPE_t* max_dist) nogil except -1:
+        return pow(
+            max_rdist{{name_suffix}}(tree, i_node, pt),
+            1. / tree.dist_metric.p
+        )
+
+
+cdef inline int min_max_dist{{name_suffix}}(
+    BinaryTree{{name_suffix}} tree,
+    intp_t i_node,
+    const {{INPUT_DTYPE_t}}* pt,
+    float64_t* min_dist,
+    float64_t* max_dist,
+) except -1 nogil:
     """Compute the minimum and maximum distance between a point and a node"""
-    cdef ITYPE_t n_features = tree.data.shape[1]
+    cdef intp_t n_features = tree.data.shape[1]
 
-    cdef DTYPE_t d, d_lo, d_hi
-    cdef ITYPE_t j
+    cdef float64_t d, d_lo, d_hi
+    cdef intp_t j
 
     min_dist[0] = 0.0
     max_dist[0] = 0.0
@@ -165,10 +223,8 @@ cdef inline int min_max_dist(BinaryTree tree, ITYPE_t i_node, DTYPE_t* pt,
             d_hi = pt[j] - tree.node_bounds[1, i_node, j]
             d = (d_lo + fabs(d_lo)) + (d_hi + fabs(d_hi))
             min_dist[0] = fmax(min_dist[0], 0.5 * d)
-            max_dist[0] = fmax(max_dist[0],
-                               fabs(pt[j] - tree.node_bounds[0, i_node, j]))
-            max_dist[0] = fmax(max_dist[0],
-                               fabs(pt[j] - tree.node_bounds[1, i_node, j]))
+            max_dist[0] = fmax(max_dist[0], fabs(d_lo))
+            max_dist[0] = fmax(max_dist[0], fabs(d_hi))
     else:
         # as above, use the fact that x + abs(x) = 2 * max(x, 0)
         for j in range(n_features):
@@ -185,14 +241,17 @@ cdef inline int min_max_dist(BinaryTree tree, ITYPE_t i_node, DTYPE_t* pt,
     return 0
 
 
-cdef inline DTYPE_t min_rdist_dual(BinaryTree tree1, ITYPE_t i_node1,
-                                   BinaryTree tree2, ITYPE_t i_node2) except -1:
+cdef inline float64_t min_rdist_dual{{name_suffix}}(
+    BinaryTree{{name_suffix}} tree1,
+    intp_t i_node1,
+    BinaryTree{{name_suffix}} tree2,
+    intp_t i_node2,
+) except -1:
     """Compute the minimum reduced distance between two nodes"""
-    cdef ITYPE_t n_features = tree1.data.shape[1]
+    cdef intp_t n_features = tree1.data.shape[1]
 
-    cdef DTYPE_t d, d1, d2, rdist=0.0
-    cdef DTYPE_t zero = 0.0
-    cdef ITYPE_t j
+    cdef float64_t d, d1, d2, rdist=0.0
+    cdef intp_t j
 
     if tree1.dist_metric.p == INF:
         for j in range(n_features):
@@ -217,21 +276,29 @@ cdef inline DTYPE_t min_rdist_dual(BinaryTree tree1, ITYPE_t i_node1,
     return rdist
 
 
-cdef inline DTYPE_t min_dist_dual(BinaryTree tree1, ITYPE_t i_node1,
-                                  BinaryTree tree2, ITYPE_t i_node2) except -1:
+cdef inline float64_t min_dist_dual{{name_suffix}}(
+    BinaryTree{{name_suffix}} tree1,
+    intp_t i_node1,
+    BinaryTree{{name_suffix}} tree2,
+    intp_t i_node2,
+) except -1:
     """Compute the minimum distance between two nodes"""
-    return tree1.dist_metric._rdist_to_dist(min_rdist_dual(tree1, i_node1,
-                                                           tree2, i_node2))
+    return tree1.dist_metric._rdist_to_dist(
+        min_rdist_dual{{name_suffix}}(tree1, i_node1, tree2, i_node2)
+    )
 
 
-cdef inline DTYPE_t max_rdist_dual(BinaryTree tree1, ITYPE_t i_node1,
-                                   BinaryTree tree2, ITYPE_t i_node2) except -1:
+cdef inline float64_t max_rdist_dual{{name_suffix}}(
+    BinaryTree{{name_suffix}} tree1,
+    intp_t i_node1,
+    BinaryTree{{name_suffix}} tree2,
+    intp_t i_node2,
+) except -1:
     """Compute the maximum reduced distance between two nodes"""
-    cdef ITYPE_t n_features = tree1.data.shape[1]
+    cdef intp_t n_features = tree1.data.shape[1]
 
-    cdef DTYPE_t d, d1, d2, rdist=0.0
-    cdef DTYPE_t zero = 0.0
-    cdef ITYPE_t j
+    cdef float64_t d1, d2, rdist=0.0
+    cdef intp_t j
 
     if tree1.dist_metric.p == INF:
         for j in range(n_features):
@@ -250,8 +317,20 @@ cdef inline DTYPE_t max_rdist_dual(BinaryTree tree1, ITYPE_t i_node1,
     return rdist
 
 
-cdef inline DTYPE_t max_dist_dual(BinaryTree tree1, ITYPE_t i_node1,
-                                  BinaryTree tree2, ITYPE_t i_node2) except -1:
+cdef inline float64_t max_dist_dual{{name_suffix}}(
+    BinaryTree{{name_suffix}} tree1,
+    intp_t i_node1,
+    BinaryTree{{name_suffix}} tree2,
+    intp_t i_node2,
+) except -1:
     """Compute the maximum distance between two nodes"""
-    return tree1.dist_metric._rdist_to_dist(max_rdist_dual(tree1, i_node1,
-                                                           tree2, i_node2))
+    return tree1.dist_metric._rdist_to_dist(
+        max_rdist_dual{{name_suffix}}(tree1, i_node1, tree2, i_node2)
+    )
+
+{{endfor}}
+
+
+class KDTree(KDTree64):
+    __doc__ = CLASS_DOC.format(BinaryTree="KDTree")
+    pass
diff --git a/sklearn/neighbors/_kde.py b/sklearn/neighbors/_kde.py
index 57f80f83762fb..7661308db2e01 100644
--- a/sklearn/neighbors/_kde.py
+++ b/sklearn/neighbors/_kde.py
@@ -2,27 +2,38 @@
 Kernel Density Estimation
 -------------------------
 """
-# Author: Jake Vanderplas 
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+import itertools
+from numbers import Integral, Real
 
 import numpy as np
 from scipy.special import gammainc
-from ..base import BaseEstimator
-from ..utils import check_random_state
-from ..utils.validation import _check_sample_weight, check_is_fitted
-from ..utils.validation import _deprecate_positional_args
 
+from ..base import BaseEstimator, _fit_context
+from ..neighbors._base import VALID_METRICS
+from ..utils import check_random_state
+from ..utils._param_validation import Interval, StrOptions
 from ..utils.extmath import row_norms
-from ._ball_tree import BallTree, DTYPE
+from ..utils.validation import _check_sample_weight, check_is_fitted, validate_data
+from ._ball_tree import BallTree
 from ._kd_tree import KDTree
 
+VALID_KERNELS = [
+    "gaussian",
+    "tophat",
+    "epanechnikov",
+    "exponential",
+    "linear",
+    "cosine",
+]
 
-VALID_KERNELS = ['gaussian', 'tophat', 'epanechnikov', 'exponential', 'linear',
-                 'cosine']
-TREE_DICT = {'ball_tree': BallTree, 'kd_tree': KDTree}
+TREE_DICT = {"ball_tree": BallTree, "kd_tree": KDTree}
 
 
 # TODO: implement a brute force version for testing purposes
-# TODO: bandwidth estimation
 # TODO: create a density estimation base class?
 class KernelDensity(BaseEstimator):
     """Kernel Density Estimation.
@@ -31,8 +42,10 @@ class KernelDensity(BaseEstimator):
 
     Parameters
     ----------
-    bandwidth : float, default=1.0
-        The bandwidth of the kernel.
+    bandwidth : float or {"scott", "silverman"}, default=1.0
+        The bandwidth of the kernel. If bandwidth is a float, it defines the
+        bandwidth of the kernel. If bandwidth is a string, one of the estimation
+        methods is implemented.
 
     algorithm : {'kd_tree', 'ball_tree', 'auto'}, default='auto'
         The tree algorithm to use.
@@ -41,13 +54,18 @@ class KernelDensity(BaseEstimator):
                  'cosine'}, default='gaussian'
         The kernel to use.
 
-    metric : str, default='euclidian'
-        The distance metric to use.  Note that not all metrics are
-        valid with all algorithms.  Refer to the documentation of
-        :class:`BallTree` and :class:`KDTree` for a description of
-        available algorithms.  Note that the normalization of the density
-        output is correct only for the Euclidean distance metric. Default
-        is 'euclidean'.
+    metric : str, default='euclidean'
+        Metric to use for distance computation. See the
+        documentation of `scipy.spatial.distance
+        `_ and
+        the metrics listed in
+        :class:`~sklearn.metrics.pairwise.distance_metrics` for valid metric
+        values.
+
+        Not all metrics are valid with all algorithms: refer to the
+        documentation of :class:`BallTree` and :class:`KDTree`. Note that the
+        normalization of the density output is correct only for the Euclidean
+        distance metric.
 
     atol : float, default=0
         The desired absolute tolerance of the result.  A larger tolerance will
@@ -72,9 +90,24 @@ class KernelDensity(BaseEstimator):
 
     Attributes
     ----------
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
     tree_ : ``BinaryTree`` instance
         The tree algorithm for fast generalized N-point problems.
 
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+    bandwidth_ : float
+        Value of the bandwidth, given directly by the bandwidth parameter or
+        estimated using the 'scott' or 'silverman' method.
+
+        .. versionadded:: 1.0
+
     See Also
     --------
     sklearn.neighbors.KDTree : K-dimensional tree for fast generalized N-point
@@ -86,6 +119,7 @@ class KernelDensity(BaseEstimator):
     --------
     Compute a gaussian kernel density estimate with a fixed bandwidth.
 
+    >>> from sklearn.neighbors import KernelDensity
     >>> import numpy as np
     >>> rng = np.random.RandomState(42)
     >>> X = rng.random_sample((100, 3))
@@ -94,10 +128,39 @@ class KernelDensity(BaseEstimator):
     >>> log_density
     array([-1.52955942, -1.51462041, -1.60244657])
     """
-    @_deprecate_positional_args
-    def __init__(self, *, bandwidth=1.0, algorithm='auto',
-                 kernel='gaussian', metric="euclidean", atol=0, rtol=0,
-                 breadth_first=True, leaf_size=40, metric_params=None):
+
+    _parameter_constraints: dict = {
+        "bandwidth": [
+            Interval(Real, 0, None, closed="neither"),
+            StrOptions({"scott", "silverman"}),
+        ],
+        "algorithm": [StrOptions(set(TREE_DICT.keys()) | {"auto"})],
+        "kernel": [StrOptions(set(VALID_KERNELS))],
+        "metric": [
+            StrOptions(
+                set(itertools.chain(*[VALID_METRICS[alg] for alg in TREE_DICT.keys()]))
+            )
+        ],
+        "atol": [Interval(Real, 0, None, closed="left")],
+        "rtol": [Interval(Real, 0, None, closed="left")],
+        "breadth_first": ["boolean"],
+        "leaf_size": [Interval(Integral, 1, None, closed="left")],
+        "metric_params": [None, dict],
+    }
+
+    def __init__(
+        self,
+        *,
+        bandwidth=1.0,
+        algorithm="auto",
+        kernel="gaussian",
+        metric="euclidean",
+        atol=0,
+        rtol=0,
+        breadth_first=True,
+        leaf_size=40,
+        metric_params=None,
+    ):
         self.algorithm = algorithm
         self.bandwidth = bandwidth
         self.kernel = kernel
@@ -108,36 +171,26 @@ def __init__(self, *, bandwidth=1.0, algorithm='auto',
         self.leaf_size = leaf_size
         self.metric_params = metric_params
 
-        # run the choose algorithm code so that exceptions will happen here
-        # we're using clone() in the GenerativeBayes classifier,
-        # so we can't do this kind of logic in __init__
-        self._choose_algorithm(self.algorithm, self.metric)
-
-        if bandwidth <= 0:
-            raise ValueError("bandwidth must be positive")
-        if kernel not in VALID_KERNELS:
-            raise ValueError("invalid kernel: '{0}'".format(kernel))
-
     def _choose_algorithm(self, algorithm, metric):
         # given the algorithm string + metric string, choose the optimal
         # algorithm to compute the result.
-        if algorithm == 'auto':
+        if algorithm == "auto":
             # use KD Tree if possible
             if metric in KDTree.valid_metrics:
-                return 'kd_tree'
+                return "kd_tree"
             elif metric in BallTree.valid_metrics:
-                return 'ball_tree'
-            else:
-                raise ValueError("invalid metric: '{0}'".format(metric))
-        elif algorithm in TREE_DICT:
+                return "ball_tree"
+        else:  # kd_tree or ball_tree
             if metric not in TREE_DICT[algorithm].valid_metrics:
-                raise ValueError("invalid metric for {0}: "
-                                 "'{1}'".format(TREE_DICT[algorithm],
-                                                metric))
+                raise ValueError(
+                    "invalid metric for {0}: '{1}'".format(TREE_DICT[algorithm], metric)
+                )
             return algorithm
-        else:
-            raise ValueError("invalid algorithm: '{0}'".format(algorithm))
 
+    @_fit_context(
+        # KernelDensity.metric is not validated yet
+        prefer_skip_nested_validation=False
+    )
     def fit(self, X, y=None, sample_weight=None):
         """Fit the Kernel Density model on the data.
 
@@ -159,27 +212,41 @@ def fit(self, X, y=None, sample_weight=None):
         Returns
         -------
         self : object
-            Returns instance of object.
+            Returns the instance itself.
         """
         algorithm = self._choose_algorithm(self.algorithm, self.metric)
-        X = self._validate_data(X, order='C', dtype=DTYPE)
+
+        if isinstance(self.bandwidth, str):
+            if self.bandwidth == "scott":
+                self.bandwidth_ = X.shape[0] ** (-1 / (X.shape[1] + 4))
+            elif self.bandwidth == "silverman":
+                self.bandwidth_ = (X.shape[0] * (X.shape[1] + 2) / 4) ** (
+                    -1 / (X.shape[1] + 4)
+                )
+        else:
+            self.bandwidth_ = self.bandwidth
+
+        X = validate_data(self, X, order="C", dtype=np.float64)
 
         if sample_weight is not None:
-            sample_weight = _check_sample_weight(sample_weight, X, DTYPE)
-            if sample_weight.min() <= 0:
-                raise ValueError("sample_weight must have positive values")
+            sample_weight = _check_sample_weight(
+                sample_weight, X, dtype=np.float64, ensure_non_negative=True
+            )
 
         kwargs = self.metric_params
         if kwargs is None:
             kwargs = {}
-        self.tree_ = TREE_DICT[algorithm](X, metric=self.metric,
-                                          leaf_size=self.leaf_size,
-                                          sample_weight=sample_weight,
-                                          **kwargs)
+        self.tree_ = TREE_DICT[algorithm](
+            X,
+            metric=self.metric,
+            leaf_size=self.leaf_size,
+            sample_weight=sample_weight,
+            **kwargs,
+        )
         return self
 
     def score_samples(self, X):
-        """Evaluate the log density model on the data.
+        """Compute the log-likelihood of each sample under the model.
 
         Parameters
         ----------
@@ -190,7 +257,7 @@ def score_samples(self, X):
         Returns
         -------
         density : ndarray of shape (n_samples,)
-            The array of log(density) evaluations. These are normalized to be
+            Log-likelihood of each sample in `X`. These are normalized to be
             probability densities, so values will be low for high-dimensional
             data.
         """
@@ -198,20 +265,26 @@ def score_samples(self, X):
         # The returned density is normalized to the number of points.
         # For it to be a probability, we must scale it.  For this reason
         # we'll also scale atol.
-        X = self._validate_data(X, order='C', dtype=DTYPE, reset=False)
+        X = validate_data(self, X, order="C", dtype=np.float64, reset=False)
         if self.tree_.sample_weight is None:
             N = self.tree_.data.shape[0]
         else:
             N = self.tree_.sum_weight
         atol_N = self.atol * N
         log_density = self.tree_.kernel_density(
-            X, h=self.bandwidth, kernel=self.kernel, atol=atol_N,
-            rtol=self.rtol, breadth_first=self.breadth_first, return_log=True)
+            X,
+            h=self.bandwidth_,
+            kernel=self.kernel,
+            atol=atol_N,
+            rtol=self.rtol,
+            breadth_first=self.breadth_first,
+            return_log=True,
+        )
         log_density -= np.log(N)
         return log_density
 
     def score(self, X, y=None):
-        """Compute the total log probability density under the model.
+        """Compute the total log-likelihood under the model.
 
         Parameters
         ----------
@@ -246,7 +319,7 @@ def sample(self, n_samples=1, random_state=None):
             Determines random number generation used to generate
             random samples. Pass an int for reproducible results
             across multiple function calls.
-            See :term: `Glossary `.
+            See :term:`Glossary `.
 
         Returns
         -------
@@ -255,7 +328,7 @@ def sample(self, n_samples=1, random_state=None):
         """
         check_is_fitted(self)
         # TODO: implement sampling for other valid kernel shapes
-        if self.kernel not in ['gaussian', 'tophat']:
+        if self.kernel not in ["gaussian", "tophat"]:
             raise NotImplementedError()
 
         data = np.asarray(self.tree_.data)
@@ -268,24 +341,19 @@ def sample(self, n_samples=1, random_state=None):
             cumsum_weight = np.cumsum(np.asarray(self.tree_.sample_weight))
             sum_weight = cumsum_weight[-1]
             i = np.searchsorted(cumsum_weight, u * sum_weight)
-        if self.kernel == 'gaussian':
-            return np.atleast_2d(rng.normal(data[i], self.bandwidth))
+        if self.kernel == "gaussian":
+            return np.atleast_2d(rng.normal(data[i], self.bandwidth_))
 
-        elif self.kernel == 'tophat':
+        elif self.kernel == "tophat":
             # we first draw points from a d-dimensional normal distribution,
             # then use an incomplete gamma function to map them to a uniform
             # d-dimensional tophat distribution.
             dim = data.shape[1]
             X = rng.normal(size=(n_samples, dim))
             s_sq = row_norms(X, squared=True)
-            correction = (gammainc(0.5 * dim, 0.5 * s_sq) ** (1. / dim)
-                          * self.bandwidth / np.sqrt(s_sq))
+            correction = (
+                gammainc(0.5 * dim, 0.5 * s_sq) ** (1.0 / dim)
+                * self.bandwidth_
+                / np.sqrt(s_sq)
+            )
             return data[i] + X * correction[:, np.newaxis]
-
-    def _more_tags(self):
-        return {
-            '_xfail_checks': {
-                'check_sample_weights_invariance':
-                'sample_weight must have positive values',
-            }
-        }
diff --git a/sklearn/neighbors/_lof.py b/sklearn/neighbors/_lof.py
index 29bf1a5e73f91..d9f00be42570e 100644
--- a/sklearn/neighbors/_lof.py
+++ b/sklearn/neighbors/_lof.py
@@ -1,36 +1,34 @@
-# Authors: Nicolas Goix 
-#          Alexandre Gramfort 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-import numpy as np
 import warnings
+from numbers import Real
 
-from ._base import NeighborsBase
-from ._base import KNeighborsMixin
-from ..base import OutlierMixin
+import numpy as np
 
-from ..utils.validation import check_is_fitted
-from ..utils.validation import _deprecate_positional_args
+from ..base import OutlierMixin, _fit_context
 from ..utils import check_array
+from ..utils._param_validation import Interval, StrOptions
+from ..utils.metaestimators import available_if
+from ..utils.validation import check_is_fitted
+from ._base import KNeighborsMixin, NeighborsBase
 
 __all__ = ["LocalOutlierFactor"]
 
 
-class LocalOutlierFactor(KNeighborsMixin,
-                         OutlierMixin,
-                         NeighborsBase):
-    """Unsupervised Outlier Detection using Local Outlier Factor (LOF)
+class LocalOutlierFactor(KNeighborsMixin, OutlierMixin, NeighborsBase):
+    """Unsupervised Outlier Detection using the Local Outlier Factor (LOF).
 
-    The anomaly score of each sample is called Local Outlier Factor.
-    It measures the local deviation of density of a given sample with
-    respect to its neighbors.
+    The anomaly score of each sample is called the Local Outlier Factor.
+    It measures the local deviation of the density of a given sample with respect
+    to its neighbors.
     It is local in that the anomaly score depends on how isolated the object
     is with respect to the surrounding neighborhood.
     More precisely, locality is given by k-nearest neighbors, whose distance
     is used to estimate the local density.
-    By comparing the local density of a sample to the local densities of
-    its neighbors, one can identify samples that have a substantially lower
-    density than their neighbors. These are considered outliers.
+    By comparing the local density of a sample to the local densities of its
+    neighbors, one can identify samples that have a substantially lower density
+    than their neighbors. These are considered outliers.
 
     .. versionadded:: 0.19
 
@@ -54,43 +52,32 @@ class LocalOutlierFactor(KNeighborsMixin,
         this parameter, using brute force.
 
     leaf_size : int, default=30
-        Leaf size passed to :class:`BallTree` or :class:`KDTree`. This can
+        Leaf is size passed to :class:`BallTree` or :class:`KDTree`. This can
         affect the speed of the construction and query, as well as the memory
         required to store the tree. The optimal value depends on the
         nature of the problem.
 
     metric : str or callable, default='minkowski'
-        metric used for the distance computation. Any metric from scikit-learn
-        or scipy.spatial.distance can be used.
+        Metric to use for distance computation. Default is "minkowski", which
+        results in the standard Euclidean distance when p = 2. See the
+        documentation of `scipy.spatial.distance
+        `_ and
+        the metrics listed in
+        :class:`~sklearn.metrics.pairwise.distance_metrics` for valid metric
+        values.
 
         If metric is "precomputed", X is assumed to be a distance matrix and
-        must be square. X may be a sparse matrix, in which case only "nonzero"
-        elements may be considered neighbors.
+        must be square during fit. X may be a :term:`sparse graph`, in which
+        case only "nonzero" elements may be considered neighbors.
 
-        If metric is a callable function, it is called on each
-        pair of instances (rows) and the resulting value recorded. The callable
-        should take two arrays as input and return one value indicating the
-        distance between them. This works for Scipy's metrics, but is less
+        If metric is a callable function, it takes two arrays representing 1D
+        vectors as inputs and must return one value indicating the distance
+        between those vectors. This works for Scipy's metrics, but is less
         efficient than passing the metric name as a string.
 
-        Valid values for metric are:
-
-        - from scikit-learn: ['cityblock', 'cosine', 'euclidean', 'l1', 'l2',
-          'manhattan']
-
-        - from scipy.spatial.distance: ['braycurtis', 'canberra', 'chebyshev',
-          'correlation', 'dice', 'hamming', 'jaccard', 'kulsinski',
-          'mahalanobis', 'minkowski', 'rogerstanimoto', 'russellrao',
-          'seuclidean', 'sokalmichener', 'sokalsneath', 'sqeuclidean',
-          'yule']
-
-        See the documentation for scipy.spatial.distance for details on these
-        metrics:
-        https://docs.scipy.org/doc/scipy/reference/spatial.distance.html
-
-    p : int, default=2
+    p : float, default=2
         Parameter for the Minkowski metric from
-        :func:`sklearn.metrics.pairwise.pairwise_distances`. When p = 1, this
+        :func:`sklearn.metrics.pairwise_distances`. When p = 1, this
         is equivalent to using manhattan_distance (l1), and euclidean_distance
         (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used.
 
@@ -115,7 +102,8 @@ class LocalOutlierFactor(KNeighborsMixin,
         detection (novelty=False). Set novelty to True if you want to use
         LocalOutlierFactor for novelty detection. In this case be aware that
         you should only use predict, decision_function and score_samples
-        on new unseen data and not on the training set.
+        on new unseen data and not on the training set; and note that the
+        results obtained this way may differ from the standard LOF results.
 
         .. versionadded:: 0.20
 
@@ -158,9 +146,30 @@ class LocalOutlierFactor(KNeighborsMixin,
     effective_metric_params_ : dict
         The effective additional keyword arguments for the metric function.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     n_samples_fit_ : int
         It is the number of samples in the fitted data.
 
+    See Also
+    --------
+    sklearn.svm.OneClassSVM: Unsupervised Outlier Detection using
+        Support Vector Machine.
+
+    References
+    ----------
+    .. [1] Breunig, M. M., Kriegel, H. P., Ng, R. T., & Sander, J. (2000, May).
+           LOF: identifying density-based local outliers. In ACM sigmod record.
+
     Examples
     --------
     >>> import numpy as np
@@ -170,28 +179,56 @@ class LocalOutlierFactor(KNeighborsMixin,
     >>> clf.fit_predict(X)
     array([ 1,  1, -1,  1])
     >>> clf.negative_outlier_factor_
-    array([ -0.9821...,  -1.0370..., -73.3697...,  -0.9821...])
-
-    References
-    ----------
-    .. [1] Breunig, M. M., Kriegel, H. P., Ng, R. T., & Sander, J. (2000, May).
-           LOF: identifying density-based local outliers. In ACM sigmod record.
+    array([ -0.9821,  -1.0370, -73.3697,  -0.9821])
     """
-    @_deprecate_positional_args
-    def __init__(self, n_neighbors=20, *, algorithm='auto', leaf_size=30,
-                 metric='minkowski', p=2, metric_params=None,
-                 contamination="auto", novelty=False, n_jobs=None):
+
+    _parameter_constraints: dict = {
+        **NeighborsBase._parameter_constraints,
+        "contamination": [
+            StrOptions({"auto"}),
+            Interval(Real, 0, 0.5, closed="right"),
+        ],
+        "novelty": ["boolean"],
+    }
+    _parameter_constraints.pop("radius")
+
+    def __init__(
+        self,
+        n_neighbors=20,
+        *,
+        algorithm="auto",
+        leaf_size=30,
+        metric="minkowski",
+        p=2,
+        metric_params=None,
+        contamination="auto",
+        novelty=False,
+        n_jobs=None,
+    ):
         super().__init__(
             n_neighbors=n_neighbors,
             algorithm=algorithm,
-            leaf_size=leaf_size, metric=metric, p=p,
-            metric_params=metric_params, n_jobs=n_jobs)
+            leaf_size=leaf_size,
+            metric=metric,
+            p=p,
+            metric_params=metric_params,
+            n_jobs=n_jobs,
+        )
         self.contamination = contamination
         self.novelty = novelty
 
-    @property
-    def fit_predict(self):
-        """Fits the model to the training set X and returns the labels.
+    def _check_novelty_fit_predict(self):
+        if self.novelty:
+            msg = (
+                "fit_predict is not available when novelty=True. Use "
+                "novelty=False if you want to predict on the training set."
+            )
+            raise AttributeError(msg)
+        return True
+
+    @available_if(_check_novelty_fit_predict)
+    def fit_predict(self, X, y=None):
+        """Fit the model to the training set X and return the labels.
 
         **Not available for novelty detection (when novelty is set to True).**
         Label is 1 for an inlier and -1 for an outlier according to the LOF
@@ -199,9 +236,9 @@ def fit_predict(self):
 
         Parameters
         ----------
-        X : array-like of shape (n_samples, n_features), default=None
+        X : {array-like, sparse matrix} of shape (n_samples, n_features), default=None
             The query sample or samples to compute the Local Outlier Factor
-            w.r.t. to the training samples.
+            w.r.t. the training samples.
 
         y : Ignored
             Not used, present for API consistency by convention.
@@ -215,36 +252,12 @@ def fit_predict(self):
         # As fit_predict would be different from fit.predict, fit_predict is
         # only available for outlier detection (novelty=False)
 
-        if self.novelty:
-            msg = ('fit_predict is not available when novelty=True. Use '
-                   'novelty=False if you want to predict on the training set.')
-            raise AttributeError(msg)
-
-        return self._fit_predict
-
-    def _fit_predict(self, X, y=None):
-        """Fits the model to the training set X and returns the labels.
-
-        Label is 1 for an inlier and -1 for an outlier according to the LOF
-        score and the contamination parameter.
-
-        Parameters
-        ----------
-        X : array-like of shape (n_samples, n_features), default=None
-            The query sample or samples to compute the Local Outlier Factor
-            w.r.t. to the training samples.
-
-        Returns
-        -------
-        is_inlier : ndarray of shape (n_samples,)
-            Returns -1 for anomalies/outliers and 1 for inliers.
-        """
-
-        # As fit_predict would be different from fit.predict, fit_predict is
-        # only available for outlier detection (novelty=False)
-
         return self.fit(X)._predict()
 
+    @_fit_context(
+        # LocalOutlierFactor.metric is not validated yet
+        prefer_skip_nested_validation=False
+    )
     def fit(self, X, y=None):
         """Fit the local outlier factor detector from the training dataset.
 
@@ -264,28 +277,34 @@ def fit(self, X, y=None):
         """
         self._fit(X)
 
-        if self.contamination != 'auto':
-            if not(0. < self.contamination <= .5):
-                raise ValueError("contamination must be in (0, 0.5], "
-                                 "got: %f" % self.contamination)
-
         n_samples = self.n_samples_fit_
         if self.n_neighbors > n_samples:
-            warnings.warn("n_neighbors (%s) is greater than the "
-                          "total number of samples (%s). n_neighbors "
-                          "will be set to (n_samples - 1) for estimation."
-                          % (self.n_neighbors, n_samples))
+            warnings.warn(
+                "n_neighbors (%s) is greater than the "
+                "total number of samples (%s). n_neighbors "
+                "will be set to (n_samples - 1) for estimation."
+                % (self.n_neighbors, n_samples)
+            )
         self.n_neighbors_ = max(1, min(self.n_neighbors, n_samples - 1))
 
         self._distances_fit_X_, _neighbors_indices_fit_X_ = self.kneighbors(
-            n_neighbors=self.n_neighbors_)
+            n_neighbors=self.n_neighbors_
+        )
+
+        if self._fit_X.dtype == np.float32:
+            self._distances_fit_X_ = self._distances_fit_X_.astype(
+                self._fit_X.dtype,
+                copy=False,
+            )
 
         self._lrd = self._local_reachability_density(
-            self._distances_fit_X_, _neighbors_indices_fit_X_)
+            self._distances_fit_X_, _neighbors_indices_fit_X_
+        )
 
         # Compute lof score over training samples to define offset_:
-        lrd_ratios_array = (self._lrd[_neighbors_indices_fit_X_] /
-                            self._lrd[:, np.newaxis])
+        lrd_ratios_array = (
+            self._lrd[_neighbors_indices_fit_X_] / self._lrd[:, np.newaxis]
+        )
 
         self.negative_outlier_factor_ = -np.mean(lrd_ratios_array, axis=1)
 
@@ -293,38 +312,53 @@ def fit(self, X, y=None):
             # inliers score around -1 (the higher, the less abnormal).
             self.offset_ = -1.5
         else:
-            self.offset_ = np.percentile(self.negative_outlier_factor_,
-                                         100. * self.contamination)
+            self.offset_ = np.percentile(
+                self.negative_outlier_factor_, 100.0 * self.contamination
+            )
+
+        # Verify if negative_outlier_factor_ values are within acceptable range.
+        # Novelty must also be false to detect outliers
+        if np.min(self.negative_outlier_factor_) < -1e7 and not self.novelty:
+            warnings.warn(
+                "Duplicate values are leading to incorrect results. "
+                "Increase the number of neighbors for more accurate results."
+            )
 
         return self
 
-    @property
-    def predict(self):
+    def _check_novelty_predict(self):
+        if not self.novelty:
+            msg = (
+                "predict is not available when novelty=False, use "
+                "fit_predict if you want to predict on training data. Use "
+                "novelty=True if you want to use LOF for novelty detection "
+                "and predict on new unseen data."
+            )
+            raise AttributeError(msg)
+        return True
+
+    @available_if(_check_novelty_predict)
+    def predict(self, X=None):
         """Predict the labels (1 inlier, -1 outlier) of X according to LOF.
 
         **Only available for novelty detection (when novelty is set to True).**
         This method allows to generalize prediction to *new observations* (not
-        in the training set).
+        in the training set). Note that the result of ``clf.fit(X)`` then
+        ``clf.predict(X)`` with ``novelty=True`` may differ from the result
+        obtained by ``clf.fit_predict(X)`` with ``novelty=False``.
 
         Parameters
         ----------
-        X : array-like of shape (n_samples, n_features)
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
             The query sample or samples to compute the Local Outlier Factor
-            w.r.t. to the training samples.
+            w.r.t. the training samples.
 
         Returns
         -------
         is_inlier : ndarray of shape (n_samples,)
             Returns -1 for anomalies/outliers and +1 for inliers.
         """
-        if not self.novelty:
-            msg = ('predict is not available when novelty=False, use '
-                   'fit_predict if you want to predict on training data. Use '
-                   'novelty=True if you want to use LOF for novelty detection '
-                   'and predict on new unseen data.')
-            raise AttributeError(msg)
-
-        return self._predict
+        return self._predict(X)
 
     def _predict(self, X=None):
         """Predict the labels (1 inlier, -1 outlier) of X according to LOF.
@@ -333,9 +367,9 @@ def _predict(self, X=None):
 
         Parameters
         ----------
-        X : array-like of shape (n_samples, n_features), default=None
+        X : {array-like, sparse matrix} of shape (n_samples, n_features), default=None
             The query sample or samples to compute the Local Outlier Factor
-            w.r.t. to the training samples. If None, makes prediction on the
+            w.r.t. the training samples. If None, makes prediction on the
             training data without considering them as their own neighbors.
 
         Returns
@@ -346,53 +380,30 @@ def _predict(self, X=None):
         check_is_fitted(self)
 
         if X is not None:
-            X = check_array(X, accept_sparse='csr')
-            is_inlier = np.ones(X.shape[0], dtype=int)
-            is_inlier[self.decision_function(X) < 0] = -1
+            shifted_opposite_lof_scores = self.decision_function(X)
+            is_inlier = np.ones(shifted_opposite_lof_scores.shape[0], dtype=int)
+            is_inlier[shifted_opposite_lof_scores < 0] = -1
         else:
             is_inlier = np.ones(self.n_samples_fit_, dtype=int)
             is_inlier[self.negative_outlier_factor_ < self.offset_] = -1
 
         return is_inlier
 
-    @property
-    def decision_function(self):
-        """Shifted opposite of the Local Outlier Factor of X.
-
-        Bigger is better, i.e. large values correspond to inliers.
-
-        **Only available for novelty detection (when novelty is set to True).**
-        The shift offset allows a zero threshold for being an outlier.
-        The argument X is supposed to contain *new data*: if X contains a
-        point from training, it considers the later in its own neighborhood.
-        Also, the samples in X are not considered in the neighborhood of any
-        point.
-
-        Parameters
-        ----------
-        X : array-like of shape (n_samples, n_features)
-            The query sample or samples to compute the Local Outlier Factor
-            w.r.t. the training samples.
-
-        Returns
-        -------
-        shifted_opposite_lof_scores : ndarray of shape (n_samples,)
-            The shifted opposite of the Local Outlier Factor of each input
-            samples. The lower, the more abnormal. Negative scores represent
-            outliers, positive scores represent inliers.
-        """
+    def _check_novelty_decision_function(self):
         if not self.novelty:
-            msg = ('decision_function is not available when novelty=False. '
-                   'Use novelty=True if you want to use LOF for novelty '
-                   'detection and compute decision_function for new unseen '
-                   'data. Note that the opposite LOF of the training samples '
-                   'is always available by considering the '
-                   'negative_outlier_factor_ attribute.')
+            msg = (
+                "decision_function is not available when novelty=False. "
+                "Use novelty=True if you want to use LOF for novelty "
+                "detection and compute decision_function for new unseen "
+                "data. Note that the opposite LOF of the training samples "
+                "is always available by considering the "
+                "negative_outlier_factor_ attribute."
+            )
             raise AttributeError(msg)
+        return True
 
-        return self._decision_function
-
-    def _decision_function(self, X):
+    @available_if(_check_novelty_decision_function)
+    def decision_function(self, X):
         """Shifted opposite of the Local Outlier Factor of X.
 
         Bigger is better, i.e. large values correspond to inliers.
@@ -406,7 +417,7 @@ def _decision_function(self, X):
 
         Parameters
         ----------
-        X : array-like of shape (n_samples, n_features)
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
             The query sample or samples to compute the Local Outlier Factor
             w.r.t. the training samples.
 
@@ -417,47 +428,22 @@ def _decision_function(self, X):
             samples. The lower, the more abnormal. Negative scores represent
             outliers, positive scores represent inliers.
         """
+        return self.score_samples(X) - self.offset_
 
-        return self._score_samples(X) - self.offset_
-
-    @property
-    def score_samples(self):
-        """Opposite of the Local Outlier Factor of X.
-
-        It is the opposite as bigger is better, i.e. large values correspond
-        to inliers.
-
-        **Only available for novelty detection (when novelty is set to True).**
-        The argument X is supposed to contain *new data*: if X contains a
-        point from training, it considers the later in its own neighborhood.
-        Also, the samples in X are not considered in the neighborhood of any
-        point.
-        The score_samples on training data is available by considering the
-        the ``negative_outlier_factor_`` attribute.
-
-        Parameters
-        ----------
-        X : array-like of shape (n_samples, n_features)
-            The query sample or samples to compute the Local Outlier Factor
-            w.r.t. the training samples.
-
-        Returns
-        -------
-        opposite_lof_scores : ndarray of shape (n_samples,)
-            The opposite of the Local Outlier Factor of each input samples.
-            The lower, the more abnormal.
-        """
+    def _check_novelty_score_samples(self):
         if not self.novelty:
-            msg = ('score_samples is not available when novelty=False. The '
-                   'scores of the training samples are always available '
-                   'through the negative_outlier_factor_ attribute. Use '
-                   'novelty=True if you want to use LOF for novelty detection '
-                   'and compute score_samples for new unseen data.')
+            msg = (
+                "score_samples is not available when novelty=False. The "
+                "scores of the training samples are always available "
+                "through the negative_outlier_factor_ attribute. Use "
+                "novelty=True if you want to use LOF for novelty detection "
+                "and compute score_samples for new unseen data."
+            )
             raise AttributeError(msg)
+        return True
 
-        return self._score_samples
-
-    def _score_samples(self, X):
+    @available_if(_check_novelty_score_samples)
+    def score_samples(self, X):
         """Opposite of the Local Outlier Factor of X.
 
         It is the opposite as bigger is better, i.e. large values correspond
@@ -467,13 +453,14 @@ def _score_samples(self, X):
         The argument X is supposed to contain *new data*: if X contains a
         point from training, it considers the later in its own neighborhood.
         Also, the samples in X are not considered in the neighborhood of any
-        point.
-        The score_samples on training data is available by considering the
-        the ``negative_outlier_factor_`` attribute.
+        point. Because of this, the scores obtained via ``score_samples`` may
+        differ from the standard LOF scores.
+        The standard LOF scores for the training data is available via the
+        ``negative_outlier_factor_`` attribute.
 
         Parameters
         ----------
-        X : array-like of shape (n_samples, n_features)
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
             The query sample or samples to compute the Local Outlier Factor
             w.r.t. the training samples.
 
@@ -484,15 +471,21 @@ def _score_samples(self, X):
             The lower, the more abnormal.
         """
         check_is_fitted(self)
-        X = check_array(X, accept_sparse='csr')
+        X = check_array(X, accept_sparse="csr")
+
+        distances_X, neighbors_indices_X = self.kneighbors(
+            X, n_neighbors=self.n_neighbors_
+        )
+
+        if X.dtype == np.float32:
+            distances_X = distances_X.astype(X.dtype, copy=False)
 
-        distances_X, neighbors_indices_X = (
-            self.kneighbors(X, n_neighbors=self.n_neighbors_))
-        X_lrd = self._local_reachability_density(distances_X,
-                                                 neighbors_indices_X)
+        X_lrd = self._local_reachability_density(
+            distances_X,
+            neighbors_indices_X,
+        )
 
-        lrd_ratios_array = (self._lrd[neighbors_indices_X] /
-                            X_lrd[:, np.newaxis])
+        lrd_ratios_array = self._lrd[neighbors_indices_X] / X_lrd[:, np.newaxis]
 
         # as bigger is better:
         return -np.mean(lrd_ratios_array, axis=1)
@@ -518,9 +511,8 @@ def _local_reachability_density(self, distances_X, neighbors_indices):
         local_reachability_density : ndarray of shape (n_queries,)
             The local reachability density of each sample.
         """
-        dist_k = self._distances_fit_X_[neighbors_indices,
-                                        self.n_neighbors_ - 1]
+        dist_k = self._distances_fit_X_[neighbors_indices, self.n_neighbors_ - 1]
         reach_dist_array = np.maximum(distances_X, dist_k)
 
         # 1e-10 to avoid `nan' when nb of duplicates > n_neighbors_:
-        return 1. / (np.mean(reach_dist_array, axis=1) + 1e-10)
+        return 1.0 / (np.mean(reach_dist_array, axis=1) + 1e-10)
diff --git a/sklearn/neighbors/_nca.py b/sklearn/neighbors/_nca.py
index a4ef02b687d97..a4ef3c303b851 100644
--- a/sklearn/neighbors/_nca.py
+++ b/sklearn/neighbors/_nca.py
@@ -1,34 +1,39 @@
-# coding: utf-8
 """
 Neighborhood Component Analysis
 """
 
-# Authors: William de Vazelhes 
-#          John Chiotellis 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-from __future__ import print_function
-
-from warnings import warn
-import numpy as np
 import sys
 import time
-import numbers
+from numbers import Integral, Real
+from warnings import warn
+
+import numpy as np
 from scipy.optimize import minimize
-from ..utils.extmath import softmax
+
+from ..base import (
+    BaseEstimator,
+    ClassNamePrefixFeaturesOutMixin,
+    TransformerMixin,
+    _fit_context,
+)
+from ..decomposition import PCA
+from ..exceptions import ConvergenceWarning
 from ..metrics import pairwise_distances
-from ..base import BaseEstimator, TransformerMixin
 from ..preprocessing import LabelEncoder
-from ..decomposition import PCA
+from ..utils._param_validation import Interval, StrOptions
+from ..utils.extmath import softmax
 from ..utils.multiclass import check_classification_targets
 from ..utils.random import check_random_state
-from ..utils.validation import check_is_fitted, check_array, check_scalar
-from ..utils.validation import _deprecate_positional_args
-from ..exceptions import ConvergenceWarning
+from ..utils.validation import check_array, check_is_fitted, validate_data
 
 
-class NeighborhoodComponentsAnalysis(TransformerMixin, BaseEstimator):
-    """Neighborhood Components Analysis
+class NeighborhoodComponentsAnalysis(
+    ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator
+):
+    """Neighborhood Components Analysis.
 
     Neighborhood Component Analysis (NCA) is a machine learning algorithm for
     metric learning. It learns a linear transformation in a supervised fashion
@@ -41,53 +46,53 @@ class NeighborhoodComponentsAnalysis(TransformerMixin, BaseEstimator):
     ----------
     n_components : int, default=None
         Preferred dimensionality of the projected space.
-        If None it will be set to ``n_features``.
+        If None it will be set to `n_features`.
 
     init : {'auto', 'pca', 'lda', 'identity', 'random'} or ndarray of shape \
             (n_features_a, n_features_b), default='auto'
         Initialization of the linear transformation. Possible options are
-        'auto', 'pca', 'lda', 'identity', 'random', and a numpy array of shape
-        (n_features_a, n_features_b).
-
-        'auto'
-            Depending on ``n_components``, the most reasonable initialization
-            will be chosen. If ``n_components <= n_classes`` we use 'lda', as
-            it uses labels information. If not, but
-            ``n_components < min(n_features, n_samples)``, we use 'pca', as
+        `'auto'`, `'pca'`, `'lda'`, `'identity'`, `'random'`, and a numpy
+        array of shape `(n_features_a, n_features_b)`.
+
+        - `'auto'`
+            Depending on `n_components`, the most reasonable initialization
+            is chosen. If `n_components <= min(n_features, n_classes - 1)`
+            we use `'lda'`, as it uses labels information. If not, but
+            `n_components < min(n_features, n_samples)`, we use `'pca'`, as
             it projects data in meaningful directions (those of higher
-            variance). Otherwise, we just use 'identity'.
+            variance). Otherwise, we just use `'identity'`.
 
-        'pca'
-            ``n_components`` principal components of the inputs passed
+        - `'pca'`
+            `n_components` principal components of the inputs passed
             to :meth:`fit` will be used to initialize the transformation.
             (See :class:`~sklearn.decomposition.PCA`)
 
-        'lda'
-            ``min(n_components, n_classes)`` most discriminative
+        - `'lda'`
+            `min(n_components, n_classes)` most discriminative
             components of the inputs passed to :meth:`fit` will be used to
-            initialize the transformation. (If ``n_components > n_classes``,
+            initialize the transformation. (If `n_components > n_classes`,
             the rest of the components will be zero.) (See
             :class:`~sklearn.discriminant_analysis.LinearDiscriminantAnalysis`)
 
-        'identity'
-            If ``n_components`` is strictly smaller than the
+        - `'identity'`
+            If `n_components` is strictly smaller than the
             dimensionality of the inputs passed to :meth:`fit`, the identity
-            matrix will be truncated to the first ``n_components`` rows.
+            matrix will be truncated to the first `n_components` rows.
 
-        'random'
+        - `'random'`
             The initial transformation will be a random array of shape
             `(n_components, n_features)`. Each value is sampled from the
             standard normal distribution.
 
-        numpy array
-            n_features_b must match the dimensionality of the inputs passed to
-            :meth:`fit` and n_features_a must be less than or equal to that.
-            If ``n_components`` is not None, n_features_a must match it.
+        - numpy array
+            `n_features_b` must match the dimensionality of the inputs passed
+            to :meth:`fit` and n_features_a must be less than or equal to that.
+            If `n_components` is not `None`, `n_features_a` must match it.
 
     warm_start : bool, default=False
-        If True and :meth:`fit` has been called before, the solution of the
+        If `True` and :meth:`fit` has been called before, the solution of the
         previous call to :meth:`fit` is used as the initial linear
-        transformation (``n_components`` and ``init`` will be ignored).
+        transformation (`n_components` and `init` will be ignored).
 
     max_iter : int, default=50
         Maximum number of iterations in the optimization.
@@ -96,7 +101,7 @@ class NeighborhoodComponentsAnalysis(TransformerMixin, BaseEstimator):
         Convergence tolerance for the optimization.
 
     callback : callable, default=None
-        If not None, this function is called after every iteration of the
+        If not `None`, this function is called after every iteration of the
         optimizer, taking as arguments the current solution (flattened
         transformation matrix) and the number of iterations. This might be
         useful in case one wants to examine or store the transformation
@@ -105,29 +110,56 @@ class NeighborhoodComponentsAnalysis(TransformerMixin, BaseEstimator):
     verbose : int, default=0
         If 0, no progress messages will be printed.
         If 1, progress messages will be printed to stdout.
-        If > 1, progress messages will be printed and the ``disp``
+        If > 1, progress messages will be printed and the `disp`
         parameter of :func:`scipy.optimize.minimize` will be set to
-        ``verbose - 2``.
+        `verbose - 2`.
 
     random_state : int or numpy.RandomState, default=None
         A pseudo random number generator object or a seed for it if int. If
-        ``init='random'``, ``random_state`` is used to initialize the random
-        transformation. If ``init='pca'``, ``random_state`` is passed as an
+        `init='random'`, `random_state` is used to initialize the random
+        transformation. If `init='pca'`, `random_state` is passed as an
         argument to PCA when initializing the transformation. Pass an int
         for reproducible results across multiple function calls.
-        See :term: `Glossary `.
+        See :term:`Glossary `.
 
     Attributes
     ----------
     components_ : ndarray of shape (n_components, n_features)
         The linear transformation learned during fitting.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
     n_iter_ : int
         Counts the number of iterations performed by the optimizer.
 
     random_state_ : numpy.RandomState
         Pseudo random number generator object used during initialization.
 
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    sklearn.discriminant_analysis.LinearDiscriminantAnalysis : Linear
+        Discriminant Analysis.
+    sklearn.decomposition.PCA : Principal component analysis (PCA).
+
+    References
+    ----------
+    .. [1] J. Goldberger, G. Hinton, S. Roweis, R. Salakhutdinov.
+           "Neighbourhood Components Analysis". Advances in Neural Information
+           Processing Systems. 17, 513-520, 2005.
+           http://www.cs.nyu.edu/~roweis/papers/ncanips.pdf
+
+    .. [2] Wikipedia entry on Neighborhood Components Analysis
+           https://en.wikipedia.org/wiki/Neighbourhood_components_analysis
+
     Examples
     --------
     >>> from sklearn.neighbors import NeighborhoodComponentsAnalysis
@@ -149,23 +181,37 @@ class NeighborhoodComponentsAnalysis(TransformerMixin, BaseEstimator):
     KNeighborsClassifier(...)
     >>> print(knn.score(nca.transform(X_test), y_test))
     0.961904...
-
-    References
-    ----------
-    .. [1] J. Goldberger, G. Hinton, S. Roweis, R. Salakhutdinov.
-           "Neighbourhood Components Analysis". Advances in Neural Information
-           Processing Systems. 17, 513-520, 2005.
-           http://www.cs.nyu.edu/~roweis/papers/ncanips.pdf
-
-    .. [2] Wikipedia entry on Neighborhood Components Analysis
-           https://en.wikipedia.org/wiki/Neighbourhood_components_analysis
-
     """
 
-    @_deprecate_positional_args
-    def __init__(self, n_components=None, *, init='auto', warm_start=False,
-                 max_iter=50, tol=1e-5, callback=None, verbose=0,
-                 random_state=None):
+    _parameter_constraints: dict = {
+        "n_components": [
+            Interval(Integral, 1, None, closed="left"),
+            None,
+        ],
+        "init": [
+            StrOptions({"auto", "pca", "lda", "identity", "random"}),
+            np.ndarray,
+        ],
+        "warm_start": ["boolean"],
+        "max_iter": [Interval(Integral, 1, None, closed="left")],
+        "tol": [Interval(Real, 0, None, closed="left")],
+        "callback": [callable, None],
+        "verbose": ["verbose"],
+        "random_state": ["random_state"],
+    }
+
+    def __init__(
+        self,
+        n_components=None,
+        *,
+        init="auto",
+        warm_start=False,
+        max_iter=50,
+        tol=1e-5,
+        callback=None,
+        verbose=0,
+        random_state=None,
+    ):
         self.n_components = n_components
         self.init = init
         self.warm_start = warm_start
@@ -175,6 +221,7 @@ def __init__(self, n_components=None, *, init='auto', warm_start=False,
         self.verbose = verbose
         self.random_state = random_state
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y):
         """Fit the model according to the given training data.
 
@@ -189,12 +236,59 @@ def fit(self, X, y):
         Returns
         -------
         self : object
-            returns a trained NeighborhoodComponentsAnalysis model.
+            Fitted estimator.
         """
+        # Validate the inputs X and y, and converts y to numerical classes.
+        X, y = validate_data(self, X, y, ensure_min_samples=2)
+        check_classification_targets(y)
+        y = LabelEncoder().fit_transform(y)
 
-        # Verify inputs X and y and NCA parameters, and transform a copy if
-        # needed
-        X, y, init = self._validate_params(X, y)
+        # Check the preferred dimensionality of the projected space
+        if self.n_components is not None and self.n_components > X.shape[1]:
+            raise ValueError(
+                "The preferred dimensionality of the "
+                f"projected space `n_components` ({self.n_components}) cannot "
+                "be greater than the given data "
+                f"dimensionality ({X.shape[1]})!"
+            )
+        # If warm_start is enabled, check that the inputs are consistent
+        if (
+            self.warm_start
+            and hasattr(self, "components_")
+            and self.components_.shape[1] != X.shape[1]
+        ):
+            raise ValueError(
+                f"The new inputs dimensionality ({X.shape[1]}) does not "
+                "match the input dimensionality of the "
+                f"previously learned transformation ({self.components_.shape[1]})."
+            )
+        # Check how the linear transformation should be initialized
+        init = self.init
+        if isinstance(init, np.ndarray):
+            init = check_array(init)
+            # Assert that init.shape[1] = X.shape[1]
+            if init.shape[1] != X.shape[1]:
+                raise ValueError(
+                    f"The input dimensionality ({init.shape[1]}) of the given "
+                    "linear transformation `init` must match the "
+                    f"dimensionality of the given inputs `X` ({X.shape[1]})."
+                )
+            # Assert that init.shape[0] <= init.shape[1]
+            if init.shape[0] > init.shape[1]:
+                raise ValueError(
+                    f"The output dimensionality ({init.shape[0]}) of the given "
+                    "linear transformation `init` cannot be "
+                    f"greater than its input dimensionality ({init.shape[1]})."
+                )
+            # Assert that self.n_components = init.shape[0]
+            if self.n_components is not None and self.n_components != init.shape[0]:
+                raise ValueError(
+                    "The preferred dimensionality of the "
+                    f"projected space `n_components` ({self.n_components}) does"
+                    " not match the output dimensionality of "
+                    "the given linear transformation "
+                    f"`init` ({init.shape[0]})!"
+                )
 
         # Initialize the random generator
         self.random_state_ = check_random_state(self.random_state)
@@ -207,19 +301,20 @@ def fit(self, X, y):
         # (n_samples, n_samples)
 
         # Initialize the transformation
-        transformation = self._initialize(X, y, init)
+        transformation = np.ravel(self._initialize(X, y, init))
 
         # Create a dictionary of parameters to be passed to the optimizer
         disp = self.verbose - 2 if self.verbose > 1 else -1
-        optimizer_params = {'method': 'L-BFGS-B',
-                            'fun': self._loss_grad_lbfgs,
-                            'args': (X, same_class_mask, -1.0),
-                            'jac': True,
-                            'x0': transformation,
-                            'tol': self.tol,
-                            'options': dict(maxiter=self.max_iter, disp=disp),
-                            'callback': self._callback
-                            }
+        optimizer_params = {
+            "method": "L-BFGS-B",
+            "fun": self._loss_grad_lbfgs,
+            "args": (X, same_class_mask, -1.0),
+            "jac": True,
+            "x0": transformation,
+            "tol": self.tol,
+            "options": dict(maxiter=self.max_iter, disp=disp),
+            "callback": self._callback,
+        }
 
         # Call the optimizer
         self.n_iter_ = 0
@@ -235,16 +330,19 @@ def fit(self, X, y):
 
             # Warn the user if the algorithm did not converge
             if not opt_result.success:
-                warn('[{}] NCA did not converge: {}'.format(
-                    cls_name, opt_result.message),
-                     ConvergenceWarning)
+                warn(
+                    "[{}] NCA did not converge: {}".format(
+                        cls_name, opt_result.message
+                    ),
+                    ConvergenceWarning,
+                )
 
-            print('[{}] Training took {:8.2f}s.'.format(cls_name, t_train))
+            print("[{}] Training took {:8.2f}s.".format(cls_name, t_train))
 
         return self
 
     def transform(self, X):
-        """Applies the learned transformation to the given data.
+        """Apply the learned transformation to the given data.
 
         Parameters
         ----------
@@ -263,119 +361,10 @@ def transform(self, X):
         """
 
         check_is_fitted(self)
-        X = self._validate_data(X, reset=False)
+        X = validate_data(self, X, reset=False)
 
         return np.dot(X, self.components_.T)
 
-    def _validate_params(self, X, y):
-        """Validate parameters as soon as :meth:`fit` is called.
-
-        Parameters
-        ----------
-        X : array-like of shape (n_samples, n_features)
-            The training samples.
-
-        y : array-like of shape (n_samples,)
-            The corresponding training labels.
-
-        Returns
-        -------
-        X : ndarray of shape (n_samples, n_features)
-            The validated training samples.
-
-        y : ndarray of shape (n_samples,)
-            The validated training labels, encoded to be integers in
-            the range(0, n_classes).
-
-        init : str or ndarray of shape (n_features_a, n_features_b)
-            The validated initialization of the linear transformation.
-
-        Raises
-        -------
-        TypeError
-            If a parameter is not an instance of the desired type.
-
-        ValueError
-            If a parameter's value violates its legal value range or if the
-            combination of two or more given parameters is incompatible.
-        """
-
-        # Validate the inputs X and y, and converts y to numerical classes.
-        X, y = self._validate_data(X, y, ensure_min_samples=2)
-        check_classification_targets(y)
-        y = LabelEncoder().fit_transform(y)
-
-        # Check the preferred dimensionality of the projected space
-        if self.n_components is not None:
-            check_scalar(
-                self.n_components, 'n_components', numbers.Integral, min_val=1)
-
-            if self.n_components > X.shape[1]:
-                raise ValueError('The preferred dimensionality of the '
-                                 'projected space `n_components` ({}) cannot '
-                                 'be greater than the given data '
-                                 'dimensionality ({})!'
-                                 .format(self.n_components, X.shape[1]))
-
-        # If warm_start is enabled, check that the inputs are consistent
-        check_scalar(self.warm_start, 'warm_start', bool)
-        if self.warm_start and hasattr(self, 'components_'):
-            if self.components_.shape[1] != X.shape[1]:
-                raise ValueError('The new inputs dimensionality ({}) does not '
-                                 'match the input dimensionality of the '
-                                 'previously learned transformation ({}).'
-                                 .format(X.shape[1],
-                                         self.components_.shape[1]))
-
-        check_scalar(self.max_iter, 'max_iter', numbers.Integral, min_val=1)
-        check_scalar(self.tol, 'tol', numbers.Real, min_val=0.)
-        check_scalar(self.verbose, 'verbose', numbers.Integral, min_val=0)
-
-        if self.callback is not None:
-            if not callable(self.callback):
-                raise ValueError('`callback` is not callable.')
-
-        # Check how the linear transformation should be initialized
-        init = self.init
-
-        if isinstance(init, np.ndarray):
-            init = check_array(init)
-
-            # Assert that init.shape[1] = X.shape[1]
-            if init.shape[1] != X.shape[1]:
-                raise ValueError(
-                    'The input dimensionality ({}) of the given '
-                    'linear transformation `init` must match the '
-                    'dimensionality of the given inputs `X` ({}).'
-                    .format(init.shape[1], X.shape[1]))
-
-            # Assert that init.shape[0] <= init.shape[1]
-            if init.shape[0] > init.shape[1]:
-                raise ValueError(
-                    'The output dimensionality ({}) of the given '
-                    'linear transformation `init` cannot be '
-                    'greater than its input dimensionality ({}).'
-                    .format(init.shape[0], init.shape[1]))
-
-            if self.n_components is not None:
-                # Assert that self.n_components = init.shape[0]
-                if self.n_components != init.shape[0]:
-                    raise ValueError('The preferred dimensionality of the '
-                                     'projected space `n_components` ({}) does'
-                                     ' not match the output dimensionality of '
-                                     'the given linear transformation '
-                                     '`init` ({})!'
-                                     .format(self.n_components,
-                                             init.shape[0]))
-        elif init in ['auto', 'pca', 'lda', 'identity', 'random']:
-            pass
-        else:
-            raise ValueError(
-                "`init` must be 'auto', 'pca', 'lda', 'identity', 'random' "
-                "or a numpy array of shape (n_components, n_features).")
-
-        return X, y, init
-
     def _initialize(self, X, y, init):
         """Initialize the transformation.
 
@@ -398,48 +387,49 @@ def _initialize(self, X, y, init):
         """
 
         transformation = init
-        if self.warm_start and hasattr(self, 'components_'):
+        if self.warm_start and hasattr(self, "components_"):
             transformation = self.components_
         elif isinstance(init, np.ndarray):
             pass
         else:
             n_samples, n_features = X.shape
             n_components = self.n_components or n_features
-            if init == 'auto':
+            if init == "auto":
                 n_classes = len(np.unique(y))
                 if n_components <= min(n_features, n_classes - 1):
-                    init = 'lda'
+                    init = "lda"
                 elif n_components < min(n_features, n_samples):
-                    init = 'pca'
+                    init = "pca"
                 else:
-                    init = 'identity'
-            if init == 'identity':
+                    init = "identity"
+            if init == "identity":
                 transformation = np.eye(n_components, X.shape[1])
-            elif init == 'random':
-                transformation = self.random_state_.randn(n_components,
-                                                          X.shape[1])
-            elif init in {'pca', 'lda'}:
+            elif init == "random":
+                transformation = self.random_state_.standard_normal(
+                    size=(n_components, X.shape[1])
+                )
+            elif init in {"pca", "lda"}:
                 init_time = time.time()
-                if init == 'pca':
-                    pca = PCA(n_components=n_components,
-                              random_state=self.random_state_)
+                if init == "pca":
+                    pca = PCA(
+                        n_components=n_components, random_state=self.random_state_
+                    )
                     if self.verbose:
-                        print('Finding principal components... ', end='')
+                        print("Finding principal components... ", end="")
                         sys.stdout.flush()
                     pca.fit(X)
                     transformation = pca.components_
-                elif init == 'lda':
-                    from ..discriminant_analysis import (
-                        LinearDiscriminantAnalysis)
+                elif init == "lda":
+                    from ..discriminant_analysis import LinearDiscriminantAnalysis
+
                     lda = LinearDiscriminantAnalysis(n_components=n_components)
                     if self.verbose:
-                        print('Finding most discriminative components... ',
-                              end='')
+                        print("Finding most discriminative components... ", end="")
                         sys.stdout.flush()
                     lda.fit(X, y)
                     transformation = lda.scalings_.T[:n_components]
                 if self.verbose:
-                    print('done in {:5.2f}s'.format(time.time() - init_time))
+                    print("done in {:5.2f}s".format(time.time() - init_time))
         return transformation
 
     def _callback(self, transformation):
@@ -456,7 +446,7 @@ def _callback(self, transformation):
         self.n_iter_ += 1
 
     def _loss_grad_lbfgs(self, transformation, X, same_class_mask, sign=1.0):
-        """Compute the loss and the loss gradient w.r.t. ``transformation``.
+        """Compute the loss and the loss gradient w.r.t. `transformation`.
 
         Parameters
         ----------
@@ -468,8 +458,8 @@ def _loss_grad_lbfgs(self, transformation, X, same_class_mask, sign=1.0):
             The training samples.
 
         same_class_mask : ndarray of shape (n_samples, n_samples)
-            A mask where ``mask[i, j] == 1`` if ``X[i]`` and ``X[j]`` belong
-            to the same class, and ``0`` otherwise.
+            A mask where `mask[i, j] == 1` if `X[i]` and `X[j]` belong
+            to the same class, and `0` otherwise.
 
         Returns
         -------
@@ -483,13 +473,16 @@ def _loss_grad_lbfgs(self, transformation, X, same_class_mask, sign=1.0):
         if self.n_iter_ == 0:
             self.n_iter_ += 1
             if self.verbose:
-                header_fields = ['Iteration', 'Objective Value', 'Time(s)']
-                header_fmt = '{:>10} {:>20} {:>10}'
+                header_fields = ["Iteration", "Objective Value", "Time(s)"]
+                header_fmt = "{:>10} {:>20} {:>10}"
                 header = header_fmt.format(*header_fields)
                 cls_name = self.__class__.__name__
-                print('[{}]'.format(cls_name))
-                print('[{}] {}\n[{}] {}'.format(cls_name, header,
-                                                cls_name, '-' * len(header)))
+                print("[{}]".format(cls_name))
+                print(
+                    "[{}] {}\n[{}] {}".format(
+                        cls_name, header, cls_name, "-" * len(header)
+                    )
+                )
 
         t_funcall = time.time()
 
@@ -516,12 +509,22 @@ def _loss_grad_lbfgs(self, transformation, X, same_class_mask, sign=1.0):
 
         if self.verbose:
             t_funcall = time.time() - t_funcall
-            values_fmt = '[{}] {:>10} {:>20.6e} {:>10.2f}'
-            print(values_fmt.format(self.__class__.__name__, self.n_iter_,
-                                    loss, t_funcall))
+            values_fmt = "[{}] {:>10} {:>20.6e} {:>10.2f}"
+            print(
+                values_fmt.format(
+                    self.__class__.__name__, self.n_iter_, loss, t_funcall
+                )
+            )
             sys.stdout.flush()
 
         return sign * loss, sign * gradient.ravel()
 
-    def _more_tags(self):
-        return {'requires_y': True}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.target_tags.required = True
+        return tags
+
+    @property
+    def _n_features_out(self):
+        """Number of transformed output features."""
+        return self.components_.shape[0]
diff --git a/sklearn/neighbors/_nearest_centroid.py b/sklearn/neighbors/_nearest_centroid.py
index 0c726cdc0a62c..a780c27587792 100644
--- a/sklearn/neighbors/_nearest_centroid.py
+++ b/sklearn/neighbors/_nearest_centroid.py
@@ -1,27 +1,34 @@
-# -*- coding: utf-8 -*-
 """
 Nearest Centroid Classification
 """
 
-# Author: Robert Layton 
-#         Olivier Grisel 
-#
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import warnings
+from numbers import Real
+
 import numpy as np
 from scipy import sparse as sp
 
-from ..base import BaseEstimator, ClassifierMixin
-from ..metrics.pairwise import pairwise_distances
+from ..base import BaseEstimator, ClassifierMixin, _fit_context
+from ..discriminant_analysis import DiscriminantAnalysisPredictionMixin
+from ..metrics.pairwise import (
+    pairwise_distances,
+    pairwise_distances_argmin,
+)
 from ..preprocessing import LabelEncoder
-from ..utils.validation import check_is_fitted
-from ..utils.validation import _deprecate_positional_args
-from ..utils.sparsefuncs import csc_median_axis_0
+from ..utils import get_tags
+from ..utils._available_if import available_if
+from ..utils._param_validation import Interval, StrOptions
 from ..utils.multiclass import check_classification_targets
+from ..utils.sparsefuncs import csc_median_axis_0
+from ..utils.validation import check_is_fitted, validate_data
 
 
-class NearestCentroid(ClassifierMixin, BaseEstimator):
+class NearestCentroid(
+    DiscriminantAnalysisPredictionMixin, ClassifierMixin, BaseEstimator
+):
     """Nearest centroid classifier.
 
     Each class is represented by its centroid, with test samples classified to
@@ -31,23 +38,31 @@ class NearestCentroid(ClassifierMixin, BaseEstimator):
 
     Parameters
     ----------
-    metric : str or callable
-        The metric to use when calculating distance between instances in a
-        feature array. If metric is a string or callable, it must be one of
-        the options allowed by metrics.pairwise.pairwise_distances for its
-        metric parameter.
-        The centroids for the samples corresponding to each class is the point
-        from which the sum of the distances (according to the metric) of all
-        samples that belong to that particular class are minimized.
-        If the "manhattan" metric is provided, this centroid is the median and
-        for all other metrics, the centroid is now set to be the mean.
+    metric : {"euclidean", "manhattan"}, default="euclidean"
+        Metric to use for distance computation.
+
+        If `metric="euclidean"`, the centroid for the samples corresponding to each
+        class is the arithmetic mean, which minimizes the sum of squared L1 distances.
+        If `metric="manhattan"`, the centroid is the feature-wise median, which
+        minimizes the sum of L1 distances.
+
+        .. versionchanged:: 1.5
+            All metrics but `"euclidean"` and `"manhattan"` were deprecated and
+            now raise an error.
 
         .. versionchanged:: 0.19
-            ``metric='precomputed'`` was deprecated and now raises an error
+            `metric='precomputed'` was deprecated and now raises an error
 
     shrink_threshold : float, default=None
         Threshold for shrinking centroids to remove features.
 
+    priors : {"uniform", "empirical"} or array-like of shape (n_classes,), \
+        default="uniform"
+        The class prior probabilities. By default, the class proportions are
+        inferred from the training data.
+
+        .. versionadded:: 1.6
+
     Attributes
     ----------
     centroids_ : array-like of shape (n_classes, n_features)
@@ -56,17 +71,34 @@ class NearestCentroid(ClassifierMixin, BaseEstimator):
     classes_ : array of shape (n_classes,)
         The unique classes labels.
 
-    Examples
-    --------
-    >>> from sklearn.neighbors import NearestCentroid
-    >>> import numpy as np
-    >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
-    >>> y = np.array([1, 1, 1, 2, 2, 2])
-    >>> clf = NearestCentroid()
-    >>> clf.fit(X, y)
-    NearestCentroid()
-    >>> print(clf.predict([[-0.8, -1]]))
-    [1]
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    deviations_ : ndarray of shape (n_classes, n_features)
+        Deviations (or shrinkages) of the centroids of each class from the
+        overall centroid. Equal to eq. (18.4) if `shrink_threshold=None`,
+        else (18.5) p. 653 of [2]. Can be used to identify features used
+        for classification.
+
+        .. versionadded:: 1.6
+
+    within_class_std_dev_ : ndarray of shape (n_features,)
+        Pooled or within-class standard deviation of input data.
+
+        .. versionadded:: 1.6
+
+    class_prior_ : ndarray of shape (n_classes,)
+        The class prior probabilities.
+
+        .. versionadded:: 1.6
 
     See Also
     --------
@@ -79,18 +111,45 @@ class NearestCentroid(ClassifierMixin, BaseEstimator):
 
     References
     ----------
-    Tibshirani, R., Hastie, T., Narasimhan, B., & Chu, G. (2002). Diagnosis of
+    [1] Tibshirani, R., Hastie, T., Narasimhan, B., & Chu, G. (2002). Diagnosis of
     multiple cancer types by shrunken centroids of gene expression. Proceedings
     of the National Academy of Sciences of the United States of America,
     99(10), 6567-6572. The National Academy of Sciences.
 
+    [2] Hastie, T., Tibshirani, R., Friedman, J. (2009). The Elements of Statistical
+    Learning Data Mining, Inference, and Prediction. 2nd Edition. New York, Springer.
+
+    Examples
+    --------
+    >>> from sklearn.neighbors import NearestCentroid
+    >>> import numpy as np
+    >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
+    >>> y = np.array([1, 1, 1, 2, 2, 2])
+    >>> clf = NearestCentroid()
+    >>> clf.fit(X, y)
+    NearestCentroid()
+    >>> print(clf.predict([[-0.8, -1]]))
+    [1]
     """
 
-    @_deprecate_positional_args
-    def __init__(self, metric='euclidean', *, shrink_threshold=None):
+    _parameter_constraints: dict = {
+        "metric": [StrOptions({"manhattan", "euclidean"})],
+        "shrink_threshold": [Interval(Real, 0, None, closed="neither"), None],
+        "priors": ["array-like", StrOptions({"empirical", "uniform"})],
+    }
+
+    def __init__(
+        self,
+        metric="euclidean",
+        *,
+        shrink_threshold=None,
+        priors="uniform",
+    ):
         self.metric = metric
         self.shrink_threshold = shrink_threshold
+        self.priors = priors
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y):
         """
         Fit the NearestCentroid model according to the given training data.
@@ -98,24 +157,33 @@ def fit(self, X, y):
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
-            Training vector, where n_samples is the number of samples and
-            n_features is the number of features.
+            Training vector, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
             Note that centroid shrinking cannot be used with sparse matrices.
         y : array-like of shape (n_samples,)
-            Target values (integers)
+            Target values.
+
+        Returns
+        -------
+        self : object
+            Fitted estimator.
         """
-        if self.metric == 'precomputed':
-            raise ValueError("Precomputed is not supported.")
         # If X is sparse and the metric is "manhattan", store it in a csc
         # format is easier to calculate the median.
-        if self.metric == 'manhattan':
-            X, y = self._validate_data(X, y, accept_sparse=['csc'])
+        if self.metric == "manhattan":
+            X, y = validate_data(self, X, y, accept_sparse=["csc"])
         else:
-            X, y = self._validate_data(X, y, accept_sparse=['csr', 'csc'])
+            ensure_all_finite = (
+                "allow-nan" if get_tags(self).input_tags.allow_nan else True
+            )
+            X, y = validate_data(
+                self,
+                X,
+                y,
+                ensure_all_finite=ensure_all_finite,
+                accept_sparse=["csr", "csc"],
+            )
         is_X_sparse = sp.issparse(X)
-        if is_X_sparse and self.shrink_threshold:
-            raise ValueError("threshold shrinking not supported"
-                             " for sparse input")
         check_classification_targets(y)
 
         n_samples, n_features = X.shape
@@ -124,11 +192,31 @@ def fit(self, X, y):
         self.classes_ = classes = le.classes_
         n_classes = classes.size
         if n_classes < 2:
-            raise ValueError('The number of classes has to be greater than'
-                             ' one; got %d class' % (n_classes))
+            raise ValueError(
+                "The number of classes has to be greater than one; got %d class"
+                % (n_classes)
+            )
+
+        if self.priors == "empirical":  # estimate priors from sample
+            _, class_counts = np.unique(y, return_inverse=True)  # non-negative ints
+            self.class_prior_ = np.bincount(class_counts) / float(len(y))
+        elif self.priors == "uniform":
+            self.class_prior_ = np.asarray([1 / n_classes] * n_classes)
+        else:
+            self.class_prior_ = np.asarray(self.priors)
+
+        if (self.class_prior_ < 0).any():
+            raise ValueError("priors must be non-negative")
+        if not np.isclose(self.class_prior_.sum(), 1.0):
+            warnings.warn(
+                "The priors do not sum to 1. Normalizing such that it sums to one.",
+                UserWarning,
+            )
+            self.class_prior_ = self.class_prior_ / self.class_prior_.sum()
 
         # Mask mapping each class to its members.
         self.centroids_ = np.empty((n_classes, n_features), dtype=np.float64)
+
         # Number of clusters in each class.
         nk = np.zeros(n_classes)
 
@@ -138,69 +226,134 @@ def fit(self, X, y):
             if is_X_sparse:
                 center_mask = np.where(center_mask)[0]
 
-            # XXX: Update other averaging methods according to the metrics.
             if self.metric == "manhattan":
                 # NumPy does not calculate median of sparse matrices.
                 if not is_X_sparse:
                     self.centroids_[cur_class] = np.median(X[center_mask], axis=0)
                 else:
                     self.centroids_[cur_class] = csc_median_axis_0(X[center_mask])
-            else:
-                if self.metric != 'euclidean':
-                    warnings.warn("Averaging for metrics other than "
-                                  "euclidean and manhattan not supported. "
-                                  "The average is set to be the mean."
-                                  )
+            else:  # metric == "euclidean"
                 self.centroids_[cur_class] = X[center_mask].mean(axis=0)
 
+        # Compute within-class std_dev with unshrunked centroids
+        variance = np.array(X - self.centroids_[y_ind], copy=False) ** 2
+        self.within_class_std_dev_ = np.array(
+            np.sqrt(variance.sum(axis=0) / (n_samples - n_classes)), copy=False
+        )
+        if any(self.within_class_std_dev_ == 0):
+            warnings.warn(
+                "self.within_class_std_dev_ has at least 1 zero standard deviation."
+                "Inputs within the same classes for at least 1 feature are identical."
+            )
+
+        err_msg = "All features have zero variance. Division by zero."
+        if is_X_sparse and np.all((X.max(axis=0) - X.min(axis=0)).toarray() == 0):
+            raise ValueError(err_msg)
+        elif not is_X_sparse and np.all(np.ptp(X, axis=0) == 0):
+            raise ValueError(err_msg)
+
+        dataset_centroid_ = X.mean(axis=0)
+        # m parameter for determining deviation
+        m = np.sqrt((1.0 / nk) - (1.0 / n_samples))
+        # Calculate deviation using the standard deviation of centroids.
+        # To deter outliers from affecting the results.
+        s = self.within_class_std_dev_ + np.median(self.within_class_std_dev_)
+        mm = m.reshape(len(m), 1)  # Reshape to allow broadcasting.
+        ms = mm * s
+        self.deviations_ = np.array(
+            (self.centroids_ - dataset_centroid_) / ms, copy=False
+        )
+        # Soft thresholding: if the deviation crosses 0 during shrinking,
+        # it becomes zero.
         if self.shrink_threshold:
-            if np.all(np.ptp(X, axis=0) == 0):
-                raise ValueError("All features have zero variance. "
-                                 "Division by zero.")
-            dataset_centroid_ = np.mean(X, axis=0)
-
-            # m parameter for determining deviation
-            m = np.sqrt((1. / nk) - (1. / n_samples))
-            # Calculate deviation using the standard deviation of centroids.
-            variance = (X - self.centroids_[y_ind]) ** 2
-            variance = variance.sum(axis=0)
-            s = np.sqrt(variance / (n_samples - n_classes))
-            s += np.median(s)  # To deter outliers from affecting the results.
-            mm = m.reshape(len(m), 1)  # Reshape to allow broadcasting.
-            ms = mm * s
-            deviation = ((self.centroids_ - dataset_centroid_) / ms)
-            # Soft thresholding: if the deviation crosses 0 during shrinking,
-            # it becomes zero.
-            signs = np.sign(deviation)
-            deviation = (np.abs(deviation) - self.shrink_threshold)
-            np.clip(deviation, 0, None, out=deviation)
-            deviation *= signs
+            signs = np.sign(self.deviations_)
+            self.deviations_ = np.abs(self.deviations_) - self.shrink_threshold
+            np.clip(self.deviations_, 0, None, out=self.deviations_)
+            self.deviations_ *= signs
             # Now adjust the centroids using the deviation
-            msd = ms * deviation
-            self.centroids_ = dataset_centroid_[np.newaxis, :] + msd
+            msd = ms * self.deviations_
+            self.centroids_ = np.array(dataset_centroid_ + msd, copy=False)
         return self
 
     def predict(self, X):
-        """Perform classification on an array of test vectors X.
+        """Perform classification on an array of test vectors `X`.
 
-        The predicted class C for each sample in X is returned.
+        The predicted class `C` for each sample in `X` is returned.
 
         Parameters
         ----------
-        X : array-like of shape (n_samples, n_features)
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Input data.
 
         Returns
         -------
-        C : ndarray of shape (n_samples,)
-
-        Notes
-        -----
-        If the metric constructor parameter is "precomputed", X is assumed to
-        be the distance matrix between the data to be predicted and
-        ``self.centroids_``.
+        y_pred : ndarray of shape (n_samples,)
+            The predicted classes.
         """
         check_is_fitted(self)
-
-        X = self._validate_data(X, accept_sparse='csr', reset=False)
-        return self.classes_[pairwise_distances(
-            X, self.centroids_, metric=self.metric).argmin(axis=1)]
+        if np.isclose(self.class_prior_, 1 / len(self.classes_)).all():
+            # `validate_data` is called here since we are not calling `super()`
+            ensure_all_finite = (
+                "allow-nan" if get_tags(self).input_tags.allow_nan else True
+            )
+            X = validate_data(
+                self,
+                X,
+                ensure_all_finite=ensure_all_finite,
+                accept_sparse="csr",
+                reset=False,
+            )
+            return self.classes_[
+                pairwise_distances_argmin(X, self.centroids_, metric=self.metric)
+            ]
+        else:
+            return super().predict(X)
+
+    def _decision_function(self, X):
+        # return discriminant scores, see eq. (18.2) p. 652 of the ESL.
+        check_is_fitted(self, "centroids_")
+
+        X_normalized = validate_data(
+            self, X, copy=True, reset=False, accept_sparse="csr", dtype=np.float64
+        )
+
+        discriminant_score = np.empty(
+            (X_normalized.shape[0], self.classes_.size), dtype=np.float64
+        )
+
+        mask = self.within_class_std_dev_ != 0
+        X_normalized[:, mask] /= self.within_class_std_dev_[mask]
+        centroids_normalized = self.centroids_.copy()
+        centroids_normalized[:, mask] /= self.within_class_std_dev_[mask]
+
+        for class_idx in range(self.classes_.size):
+            distances = pairwise_distances(
+                X_normalized, centroids_normalized[[class_idx]], metric=self.metric
+            ).ravel()
+            distances **= 2
+            discriminant_score[:, class_idx] = np.squeeze(
+                -distances + 2.0 * np.log(self.class_prior_[class_idx])
+            )
+
+        return discriminant_score
+
+    def _check_euclidean_metric(self):
+        return self.metric == "euclidean"
+
+    decision_function = available_if(_check_euclidean_metric)(
+        DiscriminantAnalysisPredictionMixin.decision_function
+    )
+
+    predict_proba = available_if(_check_euclidean_metric)(
+        DiscriminantAnalysisPredictionMixin.predict_proba
+    )
+
+    predict_log_proba = available_if(_check_euclidean_metric)(
+        DiscriminantAnalysisPredictionMixin.predict_log_proba
+    )
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.allow_nan = self.metric == "nan_euclidean"
+        tags.input_tags.sparse = True
+        return tags
diff --git a/sklearn/neighbors/_partition_nodes.pxd b/sklearn/neighbors/_partition_nodes.pxd
new file mode 100644
index 0000000000000..bd2160cc3b26f
--- /dev/null
+++ b/sklearn/neighbors/_partition_nodes.pxd
@@ -0,0 +1,10 @@
+from cython cimport floating
+from ..utils._typedefs cimport float64_t, intp_t
+
+cdef int partition_node_indices(
+        const floating *data,
+        intp_t *node_indices,
+        intp_t split_dim,
+        intp_t split_index,
+        intp_t n_features,
+        intp_t n_points) except -1
diff --git a/sklearn/neighbors/_partition_nodes.pyx b/sklearn/neighbors/_partition_nodes.pyx
new file mode 100644
index 0000000000000..111353c49a22b
--- /dev/null
+++ b/sklearn/neighbors/_partition_nodes.pyx
@@ -0,0 +1,122 @@
+# BinaryTrees rely on partial sorts to partition their nodes during their
+# initialisation.
+#
+# The C++ std library exposes nth_element, an efficient partial sort for this
+# situation which has a linear time complexity as well as the best performances.
+#
+# To use std::algorithm::nth_element, a few fixture are defined using Cython:
+# - partition_node_indices, a Cython function used in BinaryTrees, that calls
+# - partition_node_indices_inner, a C++ function that wraps nth_element and uses
+# - an IndexComparator to state how to compare KDTrees' indices
+#
+# IndexComparator has been defined so that partial sorts are stable with
+# respect to the nodes initial indices.
+#
+# See for reference:
+#  - https://en.cppreference.com/w/cpp/algorithm/nth_element.
+#  - https://github.com/scikit-learn/scikit-learn/pull/11103
+#  - https://github.com/scikit-learn/scikit-learn/pull/19473
+from cython cimport floating
+
+
+cdef extern from *:
+    """
+    #include 
+
+    template
+    class IndexComparator {
+    private:
+        const D *data;
+        I split_dim, n_features;
+    public:
+        IndexComparator(const D *data, const I &split_dim, const I &n_features):
+            data(data), split_dim(split_dim), n_features(n_features) {}
+
+        bool operator()(const I &a, const I &b) const {
+            D a_value = data[a * n_features + split_dim];
+            D b_value = data[b * n_features + split_dim];
+            return a_value == b_value ? a < b : a_value < b_value;
+        }
+    };
+
+    template
+    void partition_node_indices_inner(
+        const D *data,
+        I *node_indices,
+        const I &split_dim,
+        const I &split_index,
+        const I &n_features,
+        const I &n_points) {
+        IndexComparator index_comparator(data, split_dim, n_features);
+        std::nth_element(
+            node_indices,
+            node_indices + split_index,
+            node_indices + n_points,
+            index_comparator);
+    }
+    """
+    void partition_node_indices_inner[D, I](
+                const D *data,
+                I *node_indices,
+                I split_dim,
+                I split_index,
+                I n_features,
+                I n_points) except +
+
+
+cdef int partition_node_indices(
+        const floating *data,
+        intp_t *node_indices,
+        intp_t split_dim,
+        intp_t split_index,
+        intp_t n_features,
+        intp_t n_points) except -1:
+    """Partition points in the node into two equal-sized groups.
+
+    Upon return, the values in node_indices will be rearranged such that
+    (assuming numpy-style indexing):
+
+        data[node_indices[0:split_index], split_dim]
+          <= data[node_indices[split_index], split_dim]
+
+    and
+
+        data[node_indices[split_index], split_dim]
+          <= data[node_indices[split_index:n_points], split_dim]
+
+    The algorithm is essentially a partial in-place quicksort around a
+    set pivot.
+
+    Parameters
+    ----------
+    data : double pointer
+        Pointer to a 2D array of the training data, of shape [N, n_features].
+        N must be greater than any of the values in node_indices.
+    node_indices : int pointer
+        Pointer to a 1D array of length n_points.  This lists the indices of
+        each of the points within the current node.  This will be modified
+        in-place.
+    split_dim : int
+        the dimension on which to split.  This will usually be computed via
+        the routine ``find_node_split_dim``.
+    split_index : int
+        the index within node_indices around which to split the points.
+    n_features: int
+        the number of features (i.e columns) in the 2D array pointed by data.
+    n_points : int
+        the length of node_indices. This is also the number of points in
+        the original dataset.
+    Returns
+    -------
+    status : int
+        integer exit status.  On return, the contents of node_indices are
+        modified as noted above.
+    """
+    partition_node_indices_inner(
+        data,
+        node_indices,
+        split_dim,
+        split_index,
+        n_features,
+        n_points)
+    return 0
diff --git a/sklearn/neighbors/_quad_tree.pxd b/sklearn/neighbors/_quad_tree.pxd
index 7fb65ec2e4fb6..e7e817902f103 100644
--- a/sklearn/neighbors/_quad_tree.pxd
+++ b/sklearn/neighbors/_quad_tree.pxd
@@ -1,19 +1,10 @@
-# cython: boundscheck=False
-# cython: wraparound=False
-# cython: cdivision=True
-#
-# Author: Thomas Moreau 
-# Author: Olivier Grisel 
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 # See quad_tree.pyx for details.
 
-import numpy as np
-cimport numpy as np
-
-ctypedef np.npy_float32 DTYPE_t          # Type of X
-ctypedef np.npy_intp SIZE_t              # Type for indices and counters
-ctypedef np.npy_int32 INT32_t            # Signed 32 bit integer
-ctypedef np.npy_uint32 UINT32_t          # Unsigned 32 bit integer
+cimport numpy as cnp
+from ..utils._typedefs cimport float32_t, intp_t
 
 # This is effectively an ifdef statement in Cython
 # It allows us to write printf debugging lines
@@ -30,26 +21,26 @@ cdef struct Cell:
     # Base storage structure for cells in a QuadTree object
 
     # Tree structure
-    SIZE_t parent              # Parent cell of this cell
-    SIZE_t[8] children         # Array pointing to childrens of this cell
+    intp_t parent                # Parent cell of this cell
+    intp_t[8] children           # Array pointing to children of this cell
 
     # Cell description
-    SIZE_t cell_id             # Id of the cell in the cells array in the Tree
-    SIZE_t point_index         # Index of the point at this cell (only defined
-                               # in non empty leaf)
-    bint is_leaf               # Does this cell have children?
-    DTYPE_t squared_max_width  # Squared value of the maximum width w
-    SIZE_t depth               # Depth of the cell in the tree
-    SIZE_t cumulative_size     # Number of points included in the subtree with
-                               # this cell as a root.
+    intp_t cell_id               # Id of the cell in the cells array in the Tree
+    intp_t point_index           # Index of the point at this cell (only defined
+    #                            # in non empty leaf)
+    bint is_leaf                 # Does this cell have children?
+    float32_t squared_max_width  # Squared value of the maximum width w
+    intp_t depth                 # Depth of the cell in the tree
+    intp_t cumulative_size       # Number of points included in the subtree with
+    #                            # this cell as a root.
 
     # Internal constants
-    DTYPE_t[3] center          # Store the center for quick split of cells
-    DTYPE_t[3] barycenter      # Keep track of the center of mass of the cell
+    float32_t[3] center          # Store the center for quick split of cells
+    float32_t[3] barycenter      # Keep track of the center of mass of the cell
 
     # Cell boundaries
-    DTYPE_t[3] min_bounds      # Inferior boundaries of this cell (inclusive)
-    DTYPE_t[3] max_bounds      # Superior boundaries of this cell (exclusive)
+    float32_t[3] min_bounds      # Inferior boundaries of this cell (inclusive)
+    float32_t[3] max_bounds      # Superior boundaries of this cell (exclusive)
 
 
 cdef class _QuadTree:
@@ -62,40 +53,40 @@ cdef class _QuadTree:
     # Parameters of the tree
     cdef public int n_dimensions         # Number of dimensions in X
     cdef public int verbose              # Verbosity of the output
-    cdef SIZE_t n_cells_per_cell         # Number of children per node. (2 ** n_dimension)
+    cdef intp_t n_cells_per_cell         # Number of children per node. (2 ** n_dimension)
 
     # Tree inner structure
-    cdef public SIZE_t max_depth         # Max depth of the tree
-    cdef public SIZE_t cell_count        # Counter for node IDs
-    cdef public SIZE_t capacity          # Capacity of tree, in terms of nodes
-    cdef public SIZE_t n_points          # Total number of points
+    cdef public intp_t max_depth         # Max depth of the tree
+    cdef public intp_t cell_count        # Counter for node IDs
+    cdef public intp_t capacity          # Capacity of tree, in terms of nodes
+    cdef public intp_t n_points          # Total number of points
     cdef Cell* cells                     # Array of nodes
 
     # Point insertion methods
-    cdef int insert_point(self, DTYPE_t[3] point, SIZE_t point_index,
-                          SIZE_t cell_id=*) nogil except -1
-    cdef SIZE_t _insert_point_in_new_child(self, DTYPE_t[3] point, Cell* cell,
-                                           SIZE_t point_index, SIZE_t size=*
-                                           ) nogil
-    cdef SIZE_t _select_child(self, DTYPE_t[3] point, Cell* cell) nogil
-    cdef bint _is_duplicate(self, DTYPE_t[3] point1, DTYPE_t[3] point2) nogil
+    cdef int insert_point(self, float32_t[3] point, intp_t point_index,
+                          intp_t cell_id=*) except -1 nogil
+    cdef intp_t _insert_point_in_new_child(self, float32_t[3] point, Cell* cell,
+                                           intp_t point_index, intp_t size=*
+                                           ) noexcept nogil
+    cdef intp_t _select_child(self, float32_t[3] point, Cell* cell) noexcept nogil
+    cdef bint _is_duplicate(self, float32_t[3] point1, float32_t[3] point2) noexcept nogil
 
     # Create a summary of the Tree compare to a query point
-    cdef long summarize(self, DTYPE_t[3] point, DTYPE_t* results,
-                        float squared_theta=*, SIZE_t cell_id=*, long idx=*
-                        ) nogil
+    cdef long summarize(self, float32_t[3] point, float32_t* results,
+                        float squared_theta=*, intp_t cell_id=*, long idx=*
+                        ) noexcept nogil
 
     # Internal cell initialization methods
-    cdef void _init_cell(self, Cell* cell, SIZE_t parent, SIZE_t depth) nogil
-    cdef void _init_root(self, DTYPE_t[3] min_bounds, DTYPE_t[3] max_bounds
-                         ) nogil
+    cdef void _init_cell(self, Cell* cell, intp_t parent, intp_t depth) noexcept nogil
+    cdef void _init_root(self, float32_t[3] min_bounds, float32_t[3] max_bounds
+                         ) noexcept nogil
 
     # Private methods
-    cdef int _check_point_in_cell(self, DTYPE_t[3] point, Cell* cell
-                                  ) nogil except -1
+    cdef int _check_point_in_cell(self, float32_t[3] point, Cell* cell
+                                  ) except -1 nogil
 
     # Private array manipulation to manage the ``cells`` array
-    cdef int _resize(self, SIZE_t capacity) nogil except -1
-    cdef int _resize_c(self, SIZE_t capacity=*) nogil except -1
-    cdef int _get_cell(self, DTYPE_t[3] point, SIZE_t cell_id=*) nogil except -1
-    cdef np.ndarray _get_cell_ndarray(self)
+    cdef int _resize(self, intp_t capacity) except -1 nogil
+    cdef int _resize_c(self, intp_t capacity=*) except -1 nogil
+    cdef int _get_cell(self, float32_t[3] point, intp_t cell_id=*) except -1 nogil
+    cdef Cell[:] _get_cell_ndarray(self)
diff --git a/sklearn/neighbors/_quad_tree.pyx b/sklearn/neighbors/_quad_tree.pyx
index 0bcc7aeb6af11..aec79da505f52 100644
--- a/sklearn/neighbors/_quad_tree.pyx
+++ b/sklearn/neighbors/_quad_tree.pyx
@@ -1,40 +1,33 @@
-# cython: boundscheck=False
-# cython: wraparound=False
-# cython: cdivision=True
-#
-# Author: Thomas Moreau 
-# Author: Olivier Grisel 
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 
 from cpython cimport Py_INCREF, PyObject, PyTypeObject
 
-from libc.stdlib cimport malloc, free
+from libc.math cimport fabsf
+from libc.stdlib cimport free
 from libc.string cimport memcpy
 from libc.stdio cimport printf
 from libc.stdint cimport SIZE_MAX
 
-from ..tree._utils cimport safe_realloc, sizet_ptr_to_ndarray
-from ..utils import check_array
+from ..tree._utils cimport safe_realloc
 
 import numpy as np
-cimport numpy as np
-np.import_array()
-
-cdef extern from "math.h":
-    float fabsf(float x) nogil
+cimport numpy as cnp
+cnp.import_array()
 
 cdef extern from "numpy/arrayobject.h":
-    object PyArray_NewFromDescr(PyTypeObject* subtype, np.dtype descr,
-                                int nd, np.npy_intp* dims,
-                                np.npy_intp* strides,
+    object PyArray_NewFromDescr(PyTypeObject* subtype, cnp.dtype descr,
+                                int nd, cnp.npy_intp* dims,
+                                cnp.npy_intp* strides,
                                 void* data, int flags, object obj)
-    int PyArray_SetBaseObject(np.ndarray arr, PyObject* obj)
+    int PyArray_SetBaseObject(cnp.ndarray arr, PyObject* obj)
 
 # Build the corresponding numpy dtype for Cell.
 # This works by casting `dummy` to an array of Cell of length 1, which numpy
 # can construct a `dtype`-object for. See https://stackoverflow.com/q/62448946
 # for a more detailed explanation.
-cdef Cell dummy;
+cdef Cell dummy
 CELL_DTYPE = np.asarray((&dummy)).dtype
 
 assert CELL_DTYPE.itemsize == sizeof(Cell)
@@ -57,7 +50,7 @@ cdef class _QuadTree:
         # Parameters of the tree
         self.n_dimensions = n_dimensions
         self.verbose = verbose
-        self.n_cells_per_cell = 2 ** self.n_dimensions
+        self.n_cells_per_cell =  (2 ** self.n_dimensions)
 
         # Inner structures
         self.max_depth = 0
@@ -71,23 +64,25 @@ cdef class _QuadTree:
         # Free all inner structures
         free(self.cells)
 
-    property cumulative_size:
-        def __get__(self):
-            return self._get_cell_ndarray()['cumulative_size'][:self.cell_count]
+    @property
+    def cumulative_size(self):
+        cdef Cell[:] cell_mem_view = self._get_cell_ndarray()
+        return cell_mem_view.base['cumulative_size'][:self.cell_count]
 
-    property leafs:
-        def __get__(self):
-            return self._get_cell_ndarray()['is_leaf'][:self.cell_count]
+    @property
+    def leafs(self):
+        cdef Cell[:] cell_mem_view = self._get_cell_ndarray()
+        return cell_mem_view.base['is_leaf'][:self.cell_count]
 
     def build_tree(self, X):
         """Build a tree from an array of points X."""
         cdef:
             int i
-            DTYPE_t[3] pt
-            DTYPE_t[3] min_bounds, max_bounds
+            float32_t[3] pt
+            float32_t[3] min_bounds, max_bounds
 
         # validate X and prepare for query
-        # X = check_array(X, dtype=DTYPE_t, order='C')
+        # X = check_array(X, dtype=float32_t, order='C')
         n_samples = X.shape[0]
 
         capacity = 100
@@ -116,14 +111,13 @@ cdef class _QuadTree:
         # Shrink the cells array to reduce memory usage
         self._resize(capacity=self.cell_count)
 
-    cdef int insert_point(self, DTYPE_t[3] point, SIZE_t point_index,
-                          SIZE_t cell_id=0) nogil except -1:
+    cdef int insert_point(self, float32_t[3] point, intp_t point_index,
+                          intp_t cell_id=0) except -1 nogil:
         """Insert a point in the QuadTree."""
         cdef int ax
-        cdef DTYPE_t n_frac
-        cdef SIZE_t selected_child
+        cdef intp_t selected_child
         cdef Cell* cell = &self.cells[cell_id]
-        cdef SIZE_t n_point = cell.cumulative_size
+        cdef intp_t n_point = cell.cumulative_size
 
         if self.verbose > 10:
             printf("[QuadTree] Inserting depth %li\n", cell.depth)
@@ -181,16 +175,16 @@ cdef class _QuadTree:
         return self.insert_point(point, point_index, cell_id)
 
     # XXX: This operation is not Thread safe
-    cdef SIZE_t _insert_point_in_new_child(self, DTYPE_t[3] point, Cell* cell,
-                                          SIZE_t point_index, SIZE_t size=1
-                                          ) nogil:
+    cdef intp_t _insert_point_in_new_child(
+        self, float32_t[3] point, Cell* cell, intp_t point_index, intp_t size=1
+    ) noexcept nogil:
         """Create a child of cell which will contain point."""
 
         # Local variable definition
         cdef:
-            SIZE_t cell_id, cell_child_id, parent_id
-            DTYPE_t[3] save_point
-            DTYPE_t width
+            intp_t cell_id, cell_child_id, parent_id
+            float32_t[3] save_point
+            float32_t width
             Cell* child
             int i
 
@@ -208,7 +202,7 @@ cdef class _QuadTree:
         # Get an empty cell and initialize it
         cell_id = self.cell_count
         self.cell_count += 1
-        child  = &self.cells[cell_id]
+        child = &self.cells[cell_id]
 
         self._init_cell(child, cell.cell_id, cell.depth + 1)
         child.cell_id = cell_id
@@ -251,8 +245,7 @@ cdef class _QuadTree:
 
         return cell_id
 
-
-    cdef bint _is_duplicate(self, DTYPE_t[3] point1, DTYPE_t[3] point2) nogil:
+    cdef bint _is_duplicate(self, float32_t[3] point1, float32_t[3] point2) noexcept nogil:
         """Check if the two given points are equals."""
         cdef int i
         cdef bint res = True
@@ -261,12 +254,11 @@ cdef class _QuadTree:
             res &= fabsf(point1[i] - point2[i]) <= EPSILON
         return res
 
-
-    cdef SIZE_t _select_child(self, DTYPE_t[3] point, Cell* cell) nogil:
+    cdef intp_t _select_child(self, float32_t[3] point, Cell* cell) noexcept nogil:
         """Select the child of cell which contains the given query point."""
         cdef:
             int i
-            SIZE_t selected_child = 0
+            intp_t selected_child = 0
 
         for i in range(self.n_dimensions):
             # Select the correct child cell to insert the point by comparing
@@ -276,7 +268,7 @@ cdef class _QuadTree:
                 selected_child += 1
         return cell.children[selected_child]
 
-    cdef void _init_cell(self, Cell* cell, SIZE_t parent, SIZE_t depth) nogil:
+    cdef void _init_cell(self, Cell* cell, intp_t parent, intp_t depth) noexcept nogil:
         """Initialize a cell structure with some constants."""
         cell.parent = parent
         cell.is_leaf = True
@@ -286,12 +278,12 @@ cdef class _QuadTree:
         for i in range(self.n_cells_per_cell):
             cell.children[i] = SIZE_MAX
 
-    cdef void _init_root(self, DTYPE_t[3] min_bounds, DTYPE_t[3] max_bounds
-                         ) nogil:
+    cdef void _init_root(self, float32_t[3] min_bounds, float32_t[3] max_bounds
+                         ) noexcept nogil:
         """Initialize the root node with the given space boundaries"""
         cdef:
             int i
-            DTYPE_t width
+            float32_t width
             Cell* root = &self.cells[0]
 
         self._init_cell(root, -1, 0)
@@ -305,24 +297,24 @@ cdef class _QuadTree:
 
         self.cell_count += 1
 
-    cdef int _check_point_in_cell(self, DTYPE_t[3] point, Cell* cell
-                                  ) nogil except -1:
+    cdef int _check_point_in_cell(self, float32_t[3] point, Cell* cell
+                                  ) except -1 nogil:
         """Check that the given point is in the cell boundaries."""
 
         if self.verbose >= 50:
             if self.n_dimensions == 3:
                 printf("[QuadTree] Checking point (%f, %f, %f) in cell %li "
-                        "([%f/%f, %f/%f, %f/%f], size %li)\n",
-                        point[0], point[1], point[2], cell.cell_id,
-                        cell.min_bounds[0], cell.max_bounds[0], cell.min_bounds[1],
-                        cell.max_bounds[1], cell.min_bounds[2], cell.max_bounds[2],
-                        cell.cumulative_size)
+                       "([%f/%f, %f/%f, %f/%f], size %li)\n",
+                       point[0], point[1], point[2], cell.cell_id,
+                       cell.min_bounds[0], cell.max_bounds[0], cell.min_bounds[1],
+                       cell.max_bounds[1], cell.min_bounds[2], cell.max_bounds[2],
+                       cell.cumulative_size)
             else:
                 printf("[QuadTree] Checking point (%f, %f) in cell %li "
-                        "([%f/%f, %f/%f], size %li)\n",
-                        point[0], point[1],cell.cell_id, cell.min_bounds[0],
-                        cell.max_bounds[0], cell.min_bounds[1],
-                        cell.max_bounds[1], cell.cumulative_size)
+                       "([%f/%f, %f/%f], size %li)\n",
+                       point[0], point[1], cell.cell_id, cell.min_bounds[0],
+                       cell.max_bounds[0], cell.min_bounds[1],
+                       cell.max_bounds[1], cell.cumulative_size)
 
         for i in range(self.n_dimensions):
             if (cell.min_bounds[i] > point[i] or
@@ -356,7 +348,7 @@ cdef class _QuadTree:
                         child = self.cells[child_id]
                         n_points += child.cumulative_size
                         assert child.cell_id == child_id, (
-                            "Cell id not correctly initiliazed.")
+                            "Cell id not correctly initialized.")
                 if n_points != cell.cumulative_size:
                     raise ValueError(
                         "Cell {} is incoherent. Size={} but found {} points "
@@ -365,16 +357,16 @@ cdef class _QuadTree:
                                 n_points, cell.children))
 
         # Make sure that the number of point in the tree correspond to the
-        # cummulative size in root cell.
+        # cumulative size in root cell.
         if self.n_points != self.cells[0].cumulative_size:
             raise ValueError(
                 "QuadTree is incoherent. Size={} but found {} points "
                 "in children."
                 .format(self.n_points, self.cells[0].cumulative_size))
 
-    cdef long summarize(self, DTYPE_t[3] point, DTYPE_t* results,
-                        float squared_theta=.5, SIZE_t cell_id=0, long idx=0
-                        ) nogil:
+    cdef long summarize(self, float32_t[3] point, float32_t* results,
+                        float squared_theta=.5, intp_t cell_id=0, long idx=0
+                        ) noexcept nogil:
         """Summarize the tree compared to a query point.
 
         Input arguments
@@ -430,12 +422,12 @@ cdef class _QuadTree:
 
         # Check whether we can use this node as a summary
         # It's a summary node if the angular size as measured from the point
-        # is relatively small (w.r.t. to theta) or if it is a leaf node.
+        # is relatively small (w.r.t. theta) or if it is a leaf node.
         # If it can be summarized, we use the cell center of mass
         # Otherwise, we go a higher level of resolution and into the leaves.
         if cell.is_leaf or (
                 (cell.squared_max_width / results[idx_d]) < squared_theta):
-            results[idx_d + 1] =  cell.cumulative_size
+            results[idx_d + 1] =  cell.cumulative_size
             return idx + self.n_dimensions + 2
 
         else:
@@ -452,7 +444,7 @@ cdef class _QuadTree:
         """return the id of the cell containing the query point or raise
         ValueError if the point is not in the tree
         """
-        cdef DTYPE_t[3] query_pt
+        cdef float32_t[3] query_pt
         cdef int i
 
         assert len(point) == self.n_dimensions, (
@@ -464,14 +456,14 @@ cdef class _QuadTree:
 
         return self._get_cell(query_pt, 0)
 
-    cdef int _get_cell(self, DTYPE_t[3] point, SIZE_t cell_id=0
-                       ) nogil except -1:
+    cdef int _get_cell(self, float32_t[3] point, intp_t cell_id=0
+                       ) except -1 nogil:
         """guts of get_cell.
 
         Return the id of the cell containing the query point or raise ValueError
         if the point is not in the tree"""
         cdef:
-            SIZE_t selected_child
+            intp_t selected_child
             Cell* cell = &self.cells[cell_id]
 
         if cell.is_leaf:
@@ -495,8 +487,7 @@ cdef class _QuadTree:
 
     def __reduce__(self):
         """Reduce re-implementation, for pickling."""
-        return (_QuadTree, (self.n_dimensions, self.verbose),
-                           self.__getstate__())
+        return (_QuadTree, (self.n_dimensions, self.verbose), self.__getstate__())
 
     def __getstate__(self):
         """Getstate re-implementation, for pickling."""
@@ -506,7 +497,7 @@ cdef class _QuadTree:
         d["cell_count"] = self.cell_count
         d["capacity"] = self.capacity
         d["n_points"] = self.n_points
-        d["cells"] = self._get_cell_ndarray()
+        d["cells"] = self._get_cell_ndarray().base
         return d
 
     def __setstate__(self, d):
@@ -531,36 +522,45 @@ cdef class _QuadTree:
         if self._resize_c(self.capacity) != 0:
             raise MemoryError("resizing tree to %d" % self.capacity)
 
-        cells = memcpy(self.cells, ( cell_ndarray).data,
-                       self.capacity * sizeof(Cell))
-
+        cdef Cell[:] cell_mem_view = cell_ndarray
+        memcpy(
+            pto=self.cells,
+            pfrom=&cell_mem_view[0],
+            size=self.capacity * sizeof(Cell),
+        )
 
     # Array manipulation methods, to convert it to numpy or to resize
     # self.cells array
 
-    cdef np.ndarray _get_cell_ndarray(self):
+    cdef Cell[:] _get_cell_ndarray(self):
         """Wraps nodes as a NumPy struct array.
 
         The array keeps a reference to this Tree, which manages the underlying
         memory. Individual fields are publicly accessible as properties of the
         Tree.
         """
-        cdef np.npy_intp shape[1]
-        shape[0] =  self.cell_count
-        cdef np.npy_intp strides[1]
+        cdef cnp.npy_intp shape[1]
+        shape[0] =  self.cell_count
+        cdef cnp.npy_intp strides[1]
         strides[0] = sizeof(Cell)
-        cdef np.ndarray arr
+        cdef Cell[:] arr
         Py_INCREF(CELL_DTYPE)
-        arr = PyArray_NewFromDescr( np.ndarray,
-                                   CELL_DTYPE, 1, shape,
-                                   strides,  self.cells,
-                                   np.NPY_DEFAULT, None)
+        arr = PyArray_NewFromDescr(
+            subtype= np.ndarray,
+            descr=CELL_DTYPE,
+            nd=1,
+            dims=shape,
+            strides=strides,
+            data= self.cells,
+            flags=cnp.NPY_ARRAY_DEFAULT,
+            obj=None,
+        )
         Py_INCREF(self)
-        if PyArray_SetBaseObject(arr,  self) < 0:
-            raise ValueError("Can't intialize array!")
+        if PyArray_SetBaseObject(arr.base,  self) < 0:
+            raise ValueError("Can't initialize array!")
         return arr
 
-    cdef int _resize(self, SIZE_t capacity) nogil except -1:
+    cdef int _resize(self, intp_t capacity) except -1 nogil:
         """Resize all inner arrays to `capacity`, if `capacity` == -1, then
            double the size of the inner arrays.
 
@@ -572,7 +572,7 @@ cdef class _QuadTree:
             with gil:
                 raise MemoryError()
 
-    cdef int _resize_c(self, SIZE_t capacity=SIZE_MAX) nogil except -1:
+    cdef int _resize_c(self, intp_t capacity=SIZE_MAX) except -1 nogil:
         """Guts of _resize
 
         Returns -1 in case of failure to allocate memory (and raise MemoryError)
@@ -581,7 +581,7 @@ cdef class _QuadTree:
         if capacity == self.capacity and self.cells != NULL:
             return 0
 
-        if capacity == SIZE_MAX:
+        if  capacity == SIZE_MAX:
             if self.capacity == 0:
                 capacity = 9  # default initial value to min
             else:
@@ -596,14 +596,13 @@ cdef class _QuadTree:
         self.capacity = capacity
         return 0
 
-    def _py_summarize(self, DTYPE_t[:] query_pt, DTYPE_t[:, :] X, float angle):
+    def _py_summarize(self, float32_t[:] query_pt, float32_t[:, :] X, float angle):
         # Used for testing summarize
         cdef:
-            DTYPE_t[:] summary
-            int n_samples, n_dimensions
+            float32_t[:] summary
+            int n_samples
 
         n_samples = X.shape[0]
-        n_dimensions = X.shape[1]
         summary = np.empty(4 * n_samples, dtype=np.float32)
 
         idx = self.summarize(&query_pt[0], &summary[0], angle * angle)
diff --git a/sklearn/neighbors/_regression.py b/sklearn/neighbors/_regression.py
index d3878cd54aa06..0ee0a340b8153 100644
--- a/sklearn/neighbors/_regression.py
+++ b/sklearn/neighbors/_regression.py
@@ -1,29 +1,19 @@
-"""Nearest Neighbor Regression"""
-
-# Authors: Jake Vanderplas 
-#          Fabian Pedregosa 
-#          Alexandre Gramfort 
-#          Sparseness support by Lars Buitinck
-#          Multi-output support by Arnaud Joly 
-#          Empty radius support by Andreas Bjerre-Nielsen
-#
-# License: BSD 3 clause (C) INRIA, University of Amsterdam,
-#                           University of Copenhagen
+"""Nearest Neighbor Regression."""
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import warnings
 
 import numpy as np
 
-from ._base import _get_weights, _check_weights
-from ._base import NeighborsBase, KNeighborsMixin, RadiusNeighborsMixin
-from ..base import RegressorMixin
-from ..utils.validation import _deprecate_positional_args
-from ..utils.deprecation import deprecated
+from ..base import RegressorMixin, _fit_context
+from ..metrics import DistanceMetric
+from ..utils._param_validation import StrOptions
+from ._base import KNeighborsMixin, NeighborsBase, RadiusNeighborsMixin, _get_weights
 
 
-class KNeighborsRegressor(KNeighborsMixin,
-                          RegressorMixin,
-                          NeighborsBase):
+class KNeighborsRegressor(KNeighborsMixin, RegressorMixin, NeighborsBase):
     """Regression based on k-nearest neighbors.
 
     The target is predicted by local interpolation of the targets
@@ -38,8 +28,8 @@ class KNeighborsRegressor(KNeighborsMixin,
     n_neighbors : int, default=5
         Number of neighbors to use by default for :meth:`kneighbors` queries.
 
-    weights : {'uniform', 'distance'} or callable, default='uniform'
-        weight function used in prediction.  Possible values:
+    weights : {'uniform', 'distance'}, callable or None, default='uniform'
+        Weight function used in prediction.  Possible values:
 
         - 'uniform' : uniform weights.  All points in each neighborhood
           are weighted equally.
@@ -52,6 +42,10 @@ class KNeighborsRegressor(KNeighborsMixin,
 
         Uniform weights are used by default.
 
+        See the following example for a demonstration of the impact of
+        different weighting schemes on predictions:
+        :ref:`sphx_glr_auto_examples_neighbors_plot_regression.py`.
+
     algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, default='auto'
         Algorithm used to compute the nearest neighbors:
 
@@ -70,19 +64,31 @@ class KNeighborsRegressor(KNeighborsMixin,
         required to store the tree.  The optimal value depends on the
         nature of the problem.
 
-    p : int, default=2
+    p : float, default=2
         Power parameter for the Minkowski metric. When p = 1, this is
         equivalent to using manhattan_distance (l1), and euclidean_distance
         (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used.
 
-    metric : str or callable, default='minkowski'
-        the distance metric to use for the tree.  The default metric is
-        minkowski, and with p=2 is equivalent to the standard Euclidean
-        metric. See the documentation of :class:`DistanceMetric` for a
-        list of available metrics.
+    metric : str, DistanceMetric object or callable, default='minkowski'
+        Metric to use for distance computation. Default is "minkowski", which
+        results in the standard Euclidean distance when p = 2. See the
+        documentation of `scipy.spatial.distance
+        `_ and
+        the metrics listed in
+        :class:`~sklearn.metrics.pairwise.distance_metrics` for valid metric
+        values.
+
         If metric is "precomputed", X is assumed to be a distance matrix and
-        must be square during fit. X may be a :term:`sparse graph`,
-        in which case only "nonzero" elements may be considered neighbors.
+        must be square during fit. X may be a :term:`sparse graph`, in which
+        case only "nonzero" elements may be considered neighbors.
+
+        If metric is a callable function, it takes two arrays representing 1D
+        vectors as inputs and must return one value indicating the distance
+        between those vectors. This works for Scipy's metrics, but is less
+        efficient than passing the metric name as a string.
+
+        If metric is a DistanceMetric object, it will be passed directly to
+        the underlying computation routines.
 
     metric_params : dict, default=None
         Additional keyword arguments for the metric function.
@@ -107,26 +113,27 @@ class KNeighborsRegressor(KNeighborsMixin,
         `p` parameter value if the `effective_metric_` attribute is set to
         'minkowski'.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     n_samples_fit_ : int
         Number of samples in the fitted data.
 
-    Examples
-    --------
-    >>> X = [[0], [1], [2], [3]]
-    >>> y = [0, 0, 1, 1]
-    >>> from sklearn.neighbors import KNeighborsRegressor
-    >>> neigh = KNeighborsRegressor(n_neighbors=2)
-    >>> neigh.fit(X, y)
-    KNeighborsRegressor(...)
-    >>> print(neigh.predict([[1.5]]))
-    [0.5]
-
     See Also
     --------
-    NearestNeighbors
-    RadiusNeighborsRegressor
-    KNeighborsClassifier
-    RadiusNeighborsClassifier
+    NearestNeighbors : Unsupervised learner for implementing neighbor searches.
+    RadiusNeighborsRegressor : Regression based on neighbors within a fixed radius.
+    KNeighborsClassifier : Classifier implementing the k-nearest neighbors vote.
+    RadiusNeighborsClassifier : Classifier implementing
+        a vote among neighbors within a given radius.
 
     Notes
     -----
@@ -140,34 +147,60 @@ class KNeighborsRegressor(KNeighborsMixin,
        different labels, the results will depend on the ordering of the
        training data.
 
-    https://en.wikipedia.org/wiki/K-nearest_neighbor_algorithm
+    https://en.wikipedia.org/wiki/K-nearest_neighbors_algorithm
+
+    Examples
+    --------
+    >>> X = [[0], [1], [2], [3]]
+    >>> y = [0, 0, 1, 1]
+    >>> from sklearn.neighbors import KNeighborsRegressor
+    >>> neigh = KNeighborsRegressor(n_neighbors=2)
+    >>> neigh.fit(X, y)
+    KNeighborsRegressor(...)
+    >>> print(neigh.predict([[1.5]]))
+    [0.5]
     """
 
-    @_deprecate_positional_args
-    def __init__(self, n_neighbors=5, *, weights='uniform',
-                 algorithm='auto', leaf_size=30,
-                 p=2, metric='minkowski', metric_params=None, n_jobs=None,
-                 **kwargs):
+    _parameter_constraints: dict = {
+        **NeighborsBase._parameter_constraints,
+        "weights": [StrOptions({"uniform", "distance"}), callable, None],
+    }
+    _parameter_constraints["metric"].append(DistanceMetric)
+    _parameter_constraints.pop("radius")
+
+    def __init__(
+        self,
+        n_neighbors=5,
+        *,
+        weights="uniform",
+        algorithm="auto",
+        leaf_size=30,
+        p=2,
+        metric="minkowski",
+        metric_params=None,
+        n_jobs=None,
+    ):
         super().__init__(
-              n_neighbors=n_neighbors,
-              algorithm=algorithm,
-              leaf_size=leaf_size, metric=metric, p=p,
-              metric_params=metric_params, n_jobs=n_jobs, **kwargs)
-        self.weights = _check_weights(weights)
-
-    def _more_tags(self):
-        # For cross-validation routines to split data correctly
-        return {'pairwise': self.metric == 'precomputed'}
-
-    # TODO: Remove in 1.1
-    # mypy error: Decorated property not supported
-    @deprecated("Attribute _pairwise was deprecated in "  # type: ignore
-                "version 0.24 and will be removed in 1.1 (renaming of 0.26).")
-    @property
-    def _pairwise(self):
+            n_neighbors=n_neighbors,
+            algorithm=algorithm,
+            leaf_size=leaf_size,
+            metric=metric,
+            p=p,
+            metric_params=metric_params,
+            n_jobs=n_jobs,
+        )
+        self.weights = weights
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
         # For cross-validation routines to split data correctly
-        return self.metric == 'precomputed'
+        tags.input_tags.pairwise = self.metric == "precomputed"
+        return tags
 
+    @_fit_context(
+        # KNeighborsRegressor.metric is not validated yet
+        prefer_skip_nested_validation=False
+    )
     def fit(self, X, y):
         """Fit the k-nearest neighbors regressor from the training dataset.
 
@@ -189,22 +222,28 @@ def fit(self, X, y):
         return self._fit(X, y)
 
     def predict(self, X):
-        """Predict the target for the provided data
+        """Predict the target for the provided data.
 
         Parameters
         ----------
-        X : array-like of shape (n_queries, n_features), \
-                or (n_queries, n_indexed) if metric == 'precomputed'
-            Test samples.
+        X : {array-like, sparse matrix} of shape (n_queries, n_features), \
+                or (n_queries, n_indexed) if metric == 'precomputed', or None
+            Test samples. If `None`, predictions for all indexed points are
+            returned; in this case, points are not considered their own
+            neighbors.
 
         Returns
         -------
         y : ndarray of shape (n_queries,) or (n_queries, n_outputs), dtype=int
             Target values.
         """
-        X = self._validate_data(X, accept_sparse='csr', reset=False)
-
-        neigh_dist, neigh_ind = self.kneighbors(X)
+        if self.weights == "uniform":
+            # In that case, we do not need the distances to perform
+            # the weighting so we do not compute them.
+            neigh_ind = self.kneighbors(X, return_distance=False)
+            neigh_dist = None
+        else:
+            neigh_dist, neigh_ind = self.kneighbors(X)
 
         weights = _get_weights(neigh_dist, self.weights)
 
@@ -215,7 +254,7 @@ def predict(self, X):
         if weights is None:
             y_pred = np.mean(_y[neigh_ind], axis=1)
         else:
-            y_pred = np.empty((X.shape[0], _y.shape[1]), dtype=np.float64)
+            y_pred = np.empty((neigh_dist.shape[0], _y.shape[1]), dtype=np.float64)
             denom = np.sum(weights, axis=1)
 
             for j in range(_y.shape[1]):
@@ -228,9 +267,7 @@ def predict(self, X):
         return y_pred
 
 
-class RadiusNeighborsRegressor(RadiusNeighborsMixin,
-                               RegressorMixin,
-                               NeighborsBase):
+class RadiusNeighborsRegressor(RadiusNeighborsMixin, RegressorMixin, NeighborsBase):
     """Regression based on neighbors within a fixed radius.
 
     The target is predicted by local interpolation of the targets
@@ -246,8 +283,8 @@ class RadiusNeighborsRegressor(RadiusNeighborsMixin,
         Range of parameter space to use by default for :meth:`radius_neighbors`
         queries.
 
-    weights : {'uniform', 'distance'} or callable, default='uniform'
-        weight function used in prediction.  Possible values:
+    weights : {'uniform', 'distance'}, callable or None, default='uniform'
+        Weight function used in prediction.  Possible values:
 
         - 'uniform' : uniform weights.  All points in each neighborhood
           are weighted equally.
@@ -278,19 +315,28 @@ class RadiusNeighborsRegressor(RadiusNeighborsMixin,
         required to store the tree.  The optimal value depends on the
         nature of the problem.
 
-    p : int, default=2
+    p : float, default=2
         Power parameter for the Minkowski metric. When p = 1, this is
         equivalent to using manhattan_distance (l1), and euclidean_distance
         (l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used.
 
     metric : str or callable, default='minkowski'
-        the distance metric to use for the tree.  The default metric is
-        minkowski, and with p=2 is equivalent to the standard Euclidean
-        metric. See the documentation of :class:`DistanceMetric` for a
-        list of available metrics.
+        Metric to use for distance computation. Default is "minkowski", which
+        results in the standard Euclidean distance when p = 2. See the
+        documentation of `scipy.spatial.distance
+        `_ and
+        the metrics listed in
+        :class:`~sklearn.metrics.pairwise.distance_metrics` for valid metric
+        values.
+
         If metric is "precomputed", X is assumed to be a distance matrix and
-        must be square during fit. X may be a :term:`sparse graph`,
-        in which case only "nonzero" elements may be considered neighbors.
+        must be square during fit. X may be a :term:`sparse graph`, in which
+        case only "nonzero" elements may be considered neighbors.
+
+        If metric is a callable function, it takes two arrays representing 1D
+        vectors as inputs and must return one value indicating the distance
+        between those vectors. This works for Scipy's metrics, but is less
+        efficient than passing the metric name as a string.
 
     metric_params : dict, default=None
         Additional keyword arguments for the metric function.
@@ -314,9 +360,34 @@ class RadiusNeighborsRegressor(RadiusNeighborsMixin,
         `p` parameter value if the `effective_metric_` attribute is set to
         'minkowski'.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     n_samples_fit_ : int
         Number of samples in the fitted data.
 
+    See Also
+    --------
+    NearestNeighbors : Unsupervised learner for implementing neighbor searches.
+    KNeighborsRegressor : Regression based on k-nearest neighbors.
+    KNeighborsClassifier : Classifier based on the k-nearest neighbors.
+    RadiusNeighborsClassifier : Classifier based on neighbors within a given radius.
+
+    Notes
+    -----
+    See :ref:`Nearest Neighbors ` in the online documentation
+    for a discussion of the choice of ``algorithm`` and ``leaf_size``.
+
+    https://en.wikipedia.org/wiki/K-nearest_neighbor_algorithm
+
     Examples
     --------
     >>> X = [[0], [1], [2], [3]]
@@ -327,35 +398,41 @@ class RadiusNeighborsRegressor(RadiusNeighborsMixin,
     RadiusNeighborsRegressor(...)
     >>> print(neigh.predict([[1.5]]))
     [0.5]
-
-    See Also
-    --------
-    NearestNeighbors
-    KNeighborsRegressor
-    KNeighborsClassifier
-    RadiusNeighborsClassifier
-
-    Notes
-    -----
-    See :ref:`Nearest Neighbors ` in the online documentation
-    for a discussion of the choice of ``algorithm`` and ``leaf_size``.
-
-    https://en.wikipedia.org/wiki/K-nearest_neighbor_algorithm
     """
 
-    @_deprecate_positional_args
-    def __init__(self, radius=1.0, *, weights='uniform',
-                 algorithm='auto', leaf_size=30,
-                 p=2, metric='minkowski', metric_params=None, n_jobs=None,
-                 **kwargs):
+    _parameter_constraints: dict = {
+        **NeighborsBase._parameter_constraints,
+        "weights": [StrOptions({"uniform", "distance"}), callable, None],
+    }
+    _parameter_constraints.pop("n_neighbors")
+
+    def __init__(
+        self,
+        radius=1.0,
+        *,
+        weights="uniform",
+        algorithm="auto",
+        leaf_size=30,
+        p=2,
+        metric="minkowski",
+        metric_params=None,
+        n_jobs=None,
+    ):
         super().__init__(
-              radius=radius,
-              algorithm=algorithm,
-              leaf_size=leaf_size,
-              p=p, metric=metric, metric_params=metric_params,
-              n_jobs=n_jobs, **kwargs)
-        self.weights = _check_weights(weights)
-
+            radius=radius,
+            algorithm=algorithm,
+            leaf_size=leaf_size,
+            p=p,
+            metric=metric,
+            metric_params=metric_params,
+            n_jobs=n_jobs,
+        )
+        self.weights = weights
+
+    @_fit_context(
+        # RadiusNeighborsRegressor.metric is not validated yet
+        prefer_skip_nested_validation=False
+    )
     def fit(self, X, y):
         """Fit the radius neighbors regressor from the training dataset.
 
@@ -377,13 +454,15 @@ def fit(self, X, y):
         return self._fit(X, y)
 
     def predict(self, X):
-        """Predict the target for the provided data
+        """Predict the target for the provided data.
 
         Parameters
         ----------
-        X : array-like of shape (n_queries, n_features), \
-                or (n_queries, n_indexed) if metric == 'precomputed'
-            Test samples.
+        X : {array-like, sparse matrix} of shape (n_queries, n_features), \
+                or (n_queries, n_indexed) if metric == 'precomputed', or None
+            Test samples. If `None`, predictions for all indexed points are
+            returned; in this case, points are not considered their own
+            neighbors.
 
         Returns
         -------
@@ -391,8 +470,6 @@ def predict(self, X):
                 dtype=double
             Target values.
         """
-        X = self._validate_data(X, accept_sparse='csr', reset=False)
-
         neigh_dist, neigh_ind = self.radius_neighbors(X)
 
         weights = _get_weights(neigh_dist, self.weights)
@@ -404,19 +481,30 @@ def predict(self, X):
         empty_obs = np.full_like(_y[0], np.nan)
 
         if weights is None:
-            y_pred = np.array([np.mean(_y[ind, :], axis=0)
-                               if len(ind) else empty_obs
-                               for (i, ind) in enumerate(neigh_ind)])
+            y_pred = np.array(
+                [
+                    np.mean(_y[ind, :], axis=0) if len(ind) else empty_obs
+                    for (i, ind) in enumerate(neigh_ind)
+                ]
+            )
 
         else:
-            y_pred = np.array([np.average(_y[ind, :], axis=0,
-                               weights=weights[i])
-                               if len(ind) else empty_obs
-                               for (i, ind) in enumerate(neigh_ind)])
+            y_pred = np.array(
+                [
+                    (
+                        np.average(_y[ind, :], axis=0, weights=weights[i])
+                        if len(ind)
+                        else empty_obs
+                    )
+                    for (i, ind) in enumerate(neigh_ind)
+                ]
+            )
 
         if np.any(np.isnan(y_pred)):
-            empty_warning_msg = ("One or more samples have no neighbors "
-                                 "within specified radius; predicting NaN.")
+            empty_warning_msg = (
+                "One or more samples have no neighbors "
+                "within specified radius; predicting NaN."
+            )
             warnings.warn(empty_warning_msg)
 
         if self._y.ndim == 1:
diff --git a/sklearn/neighbors/_typedefs.pxd b/sklearn/neighbors/_typedefs.pxd
deleted file mode 100644
index b121953d15e4e..0000000000000
--- a/sklearn/neighbors/_typedefs.pxd
+++ /dev/null
@@ -1,18 +0,0 @@
-#!python
-cimport numpy as np
-
-# Floating point/data type
-ctypedef np.float64_t DTYPE_t  # WARNING: should match DTYPE in typedefs.pyx
-
-cdef enum:
-    DTYPECODE = np.NPY_FLOAT64
-    ITYPECODE = np.NPY_INTP
-
-# Index/integer type.
-#  WARNING: ITYPE_t must be a signed integer type or you will have a bad time!
-ctypedef np.intp_t ITYPE_t  # WARNING: should match ITYPE in typedefs.pyx
-
-# Fused type for certain operations
-ctypedef fused DITYPE_t:
-    ITYPE_t
-    DTYPE_t
diff --git a/sklearn/neighbors/_typedefs.pyx b/sklearn/neighbors/_typedefs.pyx
deleted file mode 100644
index 789afb4997dd1..0000000000000
--- a/sklearn/neighbors/_typedefs.pyx
+++ /dev/null
@@ -1,26 +0,0 @@
-#!python
-
-import numpy as np
-cimport numpy as np
-from libc.math cimport sqrt
-
-np.import_array()
-
-
-# use a hack to determine the associated numpy data types
-# NOTE: the following requires the buffer interface, only available in
-#       numpy 1.5+.  We'll choose the DTYPE by hand instead.
-#cdef ITYPE_t idummy
-#cdef ITYPE_t[:] idummy_view =  &idummy
-#ITYPE = np.asarray(idummy_view).dtype
-ITYPE = np.intp  # WARNING: this should match ITYPE_t in typedefs.pxd
-
-#cdef DTYPE_t ddummy
-#cdef DTYPE_t[:] ddummy_view =  &ddummy
-#DTYPE = np.asarray(ddummy_view).dtype
-DTYPE = np.float64  # WARNING: this should match DTYPE_t in typedefs.pxd
-
-# some handy constants
-cdef DTYPE_t INF = np.inf
-cdef DTYPE_t PI = np.pi
-cdef DTYPE_t ROOT_2PI = sqrt(2 * PI)
diff --git a/sklearn/neighbors/_unsupervised.py b/sklearn/neighbors/_unsupervised.py
index 822a30f503bd2..8888fe18483c6 100644
--- a/sklearn/neighbors/_unsupervised.py
+++ b/sklearn/neighbors/_unsupervised.py
@@ -1,13 +1,13 @@
 """Unsupervised nearest neighbors learner"""
-from ._base import NeighborsBase
-from ._base import KNeighborsMixin
-from ._base import RadiusNeighborsMixin
-from ..utils.validation import _deprecate_positional_args
 
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-class NearestNeighbors(KNeighborsMixin,
-                       RadiusNeighborsMixin,
-                       NeighborsBase):
+from ..base import _fit_context
+from ._base import KNeighborsMixin, NeighborsBase, RadiusNeighborsMixin
+
+
+class NearestNeighbors(KNeighborsMixin, RadiusNeighborsMixin, NeighborsBase):
     """Unsupervised learner for implementing neighbor searches.
 
     Read more in the :ref:`User Guide `.
@@ -42,15 +42,24 @@ class NearestNeighbors(KNeighborsMixin,
         nature of the problem.
 
     metric : str or callable, default='minkowski'
-        the distance metric to use for the tree.  The default metric is
-        minkowski, and with p=2 is equivalent to the standard Euclidean
-        metric. See the documentation of :class:`DistanceMetric` for a
-        list of available metrics.
+        Metric to use for distance computation. Default is "minkowski", which
+        results in the standard Euclidean distance when p = 2. See the
+        documentation of `scipy.spatial.distance
+        `_ and
+        the metrics listed in
+        :class:`~sklearn.metrics.pairwise.distance_metrics` for valid metric
+        values.
+
         If metric is "precomputed", X is assumed to be a distance matrix and
-        must be square during fit. X may be a :term:`sparse graph`,
-        in which case only "nonzero" elements may be considered neighbors.
+        must be square during fit. X may be a :term:`sparse graph`, in which
+        case only "nonzero" elements may be considered neighbors.
+
+        If metric is a callable function, it takes two arrays representing 1D
+        vectors as inputs and must return one value indicating the distance
+        between those vectors. This works for Scipy's metrics, but is less
+        efficient than passing the metric name as a string.
 
-    p : int, default=2
+    p : float (positive), default=2
         Parameter for the Minkowski metric from
         sklearn.metrics.pairwise.pairwise_distances. When p = 1, this is
         equivalent to using manhattan_distance (l1), and euclidean_distance
@@ -73,55 +82,83 @@ class NearestNeighbors(KNeighborsMixin,
     effective_metric_params_ : dict
         Parameters for the metric used to compute distances to neighbors.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     n_samples_fit_ : int
         Number of samples in the fitted data.
 
+    See Also
+    --------
+    KNeighborsClassifier : Classifier implementing the k-nearest neighbors
+        vote.
+    RadiusNeighborsClassifier : Classifier implementing a vote among neighbors
+        within a given radius.
+    KNeighborsRegressor : Regression based on k-nearest neighbors.
+    RadiusNeighborsRegressor : Regression based on neighbors within a fixed
+        radius.
+    BallTree : Space partitioning data structure for organizing points in a
+        multi-dimensional space, used for nearest neighbor search.
+
+    Notes
+    -----
+    See :ref:`Nearest Neighbors ` in the online documentation
+    for a discussion of the choice of ``algorithm`` and ``leaf_size``.
+
+    https://en.wikipedia.org/wiki/K-nearest_neighbors_algorithm
+
     Examples
     --------
     >>> import numpy as np
     >>> from sklearn.neighbors import NearestNeighbors
     >>> samples = [[0, 0, 2], [1, 0, 0], [0, 0, 1]]
-
     >>> neigh = NearestNeighbors(n_neighbors=2, radius=0.4)
     >>> neigh.fit(samples)
     NearestNeighbors(...)
-
     >>> neigh.kneighbors([[0, 0, 1.3]], 2, return_distance=False)
     array([[2, 0]]...)
-
     >>> nbrs = neigh.radius_neighbors(
     ...    [[0, 0, 1.3]], 0.4, return_distance=False
     ... )
     >>> np.asarray(nbrs[0][0])
     array(2)
-
-    See Also
-    --------
-    KNeighborsClassifier
-    RadiusNeighborsClassifier
-    KNeighborsRegressor
-    RadiusNeighborsRegressor
-    BallTree
-
-    Notes
-    -----
-    See :ref:`Nearest Neighbors ` in the online documentation
-    for a discussion of the choice of ``algorithm`` and ``leaf_size``.
-
-    https://en.wikipedia.org/wiki/K-nearest_neighbor_algorithm
     """
 
-    @_deprecate_positional_args
-    def __init__(self, *, n_neighbors=5, radius=1.0,
-                 algorithm='auto', leaf_size=30, metric='minkowski',
-                 p=2, metric_params=None, n_jobs=None):
+    def __init__(
+        self,
+        *,
+        n_neighbors=5,
+        radius=1.0,
+        algorithm="auto",
+        leaf_size=30,
+        metric="minkowski",
+        p=2,
+        metric_params=None,
+        n_jobs=None,
+    ):
         super().__init__(
-              n_neighbors=n_neighbors,
-              radius=radius,
-              algorithm=algorithm,
-              leaf_size=leaf_size, metric=metric, p=p,
-              metric_params=metric_params, n_jobs=n_jobs)
-
+            n_neighbors=n_neighbors,
+            radius=radius,
+            algorithm=algorithm,
+            leaf_size=leaf_size,
+            metric=metric,
+            p=p,
+            metric_params=metric_params,
+            n_jobs=n_jobs,
+        )
+
+    @_fit_context(
+        # NearestNeighbors.metric is not validated yet
+        prefer_skip_nested_validation=False
+    )
     def fit(self, X, y=None):
         """Fit the nearest neighbors estimator from the training dataset.
 
diff --git a/sklearn/neighbors/meson.build b/sklearn/neighbors/meson.build
new file mode 100644
index 0000000000000..7993421896218
--- /dev/null
+++ b/sklearn/neighbors/meson.build
@@ -0,0 +1,53 @@
+_binary_tree_pxi = custom_target(
+  '_binary_tree_pxi',
+  output: '_binary_tree.pxi',
+  input: '_binary_tree.pxi.tp',
+  command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
+)
+
+# .pyx is generated so this is needed to make Cython compilation work. The pxi
+# file is included avoid "missing dependency paths" with ninja -t missindeps
+neighbors_cython_tree = [
+  fs.copyfile('__init__.py'),
+  fs.copyfile('_partition_nodes.pxd'),
+  _binary_tree_pxi,
+]
+
+name_list = ['_ball_tree', '_kd_tree']
+
+foreach name: name_list
+  pyx = custom_target(
+    name + '_pyx',
+    output: name + '.pyx',
+    input: name + '.pyx.tp',
+    command: [tempita, '@INPUT@', '-o', '@OUTDIR@'],
+    # TODO in principle this should go in py.exension_module below. This is
+    # temporary work-around for dependency issue with .pyx.tp files. For more
+    # details, see https://github.com/mesonbuild/meson/issues/13212
+    depends: [neighbors_cython_tree, utils_cython_tree, metrics_cython_tree],
+  )
+  py.extension_module(
+    name,
+    cython_gen.process(pyx),
+    dependencies: [np_dep],
+    subdir: 'sklearn/neighbors',
+    install: true
+)
+endforeach
+
+neighbors_extension_metadata = {
+  '_partition_nodes':
+      {'sources': [cython_gen_cpp.process('_partition_nodes.pyx')],
+       'dependencies': [np_dep]},
+  '_quad_tree': {'sources': [cython_gen.process('_quad_tree.pyx')], 'dependencies': [np_dep]},
+}
+
+foreach ext_name, ext_dict : neighbors_extension_metadata
+  py.extension_module(
+    ext_name,
+    [ext_dict.get('sources'), utils_cython_tree],
+    dependencies: ext_dict.get('dependencies'),
+    subdir: 'sklearn/neighbors',
+    install: true
+  )
+endforeach
diff --git a/sklearn/neighbors/setup.py b/sklearn/neighbors/setup.py
deleted file mode 100644
index 9264044678193..0000000000000
--- a/sklearn/neighbors/setup.py
+++ /dev/null
@@ -1,41 +0,0 @@
-import os
-
-
-def configuration(parent_package='', top_path=None):
-    import numpy
-    from numpy.distutils.misc_util import Configuration
-
-    config = Configuration('neighbors', parent_package, top_path)
-    libraries = []
-    if os.name == 'posix':
-        libraries.append('m')
-
-    config.add_extension('_ball_tree',
-                         sources=['_ball_tree.pyx'],
-                         include_dirs=[numpy.get_include()],
-                         libraries=libraries)
-
-    config.add_extension('_kd_tree',
-                         sources=['_kd_tree.pyx'],
-                         include_dirs=[numpy.get_include()],
-                         libraries=libraries)
-
-    config.add_extension('_dist_metrics',
-                         sources=['_dist_metrics.pyx'],
-                         include_dirs=[numpy.get_include(),
-                                       os.path.join(numpy.get_include(),
-                                                    'numpy')],
-                         libraries=libraries)
-
-    config.add_extension('_typedefs',
-                         sources=['_typedefs.pyx'],
-                         include_dirs=[numpy.get_include()],
-                         libraries=libraries)
-    config.add_extension("_quad_tree",
-                         sources=["_quad_tree.pyx"],
-                         include_dirs=[numpy.get_include()],
-                         libraries=libraries)
-
-    config.add_subpackage('tests')
-
-    return config
diff --git a/sklearn/neighbors/tests/test_ball_tree.py b/sklearn/neighbors/tests/test_ball_tree.py
index ae88c71ff497b..5263f201f320b 100644
--- a/sklearn/neighbors/tests/test_ball_tree.py
+++ b/sklearn/neighbors/tests/test_ball_tree.py
@@ -2,12 +2,12 @@
 
 import numpy as np
 import pytest
-from numpy.testing import assert_array_almost_equal
-from sklearn.neighbors._ball_tree import BallTree
-from sklearn.neighbors import DistanceMetric
+from numpy.testing import assert_allclose, assert_array_almost_equal, assert_equal
+
+from sklearn.neighbors._ball_tree import BallTree, BallTree32, BallTree64
 from sklearn.utils import check_random_state
-from sklearn.utils.validation import check_array
 from sklearn.utils._testing import _convert_container
+from sklearn.utils.validation import check_array
 
 rng = np.random.RandomState(10)
 V_mahalanobis = rng.rand(3, 3)
@@ -15,24 +15,33 @@
 
 DIMENSION = 3
 
-METRICS = {'euclidean': {},
-           'manhattan': {},
-           'minkowski': dict(p=3),
-           'chebyshev': {},
-           'seuclidean': dict(V=rng.random_sample(DIMENSION)),
-           'wminkowski': dict(p=3, w=rng.random_sample(DIMENSION)),
-           'mahalanobis': dict(V=V_mahalanobis)}
+METRICS = {
+    "euclidean": {},
+    "manhattan": {},
+    "minkowski": dict(p=3),
+    "chebyshev": {},
+}
 
-DISCRETE_METRICS = ['hamming',
-                    'canberra',
-                    'braycurtis']
+DISCRETE_METRICS = ["hamming", "canberra", "braycurtis"]
 
-BOOLEAN_METRICS = ['matching', 'jaccard', 'dice', 'kulsinski',
-                   'rogerstanimoto', 'russellrao', 'sokalmichener',
-                   'sokalsneath']
+BOOLEAN_METRICS = [
+    "jaccard",
+    "dice",
+    "rogerstanimoto",
+    "russellrao",
+    "sokalmichener",
+    "sokalsneath",
+]
+
+BALL_TREE_CLASSES = [
+    BallTree64,
+    BallTree32,
+]
 
 
 def brute_force_neighbors(X, Y, k, metric, **kwargs):
+    from sklearn.metrics import DistanceMetric
+
     X, Y = check_array(X), check_array(Y)
     D = DistanceMetric.get_metric(metric, **kwargs).pairwise(Y, X)
     ind = np.argsort(D, axis=1)[:, :k]
@@ -40,12 +49,14 @@ def brute_force_neighbors(X, Y, k, metric, **kwargs):
     return dist, ind
 
 
-@pytest.mark.parametrize(
-    'metric',
-    itertools.chain(BOOLEAN_METRICS, DISCRETE_METRICS)
-)
+def test_BallTree_is_BallTree64_subclass():
+    assert issubclass(BallTree, BallTree64)
+
+
+@pytest.mark.parametrize("metric", itertools.chain(BOOLEAN_METRICS, DISCRETE_METRICS))
 @pytest.mark.parametrize("array_type", ["list", "array"])
-def test_ball_tree_query_metrics(metric, array_type):
+@pytest.mark.parametrize("BallTreeImplementation", BALL_TREE_CLASSES)
+def test_ball_tree_query_metrics(metric, array_type, BallTreeImplementation):
     rng = check_random_state(0)
     if metric in BOOLEAN_METRICS:
         X = rng.random_sample((40, 10)).round(0)
@@ -58,28 +69,132 @@ def test_ball_tree_query_metrics(metric, array_type):
 
     k = 5
 
-    bt = BallTree(X, leaf_size=1, metric=metric)
+    bt = BallTreeImplementation(X, leaf_size=1, metric=metric)
     dist1, ind1 = bt.query(Y, k)
     dist2, ind2 = brute_force_neighbors(X, Y, k, metric)
     assert_array_almost_equal(dist1, dist2)
 
 
-def test_query_haversine():
+@pytest.mark.parametrize(
+    "BallTreeImplementation, decimal_tol", zip(BALL_TREE_CLASSES, [6, 5])
+)
+def test_query_haversine(BallTreeImplementation, decimal_tol):
     rng = check_random_state(0)
     X = 2 * np.pi * rng.random_sample((40, 2))
-    bt = BallTree(X, leaf_size=1, metric='haversine')
+    bt = BallTreeImplementation(X, leaf_size=1, metric="haversine")
     dist1, ind1 = bt.query(X, k=5)
-    dist2, ind2 = brute_force_neighbors(X, X, k=5, metric='haversine')
+    dist2, ind2 = brute_force_neighbors(X, X, k=5, metric="haversine")
 
-    assert_array_almost_equal(dist1, dist2)
+    assert_array_almost_equal(dist1, dist2, decimal=decimal_tol)
     assert_array_almost_equal(ind1, ind2)
 
 
-def test_array_object_type():
+@pytest.mark.parametrize("BallTreeImplementation", BALL_TREE_CLASSES)
+def test_array_object_type(BallTreeImplementation):
     """Check that we do not accept object dtype array."""
     X = np.array([(1, 2, 3), (2, 5), (5, 5, 1, 2)], dtype=object)
-    with pytest.raises(
-        ValueError,
-        match="setting an array element with a sequence"
-    ):
-        BallTree(X)
+    with pytest.raises(ValueError, match="setting an array element with a sequence"):
+        BallTreeImplementation(X)
+
+
+@pytest.mark.parametrize("BallTreeImplementation", BALL_TREE_CLASSES)
+def test_bad_pyfunc_metric(BallTreeImplementation):
+    def wrong_returned_value(x, y):
+        return "1"
+
+    def one_arg_func(x):
+        return 1.0  # pragma: no cover
+
+    X = np.ones((5, 2))
+    msg = "Custom distance function must accept two vectors and return a float."
+    with pytest.raises(TypeError, match=msg):
+        BallTreeImplementation(X, metric=wrong_returned_value)
+
+    msg = "takes 1 positional argument but 2 were given"
+    with pytest.raises(TypeError, match=msg):
+        BallTreeImplementation(X, metric=one_arg_func)
+
+
+@pytest.mark.parametrize("metric", itertools.chain(METRICS, BOOLEAN_METRICS))
+def test_ball_tree_numerical_consistency(global_random_seed, metric):
+    # Results on float64 and float32 versions of a dataset must be
+    # numerically close.
+    X_64, X_32, Y_64, Y_32 = get_dataset_for_binary_tree(
+        random_seed=global_random_seed, features=50
+    )
+
+    metric_params = METRICS.get(metric, {})
+    bt_64 = BallTree64(X_64, leaf_size=1, metric=metric, **metric_params)
+    bt_32 = BallTree32(X_32, leaf_size=1, metric=metric, **metric_params)
+
+    # Test consistency with respect to the `query` method
+    k = 5
+    dist_64, ind_64 = bt_64.query(Y_64, k=k)
+    dist_32, ind_32 = bt_32.query(Y_32, k=k)
+    assert_allclose(dist_64, dist_32, rtol=1e-5)
+    assert_equal(ind_64, ind_32)
+    assert dist_64.dtype == np.float64
+    assert dist_32.dtype == np.float32
+
+    # Test consistency with respect to the `query_radius` method
+    r = 2.38
+    ind_64 = bt_64.query_radius(Y_64, r=r)
+    ind_32 = bt_32.query_radius(Y_32, r=r)
+    for _ind64, _ind32 in zip(ind_64, ind_32):
+        assert_equal(_ind64, _ind32)
+
+    # Test consistency with respect to the `query_radius` method
+    # with return distances being true
+    ind_64, dist_64 = bt_64.query_radius(Y_64, r=r, return_distance=True)
+    ind_32, dist_32 = bt_32.query_radius(Y_32, r=r, return_distance=True)
+    for _ind64, _ind32, _dist_64, _dist_32 in zip(ind_64, ind_32, dist_64, dist_32):
+        assert_equal(_ind64, _ind32)
+        assert_allclose(_dist_64, _dist_32, rtol=1e-5)
+        assert _dist_64.dtype == np.float64
+        assert _dist_32.dtype == np.float32
+
+
+@pytest.mark.parametrize("metric", itertools.chain(METRICS, BOOLEAN_METRICS))
+def test_kernel_density_numerical_consistency(global_random_seed, metric):
+    # Test consistency with respect to the `kernel_density` method
+    X_64, X_32, Y_64, Y_32 = get_dataset_for_binary_tree(random_seed=global_random_seed)
+
+    metric_params = METRICS.get(metric, {})
+    bt_64 = BallTree64(X_64, leaf_size=1, metric=metric, **metric_params)
+    bt_32 = BallTree32(X_32, leaf_size=1, metric=metric, **metric_params)
+
+    kernel = "gaussian"
+    h = 0.1
+    density64 = bt_64.kernel_density(Y_64, h=h, kernel=kernel, breadth_first=True)
+    density32 = bt_32.kernel_density(Y_32, h=h, kernel=kernel, breadth_first=True)
+    assert_allclose(density64, density32, rtol=1e-5)
+    assert density64.dtype == np.float64
+    assert density32.dtype == np.float32
+
+
+def test_two_point_correlation_numerical_consistency(global_random_seed):
+    # Test consistency with respect to the `two_point_correlation` method
+    X_64, X_32, Y_64, Y_32 = get_dataset_for_binary_tree(random_seed=global_random_seed)
+
+    bt_64 = BallTree64(X_64, leaf_size=10)
+    bt_32 = BallTree32(X_32, leaf_size=10)
+
+    r = np.linspace(0, 1, 10)
+
+    counts_64 = bt_64.two_point_correlation(Y_64, r=r, dualtree=True)
+    counts_32 = bt_32.two_point_correlation(Y_32, r=r, dualtree=True)
+    assert_allclose(counts_64, counts_32)
+
+
+def get_dataset_for_binary_tree(random_seed, features=3):
+    rng = np.random.RandomState(random_seed)
+    _X = rng.rand(100, features)
+    _Y = rng.rand(5, features)
+
+    X_64 = _X.astype(dtype=np.float64, copy=False)
+    Y_64 = _Y.astype(dtype=np.float64, copy=False)
+
+    X_32 = _X.astype(dtype=np.float32, copy=False)
+    Y_32 = _Y.astype(dtype=np.float32, copy=False)
+
+    return X_64, X_32, Y_64, Y_32
diff --git a/sklearn/neighbors/tests/test_dist_metrics.py b/sklearn/neighbors/tests/test_dist_metrics.py
deleted file mode 100644
index 05e0f4294ebb6..0000000000000
--- a/sklearn/neighbors/tests/test_dist_metrics.py
+++ /dev/null
@@ -1,226 +0,0 @@
-import itertools
-import pickle
-
-import numpy as np
-from numpy.testing import assert_array_almost_equal
-
-import pytest
-
-from scipy.spatial.distance import cdist
-from sklearn.neighbors import DistanceMetric
-from sklearn.neighbors import BallTree
-from sklearn.utils import check_random_state
-from sklearn.utils.fixes import sp_version, parse_version
-
-
-def dist_func(x1, x2, p):
-    return np.sum((x1 - x2) ** p) ** (1. / p)
-
-
-rng = check_random_state(0)
-d = 4
-n1 = 20
-n2 = 25
-X1 = rng.random_sample((n1, d)).astype('float64', copy=False)
-X2 = rng.random_sample((n2, d)).astype('float64', copy=False)
-
-# make boolean arrays: ones and zeros
-X1_bool = X1.round(0)
-X2_bool = X2.round(0)
-
-V = rng.random_sample((d, d))
-VI = np.dot(V, V.T)
-
-BOOL_METRICS = ['matching', 'jaccard', 'dice',
-                'kulsinski', 'rogerstanimoto', 'russellrao',
-                'sokalmichener', 'sokalsneath']
-
-METRICS_DEFAULT_PARAMS = {'euclidean': {},
-                          'cityblock': {},
-                          'minkowski': dict(p=(1, 1.5, 2, 3)),
-                          'chebyshev': {},
-                          'seuclidean': dict(V=(rng.random_sample(d),)),
-                          'wminkowski': dict(p=(1, 1.5, 3),
-                                             w=(rng.random_sample(d),)),
-                          'mahalanobis': dict(VI=(VI,)),
-                          'hamming': {},
-                          'canberra': {},
-                          'braycurtis': {}}
-
-
-@pytest.mark.parametrize('metric', METRICS_DEFAULT_PARAMS)
-def test_cdist(metric):
-    argdict = METRICS_DEFAULT_PARAMS[metric]
-    keys = argdict.keys()
-    for vals in itertools.product(*argdict.values()):
-        kwargs = dict(zip(keys, vals))
-        if metric == "wminkowski":
-            if sp_version >= parse_version("1.8.0"):
-                pytest.skip("wminkowski will be removed in SciPy 1.8.0")
-
-            # wminkoski is deprecated in SciPy 1.6.0 and removed in 1.8.0
-            ExceptionToAssert = None
-            if sp_version >= parse_version("1.6.0"):
-                ExceptionToAssert = DeprecationWarning
-            with pytest.warns(ExceptionToAssert):
-                D_true = cdist(X1, X2, metric, **kwargs)
-        else:
-            D_true = cdist(X1, X2, metric, **kwargs)
-
-        check_cdist(metric, kwargs, D_true)
-
-
-@pytest.mark.parametrize('metric', BOOL_METRICS)
-def test_cdist_bool_metric(metric):
-    D_true = cdist(X1_bool, X2_bool, metric)
-    check_cdist_bool(metric, D_true)
-
-
-def check_cdist(metric, kwargs, D_true):
-    dm = DistanceMetric.get_metric(metric, **kwargs)
-    D12 = dm.pairwise(X1, X2)
-    assert_array_almost_equal(D12, D_true)
-
-
-def check_cdist_bool(metric, D_true):
-    dm = DistanceMetric.get_metric(metric)
-    D12 = dm.pairwise(X1_bool, X2_bool)
-    assert_array_almost_equal(D12, D_true)
-
-
-@pytest.mark.parametrize('metric', METRICS_DEFAULT_PARAMS)
-def test_pdist(metric):
-    argdict = METRICS_DEFAULT_PARAMS[metric]
-    keys = argdict.keys()
-    for vals in itertools.product(*argdict.values()):
-        kwargs = dict(zip(keys, vals))
-        if metric == "wminkowski":
-            if sp_version >= parse_version("1.8.0"):
-                pytest.skip("wminkowski will be removed in SciPy 1.8.0")
-
-            # wminkoski is deprecated in SciPy 1.6.0 and removed in 1.8.0
-            ExceptionToAssert = None
-            if sp_version >= parse_version("1.6.0"):
-                ExceptionToAssert = DeprecationWarning
-            with pytest.warns(ExceptionToAssert):
-                D_true = cdist(X1, X1, metric, **kwargs)
-        else:
-            D_true = cdist(X1, X1, metric, **kwargs)
-
-        check_pdist(metric, kwargs, D_true)
-
-
-@pytest.mark.parametrize('metric', BOOL_METRICS)
-def test_pdist_bool_metrics(metric):
-    D_true = cdist(X1_bool, X1_bool, metric)
-    check_pdist_bool(metric, D_true)
-
-
-def check_pdist(metric, kwargs, D_true):
-    dm = DistanceMetric.get_metric(metric, **kwargs)
-    D12 = dm.pairwise(X1)
-    assert_array_almost_equal(D12, D_true)
-
-
-def check_pdist_bool(metric, D_true):
-    dm = DistanceMetric.get_metric(metric)
-    D12 = dm.pairwise(X1_bool)
-    # Based on https://github.com/scipy/scipy/pull/7373
-    # When comparing two all-zero vectors, scipy>=1.2.0 jaccard metric
-    # was changed to return 0, instead of nan.
-    if metric == 'jaccard' and sp_version < parse_version('1.2.0'):
-        D_true[np.isnan(D_true)] = 0
-    assert_array_almost_equal(D12, D_true)
-
-
-@pytest.mark.parametrize('metric', METRICS_DEFAULT_PARAMS)
-def test_pickle(metric):
-    argdict = METRICS_DEFAULT_PARAMS[metric]
-    keys = argdict.keys()
-    for vals in itertools.product(*argdict.values()):
-        kwargs = dict(zip(keys, vals))
-        check_pickle(metric, kwargs)
-
-
-@pytest.mark.parametrize('metric', BOOL_METRICS)
-def test_pickle_bool_metrics(metric):
-    dm = DistanceMetric.get_metric(metric)
-    D1 = dm.pairwise(X1_bool)
-    dm2 = pickle.loads(pickle.dumps(dm))
-    D2 = dm2.pairwise(X1_bool)
-    assert_array_almost_equal(D1, D2)
-
-
-def check_pickle(metric, kwargs):
-    dm = DistanceMetric.get_metric(metric, **kwargs)
-    D1 = dm.pairwise(X1)
-    dm2 = pickle.loads(pickle.dumps(dm))
-    D2 = dm2.pairwise(X1)
-    assert_array_almost_equal(D1, D2)
-
-
-def test_haversine_metric():
-    def haversine_slow(x1, x2):
-        return 2 * np.arcsin(np.sqrt(np.sin(0.5 * (x1[0] - x2[0])) ** 2
-                                     + np.cos(x1[0]) * np.cos(x2[0]) *
-                                     np.sin(0.5 * (x1[1] - x2[1])) ** 2))
-
-    X = np.random.random((10, 2))
-
-    haversine = DistanceMetric.get_metric("haversine")
-
-    D1 = haversine.pairwise(X)
-    D2 = np.zeros_like(D1)
-    for i, x1 in enumerate(X):
-        for j, x2 in enumerate(X):
-            D2[i, j] = haversine_slow(x1, x2)
-
-    assert_array_almost_equal(D1, D2)
-    assert_array_almost_equal(haversine.dist_to_rdist(D1),
-                              np.sin(0.5 * D2) ** 2)
-
-
-def test_pyfunc_metric():
-    X = np.random.random((10, 3))
-
-    euclidean = DistanceMetric.get_metric("euclidean")
-    pyfunc = DistanceMetric.get_metric("pyfunc", func=dist_func, p=2)
-
-    # Check if both callable metric and predefined metric initialized
-    # DistanceMetric object is picklable
-    euclidean_pkl = pickle.loads(pickle.dumps(euclidean))
-    pyfunc_pkl = pickle.loads(pickle.dumps(pyfunc))
-
-    D1 = euclidean.pairwise(X)
-    D2 = pyfunc.pairwise(X)
-
-    D1_pkl = euclidean_pkl.pairwise(X)
-    D2_pkl = pyfunc_pkl.pairwise(X)
-
-    assert_array_almost_equal(D1, D2)
-    assert_array_almost_equal(D1_pkl, D2_pkl)
-
-
-def test_bad_pyfunc_metric():
-    def wrong_distance(x, y):
-        return "1"
-
-    X = np.ones((5, 2))
-    msg = "Custom distance function must accept two vectors"
-    with pytest.raises(TypeError, match=msg):
-        BallTree(X, metric=wrong_distance)
-
-
-def test_input_data_size():
-    # Regression test for #6288
-    # Previously, a metric requiring a particular input dimension would fail
-    def custom_metric(x, y):
-        assert x.shape[0] == 3
-        return np.sum((x - y) ** 2)
-
-    rng = check_random_state(0)
-    X = rng.rand(10, 3)
-
-    pyfunc = DistanceMetric.get_metric("pyfunc", func=custom_metric)
-    eucl = DistanceMetric.get_metric("euclidean")
-    assert_array_almost_equal(pyfunc.pairwise(X), eucl.pairwise(X) ** 2)
diff --git a/sklearn/neighbors/tests/test_graph.py b/sklearn/neighbors/tests/test_graph.py
index 3654a26cfc785..fb593485d17a8 100644
--- a/sklearn/neighbors/tests/test_graph.py
+++ b/sklearn/neighbors/tests/test_graph.py
@@ -1,8 +1,10 @@
 import numpy as np
+import pytest
 
 from sklearn.metrics import euclidean_distances
 from sklearn.neighbors import KNeighborsTransformer, RadiusNeighborsTransformer
 from sklearn.neighbors._base import _is_sorted_by_data
+from sklearn.utils._testing import assert_array_equal
 
 
 def test_transformer_result():
@@ -18,35 +20,35 @@ def test_transformer_result():
     radius = np.percentile(euclidean_distances(X), 10)
 
     # with n_neighbors
-    for mode in ['distance', 'connectivity']:
-        add_one = mode == 'distance'
+    for mode in ["distance", "connectivity"]:
+        add_one = mode == "distance"
         nnt = KNeighborsTransformer(n_neighbors=n_neighbors, mode=mode)
         Xt = nnt.fit_transform(X)
         assert Xt.shape == (n_samples_fit, n_samples_fit)
-        assert Xt.data.shape == (n_samples_fit * (n_neighbors + add_one), )
-        assert Xt.format == 'csr'
+        assert Xt.data.shape == (n_samples_fit * (n_neighbors + add_one),)
+        assert Xt.format == "csr"
         assert _is_sorted_by_data(Xt)
 
         X2t = nnt.transform(X2)
         assert X2t.shape == (n_queries, n_samples_fit)
-        assert X2t.data.shape == (n_queries * (n_neighbors + add_one), )
-        assert X2t.format == 'csr'
+        assert X2t.data.shape == (n_queries * (n_neighbors + add_one),)
+        assert X2t.format == "csr"
         assert _is_sorted_by_data(X2t)
 
     # with radius
-    for mode in ['distance', 'connectivity']:
-        add_one = mode == 'distance'
+    for mode in ["distance", "connectivity"]:
+        add_one = mode == "distance"
         nnt = RadiusNeighborsTransformer(radius=radius, mode=mode)
         Xt = nnt.fit_transform(X)
         assert Xt.shape == (n_samples_fit, n_samples_fit)
-        assert not Xt.data.shape == (n_samples_fit * (n_neighbors + add_one), )
-        assert Xt.format == 'csr'
+        assert not Xt.data.shape == (n_samples_fit * (n_neighbors + add_one),)
+        assert Xt.format == "csr"
         assert _is_sorted_by_data(Xt)
 
         X2t = nnt.transform(X2)
         assert X2t.shape == (n_queries, n_samples_fit)
-        assert not X2t.data.shape == (n_queries * (n_neighbors + add_one), )
-        assert X2t.format == 'csr'
+        assert not X2t.data.shape == (n_queries * (n_neighbors + add_one),)
+        assert X2t.format == "csr"
         assert _is_sorted_by_data(X2t)
 
 
@@ -77,3 +79,23 @@ def test_explicit_diagonal():
     # Using transform on new data should not always have zero diagonal
     X2t = nnt.transform(X2)
     assert not _has_explicit_diagonal(X2t)
+
+
+@pytest.mark.parametrize("Klass", [KNeighborsTransformer, RadiusNeighborsTransformer])
+def test_graph_feature_names_out(Klass):
+    """Check `get_feature_names_out` for transformers defined in `_graph.py`."""
+
+    n_samples_fit = 20
+    n_features = 10
+    rng = np.random.RandomState(42)
+    X = rng.randn(n_samples_fit, n_features)
+
+    est = Klass().fit(X)
+    names_out = est.get_feature_names_out()
+
+    class_name_lower = Klass.__name__.lower()
+    expected_names_out = np.array(
+        [f"{class_name_lower}{i}" for i in range(est.n_samples_fit_)],
+        dtype=object,
+    )
+    assert_array_equal(names_out, expected_names_out)
diff --git a/sklearn/neighbors/tests/test_kd_tree.py b/sklearn/neighbors/tests/test_kd_tree.py
index 8b013cae522b8..749601baaf66f 100644
--- a/sklearn/neighbors/tests/test_kd_tree.py
+++ b/sklearn/neighbors/tests/test_kd_tree.py
@@ -1,21 +1,100 @@
 import numpy as np
 import pytest
+from numpy.testing import assert_allclose, assert_equal
 
-from sklearn.neighbors._kd_tree import KDTree
+from sklearn.neighbors._kd_tree import KDTree, KDTree32, KDTree64
+from sklearn.neighbors.tests.test_ball_tree import get_dataset_for_binary_tree
+from sklearn.utils.parallel import Parallel, delayed
 
 DIMENSION = 3
 
-METRICS = {'euclidean': {},
-           'manhattan': {},
-           'chebyshev': {},
-           'minkowski': dict(p=3)}
+METRICS = {"euclidean": {}, "manhattan": {}, "chebyshev": {}, "minkowski": dict(p=3)}
 
+KD_TREE_CLASSES = [
+    KDTree64,
+    KDTree32,
+]
 
-def test_array_object_type():
+
+def test_KDTree_is_KDTree64_subclass():
+    assert issubclass(KDTree, KDTree64)
+
+
+@pytest.mark.parametrize("BinarySearchTree", KD_TREE_CLASSES)
+def test_array_object_type(BinarySearchTree):
     """Check that we do not accept object dtype array."""
     X = np.array([(1, 2, 3), (2, 5), (5, 5, 1, 2)], dtype=object)
-    with pytest.raises(
-        ValueError,
-        match="setting an array element with a sequence"
-    ):
-        KDTree(X)
+    with pytest.raises(ValueError, match="setting an array element with a sequence"):
+        BinarySearchTree(X)
+
+
+@pytest.mark.parametrize("BinarySearchTree", KD_TREE_CLASSES)
+def test_kdtree_picklable_with_joblib(BinarySearchTree):
+    """Make sure that KDTree queries work when joblib memmaps.
+
+    Non-regression test for #21685 and #21228."""
+    rng = np.random.RandomState(0)
+    X = rng.random_sample((10, 3))
+    tree = BinarySearchTree(X, leaf_size=2)
+
+    # Call Parallel with max_nbytes=1 to trigger readonly memory mapping that
+    # use to raise "ValueError: buffer source array is read-only" in a previous
+    # version of the Cython code.
+    Parallel(n_jobs=2, max_nbytes=1)(delayed(tree.query)(data) for data in 2 * [X])
+
+
+@pytest.mark.parametrize("metric", METRICS)
+def test_kd_tree_numerical_consistency(global_random_seed, metric):
+    # Results on float64 and float32 versions of a dataset must be
+    # numerically close.
+    X_64, X_32, Y_64, Y_32 = get_dataset_for_binary_tree(
+        random_seed=global_random_seed, features=50
+    )
+
+    metric_params = METRICS.get(metric, {})
+    kd_64 = KDTree64(X_64, leaf_size=2, metric=metric, **metric_params)
+    kd_32 = KDTree32(X_32, leaf_size=2, metric=metric, **metric_params)
+
+    # Test consistency with respect to the `query` method
+    k = 4
+    dist_64, ind_64 = kd_64.query(Y_64, k=k)
+    dist_32, ind_32 = kd_32.query(Y_32, k=k)
+    assert_allclose(dist_64, dist_32, rtol=1e-5)
+    assert_equal(ind_64, ind_32)
+    assert dist_64.dtype == np.float64
+    assert dist_32.dtype == np.float32
+
+    # Test consistency with respect to the `query_radius` method
+    r = 2.38
+    ind_64 = kd_64.query_radius(Y_64, r=r)
+    ind_32 = kd_32.query_radius(Y_32, r=r)
+    for _ind64, _ind32 in zip(ind_64, ind_32):
+        assert_equal(_ind64, _ind32)
+
+    # Test consistency with respect to the `query_radius` method
+    # with return distances being true
+    ind_64, dist_64 = kd_64.query_radius(Y_64, r=r, return_distance=True)
+    ind_32, dist_32 = kd_32.query_radius(Y_32, r=r, return_distance=True)
+    for _ind64, _ind32, _dist_64, _dist_32 in zip(ind_64, ind_32, dist_64, dist_32):
+        assert_equal(_ind64, _ind32)
+        assert_allclose(_dist_64, _dist_32, rtol=1e-5)
+        assert _dist_64.dtype == np.float64
+        assert _dist_32.dtype == np.float32
+
+
+@pytest.mark.parametrize("metric", METRICS)
+def test_kernel_density_numerical_consistency(global_random_seed, metric):
+    # Test consistency with respect to the `kernel_density` method
+    X_64, X_32, Y_64, Y_32 = get_dataset_for_binary_tree(random_seed=global_random_seed)
+
+    metric_params = METRICS.get(metric, {})
+    kd_64 = KDTree64(X_64, leaf_size=2, metric=metric, **metric_params)
+    kd_32 = KDTree32(X_32, leaf_size=2, metric=metric, **metric_params)
+
+    kernel = "gaussian"
+    h = 0.1
+    density64 = kd_64.kernel_density(Y_64, h=h, kernel=kernel, breadth_first=True)
+    density32 = kd_32.kernel_density(Y_32, h=h, kernel=kernel, breadth_first=True)
+    assert_allclose(density64, density32, rtol=1e-5)
+    assert density64.dtype == np.float64
+    assert density32.dtype == np.float32
diff --git a/sklearn/neighbors/tests/test_kde.py b/sklearn/neighbors/tests/test_kde.py
index 90ce667e5c284..b6bf09d01b672 100644
--- a/sklearn/neighbors/tests/test_kde.py
+++ b/sklearn/neighbors/tests/test_kde.py
@@ -1,55 +1,56 @@
+import joblib
 import numpy as np
-
 import pytest
 
-from sklearn.utils._testing import assert_allclose
-from sklearn.neighbors import KernelDensity, KDTree, NearestNeighbors
-from sklearn.neighbors._ball_tree import kernel_norm
-from sklearn.pipeline import make_pipeline
 from sklearn.datasets import make_blobs
+from sklearn.exceptions import NotFittedError
 from sklearn.model_selection import GridSearchCV
+from sklearn.neighbors import KDTree, KernelDensity, NearestNeighbors
+from sklearn.neighbors._ball_tree import kernel_norm
+from sklearn.pipeline import make_pipeline
 from sklearn.preprocessing import StandardScaler
-from sklearn.exceptions import NotFittedError
-import joblib
+from sklearn.utils._testing import assert_allclose
 
 
 # XXX Duplicated in test_neighbors_tree, test_kde
 def compute_kernel_slow(Y, X, kernel, h):
+    if h == "scott":
+        h = X.shape[0] ** (-1 / (X.shape[1] + 4))
+    elif h == "silverman":
+        h = (X.shape[0] * (X.shape[1] + 2) / 4) ** (-1 / (X.shape[1] + 4))
+
     d = np.sqrt(((Y[:, None, :] - X) ** 2).sum(-1))
     norm = kernel_norm(h, X.shape[1], kernel) / X.shape[0]
 
-    if kernel == 'gaussian':
+    if kernel == "gaussian":
         return norm * np.exp(-0.5 * (d * d) / (h * h)).sum(-1)
-    elif kernel == 'tophat':
+    elif kernel == "tophat":
         return norm * (d < h).sum(-1)
-    elif kernel == 'epanechnikov':
+    elif kernel == "epanechnikov":
         return norm * ((1.0 - (d * d) / (h * h)) * (d < h)).sum(-1)
-    elif kernel == 'exponential':
+    elif kernel == "exponential":
         return norm * (np.exp(-d / h)).sum(-1)
-    elif kernel == 'linear':
+    elif kernel == "linear":
         return norm * ((1 - d / h) * (d < h)).sum(-1)
-    elif kernel == 'cosine':
+    elif kernel == "cosine":
         return norm * (np.cos(0.5 * np.pi * d / h) * (d < h)).sum(-1)
     else:
-        raise ValueError('kernel not recognized')
+        raise ValueError("kernel not recognized")
 
 
 def check_results(kernel, bandwidth, atol, rtol, X, Y, dens_true):
-    kde = KernelDensity(kernel=kernel, bandwidth=bandwidth,
-                        atol=atol, rtol=rtol)
+    kde = KernelDensity(kernel=kernel, bandwidth=bandwidth, atol=atol, rtol=rtol)
     log_dens = kde.fit(X).score_samples(Y)
-    assert_allclose(np.exp(log_dens), dens_true,
-                    atol=atol, rtol=max(1E-7, rtol))
-    assert_allclose(np.exp(kde.score(Y)),
-                    np.prod(dens_true),
-                    atol=atol, rtol=max(1E-7, rtol))
+    assert_allclose(np.exp(log_dens), dens_true, atol=atol, rtol=max(1e-7, rtol))
+    assert_allclose(
+        np.exp(kde.score(Y)), np.prod(dens_true), atol=atol, rtol=max(1e-7, rtol)
+    )
 
 
 @pytest.mark.parametrize(
-        'kernel',
-        ['gaussian', 'tophat', 'epanechnikov',
-         'exponential', 'linear', 'cosine'])
-@pytest.mark.parametrize('bandwidth', [0.01, 0.1, 1])
+    "kernel", ["gaussian", "tophat", "epanechnikov", "exponential", "linear", "cosine"]
+)
+@pytest.mark.parametrize("bandwidth", [0.01, 0.1, 1, "scott", "silverman"])
 def test_kernel_density(kernel, bandwidth):
     n_samples, n_features = (100, 3)
 
@@ -59,11 +60,10 @@ def test_kernel_density(kernel, bandwidth):
 
     dens_true = compute_kernel_slow(Y, X, kernel, bandwidth)
 
-    for rtol in [0, 1E-5]:
-        for atol in [1E-6, 1E-2]:
+    for rtol in [0, 1e-5]:
+        for atol in [1e-6, 1e-2]:
             for breadth_first in (True, False):
-                check_results(kernel, bandwidth, atol, rtol,
-                              X, Y, dens_true)
+                check_results(kernel, bandwidth, atol, rtol, X, Y, dens_true)
 
 
 def test_kernel_density_sampling(n_samples=100, n_features=3):
@@ -72,7 +72,7 @@ def test_kernel_density_sampling(n_samples=100, n_features=3):
 
     bandwidth = 0.2
 
-    for kernel in ['gaussian', 'tophat']:
+    for kernel in ["gaussian", "tophat"]:
         # draw a tophat sample
         kde = KernelDensity(bandwidth=bandwidth, kernel=kernel).fit(X)
         samp = kde.sample(100)
@@ -82,15 +82,15 @@ def test_kernel_density_sampling(n_samples=100, n_features=3):
         nbrs = NearestNeighbors(n_neighbors=1).fit(X)
         dist, ind = nbrs.kneighbors(X, return_distance=True)
 
-        if kernel == 'tophat':
+        if kernel == "tophat":
             assert np.all(dist < bandwidth)
-        elif kernel == 'gaussian':
+        elif kernel == "gaussian":
             # 5 standard deviations is safe for 100 samples, but there's a
             # very small chance this test could fail.
             assert np.all(dist < 5 * bandwidth)
 
     # check unsupported kernels
-    for kernel in ['epanechnikov', 'exponential', 'linear', 'cosine']:
+    for kernel in ["epanechnikov", "exponential", "linear", "cosine"]:
         kde = KernelDensity(bandwidth=bandwidth, kernel=kernel).fit(X)
         with pytest.raises(NotImplementedError):
             kde.sample(100)
@@ -101,21 +101,22 @@ def test_kernel_density_sampling(n_samples=100, n_features=3):
     assert kde.sample().shape == (1, 1)
 
 
-@pytest.mark.parametrize('algorithm', ['auto', 'ball_tree', 'kd_tree'])
-@pytest.mark.parametrize('metric',
-                         ['euclidean', 'minkowski', 'manhattan',
-                          'chebyshev', 'haversine'])
+@pytest.mark.parametrize("algorithm", ["auto", "ball_tree", "kd_tree"])
+@pytest.mark.parametrize(
+    "metric", ["euclidean", "minkowski", "manhattan", "chebyshev", "haversine"]
+)
 def test_kde_algorithm_metric_choice(algorithm, metric):
     # Smoke test for various metrics and algorithms
     rng = np.random.RandomState(0)
-    X = rng.randn(10, 2)    # 2 features required for haversine dist.
+    X = rng.randn(10, 2)  # 2 features required for haversine dist.
     Y = rng.randn(10, 2)
 
-    if algorithm == 'kd_tree' and metric not in KDTree.valid_metrics:
-        with pytest.raises(ValueError):
-            KernelDensity(algorithm=algorithm, metric=metric)
+    kde = KernelDensity(algorithm=algorithm, metric=metric)
+
+    if algorithm == "kd_tree" and metric not in KDTree.valid_metrics:
+        with pytest.raises(ValueError, match="invalid metric"):
+            kde.fit(X)
     else:
-        kde = KernelDensity(algorithm=algorithm, metric=metric)
         kde.fit(X)
         y_dens = kde.score_samples(Y)
         assert y_dens.shape == Y.shape[:1]
@@ -129,42 +130,31 @@ def test_kde_score(n_samples=100, n_features=3):
     # Y = rng.random_sample((n_samples, n_features))
 
 
-def test_kde_badargs():
-    with pytest.raises(ValueError):
-        KernelDensity(algorithm='blah')
-    with pytest.raises(ValueError):
-        KernelDensity(bandwidth=0)
-    with pytest.raises(ValueError):
-        KernelDensity(kernel='blah')
-    with pytest.raises(ValueError):
-        KernelDensity(metric='blah')
-    with pytest.raises(ValueError):
-        KernelDensity(algorithm='kd_tree', metric='blah')
+def test_kde_sample_weights_error():
     kde = KernelDensity()
     with pytest.raises(ValueError):
-        kde.fit(np.random.random((200, 10)),
-                sample_weight=np.random.random((200, 10)))
+        kde.fit(np.random.random((200, 10)), sample_weight=np.random.random((200, 10)))
     with pytest.raises(ValueError):
-        kde.fit(np.random.random((200, 10)),
-                sample_weight=-np.random.random(200))
+        kde.fit(np.random.random((200, 10)), sample_weight=-np.random.random(200))
 
 
 def test_kde_pipeline_gridsearch():
     # test that kde plays nice in pipelines and grid-searches
-    X, _ = make_blobs(cluster_std=.1, random_state=1,
-                      centers=[[0, 1], [1, 0], [0, 0]])
-    pipe1 = make_pipeline(StandardScaler(with_mean=False, with_std=False),
-                          KernelDensity(kernel="gaussian"))
+    X, _ = make_blobs(cluster_std=0.1, random_state=1, centers=[[0, 1], [1, 0], [0, 0]])
+    pipe1 = make_pipeline(
+        StandardScaler(with_mean=False, with_std=False),
+        KernelDensity(kernel="gaussian"),
+    )
     params = dict(kerneldensity__bandwidth=[0.001, 0.01, 0.1, 1, 10])
     search = GridSearchCV(pipe1, param_grid=params)
     search.fit(X)
-    assert search.best_params_['kerneldensity__bandwidth'] == .1
+    assert search.best_params_["kerneldensity__bandwidth"] == 0.1
 
 
 def test_kde_sample_weights():
     n_samples = 400
     size_test = 20
-    weights_neutral = np.full(n_samples, 3.)
+    weights_neutral = np.full(n_samples, 3.0)
     for d in [1, 2, 10]:
         rng = np.random.RandomState(0)
         X = rng.rand(n_samples, d)
@@ -172,10 +162,9 @@ def test_kde_sample_weights():
         X_repetitions = np.repeat(X, weights, axis=0)
         n_samples_test = size_test // d
         test_points = rng.rand(n_samples_test, d)
-        for algorithm in ['auto', 'ball_tree', 'kd_tree']:
-            for metric in ['euclidean', 'minkowski', 'manhattan',
-                           'chebyshev']:
-                if algorithm != 'kd_tree' or metric in KDTree.valid_metrics:
+        for algorithm in ["auto", "ball_tree", "kd_tree"]:
+            for metric in ["euclidean", "minkowski", "manhattan", "chebyshev"]:
+                if algorithm != "kd_tree" or metric in KDTree.valid_metrics:
                     kde = KernelDensity(algorithm=algorithm, metric=metric)
 
                     # Test that adding a constant sample weight has no effect
@@ -209,31 +198,20 @@ def test_kde_sample_weights():
                     assert_allclose(scores_scaled_weight, scores_weight)
 
 
-def test_sample_weight_invalid():
-    # Check sample weighting raises errors.
-    kde = KernelDensity()
-    data = np.reshape([1., 2., 3.], (-1, 1))
-
-    sample_weight = [0.1, -0.2, 0.3]
-    expected_err = "sample_weight must have positive values"
-    with pytest.raises(ValueError, match=expected_err):
-        kde.fit(data, sample_weight=sample_weight)
-
-
-@pytest.mark.parametrize('sample_weight', [None, [0.1, 0.2, 0.3]])
+@pytest.mark.parametrize("sample_weight", [None, [0.1, 0.2, 0.3]])
 def test_pickling(tmpdir, sample_weight):
     # Make sure that predictions are the same before and after pickling. Used
     # to be a bug because sample_weights wasn't pickled and the resulting tree
     # would miss some info.
 
     kde = KernelDensity()
-    data = np.reshape([1., 2., 3.], (-1, 1))
+    data = np.reshape([1.0, 2.0, 3.0], (-1, 1))
     kde.fit(data, sample_weight=sample_weight)
 
     X = np.reshape([1.1, 2.1], (-1, 1))
     scores = kde.score_samples(X)
 
-    file_path = str(tmpdir.join('dump.pkl'))
+    file_path = str(tmpdir.join("dump.pkl"))
     joblib.dump(kde, file_path)
     kde = joblib.load(file_path)
     scores_pickled = kde.score_samples(X)
@@ -241,7 +219,7 @@ def test_pickling(tmpdir, sample_weight):
     assert_allclose(scores, scores_pickled)
 
 
-@pytest.mark.parametrize('method', ['score_samples', 'sample'])
+@pytest.mark.parametrize("method", ["score_samples", "sample"])
 def test_check_is_fitted(method):
     # Check that predict raises an exception in an unfitted estimator.
     # Unfitted estimators should raise a NotFittedError.
@@ -251,3 +229,24 @@ def test_check_is_fitted(method):
 
     with pytest.raises(NotFittedError):
         getattr(kde, method)(X)
+
+
+@pytest.mark.parametrize("bandwidth", ["scott", "silverman", 0.1])
+def test_bandwidth(bandwidth):
+    n_samples, n_features = (100, 3)
+    rng = np.random.RandomState(0)
+    X = rng.randn(n_samples, n_features)
+    kde = KernelDensity(bandwidth=bandwidth).fit(X)
+    samp = kde.sample(100)
+    kde_sc = kde.score_samples(X)
+    assert X.shape == samp.shape
+    assert kde_sc.shape == (n_samples,)
+
+    # Test that the attribute self.bandwidth_ has the expected value
+    if bandwidth == "scott":
+        h = X.shape[0] ** (-1 / (X.shape[1] + 4))
+    elif bandwidth == "silverman":
+        h = (X.shape[0] * (X.shape[1] + 2) / 4) ** (-1 / (X.shape[1] + 4))
+    else:
+        h = bandwidth
+    assert kde.bandwidth_ == pytest.approx(h)
diff --git a/sklearn/neighbors/tests/test_lof.py b/sklearn/neighbors/tests/test_lof.py
index 5d479d5b141f7..140d0d9ba6dff 100644
--- a/sklearn/neighbors/tests/test_lof.py
+++ b/sklearn/neighbors/tests/test_lof.py
@@ -1,25 +1,22 @@
-# Authors: Nicolas Goix 
-#          Alexandre Gramfort 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
+import re
 from math import sqrt
 
 import numpy as np
-from sklearn import neighbors
-import re
 import pytest
-from numpy.testing import assert_array_equal
 
-from sklearn import metrics
+from sklearn import metrics, neighbors
+from sklearn.datasets import load_iris
 from sklearn.metrics import roc_auc_score
-
 from sklearn.utils import check_random_state
-from sklearn.utils._testing import assert_array_almost_equal
-from sklearn.utils.estimator_checks import check_estimator
-from sklearn.utils.estimator_checks import check_outlier_corruption
-
-from sklearn.datasets import load_iris
-
+from sklearn.utils._testing import assert_allclose, assert_array_equal
+from sklearn.utils.estimator_checks import (
+    check_outlier_corruption,
+    parametrize_with_checks,
+)
+from sklearn.utils.fixes import CSR_CONTAINERS
 
 # load the iris dataset
 # and randomly permute it
@@ -30,9 +27,12 @@
 iris.target = iris.target[perm]
 
 
-def test_lof():
+def test_lof(global_dtype):
     # Toy sample (the last two samples are outliers):
-    X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1], [5, 3], [-4, 2]]
+    X = np.asarray(
+        [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1], [5, 3], [-4, 2]],
+        dtype=global_dtype,
+    )
 
     # Test LocalOutlierFactor:
     clf = neighbors.LocalOutlierFactor(n_neighbors=5)
@@ -43,20 +43,22 @@ def test_lof():
     assert np.min(score[:-2]) > np.max(score[-2:])
 
     # Assert predict() works:
-    clf = neighbors.LocalOutlierFactor(contamination=0.25,
-                                       n_neighbors=5).fit(X)
-    assert_array_equal(clf._predict(), 6 * [1] + 2 * [-1])
-    assert_array_equal(clf.fit_predict(X), 6 * [1] + 2 * [-1])
+    clf = neighbors.LocalOutlierFactor(contamination=0.25, n_neighbors=5).fit(X)
+    expected_predictions = 6 * [1] + 2 * [-1]
+    assert_array_equal(clf._predict(), expected_predictions)
+    assert_array_equal(clf.fit_predict(X), expected_predictions)
 
 
-def test_lof_performance():
+def test_lof_performance(global_dtype):
     # Generate train/test data
     rng = check_random_state(2)
-    X = 0.3 * rng.randn(120, 2)
+    X = 0.3 * rng.randn(120, 2).astype(global_dtype, copy=False)
     X_train = X[:100]
 
     # Generate some abnormal novel observations
-    X_outliers = rng.uniform(low=-4, high=4, size=(20, 2))
+    X_outliers = rng.uniform(low=-4, high=4, size=(20, 2)).astype(
+        global_dtype, copy=False
+    )
     X_test = np.r_[X[100:], X_outliers]
     y_test = np.array([0] * 20 + [1] * 20)
 
@@ -67,38 +69,37 @@ def test_lof_performance():
     y_pred = -clf.decision_function(X_test)
 
     # check that roc_auc is good
-    assert roc_auc_score(y_test, y_pred) > .99
+    assert roc_auc_score(y_test, y_pred) > 0.99
 
 
-def test_lof_values():
+def test_lof_values(global_dtype):
     # toy samples:
-    X_train = [[1, 1], [1, 2], [2, 1]]
-    clf1 = neighbors.LocalOutlierFactor(n_neighbors=2,
-                                        contamination=0.1,
-                                        novelty=True).fit(X_train)
-    clf2 = neighbors.LocalOutlierFactor(n_neighbors=2,
-                                        novelty=True).fit(X_train)
-    s_0 = 2. * sqrt(2.) / (1. + sqrt(2.))
-    s_1 = (1. + sqrt(2)) * (1. / (4. * sqrt(2.)) + 1. / (2. + 2. * sqrt(2)))
+    X_train = np.asarray([[1, 1], [1, 2], [2, 1]], dtype=global_dtype)
+    clf1 = neighbors.LocalOutlierFactor(
+        n_neighbors=2, contamination=0.1, novelty=True
+    ).fit(X_train)
+    clf2 = neighbors.LocalOutlierFactor(n_neighbors=2, novelty=True).fit(X_train)
+    s_0 = 2.0 * sqrt(2.0) / (1.0 + sqrt(2.0))
+    s_1 = (1.0 + sqrt(2)) * (1.0 / (4.0 * sqrt(2.0)) + 1.0 / (2.0 + 2.0 * sqrt(2)))
     # check predict()
-    assert_array_almost_equal(-clf1.negative_outlier_factor_, [s_0, s_1, s_1])
-    assert_array_almost_equal(-clf2.negative_outlier_factor_, [s_0, s_1, s_1])
+    assert_allclose(-clf1.negative_outlier_factor_, [s_0, s_1, s_1])
+    assert_allclose(-clf2.negative_outlier_factor_, [s_0, s_1, s_1])
     # check predict(one sample not in train)
-    assert_array_almost_equal(-clf1.score_samples([[2., 2.]]), [s_0])
-    assert_array_almost_equal(-clf2.score_samples([[2., 2.]]), [s_0])
+    assert_allclose(-clf1.score_samples([[2.0, 2.0]]), [s_0])
+    assert_allclose(-clf2.score_samples([[2.0, 2.0]]), [s_0])
     # check predict(one sample already in train)
-    assert_array_almost_equal(-clf1.score_samples([[1., 1.]]), [s_1])
-    assert_array_almost_equal(-clf2.score_samples([[1., 1.]]), [s_1])
+    assert_allclose(-clf1.score_samples([[1.0, 1.0]]), [s_1])
+    assert_allclose(-clf2.score_samples([[1.0, 1.0]]), [s_1])
 
 
-def test_lof_precomputed(random_state=42):
+def test_lof_precomputed(global_dtype, random_state=42):
     """Tests LOF with a distance matrix."""
     # Note: smaller samples may result in spurious test success
     rng = np.random.RandomState(random_state)
-    X = rng.random_sample((10, 4))
-    Y = rng.random_sample((3, 4))
-    DXX = metrics.pairwise_distances(X, metric='euclidean')
-    DYX = metrics.pairwise_distances(Y, X, metric='euclidean')
+    X = rng.random_sample((10, 4)).astype(global_dtype, copy=False)
+    Y = rng.random_sample((3, 4)).astype(global_dtype, copy=False)
+    DXX = metrics.pairwise_distances(X, metric="euclidean")
+    DYX = metrics.pairwise_distances(Y, X, metric="euclidean")
     # As a feature matrix (n_samples by n_features)
     lof_X = neighbors.LocalOutlierFactor(n_neighbors=3, novelty=True)
     lof_X.fit(X)
@@ -106,14 +107,15 @@ def test_lof_precomputed(random_state=42):
     pred_X_Y = lof_X.predict(Y)
 
     # As a dense distance matrix (n_samples by n_samples)
-    lof_D = neighbors.LocalOutlierFactor(n_neighbors=3, algorithm='brute',
-                                         metric='precomputed', novelty=True)
+    lof_D = neighbors.LocalOutlierFactor(
+        n_neighbors=3, algorithm="brute", metric="precomputed", novelty=True
+    )
     lof_D.fit(DXX)
     pred_D_X = lof_D._predict()
     pred_D_Y = lof_D.predict(DYX)
 
-    assert_array_almost_equal(pred_X_X, pred_D_X)
-    assert_array_almost_equal(pred_X_Y, pred_D_Y)
+    assert_allclose(pred_X_X, pred_D_X)
+    assert_allclose(pred_X_Y, pred_D_Y)
 
 
 def test_n_neighbors_attribute():
@@ -128,26 +130,29 @@ def test_n_neighbors_attribute():
     assert clf.n_neighbors_ == X.shape[0] - 1
 
 
-def test_score_samples():
-    X_train = [[1, 1], [1, 2], [2, 1]]
-    clf1 = neighbors.LocalOutlierFactor(n_neighbors=2,
-                                        contamination=0.1,
-                                        novelty=True).fit(X_train)
-    clf2 = neighbors.LocalOutlierFactor(n_neighbors=2,
-                                        novelty=True).fit(X_train)
-    assert_array_equal(clf1.score_samples([[2., 2.]]),
-                       clf1.decision_function([[2., 2.]]) + clf1.offset_)
-    assert_array_equal(clf2.score_samples([[2., 2.]]),
-                       clf2.decision_function([[2., 2.]]) + clf2.offset_)
-    assert_array_equal(clf1.score_samples([[2., 2.]]),
-                       clf2.score_samples([[2., 2.]]))
-
-
-def test_contamination():
-    X = [[1, 1], [1, 0]]
-    clf = neighbors.LocalOutlierFactor(contamination=0.6)
-    with pytest.raises(ValueError):
-        clf.fit(X)
+def test_score_samples(global_dtype):
+    X_train = np.asarray([[1, 1], [1, 2], [2, 1]], dtype=global_dtype)
+    X_test = np.asarray([[2.0, 2.0]], dtype=global_dtype)
+    clf1 = neighbors.LocalOutlierFactor(
+        n_neighbors=2, contamination=0.1, novelty=True
+    ).fit(X_train)
+    clf2 = neighbors.LocalOutlierFactor(n_neighbors=2, novelty=True).fit(X_train)
+
+    clf1_scores = clf1.score_samples(X_test)
+    clf1_decisions = clf1.decision_function(X_test)
+
+    clf2_scores = clf2.score_samples(X_test)
+    clf2_decisions = clf2.decision_function(X_test)
+
+    assert_allclose(
+        clf1_scores,
+        clf1_decisions + clf1.offset_,
+    )
+    assert_allclose(
+        clf2_scores,
+        clf2_decisions + clf2.offset_,
+    )
+    assert_allclose(clf1_scores, clf2_scores)
 
 
 def test_novelty_errors():
@@ -157,22 +162,31 @@ def test_novelty_errors():
     clf = neighbors.LocalOutlierFactor()
     clf.fit(X)
     # predict, decision_function and score_samples raise ValueError
-    for method in ['predict', 'decision_function', 'score_samples']:
-        msg = ('{} is not available when novelty=False'.format(method))
-        with pytest.raises(AttributeError, match=msg):
+    for method in ["predict", "decision_function", "score_samples"]:
+        outer_msg = f"'LocalOutlierFactor' has no attribute '{method}'"
+        inner_msg = "{} is not available when novelty=False".format(method)
+        with pytest.raises(AttributeError, match=outer_msg) as exec_info:
             getattr(clf, method)
 
+        assert isinstance(exec_info.value.__cause__, AttributeError)
+        assert inner_msg in str(exec_info.value.__cause__)
+
     # check errors for novelty=True
     clf = neighbors.LocalOutlierFactor(novelty=True)
-    msg = 'fit_predict is not available when novelty=True'
-    with pytest.raises(AttributeError, match=msg):
-        getattr(clf, 'fit_predict')
+
+    outer_msg = "'LocalOutlierFactor' has no attribute 'fit_predict'"
+    inner_msg = "fit_predict is not available when novelty=True"
+    with pytest.raises(AttributeError, match=outer_msg) as exec_info:
+        getattr(clf, "fit_predict")
+
+    assert isinstance(exec_info.value.__cause__, AttributeError)
+    assert inner_msg in str(exec_info.value.__cause__)
 
 
-def test_novelty_training_scores():
+def test_novelty_training_scores(global_dtype):
     # check that the scores of the training samples are still accessible
     # when novelty=True through the negative_outlier_factor_ attribute
-    X = iris.data
+    X = iris.data.astype(global_dtype)
 
     # fit with novelty=False
     clf_1 = neighbors.LocalOutlierFactor()
@@ -184,7 +198,7 @@ def test_novelty_training_scores():
     clf_2.fit(X)
     scores_2 = clf_2.negative_outlier_factor_
 
-    assert_array_almost_equal(scores_1, scores_2)
+    assert_allclose(scores_1, scores_2)
 
 
 def test_hasattr_prediction():
@@ -194,34 +208,34 @@ def test_hasattr_prediction():
     # when novelty=True
     clf = neighbors.LocalOutlierFactor(novelty=True)
     clf.fit(X)
-    assert hasattr(clf, 'predict')
-    assert hasattr(clf, 'decision_function')
-    assert hasattr(clf, 'score_samples')
-    assert not hasattr(clf, 'fit_predict')
+    assert hasattr(clf, "predict")
+    assert hasattr(clf, "decision_function")
+    assert hasattr(clf, "score_samples")
+    assert not hasattr(clf, "fit_predict")
 
     # when novelty=False
     clf = neighbors.LocalOutlierFactor(novelty=False)
     clf.fit(X)
-    assert hasattr(clf, 'fit_predict')
-    assert not hasattr(clf, 'predict')
-    assert not hasattr(clf, 'decision_function')
-    assert not hasattr(clf, 'score_samples')
-
+    assert hasattr(clf, "fit_predict")
+    assert not hasattr(clf, "predict")
+    assert not hasattr(clf, "decision_function")
+    assert not hasattr(clf, "score_samples")
 
-def test_novelty_true_common_tests():
 
+@parametrize_with_checks([neighbors.LocalOutlierFactor(novelty=True)])
+def test_novelty_true_common_tests(estimator, check):
     # the common tests are run for the default LOF (novelty=False).
     # here we run these common tests for LOF when novelty=True
-    check_estimator(neighbors.LocalOutlierFactor(novelty=True))
+    check(estimator)
 
 
-@pytest.mark.parametrize('expected_outliers', [30, 53])
+@pytest.mark.parametrize("expected_outliers", [30, 53])
 def test_predicted_outlier_number(expected_outliers):
     # the number of predicted outliers should be equal to the number of
     # expected outliers unless there are ties in the abnormality scores.
     X = iris.data
     n_samples = X.shape[0]
-    contamination = float(expected_outliers)/n_samples
+    contamination = float(expected_outliers) / n_samples
 
     clf = neighbors.LocalOutlierFactor(contamination=contamination)
     y_pred = clf.fit_predict(X)
@@ -230,3 +244,151 @@ def test_predicted_outlier_number(expected_outliers):
     if num_outliers != expected_outliers:
         y_dec = clf.negative_outlier_factor_
         check_outlier_corruption(num_outliers, expected_outliers, y_dec)
+
+
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sparse(csr_container):
+    # LocalOutlierFactor must support CSR inputs
+    # TODO: compare results on dense and sparse data as proposed in:
+    # https://github.com/scikit-learn/scikit-learn/pull/23585#discussion_r968388186
+    X = csr_container(iris.data)
+
+    lof = neighbors.LocalOutlierFactor(novelty=True)
+    lof.fit(X)
+    lof.predict(X)
+    lof.score_samples(X)
+    lof.decision_function(X)
+
+    lof = neighbors.LocalOutlierFactor(novelty=False)
+    lof.fit_predict(X)
+
+
+def test_lof_error_n_neighbors_too_large():
+    """Check that we raise a proper error message when n_neighbors == n_samples.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/17207
+    """
+    X = np.ones((7, 7))
+
+    msg = (
+        "Expected n_neighbors < n_samples_fit, but n_neighbors = 1, "
+        "n_samples_fit = 1, n_samples = 1"
+    )
+    with pytest.raises(ValueError, match=msg):
+        lof = neighbors.LocalOutlierFactor(n_neighbors=1).fit(X[:1])
+
+    lof = neighbors.LocalOutlierFactor(n_neighbors=2).fit(X[:2])
+    assert lof.n_samples_fit_ == 2
+
+    msg = (
+        "Expected n_neighbors < n_samples_fit, but n_neighbors = 2, "
+        "n_samples_fit = 2, n_samples = 2"
+    )
+    with pytest.raises(ValueError, match=msg):
+        lof.kneighbors(None, n_neighbors=2)
+
+    distances, indices = lof.kneighbors(None, n_neighbors=1)
+    assert distances.shape == (2, 1)
+    assert indices.shape == (2, 1)
+
+    msg = (
+        "Expected n_neighbors <= n_samples_fit, but n_neighbors = 3, "
+        "n_samples_fit = 2, n_samples = 7"
+    )
+    with pytest.raises(ValueError, match=msg):
+        lof.kneighbors(X, n_neighbors=3)
+
+    (
+        distances,
+        indices,
+    ) = lof.kneighbors(X, n_neighbors=2)
+    assert distances.shape == (7, 2)
+    assert indices.shape == (7, 2)
+
+
+@pytest.mark.parametrize("algorithm", ["auto", "ball_tree", "kd_tree", "brute"])
+@pytest.mark.parametrize("novelty", [True, False])
+@pytest.mark.parametrize("contamination", [0.5, "auto"])
+def test_lof_input_dtype_preservation(global_dtype, algorithm, contamination, novelty):
+    """Check that the fitted attributes are stored using the data type of X."""
+    X = iris.data.astype(global_dtype, copy=False)
+
+    iso = neighbors.LocalOutlierFactor(
+        n_neighbors=5, algorithm=algorithm, contamination=contamination, novelty=novelty
+    )
+    iso.fit(X)
+
+    assert iso.negative_outlier_factor_.dtype == global_dtype
+
+    for method in ("score_samples", "decision_function"):
+        if hasattr(iso, method):
+            y_pred = getattr(iso, method)(X)
+            assert y_pred.dtype == global_dtype
+
+
+@pytest.mark.parametrize("algorithm", ["auto", "ball_tree", "kd_tree", "brute"])
+@pytest.mark.parametrize("novelty", [True, False])
+@pytest.mark.parametrize("contamination", [0.5, "auto"])
+def test_lof_dtype_equivalence(algorithm, novelty, contamination):
+    """Check the equivalence of the results with 32 and 64 bits input."""
+
+    inliers = iris.data[:50]  # setosa iris are really distinct from others
+    outliers = iris.data[-5:]  # virginica will be considered as outliers
+    # lower the precision of the input data to check that we have an equivalence when
+    # making the computation in 32 and 64 bits.
+    X = np.concatenate([inliers, outliers], axis=0).astype(np.float32)
+
+    lof_32 = neighbors.LocalOutlierFactor(
+        algorithm=algorithm, novelty=novelty, contamination=contamination
+    )
+    X_32 = X.astype(np.float32, copy=True)
+    lof_32.fit(X_32)
+
+    lof_64 = neighbors.LocalOutlierFactor(
+        algorithm=algorithm, novelty=novelty, contamination=contamination
+    )
+    X_64 = X.astype(np.float64, copy=True)
+    lof_64.fit(X_64)
+
+    assert_allclose(lof_32.negative_outlier_factor_, lof_64.negative_outlier_factor_)
+
+    for method in ("score_samples", "decision_function", "predict", "fit_predict"):
+        if hasattr(lof_32, method):
+            y_pred_32 = getattr(lof_32, method)(X_32)
+            y_pred_64 = getattr(lof_64, method)(X_64)
+            assert_allclose(y_pred_32, y_pred_64, atol=0.0002)
+
+
+def test_lof_duplicate_samples():
+    """
+    Check that LocalOutlierFactor raises a warning when duplicate values
+    in the training data cause inaccurate results.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/27839
+    """
+
+    rng = np.random.default_rng(0)
+
+    x = rng.permutation(
+        np.hstack(
+            [
+                [0.1] * 1000,  # constant values
+                np.linspace(0.1, 0.3, num=3000),
+                rng.random(500) * 100,  # the clear outliers
+            ]
+        )
+    )
+    X = x.reshape(-1, 1)
+
+    error_msg = (
+        "Duplicate values are leading to incorrect results. "
+        "Increase the number of neighbors for more accurate results."
+    )
+
+    lof = neighbors.LocalOutlierFactor(n_neighbors=5, contamination=0.1)
+
+    # Catch the warning
+    with pytest.warns(UserWarning, match=re.escape(error_msg)):
+        lof.fit_predict(X)
diff --git a/sklearn/neighbors/tests/test_nca.py b/sklearn/neighbors/tests/test_nca.py
index e7fc741899209..ebfb01d12e3ac 100644
--- a/sklearn/neighbors/tests/test_nca.py
+++ b/sklearn/neighbors/tests/test_nca.py
@@ -1,31 +1,35 @@
-# coding: utf-8
 """
 Testing for Neighborhood Component Analysis module (sklearn.neighbors.nca)
 """
 
-# Authors: William de Vazelhes 
-#          John Chiotellis 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-import pytest
 import re
+
 import numpy as np
-from numpy.testing import assert_array_equal, assert_array_almost_equal
+import pytest
+from numpy.testing import assert_array_almost_equal, assert_array_equal
 from scipy.optimize import check_grad
+
 from sklearn import clone
+from sklearn.datasets import load_iris, make_blobs, make_classification
 from sklearn.exceptions import ConvergenceWarning
-from sklearn.utils import check_random_state
-from sklearn.datasets import load_iris, make_classification, make_blobs
-from sklearn.neighbors import NeighborhoodComponentsAnalysis
 from sklearn.metrics import pairwise_distances
-
+from sklearn.neighbors import NeighborhoodComponentsAnalysis
+from sklearn.preprocessing import LabelEncoder
+from sklearn.utils import check_random_state
+from sklearn.utils.validation import validate_data
 
 rng = check_random_state(0)
-# load and shuffle iris dataset
+# Load and shuffle the iris dataset.
 iris = load_iris()
 perm = rng.permutation(iris.target.size)
 iris_data = iris.data[perm]
 iris_target = iris.target[perm]
+# Avoid having test data introducing dependencies between tests.
+iris_data.flags.writeable = False
+iris_target.flags.writeable = False
 EPS = np.finfo(float).eps
 
 
@@ -39,12 +43,12 @@ def test_simple_example():
     """
     X = np.array([[0, 0], [0, 1], [2, 0], [2, 1]])
     y = np.array([1, 0, 1, 0])
-    nca = NeighborhoodComponentsAnalysis(n_components=2, init='identity',
-                                         random_state=42)
+    nca = NeighborhoodComponentsAnalysis(
+        n_components=2, init="identity", random_state=42
+    )
     nca.fit(X, y)
     X_t = nca.transform(X)
-    assert_array_equal(pairwise_distances(X_t).argsort()[:, 1],
-                       np.array([2, 3, 0, 1]))
+    assert_array_equal(pairwise_distances(X_t).argsort()[:, 1], np.array([2, 3, 0, 1]))
 
 
 def test_toy_example_collapse_points():
@@ -65,43 +69,40 @@ def test_toy_example_collapse_points():
     y = [0, 0, 1]
 
     class LossStorer:
-
         def __init__(self, X, y):
             self.loss = np.inf  # initialize the loss to very high
             # Initialize a fake NCA and variables needed to compute the loss:
             self.fake_nca = NeighborhoodComponentsAnalysis()
             self.fake_nca.n_iter_ = np.inf
-            self.X, y, _ = self.fake_nca._validate_params(X, y)
+            self.X, y = validate_data(self.fake_nca, X, y, ensure_min_samples=2)
+            y = LabelEncoder().fit_transform(y)
             self.same_class_mask = y[:, np.newaxis] == y[np.newaxis, :]
 
         def callback(self, transformation, n_iter):
             """Stores the last value of the loss function"""
-            self.loss, _ = self.fake_nca._loss_grad_lbfgs(transformation,
-                                                          self.X,
-                                                          self.same_class_mask,
-                                                          -1.0)
+            self.loss, _ = self.fake_nca._loss_grad_lbfgs(
+                transformation, self.X, self.same_class_mask, -1.0
+            )
 
     loss_storer = LossStorer(X, y)
-    nca = NeighborhoodComponentsAnalysis(random_state=42,
-                                         callback=loss_storer.callback)
+    nca = NeighborhoodComponentsAnalysis(random_state=42, callback=loss_storer.callback)
     X_t = nca.fit_transform(X, y)
     print(X_t)
     # test that points are collapsed into one point
-    assert_array_almost_equal(X_t - X_t[0], 0.)
+    assert_array_almost_equal(X_t - X_t[0], 0.0)
     assert abs(loss_storer.loss + 1) < 1e-10
 
 
-def test_finite_differences():
+def test_finite_differences(global_random_seed):
     """Test gradient of loss function
 
     Assert that the gradient is almost equal to its finite differences
     approximation.
     """
     # Initialize the transformation `M`, as well as `X` and `y` and `NCA`
-    rng = np.random.RandomState(42)
-    X, y = make_classification()
-    M = rng.randn(rng.randint(1, X.shape[1] + 1),
-                  X.shape[1])
+    rng = np.random.RandomState(global_random_seed)
+    X, y = make_classification(random_state=global_random_seed)
+    M = rng.randn(rng.randint(1, X.shape[1] + 1), X.shape[1])
     nca = NeighborhoodComponentsAnalysis()
     nca.n_iter_ = 0
     mask = y[:, np.newaxis] == y[np.newaxis, :]
@@ -112,9 +113,9 @@ def fun(M):
     def grad(M):
         return nca._loss_grad_lbfgs(M, X, mask)[1]
 
-    # compute relative error
-    rel_diff = check_grad(fun, grad, M.ravel()) / np.linalg.norm(grad(M))
-    np.testing.assert_almost_equal(rel_diff, 0., decimal=5)
+    # compare the gradient to a finite difference approximation
+    diff = check_grad(fun, grad, M.ravel())
+    assert diff == pytest.approx(0.0, abs=1e-4)
 
 
 def test_params_validation():
@@ -124,27 +125,6 @@ def test_params_validation():
     NCA = NeighborhoodComponentsAnalysis
     rng = np.random.RandomState(42)
 
-    # TypeError
-    with pytest.raises(TypeError):
-        NCA(max_iter='21').fit(X, y)
-    with pytest.raises(TypeError):
-        NCA(verbose='true').fit(X, y)
-    with pytest.raises(TypeError):
-        NCA(tol='1').fit(X, y)
-    with pytest.raises(TypeError):
-        NCA(n_components='invalid').fit(X, y)
-    with pytest.raises(TypeError):
-        NCA(warm_start=1).fit(X, y)
-
-    # ValueError
-    msg = (
-        r"`init` must be 'auto', 'pca', 'lda', 'identity', 'random' or a "
-        r"numpy array of shape (n_components, n_features)."
-    )
-    with pytest.raises(ValueError, match=re.escape(msg)):
-        NCA(init=1).fit(X, y)
-    with pytest.raises(ValueError, match='`max_iter`= -1, must be >= 1.'):
-        NCA(max_iter=-1).fit(X, y)
     init = rng.rand(5, 3)
     msg = (
         f"The output dimensionality ({init.shape[0]}) "
@@ -215,7 +195,7 @@ def test_n_components():
         nca.fit(X, y)
 
     # n_components < X.shape[1]
-    nca = NeighborhoodComponentsAnalysis(n_components=2, init='identity')
+    nca = NeighborhoodComponentsAnalysis(n_components=2, init="identity")
     nca.fit(X, y)
 
 
@@ -224,23 +204,23 @@ def test_init_transformation():
     X, y = make_blobs(n_samples=30, centers=6, n_features=5, random_state=0)
 
     # Start learning from scratch
-    nca = NeighborhoodComponentsAnalysis(init='identity')
+    nca = NeighborhoodComponentsAnalysis(init="identity")
     nca.fit(X, y)
 
     # Initialize with random
-    nca_random = NeighborhoodComponentsAnalysis(init='random')
+    nca_random = NeighborhoodComponentsAnalysis(init="random")
     nca_random.fit(X, y)
 
     # Initialize with auto
-    nca_auto = NeighborhoodComponentsAnalysis(init='auto')
+    nca_auto = NeighborhoodComponentsAnalysis(init="auto")
     nca_auto.fit(X, y)
 
     # Initialize with PCA
-    nca_pca = NeighborhoodComponentsAnalysis(init='pca')
+    nca_pca = NeighborhoodComponentsAnalysis(init="pca")
     nca_pca.fit(X, y)
 
     # Initialize with LDA
-    nca_lda = NeighborhoodComponentsAnalysis(init='lda')
+    nca_lda = NeighborhoodComponentsAnalysis(init="lda")
     nca_lda.fit(X, y)
 
     init = rng.rand(X.shape[1], X.shape[1])
@@ -283,18 +263,17 @@ def test_init_transformation():
         nca.fit(X, y)
 
 
-@pytest.mark.parametrize('n_samples', [3, 5, 7, 11])
-@pytest.mark.parametrize('n_features', [3, 5, 7, 11])
-@pytest.mark.parametrize('n_classes', [5, 7, 11])
-@pytest.mark.parametrize('n_components', [3, 5, 7, 11])
+@pytest.mark.parametrize("n_samples", [3, 5, 7, 11])
+@pytest.mark.parametrize("n_features", [3, 5, 7, 11])
+@pytest.mark.parametrize("n_classes", [5, 7, 11])
+@pytest.mark.parametrize("n_components", [3, 5, 7, 11])
 def test_auto_init(n_samples, n_features, n_classes, n_components):
     # Test that auto choose the init as expected with every configuration
     # of order of n_samples, n_features, n_classes and n_components.
     rng = np.random.RandomState(42)
-    nca_base = NeighborhoodComponentsAnalysis(init='auto',
-                                              n_components=n_components,
-                                              max_iter=1,
-                                              random_state=rng)
+    nca_base = NeighborhoodComponentsAnalysis(
+        init="auto", n_components=n_components, max_iter=1, random_state=rng
+    )
     if n_classes >= n_samples:
         pass
         # n_classes > n_samples is impossible, and n_classes == n_samples
@@ -310,25 +289,36 @@ def test_auto_init(n_samples, n_features, n_classes, n_components):
             nca = clone(nca_base)
             nca.fit(X, y)
             if n_components <= min(n_classes - 1, n_features):
-                nca_other = clone(nca_base).set_params(init='lda')
+                nca_other = clone(nca_base).set_params(init="lda")
             elif n_components < min(n_features, n_samples):
-                nca_other = clone(nca_base).set_params(init='pca')
+                nca_other = clone(nca_base).set_params(init="pca")
             else:
-                nca_other = clone(nca_base).set_params(init='identity')
+                nca_other = clone(nca_base).set_params(init="identity")
             nca_other.fit(X, y)
             assert_array_almost_equal(nca.components_, nca_other.components_)
 
 
 def test_warm_start_validation():
-    X, y = make_classification(n_samples=30, n_features=5, n_classes=4,
-                               n_redundant=0, n_informative=5, random_state=0)
+    X, y = make_classification(
+        n_samples=30,
+        n_features=5,
+        n_classes=4,
+        n_redundant=0,
+        n_informative=5,
+        random_state=0,
+    )
 
     nca = NeighborhoodComponentsAnalysis(warm_start=True, max_iter=5)
     nca.fit(X, y)
 
-    X_less_features, y = make_classification(n_samples=30, n_features=4,
-                                             n_classes=4, n_redundant=0,
-                                             n_informative=4, random_state=0)
+    X_less_features, y = make_classification(
+        n_samples=30,
+        n_features=4,
+        n_classes=4,
+        n_redundant=0,
+        n_informative=4,
+        random_state=0,
+    )
     msg = (
         f"The new inputs dimensionality ({X_less_features.shape[1]}) "
         "does not match the input dimensionality of the previously learned "
@@ -356,29 +346,34 @@ def test_warm_start_effectiveness():
     nca_cold.fit(iris_data, iris_target)
     transformation_cold_plus_one = nca_cold.components_
 
-    diff_warm = np.sum(np.abs(transformation_warm_plus_one -
-                              transformation_warm))
-    diff_cold = np.sum(np.abs(transformation_cold_plus_one -
-                              transformation_cold))
-    assert diff_warm < 3.0, ("Transformer changed significantly after one "
-                             "iteration even though it was warm-started.")
+    diff_warm = np.sum(np.abs(transformation_warm_plus_one - transformation_warm))
+    diff_cold = np.sum(np.abs(transformation_cold_plus_one - transformation_cold))
+    assert diff_warm < 3.0, (
+        "Transformer changed significantly after one "
+        "iteration even though it was warm-started."
+    )
 
-    assert diff_cold > diff_warm, ("Cold-started transformer changed less "
-                                   "significantly than warm-started "
-                                   "transformer after one iteration.")
+    assert diff_cold > diff_warm, (
+        "Cold-started transformer changed less "
+        "significantly than warm-started "
+        "transformer after one iteration."
+    )
 
 
-@pytest.mark.parametrize('init_name', ['pca', 'lda', 'identity', 'random',
-                                       'precomputed'])
+@pytest.mark.parametrize(
+    "init_name", ["pca", "lda", "identity", "random", "precomputed"]
+)
 def test_verbose(init_name, capsys):
     # assert there is proper output when verbose = 1, for every initialization
     # except auto because auto will call one of the others
     rng = np.random.RandomState(42)
     X, y = make_blobs(n_samples=30, centers=6, n_features=5, random_state=0)
-    regexp_init = r'... done in \ *\d+\.\d{2}s'
-    msgs = {'pca': "Finding principal components" + regexp_init,
-            'lda': "Finding most discriminative components" + regexp_init}
-    if init_name == 'precomputed':
+    regexp_init = r"... done in \ *\d+\.\d{2}s"
+    msgs = {
+        "pca": "Finding principal components" + regexp_init,
+        "lda": "Finding most discriminative components" + regexp_init,
+    }
+    if init_name == "precomputed":
         init = rng.randn(X.shape[1], X.shape[1])
     else:
         init = init_name
@@ -387,26 +382,29 @@ def test_verbose(init_name, capsys):
     out, _ = capsys.readouterr()
 
     # check output
-    lines = re.split('\n+', out)
+    lines = re.split("\n+", out)
     # if pca or lda init, an additional line is printed, so we test
     # it and remove it to test the rest equally among initializations
-    if init_name in ['pca', 'lda']:
+    if init_name in ["pca", "lda"]:
         assert re.match(msgs[init_name], lines[0])
         lines = lines[1:]
-    assert lines[0] == '[NeighborhoodComponentsAnalysis]'
-    header = '{:>10} {:>20} {:>10}'.format('Iteration', 'Objective Value',
-                                           'Time(s)')
-    assert lines[1] == '[NeighborhoodComponentsAnalysis] {}'.format(header)
-    assert lines[2] == ('[NeighborhoodComponentsAnalysis] {}'
-                        .format('-' * len(header)))
+    assert lines[0] == "[NeighborhoodComponentsAnalysis]"
+    header = "{:>10} {:>20} {:>10}".format("Iteration", "Objective Value", "Time(s)")
+    assert lines[1] == "[NeighborhoodComponentsAnalysis] {}".format(header)
+    assert lines[2] == "[NeighborhoodComponentsAnalysis] {}".format("-" * len(header))
     for line in lines[3:-2]:
         # The following regex will match for instance:
         # '[NeighborhoodComponentsAnalysis]  0    6.988936e+01   0.01'
-        assert re.match(r'\[NeighborhoodComponentsAnalysis\] *\d+ *\d\.\d{6}e'
-                        r'[+|-]\d+\ *\d+\.\d{2}', line)
-    assert re.match(r'\[NeighborhoodComponentsAnalysis\] Training took\ *'
-                    r'\d+\.\d{2}s\.', lines[-2])
-    assert lines[-1] == ''
+        assert re.match(
+            r"\[NeighborhoodComponentsAnalysis\] *\d+ *\d\.\d{6}e"
+            r"[+|-]\d+\ *\d+\.\d{2}",
+            line,
+        )
+    assert re.match(
+        r"\[NeighborhoodComponentsAnalysis\] Training took\ *\d+\.\d{2}s\.",
+        lines[-2],
+    )
+    assert lines[-1] == ""
 
 
 def test_no_verbose(capsys):
@@ -415,16 +413,16 @@ def test_no_verbose(capsys):
     nca.fit(iris_data, iris_target)
     out, _ = capsys.readouterr()
     # check output
-    assert(out == '')
+    assert out == ""
 
 
 def test_singleton_class():
-    X = iris_data
-    y = iris_target
+    X = iris_data.copy()
+    y = iris_target.copy()
 
     # one singleton class
     singleton_class = 1
-    ind_singleton, = np.where(y == singleton_class)
+    (ind_singleton,) = np.where(y == singleton_class)
     y[ind_singleton] = 2
     y[ind_singleton[0]] = singleton_class
 
@@ -432,8 +430,8 @@ def test_singleton_class():
     nca.fit(X, y)
 
     # One non-singleton class
-    ind_1, = np.where(y == 1)
-    ind_2, = np.where(y == 2)
+    (ind_1,) = np.where(y == 1)
+    (ind_2,) = np.where(y == 2)
     y[ind_1] = 0
     y[ind_1[0]] = 1
     y[ind_2] = 0
@@ -443,13 +441,13 @@ def test_singleton_class():
     nca.fit(X, y)
 
     # Only singleton classes
-    ind_0, = np.where(y == 0)
-    ind_1, = np.where(y == 1)
-    ind_2, = np.where(y == 2)
+    (ind_0,) = np.where(y == 0)
+    (ind_1,) = np.where(y == 1)
+    (ind_2,) = np.where(y == 2)
     X = X[[ind_0[0], ind_1[0], ind_2[0]]]
     y = y[[ind_0[0], ind_1[0], ind_2[0]]]
 
-    nca = NeighborhoodComponentsAnalysis(init='identity', max_iter=30)
+    nca = NeighborhoodComponentsAnalysis(init="identity", max_iter=30)
     nca.fit(X, y)
     assert_array_equal(X, nca.transform(X))
 
@@ -458,36 +456,28 @@ def test_one_class():
     X = iris_data[iris_target == 0]
     y = iris_target[iris_target == 0]
 
-    nca = NeighborhoodComponentsAnalysis(max_iter=30,
-                                         n_components=X.shape[1],
-                                         init='identity')
+    nca = NeighborhoodComponentsAnalysis(
+        max_iter=30, n_components=X.shape[1], init="identity"
+    )
     nca.fit(X, y)
     assert_array_equal(X, nca.transform(X))
 
 
 def test_callback(capsys):
-    X = iris_data
-    y = iris_target
-
-    nca = NeighborhoodComponentsAnalysis(callback='my_cb')
-    with pytest.raises(ValueError):
-        nca.fit(X, y)
-
     max_iter = 10
 
     def my_cb(transformation, n_iter):
-        assert transformation.shape == (iris_data.shape[1]**2,)
+        assert transformation.shape == (iris_data.shape[1] ** 2,)
         rem_iter = max_iter - n_iter
-        print('{} iterations remaining...'.format(rem_iter))
+        print("{} iterations remaining...".format(rem_iter))
 
     # assert that my_cb is called
-    nca = NeighborhoodComponentsAnalysis(max_iter=max_iter,
-                                         callback=my_cb, verbose=1)
+    nca = NeighborhoodComponentsAnalysis(max_iter=max_iter, callback=my_cb, verbose=1)
     nca.fit(iris_data, iris_target)
     out, _ = capsys.readouterr()
 
     # check output
-    assert('{} iterations remaining...'.format(max_iter - 1) in out)
+    assert "{} iterations remaining...".format(max_iter - 1) in out
 
 
 def test_expected_transformation_shape():
@@ -496,13 +486,13 @@ def test_expected_transformation_shape():
     y = iris_target
 
     class TransformationStorer:
-
         def __init__(self, X, y):
             # Initialize a fake NCA and variables needed to call the loss
             # function:
             self.fake_nca = NeighborhoodComponentsAnalysis()
             self.fake_nca.n_iter_ = np.inf
-            self.X, y, _ = self.fake_nca._validate_params(X, y)
+            self.X, y = validate_data(self.fake_nca, X, y, ensure_min_samples=2)
+            y = LabelEncoder().fit_transform(y)
             self.same_class_mask = y[:, np.newaxis] == y[np.newaxis, :]
 
         def callback(self, transformation, n_iter):
@@ -514,20 +504,25 @@ def callback(self, transformation, n_iter):
     cb = transformation_storer.callback
     nca = NeighborhoodComponentsAnalysis(max_iter=5, callback=cb)
     nca.fit(X, y)
-    assert transformation_storer.transformation.size == X.shape[1]**2
+    assert transformation_storer.transformation.size == X.shape[1] ** 2
 
 
 def test_convergence_warning():
     nca = NeighborhoodComponentsAnalysis(max_iter=2, verbose=1)
     cls_name = nca.__class__.__name__
-    msg = '[{}] NCA did not converge'.format(cls_name)
+    msg = "[{}] NCA did not converge".format(cls_name)
     with pytest.warns(ConvergenceWarning, match=re.escape(msg)):
         nca.fit(iris_data, iris_target)
 
 
-@pytest.mark.parametrize('param, value', [('n_components', np.int32(3)),
-                                          ('max_iter', np.int32(100)),
-                                          ('tol', np.float32(0.0001))])
+@pytest.mark.parametrize(
+    "param, value",
+    [
+        ("n_components", np.int32(3)),
+        ("max_iter", np.int32(100)),
+        ("tol", np.float32(0.0001)),
+    ],
+)
 def test_parameters_valid_types(param, value):
     # check that no error is raised when parameters have numpy integer or
     # floating types.
@@ -537,3 +532,32 @@ def test_parameters_valid_types(param, value):
     y = iris_target
 
     nca.fit(X, y)
+
+
+@pytest.mark.parametrize("n_components", [None, 2])
+def test_nca_feature_names_out(n_components):
+    """Check `get_feature_names_out` for `NeighborhoodComponentsAnalysis`.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/28293
+    """
+
+    X = iris_data
+    y = iris_target
+
+    est = NeighborhoodComponentsAnalysis(n_components=n_components).fit(X, y)
+    names_out = est.get_feature_names_out()
+
+    class_name_lower = est.__class__.__name__.lower()
+
+    if n_components is not None:
+        expected_n_features = n_components
+    else:
+        expected_n_features = X.shape[1]
+
+    expected_names_out = np.array(
+        [f"{class_name_lower}{i}" for i in range(expected_n_features)],
+        dtype=object,
+    )
+
+    assert_array_equal(names_out, expected_names_out)
diff --git a/sklearn/neighbors/tests/test_nearest_centroid.py b/sklearn/neighbors/tests/test_nearest_centroid.py
index 9af02b07e2a96..1aa9274cd28a8 100644
--- a/sklearn/neighbors/tests/test_nearest_centroid.py
+++ b/sklearn/neighbors/tests/test_nearest_centroid.py
@@ -1,21 +1,29 @@
 """
 Testing for the nearest centroid module.
 """
+
 import numpy as np
 import pytest
-from scipy import sparse as sp
-from numpy.testing import assert_array_equal
 
-from sklearn.neighbors import NearestCentroid
 from sklearn import datasets
+from sklearn.neighbors import NearestCentroid
+from sklearn.utils._testing import (
+    assert_allclose,
+    assert_array_almost_equal,
+    assert_array_equal,
+)
+from sklearn.utils.fixes import CSR_CONTAINERS
 
 # toy sample
 X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]]
-X_csr = sp.csr_matrix(X)  # Sparse matrix
 y = [-1, -1, -1, 1, 1, 1]
 T = [[-1, -1], [2, 2], [3, 2]]
-T_csr = sp.csr_matrix(T)
 true_result = [-1, 1, 1]
+true_result_prior1 = [-1, 1, 1]
+
+true_discriminant_scores = [-32, 64, 80]
+true_proba = [[1, 1.26642e-14], [1.60381e-28, 1], [1.80485e-35, 1]]
+
 
 # also load the iris dataset
 # and randomly permute it
@@ -26,11 +34,36 @@
 iris.target = iris.target[perm]
 
 
-def test_classification_toy():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_classification_toy(csr_container):
+    # Check classification on a toy dataset, including sparse versions.
+    X_csr = csr_container(X)
+    T_csr = csr_container(T)
+
     # Check classification on a toy dataset, including sparse versions.
     clf = NearestCentroid()
     clf.fit(X, y)
     assert_array_equal(clf.predict(T), true_result)
+    assert_array_almost_equal(clf.decision_function(T), true_discriminant_scores)
+    assert_array_almost_equal(clf.predict_proba(T), true_proba)
+
+    # Test uniform priors
+    clf = NearestCentroid(priors="uniform")
+    clf.fit(X, y)
+    assert_array_equal(clf.predict(T), true_result)
+    assert_array_almost_equal(clf.decision_function(T), true_discriminant_scores)
+    assert_array_almost_equal(clf.predict_proba(T), true_proba)
+
+    clf = NearestCentroid(priors="empirical")
+    clf.fit(X, y)
+    assert_array_equal(clf.predict(T), true_result)
+    assert_array_almost_equal(clf.decision_function(T), true_discriminant_scores)
+    assert_array_almost_equal(clf.predict_proba(T), true_proba)
+
+    # Test custom priors
+    clf = NearestCentroid(priors=[0.25, 0.75])
+    clf.fit(X, y)
+    assert_array_equal(clf.predict(T), true_result_prior1)
 
     # Same test, but with a sparse matrix to fit and test.
     clf = NearestCentroid()
@@ -53,15 +86,9 @@ def test_classification_toy():
     assert_array_equal(clf.predict(T_csr.tolil()), true_result)
 
 
-def test_precomputed():
-    clf = NearestCentroid(metric='precomputed')
-    with pytest.raises(ValueError):
-        clf.fit(X, y)
-
-
 def test_iris():
     # Check consistency on dataset iris.
-    for metric in ('euclidean', 'cosine'):
+    for metric in ("euclidean", "manhattan"):
         clf = NearestCentroid(metric=metric).fit(iris.data, iris.target)
         score = np.mean(clf.predict(iris.data) == iris.target)
         assert score > 0.9, "Failed with score = " + str(score)
@@ -69,10 +96,9 @@ def test_iris():
 
 def test_iris_shrinkage():
     # Check consistency on dataset iris, when using shrinkage.
-    for metric in ('euclidean', 'cosine'):
+    for metric in ("euclidean", "manhattan"):
         for shrink_threshold in [None, 0.1, 0.5]:
-            clf = NearestCentroid(metric=metric,
-                                  shrink_threshold=shrink_threshold)
+            clf = NearestCentroid(metric=metric, shrink_threshold=shrink_threshold)
             clf = clf.fit(iris.data, iris.target)
             score = np.mean(clf.predict(iris.data) == iris.target)
             assert score > 0.8, "Failed with score = " + str(score)
@@ -90,9 +116,11 @@ def test_pickle():
     obj2 = pickle.loads(s)
     assert type(obj2) == obj.__class__
     score2 = obj2.score(iris.data, iris.target)
-    assert_array_equal(score, score2,
-                       "Failed to generate same score"
-                       " after pickling (classification).")
+    assert_array_equal(
+        score,
+        score2,
+        "Failed to generate same score after pickling (classification).",
+    )
 
 
 def test_shrinkage_correct():
@@ -136,10 +164,12 @@ def test_predict_translated_data():
     assert_array_equal(y_init, y_translate)
 
 
-def test_manhattan_metric():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_manhattan_metric(csr_container):
     # Test the manhattan metric.
+    X_csr = csr_container(X)
 
-    clf = NearestCentroid(metric='manhattan')
+    clf = NearestCentroid(metric="manhattan")
     clf.fit(X, y)
     dense_centroid = clf.centroids_
     clf.fit(X_csr, y)
@@ -159,3 +189,49 @@ def test_features_zero_var():
     clf = NearestCentroid(shrink_threshold=0.1)
     with pytest.raises(ValueError):
         clf.fit(X, y)
+
+
+def test_negative_priors_error():
+    """Check that we raise an error when the user-defined priors are negative."""
+    clf = NearestCentroid(priors=[-2, 4])
+    with pytest.raises(ValueError, match="priors must be non-negative"):
+        clf.fit(X, y)
+
+
+def test_warn_non_normalized_priors():
+    """Check that we raise a warning and normalize the user-defined priors when they
+    don't sum to 1.
+    """
+    priors = [2, 4]
+    clf = NearestCentroid(priors=priors)
+    with pytest.warns(
+        UserWarning,
+        match="The priors do not sum to 1. Normalizing such that it sums to one.",
+    ):
+        clf.fit(X, y)
+
+    assert_allclose(clf.class_prior_, np.asarray(priors) / np.asarray(priors).sum())
+
+
+@pytest.mark.parametrize(
+    "response_method", ["decision_function", "predict_proba", "predict_log_proba"]
+)
+def test_method_not_available_with_manhattan(response_method):
+    """Check that we raise an AttributeError with Manhattan metric when trying
+    to call a non-thresholded response method.
+    """
+    clf = NearestCentroid(metric="manhattan").fit(X, y)
+    with pytest.raises(AttributeError):
+        getattr(clf, response_method)(T)
+
+
+@pytest.mark.parametrize("array_constructor", [np.array] + CSR_CONTAINERS)
+def test_error_zero_variances(array_constructor):
+    """Check that we raise an error when the variance for all features is zero."""
+    X = np.ones((len(y), 2))
+    X[:, 1] *= 2
+    X = array_constructor(X)
+
+    clf = NearestCentroid()
+    with pytest.raises(ValueError, match="All features have zero variance"):
+        clf.fit(X, y)
diff --git a/sklearn/neighbors/tests/test_neighbors.py b/sklearn/neighbors/tests/test_neighbors.py
index 8ce52119faa02..ae589b30dd743 100644
--- a/sklearn/neighbors/tests/test_neighbors.py
+++ b/sklearn/neighbors/tests/test_neighbors.py
@@ -1,30 +1,61 @@
+import re
+import warnings
 from itertools import product
 
-import pytest
-import re
+import joblib
 import numpy as np
-from scipy.sparse import (bsr_matrix, coo_matrix, csc_matrix, csr_matrix,
-                          dok_matrix, lil_matrix, issparse)
-
-from sklearn import metrics
-from sklearn import neighbors, datasets
+import pytest
+from scipy.sparse import issparse
+
+from sklearn import (
+    config_context,
+    datasets,
+    metrics,
+    neighbors,
+)
 from sklearn.base import clone
-from sklearn.exceptions import DataConversionWarning
-from sklearn.exceptions import EfficiencyWarning
-from sklearn.exceptions import NotFittedError
-from sklearn.metrics.pairwise import pairwise_distances
-from sklearn.model_selection import cross_val_score
-from sklearn.model_selection import train_test_split
-from sklearn.neighbors import VALID_METRICS_SPARSE, VALID_METRICS
-from sklearn.neighbors._base import _is_sorted_by_data, _check_precomputed
+from sklearn.exceptions import EfficiencyWarning, NotFittedError
+from sklearn.metrics._dist_metrics import (
+    DistanceMetric,
+)
+from sklearn.metrics.pairwise import PAIRWISE_BOOLEAN_FUNCTIONS, pairwise_distances
+from sklearn.metrics.tests.test_dist_metrics import BOOL_METRICS
+from sklearn.metrics.tests.test_pairwise_distances_reduction import (
+    assert_compatible_argkmin_results,
+    assert_compatible_radius_results,
+)
+from sklearn.model_selection import (
+    LeaveOneOut,
+    cross_val_predict,
+    cross_val_score,
+    train_test_split,
+)
+from sklearn.neighbors import (
+    VALID_METRICS_SPARSE,
+    KNeighborsRegressor,
+)
+from sklearn.neighbors._base import (
+    KNeighborsMixin,
+    _check_precomputed,
+    _is_sorted_by_data,
+    sort_graph_by_row_values,
+)
 from sklearn.pipeline import make_pipeline
-from sklearn.utils._testing import assert_array_almost_equal
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils._testing import ignore_warnings
+from sklearn.utils._testing import (
+    assert_allclose,
+    assert_array_equal,
+    ignore_warnings,
+)
+from sklearn.utils.fixes import (
+    BSR_CONTAINERS,
+    COO_CONTAINERS,
+    CSC_CONTAINERS,
+    CSR_CONTAINERS,
+    DIA_CONTAINERS,
+    DOK_CONTAINERS,
+    LIL_CONTAINERS,
+)
 from sklearn.utils.validation import check_random_state
-from sklearn.utils.fixes import sp_version, parse_version
-
-import joblib
 
 rng = np.random.RandomState(0)
 # load and shuffle iris dataset
@@ -39,97 +70,289 @@
 digits.data = digits.data[perm]
 digits.target = digits.target[perm]
 
-SPARSE_TYPES = (bsr_matrix, coo_matrix, csc_matrix, csr_matrix, dok_matrix,
-                lil_matrix)
+SPARSE_TYPES = tuple(
+    BSR_CONTAINERS
+    + COO_CONTAINERS
+    + CSC_CONTAINERS
+    + CSR_CONTAINERS
+    + DOK_CONTAINERS
+    + LIL_CONTAINERS
+)
 SPARSE_OR_DENSE = SPARSE_TYPES + (np.asarray,)
 
-ALGORITHMS = ('ball_tree', 'brute', 'kd_tree', 'auto')
+ALGORITHMS = ("ball_tree", "brute", "kd_tree", "auto")
+COMMON_VALID_METRICS = sorted(
+    set.intersection(*map(set, neighbors.VALID_METRICS.values()))
+)
+
 P = (1, 2, 3, 4, np.inf)
-JOBLIB_BACKENDS = list(joblib.parallel.BACKENDS.keys())
 
 # Filter deprecation warnings.
 neighbors.kneighbors_graph = ignore_warnings(neighbors.kneighbors_graph)
-neighbors.radius_neighbors_graph = ignore_warnings(
-    neighbors.radius_neighbors_graph)
+neighbors.radius_neighbors_graph = ignore_warnings(neighbors.radius_neighbors_graph)
+
+# A list containing metrics where the string specifies the use of the
+# DistanceMetric object directly (as resolved in _parse_metric)
+DISTANCE_METRIC_OBJS = ["DM_euclidean"]
+
+
+def _parse_metric(metric: str, dtype=None):
+    """
+    Helper function for properly building a type-specialized DistanceMetric instances.
+
+    Constructs a type-specialized DistanceMetric instance from a string
+    beginning with "DM_" while allowing a pass-through for other metric-specifying
+    strings. This is necessary since we wish to parameterize dtype independent of
+    metric, yet DistanceMetric requires it for construction.
+
+    """
+    if metric[:3] == "DM_":
+        return DistanceMetric.get_metric(metric[3:], dtype=dtype)
+    return metric
+
+
+def _generate_test_params_for(metric: str, n_features: int):
+    """Return list of DistanceMetric kwargs for tests."""
+
+    # Distinguishing on cases not to compute unneeded datastructures.
+    rng = np.random.RandomState(1)
+
+    if metric == "minkowski":
+        return [
+            dict(p=1.5),
+            dict(p=2),
+            dict(p=3),
+            dict(p=np.inf),
+            dict(p=3, w=rng.rand(n_features)),
+        ]
+
+    if metric == "seuclidean":
+        return [dict(V=rng.rand(n_features))]
+
+    if metric == "mahalanobis":
+        A = rng.rand(n_features, n_features)
+        # Make the matrix symmetric positive definite
+        VI = A + A.T + 3 * np.eye(n_features)
+        return [dict(VI=VI)]
+
+    # Case of: "euclidean", "manhattan", "chebyshev", "haversine" or any other metric.
+    # In those cases, no kwargs are needed.
+    return [{}]
 
 
 def _weight_func(dist):
-    """ Weight function to replace lambda d: d ** -2.
+    """Weight function to replace lambda d: d ** -2.
     The lambda function is not valid because:
-    if d==0 then 0^-2 is not valid. """
+    if d==0 then 0^-2 is not valid."""
 
     # Dist could be multidimensional, flatten it so all values
     # can be looped
-    with np.errstate(divide='ignore'):
-        retval = 1. / dist
-    return retval ** 2
+    with np.errstate(divide="ignore"):
+        retval = 1.0 / dist
+    return retval**2
+
+
+WEIGHTS = ["uniform", "distance", _weight_func]
+
+
+@pytest.mark.parametrize(
+    "n_samples, n_features, n_query_pts, n_neighbors",
+    [
+        (100, 100, 10, 100),
+        (1000, 5, 100, 1),
+    ],
+)
+@pytest.mark.parametrize("query_is_train", [False, True])
+@pytest.mark.parametrize("metric", COMMON_VALID_METRICS + DISTANCE_METRIC_OBJS)
+def test_unsupervised_kneighbors(
+    global_dtype,
+    n_samples,
+    n_features,
+    n_query_pts,
+    n_neighbors,
+    query_is_train,
+    metric,
+):
+    # The different algorithms must return identical results
+    # on their common metrics, with and without returning
+    # distances
+
+    metric = _parse_metric(metric, global_dtype)
+
+    # Redefining the rng locally to use the same generated X
+    local_rng = np.random.RandomState(0)
+    X = local_rng.rand(n_samples, n_features).astype(global_dtype, copy=False)
+
+    query = (
+        X
+        if query_is_train
+        else local_rng.rand(n_query_pts, n_features).astype(global_dtype, copy=False)
+    )
 
+    results_nodist = []
+    results = []
 
-def test_unsupervised_kneighbors(n_samples=20, n_features=5,
-                                 n_query_pts=2, n_neighbors=5):
-    # Test unsupervised neighbors methods
-    X = rng.rand(n_samples, n_features)
+    for algorithm in ALGORITHMS:
+        if isinstance(metric, DistanceMetric) and global_dtype == np.float32:
+            if "tree" in algorithm:  # pragma: nocover
+                pytest.skip(
+                    "Neither KDTree nor BallTree support 32-bit distance metric"
+                    " objects."
+                )
+        neigh = neighbors.NearestNeighbors(
+            n_neighbors=n_neighbors, algorithm=algorithm, metric=metric
+        )
+        neigh.fit(X)
+
+        results_nodist.append(neigh.kneighbors(query, return_distance=False))
+        results.append(neigh.kneighbors(query, return_distance=True))
+
+    for i in range(len(results) - 1):
+        algorithm = ALGORITHMS[i]
+        next_algorithm = ALGORITHMS[i + 1]
+
+        indices_no_dist = results_nodist[i]
+        distances, next_distances = results[i][0], results[i + 1][0]
+        indices, next_indices = results[i][1], results[i + 1][1]
+        assert_array_equal(
+            indices_no_dist,
+            indices,
+            err_msg=(
+                f"The '{algorithm}' algorithm returns different"
+                "indices depending on 'return_distances'."
+            ),
+        )
+        assert_array_equal(
+            indices,
+            next_indices,
+            err_msg=(
+                f"The '{algorithm}' and '{next_algorithm}' "
+                "algorithms return different indices."
+            ),
+        )
+        assert_allclose(
+            distances,
+            next_distances,
+            err_msg=(
+                f"The '{algorithm}' and '{next_algorithm}' "
+                "algorithms return different distances."
+            ),
+            atol=1e-6,
+        )
+
+
+@pytest.mark.parametrize(
+    "n_samples, n_features, n_query_pts",
+    [
+        (100, 100, 10),
+        (1000, 5, 100),
+    ],
+)
+@pytest.mark.parametrize("metric", COMMON_VALID_METRICS + DISTANCE_METRIC_OBJS)
+@pytest.mark.parametrize("n_neighbors, radius", [(1, 100), (50, 500), (100, 1000)])
+@pytest.mark.parametrize(
+    "NeighborsMixinSubclass",
+    [
+        neighbors.KNeighborsClassifier,
+        neighbors.KNeighborsRegressor,
+        neighbors.RadiusNeighborsClassifier,
+        neighbors.RadiusNeighborsRegressor,
+    ],
+)
+def test_neigh_predictions_algorithm_agnosticity(
+    global_dtype,
+    n_samples,
+    n_features,
+    n_query_pts,
+    metric,
+    n_neighbors,
+    radius,
+    NeighborsMixinSubclass,
+):
+    # The different algorithms must return identical predictions results
+    # on their common metrics.
+
+    metric = _parse_metric(metric, global_dtype)
+    if isinstance(metric, DistanceMetric):
+        if "Classifier" in NeighborsMixinSubclass.__name__:
+            pytest.skip(
+                "Metrics of type `DistanceMetric` are not yet supported for"
+                " classifiers."
+            )
+        if "Radius" in NeighborsMixinSubclass.__name__:
+            pytest.skip(
+                "Metrics of type `DistanceMetric` are not yet supported for"
+                " radius-neighbor estimators."
+            )
+
+    # Redefining the rng locally to use the same generated X
+    local_rng = np.random.RandomState(0)
+    X = local_rng.rand(n_samples, n_features).astype(global_dtype, copy=False)
+    y = local_rng.randint(3, size=n_samples)
+
+    query = local_rng.rand(n_query_pts, n_features).astype(global_dtype, copy=False)
+
+    predict_results = []
+
+    parameter = (
+        n_neighbors if issubclass(NeighborsMixinSubclass, KNeighborsMixin) else radius
+    )
 
-    test = rng.rand(n_query_pts, n_features)
+    for algorithm in ALGORITHMS:
+        if isinstance(metric, DistanceMetric) and global_dtype == np.float32:
+            if "tree" in algorithm:  # pragma: nocover
+                pytest.skip(
+                    "Neither KDTree nor BallTree support 32-bit distance metric"
+                    " objects."
+                )
+        neigh = NeighborsMixinSubclass(parameter, algorithm=algorithm, metric=metric)
+        neigh.fit(X, y)
 
-    for p in P:
-        results_nodist = []
-        results = []
+        predict_results.append(neigh.predict(query))
 
-        for algorithm in ALGORITHMS:
-            neigh = neighbors.NearestNeighbors(n_neighbors=n_neighbors,
-                                               algorithm=algorithm,
-                                               p=p)
-            neigh.fit(X)
+    for i in range(len(predict_results) - 1):
+        algorithm = ALGORITHMS[i]
+        next_algorithm = ALGORITHMS[i + 1]
 
-            results_nodist.append(neigh.kneighbors(test,
-                                                   return_distance=False))
-            results.append(neigh.kneighbors(test, return_distance=True))
+        predictions, next_predictions = predict_results[i], predict_results[i + 1]
 
-        for i in range(len(results) - 1):
-            assert_array_almost_equal(results_nodist[i], results[i][1])
-            assert_array_almost_equal(results[i][0], results[i + 1][0])
-            assert_array_almost_equal(results[i][1], results[i + 1][1])
+        assert_allclose(
+            predictions,
+            next_predictions,
+            err_msg=(
+                f"The '{algorithm}' and '{next_algorithm}' "
+                "algorithms return different predictions."
+            ),
+        )
 
 
-@pytest.mark.parametrize("NearestNeighbors", [neighbors.KNeighborsClassifier,
-                                              neighbors.KNeighborsRegressor,
-                                              neighbors.NearestNeighbors])
-def test_unsupervised_inputs(NearestNeighbors):
+@pytest.mark.parametrize(
+    "KNeighborsMixinSubclass",
+    [
+        neighbors.KNeighborsClassifier,
+        neighbors.KNeighborsRegressor,
+        neighbors.NearestNeighbors,
+    ],
+)
+def test_unsupervised_inputs(global_dtype, KNeighborsMixinSubclass):
     # Test unsupervised inputs for neighbors estimators
 
-    X = rng.random_sample((10, 3))
+    X = rng.random_sample((10, 3)).astype(global_dtype, copy=False)
     y = rng.randint(3, size=10)
     nbrs_fid = neighbors.NearestNeighbors(n_neighbors=1)
     nbrs_fid.fit(X)
 
     dist1, ind1 = nbrs_fid.kneighbors(X)
 
-    nbrs = NearestNeighbors(n_neighbors=1)
+    nbrs = KNeighborsMixinSubclass(n_neighbors=1)
 
     for data in (nbrs_fid, neighbors.BallTree(X), neighbors.KDTree(X)):
         nbrs.fit(data, y)
 
         dist2, ind2 = nbrs.kneighbors(X)
 
-        assert_array_almost_equal(dist1, dist2)
-        assert_array_almost_equal(ind1, ind2)
-
-
-def test_n_neighbors_datatype():
-    # Test to check whether n_neighbors is integer
-    X = [[1, 1], [1, 1], [1, 1]]
-    expected_msg = "n_neighbors does not take .*float.* " \
-                   "value, enter integer value"
-    msg = "Expected n_neighbors > 0. Got -3"
-
-    neighbors_ = neighbors.NearestNeighbors(n_neighbors=3.)
-    with pytest.raises(TypeError, match=expected_msg):
-        neighbors_.fit(X)
-    with pytest.raises(ValueError, match=msg):
-        neighbors_.kneighbors(X=X, n_neighbors=-3)
-    with pytest.raises(TypeError, match=expected_msg):
-        neighbors_.kneighbors(X=X, n_neighbors=3.)
+        assert_allclose(dist1, dist2)
+        assert_array_equal(ind1, ind2)
 
 
 def test_not_fitted_error_gets_raised():
@@ -141,7 +364,7 @@ def test_not_fitted_error_gets_raised():
         neighbors_.radius_neighbors_graph(X)
 
 
-@ignore_warnings(category=EfficiencyWarning)
+@pytest.mark.filterwarnings("ignore:EfficiencyWarning")
 def check_precomputed(make_train_test, estimators):
     """Tests unsupervised NearestNeighbors with a distance matrix."""
     # Note: smaller samples may result in spurious test success
@@ -149,7 +372,9 @@ def check_precomputed(make_train_test, estimators):
     X = rng.random_sample((10, 4))
     Y = rng.random_sample((3, 4))
     DXX, DYX = make_train_test(X, Y)
-    for method in ['kneighbors', ]:
+    for method in [
+        "kneighbors",
+    ]:
         # TODO: also test radius_neighbors, but requires different assertion
 
         # As a feature matrix (n_samples by n_features)
@@ -158,26 +383,28 @@ def check_precomputed(make_train_test, estimators):
         dist_X, ind_X = getattr(nbrs_X, method)(Y)
 
         # As a dense distance matrix (n_samples by n_samples)
-        nbrs_D = neighbors.NearestNeighbors(n_neighbors=3, algorithm='brute',
-                                            metric='precomputed')
+        nbrs_D = neighbors.NearestNeighbors(
+            n_neighbors=3, algorithm="brute", metric="precomputed"
+        )
         nbrs_D.fit(DXX)
         dist_D, ind_D = getattr(nbrs_D, method)(DYX)
-        assert_array_almost_equal(dist_X, dist_D)
-        assert_array_almost_equal(ind_X, ind_D)
+        assert_allclose(dist_X, dist_D)
+        assert_array_equal(ind_X, ind_D)
 
         # Check auto works too
-        nbrs_D = neighbors.NearestNeighbors(n_neighbors=3, algorithm='auto',
-                                            metric='precomputed')
+        nbrs_D = neighbors.NearestNeighbors(
+            n_neighbors=3, algorithm="auto", metric="precomputed"
+        )
         nbrs_D.fit(DXX)
         dist_D, ind_D = getattr(nbrs_D, method)(DYX)
-        assert_array_almost_equal(dist_X, dist_D)
-        assert_array_almost_equal(ind_X, ind_D)
+        assert_allclose(dist_X, dist_D)
+        assert_array_equal(ind_X, ind_D)
 
         # Check X=None in prediction
         dist_X, ind_X = getattr(nbrs_X, method)(None)
         dist_D, ind_D = getattr(nbrs_D, method)(None)
-        assert_array_almost_equal(dist_X, dist_D)
-        assert_array_almost_equal(ind_X, ind_D)
+        assert_allclose(dist_X, dist_D)
+        assert_array_equal(ind_X, ind_D)
 
         # Must raise a ValueError if the matrix is not of correct shape
         with pytest.raises(ValueError):
@@ -185,32 +412,38 @@ def check_precomputed(make_train_test, estimators):
 
     target = np.arange(X.shape[0])
     for Est in estimators:
-        est = Est(metric='euclidean')
+        est = Est(metric="euclidean")
         est.radius = est.n_neighbors = 1
         pred_X = est.fit(X, target).predict(Y)
-        est.metric = 'precomputed'
+        est.metric = "precomputed"
         pred_D = est.fit(DXX, target).predict(DYX)
-        assert_array_almost_equal(pred_X, pred_D)
+        assert_allclose(pred_X, pred_D)
 
 
 def test_precomputed_dense():
     def make_train_test(X_train, X_test):
-        return (metrics.pairwise_distances(X_train),
-                metrics.pairwise_distances(X_test, X_train))
+        return (
+            metrics.pairwise_distances(X_train),
+            metrics.pairwise_distances(X_test, X_train),
+        )
 
     estimators = [
-        neighbors.KNeighborsClassifier, neighbors.KNeighborsRegressor,
-        neighbors.RadiusNeighborsClassifier, neighbors.RadiusNeighborsRegressor
+        neighbors.KNeighborsClassifier,
+        neighbors.KNeighborsRegressor,
+        neighbors.RadiusNeighborsClassifier,
+        neighbors.RadiusNeighborsRegressor,
     ]
     check_precomputed(make_train_test, estimators)
 
 
-@pytest.mark.parametrize('fmt', ['csr', 'lil'])
+@pytest.mark.parametrize("fmt", ["csr", "lil"])
 def test_precomputed_sparse_knn(fmt):
     def make_train_test(X_train, X_test):
         nn = neighbors.NearestNeighbors(n_neighbors=3 + 1).fit(X_train)
-        return (nn.kneighbors_graph(X_train, mode='distance').asformat(fmt),
-                nn.kneighbors_graph(X_test, mode='distance').asformat(fmt))
+        return (
+            nn.kneighbors_graph(X_train, mode="distance").asformat(fmt),
+            nn.kneighbors_graph(X_test, mode="distance").asformat(fmt),
+        )
 
     # We do not test RadiusNeighborsClassifier and RadiusNeighborsRegressor
     # since the precomputed neighbors graph is built with k neighbors only.
@@ -221,14 +454,14 @@ def make_train_test(X_train, X_test):
     check_precomputed(make_train_test, estimators)
 
 
-@pytest.mark.parametrize('fmt', ['csr', 'lil'])
+@pytest.mark.parametrize("fmt", ["csr", "lil"])
 def test_precomputed_sparse_radius(fmt):
     def make_train_test(X_train, X_test):
         nn = neighbors.NearestNeighbors(radius=1).fit(X_train)
-        return (nn.radius_neighbors_graph(X_train,
-                                          mode='distance').asformat(fmt),
-                nn.radius_neighbors_graph(X_test,
-                                          mode='distance').asformat(fmt))
+        return (
+            nn.radius_neighbors_graph(X_train, mode="distance").asformat(fmt),
+            nn.radius_neighbors_graph(X_test, mode="distance").asformat(fmt),
+        )
 
     # We do not test KNeighborsClassifier and KNeighborsRegressor
     # since the precomputed neighbors graph is built with a radius.
@@ -239,14 +472,15 @@ def make_train_test(X_train, X_test):
     check_precomputed(make_train_test, estimators)
 
 
-def test_is_sorted_by_data():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_is_sorted_by_data(csr_container):
     # Test that _is_sorted_by_data works as expected. In CSR sparse matrix,
     # entries in each row can be sorted by indices, by data, or unsorted.
     # _is_sorted_by_data should return True when entries are sorted by data,
     # and False in all other cases.
 
-    # Test with sorted 1D array
-    X = csr_matrix(np.arange(10))
+    # Test with sorted single row sparse array
+    X = csr_container(np.arange(10).reshape(1, 10))
     assert _is_sorted_by_data(X)
     # Test with unsorted 1D array
     X[0, 2] = 5
@@ -254,53 +488,117 @@ def test_is_sorted_by_data():
 
     # Test when the data is sorted in each sample, but not necessarily
     # between samples
-    X = csr_matrix([[0, 1, 2], [3, 0, 0], [3, 4, 0], [1, 0, 2]])
+    X = csr_container([[0, 1, 2], [3, 0, 0], [3, 4, 0], [1, 0, 2]])
     assert _is_sorted_by_data(X)
 
     # Test with duplicates entries in X.indptr
     data, indices, indptr = [0, 4, 2, 2], [0, 1, 1, 1], [0, 2, 2, 4]
-    X = csr_matrix((data, indices, indptr), shape=(3, 3))
+    X = csr_container((data, indices, indptr), shape=(3, 3))
     assert _is_sorted_by_data(X)
 
 
-@ignore_warnings(category=EfficiencyWarning)
-def test_check_precomputed():
-    # Test that _check_precomputed returns a graph sorted by data
-    X = csr_matrix(np.abs(np.random.RandomState(42).randn(10, 10)))
+@pytest.mark.filterwarnings("ignore:EfficiencyWarning")
+@pytest.mark.parametrize("function", [sort_graph_by_row_values, _check_precomputed])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sort_graph_by_row_values(function, csr_container):
+    # Test that sort_graph_by_row_values returns a graph sorted by row values
+    X = csr_container(np.abs(np.random.RandomState(42).randn(10, 10)))
     assert not _is_sorted_by_data(X)
-    Xt = _check_precomputed(X)
+    Xt = function(X)
     assert _is_sorted_by_data(Xt)
 
-    # est with a different number of nonzero entries for each sample
+    # test with a different number of nonzero entries for each sample
     mask = np.random.RandomState(42).randint(2, size=(10, 10))
     X = X.toarray()
     X[mask == 1] = 0
-    X = csr_matrix(X)
+    X = csr_container(X)
     assert not _is_sorted_by_data(X)
-    Xt = _check_precomputed(X)
+    Xt = function(X)
     assert _is_sorted_by_data(Xt)
 
 
-@ignore_warnings(category=EfficiencyWarning)
-def test_precomputed_sparse_invalid():
-    dist = np.array([[0., 2., 1.], [2., 0., 3.], [1., 3., 0.]])
-    dist_csr = csr_matrix(dist)
+@pytest.mark.filterwarnings("ignore:EfficiencyWarning")
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sort_graph_by_row_values_copy(csr_container):
+    # Test if the sorting is done inplace if X is CSR, so that Xt is X.
+    X_ = csr_container(np.abs(np.random.RandomState(42).randn(10, 10)))
+    assert not _is_sorted_by_data(X_)
+
+    # sort_graph_by_row_values is done inplace if copy=False
+    X = X_.copy()
+    assert sort_graph_by_row_values(X).data is X.data
+
+    X = X_.copy()
+    assert sort_graph_by_row_values(X, copy=False).data is X.data
+
+    X = X_.copy()
+    assert sort_graph_by_row_values(X, copy=True).data is not X.data
+
+    # _check_precomputed is never done inplace
+    X = X_.copy()
+    assert _check_precomputed(X).data is not X.data
+
+    # do not raise if X is not CSR and copy=True
+    sort_graph_by_row_values(X.tocsc(), copy=True)
+
+    # raise if X is not CSR and copy=False
+    with pytest.raises(ValueError, match="Use copy=True to allow the conversion"):
+        sort_graph_by_row_values(X.tocsc(), copy=False)
+
+
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sort_graph_by_row_values_warning(csr_container):
+    # Test that the parameter warn_when_not_sorted works as expected.
+    X = csr_container(np.abs(np.random.RandomState(42).randn(10, 10)))
+    assert not _is_sorted_by_data(X)
+
+    # warning
+    with pytest.warns(EfficiencyWarning, match="was not sorted by row values"):
+        sort_graph_by_row_values(X, copy=True)
+    with pytest.warns(EfficiencyWarning, match="was not sorted by row values"):
+        sort_graph_by_row_values(X, copy=True, warn_when_not_sorted=True)
+    with pytest.warns(EfficiencyWarning, match="was not sorted by row values"):
+        _check_precomputed(X)
+
+    # no warning
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        sort_graph_by_row_values(X, copy=True, warn_when_not_sorted=False)
+
+
+@pytest.mark.parametrize(
+    "sparse_container", DOK_CONTAINERS + BSR_CONTAINERS + DIA_CONTAINERS
+)
+def test_sort_graph_by_row_values_bad_sparse_format(sparse_container):
+    # Test that sort_graph_by_row_values and _check_precomputed error on bad formats
+    X = sparse_container(np.abs(np.random.RandomState(42).randn(10, 10)))
+    with pytest.raises(TypeError, match="format is not supported"):
+        sort_graph_by_row_values(X)
+    with pytest.raises(TypeError, match="format is not supported"):
+        _check_precomputed(X)
+
+
+@pytest.mark.filterwarnings("ignore:EfficiencyWarning")
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_precomputed_sparse_invalid(csr_container):
+    dist = np.array([[0.0, 2.0, 1.0], [2.0, 0.0, 3.0], [1.0, 3.0, 0.0]])
+    dist_csr = csr_container(dist)
     neigh = neighbors.NearestNeighbors(n_neighbors=1, metric="precomputed")
     neigh.fit(dist_csr)
     neigh.kneighbors(None, n_neighbors=1)
-    neigh.kneighbors(np.array([[0., 0., 0.]]), n_neighbors=2)
+    neigh.kneighbors(np.array([[0.0, 0.0, 0.0]]), n_neighbors=2)
 
     # Ensures enough number of nearest neighbors
-    dist = np.array([[0., 2., 0.], [2., 0., 3.], [0., 3., 0.]])
-    dist_csr = csr_matrix(dist)
+    dist = np.array([[0.0, 2.0, 0.0], [2.0, 0.0, 3.0], [0.0, 3.0, 0.0]])
+    dist_csr = csr_container(dist)
     neigh.fit(dist_csr)
     msg = "2 neighbors per samples are required, but some samples have only 1"
     with pytest.raises(ValueError, match=msg):
         neigh.kneighbors(None, n_neighbors=1)
 
     # Checks error with inconsistent distance matrix
-    dist = np.array([[5., 2., 1.], [-2., 0., 3.], [1., 3., 0.]])
-    dist_csr = csr_matrix(dist)
+    dist = np.array([[5.0, 2.0, 1.0], [-2.0, 0.0, 3.0], [1.0, 3.0, 0.0]])
+    dist_csr = csr_container(dist)
     msg = "Negative values in data passed to precomputed distance matrix."
     with pytest.raises(ValueError, match=msg):
         neigh.kneighbors(dist_csr, n_neighbors=1)
@@ -310,34 +608,34 @@ def test_precomputed_cross_validation():
     # Ensure array is split correctly
     rng = np.random.RandomState(0)
     X = rng.rand(20, 2)
-    D = pairwise_distances(X, metric='euclidean')
+    D = pairwise_distances(X, metric="euclidean")
     y = rng.randint(3, size=20)
-    for Est in (neighbors.KNeighborsClassifier,
-                neighbors.RadiusNeighborsClassifier,
-                neighbors.KNeighborsRegressor,
-                neighbors.RadiusNeighborsRegressor):
+    for Est in (
+        neighbors.KNeighborsClassifier,
+        neighbors.RadiusNeighborsClassifier,
+        neighbors.KNeighborsRegressor,
+        neighbors.RadiusNeighborsRegressor,
+    ):
         metric_score = cross_val_score(Est(), X, y)
-        precomp_score = cross_val_score(Est(metric='precomputed'), D, y)
+        precomp_score = cross_val_score(Est(metric="precomputed"), D, y)
         assert_array_equal(metric_score, precomp_score)
 
 
-def test_unsupervised_radius_neighbors(n_samples=20, n_features=5,
-                                       n_query_pts=2, radius=0.5,
-                                       random_state=0):
+def test_unsupervised_radius_neighbors(
+    global_dtype, n_samples=20, n_features=5, n_query_pts=2, radius=0.5, random_state=0
+):
     # Test unsupervised radius-based query
     rng = np.random.RandomState(random_state)
 
-    X = rng.rand(n_samples, n_features)
+    X = rng.rand(n_samples, n_features).astype(global_dtype, copy=False)
 
-    test = rng.rand(n_query_pts, n_features)
+    test = rng.rand(n_query_pts, n_features).astype(global_dtype, copy=False)
 
     for p in P:
         results = []
 
         for algorithm in ALGORITHMS:
-            neigh = neighbors.NearestNeighbors(radius=radius,
-                                               algorithm=algorithm,
-                                               p=p)
+            neigh = neighbors.NearestNeighbors(radius=radius, algorithm=algorithm, p=p)
             neigh.fit(X)
 
             ind1 = neigh.radius_neighbors(test, return_distance=False)
@@ -345,58 +643,69 @@ def test_unsupervised_radius_neighbors(n_samples=20, n_features=5,
             # sort the results: this is not done automatically for
             # radius searches
             dist, ind = neigh.radius_neighbors(test, return_distance=True)
-            for (d, i, i1) in zip(dist, ind, ind1):
+            for d, i, i1 in zip(dist, ind, ind1):
                 j = d.argsort()
                 d[:] = d[j]
                 i[:] = i[j]
                 i1[:] = i1[j]
             results.append((dist, ind))
 
-            assert_array_almost_equal(np.concatenate(list(ind)),
-                                      np.concatenate(list(ind1)))
+            assert_allclose(np.concatenate(list(ind)), np.concatenate(list(ind1)))
 
         for i in range(len(results) - 1):
-            assert_array_almost_equal(np.concatenate(list(results[i][0])),
-                                      np.concatenate(list(results[i + 1][0]))),
-            assert_array_almost_equal(np.concatenate(list(results[i][1])),
-                                      np.concatenate(list(results[i + 1][1])))
-
-
-def test_kneighbors_classifier(n_samples=40,
-                               n_features=5,
-                               n_test_pts=10,
-                               n_neighbors=5,
-                               random_state=0):
+            assert_allclose(
+                np.concatenate(list(results[i][0])),
+                np.concatenate(list(results[i + 1][0])),
+            )
+            assert_allclose(
+                np.concatenate(list(results[i][1])),
+                np.concatenate(list(results[i + 1][1])),
+            )
+
+
+@pytest.mark.parametrize("algorithm", ALGORITHMS)
+@pytest.mark.parametrize("weights", WEIGHTS)
+def test_kneighbors_classifier(
+    global_dtype,
+    algorithm,
+    weights,
+    n_samples=40,
+    n_features=5,
+    n_test_pts=10,
+    n_neighbors=5,
+    random_state=0,
+):
     # Test k-neighbors classification
     rng = np.random.RandomState(random_state)
-    X = 2 * rng.rand(n_samples, n_features) - 1
-    y = ((X ** 2).sum(axis=1) < .5).astype(int)
+    X = 2 * rng.rand(n_samples, n_features).astype(global_dtype, copy=False) - 1
+    y = ((X**2).sum(axis=1) < 0.5).astype(int)
     y_str = y.astype(str)
 
-    weight_func = _weight_func
-
-    for algorithm in ALGORITHMS:
-        for weights in ['uniform', 'distance', weight_func]:
-            knn = neighbors.KNeighborsClassifier(n_neighbors=n_neighbors,
-                                                 weights=weights,
-                                                 algorithm=algorithm)
-            knn.fit(X, y)
-            epsilon = 1e-5 * (2 * rng.rand(1, n_features) - 1)
-            y_pred = knn.predict(X[:n_test_pts] + epsilon)
-            assert_array_equal(y_pred, y[:n_test_pts])
-            # Test prediction with y_str
-            knn.fit(X, y_str)
-            y_pred = knn.predict(X[:n_test_pts] + epsilon)
-            assert_array_equal(y_pred, y_str[:n_test_pts])
+    knn = neighbors.KNeighborsClassifier(
+        n_neighbors=n_neighbors, weights=weights, algorithm=algorithm
+    )
+    knn.fit(X, y)
+    epsilon = 1e-5 * (2 * rng.rand(1, n_features) - 1)
+    y_pred = knn.predict(X[:n_test_pts] + epsilon)
+    assert_array_equal(y_pred, y[:n_test_pts])
+    # Test prediction with y_str
+    knn.fit(X, y_str)
+    y_pred = knn.predict(X[:n_test_pts] + epsilon)
+    assert_array_equal(y_pred, y_str[:n_test_pts])
 
 
-def test_kneighbors_classifier_float_labels(n_samples=40, n_features=5,
-                                            n_test_pts=10, n_neighbors=5,
-                                            random_state=0):
+def test_kneighbors_classifier_float_labels(
+    global_dtype,
+    n_samples=40,
+    n_features=5,
+    n_test_pts=10,
+    n_neighbors=5,
+    random_state=0,
+):
     # Test k-neighbors classification
     rng = np.random.RandomState(random_state)
-    X = 2 * rng.rand(n_samples, n_features) - 1
-    y = ((X ** 2).sum(axis=1) < .5).astype(int)
+    X = 2 * rng.rand(n_samples, n_features).astype(global_dtype, copy=False) - 1
+    y = ((X**2).sum(axis=1) < 0.5).astype(int)
 
     knn = neighbors.KNeighborsClassifier(n_neighbors=n_neighbors)
     knn.fit(X, y.astype(float))
@@ -405,141 +714,160 @@ def test_kneighbors_classifier_float_labels(n_samples=40, n_features=5,
     assert_array_equal(y_pred, y[:n_test_pts])
 
 
-def test_kneighbors_classifier_predict_proba():
+def test_kneighbors_classifier_predict_proba(global_dtype):
     # Test KNeighborsClassifier.predict_proba() method
-    X = np.array([[0, 2, 0],
-                  [0, 2, 1],
-                  [2, 0, 0],
-                  [2, 2, 0],
-                  [0, 0, 2],
-                  [0, 0, 1]])
+    X = np.array(
+        [[0, 2, 0], [0, 2, 1], [2, 0, 0], [2, 2, 0], [0, 0, 2], [0, 0, 1]]
+    ).astype(global_dtype, copy=False)
     y = np.array([4, 4, 5, 5, 1, 1])
     cls = neighbors.KNeighborsClassifier(n_neighbors=3, p=1)  # cityblock dist
     cls.fit(X, y)
     y_prob = cls.predict_proba(X)
-    real_prob = np.array([[0, 2. / 3, 1. / 3],
-                          [1. / 3, 2. / 3, 0],
-                          [1. / 3, 0, 2. / 3],
-                          [0, 1. / 3, 2. / 3],
-                          [2. / 3, 1. / 3, 0],
-                          [2. / 3, 1. / 3, 0]])
+    real_prob = (
+        np.array(
+            [
+                [0, 2, 1],
+                [1, 2, 0],
+                [1, 0, 2],
+                [0, 1, 2],
+                [2, 1, 0],
+                [2, 1, 0],
+            ]
+        )
+        / 3.0
+    )
     assert_array_equal(real_prob, y_prob)
     # Check that it also works with non integer labels
     cls.fit(X, y.astype(str))
     y_prob = cls.predict_proba(X)
     assert_array_equal(real_prob, y_prob)
     # Check that it works with weights='distance'
-    cls = neighbors.KNeighborsClassifier(
-        n_neighbors=2, p=1, weights='distance')
+    cls = neighbors.KNeighborsClassifier(n_neighbors=2, p=1, weights="distance")
     cls.fit(X, y)
     y_prob = cls.predict_proba(np.array([[0, 2, 0], [2, 2, 2]]))
     real_prob = np.array([[0, 1, 0], [0, 0.4, 0.6]])
-    assert_array_almost_equal(real_prob, y_prob)
-
-
-def test_radius_neighbors_classifier(n_samples=40,
-                                     n_features=5,
-                                     n_test_pts=10,
-                                     radius=0.5,
-                                     random_state=0):
+    assert_allclose(real_prob, y_prob)
+
+
+@pytest.mark.parametrize("algorithm", ALGORITHMS)
+@pytest.mark.parametrize("weights", WEIGHTS)
+def test_radius_neighbors_classifier(
+    global_dtype,
+    algorithm,
+    weights,
+    n_samples=40,
+    n_features=5,
+    n_test_pts=10,
+    radius=0.5,
+    random_state=0,
+):
     # Test radius-based classification
     rng = np.random.RandomState(random_state)
-    X = 2 * rng.rand(n_samples, n_features) - 1
-    y = ((X ** 2).sum(axis=1) < .5).astype(int)
+    X = 2 * rng.rand(n_samples, n_features).astype(global_dtype, copy=False) - 1
+    y = ((X**2).sum(axis=1) < radius).astype(int)
     y_str = y.astype(str)
 
-    weight_func = _weight_func
-
-    for algorithm in ALGORITHMS:
-        for weights in ['uniform', 'distance', weight_func]:
-            neigh = neighbors.RadiusNeighborsClassifier(radius=radius,
-                                                        weights=weights,
-                                                        algorithm=algorithm)
-            neigh.fit(X, y)
-            epsilon = 1e-5 * (2 * rng.rand(1, n_features) - 1)
-            y_pred = neigh.predict(X[:n_test_pts] + epsilon)
-            assert_array_equal(y_pred, y[:n_test_pts])
-            neigh.fit(X, y_str)
-            y_pred = neigh.predict(X[:n_test_pts] + epsilon)
-            assert_array_equal(y_pred, y_str[:n_test_pts])
+    neigh = neighbors.RadiusNeighborsClassifier(
+        radius=radius, weights=weights, algorithm=algorithm
+    )
+    neigh.fit(X, y)
+    epsilon = 1e-5 * (2 * rng.rand(1, n_features) - 1)
+    y_pred = neigh.predict(X[:n_test_pts] + epsilon)
+    assert_array_equal(y_pred, y[:n_test_pts])
+    neigh.fit(X, y_str)
+    y_pred = neigh.predict(X[:n_test_pts] + epsilon)
+    assert_array_equal(y_pred, y_str[:n_test_pts])
 
 
-def test_radius_neighbors_classifier_when_no_neighbors():
+@pytest.mark.parametrize("algorithm", ALGORITHMS)
+@pytest.mark.parametrize("weights", WEIGHTS)
+@pytest.mark.parametrize("outlier_label", [0, -1, None])
+def test_radius_neighbors_classifier_when_no_neighbors(
+    global_dtype, algorithm, weights, outlier_label
+):
     # Test radius-based classifier when no neighbors found.
     # In this case it should rise an informative exception
 
-    X = np.array([[1.0, 1.0], [2.0, 2.0]])
+    X = np.array([[1.0, 1.0], [2.0, 2.0]], dtype=global_dtype)
     y = np.array([1, 2])
     radius = 0.1
 
-    z1 = np.array([[1.01, 1.01], [2.01, 2.01]])  # no outliers
-    z2 = np.array([[1.01, 1.01], [1.4, 1.4]])    # one outlier
+    # no outliers
+    z1 = np.array([[1.01, 1.01], [2.01, 2.01]], dtype=global_dtype)
 
-    weight_func = _weight_func
+    # one outlier
+    z2 = np.array([[1.01, 1.01], [1.4, 1.4]], dtype=global_dtype)
 
-    for outlier_label in [0, -1, None]:
-        for algorithm in ALGORITHMS:
-            for weights in ['uniform', 'distance', weight_func]:
-                rnc = neighbors.RadiusNeighborsClassifier
-                clf = rnc(radius=radius, weights=weights, algorithm=algorithm,
-                          outlier_label=outlier_label)
-                clf.fit(X, y)
-                assert_array_equal(np.array([1, 2]),
-                                   clf.predict(z1))
-                if outlier_label is None:
-                    with pytest.raises(ValueError):
-                        clf.predict(z2)
-
-
-def test_radius_neighbors_classifier_outlier_labeling():
+    rnc = neighbors.RadiusNeighborsClassifier
+    clf = rnc(
+        radius=radius,
+        weights=weights,
+        algorithm=algorithm,
+        outlier_label=outlier_label,
+    )
+    clf.fit(X, y)
+    assert_array_equal(np.array([1, 2]), clf.predict(z1))
+    if outlier_label is None:
+        with pytest.raises(ValueError):
+            clf.predict(z2)
+
+
+@pytest.mark.parametrize("algorithm", ALGORITHMS)
+@pytest.mark.parametrize("weights", WEIGHTS)
+def test_radius_neighbors_classifier_outlier_labeling(global_dtype, algorithm, weights):
     # Test radius-based classifier when no neighbors found and outliers
     # are labeled.
 
-    X = np.array([[1.0, 1.0], [2.0, 2.0], [0.99, 0.99],
-                  [0.98, 0.98], [2.01, 2.01]])
+    X = np.array(
+        [[1.0, 1.0], [2.0, 2.0], [0.99, 0.99], [0.98, 0.98], [2.01, 2.01]],
+        dtype=global_dtype,
+    )
     y = np.array([1, 2, 1, 1, 2])
     radius = 0.1
 
-    z1 = np.array([[1.01, 1.01], [2.01, 2.01]])  # no outliers
-    z2 = np.array([[1.4, 1.4], [1.01, 1.01], [2.01, 2.01]])    # one outlier
+    # no outliers
+    z1 = np.array([[1.01, 1.01], [2.01, 2.01]], dtype=global_dtype)
+
+    # one outlier
+    z2 = np.array([[1.4, 1.4], [1.01, 1.01], [2.01, 2.01]], dtype=global_dtype)
+
     correct_labels1 = np.array([1, 2])
     correct_labels2 = np.array([-1, 1, 2])
     outlier_proba = np.array([0, 0])
 
-    weight_func = _weight_func
-
-    for algorithm in ALGORITHMS:
-        for weights in ['uniform', 'distance', weight_func]:
-            clf = neighbors.RadiusNeighborsClassifier(radius=radius,
-                                                      weights=weights,
-                                                      algorithm=algorithm,
-                                                      outlier_label=-1)
-            clf.fit(X, y)
-            assert_array_equal(correct_labels1, clf.predict(z1))
-            assert_array_equal(correct_labels2, clf.predict(z2))
-            assert_array_equal(outlier_proba, clf.predict_proba(z2)[0])
+    clf = neighbors.RadiusNeighborsClassifier(
+        radius=radius, weights=weights, algorithm=algorithm, outlier_label=-1
+    )
+    clf.fit(X, y)
+    assert_array_equal(correct_labels1, clf.predict(z1))
+    with pytest.warns(UserWarning, match="Outlier label -1 is not in training classes"):
+        assert_array_equal(correct_labels2, clf.predict(z2))
+    with pytest.warns(UserWarning, match="Outlier label -1 is not in training classes"):
+        assert_allclose(outlier_proba, clf.predict_proba(z2)[0])
 
     # test outlier_labeling of using predict_proba()
     RNC = neighbors.RadiusNeighborsClassifier
-    X = np.array([[0], [1], [2], [3], [4], [5], [6], [7], [8], [9]])
+    X = np.array([[0], [1], [2], [3], [4], [5], [6], [7], [8], [9]], dtype=global_dtype)
     y = np.array([0, 2, 2, 1, 1, 1, 3, 3, 3, 3])
 
     # test outlier_label scalar verification
     def check_array_exception():
         clf = RNC(radius=1, outlier_label=[[5]])
         clf.fit(X, y)
+
     with pytest.raises(TypeError):
         check_array_exception()
 
     # test invalid outlier_label dtype
     def check_dtype_exception():
-        clf = RNC(radius=1, outlier_label='a')
+        clf = RNC(radius=1, outlier_label="a")
         clf.fit(X, y)
+
     with pytest.raises(TypeError):
         check_dtype_exception()
 
     # test most frequent
-    clf = RNC(radius=1, outlier_label='most_frequent')
+    clf = RNC(radius=1, outlier_label="most_frequent")
     clf.fit(X, y)
     proba = clf.predict_proba([[1], [15]])
     assert_array_equal(proba[1, :], [0, 0, 0, 1])
@@ -557,12 +885,23 @@ def check_warning():
         clf = RNC(radius=1, outlier_label=4)
         clf.fit(X, y)
         clf.predict_proba([[1], [15]])
+
     with pytest.warns(UserWarning):
         check_warning()
 
     # test multi output same outlier label
-    y_multi = [[0, 1], [2, 1], [2, 2], [1, 2], [1, 2],
-               [1, 3], [3, 3], [3, 3], [3, 0], [3, 0]]
+    y_multi = [
+        [0, 1],
+        [2, 1],
+        [2, 2],
+        [1, 2],
+        [1, 2],
+        [1, 3],
+        [3, 3],
+        [3, 3],
+        [3, 0],
+        [3, 0],
+    ]
     clf = RNC(radius=1, outlier_label=1)
     clf.fit(X, y_multi)
     proba = clf.predict_proba([[7], [15]])
@@ -571,8 +910,18 @@ def check_warning():
     assert_array_equal(pred[1, :], [1, 1])
 
     # test multi output different outlier label
-    y_multi = [[0, 0], [2, 2], [2, 2], [1, 1], [1, 1],
-               [1, 1], [3, 3], [3, 3], [3, 3], [3, 3]]
+    y_multi = [
+        [0, 0],
+        [2, 2],
+        [2, 2],
+        [1, 1],
+        [1, 1],
+        [1, 1],
+        [3, 3],
+        [3, 3],
+        [3, 3],
+        [3, 3],
+    ]
     clf = RNC(radius=1, outlier_label=[0, 1])
     clf.fit(X, y_multi)
     proba = clf.predict_proba([[7], [15]])
@@ -585,6 +934,7 @@ def check_warning():
     def check_exception():
         clf = RNC(radius=1, outlier_label=[0, 1, 2])
         clf.fit(X, y_multi)
+
     with pytest.raises(ValueError):
         check_exception()
 
@@ -602,10 +952,10 @@ def test_radius_neighbors_classifier_zero_distance():
     weight_func = _weight_func
 
     for algorithm in ALGORITHMS:
-        for weights in ['uniform', 'distance', weight_func]:
-            clf = neighbors.RadiusNeighborsClassifier(radius=radius,
-                                                      weights=weights,
-                                                      algorithm=algorithm)
+        for weights in ["uniform", "distance", weight_func]:
+            clf = neighbors.RadiusNeighborsClassifier(
+                radius=radius, weights=weights, algorithm=algorithm
+            )
             clf.fit(X, y)
             with np.errstate(invalid="ignore"):
                 # Ignore the warning raised in _weight_func when making
@@ -629,20 +979,21 @@ def test_neighbors_regressors_zero_distance():
     for algorithm in ALGORITHMS:
         # we don't test for weights=_weight_func since user will be expected
         # to handle zero distances themselves in the function.
-        for weights in ['uniform', 'distance']:
-            rnn = neighbors.RadiusNeighborsRegressor(radius=radius,
-                                                     weights=weights,
-                                                     algorithm=algorithm)
+        for weights in ["uniform", "distance"]:
+            rnn = neighbors.RadiusNeighborsRegressor(
+                radius=radius, weights=weights, algorithm=algorithm
+            )
             rnn.fit(X, y)
-            assert_array_almost_equal(rnn_correct_labels, rnn.predict(z))
-
-        for weights, corr_labels in zip(['uniform', 'distance'],
-                                        [knn_correct_unif, knn_correct_dist]):
-            knn = neighbors.KNeighborsRegressor(n_neighbors=2,
-                                                weights=weights,
-                                                algorithm=algorithm)
+            assert_allclose(rnn_correct_labels, rnn.predict(z))
+
+        for weights, corr_labels in zip(
+            ["uniform", "distance"], [knn_correct_unif, knn_correct_dist]
+        ):
+            knn = neighbors.KNeighborsRegressor(
+                n_neighbors=2, weights=weights, algorithm=algorithm
+            )
             knn.fit(X, y)
-            assert_array_almost_equal(corr_labels, knn.predict(z))
+            assert_allclose(corr_labels, knn.predict(z))
 
 
 def test_radius_neighbors_boundary_handling():
@@ -656,44 +1007,73 @@ def test_radius_neighbors_boundary_handling():
     radius = 3.0
 
     for algorithm in ALGORITHMS:
-        nbrs = neighbors.NearestNeighbors(radius=radius,
-                                          algorithm=algorithm).fit(X)
+        nbrs = neighbors.NearestNeighbors(radius=radius, algorithm=algorithm).fit(X)
         results = nbrs.radius_neighbors([[0.0]], return_distance=False)
         assert results.shape == (1,)
         assert results.dtype == object
         assert_array_equal(results[0], [0, 1])
 
 
-def test_radius_neighbors_returns_array_of_objects():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_radius_neighbors_returns_array_of_objects(csr_container):
     # check that we can pass precomputed distances to
     # NearestNeighbors.radius_neighbors()
     # non-regression test for
     # https://github.com/scikit-learn/scikit-learn/issues/16036
-    X = csr_matrix(np.ones((4, 4)))
+    X = csr_container(np.ones((4, 4)))
     X.setdiag([0, 0, 0, 0])
 
-    nbrs = neighbors.NearestNeighbors(radius=0.5, algorithm='auto',
-                                      leaf_size=30,
-                                      metric='precomputed').fit(X)
+    nbrs = neighbors.NearestNeighbors(
+        radius=0.5, algorithm="auto", leaf_size=30, metric="precomputed"
+    ).fit(X)
     neigh_dist, neigh_ind = nbrs.radius_neighbors(X, return_distance=True)
 
     expected_dist = np.empty(X.shape[0], dtype=object)
-    expected_dist[:] = [np.array([0]), np.array([0]), np.array([0]),
-                        np.array([0])]
+    expected_dist[:] = [np.array([0]), np.array([0]), np.array([0]), np.array([0])]
     expected_ind = np.empty(X.shape[0], dtype=object)
-    expected_ind[:] = [np.array([0]), np.array([1]), np.array([2]),
-                       np.array([3])]
+    expected_ind[:] = [np.array([0]), np.array([1]), np.array([2]), np.array([3])]
 
     assert_array_equal(neigh_dist, expected_dist)
     assert_array_equal(neigh_ind, expected_ind)
 
 
-@pytest.mark.parametrize(["algorithm", "metric"], [("ball_tree", "euclidean"),
-                                                   ("kd_tree", "euclidean"),
-                                                   ("brute", "euclidean"),
-                                                   ("brute", "precomputed")])
+@pytest.mark.parametrize("algorithm", ["ball_tree", "kd_tree", "brute"])
+def test_query_equidistant_kth_nn(algorithm):
+    # For several candidates for the k-th nearest neighbor position,
+    # the first candidate should be chosen
+    query_point = np.array([[0, 0]])
+    equidistant_points = np.array([[1, 0], [0, 1], [-1, 0], [0, -1]])
+    # The 3rd and 4th points should not replace the 2nd point
+    # for the 2th nearest neighbor position
+    k = 2
+    knn_indices = np.array([[0, 1]])
+    nn = neighbors.NearestNeighbors(algorithm=algorithm).fit(equidistant_points)
+    indices = np.sort(nn.kneighbors(query_point, n_neighbors=k, return_distance=False))
+    assert_array_equal(indices, knn_indices)
+
+
+@pytest.mark.parametrize(
+    ["algorithm", "metric"],
+    list(
+        product(
+            ("kd_tree", "ball_tree", "brute"),
+            ("euclidean", *DISTANCE_METRIC_OBJS),
+        )
+    )
+    + [
+        ("brute", "euclidean"),
+        ("brute", "precomputed"),
+    ],
+)
 def test_radius_neighbors_sort_results(algorithm, metric):
     # Test radius_neighbors[_graph] output when sort_result is True
+
+    metric = _parse_metric(metric, np.float64)
+    if isinstance(metric, DistanceMetric):
+        pytest.skip(
+            "Metrics of type `DistanceMetric` are not yet supported for radius-neighbor"
+            " estimators."
+        )
     n_samples = 10
     rng = np.random.RandomState(42)
     X = rng.random_sample((n_samples, 4))
@@ -704,20 +1084,21 @@ def test_radius_neighbors_sort_results(algorithm, metric):
     model.fit(X)
 
     # self.radius_neighbors
-    distances, indices = model.radius_neighbors(X=X, radius=np.inf,
-                                                sort_results=True)
+    distances, indices = model.radius_neighbors(X=X, radius=np.inf, sort_results=True)
     for ii in range(n_samples):
         assert_array_equal(distances[ii], np.sort(distances[ii]))
 
     # sort_results=True and return_distance=False
     if metric != "precomputed":  # no need to raise with precomputed graph
         with pytest.raises(ValueError, match="return_distance must be True"):
-            model.radius_neighbors(X=X, radius=np.inf, sort_results=True,
-                                   return_distance=False)
+            model.radius_neighbors(
+                X=X, radius=np.inf, sort_results=True, return_distance=False
+            )
 
     # self.radius_neighbors_graph
-    graph = model.radius_neighbors_graph(X=X, radius=np.inf, mode="distance",
-                                         sort_results=True)
+    graph = model.radius_neighbors_graph(
+        X=X, radius=np.inf, mode="distance", sort_results=True
+    )
     assert _is_sorted_by_data(graph)
 
 
@@ -733,14 +1114,15 @@ def test_RadiusNeighborsClassifier_multioutput():
 
     X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
 
-    weights = [None, 'uniform', 'distance', _weight_func]
+    weights = [None, "uniform", "distance", _weight_func]
 
     for algorithm, weights in product(ALGORITHMS, weights):
         # Stack single output prediction
         y_pred_so = []
         for o in range(n_output):
-            rnn = neighbors.RadiusNeighborsClassifier(weights=weights,
-                                                      algorithm=algorithm)
+            rnn = neighbors.RadiusNeighborsClassifier(
+                weights=weights, algorithm=algorithm
+            )
             rnn.fit(X_train, y_train[:, o])
             y_pred_so.append(rnn.predict(X_test))
 
@@ -748,30 +1130,28 @@ def test_RadiusNeighborsClassifier_multioutput():
         assert y_pred_so.shape == y_test.shape
 
         # Multioutput prediction
-        rnn_mo = neighbors.RadiusNeighborsClassifier(weights=weights,
-                                                     algorithm=algorithm)
+        rnn_mo = neighbors.RadiusNeighborsClassifier(
+            weights=weights, algorithm=algorithm
+        )
         rnn_mo.fit(X_train, y_train)
         y_pred_mo = rnn_mo.predict(X_test)
 
         assert y_pred_mo.shape == y_test.shape
-        assert_array_almost_equal(y_pred_mo, y_pred_so)
+        assert_array_equal(y_pred_mo, y_pred_so)
 
 
-def test_kneighbors_classifier_sparse(n_samples=40,
-                                      n_features=5,
-                                      n_test_pts=10,
-                                      n_neighbors=5,
-                                      random_state=0):
+def test_kneighbors_classifier_sparse(
+    n_samples=40, n_features=5, n_test_pts=10, n_neighbors=5, random_state=0
+):
     # Test k-NN classifier on sparse matrices
     # Like the above, but with various types of sparse matrices
     rng = np.random.RandomState(random_state)
     X = 2 * rng.rand(n_samples, n_features) - 1
-    X *= X > .2
-    y = ((X ** 2).sum(axis=1) < .5).astype(int)
+    X *= X > 0.2
+    y = ((X**2).sum(axis=1) < 0.5).astype(int)
 
     for sparsemat in SPARSE_TYPES:
-        knn = neighbors.KNeighborsClassifier(n_neighbors=n_neighbors,
-                                             algorithm='auto')
+        knn = neighbors.KNeighborsClassifier(n_neighbors=n_neighbors, algorithm="auto")
         knn.fit(sparsemat(X), y)
         epsilon = 1e-5 * (2 * rng.rand(1, n_features) - 1)
         for sparsev in SPARSE_TYPES + (np.asarray,):
@@ -792,15 +1172,14 @@ def test_KNeighborsClassifier_multioutput():
 
     X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
 
-    weights = [None, 'uniform', 'distance', _weight_func]
+    weights = [None, "uniform", "distance", _weight_func]
 
     for algorithm, weights in product(ALGORITHMS, weights):
         # Stack single output prediction
         y_pred_so = []
         y_pred_proba_so = []
         for o in range(n_output):
-            knn = neighbors.KNeighborsClassifier(weights=weights,
-                                                 algorithm=algorithm)
+            knn = neighbors.KNeighborsClassifier(weights=weights, algorithm=algorithm)
             knn.fit(X_train, y_train[:, o])
             y_pred_so.append(knn.predict(X_test))
             y_pred_proba_so.append(knn.predict_proba(X_test))
@@ -810,31 +1189,28 @@ def test_KNeighborsClassifier_multioutput():
         assert len(y_pred_proba_so) == n_output
 
         # Multioutput prediction
-        knn_mo = neighbors.KNeighborsClassifier(weights=weights,
-                                                algorithm=algorithm)
+        knn_mo = neighbors.KNeighborsClassifier(weights=weights, algorithm=algorithm)
         knn_mo.fit(X_train, y_train)
         y_pred_mo = knn_mo.predict(X_test)
 
         assert y_pred_mo.shape == y_test.shape
-        assert_array_almost_equal(y_pred_mo, y_pred_so)
+        assert_array_equal(y_pred_mo, y_pred_so)
 
         # Check proba
         y_pred_proba_mo = knn_mo.predict_proba(X_test)
         assert len(y_pred_proba_mo) == n_output
 
         for proba_mo, proba_so in zip(y_pred_proba_mo, y_pred_proba_so):
-            assert_array_almost_equal(proba_mo, proba_so)
+            assert_array_equal(proba_mo, proba_so)
 
 
-def test_kneighbors_regressor(n_samples=40,
-                              n_features=5,
-                              n_test_pts=10,
-                              n_neighbors=3,
-                              random_state=0):
+def test_kneighbors_regressor(
+    n_samples=40, n_features=5, n_test_pts=10, n_neighbors=3, random_state=0
+):
     # Test k-neighbors regression
     rng = np.random.RandomState(random_state)
     X = 2 * rng.rand(n_samples, n_features) - 1
-    y = np.sqrt((X ** 2).sum(1))
+    y = np.sqrt((X**2).sum(1))
     y /= y.max()
 
     y_target = y[:n_test_pts]
@@ -842,12 +1218,12 @@ def test_kneighbors_regressor(n_samples=40,
     weight_func = _weight_func
 
     for algorithm in ALGORITHMS:
-        for weights in ['uniform', 'distance', weight_func]:
-            knn = neighbors.KNeighborsRegressor(n_neighbors=n_neighbors,
-                                                weights=weights,
-                                                algorithm=algorithm)
+        for weights in ["uniform", "distance", weight_func]:
+            knn = neighbors.KNeighborsRegressor(
+                n_neighbors=n_neighbors, weights=weights, algorithm=algorithm
+            )
             knn.fit(X, y)
-            epsilon = 1E-5 * (2 * rng.rand(1, n_features) - 1)
+            epsilon = 1e-5 * (2 * rng.rand(1, n_features) - 1)
             y_pred = knn.predict(X[:n_test_pts] + epsilon)
             assert np.all(abs(y_pred - y_target) < 0.3)
 
@@ -863,58 +1239,52 @@ def test_KNeighborsRegressor_multioutput_uniform_weight():
     y = rng.rand(n_samples, n_output)
 
     X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
-    for algorithm, weights in product(ALGORITHMS, [None, 'uniform']):
-        knn = neighbors.KNeighborsRegressor(weights=weights,
-                                            algorithm=algorithm)
+    for algorithm, weights in product(ALGORITHMS, [None, "uniform"]):
+        knn = neighbors.KNeighborsRegressor(weights=weights, algorithm=algorithm)
         knn.fit(X_train, y_train)
 
         neigh_idx = knn.kneighbors(X_test, return_distance=False)
-        y_pred_idx = np.array([np.mean(y_train[idx], axis=0)
-                               for idx in neigh_idx])
+        y_pred_idx = np.array([np.mean(y_train[idx], axis=0) for idx in neigh_idx])
 
         y_pred = knn.predict(X_test)
 
         assert y_pred.shape == y_test.shape
         assert y_pred_idx.shape == y_test.shape
-        assert_array_almost_equal(y_pred, y_pred_idx)
+        assert_allclose(y_pred, y_pred_idx)
 
 
-def test_kneighbors_regressor_multioutput(n_samples=40,
-                                          n_features=5,
-                                          n_test_pts=10,
-                                          n_neighbors=3,
-                                          random_state=0):
+def test_kneighbors_regressor_multioutput(
+    n_samples=40, n_features=5, n_test_pts=10, n_neighbors=3, random_state=0
+):
     # Test k-neighbors in multi-output regression
     rng = np.random.RandomState(random_state)
     X = 2 * rng.rand(n_samples, n_features) - 1
-    y = np.sqrt((X ** 2).sum(1))
+    y = np.sqrt((X**2).sum(1))
     y /= y.max()
     y = np.vstack([y, y]).T
 
     y_target = y[:n_test_pts]
 
-    weights = ['uniform', 'distance', _weight_func]
+    weights = ["uniform", "distance", _weight_func]
     for algorithm, weights in product(ALGORITHMS, weights):
-        knn = neighbors.KNeighborsRegressor(n_neighbors=n_neighbors,
-                                            weights=weights,
-                                            algorithm=algorithm)
+        knn = neighbors.KNeighborsRegressor(
+            n_neighbors=n_neighbors, weights=weights, algorithm=algorithm
+        )
         knn.fit(X, y)
-        epsilon = 1E-5 * (2 * rng.rand(1, n_features) - 1)
+        epsilon = 1e-5 * (2 * rng.rand(1, n_features) - 1)
         y_pred = knn.predict(X[:n_test_pts] + epsilon)
         assert y_pred.shape == y_target.shape
 
         assert np.all(np.abs(y_pred - y_target) < 0.3)
 
 
-def test_radius_neighbors_regressor(n_samples=40,
-                                    n_features=3,
-                                    n_test_pts=10,
-                                    radius=0.5,
-                                    random_state=0):
+def test_radius_neighbors_regressor(
+    n_samples=40, n_features=3, n_test_pts=10, radius=0.5, random_state=0
+):
     # Test radius-based neighbors regression
     rng = np.random.RandomState(random_state)
     X = 2 * rng.rand(n_samples, n_features) - 1
-    y = np.sqrt((X ** 2).sum(1))
+    y = np.sqrt((X**2).sum(1))
     y /= y.max()
 
     y_target = y[:n_test_pts]
@@ -922,24 +1292,26 @@ def test_radius_neighbors_regressor(n_samples=40,
     weight_func = _weight_func
 
     for algorithm in ALGORITHMS:
-        for weights in ['uniform', 'distance', weight_func]:
-            neigh = neighbors.RadiusNeighborsRegressor(radius=radius,
-                                                       weights=weights,
-                                                       algorithm=algorithm)
+        for weights in ["uniform", "distance", weight_func]:
+            neigh = neighbors.RadiusNeighborsRegressor(
+                radius=radius, weights=weights, algorithm=algorithm
+            )
             neigh.fit(X, y)
-            epsilon = 1E-5 * (2 * rng.rand(1, n_features) - 1)
+            epsilon = 1e-5 * (2 * rng.rand(1, n_features) - 1)
             y_pred = neigh.predict(X[:n_test_pts] + epsilon)
             assert np.all(abs(y_pred - y_target) < radius / 2)
 
     # test that nan is returned when no nearby observations
-    for weights in ['uniform', 'distance']:
-        neigh = neighbors.RadiusNeighborsRegressor(radius=radius,
-                                                   weights=weights,
-                                                   algorithm='auto')
+    for weights in ["uniform", "distance"]:
+        neigh = neighbors.RadiusNeighborsRegressor(
+            radius=radius, weights=weights, algorithm="auto"
+        )
         neigh.fit(X, y)
-        X_test_nan = np.full((1, n_features), -1.)
-        empty_warning_msg = ("One or more samples have no neighbors "
-                             "within specified radius; predicting NaN.")
+        X_test_nan = np.full((1, n_features), -1.0)
+        empty_warning_msg = (
+            "One or more samples have no neighbors "
+            "within specified radius; predicting NaN."
+        )
         with pytest.warns(UserWarning, match=re.escape(empty_warning_msg)):
             pred = neigh.predict(X_test_nan)
         assert np.all(np.isnan(pred))
@@ -957,78 +1329,74 @@ def test_RadiusNeighborsRegressor_multioutput_with_uniform_weight():
     y = rng.rand(n_samples, n_output)
     X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
 
-    for algorithm, weights in product(ALGORITHMS, [None, 'uniform']):
-
-        rnn = neighbors. RadiusNeighborsRegressor(weights=weights,
-                                                  algorithm=algorithm)
+    for algorithm, weights in product(ALGORITHMS, [None, "uniform"]):
+        rnn = neighbors.RadiusNeighborsRegressor(weights=weights, algorithm=algorithm)
         rnn.fit(X_train, y_train)
 
         neigh_idx = rnn.radius_neighbors(X_test, return_distance=False)
-        y_pred_idx = np.array([np.mean(y_train[idx], axis=0)
-                               for idx in neigh_idx])
+        y_pred_idx = np.array([np.mean(y_train[idx], axis=0) for idx in neigh_idx])
 
         y_pred_idx = np.array(y_pred_idx)
         y_pred = rnn.predict(X_test)
 
         assert y_pred_idx.shape == y_test.shape
         assert y_pred.shape == y_test.shape
-        assert_array_almost_equal(y_pred, y_pred_idx)
+        assert_allclose(y_pred, y_pred_idx)
 
 
-def test_RadiusNeighborsRegressor_multioutput(n_samples=40,
-                                              n_features=5,
-                                              n_test_pts=10,
-                                              n_neighbors=3,
-                                              random_state=0):
+def test_RadiusNeighborsRegressor_multioutput(
+    n_samples=40, n_features=5, n_test_pts=10, random_state=0
+):
     # Test k-neighbors in multi-output regression with various weight
     rng = np.random.RandomState(random_state)
     X = 2 * rng.rand(n_samples, n_features) - 1
-    y = np.sqrt((X ** 2).sum(1))
+    y = np.sqrt((X**2).sum(1))
     y /= y.max()
     y = np.vstack([y, y]).T
 
     y_target = y[:n_test_pts]
-    weights = ['uniform', 'distance', _weight_func]
+    weights = ["uniform", "distance", _weight_func]
 
     for algorithm, weights in product(ALGORITHMS, weights):
-        rnn = neighbors.RadiusNeighborsRegressor(n_neighbors=n_neighbors,
-                                                 weights=weights,
-                                                 algorithm=algorithm)
+        rnn = neighbors.RadiusNeighborsRegressor(weights=weights, algorithm=algorithm)
         rnn.fit(X, y)
-        epsilon = 1E-5 * (2 * rng.rand(1, n_features) - 1)
+        epsilon = 1e-5 * (2 * rng.rand(1, n_features) - 1)
         y_pred = rnn.predict(X[:n_test_pts] + epsilon)
 
         assert y_pred.shape == y_target.shape
         assert np.all(np.abs(y_pred - y_target) < 0.3)
 
 
-@ignore_warnings(category=EfficiencyWarning)
-def test_kneighbors_regressor_sparse(n_samples=40,
-                                     n_features=5,
-                                     n_test_pts=10,
-                                     n_neighbors=5,
-                                     random_state=0):
+@pytest.mark.filterwarnings("ignore:EfficiencyWarning")
+def test_kneighbors_regressor_sparse(
+    n_samples=40, n_features=5, n_test_pts=10, n_neighbors=5, random_state=0
+):
     # Test radius-based regression on sparse matrices
     # Like the above, but with various types of sparse matrices
     rng = np.random.RandomState(random_state)
     X = 2 * rng.rand(n_samples, n_features) - 1
-    y = ((X ** 2).sum(axis=1) < .25).astype(int)
+    y = ((X**2).sum(axis=1) < 0.25).astype(int)
 
     for sparsemat in SPARSE_TYPES:
-        knn = neighbors.KNeighborsRegressor(n_neighbors=n_neighbors,
-                                            algorithm='auto')
+        knn = neighbors.KNeighborsRegressor(n_neighbors=n_neighbors, algorithm="auto")
         knn.fit(sparsemat(X), y)
 
-        knn_pre = neighbors.KNeighborsRegressor(n_neighbors=n_neighbors,
-                                                metric='precomputed')
-        knn_pre.fit(pairwise_distances(X, metric='euclidean'), y)
+        knn_pre = neighbors.KNeighborsRegressor(
+            n_neighbors=n_neighbors, metric="precomputed"
+        )
+        knn_pre.fit(pairwise_distances(X, metric="euclidean"), y)
 
         for sparsev in SPARSE_OR_DENSE:
             X2 = sparsev(X)
             assert np.mean(knn.predict(X2).round() == y) > 0.95
 
-            X2_pre = sparsev(pairwise_distances(X, metric='euclidean'))
-            assert np.mean(knn_pre.predict(X2_pre).round() == y) > 0.95
+            X2_pre = sparsev(pairwise_distances(X, metric="euclidean"))
+            if sparsev in DOK_CONTAINERS + BSR_CONTAINERS:
+                msg = "not supported due to its handling of explicit zeros"
+                with pytest.raises(TypeError, match=msg):
+                    knn_pre.predict(X2_pre)
+            else:
+                assert np.mean(knn_pre.predict(X2_pre).round() == y) > 0.95
 
 
 def test_neighbors_iris():
@@ -1037,8 +1405,7 @@ def test_neighbors_iris():
     # nearest neighbor query on points near the decision boundary.
 
     for algorithm in ALGORITHMS:
-        clf = neighbors.KNeighborsClassifier(n_neighbors=1,
-                                             algorithm=algorithm)
+        clf = neighbors.KNeighborsClassifier(n_neighbors=1, algorithm=algorithm)
         clf.fit(iris.data, iris.target)
         assert_array_equal(clf.predict(iris.data), iris.target)
 
@@ -1048,7 +1415,7 @@ def test_neighbors_iris():
 
         rgs = neighbors.KNeighborsRegressor(n_neighbors=5, algorithm=algorithm)
         rgs.fit(iris.data, iris.target)
-        assert (np.mean(rgs.predict(iris.data).round() == iris.target) > 0.95)
+        assert np.mean(rgs.predict(iris.data).round() == iris.target) > 0.95
 
 
 def test_neighbors_digits():
@@ -1056,7 +1423,7 @@ def test_neighbors_digits():
     # the 'brute' algorithm has been observed to fail if the input
     # dtype is uint8 due to overflow in distance calculations.
 
-    X = digits.data.astype('uint8')
+    X = digits.data.astype("uint8")
     Y = digits.target
     (n_samples, n_features) = X.shape
     train_test_boundary = int(n_samples * 0.8)
@@ -1064,242 +1431,372 @@ def test_neighbors_digits():
     test = np.arange(train_test_boundary, n_samples)
     (X_train, Y_train, X_test, Y_test) = X[train], Y[train], X[test], Y[test]
 
-    clf = neighbors.KNeighborsClassifier(n_neighbors=1, algorithm='brute')
+    clf = neighbors.KNeighborsClassifier(n_neighbors=1, algorithm="brute")
     score_uint8 = clf.fit(X_train, Y_train).score(X_test, Y_test)
     score_float = clf.fit(X_train.astype(float, copy=False), Y_train).score(
-        X_test.astype(float, copy=False), Y_test)
+        X_test.astype(float, copy=False), Y_test
+    )
     assert score_uint8 == score_float
 
 
 def test_kneighbors_graph():
     # Test kneighbors_graph to build the k-Nearest Neighbor graph.
-    X = np.array([[0, 1], [1.01, 1.], [2, 0]])
+    X = np.array([[0, 1], [1.01, 1.0], [2, 0]])
 
     # n_neighbors = 1
-    A = neighbors.kneighbors_graph(X, 1, mode='connectivity',
-                                   include_self=True)
+    A = neighbors.kneighbors_graph(X, 1, mode="connectivity", include_self=True)
     assert_array_equal(A.toarray(), np.eye(A.shape[0]))
 
-    A = neighbors.kneighbors_graph(X, 1, mode='distance')
-    assert_array_almost_equal(
-        A.toarray(),
-        [[0.00, 1.01, 0.],
-         [1.01, 0., 0.],
-         [0.00, 1.40716026, 0.]])
+    A = neighbors.kneighbors_graph(X, 1, mode="distance")
+    assert_allclose(
+        A.toarray(), [[0.00, 1.01, 0.0], [1.01, 0.0, 0.0], [0.00, 1.40716026, 0.0]]
+    )
 
     # n_neighbors = 2
-    A = neighbors.kneighbors_graph(X, 2, mode='connectivity',
-                                   include_self=True)
-    assert_array_equal(
-        A.toarray(),
-        [[1., 1., 0.],
-         [1., 1., 0.],
-         [0., 1., 1.]])
+    A = neighbors.kneighbors_graph(X, 2, mode="connectivity", include_self=True)
+    assert_array_equal(A.toarray(), [[1.0, 1.0, 0.0], [1.0, 1.0, 0.0], [0.0, 1.0, 1.0]])
 
-    A = neighbors.kneighbors_graph(X, 2, mode='distance')
-    assert_array_almost_equal(
+    A = neighbors.kneighbors_graph(X, 2, mode="distance")
+    assert_allclose(
         A.toarray(),
-        [[0., 1.01, 2.23606798],
-         [1.01, 0., 1.40716026],
-         [2.23606798, 1.40716026, 0.]])
+        [
+            [0.0, 1.01, 2.23606798],
+            [1.01, 0.0, 1.40716026],
+            [2.23606798, 1.40716026, 0.0],
+        ],
+    )
 
     # n_neighbors = 3
-    A = neighbors.kneighbors_graph(X, 3, mode='connectivity',
-                                   include_self=True)
-    assert_array_almost_equal(
-        A.toarray(),
-        [[1, 1, 1], [1, 1, 1], [1, 1, 1]])
+    A = neighbors.kneighbors_graph(X, 3, mode="connectivity", include_self=True)
+    assert_allclose(A.toarray(), [[1, 1, 1], [1, 1, 1], [1, 1, 1]])
 
 
-def test_kneighbors_graph_sparse(seed=36):
+@pytest.mark.parametrize("n_neighbors", [1, 2, 3])
+@pytest.mark.parametrize("mode", ["connectivity", "distance"])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_kneighbors_graph_sparse(n_neighbors, mode, csr_container, seed=36):
     # Test kneighbors_graph to build the k-Nearest Neighbor graph
     # for sparse input.
     rng = np.random.RandomState(seed)
     X = rng.randn(10, 10)
-    Xcsr = csr_matrix(X)
+    Xcsr = csr_container(X)
 
-    for n_neighbors in [1, 2, 3]:
-        for mode in ["connectivity", "distance"]:
-            assert_array_almost_equal(
-                neighbors.kneighbors_graph(X,
-                                           n_neighbors,
-                                           mode=mode).toarray(),
-                neighbors.kneighbors_graph(Xcsr,
-                                           n_neighbors,
-                                           mode=mode).toarray())
+    assert_allclose(
+        neighbors.kneighbors_graph(X, n_neighbors, mode=mode).toarray(),
+        neighbors.kneighbors_graph(Xcsr, n_neighbors, mode=mode).toarray(),
+    )
 
 
 def test_radius_neighbors_graph():
     # Test radius_neighbors_graph to build the Nearest Neighbor graph.
-    X = np.array([[0, 1], [1.01, 1.], [2, 0]])
+    X = np.array([[0, 1], [1.01, 1.0], [2, 0]])
 
-    A = neighbors.radius_neighbors_graph(X, 1.5, mode='connectivity',
-                                         include_self=True)
-    assert_array_equal(
-        A.toarray(),
-        [[1., 1., 0.],
-         [1., 1., 1.],
-         [0., 1., 1.]])
+    A = neighbors.radius_neighbors_graph(X, 1.5, mode="connectivity", include_self=True)
+    assert_array_equal(A.toarray(), [[1.0, 1.0, 0.0], [1.0, 1.0, 1.0], [0.0, 1.0, 1.0]])
 
-    A = neighbors.radius_neighbors_graph(X, 1.5, mode='distance')
-    assert_array_almost_equal(
-        A.toarray(),
-        [[0., 1.01, 0.],
-         [1.01, 0., 1.40716026],
-         [0., 1.40716026, 0.]])
+    A = neighbors.radius_neighbors_graph(X, 1.5, mode="distance")
+    assert_allclose(
+        A.toarray(), [[0.0, 1.01, 0.0], [1.01, 0.0, 1.40716026], [0.0, 1.40716026, 0.0]]
+    )
 
 
-def test_radius_neighbors_graph_sparse(seed=36):
+@pytest.mark.parametrize("n_neighbors", [1, 2, 3])
+@pytest.mark.parametrize("mode", ["connectivity", "distance"])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_radius_neighbors_graph_sparse(n_neighbors, mode, csr_container, seed=36):
     # Test radius_neighbors_graph to build the Nearest Neighbor graph
     # for sparse input.
     rng = np.random.RandomState(seed)
     X = rng.randn(10, 10)
-    Xcsr = csr_matrix(X)
-
-    for n_neighbors in [1, 2, 3]:
-        for mode in ["connectivity", "distance"]:
-            assert_array_almost_equal(
-                neighbors.radius_neighbors_graph(X,
-                                                 n_neighbors,
-                                                 mode=mode).toarray(),
-                neighbors.radius_neighbors_graph(Xcsr,
-                                                 n_neighbors,
-                                                 mode=mode).toarray())
+    Xcsr = csr_container(X)
 
+    assert_allclose(
+        neighbors.radius_neighbors_graph(X, n_neighbors, mode=mode).toarray(),
+        neighbors.radius_neighbors_graph(Xcsr, n_neighbors, mode=mode).toarray(),
+    )
 
-def test_neighbors_badargs():
-    # Test bad argument values: these should all raise ValueErrors
-    with pytest.raises(ValueError):
-        neighbors.NearestNeighbors(algorithm='blah')
 
+@pytest.mark.parametrize(
+    "Estimator",
+    [
+        neighbors.KNeighborsClassifier,
+        neighbors.RadiusNeighborsClassifier,
+        neighbors.KNeighborsRegressor,
+        neighbors.RadiusNeighborsRegressor,
+    ],
+)
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_neighbors_validate_parameters(Estimator, csr_container):
+    """Additional parameter validation for *Neighbors* estimators not covered by common
+    validation."""
     X = rng.random_sample((10, 2))
-    Xsparse = csr_matrix(X)
+    Xsparse = csr_container(X)
     X3 = rng.random_sample((10, 3))
     y = np.ones(10)
 
-    for cls in (neighbors.KNeighborsClassifier,
-                neighbors.RadiusNeighborsClassifier,
-                neighbors.KNeighborsRegressor,
-                neighbors.RadiusNeighborsRegressor):
-        with pytest.raises(ValueError):
-            cls(weights='blah')
-        with pytest.raises(ValueError):
-            cls(p=-1)
-        with pytest.raises(ValueError):
-            cls(algorithm='blah')
+    nbrs = Estimator(algorithm="ball_tree", metric="haversine")
+    msg = "instance is not fitted yet"
+    with pytest.raises(ValueError, match=msg):
+        nbrs.predict(X)
+    msg = "Metric 'haversine' not valid for sparse input."
+    with pytest.raises(ValueError, match=msg):
+        ignore_warnings(nbrs.fit(Xsparse, y))
 
-        nbrs = cls(algorithm='ball_tree', metric='haversine')
-        with pytest.raises(ValueError):
-            nbrs.predict(X)
-        with pytest.raises(ValueError):
-            ignore_warnings(nbrs.fit(Xsparse, y))
+    nbrs = Estimator(metric="haversine", algorithm="brute")
+    nbrs.fit(X3, y)
+    msg = "Haversine distance only valid in 2 dimensions"
+    with pytest.raises(ValueError, match=msg):
+        nbrs.predict(X3)
 
-        nbrs = cls(metric='haversine', algorithm='brute')
-        nbrs.fit(X3, y)
-        msg = "Haversine distance only valid in 2 dimensions"
-        with pytest.raises(ValueError, match=msg):
-            nbrs.predict(X3)
+    nbrs = Estimator()
+    msg = re.escape("Found array with 0 sample(s)")
+    with pytest.raises(ValueError, match=msg):
+        nbrs.fit(np.ones((0, 2)), np.ones(0))
 
-        nbrs = cls()
-        with pytest.raises(ValueError):
-            nbrs.fit(np.ones((0, 2)), np.ones(0))
-        with pytest.raises(ValueError):
-            nbrs.fit(X[:, :, None], y)
-        nbrs.fit(X, y)
-        with pytest.raises(ValueError):
-            nbrs.predict([[]])
-        if (issubclass(cls, neighbors.KNeighborsClassifier) or
-                issubclass(cls, neighbors.KNeighborsRegressor)):
-            nbrs = cls(n_neighbors=-1)
-            with pytest.raises(ValueError):
-                nbrs.fit(X, y)
+    msg = "Found array with dim 3"
+    with pytest.raises(ValueError, match=msg):
+        nbrs.fit(X[:, :, None], y)
+    nbrs.fit(X, y)
 
-    nbrs = neighbors.NearestNeighbors().fit(X)
+    msg = re.escape("Found array with 0 feature(s)")
+    with pytest.raises(ValueError, match=msg):
+        nbrs.predict([[]])
 
-    with pytest.raises(ValueError):
-        nbrs.kneighbors_graph(X, mode='blah')
-    with pytest.raises(ValueError):
-        nbrs.radius_neighbors_graph(X, mode='blah')
 
+@pytest.mark.parametrize(
+    "Estimator",
+    [
+        neighbors.KNeighborsClassifier,
+        neighbors.RadiusNeighborsClassifier,
+        neighbors.KNeighborsRegressor,
+        neighbors.RadiusNeighborsRegressor,
+    ],
+)
+@pytest.mark.parametrize("n_features", [2, 100])
+@pytest.mark.parametrize("algorithm", ["auto", "brute"])
+def test_neighbors_minkowski_semimetric_algo_warn(Estimator, n_features, algorithm):
+    """
+    Validation of all classes extending NeighborsBase with
+    Minkowski semi-metrics (i.e. when 0 < p < 1). That proper
+    Warning is raised for `algorithm="auto"` and "brute".
+    """
+    X = rng.random_sample((10, n_features))
+    y = np.ones(10)
 
-def test_neighbors_metrics(n_samples=20, n_features=3,
-                           n_query_pts=2, n_neighbors=5):
-    # Test computing the neighbors for various metrics
-    # create a symmetric matrix
-    V = rng.rand(n_features, n_features)
-    VI = np.dot(V, V.T)
-
-    metrics = [('euclidean', {}),
-               ('manhattan', {}),
-               ('minkowski', dict(p=1)),
-               ('minkowski', dict(p=2)),
-               ('minkowski', dict(p=3)),
-               ('minkowski', dict(p=np.inf)),
-               ('chebyshev', {}),
-               ('seuclidean', dict(V=rng.rand(n_features))),
-               ('wminkowski', dict(p=3, w=rng.rand(n_features))),
-               ('mahalanobis', dict(VI=VI)),
-               ('haversine', {})]
-    algorithms = ['brute', 'ball_tree', 'kd_tree']
-    X = rng.rand(n_samples, n_features)
+    model = Estimator(p=0.1, algorithm=algorithm)
+    msg = (
+        "Mind that for 0 < p < 1, Minkowski metrics are not distance"
+        " metrics. Continuing the execution with `algorithm='brute'`."
+    )
+    with pytest.warns(UserWarning, match=msg):
+        model.fit(X, y)
+
+    assert model._fit_method == "brute"
+
+
+@pytest.mark.parametrize(
+    "Estimator",
+    [
+        neighbors.KNeighborsClassifier,
+        neighbors.RadiusNeighborsClassifier,
+        neighbors.KNeighborsRegressor,
+        neighbors.RadiusNeighborsRegressor,
+    ],
+)
+@pytest.mark.parametrize("n_features", [2, 100])
+@pytest.mark.parametrize("algorithm", ["kd_tree", "ball_tree"])
+def test_neighbors_minkowski_semimetric_algo_error(Estimator, n_features, algorithm):
+    """Check that we raise a proper error if `algorithm!='brute'` and `p<1`."""
+    X = rng.random_sample((10, 2))
+    y = np.ones(10)
 
-    test = rng.rand(n_query_pts, n_features)
+    model = Estimator(algorithm=algorithm, p=0.1)
+    msg = (
+        f'algorithm="{algorithm}" does not support 0 < p < 1 for '
+        "the Minkowski metric. To resolve this problem either "
+        'set p >= 1 or algorithm="brute".'
+    )
+    with pytest.raises(ValueError, match=msg):
+        model.fit(X, y)
 
-    for metric, metric_params in metrics:
-        if metric == "wminkowski" and sp_version >= parse_version("1.8.0"):
-            # wminkowski will be removed in SciPy 1.8.0
-            continue
+
+# TODO: remove when NearestNeighbors methods uses parameter validation mechanism
+def test_nearest_neighbors_validate_params():
+    """Validate parameter of NearestNeighbors."""
+    X = rng.random_sample((10, 2))
+
+    nbrs = neighbors.NearestNeighbors().fit(X)
+    msg = (
+        'Unsupported mode, must be one of "connectivity", or "distance" but got "blah"'
+        " instead"
+    )
+    with pytest.raises(ValueError, match=msg):
+        nbrs.kneighbors_graph(X, mode="blah")
+    with pytest.raises(ValueError, match=msg):
+        nbrs.radius_neighbors_graph(X, mode="blah")
+
+
+@pytest.mark.parametrize(
+    "metric",
+    sorted(
+        set(neighbors.VALID_METRICS["ball_tree"]).intersection(
+            neighbors.VALID_METRICS["brute"]
+        )
+        - set(["pyfunc", *BOOL_METRICS])
+    )
+    + DISTANCE_METRIC_OBJS,
+)
+def test_neighbors_metrics(
+    global_dtype,
+    global_random_seed,
+    metric,
+    n_samples=20,
+    n_features=3,
+    n_query_pts=2,
+    n_neighbors=5,
+):
+    rng = np.random.RandomState(global_random_seed)
+
+    metric = _parse_metric(metric, global_dtype)
+
+    # Test computing the neighbors for various metrics
+    algorithms = ["brute", "ball_tree", "kd_tree"]
+    X_train = rng.rand(n_samples, n_features).astype(global_dtype, copy=False)
+    X_test = rng.rand(n_query_pts, n_features).astype(global_dtype, copy=False)
+
+    metric_params_list = _generate_test_params_for(metric, n_features)
+
+    for metric_params in metric_params_list:
+        # Some metric (e.g. Weighted minkowski) are not supported by KDTree
+        exclude_kd_tree = (
+            False
+            if isinstance(metric, DistanceMetric)
+            else metric not in neighbors.VALID_METRICS["kd_tree"]
+            or ("minkowski" in metric and "w" in metric_params)
+        )
         results = {}
-        p = metric_params.pop('p', 2)
+        p = metric_params.pop("p", 2)
         for algorithm in algorithms:
-            # KD tree doesn't support all metrics
-            if (algorithm == 'kd_tree' and
-                    metric not in neighbors.KDTree.valid_metrics):
+            if isinstance(metric, DistanceMetric) and global_dtype == np.float32:
+                if "tree" in algorithm:  # pragma: nocover
+                    pytest.skip(
+                        "Neither KDTree nor BallTree support 32-bit distance metric"
+                        " objects."
+                    )
+            neigh = neighbors.NearestNeighbors(
+                n_neighbors=n_neighbors,
+                algorithm=algorithm,
+                metric=metric,
+                p=p,
+                metric_params=metric_params,
+            )
+
+            if exclude_kd_tree and algorithm == "kd_tree":
                 with pytest.raises(ValueError):
-                    neighbors.NearestNeighbors(algorithm=algorithm,
-                                               metric=metric,
-                                               metric_params=metric_params)
+                    neigh.fit(X_train)
                 continue
-            neigh = neighbors.NearestNeighbors(n_neighbors=n_neighbors,
-                                               algorithm=algorithm,
-                                               metric=metric, p=p,
-                                               metric_params=metric_params)
 
             # Haversine distance only accepts 2D data
-            feature_sl = (slice(None, 2)
-                          if metric == 'haversine' else slice(None))
-
-            neigh.fit(X[:, feature_sl])
-
-            # wminkoski is deprecated in SciPy 1.6.0 and removed in 1.8.0
-            ExceptionToAssert = None
-            if (metric == "wminkowski" and algorithm == 'brute'
-                    and sp_version >= parse_version("1.6.0")):
-                ExceptionToAssert = DeprecationWarning
-
-            with pytest.warns(ExceptionToAssert):
-                results[algorithm] = neigh.kneighbors(test[:, feature_sl],
-                                                      return_distance=True)
-
-        assert_array_almost_equal(results['brute'][0], results['ball_tree'][0])
-        assert_array_almost_equal(results['brute'][1], results['ball_tree'][1])
-        if 'kd_tree' in results:
-            assert_array_almost_equal(results['brute'][0],
-                                      results['kd_tree'][0])
-            assert_array_almost_equal(results['brute'][1],
-                                      results['kd_tree'][1])
+            if metric == "haversine":
+                feature_sl = slice(None, 2)
+                X_train = np.ascontiguousarray(X_train[:, feature_sl])
+                X_test = np.ascontiguousarray(X_test[:, feature_sl])
+
+            neigh.fit(X_train)
+            results[algorithm] = neigh.kneighbors(X_test, return_distance=True)
+
+        brute_dst, brute_idx = results["brute"]
+        ball_tree_dst, ball_tree_idx = results["ball_tree"]
+
+        # The returned distances are always in float64 regardless of the input dtype
+        # We need to adjust the tolerance w.r.t the input dtype
+        rtol = 1e-7 if global_dtype == np.float64 else 1e-4
+
+        assert_allclose(brute_dst, ball_tree_dst, rtol=rtol)
+        assert_array_equal(brute_idx, ball_tree_idx)
+
+        if not exclude_kd_tree:
+            kd_tree_dst, kd_tree_idx = results["kd_tree"]
+            assert_allclose(brute_dst, kd_tree_dst, rtol=rtol)
+            assert_array_equal(brute_idx, kd_tree_idx)
+
+            assert_allclose(ball_tree_dst, kd_tree_dst, rtol=rtol)
+            assert_array_equal(ball_tree_idx, kd_tree_idx)
+
+
+# TODO: Remove ignore_warnings when minimum supported SciPy version is 1.17
+# Some scipy metrics are deprecated (depending on the scipy version) but we
+# still want to test them.
+@ignore_warnings(category=DeprecationWarning)
+@pytest.mark.parametrize(
+    "metric", sorted(set(neighbors.VALID_METRICS["brute"]) - set(["precomputed"]))
+)
+def test_kneighbors_brute_backend(
+    metric,
+    global_dtype,
+    global_random_seed,
+    n_samples=2000,
+    n_features=30,
+    n_query_pts=5,
+    n_neighbors=5,
+):
+    rng = np.random.RandomState(global_random_seed)
+    # Both backend for the 'brute' algorithm of kneighbors must give identical results.
+    X_train = rng.rand(n_samples, n_features).astype(global_dtype, copy=False)
+    X_test = rng.rand(n_query_pts, n_features).astype(global_dtype, copy=False)
+
+    # Haversine distance only accepts 2D data
+    if metric == "haversine":
+        feature_sl = slice(None, 2)
+        X_train = np.ascontiguousarray(X_train[:, feature_sl])
+        X_test = np.ascontiguousarray(X_test[:, feature_sl])
+
+    if metric in PAIRWISE_BOOLEAN_FUNCTIONS:
+        X_train = X_train > 0.5
+        X_test = X_test > 0.5
+
+    metric_params_list = _generate_test_params_for(metric, n_features)
+
+    for metric_params in metric_params_list:
+        p = metric_params.pop("p", 2)
+
+        neigh = neighbors.NearestNeighbors(
+            n_neighbors=n_neighbors,
+            algorithm="brute",
+            metric=metric,
+            p=p,
+            metric_params=metric_params,
+        )
+
+        neigh.fit(X_train)
+
+        with config_context(enable_cython_pairwise_dist=False):
+            # Use the legacy backend for brute
+            legacy_brute_dst, legacy_brute_idx = neigh.kneighbors(
+                X_test, return_distance=True
+            )
+        with config_context(enable_cython_pairwise_dist=True):
+            # Use the pairwise-distances reduction backend for brute
+            pdr_brute_dst, pdr_brute_idx = neigh.kneighbors(
+                X_test, return_distance=True
+            )
+
+        assert_compatible_argkmin_results(
+            legacy_brute_dst, pdr_brute_dst, legacy_brute_idx, pdr_brute_idx
+        )
 
 
 def test_callable_metric():
-
     def custom_metric(x1, x2):
-        return np.sqrt(np.sum(x1 ** 2 + x2 ** 2))
+        return np.sqrt(np.sum(x1**2 + x2**2))
 
     X = np.random.RandomState(42).rand(20, 2)
-    nbrs1 = neighbors.NearestNeighbors(n_neighbors=3, algorithm='auto',
-                                       metric=custom_metric)
-    nbrs2 = neighbors.NearestNeighbors(n_neighbors=3, algorithm='brute',
-                                       metric=custom_metric)
+    nbrs1 = neighbors.NearestNeighbors(
+        n_neighbors=3, algorithm="auto", metric=custom_metric
+    )
+    nbrs2 = neighbors.NearestNeighbors(
+        n_neighbors=3, algorithm="brute", metric=custom_metric
+    )
 
     nbrs1.fit(X)
     nbrs2.fit(X)
@@ -1307,74 +1804,76 @@ def custom_metric(x1, x2):
     dist1, ind1 = nbrs1.kneighbors(X)
     dist2, ind2 = nbrs2.kneighbors(X)
 
-    assert_array_almost_equal(dist1, dist2)
+    assert_allclose(dist1, dist2)
 
 
-def test_valid_brute_metric_for_auto_algorithm():
-    X = rng.rand(12, 12)
-    Xcsr = csr_matrix(X)
+@pytest.mark.parametrize(
+    "metric", neighbors.VALID_METRICS["brute"] + DISTANCE_METRIC_OBJS
+)
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_valid_brute_metric_for_auto_algorithm(
+    global_dtype, metric, csr_container, n_samples=20, n_features=12
+):
+    metric = _parse_metric(metric, global_dtype)
 
-    # check that there is a metric that is valid for brute
-    # but not ball_tree (so we actually test something)
-    assert "cosine" in VALID_METRICS['brute']
-    assert "cosine" not in VALID_METRICS['ball_tree']
+    X = rng.rand(n_samples, n_features).astype(global_dtype, copy=False)
+    Xcsr = csr_container(X)
 
-    # Metric which don't required any additional parameter
-    require_params = ['mahalanobis', 'wminkowski', 'seuclidean']
-    for metric in VALID_METRICS['brute']:
-        if metric != 'precomputed' and metric not in require_params:
-            nn = neighbors.NearestNeighbors(n_neighbors=3,
-                                            algorithm='auto',
-                                            metric=metric)
-            if metric != 'haversine':
-                nn.fit(X)
-                nn.kneighbors(X)
-            else:
-                nn.fit(X[:, :2])
-                nn.kneighbors(X[:, :2])
-        elif metric == 'precomputed':
-            X_precomputed = rng.random_sample((10, 4))
-            Y_precomputed = rng.random_sample((3, 4))
-            DXX = metrics.pairwise_distances(X_precomputed, metric='euclidean')
-            DYX = metrics.pairwise_distances(Y_precomputed, X_precomputed,
-                                             metric='euclidean')
-            nb_p = neighbors.NearestNeighbors(n_neighbors=3)
-            nb_p.fit(DXX)
-            nb_p.kneighbors(DYX)
-
-    for metric in VALID_METRICS_SPARSE['brute']:
-        if metric != 'precomputed' and metric not in require_params:
-            nn = neighbors.NearestNeighbors(n_neighbors=3, algorithm='auto',
-                                            metric=metric).fit(Xcsr)
-            nn.kneighbors(Xcsr)
-
-    # Metric with parameter
-    VI = np.dot(X, X.T)
-    list_metrics = [('seuclidean', dict(V=rng.rand(12))),
-                    ('wminkowski', dict(w=rng.rand(12))),
-                    ('mahalanobis', dict(VI=VI))]
-    for metric, params in list_metrics:
-        nn = neighbors.NearestNeighbors(n_neighbors=3, algorithm='auto',
-                                        metric=metric,
-                                        metric_params=params).fit(X)
-        nn.kneighbors(X)
+    metric_params_list = _generate_test_params_for(metric, n_features)
+
+    if metric == "precomputed":
+        X_precomputed = rng.random_sample((10, 4))
+        Y_precomputed = rng.random_sample((3, 4))
+        DXX = metrics.pairwise_distances(X_precomputed, metric="euclidean")
+        DYX = metrics.pairwise_distances(
+            Y_precomputed, X_precomputed, metric="euclidean"
+        )
+        nb_p = neighbors.NearestNeighbors(n_neighbors=3, metric="precomputed")
+        nb_p.fit(DXX)
+        nb_p.kneighbors(DYX)
+
+    else:
+        for metric_params in metric_params_list:
+            nn = neighbors.NearestNeighbors(
+                n_neighbors=3,
+                algorithm="auto",
+                metric=metric,
+                metric_params=metric_params,
+            )
+            # Haversine distance only accepts 2D data
+            if metric == "haversine":
+                feature_sl = slice(None, 2)
+                X = np.ascontiguousarray(X[:, feature_sl])
+
+            nn.fit(X)
+            nn.kneighbors(X)
+
+            if metric in VALID_METRICS_SPARSE["brute"]:
+                nn = neighbors.NearestNeighbors(
+                    n_neighbors=3, algorithm="auto", metric=metric
+                ).fit(Xcsr)
+                nn.kneighbors(Xcsr)
 
 
 def test_metric_params_interface():
+    X = rng.rand(5, 5)
+    y = rng.randint(0, 2, 5)
+    est = neighbors.KNeighborsClassifier(metric_params={"p": 3})
     with pytest.warns(SyntaxWarning):
-        neighbors.KNeighborsClassifier(metric_params={'p': 3})
+        est.fit(X, y)
 
 
-def test_predict_sparse_ball_kd_tree():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_predict_sparse_ball_kd_tree(csr_container):
     rng = np.random.RandomState(0)
     X = rng.rand(5, 5)
     y = rng.randint(0, 2, 5)
-    nbrs1 = neighbors.KNeighborsClassifier(1, algorithm='kd_tree')
-    nbrs2 = neighbors.KNeighborsRegressor(1, algorithm='ball_tree')
+    nbrs1 = neighbors.KNeighborsClassifier(1, algorithm="kd_tree")
+    nbrs2 = neighbors.KNeighborsRegressor(1, algorithm="ball_tree")
     for model in [nbrs1, nbrs2]:
         model.fit(X, y)
         with pytest.raises(ValueError):
-            model.predict(csr_matrix(X))
+            model.predict(csr_container(X))
 
 
 def test_non_euclidean_kneighbors():
@@ -1387,30 +1886,30 @@ def test_non_euclidean_kneighbors():
     radius = dist_array[15]
 
     # Test kneighbors_graph
-    for metric in ['manhattan', 'chebyshev']:
+    for metric in ["manhattan", "chebyshev"]:
         nbrs_graph = neighbors.kneighbors_graph(
-            X, 3, metric=metric, mode='connectivity',
-            include_self=True).toarray()
+            X, 3, metric=metric, mode="connectivity", include_self=True
+        ).toarray()
         nbrs1 = neighbors.NearestNeighbors(n_neighbors=3, metric=metric).fit(X)
         assert_array_equal(nbrs_graph, nbrs1.kneighbors_graph(X).toarray())
 
     # Test radiusneighbors_graph
-    for metric in ['manhattan', 'chebyshev']:
+    for metric in ["manhattan", "chebyshev"]:
         nbrs_graph = neighbors.radius_neighbors_graph(
-            X, radius, metric=metric, mode='connectivity',
-            include_self=True).toarray()
+            X, radius, metric=metric, mode="connectivity", include_self=True
+        ).toarray()
         nbrs1 = neighbors.NearestNeighbors(metric=metric, radius=radius).fit(X)
-        assert_array_equal(nbrs_graph, nbrs1.radius_neighbors_graph(X).A)
+        assert_array_equal(nbrs_graph, nbrs1.radius_neighbors_graph(X).toarray())
 
     # Raise error when wrong parameters are supplied,
-    X_nbrs = neighbors.NearestNeighbors(n_neighbors=3, metric='manhattan')
+    X_nbrs = neighbors.NearestNeighbors(n_neighbors=3, metric="manhattan")
     X_nbrs.fit(X)
     with pytest.raises(ValueError):
-        neighbors.kneighbors_graph(X_nbrs, 3, metric='euclidean')
-    X_nbrs = neighbors.NearestNeighbors(radius=radius, metric='manhattan')
+        neighbors.kneighbors_graph(X_nbrs, 3, metric="euclidean")
+    X_nbrs = neighbors.NearestNeighbors(radius=radius, metric="manhattan")
     X_nbrs.fit(X)
     with pytest.raises(ValueError):
-        neighbors.radius_neighbors_graph(X_nbrs, radius, metric='euclidean')
+        neighbors.radius_neighbors_graph(X_nbrs, radius, metric="euclidean")
 
 
 def check_object_arrays(nparray, list_check):
@@ -1422,7 +1921,6 @@ def test_k_and_radius_neighbors_train_is_not_query():
     # Test kneighbors et.al when query is not training data
 
     for algorithm in ALGORITHMS:
-
         nn = neighbors.NearestNeighbors(n_neighbors=1, algorithm=algorithm)
 
         X = [[0], [1]]
@@ -1439,222 +1937,220 @@ def test_k_and_radius_neighbors_train_is_not_query():
 
         # Test the graph variants.
         assert_array_equal(
-            nn.kneighbors_graph(test_data).A, [[0., 1.], [0., 1.]])
+            nn.kneighbors_graph(test_data).toarray(), [[0.0, 1.0], [0.0, 1.0]]
+        )
         assert_array_equal(
-            nn.kneighbors_graph([[2], [1]], mode='distance').A,
-            np.array([[0., 1.], [0., 0.]]))
+            nn.kneighbors_graph([[2], [1]], mode="distance").toarray(),
+            np.array([[0.0, 1.0], [0.0, 0.0]]),
+        )
         rng = nn.radius_neighbors_graph([[2], [1]], radius=1.5)
-        assert_array_equal(rng.A, [[0, 1], [1, 1]])
+        assert_array_equal(rng.toarray(), [[0, 1], [1, 1]])
 
 
-def test_k_and_radius_neighbors_X_None():
+@pytest.mark.parametrize("algorithm", ALGORITHMS)
+def test_k_and_radius_neighbors_X_None(algorithm):
     # Test kneighbors et.al when query is None
-    for algorithm in ALGORITHMS:
-
-        nn = neighbors.NearestNeighbors(n_neighbors=1, algorithm=algorithm)
-
-        X = [[0], [1]]
-        nn.fit(X)
+    nn = neighbors.NearestNeighbors(n_neighbors=1, algorithm=algorithm)
+
+    X = [[0], [1]]
+    nn.fit(X)
+
+    dist, ind = nn.kneighbors()
+    assert_array_equal(dist, [[1], [1]])
+    assert_array_equal(ind, [[1], [0]])
+    dist, ind = nn.radius_neighbors(None, radius=1.5)
+    check_object_arrays(dist, [[1], [1]])
+    check_object_arrays(ind, [[1], [0]])
+
+    # Test the graph variants.
+    rng = nn.radius_neighbors_graph(None, radius=1.5)
+    kng = nn.kneighbors_graph(None)
+    for graph in [rng, kng]:
+        assert_array_equal(graph.toarray(), [[0, 1], [1, 0]])
+        assert_array_equal(graph.data, [1, 1])
+        assert_array_equal(graph.indices, [1, 0])
+
+    X = [[0, 1], [0, 1], [1, 1]]
+    nn = neighbors.NearestNeighbors(n_neighbors=2, algorithm=algorithm)
+    nn.fit(X)
+    assert_array_equal(
+        nn.kneighbors_graph().toarray(),
+        np.array([[0.0, 1.0, 1.0], [1.0, 0.0, 1.0], [1.0, 1.0, 0]]),
+    )
 
-        dist, ind = nn.kneighbors()
-        assert_array_equal(dist, [[1], [1]])
-        assert_array_equal(ind, [[1], [0]])
-        dist, ind = nn.radius_neighbors(None, radius=1.5)
-        check_object_arrays(dist, [[1], [1]])
-        check_object_arrays(ind, [[1], [0]])
 
-        # Test the graph variants.
-        rng = nn.radius_neighbors_graph(None, radius=1.5)
-        kng = nn.kneighbors_graph(None)
-        for graph in [rng, kng]:
-            assert_array_equal(graph.A, [[0, 1], [1, 0]])
-            assert_array_equal(graph.data, [1, 1])
-            assert_array_equal(graph.indices, [1, 0])
-
-        X = [[0, 1], [0, 1], [1, 1]]
-        nn = neighbors.NearestNeighbors(n_neighbors=2, algorithm=algorithm)
-        nn.fit(X)
-        assert_array_equal(
-            nn.kneighbors_graph().A,
-            np.array([[0., 1., 1.], [1., 0., 1.], [1., 1., 0]]))
+@pytest.mark.parametrize("algorithm", ALGORITHMS)
+def test_k_and_radius_neighbors_duplicates(algorithm):
+    # Test behavior of kneighbors when duplicates are present in query
+    nn = neighbors.NearestNeighbors(n_neighbors=1, algorithm=algorithm)
+    duplicates = [[0], [1], [3]]
 
+    nn.fit(duplicates)
 
-def test_k_and_radius_neighbors_duplicates():
-    # Test behavior of kneighbors when duplicates are present in query
+    # Do not do anything special to duplicates.
+    kng = nn.kneighbors_graph(duplicates, mode="distance")
+    assert_allclose(
+        kng.toarray(), np.array([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]])
+    )
+    assert_allclose(kng.data, [0.0, 0.0, 0.0])
+    assert_allclose(kng.indices, [0, 1, 2])
 
-    for algorithm in ALGORITHMS:
-        nn = neighbors.NearestNeighbors(n_neighbors=1, algorithm=algorithm)
-        nn.fit([[0], [1]])
+    dist, ind = nn.radius_neighbors([[0], [1]], radius=1.5)
+    check_object_arrays(dist, [[0, 1], [1, 0]])
+    check_object_arrays(ind, [[0, 1], [0, 1]])
 
-        # Do not do anything special to duplicates.
-        kng = nn.kneighbors_graph([[0], [1]], mode='distance')
-        assert_array_equal(
-            kng.A,
-            np.array([[0., 0.], [0., 0.]]))
-        assert_array_equal(kng.data, [0., 0.])
-        assert_array_equal(kng.indices, [0, 1])
-
-        dist, ind = nn.radius_neighbors([[0], [1]], radius=1.5)
-        check_object_arrays(dist, [[0, 1], [1, 0]])
-        check_object_arrays(ind, [[0, 1], [0, 1]])
-
-        rng = nn.radius_neighbors_graph([[0], [1]], radius=1.5)
-        assert_array_equal(rng.A, np.ones((2, 2)))
-
-        rng = nn.radius_neighbors_graph([[0], [1]], radius=1.5,
-                                        mode='distance')
-        rng.sort_indices()
-        assert_array_equal(rng.A, [[0, 1], [1, 0]])
-        assert_array_equal(rng.indices, [0, 1, 0, 1])
-        assert_array_equal(rng.data, [0, 1, 1, 0])
-
-        # Mask the first duplicates when n_duplicates > n_neighbors.
-        X = np.ones((3, 1))
-        nn = neighbors.NearestNeighbors(n_neighbors=1, algorithm='brute')
-        nn.fit(X)
-        dist, ind = nn.kneighbors()
-        assert_array_equal(dist, np.zeros((3, 1)))
-        assert_array_equal(ind, [[1], [0], [1]])
+    rng = nn.radius_neighbors_graph(duplicates, radius=1.5)
+    assert_allclose(
+        rng.toarray(), np.array([[1.0, 1.0, 0.0], [1.0, 1.0, 0.0], [0.0, 0.0, 1.0]])
+    )
 
-        # Test that zeros are explicitly marked in kneighbors_graph.
-        kng = nn.kneighbors_graph(mode='distance')
-        assert_array_equal(
-            kng.A, np.zeros((3, 3)))
-        assert_array_equal(kng.data, np.zeros(3))
-        assert_array_equal(kng.indices, [1., 0., 1.])
-        assert_array_equal(
-            nn.kneighbors_graph().A,
-            np.array([[0., 1., 0.], [1., 0., 0.], [0., 1., 0.]]))
+    rng = nn.radius_neighbors_graph([[0], [1]], radius=1.5, mode="distance")
+    rng.sort_indices()
+    assert_allclose(rng.toarray(), [[0, 1, 0], [1, 0, 0]])
+    assert_allclose(rng.indices, [0, 1, 0, 1])
+    assert_allclose(rng.data, [0, 1, 1, 0])
+
+    # Mask the first duplicates when n_duplicates > n_neighbors.
+    X = np.ones((3, 1))
+    nn = neighbors.NearestNeighbors(n_neighbors=1, algorithm="brute")
+    nn.fit(X)
+    dist, ind = nn.kneighbors()
+    assert_allclose(dist, np.zeros((3, 1)))
+    assert_allclose(ind, [[1], [0], [1]])
+
+    # Test that zeros are explicitly marked in kneighbors_graph.
+    kng = nn.kneighbors_graph(mode="distance")
+    assert_allclose(kng.toarray(), np.zeros((3, 3)))
+    assert_allclose(kng.data, np.zeros(3))
+    assert_allclose(kng.indices, [1, 0, 1])
+    assert_allclose(
+        nn.kneighbors_graph().toarray(),
+        np.array([[0.0, 1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0]]),
+    )
 
 
 def test_include_self_neighbors_graph():
     # Test include_self parameter in neighbors_graph
     X = [[2, 3], [4, 5]]
-    kng = neighbors.kneighbors_graph(X, 1, include_self=True).A
-    kng_not_self = neighbors.kneighbors_graph(X, 1, include_self=False).A
-    assert_array_equal(kng, [[1., 0.], [0., 1.]])
-    assert_array_equal(kng_not_self, [[0., 1.], [1., 0.]])
+    kng = neighbors.kneighbors_graph(X, 1, include_self=True).toarray()
+    kng_not_self = neighbors.kneighbors_graph(X, 1, include_self=False).toarray()
+    assert_array_equal(kng, [[1.0, 0.0], [0.0, 1.0]])
+    assert_array_equal(kng_not_self, [[0.0, 1.0], [1.0, 0.0]])
 
-    rng = neighbors.radius_neighbors_graph(X, 5.0, include_self=True).A
+    rng = neighbors.radius_neighbors_graph(X, 5.0, include_self=True).toarray()
     rng_not_self = neighbors.radius_neighbors_graph(
-        X, 5.0, include_self=False).A
-    assert_array_equal(rng, [[1., 1.], [1., 1.]])
-    assert_array_equal(rng_not_self, [[0., 1.], [1., 0.]])
+        X, 5.0, include_self=False
+    ).toarray()
+    assert_array_equal(rng, [[1.0, 1.0], [1.0, 1.0]])
+    assert_array_equal(rng_not_self, [[0.0, 1.0], [1.0, 0.0]])
 
 
-@pytest.mark.parametrize('algorithm', ALGORITHMS)
+@pytest.mark.parametrize("algorithm", ALGORITHMS)
 def test_same_knn_parallel(algorithm):
-    X, y = datasets.make_classification(n_samples=30, n_features=5,
-                                        n_redundant=0, random_state=0)
+    X, y = datasets.make_classification(
+        n_samples=30, n_features=5, n_redundant=0, random_state=0
+    )
     X_train, X_test, y_train, y_test = train_test_split(X, y)
 
-    clf = neighbors.KNeighborsClassifier(n_neighbors=3,
-                                         algorithm=algorithm)
+    clf = neighbors.KNeighborsClassifier(n_neighbors=3, algorithm=algorithm)
     clf.fit(X_train, y_train)
     y = clf.predict(X_test)
     dist, ind = clf.kneighbors(X_test)
-    graph = clf.kneighbors_graph(X_test, mode='distance').toarray()
+    graph = clf.kneighbors_graph(X_test, mode="distance").toarray()
 
     clf.set_params(n_jobs=3)
     clf.fit(X_train, y_train)
     y_parallel = clf.predict(X_test)
     dist_parallel, ind_parallel = clf.kneighbors(X_test)
-    graph_parallel = \
-        clf.kneighbors_graph(X_test, mode='distance').toarray()
+    graph_parallel = clf.kneighbors_graph(X_test, mode="distance").toarray()
 
     assert_array_equal(y, y_parallel)
-    assert_array_almost_equal(dist, dist_parallel)
+    assert_allclose(dist, dist_parallel)
     assert_array_equal(ind, ind_parallel)
-    assert_array_almost_equal(graph, graph_parallel)
+    assert_allclose(graph, graph_parallel)
 
 
-@pytest.mark.parametrize('algorithm', ALGORITHMS)
+@pytest.mark.parametrize("algorithm", ALGORITHMS)
 def test_same_radius_neighbors_parallel(algorithm):
-    X, y = datasets.make_classification(n_samples=30, n_features=5,
-                                        n_redundant=0, random_state=0)
+    X, y = datasets.make_classification(
+        n_samples=30, n_features=5, n_redundant=0, random_state=0
+    )
     X_train, X_test, y_train, y_test = train_test_split(X, y)
 
-    clf = neighbors.RadiusNeighborsClassifier(radius=10,
-                                              algorithm=algorithm)
+    clf = neighbors.RadiusNeighborsClassifier(radius=10, algorithm=algorithm)
     clf.fit(X_train, y_train)
     y = clf.predict(X_test)
     dist, ind = clf.radius_neighbors(X_test)
-    graph = clf.radius_neighbors_graph(X_test, mode='distance').toarray()
+    graph = clf.radius_neighbors_graph(X_test, mode="distance").toarray()
 
     clf.set_params(n_jobs=3)
     clf.fit(X_train, y_train)
     y_parallel = clf.predict(X_test)
     dist_parallel, ind_parallel = clf.radius_neighbors(X_test)
-    graph_parallel = \
-        clf.radius_neighbors_graph(X_test, mode='distance').toarray()
+    graph_parallel = clf.radius_neighbors_graph(X_test, mode="distance").toarray()
 
     assert_array_equal(y, y_parallel)
     for i in range(len(dist)):
-        assert_array_almost_equal(dist[i], dist_parallel[i])
+        assert_allclose(dist[i], dist_parallel[i])
         assert_array_equal(ind[i], ind_parallel[i])
-    assert_array_almost_equal(graph, graph_parallel)
+    assert_allclose(graph, graph_parallel)
 
 
-@pytest.mark.parametrize('backend', JOBLIB_BACKENDS)
-@pytest.mark.parametrize('algorithm', ALGORITHMS)
+@pytest.mark.parametrize("backend", ["threading", "loky"])
+@pytest.mark.parametrize("algorithm", ALGORITHMS)
 def test_knn_forcing_backend(backend, algorithm):
-    # Non-regression test which ensure the knn methods are properly working
+    # Non-regression test which ensures the knn methods are properly working
     # even when forcing the global joblib backend.
     with joblib.parallel_backend(backend):
-        X, y = datasets.make_classification(n_samples=30, n_features=5,
-                                            n_redundant=0, random_state=0)
+        X, y = datasets.make_classification(
+            n_samples=30, n_features=5, n_redundant=0, random_state=0
+        )
         X_train, X_test, y_train, y_test = train_test_split(X, y)
 
-        clf = neighbors.KNeighborsClassifier(n_neighbors=3,
-                                             algorithm=algorithm,
-                                             n_jobs=3)
+        clf = neighbors.KNeighborsClassifier(
+            n_neighbors=3, algorithm=algorithm, n_jobs=2
+        )
         clf.fit(X_train, y_train)
         clf.predict(X_test)
         clf.kneighbors(X_test)
-        clf.kneighbors_graph(X_test, mode='distance').toarray()
+        clf.kneighbors_graph(X_test, mode="distance")
 
 
 def test_dtype_convert():
     classifier = neighbors.KNeighborsClassifier(n_neighbors=1)
     CLASSES = 15
     X = np.eye(CLASSES)
-    y = [ch for ch in 'ABCDEFGHIJKLMNOPQRSTU'[:CLASSES]]
+    y = [ch for ch in "ABCDEFGHIJKLMNOPQRSTU"[:CLASSES]]
 
     result = classifier.fit(X, y).predict(X)
     assert_array_equal(result, y)
 
 
-def test_sparse_metric_callable():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sparse_metric_callable(csr_container):
     def sparse_metric(x, y):  # Metric accepting sparse matrix input (only)
         assert issparse(x) and issparse(y)
-        return x.dot(y.T).A.item()
+        return x.dot(y.T).toarray().item()
 
-    X = csr_matrix([  # Population matrix
-        [1, 1, 1, 1, 1],
-        [1, 0, 1, 0, 1],
-        [0, 0, 1, 0, 0]
-    ])
+    X = csr_container(
+        [[1, 1, 1, 1, 1], [1, 0, 1, 0, 1], [0, 0, 1, 0, 0]]  # Population matrix
+    )
 
-    Y = csr_matrix([  # Query matrix
-        [1, 1, 0, 1, 1],
-        [1, 0, 0, 0, 1]
-    ])
+    Y = csr_container([[1, 1, 0, 1, 1], [1, 0, 0, 1, 1]])  # Query matrix
 
-    nn = neighbors.NearestNeighbors(algorithm='brute', n_neighbors=2,
-                                    metric=sparse_metric).fit(X)
+    nn = neighbors.NearestNeighbors(
+        algorithm="brute", n_neighbors=2, metric=sparse_metric
+    ).fit(X)
     N = nn.kneighbors(Y, return_distance=False)
 
     # GS indices of nearest neighbours in `X` for `sparse_metric`
-    gold_standard_nn = np.array([
-        [2, 1],
-        [2, 1]
-    ])
+    gold_standard_nn = np.array([[2, 1], [2, 1]])
 
     assert_array_equal(N, gold_standard_nn)
 
 
 # ignore conversion to boolean in pairwise_distances
-@ignore_warnings(category=DataConversionWarning)
+@pytest.mark.filterwarnings("ignore::sklearn.exceptions.DataConversionWarning")
 def test_pairwise_boolean_distance():
     # Non-regression test for #4523
     # 'brute': uses scipy.spatial.distance through pairwise_distances
@@ -1663,26 +2159,29 @@ def test_pairwise_boolean_distance():
     X = rng.uniform(size=(6, 5))
     NN = neighbors.NearestNeighbors
 
-    nn1 = NN(metric="jaccard", algorithm='brute').fit(X)
-    nn2 = NN(metric="jaccard", algorithm='ball_tree').fit(X)
+    nn1 = NN(metric="jaccard", algorithm="brute").fit(X)
+    nn2 = NN(metric="jaccard", algorithm="ball_tree").fit(X)
     assert_array_equal(nn1.kneighbors(X)[0], nn2.kneighbors(X)[0])
 
 
 def test_radius_neighbors_predict_proba():
     for seed in range(5):
-        X, y = datasets.make_classification(n_samples=50, n_features=5,
-                                            n_informative=3, n_redundant=0,
-                                            n_classes=3, random_state=seed)
+        X, y = datasets.make_classification(
+            n_samples=50,
+            n_features=5,
+            n_informative=3,
+            n_redundant=0,
+            n_classes=3,
+            random_state=seed,
+        )
         X_tr, X_te, y_tr, y_te = train_test_split(X, y, random_state=0)
         outlier_label = int(2 - seed)
-        clf = neighbors.RadiusNeighborsClassifier(radius=2,
-                                                  outlier_label=outlier_label)
+        clf = neighbors.RadiusNeighborsClassifier(radius=2, outlier_label=outlier_label)
         clf.fit(X_tr, y_tr)
         pred = clf.predict(X_te)
         proba = clf.predict_proba(X_te)
         proba_label = proba.argmax(axis=1)
-        proba_label = np.where(proba.sum(axis=1) == 0,
-                               outlier_label, proba_label)
+        proba_label = np.where(proba.sum(axis=1) == 0, outlier_label, proba_label)
         assert_array_equal(pred, proba_label)
 
 
@@ -1699,60 +2198,306 @@ def test_pipeline_with_nearest_neighbors_transformer():
     # k-neighbors estimator after radius-neighbors transformer, and vice-versa.
     factor = 2
 
-    k_trans = neighbors.KNeighborsTransformer(
-        n_neighbors=n_neighbors, mode='distance')
+    k_trans = neighbors.KNeighborsTransformer(n_neighbors=n_neighbors, mode="distance")
     k_trans_factor = neighbors.KNeighborsTransformer(
-        n_neighbors=int(n_neighbors * factor), mode='distance')
+        n_neighbors=int(n_neighbors * factor), mode="distance"
+    )
 
-    r_trans = neighbors.RadiusNeighborsTransformer(
-        radius=radius, mode='distance')
+    r_trans = neighbors.RadiusNeighborsTransformer(radius=radius, mode="distance")
     r_trans_factor = neighbors.RadiusNeighborsTransformer(
-        radius=int(radius * factor), mode='distance')
+        radius=int(radius * factor), mode="distance"
+    )
 
     k_reg = neighbors.KNeighborsRegressor(n_neighbors=n_neighbors)
     r_reg = neighbors.RadiusNeighborsRegressor(radius=radius)
 
-    test_list = [(k_trans, k_reg), (k_trans_factor, r_reg),
-                 (r_trans, r_reg), (r_trans_factor, k_reg), ]
+    test_list = [
+        (k_trans, k_reg),
+        (k_trans_factor, r_reg),
+        (r_trans, r_reg),
+        (r_trans_factor, k_reg),
+    ]
 
     for trans, reg in test_list:
         # compare the chained version and the compact version
         reg_compact = clone(reg)
         reg_precomp = clone(reg)
-        reg_precomp.set_params(metric='precomputed')
+        reg_precomp.set_params(metric="precomputed")
 
         reg_chain = make_pipeline(clone(trans), reg_precomp)
 
         y_pred_chain = reg_chain.fit(X, y).predict(X2)
         y_pred_compact = reg_compact.fit(X, y).predict(X2)
-        assert_array_almost_equal(y_pred_chain, y_pred_compact)
-
-
-@pytest.mark.parametrize('X, metric, metric_params, expected_algo', [
-    (np.random.randint(10, size=(10, 10)), 'precomputed', None, 'brute'),
-    (np.random.randn(10, 20), 'euclidean', None, 'brute'),
-    (np.random.randn(8, 5), 'euclidean', None, 'brute'),
-    (np.random.randn(10, 5), 'euclidean', None, 'kd_tree'),
-    (np.random.randn(10, 5), 'seuclidean', {'V': [2]*5}, 'ball_tree'),
-    (np.random.randn(10, 5), 'correlation', None, 'brute'),
-])
+        assert_allclose(y_pred_chain, y_pred_compact)
+
+
+@pytest.mark.parametrize(
+    "X, metric, metric_params, expected_algo",
+    [
+        (np.random.randint(10, size=(10, 10)), "precomputed", None, "brute"),
+        (np.random.randn(10, 20), "euclidean", None, "brute"),
+        (np.random.randn(8, 5), "euclidean", None, "brute"),
+        (np.random.randn(10, 5), "euclidean", None, "kd_tree"),
+        (np.random.randn(10, 5), "seuclidean", {"V": [2] * 5}, "ball_tree"),
+        (np.random.randn(10, 5), "correlation", None, "brute"),
+    ],
+)
 def test_auto_algorithm(X, metric, metric_params, expected_algo):
     model = neighbors.NearestNeighbors(
-        n_neighbors=4,
-        algorithm='auto',
-        metric=metric,
-        metric_params=metric_params
+        n_neighbors=4, algorithm="auto", metric=metric, metric_params=metric_params
     )
     model.fit(X)
     assert model._fit_method == expected_algo
 
 
-# TODO: Remove in 1.1
-@pytest.mark.parametrize("NearestNeighbors", [neighbors.KNeighborsClassifier,
-                                              neighbors.KNeighborsRegressor,
-                                              neighbors.NearestNeighbors])
-def test_pairwise_deprecated(NearestNeighbors):
-    nn = NearestNeighbors(metric='precomputed')
-    msg = r"Attribute _pairwise was deprecated in version 0\.24"
-    with pytest.warns(FutureWarning, match=msg):
-        nn._pairwise
+# TODO: Remove ignore_warnings when minimum supported SciPy version is 1.17
+# Some scipy metrics are deprecated (depending on the scipy version) but we
+# still want to test them.
+@ignore_warnings(category=DeprecationWarning)
+@pytest.mark.parametrize(
+    "metric", sorted(set(neighbors.VALID_METRICS["brute"]) - set(["precomputed"]))
+)
+def test_radius_neighbors_brute_backend(
+    metric,
+    global_random_seed,
+    global_dtype,
+    n_samples=2000,
+    n_features=30,
+    n_query_pts=5,
+    radius=1.0,
+):
+    rng = np.random.RandomState(global_random_seed)
+    # Both backends for the 'brute' algorithm of radius_neighbors
+    # must give identical results.
+    X_train = rng.rand(n_samples, n_features).astype(global_dtype, copy=False)
+    X_test = rng.rand(n_query_pts, n_features).astype(global_dtype, copy=False)
+
+    # Haversine distance only accepts 2D data
+    if metric == "haversine":
+        feature_sl = slice(None, 2)
+        X_train = np.ascontiguousarray(X_train[:, feature_sl])
+        X_test = np.ascontiguousarray(X_test[:, feature_sl])
+
+    metric_params_list = _generate_test_params_for(metric, n_features)
+
+    for metric_params in metric_params_list:
+        p = metric_params.pop("p", 2)
+
+        neigh = neighbors.NearestNeighbors(
+            radius=radius,
+            algorithm="brute",
+            metric=metric,
+            p=p,
+            metric_params=metric_params,
+        )
+
+        neigh.fit(X_train)
+
+        with config_context(enable_cython_pairwise_dist=False):
+            # Use the legacy backend for brute
+            legacy_brute_dst, legacy_brute_idx = neigh.radius_neighbors(
+                X_test, return_distance=True
+            )
+        with config_context(enable_cython_pairwise_dist=True):
+            # Use the pairwise-distances reduction backend for brute
+            pdr_brute_dst, pdr_brute_idx = neigh.radius_neighbors(
+                X_test, return_distance=True
+            )
+
+        assert_compatible_radius_results(
+            legacy_brute_dst,
+            pdr_brute_dst,
+            legacy_brute_idx,
+            pdr_brute_idx,
+            radius=radius,
+            check_sorted=False,
+        )
+
+
+def test_valid_metrics_has_no_duplicate():
+    for val in neighbors.VALID_METRICS.values():
+        assert len(val) == len(set(val))
+
+
+def test_regressor_predict_on_arraylikes():
+    """Ensures that `predict` works for array-likes when `weights` is a callable.
+
+    Non-regression test for #22687.
+    """
+    X = [[5, 1], [3, 1], [4, 3], [0, 3]]
+    y = [2, 3, 5, 6]
+
+    def _weights(dist):
+        return np.ones_like(dist)
+
+    est = KNeighborsRegressor(n_neighbors=1, algorithm="brute", weights=_weights)
+    est.fit(X, y)
+    assert_allclose(est.predict([[0, 2.5]]), [6])
+
+
+@pytest.mark.parametrize(
+    "Estimator, params",
+    [
+        (neighbors.KNeighborsClassifier, {"n_neighbors": 2}),
+        (neighbors.KNeighborsRegressor, {"n_neighbors": 2}),
+        (neighbors.RadiusNeighborsRegressor, {}),
+        (neighbors.RadiusNeighborsClassifier, {}),
+        (neighbors.KNeighborsTransformer, {"n_neighbors": 2}),
+        (neighbors.RadiusNeighborsTransformer, {"radius": 1.5}),
+        (neighbors.LocalOutlierFactor, {"n_neighbors": 1}),
+    ],
+)
+def test_nan_euclidean_support(Estimator, params):
+    """Check that the different neighbor estimators are lenient towards `nan`
+    values if using `metric="nan_euclidean"`.
+    """
+
+    X = [[0, 1], [1, np.nan], [2, 3], [3, 5]]
+    y = [0, 0, 1, 1]
+
+    params.update({"metric": "nan_euclidean"})
+    estimator = Estimator().set_params(**params).fit(X, y)
+
+    for response_method in ("kneighbors", "predict", "transform", "fit_predict"):
+        if hasattr(estimator, response_method):
+            output = getattr(estimator, response_method)(X)
+            if hasattr(output, "toarray"):
+                assert not np.isnan(output.data).any()
+            else:
+                assert not np.isnan(output).any()
+
+
+def test_predict_dataframe():
+    """Check that KNN predict works with dataframes
+
+    non-regression test for issue #26768
+    """
+    pd = pytest.importorskip("pandas")
+
+    X = pd.DataFrame(np.array([[1, 2], [3, 4], [5, 6], [7, 8]]), columns=["a", "b"])
+    y = np.array([1, 2, 3, 4])
+
+    knn = neighbors.KNeighborsClassifier(n_neighbors=2).fit(X, y)
+    knn.predict(X)
+
+
+def test_nearest_neighbours_works_with_p_less_than_1():
+    """Check that NearestNeighbors works with :math:`p \\in (0,1)` when `algorithm`
+    is `"auto"` or `"brute"` regardless of the dtype of X.
+
+    Non-regression test for issue #26548
+    """
+    X = np.array([[1.0, 0.0], [0.0, 0.0], [0.0, 1.0]])
+    neigh = neighbors.NearestNeighbors(
+        n_neighbors=3, algorithm="brute", metric_params={"p": 0.5}
+    )
+    neigh.fit(X)
+
+    y = neigh.radius_neighbors(X[0].reshape(1, -1), radius=4, return_distance=False)
+    assert_allclose(y[0], [0, 1, 2])
+
+    y = neigh.kneighbors(X[0].reshape(1, -1), return_distance=False)
+    assert_allclose(y[0], [0, 1, 2])
+
+
+def test_KNeighborsClassifier_raise_on_all_zero_weights():
+    """Check that `predict` and `predict_proba` raises on sample of all zeros weights.
+
+    Related to Issue #25854.
+    """
+    X = [[0, 1], [1, 2], [2, 3], [3, 4]]
+    y = [0, 0, 1, 1]
+
+    def _weights(dist):
+        return np.vectorize(lambda x: 0 if x > 0.5 else 1)(dist)
+
+    est = neighbors.KNeighborsClassifier(n_neighbors=3, weights=_weights)
+    est.fit(X, y)
+
+    msg = (
+        "All neighbors of some sample is getting zero weights. "
+        "Please modify 'weights' to avoid this case if you are "
+        "using a user-defined function."
+    )
+
+    with pytest.raises(ValueError, match=msg):
+        est.predict([[1.1, 1.1]])
+
+    with pytest.raises(ValueError, match=msg):
+        est.predict_proba([[1.1, 1.1]])
+
+
+@pytest.mark.parametrize(
+    "nn_model",
+    [
+        neighbors.KNeighborsClassifier(n_neighbors=10),
+        neighbors.RadiusNeighborsClassifier(),
+    ],
+)
+@pytest.mark.parametrize("algorithm", ALGORITHMS)
+def test_neighbor_classifiers_loocv(nn_model, algorithm):
+    """Check that `predict` and related functions work fine with X=None
+
+    Calling predict with X=None computes a prediction for each training point
+    from the labels of its neighbors (without the label of the data point being
+    predicted upon). This is therefore mathematically equivalent to
+    leave-one-out cross-validation without having do any retraining (rebuilding
+    a KD-tree or Ball-tree index) or any data reshuffling.
+    """
+    X, y = datasets.make_blobs(n_samples=15, centers=5, n_features=2, random_state=0)
+
+    nn_model = clone(nn_model).set_params(algorithm=algorithm)
+
+    # Set the radius for RadiusNeighborsRegressor to some percentile of the
+    # empirical pairwise distances to avoid trivial test cases and warnings for
+    # predictions with no neighbors within the radius.
+    if "radius" in nn_model.get_params():
+        dists = pairwise_distances(X).ravel()
+        dists = dists[dists > 0]
+        nn_model.set_params(radius=np.percentile(dists, 80))
+
+    loocv = cross_val_score(nn_model, X, y, cv=LeaveOneOut())
+    nn_model.fit(X, y)
+
+    assert_allclose(loocv, nn_model.predict(None) == y)
+    assert np.mean(loocv) == pytest.approx(nn_model.score(None, y))
+
+    # Evaluating `nn_model` on its "training" set should lead to a higher
+    # accuracy value than leaving out each data point in turn because the
+    # former can overfit while the latter cannot by construction.
+    assert nn_model.score(None, y) < nn_model.score(X, y)
+
+
+@pytest.mark.parametrize(
+    "nn_model",
+    [
+        neighbors.KNeighborsRegressor(n_neighbors=10),
+        neighbors.RadiusNeighborsRegressor(),
+    ],
+)
+@pytest.mark.parametrize("algorithm", ALGORITHMS)
+def test_neighbor_regressors_loocv(nn_model, algorithm):
+    """Check that `predict` and related functions work fine with X=None"""
+    X, y = datasets.make_regression(n_samples=15, n_features=2, random_state=0)
+
+    # Only checking cross_val_predict and not cross_val_score because
+    # cross_val_score does not work with LeaveOneOut() for a regressor: the
+    # default score method implements R2 score which is not well defined for a
+    # single data point.
+    #
+    # TODO: if score is refactored to evaluate models for other scoring
+    # functions, then this test can be extended to check cross_val_score as
+    # well.
+    nn_model = clone(nn_model).set_params(algorithm=algorithm)
+
+    # Set the radius for RadiusNeighborsRegressor to some percentile of the
+    # empirical pairwise distances to avoid trivial test cases and warnings for
+    # predictions with no neighbors within the radius.
+    if "radius" in nn_model.get_params():
+        dists = pairwise_distances(X).ravel()
+        dists = dists[dists > 0]
+        nn_model.set_params(radius=np.percentile(dists, 80))
+
+    loocv = cross_val_predict(nn_model, X, y, cv=LeaveOneOut())
+    nn_model.fit(X, y)
+    assert_allclose(loocv, nn_model.predict(None))
diff --git a/sklearn/neighbors/tests/test_neighbors_pipeline.py b/sklearn/neighbors/tests/test_neighbors_pipeline.py
index f8f9472bdac48..6ad78824489ca 100644
--- a/sklearn/neighbors/tests/test_neighbors_pipeline.py
+++ b/sklearn/neighbors/tests/test_neighbors_pipeline.py
@@ -7,23 +7,20 @@
 
 import numpy as np
 
-from sklearn.utils._testing import assert_array_almost_equal
+from sklearn.base import clone
+from sklearn.cluster import DBSCAN, SpectralClustering
 from sklearn.cluster.tests.common import generate_clustered_data
 from sklearn.datasets import make_blobs
+from sklearn.manifold import TSNE, Isomap, SpectralEmbedding
+from sklearn.neighbors import (
+    KNeighborsRegressor,
+    KNeighborsTransformer,
+    LocalOutlierFactor,
+    RadiusNeighborsRegressor,
+    RadiusNeighborsTransformer,
+)
 from sklearn.pipeline import make_pipeline
-from sklearn.base import clone
-
-from sklearn.neighbors import KNeighborsTransformer
-from sklearn.neighbors import RadiusNeighborsTransformer
-
-from sklearn.cluster import DBSCAN
-from sklearn.cluster import SpectralClustering
-from sklearn.neighbors import KNeighborsRegressor
-from sklearn.neighbors import RadiusNeighborsRegressor
-from sklearn.neighbors import LocalOutlierFactor
-from sklearn.manifold import SpectralEmbedding
-from sklearn.manifold import Isomap
-from sklearn.manifold import TSNE
+from sklearn.utils._testing import assert_array_almost_equal
 
 
 def test_spectral_clustering():
@@ -33,11 +30,14 @@ def test_spectral_clustering():
 
     # compare the chained version and the compact version
     est_chain = make_pipeline(
-        KNeighborsTransformer(n_neighbors=n_neighbors, mode='connectivity'),
-        SpectralClustering(n_neighbors=n_neighbors, affinity='precomputed',
-                           random_state=42))
+        KNeighborsTransformer(n_neighbors=n_neighbors, mode="connectivity"),
+        SpectralClustering(
+            n_neighbors=n_neighbors, affinity="precomputed", random_state=42
+        ),
+    )
     est_compact = SpectralClustering(
-        n_neighbors=n_neighbors, affinity='nearest_neighbors', random_state=42)
+        n_neighbors=n_neighbors, affinity="nearest_neighbors", random_state=42
+    )
     labels_compact = est_compact.fit_predict(X)
     labels_chain = est_chain.fit_predict(X)
     assert_array_almost_equal(labels_chain, labels_compact)
@@ -48,21 +48,27 @@ def test_spectral_embedding():
     n_neighbors = 5
 
     n_samples = 1000
-    centers = np.array([
-        [0.0, 5.0, 0.0, 0.0, 0.0],
-        [0.0, 0.0, 4.0, 0.0, 0.0],
-        [1.0, 0.0, 0.0, 5.0, 1.0],
-    ])
-    S, true_labels = make_blobs(n_samples=n_samples, centers=centers,
-                                cluster_std=1., random_state=42)
+    centers = np.array(
+        [
+            [0.0, 5.0, 0.0, 0.0, 0.0],
+            [0.0, 0.0, 4.0, 0.0, 0.0],
+            [1.0, 0.0, 0.0, 5.0, 1.0],
+        ]
+    )
+    S, true_labels = make_blobs(
+        n_samples=n_samples, centers=centers, cluster_std=1.0, random_state=42
+    )
 
     # compare the chained version and the compact version
     est_chain = make_pipeline(
-        KNeighborsTransformer(n_neighbors=n_neighbors, mode='connectivity'),
-        SpectralEmbedding(n_neighbors=n_neighbors, affinity='precomputed',
-                          random_state=42))
+        KNeighborsTransformer(n_neighbors=n_neighbors, mode="connectivity"),
+        SpectralEmbedding(
+            n_neighbors=n_neighbors, affinity="precomputed", random_state=42
+        ),
+    )
     est_compact = SpectralEmbedding(
-        n_neighbors=n_neighbors, affinity='nearest_neighbors', random_state=42)
+        n_neighbors=n_neighbors, affinity="nearest_neighbors", random_state=42
+    )
     St_compact = est_compact.fit_transform(S)
     St_chain = est_chain.fit_transform(S)
     assert_array_almost_equal(St_chain, St_compact)
@@ -76,8 +82,9 @@ def test_dbscan():
 
     # compare the chained version and the compact version
     est_chain = make_pipeline(
-        RadiusNeighborsTransformer(radius=radius, mode='distance'),
-        DBSCAN(metric='precomputed', eps=radius))
+        RadiusNeighborsTransformer(radius=radius, mode="distance"),
+        DBSCAN(metric="precomputed", eps=radius),
+    )
     est_compact = DBSCAN(eps=radius)
 
     labels_chain = est_chain.fit_predict(X)
@@ -88,7 +95,7 @@ def test_dbscan():
 def test_isomap():
     # Test chaining KNeighborsTransformer and Isomap with
     # neighbors_algorithm='precomputed'
-    algorithm = 'auto'
+    algorithm = "auto"
     n_neighbors = 10
 
     X, _ = make_blobs(random_state=0)
@@ -96,11 +103,12 @@ def test_isomap():
 
     # compare the chained version and the compact version
     est_chain = make_pipeline(
-        KNeighborsTransformer(n_neighbors=n_neighbors, algorithm=algorithm,
-                              mode='distance'),
-        Isomap(n_neighbors=n_neighbors, metric='precomputed'))
-    est_compact = Isomap(n_neighbors=n_neighbors,
-                         neighbors_algorithm=algorithm)
+        KNeighborsTransformer(
+            n_neighbors=n_neighbors, algorithm=algorithm, mode="distance"
+        ),
+        Isomap(n_neighbors=n_neighbors, metric="precomputed"),
+    )
+    est_compact = Isomap(n_neighbors=n_neighbors, neighbors_algorithm=algorithm)
 
     Xt_chain = est_chain.fit_transform(X)
     Xt_compact = est_compact.fit_transform(X)
@@ -113,25 +121,36 @@ def test_isomap():
 
 def test_tsne():
     # Test chaining KNeighborsTransformer and TSNE
-    n_iter = 250
+    max_iter = 250
     perplexity = 5
-    n_neighbors = int(3. * perplexity + 1)
+    n_neighbors = int(3.0 * perplexity + 1)
 
     rng = np.random.RandomState(0)
     X = rng.randn(20, 2)
 
-    for metric in ['minkowski', 'sqeuclidean']:
-
+    for metric in ["minkowski", "sqeuclidean"]:
         # compare the chained version and the compact version
         est_chain = make_pipeline(
-            KNeighborsTransformer(n_neighbors=n_neighbors, mode='distance',
-                                  metric=metric),
-            TSNE(metric='precomputed', perplexity=perplexity,
-                 method="barnes_hut", random_state=42, n_iter=n_iter,
-                 square_distances=True))
-        est_compact = TSNE(metric=metric, perplexity=perplexity, n_iter=n_iter,
-                           method="barnes_hut", random_state=42,
-                           square_distances=True)
+            KNeighborsTransformer(
+                n_neighbors=n_neighbors, mode="distance", metric=metric
+            ),
+            TSNE(
+                init="random",
+                metric="precomputed",
+                perplexity=perplexity,
+                method="barnes_hut",
+                random_state=42,
+                max_iter=max_iter,
+            ),
+        )
+        est_compact = TSNE(
+            init="random",
+            metric=metric,
+            perplexity=perplexity,
+            max_iter=max_iter,
+            method="barnes_hut",
+            random_state=42,
+        )
 
         Xt_chain = est_chain.fit_transform(X)
         Xt_compact = est_compact.fit_transform(X)
@@ -147,11 +166,17 @@ def test_lof_novelty_false():
 
     # compare the chained version and the compact version
     est_chain = make_pipeline(
-        KNeighborsTransformer(n_neighbors=n_neighbors, mode='distance'),
-        LocalOutlierFactor(metric='precomputed', n_neighbors=n_neighbors,
-                           novelty=False, contamination="auto"))
-    est_compact = LocalOutlierFactor(n_neighbors=n_neighbors, novelty=False,
-                                     contamination="auto")
+        KNeighborsTransformer(n_neighbors=n_neighbors, mode="distance"),
+        LocalOutlierFactor(
+            metric="precomputed",
+            n_neighbors=n_neighbors,
+            novelty=False,
+            contamination="auto",
+        ),
+    )
+    est_compact = LocalOutlierFactor(
+        n_neighbors=n_neighbors, novelty=False, contamination="auto"
+    )
 
     pred_chain = est_chain.fit_predict(X)
     pred_compact = est_compact.fit_predict(X)
@@ -168,11 +193,17 @@ def test_lof_novelty_true():
 
     # compare the chained version and the compact version
     est_chain = make_pipeline(
-        KNeighborsTransformer(n_neighbors=n_neighbors, mode='distance'),
-        LocalOutlierFactor(metric='precomputed', n_neighbors=n_neighbors,
-                           novelty=True, contamination="auto"))
-    est_compact = LocalOutlierFactor(n_neighbors=n_neighbors, novelty=True,
-                                     contamination="auto")
+        KNeighborsTransformer(n_neighbors=n_neighbors, mode="distance"),
+        LocalOutlierFactor(
+            metric="precomputed",
+            n_neighbors=n_neighbors,
+            novelty=True,
+            contamination="auto",
+        ),
+    )
+    est_compact = LocalOutlierFactor(
+        n_neighbors=n_neighbors, novelty=True, contamination="auto"
+    )
 
     pred_chain = est_chain.fit(X1).predict(X2)
     pred_compact = est_compact.fit(X1).predict(X2)
@@ -192,13 +223,15 @@ def test_kneighbors_regressor():
     # k-neighbors estimator after radius-neighbors transformer, and vice-versa.
     factor = 2
 
-    k_trans = KNeighborsTransformer(n_neighbors=n_neighbors, mode='distance')
-    k_trans_factor = KNeighborsTransformer(n_neighbors=int(
-        n_neighbors * factor), mode='distance')
+    k_trans = KNeighborsTransformer(n_neighbors=n_neighbors, mode="distance")
+    k_trans_factor = KNeighborsTransformer(
+        n_neighbors=int(n_neighbors * factor), mode="distance"
+    )
 
-    r_trans = RadiusNeighborsTransformer(radius=radius, mode='distance')
-    r_trans_factor = RadiusNeighborsTransformer(radius=int(
-        radius * factor), mode='distance')
+    r_trans = RadiusNeighborsTransformer(radius=radius, mode="distance")
+    r_trans_factor = RadiusNeighborsTransformer(
+        radius=int(radius * factor), mode="distance"
+    )
 
     k_reg = KNeighborsRegressor(n_neighbors=n_neighbors)
     r_reg = RadiusNeighborsRegressor(radius=radius)
@@ -214,7 +247,7 @@ def test_kneighbors_regressor():
         # compare the chained version and the compact version
         reg_compact = clone(reg)
         reg_precomp = clone(reg)
-        reg_precomp.set_params(metric='precomputed')
+        reg_precomp.set_params(metric="precomputed")
 
         reg_chain = make_pipeline(clone(trans), reg_precomp)
 
diff --git a/sklearn/neighbors/tests/test_neighbors_tree.py b/sklearn/neighbors/tests/test_neighbors_tree.py
index 6609d9af2656f..de19152e8b7f2 100644
--- a/sklearn/neighbors/tests/test_neighbors_tree.py
+++ b/sklearn/neighbors/tests/test_neighbors_tree.py
@@ -1,24 +1,39 @@
-# License: BSD 3 clause
+# SPDX-License-Identifier: BSD-3-Clause
 
-import pickle
 import itertools
+import pickle
 
 import numpy as np
 import pytest
+from numpy.testing import assert_allclose, assert_array_almost_equal
 
-from sklearn.neighbors import DistanceMetric
+from sklearn.metrics import DistanceMetric
+from sklearn.neighbors._ball_tree import (
+    BallTree,
+    kernel_norm,
+)
+from sklearn.neighbors._ball_tree import (
+    NeighborsHeap64 as NeighborsHeapBT,
+)
+from sklearn.neighbors._ball_tree import (
+    nodeheap_sort as nodeheap_sort_bt,
+)
 from sklearn.neighbors._ball_tree import (
-    BallTree, kernel_norm, DTYPE, ITYPE,
-    NeighborsHeap as NeighborsHeapBT,
     simultaneous_sort as simultaneous_sort_bt,
-    nodeheap_sort as nodeheap_sort_bt)
+)
+from sklearn.neighbors._kd_tree import (
+    KDTree,
+)
+from sklearn.neighbors._kd_tree import (
+    NeighborsHeap64 as NeighborsHeapKDT,
+)
+from sklearn.neighbors._kd_tree import (
+    nodeheap_sort as nodeheap_sort_kdt,
+)
 from sklearn.neighbors._kd_tree import (
-    KDTree, NeighborsHeap as NeighborsHeapKDT,
     simultaneous_sort as simultaneous_sort_kdt,
-    nodeheap_sort as nodeheap_sort_kdt)
-
+)
 from sklearn.utils import check_random_state
-from numpy.testing import assert_array_almost_equal, assert_allclose
 
 rng = np.random.RandomState(42)
 V_mahalanobis = rng.rand(3, 3)
@@ -26,40 +41,41 @@
 
 DIMENSION = 3
 
-METRICS = {'euclidean': {},
-           'manhattan': {},
-           'minkowski': dict(p=3),
-           'chebyshev': {},
-           'seuclidean': dict(V=rng.random_sample(DIMENSION)),
-           'wminkowski': dict(p=3, w=rng.random_sample(DIMENSION)),
-           'mahalanobis': dict(V=V_mahalanobis)}
+METRICS = {
+    "euclidean": {},
+    "manhattan": {},
+    "minkowski": dict(p=3),
+    "chebyshev": {},
+    "seuclidean": dict(V=rng.random_sample(DIMENSION)),
+    "mahalanobis": dict(V=V_mahalanobis),
+}
 
-KD_TREE_METRICS = ['euclidean', 'manhattan', 'chebyshev', 'minkowski']
+KD_TREE_METRICS = ["euclidean", "manhattan", "chebyshev", "minkowski"]
 BALL_TREE_METRICS = list(METRICS)
 
 
 def dist_func(x1, x2, p):
-    return np.sum((x1 - x2) ** p) ** (1. / p)
+    return np.sum((x1 - x2) ** p) ** (1.0 / p)
 
 
 def compute_kernel_slow(Y, X, kernel, h):
     d = np.sqrt(((Y[:, None, :] - X) ** 2).sum(-1))
     norm = kernel_norm(h, X.shape[1], kernel)
 
-    if kernel == 'gaussian':
+    if kernel == "gaussian":
         return norm * np.exp(-0.5 * (d * d) / (h * h)).sum(-1)
-    elif kernel == 'tophat':
+    elif kernel == "tophat":
         return norm * (d < h).sum(-1)
-    elif kernel == 'epanechnikov':
+    elif kernel == "epanechnikov":
         return norm * ((1.0 - (d * d) / (h * h)) * (d < h)).sum(-1)
-    elif kernel == 'exponential':
+    elif kernel == "exponential":
         return norm * (np.exp(-d / h)).sum(-1)
-    elif kernel == 'linear':
+    elif kernel == "linear":
         return norm * ((1 - d / h) * (d < h)).sum(-1)
-    elif kernel == 'cosine':
+    elif kernel == "cosine":
         return norm * (np.cos(0.5 * np.pi * d / h) * (d < h)).sum(-1)
     else:
-        raise ValueError('kernel not recognized')
+        raise ValueError("kernel not recognized")
 
 
 def brute_force_neighbors(X, Y, k, metric, **kwargs):
@@ -69,35 +85,36 @@ def brute_force_neighbors(X, Y, k, metric, **kwargs):
     return dist, ind
 
 
-@pytest.mark.parametrize('Cls', [KDTree, BallTree])
-@pytest.mark.parametrize("kernel", ['gaussian', 'tophat', 'epanechnikov',
-                                    'exponential', 'linear', 'cosine'])
+@pytest.mark.parametrize("Cls", [KDTree, BallTree])
+@pytest.mark.parametrize(
+    "kernel", ["gaussian", "tophat", "epanechnikov", "exponential", "linear", "cosine"]
+)
 @pytest.mark.parametrize("h", [0.01, 0.1, 1])
-@pytest.mark.parametrize("rtol", [0, 1E-5])
-@pytest.mark.parametrize("atol", [1E-6, 1E-2])
+@pytest.mark.parametrize("rtol", [0, 1e-5])
+@pytest.mark.parametrize("atol", [1e-6, 1e-2])
 @pytest.mark.parametrize("breadth_first", [True, False])
-def test_kernel_density(Cls, kernel, h, rtol, atol, breadth_first,
-                        n_samples=100, n_features=3):
+def test_kernel_density(
+    Cls, kernel, h, rtol, atol, breadth_first, n_samples=100, n_features=3
+):
     rng = check_random_state(1)
     X = rng.random_sample((n_samples, n_features))
     Y = rng.random_sample((n_samples, n_features))
     dens_true = compute_kernel_slow(Y, X, kernel, h)
 
     tree = Cls(X, leaf_size=10)
-    dens = tree.kernel_density(Y, h, atol=atol, rtol=rtol,
-                               kernel=kernel,
-                               breadth_first=breadth_first)
-    assert_allclose(dens, dens_true,
-                    atol=atol, rtol=max(rtol, 1e-7))
+    dens = tree.kernel_density(
+        Y, h, atol=atol, rtol=rtol, kernel=kernel, breadth_first=breadth_first
+    )
+    assert_allclose(dens, dens_true, atol=atol, rtol=max(rtol, 1e-7))
 
 
-@pytest.mark.parametrize('Cls', [KDTree, BallTree])
+@pytest.mark.parametrize("Cls", [KDTree, BallTree])
 def test_neighbor_tree_query_radius(Cls, n_samples=100, n_features=10):
     rng = check_random_state(0)
     X = 2 * rng.random_sample(size=(n_samples, n_features)) - 1
     query_pt = np.zeros(n_features, dtype=float)
 
-    eps = 1E-15  # roundoff error can cause test to fail
+    eps = 1e-15  # roundoff error can cause test to fail
     tree = Cls(X, leaf_size=5)
     rad = np.sqrt(((X - query_pt) ** 2).sum(1))
 
@@ -111,20 +128,18 @@ def test_neighbor_tree_query_radius(Cls, n_samples=100, n_features=10):
         assert_array_almost_equal(i, ind)
 
 
-@pytest.mark.parametrize('Cls', [KDTree, BallTree])
-def test_neighbor_tree_query_radius_distance(Cls, n_samples=100,
-                                             n_features=10):
+@pytest.mark.parametrize("Cls", [KDTree, BallTree])
+def test_neighbor_tree_query_radius_distance(Cls, n_samples=100, n_features=10):
     rng = check_random_state(0)
     X = 2 * rng.random_sample(size=(n_samples, n_features)) - 1
     query_pt = np.zeros(n_features, dtype=float)
 
-    eps = 1E-15  # roundoff error can cause test to fail
+    eps = 1e-15  # roundoff error can cause test to fail
     tree = Cls(X, leaf_size=5)
     rad = np.sqrt(((X - query_pt) ** 2).sum(1))
 
     for r in np.linspace(rad[0], rad[-1], 100):
-        ind, dist = tree.query_radius([query_pt], r + eps,
-                                      return_distance=True)
+        ind, dist = tree.query_radius([query_pt], r + eps, return_distance=True)
 
         ind = ind[0]
         dist = dist[0]
@@ -134,8 +149,8 @@ def test_neighbor_tree_query_radius_distance(Cls, n_samples=100,
         assert_array_almost_equal(d, dist)
 
 
-@pytest.mark.parametrize('Cls', [KDTree, BallTree])
-@pytest.mark.parametrize('dualtree', (True, False))
+@pytest.mark.parametrize("Cls", [KDTree, BallTree])
+@pytest.mark.parametrize("dualtree", (True, False))
 def test_neighbor_tree_two_point(Cls, dualtree, n_samples=100, n_features=3):
     rng = check_random_state(0)
     X = rng.random_sample((n_samples, n_features))
@@ -150,14 +165,14 @@ def test_neighbor_tree_two_point(Cls, dualtree, n_samples=100, n_features=3):
     assert_array_almost_equal(counts, counts_true)
 
 
-@pytest.mark.parametrize('NeighborsHeap', [NeighborsHeapBT, NeighborsHeapKDT])
+@pytest.mark.parametrize("NeighborsHeap", [NeighborsHeapBT, NeighborsHeapKDT])
 def test_neighbors_heap(NeighborsHeap, n_pts=5, n_nbrs=10):
     heap = NeighborsHeap(n_pts, n_nbrs)
     rng = check_random_state(0)
 
     for row in range(n_pts):
-        d_in = rng.random_sample(2 * n_nbrs).astype(DTYPE, copy=False)
-        i_in = np.arange(2 * n_nbrs, dtype=ITYPE)
+        d_in = rng.random_sample(2 * n_nbrs).astype(np.float64, copy=False)
+        i_in = np.arange(2 * n_nbrs, dtype=np.intp)
         for d, i in zip(d_in, i_in):
             heap.push(row, d, i)
 
@@ -171,11 +186,10 @@ def test_neighbors_heap(NeighborsHeap, n_pts=5, n_nbrs=10):
         assert_array_almost_equal(i_in[:n_nbrs], i_heap[row])
 
 
-@pytest.mark.parametrize('nodeheap_sort', [nodeheap_sort_bt,
-                                           nodeheap_sort_kdt])
+@pytest.mark.parametrize("nodeheap_sort", [nodeheap_sort_bt, nodeheap_sort_kdt])
 def test_node_heap(nodeheap_sort, n_nodes=50):
     rng = check_random_state(0)
-    vals = rng.random_sample(n_nodes).astype(DTYPE, copy=False)
+    vals = rng.random_sample(n_nodes).astype(np.float64, copy=False)
 
     i1 = np.argsort(vals)
     vals2, i2 = nodeheap_sort(vals)
@@ -184,12 +198,13 @@ def test_node_heap(nodeheap_sort, n_nodes=50):
     assert_array_almost_equal(vals[i1], vals2)
 
 
-@pytest.mark.parametrize('simultaneous_sort', [simultaneous_sort_bt,
-                                               simultaneous_sort_kdt])
+@pytest.mark.parametrize(
+    "simultaneous_sort", [simultaneous_sort_bt, simultaneous_sort_kdt]
+)
 def test_simultaneous_sort(simultaneous_sort, n_rows=10, n_pts=201):
     rng = check_random_state(0)
-    dist = rng.random_sample((n_rows, n_pts)).astype(DTYPE, copy=False)
-    ind = (np.arange(n_pts) + np.zeros((n_rows, 1))).astype(ITYPE, copy=False)
+    dist = rng.random_sample((n_rows, n_pts)).astype(np.float64, copy=False)
+    ind = (np.arange(n_pts) + np.zeros((n_rows, 1))).astype(np.intp, copy=False)
 
     dist2 = dist.copy()
     ind2 = ind.copy()
@@ -207,10 +222,11 @@ def test_simultaneous_sort(simultaneous_sort, n_rows=10, n_pts=201):
     assert_array_almost_equal(ind, ind2)
 
 
-@pytest.mark.parametrize('Cls', [KDTree, BallTree])
+@pytest.mark.parametrize("Cls", [KDTree, BallTree])
 def test_gaussian_kde(Cls, n_samples=1000):
     # Compare gaussian KDE results to scipy.stats.gaussian_kde
     from scipy.stats import gaussian_kde
+
     rng = check_random_state(0)
     x_in = rng.normal(0, 1, n_samples)
     x_out = np.linspace(-5, 5, 30)
@@ -226,13 +242,15 @@ def test_gaussian_kde(Cls, n_samples=1000):
 
 
 @pytest.mark.parametrize(
-        'Cls, metric',
-        itertools.chain(
-            [(KDTree, metric) for metric in KD_TREE_METRICS],
-            [(BallTree, metric) for metric in BALL_TREE_METRICS]))
-@pytest.mark.parametrize('k', (1, 3, 5))
-@pytest.mark.parametrize('dualtree', (True, False))
-@pytest.mark.parametrize('breadth_first', (True, False))
+    "Cls, metric",
+    itertools.chain(
+        [(KDTree, metric) for metric in KD_TREE_METRICS],
+        [(BallTree, metric) for metric in BALL_TREE_METRICS],
+    ),
+)
+@pytest.mark.parametrize("k", (1, 3, 5))
+@pytest.mark.parametrize("dualtree", (True, False))
+@pytest.mark.parametrize("breadth_first", (True, False))
 def test_nn_tree_query(Cls, metric, k, dualtree, breadth_first):
     rng = check_random_state(0)
     X = rng.random_sample((40, DIMENSION))
@@ -241,8 +259,7 @@ def test_nn_tree_query(Cls, metric, k, dualtree, breadth_first):
     kwargs = METRICS[metric]
 
     kdt = Cls(X, leaf_size=1, metric=metric, **kwargs)
-    dist1, ind1 = kdt.query(Y, k, dualtree=dualtree,
-                            breadth_first=breadth_first)
+    dist1, ind1 = kdt.query(Y, k, dualtree=dualtree, breadth_first=breadth_first)
     dist2, ind2 = brute_force_neighbors(X, Y, k, metric, **kwargs)
 
     # don't check indices here: if there are any duplicate distances,
@@ -251,16 +268,16 @@ def test_nn_tree_query(Cls, metric, k, dualtree, breadth_first):
 
 
 @pytest.mark.parametrize(
-        "Cls, metric",
-        [(KDTree, 'euclidean'), (BallTree, 'euclidean'),
-         (BallTree, dist_func)])
-@pytest.mark.parametrize('protocol', (0, 1, 2))
+    "Cls, metric",
+    [(KDTree, "euclidean"), (BallTree, "euclidean"), (BallTree, dist_func)],
+)
+@pytest.mark.parametrize("protocol", (0, 1, 2))
 def test_pickle(Cls, metric, protocol):
     rng = check_random_state(0)
     X = rng.random_sample((10, 3))
 
-    if hasattr(metric, '__call__'):
-        kwargs = {'p': 2}
+    if hasattr(metric, "__call__"):
+        kwargs = {"p": 2}
     else:
         kwargs = {}
 
diff --git a/sklearn/neighbors/tests/test_quad_tree.py b/sklearn/neighbors/tests/test_quad_tree.py
index abdb2f118a928..be9a4c5fe549d 100644
--- a/sklearn/neighbors/tests/test_quad_tree.py
+++ b/sklearn/neighbors/tests/test_quad_tree.py
@@ -1,6 +1,6 @@
 import pickle
-import numpy as np
 
+import numpy as np
 import pytest
 
 from sklearn.neighbors._quad_tree import _QuadTree
@@ -42,18 +42,17 @@ def test_quadtree_similar_point():
     # check the case where points are arbitrarily close on Y axis
     Xs.append(np.array([[1.0, 2.00001], [3.0, 2.00002]], dtype=np.float32))
     # check the case where points are arbitrarily close on both axes
-    Xs.append(np.array([[1.00001, 2.00001], [1.00002, 2.00002]],
-              dtype=np.float32))
+    Xs.append(np.array([[1.00001, 2.00001], [1.00002, 2.00002]], dtype=np.float32))
 
     # check the case where points are arbitrarily close on both axes
     # close to machine epsilon - x axis
-    Xs.append(np.array([[1, 0.0003817754041], [2, 0.0003817753750]],
-              dtype=np.float32))
+    Xs.append(np.array([[1, 0.0003817754041], [2, 0.0003817753750]], dtype=np.float32))
 
     # check the case where points are arbitrarily close on both axes
     # close to machine epsilon - y axis
-    Xs.append(np.array([[0.0003817754041, 1.0], [0.0003817753750, 2.0]],
-              dtype=np.float32))
+    Xs.append(
+        np.array([[0.0003817754041, 1.0], [0.0003817753750, 2.0]], dtype=np.float32)
+    )
 
     for X in Xs:
         tree = _QuadTree(n_dimensions=2, verbose=0)
@@ -61,8 +60,8 @@ def test_quadtree_similar_point():
         tree._check_coherence()
 
 
-@pytest.mark.parametrize('n_dimensions', (2, 3))
-@pytest.mark.parametrize('protocol', (0, 1, 2))
+@pytest.mark.parametrize("n_dimensions", (2, 3))
+@pytest.mark.parametrize("protocol", (0, 1, 2))
 def test_quad_tree_pickle(n_dimensions, protocol):
     rng = check_random_state(0)
 
@@ -80,7 +79,7 @@ def test_quad_tree_pickle(n_dimensions, protocol):
         assert cell_x_tree == cell_x_bt2
 
 
-@pytest.mark.parametrize('n_dimensions', (2, 3))
+@pytest.mark.parametrize("n_dimensions", (2, 3))
 def test_qt_insert_duplicate(n_dimensions):
     rng = check_random_state(0)
 
@@ -104,8 +103,9 @@ def test_summarize():
     # Simple check for quad tree's summarize
 
     angle = 0.9
-    X = np.array([[-10., -10.], [9., 10.], [10., 9.], [10., 10.]],
-                 dtype=np.float32)
+    X = np.array(
+        [[-10.0, -10.0], [9.0, 10.0], [10.0, 9.0], [10.0, 10.0]], dtype=np.float32
+    )
     query_pt = X[0, :]
     n_dimensions = X.shape[1]
     offset = n_dimensions + 2
@@ -129,7 +129,7 @@ def test_summarize():
 
     # Summary should contain all 3 node with size 1 and distance to
     # each point in X[1:] for ``angle=0``
-    idx, summary = qt._py_summarize(query_pt, X, 0.)
+    idx, summary = qt._py_summarize(query_pt, X, 0.0)
     barycenter = X[1:].mean(axis=0)
     ds2c = ((X[0] - barycenter) ** 2).sum()
 
diff --git a/sklearn/neural_network/__init__.py b/sklearn/neural_network/__init__.py
index 722b1453e08ec..fa5980ce24f5c 100644
--- a/sklearn/neural_network/__init__.py
+++ b/sklearn/neural_network/__init__.py
@@ -1,15 +1,9 @@
-"""
-The :mod:`sklearn.neural_network` module includes models based on neural
-networks.
-"""
+"""Models based on neural networks."""
 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
+from ._multilayer_perceptron import MLPClassifier, MLPRegressor
 from ._rbm import BernoulliRBM
 
-from ._multilayer_perceptron import MLPClassifier
-from ._multilayer_perceptron import MLPRegressor
-
-__all__ = ["BernoulliRBM",
-           "MLPClassifier",
-           "MLPRegressor"]
+__all__ = ["BernoulliRBM", "MLPClassifier", "MLPRegressor"]
diff --git a/sklearn/neural_network/_base.py b/sklearn/neural_network/_base.py
index 6afe8a23db446..25f0b0a18512b 100644
--- a/sklearn/neural_network/_base.py
+++ b/sklearn/neural_network/_base.py
@@ -1,11 +1,9 @@
-"""Utilities for the neural network modules
-"""
+"""Utilities for the neural network modules"""
 
-# Author: Issam H. Laradji 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import numpy as np
-
 from scipy.special import expit as logistic_sigmoid
 from scipy.special import xlogy
 
@@ -16,12 +14,23 @@ def inplace_identity(X):
     Parameters
     ----------
     X : {array-like, sparse matrix}, shape (n_samples, n_features)
-        Data, where n_samples is the number of samples
-        and n_features is the number of features.
+        Data, where `n_samples` is the number of samples
+        and `n_features` is the number of features.
     """
     # Nothing to do
 
 
+def inplace_exp(X):
+    """Compute the exponential inplace.
+
+    Parameters
+    ----------
+    X : {array-like, sparse matrix}, shape (n_samples, n_features)
+        The input data.
+    """
+    np.exp(X, out=X)
+
+
 def inplace_logistic(X):
     """Compute the logistic function inplace.
 
@@ -68,11 +77,14 @@ def inplace_softmax(X):
     X /= X.sum(axis=1)[:, np.newaxis]
 
 
-ACTIVATIONS = {'identity': inplace_identity,
-               'tanh': inplace_tanh,
-               'logistic': inplace_logistic,
-               'relu': inplace_relu,
-               'softmax': inplace_softmax}
+ACTIVATIONS = {
+    "identity": inplace_identity,
+    "exp": inplace_exp,
+    "tanh": inplace_tanh,
+    "logistic": inplace_logistic,
+    "relu": inplace_relu,
+    "softmax": inplace_softmax,
+}
 
 
 def inplace_identity_derivative(Z, delta):
@@ -106,7 +118,7 @@ def inplace_logistic_derivative(Z, delta):
          The backpropagated error signal to be modified inplace.
     """
     delta *= Z
-    delta *= (1 - Z)
+    delta *= 1 - Z
 
 
 def inplace_tanh_derivative(Z, delta):
@@ -124,7 +136,7 @@ def inplace_tanh_derivative(Z, delta):
     delta : {array-like}, shape (n_samples, n_features)
          The backpropagated error signal to be modified inplace.
     """
-    delta *= (1 - Z ** 2)
+    delta *= 1 - Z**2
 
 
 def inplace_relu_derivative(Z, delta):
@@ -145,13 +157,15 @@ def inplace_relu_derivative(Z, delta):
     delta[Z == 0] = 0
 
 
-DERIVATIVES = {'identity': inplace_identity_derivative,
-               'tanh': inplace_tanh_derivative,
-               'logistic': inplace_logistic_derivative,
-               'relu': inplace_relu_derivative}
+DERIVATIVES = {
+    "identity": inplace_identity_derivative,
+    "tanh": inplace_tanh_derivative,
+    "logistic": inplace_logistic_derivative,
+    "relu": inplace_relu_derivative,
+}
 
 
-def squared_loss(y_true, y_pred):
+def squared_loss(y_true, y_pred, sample_weight=None):
     """Compute the squared loss for regression.
 
     Parameters
@@ -162,15 +176,47 @@ def squared_loss(y_true, y_pred):
     y_pred : array-like or label indicator matrix
         Predicted values, as returned by a regression estimator.
 
+    sample_weight : array-like of shape (n_samples,), default=None
+        Sample weights.
+
     Returns
     -------
     loss : float
         The degree to which the samples are correctly predicted.
     """
-    return ((y_true - y_pred) ** 2).mean() / 2
+    return (
+        0.5 * np.average((y_true - y_pred) ** 2, weights=sample_weight, axis=0).mean()
+    )
+
+
+def poisson_loss(y_true, y_pred, sample_weight=None):
+    """Compute (half of the) Poisson deviance loss for regression.
 
+    Parameters
+    ----------
+    y_true : array-like or label indicator matrix
+        Ground truth (correct) labels.
 
-def log_loss(y_true, y_prob):
+    y_pred : array-like or label indicator matrix
+        Predicted values, as returned by a regression estimator.
+
+    sample_weight : array-like of shape (n_samples,), default=None
+        Sample weights.
+
+    Returns
+    -------
+    loss : float
+        The degree to which the samples are correctly predicted.
+    """
+    # TODO: Decide what to do with the term `xlogy(y_true, y_true) - y_true`. For now,
+    # it is included. But the _loss module doesn't use it (for performance reasons) and
+    # only adds it as return of constant_to_optimal_zero (mainly for testing).
+    return np.average(
+        xlogy(y_true, y_true / y_pred) - y_true + y_pred, weights=sample_weight, axis=0
+    ).sum()
+
+
+def log_loss(y_true, y_prob, sample_weight=None):
     """Compute Logistic loss for classification.
 
     Parameters
@@ -182,6 +228,9 @@ def log_loss(y_true, y_prob):
         Predicted probabilities, as returned by a classifier's
         predict_proba method.
 
+    sample_weight : array-like of shape (n_samples,), default=None
+        Sample weights.
+
     Returns
     -------
     loss : float
@@ -195,10 +244,10 @@ def log_loss(y_true, y_prob):
     if y_true.shape[1] == 1:
         y_true = np.append(1 - y_true, y_true, axis=1)
 
-    return - xlogy(y_true, y_prob).sum() / y_prob.shape[0]
+    return -np.average(xlogy(y_true, y_prob), weights=sample_weight, axis=0).sum()
 
 
-def binary_log_loss(y_true, y_prob):
+def binary_log_loss(y_true, y_prob, sample_weight=None):
     """Compute binary logistic loss for classification.
 
     This is identical to log_loss in binary classification case,
@@ -213,6 +262,9 @@ def binary_log_loss(y_true, y_prob):
         Predicted probabilities, as returned by a classifier's
         predict_proba method.
 
+    sample_weight : array-like of shape (n_samples,), default=None
+        Sample weights.
+
     Returns
     -------
     loss : float
@@ -220,9 +272,16 @@ def binary_log_loss(y_true, y_prob):
     """
     eps = np.finfo(y_prob.dtype).eps
     y_prob = np.clip(y_prob, eps, 1 - eps)
-    return -(xlogy(y_true, y_prob).sum() +
-             xlogy(1 - y_true, 1 - y_prob).sum()) / y_prob.shape[0]
-
-
-LOSS_FUNCTIONS = {'squared_loss': squared_loss, 'log_loss': log_loss,
-                  'binary_log_loss': binary_log_loss}
+    return -np.average(
+        xlogy(y_true, y_prob) + xlogy(1 - y_true, 1 - y_prob),
+        weights=sample_weight,
+        axis=0,
+    ).sum()
+
+
+LOSS_FUNCTIONS = {
+    "squared_error": squared_loss,
+    "poisson": poisson_loss,
+    "log_loss": log_loss,
+    "binary_log_loss": binary_log_loss,
+}
diff --git a/sklearn/neural_network/_multilayer_perceptron.py b/sklearn/neural_network/_multilayer_perceptron.py
index ae06502d3ce1a..a8a00fe3b4ac5 100644
--- a/sklearn/neural_network/_multilayer_perceptron.py
+++ b/sklearn/neural_network/_multilayer_perceptron.py
@@ -1,37 +1,48 @@
-"""Multi-layer Perceptron
-"""
+"""Multi-layer Perceptron"""
 
-# Authors: Issam H. Laradji 
-#          Andreas Mueller
-#          Jiyuan Qian
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-import numpy as np
-
-from abc import ABCMeta, abstractmethod
 import warnings
+from abc import ABC, abstractmethod
+from itertools import chain, pairwise
+from numbers import Integral, Real
 
+import numpy as np
 import scipy.optimize
 
-from ..base import BaseEstimator, ClassifierMixin, RegressorMixin
-from ..base import is_classifier
-from ._base import ACTIVATIONS, DERIVATIVES, LOSS_FUNCTIONS
-from ._stochastic_optimizers import SGDOptimizer, AdamOptimizer
+from ..base import (
+    BaseEstimator,
+    ClassifierMixin,
+    RegressorMixin,
+    _fit_context,
+    is_classifier,
+)
+from ..exceptions import ConvergenceWarning
+from ..metrics import accuracy_score, r2_score
 from ..model_selection import train_test_split
 from ..preprocessing import LabelBinarizer
-from ..utils import gen_batches, check_random_state
-from ..utils import shuffle
-from ..utils import _safe_indexing
-from ..utils import column_or_1d
-from ..exceptions import ConvergenceWarning
+from ..utils import (
+    _safe_indexing,
+    check_random_state,
+    column_or_1d,
+    gen_batches,
+    shuffle,
+)
+from ..utils._param_validation import Interval, Options, StrOptions
 from ..utils.extmath import safe_sparse_dot
-from ..utils.validation import check_is_fitted, _deprecate_positional_args
-from ..utils.multiclass import _check_partial_fit_first_call, unique_labels
-from ..utils.multiclass import type_of_target
+from ..utils.metaestimators import available_if
+from ..utils.multiclass import (
+    _check_partial_fit_first_call,
+    type_of_target,
+    unique_labels,
+)
 from ..utils.optimize import _check_optimize_result
+from ..utils.validation import _check_sample_weight, check_is_fitted, validate_data
+from ._base import ACTIVATIONS, DERIVATIVES, LOSS_FUNCTIONS
+from ._stochastic_optimizers import AdamOptimizer, SGDOptimizer
 
-
-_STOCHASTIC_SOLVERS = ['sgd', 'adam']
+_STOCHASTIC_SOLVERS = ["sgd", "adam"]
 
 
 def _pack(coefs_, intercepts_):
@@ -39,7 +50,7 @@ def _pack(coefs_, intercepts_):
     return np.hstack([l.ravel() for l in coefs_ + intercepts_])
 
 
-class BaseMultilayerPerceptron(BaseEstimator, metaclass=ABCMeta):
+class BaseMultilayerPerceptron(BaseEstimator, ABC):
     """Base class for MLP classification and regression.
 
     Warning: This class should not be used directly.
@@ -48,13 +59,69 @@ class BaseMultilayerPerceptron(BaseEstimator, metaclass=ABCMeta):
     .. versionadded:: 0.18
     """
 
+    _parameter_constraints: dict = {
+        "hidden_layer_sizes": [
+            "array-like",
+            Interval(Integral, 1, None, closed="left"),
+        ],
+        "activation": [StrOptions({"identity", "logistic", "tanh", "relu"})],
+        "solver": [StrOptions({"lbfgs", "sgd", "adam"})],
+        "alpha": [Interval(Real, 0, None, closed="left")],
+        "batch_size": [
+            StrOptions({"auto"}),
+            Interval(Integral, 1, None, closed="left"),
+        ],
+        "learning_rate": [StrOptions({"constant", "invscaling", "adaptive"})],
+        "learning_rate_init": [Interval(Real, 0, None, closed="neither")],
+        "power_t": [Interval(Real, 0, None, closed="left")],
+        "max_iter": [Interval(Integral, 1, None, closed="left")],
+        "shuffle": ["boolean"],
+        "random_state": ["random_state"],
+        "tol": [Interval(Real, 0, None, closed="left")],
+        "verbose": ["verbose"],
+        "warm_start": ["boolean"],
+        "momentum": [Interval(Real, 0, 1, closed="both")],
+        "nesterovs_momentum": ["boolean"],
+        "early_stopping": ["boolean"],
+        "validation_fraction": [Interval(Real, 0, 1, closed="left")],
+        "beta_1": [Interval(Real, 0, 1, closed="left")],
+        "beta_2": [Interval(Real, 0, 1, closed="left")],
+        "epsilon": [Interval(Real, 0, None, closed="neither")],
+        "n_iter_no_change": [
+            Interval(Integral, 1, None, closed="left"),
+            Options(Real, {np.inf}),
+        ],
+        "max_fun": [Interval(Integral, 1, None, closed="left")],
+    }
+
     @abstractmethod
-    def __init__(self, hidden_layer_sizes, activation, solver,
-                 alpha, batch_size, learning_rate, learning_rate_init, power_t,
-                 max_iter, loss, shuffle, random_state, tol, verbose,
-                 warm_start, momentum, nesterovs_momentum, early_stopping,
-                 validation_fraction, beta_1, beta_2, epsilon,
-                 n_iter_no_change, max_fun):
+    def __init__(
+        self,
+        hidden_layer_sizes,
+        activation,
+        solver,
+        alpha,
+        batch_size,
+        learning_rate,
+        learning_rate_init,
+        power_t,
+        max_iter,
+        loss,
+        shuffle,
+        random_state,
+        tol,
+        verbose,
+        warm_start,
+        momentum,
+        nesterovs_momentum,
+        early_stopping,
+        validation_fraction,
+        beta_1,
+        beta_2,
+        epsilon,
+        n_iter_no_change,
+        max_fun,
+    ):
         self.activation = activation
         self.solver = solver
         self.alpha = alpha
@@ -101,8 +168,7 @@ def _forward_pass(self, activations):
         hidden_activation = ACTIVATIONS[self.activation]
         # Iterate over the hidden layers
         for i in range(self.n_layers_ - 1):
-            activations[i + 1] = safe_sparse_dot(activations[i],
-                                                 self.coefs_[i])
+            activations[i + 1] = safe_sparse_dot(activations[i], self.coefs_[i])
             activations[i + 1] += self.intercepts_[i]
 
             # For the hidden layers
@@ -115,7 +181,7 @@ def _forward_pass(self, activations):
 
         return activations
 
-    def _forward_pass_fast(self, X):
+    def _forward_pass_fast(self, X, check_input=True):
         """Predict using the trained model
 
         This is the same as _forward_pass but does not record the activations
@@ -126,12 +192,16 @@ def _forward_pass_fast(self, X):
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
             The input data.
 
+        check_input : bool, default=True
+            Perform input data validation or not.
+
         Returns
         -------
         y_pred : ndarray of shape (n_samples,) or (n_samples, n_outputs)
             The decision function of the samples for each class in the model.
         """
-        X = self._validate_data(X, accept_sparse=['csr', 'csc'], reset=False)
+        if check_input:
+            X = validate_data(self, X, accept_sparse=["csr", "csc"], reset=False)
 
         # Initialize first layer
         activation = X
@@ -148,22 +218,31 @@ def _forward_pass_fast(self, X):
 
         return activation
 
-    def _compute_loss_grad(self, layer, n_samples, activations, deltas,
-                           coef_grads, intercept_grads):
+    def _compute_loss_grad(
+        self, layer, sw_sum, activations, deltas, coef_grads, intercept_grads
+    ):
         """Compute the gradient of loss with respect to coefs and intercept for
         specified layer.
 
         This function does backpropagation for the specified one layer.
         """
-        coef_grads[layer] = safe_sparse_dot(activations[layer].T,
-                                            deltas[layer])
-        coef_grads[layer] += (self.alpha * self.coefs_[layer])
-        coef_grads[layer] /= n_samples
-
-        intercept_grads[layer] = np.mean(deltas[layer], 0)
-
-    def _loss_grad_lbfgs(self, packed_coef_inter, X, y, activations, deltas,
-                         coef_grads, intercept_grads):
+        coef_grads[layer] = safe_sparse_dot(activations[layer].T, deltas[layer])
+        coef_grads[layer] += self.alpha * self.coefs_[layer]
+        coef_grads[layer] /= sw_sum
+
+        intercept_grads[layer] = np.sum(deltas[layer], axis=0) / sw_sum
+
+    def _loss_grad_lbfgs(
+        self,
+        packed_coef_inter,
+        X,
+        y,
+        sample_weight,
+        activations,
+        deltas,
+        coef_grads,
+        intercept_grads,
+    ):
         """Compute the MLP loss function and its corresponding derivatives
         with respect to the different parameters given in the initialization.
 
@@ -181,6 +260,9 @@ def _loss_grad_lbfgs(self, packed_coef_inter, X, y, activations, deltas,
         y : ndarray of shape (n_samples,)
             The target values.
 
+        sample_weight : array-like of shape (n_samples,), default=None
+            Sample weights.
+
         activations : list, length = n_layers - 1
             The ith element of the list holds the values of the ith layer.
 
@@ -206,12 +288,14 @@ def _loss_grad_lbfgs(self, packed_coef_inter, X, y, activations, deltas,
         """
         self._unpack(packed_coef_inter)
         loss, coef_grads, intercept_grads = self._backprop(
-            X, y, activations, deltas, coef_grads, intercept_grads)
+            X, y, sample_weight, activations, deltas, coef_grads, intercept_grads
+        )
         grad = _pack(coef_grads, intercept_grads)
         return loss, grad
 
-    def _backprop(self, X, y, activations, deltas, coef_grads,
-                  intercept_grads):
+    def _backprop(
+        self, X, y, sample_weight, activations, deltas, coef_grads, intercept_grads
+    ):
         """Compute the MLP loss function and its corresponding derivatives
         with respect to each parameter: weights and bias vectors.
 
@@ -223,6 +307,9 @@ def _backprop(self, X, y, activations, deltas, coef_grads,
         y : ndarray of shape (n_samples,)
             The target values.
 
+        sample_weight : array-like of shape (n_samples,), default=None
+            Sample weights.
+
         activations : list, length = n_layers - 1
              The ith element of the list holds the values of the ith layer.
 
@@ -254,43 +341,54 @@ def _backprop(self, X, y, activations, deltas, coef_grads,
 
         # Get loss
         loss_func_name = self.loss
-        if loss_func_name == 'log_loss' and self.out_activation_ == 'logistic':
-            loss_func_name = 'binary_log_loss'
-        loss = LOSS_FUNCTIONS[loss_func_name](y, activations[-1])
+        if loss_func_name == "log_loss" and self.out_activation_ == "logistic":
+            loss_func_name = "binary_log_loss"
+        loss = LOSS_FUNCTIONS[loss_func_name](y, activations[-1], sample_weight)
         # Add L2 regularization term to loss
         values = 0
         for s in self.coefs_:
             s = s.ravel()
             values += np.dot(s, s)
-        loss += (0.5 * self.alpha) * values / n_samples
+        if sample_weight is None:
+            sw_sum = n_samples
+        else:
+            sw_sum = sample_weight.sum()
+        loss += (0.5 * self.alpha) * values / sw_sum
 
         # Backward propagate
         last = self.n_layers_ - 2
 
-        # The calculation of delta[last] here works with following
-        # combinations of output activation and loss function:
+        # The calculation of delta[last] is as follows:
+        #   delta[last] = d/dz loss(y, act(z)) = act(z) - y
+        # with z=x@w + b being the output of the last layer before passing through the
+        # output activation, act(z) = activations[-1].
+        # The simple formula for delta[last] here works with following (canonical
+        # loss-link) combinations of output activation and loss function:
         # sigmoid and binary cross entropy, softmax and categorical cross
         # entropy, and identity with squared loss
         deltas[last] = activations[-1] - y
+        if sample_weight is not None:
+            deltas[last] *= sample_weight.reshape(-1, 1)
 
         # Compute gradient for the last layer
         self._compute_loss_grad(
-            last, n_samples, activations, deltas, coef_grads, intercept_grads)
+            last, sw_sum, activations, deltas, coef_grads, intercept_grads
+        )
 
         inplace_derivative = DERIVATIVES[self.activation]
         # Iterate over the hidden layers
-        for i in range(self.n_layers_ - 2, 0, -1):
+        for i in range(last, 0, -1):
             deltas[i - 1] = safe_sparse_dot(deltas[i], self.coefs_[i].T)
             inplace_derivative(activations[i], deltas[i - 1])
 
             self._compute_loss_grad(
-                i - 1, n_samples, activations, deltas, coef_grads,
-                intercept_grads)
+                i - 1, sw_sum, activations, deltas, coef_grads, intercept_grads
+            )
 
         return loss, coef_grads, intercept_grads
 
     def _initialize(self, y, layer_units, dtype):
-        # set all attributes, allocate weights etc for first call
+        # set all attributes, allocate weights etc. for first call
         # Initialize parameters
         self.n_iter_ = 0
         self.t_ = 0
@@ -301,69 +399,80 @@ def _initialize(self, y, layer_units, dtype):
 
         # Output for regression
         if not is_classifier(self):
-            self.out_activation_ = 'identity'
+            if self.loss == "poisson":
+                self.out_activation_ = "exp"
+            else:
+                # loss = "squared_error"
+                self.out_activation_ = "identity"
         # Output for multi class
-        elif self._label_binarizer.y_type_ == 'multiclass':
-            self.out_activation_ = 'softmax'
+        elif self._label_binarizer.y_type_ == "multiclass":
+            self.out_activation_ = "softmax"
         # Output for binary class and multi-label
         else:
-            self.out_activation_ = 'logistic'
+            self.out_activation_ = "logistic"
 
         # Initialize coefficient and intercept layers
         self.coefs_ = []
         self.intercepts_ = []
 
         for i in range(self.n_layers_ - 1):
-            coef_init, intercept_init = self._init_coef(layer_units[i],
-                                                        layer_units[i + 1],
-                                                        dtype)
+            coef_init, intercept_init = self._init_coef(
+                layer_units[i], layer_units[i + 1], dtype
+            )
             self.coefs_.append(coef_init)
             self.intercepts_.append(intercept_init)
 
+        self._best_coefs = [c.copy() for c in self.coefs_]
+        self._best_intercepts = [i.copy() for i in self.intercepts_]
+
         if self.solver in _STOCHASTIC_SOLVERS:
             self.loss_curve_ = []
             self._no_improvement_count = 0
             if self.early_stopping:
                 self.validation_scores_ = []
                 self.best_validation_score_ = -np.inf
+                self.best_loss_ = None
             else:
                 self.best_loss_ = np.inf
+                self.validation_scores_ = None
+                self.best_validation_score_ = None
 
     def _init_coef(self, fan_in, fan_out, dtype):
         # Use the initialization method recommended by
         # Glorot et al.
-        factor = 6.
-        if self.activation == 'logistic':
-            factor = 2.
+        factor = 6.0
+        if self.activation == "logistic":
+            factor = 2.0
         init_bound = np.sqrt(factor / (fan_in + fan_out))
 
         # Generate weights and bias:
-        coef_init = self._random_state.uniform(-init_bound, init_bound,
-                                               (fan_in, fan_out))
-        intercept_init = self._random_state.uniform(-init_bound, init_bound,
-                                                    fan_out)
+        coef_init = self._random_state.uniform(
+            -init_bound, init_bound, (fan_in, fan_out)
+        )
+        intercept_init = self._random_state.uniform(-init_bound, init_bound, fan_out)
         coef_init = coef_init.astype(dtype, copy=False)
         intercept_init = intercept_init.astype(dtype, copy=False)
         return coef_init, intercept_init
 
-    def _fit(self, X, y, incremental=False):
+    def _fit(self, X, y, sample_weight=None, incremental=False):
         # Make sure self.hidden_layer_sizes is a list
         hidden_layer_sizes = self.hidden_layer_sizes
         if not hasattr(hidden_layer_sizes, "__iter__"):
             hidden_layer_sizes = [hidden_layer_sizes]
         hidden_layer_sizes = list(hidden_layer_sizes)
 
-        # Validate input parameters.
-        self._validate_hyperparameters()
         if np.any(np.array(hidden_layer_sizes) <= 0):
-            raise ValueError("hidden_layer_sizes must be > 0, got %s." %
-                             hidden_layer_sizes)
-        first_pass = (not hasattr(self, 'coefs_') or
-                      (not self.warm_start and not incremental))
+            raise ValueError(
+                "hidden_layer_sizes must be > 0, got %s." % hidden_layer_sizes
+            )
+        first_pass = not hasattr(self, "coefs_") or (
+            not self.warm_start and not incremental
+        )
 
         X, y = self._validate_input(X, y, incremental, reset=first_pass)
-
         n_samples, n_features = X.shape
+        if sample_weight is not None:
+            sample_weight = _check_sample_weight(sample_weight, X)
 
         # Ensure y is 2D
         if y.ndim == 1:
@@ -371,8 +480,7 @@ def _fit(self, X, y, incremental=False):
 
         self.n_outputs_ = y.shape[1]
 
-        layer_units = ([n_features] + hidden_layer_sizes +
-                       [self.n_outputs_])
+        layer_units = [n_features] + hidden_layer_sizes + [self.n_outputs_]
 
         # check random state
         self._random_state = check_random_state(self.random_state)
@@ -385,80 +493,63 @@ def _fit(self, X, y, incremental=False):
         activations = [X] + [None] * (len(layer_units) - 1)
         deltas = [None] * (len(activations) - 1)
 
-        coef_grads = [np.empty((n_fan_in_, n_fan_out_), dtype=X.dtype)
-                      for n_fan_in_,
-                      n_fan_out_ in zip(layer_units[:-1],
-                                        layer_units[1:])]
+        coef_grads = [
+            np.empty((n_fan_in_, n_fan_out_), dtype=X.dtype)
+            for n_fan_in_, n_fan_out_ in pairwise(layer_units)
+        ]
 
-        intercept_grads = [np.empty(n_fan_out_, dtype=X.dtype)
-                           for n_fan_out_ in
-                           layer_units[1:]]
+        intercept_grads = [
+            np.empty(n_fan_out_, dtype=X.dtype) for n_fan_out_ in layer_units[1:]
+        ]
 
         # Run the Stochastic optimization solver
         if self.solver in _STOCHASTIC_SOLVERS:
-            self._fit_stochastic(X, y, activations, deltas, coef_grads,
-                                 intercept_grads, layer_units, incremental)
+            self._fit_stochastic(
+                X,
+                y,
+                sample_weight,
+                activations,
+                deltas,
+                coef_grads,
+                intercept_grads,
+                layer_units,
+                incremental,
+            )
 
         # Run the LBFGS solver
-        elif self.solver == 'lbfgs':
-            self._fit_lbfgs(X, y, activations, deltas, coef_grads,
-                            intercept_grads, layer_units)
+        elif self.solver == "lbfgs":
+            self._fit_lbfgs(
+                X,
+                y,
+                sample_weight,
+                activations,
+                deltas,
+                coef_grads,
+                intercept_grads,
+                layer_units,
+            )
+
+        # validate parameter weights
+        weights = chain(self.coefs_, self.intercepts_)
+        if not all(np.isfinite(w).all() for w in weights):
+            raise ValueError(
+                "Solver produced non-finite parameter weights. The input data may"
+                " contain large values and need to be preprocessed."
+            )
+
         return self
 
-    def _validate_hyperparameters(self):
-        if not isinstance(self.shuffle, bool):
-            raise ValueError("shuffle must be either True or False, got %s." %
-                             self.shuffle)
-        if self.max_iter <= 0:
-            raise ValueError("max_iter must be > 0, got %s." % self.max_iter)
-        if self.max_fun <= 0:
-            raise ValueError("max_fun must be > 0, got %s." % self.max_fun)
-        if self.alpha < 0.0:
-            raise ValueError("alpha must be >= 0, got %s." % self.alpha)
-        if (self.learning_rate in ["constant", "invscaling", "adaptive"] and
-                self.learning_rate_init <= 0.0):
-            raise ValueError("learning_rate_init must be > 0, got %s." %
-                             self.learning_rate)
-        if self.momentum > 1 or self.momentum < 0:
-            raise ValueError("momentum must be >= 0 and <= 1, got %s" %
-                             self.momentum)
-        if not isinstance(self.nesterovs_momentum, bool):
-            raise ValueError("nesterovs_momentum must be either True or False,"
-                             " got %s." % self.nesterovs_momentum)
-        if not isinstance(self.early_stopping, bool):
-            raise ValueError("early_stopping must be either True or False,"
-                             " got %s." % self.early_stopping)
-        if self.validation_fraction < 0 or self.validation_fraction >= 1:
-            raise ValueError("validation_fraction must be >= 0 and < 1, "
-                             "got %s" % self.validation_fraction)
-        if self.beta_1 < 0 or self.beta_1 >= 1:
-            raise ValueError("beta_1 must be >= 0 and < 1, got %s" %
-                             self.beta_1)
-        if self.beta_2 < 0 or self.beta_2 >= 1:
-            raise ValueError("beta_2 must be >= 0 and < 1, got %s" %
-                             self.beta_2)
-        if self.epsilon <= 0.0:
-            raise ValueError("epsilon must be > 0, got %s." % self.epsilon)
-        if self.n_iter_no_change <= 0:
-            raise ValueError("n_iter_no_change must be > 0, got %s."
-                             % self.n_iter_no_change)
-
-        # raise ValueError if not registered
-        if self.activation not in ACTIVATIONS:
-            raise ValueError("The activation '%s' is not supported. Supported "
-                             "activations are %s."
-                             % (self.activation, list(sorted(ACTIVATIONS))))
-        if self.learning_rate not in ["constant", "invscaling", "adaptive"]:
-            raise ValueError("learning rate %s is not supported. " %
-                             self.learning_rate)
-        supported_solvers = _STOCHASTIC_SOLVERS + ["lbfgs"]
-        if self.solver not in supported_solvers:
-            raise ValueError("The solver %s is not supported. "
-                             " Expected one of: %s" %
-                             (self.solver, ", ".join(supported_solvers)))
-
-    def _fit_lbfgs(self, X, y, activations, deltas, coef_grads,
-                   intercept_grads, layer_units):
+    def _fit_lbfgs(
+        self,
+        X,
+        y,
+        sample_weight,
+        activations,
+        deltas,
+        coef_grads,
+        intercept_grads,
+        layer_units,
+    ):
         # Store meta information for the parameters
         self._coef_indptr = []
         self._intercept_indptr = []
@@ -479,8 +570,7 @@ def _fit_lbfgs(self, X, y, activations, deltas, coef_grads,
             start = end
 
         # Run LBFGS
-        packed_coef_inter = _pack(self.coefs_,
-                                  self.intercepts_)
+        packed_coef_inter = _pack(self.coefs_, self.intercepts_)
 
         if self.verbose is True or self.verbose >= 1:
             iprint = 1
@@ -488,102 +578,175 @@ def _fit_lbfgs(self, X, y, activations, deltas, coef_grads,
             iprint = -1
 
         opt_res = scipy.optimize.minimize(
-                self._loss_grad_lbfgs, packed_coef_inter,
-                method="L-BFGS-B", jac=True,
-                options={
-                    "maxfun": self.max_fun,
-                    "maxiter": self.max_iter,
-                    "iprint": iprint,
-                    "gtol": self.tol
-                },
-                args=(X, y, activations, deltas, coef_grads, intercept_grads))
+            self._loss_grad_lbfgs,
+            packed_coef_inter,
+            method="L-BFGS-B",
+            jac=True,
+            options={
+                "maxfun": self.max_fun,
+                "maxiter": self.max_iter,
+                "iprint": iprint,
+                "gtol": self.tol,
+            },
+            args=(
+                X,
+                y,
+                sample_weight,
+                activations,
+                deltas,
+                coef_grads,
+                intercept_grads,
+            ),
+        )
         self.n_iter_ = _check_optimize_result("lbfgs", opt_res, self.max_iter)
         self.loss_ = opt_res.fun
         self._unpack(opt_res.x)
 
-    def _fit_stochastic(self, X, y, activations, deltas, coef_grads,
-                        intercept_grads, layer_units, incremental):
-
-        if not incremental or not hasattr(self, '_optimizer'):
-            params = self.coefs_ + self.intercepts_
-
-            if self.solver == 'sgd':
+    def _fit_stochastic(
+        self,
+        X,
+        y,
+        sample_weight,
+        activations,
+        deltas,
+        coef_grads,
+        intercept_grads,
+        layer_units,
+        incremental,
+    ):
+        params = self.coefs_ + self.intercepts_
+        if not incremental or not hasattr(self, "_optimizer"):
+            if self.solver == "sgd":
                 self._optimizer = SGDOptimizer(
-                    params, self.learning_rate_init, self.learning_rate,
-                    self.momentum, self.nesterovs_momentum, self.power_t)
-            elif self.solver == 'adam':
+                    params,
+                    self.learning_rate_init,
+                    self.learning_rate,
+                    self.momentum,
+                    self.nesterovs_momentum,
+                    self.power_t,
+                )
+            elif self.solver == "adam":
                 self._optimizer = AdamOptimizer(
-                    params, self.learning_rate_init, self.beta_1, self.beta_2,
-                    self.epsilon)
+                    params,
+                    self.learning_rate_init,
+                    self.beta_1,
+                    self.beta_2,
+                    self.epsilon,
+                )
 
         # early_stopping in partial_fit doesn't make sense
-        early_stopping = self.early_stopping and not incremental
+        if self.early_stopping and incremental:
+            raise ValueError("partial_fit does not support early_stopping=True")
+        early_stopping = self.early_stopping
         if early_stopping:
             # don't stratify in multilabel classification
             should_stratify = is_classifier(self) and self.n_outputs_ == 1
             stratify = y if should_stratify else None
-            X, X_val, y, y_val = train_test_split(
-                X, y, random_state=self._random_state,
-                test_size=self.validation_fraction,
-                stratify=stratify)
+            if sample_weight is None:
+                X_train, X_val, y_train, y_val = train_test_split(
+                    X,
+                    y,
+                    random_state=self._random_state,
+                    test_size=self.validation_fraction,
+                    stratify=stratify,
+                )
+                sample_weight_train = sample_weight_val = None
+            else:
+                # TODO: incorporate sample_weight in sampling here.
+                (
+                    X_train,
+                    X_val,
+                    y_train,
+                    y_val,
+                    sample_weight_train,
+                    sample_weight_val,
+                ) = train_test_split(
+                    X,
+                    y,
+                    sample_weight,
+                    random_state=self._random_state,
+                    test_size=self.validation_fraction,
+                    stratify=stratify,
+                )
+            if X_val.shape[0] < 2:
+                raise ValueError(
+                    "The validation set is too small. Increase 'validation_fraction' "
+                    "or the size of your dataset."
+                )
+
             if is_classifier(self):
                 y_val = self._label_binarizer.inverse_transform(y_val)
         else:
-            X_val = None
-            y_val = None
+            X_train, y_train, sample_weight_train = X, y, sample_weight
+            X_val = y_val = sample_weight_val = None
 
-        n_samples = X.shape[0]
+        n_samples = X_train.shape[0]
         sample_idx = np.arange(n_samples, dtype=int)
 
-        if self.batch_size == 'auto':
+        if self.batch_size == "auto":
             batch_size = min(200, n_samples)
         else:
-            if self.batch_size < 1 or self.batch_size > n_samples:
-                warnings.warn("Got `batch_size` less than 1 or larger than "
-                              "sample size. It is going to be clipped")
+            if self.batch_size > n_samples:
+                warnings.warn(
+                    "Got `batch_size` less than 1 or larger than "
+                    "sample size. It is going to be clipped"
+                )
             batch_size = np.clip(self.batch_size, 1, n_samples)
 
         try:
+            self.n_iter_ = 0
             for it in range(self.max_iter):
                 if self.shuffle:
                     # Only shuffle the sample indices instead of X and y to
                     # reduce the memory footprint. These indices will be used
                     # to slice the X and y.
-                    sample_idx = shuffle(sample_idx,
-                                         random_state=self._random_state)
+                    sample_idx = shuffle(sample_idx, random_state=self._random_state)
 
                 accumulated_loss = 0.0
                 for batch_slice in gen_batches(n_samples, batch_size):
                     if self.shuffle:
-                        X_batch = _safe_indexing(X, sample_idx[batch_slice])
-                        y_batch = y[sample_idx[batch_slice]]
+                        batch_idx = sample_idx[batch_slice]
+                        X_batch = _safe_indexing(X_train, batch_idx)
+                    else:
+                        batch_idx = batch_slice
+                        X_batch = X_train[batch_idx]
+                    y_batch = y_train[batch_idx]
+                    if sample_weight is None:
+                        sample_weight_batch = None
                     else:
-                        X_batch = X[batch_slice]
-                        y_batch = y[batch_slice]
+                        sample_weight_batch = sample_weight_train[batch_idx]
 
                     activations[0] = X_batch
                     batch_loss, coef_grads, intercept_grads = self._backprop(
-                        X_batch, y_batch, activations, deltas,
-                        coef_grads, intercept_grads)
-                    accumulated_loss += batch_loss * (batch_slice.stop -
-                                                      batch_slice.start)
+                        X_batch,
+                        y_batch,
+                        sample_weight_batch,
+                        activations,
+                        deltas,
+                        coef_grads,
+                        intercept_grads,
+                    )
+                    accumulated_loss += batch_loss * (
+                        batch_slice.stop - batch_slice.start
+                    )
 
                     # update weights
                     grads = coef_grads + intercept_grads
-                    self._optimizer.update_params(grads)
+                    self._optimizer.update_params(params, grads)
 
                 self.n_iter_ += 1
-                self.loss_ = accumulated_loss / X.shape[0]
+                self.loss_ = accumulated_loss / X_train.shape[0]
 
                 self.t_ += n_samples
                 self.loss_curve_.append(self.loss_)
                 if self.verbose:
-                    print("Iteration %d, loss = %.8f" % (self.n_iter_,
-                                                         self.loss_))
+                    print("Iteration %d, loss = %.8f" % (self.n_iter_, self.loss_))
 
                 # update no_improvement_count based on training loss or
                 # validation score according to early_stopping
-                self._update_no_improvement_count(early_stopping, X_val, y_val)
+                self._update_no_improvement_count(
+                    early_stopping, X_val, y_val, sample_weight_val
+                )
 
                 # for learning rate that needs to be updated at iteration end
                 self._optimizer.iteration_ends(self.t_)
@@ -592,16 +755,19 @@ def _fit_stochastic(self, X, y, activations, deltas, coef_grads,
                     # not better than last `n_iter_no_change` iterations by tol
                     # stop or decrease learning rate
                     if early_stopping:
-                        msg = ("Validation score did not improve more than "
-                               "tol=%f for %d consecutive epochs." % (
-                                   self.tol, self.n_iter_no_change))
+                        msg = (
+                            "Validation score did not improve more than "
+                            "tol=%f for %d consecutive epochs."
+                            % (self.tol, self.n_iter_no_change)
+                        )
                     else:
-                        msg = ("Training loss did not improve more than tol=%f"
-                               " for %d consecutive epochs." % (
-                                   self.tol, self.n_iter_no_change))
+                        msg = (
+                            "Training loss did not improve more than tol=%f"
+                            " for %d consecutive epochs."
+                            % (self.tol, self.n_iter_no_change)
+                        )
 
-                    is_stopping = self._optimizer.trigger_stopping(
-                        msg, self.verbose)
+                    is_stopping = self._optimizer.trigger_stopping(msg, self.verbose)
                     if is_stopping:
                         break
                     else:
@@ -614,7 +780,9 @@ def _fit_stochastic(self, X, y, activations, deltas, coef_grads,
                     warnings.warn(
                         "Stochastic Optimizer: Maximum iterations (%d) "
                         "reached and the optimization hasn't converged yet."
-                        % self.max_iter, ConvergenceWarning)
+                        % self.max_iter,
+                        ConvergenceWarning,
+                    )
         except KeyboardInterrupt:
             warnings.warn("Training interrupted by user.")
 
@@ -623,10 +791,12 @@ def _fit_stochastic(self, X, y, activations, deltas, coef_grads,
             self.coefs_ = self._best_coefs
             self.intercepts_ = self._best_intercepts
 
-    def _update_no_improvement_count(self, early_stopping, X_val, y_val):
+    def _update_no_improvement_count(self, early_stopping, X, y, sample_weight):
         if early_stopping:
-            # compute validation score, use that for stopping
-            self.validation_scores_.append(self.score(X_val, y_val))
+            # compute validation score (can be NaN), use that for stopping
+            val_score = self._score(X, y, sample_weight=sample_weight)
+
+            self.validation_scores_.append(val_score)
 
             if self.verbose:
                 print("Validation score: %f" % self.validation_scores_[-1])
@@ -635,8 +805,7 @@ def _update_no_improvement_count(self, early_stopping, X_val, y_val):
             # let's hope no-one overloads .score with mse
             last_valid_score = self.validation_scores_[-1]
 
-            if last_valid_score < (self.best_validation_score_ +
-                                   self.tol):
+            if last_valid_score < (self.best_validation_score_ + self.tol):
                 self._no_improvement_count += 1
             else:
                 self._no_improvement_count = 0
@@ -644,8 +813,7 @@ def _update_no_improvement_count(self, early_stopping, X_val, y_val):
             if last_valid_score > self.best_validation_score_:
                 self.best_validation_score_ = last_valid_score
                 self._best_coefs = [c.copy() for c in self.coefs_]
-                self._best_intercepts = [i.copy()
-                                         for i in self.intercepts_]
+                self._best_intercepts = [i.copy() for i in self.intercepts_]
         else:
             if self.loss_curve_[-1] > self.best_loss_ - self.tol:
                 self._no_improvement_count += 1
@@ -654,7 +822,8 @@ def _update_no_improvement_count(self, early_stopping, X_val, y_val):
             if self.loss_curve_[-1] < self.best_loss_:
                 self.best_loss_ = self.loss_curve_[-1]
 
-    def fit(self, X, y):
+    @_fit_context(prefer_skip_nested_validation=True)
+    def fit(self, X, y, sample_weight=None):
         """Fit the model to data matrix X and target(s) y.
 
         Parameters
@@ -666,36 +835,40 @@ def fit(self, X, y):
             The target values (class labels in classification, real numbers in
             regression).
 
+        sample_weight : array-like of shape (n_samples,), default=None
+            Sample weights.
+
+            .. versionadded:: 1.7
+
         Returns
         -------
-        self : returns a trained MLP model.
+        self : object
+            Returns a trained MLP model.
         """
-        return self._fit(X, y, incremental=False)
+        return self._fit(X, y, sample_weight=sample_weight, incremental=False)
 
-    @property
-    def partial_fit(self):
-        """Update the model with a single iteration over the given data.
+    def _check_solver(self):
+        if self.solver not in _STOCHASTIC_SOLVERS:
+            raise AttributeError(
+                "partial_fit is only available for stochastic"
+                " optimizers. %s is not stochastic." % self.solver
+            )
+        return True
 
-        Parameters
-        ----------
-        X : {array-like, sparse matrix} of shape (n_samples, n_features)
-            The input data.
+    def _score_with_function(self, X, y, sample_weight, score_function):
+        """Private score method without input validation."""
+        # Input validation would remove feature names, so we disable it
+        y_pred = self._predict(X, check_input=False)
 
-        y : ndarray of shape (n_samples,)
-            The target values.
+        if np.isnan(y_pred).any() or np.isinf(y_pred).any():
+            return np.nan
 
-        Returns
-        -------
-        self : returns a trained MLP model.
-        """
-        if self.solver not in _STOCHASTIC_SOLVERS:
-            raise AttributeError("partial_fit is only available for stochastic"
-                                 " optimizers. %s is not stochastic."
-                                 % self.solver)
-        return self._partial_fit
+        return score_function(y, y_pred, sample_weight=sample_weight)
 
-    def _partial_fit(self, X, y):
-        return self._fit(X, y, incremental=True)
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
+        return tags
 
 
 class MLPClassifier(ClassifierMixin, BaseMultilayerPerceptron):
@@ -708,7 +881,7 @@ class MLPClassifier(ClassifierMixin, BaseMultilayerPerceptron):
 
     Parameters
     ----------
-    hidden_layer_sizes : tuple, length = n_layers - 2, default=(100,)
+    hidden_layer_sizes : array-like of shape(n_layers - 2,), default=(100,)
         The ith element represents the number of neurons in the ith
         hidden layer.
 
@@ -737,6 +910,9 @@ class MLPClassifier(ClassifierMixin, BaseMultilayerPerceptron):
         - 'adam' refers to a stochastic gradient-based optimizer proposed
           by Kingma, Diederik, and Jimmy Ba
 
+        For a comparison between Adam optimizer and SGD, see
+        :ref:`sphx_glr_auto_examples_neural_networks_plot_mlp_training_curves.py`.
+
         Note: The default solver 'adam' works pretty well on relatively
         large datasets (with thousands of training samples or more) in terms of
         both training time and validation score.
@@ -744,12 +920,16 @@ class MLPClassifier(ClassifierMixin, BaseMultilayerPerceptron):
         better.
 
     alpha : float, default=0.0001
-        L2 penalty (regularization term) parameter.
+        Strength of the L2 regularization term. The L2 regularization term
+        is divided by the sample size when added to the loss.
+
+        For an example usage and visualization of varying regularization, see
+        :ref:`sphx_glr_auto_examples_neural_networks_plot_mlp_alpha.py`.
 
     batch_size : int, default='auto'
         Size of minibatches for stochastic optimizers.
         If the solver is 'lbfgs', the classifier will not use minibatch.
-        When set to "auto", `batch_size=min(200, n_samples)`
+        When set to "auto", `batch_size=min(200, n_samples)`.
 
     learning_rate : {'constant', 'invscaling', 'adaptive'}, default='constant'
         Learning rate schedule for weight updates.
@@ -769,11 +949,11 @@ class MLPClassifier(ClassifierMixin, BaseMultilayerPerceptron):
 
         Only used when ``solver='sgd'``.
 
-    learning_rate_init : double, default=0.001
+    learning_rate_init : float, default=0.001
         The initial learning rate used. It controls the step-size
         in updating the weights. Only used when solver='sgd' or 'adam'.
 
-    power_t : double, default=0.5
+    power_t : float, default=0.5
         The exponent for inverse scaling learning rate.
         It is used in updating effective learning rate when the learning_rate
         is set to 'invscaling'. Only used when solver='sgd'.
@@ -822,30 +1002,33 @@ class MLPClassifier(ClassifierMixin, BaseMultilayerPerceptron):
         Whether to use early stopping to terminate training when validation
         score is not improving. If set to true, it will automatically set
         aside 10% of training data as validation and terminate training when
-        validation score is not improving by at least tol for
+        validation score is not improving by at least ``tol`` for
         ``n_iter_no_change`` consecutive epochs. The split is stratified,
         except in a multilabel setting.
-        Only effective when solver='sgd' or 'adam'
+        If early stopping is False, then the training stops when the training
+        loss does not improve by more than tol for n_iter_no_change consecutive
+        passes over the training set.
+        Only effective when solver='sgd' or 'adam'.
 
     validation_fraction : float, default=0.1
         The proportion of training data to set aside as validation set for
         early stopping. Must be between 0 and 1.
-        Only used if early_stopping is True
+        Only used if early_stopping is True.
 
     beta_1 : float, default=0.9
         Exponential decay rate for estimates of first moment vector in adam,
-        should be in [0, 1). Only used when solver='adam'
+        should be in [0, 1). Only used when solver='adam'.
 
     beta_2 : float, default=0.999
         Exponential decay rate for estimates of second moment vector in adam,
-        should be in [0, 1). Only used when solver='adam'
+        should be in [0, 1). Only used when solver='adam'.
 
     epsilon : float, default=1e-8
-        Value for numerical stability in adam. Only used when solver='adam'
+        Value for numerical stability in adam. Only used when solver='adam'.
 
     n_iter_no_change : int, default=10
         Maximum number of epochs to not meet ``tol`` improvement.
-        Only effective when solver='sgd' or 'adam'
+        Only effective when solver='sgd' or 'adam'.
 
         .. versionadded:: 0.20
 
@@ -866,12 +1049,24 @@ class MLPClassifier(ClassifierMixin, BaseMultilayerPerceptron):
     loss_ : float
         The current loss computed with the loss function.
 
-    best_loss_ : float
+    best_loss_ : float or None
         The minimum loss reached by the solver throughout fitting.
+        If `early_stopping=True`, this attribute is set to `None`. Refer to
+        the `best_validation_score_` fitted attribute instead.
 
     loss_curve_ : list of shape (`n_iter_`,)
         The ith element in the list represents the loss at the ith iteration.
 
+    validation_scores_ : list of shape (`n_iter_`,) or None
+        The score at each iteration on a held-out validation set. The score
+        reported is the accuracy score. Only available if `early_stopping=True`,
+        otherwise the attribute is set to `None`.
+
+    best_validation_score_ : float or None
+        The best validation score (i.e. accuracy score) that triggered the
+        early stopping. Only available if `early_stopping=True`, otherwise the
+        attribute is set to `None`.
+
     t_ : int
         The number of training samples seen by the solver during fitting.
 
@@ -883,8 +1078,19 @@ class MLPClassifier(ClassifierMixin, BaseMultilayerPerceptron):
         The ith element in the list represents the bias vector corresponding to
         layer i + 1.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     n_iter_ : int
-        The number of iterations the solver has ran.
+        The number of iterations the solver has run.
 
     n_layers_ : int
         Number of layers.
@@ -895,21 +1101,10 @@ class MLPClassifier(ClassifierMixin, BaseMultilayerPerceptron):
     out_activation_ : str
         Name of the output activation function.
 
-    Examples
+    See Also
     --------
-    >>> from sklearn.neural_network import MLPClassifier
-    >>> from sklearn.datasets import make_classification
-    >>> from sklearn.model_selection import train_test_split
-    >>> X, y = make_classification(n_samples=100, random_state=1)
-    >>> X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y,
-    ...                                                     random_state=1)
-    >>> clf = MLPClassifier(random_state=1, max_iter=300).fit(X_train, y_train)
-    >>> clf.predict_proba(X_test[:1])
-    array([[0.038..., 0.961...]])
-    >>> clf.predict(X_test[:5, :])
-    array([1, 0, 1, 0, 1])
-    >>> clf.score(X_test, y_test)
-    0.8...
+    MLPRegressor : Multi-layer Perceptron regressor.
+    BernoulliRBM : Bernoulli Restricted Boltzmann Machine (RBM).
 
     Notes
     -----
@@ -925,50 +1120,100 @@ class MLPClassifier(ClassifierMixin, BaseMultilayerPerceptron):
 
     References
     ----------
-    Hinton, Geoffrey E.
-        "Connectionist learning procedures." Artificial intelligence 40.1
-        (1989): 185-234.
+    Hinton, Geoffrey E. "Connectionist learning procedures."
+    Artificial intelligence 40.1 (1989): 185-234.
+
+    Glorot, Xavier, and Yoshua Bengio.
+    "Understanding the difficulty of training deep feedforward neural networks."
+    International Conference on Artificial Intelligence and Statistics. 2010.
 
-    Glorot, Xavier, and Yoshua Bengio. "Understanding the difficulty of
-        training deep feedforward neural networks." International Conference
-        on Artificial Intelligence and Statistics. 2010.
+    :arxiv:`He, Kaiming, et al (2015). "Delving deep into rectifiers:
+    Surpassing human-level performance on imagenet classification." <1502.01852>`
 
-    He, Kaiming, et al. "Delving deep into rectifiers: Surpassing human-level
-        performance on imagenet classification." arXiv preprint
-        arXiv:1502.01852 (2015).
+    :arxiv:`Kingma, Diederik, and Jimmy Ba (2014)
+    "Adam: A method for stochastic optimization." <1412.6980>`
 
-    Kingma, Diederik, and Jimmy Ba. "Adam: A method for stochastic
-        optimization." arXiv preprint arXiv:1412.6980 (2014).
+    Examples
+    --------
+    >>> from sklearn.neural_network import MLPClassifier
+    >>> from sklearn.datasets import make_classification
+    >>> from sklearn.model_selection import train_test_split
+    >>> X, y = make_classification(n_samples=100, random_state=1)
+    >>> X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y,
+    ...                                                     random_state=1)
+    >>> clf = MLPClassifier(random_state=1, max_iter=300).fit(X_train, y_train)
+    >>> clf.predict_proba(X_test[:1])
+    array([[0.0383, 0.961]])
+    >>> clf.predict(X_test[:5, :])
+    array([1, 0, 1, 0, 1])
+    >>> clf.score(X_test, y_test)
+    0.8...
     """
-    @_deprecate_positional_args
-    def __init__(self, hidden_layer_sizes=(100,), activation="relu", *,
-                 solver='adam', alpha=0.0001,
-                 batch_size='auto', learning_rate="constant",
-                 learning_rate_init=0.001, power_t=0.5, max_iter=200,
-                 shuffle=True, random_state=None, tol=1e-4,
-                 verbose=False, warm_start=False, momentum=0.9,
-                 nesterovs_momentum=True, early_stopping=False,
-                 validation_fraction=0.1, beta_1=0.9, beta_2=0.999,
-                 epsilon=1e-8, n_iter_no_change=10, max_fun=15000):
+
+    def __init__(
+        self,
+        hidden_layer_sizes=(100,),
+        activation="relu",
+        *,
+        solver="adam",
+        alpha=0.0001,
+        batch_size="auto",
+        learning_rate="constant",
+        learning_rate_init=0.001,
+        power_t=0.5,
+        max_iter=200,
+        shuffle=True,
+        random_state=None,
+        tol=1e-4,
+        verbose=False,
+        warm_start=False,
+        momentum=0.9,
+        nesterovs_momentum=True,
+        early_stopping=False,
+        validation_fraction=0.1,
+        beta_1=0.9,
+        beta_2=0.999,
+        epsilon=1e-8,
+        n_iter_no_change=10,
+        max_fun=15000,
+    ):
         super().__init__(
             hidden_layer_sizes=hidden_layer_sizes,
-            activation=activation, solver=solver, alpha=alpha,
-            batch_size=batch_size, learning_rate=learning_rate,
-            learning_rate_init=learning_rate_init, power_t=power_t,
-            max_iter=max_iter, loss='log_loss', shuffle=shuffle,
-            random_state=random_state, tol=tol, verbose=verbose,
-            warm_start=warm_start, momentum=momentum,
+            activation=activation,
+            solver=solver,
+            alpha=alpha,
+            batch_size=batch_size,
+            learning_rate=learning_rate,
+            learning_rate_init=learning_rate_init,
+            power_t=power_t,
+            max_iter=max_iter,
+            loss="log_loss",
+            shuffle=shuffle,
+            random_state=random_state,
+            tol=tol,
+            verbose=verbose,
+            warm_start=warm_start,
+            momentum=momentum,
             nesterovs_momentum=nesterovs_momentum,
             early_stopping=early_stopping,
             validation_fraction=validation_fraction,
-            beta_1=beta_1, beta_2=beta_2, epsilon=epsilon,
-            n_iter_no_change=n_iter_no_change, max_fun=max_fun)
+            beta_1=beta_1,
+            beta_2=beta_2,
+            epsilon=epsilon,
+            n_iter_no_change=n_iter_no_change,
+            max_fun=max_fun,
+        )
 
     def _validate_input(self, X, y, incremental, reset):
-        X, y = self._validate_data(X, y, accept_sparse=['csr', 'csc'],
-                                   multi_output=True,
-                                   dtype=(np.float64, np.float32),
-                                   reset=reset)
+        X, y = validate_data(
+            self,
+            X,
+            y,
+            accept_sparse=["csr", "csc"],
+            multi_output=True,
+            dtype=(np.float64, np.float32),
+            reset=reset,
+        )
         if y.ndim == 2 and y.shape[1] == 1:
             y = column_or_1d(y, warn=True)
 
@@ -990,10 +1235,7 @@ def _validate_input(self, X, y, incremental, reset):
         #
         # Note the reliance on short-circuiting here, so that the second
         # or part implies that classes_ is defined.
-        if (
-            (not hasattr(self, "classes_")) or
-            (not self.warm_start and not incremental)
-        ):
+        if (not hasattr(self, "classes_")) or (not self.warm_start and not incremental):
             self._label_binarizer = LabelBinarizer()
             self._label_binarizer.fit(y)
             self.classes_ = self._label_binarizer.classes_
@@ -1002,13 +1244,13 @@ def _validate_input(self, X, y, incremental, reset):
             if self.warm_start:
                 if set(classes) != set(self.classes_):
                     raise ValueError(
-                        f"warm_start can only be used where `y` has the same "
-                        f"classes as in the previous call to fit. Previously "
+                        "warm_start can only be used where `y` has the same "
+                        "classes as in the previous call to fit. Previously "
                         f"got {self.classes_}, `y` has {classes}"
                     )
             elif len(np.setdiff1d(classes, self.classes_, assume_unique=True)):
                 raise ValueError(
-                    f"`y` has classes not in `self.classes_`. "
+                    "`y` has classes not in `self.classes_`. "
                     f"`self.classes_` has {self.classes_}. 'y' has {classes}."
                 )
 
@@ -1018,7 +1260,7 @@ def _validate_input(self, X, y, incremental, reset):
         return X, y
 
     def predict(self, X):
-        """Predict using the multi-layer perceptron classifier
+        """Predict using the multi-layer perceptron classifier.
 
         Parameters
         ----------
@@ -1031,15 +1273,25 @@ def predict(self, X):
             The predicted classes.
         """
         check_is_fitted(self)
-        y_pred = self._forward_pass_fast(X)
+        return self._predict(X)
+
+    def _predict(self, X, check_input=True):
+        """Private predict method with optional input validation"""
+        y_pred = self._forward_pass_fast(X, check_input=check_input)
 
         if self.n_outputs_ == 1:
             y_pred = y_pred.ravel()
 
         return self._label_binarizer.inverse_transform(y_pred)
 
-    @property
-    def partial_fit(self):
+    def _score(self, X, y, sample_weight=None):
+        return super()._score_with_function(
+            X, y, sample_weight=sample_weight, score_function=accuracy_score
+        )
+
+    @available_if(lambda est: est._check_solver())
+    @_fit_context(prefer_skip_nested_validation=True)
+    def partial_fit(self, X, y, sample_weight=None, classes=None):
         """Update the model with a single iteration over the given data.
 
         Parameters
@@ -1050,6 +1302,11 @@ def partial_fit(self):
         y : array-like of shape (n_samples,)
             The target values.
 
+        sample_weight : array-like of shape (n_samples,), default=None
+            Sample weights.
+
+            .. versionadded:: 1.7
+
         classes : array of shape (n_classes,), default=None
             Classes across all calls to partial_fit.
             Can be obtained via `np.unique(y_all)`, where y_all is the
@@ -1060,25 +1317,17 @@ def partial_fit(self):
 
         Returns
         -------
-        self : returns a trained MLP model.
+        self : object
+            Trained MLP model.
         """
-        if self.solver not in _STOCHASTIC_SOLVERS:
-            raise AttributeError("partial_fit is only available for stochastic"
-                                 " optimizer. %s is not stochastic"
-                                 % self.solver)
-        return self._partial_fit
-
-    def _partial_fit(self, X, y, classes=None):
         if _check_partial_fit_first_call(self, classes):
             self._label_binarizer = LabelBinarizer()
-            if type_of_target(y).startswith('multilabel'):
+            if type_of_target(y).startswith("multilabel"):
                 self._label_binarizer.fit(y)
             else:
                 self._label_binarizer.fit(classes)
 
-        super()._partial_fit(X, y)
-
-        return self
+        return self._fit(X, y, sample_weight=sample_weight, incremental=True)
 
     def predict_log_proba(self, X):
         """Return the log of probability estimates.
@@ -1093,7 +1342,7 @@ def predict_log_proba(self, X):
         log_y_prob : ndarray of shape (n_samples, n_classes)
             The predicted log-probability of the sample for each class
             in the model, where classes are ordered as they are in
-            `self.classes_`. Equivalent to log(predict_proba(X))
+            `self.classes_`. Equivalent to `log(predict_proba(X))`.
         """
         y_prob = self.predict_proba(X)
         return np.log(y_prob, out=y_prob)
@@ -1123,18 +1372,34 @@ def predict_proba(self, X):
         else:
             return y_pred
 
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.classifier_tags.multi_label = True
+        return tags
+
 
 class MLPRegressor(RegressorMixin, BaseMultilayerPerceptron):
     """Multi-layer Perceptron regressor.
 
-    This model optimizes the squared-loss using LBFGS or stochastic gradient
+    This model optimizes the squared error using LBFGS or stochastic gradient
     descent.
 
     .. versionadded:: 0.18
 
     Parameters
     ----------
-    hidden_layer_sizes : tuple, length = n_layers - 2, default=(100,)
+    loss : {'squared_error', 'poisson'}, default='squared_error'
+        The loss function to use when training the weights. Note that the
+        "squared error" and "poisson" losses actually implement
+        "half squares error" and "half poisson deviance" to simplify the
+        computation of the gradient. Furthermore, the "poisson" loss internally uses
+        a log-link (exponential as the output activation function) and requires
+        ``y >= 0``.
+
+        .. versionchanged:: 1.7
+           Added parameter `loss` and option 'poisson'.
+
+    hidden_layer_sizes : array-like of shape(n_layers - 2,), default=(100,)
         The ith element represents the number of neurons in the ith
         hidden layer.
 
@@ -1163,6 +1428,9 @@ class MLPRegressor(RegressorMixin, BaseMultilayerPerceptron):
         - 'adam' refers to a stochastic gradient-based optimizer proposed by
           Kingma, Diederik, and Jimmy Ba
 
+        For a comparison between Adam optimizer and SGD, see
+        :ref:`sphx_glr_auto_examples_neural_networks_plot_mlp_training_curves.py`.
+
         Note: The default solver 'adam' works pretty well on relatively
         large datasets (with thousands of training samples or more) in terms of
         both training time and validation score.
@@ -1170,12 +1438,13 @@ class MLPRegressor(RegressorMixin, BaseMultilayerPerceptron):
         better.
 
     alpha : float, default=0.0001
-        L2 penalty (regularization term) parameter.
+        Strength of the L2 regularization term. The L2 regularization term
+        is divided by the sample size when added to the loss.
 
     batch_size : int, default='auto'
         Size of minibatches for stochastic optimizers.
-        If the solver is 'lbfgs', the classifier will not use minibatch.
-        When set to "auto", `batch_size=min(200, n_samples)`
+        If the solver is 'lbfgs', the regressor will not use minibatch.
+        When set to "auto", `batch_size=min(200, n_samples)`.
 
     learning_rate : {'constant', 'invscaling', 'adaptive'}, default='constant'
         Learning rate schedule for weight updates.
@@ -1195,11 +1464,11 @@ class MLPRegressor(RegressorMixin, BaseMultilayerPerceptron):
 
         Only used when solver='sgd'.
 
-    learning_rate_init : double, default=0.001
+    learning_rate_init : float, default=0.001
         The initial learning rate used. It controls the step-size
         in updating the weights. Only used when solver='sgd' or 'adam'.
 
-    power_t : double, default=0.5
+    power_t : float, default=0.5
         The exponent for inverse scaling learning rate.
         It is used in updating effective learning rate when the learning_rate
         is set to 'invscaling'. Only used when solver='sgd'.
@@ -1237,7 +1506,7 @@ class MLPRegressor(RegressorMixin, BaseMultilayerPerceptron):
         previous solution. See :term:`the Glossary `.
 
     momentum : float, default=0.9
-        Momentum for gradient descent update.  Should be between 0 and 1. Only
+        Momentum for gradient descent update. Should be between 0 and 1. Only
         used when solver='sgd'.
 
     nesterovs_momentum : bool, default=True
@@ -1246,37 +1515,37 @@ class MLPRegressor(RegressorMixin, BaseMultilayerPerceptron):
 
     early_stopping : bool, default=False
         Whether to use early stopping to terminate training when validation
-        score is not improving. If set to true, it will automatically set
-        aside 10% of training data as validation and terminate training when
-        validation score is not improving by at least ``tol`` for
-        ``n_iter_no_change`` consecutive epochs.
-        Only effective when solver='sgd' or 'adam'
+        score is not improving. If set to True, it will automatically set
+        aside ``validation_fraction`` of training data as validation and
+        terminate training when validation score is not improving by at
+        least ``tol`` for ``n_iter_no_change`` consecutive epochs.
+        Only effective when solver='sgd' or 'adam'.
 
     validation_fraction : float, default=0.1
         The proportion of training data to set aside as validation set for
         early stopping. Must be between 0 and 1.
-        Only used if early_stopping is True
+        Only used if early_stopping is True.
 
     beta_1 : float, default=0.9
         Exponential decay rate for estimates of first moment vector in adam,
-        should be in [0, 1). Only used when solver='adam'
+        should be in [0, 1). Only used when solver='adam'.
 
     beta_2 : float, default=0.999
         Exponential decay rate for estimates of second moment vector in adam,
-        should be in [0, 1). Only used when solver='adam'
+        should be in [0, 1). Only used when solver='adam'.
 
     epsilon : float, default=1e-8
-        Value for numerical stability in adam. Only used when solver='adam'
+        Value for numerical stability in adam. Only used when solver='adam'.
 
     n_iter_no_change : int, default=10
         Maximum number of epochs to not meet ``tol`` improvement.
-        Only effective when solver='sgd' or 'adam'
+        Only effective when solver='sgd' or 'adam'.
 
         .. versionadded:: 0.20
 
     max_fun : int, default=15000
         Only used when solver='lbfgs'. Maximum number of function calls.
-        The solver iterates until convergence (determined by 'tol'), number
+        The solver iterates until convergence (determined by ``tol``), number
         of iterations reaches max_iter, or this number of function calls.
         Note that number of function calls will be greater than or equal to
         the number of iterations for the MLPRegressor.
@@ -1290,12 +1559,31 @@ class MLPRegressor(RegressorMixin, BaseMultilayerPerceptron):
 
     best_loss_ : float
         The minimum loss reached by the solver throughout fitting.
+        If `early_stopping=True`, this attribute is set to `None`. Refer to
+        the `best_validation_score_` fitted attribute instead.
+        Only accessible when solver='sgd' or 'adam'.
 
     loss_curve_ : list of shape (`n_iter_`,)
+        Loss value evaluated at the end of each training step.
         The ith element in the list represents the loss at the ith iteration.
+        Only accessible when solver='sgd' or 'adam'.
+
+    validation_scores_ : list of shape (`n_iter_`,) or None
+        The score at each iteration on a held-out validation set. The score
+        reported is the R2 score. Only available if `early_stopping=True`,
+        otherwise the attribute is set to `None`.
+        Only accessible when solver='sgd' or 'adam'.
+
+    best_validation_score_ : float or None
+        The best validation score (i.e. R2 score) that triggered the
+        early stopping. Only available if `early_stopping=True`, otherwise the
+        attribute is set to `None`.
+        Only accessible when solver='sgd' or 'adam'.
 
     t_ : int
         The number of training samples seen by the solver during fitting.
+        Mathematically equals `n_iters * X.shape[0]`, it means
+        `time_step` and it is used by optimizer's learning rate scheduler.
 
     coefs_ : list of shape (n_layers - 1,)
         The ith element in the list represents the weight matrix corresponding
@@ -1305,8 +1593,19 @@ class MLPRegressor(RegressorMixin, BaseMultilayerPerceptron):
         The ith element in the list represents the bias vector corresponding to
         layer i + 1.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     n_iter_ : int
-        The number of iterations the solver has ran.
+        The number of iterations the solver has run.
 
     n_layers_ : int
         Number of layers.
@@ -1317,26 +1616,12 @@ class MLPRegressor(RegressorMixin, BaseMultilayerPerceptron):
     out_activation_ : str
         Name of the output activation function.
 
-    loss_curve_ : list of shape (n_iters,)
-        Loss value evaluated at the end of each training step.
-
-    t_ : int
-        Mathematically equals `n_iters * X.shape[0]`, it means
-        `time_step` and it is used by optimizer's learning rate scheduler.
-
-    Examples
+    See Also
     --------
-    >>> from sklearn.neural_network import MLPRegressor
-    >>> from sklearn.datasets import make_regression
-    >>> from sklearn.model_selection import train_test_split
-    >>> X, y = make_regression(n_samples=200, random_state=1)
-    >>> X_train, X_test, y_train, y_test = train_test_split(X, y,
-    ...                                                     random_state=1)
-    >>> regr = MLPRegressor(random_state=1, max_iter=500).fit(X_train, y_train)
-    >>> regr.predict(X_test[:2])
-    array([-0.9..., -7.1...])
-    >>> regr.score(X_test, y_test)
-    0.4...
+    BernoulliRBM : Bernoulli Restricted Boltzmann Machine (RBM).
+    MLPClassifier : Multi-layer Perceptron classifier.
+    sklearn.linear_model.SGDRegressor : Linear model fitted by minimizing
+        a regularized empirical loss with SGD.
 
     Notes
     -----
@@ -1352,45 +1637,95 @@ class MLPRegressor(RegressorMixin, BaseMultilayerPerceptron):
 
     References
     ----------
-    Hinton, Geoffrey E.
-        "Connectionist learning procedures." Artificial intelligence 40.1
-        (1989): 185-234.
+    Hinton, Geoffrey E. "Connectionist learning procedures."
+    Artificial intelligence 40.1 (1989): 185-234.
 
-    Glorot, Xavier, and Yoshua Bengio. "Understanding the difficulty of
-        training deep feedforward neural networks." International Conference
-        on Artificial Intelligence and Statistics. 2010.
+    Glorot, Xavier, and Yoshua Bengio.
+    "Understanding the difficulty of training deep feedforward neural networks."
+    International Conference on Artificial Intelligence and Statistics. 2010.
 
-    He, Kaiming, et al. "Delving deep into rectifiers: Surpassing human-level
-        performance on imagenet classification." arXiv preprint
-        arXiv:1502.01852 (2015).
+    :arxiv:`He, Kaiming, et al (2015). "Delving deep into rectifiers:
+    Surpassing human-level performance on imagenet classification." <1502.01852>`
 
-    Kingma, Diederik, and Jimmy Ba. "Adam: A method for stochastic
-        optimization." arXiv preprint arXiv:1412.6980 (2014).
+    :arxiv:`Kingma, Diederik, and Jimmy Ba (2014)
+    "Adam: A method for stochastic optimization." <1412.6980>`
+
+    Examples
+    --------
+    >>> from sklearn.neural_network import MLPRegressor
+    >>> from sklearn.datasets import make_regression
+    >>> from sklearn.model_selection import train_test_split
+    >>> X, y = make_regression(n_samples=200, n_features=20, random_state=1)
+    >>> X_train, X_test, y_train, y_test = train_test_split(X, y,
+    ...                                                     random_state=1)
+    >>> regr = MLPRegressor(random_state=1, max_iter=2000, tol=0.1)
+    >>> regr.fit(X_train, y_train)
+    MLPRegressor(max_iter=2000, random_state=1, tol=0.1)
+    >>> regr.predict(X_test[:2])
+    array([  28.98, -291])
+    >>> regr.score(X_test, y_test)
+    0.98
     """
-    @_deprecate_positional_args
-    def __init__(self, hidden_layer_sizes=(100,), activation="relu", *,
-                 solver='adam', alpha=0.0001,
-                 batch_size='auto', learning_rate="constant",
-                 learning_rate_init=0.001,
-                 power_t=0.5, max_iter=200, shuffle=True,
-                 random_state=None, tol=1e-4,
-                 verbose=False, warm_start=False, momentum=0.9,
-                 nesterovs_momentum=True, early_stopping=False,
-                 validation_fraction=0.1, beta_1=0.9, beta_2=0.999,
-                 epsilon=1e-8, n_iter_no_change=10, max_fun=15000):
+
+    _parameter_constraints: dict = {
+        **BaseMultilayerPerceptron._parameter_constraints,
+        "loss": [StrOptions({"squared_error", "poisson"})],
+    }
+
+    def __init__(
+        self,
+        loss="squared_error",
+        hidden_layer_sizes=(100,),
+        activation="relu",
+        *,
+        solver="adam",
+        alpha=0.0001,
+        batch_size="auto",
+        learning_rate="constant",
+        learning_rate_init=0.001,
+        power_t=0.5,
+        max_iter=200,
+        shuffle=True,
+        random_state=None,
+        tol=1e-4,
+        verbose=False,
+        warm_start=False,
+        momentum=0.9,
+        nesterovs_momentum=True,
+        early_stopping=False,
+        validation_fraction=0.1,
+        beta_1=0.9,
+        beta_2=0.999,
+        epsilon=1e-8,
+        n_iter_no_change=10,
+        max_fun=15000,
+    ):
         super().__init__(
             hidden_layer_sizes=hidden_layer_sizes,
-            activation=activation, solver=solver, alpha=alpha,
-            batch_size=batch_size, learning_rate=learning_rate,
-            learning_rate_init=learning_rate_init, power_t=power_t,
-            max_iter=max_iter, loss='squared_loss', shuffle=shuffle,
-            random_state=random_state, tol=tol, verbose=verbose,
-            warm_start=warm_start, momentum=momentum,
+            activation=activation,
+            solver=solver,
+            alpha=alpha,
+            batch_size=batch_size,
+            learning_rate=learning_rate,
+            learning_rate_init=learning_rate_init,
+            power_t=power_t,
+            max_iter=max_iter,
+            loss=loss,
+            shuffle=shuffle,
+            random_state=random_state,
+            tol=tol,
+            verbose=verbose,
+            warm_start=warm_start,
+            momentum=momentum,
             nesterovs_momentum=nesterovs_momentum,
             early_stopping=early_stopping,
             validation_fraction=validation_fraction,
-            beta_1=beta_1, beta_2=beta_2, epsilon=epsilon,
-            n_iter_no_change=n_iter_no_change, max_fun=max_fun)
+            beta_1=beta_1,
+            beta_2=beta_2,
+            epsilon=epsilon,
+            n_iter_no_change=n_iter_no_change,
+            max_fun=max_fun,
+        )
 
     def predict(self, X):
         """Predict using the multi-layer perceptron model.
@@ -1406,16 +1741,56 @@ def predict(self, X):
             The predicted values.
         """
         check_is_fitted(self)
-        y_pred = self._forward_pass_fast(X)
+        return self._predict(X)
+
+    def _predict(self, X, check_input=True):
+        """Private predict method with optional input validation"""
+        y_pred = self._forward_pass_fast(X, check_input=check_input)
         if y_pred.shape[1] == 1:
             return y_pred.ravel()
         return y_pred
 
+    def _score(self, X, y, sample_weight=None):
+        return super()._score_with_function(
+            X, y, sample_weight=sample_weight, score_function=r2_score
+        )
+
     def _validate_input(self, X, y, incremental, reset):
-        X, y = self._validate_data(X, y, accept_sparse=['csr', 'csc'],
-                                   multi_output=True, y_numeric=True,
-                                   dtype=(np.float64, np.float32),
-                                   reset=reset)
+        X, y = validate_data(
+            self,
+            X,
+            y,
+            accept_sparse=["csr", "csc"],
+            multi_output=True,
+            y_numeric=True,
+            dtype=(np.float64, np.float32),
+            reset=reset,
+        )
         if y.ndim == 2 and y.shape[1] == 1:
             y = column_or_1d(y, warn=True)
         return X, y
+
+    @available_if(lambda est: est._check_solver)
+    @_fit_context(prefer_skip_nested_validation=True)
+    def partial_fit(self, X, y, sample_weight=None):
+        """Update the model with a single iteration over the given data.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            The input data.
+
+        y : ndarray of shape (n_samples,)
+            The target values.
+
+        sample_weight : array-like of shape (n_samples,), default=None
+            Sample weights.
+
+            .. versionadded:: 1.6
+
+        Returns
+        -------
+        self : object
+            Trained MLP model.
+        """
+        return self._fit(X, y, sample_weight=sample_weight, incremental=True)
diff --git a/sklearn/neural_network/_rbm.py b/sklearn/neural_network/_rbm.py
index 7aa64c503bb21..1e1d3c2e11b7c 100644
--- a/sklearn/neural_network/_rbm.py
+++ b/sklearn/neural_network/_rbm.py
@@ -1,29 +1,28 @@
-"""Restricted Boltzmann Machine
-"""
+"""Restricted Boltzmann Machine"""
 
-# Authors: Yann N. Dauphin 
-#          Vlad Niculae
-#          Gabriel Synnaeve
-#          Lars Buitinck
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import time
+from numbers import Integral, Real
 
 import numpy as np
 import scipy.sparse as sp
 from scipy.special import expit  # logistic function
 
-from ..base import BaseEstimator
-from ..base import TransformerMixin
-from ..utils import check_array
-from ..utils import check_random_state
-from ..utils import gen_even_slices
+from ..base import (
+    BaseEstimator,
+    ClassNamePrefixFeaturesOutMixin,
+    TransformerMixin,
+    _fit_context,
+)
+from ..utils import check_random_state, gen_even_slices
+from ..utils._param_validation import Interval
 from ..utils.extmath import safe_sparse_dot
-from ..utils.extmath import log_logistic
-from ..utils.validation import check_is_fitted, _deprecate_positional_args
+from ..utils.validation import check_is_fitted, validate_data
 
 
-class BernoulliRBM(TransformerMixin, BaseEstimator):
+class BernoulliRBM(ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator):
     """Bernoulli Restricted Boltzmann Machine (RBM).
 
     A Restricted Boltzmann Machine with binary visible units and
@@ -54,7 +53,8 @@ class BernoulliRBM(TransformerMixin, BaseEstimator):
         during training.
 
     verbose : int, default=0
-        The verbosity level. The default, zero, means silent mode.
+        The verbosity level. The default, zero, means silent mode. Range
+        of values is [0, inf].
 
     random_state : int, RandomState instance or None, default=None
         Determines random number generation for:
@@ -77,23 +77,31 @@ class BernoulliRBM(TransformerMixin, BaseEstimator):
         Biases of the visible units.
 
     components_ : array-like of shape (n_components, n_features)
-        Weight matrix, where n_features in the number of
-        visible units and n_components is the number of hidden units.
+        Weight matrix, where `n_features` is the number of
+        visible units and `n_components` is the number of hidden units.
 
     h_samples_ : array-like of shape (batch_size, n_components)
         Hidden Activation sampled from the model distribution,
-        where batch_size in the number of examples per minibatch and
-        n_components is the number of hidden units.
+        where `batch_size` is the number of examples per minibatch and
+        `n_components` is the number of hidden units.
 
-    Examples
-    --------
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
 
-    >>> import numpy as np
-    >>> from sklearn.neural_network import BernoulliRBM
-    >>> X = np.array([[0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 1]])
-    >>> model = BernoulliRBM(n_components=2)
-    >>> model.fit(X)
-    BernoulliRBM(n_components=2)
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    sklearn.neural_network.MLPRegressor : Multi-layer Perceptron regressor.
+    sklearn.neural_network.MLPClassifier : Multi-layer Perceptron classifier.
+    sklearn.decomposition.PCA : An unsupervised linear dimensionality
+        reduction model.
 
     References
     ----------
@@ -105,10 +113,40 @@ class BernoulliRBM(TransformerMixin, BaseEstimator):
     [2] Tieleman, T. Training Restricted Boltzmann Machines using
         Approximations to the Likelihood Gradient. International Conference
         on Machine Learning (ICML) 2008
+
+    Examples
+    --------
+
+    >>> import numpy as np
+    >>> from sklearn.neural_network import BernoulliRBM
+    >>> X = np.array([[0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 1]])
+    >>> model = BernoulliRBM(n_components=2)
+    >>> model.fit(X)
+    BernoulliRBM(n_components=2)
+
+    For a more detailed example usage, see
+    :ref:`sphx_glr_auto_examples_neural_networks_plot_rbm_logistic_classification.py`.
     """
-    @_deprecate_positional_args
-    def __init__(self, n_components=256, *, learning_rate=0.1, batch_size=10,
-                 n_iter=10, verbose=0, random_state=None):
+
+    _parameter_constraints: dict = {
+        "n_components": [Interval(Integral, 1, None, closed="left")],
+        "learning_rate": [Interval(Real, 0, None, closed="neither")],
+        "batch_size": [Interval(Integral, 1, None, closed="left")],
+        "n_iter": [Interval(Integral, 0, None, closed="left")],
+        "verbose": ["verbose"],
+        "random_state": ["random_state"],
+    }
+
+    def __init__(
+        self,
+        n_components=256,
+        *,
+        learning_rate=0.1,
+        batch_size=10,
+        n_iter=10,
+        verbose=0,
+        random_state=None,
+    ):
         self.n_components = n_components
         self.learning_rate = learning_rate
         self.batch_size = batch_size
@@ -131,8 +169,9 @@ def transform(self, X):
         """
         check_is_fitted(self)
 
-        X = self._validate_data(X, accept_sparse='csr', reset=False,
-                                dtype=(np.float64, np.float32))
+        X = validate_data(
+            self, X, accept_sparse="csr", reset=False, dtype=(np.float64, np.float32)
+        )
         return self._mean_hiddens(X)
 
     def _mean_hiddens(self, v):
@@ -169,7 +208,7 @@ def _sample_hiddens(self, v, rng):
             Values of the hidden layer.
         """
         p = self._mean_hiddens(v)
-        return (rng.random_sample(size=p.shape) < p)
+        return rng.uniform(size=p.shape) < p
 
     def _sample_visibles(self, h, rng):
         """Sample from the distribution P(v|h).
@@ -190,7 +229,7 @@ def _sample_visibles(self, h, rng):
         p = np.dot(h, self.components_)
         p += self.intercept_visible_
         expit(p, out=p)
-        return (rng.random_sample(size=p.shape) < p)
+        return rng.uniform(size=p.shape) < p
 
     def _free_energy(self, v):
         """Computes the free energy F(v) = - log sum_h exp(-E(v,h)).
@@ -205,9 +244,9 @@ def _free_energy(self, v):
         free_energy : ndarray of shape (n_samples,)
             The value of the free energy.
         """
-        return (- safe_sparse_dot(v, self.intercept_visible_)
-                - np.logaddexp(0, safe_sparse_dot(v, self.components_.T)
-                               + self.intercept_hidden_).sum(axis=1))
+        return -safe_sparse_dot(v, self.intercept_visible_) - np.logaddexp(
+            0, safe_sparse_dot(v, self.components_.T) + self.intercept_hidden_
+        ).sum(axis=1)
 
     def gibbs(self, v):
         """Perform one Gibbs sampling step.
@@ -230,38 +269,44 @@ def gibbs(self, v):
 
         return v_
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def partial_fit(self, X, y=None):
-        """Fit the model to the data X which should contain a partial
-        segment of the data.
+        """Fit the model to the partial segment of the data X.
 
         Parameters
         ----------
         X : ndarray of shape (n_samples, n_features)
             Training data.
 
+        y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None
+            Target values (None for unsupervised transformations).
+
         Returns
         -------
         self : BernoulliRBM
             The fitted model.
         """
-        first_pass = not hasattr(self, 'components_')
-        X = self._validate_data(X, accept_sparse='csr', dtype=np.float64,
-                                reset=first_pass)
-        if not hasattr(self, 'random_state_'):
+        first_pass = not hasattr(self, "components_")
+        X = validate_data(
+            self, X, accept_sparse="csr", dtype=np.float64, reset=first_pass
+        )
+        if not hasattr(self, "random_state_"):
             self.random_state_ = check_random_state(self.random_state)
-        if not hasattr(self, 'components_'):
+        if not hasattr(self, "components_"):
             self.components_ = np.asarray(
-                self.random_state_.normal(
-                    0,
-                    0.01,
-                    (self.n_components, X.shape[1])
-                ),
-                order='F')
-        if not hasattr(self, 'intercept_hidden_'):
-            self.intercept_hidden_ = np.zeros(self.n_components, )
-        if not hasattr(self, 'intercept_visible_'):
-            self.intercept_visible_ = np.zeros(X.shape[1], )
-        if not hasattr(self, 'h_samples_'):
+                self.random_state_.normal(0, 0.01, (self.n_components, X.shape[1])),
+                order="F",
+            )
+            self._n_features_out = self.components_.shape[0]
+        if not hasattr(self, "intercept_hidden_"):
+            self.intercept_hidden_ = np.zeros(
+                self.n_components,
+            )
+        if not hasattr(self, "intercept_visible_"):
+            self.intercept_visible_ = np.zeros(
+                X.shape[1],
+            )
+        if not hasattr(self, "h_samples_"):
             self.h_samples_ = np.zeros((self.batch_size, self.n_components))
 
         self._fit(X, self.random_state_)
@@ -289,9 +334,9 @@ def _fit(self, v_pos, rng):
         update -= np.dot(h_neg.T, v_neg)
         self.components_ += lr * update
         self.intercept_hidden_ += lr * (h_pos.sum(axis=0) - h_neg.sum(axis=0))
-        self.intercept_visible_ += lr * (np.asarray(
-                                         v_pos.sum(axis=0)).squeeze() -
-                                         v_neg.sum(axis=0))
+        self.intercept_visible_ += lr * (
+            np.asarray(v_pos.sum(axis=0)).squeeze() - v_neg.sum(axis=0)
+        )
 
         h_neg[rng.uniform(size=h_neg.shape) < h_neg] = 1.0  # sample binomial
         self.h_samples_ = np.floor(h_neg, h_neg)
@@ -317,23 +362,27 @@ def score_samples(self, X):
         """
         check_is_fitted(self)
 
-        v = check_array(X, accept_sparse='csr')
+        v = validate_data(self, X, accept_sparse="csr", reset=False)
         rng = check_random_state(self.random_state)
 
         # Randomly corrupt one feature in each sample in v.
-        ind = (np.arange(v.shape[0]),
-               rng.randint(0, v.shape[1], v.shape[0]))
+        ind = (np.arange(v.shape[0]), rng.randint(0, v.shape[1], v.shape[0]))
         if sp.issparse(v):
             data = -2 * v[ind] + 1
-            v_ = v + sp.csr_matrix((data.A.ravel(), ind), shape=v.shape)
+            if isinstance(data, np.matrix):  # v is a sparse matrix
+                v_ = v + sp.csr_matrix((data.A.ravel(), ind), shape=v.shape)
+            else:  # v is a sparse array
+                v_ = v + sp.csr_array((data.ravel(), ind), shape=v.shape)
         else:
             v_ = v.copy()
             v_[ind] = 1 - v_[ind]
 
         fe = self._free_energy(v)
         fe_ = self._free_energy(v_)
-        return v.shape[1] * log_logistic(fe_ - fe)
+        # log(expit(x)) = log(1 / (1 + exp(-x)) = -np.logaddexp(0, -x)
+        return -v.shape[1] * np.logaddexp(0, -(fe_ - fe))
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y=None):
         """Fit the model to the data X.
 
@@ -342,29 +391,32 @@ def fit(self, X, y=None):
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
             Training data.
 
+        y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None
+            Target values (None for unsupervised transformations).
+
         Returns
         -------
         self : BernoulliRBM
             The fitted model.
         """
-        X = self._validate_data(
-            X, accept_sparse='csr', dtype=(np.float64, np.float32)
-        )
+        X = validate_data(self, X, accept_sparse="csr", dtype=(np.float64, np.float32))
         n_samples = X.shape[0]
         rng = check_random_state(self.random_state)
 
         self.components_ = np.asarray(
             rng.normal(0, 0.01, (self.n_components, X.shape[1])),
-            order='F',
-            dtype=X.dtype)
+            order="F",
+            dtype=X.dtype,
+        )
+        self._n_features_out = self.components_.shape[0]
         self.intercept_hidden_ = np.zeros(self.n_components, dtype=X.dtype)
         self.intercept_visible_ = np.zeros(X.shape[1], dtype=X.dtype)
-        self.h_samples_ = np.zeros((self.batch_size, self.n_components),
-                                   dtype=X.dtype)
+        self.h_samples_ = np.zeros((self.batch_size, self.n_components), dtype=X.dtype)
 
         n_batches = int(np.ceil(float(n_samples) / self.batch_size))
-        batch_slices = list(gen_even_slices(n_batches * self.batch_size,
-                                            n_batches, n_samples=n_samples))
+        batch_slices = list(
+            gen_even_slices(n_batches * self.batch_size, n_batches, n_samples=n_samples)
+        )
         verbose = self.verbose
         begin = time.time()
         for iteration in range(1, self.n_iter + 1):
@@ -373,20 +425,21 @@ def fit(self, X, y=None):
 
             if verbose:
                 end = time.time()
-                print("[%s] Iteration %d, pseudo-likelihood = %.2f,"
-                      " time = %.2fs"
-                      % (type(self).__name__, iteration,
-                         self.score_samples(X).mean(), end - begin))
+                print(
+                    "[%s] Iteration %d, pseudo-likelihood = %.2f, time = %.2fs"
+                    % (
+                        type(self).__name__,
+                        iteration,
+                        self.score_samples(X).mean(),
+                        end - begin,
+                    )
+                )
                 begin = end
 
         return self
 
-    def _more_tags(self):
-        return {
-            '_xfail_checks': {
-                'check_methods_subset_invariance':
-                'fails for the decision_function method',
-                'check_methods_sample_order_invariance':
-                'fails for the score_samples method',
-            }
-        }
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
+        tags.transformer_tags.preserves_dtype = ["float64", "float32"]
+        return tags
diff --git a/sklearn/neural_network/_stochastic_optimizers.py b/sklearn/neural_network/_stochastic_optimizers.py
index 2da9c0b278e71..52641a91ce4d3 100644
--- a/sklearn/neural_network/_stochastic_optimizers.py
+++ b/sklearn/neural_network/_stochastic_optimizers.py
@@ -1,8 +1,7 @@
-"""Stochastic optimization methods for MLP
-"""
+"""Stochastic optimization methods for MLP"""
 
-# Authors: Jiyuan Qian 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import numpy as np
 
@@ -12,10 +11,6 @@ class BaseOptimizer:
 
     Parameters
     ----------
-    params : list, length = len(coefs_) + len(intercepts_)
-        The concatenated list containing coefs_ and intercepts_ in MLP model.
-        Used for initializing velocities and updating params
-
     learning_rate_init : float, default=0.1
         The initial learning rate used. It controls the step-size in updating
         the weights
@@ -26,22 +21,25 @@ class BaseOptimizer:
         the current learning rate
     """
 
-    def __init__(self, params, learning_rate_init=0.1):
-        self.params = [param for param in params]
+    def __init__(self, learning_rate_init=0.1):
         self.learning_rate_init = learning_rate_init
         self.learning_rate = float(learning_rate_init)
 
-    def update_params(self, grads):
+    def update_params(self, params, grads):
         """Update parameters with given gradients
 
         Parameters
         ----------
-        grads : list, length = len(params)
+        params : list of length = len(coefs_) + len(intercepts_)
+            The concatenated list containing coefs_ and intercepts_ in MLP
+            model. Used for initializing velocities and updating params
+
+        grads : list of length = len(params)
             Containing gradients with respect to coefs_ and intercepts_ in MLP
             model. So length should be aligned with params
         """
         updates = self._get_updates(grads)
-        for param, update in zip(self.params, updates):
+        for param, update in zip((p for p in params), updates):
             param += update
 
     def iteration_ends(self, time_step):
@@ -119,9 +117,16 @@ class SGDOptimizer(BaseOptimizer):
         velocities that are used to update params
     """
 
-    def __init__(self, params, learning_rate_init=0.1, lr_schedule='constant',
-                 momentum=0.9, nesterov=True, power_t=0.5):
-        super().__init__(params, learning_rate_init)
+    def __init__(
+        self,
+        params,
+        learning_rate_init=0.1,
+        lr_schedule="constant",
+        momentum=0.9,
+        nesterov=True,
+        power_t=0.5,
+    ):
+        super().__init__(learning_rate_init)
 
         self.lr_schedule = lr_schedule
         self.momentum = momentum
@@ -139,12 +144,13 @@ def iteration_ends(self, time_step):
             number of training samples trained on so far, used to update
             learning rate for 'invscaling'
         """
-        if self.lr_schedule == 'invscaling':
-            self.learning_rate = (float(self.learning_rate_init) /
-                                  (time_step + 1) ** self.power_t)
+        if self.lr_schedule == "invscaling":
+            self.learning_rate = (
+                float(self.learning_rate_init) / (time_step + 1) ** self.power_t
+            )
 
     def trigger_stopping(self, msg, verbose):
-        if self.lr_schedule != 'adaptive':
+        if self.lr_schedule != "adaptive":
             if verbose:
                 print(msg + " Stopping.")
             return True
@@ -154,10 +160,9 @@ def trigger_stopping(self, msg, verbose):
                 print(msg + " Learning rate too small. Stopping.")
             return True
 
-        self.learning_rate /= 5.
+        self.learning_rate /= 5.0
         if verbose:
-            print(msg + " Setting learning rate to %f" %
-                  self.learning_rate)
+            print(msg + " Setting learning rate to %f" % self.learning_rate)
         return False
 
     def _get_updates(self, grads):
@@ -174,13 +179,17 @@ def _get_updates(self, grads):
         updates : list, length = len(grads)
             The values to add to params
         """
-        updates = [self.momentum * velocity - self.learning_rate * grad
-                   for velocity, grad in zip(self.velocities, grads)]
+        updates = [
+            self.momentum * velocity - self.learning_rate * grad
+            for velocity, grad in zip(self.velocities, grads)
+        ]
         self.velocities = updates
 
         if self.nesterov:
-            updates = [self.momentum * velocity - self.learning_rate * grad
-                       for velocity, grad in zip(self.velocities, grads)]
+            updates = [
+                self.momentum * velocity - self.learning_rate * grad
+                for velocity, grad in zip(self.velocities, grads)
+            ]
 
         return updates
 
@@ -227,14 +236,14 @@ class AdamOptimizer(BaseOptimizer):
 
     References
     ----------
-    Kingma, Diederik, and Jimmy Ba.
-    "Adam: A method for stochastic optimization."
-    arXiv preprint arXiv:1412.6980 (2014).
+    :arxiv:`Kingma, Diederik, and Jimmy Ba (2014) "Adam: A method for
+        stochastic optimization." <1412.6980>
     """
 
-    def __init__(self, params, learning_rate_init=0.001, beta_1=0.9,
-                 beta_2=0.999, epsilon=1e-8):
-        super().__init__(params, learning_rate_init)
+    def __init__(
+        self, params, learning_rate_init=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-8
+    ):
+        super().__init__(learning_rate_init)
 
         self.beta_1 = beta_1
         self.beta_2 = beta_2
@@ -258,13 +267,21 @@ def _get_updates(self, grads):
             The values to add to params
         """
         self.t += 1
-        self.ms = [self.beta_1 * m + (1 - self.beta_1) * grad
-                   for m, grad in zip(self.ms, grads)]
-        self.vs = [self.beta_2 * v + (1 - self.beta_2) * (grad ** 2)
-                   for v, grad in zip(self.vs, grads)]
-        self.learning_rate = (self.learning_rate_init *
-                              np.sqrt(1 - self.beta_2 ** self.t) /
-                              (1 - self.beta_1 ** self.t))
-        updates = [-self.learning_rate * m / (np.sqrt(v) + self.epsilon)
-                   for m, v in zip(self.ms, self.vs)]
+        self.ms = [
+            self.beta_1 * m + (1 - self.beta_1) * grad
+            for m, grad in zip(self.ms, grads)
+        ]
+        self.vs = [
+            self.beta_2 * v + (1 - self.beta_2) * (grad**2)
+            for v, grad in zip(self.vs, grads)
+        ]
+        self.learning_rate = (
+            self.learning_rate_init
+            * np.sqrt(1 - self.beta_2**self.t)
+            / (1 - self.beta_1**self.t)
+        )
+        updates = [
+            -self.learning_rate * m / (np.sqrt(v) + self.epsilon)
+            for m, v in zip(self.ms, self.vs)
+        ]
         return updates
diff --git a/sklearn/neural_network/tests/test_base.py b/sklearn/neural_network/tests/test_base.py
index c803efe561faa..598b7e6054eea 100644
--- a/sklearn/neural_network/tests/test_base.py
+++ b/sklearn/neural_network/tests/test_base.py
@@ -1,8 +1,8 @@
-import pytest
 import numpy as np
+import pytest
 
-from sklearn.neural_network._base import binary_log_loss
-from sklearn.neural_network._base import log_loss
+from sklearn._loss import HalfPoissonLoss
+from sklearn.neural_network._base import binary_log_loss, log_loss, poisson_loss
 
 
 def test_binary_log_loss_1_prob_finite():
@@ -14,13 +14,39 @@ def test_binary_log_loss_1_prob_finite():
     assert np.isfinite(loss)
 
 
-@pytest.mark.parametrize("y_true, y_prob", [
-    (np.array([[1, 0, 0], [0, 1, 0]]),
-     np.array([[0., 1., 0.], [0.9, 0.05, 0.05]])),
-    (np.array([[0, 0, 1]]).T,
-     np.array([[0.9, 1.0, 1.0]]).T),
-])
+@pytest.mark.parametrize(
+    "y_true, y_prob",
+    [
+        (
+            np.array([[1, 0, 0], [0, 1, 0]]),
+            np.array([[0.0, 1.0, 0.0], [0.9, 0.05, 0.05]]),
+        ),
+        (np.array([[0, 0, 1]]).T, np.array([[0.9, 1.0, 1.0]]).T),
+    ],
+)
 def test_log_loss_1_prob_finite(y_true, y_prob):
     # y_proba is equal to 1 should result in a finite logloss
     loss = log_loss(y_true, y_prob)
     assert np.isfinite(loss)
+
+
+def test_poisson_loss(global_random_seed):
+    """Test Poisson loss against well tested HalfPoissonLoss."""
+    n = 1000
+    rng = np.random.default_rng(global_random_seed)
+    y_true = rng.integers(low=0, high=10, size=n).astype(float)
+    y_raw = rng.standard_normal(n)
+    y_pred = np.exp(y_raw)
+    sw = rng.uniform(low=0.1, high=10, size=n)
+
+    assert 0 in y_true
+
+    loss = poisson_loss(y_true=y_true, y_pred=y_pred, sample_weight=sw)
+    pl = HalfPoissonLoss()
+    loss_ref = (
+        pl(y_true=y_true, raw_prediction=y_raw, sample_weight=sw)
+        + pl.constant_to_optimal_zero(y_true=y_true, sample_weight=sw).mean()
+        / sw.mean()
+    )
+
+    assert loss == pytest.approx(loss_ref, rel=1e-12)
diff --git a/sklearn/neural_network/tests/test_mlp.py b/sklearn/neural_network/tests/test_mlp.py
index bdadf37c39902..9dddb78223ea7 100644
--- a/sklearn/neural_network/tests/test_mlp.py
+++ b/sklearn/neural_network/tests/test_mlp.py
@@ -2,34 +2,36 @@
 Testing for Multi-layer Perceptron module (sklearn.neural_network)
 """
 
-# Author: Issam H. Laradji
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-import pytest
+import re
 import sys
 import warnings
-import re
+from io import StringIO
 
+import joblib
 import numpy as np
+import pytest
 
-from numpy.testing import (
-    assert_almost_equal,
-    assert_array_equal,
-    assert_allclose,
+from sklearn.datasets import (
+    load_digits,
+    load_iris,
+    make_multilabel_classification,
+    make_regression,
 )
-
-from sklearn.datasets import load_digits, load_iris
-from sklearn.datasets import make_regression, make_multilabel_classification
 from sklearn.exceptions import ConvergenceWarning
-from io import StringIO
+from sklearn.linear_model import PoissonRegressor
 from sklearn.metrics import roc_auc_score
-from sklearn.neural_network import MLPClassifier
-from sklearn.neural_network import MLPRegressor
-from sklearn.preprocessing import LabelBinarizer
-from sklearn.preprocessing import MinMaxScaler, scale
-from scipy.sparse import csr_matrix
-from sklearn.utils._testing import ignore_warnings
-
+from sklearn.neural_network import MLPClassifier, MLPRegressor
+from sklearn.preprocessing import LabelBinarizer, MinMaxScaler, scale
+from sklearn.utils._testing import (
+    assert_allclose,
+    assert_almost_equal,
+    assert_array_equal,
+    ignore_warnings,
+)
+from sklearn.utils.fixes import CSR_CONTAINERS
 
 ACTIVATION_TYPES = ["identity", "logistic", "tanh", "relu"]
 
@@ -43,11 +45,14 @@
 X_digits_binary = MinMaxScaler().fit_transform(X_digits[:200])
 y_digits_binary = y_digits[:200]
 
-classification_datasets = [(X_digits_multi, y_digits_multi),
-                           (X_digits_binary, y_digits_binary)]
+classification_datasets = [
+    (X_digits_multi, y_digits_multi),
+    (X_digits_binary, y_digits_binary),
+]
 
-X_reg, y_reg = make_regression(n_samples=200, n_features=10, bias=20.,
-                               noise=100., random_state=7)
+X_reg, y_reg = make_regression(
+    n_samples=200, n_features=10, bias=20.0, noise=100.0, random_state=7
+)
 y_reg = scale(y_reg)
 regression_datasets = [(X_reg, y_reg)]
 
@@ -70,8 +75,9 @@ def test_alpha():
         mlp = MLPClassifier(hidden_layer_sizes=10, alpha=alpha, random_state=1)
         with ignore_warnings(category=ConvergenceWarning):
             mlp.fit(X, y)
-        alpha_vectors.append(np.array([absolute_sum(mlp.coefs_[0]),
-                                       absolute_sum(mlp.coefs_[1])]))
+        alpha_vectors.append(
+            np.array([absolute_sum(mlp.coefs_[0]), absolute_sum(mlp.coefs_[1])])
+        )
 
     for i in range(len(alpha_values) - 1):
         assert (alpha_vectors[i] > alpha_vectors[i + 1]).all()
@@ -81,9 +87,16 @@ def test_fit():
     # Test that the algorithm solution is equal to a worked out example.
     X = np.array([[0.6, 0.8, 0.7]])
     y = np.array([0])
-    mlp = MLPClassifier(solver='sgd', learning_rate_init=0.1, alpha=0.1,
-                        activation='logistic', random_state=1, max_iter=1,
-                        hidden_layer_sizes=2, momentum=0)
+    mlp = MLPClassifier(
+        solver="sgd",
+        learning_rate_init=0.1,
+        alpha=0.1,
+        activation="logistic",
+        random_state=1,
+        max_iter=1,
+        hidden_layer_sizes=2,
+        momentum=0,
+    )
     # set weights
     mlp.coefs_ = [0] * 2
     mlp.intercepts_ = [0] * 2
@@ -107,16 +120,15 @@ def test_fit():
     mlp._coef_grads = [0] * (mlp.n_layers_ - 1)
     mlp._intercept_grads = [0] * (mlp.n_layers_ - 1)
 
-    mlp.out_activation_ = 'logistic'
+    mlp.out_activation_ = "logistic"
     mlp.t_ = 0
     mlp.best_loss_ = np.inf
     mlp.loss_curve_ = []
     mlp._no_improvement_count = 0
-    mlp._intercept_velocity = [np.zeros_like(intercepts) for
-                               intercepts in
-                               mlp.intercepts_]
-    mlp._coef_velocity = [np.zeros_like(coefs) for coefs in
-                          mlp.coefs_]
+    mlp._intercept_velocity = [
+        np.zeros_like(intercepts) for intercepts in mlp.intercepts_
+    ]
+    mlp._coef_velocity = [np.zeros_like(coefs) for coefs in mlp.coefs_]
 
     mlp.partial_fit(X, y, classes=[0, 1])
     # Manually worked out example
@@ -149,14 +161,13 @@ def test_fit():
     # b1 = b1 - eta * [b1grad1, b1grad2] = 0.1 - 0.1 * [0.01667, 0.0374]
     #         = [0.098333, 0.09626]
     # b2 = b2 - eta * b2grad = 1.0 - 0.1 * 0.765 = 0.9235
-    assert_almost_equal(mlp.coefs_[0], np.array([[0.098, 0.195756],
-                                                 [0.2956664, 0.096008],
-                                                 [0.4939998, -0.002244]]),
-                        decimal=3)
-    assert_almost_equal(mlp.coefs_[1], np.array([[0.04706], [0.154089]]),
-                        decimal=3)
-    assert_almost_equal(mlp.intercepts_[0],
-                        np.array([0.098333, 0.09626]), decimal=3)
+    assert_almost_equal(
+        mlp.coefs_[0],
+        np.array([[0.098, 0.195756], [0.2956664, 0.096008], [0.4939998, -0.002244]]),
+        decimal=3,
+    )
+    assert_almost_equal(mlp.coefs_[1], np.array([[0.04706], [0.154089]]), decimal=3)
+    assert_almost_equal(mlp.intercepts_[0], np.array([0.098333, 0.09626]), decimal=3)
     assert_almost_equal(mlp.intercepts_[1], np.array(0.9235), decimal=3)
     # Testing output
     #  h1 = g(X1 * W_i1 + b11) = g(0.6 * 0.098 + 0.8 * 0.2956664 +
@@ -184,17 +195,22 @@ def test_gradient():
         Y = LabelBinarizer().fit_transform(y)
 
         for activation in ACTIVATION_TYPES:
-            mlp = MLPClassifier(activation=activation, hidden_layer_sizes=10,
-                                solver='lbfgs', alpha=1e-5,
-                                learning_rate_init=0.2, max_iter=1,
-                                random_state=1)
-            mlp.fit(X, y)
-
-            theta = np.hstack([l.ravel() for l in mlp.coefs_ +
-                               mlp.intercepts_])
-
-            layer_units = ([X.shape[1]] + [mlp.hidden_layer_sizes] +
-                           [mlp.n_outputs_])
+            mlp = MLPClassifier(
+                activation=activation,
+                hidden_layer_sizes=10,
+                solver="lbfgs",
+                alpha=1e-5,
+                learning_rate_init=0.2,
+                max_iter=1,
+                random_state=1,
+            )
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore", ConvergenceWarning)
+                mlp.fit(X, y)
+
+            theta = np.hstack([l.ravel() for l in mlp.coefs_ + mlp.intercepts_])
+
+            layer_units = [X.shape[1]] + [mlp.hidden_layer_sizes] + [mlp.n_outputs_]
 
             activations = []
             deltas = []
@@ -203,10 +219,8 @@ def test_gradient():
 
             activations.append(X)
             for i in range(mlp.n_layers_ - 1):
-                activations.append(np.empty((X.shape[0],
-                                             layer_units[i + 1])))
-                deltas.append(np.empty((X.shape[0],
-                                        layer_units[i + 1])))
+                activations.append(np.empty((X.shape[0], layer_units[i + 1])))
+                deltas.append(np.empty((X.shape[0], layer_units[i + 1])))
 
                 fan_in = layer_units[i]
                 fan_out = layer_units[i + 1]
@@ -215,8 +229,9 @@ def test_gradient():
 
             # analytically compute the gradients
             def loss_grad_fun(t):
-                return mlp._loss_grad_lbfgs(t, X, Y, activations, deltas,
-                                            coef_grads, intercept_grads)
+                return mlp._loss_grad_lbfgs(
+                    t, X, Y, None, activations, deltas, coef_grads, intercept_grads
+                )
 
             [value, grad] = loss_grad_fun(theta)
             numgrad = np.zeros(np.size(theta))
@@ -226,13 +241,13 @@ def loss_grad_fun(t):
             # numerically compute the gradients
             for i in range(n):
                 dtheta = E[:, i] * epsilon
-                numgrad[i] = ((loss_grad_fun(theta + dtheta)[0] -
-                              loss_grad_fun(theta - dtheta)[0]) /
-                              (epsilon * 2.0))
+                numgrad[i] = (
+                    loss_grad_fun(theta + dtheta)[0] - loss_grad_fun(theta - dtheta)[0]
+                ) / (epsilon * 2.0)
             assert_almost_equal(numgrad, grad)
 
 
-@pytest.mark.parametrize('X,y', classification_datasets)
+@pytest.mark.parametrize("X,y", classification_datasets)
 def test_lbfgs_classification(X, y):
     # Test lbfgs on classification.
     # It should achieve a score higher than 0.95 for the binary and multi-class
@@ -243,106 +258,142 @@ def test_lbfgs_classification(X, y):
     expected_shape_dtype = (X_test.shape[0], y_train.dtype.kind)
 
     for activation in ACTIVATION_TYPES:
-        mlp = MLPClassifier(solver='lbfgs', hidden_layer_sizes=50,
-                            max_iter=150, shuffle=True, random_state=1,
-                            activation=activation)
+        mlp = MLPClassifier(
+            solver="lbfgs",
+            hidden_layer_sizes=50,
+            max_iter=150,
+            shuffle=True,
+            random_state=1,
+            activation=activation,
+        )
         mlp.fit(X_train, y_train)
         y_predict = mlp.predict(X_test)
         assert mlp.score(X_train, y_train) > 0.95
-        assert ((y_predict.shape[0], y_predict.dtype.kind) ==
-                expected_shape_dtype)
+        assert (y_predict.shape[0], y_predict.dtype.kind) == expected_shape_dtype
 
 
-@pytest.mark.parametrize('X,y', regression_datasets)
+@pytest.mark.parametrize("X,y", regression_datasets)
 def test_lbfgs_regression(X, y):
     # Test lbfgs on the regression dataset.
     for activation in ACTIVATION_TYPES:
-        mlp = MLPRegressor(solver='lbfgs', hidden_layer_sizes=50,
-                           max_iter=150, shuffle=True, random_state=1,
-                           activation=activation)
+        mlp = MLPRegressor(
+            solver="lbfgs",
+            hidden_layer_sizes=50,
+            max_iter=200,
+            tol=1e-3,
+            shuffle=True,
+            random_state=1,
+            activation=activation,
+        )
         mlp.fit(X, y)
-        if activation == 'identity':
+        if activation == "identity":
             assert mlp.score(X, y) > 0.80
         else:
             # Non linear models perform much better than linear bottleneck:
             assert mlp.score(X, y) > 0.98
 
 
-@pytest.mark.parametrize('X,y', classification_datasets)
+@pytest.mark.parametrize("X,y", classification_datasets)
 def test_lbfgs_classification_maxfun(X, y):
     # Test lbfgs parameter max_fun.
     # It should independently limit the number of iterations for lbfgs.
     max_fun = 10
     # classification tests
     for activation in ACTIVATION_TYPES:
-        mlp = MLPClassifier(solver='lbfgs', hidden_layer_sizes=50,
-                            max_iter=150, max_fun=max_fun, shuffle=True,
-                            random_state=1, activation=activation)
+        mlp = MLPClassifier(
+            solver="lbfgs",
+            hidden_layer_sizes=50,
+            max_iter=150,
+            max_fun=max_fun,
+            shuffle=True,
+            random_state=1,
+            activation=activation,
+        )
         with pytest.warns(ConvergenceWarning):
             mlp.fit(X, y)
             assert max_fun >= mlp.n_iter_
 
 
-@pytest.mark.parametrize('X,y', regression_datasets)
+@pytest.mark.parametrize("X,y", regression_datasets)
 def test_lbfgs_regression_maxfun(X, y):
     # Test lbfgs parameter max_fun.
     # It should independently limit the number of iterations for lbfgs.
     max_fun = 10
     # regression tests
     for activation in ACTIVATION_TYPES:
-        mlp = MLPRegressor(solver='lbfgs', hidden_layer_sizes=50, tol=0.0,
-                           max_iter=150, max_fun=max_fun, shuffle=True,
-                           random_state=1, activation=activation)
+        mlp = MLPRegressor(
+            solver="lbfgs",
+            hidden_layer_sizes=50,
+            tol=0.0,
+            max_iter=150,
+            max_fun=max_fun,
+            shuffle=True,
+            random_state=1,
+            activation=activation,
+        )
         with pytest.warns(ConvergenceWarning):
             mlp.fit(X, y)
             assert max_fun >= mlp.n_iter_
 
-    mlp.max_fun = -1
-    with pytest.raises(ValueError):
-        mlp.fit(X, y)
-
 
 def test_learning_rate_warmstart():
     # Tests that warm_start reuse past solutions.
     X = [[3, 2], [1, 6], [5, 6], [-2, -4]]
     y = [1, 1, 1, 0]
     for learning_rate in ["invscaling", "constant"]:
-        mlp = MLPClassifier(solver='sgd', hidden_layer_sizes=4,
-                            learning_rate=learning_rate, max_iter=1,
-                            power_t=0.25, warm_start=True)
+        mlp = MLPClassifier(
+            solver="sgd",
+            hidden_layer_sizes=4,
+            learning_rate=learning_rate,
+            max_iter=1,
+            power_t=0.25,
+            warm_start=True,
+        )
         with ignore_warnings(category=ConvergenceWarning):
             mlp.fit(X, y)
             prev_eta = mlp._optimizer.learning_rate
             mlp.fit(X, y)
             post_eta = mlp._optimizer.learning_rate
 
-        if learning_rate == 'constant':
+        if learning_rate == "constant":
             assert prev_eta == post_eta
-        elif learning_rate == 'invscaling':
-            assert (mlp.learning_rate_init / pow(8 + 1, mlp.power_t) ==
-                         post_eta)
+        elif learning_rate == "invscaling":
+            assert mlp.learning_rate_init / pow(8 + 1, mlp.power_t) == post_eta
 
 
 def test_multilabel_classification():
     # Test that multi-label classification works as expected.
     # test fit method
-    X, y = make_multilabel_classification(n_samples=50, random_state=0,
-                                          return_indicator=True)
-    mlp = MLPClassifier(solver='lbfgs', hidden_layer_sizes=50, alpha=1e-5,
-                        max_iter=150, random_state=0, activation='logistic',
-                        learning_rate_init=0.2)
+    X, y = make_multilabel_classification(
+        n_samples=50, random_state=0, return_indicator=True
+    )
+    mlp = MLPClassifier(
+        solver="lbfgs",
+        hidden_layer_sizes=50,
+        alpha=1e-5,
+        max_iter=150,
+        random_state=0,
+        activation="logistic",
+        learning_rate_init=0.2,
+    )
     mlp.fit(X, y)
     assert mlp.score(X, y) > 0.97
 
     # test partial fit method
-    mlp = MLPClassifier(solver='sgd', hidden_layer_sizes=50, max_iter=150,
-                        random_state=0, activation='logistic', alpha=1e-5,
-                        learning_rate_init=0.2)
+    mlp = MLPClassifier(
+        solver="sgd",
+        hidden_layer_sizes=50,
+        max_iter=150,
+        random_state=0,
+        activation="logistic",
+        alpha=1e-5,
+        learning_rate_init=0.2,
+    )
     for i in range(100):
         mlp.partial_fit(X, y, classes=[0, 1, 2, 3, 4])
     assert mlp.score(X, y) > 0.9
 
-    # Make sure early stopping still work now that spliting is stratified by
+    # Make sure early stopping still work now that splitting is stratified by
     # default (it is disabled for multilabel classification)
     mlp = MLPClassifier(early_stopping=True)
     mlp.fit(X, y).predict(X)
@@ -350,9 +401,10 @@ def test_multilabel_classification():
 
 def test_multioutput_regression():
     # Test that multi-output regression works as expected
-    X, y = make_regression(n_samples=200, n_targets=5)
-    mlp = MLPRegressor(solver='lbfgs', hidden_layer_sizes=50, max_iter=200,
-                       random_state=1)
+    X, y = make_regression(n_samples=200, n_targets=5, random_state=11)
+    mlp = MLPRegressor(
+        solver="lbfgs", hidden_layer_sizes=50, max_iter=200, tol=1e-2, random_state=1
+    )
     mlp.fit(X, y)
     assert mlp.score(X, y) > 0.9
 
@@ -361,7 +413,7 @@ def test_partial_fit_classes_error():
     # Tests that passing different classes to partial_fit raises an error
     X = [[3, 2]]
     y = [0]
-    clf = MLPClassifier(solver='sgd')
+    clf = MLPClassifier(solver="sgd")
     clf.partial_fit(X, y, classes=[0, 1])
     with pytest.raises(ValueError):
         clf.partial_fit(X, y, classes=[1, 2])
@@ -372,14 +424,21 @@ def test_partial_fit_classification():
     # `partial_fit` should yield the same results as 'fit' for binary and
     # multi-class classification.
     for X, y in classification_datasets:
-        mlp = MLPClassifier(solver='sgd', max_iter=100, random_state=1,
-                            tol=0, alpha=1e-5, learning_rate_init=0.2)
+        mlp = MLPClassifier(
+            solver="sgd",
+            max_iter=100,
+            random_state=1,
+            tol=0,
+            alpha=1e-5,
+            learning_rate_init=0.2,
+        )
 
         with ignore_warnings(category=ConvergenceWarning):
             mlp.fit(X, y)
         pred1 = mlp.predict(X)
-        mlp = MLPClassifier(solver='sgd', random_state=1, alpha=1e-5,
-                            learning_rate_init=0.2)
+        mlp = MLPClassifier(
+            solver="sgd", random_state=1, alpha=1e-5, learning_rate_init=0.2
+        )
         for i in range(100):
             mlp.partial_fit(X, y, classes=np.unique(y))
         pred2 = mlp.predict(X)
@@ -392,8 +451,7 @@ def test_partial_fit_unseen_classes():
     # Tests for labeling errors in partial fit
 
     clf = MLPClassifier(random_state=0)
-    clf.partial_fit([[1], [2], [3]], ["a", "b", "c"],
-                    classes=["a", "b", "c", "d"])
+    clf.partial_fit([[1], [2], [3]], ["a", "b", "c"], classes=["a", "b", "c", "d"])
     clf.partial_fit([[4]], ["d"])
     assert clf.score([[1], [2], [3], [4]], ["a", "b", "c", "d"]) > 0
 
@@ -404,17 +462,28 @@ def test_partial_fit_regression():
     X = X_reg
     y = y_reg
 
-    for momentum in [0, .9]:
-        mlp = MLPRegressor(solver='sgd', max_iter=100, activation='relu',
-                           random_state=1, learning_rate_init=0.01,
-                           batch_size=X.shape[0], momentum=momentum)
-        with warnings.catch_warnings(record=True):
-            # catch convergence warning
+    for momentum in [0, 0.9]:
+        mlp = MLPRegressor(
+            solver="sgd",
+            max_iter=100,
+            activation="relu",
+            random_state=1,
+            learning_rate_init=0.01,
+            batch_size=X.shape[0],
+            momentum=momentum,
+        )
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", ConvergenceWarning)
             mlp.fit(X, y)
         pred1 = mlp.predict(X)
-        mlp = MLPRegressor(solver='sgd', activation='relu',
-                           learning_rate_init=0.01, random_state=1,
-                           batch_size=X.shape[0], momentum=momentum)
+        mlp = MLPRegressor(
+            solver="sgd",
+            activation="relu",
+            learning_rate_init=0.01,
+            random_state=1,
+            batch_size=X.shape[0],
+            momentum=momentum,
+        )
         for i in range(100):
             mlp.partial_fit(X, y)
 
@@ -431,43 +500,31 @@ def test_partial_fit_errors():
 
     # no classes passed
     with pytest.raises(ValueError):
-        MLPClassifier(solver='sgd').partial_fit(X, y, classes=[2])
+        MLPClassifier(solver="sgd").partial_fit(X, y, classes=[2])
 
     # lbfgs doesn't support partial_fit
-    assert not hasattr(MLPClassifier(solver='lbfgs'), 'partial_fit')
-
-
-@pytest.mark.parametrize(
-        "args",
-        [{'hidden_layer_sizes': -1},
-         {'max_iter': -1},
-         {'shuffle': 'true'},
-         {'alpha': -1},
-         {'learning_rate_init': -1},
-         {'momentum': 2},
-         {'momentum': -0.5},
-         {'nesterovs_momentum': 'invalid'},
-         {'early_stopping': 'invalid'},
-         {'validation_fraction': 1},
-         {'validation_fraction': -0.5},
-         {'beta_1': 1},
-         {'beta_1': -0.5},
-         {'beta_2': 1},
-         {'beta_2': -0.5},
-         {'epsilon': -0.5},
-         {'n_iter_no_change': -1},
-         {'solver': 'hadoken'},
-         {'learning_rate': 'converge'},
-         {'activation': 'cloak'}]
-)
-def test_params_errors(args):
-    # Test that invalid parameters raise value error
-    X = [[3, 2], [1, 6]]
-    y = [1, 0]
-    clf = MLPClassifier
-
-    with pytest.raises(ValueError):
-        clf(**args).fit(X, y)
+    assert not hasattr(MLPClassifier(solver="lbfgs"), "partial_fit")
+
+
+def test_nonfinite_params():
+    # Check that MLPRegressor throws ValueError when dealing with non-finite
+    # parameter values
+    rng = np.random.RandomState(0)
+    n_samples = 10
+    fmax = np.finfo(np.float64).max
+    X = fmax * rng.uniform(size=(n_samples, 2))
+    y = rng.standard_normal(size=n_samples)
+
+    clf = MLPRegressor()
+    msg = (
+        "Solver produced non-finite parameter weights. The input data may contain large"
+        " values and need to be preprocessed."
+    )
+    with pytest.raises(ValueError, match=msg):
+        with warnings.catch_warnings():
+            # RuntimeWarning: overflow encountered in square
+            warnings.simplefilter("ignore")
+            clf.fit(X, y)
 
 
 def test_predict_proba_binary():
@@ -475,8 +532,7 @@ def test_predict_proba_binary():
     X = X_digits_binary[:50]
     y = y_digits_binary[:50]
 
-    clf = MLPClassifier(hidden_layer_sizes=5, activation='logistic',
-                        random_state=1)
+    clf = MLPClassifier(hidden_layer_sizes=5, activation="logistic", random_state=1)
     with ignore_warnings(category=ConvergenceWarning):
         clf.fit(X, y)
     y_proba = clf.predict_proba(X)
@@ -518,12 +574,12 @@ def test_predict_proba_multiclass():
 def test_predict_proba_multilabel():
     # Test that predict_proba works as expected for multilabel.
     # Multilabel should not use softmax which makes probabilities sum to 1
-    X, Y = make_multilabel_classification(n_samples=50, random_state=0,
-                                          return_indicator=True)
+    X, Y = make_multilabel_classification(
+        n_samples=50, random_state=0, return_indicator=True
+    )
     n_samples, n_classes = Y.shape
 
-    clf = MLPClassifier(solver='lbfgs', hidden_layer_sizes=30,
-                        random_state=0)
+    clf = MLPClassifier(solver="lbfgs", hidden_layer_sizes=30, random_state=0)
     clf.fit(X, Y)
     y_proba = clf.predict_proba(X)
 
@@ -541,38 +597,53 @@ def test_predict_proba_multilabel():
 
 def test_shuffle():
     # Test that the shuffle parameter affects the training process (it should)
-    X, y = make_regression(n_samples=50, n_features=5, n_targets=1,
-                           random_state=0)
+    X, y = make_regression(n_samples=50, n_features=5, n_targets=1, random_state=0)
 
     # The coefficients will be identical if both do or do not shuffle
     for shuffle in [True, False]:
-        mlp1 = MLPRegressor(hidden_layer_sizes=1, max_iter=1, batch_size=1,
-                            random_state=0, shuffle=shuffle)
-        mlp2 = MLPRegressor(hidden_layer_sizes=1, max_iter=1, batch_size=1,
-                            random_state=0, shuffle=shuffle)
-        mlp1.fit(X, y)
-        mlp2.fit(X, y)
+        mlp1 = MLPRegressor(
+            hidden_layer_sizes=1,
+            max_iter=1,
+            batch_size=1,
+            random_state=0,
+            shuffle=shuffle,
+        )
+        mlp2 = MLPRegressor(
+            hidden_layer_sizes=1,
+            max_iter=1,
+            batch_size=1,
+            random_state=0,
+            shuffle=shuffle,
+        )
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", ConvergenceWarning)
+            mlp1.fit(X, y)
+            mlp2.fit(X, y)
 
         assert np.array_equal(mlp1.coefs_[0], mlp2.coefs_[0])
 
     # The coefficients will be slightly different if shuffle=True
-    mlp1 = MLPRegressor(hidden_layer_sizes=1, max_iter=1, batch_size=1,
-                        random_state=0, shuffle=True)
-    mlp2 = MLPRegressor(hidden_layer_sizes=1, max_iter=1, batch_size=1,
-                        random_state=0, shuffle=False)
-    mlp1.fit(X, y)
-    mlp2.fit(X, y)
+    mlp1 = MLPRegressor(
+        hidden_layer_sizes=1, max_iter=1, batch_size=1, random_state=0, shuffle=True
+    )
+    mlp2 = MLPRegressor(
+        hidden_layer_sizes=1, max_iter=1, batch_size=1, random_state=0, shuffle=False
+    )
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", ConvergenceWarning)
+        mlp1.fit(X, y)
+        mlp2.fit(X, y)
 
     assert not np.array_equal(mlp1.coefs_[0], mlp2.coefs_[0])
 
 
-def test_sparse_matrices():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sparse_matrices(csr_container):
     # Test that sparse and dense input matrices output the same results.
     X = X_digits_binary[:50]
     y = y_digits_binary[:50]
-    X_sparse = csr_matrix(X)
-    mlp = MLPClassifier(solver='lbfgs', hidden_layer_sizes=15,
-                        random_state=1)
+    X_sparse = csr_container(X)
+    mlp = MLPClassifier(solver="lbfgs", hidden_layer_sizes=15, random_state=1)
     mlp.fit(X, y)
     pred1 = mlp.predict(X)
     mlp.fit(X_sparse, y)
@@ -588,7 +659,7 @@ def test_tolerance():
     # It should force the solver to exit the loop when it converges.
     X = [[3, 2], [1, 6]]
     y = [1, 0]
-    clf = MLPClassifier(tol=0.5, max_iter=3000, solver='sgd')
+    clf = MLPClassifier(tol=0.5, max_iter=3000, solver="sgd")
     clf.fit(X, y)
     assert clf.max_iter > clf.n_iter_
 
@@ -597,8 +668,7 @@ def test_verbose_sgd():
     # Test verbose.
     X = [[3, 2], [1, 6]]
     y = [1, 0]
-    clf = MLPClassifier(solver='sgd', max_iter=2, verbose=10,
-                        hidden_layer_sizes=2)
+    clf = MLPClassifier(solver="sgd", max_iter=2, verbose=10, hidden_layer_sizes=2)
     old_stdout = sys.stdout
     sys.stdout = output = StringIO()
 
@@ -607,36 +677,49 @@ def test_verbose_sgd():
     clf.partial_fit(X, y)
 
     sys.stdout = old_stdout
-    assert 'Iteration' in output.getvalue()
+    assert "Iteration" in output.getvalue()
 
 
-def test_early_stopping():
+@pytest.mark.parametrize("MLPEstimator", [MLPClassifier, MLPRegressor])
+def test_early_stopping(MLPEstimator):
     X = X_digits_binary[:100]
     y = y_digits_binary[:100]
     tol = 0.2
-    clf = MLPClassifier(tol=tol, max_iter=3000, solver='sgd',
-                        early_stopping=True)
-    clf.fit(X, y)
-    assert clf.max_iter > clf.n_iter_
+    mlp_estimator = MLPEstimator(
+        tol=tol, max_iter=3000, solver="sgd", early_stopping=True
+    )
+    mlp_estimator.fit(X, y)
+    assert mlp_estimator.max_iter > mlp_estimator.n_iter_
+
+    assert mlp_estimator.best_loss_ is None
+    assert isinstance(mlp_estimator.validation_scores_, list)
 
-    valid_scores = clf.validation_scores_
-    best_valid_score = clf.best_validation_score_
+    valid_scores = mlp_estimator.validation_scores_
+    best_valid_score = mlp_estimator.best_validation_score_
     assert max(valid_scores) == best_valid_score
     assert best_valid_score + tol > valid_scores[-2]
     assert best_valid_score + tol > valid_scores[-1]
 
+    # check that the attributes `validation_scores_` and `best_validation_score_`
+    # are set to None when `early_stopping=False`
+    mlp_estimator = MLPEstimator(
+        tol=tol, max_iter=3000, solver="sgd", early_stopping=False
+    )
+    mlp_estimator.fit(X, y)
+    assert mlp_estimator.validation_scores_ is None
+    assert mlp_estimator.best_validation_score_ is None
+    assert mlp_estimator.best_loss_ is not None
+
 
 def test_adaptive_learning_rate():
     X = [[3, 2], [1, 6]]
     y = [1, 0]
-    clf = MLPClassifier(tol=0.5, max_iter=3000, solver='sgd',
-                        learning_rate='adaptive')
+    clf = MLPClassifier(tol=0.5, max_iter=3000, solver="sgd", learning_rate="adaptive")
     clf.fit(X, y)
     assert clf.max_iter > clf.n_iter_
     assert 1e-6 > clf._optimizer.learning_rate
 
 
-@ignore_warnings(category=RuntimeWarning)
 def test_warm_start():
     X = X_iris
     y = y_iris
@@ -648,17 +731,25 @@ def test_warm_start():
     y_5classes = np.array([0] * 30 + [1] * 30 + [2] * 30 + [3] * 30 + [4] * 30)
 
     # No error raised
-    clf = MLPClassifier(hidden_layer_sizes=2, solver='lbfgs',
-                        warm_start=True).fit(X, y)
+    clf = MLPClassifier(
+        hidden_layer_sizes=2, solver="lbfgs", warm_start=True, random_state=42, tol=1e-2
+    ).fit(X, y)
     clf.fit(X, y)
     clf.fit(X, y_3classes)
 
     for y_i in (y_2classes, y_3classes_alt, y_4classes, y_5classes):
-        clf = MLPClassifier(hidden_layer_sizes=2, solver='lbfgs',
-                            warm_start=True).fit(X, y)
-        message = ('warm_start can only be used where `y` has the same '
-                   'classes as in the previous call to fit.'
-                   ' Previously got [0 1 2], `y` has %s' % np.unique(y_i))
+        clf = MLPClassifier(
+            hidden_layer_sizes=2,
+            solver="lbfgs",
+            warm_start=True,
+            random_state=42,
+            tol=1e-2,
+        ).fit(X, y)
+        message = (
+            "warm_start can only be used where `y` has the same "
+            "classes as in the previous call to fit."
+            " Previously got [0 1 2], `y` has %s" % np.unique(y_i)
+        )
         with pytest.raises(ValueError, match=re.escape(message)):
             clf.fit(X, y_i)
 
@@ -672,12 +763,14 @@ def test_warm_start_full_iteration(MLPEstimator):
     X, y = X_iris, y_iris
     max_iter = 3
     clf = MLPEstimator(
-        hidden_layer_sizes=2, solver='sgd', warm_start=True, max_iter=max_iter
+        hidden_layer_sizes=2, solver="sgd", warm_start=True, max_iter=max_iter
     )
-    clf.fit(X, y)
-    assert max_iter == clf.n_iter_
-    clf.fit(X, y)
-    assert 2 * max_iter == clf.n_iter_
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", ConvergenceWarning)
+        clf.fit(X, y)
+        assert max_iter == clf.n_iter_
+        clf.fit(X, y)
+        assert max_iter == clf.n_iter_
 
 
 def test_n_iter_no_change():
@@ -690,8 +783,9 @@ def test_n_iter_no_change():
 
     # test multiple n_iter_no_change
     for n_iter_no_change in [2, 5, 10, 50, 100]:
-        clf = MLPClassifier(tol=tol, max_iter=max_iter, solver='sgd',
-                            n_iter_no_change=n_iter_no_change)
+        clf = MLPClassifier(
+            tol=tol, max_iter=max_iter, solver="sgd", n_iter_no_change=n_iter_no_change
+        )
         clf.fit(X, y)
 
         # validate n_iter_no_change
@@ -699,7 +793,7 @@ def test_n_iter_no_change():
         assert max_iter > clf.n_iter_
 
 
-@ignore_warnings(category=ConvergenceWarning)
+@pytest.mark.filterwarnings("ignore::sklearn.exceptions.ConvergenceWarning")
 def test_n_iter_no_change_inf():
     # test n_iter_no_change using binary data set
     # the fitting process should go to max_iter iterations
@@ -713,8 +807,9 @@ def test_n_iter_no_change_inf():
     # fit
     n_iter_no_change = np.inf
     max_iter = 3000
-    clf = MLPClassifier(tol=tol, max_iter=max_iter, solver='sgd',
-                        n_iter_no_change=n_iter_no_change)
+    clf = MLPClassifier(
+        tol=tol, max_iter=max_iter, solver="sgd", n_iter_no_change=n_iter_no_change
+    )
     clf.fit(X, y)
 
     # validate n_iter_no_change doesn't cause early stopping
@@ -731,23 +826,23 @@ def test_early_stopping_stratified():
 
     mlp = MLPClassifier(early_stopping=True)
     with pytest.raises(
-            ValueError,
-            match='The least populated class in y has only 1 member'):
+        ValueError, match="The least populated class in y has only 1 member"
+    ):
         mlp.fit(X, y)
 
 
 def test_mlp_classifier_dtypes_casting():
     # Compare predictions for different dtypes
-    mlp_64 = MLPClassifier(alpha=1e-5,
-                           hidden_layer_sizes=(5, 3),
-                           random_state=1, max_iter=50)
+    mlp_64 = MLPClassifier(
+        alpha=1e-5, hidden_layer_sizes=(5, 3), random_state=1, max_iter=100, tol=1e-1
+    )
     mlp_64.fit(X_digits[:300], y_digits[:300])
     pred_64 = mlp_64.predict(X_digits[300:])
     proba_64 = mlp_64.predict_proba(X_digits[300:])
 
-    mlp_32 = MLPClassifier(alpha=1e-5,
-                           hidden_layer_sizes=(5, 3),
-                           random_state=1, max_iter=50)
+    mlp_32 = MLPClassifier(
+        alpha=1e-5, hidden_layer_sizes=(5, 3), random_state=1, max_iter=100, tol=1e-1
+    )
     mlp_32.fit(X_digits[:300].astype(np.float32), y_digits[:300])
     pred_32 = mlp_32.predict(X_digits[300:].astype(np.float32))
     proba_32 = mlp_32.predict_proba(X_digits[300:].astype(np.float32))
@@ -757,38 +852,243 @@ def test_mlp_classifier_dtypes_casting():
 
 
 def test_mlp_regressor_dtypes_casting():
-    mlp_64 = MLPRegressor(alpha=1e-5,
-                          hidden_layer_sizes=(5, 3),
-                          random_state=1, max_iter=50)
+    mlp_64 = MLPRegressor(
+        alpha=1e-5, hidden_layer_sizes=(5, 3), random_state=1, max_iter=150, tol=1e-3
+    )
     mlp_64.fit(X_digits[:300], y_digits[:300])
     pred_64 = mlp_64.predict(X_digits[300:])
 
-    mlp_32 = MLPRegressor(alpha=1e-5,
-                          hidden_layer_sizes=(5, 3),
-                          random_state=1, max_iter=50)
+    mlp_32 = MLPRegressor(
+        alpha=1e-5, hidden_layer_sizes=(5, 3), random_state=1, max_iter=150, tol=1e-3
+    )
     mlp_32.fit(X_digits[:300].astype(np.float32), y_digits[:300])
     pred_32 = mlp_32.predict(X_digits[300:].astype(np.float32))
 
-    assert_allclose(pred_64, pred_32, rtol=1e-04)
+    assert_allclose(pred_64, pred_32, rtol=5e-04)
 
 
-@pytest.mark.parametrize('dtype', [np.float32, np.float64])
-@pytest.mark.parametrize('Estimator', [MLPClassifier, MLPRegressor])
+@pytest.mark.parametrize("dtype", [np.float32, np.float64])
+@pytest.mark.parametrize("Estimator", [MLPClassifier, MLPRegressor])
 def test_mlp_param_dtypes(dtype, Estimator):
     # Checks if input dtype is used for network parameters
     # and predictions
     X, y = X_digits.astype(dtype), y_digits
-    mlp = Estimator(alpha=1e-5,
-                    hidden_layer_sizes=(5, 3),
-                    random_state=1, max_iter=50)
+    mlp = Estimator(
+        alpha=1e-5, hidden_layer_sizes=(5, 3), random_state=1, max_iter=50, tol=1e-1
+    )
     mlp.fit(X[:300], y[:300])
     pred = mlp.predict(X[300:])
 
-    assert all([intercept.dtype == dtype
-                for intercept in mlp.intercepts_])
+    assert all([intercept.dtype == dtype for intercept in mlp.intercepts_])
 
-    assert all([coef.dtype == dtype
-                for coef in mlp.coefs_])
+    assert all([coef.dtype == dtype for coef in mlp.coefs_])
 
     if Estimator == MLPRegressor:
         assert pred.dtype == dtype
+
+
+def test_mlp_loading_from_joblib_partial_fit(tmp_path):
+    """Loading from MLP and partial fitting updates weights. Non-regression
+    test for #19626."""
+    pre_trained_estimator = MLPRegressor(
+        hidden_layer_sizes=(42,), random_state=42, learning_rate_init=0.01, max_iter=200
+    )
+    features, target = [[2]], [4]
+
+    # Fit on x=2, y=4
+    pre_trained_estimator.fit(features, target)
+
+    # dump and load model
+    pickled_file = tmp_path / "mlp.pkl"
+    joblib.dump(pre_trained_estimator, pickled_file)
+    load_estimator = joblib.load(pickled_file)
+
+    # Train for a more epochs on point x=2, y=1
+    fine_tune_features, fine_tune_target = [[2]], [1]
+
+    for _ in range(200):
+        load_estimator.partial_fit(fine_tune_features, fine_tune_target)
+
+    # finetuned model learned the new target
+    predicted_value = load_estimator.predict(fine_tune_features)
+    assert_allclose(predicted_value, fine_tune_target, rtol=1e-4)
+
+
+@pytest.mark.parametrize("Estimator", [MLPClassifier, MLPRegressor])
+def test_preserve_feature_names(Estimator):
+    """Check that feature names are preserved when early stopping is enabled.
+
+    Feature names are required for consistency checks during scoring.
+
+    Non-regression test for gh-24846
+    """
+    pd = pytest.importorskip("pandas")
+    rng = np.random.RandomState(0)
+
+    X = pd.DataFrame(data=rng.randn(10, 2), columns=["colname_a", "colname_b"])
+    y = pd.Series(data=np.full(10, 1), name="colname_y")
+
+    model = Estimator(early_stopping=True, validation_fraction=0.2)
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", UserWarning)
+        model.fit(X, y)
+
+
+@pytest.mark.parametrize("MLPEstimator", [MLPClassifier, MLPRegressor])
+def test_mlp_warm_start_with_early_stopping(MLPEstimator):
+    """Check that early stopping works with warm start."""
+    mlp = MLPEstimator(
+        max_iter=10, random_state=0, warm_start=True, early_stopping=True
+    )
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", ConvergenceWarning)
+        mlp.fit(X_iris, y_iris)
+        n_validation_scores = len(mlp.validation_scores_)
+        mlp.set_params(max_iter=20)
+        mlp.fit(X_iris, y_iris)
+    assert len(mlp.validation_scores_) > n_validation_scores
+
+
+@pytest.mark.parametrize("MLPEstimator", [MLPClassifier, MLPRegressor])
+@pytest.mark.parametrize("solver", ["sgd", "adam", "lbfgs"])
+def test_mlp_warm_start_no_convergence(MLPEstimator, solver):
+    """Check that we stop the number of iteration at `max_iter` when warm starting.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/24764
+    """
+    model = MLPEstimator(
+        solver=solver,
+        warm_start=True,
+        early_stopping=False,
+        max_iter=10,
+        n_iter_no_change=np.inf,
+        random_state=0,
+    )
+
+    with pytest.warns(ConvergenceWarning):
+        model.fit(X_iris, y_iris)
+    assert model.n_iter_ == 10
+
+    model.set_params(max_iter=20)
+    with pytest.warns(ConvergenceWarning):
+        model.fit(X_iris, y_iris)
+    assert model.n_iter_ == 20
+
+
+@pytest.mark.parametrize("MLPEstimator", [MLPClassifier, MLPRegressor])
+def test_mlp_partial_fit_after_fit(MLPEstimator):
+    """Check partial fit does not fail after fit when early_stopping=True.
+
+    Non-regression test for gh-25693.
+    """
+    mlp = MLPEstimator(early_stopping=True, random_state=0).fit(X_iris, y_iris)
+
+    msg = "partial_fit does not support early_stopping=True"
+    with pytest.raises(ValueError, match=msg):
+        mlp.partial_fit(X_iris, y_iris)
+
+
+def test_mlp_diverging_loss():
+    """Test that a diverging model does not raise errors when early stopping is enabled.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/29504
+    """
+    mlp = MLPRegressor(
+        hidden_layer_sizes=100,
+        activation="identity",
+        solver="sgd",
+        alpha=0.0001,
+        learning_rate="constant",
+        learning_rate_init=1,
+        shuffle=True,
+        max_iter=20,
+        early_stopping=True,
+        n_iter_no_change=10,
+        random_state=0,
+    )
+
+    with warnings.catch_warnings():
+        # RuntimeWarning: overflow encountered in matmul
+        # ConvergenceWarning: Stochastic Optimizer: Maximum iteration
+        warnings.simplefilter("ignore", RuntimeWarning)
+        warnings.simplefilter("ignore", ConvergenceWarning)
+        mlp.fit(X_iris, y_iris)
+
+    # In python, float("nan") != float("nan")
+    assert str(mlp.validation_scores_[-1]) == str(np.nan)
+    assert isinstance(mlp.validation_scores_[-1], float)
+
+
+def test_mlp_sample_weight_with_early_stopping():
+    # Test code path for inner validation set splitting.
+    X, y = make_regression(
+        n_samples=100,
+        n_features=2,
+        n_informative=2,
+        random_state=42,
+    )
+    sw = np.ones_like(y)
+    params = dict(
+        hidden_layer_sizes=10,
+        solver="adam",
+        early_stopping=True,
+        tol=1e-2,
+        learning_rate_init=0.01,
+        batch_size=10,
+        random_state=42,
+    )
+    m1 = MLPRegressor(
+        **params,
+    )
+    m1.fit(X, y, sample_weight=sw)
+
+    m2 = MLPRegressor(**params).fit(X, y, sample_weight=None)
+    assert_allclose(m1.predict(X), m2.predict(X))
+
+
+def test_mlp_vs_poisson_glm_equivalent(global_random_seed):
+    """Test MLP with Poisson loss and no hidden layer equals GLM."""
+    n = 100
+    rng = np.random.default_rng(global_random_seed)
+    X = np.linspace(0, 1, n)
+    y = rng.poisson(np.exp(X + 1))
+    X = X.reshape(n, -1)
+    glm = PoissonRegressor(alpha=0, tol=1e-7).fit(X, y)
+    # Unfortunately, we can't set a zero hidden_layer_size, so we use a trick by using
+    # just one hidden layer node with an identity activation. Coefficients will
+    # therefore be different, but predictions are the same.
+    mlp = MLPRegressor(
+        loss="poisson",
+        hidden_layer_sizes=(1,),
+        activation="identity",
+        alpha=0,
+        solver="lbfgs",
+        tol=1e-7,
+        random_state=np.random.RandomState(global_random_seed + 1),
+    ).fit(X, y)
+
+    assert_allclose(mlp.predict(X), glm.predict(X), rtol=1e-4)
+
+    # The same does not work with the squared error because the output activation is
+    # the identity instead of the exponential.
+    mlp = MLPRegressor(
+        loss="squared_error",
+        hidden_layer_sizes=(1,),
+        activation="identity",
+        alpha=0,
+        solver="lbfgs",
+        tol=1e-7,
+        random_state=np.random.RandomState(global_random_seed + 1),
+    ).fit(X, y)
+    assert not np.allclose(mlp.predict(X), glm.predict(X), rtol=1e-4)
+
+
+def test_minimum_input_sample_size():
+    """Check error message when the validation set is too small."""
+    X, y = make_regression(n_samples=2, n_features=5, random_state=0)
+    model = MLPRegressor(early_stopping=True, random_state=0)
+    with pytest.raises(ValueError, match="The validation set is too small"):
+        model.fit(X, y)
diff --git a/sklearn/neural_network/tests/test_rbm.py b/sklearn/neural_network/tests/test_rbm.py
index 724868dc8bba9..8211c9735923d 100644
--- a/sklearn/neural_network/tests/test_rbm.py
+++ b/sklearn/neural_network/tests/test_rbm.py
@@ -1,15 +1,18 @@
-import sys
 import re
-import pytest
+import sys
+from io import StringIO
 
 import numpy as np
-from scipy.sparse import csc_matrix, csr_matrix, lil_matrix
-from sklearn.utils._testing import (assert_almost_equal, assert_array_equal,
-                                    assert_allclose)
+import pytest
 
 from sklearn.datasets import load_digits
-from io import StringIO
 from sklearn.neural_network import BernoulliRBM
+from sklearn.utils._testing import (
+    assert_allclose,
+    assert_almost_equal,
+    assert_array_equal,
+)
+from sklearn.utils.fixes import CSC_CONTAINERS, CSR_CONTAINERS, LIL_CONTAINERS
 from sklearn.utils.validation import assert_all_finite
 
 Xdigits, _ = load_digits(return_X_y=True)
@@ -20,11 +23,12 @@
 def test_fit():
     X = Xdigits.copy()
 
-    rbm = BernoulliRBM(n_components=64, learning_rate=0.1,
-                       batch_size=10, n_iter=7, random_state=9)
+    rbm = BernoulliRBM(
+        n_components=64, learning_rate=0.1, batch_size=10, n_iter=7, random_state=9
+    )
     rbm.fit(X)
 
-    assert_almost_equal(rbm.score_samples(X).mean(), -21., decimal=0)
+    assert_almost_equal(rbm.score_samples(X).mean(), -21.0, decimal=0)
 
     # in-place tricks shouldn't have modified X
     assert_array_equal(X, Xdigits)
@@ -32,8 +36,9 @@ def test_fit():
 
 def test_partial_fit():
     X = Xdigits.copy()
-    rbm = BernoulliRBM(n_components=64, learning_rate=0.1,
-                       batch_size=20, random_state=9)
+    rbm = BernoulliRBM(
+        n_components=64, learning_rate=0.1, batch_size=20, random_state=9
+    )
     n_samples = X.shape[0]
     n_batches = int(np.ceil(float(n_samples) / rbm.batch_size))
     batch_slices = np.array_split(X, n_batches)
@@ -42,14 +47,13 @@ def test_partial_fit():
         for batch in batch_slices:
             rbm.partial_fit(batch)
 
-    assert_almost_equal(rbm.score_samples(X).mean(), -21., decimal=0)
+    assert_almost_equal(rbm.score_samples(X).mean(), -21.0, decimal=0)
     assert_array_equal(X, Xdigits)
 
 
 def test_transform():
     X = Xdigits[:100]
-    rbm1 = BernoulliRBM(n_components=16, batch_size=5,
-                        n_iter=5, random_state=42)
+    rbm1 = BernoulliRBM(n_components=16, batch_size=5, n_iter=5, random_state=42)
     rbm1.fit(X)
 
     Xt1 = rbm1.transform(X)
@@ -58,35 +62,37 @@ def test_transform():
     assert_array_equal(Xt1, Xt2)
 
 
-def test_small_sparse():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_small_sparse(csr_container):
     # BernoulliRBM should work on small sparse matrices.
-    X = csr_matrix(Xdigits[:4])
-    BernoulliRBM().fit(X)       # no exception
+    X = csr_container(Xdigits[:4])
+    BernoulliRBM().fit(X)  # no exception
 
 
-def test_small_sparse_partial_fit():
-    for sparse in [csc_matrix, csr_matrix]:
-        X_sparse = sparse(Xdigits[:100])
-        X = Xdigits[:100].copy()
+@pytest.mark.parametrize("sparse_container", CSC_CONTAINERS + CSR_CONTAINERS)
+def test_small_sparse_partial_fit(sparse_container):
+    X_sparse = sparse_container(Xdigits[:100])
+    X = Xdigits[:100].copy()
 
-        rbm1 = BernoulliRBM(n_components=64, learning_rate=0.1,
-                            batch_size=10, random_state=9)
-        rbm2 = BernoulliRBM(n_components=64, learning_rate=0.1,
-                            batch_size=10, random_state=9)
+    rbm1 = BernoulliRBM(
+        n_components=64, learning_rate=0.1, batch_size=10, random_state=9
+    )
+    rbm2 = BernoulliRBM(
+        n_components=64, learning_rate=0.1, batch_size=10, random_state=9
+    )
 
-        rbm1.partial_fit(X_sparse)
-        rbm2.partial_fit(X)
+    rbm1.partial_fit(X_sparse)
+    rbm2.partial_fit(X)
 
-        assert_almost_equal(rbm1.score_samples(X).mean(),
-                            rbm2.score_samples(X).mean(),
-                            decimal=0)
+    assert_almost_equal(
+        rbm1.score_samples(X).mean(), rbm2.score_samples(X).mean(), decimal=0
+    )
 
 
 def test_sample_hiddens():
     rng = np.random.RandomState(0)
     X = Xdigits[:100]
-    rbm1 = BernoulliRBM(n_components=2, batch_size=5,
-                        n_iter=5, random_state=42)
+    rbm1 = BernoulliRBM(n_components=2, batch_size=5, n_iter=5, random_state=42)
     rbm1.fit(X)
 
     h = rbm1._mean_hiddens(X[0])
@@ -95,33 +101,31 @@ def test_sample_hiddens():
     assert_almost_equal(h, hs, decimal=1)
 
 
-def test_fit_gibbs():
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+def test_fit_gibbs(csc_container):
+    # XXX: this test is very seed-dependent! It probably needs to be rewritten.
+
     # Gibbs on the RBM hidden layer should be able to recreate [[0], [1]]
     # from the same input
     rng = np.random.RandomState(42)
-    X = np.array([[0.], [1.]])
-    rbm1 = BernoulliRBM(n_components=2, batch_size=2,
-                        n_iter=42, random_state=rng)
+    X = np.array([[0.0], [1.0]])
+    rbm1 = BernoulliRBM(n_components=2, batch_size=2, n_iter=42, random_state=rng)
     # you need that much iters
     rbm1.fit(X)
-    assert_almost_equal(rbm1.components_,
-                        np.array([[0.02649814], [0.02009084]]), decimal=4)
+    assert_almost_equal(
+        rbm1.components_, np.array([[0.02649814], [0.02009084]]), decimal=4
+    )
     assert_almost_equal(rbm1.gibbs(X), X)
-    return rbm1
-
 
-def test_fit_gibbs_sparse():
     # Gibbs on the RBM hidden layer should be able to recreate [[0], [1]] from
     # the same input even when the input is sparse, and test against non-sparse
-    rbm1 = test_fit_gibbs()
     rng = np.random.RandomState(42)
-    from scipy.sparse import csc_matrix
-    X = csc_matrix([[0.], [1.]])
-    rbm2 = BernoulliRBM(n_components=2, batch_size=2,
-                        n_iter=42, random_state=rng)
+    X = csc_container([[0.0], [1.0]])
+    rbm2 = BernoulliRBM(n_components=2, batch_size=2, n_iter=42, random_state=rng)
     rbm2.fit(X)
-    assert_almost_equal(rbm2.components_,
-                        np.array([[0.02649814], [0.02009084]]), decimal=4)
+    assert_almost_equal(
+        rbm2.components_, np.array([[0.02649814], [0.02009084]]), decimal=4
+    )
     assert_almost_equal(rbm2.gibbs(X), X.toarray())
     assert_almost_equal(rbm1.components_, rbm2.components_)
 
@@ -130,8 +134,7 @@ def test_gibbs_smoke():
     # Check if we don't get NaNs sampling the full digits dataset.
     # Also check that sampling again will yield different results.
     X = Xdigits
-    rbm1 = BernoulliRBM(n_components=42, batch_size=40,
-                        n_iter=20, random_state=42)
+    rbm1 = BernoulliRBM(n_components=42, batch_size=40, n_iter=20, random_state=42)
     rbm1.fit(X)
     X_sampled = rbm1.gibbs(X)
     assert_all_finite(X_sampled)
@@ -139,14 +142,14 @@ def test_gibbs_smoke():
     assert np.all((X_sampled != X_sampled2).max(axis=1))
 
 
-def test_score_samples():
+@pytest.mark.parametrize("lil_containers", LIL_CONTAINERS)
+def test_score_samples(lil_containers):
     # Test score_samples (pseudo-likelihood) method.
     # Assert that pseudo-likelihood is computed without clipping.
     # See Fabian's blog, http://bit.ly/1iYefRk
     rng = np.random.RandomState(42)
     X = np.vstack([np.zeros(1000), np.ones(1000)])
-    rbm1 = BernoulliRBM(n_components=10, batch_size=2,
-                        n_iter=10, random_state=rng)
+    rbm1 = BernoulliRBM(n_components=10, batch_size=2, n_iter=10, random_state=rng)
     rbm1.fit(X)
     assert (rbm1.score_samples(X) < -300).all()
 
@@ -155,12 +158,12 @@ def test_score_samples():
     rbm1.random_state = 42
     d_score = rbm1.score_samples(X)
     rbm1.random_state = 42
-    s_score = rbm1.score_samples(lil_matrix(X))
+    s_score = rbm1.score_samples(lil_containers(X))
     assert_almost_equal(d_score, s_score)
 
     # Test numerical stability (#2785): would previously generate infinities
     # and crash with an exception.
-    with np.errstate(under='ignore'):
+    with np.errstate(under="ignore"):
         rbm1.score_samples([np.arange(1000) * 100])
 
 
@@ -174,60 +177,75 @@ def test_rbm_verbose():
         sys.stdout = old_stdout
 
 
-def test_sparse_and_verbose():
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+def test_sparse_and_verbose(csc_container):
     # Make sure RBM works with sparse input when verbose=True
     old_stdout = sys.stdout
     sys.stdout = StringIO()
-    from scipy.sparse import csc_matrix
-    X = csc_matrix([[0.], [1.]])
-    rbm = BernoulliRBM(n_components=2, batch_size=2, n_iter=1,
-                       random_state=42, verbose=True)
+
+    X = csc_container([[0.0], [1.0]])
+    rbm = BernoulliRBM(
+        n_components=2, batch_size=2, n_iter=1, random_state=42, verbose=True
+    )
     try:
         rbm.fit(X)
         s = sys.stdout.getvalue()
         # make sure output is sound
-        assert re.match(r"\[BernoulliRBM\] Iteration 1,"
-                        r" pseudo-likelihood = -?(\d)+(\.\d+)?,"
-                        r" time = (\d|\.)+s", s)
+        assert re.match(
+            r"\[BernoulliRBM\] Iteration 1,"
+            r" pseudo-likelihood = -?(\d)+(\.\d+)?,"
+            r" time = (\d|\.)+s",
+            s,
+        )
     finally:
         sys.stdout = old_stdout
 
 
-@pytest.mark.parametrize("dtype_in, dtype_out", [
-    (np.float32, np.float32),
-    (np.float64, np.float64),
-    (int, np.float64)])
+@pytest.mark.parametrize(
+    "dtype_in, dtype_out",
+    [(np.float32, np.float32), (np.float64, np.float64), (int, np.float64)],
+)
 def test_transformer_dtypes_casting(dtype_in, dtype_out):
     X = Xdigits[:100].astype(dtype_in)
-    rbm = BernoulliRBM(n_components=16, batch_size=5, n_iter=5,
-                       random_state=42)
+    rbm = BernoulliRBM(n_components=16, batch_size=5, n_iter=5, random_state=42)
     Xt = rbm.fit_transform(X)
 
     # dtype_in and dtype_out should be consistent
-    assert Xt.dtype == dtype_out, ('transform dtype: {} - original dtype: {}'
-                                   .format(Xt.dtype, X.dtype))
+    assert Xt.dtype == dtype_out, "transform dtype: {} - original dtype: {}".format(
+        Xt.dtype, X.dtype
+    )
 
 
 def test_convergence_dtype_consistency():
     # float 64 transformer
     X_64 = Xdigits[:100].astype(np.float64)
-    rbm_64 = BernoulliRBM(n_components=16, batch_size=5, n_iter=5,
-                          random_state=42)
+    rbm_64 = BernoulliRBM(n_components=16, batch_size=5, n_iter=5, random_state=42)
     Xt_64 = rbm_64.fit_transform(X_64)
 
     # float 32 transformer
     X_32 = Xdigits[:100].astype(np.float32)
-    rbm_32 = BernoulliRBM(n_components=16, batch_size=5, n_iter=5,
-                          random_state=42)
+    rbm_32 = BernoulliRBM(n_components=16, batch_size=5, n_iter=5, random_state=42)
     Xt_32 = rbm_32.fit_transform(X_32)
 
     # results and attributes should be close enough in 32 bit and 64 bit
-    assert_allclose(Xt_64, Xt_32,
-                    rtol=1e-06, atol=0)
-    assert_allclose(rbm_64.intercept_hidden_, rbm_32.intercept_hidden_,
-                    rtol=1e-06, atol=0)
-    assert_allclose(rbm_64.intercept_visible_, rbm_32.intercept_visible_,
-                    rtol=1e-05, atol=0)
-    assert_allclose(rbm_64.components_, rbm_32.components_,
-                    rtol=1e-03, atol=0)
+    assert_allclose(Xt_64, Xt_32, rtol=1e-06, atol=0)
+    assert_allclose(
+        rbm_64.intercept_hidden_, rbm_32.intercept_hidden_, rtol=1e-06, atol=0
+    )
+    assert_allclose(
+        rbm_64.intercept_visible_, rbm_32.intercept_visible_, rtol=1e-05, atol=0
+    )
+    assert_allclose(rbm_64.components_, rbm_32.components_, rtol=1e-03, atol=0)
     assert_allclose(rbm_64.h_samples_, rbm_32.h_samples_)
+
+
+@pytest.mark.parametrize("method", ["fit", "partial_fit"])
+def test_feature_names_out(method):
+    """Check `get_feature_names_out` for `BernoulliRBM`."""
+    n_components = 10
+    rbm = BernoulliRBM(n_components=n_components)
+    getattr(rbm, method)(Xdigits)
+
+    names = rbm.get_feature_names_out()
+    expected_names = [f"bernoullirbm{i}" for i in range(n_components)]
+    assert_array_equal(expected_names, names)
diff --git a/sklearn/neural_network/tests/test_stochastic_optimizers.py b/sklearn/neural_network/tests/test_stochastic_optimizers.py
index 253dfd175d024..58a9f0c7dda13 100644
--- a/sklearn/neural_network/tests/test_stochastic_optimizers.py
+++ b/sklearn/neural_network/tests/test_stochastic_optimizers.py
@@ -1,79 +1,84 @@
 import numpy as np
 
-from sklearn.neural_network._stochastic_optimizers import (BaseOptimizer,
-                                                           SGDOptimizer,
-                                                           AdamOptimizer)
+from sklearn.neural_network._stochastic_optimizers import (
+    AdamOptimizer,
+    BaseOptimizer,
+    SGDOptimizer,
+)
 from sklearn.utils._testing import assert_array_equal
 
-
 shapes = [(4, 6), (6, 8), (7, 8, 9)]
 
 
 def test_base_optimizer():
-    params = [np.zeros(shape) for shape in shapes]
-
-    for lr in [10 ** i for i in range(-3, 4)]:
-        optimizer = BaseOptimizer(params, lr)
-        assert optimizer.trigger_stopping('', False)
+    for lr in [10**i for i in range(-3, 4)]:
+        optimizer = BaseOptimizer(lr)
+        assert optimizer.trigger_stopping("", False)
 
 
 def test_sgd_optimizer_no_momentum():
     params = [np.zeros(shape) for shape in shapes]
+    rng = np.random.RandomState(0)
 
-    for lr in [10 ** i for i in range(-3, 4)]:
+    for lr in [10**i for i in range(-3, 4)]:
         optimizer = SGDOptimizer(params, lr, momentum=0, nesterov=False)
-        grads = [np.random.random(shape) for shape in shapes]
+        grads = [rng.random_sample(shape) for shape in shapes]
         expected = [param - lr * grad for param, grad in zip(params, grads)]
-        optimizer.update_params(grads)
+        optimizer.update_params(params, grads)
 
-        for exp, param in zip(expected, optimizer.params):
+        for exp, param in zip(expected, params):
             assert_array_equal(exp, param)
 
 
 def test_sgd_optimizer_momentum():
     params = [np.zeros(shape) for shape in shapes]
     lr = 0.1
+    rng = np.random.RandomState(0)
 
     for momentum in np.arange(0.5, 0.9, 0.1):
         optimizer = SGDOptimizer(params, lr, momentum=momentum, nesterov=False)
-        velocities = [np.random.random(shape) for shape in shapes]
+        velocities = [rng.random_sample(shape) for shape in shapes]
         optimizer.velocities = velocities
-        grads = [np.random.random(shape) for shape in shapes]
-        updates = [momentum * velocity - lr * grad
-                   for velocity, grad in zip(velocities, grads)]
+        grads = [rng.random_sample(shape) for shape in shapes]
+        updates = [
+            momentum * velocity - lr * grad for velocity, grad in zip(velocities, grads)
+        ]
         expected = [param + update for param, update in zip(params, updates)]
-        optimizer.update_params(grads)
+        optimizer.update_params(params, grads)
 
-        for exp, param in zip(expected, optimizer.params):
+        for exp, param in zip(expected, params):
             assert_array_equal(exp, param)
 
 
 def test_sgd_optimizer_trigger_stopping():
     params = [np.zeros(shape) for shape in shapes]
     lr = 2e-6
-    optimizer = SGDOptimizer(params, lr, lr_schedule='adaptive')
-    assert not optimizer.trigger_stopping('', False)
+    optimizer = SGDOptimizer(params, lr, lr_schedule="adaptive")
+    assert not optimizer.trigger_stopping("", False)
     assert lr / 5 == optimizer.learning_rate
-    assert optimizer.trigger_stopping('', False)
+    assert optimizer.trigger_stopping("", False)
 
 
 def test_sgd_optimizer_nesterovs_momentum():
     params = [np.zeros(shape) for shape in shapes]
     lr = 0.1
+    rng = np.random.RandomState(0)
 
     for momentum in np.arange(0.5, 0.9, 0.1):
         optimizer = SGDOptimizer(params, lr, momentum=momentum, nesterov=True)
-        velocities = [np.random.random(shape) for shape in shapes]
+        velocities = [rng.random_sample(shape) for shape in shapes]
         optimizer.velocities = velocities
-        grads = [np.random.random(shape) for shape in shapes]
-        updates = [momentum * velocity - lr * grad
-                   for velocity, grad in zip(velocities, grads)]
-        updates = [momentum * update - lr * grad
-                   for update, grad in zip(updates, grads)]
+        grads = [rng.random_sample(shape) for shape in shapes]
+        updates = [
+            momentum * velocity - lr * grad for velocity, grad in zip(velocities, grads)
+        ]
+        updates = [
+            momentum * update - lr * grad for update, grad in zip(updates, grads)
+        ]
         expected = [param + update for param, update in zip(params, updates)]
-        optimizer.update_params(grads)
+        optimizer.update_params(params, grads)
 
-        for exp, param in zip(expected, optimizer.params):
+        for exp, param in zip(expected, params):
             assert_array_equal(exp, param)
 
 
@@ -81,28 +86,27 @@ def test_adam_optimizer():
     params = [np.zeros(shape) for shape in shapes]
     lr = 0.001
     epsilon = 1e-8
+    rng = np.random.RandomState(0)
 
     for beta_1 in np.arange(0.9, 1.0, 0.05):
         for beta_2 in np.arange(0.995, 1.0, 0.001):
             optimizer = AdamOptimizer(params, lr, beta_1, beta_2, epsilon)
-            ms = [np.random.random(shape) for shape in shapes]
-            vs = [np.random.random(shape) for shape in shapes]
+            ms = [rng.random_sample(shape) for shape in shapes]
+            vs = [rng.random_sample(shape) for shape in shapes]
             t = 10
             optimizer.ms = ms
             optimizer.vs = vs
             optimizer.t = t - 1
-            grads = [np.random.random(shape) for shape in shapes]
-
-            ms = [beta_1 * m + (1 - beta_1) * grad
-                  for m, grad in zip(ms, grads)]
-            vs = [beta_2 * v + (1 - beta_2) * (grad ** 2)
-                  for v, grad in zip(vs, grads)]
-            learning_rate = lr * np.sqrt(1 - beta_2 ** t) / (1 - beta_1**t)
-            updates = [-learning_rate * m / (np.sqrt(v) + epsilon)
-                       for m, v in zip(ms, vs)]
-            expected = [param + update
-                        for param, update in zip(params, updates)]
-
-            optimizer.update_params(grads)
-            for exp, param in zip(expected, optimizer.params):
+            grads = [rng.random_sample(shape) for shape in shapes]
+
+            ms = [beta_1 * m + (1 - beta_1) * grad for m, grad in zip(ms, grads)]
+            vs = [beta_2 * v + (1 - beta_2) * (grad**2) for v, grad in zip(vs, grads)]
+            learning_rate = lr * np.sqrt(1 - beta_2**t) / (1 - beta_1**t)
+            updates = [
+                -learning_rate * m / (np.sqrt(v) + epsilon) for m, v in zip(ms, vs)
+            ]
+            expected = [param + update for param, update in zip(params, updates)]
+
+            optimizer.update_params(params, grads)
+            for exp, param in zip(expected, params):
                 assert_array_equal(exp, param)
diff --git a/sklearn/pipeline.py b/sklearn/pipeline.py
index f466b735c4fa6..b291d970b1c79 100644
--- a/sklearn/pipeline.py
+++ b/sklearn/pipeline.py
@@ -1,56 +1,151 @@
-"""
-The :mod:`sklearn.pipeline` module implements utilities to build a composite
-estimator, as a chain of transforms and estimators.
-"""
-# Author: Edouard Duchesnay
-#         Gael Varoquaux
-#         Virgile Fritsch
-#         Alexandre Gramfort
-#         Lars Buitinck
-# License: BSD
-
-from collections import defaultdict
-from itertools import islice
+"""Utilities to build a composite estimator as a chain of transforms and estimators."""
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+import warnings
+from collections import Counter, defaultdict
+from contextlib import contextmanager
+from copy import deepcopy
+from itertools import chain, islice
 
 import numpy as np
 from scipy import sparse
-from joblib import Parallel
-
-from .base import clone, TransformerMixin
-from .utils._estimator_html_repr import _VisualBlock
-from .utils.metaestimators import if_delegate_has_method
-from .utils import (
-    Bunch,
-    _print_elapsed_time,
+
+from .base import TransformerMixin, _fit_context, clone
+from .exceptions import NotFittedError
+from .preprocessing import FunctionTransformer
+from .utils import Bunch
+from .utils._metadata_requests import METHODS
+from .utils._param_validation import HasMethods, Hidden
+from .utils._repr_html.estimator import _VisualBlock
+from .utils._set_output import (
+    _get_container_adapter,
+    _safe_set_output,
 )
-from .utils.deprecation import deprecated
-from .utils._tags import _safe_tags
-from .utils.validation import check_memory
-from .utils.validation import _deprecate_positional_args
-from .utils.fixes import delayed
+from .utils._tags import get_tags
+from .utils._user_interface import _print_elapsed_time
+from .utils.metadata_routing import (
+    MetadataRouter,
+    MethodMapping,
+    _raise_for_params,
+    _routing_enabled,
+    get_routing_for_object,
+    process_routing,
+)
+from .utils.metaestimators import _BaseComposition, available_if
+from .utils.parallel import Parallel, delayed
+from .utils.validation import check_is_fitted, check_memory
+
+__all__ = ["FeatureUnion", "Pipeline", "make_pipeline", "make_union"]
+
+
+@contextmanager
+def _raise_or_warn_if_not_fitted(estimator):
+    """A context manager to make sure a NotFittedError is raised, if a sub-estimator
+    raises the error.
+
+    Otherwise, we raise a warning if the pipeline is not fitted, with the deprecation.
+
+    TODO(1.8): remove this context manager and replace with check_is_fitted.
+    """
+    try:
+        yield
+    except NotFittedError as exc:
+        raise NotFittedError("Pipeline is not fitted yet.") from exc
+
+    # we only get here if the above didn't raise
+    try:
+        check_is_fitted(estimator)
+    except NotFittedError:
+        warnings.warn(
+            "This Pipeline instance is not fitted yet. Call 'fit' with "
+            "appropriate arguments before using other methods such as transform, "
+            "predict, etc. This will raise an error in 1.8 instead of the current "
+            "warning.",
+            FutureWarning,
+        )
+
+
+def _final_estimator_has(attr):
+    """Check that final_estimator has `attr`.
+
+    Used together with `available_if` in `Pipeline`."""
+
+    def check(self):
+        # raise original `AttributeError` if `attr` does not exist
+        getattr(self._final_estimator, attr)
+        return True
+
+    return check
+
+
+def _cached_transform(
+    sub_pipeline, *, cache, param_name, param_value, transform_params
+):
+    """Transform a parameter value using a sub-pipeline and cache the result.
+
+    Parameters
+    ----------
+    sub_pipeline : Pipeline
+        The sub-pipeline to be used for transformation.
+    cache : dict
+        The cache dictionary to store the transformed values.
+    param_name : str
+        The name of the parameter to be transformed.
+    param_value : object
+        The value of the parameter to be transformed.
+    transform_params : dict
+        The metadata to be used for transformation. This passed to the
+        `transform` method of the sub-pipeline.
 
-from .utils.metaestimators import _BaseComposition
+    Returns
+    -------
+    transformed_value : object
+        The transformed value of the parameter.
+    """
+    if param_name not in cache:
+        # If the parameter is a tuple, transform each element of the
+        # tuple. This is needed to support the pattern present in
+        # `lightgbm` and `xgboost` where users can pass multiple
+        # validation sets.
+        if isinstance(param_value, tuple):
+            cache[param_name] = tuple(
+                sub_pipeline.transform(element, **transform_params)
+                for element in param_value
+            )
+        else:
+            cache[param_name] = sub_pipeline.transform(param_value, **transform_params)
 
-__all__ = ['Pipeline', 'FeatureUnion', 'make_pipeline', 'make_union']
+    return cache[param_name]
 
 
 class Pipeline(_BaseComposition):
     """
-    Pipeline of transforms with a final estimator.
+    A sequence of data transformers with an optional final predictor.
+
+    `Pipeline` allows you to sequentially apply a list of transformers to
+    preprocess the data and, if desired, conclude the sequence with a final
+    :term:`predictor` for predictive modeling.
 
-    Sequentially apply a list of transforms and a final estimator.
-    Intermediate steps of the pipeline must be 'transforms', that is, they
-    must implement fit and transform methods.
-    The final estimator only needs to implement fit.
+    Intermediate steps of the pipeline must be transformers, that is, they
+    must implement `fit` and `transform` methods.
+    The final :term:`estimator` only needs to implement `fit`.
     The transformers in the pipeline can be cached using ``memory`` argument.
 
     The purpose of the pipeline is to assemble several steps that can be
-    cross-validated together while setting different parameters.
-    For this, it enables setting parameters of the various steps using their
-    names and the parameter name separated by a '__', as in the example below.
-    A step's estimator may be replaced entirely by setting the parameter
-    with its name to another estimator, or a transformer removed by setting
-    it to 'passthrough' or ``None``.
+    cross-validated together while setting different parameters. For this, it
+    enables setting parameters of the various steps using their names and the
+    parameter name separated by a `'__'`, as in the example below. A step's
+    estimator may be replaced entirely by setting the parameter with its name
+    to another estimator, or a transformer removed by setting it to
+    `'passthrough'` or `None`.
+
+    For an example use case of `Pipeline` combined with
+    :class:`~sklearn.model_selection.GridSearchCV`, refer to
+    :ref:`sphx_glr_auto_examples_compose_plot_compare_reduction.py`. The
+    example :ref:`sphx_glr_auto_examples_compose_plot_digits_pipe.py` shows how
+    to grid search on a pipeline using `'__'` as a separator in the parameter names.
 
     Read more in the :ref:`User Guide `.
 
@@ -58,20 +153,37 @@ class Pipeline(_BaseComposition):
 
     Parameters
     ----------
-    steps : list
-        List of (name, transform) tuples (implementing fit/transform) that are
-        chained, in the order in which they are chained, with the last object
-        an estimator.
+    steps : list of tuples
+        List of (name of step, estimator) tuples that are to be chained in
+        sequential order. To be compatible with the scikit-learn API, all steps
+        must define `fit`. All non-last steps must also define `transform`. See
+        :ref:`Combining Estimators ` for more details.
+
+    transform_input : list of str, default=None
+        The names of the :term:`metadata` parameters that should be transformed by the
+        pipeline before passing it to the step consuming it.
+
+        This enables transforming some input arguments to ``fit`` (other than ``X``)
+        to be transformed by the steps of the pipeline up to the step which requires
+        them. Requirement is defined via :ref:`metadata routing `.
+        For instance, this can be used to pass a validation set through the pipeline.
+
+        You can only set this if metadata routing is enabled, which you
+        can enable using ``sklearn.set_config(enable_metadata_routing=True)``.
+
+        .. versionadded:: 1.6
 
     memory : str or object with the joblib.Memory interface, default=None
-        Used to cache the fitted transformers of the pipeline. By default,
-        no caching is performed. If a string is given, it is the path to
-        the caching directory. Enabling caching triggers a clone of
-        the transformers before fitting. Therefore, the transformer
-        instance given to the pipeline cannot be inspected
-        directly. Use the attribute ``named_steps`` or ``steps`` to
-        inspect estimators within the pipeline. Caching the
-        transformers is advantageous when fitting is time consuming.
+        Used to cache the fitted transformers of the pipeline. The last step
+        will never be cached, even if it is a transformer. By default, no
+        caching is performed. If a string is given, it is the path to the
+        caching directory. Enabling caching triggers a clone of the transformers
+        before fitting. Therefore, the transformer instance given to the
+        pipeline cannot be inspected directly. Use the attribute ``named_steps``
+        or ``steps`` to inspect estimators within the pipeline. Caching the
+        transformers is advantageous when fitting is time consuming. See
+        :ref:`sphx_glr_auto_examples_neighbors_plot_caching_nearest_neighbors.py`
+        for an example on how to enable caching.
 
     verbose : bool, default=False
         If True, the time elapsed while fitting each step will be printed as it
@@ -84,6 +196,23 @@ class Pipeline(_BaseComposition):
         Read-only attribute to access any step parameter by user given name.
         Keys are step names and values are steps parameters.
 
+    classes_ : ndarray of shape (n_classes,)
+        The classes labels. Only exist if the last step of the pipeline is a
+        classifier.
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`. Only defined if the
+        underlying first estimator in `steps` exposes such an attribute
+        when fit.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Only defined if the
+        underlying estimator exposes such an attribute when fit.
+
+        .. versionadded:: 1.0
+
     See Also
     --------
     make_pipeline : Convenience function for simplified pipeline construction.
@@ -101,21 +230,53 @@ class Pipeline(_BaseComposition):
     >>> pipe = Pipeline([('scaler', StandardScaler()), ('svc', SVC())])
     >>> # The pipeline can be used as any other estimator
     >>> # and avoids leaking the test set into the train set
-    >>> pipe.fit(X_train, y_train)
-    Pipeline(steps=[('scaler', StandardScaler()), ('svc', SVC())])
-    >>> pipe.score(X_test, y_test)
+    >>> pipe.fit(X_train, y_train).score(X_test, y_test)
     0.88
+    >>> # An estimator's parameter can be set using '__' syntax
+    >>> pipe.set_params(svc__C=10).fit(X_train, y_train).score(X_test, y_test)
+    0.76
     """
 
     # BaseEstimator interface
-    _required_parameters = ['steps']
-
-    @_deprecate_positional_args
-    def __init__(self, steps, *, memory=None, verbose=False):
+    _parameter_constraints: dict = {
+        "steps": [list, Hidden(tuple)],
+        "transform_input": [list, None],
+        "memory": [None, str, HasMethods(["cache"])],
+        "verbose": ["boolean"],
+    }
+
+    def __init__(self, steps, *, transform_input=None, memory=None, verbose=False):
         self.steps = steps
+        self.transform_input = transform_input
         self.memory = memory
         self.verbose = verbose
-        self._validate_steps()
+
+    def set_output(self, *, transform=None):
+        """Set the output container when `"transform"` and `"fit_transform"` are called.
+
+        Calling `set_output` will set the output of all estimators in `steps`.
+
+        Parameters
+        ----------
+        transform : {"default", "pandas", "polars"}, default=None
+            Configure output of `transform` and `fit_transform`.
+
+            - `"default"`: Default output format of a transformer
+            - `"pandas"`: DataFrame output
+            - `"polars"`: Polars output
+            - `None`: Transform configuration is unchanged
+
+            .. versionadded:: 1.4
+                `"polars"` option was added.
+
+        Returns
+        -------
+        self : estimator instance
+            Estimator instance.
+        """
+        for _, _, step in self._iter():
+            _safe_set_output(step, transform=transform)
+        return self
 
     def get_params(self, deep=True):
         """Get parameters for this estimator.
@@ -134,7 +295,7 @@ def get_params(self, deep=True):
         params : mapping of string to any
             Parameter names mapped to their values.
         """
-        return self._get_params('steps', deep=deep)
+        return self._get_params("steps", deep=deep)
 
     def set_params(self, **kwargs):
         """Set the parameters of this estimator.
@@ -143,11 +304,19 @@ def set_params(self, **kwargs):
         you can directly set the parameters of the estimators contained in
         `steps`.
 
+        Parameters
+        ----------
+        **kwargs : dict
+            Parameters of this estimator or parameters of estimators contained
+            in `steps`. Parameters of the steps may be set using its name and
+            the parameter name separated by a '__'.
+
         Returns
         -------
-        self
+        self : object
+            Pipeline class instance.
         """
-        self._set_params('steps', **kwargs)
+        self._set_params("steps", **kwargs)
         return self
 
     def _validate_steps(self):
@@ -161,22 +330,29 @@ def _validate_steps(self):
         estimator = estimators[-1]
 
         for t in transformers:
-            if t is None or t == 'passthrough':
+            if t is None or t == "passthrough":
                 continue
-            if (not (hasattr(t, "fit") or hasattr(t, "fit_transform")) or not
-                    hasattr(t, "transform")):
-                raise TypeError("All intermediate steps should be "
-                                "transformers and implement fit and transform "
-                                "or be the string 'passthrough' "
-                                "'%s' (type %s) doesn't" % (t, type(t)))
+            if not (hasattr(t, "fit") or hasattr(t, "fit_transform")) or not hasattr(
+                t, "transform"
+            ):
+                raise TypeError(
+                    "All intermediate steps should be "
+                    "transformers and implement fit and transform "
+                    "or be the string 'passthrough' "
+                    "'%s' (type %s) doesn't" % (t, type(t))
+                )
 
         # We allow last estimator to be None as an identity transformation
-        if (estimator is not None and estimator != 'passthrough'
-                and not hasattr(estimator, "fit")):
+        if (
+            estimator is not None
+            and estimator != "passthrough"
+            and not hasattr(estimator, "fit")
+        ):
             raise TypeError(
                 "Last step of Pipeline should implement fit "
                 "or be the string 'passthrough'. "
-                "'%s' (type %s) doesn't" % (estimator, type(estimator)))
+                "'%s' (type %s) doesn't" % (estimator, type(estimator))
+            )
 
     def _iter(self, with_final=True, filter_passthrough=True):
         """
@@ -192,7 +368,7 @@ def _iter(self, with_final=True, filter_passthrough=True):
         for idx, (name, trans) in enumerate(islice(self.steps, 0, stop)):
             if not filter_passthrough:
                 yield idx, name, trans
-            elif trans is not None and trans != 'passthrough':
+            elif trans is not None and trans != "passthrough":
                 yield idx, name, trans
 
     def __len__(self):
@@ -202,13 +378,17 @@ def __len__(self):
         return len(self.steps)
 
     def __getitem__(self, ind):
-        """Returns a sub-pipeline or a single esimtator in the pipeline
+        """Returns a sub-pipeline or a single estimator in the pipeline
 
         Indexing with an integer will return an estimator; using a slice
         returns another Pipeline instance which copies a slice of this
         Pipeline. This copy is shallow: modifying (or fitting) estimators in
         the sub-pipeline will affect the larger pipeline and vice-versa.
         However, replacing a value in `step` will not affect a copy.
+
+        See
+        :ref:`sphx_glr_auto_examples_feature_selection_plot_feature_selection_pipeline.py`
+        for an example of how to use slicing to inspect part of a pipeline.
         """
         if isinstance(ind, slice):
             if ind.step not in (1, None):
@@ -223,47 +403,159 @@ def __getitem__(self, ind):
             return self.named_steps[ind]
         return est
 
+    # TODO(1.8): Remove this property
     @property
     def _estimator_type(self):
+        """Return the estimator type of the last step in the pipeline."""
+
+        if not self.steps:
+            return None
+
         return self.steps[-1][1]._estimator_type
 
     @property
     def named_steps(self):
+        """Access the steps by name.
+
+        Read-only attribute to access any step by given name.
+        Keys are steps names and values are the steps objects."""
         # Use Bunch object to improve autocomplete
         return Bunch(**dict(self.steps))
 
     @property
     def _final_estimator(self):
-        estimator = self.steps[-1][1]
-        return 'passthrough' if estimator is None else estimator
+        try:
+            estimator = self.steps[-1][1]
+            return "passthrough" if estimator is None else estimator
+        except (ValueError, AttributeError, TypeError):
+            # This condition happens when a call to a method is first calling
+            # `_available_if` and `fit` did not validate `steps` yet. We
+            # return `None` and an `InvalidParameterError` will be raised
+            # right after.
+            return None
 
     def _log_message(self, step_idx):
         if not self.verbose:
             return None
         name, _ = self.steps[step_idx]
 
-        return '(step %d of %d) Processing %s' % (step_idx + 1,
-                                                  len(self.steps),
-                                                  name)
+        return "(step %d of %d) Processing %s" % (step_idx + 1, len(self.steps), name)
 
-    def _check_fit_params(self, **fit_params):
-        fit_params_steps = {name: {} for name, step in self.steps
-                            if step is not None}
-        for pname, pval in fit_params.items():
-            if '__' not in pname:
-                raise ValueError(
-                    "Pipeline.fit does not accept the {} parameter. "
-                    "You can pass parameters to specific steps of your "
-                    "pipeline using the stepname__parameter format, e.g. "
-                    "`Pipeline.fit(X, y, logisticregression__sample_weight"
-                    "=sample_weight)`.".format(pname))
-            step, param = pname.split('__', 1)
-            fit_params_steps[step][param] = pval
-        return fit_params_steps
+    def _check_method_params(self, method, props, **kwargs):
+        if _routing_enabled():
+            routed_params = process_routing(self, method, **props, **kwargs)
+            return routed_params
+        else:
+            fit_params_steps = Bunch(
+                **{
+                    name: Bunch(**{method: {} for method in METHODS})
+                    for name, step in self.steps
+                    if step is not None
+                }
+            )
+            for pname, pval in props.items():
+                if "__" not in pname:
+                    raise ValueError(
+                        "Pipeline.fit does not accept the {} parameter. "
+                        "You can pass parameters to specific steps of your "
+                        "pipeline using the stepname__parameter format, e.g. "
+                        "`Pipeline.fit(X, y, logisticregression__sample_weight"
+                        "=sample_weight)`.".format(pname)
+                    )
+                step, param = pname.split("__", 1)
+                fit_params_steps[step]["fit"][param] = pval
+                # without metadata routing, fit_transform and fit_predict
+                # get all the same params and pass it to the last fit.
+                fit_params_steps[step]["fit_transform"][param] = pval
+                fit_params_steps[step]["fit_predict"][param] = pval
+            return fit_params_steps
+
+    def _get_metadata_for_step(self, *, step_idx, step_params, all_params):
+        """Get params (metadata) for step `name`.
+
+        This transforms the metadata up to this step if required, which is
+        indicated by the `transform_input` parameter.
+
+        If a param in `step_params` is included in the `transform_input` list,
+        it will be transformed.
+
+        Parameters
+        ----------
+        step_idx : int
+            Index of the step in the pipeline.
+
+        step_params : dict
+            Parameters specific to the step. These are routed parameters, e.g.
+            `routed_params[name]`. If a parameter name here is included in the
+            `pipeline.transform_input`, then it will be transformed. Note that
+            these parameters are *after* routing, so the aliases are already
+            resolved.
+
+        all_params : dict
+            All parameters passed by the user. Here this is used to call
+            `transform` on the slice of the pipeline itself.
+
+        Returns
+        -------
+        dict
+            Parameters to be passed to the step. The ones which should be
+            transformed are transformed.
+        """
+        if (
+            self.transform_input is None
+            or not all_params
+            or not step_params
+            or step_idx == 0
+        ):
+            # we only need to process step_params if transform_input is set
+            # and metadata is given by the user.
+            return step_params
+
+        sub_pipeline = self[:step_idx]
+        sub_metadata_routing = get_routing_for_object(sub_pipeline)
+        # here we get the metadata required by sub_pipeline.transform
+        transform_params = {
+            key: value
+            for key, value in all_params.items()
+            if key
+            in sub_metadata_routing.consumes(
+                method="transform", params=all_params.keys()
+            )
+        }
+        transformed_params = dict()  # this is to be returned
+        transformed_cache = dict()  # used to transform each param once
+        # `step_params` is the output of `process_routing`, so it has a dict for each
+        # method (e.g. fit, transform, predict), which are the args to be passed to
+        # those methods. We need to transform the parameters which are in the
+        # `transform_input`, before returning these dicts.
+        for method, method_params in step_params.items():
+            transformed_params[method] = Bunch()
+            for param_name, param_value in method_params.items():
+                # An example of `(param_name, param_value)` is
+                # `('sample_weight', array([0.5, 0.5, ...]))`
+                if param_name in self.transform_input:
+                    # This parameter now needs to be transformed by the sub_pipeline, to
+                    # this step. We cache these computations to avoid repeating them.
+                    transformed_params[method][param_name] = _cached_transform(
+                        sub_pipeline,
+                        cache=transformed_cache,
+                        param_name=param_name,
+                        param_value=param_value,
+                        transform_params=transform_params,
+                    )
+                else:
+                    transformed_params[method][param_name] = param_value
+        return transformed_params
 
     # Estimator interface
 
-    def _fit(self, X, y=None, **fit_params_steps):
+    def _fit(self, X, y=None, routed_params=None, raw_params=None):
+        """Fit the pipeline except the last step.
+
+        routed_params is the output of `process_routing`
+        raw_params is the parameters passed by the user, used when `transform_input`
+            is set by the user, to transform metadata using a sub-pipeline.
+        """
         # shallow copy of steps - this should really be steps_
         self.steps = list(self.steps)
         self._validate_steps()
@@ -272,50 +564,50 @@ def _fit(self, X, y=None, **fit_params_steps):
 
         fit_transform_one_cached = memory.cache(_fit_transform_one)
 
-        for (step_idx,
-             name,
-             transformer) in self._iter(with_final=False,
-                                        filter_passthrough=False):
-            if (transformer is None or transformer == 'passthrough'):
-                with _print_elapsed_time('Pipeline',
-                                         self._log_message(step_idx)):
+        for step_idx, name, transformer in self._iter(
+            with_final=False, filter_passthrough=False
+        ):
+            if transformer is None or transformer == "passthrough":
+                with _print_elapsed_time("Pipeline", self._log_message(step_idx)):
                     continue
 
-            if hasattr(memory, 'location'):
-                # joblib >= 0.12
-                if memory.location is None:
-                    # we do not clone when caching is disabled to
-                    # preserve backward compatibility
-                    cloned_transformer = transformer
-                else:
-                    cloned_transformer = clone(transformer)
-            elif hasattr(memory, 'cachedir'):
-                # joblib < 0.11
-                if memory.cachedir is None:
-                    # we do not clone when caching is disabled to
-                    # preserve backward compatibility
-                    cloned_transformer = transformer
-                else:
-                    cloned_transformer = clone(transformer)
+            if hasattr(memory, "location") and memory.location is None:
+                # we do not clone when caching is disabled to
+                # preserve backward compatibility
+                cloned_transformer = transformer
             else:
                 cloned_transformer = clone(transformer)
             # Fit or load from cache the current transformer
+            step_params = self._get_metadata_for_step(
+                step_idx=step_idx,
+                step_params=routed_params[name],
+                all_params=raw_params,
+            )
+
             X, fitted_transformer = fit_transform_one_cached(
-                cloned_transformer, X, y, None,
-                message_clsname='Pipeline',
+                cloned_transformer,
+                X,
+                y,
+                weight=None,
+                message_clsname="Pipeline",
                 message=self._log_message(step_idx),
-                **fit_params_steps[name])
+                params=step_params,
+            )
             # Replace the transformer of the step with the fitted
             # transformer. This is necessary when loading the transformer
             # from the cache.
             self.steps[step_idx] = (name, fitted_transformer)
         return X
 
-    def fit(self, X, y=None, **fit_params):
-        """Fit the model
+    @_fit_context(
+        # estimators in Pipeline.steps are not validated yet
+        prefer_skip_nested_validation=False
+    )
+    def fit(self, X, y=None, **params):
+        """Fit the model.
 
-        Fit all the transforms one after the other and transform the
-        data, then fit the transformed data using the final estimator.
+        Fit all the transformers one after the other and sequentially transform the
+        data. Finally, fit the transformed data using the final estimator.
 
         Parameters
         ----------
@@ -327,32 +619,67 @@ def fit(self, X, y=None, **fit_params):
             Training targets. Must fulfill label requirements for all steps of
             the pipeline.
 
-        **fit_params : dict of string -> object
-            Parameters passed to the ``fit`` method of each step, where
-            each parameter name is prefixed such that parameter ``p`` for step
-            ``s`` has key ``s__p``.
+        **params : dict of str -> object
+            - If `enable_metadata_routing=False` (default): Parameters passed to the
+              ``fit`` method of each step, where each parameter name is prefixed such
+              that parameter ``p`` for step ``s`` has key ``s__p``.
+
+            - If `enable_metadata_routing=True`: Parameters requested and accepted by
+              steps. Each step must have requested certain metadata for these parameters
+              to be forwarded to them.
+
+            .. versionchanged:: 1.4
+                Parameters are now passed to the ``transform`` method of the
+                intermediate steps as well, if requested, and if
+                `enable_metadata_routing=True` is set via
+                :func:`~sklearn.set_config`.
+
+            See :ref:`Metadata Routing User Guide ` for more
+            details.
 
         Returns
         -------
-        self : Pipeline
-            This estimator
+        self : object
+            Pipeline with fitted steps.
         """
-        fit_params_steps = self._check_fit_params(**fit_params)
-        Xt = self._fit(X, y, **fit_params_steps)
-        with _print_elapsed_time('Pipeline',
-                                 self._log_message(len(self.steps) - 1)):
-            if self._final_estimator != 'passthrough':
-                fit_params_last_step = fit_params_steps[self.steps[-1][0]]
-                self._final_estimator.fit(Xt, y, **fit_params_last_step)
+        if not _routing_enabled() and self.transform_input is not None:
+            raise ValueError(
+                "The `transform_input` parameter can only be set if metadata "
+                "routing is enabled. You can enable metadata routing using "
+                "`sklearn.set_config(enable_metadata_routing=True)`."
+            )
+
+        routed_params = self._check_method_params(method="fit", props=params)
+        Xt = self._fit(X, y, routed_params, raw_params=params)
+        with _print_elapsed_time("Pipeline", self._log_message(len(self.steps) - 1)):
+            if self._final_estimator != "passthrough":
+                last_step_params = self._get_metadata_for_step(
+                    step_idx=len(self) - 1,
+                    step_params=routed_params[self.steps[-1][0]],
+                    all_params=params,
+                )
+                self._final_estimator.fit(Xt, y, **last_step_params["fit"])
 
         return self
 
-    def fit_transform(self, X, y=None, **fit_params):
-        """Fit the model and transform with the final estimator
-
-        Fits all the transforms one after the other and transforms the
-        data, then uses fit_transform on transformed data with the final
-        estimator.
+    def _can_fit_transform(self):
+        return (
+            self._final_estimator == "passthrough"
+            or hasattr(self._final_estimator, "transform")
+            or hasattr(self._final_estimator, "fit_transform")
+        )
+
+    @available_if(_can_fit_transform)
+    @_fit_context(
+        # estimators in Pipeline.steps are not validated yet
+        prefer_skip_nested_validation=False
+    )
+    def fit_transform(self, X, y=None, **params):
+        """Fit the model and transform with the final estimator.
+
+        Fit all the transformers one after the other and sequentially transform
+        the data. Only valid if the final estimator either implements
+        `fit_transform` or `fit` and `transform`.
 
         Parameters
         ----------
@@ -364,34 +691,56 @@ def fit_transform(self, X, y=None, **fit_params):
             Training targets. Must fulfill label requirements for all steps of
             the pipeline.
 
-        **fit_params : dict of string -> object
-            Parameters passed to the ``fit`` method of each step, where
-            each parameter name is prefixed such that parameter ``p`` for step
-            ``s`` has key ``s__p``.
+        **params : dict of str -> object
+            - If `enable_metadata_routing=False` (default): Parameters passed to the
+              ``fit`` method of each step, where each parameter name is prefixed such
+              that parameter ``p`` for step ``s`` has key ``s__p``.
+
+            - If `enable_metadata_routing=True`: Parameters requested and accepted by
+              steps. Each step must have requested certain metadata for these parameters
+              to be forwarded to them.
+
+            .. versionchanged:: 1.4
+                Parameters are now passed to the ``transform`` method of the
+                intermediate steps as well, if requested, and if
+                `enable_metadata_routing=True`.
+
+            See :ref:`Metadata Routing User Guide ` for more
+            details.
 
         Returns
         -------
-        Xt : array-like of shape  (n_samples, n_transformed_features)
-            Transformed samples
+        Xt : ndarray of shape (n_samples, n_transformed_features)
+            Transformed samples.
         """
-        fit_params_steps = self._check_fit_params(**fit_params)
-        Xt = self._fit(X, y, **fit_params_steps)
+        routed_params = self._check_method_params(method="fit_transform", props=params)
+        Xt = self._fit(X, y, routed_params)
 
         last_step = self._final_estimator
-        with _print_elapsed_time('Pipeline',
-                                 self._log_message(len(self.steps) - 1)):
-            if last_step == 'passthrough':
+        with _print_elapsed_time("Pipeline", self._log_message(len(self.steps) - 1)):
+            if last_step == "passthrough":
                 return Xt
-            fit_params_last_step = fit_params_steps[self.steps[-1][0]]
-            if hasattr(last_step, 'fit_transform'):
-                return last_step.fit_transform(Xt, y, **fit_params_last_step)
+            last_step_params = self._get_metadata_for_step(
+                step_idx=len(self) - 1,
+                step_params=routed_params[self.steps[-1][0]],
+                all_params=params,
+            )
+            if hasattr(last_step, "fit_transform"):
+                return last_step.fit_transform(
+                    Xt, y, **last_step_params["fit_transform"]
+                )
             else:
-                return last_step.fit(Xt, y,
-                                     **fit_params_last_step).transform(Xt)
+                return last_step.fit(Xt, y, **last_step_params["fit"]).transform(
+                    Xt, **last_step_params["transform"]
+                )
 
-    @if_delegate_has_method(delegate='_final_estimator')
-    def predict(self, X, **predict_params):
-        """Apply transforms to the data, and predict with the final estimator
+    @available_if(_final_estimator_has("predict"))
+    def predict(self, X, **params):
+        """Transform the data, and apply `predict` with the final estimator.
+
+        Call `transform` of each transformer in the pipeline. The transformed
+        data are finally passed to the final estimator that calls `predict`
+        method. Only valid if the final estimator implements `predict`.
 
         Parameters
         ----------
@@ -399,32 +748,64 @@ def predict(self, X, **predict_params):
             Data to predict on. Must fulfill input requirements of first step
             of the pipeline.
 
-        **predict_params : dict of string -> object
-            Parameters to the ``predict`` called at the end of all
-            transformations in the pipeline. Note that while this may be
-            used to return uncertainties from some models with return_std
-            or return_cov, uncertainties that are generated by the
-            transformations in the pipeline are not propagated to the
-            final estimator.
+        **params : dict of str -> object
+            - If `enable_metadata_routing=False` (default): Parameters to the
+              ``predict`` called at the end of all transformations in the pipeline.
+
+            - If `enable_metadata_routing=True`: Parameters requested and accepted by
+              steps. Each step must have requested certain metadata for these parameters
+              to be forwarded to them.
 
             .. versionadded:: 0.20
 
+            .. versionchanged:: 1.4
+                Parameters are now passed to the ``transform`` method of the
+                intermediate steps as well, if requested, and if
+                `enable_metadata_routing=True` is set via
+                :func:`~sklearn.set_config`.
+
+            See :ref:`Metadata Routing User Guide ` for more
+            details.
+
+            Note that while this may be used to return uncertainties from some
+            models with ``return_std`` or ``return_cov``, uncertainties that are
+            generated by the transformations in the pipeline are not propagated
+            to the final estimator.
+
         Returns
         -------
-        y_pred : array-like
+        y_pred : ndarray
+            Result of calling `predict` on the final estimator.
         """
-        Xt = X
-        for _, name, transform in self._iter(with_final=False):
-            Xt = transform.transform(Xt)
-        return self.steps[-1][-1].predict(Xt, **predict_params)
+        # TODO(1.8): Remove the context manager and use check_is_fitted(self)
+        with _raise_or_warn_if_not_fitted(self):
+            Xt = X
+
+            if not _routing_enabled():
+                for _, name, transform in self._iter(with_final=False):
+                    Xt = transform.transform(Xt)
+                return self.steps[-1][1].predict(Xt, **params)
+
+            # metadata routing enabled
+            routed_params = process_routing(self, "predict", **params)
+            for _, name, transform in self._iter(with_final=False):
+                Xt = transform.transform(Xt, **routed_params[name].transform)
+            return self.steps[-1][1].predict(
+                Xt, **routed_params[self.steps[-1][0]].predict
+            )
 
-    @if_delegate_has_method(delegate='_final_estimator')
-    def fit_predict(self, X, y=None, **fit_params):
-        """Applies fit_predict of last step in pipeline after transforms.
+    @available_if(_final_estimator_has("fit_predict"))
+    @_fit_context(
+        # estimators in Pipeline.steps are not validated yet
+        prefer_skip_nested_validation=False
+    )
+    def fit_predict(self, X, y=None, **params):
+        """Transform the data, and apply `fit_predict` with the final estimator.
 
-        Applies fit_transforms of a pipeline to the data, followed by the
-        fit_predict method of the final estimator in the pipeline. Valid
-        only if the final estimator implements fit_predict.
+        Call `fit_transform` of each transformer in the pipeline. The
+        transformed data are finally passed to the final estimator that calls
+        `fit_predict` method. Only valid if the final estimator implements
+        `fit_predict`.
 
         Parameters
         ----------
@@ -436,28 +817,52 @@ def fit_predict(self, X, y=None, **fit_params):
             Training targets. Must fulfill label requirements for all steps
             of the pipeline.
 
-        **fit_params : dict of string -> object
-            Parameters passed to the ``fit`` method of each step, where
-            each parameter name is prefixed such that parameter ``p`` for step
-            ``s`` has key ``s__p``.
+        **params : dict of str -> object
+            - If `enable_metadata_routing=False` (default): Parameters to the
+              ``predict`` called at the end of all transformations in the pipeline.
+
+            - If `enable_metadata_routing=True`: Parameters requested and accepted by
+              steps. Each step must have requested certain metadata for these parameters
+              to be forwarded to them.
+
+            .. versionadded:: 0.20
+
+            .. versionchanged:: 1.4
+                Parameters are now passed to the ``transform`` method of the
+                intermediate steps as well, if requested, and if
+                `enable_metadata_routing=True`.
+
+            See :ref:`Metadata Routing User Guide ` for more
+            details.
+
+            Note that while this may be used to return uncertainties from some
+            models with ``return_std`` or ``return_cov``, uncertainties that are
+            generated by the transformations in the pipeline are not propagated
+            to the final estimator.
 
         Returns
         -------
-        y_pred : array-like
+        y_pred : ndarray
+            Result of calling `fit_predict` on the final estimator.
         """
-        fit_params_steps = self._check_fit_params(**fit_params)
-        Xt = self._fit(X, y, **fit_params_steps)
-
-        fit_params_last_step = fit_params_steps[self.steps[-1][0]]
-        with _print_elapsed_time('Pipeline',
-                                 self._log_message(len(self.steps) - 1)):
-            y_pred = self.steps[-1][-1].fit_predict(Xt, y,
-                                                    **fit_params_last_step)
+        routed_params = self._check_method_params(method="fit_predict", props=params)
+        Xt = self._fit(X, y, routed_params)
+
+        params_last_step = routed_params[self.steps[-1][0]]
+        with _print_elapsed_time("Pipeline", self._log_message(len(self.steps) - 1)):
+            y_pred = self.steps[-1][1].fit_predict(
+                Xt, y, **params_last_step.get("fit_predict", {})
+            )
         return y_pred
 
-    @if_delegate_has_method(delegate='_final_estimator')
-    def predict_proba(self, X):
-        """Apply transforms, and predict_proba of the final estimator
+    @available_if(_final_estimator_has("predict_proba"))
+    def predict_proba(self, X, **params):
+        """Transform the data, and apply `predict_proba` with the final estimator.
+
+        Call `transform` of each transformer in the pipeline. The transformed
+        data are finally passed to the final estimator that calls
+        `predict_proba` method. Only valid if the final estimator implements
+        `predict_proba`.
 
         Parameters
         ----------
@@ -465,18 +870,54 @@ def predict_proba(self, X):
             Data to predict on. Must fulfill input requirements of first step
             of the pipeline.
 
+        **params : dict of str -> object
+            - If `enable_metadata_routing=False` (default): Parameters to the
+              `predict_proba` called at the end of all transformations in the pipeline.
+
+            - If `enable_metadata_routing=True`: Parameters requested and accepted by
+              steps. Each step must have requested certain metadata for these parameters
+              to be forwarded to them.
+
+            .. versionadded:: 0.20
+
+            .. versionchanged:: 1.4
+                Parameters are now passed to the ``transform`` method of the
+                intermediate steps as well, if requested, and if
+                `enable_metadata_routing=True`.
+
+            See :ref:`Metadata Routing User Guide ` for more
+            details.
+
         Returns
         -------
-        y_proba : array-like of shape (n_samples, n_classes)
+        y_proba : ndarray of shape (n_samples, n_classes)
+            Result of calling `predict_proba` on the final estimator.
         """
-        Xt = X
-        for _, name, transform in self._iter(with_final=False):
-            Xt = transform.transform(Xt)
-        return self.steps[-1][-1].predict_proba(Xt)
+        # TODO(1.8): Remove the context manager and use check_is_fitted(self)
+        with _raise_or_warn_if_not_fitted(self):
+            Xt = X
+
+            if not _routing_enabled():
+                for _, name, transform in self._iter(with_final=False):
+                    Xt = transform.transform(Xt)
+                return self.steps[-1][1].predict_proba(Xt, **params)
+
+            # metadata routing enabled
+            routed_params = process_routing(self, "predict_proba", **params)
+            for _, name, transform in self._iter(with_final=False):
+                Xt = transform.transform(Xt, **routed_params[name].transform)
+            return self.steps[-1][1].predict_proba(
+                Xt, **routed_params[self.steps[-1][0]].predict_proba
+            )
+
+    @available_if(_final_estimator_has("decision_function"))
+    def decision_function(self, X, **params):
+        """Transform the data, and apply `decision_function` with the final estimator.
 
-    @if_delegate_has_method(delegate='_final_estimator')
-    def decision_function(self, X):
-        """Apply transforms, and decision_function of the final estimator
+        Call `transform` of each transformer in the pipeline. The transformed
+        data are finally passed to the final estimator that calls
+        `decision_function` method. Only valid if the final estimator
+        implements `decision_function`.
 
         Parameters
         ----------
@@ -484,18 +925,47 @@ def decision_function(self, X):
             Data to predict on. Must fulfill input requirements of first step
             of the pipeline.
 
+        **params : dict of string -> object
+            Parameters requested and accepted by steps. Each step must have
+            requested certain metadata for these parameters to be forwarded to
+            them.
+
+            .. versionadded:: 1.4
+                Only available if `enable_metadata_routing=True`. See
+                :ref:`Metadata Routing User Guide ` for more
+                details.
+
         Returns
         -------
-        y_score : array-like of shape (n_samples, n_classes)
+        y_score : ndarray of shape (n_samples, n_classes)
+            Result of calling `decision_function` on the final estimator.
         """
-        Xt = X
-        for _, name, transform in self._iter(with_final=False):
-            Xt = transform.transform(Xt)
-        return self.steps[-1][-1].decision_function(Xt)
+        # TODO(1.8): Remove the context manager and use check_is_fitted(self)
+        with _raise_or_warn_if_not_fitted(self):
+            _raise_for_params(params, self, "decision_function")
+
+            # not branching here since params is only available if
+            # enable_metadata_routing=True
+            routed_params = process_routing(self, "decision_function", **params)
+
+            Xt = X
+            for _, name, transform in self._iter(with_final=False):
+                Xt = transform.transform(
+                    Xt, **routed_params.get(name, {}).get("transform", {})
+                )
+            return self.steps[-1][1].decision_function(
+                Xt,
+                **routed_params.get(self.steps[-1][0], {}).get("decision_function", {}),
+            )
 
-    @if_delegate_has_method(delegate='_final_estimator')
+    @available_if(_final_estimator_has("score_samples"))
     def score_samples(self, X):
-        """Apply transforms, and score_samples of the final estimator.
+        """Transform the data, and apply `score_samples` with the final estimator.
+
+        Call `transform` of each transformer in the pipeline. The transformed
+        data are finally passed to the final estimator that calls
+        `score_samples` method. Only valid if the final estimator implements
+        `score_samples`.
 
         Parameters
         ----------
@@ -506,15 +976,23 @@ def score_samples(self, X):
         Returns
         -------
         y_score : ndarray of shape (n_samples,)
+            Result of calling `score_samples` on the final estimator.
         """
-        Xt = X
-        for _, _, transformer in self._iter(with_final=False):
-            Xt = transformer.transform(Xt)
-        return self.steps[-1][-1].score_samples(Xt)
-
-    @if_delegate_has_method(delegate='_final_estimator')
-    def predict_log_proba(self, X):
-        """Apply transforms, and predict_log_proba of the final estimator
+        # TODO(1.8): Remove the context manager and use check_is_fitted(self)
+        with _raise_or_warn_if_not_fitted(self):
+            Xt = X
+            for _, _, transformer in self._iter(with_final=False):
+                Xt = transformer.transform(Xt)
+            return self.steps[-1][1].score_samples(Xt)
+
+    @available_if(_final_estimator_has("predict_log_proba"))
+    def predict_log_proba(self, X, **params):
+        """Transform the data, and apply `predict_log_proba` with the final estimator.
+
+        Call `transform` of each transformer in the pipeline. The transformed
+        data are finally passed to the final estimator that calls
+        `predict_log_proba` method. Only valid if the final estimator
+        implements `predict_log_proba`.
 
         Parameters
         ----------
@@ -522,20 +1000,62 @@ def predict_log_proba(self, X):
             Data to predict on. Must fulfill input requirements of first step
             of the pipeline.
 
+        **params : dict of str -> object
+            - If `enable_metadata_routing=False` (default): Parameters to the
+              `predict_log_proba` called at the end of all transformations in the
+              pipeline.
+
+            - If `enable_metadata_routing=True`: Parameters requested and accepted by
+              steps. Each step must have requested certain metadata for these parameters
+              to be forwarded to them.
+
+            .. versionadded:: 0.20
+
+            .. versionchanged:: 1.4
+                Parameters are now passed to the ``transform`` method of the
+                intermediate steps as well, if requested, and if
+                `enable_metadata_routing=True`.
+
+            See :ref:`Metadata Routing User Guide ` for more
+            details.
+
         Returns
         -------
-        y_score : array-like of shape (n_samples, n_classes)
+        y_log_proba : ndarray of shape (n_samples, n_classes)
+            Result of calling `predict_log_proba` on the final estimator.
         """
-        Xt = X
-        for _, name, transform in self._iter(with_final=False):
-            Xt = transform.transform(Xt)
-        return self.steps[-1][-1].predict_log_proba(Xt)
+        # TODO(1.8): Remove the context manager and use check_is_fitted(self)
+        with _raise_or_warn_if_not_fitted(self):
+            Xt = X
+
+            if not _routing_enabled():
+                for _, name, transform in self._iter(with_final=False):
+                    Xt = transform.transform(Xt)
+                return self.steps[-1][1].predict_log_proba(Xt, **params)
+
+            # metadata routing enabled
+            routed_params = process_routing(self, "predict_log_proba", **params)
+            for _, name, transform in self._iter(with_final=False):
+                Xt = transform.transform(Xt, **routed_params[name].transform)
+            return self.steps[-1][1].predict_log_proba(
+                Xt, **routed_params[self.steps[-1][0]].predict_log_proba
+            )
 
-    @property
-    def transform(self):
-        """Apply transforms, and transform with the final estimator
+    def _can_transform(self):
+        return self._final_estimator == "passthrough" or hasattr(
+            self._final_estimator, "transform"
+        )
+
+    @available_if(_can_transform)
+    def transform(self, X, **params):
+        """Transform the data, and apply `transform` with the final estimator.
 
-        This also works where final estimator is ``None``: all prior
+        Call `transform` of each transformer in the pipeline. The transformed
+        data are finally passed to the final estimator that calls
+        `transform` method. Only valid if the final estimator
+        implements `transform`.
+
+        This also works where final estimator is `None` in which case all prior
         transformations are applied.
 
         Parameters
@@ -544,56 +1064,87 @@ def transform(self):
             Data to transform. Must fulfill input requirements of first step
             of the pipeline.
 
+        **params : dict of str -> object
+            Parameters requested and accepted by steps. Each step must have
+            requested certain metadata for these parameters to be forwarded to
+            them.
+
+            .. versionadded:: 1.4
+                Only available if `enable_metadata_routing=True`. See
+                :ref:`Metadata Routing User Guide ` for more
+                details.
+
         Returns
         -------
-        Xt : array-like of shape  (n_samples, n_transformed_features)
+        Xt : ndarray of shape (n_samples, n_transformed_features)
+            Transformed data.
         """
-        # _final_estimator is None or has transform, otherwise attribute error
-        # XXX: Handling the None case means we can't use if_delegate_has_method
-        if self._final_estimator != 'passthrough':
-            self._final_estimator.transform
-        return self._transform
-
-    def _transform(self, X):
-        Xt = X
-        for _, _, transform in self._iter():
-            Xt = transform.transform(Xt)
-        return Xt
+        # TODO(1.8): Remove the context manager and use check_is_fitted(self)
+        with _raise_or_warn_if_not_fitted(self):
+            _raise_for_params(params, self, "transform")
 
-    @property
-    def inverse_transform(self):
-        """Apply inverse transformations in reverse order
+            # not branching here since params is only available if
+            # enable_metadata_routing=True
+            routed_params = process_routing(self, "transform", **params)
+            Xt = X
+            for _, name, transform in self._iter():
+                Xt = transform.transform(Xt, **routed_params[name].transform)
+            return Xt
+
+    def _can_inverse_transform(self):
+        return all(hasattr(t, "inverse_transform") for _, _, t in self._iter())
 
-        All estimators in the pipeline must support ``inverse_transform``.
+    @available_if(_can_inverse_transform)
+    def inverse_transform(self, X, **params):
+        """Apply `inverse_transform` for each step in a reverse order.
+
+        All estimators in the pipeline must support `inverse_transform`.
 
         Parameters
         ----------
-        Xt : array-like of shape  (n_samples, n_transformed_features)
+        X : array-like of shape (n_samples, n_transformed_features)
             Data samples, where ``n_samples`` is the number of samples and
             ``n_features`` is the number of features. Must fulfill
             input requirements of last step of pipeline's
             ``inverse_transform`` method.
 
+        **params : dict of str -> object
+            Parameters requested and accepted by steps. Each step must have
+            requested certain metadata for these parameters to be forwarded to
+            them.
+
+            .. versionadded:: 1.4
+                Only available if `enable_metadata_routing=True`. See
+                :ref:`Metadata Routing User Guide ` for more
+                details.
+
         Returns
         -------
-        Xt : array-like of shape (n_samples, n_features)
+        X_original : ndarray of shape (n_samples, n_features)
+            Inverse transformed data, that is, data in the original feature
+            space.
         """
-        # raise AttributeError if necessary for hasattr behaviour
-        # XXX: Handling the None case means we can't use if_delegate_has_method
-        for _, _, transform in self._iter():
-            transform.inverse_transform
-        return self._inverse_transform
-
-    def _inverse_transform(self, X):
-        Xt = X
-        reverse_iter = reversed(list(self._iter()))
-        for _, _, transform in reverse_iter:
-            Xt = transform.inverse_transform(Xt)
-        return Xt
-
-    @if_delegate_has_method(delegate='_final_estimator')
-    def score(self, X, y=None, sample_weight=None):
-        """Apply transforms, and score with the final estimator
+        # TODO(1.8): Remove the context manager and use check_is_fitted(self)
+        with _raise_or_warn_if_not_fitted(self):
+            _raise_for_params(params, self, "inverse_transform")
+
+            # we don't have to branch here, since params is only non-empty if
+            # enable_metadata_routing=True.
+            routed_params = process_routing(self, "inverse_transform", **params)
+            reverse_iter = reversed(list(self._iter()))
+            for _, name, transform in reverse_iter:
+                X = transform.inverse_transform(
+                    X, **routed_params[name].inverse_transform
+                )
+            return X
+
+    @available_if(_final_estimator_has("score"))
+    def score(self, X, y=None, sample_weight=None, **params):
+        """Transform the data, and apply `score` with the final estimator.
+
+        Call `transform` of each transformer in the pipeline. The transformed
+        data are finally passed to the final estimator that calls
+        `score` method. Only valid if the final estimator implements `score`.
 
         Parameters
         ----------
@@ -609,62 +1160,257 @@ def score(self, X, y=None, sample_weight=None):
             If not None, this argument is passed as ``sample_weight`` keyword
             argument to the ``score`` method of the final estimator.
 
+        **params : dict of str -> object
+            Parameters requested and accepted by steps. Each step must have
+            requested certain metadata for these parameters to be forwarded to
+            them.
+
+            .. versionadded:: 1.4
+                Only available if `enable_metadata_routing=True`. See
+                :ref:`Metadata Routing User Guide ` for more
+                details.
+
         Returns
         -------
         score : float
+            Result of calling `score` on the final estimator.
         """
-        Xt = X
-        for _, name, transform in self._iter(with_final=False):
-            Xt = transform.transform(Xt)
-        score_params = {}
-        if sample_weight is not None:
-            score_params['sample_weight'] = sample_weight
-        return self.steps[-1][-1].score(Xt, y, **score_params)
+        # TODO(1.8): Remove the context manager and use check_is_fitted(self)
+        with _raise_or_warn_if_not_fitted(self):
+            Xt = X
+            if not _routing_enabled():
+                for _, name, transform in self._iter(with_final=False):
+                    Xt = transform.transform(Xt)
+                score_params = {}
+                if sample_weight is not None:
+                    score_params["sample_weight"] = sample_weight
+                return self.steps[-1][1].score(Xt, y, **score_params)
+
+            # metadata routing is enabled.
+            routed_params = process_routing(
+                self, "score", sample_weight=sample_weight, **params
+            )
+
+            Xt = X
+            for _, name, transform in self._iter(with_final=False):
+                Xt = transform.transform(Xt, **routed_params[name].transform)
+            return self.steps[-1][1].score(
+                Xt, y, **routed_params[self.steps[-1][0]].score
+            )
 
     @property
     def classes_(self):
-        return self.steps[-1][-1].classes_
+        """The classes labels. Only exist if the last step is a classifier."""
+        return self.steps[-1][1].classes_
 
-    def _more_tags(self):
-        # check if first estimator expects pairwise input
-        return {'pairwise': _safe_tags(self.steps[0][1], "pairwise")}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
 
-    # TODO: Remove in 1.1
-    # mypy error: Decorated property not supported
-    @deprecated("Attribute _pairwise was deprecated in "  # type: ignore
-                "version 0.24 and will be removed in 1.1 (renaming of 0.26).")
-    @property
-    def _pairwise(self):
-        # check if first estimator expects pairwise input
-        return getattr(self.steps[0][1], '_pairwise', False)
+        if not self.steps:
+            return tags
+
+        try:
+            if self.steps[0][1] is not None and self.steps[0][1] != "passthrough":
+                tags.input_tags.pairwise = get_tags(
+                    self.steps[0][1]
+                ).input_tags.pairwise
+            # WARNING: the sparse tag can be incorrect.
+            # Some Pipelines accepting sparse data are wrongly tagged sparse=False.
+            # For example Pipeline([PCA(), estimator]) accepts sparse data
+            # even if the estimator doesn't as PCA outputs a dense array.
+            tags.input_tags.sparse = all(
+                get_tags(step).input_tags.sparse
+                for name, step in self.steps
+                if step is not None and step != "passthrough"
+            )
+        except (ValueError, AttributeError, TypeError):
+            # This happens when the `steps` is not a list of (name, estimator)
+            # tuples and `fit` is not called yet to validate the steps.
+            pass
+
+        try:
+            if self.steps[-1][1] is not None and self.steps[-1][1] != "passthrough":
+                last_step_tags = get_tags(self.steps[-1][1])
+                tags.estimator_type = last_step_tags.estimator_type
+                tags.target_tags.multi_output = last_step_tags.target_tags.multi_output
+                tags.classifier_tags = deepcopy(last_step_tags.classifier_tags)
+                tags.regressor_tags = deepcopy(last_step_tags.regressor_tags)
+                tags.transformer_tags = deepcopy(last_step_tags.transformer_tags)
+        except (ValueError, AttributeError, TypeError):
+            # This happens when the `steps` is not a list of (name, estimator)
+            # tuples and `fit` is not called yet to validate the steps.
+            pass
+
+        return tags
+
+    def get_feature_names_out(self, input_features=None):
+        """Get output feature names for transformation.
+
+        Transform input features using the pipeline.
+
+        Parameters
+        ----------
+        input_features : array-like of str or None, default=None
+            Input features.
+
+        Returns
+        -------
+        feature_names_out : ndarray of str objects
+            Transformed feature names.
+        """
+        feature_names_out = input_features
+        for _, name, transform in self._iter():
+            if not hasattr(transform, "get_feature_names_out"):
+                raise AttributeError(
+                    "Estimator {} does not provide get_feature_names_out. "
+                    "Did you mean to call pipeline[:-1].get_feature_names_out"
+                    "()?".format(name)
+                )
+            feature_names_out = transform.get_feature_names_out(feature_names_out)
+        return feature_names_out
 
     @property
     def n_features_in_(self):
-        # delegate to first step (which will call _check_is_fitted)
+        """Number of features seen during first step `fit` method."""
+        # delegate to first step (which will call check_is_fitted)
         return self.steps[0][1].n_features_in_
 
+    @property
+    def feature_names_in_(self):
+        """Names of features seen during first step `fit` method."""
+        # delegate to first step (which will call check_is_fitted)
+        return self.steps[0][1].feature_names_in_
+
+    def __sklearn_is_fitted__(self):
+        """Indicate whether pipeline has been fit.
+
+        This is done by checking whether the last non-`passthrough` step of the
+        pipeline is fitted.
+
+        An empty pipeline is considered fitted.
+        """
+
+        # First find the last step that is not 'passthrough'
+        last_step = None
+        for _, estimator in reversed(self.steps):
+            if estimator != "passthrough":
+                last_step = estimator
+                break
+
+        if last_step is None:
+            # All steps are 'passthrough', so the pipeline is considered fitted
+            return True
+
+        try:
+            # check if the last step of the pipeline is fitted
+            # we only check the last step since if the last step is fit, it
+            # means the previous steps should also be fit. This is faster than
+            # checking if every step of the pipeline is fit.
+            check_is_fitted(last_step)
+            return True
+        except NotFittedError:
+            return False
+
     def _sk_visual_block_(self):
         _, estimators = zip(*self.steps)
 
         def _get_name(name, est):
-            if est is None or est == 'passthrough':
-                return f'{name}: passthrough'
+            if est is None or est == "passthrough":
+                return f"{name}: passthrough"
             # Is an estimator
-            return f'{name}: {est.__class__.__name__}'
+            return f"{name}: {est.__class__.__name__}"
+
         names = [_get_name(name, est) for name, est in self.steps]
         name_details = [str(est) for est in estimators]
-        return _VisualBlock('serial', estimators,
-                            names=names,
-                            name_details=name_details,
-                            dash_wrapped=False)
+        return _VisualBlock(
+            "serial",
+            estimators,
+            names=names,
+            name_details=name_details,
+            dash_wrapped=False,
+        )
+
+    def get_metadata_routing(self):
+        """Get metadata routing of this object.
+
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
+
+        Returns
+        -------
+        routing : MetadataRouter
+            A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
+            routing information.
+        """
+        router = MetadataRouter(owner=self.__class__.__name__)
+
+        # first we add all steps except the last one
+        for _, name, trans in self._iter(with_final=False, filter_passthrough=True):
+            method_mapping = MethodMapping()
+            # fit, fit_predict, and fit_transform call fit_transform if it
+            # exists, or else fit and transform
+            if hasattr(trans, "fit_transform"):
+                (
+                    method_mapping.add(caller="fit", callee="fit_transform")
+                    .add(caller="fit_transform", callee="fit_transform")
+                    .add(caller="fit_predict", callee="fit_transform")
+                )
+            else:
+                (
+                    method_mapping.add(caller="fit", callee="fit")
+                    .add(caller="fit", callee="transform")
+                    .add(caller="fit_transform", callee="fit")
+                    .add(caller="fit_transform", callee="transform")
+                    .add(caller="fit_predict", callee="fit")
+                    .add(caller="fit_predict", callee="transform")
+                )
+
+            (
+                method_mapping.add(caller="predict", callee="transform")
+                .add(caller="predict", callee="transform")
+                .add(caller="predict_proba", callee="transform")
+                .add(caller="decision_function", callee="transform")
+                .add(caller="predict_log_proba", callee="transform")
+                .add(caller="transform", callee="transform")
+                .add(caller="inverse_transform", callee="inverse_transform")
+                .add(caller="score", callee="transform")
+            )
+
+            router.add(method_mapping=method_mapping, **{name: trans})
+
+        final_name, final_est = self.steps[-1]
+        if final_est is None or final_est == "passthrough":
+            return router
+
+        # then we add the last step
+        method_mapping = MethodMapping()
+        if hasattr(final_est, "fit_transform"):
+            method_mapping.add(caller="fit_transform", callee="fit_transform")
+        else:
+            method_mapping.add(caller="fit", callee="fit").add(
+                caller="fit", callee="transform"
+            )
+        (
+            method_mapping.add(caller="fit", callee="fit")
+            .add(caller="predict", callee="predict")
+            .add(caller="fit_predict", callee="fit_predict")
+            .add(caller="predict_proba", callee="predict_proba")
+            .add(caller="decision_function", callee="decision_function")
+            .add(caller="predict_log_proba", callee="predict_log_proba")
+            .add(caller="transform", callee="transform")
+            .add(caller="inverse_transform", callee="inverse_transform")
+            .add(caller="score", callee="score")
+        )
+
+        router.add(method_mapping=method_mapping, **{final_name: final_est})
+        return router
 
 
 def _name_estimators(estimators):
     """Generate names for estimators."""
 
     names = [
-        estimator
-        if isinstance(estimator, str) else type(estimator).__name__.lower()
+        estimator if isinstance(estimator, str) else type(estimator).__name__.lower()
         for estimator in estimators
     ]
     namecount = defaultdict(int)
@@ -684,31 +1430,48 @@ def _name_estimators(estimators):
     return list(zip(names, estimators))
 
 
-def make_pipeline(*steps, memory=None, verbose=False):
-    """Construct a Pipeline from the given estimators.
+def make_pipeline(*steps, memory=None, transform_input=None, verbose=False):
+    """Construct a :class:`Pipeline` from the given estimators.
 
-    This is a shorthand for the Pipeline constructor; it does not require, and
-    does not permit, naming the estimators. Instead, their names will be set
-    to the lowercase of their types automatically.
+    This is a shorthand for the :class:`Pipeline` constructor; it does not
+    require, and does not permit, naming the estimators. Instead, their names
+    will be set to the lowercase of their types automatically.
 
     Parameters
     ----------
-    *steps : list of estimators.
+    *steps : list of Estimator objects
+        List of the scikit-learn estimators that are chained together.
 
     memory : str or object with the joblib.Memory interface, default=None
-        Used to cache the fitted transformers of the pipeline. By default,
-        no caching is performed. If a string is given, it is the path to
-        the caching directory. Enabling caching triggers a clone of
-        the transformers before fitting. Therefore, the transformer
-        instance given to the pipeline cannot be inspected
-        directly. Use the attribute ``named_steps`` or ``steps`` to
-        inspect estimators within the pipeline. Caching the
+        Used to cache the fitted transformers of the pipeline. The last step
+        will never be cached, even if it is a transformer. By default, no
+        caching is performed. If a string is given, it is the path to the
+        caching directory. Enabling caching triggers a clone of the transformers
+        before fitting. Therefore, the transformer instance given to the
+        pipeline cannot be inspected directly. Use the attribute ``named_steps``
+        or ``steps`` to inspect estimators within the pipeline. Caching the
         transformers is advantageous when fitting is time consuming.
 
+    transform_input : list of str, default=None
+        This enables transforming some input arguments to ``fit`` (other than ``X``)
+        to be transformed by the steps of the pipeline up to the step which requires
+        them. Requirement is defined via :ref:`metadata routing `.
+        This can be used to pass a validation set through the pipeline for instance.
+
+        You can only set this if metadata routing is enabled, which you
+        can enable using ``sklearn.set_config(enable_metadata_routing=True)``.
+
+        .. versionadded:: 1.6
+
     verbose : bool, default=False
         If True, the time elapsed while fitting each step will be printed as it
         is completed.
 
+    Returns
+    -------
+    p : Pipeline
+        Returns a scikit-learn :class:`Pipeline` object.
+
     See Also
     --------
     Pipeline : Class for creating a pipeline of transforms with a final
@@ -718,60 +1481,78 @@ def make_pipeline(*steps, memory=None, verbose=False):
     --------
     >>> from sklearn.naive_bayes import GaussianNB
     >>> from sklearn.preprocessing import StandardScaler
+    >>> from sklearn.pipeline import make_pipeline
     >>> make_pipeline(StandardScaler(), GaussianNB(priors=None))
     Pipeline(steps=[('standardscaler', StandardScaler()),
                     ('gaussiannb', GaussianNB())])
-
-    Returns
-    -------
-    p : Pipeline
     """
-    return Pipeline(_name_estimators(steps), memory=memory, verbose=verbose)
+    return Pipeline(
+        _name_estimators(steps),
+        transform_input=transform_input,
+        memory=memory,
+        verbose=verbose,
+    )
 
 
-def _transform_one(transformer, X, y, weight, **fit_params):
-    res = transformer.transform(X)
+def _transform_one(transformer, X, y, weight, params):
+    """Call transform and apply weight to output.
+
+    Parameters
+    ----------
+    transformer : estimator
+        Estimator to be used for transformation.
+
+    X : {array-like, sparse matrix} of shape (n_samples, n_features)
+        Input data to be transformed.
+
+    y : ndarray of shape (n_samples,)
+        Ignored.
+
+    weight : float
+        Weight to be applied to the output of the transformation.
+
+    params : dict
+        Parameters to be passed to the transformer's ``transform`` method.
+
+        This should be of the form ``process_routing()["step_name"]``.
+    """
+    res = transformer.transform(X, **params.transform)
     # if we have a weight for this transformer, multiply output
     if weight is None:
         return res
     return res * weight
 
 
-def _fit_transform_one(transformer,
-                       X,
-                       y,
-                       weight,
-                       message_clsname='',
-                       message=None,
-                       **fit_params):
+def _fit_transform_one(
+    transformer, X, y, weight, message_clsname="", message=None, params=None
+):
     """
     Fits ``transformer`` to ``X`` and ``y``. The transformed result is returned
     with the fitted transformer. If ``weight`` is not ``None``, the result will
     be multiplied by ``weight``.
+
+    ``params`` needs to be of the form ``process_routing()["step_name"]``.
     """
+    params = params or {}
     with _print_elapsed_time(message_clsname, message):
-        if hasattr(transformer, 'fit_transform'):
-            res = transformer.fit_transform(X, y, **fit_params)
+        if hasattr(transformer, "fit_transform"):
+            res = transformer.fit_transform(X, y, **params.get("fit_transform", {}))
         else:
-            res = transformer.fit(X, y, **fit_params).transform(X)
+            res = transformer.fit(X, y, **params.get("fit", {})).transform(
+                X, **params.get("transform", {})
+            )
 
     if weight is None:
         return res, transformer
     return res * weight, transformer
 
 
-def _fit_one(transformer,
-             X,
-             y,
-             weight,
-             message_clsname='',
-             message=None,
-             **fit_params):
+def _fit_one(transformer, X, y, weight, message_clsname="", message=None, params=None):
     """
     Fits ``transformer`` to ``X`` and ``y``.
     """
     with _print_elapsed_time(message_clsname, message):
-        return transformer.fit(X, y, **fit_params)
+        return transformer.fit(X, y, **params["fit"])
 
 
 class FeatureUnion(TransformerMixin, _BaseComposition):
@@ -783,8 +1564,9 @@ class FeatureUnion(TransformerMixin, _BaseComposition):
 
     Parameters of the transformers may be set using its name and the parameter
     name separated by a '__'. A transformer may be replaced entirely by
-    setting the parameter with its name to another transformer,
-    or removed by setting to 'drop'.
+    setting the parameter with its name to another transformer, removed by
+    setting to 'drop' or disabled by setting to 'passthrough' (features are
+    passed without transformation).
 
     Read more in the :ref:`User Guide `.
 
@@ -792,10 +1574,14 @@ class FeatureUnion(TransformerMixin, _BaseComposition):
 
     Parameters
     ----------
-    transformer_list : list of (string, transformer) tuples
+    transformer_list : list of (str, transformer) tuples
         List of transformer objects to be applied to the data. The first
-        half of each tuple is the name of the transformer. The tranformer can
-        be 'drop' for it to be ignored.
+        half of each tuple is the name of the transformer. The transformer can
+        be 'drop' for it to be ignored or can be 'passthrough' for features to
+        be passed unchanged.
+
+        .. versionadded:: 1.1
+           Added the option `"passthrough"`.
 
         .. versionchanged:: 0.22
            Deprecated `None` as a transformer in favor of 'drop'.
@@ -818,6 +1604,37 @@ class FeatureUnion(TransformerMixin, _BaseComposition):
         If True, the time elapsed while fitting each transformer will be
         printed as it is completed.
 
+    verbose_feature_names_out : bool, default=True
+        If True, :meth:`get_feature_names_out` will prefix all feature names
+        with the name of the transformer that generated that feature.
+        If False, :meth:`get_feature_names_out` will not prefix any feature
+        names and will error if feature names are not unique.
+
+        .. versionadded:: 1.5
+
+    Attributes
+    ----------
+    named_transformers : :class:`~sklearn.utils.Bunch`
+        Dictionary-like object, with the following attributes.
+        Read-only attribute to access any transformer parameter by user
+        given name. Keys are transformer names and values are
+        transformer parameters.
+
+        .. versionadded:: 1.2
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`. Only defined if the
+        underlying first transformer in `transformer_list` exposes such an
+        attribute when fit.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when
+        `X` has feature names that are all strings.
+
+        .. versionadded:: 1.3
+
     See Also
     --------
     make_union : Convenience function for simplified feature union
@@ -831,19 +1648,61 @@ class FeatureUnion(TransformerMixin, _BaseComposition):
     ...                       ("svd", TruncatedSVD(n_components=2))])
     >>> X = [[0., 1., 3], [2., 2., 5]]
     >>> union.fit_transform(X)
-    array([[ 1.5       ,  3.0...,  0.8...],
-           [-1.5       ,  5.7..., -0.4...]])
+    array([[-1.5       ,  3.04, -0.872],
+           [ 1.5       ,  5.72,  0.463]])
+    >>> # An estimator's parameter can be set using '__' syntax
+    >>> union.set_params(svd__n_components=1).fit_transform(X)
+    array([[-1.5       ,  3.04],
+           [ 1.5       ,  5.72]])
+
+    For a more detailed example of usage, see
+    :ref:`sphx_glr_auto_examples_compose_plot_feature_union.py`.
     """
-    _required_parameters = ["transformer_list"]
 
-    @_deprecate_positional_args
-    def __init__(self, transformer_list, *, n_jobs=None,
-                 transformer_weights=None, verbose=False):
+    def __init__(
+        self,
+        transformer_list,
+        *,
+        n_jobs=None,
+        transformer_weights=None,
+        verbose=False,
+        verbose_feature_names_out=True,
+    ):
         self.transformer_list = transformer_list
         self.n_jobs = n_jobs
         self.transformer_weights = transformer_weights
         self.verbose = verbose
-        self._validate_transformers()
+        self.verbose_feature_names_out = verbose_feature_names_out
+
+    def set_output(self, *, transform=None):
+        """Set the output container when `"transform"` and `"fit_transform"` are called.
+
+        `set_output` will set the output of all estimators in `transformer_list`.
+
+        Parameters
+        ----------
+        transform : {"default", "pandas", "polars"}, default=None
+            Configure output of `transform` and `fit_transform`.
+
+            - `"default"`: Default output format of a transformer
+            - `"pandas"`: DataFrame output
+            - `"polars"`: Polars output
+            - `None`: Transform configuration is unchanged
+
+        Returns
+        -------
+        self : estimator instance
+            Estimator instance.
+        """
+        super().set_output(transform=transform)
+        for _, step, _ in self._iter():
+            _safe_set_output(step, transform=transform)
+        return self
+
+    @property
+    def named_transformers(self):
+        # Use Bunch object to improve autocomplete
+        return Bunch(**dict(self.transformer_list))
 
     def get_params(self, deep=True):
         """Get parameters for this estimator.
@@ -863,20 +1722,28 @@ def get_params(self, deep=True):
         params : mapping of string to any
             Parameter names mapped to their values.
         """
-        return self._get_params('transformer_list', deep=deep)
+        return self._get_params("transformer_list", deep=deep)
 
     def set_params(self, **kwargs):
         """Set the parameters of this estimator.
 
         Valid parameter keys can be listed with ``get_params()``. Note that
         you can directly set the parameters of the estimators contained in
-        `tranformer_list`.
+        `transformer_list`.
+
+        Parameters
+        ----------
+        **kwargs : dict
+            Parameters of this estimator or parameters of estimators contained
+            in `transform_list`. Parameters of the transformers may be set
+            using its name and the parameter name separated by a '__'.
 
         Returns
         -------
-        self
+        self : object
+            FeatureUnion class instance.
         """
-        self._set_params('transformer_list', **kwargs)
+        self._set_params("transformer_list", **kwargs)
         return self
 
     def _validate_transformers(self):
@@ -887,13 +1754,15 @@ def _validate_transformers(self):
 
         # validate estimators
         for t in transformers:
-            if t == 'drop':
+            if t in ("drop", "passthrough"):
                 continue
-            if (not (hasattr(t, "fit") or hasattr(t, "fit_transform")) or not
-                    hasattr(t, "transform")):
-                raise TypeError("All estimators should implement fit and "
-                                "transform. '%s' (type %s) doesn't" %
-                                (t, type(t)))
+            if not (hasattr(t, "fit") or hasattr(t, "fit_transform")) or not hasattr(
+                t, "transform"
+            ):
+                raise TypeError(
+                    "All estimators should implement fit and "
+                    "transform. '%s' (type %s) doesn't" % (t, type(t))
+                )
 
     def _validate_transformer_weights(self):
         if not self.transformer_weights:
@@ -904,7 +1773,7 @@ def _validate_transformer_weights(self):
             if name not in transformer_names:
                 raise ValueError(
                     f'Attempting to weight transformer "{name}", '
-                    'but it is not present in transformer_list.'
+                    "but it is not present in transformer_list."
                 )
 
     def _iter(self):
@@ -912,28 +1781,91 @@ def _iter(self):
         Generate (name, trans, weight) tuples excluding None and
         'drop' transformers.
         """
+
         get_weight = (self.transformer_weights or {}).get
-        return ((name, trans, get_weight(name))
-                for name, trans in self.transformer_list
-                if trans != 'drop')
 
-    def get_feature_names(self):
-        """Get feature names from all transformers.
+        for name, trans in self.transformer_list:
+            if trans == "drop":
+                continue
+            if trans == "passthrough":
+                trans = FunctionTransformer(feature_names_out="one-to-one")
+            yield (name, trans, get_weight(name))
+
+    def get_feature_names_out(self, input_features=None):
+        """Get output feature names for transformation.
+
+        Parameters
+        ----------
+        input_features : array-like of str or None, default=None
+            Input features.
 
         Returns
         -------
-        feature_names : list of strings
-            Names of the features produced by transform.
+        feature_names_out : ndarray of str objects
+            Transformed feature names.
         """
-        feature_names = []
-        for name, trans, weight in self._iter():
-            if not hasattr(trans, 'get_feature_names'):
-                raise AttributeError("Transformer %s (type %s) does not "
-                                     "provide get_feature_names."
-                                     % (str(name), type(trans).__name__))
-            feature_names.extend([name + "__" + f for f in
-                                  trans.get_feature_names()])
-        return feature_names
+        # List of tuples (name, feature_names_out)
+        transformer_with_feature_names_out = []
+        for name, trans, _ in self._iter():
+            if not hasattr(trans, "get_feature_names_out"):
+                raise AttributeError(
+                    "Transformer %s (type %s) does not provide get_feature_names_out."
+                    % (str(name), type(trans).__name__)
+                )
+            feature_names_out = trans.get_feature_names_out(input_features)
+            transformer_with_feature_names_out.append((name, feature_names_out))
+
+        return self._add_prefix_for_feature_names_out(
+            transformer_with_feature_names_out
+        )
+
+    def _add_prefix_for_feature_names_out(self, transformer_with_feature_names_out):
+        """Add prefix for feature names out that includes the transformer names.
+
+        Parameters
+        ----------
+        transformer_with_feature_names_out : list of tuples of (str, array-like of str)
+            The tuple consistent of the transformer's name and its feature names out.
+
+        Returns
+        -------
+        feature_names_out : ndarray of shape (n_features,), dtype=str
+            Transformed feature names.
+        """
+        if self.verbose_feature_names_out:
+            # Prefix the feature names out with the transformers name
+            names = list(
+                chain.from_iterable(
+                    (f"{name}__{i}" for i in feature_names_out)
+                    for name, feature_names_out in transformer_with_feature_names_out
+                )
+            )
+            return np.asarray(names, dtype=object)
+
+        # verbose_feature_names_out is False
+        # Check that names are all unique without a prefix
+        feature_names_count = Counter(
+            chain.from_iterable(s for _, s in transformer_with_feature_names_out)
+        )
+        top_6_overlap = [
+            name for name, count in feature_names_count.most_common(6) if count > 1
+        ]
+        top_6_overlap.sort()
+        if top_6_overlap:
+            if len(top_6_overlap) == 6:
+                # There are more than 5 overlapping names, we only show the 5
+                # of the feature names
+                names_repr = str(top_6_overlap[:5])[:-1] + ", ...]"
+            else:
+                names_repr = str(top_6_overlap)
+            raise ValueError(
+                f"Output feature names: {names_repr} are not unique. Please set "
+                "verbose_feature_names_out=True to add prefixes to feature names"
+            )
+
+        return np.concatenate(
+            [name for _, name in transformer_with_feature_names_out],
+        )
 
     def fit(self, X, y=None, **fit_params):
         """Fit all transformers using X.
@@ -946,12 +1878,35 @@ def fit(self, X, y=None, **fit_params):
         y : array-like of shape (n_samples, n_outputs), default=None
             Targets for supervised learning.
 
+        **fit_params : dict, default=None
+            - If `enable_metadata_routing=False` (default):
+              Parameters directly passed to the `fit` methods of the
+              sub-transformers.
+
+            - If `enable_metadata_routing=True`:
+              Parameters safely routed to the `fit` methods of the
+              sub-transformers. See :ref:`Metadata Routing User Guide
+              ` for more details.
+
+            .. versionchanged:: 1.5
+                `**fit_params` can be routed via metadata routing API.
+
         Returns
         -------
-        self : FeatureUnion
-            This estimator
+        self : object
+            FeatureUnion class instance.
         """
-        transformers = self._parallel_func(X, y, fit_params, _fit_one)
+        if _routing_enabled():
+            routed_params = process_routing(self, "fit", **fit_params)
+        else:
+            # TODO(SLEP6): remove when metadata routing cannot be disabled.
+            routed_params = Bunch()
+            for name, _ in self.transformer_list:
+                routed_params[name] = Bunch(fit={})
+                routed_params[name].fit = fit_params
+
+        transformers = self._parallel_func(X, y, _fit_one, routed_params)
+
         if not transformers:
             # All transformers are None
             return self
@@ -959,7 +1914,7 @@ def fit(self, X, y=None, **fit_params):
         self._update_transformer_list(transformers)
         return self
 
-    def fit_transform(self, X, y=None, **fit_params):
+    def fit_transform(self, X, y=None, **params):
         """Fit all transformers, transform the data and concatenate results.
 
         Parameters
@@ -970,14 +1925,41 @@ def fit_transform(self, X, y=None, **fit_params):
         y : array-like of shape (n_samples, n_outputs), default=None
             Targets for supervised learning.
 
+        **params : dict, default=None
+            - If `enable_metadata_routing=False` (default):
+              Parameters directly passed to the `fit` methods of the
+              sub-transformers.
+
+            - If `enable_metadata_routing=True`:
+              Parameters safely routed to the `fit` methods of the
+              sub-transformers. See :ref:`Metadata Routing User Guide
+              ` for more details.
+
+            .. versionchanged:: 1.5
+                `**params` can now be routed via metadata routing API.
+
         Returns
         -------
         X_t : array-like or sparse matrix of \
                 shape (n_samples, sum_n_components)
-            hstack of results of transformers. sum_n_components is the
-            sum of n_components (output dimension) over transformers.
+            The `hstack` of results of transformers. `sum_n_components` is the
+            sum of `n_components` (output dimension) over transformers.
         """
-        results = self._parallel_func(X, y, fit_params, _fit_transform_one)
+        if _routing_enabled():
+            routed_params = process_routing(self, "fit_transform", **params)
+        else:
+            # TODO(SLEP6): remove when metadata routing cannot be disabled.
+            routed_params = Bunch()
+            for name, obj in self.transformer_list:
+                if hasattr(obj, "fit_transform"):
+                    routed_params[name] = Bunch(fit_transform={})
+                    routed_params[name].fit_transform = params
+                else:
+                    routed_params[name] = Bunch(fit={})
+                    routed_params[name] = Bunch(transform={})
+                    routed_params[name].fit = params
+
+        results = self._parallel_func(X, y, _fit_transform_one, routed_params)
         if not results:
             # All transformers are None
             return np.zeros((X.shape[0], 0))
@@ -990,23 +1972,29 @@ def fit_transform(self, X, y=None, **fit_params):
     def _log_message(self, name, idx, total):
         if not self.verbose:
             return None
-        return '(step %d of %d) Processing %s' % (idx, total, name)
+        return "(step %d of %d) Processing %s" % (idx, total, name)
 
-    def _parallel_func(self, X, y, fit_params, func):
+    def _parallel_func(self, X, y, func, routed_params):
         """Runs func in parallel on X and y"""
         self.transformer_list = list(self.transformer_list)
         self._validate_transformers()
         self._validate_transformer_weights()
         transformers = list(self._iter())
 
-        return Parallel(n_jobs=self.n_jobs)(delayed(func)(
-            transformer, X, y, weight,
-            message_clsname='FeatureUnion',
-            message=self._log_message(name, idx, len(transformers)),
-            **fit_params) for idx, (name, transformer,
-                                    weight) in enumerate(transformers, 1))
+        return Parallel(n_jobs=self.n_jobs)(
+            delayed(func)(
+                transformer,
+                X,
+                y,
+                weight,
+                message_clsname="FeatureUnion",
+                message=self._log_message(name, idx, len(transformers)),
+                params=routed_params[name],
+            )
+            for idx, (name, transformer, weight) in enumerate(transformers, 1)
+        )
 
-    def transform(self, X):
+    def transform(self, X, **params):
         """Transform X separately by each transformer, concatenate results.
 
         Parameters
@@ -1014,16 +2002,34 @@ def transform(self, X):
         X : iterable or array-like, depending on transformers
             Input data to be transformed.
 
+        **params : dict, default=None
+
+            Parameters routed to the `transform` method of the sub-transformers via the
+            metadata routing API. See :ref:`Metadata Routing User Guide
+            ` for more details.
+
+            .. versionadded:: 1.5
+
         Returns
         -------
-        X_t : array-like or sparse matrix of \
-                shape (n_samples, sum_n_components)
-            hstack of results of transformers. sum_n_components is the
-            sum of n_components (output dimension) over transformers.
+        X_t : array-like or sparse matrix of shape (n_samples, sum_n_components)
+            The `hstack` of results of transformers. `sum_n_components` is the
+            sum of `n_components` (output dimension) over transformers.
         """
+        _raise_for_params(params, self, "transform")
+
+        if _routing_enabled():
+            routed_params = process_routing(self, "transform", **params)
+        else:
+            # TODO(SLEP6): remove when metadata routing cannot be disabled.
+            routed_params = Bunch()
+            for name, _ in self.transformer_list:
+                routed_params[name] = Bunch(transform={})
+
         Xs = Parallel(n_jobs=self.n_jobs)(
-            delayed(_transform_one)(trans, X, None, weight)
-            for name, trans, weight in self._iter())
+            delayed(_transform_one)(trans, X, None, weight, params=routed_params[name])
+            for name, trans, weight in self._iter()
+        )
         if not Xs:
             # All transformers are None
             return np.zeros((X.shape[0], 0))
@@ -1031,6 +2037,10 @@ def transform(self, X):
         return self._hstack(Xs)
 
     def _hstack(self, Xs):
+        adapter = _get_container_adapter("transform", self)
+        if adapter and all(adapter.is_supported_container(X) for X in Xs):
+            return adapter.hstack(Xs)
+
         if any(sparse.issparse(f) for f in Xs):
             Xs = sparse.hstack(Xs).tocsr()
         else:
@@ -1039,31 +2049,99 @@ def _hstack(self, Xs):
 
     def _update_transformer_list(self, transformers):
         transformers = iter(transformers)
-        self.transformer_list[:] = [(name, old if old == 'drop'
-                                     else next(transformers))
-                                    for name, old in self.transformer_list]
+        self.transformer_list[:] = [
+            (name, old if old == "drop" else next(transformers))
+            for name, old in self.transformer_list
+        ]
 
     @property
     def n_features_in_(self):
+        """Number of features seen during :term:`fit`."""
+
         # X is passed to all transformers so we just delegate to the first one
         return self.transformer_list[0][1].n_features_in_
 
+    @property
+    def feature_names_in_(self):
+        """Names of features seen during :term:`fit`."""
+        # X is passed to all transformers -- delegate to the first one
+        return self.transformer_list[0][1].feature_names_in_
+
+    def __sklearn_is_fitted__(self):
+        # Delegate whether feature union was fitted
+        for _, transformer, _ in self._iter():
+            check_is_fitted(transformer)
+        return True
+
     def _sk_visual_block_(self):
         names, transformers = zip(*self.transformer_list)
-        return _VisualBlock('parallel', transformers, names=names)
+        return _VisualBlock("parallel", transformers, names=names)
 
+    def __getitem__(self, name):
+        """Return transformer with name."""
+        if not isinstance(name, str):
+            raise KeyError("Only string keys are supported")
+        return self.named_transformers[name]
 
-def make_union(*transformers, n_jobs=None, verbose=False):
-    """
-    Construct a FeatureUnion from the given transformers.
+    def get_metadata_routing(self):
+        """Get metadata routing of this object.
+
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
+
+        .. versionadded:: 1.5
+
+        Returns
+        -------
+        routing : MetadataRouter
+            A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
+            routing information.
+        """
+        router = MetadataRouter(owner=self.__class__.__name__)
+
+        for name, transformer in self.transformer_list:
+            router.add(
+                **{name: transformer},
+                method_mapping=MethodMapping()
+                .add(caller="fit", callee="fit")
+                .add(caller="fit_transform", callee="fit_transform")
+                .add(caller="fit_transform", callee="fit")
+                .add(caller="fit_transform", callee="transform")
+                .add(caller="transform", callee="transform"),
+            )
 
-    This is a shorthand for the FeatureUnion constructor; it does not require,
-    and does not permit, naming the transformers. Instead, they will be given
-    names automatically based on their types. It also does not allow weighting.
+        return router
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        try:
+            tags.input_tags.sparse = all(
+                get_tags(trans).input_tags.sparse
+                for name, trans in self.transformer_list
+                if trans not in {"passthrough", "drop"}
+            )
+        except Exception:
+            # If `transformer_list` does not comply with our API (list of tuples)
+            # then it will fail. In this case, we assume that `sparse` is False
+            # but the parameter validation will raise an error during `fit`.
+            pass  # pragma: no cover
+        return tags
+
+
+def make_union(
+    *transformers, n_jobs=None, verbose=False, verbose_feature_names_out=True
+):
+    """Construct a :class:`FeatureUnion` from the given transformers.
+
+    This is a shorthand for the :class:`FeatureUnion` constructor; it does not
+    require, and does not permit, naming the transformers. Instead, they will
+    be given names automatically based on their types. It also does not allow
+    weighting.
 
     Parameters
     ----------
     *transformers : list of estimators
+        One or more estimators.
 
     n_jobs : int, default=None
         Number of jobs to run in parallel.
@@ -1072,15 +2150,21 @@ def make_union(*transformers, n_jobs=None, verbose=False):
         for more details.
 
         .. versionchanged:: v0.20
-           `n_jobs` default changed from 1 to None
+           `n_jobs` default changed from 1 to None.
 
     verbose : bool, default=False
         If True, the time elapsed while fitting each transformer will be
         printed as it is completed.
 
+    verbose_feature_names_out : bool, default=True
+        If True, the feature names generated by `get_feature_names_out` will
+        include prefixes derived from the transformer names.
+
     Returns
     -------
     f : FeatureUnion
+        A :class:`FeatureUnion` object for concatenating the results of multiple
+        transformer objects.
 
     See Also
     --------
@@ -1096,4 +2180,8 @@ def make_union(*transformers, n_jobs=None, verbose=False):
                                    ('truncatedsvd', TruncatedSVD())])
     """
     return FeatureUnion(
-        _name_estimators(transformers), n_jobs=n_jobs, verbose=verbose)
+        _name_estimators(transformers),
+        n_jobs=n_jobs,
+        verbose=verbose,
+        verbose_feature_names_out=verbose_feature_names_out,
+    )
diff --git a/sklearn/preprocessing/__init__.py b/sklearn/preprocessing/__init__.py
index 076b9e85e1150..48bb3aa6a7a4e 100644
--- a/sklearn/preprocessing/__init__.py
+++ b/sklearn/preprocessing/__init__.py
@@ -1,70 +1,63 @@
-"""
-The :mod:`sklearn.preprocessing` module includes scaling, centering,
-normalization, binarization methods.
-"""
-
-from ._function_transformer import FunctionTransformer
-
-from ._data import Binarizer
-from ._data import KernelCenterer
-from ._data import MinMaxScaler
-from ._data import MaxAbsScaler
-from ._data import Normalizer
-from ._data import RobustScaler
-from ._data import StandardScaler
-from ._data import QuantileTransformer
-from ._data import add_dummy_feature
-from ._data import binarize
-from ._data import normalize
-from ._data import scale
-from ._data import robust_scale
-from ._data import maxabs_scale
-from ._data import minmax_scale
-from ._data import quantile_transform
-from ._data import power_transform
-from ._data import PowerTransformer
-from ._data import PolynomialFeatures
-
-from ._encoders import OneHotEncoder
-from ._encoders import OrdinalEncoder
-
-from ._label import label_binarize
-from ._label import LabelBinarizer
-from ._label import LabelEncoder
-from ._label import MultiLabelBinarizer
-
+"""Methods for scaling, centering, normalization, binarization, and more."""
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+from ._data import (
+    Binarizer,
+    KernelCenterer,
+    MaxAbsScaler,
+    MinMaxScaler,
+    Normalizer,
+    PowerTransformer,
+    QuantileTransformer,
+    RobustScaler,
+    StandardScaler,
+    add_dummy_feature,
+    binarize,
+    maxabs_scale,
+    minmax_scale,
+    normalize,
+    power_transform,
+    quantile_transform,
+    robust_scale,
+    scale,
+)
 from ._discretization import KBinsDiscretizer
-
-from ._polynomial import SplineTransformer
-
+from ._encoders import OneHotEncoder, OrdinalEncoder
+from ._function_transformer import FunctionTransformer
+from ._label import LabelBinarizer, LabelEncoder, MultiLabelBinarizer, label_binarize
+from ._polynomial import PolynomialFeatures, SplineTransformer
+from ._target_encoder import TargetEncoder
 
 __all__ = [
-    'Binarizer',
-    'FunctionTransformer',
-    'KBinsDiscretizer',
-    'KernelCenterer',
-    'LabelBinarizer',
-    'LabelEncoder',
-    'MultiLabelBinarizer',
-    'MinMaxScaler',
-    'MaxAbsScaler',
-    'QuantileTransformer',
-    'Normalizer',
-    'OneHotEncoder',
-    'OrdinalEncoder',
-    'PowerTransformer',
-    'RobustScaler',
-    'SplineTransformer',
-    'StandardScaler',
-    'add_dummy_feature',
-    'PolynomialFeatures',
-    'binarize',
-    'normalize',
-    'scale',
-    'robust_scale',
-    'maxabs_scale',
-    'minmax_scale',
-    'label_binarize',
-    'quantile_transform',
-    'power_transform',
+    "Binarizer",
+    "FunctionTransformer",
+    "KBinsDiscretizer",
+    "KernelCenterer",
+    "LabelBinarizer",
+    "LabelEncoder",
+    "MaxAbsScaler",
+    "MinMaxScaler",
+    "MultiLabelBinarizer",
+    "Normalizer",
+    "OneHotEncoder",
+    "OrdinalEncoder",
+    "PolynomialFeatures",
+    "PowerTransformer",
+    "QuantileTransformer",
+    "RobustScaler",
+    "SplineTransformer",
+    "StandardScaler",
+    "TargetEncoder",
+    "add_dummy_feature",
+    "binarize",
+    "label_binarize",
+    "maxabs_scale",
+    "minmax_scale",
+    "normalize",
+    "power_transform",
+    "quantile_transform",
+    "robust_scale",
+    "scale",
 ]
diff --git a/sklearn/preprocessing/_csr_polynomial_expansion.pyx b/sklearn/preprocessing/_csr_polynomial_expansion.pyx
index 84fef3f042dc7..38e5c3069d252 100644
--- a/sklearn/preprocessing/_csr_polynomial_expansion.pyx
+++ b/sklearn/preprocessing/_csr_polynomial_expansion.pyx
@@ -1,79 +1,206 @@
-# cython: cdivision=True
-# cython: boundscheck=False
-# cython: wraparound=False
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-# Author: Andrew nystrom 
+from ..utils._typedefs cimport uint8_t, int64_t, intp_t
 
-from scipy.sparse import csr_matrix
-from numpy cimport ndarray
-cimport numpy as np
+ctypedef uint8_t FLAG_t
 
-np.import_array()
-ctypedef np.int32_t INDEX_T
+# We use the following verbatim block to determine whether the current
+# platform's compiler supports 128-bit integer values intrinsically.
+# This should work for GCC and CLANG on 64-bit architectures, but doesn't for
+# MSVC on any architecture. We prefer to use 128-bit integers when possible
+# because the intermediate calculations have a non-trivial risk of overflow. It
+# is, however, very unlikely to come up on an average use case, hence 64-bit
+# integers (i.e. `long long`) are "good enough" for most common cases. There is
+# not much we can do to efficiently mitigate  the overflow risk on the Windows
+# platform at this time. Consider this a "best effort" design decision that
+# could be revisited later in case someone comes up with a safer option that
+# does not hurt the performance of the common cases.
+# See `test_sizeof_LARGEST_INT_t()`for more information on exact type expectations.
+cdef extern from *:
+    """
+    #ifdef __SIZEOF_INT128__
+        typedef __int128 LARGEST_INT_t;
+    #elif (__clang__ || __EMSCRIPTEN__) && !__i386__
+        typedef _BitInt(128) LARGEST_INT_t;
+    #else
+        typedef long long LARGEST_INT_t;
+    #endif
+    """
+    ctypedef long long LARGEST_INT_t
+
+
+# Determine the size of `LARGEST_INT_t` at runtime.
+# Used in `test_sizeof_LARGEST_INT_t`.
+def _get_sizeof_LARGEST_INT_t():
+    return sizeof(LARGEST_INT_t)
 
-ctypedef fused DATA_T:
-    np.float32_t
-    np.float64_t
-    np.int32_t
-    np.int64_t
 
+# TODO: use `{int,float}{32,64}_t` when cython#5230 is resolved:
+# https://github.com/cython/cython/issues/5230
+ctypedef fused DATA_t:
+    float
+    double
+    int
+    long long
+# INDEX_{A,B}_t are defined to generate a proper Cartesian product
+# of types through Cython fused-type expansion.
+ctypedef fused INDEX_A_t:
+    signed int
+    signed long long
+ctypedef fused INDEX_B_t:
+    signed int
+    signed long long
 
-cdef inline INDEX_T _deg2_column(INDEX_T d, INDEX_T i, INDEX_T j,
-                                 INDEX_T interaction_only) nogil:
+cdef inline int64_t _deg2_column(
+    LARGEST_INT_t n_features,
+    LARGEST_INT_t i,
+    LARGEST_INT_t j,
+    FLAG_t interaction_only
+) nogil:
     """Compute the index of the column for a degree 2 expansion
 
-    d is the dimensionality of the input data, i and j are the indices
+    n_features is the dimensionality of the input data, i and j are the indices
     for the columns involved in the expansion.
     """
     if interaction_only:
-        return d * i - (i**2 + 3 * i) / 2 - 1 + j
+        return n_features * i - i * (i + 3) / 2 - 1 + j
     else:
-        return d * i - (i**2 + i) / 2 + j
+        return n_features * i - i* (i + 1) / 2 + j
 
 
-cdef inline INDEX_T _deg3_column(INDEX_T d, INDEX_T i, INDEX_T j, INDEX_T k,
-                                 INDEX_T interaction_only) nogil:
+cdef inline int64_t _deg3_column(
+    LARGEST_INT_t n_features,
+    LARGEST_INT_t i,
+    LARGEST_INT_t j,
+    LARGEST_INT_t k,
+    FLAG_t interaction_only
+) nogil:
     """Compute the index of the column for a degree 3 expansion
 
-    d is the dimensionality of the input data, i, j and k are the indices
+    n_features is the dimensionality of the input data, i, j and k are the indices
     for the columns involved in the expansion.
     """
     if interaction_only:
-        return ((3 * d**2 * i - 3 * d * i**2 + i**3
-                 + 11 * i - 3 * j**2 - 9 * j) / 6
-                + i**2 - 2 * d * i + d * j - d + k)
+        return (
+            (
+                (3 * n_features) * (n_features * i - i**2)
+                + i * (i**2 + 11) - (3 * j) * (j + 3)
+            ) / 6 + i**2 + n_features * (j - 1 - 2 * i) + k
+        )
+    else:
+        return (
+            (
+                (3 * n_features) * (n_features * i - i**2)
+                + i ** 3 - i - (3 * j) * (j + 1)
+            ) / 6 + n_features * j + k
+        )
+
+
+def py_calc_expanded_nnz_deg2(n, interaction_only):
+    return n * (n + 1) // 2 - interaction_only * n
+
+
+def py_calc_expanded_nnz_deg3(n, interaction_only):
+    return n * (n**2 + 3 * n + 2) // 6 - interaction_only * n**2
+
+
+cpdef int64_t _calc_expanded_nnz(
+    LARGEST_INT_t n,
+    FLAG_t interaction_only,
+    LARGEST_INT_t degree
+):
+    """
+    Calculates the number of non-zero interaction terms generated by the
+    non-zero elements of a single row.
+    """
+    # This is the maximum value before the intermediate computation
+    # d**2 + d overflows
+    # Solution to d**2 + d = maxint64
+    # SymPy: solve(x**2 + x - int64_max, x)
+    cdef int64_t MAX_SAFE_INDEX_CALC_DEG2 = 3037000499
+
+    # This is the maximum value before the intermediate computation
+    # d**3 + 3 * d**2 + 2*d overflows
+    # Solution to d**3 + 3 * d**2 + 2*d = maxint64
+    # SymPy: solve(x * (x**2 + 3 * x + 2) - int64_max, x)
+    cdef int64_t MAX_SAFE_INDEX_CALC_DEG3 = 2097151
+
+    if degree == 2:
+        # Only need to check when not using 128-bit integers
+        if sizeof(LARGEST_INT_t) < 16 and n <= MAX_SAFE_INDEX_CALC_DEG2:
+            return n * (n + 1) / 2 - interaction_only * n
+        return  py_calc_expanded_nnz_deg2(n, interaction_only)
     else:
-        return ((3 * d**2 * i - 3 * d * i**2 + i ** 3 - i
-                 - 3 * j**2 - 3 * j) / 6
-                + d * j + k)
+        # Only need to check when not using 128-bit integers
+        if sizeof(LARGEST_INT_t) < 16 and n <= MAX_SAFE_INDEX_CALC_DEG3:
+            return n * (n**2 + 3 * n + 2) / 6 - interaction_only * n**2
+        return  py_calc_expanded_nnz_deg3(n, interaction_only)
 
+cpdef int64_t _calc_total_nnz(
+    INDEX_A_t[:] indptr,
+    FLAG_t interaction_only,
+    int64_t degree,
+):
+    """
+    Calculates the number of non-zero interaction terms generated by the
+    non-zero elements across all rows for a single degree.
+    """
+    cdef int64_t total_nnz=0
+    cdef intp_t row_idx
+    for row_idx in range(len(indptr) - 1):
+        total_nnz += _calc_expanded_nnz(
+            indptr[row_idx + 1] - indptr[row_idx],
+            interaction_only,
+            degree
+        )
+    return total_nnz
 
-def _csr_polynomial_expansion(ndarray[DATA_T, ndim=1] data,
-                              ndarray[INDEX_T, ndim=1] indices,
-                              ndarray[INDEX_T, ndim=1] indptr,
-                              INDEX_T d, INDEX_T interaction_only,
-                              INDEX_T degree):
+
+cpdef void _csr_polynomial_expansion(
+    const DATA_t[:] data,           # IN READ-ONLY
+    const INDEX_A_t[:] indices,     # IN READ-ONLY
+    const INDEX_A_t[:] indptr,      # IN READ-ONLY
+    INDEX_A_t n_features,
+    DATA_t[:] result_data,          # OUT
+    INDEX_B_t[:] result_indices,    # OUT
+    INDEX_B_t[:] result_indptr,     # OUT
+    FLAG_t interaction_only,
+    FLAG_t degree
+):
     """
-    Perform a second-degree polynomial or interaction expansion on a scipy
+    Perform a second or third degree polynomial or interaction expansion on a
     compressed sparse row (CSR) matrix. The method used only takes products of
-    non-zero features. For a matrix with density d, this results in a speedup
-    on the order of d^k where k is the degree of the expansion, assuming all
-    rows are of similar density.
+    non-zero features. For a matrix with density :math:`d`, this results in a
+    speedup on the order of :math:`(1/d)^k` where :math:`k` is the degree of
+    the expansion, assuming all rows are of similar density.
 
     Parameters
     ----------
-    data : nd-array
+    data : memory view on nd-array
         The "data" attribute of the input CSR matrix.
 
-    indices : nd-array
+    indices : memory view on nd-array
         The "indices" attribute of the input CSR matrix.
 
-    indptr : nd-array
+    indptr : memory view on nd-array
         The "indptr" attribute of the input CSR matrix.
 
-    d : int
+    n_features : int
         The dimensionality of the input CSR matrix.
 
+    result_data : nd-array
+        The output CSR matrix's "data" attribute.
+        It is modified by this routine.
+
+    result_indices : nd-array
+        The output CSR matrix's "indices" attribute.
+        It is modified by this routine.
+
+    result_indptr : nd-array
+        The output CSR matrix's "indptr" attribute.
+        It is modified by this routine.
+
     interaction_only : int
         0 for a polynomial expansion, 1 for an interaction expansion.
 
@@ -86,44 +213,11 @@ def _csr_polynomial_expansion(ndarray[DATA_T, ndim=1] data,
     Matrices Using K-Simplex Numbers" by Andrew Nystrom and John Hughes.
     """
 
-    assert degree in (2, 3)
-
-    if degree == 2:
-        expanded_dimensionality = int((d**2 + d) / 2 - interaction_only*d)
-    else:
-        expanded_dimensionality = int((d**3 + 3*d**2 + 2*d) / 6
-                                      - interaction_only*d**2)
-    if expanded_dimensionality == 0:
-        return None
-    assert expanded_dimensionality > 0
-
-    cdef INDEX_T total_nnz = 0, row_i, nnz
-
-    # Count how many nonzero elements the expanded matrix will contain.
-    for row_i in range(indptr.shape[0]-1):
-        # nnz is the number of nonzero elements in this row.
-        nnz = indptr[row_i + 1] - indptr[row_i]
-        if degree == 2:
-            total_nnz += (nnz ** 2 + nnz) / 2 - interaction_only * nnz
-        else:
-            total_nnz += ((nnz ** 3 + 3 * nnz ** 2 + 2 * nnz) / 6
-                          - interaction_only * nnz ** 2)
-
     # Make the arrays that will form the CSR matrix of the expansion.
-    cdef ndarray[DATA_T, ndim=1] expanded_data = ndarray(
-        shape=total_nnz, dtype=data.dtype)
-    cdef ndarray[INDEX_T, ndim=1] expanded_indices = ndarray(
-        shape=total_nnz, dtype=indices.dtype)
-    cdef INDEX_T num_rows = indptr.shape[0] - 1
-    cdef ndarray[INDEX_T, ndim=1] expanded_indptr = ndarray(
-        shape=num_rows + 1, dtype=indptr.dtype)
-
-    cdef INDEX_T expanded_index = 0, row_starts, row_ends, i, j, k, \
-                 i_ptr, j_ptr, k_ptr, num_cols_in_row,  \
-                 expanded_column
-
+    cdef INDEX_A_t row_i, row_starts, row_ends, i, j, k, i_ptr, j_ptr, k_ptr
+    cdef INDEX_B_t expanded_index=0, num_cols_in_row, col
     with nogil:
-        expanded_indptr[0] = indptr[0]
+        result_indptr[0] = indptr[0]
         for row_i in range(indptr.shape[0]-1):
             row_starts = indptr[row_i]
             row_ends = indptr[row_i + 1]
@@ -133,25 +227,32 @@ def _csr_polynomial_expansion(ndarray[DATA_T, ndim=1] data,
                 for j_ptr in range(i_ptr + interaction_only, row_ends):
                     j = indices[j_ptr]
                     if degree == 2:
-                        col = _deg2_column(d, i, j, interaction_only)
-                        expanded_indices[expanded_index] = col
-                        expanded_data[expanded_index] = (
-                            data[i_ptr] * data[j_ptr])
+                        col =  _deg2_column(
+                            n_features,
+                            i, j,
+                            interaction_only
+                        )
+                        result_indices[expanded_index] = col
+                        result_data[expanded_index] = (
+                            data[i_ptr] * data[j_ptr]
+                        )
                         expanded_index += 1
                         num_cols_in_row += 1
                     else:
                         # degree == 3
-                        for k_ptr in range(j_ptr + interaction_only,
-                                            row_ends):
+                        for k_ptr in range(j_ptr + interaction_only, row_ends):
                             k = indices[k_ptr]
-                            col = _deg3_column(d, i, j, k, interaction_only)
-                            expanded_indices[expanded_index] = col
-                            expanded_data[expanded_index] = (
-                                data[i_ptr] * data[j_ptr] * data[k_ptr])
+                            col =  _deg3_column(
+                                n_features,
+                                i, j, k,
+                                interaction_only
+                            )
+                            result_indices[expanded_index] = col
+                            result_data[expanded_index] = (
+                                data[i_ptr] * data[j_ptr] * data[k_ptr]
+                            )
                             expanded_index += 1
                             num_cols_in_row += 1
 
-            expanded_indptr[row_i+1] = expanded_indptr[row_i] + num_cols_in_row
-
-    return csr_matrix((expanded_data, expanded_indices, expanded_indptr),
-                      shape=(num_rows, expanded_dimensionality))
+            result_indptr[row_i+1] = result_indptr[row_i] + num_cols_in_row
+    return
diff --git a/sklearn/preprocessing/_data.py b/sklearn/preprocessing/_data.py
index 29190dd6e2b67..fe138cda73803 100644
--- a/sklearn/preprocessing/_data.py
+++ b/sklearn/preprocessing/_data.py
@@ -1,65 +1,94 @@
-# Authors: Alexandre Gramfort 
-#          Mathieu Blondel 
-#          Olivier Grisel 
-#          Andreas Mueller 
-#          Eric Martin 
-#          Giorgio Patrini 
-#          Eric Chang 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 
-from itertools import chain, combinations
 import warnings
-from itertools import combinations_with_replacement as combinations_w_r
+from numbers import Integral, Real
 
 import numpy as np
-from scipy import sparse
-from scipy import stats
-from scipy import optimize
-from scipy.special import boxcox
-
-from ..base import BaseEstimator, TransformerMixin
-from ..utils import check_array
-from ..utils.deprecation import deprecated
-from ..utils.extmath import row_norms
-from ..utils.extmath import _incremental_mean_and_var
-from ..utils.sparsefuncs_fast import (inplace_csr_row_normalize_l1,
-                                      inplace_csr_row_normalize_l2)
-from ..utils.sparsefuncs import (inplace_column_scale,
-                                 mean_variance_axis, incr_mean_variance_axis,
-                                 min_max_axis)
-from ..utils.validation import (check_is_fitted, check_random_state,
-                                _check_sample_weight,
-                                FLOAT_DTYPES, _deprecate_positional_args)
-from ._csr_polynomial_expansion import _csr_polynomial_expansion
-
+from scipy import sparse, stats
+from scipy.special import boxcox, inv_boxcox
+
+from sklearn.utils import metadata_routing
+
+from ..base import (
+    BaseEstimator,
+    ClassNamePrefixFeaturesOutMixin,
+    OneToOneFeatureMixin,
+    TransformerMixin,
+    _fit_context,
+)
+from ..utils import _array_api, check_array, resample
+from ..utils._array_api import (
+    _find_matching_floating_dtype,
+    _modify_in_place_if_numpy,
+    device,
+    get_namespace,
+    get_namespace_and_device,
+)
+from ..utils._param_validation import Interval, Options, StrOptions, validate_params
+from ..utils.extmath import _incremental_mean_and_var, row_norms
+from ..utils.fixes import _yeojohnson_lambda
+from ..utils.sparsefuncs import (
+    incr_mean_variance_axis,
+    inplace_column_scale,
+    mean_variance_axis,
+    min_max_axis,
+)
+from ..utils.sparsefuncs_fast import (
+    inplace_csr_row_normalize_l1,
+    inplace_csr_row_normalize_l2,
+)
+from ..utils.validation import (
+    FLOAT_DTYPES,
+    _check_sample_weight,
+    check_is_fitted,
+    check_random_state,
+    validate_data,
+)
 from ._encoders import OneHotEncoder
 
 BOUNDS_THRESHOLD = 1e-7
 
 __all__ = [
-    'Binarizer',
-    'KernelCenterer',
-    'MinMaxScaler',
-    'MaxAbsScaler',
-    'Normalizer',
-    'OneHotEncoder',
-    'RobustScaler',
-    'StandardScaler',
-    'QuantileTransformer',
-    'PowerTransformer',
-    'add_dummy_feature',
-    'binarize',
-    'normalize',
-    'scale',
-    'robust_scale',
-    'maxabs_scale',
-    'minmax_scale',
-    'quantile_transform',
-    'power_transform',
+    "Binarizer",
+    "KernelCenterer",
+    "MaxAbsScaler",
+    "MinMaxScaler",
+    "Normalizer",
+    "OneHotEncoder",
+    "PowerTransformer",
+    "QuantileTransformer",
+    "RobustScaler",
+    "StandardScaler",
+    "add_dummy_feature",
+    "binarize",
+    "maxabs_scale",
+    "minmax_scale",
+    "normalize",
+    "power_transform",
+    "quantile_transform",
+    "robust_scale",
+    "scale",
 ]
 
 
+def _is_constant_feature(var, mean, n_samples):
+    """Detect if a feature is indistinguishable from a constant feature.
+
+    The detection is based on its computed variance and on the theoretical
+    error bounds of the '2 pass algorithm' for variance computation.
+
+    See "Algorithms for computing the sample variance: analysis and
+    recommendations", by Chan, Golub, and LeVeque.
+    """
+    # In scikit-learn, variance is always computed using float64 accumulators.
+    eps = np.finfo(np.float64).eps
+
+    upper_bound = n_samples * eps * var + (n_samples * mean * eps) ** 2
+    return var <= upper_bound
+
+
 def _handle_zeros_in_scale(scale, copy=True, constant_mask=None):
     """Set scales of near constant features to 1.
 
@@ -76,24 +105,35 @@ def _handle_zeros_in_scale(scale, copy=True, constant_mask=None):
     """
     # if we are fitting on 1D arrays, scale might be a scalar
     if np.isscalar(scale):
-        if scale == .0:
-            scale = 1.
+        if scale == 0.0:
+            scale = 1.0
         return scale
-    elif isinstance(scale, np.ndarray):
+    # scale is an array
+    else:
+        xp, _ = get_namespace(scale)
         if constant_mask is None:
             # Detect near constant values to avoid dividing by a very small
-            # value that could lead to suprising results and numerical
+            # value that could lead to surprising results and numerical
             # stability issues.
-            constant_mask = scale < 10 * np.finfo(scale.dtype).eps
+            constant_mask = scale < 10 * xp.finfo(scale.dtype).eps
 
         if copy:
             # New array to avoid side-effects
-            scale = scale.copy()
+            scale = xp.asarray(scale, copy=True)
         scale[constant_mask] = 1.0
         return scale
 
 
-@_deprecate_positional_args
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "axis": [Options(Integral, {0, 1})],
+        "with_mean": ["boolean"],
+        "with_std": ["boolean"],
+        "copy": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
 def scale(X, *, axis=0, with_mean=True, with_std=True, copy=True):
     """Standardize a dataset along any axis.
 
@@ -106,8 +146,8 @@ def scale(X, *, axis=0, with_mean=True, with_std=True, copy=True):
     X : {array-like, sparse matrix} of shape (n_samples, n_features)
         The data to center and scale.
 
-    axis : int, default=0
-        axis used to compute the means and standard deviations along. If 0,
+    axis : {0, 1}, default=0
+        Axis used to compute the means and standard deviations along. If 0,
         independently standardize each feature, otherwise (if 1) standardize
         each sample.
 
@@ -119,15 +159,22 @@ def scale(X, *, axis=0, with_mean=True, with_std=True, copy=True):
         unit standard deviation).
 
     copy : bool, default=True
-        set to False to perform inplace row normalization and avoid a
-        copy (if the input is already a numpy array or a scipy.sparse
-        CSC matrix and if axis is 1).
+        If False, try to avoid a copy and scale in place.
+        This is not guaranteed to always work in place; e.g. if the data is
+        a numpy array with an int dtype, a copy will be returned even with
+        copy=False.
 
     Returns
     -------
     X_tr : {ndarray, sparse matrix} of shape (n_samples, n_features)
         The transformed data.
 
+    See Also
+    --------
+    StandardScaler : Performs scaling to unit variance using the Transformer
+        API (e.g. as part of a preprocessing
+        :class:`~sklearn.pipeline.Pipeline`).
+
     Notes
     -----
     This implementation will refuse to center scipy.sparse matrices
@@ -149,8 +196,7 @@ def scale(X, *, axis=0, with_mean=True, with_std=True, copy=True):
     affect model performance.
 
     For a comparison of the different scalers, transformers, and normalizers,
-    see :ref:`examples/preprocessing/plot_all_scaling.py
-    `.
+    see: :ref:`sphx_glr_auto_examples_preprocessing_plot_all_scaling.py`.
 
     .. warning:: Risk of data leak
 
@@ -164,24 +210,36 @@ def scale(X, *, axis=0, with_mean=True, with_std=True, copy=True):
         :ref:`Pipeline ` in order to prevent most risks of data
         leaking: `pipe = make_pipeline(StandardScaler(), LogisticRegression())`.
 
-    See Also
+    Examples
     --------
-    StandardScaler : Performs scaling to unit variance using the Transformer
-        API (e.g. as part of a preprocessing
-        :class:`~sklearn.pipeline.Pipeline`).
-
-    """  # noqa
-    X = check_array(X, accept_sparse='csc', copy=copy, ensure_2d=False,
-                    estimator='the scale function', dtype=FLOAT_DTYPES,
-                    force_all_finite='allow-nan')
+    >>> from sklearn.preprocessing import scale
+    >>> X = [[-2, 1, 2], [-1, 0, 1]]
+    >>> scale(X, axis=0)  # scaling each column independently
+    array([[-1.,  1.,  1.],
+           [ 1., -1., -1.]])
+    >>> scale(X, axis=1)  # scaling each row independently
+    array([[-1.37,  0.39,  0.98],
+           [-1.22,  0.     ,  1.22]])
+    """
+    X = check_array(
+        X,
+        accept_sparse="csc",
+        copy=copy,
+        ensure_2d=False,
+        estimator="the scale function",
+        dtype=FLOAT_DTYPES,
+        ensure_all_finite="allow-nan",
+    )
     if sparse.issparse(X):
         if with_mean:
             raise ValueError(
                 "Cannot center sparse matrices: pass `with_mean=False` instead"
-                " See docstring for motivation and alternatives.")
+                " See docstring for motivation and alternatives."
+            )
         if axis != 0:
-            raise ValueError("Can only scale sparse matrix on axis=0, "
-                             " got axis=%d" % axis)
+            raise ValueError(
+                "Can only scale sparse matrix on axis=0,  got axis=%d" % axis
+            )
         if with_std:
             _, var = mean_variance_axis(X, axis=0)
             var = _handle_zeros_in_scale(var, copy=False)
@@ -204,11 +262,13 @@ def scale(X, *, axis=0, with_mean=True, with_std=True, copy=True):
             # concerned feature is efficient, for instance by its mean or
             # maximum.
             if not np.allclose(mean_1, 0):
-                warnings.warn("Numerical issues were encountered "
-                              "when centering the data "
-                              "and might not be solved. Dataset may "
-                              "contain too large values. You may need "
-                              "to prescale your features.")
+                warnings.warn(
+                    "Numerical issues were encountered "
+                    "when centering the data "
+                    "and might not be solved. Dataset may "
+                    "contain too large values. You may need "
+                    "to prescale your features."
+                )
                 Xr -= mean_1
         if with_std:
             scale_ = _handle_zeros_in_scale(scale_, copy=False)
@@ -221,16 +281,18 @@ def scale(X, *, axis=0, with_mean=True, with_std=True, copy=True):
                 # due to the lack of precision of mean_. A solution is then to
                 # subtract the mean again:
                 if not np.allclose(mean_2, 0):
-                    warnings.warn("Numerical issues were encountered "
-                                  "when scaling the data "
-                                  "and might not be solved. The standard "
-                                  "deviation of the data is probably "
-                                  "very close to 0. ")
+                    warnings.warn(
+                        "Numerical issues were encountered "
+                        "when scaling the data "
+                        "and might not be solved. The standard "
+                        "deviation of the data is probably "
+                        "very close to 0. "
+                    )
                     Xr -= mean_2
     return X
 
 
-class MinMaxScaler(TransformerMixin, BaseEstimator):
+class MinMaxScaler(OneToOneFeatureMixin, TransformerMixin, BaseEstimator):
     """Transform features by scaling each feature to a given range.
 
     This estimator scales and translates each feature individually such
@@ -247,6 +309,12 @@ class MinMaxScaler(TransformerMixin, BaseEstimator):
     This transformation is often used as an alternative to zero mean,
     unit variance scaling.
 
+    `MinMaxScaler` doesn't reduce the effect of outliers, but it linearly
+    scales them down into a fixed range, where the largest occurring data point
+    corresponds to the maximum value and the smallest one corresponds to the
+    minimum value. For an example visualization, refer to :ref:`Compare
+    MinMaxScaler with other scalers `.
+
     Read more in the :ref:`User Guide `.
 
     Parameters
@@ -295,11 +363,31 @@ class MinMaxScaler(TransformerMixin, BaseEstimator):
         .. versionadded:: 0.17
            *data_range_*
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
     n_samples_seen_ : int
         The number of samples processed by the estimator.
         It will be reset on new calls to fit, but increments across
         ``partial_fit`` calls.
 
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    minmax_scale : Equivalent function without the estimator API.
+
+    Notes
+    -----
+    NaNs are treated as missing values: disregarded in fit, and maintained in
+    transform.
+
     Examples
     --------
     >>> from sklearn.preprocessing import MinMaxScaler
@@ -316,22 +404,14 @@ class MinMaxScaler(TransformerMixin, BaseEstimator):
      [1.   1.  ]]
     >>> print(scaler.transform([[2, 2]]))
     [[1.5 0. ]]
-
-    See Also
-    --------
-    minmax_scale : Equivalent function without the estimator API.
-
-    Notes
-    -----
-    NaNs are treated as missing values: disregarded in fit, and maintained in
-    transform.
-
-    For a comparison of the different scalers, transformers, and normalizers,
-    see :ref:`examples/preprocessing/plot_all_scaling.py
-    `.
     """
 
-    @_deprecate_positional_args
+    _parameter_constraints: dict = {
+        "feature_range": [tuple],
+        "copy": ["boolean"],
+        "clip": ["boolean"],
+    }
+
     def __init__(self, feature_range=(0, 1), *, copy=True, clip=False):
         self.feature_range = feature_range
         self.copy = copy
@@ -342,10 +422,9 @@ def _reset(self):
 
         __init__ parameters are not touched.
         """
-
-        # Checking one attribute is enough, becase they are all set together
+        # Checking one attribute is enough, because they are all set together
         # in partial_fit
-        if hasattr(self, 'scale_'):
+        if hasattr(self, "scale_"):
             del self.scale_
             del self.min_
             del self.n_samples_seen_
@@ -370,11 +449,11 @@ def fit(self, X, y=None):
         self : object
             Fitted scaler.
         """
-
         # Reset internal state before fitting
         self._reset()
         return self.partial_fit(X, y)
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def partial_fit(self, X, y=None):
         """Online computation of min and max on X for later scaling.
 
@@ -398,31 +477,48 @@ def partial_fit(self, X, y=None):
         """
         feature_range = self.feature_range
         if feature_range[0] >= feature_range[1]:
-            raise ValueError("Minimum of desired feature range must be smaller"
-                             " than maximum. Got %s." % str(feature_range))
+            raise ValueError(
+                "Minimum of desired feature range must be smaller than maximum. Got %s."
+                % str(feature_range)
+            )
 
         if sparse.issparse(X):
-            raise TypeError("MinMaxScaler does not support sparse input. "
-                            "Consider using MaxAbsScaler instead.")
-
-        first_pass = not hasattr(self, 'n_samples_seen_')
-        X = self._validate_data(X, reset=first_pass,
-                                estimator=self, dtype=FLOAT_DTYPES,
-                                force_all_finite="allow-nan")
-
-        data_min = np.nanmin(X, axis=0)
-        data_max = np.nanmax(X, axis=0)
+            raise TypeError(
+                "MinMaxScaler does not support sparse input. "
+                "Consider using MaxAbsScaler instead."
+            )
+
+        xp, _ = get_namespace(X)
+
+        first_pass = not hasattr(self, "n_samples_seen_")
+        X = validate_data(
+            self,
+            X,
+            reset=first_pass,
+            dtype=_array_api.supported_float_dtypes(xp),
+            ensure_all_finite="allow-nan",
+        )
+
+        device_ = device(X)
+        feature_range = (
+            xp.asarray(feature_range[0], dtype=X.dtype, device=device_),
+            xp.asarray(feature_range[1], dtype=X.dtype, device=device_),
+        )
+
+        data_min = _array_api._nanmin(X, axis=0, xp=xp)
+        data_max = _array_api._nanmax(X, axis=0, xp=xp)
 
         if first_pass:
             self.n_samples_seen_ = X.shape[0]
         else:
-            data_min = np.minimum(self.data_min_, data_min)
-            data_max = np.maximum(self.data_max_, data_max)
+            data_min = xp.minimum(self.data_min_, data_min)
+            data_max = xp.maximum(self.data_max_, data_max)
             self.n_samples_seen_ += X.shape[0]
 
         data_range = data_max - data_min
-        self.scale_ = ((feature_range[1] - feature_range[0]) /
-                       _handle_zeros_in_scale(data_range, copy=True))
+        self.scale_ = (feature_range[1] - feature_range[0]) / _handle_zeros_in_scale(
+            data_range, copy=True
+        )
         self.min_ = feature_range[0] - data_min * self.scale_
         self.data_min_ = data_min
         self.data_max_ = data_max
@@ -444,13 +540,30 @@ def transform(self, X):
         """
         check_is_fitted(self)
 
-        X = self._validate_data(X, copy=self.copy, dtype=FLOAT_DTYPES,
-                                force_all_finite="allow-nan", reset=False)
+        xp, _ = get_namespace(X)
+
+        X = validate_data(
+            self,
+            X,
+            copy=self.copy,
+            dtype=_array_api.supported_float_dtypes(xp),
+            force_writeable=True,
+            ensure_all_finite="allow-nan",
+            reset=False,
+        )
 
         X *= self.scale_
         X += self.min_
         if self.clip:
-            np.clip(X, self.feature_range[0], self.feature_range[1], out=X)
+            device_ = device(X)
+            X = _modify_in_place_if_numpy(
+                xp,
+                xp.clip,
+                X,
+                xp.asarray(self.feature_range[0], dtype=X.dtype, device=device_),
+                xp.asarray(self.feature_range[1], dtype=X.dtype, device=device_),
+                out=X,
+            )
         return X
 
     def inverse_transform(self, X):
@@ -463,23 +576,39 @@ def inverse_transform(self, X):
 
         Returns
         -------
-        Xt : ndarray of shape (n_samples, n_features)
+        X_original : ndarray of shape (n_samples, n_features)
             Transformed data.
         """
         check_is_fitted(self)
 
-        X = check_array(X, copy=self.copy, dtype=FLOAT_DTYPES,
-                        force_all_finite="allow-nan")
+        xp, _ = get_namespace(X)
+
+        X = check_array(
+            X,
+            copy=self.copy,
+            dtype=_array_api.supported_float_dtypes(xp),
+            force_writeable=True,
+            ensure_all_finite="allow-nan",
+        )
 
         X -= self.min_
         X /= self.scale_
         return X
 
-    def _more_tags(self):
-        return {'allow_nan': True}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.allow_nan = True
+        tags.array_api_support = True
+        return tags
 
 
-@_deprecate_positional_args
+@validate_params(
+    {
+        "X": ["array-like"],
+        "axis": [Options(Integral, {0, 1})],
+    },
+    prefer_skip_nested_validation=False,
+)
 def minmax_scale(X, feature_range=(0, 1), *, axis=0, copy=True):
     """Transform features by scaling each feature to a given range.
 
@@ -516,13 +645,15 @@ def minmax_scale(X, feature_range=(0, 1), *, axis=0, copy=True):
     feature_range : tuple (min, max), default=(0, 1)
         Desired range of transformed data.
 
-    axis : int, default=0
+    axis : {0, 1}, default=0
         Axis used to scale along. If 0, independently scale each feature,
         otherwise (if 1) scale each sample.
 
     copy : bool, default=True
-        Set to False to perform inplace scaling and avoid a copy (if the input
-        is already a numpy array).
+        If False, try to avoid a copy and scale in place.
+        This is not guaranteed to always work in place; e.g. if the data is
+        a numpy array with an int dtype, a copy will be returned even with
+        copy=False.
 
     Returns
     -------
@@ -550,13 +681,28 @@ def minmax_scale(X, feature_range=(0, 1), *, axis=0, copy=True):
     Notes
     -----
     For a comparison of the different scalers, transformers, and normalizers,
-    see :ref:`examples/preprocessing/plot_all_scaling.py
-    `.
-    """  # noqa
+    see: :ref:`sphx_glr_auto_examples_preprocessing_plot_all_scaling.py`.
+
+    Examples
+    --------
+    >>> from sklearn.preprocessing import minmax_scale
+    >>> X = [[-2, 1, 2], [-1, 0, 1]]
+    >>> minmax_scale(X, axis=0)  # scale each column independently
+    array([[0., 1., 1.],
+           [1., 0., 0.]])
+    >>> minmax_scale(X, axis=1)  # scale each row independently
+    array([[0.  , 0.75, 1.  ],
+           [0.  , 0.5 , 1.  ]])
+    """
     # Unlike the scaler object, this function allows 1d input.
     # If copy is required, it will be done inside the scaler object.
-    X = check_array(X, copy=False, ensure_2d=False,
-                    dtype=FLOAT_DTYPES, force_all_finite='allow-nan')
+    X = check_array(
+        X,
+        copy=False,
+        ensure_2d=False,
+        dtype=FLOAT_DTYPES,
+        ensure_all_finite="allow-nan",
+    )
     original_ndim = X.ndim
 
     if original_ndim == 1:
@@ -574,11 +720,13 @@ def minmax_scale(X, feature_range=(0, 1), *, axis=0, copy=True):
     return X
 
 
-class StandardScaler(TransformerMixin, BaseEstimator):
-    """Standardize features by removing the mean and scaling to unit variance
+class StandardScaler(OneToOneFeatureMixin, TransformerMixin, BaseEstimator):
+    """Standardize features by removing the mean and scaling to unit variance.
 
     The standard score of a sample `x` is calculated as:
 
+    .. code-block:: text
+
         z = (x - u) / s
 
     where `u` is the mean of the training samples or zero if `with_mean=False`,
@@ -600,9 +748,14 @@ class StandardScaler(TransformerMixin, BaseEstimator):
     Machines or the L1 and L2 regularizers of linear models) assume that
     all features are centered around 0 and have variance in the same
     order. If a feature has a variance that is orders of magnitude larger
-    that others, it might dominate the objective function and make the
+    than others, it might dominate the objective function and make the
     estimator unable to learn from other features correctly as expected.
 
+    `StandardScaler` is sensitive to outliers, and the features may scale
+    differently from each other in the presence of outliers. For an example
+    visualization, refer to :ref:`Compare StandardScaler with other scalers
+    `.
+
     This scaler can also be applied to sparse CSR or CSC matrices by passing
     `with_mean=False` to avoid breaking the sparsity structure of the data.
 
@@ -641,11 +794,23 @@ class StandardScaler(TransformerMixin, BaseEstimator):
 
     mean_ : ndarray of shape (n_features,) or None
         The mean value for each feature in the training set.
-        Equal to ``None`` when ``with_mean=False``.
+        Equal to ``None`` when ``with_mean=False`` and ``with_std=False``.
 
     var_ : ndarray of shape (n_features,) or None
         The variance for each feature in the training set. Used to compute
-        `scale_`. Equal to ``None`` when ``with_std=False``.
+        `scale_`. Equal to ``None`` when ``with_mean=False`` and
+        ``with_std=False``.
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
 
     n_samples_seen_ : int or ndarray of shape (n_features,)
         The number of samples processed by the estimator for each feature.
@@ -656,6 +821,22 @@ class StandardScaler(TransformerMixin, BaseEstimator):
         Will be reset on new calls to fit, but increments across
         ``partial_fit`` calls.
 
+    See Also
+    --------
+    scale : Equivalent function without the estimator API.
+
+    :class:`~sklearn.decomposition.PCA` : Further removes the linear
+        correlation across features with 'whiten=True'.
+
+    Notes
+    -----
+    NaNs are treated as missing values: disregarded in fit, and maintained in
+    transform.
+
+    We use a biased estimator for the standard deviation, equivalent to
+    `numpy.std(x, ddof=0)`. Note that the choice of `ddof` is unlikely to
+    affect model performance.
+
     Examples
     --------
     >>> from sklearn.preprocessing import StandardScaler
@@ -672,29 +853,14 @@ class StandardScaler(TransformerMixin, BaseEstimator):
      [ 1.  1.]]
     >>> print(scaler.transform([[2, 2]]))
     [[3. 3.]]
+    """
 
-    See Also
-    --------
-    scale : Equivalent function without the estimator API.
-
-    :class:`~sklearn.decomposition.PCA` : Further removes the linear
-        correlation across features with 'whiten=True'.
-
-    Notes
-    -----
-    NaNs are treated as missing values: disregarded in fit, and maintained in
-    transform.
-
-    We use a biased estimator for the standard deviation, equivalent to
-    `numpy.std(x, ddof=0)`. Note that the choice of `ddof` is unlikely to
-    affect model performance.
-
-    For a comparison of the different scalers, transformers, and normalizers,
-    see :ref:`examples/preprocessing/plot_all_scaling.py
-    `.
-    """  # noqa
+    _parameter_constraints: dict = {
+        "copy": ["boolean"],
+        "with_mean": ["boolean"],
+        "with_std": ["boolean"],
+    }
 
-    @_deprecate_positional_args
     def __init__(self, *, copy=True, with_mean=True, with_std=True):
         self.with_mean = with_mean
         self.with_std = with_std
@@ -705,10 +871,9 @@ def _reset(self):
 
         __init__ parameters are not touched.
         """
-
-        # Checking one attribute is enough, becase they are all set together
+        # Checking one attribute is enough, because they are all set together
         # in partial_fit
-        if hasattr(self, 'scale_'):
+        if hasattr(self, "scale_"):
             del self.scale_
             del self.n_samples_seen_
             del self.mean_
@@ -737,14 +902,13 @@ def fit(self, X, y=None, sample_weight=None):
         self : object
             Fitted scaler.
         """
-
         # Reset internal state before fitting
         self._reset()
         return self.partial_fit(X, y, sample_weight)
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def partial_fit(self, X, y=None, sample_weight=None):
-        """
-        Online computation of mean and std on X for later scaling.
+        """Online computation of mean and std on X for later scaling.
 
         All of X is processed as a single batch. This is intended for cases
         when :meth:`fit` is not feasible due to very large number of
@@ -776,14 +940,18 @@ def partial_fit(self, X, y=None, sample_weight=None):
             Fitted scaler.
         """
         first_call = not hasattr(self, "n_samples_seen_")
-        X = self._validate_data(X, accept_sparse=('csr', 'csc'),
-                                estimator=self, dtype=FLOAT_DTYPES,
-                                force_all_finite='allow-nan', reset=first_call)
+        X = validate_data(
+            self,
+            X,
+            accept_sparse=("csr", "csc"),
+            dtype=FLOAT_DTYPES,
+            ensure_all_finite="allow-nan",
+            reset=first_call,
+        )
         n_features = X.shape[1]
 
         if sample_weight is not None:
-            sample_weight = _check_sample_weight(sample_weight, X,
-                                                 dtype=X.dtype)
+            sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype)
 
         # Even in the case of `with_mean=False`, we update the mean anyway
         # This is needed for the incremental computation of the var
@@ -793,36 +961,42 @@ def partial_fit(self, X, y=None, sample_weight=None):
         # transform it to a NumPy array of shape (n_features,) required by
         # incr_mean_variance_axis and _incremental_variance_axis
         dtype = np.int64 if sample_weight is None else X.dtype
-        if not hasattr(self, 'n_samples_seen_'):
+        if not hasattr(self, "n_samples_seen_"):
             self.n_samples_seen_ = np.zeros(n_features, dtype=dtype)
         elif np.size(self.n_samples_seen_) == 1:
-            self.n_samples_seen_ = np.repeat(
-                self.n_samples_seen_, X.shape[1])
-            self.n_samples_seen_ = \
-                self.n_samples_seen_.astype(dtype, copy=False)
+            self.n_samples_seen_ = np.repeat(self.n_samples_seen_, X.shape[1])
+            self.n_samples_seen_ = self.n_samples_seen_.astype(dtype, copy=False)
 
         if sparse.issparse(X):
             if self.with_mean:
                 raise ValueError(
                     "Cannot center sparse matrices: pass `with_mean=False` "
-                    "instead. See docstring for motivation and alternatives.")
-            sparse_constructor = (sparse.csr_matrix
-                                  if X.format == 'csr' else sparse.csc_matrix)
+                    "instead. See docstring for motivation and alternatives."
+                )
+            sparse_constructor = (
+                sparse.csr_matrix if X.format == "csr" else sparse.csc_matrix
+            )
 
             if self.with_std:
                 # First pass
-                if not hasattr(self, 'scale_'):
-                    self.mean_, self.var_, self.n_samples_seen_ = \
-                        mean_variance_axis(X, axis=0, weights=sample_weight,
-                                           return_sum_weights=True)
+                if not hasattr(self, "scale_"):
+                    self.mean_, self.var_, self.n_samples_seen_ = mean_variance_axis(
+                        X, axis=0, weights=sample_weight, return_sum_weights=True
+                    )
                 # Next passes
                 else:
-                    self.mean_, self.var_, self.n_samples_seen_ = \
-                        incr_mean_variance_axis(X, axis=0,
-                                                last_mean=self.mean_,
-                                                last_var=self.var_,
-                                                last_n=self.n_samples_seen_,
-                                                weights=sample_weight)
+                    (
+                        self.mean_,
+                        self.var_,
+                        self.n_samples_seen_,
+                    ) = incr_mean_variance_axis(
+                        X,
+                        axis=0,
+                        last_mean=self.mean_,
+                        last_var=self.var_,
+                        last_n=self.n_samples_seen_,
+                        weights=sample_weight,
+                    )
                 # We force the mean and variance to float64 for large arrays
                 # See https://github.com/scikit-learn/scikit-learn/pull/12338
                 self.mean_ = self.mean_.astype(np.float64, copy=False)
@@ -832,17 +1006,17 @@ def partial_fit(self, X, y=None, sample_weight=None):
                 self.var_ = None
                 weights = _check_sample_weight(sample_weight, X)
                 sum_weights_nan = weights @ sparse_constructor(
-                    (np.isnan(X.data), X.indices, X.indptr),
-                    shape=X.shape)
-                self.n_samples_seen_ += (
-                    (np.sum(weights) - sum_weights_nan).astype(dtype)
+                    (np.isnan(X.data), X.indices, X.indptr), shape=X.shape
+                )
+                self.n_samples_seen_ += (np.sum(weights) - sum_weights_nan).astype(
+                    dtype
                 )
         else:
             # First pass
-            if not hasattr(self, 'scale_'):
-                self.mean_ = .0
+            if not hasattr(self, "scale_"):
+                self.mean_ = 0.0
                 if self.with_std:
-                    self.var_ = .0
+                    self.var_ = 0.0
                 else:
                     self.var_ = None
 
@@ -852,10 +1026,13 @@ def partial_fit(self, X, y=None, sample_weight=None):
                 self.n_samples_seen_ += X.shape[0] - np.isnan(X).sum(axis=0)
 
             else:
-                self.mean_, self.var_, self.n_samples_seen_ = \
-                    _incremental_mean_and_var(X, self.mean_, self.var_,
-                                              self.n_samples_seen_,
-                                              sample_weight=sample_weight)
+                self.mean_, self.var_, self.n_samples_seen_ = _incremental_mean_and_var(
+                    X,
+                    self.mean_,
+                    self.var_,
+                    self.n_samples_seen_,
+                    sample_weight=sample_weight,
+                )
 
         # for backward-compatibility, reduce n_samples_seen_ to an integer
         # if the number of samples is the same for each feature (i.e. no
@@ -866,16 +1043,19 @@ def partial_fit(self, X, y=None, sample_weight=None):
         if self.with_std:
             # Extract the list of near constant features on the raw variances,
             # before taking the square root.
-            constant_mask = self.var_ < 10 * np.finfo(X.dtype).eps
+            constant_mask = _is_constant_feature(
+                self.var_, self.mean_, self.n_samples_seen_
+            )
             self.scale_ = _handle_zeros_in_scale(
-                np.sqrt(self.var_), copy=False, constant_mask=constant_mask)
+                np.sqrt(self.var_), copy=False, constant_mask=constant_mask
+            )
         else:
             self.scale_ = None
 
         return self
 
     def transform(self, X, copy=None):
-        """Perform standardization by centering and scaling
+        """Perform standardization by centering and scaling.
 
         Parameters
         ----------
@@ -892,16 +1072,23 @@ def transform(self, X, copy=None):
         check_is_fitted(self)
 
         copy = copy if copy is not None else self.copy
-        X = self._validate_data(X, reset=False,
-                                accept_sparse='csr', copy=copy,
-                                estimator=self, dtype=FLOAT_DTYPES,
-                                force_all_finite='allow-nan')
+        X = validate_data(
+            self,
+            X,
+            reset=False,
+            accept_sparse="csr",
+            copy=copy,
+            dtype=FLOAT_DTYPES,
+            force_writeable=True,
+            ensure_all_finite="allow-nan",
+        )
 
         if sparse.issparse(X):
             if self.with_mean:
                 raise ValueError(
                     "Cannot center sparse matrices: pass `with_mean=False` "
-                    "instead. See docstring for motivation and alternatives.")
+                    "instead. See docstring for motivation and alternatives."
+                )
             if self.scale_ is not None:
                 inplace_column_scale(X, 1 / self.scale_)
         else:
@@ -912,51 +1099,57 @@ def transform(self, X, copy=None):
         return X
 
     def inverse_transform(self, X, copy=None):
-        """Scale back the data to the original representation
+        """Scale back the data to the original representation.
 
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
             The data used to scale along the features axis.
+
         copy : bool, default=None
-            Copy the input X or not.
+            Copy the input `X` or not.
 
         Returns
         -------
-        X_tr : {ndarray, sparse matrix} of shape (n_samples, n_features)
+        X_original : {ndarray, sparse matrix} of shape (n_samples, n_features)
             Transformed array.
         """
         check_is_fitted(self)
 
         copy = copy if copy is not None else self.copy
+        X = check_array(
+            X,
+            accept_sparse="csr",
+            copy=copy,
+            dtype=FLOAT_DTYPES,
+            force_writeable=True,
+            ensure_all_finite="allow-nan",
+        )
+
         if sparse.issparse(X):
             if self.with_mean:
                 raise ValueError(
                     "Cannot uncenter sparse matrices: pass `with_mean=False` "
-                    "instead See docstring for motivation and alternatives.")
-            if not sparse.isspmatrix_csr(X):
-                X = X.tocsr()
-                copy = False
-            if copy:
-                X = X.copy()
+                    "instead See docstring for motivation and alternatives."
+                )
             if self.scale_ is not None:
                 inplace_column_scale(X, self.scale_)
         else:
-            X = np.asarray(X)
-            if copy:
-                X = X.copy()
             if self.with_std:
                 X *= self.scale_
             if self.with_mean:
                 X += self.mean_
         return X
 
-    def _more_tags(self):
-        return {'allow_nan': True,
-                'preserves_dtype': [np.float64, np.float32]}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.allow_nan = True
+        tags.input_tags.sparse = not self.with_mean
+        tags.transformer_tags.preserves_dtype = ["float64", "float32"]
+        return tags
 
 
-class MaxAbsScaler(TransformerMixin, BaseEstimator):
+class MaxAbsScaler(OneToOneFeatureMixin, TransformerMixin, BaseEstimator):
     """Scale each feature by its maximum absolute value.
 
     This estimator scales and translates each feature individually such
@@ -966,6 +1159,10 @@ class MaxAbsScaler(TransformerMixin, BaseEstimator):
 
     This scaler can also be applied to sparse CSR or CSC matrices.
 
+    `MaxAbsScaler` doesn't reduce the effect of outliers; it only linearly
+    scales them down. For an example visualization, refer to :ref:`Compare
+    MaxAbsScaler with other scalers `.
+
     .. versionadded:: 0.17
 
     Parameters
@@ -985,10 +1182,30 @@ class MaxAbsScaler(TransformerMixin, BaseEstimator):
     max_abs_ : ndarray of shape (n_features,)
         Per feature maximum absolute value.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     n_samples_seen_ : int
         The number of samples processed by the estimator. Will be reset on
         new calls to fit, but increments across ``partial_fit`` calls.
 
+    See Also
+    --------
+    maxabs_scale : Equivalent function without the estimator API.
+
+    Notes
+    -----
+    NaNs are treated as missing values: disregarded in fit, and maintained in
+    transform.
+
     Examples
     --------
     >>> from sklearn.preprocessing import MaxAbsScaler
@@ -1002,22 +1219,10 @@ class MaxAbsScaler(TransformerMixin, BaseEstimator):
     array([[ 0.5, -1. ,  1. ],
            [ 1. ,  0. ,  0. ],
            [ 0. ,  1. , -0.5]])
-
-    See Also
-    --------
-    maxabs_scale : Equivalent function without the estimator API.
-
-    Notes
-    -----
-    NaNs are treated as missing values: disregarded in fit, and maintained in
-    transform.
-
-    For a comparison of the different scalers, transformers, and normalizers,
-    see :ref:`examples/preprocessing/plot_all_scaling.py
-    `.
     """
 
-    @_deprecate_positional_args
+    _parameter_constraints: dict = {"copy": ["boolean"]}
+
     def __init__(self, *, copy=True):
         self.copy = copy
 
@@ -1026,10 +1231,9 @@ def _reset(self):
 
         __init__ parameters are not touched.
         """
-
-        # Checking one attribute is enough, becase they are all set together
+        # Checking one attribute is enough, because they are all set together
         # in partial_fit
-        if hasattr(self, 'scale_'):
+        if hasattr(self, "scale_"):
             del self.scale_
             del self.n_samples_seen_
             del self.max_abs_
@@ -1055,9 +1259,9 @@ def fit(self, X, y=None):
         self._reset()
         return self.partial_fit(X, y)
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def partial_fit(self, X, y=None):
-        """
-        Online computation of max absolute value of X for later scaling.
+        """Online computation of max absolute value of X for later scaling.
 
         All of X is processed as a single batch. This is intended for cases
         when :meth:`fit` is not feasible due to very large number of
@@ -1077,22 +1281,28 @@ def partial_fit(self, X, y=None):
         self : object
             Fitted scaler.
         """
-        first_pass = not hasattr(self, 'n_samples_seen_')
-        X = self._validate_data(X, reset=first_pass,
-                                accept_sparse=('csr', 'csc'), estimator=self,
-                                dtype=FLOAT_DTYPES,
-                                force_all_finite='allow-nan')
+        xp, _ = get_namespace(X)
+
+        first_pass = not hasattr(self, "n_samples_seen_")
+        X = validate_data(
+            self,
+            X,
+            reset=first_pass,
+            accept_sparse=("csr", "csc"),
+            dtype=_array_api.supported_float_dtypes(xp),
+            ensure_all_finite="allow-nan",
+        )
 
         if sparse.issparse(X):
             mins, maxs = min_max_axis(X, axis=0, ignore_nan=True)
             max_abs = np.maximum(np.abs(mins), np.abs(maxs))
         else:
-            max_abs = np.nanmax(np.abs(X), axis=0)
+            max_abs = _array_api._nanmax(xp.abs(X), axis=0, xp=xp)
 
         if first_pass:
             self.n_samples_seen_ = X.shape[0]
         else:
-            max_abs = np.maximum(self.max_abs_, max_abs)
+            max_abs = xp.maximum(self.max_abs_, max_abs)
             self.n_samples_seen_ += X.shape[0]
 
         self.max_abs_ = max_abs
@@ -1100,7 +1310,7 @@ def partial_fit(self, X, y=None):
         return self
 
     def transform(self, X):
-        """Scale the data
+        """Scale the data.
 
         Parameters
         ----------
@@ -1113,10 +1323,19 @@ def transform(self, X):
             Transformed array.
         """
         check_is_fitted(self)
-        X = self._validate_data(X, accept_sparse=('csr', 'csc'),
-                                copy=self.copy, reset=False,
-                                estimator=self, dtype=FLOAT_DTYPES,
-                                force_all_finite='allow-nan')
+
+        xp, _ = get_namespace(X)
+
+        X = validate_data(
+            self,
+            X,
+            accept_sparse=("csr", "csc"),
+            copy=self.copy,
+            reset=False,
+            dtype=_array_api.supported_float_dtypes(xp),
+            force_writeable=True,
+            ensure_all_finite="allow-nan",
+        )
 
         if sparse.issparse(X):
             inplace_column_scale(X, 1.0 / self.scale_)
@@ -1125,7 +1344,7 @@ def transform(self, X):
         return X
 
     def inverse_transform(self, X):
-        """Scale back the data to the original representation
+        """Scale back the data to the original representation.
 
         Parameters
         ----------
@@ -1134,13 +1353,21 @@ def inverse_transform(self, X):
 
         Returns
         -------
-        X_tr : {ndarray, sparse matrix} of shape (n_samples, n_features)
+        X_original : {ndarray, sparse matrix} of shape (n_samples, n_features)
             Transformed array.
         """
         check_is_fitted(self)
-        X = check_array(X, accept_sparse=('csr', 'csc'), copy=self.copy,
-                        estimator=self, dtype=FLOAT_DTYPES,
-                        force_all_finite='allow-nan')
+
+        xp, _ = get_namespace(X)
+
+        X = check_array(
+            X,
+            accept_sparse=("csr", "csc"),
+            copy=self.copy,
+            dtype=_array_api.supported_float_dtypes(xp),
+            force_writeable=True,
+            ensure_all_finite="allow-nan",
+        )
 
         if sparse.issparse(X):
             inplace_column_scale(X, self.scale_)
@@ -1148,11 +1375,20 @@ def inverse_transform(self, X):
             X *= self.scale_
         return X
 
-    def _more_tags(self):
-        return {'allow_nan': True}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.allow_nan = True
+        tags.input_tags.sparse = True
+        return tags
 
 
-@_deprecate_positional_args
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "axis": [Options(Integral, {0, 1})],
+    },
+    prefer_skip_nested_validation=False,
+)
 def maxabs_scale(X, *, axis=0, copy=True):
     """Scale each feature to the [-1, 1] range without breaking the sparsity.
 
@@ -1167,13 +1403,15 @@ def maxabs_scale(X, *, axis=0, copy=True):
     X : {array-like, sparse matrix} of shape (n_samples, n_features)
         The data.
 
-    axis : int, default=0
-        axis used to scale along. If 0, independently scale each feature,
+    axis : {0, 1}, default=0
+        Axis used to scale along. If 0, independently scale each feature,
         otherwise (if 1) scale each sample.
 
     copy : bool, default=True
-        Set to False to perform inplace scaling and avoid a copy (if the input
-        is already a numpy array).
+        If False, try to avoid a copy and scale in place.
+        This is not guaranteed to always work in place; e.g. if the data is
+        a numpy array with an int dtype, a copy will be returned even with
+        copy=False.
 
     Returns
     -------
@@ -1182,8 +1420,8 @@ def maxabs_scale(X, *, axis=0, copy=True):
 
     .. warning:: Risk of data leak
 
-        Do not use :func:`~sklearn.preprocessing.maxabs_scale` unless you know what
-        you are doing. A common mistake is to apply it to the entire data
+        Do not use :func:`~sklearn.preprocessing.maxabs_scale` unless you know
+        what you are doing. A common mistake is to apply it to the entire data
         *before* splitting into training and test sets. This will bias the
         model evaluation because information would have leaked from the test
         set to the training set.
@@ -1204,15 +1442,30 @@ def maxabs_scale(X, *, axis=0, copy=True):
     and maintained during the data transformation.
 
     For a comparison of the different scalers, transformers, and normalizers,
-    see :ref:`examples/preprocessing/plot_all_scaling.py
-    `.
-    """  # noqa
+    see: :ref:`sphx_glr_auto_examples_preprocessing_plot_all_scaling.py`.
+
+    Examples
+    --------
+    >>> from sklearn.preprocessing import maxabs_scale
+    >>> X = [[-2, 1, 2], [-1, 0, 1]]
+    >>> maxabs_scale(X, axis=0)  # scale each column independently
+    array([[-1. ,  1. ,  1. ],
+           [-0.5,  0. ,  0.5]])
+    >>> maxabs_scale(X, axis=1)  # scale each row independently
+    array([[-1. ,  0.5,  1. ],
+           [-1. ,  0. ,  1. ]])
+    """
     # Unlike the scaler object, this function allows 1d input.
 
     # If copy is required, it will be done inside the scaler object.
-    X = check_array(X, accept_sparse=('csr', 'csc'), copy=False,
-                    ensure_2d=False, dtype=FLOAT_DTYPES,
-                    force_all_finite='allow-nan')
+    X = check_array(
+        X,
+        accept_sparse=("csr", "csc"),
+        copy=False,
+        ensure_2d=False,
+        dtype=FLOAT_DTYPES,
+        ensure_all_finite="allow-nan",
+    )
     original_ndim = X.ndim
 
     if original_ndim == 1:
@@ -1230,7 +1483,7 @@ def maxabs_scale(X, *, axis=0, copy=True):
     return X
 
 
-class RobustScaler(TransformerMixin, BaseEstimator):
+class RobustScaler(OneToOneFeatureMixin, TransformerMixin, BaseEstimator):
     """Scale features using statistics that are robust to outliers.
 
     This Scaler removes the median and scales the data according to
@@ -1241,13 +1494,15 @@ class RobustScaler(TransformerMixin, BaseEstimator):
     Centering and scaling happen independently on each feature by
     computing the relevant statistics on the samples in the training
     set. Median and interquartile range are then stored to be used on
-    later data using the ``transform`` method.
+    later data using the :meth:`transform` method.
 
-    Standardization of a dataset is a common requirement for many
-    machine learning estimators. Typically this is done by removing the mean
-    and scaling to unit variance. However, outliers can often influence the
-    sample mean / variance in a negative way. In such cases, the median and
-    the interquartile range often give better results.
+    Standardization of a dataset is a common preprocessing for many machine
+    learning estimators. Typically this is done by removing the mean and
+    scaling to unit variance. However, outliers can often influence the sample
+    mean / variance in a negative way. In such cases, using the median and the
+    interquartile range often give better results. For an example visualization
+    and comparison to other scalers, refer to :ref:`Compare RobustScaler with
+    other scalers `.
 
     .. versionadded:: 0.17
 
@@ -1256,31 +1511,33 @@ class RobustScaler(TransformerMixin, BaseEstimator):
     Parameters
     ----------
     with_centering : bool, default=True
-        If True, center the data before scaling.
-        This will cause ``transform`` to raise an exception when attempted on
-        sparse matrices, because centering them entails building a dense
+        If `True`, center the data before scaling.
+        This will cause :meth:`transform` to raise an exception when attempted
+        on sparse matrices, because centering them entails building a dense
         matrix which in common use cases is likely to be too large to fit in
         memory.
 
     with_scaling : bool, default=True
-        If True, scale the data to interquartile range.
+        If `True`, scale the data to interquartile range.
 
     quantile_range : tuple (q_min, q_max), 0.0 < q_min < q_max < 100.0, \
-        default=(25.0, 75.0), == (1st quantile, 3rd quantile), == IQR
-        Quantile range used to calculate ``scale_``.
+        default=(25.0, 75.0)
+        Quantile range used to calculate `scale_`. By default this is equal to
+        the IQR, i.e., `q_min` is the first quantile and `q_max` is the third
+        quantile.
 
         .. versionadded:: 0.18
 
     copy : bool, default=True
-        If False, try to avoid a copy and do inplace scaling instead.
+        If `False`, try to avoid a copy and do inplace scaling instead.
         This is not guaranteed to always work inplace; e.g. if the data is
         not a NumPy array or scipy.sparse CSR matrix, a copy may still be
         returned.
 
     unit_variance : bool, default=False
-        If True, scale data so that normally distributed features have a
+        If `True`, scale data so that normally distributed features have a
         variance of 1. In general, if the difference between the x-values of
-        ``q_max`` and ``q_min`` for a standard normal distribution is greater
+        `q_max` and `q_min` for a standard normal distribution is greater
         than 1, the dataset will be scaled down. If less than 1, the dataset
         will be scaled up.
 
@@ -1297,6 +1554,29 @@ class RobustScaler(TransformerMixin, BaseEstimator):
         .. versionadded:: 0.17
            *scale_* attribute.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    robust_scale : Equivalent function without the estimator API.
+    sklearn.decomposition.PCA : Further removes the linear correlation across
+        features with 'whiten=True'.
+
+    Notes
+    -----
+
+    https://en.wikipedia.org/wiki/Median
+    https://en.wikipedia.org/wiki/Interquartile_range
+
     Examples
     --------
     >>> from sklearn.preprocessing import RobustScaler
@@ -1310,33 +1590,32 @@ class RobustScaler(TransformerMixin, BaseEstimator):
     array([[ 0. , -2. ,  0. ],
            [-1. ,  0. ,  0.4],
            [ 1. ,  0. , -1.6]])
-
-    See Also
-    --------
-    robust_scale : Equivalent function without the estimator API.
-
-    :class:`~sklearn.decomposition.PCA`
-        Further removes the linear correlation across features with
-        'whiten=True'.
-
-    Notes
-    -----
-    For a comparison of the different scalers, transformers, and normalizers,
-    see :ref:`examples/preprocessing/plot_all_scaling.py
-    `.
-
-    https://en.wikipedia.org/wiki/Median
-    https://en.wikipedia.org/wiki/Interquartile_range
     """
-    @_deprecate_positional_args
-    def __init__(self, *, with_centering=True, with_scaling=True,
-                 quantile_range=(25.0, 75.0), copy=True, unit_variance=False):
+
+    _parameter_constraints: dict = {
+        "with_centering": ["boolean"],
+        "with_scaling": ["boolean"],
+        "quantile_range": [tuple],
+        "copy": ["boolean"],
+        "unit_variance": ["boolean"],
+    }
+
+    def __init__(
+        self,
+        *,
+        with_centering=True,
+        with_scaling=True,
+        quantile_range=(25.0, 75.0),
+        copy=True,
+        unit_variance=False,
+    ):
         self.with_centering = with_centering
         self.with_scaling = with_scaling
         self.quantile_range = quantile_range
         self.unit_variance = unit_variance
         self.copy = copy
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y=None):
         """Compute the median and quantiles to be used for scaling.
 
@@ -1346,8 +1625,8 @@ def fit(self, X, y=None):
             The data used to compute the median and quantiles
             used for later scaling along the features axis.
 
-        y : None
-            Ignored.
+        y : Ignored
+            Not used, present here for API consistency by convention.
 
         Returns
         -------
@@ -1356,20 +1635,24 @@ def fit(self, X, y=None):
         """
         # at fit, convert sparse matrices to csc for optimized computation of
         # the quantiles
-        X = self._validate_data(X, accept_sparse='csc', estimator=self,
-                                dtype=FLOAT_DTYPES,
-                                force_all_finite='allow-nan')
+        X = validate_data(
+            self,
+            X,
+            accept_sparse="csc",
+            dtype=FLOAT_DTYPES,
+            ensure_all_finite="allow-nan",
+        )
 
         q_min, q_max = self.quantile_range
         if not 0 <= q_min <= q_max <= 100:
-            raise ValueError("Invalid quantile range: %s" %
-                             str(self.quantile_range))
+            raise ValueError("Invalid quantile range: %s" % str(self.quantile_range))
 
         if self.with_centering:
             if sparse.issparse(X):
                 raise ValueError(
                     "Cannot center sparse matrices: use `with_centering=False`"
-                    " instead. See docstring for motivation and alternatives.")
+                    " instead. See docstring for motivation and alternatives."
+                )
             self.center_ = np.nanmedian(X, axis=0)
         else:
             self.center_ = None
@@ -1378,23 +1661,22 @@ def fit(self, X, y=None):
             quantiles = []
             for feature_idx in range(X.shape[1]):
                 if sparse.issparse(X):
-                    column_nnz_data = X.data[X.indptr[feature_idx]:
-                                             X.indptr[feature_idx + 1]]
+                    column_nnz_data = X.data[
+                        X.indptr[feature_idx] : X.indptr[feature_idx + 1]
+                    ]
                     column_data = np.zeros(shape=X.shape[0], dtype=X.dtype)
-                    column_data[:len(column_nnz_data)] = column_nnz_data
+                    column_data[: len(column_nnz_data)] = column_nnz_data
                 else:
                     column_data = X[:, feature_idx]
 
-                quantiles.append(np.nanpercentile(column_data,
-                                                  self.quantile_range))
+                quantiles.append(np.nanpercentile(column_data, self.quantile_range))
 
             quantiles = np.transpose(quantiles)
 
             self.scale_ = quantiles[1] - quantiles[0]
             self.scale_ = _handle_zeros_in_scale(self.scale_, copy=False)
             if self.unit_variance:
-                adjust = (stats.norm.ppf(q_max / 100.0) -
-                          stats.norm.ppf(q_min / 100.0))
+                adjust = stats.norm.ppf(q_max / 100.0) - stats.norm.ppf(q_min / 100.0)
                 self.scale_ = self.scale_ / adjust
         else:
             self.scale_ = None
@@ -1415,10 +1697,16 @@ def transform(self, X):
             Transformed array.
         """
         check_is_fitted(self)
-        X = self._validate_data(X, accept_sparse=('csr', 'csc'),
-                                copy=self.copy, estimator=self,
-                                dtype=FLOAT_DTYPES, reset=False,
-                                force_all_finite='allow-nan')
+        X = validate_data(
+            self,
+            X,
+            accept_sparse=("csr", "csc"),
+            copy=self.copy,
+            dtype=FLOAT_DTYPES,
+            force_writeable=True,
+            reset=False,
+            ensure_all_finite="allow-nan",
+        )
 
         if sparse.issparse(X):
             if self.with_scaling:
@@ -1431,7 +1719,7 @@ def transform(self, X):
         return X
 
     def inverse_transform(self, X):
-        """Scale back the data to the original representation
+        """Scale back the data to the original representation.
 
         Parameters
         ----------
@@ -1440,13 +1728,18 @@ def inverse_transform(self, X):
 
         Returns
         -------
-        X_tr : {ndarray, sparse matrix} of shape (n_samples, n_features)
+        X_original : {ndarray, sparse matrix} of shape (n_samples, n_features)
             Transformed array.
         """
         check_is_fitted(self)
-        X = check_array(X, accept_sparse=('csr', 'csc'), copy=self.copy,
-                        estimator=self, dtype=FLOAT_DTYPES,
-                        force_all_finite='allow-nan')
+        X = check_array(
+            X,
+            accept_sparse=("csr", "csc"),
+            copy=self.copy,
+            dtype=FLOAT_DTYPES,
+            force_writeable=True,
+            ensure_all_finite="allow-nan",
+        )
 
         if sparse.issparse(X):
             if self.with_scaling:
@@ -1458,14 +1751,28 @@ def inverse_transform(self, X):
                 X += self.center_
         return X
 
-    def _more_tags(self):
-        return {'allow_nan': True}
-
-
-@_deprecate_positional_args
-def robust_scale(X, *, axis=0, with_centering=True, with_scaling=True,
-                 quantile_range=(25.0, 75.0), copy=True, unit_variance=False):
-    """Standardize a dataset along any axis
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = not self.with_centering
+        tags.input_tags.allow_nan = True
+        return tags
+
+
+@validate_params(
+    {"X": ["array-like", "sparse matrix"], "axis": [Options(Integral, {0, 1})]},
+    prefer_skip_nested_validation=False,
+)
+def robust_scale(
+    X,
+    *,
+    axis=0,
+    with_centering=True,
+    with_scaling=True,
+    quantile_range=(25.0, 75.0),
+    copy=True,
+    unit_variance=False,
+):
+    """Standardize a dataset along any axis.
 
     Center to the median and component wise scale
     according to the interquartile range.
@@ -1478,32 +1785,35 @@ def robust_scale(X, *, axis=0, with_centering=True, with_scaling=True,
         The data to center and scale.
 
     axis : int, default=0
-        axis used to compute the medians and IQR along. If 0,
+        Axis used to compute the medians and IQR along. If 0,
         independently scale each feature, otherwise (if 1) scale
         each sample.
 
     with_centering : bool, default=True
-        If True, center the data before scaling.
+        If `True`, center the data before scaling.
 
     with_scaling : bool, default=True
-        If True, scale the data to unit variance (or equivalently,
+        If `True`, scale the data to unit variance (or equivalently,
         unit standard deviation).
 
-    quantile_range : tuple (q_min, q_max), 0.0 < q_min < q_max < 100.0
-        default=(25.0, 75.0), == (1st quantile, 3rd quantile), == IQR
-        Quantile range used to calculate ``scale_``.
+    quantile_range : tuple (q_min, q_max), 0.0 < q_min < q_max < 100.0,\
+        default=(25.0, 75.0)
+        Quantile range used to calculate `scale_`. By default this is equal to
+        the IQR, i.e., `q_min` is the first quantile and `q_max` is the third
+        quantile.
 
         .. versionadded:: 0.18
 
     copy : bool, default=True
-        set to False to perform inplace row normalization and avoid a
-        copy (if the input is already a numpy array or a scipy.sparse
-        CSR matrix and if axis is 1).
+        If False, try to avoid a copy and scale in place.
+        This is not guaranteed to always work in place; e.g. if the data is
+        a numpy array with an int dtype, a copy will be returned even with
+        copy=False.
 
     unit_variance : bool, default=False
-        If True, scale data so that normally distributed features have a
+        If `True`, scale data so that normally distributed features have a
         variance of 1. In general, if the difference between the x-values of
-        ``q_max`` and ``q_min`` for a standard normal distribution is greater
+        `q_max` and `q_min` for a standard normal distribution is greater
         than 1, the dataset will be scaled down. If less than 1, the dataset
         will be scaled up.
 
@@ -1514,6 +1824,11 @@ def robust_scale(X, *, axis=0, with_centering=True, with_scaling=True,
     X_tr : {ndarray, sparse matrix} of shape (n_samples, n_features)
         The transformed data.
 
+    See Also
+    --------
+    RobustScaler : Performs centering and scaling using the Transformer API
+        (e.g. as part of a preprocessing :class:`~sklearn.pipeline.Pipeline`).
+
     Notes
     -----
     This implementation will refuse to center scipy.sparse matrices
@@ -1528,8 +1843,7 @@ def robust_scale(X, *, axis=0, with_centering=True, with_scaling=True,
     To avoid memory copy the caller should pass a CSR matrix.
 
     For a comparison of the different scalers, transformers, and normalizers,
-    see :ref:`examples/preprocessing/plot_all_scaling.py
-    `.
+    see: :ref:`sphx_glr_auto_examples_preprocessing_plot_all_scaling.py`.
 
     .. warning:: Risk of data leak
 
@@ -1543,22 +1857,37 @@ def robust_scale(X, *, axis=0, with_centering=True, with_scaling=True,
         :ref:`Pipeline ` in order to prevent most risks of data
         leaking: `pipe = make_pipeline(RobustScaler(), LogisticRegression())`.
 
-    See Also
+    Examples
     --------
-    RobustScaler : Performs centering and scaling using the Transformer API
-        (e.g. as part of a preprocessing :class:`~sklearn.pipeline.Pipeline`).
+    >>> from sklearn.preprocessing import robust_scale
+    >>> X = [[-2, 1, 2], [-1, 0, 1]]
+    >>> robust_scale(X, axis=0)  # scale each column independently
+    array([[-1.,  1.,  1.],
+           [ 1., -1., -1.]])
+    >>> robust_scale(X, axis=1)  # scale each row independently
+    array([[-1.5,  0. ,  0.5],
+           [-1. ,  0. ,  1. ]])
     """
-    X = check_array(X, accept_sparse=('csr', 'csc'), copy=False,
-                    ensure_2d=False, dtype=FLOAT_DTYPES,
-                    force_all_finite='allow-nan')
+    X = check_array(
+        X,
+        accept_sparse=("csr", "csc"),
+        copy=False,
+        ensure_2d=False,
+        dtype=FLOAT_DTYPES,
+        ensure_all_finite="allow-nan",
+    )
     original_ndim = X.ndim
 
     if original_ndim == 1:
         X = X.reshape(X.shape[0], 1)
 
-    s = RobustScaler(with_centering=with_centering, with_scaling=with_scaling,
-                     quantile_range=quantile_range,
-                     unit_variance=unit_variance, copy=copy)
+    s = RobustScaler(
+        with_centering=with_centering,
+        with_scaling=with_scaling,
+        quantile_range=quantile_range,
+        unit_variance=unit_variance,
+        copy=copy,
+    )
     if axis == 0:
         X = s.fit_transform(X)
     else:
@@ -1570,383 +1899,120 @@ def robust_scale(X, *, axis=0, with_centering=True, with_scaling=True,
     return X
 
 
-class PolynomialFeatures(TransformerMixin, BaseEstimator):
-    """Generate polynomial and interaction features.
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "norm": [StrOptions({"l1", "l2", "max"})],
+        "axis": [Options(Integral, {0, 1})],
+        "copy": ["boolean"],
+        "return_norm": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def normalize(X, norm="l2", *, axis=1, copy=True, return_norm=False):
+    """Scale input vectors individually to unit norm (vector length).
 
-    Generate a new feature matrix consisting of all polynomial combinations
-    of the features with degree less than or equal to the specified degree.
-    For example, if an input sample is two dimensional and of the form
-    [a, b], the degree-2 polynomial features are [1, a, b, a^2, ab, b^2].
+    Read more in the :ref:`User Guide `.
 
     Parameters
     ----------
-    degree : int, default=2
-        The degree of the polynomial features.
-
-    interaction_only : bool, default=False
-        If true, only interaction features are produced: features that are
-        products of at most ``degree`` *distinct* input features (so not
-        ``x[1] ** 2``, ``x[0] * x[2] ** 3``, etc.).
-
-    include_bias : bool, default=True
-        If True (default), then include a bias column, the feature in which
-        all polynomial powers are zero (i.e. a column of ones - acts as an
-        intercept term in a linear model).
+    X : {array-like, sparse matrix} of shape (n_samples, n_features)
+        The data to normalize, element by element.
+        scipy.sparse matrices should be in CSR format to avoid an
+        un-necessary copy.
 
-    order : {'C', 'F'}, default='C'
-        Order of output array in the dense case. 'F' order is faster to
-        compute, but may slow down subsequent estimators.
+    norm : {'l1', 'l2', 'max'}, default='l2'
+        The norm to use to normalize each non zero sample (or each non-zero
+        feature if axis is 0).
 
-        .. versionadded:: 0.21
+    axis : {0, 1}, default=1
+        Define axis used to normalize the data along. If 1, independently
+        normalize each sample, otherwise (if 0) normalize each feature.
 
-    Examples
-    --------
-    >>> import numpy as np
-    >>> from sklearn.preprocessing import PolynomialFeatures
-    >>> X = np.arange(6).reshape(3, 2)
-    >>> X
-    array([[0, 1],
-           [2, 3],
-           [4, 5]])
-    >>> poly = PolynomialFeatures(2)
-    >>> poly.fit_transform(X)
-    array([[ 1.,  0.,  1.,  0.,  0.,  1.],
-           [ 1.,  2.,  3.,  4.,  6.,  9.],
-           [ 1.,  4.,  5., 16., 20., 25.]])
-    >>> poly = PolynomialFeatures(interaction_only=True)
-    >>> poly.fit_transform(X)
-    array([[ 1.,  0.,  1.,  0.],
-           [ 1.,  2.,  3.,  6.],
-           [ 1.,  4.,  5., 20.]])
+    copy : bool, default=True
+        If False, try to avoid a copy and normalize in place.
+        This is not guaranteed to always work in place; e.g. if the data is
+        a numpy array with an int dtype, a copy will be returned even with
+        copy=False.
 
-    Attributes
-    ----------
-    powers_ : ndarray of shape (n_output_features, n_input_features)
-        powers_[i, j] is the exponent of the jth input in the ith output.
+    return_norm : bool, default=False
+        Whether to return the computed norms.
 
-    n_input_features_ : int
-        The total number of input features.
+    Returns
+    -------
+    X : {ndarray, sparse matrix} of shape (n_samples, n_features)
+        Normalized input X.
 
-    n_output_features_ : int
-        The total number of polynomial output features. The number of output
-        features is computed by iterating over all suitably sized combinations
-        of input features.
+    norms : ndarray of shape (n_samples, ) if axis=1 else (n_features, )
+        An array of norms along given axis for X.
+        When X is sparse, a NotImplementedError will be raised
+        for norm 'l1' or 'l2'.
 
     See Also
     --------
-    SplineTransformer : Transformer that generates univariate B-spline bases
-        for features
+    Normalizer : Performs normalization using the Transformer API
+        (e.g. as part of a preprocessing :class:`~sklearn.pipeline.Pipeline`).
 
     Notes
     -----
-    Be aware that the number of features in the output array scales
-    polynomially in the number of features of the input array, and
-    exponentially in the degree. High degrees can cause overfitting.
+    For a comparison of the different scalers, transformers, and normalizers,
+    see: :ref:`sphx_glr_auto_examples_preprocessing_plot_all_scaling.py`.
 
-    See :ref:`examples/linear_model/plot_polynomial_interpolation.py
-    `
+    Examples
+    --------
+    >>> from sklearn.preprocessing import normalize
+    >>> X = [[-2, 1, 2], [-1, 0, 1]]
+    >>> normalize(X, norm="l1")  # L1 normalization each row independently
+    array([[-0.4,  0.2,  0.4],
+           [-0.5,  0. ,  0.5]])
+    >>> normalize(X, norm="l2")  # L2 normalization each row independently
+    array([[-0.67, 0.33, 0.67],
+           [-0.71, 0.  , 0.71]])
     """
-    @_deprecate_positional_args
-    def __init__(self, degree=2, *, interaction_only=False, include_bias=True,
-                 order='C'):
-        self.degree = degree
-        self.interaction_only = interaction_only
-        self.include_bias = include_bias
-        self.order = order
-
-    @staticmethod
-    def _combinations(n_features, degree, interaction_only, include_bias):
-        comb = (combinations if interaction_only else combinations_w_r)
-        start = int(not include_bias)
-        return chain.from_iterable(comb(range(n_features), i)
-                                   for i in range(start, degree + 1))
-
-    @property
-    def powers_(self):
-        check_is_fitted(self)
-
-        combinations = self._combinations(self.n_input_features_, self.degree,
-                                          self.interaction_only,
-                                          self.include_bias)
-        return np.vstack([np.bincount(c, minlength=self.n_input_features_)
-                          for c in combinations])
-
-    def get_feature_names(self, input_features=None):
-        """
-        Return feature names for output features
-
-        Parameters
-        ----------
-        input_features : list of str of shape (n_features,), default=None
-            String names for input features if available. By default,
-            "x0", "x1", ... "xn_features" is used.
-
-        Returns
-        -------
-        output_feature_names : list of str of shape (n_output_features,)
-        """
-        powers = self.powers_
-        if input_features is None:
-            input_features = ['x%d' % i for i in range(powers.shape[1])]
-        feature_names = []
-        for row in powers:
-            inds = np.where(row)[0]
-            if len(inds):
-                name = " ".join("%s^%d" % (input_features[ind], exp)
-                                if exp != 1 else input_features[ind]
-                                for ind, exp in zip(inds, row[inds]))
-            else:
-                name = "1"
-            feature_names.append(name)
-        return feature_names
-
-    def fit(self, X, y=None):
-        """
-        Compute number of output features.
-
-
-        Parameters
-        ----------
-        X : {array-like, sparse matrix} of shape (n_samples, n_features)
-            The data.
-
-        y : None
-            Ignored.
-
-        Returns
-        -------
-        self : object
-            Fitted transformer.
-        """
-        n_samples, n_features = self._validate_data(
-            X, accept_sparse=True).shape
-        combinations = self._combinations(n_features, self.degree,
-                                          self.interaction_only,
-                                          self.include_bias)
-        self.n_input_features_ = n_features
-        self.n_output_features_ = sum(1 for _ in combinations)
-        return self
-
-    def transform(self, X):
-        """Transform data to polynomial features.
-
-        Parameters
-        ----------
-        X : {array-like, sparse matrix} of shape (n_samples, n_features)
-            The data to transform, row by row.
-
-            Prefer CSR over CSC for sparse input (for speed), but CSC is
-            required if the degree is 4 or higher. If the degree is less than
-            4 and the input format is CSC, it will be converted to CSR, have
-            its polynomial features generated, then converted back to CSC.
-
-            If the degree is 2 or 3, the method described in "Leveraging
-            Sparsity to Speed Up Polynomial Feature Expansions of CSR Matrices
-            Using K-Simplex Numbers" by Andrew Nystrom and John Hughes is
-            used, which is much faster than the method used on CSC input. For
-            this reason, a CSC input will be converted to CSR, and the output
-            will be converted back to CSC prior to being returned, hence the
-            preference of CSR.
-
-        Returns
-        -------
-        XP : {ndarray, sparse matrix} of shape (n_samples, NP)
-            The matrix of features, where NP is the number of polynomial
-            features generated from the combination of inputs. If a sparse
-            matrix is provided, it will be converted into a sparse
-            ``csr_matrix``.
-        """
-        check_is_fitted(self)
-
-        X = self._validate_data(X, order='F', dtype=FLOAT_DTYPES, reset=False,
-                                accept_sparse=('csr', 'csc'))
-
-        n_samples, n_features = X.shape
-
-        if n_features != self.n_input_features_:
-            raise ValueError("X shape does not match training shape")
-
-        if sparse.isspmatrix_csr(X):
-            if self.degree > 3:
-                return self.transform(X.tocsc()).tocsr()
-            to_stack = []
-            if self.include_bias:
-                to_stack.append(np.ones(shape=(n_samples, 1), dtype=X.dtype))
-            to_stack.append(X)
-            for deg in range(2, self.degree+1):
-                Xp_next = _csr_polynomial_expansion(X.data, X.indices,
-                                                    X.indptr, X.shape[1],
-                                                    self.interaction_only,
-                                                    deg)
-                if Xp_next is None:
-                    break
-                to_stack.append(Xp_next)
-            XP = sparse.hstack(to_stack, format='csr')
-        elif sparse.isspmatrix_csc(X) and self.degree < 4:
-            return self.transform(X.tocsr()).tocsc()
-        else:
-            if sparse.isspmatrix(X):
-                combinations = self._combinations(n_features, self.degree,
-                                                  self.interaction_only,
-                                                  self.include_bias)
-                columns = []
-                for comb in combinations:
-                    if comb:
-                        out_col = 1
-                        for col_idx in comb:
-                            out_col = X[:, col_idx].multiply(out_col)
-                        columns.append(out_col)
-                    else:
-                        bias = sparse.csc_matrix(np.ones((X.shape[0], 1)))
-                        columns.append(bias)
-                XP = sparse.hstack(columns, dtype=X.dtype).tocsc()
-            else:
-                XP = np.empty((n_samples, self.n_output_features_),
-                              dtype=X.dtype, order=self.order)
-
-                # What follows is a faster implementation of:
-                # for i, comb in enumerate(combinations):
-                #     XP[:, i] = X[:, comb].prod(1)
-                # This implementation uses two optimisations.
-                # First one is broadcasting,
-                # multiply ([X1, ..., Xn], X1) -> [X1 X1, ..., Xn X1]
-                # multiply ([X2, ..., Xn], X2) -> [X2 X2, ..., Xn X2]
-                # ...
-                # multiply ([X[:, start:end], X[:, start]) -> ...
-                # Second optimisation happens for degrees >= 3.
-                # Xi^3 is computed reusing previous computation:
-                # Xi^3 = Xi^2 * Xi.
-
-                if self.include_bias:
-                    XP[:, 0] = 1
-                    current_col = 1
-                else:
-                    current_col = 0
-
-                # d = 0
-                XP[:, current_col:current_col + n_features] = X
-                index = list(range(current_col,
-                                   current_col + n_features))
-                current_col += n_features
-                index.append(current_col)
-
-                # d >= 1
-                for _ in range(1, self.degree):
-                    new_index = []
-                    end = index[-1]
-                    for feature_idx in range(n_features):
-                        start = index[feature_idx]
-                        new_index.append(current_col)
-                        if self.interaction_only:
-                            start += (index[feature_idx + 1] -
-                                      index[feature_idx])
-                        next_col = current_col + end - start
-                        if next_col <= current_col:
-                            break
-                        # XP[:, start:end] are terms of degree d - 1
-                        # that exclude feature #feature_idx.
-                        np.multiply(XP[:, start:end],
-                                    X[:, feature_idx:feature_idx + 1],
-                                    out=XP[:, current_col:next_col],
-                                    casting='no')
-                        current_col = next_col
-
-                    new_index.append(current_col)
-                    index = new_index
-
-        return XP
-
-
-@_deprecate_positional_args
-def normalize(X, norm='l2', *, axis=1, copy=True, return_norm=False):
-    """Scale input vectors individually to unit norm (vector length).
-
-    Read more in the :ref:`User Guide `.
-
-    Parameters
-    ----------
-    X : {array-like, sparse matrix} of shape (n_samples, n_features)
-        The data to normalize, element by element.
-        scipy.sparse matrices should be in CSR format to avoid an
-        un-necessary copy.
-
-    norm : {'l1', 'l2', 'max'}, default='l2'
-        The norm to use to normalize each non zero sample (or each non-zero
-        feature if axis is 0).
-
-    axis : {0, 1}, default=1
-        axis used to normalize the data along. If 1, independently normalize
-        each sample, otherwise (if 0) normalize each feature.
-
-    copy : bool, default=True
-        set to False to perform inplace row normalization and avoid a
-        copy (if the input is already a numpy array or a scipy.sparse
-        CSR matrix and if axis is 1).
-
-    return_norm : bool, default=False
-        whether to return the computed norms
-
-    Returns
-    -------
-    X : {ndarray, sparse matrix} of shape (n_samples, n_features)
-        Normalized input X.
-
-    norms : ndarray of shape (n_samples, ) if axis=1 else (n_features, )
-        An array of norms along given axis for X.
-        When X is sparse, a NotImplementedError will be raised
-        for norm 'l1' or 'l2'.
-
-    See Also
-    --------
-    Normalizer : Performs normalization using the Transformer API
-        (e.g. as part of a preprocessing :class:`~sklearn.pipeline.Pipeline`).
-
-    Notes
-    -----
-    For a comparison of the different scalers, transformers, and normalizers,
-    see :ref:`examples/preprocessing/plot_all_scaling.py
-    `.
-
-    """
-    if norm not in ('l1', 'l2', 'max'):
-        raise ValueError("'%s' is not a supported norm" % norm)
-
-    if axis == 0:
-        sparse_format = 'csc'
-    elif axis == 1:
-        sparse_format = 'csr'
-    else:
-        raise ValueError("'%d' is not a supported axis" % axis)
-
-    X = check_array(X, accept_sparse=sparse_format, copy=copy,
-                    estimator='the normalize function', dtype=FLOAT_DTYPES)
-    if axis == 0:
-        X = X.T
+    if axis == 0:
+        sparse_format = "csc"
+    else:  # axis == 1:
+        sparse_format = "csr"
+
+    xp, _ = get_namespace(X)
+
+    X = check_array(
+        X,
+        accept_sparse=sparse_format,
+        copy=copy,
+        estimator="the normalize function",
+        dtype=_array_api.supported_float_dtypes(xp),
+        force_writeable=True,
+    )
+    if axis == 0:
+        X = X.T
 
     if sparse.issparse(X):
-        if return_norm and norm in ('l1', 'l2'):
-            raise NotImplementedError("return_norm=True is not implemented "
-                                      "for sparse matrices with norm 'l1' "
-                                      "or norm 'l2'")
-        if norm == 'l1':
+        if return_norm and norm in ("l1", "l2"):
+            raise NotImplementedError(
+                "return_norm=True is not implemented "
+                "for sparse matrices with norm 'l1' "
+                "or norm 'l2'"
+            )
+        if norm == "l1":
             inplace_csr_row_normalize_l1(X)
-        elif norm == 'l2':
+        elif norm == "l2":
             inplace_csr_row_normalize_l2(X)
-        elif norm == 'max':
+        elif norm == "max":
             mins, maxes = min_max_axis(X, 1)
             norms = np.maximum(abs(mins), maxes)
             norms_elementwise = norms.repeat(np.diff(X.indptr))
             mask = norms_elementwise != 0
             X.data[mask] /= norms_elementwise[mask]
     else:
-        if norm == 'l1':
-            norms = np.abs(X).sum(axis=1)
-        elif norm == 'l2':
+        if norm == "l1":
+            norms = xp.sum(xp.abs(X), axis=1)
+        elif norm == "l2":
             norms = row_norms(X)
-        elif norm == 'max':
-            norms = np.max(abs(X), axis=1)
+        elif norm == "max":
+            norms = xp.max(xp.abs(X), axis=1)
         norms = _handle_zeros_in_scale(norms, copy=False)
-        X /= norms[:, np.newaxis]
+        X /= norms[:, None]
 
     if axis == 0:
         X = X.T
@@ -1957,7 +2023,7 @@ def normalize(X, norm='l2', *, axis=1, copy=True, return_norm=False):
         return X
 
 
-class Normalizer(TransformerMixin, BaseEstimator):
+class Normalizer(OneToOneFeatureMixin, TransformerMixin, BaseEstimator):
     """Normalize samples individually to unit norm.
 
     Each sample (i.e. each row of the data matrix) with at least one
@@ -1974,6 +2040,9 @@ class Normalizer(TransformerMixin, BaseEstimator):
     of the vectors and is the base similarity metric for the Vector
     Space Model commonly used by the Information Retrieval community.
 
+    For an example visualization, refer to :ref:`Compare Normalizer with other
+    scalers `.
+
     Read more in the :ref:`User Guide `.
 
     Parameters
@@ -1984,10 +2053,34 @@ class Normalizer(TransformerMixin, BaseEstimator):
         values.
 
     copy : bool, default=True
-        set to False to perform inplace row normalization and avoid a
+        Set to False to perform inplace row normalization and avoid a
         copy (if the input is already a numpy array or a scipy.sparse
         CSR matrix).
 
+    Attributes
+    ----------
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    normalize : Equivalent function without the estimator API.
+
+    Notes
+    -----
+    This estimator is :term:`stateless` and does not need to be fitted.
+    However, we recommend to call :meth:`fit_transform` instead of
+    :meth:`transform`, as parameter validation is only performed in
+    :meth:`fit`.
+
     Examples
     --------
     >>> from sklearn.preprocessing import Normalizer
@@ -2001,50 +2094,42 @@ class Normalizer(TransformerMixin, BaseEstimator):
     array([[0.8, 0.2, 0.4, 0.4],
            [0.1, 0.3, 0.9, 0.3],
            [0.5, 0.7, 0.5, 0.1]])
-
-    Notes
-    -----
-    This estimator is stateless (besides constructor parameters), the
-    fit method does nothing but is useful when used in a pipeline.
-
-    For a comparison of the different scalers, transformers, and normalizers,
-    see :ref:`examples/preprocessing/plot_all_scaling.py
-    `.
-
-    See Also
-    --------
-    normalize : Equivalent function without the estimator API.
     """
 
-    @_deprecate_positional_args
-    def __init__(self, norm='l2', *, copy=True):
+    _parameter_constraints: dict = {
+        "norm": [StrOptions({"l1", "l2", "max"})],
+        "copy": ["boolean"],
+    }
+
+    def __init__(self, norm="l2", *, copy=True):
         self.norm = norm
         self.copy = copy
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y=None):
-        """Do nothing and return the estimator unchanged
+        """Only validates estimator's parameters.
 
-        This method is just there to implement the usual API and hence
-        work in pipelines.
+        This method allows to: (i) validate the estimator's parameters and
+        (ii) be consistent with the scikit-learn transformer API.
 
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
             The data to estimate the normalization parameters.
 
-        y : None
-            Ignored.
+        y : Ignored
+            Not used, present here for API consistency by convention.
 
         Returns
         -------
         self : object
             Fitted transformer.
         """
-        self._validate_data(X, accept_sparse='csr')
+        validate_data(self, X, accept_sparse="csr")
         return self
 
     def transform(self, X, copy=None):
-        """Scale each non zero row of X to unit norm
+        """Scale each non zero row of X to unit norm.
 
         Parameters
         ----------
@@ -2061,14 +2146,27 @@ def transform(self, X, copy=None):
             Transformed array.
         """
         copy = copy if copy is not None else self.copy
-        X = self._validate_data(X, accept_sparse='csr', reset=False)
-        return normalize(X, norm=self.norm, axis=1, copy=copy)
-
-    def _more_tags(self):
-        return {'stateless': True}
-
-
-@_deprecate_positional_args
+        X = validate_data(
+            self, X, accept_sparse="csr", force_writeable=True, copy=copy, reset=False
+        )
+        return normalize(X, norm=self.norm, axis=1, copy=False)
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
+        tags.requires_fit = False
+        tags.array_api_support = True
+        return tags
+
+
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "threshold": [Interval(Real, None, None, closed="neither")],
+        "copy": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
 def binarize(X, *, threshold=0.0, copy=True):
     """Boolean thresholding of array-like or scipy.sparse matrix.
 
@@ -2086,9 +2184,10 @@ def binarize(X, *, threshold=0.0, copy=True):
         Threshold may not be less than 0 for operations on sparse matrices.
 
     copy : bool, default=True
-        set to False to perform inplace binarization and avoid a copy
-        (if the input is already a numpy array or a scipy.sparse CSR / CSC
-        matrix and if axis is 1).
+        If False, try to avoid a copy and binarize in place.
+        This is not guaranteed to always work in place; e.g. if the data is
+        a numpy array with an object dtype, a copy will be returned even with
+        copy=False.
 
     Returns
     -------
@@ -2099,26 +2198,35 @@ def binarize(X, *, threshold=0.0, copy=True):
     --------
     Binarizer : Performs binarization using the Transformer API
         (e.g. as part of a preprocessing :class:`~sklearn.pipeline.Pipeline`).
+
+    Examples
+    --------
+    >>> from sklearn.preprocessing import binarize
+    >>> X = [[0.4, 0.6, 0.5], [0.6, 0.1, 0.2]]
+    >>> binarize(X, threshold=0.5)
+    array([[0., 1., 0.],
+           [1., 0., 0.]])
     """
-    X = check_array(X, accept_sparse=['csr', 'csc'], copy=copy)
+    X = check_array(X, accept_sparse=["csr", "csc"], force_writeable=True, copy=copy)
     if sparse.issparse(X):
         if threshold < 0:
-            raise ValueError('Cannot binarize a sparse matrix with threshold '
-                             '< 0')
+            raise ValueError("Cannot binarize a sparse matrix with threshold < 0")
         cond = X.data > threshold
         not_cond = np.logical_not(cond)
         X.data[cond] = 1
         X.data[not_cond] = 0
         X.eliminate_zeros()
     else:
-        cond = X > threshold
-        not_cond = np.logical_not(cond)
+        xp, _, device = get_namespace_and_device(X)
+        float_dtype = _find_matching_floating_dtype(X, threshold, xp=xp)
+        cond = xp.astype(X, float_dtype, copy=False) > threshold
+        not_cond = xp.logical_not(cond)
         X[cond] = 1
         X[not_cond] = 0
     return X
 
 
-class Binarizer(TransformerMixin, BaseEstimator):
+class Binarizer(OneToOneFeatureMixin, TransformerMixin, BaseEstimator):
     """Binarize data (set feature values to 0 or 1) according to a threshold.
 
     Values greater than the threshold map to 1, while values less than
@@ -2142,9 +2250,38 @@ class Binarizer(TransformerMixin, BaseEstimator):
         Threshold may not be less than 0 for operations on sparse matrices.
 
     copy : bool, default=True
-        set to False to perform inplace binarization and avoid a copy (if
+        Set to False to perform inplace binarization and avoid a copy (if
         the input is already a numpy array or a scipy.sparse CSR matrix).
 
+    Attributes
+    ----------
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    binarize : Equivalent function without the estimator API.
+    KBinsDiscretizer : Bin continuous data into intervals.
+    OneHotEncoder : Encode categorical features as a one-hot numeric array.
+
+    Notes
+    -----
+    If the input is a sparse matrix, only the non-zero values are subject
+    to update by the :class:`Binarizer` class.
+
+    This estimator is :term:`stateless` and does not need to be fitted.
+    However, we recommend to call :meth:`fit_transform` instead of
+    :meth:`transform`, as parameter validation is only performed in
+    :meth:`fit`.
+
     Examples
     --------
     >>> from sklearn.preprocessing import Binarizer
@@ -2158,30 +2295,23 @@ class Binarizer(TransformerMixin, BaseEstimator):
     array([[1., 0., 1.],
            [1., 0., 0.],
            [0., 1., 0.]])
-
-    Notes
-    -----
-    If the input is a sparse matrix, only the non-zero values are subject
-    to update by the Binarizer class.
-
-    This estimator is stateless (besides constructor parameters), the
-    fit method does nothing but is useful when used in a pipeline.
-
-    See Also
-    --------
-    binarize : Equivalent function without the estimator API.
     """
 
-    @_deprecate_positional_args
+    _parameter_constraints: dict = {
+        "threshold": [Real],
+        "copy": ["boolean"],
+    }
+
     def __init__(self, *, threshold=0.0, copy=True):
         self.threshold = threshold
         self.copy = copy
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y=None):
-        """Do nothing and return the estimator unchanged.
+        """Only validates estimator's parameters.
 
-        This method is just there to implement the usual API and hence
-        work in pipelines.
+        This method allows to: (i) validate the estimator's parameters and
+        (ii) be consistent with the scikit-learn transformer API.
 
         Parameters
         ----------
@@ -2196,7 +2326,7 @@ def fit(self, X, y=None):
         self : object
             Fitted transformer.
         """
-        self._validate_data(X, accept_sparse='csr')
+        validate_data(self, X, accept_sparse="csr")
         return self
 
     def transform(self, X, copy=None):
@@ -2220,33 +2350,81 @@ def transform(self, X, copy=None):
         copy = copy if copy is not None else self.copy
         # TODO: This should be refactored because binarize also calls
         # check_array
-        X = self._validate_data(X, accept_sparse=['csr', 'csc'], copy=copy,
-                                reset=False)
+        X = validate_data(
+            self,
+            X,
+            accept_sparse=["csr", "csc"],
+            force_writeable=True,
+            copy=copy,
+            reset=False,
+        )
         return binarize(X, threshold=self.threshold, copy=False)
 
-    def _more_tags(self):
-        return {'stateless': True}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.requires_fit = False
+        tags.array_api_support = True
+        tags.input_tags.sparse = True
+        return tags
+
+
+class KernelCenterer(ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator):
+    r"""Center an arbitrary kernel matrix :math:`K`.
+
+    Let define a kernel :math:`K` such that:
+
+    .. math::
+        K(X, Y) = \phi(X) . \phi(Y)^{T}
+
+    :math:`\phi(X)` is a function mapping of rows of :math:`X` to a
+    Hilbert space and :math:`K` is of shape `(n_samples, n_samples)`.
+
+    This class allows to compute :math:`\tilde{K}(X, Y)` such that:
 
+    .. math::
+        \tilde{K(X, Y)} = \tilde{\phi}(X) . \tilde{\phi}(Y)^{T}
 
-class KernelCenterer(TransformerMixin, BaseEstimator):
-    """Center a kernel matrix.
+    :math:`\tilde{\phi}(X)` is the centered mapped data in the Hilbert
+    space.
 
-    Let K(x, z) be a kernel defined by phi(x)^T phi(z), where phi is a
-    function mapping x to a Hilbert space. KernelCenterer centers (i.e.,
-    normalize to have zero mean) the data without explicitly computing phi(x).
-    It is equivalent to centering phi(x) with
-    sklearn.preprocessing.StandardScaler(with_std=False).
+    `KernelCenterer` centers the features without explicitly computing the
+    mapping :math:`\phi(\cdot)`. Working with centered kernels is sometime
+    expected when dealing with algebra computation such as eigendecomposition
+    for :class:`~sklearn.decomposition.KernelPCA` for instance.
 
     Read more in the :ref:`User Guide `.
 
     Attributes
     ----------
-    K_fit_rows_ : array of shape (n_samples,)
+    K_fit_rows_ : ndarray of shape (n_samples,)
         Average of each column of kernel matrix.
 
     K_fit_all_ : float
         Average of kernel matrix.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    sklearn.kernel_approximation.Nystroem : Approximate a kernel map
+        using a subset of the training data.
+
+    References
+    ----------
+    .. [1] `SchÃļlkopf, Bernhard, Alexander Smola, and Klaus-Robert MÃŧller.
+       "Nonlinear component analysis as a kernel eigenvalue problem."
+       Neural computation 10.5 (1998): 1299-1319.
+       `_
+
     Examples
     --------
     >>> from sklearn.preprocessing import KernelCenterer
@@ -2268,12 +2446,12 @@ class KernelCenterer(TransformerMixin, BaseEstimator):
            [ -5., -14.,  19.]])
     """
 
-    def __init__(self):
-        # Needed for backported inspect.signature compatibility with PyPy
-        pass
+    # X is called K in these methods.
+    __metadata_request__transform = {"K": metadata_routing.UNUSED}
+    __metadata_request__fit = {"K": metadata_routing.UNUSED}
 
     def fit(self, K, y=None):
-        """Fit KernelCenterer
+        """Fit KernelCenterer.
 
         Parameters
         ----------
@@ -2286,19 +2464,21 @@ def fit(self, K, y=None):
         Returns
         -------
         self : object
-            Fitted transformer.
+            Returns the instance itself.
         """
+        xp, _ = get_namespace(K)
 
-        K = self._validate_data(K, dtype=FLOAT_DTYPES)
+        K = validate_data(self, K, dtype=_array_api.supported_float_dtypes(xp))
 
         if K.shape[0] != K.shape[1]:
-            raise ValueError("Kernel matrix must be a square matrix."
-                             " Input is a {}x{} matrix."
-                             .format(K.shape[0], K.shape[1]))
+            raise ValueError(
+                "Kernel matrix must be a square matrix."
+                " Input is a {}x{} matrix.".format(K.shape[0], K.shape[1])
+            )
 
         n_samples = K.shape[0]
-        self.K_fit_rows_ = np.sum(K, axis=0) / n_samples
-        self.K_fit_all_ = self.K_fit_rows_.sum() / n_samples
+        self.K_fit_rows_ = xp.sum(K, axis=0) / n_samples
+        self.K_fit_all_ = xp.sum(self.K_fit_rows_) / n_samples
         return self
 
     def transform(self, K, copy=True):
@@ -2315,13 +2495,22 @@ def transform(self, K, copy=True):
         Returns
         -------
         K_new : ndarray of shape (n_samples1, n_samples2)
+            Returns the instance itself.
         """
         check_is_fitted(self)
 
-        K = self._validate_data(K, copy=copy, dtype=FLOAT_DTYPES, reset=False)
+        xp, _ = get_namespace(K)
 
-        K_pred_cols = (np.sum(K, axis=1) /
-                       self.K_fit_rows_.shape[0])[:, np.newaxis]
+        K = validate_data(
+            self,
+            K,
+            copy=copy,
+            force_writeable=True,
+            dtype=_array_api.supported_float_dtypes(xp),
+            reset=False,
+        )
+
+        K_pred_cols = (xp.sum(K, axis=1) / self.K_fit_rows_.shape[0])[:, None]
 
         K -= self.K_fit_rows_
         K -= K_pred_cols
@@ -2329,18 +2518,29 @@ def transform(self, K, copy=True):
 
         return K
 
-    def _more_tags(self):
-        return {'pairwise': True}
-
-    # TODO: Remove in 1.1
-    # mypy error: Decorated property not supported
-    @deprecated("Attribute _pairwise was deprecated in "  # type: ignore
-                "version 0.24 and will be removed in 1.1.")
     @property
-    def _pairwise(self):
-        return True
-
-
+    def _n_features_out(self):
+        """Number of transformed output features."""
+        # Used by ClassNamePrefixFeaturesOutMixin. This model preserves the
+        # number of input features but this is not a one-to-one mapping in the
+        # usual sense. Hence the choice not to use OneToOneFeatureMixin to
+        # implement get_feature_names_out for this class.
+        return self.n_features_in_
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.pairwise = True
+        tags.array_api_support = True
+        return tags
+
+
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "value": [Interval(Real, None, None, closed="neither")],
+    },
+    prefer_skip_nested_validation=True,
+)
 def add_dummy_feature(X, value=1.0):
     """Augment dataset with an additional dummy feature.
 
@@ -2367,11 +2567,11 @@ def add_dummy_feature(X, value=1.0):
     array([[1., 0., 1.],
            [1., 1., 0.]])
     """
-    X = check_array(X, accept_sparse=['csc', 'csr', 'coo'], dtype=FLOAT_DTYPES)
+    X = check_array(X, accept_sparse=["csc", "csr", "coo"], dtype=FLOAT_DTYPES)
     n_samples, n_features = X.shape
     shape = (n_samples, n_features + 1)
     if sparse.issparse(X):
-        if sparse.isspmatrix_coo(X):
+        if X.format == "coo":
             # Shift columns to the right.
             col = X.col + 1
             # Column indices of dummy feature are 0 everywhere.
@@ -2381,7 +2581,7 @@ def add_dummy_feature(X, value=1.0):
             # Prepend the dummy feature n_samples times.
             data = np.concatenate((np.full(n_samples, value), X.data))
             return sparse.coo_matrix((data, (row, col)), shape)
-        elif sparse.isspmatrix_csc(X):
+        elif X.format == "csc":
             # Shift index pointers since we need to add n_samples elements.
             indptr = X.indptr + n_samples
             # indptr[0] must be 0.
@@ -2398,7 +2598,7 @@ def add_dummy_feature(X, value=1.0):
         return np.hstack((np.full((n_samples, 1), value), X))
 
 
-class QuantileTransformer(TransformerMixin, BaseEstimator):
+class QuantileTransformer(OneToOneFeatureMixin, TransformerMixin, BaseEstimator):
     """Transform features using quantiles information.
 
     This method transforms the features to follow a uniform or a normal
@@ -2416,6 +2616,9 @@ class QuantileTransformer(TransformerMixin, BaseEstimator):
     correlations between variables measured at the same scale but renders
     variables measured at different scales more directly comparable.
 
+    For example visualizations, refer to :ref:`Compare QuantileTransformer with
+    other scalers `.
+
     Read more in the :ref:`User Guide `.
 
     .. versionadded:: 0.19
@@ -2439,17 +2642,21 @@ class QuantileTransformer(TransformerMixin, BaseEstimator):
         matrix are discarded to compute the quantile statistics. If False,
         these entries are treated as zeros.
 
-    subsample : int, default=1e5
+    subsample : int or None, default=10_000
         Maximum number of samples used to estimate the quantiles for
         computational efficiency. Note that the subsampling procedure may
         differ for value-identical sparse and dense matrices.
+        Disable subsampling by setting `subsample=None`.
+
+        .. versionadded:: 1.5
+           The option `None` to disable subsampling was added.
 
     random_state : int, RandomState instance or None, default=None
         Determines random number generation for subsampling and smoothing
         noise.
         Please see ``subsample`` for more details.
         Pass an int for reproducible results across multiple function calls.
-        See :term:`Glossary `
+        See :term:`Glossary `.
 
     copy : bool, default=True
         Set to False to perform inplace transformation and avoid a copy (if the
@@ -2467,15 +2674,16 @@ class QuantileTransformer(TransformerMixin, BaseEstimator):
     references_ : ndarray of shape (n_quantiles, )
         Quantiles of references.
 
-    Examples
-    --------
-    >>> import numpy as np
-    >>> from sklearn.preprocessing import QuantileTransformer
-    >>> rng = np.random.RandomState(0)
-    >>> X = np.sort(rng.normal(loc=0.5, scale=0.25, size=(25, 1)), axis=0)
-    >>> qt = QuantileTransformer(n_quantiles=10, random_state=0)
-    >>> qt.fit_transform(X)
-    array([...])
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
 
     See Also
     --------
@@ -2492,15 +2700,36 @@ class QuantileTransformer(TransformerMixin, BaseEstimator):
     NaNs are treated as missing values: disregarded in fit, and maintained in
     transform.
 
-    For a comparison of the different scalers, transformers, and normalizers,
-    see :ref:`examples/preprocessing/plot_all_scaling.py
-    `.
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from sklearn.preprocessing import QuantileTransformer
+    >>> rng = np.random.RandomState(0)
+    >>> X = np.sort(rng.normal(loc=0.5, scale=0.25, size=(25, 1)), axis=0)
+    >>> qt = QuantileTransformer(n_quantiles=10, random_state=0)
+    >>> qt.fit_transform(X)
+    array([...])
     """
 
-    @_deprecate_positional_args
-    def __init__(self, *, n_quantiles=1000, output_distribution='uniform',
-                 ignore_implicit_zeros=False, subsample=int(1e5),
-                 random_state=None, copy=True):
+    _parameter_constraints: dict = {
+        "n_quantiles": [Interval(Integral, 1, None, closed="left")],
+        "output_distribution": [StrOptions({"uniform", "normal"})],
+        "ignore_implicit_zeros": ["boolean"],
+        "subsample": [Interval(Integral, 1, None, closed="left"), None],
+        "random_state": ["random_state"],
+        "copy": ["boolean"],
+    }
+
+    def __init__(
+        self,
+        *,
+        n_quantiles=1000,
+        output_distribution="uniform",
+        ignore_implicit_zeros=False,
+        subsample=10_000,
+        random_state=None,
+        copy=True,
+    ):
         self.n_quantiles = n_quantiles
         self.output_distribution = output_distribution
         self.ignore_implicit_zeros = ignore_implicit_zeros
@@ -2517,21 +2746,21 @@ def _dense_fit(self, X, random_state):
             The data used to scale along the features axis.
         """
         if self.ignore_implicit_zeros:
-            warnings.warn("'ignore_implicit_zeros' takes effect only with"
-                          " sparse matrix. This parameter has no effect.")
+            warnings.warn(
+                "'ignore_implicit_zeros' takes effect only with"
+                " sparse matrix. This parameter has no effect."
+            )
 
         n_samples, n_features = X.shape
         references = self.references_ * 100
 
-        self.quantiles_ = []
-        for col in X.T:
-            if self.subsample < n_samples:
-                subsample_idx = random_state.choice(n_samples,
-                                                    size=self.subsample,
-                                                    replace=False)
-                col = col.take(subsample_idx, mode='clip')
-            self.quantiles_.append(np.nanpercentile(col, references))
-        self.quantiles_ = np.transpose(self.quantiles_)
+        if self.subsample is not None and self.subsample < n_samples:
+            # Take a subsample of `X`
+            X = resample(
+                X, replace=False, n_samples=self.subsample, random_state=random_state
+            )
+
+        self.quantiles_ = np.nanpercentile(X, references, axis=0)
         # Due to floating-point precision error in `np.nanpercentile`,
         # make sure that quantiles are monotonically increasing.
         # Upstream issue in numpy:
@@ -2553,33 +2782,29 @@ def _sparse_fit(self, X, random_state):
 
         self.quantiles_ = []
         for feature_idx in range(n_features):
-            column_nnz_data = X.data[X.indptr[feature_idx]:
-                                     X.indptr[feature_idx + 1]]
-            if len(column_nnz_data) > self.subsample:
-                column_subsample = (self.subsample * len(column_nnz_data) //
-                                    n_samples)
+            column_nnz_data = X.data[X.indptr[feature_idx] : X.indptr[feature_idx + 1]]
+            if self.subsample is not None and len(column_nnz_data) > self.subsample:
+                column_subsample = self.subsample * len(column_nnz_data) // n_samples
                 if self.ignore_implicit_zeros:
-                    column_data = np.zeros(shape=column_subsample,
-                                           dtype=X.dtype)
+                    column_data = np.zeros(shape=column_subsample, dtype=X.dtype)
                 else:
                     column_data = np.zeros(shape=self.subsample, dtype=X.dtype)
                 column_data[:column_subsample] = random_state.choice(
-                    column_nnz_data, size=column_subsample, replace=False)
+                    column_nnz_data, size=column_subsample, replace=False
+                )
             else:
                 if self.ignore_implicit_zeros:
-                    column_data = np.zeros(shape=len(column_nnz_data),
-                                           dtype=X.dtype)
+                    column_data = np.zeros(shape=len(column_nnz_data), dtype=X.dtype)
                 else:
                     column_data = np.zeros(shape=n_samples, dtype=X.dtype)
-                column_data[:len(column_nnz_data)] = column_nnz_data
+                column_data[: len(column_nnz_data)] = column_nnz_data
 
             if not column_data.size:
                 # if no nnz, an error will be raised for computing the
                 # quantiles. Force the quantiles to be zeros.
                 self.quantiles_.append([0] * len(references))
             else:
-                self.quantiles_.append(
-                        np.nanpercentile(column_data, references))
+                self.quantiles_.append(np.nanpercentile(column_data, references))
         self.quantiles_ = np.transpose(self.quantiles_)
         # due to floating-point precision error in `np.nanpercentile`,
         # make sure the quantiles are monotonically increasing
@@ -2587,6 +2812,7 @@ def _sparse_fit(self, X, random_state):
         # https://github.com/numpy/numpy/issues/14685
         self.quantiles_ = np.maximum.accumulate(self.quantiles_)
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y=None):
         """Compute the quantiles used for transforming.
 
@@ -2606,37 +2832,28 @@ def fit(self, X, y=None):
         self : object
            Fitted transformer.
         """
-        if self.n_quantiles <= 0:
-            raise ValueError("Invalid value for 'n_quantiles': %d. "
-                             "The number of quantiles must be at least one."
-                             % self.n_quantiles)
-
-        if self.subsample <= 0:
-            raise ValueError("Invalid value for 'subsample': %d. "
-                             "The number of subsamples must be at least one."
-                             % self.subsample)
-
-        if self.n_quantiles > self.subsample:
-            raise ValueError("The number of quantiles cannot be greater than"
-                             " the number of samples used. Got {} quantiles"
-                             " and {} samples.".format(self.n_quantiles,
-                                                       self.subsample))
+        if self.subsample is not None and self.n_quantiles > self.subsample:
+            raise ValueError(
+                "The number of quantiles cannot be greater than"
+                " the number of samples used. Got {} quantiles"
+                " and {} samples.".format(self.n_quantiles, self.subsample)
+            )
 
         X = self._check_inputs(X, in_fit=True, copy=False)
         n_samples = X.shape[0]
 
         if self.n_quantiles > n_samples:
-            warnings.warn("n_quantiles (%s) is greater than the total number "
-                          "of samples (%s). n_quantiles is set to "
-                          "n_samples."
-                          % (self.n_quantiles, n_samples))
+            warnings.warn(
+                "n_quantiles (%s) is greater than the total number "
+                "of samples (%s). n_quantiles is set to "
+                "n_samples." % (self.n_quantiles, n_samples)
+            )
         self.n_quantiles_ = max(1, min(self.n_quantiles, n_samples))
 
         rng = check_random_state(self.random_state)
 
         # Create the quantiles of reference
-        self.references_ = np.linspace(0, 1, self.n_quantiles_,
-                                       endpoint=True)
+        self.references_ = np.linspace(0, 1, self.n_quantiles_, endpoint=True)
         if sparse.issparse(X):
             self._sparse_fit(X, rng)
         else:
@@ -2660,21 +2877,19 @@ def _transform_col(self, X_col, quantiles, inverse):
             lower_bound_y = quantiles[0]
             upper_bound_y = quantiles[-1]
             # for inverse transform, match a uniform distribution
-            with np.errstate(invalid='ignore'):  # hide NaN comparison warnings
-                if output_distribution == 'normal':
+            with np.errstate(invalid="ignore"):  # hide NaN comparison warnings
+                if output_distribution == "normal":
                     X_col = stats.norm.cdf(X_col)
                 # else output distribution is already a uniform distribution
 
         # find index for lower and higher bounds
-        with np.errstate(invalid='ignore'):  # hide NaN comparison warnings
-            if output_distribution == 'normal':
-                lower_bounds_idx = (X_col - BOUNDS_THRESHOLD <
-                                    lower_bound_x)
-                upper_bounds_idx = (X_col + BOUNDS_THRESHOLD >
-                                    upper_bound_x)
-            if output_distribution == 'uniform':
-                lower_bounds_idx = (X_col == lower_bound_x)
-                upper_bounds_idx = (X_col == upper_bound_x)
+        with np.errstate(invalid="ignore"):  # hide NaN comparison warnings
+            if output_distribution == "normal":
+                lower_bounds_idx = X_col - BOUNDS_THRESHOLD < lower_bound_x
+                upper_bounds_idx = X_col + BOUNDS_THRESHOLD > upper_bound_x
+            if output_distribution == "uniform":
+                lower_bounds_idx = X_col == lower_bound_x
+                upper_bounds_idx = X_col == upper_bound_x
 
         isfinite_mask = ~np.isnan(X_col)
         X_col_finite = X_col[isfinite_mask]
@@ -2686,53 +2901,56 @@ def _transform_col(self, X_col, quantiles, inverse):
             # If we don't do this, only one extreme of the duplicated is
             # used (the upper when we do ascending, and the
             # lower for descending). We take the mean of these two
-            X_col[isfinite_mask] = .5 * (
+            X_col[isfinite_mask] = 0.5 * (
                 np.interp(X_col_finite, quantiles, self.references_)
-                - np.interp(-X_col_finite, -quantiles[::-1],
-                            -self.references_[::-1]))
+                - np.interp(-X_col_finite, -quantiles[::-1], -self.references_[::-1])
+            )
         else:
-            X_col[isfinite_mask] = np.interp(X_col_finite,
-                                             self.references_, quantiles)
+            X_col[isfinite_mask] = np.interp(X_col_finite, self.references_, quantiles)
 
         X_col[upper_bounds_idx] = upper_bound_y
         X_col[lower_bounds_idx] = lower_bound_y
         # for forward transform, match the output distribution
         if not inverse:
-            with np.errstate(invalid='ignore'):  # hide NaN comparison warnings
-                if output_distribution == 'normal':
+            with np.errstate(invalid="ignore"):  # hide NaN comparison warnings
+                if output_distribution == "normal":
                     X_col = stats.norm.ppf(X_col)
                     # find the value to clip the data to avoid mapping to
                     # infinity. Clip such that the inverse transform will be
                     # consistent
                     clip_min = stats.norm.ppf(BOUNDS_THRESHOLD - np.spacing(1))
-                    clip_max = stats.norm.ppf(1 - (BOUNDS_THRESHOLD -
-                                                   np.spacing(1)))
+                    clip_max = stats.norm.ppf(1 - (BOUNDS_THRESHOLD - np.spacing(1)))
                     X_col = np.clip(X_col, clip_min, clip_max)
                 # else output distribution is uniform and the ppf is the
                 # identity function so we let X_col unchanged
 
         return X_col
 
-    def _check_inputs(self, X, in_fit, accept_sparse_negative=False,
-                      copy=False):
+    def _check_inputs(self, X, in_fit, accept_sparse_negative=False, copy=False):
         """Check inputs before fit and transform."""
-        X = self._validate_data(X, reset=in_fit,
-                                accept_sparse='csc', copy=copy,
-                                dtype=FLOAT_DTYPES,
-                                force_all_finite='allow-nan')
+        X = validate_data(
+            self,
+            X,
+            reset=in_fit,
+            accept_sparse="csc",
+            copy=copy,
+            dtype=FLOAT_DTYPES,
+            # only set force_writeable for the validation at transform time because
+            # it's the only place where QuantileTransformer performs inplace operations.
+            force_writeable=True if not in_fit else None,
+            ensure_all_finite="allow-nan",
+        )
         # we only accept positive sparse matrix when ignore_implicit_zeros is
         # false and that we call fit or transform.
-        with np.errstate(invalid='ignore'):  # hide NaN comparison warnings
-            if (not accept_sparse_negative and not self.ignore_implicit_zeros
-                    and (sparse.issparse(X) and np.any(X.data < 0))):
-                raise ValueError('QuantileTransformer only accepts'
-                                 ' non-negative sparse matrices.')
-
-        # check the output distribution
-        if self.output_distribution not in ('normal', 'uniform'):
-            raise ValueError("'output_distribution' has to be either 'normal'"
-                             " or 'uniform'. Got '{}' instead.".format(
-                                 self.output_distribution))
+        with np.errstate(invalid="ignore"):  # hide NaN comparison warnings
+            if (
+                not accept_sparse_negative
+                and not self.ignore_implicit_zeros
+                and (sparse.issparse(X) and np.any(X.data < 0))
+            ):
+                raise ValueError(
+                    "QuantileTransformer only accepts non-negative sparse matrices."
+                )
 
         return X
 
@@ -2753,19 +2971,17 @@ def _transform(self, X, inverse=False):
         X : ndarray of shape (n_samples, n_features)
             Projected data.
         """
-
         if sparse.issparse(X):
             for feature_idx in range(X.shape[1]):
-                column_slice = slice(X.indptr[feature_idx],
-                                     X.indptr[feature_idx + 1])
+                column_slice = slice(X.indptr[feature_idx], X.indptr[feature_idx + 1])
                 X.data[column_slice] = self._transform_col(
-                    X.data[column_slice], self.quantiles_[:, feature_idx],
-                    inverse)
+                    X.data[column_slice], self.quantiles_[:, feature_idx], inverse
+                )
         else:
             for feature_idx in range(X.shape[1]):
                 X[:, feature_idx] = self._transform_col(
-                    X[:, feature_idx], self.quantiles_[:, feature_idx],
-                    inverse)
+                    X[:, feature_idx], self.quantiles_[:, feature_idx], inverse
+                )
 
         return X
 
@@ -2803,26 +3019,38 @@ def inverse_transform(self, X):
 
         Returns
         -------
-        Xt : {ndarray, sparse matrix} of (n_samples, n_features)
+        X_original : {ndarray, sparse matrix} of (n_samples, n_features)
             The projected data.
         """
         check_is_fitted(self)
-        X = self._check_inputs(X, in_fit=False, accept_sparse_negative=True,
-                               copy=self.copy)
+        X = self._check_inputs(
+            X, in_fit=False, accept_sparse_negative=True, copy=self.copy
+        )
 
         return self._transform(X, inverse=True)
 
-    def _more_tags(self):
-        return {'allow_nan': True}
-
-
-@_deprecate_positional_args
-def quantile_transform(X, *, axis=0, n_quantiles=1000,
-                       output_distribution='uniform',
-                       ignore_implicit_zeros=False,
-                       subsample=int(1e5),
-                       random_state=None,
-                       copy=True):
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
+        tags.input_tags.allow_nan = True
+        return tags
+
+
+@validate_params(
+    {"X": ["array-like", "sparse matrix"], "axis": [Options(Integral, {0, 1})]},
+    prefer_skip_nested_validation=False,
+)
+def quantile_transform(
+    X,
+    *,
+    axis=0,
+    n_quantiles=1000,
+    output_distribution="uniform",
+    ignore_implicit_zeros=False,
+    subsample=int(1e5),
+    random_state=None,
+    copy=True,
+):
     """Transform features using quantiles information.
 
     This method transforms the features to follow a uniform or a normal
@@ -2868,24 +3096,29 @@ def quantile_transform(X, *, axis=0, n_quantiles=1000,
         matrix are discarded to compute the quantile statistics. If False,
         these entries are treated as zeros.
 
-    subsample : int, default=1e5
+    subsample : int or None, default=1e5
         Maximum number of samples used to estimate the quantiles for
         computational efficiency. Note that the subsampling procedure may
         differ for value-identical sparse and dense matrices.
+        Disable subsampling by setting `subsample=None`.
+
+        .. versionadded:: 1.5
+           The option `None` to disable subsampling was added.
 
     random_state : int, RandomState instance or None, default=None
         Determines random number generation for subsampling and smoothing
         noise.
         Please see ``subsample`` for more details.
         Pass an int for reproducible results across multiple function calls.
-        See :term:`Glossary `
+        See :term:`Glossary `.
 
     copy : bool, default=True
-        Set to False to perform inplace transformation and avoid a copy (if the
-        input is already a numpy array). If True, a copy of `X` is transformed,
-        leaving the original `X` unchanged
+        If False, try to avoid a copy and transform in place.
+        This is not guaranteed to always work in place; e.g. if the data is
+        a numpy array with an int dtype, a copy will be returned even with
+        copy=False.
 
-        ..versionchanged:: 0.23
+        .. versionchanged:: 0.23
             The default value of `copy` changed from False to True in 0.23.
 
     Returns
@@ -2893,15 +3126,6 @@ def quantile_transform(X, *, axis=0, n_quantiles=1000,
     Xt : {ndarray, sparse matrix} of shape (n_samples, n_features)
         The transformed data.
 
-    Examples
-    --------
-    >>> import numpy as np
-    >>> from sklearn.preprocessing import quantile_transform
-    >>> rng = np.random.RandomState(0)
-    >>> X = np.sort(rng.normal(loc=0.5, scale=0.25, size=(25, 1)), axis=0)
-    >>> quantile_transform(X, n_quantiles=10, random_state=0, copy=True)
-    array([...])
-
     See Also
     --------
     QuantileTransformer : Performs quantile-based scaling using the
@@ -2934,25 +3158,33 @@ def quantile_transform(X, *, axis=0, n_quantiles=1000,
         LogisticRegression())`.
 
     For a comparison of the different scalers, transformers, and normalizers,
-    see :ref:`examples/preprocessing/plot_all_scaling.py
-    `.
+    see: :ref:`sphx_glr_auto_examples_preprocessing_plot_all_scaling.py`.
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from sklearn.preprocessing import quantile_transform
+    >>> rng = np.random.RandomState(0)
+    >>> X = np.sort(rng.normal(loc=0.5, scale=0.25, size=(25, 1)), axis=0)
+    >>> quantile_transform(X, n_quantiles=10, random_state=0, copy=True)
+    array([...])
     """
-    n = QuantileTransformer(n_quantiles=n_quantiles,
-                            output_distribution=output_distribution,
-                            subsample=subsample,
-                            ignore_implicit_zeros=ignore_implicit_zeros,
-                            random_state=random_state,
-                            copy=copy)
+    n = QuantileTransformer(
+        n_quantiles=n_quantiles,
+        output_distribution=output_distribution,
+        subsample=subsample,
+        ignore_implicit_zeros=ignore_implicit_zeros,
+        random_state=random_state,
+        copy=copy,
+    )
     if axis == 0:
-        return n.fit_transform(X)
-    elif axis == 1:
-        return n.fit_transform(X.T).T
-    else:
-        raise ValueError("axis should be either equal to 0 or 1. Got"
-                         " axis={}".format(axis))
+        X = n.fit_transform(X)
+    else:  # axis == 1
+        X = n.fit_transform(X.T).T
+    return X
 
 
-class PowerTransformer(TransformerMixin, BaseEstimator):
+class PowerTransformer(OneToOneFeatureMixin, TransformerMixin, BaseEstimator):
     """Apply a power transform featurewise to make data more Gaussian-like.
 
     Power transforms are a family of parametric, monotonic transformations
@@ -2970,6 +3202,12 @@ class PowerTransformer(TransformerMixin, BaseEstimator):
     By default, zero-mean, unit-variance normalization is applied to the
     transformed data.
 
+    For an example visualization, refer to :ref:`Compare PowerTransformer with
+    other scalers `. To see the
+    effect of Box-Cox and Yeo-Johnson transformations on different
+    distributions, see:
+    :ref:`sphx_glr_auto_examples_preprocessing_plot_map_data_to_normal.py`.
+
     Read more in the :ref:`User Guide `.
 
     .. versionadded:: 0.20
@@ -2994,20 +3232,16 @@ class PowerTransformer(TransformerMixin, BaseEstimator):
     lambdas_ : ndarray of float of shape (n_features,)
         The parameters of the power transformation for the selected features.
 
-    Examples
-    --------
-    >>> import numpy as np
-    >>> from sklearn.preprocessing import PowerTransformer
-    >>> pt = PowerTransformer()
-    >>> data = [[1, 2], [3, 2], [4, 5]]
-    >>> print(pt.fit(data))
-    PowerTransformer()
-    >>> print(pt.lambdas_)
-    [ 1.386... -3.100...]
-    >>> print(pt.transform(data))
-    [[-1.316... -0.707...]
-     [ 0.209... -0.707...]
-     [ 1.106...  1.414...]]
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
 
     See Also
     --------
@@ -3021,26 +3255,45 @@ class PowerTransformer(TransformerMixin, BaseEstimator):
     NaNs are treated as missing values: disregarded in ``fit``, and maintained
     in ``transform``.
 
-    For a comparison of the different scalers, transformers, and normalizers,
-    see :ref:`examples/preprocessing/plot_all_scaling.py
-    `.
-
     References
     ----------
 
-    .. [1] I.K. Yeo and R.A. Johnson, "A new family of power transformations to
-           improve normality or symmetry." Biometrika, 87(4), pp.954-959,
-           (2000).
+    .. [1] :doi:`I.K. Yeo and R.A. Johnson, "A new family of power
+           transformations to improve normality or symmetry." Biometrika,
+           87(4), pp.954-959, (2000). <10.1093/biomet/87.4.954>`
 
-    .. [2] G.E.P. Box and D.R. Cox, "An Analysis of Transformations", Journal
-           of the Royal Statistical Society B, 26, 211-252 (1964).
+    .. [2] :doi:`G.E.P. Box and D.R. Cox, "An Analysis of Transformations",
+           Journal of the Royal Statistical Society B, 26, 211-252 (1964).
+           <10.1111/j.2517-6161.1964.tb00553.x>`
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from sklearn.preprocessing import PowerTransformer
+    >>> pt = PowerTransformer()
+    >>> data = [[1, 2], [3, 2], [4, 5]]
+    >>> print(pt.fit(data))
+    PowerTransformer()
+    >>> print(pt.lambdas_)
+    [ 1.386 -3.100]
+    >>> print(pt.transform(data))
+    [[-1.316 -0.707]
+     [ 0.209 -0.707]
+     [ 1.106  1.414]]
     """
-    @_deprecate_positional_args
-    def __init__(self, method='yeo-johnson', *, standardize=True, copy=True):
+
+    _parameter_constraints: dict = {
+        "method": [StrOptions({"yeo-johnson", "box-cox"})],
+        "standardize": ["boolean"],
+        "copy": ["boolean"],
+    }
+
+    def __init__(self, method="yeo-johnson", *, standardize=True, copy=True):
         self.method = method
         self.standardize = standardize
         self.copy = copy
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y=None):
         """Estimate the optimal parameter lambda for each feature.
 
@@ -3063,32 +3316,63 @@ def fit(self, X, y=None):
         self._fit(X, y=y, force_transform=False)
         return self
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit_transform(self, X, y=None):
+        """Fit `PowerTransformer` to `X`, then transform `X`.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            The data used to estimate the optimal transformation parameters
+            and to be transformed using a power transformation.
+
+        y : Ignored
+            Not used, present for API consistency by convention.
+
+        Returns
+        -------
+        X_new : ndarray of shape (n_samples, n_features)
+            Transformed data.
+        """
         return self._fit(X, y, force_transform=True)
 
     def _fit(self, X, y=None, force_transform=False):
-        X = self._check_input(X, in_fit=True, check_positive=True,
-                              check_method=True)
+        X = self._check_input(X, in_fit=True, check_positive=True)
 
         if not self.copy and not force_transform:  # if call from fit()
             X = X.copy()  # force copy so that fit does not change X inplace
 
-        optim_function = {'box-cox': self._box_cox_optimize,
-                          'yeo-johnson': self._yeo_johnson_optimize
-                          }[self.method]
-        with np.errstate(invalid='ignore'):  # hide NaN warnings
-            self.lambdas_ = np.array([optim_function(col) for col in X.T])
-
-        if self.standardize or force_transform:
-            transform_function = {'box-cox': boxcox,
-                                  'yeo-johnson': self._yeo_johnson_transform
-                                  }[self.method]
-            for i, lmbda in enumerate(self.lambdas_):
-                with np.errstate(invalid='ignore'):  # hide NaN warnings
-                    X[:, i] = transform_function(X[:, i], lmbda)
+        n_samples = X.shape[0]
+        mean = np.mean(X, axis=0, dtype=np.float64)
+        var = np.var(X, axis=0, dtype=np.float64)
+
+        optim_function = {
+            "box-cox": self._box_cox_optimize,
+            "yeo-johnson": self._yeo_johnson_optimize,
+        }[self.method]
+
+        transform_function = {
+            "box-cox": boxcox,
+            "yeo-johnson": self._yeo_johnson_transform,
+        }[self.method]
+
+        with np.errstate(invalid="ignore"):  # hide NaN warnings
+            self.lambdas_ = np.empty(X.shape[1], dtype=X.dtype)
+            for i, col in enumerate(X.T):
+                # For yeo-johnson, leave constant features unchanged
+                # lambda=1 corresponds to the identity transformation
+                is_constant_feature = _is_constant_feature(var[i], mean[i], n_samples)
+                if self.method == "yeo-johnson" and is_constant_feature:
+                    self.lambdas_[i] = 1.0
+                    continue
+
+                self.lambdas_[i] = optim_function(col)
+
+                if self.standardize or force_transform:
+                    X[:, i] = transform_function(X[:, i], self.lambdas_[i])
 
         if self.standardize:
-            self._scaler = StandardScaler(copy=False)
+            self._scaler = StandardScaler(copy=False).set_output(transform="default")
             if force_transform:
                 X = self._scaler.fit_transform(X)
             else:
@@ -3110,14 +3394,14 @@ def transform(self, X):
             The transformed data.
         """
         check_is_fitted(self)
-        X = self._check_input(X, in_fit=False, check_positive=True,
-                              check_shape=True)
+        X = self._check_input(X, in_fit=False, check_positive=True, check_shape=True)
 
-        transform_function = {'box-cox': boxcox,
-                              'yeo-johnson': self._yeo_johnson_transform
-                              }[self.method]
+        transform_function = {
+            "box-cox": boxcox,
+            "yeo-johnson": self._yeo_johnson_transform,
+        }[self.method]
         for i, lmbda in enumerate(self.lambdas_):
-            with np.errstate(invalid='ignore'):  # hide NaN warnings
+            with np.errstate(invalid="ignore"):  # hide NaN warnings
                 X[:, i] = transform_function(X[:, i], lmbda)
 
         if self.standardize:
@@ -3131,20 +3415,20 @@ def inverse_transform(self, X):
         The inverse of the Box-Cox transformation is given by::
 
             if lambda_ == 0:
-                X = exp(X_trans)
+                X_original = exp(X_trans)
             else:
-                X = (X_trans * lambda_ + 1) ** (1 / lambda_)
+                X_original = (X * lambda_ + 1) ** (1 / lambda_)
 
         The inverse of the Yeo-Johnson transformation is given by::
 
             if X >= 0 and lambda_ == 0:
-                X = exp(X_trans) - 1
+                X_original = exp(X) - 1
             elif X >= 0 and lambda_ != 0:
-                X = (X_trans * lambda_ + 1) ** (1 / lambda_) - 1
+                X_original = (X * lambda_ + 1) ** (1 / lambda_) - 1
             elif X < 0 and lambda_ != 2:
-                X = 1 - (-(2 - lambda_) * X_trans + 1) ** (1 / (2 - lambda_))
+                X_original = 1 - (-(2 - lambda_) * X + 1) ** (1 / (2 - lambda_))
             elif X < 0 and lambda_ == 2:
-                X = 1 - exp(-X_trans)
+                X_original = 1 - exp(-X)
 
         Parameters
         ----------
@@ -3153,7 +3437,7 @@ def inverse_transform(self, X):
 
         Returns
         -------
-        X : ndarray of shape (n_samples, n_features)
+        X_original : ndarray of shape (n_samples, n_features)
             The original data.
         """
         check_is_fitted(self)
@@ -3162,26 +3446,16 @@ def inverse_transform(self, X):
         if self.standardize:
             X = self._scaler.inverse_transform(X)
 
-        inv_fun = {'box-cox': self._box_cox_inverse_tranform,
-                   'yeo-johnson': self._yeo_johnson_inverse_transform
-                   }[self.method]
+        inv_fun = {
+            "box-cox": inv_boxcox,
+            "yeo-johnson": self._yeo_johnson_inverse_transform,
+        }[self.method]
         for i, lmbda in enumerate(self.lambdas_):
-            with np.errstate(invalid='ignore'):  # hide NaN warnings
+            with np.errstate(invalid="ignore"):  # hide NaN warnings
                 X[:, i] = inv_fun(X[:, i], lmbda)
 
         return X
 
-    def _box_cox_inverse_tranform(self, x, lmbda):
-        """Return inverse-transformed input x following Box-Cox inverse
-        transform with parameter lambda.
-        """
-        if lmbda == 0:
-            x_inv = np.exp(x)
-        else:
-            x_inv = (x * lmbda + 1) ** (1 / lmbda)
-
-        return x_inv
-
     def _yeo_johnson_inverse_transform(self, x, lmbda):
         """Return inverse-transformed input x following Yeo-Johnson inverse
         transform with parameter lambda.
@@ -3190,15 +3464,14 @@ def _yeo_johnson_inverse_transform(self, x, lmbda):
         pos = x >= 0
 
         # when x >= 0
-        if abs(lmbda) < np.spacing(1.):
+        if abs(lmbda) < np.spacing(1.0):
             x_inv[pos] = np.exp(x[pos]) - 1
         else:  # lmbda != 0
             x_inv[pos] = np.power(x[pos] * lmbda + 1, 1 / lmbda) - 1
 
         # when x < 0
-        if abs(lmbda - 2) > np.spacing(1.):
-            x_inv[~pos] = 1 - np.power(-(2 - lmbda) * x[~pos] + 1,
-                                       1 / (2 - lmbda))
+        if abs(lmbda - 2) > np.spacing(1.0):
+            x_inv[~pos] = 1 - np.power(-(2 - lmbda) * x[~pos] + 1, 1 / (2 - lmbda))
         else:  # lmbda == 2
             x_inv[~pos] = 1 - np.exp(-x[~pos])
 
@@ -3213,13 +3486,13 @@ def _yeo_johnson_transform(self, x, lmbda):
         pos = x >= 0  # binary mask
 
         # when x >= 0
-        if abs(lmbda) < np.spacing(1.):
+        if abs(lmbda) < np.spacing(1.0):
             out[pos] = np.log1p(x[pos])
         else:  # lmbda != 0
             out[pos] = (np.power(x[pos] + 1, lmbda) - 1) / lmbda
 
         # when x < 0
-        if abs(lmbda - 2) > np.spacing(1.):
+        if abs(lmbda - 2) > np.spacing(1.0):
             out[~pos] = -(np.power(-x[~pos] + 1, 2 - lmbda) - 1) / (2 - lmbda)
         else:  # lmbda == 2
             out[~pos] = -np.log1p(-x[~pos])
@@ -3232,9 +3505,13 @@ def _box_cox_optimize(self, x):
 
         We here use scipy builtins which uses the brent optimizer.
         """
+        mask = np.isnan(x)
+        if np.all(mask):
+            raise ValueError("Column must not be all nan.")
+
         # the computation of lambda is influenced by NaNs so we need to
         # get rid of them
-        _, lmbda = stats.boxcox(x[~np.isnan(x)], lmbda=None)
+        _, lmbda = stats.boxcox(x[~mask], lmbda=None)
 
         return lmbda
 
@@ -3244,14 +3521,21 @@ def _yeo_johnson_optimize(self, x):
 
         Like for Box-Cox, MLE is done via the brent optimizer.
         """
+        x_tiny = np.finfo(np.float64).tiny
 
         def _neg_log_likelihood(lmbda):
             """Return the negative log likelihood of the observed data x as a
             function of lambda."""
             x_trans = self._yeo_johnson_transform(x, lmbda)
             n_samples = x.shape[0]
+            x_trans_var = x_trans.var()
 
-            loglike = -n_samples / 2 * np.log(x_trans.var())
+            # Reject transformed data that would raise a RuntimeWarning in np.log
+            if x_trans_var < x_tiny:
+                return np.inf
+
+            log_var = np.log(x_trans_var)
+            loglike = -n_samples / 2 * log_var
             loglike += (lmbda - 1) * (np.sign(x) * np.log1p(np.abs(x))).sum()
 
             return -loglike
@@ -3259,11 +3543,10 @@ def _neg_log_likelihood(lmbda):
         # the computation of lambda is influenced by NaNs so we need to
         # get rid of them
         x = x[~np.isnan(x)]
-        # choosing bracket -2, 2 like for boxcox
-        return optimize.brent(_neg_log_likelihood, brack=(-2, 2))
 
-    def _check_input(self, X, in_fit, check_positive=False, check_shape=False,
-                     check_method=False):
+        return _yeojohnson_lambda(_neg_log_likelihood, x)
+
+    def _check_input(self, X, in_fit, check_positive=False, check_shape=False):
         """Validate the input before fit and transform.
 
         Parameters
@@ -3280,42 +3563,49 @@ def _check_input(self, X, in_fit, check_positive=False, check_shape=False,
 
         check_shape : bool, default=False
             If True, check that n_features matches the length of self.lambdas_
-
-        check_method : bool, default=False
-            If True, check that the transformation method is valid.
         """
-        X = self._validate_data(X, ensure_2d=True, dtype=FLOAT_DTYPES,
-                                copy=self.copy, force_all_finite='allow-nan',
-                                reset=in_fit)
-
-        with np.warnings.catch_warnings():
-            np.warnings.filterwarnings(
-                'ignore', r'All-NaN (slice|axis) encountered')
-            if (check_positive and self.method == 'box-cox' and
-                    np.nanmin(X) <= 0):
-                raise ValueError("The Box-Cox transformation can only be "
-                                 "applied to strictly positive data")
+        X = validate_data(
+            self,
+            X,
+            ensure_2d=True,
+            dtype=FLOAT_DTYPES,
+            force_writeable=True,
+            copy=self.copy,
+            ensure_all_finite="allow-nan",
+            reset=in_fit,
+        )
+
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", r"All-NaN (slice|axis) encountered")
+            if check_positive and self.method == "box-cox" and np.nanmin(X) <= 0:
+                raise ValueError(
+                    "The Box-Cox transformation can only be "
+                    "applied to strictly positive data"
+                )
 
         if check_shape and not X.shape[1] == len(self.lambdas_):
-            raise ValueError("Input data has a different number of features "
-                             "than fitting data. Should have {n}, data has {m}"
-                             .format(n=len(self.lambdas_), m=X.shape[1]))
-
-        valid_methods = ('box-cox', 'yeo-johnson')
-        if check_method and self.method not in valid_methods:
-            raise ValueError("'method' must be one of {}, "
-                             "got {} instead."
-                             .format(valid_methods, self.method))
+            raise ValueError(
+                "Input data has a different number of features "
+                "than fitting data. Should have {n}, data has {m}".format(
+                    n=len(self.lambdas_), m=X.shape[1]
+                )
+            )
 
         return X
 
-    def _more_tags(self):
-        return {'allow_nan': True}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.allow_nan = True
+        return tags
 
 
-@_deprecate_positional_args
-def power_transform(X, method='yeo-johnson', *, standardize=True, copy=True):
-    """
+@validate_params(
+    {"X": ["array-like"]},
+    prefer_skip_nested_validation=False,
+)
+def power_transform(X, method="yeo-johnson", *, standardize=True, copy=True):
+    """Parametric, monotonic transformation to make data more Gaussian-like.
+
     Power transforms are a family of parametric, monotonic transformations
     that are applied to make data more Gaussian-like. This is useful for
     modeling issues related to heteroscedasticity (non-constant variance),
@@ -3333,7 +3623,6 @@ def power_transform(X, method='yeo-johnson', *, standardize=True, copy=True):
 
     Read more in the :ref:`User Guide `.
 
-
     Parameters
     ----------
     X : array-like of shape (n_samples, n_features)
@@ -3354,35 +3643,16 @@ def power_transform(X, method='yeo-johnson', *, standardize=True, copy=True):
         transformed output.
 
     copy : bool, default=True
-        Set to False to perform inplace computation during transformation.
+        If False, try to avoid a copy and transform in place.
+        This is not guaranteed to always work in place; e.g. if the data is
+        a numpy array with an int dtype, a copy will be returned even with
+        copy=False.
 
     Returns
     -------
     X_trans : ndarray of shape (n_samples, n_features)
         The transformed data.
 
-    Examples
-    --------
-    >>> import numpy as np
-    >>> from sklearn.preprocessing import power_transform
-    >>> data = [[1, 2], [3, 2], [4, 5]]
-    >>> print(power_transform(data, method='box-cox'))
-    [[-1.332... -0.707...]
-     [ 0.256... -0.707...]
-     [ 1.076...  1.414...]]
-
-    .. warning:: Risk of data leak.
-        Do not use :func:`~sklearn.preprocessing.power_transform` unless you
-        know what you are doing. A common mistake is to apply it to the entire
-        data *before* splitting into training and test sets. This will bias the
-        model evaluation because information would have leaked from the test
-        set to the training set.
-        In general, we recommend using
-        :class:`~sklearn.preprocessing.PowerTransformer` within a
-        :ref:`Pipeline ` in order to prevent most risks of data
-        leaking, e.g.: `pipe = make_pipeline(PowerTransformer(),
-        LogisticRegression())`.
-
     See Also
     --------
     PowerTransformer : Equivalent transformation with the
@@ -3398,8 +3668,7 @@ def power_transform(X, method='yeo-johnson', *, standardize=True, copy=True):
     in ``transform``.
 
     For a comparison of the different scalers, transformers, and normalizers,
-    see :ref:`examples/preprocessing/plot_all_scaling.py
-    `.
+    see: :ref:`sphx_glr_auto_examples_preprocessing_plot_all_scaling.py`.
 
     References
     ----------
@@ -3410,6 +3679,28 @@ def power_transform(X, method='yeo-johnson', *, standardize=True, copy=True):
 
     .. [2] G.E.P. Box and D.R. Cox, "An Analysis of Transformations", Journal
            of the Royal Statistical Society B, 26, 211-252 (1964).
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from sklearn.preprocessing import power_transform
+    >>> data = [[1, 2], [3, 2], [4, 5]]
+    >>> print(power_transform(data, method='box-cox'))
+    [[-1.332 -0.707]
+     [ 0.256 -0.707]
+     [ 1.076  1.414]]
+
+    .. warning:: Risk of data leak.
+        Do not use :func:`~sklearn.preprocessing.power_transform` unless you
+        know what you are doing. A common mistake is to apply it to the entire
+        data *before* splitting into training and test sets. This will bias the
+        model evaluation because information would have leaked from the test
+        set to the training set.
+        In general, we recommend using
+        :class:`~sklearn.preprocessing.PowerTransformer` within a
+        :ref:`Pipeline ` in order to prevent most risks of data
+        leaking, e.g.: `pipe = make_pipeline(PowerTransformer(),
+        LogisticRegression())`.
     """
     pt = PowerTransformer(method=method, standardize=standardize, copy=copy)
     return pt.fit_transform(X)
diff --git a/sklearn/preprocessing/_discretization.py b/sklearn/preprocessing/_discretization.py
index 22fa236f3314e..ef5081080bda1 100644
--- a/sklearn/preprocessing/_discretization.py
+++ b/sklearn/preprocessing/_discretization.py
@@ -1,21 +1,24 @@
-# -*- coding: utf-8 -*-
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-# Author: Henry Lin 
-#         Tom DuprÊ la Tour
 
-# License: BSD
-
-
-import numbers
-import numpy as np
 import warnings
+from numbers import Integral
 
-from . import OneHotEncoder
+import numpy as np
 
-from ..base import BaseEstimator, TransformerMixin
-from ..utils.validation import check_array
-from ..utils.validation import check_is_fitted
-from ..utils.validation import _deprecate_positional_args
+from ..base import BaseEstimator, TransformerMixin, _fit_context
+from ..utils import resample
+from ..utils._param_validation import Interval, Options, StrOptions
+from ..utils.stats import _averaged_weighted_percentile, _weighted_percentile
+from ..utils.validation import (
+    _check_feature_names_in,
+    _check_sample_weight,
+    check_array,
+    check_is_fitted,
+    validate_data,
+)
+from ._encoders import OneHotEncoder
 
 
 class KBinsDiscretizer(TransformerMixin, BaseEstimator):
@@ -34,27 +37,35 @@ class KBinsDiscretizer(TransformerMixin, BaseEstimator):
     encode : {'onehot', 'onehot-dense', 'ordinal'}, default='onehot'
         Method used to encode the transformed result.
 
-        onehot
-            Encode the transformed result with one-hot encoding
-            and return a sparse matrix. Ignored features are always
-            stacked to the right.
-        onehot-dense
-            Encode the transformed result with one-hot encoding
-            and return a dense array. Ignored features are always
-            stacked to the right.
-        ordinal
-            Return the bin identifier encoded as an integer value.
+        - 'onehot': Encode the transformed result with one-hot encoding
+          and return a sparse matrix. Ignored features are always
+          stacked to the right.
+        - 'onehot-dense': Encode the transformed result with one-hot encoding
+          and return a dense array. Ignored features are always
+          stacked to the right.
+        - 'ordinal': Return the bin identifier encoded as an integer value.
 
     strategy : {'uniform', 'quantile', 'kmeans'}, default='quantile'
         Strategy used to define the widths of the bins.
 
-        uniform
-            All bins in each feature have identical widths.
-        quantile
-            All bins in each feature have the same number of points.
-        kmeans
-            Values in each bin have the same nearest center of a 1D k-means
-            cluster.
+        - 'uniform': All bins in each feature have identical widths.
+        - 'quantile': All bins in each feature have the same number of points.
+        - 'kmeans': Values in each bin have the same nearest center of a 1D
+          k-means cluster.
+
+        For an example of the different strategies see:
+        :ref:`sphx_glr_auto_examples_preprocessing_plot_discretization_strategies.py`.
+
+    quantile_method : {"inverted_cdf", "averaged_inverted_cdf",
+            "closest_observation", "interpolated_inverted_cdf", "hazen",
+            "weibull", "linear", "median_unbiased", "normal_unbiased"},
+            default="linear"
+            Method to pass on to np.percentile calculation when using
+            strategy="quantile". Only `averaged_inverted_cdf` and `inverted_cdf`
+            support the use of `sample_weight != None` when subsampling is not
+            active.
+
+            .. versionadded:: 1.7
 
     dtype : {np.float32, np.float64}, default=None
         The desired data-type for the output. If None, output dtype is
@@ -63,16 +74,53 @@ class KBinsDiscretizer(TransformerMixin, BaseEstimator):
 
         .. versionadded:: 0.24
 
+    subsample : int or None, default=200_000
+        Maximum number of samples, used to fit the model, for computational
+        efficiency.
+        `subsample=None` means that all the training samples are used when
+        computing the quantiles that determine the binning thresholds.
+        Since quantile computation relies on sorting each column of `X` and
+        that sorting has an `n log(n)` time complexity,
+        it is recommended to use subsampling on datasets with a
+        very large number of samples.
+
+        .. versionchanged:: 1.3
+            The default value of `subsample` changed from `None` to `200_000` when
+            `strategy="quantile"`.
+
+        .. versionchanged:: 1.5
+            The default value of `subsample` changed from `None` to `200_000` when
+            `strategy="uniform"` or `strategy="kmeans"`.
+
+    random_state : int, RandomState instance or None, default=None
+        Determines random number generation for subsampling.
+        Pass an int for reproducible results across multiple function calls.
+        See the `subsample` parameter for more details.
+        See :term:`Glossary `.
+
+        .. versionadded:: 1.1
+
     Attributes
     ----------
-    n_bins_ : ndarray of shape (n_features,), dtype=np.int_
-        Number of bins per feature. Bins whose width are too small
-        (i.e., <= 1e-8) are removed with a warning.
-
     bin_edges_ : ndarray of ndarray of shape (n_features,)
         The edges of each bin. Contain arrays of varying shapes ``(n_bins_, )``
         Ignored features will have empty arrays.
 
+    n_bins_ : ndarray of shape (n_features,), dtype=np.int64
+        Number of bins per feature. Bins whose width are too small
+        (i.e., <= 1e-8) are removed with a warning.
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     See Also
     --------
     Binarizer : Class used to bin values as ``0`` or
@@ -80,6 +128,12 @@ class KBinsDiscretizer(TransformerMixin, BaseEstimator):
 
     Notes
     -----
+
+    For a visualization of discretization on different datasets refer to
+    :ref:`sphx_glr_auto_examples_preprocessing_plot_discretization_classification.py`.
+    On the effect of discretization on linear models see:
+    :ref:`sphx_glr_auto_examples_preprocessing_plot_discretization.py`.
+
     In bin edges for feature ``i``, the first and last values are used only for
     ``inverse_transform``. During transform, bin edges are extended to::
 
@@ -96,11 +150,14 @@ class KBinsDiscretizer(TransformerMixin, BaseEstimator):
 
     Examples
     --------
+    >>> from sklearn.preprocessing import KBinsDiscretizer
     >>> X = [[-2, 1, -4,   -1],
     ...      [-1, 2, -3, -0.5],
     ...      [ 0, 3, -2,  0.5],
     ...      [ 1, 4, -1,    2]]
-    >>> est = KBinsDiscretizer(n_bins=3, encode='ordinal', strategy='uniform')
+    >>> est = KBinsDiscretizer(
+    ...     n_bins=3, encode='ordinal', strategy='uniform'
+    ... )
     >>> est.fit(X)
     KBinsDiscretizer(...)
     >>> Xt = est.transform(X)
@@ -122,18 +179,54 @@ class KBinsDiscretizer(TransformerMixin, BaseEstimator):
            [-0.5,  2.5, -2.5, -0.5],
            [ 0.5,  3.5, -1.5,  0.5],
            [ 0.5,  3.5, -1.5,  1.5]])
-
     """
 
-    @_deprecate_positional_args
-    def __init__(self, n_bins=5, *, encode='onehot', strategy='quantile',
-                 dtype=None):
+    _parameter_constraints: dict = {
+        "n_bins": [Interval(Integral, 2, None, closed="left"), "array-like"],
+        "encode": [StrOptions({"onehot", "onehot-dense", "ordinal"})],
+        "strategy": [StrOptions({"uniform", "quantile", "kmeans"})],
+        "quantile_method": [
+            StrOptions(
+                {
+                    "warn",
+                    "inverted_cdf",
+                    "averaged_inverted_cdf",
+                    "closest_observation",
+                    "interpolated_inverted_cdf",
+                    "hazen",
+                    "weibull",
+                    "linear",
+                    "median_unbiased",
+                    "normal_unbiased",
+                }
+            )
+        ],
+        "dtype": [Options(type, {np.float64, np.float32}), None],
+        "subsample": [Interval(Integral, 1, None, closed="left"), None],
+        "random_state": ["random_state"],
+    }
+
+    def __init__(
+        self,
+        n_bins=5,
+        *,
+        encode="onehot",
+        strategy="quantile",
+        quantile_method="warn",
+        dtype=None,
+        subsample=200_000,
+        random_state=None,
+    ):
         self.n_bins = n_bins
         self.encode = encode
         self.strategy = strategy
+        self.quantile_method = quantile_method
         self.dtype = dtype
+        self.subsample = subsample
+        self.random_state = random_state
 
-    def fit(self, X, y=None):
+    @_fit_context(prefer_skip_nested_validation=True)
+    def fit(self, X, y=None, sample_weight=None):
         """
         Fit the estimator.
 
@@ -146,58 +239,140 @@ def fit(self, X, y=None):
             Ignored. This parameter exists only for compatibility with
             :class:`~sklearn.pipeline.Pipeline`.
 
+        sample_weight : ndarray of shape (n_samples,)
+            Contains weight values to be associated with each sample.
+
+            .. versionadded:: 1.3
+
+            .. versionchanged:: 1.7
+               Added support for strategy="uniform".
+
         Returns
         -------
-        self
+        self : object
+            Returns the instance itself.
         """
-        X = self._validate_data(X, dtype='numeric')
+        X = validate_data(self, X, dtype="numeric")
 
-        supported_dtype = (np.float64, np.float32)
-        if self.dtype in supported_dtype:
+        if self.dtype in (np.float64, np.float32):
             output_dtype = self.dtype
-        elif self.dtype is None:
+        else:  # self.dtype is None
             output_dtype = X.dtype
-        else:
-            raise ValueError(
-                f"Valid options for 'dtype' are "
-                f"{supported_dtype + (None,)}. Got dtype={self.dtype} "
-                f" instead."
-            )
 
-        valid_encode = ('onehot', 'onehot-dense', 'ordinal')
-        if self.encode not in valid_encode:
-            raise ValueError("Valid options for 'encode' are {}. "
-                             "Got encode={!r} instead."
-                             .format(valid_encode, self.encode))
-        valid_strategy = ('uniform', 'quantile', 'kmeans')
-        if self.strategy not in valid_strategy:
-            raise ValueError("Valid options for 'strategy' are {}. "
-                             "Got strategy={!r} instead."
-                             .format(valid_strategy, self.strategy))
+        n_samples, n_features = X.shape
+
+        if sample_weight is not None:
+            sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype)
+
+        if self.subsample is not None and n_samples > self.subsample:
+            # Take a subsample of `X`
+            # When resampling, it is important to subsample **with replacement** to
+            # preserve the distribution, in particular in the presence of a few data
+            # points with large weights. You can check this by setting `replace=False`
+            # in sklearn.utils.test.test_indexing.test_resample_weighted and check that
+            # it fails as a justification for this claim.
+            X = resample(
+                X,
+                replace=True,
+                n_samples=self.subsample,
+                random_state=self.random_state,
+                sample_weight=sample_weight,
+            )
+            # Since we already used the weights when resampling when provided,
+            # we set them back to `None` to avoid accounting for the weights twice
+            # in subsequent operations to compute weight-aware bin edges with
+            # quantiles or k-means.
+            sample_weight = None
 
         n_features = X.shape[1]
         n_bins = self._validate_n_bins(n_features)
 
         bin_edges = np.zeros(n_features, dtype=object)
+
+        # TODO(1.9): remove and switch to quantile_method="averaged_inverted_cdf"
+        # by default.
+        quantile_method = self.quantile_method
+        if self.strategy == "quantile" and quantile_method == "warn":
+            warnings.warn(
+                "The current default behavior, quantile_method='linear', will be "
+                "changed to quantile_method='averaged_inverted_cdf' in "
+                "scikit-learn version 1.9 to naturally support sample weight "
+                "equivalence properties by default. Pass "
+                "quantile_method='averaged_inverted_cdf' explicitly to silence this "
+                "warning.",
+                FutureWarning,
+            )
+            quantile_method = "linear"
+
+        if (
+            self.strategy == "quantile"
+            and quantile_method not in ["inverted_cdf", "averaged_inverted_cdf"]
+            and sample_weight is not None
+        ):
+            raise ValueError(
+                "When fitting with strategy='quantile' and sample weights, "
+                "quantile_method should either be set to 'averaged_inverted_cdf' or "
+                f"'inverted_cdf', got quantile_method='{quantile_method}' instead."
+            )
+
+        if self.strategy != "quantile" and sample_weight is not None:
+            # Prepare a mask to filter out zero-weight samples when extracting
+            # the min and max values of each columns which are needed for the
+            # "uniform" and "kmeans" strategies.
+            nnz_weight_mask = sample_weight != 0
+        else:
+            # Otherwise, all samples are used. Use a slice to avoid creating a
+            # new array.
+            nnz_weight_mask = slice(None)
+
         for jj in range(n_features):
             column = X[:, jj]
-            col_min, col_max = column.min(), column.max()
+            col_min = column[nnz_weight_mask].min()
+            col_max = column[nnz_weight_mask].max()
 
             if col_min == col_max:
-                warnings.warn("Feature %d is constant and will be "
-                              "replaced with 0." % jj)
+                warnings.warn(
+                    "Feature %d is constant and will be replaced with 0." % jj
+                )
                 n_bins[jj] = 1
                 bin_edges[jj] = np.array([-np.inf, np.inf])
                 continue
 
-            if self.strategy == 'uniform':
+            if self.strategy == "uniform":
                 bin_edges[jj] = np.linspace(col_min, col_max, n_bins[jj] + 1)
 
-            elif self.strategy == 'quantile':
-                quantiles = np.linspace(0, 100, n_bins[jj] + 1)
-                bin_edges[jj] = np.asarray(np.percentile(column, quantiles))
-
-            elif self.strategy == 'kmeans':
+            elif self.strategy == "quantile":
+                percentile_levels = np.linspace(0, 100, n_bins[jj] + 1)
+
+                # method="linear" is the implicit default for any numpy
+                # version. So we keep it version independent in that case by
+                # using an empty param dict.
+                percentile_kwargs = {}
+                if quantile_method != "linear" and sample_weight is None:
+                    percentile_kwargs["method"] = quantile_method
+
+                if sample_weight is None:
+                    bin_edges[jj] = np.asarray(
+                        np.percentile(column, percentile_levels, **percentile_kwargs),
+                        dtype=np.float64,
+                    )
+                else:
+                    # TODO: make _weighted_percentile and
+                    # _averaged_weighted_percentile accept an array of
+                    # quantiles instead of calling it multiple times and
+                    # sorting the column multiple times as a result.
+                    percentile_func = {
+                        "inverted_cdf": _weighted_percentile,
+                        "averaged_inverted_cdf": _averaged_weighted_percentile,
+                    }[quantile_method]
+                    bin_edges[jj] = np.asarray(
+                        [
+                            percentile_func(column, sample_weight, percentile_rank=p)
+                            for p in percentile_levels
+                        ],
+                        dtype=np.float64,
+                    )
+            elif self.strategy == "kmeans":
                 from ..cluster import KMeans  # fixes import loops
 
                 # Deterministic initialization with uniform spacing
@@ -206,30 +381,35 @@ def fit(self, X, y=None):
 
                 # 1D k-means procedure
                 km = KMeans(n_clusters=n_bins[jj], init=init, n_init=1)
-                centers = km.fit(column[:, None]).cluster_centers_[:, 0]
+                centers = km.fit(
+                    column[:, None], sample_weight=sample_weight
+                ).cluster_centers_[:, 0]
                 # Must sort, centers may be unsorted even with sorted init
                 centers.sort()
                 bin_edges[jj] = (centers[1:] + centers[:-1]) * 0.5
                 bin_edges[jj] = np.r_[col_min, bin_edges[jj], col_max]
 
             # Remove bins whose width are too small (i.e., <= 1e-8)
-            if self.strategy in ('quantile', 'kmeans'):
+            if self.strategy in ("quantile", "kmeans"):
                 mask = np.ediff1d(bin_edges[jj], to_begin=np.inf) > 1e-8
                 bin_edges[jj] = bin_edges[jj][mask]
                 if len(bin_edges[jj]) - 1 != n_bins[jj]:
-                    warnings.warn('Bins whose width are too small (i.e., <= '
-                                  '1e-8) in feature %d are removed. Consider '
-                                  'decreasing the number of bins.' % jj)
+                    warnings.warn(
+                        "Bins whose width are too small (i.e., <= "
+                        "1e-8) in feature %d are removed. Consider "
+                        "decreasing the number of bins." % jj
+                    )
                     n_bins[jj] = len(bin_edges[jj]) - 1
 
         self.bin_edges_ = bin_edges
         self.n_bins_ = n_bins
 
-        if 'onehot' in self.encode:
+        if "onehot" in self.encode:
             self._encoder = OneHotEncoder(
                 categories=[np.arange(i) for i in self.n_bins_],
-                sparse=self.encode == 'onehot',
-                dtype=output_dtype)
+                sparse_output=self.encode == "onehot",
+                dtype=output_dtype,
+            )
             # Fit the OneHotEncoder with toy datasets
             # so that it's ready for use after the KBinsDiscretizer is fitted
             self._encoder.fit(np.zeros((1, len(self.n_bins_))))
@@ -237,37 +417,28 @@ def fit(self, X, y=None):
         return self
 
     def _validate_n_bins(self, n_features):
-        """Returns n_bins_, the number of bins per feature.
-        """
+        """Returns n_bins_, the number of bins per feature."""
         orig_bins = self.n_bins
-        if isinstance(orig_bins, numbers.Number):
-            if not isinstance(orig_bins, numbers.Integral):
-                raise ValueError("{} received an invalid n_bins type. "
-                                 "Received {}, expected int."
-                                 .format(KBinsDiscretizer.__name__,
-                                         type(orig_bins).__name__))
-            if orig_bins < 2:
-                raise ValueError("{} received an invalid number "
-                                 "of bins. Received {}, expected at least 2."
-                                 .format(KBinsDiscretizer.__name__, orig_bins))
+        if isinstance(orig_bins, Integral):
             return np.full(n_features, orig_bins, dtype=int)
 
-        n_bins = check_array(orig_bins, dtype=int, copy=True,
-                             ensure_2d=False)
+        n_bins = check_array(orig_bins, dtype=int, copy=True, ensure_2d=False)
 
         if n_bins.ndim > 1 or n_bins.shape[0] != n_features:
-            raise ValueError("n_bins must be a scalar or array "
-                             "of shape (n_features,).")
+            raise ValueError("n_bins must be a scalar or array of shape (n_features,).")
 
         bad_nbins_value = (n_bins < 2) | (n_bins != orig_bins)
 
         violating_indices = np.where(bad_nbins_value)[0]
         if violating_indices.shape[0] > 0:
             indices = ", ".join(str(i) for i in violating_indices)
-            raise ValueError("{} received an invalid number "
-                             "of bins at indices {}. Number of bins "
-                             "must be at least 2, and must be an int."
-                             .format(KBinsDiscretizer.__name__, indices))
+            raise ValueError(
+                "{} received an invalid number "
+                "of bins at indices {}. Number of bins "
+                "must be at least 2, and must be an int.".format(
+                    KBinsDiscretizer.__name__, indices
+                )
+            )
         return n_bins
 
     def transform(self, X):
@@ -289,25 +460,17 @@ def transform(self, X):
 
         # check input and attribute dtypes
         dtype = (np.float64, np.float32) if self.dtype is None else self.dtype
-        Xt = self._validate_data(X, copy=True, dtype=dtype, reset=False)
+        Xt = validate_data(self, X, copy=True, dtype=dtype, reset=False)
 
         bin_edges = self.bin_edges_
         for jj in range(Xt.shape[1]):
-            # Values which are close to a bin edge are susceptible to numeric
-            # instability. Add eps to X so these values are binned correctly
-            # with respect to their decimal truncation. See documentation of
-            # numpy.isclose for an explanation of ``rtol`` and ``atol``.
-            rtol = 1.e-5
-            atol = 1.e-8
-            eps = atol + rtol * np.abs(Xt[:, jj])
-            Xt[:, jj] = np.digitize(Xt[:, jj] + eps, bin_edges[jj][1:])
-        np.clip(Xt, 0, self.n_bins_ - 1, out=Xt)
-
-        if self.encode == 'ordinal':
+            Xt[:, jj] = np.searchsorted(bin_edges[jj][1:-1], Xt[:, jj], side="right")
+
+        if self.encode == "ordinal":
             return Xt
 
         dtype_init = None
-        if 'onehot' in self.encode:
+        if "onehot" in self.encode:
             dtype_init = self._encoder.dtype
             self._encoder.dtype = Xt.dtype
         try:
@@ -317,7 +480,7 @@ def transform(self, X):
             self._encoder.dtype = dtype_init
         return Xt_enc
 
-    def inverse_transform(self, Xt):
+    def inverse_transform(self, X):
         """
         Transform discretized data back to original feature space.
 
@@ -326,28 +489,60 @@ def inverse_transform(self, Xt):
 
         Parameters
         ----------
-        Xt : array-like of shape (n_samples, n_features)
+        X : array-like of shape (n_samples, n_features)
             Transformed data in the binned space.
 
         Returns
         -------
-        Xinv : ndarray, dtype={np.float32, np.float64}
+        X_original : ndarray, dtype={np.float32, np.float64}
             Data in the original feature space.
         """
+
         check_is_fitted(self)
 
-        if 'onehot' in self.encode:
-            Xt = self._encoder.inverse_transform(Xt)
+        if "onehot" in self.encode:
+            X = self._encoder.inverse_transform(X)
 
-        Xinv = check_array(Xt, copy=True, dtype=(np.float64, np.float32))
+        Xinv = check_array(X, copy=True, dtype=(np.float64, np.float32))
         n_features = self.n_bins_.shape[0]
         if Xinv.shape[1] != n_features:
-            raise ValueError("Incorrect number of features. Expecting {}, "
-                             "received {}.".format(n_features, Xinv.shape[1]))
+            raise ValueError(
+                "Incorrect number of features. Expecting {}, received {}.".format(
+                    n_features, Xinv.shape[1]
+                )
+            )
 
         for jj in range(n_features):
             bin_edges = self.bin_edges_[jj]
             bin_centers = (bin_edges[1:] + bin_edges[:-1]) * 0.5
-            Xinv[:, jj] = bin_centers[np.int_(Xinv[:, jj])]
+            Xinv[:, jj] = bin_centers[(Xinv[:, jj]).astype(np.int64)]
 
         return Xinv
+
+    def get_feature_names_out(self, input_features=None):
+        """Get output feature names.
+
+        Parameters
+        ----------
+        input_features : array-like of str or None, default=None
+            Input features.
+
+            - If `input_features` is `None`, then `feature_names_in_` is
+              used as feature names in. If `feature_names_in_` is not defined,
+              then the following input feature names are generated:
+              `["x0", "x1", ..., "x(n_features_in_ - 1)"]`.
+            - If `input_features` is an array-like, then `input_features` must
+              match `feature_names_in_` if `feature_names_in_` is defined.
+
+        Returns
+        -------
+        feature_names_out : ndarray of str objects
+            Transformed feature names.
+        """
+        check_is_fitted(self, "n_features_in_")
+        input_features = _check_feature_names_in(self, input_features)
+        if hasattr(self, "_encoder"):
+            return self._encoder.get_feature_names_out(input_features)
+
+        # ordinal encoding
+        return input_features
diff --git a/sklearn/preprocessing/_encoders.py b/sklearn/preprocessing/_encoders.py
index 043f9fc40ef53..5f41c9d0c6d22 100644
--- a/sklearn/preprocessing/_encoders.py
+++ b/sklearn/preprocessing/_encoders.py
@@ -1,24 +1,28 @@
-# Authors: Andreas Mueller 
-#          Joris Van den Bossche 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+import numbers
+import warnings
+from numbers import Integral
 
 import numpy as np
 from scipy import sparse
-import numbers
 
-from ..base import BaseEstimator, TransformerMixin
-from ..utils import check_array, is_scalar_nan
-from ..utils.validation import check_is_fitted
-from ..utils.validation import _deprecate_positional_args
+from ..base import BaseEstimator, OneToOneFeatureMixin, TransformerMixin, _fit_context
+from ..utils import _safe_indexing, check_array
+from ..utils._encode import _check_unknown, _encode, _get_counts, _unique
 from ..utils._mask import _get_mask
+from ..utils._missing import is_scalar_nan
+from ..utils._param_validation import Interval, RealNotInt, StrOptions
+from ..utils._set_output import _get_output_config
+from ..utils.validation import (
+    _check_feature_names,
+    _check_feature_names_in,
+    _check_n_features,
+    check_is_fitted,
+)
 
-from ..utils._encode import _encode, _check_unknown, _unique
-
-
-__all__ = [
-    'OneHotEncoder',
-    'OrdinalEncoder'
-]
+__all__ = ["OneHotEncoder", "OrdinalEncoder"]
 
 
 class _BaseEncoder(TransformerMixin, BaseEstimator):
@@ -28,7 +32,7 @@ class _BaseEncoder(TransformerMixin, BaseEstimator):
 
     """
 
-    def _check_X(self, X, force_all_finite=True):
+    def _check_X(self, X, ensure_all_finite=True):
         """
         Perform custom check_array:
         - convert list of strings to object dtype
@@ -37,127 +41,425 @@ def _check_X(self, X, force_all_finite=True):
         - return list of features (arrays): this list of features is
           constructed feature by feature to preserve the data types
           of pandas DataFrame columns, as otherwise information is lost
-          and cannot be used, eg for the `categories_` attribute.
+          and cannot be used, e.g. for the `categories_` attribute.
 
         """
-        if not (hasattr(X, 'iloc') and getattr(X, 'ndim', 0) == 2):
+        if not (hasattr(X, "iloc") and getattr(X, "ndim", 0) == 2):
             # if not a dataframe, do normal check_array validation
-            X_temp = check_array(X, dtype=None,
-                                 force_all_finite=force_all_finite)
-            if (not hasattr(X, 'dtype')
-                    and np.issubdtype(X_temp.dtype, np.str_)):
-                X = check_array(X, dtype=object,
-                                force_all_finite=force_all_finite)
+            X_temp = check_array(X, dtype=None, ensure_all_finite=ensure_all_finite)
+            if not hasattr(X, "dtype") and np.issubdtype(X_temp.dtype, np.str_):
+                X = check_array(X, dtype=object, ensure_all_finite=ensure_all_finite)
             else:
                 X = X_temp
             needs_validation = False
         else:
             # pandas dataframe, do validation later column by column, in order
             # to keep the dtype information to be used in the encoder.
-            needs_validation = force_all_finite
+            needs_validation = ensure_all_finite
 
         n_samples, n_features = X.shape
         X_columns = []
 
         for i in range(n_features):
-            Xi = self._get_feature(X, feature_idx=i)
-            Xi = check_array(Xi, ensure_2d=False, dtype=None,
-                             force_all_finite=needs_validation)
+            Xi = _safe_indexing(X, indices=i, axis=1)
+            Xi = check_array(
+                Xi, ensure_2d=False, dtype=None, ensure_all_finite=needs_validation
+            )
             X_columns.append(Xi)
 
         return X_columns, n_samples, n_features
 
-    def _get_feature(self, X, feature_idx):
-        if hasattr(X, 'iloc'):
-            # pandas dataframes
-            return X.iloc[:, feature_idx]
-        # numpy arrays, sparse arrays
-        return X[:, feature_idx]
-
-    def _fit(self, X, handle_unknown='error', force_all_finite=True):
+    def _fit(
+        self,
+        X,
+        handle_unknown="error",
+        ensure_all_finite=True,
+        return_counts=False,
+        return_and_ignore_missing_for_infrequent=False,
+    ):
+        self._check_infrequent_enabled()
+        _check_n_features(self, X, reset=True)
+        _check_feature_names(self, X, reset=True)
         X_list, n_samples, n_features = self._check_X(
-            X, force_all_finite=force_all_finite)
+            X, ensure_all_finite=ensure_all_finite
+        )
+        self.n_features_in_ = n_features
 
-        if self.categories != 'auto':
+        if self.categories != "auto":
             if len(self.categories) != n_features:
-                raise ValueError("Shape mismatch: if categories is an array,"
-                                 " it has to be of shape (n_features,).")
+                raise ValueError(
+                    "Shape mismatch: if categories is an array,"
+                    " it has to be of shape (n_features,)."
+                )
 
         self.categories_ = []
+        category_counts = []
+        compute_counts = return_counts or self._infrequent_enabled
 
         for i in range(n_features):
             Xi = X_list[i]
-            if self.categories == 'auto':
-                cats = _unique(Xi)
+
+            if self.categories == "auto":
+                result = _unique(Xi, return_counts=compute_counts)
+                if compute_counts:
+                    cats, counts = result
+                    category_counts.append(counts)
+                else:
+                    cats = result
             else:
-                cats = np.array(self.categories[i], dtype=Xi.dtype)
-                if Xi.dtype.kind not in 'OU':
+                if np.issubdtype(Xi.dtype, np.str_):
+                    # Always convert string categories to objects to avoid
+                    # unexpected string truncation for longer category labels
+                    # passed in the constructor.
+                    Xi_dtype = object
+                else:
+                    Xi_dtype = Xi.dtype
+
+                cats = np.array(self.categories[i], dtype=Xi_dtype)
+                if (
+                    cats.dtype == object
+                    and isinstance(cats[0], bytes)
+                    and Xi.dtype.kind != "S"
+                ):
+                    msg = (
+                        f"In column {i}, the predefined categories have type 'bytes'"
+                        " which is incompatible with values of type"
+                        f" '{type(Xi[0]).__name__}'."
+                    )
+                    raise ValueError(msg)
+
+                # `nan` must be the last stated category
+                for category in cats[:-1]:
+                    if is_scalar_nan(category):
+                        raise ValueError(
+                            "Nan should be the last element in user"
+                            f" provided categories, see categories {cats}"
+                            f" in column #{i}"
+                        )
+
+                if cats.size != len(_unique(cats)):
+                    msg = (
+                        f"In column {i}, the predefined categories"
+                        " contain duplicate elements."
+                    )
+                    raise ValueError(msg)
+
+                if Xi.dtype.kind not in "OUS":
                     sorted_cats = np.sort(cats)
-                    error_msg = ("Unsorted categories are not "
-                                 "supported for numerical categories")
+                    error_msg = (
+                        "Unsorted categories are not supported for numerical categories"
+                    )
                     # if there are nans, nan should be the last element
                     stop_idx = -1 if np.isnan(sorted_cats[-1]) else None
-                    if (np.any(sorted_cats[:stop_idx] != cats[:stop_idx]) or
-                        (np.isnan(sorted_cats[-1]) and
-                         not np.isnan(sorted_cats[-1]))):
+                    if np.any(sorted_cats[:stop_idx] != cats[:stop_idx]):
                         raise ValueError(error_msg)
 
-                if handle_unknown == 'error':
+                if handle_unknown == "error":
                     diff = _check_unknown(Xi, cats)
                     if diff:
-                        msg = ("Found unknown categories {0} in column {1}"
-                               " during fit".format(diff, i))
+                        msg = (
+                            "Found unknown categories {0} in column {1}"
+                            " during fit".format(diff, i)
+                        )
                         raise ValueError(msg)
+                if compute_counts:
+                    category_counts.append(_get_counts(Xi, cats))
+
             self.categories_.append(cats)
 
-    def _transform(self, X, handle_unknown='error', force_all_finite=True):
+        output = {"n_samples": n_samples}
+        if return_counts:
+            output["category_counts"] = category_counts
+
+        missing_indices = {}
+        if return_and_ignore_missing_for_infrequent:
+            for feature_idx, categories_for_idx in enumerate(self.categories_):
+                if is_scalar_nan(categories_for_idx[-1]):
+                    # `nan` values can only be placed in the latest position
+                    missing_indices[feature_idx] = categories_for_idx.size - 1
+            output["missing_indices"] = missing_indices
+
+        if self._infrequent_enabled:
+            self._fit_infrequent_category_mapping(
+                n_samples,
+                category_counts,
+                missing_indices,
+            )
+        return output
+
+    def _transform(
+        self,
+        X,
+        handle_unknown="error",
+        ensure_all_finite=True,
+        warn_on_unknown=False,
+        ignore_category_indices=None,
+    ):
         X_list, n_samples, n_features = self._check_X(
-            X, force_all_finite=force_all_finite)
+            X, ensure_all_finite=ensure_all_finite
+        )
+        _check_feature_names(self, X, reset=False)
+        _check_n_features(self, X, reset=False)
 
         X_int = np.zeros((n_samples, n_features), dtype=int)
         X_mask = np.ones((n_samples, n_features), dtype=bool)
 
-        if n_features != len(self.categories_):
-            raise ValueError(
-                "The number of features in X is different to the number of "
-                "features of the fitted data. The fitted data had {} features "
-                "and the X has {} features."
-                .format(len(self.categories_,), n_features)
-            )
-
+        columns_with_unknown = []
         for i in range(n_features):
             Xi = X_list[i]
-            diff, valid_mask = _check_unknown(Xi, self.categories_[i],
-                                              return_mask=True)
+            diff, valid_mask = _check_unknown(Xi, self.categories_[i], return_mask=True)
 
             if not np.all(valid_mask):
-                if handle_unknown == 'error':
-                    msg = ("Found unknown categories {0} in column {1}"
-                           " during transform".format(diff, i))
+                if handle_unknown == "error":
+                    msg = (
+                        "Found unknown categories {0} in column {1}"
+                        " during transform".format(diff, i)
+                    )
                     raise ValueError(msg)
                 else:
+                    if warn_on_unknown:
+                        columns_with_unknown.append(i)
                     # Set the problematic rows to an acceptable value and
                     # continue `The rows are marked `X_mask` and will be
                     # removed later.
                     X_mask[:, i] = valid_mask
                     # cast Xi into the largest string type necessary
                     # to handle different lengths of numpy strings
-                    if (self.categories_[i].dtype.kind in ('U', 'S')
-                            and self.categories_[i].itemsize > Xi.itemsize):
+                    if (
+                        self.categories_[i].dtype.kind in ("U", "S")
+                        and self.categories_[i].itemsize > Xi.itemsize
+                    ):
                         Xi = Xi.astype(self.categories_[i].dtype)
+                    elif self.categories_[i].dtype.kind == "O" and Xi.dtype.kind == "U":
+                        # categories are objects and Xi are numpy strings.
+                        # Cast Xi to an object dtype to prevent truncation
+                        # when setting invalid values.
+                        Xi = Xi.astype("O")
                     else:
                         Xi = Xi.copy()
 
                     Xi[~valid_mask] = self.categories_[i][0]
             # We use check_unknown=False, since _check_unknown was
             # already called above.
-            X_int[:, i] = _encode(Xi, uniques=self.categories_[i],
-                                  check_unknown=False)
+            X_int[:, i] = _encode(Xi, uniques=self.categories_[i], check_unknown=False)
+        if columns_with_unknown:
+            warnings.warn(
+                (
+                    "Found unknown categories in columns "
+                    f"{columns_with_unknown} during transform. These "
+                    "unknown categories will be encoded as all zeros"
+                ),
+                UserWarning,
+            )
 
+        self._map_infrequent_categories(X_int, X_mask, ignore_category_indices)
         return X_int, X_mask
 
-    def _more_tags(self):
-        return {'X_types': ['categorical']}
+    @property
+    def infrequent_categories_(self):
+        """Infrequent categories for each feature."""
+        # raises an AttributeError if `_infrequent_indices` is not defined
+        infrequent_indices = self._infrequent_indices
+        return [
+            None if indices is None else category[indices]
+            for category, indices in zip(self.categories_, infrequent_indices)
+        ]
+
+    def _check_infrequent_enabled(self):
+        """
+        This functions checks whether _infrequent_enabled is True or False.
+        This has to be called after parameter validation in the fit function.
+        """
+        max_categories = getattr(self, "max_categories", None)
+        min_frequency = getattr(self, "min_frequency", None)
+        self._infrequent_enabled = (
+            max_categories is not None and max_categories >= 1
+        ) or min_frequency is not None
+
+    def _identify_infrequent(self, category_count, n_samples, col_idx):
+        """Compute the infrequent indices.
+
+        Parameters
+        ----------
+        category_count : ndarray of shape (n_cardinality,)
+            Category counts.
+
+        n_samples : int
+            Number of samples.
+
+        col_idx : int
+            Index of the current category. Only used for the error message.
+
+        Returns
+        -------
+        output : ndarray of shape (n_infrequent_categories,) or None
+            If there are infrequent categories, indices of infrequent
+            categories. Otherwise None.
+        """
+        if isinstance(self.min_frequency, numbers.Integral):
+            infrequent_mask = category_count < self.min_frequency
+        elif isinstance(self.min_frequency, numbers.Real):
+            min_frequency_abs = n_samples * self.min_frequency
+            infrequent_mask = category_count < min_frequency_abs
+        else:
+            infrequent_mask = np.zeros(category_count.shape[0], dtype=bool)
+
+        n_current_features = category_count.size - infrequent_mask.sum() + 1
+        if self.max_categories is not None and self.max_categories < n_current_features:
+            # max_categories includes the one infrequent category
+            frequent_category_count = self.max_categories - 1
+            if frequent_category_count == 0:
+                # All categories are infrequent
+                infrequent_mask[:] = True
+            else:
+                # stable sort to preserve original count order
+                smallest_levels = np.argsort(category_count, kind="mergesort")[
+                    :-frequent_category_count
+                ]
+                infrequent_mask[smallest_levels] = True
+
+        output = np.flatnonzero(infrequent_mask)
+        return output if output.size > 0 else None
+
+    def _fit_infrequent_category_mapping(
+        self, n_samples, category_counts, missing_indices
+    ):
+        """Fit infrequent categories.
+
+        Defines the private attribute: `_default_to_infrequent_mappings`. For
+        feature `i`, `_default_to_infrequent_mappings[i]` defines the mapping
+        from the integer encoding returned by `super().transform()` into
+        infrequent categories. If `_default_to_infrequent_mappings[i]` is None,
+        there were no infrequent categories in the training set.
+
+        For example if categories 0, 2 and 4 were frequent, while categories
+        1, 3, 5 were infrequent for feature 7, then these categories are mapped
+        to a single output:
+        `_default_to_infrequent_mappings[7] = array([0, 3, 1, 3, 2, 3])`
+
+        Defines private attribute: `_infrequent_indices`. `_infrequent_indices[i]`
+        is an array of indices such that
+        `categories_[i][_infrequent_indices[i]]` are all the infrequent category
+        labels. If the feature `i` has no infrequent categories
+        `_infrequent_indices[i]` is None.
+
+        .. versionadded:: 1.1
+
+        Parameters
+        ----------
+        n_samples : int
+            Number of samples in training set.
+        category_counts: list of ndarray
+            `category_counts[i]` is the category counts corresponding to
+            `self.categories_[i]`.
+        missing_indices : dict
+            Dict mapping from feature_idx to category index with a missing value.
+        """
+        # Remove missing value from counts, so it is not considered as infrequent
+        if missing_indices:
+            category_counts_ = []
+            for feature_idx, count in enumerate(category_counts):
+                if feature_idx in missing_indices:
+                    category_counts_.append(
+                        np.delete(count, missing_indices[feature_idx])
+                    )
+                else:
+                    category_counts_.append(count)
+        else:
+            category_counts_ = category_counts
+
+        self._infrequent_indices = [
+            self._identify_infrequent(category_count, n_samples, col_idx)
+            for col_idx, category_count in enumerate(category_counts_)
+        ]
+
+        # compute mapping from default mapping to infrequent mapping
+        self._default_to_infrequent_mappings = []
+
+        for feature_idx, infreq_idx in enumerate(self._infrequent_indices):
+            cats = self.categories_[feature_idx]
+            # no infrequent categories
+            if infreq_idx is None:
+                self._default_to_infrequent_mappings.append(None)
+                continue
+
+            n_cats = len(cats)
+            if feature_idx in missing_indices:
+                # Missing index was removed from this category when computing
+                # infrequent indices, thus we need to decrease the number of
+                # total categories when considering the infrequent mapping.
+                n_cats -= 1
+
+            # infrequent indices exist
+            mapping = np.empty(n_cats, dtype=np.int64)
+            n_infrequent_cats = infreq_idx.size
+
+            # infrequent categories are mapped to the last element.
+            n_frequent_cats = n_cats - n_infrequent_cats
+            mapping[infreq_idx] = n_frequent_cats
+
+            frequent_indices = np.setdiff1d(np.arange(n_cats), infreq_idx)
+            mapping[frequent_indices] = np.arange(n_frequent_cats)
+
+            self._default_to_infrequent_mappings.append(mapping)
+
+    def _map_infrequent_categories(self, X_int, X_mask, ignore_category_indices):
+        """Map infrequent categories to integer representing the infrequent category.
+
+        This modifies X_int in-place. Values that were invalid based on `X_mask`
+        are mapped to the infrequent category if there was an infrequent
+        category for that feature.
+
+        Parameters
+        ----------
+        X_int: ndarray of shape (n_samples, n_features)
+            Integer encoded categories.
+
+        X_mask: ndarray of shape (n_samples, n_features)
+            Bool mask for valid values in `X_int`.
+
+        ignore_category_indices : dict
+            Dictionary mapping from feature_idx to category index to ignore.
+            Ignored indexes will not be grouped and the original ordinal encoding
+            will remain.
+        """
+        if not self._infrequent_enabled:
+            return
+
+        ignore_category_indices = ignore_category_indices or {}
+
+        for col_idx in range(X_int.shape[1]):
+            infrequent_idx = self._infrequent_indices[col_idx]
+            if infrequent_idx is None:
+                continue
+
+            X_int[~X_mask[:, col_idx], col_idx] = infrequent_idx[0]
+            if self.handle_unknown == "infrequent_if_exist":
+                # All the unknown values are now mapped to the
+                # infrequent_idx[0], which makes the unknown values valid
+                # This is needed in `transform` when the encoding is formed
+                # using `X_mask`.
+                X_mask[:, col_idx] = True
+
+        # Remaps encoding in `X_int` where the infrequent categories are
+        # grouped together.
+        for i, mapping in enumerate(self._default_to_infrequent_mappings):
+            if mapping is None:
+                continue
+
+            if i in ignore_category_indices:
+                # Update rows that are **not** ignored
+                rows_to_update = X_int[:, i] != ignore_category_indices[i]
+            else:
+                rows_to_update = slice(None)
+
+            X_int[rows_to_update, i] = np.take(mapping, X_int[rows_to_update, i])
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.categorical = True
+        tags.input_tags.allow_nan = True
+        return tags
 
 
 class OneHotEncoder(_BaseEncoder):
@@ -168,8 +470,8 @@ class OneHotEncoder(_BaseEncoder):
     strings, denoting the values taken on by categorical (discrete) features.
     The features are encoded using a one-hot (aka 'one-of-K' or 'dummy')
     encoding scheme. This creates a binary column for each category and
-    returns a sparse matrix or dense array (depending on the ``sparse``
-    parameter)
+    returns a sparse matrix or dense array (depending on the ``sparse_output``
+    parameter).
 
     By default, the encoder derives the categories based on the unique values
     in each feature. Alternatively, you can also specify the `categories`
@@ -182,8 +484,8 @@ class OneHotEncoder(_BaseEncoder):
     instead.
 
     Read more in the :ref:`User Guide `.
-
-    .. versionchanged:: 0.20
+    For a comparison of different encoders, refer to:
+    :ref:`sphx_glr_auto_examples_preprocessing_plot_target_encoder.py`.
 
     Parameters
     ----------
@@ -200,12 +502,12 @@ class OneHotEncoder(_BaseEncoder):
 
         .. versionadded:: 0.20
 
-    drop : {'first', 'if_binary'} or a array-like of shape (n_features,), \
+    drop : {'first', 'if_binary'} or an array-like of shape (n_features,), \
             default=None
         Specifies a methodology to use to drop one of the categories per
         feature. This is useful in situations where perfectly collinear
         features cause problems, such as when feeding the resulting data
-        into a neural network or an unregularized regression.
+        into an unregularized linear regression model.
 
         However, dropping one category breaks the symmetry of the original
         representation and can therefore induce a bias in downstream models,
@@ -220,22 +522,93 @@ class OneHotEncoder(_BaseEncoder):
         - array : ``drop[i]`` is the category in feature ``X[:, i]`` that
           should be dropped.
 
+        When `max_categories` or `min_frequency` is configured to group
+        infrequent categories, the dropping behavior is handled after the
+        grouping.
+
+        .. versionadded:: 0.21
+           The parameter `drop` was added in 0.21.
+
         .. versionchanged:: 0.23
-           Added option 'if_binary'.
+           The option `drop='if_binary'` was added in 0.23.
 
-    sparse : bool, default=True
-        Will return sparse matrix if set True else will return an array.
+        .. versionchanged:: 1.1
+            Support for dropping infrequent categories.
 
-    dtype : number type, default=float
+    sparse_output : bool, default=True
+        When ``True``, it returns a :class:`scipy.sparse.csr_matrix`,
+        i.e. a sparse matrix in "Compressed Sparse Row" (CSR) format.
+
+        .. versionadded:: 1.2
+           `sparse` was renamed to `sparse_output`
+
+    dtype : number type, default=np.float64
         Desired dtype of output.
 
-    handle_unknown : {'error', 'ignore'}, default='error'
-        Whether to raise an error or ignore if an unknown categorical feature
-        is present during transform (default is to raise). When this parameter
-        is set to 'ignore' and an unknown category is encountered during
-        transform, the resulting one-hot encoded columns for this feature
-        will be all zeros. In the inverse transform, an unknown category
-        will be denoted as None.
+    handle_unknown : {'error', 'ignore', 'infrequent_if_exist', 'warn'}, \
+                     default='error'
+        Specifies the way unknown categories are handled during :meth:`transform`.
+
+        - 'error' : Raise an error if an unknown category is present during transform.
+        - 'ignore' : When an unknown category is encountered during
+          transform, the resulting one-hot encoded columns for this feature
+          will be all zeros. In the inverse transform, an unknown category
+          will be denoted as None.
+        - 'infrequent_if_exist' : When an unknown category is encountered
+          during transform, the resulting one-hot encoded columns for this
+          feature will map to the infrequent category if it exists. The
+          infrequent category will be mapped to the last position in the
+          encoding. During inverse transform, an unknown category will be
+          mapped to the category denoted `'infrequent'` if it exists. If the
+          `'infrequent'` category does not exist, then :meth:`transform` and
+          :meth:`inverse_transform` will handle an unknown category as with
+          `handle_unknown='ignore'`. Infrequent categories exist based on
+          `min_frequency` and `max_categories`. Read more in the
+          :ref:`User Guide `.
+        - 'warn' : When an unknown category is encountered during transform
+          a warning is issued, and the encoding then proceeds as described for
+          `handle_unknown="infrequent_if_exist"`.
+
+        .. versionchanged:: 1.1
+            `'infrequent_if_exist'` was added to automatically handle unknown
+            categories and infrequent categories.
+
+        .. versionadded:: 1.6
+           The option `"warn"` was added in 1.6.
+
+    min_frequency : int or float, default=None
+        Specifies the minimum frequency below which a category will be
+        considered infrequent.
+
+        - If `int`, categories with a smaller cardinality will be considered
+          infrequent.
+
+        - If `float`, categories with a smaller cardinality than
+          `min_frequency * n_samples`  will be considered infrequent.
+
+        .. versionadded:: 1.1
+            Read more in the :ref:`User Guide `.
+
+    max_categories : int, default=None
+        Specifies an upper limit to the number of output features for each input
+        feature when considering infrequent categories. If there are infrequent
+        categories, `max_categories` includes the category representing the
+        infrequent categories along with the frequent categories. If `None`,
+        there is no limit to the number of output features.
+
+        .. versionadded:: 1.1
+            Read more in the :ref:`User Guide `.
+
+    feature_name_combiner : "concat" or callable, default="concat"
+        Callable with signature `def callable(input_feature, category)` that returns a
+        string. This is used to create feature names to be returned by
+        :meth:`get_feature_names_out`.
+
+        `"concat"` concatenates encoded feature name and category with
+        `feature + "_" + str(category)`.E.g. feature X with values 1, 6, 7 create
+        feature names `X_1, X_6, X_7`.
+
+        .. versionadded:: 1.3
 
     Attributes
     ----------
@@ -246,7 +619,7 @@ class OneHotEncoder(_BaseEncoder):
         (if any).
 
     drop_idx_ : array of shape (n_features,)
-        - ``drop_idx_[i]`` is the index in ``categories_[i]`` of the category
+        - ``drop_idx_[i]`` is the index in ``categories_[i]`` of the category
           to be dropped for each feature.
         - ``drop_idx_[i] = None`` if no category is to be dropped from the
           feature with index ``i``, e.g. when `drop='if_binary'` and the
@@ -254,13 +627,46 @@ class OneHotEncoder(_BaseEncoder):
         - ``drop_idx_ = None`` if all the transformed features will be
           retained.
 
+        If infrequent categories are enabled by setting `min_frequency` or
+        `max_categories` to a non-default value and `drop_idx[i]` corresponds
+        to a infrequent category, then the entire infrequent category is
+        dropped.
+
         .. versionchanged:: 0.23
            Added the possibility to contain `None` values.
 
+    infrequent_categories_ : list of ndarray
+        Defined only if infrequent categories are enabled by setting
+        `min_frequency` or `max_categories` to a non-default value.
+        `infrequent_categories_[i]` are the infrequent categories for feature
+        `i`. If the feature `i` has no infrequent categories
+        `infrequent_categories_[i]` is None.
+
+        .. versionadded:: 1.1
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 1.0
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    feature_name_combiner : callable or None
+        Callable with signature `def callable(input_feature, category)` that returns a
+        string. This is used to create feature names to be returned by
+        :meth:`get_feature_names_out`.
+
+        .. versionadded:: 1.3
+
     See Also
     --------
     OrdinalEncoder : Performs an ordinal (integer)
       encoding of the categorical features.
+    TargetEncoder : Encodes categorical features using the target.
     sklearn.feature_extraction.DictVectorizer : Performs a one-hot encoding of
       dictionary items (also handles string-valued features).
     sklearn.feature_extraction.FeatureHasher : Performs an approximate one-hot
@@ -292,9 +698,8 @@ class OneHotEncoder(_BaseEncoder):
     >>> enc.inverse_transform([[0, 1, 1, 0, 0], [0, 0, 0, 1, 0]])
     array([['Male', 1],
            [None, 2]], dtype=object)
-    >>> enc.get_feature_names(['gender', 'group'])
-    array(['gender_Female', 'gender_Male', 'group_1', 'group_2', 'group_3'],
-      dtype=object)
+    >>> enc.get_feature_names_out(['gender', 'group'])
+    array(['gender_Female', 'gender_Male', 'group_1', 'group_2', 'group_3'], ...)
 
     One can always drop the first column for each feature:
 
@@ -311,125 +716,268 @@ class OneHotEncoder(_BaseEncoder):
     >>> drop_binary_enc.transform([['Female', 1], ['Male', 2]]).toarray()
     array([[0., 1., 0., 0.],
            [1., 0., 1., 0.]])
+
+    One can change the way feature names are created.
+
+    >>> def custom_combiner(feature, category):
+    ...     return str(feature) + "_" + type(category).__name__ + "_" + str(category)
+    >>> custom_fnames_enc = OneHotEncoder(feature_name_combiner=custom_combiner).fit(X)
+    >>> custom_fnames_enc.get_feature_names_out()
+    array(['x0_str_Female', 'x0_str_Male', 'x1_int_1', 'x1_int_2', 'x1_int_3'],
+          dtype=object)
+
+    Infrequent categories are enabled by setting `max_categories` or `min_frequency`.
+
+    >>> import numpy as np
+    >>> X = np.array([["a"] * 5 + ["b"] * 20 + ["c"] * 10 + ["d"] * 3], dtype=object).T
+    >>> ohe = OneHotEncoder(max_categories=3, sparse_output=False).fit(X)
+    >>> ohe.infrequent_categories_
+    [array(['a', 'd'], dtype=object)]
+    >>> ohe.transform([["a"], ["b"]])
+    array([[0., 0., 1.],
+           [1., 0., 0.]])
     """
 
-    @_deprecate_positional_args
-    def __init__(self, *, categories='auto', drop=None, sparse=True,
-                 dtype=np.float64, handle_unknown='error'):
+    _parameter_constraints: dict = {
+        "categories": [StrOptions({"auto"}), list],
+        "drop": [StrOptions({"first", "if_binary"}), "array-like", None],
+        "dtype": "no_validation",  # validation delegated to numpy
+        "handle_unknown": [
+            StrOptions({"error", "ignore", "infrequent_if_exist", "warn"})
+        ],
+        "max_categories": [Interval(Integral, 1, None, closed="left"), None],
+        "min_frequency": [
+            Interval(Integral, 1, None, closed="left"),
+            Interval(RealNotInt, 0, 1, closed="neither"),
+            None,
+        ],
+        "sparse_output": ["boolean"],
+        "feature_name_combiner": [StrOptions({"concat"}), callable],
+    }
+
+    def __init__(
+        self,
+        *,
+        categories="auto",
+        drop=None,
+        sparse_output=True,
+        dtype=np.float64,
+        handle_unknown="error",
+        min_frequency=None,
+        max_categories=None,
+        feature_name_combiner="concat",
+    ):
         self.categories = categories
-        self.sparse = sparse
+        self.sparse_output = sparse_output
         self.dtype = dtype
         self.handle_unknown = handle_unknown
         self.drop = drop
+        self.min_frequency = min_frequency
+        self.max_categories = max_categories
+        self.feature_name_combiner = feature_name_combiner
 
-    def _validate_keywords(self):
-        if self.handle_unknown not in ('error', 'ignore'):
-            msg = ("handle_unknown should be either 'error' or 'ignore', "
-                   "got {0}.".format(self.handle_unknown))
-            raise ValueError(msg)
-        # If we have both dropped columns and ignored unknown
-        # values, there will be ambiguous cells. This creates difficulties
-        # in interpreting the model.
-        if self.drop is not None and self.handle_unknown != 'error':
-            raise ValueError(
-                "`handle_unknown` must be 'error' when the drop parameter is "
-                "specified, as both would create categories that are all "
-                "zero.")
+    def _map_drop_idx_to_infrequent(self, feature_idx, drop_idx):
+        """Convert `drop_idx` into the index for infrequent categories.
+
+        If there are no infrequent categories, then `drop_idx` is
+        returned. This method is called in `_set_drop_idx` when the `drop`
+        parameter is an array-like.
+        """
+        if not self._infrequent_enabled:
+            return drop_idx
 
-    def _compute_drop_idx(self):
+        default_to_infrequent = self._default_to_infrequent_mappings[feature_idx]
+        if default_to_infrequent is None:
+            return drop_idx
+
+        # Raise error when explicitly dropping a category that is infrequent
+        infrequent_indices = self._infrequent_indices[feature_idx]
+        if infrequent_indices is not None and drop_idx in infrequent_indices:
+            categories = self.categories_[feature_idx]
+            raise ValueError(
+                f"Unable to drop category {categories[drop_idx].item()!r} from"
+                f" feature {feature_idx} because it is infrequent"
+            )
+        return default_to_infrequent[drop_idx]
+
+    def _set_drop_idx(self):
+        """Compute the drop indices associated with `self.categories_`.
+
+        If `self.drop` is:
+        - `None`, No categories have been dropped.
+        - `'first'`, All zeros to drop the first category.
+        - `'if_binary'`, All zeros if the category is binary and `None`
+          otherwise.
+        - array-like, The indices of the categories that match the
+          categories in `self.drop`. If the dropped category is an infrequent
+          category, then the index for the infrequent category is used. This
+          means that the entire infrequent category is dropped.
+
+        This methods defines a public `drop_idx_` and a private
+        `_drop_idx_after_grouping`.
+
+        - `drop_idx_`: Public facing API that references the drop category in
+          `self.categories_`.
+        - `_drop_idx_after_grouping`: Used internally to drop categories *after* the
+          infrequent categories are grouped together.
+
+        If there are no infrequent categories or drop is `None`, then
+        `drop_idx_=_drop_idx_after_grouping`.
+        """
         if self.drop is None:
-            return None
+            drop_idx_after_grouping = None
         elif isinstance(self.drop, str):
-            if self.drop == 'first':
-                return np.zeros(len(self.categories_), dtype=object)
-            elif self.drop == 'if_binary':
-                return np.array([0 if len(cats) == 2 else None
-                                for cats in self.categories_], dtype=object)
-            else:
-                msg = (
-                    "Wrong input for parameter `drop`. Expected "
-                    "'first', 'if_binary', None or array of objects, got {}"
-                    )
-                raise ValueError(msg.format(type(self.drop)))
+            if self.drop == "first":
+                drop_idx_after_grouping = np.zeros(len(self.categories_), dtype=object)
+            elif self.drop == "if_binary":
+                n_features_out_no_drop = [len(cat) for cat in self.categories_]
+                if self._infrequent_enabled:
+                    for i, infreq_idx in enumerate(self._infrequent_indices):
+                        if infreq_idx is None:
+                            continue
+                        n_features_out_no_drop[i] -= infreq_idx.size - 1
+
+                drop_idx_after_grouping = np.array(
+                    [
+                        0 if n_features_out == 2 else None
+                        for n_features_out in n_features_out_no_drop
+                    ],
+                    dtype=object,
+                )
 
         else:
-            try:
-                self.drop = np.asarray(self.drop, dtype=object)
-                droplen = len(self.drop)
-            except (ValueError, TypeError):
-                msg = (
-                    "Wrong input for parameter `drop`. Expected "
-                    "'first', 'if_binary', None or array of objects, got {}"
-                    )
-                raise ValueError(msg.format(type(self.drop)))
+            drop_array = np.asarray(self.drop, dtype=object)
+            droplen = len(drop_array)
+
             if droplen != len(self.categories_):
-                msg = ("`drop` should have length equal to the number "
-                       "of features ({}), got {}")
-                raise ValueError(msg.format(len(self.categories_),
-                                            len(self.drop)))
+                msg = (
+                    "`drop` should have length equal to the number "
+                    "of features ({}), got {}"
+                )
+                raise ValueError(msg.format(len(self.categories_), droplen))
             missing_drops = []
             drop_indices = []
-            for col_idx, (val, cat_list) in enumerate(zip(self.drop,
-                                                          self.categories_)):
-                if not is_scalar_nan(val):
-                    drop_idx = np.where(cat_list == val)[0]
+            for feature_idx, (drop_val, cat_list) in enumerate(
+                zip(drop_array, self.categories_)
+            ):
+                if not is_scalar_nan(drop_val):
+                    drop_idx = np.where(cat_list == drop_val)[0]
                     if drop_idx.size:  # found drop idx
-                        drop_indices.append(drop_idx[0])
+                        drop_indices.append(
+                            self._map_drop_idx_to_infrequent(feature_idx, drop_idx[0])
+                        )
                     else:
-                        missing_drops.append((col_idx, val))
+                        missing_drops.append((feature_idx, drop_val))
                     continue
 
-                # val is nan, find nan in categories manually
-                for cat_idx, cat in enumerate(cat_list):
-                    if is_scalar_nan(cat):
-                        drop_indices.append(cat_idx)
-                        break
-                else:  # loop did not break thus drop is missing
-                    missing_drops.append((col_idx, val))
+                # drop_val is nan, find nan in categories manually
+                if is_scalar_nan(cat_list[-1]):
+                    drop_indices.append(
+                        self._map_drop_idx_to_infrequent(feature_idx, cat_list.size - 1)
+                    )
+                else:  # nan is missing
+                    missing_drops.append((feature_idx, drop_val))
 
             if any(missing_drops):
-                msg = ("The following categories were supposed to be "
-                       "dropped, but were not found in the training "
-                       "data.\n{}".format(
-                           "\n".join(
-                                ["Category: {}, Feature: {}".format(c, v)
-                                    for c, v in missing_drops])))
+                msg = (
+                    "The following categories were supposed to be "
+                    "dropped, but were not found in the training "
+                    "data.\n{}".format(
+                        "\n".join(
+                            [
+                                "Category: {}, Feature: {}".format(c, v)
+                                for c, v in missing_drops
+                            ]
+                        )
+                    )
+                )
                 raise ValueError(msg)
-            return np.array(drop_indices, dtype=object)
+            drop_idx_after_grouping = np.array(drop_indices, dtype=object)
 
-    def fit(self, X, y=None):
-        """
-        Fit OneHotEncoder to X.
+        # `_drop_idx_after_grouping` are the categories to drop *after* the infrequent
+        # categories are grouped together. If needed, we remap `drop_idx` back
+        # to the categories seen in `self.categories_`.
+        self._drop_idx_after_grouping = drop_idx_after_grouping
 
-        Parameters
-        ----------
-        X : array-like, shape [n_samples, n_features]
-            The data to determine the categories of each feature.
+        if not self._infrequent_enabled or drop_idx_after_grouping is None:
+            self.drop_idx_ = self._drop_idx_after_grouping
+        else:
+            drop_idx_ = []
+            for feature_idx, drop_idx in enumerate(drop_idx_after_grouping):
+                default_to_infrequent = self._default_to_infrequent_mappings[
+                    feature_idx
+                ]
+                if drop_idx is None or default_to_infrequent is None:
+                    orig_drop_idx = drop_idx
+                else:
+                    orig_drop_idx = np.flatnonzero(default_to_infrequent == drop_idx)[0]
 
-        y : None
-            Ignored. This parameter exists only for compatibility with
-            :class:`~sklearn.pipeline.Pipeline`.
+                drop_idx_.append(orig_drop_idx)
 
-        Returns
-        -------
-        self
-        """
-        self._validate_keywords()
-        self._fit(X, handle_unknown=self.handle_unknown,
-                  force_all_finite='allow-nan')
-        self.drop_idx_ = self._compute_drop_idx()
-        return self
+            self.drop_idx_ = np.asarray(drop_idx_, dtype=object)
+
+    def _compute_transformed_categories(self, i, remove_dropped=True):
+        """Compute the transformed categories used for column `i`.
 
-    def fit_transform(self, X, y=None):
+        1. If there are infrequent categories, the category is named
+        'infrequent_sklearn'.
+        2. Dropped columns are removed when remove_dropped=True.
         """
-        Fit OneHotEncoder to X, then transform X.
+        cats = self.categories_[i]
+
+        if self._infrequent_enabled:
+            infreq_map = self._default_to_infrequent_mappings[i]
+            if infreq_map is not None:
+                frequent_mask = infreq_map < infreq_map.max()
+                infrequent_cat = "infrequent_sklearn"
+                # infrequent category is always at the end
+                cats = np.concatenate(
+                    (cats[frequent_mask], np.array([infrequent_cat], dtype=object))
+                )
+
+        if remove_dropped:
+            cats = self._remove_dropped_categories(cats, i)
+        return cats
+
+    def _remove_dropped_categories(self, categories, i):
+        """Remove dropped categories."""
+        if (
+            self._drop_idx_after_grouping is not None
+            and self._drop_idx_after_grouping[i] is not None
+        ):
+            return np.delete(categories, self._drop_idx_after_grouping[i])
+        return categories
+
+    def _compute_n_features_outs(self):
+        """Compute the n_features_out for each input feature."""
+        output = [len(cats) for cats in self.categories_]
+
+        if self._drop_idx_after_grouping is not None:
+            for i, drop_idx in enumerate(self._drop_idx_after_grouping):
+                if drop_idx is not None:
+                    output[i] -= 1
+
+        if not self._infrequent_enabled:
+            return output
+
+        # infrequent is enabled, the number of features out are reduced
+        # because the infrequent categories are grouped together
+        for i, infreq_idx in enumerate(self._infrequent_indices):
+            if infreq_idx is None:
+                continue
+            output[i] -= infreq_idx.size - 1
+
+        return output
 
-        Equivalent to fit(X).transform(X) but more convenient.
+    @_fit_context(prefer_skip_nested_validation=True)
+    def fit(self, X, y=None):
+        """
+        Fit OneHotEncoder to X.
 
         Parameters
         ----------
-        X : array-like, shape [n_samples, n_features]
-            The data to encode.
+        X : array-like of shape (n_samples, n_features)
+            The data to determine the categories of each feature.
 
         y : None
             Ignored. This parameter exists only for compatibility with
@@ -437,70 +985,101 @@ def fit_transform(self, X, y=None):
 
         Returns
         -------
-        X_out : sparse matrix if sparse=True else a 2-d array
-            Transformed input.
+        self
+            Fitted encoder.
         """
-        self._validate_keywords()
-        return super().fit_transform(X, y)
+        self._fit(
+            X,
+            handle_unknown=self.handle_unknown,
+            ensure_all_finite="allow-nan",
+        )
+        self._set_drop_idx()
+        self._n_features_outs = self._compute_n_features_outs()
+        return self
 
     def transform(self, X):
         """
         Transform X using one-hot encoding.
 
+        If `sparse_output=True` (default), it returns an instance of
+        :class:`scipy.sparse._csr.csr_matrix` (CSR format).
+
+        If there are infrequent categories for a feature, set by specifying
+        `max_categories` or `min_frequency`, the infrequent categories are
+        grouped into a single category.
+
         Parameters
         ----------
-        X : array-like, shape [n_samples, n_features]
+        X : array-like of shape (n_samples, n_features)
             The data to encode.
 
         Returns
         -------
-        X_out : sparse matrix if sparse=True else a 2-d array
-            Transformed input.
+        X_out : {ndarray, sparse matrix} of shape \
+                (n_samples, n_encoded_features)
+            Transformed input. If `sparse_output=True`, a sparse matrix will be
+            returned.
         """
         check_is_fitted(self)
+        transform_output = _get_output_config("transform", estimator=self)["dense"]
+        if transform_output != "default" and self.sparse_output:
+            capitalize_transform_output = transform_output.capitalize()
+            raise ValueError(
+                f"{capitalize_transform_output} output does not support sparse data."
+                f" Set sparse_output=False to output {transform_output} dataframes or"
+                f" disable {capitalize_transform_output} output via"
+                '` ohe.set_output(transform="default").'
+            )
+
         # validation of X happens in _check_X called by _transform
-        X_int, X_mask = self._transform(X, handle_unknown=self.handle_unknown,
-                                        force_all_finite='allow-nan')
+        if self.handle_unknown == "warn":
+            warn_on_unknown, handle_unknown = True, "infrequent_if_exist"
+        else:
+            warn_on_unknown = self.drop is not None and self.handle_unknown in {
+                "ignore",
+                "infrequent_if_exist",
+            }
+            handle_unknown = self.handle_unknown
+        X_int, X_mask = self._transform(
+            X,
+            handle_unknown=handle_unknown,
+            ensure_all_finite="allow-nan",
+            warn_on_unknown=warn_on_unknown,
+        )
 
         n_samples, n_features = X_int.shape
 
-        if self.drop_idx_ is not None:
-            to_drop = self.drop_idx_.copy()
+        if self._drop_idx_after_grouping is not None:
+            to_drop = self._drop_idx_after_grouping.copy()
             # We remove all the dropped categories from mask, and decrement all
             # categories that occur after them to avoid an empty column.
             keep_cells = X_int != to_drop
-            n_values = []
             for i, cats in enumerate(self.categories_):
-                n_cats = len(cats)
-
                 # drop='if_binary' but feature isn't binary
                 if to_drop[i] is None:
                     # set to cardinality to not drop from X_int
-                    to_drop[i] = n_cats
-                    n_values.append(n_cats)
-                else:  # dropped
-                    n_values.append(n_cats - 1)
+                    to_drop[i] = len(cats)
 
             to_drop = to_drop.reshape(1, -1)
             X_int[X_int > to_drop] -= 1
             X_mask &= keep_cells
-        else:
-            n_values = [len(cats) for cats in self.categories_]
 
         mask = X_mask.ravel()
-        feature_indices = np.cumsum([0] + n_values)
+        feature_indices = np.cumsum([0] + self._n_features_outs)
         indices = (X_int + feature_indices[:-1]).ravel()[mask]
 
         indptr = np.empty(n_samples + 1, dtype=int)
         indptr[0] = 0
-        np.sum(X_mask, axis=1, out=indptr[1:])
+        np.sum(X_mask, axis=1, out=indptr[1:], dtype=indptr.dtype)
         np.cumsum(indptr[1:], out=indptr[1:])
         data = np.ones(indptr[-1])
 
-        out = sparse.csr_matrix((data, indices, indptr),
-                                shape=(n_samples, feature_indices[-1]),
-                                dtype=self.dtype)
-        if not self.sparse:
+        out = sparse.csr_matrix(
+            (data, indices, indptr),
+            shape=(n_samples, feature_indices[-1]),
+            dtype=self.dtype,
+        )
+        if not self.sparse_output:
             return out.toarray()
         else:
             return out
@@ -509,83 +1088,107 @@ def inverse_transform(self, X):
         """
         Convert the data back to the original representation.
 
-        In case unknown categories are encountered (all zeros in the
-        one-hot encoding), ``None`` is used to represent this category.
+        When unknown categories are encountered (all zeros in the
+        one-hot encoding), ``None`` is used to represent this category. If the
+        feature with the unknown category has a dropped category, the dropped
+        category will be its inverse.
+
+        For a given input feature, if there is an infrequent category,
+        'infrequent_sklearn' will be used to represent the infrequent category.
 
         Parameters
         ----------
-        X : array-like or sparse matrix, shape [n_samples, n_encoded_features]
+        X : {array-like, sparse matrix} of shape \
+                (n_samples, n_encoded_features)
             The transformed data.
 
         Returns
         -------
-        X_tr : array-like, shape [n_samples, n_features]
+        X_original : ndarray of shape (n_samples, n_features)
             Inverse transformed array.
         """
         check_is_fitted(self)
-        X = check_array(X, accept_sparse='csr')
+        X = check_array(X, accept_sparse="csr")
 
         n_samples, _ = X.shape
         n_features = len(self.categories_)
-        if self.drop_idx_ is None:
-            n_transformed_features = sum(len(cats)
-                                         for cats in self.categories_)
-        else:
-            n_transformed_features = sum(
-                len(cats) - 1 if to_drop is not None else len(cats)
-                for cats, to_drop in zip(self.categories_, self.drop_idx_)
-            )
+
+        n_features_out = np.sum(self._n_features_outs)
 
         # validate shape of passed X
-        msg = ("Shape of the passed X data is not correct. Expected {0} "
-               "columns, got {1}.")
-        if X.shape[1] != n_transformed_features:
-            raise ValueError(msg.format(n_transformed_features, X.shape[1]))
+        msg = (
+            "Shape of the passed X data is not correct. Expected {0} columns, got {1}."
+        )
+        if X.shape[1] != n_features_out:
+            raise ValueError(msg.format(n_features_out, X.shape[1]))
+
+        transformed_features = [
+            self._compute_transformed_categories(i, remove_dropped=False)
+            for i, _ in enumerate(self.categories_)
+        ]
 
         # create resulting array of appropriate dtype
-        dt = np.find_common_type([cat.dtype for cat in self.categories_], [])
+        dt = np.result_type(*[cat.dtype for cat in transformed_features])
         X_tr = np.empty((n_samples, n_features), dtype=dt)
 
         j = 0
         found_unknown = {}
 
+        if self._infrequent_enabled:
+            infrequent_indices = self._infrequent_indices
+        else:
+            infrequent_indices = [None] * n_features
+
         for i in range(n_features):
-            if self.drop_idx_ is None or self.drop_idx_[i] is None:
-                cats = self.categories_[i]
-            else:
-                cats = np.delete(self.categories_[i], self.drop_idx_[i])
-            n_categories = len(cats)
+            cats_wo_dropped = self._remove_dropped_categories(
+                transformed_features[i], i
+            )
+            n_categories = cats_wo_dropped.shape[0]
 
             # Only happens if there was a column with a unique
             # category. In this case we just fill the column with this
             # unique category value.
             if n_categories == 0:
-                X_tr[:, i] = self.categories_[i][self.drop_idx_[i]]
+                X_tr[:, i] = self.categories_[i][self._drop_idx_after_grouping[i]]
                 j += n_categories
                 continue
-            sub = X[:, j:j + n_categories]
+            sub = X[:, j : j + n_categories]
             # for sparse X argmax returns 2D matrix, ensure 1D array
             labels = np.asarray(sub.argmax(axis=1)).flatten()
-            X_tr[:, i] = cats[labels]
-            if self.handle_unknown == 'ignore':
+            X_tr[:, i] = cats_wo_dropped[labels]
+
+            if self.handle_unknown == "ignore" or (
+                self.handle_unknown in ("infrequent_if_exist", "warn")
+                and infrequent_indices[i] is None
+            ):
                 unknown = np.asarray(sub.sum(axis=1) == 0).flatten()
                 # ignored unknown categories: we have a row of all zero
                 if unknown.any():
-                    found_unknown[i] = unknown
+                    # if categories were dropped then unknown categories will
+                    # be mapped to the dropped category
+                    if (
+                        self._drop_idx_after_grouping is None
+                        or self._drop_idx_after_grouping[i] is None
+                    ):
+                        found_unknown[i] = unknown
+                    else:
+                        X_tr[unknown, i] = self.categories_[i][
+                            self._drop_idx_after_grouping[i]
+                        ]
             else:
                 dropped = np.asarray(sub.sum(axis=1) == 0).flatten()
                 if dropped.any():
-                    if self.drop_idx_ is None:
+                    if self._drop_idx_after_grouping is None:
                         all_zero_samples = np.flatnonzero(dropped)
                         raise ValueError(
                             f"Samples {all_zero_samples} can not be inverted "
                             "when drop=None and handle_unknown='error' "
-                            "because they contain all zeros")
+                            "because they contain all zeros"
+                        )
                     # we can safely assume that all of the nulls in each column
                     # are the dropped value
-                    X_tr[dropped, i] = self.categories_[i][
-                        self.drop_idx_[i]
-                    ]
+                    drop_idx = self._drop_idx_after_grouping[i]
+                    X_tr[dropped, i] = transformed_features[i][drop_idx]
 
             j += n_categories
 
@@ -600,43 +1203,55 @@ def inverse_transform(self, X):
 
         return X_tr
 
-    def get_feature_names(self, input_features=None):
-        """
-        Return feature names for output features.
+    def get_feature_names_out(self, input_features=None):
+        """Get output feature names for transformation.
 
         Parameters
         ----------
-        input_features : list of str of shape (n_features,)
-            String names for input features if available. By default,
-            "x0", "x1", ... "xn_features" is used.
+        input_features : array-like of str or None, default=None
+            Input features.
+
+            - If `input_features` is `None`, then `feature_names_in_` is
+              used as feature names in. If `feature_names_in_` is not defined,
+              then the following input feature names are generated:
+              `["x0", "x1", ..., "x(n_features_in_ - 1)"]`.
+            - If `input_features` is an array-like, then `input_features` must
+              match `feature_names_in_` if `feature_names_in_` is defined.
 
         Returns
         -------
-        output_feature_names : ndarray of shape (n_output_features,)
-            Array of feature names.
+        feature_names_out : ndarray of str objects
+            Transformed feature names.
         """
         check_is_fitted(self)
-        cats = self.categories_
-        if input_features is None:
-            input_features = ['x%d' % i for i in range(len(cats))]
-        elif len(input_features) != len(self.categories_):
-            raise ValueError(
-                "input_features should have length equal to number of "
-                "features ({}), got {}".format(len(self.categories_),
-                                               len(input_features)))
+        input_features = _check_feature_names_in(self, input_features)
+        cats = [
+            self._compute_transformed_categories(i)
+            for i, _ in enumerate(self.categories_)
+        ]
 
+        name_combiner = self._check_get_feature_name_combiner()
         feature_names = []
         for i in range(len(cats)):
-            names = [
-                input_features[i] + '_' + str(t) for t in cats[i]]
-            if self.drop_idx_ is not None and self.drop_idx_[i] is not None:
-                names.pop(self.drop_idx_[i])
+            names = [name_combiner(input_features[i], t) for t in cats[i]]
             feature_names.extend(names)
 
         return np.array(feature_names, dtype=object)
 
+    def _check_get_feature_name_combiner(self):
+        if self.feature_name_combiner == "concat":
+            return lambda feature, category: feature + "_" + str(category)
+        else:  # callable
+            dry_run_combiner = self.feature_name_combiner("feature", "category")
+            if not isinstance(dry_run_combiner, str):
+                raise TypeError(
+                    "When `feature_name_combiner` is a callable, it should return a "
+                    f"Python string. Got {type(dry_run_combiner)} instead."
+                )
+            return self.feature_name_combiner
+
 
-class OrdinalEncoder(_BaseEncoder):
+class OrdinalEncoder(OneToOneFeatureMixin, _BaseEncoder):
     """
     Encode categorical features as an integer array.
 
@@ -646,6 +1261,8 @@ class OrdinalEncoder(_BaseEncoder):
     a single column of integers (0 to n_categories - 1) per feature.
 
     Read more in the :ref:`User Guide `.
+    For a comparison of different encoders, refer to:
+    :ref:`sphx_glr_auto_examples_preprocessing_plot_target_encoder.py`.
 
     .. versionadded:: 0.20
 
@@ -661,7 +1278,7 @@ class OrdinalEncoder(_BaseEncoder):
 
         The used categories can be found in the ``categories_`` attribute.
 
-    dtype : number type, default np.float64
+    dtype : number type, default=np.float64
         Desired dtype of output.
 
     handle_unknown : {'error', 'use_encoded_value'}, default='error'
@@ -682,6 +1299,40 @@ class OrdinalEncoder(_BaseEncoder):
 
         .. versionadded:: 0.24
 
+    encoded_missing_value : int or np.nan, default=np.nan
+        Encoded value of missing categories. If set to `np.nan`, then the `dtype`
+        parameter must be a float dtype.
+
+        .. versionadded:: 1.1
+
+    min_frequency : int or float, default=None
+        Specifies the minimum frequency below which a category will be
+        considered infrequent.
+
+        - If `int`, categories with a smaller cardinality will be considered
+          infrequent.
+
+        - If `float`, categories with a smaller cardinality than
+          `min_frequency * n_samples`  will be considered infrequent.
+
+        .. versionadded:: 1.3
+            Read more in the :ref:`User Guide `.
+
+    max_categories : int, default=None
+        Specifies an upper limit to the number of output categories for each input
+        feature when considering infrequent categories. If there are infrequent
+        categories, `max_categories` includes the category representing the
+        infrequent categories along with the frequent categories. If `None`,
+        there is no limit to the number of output features.
+
+        `max_categories` do **not** take into account missing or unknown
+        categories. Setting `unknown_value` or `encoded_missing_value` to an
+        integer will increase the number of unique integer codes by one each.
+        This can result in up to `max_categories + 2` integer codes.
+
+        .. versionadded:: 1.3
+            Read more in the :ref:`User Guide `.
+
     Attributes
     ----------
     categories_ : list of arrays
@@ -689,12 +1340,44 @@ class OrdinalEncoder(_BaseEncoder):
         the features in X and corresponding with the output of ``transform``).
         This does not include categories that weren't seen during ``fit``.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 1.0
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    infrequent_categories_ : list of ndarray
+        Defined only if infrequent categories are enabled by setting
+        `min_frequency` or `max_categories` to a non-default value.
+        `infrequent_categories_[i]` are the infrequent categories for feature
+        `i`. If the feature `i` has no infrequent categories
+        `infrequent_categories_[i]` is None.
+
+        .. versionadded:: 1.3
+
     See Also
     --------
-    OneHotEncoder : Performs a one-hot encoding of categorical features.
+    OneHotEncoder : Performs a one-hot encoding of categorical features. This encoding
+        is suitable for low to medium cardinality categorical variables, both in
+        supervised and unsupervised settings.
+    TargetEncoder : Encodes categorical features using supervised signal
+        in a classification or regression pipeline. This encoding is typically
+        suitable for high cardinality categorical variables.
     LabelEncoder : Encodes target labels with values between 0 and
         ``n_classes-1``.
 
+    Notes
+    -----
+    With a high proportion of `nan` values, inferring categories becomes slow with
+    Python versions before 3.10. The handling of `nan` values was improved
+    from Python 3.10 onwards, (c.f.
+    `bpo-43475 `_).
+
     Examples
     --------
     Given a dataset with two features, we let the encoder find the unique
@@ -714,23 +1397,87 @@ class OrdinalEncoder(_BaseEncoder):
     >>> enc.inverse_transform([[1, 0], [0, 1]])
     array([['Male', 1],
            ['Female', 2]], dtype=object)
+
+    By default, :class:`OrdinalEncoder` is lenient towards missing values by
+    propagating them.
+
+    >>> import numpy as np
+    >>> X = [['Male', 1], ['Female', 3], ['Female', np.nan]]
+    >>> enc.fit_transform(X)
+    array([[ 1.,  0.],
+           [ 0.,  1.],
+           [ 0., nan]])
+
+    You can use the parameter `encoded_missing_value` to encode missing values.
+
+    >>> enc.set_params(encoded_missing_value=-1).fit_transform(X)
+    array([[ 1.,  0.],
+           [ 0.,  1.],
+           [ 0., -1.]])
+
+    Infrequent categories are enabled by setting `max_categories` or `min_frequency`.
+    In the following example, "a" and "d" are considered infrequent and grouped
+    together into a single category, "b" and "c" are their own categories, unknown
+    values are encoded as 3 and missing values are encoded as 4.
+
+    >>> X_train = np.array(
+    ...     [["a"] * 5 + ["b"] * 20 + ["c"] * 10 + ["d"] * 3 + [np.nan]],
+    ...     dtype=object).T
+    >>> enc = OrdinalEncoder(
+    ...     handle_unknown="use_encoded_value", unknown_value=3,
+    ...     max_categories=3, encoded_missing_value=4)
+    >>> _ = enc.fit(X_train)
+    >>> X_test = np.array([["a"], ["b"], ["c"], ["d"], ["e"], [np.nan]], dtype=object)
+    >>> enc.transform(X_test)
+    array([[2.],
+           [0.],
+           [1.],
+           [2.],
+           [3.],
+           [4.]])
     """
 
-    @_deprecate_positional_args
-    def __init__(self, *, categories='auto', dtype=np.float64,
-                 handle_unknown='error', unknown_value=None):
+    _parameter_constraints: dict = {
+        "categories": [StrOptions({"auto"}), list],
+        "dtype": "no_validation",  # validation delegated to numpy
+        "encoded_missing_value": [Integral, type(np.nan)],
+        "handle_unknown": [StrOptions({"error", "use_encoded_value"})],
+        "unknown_value": [Integral, type(np.nan), None],
+        "max_categories": [Interval(Integral, 1, None, closed="left"), None],
+        "min_frequency": [
+            Interval(Integral, 1, None, closed="left"),
+            Interval(RealNotInt, 0, 1, closed="neither"),
+            None,
+        ],
+    }
+
+    def __init__(
+        self,
+        *,
+        categories="auto",
+        dtype=np.float64,
+        handle_unknown="error",
+        unknown_value=None,
+        encoded_missing_value=np.nan,
+        min_frequency=None,
+        max_categories=None,
+    ):
         self.categories = categories
         self.dtype = dtype
         self.handle_unknown = handle_unknown
         self.unknown_value = unknown_value
+        self.encoded_missing_value = encoded_missing_value
+        self.min_frequency = min_frequency
+        self.max_categories = max_categories
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y=None):
         """
         Fit the OrdinalEncoder to X.
 
         Parameters
         ----------
-        X : array-like, shape [n_samples, n_features]
+        X : array-like of shape (n_samples, n_features)
             The data to determine the categories of each feature.
 
         y : None
@@ -739,57 +1486,96 @@ def fit(self, X, y=None):
 
         Returns
         -------
-        self
+        self : object
+            Fitted encoder.
         """
-        handle_unknown_strategies = ("error", "use_encoded_value")
-        if self.handle_unknown not in handle_unknown_strategies:
-            raise ValueError(
-                f"handle_unknown should be either 'error' or "
-                f"'use_encoded_value', got {self.handle_unknown}."
-            )
-
-        if self.handle_unknown == 'use_encoded_value':
+        if self.handle_unknown == "use_encoded_value":
             if is_scalar_nan(self.unknown_value):
-                if np.dtype(self.dtype).kind != 'f':
+                if np.dtype(self.dtype).kind != "f":
                     raise ValueError(
-                        f"When unknown_value is np.nan, the dtype "
-                        f"parameter should be "
+                        "When unknown_value is np.nan, the dtype "
+                        "parameter should be "
                         f"a float dtype. Got {self.dtype}."
                     )
             elif not isinstance(self.unknown_value, numbers.Integral):
-                raise TypeError(f"unknown_value should be an integer or "
-                                f"np.nan when "
-                                f"handle_unknown is 'use_encoded_value', "
-                                f"got {self.unknown_value}.")
+                raise TypeError(
+                    "unknown_value should be an integer or "
+                    "np.nan when "
+                    "handle_unknown is 'use_encoded_value', "
+                    f"got {self.unknown_value}."
+                )
         elif self.unknown_value is not None:
-            raise TypeError(f"unknown_value should only be set when "
-                            f"handle_unknown is 'use_encoded_value', "
-                            f"got {self.unknown_value}.")
-
-        self._fit(X, force_all_finite='allow-nan')
-
-        if self.handle_unknown == 'use_encoded_value':
-            for feature_cats in self.categories_:
-                if 0 <= self.unknown_value < len(feature_cats):
-                    raise ValueError(f"The used value for unknown_value "
-                                     f"{self.unknown_value} is one of the "
-                                     f"values already used for encoding the "
-                                     f"seen categories.")
-
-        # stores the missing indices per category
-        self._missing_indices = {}
+            raise TypeError(
+                "unknown_value should only be set when "
+                "handle_unknown is 'use_encoded_value', "
+                f"got {self.unknown_value}."
+            )
+
+        # `_fit` will only raise an error when `self.handle_unknown="error"`
+        fit_results = self._fit(
+            X,
+            handle_unknown=self.handle_unknown,
+            ensure_all_finite="allow-nan",
+            return_and_ignore_missing_for_infrequent=True,
+        )
+        self._missing_indices = fit_results["missing_indices"]
+
+        cardinalities = [len(categories) for categories in self.categories_]
+        if self._infrequent_enabled:
+            # Cardinality decreases because the infrequent categories are grouped
+            # together
+            for feature_idx, infrequent in enumerate(self.infrequent_categories_):
+                if infrequent is not None:
+                    cardinalities[feature_idx] -= len(infrequent)
+
+        # missing values are not considered part of the cardinality
+        # when considering unknown categories or encoded_missing_value
         for cat_idx, categories_for_idx in enumerate(self.categories_):
-            for i, cat in enumerate(categories_for_idx):
-                if is_scalar_nan(cat):
-                    self._missing_indices[cat_idx] = i
-                    continue
+            if is_scalar_nan(categories_for_idx[-1]):
+                cardinalities[cat_idx] -= 1
 
-        if np.dtype(self.dtype).kind != 'f' and self._missing_indices:
-            raise ValueError(
-                "There are missing values in features "
-                f"{list(self._missing_indices)}. For OrdinalEncoder to "
-                "passthrough missing values, the dtype parameter must be a "
-                "float")
+        if self.handle_unknown == "use_encoded_value":
+            for cardinality in cardinalities:
+                if 0 <= self.unknown_value < cardinality:
+                    raise ValueError(
+                        "The used value for unknown_value "
+                        f"{self.unknown_value} is one of the "
+                        "values already used for encoding the "
+                        "seen categories."
+                    )
+
+        if self._missing_indices:
+            if np.dtype(self.dtype).kind != "f" and is_scalar_nan(
+                self.encoded_missing_value
+            ):
+                raise ValueError(
+                    "There are missing values in features "
+                    f"{list(self._missing_indices)}. For OrdinalEncoder to "
+                    f"encode missing values with dtype: {self.dtype}, set "
+                    "encoded_missing_value to a non-nan value, or "
+                    "set dtype to a float"
+                )
+
+            if not is_scalar_nan(self.encoded_missing_value):
+                # Features are invalid when they contain a missing category
+                # and encoded_missing_value was already used to encode a
+                # known category
+                invalid_features = [
+                    cat_idx
+                    for cat_idx, cardinality in enumerate(cardinalities)
+                    if cat_idx in self._missing_indices
+                    and 0 <= self.encoded_missing_value < cardinality
+                ]
+
+                if invalid_features:
+                    # Use feature names if they are available
+                    if hasattr(self, "feature_names_in_"):
+                        invalid_features = self.feature_names_in_[invalid_features]
+                    raise ValueError(
+                        f"encoded_missing_value ({self.encoded_missing_value}) "
+                        "is already used to encode a known category in features: "
+                        f"{invalid_features}"
+                    )
 
         return self
 
@@ -799,24 +1585,29 @@ def transform(self, X):
 
         Parameters
         ----------
-        X : array-like, shape [n_samples, n_features]
+        X : array-like of shape (n_samples, n_features)
             The data to encode.
 
         Returns
         -------
-        X_out : sparse matrix or a 2-d array
+        X_out : ndarray of shape (n_samples, n_features)
             Transformed input.
         """
-        X_int, X_mask = self._transform(X, handle_unknown=self.handle_unknown,
-                                        force_all_finite='allow-nan')
+        check_is_fitted(self, "categories_")
+        X_int, X_mask = self._transform(
+            X,
+            handle_unknown=self.handle_unknown,
+            ensure_all_finite="allow-nan",
+            ignore_category_indices=self._missing_indices,
+        )
         X_trans = X_int.astype(self.dtype, copy=False)
 
         for cat_idx, missing_idx in self._missing_indices.items():
             X_missing_mask = X_int[:, cat_idx] == missing_idx
-            X_trans[X_missing_mask, cat_idx] = np.nan
+            X_trans[X_missing_mask, cat_idx] = self.encoded_missing_value
 
         # create separate category for unknown values
-        if self.handle_unknown == 'use_encoded_value':
+        if self.handle_unknown == "use_encoded_value":
             X_trans[~X_mask] = self.unknown_value
         return X_trans
 
@@ -826,53 +1617,82 @@ def inverse_transform(self, X):
 
         Parameters
         ----------
-        X : array-like or sparse matrix, shape [n_samples, n_encoded_features]
+        X : array-like of shape (n_samples, n_encoded_features)
             The transformed data.
 
         Returns
         -------
-        X_tr : array-like, shape [n_samples, n_features]
+        X_original : ndarray of shape (n_samples, n_features)
             Inverse transformed array.
         """
         check_is_fitted(self)
-        X = check_array(X, accept_sparse='csr', force_all_finite='allow-nan')
+        X = check_array(X, ensure_all_finite="allow-nan")
 
         n_samples, _ = X.shape
         n_features = len(self.categories_)
 
         # validate shape of passed X
-        msg = ("Shape of the passed X data is not correct. Expected {0} "
-               "columns, got {1}.")
+        msg = (
+            "Shape of the passed X data is not correct. Expected {0} columns, got {1}."
+        )
         if X.shape[1] != n_features:
             raise ValueError(msg.format(n_features, X.shape[1]))
 
         # create resulting array of appropriate dtype
-        dt = np.find_common_type([cat.dtype for cat in self.categories_], [])
+        dt = np.result_type(*[cat.dtype for cat in self.categories_])
         X_tr = np.empty((n_samples, n_features), dtype=dt)
 
         found_unknown = {}
+        infrequent_masks = {}
+
+        infrequent_indices = getattr(self, "_infrequent_indices", None)
 
         for i in range(n_features):
-            labels = X[:, i].astype('int64', copy=False)
+            labels = X[:, i]
 
             # replace values of X[:, i] that were nan with actual indices
             if i in self._missing_indices:
-                X_i_mask = _get_mask(X[:, i], np.nan)
+                X_i_mask = _get_mask(labels, self.encoded_missing_value)
                 labels[X_i_mask] = self._missing_indices[i]
 
-            if self.handle_unknown == 'use_encoded_value':
-                unknown_labels = labels == self.unknown_value
-                X_tr[:, i] = self.categories_[i][np.where(
-                    unknown_labels, 0, labels)]
+            rows_to_update = slice(None)
+            categories = self.categories_[i]
+
+            if infrequent_indices is not None and infrequent_indices[i] is not None:
+                # Compute mask for frequent categories
+                infrequent_encoding_value = len(categories) - len(infrequent_indices[i])
+                infrequent_masks[i] = labels == infrequent_encoding_value
+                rows_to_update = ~infrequent_masks[i]
+
+                # Remap categories to be only frequent categories. The infrequent
+                # categories will be mapped to "infrequent_sklearn" later
+                frequent_categories_mask = np.ones_like(categories, dtype=bool)
+                frequent_categories_mask[infrequent_indices[i]] = False
+                categories = categories[frequent_categories_mask]
+
+            if self.handle_unknown == "use_encoded_value":
+                unknown_labels = _get_mask(labels, self.unknown_value)
                 found_unknown[i] = unknown_labels
-            else:
-                X_tr[:, i] = self.categories_[i][labels]
 
-        # insert None values for unknown values
-        if found_unknown:
+                known_labels = ~unknown_labels
+                if isinstance(rows_to_update, np.ndarray):
+                    rows_to_update &= known_labels
+                else:
+                    rows_to_update = known_labels
+
+            labels_int = labels[rows_to_update].astype("int64", copy=False)
+            X_tr[rows_to_update, i] = categories[labels_int]
+
+        if found_unknown or infrequent_masks:
             X_tr = X_tr.astype(object, copy=False)
 
+        # insert None values for unknown values
+        if found_unknown:
             for idx, mask in found_unknown.items():
                 X_tr[mask, idx] = None
 
+        if infrequent_masks:
+            for idx, mask in infrequent_masks.items():
+                X_tr[mask, idx] = "infrequent_sklearn"
+
         return X_tr
diff --git a/sklearn/preprocessing/_function_transformer.py b/sklearn/preprocessing/_function_transformer.py
index ca176aeb87a10..3d7592b17e2af 100644
--- a/sklearn/preprocessing/_function_transformer.py
+++ b/sklearn/preprocessing/_function_transformer.py
@@ -1,13 +1,34 @@
-import warnings
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-from ..base import BaseEstimator, TransformerMixin
-from ..utils.validation import _allclose_dense_sparse
-from ..utils.validation import _deprecate_positional_args
+import warnings
+from functools import partial
+
+import numpy as np
+
+from ..base import BaseEstimator, TransformerMixin, _fit_context
+from ..utils._param_validation import StrOptions
+from ..utils._repr_html.estimator import _VisualBlock
+from ..utils._set_output import (
+    _get_adapter_from_container,
+    _get_output_config,
+)
+from ..utils.metaestimators import available_if
+from ..utils.validation import (
+    _allclose_dense_sparse,
+    _check_feature_names,
+    _check_feature_names_in,
+    _check_n_features,
+    _get_feature_names,
+    _is_pandas_df,
+    _is_polars_df,
+    check_array,
+    validate_data,
+)
 
 
 def _identity(X):
-    """The identity function.
-    """
+    """The identity function."""
     return X
 
 
@@ -63,6 +84,20 @@ class FunctionTransformer(TransformerMixin, BaseEstimator):
 
        .. versionadded:: 0.20
 
+    feature_names_out : callable, 'one-to-one' or None, default=None
+        Determines the list of feature names that will be returned by the
+        `get_feature_names_out` method. If it is 'one-to-one', then the output
+        feature names will be equal to the input feature names. If it is a
+        callable, then it must take two positional arguments: this
+        `FunctionTransformer` (`self`) and an array-like of input feature names
+        (`input_features`). It must return an array-like of output feature
+        names. The `get_feature_names_out` method is only defined if
+        `feature_names_out` is not None.
+
+        See ``get_feature_names_out`` for more details.
+
+        .. versionadded:: 1.1
+
     kw_args : dict, default=None
         Dictionary of additional keyword arguments to pass to func.
 
@@ -73,6 +108,33 @@ class FunctionTransformer(TransformerMixin, BaseEstimator):
 
         .. versionadded:: 0.18
 
+    Attributes
+    ----------
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X` has feature
+        names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    MaxAbsScaler : Scale each feature by its maximum absolute value.
+    StandardScaler : Standardize features by removing the mean and
+        scaling to unit variance.
+    LabelBinarizer : Binarize labels in a one-vs-all fashion.
+    MultiLabelBinarizer : Transform between iterable of iterables
+        and a multilabel format.
+
+    Notes
+    -----
+    If `func` returns an output with a `columns` attribute, then the columns is enforced
+    to be consistent with the output of `get_feature_names_out`.
+
     Examples
     --------
     >>> import numpy as np
@@ -80,37 +142,82 @@ class FunctionTransformer(TransformerMixin, BaseEstimator):
     >>> transformer = FunctionTransformer(np.log1p)
     >>> X = np.array([[0, 1], [2, 3]])
     >>> transformer.transform(X)
-    array([[0.       , 0.6931...],
-           [1.0986..., 1.3862...]])
+    array([[0.       , 0.6931],
+           [1.0986, 1.3862]])
     """
 
-    @_deprecate_positional_args
-    def __init__(self, func=None, inverse_func=None, *, validate=False,
-                 accept_sparse=False, check_inverse=True, kw_args=None,
-                 inv_kw_args=None):
+    _parameter_constraints: dict = {
+        "func": [callable, None],
+        "inverse_func": [callable, None],
+        "validate": ["boolean"],
+        "accept_sparse": ["boolean"],
+        "check_inverse": ["boolean"],
+        "feature_names_out": [callable, StrOptions({"one-to-one"}), None],
+        "kw_args": [dict, None],
+        "inv_kw_args": [dict, None],
+    }
+
+    def __init__(
+        self,
+        func=None,
+        inverse_func=None,
+        *,
+        validate=False,
+        accept_sparse=False,
+        check_inverse=True,
+        feature_names_out=None,
+        kw_args=None,
+        inv_kw_args=None,
+    ):
         self.func = func
         self.inverse_func = inverse_func
         self.validate = validate
         self.accept_sparse = accept_sparse
         self.check_inverse = check_inverse
+        self.feature_names_out = feature_names_out
         self.kw_args = kw_args
         self.inv_kw_args = inv_kw_args
 
-    def _check_input(self, X):
+    def _check_input(self, X, *, reset):
         if self.validate:
-            return self._validate_data(X, accept_sparse=self.accept_sparse)
+            return validate_data(self, X, accept_sparse=self.accept_sparse, reset=reset)
+        elif reset:
+            # Set feature_names_in_ and n_features_in_ even if validate=False
+            # We run this only when reset==True to store the attributes but not
+            # validate them, because validate=False
+            _check_n_features(self, X, reset=reset)
+            _check_feature_names(self, X, reset=reset)
         return X
 
     def _check_inverse_transform(self, X):
         """Check that func and inverse_func are the inverse."""
         idx_selected = slice(None, None, max(1, X.shape[0] // 100))
         X_round_trip = self.inverse_transform(self.transform(X[idx_selected]))
-        if not _allclose_dense_sparse(X[idx_selected], X_round_trip):
-            warnings.warn("The provided functions are not strictly"
-                          " inverse of each other. If you are sure you"
-                          " want to proceed regardless, set"
-                          " 'check_inverse=False'.", UserWarning)
 
+        if hasattr(X, "dtype"):
+            dtypes = [X.dtype]
+        elif hasattr(X, "dtypes"):
+            # Dataframes can have multiple dtypes
+            dtypes = X.dtypes
+
+        if not all(np.issubdtype(d, np.number) for d in dtypes):
+            raise ValueError(
+                "'check_inverse' is only supported when all the elements in `X` is"
+                " numerical."
+            )
+
+        if not _allclose_dense_sparse(X[idx_selected], X_round_trip):
+            warnings.warn(
+                (
+                    "The provided functions are not strictly"
+                    " inverse of each other. If you are sure you"
+                    " want to proceed regardless, set"
+                    " 'check_inverse=False'."
+                ),
+                UserWarning,
+            )
+
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y=None):
         """Fit transformer by checking X.
 
@@ -118,16 +225,20 @@ def fit(self, X, y=None):
 
         Parameters
         ----------
-        X : array-like, shape (n_samples, n_features)
+        X : {array-like, sparse-matrix} of shape (n_samples, n_features) \
+                if `validate=True` else any object that `func` can handle
             Input array.
 
+        y : Ignored
+            Not used, present here for API consistency by convention.
+
         Returns
         -------
-        self
+        self : object
+            FunctionTransformer class instance.
         """
-        X = self._check_input(X)
-        if (self.check_inverse and not (self.func is None or
-                                        self.inverse_func is None)):
+        X = self._check_input(X, reset=True)
+        if self.check_inverse and not (self.func is None or self.inverse_func is None):
             self._check_inverse_transform(X)
         return self
 
@@ -136,7 +247,8 @@ def transform(self, X):
 
         Parameters
         ----------
-        X : array-like, shape (n_samples, n_features)
+        X : {array-like, sparse-matrix} of shape (n_samples, n_features) \
+                if `validate=True` else any object that `func` can handle
             Input array.
 
         Returns
@@ -144,32 +256,191 @@ def transform(self, X):
         X_out : array-like, shape (n_samples, n_features)
             Transformed input.
         """
-        return self._transform(X, func=self.func, kw_args=self.kw_args)
+        X = self._check_input(X, reset=False)
+        out = self._transform(X, func=self.func, kw_args=self.kw_args)
+        output_config = _get_output_config("transform", self)["dense"]
+
+        if hasattr(out, "columns") and self.feature_names_out is not None:
+            # check the consistency between the column provided by `transform` and
+            # the column names provided by `get_feature_names_out`.
+            feature_names_out = self.get_feature_names_out()
+            if list(out.columns) != list(feature_names_out):
+                # we can override the column names of the output if it is inconsistent
+                # with the column names provided by `get_feature_names_out` in the
+                # following cases:
+                # * `func` preserved the column names between the input and the output
+                # * the input column names are all numbers
+                # * the output is requested to be a DataFrame (pandas or polars)
+                feature_names_in = getattr(
+                    X, "feature_names_in_", _get_feature_names(X)
+                )
+                same_feature_names_in_out = feature_names_in is not None and list(
+                    feature_names_in
+                ) == list(out.columns)
+                not_all_str_columns = not all(
+                    isinstance(col, str) for col in out.columns
+                )
+                if same_feature_names_in_out or not_all_str_columns:
+                    adapter = _get_adapter_from_container(out)
+                    out = adapter.create_container(
+                        X_output=out,
+                        X_original=out,
+                        columns=feature_names_out,
+                        inplace=False,
+                    )
+                else:
+                    raise ValueError(
+                        "The output generated by `func` have different column names "
+                        "than the ones provided by `get_feature_names_out`. "
+                        f"Got output with columns names: {list(out.columns)} and "
+                        "`get_feature_names_out` returned: "
+                        f"{list(self.get_feature_names_out())}. "
+                        "The column names can be overridden by setting "
+                        "`set_output(transform='pandas')` or "
+                        "`set_output(transform='polars')` such that the column names "
+                        "are set to the names provided by `get_feature_names_out`."
+                    )
+
+        if self.feature_names_out is None:
+            warn_msg = (
+                "When `set_output` is configured to be '{0}', `func` should return "
+                "a {0} DataFrame to follow the `set_output` API  or `feature_names_out`"
+                " should be defined."
+            )
+            if output_config == "pandas" and not _is_pandas_df(out):
+                warnings.warn(warn_msg.format("pandas"))
+            elif output_config == "polars" and not _is_polars_df(out):
+                warnings.warn(warn_msg.format("polars"))
+
+        return out
 
     def inverse_transform(self, X):
         """Transform X using the inverse function.
 
         Parameters
         ----------
-        X : array-like, shape (n_samples, n_features)
+        X : {array-like, sparse-matrix} of shape (n_samples, n_features) \
+                if `validate=True` else any object that `inverse_func` can handle
             Input array.
 
         Returns
         -------
-        X_out : array-like, shape (n_samples, n_features)
+        X_original : array-like, shape (n_samples, n_features)
             Transformed input.
         """
-        return self._transform(X, func=self.inverse_func,
-                               kw_args=self.inv_kw_args)
+        if self.validate:
+            X = check_array(X, accept_sparse=self.accept_sparse)
+        return self._transform(X, func=self.inverse_func, kw_args=self.inv_kw_args)
 
-    def _transform(self, X, func=None, kw_args=None):
-        X = self._check_input(X)
+    @available_if(lambda self: self.feature_names_out is not None)
+    def get_feature_names_out(self, input_features=None):
+        """Get output feature names for transformation.
+
+        This method is only defined if `feature_names_out` is not None.
+
+        Parameters
+        ----------
+        input_features : array-like of str or None, default=None
+            Input feature names.
+
+            - If `input_features` is None, then `feature_names_in_` is
+              used as the input feature names. If `feature_names_in_` is not
+              defined, then names are generated:
+              `[x0, x1, ..., x(n_features_in_ - 1)]`.
+            - If `input_features` is array-like, then `input_features` must
+              match `feature_names_in_` if `feature_names_in_` is defined.
+
+        Returns
+        -------
+        feature_names_out : ndarray of str objects
+            Transformed feature names.
+
+            - If `feature_names_out` is 'one-to-one', the input feature names
+              are returned (see `input_features` above). This requires
+              `feature_names_in_` and/or `n_features_in_` to be defined, which
+              is done automatically if `validate=True`. Alternatively, you can
+              set them in `func`.
+            - If `feature_names_out` is a callable, then it is called with two
+              arguments, `self` and `input_features`, and its return value is
+              returned by this method.
+        """
+        if hasattr(self, "n_features_in_") or input_features is not None:
+            input_features = _check_feature_names_in(self, input_features)
+        if self.feature_names_out == "one-to-one":
+            names_out = input_features
+        elif callable(self.feature_names_out):
+            names_out = self.feature_names_out(self, input_features)
+        else:
+            raise ValueError(
+                f"feature_names_out={self.feature_names_out!r} is invalid. "
+                'It must either be "one-to-one" or a callable with two '
+                "arguments: the function transformer and an array-like of "
+                "input feature names. The callable must return an array-like "
+                "of output feature names."
+            )
+        return np.asarray(names_out, dtype=object)
 
+    def _transform(self, X, func=None, kw_args=None):
         if func is None:
             func = _identity
 
         return func(X, **(kw_args if kw_args else {}))
 
-    def _more_tags(self):
-        return {'no_validation': not self.validate,
-                'stateless': True}
+    def __sklearn_is_fitted__(self):
+        """Return True since FunctionTransfomer is stateless."""
+        return True
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.no_validation = not self.validate
+        tags.requires_fit = False
+        tags.input_tags.sparse = not self.validate or self.accept_sparse
+        return tags
+
+    def set_output(self, *, transform=None):
+        """Set output container.
+
+        See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py`
+        for an example on how to use the API.
+
+        Parameters
+        ----------
+        transform : {"default", "pandas", "polars"}, default=None
+            Configure output of `transform` and `fit_transform`.
+
+            - `"default"`: Default output format of a transformer
+            - `"pandas"`: DataFrame output
+            - `"polars"`: Polars output
+            - `None`: Transform configuration is unchanged
+
+            .. versionadded:: 1.4
+                `"polars"` option was added.
+
+        Returns
+        -------
+        self : estimator instance
+            Estimator instance.
+        """
+        if not hasattr(self, "_sklearn_output_config"):
+            self._sklearn_output_config = {}
+
+        self._sklearn_output_config["transform"] = transform
+        return self
+
+    def _get_function_name(self):
+        """Get the name display of the `func` used in HTML representation."""
+        if hasattr(self.func, "__name__"):
+            return self.func.__name__
+        if isinstance(self.func, partial):
+            return self.func.func.__name__
+        return f"{self.func.__class__.__name__}(...)"
+
+    def _sk_visual_block_(self):
+        return _VisualBlock(
+            "single",
+            self,
+            names=self._get_function_name(),
+            name_details=str(self),
+            name_caption="FunctionTransformer",
+            doc_link_label="FunctionTransformer",
+        )
diff --git a/sklearn/preprocessing/_label.py b/sklearn/preprocessing/_label.py
index 2b43dfffe716d..dd721b35a3521 100644
--- a/sklearn/preprocessing/_label.py
+++ b/sklearn/preprocessing/_label.py
@@ -1,41 +1,33 @@
-# Authors: Alexandre Gramfort 
-#          Mathieu Blondel 
-#          Olivier Grisel 
-#          Andreas Mueller 
-#          Joel Nothman 
-#          Hamzeh Alsalhi 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-from collections import defaultdict
-import itertools
 import array
+import itertools
 import warnings
+from collections import defaultdict
+from numbers import Integral
 
 import numpy as np
 import scipy.sparse as sp
 
-from ..base import BaseEstimator, TransformerMixin
-
-from ..utils.sparsefuncs import min_max_axis
+from ..base import BaseEstimator, TransformerMixin, _fit_context
 from ..utils import column_or_1d
-from ..utils.validation import check_array
-from ..utils.validation import check_is_fitted
-from ..utils.validation import _num_samples
-from ..utils.validation import _deprecate_positional_args
-from ..utils.multiclass import unique_labels
-from ..utils.multiclass import type_of_target
+from ..utils._array_api import device, get_namespace, xpx
 from ..utils._encode import _encode, _unique
-
+from ..utils._param_validation import Interval, validate_params
+from ..utils.multiclass import type_of_target, unique_labels
+from ..utils.sparsefuncs import min_max_axis
+from ..utils.validation import _num_samples, check_array, check_is_fitted
 
 __all__ = [
-    'label_binarize',
-    'LabelBinarizer',
-    'LabelEncoder',
-    'MultiLabelBinarizer',
+    "LabelBinarizer",
+    "LabelEncoder",
+    "MultiLabelBinarizer",
+    "label_binarize",
 ]
 
 
-class LabelEncoder(TransformerMixin, BaseEstimator):
+class LabelEncoder(TransformerMixin, BaseEstimator, auto_wrap_output_keys=None):
     """Encode target labels with value between 0 and n_classes-1.
 
     This transformer should be used to encode target values, *i.e.* `y`, and
@@ -50,12 +42,18 @@ class LabelEncoder(TransformerMixin, BaseEstimator):
     classes_ : ndarray of shape (n_classes,)
         Holds the label for each class.
 
+    See Also
+    --------
+    OrdinalEncoder : Encode categorical features using an ordinal encoding
+        scheme.
+    OneHotEncoder : Encode categorical features as a one-hot numeric array.
+
     Examples
     --------
     `LabelEncoder` can be used to normalize labels.
 
-    >>> from sklearn import preprocessing
-    >>> le = preprocessing.LabelEncoder()
+    >>> from sklearn.preprocessing import LabelEncoder
+    >>> le = LabelEncoder()
     >>> le.fit([1, 2, 2, 6])
     LabelEncoder()
     >>> le.classes_
@@ -68,21 +66,15 @@ class LabelEncoder(TransformerMixin, BaseEstimator):
     It can also be used to transform non-numerical labels (as long as they are
     hashable and comparable) to numerical labels.
 
-    >>> le = preprocessing.LabelEncoder()
+    >>> le = LabelEncoder()
     >>> le.fit(["paris", "paris", "tokyo", "amsterdam"])
     LabelEncoder()
     >>> list(le.classes_)
-    ['amsterdam', 'paris', 'tokyo']
+    [np.str_('amsterdam'), np.str_('paris'), np.str_('tokyo')]
     >>> le.transform(["tokyo", "tokyo", "paris"])
     array([2, 2, 1]...)
     >>> list(le.inverse_transform([2, 2, 1]))
-    ['tokyo', 'tokyo', 'paris']
-
-    See Also
-    --------
-    OrdinalEncoder : Encode categorical features using an ordinal encoding
-        scheme.
-    OneHotEncoder : Encode categorical features as a one-hot numeric array.
+    [np.str_('tokyo'), np.str_('tokyo'), np.str_('paris')]
     """
 
     def fit(self, y):
@@ -96,6 +88,7 @@ def fit(self, y):
         Returns
         -------
         self : returns an instance of self.
+            Fitted label encoder.
         """
         y = column_or_1d(y, warn=True)
         self.classes_ = _unique(y)
@@ -112,6 +105,7 @@ def fit_transform(self, y):
         Returns
         -------
         y : array-like of shape (n_samples,)
+            Encoded labels.
         """
         y = column_or_1d(y, warn=True)
         self.classes_, y = _unique(y, return_inverse=True)
@@ -128,12 +122,14 @@ def transform(self, y):
         Returns
         -------
         y : array-like of shape (n_samples,)
+            Labels as normalized encodings.
         """
         check_is_fitted(self)
-        y = column_or_1d(y, warn=True)
+        xp, _ = get_namespace(y)
+        y = column_or_1d(y, dtype=self.classes_.dtype, warn=True)
         # transform of empty array is empty array
         if _num_samples(y) == 0:
-            return np.array([])
+            return xp.asarray([])
 
         return _encode(y, uniques=self.classes_)
 
@@ -142,31 +138,40 @@ def inverse_transform(self, y):
 
         Parameters
         ----------
-        y : ndarray of shape (n_samples,)
+        y : array-like of shape (n_samples,)
             Target values.
 
         Returns
         -------
-        y : ndarray of shape (n_samples,)
+        y_original : ndarray of shape (n_samples,)
+            Original encoding.
         """
         check_is_fitted(self)
+        xp, _ = get_namespace(y)
         y = column_or_1d(y, warn=True)
         # inverse transform of empty array is empty array
         if _num_samples(y) == 0:
-            return np.array([])
-
-        diff = np.setdiff1d(y, np.arange(len(self.classes_)))
-        if len(diff):
-            raise ValueError(
-                    "y contains previously unseen labels: %s" % str(diff))
-        y = np.asarray(y)
-        return self.classes_[y]
-
-    def _more_tags(self):
-        return {'X_types': ['1dlabels']}
-
-
-class LabelBinarizer(TransformerMixin, BaseEstimator):
+            return xp.asarray([])
+
+        diff = xpx.setdiff1d(
+            y,
+            xp.arange(self.classes_.shape[0], device=device(y)),
+            xp=xp,
+        )
+        if diff.shape[0]:
+            raise ValueError("y contains previously unseen labels: %s" % str(diff))
+        y = xp.asarray(y)
+        return xp.take(self.classes_, y, axis=0)
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.array_api_support = True
+        tags.input_tags.two_d_array = False
+        tags.target_tags.one_d_labels = True
+        return tags
+
+
+class LabelBinarizer(TransformerMixin, BaseEstimator, auto_wrap_output_keys=None):
     """Binarize labels in a one-vs-all fashion.
 
     Several regression and binary classification algorithms are
@@ -177,18 +182,17 @@ class LabelBinarizer(TransformerMixin, BaseEstimator):
     At learning time, this simply consists in learning one regressor
     or binary classifier per class. In doing so, one needs to convert
     multi-class labels to binary labels (belong or does not belong
-    to the class). LabelBinarizer makes this process easy with the
+    to the class). `LabelBinarizer` makes this process easy with the
     transform method.
 
     At prediction time, one assigns the class for which the corresponding
-    model gave the greatest confidence. LabelBinarizer makes this easy
-    with the inverse_transform method.
+    model gave the greatest confidence. `LabelBinarizer` makes this easy
+    with the :meth:`inverse_transform` method.
 
     Read more in the :ref:`User Guide `.
 
     Parameters
     ----------
-
     neg_label : int, default=0
         Value with which negative labels must be encoded.
 
@@ -201,24 +205,30 @@ class LabelBinarizer(TransformerMixin, BaseEstimator):
 
     Attributes
     ----------
-
     classes_ : ndarray of shape (n_classes,)
         Holds the label for each class.
 
     y_type_ : str
         Represents the type of the target data as evaluated by
-        utils.multiclass.type_of_target. Possible type are 'continuous',
-        'continuous-multioutput', 'binary', 'multiclass',
+        :func:`~sklearn.utils.multiclass.type_of_target`. Possible type are
+        'continuous', 'continuous-multioutput', 'binary', 'multiclass',
         'multiclass-multioutput', 'multilabel-indicator', and 'unknown'.
 
     sparse_input_ : bool
-        True if the input data to transform is given as a sparse matrix, False
-        otherwise.
+        `True` if the input data to transform is given as a sparse matrix,
+         `False` otherwise.
+
+    See Also
+    --------
+    label_binarize : Function to perform the transform operation of
+        LabelBinarizer with fixed classes.
+    OneHotEncoder : Encode categorical features using a one-hot aka one-of-K
+        scheme.
 
     Examples
     --------
-    >>> from sklearn import preprocessing
-    >>> lb = preprocessing.LabelBinarizer()
+    >>> from sklearn.preprocessing import LabelBinarizer
+    >>> lb = LabelBinarizer()
     >>> lb.fit([1, 2, 6, 4, 2])
     LabelBinarizer()
     >>> lb.classes_
@@ -229,7 +239,7 @@ class LabelBinarizer(TransformerMixin, BaseEstimator):
 
     Binary targets transform to a column vector
 
-    >>> lb = preprocessing.LabelBinarizer()
+    >>> lb = LabelBinarizer()
     >>> lb.fit_transform(['yes', 'no', 'no', 'yes'])
     array([[1],
            [0],
@@ -248,31 +258,20 @@ class LabelBinarizer(TransformerMixin, BaseEstimator):
            [0, 1, 0],
            [0, 0, 1],
            [0, 1, 0]])
-
-    See Also
-    --------
-    label_binarize : Function to perform the transform operation of
-        LabelBinarizer with fixed classes.
-    OneHotEncoder : Encode categorical features using a one-hot aka one-of-K
-        scheme.
     """
 
-    @_deprecate_positional_args
-    def __init__(self, *, neg_label=0, pos_label=1, sparse_output=False):
-        if neg_label >= pos_label:
-            raise ValueError("neg_label={0} must be strictly less than "
-                             "pos_label={1}.".format(neg_label, pos_label))
-
-        if sparse_output and (pos_label == 0 or neg_label != 0):
-            raise ValueError("Sparse binarization is only supported with non "
-                             "zero pos_label and zero neg_label, got "
-                             "pos_label={0} and neg_label={1}"
-                             "".format(pos_label, neg_label))
+    _parameter_constraints: dict = {
+        "neg_label": [Integral],
+        "pos_label": [Integral],
+        "sparse_output": ["boolean"],
+    }
 
+    def __init__(self, *, neg_label=0, pos_label=1, sparse_output=False):
         self.neg_label = neg_label
         self.pos_label = pos_label
         self.sparse_output = sparse_output
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, y):
         """Fit label binarizer.
 
@@ -284,22 +283,37 @@ def fit(self, y):
 
         Returns
         -------
-        self : returns an instance of self.
+        self : object
+            Returns the instance itself.
         """
-        self.y_type_ = type_of_target(y)
-        if 'multioutput' in self.y_type_:
-            raise ValueError("Multioutput target data is not supported with "
-                             "label binarization")
+        if self.neg_label >= self.pos_label:
+            raise ValueError(
+                f"neg_label={self.neg_label} must be strictly less than "
+                f"pos_label={self.pos_label}."
+            )
+
+        if self.sparse_output and (self.pos_label == 0 or self.neg_label != 0):
+            raise ValueError(
+                "Sparse binarization is only supported with non "
+                "zero pos_label and zero neg_label, got "
+                f"pos_label={self.pos_label} and neg_label={self.neg_label}"
+            )
+
+        self.y_type_ = type_of_target(y, input_name="y")
+
+        if "multioutput" in self.y_type_:
+            raise ValueError(
+                "Multioutput target data is not supported with label binarization"
+            )
         if _num_samples(y) == 0:
-            raise ValueError('y has 0 samples: %r' % y)
+            raise ValueError("y has 0 samples: %r" % y)
 
         self.sparse_input_ = sp.issparse(y)
         self.classes_ = unique_labels(y)
         return self
 
     def fit_transform(self, y):
-        """Fit label binarizer and transform multi-class labels to binary
-        labels.
+        """Fit label binarizer/transform multi-class labels to binary labels.
 
         The output of transform is sometimes referred to as
         the 1-of-K coding scheme.
@@ -342,15 +356,17 @@ def transform(self, y):
         """
         check_is_fitted(self)
 
-        y_is_multilabel = type_of_target(y).startswith('multilabel')
-        if y_is_multilabel and not self.y_type_.startswith('multilabel'):
-            raise ValueError("The object was not fitted with multilabel"
-                             " input.")
+        y_is_multilabel = type_of_target(y).startswith("multilabel")
+        if y_is_multilabel and not self.y_type_.startswith("multilabel"):
+            raise ValueError("The object was not fitted with multilabel input.")
 
-        return label_binarize(y, classes=self.classes_,
-                              pos_label=self.pos_label,
-                              neg_label=self.neg_label,
-                              sparse_output=self.sparse_output)
+        return label_binarize(
+            y,
+            classes=self.classes_,
+            pos_label=self.pos_label,
+            neg_label=self.neg_label,
+            sparse_output=self.sparse_output,
+        )
 
     def inverse_transform(self, Y, threshold=None):
         """Transform binary labels back to multi-class labels.
@@ -364,36 +380,37 @@ def inverse_transform(self, Y, threshold=None):
         threshold : float, default=None
             Threshold used in the binary and multi-label cases.
 
-            Use 0 when ``Y`` contains the output of decision_function
+            Use 0 when ``Y`` contains the output of :term:`decision_function`
             (classifier).
-            Use 0.5 when ``Y`` contains the output of predict_proba.
+            Use 0.5 when ``Y`` contains the output of :term:`predict_proba`.
 
             If None, the threshold is assumed to be half way between
             neg_label and pos_label.
 
         Returns
         -------
-        y : {ndarray, sparse matrix} of shape (n_samples,)
+        y_original : {ndarray, sparse matrix} of shape (n_samples,)
             Target values. Sparse matrix will be of CSR format.
 
         Notes
         -----
         In the case when the binary labels are fractional
-        (probabilistic), inverse_transform chooses the class with the
+        (probabilistic), :meth:`inverse_transform` chooses the class with the
         greatest value. Typically, this allows to use the output of a
-        linear model's decision_function method directly as the input
-        of inverse_transform.
+        linear model's :term:`decision_function` method directly as the input
+        of :meth:`inverse_transform`.
         """
         check_is_fitted(self)
 
         if threshold is None:
-            threshold = (self.pos_label + self.neg_label) / 2.
+            threshold = (self.pos_label + self.neg_label) / 2.0
 
         if self.y_type_ == "multiclass":
             y_inv = _inverse_binarize_multiclass(Y, self.classes_)
         else:
-            y_inv = _inverse_binarize_thresholding(Y, self.y_type_,
-                                                   self.classes_, threshold)
+            y_inv = _inverse_binarize_thresholding(
+                Y, self.y_type_, self.classes_, threshold
+            )
 
         if self.sparse_input_:
             y_inv = sp.csr_matrix(y_inv)
@@ -402,13 +419,24 @@ def inverse_transform(self, Y, threshold=None):
 
         return y_inv
 
-    def _more_tags(self):
-        return {'X_types': ['1dlabels']}
-
-
-@_deprecate_positional_args
-def label_binarize(y, *, classes, neg_label=0, pos_label=1,
-                   sparse_output=False):
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.two_d_array = False
+        tags.target_tags.one_d_labels = True
+        return tags
+
+
+@validate_params(
+    {
+        "y": ["array-like", "sparse matrix"],
+        "classes": ["array-like"],
+        "neg_label": [Interval(Integral, None, None, closed="neither")],
+        "pos_label": [Interval(Integral, None, None, closed="neither")],
+        "sparse_output": ["boolean"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def label_binarize(y, *, classes, neg_label=0, pos_label=1, sparse_output=False):
     """Binarize labels in a one-vs-all fashion.
 
     Several regression and binary classification algorithms are
@@ -421,7 +449,7 @@ def label_binarize(y, *, classes, neg_label=0, pos_label=1,
 
     Parameters
     ----------
-    y : array-like
+    y : array-like or sparse matrix
         Sequence of integer labels or multilabel data to encode.
 
     classes : array-like of shape (n_classes,)
@@ -442,6 +470,11 @@ def label_binarize(y, *, classes, neg_label=0, pos_label=1,
         Shape will be (n_samples, 1) for binary problems. Sparse matrix will
         be of CSR format.
 
+    See Also
+    --------
+    LabelBinarizer : Class used to wrap the functionality of label_binarize and
+        allow for fitting to classes independently of the transform operation.
+
     Examples
     --------
     >>> from sklearn.preprocessing import label_binarize
@@ -462,28 +495,30 @@ def label_binarize(y, *, classes, neg_label=0, pos_label=1,
            [0],
            [0],
            [1]])
-
-    See Also
-    --------
-    LabelBinarizer : Class used to wrap the functionality of label_binarize and
-        allow for fitting to classes independently of the transform operation.
     """
     if not isinstance(y, list):
         # XXX Workaround that will be removed when list of list format is
         # dropped
-        y = check_array(y, accept_sparse='csr', ensure_2d=False, dtype=None)
+        y = check_array(
+            y, input_name="y", accept_sparse="csr", ensure_2d=False, dtype=None
+        )
     else:
         if _num_samples(y) == 0:
-            raise ValueError('y has 0 samples: %r' % y)
+            raise ValueError("y has 0 samples: %r" % y)
     if neg_label >= pos_label:
-        raise ValueError("neg_label={0} must be strictly less than "
-                         "pos_label={1}.".format(neg_label, pos_label))
-
-    if (sparse_output and (pos_label == 0 or neg_label != 0)):
-        raise ValueError("Sparse binarization is only supported with non "
-                         "zero pos_label and zero neg_label, got "
-                         "pos_label={0} and neg_label={1}"
-                         "".format(pos_label, neg_label))
+        raise ValueError(
+            "neg_label={0} must be strictly less than pos_label={1}.".format(
+                neg_label, pos_label
+            )
+        )
+
+    if sparse_output and (pos_label == 0 or neg_label != 0):
+        raise ValueError(
+            "Sparse binarization is only supported with non "
+            "zero pos_label and zero neg_label, got "
+            "pos_label={0} and neg_label={1}"
+            "".format(pos_label, neg_label)
+        )
 
     # To account for pos_label == 0 in the dense case
     pos_switch = pos_label == 0
@@ -491,10 +526,11 @@ def label_binarize(y, *, classes, neg_label=0, pos_label=1,
         pos_label = -neg_label
 
     y_type = type_of_target(y)
-    if 'multioutput' in y_type:
-        raise ValueError("Multioutput target data is not supported with label "
-                         "binarization")
-    if y_type == 'unknown':
+    if "multioutput" in y_type:
+        raise ValueError(
+            "Multioutput target data is not supported with label binarization"
+        )
+    if y_type == "unknown":
         raise ValueError("The type of target data is not known")
 
     n_samples = y.shape[0] if sp.issparse(y) else len(y)
@@ -514,25 +550,26 @@ def label_binarize(y, *, classes, neg_label=0, pos_label=1,
 
     sorted_class = np.sort(classes)
     if y_type == "multilabel-indicator":
-        y_n_classes = y.shape[1] if hasattr(y, 'shape') else len(y[0])
+        y_n_classes = y.shape[1] if hasattr(y, "shape") else len(y[0])
         if classes.size != y_n_classes:
-            raise ValueError("classes {0} mismatch with the labels {1}"
-                             " found in the data"
-                             .format(classes, unique_labels(y)))
+            raise ValueError(
+                "classes {0} mismatch with the labels {1} found in the data".format(
+                    classes, unique_labels(y)
+                )
+            )
 
     if y_type in ("binary", "multiclass"):
         y = column_or_1d(y)
 
         # pick out the known labels from y
-        y_in_classes = np.in1d(y, classes)
+        y_in_classes = np.isin(y, classes)
         y_seen = y[y_in_classes]
         indices = np.searchsorted(sorted_class, y_seen)
         indptr = np.hstack((0, np.cumsum(y_in_classes)))
 
         data = np.empty_like(indices)
         data.fill(pos_label)
-        Y = sp.csr_matrix((data, indices, indptr),
-                          shape=(n_samples, n_classes))
+        Y = sp.csr_matrix((data, indices, indptr), shape=(n_samples, n_classes))
     elif y_type == "multilabel-indicator":
         Y = sp.csr_matrix(y)
         if pos_label != 1:
@@ -540,8 +577,9 @@ def label_binarize(y, *, classes, neg_label=0, pos_label=1,
             data.fill(pos_label)
             Y.data = data
     else:
-        raise ValueError("%s target data is not supported with label "
-                         "binarization" % y_type)
+        raise ValueError(
+            "%s target data is not supported with label binarization" % y_type
+        )
 
     if not sparse_output:
         Y = Y.toarray()
@@ -562,7 +600,7 @@ def label_binarize(y, *, classes, neg_label=0, pos_label=1,
 
     if y_type == "binary":
         if sparse_output:
-            Y = Y.getcol(-1)
+            Y = Y[:, [-1]]
         else:
             Y = Y[:, -1].reshape((-1, 1))
 
@@ -602,10 +640,9 @@ def _inverse_binarize_multiclass(y, classes):
         y_i_argmax[np.where(row_nnz == 0)[0]] = 0
 
         # Handles rows with max of 0 that contain negative numbers
-        samples = np.arange(n_samples)[(row_nnz > 0) &
-                                       (row_max.ravel() == 0)]
+        samples = np.arange(n_samples)[(row_nnz > 0) & (row_max.ravel() == 0)]
         for i in samples:
-            ind = y.indices[y.indptr[i]:y.indptr[i + 1]]
+            ind = y.indices[y.indptr[i] : y.indptr[i + 1]]
             y_i_argmax[i] = classes[np.setdiff1d(outputs, ind)][0]
 
         return classes[y_i_argmax]
@@ -617,19 +654,19 @@ def _inverse_binarize_thresholding(y, output_type, classes, threshold):
     """Inverse label binarization transformation using thresholding."""
 
     if output_type == "binary" and y.ndim == 2 and y.shape[1] > 2:
-        raise ValueError("output_type='binary', but y.shape = {0}".
-                         format(y.shape))
+        raise ValueError("output_type='binary', but y.shape = {0}".format(y.shape))
 
     if output_type != "binary" and y.shape[1] != len(classes):
-        raise ValueError("The number of class is not equal to the number of "
-                         "dimension of y.")
+        raise ValueError(
+            "The number of class is not equal to the number of dimension of y."
+        )
 
     classes = np.asarray(classes)
 
     # Perform thresholding
     if sp.issparse(y):
         if threshold > 0:
-            if y.format not in ('csr', 'csc'):
+            if y.format not in ("csr", "csc"):
                 y = y.tocsr()
             y.data = np.array(y.data > threshold, dtype=int)
             y.eliminate_zeros()
@@ -657,7 +694,7 @@ def _inverse_binarize_thresholding(y, output_type, classes, threshold):
         raise ValueError("{0} format is not supported".format(output_type))
 
 
-class MultiLabelBinarizer(TransformerMixin, BaseEstimator):
+class MultiLabelBinarizer(TransformerMixin, BaseEstimator, auto_wrap_output_keys=None):
     """Transform between iterable of iterables and a multilabel format.
 
     Although a list of sets or tuples is a very intuitive format for multilabel
@@ -681,6 +718,11 @@ class MultiLabelBinarizer(TransformerMixin, BaseEstimator):
         Otherwise it corresponds to the sorted set of classes found
         when fitting.
 
+    See Also
+    --------
+    OneHotEncoder : Encode categorical features using a one-hot aka one-of-K
+        scheme.
+
     Examples
     --------
     >>> from sklearn.preprocessing import MultiLabelBinarizer
@@ -713,18 +755,18 @@ class MultiLabelBinarizer(TransformerMixin, BaseEstimator):
     MultiLabelBinarizer()
     >>> mlb.classes_
     array(['comedy', 'sci-fi', 'thriller'], dtype=object)
-
-    See Also
-    --------
-    OneHotEncoder : Encode categorical features using a one-hot aka one-of-K
-        scheme.
     """
 
-    @_deprecate_positional_args
+    _parameter_constraints: dict = {
+        "classes": ["array-like", None],
+        "sparse_output": ["boolean"],
+    }
+
     def __init__(self, *, classes=None, sparse_output=False):
         self.classes = classes
         self.sparse_output = sparse_output
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, y):
         """Fit the label sets binarizer, storing :term:`classes_`.
 
@@ -737,15 +779,19 @@ def fit(self, y):
 
         Returns
         -------
-        self : returns this MultiLabelBinarizer instance
+        self : object
+            Fitted estimator.
         """
         self._cached_dict = None
+
         if self.classes is None:
             classes = sorted(set(itertools.chain.from_iterable(y)))
         elif len(set(self.classes)) < len(self.classes):
-            raise ValueError("The classes argument contains duplicate "
-                             "classes. Remove these duplicates before passing "
-                             "them to MultiLabelBinarizer.")
+            raise ValueError(
+                "The classes argument contains duplicate "
+                "classes. Remove these duplicates before passing "
+                "them to MultiLabelBinarizer."
+            )
         else:
             classes = self.classes
         dtype = int if all(isinstance(c, int) for c in classes) else object
@@ -753,6 +799,7 @@ def fit(self, y):
         self.classes_[:] = classes
         return self
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit_transform(self, y):
         """Fit the label sets binarizer and transform the given label sets.
 
@@ -766,15 +813,15 @@ def fit_transform(self, y):
         Returns
         -------
         y_indicator : {ndarray, sparse matrix} of shape (n_samples, n_classes)
-            A matrix such that `y_indicator[i, j] = 1` i.f.f. `classes_[j]`
+            A matrix such that `y_indicator[i, j] = 1` iff `classes_[j]`
             is in `y[i]`, and 0 otherwise. Sparse matrix will be of CSR
             format.
         """
-        self._cached_dict = None
-
         if self.classes is not None:
             return self.fit(y).transform(y)
 
+        self._cached_dict = None
+
         # Automatically increment on new class
         class_mapping = defaultdict(int)
         class_mapping.default_factory = class_mapping.__len__
@@ -789,8 +836,7 @@ def fit_transform(self, y):
         class_mapping[:] = tmp
         self.classes_, inverse = np.unique(class_mapping, return_inverse=True)
         # ensure yt.indices keeps its current dtype
-        yt.indices = np.array(inverse[yt.indices], dtype=yt.indices.dtype,
-                              copy=False)
+        yt.indices = np.asarray(inverse[yt.indices], dtype=yt.indices.dtype)
 
         if not self.sparse_output:
             yt = yt.toarray()
@@ -825,17 +871,20 @@ def transform(self, y):
 
     def _build_cache(self):
         if self._cached_dict is None:
-            self._cached_dict = dict(zip(self.classes_,
-                                         range(len(self.classes_))))
+            self._cached_dict = dict(zip(self.classes_, range(len(self.classes_))))
 
         return self._cached_dict
 
     def _transform(self, y, class_mapping):
-        """Transforms the label sets with a given mapping
+        """Transforms the label sets with a given mapping.
 
         Parameters
         ----------
         y : iterable of iterables
+            A set of labels (any orderable and hashable object) for each
+            sample. If the `classes` parameter is set, `y` will not be
+            iterated.
+
         class_mapping : Mapping
             Maps from label to column index in label indicator matrix.
 
@@ -844,8 +893,8 @@ def _transform(self, y, class_mapping):
         y_indicator : sparse matrix of shape (n_samples, n_classes)
             Label indicator matrix. Will be of CSR format.
         """
-        indices = array.array('i')
-        indptr = array.array('i', [0])
+        indices = array.array("i")
+        indptr = array.array("i", [0])
         unknown = set()
         for labels in y:
             index = set()
@@ -857,12 +906,14 @@ def _transform(self, y, class_mapping):
             indices.extend(index)
             indptr.append(len(indices))
         if unknown:
-            warnings.warn('unknown class(es) {0} will be ignored'
-                          .format(sorted(unknown, key=str)))
+            warnings.warn(
+                "unknown class(es) {0} will be ignored".format(sorted(unknown, key=str))
+            )
         data = np.ones(len(indices), dtype=int)
 
-        return sp.csr_matrix((data, indices, indptr),
-                             shape=(len(indptr) - 1, len(class_mapping)))
+        return sp.csr_matrix(
+            (data, indices, indptr), shape=(len(indptr) - 1, len(class_mapping))
+        )
 
     def inverse_transform(self, yt):
         """Transform the given indicator matrix into label sets.
@@ -874,29 +925,39 @@ def inverse_transform(self, yt):
 
         Returns
         -------
-        y : list of tuples
+        y_original : list of tuples
             The set of labels for each sample such that `y[i]` consists of
             `classes_[j]` for each `yt[i, j] == 1`.
         """
         check_is_fitted(self)
 
         if yt.shape[1] != len(self.classes_):
-            raise ValueError('Expected indicator for {0} classes, but got {1}'
-                             .format(len(self.classes_), yt.shape[1]))
+            raise ValueError(
+                "Expected indicator for {0} classes, but got {1}".format(
+                    len(self.classes_), yt.shape[1]
+                )
+            )
 
         if sp.issparse(yt):
             yt = yt.tocsr()
             if len(yt.data) != 0 and len(np.setdiff1d(yt.data, [0, 1])) > 0:
-                raise ValueError('Expected only 0s and 1s in label indicator.')
-            return [tuple(self.classes_.take(yt.indices[start:end]))
-                    for start, end in zip(yt.indptr[:-1], yt.indptr[1:])]
+                raise ValueError("Expected only 0s and 1s in label indicator.")
+            return [
+                tuple(self.classes_.take(yt.indices[start:end]))
+                for start, end in zip(yt.indptr[:-1], yt.indptr[1:])
+            ]
         else:
             unexpected = np.setdiff1d(yt, [0, 1])
             if len(unexpected) > 0:
-                raise ValueError('Expected only 0s and 1s in label indicator. '
-                                 'Also got {0}'.format(unexpected))
-            return [tuple(self.classes_.compress(indicators)) for indicators
-                    in yt]
-
-    def _more_tags(self):
-        return {'X_types': ['2dlabels']}
+                raise ValueError(
+                    "Expected only 0s and 1s in label indicator. Also got {0}".format(
+                        unexpected
+                    )
+                )
+            return [tuple(self.classes_.compress(indicators)) for indicators in yt]
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.two_d_array = False
+        tags.target_tags.two_d_labels = True
+        return tags
diff --git a/sklearn/preprocessing/_polynomial.py b/sklearn/preprocessing/_polynomial.py
index 26587e7f05823..69bfe7b212bba 100644
--- a/sklearn/preprocessing/_polynomial.py
+++ b/sklearn/preprocessing/_polynomial.py
@@ -1,31 +1,588 @@
 """
 This file contains preprocessing tools based on polynomials.
 """
-import numbers
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+import collections
+from itertools import chain, combinations
+from itertools import combinations_with_replacement as combinations_w_r
+from numbers import Integral
 
 import numpy as np
+from scipy import sparse
 from scipy.interpolate import BSpline
+from scipy.special import comb
 
-from ..base import BaseEstimator, TransformerMixin
+from ..base import BaseEstimator, TransformerMixin, _fit_context
 from ..utils import check_array
-from ..utils.fixes import linspace
-from ..utils.validation import check_is_fitted, FLOAT_DTYPES
-
+from ..utils._param_validation import Interval, StrOptions
+from ..utils.fixes import parse_version, sp_version
+from ..utils.stats import _weighted_percentile
+from ..utils.validation import (
+    FLOAT_DTYPES,
+    _check_feature_names_in,
+    _check_sample_weight,
+    check_is_fitted,
+    validate_data,
+)
+from ._csr_polynomial_expansion import (
+    _calc_expanded_nnz,
+    _calc_total_nnz,
+    _csr_polynomial_expansion,
+)
 
 __all__ = [
+    "PolynomialFeatures",
     "SplineTransformer",
 ]
 
 
-# TODO:
-# - sparse support (either scipy or own cython solution)?
-# - extrapolation (cyclic)
+def _create_expansion(X, interaction_only, deg, n_features, cumulative_size=0):
+    """Helper function for creating and appending sparse expansion matrices"""
+
+    total_nnz = _calc_total_nnz(X.indptr, interaction_only, deg)
+    expanded_col = _calc_expanded_nnz(n_features, interaction_only, deg)
+
+    if expanded_col == 0:
+        return None
+    # This only checks whether each block needs 64bit integers upon
+    # expansion. We prefer to keep int32 indexing where we can,
+    # since currently SciPy's CSR construction downcasts when possible,
+    # so we prefer to avoid an unnecessary cast. The dtype may still
+    # change in the concatenation process if needed.
+    # See: https://github.com/scipy/scipy/issues/16569
+    max_indices = expanded_col - 1
+    max_indptr = total_nnz
+    max_int32 = np.iinfo(np.int32).max
+    needs_int64 = max(max_indices, max_indptr) > max_int32
+    index_dtype = np.int64 if needs_int64 else np.int32
+
+    # Result of the expansion, modified in place by the
+    # `_csr_polynomial_expansion` routine.
+    expanded_data = np.empty(shape=total_nnz, dtype=X.data.dtype)
+    expanded_indices = np.empty(shape=total_nnz, dtype=index_dtype)
+    expanded_indptr = np.empty(shape=X.indptr.shape[0], dtype=index_dtype)
+    _csr_polynomial_expansion(
+        X.data,
+        X.indices,
+        X.indptr,
+        X.shape[1],
+        expanded_data,
+        expanded_indices,
+        expanded_indptr,
+        interaction_only,
+        deg,
+    )
+    return sparse.csr_matrix(
+        (expanded_data, expanded_indices, expanded_indptr),
+        shape=(X.indptr.shape[0] - 1, expanded_col),
+        dtype=X.dtype,
+    )
+
+
+class PolynomialFeatures(TransformerMixin, BaseEstimator):
+    """Generate polynomial and interaction features.
+
+    Generate a new feature matrix consisting of all polynomial combinations
+    of the features with degree less than or equal to the specified degree.
+    For example, if an input sample is two dimensional and of the form
+    [a, b], the degree-2 polynomial features are [1, a, b, a^2, ab, b^2].
+
+    Read more in the :ref:`User Guide `.
+
+    Parameters
+    ----------
+    degree : int or tuple (min_degree, max_degree), default=2
+        If a single int is given, it specifies the maximal degree of the
+        polynomial features. If a tuple `(min_degree, max_degree)` is passed,
+        then `min_degree` is the minimum and `max_degree` is the maximum
+        polynomial degree of the generated features. Note that `min_degree=0`
+        and `min_degree=1` are equivalent as outputting the degree zero term is
+        determined by `include_bias`.
+
+    interaction_only : bool, default=False
+        If `True`, only interaction features are produced: features that are
+        products of at most `degree` *distinct* input features, i.e. terms with
+        power of 2 or higher of the same input feature are excluded:
+
+        - included: `x[0]`, `x[1]`, `x[0] * x[1]`, etc.
+        - excluded: `x[0] ** 2`, `x[0] ** 2 * x[1]`, etc.
+
+    include_bias : bool, default=True
+        If `True` (default), then include a bias column, the feature in which
+        all polynomial powers are zero (i.e. a column of ones - acts as an
+        intercept term in a linear model).
+
+    order : {'C', 'F'}, default='C'
+        Order of output array in the dense case. `'F'` order is faster to
+        compute, but may slow down subsequent estimators.
+
+        .. versionadded:: 0.21
+
+    Attributes
+    ----------
+    powers_ : ndarray of shape (`n_output_features_`, `n_features_in_`)
+        `powers_[i, j]` is the exponent of the jth input in the ith output.
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    n_output_features_ : int
+        The total number of polynomial output features. The number of output
+        features is computed by iterating over all suitably sized combinations
+        of input features.
+
+    See Also
+    --------
+    SplineTransformer : Transformer that generates univariate B-spline bases
+        for features.
+
+    Notes
+    -----
+    Be aware that the number of features in the output array scales
+    polynomially in the number of features of the input array, and
+    exponentially in the degree. High degrees can cause overfitting.
+
+    See :ref:`examples/linear_model/plot_polynomial_interpolation.py
+    `
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from sklearn.preprocessing import PolynomialFeatures
+    >>> X = np.arange(6).reshape(3, 2)
+    >>> X
+    array([[0, 1],
+           [2, 3],
+           [4, 5]])
+    >>> poly = PolynomialFeatures(2)
+    >>> poly.fit_transform(X)
+    array([[ 1.,  0.,  1.,  0.,  0.,  1.],
+           [ 1.,  2.,  3.,  4.,  6.,  9.],
+           [ 1.,  4.,  5., 16., 20., 25.]])
+    >>> poly = PolynomialFeatures(interaction_only=True)
+    >>> poly.fit_transform(X)
+    array([[ 1.,  0.,  1.,  0.],
+           [ 1.,  2.,  3.,  6.],
+           [ 1.,  4.,  5., 20.]])
+    """
+
+    _parameter_constraints: dict = {
+        "degree": [Interval(Integral, 0, None, closed="left"), "array-like"],
+        "interaction_only": ["boolean"],
+        "include_bias": ["boolean"],
+        "order": [StrOptions({"C", "F"})],
+    }
+
+    def __init__(
+        self, degree=2, *, interaction_only=False, include_bias=True, order="C"
+    ):
+        self.degree = degree
+        self.interaction_only = interaction_only
+        self.include_bias = include_bias
+        self.order = order
+
+    @staticmethod
+    def _combinations(
+        n_features, min_degree, max_degree, interaction_only, include_bias
+    ):
+        comb = combinations if interaction_only else combinations_w_r
+        start = max(1, min_degree)
+        iter = chain.from_iterable(
+            comb(range(n_features), i) for i in range(start, max_degree + 1)
+        )
+        if include_bias:
+            iter = chain(comb(range(n_features), 0), iter)
+        return iter
+
+    @staticmethod
+    def _num_combinations(
+        n_features, min_degree, max_degree, interaction_only, include_bias
+    ):
+        """Calculate number of terms in polynomial expansion
+
+        This should be equivalent to counting the number of terms returned by
+        _combinations(...) but much faster.
+        """
+
+        if interaction_only:
+            combinations = sum(
+                [
+                    comb(n_features, i, exact=True)
+                    for i in range(max(1, min_degree), min(max_degree, n_features) + 1)
+                ]
+            )
+        else:
+            combinations = comb(n_features + max_degree, max_degree, exact=True) - 1
+            if min_degree > 0:
+                d = min_degree - 1
+                combinations -= comb(n_features + d, d, exact=True) - 1
+
+        if include_bias:
+            combinations += 1
+
+        return combinations
+
+    @property
+    def powers_(self):
+        """Exponent for each of the inputs in the output."""
+        check_is_fitted(self)
+
+        combinations = self._combinations(
+            n_features=self.n_features_in_,
+            min_degree=self._min_degree,
+            max_degree=self._max_degree,
+            interaction_only=self.interaction_only,
+            include_bias=self.include_bias,
+        )
+        return np.vstack(
+            [np.bincount(c, minlength=self.n_features_in_) for c in combinations]
+        )
+
+    def get_feature_names_out(self, input_features=None):
+        """Get output feature names for transformation.
+
+        Parameters
+        ----------
+        input_features : array-like of str or None, default=None
+            Input features.
+
+            - If `input_features is None`, then `feature_names_in_` is
+              used as feature names in. If `feature_names_in_` is not defined,
+              then the following input feature names are generated:
+              `["x0", "x1", ..., "x(n_features_in_ - 1)"]`.
+            - If `input_features` is an array-like, then `input_features` must
+              match `feature_names_in_` if `feature_names_in_` is defined.
+
+        Returns
+        -------
+        feature_names_out : ndarray of str objects
+            Transformed feature names.
+        """
+        powers = self.powers_
+        input_features = _check_feature_names_in(self, input_features)
+        feature_names = []
+        for row in powers:
+            inds = np.where(row)[0]
+            if len(inds):
+                name = " ".join(
+                    (
+                        "%s^%d" % (input_features[ind], exp)
+                        if exp != 1
+                        else input_features[ind]
+                    )
+                    for ind, exp in zip(inds, row[inds])
+                )
+            else:
+                name = "1"
+            feature_names.append(name)
+        return np.asarray(feature_names, dtype=object)
+
+    @_fit_context(prefer_skip_nested_validation=True)
+    def fit(self, X, y=None):
+        """
+        Compute number of output features.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            The data.
+
+        y : Ignored
+            Not used, present here for API consistency by convention.
+
+        Returns
+        -------
+        self : object
+            Fitted transformer.
+        """
+        _, n_features = validate_data(self, X, accept_sparse=True).shape
+
+        if isinstance(self.degree, Integral):
+            if self.degree == 0 and not self.include_bias:
+                raise ValueError(
+                    "Setting degree to zero and include_bias to False would result in"
+                    " an empty output array."
+                )
+
+            self._min_degree = 0
+            self._max_degree = self.degree
+        elif (
+            isinstance(self.degree, collections.abc.Iterable) and len(self.degree) == 2
+        ):
+            self._min_degree, self._max_degree = self.degree
+            if not (
+                isinstance(self._min_degree, Integral)
+                and isinstance(self._max_degree, Integral)
+                and self._min_degree >= 0
+                and self._min_degree <= self._max_degree
+            ):
+                raise ValueError(
+                    "degree=(min_degree, max_degree) must "
+                    "be non-negative integers that fulfil "
+                    "min_degree <= max_degree, got "
+                    f"{self.degree}."
+                )
+            elif self._max_degree == 0 and not self.include_bias:
+                raise ValueError(
+                    "Setting both min_degree and max_degree to zero and include_bias to"
+                    " False would result in an empty output array."
+                )
+        else:
+            raise ValueError(
+                "degree must be a non-negative int or tuple "
+                "(min_degree, max_degree), got "
+                f"{self.degree}."
+            )
+
+        self.n_output_features_ = self._num_combinations(
+            n_features=n_features,
+            min_degree=self._min_degree,
+            max_degree=self._max_degree,
+            interaction_only=self.interaction_only,
+            include_bias=self.include_bias,
+        )
+        if self.n_output_features_ > np.iinfo(np.intp).max:
+            msg = (
+                "The output that would result from the current configuration would"
+                f" have {self.n_output_features_} features which is too large to be"
+                f" indexed by {np.intp().dtype.name}. Please change some or all of the"
+                " following:\n- The number of features in the input, currently"
+                f" {n_features=}\n- The range of degrees to calculate, currently"
+                f" [{self._min_degree}, {self._max_degree}]\n- Whether to include only"
+                f" interaction terms, currently {self.interaction_only}\n- Whether to"
+                f" include a bias term, currently {self.include_bias}."
+            )
+            if (
+                np.intp == np.int32
+                and self.n_output_features_ <= np.iinfo(np.int64).max
+            ):  # pragma: nocover
+                msg += (
+                    "\nNote that the current Python runtime has a limited 32 bit "
+                    "address space and that this configuration would have been "
+                    "admissible if run on a 64 bit Python runtime."
+                )
+            raise ValueError(msg)
+        # We also record the number of output features for
+        # _min_degree = 0
+        self._n_out_full = self._num_combinations(
+            n_features=n_features,
+            min_degree=0,
+            max_degree=self._max_degree,
+            interaction_only=self.interaction_only,
+            include_bias=self.include_bias,
+        )
+
+        return self
+
+    def transform(self, X):
+        """Transform data to polynomial features.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            The data to transform, row by row.
+
+            Prefer CSR over CSC for sparse input (for speed), but CSC is
+            required if the degree is 4 or higher. If the degree is less than
+            4 and the input format is CSC, it will be converted to CSR, have
+            its polynomial features generated, then converted back to CSC.
+
+            If the degree is 2 or 3, the method described in "Leveraging
+            Sparsity to Speed Up Polynomial Feature Expansions of CSR Matrices
+            Using K-Simplex Numbers" by Andrew Nystrom and John Hughes is
+            used, which is much faster than the method used on CSC input. For
+            this reason, a CSC input will be converted to CSR, and the output
+            will be converted back to CSC prior to being returned, hence the
+            preference of CSR.
+
+        Returns
+        -------
+        XP : {ndarray, sparse matrix} of shape (n_samples, NP)
+            The matrix of features, where `NP` is the number of polynomial
+            features generated from the combination of inputs. If a sparse
+            matrix is provided, it will be converted into a sparse
+            `csr_matrix`.
+        """
+        check_is_fitted(self)
+
+        X = validate_data(
+            self,
+            X,
+            order="F",
+            dtype=FLOAT_DTYPES,
+            reset=False,
+            accept_sparse=("csr", "csc"),
+        )
+
+        n_samples, n_features = X.shape
+        max_int32 = np.iinfo(np.int32).max
+        if sparse.issparse(X) and X.format == "csr":
+            if self._max_degree > 3:
+                return self.transform(X.tocsc()).tocsr()
+            to_stack = []
+            if self.include_bias:
+                to_stack.append(
+                    sparse.csr_matrix(np.ones(shape=(n_samples, 1), dtype=X.dtype))
+                )
+            if self._min_degree <= 1 and self._max_degree > 0:
+                to_stack.append(X)
+
+            cumulative_size = sum(mat.shape[1] for mat in to_stack)
+            for deg in range(max(2, self._min_degree), self._max_degree + 1):
+                expanded = _create_expansion(
+                    X=X,
+                    interaction_only=self.interaction_only,
+                    deg=deg,
+                    n_features=n_features,
+                    cumulative_size=cumulative_size,
+                )
+                if expanded is not None:
+                    to_stack.append(expanded)
+                    cumulative_size += expanded.shape[1]
+            if len(to_stack) == 0:
+                # edge case: deal with empty matrix
+                XP = sparse.csr_matrix((n_samples, 0), dtype=X.dtype)
+            else:
+                # `scipy.sparse.hstack` breaks in scipy<1.9.2
+                # when `n_output_features_ > max_int32`
+                all_int32 = all(mat.indices.dtype == np.int32 for mat in to_stack)
+                if (
+                    sp_version < parse_version("1.9.2")
+                    and self.n_output_features_ > max_int32
+                    and all_int32
+                ):
+                    raise ValueError(  # pragma: no cover
+                        "In scipy versions `<1.9.2`, the function `scipy.sparse.hstack`"
+                        " produces negative columns when:\n1. The output shape contains"
+                        " `n_cols` too large to be represented by a 32bit signed"
+                        " integer.\n2. All sub-matrices to be stacked have indices of"
+                        " dtype `np.int32`.\nTo avoid this error, either use a version"
+                        " of scipy `>=1.9.2` or alter the `PolynomialFeatures`"
+                        " transformer to produce fewer than 2^31 output features"
+                    )
+                XP = sparse.hstack(to_stack, dtype=X.dtype, format="csr")
+        elif sparse.issparse(X) and X.format == "csc" and self._max_degree < 4:
+            return self.transform(X.tocsr()).tocsc()
+        elif sparse.issparse(X):
+            combinations = self._combinations(
+                n_features=n_features,
+                min_degree=self._min_degree,
+                max_degree=self._max_degree,
+                interaction_only=self.interaction_only,
+                include_bias=self.include_bias,
+            )
+            columns = []
+            for combi in combinations:
+                if combi:
+                    out_col = 1
+                    for col_idx in combi:
+                        out_col = X[:, [col_idx]].multiply(out_col)
+                    columns.append(out_col)
+                else:
+                    bias = sparse.csc_matrix(np.ones((X.shape[0], 1)))
+                    columns.append(bias)
+            XP = sparse.hstack(columns, dtype=X.dtype).tocsc()
+        else:
+            # Do as if _min_degree = 0 and cut down array after the
+            # computation, i.e. use _n_out_full instead of n_output_features_.
+            XP = np.empty(
+                shape=(n_samples, self._n_out_full), dtype=X.dtype, order=self.order
+            )
+
+            # What follows is a faster implementation of:
+            # for i, comb in enumerate(combinations):
+            #     XP[:, i] = X[:, comb].prod(1)
+            # This implementation uses two optimisations.
+            # First one is broadcasting,
+            # multiply ([X1, ..., Xn], X1) -> [X1 X1, ..., Xn X1]
+            # multiply ([X2, ..., Xn], X2) -> [X2 X2, ..., Xn X2]
+            # ...
+            # multiply ([X[:, start:end], X[:, start]) -> ...
+            # Second optimisation happens for degrees >= 3.
+            # Xi^3 is computed reusing previous computation:
+            # Xi^3 = Xi^2 * Xi.
+
+            # degree 0 term
+            if self.include_bias:
+                XP[:, 0] = 1
+                current_col = 1
+            else:
+                current_col = 0
+
+            if self._max_degree == 0:
+                return XP
+
+            # degree 1 term
+            XP[:, current_col : current_col + n_features] = X
+            index = list(range(current_col, current_col + n_features))
+            current_col += n_features
+            index.append(current_col)
+
+            # loop over degree >= 2 terms
+            for _ in range(2, self._max_degree + 1):
+                new_index = []
+                end = index[-1]
+                for feature_idx in range(n_features):
+                    start = index[feature_idx]
+                    new_index.append(current_col)
+                    if self.interaction_only:
+                        start += index[feature_idx + 1] - index[feature_idx]
+                    next_col = current_col + end - start
+                    if next_col <= current_col:
+                        break
+                    # XP[:, start:end] are terms of degree d - 1
+                    # that exclude feature #feature_idx.
+                    np.multiply(
+                        XP[:, start:end],
+                        X[:, feature_idx : feature_idx + 1],
+                        out=XP[:, current_col:next_col],
+                        casting="no",
+                    )
+                    current_col = next_col
+
+                new_index.append(current_col)
+                index = new_index
+
+            if self._min_degree > 1:
+                n_XP, n_Xout = self._n_out_full, self.n_output_features_
+                if self.include_bias:
+                    Xout = np.empty(
+                        shape=(n_samples, n_Xout), dtype=XP.dtype, order=self.order
+                    )
+                    Xout[:, 0] = 1
+                    Xout[:, 1:] = XP[:, n_XP - n_Xout + 1 :]
+                else:
+                    Xout = XP[:, n_XP - n_Xout :].copy()
+                XP = Xout
+        return XP
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
+        return tags
+
+
 class SplineTransformer(TransformerMixin, BaseEstimator):
     """Generate univariate B-spline bases for features.
 
     Generate a new feature matrix consisting of
-    `n_splines=n_knots + degree - 1` spline basis functions (B-splines) of
-    polynomial order=`degree` for each feature.
+    `n_splines=n_knots + degree - 1` (`n_knots - 1` for
+    `extrapolation="periodic"`) spline basis functions
+    (B-splines) of polynomial order=`degree` for each feature.
+
+    In order to learn more about the SplineTransformer class go to:
+    :ref:`sphx_glr_auto_examples_applications_plot_cyclical_feature_engineering.py`
 
     Read more in the :ref:`User Guide `.
 
@@ -35,7 +592,8 @@ class SplineTransformer(TransformerMixin, BaseEstimator):
     ----------
     n_knots : int, default=5
         Number of knots of the splines if `knots` equals one of
-        {'uniform', 'quantile'}. Must be larger or equal 2.
+        {'uniform', 'quantile'}. Must be larger or equal 2. Ignored if `knots`
+        is array-like.
 
     degree : int, default=3
         The polynomial degree of the spline basis. Must be a non-negative
@@ -54,24 +612,36 @@ class SplineTransformer(TransformerMixin, BaseEstimator):
           `degree` number of knots are added before the first knot, the same
           after the last knot.
 
-    extrapolation : {'error', 'constant', 'linear', 'continue'}, \
+    extrapolation : {'error', 'constant', 'linear', 'continue', 'periodic'}, \
         default='constant'
         If 'error', values outside the min and max values of the training
         features raises a `ValueError`. If 'constant', the value of the
         splines at minimum and maximum value of the features is used as
         constant extrapolation. If 'linear', a linear extrapolation is used.
         If 'continue', the splines are extrapolated as is, i.e. option
-        `extrapolate=True` in :class:`scipy.interpolate.BSpline`.
+        `extrapolate=True` in :class:`scipy.interpolate.BSpline`. If
+        'periodic', periodic splines with a periodicity equal to the distance
+        between the first and last knot are used. Periodic splines enforce
+        equal function values and derivatives at the first and last knot.
+        For example, this makes it possible to avoid introducing an arbitrary
+        jump between Dec 31st and Jan 1st in spline features derived from a
+        naturally periodic "day-of-year" input feature. In this case it is
+        recommended to manually set the knot values to control the period.
 
     include_bias : bool, default=True
-        If True (default), then the last spline element inside the data range
+        If False, then the last spline element inside the data range
         of a feature is dropped. As B-splines sum to one over the spline basis
         functions for each data point, they implicitly include a bias term,
         i.e. a column of ones. It acts as an intercept term in a linear models.
 
     order : {'C', 'F'}, default='C'
-        Order of output array. 'F' order is faster to compute, but may slow
-        down subsequent estimators.
+        Order of output array in the dense case. `'F'` order is faster to compute, but
+        may slow down subsequent estimators.
+
+    sparse_output : bool, default=False
+        Will return sparse CSR matrix if set True else will return an array.
+
+        .. versionadded:: 1.2
 
     Attributes
     ----------
@@ -81,10 +651,18 @@ class SplineTransformer(TransformerMixin, BaseEstimator):
     n_features_in_ : int
         The total number of input features.
 
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     n_features_out_ : int
         The total number of output features, which is computed as
         `n_features * n_splines`, where `n_splines` is
-        the number of bases elements of the B-splines, `n_knots + degree - 1`.
+        the number of bases elements of the B-splines,
+        `n_knots + degree - 1` for non-periodic splines and
+        `n_knots - 1` for periodic ones.
         If `include_bias=False`, then it is only
         `n_features * (n_splines - 1)`.
 
@@ -117,6 +695,18 @@ class SplineTransformer(TransformerMixin, BaseEstimator):
            [0.  , 0.  , 0.5 , 0.5 ]])
     """
 
+    _parameter_constraints: dict = {
+        "n_knots": [Interval(Integral, 2, None, closed="left")],
+        "degree": [Interval(Integral, 0, None, closed="left")],
+        "knots": [StrOptions({"uniform", "quantile"}), "array-like"],
+        "extrapolation": [
+            StrOptions({"error", "constant", "linear", "continue", "periodic"})
+        ],
+        "include_bias": ["boolean"],
+        "order": [StrOptions({"C", "F"})],
+        "sparse_output": ["boolean"],
+    }
+
     def __init__(
         self,
         n_knots=5,
@@ -126,6 +716,7 @@ def __init__(
         extrapolation="constant",
         include_bias=True,
         order="C",
+        sparse_output=False,
     ):
         self.n_knots = n_knots
         self.degree = degree
@@ -133,9 +724,10 @@ def __init__(
         self.extrapolation = extrapolation
         self.include_bias = include_bias
         self.order = order
+        self.sparse_output = sparse_output
 
     @staticmethod
-    def _get_base_knot_positions(X, n_knots=10, knots="uniform"):
+    def _get_base_knot_positions(X, n_knots=10, knots="uniform", sample_weight=None):
         """Calculate base knot positions.
 
         Base knots such that first knot <= feature <= last knot. For the
@@ -148,19 +740,30 @@ def _get_base_knot_positions(X, n_knots=10, knots="uniform"):
             Knot positions (points) of base interval.
         """
         if knots == "quantile":
-            knots = np.percentile(
-                X,
-                100
-                * np.linspace(start=0, stop=1, num=n_knots, dtype=np.float64),
-                axis=0,
+            percentile_ranks = 100 * np.linspace(
+                start=0, stop=1, num=n_knots, dtype=np.float64
             )
+
+            if sample_weight is None:
+                knots = np.percentile(X, percentile_ranks, axis=0)
+            else:
+                knots = np.array(
+                    [
+                        _weighted_percentile(X, sample_weight, percentile_rank)
+                        for percentile_rank in percentile_ranks
+                    ]
+                )
+
         else:
             # knots == 'uniform':
             # Note that the variable `knots` has already been validated and
             # `else` is therefore safe.
-            x_min = np.amin(X, axis=0)
-            x_max = np.amax(X, axis=0)
-            knots = linspace(
+            # Disregard observations with zero weight.
+            mask = slice(None, None, 1) if sample_weight is None else sample_weight > 0
+            x_min = np.amin(X[mask], axis=0)
+            x_max = np.amax(X[mask], axis=0)
+
+            knots = np.linspace(
                 start=x_min,
                 stop=x_max,
                 num=n_knots,
@@ -170,29 +773,38 @@ def _get_base_knot_positions(X, n_knots=10, knots="uniform"):
 
         return knots
 
-    def get_feature_names(self, input_features=None):
-        """Return feature names for output features.
+    def get_feature_names_out(self, input_features=None):
+        """Get output feature names for transformation.
 
         Parameters
         ----------
-        input_features : list of str of shape (n_features,), default=None
-            String names for input features if available. By default,
-            "x0", "x1", ... "xn_features" is used.
+        input_features : array-like of str or None, default=None
+            Input features.
+
+            - If `input_features` is `None`, then `feature_names_in_` is
+              used as feature names in. If `feature_names_in_` is not defined,
+              then the following input feature names are generated:
+              `["x0", "x1", ..., "x(n_features_in_ - 1)"]`.
+            - If `input_features` is an array-like, then `input_features` must
+              match `feature_names_in_` if `feature_names_in_` is defined.
 
         Returns
         -------
-        output_feature_names : list of str of shape (n_output_features,)
+        feature_names_out : ndarray of str objects
+            Transformed feature names.
         """
-        n_splines = self.bsplines_[0].c.shape[0]
-        if input_features is None:
-            input_features = ["x%d" % i for i in range(self.n_features_in_)]
+        check_is_fitted(self, "n_features_in_")
+        n_splines = self.bsplines_[0].c.shape[1]
+
+        input_features = _check_feature_names_in(self, input_features)
         feature_names = []
         for i in range(self.n_features_in_):
             for j in range(n_splines - 1 + self.include_bias):
                 feature_names.append(f"{input_features[i]}_sp_{j}")
-        return feature_names
+        return np.asarray(feature_names, dtype=object)
 
-    def fit(self, X, y=None):
+    @_fit_context(prefer_skip_nested_validation=True)
+    def fit(self, X, y=None, sample_weight=None):
         """Compute knot positions of splines.
 
         Parameters
@@ -203,102 +815,112 @@ def fit(self, X, y=None):
         y : None
             Ignored.
 
+        sample_weight : array-like of shape (n_samples,), default = None
+            Individual weights for each sample. Used to calculate quantiles if
+            `knots="quantile"`. For `knots="uniform"`, zero weighted
+            observations are ignored for finding the min and max of `X`.
+
         Returns
         -------
         self : object
             Fitted transformer.
         """
-        X = self._validate_data(
+        X = validate_data(
+            self,
             X,
             reset=True,
             accept_sparse=False,
             ensure_min_samples=2,
             ensure_2d=True,
         )
-        n_samples, n_features = X.shape
+        if sample_weight is not None:
+            sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype)
 
-        if not (
-            isinstance(self.degree, numbers.Integral) and self.degree >= 0
-        ):
-            raise ValueError("degree must be a non-negative integer.")
+        _, n_features = X.shape
 
-        if not (
-            isinstance(self.n_knots, numbers.Integral) and self.n_knots >= 2
-        ):
-            raise ValueError("n_knots must be a positive integer >= 2.")
-
-        if isinstance(self.knots, str) and self.knots in [
-            "uniform",
-            "quantile",
-        ]:
+        if isinstance(self.knots, str):
             base_knots = self._get_base_knot_positions(
-                X, n_knots=self.n_knots, knots=self.knots
+                X, n_knots=self.n_knots, knots=self.knots, sample_weight=sample_weight
             )
         else:
-            base_knots = check_array(self.knots)
+            base_knots = check_array(self.knots, dtype=np.float64)
             if base_knots.shape[0] < 2:
-                raise ValueError(
-                    "Number of knots, knots.shape[0], must be >= " "2."
-                )
+                raise ValueError("Number of knots, knots.shape[0], must be >= 2.")
             elif base_knots.shape[1] != n_features:
                 raise ValueError("knots.shape[1] == n_features is violated.")
             elif not np.all(np.diff(base_knots, axis=0) > 0):
                 raise ValueError("knots must be sorted without duplicates.")
 
-        if self.extrapolation not in (
-            "error",
-            "constant",
-            "linear",
-            "continue",
-        ):
+        # number of knots for base interval
+        n_knots = base_knots.shape[0]
+
+        if self.extrapolation == "periodic" and n_knots <= self.degree:
             raise ValueError(
-                "extrapolation must be one of 'error', "
-                "'constant', 'linear' or 'continue'."
+                "Periodic splines require degree < n_knots. Got n_knots="
+                f"{n_knots} and degree={self.degree}."
             )
 
-        if not isinstance(self.include_bias, (bool, np.bool_)):
-            raise ValueError("include_bias must be bool.")
-
-        # number of knots for base interval
-        n_knots = base_knots.shape[0]
         # number of splines basis functions
-        n_splines = n_knots + self.degree - 1
+        if self.extrapolation != "periodic":
+            n_splines = n_knots + self.degree - 1
+        else:
+            # periodic splines have self.degree less degrees of freedom
+            n_splines = n_knots - 1
+
         degree = self.degree
         n_out = n_features * n_splines
         # We have to add degree number of knots below, and degree number knots
         # above the base knots in order to make the spline basis complete.
-        # Eilers & Marx in "Flexible smoothing with B-splines and  penalties"
-        # https://doi.org/10.1214/ss/1038425655 advice against repeating first
-        # and last knot several times, which would have inferior behaviour at
-        # boundaries if combined with a penalty (hence P-Spline). We follow
-        # this advice even if our splines are unpenalized.
-        # Meaning we do not:
-        # knots = np.r_[np.tile(base_knots.min(axis=0), reps=[degree, 1]),
-        #              base_knots,
-        #              np.tile(base_knots.max(axis=0), reps=[degree, 1])
-        #              ]
-        # Instead, we reuse the distance of the 2 fist/last knots.
-        dist_min = base_knots[1] - base_knots[0]
-        dist_max = base_knots[-1] - base_knots[-2]
-        knots = np.r_[
-            linspace(
-                base_knots[0] - degree * dist_min,
-                base_knots[0] - dist_min,
-                num=degree,
-            ),
-            base_knots,
-            linspace(
-                base_knots[-1] + dist_max,
-                base_knots[-1] + degree * dist_max,
-                num=degree,
-            ),
-        ]
+        if self.extrapolation == "periodic":
+            # For periodic splines the spacing of the first / last degree knots
+            # needs to be a continuation of the spacing of the last / first
+            # base knots.
+            period = base_knots[-1] - base_knots[0]
+            knots = np.r_[
+                base_knots[-(degree + 1) : -1] - period,
+                base_knots,
+                base_knots[1 : (degree + 1)] + period,
+            ]
+
+        else:
+            # Eilers & Marx in "Flexible smoothing with B-splines and
+            # penalties" https://doi.org/10.1214/ss/1038425655 advice
+            # against repeating first and last knot several times, which
+            # would have inferior behaviour at boundaries if combined with
+            # a penalty (hence P-Spline). We follow this advice even if our
+            # splines are unpenalized. Meaning we do not:
+            # knots = np.r_[
+            #     np.tile(base_knots.min(axis=0), reps=[degree, 1]),
+            #     base_knots,
+            #     np.tile(base_knots.max(axis=0), reps=[degree, 1])
+            # ]
+            # Instead, we reuse the distance of the 2 fist/last knots.
+            dist_min = base_knots[1] - base_knots[0]
+            dist_max = base_knots[-1] - base_knots[-2]
+
+            knots = np.r_[
+                np.linspace(
+                    base_knots[0] - degree * dist_min,
+                    base_knots[0] - dist_min,
+                    num=degree,
+                ),
+                base_knots,
+                np.linspace(
+                    base_knots[-1] + dist_max,
+                    base_knots[-1] + degree * dist_max,
+                    num=degree,
+                ),
+            ]
 
         # With a diagonal coefficient matrix, we get back the spline basis
         # elements, i.e. the design matrix of the spline.
         # Note, BSpline appreciates C-contiguous float64 arrays as c=coef.
-        coef = np.eye(n_knots + self.degree - 1, dtype=np.float64)
-        extrapolate = self.extrapolation == "continue"
+        coef = np.eye(n_splines, dtype=np.float64)
+        if self.extrapolation == "periodic":
+            coef = np.concatenate((coef, coef[:degree, :]))
+
+        extrapolate = self.extrapolation in ["periodic", "continue"]
+
         bsplines = [
             BSpline.construct_fast(
                 knots[:, i], coef, self.degree, extrapolate=extrapolate
@@ -320,20 +942,31 @@ def transform(self, X):
 
         Returns
         -------
-        XBS : ndarray of shape (n_samples, n_features * n_splines)
+        XBS : {ndarray, sparse matrix} of shape (n_samples, n_features * n_splines)
             The matrix of features, where n_splines is the number of bases
             elements of the B-splines, n_knots + degree - 1.
         """
         check_is_fitted(self)
 
-        X = self._validate_data(
-            X, reset=False, accept_sparse=False, ensure_2d=True
-        )
+        X = validate_data(self, X, reset=False, accept_sparse=False, ensure_2d=True)
 
         n_samples, n_features = X.shape
-        n_splines = self.bsplines_[0].c.shape[0]
+        n_splines = self.bsplines_[0].c.shape[1]
         degree = self.degree
 
+        # TODO: Remove this condition, once scipy 1.10 is the minimum version.
+        #       Only scipy => 1.10 supports design_matrix(.., extrapolate=..).
+        #       The default (implicit in scipy < 1.10) is extrapolate=False.
+        scipy_1_10 = sp_version >= parse_version("1.10.0")
+        # Note: self.bsplines_[0].extrapolate is True for extrapolation in
+        # ["periodic", "continue"]
+        if scipy_1_10:
+            use_sparse = self.sparse_output
+            kwargs_extrapolate = {"extrapolate": self.bsplines_[0].extrapolate}
+        else:
+            use_sparse = self.sparse_output and not self.bsplines_[0].extrapolate
+            kwargs_extrapolate = dict()
+
         # Note that scipy BSpline returns float64 arrays and converts input
         # x=X[:, i] to c-contiguous float64.
         n_out = self.n_features_out_ + n_features * (1 - self.include_bias)
@@ -341,28 +974,74 @@ def transform(self, X):
             dtype = X.dtype
         else:
             dtype = np.float64
-        XBS = np.zeros((n_samples, n_out), dtype=dtype, order=self.order)
+        if use_sparse:
+            output_list = []
+        else:
+            XBS = np.zeros((n_samples, n_out), dtype=dtype, order=self.order)
 
         for i in range(n_features):
             spl = self.bsplines_[i]
 
-            if self.extrapolation in ("continue", "error"):
-                XBS[:, (i * n_splines):((i + 1) * n_splines)] = spl(X[:, i])
-            else:
-                xmin = spl.t[degree]
-                xmax = spl.t[-degree - 1]
+            if self.extrapolation in ("continue", "error", "periodic"):
+                if self.extrapolation == "periodic":
+                    # With periodic extrapolation we map x to the segment
+                    # [spl.t[k], spl.t[n]].
+                    # This is equivalent to BSpline(.., extrapolate="periodic")
+                    # for scipy>=1.0.0.
+                    n = spl.t.size - spl.k - 1
+                    # Assign to new array to avoid inplace operation
+                    x = spl.t[spl.k] + (X[:, i] - spl.t[spl.k]) % (
+                        spl.t[n] - spl.t[spl.k]
+                    )
+                else:
+                    x = X[:, i]
+
+                if use_sparse:
+                    XBS_sparse = BSpline.design_matrix(
+                        x, spl.t, spl.k, **kwargs_extrapolate
+                    )
+                    if self.extrapolation == "periodic":
+                        # See the construction of coef in fit. We need to add the last
+                        # degree spline basis function to the first degree ones and
+                        # then drop the last ones.
+                        # Note: See comment about SparseEfficiencyWarning below.
+                        XBS_sparse = XBS_sparse.tolil()
+                        XBS_sparse[:, :degree] += XBS_sparse[:, -degree:]
+                        XBS_sparse = XBS_sparse[:, :-degree]
+                else:
+                    XBS[:, (i * n_splines) : ((i + 1) * n_splines)] = spl(x)
+            else:  # extrapolation in ("constant", "linear")
+                xmin, xmax = spl.t[degree], spl.t[-degree - 1]
+                # spline values at boundaries
+                f_min, f_max = spl(xmin), spl(xmax)
                 mask = (xmin <= X[:, i]) & (X[:, i] <= xmax)
-                XBS[mask, (i * n_splines):((i + 1) * n_splines)] = spl(
-                    X[mask, i]
-                )
+                if use_sparse:
+                    mask_inv = ~mask
+                    x = X[:, i].copy()
+                    # Set some arbitrary values outside boundary that will be reassigned
+                    # later.
+                    x[mask_inv] = spl.t[self.degree]
+                    XBS_sparse = BSpline.design_matrix(x, spl.t, spl.k)
+                    # Note: Without converting to lil_matrix we would get:
+                    # scipy.sparse._base.SparseEfficiencyWarning: Changing the sparsity
+                    # structure of a csr_matrix is expensive. lil_matrix is more
+                    # efficient.
+                    if np.any(mask_inv):
+                        XBS_sparse = XBS_sparse.tolil()
+                        XBS_sparse[mask_inv, :] = 0
+                else:
+                    XBS[mask, (i * n_splines) : ((i + 1) * n_splines)] = spl(X[mask, i])
 
             # Note for extrapolation:
             # 'continue' is already returned as is by scipy BSplines
             if self.extrapolation == "error":
                 # BSpline with extrapolate=False does not raise an error, but
-                # output np.nan.
-                if np.any(
-                    np.isnan(XBS[:, (i * n_splines):((i + 1) * n_splines)])
+                # outputs np.nan.
+                if (use_sparse and np.any(np.isnan(XBS_sparse.data))) or (
+                    not use_sparse
+                    and np.any(
+                        np.isnan(XBS[:, (i * n_splines) : ((i + 1) * n_splines)])
+                    )
                 ):
                     raise ValueError(
                         "X contains values beyond the limits of the knots."
@@ -373,21 +1052,30 @@ def transform(self, X):
                 # Only the first degree and last degree number of splines
                 # have non-zero values at the boundaries.
 
-                # spline values at boundaries
-                f_min = spl(xmin)
-                f_max = spl(xmax)
                 mask = X[:, i] < xmin
                 if np.any(mask):
-                    XBS[
-                        mask, (i * n_splines):(i * n_splines + degree)
-                    ] = f_min[:degree]
+                    if use_sparse:
+                        # Note: See comment about SparseEfficiencyWarning above.
+                        XBS_sparse = XBS_sparse.tolil()
+                        XBS_sparse[mask, :degree] = f_min[:degree]
+
+                    else:
+                        XBS[mask, (i * n_splines) : (i * n_splines + degree)] = f_min[
+                            :degree
+                        ]
 
                 mask = X[:, i] > xmax
                 if np.any(mask):
-                    XBS[
-                        mask,
-                        ((i + 1) * n_splines - degree):((i + 1) * n_splines),
-                    ] = f_max[-degree:]
+                    if use_sparse:
+                        # Note: See comment about SparseEfficiencyWarning above.
+                        XBS_sparse = XBS_sparse.tolil()
+                        XBS_sparse[mask, -degree:] = f_max[-degree:]
+                    else:
+                        XBS[
+                            mask,
+                            ((i + 1) * n_splines - degree) : ((i + 1) * n_splines),
+                        ] = f_max[-degree:]
+
             elif self.extrapolation == "linear":
                 # Continue the degree first and degree last spline bases
                 # linearly beyond the boundaries, with slope = derivative at
@@ -395,8 +1083,6 @@ def transform(self, X):
                 # Note that all others have derivative = value = 0 at the
                 # boundaries.
 
-                # spline values at boundaries
-                f_min, f_max = spl(xmin), spl(xmax)
                 # spline derivatives = slopes at boundaries
                 fp_min, fp_max = spl(xmin, nu=1), spl(xmax, nu=1)
                 # Compute the linear continuation.
@@ -407,23 +1093,61 @@ def transform(self, X):
                 for j in range(degree):
                     mask = X[:, i] < xmin
                     if np.any(mask):
-                        XBS[mask, i * n_splines + j] = (
-                            f_min[j] + (X[mask, i] - xmin) * fp_min[j]
-                        )
+                        linear_extr = f_min[j] + (X[mask, i] - xmin) * fp_min[j]
+                        if use_sparse:
+                            # Note: See comment about SparseEfficiencyWarning above.
+                            XBS_sparse = XBS_sparse.tolil()
+                            XBS_sparse[mask, j] = linear_extr
+                        else:
+                            XBS[mask, i * n_splines + j] = linear_extr
 
                     mask = X[:, i] > xmax
                     if np.any(mask):
                         k = n_splines - 1 - j
-                        XBS[mask, i * n_splines + k] = (
-                            f_max[k] + (X[mask, i] - xmax) * fp_max[k]
-                        )
+                        linear_extr = f_max[k] + (X[mask, i] - xmax) * fp_max[k]
+                        if use_sparse:
+                            # Note: See comment about SparseEfficiencyWarning above.
+                            XBS_sparse = XBS_sparse.tolil()
+                            XBS_sparse[mask, k : k + 1] = linear_extr[:, None]
+                        else:
+                            XBS[mask, i * n_splines + k] = linear_extr
+
+            if use_sparse:
+                XBS_sparse = XBS_sparse.tocsr()
+                output_list.append(XBS_sparse)
+
+        if use_sparse:
+            # TODO: Remove this conditional error when the minimum supported version of
+            # SciPy is 1.9.2
+            # `scipy.sparse.hstack` breaks in scipy<1.9.2
+            # when `n_features_out_ > max_int32`
+            max_int32 = np.iinfo(np.int32).max
+            all_int32 = True
+            for mat in output_list:
+                all_int32 &= mat.indices.dtype == np.int32
+            if (
+                sp_version < parse_version("1.9.2")
+                and self.n_features_out_ > max_int32
+                and all_int32
+            ):
+                raise ValueError(
+                    "In scipy versions `<1.9.2`, the function `scipy.sparse.hstack`"
+                    " produces negative columns when:\n1. The output shape contains"
+                    " `n_cols` too large to be represented by a 32bit signed"
+                    " integer.\n. All sub-matrices to be stacked have indices of"
+                    " dtype `np.int32`.\nTo avoid this error, either use a version"
+                    " of scipy `>=1.9.2` or alter the `SplineTransformer`"
+                    " transformer to produce fewer than 2^31 output features"
+                )
+            XBS = sparse.hstack(output_list, format="csr")
+        elif self.sparse_output:
+            # TODO: Remove ones scipy 1.10 is the minimum version. See comments above.
+            XBS = sparse.csr_matrix(XBS)
 
         if self.include_bias:
             return XBS
         else:
             # We throw away one spline basis per feature.
             # We chose the last one.
-            indices = [
-                j for j in range(XBS.shape[1]) if (j + 1) % n_splines != 0
-            ]
+            indices = [j for j in range(XBS.shape[1]) if (j + 1) % n_splines != 0]
             return XBS[:, indices]
diff --git a/sklearn/preprocessing/_target_encoder.py b/sklearn/preprocessing/_target_encoder.py
new file mode 100644
index 0000000000000..77b404e3e39e9
--- /dev/null
+++ b/sklearn/preprocessing/_target_encoder.py
@@ -0,0 +1,534 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+from numbers import Integral, Real
+
+import numpy as np
+
+from ..base import OneToOneFeatureMixin, _fit_context
+from ..utils._param_validation import Interval, StrOptions
+from ..utils.multiclass import type_of_target
+from ..utils.validation import (
+    _check_feature_names_in,
+    _check_y,
+    check_consistent_length,
+    check_is_fitted,
+)
+from ._encoders import _BaseEncoder
+from ._target_encoder_fast import _fit_encoding_fast, _fit_encoding_fast_auto_smooth
+
+
+class TargetEncoder(OneToOneFeatureMixin, _BaseEncoder):
+    """Target Encoder for regression and classification targets.
+
+    Each category is encoded based on a shrunk estimate of the average target
+    values for observations belonging to the category. The encoding scheme mixes
+    the global target mean with the target mean conditioned on the value of the
+    category (see [MIC]_).
+
+    When the target type is "multiclass", encodings are based
+    on the conditional probability estimate for each class. The target is first
+    binarized using the "one-vs-all" scheme via
+    :class:`~sklearn.preprocessing.LabelBinarizer`, then the average target
+    value for each class and each category is used for encoding, resulting in
+    `n_features` * `n_classes` encoded output features.
+
+    :class:`TargetEncoder` considers missing values, such as `np.nan` or `None`,
+    as another category and encodes them like any other category. Categories
+    that are not seen during :meth:`fit` are encoded with the target mean, i.e.
+    `target_mean_`.
+
+    For a demo on the importance of the `TargetEncoder` internal cross-fitting,
+    see
+    :ref:`sphx_glr_auto_examples_preprocessing_plot_target_encoder_cross_val.py`.
+    For a comparison of different encoders, refer to
+    :ref:`sphx_glr_auto_examples_preprocessing_plot_target_encoder.py`. Read
+    more in the :ref:`User Guide `.
+
+    .. note::
+        `fit(X, y).transform(X)` does not equal `fit_transform(X, y)` because a
+        :term:`cross fitting` scheme is used in `fit_transform` for encoding.
+        See the :ref:`User Guide ` for details.
+
+    .. versionadded:: 1.3
+
+    Parameters
+    ----------
+    categories : "auto" or list of shape (n_features,) of array-like, default="auto"
+        Categories (unique values) per feature:
+
+        - `"auto"` : Determine categories automatically from the training data.
+        - list : `categories[i]` holds the categories expected in the i-th column. The
+          passed categories should not mix strings and numeric values within a single
+          feature, and should be sorted in case of numeric values.
+
+        The used categories are stored in the `categories_` fitted attribute.
+
+    target_type : {"auto", "continuous", "binary", "multiclass"}, default="auto"
+        Type of target.
+
+        - `"auto"` : Type of target is inferred with
+          :func:`~sklearn.utils.multiclass.type_of_target`.
+        - `"continuous"` : Continuous target
+        - `"binary"` : Binary target
+        - `"multiclass"` : Multiclass target
+
+        .. note::
+            The type of target inferred with `"auto"` may not be the desired target
+            type used for modeling. For example, if the target consisted of integers
+            between 0 and 100, then :func:`~sklearn.utils.multiclass.type_of_target`
+            will infer the target as `"multiclass"`. In this case, setting
+            `target_type="continuous"` will specify the target as a regression
+            problem. The `target_type_` attribute gives the target type used by the
+            encoder.
+
+        .. versionchanged:: 1.4
+           Added the option 'multiclass'.
+
+    smooth : "auto" or float, default="auto"
+        The amount of mixing of the target mean conditioned on the value of the
+        category with the global target mean. A larger `smooth` value will put
+        more weight on the global target mean.
+        If `"auto"`, then `smooth` is set to an empirical Bayes estimate.
+
+    cv : int, default=5
+        Determines the number of folds in the :term:`cross fitting` strategy used in
+        :meth:`fit_transform`. For classification targets, `StratifiedKFold` is used
+        and for continuous targets, `KFold` is used.
+
+    shuffle : bool, default=True
+        Whether to shuffle the data in :meth:`fit_transform` before splitting into
+        folds. Note that the samples within each split will not be shuffled.
+
+    random_state : int, RandomState instance or None, default=None
+        When `shuffle` is True, `random_state` affects the ordering of the
+        indices, which controls the randomness of each fold. Otherwise, this
+        parameter has no effect.
+        Pass an int for reproducible output across multiple function calls.
+        See :term:`Glossary `.
+
+    Attributes
+    ----------
+    encodings_ : list of shape (n_features,) or (n_features * n_classes) of \
+                    ndarray
+        Encodings learnt on all of `X`.
+        For feature `i`, `encodings_[i]` are the encodings matching the
+        categories listed in `categories_[i]`. When `target_type_` is
+        "multiclass", the encoding for feature `i` and class `j` is stored in
+        `encodings_[j + (i * len(classes_))]`. E.g., for 2 features (f) and
+        3 classes (c), encodings are ordered:
+        f0_c0, f0_c1, f0_c2, f1_c0, f1_c1, f1_c2,
+
+    categories_ : list of shape (n_features,) of ndarray
+        The categories of each input feature determined during fitting or
+        specified in `categories`
+        (in order of the features in `X` and corresponding with the output
+        of :meth:`transform`).
+
+    target_type_ : str
+        Type of target.
+
+    target_mean_ : float
+        The overall mean of the target. This value is only used in :meth:`transform`
+        to encode categories.
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+    classes_ : ndarray or None
+        If `target_type_` is 'binary' or 'multiclass', holds the label for each class,
+        otherwise `None`.
+
+    See Also
+    --------
+    OrdinalEncoder : Performs an ordinal (integer) encoding of the categorical features.
+        Contrary to TargetEncoder, this encoding is not supervised. Treating the
+        resulting encoding as a numerical features therefore lead arbitrarily
+        ordered values and therefore typically lead to lower predictive performance
+        when used as preprocessing for a classifier or regressor.
+    OneHotEncoder : Performs a one-hot encoding of categorical features. This
+        unsupervised encoding is better suited for low cardinality categorical
+        variables as it generate one new feature per unique category.
+
+    References
+    ----------
+    .. [MIC] :doi:`Micci-Barreca, Daniele. "A preprocessing scheme for high-cardinality
+       categorical attributes in classification and prediction problems"
+       SIGKDD Explor. Newsl. 3, 1 (July 2001), 27–32. <10.1145/507533.507538>`
+
+    Examples
+    --------
+    With `smooth="auto"`, the smoothing parameter is set to an empirical Bayes estimate:
+
+    >>> import numpy as np
+    >>> from sklearn.preprocessing import TargetEncoder
+    >>> X = np.array([["dog"] * 20 + ["cat"] * 30 + ["snake"] * 38], dtype=object).T
+    >>> y = [90.3] * 5 + [80.1] * 15 + [20.4] * 5 + [20.1] * 25 + [21.2] * 8 + [49] * 30
+    >>> enc_auto = TargetEncoder(smooth="auto")
+    >>> X_trans = enc_auto.fit_transform(X, y)
+
+    >>> # A high `smooth` parameter puts more weight on global mean on the categorical
+    >>> # encodings:
+    >>> enc_high_smooth = TargetEncoder(smooth=5000.0).fit(X, y)
+    >>> enc_high_smooth.target_mean_
+    np.float64(44.3)
+    >>> enc_high_smooth.encodings_
+    [array([44.1, 44.4, 44.3])]
+
+    >>> # On the other hand, a low `smooth` parameter puts more weight on target
+    >>> # conditioned on the value of the categorical:
+    >>> enc_low_smooth = TargetEncoder(smooth=1.0).fit(X, y)
+    >>> enc_low_smooth.encodings_
+    [array([21, 80.8, 43.2])]
+    """
+
+    _parameter_constraints: dict = {
+        "categories": [StrOptions({"auto"}), list],
+        "target_type": [StrOptions({"auto", "continuous", "binary", "multiclass"})],
+        "smooth": [StrOptions({"auto"}), Interval(Real, 0, None, closed="left")],
+        "cv": [Interval(Integral, 2, None, closed="left")],
+        "shuffle": ["boolean"],
+        "random_state": ["random_state"],
+    }
+
+    def __init__(
+        self,
+        categories="auto",
+        target_type="auto",
+        smooth="auto",
+        cv=5,
+        shuffle=True,
+        random_state=None,
+    ):
+        self.categories = categories
+        self.smooth = smooth
+        self.target_type = target_type
+        self.cv = cv
+        self.shuffle = shuffle
+        self.random_state = random_state
+
+    @_fit_context(prefer_skip_nested_validation=True)
+    def fit(self, X, y):
+        """Fit the :class:`TargetEncoder` to X and y.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            The data to determine the categories of each feature.
+
+        y : array-like of shape (n_samples,)
+            The target data used to encode the categories.
+
+        Returns
+        -------
+        self : object
+            Fitted encoder.
+        """
+        self._fit_encodings_all(X, y)
+        return self
+
+    @_fit_context(prefer_skip_nested_validation=True)
+    def fit_transform(self, X, y):
+        """Fit :class:`TargetEncoder` and transform X with the target encoding.
+
+        .. note::
+            `fit(X, y).transform(X)` does not equal `fit_transform(X, y)` because a
+            :term:`cross fitting` scheme is used in `fit_transform` for encoding.
+            See the :ref:`User Guide `. for details.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            The data to determine the categories of each feature.
+
+        y : array-like of shape (n_samples,)
+            The target data used to encode the categories.
+
+        Returns
+        -------
+        X_trans : ndarray of shape (n_samples, n_features) or \
+                    (n_samples, (n_features * n_classes))
+            Transformed input.
+        """
+        from ..model_selection import KFold, StratifiedKFold  # avoid circular import
+
+        X_ordinal, X_known_mask, y_encoded, n_categories = self._fit_encodings_all(X, y)
+
+        # The cv splitter is voluntarily restricted to *KFold to enforce non
+        # overlapping validation folds, otherwise the fit_transform output will
+        # not be well-specified.
+        if self.target_type_ == "continuous":
+            cv = KFold(self.cv, shuffle=self.shuffle, random_state=self.random_state)
+        else:
+            cv = StratifiedKFold(
+                self.cv, shuffle=self.shuffle, random_state=self.random_state
+            )
+
+        # If 'multiclass' multiply axis=1 by num classes else keep shape the same
+        if self.target_type_ == "multiclass":
+            X_out = np.empty(
+                (X_ordinal.shape[0], X_ordinal.shape[1] * len(self.classes_)),
+                dtype=np.float64,
+            )
+        else:
+            X_out = np.empty_like(X_ordinal, dtype=np.float64)
+
+        for train_idx, test_idx in cv.split(X, y):
+            X_train, y_train = X_ordinal[train_idx, :], y_encoded[train_idx]
+            y_train_mean = np.mean(y_train, axis=0)
+
+            if self.target_type_ == "multiclass":
+                encodings = self._fit_encoding_multiclass(
+                    X_train,
+                    y_train,
+                    n_categories,
+                    y_train_mean,
+                )
+            else:
+                encodings = self._fit_encoding_binary_or_continuous(
+                    X_train,
+                    y_train,
+                    n_categories,
+                    y_train_mean,
+                )
+            self._transform_X_ordinal(
+                X_out,
+                X_ordinal,
+                ~X_known_mask,
+                test_idx,
+                encodings,
+                y_train_mean,
+            )
+        return X_out
+
+    def transform(self, X):
+        """Transform X with the target encoding.
+
+        .. note::
+            `fit(X, y).transform(X)` does not equal `fit_transform(X, y)` because a
+            :term:`cross fitting` scheme is used in `fit_transform` for encoding.
+            See the :ref:`User Guide `. for details.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            The data to determine the categories of each feature.
+
+        Returns
+        -------
+        X_trans : ndarray of shape (n_samples, n_features) or \
+                    (n_samples, (n_features * n_classes))
+            Transformed input.
+        """
+        X_ordinal, X_known_mask = self._transform(
+            X, handle_unknown="ignore", ensure_all_finite="allow-nan"
+        )
+
+        # If 'multiclass' multiply axis=1 by num of classes else keep shape the same
+        if self.target_type_ == "multiclass":
+            X_out = np.empty(
+                (X_ordinal.shape[0], X_ordinal.shape[1] * len(self.classes_)),
+                dtype=np.float64,
+            )
+        else:
+            X_out = np.empty_like(X_ordinal, dtype=np.float64)
+
+        self._transform_X_ordinal(
+            X_out,
+            X_ordinal,
+            ~X_known_mask,
+            slice(None),
+            self.encodings_,
+            self.target_mean_,
+        )
+        return X_out
+
+    def _fit_encodings_all(self, X, y):
+        """Fit a target encoding with all the data."""
+        # avoid circular import
+        from ..preprocessing import (
+            LabelBinarizer,
+            LabelEncoder,
+        )
+
+        check_consistent_length(X, y)
+        self._fit(X, handle_unknown="ignore", ensure_all_finite="allow-nan")
+
+        if self.target_type == "auto":
+            accepted_target_types = ("binary", "multiclass", "continuous")
+            inferred_type_of_target = type_of_target(y, input_name="y")
+            if inferred_type_of_target not in accepted_target_types:
+                raise ValueError(
+                    "Unknown label type: Target type was inferred to be "
+                    f"{inferred_type_of_target!r}. Only {accepted_target_types} are "
+                    "supported."
+                )
+            self.target_type_ = inferred_type_of_target
+        else:
+            self.target_type_ = self.target_type
+
+        self.classes_ = None
+        if self.target_type_ == "binary":
+            label_encoder = LabelEncoder()
+            y = label_encoder.fit_transform(y)
+            self.classes_ = label_encoder.classes_
+        elif self.target_type_ == "multiclass":
+            label_binarizer = LabelBinarizer()
+            y = label_binarizer.fit_transform(y)
+            self.classes_ = label_binarizer.classes_
+        else:  # continuous
+            y = _check_y(y, y_numeric=True, estimator=self)
+
+        self.target_mean_ = np.mean(y, axis=0)
+
+        X_ordinal, X_known_mask = self._transform(
+            X, handle_unknown="ignore", ensure_all_finite="allow-nan"
+        )
+        n_categories = np.fromiter(
+            (len(category_for_feature) for category_for_feature in self.categories_),
+            dtype=np.int64,
+            count=len(self.categories_),
+        )
+        if self.target_type_ == "multiclass":
+            encodings = self._fit_encoding_multiclass(
+                X_ordinal,
+                y,
+                n_categories,
+                self.target_mean_,
+            )
+        else:
+            encodings = self._fit_encoding_binary_or_continuous(
+                X_ordinal,
+                y,
+                n_categories,
+                self.target_mean_,
+            )
+        self.encodings_ = encodings
+
+        return X_ordinal, X_known_mask, y, n_categories
+
+    def _fit_encoding_binary_or_continuous(
+        self, X_ordinal, y, n_categories, target_mean
+    ):
+        """Learn target encodings."""
+        if self.smooth == "auto":
+            y_variance = np.var(y)
+            encodings = _fit_encoding_fast_auto_smooth(
+                X_ordinal,
+                y,
+                n_categories,
+                target_mean,
+                y_variance,
+            )
+        else:
+            encodings = _fit_encoding_fast(
+                X_ordinal,
+                y,
+                n_categories,
+                self.smooth,
+                target_mean,
+            )
+        return encodings
+
+    def _fit_encoding_multiclass(self, X_ordinal, y, n_categories, target_mean):
+        """Learn multiclass encodings.
+
+        Learn encodings for each class (c) then reorder encodings such that
+        the same features (f) are grouped together. `reorder_index` enables
+        converting from:
+        f0_c0, f1_c0, f0_c1, f1_c1, f0_c2, f1_c2
+        to:
+        f0_c0, f0_c1, f0_c2, f1_c0, f1_c1, f1_c2
+        """
+        n_features = self.n_features_in_
+        n_classes = len(self.classes_)
+
+        encodings = []
+        for i in range(n_classes):
+            y_class = y[:, i]
+            encoding = self._fit_encoding_binary_or_continuous(
+                X_ordinal,
+                y_class,
+                n_categories,
+                target_mean[i],
+            )
+            encodings.extend(encoding)
+
+        reorder_index = (
+            idx
+            for start in range(n_features)
+            for idx in range(start, (n_classes * n_features), n_features)
+        )
+        return [encodings[idx] for idx in reorder_index]
+
+    def _transform_X_ordinal(
+        self,
+        X_out,
+        X_ordinal,
+        X_unknown_mask,
+        row_indices,
+        encodings,
+        target_mean,
+    ):
+        """Transform X_ordinal using encodings.
+
+        In the multiclass case, `X_ordinal` and `X_unknown_mask` have column
+        (axis=1) size `n_features`, while `encodings` has length of size
+        `n_features * n_classes`. `feat_idx` deals with this by repeating
+        feature indices by `n_classes` E.g., for 3 features, 2 classes:
+        0,0,1,1,2,2
+
+        Additionally, `target_mean` is of shape (`n_classes`,) so `mean_idx`
+        cycles through 0 to `n_classes` - 1, `n_features` times.
+        """
+        if self.target_type_ == "multiclass":
+            n_classes = len(self.classes_)
+            for e_idx, encoding in enumerate(encodings):
+                # Repeat feature indices by n_classes
+                feat_idx = e_idx // n_classes
+                # Cycle through each class
+                mean_idx = e_idx % n_classes
+                X_out[row_indices, e_idx] = encoding[X_ordinal[row_indices, feat_idx]]
+                X_out[X_unknown_mask[:, feat_idx], e_idx] = target_mean[mean_idx]
+        else:
+            for e_idx, encoding in enumerate(encodings):
+                X_out[row_indices, e_idx] = encoding[X_ordinal[row_indices, e_idx]]
+                X_out[X_unknown_mask[:, e_idx], e_idx] = target_mean
+
+    def get_feature_names_out(self, input_features=None):
+        """Get output feature names for transformation.
+
+        Parameters
+        ----------
+        input_features : array-like of str or None, default=None
+            Not used, present here for API consistency by convention.
+
+        Returns
+        -------
+        feature_names_out : ndarray of str objects
+            Transformed feature names. `feature_names_in_` is used unless it is
+            not defined, in which case the following input feature names are
+            generated: `["x0", "x1", ..., "x(n_features_in_ - 1)"]`.
+            When `type_of_target_` is "multiclass" the names are of the format
+            '_'.
+        """
+        check_is_fitted(self, "n_features_in_")
+        feature_names = _check_feature_names_in(self, input_features)
+        if self.target_type_ == "multiclass":
+            feature_names = [
+                f"{feature_name}_{class_name}"
+                for feature_name in feature_names
+                for class_name in self.classes_
+            ]
+            return np.asarray(feature_names, dtype=object)
+        else:
+            return feature_names
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.target_tags.required = True
+        return tags
diff --git a/sklearn/preprocessing/_target_encoder_fast.pyx b/sklearn/preprocessing/_target_encoder_fast.pyx
new file mode 100644
index 0000000000000..dca5f78e8d60f
--- /dev/null
+++ b/sklearn/preprocessing/_target_encoder_fast.pyx
@@ -0,0 +1,167 @@
+from libc.math cimport isnan
+from libcpp.vector cimport vector
+
+from ..utils._typedefs cimport float32_t, float64_t, int32_t, int64_t
+
+import numpy as np
+
+
+ctypedef fused INT_DTYPE:
+    int64_t
+    int32_t
+
+ctypedef fused Y_DTYPE:
+    int64_t
+    int32_t
+    float64_t
+    float32_t
+
+
+def _fit_encoding_fast(
+    INT_DTYPE[:, ::1] X_int,
+    const Y_DTYPE[:] y,
+    int64_t[::1] n_categories,
+    double smooth,
+    double y_mean,
+):
+    """Fit a target encoding on X_int and y.
+
+    This implementation uses Eq 7 from [1] to compute the encoding.
+    As stated in the paper, Eq 7 is the same as Eq 3.
+
+    [1]: Micci-Barreca, Daniele. "A preprocessing scheme for high-cardinality
+         categorical attributes in classification and prediction problems"
+    """
+    cdef:
+        int64_t sample_idx, feat_idx, cat_idx, n_cats
+        INT_DTYPE X_int_tmp
+        int n_samples = X_int.shape[0]
+        int n_features = X_int.shape[1]
+        double smooth_sum = smooth * y_mean
+        int64_t max_n_cats = np.max(n_categories)
+        double[::1] sums = np.empty(max_n_cats, dtype=np.float64)
+        double[::1] counts = np.empty(max_n_cats, dtype=np.float64)
+        list encodings = []
+        double[::1] current_encoding
+        # Gives access to encodings without gil
+        vector[double*] encoding_vec
+
+    encoding_vec.resize(n_features)
+    for feat_idx in range(n_features):
+        current_encoding = np.empty(shape=n_categories[feat_idx], dtype=np.float64)
+        encoding_vec[feat_idx] = ¤t_encoding[0]
+        encodings.append(np.asarray(current_encoding))
+
+    with nogil:
+        for feat_idx in range(n_features):
+            n_cats = n_categories[feat_idx]
+
+            for cat_idx in range(n_cats):
+                sums[cat_idx] = smooth_sum
+                counts[cat_idx] = smooth
+
+            for sample_idx in range(n_samples):
+                X_int_tmp = X_int[sample_idx, feat_idx]
+                # -1 are unknown categories, which are not counted
+                if X_int_tmp == -1:
+                    continue
+                sums[X_int_tmp] += y[sample_idx]
+                counts[X_int_tmp] += 1.0
+
+            for cat_idx in range(n_cats):
+                if counts[cat_idx] == 0:
+                    encoding_vec[feat_idx][cat_idx] = y_mean
+                else:
+                    encoding_vec[feat_idx][cat_idx] = sums[cat_idx] / counts[cat_idx]
+
+    return encodings
+
+
+def _fit_encoding_fast_auto_smooth(
+    INT_DTYPE[:, ::1] X_int,
+    const Y_DTYPE[:] y,
+    int64_t[::1] n_categories,
+    double y_mean,
+    double y_variance,
+):
+    """Fit a target encoding on X_int and y with auto smoothing.
+
+    This implementation uses Eq 5 and 6 from [1].
+
+    [1]: Micci-Barreca, Daniele. "A preprocessing scheme for high-cardinality
+         categorical attributes in classification and prediction problems"
+    """
+    cdef:
+        int64_t sample_idx, feat_idx, cat_idx, n_cats
+        INT_DTYPE X_int_tmp
+        double diff
+        int n_samples = X_int.shape[0]
+        int n_features = X_int.shape[1]
+        int64_t max_n_cats = np.max(n_categories)
+        double[::1] means = np.empty(max_n_cats, dtype=np.float64)
+        int64_t[::1] counts = np.empty(max_n_cats, dtype=np.int64)
+        double[::1] sum_of_squared_diffs = np.empty(max_n_cats, dtype=np.float64)
+        double lambda_
+        list encodings = []
+        double[::1] current_encoding
+        # Gives access to encodings without gil
+        vector[double*] encoding_vec
+
+    encoding_vec.resize(n_features)
+    for feat_idx in range(n_features):
+        current_encoding = np.empty(shape=n_categories[feat_idx], dtype=np.float64)
+        encoding_vec[feat_idx] = ¤t_encoding[0]
+        encodings.append(np.asarray(current_encoding))
+
+    # TODO: parallelize this with OpenMP prange. When n_features >= n_threads, it's
+    # probably good to parallelize the outer loop. When n_features is too small,
+    # then it would probably better to parallelize the nested loops on n_samples and
+    # n_cats, but the code to handle thread-local temporary variables might be
+    # significantly more complex.
+    with nogil:
+        for feat_idx in range(n_features):
+            n_cats = n_categories[feat_idx]
+
+            for cat_idx in range(n_cats):
+                means[cat_idx] = 0.0
+                counts[cat_idx] = 0
+                sum_of_squared_diffs[cat_idx] = 0.0
+
+            # first pass to compute the mean
+            for sample_idx in range(n_samples):
+                X_int_tmp = X_int[sample_idx, feat_idx]
+
+                # -1 are unknown categories, which are not counted
+                if X_int_tmp == -1:
+                    continue
+                counts[X_int_tmp] += 1
+                means[X_int_tmp] += y[sample_idx]
+
+            for cat_idx in range(n_cats):
+                means[cat_idx] /= counts[cat_idx]
+
+            # second pass to compute the sum of squared differences
+            for sample_idx in range(n_samples):
+                X_int_tmp = X_int[sample_idx, feat_idx]
+                if X_int_tmp == -1:
+                    continue
+                diff = y[sample_idx] - means[X_int_tmp]
+                sum_of_squared_diffs[X_int_tmp] += diff * diff
+
+            for cat_idx in range(n_cats):
+                lambda_ = (
+                    y_variance * counts[cat_idx] /
+                    (y_variance * counts[cat_idx] + sum_of_squared_diffs[cat_idx] /
+                     counts[cat_idx])
+                )
+                if isnan(lambda_):
+                    # A nan can happen when:
+                    # 1. counts[cat_idx] == 0
+                    # 2. y_variance == 0 and sum_of_squared_diffs[cat_idx] == 0
+                    encoding_vec[feat_idx][cat_idx] = y_mean
+                else:
+                    encoding_vec[feat_idx][cat_idx] = (
+                        lambda_ * means[cat_idx] + (1 - lambda_) * y_mean
+                    )
+
+    return encodings
diff --git a/sklearn/preprocessing/meson.build b/sklearn/preprocessing/meson.build
new file mode 100644
index 0000000000000..052c4a6766ad4
--- /dev/null
+++ b/sklearn/preprocessing/meson.build
@@ -0,0 +1,13 @@
+py.extension_module(
+  '_csr_polynomial_expansion',
+  [cython_gen.process('_csr_polynomial_expansion.pyx'), utils_cython_tree],
+  subdir: 'sklearn/preprocessing',
+  install: true
+)
+
+py.extension_module(
+  '_target_encoder_fast',
+  [cython_gen_cpp.process('_target_encoder_fast.pyx'), utils_cython_tree],
+  subdir: 'sklearn/preprocessing',
+  install: true
+)
diff --git a/sklearn/preprocessing/setup.py b/sklearn/preprocessing/setup.py
deleted file mode 100644
index 29dae9b8faa34..0000000000000
--- a/sklearn/preprocessing/setup.py
+++ /dev/null
@@ -1,20 +0,0 @@
-import os
-
-
-def configuration(parent_package='', top_path=None):
-    import numpy
-    from numpy.distutils.misc_util import Configuration
-
-    config = Configuration('preprocessing', parent_package, top_path)
-    libraries = []
-    if os.name == 'posix':
-        libraries.append('m')
-
-    config.add_extension('_csr_polynomial_expansion',
-                         sources=['_csr_polynomial_expansion.pyx'],
-                         include_dirs=[numpy.get_include()],
-                         libraries=libraries)
-
-    config.add_subpackage('tests')
-
-    return config
diff --git a/sklearn/preprocessing/tests/test_common.py b/sklearn/preprocessing/tests/test_common.py
index a00dd2b6cb025..09f702f64ce23 100644
--- a/sklearn/preprocessing/tests/test_common.py
+++ b/sklearn/preprocessing/tests/test_common.py
@@ -1,31 +1,35 @@
 import warnings
 
-import pytest
 import numpy as np
+import pytest
 
-from scipy import sparse
-
+from sklearn.base import clone
 from sklearn.datasets import load_iris
 from sklearn.model_selection import train_test_split
-
-from sklearn.base import clone
-
-from sklearn.preprocessing import maxabs_scale
-from sklearn.preprocessing import minmax_scale
-from sklearn.preprocessing import scale
-from sklearn.preprocessing import power_transform
-from sklearn.preprocessing import quantile_transform
-from sklearn.preprocessing import robust_scale
-
-from sklearn.preprocessing import MaxAbsScaler
-from sklearn.preprocessing import MinMaxScaler
-from sklearn.preprocessing import StandardScaler
-from sklearn.preprocessing import PowerTransformer
-from sklearn.preprocessing import QuantileTransformer
-from sklearn.preprocessing import RobustScaler
-
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils._testing import assert_allclose
+from sklearn.preprocessing import (
+    MaxAbsScaler,
+    MinMaxScaler,
+    PowerTransformer,
+    QuantileTransformer,
+    RobustScaler,
+    StandardScaler,
+    maxabs_scale,
+    minmax_scale,
+    power_transform,
+    quantile_transform,
+    robust_scale,
+    scale,
+)
+from sklearn.utils._testing import assert_allclose, assert_array_equal
+from sklearn.utils.fixes import (
+    BSR_CONTAINERS,
+    COO_CONTAINERS,
+    CSC_CONTAINERS,
+    CSR_CONTAINERS,
+    DIA_CONTAINERS,
+    DOK_CONTAINERS,
+    LIL_CONTAINERS,
+)
 
 iris = load_iris()
 
@@ -37,25 +41,28 @@ def _get_valid_samples_by_column(X, col):
 
 @pytest.mark.parametrize(
     "est, func, support_sparse, strictly_positive, omit_kwargs",
-    [(MaxAbsScaler(), maxabs_scale, True, False, []),
-     (MinMaxScaler(), minmax_scale, False, False, ['clip']),
-     (StandardScaler(), scale, False, False, []),
-     (StandardScaler(with_mean=False), scale, True, False, []),
-     (PowerTransformer('yeo-johnson'), power_transform, False, False, []),
-     (PowerTransformer('box-cox'), power_transform, False, True, []),
-     (QuantileTransformer(n_quantiles=10), quantile_transform, True, False,
-     []),
-     (RobustScaler(), robust_scale, False, False, []),
-     (RobustScaler(with_centering=False), robust_scale, True, False, [])]
+    [
+        (MaxAbsScaler(), maxabs_scale, True, False, []),
+        (MinMaxScaler(), minmax_scale, False, False, ["clip"]),
+        (StandardScaler(), scale, False, False, []),
+        (StandardScaler(with_mean=False), scale, True, False, []),
+        (PowerTransformer("yeo-johnson"), power_transform, False, False, []),
+        (PowerTransformer("box-cox"), power_transform, False, True, []),
+        (QuantileTransformer(n_quantiles=10), quantile_transform, True, False, []),
+        (RobustScaler(), robust_scale, False, False, []),
+        (RobustScaler(with_centering=False), robust_scale, True, False, []),
+    ],
 )
-def test_missing_value_handling(est, func, support_sparse, strictly_positive,
-                                omit_kwargs):
+def test_missing_value_handling(
+    est, func, support_sparse, strictly_positive, omit_kwargs
+):
     # check that the preprocessing method let pass nan
     rng = np.random.RandomState(42)
     X = iris.data.copy()
     n_missing = 50
-    X[rng.randint(X.shape[0], size=n_missing),
-      rng.randint(X.shape[1], size=n_missing)] = np.nan
+    X[
+        rng.randint(X.shape[0], size=n_missing), rng.randint(X.shape[1], size=n_missing)
+    ] = np.nan
     if strictly_positive:
         X += np.nanmin(X) + 0.1
     X_train, X_test = train_test_split(X, random_state=1)
@@ -65,20 +72,20 @@ def test_missing_value_handling(est, func, support_sparse, strictly_positive,
     assert np.any(np.isnan(X_test), axis=0).all()
     X_test[:, 0] = np.nan  # make sure this boundary case is tested
 
-    with pytest.warns(None) as records:
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", RuntimeWarning)
         Xt = est.fit(X_train).transform(X_test)
     # ensure no warnings are raised
-    assert len(records) == 0
     # missing values should still be missing, and only them
     assert_array_equal(np.isnan(Xt), np.isnan(X_test))
 
     # check that the function leads to the same results as the class
-    with pytest.warns(None) as records:
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", RuntimeWarning)
         Xt_class = est.transform(X_train)
-    assert len(records) == 0
     kwargs = est.get_params()
     # remove the parameters which should be omitted because they
-    # are not defined in the sister function of the preprocessing class
+    # are not defined in the counterpart function of the preprocessing class
     for kwarg in omit_kwargs:
         _ = kwargs.pop(kwarg)
     Xt_func = func(X_train, **kwargs)
@@ -96,68 +103,83 @@ def test_missing_value_handling(est, func, support_sparse, strictly_positive,
         # train only on non-NaN
         est.fit(_get_valid_samples_by_column(X_train, i))
         # check transforming with NaN works even when training without NaN
-        with pytest.warns(None) as records:
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", RuntimeWarning)
             Xt_col = est.transform(X_test[:, [i]])
-        assert len(records) == 0
         assert_allclose(Xt_col, Xt[:, [i]])
         # check non-NaN is handled as before - the 1st column is all nan
         if not np.isnan(X_test[:, i]).all():
-            Xt_col_nonan = est.transform(
-                _get_valid_samples_by_column(X_test, i))
-            assert_array_equal(Xt_col_nonan,
-                               Xt_col[~np.isnan(Xt_col.squeeze())])
+            Xt_col_nonan = est.transform(_get_valid_samples_by_column(X_test, i))
+            assert_array_equal(Xt_col_nonan, Xt_col[~np.isnan(Xt_col.squeeze())])
 
     if support_sparse:
         est_dense = clone(est)
         est_sparse = clone(est)
 
-        with pytest.warns(None) as records:
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", RuntimeWarning)
             Xt_dense = est_dense.fit(X_train).transform(X_test)
             Xt_inv_dense = est_dense.inverse_transform(Xt_dense)
-        assert len(records) == 0
-        for sparse_constructor in (sparse.csr_matrix, sparse.csc_matrix,
-                                   sparse.bsr_matrix, sparse.coo_matrix,
-                                   sparse.dia_matrix, sparse.dok_matrix,
-                                   sparse.lil_matrix):
+
+        for sparse_container in (
+            BSR_CONTAINERS
+            + COO_CONTAINERS
+            + CSC_CONTAINERS
+            + CSR_CONTAINERS
+            + DIA_CONTAINERS
+            + DOK_CONTAINERS
+            + LIL_CONTAINERS
+        ):
             # check that the dense and sparse inputs lead to the same results
             # precompute the matrix to avoid catching side warnings
-            X_train_sp = sparse_constructor(X_train)
-            X_test_sp = sparse_constructor(X_test)
-            with pytest.warns(None) as records:
-                warnings.simplefilter('ignore', PendingDeprecationWarning)
+            X_train_sp = sparse_container(X_train)
+            X_test_sp = sparse_container(X_test)
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore", PendingDeprecationWarning)
+                warnings.simplefilter("error", RuntimeWarning)
                 Xt_sp = est_sparse.fit(X_train_sp).transform(X_test_sp)
-            assert len(records) == 0
-            assert_allclose(Xt_sp.A, Xt_dense)
-            with pytest.warns(None) as records:
-                warnings.simplefilter('ignore', PendingDeprecationWarning)
+
+            assert_allclose(Xt_sp.toarray(), Xt_dense)
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore", PendingDeprecationWarning)
+                warnings.simplefilter("error", RuntimeWarning)
                 Xt_inv_sp = est_sparse.inverse_transform(Xt_sp)
-            assert len(records) == 0
-            assert_allclose(Xt_inv_sp.A, Xt_inv_dense)
+
+            assert_allclose(Xt_inv_sp.toarray(), Xt_inv_dense)
 
 
 @pytest.mark.parametrize(
     "est, func",
-    [(MaxAbsScaler(), maxabs_scale),
-     (MinMaxScaler(), minmax_scale),
-     (StandardScaler(), scale),
-     (StandardScaler(with_mean=False), scale),
-     (PowerTransformer('yeo-johnson'), power_transform),
-     (PowerTransformer('box-cox'), power_transform,),
-     (QuantileTransformer(n_quantiles=3), quantile_transform),
-     (RobustScaler(), robust_scale),
-     (RobustScaler(with_centering=False), robust_scale)]
+    [
+        (MaxAbsScaler(), maxabs_scale),
+        (MinMaxScaler(), minmax_scale),
+        (StandardScaler(), scale),
+        (StandardScaler(with_mean=False), scale),
+        (PowerTransformer("yeo-johnson"), power_transform),
+        (
+            PowerTransformer("box-cox"),
+            power_transform,
+        ),
+        (QuantileTransformer(n_quantiles=3), quantile_transform),
+        (RobustScaler(), robust_scale),
+        (RobustScaler(with_centering=False), robust_scale),
+    ],
 )
 def test_missing_value_pandas_na_support(est, func):
     # Test pandas IntegerArray with pd.NA
-    pd = pytest.importorskip('pandas', minversion="1.0")
+    pd = pytest.importorskip("pandas")
 
-    X = np.array([[1, 2, 3, np.nan, np.nan, 4, 5, 1],
-                  [np.nan, np.nan, 8, 4, 6, np.nan, np.nan, 8],
-                  [1, 2, 3, 4, 5, 6, 7, 8]]).T
+    X = np.array(
+        [
+            [1, 2, 3, np.nan, np.nan, 4, 5, 1],
+            [np.nan, np.nan, 8, 4, 6, np.nan, np.nan, 8],
+            [1, 2, 3, 4, 5, 6, 7, 8],
+        ]
+    ).T
 
     # Creates dataframe with IntegerArrays with pd.NA
-    X_df = pd.DataFrame(X, dtype="Int16", columns=['a', 'b', 'c'])
-    X_df['c'] = X_df['c'].astype('int')
+    X_df = pd.DataFrame(X, dtype="Int16", columns=["a", "b", "c"])
+    X_df["c"] = X_df["c"].astype("int")
 
     X_trans = est.fit_transform(X)
     X_df_trans = est.fit_transform(X_df)
diff --git a/sklearn/preprocessing/tests/test_data.py b/sklearn/preprocessing/tests/test_data.py
index fdd88be0ccff4..a618d426a7dcb 100644
--- a/sklearn/preprocessing/tests/test_data.py
+++ b/sklearn/preprocessing/tests/test_data.py
@@ -1,64 +1,71 @@
-# Authors:
-#
-#          Giorgio Patrini
-#
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
+import re
 import warnings
-import itertools
 
 import numpy as np
 import numpy.linalg as la
-from scipy import sparse, stats
-from scipy.sparse import random as sparse_random
-
 import pytest
+from scipy import sparse, stats
 
-from sklearn.utils import gen_batches
-
-from sklearn.utils._testing import assert_almost_equal
-from sklearn.utils._testing import assert_array_almost_equal
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils._testing import assert_array_less
-from sklearn.utils._testing import assert_warns_message
-from sklearn.utils._testing import assert_no_warnings
-from sklearn.utils._testing import assert_allclose
-from sklearn.utils._testing import assert_allclose_dense_sparse
-from sklearn.utils._testing import skip_if_32bit
-from sklearn.utils._testing import _convert_container
-
-from sklearn.utils.sparsefuncs import mean_variance_axis
-from sklearn.preprocessing import Binarizer
-from sklearn.preprocessing import KernelCenterer
-from sklearn.preprocessing import Normalizer
-from sklearn.preprocessing import normalize
-from sklearn.preprocessing import StandardScaler
-from sklearn.preprocessing import scale
-from sklearn.preprocessing import MinMaxScaler
-from sklearn.preprocessing import minmax_scale
-from sklearn.preprocessing import QuantileTransformer
-from sklearn.preprocessing import quantile_transform
-from sklearn.preprocessing import MaxAbsScaler
-from sklearn.preprocessing import maxabs_scale
-from sklearn.preprocessing import RobustScaler
-from sklearn.preprocessing import robust_scale
-from sklearn.preprocessing import add_dummy_feature
-from sklearn.preprocessing import PolynomialFeatures
-from sklearn.preprocessing import PowerTransformer
-from sklearn.preprocessing import power_transform
-from sklearn.preprocessing._data import _handle_zeros_in_scale
-from sklearn.preprocessing._data import BOUNDS_THRESHOLD
-
-from sklearn.exceptions import NotFittedError
-
+from sklearn import config_context, datasets
 from sklearn.base import clone
-from sklearn.pipeline import Pipeline
+from sklearn.exceptions import NotFittedError
+from sklearn.externals._packaging.version import parse as parse_version
+from sklearn.metrics.pairwise import linear_kernel
 from sklearn.model_selection import cross_val_predict
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import (
+    Binarizer,
+    KernelCenterer,
+    MaxAbsScaler,
+    MinMaxScaler,
+    Normalizer,
+    PowerTransformer,
+    QuantileTransformer,
+    RobustScaler,
+    StandardScaler,
+    add_dummy_feature,
+    maxabs_scale,
+    minmax_scale,
+    normalize,
+    power_transform,
+    quantile_transform,
+    robust_scale,
+    scale,
+)
+from sklearn.preprocessing._data import BOUNDS_THRESHOLD, _handle_zeros_in_scale
 from sklearn.svm import SVR
-from sklearn.utils import shuffle
-
-from sklearn import datasets
-
+from sklearn.utils import gen_batches, shuffle
+from sklearn.utils._array_api import (
+    _convert_to_numpy,
+    _get_namespace_device_dtype_ids,
+    yield_namespace_device_dtype_combinations,
+)
+from sklearn.utils._test_common.instance_generator import _get_check_estimator_ids
+from sklearn.utils._testing import (
+    _array_api_for_tests,
+    _convert_container,
+    assert_allclose,
+    assert_allclose_dense_sparse,
+    assert_almost_equal,
+    assert_array_almost_equal,
+    assert_array_equal,
+    assert_array_less,
+    skip_if_32bit,
+)
+from sklearn.utils.estimator_checks import (
+    check_array_api_input_and_values,
+)
+from sklearn.utils.fixes import (
+    COO_CONTAINERS,
+    CSC_CONTAINERS,
+    CSR_CONTAINERS,
+    LIL_CONTAINERS,
+    sp_version,
+)
+from sklearn.utils.sparsefuncs import mean_variance_axis
 
 iris = datasets.load_iris()
 
@@ -85,210 +92,11 @@ def _check_dim_1axis(a):
     return np.asarray(a).shape[0]
 
 
-def assert_correct_incr(i, batch_start, batch_stop, n, chunk_size,
-                        n_samples_seen):
+def assert_correct_incr(i, batch_start, batch_stop, n, chunk_size, n_samples_seen):
     if batch_stop != n:
         assert (i + 1) * chunk_size == n_samples_seen
     else:
-        assert (i * chunk_size + (batch_stop - batch_start) ==
-                n_samples_seen)
-
-
-def test_polynomial_features():
-    # Test Polynomial Features
-    X1 = np.arange(6)[:, np.newaxis]
-    P1 = np.hstack([np.ones_like(X1),
-                    X1, X1 ** 2, X1 ** 3])
-    deg1 = 3
-
-    X2 = np.arange(6).reshape((3, 2))
-    x1 = X2[:, :1]
-    x2 = X2[:, 1:]
-    P2 = np.hstack([x1 ** 0 * x2 ** 0,
-                    x1 ** 1 * x2 ** 0,
-                    x1 ** 0 * x2 ** 1,
-                    x1 ** 2 * x2 ** 0,
-                    x1 ** 1 * x2 ** 1,
-                    x1 ** 0 * x2 ** 2])
-    deg2 = 2
-
-    for (deg, X, P) in [(deg1, X1, P1), (deg2, X2, P2)]:
-        P_test = PolynomialFeatures(deg, include_bias=True).fit_transform(X)
-        assert_array_almost_equal(P_test, P)
-
-        P_test = PolynomialFeatures(deg, include_bias=False).fit_transform(X)
-        assert_array_almost_equal(P_test, P[:, 1:])
-
-    interact = PolynomialFeatures(2, interaction_only=True, include_bias=True)
-    X_poly = interact.fit_transform(X)
-    assert_array_almost_equal(X_poly, P2[:, [0, 1, 2, 4]])
-
-    assert interact.powers_.shape == (interact.n_output_features_,
-                                      interact.n_input_features_)
-
-
-def test_polynomial_feature_names():
-    X = np.arange(30).reshape(10, 3)
-    poly = PolynomialFeatures(degree=2, include_bias=True).fit(X)
-    feature_names = poly.get_feature_names()
-    assert_array_equal(['1', 'x0', 'x1', 'x2', 'x0^2', 'x0 x1',
-                        'x0 x2', 'x1^2', 'x1 x2', 'x2^2'],
-                       feature_names)
-
-    poly = PolynomialFeatures(degree=3, include_bias=False).fit(X)
-    feature_names = poly.get_feature_names(["a", "b", "c"])
-    assert_array_equal(['a', 'b', 'c', 'a^2', 'a b', 'a c', 'b^2',
-                        'b c', 'c^2', 'a^3', 'a^2 b', 'a^2 c',
-                        'a b^2', 'a b c', 'a c^2', 'b^3', 'b^2 c',
-                        'b c^2', 'c^3'], feature_names)
-    # test some unicode
-    poly = PolynomialFeatures(degree=1, include_bias=True).fit(X)
-    feature_names = poly.get_feature_names(
-        ["\u0001F40D", "\u262E", "\u05D0"])
-    assert_array_equal(["1", "\u0001F40D", "\u262E", "\u05D0"],
-                       feature_names)
-
-
-def test_polynomial_feature_array_order():
-    """Test that output array has the given order."""
-    X = np.arange(10).reshape(5, 2)
-
-    def is_c_contiguous(a):
-        return np.isfortran(a.T)
-
-    assert is_c_contiguous(PolynomialFeatures().fit_transform(X))
-    assert is_c_contiguous(PolynomialFeatures(order='C').fit_transform(X))
-    assert np.isfortran(PolynomialFeatures(order='F').fit_transform(X))
-
-
-@pytest.mark.parametrize(['deg', 'include_bias', 'interaction_only', 'dtype'],
-                         [(1, True, False, int),
-                          (2, True, False, int),
-                          (2, True, False, np.float32),
-                          (2, True, False, np.float64),
-                          (3, False, False, np.float64),
-                          (3, False, True, np.float64),
-                          (4, False, False, np.float64),
-                          (4, False, True, np.float64)])
-def test_polynomial_features_csc_X(deg, include_bias, interaction_only, dtype):
-    rng = np.random.RandomState(0)
-    X = rng.randint(0, 2, (100, 2))
-    X_csc = sparse.csc_matrix(X)
-
-    est = PolynomialFeatures(deg, include_bias=include_bias,
-                             interaction_only=interaction_only)
-    Xt_csc = est.fit_transform(X_csc.astype(dtype))
-    Xt_dense = est.fit_transform(X.astype(dtype))
-
-    assert isinstance(Xt_csc, sparse.csc_matrix)
-    assert Xt_csc.dtype == Xt_dense.dtype
-    assert_array_almost_equal(Xt_csc.A, Xt_dense)
-
-
-@pytest.mark.parametrize(['deg', 'include_bias', 'interaction_only', 'dtype'],
-                         [(1, True, False, int),
-                          (2, True, False, int),
-                          (2, True, False, np.float32),
-                          (2, True, False, np.float64),
-                          (3, False, False, np.float64),
-                          (3, False, True, np.float64)])
-def test_polynomial_features_csr_X(deg, include_bias, interaction_only, dtype):
-    rng = np.random.RandomState(0)
-    X = rng.randint(0, 2, (100, 2))
-    X_csr = sparse.csr_matrix(X)
-
-    est = PolynomialFeatures(deg, include_bias=include_bias,
-                             interaction_only=interaction_only)
-    Xt_csr = est.fit_transform(X_csr.astype(dtype))
-    Xt_dense = est.fit_transform(X.astype(dtype, copy=False))
-
-    assert isinstance(Xt_csr, sparse.csr_matrix)
-    assert Xt_csr.dtype == Xt_dense.dtype
-    assert_array_almost_equal(Xt_csr.A, Xt_dense)
-
-
-@pytest.mark.parametrize(['deg', 'include_bias', 'interaction_only', 'dtype'],
-                         [(2, True, False, np.float32),
-                          (2, True, False, np.float64),
-                          (3, False, False, np.float64),
-                          (3, False, True, np.float64)])
-def test_polynomial_features_csr_X_floats(deg, include_bias,
-                                          interaction_only, dtype):
-    X_csr = sparse_random(1000, 10, 0.5, random_state=0).tocsr()
-    X = X_csr.toarray()
-
-    est = PolynomialFeatures(deg, include_bias=include_bias,
-                             interaction_only=interaction_only)
-    Xt_csr = est.fit_transform(X_csr.astype(dtype))
-    Xt_dense = est.fit_transform(X.astype(dtype))
-
-    assert isinstance(Xt_csr, sparse.csr_matrix)
-    assert Xt_csr.dtype == Xt_dense.dtype
-    assert_array_almost_equal(Xt_csr.A, Xt_dense)
-
-
-@pytest.mark.parametrize(['zero_row_index', 'deg', 'interaction_only'],
-                         [(0, 2, True), (1, 2, True), (2, 2, True),
-                          (0, 3, True), (1, 3, True), (2, 3, True),
-                          (0, 2, False), (1, 2, False), (2, 2, False),
-                          (0, 3, False), (1, 3, False), (2, 3, False)])
-def test_polynomial_features_csr_X_zero_row(zero_row_index, deg,
-                                            interaction_only):
-    X_csr = sparse_random(3, 10, 1.0, random_state=0).tocsr()
-    X_csr[zero_row_index, :] = 0.0
-    X = X_csr.toarray()
-
-    est = PolynomialFeatures(deg, include_bias=False,
-                             interaction_only=interaction_only)
-    Xt_csr = est.fit_transform(X_csr)
-    Xt_dense = est.fit_transform(X)
-
-    assert isinstance(Xt_csr, sparse.csr_matrix)
-    assert Xt_csr.dtype == Xt_dense.dtype
-    assert_array_almost_equal(Xt_csr.A, Xt_dense)
-
-
-# This degree should always be one more than the highest degree supported by
-# _csr_expansion.
-@pytest.mark.parametrize(['include_bias', 'interaction_only'],
-                         [(True, True), (True, False),
-                          (False, True), (False, False)])
-def test_polynomial_features_csr_X_degree_4(include_bias, interaction_only):
-    X_csr = sparse_random(1000, 10, 0.5, random_state=0).tocsr()
-    X = X_csr.toarray()
-
-    est = PolynomialFeatures(4, include_bias=include_bias,
-                             interaction_only=interaction_only)
-    Xt_csr = est.fit_transform(X_csr)
-    Xt_dense = est.fit_transform(X)
-
-    assert isinstance(Xt_csr, sparse.csr_matrix)
-    assert Xt_csr.dtype == Xt_dense.dtype
-    assert_array_almost_equal(Xt_csr.A, Xt_dense)
-
-
-@pytest.mark.parametrize(['deg', 'dim', 'interaction_only'],
-                         [(2, 1, True),
-                          (2, 2, True),
-                          (3, 1, True),
-                          (3, 2, True),
-                          (3, 3, True),
-                          (2, 1, False),
-                          (2, 2, False),
-                          (3, 1, False),
-                          (3, 2, False),
-                          (3, 3, False)])
-def test_polynomial_features_csr_X_dim_edges(deg, dim, interaction_only):
-    X_csr = sparse_random(1000, dim, 0.5, random_state=0).tocsr()
-    X = X_csr.toarray()
-
-    est = PolynomialFeatures(deg, interaction_only=interaction_only)
-    Xt_csr = est.fit_transform(X_csr)
-    Xt_dense = est.fit_transform(X)
-
-    assert isinstance(Xt_csr, sparse.csr_matrix)
-    assert Xt_csr.dtype == Xt_dense.dtype
-    assert_array_almost_equal(Xt_csr.A, Xt_dense)
+        assert i * chunk_size + (batch_stop - batch_start) == n_samples_seen
 
 
 def test_raises_value_error_if_sample_weights_greater_than_1d():
@@ -298,7 +106,6 @@ def test_raises_value_error_if_sample_weights_greater_than_1d():
     n_featuress = [3, 2]
 
     for n_samples, n_features in zip(n_sampless, n_featuress):
-
         X = rng.randn(n_samples, n_features)
         y = rng.randn(n_samples)
 
@@ -310,23 +117,29 @@ def test_raises_value_error_if_sample_weights_greater_than_1d():
             scaler.fit(X, y, sample_weight=sample_weight_notOK)
 
 
-@pytest.mark.parametrize(['Xw', 'X', 'sample_weight'],
-                         [([[1, 2, 3], [4, 5, 6]],
-                           [[1, 2, 3], [1, 2, 3], [4, 5, 6]],
-                           [2., 1.]),
-                          ([[1, 0, 1], [0, 0, 1]],
-                           [[1, 0, 1], [0, 0, 1], [0, 0, 1], [0, 0, 1]],
-                           np.array([1, 3])),
-                          ([[1, np.nan, 1], [np.nan, np.nan, 1]],
-                           [[1, np.nan, 1], [np.nan, np.nan, 1],
-                            [np.nan, np.nan, 1], [np.nan, np.nan, 1]],
-                           np.array([1, 3])),
-                          ])
 @pytest.mark.parametrize(
-    "array_constructor", ["array", "sparse_csr", "sparse_csc"]
+    ["Xw", "X", "sample_weight"],
+    [
+        ([[1, 2, 3], [4, 5, 6]], [[1, 2, 3], [1, 2, 3], [4, 5, 6]], [2.0, 1.0]),
+        (
+            [[1, 0, 1], [0, 0, 1]],
+            [[1, 0, 1], [0, 0, 1], [0, 0, 1], [0, 0, 1]],
+            np.array([1, 3]),
+        ),
+        (
+            [[1, np.nan, 1], [np.nan, np.nan, 1]],
+            [
+                [1, np.nan, 1],
+                [np.nan, np.nan, 1],
+                [np.nan, np.nan, 1],
+                [np.nan, np.nan, 1],
+            ],
+            np.array([1, 3]),
+        ),
+    ],
 )
-def test_standard_scaler_sample_weight(
-        Xw, X, sample_weight, array_constructor):
+@pytest.mark.parametrize("array_constructor", ["array", "sparse_csr", "sparse_csc"])
+def test_standard_scaler_sample_weight(Xw, X, sample_weight, array_constructor):
     with_mean = not array_constructor.startswith("sparse")
     X = _convert_container(X, array_constructor)
     Xw = _convert_container(Xw, array_constructor)
@@ -360,17 +173,14 @@ def test_standard_scaler_1d():
         if _check_dim_1axis(X) == 1:
             assert_almost_equal(scaler.mean_, X.ravel())
             assert_almost_equal(scaler.scale_, np.ones(n_features))
-            assert_array_almost_equal(X_scaled.mean(axis=0),
-                                      np.zeros_like(n_features))
-            assert_array_almost_equal(X_scaled.std(axis=0),
-                                      np.zeros_like(n_features))
+            assert_array_almost_equal(X_scaled.mean(axis=0), np.zeros_like(n_features))
+            assert_array_almost_equal(X_scaled.std(axis=0), np.zeros_like(n_features))
         else:
             assert_almost_equal(scaler.mean_, X.mean())
             assert_almost_equal(scaler.scale_, X.std())
-            assert_array_almost_equal(X_scaled.mean(axis=0),
-                                      np.zeros_like(n_features))
-            assert_array_almost_equal(X_scaled.mean(axis=0), .0)
-            assert_array_almost_equal(X_scaled.std(axis=0), 1.)
+            assert_array_almost_equal(X_scaled.mean(axis=0), np.zeros_like(n_features))
+            assert_array_almost_equal(X_scaled.mean(axis=0), 0.0)
+            assert_array_almost_equal(X_scaled.std(axis=0), 1.0)
         assert scaler.n_samples_seen_ == X.shape[0]
 
         # check inverse transform
@@ -381,17 +191,16 @@ def test_standard_scaler_1d():
     X = np.ones((5, 1))
     scaler = StandardScaler()
     X_scaled = scaler.fit(X).transform(X, copy=True)
-    assert_almost_equal(scaler.mean_, 1.)
-    assert_almost_equal(scaler.scale_, 1.)
-    assert_array_almost_equal(X_scaled.mean(axis=0), .0)
-    assert_array_almost_equal(X_scaled.std(axis=0), .0)
+    assert_almost_equal(scaler.mean_, 1.0)
+    assert_almost_equal(scaler.scale_, 1.0)
+    assert_array_almost_equal(X_scaled.mean(axis=0), 0.0)
+    assert_array_almost_equal(X_scaled.std(axis=0), 0.0)
     assert scaler.n_samples_seen_ == X.shape[0]
 
 
-@pytest.mark.parametrize("sparse_constructor",
-                         [None, sparse.csc_matrix, sparse.csr_matrix])
+@pytest.mark.parametrize("sparse_container", [None] + CSC_CONTAINERS + CSR_CONTAINERS)
 @pytest.mark.parametrize("add_sample_weight", [False, True])
-def test_standard_scaler_dtype(add_sample_weight, sparse_constructor):
+def test_standard_scaler_dtype(add_sample_weight, sparse_container):
     # Ensure scaling does not affect dtype
     rng = np.random.RandomState(0)
     n_samples = 10
@@ -401,10 +210,16 @@ def test_standard_scaler_dtype(add_sample_weight, sparse_constructor):
     else:
         sample_weight = None
     with_mean = True
-    for dtype in [np.float16, np.float32, np.float64]:
+    if sparse_container is not None:
+        # scipy sparse containers do not support float16, see
+        # https://github.com/scipy/scipy/issues/7408 for more details.
+        supported_dtype = [np.float64, np.float32]
+    else:
+        supported_dtype = [np.float64, np.float32, np.float16]
+    for dtype in supported_dtype:
         X = rng.randn(n_samples, n_features).astype(dtype)
-        if sparse_constructor is not None:
-            X = sparse_constructor(X)
+        if sparse_container is not None:
+            X = sparse_container(X)
             with_mean = False
 
         scaler = StandardScaler(with_mean=with_mean)
@@ -414,28 +229,22 @@ def test_standard_scaler_dtype(add_sample_weight, sparse_constructor):
         assert scaler.scale_.dtype == np.float64
 
 
-@pytest.mark.parametrize("scaler", [
-    StandardScaler(with_mean=False),
-    RobustScaler(with_centering=False),
-])
-@pytest.mark.parametrize("sparse_constructor",
-                         [np.asarray, sparse.csc_matrix, sparse.csr_matrix])
+@pytest.mark.parametrize(
+    "scaler",
+    [
+        StandardScaler(with_mean=False),
+        RobustScaler(with_centering=False),
+    ],
+)
+@pytest.mark.parametrize("sparse_container", [None] + CSC_CONTAINERS + CSR_CONTAINERS)
 @pytest.mark.parametrize("add_sample_weight", [False, True])
 @pytest.mark.parametrize("dtype", [np.float32, np.float64])
-@pytest.mark.parametrize("constant", [0, 1., 100.])
+@pytest.mark.parametrize("constant", [0, 1.0, 100.0])
 def test_standard_scaler_constant_features(
-        scaler, add_sample_weight, sparse_constructor, dtype, constant):
-    if (isinstance(scaler, StandardScaler)
-            and constant > 1
-            and sparse_constructor is not np.asarray
-            and add_sample_weight):
-        # https://github.com/scikit-learn/scikit-learn/issues/19546
-        pytest.xfail("Computation of weighted variance is numerically unstable"
-                     " for sparse data. See: #19546.")
-
+    scaler, add_sample_weight, sparse_container, dtype, constant
+):
     if isinstance(scaler, RobustScaler) and add_sample_weight:
-        pytest.skip(f"{scaler.__class__.__name__} does not yet support"
-                    f" sample_weight")
+        pytest.skip(f"{scaler.__class__.__name__} does not yet support sample_weight")
 
     rng = np.random.RandomState(0)
     n_samples = 100
@@ -444,9 +253,8 @@ def test_standard_scaler_constant_features(
         fit_params = dict(sample_weight=rng.uniform(size=n_samples) * 2)
     else:
         fit_params = {}
-    X_array = np.full(shape=(n_samples, n_features), fill_value=constant,
-                      dtype=dtype)
-    X = sparse_constructor(X_array)
+    X_array = np.full(shape=(n_samples, n_features), fill_value=constant, dtype=dtype)
+    X = X_array if sparse_container is None else sparse_container(X_array)
     X_scaled = scaler.fit(X, **fit_params).transform(X)
 
     if isinstance(scaler, StandardScaler):
@@ -456,23 +264,73 @@ def test_standard_scaler_constant_features(
     # Constant features should not be scaled (scale of 1.):
     assert_allclose(scaler.scale_, np.ones(X.shape[1]))
 
-    if hasattr(X_scaled, "toarray"):
-        assert_allclose(X_scaled.toarray(), X_array)
-    else:
-        assert_allclose(X_scaled, X)
+    assert X_scaled is not X  # make sure we make a copy
+    assert_allclose_dense_sparse(X_scaled, X)
 
     if isinstance(scaler, StandardScaler) and not add_sample_weight:
         # Also check consistency with the standard scale function.
         X_scaled_2 = scale(X, with_mean=scaler.with_mean)
-        if hasattr(X_scaled_2, "toarray"):
-            assert_allclose(X_scaled_2.toarray(), X_scaled_2.toarray())
-        else:
-            assert_allclose(X_scaled_2, X_scaled_2)
+        assert X_scaled_2 is not X  # make sure we did a copy
+        assert_allclose_dense_sparse(X_scaled_2, X)
+
+
+@pytest.mark.parametrize("n_samples", [10, 100, 10_000])
+@pytest.mark.parametrize("average", [1e-10, 1, 1e10])
+@pytest.mark.parametrize("dtype", [np.float32, np.float64])
+@pytest.mark.parametrize("sparse_container", [None] + CSC_CONTAINERS + CSR_CONTAINERS)
+def test_standard_scaler_near_constant_features(
+    n_samples, sparse_container, average, dtype
+):
+    # Check that when the variance is too small (var << mean**2) the feature
+    # is considered constant and not scaled.
+
+    scale_min, scale_max = -30, 19
+    scales = np.array([10**i for i in range(scale_min, scale_max + 1)], dtype=dtype)
+
+    n_features = scales.shape[0]
+    X = np.empty((n_samples, n_features), dtype=dtype)
+    # Make a dataset of known var = scales**2 and mean = average
+    X[: n_samples // 2, :] = average + scales
+    X[n_samples // 2 :, :] = average - scales
+    X_array = X if sparse_container is None else sparse_container(X)
+
+    scaler = StandardScaler(with_mean=False).fit(X_array)
+
+    # StandardScaler uses float64 accumulators even if the data has a float32
+    # dtype.
+    eps = np.finfo(np.float64).eps
+
+    # if var < bound = N.eps.var + N².eps².mean², the feature is considered
+    # constant and the scale_ attribute is set to 1.
+    bounds = n_samples * eps * scales**2 + n_samples**2 * eps**2 * average**2
+    within_bounds = scales**2 <= bounds
+
+    # Check that scale_min is small enough to have some scales below the
+    # bound and therefore detected as constant:
+    assert np.any(within_bounds)
+
+    # Check that such features are actually treated as constant by the scaler:
+    assert all(scaler.var_[within_bounds] <= bounds[within_bounds])
+    assert_allclose(scaler.scale_[within_bounds], 1.0)
+
+    # Depending the on the dtype of X, some features might not actually be
+    # representable as non constant for small scales (even if above the
+    # precision bound of the float64 variance estimate). Such feature should
+    # be correctly detected as constants with 0 variance by StandardScaler.
+    representable_diff = X[0, :] - X[-1, :] != 0
+    assert_allclose(scaler.var_[np.logical_not(representable_diff)], 0)
+    assert_allclose(scaler.scale_[np.logical_not(representable_diff)], 1)
+
+    # The other features are scaled and scale_ is equal to sqrt(var_) assuming
+    # that scales are large enough for average + scale and average - scale to
+    # be distinct in X (depending on X's dtype).
+    common_mask = np.logical_and(scales**2 > bounds, representable_diff)
+    assert_allclose(scaler.scale_[common_mask], np.sqrt(scaler.var_)[common_mask])
 
 
 def test_scale_1d():
     # 1-d inputs
-    X_list = [1., 3., 5., 0.]
+    X_list = [1.0, 3.0, 5.0, 0.0]
     X_arr = np.array(X_list)
 
     for X in [X_list, X_arr]:
@@ -490,28 +348,33 @@ def test_standard_scaler_numerical_stability():
     x = np.full(8, np.log(1e-5), dtype=np.float64)
     # This does not raise a warning as the number of samples is too low
     # to trigger the problem in recent numpy
-    x_scaled = assert_no_warnings(scale, x)
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", UserWarning)
+        scale(x)
     assert_array_almost_equal(scale(x), np.zeros(8))
 
     # with 2 more samples, the std computation run into numerical issues:
     x = np.full(10, np.log(1e-5), dtype=np.float64)
-    w = "standard deviation of the data is probably very close to 0"
-    x_scaled = assert_warns_message(UserWarning, w, scale, x)
+    warning_message = "standard deviation of the data is probably very close to 0"
+    with pytest.warns(UserWarning, match=warning_message):
+        x_scaled = scale(x)
     assert_array_almost_equal(x_scaled, np.zeros(10))
 
     x = np.full(10, 1e-100, dtype=np.float64)
-    x_small_scaled = assert_no_warnings(scale, x)
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", UserWarning)
+        x_small_scaled = scale(x)
     assert_array_almost_equal(x_small_scaled, np.zeros(10))
 
     # Large values can cause (often recoverable) numerical stability issues:
     x_big = np.full(10, 1e100, dtype=np.float64)
-    w = "Dataset may contain too large values"
-    x_big_scaled = assert_warns_message(UserWarning, w, scale, x_big)
+    warning_message = "Dataset may contain too large values"
+    with pytest.warns(UserWarning, match=warning_message):
+        x_big_scaled = scale(x_big)
     assert_array_almost_equal(x_big_scaled, np.zeros(10))
     assert_array_almost_equal(x_big_scaled, x_small_scaled)
-
-    x_big_centered = assert_warns_message(UserWarning, w, scale, x_big,
-                                          with_std=False)
+    with pytest.warns(UserWarning, match=warning_message):
+        x_big_centered = scale(x_big, with_std=False)
     assert_array_almost_equal(x_big_centered, np.zeros(10))
     assert_array_almost_equal(x_big_centered, x_small_scaled)
 
@@ -530,7 +393,7 @@ def test_scaler_2d_arrays():
     assert scaler.n_samples_seen_ == n_samples
 
     assert_array_almost_equal(X_scaled.mean(axis=0), n_features * [0.0])
-    assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.])
+    assert_array_almost_equal(X_scaled.std(axis=0), [0.0, 1.0, 1.0, 1.0, 1.0])
     # Check that X has been copied
     assert X_scaled is not X
 
@@ -553,7 +416,7 @@ def test_scaler_2d_arrays():
     X_scaled = scaler.fit(X).transform(X, copy=False)
     assert not np.any(np.isnan(X_scaled))
     assert_array_almost_equal(X_scaled.mean(axis=0), n_features * [0.0])
-    assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.])
+    assert_array_almost_equal(X_scaled.std(axis=0), [0.0, 1.0, 1.0, 1.0, 1.0])
     # Check that X has not been copied
     assert X_scaled is X
 
@@ -563,7 +426,7 @@ def test_scaler_2d_arrays():
     X_scaled = scaler.fit(X).transform(X, copy=True)
     assert not np.any(np.isnan(X_scaled))
     assert_array_almost_equal(X_scaled.mean(axis=0), n_features * [0.0])
-    assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.])
+    assert_array_almost_equal(X_scaled.std(axis=0), [0.0, 1.0, 1.0, 1.0, 1.0])
     # Check that X has not been copied
     assert X_scaled is not X
 
@@ -575,7 +438,7 @@ def test_scaler_float16_overflow():
     # which is enough to overflow the data type
     X = rng.uniform(5, 10, [200000, 1]).astype(np.float16)
 
-    with np.errstate(over='raise'):
+    with np.errstate(over="raise"):
         scaler = StandardScaler().fit(X)
         X_scaled = scaler.transform(X)
 
@@ -615,13 +478,10 @@ def test_minmax_scaler_partial_fit():
         for batch in gen_batches(n_samples, chunk_size):
             scaler_incr = scaler_incr.partial_fit(X[batch])
 
-        assert_array_almost_equal(scaler_batch.data_min_,
-                                  scaler_incr.data_min_)
-        assert_array_almost_equal(scaler_batch.data_max_,
-                                  scaler_incr.data_max_)
+        assert_array_almost_equal(scaler_batch.data_min_, scaler_incr.data_min_)
+        assert_array_almost_equal(scaler_batch.data_max_, scaler_incr.data_max_)
         assert scaler_batch.n_samples_seen_ == scaler_incr.n_samples_seen_
-        assert_array_almost_equal(scaler_batch.data_range_,
-                                  scaler_incr.data_range_)
+        assert_array_almost_equal(scaler_batch.data_range_, scaler_incr.data_range_)
         assert_array_almost_equal(scaler_batch.scale_, scaler_incr.scale_)
         assert_array_almost_equal(scaler_batch.min_, scaler_incr.min_)
 
@@ -630,13 +490,10 @@ def test_minmax_scaler_partial_fit():
         scaler_batch = MinMaxScaler().fit(X[batch0])
         scaler_incr = MinMaxScaler().partial_fit(X[batch0])
 
-        assert_array_almost_equal(scaler_batch.data_min_,
-                                  scaler_incr.data_min_)
-        assert_array_almost_equal(scaler_batch.data_max_,
-                                  scaler_incr.data_max_)
+        assert_array_almost_equal(scaler_batch.data_min_, scaler_incr.data_min_)
+        assert_array_almost_equal(scaler_batch.data_max_, scaler_incr.data_max_)
         assert scaler_batch.n_samples_seen_ == scaler_incr.n_samples_seen_
-        assert_array_almost_equal(scaler_batch.data_range_,
-                                  scaler_incr.data_range_)
+        assert_array_almost_equal(scaler_batch.data_range_, scaler_incr.data_range_)
         assert_array_almost_equal(scaler_batch.scale_, scaler_incr.scale_)
         assert_array_almost_equal(scaler_batch.min_, scaler_incr.min_)
 
@@ -645,10 +502,14 @@ def test_minmax_scaler_partial_fit():
         scaler_incr = MinMaxScaler()  # Clean estimator
         for i, batch in enumerate(gen_batches(n_samples, chunk_size)):
             scaler_incr = scaler_incr.partial_fit(X[batch])
-            assert_correct_incr(i, batch_start=batch.start,
-                                batch_stop=batch.stop, n=n,
-                                chunk_size=chunk_size,
-                                n_samples_seen=scaler_incr.n_samples_seen_)
+            assert_correct_incr(
+                i,
+                batch_start=batch.start,
+                batch_stop=batch.stop,
+                n=n,
+                chunk_size=chunk_size,
+                n_samples_seen=scaler_incr.n_samples_seen_,
+            )
 
 
 def test_standard_scaler_partial_fit():
@@ -672,31 +533,38 @@ def test_standard_scaler_partial_fit():
         batch0 = slice(0, chunk_size)
         scaler_incr = StandardScaler().partial_fit(X[batch0])
         if chunk_size == 1:
-            assert_array_almost_equal(np.zeros(n_features, dtype=np.float64),
-                                      scaler_incr.var_)
-            assert_array_almost_equal(np.ones(n_features, dtype=np.float64),
-                                      scaler_incr.scale_)
+            assert_array_almost_equal(
+                np.zeros(n_features, dtype=np.float64), scaler_incr.var_
+            )
+            assert_array_almost_equal(
+                np.ones(n_features, dtype=np.float64), scaler_incr.scale_
+            )
         else:
-            assert_array_almost_equal(np.var(X[batch0], axis=0),
-                                      scaler_incr.var_)
-            assert_array_almost_equal(np.std(X[batch0], axis=0),
-                                      scaler_incr.scale_)  # no constants
+            assert_array_almost_equal(np.var(X[batch0], axis=0), scaler_incr.var_)
+            assert_array_almost_equal(
+                np.std(X[batch0], axis=0), scaler_incr.scale_
+            )  # no constants
 
         # Test std until the end of partial fits, and
         scaler_batch = StandardScaler().fit(X)
         scaler_incr = StandardScaler()  # Clean estimator
         for i, batch in enumerate(gen_batches(n_samples, chunk_size)):
             scaler_incr = scaler_incr.partial_fit(X[batch])
-            assert_correct_incr(i, batch_start=batch.start,
-                                batch_stop=batch.stop, n=n,
-                                chunk_size=chunk_size,
-                                n_samples_seen=scaler_incr.n_samples_seen_)
+            assert_correct_incr(
+                i,
+                batch_start=batch.start,
+                batch_stop=batch.stop,
+                n=n,
+                chunk_size=chunk_size,
+                n_samples_seen=scaler_incr.n_samples_seen_,
+            )
 
         assert_array_almost_equal(scaler_batch.var_, scaler_incr.var_)
         assert scaler_batch.n_samples_seen_ == scaler_incr.n_samples_seen_
 
 
-def test_standard_scaler_partial_fit_numerical_stability():
+@pytest.mark.parametrize("sparse_container", CSC_CONTAINERS + CSR_CONTAINERS)
+def test_standard_scaler_partial_fit_numerical_stability(sparse_container):
     # Test if the incremental computation introduces significative errors
     # for large datasets with values of large magniture
     rng = np.random.RandomState(0)
@@ -722,45 +590,41 @@ def test_standard_scaler_partial_fit_numerical_stability():
     # Sparse input
     size = (100, 3)
     scale = 1e20
-    X = rng.randint(0, 2, size).astype(np.float64) * scale
-    X_csr = sparse.csr_matrix(X)
-    X_csc = sparse.csc_matrix(X)
+    X = sparse_container(rng.randint(0, 2, size).astype(np.float64) * scale)
 
-    for X in [X_csr, X_csc]:
-        # with_mean=False is required with sparse input
-        scaler = StandardScaler(with_mean=False).fit(X)
-        scaler_incr = StandardScaler(with_mean=False)
+    # with_mean=False is required with sparse input
+    scaler = StandardScaler(with_mean=False).fit(X)
+    scaler_incr = StandardScaler(with_mean=False)
 
-        for chunk in X:
-            # chunk = sparse.csr_matrix(data_chunks)
-            scaler_incr = scaler_incr.partial_fit(chunk)
+    for chunk in X:
+        if chunk.ndim == 1:
+            # Sparse arrays can be 1D (in scipy 1.14 and later) while old
+            # sparse matrix instances are always 2D.
+            chunk = chunk.reshape(1, -1)
+        scaler_incr = scaler_incr.partial_fit(chunk)
 
-        # Regardless of magnitude, they must not differ more than of 6 digits
-        tol = 10 ** (-6)
-        assert scaler.mean_ is not None
-        assert_allclose(scaler_incr.var_, scaler.var_, rtol=tol)
-        assert_allclose(scaler_incr.scale_, scaler.scale_, rtol=tol)
+    # Regardless of magnitude, they must not differ more than of 6 digits
+    tol = 10 ** (-6)
+    assert scaler.mean_ is not None
+    assert_allclose(scaler_incr.var_, scaler.var_, rtol=tol)
+    assert_allclose(scaler_incr.scale_, scaler.scale_, rtol=tol)
 
 
 @pytest.mark.parametrize("sample_weight", [True, None])
-def test_partial_fit_sparse_input(sample_weight):
+@pytest.mark.parametrize("sparse_container", CSC_CONTAINERS + CSR_CONTAINERS)
+def test_partial_fit_sparse_input(sample_weight, sparse_container):
     # Check that sparsity is not destroyed
-    X = np.array([[1.], [0.], [0.], [5.]])
-    X_csr = sparse.csr_matrix(X)
-    X_csc = sparse.csc_matrix(X)
+    X = sparse_container(np.array([[1.0], [0.0], [0.0], [5.0]]))
 
     if sample_weight:
-        sample_weight = rng.rand(X_csc.shape[0])
+        sample_weight = rng.rand(X.shape[0])
 
     null_transform = StandardScaler(with_mean=False, with_std=False, copy=True)
-    for X in [X_csr, X_csc]:
-
-        X_null = null_transform.partial_fit(
-            X, sample_weight=sample_weight).transform(X)
-        assert_array_equal(X_null.toarray(), X.toarray())
-        X_orig = null_transform.inverse_transform(X_null)
-        assert_array_equal(X_orig.toarray(), X_null.toarray())
-        assert_array_equal(X_orig.toarray(), X.toarray())
+    X_null = null_transform.partial_fit(X, sample_weight=sample_weight).transform(X)
+    assert_array_equal(X_null.toarray(), X.toarray())
+    X_orig = null_transform.inverse_transform(X_null)
+    assert_array_equal(X_orig.toarray(), X_null.toarray())
+    assert_array_equal(X_orig.toarray(), X.toarray())
 
 
 @pytest.mark.parametrize("sample_weight", [True, None])
@@ -773,17 +637,18 @@ def test_standard_scaler_trasform_with_partial_fit(sample_weight):
 
     scaler_incr = StandardScaler()
     for i, batch in enumerate(gen_batches(X.shape[0], 1)):
-
-        X_sofar = X[:(i + 1), :]
+        X_sofar = X[: (i + 1), :]
         chunks_copy = X_sofar.copy()
         if sample_weight is None:
             scaled_batch = StandardScaler().fit_transform(X_sofar)
             scaler_incr = scaler_incr.partial_fit(X[batch])
         else:
             scaled_batch = StandardScaler().fit_transform(
-                X_sofar, sample_weight=sample_weight[:i + 1])
+                X_sofar, sample_weight=sample_weight[: i + 1]
+            )
             scaler_incr = scaler_incr.partial_fit(
-                X[batch], sample_weight=sample_weight[batch])
+                X[batch], sample_weight=sample_weight[batch]
+            )
         scaled_incr = scaler_incr.transform(X_sofar)
 
         assert_array_almost_equal(scaled_batch, scaled_incr)
@@ -799,12 +664,66 @@ def test_standard_scaler_trasform_with_partial_fit(sample_weight):
             # (i+1) because the Scaler has been already fitted
             assert (i + 1) == scaler_incr.n_samples_seen_
         else:
-            assert (
-                np.sum(sample_weight[:i + 1]) ==
-                pytest.approx(scaler_incr.n_samples_seen_)
+            assert np.sum(sample_weight[: i + 1]) == pytest.approx(
+                scaler_incr.n_samples_seen_
             )
 
 
+def test_standard_check_array_of_inverse_transform():
+    # Check if StandardScaler inverse_transform is
+    # converting the integer array to float
+    x = np.array(
+        [
+            [1, 1, 1, 0, 1, 0],
+            [1, 1, 1, 0, 1, 0],
+            [0, 8, 0, 1, 0, 0],
+            [1, 4, 1, 1, 0, 0],
+            [0, 1, 0, 0, 1, 0],
+            [0, 4, 0, 1, 0, 1],
+        ],
+        dtype=np.int32,
+    )
+
+    scaler = StandardScaler()
+    scaler.fit(x)
+
+    # The of inverse_transform should be converted
+    # to a float array.
+    # If not X *= self.scale_ will fail.
+    scaler.inverse_transform(x)
+
+
+@pytest.mark.parametrize(
+    "array_namespace, device, dtype_name",
+    yield_namespace_device_dtype_combinations(),
+    ids=_get_namespace_device_dtype_ids,
+)
+@pytest.mark.parametrize(
+    "check",
+    [check_array_api_input_and_values],
+    ids=_get_check_estimator_ids,
+)
+@pytest.mark.parametrize(
+    "estimator",
+    [
+        MaxAbsScaler(),
+        MinMaxScaler(),
+        MinMaxScaler(clip=True),
+        KernelCenterer(),
+        Normalizer(norm="l1"),
+        Normalizer(norm="l2"),
+        Normalizer(norm="max"),
+        Binarizer(),
+    ],
+    ids=_get_check_estimator_ids,
+)
+def test_preprocessing_array_api_compliance(
+    estimator, check, array_namespace, device, dtype_name
+):
+    name = estimator.__class__.__name__
+    check(name, estimator, array_namespace, device=device, dtype_name=dtype_name)
+
+
 def test_min_max_scaler_iris():
     X = iris.data
     scaler = MinMaxScaler()
@@ -824,10 +743,10 @@ def test_min_max_scaler_iris():
     assert_array_almost_equal(X, X_trans_inv)
 
     # min=-.5, max=.6
-    scaler = MinMaxScaler(feature_range=(-.5, .6))
+    scaler = MinMaxScaler(feature_range=(-0.5, 0.6))
     X_trans = scaler.fit_transform(X)
-    assert_array_almost_equal(X_trans.min(axis=0), -.5)
-    assert_array_almost_equal(X_trans.max(axis=0), .6)
+    assert_array_almost_equal(X_trans.min(axis=0), -0.5)
+    assert_array_almost_equal(X_trans.max(axis=0), 0.6)
     X_trans_inv = scaler.inverse_transform(X_trans)
     assert_array_almost_equal(X, X_trans_inv)
 
@@ -839,36 +758,26 @@ def test_min_max_scaler_iris():
 
 def test_min_max_scaler_zero_variance_features():
     # Check min max scaler on toy data with zero variance features
-    X = [[0., 1., +0.5],
-         [0., 1., -0.1],
-         [0., 1., +1.1]]
+    X = [[0.0, 1.0, +0.5], [0.0, 1.0, -0.1], [0.0, 1.0, +1.1]]
 
-    X_new = [[+0., 2., 0.5],
-             [-1., 1., 0.0],
-             [+0., 1., 1.5]]
+    X_new = [[+0.0, 2.0, 0.5], [-1.0, 1.0, 0.0], [+0.0, 1.0, 1.5]]
 
     # default params
     scaler = MinMaxScaler()
     X_trans = scaler.fit_transform(X)
-    X_expected_0_1 = [[0., 0., 0.5],
-                      [0., 0., 0.0],
-                      [0., 0., 1.0]]
+    X_expected_0_1 = [[0.0, 0.0, 0.5], [0.0, 0.0, 0.0], [0.0, 0.0, 1.0]]
     assert_array_almost_equal(X_trans, X_expected_0_1)
     X_trans_inv = scaler.inverse_transform(X_trans)
     assert_array_almost_equal(X, X_trans_inv)
 
     X_trans_new = scaler.transform(X_new)
-    X_expected_0_1_new = [[+0., 1., 0.500],
-                          [-1., 0., 0.083],
-                          [+0., 0., 1.333]]
+    X_expected_0_1_new = [[+0.0, 1.0, 0.500], [-1.0, 0.0, 0.083], [+0.0, 0.0, 1.333]]
     assert_array_almost_equal(X_trans_new, X_expected_0_1_new, decimal=2)
 
     # not default params
     scaler = MinMaxScaler(feature_range=(1, 2))
     X_trans = scaler.fit_transform(X)
-    X_expected_1_2 = [[1., 1., 1.5],
-                      [1., 1., 1.0],
-                      [1., 1., 2.0]]
+    X_expected_1_2 = [[1.0, 1.0, 1.5], [1.0, 1.0, 1.0], [1.0, 1.0, 2.0]]
     assert_array_almost_equal(X_trans, X_expected_1_2)
 
     # function interface
@@ -888,7 +797,6 @@ def test_minmax_scale_axis1():
 def test_min_max_scaler_1d():
     # Test scaling of dataset along single axis
     for X in [X_1row, X_1col, X_list_1row, X_list_1row]:
-
         scaler = MinMaxScaler(copy=True)
         X_scaled = scaler.fit(X).transform(X)
 
@@ -896,13 +804,11 @@ def test_min_max_scaler_1d():
             X = np.array(X)  # cast only after scaling done
 
         if _check_dim_1axis(X) == 1:
-            assert_array_almost_equal(X_scaled.min(axis=0),
-                                      np.zeros(n_features))
-            assert_array_almost_equal(X_scaled.max(axis=0),
-                                      np.zeros(n_features))
+            assert_array_almost_equal(X_scaled.min(axis=0), np.zeros(n_features))
+            assert_array_almost_equal(X_scaled.max(axis=0), np.zeros(n_features))
         else:
-            assert_array_almost_equal(X_scaled.min(axis=0), .0)
-            assert_array_almost_equal(X_scaled.max(axis=0), 1.)
+            assert_array_almost_equal(X_scaled.min(axis=0), 0.0)
+            assert_array_almost_equal(X_scaled.max(axis=0), 1.0)
         assert scaler.n_samples_seen_ == X.shape[0]
 
         # check inverse transform
@@ -913,109 +819,89 @@ def test_min_max_scaler_1d():
     X = np.ones((5, 1))
     scaler = MinMaxScaler()
     X_scaled = scaler.fit(X).transform(X)
-    assert X_scaled.min() >= 0.
-    assert X_scaled.max() <= 1.
+    assert X_scaled.min() >= 0.0
+    assert X_scaled.max() <= 1.0
     assert scaler.n_samples_seen_ == X.shape[0]
 
     # Function interface
     X_1d = X_1row.ravel()
     min_ = X_1d.min()
     max_ = X_1d.max()
-    assert_array_almost_equal((X_1d - min_) / (max_ - min_),
-                              minmax_scale(X_1d, copy=True))
+    assert_array_almost_equal(
+        (X_1d - min_) / (max_ - min_), minmax_scale(X_1d, copy=True)
+    )
 
 
 @pytest.mark.parametrize("sample_weight", [True, None])
-def test_scaler_without_centering(sample_weight):
+@pytest.mark.parametrize("sparse_container", CSC_CONTAINERS + CSR_CONTAINERS)
+def test_scaler_without_centering(sample_weight, sparse_container):
     rng = np.random.RandomState(42)
     X = rng.randn(4, 5)
     X[:, 0] = 0.0  # first feature is always of zero
-    X_csr = sparse.csr_matrix(X)
-    X_csc = sparse.csc_matrix(X)
+    X_sparse = sparse_container(X)
 
     if sample_weight:
         sample_weight = rng.rand(X.shape[0])
 
     with pytest.raises(ValueError):
-        StandardScaler().fit(X_csr)
-    with pytest.raises(ValueError):
-        StandardScaler().fit(X_csc)
-
-    null_transform = StandardScaler(with_mean=False, with_std=False, copy=True)
-    X_null = null_transform.fit_transform(X_csr)
-    assert_array_equal(X_null.data, X_csr.data)
-    X_orig = null_transform.inverse_transform(X_null)
-    assert_array_equal(X_orig.data, X_csr.data)
+        StandardScaler().fit(X_sparse)
 
-    scaler = StandardScaler(with_mean=False).fit(
-        X, sample_weight=sample_weight)
+    scaler = StandardScaler(with_mean=False).fit(X, sample_weight=sample_weight)
     X_scaled = scaler.transform(X, copy=True)
     assert not np.any(np.isnan(X_scaled))
 
-    scaler_csr = StandardScaler(with_mean=False).fit(
-        X_csr, sample_weight=sample_weight)
-    X_csr_scaled = scaler_csr.transform(X_csr, copy=True)
-    assert not np.any(np.isnan(X_csr_scaled.data))
-
-    scaler_csc = StandardScaler(with_mean=False).fit(
-        X_csc, sample_weight=sample_weight)
-    X_csc_scaled = scaler_csc.transform(X_csc, copy=True)
-    assert not np.any(np.isnan(X_csc_scaled.data))
-
-    assert_array_almost_equal(scaler.mean_, scaler_csr.mean_)
-    assert_array_almost_equal(scaler.var_, scaler_csr.var_)
-    assert_array_almost_equal(scaler.scale_, scaler_csr.scale_)
-    assert_array_almost_equal(scaler.n_samples_seen_,
-                              scaler_csr.n_samples_seen_)
+    scaler_sparse = StandardScaler(with_mean=False).fit(
+        X_sparse, sample_weight=sample_weight
+    )
+    X_sparse_scaled = scaler_sparse.transform(X_sparse, copy=True)
+    assert not np.any(np.isnan(X_sparse_scaled.data))
 
-    assert_array_almost_equal(scaler.mean_, scaler_csc.mean_)
-    assert_array_almost_equal(scaler.var_, scaler_csc.var_)
-    assert_array_almost_equal(scaler.scale_, scaler_csc.scale_)
-    assert_array_almost_equal(scaler.n_samples_seen_,
-                              scaler_csc.n_samples_seen_)
+    assert_array_almost_equal(scaler.mean_, scaler_sparse.mean_)
+    assert_array_almost_equal(scaler.var_, scaler_sparse.var_)
+    assert_array_almost_equal(scaler.scale_, scaler_sparse.scale_)
+    assert_array_almost_equal(scaler.n_samples_seen_, scaler_sparse.n_samples_seen_)
 
     if sample_weight is None:
         assert_array_almost_equal(
-            X_scaled.mean(axis=0), [0., -0.01, 2.24, -0.35, -0.78], 2)
-        assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.])
+            X_scaled.mean(axis=0), [0.0, -0.01, 2.24, -0.35, -0.78], 2
+        )
+        assert_array_almost_equal(X_scaled.std(axis=0), [0.0, 1.0, 1.0, 1.0, 1.0])
 
-    X_csr_scaled_mean, X_csr_scaled_var = \
-        mean_variance_axis(X_csr_scaled, 0)
-    assert_array_almost_equal(X_csr_scaled_mean, X_scaled.mean(axis=0))
-    assert_array_almost_equal(X_csr_scaled_var, X_scaled.var(axis=0))
+    X_sparse_scaled_mean, X_sparse_scaled_var = mean_variance_axis(X_sparse_scaled, 0)
+    assert_array_almost_equal(X_sparse_scaled_mean, X_scaled.mean(axis=0))
+    assert_array_almost_equal(X_sparse_scaled_var, X_scaled.var(axis=0))
 
     # Check that X has not been modified (copy)
     assert X_scaled is not X
-    assert X_csr_scaled is not X_csr
+    assert X_sparse_scaled is not X_sparse
 
     X_scaled_back = scaler.inverse_transform(X_scaled)
     assert X_scaled_back is not X
     assert X_scaled_back is not X_scaled
     assert_array_almost_equal(X_scaled_back, X)
 
-    X_csr_scaled_back = scaler_csr.inverse_transform(X_csr_scaled)
-    assert X_csr_scaled_back is not X_csr
-    assert X_csr_scaled_back is not X_csr_scaled
-    assert_array_almost_equal(X_csr_scaled_back.toarray(), X)
+    X_sparse_scaled_back = scaler_sparse.inverse_transform(X_sparse_scaled)
+    assert X_sparse_scaled_back is not X_sparse
+    assert X_sparse_scaled_back is not X_sparse_scaled
+    assert_array_almost_equal(X_sparse_scaled_back.toarray(), X)
 
-    X_csc_scaled_back = scaler_csr.inverse_transform(X_csc_scaled.tocsc())
-    assert X_csc_scaled_back is not X_csc
-    assert X_csc_scaled_back is not X_csc_scaled
-    assert_array_almost_equal(X_csc_scaled_back.toarray(), X)
+    if sparse_container in CSR_CONTAINERS:
+        null_transform = StandardScaler(with_mean=False, with_std=False, copy=True)
+        X_null = null_transform.fit_transform(X_sparse)
+        assert_array_equal(X_null.data, X_sparse.data)
+        X_orig = null_transform.inverse_transform(X_null)
+        assert_array_equal(X_orig.data, X_sparse.data)
 
 
 @pytest.mark.parametrize("with_mean", [True, False])
 @pytest.mark.parametrize("with_std", [True, False])
-@pytest.mark.parametrize("array_constructor",
-                         [np.asarray, sparse.csc_matrix, sparse.csr_matrix])
-def test_scaler_n_samples_seen_with_nan(with_mean, with_std,
-                                        array_constructor):
-    X = np.array([[0, 1, 3],
-                  [np.nan, 6, 10],
-                  [5, 4, np.nan],
-                  [8, 0, np.nan]],
-                 dtype=np.float64)
-    X = array_constructor(X)
+@pytest.mark.parametrize("sparse_container", [None] + CSC_CONTAINERS + CSR_CONTAINERS)
+def test_scaler_n_samples_seen_with_nan(with_mean, with_std, sparse_container):
+    X = np.array(
+        [[0, 1, 3], [np.nan, 6, 10], [5, 4, np.nan], [8, 0, np.nan]], dtype=np.float64
+    )
+    if sparse_container is not None:
+        X = sparse_container(X)
 
     if sparse.issparse(X) and with_mean:
         pytest.skip("'with_mean=True' cannot be used with sparse matrix.")
@@ -1033,71 +919,40 @@ def _check_identity_scalers_attributes(scaler_1, scaler_2):
     assert scaler_1.n_samples_seen_ == scaler_2.n_samples_seen_
 
 
-def test_scaler_return_identity():
+@pytest.mark.parametrize("sparse_container", CSC_CONTAINERS + CSR_CONTAINERS)
+def test_scaler_return_identity(sparse_container):
     # test that the scaler return identity when with_mean and with_std are
     # False
-    X_dense = np.array([[0, 1, 3],
-                        [5, 6, 0],
-                        [8, 0, 10]],
-                       dtype=np.float64)
-    X_csr = sparse.csr_matrix(X_dense)
-    X_csc = X_csr.tocsc()
+    X_dense = np.array([[0, 1, 3], [5, 6, 0], [8, 0, 10]], dtype=np.float64)
+    X_sparse = sparse_container(X_dense)
 
     transformer_dense = StandardScaler(with_mean=False, with_std=False)
     X_trans_dense = transformer_dense.fit_transform(X_dense)
-
-    transformer_csr = clone(transformer_dense)
-    X_trans_csr = transformer_csr.fit_transform(X_csr)
-
-    transformer_csc = clone(transformer_dense)
-    X_trans_csc = transformer_csc.fit_transform(X_csc)
-
-    assert_allclose_dense_sparse(X_trans_csr, X_csr)
-    assert_allclose_dense_sparse(X_trans_csc, X_csc)
     assert_allclose(X_trans_dense, X_dense)
 
-    for trans_1, trans_2 in itertools.combinations([transformer_dense,
-                                                    transformer_csr,
-                                                    transformer_csc],
-                                                   2):
-        _check_identity_scalers_attributes(trans_1, trans_2)
+    transformer_sparse = clone(transformer_dense)
+    X_trans_sparse = transformer_sparse.fit_transform(X_sparse)
+    assert_allclose_dense_sparse(X_trans_sparse, X_sparse)
 
-    transformer_dense.partial_fit(X_dense)
-    transformer_csr.partial_fit(X_csr)
-    transformer_csc.partial_fit(X_csc)
+    _check_identity_scalers_attributes(transformer_dense, transformer_sparse)
 
-    for trans_1, trans_2 in itertools.combinations([transformer_dense,
-                                                    transformer_csr,
-                                                    transformer_csc],
-                                                   2):
-        _check_identity_scalers_attributes(trans_1, trans_2)
+    transformer_dense.partial_fit(X_dense)
+    transformer_sparse.partial_fit(X_sparse)
+    _check_identity_scalers_attributes(transformer_dense, transformer_sparse)
 
     transformer_dense.fit(X_dense)
-    transformer_csr.fit(X_csr)
-    transformer_csc.fit(X_csc)
+    transformer_sparse.fit(X_sparse)
+    _check_identity_scalers_attributes(transformer_dense, transformer_sparse)
 
-    for trans_1, trans_2 in itertools.combinations([transformer_dense,
-                                                    transformer_csr,
-                                                    transformer_csc],
-                                                   2):
-        _check_identity_scalers_attributes(trans_1, trans_2)
 
-
-def test_scaler_int():
+@pytest.mark.parametrize("sparse_container", CSC_CONTAINERS + CSR_CONTAINERS)
+def test_scaler_int(sparse_container):
     # test that scaler converts integer input to floating
     # for both sparse and dense matrices
     rng = np.random.RandomState(42)
     X = rng.randint(20, size=(4, 5))
     X[:, 0] = 0  # first feature is always of zero
-    X_csr = sparse.csr_matrix(X)
-    X_csc = sparse.csc_matrix(X)
-
-    null_transform = StandardScaler(with_mean=False, with_std=False, copy=True)
-    with warnings.catch_warnings(record=True):
-        X_null = null_transform.fit_transform(X_csr)
-    assert_array_equal(X_null.data, X_csr.data)
-    X_orig = null_transform.inverse_transform(X_null)
-    assert_array_equal(X_orig.data, X_csr.data)
+    X_sparse = sparse_container(X)
 
     with warnings.catch_warnings(record=True):
         scaler = StandardScaler(with_mean=False).fit(X)
@@ -1105,112 +960,93 @@ def test_scaler_int():
     assert not np.any(np.isnan(X_scaled))
 
     with warnings.catch_warnings(record=True):
-        scaler_csr = StandardScaler(with_mean=False).fit(X_csr)
-        X_csr_scaled = scaler_csr.transform(X_csr, copy=True)
-    assert not np.any(np.isnan(X_csr_scaled.data))
-
-    with warnings.catch_warnings(record=True):
-        scaler_csc = StandardScaler(with_mean=False).fit(X_csc)
-        X_csc_scaled = scaler_csc.transform(X_csc, copy=True)
-    assert not np.any(np.isnan(X_csc_scaled.data))
+        scaler_sparse = StandardScaler(with_mean=False).fit(X_sparse)
+        X_sparse_scaled = scaler_sparse.transform(X_sparse, copy=True)
+    assert not np.any(np.isnan(X_sparse_scaled.data))
 
-    assert_array_almost_equal(scaler.mean_, scaler_csr.mean_)
-    assert_array_almost_equal(scaler.var_, scaler_csr.var_)
-    assert_array_almost_equal(scaler.scale_, scaler_csr.scale_)
-
-    assert_array_almost_equal(scaler.mean_, scaler_csc.mean_)
-    assert_array_almost_equal(scaler.var_, scaler_csc.var_)
-    assert_array_almost_equal(scaler.scale_, scaler_csc.scale_)
+    assert_array_almost_equal(scaler.mean_, scaler_sparse.mean_)
+    assert_array_almost_equal(scaler.var_, scaler_sparse.var_)
+    assert_array_almost_equal(scaler.scale_, scaler_sparse.scale_)
 
     assert_array_almost_equal(
-        X_scaled.mean(axis=0),
-        [0., 1.109, 1.856, 21., 1.559], 2)
-    assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.])
+        X_scaled.mean(axis=0), [0.0, 1.109, 1.856, 21.0, 1.559], 2
+    )
+    assert_array_almost_equal(X_scaled.std(axis=0), [0.0, 1.0, 1.0, 1.0, 1.0])
 
-    X_csr_scaled_mean, X_csr_scaled_std = mean_variance_axis(
-        X_csr_scaled.astype(float), 0)
-    assert_array_almost_equal(X_csr_scaled_mean, X_scaled.mean(axis=0))
-    assert_array_almost_equal(X_csr_scaled_std, X_scaled.std(axis=0))
+    X_sparse_scaled_mean, X_sparse_scaled_std = mean_variance_axis(
+        X_sparse_scaled.astype(float), 0
+    )
+    assert_array_almost_equal(X_sparse_scaled_mean, X_scaled.mean(axis=0))
+    assert_array_almost_equal(X_sparse_scaled_std, X_scaled.std(axis=0))
 
     # Check that X has not been modified (copy)
     assert X_scaled is not X
-    assert X_csr_scaled is not X_csr
+    assert X_sparse_scaled is not X_sparse
 
     X_scaled_back = scaler.inverse_transform(X_scaled)
     assert X_scaled_back is not X
     assert X_scaled_back is not X_scaled
     assert_array_almost_equal(X_scaled_back, X)
 
-    X_csr_scaled_back = scaler_csr.inverse_transform(X_csr_scaled)
-    assert X_csr_scaled_back is not X_csr
-    assert X_csr_scaled_back is not X_csr_scaled
-    assert_array_almost_equal(X_csr_scaled_back.toarray(), X)
+    X_sparse_scaled_back = scaler_sparse.inverse_transform(X_sparse_scaled)
+    assert X_sparse_scaled_back is not X_sparse
+    assert X_sparse_scaled_back is not X_sparse_scaled
+    assert_array_almost_equal(X_sparse_scaled_back.toarray(), X)
 
-    X_csc_scaled_back = scaler_csr.inverse_transform(X_csc_scaled.tocsc())
-    assert X_csc_scaled_back is not X_csc
-    assert X_csc_scaled_back is not X_csc_scaled
-    assert_array_almost_equal(X_csc_scaled_back.toarray(), X)
+    if sparse_container in CSR_CONTAINERS:
+        null_transform = StandardScaler(with_mean=False, with_std=False, copy=True)
+        with warnings.catch_warnings(record=True):
+            X_null = null_transform.fit_transform(X_sparse)
+        assert_array_equal(X_null.data, X_sparse.data)
+        X_orig = null_transform.inverse_transform(X_null)
+        assert_array_equal(X_orig.data, X_sparse.data)
 
 
-def test_scaler_without_copy():
+@pytest.mark.parametrize("sparse_container", CSR_CONTAINERS + CSC_CONTAINERS)
+def test_scaler_without_copy(sparse_container):
     # Check that StandardScaler.fit does not change input
     rng = np.random.RandomState(42)
     X = rng.randn(4, 5)
     X[:, 0] = 0.0  # first feature is always of zero
-    X_csr = sparse.csr_matrix(X)
-    X_csc = sparse.csc_matrix(X)
+    X_sparse = sparse_container(X)
 
     X_copy = X.copy()
     StandardScaler(copy=False).fit(X)
     assert_array_equal(X, X_copy)
 
-    X_csr_copy = X_csr.copy()
-    StandardScaler(with_mean=False, copy=False).fit(X_csr)
-    assert_array_equal(X_csr.toarray(), X_csr_copy.toarray())
-
-    X_csc_copy = X_csc.copy()
-    StandardScaler(with_mean=False, copy=False).fit(X_csc)
-    assert_array_equal(X_csc.toarray(), X_csc_copy.toarray())
+    X_sparse_copy = X_sparse.copy()
+    StandardScaler(with_mean=False, copy=False).fit(X_sparse)
+    assert_array_equal(X_sparse.toarray(), X_sparse_copy.toarray())
 
 
-def test_scale_sparse_with_mean_raise_exception():
+@pytest.mark.parametrize("sparse_container", CSR_CONTAINERS + CSC_CONTAINERS)
+def test_scale_sparse_with_mean_raise_exception(sparse_container):
     rng = np.random.RandomState(42)
     X = rng.randn(4, 5)
-    X_csr = sparse.csr_matrix(X)
-    X_csc = sparse.csc_matrix(X)
+    X_sparse = sparse_container(X)
 
     # check scaling and fit with direct calls on sparse data
     with pytest.raises(ValueError):
-        scale(X_csr, with_mean=True)
+        scale(X_sparse, with_mean=True)
     with pytest.raises(ValueError):
-        StandardScaler(with_mean=True).fit(X_csr)
-
-    with pytest.raises(ValueError):
-        scale(X_csc, with_mean=True)
-    with pytest.raises(ValueError):
-        StandardScaler(with_mean=True).fit(X_csc)
+        StandardScaler(with_mean=True).fit(X_sparse)
 
     # check transform and inverse_transform after a fit on a dense array
     scaler = StandardScaler(with_mean=True).fit(X)
     with pytest.raises(ValueError):
-        scaler.transform(X_csr)
-    with pytest.raises(ValueError):
-        scaler.transform(X_csc)
-
-    X_transformed_csr = sparse.csr_matrix(scaler.transform(X))
-    with pytest.raises(ValueError):
-        scaler.inverse_transform(X_transformed_csr)
+        scaler.transform(X_sparse)
 
-    X_transformed_csc = sparse.csc_matrix(scaler.transform(X))
+    X_transformed_sparse = sparse_container(scaler.transform(X))
     with pytest.raises(ValueError):
-        scaler.inverse_transform(X_transformed_csc)
+        scaler.inverse_transform(X_transformed_sparse)
 
 
 def test_scale_input_finiteness_validation():
     # Check if non finite inputs raise ValueError
     X = [[np.inf, 5, 6, 7, 8]]
-    with pytest.raises(ValueError, match="Input contains infinity "
-                       "or a value too large"):
+    with pytest.raises(
+        ValueError, match="Input contains infinity or a value too large"
+    ):
         scale(X)
 
 
@@ -1224,15 +1060,13 @@ def test_robust_scaler_error_sparse():
 
 @pytest.mark.parametrize("with_centering", [True, False])
 @pytest.mark.parametrize("with_scaling", [True, False])
-@pytest.mark.parametrize("X", [np.random.randn(10, 3),
-                               sparse.rand(10, 3, density=0.5)])
+@pytest.mark.parametrize("X", [np.random.randn(10, 3), sparse.rand(10, 3, density=0.5)])
 def test_robust_scaler_attributes(X, with_centering, with_scaling):
     # check consistent type of attributes
     if with_centering and sparse.issparse(X):
         pytest.skip("RobustScaler cannot center sparse matrix")
 
-    scaler = RobustScaler(with_centering=with_centering,
-                          with_scaling=with_scaling)
+    scaler = RobustScaler(with_centering=with_centering, with_scaling=with_scaling)
     scaler.fit(X)
 
     if with_centering:
@@ -1245,19 +1079,20 @@ def test_robust_scaler_attributes(X, with_centering, with_scaling):
         assert scaler.scale_ is None
 
 
-def test_robust_scaler_col_zero_sparse():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_robust_scaler_col_zero_sparse(csr_container):
     # check that the scaler is working when there is not data materialized in a
     # column of a sparse matrix
     X = np.random.randn(10, 5)
     X[:, 0] = 0
-    X = sparse.csr_matrix(X)
+    X = csr_container(X)
 
     scaler = RobustScaler(with_centering=False)
     scaler.fit(X)
     assert scaler.scale_[0] == pytest.approx(1)
 
     X_trans = scaler.transform(X)
-    assert_allclose(X[:, 0].toarray(), X_trans[:, 0].toarray())
+    assert_allclose(X[:, [0]].toarray(), X_trans[:, [0]].toarray())
 
 
 def test_robust_scaler_2d_arrays():
@@ -1274,16 +1109,15 @@ def test_robust_scaler_2d_arrays():
 
 
 @pytest.mark.parametrize("density", [0, 0.05, 0.1, 0.5, 1])
-@pytest.mark.parametrize("strictly_signed",
-                         ['positive', 'negative', 'zeros', None])
+@pytest.mark.parametrize("strictly_signed", ["positive", "negative", "zeros", None])
 def test_robust_scaler_equivalence_dense_sparse(density, strictly_signed):
     # Check the equivalence of the fitting with dense and sparse matrices
     X_sparse = sparse.rand(1000, 5, density=density).tocsc()
-    if strictly_signed == 'positive':
+    if strictly_signed == "positive":
         X_sparse.data = np.abs(X_sparse.data)
-    elif strictly_signed == 'negative':
-        X_sparse.data = - np.abs(X_sparse.data)
-    elif strictly_signed == 'zeros':
+    elif strictly_signed == "negative":
+        X_sparse.data = -np.abs(X_sparse.data)
+    elif strictly_signed == "zeros":
         X_sparse.data = np.zeros(X_sparse.data.shape, dtype=np.float64)
     X_dense = X_sparse.toarray()
 
@@ -1296,14 +1130,15 @@ def test_robust_scaler_equivalence_dense_sparse(density, strictly_signed):
     assert_allclose(scaler_sparse.scale_, scaler_dense.scale_)
 
 
-def test_robust_scaler_transform_one_row_csr():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_robust_scaler_transform_one_row_csr(csr_container):
     # Check RobustScaler on transforming csr matrix with one row
     rng = np.random.RandomState(0)
     X = rng.randn(4, 5)
-    single_row = np.array([[0.1, 1., 2., 0., -1.]])
+    single_row = np.array([[0.1, 1.0, 2.0, 0.0, -1.0]])
     scaler = RobustScaler(with_centering=False)
     scaler = scaler.fit(X)
-    row_trans = scaler.transform(sparse.csr_matrix(single_row))
+    row_trans = scaler.transform(csr_container(single_row))
     row_expected = single_row / scaler.scale_
     assert_array_almost_equal(row_trans.toarray(), row_expected)
     row_scaled_back = scaler.inverse_transform(row_trans)
@@ -1334,7 +1169,8 @@ def test_robust_scaler_iris_quantiles():
     assert_array_almost_equal(q_range, 1)
 
 
-def test_quantile_transform_iris():
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+def test_quantile_transform_iris(csc_container):
     X = iris.data
     # uniform output distribution
     transformer = QuantileTransformer(n_quantiles=30)
@@ -1342,38 +1178,42 @@ def test_quantile_transform_iris():
     X_trans_inv = transformer.inverse_transform(X_trans)
     assert_array_almost_equal(X, X_trans_inv)
     # normal output distribution
-    transformer = QuantileTransformer(n_quantiles=30,
-                                      output_distribution='normal')
+    transformer = QuantileTransformer(n_quantiles=30, output_distribution="normal")
     X_trans = transformer.fit_transform(X)
     X_trans_inv = transformer.inverse_transform(X_trans)
     assert_array_almost_equal(X, X_trans_inv)
     # make sure it is possible to take the inverse of a sparse matrix
     # which contain negative value; this is the case in the iris dataset
-    X_sparse = sparse.csc_matrix(X)
+    X_sparse = csc_container(X)
     X_sparse_tran = transformer.fit_transform(X_sparse)
     X_sparse_tran_inv = transformer.inverse_transform(X_sparse_tran)
-    assert_array_almost_equal(X_sparse.A, X_sparse_tran_inv.A)
+    assert_array_almost_equal(X_sparse.toarray(), X_sparse_tran_inv.toarray())
 
 
-def test_quantile_transform_check_error():
-    X = np.transpose([[0, 25, 50, 0, 0, 0, 75, 0, 0, 100],
-                      [2, 4, 0, 0, 6, 8, 0, 10, 0, 0],
-                      [0, 0, 2.6, 4.1, 0, 0, 2.3, 0, 9.5, 0.1]])
-    X = sparse.csc_matrix(X)
-    X_neg = np.transpose([[0, 25, 50, 0, 0, 0, 75, 0, 0, 100],
-                          [-2, 4, 0, 0, 6, 8, 0, 10, 0, 0],
-                          [0, 0, 2.6, 4.1, 0, 0, 2.3, 0, 9.5, 0.1]])
-    X_neg = sparse.csc_matrix(X_neg)
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+def test_quantile_transform_check_error(csc_container):
+    X = np.transpose(
+        [
+            [0, 25, 50, 0, 0, 0, 75, 0, 0, 100],
+            [2, 4, 0, 0, 6, 8, 0, 10, 0, 0],
+            [0, 0, 2.6, 4.1, 0, 0, 2.3, 0, 9.5, 0.1],
+        ]
+    )
+    X = csc_container(X)
+    X_neg = np.transpose(
+        [
+            [0, 25, 50, 0, 0, 0, 75, 0, 0, 100],
+            [-2, 4, 0, 0, 6, 8, 0, 10, 0, 0],
+            [0, 0, 2.6, 4.1, 0, 0, 2.3, 0, 9.5, 0.1],
+        ]
+    )
+    X_neg = csc_container(X_neg)
 
-    err_msg = "Invalid value for 'n_quantiles': 0."
-    with pytest.raises(ValueError, match=err_msg):
-        QuantileTransformer(n_quantiles=0).fit(X)
-    err_msg = "Invalid value for 'subsample': 0."
-    with pytest.raises(ValueError, match=err_msg):
-        QuantileTransformer(subsample=0).fit(X)
-    err_msg = ("The number of quantiles cannot be greater than "
-               "the number of samples used. Got 1000 quantiles "
-               "and 10 samples.")
+    err_msg = (
+        "The number of quantiles cannot be greater than "
+        "the number of samples used. Got 1000 quantiles "
+        "and 10 samples."
+    )
     with pytest.raises(ValueError, match=err_msg):
         QuantileTransformer(subsample=10).fit(X)
 
@@ -1386,37 +1226,18 @@ def test_quantile_transform_check_error():
     with pytest.raises(ValueError, match=err_msg):
         transformer.transform(X_neg)
 
-    X_bad_feat = np.transpose([[0, 25, 50, 0, 0, 0, 75, 0, 0, 100],
-                               [0, 0, 2.6, 4.1, 0, 0, 2.3, 0, 9.5, 0.1]])
-    err_msg = ("X has 2 features, but QuantileTransformer is expecting "
-               "3 features as input.")
+    X_bad_feat = np.transpose(
+        [[0, 25, 50, 0, 0, 0, 75, 0, 0, 100], [0, 0, 2.6, 4.1, 0, 0, 2.3, 0, 9.5, 0.1]]
+    )
+    err_msg = (
+        "X has 2 features, but QuantileTransformer is expecting 3 features as input."
+    )
     with pytest.raises(ValueError, match=err_msg):
         transformer.inverse_transform(X_bad_feat)
 
-    transformer = QuantileTransformer(n_quantiles=10,
-                                      output_distribution='rnd')
-    # check that an error is raised at fit time
-    err_msg = ("'output_distribution' has to be either 'normal' or "
-               "'uniform'. Got 'rnd' instead.")
-    with pytest.raises(ValueError, match=err_msg):
-        transformer.fit(X)
-    # check that an error is raised at transform time
-    transformer.output_distribution = 'uniform'
-    transformer.fit(X)
-    X_tran = transformer.transform(X)
-    transformer.output_distribution = 'rnd'
-    err_msg = ("'output_distribution' has to be either 'normal' or 'uniform'."
-               " Got 'rnd' instead.")
-    with pytest.raises(ValueError, match=err_msg):
-        transformer.transform(X)
-    # check that an error is raised at inverse_transform time
-    err_msg = ("'output_distribution' has to be either 'normal' or 'uniform'."
-               " Got 'rnd' instead.")
-    with pytest.raises(ValueError, match=err_msg):
-        transformer.inverse_transform(X_tran)
+    transformer = QuantileTransformer(n_quantiles=10).fit(X)
     # check that an error is raised if input is scalar
-    with pytest.raises(ValueError,
-                       match='Expected 2D array, got scalar array instead'):
+    with pytest.raises(ValueError, match="Expected 2D array, got scalar array instead"):
         transformer.transform(10)
     # check that a warning is raised is n_quantiles > n_samples
     transformer = QuantileTransformer(n_quantiles=100)
@@ -1427,98 +1248,98 @@ def test_quantile_transform_check_error():
     assert transformer.n_quantiles_ == X.shape[0]
 
 
-def test_quantile_transform_sparse_ignore_zeros():
-    X = np.array([[0, 1],
-                  [0, 0],
-                  [0, 2],
-                  [0, 2],
-                  [0, 1]])
-    X_sparse = sparse.csc_matrix(X)
-    transformer = QuantileTransformer(ignore_implicit_zeros=True,
-                                      n_quantiles=5)
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+def test_quantile_transform_sparse_ignore_zeros(csc_container):
+    X = np.array([[0, 1], [0, 0], [0, 2], [0, 2], [0, 1]])
+    X_sparse = csc_container(X)
+    transformer = QuantileTransformer(ignore_implicit_zeros=True, n_quantiles=5)
 
     # dense case -> warning raise
-    assert_warns_message(UserWarning, "'ignore_implicit_zeros' takes effect"
-                         " only with sparse matrix. This parameter has no"
-                         " effect.", transformer.fit, X)
-
-    X_expected = np.array([[0, 0],
-                           [0, 0],
-                           [0, 1],
-                           [0, 1],
-                           [0, 0]])
+    warning_message = (
+        "'ignore_implicit_zeros' takes effect"
+        " only with sparse matrix. This parameter has no"
+        " effect."
+    )
+    with pytest.warns(UserWarning, match=warning_message):
+        transformer.fit(X)
+
+    X_expected = np.array([[0, 0], [0, 0], [0, 1], [0, 1], [0, 0]])
     X_trans = transformer.fit_transform(X_sparse)
-    assert_almost_equal(X_expected, X_trans.A)
+    assert_almost_equal(X_expected, X_trans.toarray())
 
     # consider the case where sparse entries are missing values and user-given
     # zeros are to be considered
     X_data = np.array([0, 0, 1, 0, 2, 2, 1, 0, 1, 2, 0])
     X_col = np.array([0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1])
     X_row = np.array([0, 4, 0, 1, 2, 3, 4, 5, 6, 7, 8])
-    X_sparse = sparse.csc_matrix((X_data, (X_row, X_col)))
+    X_sparse = csc_container((X_data, (X_row, X_col)))
     X_trans = transformer.fit_transform(X_sparse)
-    X_expected = np.array([[0., 0.5],
-                           [0., 0.],
-                           [0., 1.],
-                           [0., 1.],
-                           [0., 0.5],
-                           [0., 0.],
-                           [0., 0.5],
-                           [0., 1.],
-                           [0., 0.]])
-    assert_almost_equal(X_expected, X_trans.A)
-
-    transformer = QuantileTransformer(ignore_implicit_zeros=True,
-                                      n_quantiles=5)
+    X_expected = np.array(
+        [
+            [0.0, 0.5],
+            [0.0, 0.0],
+            [0.0, 1.0],
+            [0.0, 1.0],
+            [0.0, 0.5],
+            [0.0, 0.0],
+            [0.0, 0.5],
+            [0.0, 1.0],
+            [0.0, 0.0],
+        ]
+    )
+    assert_almost_equal(X_expected, X_trans.toarray())
+
+    transformer = QuantileTransformer(ignore_implicit_zeros=True, n_quantiles=5)
     X_data = np.array([-1, -1, 1, 0, 0, 0, 1, -1, 1])
     X_col = np.array([0, 0, 1, 1, 1, 1, 1, 1, 1])
     X_row = np.array([0, 4, 0, 1, 2, 3, 4, 5, 6])
-    X_sparse = sparse.csc_matrix((X_data, (X_row, X_col)))
+    X_sparse = csc_container((X_data, (X_row, X_col)))
     X_trans = transformer.fit_transform(X_sparse)
-    X_expected = np.array([[0, 1],
-                           [0, 0.375],
-                           [0, 0.375],
-                           [0, 0.375],
-                           [0, 1],
-                           [0, 0],
-                           [0, 1]])
-    assert_almost_equal(X_expected, X_trans.A)
-    assert_almost_equal(X_sparse.A, transformer.inverse_transform(X_trans).A)
+    X_expected = np.array(
+        [[0, 1], [0, 0.375], [0, 0.375], [0, 0.375], [0, 1], [0, 0], [0, 1]]
+    )
+    assert_almost_equal(X_expected, X_trans.toarray())
+    assert_almost_equal(
+        X_sparse.toarray(), transformer.inverse_transform(X_trans).toarray()
+    )
 
     # check in conjunction with subsampling
-    transformer = QuantileTransformer(ignore_implicit_zeros=True,
-                                      n_quantiles=5,
-                                      subsample=8,
-                                      random_state=0)
+    transformer = QuantileTransformer(
+        ignore_implicit_zeros=True, n_quantiles=5, subsample=8, random_state=0
+    )
     X_trans = transformer.fit_transform(X_sparse)
-    assert_almost_equal(X_expected, X_trans.A)
-    assert_almost_equal(X_sparse.A, transformer.inverse_transform(X_trans).A)
+    assert_almost_equal(X_expected, X_trans.toarray())
+    assert_almost_equal(
+        X_sparse.toarray(), transformer.inverse_transform(X_trans).toarray()
+    )
 
 
 def test_quantile_transform_dense_toy():
-    X = np.array([[0, 2, 2.6],
-                  [25, 4, 4.1],
-                  [50, 6, 2.3],
-                  [75, 8, 9.5],
-                  [100, 10, 0.1]])
+    X = np.array(
+        [[0, 2, 2.6], [25, 4, 4.1], [50, 6, 2.3], [75, 8, 9.5], [100, 10, 0.1]]
+    )
 
     transformer = QuantileTransformer(n_quantiles=5)
     transformer.fit(X)
 
-    # using the a uniform output, each entry of X should be map between 0 and 1
+    # using a uniform output, each entry of X should be map between 0 and 1
     # and equally spaced
     X_trans = transformer.fit_transform(X)
     X_expected = np.tile(np.linspace(0, 1, num=5), (3, 1)).T
     assert_almost_equal(np.sort(X_trans, axis=0), X_expected)
 
-    X_test = np.array([
-        [-1, 1, 0],
-        [101, 11, 10],
-    ])
-    X_expected = np.array([
-        [0, 0, 0],
-        [1, 1, 1],
-    ])
+    X_test = np.array(
+        [
+            [-1, 1, 0],
+            [101, 11, 10],
+        ]
+    )
+    X_expected = np.array(
+        [
+            [0, 0, 0],
+            [1, 1, 1],
+        ]
+    )
     assert_array_almost_equal(transformer.transform(X_test), X_expected)
 
     X_trans_inv = transformer.inverse_transform(X_trans)
@@ -1538,12 +1359,13 @@ def test_quantile_transform_subsampling():
     ROUND = 5
     inf_norm_arr = []
     for random_state in range(ROUND):
-        transformer = QuantileTransformer(random_state=random_state,
-                                          n_quantiles=n_quantiles,
-                                          subsample=n_samples // 10)
+        transformer = QuantileTransformer(
+            random_state=random_state,
+            n_quantiles=n_quantiles,
+            subsample=n_samples // 10,
+        )
         transformer.fit(X)
-        diff = (np.linspace(0, 1, n_quantiles) -
-                np.ravel(transformer.quantiles_))
+        diff = np.linspace(0, 1, n_quantiles) - np.ravel(transformer.quantiles_)
         inf_norm = np.max(np.abs(diff))
         assert inf_norm < 1e-2
         inf_norm_arr.append(inf_norm)
@@ -1553,15 +1375,16 @@ def test_quantile_transform_subsampling():
 
     # sparse support
 
-    X = sparse.rand(n_samples, 1, density=.99, format='csc', random_state=0)
+    X = sparse.rand(n_samples, 1, density=0.99, format="csc", random_state=0)
     inf_norm_arr = []
     for random_state in range(ROUND):
-        transformer = QuantileTransformer(random_state=random_state,
-                                          n_quantiles=n_quantiles,
-                                          subsample=n_samples // 10)
+        transformer = QuantileTransformer(
+            random_state=random_state,
+            n_quantiles=n_quantiles,
+            subsample=n_samples // 10,
+        )
         transformer.fit(X)
-        diff = (np.linspace(0, 1, n_quantiles) -
-                np.ravel(transformer.quantiles_))
+        diff = np.linspace(0, 1, n_quantiles) - np.ravel(transformer.quantiles_)
         inf_norm = np.max(np.abs(diff))
         assert inf_norm < 1e-1
         inf_norm_arr.append(inf_norm)
@@ -1570,76 +1393,86 @@ def test_quantile_transform_subsampling():
     assert len(np.unique(inf_norm_arr)) == len(inf_norm_arr)
 
 
-def test_quantile_transform_sparse_toy():
-    X = np.array([[0., 2., 0.],
-                  [25., 4., 0.],
-                  [50., 0., 2.6],
-                  [0., 0., 4.1],
-                  [0., 6., 0.],
-                  [0., 8., 0.],
-                  [75., 0., 2.3],
-                  [0., 10., 0.],
-                  [0., 0., 9.5],
-                  [100., 0., 0.1]])
+def test_quantile_transform_subsampling_disabled():
+    """Check the behaviour of `QuantileTransformer` when `subsample=None`."""
+    X = np.random.RandomState(0).normal(size=(200, 1))
+
+    n_quantiles = 5
+    transformer = QuantileTransformer(n_quantiles=n_quantiles, subsample=None).fit(X)
+
+    expected_references = np.linspace(0, 1, n_quantiles)
+    assert_allclose(transformer.references_, expected_references)
+    expected_quantiles = np.quantile(X.ravel(), expected_references)
+    assert_allclose(transformer.quantiles_.ravel(), expected_quantiles)
+
+
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+def test_quantile_transform_sparse_toy(csc_container):
+    X = np.array(
+        [
+            [0.0, 2.0, 0.0],
+            [25.0, 4.0, 0.0],
+            [50.0, 0.0, 2.6],
+            [0.0, 0.0, 4.1],
+            [0.0, 6.0, 0.0],
+            [0.0, 8.0, 0.0],
+            [75.0, 0.0, 2.3],
+            [0.0, 10.0, 0.0],
+            [0.0, 0.0, 9.5],
+            [100.0, 0.0, 0.1],
+        ]
+    )
 
-    X = sparse.csc_matrix(X)
+    X = csc_container(X)
 
     transformer = QuantileTransformer(n_quantiles=10)
     transformer.fit(X)
 
     X_trans = transformer.fit_transform(X)
-    assert_array_almost_equal(np.min(X_trans.toarray(), axis=0), 0.)
-    assert_array_almost_equal(np.max(X_trans.toarray(), axis=0), 1.)
+    assert_array_almost_equal(np.min(X_trans.toarray(), axis=0), 0.0)
+    assert_array_almost_equal(np.max(X_trans.toarray(), axis=0), 1.0)
 
     X_trans_inv = transformer.inverse_transform(X_trans)
     assert_array_almost_equal(X.toarray(), X_trans_inv.toarray())
 
-    transformer_dense = QuantileTransformer(n_quantiles=10).fit(
-        X.toarray())
+    transformer_dense = QuantileTransformer(n_quantiles=10).fit(X.toarray())
 
     X_trans = transformer_dense.transform(X)
-    assert_array_almost_equal(np.min(X_trans.toarray(), axis=0), 0.)
-    assert_array_almost_equal(np.max(X_trans.toarray(), axis=0), 1.)
+    assert_array_almost_equal(np.min(X_trans.toarray(), axis=0), 0.0)
+    assert_array_almost_equal(np.max(X_trans.toarray(), axis=0), 1.0)
 
     X_trans_inv = transformer_dense.inverse_transform(X_trans)
     assert_array_almost_equal(X.toarray(), X_trans_inv.toarray())
 
 
 def test_quantile_transform_axis1():
-    X = np.array([[0, 25, 50, 75, 100],
-                  [2, 4, 6, 8, 10],
-                  [2.6, 4.1, 2.3, 9.5, 0.1]])
+    X = np.array([[0, 25, 50, 75, 100], [2, 4, 6, 8, 10], [2.6, 4.1, 2.3, 9.5, 0.1]])
 
     X_trans_a0 = quantile_transform(X.T, axis=0, n_quantiles=5)
     X_trans_a1 = quantile_transform(X, axis=1, n_quantiles=5)
     assert_array_almost_equal(X_trans_a0, X_trans_a1.T)
 
 
-def test_quantile_transform_bounds():
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+def test_quantile_transform_bounds(csc_container):
     # Lower and upper bounds are manually mapped. We checked that in the case
     # of a constant feature and binary feature, the bounds are properly mapped.
-    X_dense = np.array([[0, 0],
-                        [0, 0],
-                        [1, 0]])
-    X_sparse = sparse.csc_matrix(X_dense)
+    X_dense = np.array([[0, 0], [0, 0], [1, 0]])
+    X_sparse = csc_container(X_dense)
 
     # check sparse and dense are consistent
-    X_trans = QuantileTransformer(n_quantiles=3,
-                                  random_state=0).fit_transform(X_dense)
+    X_trans = QuantileTransformer(n_quantiles=3, random_state=0).fit_transform(X_dense)
     assert_array_almost_equal(X_trans, X_dense)
-    X_trans_sp = QuantileTransformer(n_quantiles=3,
-                                     random_state=0).fit_transform(X_sparse)
-    assert_array_almost_equal(X_trans_sp.A, X_dense)
-    assert_array_almost_equal(X_trans, X_trans_sp.A)
+    X_trans_sp = QuantileTransformer(n_quantiles=3, random_state=0).fit_transform(
+        X_sparse
+    )
+    assert_array_almost_equal(X_trans_sp.toarray(), X_dense)
+    assert_array_almost_equal(X_trans, X_trans_sp.toarray())
 
     # check the consistency of the bounds by learning on 1 matrix
     # and transforming another
-    X = np.array([[0, 1],
-                  [0, 0.5],
-                  [1, 0]])
-    X1 = np.array([[0, 0.1],
-                   [0, 0.5],
-                   [1, 0.1]])
+    X = np.array([[0, 1], [0, 0.5], [1, 0]])
+    X1 = np.array([[0, 0.1], [0, 0.5], [1, 0.1]])
     transformer = QuantileTransformer(n_quantiles=3).fit(X)
     X_trans = transformer.transform(X1)
     assert_array_almost_equal(X_trans, X1)
@@ -1648,19 +1481,19 @@ def test_quantile_transform_bounds():
     X = np.random.random((1000, 1))
     transformer = QuantileTransformer()
     transformer.fit(X)
-    assert (transformer.transform([[-10]]) ==
-            transformer.transform([[np.min(X)]]))
-    assert (transformer.transform([[10]]) ==
-            transformer.transform([[np.max(X)]]))
-    assert (transformer.inverse_transform([[-10]]) ==
-            transformer.inverse_transform([[np.min(transformer.references_)]]))
-    assert (transformer.inverse_transform([[10]]) ==
-            transformer.inverse_transform([[np.max(transformer.references_)]]))
+    assert transformer.transform([[-10]]) == transformer.transform([[np.min(X)]])
+    assert transformer.transform([[10]]) == transformer.transform([[np.max(X)]])
+    assert transformer.inverse_transform([[-10]]) == transformer.inverse_transform(
+        [[np.min(transformer.references_)]]
+    )
+    assert transformer.inverse_transform([[10]]) == transformer.inverse_transform(
+        [[np.max(transformer.references_)]]
+    )
 
 
 def test_quantile_transform_and_inverse():
     X_1 = iris.data
-    X_2 = np.array([[0.], [BOUNDS_THRESHOLD / 10], [1.5], [2], [3], [3], [4]])
+    X_2 = np.array([[0.0], [BOUNDS_THRESHOLD / 10], [1.5], [2], [3], [3], [4]])
     for X in [X_1, X_2]:
         transformer = QuantileTransformer(n_quantiles=1000, random_state=0)
         X_trans = transformer.fit_transform(X)
@@ -1669,9 +1502,7 @@ def test_quantile_transform_and_inverse():
 
 
 def test_quantile_transform_nan():
-    X = np.array([[np.nan, 0,  0, 1],
-                  [np.nan, np.nan, 0, 0.5],
-                  [np.nan, 1, 1, 0]])
+    X = np.array([[np.nan, 0, 0, 1], [np.nan, np.nan, 0, 0.5], [np.nan, 1, 1, 0]])
 
     transformer = QuantileTransformer(n_quantiles=10, random_state=42)
     transformer.fit_transform(X)
@@ -1682,7 +1513,7 @@ def test_quantile_transform_nan():
     assert not np.isnan(transformer.quantiles_[:, 1:]).any()
 
 
-@pytest.mark.parametrize("array_type", ['array', 'sparse'])
+@pytest.mark.parametrize("array_type", ["array", "sparse"])
 def test_quantile_transformer_sorted_quantiles(array_type):
     # Non-regression test for:
     # https://github.com/scikit-learn/scikit-learn/issues/15733
@@ -1695,7 +1526,7 @@ def test_quantile_transformer_sorted_quantiles(array_type):
     n_quantiles = 100
     qt = QuantileTransformer(n_quantiles=n_quantiles).fit(X)
 
-    # Check that the estimated quantile threasholds are monotically
+    # Check that the estimated quantile thresholds are monotically
     # increasing:
     quantiles = qt.quantiles_[:, 0]
     assert len(quantiles) == 100
@@ -1712,15 +1543,16 @@ def test_robust_scaler_invalid_range():
     ]:
         scaler = RobustScaler(quantile_range=range_)
 
-        with pytest.raises(ValueError, match=r'Invalid quantile range: \('):
+        with pytest.raises(ValueError, match=r"Invalid quantile range: \("):
             scaler.fit(iris.data)
 
 
-def test_scale_function_without_centering():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_scale_function_without_centering(csr_container):
     rng = np.random.RandomState(42)
     X = rng.randn(4, 5)
     X[:, 0] = 0.0  # first feature is always of zero
-    X_csr = sparse.csr_matrix(X)
+    X_csr = csr_container(X)
 
     X_scaled = scale(X, with_mean=False)
     assert not np.any(np.isnan(X_scaled))
@@ -1736,9 +1568,10 @@ def test_scale_function_without_centering():
     with pytest.raises(ValueError):
         scale(X_csr, with_mean=False, axis=1)
 
-    assert_array_almost_equal(X_scaled.mean(axis=0),
-                              [0., -0.01, 2.24, -0.35, -0.78], 2)
-    assert_array_almost_equal(X_scaled.std(axis=0), [0., 1., 1., 1., 1.])
+    assert_array_almost_equal(
+        X_scaled.mean(axis=0), [0.0, -0.01, 2.24, -0.35, -0.78], 2
+    )
+    assert_array_almost_equal(X_scaled.std(axis=0), [0.0, 1.0, 1.0, 1.0, 1.0])
     # Check that X has not been copied
     assert X_scaled is not X
 
@@ -1771,9 +1604,7 @@ def test_robust_scale_1d_array():
 
 def test_robust_scaler_zero_variance_features():
     # Check RobustScaler on toy data with zero variance features
-    X = [[0., 1., +0.5],
-         [0., 1., -0.1],
-         [0., 1., +1.1]]
+    X = [[0.0, 1.0, +0.5], [0.0, 1.0, -0.1], [0.0, 1.0, +1.1]]
 
     scaler = RobustScaler()
     X_trans = scaler.fit_transform(X)
@@ -1784,21 +1615,15 @@ def test_robust_scaler_zero_variance_features():
     # using numpy 1.9 Calculating quantiles with
     # scipy.stats.mstats.scoreatquantile or scipy.stats.mstats.mquantiles
     # would yield very different results!
-    X_expected = [[0., 0., +0.0],
-                  [0., 0., -1.0],
-                  [0., 0., +1.0]]
+    X_expected = [[0.0, 0.0, +0.0], [0.0, 0.0, -1.0], [0.0, 0.0, +1.0]]
     assert_array_almost_equal(X_trans, X_expected)
     X_trans_inv = scaler.inverse_transform(X_trans)
     assert_array_almost_equal(X, X_trans_inv)
 
     # make sure new data gets transformed correctly
-    X_new = [[+0., 2., 0.5],
-             [-1., 1., 0.0],
-             [+0., 1., 1.5]]
+    X_new = [[+0.0, 2.0, 0.5], [-1.0, 1.0, 0.0], [+0.0, 1.0, 1.5]]
     X_trans_new = scaler.transform(X_new)
-    X_expected_new = [[+0., 1., +0.],
-                      [-1., 0., -0.83333],
-                      [+0., 0., +1.66667]]
+    X_expected_new = [[+0.0, 1.0, +0.0], [-1.0, 0.0, -0.83333], [+0.0, 0.0, +1.66667]]
     assert_array_almost_equal(X_trans_new, X_expected_new, decimal=3)
 
 
@@ -1807,14 +1632,12 @@ def test_robust_scaler_unit_variance():
     # outliers
     rng = np.random.RandomState(42)
     X = rng.randn(1000000, 1)
-    X_with_outliers = np.vstack(
-        [X, np.ones((100, 1)) * 100, np.ones((100, 1)) * -100]
-    )
+    X_with_outliers = np.vstack([X, np.ones((100, 1)) * 100, np.ones((100, 1)) * -100])
 
     quantile_range = (1, 99)
-    robust_scaler = RobustScaler(
-        quantile_range=quantile_range, unit_variance=True
-    ).fit(X_with_outliers)
+    robust_scaler = RobustScaler(quantile_range=quantile_range, unit_variance=True).fit(
+        X_with_outliers
+    )
     X_trans = robust_scaler.transform(X)
 
     assert robust_scaler.center_ == pytest.approx(0, abs=1e-3)
@@ -1822,31 +1645,27 @@ def test_robust_scaler_unit_variance():
     assert X_trans.std() == pytest.approx(1, abs=1e-2)
 
 
-def test_maxabs_scaler_zero_variance_features():
+@pytest.mark.parametrize("sparse_container", CSC_CONTAINERS + CSR_CONTAINERS)
+def test_maxabs_scaler_zero_variance_features(sparse_container):
     # Check MaxAbsScaler on toy data with zero variance features
-    X = [[0., 1., +0.5],
-         [0., 1., -0.3],
-         [0., 1., +1.5],
-         [0., 0., +0.0]]
+    X = [[0.0, 1.0, +0.5], [0.0, 1.0, -0.3], [0.0, 1.0, +1.5], [0.0, 0.0, +0.0]]
 
     scaler = MaxAbsScaler()
     X_trans = scaler.fit_transform(X)
-    X_expected = [[0., 1., 1.0 / 3.0],
-                  [0., 1., -0.2],
-                  [0., 1., 1.0],
-                  [0., 0., 0.0]]
+    X_expected = [
+        [0.0, 1.0, 1.0 / 3.0],
+        [0.0, 1.0, -0.2],
+        [0.0, 1.0, 1.0],
+        [0.0, 0.0, 0.0],
+    ]
     assert_array_almost_equal(X_trans, X_expected)
     X_trans_inv = scaler.inverse_transform(X_trans)
     assert_array_almost_equal(X, X_trans_inv)
 
     # make sure new data gets transformed correctly
-    X_new = [[+0., 2., 0.5],
-             [-1., 1., 0.0],
-             [+0., 1., 1.5]]
+    X_new = [[+0.0, 2.0, 0.5], [-1.0, 1.0, 0.0], [+0.0, 1.0, 1.5]]
     X_trans_new = scaler.transform(X_new)
-    X_expected_new = [[+0., 2.0, 1.0 / 3.0],
-                      [-1., 1.0, 0.0],
-                      [+0., 1.0, 1.0]]
+    X_expected_new = [[+0.0, 2.0, 1.0 / 3.0], [-1.0, 1.0, 0.0], [+0.0, 1.0, 1.0]]
 
     assert_array_almost_equal(X_trans_new, X_expected_new, decimal=2)
 
@@ -1855,45 +1674,47 @@ def test_maxabs_scaler_zero_variance_features():
     assert_array_almost_equal(X_trans, X_expected)
 
     # sparse data
-    X_csr = sparse.csr_matrix(X)
-    X_csc = sparse.csc_matrix(X)
-    X_trans_csr = scaler.fit_transform(X_csr)
-    X_trans_csc = scaler.fit_transform(X_csc)
-    X_expected = [[0., 1., 1.0 / 3.0],
-                  [0., 1., -0.2],
-                  [0., 1., 1.0],
-                  [0., 0., 0.0]]
-    assert_array_almost_equal(X_trans_csr.A, X_expected)
-    assert_array_almost_equal(X_trans_csc.A, X_expected)
-    X_trans_csr_inv = scaler.inverse_transform(X_trans_csr)
-    X_trans_csc_inv = scaler.inverse_transform(X_trans_csc)
-    assert_array_almost_equal(X, X_trans_csr_inv.A)
-    assert_array_almost_equal(X, X_trans_csc_inv.A)
+    X_sparse = sparse_container(X)
+    X_trans_sparse = scaler.fit_transform(X_sparse)
+    X_expected = [
+        [0.0, 1.0, 1.0 / 3.0],
+        [0.0, 1.0, -0.2],
+        [0.0, 1.0, 1.0],
+        [0.0, 0.0, 0.0],
+    ]
+    assert_array_almost_equal(X_trans_sparse.toarray(), X_expected)
+    X_trans_sparse_inv = scaler.inverse_transform(X_trans_sparse)
+    assert_array_almost_equal(X, X_trans_sparse_inv.toarray())
 
 
 def test_maxabs_scaler_large_negative_value():
     # Check MaxAbsScaler on toy data with a large negative value
-    X = [[0., 1.,   +0.5, -1.0],
-         [0., 1.,   -0.3, -0.5],
-         [0., 1., -100.0,  0.0],
-         [0., 0.,   +0.0, -2.0]]
+    X = [
+        [0.0, 1.0, +0.5, -1.0],
+        [0.0, 1.0, -0.3, -0.5],
+        [0.0, 1.0, -100.0, 0.0],
+        [0.0, 0.0, +0.0, -2.0],
+    ]
 
     scaler = MaxAbsScaler()
     X_trans = scaler.fit_transform(X)
-    X_expected = [[0., 1.,  0.005,    -0.5],
-                  [0., 1., -0.003,    -0.25],
-                  [0., 1., -1.0,       0.0],
-                  [0., 0.,  0.0,      -1.0]]
+    X_expected = [
+        [0.0, 1.0, 0.005, -0.5],
+        [0.0, 1.0, -0.003, -0.25],
+        [0.0, 1.0, -1.0, 0.0],
+        [0.0, 0.0, 0.0, -1.0],
+    ]
     assert_array_almost_equal(X_trans, X_expected)
 
 
-def test_maxabs_scaler_transform_one_row_csr():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_maxabs_scaler_transform_one_row_csr(csr_container):
     # Check MaxAbsScaler on transforming csr matrix with one row
-    X = sparse.csr_matrix([[0.5, 1., 1.]])
+    X = csr_container([[0.5, 1.0, 1.0]])
     scaler = MaxAbsScaler()
     scaler = scaler.fit(X)
     X_trans = scaler.transform(X)
-    X_expected = sparse.csr_matrix([[1., 1., 1.]])
+    X_expected = csr_container([[1.0, 1.0, 1.0]])
     assert_array_almost_equal(X_trans.toarray(), X_expected.toarray())
     X_scaled_back = scaler.inverse_transform(X_trans)
     assert_array_almost_equal(X.toarray(), X_scaled_back.toarray())
@@ -1902,7 +1723,6 @@ def test_maxabs_scaler_transform_one_row_csr():
 def test_maxabs_scaler_1d():
     # Test scaling of dataset along single axis
     for X in [X_1row, X_1col, X_list_1row, X_list_1row]:
-
         scaler = MaxAbsScaler(copy=True)
         X_scaled = scaler.fit(X).transform(X)
 
@@ -1910,10 +1730,9 @@ def test_maxabs_scaler_1d():
             X = np.array(X)  # cast only after scaling done
 
         if _check_dim_1axis(X) == 1:
-            assert_array_almost_equal(np.abs(X_scaled.max(axis=0)),
-                                      np.ones(n_features))
+            assert_array_almost_equal(np.abs(X_scaled.max(axis=0)), np.ones(n_features))
         else:
-            assert_array_almost_equal(np.abs(X_scaled.max(axis=0)), 1.)
+            assert_array_almost_equal(np.abs(X_scaled.max(axis=0)), 1.0)
         assert scaler.n_samples_seen_ == X.shape[0]
 
         # check inverse transform
@@ -1924,7 +1743,7 @@ def test_maxabs_scaler_1d():
     X = np.ones((5, 1))
     scaler = MaxAbsScaler()
     X_scaled = scaler.fit(X).transform(X)
-    assert_array_almost_equal(np.abs(X_scaled.max(axis=0)), 1.)
+    assert_array_almost_equal(np.abs(X_scaled.max(axis=0)), 1.0)
     assert scaler.n_samples_seen_ == X.shape[0]
 
     # function interface
@@ -1933,7 +1752,8 @@ def test_maxabs_scaler_1d():
     assert_array_almost_equal(X_1d / max_abs, maxabs_scale(X_1d, copy=True))
 
 
-def test_maxabs_scaler_partial_fit():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_maxabs_scaler_partial_fit(csr_container):
     # Test if partial_fit run over many batches of size 1 and 50
     # gives the same results as fit
     X = X_2d[:100, :]
@@ -1948,26 +1768,21 @@ def test_maxabs_scaler_partial_fit():
         scaler_incr_csc = MaxAbsScaler()
         for batch in gen_batches(n, chunk_size):
             scaler_incr = scaler_incr.partial_fit(X[batch])
-            X_csr = sparse.csr_matrix(X[batch])
+            X_csr = csr_container(X[batch])
             scaler_incr_csr = scaler_incr_csr.partial_fit(X_csr)
-            X_csc = sparse.csc_matrix(X[batch])
+            X_csc = csr_container(X[batch])
             scaler_incr_csc = scaler_incr_csc.partial_fit(X_csc)
 
         assert_array_almost_equal(scaler_batch.max_abs_, scaler_incr.max_abs_)
-        assert_array_almost_equal(scaler_batch.max_abs_,
-                                  scaler_incr_csr.max_abs_)
-        assert_array_almost_equal(scaler_batch.max_abs_,
-                                  scaler_incr_csc.max_abs_)
+        assert_array_almost_equal(scaler_batch.max_abs_, scaler_incr_csr.max_abs_)
+        assert_array_almost_equal(scaler_batch.max_abs_, scaler_incr_csc.max_abs_)
         assert scaler_batch.n_samples_seen_ == scaler_incr.n_samples_seen_
-        assert (scaler_batch.n_samples_seen_ ==
-                scaler_incr_csr.n_samples_seen_)
-        assert (scaler_batch.n_samples_seen_ ==
-                scaler_incr_csc.n_samples_seen_)
+        assert scaler_batch.n_samples_seen_ == scaler_incr_csr.n_samples_seen_
+        assert scaler_batch.n_samples_seen_ == scaler_incr_csc.n_samples_seen_
         assert_array_almost_equal(scaler_batch.scale_, scaler_incr.scale_)
         assert_array_almost_equal(scaler_batch.scale_, scaler_incr_csr.scale_)
         assert_array_almost_equal(scaler_batch.scale_, scaler_incr_csc.scale_)
-        assert_array_almost_equal(scaler_batch.transform(X),
-                                  scaler_incr.transform(X))
+        assert_array_almost_equal(scaler_batch.transform(X), scaler_incr.transform(X))
 
         # Test std after 1 step
         batch0 = slice(0, chunk_size)
@@ -1977,73 +1792,50 @@ def test_maxabs_scaler_partial_fit():
         assert_array_almost_equal(scaler_batch.max_abs_, scaler_incr.max_abs_)
         assert scaler_batch.n_samples_seen_ == scaler_incr.n_samples_seen_
         assert_array_almost_equal(scaler_batch.scale_, scaler_incr.scale_)
-        assert_array_almost_equal(scaler_batch.transform(X),
-                                  scaler_incr.transform(X))
+        assert_array_almost_equal(scaler_batch.transform(X), scaler_incr.transform(X))
 
         # Test std until the end of partial fits, and
         scaler_batch = MaxAbsScaler().fit(X)
         scaler_incr = MaxAbsScaler()  # Clean estimator
         for i, batch in enumerate(gen_batches(n, chunk_size)):
             scaler_incr = scaler_incr.partial_fit(X[batch])
-            assert_correct_incr(i, batch_start=batch.start,
-                                batch_stop=batch.stop, n=n,
-                                chunk_size=chunk_size,
-                                n_samples_seen=scaler_incr.n_samples_seen_)
-
-
-def test_normalizer_l1():
-    rng = np.random.RandomState(0)
-    X_dense = rng.randn(4, 5)
-    X_sparse_unpruned = sparse.csr_matrix(X_dense)
-
-    # set the row number 3 to zero
-    X_dense[3, :] = 0.0
-
-    # set the row number 3 to zero without pruning (can happen in real life)
-    indptr_3 = X_sparse_unpruned.indptr[3]
-    indptr_4 = X_sparse_unpruned.indptr[4]
-    X_sparse_unpruned.data[indptr_3:indptr_4] = 0.0
-
-    # build the pruned variant using the regular constructor
-    X_sparse_pruned = sparse.csr_matrix(X_dense)
-
-    # check inputs that support the no-copy optim
-    for X in (X_dense, X_sparse_pruned, X_sparse_unpruned):
-
-        normalizer = Normalizer(norm='l1', copy=True)
-        X_norm = normalizer.transform(X)
-        assert X_norm is not X
-        X_norm1 = toarray(X_norm)
-
-        normalizer = Normalizer(norm='l1', copy=False)
-        X_norm = normalizer.transform(X)
-        assert X_norm is X
-        X_norm2 = toarray(X_norm)
-
-        for X_norm in (X_norm1, X_norm2):
-            row_sums = np.abs(X_norm).sum(axis=1)
-            for i in range(3):
-                assert_almost_equal(row_sums[i], 1.0)
-            assert_almost_equal(row_sums[3], 0.0)
-
-    # check input for which copy=False won't prevent a copy
-    for init in (sparse.coo_matrix, sparse.csc_matrix, sparse.lil_matrix):
-        X = init(X_dense)
-        X_norm = normalizer = Normalizer(norm='l2', copy=False).transform(X)
+            assert_correct_incr(
+                i,
+                batch_start=batch.start,
+                batch_stop=batch.stop,
+                n=n,
+                chunk_size=chunk_size,
+                n_samples_seen=scaler_incr.n_samples_seen_,
+            )
 
-        assert X_norm is not X
-        assert isinstance(X_norm, sparse.csr_matrix)
 
-        X_norm = toarray(X_norm)
+def check_normalizer(norm, X_norm):
+    """
+    Convenient checking function for `test_normalizer_l1_l2_max` and
+    `test_normalizer_l1_l2_max_non_csr`
+    """
+    if norm == "l1":
+        row_sums = np.abs(X_norm).sum(axis=1)
         for i in range(3):
             assert_almost_equal(row_sums[i], 1.0)
+        assert_almost_equal(row_sums[3], 0.0)
+    elif norm == "l2":
+        for i in range(3):
+            assert_almost_equal(la.norm(X_norm[i]), 1.0)
         assert_almost_equal(la.norm(X_norm[3]), 0.0)
+    elif norm == "max":
+        row_maxs = abs(X_norm).max(axis=1)
+        for i in range(3):
+            assert_almost_equal(row_maxs[i], 1.0)
+        assert_almost_equal(row_maxs[3], 0.0)
 
 
-def test_normalizer_l2():
+@pytest.mark.parametrize("norm", ["l1", "l2", "max"])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_normalizer_l1_l2_max(norm, csr_container):
     rng = np.random.RandomState(0)
     X_dense = rng.randn(4, 5)
-    X_sparse_unpruned = sparse.csr_matrix(X_dense)
+    X_sparse_unpruned = csr_container(X_dense)
 
     # set the row number 3 to zero
     X_dense[3, :] = 0.0
@@ -2054,90 +1846,47 @@ def test_normalizer_l2():
     X_sparse_unpruned.data[indptr_3:indptr_4] = 0.0
 
     # build the pruned variant using the regular constructor
-    X_sparse_pruned = sparse.csr_matrix(X_dense)
+    X_sparse_pruned = csr_container(X_dense)
 
     # check inputs that support the no-copy optim
     for X in (X_dense, X_sparse_pruned, X_sparse_unpruned):
-
-        normalizer = Normalizer(norm='l2', copy=True)
+        normalizer = Normalizer(norm=norm, copy=True)
         X_norm1 = normalizer.transform(X)
         assert X_norm1 is not X
         X_norm1 = toarray(X_norm1)
 
-        normalizer = Normalizer(norm='l2', copy=False)
+        normalizer = Normalizer(norm=norm, copy=False)
         X_norm2 = normalizer.transform(X)
         assert X_norm2 is X
         X_norm2 = toarray(X_norm2)
 
         for X_norm in (X_norm1, X_norm2):
-            for i in range(3):
-                assert_almost_equal(la.norm(X_norm[i]), 1.0)
-            assert_almost_equal(la.norm(X_norm[3]), 0.0)
-
-    # check input for which copy=False won't prevent a copy
-    for init in (sparse.coo_matrix, sparse.csc_matrix, sparse.lil_matrix):
-        X = init(X_dense)
-        X_norm = normalizer = Normalizer(norm='l2', copy=False).transform(X)
-
-        assert X_norm is not X
-        assert isinstance(X_norm, sparse.csr_matrix)
-
-        X_norm = toarray(X_norm)
-        for i in range(3):
-            assert_almost_equal(la.norm(X_norm[i]), 1.0)
-        assert_almost_equal(la.norm(X_norm[3]), 0.0)
+            check_normalizer(norm, X_norm)
 
 
-def test_normalizer_max():
+@pytest.mark.parametrize("norm", ["l1", "l2", "max"])
+@pytest.mark.parametrize(
+    "sparse_container", COO_CONTAINERS + CSC_CONTAINERS + LIL_CONTAINERS
+)
+def test_normalizer_l1_l2_max_non_csr(norm, sparse_container):
     rng = np.random.RandomState(0)
     X_dense = rng.randn(4, 5)
-    X_sparse_unpruned = sparse.csr_matrix(X_dense)
 
     # set the row number 3 to zero
     X_dense[3, :] = 0.0
 
-    # set the row number 3 to zero without pruning (can happen in real life)
-    indptr_3 = X_sparse_unpruned.indptr[3]
-    indptr_4 = X_sparse_unpruned.indptr[4]
-    X_sparse_unpruned.data[indptr_3:indptr_4] = 0.0
-
-    # build the pruned variant using the regular constructor
-    X_sparse_pruned = sparse.csr_matrix(X_dense)
-
-    # check inputs that support the no-copy optim
-    for X in (X_dense, X_sparse_pruned, X_sparse_unpruned):
-
-        normalizer = Normalizer(norm='max', copy=True)
-        X_norm1 = normalizer.transform(X)
-        assert X_norm1 is not X
-        X_norm1 = toarray(X_norm1)
-
-        normalizer = Normalizer(norm='max', copy=False)
-        X_norm2 = normalizer.transform(X)
-        assert X_norm2 is X
-        X_norm2 = toarray(X_norm2)
-
-        for X_norm in (X_norm1, X_norm2):
-            row_maxs = abs(X_norm).max(axis=1)
-            for i in range(3):
-                assert_almost_equal(row_maxs[i], 1.0)
-            assert_almost_equal(row_maxs[3], 0.0)
+    X = sparse_container(X_dense)
+    X_norm = Normalizer(norm=norm, copy=False).transform(X)
 
-    # check input for which copy=False won't prevent a copy
-    for init in (sparse.coo_matrix, sparse.csc_matrix, sparse.lil_matrix):
-        X = init(X_dense)
-        X_norm = normalizer = Normalizer(norm='l2', copy=False).transform(X)
-
-        assert X_norm is not X
-        assert isinstance(X_norm, sparse.csr_matrix)
+    assert X_norm is not X
+    assert sparse.issparse(X_norm) and X_norm.format == "csr"
 
-        X_norm = toarray(X_norm)
-        for i in range(3):
-            assert_almost_equal(row_maxs[i], 1.0)
-        assert_almost_equal(la.norm(X_norm[3]), 0.0)
+    X_norm = toarray(X_norm)
+    check_normalizer(norm, X_norm)
 
 
-def test_normalizer_max_sign():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_normalizer_max_sign(csr_container):
     # check that we normalize by a positive number even for negative data
     rng = np.random.RandomState(0)
     X_dense = rng.randn(4, 5)
@@ -2147,41 +1896,36 @@ def test_normalizer_max_sign():
     # largest magnitude is negative
     X_dense[2, abs(X_dense[2, :]).argmax()] *= -1
     X_all_neg = -np.abs(X_dense)
-    X_all_neg_sparse = sparse.csr_matrix(X_all_neg)
+    X_all_neg_sparse = csr_container(X_all_neg)
 
     for X in (X_dense, X_all_neg, X_all_neg_sparse):
-        normalizer = Normalizer(norm='max')
+        normalizer = Normalizer(norm="max")
         X_norm = normalizer.transform(X)
         assert X_norm is not X
         X_norm = toarray(X_norm)
-        assert_array_equal(
-            np.sign(X_norm), np.sign(toarray(X)))
+        assert_array_equal(np.sign(X_norm), np.sign(toarray(X)))
 
 
-def test_normalize():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_normalize(csr_container):
     # Test normalize function
     # Only tests functionality not used by the tests for Normalizer.
     X = np.random.RandomState(37).randn(3, 2)
-    assert_array_equal(normalize(X, copy=False),
-                       normalize(X.T, axis=0, copy=False).T)
-    with pytest.raises(ValueError):
-        normalize([[0]], axis=2)
-    with pytest.raises(ValueError):
-        normalize([[0]], norm='l3')
+    assert_array_equal(normalize(X, copy=False), normalize(X.T, axis=0, copy=False).T)
 
     rs = np.random.RandomState(0)
     X_dense = rs.randn(10, 5)
-    X_sparse = sparse.csr_matrix(X_dense)
+    X_sparse = csr_container(X_dense)
     ones = np.ones((10))
     for X in (X_dense, X_sparse):
         for dtype in (np.float32, np.float64):
-            for norm in ('l1', 'l2'):
+            for norm in ("l1", "l2"):
                 X = X.astype(dtype)
                 X_norm = normalize(X, norm=norm)
                 assert X_norm.dtype == dtype
 
                 X_norm = toarray(X_norm)
-                if norm == 'l1':
+                if norm == "l1":
                     row_sums = np.abs(X_norm).sum(axis=1)
                 else:
                     X_norm_squared = X_norm**2
@@ -2191,68 +1935,68 @@ def test_normalize():
 
     # Test return_norm
     X_dense = np.array([[3.0, 0, 4.0], [1.0, 0.0, 0.0], [2.0, 3.0, 0.0]])
-    for norm in ('l1', 'l2', 'max'):
+    for norm in ("l1", "l2", "max"):
         _, norms = normalize(X_dense, norm=norm, return_norm=True)
-        if norm == 'l1':
+        if norm == "l1":
             assert_array_almost_equal(norms, np.array([7.0, 1.0, 5.0]))
-        elif norm == 'l2':
+        elif norm == "l2":
             assert_array_almost_equal(norms, np.array([5.0, 1.0, 3.60555127]))
         else:
             assert_array_almost_equal(norms, np.array([4.0, 1.0, 3.0]))
 
-    X_sparse = sparse.csr_matrix(X_dense)
-    for norm in ('l1', 'l2'):
+    X_sparse = csr_container(X_dense)
+    for norm in ("l1", "l2"):
         with pytest.raises(NotImplementedError):
             normalize(X_sparse, norm=norm, return_norm=True)
-    _, norms = normalize(X_sparse, norm='max', return_norm=True)
+    _, norms = normalize(X_sparse, norm="max", return_norm=True)
     assert_array_almost_equal(norms, np.array([4.0, 1.0, 3.0]))
 
 
-def test_binarizer():
+@pytest.mark.parametrize(
+    "constructor", [np.array, list] + CSC_CONTAINERS + CSR_CONTAINERS
+)
+def test_binarizer(constructor):
     X_ = np.array([[1, 0, 5], [2, 3, -1]])
-
-    for init in (np.array, list, sparse.csr_matrix, sparse.csc_matrix):
-
-        X = init(X_.copy())
-
-        binarizer = Binarizer(threshold=2.0, copy=True)
-        X_bin = toarray(binarizer.transform(X))
-        assert np.sum(X_bin == 0) == 4
-        assert np.sum(X_bin == 1) == 2
-        X_bin = binarizer.transform(X)
-        assert sparse.issparse(X) == sparse.issparse(X_bin)
-
-        binarizer = Binarizer(copy=True).fit(X)
-        X_bin = toarray(binarizer.transform(X))
-        assert X_bin is not X
-        assert np.sum(X_bin == 0) == 2
-        assert np.sum(X_bin == 1) == 4
-
-        binarizer = Binarizer(copy=True)
-        X_bin = binarizer.transform(X)
-        assert X_bin is not X
-        X_bin = toarray(X_bin)
-        assert np.sum(X_bin == 0) == 2
-        assert np.sum(X_bin == 1) == 4
-
-        binarizer = Binarizer(copy=False)
-        X_bin = binarizer.transform(X)
-        if init is not list:
-            assert X_bin is X
-
-        binarizer = Binarizer(copy=False)
-        X_float = np.array([[1, 0, 5], [2, 3, -1]], dtype=np.float64)
-        X_bin = binarizer.transform(X_float)
-        if init is not list:
-            assert X_bin is X_float
-
-        X_bin = toarray(X_bin)
-        assert np.sum(X_bin == 0) == 2
-        assert np.sum(X_bin == 1) == 4
+    X = constructor(X_.copy())
+
+    binarizer = Binarizer(threshold=2.0, copy=True)
+    X_bin = toarray(binarizer.transform(X))
+    assert np.sum(X_bin == 0) == 4
+    assert np.sum(X_bin == 1) == 2
+    X_bin = binarizer.transform(X)
+    assert sparse.issparse(X) == sparse.issparse(X_bin)
+
+    binarizer = Binarizer(copy=True).fit(X)
+    X_bin = toarray(binarizer.transform(X))
+    assert X_bin is not X
+    assert np.sum(X_bin == 0) == 2
+    assert np.sum(X_bin == 1) == 4
+
+    binarizer = Binarizer(copy=True)
+    X_bin = binarizer.transform(X)
+    assert X_bin is not X
+    X_bin = toarray(X_bin)
+    assert np.sum(X_bin == 0) == 2
+    assert np.sum(X_bin == 1) == 4
+
+    binarizer = Binarizer(copy=False)
+    X_bin = binarizer.transform(X)
+    if constructor is not list:
+        assert X_bin is X
+
+    binarizer = Binarizer(copy=False)
+    X_float = np.array([[1, 0, 5], [2, 3, -1]], dtype=np.float64)
+    X_bin = binarizer.transform(X_float)
+    if constructor is not list:
+        assert X_bin is X_float
+
+    X_bin = toarray(X_bin)
+    assert np.sum(X_bin == 0) == 2
+    assert np.sum(X_bin == 1) == 4
 
     binarizer = Binarizer(threshold=-0.5, copy=True)
-    for init in (np.array, list):
-        X = init(X_.copy())
+    if constructor in (np.array, list):
+        X = constructor(X_.copy())
 
         X_bin = toarray(binarizer.transform(X))
         assert np.sum(X_bin == 0) == 1
@@ -2260,8 +2004,24 @@ def test_binarizer():
         X_bin = binarizer.transform(X)
 
     # Cannot use threshold < 0 for sparse
-    with pytest.raises(ValueError):
-        binarizer.transform(sparse.csc_matrix(X))
+    if constructor in CSC_CONTAINERS:
+        with pytest.raises(ValueError):
+            binarizer.transform(constructor(X))
+
+
+@pytest.mark.parametrize(
+    "array_namespace, device, dtype_name", yield_namespace_device_dtype_combinations()
+)
+def test_binarizer_array_api_int(array_namespace, device, dtype_name):
+    # Checks that Binarizer works with integer elements and float threshold
+    xp = _array_api_for_tests(array_namespace, device)
+    for dtype_name_ in [dtype_name, "int32", "int64"]:
+        X_np = np.reshape(np.asarray([0, 1, 2, 3, 4], dtype=dtype_name_), (-1, 1))
+        X_xp = xp.asarray(X_np, device=device)
+        binarized_np = Binarizer(threshold=2.5).fit_transform(X_np)
+        with config_context(array_api_dispatch=True):
+            binarized_xp = Binarizer(threshold=2.5).fit_transform(X_xp)
+        assert_array_equal(_convert_to_numpy(binarized_xp, xp), binarized_np)
 
 
 def test_center_kernel():
@@ -2288,6 +2048,79 @@ def test_center_kernel():
     K_pred_centered2 = centerer.transform(K_pred)
     assert_array_almost_equal(K_pred_centered, K_pred_centered2)
 
+    # check the results coherence with the method proposed in:
+    # B. SchÃļlkopf, A. Smola, and K.R. MÃŧller,
+    # "Nonlinear component analysis as a kernel eigenvalue problem"
+    # equation (B.3)
+
+    # K_centered3 = (I - 1_M) K (I - 1_M)
+    #             =  K - 1_M K - K 1_M + 1_M K 1_M
+    ones_M = np.ones_like(K_fit) / K_fit.shape[0]
+    K_fit_centered3 = K_fit - ones_M @ K_fit - K_fit @ ones_M + ones_M @ K_fit @ ones_M
+    assert_allclose(K_fit_centered, K_fit_centered3)
+
+    # K_test_centered3 = (K_test - 1'_M K)(I - 1_M)
+    #                  = K_test - 1'_M K - K_test 1_M + 1'_M K 1_M
+    ones_prime_M = np.ones_like(K_pred) / K_fit.shape[0]
+    K_pred_centered3 = (
+        K_pred - ones_prime_M @ K_fit - K_pred @ ones_M + ones_prime_M @ K_fit @ ones_M
+    )
+    assert_allclose(K_pred_centered, K_pred_centered3)
+
+
+def test_kernelcenterer_non_linear_kernel():
+    """Check kernel centering for non-linear kernel."""
+    rng = np.random.RandomState(0)
+    X, X_test = rng.randn(100, 50), rng.randn(20, 50)
+
+    def phi(X):
+        """Our mapping function phi."""
+        return np.vstack(
+            [
+                np.clip(X, a_min=0, a_max=None),
+                -np.clip(X, a_min=None, a_max=0),
+            ]
+        )
+
+    phi_X = phi(X)
+    phi_X_test = phi(X_test)
+
+    # centered the projection
+    scaler = StandardScaler(with_std=False)
+    phi_X_center = scaler.fit_transform(phi_X)
+    phi_X_test_center = scaler.transform(phi_X_test)
+
+    # create the different kernel
+    K = phi_X @ phi_X.T
+    K_test = phi_X_test @ phi_X.T
+    K_center = phi_X_center @ phi_X_center.T
+    K_test_center = phi_X_test_center @ phi_X_center.T
+
+    kernel_centerer = KernelCenterer()
+    kernel_centerer.fit(K)
+
+    assert_allclose(kernel_centerer.transform(K), K_center)
+    assert_allclose(kernel_centerer.transform(K_test), K_test_center)
+
+    # check the results coherence with the method proposed in:
+    # B. SchÃļlkopf, A. Smola, and K.R. MÃŧller,
+    # "Nonlinear component analysis as a kernel eigenvalue problem"
+    # equation (B.3)
+
+    # K_centered = (I - 1_M) K (I - 1_M)
+    #            =  K - 1_M K - K 1_M + 1_M K 1_M
+    ones_M = np.ones_like(K) / K.shape[0]
+    K_centered = K - ones_M @ K - K @ ones_M + ones_M @ K @ ones_M
+    assert_allclose(kernel_centerer.transform(K), K_centered)
+
+    # K_test_centered = (K_test - 1'_M K)(I - 1_M)
+    #                 = K_test - 1'_M K - K_test 1_M + 1'_M K 1_M
+    ones_prime_M = np.ones_like(K_test) / K.shape[0]
+    K_test_centered = (
+        K_test - ones_prime_M @ K - K_test @ ones_M + ones_prime_M @ K @ ones_M
+    )
+    assert_allclose(kernel_centerer.transform(K_test), K_test_centered)
+
 
 def test_cv_pipeline_precomputed():
     # Cross-validate a regression on four coplanar points with the same
@@ -2300,12 +2133,7 @@ def test_cv_pipeline_precomputed():
     pipeline = Pipeline([("kernel_centerer", kcent), ("svr", SVR())])
 
     # did the pipeline set the pairwise attribute?
-    assert pipeline._get_tags()['pairwise']
-
-    # TODO: Remove in 1.1
-    msg = r"Attribute _pairwise was deprecated in version 0\.24"
-    with pytest.warns(FutureWarning, match=msg):
-        assert pipeline._pairwise
+    assert pipeline.__sklearn_tags__().input_tags.pairwise
 
     # test cross-validation, score should be almost perfect
     # NB: this test is pretty vacuous -- it's mainly to test integration
@@ -2314,18 +2142,10 @@ def test_cv_pipeline_precomputed():
     assert_array_almost_equal(y_true, y_pred)
 
 
-# TODO: Remove in 1.1
-def test_pairwise_deprecated():
-    kcent = KernelCenterer()
-    msg = r"Attribute _pairwise was deprecated in version 0\.24"
-    with pytest.warns(FutureWarning, match=msg):
-        kcent._pairwise
-
-
 def test_fit_transform():
     rng = np.random.RandomState(0)
     X = rng.random_sample((5, 4))
-    for obj in ((StandardScaler(), Normalizer(), Binarizer())):
+    for obj in (StandardScaler(), Normalizer(), Binarizer()):
         X_transformed = obj.fit(X).transform(X)
         X_transformed2 = obj.fit_transform(X)
         assert_array_equal(X_transformed, X_transformed2)
@@ -2337,24 +2157,14 @@ def test_add_dummy_feature():
     assert_array_equal(X, [[1, 1, 0], [1, 0, 1], [1, 0, 1]])
 
 
-def test_add_dummy_feature_coo():
-    X = sparse.coo_matrix([[1, 0], [0, 1], [0, 1]])
-    X = add_dummy_feature(X)
-    assert sparse.isspmatrix_coo(X), X
-    assert_array_equal(X.toarray(), [[1, 1, 0], [1, 0, 1], [1, 0, 1]])
-
-
-def test_add_dummy_feature_csc():
-    X = sparse.csc_matrix([[1, 0], [0, 1], [0, 1]])
-    X = add_dummy_feature(X)
-    assert sparse.isspmatrix_csc(X), X
-    assert_array_equal(X.toarray(), [[1, 1, 0], [1, 0, 1], [1, 0, 1]])
-
-
-def test_add_dummy_feature_csr():
-    X = sparse.csr_matrix([[1, 0], [0, 1], [0, 1]])
+@pytest.mark.parametrize(
+    "sparse_container", COO_CONTAINERS + CSC_CONTAINERS + CSR_CONTAINERS
+)
+def test_add_dummy_feature_sparse(sparse_container):
+    X = sparse_container([[1, 0], [0, 1], [0, 1]])
+    desired_format = X.format
     X = add_dummy_feature(X)
-    assert sparse.isspmatrix_csr(X), X
+    assert sparse.issparse(X) and X.format == desired_format, X
     assert_array_equal(X.toarray(), [[1, 1, 0], [1, 0, 1], [1, 0, 1]])
 
 
@@ -2363,9 +2173,11 @@ def test_fit_cold_start():
     X_2d = X[:, :2]
 
     # Scalers that have a partial_fit method
-    scalers = [StandardScaler(with_mean=False, with_std=False),
-               MinMaxScaler(),
-               MaxAbsScaler()]
+    scalers = [
+        StandardScaler(with_mean=False, with_std=False),
+        MinMaxScaler(),
+        MaxAbsScaler(),
+    ]
 
     for scaler in scalers:
         scaler.fit_transform(X)
@@ -2374,17 +2186,7 @@ def test_fit_cold_start():
         scaler.fit_transform(X_2d)
 
 
-def test_quantile_transform_valid_axis():
-    X = np.array([[0, 25, 50, 75, 100],
-                  [2, 4, 6, 8, 10],
-                  [2.6, 4.1, 2.3, 9.5, 0.1]])
-
-    with pytest.raises(ValueError, match="axis should be either equal "
-                                         "to 0 or 1. Got axis=2"):
-        quantile_transform(X.T, axis=2)
-
-
-@pytest.mark.parametrize("method", ['box-cox', 'yeo-johnson'])
+@pytest.mark.parametrize("method", ["box-cox", "yeo-johnson"])
 def test_power_transformer_notfitted(method):
     pt = PowerTransformer(method=method)
     X = np.abs(X_1col)
@@ -2394,13 +2196,13 @@ def test_power_transformer_notfitted(method):
         pt.inverse_transform(X)
 
 
-@pytest.mark.parametrize('method', ['box-cox', 'yeo-johnson'])
-@pytest.mark.parametrize('standardize', [True, False])
-@pytest.mark.parametrize('X', [X_1col, X_2d])
+@pytest.mark.parametrize("method", ["box-cox", "yeo-johnson"])
+@pytest.mark.parametrize("standardize", [True, False])
+@pytest.mark.parametrize("X", [X_1col, X_2d])
 def test_power_transformer_inverse(method, standardize, X):
     # Make sure we get the original input when applying transform and then
     # inverse transform
-    X = np.abs(X) if method == 'box-cox' else X
+    X = np.abs(X) if method == "box-cox" else X
     pt = PowerTransformer(method=method, standardize=standardize)
     X_trans = pt.fit_transform(X)
     assert_almost_equal(X, pt.inverse_transform(X_trans))
@@ -2410,13 +2212,10 @@ def test_power_transformer_1d():
     X = np.abs(X_1col)
 
     for standardize in [True, False]:
-        pt = PowerTransformer(method='box-cox', standardize=standardize)
+        pt = PowerTransformer(method="box-cox", standardize=standardize)
 
         X_trans = pt.fit_transform(X)
-        X_trans_func = power_transform(
-            X, method='box-cox',
-            standardize=standardize
-        )
+        X_trans_func = power_transform(X, method="box-cox", standardize=standardize)
 
         X_expected, lambda_expected = stats.boxcox(X.flatten())
 
@@ -2437,13 +2236,10 @@ def test_power_transformer_2d():
     X = np.abs(X_2d)
 
     for standardize in [True, False]:
-        pt = PowerTransformer(method='box-cox', standardize=standardize)
+        pt = PowerTransformer(method="box-cox", standardize=standardize)
 
         X_trans_class = pt.fit_transform(X)
-        X_trans_func = power_transform(
-            X, method='box-cox',
-            standardize=standardize
-        )
+        X_trans_func = power_transform(X, method="box-cox", standardize=standardize)
 
         for X_trans in [X_trans_class, X_trans_func]:
             for j in range(X_trans.shape[1]):
@@ -2467,10 +2263,10 @@ def test_power_transformer_boxcox_strictly_positive_exception():
     # Exceptions should be raised for negative arrays and zero arrays when
     # method is boxcox
 
-    pt = PowerTransformer(method='box-cox')
+    pt = PowerTransformer(method="box-cox")
     pt.fit(np.abs(X_2d))
     X_with_negatives = X_2d
-    not_positive_message = 'strictly positive'
+    not_positive_message = "strictly positive"
 
     with pytest.raises(ValueError, match=not_positive_message):
         pt.transform(X_with_negatives)
@@ -2479,7 +2275,7 @@ def test_power_transformer_boxcox_strictly_positive_exception():
         pt.fit(X_with_negatives)
 
     with pytest.raises(ValueError, match=not_positive_message):
-        power_transform(X_with_negatives, method='box-cox')
+        power_transform(X_with_negatives, method="box-cox")
 
     with pytest.raises(ValueError, match=not_positive_message):
         pt.transform(np.zeros(X_2d.shape))
@@ -2488,17 +2284,16 @@ def test_power_transformer_boxcox_strictly_positive_exception():
         pt.fit(np.zeros(X_2d.shape))
 
     with pytest.raises(ValueError, match=not_positive_message):
-        power_transform(np.zeros(X_2d.shape), method='box-cox')
+        power_transform(np.zeros(X_2d.shape), method="box-cox")
 
 
-@pytest.mark.parametrize('X', [X_2d, np.abs(X_2d), -np.abs(X_2d),
-                               np.zeros(X_2d.shape)])
+@pytest.mark.parametrize("X", [X_2d, np.abs(X_2d), -np.abs(X_2d), np.zeros(X_2d.shape)])
 def test_power_transformer_yeojohnson_any_input(X):
     # Yeo-Johnson method should support any kind of input
-    power_transform(X, method='yeo-johnson')
+    power_transform(X, method="yeo-johnson")
 
 
-@pytest.mark.parametrize("method", ['box-cox', 'yeo-johnson'])
+@pytest.mark.parametrize("method", ["box-cox", "yeo-johnson"])
 def test_power_transformer_shape_exception(method):
     pt = PowerTransformer(method=method)
     X = np.abs(X_2d)
@@ -2506,8 +2301,9 @@ def test_power_transformer_shape_exception(method):
 
     # Exceptions should be raised for arrays with different num_columns
     # than during fitting
-    wrong_shape_message = (r"X has \d+ features, but PowerTransformer is "
-                           r"expecting \d+ features")
+    wrong_shape_message = (
+        r"X has \d+ features, but PowerTransformer is expecting \d+ features"
+    )
 
     with pytest.raises(ValueError, match=wrong_shape_message):
         pt.transform(X[:, 0:1])
@@ -2516,18 +2312,8 @@ def test_power_transformer_shape_exception(method):
         pt.inverse_transform(X[:, 0:1])
 
 
-def test_power_transformer_method_exception():
-    pt = PowerTransformer(method='monty-python')
-    X = np.abs(X_2d)
-
-    # An exception should be raised if PowerTransformer.method isn't valid
-    bad_method_message = "'method' must be one of"
-    with pytest.raises(ValueError, match=bad_method_message):
-        pt.fit(X)
-
-
 def test_power_transformer_lambda_zero():
-    pt = PowerTransformer(method='box-cox', standardize=False)
+    pt = PowerTransformer(method="box-cox", standardize=False)
     X = np.abs(X_2d)[:, 0:1]
 
     # Test the lambda = 0 case
@@ -2538,7 +2324,7 @@ def test_power_transformer_lambda_zero():
 
 def test_power_transformer_lambda_one():
     # Make sure lambda = 1 corresponds to the identity for yeo-johnson
-    pt = PowerTransformer(method='yeo-johnson', standardize=False)
+    pt = PowerTransformer(method="yeo-johnson", standardize=False)
     X = np.abs(X_2d)[:, 0:1]
 
     pt.lambdas_ = np.array([1])
@@ -2546,12 +2332,16 @@ def test_power_transformer_lambda_one():
     assert_array_almost_equal(X_trans, X)
 
 
-@pytest.mark.parametrize("method, lmbda", [('box-cox', .1),
-                                           ('box-cox', .5),
-                                           ('yeo-johnson', .1),
-                                           ('yeo-johnson', .5),
-                                           ('yeo-johnson', 1.),
-                                           ])
+@pytest.mark.parametrize(
+    "method, lmbda",
+    [
+        ("box-cox", 0.1),
+        ("box-cox", 0.5),
+        ("yeo-johnson", 0.1),
+        ("yeo-johnson", 0.5),
+        ("yeo-johnson", 1.0),
+    ],
+)
 def test_optimization_power_transformer(method, lmbda):
     # Test the optimization procedure:
     # - set a predefined value for lambda
@@ -2563,6 +2353,11 @@ def test_optimization_power_transformer(method, lmbda):
     n_samples = 20000
     X = rng.normal(loc=0, scale=1, size=(n_samples, 1))
 
+    if method == "box-cox":
+        # For box-cox, means that lmbda * y + 1 > 0 or y > - 1 / lmbda
+        # Clip the data here to make sure the inequality is valid.
+        X = np.clip(X, -1 / lmbda + 1e-5, None)
+
     pt = PowerTransformer(method=method, standardize=False)
     pt.lambdas_ = [lmbda]
     X_inv = pt.inverse_transform(X)
@@ -2570,23 +2365,29 @@ def test_optimization_power_transformer(method, lmbda):
     pt = PowerTransformer(method=method, standardize=False)
     X_inv_trans = pt.fit_transform(X_inv)
 
-    assert_almost_equal(0, np.linalg.norm(X - X_inv_trans) / n_samples,
-                        decimal=2)
+    assert_almost_equal(0, np.linalg.norm(X - X_inv_trans) / n_samples, decimal=2)
     assert_almost_equal(0, X_inv_trans.mean(), decimal=1)
     assert_almost_equal(1, X_inv_trans.std(), decimal=1)
 
 
+def test_invserse_box_cox():
+    # output nan if the input is invalid
+    pt = PowerTransformer(method="box-cox", standardize=False)
+    pt.lambdas_ = [0.5]
+    X_inv = pt.inverse_transform([[-2.1]])
+    assert np.isnan(X_inv)
+
+
 def test_yeo_johnson_darwin_example():
     # test from original paper "A new family of power transformations to
     # improve normality or symmetry" by Yeo and Johnson.
-    X = [6.1, -8.4, 1.0, 2.0, 0.7, 2.9, 3.5, 5.1, 1.8, 3.6, 7.0, 3.0, 9.3,
-         7.5, -6.0]
+    X = [6.1, -8.4, 1.0, 2.0, 0.7, 2.9, 3.5, 5.1, 1.8, 3.6, 7.0, 3.0, 9.3, 7.5, -6.0]
     X = np.array(X).reshape(-1, 1)
-    lmbda = PowerTransformer(method='yeo-johnson').fit(X).lambdas_
+    lmbda = PowerTransformer(method="yeo-johnson").fit(X).lambdas_
     assert np.allclose(lmbda, 1.305, atol=1e-3)
 
 
-@pytest.mark.parametrize('method', ['box-cox', 'yeo-johnson'])
+@pytest.mark.parametrize("method", ["box-cox", "yeo-johnson"])
 def test_power_transformer_nans(method):
     # Make sure lambda estimation is not influenced by NaN values
     # and that transform() supports NaN silently
@@ -2609,25 +2410,25 @@ def test_power_transformer_nans(method):
     assert_array_equal(np.isnan(X_trans), np.isnan(X))
 
 
-@pytest.mark.parametrize('method', ['box-cox', 'yeo-johnson'])
-@pytest.mark.parametrize('standardize', [True, False])
+@pytest.mark.parametrize("method", ["box-cox", "yeo-johnson"])
+@pytest.mark.parametrize("standardize", [True, False])
 def test_power_transformer_fit_transform(method, standardize):
     # check that fit_transform() and fit().transform() return the same values
     X = X_1col
-    if method == 'box-cox':
+    if method == "box-cox":
         X = np.abs(X)
 
     pt = PowerTransformer(method, standardize=standardize)
     assert_array_almost_equal(pt.fit(X).transform(X), pt.fit_transform(X))
 
 
-@pytest.mark.parametrize('method', ['box-cox', 'yeo-johnson'])
-@pytest.mark.parametrize('standardize', [True, False])
+@pytest.mark.parametrize("method", ["box-cox", "yeo-johnson"])
+@pytest.mark.parametrize("standardize", [True, False])
 def test_power_transformer_copy_True(method, standardize):
     # Check that neither fit, transform, fit_transform nor inverse_transform
     # modify X inplace when copy=True
     X = X_1col
-    if method == 'box-cox':
+    if method == "box-cox":
         X = np.abs(X)
 
     X_original = X.copy()
@@ -2649,13 +2450,13 @@ def test_power_transformer_copy_True(method, standardize):
     assert X_trans is not X_inv_trans
 
 
-@pytest.mark.parametrize('method', ['box-cox', 'yeo-johnson'])
-@pytest.mark.parametrize('standardize', [True, False])
+@pytest.mark.parametrize("method", ["box-cox", "yeo-johnson"])
+@pytest.mark.parametrize("standardize", [True, False])
 def test_power_transformer_copy_False(method, standardize):
     # check that when copy=False fit doesn't change X inplace but transform,
     # fit_transform and inverse_transform do.
     X = X_1col
-    if method == 'box-cox':
+    if method == "box-cox":
         X = np.abs(X)
 
     X_original = X.copy()
@@ -2670,7 +2471,7 @@ def test_power_transformer_copy_False(method, standardize):
     X_trans = pt.transform(X)
     assert X_trans is X
 
-    if method == 'box-cox':
+    if method == "box-cox":
         X = np.abs(X)
     X_trans = pt.fit_transform(X)
     assert X_trans is X
@@ -2679,10 +2480,28 @@ def test_power_transformer_copy_False(method, standardize):
     assert X_trans is X_inv_trans
 
 
+def test_power_transformer_box_cox_raise_all_nans_col():
+    """Check that box-cox raises informative when a column contains all nans.
+
+    Non-regression test for gh-26303
+    """
+    X = rng.random_sample((4, 5))
+    X[:, 0] = np.nan
+
+    err_msg = "Column must not be all nan."
+
+    pt = PowerTransformer(method="box-cox")
+    with pytest.raises(ValueError, match=err_msg):
+        pt.fit_transform(X)
+
+
 @pytest.mark.parametrize(
     "X_2",
-    [sparse.random(10, 1, density=0.8, random_state=0),
-     sparse.csr_matrix(np.full((10, 1), fill_value=np.nan))]
+    [sparse.random(10, 1, density=0.8, random_state=0)]
+    + [
+        csr_container(np.full((10, 1), fill_value=np.nan))
+        for csr_container in CSR_CONTAINERS
+    ],
 )
 def test_standard_scaler_sparse_partial_fit_finite_variance(X_2):
     # non-regression test for:
@@ -2693,11 +2512,9 @@ def test_standard_scaler_sparse_partial_fit_finite_variance(X_2):
     assert np.isfinite(scaler.var_[0])
 
 
-@pytest.mark.parametrize(
-    "feature_range", [(0, 1), (-10, 10)]
-)
+@pytest.mark.parametrize("feature_range", [(0, 1), (-10, 10)])
 def test_minmax_scaler_clip(feature_range):
-    # test behaviour of the paramter 'clip' in MinMaxScaler
+    # test behaviour of the parameter 'clip' in MinMaxScaler
     X = iris.data
     scaler = MinMaxScaler(feature_range=feature_range, clip=True).fit(X)
     X_min, X_max = np.min(X, axis=0), np.max(X, axis=0)
@@ -2705,5 +2522,172 @@ def test_minmax_scaler_clip(feature_range):
     X_transformed = scaler.transform(X_test)
     assert_allclose(
         X_transformed,
-        [[feature_range[0], feature_range[0],
-          feature_range[1], feature_range[1]]])
+        [[feature_range[0], feature_range[0], feature_range[1], feature_range[1]]],
+    )
+
+
+def test_standard_scaler_raise_error_for_1d_input():
+    """Check that `inverse_transform` from `StandardScaler` raises an error
+    with 1D array.
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/19518
+    """
+    scaler = StandardScaler().fit(X_2d)
+    err_msg = "Expected 2D array, got 1D array instead"
+    with pytest.raises(ValueError, match=err_msg):
+        scaler.inverse_transform(X_2d[:, 0])
+
+
+def test_power_transformer_significantly_non_gaussian():
+    """Check that significantly non-Gaussian data before transforms correctly.
+
+    For some explored lambdas, the transformed data may be constant and will
+    be rejected. Non-regression test for
+    https://github.com/scikit-learn/scikit-learn/issues/14959
+    """
+
+    X_non_gaussian = 1e6 * np.array(
+        [0.6, 2.0, 3.0, 4.0] * 4 + [11, 12, 12, 16, 17, 20, 85, 90], dtype=np.float64
+    ).reshape(-1, 1)
+    pt = PowerTransformer()
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", RuntimeWarning)
+        X_trans = pt.fit_transform(X_non_gaussian)
+
+    assert not np.any(np.isnan(X_trans))
+    assert X_trans.mean() == pytest.approx(0.0)
+    assert X_trans.std() == pytest.approx(1.0)
+    assert X_trans.min() > -2
+    assert X_trans.max() < 2
+
+
+@pytest.mark.parametrize(
+    "Transformer",
+    [
+        MinMaxScaler,
+        MaxAbsScaler,
+        RobustScaler,
+        StandardScaler,
+        QuantileTransformer,
+        PowerTransformer,
+    ],
+)
+def test_one_to_one_features(Transformer):
+    """Check one-to-one transformers give correct feature names."""
+    tr = Transformer().fit(iris.data)
+    names_out = tr.get_feature_names_out(iris.feature_names)
+    assert_array_equal(names_out, iris.feature_names)
+
+
+@pytest.mark.parametrize(
+    "Transformer",
+    [
+        MinMaxScaler,
+        MaxAbsScaler,
+        RobustScaler,
+        StandardScaler,
+        QuantileTransformer,
+        PowerTransformer,
+        Normalizer,
+        Binarizer,
+    ],
+)
+def test_one_to_one_features_pandas(Transformer):
+    """Check one-to-one transformers give correct feature names."""
+    pd = pytest.importorskip("pandas")
+
+    df = pd.DataFrame(iris.data, columns=iris.feature_names)
+    tr = Transformer().fit(df)
+
+    names_out_df_default = tr.get_feature_names_out()
+    assert_array_equal(names_out_df_default, iris.feature_names)
+
+    names_out_df_valid_in = tr.get_feature_names_out(iris.feature_names)
+    assert_array_equal(names_out_df_valid_in, iris.feature_names)
+
+    msg = re.escape("input_features is not equal to feature_names_in_")
+    with pytest.raises(ValueError, match=msg):
+        invalid_names = list("abcd")
+        tr.get_feature_names_out(invalid_names)
+
+
+def test_kernel_centerer_feature_names_out():
+    """Test that kernel centerer `feature_names_out`."""
+
+    rng = np.random.RandomState(0)
+    X = rng.random_sample((6, 4))
+    X_pairwise = linear_kernel(X)
+    centerer = KernelCenterer().fit(X_pairwise)
+
+    names_out = centerer.get_feature_names_out()
+    samples_out2 = X_pairwise.shape[1]
+    assert_array_equal(names_out, [f"kernelcenterer{i}" for i in range(samples_out2)])
+
+
+@pytest.mark.parametrize("standardize", [True, False])
+def test_power_transformer_constant_feature(standardize):
+    """Check that PowerTransfomer leaves constant features unchanged."""
+    X = [[-2, 0, 2], [-2, 0, 2], [-2, 0, 2]]
+
+    pt = PowerTransformer(method="yeo-johnson", standardize=standardize).fit(X)
+
+    assert_allclose(pt.lambdas_, [1, 1, 1])
+
+    Xft = pt.fit_transform(X)
+    Xt = pt.transform(X)
+
+    for Xt_ in [Xft, Xt]:
+        if standardize:
+            assert_allclose(Xt_, np.zeros_like(X))
+        else:
+            assert_allclose(Xt_, X)
+
+
+@pytest.mark.skipif(
+    sp_version < parse_version("1.12"),
+    reason="scipy version 1.12 required for stable yeo-johnson",
+)
+def test_power_transformer_no_warnings():
+    """Verify that PowerTransformer operates without raising any warnings on valid data.
+
+    This test addresses numerical issues with floating point numbers (mostly
+    overflows) with the Yeo-Johnson transform, see
+    https://github.com/scikit-learn/scikit-learn/issues/23319#issuecomment-1464933635
+    """
+    x = np.array(
+        [
+            2003.0,
+            1950.0,
+            1997.0,
+            2000.0,
+            2009.0,
+            2009.0,
+            1980.0,
+            1999.0,
+            2007.0,
+            1991.0,
+        ]
+    )
+
+    def _test_no_warnings(data):
+        """Internal helper to test for unexpected warnings."""
+        with warnings.catch_warnings(record=True) as caught_warnings:
+            warnings.simplefilter("always")  # Ensure all warnings are captured
+            PowerTransformer(method="yeo-johnson", standardize=True).fit_transform(data)
+
+        assert not caught_warnings, "Unexpected warnings were raised:\n" + "\n".join(
+            str(w.message) for w in caught_warnings
+        )
+
+    # Full dataset: Should not trigger overflow in variance calculation.
+    _test_no_warnings(x.reshape(-1, 1))
+
+    # Subset of data: Should not trigger overflow in power calculation.
+    _test_no_warnings(x[:5].reshape(-1, 1))
+
+
+def test_yeojohnson_for_different_scipy_version():
+    """Check that the results are consistent across different SciPy versions."""
+    pt = PowerTransformer(method="yeo-johnson").fit(X_1col)
+    pt.lambdas_[0] == pytest.approx(0.99546157, rel=1e-7)
diff --git a/sklearn/preprocessing/tests/test_discretization.py b/sklearn/preprocessing/tests/test_discretization.py
index 9d607c82d5831..7463a8608291c 100644
--- a/sklearn/preprocessing/tests/test_discretization.py
+++ b/sklearn/preprocessing/tests/test_discretization.py
@@ -1,117 +1,266 @@
+import warnings
 
-import pytest
 import numpy as np
+import pytest
 import scipy.sparse as sp
-import warnings
 
-from sklearn.preprocessing import KBinsDiscretizer
-from sklearn.preprocessing import OneHotEncoder
+from sklearn import clone
+from sklearn.preprocessing import KBinsDiscretizer, OneHotEncoder
 from sklearn.utils._testing import (
+    assert_allclose,
+    assert_allclose_dense_sparse,
     assert_array_almost_equal,
     assert_array_equal,
-    assert_warns_message,
-    assert_allclose_dense_sparse
+    ignore_warnings,
 )
 
-X = [[-2, 1.5, -4, -1],
-     [-1, 2.5, -3, -0.5],
-     [0, 3.5, -2, 0.5],
-     [1, 4.5, -1, 2]]
+X = [[-2, 1.5, -4, -1], [-1, 2.5, -3, -0.5], [0, 3.5, -2, 0.5], [1, 4.5, -1, 2]]
 
 
 @pytest.mark.parametrize(
-    'strategy, expected',
-    [('uniform', [[0, 0, 0, 0], [1, 1, 1, 0], [2, 2, 2, 1], [2, 2, 2, 2]]),
-     ('kmeans', [[0, 0, 0, 0], [0, 0, 0, 0], [1, 1, 1, 1], [2, 2, 2, 2]]),
-     ('quantile', [[0, 0, 0, 0], [1, 1, 1, 1], [2, 2, 2, 2], [2, 2, 2, 2]])])
-def test_fit_transform(strategy, expected):
-    est = KBinsDiscretizer(n_bins=3, encode='ordinal', strategy=strategy)
-    est.fit(X)
-    assert_array_equal(expected, est.transform(X))
+    "strategy, quantile_method, expected, sample_weight",
+    [
+        (
+            "uniform",
+            "warn",  # default, will not warn when strategy != "quantile"
+            [[0, 0, 0, 0], [1, 1, 1, 0], [2, 2, 2, 1], [2, 2, 2, 2]],
+            None,
+        ),
+        (
+            "kmeans",
+            "warn",  # default, will not warn when strategy != "quantile"
+            [[0, 0, 0, 0], [0, 0, 0, 0], [1, 1, 1, 1], [2, 2, 2, 2]],
+            None,
+        ),
+        (
+            "quantile",
+            "averaged_inverted_cdf",
+            [[0, 0, 0, 0], [1, 1, 1, 1], [2, 2, 2, 2], [2, 2, 2, 2]],
+            None,
+        ),
+        (
+            "uniform",
+            "warn",  # default, will not warn when strategy != "quantile"
+            [[0, 0, 0, 0], [1, 1, 1, 0], [2, 2, 2, 1], [2, 2, 2, 2]],
+            [1, 1, 2, 1],
+        ),
+        (
+            "uniform",
+            "warn",  # default, will not warn when strategy != "quantile"
+            [[0, 0, 0, 0], [1, 1, 1, 0], [2, 2, 2, 1], [2, 2, 2, 2]],
+            [1, 1, 1, 1],
+        ),
+        (
+            "quantile",
+            "averaged_inverted_cdf",
+            [[0, 0, 0, 0], [1, 1, 1, 1], [2, 2, 2, 2], [2, 2, 2, 2]],
+            [1, 1, 2, 1],
+        ),
+        (
+            "quantile",
+            "averaged_inverted_cdf",
+            [[0, 0, 0, 0], [1, 1, 1, 1], [2, 2, 2, 2], [2, 2, 2, 2]],
+            [1, 1, 1, 1],
+        ),
+        (
+            "quantile",
+            "averaged_inverted_cdf",
+            [[0, 0, 0, 0], [0, 0, 0, 0], [1, 1, 1, 1], [2, 2, 2, 2]],
+            [0, 1, 1, 1],
+        ),
+        (
+            "kmeans",
+            "warn",  # default, will not warn when strategy != "quantile"
+            [[0, 0, 0, 0], [1, 1, 1, 0], [1, 1, 1, 1], [2, 2, 2, 2]],
+            [1, 0, 3, 1],
+        ),
+        (
+            "kmeans",
+            "warn",  # default, will not warn when strategy != "quantile"
+            [[0, 0, 0, 0], [0, 0, 0, 0], [1, 1, 1, 1], [2, 2, 2, 2]],
+            [1, 1, 1, 1],
+        ),
+    ],
+)
+def test_fit_transform(strategy, quantile_method, expected, sample_weight):
+    est = KBinsDiscretizer(
+        n_bins=3, encode="ordinal", strategy=strategy, quantile_method=quantile_method
+    )
+    with ignore_warnings(category=UserWarning):
+        # Ignore the warning on removed small bins.
+        est.fit(X, sample_weight=sample_weight)
+    assert_array_equal(est.transform(X), expected)
 
 
 def test_valid_n_bins():
-    KBinsDiscretizer(n_bins=2).fit_transform(X)
-    KBinsDiscretizer(n_bins=np.array([2])[0]).fit_transform(X)
-    assert KBinsDiscretizer(n_bins=2).fit(X).n_bins_.dtype == np.dtype(int)
-
-
-def test_invalid_n_bins():
-    est = KBinsDiscretizer(n_bins=1)
-    err_msg = ("KBinsDiscretizer received an invalid "
-               "number of bins. Received 1, expected at least 2.")
-    with pytest.raises(ValueError, match=err_msg):
-        est.fit_transform(X)
-
-    est = KBinsDiscretizer(n_bins=1.1)
-    err_msg = ("KBinsDiscretizer received an invalid "
-               "n_bins type. Received float, expected int.")
-    with pytest.raises(ValueError, match=err_msg):
-        est.fit_transform(X)
+    KBinsDiscretizer(n_bins=2, quantile_method="averaged_inverted_cdf").fit_transform(X)
+    KBinsDiscretizer(
+        n_bins=np.array([2])[0], quantile_method="averaged_inverted_cdf"
+    ).fit_transform(X)
+    assert KBinsDiscretizer(n_bins=2, quantile_method="averaged_inverted_cdf").fit(
+        X
+    ).n_bins_.dtype == np.dtype(int)
 
 
 def test_invalid_n_bins_array():
     # Bad shape
-    n_bins = np.full((2, 4), 2.)
-    est = KBinsDiscretizer(n_bins=n_bins)
+    n_bins = np.full((2, 4), 2.0)
+    est = KBinsDiscretizer(n_bins=n_bins, quantile_method="averaged_inverted_cdf")
     err_msg = r"n_bins must be a scalar or array of shape \(n_features,\)."
     with pytest.raises(ValueError, match=err_msg):
         est.fit_transform(X)
 
     # Incorrect number of features
     n_bins = [1, 2, 2]
-    est = KBinsDiscretizer(n_bins=n_bins)
+    est = KBinsDiscretizer(n_bins=n_bins, quantile_method="averaged_inverted_cdf")
     err_msg = r"n_bins must be a scalar or array of shape \(n_features,\)."
     with pytest.raises(ValueError, match=err_msg):
         est.fit_transform(X)
 
     # Bad bin values
     n_bins = [1, 2, 2, 1]
-    est = KBinsDiscretizer(n_bins=n_bins)
-    err_msg = ("KBinsDiscretizer received an invalid number of bins "
-               "at indices 0, 3. Number of bins must be at least 2, "
-               "and must be an int.")
+    est = KBinsDiscretizer(n_bins=n_bins, quantile_method="averaged_inverted_cdf")
+    err_msg = (
+        "KBinsDiscretizer received an invalid number of bins "
+        "at indices 0, 3. Number of bins must be at least 2, "
+        "and must be an int."
+    )
     with pytest.raises(ValueError, match=err_msg):
         est.fit_transform(X)
 
     # Float bin values
     n_bins = [2.1, 2, 2.1, 2]
-    est = KBinsDiscretizer(n_bins=n_bins)
-    err_msg = ("KBinsDiscretizer received an invalid number of bins "
-               "at indices 0, 2. Number of bins must be at least 2, "
-               "and must be an int.")
+    est = KBinsDiscretizer(n_bins=n_bins, quantile_method="averaged_inverted_cdf")
+    err_msg = (
+        "KBinsDiscretizer received an invalid number of bins "
+        "at indices 0, 2. Number of bins must be at least 2, "
+        "and must be an int."
+    )
     with pytest.raises(ValueError, match=err_msg):
         est.fit_transform(X)
 
 
 @pytest.mark.parametrize(
-    'strategy, expected',
-    [('uniform', [[0, 0, 0, 0], [0, 1, 1, 0], [1, 2, 2, 1], [1, 2, 2, 2]]),
-     ('kmeans', [[0, 0, 0, 0], [0, 0, 0, 0], [1, 1, 1, 1], [1, 2, 2, 2]]),
-     ('quantile', [[0, 0, 0, 0], [0, 1, 1, 1], [1, 2, 2, 2], [1, 2, 2, 2]])])
-def test_fit_transform_n_bins_array(strategy, expected):
-    est = KBinsDiscretizer(n_bins=[2, 3, 3, 3], encode='ordinal',
-                           strategy=strategy).fit(X)
-    assert_array_equal(expected, est.transform(X))
+    "strategy, quantile_method, expected, sample_weight",
+    [
+        (
+            "uniform",
+            "warn",  # default, will not warn when strategy != "quantile"
+            [[0, 0, 0, 0], [0, 1, 1, 0], [1, 2, 2, 1], [1, 2, 2, 2]],
+            None,
+        ),
+        (
+            "kmeans",
+            "warn",  # default, will not warn when strategy != "quantile"
+            [[0, 0, 0, 0], [0, 0, 0, 0], [1, 1, 1, 1], [1, 2, 2, 2]],
+            None,
+        ),
+        (
+            "quantile",
+            "linear",
+            [[0, 0, 0, 0], [0, 1, 1, 1], [1, 2, 2, 2], [1, 2, 2, 2]],
+            None,
+        ),
+        (
+            "quantile",
+            "averaged_inverted_cdf",
+            [[0, 0, 0, 0], [0, 1, 1, 1], [1, 2, 2, 2], [1, 2, 2, 2]],
+            None,
+        ),
+        (
+            "quantile",
+            "averaged_inverted_cdf",
+            [[0, 0, 0, 0], [0, 1, 1, 1], [1, 2, 2, 2], [1, 2, 2, 2]],
+            [1, 1, 1, 1],
+        ),
+        (
+            "quantile",
+            "averaged_inverted_cdf",
+            [[0, 0, 0, 0], [0, 0, 0, 0], [1, 1, 1, 1], [1, 1, 1, 1]],
+            [0, 1, 3, 1],
+        ),
+        (
+            "quantile",
+            "averaged_inverted_cdf",
+            [[0, 0, 0, 0], [0, 0, 0, 0], [1, 2, 2, 2], [1, 2, 2, 2]],
+            [1, 1, 3, 1],
+        ),
+        (
+            "kmeans",
+            "warn",  # default, will not warn when strategy != "quantile"
+            [[0, 0, 0, 0], [0, 1, 1, 0], [1, 1, 1, 1], [1, 2, 2, 2]],
+            [1, 0, 3, 1],
+        ),
+    ],
+)
+def test_fit_transform_n_bins_array(strategy, quantile_method, expected, sample_weight):
+    est = KBinsDiscretizer(
+        n_bins=[2, 3, 3, 3],
+        encode="ordinal",
+        strategy=strategy,
+        quantile_method=quantile_method,
+    ).fit(X, sample_weight=sample_weight)
+    assert_array_equal(est.transform(X), expected)
 
     # test the shape of bin_edges_
     n_features = np.array(X).shape[1]
-    assert est.bin_edges_.shape == (n_features, )
+    assert est.bin_edges_.shape == (n_features,)
     for bin_edges, n_bins in zip(est.bin_edges_, est.n_bins_):
-        assert bin_edges.shape == (n_bins + 1, )
+        assert bin_edges.shape == (n_bins + 1,)
+
+
+@pytest.mark.filterwarnings("ignore: Bins whose width are too small")
+def test_kbinsdiscretizer_effect_sample_weight():
+    """Check the impact of `sample_weight` one computed quantiles."""
+    X = np.array([[-2], [-1], [1], [3], [500], [1000]])
+    # add a large number of bins such that each sample with a non-null weight
+    # will be used as bin edge
+    est = KBinsDiscretizer(
+        n_bins=10,
+        encode="ordinal",
+        strategy="quantile",
+        quantile_method="averaged_inverted_cdf",
+    )
+    est.fit(X, sample_weight=[1, 1, 1, 1, 0, 0])
+    assert_allclose(est.bin_edges_[0], [-2, -1, 0, 1, 3])
+    assert_allclose(est.transform(X), [[0.0], [1.0], [3.0], [3.0], [3.0], [3.0]])
+
+
+@pytest.mark.parametrize("strategy", ["kmeans", "quantile"])
+def test_kbinsdiscretizer_no_mutating_sample_weight(strategy):
+    """Make sure that `sample_weight` is not changed in place."""
+
+    if strategy == "quantile":
+        est = KBinsDiscretizer(
+            n_bins=3,
+            encode="ordinal",
+            strategy=strategy,
+            quantile_method="averaged_inverted_cdf",
+        )
+    else:
+        est = KBinsDiscretizer(n_bins=3, encode="ordinal", strategy=strategy)
+    sample_weight = np.array([1, 3, 1, 2], dtype=np.float64)
+    sample_weight_copy = np.copy(sample_weight)
+    est.fit(X, sample_weight=sample_weight)
+    assert_allclose(sample_weight, sample_weight_copy)
 
 
-@pytest.mark.parametrize('strategy', ['uniform', 'kmeans', 'quantile'])
+@pytest.mark.parametrize("strategy", ["uniform", "kmeans", "quantile"])
 def test_same_min_max(strategy):
     warnings.simplefilter("always")
-    X = np.array([[1, -2],
-                  [1, -1],
-                  [1, 0],
-                  [1, 1]])
-    est = KBinsDiscretizer(strategy=strategy, n_bins=3, encode='ordinal')
-    assert_warns_message(UserWarning,
-                         "Feature 0 is constant and will be replaced "
-                         "with 0.", est.fit, X)
+    X = np.array([[1, -2], [1, -1], [1, 0], [1, 1]])
+    if strategy == "quantile":
+        est = KBinsDiscretizer(
+            strategy=strategy,
+            n_bins=3,
+            encode="ordinal",
+            quantile_method="averaged_inverted_cdf",
+        )
+    else:
+        est = KBinsDiscretizer(strategy=strategy, n_bins=3, encode="ordinal")
+    warning_message = "Feature 0 is constant and will be replaced with 0."
+    with pytest.warns(UserWarning, match=warning_message):
+        est.fit(X)
     assert est.n_bins_[0] == 1
     # replace the feature with zeros
     Xt = est.transform(X)
@@ -120,115 +269,161 @@ def test_same_min_max(strategy):
 
 def test_transform_1d_behavior():
     X = np.arange(4)
-    est = KBinsDiscretizer(n_bins=2)
+    est = KBinsDiscretizer(n_bins=2, quantile_method="averaged_inverted_cdf")
     with pytest.raises(ValueError):
         est.fit(X)
 
-    est = KBinsDiscretizer(n_bins=2)
+    est = KBinsDiscretizer(n_bins=2, quantile_method="averaged_inverted_cdf")
     est.fit(X.reshape(-1, 1))
     with pytest.raises(ValueError):
         est.transform(X)
 
 
-@pytest.mark.parametrize('i', range(1, 9))
+@pytest.mark.parametrize("i", range(1, 9))
 def test_numeric_stability(i):
-    X_init = np.array([2., 4., 6., 8., 10.]).reshape(-1, 1)
+    X_init = np.array([2.0, 4.0, 6.0, 8.0, 10.0]).reshape(-1, 1)
     Xt_expected = np.array([0, 0, 1, 1, 1]).reshape(-1, 1)
 
     # Test up to discretizing nano units
     X = X_init / 10**i
-    Xt = KBinsDiscretizer(n_bins=2, encode='ordinal').fit_transform(X)
+    Xt = KBinsDiscretizer(
+        n_bins=2, encode="ordinal", quantile_method="averaged_inverted_cdf"
+    ).fit_transform(X)
     assert_array_equal(Xt_expected, Xt)
 
 
-def test_invalid_encode_option():
-    est = KBinsDiscretizer(n_bins=[2, 3, 3, 3], encode='invalid-encode')
-    err_msg = (r"Valid options for 'encode' are "
-               r"\('onehot', 'onehot-dense', 'ordinal'\). "
-               r"Got encode='invalid-encode' instead.")
-    with pytest.raises(ValueError, match=err_msg):
-        est.fit(X)
-
-
 def test_encode_options():
-    est = KBinsDiscretizer(n_bins=[2, 3, 3, 3],
-                           encode='ordinal').fit(X)
+    est = KBinsDiscretizer(
+        n_bins=[2, 3, 3, 3], encode="ordinal", quantile_method="averaged_inverted_cdf"
+    ).fit(X)
     Xt_1 = est.transform(X)
-    est = KBinsDiscretizer(n_bins=[2, 3, 3, 3],
-                           encode='onehot-dense').fit(X)
+    est = KBinsDiscretizer(
+        n_bins=[2, 3, 3, 3],
+        encode="onehot-dense",
+        quantile_method="averaged_inverted_cdf",
+    ).fit(X)
     Xt_2 = est.transform(X)
     assert not sp.issparse(Xt_2)
-    assert_array_equal(OneHotEncoder(
-                           categories=[np.arange(i) for i in [2, 3, 3, 3]],
-                           sparse=False)
-                       .fit_transform(Xt_1), Xt_2)
-    est = KBinsDiscretizer(n_bins=[2, 3, 3, 3],
-                           encode='onehot').fit(X)
+    assert_array_equal(
+        OneHotEncoder(
+            categories=[np.arange(i) for i in [2, 3, 3, 3]], sparse_output=False
+        ).fit_transform(Xt_1),
+        Xt_2,
+    )
+    est = KBinsDiscretizer(
+        n_bins=[2, 3, 3, 3], encode="onehot", quantile_method="averaged_inverted_cdf"
+    ).fit(X)
     Xt_3 = est.transform(X)
     assert sp.issparse(Xt_3)
-    assert_array_equal(OneHotEncoder(
-                           categories=[np.arange(i) for i in [2, 3, 3, 3]],
-                           sparse=True)
-                       .fit_transform(Xt_1).toarray(),
-                       Xt_3.toarray())
-
-
-def test_invalid_strategy_option():
-    est = KBinsDiscretizer(n_bins=[2, 3, 3, 3], strategy='invalid-strategy')
-    err_msg = (r"Valid options for 'strategy' are "
-               r"\('uniform', 'quantile', 'kmeans'\). "
-               r"Got strategy='invalid-strategy' instead.")
-    with pytest.raises(ValueError, match=err_msg):
-        est.fit(X)
+    assert_array_equal(
+        OneHotEncoder(
+            categories=[np.arange(i) for i in [2, 3, 3, 3]], sparse_output=True
+        )
+        .fit_transform(Xt_1)
+        .toarray(),
+        Xt_3.toarray(),
+    )
 
 
 @pytest.mark.parametrize(
-    'strategy, expected_2bins, expected_3bins, expected_5bins',
-    [('uniform', [0, 0, 0, 0, 1, 1], [0, 0, 0, 0, 2, 2], [0, 0, 1, 1, 4, 4]),
-     ('kmeans', [0, 0, 0, 0, 1, 1], [0, 0, 1, 1, 2, 2], [0, 0, 1, 2, 3, 4]),
-     ('quantile', [0, 0, 0, 1, 1, 1], [0, 0, 1, 1, 2, 2], [0, 1, 2, 3, 4, 4])])
+    "strategy, quantile_method, expected_2bins, expected_3bins, expected_5bins",
+    [
+        ("uniform", "warn", [0, 0, 0, 0, 1, 1], [0, 0, 0, 0, 2, 2], [0, 0, 1, 1, 4, 4]),
+        ("kmeans", "warn", [0, 0, 0, 0, 1, 1], [0, 0, 1, 1, 2, 2], [0, 0, 1, 2, 3, 4]),
+        (
+            "quantile",
+            "averaged_inverted_cdf",
+            [0, 0, 0, 1, 1, 1],
+            [0, 0, 1, 1, 2, 2],
+            [0, 1, 2, 3, 4, 4],
+        ),
+    ],
+)
 def test_nonuniform_strategies(
-        strategy, expected_2bins, expected_3bins, expected_5bins):
+    strategy, quantile_method, expected_2bins, expected_3bins, expected_5bins
+):
     X = np.array([0, 0.5, 2, 3, 9, 10]).reshape(-1, 1)
 
     # with 2 bins
-    est = KBinsDiscretizer(n_bins=2, strategy=strategy, encode='ordinal')
+    est = KBinsDiscretizer(
+        n_bins=2, strategy=strategy, quantile_method=quantile_method, encode="ordinal"
+    )
     Xt = est.fit_transform(X)
     assert_array_equal(expected_2bins, Xt.ravel())
 
     # with 3 bins
-    est = KBinsDiscretizer(n_bins=3, strategy=strategy, encode='ordinal')
+    est = KBinsDiscretizer(
+        n_bins=3, strategy=strategy, quantile_method=quantile_method, encode="ordinal"
+    )
     Xt = est.fit_transform(X)
     assert_array_equal(expected_3bins, Xt.ravel())
 
     # with 5 bins
-    est = KBinsDiscretizer(n_bins=5, strategy=strategy, encode='ordinal')
+    est = KBinsDiscretizer(
+        n_bins=5, strategy=strategy, quantile_method=quantile_method, encode="ordinal"
+    )
     Xt = est.fit_transform(X)
     assert_array_equal(expected_5bins, Xt.ravel())
 
 
 @pytest.mark.parametrize(
-    'strategy, expected_inv',
-    [('uniform', [[-1.5, 2., -3.5, -0.5], [-0.5, 3., -2.5, -0.5],
-                  [0.5, 4., -1.5, 0.5], [0.5, 4., -1.5, 1.5]]),
-     ('kmeans', [[-1.375, 2.125, -3.375, -0.5625],
-                 [-1.375, 2.125, -3.375, -0.5625],
-                 [-0.125, 3.375, -2.125, 0.5625],
-                 [0.75, 4.25, -1.25, 1.625]]),
-     ('quantile', [[-1.5, 2., -3.5, -0.75], [-0.5, 3., -2.5, 0.],
-                   [0.5, 4., -1.5, 1.25], [0.5, 4., -1.5, 1.25]])])
-@pytest.mark.parametrize('encode', ['ordinal', 'onehot', 'onehot-dense'])
-def test_inverse_transform(strategy, encode, expected_inv):
-    kbd = KBinsDiscretizer(n_bins=3, strategy=strategy, encode=encode)
+    "strategy, expected_inv,quantile_method",
+    [
+        (
+            "uniform",
+            [
+                [-1.5, 2.0, -3.5, -0.5],
+                [-0.5, 3.0, -2.5, -0.5],
+                [0.5, 4.0, -1.5, 0.5],
+                [0.5, 4.0, -1.5, 1.5],
+            ],
+            "warn",  # default, will not warn when strategy != "quantile"
+        ),
+        (
+            "kmeans",
+            [
+                [-1.375, 2.125, -3.375, -0.5625],
+                [-1.375, 2.125, -3.375, -0.5625],
+                [-0.125, 3.375, -2.125, 0.5625],
+                [0.75, 4.25, -1.25, 1.625],
+            ],
+            "warn",  # default, will not warn when strategy != "quantile"
+        ),
+        (
+            "quantile",
+            [
+                [-1.5, 2.0, -3.5, -0.75],
+                [-0.5, 3.0, -2.5, 0.0],
+                [0.5, 4.0, -1.5, 1.25],
+                [0.5, 4.0, -1.5, 1.25],
+            ],
+            "averaged_inverted_cdf",
+        ),
+    ],
+)
+@pytest.mark.parametrize("encode", ["ordinal", "onehot", "onehot-dense"])
+def test_inverse_transform(strategy, encode, expected_inv, quantile_method):
+    kbd = KBinsDiscretizer(
+        n_bins=3, strategy=strategy, quantile_method=quantile_method, encode=encode
+    )
     Xt = kbd.fit_transform(X)
     Xinv = kbd.inverse_transform(Xt)
     assert_array_almost_equal(expected_inv, Xinv)
 
 
-@pytest.mark.parametrize('strategy', ['uniform', 'kmeans', 'quantile'])
+@pytest.mark.parametrize("strategy", ["uniform", "kmeans", "quantile"])
 def test_transform_outside_fit_range(strategy):
     X = np.array([0, 1, 2, 3])[:, None]
-    kbd = KBinsDiscretizer(n_bins=4, strategy=strategy, encode='ordinal')
+
+    if strategy == "quantile":
+        kbd = KBinsDiscretizer(
+            n_bins=4,
+            strategy=strategy,
+            encode="ordinal",
+            quantile_method="averaged_inverted_cdf",
+        )
+    else:
+        kbd = KBinsDiscretizer(n_bins=4, strategy=strategy, encode="ordinal")
     kbd.fit(X)
 
     X2 = np.array([-2, 5])[:, None]
@@ -241,7 +436,9 @@ def test_overwrite():
     X = np.array([0, 1, 2, 3])[:, None]
     X_before = X.copy()
 
-    est = KBinsDiscretizer(n_bins=3, encode="ordinal")
+    est = KBinsDiscretizer(
+        n_bins=3, quantile_method="averaged_inverted_cdf", encode="ordinal"
+    )
     Xt = est.fit_transform(X)
     assert_array_equal(X, X_before)
 
@@ -252,14 +449,21 @@ def test_overwrite():
 
 
 @pytest.mark.parametrize(
-    'strategy, expected_bin_edges',
-    [('quantile', [0, 1, 3]), ('kmeans', [0, 1.5, 3])])
-def test_redundant_bins(strategy, expected_bin_edges):
+    "strategy, expected_bin_edges, quantile_method",
+    [
+        ("quantile", [0, 1.5, 3], "averaged_inverted_cdf"),
+        ("kmeans", [0, 1.5, 3], "warn"),
+    ],
+)
+def test_redundant_bins(strategy, expected_bin_edges, quantile_method):
     X = [[0], [0], [0], [0], [3], [3]]
-    kbd = KBinsDiscretizer(n_bins=3, strategy=strategy)
-    msg = ("Bins whose width are too small (i.e., <= 1e-8) in feature 0 "
-           "are removed. Consider decreasing the number of bins.")
-    assert_warns_message(UserWarning, msg, kbd.fit, X)
+    kbd = KBinsDiscretizer(
+        n_bins=3, strategy=strategy, quantile_method=quantile_method, subsample=None
+    )
+    warning_message = "Consider decreasing the number of bins."
+    with pytest.warns(UserWarning, match=warning_message):
+        kbd.fit(X)
+
     assert_array_almost_equal(kbd.bin_edges_[0], expected_bin_edges)
 
 
@@ -267,57 +471,195 @@ def test_percentile_numeric_stability():
     X = np.array([0.05, 0.05, 0.95]).reshape(-1, 1)
     bin_edges = np.array([0.05, 0.23, 0.41, 0.59, 0.77, 0.95])
     Xt = np.array([0, 0, 4]).reshape(-1, 1)
-    kbd = KBinsDiscretizer(n_bins=10, encode='ordinal',
-                           strategy='quantile')
-    msg = ("Bins whose width are too small (i.e., <= 1e-8) in feature 0 "
-           "are removed. Consider decreasing the number of bins.")
-    assert_warns_message(UserWarning, msg, kbd.fit, X)
+    kbd = KBinsDiscretizer(
+        n_bins=10,
+        encode="ordinal",
+        strategy="quantile",
+        quantile_method="linear",
+    )
+    ## TODO: change to averaged inverted cdf, but that means we only get bin
+    ## edges of 0.05 and 0.95 and nothing in between
+
+    warning_message = "Consider decreasing the number of bins."
+    with pytest.warns(UserWarning, match=warning_message):
+        kbd.fit(X)
+
     assert_array_almost_equal(kbd.bin_edges_[0], bin_edges)
     assert_array_almost_equal(kbd.transform(X), Xt)
 
 
 @pytest.mark.parametrize("in_dtype", [np.float16, np.float32, np.float64])
-@pytest.mark.parametrize("out_dtype", [None, np.float16, np.float32,
-                                       np.float64])
-@pytest.mark.parametrize('encode', ['ordinal', 'onehot', 'onehot-dense'])
+@pytest.mark.parametrize("out_dtype", [None, np.float32, np.float64])
+@pytest.mark.parametrize("encode", ["ordinal", "onehot", "onehot-dense"])
 def test_consistent_dtype(in_dtype, out_dtype, encode):
     X_input = np.array(X, dtype=in_dtype)
-    kbd = KBinsDiscretizer(n_bins=3, encode=encode, dtype=out_dtype)
-
-    # a error is raised if a wrong dtype is define for the model
-    if out_dtype not in [None, np.float32, np.float64]:
-        with pytest.raises(ValueError, match="Valid options for 'dtype' are"):
-            kbd.fit(X_input)
+    kbd = KBinsDiscretizer(
+        n_bins=3,
+        encode=encode,
+        quantile_method="averaged_inverted_cdf",
+        dtype=out_dtype,
+    )
+    kbd.fit(X_input)
+
+    # test output dtype
+    if out_dtype is not None:
+        expected_dtype = out_dtype
+    elif out_dtype is None and X_input.dtype == np.float16:
+        # wrong numeric input dtype are cast in np.float64
+        expected_dtype = np.float64
     else:
-        kbd.fit(X_input)
-
-        # test output dtype
-        if out_dtype is not None:
-            expected_dtype = out_dtype
-        elif out_dtype is None and X_input.dtype == np.float16:
-            # wrong numeric input dtype are cast in np.float64
-            expected_dtype = np.float64
-        else:
-            expected_dtype = X_input.dtype
-        Xt = kbd.transform(X_input)
-        assert Xt.dtype == expected_dtype
-
-
-@pytest.mark.parametrize('input_dtype', [np.float16, np.float32, np.float64])
-@pytest.mark.parametrize('encode', ['ordinal', 'onehot', 'onehot-dense'])
+        expected_dtype = X_input.dtype
+    Xt = kbd.transform(X_input)
+    assert Xt.dtype == expected_dtype
+
+
+@pytest.mark.parametrize("input_dtype", [np.float16, np.float32, np.float64])
+@pytest.mark.parametrize("encode", ["ordinal", "onehot", "onehot-dense"])
 def test_32_equal_64(input_dtype, encode):
     # TODO this check is redundant with common checks and can be removed
     #  once #16290 is merged
     X_input = np.array(X, dtype=input_dtype)
 
     # 32 bit output
-    kbd_32 = KBinsDiscretizer(n_bins=3, encode=encode, dtype=np.float32)
+    kbd_32 = KBinsDiscretizer(
+        n_bins=3,
+        encode=encode,
+        quantile_method="averaged_inverted_cdf",
+        dtype=np.float32,
+    )
     kbd_32.fit(X_input)
     Xt_32 = kbd_32.transform(X_input)
 
     # 64 bit output
-    kbd_64 = KBinsDiscretizer(n_bins=3, encode=encode, dtype=np.float64)
+    kbd_64 = KBinsDiscretizer(
+        n_bins=3,
+        encode=encode,
+        quantile_method="averaged_inverted_cdf",
+        dtype=np.float64,
+    )
     kbd_64.fit(X_input)
     Xt_64 = kbd_64.transform(X_input)
 
     assert_allclose_dense_sparse(Xt_32, Xt_64)
+
+
+def test_kbinsdiscretizer_subsample_default():
+    # Since the size of X is small (< 2e5), subsampling will not take place.
+    X = np.array([-2, 1.5, -4, -1]).reshape(-1, 1)
+    kbd_default = KBinsDiscretizer(
+        n_bins=10,
+        encode="ordinal",
+        strategy="quantile",
+        quantile_method="averaged_inverted_cdf",
+    )
+    kbd_default.fit(X)
+
+    kbd_without_subsampling = clone(kbd_default)
+    kbd_without_subsampling.set_params(subsample=None)
+    kbd_without_subsampling.fit(X)
+
+    for bin_kbd_default, bin_kbd_with_subsampling in zip(
+        kbd_default.bin_edges_[0], kbd_without_subsampling.bin_edges_[0]
+    ):
+        np.testing.assert_allclose(bin_kbd_default, bin_kbd_with_subsampling)
+    assert kbd_default.bin_edges_.shape == kbd_without_subsampling.bin_edges_.shape
+
+
+@pytest.mark.parametrize(
+    "encode, expected_names",
+    [
+        (
+            "onehot",
+            [
+                f"feat{col_id}_{float(bin_id)}"
+                for col_id in range(3)
+                for bin_id in range(4)
+            ],
+        ),
+        (
+            "onehot-dense",
+            [
+                f"feat{col_id}_{float(bin_id)}"
+                for col_id in range(3)
+                for bin_id in range(4)
+            ],
+        ),
+        ("ordinal", [f"feat{col_id}" for col_id in range(3)]),
+    ],
+)
+def test_kbinsdiscrtizer_get_feature_names_out(encode, expected_names):
+    """Check get_feature_names_out for different settings.
+    Non-regression test for #22731
+    """
+    X = [[-2, 1, -4], [-1, 2, -3], [0, 3, -2], [1, 4, -1]]
+
+    kbd = KBinsDiscretizer(
+        n_bins=4, encode=encode, quantile_method="averaged_inverted_cdf"
+    ).fit(X)
+    Xt = kbd.transform(X)
+
+    input_features = [f"feat{i}" for i in range(3)]
+    output_names = kbd.get_feature_names_out(input_features)
+    assert Xt.shape[1] == output_names.shape[0]
+
+    assert_array_equal(output_names, expected_names)
+
+
+@pytest.mark.parametrize("strategy", ["uniform", "kmeans", "quantile"])
+def test_kbinsdiscretizer_subsample(strategy, global_random_seed):
+    # Check that the bin edges are almost the same when subsampling is used.
+    X = np.random.RandomState(global_random_seed).random_sample((100000, 1)) + 1
+
+    if strategy == "quantile":
+        kbd_subsampling = KBinsDiscretizer(
+            strategy=strategy,
+            subsample=50000,
+            random_state=global_random_seed,
+            quantile_method="averaged_inverted_cdf",
+        )
+    else:
+        kbd_subsampling = KBinsDiscretizer(
+            strategy=strategy, subsample=50000, random_state=global_random_seed
+        )
+    kbd_subsampling.fit(X)
+
+    kbd_no_subsampling = clone(kbd_subsampling)
+    kbd_no_subsampling.set_params(subsample=None)
+    kbd_no_subsampling.fit(X)
+
+    # We use a large tolerance because we can't expect the bin edges to be exactly the
+    # same when subsampling is used.
+    assert_allclose(
+        kbd_subsampling.bin_edges_[0], kbd_no_subsampling.bin_edges_[0], rtol=1e-2
+    )
+
+
+def test_quantile_method_future_warnings():
+    X = [[-2, 1, -4], [-1, 2, -3], [0, 3, -2], [1, 4, -1]]
+    with pytest.warns(
+        FutureWarning,
+        match="The current default behavior, quantile_method='linear', will be "
+        "changed to quantile_method='averaged_inverted_cdf' in "
+        "scikit-learn version 1.9 to naturally support sample weight "
+        "equivalence properties by default. Pass "
+        "quantile_method='averaged_inverted_cdf' explicitly to silence this "
+        "warning.",
+    ):
+        KBinsDiscretizer(strategy="quantile").fit(X)
+
+
+def test_invalid_quantile_method_with_sample_weight():
+    X = [[-2, 1, -4], [-1, 2, -3], [0, 3, -2], [1, 4, -1]]
+    expected_msg = (
+        "When fitting with strategy='quantile' and sample weights, "
+        "quantile_method should either be set to 'averaged_inverted_cdf' or "
+        "'inverted_cdf', got quantile_method='linear' instead."
+    )
+    with pytest.raises(
+        ValueError,
+        match=expected_msg,
+    ):
+        KBinsDiscretizer(strategy="quantile", quantile_method="linear").fit(
+            X,
+            sample_weight=[1, 1, 2, 2],
+        )
diff --git a/sklearn/preprocessing/tests/test_encoders.py b/sklearn/preprocessing/tests/test_encoders.py
index b1eff0cad21e0..dc7bbd2ec03b6 100644
--- a/sklearn/preprocessing/tests/test_encoders.py
+++ b/sklearn/preprocessing/tests/test_encoders.py
@@ -1,19 +1,19 @@
-# -*- coding: utf-8 -*-
-
 import re
+import warnings
 
 import numpy as np
-from scipy import sparse
 import pytest
+from scipy import sparse
 
 from sklearn.exceptions import NotFittedError
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils._testing import assert_allclose
-from sklearn.utils._testing import _convert_container
-from sklearn.utils import is_scalar_nan
-
-from sklearn.preprocessing import OneHotEncoder
-from sklearn.preprocessing import OrdinalEncoder
+from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder
+from sklearn.utils._missing import is_scalar_nan
+from sklearn.utils._testing import (
+    _convert_container,
+    assert_allclose,
+    assert_array_equal,
+)
+from sklearn.utils.fixes import CSR_CONTAINERS
 
 
 def test_one_hot_encoder_sparse_dense():
@@ -21,7 +21,7 @@ def test_one_hot_encoder_sparse_dense():
 
     X = np.array([[3, 2, 1], [0, 1, 1]])
     enc_sparse = OneHotEncoder()
-    enc_dense = OneHotEncoder(sparse=False)
+    enc_dense = OneHotEncoder(sparse_output=False)
 
     X_trans_sparse = enc_sparse.fit_transform(X)
     X_trans_dense = enc_dense.fit_transform(X)
@@ -33,72 +33,51 @@ def test_one_hot_encoder_sparse_dense():
     assert not sparse.issparse(X_trans_dense)
 
     # check outcome
-    assert_array_equal(X_trans_sparse.toarray(), [[0., 1., 0., 1., 1.],
-                                                  [1., 0., 1., 0., 1.]])
+    assert_array_equal(
+        X_trans_sparse.toarray(), [[0.0, 1.0, 0.0, 1.0, 1.0], [1.0, 0.0, 1.0, 0.0, 1.0]]
+    )
     assert_array_equal(X_trans_sparse.toarray(), X_trans_dense)
 
 
-def test_one_hot_encoder_diff_n_features():
-    X = np.array([[0, 2, 1], [1, 0, 3], [1, 0, 2]])
-    X2 = np.array([[1, 0]])
-    enc = OneHotEncoder()
-    enc.fit(X)
-    err_msg = ("The number of features in X is different to the number of "
-               "features of the fitted data.")
-    with pytest.raises(ValueError, match=err_msg):
-        enc.transform(X2)
-
-
-def test_one_hot_encoder_handle_unknown():
+@pytest.mark.parametrize("handle_unknown", ["ignore", "infrequent_if_exist", "warn"])
+def test_one_hot_encoder_handle_unknown(handle_unknown):
     X = np.array([[0, 2, 1], [1, 0, 3], [1, 0, 2]])
     X2 = np.array([[4, 1, 1]])
 
     # Test that one hot encoder raises error for unknown features
     # present during transform.
-    oh = OneHotEncoder(handle_unknown='error')
+    oh = OneHotEncoder(handle_unknown="error")
     oh.fit(X)
-    with pytest.raises(ValueError, match='Found unknown categories'):
+    with pytest.raises(ValueError, match="Found unknown categories"):
         oh.transform(X2)
 
     # Test the ignore option, ignores unknown features (giving all 0's)
-    oh = OneHotEncoder(handle_unknown='ignore')
+    oh = OneHotEncoder(handle_unknown=handle_unknown)
     oh.fit(X)
     X2_passed = X2.copy()
     assert_array_equal(
         oh.transform(X2_passed).toarray(),
-        np.array([[0.,  0.,  0.,  0.,  1.,  0.,  0.]]))
+        np.array([[0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0]]),
+    )
     # ensure transformed data was not modified in place
     assert_allclose(X2, X2_passed)
 
-    # Raise error if handle_unknown is neither ignore or error.
-    oh = OneHotEncoder(handle_unknown='42')
-    with pytest.raises(ValueError, match='handle_unknown should be either'):
-        oh.fit(X)
-
 
-def test_one_hot_encoder_not_fitted():
-    X = np.array([['a'], ['b']])
-    enc = OneHotEncoder(categories=['a', 'b'])
-    msg = ("This OneHotEncoder instance is not fitted yet. "
-           "Call 'fit' with appropriate arguments before using this "
-           "estimator.")
-    with pytest.raises(NotFittedError, match=msg):
-        enc.transform(X)
-
-
-def test_one_hot_encoder_handle_unknown_strings():
-    X = np.array(['11111111', '22', '333', '4444']).reshape((-1, 1))
-    X2 = np.array(['55555', '22']).reshape((-1, 1))
+@pytest.mark.parametrize("handle_unknown", ["ignore", "infrequent_if_exist", "warn"])
+def test_one_hot_encoder_handle_unknown_strings(handle_unknown):
+    X = np.array(["11111111", "22", "333", "4444"]).reshape((-1, 1))
+    X2 = np.array(["55555", "22"]).reshape((-1, 1))
     # Non Regression test for the issue #12470
     # Test the ignore option, when categories are numpy string dtype
     # particularly when the known category strings are larger
     # than the unknown category strings
-    oh = OneHotEncoder(handle_unknown='ignore')
+    oh = OneHotEncoder(handle_unknown=handle_unknown)
     oh.fit(X)
     X2_passed = X2.copy()
     assert_array_equal(
         oh.transform(X2_passed).toarray(),
-        np.array([[0.,  0.,  0.,  0.], [0.,  1.,  0.,  0.]]))
+        np.array([[0.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0]]),
+    )
     # ensure transformed data was not modified in place
     assert_array_equal(X2, X2_passed)
 
@@ -109,70 +88,125 @@ def test_one_hot_encoder_dtype(input_dtype, output_dtype):
     X = np.asarray([[0, 1]], dtype=input_dtype).T
     X_expected = np.asarray([[1, 0], [0, 1]], dtype=output_dtype)
 
-    oh = OneHotEncoder(categories='auto', dtype=output_dtype)
+    oh = OneHotEncoder(categories="auto", dtype=output_dtype)
     assert_array_equal(oh.fit_transform(X).toarray(), X_expected)
     assert_array_equal(oh.fit(X).transform(X).toarray(), X_expected)
 
-    oh = OneHotEncoder(categories='auto', dtype=output_dtype, sparse=False)
+    oh = OneHotEncoder(categories="auto", dtype=output_dtype, sparse_output=False)
     assert_array_equal(oh.fit_transform(X), X_expected)
     assert_array_equal(oh.fit(X).transform(X), X_expected)
 
 
 @pytest.mark.parametrize("output_dtype", [np.int32, np.float32, np.float64])
 def test_one_hot_encoder_dtype_pandas(output_dtype):
-    pd = pytest.importorskip('pandas')
+    pd = pytest.importorskip("pandas")
 
-    X_df = pd.DataFrame({'A': ['a', 'b'], 'B': [1, 2]})
+    X_df = pd.DataFrame({"A": ["a", "b"], "B": [1, 2]})
     X_expected = np.array([[1, 0, 1, 0], [0, 1, 0, 1]], dtype=output_dtype)
 
     oh = OneHotEncoder(dtype=output_dtype)
     assert_array_equal(oh.fit_transform(X_df).toarray(), X_expected)
     assert_array_equal(oh.fit(X_df).transform(X_df).toarray(), X_expected)
 
-    oh = OneHotEncoder(dtype=output_dtype, sparse=False)
+    oh = OneHotEncoder(dtype=output_dtype, sparse_output=False)
     assert_array_equal(oh.fit_transform(X_df), X_expected)
     assert_array_equal(oh.fit(X_df).transform(X_df), X_expected)
 
 
 def test_one_hot_encoder_feature_names():
     enc = OneHotEncoder()
-    X = [['Male', 1, 'girl', 2, 3],
-         ['Female', 41, 'girl', 1, 10],
-         ['Male', 51, 'boy', 12, 3],
-         ['Male', 91, 'girl', 21, 30]]
+    X = [
+        ["Male", 1, "girl", 2, 3],
+        ["Female", 41, "girl", 1, 10],
+        ["Male", 51, "boy", 12, 3],
+        ["Male", 91, "girl", 21, 30],
+    ]
 
     enc.fit(X)
-    feature_names = enc.get_feature_names()
-    assert isinstance(feature_names, np.ndarray)
+    feature_names = enc.get_feature_names_out()
 
-    assert_array_equal(['x0_Female', 'x0_Male',
-                        'x1_1', 'x1_41', 'x1_51', 'x1_91',
-                        'x2_boy', 'x2_girl',
-                        'x3_1', 'x3_2', 'x3_12', 'x3_21',
-                        'x4_3',
-                        'x4_10', 'x4_30'], feature_names)
+    assert_array_equal(
+        [
+            "x0_Female",
+            "x0_Male",
+            "x1_1",
+            "x1_41",
+            "x1_51",
+            "x1_91",
+            "x2_boy",
+            "x2_girl",
+            "x3_1",
+            "x3_2",
+            "x3_12",
+            "x3_21",
+            "x4_3",
+            "x4_10",
+            "x4_30",
+        ],
+        feature_names,
+    )
 
-    feature_names2 = enc.get_feature_names(['one', 'two',
-                                            'three', 'four', 'five'])
+    feature_names2 = enc.get_feature_names_out(["one", "two", "three", "four", "five"])
 
-    assert_array_equal(['one_Female', 'one_Male',
-                        'two_1', 'two_41', 'two_51', 'two_91',
-                        'three_boy', 'three_girl',
-                        'four_1', 'four_2', 'four_12', 'four_21',
-                        'five_3', 'five_10', 'five_30'], feature_names2)
+    assert_array_equal(
+        [
+            "one_Female",
+            "one_Male",
+            "two_1",
+            "two_41",
+            "two_51",
+            "two_91",
+            "three_boy",
+            "three_girl",
+            "four_1",
+            "four_2",
+            "four_12",
+            "four_21",
+            "five_3",
+            "five_10",
+            "five_30",
+        ],
+        feature_names2,
+    )
 
     with pytest.raises(ValueError, match="input_features should have length"):
-        enc.get_feature_names(['one', 'two'])
+        enc.get_feature_names_out(["one", "two"])
 
 
 def test_one_hot_encoder_feature_names_unicode():
     enc = OneHotEncoder()
-    X = np.array([['c❤t1', 'dat2']], dtype=object).T
+    X = np.array([["c❤t1", "dat2"]], dtype=object).T
     enc.fit(X)
-    feature_names = enc.get_feature_names()
-    assert_array_equal(['x0_c❤t1', 'x0_dat2'], feature_names)
-    feature_names = enc.get_feature_names(input_features=['n👍me'])
-    assert_array_equal(['n👍me_c❤t1', 'n👍me_dat2'], feature_names)
+    feature_names = enc.get_feature_names_out()
+    assert_array_equal(["x0_c❤t1", "x0_dat2"], feature_names)
+    feature_names = enc.get_feature_names_out(input_features=["n👍me"])
+    assert_array_equal(["n👍me_c❤t1", "n👍me_dat2"], feature_names)
+
+
+def test_one_hot_encoder_custom_feature_name_combiner():
+    """Check the behaviour of `feature_name_combiner` as a callable."""
+
+    def name_combiner(feature, category):
+        return feature + "_" + repr(category)
+
+    enc = OneHotEncoder(feature_name_combiner=name_combiner)
+    X = np.array([["None", None]], dtype=object).T
+    enc.fit(X)
+    feature_names = enc.get_feature_names_out()
+    assert_array_equal(["x0_'None'", "x0_None"], feature_names)
+    feature_names = enc.get_feature_names_out(input_features=["a"])
+    assert_array_equal(["a_'None'", "a_None"], feature_names)
+
+    def wrong_combiner(feature, category):
+        # we should be returning a Python string
+        return 0
+
+    enc = OneHotEncoder(feature_name_combiner=wrong_combiner).fit(X)
+    err_msg = (
+        "When `feature_name_combiner` is a callable, it should return a Python string."
+    )
+    with pytest.raises(TypeError, match=err_msg):
+        enc.get_feature_names_out()
 
 
 def test_one_hot_encoder_set_params():
@@ -180,7 +214,7 @@ def test_one_hot_encoder_set_params():
     oh = OneHotEncoder()
     # set params on not yet fitted object
     oh.set_params(categories=[[0, 1, 2, 3]])
-    assert oh.get_params()['categories'] == [[0, 1, 2, 3]]
+    assert oh.get_params()["categories"] == [[0, 1, 2, 3]]
     assert oh.fit_transform(X).toarray().shape == (2, 4)
     # set params on already fitted object
     oh.set_params(categories=[[0, 1, 2, 3, 4]])
@@ -188,29 +222,41 @@ def test_one_hot_encoder_set_params():
 
 
 def check_categorical_onehot(X):
-    enc = OneHotEncoder(categories='auto')
+    enc = OneHotEncoder(categories="auto")
     Xtr1 = enc.fit_transform(X)
 
-    enc = OneHotEncoder(categories='auto', sparse=False)
+    enc = OneHotEncoder(categories="auto", sparse_output=False)
     Xtr2 = enc.fit_transform(X)
 
     assert_allclose(Xtr1.toarray(), Xtr2)
 
-    assert sparse.isspmatrix_csr(Xtr1)
+    assert sparse.issparse(Xtr1) and Xtr1.format == "csr"
     return Xtr1.toarray()
 
 
-@pytest.mark.parametrize("X", [
-    [['def', 1, 55], ['abc', 2, 55]],
-    np.array([[10, 1, 55], [5, 2, 55]]),
-    np.array([['b', 'A', 'cat'], ['a', 'B', 'cat']], dtype=object),
-    np.array([['b', 1, 'cat'], ['a', np.nan, 'cat']], dtype=object),
-    np.array([['b', 1, 'cat'], ['a', float('nan'), 'cat']], dtype=object),
-    np.array([[None, 1, 'cat'], ['a', 2, 'cat']], dtype=object),
-    np.array([[None, 1, None], ['a', np.nan, None]], dtype=object),
-    np.array([[None, 1, None], ['a', float('nan'), None]], dtype=object),
-    ], ids=['mixed', 'numeric', 'object', 'mixed-nan', 'mixed-float-nan',
-            'mixed-None', 'mixed-None-nan', 'mixed-None-float-nan'])
+@pytest.mark.parametrize(
+    "X",
+    [
+        [["def", 1, 55], ["abc", 2, 55]],
+        np.array([[10, 1, 55], [5, 2, 55]]),
+        np.array([["b", "A", "cat"], ["a", "B", "cat"]], dtype=object),
+        np.array([["b", 1, "cat"], ["a", np.nan, "cat"]], dtype=object),
+        np.array([["b", 1, "cat"], ["a", float("nan"), "cat"]], dtype=object),
+        np.array([[None, 1, "cat"], ["a", 2, "cat"]], dtype=object),
+        np.array([[None, 1, None], ["a", np.nan, None]], dtype=object),
+        np.array([[None, 1, None], ["a", float("nan"), None]], dtype=object),
+    ],
+    ids=[
+        "mixed",
+        "numeric",
+        "object",
+        "mixed-nan",
+        "mixed-float-nan",
+        "mixed-None",
+        "mixed-None-nan",
+        "mixed-None-float-nan",
+    ],
+)
 def test_one_hot_encoder(X):
     Xtr = check_categorical_onehot(np.array(X)[:, [0]])
     assert_allclose(Xtr, [[0, 1], [1, 0]])
@@ -218,22 +264,22 @@ def test_one_hot_encoder(X):
     Xtr = check_categorical_onehot(np.array(X)[:, [0, 1]])
     assert_allclose(Xtr, [[0, 1, 1, 0], [1, 0, 0, 1]])
 
-    Xtr = OneHotEncoder(categories='auto').fit_transform(X)
-    assert_allclose(Xtr.toarray(), [[0, 1, 1, 0,  1], [1, 0, 0, 1, 1]])
+    Xtr = OneHotEncoder(categories="auto").fit_transform(X)
+    assert_allclose(Xtr.toarray(), [[0, 1, 1, 0, 1], [1, 0, 0, 1, 1]])
 
 
-@pytest.mark.parametrize('sparse_', [False, True])
-@pytest.mark.parametrize('drop', [None, 'first'])
-def test_one_hot_encoder_inverse(sparse_, drop):
-    X = [['abc', 2, 55], ['def', 1, 55], ['abc', 3, 55]]
-    enc = OneHotEncoder(sparse=sparse_, drop=drop)
+@pytest.mark.parametrize("handle_unknown", ["ignore", "infrequent_if_exist", "warn"])
+@pytest.mark.parametrize("sparse_", [False, True])
+@pytest.mark.parametrize("drop", [None, "first"])
+def test_one_hot_encoder_inverse(handle_unknown, sparse_, drop):
+    X = [["abc", 2, 55], ["def", 1, 55], ["abc", 3, 55]]
+    enc = OneHotEncoder(sparse_output=sparse_, drop=drop)
     X_tr = enc.fit_transform(X)
     exp = np.array(X, dtype=object)
     assert_array_equal(enc.inverse_transform(X_tr), exp)
 
     X = [[2, 55], [1, 55], [3, 55]]
-    enc = OneHotEncoder(sparse=sparse_, categories='auto',
-                        drop=drop)
+    enc = OneHotEncoder(sparse_output=sparse_, categories="auto", drop=drop)
     X_tr = enc.fit_transform(X)
     exp = np.array(X)
     assert_array_equal(enc.inverse_transform(X_tr), exp)
@@ -241,10 +287,12 @@ def test_one_hot_encoder_inverse(sparse_, drop):
     if drop is None:
         # with unknown categories
         # drop is incompatible with handle_unknown=ignore
-        X = [['abc', 2, 55], ['def', 1, 55], ['abc', 3, 55]]
-        enc = OneHotEncoder(sparse=sparse_, handle_unknown='ignore',
-                            categories=[['abc', 'def'], [1, 2],
-                                        [54, 55, 56]])
+        X = [["abc", 2, 55], ["def", 1, 55], ["abc", 3, 55]]
+        enc = OneHotEncoder(
+            sparse_output=sparse_,
+            handle_unknown=handle_unknown,
+            categories=[["abc", "def"], [1, 2], [54, 55, 56]],
+        )
         X_tr = enc.fit_transform(X)
         exp = np.array(X, dtype=object)
         exp[2, 1] = None
@@ -252,8 +300,11 @@ def test_one_hot_encoder_inverse(sparse_, drop):
 
         # with an otherwise numerical output, still object if unknown
         X = [[2, 55], [1, 55], [3, 55]]
-        enc = OneHotEncoder(sparse=sparse_, categories=[[1, 2], [54, 56]],
-                            handle_unknown='ignore')
+        enc = OneHotEncoder(
+            sparse_output=sparse_,
+            categories=[[1, 2], [54, 56]],
+            handle_unknown=handle_unknown,
+        )
         X_tr = enc.fit_transform(X)
         exp = np.array(X, dtype=object)
         exp[2, 0] = None
@@ -262,19 +313,21 @@ def test_one_hot_encoder_inverse(sparse_, drop):
 
     # incorrect shape raises
     X_tr = np.array([[0, 1, 1], [1, 0, 1]])
-    msg = re.escape('Shape of the passed X data is not correct')
+    msg = re.escape("Shape of the passed X data is not correct")
     with pytest.raises(ValueError, match=msg):
         enc.inverse_transform(X_tr)
 
 
-@pytest.mark.parametrize('sparse_', [False, True])
+@pytest.mark.parametrize("sparse_", [False, True])
 @pytest.mark.parametrize(
     "X, X_trans",
     [
         ([[2, 55], [1, 55], [2, 55]], [[0, 1, 1], [0, 0, 0], [0, 1, 1]]),
-        ([['one', 'a'], ['two', 'a'], ['three', 'b'], ['two', 'a']],
-         [[0, 0, 0, 0, 0], [0, 0, 0, 0, 1], [0, 1, 0, 0, 0]]),
-    ]
+        (
+            [["one", "a"], ["two", "a"], ["three", "b"], ["two", "a"]],
+            [[0, 0, 0, 0, 0], [0, 0, 0, 0, 1], [0, 1, 0, 0, 0]],
+        ),
+    ],
 )
 def test_one_hot_encoder_inverse_transform_raise_error_with_unknown(
     X, X_trans, sparse_
@@ -284,7 +337,7 @@ def test_one_hot_encoder_inverse_transform_raise_error_with_unknown(
     Non-regression test for:
     https://github.com/scikit-learn/scikit-learn/issues/14934
     """
-    enc = OneHotEncoder(sparse=sparse_).fit(X)
+    enc = OneHotEncoder(sparse_output=sparse_).fit(X)
     msg = (
         r"Samples \[(\d )*\d\] can not be inverted when drop=None and "
         r"handle_unknown='error' because they contain all zeros"
@@ -298,73 +351,85 @@ def test_one_hot_encoder_inverse_transform_raise_error_with_unknown(
 
 
 def test_one_hot_encoder_inverse_if_binary():
-    X = np.array([['Male', 1],
-                  ['Female', 3],
-                  ['Female', 2]], dtype=object)
-    ohe = OneHotEncoder(drop='if_binary', sparse=False)
+    X = np.array([["Male", 1], ["Female", 3], ["Female", 2]], dtype=object)
+    ohe = OneHotEncoder(drop="if_binary", sparse_output=False)
     X_tr = ohe.fit_transform(X)
     assert_array_equal(ohe.inverse_transform(X_tr), X)
 
 
-# check that resetting drop option without refitting does not throw an error
-@pytest.mark.parametrize('drop', ['if_binary', 'first', None])
-@pytest.mark.parametrize('reset_drop', ['if_binary', 'first', None])
+@pytest.mark.parametrize("drop", ["if_binary", "first", None])
+@pytest.mark.parametrize("reset_drop", ["if_binary", "first", None])
 def test_one_hot_encoder_drop_reset(drop, reset_drop):
-    X = np.array([['Male', 1],
-                  ['Female', 3],
-                  ['Female', 2]], dtype=object)
-    ohe = OneHotEncoder(drop=drop, sparse=False)
+    # check that resetting drop option without refitting does not throw an error
+    X = np.array([["Male", 1], ["Female", 3], ["Female", 2]], dtype=object)
+    ohe = OneHotEncoder(drop=drop, sparse_output=False)
     ohe.fit(X)
     X_tr = ohe.transform(X)
-    feature_names = ohe.get_feature_names()
+    feature_names = ohe.get_feature_names_out()
     ohe.set_params(drop=reset_drop)
     assert_array_equal(ohe.inverse_transform(X_tr), X)
     assert_allclose(ohe.transform(X), X_tr)
-    assert_array_equal(ohe.get_feature_names(), feature_names)
+    assert_array_equal(ohe.get_feature_names_out(), feature_names)
 
 
-@pytest.mark.parametrize("method", ['fit', 'fit_transform'])
-@pytest.mark.parametrize("X", [
-    [1, 2],
-    np.array([3., 4.])
-    ])
+@pytest.mark.parametrize("method", ["fit", "fit_transform"])
+@pytest.mark.parametrize("X", [[1, 2], np.array([3.0, 4.0])])
 def test_X_is_not_1D(X, method):
     oh = OneHotEncoder()
 
-    msg = ("Expected 2D array, got 1D array instead")
+    msg = "Expected 2D array, got 1D array instead"
     with pytest.raises(ValueError, match=msg):
         getattr(oh, method)(X)
 
 
-@pytest.mark.parametrize("method", ['fit', 'fit_transform'])
+@pytest.mark.parametrize("method", ["fit", "fit_transform"])
 def test_X_is_not_1D_pandas(method):
-    pd = pytest.importorskip('pandas')
+    pd = pytest.importorskip("pandas")
     X = pd.Series([6, 3, 4, 6])
     oh = OneHotEncoder()
 
-    msg = ("Expected 2D array, got 1D array instead")
+    msg = f"Expected a 2-dimensional container but got {type(X)} instead."
     with pytest.raises(ValueError, match=msg):
         getattr(oh, method)(X)
 
 
-@pytest.mark.parametrize("X, cat_exp, cat_dtype", [
-    ([['abc', 55], ['def', 55]], [['abc', 'def'], [55]], np.object_),
-    (np.array([[1, 2], [3, 2]]), [[1, 3], [2]], np.integer),
-    (np.array([['A', 'cat'], ['B', 'cat']], dtype=object),
-     [['A', 'B'], ['cat']], np.object_),
-    (np.array([['A', 'cat'], ['B', 'cat']]),
-     [['A', 'B'], ['cat']], np.str_),
-    (np.array([[1, 2], [np.nan, 2]]), [[1, np.nan], [2]], np.float_),
-    (np.array([['A', np.nan], [None, np.nan]], dtype=object),
-     [['A', None], [np.nan]], np.object_),
-    (np.array([['A', float('nan')], [None, float('nan')]], dtype=object),
-     [['A', None], [float('nan')]], np.object_),
-    ], ids=['mixed', 'numeric', 'object', 'string', 'missing-float',
-            'missing-np.nan-object', 'missing-float-nan-object'])
+@pytest.mark.parametrize(
+    "X, cat_exp, cat_dtype",
+    [
+        ([["abc", 55], ["def", 55]], [["abc", "def"], [55]], np.object_),
+        (np.array([[1, 2], [3, 2]]), [[1, 3], [2]], np.integer),
+        (
+            np.array([["A", "cat"], ["B", "cat"]], dtype=object),
+            [["A", "B"], ["cat"]],
+            np.object_,
+        ),
+        (np.array([["A", "cat"], ["B", "cat"]]), [["A", "B"], ["cat"]], np.str_),
+        (np.array([[1, 2], [np.nan, 2]]), [[1, np.nan], [2]], np.float64),
+        (
+            np.array([["A", np.nan], [None, np.nan]], dtype=object),
+            [["A", None], [np.nan]],
+            np.object_,
+        ),
+        (
+            np.array([["A", float("nan")], [None, float("nan")]], dtype=object),
+            [["A", None], [float("nan")]],
+            np.object_,
+        ),
+    ],
+    ids=[
+        "mixed",
+        "numeric",
+        "object",
+        "string",
+        "missing-float",
+        "missing-np.nan-object",
+        "missing-float-nan-object",
+    ],
+)
 def test_one_hot_encoder_categories(X, cat_exp, cat_dtype):
     # order of categories should not depend on order of samples
     for Xi in [X, X[::-1]]:
-        enc = OneHotEncoder(categories='auto')
+        enc = OneHotEncoder(categories="auto")
         enc.fit(Xi)
         # assert enc.categories == 'auto'
         assert isinstance(enc.categories_, list)
@@ -378,35 +443,59 @@ def test_one_hot_encoder_categories(X, cat_exp, cat_dtype):
             assert np.issubdtype(res.dtype, cat_dtype)
 
 
-@pytest.mark.parametrize("X, X2, cats, cat_dtype", [
-    (np.array([['a', 'b']], dtype=object).T,
-     np.array([['a', 'd']], dtype=object).T,
-     [['a', 'b', 'c']], np.object_),
-    (np.array([[1, 2]], dtype='int64').T,
-     np.array([[1, 4]], dtype='int64').T,
-     [[1, 2, 3]], np.int64),
-    (np.array([['a', 'b']], dtype=object).T,
-     np.array([['a', 'd']], dtype=object).T,
-     [np.array(['a', 'b', 'c'])], np.object_),
-    (np.array([[None, 'a']], dtype=object).T,
-     np.array([[None, 'b']], dtype=object).T,
-     [[None, 'a', 'z']], object),
-    (np.array([['a', 'b']], dtype=object).T,
-     np.array([['a', np.nan]], dtype=object).T,
-     [['a', 'b', 'z']], object),
-    (np.array([['a', None]], dtype=object).T,
-     np.array([['a', np.nan]], dtype=object).T,
-     [['a', None, 'z']], object),
-    (np.array([['a', np.nan]], dtype=object).T,
-     np.array([['a', None]], dtype=object).T,
-     [['a', np.nan, 'z']], object),
-    ], ids=['object', 'numeric', 'object-string',
-            'object-string-none', 'object-string-nan',
-            'object-None-and-nan', 'object-nan-and-None'])
-def test_one_hot_encoder_specified_categories(X, X2, cats, cat_dtype):
+@pytest.mark.parametrize("handle_unknown", ["ignore", "infrequent_if_exist", "warn"])
+@pytest.mark.parametrize(
+    "X, X2, cats, cat_dtype",
+    [
+        (
+            np.array([["a", "b"]], dtype=object).T,
+            np.array([["a", "d"]], dtype=object).T,
+            [["a", "b", "c"]],
+            np.object_,
+        ),
+        (
+            np.array([[1, 2]], dtype="int64").T,
+            np.array([[1, 4]], dtype="int64").T,
+            [[1, 2, 3]],
+            np.int64,
+        ),
+        (
+            np.array([["a", "b"]], dtype=object).T,
+            np.array([["a", "d"]], dtype=object).T,
+            [np.array(["a", "b", "c"])],
+            np.object_,
+        ),
+        (
+            np.array([[None, "a"]], dtype=object).T,
+            np.array([[None, "b"]], dtype=object).T,
+            [[None, "a", "z"]],
+            object,
+        ),
+        (
+            np.array([["a", "b"]], dtype=object).T,
+            np.array([["a", np.nan]], dtype=object).T,
+            [["a", "b", "z"]],
+            object,
+        ),
+        (
+            np.array([["a", None]], dtype=object).T,
+            np.array([["a", np.nan]], dtype=object).T,
+            [["a", None, "z"]],
+            object,
+        ),
+    ],
+    ids=[
+        "object",
+        "numeric",
+        "object-string",
+        "object-string-none",
+        "object-string-nan",
+        "object-None-and-nan",
+    ],
+)
+def test_one_hot_encoder_specified_categories(X, X2, cats, cat_dtype, handle_unknown):
     enc = OneHotEncoder(categories=cats)
-    exp = np.array([[1., 0., 0.],
-                    [0., 1., 0.]])
+    exp = np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])
     assert_array_equal(enc.fit_transform(X).toarray(), exp)
     assert list(enc.categories[0]) == list(cats[0])
     assert enc.categories_[0].tolist() == list(cats[0])
@@ -419,44 +508,50 @@ def test_one_hot_encoder_specified_categories(X, X2, cats, cat_dtype):
     enc = OneHotEncoder(categories=cats)
     with pytest.raises(ValueError, match="Found unknown categories"):
         enc.fit(X2)
-    enc = OneHotEncoder(categories=cats, handle_unknown='ignore')
-    exp = np.array([[1., 0., 0.], [0., 0., 0.]])
+    enc = OneHotEncoder(categories=cats, handle_unknown=handle_unknown)
+    exp = np.array([[1.0, 0.0, 0.0], [0.0, 0.0, 0.0]])
     assert_array_equal(enc.fit(X2).transform(X2).toarray(), exp)
 
 
 def test_one_hot_encoder_unsorted_categories():
-    X = np.array([['a', 'b']], dtype=object).T
+    X = np.array([["a", "b"]], dtype=object).T
 
-    enc = OneHotEncoder(categories=[['b', 'a', 'c']])
-    exp = np.array([[0., 1., 0.],
-                    [1., 0., 0.]])
+    enc = OneHotEncoder(categories=[["b", "a", "c"]])
+    exp = np.array([[0.0, 1.0, 0.0], [1.0, 0.0, 0.0]])
     assert_array_equal(enc.fit(X).transform(X).toarray(), exp)
     assert_array_equal(enc.fit_transform(X).toarray(), exp)
-    assert enc.categories_[0].tolist() == ['b', 'a', 'c']
+    assert enc.categories_[0].tolist() == ["b", "a", "c"]
     assert np.issubdtype(enc.categories_[0].dtype, np.object_)
 
     # unsorted passed categories still raise for numerical values
     X = np.array([[1, 2]]).T
     enc = OneHotEncoder(categories=[[2, 1, 3]])
-    msg = 'Unsorted categories are not supported'
+    msg = "Unsorted categories are not supported"
     with pytest.raises(ValueError, match=msg):
         enc.fit_transform(X)
 
-    # np.nan must be the last category in categories[0] to be considered sorted
-    X = np.array([[1, 2, np.nan]]).T
-    enc = OneHotEncoder(categories=[[1, np.nan, 2]])
-    with pytest.raises(ValueError, match=msg):
-        enc.fit_transform(X)
+
+@pytest.mark.parametrize("Encoder", [OneHotEncoder, OrdinalEncoder])
+def test_encoder_nan_ending_specified_categories(Encoder):
+    """Test encoder for specified categories that nan is at the end.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/27088
+    """
+    cats = [np.array([0, np.nan, 1])]
+    enc = Encoder(categories=cats)
+    X = np.array([[0, 1]], dtype=object).T
+    with pytest.raises(ValueError, match="Nan should be the last element"):
+        enc.fit(X)
 
 
 def test_one_hot_encoder_specified_categories_mixed_columns():
     # multiple columns
-    X = np.array([['a', 'b'], [0, 2]], dtype=object).T
-    enc = OneHotEncoder(categories=[['a', 'b', 'c'], [0, 1, 2]])
-    exp = np.array([[1., 0., 0., 1., 0., 0.],
-                    [0., 1., 0., 0., 0., 1.]])
+    X = np.array([["a", "b"], [0, 2]], dtype=object).T
+    enc = OneHotEncoder(categories=[["a", "b", "c"], [0, 1, 2]])
+    exp = np.array([[1.0, 0.0, 0.0, 1.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0, 0.0, 1.0]])
     assert_array_equal(enc.fit_transform(X).toarray(), exp)
-    assert enc.categories_[0].tolist() == ['a', 'b', 'c']
+    assert enc.categories_[0].tolist() == ["a", "b", "c"]
     assert np.issubdtype(enc.categories_[0].dtype, np.object_)
     assert enc.categories_[1].tolist() == [0, 1, 2]
     # integer categories but from object dtype data
@@ -464,88 +559,100 @@ def test_one_hot_encoder_specified_categories_mixed_columns():
 
 
 def test_one_hot_encoder_pandas():
-    pd = pytest.importorskip('pandas')
+    pd = pytest.importorskip("pandas")
 
-    X_df = pd.DataFrame({'A': ['a', 'b'], 'B': [1, 2]})
+    X_df = pd.DataFrame({"A": ["a", "b"], "B": [1, 2]})
 
     Xtr = check_categorical_onehot(X_df)
     assert_allclose(Xtr, [[1, 0, 1, 0], [0, 1, 0, 1]])
 
 
-@pytest.mark.parametrize("drop, expected_names",
-                         [('first', ['x0_c', 'x2_b']),
-                          ('if_binary', ['x0_c', 'x1_2', 'x2_b']),
-                          (['c', 2, 'b'], ['x0_b', 'x2_a'])],
-                         ids=['first', 'binary', 'manual'])
+@pytest.mark.parametrize(
+    "drop, expected_names",
+    [
+        ("first", ["x0_c", "x2_b"]),
+        ("if_binary", ["x0_c", "x1_2", "x2_b"]),
+        (["c", 2, "b"], ["x0_b", "x2_a"]),
+    ],
+    ids=["first", "binary", "manual"],
+)
 def test_one_hot_encoder_feature_names_drop(drop, expected_names):
-    X = [['c', 2, 'a'],
-         ['b', 2, 'b']]
+    X = [["c", 2, "a"], ["b", 2, "b"]]
 
     ohe = OneHotEncoder(drop=drop)
     ohe.fit(X)
-    feature_names = ohe.get_feature_names()
-    assert isinstance(feature_names, np.ndarray)
+    feature_names = ohe.get_feature_names_out()
     assert_array_equal(expected_names, feature_names)
 
 
 def test_one_hot_encoder_drop_equals_if_binary():
     # Canonical case
-    X = [[10, 'yes'],
-         [20, 'no'],
-         [30, 'yes']]
-    expected = np.array([[1., 0., 0., 1.],
-                         [0., 1., 0., 0.],
-                         [0., 0., 1., 1.]])
+    X = [[10, "yes"], [20, "no"], [30, "yes"]]
+    expected = np.array(
+        [[1.0, 0.0, 0.0, 1.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 1.0]]
+    )
     expected_drop_idx = np.array([None, 0])
 
-    ohe = OneHotEncoder(drop='if_binary', sparse=False)
+    ohe = OneHotEncoder(drop="if_binary", sparse_output=False)
     result = ohe.fit_transform(X)
     assert_array_equal(ohe.drop_idx_, expected_drop_idx)
     assert_allclose(result, expected)
 
     # with only one cat, the behaviour is equivalent to drop=None
-    X = [['true', 'a'],
-         ['false', 'a'],
-         ['false', 'a']]
-    expected = np.array([[1., 1.],
-                         [0., 1.],
-                         [0., 1.]])
+    X = [["true", "a"], ["false", "a"], ["false", "a"]]
+    expected = np.array([[1.0, 1.0], [0.0, 1.0], [0.0, 1.0]])
     expected_drop_idx = np.array([0, None])
 
-    ohe = OneHotEncoder(drop='if_binary', sparse=False)
+    ohe = OneHotEncoder(drop="if_binary", sparse_output=False)
     result = ohe.fit_transform(X)
     assert_array_equal(ohe.drop_idx_, expected_drop_idx)
     assert_allclose(result, expected)
 
 
-@pytest.mark.parametrize("X", [
-    [['abc', 2, 55], ['def', 1, 55]],
-    np.array([[10, 2, 55], [20, 1, 55]]),
-    np.array([['a', 'B', 'cat'], ['b', 'A', 'cat']], dtype=object)
-    ], ids=['mixed', 'numeric', 'object'])
+@pytest.mark.parametrize(
+    "X",
+    [
+        [["abc", 2, 55], ["def", 1, 55]],
+        np.array([[10, 2, 55], [20, 1, 55]]),
+        np.array([["a", "B", "cat"], ["b", "A", "cat"]], dtype=object),
+    ],
+    ids=["mixed", "numeric", "object"],
+)
 def test_ordinal_encoder(X):
     enc = OrdinalEncoder()
-    exp = np.array([[0, 1, 0],
-                    [1, 0, 0]], dtype='int64')
-    assert_array_equal(enc.fit_transform(X), exp.astype('float64'))
-    enc = OrdinalEncoder(dtype='int64')
+    exp = np.array([[0, 1, 0], [1, 0, 0]], dtype="int64")
+    assert_array_equal(enc.fit_transform(X), exp.astype("float64"))
+    enc = OrdinalEncoder(dtype="int64")
     assert_array_equal(enc.fit_transform(X), exp)
 
 
-@pytest.mark.parametrize("X, X2, cats, cat_dtype", [
-    (np.array([['a', 'b']], dtype=object).T,
-     np.array([['a', 'd']], dtype=object).T,
-     [['a', 'b', 'c']], np.object_),
-    (np.array([[1, 2]], dtype='int64').T,
-     np.array([[1, 4]], dtype='int64').T,
-     [[1, 2, 3]], np.int64),
-    (np.array([['a', 'b']], dtype=object).T,
-     np.array([['a', 'd']], dtype=object).T,
-     [np.array(['a', 'b', 'c'])], np.object_),
-    ], ids=['object', 'numeric', 'object-string-cat'])
+@pytest.mark.parametrize(
+    "X, X2, cats, cat_dtype",
+    [
+        (
+            np.array([["a", "b"]], dtype=object).T,
+            np.array([["a", "d"]], dtype=object).T,
+            [["a", "b", "c"]],
+            np.object_,
+        ),
+        (
+            np.array([[1, 2]], dtype="int64").T,
+            np.array([[1, 4]], dtype="int64").T,
+            [[1, 2, 3]],
+            np.int64,
+        ),
+        (
+            np.array([["a", "b"]], dtype=object).T,
+            np.array([["a", "d"]], dtype=object).T,
+            [np.array(["a", "b", "c"])],
+            np.object_,
+        ),
+    ],
+    ids=["object", "numeric", "object-string-cat"],
+)
 def test_ordinal_encoder_specified_categories(X, X2, cats, cat_dtype):
     enc = OrdinalEncoder(categories=cats)
-    exp = np.array([[0.], [1.]])
+    exp = np.array([[0.0], [1.0]])
     assert_array_equal(enc.fit_transform(X), exp)
     assert list(enc.categories[0]) == list(cats[0])
     assert enc.categories_[0].tolist() == list(cats[0])
@@ -561,7 +668,7 @@ def test_ordinal_encoder_specified_categories(X, X2, cats, cat_dtype):
 
 
 def test_ordinal_encoder_inverse():
-    X = [['abc', 2, 55], ['def', 1, 55]]
+    X = [["abc", 2, 55], ["def", 1, 55]]
     enc = OrdinalEncoder()
     X_tr = enc.fit_transform(X)
     exp = np.array(X, dtype=object)
@@ -569,36 +676,35 @@ def test_ordinal_encoder_inverse():
 
     # incorrect shape raises
     X_tr = np.array([[0, 1, 1, 2], [1, 0, 1, 0]])
-    msg = re.escape('Shape of the passed X data is not correct')
+    msg = re.escape("Shape of the passed X data is not correct")
     with pytest.raises(ValueError, match=msg):
         enc.inverse_transform(X_tr)
 
 
 def test_ordinal_encoder_handle_unknowns_string():
-    enc = OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=-2)
-    X_fit = np.array([['a', 'x'], ['b', 'y'], ['c', 'z']], dtype=object)
-    X_trans = np.array([['c', 'xy'], ['bla', 'y'], ['a', 'x']], dtype=object)
+    enc = OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=-2)
+    X_fit = np.array([["a", "x"], ["b", "y"], ["c", "z"]], dtype=object)
+    X_trans = np.array([["c", "xy"], ["bla", "y"], ["a", "x"]], dtype=object)
     enc.fit(X_fit)
 
     X_trans_enc = enc.transform(X_trans)
-    exp = np.array([[2, -2], [-2, 1], [0, 0]], dtype='int64')
+    exp = np.array([[2, -2], [-2, 1], [0, 0]], dtype="int64")
     assert_array_equal(X_trans_enc, exp)
 
     X_trans_inv = enc.inverse_transform(X_trans_enc)
-    inv_exp = np.array([['c', None], [None, 'y'], ['a', 'x']], dtype=object)
+    inv_exp = np.array([["c", None], [None, "y"], ["a", "x"]], dtype=object)
     assert_array_equal(X_trans_inv, inv_exp)
 
 
-@pytest.mark.parametrize('dtype', [float, int])
+@pytest.mark.parametrize("dtype", [float, int])
 def test_ordinal_encoder_handle_unknowns_numeric(dtype):
-    enc = OrdinalEncoder(handle_unknown='use_encoded_value',
-                         unknown_value=-999)
+    enc = OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=-999)
     X_fit = np.array([[1, 7], [2, 8], [3, 9]], dtype=dtype)
     X_trans = np.array([[3, 12], [23, 8], [1, 7]], dtype=dtype)
     enc.fit(X_fit)
 
     X_trans_enc = enc.transform(X_trans)
-    exp = np.array([[2, -999], [-999, 1], [0, 0]], dtype='int64')
+    exp = np.array([[2, -999], [-999, 1], [0, 0]], dtype="int64")
     assert_array_equal(X_trans_enc, exp)
 
     X_trans_inv = enc.inverse_transform(X_trans_enc)
@@ -606,55 +712,10 @@ def test_ordinal_encoder_handle_unknowns_numeric(dtype):
     assert_array_equal(X_trans_inv, inv_exp)
 
 
-@pytest.mark.parametrize(
-    "params, err_type, err_msg",
-    [
-        (
-            {"handle_unknown": "use_encoded_value"},
-            TypeError,
-            "unknown_value should be an integer or np.nan when handle_unknown "
-            "is 'use_encoded_value', got None.",
-        ),
-        (
-            {"unknown_value": -2},
-            TypeError,
-            "unknown_value should only be set when handle_unknown is "
-            "'use_encoded_value', got -2.",
-        ),
-        (
-            {"handle_unknown": "use_encoded_value", "unknown_value": "bla"},
-            TypeError,
-            "unknown_value should be an integer or np.nan when handle_unknown "
-            "is 'use_encoded_value', got bla.",
-        ),
-        (
-            {"handle_unknown": "use_encoded_value", "unknown_value": 1},
-            ValueError,
-            "The used value for unknown_value (1) is one of the values "
-            "already used for encoding the seen categories.",
-        ),
-        (
-            {"handle_unknown": "ignore"},
-            ValueError,
-            "handle_unknown should be either 'error' or 'use_encoded_value', "
-            "got ignore.",
-        ),
-    ],
-)
-def test_ordinal_encoder_handle_unknowns_raise(params, err_type, err_msg):
-    # Check error message when validating input parameters
-    X = np.array([['a', 'x'], ['b', 'y']], dtype=object)
-
-    encoder = OrdinalEncoder(**params)
-    with pytest.raises(err_type, match=err_msg):
-        encoder.fit(X)
-
-
 def test_ordinal_encoder_handle_unknowns_nan():
     # Make sure unknown_value=np.nan properly works
 
-    enc = OrdinalEncoder(handle_unknown='use_encoded_value',
-                         unknown_value=np.nan)
+    enc = OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=np.nan)
 
     X_fit = np.array([[1], [2], [3]])
     enc.fit(X_fit)
@@ -665,21 +726,20 @@ def test_ordinal_encoder_handle_unknowns_nan():
 def test_ordinal_encoder_handle_unknowns_nan_non_float_dtype():
     # Make sure an error is raised when unknown_value=np.nan and the dtype
     # isn't a float dtype
-    enc = OrdinalEncoder(handle_unknown='use_encoded_value',
-                         unknown_value=np.nan, dtype=int)
+    enc = OrdinalEncoder(
+        handle_unknown="use_encoded_value", unknown_value=np.nan, dtype=int
+    )
 
     X_fit = np.array([[1], [2], [3]])
-    with pytest.raises(ValueError,
-                       match="dtype parameter should be a float dtype"):
+    with pytest.raises(ValueError, match="dtype parameter should be a float dtype"):
         enc.fit(X_fit)
 
 
 def test_ordinal_encoder_raise_categories_shape():
-
-    X = np.array([['Low', 'Medium', 'High', 'Medium', 'Low']], dtype=object).T
-    cats = ['Low', 'Medium', 'High']
+    X = np.array([["Low", "Medium", "High", "Medium", "Low"]], dtype=object).T
+    cats = ["Low", "Medium", "High"]
     enc = OrdinalEncoder(categories=cats)
-    msg = ("Shape mismatch: if categories is an array,")
+    msg = "Shape mismatch: if categories is an array,"
 
     with pytest.raises(ValueError, match=msg):
         enc.fit(X)
@@ -687,44 +747,48 @@ def test_ordinal_encoder_raise_categories_shape():
 
 def test_encoder_dtypes():
     # check that dtypes are preserved when determining categories
-    enc = OneHotEncoder(categories='auto')
-    exp = np.array([[1., 0., 1., 0.], [0., 1., 0., 1.]], dtype='float64')
-
-    for X in [np.array([[1, 2], [3, 4]], dtype='int64'),
-              np.array([[1, 2], [3, 4]], dtype='float64'),
-              np.array([['a', 'b'], ['c', 'd']]),  # string dtype
-              np.array([[1, 'a'], [3, 'b']], dtype='object')]:
+    enc = OneHotEncoder(categories="auto")
+    exp = np.array([[1.0, 0.0, 1.0, 0.0], [0.0, 1.0, 0.0, 1.0]], dtype="float64")
+
+    for X in [
+        np.array([[1, 2], [3, 4]], dtype="int64"),
+        np.array([[1, 2], [3, 4]], dtype="float64"),
+        np.array([["a", "b"], ["c", "d"]]),  # str dtype
+        np.array([[b"a", b"b"], [b"c", b"d"]]),  # bytes dtype
+        np.array([[1, "a"], [3, "b"]], dtype="object"),
+    ]:
         enc.fit(X)
         assert all([enc.categories_[i].dtype == X.dtype for i in range(2)])
         assert_array_equal(enc.transform(X).toarray(), exp)
 
     X = [[1, 2], [3, 4]]
     enc.fit(X)
-    assert all([np.issubdtype(enc.categories_[i].dtype, np.integer)
-                for i in range(2)])
+    assert all([np.issubdtype(enc.categories_[i].dtype, np.integer) for i in range(2)])
     assert_array_equal(enc.transform(X).toarray(), exp)
 
-    X = [[1, 'a'], [3, 'b']]
+    X = [[1, "a"], [3, "b"]]
     enc.fit(X)
-    assert all([enc.categories_[i].dtype == 'object' for i in range(2)])
+    assert all([enc.categories_[i].dtype == "object" for i in range(2)])
     assert_array_equal(enc.transform(X).toarray(), exp)
 
 
 def test_encoder_dtypes_pandas():
     # check dtype (similar to test_categorical_encoder_dtypes for dataframes)
-    pd = pytest.importorskip('pandas')
+    pd = pytest.importorskip("pandas")
 
-    enc = OneHotEncoder(categories='auto')
-    exp = np.array([[1., 0., 1., 0., 1., 0.],
-                    [0., 1., 0., 1., 0., 1.]], dtype='float64')
+    enc = OneHotEncoder(categories="auto")
+    exp = np.array(
+        [[1.0, 0.0, 1.0, 0.0, 1.0, 0.0], [0.0, 1.0, 0.0, 1.0, 0.0, 1.0]],
+        dtype="float64",
+    )
 
-    X = pd.DataFrame({'A': [1, 2], 'B': [3, 4], 'C': [5, 6]}, dtype='int64')
+    X = pd.DataFrame({"A": [1, 2], "B": [3, 4], "C": [5, 6]}, dtype="int64")
     enc.fit(X)
-    assert all([enc.categories_[i].dtype == 'int64' for i in range(2)])
+    assert all([enc.categories_[i].dtype == "int64" for i in range(2)])
     assert_array_equal(enc.transform(X).toarray(), exp)
 
-    X = pd.DataFrame({'A': [1, 2], 'B': ['a', 'b'], 'C': [3., 4.]})
-    X_type = [X['A'].dtype, X['B'].dtype, X['C'].dtype]
+    X = pd.DataFrame({"A": [1, 2], "B": ["a", "b"], "C": [3.0, 4.0]})
+    X_type = [X["A"].dtype, X["B"].dtype, X["C"].dtype]
     enc.fit(X)
     assert all([enc.categories_[i].dtype == X_type[i] for i in range(3)])
     assert_array_equal(enc.transform(X).toarray(), exp)
@@ -732,25 +796,55 @@ def test_encoder_dtypes_pandas():
 
 def test_one_hot_encoder_warning():
     enc = OneHotEncoder()
-    X = [['Male', 1], ['Female', 3]]
-    np.testing.assert_no_warnings(enc.fit_transform, X)
+    X = [["Male", 1], ["Female", 3]]
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        enc.fit_transform(X)
+
+
+@pytest.mark.parametrize("drop", ["if_binary", "first"])
+def test_ohe_handle_unknown_warn(drop):
+    """Check handle_unknown='warn' works correctly."""
+
+    X = [["a", 0], ["b", 2], ["b", 1]]
+
+    ohe = OneHotEncoder(
+        drop=drop,
+        sparse_output=False,
+        handle_unknown="warn",
+        categories=[["b", "a"], [1, 2]],
+    )
+    ohe.fit(X)
+
+    X_test = [["c", 1]]
+    X_expected = np.array([[0, 0]])
+
+    warn_msg = (
+        r"Found unknown categories in columns \[0\] during transform. "
+        r"These unknown categories will be encoded as all zeros"
+    )
+    with pytest.warns(UserWarning, match=warn_msg):
+        X_trans = ohe.transform(X_test)
+    assert_allclose(X_trans, X_expected)
 
 
-@pytest.mark.parametrize("missing_value", [np.nan, None, float('nan')])
+@pytest.mark.parametrize("missing_value", [np.nan, None, float("nan")])
 def test_one_hot_encoder_drop_manual(missing_value):
-    cats_to_drop = ['def', 12, 3, 56, missing_value]
+    cats_to_drop = ["def", 12, 3, 56, missing_value]
     enc = OneHotEncoder(drop=cats_to_drop)
-    X = [['abc', 12, 2, 55, 'a'],
-         ['def', 12, 1, 55, 'a'],
-         ['def', 12, 3, 56, missing_value]]
+    X = [
+        ["abc", 12, 2, 55, "a"],
+        ["def", 12, 1, 55, "a"],
+        ["def", 12, 3, 56, missing_value],
+    ]
     trans = enc.fit_transform(X).toarray()
-    exp = [[1, 0, 1, 1, 1],
-           [0, 1, 0, 1, 1],
-           [0, 0, 0, 0, 0]]
+    exp = [[1, 0, 1, 1, 1], [0, 1, 0, 1, 1], [0, 0, 0, 0, 0]]
     assert_array_equal(trans, exp)
-    dropped_cats = [cat[feature]
-                    for cat, feature in zip(enc.categories_,
-                                            enc.drop_idx_)]
+    assert enc.drop is cats_to_drop
+
+    dropped_cats = [
+        cat[feature] for cat, feature in zip(enc.categories_, enc.drop_idx_)
+    ]
     X_inv_trans = enc.inverse_transform(trans)
     X_array = np.array(X, dtype=object)
 
@@ -771,77 +865,540 @@ def test_one_hot_encoder_drop_manual(missing_value):
         assert_array_equal(X_array, X_inv_trans)
 
 
-@pytest.mark.parametrize(
-    "X_fit, params, err_msg",
-    [([["Male"], ["Female"]], {'drop': 'second'},
-     "Wrong input for parameter `drop`"),
-     ([["Male"], ["Female"]], {'drop': 'first', 'handle_unknown': 'ignore'},
-     "`handle_unknown` must be 'error'"),
-     ([['abc', 2, 55], ['def', 1, 55], ['def', 3, 59]],
-      {'drop': np.asarray('b', dtype=object)},
-     "Wrong input for parameter `drop`"),
-     ([['abc', 2, 55], ['def', 1, 55], ['def', 3, 59]],
-      {'drop': ['ghi', 3, 59]},
-     "The following categories were supposed")]
-)
-def test_one_hot_encoder_invalid_params(X_fit, params, err_msg):
-    enc = OneHotEncoder(**params)
-    with pytest.raises(ValueError, match=err_msg):
-        enc.fit(X_fit)
-
-
-@pytest.mark.parametrize('drop', [['abc', 3], ['abc', 3, 41, 'a']])
+@pytest.mark.parametrize("drop", [["abc", 3], ["abc", 3, 41, "a"]])
 def test_invalid_drop_length(drop):
     enc = OneHotEncoder(drop=drop)
     err_msg = "`drop` should have length equal to the number"
     with pytest.raises(ValueError, match=err_msg):
-        enc.fit([['abc', 2, 55], ['def', 1, 55], ['def', 3, 59]])
+        enc.fit([["abc", 2, 55], ["def", 1, 55], ["def", 3, 59]])
 
 
-@pytest.mark.parametrize("density", [True, False],
-                         ids=['sparse', 'dense'])
-@pytest.mark.parametrize("drop", ['first',
-                                  ['a', 2, 'b']],
-                         ids=['first', 'manual'])
+@pytest.mark.parametrize("density", [True, False], ids=["sparse", "dense"])
+@pytest.mark.parametrize("drop", ["first", ["a", 2, "b"]], ids=["first", "manual"])
 def test_categories(density, drop):
-    ohe_base = OneHotEncoder(sparse=density)
-    ohe_test = OneHotEncoder(sparse=density, drop=drop)
-    X = [['c', 1, 'a'],
-         ['a', 2, 'b']]
+    ohe_base = OneHotEncoder(sparse_output=density)
+    ohe_test = OneHotEncoder(sparse_output=density, drop=drop)
+    X = [["c", 1, "a"], ["a", 2, "b"]]
     ohe_base.fit(X)
     ohe_test.fit(X)
     assert_array_equal(ohe_base.categories_, ohe_test.categories_)
-    if drop == 'first':
+    if drop == "first":
         assert_array_equal(ohe_test.drop_idx_, 0)
     else:
-        for drop_cat, drop_idx, cat_list in zip(drop,
-                                                ohe_test.drop_idx_,
-                                                ohe_test.categories_):
+        for drop_cat, drop_idx, cat_list in zip(
+            drop, ohe_test.drop_idx_, ohe_test.categories_
+        ):
             assert cat_list[int(drop_idx)] == drop_cat
     assert isinstance(ohe_test.drop_idx_, np.ndarray)
     assert ohe_test.drop_idx_.dtype == object
 
 
-@pytest.mark.parametrize('Encoder', [OneHotEncoder, OrdinalEncoder])
+@pytest.mark.parametrize("Encoder", [OneHotEncoder, OrdinalEncoder])
 def test_encoders_has_categorical_tags(Encoder):
-    assert 'categorical' in Encoder()._get_tags()['X_types']
+    assert Encoder().__sklearn_tags__().input_tags.categorical
+
+
+@pytest.mark.parametrize(
+    "kwargs",
+    [
+        {"max_categories": 2},
+        {"min_frequency": 11},
+        {"min_frequency": 0.29},
+        {"max_categories": 2, "min_frequency": 6},
+        {"max_categories": 4, "min_frequency": 12},
+    ],
+)
+@pytest.mark.parametrize("categories", ["auto", [["a", "b", "c", "d"]]])
+def test_ohe_infrequent_two_levels(kwargs, categories):
+    """Test that different parameters for combine 'a', 'c', and 'd' into
+    the infrequent category works as expected."""
+
+    X_train = np.array([["a"] * 5 + ["b"] * 20 + ["c"] * 10 + ["d"] * 3]).T
+    ohe = OneHotEncoder(
+        categories=categories,
+        handle_unknown="infrequent_if_exist",
+        sparse_output=False,
+        **kwargs,
+    ).fit(X_train)
+    assert_array_equal(ohe.infrequent_categories_, [["a", "c", "d"]])
+
+    X_test = [["b"], ["a"], ["c"], ["d"], ["e"]]
+    expected = np.array([[1, 0], [0, 1], [0, 1], [0, 1], [0, 1]])
+
+    X_trans = ohe.transform(X_test)
+    assert_allclose(expected, X_trans)
+
+    expected_inv = [[col] for col in ["b"] + ["infrequent_sklearn"] * 4]
+    X_inv = ohe.inverse_transform(X_trans)
+    assert_array_equal(expected_inv, X_inv)
+
+    feature_names = ohe.get_feature_names_out()
+    assert_array_equal(["x0_b", "x0_infrequent_sklearn"], feature_names)
+
+
+@pytest.mark.parametrize("drop", ["if_binary", "first", ["b"]])
+def test_ohe_infrequent_two_levels_drop_frequent(drop):
+    """Test two levels and dropping the frequent category."""
+
+    X_train = np.array([["a"] * 5 + ["b"] * 20 + ["c"] * 10 + ["d"] * 3]).T
+    ohe = OneHotEncoder(
+        handle_unknown="infrequent_if_exist",
+        sparse_output=False,
+        max_categories=2,
+        drop=drop,
+    ).fit(X_train)
+    assert ohe.categories_[0][ohe.drop_idx_[0]] == "b"
+
+    X_test = np.array([["b"], ["c"]])
+    X_trans = ohe.transform(X_test)
+    assert_allclose([[0], [1]], X_trans)
+
+    feature_names = ohe.get_feature_names_out()
+    assert_array_equal(["x0_infrequent_sklearn"], feature_names)
+
+    X_inverse = ohe.inverse_transform(X_trans)
+    assert_array_equal([["b"], ["infrequent_sklearn"]], X_inverse)
+
+
+@pytest.mark.parametrize("drop", [["a"], ["d"]])
+def test_ohe_infrequent_two_levels_drop_infrequent_errors(drop):
+    """Test two levels and dropping any infrequent category removes the
+    whole infrequent category."""
+
+    X_train = np.array([["a"] * 5 + ["b"] * 20 + ["c"] * 10 + ["d"] * 3]).T
+    ohe = OneHotEncoder(
+        handle_unknown="infrequent_if_exist",
+        sparse_output=False,
+        max_categories=2,
+        drop=drop,
+    )
+
+    msg = f"Unable to drop category {drop[0]!r} from feature 0 because it is infrequent"
+    with pytest.raises(ValueError, match=msg):
+        ohe.fit(X_train)
+
+
+@pytest.mark.parametrize(
+    "kwargs",
+    [
+        {"max_categories": 3},
+        {"min_frequency": 6},
+        {"min_frequency": 9},
+        {"min_frequency": 0.24},
+        {"min_frequency": 0.16},
+        {"max_categories": 3, "min_frequency": 8},
+        {"max_categories": 4, "min_frequency": 6},
+    ],
+)
+def test_ohe_infrequent_three_levels(kwargs):
+    """Test that different parameters for combing 'a', and 'd' into
+    the infrequent category works as expected."""
+
+    X_train = np.array([["a"] * 5 + ["b"] * 20 + ["c"] * 10 + ["d"] * 3]).T
+    ohe = OneHotEncoder(
+        handle_unknown="infrequent_if_exist", sparse_output=False, **kwargs
+    ).fit(X_train)
+    assert_array_equal(ohe.infrequent_categories_, [["a", "d"]])
+
+    X_test = [["b"], ["a"], ["c"], ["d"], ["e"]]
+    expected = np.array([[1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 0, 1], [0, 0, 1]])
+
+    X_trans = ohe.transform(X_test)
+    assert_allclose(expected, X_trans)
+
+    expected_inv = [
+        ["b"],
+        ["infrequent_sklearn"],
+        ["c"],
+        ["infrequent_sklearn"],
+        ["infrequent_sklearn"],
+    ]
+    X_inv = ohe.inverse_transform(X_trans)
+    assert_array_equal(expected_inv, X_inv)
+
+    feature_names = ohe.get_feature_names_out()
+    assert_array_equal(["x0_b", "x0_c", "x0_infrequent_sklearn"], feature_names)
+
+
+@pytest.mark.parametrize("drop", ["first", ["b"]])
+def test_ohe_infrequent_three_levels_drop_frequent(drop):
+    """Test three levels and dropping the frequent category."""
+
+    X_train = np.array([["a"] * 5 + ["b"] * 20 + ["c"] * 10 + ["d"] * 3]).T
+    ohe = OneHotEncoder(
+        handle_unknown="infrequent_if_exist",
+        sparse_output=False,
+        max_categories=3,
+        drop=drop,
+    ).fit(X_train)
+
+    X_test = np.array([["b"], ["c"], ["d"]])
+    assert_allclose([[0, 0], [1, 0], [0, 1]], ohe.transform(X_test))
+
+    # Check handle_unknown="ignore"
+    ohe.set_params(handle_unknown="ignore").fit(X_train)
+    msg = "Found unknown categories"
+    with pytest.warns(UserWarning, match=msg):
+        X_trans = ohe.transform([["b"], ["e"]])
+
+    assert_allclose([[0, 0], [0, 0]], X_trans)
+
+
+@pytest.mark.parametrize("drop", [["a"], ["d"]])
+def test_ohe_infrequent_three_levels_drop_infrequent_errors(drop):
+    """Test three levels and dropping the infrequent category."""
+    X_train = np.array([["a"] * 5 + ["b"] * 20 + ["c"] * 10 + ["d"] * 3]).T
+    ohe = OneHotEncoder(
+        handle_unknown="infrequent_if_exist",
+        sparse_output=False,
+        max_categories=3,
+        drop=drop,
+    )
+
+    msg = f"Unable to drop category {drop[0]!r} from feature 0 because it is infrequent"
+    with pytest.raises(ValueError, match=msg):
+        ohe.fit(X_train)
+
+
+def test_ohe_infrequent_handle_unknown_error():
+    """Test that different parameters for combining 'a', and 'd' into
+    the infrequent category works as expected."""
+
+    X_train = np.array([["a"] * 5 + ["b"] * 20 + ["c"] * 10 + ["d"] * 3]).T
+    ohe = OneHotEncoder(
+        handle_unknown="error", sparse_output=False, max_categories=3
+    ).fit(X_train)
+    assert_array_equal(ohe.infrequent_categories_, [["a", "d"]])
+
+    # all categories are known
+    X_test = [["b"], ["a"], ["c"], ["d"]]
+    expected = np.array([[1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 0, 1]])
+
+    X_trans = ohe.transform(X_test)
+    assert_allclose(expected, X_trans)
+
+    # 'bad' is not known and will error
+    X_test = [["bad"]]
+    msg = r"Found unknown categories \['bad'\] in column 0"
+    with pytest.raises(ValueError, match=msg):
+        ohe.transform(X_test)
+
+
+@pytest.mark.parametrize(
+    "kwargs", [{"max_categories": 3, "min_frequency": 1}, {"min_frequency": 4}]
+)
+def test_ohe_infrequent_two_levels_user_cats_one_frequent(kwargs):
+    """'a' is the only frequent category, all other categories are infrequent."""
+
+    X_train = np.array([["a"] * 5 + ["e"] * 30], dtype=object).T
+    ohe = OneHotEncoder(
+        categories=[["c", "d", "a", "b"]],
+        sparse_output=False,
+        handle_unknown="infrequent_if_exist",
+        **kwargs,
+    ).fit(X_train)
+
+    X_test = [["a"], ["b"], ["c"], ["d"], ["e"]]
+    expected = np.array([[1, 0], [0, 1], [0, 1], [0, 1], [0, 1]])
+
+    X_trans = ohe.transform(X_test)
+    assert_allclose(expected, X_trans)
+
+    # 'a' is dropped
+    drops = ["first", "if_binary", ["a"]]
+    X_test = [["a"], ["c"]]
+    for drop in drops:
+        ohe.set_params(drop=drop).fit(X_train)
+        assert_allclose([[0], [1]], ohe.transform(X_test))
+
+
+def test_ohe_infrequent_two_levels_user_cats():
+    """Test that the order of the categories provided by a user is respected."""
+    X_train = np.array(
+        [["a"] * 5 + ["b"] * 20 + ["c"] * 10 + ["d"] * 3], dtype=object
+    ).T
+    ohe = OneHotEncoder(
+        categories=[["c", "d", "a", "b"]],
+        sparse_output=False,
+        handle_unknown="infrequent_if_exist",
+        max_categories=2,
+    ).fit(X_train)
+
+    assert_array_equal(ohe.infrequent_categories_, [["c", "d", "a"]])
+
+    X_test = [["b"], ["a"], ["c"], ["d"], ["e"]]
+    expected = np.array([[1, 0], [0, 1], [0, 1], [0, 1], [0, 1]])
+
+    X_trans = ohe.transform(X_test)
+    assert_allclose(expected, X_trans)
+
+    # 'infrequent' is used to denote the infrequent categories for
+    # `inverse_transform`
+    expected_inv = [[col] for col in ["b"] + ["infrequent_sklearn"] * 4]
+    X_inv = ohe.inverse_transform(X_trans)
+    assert_array_equal(expected_inv, X_inv)
+
+
+def test_ohe_infrequent_three_levels_user_cats():
+    """Test that the order of the categories provided by a user is respected.
+    In this case 'c' is encoded as the first category and 'b' is encoded
+    as the second one."""
+
+    X_train = np.array(
+        [["a"] * 5 + ["b"] * 20 + ["c"] * 10 + ["d"] * 3], dtype=object
+    ).T
+    ohe = OneHotEncoder(
+        categories=[["c", "d", "b", "a"]],
+        sparse_output=False,
+        handle_unknown="infrequent_if_exist",
+        max_categories=3,
+    ).fit(X_train)
+
+    assert_array_equal(ohe.infrequent_categories_, [["d", "a"]])
+
+    X_test = [["b"], ["a"], ["c"], ["d"], ["e"]]
+    expected = np.array([[0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 0, 1]])
+
+    X_trans = ohe.transform(X_test)
+    assert_allclose(expected, X_trans)
+
+    # 'infrequent' is used to denote the infrequent categories for
+    # `inverse_transform`
+    expected_inv = [
+        ["b"],
+        ["infrequent_sklearn"],
+        ["c"],
+        ["infrequent_sklearn"],
+        ["infrequent_sklearn"],
+    ]
+    X_inv = ohe.inverse_transform(X_trans)
+    assert_array_equal(expected_inv, X_inv)
+
+
+def test_ohe_infrequent_mixed():
+    """Test infrequent categories where feature 0 has infrequent categories,
+    and feature 1 does not."""
+
+    # X[:, 0] 1 and 2 are infrequent
+    # X[:, 1] nothing is infrequent
+    X = np.c_[[0, 1, 3, 3, 3, 3, 2, 0, 3], [0, 0, 0, 0, 1, 1, 1, 1, 1]]
+
+    ohe = OneHotEncoder(max_categories=3, drop="if_binary", sparse_output=False)
+    ohe.fit(X)
+
+    X_test = [[3, 0], [1, 1]]
+    X_trans = ohe.transform(X_test)
+
+    # feature 1 is binary so it drops a category 0
+    assert_allclose(X_trans, [[0, 1, 0, 0], [0, 0, 1, 1]])
+
+
+def test_ohe_infrequent_multiple_categories():
+    """Test infrequent categories with feature matrix with 3 features."""
+
+    X = np.c_[
+        [0, 1, 3, 3, 3, 3, 2, 0, 3],
+        [0, 0, 5, 1, 1, 10, 5, 5, 0],
+        [1, 0, 1, 0, 1, 0, 1, 0, 1],
+    ]
+
+    ohe = OneHotEncoder(
+        categories="auto", max_categories=3, handle_unknown="infrequent_if_exist"
+    )
+    # X[:, 0] 1 and 2 are infrequent
+    # X[:, 1] 1 and 10 are infrequent
+    # X[:, 2] nothing is infrequent
+
+    X_trans = ohe.fit_transform(X).toarray()
+    assert_array_equal(ohe.infrequent_categories_[0], [1, 2])
+    assert_array_equal(ohe.infrequent_categories_[1], [1, 10])
+    assert_array_equal(ohe.infrequent_categories_[2], None)
+
+    # 'infrequent' is used to denote the infrequent categories
+    # For the first column, 1 and 2 have the same frequency. In this case,
+    # 1 will be chosen to be the feature name because is smaller lexiconically
+    feature_names = ohe.get_feature_names_out()
+    assert_array_equal(
+        [
+            "x0_0",
+            "x0_3",
+            "x0_infrequent_sklearn",
+            "x1_0",
+            "x1_5",
+            "x1_infrequent_sklearn",
+            "x2_0",
+            "x2_1",
+        ],
+        feature_names,
+    )
+
+    expected = [
+        [1, 0, 0, 1, 0, 0, 0, 1],
+        [0, 0, 1, 1, 0, 0, 1, 0],
+        [0, 1, 0, 0, 1, 0, 0, 1],
+        [0, 1, 0, 0, 0, 1, 1, 0],
+        [0, 1, 0, 0, 0, 1, 0, 1],
+        [0, 1, 0, 0, 0, 1, 1, 0],
+        [0, 0, 1, 0, 1, 0, 0, 1],
+        [1, 0, 0, 0, 1, 0, 1, 0],
+        [0, 1, 0, 1, 0, 0, 0, 1],
+    ]
+
+    assert_allclose(expected, X_trans)
+
+    X_test = [[3, 1, 2], [4, 0, 3]]
+
+    X_test_trans = ohe.transform(X_test)
+
+    # X[:, 2] does not have an infrequent category, thus it is encoded as all
+    # zeros
+    expected = [[0, 1, 0, 0, 0, 1, 0, 0], [0, 0, 1, 1, 0, 0, 0, 0]]
+    assert_allclose(expected, X_test_trans.toarray())
+
+    X_inv = ohe.inverse_transform(X_test_trans)
+    expected_inv = np.array(
+        [[3, "infrequent_sklearn", None], ["infrequent_sklearn", 0, None]], dtype=object
+    )
+    assert_array_equal(expected_inv, X_inv)
+
+    # error for unknown categories
+    ohe = OneHotEncoder(
+        categories="auto", max_categories=3, handle_unknown="error"
+    ).fit(X)
+    with pytest.raises(ValueError, match="Found unknown categories"):
+        ohe.transform(X_test)
+
+    # only infrequent or known categories
+    X_test = [[1, 1, 1], [3, 10, 0]]
+    X_test_trans = ohe.transform(X_test)
+
+    expected = [[0, 0, 1, 0, 0, 1, 0, 1], [0, 1, 0, 0, 0, 1, 1, 0]]
+    assert_allclose(expected, X_test_trans.toarray())
+
+    X_inv = ohe.inverse_transform(X_test_trans)
+
+    expected_inv = np.array(
+        [["infrequent_sklearn", "infrequent_sklearn", 1], [3, "infrequent_sklearn", 0]],
+        dtype=object,
+    )
+    assert_array_equal(expected_inv, X_inv)
+
+
+def test_ohe_infrequent_multiple_categories_dtypes():
+    """Test infrequent categories with a pandas dataframe with multiple dtypes."""
+
+    pd = pytest.importorskip("pandas")
+    X = pd.DataFrame(
+        {
+            "str": ["a", "f", "c", "f", "f", "a", "c", "b", "b"],
+            "int": [5, 3, 0, 10, 10, 12, 0, 3, 5],
+        },
+        columns=["str", "int"],
+    )
+
+    ohe = OneHotEncoder(
+        categories="auto", max_categories=3, handle_unknown="infrequent_if_exist"
+    )
+    # X[:, 0] 'a', 'b', 'c' have the same frequency. 'a' and 'b' will be
+    # considered infrequent because they are greater
+
+    # X[:, 1] 0, 3, 5, 10 has frequency 2 and 12 has frequency 1.
+    # 0, 3, 12 will be considered infrequent
+
+    X_trans = ohe.fit_transform(X).toarray()
+    assert_array_equal(ohe.infrequent_categories_[0], ["a", "b"])
+    assert_array_equal(ohe.infrequent_categories_[1], [0, 3, 12])
+
+    expected = [
+        [0, 0, 1, 1, 0, 0],
+        [0, 1, 0, 0, 0, 1],
+        [1, 0, 0, 0, 0, 1],
+        [0, 1, 0, 0, 1, 0],
+        [0, 1, 0, 0, 1, 0],
+        [0, 0, 1, 0, 0, 1],
+        [1, 0, 0, 0, 0, 1],
+        [0, 0, 1, 0, 0, 1],
+        [0, 0, 1, 1, 0, 0],
+    ]
+
+    assert_allclose(expected, X_trans)
+
+    X_test = pd.DataFrame({"str": ["b", "f"], "int": [14, 12]}, columns=["str", "int"])
+
+    expected = [[0, 0, 1, 0, 0, 1], [0, 1, 0, 0, 0, 1]]
+    X_test_trans = ohe.transform(X_test)
+    assert_allclose(expected, X_test_trans.toarray())
+
+    X_inv = ohe.inverse_transform(X_test_trans)
+    expected_inv = np.array(
+        [["infrequent_sklearn", "infrequent_sklearn"], ["f", "infrequent_sklearn"]],
+        dtype=object,
+    )
+    assert_array_equal(expected_inv, X_inv)
+
+    # only infrequent or known categories
+    X_test = pd.DataFrame({"str": ["c", "b"], "int": [12, 5]}, columns=["str", "int"])
+    X_test_trans = ohe.transform(X_test).toarray()
+    expected = [[1, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0]]
+    assert_allclose(expected, X_test_trans)
+
+    X_inv = ohe.inverse_transform(X_test_trans)
+    expected_inv = np.array(
+        [["c", "infrequent_sklearn"], ["infrequent_sklearn", 5]], dtype=object
+    )
+    assert_array_equal(expected_inv, X_inv)
+
+
+@pytest.mark.parametrize("kwargs", [{"min_frequency": 21, "max_categories": 1}])
+def test_ohe_infrequent_one_level_errors(kwargs):
+    """All user provided categories are infrequent."""
+    X_train = np.array([["a"] * 5 + ["b"] * 20 + ["c"] * 10 + ["d"] * 2]).T
+
+    ohe = OneHotEncoder(
+        handle_unknown="infrequent_if_exist", sparse_output=False, **kwargs
+    )
+    ohe.fit(X_train)
+
+    X_trans = ohe.transform([["a"]])
+    assert_allclose(X_trans, [[1]])
+
+
+@pytest.mark.parametrize("kwargs", [{"min_frequency": 2, "max_categories": 3}])
+def test_ohe_infrequent_user_cats_unknown_training_errors(kwargs):
+    """All user provided categories are infrequent."""
 
+    X_train = np.array([["e"] * 3], dtype=object).T
+    ohe = OneHotEncoder(
+        categories=[["c", "d", "a", "b"]],
+        sparse_output=False,
+        handle_unknown="infrequent_if_exist",
+        **kwargs,
+    ).fit(X_train)
 
-@pytest.mark.parametrize('input_dtype', ['O', 'U'])
-@pytest.mark.parametrize('category_dtype', ['O', 'U'])
-@pytest.mark.parametrize('array_type', ['list', 'array', 'dataframe'])
-def test_encoders_unicode_categories(input_dtype, category_dtype, array_type):
-    """Check that encoding work with string and object dtypes.
+    X_trans = ohe.transform([["a"], ["e"]])
+    assert_allclose(X_trans, [[1], [1]])
+
+
+# deliberately omit 'OS' as an invalid combo
+@pytest.mark.parametrize(
+    "input_dtype, category_dtype", ["OO", "OU", "UO", "UU", "SO", "SU", "SS"]
+)
+@pytest.mark.parametrize("array_type", ["list", "array", "dataframe"])
+def test_encoders_string_categories(input_dtype, category_dtype, array_type):
+    """Check that encoding work with object, unicode, and byte string dtypes.
     Non-regression test for:
     https://github.com/scikit-learn/scikit-learn/issues/15616
     https://github.com/scikit-learn/scikit-learn/issues/15726
+    https://github.com/scikit-learn/scikit-learn/issues/19677
     """
 
-    X = np.array([['b'], ['a']], dtype=input_dtype)
-    categories = [np.array(['b', 'a'], dtype=category_dtype)]
-    ohe = OneHotEncoder(categories=categories, sparse=False).fit(X)
+    X = np.array([["b"], ["a"]], dtype=input_dtype)
+    categories = [np.array(["b", "a"], dtype=category_dtype)]
+    ohe = OneHotEncoder(categories=categories, sparse_output=False).fit(X)
 
-    X_test = _convert_container([['a'], ['a'], ['b'], ['a']], array_type)
+    X_test = _convert_container(
+        [["a"], ["a"], ["b"], ["a"]], array_type, dtype=input_dtype
+    )
     X_trans = ohe.transform(X_test)
 
     expected = np.array([[0, 1], [0, 1], [1, 0], [0, 1]])
@@ -854,145 +1411,318 @@ def test_encoders_unicode_categories(input_dtype, category_dtype, array_type):
     assert_array_equal(X_trans, expected)
 
 
+def test_mixed_string_bytes_categoricals():
+    """Check that this mixture of predefined categories and X raises an error.
+
+    Categories defined as bytes can not easily be compared to data that is
+    a string.
+    """
+    # data as unicode
+    X = np.array([["b"], ["a"]], dtype="U")
+    # predefined categories as bytes
+    categories = [np.array(["b", "a"], dtype="S")]
+    ohe = OneHotEncoder(categories=categories, sparse_output=False)
+
+    msg = re.escape(
+        "In column 0, the predefined categories have type 'bytes' which is incompatible"
+        " with values of type 'str_'."
+    )
+
+    with pytest.raises(ValueError, match=msg):
+        ohe.fit(X)
+
+
 @pytest.mark.parametrize("missing_value", [np.nan, None])
 def test_ohe_missing_values_get_feature_names(missing_value):
     # encoder with missing values with object dtypes
-    X = np.array([['a', 'b', missing_value, 'a', missing_value]],
-                 dtype=object).T
-    ohe = OneHotEncoder(sparse=False, handle_unknown='ignore').fit(X)
-    names = ohe.get_feature_names()
-    assert_array_equal(names, ['x0_a', 'x0_b', f'x0_{missing_value}'])
+    X = np.array([["a", "b", missing_value, "a", missing_value]], dtype=object).T
+    ohe = OneHotEncoder(sparse_output=False, handle_unknown="ignore").fit(X)
+    names = ohe.get_feature_names_out()
+    assert_array_equal(names, ["x0_a", "x0_b", f"x0_{missing_value}"])
 
 
 def test_ohe_missing_value_support_pandas():
     # check support for pandas with mixed dtypes and missing values
-    pd = pytest.importorskip('pandas')
-    df = pd.DataFrame({
-        'col1': ['dog', 'cat', None, 'cat'],
-        'col2': np.array([3, 0, 4, np.nan], dtype=float)
-    }, columns=['col1', 'col2'])
-    expected_df_trans = np.array([
-        [0, 1, 0, 0, 1, 0, 0],
-        [1, 0, 0, 1, 0, 0, 0],
-        [0, 0, 1, 0, 0, 1, 0],
-        [1, 0, 0, 0, 0, 0, 1],
-    ])
+    pd = pytest.importorskip("pandas")
+    df = pd.DataFrame(
+        {
+            "col1": ["dog", "cat", None, "cat"],
+            "col2": np.array([3, 0, 4, np.nan], dtype=float),
+        },
+        columns=["col1", "col2"],
+    )
+    expected_df_trans = np.array(
+        [
+            [0, 1, 0, 0, 1, 0, 0],
+            [1, 0, 0, 1, 0, 0, 0],
+            [0, 0, 1, 0, 0, 1, 0],
+            [1, 0, 0, 0, 0, 0, 1],
+        ]
+    )
 
     Xtr = check_categorical_onehot(df)
     assert_allclose(Xtr, expected_df_trans)
 
 
-@pytest.mark.parametrize('pd_nan_type', ['pd.NA', 'np.nan'])
-def test_ohe_missing_value_support_pandas_categorical(pd_nan_type):
+@pytest.mark.parametrize("handle_unknown", ["ignore", "infrequent_if_exist", "warn"])
+@pytest.mark.parametrize("pd_nan_type", ["pd.NA", "np.nan"])
+def test_ohe_missing_value_support_pandas_categorical(pd_nan_type, handle_unknown):
     # checks pandas dataframe with categorical features
-    if pd_nan_type == 'pd.NA':
-        # pd.NA is in pandas 1.0
-        pd = pytest.importorskip('pandas', minversion="1.0")
-        pd_missing_value = pd.NA
-    else:  # np.nan
-        pd = pytest.importorskip('pandas')
-        pd_missing_value = np.nan
-
-    df = pd.DataFrame({
-        'col1': pd.Series(['c', 'a', pd_missing_value, 'b', 'a'],
-                          dtype='category'),
-    })
-    expected_df_trans = np.array([
-        [0, 0, 1, 0],
-        [1, 0, 0, 0],
-        [0, 0, 0, 1],
-        [0, 1, 0, 0],
-        [1, 0, 0, 0],
-    ])
-
-    ohe = OneHotEncoder(sparse=False, handle_unknown='ignore')
+    pd = pytest.importorskip("pandas")
+
+    pd_missing_value = pd.NA if pd_nan_type == "pd.NA" else np.nan
+
+    df = pd.DataFrame(
+        {
+            "col1": pd.Series(["c", "a", pd_missing_value, "b", "a"], dtype="category"),
+        }
+    )
+    expected_df_trans = np.array(
+        [
+            [0, 0, 1, 0],
+            [1, 0, 0, 0],
+            [0, 0, 0, 1],
+            [0, 1, 0, 0],
+            [1, 0, 0, 0],
+        ]
+    )
+
+    ohe = OneHotEncoder(sparse_output=False, handle_unknown=handle_unknown)
     df_trans = ohe.fit_transform(df)
     assert_allclose(expected_df_trans, df_trans)
 
     assert len(ohe.categories_) == 1
-    assert_array_equal(ohe.categories_[0][:-1], ['a', 'b', 'c'])
+    assert_array_equal(ohe.categories_[0][:-1], ["a", "b", "c"])
     assert np.isnan(ohe.categories_[0][-1])
 
 
+@pytest.mark.parametrize("handle_unknown", ["ignore", "infrequent_if_exist", "warn"])
+def test_ohe_drop_first_handle_unknown_ignore_warns(handle_unknown):
+    """Check drop='first' and handle_unknown='ignore'/'infrequent_if_exist'
+    during transform."""
+    X = [["a", 0], ["b", 2], ["b", 1]]
+
+    ohe = OneHotEncoder(
+        drop="first", sparse_output=False, handle_unknown=handle_unknown
+    )
+    X_trans = ohe.fit_transform(X)
+
+    X_expected = np.array(
+        [
+            [0, 0, 0],
+            [1, 0, 1],
+            [1, 1, 0],
+        ]
+    )
+    assert_allclose(X_trans, X_expected)
+
+    # Both categories are unknown
+    X_test = [["c", 3]]
+    X_expected = np.array([[0, 0, 0]])
+
+    warn_msg = (
+        r"Found unknown categories in columns \[0, 1\] during "
+        "transform. These unknown categories will be encoded as all "
+        "zeros"
+    )
+    with pytest.warns(UserWarning, match=warn_msg):
+        X_trans = ohe.transform(X_test)
+    assert_allclose(X_trans, X_expected)
+
+    # inverse_transform maps to None
+    X_inv = ohe.inverse_transform(X_expected)
+    assert_array_equal(X_inv, np.array([["a", 0]], dtype=object))
+
+
+@pytest.mark.parametrize("handle_unknown", ["ignore", "infrequent_if_exist", "warn"])
+def test_ohe_drop_if_binary_handle_unknown_ignore_warns(handle_unknown):
+    """Check drop='if_binary' and handle_unknown='ignore' during transform."""
+    X = [["a", 0], ["b", 2], ["b", 1]]
+
+    ohe = OneHotEncoder(
+        drop="if_binary", sparse_output=False, handle_unknown=handle_unknown
+    )
+    X_trans = ohe.fit_transform(X)
+
+    X_expected = np.array(
+        [
+            [0, 1, 0, 0],
+            [1, 0, 0, 1],
+            [1, 0, 1, 0],
+        ]
+    )
+    assert_allclose(X_trans, X_expected)
+
+    # Both categories are unknown
+    X_test = [["c", 3]]
+    X_expected = np.array([[0, 0, 0, 0]])
+
+    warn_msg = (
+        r"Found unknown categories in columns \[0, 1\] during "
+        "transform. These unknown categories will be encoded as all "
+        "zeros"
+    )
+    with pytest.warns(UserWarning, match=warn_msg):
+        X_trans = ohe.transform(X_test)
+    assert_allclose(X_trans, X_expected)
+
+    # inverse_transform maps to None
+    X_inv = ohe.inverse_transform(X_expected)
+    assert_array_equal(X_inv, np.array([["a", None]], dtype=object))
+
+
+@pytest.mark.parametrize("handle_unknown", ["ignore", "infrequent_if_exist", "warn"])
+def test_ohe_drop_first_explicit_categories(handle_unknown):
+    """Check drop='first' and handle_unknown='ignore'/'infrequent_if_exist'
+    during fit with categories passed in."""
+
+    X = [["a", 0], ["b", 2], ["b", 1]]
+
+    ohe = OneHotEncoder(
+        drop="first",
+        sparse_output=False,
+        handle_unknown=handle_unknown,
+        categories=[["b", "a"], [1, 2]],
+    )
+    ohe.fit(X)
+
+    X_test = [["c", 1]]
+    X_expected = np.array([[0, 0]])
+
+    warn_msg = (
+        r"Found unknown categories in columns \[0\] during transform. "
+        r"These unknown categories will be encoded as all zeros"
+    )
+    with pytest.warns(UserWarning, match=warn_msg):
+        X_trans = ohe.transform(X_test)
+    assert_allclose(X_trans, X_expected)
+
+
+def test_ohe_more_informative_error_message():
+    """Raise informative error message when pandas output and sparse_output=True."""
+    pd = pytest.importorskip("pandas")
+    df = pd.DataFrame({"a": [1, 2, 3], "b": ["z", "b", "b"]}, columns=["a", "b"])
+
+    ohe = OneHotEncoder(sparse_output=True)
+    ohe.set_output(transform="pandas")
+
+    msg = (
+        "Pandas output does not support sparse data. Set "
+        "sparse_output=False to output pandas dataframes or disable Pandas output"
+    )
+    with pytest.raises(ValueError, match=msg):
+        ohe.fit_transform(df)
+
+    ohe.fit(df)
+    with pytest.raises(ValueError, match=msg):
+        ohe.transform(df)
+
+
 def test_ordinal_encoder_passthrough_missing_values_float_errors_dtype():
     """Test ordinal encoder with nan passthrough fails when dtype=np.int32."""
 
     X = np.array([[np.nan, 3.0, 1.0, 3.0]]).T
     oe = OrdinalEncoder(dtype=np.int32)
 
-    msg = (r"There are missing values in features \[0\]. For OrdinalEncoder "
-           "to passthrough missing values, the dtype parameter must be a "
-           "float")
+    msg = (
+        r"There are missing values in features \[0\]. For OrdinalEncoder "
+        f"to encode missing values with dtype: {np.int32}"
+    )
     with pytest.raises(ValueError, match=msg):
         oe.fit(X)
 
 
-def test_ordinal_encoder_passthrough_missing_values_float():
+@pytest.mark.parametrize("encoded_missing_value", [np.nan, -2])
+def test_ordinal_encoder_passthrough_missing_values_float(encoded_missing_value):
     """Test ordinal encoder with nan on float dtypes."""
 
     X = np.array([[np.nan, 3.0, 1.0, 3.0]], dtype=np.float64).T
-    oe = OrdinalEncoder().fit(X)
+    oe = OrdinalEncoder(encoded_missing_value=encoded_missing_value).fit(X)
 
     assert len(oe.categories_) == 1
+
     assert_allclose(oe.categories_[0], [1.0, 3.0, np.nan])
 
     X_trans = oe.transform(X)
-    assert_allclose(X_trans, [[np.nan], [1.0], [0.0], [1.0]])
+    assert_allclose(X_trans, [[encoded_missing_value], [1.0], [0.0], [1.0]])
 
     X_inverse = oe.inverse_transform(X_trans)
     assert_allclose(X_inverse, X)
 
 
-@pytest.mark.parametrize('pd_nan_type', ['pd.NA', 'np.nan'])
-def test_ordinal_encoder_missing_value_support_pandas_categorical(pd_nan_type):
+@pytest.mark.parametrize("pd_nan_type", ["pd.NA", "np.nan"])
+@pytest.mark.parametrize("encoded_missing_value", [np.nan, -2])
+def test_ordinal_encoder_missing_value_support_pandas_categorical(
+    pd_nan_type, encoded_missing_value
+):
     """Check ordinal encoder is compatible with pandas."""
     # checks pandas dataframe with categorical features
-    if pd_nan_type == 'pd.NA':
-        # pd.NA is in pandas 1.0
-        pd = pytest.importorskip('pandas', minversion="1.0")
-        pd_missing_value = pd.NA
-    else:  # np.nan
-        pd = pytest.importorskip('pandas')
-        pd_missing_value = np.nan
-
-    df = pd.DataFrame({
-        'col1': pd.Series(['c', 'a', pd_missing_value, 'b', 'a'],
-                          dtype='category'),
-    })
-
-    oe = OrdinalEncoder().fit(df)
+    pd = pytest.importorskip("pandas")
+
+    pd_missing_value = pd.NA if pd_nan_type == "pd.NA" else np.nan
+
+    df = pd.DataFrame(
+        {
+            "col1": pd.Series(["c", "a", pd_missing_value, "b", "a"], dtype="category"),
+        }
+    )
+
+    oe = OrdinalEncoder(encoded_missing_value=encoded_missing_value).fit(df)
     assert len(oe.categories_) == 1
-    assert_array_equal(oe.categories_[0][:3], ['a', 'b', 'c'])
+    assert_array_equal(oe.categories_[0][:3], ["a", "b", "c"])
     assert np.isnan(oe.categories_[0][-1])
 
     df_trans = oe.transform(df)
 
-    assert_allclose(df_trans, [[2.0], [0.0], [np.nan], [1.0], [0.0]])
+    assert_allclose(df_trans, [[2.0], [0.0], [encoded_missing_value], [1.0], [0.0]])
 
     X_inverse = oe.inverse_transform(df_trans)
     assert X_inverse.shape == (5, 1)
-    assert_array_equal(X_inverse[:2, 0], ['c', 'a'])
-    assert_array_equal(X_inverse[3:, 0], ['b', 'a'])
+    assert_array_equal(X_inverse[:2, 0], ["c", "a"])
+    assert_array_equal(X_inverse[3:, 0], ["b", "a"])
     assert np.isnan(X_inverse[2, 0])
 
 
-@pytest.mark.parametrize("X, X2, cats, cat_dtype", [
-    ((np.array([['a', np.nan]], dtype=object).T,
-      np.array([['a', 'b']], dtype=object).T,
-     [np.array(['a', np.nan, 'd'], dtype=object)], np.object_)),
-    ((np.array([['a', np.nan]], dtype=object).T,
-      np.array([['a', 'b']], dtype=object).T,
-     [np.array(['a', np.nan, 'd'], dtype=object)], np.object_)),
-    ((np.array([[2.0, np.nan]], dtype=np.float64).T,
-      np.array([[3.0]], dtype=np.float64).T,
-     [np.array([2.0, 4.0, np.nan])], np.float64)),
-    ], ids=['object-None-missing-value', 'object-nan-missing_value',
-            'numeric-missing-value'])
+@pytest.mark.parametrize(
+    "X, X2, cats, cat_dtype",
+    [
+        (
+            (
+                np.array([["a", np.nan]], dtype=object).T,
+                np.array([["a", "b"]], dtype=object).T,
+                [np.array(["a", "d", np.nan], dtype=object)],
+                np.object_,
+            )
+        ),
+        (
+            (
+                np.array([["a", np.nan]], dtype=object).T,
+                np.array([["a", "b"]], dtype=object).T,
+                [np.array(["a", "d", np.nan], dtype=object)],
+                np.object_,
+            )
+        ),
+        (
+            (
+                np.array([[2.0, np.nan]], dtype=np.float64).T,
+                np.array([[3.0]], dtype=np.float64).T,
+                [np.array([2.0, 4.0, np.nan])],
+                np.float64,
+            )
+        ),
+    ],
+    ids=[
+        "object-None-missing-value",
+        "object-nan-missing_value",
+        "numeric-missing-value",
+    ],
+)
 def test_ordinal_encoder_specified_categories_missing_passthrough(
-        X, X2, cats, cat_dtype):
+    X, X2, cats, cat_dtype
+):
     """Test ordinal encoder for specified categories."""
     oe = OrdinalEncoder(categories=cats)
-    exp = np.array([[0.], [np.nan]])
+    exp = np.array([[0.0], [np.nan]])
     assert_array_equal(oe.fit_transform(X), exp)
     # manually specified categories should have same dtype as
     # the data when coerced from lists
@@ -1005,29 +1735,633 @@ def test_ordinal_encoder_specified_categories_missing_passthrough(
         oe.fit(X2)
 
 
-@pytest.mark.parametrize("X, expected_X_trans, X_test", [
-    (np.array([[1.0, np.nan, 3.0]]).T,
-     np.array([[0.0, np.nan, 1.0]]).T,
-     np.array([[4.0]])),
-    (np.array([[1.0, 4.0, 3.0]]).T,
-     np.array([[0.0, 2.0, 1.0]]).T,
-     np.array([[np.nan]])),
-    (np.array([['c', np.nan, 'b']], dtype=object).T,
-     np.array([[1.0, np.nan, 0.0]]).T,
-     np.array([['d']], dtype=object)),
-    (np.array([['c', 'a', 'b']], dtype=object).T,
-     np.array([[2.0, 0.0, 1.0]]).T,
-     np.array([[np.nan]], dtype=object)),
-])
-def test_ordinal_encoder_handle_missing_and_unknown(
-        X, expected_X_trans, X_test
-):
+@pytest.mark.parametrize("Encoder", [OneHotEncoder, OrdinalEncoder])
+def test_encoder_duplicate_specified_categories(Encoder):
+    """Test encoder for specified categories have duplicate values.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/27088
+    """
+    cats = [np.array(["a", "b", "a"], dtype=object)]
+    enc = Encoder(categories=cats)
+    X = np.array([["a", "b"]], dtype=object).T
+    with pytest.raises(
+        ValueError, match="the predefined categories contain duplicate elements."
+    ):
+        enc.fit(X)
+
+
+@pytest.mark.parametrize(
+    "X, expected_X_trans, X_test",
+    [
+        (
+            np.array([[1.0, np.nan, 3.0]]).T,
+            np.array([[0.0, np.nan, 1.0]]).T,
+            np.array([[4.0]]),
+        ),
+        (
+            np.array([[1.0, 4.0, 3.0]]).T,
+            np.array([[0.0, 2.0, 1.0]]).T,
+            np.array([[np.nan]]),
+        ),
+        (
+            np.array([["c", np.nan, "b"]], dtype=object).T,
+            np.array([[1.0, np.nan, 0.0]]).T,
+            np.array([["d"]], dtype=object),
+        ),
+        (
+            np.array([["c", "a", "b"]], dtype=object).T,
+            np.array([[2.0, 0.0, 1.0]]).T,
+            np.array([[np.nan]], dtype=object),
+        ),
+    ],
+)
+def test_ordinal_encoder_handle_missing_and_unknown(X, expected_X_trans, X_test):
     """Test the interaction between missing values and handle_unknown"""
 
-    oe = OrdinalEncoder(handle_unknown="use_encoded_value",
-                        unknown_value=-1)
+    oe = OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=-1)
 
     X_trans = oe.fit_transform(X)
     assert_allclose(X_trans, expected_X_trans)
 
     assert_allclose(oe.transform(X_test), [[-1.0]])
+
+
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_ordinal_encoder_sparse(csr_container):
+    """Check that we raise proper error with sparse input in OrdinalEncoder.
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/19878
+    """
+    X = np.array([[3, 2, 1], [0, 1, 1]])
+    X_sparse = csr_container(X)
+
+    encoder = OrdinalEncoder()
+
+    err_msg = "Sparse data was passed, but dense data is required"
+    with pytest.raises(TypeError, match=err_msg):
+        encoder.fit(X_sparse)
+    with pytest.raises(TypeError, match=err_msg):
+        encoder.fit_transform(X_sparse)
+
+    X_trans = encoder.fit_transform(X)
+    X_trans_sparse = csr_container(X_trans)
+    with pytest.raises(TypeError, match=err_msg):
+        encoder.inverse_transform(X_trans_sparse)
+
+
+def test_ordinal_encoder_fit_with_unseen_category():
+    """Check OrdinalEncoder.fit works with unseen category when
+    `handle_unknown="use_encoded_value"`.
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/19872
+    """
+    X = np.array([0, 0, 1, 0, 2, 5])[:, np.newaxis]
+    oe = OrdinalEncoder(
+        categories=[[-1, 0, 1]], handle_unknown="use_encoded_value", unknown_value=-999
+    )
+    oe.fit(X)
+
+    oe = OrdinalEncoder(categories=[[-1, 0, 1]], handle_unknown="error")
+    with pytest.raises(ValueError, match="Found unknown categories"):
+        oe.fit(X)
+
+
+@pytest.mark.parametrize(
+    "X_train",
+    [
+        [["AA", "B"]],
+        np.array([["AA", "B"]], dtype="O"),
+        np.array([["AA", "B"]], dtype="U"),
+    ],
+)
+@pytest.mark.parametrize(
+    "X_test",
+    [
+        [["A", "B"]],
+        np.array([["A", "B"]], dtype="O"),
+        np.array([["A", "B"]], dtype="U"),
+    ],
+)
+def test_ordinal_encoder_handle_unknown_string_dtypes(X_train, X_test):
+    """Checks that `OrdinalEncoder` transforms string dtypes.
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/19872
+    """
+    enc = OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=-9)
+    enc.fit(X_train)
+
+    X_trans = enc.transform(X_test)
+    assert_allclose(X_trans, [[-9, 0]])
+
+
+def test_ordinal_encoder_python_integer():
+    """Check that `OrdinalEncoder` accepts Python integers that are potentially
+    larger than 64 bits.
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/20721
+    """
+    X = np.array(
+        [
+            44253463435747313673,
+            9867966753463435747313673,
+            44253462342215747313673,
+            442534634357764313673,
+        ]
+    ).reshape(-1, 1)
+    encoder = OrdinalEncoder().fit(X)
+    assert_array_equal(encoder.categories_, np.sort(X, axis=0).T)
+    X_trans = encoder.transform(X)
+    assert_array_equal(X_trans, [[0], [3], [2], [1]])
+
+
+def test_ordinal_encoder_features_names_out_pandas():
+    """Check feature names out is same as the input."""
+    pd = pytest.importorskip("pandas")
+
+    names = ["b", "c", "a"]
+    X = pd.DataFrame([[1, 2, 3]], columns=names)
+    enc = OrdinalEncoder().fit(X)
+
+    feature_names_out = enc.get_feature_names_out()
+    assert_array_equal(names, feature_names_out)
+
+
+def test_ordinal_encoder_unknown_missing_interaction():
+    """Check interactions between encode_unknown and missing value encoding."""
+
+    X = np.array([["a"], ["b"], [np.nan]], dtype=object)
+
+    oe = OrdinalEncoder(
+        handle_unknown="use_encoded_value",
+        unknown_value=np.nan,
+        encoded_missing_value=-3,
+    ).fit(X)
+
+    X_trans = oe.transform(X)
+    assert_allclose(X_trans, [[0], [1], [-3]])
+
+    # "c" is unknown and is mapped to np.nan
+    # "None" is a missing value and is set to -3
+    X_test = np.array([["c"], [np.nan]], dtype=object)
+    X_test_trans = oe.transform(X_test)
+    assert_allclose(X_test_trans, [[np.nan], [-3]])
+
+    # Non-regression test for #24082
+    X_roundtrip = oe.inverse_transform(X_test_trans)
+
+    # np.nan is unknown so it maps to None
+    assert X_roundtrip[0][0] is None
+
+    # -3 is the encoded missing value so it maps back to nan
+    assert np.isnan(X_roundtrip[1][0])
+
+
+@pytest.mark.parametrize("with_pandas", [True, False])
+def test_ordinal_encoder_encoded_missing_value_error(with_pandas):
+    """Check OrdinalEncoder errors when encoded_missing_value is used by
+    an known category."""
+    X = np.array([["a", "dog"], ["b", "cat"], ["c", np.nan]], dtype=object)
+
+    # The 0-th feature has no missing values so it is not included in the list of
+    # features
+    error_msg = (
+        r"encoded_missing_value \(1\) is already used to encode a known category "
+        r"in features: "
+    )
+
+    if with_pandas:
+        pd = pytest.importorskip("pandas")
+        X = pd.DataFrame(X, columns=["letter", "pet"])
+        error_msg = error_msg + r"\['pet'\]"
+    else:
+        error_msg = error_msg + r"\[1\]"
+
+    oe = OrdinalEncoder(encoded_missing_value=1)
+
+    with pytest.raises(ValueError, match=error_msg):
+        oe.fit(X)
+
+
+@pytest.mark.parametrize(
+    "X_train, X_test_trans_expected, X_roundtrip_expected",
+    [
+        (
+            # missing value is not in training set
+            # inverse transform will considering encoded nan as unknown
+            np.array([["a"], ["1"]], dtype=object),
+            [[0], [np.nan], [np.nan]],
+            np.asarray([["1"], [None], [None]], dtype=object),
+        ),
+        (
+            # missing value in training set,
+            # inverse transform will considering encoded nan as missing
+            np.array([[np.nan], ["1"], ["a"]], dtype=object),
+            [[0], [np.nan], [np.nan]],
+            np.asarray([["1"], [np.nan], [np.nan]], dtype=object),
+        ),
+    ],
+)
+def test_ordinal_encoder_unknown_missing_interaction_both_nan(
+    X_train, X_test_trans_expected, X_roundtrip_expected
+):
+    """Check transform when unknown_value and encoded_missing_value is nan.
+
+    Non-regression test for #24082.
+    """
+    oe = OrdinalEncoder(
+        handle_unknown="use_encoded_value",
+        unknown_value=np.nan,
+        encoded_missing_value=np.nan,
+    ).fit(X_train)
+
+    X_test = np.array([["1"], [np.nan], ["b"]])
+    X_test_trans = oe.transform(X_test)
+
+    # both nan and unknown are encoded as nan
+    assert_allclose(X_test_trans, X_test_trans_expected)
+    X_roundtrip = oe.inverse_transform(X_test_trans)
+
+    n_samples = X_roundtrip_expected.shape[0]
+    for i in range(n_samples):
+        expected_val = X_roundtrip_expected[i, 0]
+        val = X_roundtrip[i, 0]
+
+        if expected_val is None:
+            assert val is None
+        elif is_scalar_nan(expected_val):
+            assert np.isnan(val)
+        else:
+            assert val == expected_val
+
+
+def test_one_hot_encoder_set_output():
+    """Check OneHotEncoder works with set_output."""
+    pd = pytest.importorskip("pandas")
+
+    X_df = pd.DataFrame({"A": ["a", "b"], "B": [1, 2]})
+    ohe = OneHotEncoder()
+
+    ohe.set_output(transform="pandas")
+
+    match = "Pandas output does not support sparse data. Set sparse_output=False"
+    with pytest.raises(ValueError, match=match):
+        ohe.fit_transform(X_df)
+
+    ohe_default = OneHotEncoder(sparse_output=False).set_output(transform="default")
+    ohe_pandas = OneHotEncoder(sparse_output=False).set_output(transform="pandas")
+
+    X_default = ohe_default.fit_transform(X_df)
+    X_pandas = ohe_pandas.fit_transform(X_df)
+
+    assert_allclose(X_pandas.to_numpy(), X_default)
+    assert_array_equal(ohe_pandas.get_feature_names_out(), X_pandas.columns)
+
+
+def test_ordinal_set_output():
+    """Check OrdinalEncoder works with set_output."""
+    pd = pytest.importorskip("pandas")
+
+    X_df = pd.DataFrame({"A": ["a", "b"], "B": [1, 2]})
+
+    ord_default = OrdinalEncoder().set_output(transform="default")
+    ord_pandas = OrdinalEncoder().set_output(transform="pandas")
+
+    X_default = ord_default.fit_transform(X_df)
+    X_pandas = ord_pandas.fit_transform(X_df)
+
+    assert_allclose(X_pandas.to_numpy(), X_default)
+    assert_array_equal(ord_pandas.get_feature_names_out(), X_pandas.columns)
+
+
+def test_predefined_categories_dtype():
+    """Check that the categories_ dtype is `object` for string categories
+
+    Regression test for gh-25171.
+    """
+    categories = [["as", "mmas", "eas", "ras", "acs"], ["1", "2"]]
+
+    enc = OneHotEncoder(categories=categories)
+
+    enc.fit([["as", "1"]])
+
+    assert len(categories) == len(enc.categories_)
+    for n, cat in enumerate(enc.categories_):
+        assert cat.dtype == object
+        assert_array_equal(categories[n], cat)
+
+
+def test_ordinal_encoder_missing_unknown_encoding_max():
+    """Check missing value or unknown encoding can equal the cardinality."""
+    X = np.array([["dog"], ["cat"], [np.nan]], dtype=object)
+    X_trans = OrdinalEncoder(encoded_missing_value=2).fit_transform(X)
+    assert_allclose(X_trans, [[1], [0], [2]])
+
+    enc = OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=2).fit(X)
+    X_test = np.array([["snake"]])
+    X_trans = enc.transform(X_test)
+    assert_allclose(X_trans, [[2]])
+
+
+def test_drop_idx_infrequent_categories():
+    """Check drop_idx is defined correctly with infrequent categories.
+
+    Non-regression test for gh-25550.
+    """
+    X = np.array(
+        [["a"] * 2 + ["b"] * 4 + ["c"] * 4 + ["d"] * 4 + ["e"] * 4], dtype=object
+    ).T
+    ohe = OneHotEncoder(min_frequency=4, sparse_output=False, drop="first").fit(X)
+    assert_array_equal(
+        ohe.get_feature_names_out(), ["x0_c", "x0_d", "x0_e", "x0_infrequent_sklearn"]
+    )
+    assert ohe.categories_[0][ohe.drop_idx_[0]] == "b"
+
+    X = np.array([["a"] * 2 + ["b"] * 2 + ["c"] * 10], dtype=object).T
+    ohe = OneHotEncoder(min_frequency=4, sparse_output=False, drop="if_binary").fit(X)
+    assert_array_equal(ohe.get_feature_names_out(), ["x0_infrequent_sklearn"])
+    assert ohe.categories_[0][ohe.drop_idx_[0]] == "c"
+
+    X = np.array(
+        [["a"] * 2 + ["b"] * 4 + ["c"] * 4 + ["d"] * 4 + ["e"] * 4], dtype=object
+    ).T
+    ohe = OneHotEncoder(min_frequency=4, sparse_output=False, drop=["d"]).fit(X)
+    assert_array_equal(
+        ohe.get_feature_names_out(), ["x0_b", "x0_c", "x0_e", "x0_infrequent_sklearn"]
+    )
+    assert ohe.categories_[0][ohe.drop_idx_[0]] == "d"
+
+    ohe = OneHotEncoder(min_frequency=4, sparse_output=False, drop=None).fit(X)
+    assert_array_equal(
+        ohe.get_feature_names_out(),
+        ["x0_b", "x0_c", "x0_d", "x0_e", "x0_infrequent_sklearn"],
+    )
+    assert ohe.drop_idx_ is None
+
+
+@pytest.mark.parametrize(
+    "kwargs",
+    [
+        {"max_categories": 3},
+        {"min_frequency": 6},
+        {"min_frequency": 9},
+        {"min_frequency": 0.24},
+        {"min_frequency": 0.16},
+        {"max_categories": 3, "min_frequency": 8},
+        {"max_categories": 4, "min_frequency": 6},
+    ],
+)
+def test_ordinal_encoder_infrequent_three_levels(kwargs):
+    """Test parameters for grouping 'a', and 'd' into the infrequent category."""
+
+    X_train = np.array([["a"] * 5 + ["b"] * 20 + ["c"] * 10 + ["d"] * 3]).T
+    ordinal = OrdinalEncoder(
+        handle_unknown="use_encoded_value", unknown_value=-1, **kwargs
+    ).fit(X_train)
+    assert_array_equal(ordinal.categories_, [["a", "b", "c", "d"]])
+    assert_array_equal(ordinal.infrequent_categories_, [["a", "d"]])
+
+    X_test = [["a"], ["b"], ["c"], ["d"], ["z"]]
+    expected_trans = [[2], [0], [1], [2], [-1]]
+
+    X_trans = ordinal.transform(X_test)
+    assert_allclose(X_trans, expected_trans)
+
+    X_inverse = ordinal.inverse_transform(X_trans)
+    expected_inverse = [
+        ["infrequent_sklearn"],
+        ["b"],
+        ["c"],
+        ["infrequent_sklearn"],
+        [None],
+    ]
+    assert_array_equal(X_inverse, expected_inverse)
+
+
+def test_ordinal_encoder_infrequent_three_levels_user_cats():
+    """Test that the order of the categories provided by a user is respected.
+
+    In this case 'c' is encoded as the first category and 'b' is encoded
+    as the second one.
+    """
+
+    X_train = np.array(
+        [["a"] * 5 + ["b"] * 20 + ["c"] * 10 + ["d"] * 3], dtype=object
+    ).T
+    ordinal = OrdinalEncoder(
+        categories=[["c", "d", "b", "a"]],
+        max_categories=3,
+        handle_unknown="use_encoded_value",
+        unknown_value=-1,
+    ).fit(X_train)
+    assert_array_equal(ordinal.categories_, [["c", "d", "b", "a"]])
+    assert_array_equal(ordinal.infrequent_categories_, [["d", "a"]])
+
+    X_test = [["a"], ["b"], ["c"], ["d"], ["z"]]
+    expected_trans = [[2], [1], [0], [2], [-1]]
+
+    X_trans = ordinal.transform(X_test)
+    assert_allclose(X_trans, expected_trans)
+
+    X_inverse = ordinal.inverse_transform(X_trans)
+    expected_inverse = [
+        ["infrequent_sklearn"],
+        ["b"],
+        ["c"],
+        ["infrequent_sklearn"],
+        [None],
+    ]
+    assert_array_equal(X_inverse, expected_inverse)
+
+
+def test_ordinal_encoder_infrequent_mixed():
+    """Test when feature 0 has infrequent categories and feature 1 does not."""
+
+    X = np.column_stack(([0, 1, 3, 3, 3, 3, 2, 0, 3], [0, 0, 0, 0, 1, 1, 1, 1, 1]))
+
+    ordinal = OrdinalEncoder(max_categories=3).fit(X)
+
+    assert_array_equal(ordinal.infrequent_categories_[0], [1, 2])
+    assert ordinal.infrequent_categories_[1] is None
+
+    X_test = [[3, 0], [1, 1]]
+    expected_trans = [[1, 0], [2, 1]]
+
+    X_trans = ordinal.transform(X_test)
+    assert_allclose(X_trans, expected_trans)
+
+    X_inverse = ordinal.inverse_transform(X_trans)
+    expected_inverse = np.array([[3, 0], ["infrequent_sklearn", 1]], dtype=object)
+    assert_array_equal(X_inverse, expected_inverse)
+
+
+def test_ordinal_encoder_infrequent_multiple_categories_dtypes():
+    """Test infrequent categories with a pandas DataFrame with multiple dtypes."""
+
+    pd = pytest.importorskip("pandas")
+    categorical_dtype = pd.CategoricalDtype(["bird", "cat", "dog", "snake"])
+    X = pd.DataFrame(
+        {
+            "str": ["a", "f", "c", "f", "f", "a", "c", "b", "b"],
+            "int": [5, 3, 0, 10, 10, 12, 0, 3, 5],
+            "categorical": pd.Series(
+                ["dog"] * 4 + ["cat"] * 3 + ["snake"] + ["bird"],
+                dtype=categorical_dtype,
+            ),
+        },
+        columns=["str", "int", "categorical"],
+    )
+
+    ordinal = OrdinalEncoder(max_categories=3).fit(X)
+    # X[:, 0] 'a', 'b', 'c' have the same frequency. 'a' and 'b' will be
+    # considered infrequent because they appear first when sorted
+
+    # X[:, 1] 0, 3, 5, 10 has frequency 2 and 12 has frequency 1.
+    # 0, 3, 12 will be considered infrequent because they appear first when
+    # sorted.
+
+    # X[:, 2] "snake" and "bird" or infrequent
+
+    assert_array_equal(ordinal.infrequent_categories_[0], ["a", "b"])
+    assert_array_equal(ordinal.infrequent_categories_[1], [0, 3, 12])
+    assert_array_equal(ordinal.infrequent_categories_[2], ["bird", "snake"])
+
+    X_test = pd.DataFrame(
+        {
+            "str": ["a", "b", "f", "c"],
+            "int": [12, 0, 10, 5],
+            "categorical": pd.Series(
+                ["cat"] + ["snake"] + ["bird"] + ["dog"],
+                dtype=categorical_dtype,
+            ),
+        },
+        columns=["str", "int", "categorical"],
+    )
+    expected_trans = [[2, 2, 0], [2, 2, 2], [1, 1, 2], [0, 0, 1]]
+
+    X_trans = ordinal.transform(X_test)
+    assert_allclose(X_trans, expected_trans)
+
+
+def test_ordinal_encoder_infrequent_custom_mapping():
+    """Check behavior of unknown_value and encoded_missing_value with infrequent."""
+    X_train = np.array(
+        [["a"] * 5 + ["b"] * 20 + ["c"] * 10 + ["d"] * 3 + [np.nan]], dtype=object
+    ).T
+
+    ordinal = OrdinalEncoder(
+        handle_unknown="use_encoded_value",
+        unknown_value=2,
+        max_categories=2,
+        encoded_missing_value=3,
+    ).fit(X_train)
+    assert_array_equal(ordinal.infrequent_categories_, [["a", "c", "d"]])
+
+    X_test = np.array([["a"], ["b"], ["c"], ["d"], ["e"], [np.nan]], dtype=object)
+    expected_trans = [[1], [0], [1], [1], [2], [3]]
+
+    X_trans = ordinal.transform(X_test)
+    assert_allclose(X_trans, expected_trans)
+
+
+@pytest.mark.parametrize(
+    "kwargs",
+    [
+        {"max_categories": 6},
+        {"min_frequency": 2},
+    ],
+)
+def test_ordinal_encoder_all_frequent(kwargs):
+    """All categories are considered frequent have same encoding as default encoder."""
+    X_train = np.array(
+        [["a"] * 5 + ["b"] * 20 + ["c"] * 10 + ["d"] * 3], dtype=object
+    ).T
+
+    adjusted_encoder = OrdinalEncoder(
+        **kwargs, handle_unknown="use_encoded_value", unknown_value=-1
+    ).fit(X_train)
+    default_encoder = OrdinalEncoder(
+        handle_unknown="use_encoded_value", unknown_value=-1
+    ).fit(X_train)
+
+    X_test = [["a"], ["b"], ["c"], ["d"], ["e"]]
+
+    assert_allclose(
+        adjusted_encoder.transform(X_test), default_encoder.transform(X_test)
+    )
+
+
+@pytest.mark.parametrize(
+    "kwargs",
+    [
+        {"max_categories": 1},
+        {"min_frequency": 100},
+    ],
+)
+def test_ordinal_encoder_all_infrequent(kwargs):
+    """When all categories are infrequent, they are all encoded as zero."""
+    X_train = np.array(
+        [["a"] * 5 + ["b"] * 20 + ["c"] * 10 + ["d"] * 3], dtype=object
+    ).T
+    encoder = OrdinalEncoder(
+        **kwargs, handle_unknown="use_encoded_value", unknown_value=-1
+    ).fit(X_train)
+
+    X_test = [["a"], ["b"], ["c"], ["d"], ["e"]]
+    assert_allclose(encoder.transform(X_test), [[0], [0], [0], [0], [-1]])
+
+
+def test_ordinal_encoder_missing_appears_frequent():
+    """Check behavior when missing value appears frequently."""
+    X = np.array(
+        [[np.nan] * 20 + ["dog"] * 10 + ["cat"] * 5 + ["snake"] + ["deer"]],
+        dtype=object,
+    ).T
+    ordinal = OrdinalEncoder(max_categories=3).fit(X)
+
+    X_test = np.array([["snake", "cat", "dog", np.nan]], dtype=object).T
+    X_trans = ordinal.transform(X_test)
+    assert_allclose(X_trans, [[2], [0], [1], [np.nan]])
+
+
+def test_ordinal_encoder_missing_appears_infrequent():
+    """Check behavior when missing value appears infrequently."""
+
+    # feature 0 has infrequent categories
+    # feature 1 has no infrequent categories
+    X = np.array(
+        [
+            [np.nan] + ["dog"] * 10 + ["cat"] * 5 + ["snake"] + ["deer"],
+            ["red"] * 9 + ["green"] * 9,
+        ],
+        dtype=object,
+    ).T
+    ordinal = OrdinalEncoder(min_frequency=4).fit(X)
+
+    X_test = np.array(
+        [
+            ["snake", "red"],
+            ["deer", "green"],
+            [np.nan, "green"],
+            ["dog", "green"],
+            ["cat", "red"],
+        ],
+        dtype=object,
+    )
+    X_trans = ordinal.transform(X_test)
+    assert_allclose(X_trans, [[2, 1], [2, 0], [np.nan, 0], [1, 0], [0, 1]])
+
+
+@pytest.mark.parametrize("Encoder", [OneHotEncoder, OrdinalEncoder])
+def test_encoder_not_fitted(Encoder):
+    """Check that we raise a `NotFittedError` by calling transform before fit with
+    the encoders.
+
+    One could expect that the passing the `categories` argument to the encoder
+    would make it stateless. However, `fit` is making a couple of check, such as the
+    position of `np.nan`.
+    """
+    X = np.array([["A"], ["B"], ["C"]], dtype=object)
+    encoder = Encoder(categories=[["A", "B", "C"]])
+    with pytest.raises(NotFittedError):
+        encoder.transform(X)
diff --git a/sklearn/preprocessing/tests/test_function_transformer.py b/sklearn/preprocessing/tests/test_function_transformer.py
index 924975fbed2e1..6bfb5d1367c8d 100644
--- a/sklearn/preprocessing/tests/test_function_transformer.py
+++ b/sklearn/preprocessing/tests/test_function_transformer.py
@@ -1,11 +1,16 @@
-import pytest
+import warnings
+
 import numpy as np
-from scipy import sparse
+import pytest
 
-from sklearn.preprocessing import FunctionTransformer
-from sklearn.utils._testing import (assert_array_equal,
-                                   assert_allclose_dense_sparse)
-from sklearn.utils._testing import assert_warns_message, assert_no_warnings
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import FunctionTransformer, StandardScaler
+from sklearn.utils._testing import (
+    _convert_container,
+    assert_allclose_dense_sparse,
+    assert_array_equal,
+)
+from sklearn.utils.fixes import CSC_CONTAINERS, CSR_CONTAINERS
 
 
 def _make_func(args_store, kwargs_store, func=lambda X, *a, **k: X):
@@ -26,15 +31,18 @@ def test_delegate_to_func():
     X = np.arange(10).reshape((5, 2))
     assert_array_equal(
         FunctionTransformer(_make_func(args_store, kwargs_store)).transform(X),
-        X, 'transform should have returned X unchanged',
+        X,
+        "transform should have returned X unchanged",
     )
 
     # The function should only have received X.
-    assert args_store == [X], ('Incorrect positional arguments passed to '
-                               'func: {args}'.format(args=args_store))
+    assert args_store == [X], (
+        "Incorrect positional arguments passed to func: {args}".format(args=args_store)
+    )
 
-    assert not kwargs_store, ('Unexpected keyword arguments passed to '
-                              'func: {args}'.format(args=kwargs_store))
+    assert not kwargs_store, (
+        "Unexpected keyword arguments passed to func: {args}".format(args=kwargs_store)
+    )
 
     # reset the argument stores.
     args_store[:] = []
@@ -43,15 +51,18 @@ def test_delegate_to_func():
         _make_func(args_store, kwargs_store),
     ).transform(X)
 
-    assert_array_equal(transformed, X,
-                       err_msg='transform should have returned X unchanged')
+    assert_array_equal(
+        transformed, X, err_msg="transform should have returned X unchanged"
+    )
 
     # The function should have received X
-    assert args_store == [X], ('Incorrect positional arguments passed '
-                               'to func: {args}'.format(args=args_store))
+    assert args_store == [X], (
+        "Incorrect positional arguments passed to func: {args}".format(args=args_store)
+    )
 
-    assert not kwargs_store, ('Unexpected keyword arguments passed to '
-                              'func: {args}'.format(args=kwargs_store))
+    assert not kwargs_store, (
+        "Unexpected keyword arguments passed to func: {args}".format(args=kwargs_store)
+    )
 
 
 def test_np_log():
@@ -70,8 +81,7 @@ def test_kw_arg():
     F = FunctionTransformer(np.around, kw_args=dict(decimals=3))
 
     # Test that rounding is correct
-    assert_array_equal(F.transform(X),
-                       np.around(X, decimals=3))
+    assert_array_equal(F.transform(X), np.around(X, decimals=3))
 
 
 def test_kw_arg_update():
@@ -79,7 +89,7 @@ def test_kw_arg_update():
 
     F = FunctionTransformer(np.around, kw_args=dict(decimals=3))
 
-    F.kw_args['decimals'] = 1
+    F.kw_args["decimals"] = 1
 
     # Test that rounding is correct
     assert_array_equal(F.transform(X), np.around(X, decimals=1))
@@ -102,7 +112,8 @@ def test_inverse_transform():
     # Test that inverse_transform works correctly
     F = FunctionTransformer(
         func=np.sqrt,
-        inverse_func=np.around, inv_kw_args=dict(decimals=3),
+        inverse_func=np.around,
+        inv_kw_args=dict(decimals=3),
     )
     assert_array_equal(
         F.inverse_transform(F.transform(X)),
@@ -110,51 +121,459 @@ def test_inverse_transform():
     )
 
 
-def test_check_inverse():
-    X_dense = np.array([1, 4, 9, 16], dtype=np.float64).reshape((2, 2))
+@pytest.mark.parametrize("sparse_container", [None] + CSC_CONTAINERS + CSR_CONTAINERS)
+def test_check_inverse(sparse_container):
+    X = np.array([1, 4, 9, 16], dtype=np.float64).reshape((2, 2))
+    if sparse_container is not None:
+        X = sparse_container(X)
+
+    trans = FunctionTransformer(
+        func=np.sqrt,
+        inverse_func=np.around,
+        accept_sparse=sparse_container is not None,
+        check_inverse=True,
+        validate=True,
+    )
+    warning_message = (
+        "The provided functions are not strictly"
+        " inverse of each other. If you are sure you"
+        " want to proceed regardless, set"
+        " 'check_inverse=False'."
+    )
+    with pytest.warns(UserWarning, match=warning_message):
+        trans.fit(X)
+
+    trans = FunctionTransformer(
+        func=np.expm1,
+        inverse_func=np.log1p,
+        accept_sparse=sparse_container is not None,
+        check_inverse=True,
+        validate=True,
+    )
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", UserWarning)
+        Xt = trans.fit_transform(X)
 
-    X_list = [X_dense,
-              sparse.csr_matrix(X_dense),
-              sparse.csc_matrix(X_dense)]
+    assert_allclose_dense_sparse(X, trans.inverse_transform(Xt))
 
-    for X in X_list:
-        if sparse.issparse(X):
-            accept_sparse = True
-        else:
-            accept_sparse = False
-        trans = FunctionTransformer(func=np.sqrt,
-                                    inverse_func=np.around,
-                                    accept_sparse=accept_sparse,
-                                    check_inverse=True,
-                                    validate=True)
-        assert_warns_message(UserWarning,
-                             "The provided functions are not strictly"
-                             " inverse of each other. If you are sure you"
-                             " want to proceed regardless, set"
-                             " 'check_inverse=False'.",
-                             trans.fit, X)
-
-        trans = FunctionTransformer(func=np.expm1,
-                                    inverse_func=np.log1p,
-                                    accept_sparse=accept_sparse,
-                                    check_inverse=True,
-                                    validate=True)
-        Xt = assert_no_warnings(trans.fit_transform, X)
-        assert_allclose_dense_sparse(X, trans.inverse_transform(Xt))
 
+def test_check_inverse_func_or_inverse_not_provided():
     # check that we don't check inverse when one of the func or inverse is not
     # provided.
-    trans = FunctionTransformer(func=np.expm1, inverse_func=None,
-                                check_inverse=True, validate=True)
-    assert_no_warnings(trans.fit, X_dense)
-    trans = FunctionTransformer(func=None, inverse_func=np.expm1,
-                                check_inverse=True, validate=True)
-    assert_no_warnings(trans.fit, X_dense)
+    X = np.array([1, 4, 9, 16], dtype=np.float64).reshape((2, 2))
+
+    trans = FunctionTransformer(
+        func=np.expm1, inverse_func=None, check_inverse=True, validate=True
+    )
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", UserWarning)
+        trans.fit(X)
+    trans = FunctionTransformer(
+        func=None, inverse_func=np.expm1, check_inverse=True, validate=True
+    )
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", UserWarning)
+        trans.fit(X)
 
 
 def test_function_transformer_frame():
-    pd = pytest.importorskip('pandas')
+    pd = pytest.importorskip("pandas")
     X_df = pd.DataFrame(np.random.randn(100, 10))
     transformer = FunctionTransformer()
     X_df_trans = transformer.fit_transform(X_df)
-    assert hasattr(X_df_trans, 'loc')
+    assert hasattr(X_df_trans, "loc")
+
+
+@pytest.mark.parametrize("X_type", ["array", "series"])
+def test_function_transformer_raise_error_with_mixed_dtype(X_type):
+    """Check that `FunctionTransformer.check_inverse` raises error on mixed dtype."""
+    mapping = {"one": 1, "two": 2, "three": 3, 5: "five", 6: "six"}
+    inverse_mapping = {value: key for key, value in mapping.items()}
+    dtype = "object"
+
+    data = ["one", "two", "three", "one", "one", 5, 6]
+    data = _convert_container(data, X_type, columns_name=["value"], dtype=dtype)
+
+    def func(X):
+        return np.array([mapping[X[i]] for i in range(X.size)], dtype=object)
+
+    def inverse_func(X):
+        return _convert_container(
+            [inverse_mapping[x] for x in X],
+            X_type,
+            columns_name=["value"],
+            dtype=dtype,
+        )
+
+    transformer = FunctionTransformer(
+        func=func, inverse_func=inverse_func, validate=False, check_inverse=True
+    )
+
+    msg = "'check_inverse' is only supported when all the elements in `X` is numerical."
+    with pytest.raises(ValueError, match=msg):
+        transformer.fit(data)
+
+
+def test_function_transformer_support_all_nummerical_dataframes_check_inverse_True():
+    """Check support for dataframes with only numerical values."""
+    pd = pytest.importorskip("pandas")
+
+    df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
+    transformer = FunctionTransformer(
+        func=lambda x: x + 2, inverse_func=lambda x: x - 2, check_inverse=True
+    )
+
+    # Does not raise an error
+    df_out = transformer.fit_transform(df)
+    assert_allclose_dense_sparse(df_out, df + 2)
+
+
+def test_function_transformer_with_dataframe_and_check_inverse_True():
+    """Check error is raised when check_inverse=True.
+
+    Non-regresion test for gh-25261.
+    """
+    pd = pytest.importorskip("pandas")
+    transformer = FunctionTransformer(
+        func=lambda x: x, inverse_func=lambda x: x, check_inverse=True
+    )
+
+    df_mixed = pd.DataFrame({"a": [1, 2, 3], "b": ["a", "b", "c"]})
+    msg = "'check_inverse' is only supported when all the elements in `X` is numerical."
+    with pytest.raises(ValueError, match=msg):
+        transformer.fit(df_mixed)
+
+
+@pytest.mark.parametrize(
+    "X, feature_names_out, input_features, expected",
+    [
+        (
+            # NumPy inputs, default behavior: generate names
+            np.random.rand(100, 3),
+            "one-to-one",
+            None,
+            ("x0", "x1", "x2"),
+        ),
+        (
+            # Pandas input, default behavior: use input feature names
+            {"a": np.random.rand(100), "b": np.random.rand(100)},
+            "one-to-one",
+            None,
+            ("a", "b"),
+        ),
+        (
+            # NumPy input, feature_names_out=callable
+            np.random.rand(100, 3),
+            lambda transformer, input_features: ("a", "b"),
+            None,
+            ("a", "b"),
+        ),
+        (
+            # Pandas input, feature_names_out=callable
+            {"a": np.random.rand(100), "b": np.random.rand(100)},
+            lambda transformer, input_features: ("c", "d", "e"),
+            None,
+            ("c", "d", "e"),
+        ),
+        (
+            # NumPy input, feature_names_out=callable – default input_features
+            np.random.rand(100, 3),
+            lambda transformer, input_features: tuple(input_features) + ("a",),
+            None,
+            ("x0", "x1", "x2", "a"),
+        ),
+        (
+            # Pandas input, feature_names_out=callable – default input_features
+            {"a": np.random.rand(100), "b": np.random.rand(100)},
+            lambda transformer, input_features: tuple(input_features) + ("c",),
+            None,
+            ("a", "b", "c"),
+        ),
+        (
+            # NumPy input, input_features=list of names
+            np.random.rand(100, 3),
+            "one-to-one",
+            ("a", "b", "c"),
+            ("a", "b", "c"),
+        ),
+        (
+            # Pandas input, input_features=list of names
+            {"a": np.random.rand(100), "b": np.random.rand(100)},
+            "one-to-one",
+            ("a", "b"),  # must match feature_names_in_
+            ("a", "b"),
+        ),
+        (
+            # NumPy input, feature_names_out=callable, input_features=list
+            np.random.rand(100, 3),
+            lambda transformer, input_features: tuple(input_features) + ("d",),
+            ("a", "b", "c"),
+            ("a", "b", "c", "d"),
+        ),
+        (
+            # Pandas input, feature_names_out=callable, input_features=list
+            {"a": np.random.rand(100), "b": np.random.rand(100)},
+            lambda transformer, input_features: tuple(input_features) + ("c",),
+            ("a", "b"),  # must match feature_names_in_
+            ("a", "b", "c"),
+        ),
+    ],
+)
+@pytest.mark.parametrize("validate", [True, False])
+def test_function_transformer_get_feature_names_out(
+    X, feature_names_out, input_features, expected, validate
+):
+    if isinstance(X, dict):
+        pd = pytest.importorskip("pandas")
+        X = pd.DataFrame(X)
+
+    transformer = FunctionTransformer(
+        feature_names_out=feature_names_out, validate=validate
+    )
+    transformer.fit(X)
+    names = transformer.get_feature_names_out(input_features)
+    assert isinstance(names, np.ndarray)
+    assert names.dtype == object
+    assert_array_equal(names, expected)
+
+
+def test_function_transformer_get_feature_names_out_without_validation():
+    transformer = FunctionTransformer(feature_names_out="one-to-one", validate=False)
+    X = np.random.rand(100, 2)
+    transformer.fit_transform(X)
+
+    names = transformer.get_feature_names_out(("a", "b"))
+    assert isinstance(names, np.ndarray)
+    assert names.dtype == object
+    assert_array_equal(names, ("a", "b"))
+
+
+def test_function_transformer_feature_names_out_is_None():
+    transformer = FunctionTransformer()
+    X = np.random.rand(100, 2)
+    transformer.fit_transform(X)
+
+    msg = "This 'FunctionTransformer' has no attribute 'get_feature_names_out'"
+    with pytest.raises(AttributeError, match=msg):
+        transformer.get_feature_names_out()
+
+
+def test_function_transformer_feature_names_out_uses_estimator():
+    def add_n_random_features(X, n):
+        return np.concatenate([X, np.random.rand(len(X), n)], axis=1)
+
+    def feature_names_out(transformer, input_features):
+        n = transformer.kw_args["n"]
+        return list(input_features) + [f"rnd{i}" for i in range(n)]
+
+    transformer = FunctionTransformer(
+        func=add_n_random_features,
+        feature_names_out=feature_names_out,
+        kw_args=dict(n=3),
+        validate=True,
+    )
+    pd = pytest.importorskip("pandas")
+    df = pd.DataFrame({"a": np.random.rand(100), "b": np.random.rand(100)})
+    transformer.fit_transform(df)
+    names = transformer.get_feature_names_out()
+
+    assert isinstance(names, np.ndarray)
+    assert names.dtype == object
+    assert_array_equal(names, ("a", "b", "rnd0", "rnd1", "rnd2"))
+
+
+def test_function_transformer_validate_inverse():
+    """Test that function transformer does not reset estimator in
+    `inverse_transform`."""
+
+    def add_constant_feature(X):
+        X_one = np.ones((X.shape[0], 1))
+        return np.concatenate((X, X_one), axis=1)
+
+    def inverse_add_constant(X):
+        return X[:, :-1]
+
+    X = np.array([[1, 2], [3, 4], [3, 4]])
+    trans = FunctionTransformer(
+        func=add_constant_feature,
+        inverse_func=inverse_add_constant,
+        validate=True,
+    )
+    X_trans = trans.fit_transform(X)
+    assert trans.n_features_in_ == X.shape[1]
+
+    trans.inverse_transform(X_trans)
+    assert trans.n_features_in_ == X.shape[1]
+
+
+@pytest.mark.parametrize(
+    "feature_names_out, expected",
+    [
+        ("one-to-one", ["pet", "color"]),
+        [lambda est, names: [f"{n}_out" for n in names], ["pet_out", "color_out"]],
+    ],
+)
+@pytest.mark.parametrize("in_pipeline", [True, False])
+def test_get_feature_names_out_dataframe_with_string_data(
+    feature_names_out, expected, in_pipeline
+):
+    """Check that get_feature_names_out works with DataFrames with string data."""
+    pd = pytest.importorskip("pandas")
+    X = pd.DataFrame({"pet": ["dog", "cat"], "color": ["red", "green"]})
+
+    def func(X):
+        if feature_names_out == "one-to-one":
+            return X
+        else:
+            name = feature_names_out(None, X.columns)
+            return X.rename(columns=dict(zip(X.columns, name)))
+
+    transformer = FunctionTransformer(func=func, feature_names_out=feature_names_out)
+    if in_pipeline:
+        transformer = make_pipeline(transformer)
+
+    X_trans = transformer.fit_transform(X)
+    assert isinstance(X_trans, pd.DataFrame)
+
+    names = transformer.get_feature_names_out()
+    assert isinstance(names, np.ndarray)
+    assert names.dtype == object
+    assert_array_equal(names, expected)
+
+
+def test_set_output_func():
+    """Check behavior of set_output with different settings."""
+    pd = pytest.importorskip("pandas")
+
+    X = pd.DataFrame({"a": [1, 2, 3], "b": [10, 20, 100]})
+
+    ft = FunctionTransformer(np.log, feature_names_out="one-to-one")
+
+    # no warning is raised when feature_names_out is defined
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", UserWarning)
+        ft.set_output(transform="pandas")
+
+    X_trans = ft.fit_transform(X)
+    assert isinstance(X_trans, pd.DataFrame)
+    assert_array_equal(X_trans.columns, ["a", "b"])
+
+    ft = FunctionTransformer(lambda x: 2 * x)
+    ft.set_output(transform="pandas")
+
+    # no warning is raised when func returns a panda dataframe
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", UserWarning)
+        X_trans = ft.fit_transform(X)
+    assert isinstance(X_trans, pd.DataFrame)
+    assert_array_equal(X_trans.columns, ["a", "b"])
+
+    # Warning is raised when func returns a ndarray
+    ft_np = FunctionTransformer(lambda x: np.asarray(x))
+
+    for transform in ("pandas", "polars"):
+        ft_np.set_output(transform=transform)
+        msg = (
+            f"When `set_output` is configured to be '{transform}'.*{transform} "
+            "DataFrame.*"
+        )
+        with pytest.warns(UserWarning, match=msg):
+            ft_np.fit_transform(X)
+
+    # default transform does not warn
+    ft_np.set_output(transform="default")
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", UserWarning)
+        ft_np.fit_transform(X)
+
+
+def test_consistence_column_name_between_steps():
+    """Check that we have a consistence between the feature names out of
+    `FunctionTransformer` and the feature names in of the next step in the pipeline.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/27695
+    """
+    pd = pytest.importorskip("pandas")
+
+    def with_suffix(_, names):
+        return [name + "__log" for name in names]
+
+    pipeline = make_pipeline(
+        FunctionTransformer(np.log1p, feature_names_out=with_suffix), StandardScaler()
+    )
+
+    df = pd.DataFrame([[1, 2], [3, 4], [5, 6]], columns=["a", "b"])
+    X_trans = pipeline.fit_transform(df)
+    assert pipeline.get_feature_names_out().tolist() == ["a__log", "b__log"]
+    # StandardScaler will convert to a numpy array
+    assert isinstance(X_trans, np.ndarray)
+
+
+@pytest.mark.parametrize("dataframe_lib", ["pandas", "polars"])
+@pytest.mark.parametrize("transform_output", ["default", "pandas", "polars"])
+def test_function_transformer_overwrite_column_names(dataframe_lib, transform_output):
+    """Check that we overwrite the column names when we should."""
+    lib = pytest.importorskip(dataframe_lib)
+    if transform_output != "numpy":
+        pytest.importorskip(transform_output)
+
+    df = lib.DataFrame({"a": [1, 2, 3], "b": [10, 20, 100]})
+
+    def with_suffix(_, names):
+        return [name + "__log" for name in names]
+
+    transformer = FunctionTransformer(feature_names_out=with_suffix).set_output(
+        transform=transform_output
+    )
+    X_trans = transformer.fit_transform(df)
+    assert_array_equal(np.asarray(X_trans), np.asarray(df))
+
+    feature_names = transformer.get_feature_names_out()
+    assert list(X_trans.columns) == with_suffix(None, df.columns)
+    assert feature_names.tolist() == with_suffix(None, df.columns)
+
+
+@pytest.mark.parametrize(
+    "feature_names_out",
+    ["one-to-one", lambda _, names: [f"{name}_log" for name in names]],
+)
+def test_function_transformer_overwrite_column_names_numerical(feature_names_out):
+    """Check the same as `test_function_transformer_overwrite_column_names`
+    but for the specific case of pandas where column names can be numerical."""
+    pd = pytest.importorskip("pandas")
+
+    df = pd.DataFrame({0: [1, 2, 3], 1: [10, 20, 100]})
+
+    transformer = FunctionTransformer(feature_names_out=feature_names_out)
+    X_trans = transformer.fit_transform(df)
+    assert_array_equal(np.asarray(X_trans), np.asarray(df))
+
+    feature_names = transformer.get_feature_names_out()
+    assert list(X_trans.columns) == list(feature_names)
+
+
+@pytest.mark.parametrize("dataframe_lib", ["pandas", "polars"])
+@pytest.mark.parametrize(
+    "feature_names_out",
+    ["one-to-one", lambda _, names: [f"{name}_log" for name in names]],
+)
+def test_function_transformer_error_column_inconsistent(
+    dataframe_lib, feature_names_out
+):
+    """Check that we raise an error when `func` returns a dataframe with new
+    column names that become inconsistent with `get_feature_names_out`."""
+    lib = pytest.importorskip(dataframe_lib)
+
+    df = lib.DataFrame({"a": [1, 2, 3], "b": [10, 20, 100]})
+
+    def func(df):
+        if dataframe_lib == "pandas":
+            return df.rename(columns={"a": "c"})
+        else:
+            return df.rename({"a": "c"})
+
+    transformer = FunctionTransformer(func=func, feature_names_out=feature_names_out)
+    err_msg = "The output generated by `func` have different column names"
+    with pytest.raises(ValueError, match=err_msg):
+        transformer.fit_transform(df).columns
diff --git a/sklearn/preprocessing/tests/test_label.py b/sklearn/preprocessing/tests/test_label.py
index aa9361d9164de..053b474e675bc 100644
--- a/sklearn/preprocessing/tests/test_label.py
+++ b/sklearn/preprocessing/tests/test_label.py
@@ -1,30 +1,35 @@
 import numpy as np
-
 import pytest
-
 from scipy.sparse import issparse
-from scipy.sparse import coo_matrix
-from scipy.sparse import csc_matrix
-from scipy.sparse import csr_matrix
-from scipy.sparse import dok_matrix
-from scipy.sparse import lil_matrix
 
+from sklearn import config_context, datasets
+from sklearn.preprocessing._label import (
+    LabelBinarizer,
+    LabelEncoder,
+    MultiLabelBinarizer,
+    _inverse_binarize_multiclass,
+    _inverse_binarize_thresholding,
+    label_binarize,
+)
+from sklearn.utils._array_api import (
+    _convert_to_numpy,
+    _get_namespace_device_dtype_ids,
+    get_namespace,
+    yield_namespace_device_dtype_combinations,
+)
+from sklearn.utils._testing import (
+    _array_api_for_tests,
+    assert_array_equal,
+)
+from sklearn.utils.fixes import (
+    COO_CONTAINERS,
+    CSC_CONTAINERS,
+    CSR_CONTAINERS,
+    DOK_CONTAINERS,
+    LIL_CONTAINERS,
+)
 from sklearn.utils.multiclass import type_of_target
-
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils._testing import assert_warns_message
-from sklearn.utils._testing import ignore_warnings
-from sklearn.utils import _to_object_array
-
-from sklearn.preprocessing._label import LabelBinarizer
-from sklearn.preprocessing._label import MultiLabelBinarizer
-from sklearn.preprocessing._label import LabelEncoder
-from sklearn.preprocessing._label import label_binarize
-
-from sklearn.preprocessing._label import _inverse_binarize_thresholding
-from sklearn.preprocessing._label import _inverse_binarize_multiclass
-
-from sklearn import datasets
+from sklearn.utils.validation import _to_object_array
 
 iris = datasets.load_iris()
 
@@ -62,21 +67,16 @@ def test_label_binarizer():
     assert_array_equal(lb.classes_, ["neg", "pos"])
     assert_array_equal(expected, got)
 
-    to_invert = np.array([[1, 0],
-                          [0, 1],
-                          [0, 1],
-                          [1, 0]])
+    to_invert = np.array([[1, 0], [0, 1], [0, 1], [1, 0]])
     assert_array_equal(lb.inverse_transform(to_invert), inp)
 
     # multi-class case
     inp = ["spam", "ham", "eggs", "ham", "0"]
-    expected = np.array([[0, 0, 0, 1],
-                         [0, 0, 1, 0],
-                         [0, 1, 0, 0],
-                         [0, 0, 1, 0],
-                         [1, 0, 0, 0]])
+    expected = np.array(
+        [[0, 0, 0, 1], [0, 0, 1, 0], [0, 1, 0, 0], [0, 0, 1, 0], [1, 0, 0, 0]]
+    )
     got = lb.fit_transform(inp)
-    assert_array_equal(lb.classes_, ['0', 'eggs', 'ham', 'spam'])
+    assert_array_equal(lb.classes_, ["0", "eggs", "ham", "spam"])
     assert_array_equal(expected, got)
     assert_array_equal(lb.inverse_transform(got), inp)
 
@@ -84,19 +84,14 @@ def test_label_binarizer():
 def test_label_binarizer_unseen_labels():
     lb = LabelBinarizer()
 
-    expected = np.array([[1, 0, 0],
-                         [0, 1, 0],
-                         [0, 0, 1]])
-    got = lb.fit_transform(['b', 'd', 'e'])
+    expected = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
+    got = lb.fit_transform(["b", "d", "e"])
     assert_array_equal(expected, got)
 
-    expected = np.array([[0, 0, 0],
-                         [1, 0, 0],
-                         [0, 0, 0],
-                         [0, 1, 0],
-                         [0, 0, 1],
-                         [0, 0, 0]])
-    got = lb.transform(['a', 'b', 'c', 'd', 'e', 'f'])
+    expected = np.array(
+        [[0, 0, 0], [1, 0, 0], [0, 0, 0], [0, 1, 0], [0, 0, 1], [0, 0, 0]]
+    )
+    got = lb.transform(["a", "b", "c", "d", "e", "f"])
     assert_array_equal(expected, got)
 
 
@@ -114,82 +109,142 @@ def test_label_binarizer_set_label_encoding():
 
     # multi-class case
     inp = np.array([3, 2, 1, 2, 0])
-    expected = np.array([[-2, -2, -2, +2],
-                         [-2, -2, +2, -2],
-                         [-2, +2, -2, -2],
-                         [-2, -2, +2, -2],
-                         [+2, -2, -2, -2]])
+    expected = np.array(
+        [
+            [-2, -2, -2, +2],
+            [-2, -2, +2, -2],
+            [-2, +2, -2, -2],
+            [-2, -2, +2, -2],
+            [+2, -2, -2, -2],
+        ]
+    )
     got = lb.fit_transform(inp)
     assert_array_equal(expected, got)
     assert_array_equal(lb.inverse_transform(got), inp)
 
 
-@ignore_warnings
+@pytest.mark.parametrize("dtype", ["Int64", "Float64", "boolean"])
+@pytest.mark.parametrize("unique_first", [True, False])
+def test_label_binarizer_pandas_nullable(dtype, unique_first):
+    """Checks that LabelBinarizer works with pandas nullable dtypes.
+
+    Non-regression test for gh-25637.
+    """
+    pd = pytest.importorskip("pandas")
+
+    y_true = pd.Series([1, 0, 0, 1, 0, 1, 1, 0, 1], dtype=dtype)
+    if unique_first:
+        # Calling unique creates a pandas array which has a different interface
+        # compared to a pandas Series. Specifically, pandas arrays do not have "iloc".
+        y_true = y_true.unique()
+    lb = LabelBinarizer().fit(y_true)
+    y_out = lb.transform([1, 0])
+
+    assert_array_equal(y_out, [[1], [0]])
+
+
 def test_label_binarizer_errors():
     # Check that invalid arguments yield ValueError
     one_class = np.array([0, 0, 0, 0])
     lb = LabelBinarizer().fit(one_class)
 
     multi_label = [(2, 3), (0,), (0, 2)]
-    with pytest.raises(ValueError):
+    err_msg = "You appear to be using a legacy multi-label data representation."
+    with pytest.raises(ValueError, match=err_msg):
         lb.transform(multi_label)
 
     lb = LabelBinarizer()
-    with pytest.raises(ValueError):
+    err_msg = "This LabelBinarizer instance is not fitted yet"
+    with pytest.raises(ValueError, match=err_msg):
         lb.transform([])
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError, match=err_msg):
         lb.inverse_transform([])
 
-    with pytest.raises(ValueError):
-        LabelBinarizer(neg_label=2, pos_label=1)
-    with pytest.raises(ValueError):
-        LabelBinarizer(neg_label=2, pos_label=2)
-
-    with pytest.raises(ValueError):
-        LabelBinarizer(neg_label=1, pos_label=2, sparse_output=True)
-
-    # Fail on y_type
-    with pytest.raises(ValueError):
-        _inverse_binarize_thresholding(y=csr_matrix([[1, 2], [2, 1]]),
-                                       output_type="foo", classes=[1, 2],
-                                       threshold=0)
+    input_labels = [0, 1, 0, 1]
+    err_msg = "neg_label=2 must be strictly less than pos_label=1."
+    lb = LabelBinarizer(neg_label=2, pos_label=1)
+    with pytest.raises(ValueError, match=err_msg):
+        lb.fit(input_labels)
+    err_msg = "neg_label=2 must be strictly less than pos_label=2."
+    lb = LabelBinarizer(neg_label=2, pos_label=2)
+    with pytest.raises(ValueError, match=err_msg):
+        lb.fit(input_labels)
+    err_msg = (
+        "Sparse binarization is only supported with non zero pos_label and zero "
+        "neg_label, got pos_label=2 and neg_label=1"
+    )
+    lb = LabelBinarizer(neg_label=1, pos_label=2, sparse_output=True)
+    with pytest.raises(ValueError, match=err_msg):
+        lb.fit(input_labels)
 
     # Sequence of seq type should raise ValueError
     y_seq_of_seqs = [[], [1, 2], [3], [0, 1, 3], [2]]
-    with pytest.raises(ValueError):
+    err_msg = "You appear to be using a legacy multi-label data representation"
+    with pytest.raises(ValueError, match=err_msg):
         LabelBinarizer().fit_transform(y_seq_of_seqs)
 
-    # Fail on the number of classes
-    with pytest.raises(ValueError):
-        _inverse_binarize_thresholding(y=csr_matrix([[1, 2], [2, 1]]),
-                                       output_type="foo",
-                                       classes=[1, 2, 3],
-                                       threshold=0)
-
     # Fail on the dimension of 'binary'
-    with pytest.raises(ValueError):
-        _inverse_binarize_thresholding(y=np.array([[1, 2, 3], [2, 1, 3]]),
-                                       output_type="binary",
-                                       classes=[1, 2, 3],
-                                       threshold=0)
+    err_msg = "output_type='binary', but y.shape"
+    with pytest.raises(ValueError, match=err_msg):
+        _inverse_binarize_thresholding(
+            y=np.array([[1, 2, 3], [2, 1, 3]]),
+            output_type="binary",
+            classes=[1, 2, 3],
+            threshold=0,
+        )
 
     # Fail on multioutput data
-    with pytest.raises(ValueError):
+    err_msg = "Multioutput target data is not supported with label binarization"
+    with pytest.raises(ValueError, match=err_msg):
         LabelBinarizer().fit(np.array([[1, 3], [2, 1]]))
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError, match=err_msg):
         label_binarize(np.array([[1, 3], [2, 1]]), classes=[1, 2, 3])
 
 
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_label_binarizer_sparse_errors(csr_container):
+    # Fail on y_type
+    err_msg = "foo format is not supported"
+    with pytest.raises(ValueError, match=err_msg):
+        _inverse_binarize_thresholding(
+            y=csr_container([[1, 2], [2, 1]]),
+            output_type="foo",
+            classes=[1, 2],
+            threshold=0,
+        )
+
+    # Fail on the number of classes
+    err_msg = "The number of class is not equal to the number of dimension of y."
+    with pytest.raises(ValueError, match=err_msg):
+        _inverse_binarize_thresholding(
+            y=csr_container([[1, 2], [2, 1]]),
+            output_type="foo",
+            classes=[1, 2, 3],
+            threshold=0,
+        )
+
+
 @pytest.mark.parametrize(
-        "values, classes, unknown",
-        [(np.array([2, 1, 3, 1, 3], dtype='int64'),
-          np.array([1, 2, 3], dtype='int64'), np.array([4], dtype='int64')),
-         (np.array(['b', 'a', 'c', 'a', 'c'], dtype=object),
-          np.array(['a', 'b', 'c'], dtype=object),
-          np.array(['d'], dtype=object)),
-         (np.array(['b', 'a', 'c', 'a', 'c']),
-          np.array(['a', 'b', 'c']), np.array(['d']))],
-        ids=['int64', 'object', 'str'])
+    "values, classes, unknown",
+    [
+        (
+            np.array([2, 1, 3, 1, 3], dtype="int64"),
+            np.array([1, 2, 3], dtype="int64"),
+            np.array([4], dtype="int64"),
+        ),
+        (
+            np.array(["b", "a", "c", "a", "c"], dtype=object),
+            np.array(["a", "b", "c"], dtype=object),
+            np.array(["d"], dtype=object),
+        ),
+        (
+            np.array(["b", "a", "c", "a", "c"]),
+            np.array(["a", "b", "c"]),
+            np.array(["d"]),
+        ),
+    ],
+    ids=["int64", "object", "str"],
+)
 def test_label_encoder(values, classes, unknown):
     # Test LabelEncoder's transform, fit_transform and
     # inverse_transform methods
@@ -210,15 +265,15 @@ def test_label_encoder_negative_ints():
     le = LabelEncoder()
     le.fit([1, 1, 4, 5, -1, 0])
     assert_array_equal(le.classes_, [-1, 0, 1, 4, 5])
-    assert_array_equal(le.transform([0, 1, 4, 4, 5, -1, -1]),
-                       [1, 2, 3, 3, 4, 0, 0])
-    assert_array_equal(le.inverse_transform([1, 2, 3, 3, 4, 0, 0]),
-                       [0, 1, 4, 4, 5, -1, -1])
+    assert_array_equal(le.transform([0, 1, 4, 4, 5, -1, -1]), [1, 2, 3, 3, 4, 0, 0])
+    assert_array_equal(
+        le.inverse_transform([1, 2, 3, 3, 4, 0, 0]), [0, 1, 4, 4, 5, -1, -1]
+    )
     with pytest.raises(ValueError):
         le.transform([0, 6])
 
 
-@pytest.mark.parametrize("dtype", ['str', 'object'])
+@pytest.mark.parametrize("dtype", ["str", "object"])
 def test_label_encoder_str_bad_shape(dtype):
     le = LabelEncoder()
     le.fit(np.array(["apple", "orange"], dtype=dtype))
@@ -251,11 +306,14 @@ def test_label_encoder_errors():
 
 
 @pytest.mark.parametrize(
-        "values",
-        [np.array([2, 1, 3, 1, 3], dtype='int64'),
-         np.array(['b', 'a', 'c', 'a', 'c'], dtype=object),
-         np.array(['b', 'a', 'c', 'a', 'c'])],
-        ids=['int64', 'object', 'str'])
+    "values",
+    [
+        np.array([2, 1, 3, 1, 3], dtype="int64"),
+        np.array(["b", "a", "c", "a", "c"], dtype=object),
+        np.array(["b", "a", "c", "a", "c"]),
+    ],
+    ids=["int64", "object", "str"],
+)
 def test_label_encoder_empty_array(values):
     le = LabelEncoder()
     le.fit(values)
@@ -274,9 +332,7 @@ def test_sparse_output_multilabel_binarizer():
         lambda: ({2, 3}, {1}, {1, 2}),
         lambda: iter([iter((2, 3)), iter((1,)), {1, 2}]),
     ]
-    indicator_mat = np.array([[0, 1, 1],
-                              [1, 0, 0],
-                              [1, 1, 0]])
+    indicator_mat = np.array([[0, 1, 1], [1, 0, 0], [1, 1, 0]])
 
     inverse = inputs[0]()
     for sparse_output in [True, False]:
@@ -305,10 +361,16 @@ def test_sparse_output_multilabel_binarizer():
             assert_array_equal([1, 2, 3], mlb.classes_)
             assert mlb.inverse_transform(got) == inverse
 
+
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sparse_output_multilabel_binarizer_errors(csr_container):
+    inp = iter([iter((2, 3)), iter((1,)), {1, 2}])
+    mlb = MultiLabelBinarizer(sparse_output=False)
+    mlb.fit(inp)
     with pytest.raises(ValueError):
-        mlb.inverse_transform(csr_matrix(np.array([[0, 1, 1],
-                                                   [2, 0, 0],
-                                                   [1, 1, 0]])))
+        mlb.inverse_transform(
+            csr_container(np.array([[0, 1, 1], [2, 0, 0], [1, 1, 0]]))
+        )
 
 
 def test_multilabel_binarizer():
@@ -318,9 +380,7 @@ def test_multilabel_binarizer():
         lambda: ({2, 3}, {1}, {1, 2}),
         lambda: iter([iter((2, 3)), iter((1,)), {1, 2}]),
     ]
-    indicator_mat = np.array([[0, 1, 1],
-                              [1, 0, 0],
-                              [1, 1, 0]])
+    indicator_mat = np.array([[0, 1, 1], [1, 0, 0], [1, 1, 0]])
     inverse = inputs[0]()
     for inp in inputs:
         # With fit_transform
@@ -341,9 +401,7 @@ def test_multilabel_binarizer():
 def test_multilabel_binarizer_empty_sample():
     mlb = MultiLabelBinarizer()
     y = [[1, 2], [1], []]
-    Y = np.array([[1, 1],
-                  [1, 0],
-                  [0, 0]])
+    Y = np.array([[1, 1], [1, 0], [0, 0]])
     assert_array_equal(mlb.fit_transform(y), Y)
 
 
@@ -351,23 +409,20 @@ def test_multilabel_binarizer_unknown_class():
     mlb = MultiLabelBinarizer()
     y = [[1, 2]]
     Y = np.array([[1, 0], [0, 1]])
-    w = 'unknown class(es) [0, 4] will be ignored'
-    matrix = assert_warns_message(UserWarning, w,
-                                  mlb.fit(y).transform, [[4, 1], [2, 0]])
-    assert_array_equal(matrix, Y)
+    warning_message = "unknown class.* will be ignored"
+    with pytest.warns(UserWarning, match=warning_message):
+        matrix = mlb.fit(y).transform([[4, 1], [2, 0]])
 
     Y = np.array([[1, 0, 0], [0, 1, 0]])
     mlb = MultiLabelBinarizer(classes=[1, 2, 3])
-    matrix = assert_warns_message(UserWarning, w,
-                                  mlb.fit(y).transform, [[4, 1], [2, 0]])
+    with pytest.warns(UserWarning, match=warning_message):
+        matrix = mlb.fit(y).transform([[4, 1], [2, 0]])
     assert_array_equal(matrix, Y)
 
 
 def test_multilabel_binarizer_given_classes():
     inp = [(2, 3), (1,), (1, 2)]
-    indicator_mat = np.array([[0, 1, 1],
-                              [1, 0, 0],
-                              [1, 0, 1]])
+    indicator_mat = np.array([[0, 1, 1], [1, 0, 0], [1, 0, 1]])
     # fit_transform()
     mlb = MultiLabelBinarizer(classes=[1, 3, 2])
     assert_array_equal(mlb.fit_transform(inp), indicator_mat)
@@ -380,8 +435,9 @@ def test_multilabel_binarizer_given_classes():
 
     # ensure works with extra class
     mlb = MultiLabelBinarizer(classes=[4, 1, 3, 2])
-    assert_array_equal(mlb.fit_transform(inp),
-                       np.hstack(([[0], [0], [0]], indicator_mat)))
+    assert_array_equal(
+        mlb.fit_transform(inp), np.hstack(([[0], [0], [0]], indicator_mat))
+    )
     assert_array_equal(mlb.classes_, [4, 1, 3, 2])
 
     # ensure fit is no-op as iterable is not consumed
@@ -390,8 +446,10 @@ def test_multilabel_binarizer_given_classes():
     assert_array_equal(mlb.fit(inp).transform(inp), indicator_mat)
 
     # ensure a ValueError is thrown if given duplicate classes
-    err_msg = "The classes argument contains duplicate classes. Remove " \
-              "these duplicates before passing them to MultiLabelBinarizer."
+    err_msg = (
+        "The classes argument contains duplicate classes. Remove "
+        "these duplicates before passing them to MultiLabelBinarizer."
+    )
     mlb = MultiLabelBinarizer(classes=[1, 3, 2, 3])
     with pytest.raises(ValueError, match=err_msg):
         mlb.fit(inp)
@@ -399,13 +457,9 @@ def test_multilabel_binarizer_given_classes():
 
 def test_multilabel_binarizer_multiple_calls():
     inp = [(2, 3), (1,), (1, 2)]
-    indicator_mat = np.array([[0, 1, 1],
-                              [1, 0, 0],
-                              [1, 0, 1]])
+    indicator_mat = np.array([[0, 1, 1], [1, 0, 0], [1, 0, 1]])
 
-    indicator_mat2 = np.array([[0, 1, 1],
-                               [1, 0, 0],
-                               [1, 1, 0]])
+    indicator_mat2 = np.array([[0, 1, 1], [1, 0, 0], [1, 1, 0]])
 
     # first call
     mlb = MultiLabelBinarizer(classes=[1, 3, 2])
@@ -418,9 +472,7 @@ def test_multilabel_binarizer_multiple_calls():
 def test_multilabel_binarizer_same_length_sequence():
     # Ensure sequences of the same length are not interpreted as a 2-d array
     inp = [[1], [0], [2]]
-    indicator_mat = np.array([[0, 1, 0],
-                              [1, 0, 0],
-                              [0, 0, 1]])
+    indicator_mat = np.array([[0, 1, 0], [1, 0, 0], [0, 0, 1]])
     # fit_transform()
     mlb = MultiLabelBinarizer()
     assert_array_equal(mlb.fit_transform(inp), indicator_mat)
@@ -435,34 +487,30 @@ def test_multilabel_binarizer_same_length_sequence():
 def test_multilabel_binarizer_non_integer_labels():
     tuple_classes = _to_object_array([(1,), (2,), (3,)])
     inputs = [
-        ([('2', '3'), ('1',), ('1', '2')], ['1', '2', '3']),
-        ([('b', 'c'), ('a',), ('a', 'b')], ['a', 'b', 'c']),
+        ([("2", "3"), ("1",), ("1", "2")], ["1", "2", "3"]),
+        ([("b", "c"), ("a",), ("a", "b")], ["a", "b", "c"]),
         ([((2,), (3,)), ((1,),), ((1,), (2,))], tuple_classes),
     ]
-    indicator_mat = np.array([[0, 1, 1],
-                              [1, 0, 0],
-                              [1, 1, 0]])
+    indicator_mat = np.array([[0, 1, 1], [1, 0, 0], [1, 1, 0]])
     for inp, classes in inputs:
         # fit_transform()
         mlb = MultiLabelBinarizer()
         inp = np.array(inp, dtype=object)
         assert_array_equal(mlb.fit_transform(inp), indicator_mat)
         assert_array_equal(mlb.classes_, classes)
-        indicator_mat_inv = np.array(mlb.inverse_transform(indicator_mat),
-                                     dtype=object)
+        indicator_mat_inv = np.array(mlb.inverse_transform(indicator_mat), dtype=object)
         assert_array_equal(indicator_mat_inv, inp)
 
         # fit().transform()
         mlb = MultiLabelBinarizer()
         assert_array_equal(mlb.fit(inp).transform(inp), indicator_mat)
         assert_array_equal(mlb.classes_, classes)
-        indicator_mat_inv = np.array(mlb.inverse_transform(indicator_mat),
-                                     dtype=object)
+        indicator_mat_inv = np.array(mlb.inverse_transform(indicator_mat), dtype=object)
         assert_array_equal(indicator_mat_inv, inp)
 
     mlb = MultiLabelBinarizer()
     with pytest.raises(TypeError):
-        mlb.fit_transform([({}), ({}, {'a': 'b'})])
+        mlb.fit_transform([({}), ({}, {"a": "b"})])
 
 
 def test_multilabel_binarizer_non_unique():
@@ -502,26 +550,31 @@ def test_label_binarize_with_class_order():
     assert_array_equal(out, expected)
 
     out = label_binarize([0, 1, 2, 3], classes=[3, 2, 0, 1])
-    expected = np.array([[0, 0, 1, 0],
-                         [0, 0, 0, 1],
-                         [0, 1, 0, 0],
-                         [1, 0, 0, 0]])
+    expected = np.array([[0, 0, 1, 0], [0, 0, 0, 1], [0, 1, 0, 0], [1, 0, 0, 0]])
     assert_array_equal(out, expected)
 
 
 def check_binarized_results(y, classes, pos_label, neg_label, expected):
     for sparse_output in [True, False]:
-        if ((pos_label == 0 or neg_label != 0) and sparse_output):
+        if (pos_label == 0 or neg_label != 0) and sparse_output:
             with pytest.raises(ValueError):
-                label_binarize(y, classes=classes, neg_label=neg_label,
-                               pos_label=pos_label,
-                               sparse_output=sparse_output)
+                label_binarize(
+                    y,
+                    classes=classes,
+                    neg_label=neg_label,
+                    pos_label=pos_label,
+                    sparse_output=sparse_output,
+                )
             continue
 
         # check label_binarize
-        binarized = label_binarize(y, classes=classes, neg_label=neg_label,
-                                   pos_label=pos_label,
-                                   sparse_output=sparse_output)
+        binarized = label_binarize(
+            y,
+            classes=classes,
+            neg_label=neg_label,
+            pos_label=pos_label,
+            sparse_output=sparse_output,
+        )
         assert_array_equal(toarray(binarized), expected)
         assert issparse(binarized) == sparse_output
 
@@ -531,18 +584,19 @@ def check_binarized_results(y, classes, pos_label, neg_label, expected):
             inversed = _inverse_binarize_multiclass(binarized, classes=classes)
 
         else:
-            inversed = _inverse_binarize_thresholding(binarized,
-                                                      output_type=y_type,
-                                                      classes=classes,
-                                                      threshold=((neg_label +
-                                                                 pos_label) /
-                                                                 2.))
+            inversed = _inverse_binarize_thresholding(
+                binarized,
+                output_type=y_type,
+                classes=classes,
+                threshold=((neg_label + pos_label) / 2.0),
+            )
 
         assert_array_equal(toarray(inversed), toarray(y))
 
         # Check label binarizer
-        lb = LabelBinarizer(neg_label=neg_label, pos_label=pos_label,
-                            sparse_output=sparse_output)
+        lb = LabelBinarizer(
+            neg_label=neg_label, pos_label=pos_label, sparse_output=sparse_output
+        )
         binarized = lb.fit_transform(y)
         assert_array_equal(toarray(binarized), expected)
         assert issparse(binarized) == sparse_output
@@ -580,27 +634,34 @@ def test_label_binarize_multiclass():
     check_binarized_results(y, classes, pos_label, neg_label, expected)
 
     with pytest.raises(ValueError):
-        label_binarize(y, classes=classes, neg_label=-1, pos_label=pos_label,
-                       sparse_output=True)
+        label_binarize(
+            y, classes=classes, neg_label=-1, pos_label=pos_label, sparse_output=True
+        )
 
 
-def test_label_binarize_multilabel():
+@pytest.mark.parametrize(
+    "arr_type",
+    [np.array]
+    + COO_CONTAINERS
+    + CSC_CONTAINERS
+    + CSR_CONTAINERS
+    + DOK_CONTAINERS
+    + LIL_CONTAINERS,
+)
+def test_label_binarize_multilabel(arr_type):
     y_ind = np.array([[0, 1, 0], [1, 1, 1], [0, 0, 0]])
     classes = [0, 1, 2]
     pos_label = 2
     neg_label = 0
     expected = pos_label * y_ind
-    y_sparse = [sparse_matrix(y_ind)
-                for sparse_matrix in [coo_matrix, csc_matrix, csr_matrix,
-                                      dok_matrix, lil_matrix]]
+    y = arr_type(y_ind)
 
-    for y in [y_ind] + y_sparse:
-        check_binarized_results(y, classes, pos_label, neg_label,
-                                expected)
+    check_binarized_results(y, classes, pos_label, neg_label, expected)
 
     with pytest.raises(ValueError):
-        label_binarize(y, classes=classes, neg_label=-1, pos_label=pos_label,
-                       sparse_output=True)
+        label_binarize(
+            y, classes=classes, neg_label=-1, pos_label=pos_label, sparse_output=True
+        )
 
 
 def test_invalid_input_label_binarize():
@@ -612,9 +673,76 @@ def test_invalid_input_label_binarize():
         label_binarize([[1, 3]], classes=[1, 2, 3])
 
 
-def test_inverse_binarize_multiclass():
-    got = _inverse_binarize_multiclass(csr_matrix([[0, 1, 0],
-                                                   [-1, 0, -1],
-                                                   [0, 0, 0]]),
-                                       np.arange(3))
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_inverse_binarize_multiclass(csr_container):
+    got = _inverse_binarize_multiclass(
+        csr_container([[0, 1, 0], [-1, 0, -1], [0, 0, 0]]), np.arange(3)
+    )
     assert_array_equal(got, np.array([1, 1, 0]))
+
+
+def test_nan_label_encoder():
+    """Check that label encoder encodes nans in transform.
+
+    Non-regression test for #22628.
+    """
+    le = LabelEncoder()
+    le.fit(["a", "a", "b", np.nan])
+
+    y_trans = le.transform([np.nan])
+    assert_array_equal(y_trans, [2])
+
+
+@pytest.mark.parametrize(
+    "encoder", [LabelEncoder(), LabelBinarizer(), MultiLabelBinarizer()]
+)
+def test_label_encoders_do_not_have_set_output(encoder):
+    """Check that label encoders do not define set_output and work with y as a kwarg.
+
+    Non-regression test for #26854.
+    """
+    assert not hasattr(encoder, "set_output")
+    y_encoded_with_kwarg = encoder.fit_transform(y=["a", "b", "c"])
+    y_encoded_positional = encoder.fit_transform(["a", "b", "c"])
+    assert_array_equal(y_encoded_with_kwarg, y_encoded_positional)
+
+
+@pytest.mark.parametrize(
+    "array_namespace, device, dtype",
+    yield_namespace_device_dtype_combinations(),
+    ids=_get_namespace_device_dtype_ids,
+)
+@pytest.mark.parametrize(
+    "y",
+    [
+        np.array([2, 1, 3, 1, 3]),
+        np.array([1, 1, 4, 5, -1, 0]),
+        np.array([3, 5, 9, 5, 9, 3]),
+    ],
+)
+def test_label_encoder_array_api_compliance(y, array_namespace, device, dtype):
+    xp = _array_api_for_tests(array_namespace, device)
+    xp_y = xp.asarray(y, device=device)
+    with config_context(array_api_dispatch=True):
+        xp_label = LabelEncoder()
+        np_label = LabelEncoder()
+        xp_label = xp_label.fit(xp_y)
+        xp_transformed = xp_label.transform(xp_y)
+        xp_inv_transformed = xp_label.inverse_transform(xp_transformed)
+        np_label = np_label.fit(y)
+        np_transformed = np_label.transform(y)
+        assert get_namespace(xp_transformed)[0].__name__ == xp.__name__
+        assert get_namespace(xp_inv_transformed)[0].__name__ == xp.__name__
+        assert get_namespace(xp_label.classes_)[0].__name__ == xp.__name__
+        assert_array_equal(_convert_to_numpy(xp_transformed, xp), np_transformed)
+        assert_array_equal(_convert_to_numpy(xp_inv_transformed, xp), y)
+        assert_array_equal(_convert_to_numpy(xp_label.classes_, xp), np_label.classes_)
+
+        xp_label = LabelEncoder()
+        np_label = LabelEncoder()
+        xp_transformed = xp_label.fit_transform(xp_y)
+        np_transformed = np_label.fit_transform(y)
+        assert get_namespace(xp_transformed)[0].__name__ == xp.__name__
+        assert get_namespace(xp_label.classes_)[0].__name__ == xp.__name__
+        assert_array_equal(_convert_to_numpy(xp_transformed, xp), np_transformed)
+        assert_array_equal(_convert_to_numpy(xp_label.classes_, xp), np_label.classes_)
diff --git a/sklearn/preprocessing/tests/test_polynomial.py b/sklearn/preprocessing/tests/test_polynomial.py
index 2ca3260f7c05e..640bf5705baad 100644
--- a/sklearn/preprocessing/tests/test_polynomial.py
+++ b/sklearn/preprocessing/tests/test_polynomial.py
@@ -1,14 +1,32 @@
+import sys
+
 import numpy as np
-from numpy.testing import assert_allclose, assert_array_equal
 import pytest
+from numpy.testing import assert_allclose, assert_array_equal
+from scipy import sparse
+from scipy.interpolate import BSpline
+from scipy.sparse import random as sparse_random
 
 from sklearn.linear_model import LinearRegression
 from sklearn.pipeline import Pipeline
-from sklearn.preprocessing import KBinsDiscretizer, SplineTransformer
+from sklearn.preprocessing import (
+    KBinsDiscretizer,
+    PolynomialFeatures,
+    SplineTransformer,
+)
+from sklearn.preprocessing._csr_polynomial_expansion import (
+    _get_sizeof_LARGEST_INT_t,
+)
+from sklearn.utils._testing import assert_array_almost_equal
+from sklearn.utils.fixes import (
+    CSC_CONTAINERS,
+    CSR_CONTAINERS,
+    parse_version,
+    sp_version,
+)
 
 
-# TODO: add PolynomialFeatures if it moves to _polynomial.py
-@pytest.mark.parametrize("est", (SplineTransformer,))
+@pytest.mark.parametrize("est", (PolynomialFeatures, SplineTransformer))
 def test_polynomial_and_spline_array_order(est):
     """Test that output array has the given order."""
     X = np.arange(10).reshape(5, 2)
@@ -24,46 +42,9 @@ def is_c_contiguous(a):
 @pytest.mark.parametrize(
     "params, err_msg",
     [
-        ({"degree": -1}, "degree must be a non-negative integer."),
-        ({"degree": 2.5}, "degree must be a non-negative integer."),
-        ({"degree": "string"}, "degree must be a non-negative integer."),
-        ({"n_knots": 1}, "n_knots must be a positive integer >= 2."),
-        ({"n_knots": 1}, "n_knots must be a positive integer >= 2."),
-        ({"n_knots": 2.5}, "n_knots must be a positive integer >= 2."),
-        ({"n_knots": "string"}, "n_knots must be a positive integer >= 2."),
-        ({"knots": "string"}, "Expected 2D array, got scalar array instead:"),
-        ({"knots": [1, 2]}, "Expected 2D array, got 1D array instead:"),
-        (
-            {"knots": [[1]]},
-            r"Number of knots, knots.shape\[0\], must be >= 2.",
-        ),
-        (
-            {"knots": [[1, 5], [2, 6]]},
-            r"knots.shape\[1\] == n_features is violated.",
-        ),
-        (
-            {"knots": [[1], [1], [2]]},
-            "knots must be sorted without duplicates.",
-        ),
-        ({"knots": [[2], [1]]}, "knots must be sorted without duplicates."),
-        (
-            {"extrapolation": None},
-            "extrapolation must be one of 'error', 'constant', 'linear' or "
-            "'continue'.",
-        ),
-        (
-            {"extrapolation": 1},
-            "extrapolation must be one of 'error', 'constant', 'linear' or "
-            "'continue'.",
-        ),
-        (
-            {"extrapolation": "string"},
-            "extrapolation must be one of 'error', 'constant', 'linear' or "
-            "'continue'.",
-        ),
-        ({"include_bias": None}, "include_bias must be bool."),
-        ({"include_bias": 1}, "include_bias must be bool."),
-        ({"include_bias": "string"}, "include_bias must be bool."),
+        ({"knots": [[1]]}, r"Number of knots, knots.shape\[0\], must be >= 2."),
+        ({"knots": [[1, 1], [2, 2]]}, r"knots.shape\[1\] == n_features is violated"),
+        ({"knots": [[1], [0]]}, "knots must be sorted without duplicates."),
     ],
 )
 def test_spline_transformer_input_validation(params, err_msg):
@@ -74,23 +55,21 @@ def test_spline_transformer_input_validation(params, err_msg):
         SplineTransformer(**params).fit(X)
 
 
-def test_spline_transformer_manual_knot_input():
-    """Test that array-like knot positions in SplineTransformer are accepted.
-    """
+@pytest.mark.parametrize("extrapolation", ["continue", "periodic"])
+def test_spline_transformer_integer_knots(extrapolation):
+    """Test that SplineTransformer accepts integer value knot positions."""
     X = np.arange(20).reshape(10, 2)
-    knots = [[0.5, 1], [1.5, 2], [5, 10]]
-    st1 = SplineTransformer(degree=3, knots=knots).fit(X)
-    knots = np.asarray(knots)
-    st2 = SplineTransformer(degree=3, knots=knots).fit(X)
-    for i in range(X.shape[1]):
-        assert_allclose(st1.bsplines_[i].t, st2.bsplines_[i].t)
+    knots = [[0, 1], [1, 2], [5, 5], [11, 10], [12, 11]]
+    _ = SplineTransformer(
+        degree=3, knots=knots, extrapolation=extrapolation
+    ).fit_transform(X)
 
 
 def test_spline_transformer_feature_names():
     """Test that SplineTransformer generates correct features name."""
     X = np.arange(20).reshape(10, 2)
     splt = SplineTransformer(n_knots=3, degree=3, include_bias=True).fit(X)
-    feature_names = splt.get_feature_names()
+    feature_names = splt.get_feature_names_out()
     assert_array_equal(
         feature_names,
         [
@@ -108,7 +87,7 @@ def test_spline_transformer_feature_names():
     )
 
     splt = SplineTransformer(n_knots=3, degree=3, include_bias=False).fit(X)
-    feature_names = splt.get_feature_names(["a", "b"])
+    feature_names = splt.get_feature_names_out(["a", "b"])
     assert_array_equal(
         feature_names,
         [
@@ -124,21 +103,48 @@ def test_spline_transformer_feature_names():
     )
 
 
+@pytest.mark.parametrize(
+    "extrapolation",
+    ["constant", "linear", "continue", "periodic"],
+)
+@pytest.mark.parametrize("degree", [2, 3])
+def test_split_transform_feature_names_extrapolation_degree(extrapolation, degree):
+    """Test feature names are correct for different extrapolations and degree.
+
+    Non-regression test for gh-25292.
+    """
+    X = np.arange(20).reshape(10, 2)
+    splt = SplineTransformer(degree=degree, extrapolation=extrapolation).fit(X)
+    feature_names = splt.get_feature_names_out(["a", "b"])
+    assert len(feature_names) == splt.n_features_out_
+
+    X_trans = splt.transform(X)
+    assert X_trans.shape[1] == len(feature_names)
+
+
 @pytest.mark.parametrize("degree", range(1, 5))
 @pytest.mark.parametrize("n_knots", range(3, 5))
 @pytest.mark.parametrize("knots", ["uniform", "quantile"])
-def test_spline_transformer_unity_decomposition(degree, n_knots, knots):
+@pytest.mark.parametrize("extrapolation", ["constant", "periodic"])
+def test_spline_transformer_unity_decomposition(degree, n_knots, knots, extrapolation):
     """Test that B-splines are indeed a decomposition of unity.
 
-    Splines basis functions must sum up to 1 per row, if we stay in between
-    boundaries.
+    Splines basis functions must sum up to 1 per row, if we stay in between boundaries.
     """
     X = np.linspace(0, 1, 100)[:, None]
     # make the boundaries 0 and 1 part of X_train, for sure.
     X_train = np.r_[[[0]], X[::2, :], [[1]]]
     X_test = X[1::2, :]
+
+    if extrapolation == "periodic":
+        n_knots = n_knots + degree  # periodic splines require degree < n_knots
+
     splt = SplineTransformer(
-        n_knots=n_knots, degree=degree, knots=knots, include_bias=True
+        n_knots=n_knots,
+        degree=degree,
+        knots=knots,
+        include_bias=True,
+        extrapolation=extrapolation,
     )
     splt.fit(X_train)
     for X in [X_train, X_test]:
@@ -168,6 +174,148 @@ def test_spline_transformer_linear_regression(bias, intercept):
     assert_allclose(pipe.predict(X), y, rtol=1e-3)
 
 
+@pytest.mark.parametrize(
+    ["knots", "n_knots", "sample_weight", "expected_knots"],
+    [
+        ("uniform", 3, None, np.array([[0, 2], [3, 8], [6, 14]])),
+        (
+            "uniform",
+            3,
+            np.array([0, 0, 1, 1, 0, 3, 1]),
+            np.array([[2, 2], [4, 8], [6, 14]]),
+        ),
+        ("uniform", 4, None, np.array([[0, 2], [2, 6], [4, 10], [6, 14]])),
+        ("quantile", 3, None, np.array([[0, 2], [3, 3], [6, 14]])),
+        (
+            "quantile",
+            3,
+            np.array([0, 0, 1, 1, 0, 3, 1]),
+            np.array([[2, 2], [5, 8], [6, 14]]),
+        ),
+    ],
+)
+def test_spline_transformer_get_base_knot_positions(
+    knots, n_knots, sample_weight, expected_knots
+):
+    """Check the behaviour to find knot positions with and without sample_weight."""
+    X = np.array([[0, 2], [0, 2], [2, 2], [3, 3], [4, 6], [5, 8], [6, 14]])
+    base_knots = SplineTransformer._get_base_knot_positions(
+        X=X, knots=knots, n_knots=n_knots, sample_weight=sample_weight
+    )
+    assert_allclose(base_knots, expected_knots)
+
+
+@pytest.mark.parametrize(["bias", "intercept"], [(True, False), (False, True)])
+def test_spline_transformer_periodic_linear_regression(bias, intercept):
+    """Test that B-splines fit a periodic curve pretty well."""
+
+    # "+ 3" to avoid the value 0 in assert_allclose
+    def f(x):
+        return np.sin(2 * np.pi * x) - np.sin(8 * np.pi * x) + 3
+
+    X = np.linspace(0, 1, 101)[:, None]
+    pipe = Pipeline(
+        steps=[
+            (
+                "spline",
+                SplineTransformer(
+                    n_knots=20,
+                    degree=3,
+                    include_bias=bias,
+                    extrapolation="periodic",
+                ),
+            ),
+            ("ols", LinearRegression(fit_intercept=intercept)),
+        ]
+    )
+    pipe.fit(X, f(X[:, 0]))
+
+    # Generate larger array to check periodic extrapolation
+    X_ = np.linspace(-1, 2, 301)[:, None]
+    predictions = pipe.predict(X_)
+    assert_allclose(predictions, f(X_[:, 0]), atol=0.01, rtol=0.01)
+    assert_allclose(predictions[0:100], predictions[100:200], rtol=1e-3)
+
+
+def test_spline_transformer_periodic_spline_backport():
+    """Test that the backport of extrapolate="periodic" works correctly"""
+    X = np.linspace(-2, 3.5, 10)[:, None]
+    degree = 2
+
+    # Use periodic extrapolation backport in SplineTransformer
+    transformer = SplineTransformer(
+        degree=degree, extrapolation="periodic", knots=[[-1.0], [0.0], [1.0]]
+    )
+    Xt = transformer.fit_transform(X)
+
+    # Use periodic extrapolation in BSpline
+    coef = np.array([[1.0, 0.0], [0.0, 1.0], [1.0, 0.0], [0.0, 1.0]])
+    spl = BSpline(np.arange(-3, 4), coef, degree, "periodic")
+    Xspl = spl(X[:, 0])
+    assert_allclose(Xt, Xspl)
+
+
+def test_spline_transformer_periodic_splines_periodicity():
+    """Test if shifted knots result in the same transformation up to permutation."""
+    X = np.linspace(0, 10, 101)[:, None]
+
+    transformer_1 = SplineTransformer(
+        degree=3,
+        extrapolation="periodic",
+        knots=[[0.0], [1.0], [3.0], [4.0], [5.0], [8.0]],
+    )
+
+    transformer_2 = SplineTransformer(
+        degree=3,
+        extrapolation="periodic",
+        knots=[[1.0], [3.0], [4.0], [5.0], [8.0], [9.0]],
+    )
+
+    Xt_1 = transformer_1.fit_transform(X)
+    Xt_2 = transformer_2.fit_transform(X)
+
+    assert_allclose(Xt_1, Xt_2[:, [4, 0, 1, 2, 3]])
+
+
+@pytest.mark.parametrize("degree", [3, 5])
+def test_spline_transformer_periodic_splines_smoothness(degree):
+    """Test that spline transformation is smooth at first / last knot."""
+    X = np.linspace(-2, 10, 10_000)[:, None]
+
+    transformer = SplineTransformer(
+        degree=degree,
+        extrapolation="periodic",
+        knots=[[0.0], [1.0], [3.0], [4.0], [5.0], [8.0]],
+    )
+    Xt = transformer.fit_transform(X)
+
+    delta = (X.max() - X.min()) / len(X)
+    tol = 10 * delta
+
+    dXt = Xt
+    # We expect splines of degree `degree` to be (`degree`-1) times
+    # continuously differentiable. I.e. for d = 0, ..., `degree` - 1 the d-th
+    # derivative should be continuous. This is the case if the (d+1)-th
+    # numerical derivative is reasonably small (smaller than `tol` in absolute
+    # value). We thus compute d-th numeric derivatives for d = 1, ..., `degree`
+    # and compare them to `tol`.
+    #
+    # Note that the 0-th derivative is the function itself, such that we are
+    # also checking its continuity.
+    for d in range(1, degree + 1):
+        # Check continuity of the (d-1)-th derivative
+        diff = np.diff(dXt, axis=0)
+        assert np.abs(diff).max() < tol
+        # Compute d-th numeric derivative
+        dXt = diff / delta
+
+    # As degree `degree` splines are not `degree` times continuously
+    # differentiable at the knots, the `degree + 1`-th numeric derivative
+    # should have spikes at the knots.
+    diff = np.diff(dXt, axis=0)
+    assert np.abs(diff).max() > 1
+
+
 @pytest.mark.parametrize(["bias", "intercept"], [(True, False), (False, True)])
 @pytest.mark.parametrize("degree", [1, 2, 3, 4, 5])
 def test_spline_transformer_extrapolation(bias, intercept, degree):
@@ -217,15 +365,16 @@ def test_spline_transformer_extrapolation(bias, intercept, degree):
         n_knots=4, degree=degree, include_bias=bias, extrapolation="error"
     )
     splt.fit(X)
-    with pytest.raises(ValueError):
+    msg = "X contains values beyond the limits of the knots"
+    with pytest.raises(ValueError, match=msg):
         splt.transform([[-10]])
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError, match=msg):
         splt.transform([[5]])
 
 
-def test_spline_transformer_kbindiscretizer():
+def test_spline_transformer_kbindiscretizer(global_random_seed):
     """Test that a B-spline of degree=0 is equivalent to KBinsDiscretizer."""
-    rng = np.random.RandomState(97531)
+    rng = np.random.RandomState(global_random_seed)
     X = rng.randn(200).reshape(200, 1)
     n_bins = 5
     n_knots = n_bins + 1
@@ -236,7 +385,10 @@ def test_spline_transformer_kbindiscretizer():
     splines = splt.fit_transform(X)
 
     kbd = KBinsDiscretizer(
-        n_bins=n_bins, encode="onehot-dense", strategy="quantile"
+        n_bins=n_bins,
+        encode="onehot-dense",
+        strategy="quantile",
+        quantile_method="averaged_inverted_cdf",
     )
     kbins = kbd.fit_transform(X)
 
@@ -245,17 +397,834 @@ def test_spline_transformer_kbindiscretizer():
     assert_allclose(splines, kbins, rtol=1e-13)
 
 
+@pytest.mark.parametrize("degree", range(1, 3))
+@pytest.mark.parametrize("knots", ["uniform", "quantile"])
+@pytest.mark.parametrize(
+    "extrapolation", ["error", "constant", "linear", "continue", "periodic"]
+)
+@pytest.mark.parametrize("include_bias", [False, True])
+def test_spline_transformer_sparse_output(
+    degree, knots, extrapolation, include_bias, global_random_seed
+):
+    rng = np.random.RandomState(global_random_seed)
+    X = rng.randn(200).reshape(40, 5)
+
+    splt_dense = SplineTransformer(
+        degree=degree,
+        knots=knots,
+        extrapolation=extrapolation,
+        include_bias=include_bias,
+        sparse_output=False,
+    )
+    splt_sparse = SplineTransformer(
+        degree=degree,
+        knots=knots,
+        extrapolation=extrapolation,
+        include_bias=include_bias,
+        sparse_output=True,
+    )
+
+    splt_dense.fit(X)
+    splt_sparse.fit(X)
+
+    X_trans_sparse = splt_sparse.transform(X)
+    X_trans_dense = splt_dense.transform(X)
+    assert sparse.issparse(X_trans_sparse) and X_trans_sparse.format == "csr"
+    assert_allclose(X_trans_dense, X_trans_sparse.toarray())
+
+    # extrapolation regime
+    X_min = np.amin(X, axis=0)
+    X_max = np.amax(X, axis=0)
+    X_extra = np.r_[
+        np.linspace(X_min - 5, X_min, 10), np.linspace(X_max, X_max + 5, 10)
+    ]
+    if extrapolation == "error":
+        msg = "X contains values beyond the limits of the knots"
+        with pytest.raises(ValueError, match=msg):
+            splt_dense.transform(X_extra)
+        msg = "Out of bounds"
+        with pytest.raises(ValueError, match=msg):
+            splt_sparse.transform(X_extra)
+    else:
+        assert_allclose(
+            splt_dense.transform(X_extra), splt_sparse.transform(X_extra).toarray()
+        )
+
+
 @pytest.mark.parametrize("n_knots", [5, 10])
 @pytest.mark.parametrize("include_bias", [True, False])
-@pytest.mark.parametrize("degree", [3, 5])
-def test_spline_transformer_n_features_out(n_knots, include_bias, degree):
+@pytest.mark.parametrize("degree", [3, 4])
+@pytest.mark.parametrize(
+    "extrapolation", ["error", "constant", "linear", "continue", "periodic"]
+)
+@pytest.mark.parametrize("sparse_output", [False, True])
+def test_spline_transformer_n_features_out(
+    n_knots, include_bias, degree, extrapolation, sparse_output
+):
     """Test that transform results in n_features_out_ features."""
     splt = SplineTransformer(
         n_knots=n_knots,
         degree=degree,
-        include_bias=include_bias
+        include_bias=include_bias,
+        extrapolation=extrapolation,
+        sparse_output=sparse_output,
     )
     X = np.linspace(0, 1, 10)[:, None]
     splt.fit(X)
 
     assert splt.transform(X).shape[1] == splt.n_features_out_
+
+
+@pytest.mark.parametrize(
+    "params, err_msg",
+    [
+        ({"degree": (-1, 2)}, r"degree=\(min_degree, max_degree\) must"),
+        ({"degree": (0, 1.5)}, r"degree=\(min_degree, max_degree\) must"),
+        ({"degree": (3, 2)}, r"degree=\(min_degree, max_degree\) must"),
+        ({"degree": (1, 2, 3)}, r"int or tuple \(min_degree, max_degree\)"),
+    ],
+)
+def test_polynomial_features_input_validation(params, err_msg):
+    """Test that we raise errors for invalid input in PolynomialFeatures."""
+    X = [[1], [2]]
+
+    with pytest.raises(ValueError, match=err_msg):
+        PolynomialFeatures(**params).fit(X)
+
+
+@pytest.fixture()
+def single_feature_degree3():
+    X = np.arange(6)[:, np.newaxis]
+    P = np.hstack([np.ones_like(X), X, X**2, X**3])
+    return X, P
+
+
+@pytest.mark.parametrize(
+    "degree, include_bias, interaction_only, indices",
+    [
+        (3, True, False, slice(None, None)),
+        (3, False, False, slice(1, None)),
+        (3, True, True, [0, 1]),
+        (3, False, True, [1]),
+        ((2, 3), True, False, [0, 2, 3]),
+        ((2, 3), False, False, [2, 3]),
+        ((2, 3), True, True, [0]),
+        ((2, 3), False, True, []),
+    ],
+)
+@pytest.mark.parametrize("X_container", [None] + CSR_CONTAINERS + CSC_CONTAINERS)
+def test_polynomial_features_one_feature(
+    single_feature_degree3,
+    degree,
+    include_bias,
+    interaction_only,
+    indices,
+    X_container,
+):
+    """Test PolynomialFeatures on single feature up to degree 3."""
+    X, P = single_feature_degree3
+    if X_container is not None:
+        X = X_container(X)
+    tf = PolynomialFeatures(
+        degree=degree, include_bias=include_bias, interaction_only=interaction_only
+    ).fit(X)
+    out = tf.transform(X)
+    if X_container is not None:
+        out = out.toarray()
+    assert_allclose(out, P[:, indices])
+    if tf.n_output_features_ > 0:
+        assert tf.powers_.shape == (tf.n_output_features_, tf.n_features_in_)
+
+
+@pytest.fixture()
+def two_features_degree3():
+    X = np.arange(6).reshape((3, 2))
+    x1 = X[:, :1]
+    x2 = X[:, 1:]
+    P = np.hstack(
+        [
+            x1**0 * x2**0,  # 0
+            x1**1 * x2**0,  # 1
+            x1**0 * x2**1,  # 2
+            x1**2 * x2**0,  # 3
+            x1**1 * x2**1,  # 4
+            x1**0 * x2**2,  # 5
+            x1**3 * x2**0,  # 6
+            x1**2 * x2**1,  # 7
+            x1**1 * x2**2,  # 8
+            x1**0 * x2**3,  # 9
+        ]
+    )
+    return X, P
+
+
+@pytest.mark.parametrize(
+    "degree, include_bias, interaction_only, indices",
+    [
+        (2, True, False, slice(0, 6)),
+        (2, False, False, slice(1, 6)),
+        (2, True, True, [0, 1, 2, 4]),
+        (2, False, True, [1, 2, 4]),
+        ((2, 2), True, False, [0, 3, 4, 5]),
+        ((2, 2), False, False, [3, 4, 5]),
+        ((2, 2), True, True, [0, 4]),
+        ((2, 2), False, True, [4]),
+        (3, True, False, slice(None, None)),
+        (3, False, False, slice(1, None)),
+        (3, True, True, [0, 1, 2, 4]),
+        (3, False, True, [1, 2, 4]),
+        ((2, 3), True, False, [0, 3, 4, 5, 6, 7, 8, 9]),
+        ((2, 3), False, False, slice(3, None)),
+        ((2, 3), True, True, [0, 4]),
+        ((2, 3), False, True, [4]),
+        ((3, 3), True, False, [0, 6, 7, 8, 9]),
+        ((3, 3), False, False, [6, 7, 8, 9]),
+        ((3, 3), True, True, [0]),
+        ((3, 3), False, True, []),  # would need 3 input features
+    ],
+)
+@pytest.mark.parametrize("X_container", [None] + CSR_CONTAINERS + CSC_CONTAINERS)
+def test_polynomial_features_two_features(
+    two_features_degree3,
+    degree,
+    include_bias,
+    interaction_only,
+    indices,
+    X_container,
+):
+    """Test PolynomialFeatures on 2 features up to degree 3."""
+    X, P = two_features_degree3
+    if X_container is not None:
+        X = X_container(X)
+    tf = PolynomialFeatures(
+        degree=degree, include_bias=include_bias, interaction_only=interaction_only
+    ).fit(X)
+    out = tf.transform(X)
+    if X_container is not None:
+        out = out.toarray()
+    assert_allclose(out, P[:, indices])
+    if tf.n_output_features_ > 0:
+        assert tf.powers_.shape == (tf.n_output_features_, tf.n_features_in_)
+
+
+def test_polynomial_feature_names():
+    X = np.arange(30).reshape(10, 3)
+    poly = PolynomialFeatures(degree=2, include_bias=True).fit(X)
+    feature_names = poly.get_feature_names_out()
+    assert_array_equal(
+        ["1", "x0", "x1", "x2", "x0^2", "x0 x1", "x0 x2", "x1^2", "x1 x2", "x2^2"],
+        feature_names,
+    )
+    assert len(feature_names) == poly.transform(X).shape[1]
+
+    poly = PolynomialFeatures(degree=3, include_bias=False).fit(X)
+    feature_names = poly.get_feature_names_out(["a", "b", "c"])
+    assert_array_equal(
+        [
+            "a",
+            "b",
+            "c",
+            "a^2",
+            "a b",
+            "a c",
+            "b^2",
+            "b c",
+            "c^2",
+            "a^3",
+            "a^2 b",
+            "a^2 c",
+            "a b^2",
+            "a b c",
+            "a c^2",
+            "b^3",
+            "b^2 c",
+            "b c^2",
+            "c^3",
+        ],
+        feature_names,
+    )
+    assert len(feature_names) == poly.transform(X).shape[1]
+
+    poly = PolynomialFeatures(degree=(2, 3), include_bias=False).fit(X)
+    feature_names = poly.get_feature_names_out(["a", "b", "c"])
+    assert_array_equal(
+        [
+            "a^2",
+            "a b",
+            "a c",
+            "b^2",
+            "b c",
+            "c^2",
+            "a^3",
+            "a^2 b",
+            "a^2 c",
+            "a b^2",
+            "a b c",
+            "a c^2",
+            "b^3",
+            "b^2 c",
+            "b c^2",
+            "c^3",
+        ],
+        feature_names,
+    )
+    assert len(feature_names) == poly.transform(X).shape[1]
+
+    poly = PolynomialFeatures(
+        degree=(3, 3), include_bias=True, interaction_only=True
+    ).fit(X)
+    feature_names = poly.get_feature_names_out(["a", "b", "c"])
+    assert_array_equal(["1", "a b c"], feature_names)
+    assert len(feature_names) == poly.transform(X).shape[1]
+
+    # test some unicode
+    poly = PolynomialFeatures(degree=1, include_bias=True).fit(X)
+    feature_names = poly.get_feature_names_out(["\u0001F40D", "\u262e", "\u05d0"])
+    assert_array_equal(["1", "\u0001F40D", "\u262e", "\u05d0"], feature_names)
+
+
+@pytest.mark.parametrize(
+    ["deg", "include_bias", "interaction_only", "dtype"],
+    [
+        (1, True, False, int),
+        (2, True, False, int),
+        (2, True, False, np.float32),
+        (2, True, False, np.float64),
+        (3, False, False, np.float64),
+        (3, False, True, np.float64),
+        (4, False, False, np.float64),
+        (4, False, True, np.float64),
+    ],
+)
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+def test_polynomial_features_csc_X(
+    deg, include_bias, interaction_only, dtype, csc_container, global_random_seed
+):
+    rng = np.random.RandomState(global_random_seed)
+    X = rng.randint(0, 2, (100, 2))
+    X_csc = csc_container(X)
+
+    est = PolynomialFeatures(
+        deg, include_bias=include_bias, interaction_only=interaction_only
+    )
+    Xt_csc = est.fit_transform(X_csc.astype(dtype))
+    Xt_dense = est.fit_transform(X.astype(dtype))
+
+    assert sparse.issparse(Xt_csc) and Xt_csc.format == "csc"
+    assert Xt_csc.dtype == Xt_dense.dtype
+    assert_array_almost_equal(Xt_csc.toarray(), Xt_dense)
+
+
+@pytest.mark.parametrize(
+    ["deg", "include_bias", "interaction_only", "dtype"],
+    [
+        (1, True, False, int),
+        (2, True, False, int),
+        (2, True, False, np.float32),
+        (2, True, False, np.float64),
+        (3, False, False, np.float64),
+        (3, False, True, np.float64),
+    ],
+)
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_polynomial_features_csr_X(
+    deg, include_bias, interaction_only, dtype, csr_container, global_random_seed
+):
+    rng = np.random.RandomState(global_random_seed)
+    X = rng.randint(0, 2, (100, 2))
+    X_csr = csr_container(X)
+
+    est = PolynomialFeatures(
+        deg, include_bias=include_bias, interaction_only=interaction_only
+    )
+    Xt_csr = est.fit_transform(X_csr.astype(dtype))
+    Xt_dense = est.fit_transform(X.astype(dtype, copy=False))
+
+    assert sparse.issparse(Xt_csr) and Xt_csr.format == "csr"
+    assert Xt_csr.dtype == Xt_dense.dtype
+    assert_array_almost_equal(Xt_csr.toarray(), Xt_dense)
+
+
+@pytest.mark.parametrize("n_features", [1, 4, 5])
+@pytest.mark.parametrize(
+    "min_degree, max_degree", [(0, 1), (0, 2), (1, 3), (0, 4), (3, 4)]
+)
+@pytest.mark.parametrize("interaction_only", [True, False])
+@pytest.mark.parametrize("include_bias", [True, False])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_num_combinations(
+    n_features, min_degree, max_degree, interaction_only, include_bias, csr_container
+):
+    """
+    Test that n_output_features_ is calculated correctly.
+    """
+    x = csr_container(([1], ([0], [n_features - 1])))
+    est = PolynomialFeatures(
+        degree=max_degree,
+        interaction_only=interaction_only,
+        include_bias=include_bias,
+    )
+    est.fit(x)
+    num_combos = est.n_output_features_
+
+    combos = PolynomialFeatures._combinations(
+        n_features=n_features,
+        min_degree=0,
+        max_degree=max_degree,
+        interaction_only=interaction_only,
+        include_bias=include_bias,
+    )
+    assert num_combos == sum([1 for _ in combos])
+
+
+@pytest.mark.parametrize(
+    ["deg", "include_bias", "interaction_only", "dtype"],
+    [
+        (2, True, False, np.float32),
+        (2, True, False, np.float64),
+        (3, False, False, np.float64),
+        (3, False, True, np.float64),
+    ],
+)
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_polynomial_features_csr_X_floats(
+    deg, include_bias, interaction_only, dtype, csr_container, global_random_seed
+):
+    X_csr = csr_container(sparse_random(1000, 10, 0.5, random_state=global_random_seed))
+    X = X_csr.toarray()
+
+    est = PolynomialFeatures(
+        deg, include_bias=include_bias, interaction_only=interaction_only
+    )
+    Xt_csr = est.fit_transform(X_csr.astype(dtype))
+    Xt_dense = est.fit_transform(X.astype(dtype))
+
+    assert sparse.issparse(Xt_csr) and Xt_csr.format == "csr"
+    assert Xt_csr.dtype == Xt_dense.dtype
+    assert_array_almost_equal(Xt_csr.toarray(), Xt_dense)
+
+
+@pytest.mark.parametrize(
+    ["zero_row_index", "deg", "interaction_only"],
+    [
+        (0, 2, True),
+        (1, 2, True),
+        (2, 2, True),
+        (0, 3, True),
+        (1, 3, True),
+        (2, 3, True),
+        (0, 2, False),
+        (1, 2, False),
+        (2, 2, False),
+        (0, 3, False),
+        (1, 3, False),
+        (2, 3, False),
+    ],
+)
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_polynomial_features_csr_X_zero_row(
+    zero_row_index, deg, interaction_only, csr_container, global_random_seed
+):
+    X_csr = csr_container(sparse_random(3, 10, 1.0, random_state=global_random_seed))
+    X_csr[zero_row_index, :] = 0.0
+    X = X_csr.toarray()
+
+    est = PolynomialFeatures(deg, include_bias=False, interaction_only=interaction_only)
+    Xt_csr = est.fit_transform(X_csr)
+    Xt_dense = est.fit_transform(X)
+
+    assert sparse.issparse(Xt_csr) and Xt_csr.format == "csr"
+    assert Xt_csr.dtype == Xt_dense.dtype
+    assert_array_almost_equal(Xt_csr.toarray(), Xt_dense)
+
+
+# This degree should always be one more than the highest degree supported by
+# _csr_expansion.
+@pytest.mark.parametrize(
+    ["include_bias", "interaction_only"],
+    [(True, True), (True, False), (False, True), (False, False)],
+)
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_polynomial_features_csr_X_degree_4(
+    include_bias, interaction_only, csr_container, global_random_seed
+):
+    X_csr = csr_container(sparse_random(1000, 10, 0.5, random_state=global_random_seed))
+    X = X_csr.toarray()
+
+    est = PolynomialFeatures(
+        4, include_bias=include_bias, interaction_only=interaction_only
+    )
+    Xt_csr = est.fit_transform(X_csr)
+    Xt_dense = est.fit_transform(X)
+
+    assert sparse.issparse(Xt_csr) and Xt_csr.format == "csr"
+    assert Xt_csr.dtype == Xt_dense.dtype
+    assert_array_almost_equal(Xt_csr.toarray(), Xt_dense)
+
+
+@pytest.mark.parametrize(
+    ["deg", "dim", "interaction_only"],
+    [
+        (2, 1, True),
+        (2, 2, True),
+        (3, 1, True),
+        (3, 2, True),
+        (3, 3, True),
+        (2, 1, False),
+        (2, 2, False),
+        (3, 1, False),
+        (3, 2, False),
+        (3, 3, False),
+    ],
+)
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_polynomial_features_csr_X_dim_edges(
+    deg, dim, interaction_only, csr_container, global_random_seed
+):
+    X_csr = csr_container(
+        sparse_random(1000, dim, 0.5, random_state=global_random_seed)
+    )
+    X = X_csr.toarray()
+
+    est = PolynomialFeatures(deg, interaction_only=interaction_only)
+    Xt_csr = est.fit_transform(X_csr)
+    Xt_dense = est.fit_transform(X)
+
+    assert sparse.issparse(Xt_csr) and Xt_csr.format == "csr"
+    assert Xt_csr.dtype == Xt_dense.dtype
+    assert_array_almost_equal(Xt_csr.toarray(), Xt_dense)
+
+
+@pytest.mark.parametrize("interaction_only", [True, False])
+@pytest.mark.parametrize("include_bias", [True, False])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_csr_polynomial_expansion_index_overflow_non_regression(
+    interaction_only, include_bias, csr_container
+):
+    """Check the automatic index dtype promotion to `np.int64` when needed.
+
+    This ensures that sufficiently large input configurations get
+    properly promoted to use `np.int64` for index and indptr representation
+    while preserving data integrity. Non-regression test for gh-16803.
+
+    Note that this is only possible for Python runtimes with a 64 bit address
+    space. On 32 bit platforms, a `ValueError` is raised instead.
+    """
+
+    def degree_2_calc(d, i, j):
+        if interaction_only:
+            return d * i - (i**2 + 3 * i) // 2 - 1 + j
+        else:
+            return d * i - (i**2 + i) // 2 + j
+
+    n_samples = 13
+    n_features = 120001
+    data_dtype = np.float32
+    data = np.arange(1, 5, dtype=np.int64)
+    row = np.array([n_samples - 2, n_samples - 2, n_samples - 1, n_samples - 1])
+    # An int64 dtype is required to avoid overflow error on Windows within the
+    # `degree_2_calc` function.
+    col = np.array(
+        [n_features - 2, n_features - 1, n_features - 2, n_features - 1], dtype=np.int64
+    )
+    X = csr_container(
+        (data, (row, col)),
+        shape=(n_samples, n_features),
+        dtype=data_dtype,
+    )
+    pf = PolynomialFeatures(
+        interaction_only=interaction_only, include_bias=include_bias, degree=2
+    )
+
+    # Calculate the number of combinations a-priori, and if needed check for
+    # the correct ValueError and terminate the test early.
+    num_combinations = pf._num_combinations(
+        n_features=n_features,
+        min_degree=0,
+        max_degree=2,
+        interaction_only=pf.interaction_only,
+        include_bias=pf.include_bias,
+    )
+    if num_combinations > np.iinfo(np.intp).max:
+        msg = (
+            r"The output that would result from the current configuration would have"
+            r" \d* features which is too large to be indexed"
+        )
+        with pytest.raises(ValueError, match=msg):
+            pf.fit(X)
+        return
+    X_trans = pf.fit_transform(X)
+    row_nonzero, col_nonzero = X_trans.nonzero()
+    n_degree_1_features_out = n_features + include_bias
+    max_degree_2_idx = (
+        degree_2_calc(n_features, col[int(not interaction_only)], col[1])
+        + n_degree_1_features_out
+    )
+
+    # Account for bias of all samples except last one which will be handled
+    # separately since there are distinct data values before it
+    data_target = [1] * (n_samples - 2) if include_bias else []
+    col_nonzero_target = [0] * (n_samples - 2) if include_bias else []
+
+    for i in range(2):
+        x = data[2 * i]
+        y = data[2 * i + 1]
+        x_idx = col[2 * i]
+        y_idx = col[2 * i + 1]
+        if include_bias:
+            data_target.append(1)
+            col_nonzero_target.append(0)
+        data_target.extend([x, y])
+        col_nonzero_target.extend(
+            [x_idx + int(include_bias), y_idx + int(include_bias)]
+        )
+        if not interaction_only:
+            data_target.extend([x * x, x * y, y * y])
+            col_nonzero_target.extend(
+                [
+                    degree_2_calc(n_features, x_idx, x_idx) + n_degree_1_features_out,
+                    degree_2_calc(n_features, x_idx, y_idx) + n_degree_1_features_out,
+                    degree_2_calc(n_features, y_idx, y_idx) + n_degree_1_features_out,
+                ]
+            )
+        else:
+            data_target.extend([x * y])
+            col_nonzero_target.append(
+                degree_2_calc(n_features, x_idx, y_idx) + n_degree_1_features_out
+            )
+
+    nnz_per_row = int(include_bias) + 3 + 2 * int(not interaction_only)
+
+    assert pf.n_output_features_ == max_degree_2_idx + 1
+    assert X_trans.dtype == data_dtype
+    assert X_trans.shape == (n_samples, max_degree_2_idx + 1)
+    assert X_trans.indptr.dtype == X_trans.indices.dtype == np.int64
+    # Ensure that dtype promotion was actually required:
+    assert X_trans.indices.max() > np.iinfo(np.int32).max
+
+    row_nonzero_target = list(range(n_samples - 2)) if include_bias else []
+    row_nonzero_target.extend(
+        [n_samples - 2] * nnz_per_row + [n_samples - 1] * nnz_per_row
+    )
+
+    assert_allclose(X_trans.data, data_target)
+    assert_array_equal(row_nonzero, row_nonzero_target)
+    assert_array_equal(col_nonzero, col_nonzero_target)
+
+
+@pytest.mark.parametrize(
+    "degree, n_features",
+    [
+        # Needs promotion to int64 when interaction_only=False
+        (2, 65535),
+        (3, 2344),
+        # This guarantees that the intermediate operation when calculating
+        # output columns would overflow a C-long, hence checks that python-
+        # longs are being used.
+        (2, int(np.sqrt(np.iinfo(np.int64).max) + 1)),
+        (3, 65535),
+        # This case tests the second clause of the overflow check which
+        # takes into account the value of `n_features` itself.
+        (2, int(np.sqrt(np.iinfo(np.int64).max))),
+    ],
+)
+@pytest.mark.parametrize("interaction_only", [True, False])
+@pytest.mark.parametrize("include_bias", [True, False])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_csr_polynomial_expansion_index_overflow(
+    degree, n_features, interaction_only, include_bias, csr_container
+):
+    """Tests known edge-cases to the dtype promotion strategy and custom
+    Cython code, including a current bug in the upstream
+    `scipy.sparse.hstack`.
+    """
+    data = [1.0]
+    # Use int32 indices as much as we can
+    indices_dtype = np.int32 if n_features - 1 <= np.iinfo(np.int32).max else np.int64
+    row = np.array([0], dtype=indices_dtype)
+    col = np.array([n_features - 1], dtype=indices_dtype)
+
+    # First degree index
+    expected_indices = [
+        n_features - 1 + int(include_bias),
+    ]
+    # Second degree index
+    expected_indices.append(n_features * (n_features + 1) // 2 + expected_indices[0])
+    # Third degree index
+    expected_indices.append(
+        n_features * (n_features + 1) * (n_features + 2) // 6 + expected_indices[1]
+    )
+
+    X = csr_container((data, (row, col)))
+    pf = PolynomialFeatures(
+        interaction_only=interaction_only, include_bias=include_bias, degree=degree
+    )
+
+    # Calculate the number of combinations a-priori, and if needed check for
+    # the correct ValueError and terminate the test early.
+    num_combinations = pf._num_combinations(
+        n_features=n_features,
+        min_degree=0,
+        max_degree=degree,
+        interaction_only=pf.interaction_only,
+        include_bias=pf.include_bias,
+    )
+    if num_combinations > np.iinfo(np.intp).max:
+        msg = (
+            r"The output that would result from the current configuration would have"
+            r" \d* features which is too large to be indexed"
+        )
+        with pytest.raises(ValueError, match=msg):
+            pf.fit(X)
+        return
+
+    # When `n_features>=65535`, `scipy.sparse.hstack` may not use the right
+    # dtype for representing indices and indptr if `n_features` is still
+    # small enough so that each block matrix's indices and indptr arrays
+    # can be represented with `np.int32`. We test `n_features==65535`
+    # since it is guaranteed to run into this bug.
+    if (
+        sp_version < parse_version("1.9.2")
+        and n_features == 65535
+        and degree == 2
+        and not interaction_only
+    ):  # pragma: no cover
+        msg = r"In scipy versions `<1.9.2`, the function `scipy.sparse.hstack`"
+        with pytest.raises(ValueError, match=msg):
+            X_trans = pf.fit_transform(X)
+        return
+    X_trans = pf.fit_transform(X)
+
+    expected_dtype = np.int64 if num_combinations > np.iinfo(np.int32).max else np.int32
+    # Terms higher than first degree
+    non_bias_terms = 1 + (degree - 1) * int(not interaction_only)
+    expected_nnz = int(include_bias) + non_bias_terms
+    assert X_trans.dtype == X.dtype
+    assert X_trans.shape == (1, pf.n_output_features_)
+    assert X_trans.indptr.dtype == X_trans.indices.dtype == expected_dtype
+    assert X_trans.nnz == expected_nnz
+
+    if include_bias:
+        assert X_trans[0, 0] == pytest.approx(1.0)
+    for idx in range(non_bias_terms):
+        assert X_trans[0, expected_indices[idx]] == pytest.approx(1.0)
+
+    offset = interaction_only * n_features
+    if degree == 3:
+        offset *= 1 + n_features
+    assert pf.n_output_features_ == expected_indices[degree - 1] + 1 - offset
+
+
+@pytest.mark.parametrize("interaction_only", [True, False])
+@pytest.mark.parametrize("include_bias", [True, False])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_csr_polynomial_expansion_too_large_to_index(
+    interaction_only, include_bias, csr_container
+):
+    n_features = np.iinfo(np.int64).max // 2
+    data = [1.0]
+    row = [0]
+    col = [n_features - 1]
+    X = csr_container((data, (row, col)))
+    pf = PolynomialFeatures(
+        interaction_only=interaction_only, include_bias=include_bias, degree=(2, 2)
+    )
+    msg = (
+        r"The output that would result from the current configuration would have \d*"
+        r" features which is too large to be indexed"
+    )
+    with pytest.raises(ValueError, match=msg):
+        pf.fit(X)
+    with pytest.raises(ValueError, match=msg):
+        pf.fit_transform(X)
+
+
+@pytest.mark.parametrize("sparse_container", CSR_CONTAINERS + CSC_CONTAINERS)
+def test_polynomial_features_behaviour_on_zero_degree(sparse_container):
+    """Check that PolynomialFeatures raises error when degree=0 and include_bias=False,
+    and output a single constant column when include_bias=True
+    """
+    X = np.ones((10, 2))
+    poly = PolynomialFeatures(degree=0, include_bias=False)
+    err_msg = (
+        "Setting degree to zero and include_bias to False would result in"
+        " an empty output array."
+    )
+    with pytest.raises(ValueError, match=err_msg):
+        poly.fit_transform(X)
+
+    poly = PolynomialFeatures(degree=(0, 0), include_bias=False)
+    err_msg = (
+        "Setting both min_degree and max_degree to zero and include_bias to"
+        " False would result in an empty output array."
+    )
+    with pytest.raises(ValueError, match=err_msg):
+        poly.fit_transform(X)
+
+    for _X in [X, sparse_container(X)]:
+        poly = PolynomialFeatures(degree=0, include_bias=True)
+        output = poly.fit_transform(_X)
+        # convert to dense array if needed
+        if sparse.issparse(output):
+            output = output.toarray()
+        assert_array_equal(output, np.ones((X.shape[0], 1)))
+
+
+def test_sizeof_LARGEST_INT_t():
+    # On Windows, scikit-learn is typically compiled with MSVC that
+    # does not support int128 arithmetic (at the time of writing):
+    # https://stackoverflow.com/a/6761962/163740
+    if sys.platform == "win32" or (
+        sys.maxsize <= 2**32 and sys.platform != "emscripten"
+    ):
+        expected_size = 8
+    else:
+        expected_size = 16
+
+    assert _get_sizeof_LARGEST_INT_t() == expected_size
+
+
+@pytest.mark.xfail(
+    sys.platform == "win32",
+    reason=(
+        "On Windows, scikit-learn is typically compiled with MSVC that does not support"
+        " int128 arithmetic (at the time of writing)"
+    ),
+    run=True,
+)
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_csr_polynomial_expansion_windows_fail(csr_container):
+    # Minimum needed to ensure integer overflow occurs while guaranteeing an
+    # int64-indexable output.
+    n_features = int(np.iinfo(np.int64).max ** (1 / 3) + 3)
+    data = [1.0]
+    row = [0]
+    col = [n_features - 1]
+
+    # First degree index
+    expected_indices = [
+        n_features - 1,
+    ]
+    # Second degree index
+    expected_indices.append(
+        int(n_features * (n_features + 1) // 2 + expected_indices[0])
+    )
+    # Third degree index
+    expected_indices.append(
+        int(n_features * (n_features + 1) * (n_features + 2) // 6 + expected_indices[1])
+    )
+
+    X = csr_container((data, (row, col)))
+    pf = PolynomialFeatures(interaction_only=False, include_bias=False, degree=3)
+    if sys.maxsize <= 2**32:
+        msg = (
+            r"The output that would result from the current configuration would"
+            r" have \d*"
+            r" features which is too large to be indexed"
+        )
+        with pytest.raises(ValueError, match=msg):
+            pf.fit_transform(X)
+    else:
+        X_trans = pf.fit_transform(X)
+        for idx in range(3):
+            assert X_trans[0, expected_indices[idx]] == pytest.approx(1.0)
diff --git a/sklearn/preprocessing/tests/test_target_encoder.py b/sklearn/preprocessing/tests/test_target_encoder.py
new file mode 100644
index 0000000000000..536f2e031bf77
--- /dev/null
+++ b/sklearn/preprocessing/tests/test_target_encoder.py
@@ -0,0 +1,714 @@
+import re
+
+import numpy as np
+import pytest
+from numpy.testing import assert_allclose, assert_array_equal
+
+from sklearn.ensemble import RandomForestRegressor
+from sklearn.linear_model import Ridge
+from sklearn.model_selection import (
+    KFold,
+    ShuffleSplit,
+    StratifiedKFold,
+    cross_val_score,
+    train_test_split,
+)
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import (
+    KBinsDiscretizer,
+    LabelBinarizer,
+    LabelEncoder,
+    TargetEncoder,
+)
+
+
+def _encode_target(X_ordinal, y_numeric, n_categories, smooth):
+    """Simple Python implementation of target encoding."""
+    cur_encodings = np.zeros(n_categories, dtype=np.float64)
+    y_mean = np.mean(y_numeric)
+
+    if smooth == "auto":
+        y_variance = np.var(y_numeric)
+        for c in range(n_categories):
+            y_subset = y_numeric[X_ordinal == c]
+            n_i = y_subset.shape[0]
+
+            if n_i == 0:
+                cur_encodings[c] = y_mean
+                continue
+
+            y_subset_variance = np.var(y_subset)
+            m = y_subset_variance / y_variance
+            lambda_ = n_i / (n_i + m)
+
+            cur_encodings[c] = lambda_ * np.mean(y_subset) + (1 - lambda_) * y_mean
+        return cur_encodings
+    else:  # float
+        for c in range(n_categories):
+            y_subset = y_numeric[X_ordinal == c]
+            current_sum = np.sum(y_subset) + y_mean * smooth
+            current_cnt = y_subset.shape[0] + smooth
+            cur_encodings[c] = current_sum / current_cnt
+        return cur_encodings
+
+
+@pytest.mark.parametrize(
+    "categories, unknown_value",
+    [
+        ([np.array([0, 1, 2], dtype=np.int64)], 4),
+        ([np.array([1.0, 3.0, np.nan], dtype=np.float64)], 6.0),
+        ([np.array(["cat", "dog", "snake"], dtype=object)], "bear"),
+        ("auto", 3),
+    ],
+)
+@pytest.mark.parametrize("smooth", [5.0, "auto"])
+@pytest.mark.parametrize("target_type", ["binary", "continuous"])
+def test_encoding(categories, unknown_value, global_random_seed, smooth, target_type):
+    """Check encoding for binary and continuous targets.
+
+    Compare the values returned by `TargetEncoder.fit_transform` against the
+    expected encodings for cv splits from a naive reference Python
+    implementation in _encode_target.
+    """
+
+    n_categories = 3
+    X_train_int_array = np.array([[0] * 20 + [1] * 30 + [2] * 40], dtype=np.int64).T
+    X_test_int_array = np.array([[0, 1, 2]], dtype=np.int64).T
+    n_samples = X_train_int_array.shape[0]
+
+    if categories == "auto":
+        X_train = X_train_int_array
+        X_test = X_test_int_array
+    else:
+        X_train = categories[0][X_train_int_array]
+        X_test = categories[0][X_test_int_array]
+
+    X_test = np.concatenate((X_test, [[unknown_value]]))
+
+    data_rng = np.random.RandomState(global_random_seed)
+    n_splits = 3
+    if target_type == "binary":
+        y_numeric = data_rng.randint(low=0, high=2, size=n_samples)
+        target_names = np.array(["cat", "dog"], dtype=object)
+        y_train = target_names[y_numeric]
+
+    else:
+        assert target_type == "continuous"
+        y_numeric = data_rng.uniform(low=-10, high=20, size=n_samples)
+        y_train = y_numeric
+
+    shuffled_idx = data_rng.permutation(n_samples)
+    X_train_int_array = X_train_int_array[shuffled_idx]
+    X_train = X_train[shuffled_idx]
+    y_train = y_train[shuffled_idx]
+    y_numeric = y_numeric[shuffled_idx]
+
+    # Define our CV splitting strategy
+    if target_type == "binary":
+        cv = StratifiedKFold(
+            n_splits=n_splits, random_state=global_random_seed, shuffle=True
+        )
+    else:
+        cv = KFold(n_splits=n_splits, random_state=global_random_seed, shuffle=True)
+
+    # Compute the expected values using our reference Python implementation of
+    # target encoding:
+    expected_X_fit_transform = np.empty_like(X_train_int_array, dtype=np.float64)
+
+    for train_idx, test_idx in cv.split(X_train_int_array, y_train):
+        X_, y_ = X_train_int_array[train_idx, 0], y_numeric[train_idx]
+        cur_encodings = _encode_target(X_, y_, n_categories, smooth)
+        expected_X_fit_transform[test_idx, 0] = cur_encodings[
+            X_train_int_array[test_idx, 0]
+        ]
+
+    # Check that we can obtain the same encodings by calling `fit_transform` on
+    # the estimator with the same CV parameters:
+    target_encoder = TargetEncoder(
+        smooth=smooth,
+        categories=categories,
+        cv=n_splits,
+        random_state=global_random_seed,
+    )
+
+    X_fit_transform = target_encoder.fit_transform(X_train, y_train)
+
+    assert target_encoder.target_type_ == target_type
+    assert_allclose(X_fit_transform, expected_X_fit_transform)
+    assert len(target_encoder.encodings_) == 1
+    if target_type == "binary":
+        assert_array_equal(target_encoder.classes_, target_names)
+    else:
+        assert target_encoder.classes_ is None
+
+    # compute encodings for all data to validate `transform`
+    y_mean = np.mean(y_numeric)
+    expected_encodings = _encode_target(
+        X_train_int_array[:, 0], y_numeric, n_categories, smooth
+    )
+    assert_allclose(target_encoder.encodings_[0], expected_encodings)
+    assert target_encoder.target_mean_ == pytest.approx(y_mean)
+
+    # Transform on test data, the last value is unknown so it is encoded as the target
+    # mean
+    expected_X_test_transform = np.concatenate(
+        (expected_encodings, np.array([y_mean]))
+    ).reshape(-1, 1)
+
+    X_test_transform = target_encoder.transform(X_test)
+    assert_allclose(X_test_transform, expected_X_test_transform)
+
+
+@pytest.mark.parametrize(
+    "categories, unknown_values",
+    [
+        ([np.array([0, 1, 2], dtype=np.int64)], "auto"),
+        ([np.array(["cat", "dog", "snake"], dtype=object)], ["bear", "rabbit"]),
+    ],
+)
+@pytest.mark.parametrize(
+    "target_labels", [np.array([1, 2, 3]), np.array(["a", "b", "c"])]
+)
+@pytest.mark.parametrize("smooth", [5.0, "auto"])
+def test_encoding_multiclass(
+    global_random_seed, categories, unknown_values, target_labels, smooth
+):
+    """Check encoding for multiclass targets."""
+    rng = np.random.RandomState(global_random_seed)
+
+    n_samples = 80
+    n_features = 2
+    feat_1_int = np.array(rng.randint(low=0, high=2, size=n_samples))
+    feat_2_int = np.array(rng.randint(low=0, high=3, size=n_samples))
+    feat_1 = categories[0][feat_1_int]
+    feat_2 = categories[0][feat_2_int]
+    X_train = np.column_stack((feat_1, feat_2))
+    X_train_int = np.column_stack((feat_1_int, feat_2_int))
+    categories_ = [[0, 1], [0, 1, 2]]
+
+    n_classes = 3
+    y_train_int = np.array(rng.randint(low=0, high=n_classes, size=n_samples))
+    y_train = target_labels[y_train_int]
+    y_train_enc = LabelBinarizer().fit_transform(y_train)
+
+    n_splits = 3
+    cv = StratifiedKFold(
+        n_splits=n_splits, random_state=global_random_seed, shuffle=True
+    )
+
+    # Manually compute encodings for cv splits to validate `fit_transform`
+    expected_X_fit_transform = np.empty(
+        (X_train_int.shape[0], X_train_int.shape[1] * n_classes),
+        dtype=np.float64,
+    )
+    for f_idx, cats in enumerate(categories_):
+        for c_idx in range(n_classes):
+            for train_idx, test_idx in cv.split(X_train, y_train):
+                y_class = y_train_enc[:, c_idx]
+                X_, y_ = X_train_int[train_idx, f_idx], y_class[train_idx]
+                current_encoding = _encode_target(X_, y_, len(cats), smooth)
+                # f_idx:   0, 0, 0, 1, 1, 1
+                # c_idx:   0, 1, 2, 0, 1, 2
+                # exp_idx: 0, 1, 2, 3, 4, 5
+                exp_idx = c_idx + (f_idx * n_classes)
+                expected_X_fit_transform[test_idx, exp_idx] = current_encoding[
+                    X_train_int[test_idx, f_idx]
+                ]
+
+    target_encoder = TargetEncoder(
+        smooth=smooth,
+        cv=n_splits,
+        random_state=global_random_seed,
+    )
+    X_fit_transform = target_encoder.fit_transform(X_train, y_train)
+
+    assert target_encoder.target_type_ == "multiclass"
+    assert_allclose(X_fit_transform, expected_X_fit_transform)
+
+    # Manually compute encoding to validate `transform`
+    expected_encodings = []
+    for f_idx, cats in enumerate(categories_):
+        for c_idx in range(n_classes):
+            y_class = y_train_enc[:, c_idx]
+            current_encoding = _encode_target(
+                X_train_int[:, f_idx], y_class, len(cats), smooth
+            )
+            expected_encodings.append(current_encoding)
+
+    assert len(target_encoder.encodings_) == n_features * n_classes
+    for i in range(n_features * n_classes):
+        assert_allclose(target_encoder.encodings_[i], expected_encodings[i])
+    assert_array_equal(target_encoder.classes_, target_labels)
+
+    # Include unknown values at the end
+    X_test_int = np.array([[0, 1], [1, 2], [4, 5]])
+    if unknown_values == "auto":
+        X_test = X_test_int
+    else:
+        X_test = np.empty_like(X_test_int[:-1, :], dtype=object)
+        for column_idx in range(X_test_int.shape[1]):
+            X_test[:, column_idx] = categories[0][X_test_int[:-1, column_idx]]
+        # Add unknown values at end
+        X_test = np.vstack((X_test, unknown_values))
+
+    y_mean = np.mean(y_train_enc, axis=0)
+    expected_X_test_transform = np.empty(
+        (X_test_int.shape[0], X_test_int.shape[1] * n_classes),
+        dtype=np.float64,
+    )
+    n_rows = X_test_int.shape[0]
+    f_idx = [0, 0, 0, 1, 1, 1]
+    # Last row are unknowns, dealt with later
+    for row_idx in range(n_rows - 1):
+        for i, enc in enumerate(expected_encodings):
+            expected_X_test_transform[row_idx, i] = enc[X_test_int[row_idx, f_idx[i]]]
+
+    # Unknowns encoded as target mean for each class
+    # `y_mean` contains target mean for each class, thus cycle through mean of
+    # each class, `n_features` times
+    mean_idx = [0, 1, 2, 0, 1, 2]
+    for i in range(n_classes * n_features):
+        expected_X_test_transform[n_rows - 1, i] = y_mean[mean_idx[i]]
+
+    X_test_transform = target_encoder.transform(X_test)
+    assert_allclose(X_test_transform, expected_X_test_transform)
+
+
+@pytest.mark.parametrize(
+    "X, categories",
+    [
+        (
+            np.array([[0] * 10 + [1] * 10 + [3]], dtype=np.int64).T,  # 3 is unknown
+            [[0, 1, 2]],
+        ),
+        (
+            np.array(
+                [["cat"] * 10 + ["dog"] * 10 + ["snake"]], dtype=object
+            ).T,  # snake is unknown
+            [["dog", "cat", "cow"]],
+        ),
+    ],
+)
+@pytest.mark.parametrize("smooth", [4.0, "auto"])
+def test_custom_categories(X, categories, smooth):
+    """Custom categories with unknown categories that are not in training data."""
+    rng = np.random.RandomState(0)
+    y = rng.uniform(low=-10, high=20, size=X.shape[0])
+    enc = TargetEncoder(categories=categories, smooth=smooth, random_state=0).fit(X, y)
+
+    # The last element is unknown and encoded as the mean
+    y_mean = y.mean()
+    X_trans = enc.transform(X[-1:])
+    assert X_trans[0, 0] == pytest.approx(y_mean)
+
+    assert len(enc.encodings_) == 1
+    # custom category that is not in training data
+    assert enc.encodings_[0][-1] == pytest.approx(y_mean)
+
+
+@pytest.mark.parametrize(
+    "y, msg",
+    [
+        ([1, 2, 0, 1], "Found input variables with inconsistent"),
+        (
+            np.array([[1, 2, 0], [1, 2, 3]]).T,
+            "Target type was inferred to be 'multiclass-multioutput'",
+        ),
+    ],
+)
+def test_errors(y, msg):
+    """Check invalidate input."""
+    X = np.array([[1, 0, 1]]).T
+
+    enc = TargetEncoder()
+    with pytest.raises(ValueError, match=msg):
+        enc.fit_transform(X, y)
+
+
+def test_use_regression_target():
+    """Check inferred and specified `target_type` on regression target."""
+    X = np.array([[0, 1, 0, 1, 0, 1]]).T
+    y = np.array([1.0, 2.0, 3.0, 2.0, 3.0, 4.0])
+
+    enc = TargetEncoder(cv=2)
+    with pytest.warns(
+        UserWarning,
+        match=re.escape(
+            "The least populated class in y has only 1 members, which is less than"
+            " n_splits=2."
+        ),
+    ):
+        enc.fit_transform(X, y)
+    assert enc.target_type_ == "multiclass"
+
+    enc = TargetEncoder(cv=2, target_type="continuous")
+    enc.fit_transform(X, y)
+    assert enc.target_type_ == "continuous"
+
+
+@pytest.mark.parametrize(
+    "y, feature_names",
+    [
+        ([1, 2] * 10, ["A", "B"]),
+        ([1, 2, 3] * 6 + [1, 2], ["A_1", "A_2", "A_3", "B_1", "B_2", "B_3"]),
+        (
+            ["y1", "y2", "y3"] * 6 + ["y1", "y2"],
+            ["A_y1", "A_y2", "A_y3", "B_y1", "B_y2", "B_y3"],
+        ),
+    ],
+)
+def test_feature_names_out_set_output(y, feature_names):
+    """Check TargetEncoder works with set_output."""
+    pd = pytest.importorskip("pandas")
+
+    X_df = pd.DataFrame({"A": ["a", "b"] * 10, "B": [1, 2] * 10})
+
+    enc_default = TargetEncoder(cv=2, smooth=3.0, random_state=0)
+    enc_default.set_output(transform="default")
+    enc_pandas = TargetEncoder(cv=2, smooth=3.0, random_state=0)
+    enc_pandas.set_output(transform="pandas")
+
+    X_default = enc_default.fit_transform(X_df, y)
+    X_pandas = enc_pandas.fit_transform(X_df, y)
+
+    assert_allclose(X_pandas.to_numpy(), X_default)
+    assert_array_equal(enc_pandas.get_feature_names_out(), feature_names)
+    assert_array_equal(enc_pandas.get_feature_names_out(), X_pandas.columns)
+
+
+@pytest.mark.parametrize("to_pandas", [True, False])
+@pytest.mark.parametrize("smooth", [1.0, "auto"])
+@pytest.mark.parametrize("target_type", ["binary-ints", "binary-str", "continuous"])
+def test_multiple_features_quick(to_pandas, smooth, target_type):
+    """Check target encoder with multiple features."""
+    X_ordinal = np.array(
+        [[1, 1], [0, 1], [1, 1], [2, 1], [1, 0], [0, 1], [1, 0], [0, 0]], dtype=np.int64
+    )
+    if target_type == "binary-str":
+        y_train = np.array(["a", "b", "a", "a", "b", "b", "a", "b"])
+        y_integer = LabelEncoder().fit_transform(y_train)
+        cv = StratifiedKFold(2, random_state=0, shuffle=True)
+    elif target_type == "binary-ints":
+        y_train = np.array([3, 4, 3, 3, 3, 4, 4, 4])
+        y_integer = LabelEncoder().fit_transform(y_train)
+        cv = StratifiedKFold(2, random_state=0, shuffle=True)
+    else:
+        y_train = np.array([3.0, 5.1, 2.4, 3.5, 4.1, 5.5, 10.3, 7.3], dtype=np.float32)
+        y_integer = y_train
+        cv = KFold(2, random_state=0, shuffle=True)
+    y_mean = np.mean(y_integer)
+    categories = [[0, 1, 2], [0, 1]]
+
+    X_test = np.array(
+        [
+            [0, 1],
+            [3, 0],  # 3 is unknown
+            [1, 10],  # 10 is unknown
+        ],
+        dtype=np.int64,
+    )
+
+    if to_pandas:
+        pd = pytest.importorskip("pandas")
+        # convert second feature to an object
+        X_train = pd.DataFrame(
+            {
+                "feat0": X_ordinal[:, 0],
+                "feat1": np.array(["cat", "dog"], dtype=object)[X_ordinal[:, 1]],
+            }
+        )
+        # "snake" is unknown
+        X_test = pd.DataFrame({"feat0": X_test[:, 0], "feat1": ["dog", "cat", "snake"]})
+    else:
+        X_train = X_ordinal
+
+    # manually compute encoding for fit_transform
+    expected_X_fit_transform = np.empty_like(X_ordinal, dtype=np.float64)
+    for f_idx, cats in enumerate(categories):
+        for train_idx, test_idx in cv.split(X_ordinal, y_integer):
+            X_, y_ = X_ordinal[train_idx, f_idx], y_integer[train_idx]
+            current_encoding = _encode_target(X_, y_, len(cats), smooth)
+            expected_X_fit_transform[test_idx, f_idx] = current_encoding[
+                X_ordinal[test_idx, f_idx]
+            ]
+
+    # manually compute encoding for transform
+    expected_encodings = []
+    for f_idx, cats in enumerate(categories):
+        current_encoding = _encode_target(
+            X_ordinal[:, f_idx], y_integer, len(cats), smooth
+        )
+        expected_encodings.append(current_encoding)
+
+    expected_X_test_transform = np.array(
+        [
+            [expected_encodings[0][0], expected_encodings[1][1]],
+            [y_mean, expected_encodings[1][0]],
+            [expected_encodings[0][1], y_mean],
+        ],
+        dtype=np.float64,
+    )
+
+    enc = TargetEncoder(smooth=smooth, cv=2, random_state=0)
+    X_fit_transform = enc.fit_transform(X_train, y_train)
+    assert_allclose(X_fit_transform, expected_X_fit_transform)
+
+    assert len(enc.encodings_) == 2
+    for i in range(2):
+        assert_allclose(enc.encodings_[i], expected_encodings[i])
+
+    X_test_transform = enc.transform(X_test)
+    assert_allclose(X_test_transform, expected_X_test_transform)
+
+
+@pytest.mark.parametrize(
+    "y, y_mean",
+    [
+        (np.array([3.4] * 20), 3.4),
+        (np.array([0] * 20), 0),
+        (np.array(["a"] * 20, dtype=object), 0),
+    ],
+    ids=["continuous", "binary", "binary-string"],
+)
+@pytest.mark.parametrize("smooth", ["auto", 4.0, 0.0])
+def test_constant_target_and_feature(y, y_mean, smooth):
+    """Check edge case where feature and target is constant."""
+    X = np.array([[1] * 20]).T
+    n_samples = X.shape[0]
+
+    enc = TargetEncoder(cv=2, smooth=smooth, random_state=0)
+    X_trans = enc.fit_transform(X, y)
+    assert_allclose(X_trans, np.repeat([[y_mean]], n_samples, axis=0))
+    assert enc.encodings_[0][0] == pytest.approx(y_mean)
+    assert enc.target_mean_ == pytest.approx(y_mean)
+
+    X_test = np.array([[1], [0]])
+    X_test_trans = enc.transform(X_test)
+    assert_allclose(X_test_trans, np.repeat([[y_mean]], 2, axis=0))
+
+
+def test_fit_transform_not_associated_with_y_if_ordinal_categorical_is_not(
+    global_random_seed,
+):
+    cardinality = 30  # not too large, otherwise we need a very large n_samples
+    n_samples = 3000
+    rng = np.random.RandomState(global_random_seed)
+    y_train = rng.normal(size=n_samples)
+    X_train = rng.randint(0, cardinality, size=n_samples).reshape(-1, 1)
+
+    # Sort by y_train to attempt to cause a leak
+    y_sorted_indices = y_train.argsort()
+    y_train = y_train[y_sorted_indices]
+    X_train = X_train[y_sorted_indices]
+
+    target_encoder = TargetEncoder(shuffle=True, random_state=global_random_seed)
+    X_encoded_train_shuffled = target_encoder.fit_transform(X_train, y_train)
+
+    target_encoder = TargetEncoder(shuffle=False)
+    X_encoded_train_no_shuffled = target_encoder.fit_transform(X_train, y_train)
+
+    # Check that no information about y_train has leaked into X_train:
+    regressor = RandomForestRegressor(
+        n_estimators=10, min_samples_leaf=20, random_state=global_random_seed
+    )
+
+    # It's impossible to learn a good predictive model on the training set when
+    # using the original representation X_train or the target encoded
+    # representation with shuffled inner CV. For the latter, no information
+    # about y_train has inadvertently leaked into the prior used to generate
+    # `X_encoded_train_shuffled`:
+    cv = ShuffleSplit(n_splits=50, random_state=global_random_seed)
+    assert cross_val_score(regressor, X_train, y_train, cv=cv).mean() < 0.1
+    assert (
+        cross_val_score(regressor, X_encoded_train_shuffled, y_train, cv=cv).mean()
+        < 0.1
+    )
+
+    # Without the inner CV shuffling, a lot of information about y_train goes into the
+    # the per-fold y_train.mean() priors: shrinkage is no longer effective in this
+    # case and would no longer be able to prevent downstream over-fitting.
+    assert (
+        cross_val_score(regressor, X_encoded_train_no_shuffled, y_train, cv=cv).mean()
+        > 0.5
+    )
+
+
+def test_smooth_zero():
+    """Check edge case with zero smoothing and cv does not contain category."""
+    X = np.array([[0, 0, 0, 0, 0, 1, 1, 1, 1, 1]]).T
+    y = np.array([2.1, 4.3, 1.2, 3.1, 1.0, 9.0, 10.3, 14.2, 13.3, 15.0])
+
+    enc = TargetEncoder(smooth=0.0, shuffle=False, cv=2)
+    X_trans = enc.fit_transform(X, y)
+
+    # With cv = 2, category 0 does not exist in the second half, thus
+    # it will be encoded as the mean of the second half
+    assert_allclose(X_trans[0], np.mean(y[5:]))
+
+    # category 1 does not exist in the first half, thus it will be encoded as
+    # the mean of the first half
+    assert_allclose(X_trans[-1], np.mean(y[:5]))
+
+
+@pytest.mark.parametrize("smooth", [0.0, 1e3, "auto"])
+def test_invariance_of_encoding_under_label_permutation(smooth, global_random_seed):
+    # Check that the encoding does not depend on the integer of the value of
+    # the integer labels. This is quite a trivial property but it is helpful
+    # to understand the following test.
+    rng = np.random.RandomState(global_random_seed)
+
+    # Random y and informative categorical X to make the test non-trivial when
+    # using smoothing.
+    y = rng.normal(size=1000)
+    n_categories = 30
+    X = KBinsDiscretizer(
+        n_bins=n_categories, quantile_method="averaged_inverted_cdf", encode="ordinal"
+    ).fit_transform(y.reshape(-1, 1))
+
+    X_train, X_test, y_train, y_test = train_test_split(
+        X, y, random_state=global_random_seed
+    )
+
+    # Shuffle the labels to make sure that the encoding is invariant to the
+    # permutation of the labels
+    permutated_labels = rng.permutation(n_categories)
+    X_train_permuted = permutated_labels[X_train.astype(np.int32)]
+    X_test_permuted = permutated_labels[X_test.astype(np.int32)]
+
+    target_encoder = TargetEncoder(smooth=smooth, random_state=global_random_seed)
+    X_train_encoded = target_encoder.fit_transform(X_train, y_train)
+    X_test_encoded = target_encoder.transform(X_test)
+
+    X_train_permuted_encoded = target_encoder.fit_transform(X_train_permuted, y_train)
+    X_test_permuted_encoded = target_encoder.transform(X_test_permuted)
+
+    assert_allclose(X_train_encoded, X_train_permuted_encoded)
+    assert_allclose(X_test_encoded, X_test_permuted_encoded)
+
+
+@pytest.mark.parametrize("smooth", [0.0, "auto"])
+def test_target_encoding_for_linear_regression(smooth, global_random_seed):
+    # Check some expected statistical properties when fitting a linear
+    # regression model on target encoded features depending on their relation
+    # with that target.
+
+    # In this test, we use the Ridge class with the "lsqr" solver and a little
+    # bit of regularization to implement a linear regression model that
+    # converges quickly for large `n_samples` and robustly in case of
+    # correlated features. Since we will fit this model on a mean centered
+    # target, we do not need to fit an intercept and this will help simplify
+    # the analysis with respect to the expected coefficients.
+    linear_regression = Ridge(alpha=1e-6, solver="lsqr", fit_intercept=False)
+
+    # Construct a random target variable. We need a large number of samples for
+    # this test to be stable across all values of the random seed.
+    n_samples = 50_000
+    rng = np.random.RandomState(global_random_seed)
+    y = rng.randn(n_samples)
+
+    # Generate a single informative ordinal feature with medium cardinality.
+    # Inject some irreducible noise to make it harder for a multivariate model
+    # to identify the informative feature from other pure noise features.
+    noise = 0.8 * rng.randn(n_samples)
+    n_categories = 100
+    X_informative = KBinsDiscretizer(
+        n_bins=n_categories,
+        encode="ordinal",
+        strategy="uniform",
+        random_state=rng,
+    ).fit_transform((y + noise).reshape(-1, 1))
+
+    # Let's permute the labels to hide the fact that this feature is
+    # informative to naive linear regression model trained on the raw ordinal
+    # values. As highlighted in the previous test, the target encoding should be
+    # invariant to such a permutation.
+    permutated_labels = rng.permutation(n_categories)
+    X_informative = permutated_labels[X_informative.astype(np.int32)]
+
+    # Generate a shuffled copy of the informative feature to destroy the
+    # relationship with the target.
+    X_shuffled = rng.permutation(X_informative)
+
+    # Also include a very high cardinality categorical feature that is by
+    # itself independent of the target variable: target encoding such a feature
+    # without internal cross-validation should cause catastrophic overfitting
+    # for the downstream regressor, even with shrinkage. This kind of features
+    # typically represents near unique identifiers of samples. In general they
+    # should be removed from a machine learning datasets but here we want to
+    # study the ability of the default behavior of TargetEncoder to mitigate
+    # them automatically.
+    X_near_unique_categories = rng.choice(
+        int(0.9 * n_samples), size=n_samples, replace=True
+    ).reshape(-1, 1)
+
+    # Assemble the dataset and do a train-test split:
+    X = np.concatenate(
+        [X_informative, X_shuffled, X_near_unique_categories],
+        axis=1,
+    )
+    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
+
+    # Let's first check that a linear regression model trained on the raw
+    # features underfits because of the meaning-less ordinal encoding of the
+    # labels.
+    raw_model = linear_regression.fit(X_train, y_train)
+    assert raw_model.score(X_train, y_train) < 0.1
+    assert raw_model.score(X_test, y_test) < 0.1
+
+    # Now do the same with target encoding using the internal CV mechanism
+    # implemented when using fit_transform.
+    model_with_cv = make_pipeline(
+        TargetEncoder(smooth=smooth, random_state=rng), linear_regression
+    ).fit(X_train, y_train)
+
+    # This model should be able to fit the data well and also generalise to the
+    # test data (assuming that the binning is fine-grained enough). The R2
+    # scores are not perfect because of the noise injected during the
+    # generation of the unique informative feature.
+    coef = model_with_cv[-1].coef_
+    assert model_with_cv.score(X_train, y_train) > 0.5, coef
+    assert model_with_cv.score(X_test, y_test) > 0.5, coef
+
+    # The target encoder recovers the linear relationship with slope 1 between
+    # the target encoded unique informative predictor and the target. Since the
+    # target encoding of the 2 other features is not informative thanks to the
+    # use of internal cross-validation, the multivariate linear regressor
+    # assigns a coef of 1 to the first feature and 0 to the other 2.
+    assert coef[0] == pytest.approx(1, abs=1e-2)
+    assert (np.abs(coef[1:]) < 0.2).all()
+
+    # Let's now disable the internal cross-validation by calling fit and then
+    # transform separately on the training set:
+    target_encoder = TargetEncoder(smooth=smooth, random_state=rng).fit(
+        X_train, y_train
+    )
+    X_enc_no_cv_train = target_encoder.transform(X_train)
+    X_enc_no_cv_test = target_encoder.transform(X_test)
+    model_no_cv = linear_regression.fit(X_enc_no_cv_train, y_train)
+
+    # The linear regression model should always overfit because it assigns
+    # too much weight to the extremely high cardinality feature relatively to
+    # the informative feature. Note that this is the case even when using
+    # the empirical Bayes smoothing which is not enough to prevent such
+    # overfitting alone.
+    coef = model_no_cv.coef_
+    assert model_no_cv.score(X_enc_no_cv_train, y_train) > 0.7, coef
+    assert model_no_cv.score(X_enc_no_cv_test, y_test) < 0.5, coef
+
+    # The model overfits because it assigns too much weight to the high
+    # cardinality yet non-informative feature instead of the lower
+    # cardinality yet informative feature:
+    assert abs(coef[0]) < abs(coef[2])
+
+
+def test_pandas_copy_on_write():
+    """
+    Test target-encoder cython code when y is read-only.
+
+    The numpy array underlying df["y"] is read-only when copy-on-write is enabled.
+    Non-regression test for gh-27879.
+    """
+    pd = pytest.importorskip("pandas", minversion="2.0")
+    with pd.option_context("mode.copy_on_write", True):
+        df = pd.DataFrame({"x": ["a", "b", "b"], "y": [4.0, 5.0, 6.0]})
+        TargetEncoder(target_type="continuous").fit(df[["x"]], df["y"])
diff --git a/sklearn/random_projection.py b/sklearn/random_projection.py
index 8e968088e8141..f98b11365dd3b 100644
--- a/sklearn/random_projection.py
+++ b/sklearn/random_projection.py
@@ -1,12 +1,11 @@
-# -*- coding: utf8
-"""Random Projection transformers.
+"""Random projection transformers.
 
-Random Projections are a simple and computationally efficient way to
+Random projections are a simple and computationally efficient way to
 reduce the dimensionality of the data by trading a controlled amount
 of accuracy (as additional variance) for faster processing times and
 smaller model sizes.
 
-The dimensions and distribution of Random Projections matrices are
+The dimensions and distribution of random projections matrices are
 controlled so as to preserve the pairwise distances between any two
 samples of the dataset.
 
@@ -21,42 +20,56 @@
   much lower dimension in such a way that distances between the points are
   nearly preserved. The map used for the embedding is at least Lipschitz,
   and can even be taken to be an orthogonal projection.
-
 """
-# Authors: Olivier Grisel ,
-#          Arnaud Joly 
-# License: BSD 3 clause
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import warnings
 from abc import ABCMeta, abstractmethod
+from numbers import Integral, Real
 
 import numpy as np
 import scipy.sparse as sp
-
-from .base import BaseEstimator, TransformerMixin
-
+from scipy import linalg
+
+from .base import (
+    BaseEstimator,
+    ClassNamePrefixFeaturesOutMixin,
+    TransformerMixin,
+    _fit_context,
+)
+from .exceptions import DataDimensionalityWarning
 from .utils import check_random_state
+from .utils._param_validation import Interval, StrOptions, validate_params
 from .utils.extmath import safe_sparse_dot
 from .utils.random import sample_without_replacement
-from .utils.validation import check_is_fitted
-from .utils.validation import _deprecate_positional_args
-from .exceptions import DataDimensionalityWarning
-
-
-__all__ = ["SparseRandomProjection",
-           "GaussianRandomProjection",
-           "johnson_lindenstrauss_min_dim"]
-
-
-@_deprecate_positional_args
+from .utils.validation import check_array, check_is_fitted, validate_data
+
+__all__ = [
+    "GaussianRandomProjection",
+    "SparseRandomProjection",
+    "johnson_lindenstrauss_min_dim",
+]
+
+
+@validate_params(
+    {
+        "n_samples": ["array-like", Interval(Real, 1, None, closed="left")],
+        "eps": ["array-like", Interval(Real, 0, 1, closed="neither")],
+    },
+    prefer_skip_nested_validation=True,
+)
 def johnson_lindenstrauss_min_dim(n_samples, *, eps=0.1):
     """Find a 'safe' number of components to randomly project to.
 
     The distortion introduced by a random projection `p` only changes the
-    distance between two points by a factor (1 +- eps) in an euclidean space
+    distance between two points by a factor (1 +- eps) in a euclidean space
     with good probability. The projection `p` is an eps-embedding as defined
     by:
 
+    .. code-block:: text
+
       (1 - eps) ||u - v||^2 < ||p(u) - p(v)||^2 < (1 + eps) ||u - v||^2
 
     Where u and v are any rows taken from a dataset of shape (n_samples,
@@ -67,6 +80,8 @@ def johnson_lindenstrauss_min_dim(n_samples, *, eps=0.1):
     The minimum number of components to guarantee the eps-embedding is
     given by:
 
+    .. code-block:: text
+
       n_components >= 4 log(n_samples) / (eps^2 / 2 - eps^3 / 3)
 
     Note that the number of dimensions is independent of the original
@@ -79,12 +94,12 @@ def johnson_lindenstrauss_min_dim(n_samples, *, eps=0.1):
     Parameters
     ----------
     n_samples : int or array-like of int
-        Number of samples that should be a integer greater than 0. If an array
+        Number of samples that should be an integer greater than 0. If an array
         is given, it will compute a safe number of components array-wise.
 
-    eps : float or ndarray of shape (n_components,), dtype=float, \
+    eps : float or array-like of shape (n_components,), dtype=float, \
             default=0.1
-        Maximum distortion rate in the range (0,1 ) as defined by the
+        Maximum distortion rate in the range (0, 1) as defined by the
         Johnson-Lindenstrauss lemma. If an array is given, it will compute a
         safe number of components array-wise.
 
@@ -94,63 +109,61 @@ def johnson_lindenstrauss_min_dim(n_samples, *, eps=0.1):
         The minimal number of components to guarantee with good probability
         an eps-embedding with n_samples.
 
+    References
+    ----------
+
+    .. [1] https://en.wikipedia.org/wiki/Johnson%E2%80%93Lindenstrauss_lemma
+
+    .. [2] `Sanjoy Dasgupta and Anupam Gupta, 1999,
+           "An elementary proof of the Johnson-Lindenstrauss Lemma."
+           `_
+
     Examples
     --------
-
+    >>> from sklearn.random_projection import johnson_lindenstrauss_min_dim
     >>> johnson_lindenstrauss_min_dim(1e6, eps=0.5)
-    663
+    np.int64(663)
 
     >>> johnson_lindenstrauss_min_dim(1e6, eps=[0.5, 0.1, 0.01])
     array([    663,   11841, 1112658])
 
     >>> johnson_lindenstrauss_min_dim([1e4, 1e5, 1e6], eps=0.1)
     array([ 7894,  9868, 11841])
-
-    References
-    ----------
-
-    .. [1] https://en.wikipedia.org/wiki/Johnson%E2%80%93Lindenstrauss_lemma
-
-    .. [2] Sanjoy Dasgupta and Anupam Gupta, 1999,
-           "An elementary proof of the Johnson-Lindenstrauss Lemma."
-           http://citeseer.ist.psu.edu/viewdoc/summary?doi=10.1.1.45.3654
-
     """
     eps = np.asarray(eps)
     n_samples = np.asarray(n_samples)
 
     if np.any(eps <= 0.0) or np.any(eps >= 1):
-        raise ValueError(
-            "The JL bound is defined for eps in ]0, 1[, got %r" % eps)
+        raise ValueError("The JL bound is defined for eps in ]0, 1[, got %r" % eps)
 
-    if np.any(n_samples) <= 0:
+    if np.any(n_samples <= 0):
         raise ValueError(
             "The JL bound is defined for n_samples greater than zero, got %r"
-            % n_samples)
+            % n_samples
+        )
 
-    denominator = (eps ** 2 / 2) - (eps ** 3 / 3)
+    denominator = (eps**2 / 2) - (eps**3 / 3)
     return (4 * np.log(n_samples) / denominator).astype(np.int64)
 
 
 def _check_density(density, n_features):
     """Factorize density check according to Li et al."""
-    if density == 'auto':
+    if density == "auto":
         density = 1 / np.sqrt(n_features)
 
     elif density <= 0 or density > 1:
-        raise ValueError("Expected density in range ]0, 1], got: %r"
-                         % density)
+        raise ValueError("Expected density in range ]0, 1], got: %r" % density)
     return density
 
 
 def _check_input_size(n_components, n_features):
     """Factorize argument checking for random matrix generation."""
     if n_components <= 0:
-        raise ValueError("n_components must be strictly positive, got %d" %
-                         n_components)
+        raise ValueError(
+            "n_components must be strictly positive, got %d" % n_components
+        )
     if n_features <= 0:
-        raise ValueError("n_features must be strictly positive, got %d" %
-                         n_features)
+        raise ValueError("n_features must be strictly positive, got %d" % n_features)
 
 
 def _gaussian_random_matrix(n_components, n_features, random_state=None):
@@ -187,14 +200,13 @@ def _gaussian_random_matrix(n_components, n_features, random_state=None):
     """
     _check_input_size(n_components, n_features)
     rng = check_random_state(random_state)
-    components = rng.normal(loc=0.0,
-                            scale=1.0 / np.sqrt(n_components),
-                            size=(n_components, n_features))
+    components = rng.normal(
+        loc=0.0, scale=1.0 / np.sqrt(n_components), size=(n_components, n_features)
+    )
     return components
 
 
-def _sparse_random_matrix(n_components, n_features, density='auto',
-                          random_state=None):
+def _sparse_random_matrix(n_components, n_features, density="auto", random_state=None):
     """Generalized Achlioptas random sparse matrix for random projection.
 
     Setting density to 1 / 3 will yield the original matrix by Dimitris
@@ -252,7 +264,7 @@ def _sparse_random_matrix(n_components, n_features, density='auto',
            https://web.stanford.edu/~hastie/Papers/Ping/KDD06_rp.pdf
 
     .. [2] D. Achlioptas, 2001, "Database-friendly random projections",
-           http://www.cs.ucsc.edu/~optas/papers/jl.pdf
+           https://cgi.di.uoa.gr/~optas/papers/jl.pdf
 
     """
     _check_input_size(n_components, n_features)
@@ -272,8 +284,9 @@ def _sparse_random_matrix(n_components, n_features, density='auto',
         for _ in range(n_components):
             # find the indices of the non-zero components for row i
             n_nonzero_i = rng.binomial(n_features, density)
-            indices_i = sample_without_replacement(n_features, n_nonzero_i,
-                                                   random_state=rng)
+            indices_i = sample_without_replacement(
+                n_features, n_nonzero_i, random_state=rng
+            )
             indices.append(indices_i)
             offset += n_nonzero_i
             indptr.append(offset)
@@ -284,25 +297,44 @@ def _sparse_random_matrix(n_components, n_features, density='auto',
         data = rng.binomial(1, 0.5, size=np.size(indices)) * 2 - 1
 
         # build the CSR structure by concatenating the rows
-        components = sp.csr_matrix((data, indices, indptr),
-                                   shape=(n_components, n_features))
+        components = sp.csr_matrix(
+            (data, indices, indptr), shape=(n_components, n_features)
+        )
 
         return np.sqrt(1 / density) / np.sqrt(n_components) * components
 
 
-class BaseRandomProjection(TransformerMixin, BaseEstimator, metaclass=ABCMeta):
+class BaseRandomProjection(
+    ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator, metaclass=ABCMeta
+):
     """Base class for random projections.
 
     Warning: This class should not be used directly.
     Use derived classes instead.
     """
 
+    _parameter_constraints: dict = {
+        "n_components": [
+            Interval(Integral, 1, None, closed="left"),
+            StrOptions({"auto"}),
+        ],
+        "eps": [Interval(Real, 0, None, closed="neither")],
+        "compute_inverse_components": ["boolean"],
+        "random_state": ["random_state"],
+    }
+
     @abstractmethod
-    def __init__(self, n_components='auto', *, eps=0.1, dense_output=False,
-                 random_state=None):
+    def __init__(
+        self,
+        n_components="auto",
+        *,
+        eps=0.1,
+        compute_inverse_components=False,
+        random_state=None,
+    ):
         self.n_components = n_components
         self.eps = eps
-        self.dense_output = dense_output
+        self.compute_inverse_components = compute_inverse_components
         self.random_state = random_state
 
     @abstractmethod
@@ -319,12 +351,19 @@ def _make_random_matrix(self, n_components, n_features):
 
         Returns
         -------
-        components : {ndarray, sparse matrix} of shape \
-                (n_components, n_features)
+        components : {ndarray, sparse matrix} of shape (n_components, n_features)
             The generated random matrix. Sparse matrix will be of CSR format.
 
         """
 
+    def _compute_inverse_components(self):
+        """Compute the pseudo-inverse of the (densified) components."""
+        components = self.components_
+        if sp.issparse(components):
+            components = components.toarray()
+        return linalg.pinv(components, check_finite=False)
+
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y=None):
         """Generate a sparse random projection matrix.
 
@@ -335,85 +374,97 @@ def fit(self, X, y=None):
             matrix dimensions based on the theory referenced in the
             afore mentioned papers.
 
-        y
-            Ignored
+        y : Ignored
+            Not used, present here for API consistency by convention.
 
         Returns
         -------
-        self
-
+        self : object
+            BaseRandomProjection class instance.
         """
-        X = self._validate_data(X, accept_sparse=['csr', 'csc'])
+        X = validate_data(
+            self, X, accept_sparse=["csr", "csc"], dtype=[np.float64, np.float32]
+        )
 
         n_samples, n_features = X.shape
 
-        if self.n_components == 'auto':
+        if self.n_components == "auto":
             self.n_components_ = johnson_lindenstrauss_min_dim(
-                n_samples=n_samples, eps=self.eps)
+                n_samples=n_samples, eps=self.eps
+            )
 
             if self.n_components_ <= 0:
                 raise ValueError(
-                    'eps=%f and n_samples=%d lead to a target dimension of '
-                    '%d which is invalid' % (
-                        self.eps, n_samples, self.n_components_))
+                    "eps=%f and n_samples=%d lead to a target dimension of "
+                    "%d which is invalid" % (self.eps, n_samples, self.n_components_)
+                )
 
             elif self.n_components_ > n_features:
                 raise ValueError(
-                    'eps=%f and n_samples=%d lead to a target dimension of '
-                    '%d which is larger than the original space with '
-                    'n_features=%d' % (self.eps, n_samples, self.n_components_,
-                                       n_features))
+                    "eps=%f and n_samples=%d lead to a target dimension of "
+                    "%d which is larger than the original space with "
+                    "n_features=%d"
+                    % (self.eps, n_samples, self.n_components_, n_features)
+                )
         else:
-            if self.n_components <= 0:
-                raise ValueError("n_components must be greater than 0, got %s"
-                                 % self.n_components)
-
-            elif self.n_components > n_features:
+            if self.n_components > n_features:
                 warnings.warn(
                     "The number of components is higher than the number of"
                     " features: n_features < n_components (%s < %s)."
                     "The dimensionality of the problem will not be reduced."
                     % (n_features, self.n_components),
-                    DataDimensionalityWarning)
+                    DataDimensionalityWarning,
+                )
 
             self.n_components_ = self.n_components
 
         # Generate a projection matrix of size [n_components, n_features]
-        self.components_ = self._make_random_matrix(self.n_components_,
-                                                    n_features)
+        self.components_ = self._make_random_matrix(
+            self.n_components_, n_features
+        ).astype(X.dtype, copy=False)
+
+        if self.compute_inverse_components:
+            self.inverse_components_ = self._compute_inverse_components()
 
-        # Check contract
-        assert self.components_.shape == (self.n_components_, n_features), (
-                'An error has occurred the self.components_ matrix has '
-                ' not the proper shape.')
+        # Required by ClassNamePrefixFeaturesOutMixin.get_feature_names_out.
+        self._n_features_out = self.n_components
 
         return self
 
-    def transform(self, X):
-        """Project the data by using matrix product with the random matrix
+    def inverse_transform(self, X):
+        """Project data back to its original space.
+
+        Returns an array X_original whose transform would be X. Note that even
+        if X is sparse, X_original is dense: this may use a lot of RAM.
+
+        If `compute_inverse_components` is False, the inverse of the components is
+        computed during each call to `inverse_transform` which can be costly.
 
         Parameters
         ----------
-        X : {ndarray, sparse matrix} of shape (n_samples, n_features)
-            The input data to project into a smaller dimensional space.
+        X : {array-like, sparse matrix} of shape (n_samples, n_components)
+            Data to be transformed back.
 
         Returns
         -------
-        X_new : {ndarray, sparse matrix} of shape (n_samples, n_components)
-            Projected array.
+        X_original : ndarray of shape (n_samples, n_features)
+            Reconstructed data.
         """
         check_is_fitted(self)
-        X = self._validate_data(X, accept_sparse=['csr', 'csc'], reset=False)
 
-        if X.shape[1] != self.components_.shape[1]:
-            raise ValueError(
-                'Impossible to perform projection:'
-                'X at fit stage had a different number of features. '
-                '(%s != %s)' % (X.shape[1], self.components_.shape[1]))
+        X = check_array(X, dtype=[np.float64, np.float32], accept_sparse=("csr", "csc"))
+
+        if self.compute_inverse_components:
+            return X @ self.inverse_components_.T
 
-        X_new = safe_sparse_dot(X, self.components_.T,
-                                dense_output=self.dense_output)
-        return X_new
+        inverse_components = self._compute_inverse_components()
+        return X @ inverse_components.T
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.transformer_tags.preserves_dtype = ["float64", "float32"]
+        tags.input_tags.sparse = True
+        return tags
 
 
 class GaussianRandomProjection(BaseRandomProjection):
@@ -447,6 +498,11 @@ class GaussianRandomProjection(BaseRandomProjection):
         Smaller values lead to better embedding and higher number of
         dimensions (n_components) in the target projection space.
 
+    compute_inverse_components : bool, default=False
+        Learn the inverse transform by computing the pseudo-inverse of the
+        components during fit. Note that computing the pseudo-inverse does not
+        scale well to large matrices.
+
     random_state : int, RandomState instance or None, default=None
         Controls the pseudo random number generator used to generate the
         projection matrix at fit time.
@@ -461,32 +517,57 @@ class GaussianRandomProjection(BaseRandomProjection):
     components_ : ndarray of shape (n_components, n_features)
         Random matrix used for the projection.
 
+    inverse_components_ : ndarray of shape (n_features, n_components)
+        Pseudo-inverse of the components, only computed if
+        `compute_inverse_components` is True.
+
+        .. versionadded:: 1.1
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    See Also
+    --------
+    SparseRandomProjection : Reduce dimensionality through sparse
+        random projection.
+
     Examples
     --------
     >>> import numpy as np
     >>> from sklearn.random_projection import GaussianRandomProjection
     >>> rng = np.random.RandomState(42)
-    >>> X = rng.rand(100, 10000)
+    >>> X = rng.rand(25, 3000)
     >>> transformer = GaussianRandomProjection(random_state=rng)
     >>> X_new = transformer.fit_transform(X)
     >>> X_new.shape
-    (100, 3947)
-
-    See Also
-    --------
-    SparseRandomProjection
-
+    (25, 2759)
     """
-    @_deprecate_positional_args
-    def __init__(self, n_components='auto', *, eps=0.1, random_state=None):
+
+    def __init__(
+        self,
+        n_components="auto",
+        *,
+        eps=0.1,
+        compute_inverse_components=False,
+        random_state=None,
+    ):
         super().__init__(
             n_components=n_components,
             eps=eps,
-            dense_output=True,
-            random_state=random_state)
+            compute_inverse_components=compute_inverse_components,
+            random_state=random_state,
+        )
 
     def _make_random_matrix(self, n_components, n_features):
-        """ Generate the random projection matrix.
+        """Generate the random projection matrix.
 
         Parameters
         ----------
@@ -498,15 +579,37 @@ def _make_random_matrix(self, n_components, n_features):
 
         Returns
         -------
-        components : {ndarray, sparse matrix} of shape \
-                (n_components, n_features)
-            The generated random matrix. Sparse matrix will be of CSR format.
-
+        components : ndarray of shape (n_components, n_features)
+            The generated random matrix.
         """
         random_state = check_random_state(self.random_state)
-        return _gaussian_random_matrix(n_components,
-                                       n_features,
-                                       random_state=random_state)
+        return _gaussian_random_matrix(
+            n_components, n_features, random_state=random_state
+        )
+
+    def transform(self, X):
+        """Project the data by using matrix product with the random matrix.
+
+        Parameters
+        ----------
+        X : {ndarray, sparse matrix} of shape (n_samples, n_features)
+            The input data to project into a smaller dimensional space.
+
+        Returns
+        -------
+        X_new : ndarray of shape (n_samples, n_components)
+            Projected array.
+        """
+        check_is_fitted(self)
+        X = validate_data(
+            self,
+            X,
+            accept_sparse=["csr", "csc"],
+            reset=False,
+            dtype=[np.float64, np.float32],
+        )
+
+        return X @ self.components_.T
 
 
 class SparseRandomProjection(BaseRandomProjection):
@@ -520,9 +623,11 @@ class SparseRandomProjection(BaseRandomProjection):
     If we note `s = 1 / density` the components of the random matrix are
     drawn from:
 
-      - -sqrt(s) / sqrt(n_components)   with probability 1 / 2s
-      -  0                              with probability 1 - 1 / s
-      - +sqrt(s) / sqrt(n_components)   with probability 1 / 2s
+    .. code-block:: text
+
+      -sqrt(s) / sqrt(n_components)   with probability 1 / 2s
+       0                              with probability 1 - 1 / s
+      +sqrt(s) / sqrt(n_components)   with probability 1 / 2s
 
     Read more in the :ref:`User Guide `.
 
@@ -571,6 +676,14 @@ class SparseRandomProjection(BaseRandomProjection):
         If False, the projected data uses a sparse representation if
         the input is sparse.
 
+    compute_inverse_components : bool, default=False
+        Learn the inverse transform by computing the pseudo-inverse of the
+        components during fit. Note that the pseudo-inverse is always a dense
+        array, even if the training data was sparse. This means that it might be
+        necessary to call `inverse_transform` on a small batch of samples at a
+        time to avoid exhausting the available memory on the host. Moreover,
+        computing the pseudo-inverse does not scale well to large matrices.
+
     random_state : int, RandomState instance or None, default=None
         Controls the pseudo random number generator used to generate the
         projection matrix at fit time.
@@ -586,26 +699,30 @@ class SparseRandomProjection(BaseRandomProjection):
         Random matrix used for the projection. Sparse matrix will be of CSR
         format.
 
+    inverse_components_ : ndarray of shape (n_features, n_components)
+        Pseudo-inverse of the components, only computed if
+        `compute_inverse_components` is True.
+
+        .. versionadded:: 1.1
+
     density_ : float in range 0.0 - 1.0
         Concrete density computed from when density = "auto".
 
-    Examples
-    --------
-    >>> import numpy as np
-    >>> from sklearn.random_projection import SparseRandomProjection
-    >>> rng = np.random.RandomState(42)
-    >>> X = rng.rand(100, 10000)
-    >>> transformer = SparseRandomProjection(random_state=rng)
-    >>> X_new = transformer.fit_transform(X)
-    >>> X_new.shape
-    (100, 3947)
-    >>> # very few components are non-zero
-    >>> np.mean(transformer.components_ != 0)
-    0.0100...
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
 
     See Also
     --------
-    GaussianRandomProjection
+    GaussianRandomProjection : Reduce dimensionality through Gaussian
+        random projection.
 
     References
     ----------
@@ -615,22 +732,51 @@ class SparseRandomProjection(BaseRandomProjection):
            https://web.stanford.edu/~hastie/Papers/Ping/KDD06_rp.pdf
 
     .. [2] D. Achlioptas, 2001, "Database-friendly random projections",
-           https://users.soe.ucsc.edu/~optas/papers/jl.pdf
+           https://cgi.di.uoa.gr/~optas/papers/jl.pdf
 
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from sklearn.random_projection import SparseRandomProjection
+    >>> rng = np.random.RandomState(42)
+    >>> X = rng.rand(25, 3000)
+    >>> transformer = SparseRandomProjection(random_state=rng)
+    >>> X_new = transformer.fit_transform(X)
+    >>> X_new.shape
+    (25, 2759)
+    >>> # very few components are non-zero
+    >>> np.mean(transformer.components_ != 0)
+    np.float64(0.0182)
     """
-    @_deprecate_positional_args
-    def __init__(self, n_components='auto', *, density='auto', eps=0.1,
-                 dense_output=False, random_state=None):
+
+    _parameter_constraints: dict = {
+        **BaseRandomProjection._parameter_constraints,
+        "density": [Interval(Real, 0.0, 1.0, closed="right"), StrOptions({"auto"})],
+        "dense_output": ["boolean"],
+    }
+
+    def __init__(
+        self,
+        n_components="auto",
+        *,
+        density="auto",
+        eps=0.1,
+        dense_output=False,
+        compute_inverse_components=False,
+        random_state=None,
+    ):
         super().__init__(
             n_components=n_components,
             eps=eps,
-            dense_output=dense_output,
-            random_state=random_state)
+            compute_inverse_components=compute_inverse_components,
+            random_state=random_state,
+        )
 
+        self.dense_output = dense_output
         self.density = density
 
     def _make_random_matrix(self, n_components, n_features):
-        """ Generate the random projection matrix
+        """Generate the random projection matrix
 
         Parameters
         ----------
@@ -642,14 +788,37 @@ def _make_random_matrix(self, n_components, n_features):
 
         Returns
         -------
-        components : {ndarray, sparse matrix} of shape \
-                (n_components, n_features)
-            The generated random matrix. Sparse matrix will be of CSR format.
+        components : sparse matrix of shape (n_components, n_features)
+            The generated random matrix in CSR format.
 
         """
         random_state = check_random_state(self.random_state)
         self.density_ = _check_density(self.density, n_features)
-        return _sparse_random_matrix(n_components,
-                                     n_features,
-                                     density=self.density_,
-                                     random_state=random_state)
+        return _sparse_random_matrix(
+            n_components, n_features, density=self.density_, random_state=random_state
+        )
+
+    def transform(self, X):
+        """Project the data by using matrix product with the random matrix.
+
+        Parameters
+        ----------
+        X : {ndarray, sparse matrix} of shape (n_samples, n_features)
+            The input data to project into a smaller dimensional space.
+
+        Returns
+        -------
+        X_new : {ndarray, sparse matrix} of shape (n_samples, n_components)
+            Projected array. It is a sparse matrix only when the input is sparse and
+            `dense_output = False`.
+        """
+        check_is_fitted(self)
+        X = validate_data(
+            self,
+            X,
+            accept_sparse=["csr", "csc"],
+            reset=False,
+            dtype=[np.float64, np.float32],
+        )
+
+        return safe_sparse_dot(X, self.components_.T, dense_output=self.dense_output)
diff --git a/sklearn/semi_supervised/__init__.py b/sklearn/semi_supervised/__init__.py
index 8fa0365bc999c..453cd5edc348b 100644
--- a/sklearn/semi_supervised/__init__.py
+++ b/sklearn/semi_supervised/__init__.py
@@ -1,11 +1,13 @@
+"""Semi-supervised learning algorithms.
+
+These algorithms utilize small amounts of labeled data and large amounts of unlabeled
+data for classification tasks.
 """
-The :mod:`sklearn.semi_supervised` module implements semi-supervised learning
-algorithms. These algorithms utilize small amounts of labeled data and large
-amounts of unlabeled data for classification tasks. This module includes Label
-Propagation.
-"""
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 from ._label_propagation import LabelPropagation, LabelSpreading
 from ._self_training import SelfTrainingClassifier
 
-__all__ = ['SelfTrainingClassifier', 'LabelPropagation', 'LabelSpreading']
+__all__ = ["LabelPropagation", "LabelSpreading", "SelfTrainingClassifier"]
diff --git a/sklearn/semi_supervised/_label_propagation.py b/sklearn/semi_supervised/_label_propagation.py
index 8ba99b9603e05..559a17a13d6ae 100644
--- a/sklearn/semi_supervised/_label_propagation.py
+++ b/sklearn/semi_supervised/_label_propagation.py
@@ -52,64 +52,82 @@
 Non-Parametric Function Induction in Semi-Supervised Learning. AISTAT 2005
 """
 
-# Authors: Clay Woolam 
-#          Utkarsh Upadhyay 
-# License: BSD
-from abc import ABCMeta, abstractmethod
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 import warnings
+from abc import ABCMeta, abstractmethod
+from numbers import Integral, Real
+
 import numpy as np
 from scipy import sparse
-from scipy.sparse import csgraph
 
-from ..base import BaseEstimator, ClassifierMixin
+from ..base import BaseEstimator, ClassifierMixin, _fit_context
+from ..exceptions import ConvergenceWarning
 from ..metrics.pairwise import rbf_kernel
 from ..neighbors import NearestNeighbors
+from ..utils._param_validation import Interval, StrOptions
 from ..utils.extmath import safe_sparse_dot
+from ..utils.fixes import laplacian as csgraph_laplacian
 from ..utils.multiclass import check_classification_targets
-from ..utils.validation import check_is_fitted
-from ..utils.validation import _deprecate_positional_args
-from ..exceptions import ConvergenceWarning
+from ..utils.validation import check_is_fitted, validate_data
 
 
 class BaseLabelPropagation(ClassifierMixin, BaseEstimator, metaclass=ABCMeta):
     """Base class for label propagation module.
 
-    Parameters
-    ----------
-    kernel : {'knn', 'rbf'} or callable, default='rbf'
-        String identifier for kernel function to use or the kernel function
-        itself. Only 'rbf' and 'knn' strings are valid inputs. The function
-        passed should take two inputs, each of shape (n_samples, n_features),
-        and return a (n_samples, n_samples) shaped weight matrix.
+     Parameters
+     ----------
+     kernel : {'knn', 'rbf'} or callable, default='rbf'
+         String identifier for kernel function to use or the kernel function
+         itself. Only 'rbf' and 'knn' strings are valid inputs. The function
+         passed should take two inputs, each of shape (n_samples, n_features),
+         and return a (n_samples, n_samples) shaped weight matrix.
 
-    gamma : float, default=20
-        Parameter for rbf kernel.
+     gamma : float, default=20
+         Parameter for rbf kernel.
 
-    n_neighbors : int, default=7
-        Parameter for knn kernel. Need to be strictly positive.
+     n_neighbors : int, default=7
+         Parameter for knn kernel. Need to be strictly positive.
 
-    alpha : float, default=1.0
-        Clamping factor.
+     alpha : float, default=1.0
+         Clamping factor.
 
-    max_iter : int, default=30
-        Change maximum number of iterations allowed.
+     max_iter : int, default=30
+         Change maximum number of iterations allowed.
 
-    tol : float, default=1e-3
-        Convergence tolerance: threshold to consider the system at steady
-        state.
+     tol : float, default=1e-3
+         Convergence tolerance: threshold to consider the system at steady
+         state.
 
-   n_jobs : int, default=None
-        The number of parallel jobs to run.
-        ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
-        ``-1`` means using all processors. See :term:`Glossary `
-        for more details.
+    n_jobs : int, default=None
+         The number of parallel jobs to run.
+         ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
+         ``-1`` means using all processors. See :term:`Glossary `
+         for more details.
     """
 
-    @_deprecate_positional_args
-    def __init__(self, kernel='rbf', *, gamma=20, n_neighbors=7,
-                 alpha=1, max_iter=30, tol=1e-3, n_jobs=None):
-
+    _parameter_constraints: dict = {
+        "kernel": [StrOptions({"knn", "rbf"}), callable],
+        "gamma": [Interval(Real, 0, None, closed="left")],
+        "n_neighbors": [Interval(Integral, 0, None, closed="neither")],
+        "alpha": [None, Interval(Real, 0, 1, closed="neither")],
+        "max_iter": [Interval(Integral, 0, None, closed="neither")],
+        "tol": [Interval(Real, 0, None, closed="left")],
+        "n_jobs": [None, Integral],
+    }
+
+    def __init__(
+        self,
+        kernel="rbf",
+        *,
+        gamma=20,
+        n_neighbors=7,
+        alpha=1,
+        max_iter=30,
+        tol=1e-3,
+        n_jobs=None,
+    ):
         self.max_iter = max_iter
         self.tol = tol
 
@@ -131,12 +149,13 @@ def _get_kernel(self, X, y=None):
                 return rbf_kernel(X, y, gamma=self.gamma)
         elif self.kernel == "knn":
             if self.nn_fit is None:
-                self.nn_fit = NearestNeighbors(n_neighbors=self.n_neighbors,
-                                               n_jobs=self.n_jobs).fit(X)
+                self.nn_fit = NearestNeighbors(
+                    n_neighbors=self.n_neighbors, n_jobs=self.n_jobs
+                ).fit(X)
             if y is None:
-                return self.nn_fit.kneighbors_graph(self.nn_fit._fit_X,
-                                                    self.n_neighbors,
-                                                    mode='connectivity')
+                return self.nn_fit.kneighbors_graph(
+                    self.nn_fit._fit_X, self.n_neighbors, mode="connectivity"
+                )
             else:
                 return self.nn_fit.kneighbors(y, return_distance=False)
         elif callable(self.kernel):
@@ -144,18 +163,15 @@ def _get_kernel(self, X, y=None):
                 return self.kernel(X, X)
             else:
                 return self.kernel(X, y)
-        else:
-            raise ValueError("%s is not a valid kernel. Only rbf and knn"
-                             " or an explicit function "
-                             " are supported at this time." % self.kernel)
 
     @abstractmethod
     def _build_graph(self):
-        raise NotImplementedError("Graph construction must be implemented"
-                                  " to fit a label propagation model.")
+        raise NotImplementedError(
+            "Graph construction must be implemented to fit a label propagation model."
+        )
 
     def predict(self, X):
-        """Performs inductive inference across the model.
+        """Perform inductive inference across the model.
 
         Parameters
         ----------
@@ -167,6 +183,10 @@ def predict(self, X):
         y : ndarray of shape (n_samples,)
             Predictions for input data.
         """
+        # Note: since `predict` does not accept semi-supervised labels as input,
+        # `fit(X, y).predict(X) != fit(X, y).transduction_`.
+        # Hence, `fit_predict` is not implemented.
+        # See https://github.com/scikit-learn/scikit-learn/pull/24898
         probas = self.predict_proba(X)
         return self.classes_[np.argmax(probas, axis=1)].ravel()
 
@@ -190,43 +210,59 @@ class labels.
         """
         check_is_fitted(self)
 
-        X_2d = self._validate_data(
-            X, accept_sparse=['csc', 'csr', 'coo', 'dok', 'bsr', 'lil', 'dia'],
-            reset=False)
+        X_2d = validate_data(
+            self,
+            X,
+            accept_sparse=["csc", "csr", "coo", "dok", "bsr", "lil", "dia"],
+            reset=False,
+        )
         weight_matrices = self._get_kernel(self.X_, X_2d)
-        if self.kernel == 'knn':
-            probabilities = np.array([
-                np.sum(self.label_distributions_[weight_matrix], axis=0)
-                for weight_matrix in weight_matrices])
+        if self.kernel == "knn":
+            probabilities = np.array(
+                [
+                    np.sum(self.label_distributions_[weight_matrix], axis=0)
+                    for weight_matrix in weight_matrices
+                ]
+            )
         else:
             weight_matrices = weight_matrices.T
-            probabilities = safe_sparse_dot(
-                    weight_matrices, self.label_distributions_)
+            probabilities = safe_sparse_dot(weight_matrices, self.label_distributions_)
         normalizer = np.atleast_2d(np.sum(probabilities, axis=1)).T
         probabilities /= normalizer
         return probabilities
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y):
-        """Fit a semi-supervised label propagation model based
+        """Fit a semi-supervised label propagation model to X.
 
-        All the input data is provided matrix X (labeled and unlabeled)
-        and corresponding label matrix y with a dedicated marker value for
-        unlabeled samples.
+        The input samples (labeled and unlabeled) are provided by matrix X,
+        and target labels are provided by matrix y. We conventionally apply the
+        label -1 to unlabeled samples in matrix y in a semi-supervised
+        classification.
 
         Parameters
         ----------
-        X : array-like of shape (n_samples, n_features)
-            A matrix of shape (n_samples, n_samples) will be created from this.
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Training data, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
 
         y : array-like of shape (n_samples,)
-            `n_labeled_samples` (unlabeled points are marked as -1)
-            All unlabeled samples will be transductively assigned labels.
+            Target class values with unlabeled points marked as -1.
+            All unlabeled samples will be transductively assigned labels
+            internally, which are stored in `transduction_`.
 
         Returns
         -------
         self : object
+            Returns the instance itself.
         """
-        X, y = self._validate_data(X, y)
+        X, y = validate_data(
+            self,
+            X,
+            y,
+            accept_sparse=["csr", "csc"],
+            reset=True,
+        )
         self.X_ = X
         check_classification_targets(y)
 
@@ -236,16 +272,11 @@ def fit(self, X, y):
         # label construction
         # construct a categorical distribution for classification only
         classes = np.unique(y)
-        classes = (classes[classes != -1])
+        classes = classes[classes != -1]
         self.classes_ = classes
 
         n_samples, n_classes = len(y), len(classes)
 
-        alpha = self.alpha
-        if self._variant == 'spreading' and \
-                (alpha is None or alpha <= 0.0 or alpha >= 1.0):
-            raise ValueError('alpha=%s is invalid: it must be inside '
-                             'the open interval (0, 1)' % alpha)
         y = np.asarray(y)
         unlabeled = y == -1
 
@@ -255,17 +286,17 @@ def fit(self, X, y):
             self.label_distributions_[y == label, classes == label] = 1
 
         y_static = np.copy(self.label_distributions_)
-        if self._variant == 'propagation':
+        if self._variant == "propagation":
             # LabelPropagation
             y_static[unlabeled] = 0
         else:
             # LabelSpreading
-            y_static *= 1 - alpha
+            y_static *= 1 - self.alpha
 
         l_previous = np.zeros((self.X_.shape[0], n_classes))
 
         unlabeled = unlabeled[:, np.newaxis]
-        if sparse.isspmatrix(graph_matrix):
+        if sparse.issparse(graph_matrix):
             graph_matrix = graph_matrix.tocsr()
 
         for self.n_iter_ in range(self.max_iter):
@@ -274,24 +305,25 @@ def fit(self, X, y):
 
             l_previous = self.label_distributions_
             self.label_distributions_ = safe_sparse_dot(
-                graph_matrix, self.label_distributions_)
+                graph_matrix, self.label_distributions_
+            )
 
-            if self._variant == 'propagation':
-                normalizer = np.sum(
-                    self.label_distributions_, axis=1)[:, np.newaxis]
+            if self._variant == "propagation":
+                normalizer = np.sum(self.label_distributions_, axis=1)[:, np.newaxis]
                 normalizer[normalizer == 0] = 1
                 self.label_distributions_ /= normalizer
-                self.label_distributions_ = np.where(unlabeled,
-                                                     self.label_distributions_,
-                                                     y_static)
+                self.label_distributions_ = np.where(
+                    unlabeled, self.label_distributions_, y_static
+                )
             else:
                 # clamp
-                self.label_distributions_ = np.multiply(
-                    alpha, self.label_distributions_) + y_static
+                self.label_distributions_ = (
+                    np.multiply(self.alpha, self.label_distributions_) + y_static
+                )
         else:
             warnings.warn(
-                'max_iter=%d was reached without convergence.' % self.max_iter,
-                category=ConvergenceWarning
+                "max_iter=%d was reached without convergence." % self.max_iter,
+                category=ConvergenceWarning,
             )
             self.n_iter_ += 1
 
@@ -300,14 +332,18 @@ def fit(self, X, y):
         self.label_distributions_ /= normalizer
 
         # set the transduction item
-        transduction = self.classes_[np.argmax(self.label_distributions_,
-                                               axis=1)]
+        transduction = self.classes_[np.argmax(self.label_distributions_, axis=1)]
         self.transduction_ = transduction.ravel()
         return self
 
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
+        return tags
+
 
 class LabelPropagation(BaseLabelPropagation):
-    """Label Propagation classifier
+    """Label Propagation classifier.
 
     Read more in the :ref:`User Guide `.
 
@@ -328,7 +364,7 @@ class LabelPropagation(BaseLabelPropagation):
     max_iter : int, default=1000
         Change maximum number of iterations allowed.
 
-    tol : float, 1e-3
+    tol : float, default=1e-3
         Convergence tolerance: threshold to consider the system at steady
         state.
 
@@ -340,7 +376,7 @@ class LabelPropagation(BaseLabelPropagation):
 
     Attributes
     ----------
-    X_ : ndarray of shape (n_samples, n_features)
+    X_ : {array-like, sparse matrix} of shape (n_samples, n_features)
         Input array.
 
     classes_ : ndarray of shape (n_classes,)
@@ -350,11 +386,32 @@ class LabelPropagation(BaseLabelPropagation):
         Categorical distribution for each item.
 
     transduction_ : ndarray of shape (n_samples)
-        Label assigned to each item via the transduction.
+        Label assigned to each item during :term:`fit`.
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
 
     n_iter_ : int
         Number of iterations run.
 
+    See Also
+    --------
+    LabelSpreading : Alternate label propagation strategy more robust to noise.
+
+    References
+    ----------
+    Xiaojin Zhu and Zoubin Ghahramani. Learning from labeled and unlabeled data
+    with label propagation. Technical Report CMU-CALD-02-107, Carnegie Mellon
+    University, 2002 http://pages.cs.wisc.edu/~jerryzhu/pub/CMU-CALD-02-107.pdf
+
     Examples
     --------
     >>> import numpy as np
@@ -368,26 +425,32 @@ class LabelPropagation(BaseLabelPropagation):
     >>> labels[random_unlabeled_points] = -1
     >>> label_prop_model.fit(iris.data, labels)
     LabelPropagation(...)
-
-    References
-    ----------
-    Xiaojin Zhu and Zoubin Ghahramani. Learning from labeled and unlabeled data
-    with label propagation. Technical Report CMU-CALD-02-107, Carnegie Mellon
-    University, 2002 http://pages.cs.wisc.edu/~jerryzhu/pub/CMU-CALD-02-107.pdf
-
-    See Also
-    --------
-    LabelSpreading : Alternate label propagation strategy more robust to noise.
     """
 
-    _variant = 'propagation'
-
-    @_deprecate_positional_args
-    def __init__(self, kernel='rbf', *, gamma=20, n_neighbors=7,
-                 max_iter=1000, tol=1e-3, n_jobs=None):
-        super().__init__(kernel=kernel, gamma=gamma,
-                         n_neighbors=n_neighbors, max_iter=max_iter,
-                         tol=tol, n_jobs=n_jobs, alpha=None)
+    _variant = "propagation"
+
+    _parameter_constraints: dict = {**BaseLabelPropagation._parameter_constraints}
+    _parameter_constraints.pop("alpha")
+
+    def __init__(
+        self,
+        kernel="rbf",
+        *,
+        gamma=20,
+        n_neighbors=7,
+        max_iter=1000,
+        tol=1e-3,
+        n_jobs=None,
+    ):
+        super().__init__(
+            kernel=kernel,
+            gamma=gamma,
+            n_neighbors=n_neighbors,
+            max_iter=max_iter,
+            tol=tol,
+            n_jobs=n_jobs,
+            alpha=None,
+        )
 
     def _build_graph(self):
         """Matrix representing a fully connected graph between each sample
@@ -395,22 +458,40 @@ def _build_graph(self):
         This basic implementation creates a non-stochastic affinity matrix, so
         class distributions will exceed 1 (normalization may be desired).
         """
-        if self.kernel == 'knn':
+        if self.kernel == "knn":
             self.nn_fit = None
         affinity_matrix = self._get_kernel(self.X_)
         normalizer = affinity_matrix.sum(axis=0)
-        if sparse.isspmatrix(affinity_matrix):
+        if sparse.issparse(affinity_matrix):
             affinity_matrix.data /= np.diag(np.array(normalizer))
         else:
             affinity_matrix /= normalizer[:, np.newaxis]
         return affinity_matrix
 
     def fit(self, X, y):
+        """Fit a semi-supervised label propagation model to X.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix} of shape (n_samples, n_features)
+            Training data, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
+
+        y : array-like of shape (n_samples,)
+            Target class values with unlabeled points marked as -1.
+            All unlabeled samples will be transductively assigned labels
+            internally, which are stored in `transduction_`.
+
+        Returns
+        -------
+        self : object
+            Returns the instance itself.
+        """
         return super().fit(X, y)
 
 
 class LabelSpreading(BaseLabelPropagation):
-    """LabelSpreading model for semi-supervised learning
+    """LabelSpreading model for semi-supervised learning.
 
     This model is similar to the basic Label Propagation algorithm,
     but uses affinity matrix based on the normalized graph Laplacian
@@ -464,11 +545,32 @@ class LabelSpreading(BaseLabelPropagation):
         Categorical distribution for each item.
 
     transduction_ : ndarray of shape (n_samples,)
-        Label assigned to each item via the transduction.
+        Label assigned to each item during :term:`fit`.
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
 
     n_iter_ : int
         Number of iterations run.
 
+    See Also
+    --------
+    LabelPropagation : Unregularized graph based semi-supervised learning.
+
+    References
+    ----------
+    `Dengyong Zhou, Olivier Bousquet, Thomas Navin Lal, Jason Weston,
+    Bernhard Schoelkopf. Learning with local and global consistency (2004)
+    `_
+
     Examples
     --------
     >>> import numpy as np
@@ -482,41 +584,47 @@ class LabelSpreading(BaseLabelPropagation):
     >>> labels[random_unlabeled_points] = -1
     >>> label_prop_model.fit(iris.data, labels)
     LabelSpreading(...)
-
-    References
-    ----------
-    Dengyong Zhou, Olivier Bousquet, Thomas Navin Lal, Jason Weston,
-    Bernhard Schoelkopf. Learning with local and global consistency (2004)
-    http://citeseer.ist.psu.edu/viewdoc/summary?doi=10.1.1.115.3219
-
-    See Also
-    --------
-    LabelPropagation : Unregularized graph based semi-supervised learning.
     """
 
-    _variant = 'spreading'
-
-    @_deprecate_positional_args
-    def __init__(self, kernel='rbf', *, gamma=20, n_neighbors=7, alpha=0.2,
-                 max_iter=30, tol=1e-3, n_jobs=None):
-
+    _variant = "spreading"
+
+    _parameter_constraints: dict = {**BaseLabelPropagation._parameter_constraints}
+    _parameter_constraints["alpha"] = [Interval(Real, 0, 1, closed="neither")]
+
+    def __init__(
+        self,
+        kernel="rbf",
+        *,
+        gamma=20,
+        n_neighbors=7,
+        alpha=0.2,
+        max_iter=30,
+        tol=1e-3,
+        n_jobs=None,
+    ):
         # this one has different base parameters
-        super().__init__(kernel=kernel, gamma=gamma,
-                         n_neighbors=n_neighbors, alpha=alpha,
-                         max_iter=max_iter, tol=tol, n_jobs=n_jobs)
+        super().__init__(
+            kernel=kernel,
+            gamma=gamma,
+            n_neighbors=n_neighbors,
+            alpha=alpha,
+            max_iter=max_iter,
+            tol=tol,
+            n_jobs=n_jobs,
+        )
 
     def _build_graph(self):
         """Graph matrix for Label Spreading computes the graph laplacian"""
         # compute affinity matrix (or gram matrix)
-        if self.kernel == 'knn':
+        if self.kernel == "knn":
             self.nn_fit = None
         n_samples = self.X_.shape[0]
         affinity_matrix = self._get_kernel(self.X_)
-        laplacian = csgraph.laplacian(affinity_matrix, normed=True)
+        laplacian = csgraph_laplacian(affinity_matrix, normed=True)
         laplacian = -laplacian
-        if sparse.isspmatrix(laplacian):
-            diag_mask = (laplacian.row == laplacian.col)
+        if sparse.issparse(laplacian):
+            diag_mask = laplacian.row == laplacian.col
             laplacian.data[diag_mask] = 0.0
         else:
-            laplacian.flat[::n_samples + 1] = 0.0  # set diag to 0.0
+            laplacian.flat[:: n_samples + 1] = 0.0  # set diag to 0.0
         return laplacian
diff --git a/sklearn/semi_supervised/_self_training.py b/sklearn/semi_supervised/_self_training.py
index 54fa9ba45e1b8..0fe6f57d6c1ed 100644
--- a/sklearn/semi_supervised/_self_training.py
+++ b/sklearn/semi_supervised/_self_training.py
@@ -1,30 +1,38 @@
 import warnings
+from numbers import Integral, Real
+from warnings import warn
 
 import numpy as np
 
-from ..base import MetaEstimatorMixin, clone, BaseEstimator
-from ..utils.validation import check_is_fitted
-from ..utils.metaestimators import if_delegate_has_method
-from ..utils import safe_mask
+from ..base import (
+    BaseEstimator,
+    ClassifierMixin,
+    MetaEstimatorMixin,
+    _fit_context,
+    clone,
+)
+from ..utils import Bunch, get_tags, safe_mask
+from ..utils._param_validation import HasMethods, Hidden, Interval, StrOptions
+from ..utils.metadata_routing import (
+    MetadataRouter,
+    MethodMapping,
+    _raise_for_params,
+    _routing_enabled,
+    process_routing,
+)
+from ..utils.metaestimators import available_if
+from ..utils.validation import _estimator_has, check_is_fitted, validate_data
 
 __all__ = ["SelfTrainingClassifier"]
 
-# Authors: Oliver Rausch   
-#          Patrice Becker  
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 
-def _validate_estimator(estimator):
-    """Make sure that an estimator implements the necessary methods."""
-    if not hasattr(estimator, "predict_proba"):
-        msg = "base_estimator ({}) should implement predict_proba!"
-        raise ValueError(msg.format(type(estimator).__name__))
-
-
-class SelfTrainingClassifier(MetaEstimatorMixin, BaseEstimator):
+class SelfTrainingClassifier(ClassifierMixin, MetaEstimatorMixin, BaseEstimator):
     """Self-training classifier.
 
-    This class allows a given supervised classifier to function as a
+    This :term:`metaestimator` allows a given supervised classifier to function as a
     semi-supervised classifier, allowing it to learn from unlabeled data. It
     does this by iteratively predicting pseudo-labels for the unlabeled data
     and adding them to the training set.
@@ -36,31 +44,43 @@ class SelfTrainingClassifier(MetaEstimatorMixin, BaseEstimator):
 
     Parameters
     ----------
+    estimator : estimator object
+        An estimator object implementing `fit` and `predict_proba`.
+        Invoking the `fit` method will fit a clone of the passed estimator,
+        which will be stored in the `estimator_` attribute.
+
+        .. versionadded:: 1.6
+            `estimator` was added to replace `base_estimator`.
+
     base_estimator : estimator object
-        An estimator object implementing ``fit`` and ``predict_proba``.
-        Invoking the ``fit`` method will fit a clone of the passed estimator,
-        which will be stored in the ``base_estimator_`` attribute.
+        An estimator object implementing `fit` and `predict_proba`.
+        Invoking the `fit` method will fit a clone of the passed estimator,
+        which will be stored in the `estimator_` attribute.
+
+        .. deprecated:: 1.6
+            `base_estimator` was deprecated in 1.6 and will be removed in 1.8.
+            Use `estimator` instead.
 
     threshold : float, default=0.75
         The decision threshold for use with `criterion='threshold'`.
-        Should be in [0, 1). When using the 'threshold' criterion, a
+        Should be in [0, 1). When using the `'threshold'` criterion, a
         :ref:`well calibrated classifier ` should be used.
 
     criterion : {'threshold', 'k_best'}, default='threshold'
         The selection criterion used to select which labels to add to the
-        training set. If 'threshold', pseudo-labels with prediction
-        probabilities above `threshold` are added to the dataset. If 'k_best',
+        training set. If `'threshold'`, pseudo-labels with prediction
+        probabilities above `threshold` are added to the dataset. If `'k_best'`,
         the `k_best` pseudo-labels with highest prediction probabilities are
         added to the dataset. When using the 'threshold' criterion, a
         :ref:`well calibrated classifier ` should be used.
 
     k_best : int, default=10
         The amount of samples to add in each iteration. Only used when
-        `criterion` is k_best'.
+        `criterion='k_best'`.
 
     max_iter : int or None, default=10
         Maximum number of iterations allowed. Should be greater than or equal
-        to 0. If it is ``None``, the classifier will continue to predict labels
+        to 0. If it is `None`, the classifier will continue to predict labels
         until no new pseudo-labels are added, or all unlabeled samples have
         been labeled.
 
@@ -69,12 +89,12 @@ class SelfTrainingClassifier(MetaEstimatorMixin, BaseEstimator):
 
     Attributes
     ----------
-    base_estimator_ : estimator object
+    estimator_ : estimator object
         The fitted estimator.
 
     classes_ : ndarray or list of ndarray of shape (n_classes,)
         Class labels for each output. (Taken from the trained
-        ``base_estimator_``).
+        `estimator_`).
 
     transduction_ : ndarray of shape (n_samples,)
         The labels used for the final fit of the classifier, including
@@ -86,6 +106,17 @@ class SelfTrainingClassifier(MetaEstimatorMixin, BaseEstimator):
         When a sample has iteration -1, the sample was not labeled in any
         iteration.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     n_iter_ : int
         The number of rounds of self-training, that is the number of times the
         base estimator is fitted on relabeled variants of the training set.
@@ -93,11 +124,24 @@ class SelfTrainingClassifier(MetaEstimatorMixin, BaseEstimator):
     termination_condition_ : {'max_iter', 'no_change', 'all_labeled'}
         The reason that fitting was stopped.
 
-        - 'max_iter': `n_iter_` reached `max_iter`.
-        - 'no_change': no new labels were predicted.
-        - 'all_labeled': all unlabeled samples were labeled before `max_iter`
+        - `'max_iter'`: `n_iter_` reached `max_iter`.
+        - `'no_change'`: no new labels were predicted.
+        - `'all_labeled'`: all unlabeled samples were labeled before `max_iter`
           was reached.
 
+    See Also
+    --------
+    LabelPropagation : Label propagation classifier.
+    LabelSpreading : Label spreading model for semi-supervised learning.
+
+    References
+    ----------
+    :doi:`David Yarowsky. 1995. Unsupervised word sense disambiguation rivaling
+    supervised methods. In Proceedings of the 33rd annual meeting on
+    Association for Computational Linguistics (ACL '95). Association for
+    Computational Linguistics, Stroudsburg, PA, USA, 189-196.
+    <10.3115/981658.981684>`
+
     Examples
     --------
     >>> import numpy as np
@@ -112,34 +156,85 @@ class SelfTrainingClassifier(MetaEstimatorMixin, BaseEstimator):
     >>> self_training_model = SelfTrainingClassifier(svc)
     >>> self_training_model.fit(iris.data, iris.target)
     SelfTrainingClassifier(...)
-
-    References
-    ----------
-    David Yarowsky. 1995. Unsupervised word sense disambiguation rivaling
-    supervised methods. In Proceedings of the 33rd annual meeting on
-    Association for Computational Linguistics (ACL '95). Association for
-    Computational Linguistics, Stroudsburg, PA, USA, 189-196. DOI:
-    https://doi.org/10.3115/981658.981684
     """
-    _estimator_type = "classifier"
-
-    def __init__(self,
-                 base_estimator,
-                 threshold=0.75,
-                 criterion='threshold',
-                 k_best=10,
-                 max_iter=10,
-                 verbose=False):
-        self.base_estimator = base_estimator
+
+    _parameter_constraints: dict = {
+        # We don't require `predic_proba` here to allow passing a meta-estimator
+        # that only exposes `predict_proba` after fitting.
+        # TODO(1.8) remove None option
+        "estimator": [None, HasMethods(["fit"])],
+        # TODO(1.8) remove
+        "base_estimator": [
+            HasMethods(["fit"]),
+            Hidden(StrOptions({"deprecated"})),
+        ],
+        "threshold": [Interval(Real, 0.0, 1.0, closed="left")],
+        "criterion": [StrOptions({"threshold", "k_best"})],
+        "k_best": [Interval(Integral, 1, None, closed="left")],
+        "max_iter": [Interval(Integral, 0, None, closed="left"), None],
+        "verbose": ["verbose"],
+    }
+
+    def __init__(
+        self,
+        estimator=None,
+        base_estimator="deprecated",
+        threshold=0.75,
+        criterion="threshold",
+        k_best=10,
+        max_iter=10,
+        verbose=False,
+    ):
+        self.estimator = estimator
         self.threshold = threshold
         self.criterion = criterion
         self.k_best = k_best
         self.max_iter = max_iter
         self.verbose = verbose
 
-    def fit(self, X, y):
+        # TODO(1.8) remove
+        self.base_estimator = base_estimator
+
+    def _get_estimator(self):
+        """Get the estimator.
+
+        Returns
+        -------
+        estimator_ : estimator object
+            The cloned estimator object.
         """
-        Fits this ``SelfTrainingClassifier`` to a dataset.
+        # TODO(1.8): remove and only keep clone(self.estimator)
+        if self.estimator is None and self.base_estimator != "deprecated":
+            estimator_ = clone(self.base_estimator)
+
+            warn(
+                (
+                    "`base_estimator` has been deprecated in 1.6 and will be removed"
+                    " in 1.8. Please use `estimator` instead."
+                ),
+                FutureWarning,
+            )
+        # TODO(1.8) remove
+        elif self.estimator is None and self.base_estimator == "deprecated":
+            raise ValueError(
+                "You must pass an estimator to SelfTrainingClassifier. Use `estimator`."
+            )
+        elif self.estimator is not None and self.base_estimator != "deprecated":
+            raise ValueError(
+                "You must pass only one estimator to SelfTrainingClassifier."
+                " Use `estimator`."
+            )
+        else:
+            estimator_ = clone(self.estimator)
+        return estimator_
+
+    @_fit_context(
+        # SelfTrainingClassifier.estimator is not validated yet
+        prefer_skip_nested_validation=False
+    )
+    def fit(self, X, y, **params):
+        """
+        Fit self-training classifier using `X`, `y` as training data.
 
         Parameters
         ----------
@@ -150,47 +245,63 @@ def fit(self, X, y):
             Array representing the labels. Unlabeled samples should have the
             label -1.
 
+        **params : dict
+            Parameters to pass to the underlying estimators.
+
+            .. versionadded:: 1.6
+                Only available if `enable_metadata_routing=True`,
+                which can be set by using
+                ``sklearn.set_config(enable_metadata_routing=True)``.
+                See :ref:`Metadata Routing User Guide ` for
+                more details.
+
         Returns
         -------
         self : object
-            Returns an instance of self.
+            Fitted estimator.
         """
-        # we need row slicing support for sparce matrices
-        X, y = self._validate_data(X, y, accept_sparse=[
-            'csr', 'csc', 'lil', 'dok'])
-
-        if self.base_estimator is None:
-            raise ValueError("base_estimator cannot be None!")
-
-        self.base_estimator_ = clone(self.base_estimator)
-
-        if self.max_iter is not None and self.max_iter < 0:
-            raise ValueError("max_iter must be >= 0 or None,"
-                             f" got {self.max_iter}")
-
-        if not (0 <= self.threshold < 1):
-            raise ValueError("threshold must be in [0,1),"
-                             f" got {self.threshold}")
-
-        if self.criterion not in ['threshold', 'k_best']:
-            raise ValueError(f"criterion must be either 'threshold' "
-                             f"or 'k_best', got {self.criterion}.")
-
-        if y.dtype.kind in ['U', 'S']:
-            raise ValueError("y has dtype string. If you wish to predict on "
-                             "string targets, use dtype object, and use -1"
-                             " as the label for unlabeled samples.")
+        _raise_for_params(params, self, "fit")
+
+        self.estimator_ = self._get_estimator()
+
+        # we need row slicing support for sparse matrices, but costly finiteness check
+        # can be delegated to the base estimator.
+        X, y = validate_data(
+            self,
+            X,
+            y,
+            accept_sparse=["csr", "csc", "lil", "dok"],
+            ensure_all_finite=False,
+        )
+
+        if y.dtype.kind in ["U", "S"]:
+            raise ValueError(
+                "y has dtype string. If you wish to predict on "
+                "string targets, use dtype object, and use -1"
+                " as the label for unlabeled samples."
+            )
 
         has_label = y != -1
 
         if np.all(has_label):
             warnings.warn("y contains no unlabeled samples", UserWarning)
 
-        if self.criterion == 'k_best' and (self.k_best > X.shape[0] -
-                                           np.sum(has_label)):
-            warnings.warn("k_best is larger than the amount of unlabeled "
-                          "samples. All unlabeled samples will be labeled in "
-                          "the first iteration", UserWarning)
+        if self.criterion == "k_best" and (
+            self.k_best > X.shape[0] - np.sum(has_label)
+        ):
+            warnings.warn(
+                (
+                    "k_best is larger than the amount of unlabeled "
+                    "samples. All unlabeled samples will be labeled in "
+                    "the first iteration"
+                ),
+                UserWarning,
+            )
+
+        if _routing_enabled():
+            routed_params = process_routing(self, "fit", **params)
+        else:
+            routed_params = Bunch(estimator=Bunch(fit={}))
 
         self.transduction_ = np.copy(y)
         self.labeled_iter_ = np.full_like(y, -1)
@@ -198,35 +309,31 @@ def fit(self, X, y):
 
         self.n_iter_ = 0
 
-        while not np.all(has_label) and (self.max_iter is None or
-                                         self.n_iter_ < self.max_iter):
+        while not np.all(has_label) and (
+            self.max_iter is None or self.n_iter_ < self.max_iter
+        ):
             self.n_iter_ += 1
-            self.base_estimator_.fit(
+            self.estimator_.fit(
                 X[safe_mask(X, has_label)],
-                self.transduction_[has_label])
-
-            # Validate the fitted estimator since `predict_proba` can be
-            # delegated to an underlying "final" fitted estimator as
-            # generally done in meta-estimator or pipeline.
-            _validate_estimator(self.base_estimator_)
+                self.transduction_[has_label],
+                **routed_params.estimator.fit,
+            )
 
             # Predict on the unlabeled samples
-            prob = self.base_estimator_.predict_proba(
-                X[safe_mask(X, ~has_label)])
-            pred = self.base_estimator_.classes_[np.argmax(prob, axis=1)]
+            prob = self.estimator_.predict_proba(X[safe_mask(X, ~has_label)])
+            pred = self.estimator_.classes_[np.argmax(prob, axis=1)]
             max_proba = np.max(prob, axis=1)
 
             # Select new labeled samples
-            if self.criterion == 'threshold':
+            if self.criterion == "threshold":
                 selected = max_proba > self.threshold
             else:
                 n_to_select = min(self.k_best, max_proba.shape[0])
                 if n_to_select == max_proba.shape[0]:
                     selected = np.ones_like(max_proba, dtype=bool)
                 else:
-                    # NB these are indicies, not a mask
-                    selected = \
-                        np.argpartition(-max_proba, n_to_select)[:n_to_select]
+                    # NB these are indices, not a mask
+                    selected = np.argpartition(-max_proba, n_to_select)[:n_to_select]
 
             # Map selected indices into original array
             selected_full = np.nonzero(~has_label)[0][selected]
@@ -242,38 +349,68 @@ def fit(self, X, y):
                 break
 
             if self.verbose:
-                print(f"End of iteration {self.n_iter_},"
-                      f" added {selected_full.shape[0]} new labels.")
+                print(
+                    f"End of iteration {self.n_iter_},"
+                    f" added {selected_full.shape[0]} new labels."
+                )
 
         if self.n_iter_ == self.max_iter:
             self.termination_condition_ = "max_iter"
         if np.all(has_label):
             self.termination_condition_ = "all_labeled"
 
-        self.base_estimator_.fit(
+        self.estimator_.fit(
             X[safe_mask(X, has_label)],
-            self.transduction_[has_label])
-        self.classes_ = self.base_estimator_.classes_
+            self.transduction_[has_label],
+            **routed_params.estimator.fit,
+        )
+        self.classes_ = self.estimator_.classes_
         return self
 
-    @if_delegate_has_method(delegate='base_estimator')
-    def predict(self, X):
-        """Predict the classes of X.
+    @available_if(_estimator_has("predict"))
+    def predict(self, X, **params):
+        """Predict the classes of `X`.
 
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
             Array representing the data.
 
+        **params : dict of str -> object
+            Parameters to pass to the underlying estimator's ``predict`` method.
+
+            .. versionadded:: 1.6
+                Only available if `enable_metadata_routing=True`,
+                which can be set by using
+                ``sklearn.set_config(enable_metadata_routing=True)``.
+                See :ref:`Metadata Routing User Guide ` for
+                more details.
+
         Returns
         -------
         y : ndarray of shape (n_samples,)
             Array with predicted labels.
         """
         check_is_fitted(self)
-        return self.base_estimator_.predict(X)
-
-    def predict_proba(self, X):
+        _raise_for_params(params, self, "predict")
+
+        if _routing_enabled():
+            # metadata routing is enabled.
+            routed_params = process_routing(self, "predict", **params)
+        else:
+            routed_params = Bunch(estimator=Bunch(predict={}))
+
+        X = validate_data(
+            self,
+            X,
+            accept_sparse=True,
+            ensure_all_finite=False,
+            reset=False,
+        )
+        return self.estimator_.predict(X, **routed_params.estimator.predict)
+
+    @available_if(_estimator_has("predict_proba"))
+    def predict_proba(self, X, **params):
         """Predict probability for each possible outcome.
 
         Parameters
@@ -281,33 +418,87 @@ def predict_proba(self, X):
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
             Array representing the data.
 
+        **params : dict of str -> object
+            Parameters to pass to the underlying estimator's
+            ``predict_proba`` method.
+
+            .. versionadded:: 1.6
+                Only available if `enable_metadata_routing=True`,
+                which can be set by using
+                ``sklearn.set_config(enable_metadata_routing=True)``.
+                See :ref:`Metadata Routing User Guide ` for
+                more details.
+
         Returns
         -------
         y : ndarray of shape (n_samples, n_features)
             Array with prediction probabilities.
         """
         check_is_fitted(self)
-        return self.base_estimator_.predict_proba(X)
-
-    @if_delegate_has_method(delegate='base_estimator')
-    def decision_function(self, X):
-        """Calls decision function of the `base_estimator`.
+        _raise_for_params(params, self, "predict_proba")
+
+        if _routing_enabled():
+            # metadata routing is enabled.
+            routed_params = process_routing(self, "predict_proba", **params)
+        else:
+            routed_params = Bunch(estimator=Bunch(predict_proba={}))
+
+        X = validate_data(
+            self,
+            X,
+            accept_sparse=True,
+            ensure_all_finite=False,
+            reset=False,
+        )
+        return self.estimator_.predict_proba(X, **routed_params.estimator.predict_proba)
+
+    @available_if(_estimator_has("decision_function"))
+    def decision_function(self, X, **params):
+        """Call decision function of the `estimator`.
 
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
             Array representing the data.
 
+        **params : dict of str -> object
+            Parameters to pass to the underlying estimator's
+            ``decision_function`` method.
+
+            .. versionadded:: 1.6
+                Only available if `enable_metadata_routing=True`,
+                which can be set by using
+                ``sklearn.set_config(enable_metadata_routing=True)``.
+                See :ref:`Metadata Routing User Guide ` for
+                more details.
+
         Returns
         -------
         y : ndarray of shape (n_samples, n_features)
-            Result of the decision function of the `base_estimator`.
+            Result of the decision function of the `estimator`.
         """
         check_is_fitted(self)
-        return self.base_estimator_.decision_function(X)
-
-    @if_delegate_has_method(delegate='base_estimator')
-    def predict_log_proba(self, X):
+        _raise_for_params(params, self, "decision_function")
+
+        if _routing_enabled():
+            # metadata routing is enabled.
+            routed_params = process_routing(self, "decision_function", **params)
+        else:
+            routed_params = Bunch(estimator=Bunch(decision_function={}))
+
+        X = validate_data(
+            self,
+            X,
+            accept_sparse=True,
+            ensure_all_finite=False,
+            reset=False,
+        )
+        return self.estimator_.decision_function(
+            X, **routed_params.estimator.decision_function
+        )
+
+    @available_if(_estimator_has("predict_log_proba"))
+    def predict_log_proba(self, X, **params):
         """Predict log probability for each possible outcome.
 
         Parameters
@@ -315,17 +506,45 @@ def predict_log_proba(self, X):
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
             Array representing the data.
 
+        **params : dict of str -> object
+            Parameters to pass to the underlying estimator's
+            ``predict_log_proba`` method.
+
+            .. versionadded:: 1.6
+                Only available if `enable_metadata_routing=True`,
+                which can be set by using
+                ``sklearn.set_config(enable_metadata_routing=True)``.
+                See :ref:`Metadata Routing User Guide ` for
+                more details.
+
         Returns
         -------
         y : ndarray of shape (n_samples, n_features)
             Array with log prediction probabilities.
         """
         check_is_fitted(self)
-        return self.base_estimator_.predict_log_proba(X)
-
-    @if_delegate_has_method(delegate='base_estimator')
-    def score(self, X, y):
-        """Calls score on the `base_estimator`.
+        _raise_for_params(params, self, "predict_log_proba")
+
+        if _routing_enabled():
+            # metadata routing is enabled.
+            routed_params = process_routing(self, "predict_log_proba", **params)
+        else:
+            routed_params = Bunch(estimator=Bunch(predict_log_proba={}))
+
+        X = validate_data(
+            self,
+            X,
+            accept_sparse=True,
+            ensure_all_finite=False,
+            reset=False,
+        )
+        return self.estimator_.predict_log_proba(
+            X, **routed_params.estimator.predict_log_proba
+        )
+
+    @available_if(_estimator_has("score"))
+    def score(self, X, y, **params):
+        """Call score on the `estimator`.
 
         Parameters
         ----------
@@ -335,10 +554,72 @@ def score(self, X, y):
         y : array-like of shape (n_samples,)
             Array representing the labels.
 
+        **params : dict of str -> object
+            Parameters to pass to the underlying estimator's ``score`` method.
+
+            .. versionadded:: 1.6
+                Only available if `enable_metadata_routing=True`,
+                which can be set by using
+                ``sklearn.set_config(enable_metadata_routing=True)``.
+                See :ref:`Metadata Routing User Guide ` for
+                more details.
+
         Returns
         -------
         score : float
-            Result of calling score on the `base_estimator`.
+            Result of calling score on the `estimator`.
         """
         check_is_fitted(self)
-        return self.base_estimator_.score(X, y)
+        _raise_for_params(params, self, "score")
+
+        if _routing_enabled():
+            # metadata routing is enabled.
+            routed_params = process_routing(self, "score", **params)
+        else:
+            routed_params = Bunch(estimator=Bunch(score={}))
+
+        X = validate_data(
+            self,
+            X,
+            accept_sparse=True,
+            ensure_all_finite=False,
+            reset=False,
+        )
+        return self.estimator_.score(X, y, **routed_params.estimator.score)
+
+    def get_metadata_routing(self):
+        """Get metadata routing of this object.
+
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
+
+        .. versionadded:: 1.6
+
+        Returns
+        -------
+        routing : MetadataRouter
+            A :class:`~sklearn.utils.metadata_routing.MetadataRouter` encapsulating
+            routing information.
+        """
+        router = MetadataRouter(owner=self.__class__.__name__)
+        router.add(
+            estimator=self.estimator,
+            method_mapping=(
+                MethodMapping()
+                .add(callee="fit", caller="fit")
+                .add(callee="score", caller="fit")
+                .add(callee="predict", caller="predict")
+                .add(callee="predict_proba", caller="predict_proba")
+                .add(callee="decision_function", caller="decision_function")
+                .add(callee="predict_log_proba", caller="predict_log_proba")
+                .add(callee="score", caller="score")
+            ),
+        )
+        return router
+
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        # TODO(1.8): remove the condition check together with base_estimator
+        if self.estimator is not None:
+            tags.input_tags.sparse = get_tags(self.estimator).input_tags.sparse
+        return tags
diff --git a/sklearn/semi_supervised/tests/test_label_propagation.py b/sklearn/semi_supervised/tests/test_label_propagation.py
index 9f355281d9881..4b046aa111250 100644
--- a/sklearn/semi_supervised/tests/test_label_propagation.py
+++ b/sklearn/semi_supervised/tests/test_label_propagation.py
@@ -1,109 +1,120 @@
-""" test the label propagation module """
+"""test the label propagation module"""
+
+import warnings
 
 import numpy as np
 import pytest
-
 from scipy.sparse import issparse
-from sklearn.semi_supervised import _label_propagation as label_propagation
+
+from sklearn.datasets import make_classification
+from sklearn.exceptions import ConvergenceWarning
 from sklearn.metrics.pairwise import rbf_kernel
 from sklearn.model_selection import train_test_split
 from sklearn.neighbors import NearestNeighbors
-from sklearn.datasets import make_classification
-from sklearn.exceptions import ConvergenceWarning
-from numpy.testing import assert_array_almost_equal
-from numpy.testing import assert_array_equal
+from sklearn.semi_supervised import _label_propagation as label_propagation
+from sklearn.utils._testing import (
+    _convert_container,
+    assert_allclose,
+    assert_array_equal,
+)
+
+CONSTRUCTOR_TYPES = ("array", "sparse_csr", "sparse_csc")
 
 ESTIMATORS = [
-    (label_propagation.LabelPropagation, {'kernel': 'rbf'}),
-    (label_propagation.LabelPropagation, {'kernel': 'knn', 'n_neighbors': 2}),
-    (label_propagation.LabelPropagation, {
-        'kernel': lambda x, y: rbf_kernel(x, y, gamma=20)
-    }),
-    (label_propagation.LabelSpreading, {'kernel': 'rbf'}),
-    (label_propagation.LabelSpreading, {'kernel': 'knn', 'n_neighbors': 2}),
-    (label_propagation.LabelSpreading, {
-        'kernel': lambda x, y: rbf_kernel(x, y, gamma=20)
-    }),
+    (label_propagation.LabelPropagation, {"kernel": "rbf"}),
+    (label_propagation.LabelPropagation, {"kernel": "knn", "n_neighbors": 2}),
+    (
+        label_propagation.LabelPropagation,
+        {"kernel": lambda x, y: rbf_kernel(x, y, gamma=20)},
+    ),
+    (label_propagation.LabelSpreading, {"kernel": "rbf"}),
+    (label_propagation.LabelSpreading, {"kernel": "knn", "n_neighbors": 2}),
+    (
+        label_propagation.LabelSpreading,
+        {"kernel": lambda x, y: rbf_kernel(x, y, gamma=20)},
+    ),
 ]
 
 
-def test_fit_transduction():
-    samples = [[1., 0.], [0., 2.], [1., 3.]]
+@pytest.mark.parametrize("Estimator, parameters", ESTIMATORS)
+def test_fit_transduction(global_dtype, Estimator, parameters):
+    samples = np.asarray([[1.0, 0.0], [0.0, 2.0], [1.0, 3.0]], dtype=global_dtype)
     labels = [0, 1, -1]
-    for estimator, parameters in ESTIMATORS:
-        clf = estimator(**parameters).fit(samples, labels)
-        assert clf.transduction_[2] == 1
+    clf = Estimator(**parameters).fit(samples, labels)
+    assert clf.transduction_[2] == 1
 
 
-def test_distribution():
-    samples = [[1., 0.], [0., 1.], [1., 1.]]
+@pytest.mark.parametrize("Estimator, parameters", ESTIMATORS)
+def test_distribution(global_dtype, Estimator, parameters):
+    if parameters["kernel"] == "knn":
+        pytest.skip(
+            "Unstable test for this configuration: changes in k-NN ordering break it."
+        )
+    samples = np.asarray([[1.0, 0.0], [0.0, 1.0], [1.0, 1.0]], dtype=global_dtype)
     labels = [0, 1, -1]
-    for estimator, parameters in ESTIMATORS:
-        clf = estimator(**parameters).fit(samples, labels)
-        if parameters['kernel'] == 'knn':
-            continue    # unstable test; changes in k-NN ordering break it
-            assert_array_almost_equal(clf.predict_proba([[1., 0.0]]),
-                                      np.array([[1., 0.]]), 2)
-        else:
-            assert_array_almost_equal(np.asarray(clf.label_distributions_[2]),
-                                      np.array([.5, .5]), 2)
-
-
-def test_predict():
-    samples = [[1., 0.], [0., 2.], [1., 3.]]
+    clf = Estimator(**parameters).fit(samples, labels)
+    assert_allclose(clf.label_distributions_[2], [0.5, 0.5], atol=1e-2)
+
+
+@pytest.mark.parametrize("Estimator, parameters", ESTIMATORS)
+def test_predict(global_dtype, Estimator, parameters):
+    samples = np.asarray([[1.0, 0.0], [0.0, 2.0], [1.0, 3.0]], dtype=global_dtype)
     labels = [0, 1, -1]
-    for estimator, parameters in ESTIMATORS:
-        clf = estimator(**parameters).fit(samples, labels)
-        assert_array_equal(clf.predict([[0.5, 2.5]]), np.array([1]))
+    clf = Estimator(**parameters).fit(samples, labels)
+    assert_array_equal(clf.predict([[0.5, 2.5]]), np.array([1]))
 
 
-def test_predict_proba():
-    samples = [[1., 0.], [0., 1.], [1., 2.5]]
+@pytest.mark.parametrize("Estimator, parameters", ESTIMATORS)
+def test_predict_proba(global_dtype, Estimator, parameters):
+    samples = np.asarray([[1.0, 0.0], [0.0, 1.0], [1.0, 2.5]], dtype=global_dtype)
     labels = [0, 1, -1]
-    for estimator, parameters in ESTIMATORS:
-        clf = estimator(**parameters).fit(samples, labels)
-        assert_array_almost_equal(clf.predict_proba([[1., 1.]]),
-                                  np.array([[0.5, 0.5]]))
+    clf = Estimator(**parameters).fit(samples, labels)
+    assert_allclose(clf.predict_proba([[1.0, 1.0]]), np.array([[0.5, 0.5]]))
 
 
-def test_label_spreading_closed_form():
+@pytest.mark.parametrize("alpha", [0.1, 0.3, 0.5, 0.7, 0.9])
+@pytest.mark.parametrize("Estimator, parameters", ESTIMATORS)
+def test_label_spreading_closed_form(global_dtype, Estimator, parameters, alpha):
     n_classes = 2
-    X, y = make_classification(n_classes=n_classes, n_samples=200,
-                               random_state=0)
+    X, y = make_classification(n_classes=n_classes, n_samples=200, random_state=0)
+    X = X.astype(global_dtype, copy=False)
     y[::3] = -1
-    clf = label_propagation.LabelSpreading().fit(X, y)
+
+    gamma = 0.1
+    clf = label_propagation.LabelSpreading(gamma=gamma).fit(X, y)
     # adopting notation from Zhou et al (2004):
     S = clf._build_graph()
-    Y = np.zeros((len(y), n_classes + 1))
+    Y = np.zeros((len(y), n_classes + 1), dtype=X.dtype)
     Y[np.arange(len(y)), y] = 1
     Y = Y[:, :-1]
-    for alpha in [0.1, 0.3, 0.5, 0.7, 0.9]:
-        expected = np.dot(np.linalg.inv(np.eye(len(S)) - alpha * S), Y)
-        expected /= expected.sum(axis=1)[:, np.newaxis]
-        clf = label_propagation.LabelSpreading(max_iter=10000, alpha=alpha)
-        clf.fit(X, y)
-        assert_array_almost_equal(expected, clf.label_distributions_, 4)
 
+    expected = np.dot(np.linalg.inv(np.eye(len(S), dtype=S.dtype) - alpha * S), Y)
+    expected /= expected.sum(axis=1)[:, np.newaxis]
+
+    clf = label_propagation.LabelSpreading(
+        max_iter=100, alpha=alpha, tol=1e-10, gamma=gamma
+    )
+    clf.fit(X, y)
+
+    assert_allclose(expected, clf.label_distributions_)
 
-def test_label_propagation_closed_form():
+
+def test_label_propagation_closed_form(global_dtype):
     n_classes = 2
-    X, y = make_classification(n_classes=n_classes, n_samples=200,
-                               random_state=0)
+    X, y = make_classification(n_classes=n_classes, n_samples=200, random_state=0)
+    X = X.astype(global_dtype, copy=False)
     y[::3] = -1
     Y = np.zeros((len(y), n_classes + 1))
     Y[np.arange(len(y)), y] = 1
     unlabelled_idx = Y[:, (-1,)].nonzero()[0]
     labelled_idx = (Y[:, (-1,)] == 0).nonzero()[0]
 
-    clf = label_propagation.LabelPropagation(max_iter=10000,
-                                             gamma=0.1)
+    clf = label_propagation.LabelPropagation(max_iter=100, tol=1e-10, gamma=0.1)
     clf.fit(X, y)
     # adopting notation from Zhu et al 2002
     T_bar = clf._build_graph()
-    Tuu = T_bar[tuple(np.meshgrid(unlabelled_idx, unlabelled_idx,
-                      indexing='ij'))]
-    Tul = T_bar[tuple(np.meshgrid(unlabelled_idx, labelled_idx,
-                                  indexing='ij'))]
+    Tuu = T_bar[tuple(np.meshgrid(unlabelled_idx, unlabelled_idx, indexing="ij"))]
+    Tul = T_bar[tuple(np.meshgrid(unlabelled_idx, labelled_idx, indexing="ij"))]
     Y = Y[:, :-1]
     Y_l = Y[labelled_idx, :]
     Y_u = np.dot(np.dot(np.linalg.inv(np.eye(Tuu.shape[0]) - Tuu), Tul), Y_l)
@@ -112,23 +123,32 @@ def test_label_propagation_closed_form():
     expected[unlabelled_idx, :] = Y_u
     expected /= expected.sum(axis=1)[:, np.newaxis]
 
-    assert_array_almost_equal(expected, clf.label_distributions_, 4)
-
-
-def test_valid_alpha():
-    n_classes = 2
-    X, y = make_classification(n_classes=n_classes, n_samples=200,
-                               random_state=0)
-    for alpha in [-0.1, 0, 1, 1.1, None]:
-        with pytest.raises(ValueError):
-            label_propagation.LabelSpreading(alpha=alpha).fit(X, y)
+    assert_allclose(expected, clf.label_distributions_, atol=1e-4)
+
+
+@pytest.mark.parametrize("accepted_sparse_type", ["sparse_csr", "sparse_csc"])
+@pytest.mark.parametrize("index_dtype", [np.int32, np.int64])
+@pytest.mark.parametrize("dtype", [np.float32, np.float64])
+@pytest.mark.parametrize("Estimator, parameters", ESTIMATORS)
+def test_sparse_input_types(
+    accepted_sparse_type, index_dtype, dtype, Estimator, parameters
+):
+    # This is non-regression test for #17085
+    X = _convert_container([[1.0, 0.0], [0.0, 2.0], [1.0, 3.0]], accepted_sparse_type)
+    X.data = X.data.astype(dtype, copy=False)
+    X.indices = X.indices.astype(index_dtype, copy=False)
+    X.indptr = X.indptr.astype(index_dtype, copy=False)
+    labels = [0, 1, -1]
+    clf = Estimator(**parameters).fit(X, labels)
+    assert_array_equal(clf.predict([[0.5, 2.5]]), np.array([1]))
 
 
-def test_convergence_speed():
+@pytest.mark.parametrize("constructor_type", CONSTRUCTOR_TYPES)
+def test_convergence_speed(constructor_type):
     # This is a non-regression test for #5774
-    X = np.array([[1., 0.], [0., 1.], [1., 2.5]])
+    X = _convert_container([[1.0, 0.0], [0.0, 1.0], [1.0, 2.5]], constructor_type)
     y = np.array([0, 1, -1])
-    mdl = label_propagation.LabelSpreading(kernel='rbf', max_iter=5000)
+    mdl = label_propagation.LabelSpreading(kernel="rbf", max_iter=5000)
     mdl.fit(X, y)
 
     # this should converge quickly:
@@ -138,56 +158,55 @@ def test_convergence_speed():
 
 def test_convergence_warning():
     # This is a non-regression test for #5774
-    X = np.array([[1., 0.], [0., 1.], [1., 2.5]])
+    X = np.array([[1.0, 0.0], [0.0, 1.0], [1.0, 2.5]])
     y = np.array([0, 1, -1])
-    mdl = label_propagation.LabelSpreading(kernel='rbf', max_iter=1)
-    warn_msg = ('max_iter=1 was reached without convergence.')
+    mdl = label_propagation.LabelSpreading(kernel="rbf", max_iter=1)
+    warn_msg = "max_iter=1 was reached without convergence."
     with pytest.warns(ConvergenceWarning, match=warn_msg):
         mdl.fit(X, y)
     assert mdl.n_iter_ == mdl.max_iter
 
-    mdl = label_propagation.LabelPropagation(kernel='rbf', max_iter=1)
+    mdl = label_propagation.LabelPropagation(kernel="rbf", max_iter=1)
     with pytest.warns(ConvergenceWarning, match=warn_msg):
         mdl.fit(X, y)
     assert mdl.n_iter_ == mdl.max_iter
 
-    mdl = label_propagation.LabelSpreading(kernel='rbf', max_iter=500)
-    with pytest.warns(None) as record:
+    mdl = label_propagation.LabelSpreading(kernel="rbf", max_iter=500)
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", ConvergenceWarning)
         mdl.fit(X, y)
-    assert len(record) == 0
 
-    mdl = label_propagation.LabelPropagation(kernel='rbf', max_iter=500)
-    with pytest.warns(None) as record:
+    mdl = label_propagation.LabelPropagation(kernel="rbf", max_iter=500)
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", ConvergenceWarning)
         mdl.fit(X, y)
-    assert len(record) == 0
 
 
-@pytest.mark.parametrize("LabelPropagationCls",
-                         [label_propagation.LabelSpreading,
-                          label_propagation.LabelPropagation])
+@pytest.mark.parametrize(
+    "LabelPropagationCls",
+    [label_propagation.LabelSpreading, label_propagation.LabelPropagation],
+)
 def test_label_propagation_non_zero_normalizer(LabelPropagationCls):
     # check that we don't divide by zero in case of null normalizer
     # non-regression test for
     # https://github.com/scikit-learn/scikit-learn/pull/15946
     # https://github.com/scikit-learn/scikit-learn/issues/9292
-    X = np.array([[100., 100.], [100., 100.], [0., 0.], [0., 0.]])
+    X = np.array([[100.0, 100.0], [100.0, 100.0], [0.0, 0.0], [0.0, 0.0]])
     y = np.array([0, 1, -1, -1])
-    mdl = LabelPropagationCls(kernel='knn',
-                              max_iter=100,
-                              n_neighbors=1)
-    with pytest.warns(None) as record:
+    mdl = LabelPropagationCls(kernel="knn", max_iter=100, n_neighbors=1)
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", RuntimeWarning)
         mdl.fit(X, y)
-    assert len(record) == 0
 
 
-def test_predict_sparse_callable_kernel():
+def test_predict_sparse_callable_kernel(global_dtype):
     # This is a non-regression test for #15866
 
     # Custom sparse kernel (top-K RBF)
     def topk_rbf(X, Y=None, n_neighbors=10, gamma=1e-5):
-        nn = NearestNeighbors(n_neighbors=10, metric='euclidean', n_jobs=-1)
+        nn = NearestNeighbors(n_neighbors=10, metric="euclidean", n_jobs=2)
         nn.fit(X)
-        W = -1 * nn.kneighbors_graph(Y, mode='distance').power(2) * gamma
+        W = -1 * nn.kneighbors_graph(Y, mode="distance").power(2) * gamma
         np.exp(W.data, out=W.data)
         assert issparse(W)
         return W.T
@@ -195,17 +214,20 @@ def topk_rbf(X, Y=None, n_neighbors=10, gamma=1e-5):
     n_classes = 4
     n_samples = 500
     n_test = 10
-    X, y = make_classification(n_classes=n_classes,
-                               n_samples=n_samples,
-                               n_features=20,
-                               n_informative=20,
-                               n_redundant=0,
-                               n_repeated=0,
-                               random_state=0)
-
-    X_train, X_test, y_train, y_test = train_test_split(X, y,
-                                                        test_size=n_test,
-                                                        random_state=0)
+    X, y = make_classification(
+        n_classes=n_classes,
+        n_samples=n_samples,
+        n_features=20,
+        n_informative=20,
+        n_redundant=0,
+        n_repeated=0,
+        random_state=0,
+    )
+    X = X.astype(global_dtype)
+
+    X_train, X_test, y_train, y_test = train_test_split(
+        X, y, test_size=n_test, random_state=0
+    )
 
     model = label_propagation.LabelSpreading(kernel=topk_rbf)
     model.fit(X_train, y_train)
diff --git a/sklearn/semi_supervised/tests/test_self_training.py b/sklearn/semi_supervised/tests/test_self_training.py
index 7c5287be9974c..02244063994d5 100644
--- a/sklearn/semi_supervised/tests/test_self_training.py
+++ b/sklearn/semi_supervised/tests/test_self_training.py
@@ -1,108 +1,69 @@
 from math import ceil
 
 import numpy as np
-from numpy.testing import assert_array_equal
 import pytest
+from numpy.testing import assert_array_equal
 
+from sklearn.datasets import load_iris, make_blobs
 from sklearn.ensemble import StackingClassifier
 from sklearn.exceptions import NotFittedError
-from sklearn.neighbors import KNeighborsClassifier
-from sklearn.svm import SVC
-from sklearn.model_selection import train_test_split
-from sklearn.datasets import load_iris, make_blobs
 from sklearn.metrics import accuracy_score
-
+from sklearn.model_selection import train_test_split
+from sklearn.neighbors import KNeighborsClassifier
 from sklearn.semi_supervised import SelfTrainingClassifier
+from sklearn.svm import SVC
+from sklearn.tests.test_pipeline import SimpleEstimator
+from sklearn.tree import DecisionTreeClassifier
 
-# Author: Oliver Rausch 
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 # load the iris dataset and randomly permute it
 iris = load_iris()
-X_train, X_test, y_train, y_test = train_test_split(iris.data,
-                                                    iris.target,
-                                                    random_state=0)
+X_train, X_test, y_train, y_test = train_test_split(
+    iris.data, iris.target, random_state=0
+)
 
 n_labeled_samples = 50
 
 y_train_missing_labels = y_train.copy()
 y_train_missing_labels[n_labeled_samples:] = -1
-mapping = {0: 'A', 1: 'B', 2: 'C', -1: '-1'}
-y_train_missing_strings = np.vectorize(mapping.get)(
-    y_train_missing_labels).astype(object)
+mapping = {0: "A", 1: "B", 2: "C", -1: "-1"}
+y_train_missing_strings = np.vectorize(mapping.get)(y_train_missing_labels).astype(
+    object
+)
 y_train_missing_strings[y_train_missing_labels == -1] = -1
 
 
-def test_missing_predict_proba():
-    # Check that an error is thrown if predict_proba is not implemented
-    base_estimator = SVC(probability=False, gamma='scale')
-    self_training = SelfTrainingClassifier(base_estimator)
-
-    with pytest.raises(ValueError, match=r"base_estimator \(SVC\) should"):
-        self_training.fit(X_train, y_train_missing_labels)
-
-
-def test_none_classifier():
-    st = SelfTrainingClassifier(None)
-    with pytest.raises(ValueError, match="base_estimator cannot be None"):
-        st.fit(X_train, y_train_missing_labels)
-
-
-@pytest.mark.parametrize("max_iter, threshold",
-                         [(-1, 1.0), (-100, -2), (-10, 10)])
-def test_invalid_params(max_iter, threshold):
-    # Test negative iterations
-    base_estimator = SVC(gamma="scale", probability=True)
-    st = SelfTrainingClassifier(base_estimator, max_iter=max_iter)
-    with pytest.raises(ValueError, match="max_iter must be >= 0 or None"):
-        st.fit(X_train, y_train)
-
-    base_estimator = SVC(gamma="scale", probability=True)
-    st = SelfTrainingClassifier(base_estimator, threshold=threshold)
-    with pytest.raises(ValueError, match="threshold must be in"):
-        st.fit(X_train, y_train)
-
-
-def test_invalid_params_selection_crit():
-    st = SelfTrainingClassifier(KNeighborsClassifier(),
-                                criterion='foo')
-
-    with pytest.raises(ValueError, match="criterion must be either"):
-        st.fit(X_train, y_train)
-
-
 def test_warns_k_best():
-    st = SelfTrainingClassifier(KNeighborsClassifier(),
-                                criterion='k_best',
-                                k_best=1000)
+    st = SelfTrainingClassifier(KNeighborsClassifier(), criterion="k_best", k_best=1000)
     with pytest.warns(UserWarning, match="k_best is larger than"):
         st.fit(X_train, y_train_missing_labels)
 
-    assert st.termination_condition_ == 'all_labeled'
+    assert st.termination_condition_ == "all_labeled"
 
 
-@pytest.mark.parametrize("base_estimator",
-                         [KNeighborsClassifier(),
-                          SVC(gamma="scale", probability=True,
-                              random_state=0)])
-@pytest.mark.parametrize("selection_crit",
-                         ['threshold', 'k_best'])
-def test_classification(base_estimator, selection_crit):
+@pytest.mark.parametrize(
+    "estimator",
+    [KNeighborsClassifier(), SVC(gamma="scale", probability=True, random_state=0)],
+)
+@pytest.mark.parametrize("selection_crit", ["threshold", "k_best"])
+def test_classification(estimator, selection_crit):
     # Check classification for various parameter settings.
     # Also assert that predictions for strings and numerical labels are equal.
     # Also test for multioutput classification
     threshold = 0.75
     max_iter = 10
-    st = SelfTrainingClassifier(base_estimator, max_iter=max_iter,
-                                threshold=threshold,
-                                criterion=selection_crit)
+    st = SelfTrainingClassifier(
+        estimator, max_iter=max_iter, threshold=threshold, criterion=selection_crit
+    )
     st.fit(X_train, y_train_missing_labels)
     pred = st.predict(X_test)
     proba = st.predict_proba(X_test)
 
-    st_string = SelfTrainingClassifier(base_estimator, max_iter=max_iter,
-                                       criterion=selection_crit,
-                                       threshold=threshold)
+    st_string = SelfTrainingClassifier(
+        estimator, max_iter=max_iter, criterion=selection_crit, threshold=threshold
+    )
     st_string.fit(X_train, y_train_missing_strings)
     pred_string = st_string.predict(X_test)
     proba_string = st_string.predict_proba(X_test)
@@ -116,8 +77,7 @@ def test_classification(base_estimator, selection_crit):
     # assert that labeled samples have labeled_iter = 0
     assert_array_equal(st.labeled_iter_ == 0, labeled)
     # assert that labeled samples do not change label during training
-    assert_array_equal(y_train_missing_labels[labeled],
-                       st.transduction_[labeled])
+    assert_array_equal(y_train_missing_labels[labeled], st.transduction_[labeled])
 
     # assert that the max of the iterations is less than the total amount of
     # iterations
@@ -130,10 +90,12 @@ def test_classification(base_estimator, selection_crit):
 
 
 def test_k_best():
-    st = SelfTrainingClassifier(KNeighborsClassifier(n_neighbors=1),
-                                criterion='k_best',
-                                k_best=10,
-                                max_iter=None)
+    st = SelfTrainingClassifier(
+        KNeighborsClassifier(n_neighbors=1),
+        criterion="k_best",
+        k_best=10,
+        max_iter=None,
+    )
     y_train_only_one_label = np.copy(y_train)
     y_train_only_one_label[1:] = -1
     n_samples = y_train.shape[0]
@@ -147,20 +109,19 @@ def test_k_best():
     for i in range(1, n_expected_iter):
         assert np.sum(st.labeled_iter_ == i) == 10
     assert np.sum(st.labeled_iter_ == n_expected_iter) == (n_samples - 1) % 10
-    assert st.termination_condition_ == 'all_labeled'
+    assert st.termination_condition_ == "all_labeled"
 
 
 def test_sanity_classification():
-    base_estimator = SVC(gamma="scale", probability=True)
-    base_estimator.fit(X_train[n_labeled_samples:],
-                       y_train[n_labeled_samples:])
+    estimator = SVC(gamma="scale", probability=True)
+    estimator.fit(X_train[n_labeled_samples:], y_train[n_labeled_samples:])
 
-    st = SelfTrainingClassifier(base_estimator)
+    st = SelfTrainingClassifier(estimator)
     st.fit(X_train, y_train_missing_labels)
 
-    pred1, pred2 = base_estimator.predict(X_test), st.predict(X_test)
+    pred1, pred2 = estimator.predict(X_test), st.predict(X_test)
     assert not np.array_equal(pred1, pred2)
-    score_supervised = accuracy_score(base_estimator.predict(X_test), y_test)
+    score_supervised = accuracy_score(estimator.predict(X_test), y_test)
     score_self_training = accuracy_score(st.predict(X_test), y_test)
 
     assert score_self_training > score_supervised
@@ -169,32 +130,29 @@ def test_sanity_classification():
 def test_none_iter():
     # Check that the all samples were labeled after a 'reasonable' number of
     # iterations.
-    st = SelfTrainingClassifier(KNeighborsClassifier(), threshold=.55,
-                                max_iter=None)
+    st = SelfTrainingClassifier(KNeighborsClassifier(), threshold=0.55, max_iter=None)
     st.fit(X_train, y_train_missing_labels)
 
     assert st.n_iter_ < 10
     assert st.termination_condition_ == "all_labeled"
 
 
-@pytest.mark.parametrize("base_estimator",
-                         [KNeighborsClassifier(),
-                          SVC(gamma="scale", probability=True,
-                              random_state=0)])
-@pytest.mark.parametrize("y", [y_train_missing_labels,
-                               y_train_missing_strings])
-def test_zero_iterations(base_estimator, y):
+@pytest.mark.parametrize(
+    "estimator",
+    [KNeighborsClassifier(), SVC(gamma="scale", probability=True, random_state=0)],
+)
+@pytest.mark.parametrize("y", [y_train_missing_labels, y_train_missing_strings])
+def test_zero_iterations(estimator, y):
     # Check classification for zero iterations.
     # Fitting a SelfTrainingClassifier with zero iterations should give the
     # same results as fitting a supervised classifier.
     # This also asserts that string arrays work as expected.
 
-    clf1 = SelfTrainingClassifier(base_estimator, max_iter=0)
+    clf1 = SelfTrainingClassifier(estimator, max_iter=0)
 
     clf1.fit(X_train, y)
 
-    clf2 = base_estimator.fit(X_train[:n_labeled_samples],
-                              y[:n_labeled_samples])
+    clf2 = estimator.fit(X_train[:n_labeled_samples], y[:n_labeled_samples])
 
     assert_array_equal(clf1.predict(X_test), clf2.predict(X_test))
     assert clf1.termination_condition_ == "max_iter"
@@ -206,8 +164,10 @@ def test_prefitted_throws_error():
     knn = KNeighborsClassifier()
     knn.fit(X_train, y_train)
     st = SelfTrainingClassifier(knn)
-    with pytest.raises(NotFittedError, match="This SelfTrainingClassifier"
-                       " instance is not fitted yet"):
+    with pytest.raises(
+        NotFittedError,
+        match="This SelfTrainingClassifier instance is not fitted yet",
+    ):
         st.predict(X_train)
 
 
@@ -241,7 +201,7 @@ def test_no_unlabeled():
 
 
 def test_early_stopping():
-    svc = SVC(gamma='scale', probability=True)
+    svc = SVC(gamma="scale", probability=True)
     st = SelfTrainingClassifier(svc)
     X_train_easy = [[1], [0], [1], [0.5]]
     y_train_easy = [1, 0, -1, -1]
@@ -249,13 +209,12 @@ def test_early_stopping():
     # stops early
     st.fit(X_train_easy, y_train_easy)
     assert st.n_iter_ == 1
-    assert st.termination_condition_ == 'no_change'
+    assert st.termination_condition_ == "no_change"
 
 
 def test_strings_dtype():
     clf = SelfTrainingClassifier(KNeighborsClassifier())
-    X, y = make_blobs(n_samples=30, random_state=0,
-                      cluster_std=0.1)
+    X, y = make_blobs(n_samples=30, random_state=0, cluster_std=0.1)
     labels_multiclass = ["one", "two", "three"]
 
     y_strings = np.take(labels_multiclass, y)
@@ -272,16 +231,19 @@ def test_verbose(capsys, verbose):
     captured = capsys.readouterr()
 
     if verbose:
-        assert 'iteration' in captured.out
+        assert "iteration" in captured.out
     else:
-        assert 'iteration' not in captured.out
+        assert "iteration" not in captured.out
 
 
 def test_verbose_k_best(capsys):
-    st = SelfTrainingClassifier(KNeighborsClassifier(n_neighbors=1),
-                                criterion='k_best',
-                                k_best=10, verbose=True,
-                                max_iter=None)
+    st = SelfTrainingClassifier(
+        KNeighborsClassifier(n_neighbors=1),
+        criterion="k_best",
+        k_best=10,
+        verbose=True,
+        max_iter=None,
+    )
 
     y_train_only_one_label = np.copy(y_train)
     y_train_only_one_label[1:] = -1
@@ -292,20 +254,17 @@ def test_verbose_k_best(capsys):
 
     captured = capsys.readouterr()
 
-    msg = 'End of iteration {}, added {} new labels.'
+    msg = "End of iteration {}, added {} new labels."
     for i in range(1, n_expected_iter):
         assert msg.format(i, 10) in captured.out
 
-    assert msg.format(n_expected_iter,
-                      (n_samples - 1) % 10) in captured.out
+    assert msg.format(n_expected_iter, (n_samples - 1) % 10) in captured.out
 
 
 def test_k_best_selects_best():
     # Tests that the labels added by st really are the 10 best labels.
-    svc = SVC(gamma='scale', probability=True, random_state=0)
-    st = SelfTrainingClassifier(svc,
-                                criterion='k_best',
-                                max_iter=1, k_best=10)
+    svc = SVC(gamma="scale", probability=True, random_state=0)
+    st = SelfTrainingClassifier(svc, criterion="k_best", max_iter=1, k_best=10)
     has_label = y_train_missing_labels != -1
     st.fit(X_train, y_train_missing_labels)
 
@@ -322,24 +281,115 @@ def test_k_best_selects_best():
         assert row in added_by_st
 
 
-def test_base_estimator_meta_estimator():
+def test_estimator_meta_estimator():
     # Check that a meta-estimator relying on an estimator implementing
-    # `predict_proba` will work even if it does expose this method before being
+    # `predict_proba` will work even if it does not expose this method before being
     # fitted.
     # Non-regression test for:
     # https://github.com/scikit-learn/scikit-learn/issues/19119
 
-    base_estimator = StackingClassifier(
+    estimator = StackingClassifier(
         estimators=[
-            ("svc_1", SVC(probability=True)), ("svc_2", SVC(probability=True)),
+            ("svc_1", SVC(probability=True)),
+            ("svc_2", SVC(probability=True)),
         ],
-        final_estimator=SVC(probability=True), cv=2
+        final_estimator=SVC(probability=True),
+        cv=2,
     )
 
-    # make sure that the `base_estimator` does not expose `predict_proba`
-    # without being fitted
-    assert not hasattr(base_estimator, "predict_proba")
-
-    clf = SelfTrainingClassifier(base_estimator=base_estimator)
+    assert hasattr(estimator, "predict_proba")
+    clf = SelfTrainingClassifier(estimator=estimator)
     clf.fit(X_train, y_train_missing_labels)
     clf.predict_proba(X_test)
+
+    estimator = StackingClassifier(
+        estimators=[
+            ("svc_1", SVC(probability=False)),
+            ("svc_2", SVC(probability=False)),
+        ],
+        final_estimator=SVC(probability=False),
+        cv=2,
+    )
+
+    assert not hasattr(estimator, "predict_proba")
+    clf = SelfTrainingClassifier(estimator=estimator)
+    with pytest.raises(AttributeError):
+        clf.fit(X_train, y_train_missing_labels)
+
+
+def test_self_training_estimator_attribute_error():
+    """Check that we raise the proper AttributeErrors when the `estimator`
+    does not implement the `predict_proba` method, which is called from within
+    `fit`, or `decision_function`, which is decorated with `available_if`.
+
+    Non-regression test for:
+    https://github.com/scikit-learn/scikit-learn/issues/28108
+    """
+    # `SVC` with `probability=False` does not implement 'predict_proba' that
+    # is required internally in `fit` of `SelfTrainingClassifier`. We expect
+    # an AttributeError to be raised.
+    estimator = SVC(probability=False, gamma="scale")
+    self_training = SelfTrainingClassifier(estimator)
+
+    with pytest.raises(AttributeError, match="has no attribute 'predict_proba'"):
+        self_training.fit(X_train, y_train_missing_labels)
+
+    # `DecisionTreeClassifier` does not implement 'decision_function' and
+    # should raise an AttributeError
+    self_training = SelfTrainingClassifier(estimator=DecisionTreeClassifier())
+
+    outer_msg = "This 'SelfTrainingClassifier' has no attribute 'decision_function'"
+    inner_msg = "'DecisionTreeClassifier' object has no attribute 'decision_function'"
+    with pytest.raises(AttributeError, match=outer_msg) as exec_info:
+        self_training.fit(X_train, y_train_missing_labels).decision_function(X_train)
+    assert isinstance(exec_info.value.__cause__, AttributeError)
+    assert inner_msg in str(exec_info.value.__cause__)
+
+
+# TODO(1.8): remove in 1.8
+def test_deprecation_warning_base_estimator():
+    warn_msg = "`base_estimator` has been deprecated in 1.6 and will be removed"
+    with pytest.warns(FutureWarning, match=warn_msg):
+        SelfTrainingClassifier(base_estimator=DecisionTreeClassifier()).fit(
+            X_train, y_train_missing_labels
+        )
+
+    error_msg = "You must pass an estimator to SelfTrainingClassifier"
+    with pytest.raises(ValueError, match=error_msg):
+        SelfTrainingClassifier().fit(X_train, y_train_missing_labels)
+
+    error_msg = "You must pass only one estimator to SelfTrainingClassifier."
+    with pytest.raises(ValueError, match=error_msg):
+        SelfTrainingClassifier(
+            base_estimator=DecisionTreeClassifier(), estimator=DecisionTreeClassifier()
+        ).fit(X_train, y_train_missing_labels)
+
+
+# Metadata routing tests
+# =================================================================
+
+
+@pytest.mark.filterwarnings("ignore:y contains no unlabeled samples:UserWarning")
+@pytest.mark.parametrize(
+    "method", ["decision_function", "predict_log_proba", "predict_proba", "predict"]
+)
+def test_routing_passed_metadata_not_supported(method):
+    """Test that the right error message is raised when metadata is passed while
+    not supported when `enable_metadata_routing=False`."""
+    est = SelfTrainingClassifier(estimator=SimpleEstimator())
+    with pytest.raises(
+        ValueError, match="is only supported if enable_metadata_routing=True"
+    ):
+        est.fit([[1], [1]], [1, 1], sample_weight=[1], prop="a")
+
+    est = SelfTrainingClassifier(estimator=SimpleEstimator())
+    with pytest.raises(
+        ValueError, match="is only supported if enable_metadata_routing=True"
+    ):
+        # make sure that the estimator thinks it is already fitted
+        est.fitted_params_ = True
+        getattr(est, method)([[1]], sample_weight=[1], prop="a")
+
+
+# End of routing tests
+# ====================
diff --git a/sklearn/setup.py b/sklearn/setup.py
deleted file mode 100644
index e5d7e6e26b3ab..0000000000000
--- a/sklearn/setup.py
+++ /dev/null
@@ -1,90 +0,0 @@
-import sys
-import os
-
-from sklearn._build_utils import cythonize_extensions
-
-
-def configuration(parent_package='', top_path=None):
-    from numpy.distutils.misc_util import Configuration
-    import numpy
-
-    libraries = []
-    if os.name == 'posix':
-        libraries.append('m')
-
-    config = Configuration('sklearn', parent_package, top_path)
-
-    # submodules with build utilities
-    config.add_subpackage('__check_build')
-    config.add_subpackage('_build_utils')
-
-    # submodules which do not have their own setup.py
-    # we must manually add sub-submodules & tests
-    config.add_subpackage('compose')
-    config.add_subpackage('compose/tests')
-    config.add_subpackage('covariance')
-    config.add_subpackage('covariance/tests')
-    config.add_subpackage('cross_decomposition')
-    config.add_subpackage('cross_decomposition/tests')
-    config.add_subpackage('feature_selection')
-    config.add_subpackage('feature_selection/tests')
-    config.add_subpackage('gaussian_process')
-    config.add_subpackage('gaussian_process/tests')
-    config.add_subpackage('impute')
-    config.add_subpackage('impute/tests')
-    config.add_subpackage('inspection')
-    config.add_subpackage('inspection/tests')
-    config.add_subpackage('mixture')
-    config.add_subpackage('mixture/tests')
-    config.add_subpackage('model_selection')
-    config.add_subpackage('model_selection/tests')
-    config.add_subpackage('neural_network')
-    config.add_subpackage('neural_network/tests')
-    config.add_subpackage('preprocessing')
-    config.add_subpackage('preprocessing/tests')
-    config.add_subpackage('semi_supervised')
-    config.add_subpackage('semi_supervised/tests')
-    config.add_subpackage('experimental')
-    config.add_subpackage('experimental/tests')
-    config.add_subpackage('ensemble/_hist_gradient_boosting')
-    config.add_subpackage('ensemble/_hist_gradient_boosting/tests')
-    config.add_subpackage('_loss/')
-    config.add_subpackage('_loss/tests')
-    config.add_subpackage('externals')
-
-    # submodules which have their own setup.py
-    config.add_subpackage('cluster')
-    config.add_subpackage('datasets')
-    config.add_subpackage('decomposition')
-    config.add_subpackage('ensemble')
-    config.add_subpackage('feature_extraction')
-    config.add_subpackage('manifold')
-    config.add_subpackage('metrics')
-    config.add_subpackage('neighbors')
-    config.add_subpackage('tree')
-    config.add_subpackage('utils')
-    config.add_subpackage('svm')
-    config.add_subpackage('linear_model')
-
-    # add cython extension module for isotonic regression
-    config.add_extension('_isotonic',
-                         sources=['_isotonic.pyx'],
-                         include_dirs=[numpy.get_include()],
-                         libraries=libraries,
-                         )
-
-    # add the test directory
-    config.add_subpackage('tests')
-
-    # Skip cythonization as we do not want to include the generated
-    # C/C++ files in the release tarballs as they are not necessarily
-    # forward compatible with future versions of Python for instance.
-    if 'sdist' not in sys.argv:
-        cythonize_extensions(top_path, config)
-
-    return config
-
-
-if __name__ == '__main__':
-    from numpy.distutils.core import setup
-    setup(**configuration(top_path='').todict())
diff --git a/sklearn/svm/__init__.py b/sklearn/svm/__init__.py
index b80c8716137b9..a039d2e15abdd 100644
--- a/sklearn/svm/__init__.py
+++ b/sklearn/svm/__init__.py
@@ -1,24 +1,21 @@
-"""
-The :mod:`sklearn.svm` module includes Support Vector Machine algorithms.
-"""
+"""Support vector machine algorithms."""
 
 # See http://scikit-learn.sourceforge.net/modules/svm.html for complete
 # documentation.
 
-# Author: Fabian Pedregosa  with help from
-#         the scikit-learn community. LibSVM and LibLinear are copyright
-#         of their respective owners.
-# License: BSD 3 clause (C) INRIA 2010
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-from ._classes import SVC, NuSVC, SVR, NuSVR, OneClassSVM, LinearSVC, \
-        LinearSVR
 from ._bounds import l1_min_c
+from ._classes import SVC, SVR, LinearSVC, LinearSVR, NuSVC, NuSVR, OneClassSVM
 
-__all__ = ['LinearSVC',
-           'LinearSVR',
-           'NuSVC',
-           'NuSVR',
-           'OneClassSVM',
-           'SVC',
-           'SVR',
-           'l1_min_c']
+__all__ = [
+    "SVC",
+    "SVR",
+    "LinearSVC",
+    "LinearSVR",
+    "NuSVC",
+    "NuSVR",
+    "OneClassSVM",
+    "l1_min_c",
+]
diff --git a/sklearn/svm/_base.py b/sklearn/svm/_base.py
index 67808278cc59a..db295e4e877b5 100644
--- a/sklearn/svm/_base.py
+++ b/sklearn/svm/_base.py
@@ -1,30 +1,37 @@
-import numpy as np
-import scipy.sparse as sp
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
 import warnings
 from abc import ABCMeta, abstractmethod
+from numbers import Integral, Real
 
-# mypy error: error: Module 'sklearn.svm' has no attribute '_libsvm'
-# (and same for other imports)
-from . import _libsvm as libsvm  # type: ignore
-from .import _liblinear as liblinear  # type: ignore
-from . import _libsvm_sparse as libsvm_sparse  # type: ignore
-from ..base import BaseEstimator, ClassifierMixin
+import numpy as np
+import scipy.sparse as sp
+
+from ..base import BaseEstimator, ClassifierMixin, _fit_context
+from ..exceptions import ConvergenceWarning, NotFittedError
 from ..preprocessing import LabelEncoder
-from ..utils.multiclass import _ovr_decision_function
-from ..utils import check_array, check_random_state
-from ..utils import column_or_1d
-from ..utils import compute_class_weight
-from ..utils.deprecation import deprecated
+from ..utils import check_array, check_random_state, column_or_1d, compute_class_weight
+from ..utils._param_validation import Interval, StrOptions
 from ..utils.extmath import safe_sparse_dot
-from ..utils.validation import check_is_fitted, _check_large_sparse
-from ..utils.validation import _num_samples
-from ..utils.validation import _check_sample_weight, check_consistent_length
-from ..utils.multiclass import check_classification_targets
-from ..exceptions import ConvergenceWarning
-from ..exceptions import NotFittedError
+from ..utils.metaestimators import available_if
+from ..utils.multiclass import _ovr_decision_function, check_classification_targets
+from ..utils.validation import (
+    _check_large_sparse,
+    _check_sample_weight,
+    _num_samples,
+    check_consistent_length,
+    check_is_fitted,
+    validate_data,
+)
+from . import _liblinear as liblinear  # type: ignore[attr-defined]
 
+# mypy error: error: Module 'sklearn.svm' has no attribute '_libsvm'
+# (and same for other imports)
+from . import _libsvm as libsvm  # type: ignore[attr-defined]
+from . import _libsvm_sparse as libsvm_sparse  # type: ignore[attr-defined]
 
-LIBSVM_IMPL = ['c_svc', 'nu_svc', 'one_class', 'epsilon_svr', 'nu_svr']
+LIBSVM_IMPL = ["c_svc", "nu_svc", "one_class", "epsilon_svr", "nu_svr"]
 
 
 def _one_vs_one_coef(dual_coef, n_support, support_vectors):
@@ -44,19 +51,18 @@ def _one_vs_one_coef(dual_coef, n_support, support_vectors):
     sv_locs = np.cumsum(np.hstack([[0], n_support]))
     for class1 in range(n_class):
         # SVs for class1:
-        sv1 = support_vectors[sv_locs[class1]:sv_locs[class1 + 1], :]
+        sv1 = support_vectors[sv_locs[class1] : sv_locs[class1 + 1], :]
         for class2 in range(class1 + 1, n_class):
             # SVs for class1:
-            sv2 = support_vectors[sv_locs[class2]:sv_locs[class2 + 1], :]
+            sv2 = support_vectors[sv_locs[class2] : sv_locs[class2 + 1], :]
 
             # dual coef for class1 SVs:
-            alpha1 = dual_coef[class2 - 1, sv_locs[class1]:sv_locs[class1 + 1]]
+            alpha1 = dual_coef[class2 - 1, sv_locs[class1] : sv_locs[class1 + 1]]
             # dual coef for class2 SVs:
-            alpha2 = dual_coef[class1, sv_locs[class2]:sv_locs[class2 + 1]]
+            alpha2 = dual_coef[class1, sv_locs[class2] : sv_locs[class2 + 1]]
             # build weight for class1 vs class2
 
-            coef.append(safe_sparse_dot(alpha1, sv1)
-                        + safe_sparse_dot(alpha2, sv2))
+            coef.append(safe_sparse_dot(alpha1, sv1) + safe_sparse_dot(alpha2, sv2))
     return coef
 
 
@@ -68,24 +74,58 @@ class BaseLibSVM(BaseEstimator, metaclass=ABCMeta):
     Parameter documentation is in the derived `SVC` class.
     """
 
+    _parameter_constraints: dict = {
+        "kernel": [
+            StrOptions({"linear", "poly", "rbf", "sigmoid", "precomputed"}),
+            callable,
+        ],
+        "degree": [Interval(Integral, 0, None, closed="left")],
+        "gamma": [
+            StrOptions({"scale", "auto"}),
+            Interval(Real, 0.0, None, closed="left"),
+        ],
+        "coef0": [Interval(Real, None, None, closed="neither")],
+        "tol": [Interval(Real, 0.0, None, closed="neither")],
+        "C": [Interval(Real, 0.0, None, closed="right")],
+        "nu": [Interval(Real, 0.0, 1.0, closed="right")],
+        "epsilon": [Interval(Real, 0.0, None, closed="left")],
+        "shrinking": ["boolean"],
+        "probability": ["boolean"],
+        "cache_size": [Interval(Real, 0, None, closed="neither")],
+        "class_weight": [StrOptions({"balanced"}), dict, None],
+        "verbose": ["verbose"],
+        "max_iter": [Interval(Integral, -1, None, closed="left")],
+        "random_state": ["random_state"],
+    }
+
     # The order of these must match the integer values in LibSVM.
     # XXX These are actually the same in the dense case. Need to factor
     # this out.
     _sparse_kernels = ["linear", "poly", "rbf", "sigmoid", "precomputed"]
 
     @abstractmethod
-    def __init__(self, kernel, degree, gamma, coef0,
-                 tol, C, nu, epsilon, shrinking, probability, cache_size,
-                 class_weight, verbose, max_iter, random_state):
-
+    def __init__(
+        self,
+        kernel,
+        degree,
+        gamma,
+        coef0,
+        tol,
+        C,
+        nu,
+        epsilon,
+        shrinking,
+        probability,
+        cache_size,
+        class_weight,
+        verbose,
+        max_iter,
+        random_state,
+    ):
         if self._impl not in LIBSVM_IMPL:
-            raise ValueError("impl should be one of %s, %s was given" % (
-                LIBSVM_IMPL, self._impl))
-
-        if gamma == 0:
-            msg = ("The gamma value of 0.0 is invalid. Use 'auto' to set"
-                   " gamma to a value of 1 / n_features.")
-            raise ValueError(msg)
+            raise ValueError(
+                "impl should be one of %s, %s was given" % (LIBSVM_IMPL, self._impl)
+            )
 
         self.kernel = kernel
         self.degree = degree
@@ -103,19 +143,14 @@ def __init__(self, kernel, degree, gamma, coef0,
         self.max_iter = max_iter
         self.random_state = random_state
 
-    def _more_tags(self):
-        # Used by cross_val_score.
-        return {'pairwise': self.kernel == 'precomputed'}
-
-    # TODO: Remove in 1.1
-    # mypy error: Decorated property not supported
-    @deprecated("Attribute _pairwise was deprecated in "  # type: ignore
-                "version 0.24 and will be removed in 1.1 (renaming of 0.26).")
-    @property
-    def _pairwise(self):
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
         # Used by cross_val_score.
-        return self.kernel == "precomputed"
+        tags.input_tags.pairwise = self.kernel == "precomputed"
+        tags.input_tags.sparse = self.kernel != "precomputed"
+        return tags
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y, sample_weight=None):
         """Fit the SVM model according to the given training data.
 
@@ -123,8 +158,8 @@ def fit(self, X, y, sample_weight=None):
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features) \
                 or (n_samples, n_samples)
-            Training vectors, where n_samples is the number of samples
-            and n_features is the number of features.
+            Training vectors, where `n_samples` is the number of samples
+            and `n_features` is the number of features.
             For kernel="precomputed", the expected shape of X is
             (n_samples, n_samples).
 
@@ -139,6 +174,7 @@ def fit(self, X, y, sample_weight=None):
         Returns
         -------
         self : object
+            Fitted estimator.
 
         Notes
         -----
@@ -148,95 +184,112 @@ def fit(self, X, y, sample_weight=None):
         If X is a dense array, then the other methods will not support sparse
         matrices as input.
         """
-
         rnd = check_random_state(self.random_state)
 
-        sparse = sp.isspmatrix(X)
+        sparse = sp.issparse(X)
         if sparse and self.kernel == "precomputed":
             raise TypeError("Sparse precomputed kernels are not supported.")
         self._sparse = sparse and not callable(self.kernel)
 
-        if hasattr(self, 'decision_function_shape'):
-            if self.decision_function_shape not in ('ovr', 'ovo'):
-                raise ValueError(
-                    f"decision_function_shape must be either 'ovr' or 'ovo', "
-                    f"got {self.decision_function_shape}."
-                )
-
         if callable(self.kernel):
             check_consistent_length(X, y)
         else:
-            X, y = self._validate_data(X, y, dtype=np.float64,
-                                       order='C', accept_sparse='csr',
-                                       accept_large_sparse=False)
+            X, y = validate_data(
+                self,
+                X,
+                y,
+                dtype=np.float64,
+                order="C",
+                accept_sparse="csr",
+                accept_large_sparse=False,
+            )
 
         y = self._validate_targets(y)
 
-        sample_weight = np.asarray([]
-                                   if sample_weight is None
-                                   else sample_weight, dtype=np.float64)
+        sample_weight = np.asarray(
+            [] if sample_weight is None else sample_weight, dtype=np.float64
+        )
         solver_type = LIBSVM_IMPL.index(self._impl)
 
         # input validation
         n_samples = _num_samples(X)
         if solver_type != 2 and n_samples != y.shape[0]:
-            raise ValueError("X and y have incompatible shapes.\n" +
-                             "X has %s samples, but y has %s." %
-                             (n_samples, y.shape[0]))
+            raise ValueError(
+                "X and y have incompatible shapes.\n"
+                + "X has %s samples, but y has %s." % (n_samples, y.shape[0])
+            )
 
         if self.kernel == "precomputed" and n_samples != X.shape[1]:
-            raise ValueError("Precomputed matrix must be a square matrix."
-                             " Input is a {}x{} matrix."
-                             .format(X.shape[0], X.shape[1]))
+            raise ValueError(
+                "Precomputed matrix must be a square matrix."
+                " Input is a {}x{} matrix.".format(X.shape[0], X.shape[1])
+            )
 
         if sample_weight.shape[0] > 0 and sample_weight.shape[0] != n_samples:
-            raise ValueError("sample_weight and X have incompatible shapes: "
-                             "%r vs %r\n"
-                             "Note: Sparse matrices cannot be indexed w/"
-                             "boolean masks (use `indices=True` in CV)."
-                             % (sample_weight.shape, X.shape))
+            raise ValueError(
+                "sample_weight and X have incompatible shapes: "
+                "%r vs %r\n"
+                "Note: Sparse matrices cannot be indexed w/"
+                "boolean masks (use `indices=True` in CV)."
+                % (sample_weight.shape, X.shape)
+            )
 
-        kernel = 'precomputed' if callable(self.kernel) else self.kernel
+        kernel = "precomputed" if callable(self.kernel) else self.kernel
 
-        if kernel == 'precomputed':
+        if kernel == "precomputed":
             # unused but needs to be a float for cython code that ignores
             # it anyway
-            self._gamma = 0.
+            self._gamma = 0.0
         elif isinstance(self.gamma, str):
-            if self.gamma == 'scale':
+            if self.gamma == "scale":
                 # var = E[X^2] - E[X]^2 if sparse
-                X_var = ((X.multiply(X)).mean() - (X.mean()) ** 2
-                         if sparse else X.var())
+                X_var = (X.multiply(X)).mean() - (X.mean()) ** 2 if sparse else X.var()
                 self._gamma = 1.0 / (X.shape[1] * X_var) if X_var != 0 else 1.0
-            elif self.gamma == 'auto':
+            elif self.gamma == "auto":
                 self._gamma = 1.0 / X.shape[1]
-            else:
-                raise ValueError(
-                    "When 'gamma' is a string, it should be either 'scale' or "
-                    "'auto'. Got '{}' instead.".format(self.gamma)
-                )
-        else:
+        elif isinstance(self.gamma, Real):
             self._gamma = self.gamma
 
         fit = self._sparse_fit if self._sparse else self._dense_fit
         if self.verbose:
-            print('[LibSVM]', end='')
+            print("[LibSVM]", end="")
 
-        seed = rnd.randint(np.iinfo('i').max)
+        seed = rnd.randint(np.iinfo("i").max)
         fit(X, y, sample_weight, solver_type, kernel, random_seed=seed)
         # see comment on the other call to np.iinfo in this file
 
-        self.shape_fit_ = X.shape if hasattr(X, "shape") else (n_samples, )
+        self.shape_fit_ = X.shape if hasattr(X, "shape") else (n_samples,)
 
         # In binary case, we need to flip the sign of coef, intercept and
         # decision function. Use self._intercept_ and self._dual_coef_
         # internally.
         self._intercept_ = self.intercept_.copy()
         self._dual_coef_ = self.dual_coef_
-        if self._impl in ['c_svc', 'nu_svc'] and len(self.classes_) == 2:
+        if self._impl in ["c_svc", "nu_svc"] and len(self.classes_) == 2:
             self.intercept_ *= -1
             self.dual_coef_ = -self.dual_coef_
 
+        dual_coef = self._dual_coef_.data if self._sparse else self._dual_coef_
+        intercept_finiteness = np.isfinite(self._intercept_).all()
+        dual_coef_finiteness = np.isfinite(dual_coef).all()
+        if not (intercept_finiteness and dual_coef_finiteness):
+            raise ValueError(
+                "The dual coefficients or intercepts are not finite."
+                " The input data may contain large values and need to be"
+                " preprocessed."
+            )
+
+        # Since, in the case of SVC and NuSVC, the number of models optimized by
+        # libSVM could be greater than one (depending on the input), `n_iter_`
+        # stores an ndarray.
+        # For the other sub-classes (SVR, NuSVR, and OneClassSVM), the number of
+        # models optimized by libSVM is always one, so `n_iter_` stores an
+        # integer.
+        if self._impl in ["c_svc", "nu_svc"]:
+            self.n_iter_ = self._num_iter
+        else:
+            self.n_iter_ = self._num_iter.item()
+
         return self
 
     def _validate_targets(self, y):
@@ -244,21 +297,19 @@ def _validate_targets(self, y):
 
         Default implementation for SVR and one-class; overridden in BaseSVC.
         """
-        # XXX this is ugly.
-        # Regression models should not have a class_weight_ attribute.
-        self.class_weight_ = np.empty(0)
         return column_or_1d(y, warn=True).astype(np.float64, copy=False)
 
     def _warn_from_fit_status(self):
         assert self.fit_status_ in (0, 1)
         if self.fit_status_ == 1:
-            warnings.warn('Solver terminated early (max_iter=%i).'
-                          '  Consider pre-processing your data with'
-                          ' StandardScaler or MinMaxScaler.'
-                          % self.max_iter, ConvergenceWarning)
-
-    def _dense_fit(self, X, y, sample_weight, solver_type, kernel,
-                   random_seed):
+            warnings.warn(
+                "Solver terminated early (max_iter=%i)."
+                "  Consider pre-processing your data with"
+                " StandardScaler or MinMaxScaler." % self.max_iter,
+                ConvergenceWarning,
+            )
+
+    def _dense_fit(self, X, y, sample_weight, solver_type, kernel, random_seed):
         if callable(self.kernel):
             # you must store a reference to X to compute the kernel in predict
             # TODO: add keyword copy to copy on demand
@@ -272,39 +323,80 @@ def _dense_fit(self, X, y, sample_weight, solver_type, kernel,
 
         # we don't pass **self.get_params() to allow subclasses to
         # add other parameters to __init__
-        self.support_, self.support_vectors_, self._n_support, \
-            self.dual_coef_, self.intercept_, self._probA, \
-            self._probB, self.fit_status_ = libsvm.fit(
-                X, y,
-                svm_type=solver_type, sample_weight=sample_weight,
-                class_weight=self.class_weight_, kernel=kernel, C=self.C,
-                nu=self.nu, probability=self.probability, degree=self.degree,
-                shrinking=self.shrinking, tol=self.tol,
-                cache_size=self.cache_size, coef0=self.coef0,
-                gamma=self._gamma, epsilon=self.epsilon,
-                max_iter=self.max_iter, random_seed=random_seed)
+        (
+            self.support_,
+            self.support_vectors_,
+            self._n_support,
+            self.dual_coef_,
+            self.intercept_,
+            self._probA,
+            self._probB,
+            self.fit_status_,
+            self._num_iter,
+        ) = libsvm.fit(
+            X,
+            y,
+            svm_type=solver_type,
+            sample_weight=sample_weight,
+            class_weight=getattr(self, "class_weight_", np.empty(0)),
+            kernel=kernel,
+            C=self.C,
+            nu=self.nu,
+            probability=self.probability,
+            degree=self.degree,
+            shrinking=self.shrinking,
+            tol=self.tol,
+            cache_size=self.cache_size,
+            coef0=self.coef0,
+            gamma=self._gamma,
+            epsilon=self.epsilon,
+            max_iter=self.max_iter,
+            random_seed=random_seed,
+        )
 
         self._warn_from_fit_status()
 
-    def _sparse_fit(self, X, y, sample_weight, solver_type, kernel,
-                    random_seed):
-        X.data = np.asarray(X.data, dtype=np.float64, order='C')
+    def _sparse_fit(self, X, y, sample_weight, solver_type, kernel, random_seed):
+        X.data = np.asarray(X.data, dtype=np.float64, order="C")
         X.sort_indices()
 
         kernel_type = self._sparse_kernels.index(kernel)
 
         libsvm_sparse.set_verbosity_wrap(self.verbose)
 
-        self.support_, self.support_vectors_, dual_coef_data, \
-            self.intercept_, self._n_support, \
-            self._probA, self._probB, self.fit_status_ = \
-            libsvm_sparse.libsvm_sparse_train(
-                X.shape[1], X.data, X.indices, X.indptr, y, solver_type,
-                kernel_type, self.degree, self._gamma, self.coef0, self.tol,
-                self.C, self.class_weight_,
-                sample_weight, self.nu, self.cache_size, self.epsilon,
-                int(self.shrinking), int(self.probability), self.max_iter,
-                random_seed)
+        (
+            self.support_,
+            self.support_vectors_,
+            dual_coef_data,
+            self.intercept_,
+            self._n_support,
+            self._probA,
+            self._probB,
+            self.fit_status_,
+            self._num_iter,
+        ) = libsvm_sparse.libsvm_sparse_train(
+            X.shape[1],
+            X.data,
+            X.indices,
+            X.indptr,
+            y,
+            solver_type,
+            kernel_type,
+            self.degree,
+            self._gamma,
+            self.coef0,
+            self.tol,
+            self.C,
+            getattr(self, "class_weight_", np.empty(0)),
+            sample_weight,
+            self.nu,
+            self.cache_size,
+            self.epsilon,
+            int(self.shrinking),
+            int(self.probability),
+            self.max_iter,
+            random_seed,
+        )
 
         self._warn_from_fit_status()
 
@@ -318,11 +410,12 @@ def _sparse_fit(self, X, y, sample_weight, solver_type, kernel,
         if not n_SV:
             self.dual_coef_ = sp.csr_matrix([])
         else:
-            dual_coef_indptr = np.arange(0, dual_coef_indices.size + 1,
-                                         dual_coef_indices.size / n_class)
+            dual_coef_indptr = np.arange(
+                0, dual_coef_indices.size + 1, dual_coef_indices.size / n_class
+            )
             self.dual_coef_ = sp.csr_matrix(
-                (dual_coef_data, dual_coef_indices, dual_coef_indptr),
-                (n_class, n_SV))
+                (dual_coef_data, dual_coef_indices, dual_coef_indptr), (n_class, n_SV)
+            )
 
     def predict(self, X):
         """Perform regression on samples in X.
@@ -338,6 +431,7 @@ def predict(self, X):
         Returns
         -------
         y_pred : ndarray of shape (n_samples,)
+            The predicted values.
         """
         X = self._validate_for_predict(X)
         predict = self._sparse_predict if self._sparse else self._dense_predict
@@ -346,47 +440,72 @@ def predict(self, X):
     def _dense_predict(self, X):
         X = self._compute_kernel(X)
         if X.ndim == 1:
-            X = check_array(X, order='C', accept_large_sparse=False)
+            X = check_array(X, order="C", accept_large_sparse=False)
 
         kernel = self.kernel
         if callable(self.kernel):
-            kernel = 'precomputed'
+            kernel = "precomputed"
             if X.shape[1] != self.shape_fit_[0]:
-                raise ValueError("X.shape[1] = %d should be equal to %d, "
-                                 "the number of samples at training time" %
-                                 (X.shape[1], self.shape_fit_[0]))
+                raise ValueError(
+                    "X.shape[1] = %d should be equal to %d, "
+                    "the number of samples at training time"
+                    % (X.shape[1], self.shape_fit_[0])
+                )
 
         svm_type = LIBSVM_IMPL.index(self._impl)
 
         return libsvm.predict(
-            X, self.support_, self.support_vectors_, self._n_support,
-            self._dual_coef_, self._intercept_,
-            self._probA, self._probB, svm_type=svm_type, kernel=kernel,
-            degree=self.degree, coef0=self.coef0, gamma=self._gamma,
-            cache_size=self.cache_size)
+            X,
+            self.support_,
+            self.support_vectors_,
+            self._n_support,
+            self._dual_coef_,
+            self._intercept_,
+            self._probA,
+            self._probB,
+            svm_type=svm_type,
+            kernel=kernel,
+            degree=self.degree,
+            coef0=self.coef0,
+            gamma=self._gamma,
+            cache_size=self.cache_size,
+        )
 
     def _sparse_predict(self, X):
         # Precondition: X is a csr_matrix of dtype np.float64.
         kernel = self.kernel
         if callable(kernel):
-            kernel = 'precomputed'
+            kernel = "precomputed"
 
         kernel_type = self._sparse_kernels.index(kernel)
 
         C = 0.0  # C is not useful here
 
         return libsvm_sparse.libsvm_sparse_predict(
-            X.data, X.indices, X.indptr,
+            X.data,
+            X.indices,
+            X.indptr,
             self.support_vectors_.data,
             self.support_vectors_.indices,
             self.support_vectors_.indptr,
-            self._dual_coef_.data, self._intercept_,
-            LIBSVM_IMPL.index(self._impl), kernel_type,
-            self.degree, self._gamma, self.coef0, self.tol,
-            C, self.class_weight_,
-            self.nu, self.epsilon, self.shrinking,
-            self.probability, self._n_support,
-            self._probA, self._probB)
+            self._dual_coef_.data,
+            self._intercept_,
+            LIBSVM_IMPL.index(self._impl),
+            kernel_type,
+            self.degree,
+            self._gamma,
+            self.coef0,
+            self.tol,
+            C,
+            getattr(self, "class_weight_", np.empty(0)),
+            self.nu,
+            self.epsilon,
+            self.shrinking,
+            self.probability,
+            self._n_support,
+            self._probA,
+            self._probB,
+        )
 
     def _compute_kernel(self, X):
         """Return the data transformed by a callable kernel"""
@@ -396,7 +515,7 @@ def _compute_kernel(self, X):
             kernel = self.kernel(X, self.__Xfit)
             if sp.issparse(kernel):
                 kernel = kernel.toarray()
-            X = np.asarray(kernel, dtype=np.float64, order='C')
+            X = np.asarray(kernel, dtype=np.float64, order="C")
         return X
 
     def _decision_function(self, X):
@@ -424,58 +543,85 @@ def _decision_function(self, X):
 
         # In binary case, we need to flip the sign of coef, intercept and
         # decision function.
-        if self._impl in ['c_svc', 'nu_svc'] and len(self.classes_) == 2:
+        if self._impl in ["c_svc", "nu_svc"] and len(self.classes_) == 2:
             return -dec_func.ravel()
 
         return dec_func
 
     def _dense_decision_function(self, X):
-        X = check_array(X, dtype=np.float64, order="C",
-                        accept_large_sparse=False)
+        X = check_array(X, dtype=np.float64, order="C", accept_large_sparse=False)
 
         kernel = self.kernel
         if callable(kernel):
-            kernel = 'precomputed'
+            kernel = "precomputed"
 
         return libsvm.decision_function(
-            X, self.support_, self.support_vectors_, self._n_support,
-            self._dual_coef_, self._intercept_,
-            self._probA, self._probB,
+            X,
+            self.support_,
+            self.support_vectors_,
+            self._n_support,
+            self._dual_coef_,
+            self._intercept_,
+            self._probA,
+            self._probB,
             svm_type=LIBSVM_IMPL.index(self._impl),
-            kernel=kernel, degree=self.degree, cache_size=self.cache_size,
-            coef0=self.coef0, gamma=self._gamma)
+            kernel=kernel,
+            degree=self.degree,
+            cache_size=self.cache_size,
+            coef0=self.coef0,
+            gamma=self._gamma,
+        )
 
     def _sparse_decision_function(self, X):
-        X.data = np.asarray(X.data, dtype=np.float64, order='C')
+        X.data = np.asarray(X.data, dtype=np.float64, order="C")
 
         kernel = self.kernel
-        if hasattr(kernel, '__call__'):
-            kernel = 'precomputed'
+        if hasattr(kernel, "__call__"):
+            kernel = "precomputed"
 
         kernel_type = self._sparse_kernels.index(kernel)
 
         return libsvm_sparse.libsvm_sparse_decision_function(
-            X.data, X.indices, X.indptr,
+            X.data,
+            X.indices,
+            X.indptr,
             self.support_vectors_.data,
             self.support_vectors_.indices,
             self.support_vectors_.indptr,
-            self._dual_coef_.data, self._intercept_,
-            LIBSVM_IMPL.index(self._impl), kernel_type,
-            self.degree, self._gamma, self.coef0, self.tol,
-            self.C, self.class_weight_,
-            self.nu, self.epsilon, self.shrinking,
-            self.probability, self._n_support,
-            self._probA, self._probB)
+            self._dual_coef_.data,
+            self._intercept_,
+            LIBSVM_IMPL.index(self._impl),
+            kernel_type,
+            self.degree,
+            self._gamma,
+            self.coef0,
+            self.tol,
+            self.C,
+            getattr(self, "class_weight_", np.empty(0)),
+            self.nu,
+            self.epsilon,
+            self.shrinking,
+            self.probability,
+            self._n_support,
+            self._probA,
+            self._probB,
+        )
 
     def _validate_for_predict(self, X):
         check_is_fitted(self)
 
         if not callable(self.kernel):
-            X = self._validate_data(X, accept_sparse='csr', dtype=np.float64,
-                                    order="C", accept_large_sparse=False,
-                                    reset=False)
-
-        if self._sparse and not sp.isspmatrix(X):
+            X = validate_data(
+                self,
+                X,
+                accept_sparse="csr",
+                dtype=np.float64,
+                order="C",
+                accept_large_sparse=False,
+                reset=False,
+            )
+
+        if self._sparse and not sp.issparse(X):
             X = sp.csr_matrix(X)
         if self._sparse:
             X.sort_indices()
@@ -483,20 +629,35 @@ def _validate_for_predict(self, X):
         if sp.issparse(X) and not self._sparse and not callable(self.kernel):
             raise ValueError(
                 "cannot use sparse input in %r trained on dense data"
-                % type(self).__name__)
+                % type(self).__name__
+            )
 
         if self.kernel == "precomputed":
             if X.shape[1] != self.shape_fit_[0]:
-                raise ValueError("X.shape[1] = %d should be equal to %d, "
-                                 "the number of samples at training time" %
-                                 (X.shape[1], self.shape_fit_[0]))
+                raise ValueError(
+                    "X.shape[1] = %d should be equal to %d, "
+                    "the number of samples at training time"
+                    % (X.shape[1], self.shape_fit_[0])
+                )
+        # Fixes https://nvd.nist.gov/vuln/detail/CVE-2020-28975
+        # Check that _n_support is consistent with support_vectors
+        sv = self.support_vectors_
+        if not self._sparse and sv.size > 0 and self.n_support_.sum() != sv.shape[0]:
+            raise ValueError(
+                f"The internal representation of {self.__class__.__name__} was altered"
+            )
         return X
 
     @property
     def coef_(self):
-        if self.kernel != 'linear':
-            raise AttributeError('coef_ is only available when using a '
-                                 'linear kernel')
+        """Weights assigned to the features when `kernel="linear"`.
+
+        Returns
+        -------
+        ndarray of shape (n_features, n_classes)
+        """
+        if self.kernel != "linear":
+            raise AttributeError("coef_ is only available when using a linear kernel")
 
         coef = self._get_coef()
 
@@ -515,6 +676,7 @@ def _get_coef(self):
 
     @property
     def n_support_(self):
+        """Number of support vectors for each class."""
         try:
             check_is_fitted(self)
         except NotFittedError:
@@ -531,41 +693,77 @@ def n_support_(self):
 
 class BaseSVC(ClassifierMixin, BaseLibSVM, metaclass=ABCMeta):
     """ABC for LibSVM-based classifiers."""
+
+    _parameter_constraints: dict = {
+        **BaseLibSVM._parameter_constraints,
+        "decision_function_shape": [StrOptions({"ovr", "ovo"})],
+        "break_ties": ["boolean"],
+    }
+    for unused_param in ["epsilon", "nu"]:
+        _parameter_constraints.pop(unused_param)
+
     @abstractmethod
-    def __init__(self, kernel, degree, gamma, coef0, tol, C, nu,
-                 shrinking, probability, cache_size, class_weight, verbose,
-                 max_iter, decision_function_shape, random_state,
-                 break_ties):
+    def __init__(
+        self,
+        kernel,
+        degree,
+        gamma,
+        coef0,
+        tol,
+        C,
+        nu,
+        shrinking,
+        probability,
+        cache_size,
+        class_weight,
+        verbose,
+        max_iter,
+        decision_function_shape,
+        random_state,
+        break_ties,
+    ):
         self.decision_function_shape = decision_function_shape
         self.break_ties = break_ties
         super().__init__(
-            kernel=kernel, degree=degree, gamma=gamma,
-            coef0=coef0, tol=tol, C=C, nu=nu, epsilon=0., shrinking=shrinking,
-            probability=probability, cache_size=cache_size,
-            class_weight=class_weight, verbose=verbose, max_iter=max_iter,
-            random_state=random_state)
+            kernel=kernel,
+            degree=degree,
+            gamma=gamma,
+            coef0=coef0,
+            tol=tol,
+            C=C,
+            nu=nu,
+            epsilon=0.0,
+            shrinking=shrinking,
+            probability=probability,
+            cache_size=cache_size,
+            class_weight=class_weight,
+            verbose=verbose,
+            max_iter=max_iter,
+            random_state=random_state,
+        )
 
     def _validate_targets(self, y):
         y_ = column_or_1d(y, warn=True)
         check_classification_targets(y)
         cls, y = np.unique(y_, return_inverse=True)
-        self.class_weight_ = compute_class_weight(self.class_weight,
-                                                  classes=cls, y=y_)
+        self.class_weight_ = compute_class_weight(self.class_weight, classes=cls, y=y_)
         if len(cls) < 2:
             raise ValueError(
-                "The number of classes has to be greater than one; got %d"
-                " class" % len(cls))
+                "The number of classes has to be greater than one; got %d class"
+                % len(cls)
+            )
 
         self.classes_ = cls
 
-        return np.asarray(y, dtype=np.float64, order='C')
+        return np.asarray(y, dtype=np.float64, order="C")
 
     def decision_function(self, X):
-        """Evaluates the decision function for the samples in X.
+        """Evaluate the decision function for the samples in X.
 
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
+            The input samples.
 
         Returns
         -------
@@ -587,7 +785,7 @@ def decision_function(self, X):
         transformation of ovo decision function.
         """
         dec = self._decision_function(X)
-        if self.decision_function_shape == 'ovr' and len(self.classes_) > 2:
+        if self.decision_function_shape == "ovr" and len(self.classes_) > 2:
             return _ovr_decision_function(dec < 0, -dec, len(self.classes_))
         return dec
 
@@ -609,13 +807,16 @@ def predict(self, X):
             Class labels for samples in X.
         """
         check_is_fitted(self)
-        if self.break_ties and self.decision_function_shape == 'ovo':
-            raise ValueError("break_ties must be False when "
-                             "decision_function_shape is 'ovo'")
-
-        if (self.break_ties
-                and self.decision_function_shape == 'ovr'
-                and len(self.classes_) > 2):
+        if self.break_ties and self.decision_function_shape == "ovo":
+            raise ValueError(
+                "break_ties must be False when decision_function_shape is 'ovo'"
+            )
+
+        if (
+            self.break_ties
+            and self.decision_function_shape == "ovr"
+            and len(self.classes_) > 2
+        ):
             y = np.argmax(self.decision_function(X), axis=1)
         else:
             y = super().predict(X)
@@ -627,17 +828,18 @@ def predict(self, X):
     # estimators.
     def _check_proba(self):
         if not self.probability:
-            raise AttributeError("predict_proba is not available when "
-                                 " probability=False")
-        if self._impl not in ('c_svc', 'nu_svc'):
-            raise AttributeError("predict_proba only implemented for SVC"
-                                 " and NuSVC")
-
-    @property
-    def predict_proba(self):
+            raise AttributeError(
+                "predict_proba is not available when probability=False"
+            )
+        if self._impl not in ("c_svc", "nu_svc"):
+            raise AttributeError("predict_proba only implemented for SVC and NuSVC")
+        return True
+
+    @available_if(_check_proba)
+    def predict_proba(self, X):
         """Compute probabilities of possible outcomes for samples in X.
 
-        The model need to have probability information computed at training
+        The model needs to have probability information computed at training
         time: fit with attribute `probability` set to True.
 
         Parameters
@@ -660,20 +862,18 @@ def predict_proba(self):
         predict. Also, it will produce meaningless results on very small
         datasets.
         """
-        self._check_proba()
-        return self._predict_proba
-
-    def _predict_proba(self, X):
         X = self._validate_for_predict(X)
         if self.probA_.size == 0 or self.probB_.size == 0:
-            raise NotFittedError("predict_proba is not available when fitted "
-                                 "with probability=False")
-        pred_proba = (self._sparse_predict_proba
-                      if self._sparse else self._dense_predict_proba)
+            raise NotFittedError(
+                "predict_proba is not available when fitted with probability=False"
+            )
+        pred_proba = (
+            self._sparse_predict_proba if self._sparse else self._dense_predict_proba
+        )
         return pred_proba(X)
 
-    @property
-    def predict_log_proba(self):
+    @available_if(_check_proba)
+    def predict_log_proba(self, X):
         """Compute log probabilities of possible outcomes for samples in X.
 
         The model need to have probability information computed at training
@@ -700,10 +900,6 @@ def predict_log_proba(self):
         predict. Also, it will produce meaningless results on very small
         datasets.
         """
-        self._check_proba()
-        return self._predict_log_proba
-
-    def _predict_log_proba(self, X):
         return np.log(self.predict_proba(X))
 
     def _dense_predict_proba(self, X):
@@ -711,39 +907,62 @@ def _dense_predict_proba(self, X):
 
         kernel = self.kernel
         if callable(kernel):
-            kernel = 'precomputed'
+            kernel = "precomputed"
 
         svm_type = LIBSVM_IMPL.index(self._impl)
         pprob = libsvm.predict_proba(
-            X, self.support_, self.support_vectors_, self._n_support,
-            self._dual_coef_, self._intercept_,
-            self._probA, self._probB,
-            svm_type=svm_type, kernel=kernel, degree=self.degree,
-            cache_size=self.cache_size, coef0=self.coef0, gamma=self._gamma)
+            X,
+            self.support_,
+            self.support_vectors_,
+            self._n_support,
+            self._dual_coef_,
+            self._intercept_,
+            self._probA,
+            self._probB,
+            svm_type=svm_type,
+            kernel=kernel,
+            degree=self.degree,
+            cache_size=self.cache_size,
+            coef0=self.coef0,
+            gamma=self._gamma,
+        )
 
         return pprob
 
     def _sparse_predict_proba(self, X):
-        X.data = np.asarray(X.data, dtype=np.float64, order='C')
+        X.data = np.asarray(X.data, dtype=np.float64, order="C")
 
         kernel = self.kernel
         if callable(kernel):
-            kernel = 'precomputed'
+            kernel = "precomputed"
 
         kernel_type = self._sparse_kernels.index(kernel)
 
         return libsvm_sparse.libsvm_sparse_predict_proba(
-            X.data, X.indices, X.indptr,
+            X.data,
+            X.indices,
+            X.indptr,
             self.support_vectors_.data,
             self.support_vectors_.indices,
             self.support_vectors_.indptr,
-            self._dual_coef_.data, self._intercept_,
-            LIBSVM_IMPL.index(self._impl), kernel_type,
-            self.degree, self._gamma, self.coef0, self.tol,
-            self.C, self.class_weight_,
-            self.nu, self.epsilon, self.shrinking,
-            self.probability, self._n_support,
-            self._probA, self._probB)
+            self._dual_coef_.data,
+            self._intercept_,
+            LIBSVM_IMPL.index(self._impl),
+            kernel_type,
+            self.degree,
+            self._gamma,
+            self.coef0,
+            self.tol,
+            self.C,
+            getattr(self, "class_weight_", np.empty(0)),
+            self.nu,
+            self.epsilon,
+            self.shrinking,
+            self.probability,
+            self._n_support,
+            self._probA,
+            self._probB,
+        )
 
     def _get_coef(self):
         if self.dual_coef_.shape[0] == 1:
@@ -751,8 +970,9 @@ def _get_coef(self):
             coef = safe_sparse_dot(self.dual_coef_, self.support_vectors_)
         else:
             # 1vs1 classifier
-            coef = _one_vs_one_coef(self.dual_coef_, self._n_support,
-                                    self.support_vectors_)
+            coef = _one_vs_one_coef(
+                self.dual_coef_, self._n_support, self.support_vectors_
+            )
             if sp.issparse(coef[0]):
                 coef = sp.vstack(coef).tocsr()
             else:
@@ -762,12 +982,29 @@ def _get_coef(self):
 
     @property
     def probA_(self):
+        """Parameter learned in Platt scaling when `probability=True`.
+
+        Returns
+        -------
+        ndarray of shape  (n_classes * (n_classes - 1) / 2)
+        """
         return self._probA
 
     @property
     def probB_(self):
+        """Parameter learned in Platt scaling when `probability=True`.
+
+        Returns
+        -------
+        ndarray of shape  (n_classes * (n_classes - 1) / 2)
+        """
         return self._probB
 
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = self.kernel != "precomputed"
+        return tags
+
 
 def _get_liblinear_solver_type(multi_class, penalty, loss, dual):
     """Find the liblinear magic number for the solver.
@@ -786,54 +1023,64 @@ def _get_liblinear_solver_type(multi_class, penalty, loss, dual):
     # level3: whether the dual solver is available for the specified
     # combination of loss function and penalty
     _solver_type_dict = {
-        'logistic_regression': {
-            'l1': {False: 6},
-            'l2': {False: 0, True: 7}},
-        'hinge': {
-            'l2': {True: 3}},
-        'squared_hinge': {
-            'l1': {False: 5},
-            'l2': {False: 2, True: 1}},
-        'epsilon_insensitive': {
-            'l2': {True: 13}},
-        'squared_epsilon_insensitive': {
-            'l2': {False: 11, True: 12}},
-        'crammer_singer': 4
+        "logistic_regression": {"l1": {False: 6}, "l2": {False: 0, True: 7}},
+        "hinge": {"l2": {True: 3}},
+        "squared_hinge": {"l1": {False: 5}, "l2": {False: 2, True: 1}},
+        "epsilon_insensitive": {"l2": {True: 13}},
+        "squared_epsilon_insensitive": {"l2": {False: 11, True: 12}},
+        "crammer_singer": 4,
     }
 
-    if multi_class == 'crammer_singer':
+    if multi_class == "crammer_singer":
         return _solver_type_dict[multi_class]
-    elif multi_class != 'ovr':
-        raise ValueError("`multi_class` must be one of `ovr`, "
-                         "`crammer_singer`, got %r" % multi_class)
+    elif multi_class != "ovr":
+        raise ValueError(
+            "`multi_class` must be one of `ovr`, `crammer_singer`, got %r" % multi_class
+        )
 
     _solver_pen = _solver_type_dict.get(loss, None)
     if _solver_pen is None:
-        error_string = ("loss='%s' is not supported" % loss)
+        error_string = "loss='%s' is not supported" % loss
     else:
         _solver_dual = _solver_pen.get(penalty, None)
         if _solver_dual is None:
-            error_string = ("The combination of penalty='%s' "
-                            "and loss='%s' is not supported"
-                            % (penalty, loss))
+            error_string = (
+                "The combination of penalty='%s' and loss='%s' is not supported"
+                % (penalty, loss)
+            )
         else:
             solver_num = _solver_dual.get(dual, None)
             if solver_num is None:
-                error_string = ("The combination of penalty='%s' and "
-                                "loss='%s' are not supported when dual=%s"
-                                % (penalty, loss, dual))
+                error_string = (
+                    "The combination of penalty='%s' and "
+                    "loss='%s' are not supported when dual=%s" % (penalty, loss, dual)
+                )
             else:
                 return solver_num
-    raise ValueError('Unsupported set of arguments: %s, '
-                     'Parameters: penalty=%r, loss=%r, dual=%r'
-                     % (error_string, penalty, loss, dual))
-
-
-def _fit_liblinear(X, y, C, fit_intercept, intercept_scaling, class_weight,
-                   penalty, dual, verbose, max_iter, tol,
-                   random_state=None, multi_class='ovr',
-                   loss='logistic_regression', epsilon=0.1,
-                   sample_weight=None):
+    raise ValueError(
+        "Unsupported set of arguments: %s, Parameters: penalty=%r, loss=%r, dual=%r"
+        % (error_string, penalty, loss, dual)
+    )
+
+
+def _fit_liblinear(
+    X,
+    y,
+    C,
+    fit_intercept,
+    intercept_scaling,
+    class_weight,
+    penalty,
+    dual,
+    verbose,
+    max_iter,
+    tol,
+    random_state=None,
+    multi_class="ovr",
+    loss="logistic_regression",
+    epsilon=0.1,
+    sample_weight=None,
+):
     """Used by Logistic Regression (and CV) and LinearSVC/LinearSVR.
 
     Preprocessing is done in this function before supplying it to liblinear.
@@ -841,25 +1088,33 @@ def _fit_liblinear(X, y, C, fit_intercept, intercept_scaling, class_weight,
     Parameters
     ----------
     X : {array-like, sparse matrix} of shape (n_samples, n_features)
-        Training vector, where n_samples in the number of samples and
-        n_features is the number of features.
+        Training vector, where `n_samples` is the number of samples and
+        `n_features` is the number of features.
 
     y : array-like of shape (n_samples,)
         Target vector relative to X
 
     C : float
-        Inverse of cross-validation parameter. Lower the C, the more
+        Inverse of cross-validation parameter. The lower the C, the higher
         the penalization.
 
     fit_intercept : bool
-        Whether or not to fit the intercept, that is to add a intercept
-        term to the decision function.
+        Whether or not to fit an intercept. If set to True, the feature vector
+        is extended to include an intercept term: ``[x_1, ..., x_n, 1]``, where
+        1 corresponds to the intercept. If set to False, no intercept will be
+        used in calculations (i.e. data is expected to be already centered).
 
     intercept_scaling : float
-        LibLinear internally penalizes the intercept and this term is subject
-        to regularization just like the other terms of the feature vector.
-        In order to avoid this, one should increase the intercept_scaling.
-        such that the feature vector becomes [x, intercept_scaling].
+        Liblinear internally penalizes the intercept, treating it like any
+        other term in the feature vector. To reduce the impact of the
+        regularization on the intercept, the `intercept_scaling` parameter can
+        be set to a value greater than 1; the higher the value of
+        `intercept_scaling`, the lower the impact of regularization on it.
+        Then, the weights become `[w_x_1, ..., w_x_n,
+        w_intercept*intercept_scaling]`, where `w_x_1, ..., w_x_n` represent
+        the feature weights and the intercept weight is scaled by
+        `intercept_scaling`. This scaling allows the intercept term to have a
+        different regularization behavior compared to the other features.
 
     class_weight : dict or 'balanced', default=None
         Weights associated with classes in the form ``{class_label: weight}``.
@@ -921,35 +1176,39 @@ def _fit_liblinear(X, y, C, fit_intercept, intercept_scaling, class_weight,
     intercept_ : float
         The intercept term added to the vector.
 
-    n_iter_ : int
-        Maximum number of iterations run across all classes.
+    n_iter_ : array of int
+        Number of iterations run across for each class.
     """
-    if loss not in ['epsilon_insensitive', 'squared_epsilon_insensitive']:
+    if loss not in ["epsilon_insensitive", "squared_epsilon_insensitive"]:
         enc = LabelEncoder()
         y_ind = enc.fit_transform(y)
         classes_ = enc.classes_
         if len(classes_) < 2:
-            raise ValueError("This solver needs samples of at least 2 classes"
-                             " in the data, but the data contains only one"
-                             " class: %r" % classes_[0])
-
-        class_weight_ = compute_class_weight(class_weight, classes=classes_,
-                                             y=y)
+            raise ValueError(
+                "This solver needs samples of at least 2 classes"
+                " in the data, but the data contains only one"
+                " class: %r" % classes_[0]
+            )
+        class_weight_ = compute_class_weight(
+            class_weight, classes=classes_, y=y, sample_weight=sample_weight
+        )
     else:
         class_weight_ = np.empty(0, dtype=np.float64)
         y_ind = y
     liblinear.set_verbosity_wrap(verbose)
     rnd = check_random_state(random_state)
     if verbose:
-        print('[LibLinear]', end='')
+        print("[LibLinear]", end="")
 
     # LinearSVC breaks when intercept_scaling is <= 0
     bias = -1.0
     if fit_intercept:
         if intercept_scaling <= 0:
-            raise ValueError("Intercept scaling is %r but needs to be greater "
-                             "than 0. To disable fitting an intercept,"
-                             " set fit_intercept=False." % intercept_scaling)
+            raise ValueError(
+                "Intercept scaling is %r but needs to be greater "
+                "than 0. To disable fitting an intercept,"
+                " set fit_intercept=False." % intercept_scaling
+            )
         else:
             bias = intercept_scaling
 
@@ -965,28 +1224,39 @@ def _fit_liblinear(X, y, C, fit_intercept, intercept_scaling, class_weight,
     y_ind = np.asarray(y_ind, dtype=np.float64).ravel()
     y_ind = np.require(y_ind, requirements="W")
 
-    sample_weight = _check_sample_weight(sample_weight, X,
-                                         dtype=np.float64)
+    sample_weight = _check_sample_weight(sample_weight, X, dtype=np.float64)
 
     solver_type = _get_liblinear_solver_type(multi_class, penalty, loss, dual)
     raw_coef_, n_iter_ = liblinear.train_wrap(
-        X, y_ind, sp.isspmatrix(X), solver_type, tol, bias, C,
-        class_weight_, max_iter, rnd.randint(np.iinfo('i').max),
-        epsilon, sample_weight)
+        X,
+        y_ind,
+        sp.issparse(X),
+        solver_type,
+        tol,
+        bias,
+        C,
+        class_weight_,
+        max_iter,
+        rnd.randint(np.iinfo("i").max),
+        epsilon,
+        sample_weight,
+    )
     # Regarding rnd.randint(..) in the above signature:
     # seed for srand in range [0..INT_MAX); due to limitations in Numpy
     # on 32-bit platforms, we can't get to the UINT_MAX limit that
     # srand supports
-    n_iter_ = max(n_iter_)
-    if n_iter_ >= max_iter:
-        warnings.warn("Liblinear failed to converge, increase "
-                      "the number of iterations.", ConvergenceWarning)
+    n_iter_max = max(n_iter_)
+    if n_iter_max >= max_iter:
+        warnings.warn(
+            "Liblinear failed to converge, increase the number of iterations.",
+            ConvergenceWarning,
+        )
 
     if fit_intercept:
         coef_ = raw_coef_[:, :-1]
         intercept_ = intercept_scaling * raw_coef_[:, -1]
     else:
         coef_ = raw_coef_
-        intercept_ = 0.
+        intercept_ = 0.0
 
     return coef_, intercept_, n_iter_
diff --git a/sklearn/svm/_bounds.py b/sklearn/svm/_bounds.py
index b35728041f6cf..44923cb129767 100644
--- a/sklearn/svm/_bounds.py
+++ b/sklearn/svm/_bounds.py
@@ -1,31 +1,46 @@
 """Determination of parameter bounds"""
-# Author: Paolo Losi
-# License: BSD 3 clause
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+from numbers import Real
 
 import numpy as np
 
 from ..preprocessing import LabelBinarizer
-from ..utils.validation import check_consistent_length, check_array
-from ..utils.validation import _deprecate_positional_args
+from ..utils._param_validation import Interval, StrOptions, validate_params
 from ..utils.extmath import safe_sparse_dot
+from ..utils.validation import check_array, check_consistent_length
 
 
-@_deprecate_positional_args
-def l1_min_c(X, y, *, loss='squared_hinge', fit_intercept=True,
-             intercept_scaling=1.0):
-    """
-    Return the lowest bound for C such that for C in (l1_min_C, infinity)
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "y": ["array-like"],
+        "loss": [StrOptions({"squared_hinge", "log"})],
+        "fit_intercept": ["boolean"],
+        "intercept_scaling": [Interval(Real, 0, None, closed="neither")],
+    },
+    prefer_skip_nested_validation=True,
+)
+def l1_min_c(X, y, *, loss="squared_hinge", fit_intercept=True, intercept_scaling=1.0):
+    """Return the lowest bound for `C`.
+
+    The lower bound for `C` is computed such that for `C` in `(l1_min_C, infinity)`
     the model is guaranteed not to be empty. This applies to l1 penalized
-    classifiers, such as LinearSVC with penalty='l1' and
-    linear_model.LogisticRegression with penalty='l1'.
+    classifiers, such as :class:`sklearn.svm.LinearSVC` with penalty='l1' and
+    :class:`sklearn.linear_model.LogisticRegression` with penalty='l1'.
+
+    This value is valid if `class_weight` parameter in `fit()` is not set.
 
-    This value is valid if class_weight parameter in fit() is not set.
+    For an example of how to use this function, see
+    :ref:`sphx_glr_auto_examples_linear_model_plot_logistic_path.py`.
 
     Parameters
     ----------
     X : {array-like, sparse matrix} of shape (n_samples, n_features)
-        Training vector, where n_samples in the number of samples and
-        n_features is the number of features.
+        Training vector, where `n_samples` is the number of samples and
+        `n_features` is the number of features.
 
     y : array-like of shape (n_samples,)
         Target vector relative to X.
@@ -40,7 +55,7 @@ def l1_min_c(X, y, *, loss='squared_hinge', fit_intercept=True,
         It must match the fit() method parameter.
 
     intercept_scaling : float, default=1.0
-        when fit_intercept is True, instance vector x becomes
+        When fit_intercept is True, instance vector x becomes
         [x, intercept_scaling],
         i.e. a "synthetic" feature with constant value equals to
         intercept_scaling is appended to the instance vector.
@@ -49,26 +64,35 @@ def l1_min_c(X, y, *, loss='squared_hinge', fit_intercept=True,
     Returns
     -------
     l1_min_c : float
-        minimum value for C
+        Minimum value for C.
+
+    Examples
+    --------
+    >>> from sklearn.svm import l1_min_c
+    >>> from sklearn.datasets import make_classification
+    >>> X, y = make_classification(n_samples=100, n_features=20, random_state=42)
+    >>> print(f"{l1_min_c(X, y, loss='squared_hinge', fit_intercept=True):.4f}")
+    0.0044
     """
-    if loss not in ('squared_hinge', 'log'):
-        raise ValueError('loss type not in ("squared_hinge", "log")')
 
-    X = check_array(X, accept_sparse='csc')
+    X = check_array(X, accept_sparse="csc")
     check_consistent_length(X, y)
 
     Y = LabelBinarizer(neg_label=-1).fit_transform(y).T
     # maximum absolute value over classes and features
     den = np.max(np.abs(safe_sparse_dot(Y, X)))
     if fit_intercept:
-        bias = np.full((np.size(y), 1), intercept_scaling,
-                       dtype=np.array(intercept_scaling).dtype)
+        bias = np.full(
+            (np.size(y), 1), intercept_scaling, dtype=np.array(intercept_scaling).dtype
+        )
         den = max(den, abs(np.dot(Y, bias)).max())
 
     if den == 0.0:
-        raise ValueError('Ill-posed l1_min_c calculation: l1 will always '
-                         'select zero coefficients for this data')
-    if loss == 'squared_hinge':
+        raise ValueError(
+            "Ill-posed l1_min_c calculation: l1 will always "
+            "select zero coefficients for this data"
+        )
+    if loss == "squared_hinge":
         return 0.5 / den
     else:  # loss == 'log':
         return 2.0 / den
diff --git a/sklearn/svm/_classes.py b/sklearn/svm/_classes.py
index 908ece408bb1d..277da42893eaf 100644
--- a/sklearn/svm/_classes.py
+++ b/sklearn/svm/_classes.py
@@ -1,18 +1,38 @@
-import numpy as np
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
-from ._base import _fit_liblinear, BaseSVC, BaseLibSVM
-from ..base import BaseEstimator, RegressorMixin, OutlierMixin
-from ..linear_model._base import LinearClassifierMixin, SparseCoefMixin, \
-    LinearModel
-from ..utils.validation import _num_samples
-from ..utils.validation import _deprecate_positional_args
-from ..utils.multiclass import check_classification_targets
-from ..utils.deprecation import deprecated
+from numbers import Integral, Real
 
+import numpy as np
 
-class LinearSVC(LinearClassifierMixin,
-                SparseCoefMixin,
-                BaseEstimator):
+from ..base import BaseEstimator, OutlierMixin, RegressorMixin, _fit_context
+from ..linear_model._base import LinearClassifierMixin, LinearModel, SparseCoefMixin
+from ..utils._param_validation import Interval, StrOptions
+from ..utils.multiclass import check_classification_targets
+from ..utils.validation import _num_samples, validate_data
+from ._base import BaseLibSVM, BaseSVC, _fit_liblinear, _get_liblinear_solver_type
+
+
+def _validate_dual_parameter(dual, loss, penalty, multi_class, X):
+    """Helper function to assign the value of dual parameter."""
+    if dual == "auto":
+        if X.shape[0] < X.shape[1]:
+            try:
+                _get_liblinear_solver_type(multi_class, penalty, loss, True)
+                return True
+            except ValueError:  # dual not supported for the combination
+                return False
+        else:
+            try:
+                _get_liblinear_solver_type(multi_class, penalty, loss, False)
+                return False
+            except ValueError:  # primal not supported by the combination
+                return True
+    else:
+        return dual
+
+
+class LinearSVC(LinearClassifierMixin, SparseCoefMixin, BaseEstimator):
     """Linear Support Vector Classification.
 
     Similar to SVC with parameter kernel='linear', but implemented in terms of
@@ -20,6 +40,10 @@ class LinearSVC(LinearClassifierMixin,
     penalties and loss functions and should scale better to large numbers of
     samples.
 
+    The main differences between :class:`~sklearn.svm.LinearSVC` and
+    :class:`~sklearn.svm.SVC` lie in the loss function used by default, and in
+    the handling of intercept regularization between those two implementations.
+
     This class supports both dense and sparse input and the multiclass support
     is handled according to a one-vs-the-rest scheme.
 
@@ -38,9 +62,18 @@ class LinearSVC(LinearClassifierMixin,
         square of the hinge loss. The combination of ``penalty='l1'``
         and ``loss='hinge'`` is not supported.
 
-    dual : bool, default=True
+    dual : "auto" or bool, default="auto"
         Select the algorithm to either solve the dual or primal
         optimization problem. Prefer dual=False when n_samples > n_features.
+        `dual="auto"` will choose the value of the parameter automatically,
+        based on the values of `n_samples`, `n_features`, `loss`, `multi_class`
+        and `penalty`. If `n_samples` < `n_features` and optimizer supports
+        chosen `loss`, `multi_class` and `penalty`, then dual will be set to True,
+        otherwise it will be set to False.
+
+        .. versionchanged:: 1.3
+           The `"auto"` option is added in version 1.3 and will be the default
+           in version 1.5.
 
     tol : float, default=1e-4
         Tolerance for stopping criteria.
@@ -48,6 +81,9 @@ class LinearSVC(LinearClassifierMixin,
     C : float, default=1.0
         Regularization parameter. The strength of the regularization is
         inversely proportional to C. Must be strictly positive.
+        For an intuitive visualization of the effects of scaling
+        the regularization parameter C, see
+        :ref:`sphx_glr_auto_examples_svm_plot_svm_scale_c.py`.
 
     multi_class : {'ovr', 'crammer_singer'}, default='ovr'
         Determines the multi-class strategy if `y` contains more than
@@ -61,20 +97,26 @@ class LinearSVC(LinearClassifierMixin,
         will be ignored.
 
     fit_intercept : bool, default=True
-        Whether to calculate the intercept for this model. If set
-        to false, no intercept will be used in calculations
-        (i.e. data is expected to be already centered).
-
-    intercept_scaling : float, default=1
-        When self.fit_intercept is True, instance vector x becomes
-        ``[x, self.intercept_scaling]``,
-        i.e. a "synthetic" feature with constant value equals to
-        intercept_scaling is appended to the instance vector.
-        The intercept becomes intercept_scaling * synthetic feature weight
-        Note! the synthetic feature weight is subject to l1/l2 regularization
-        as all other features.
-        To lessen the effect of regularization on synthetic feature weight
-        (and therefore on the intercept) intercept_scaling has to be increased.
+        Whether or not to fit an intercept. If set to True, the feature vector
+        is extended to include an intercept term: `[x_1, ..., x_n, 1]`, where
+        1 corresponds to the intercept. If set to False, no intercept will be
+        used in calculations (i.e. data is expected to be already centered).
+
+    intercept_scaling : float, default=1.0
+        When `fit_intercept` is True, the instance vector x becomes ``[x_1,
+        ..., x_n, intercept_scaling]``, i.e. a "synthetic" feature with a
+        constant value equal to `intercept_scaling` is appended to the instance
+        vector. The intercept becomes intercept_scaling * synthetic feature
+        weight. Note that liblinear internally penalizes the intercept,
+        treating it like any other term in the feature vector. To reduce the
+        impact of the regularization on the intercept, the `intercept_scaling`
+        parameter can be set to a value greater than 1; the higher the value of
+        `intercept_scaling`, the lower the impact of regularization on it.
+        Then, the weights become `[w_x_1, ..., w_x_n,
+        w_intercept*intercept_scaling]`, where `w_x_1, ..., w_x_n` represent
+        the feature weights and the intercept weight is scaled by
+        `intercept_scaling`. This scaling allows the intercept term to have a
+        different regularization behavior compared to the other features.
 
     class_weight : dict or 'balanced', default=None
         Set the parameter C of class i to ``class_weight[i]*C`` for
@@ -105,7 +147,7 @@ class LinearSVC(LinearClassifierMixin,
     coef_ : ndarray of shape (1, n_features) if n_classes == 2 \
             else (n_classes, n_features)
         Weights assigned to the features (coefficients in the primal
-        problem). This is only available in the case of a linear kernel.
+        problem).
 
         ``coef_`` is a readonly property derived from ``raw_coef_`` that
         follows the internal memory layout of liblinear.
@@ -116,6 +158,17 @@ class LinearSVC(LinearClassifierMixin,
     classes_ : ndarray of shape (n_classes,)
         The unique classes labels.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     n_iter_ : int
         Maximum number of iterations run across all classes.
 
@@ -172,18 +225,45 @@ class LinearSVC(LinearClassifierMixin,
                     ('linearsvc', LinearSVC(random_state=0, tol=1e-05))])
 
     >>> print(clf.named_steps['linearsvc'].coef_)
-    [[0.141...   0.526... 0.679... 0.493...]]
+    [[0.141   0.526 0.679 0.493]]
 
     >>> print(clf.named_steps['linearsvc'].intercept_)
-    [0.1693...]
+    [0.1693]
     >>> print(clf.predict([[0, 0, 0, 0]]))
     [1]
     """
-    @_deprecate_positional_args
-    def __init__(self, penalty='l2', loss='squared_hinge', *, dual=True,
-                 tol=1e-4, C=1.0, multi_class='ovr', fit_intercept=True,
-                 intercept_scaling=1, class_weight=None, verbose=0,
-                 random_state=None, max_iter=1000):
+
+    _parameter_constraints: dict = {
+        "penalty": [StrOptions({"l1", "l2"})],
+        "loss": [StrOptions({"hinge", "squared_hinge"})],
+        "dual": ["boolean", StrOptions({"auto"})],
+        "tol": [Interval(Real, 0.0, None, closed="neither")],
+        "C": [Interval(Real, 0.0, None, closed="neither")],
+        "multi_class": [StrOptions({"ovr", "crammer_singer"})],
+        "fit_intercept": ["boolean"],
+        "intercept_scaling": [Interval(Real, 0, None, closed="neither")],
+        "class_weight": [None, dict, StrOptions({"balanced"})],
+        "verbose": ["verbose"],
+        "random_state": ["random_state"],
+        "max_iter": [Interval(Integral, 0, None, closed="left")],
+    }
+
+    def __init__(
+        self,
+        penalty="l2",
+        loss="squared_hinge",
+        *,
+        dual="auto",
+        tol=1e-4,
+        C=1.0,
+        multi_class="ovr",
+        fit_intercept=True,
+        intercept_scaling=1,
+        class_weight=None,
+        verbose=0,
+        random_state=None,
+        max_iter=1000,
+    ):
         self.dual = dual
         self.tol = tol
         self.C = C
@@ -197,14 +277,15 @@ def __init__(self, penalty='l2', loss='squared_hinge', *, dual=True,
         self.penalty = penalty
         self.loss = loss
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y, sample_weight=None):
         """Fit the model according to the given training data.
 
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
-            Training vector, where n_samples in the number of samples and
-            n_features is the number of features.
+            Training vector, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
 
         y : array-like of shape (n_samples,)
             Target vector relative to X.
@@ -221,21 +302,44 @@ def fit(self, X, y, sample_weight=None):
         self : object
             An instance of the estimator.
         """
-        if self.C < 0:
-            raise ValueError("Penalty term must be positive; got (C=%r)"
-                             % self.C)
-
-        X, y = self._validate_data(X, y, accept_sparse='csr',
-                                   dtype=np.float64, order="C",
-                                   accept_large_sparse=False)
+        X, y = validate_data(
+            self,
+            X,
+            y,
+            accept_sparse="csr",
+            dtype=np.float64,
+            order="C",
+            accept_large_sparse=False,
+        )
         check_classification_targets(y)
         self.classes_ = np.unique(y)
 
-        self.coef_, self.intercept_, self.n_iter_ = _fit_liblinear(
-            X, y, self.C, self.fit_intercept, self.intercept_scaling,
-            self.class_weight, self.penalty, self.dual, self.verbose,
-            self.max_iter, self.tol, self.random_state, self.multi_class,
-            self.loss, sample_weight=sample_weight)
+        _dual = _validate_dual_parameter(
+            self.dual, self.loss, self.penalty, self.multi_class, X
+        )
+
+        self.coef_, self.intercept_, n_iter_ = _fit_liblinear(
+            X,
+            y,
+            self.C,
+            self.fit_intercept,
+            self.intercept_scaling,
+            self.class_weight,
+            self.penalty,
+            _dual,
+            self.verbose,
+            self.max_iter,
+            self.tol,
+            self.random_state,
+            self.multi_class,
+            self.loss,
+            sample_weight=sample_weight,
+        )
+        # Backward compatibility: _fit_liblinear is used both by LinearSVC/R
+        # and LogisticRegression but LogisticRegression sets a structured
+        # `n_iter_` attribute with information about the underlying OvR fits
+        # while LinearSVC/R only reports the maximum value.
+        self.n_iter_ = n_iter_.max().item()
 
         if self.multi_class == "crammer_singer" and len(self.classes_) == 2:
             self.coef_ = (self.coef_[1] - self.coef_[0]).reshape(1, -1)
@@ -245,13 +349,10 @@ def fit(self, X, y, sample_weight=None):
 
         return self
 
-    def _more_tags(self):
-        return {
-            '_xfail_checks': {
-                'check_sample_weights_invariance':
-                'zero sample_weight is not equivalent to removing samples',
-            }
-        }
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
+        return tags
 
 
 class LinearSVR(RegressorMixin, LinearModel):
@@ -262,6 +363,10 @@ class LinearSVR(RegressorMixin, LinearModel):
     penalties and loss functions and should scale better to large numbers of
     samples.
 
+    The main differences between :class:`~sklearn.svm.LinearSVR` and
+    :class:`~sklearn.svm.SVR` lie in the loss function used by default, and in
+    the handling of intercept regularization between those two implementations.
+
     This class supports both dense and sparse input.
 
     Read more in the :ref:`User Guide `.
@@ -289,24 +394,38 @@ class LinearSVR(RegressorMixin, LinearModel):
         loss ('squared_epsilon_insensitive') is the L2 loss.
 
     fit_intercept : bool, default=True
-        Whether to calculate the intercept for this model. If set
-        to false, no intercept will be used in calculations
-        (i.e. data is expected to be already centered).
-
-    intercept_scaling : float, default=1.
-        When self.fit_intercept is True, instance vector x becomes
-        [x, self.intercept_scaling],
-        i.e. a "synthetic" feature with constant value equals to
-        intercept_scaling is appended to the instance vector.
-        The intercept becomes intercept_scaling * synthetic feature weight
-        Note! the synthetic feature weight is subject to l1/l2 regularization
-        as all other features.
-        To lessen the effect of regularization on synthetic feature weight
-        (and therefore on the intercept) intercept_scaling has to be increased.
-
-    dual : bool, default=True
+        Whether or not to fit an intercept. If set to True, the feature vector
+        is extended to include an intercept term: `[x_1, ..., x_n, 1]`, where
+        1 corresponds to the intercept. If set to False, no intercept will be
+        used in calculations (i.e. data is expected to be already centered).
+
+    intercept_scaling : float, default=1.0
+        When `fit_intercept` is True, the instance vector x becomes `[x_1, ...,
+        x_n, intercept_scaling]`, i.e. a "synthetic" feature with a constant
+        value equal to `intercept_scaling` is appended to the instance vector.
+        The intercept becomes intercept_scaling * synthetic feature weight.
+        Note that liblinear internally penalizes the intercept, treating it
+        like any other term in the feature vector. To reduce the impact of the
+        regularization on the intercept, the `intercept_scaling` parameter can
+        be set to a value greater than 1; the higher the value of
+        `intercept_scaling`, the lower the impact of regularization on it.
+        Then, the weights become `[w_x_1, ..., w_x_n,
+        w_intercept*intercept_scaling]`, where `w_x_1, ..., w_x_n` represent
+        the feature weights and the intercept weight is scaled by
+        `intercept_scaling`. This scaling allows the intercept term to have a
+        different regularization behavior compared to the other features.
+
+    dual : "auto" or bool, default="auto"
         Select the algorithm to either solve the dual or primal
         optimization problem. Prefer dual=False when n_samples > n_features.
+        `dual="auto"` will choose the value of the parameter automatically,
+        based on the values of `n_samples`, `n_features` and `loss`. If
+        `n_samples` < `n_features` and optimizer supports chosen `loss`,
+        then dual will be set to True, otherwise it will be set to False.
+
+        .. versionchanged:: 1.3
+           The `"auto"` option is added in version 1.3 and will be the default
+           in version 1.5.
 
     verbose : int, default=0
         Enable verbose output. Note that this setting takes advantage of a
@@ -326,7 +445,7 @@ class LinearSVR(RegressorMixin, LinearModel):
     coef_ : ndarray of shape (n_features) if n_classes == 2 \
             else (n_classes, n_features)
         Weights assigned to the features (coefficients in the primal
-        problem). This is only available in the case of a linear kernel.
+        problem).
 
         `coef_` is a readonly property derived from `raw_coef_` that
         follows the internal memory layout of liblinear.
@@ -334,9 +453,35 @@ class LinearSVR(RegressorMixin, LinearModel):
     intercept_ : ndarray of shape (1) if n_classes == 2 else (n_classes)
         Constants in decision function.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
     n_iter_ : int
         Maximum number of iterations run across all classes.
 
+    See Also
+    --------
+    LinearSVC : Implementation of Support Vector Machine classifier using the
+        same library as this class (liblinear).
+
+    SVR : Implementation of Support Vector Machine regression using libsvm:
+        the kernel can be non-linear but its SMO algorithm does not scale to
+        large number of samples as :class:`~sklearn.svm.LinearSVR` does.
+
+    sklearn.linear_model.SGDRegressor : SGDRegressor can optimize the same cost
+        function as LinearSVR
+        by adjusting the penalty and loss parameters. In addition it requires
+        less memory, allows incremental (online) learning, and implements
+        various loss functions and regularization regimes.
+
     Examples
     --------
     >>> from sklearn.svm import LinearSVR
@@ -351,34 +496,40 @@ class LinearSVR(RegressorMixin, LinearModel):
                     ('linearsvr', LinearSVR(random_state=0, tol=1e-05))])
 
     >>> print(regr.named_steps['linearsvr'].coef_)
-    [18.582... 27.023... 44.357... 64.522...]
+    [18.582 27.023 44.357 64.522]
     >>> print(regr.named_steps['linearsvr'].intercept_)
-    [-4...]
+    [-4.]
     >>> print(regr.predict([[0, 0, 0, 0]]))
-    [-2.384...]
-
-
-    See Also
-    --------
-    LinearSVC : Implementation of Support Vector Machine classifier using the
-        same library as this class (liblinear).
-
-    SVR : Implementation of Support Vector Machine regression using libsvm:
-        the kernel can be non-linear but its SMO algorithm does not
-        scale to large number of samples as LinearSVC does.
-
-    sklearn.linear_model.SGDRegressor : SGDRegressor can optimize the same cost
-        function as LinearSVR
-        by adjusting the penalty and loss parameters. In addition it requires
-        less memory, allows incremental (online) learning, and implements
-        various loss functions and regularization regimes.
+    [-2.384]
     """
 
-    @_deprecate_positional_args
-    def __init__(self, *, epsilon=0.0, tol=1e-4, C=1.0,
-                 loss='epsilon_insensitive', fit_intercept=True,
-                 intercept_scaling=1., dual=True, verbose=0,
-                 random_state=None, max_iter=1000):
+    _parameter_constraints: dict = {
+        "epsilon": [Real],
+        "tol": [Interval(Real, 0.0, None, closed="neither")],
+        "C": [Interval(Real, 0.0, None, closed="neither")],
+        "loss": [StrOptions({"epsilon_insensitive", "squared_epsilon_insensitive"})],
+        "fit_intercept": ["boolean"],
+        "intercept_scaling": [Interval(Real, 0, None, closed="neither")],
+        "dual": ["boolean", StrOptions({"auto"})],
+        "verbose": ["verbose"],
+        "random_state": ["random_state"],
+        "max_iter": [Interval(Integral, 0, None, closed="left")],
+    }
+
+    def __init__(
+        self,
+        *,
+        epsilon=0.0,
+        tol=1e-4,
+        C=1.0,
+        loss="epsilon_insensitive",
+        fit_intercept=True,
+        intercept_scaling=1.0,
+        dual="auto",
+        verbose=0,
+        random_state=None,
+        max_iter=1000,
+    ):
         self.tol = tol
         self.C = C
         self.epsilon = epsilon
@@ -390,17 +541,18 @@ def __init__(self, *, epsilon=0.0, tol=1e-4, C=1.0,
         self.dual = dual
         self.loss = loss
 
+    @_fit_context(prefer_skip_nested_validation=True)
     def fit(self, X, y, sample_weight=None):
         """Fit the model according to the given training data.
 
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
-            Training vector, where n_samples in the number of samples and
-            n_features is the number of features.
+            Training vector, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
 
         y : array-like of shape (n_samples,)
-            Target vector relative to X
+            Target vector relative to X.
 
         sample_weight : array-like of shape (n_samples,), default=None
             Array of weights that are assigned to individual
@@ -414,30 +566,49 @@ def fit(self, X, y, sample_weight=None):
         self : object
             An instance of the estimator.
         """
-        if self.C < 0:
-            raise ValueError("Penalty term must be positive; got (C=%r)"
-                             % self.C)
-
-        X, y = self._validate_data(X, y, accept_sparse='csr',
-                                   dtype=np.float64, order="C",
-                                   accept_large_sparse=False)
-        penalty = 'l2'  # SVR only accepts l2 penalty
-        self.coef_, self.intercept_, self.n_iter_ = _fit_liblinear(
-            X, y, self.C, self.fit_intercept, self.intercept_scaling,
-            None, penalty, self.dual, self.verbose,
-            self.max_iter, self.tol, self.random_state, loss=self.loss,
-            epsilon=self.epsilon, sample_weight=sample_weight)
+        X, y = validate_data(
+            self,
+            X,
+            y,
+            accept_sparse="csr",
+            dtype=np.float64,
+            order="C",
+            accept_large_sparse=False,
+        )
+        penalty = "l2"  # SVR only accepts l2 penalty
+
+        _dual = _validate_dual_parameter(self.dual, self.loss, penalty, "ovr", X)
+
+        self.coef_, self.intercept_, n_iter_ = _fit_liblinear(
+            X,
+            y,
+            self.C,
+            self.fit_intercept,
+            self.intercept_scaling,
+            None,
+            penalty,
+            _dual,
+            self.verbose,
+            self.max_iter,
+            self.tol,
+            self.random_state,
+            loss=self.loss,
+            epsilon=self.epsilon,
+            sample_weight=sample_weight,
+        )
         self.coef_ = self.coef_.ravel()
+        # Backward compatibility: _fit_liblinear is used both by LinearSVC/R
+        # and LogisticRegression but LogisticRegression sets a structured
+        # `n_iter_` attribute with information about the underlying OvR fits
+        # while LinearSVC/R only reports the maximum value.
+        self.n_iter_ = n_iter_.max().item()
 
         return self
 
-    def _more_tags(self):
-        return {
-            '_xfail_checks': {
-                'check_sample_weights_invariance':
-                'zero sample_weight is not equivalent to removing samples',
-            }
-        }
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.sparse = True
+        return tags
 
 
 class SVC(BaseSVC):
@@ -448,7 +619,8 @@ class SVC(BaseSVC):
     beyond tens of thousands of samples. For large datasets
     consider using :class:`~sklearn.svm.LinearSVC` or
     :class:`~sklearn.linear_model.SGDClassifier` instead, possibly after a
-    :class:`~sklearn.kernel_approximation.Nystroem` transformer.
+    :class:`~sklearn.kernel_approximation.Nystroem` transformer or
+    other :ref:`kernel_approximation`.
 
     The multiclass support is handled according to a one-vs-one scheme.
 
@@ -457,6 +629,9 @@ class SVC(BaseSVC):
     other, see the corresponding section in the narrative documentation:
     :ref:`svm_kernels`.
 
+    To learn how to tune SVC's hyperparameters, see the following example:
+    :ref:`sphx_glr_auto_examples_model_selection_plot_nested_cross_validation_iris.py`
+
     Read more in the :ref:`User Guide `.
 
     Parameters
@@ -464,26 +639,30 @@ class SVC(BaseSVC):
     C : float, default=1.0
         Regularization parameter. The strength of the regularization is
         inversely proportional to C. Must be strictly positive. The penalty
-        is a squared l2 penalty.
-
-    kernel : {'linear', 'poly', 'rbf', 'sigmoid', 'precomputed'}, default='rbf'
-        Specifies the kernel type to be used in the algorithm.
-        It must be one of 'linear', 'poly', 'rbf', 'sigmoid', 'precomputed' or
-        a callable.
-        If none is given, 'rbf' will be used. If a callable is given it is
-        used to pre-compute the kernel matrix from data matrices; that matrix
-        should be an array of shape ``(n_samples, n_samples)``.
+        is a squared l2 penalty. For an intuitive visualization of the effects
+        of scaling the regularization parameter C, see
+        :ref:`sphx_glr_auto_examples_svm_plot_svm_scale_c.py`.
+
+    kernel : {'linear', 'poly', 'rbf', 'sigmoid', 'precomputed'} or callable,  \
+        default='rbf'
+        Specifies the kernel type to be used in the algorithm. If
+        none is given, 'rbf' will be used. If a callable is given it is used to
+        pre-compute the kernel matrix from data matrices; that matrix should be
+        an array of shape ``(n_samples, n_samples)``. For an intuitive
+        visualization of different kernel types see
+        :ref:`sphx_glr_auto_examples_svm_plot_svm_kernels.py`.
 
     degree : int, default=3
         Degree of the polynomial kernel function ('poly').
-        Ignored by all other kernels.
+        Must be non-negative. Ignored by all other kernels.
 
     gamma : {'scale', 'auto'} or float, default='scale'
         Kernel coefficient for 'rbf', 'poly' and 'sigmoid'.
 
         - if ``gamma='scale'`` (default) is passed then it uses
           1 / (n_features * X.var()) as value of gamma,
-        - if 'auto', uses 1 / n_features.
+        - if 'auto', uses 1 / n_features
+        - if float, must be non-negative.
 
         .. versionchanged:: 0.22
            The default value of ``gamma`` changed from 'auto' to 'scale'.
@@ -514,7 +693,7 @@ class SVC(BaseSVC):
         weight one.
         The "balanced" mode uses the values of y to automatically adjust
         weights inversely proportional to class frequencies in the input data
-        as ``n_samples / (n_classes * np.bincount(y))``
+        as ``n_samples / (n_classes * np.bincount(y))``.
 
     verbose : bool, default=False
         Enable verbose output. Note that this setting takes advantage of a
@@ -528,9 +707,10 @@ class SVC(BaseSVC):
         Whether to return a one-vs-rest ('ovr') decision function of shape
         (n_samples, n_classes) as all other classifiers, or the original
         one-vs-one ('ovo') decision function of libsvm which has shape
-        (n_samples, n_classes * (n_classes - 1) / 2). However, one-vs-one
-        ('ovo') is always used as multi-class strategy. The parameter is
-        ignored for binary classification.
+        (n_samples, n_classes * (n_classes - 1) / 2). However, note that
+        internally, one-vs-one ('ovo') is always used as a multi-class strategy
+        to train models; an ovr matrix is only constructed from the ovo matrix.
+        The parameter is ignored for binary classification.
 
         .. versionchanged:: 0.19
             decision_function_shape is 'ovr' by default.
@@ -546,7 +726,9 @@ class SVC(BaseSVC):
         :term:`predict` will break ties according to the confidence values of
         :term:`decision_function`; otherwise the first class among the tied
         classes is returned. Please note that breaking ties comes at a
-        relatively high computational cost compared to a simple predict.
+        relatively high computational cost compared to a simple predict. See
+        :ref:`sphx_glr_auto_examples_svm_plot_svm_tie_breaking.py` for an
+        example of its usage with ``decision_function_shape='ovr'``.
 
         .. versionadded:: 0.22
 
@@ -587,11 +769,29 @@ class SVC(BaseSVC):
     intercept_ : ndarray of shape (n_classes * (n_classes - 1) / 2,)
         Constants in decision function.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    n_iter_ : ndarray of shape (n_classes * (n_classes - 1) // 2,)
+        Number of iterations run by the optimization routine to fit the model.
+        The shape of this attribute depends on the number of models optimized
+        which in turn depends on the number of classes.
+
+        .. versionadded:: 1.1
+
     support_ : ndarray of shape (n_SV)
         Indices of support vectors.
 
     support_vectors_ : ndarray of shape (n_SV, n_features)
-        Support vectors.
+        Support vectors. An empty array if kernel is precomputed.
 
     n_support_ : ndarray of shape (n_classes,), dtype=int32
         Number of support vectors for each class.
@@ -610,6 +810,23 @@ class SVC(BaseSVC):
     shape_fit_ : tuple of int of shape (n_dimensions_of_X,)
         Array dimensions of training vector ``X``.
 
+    See Also
+    --------
+    SVR : Support Vector Machine for Regression implemented using libsvm.
+
+    LinearSVC : Scalable Linear Support Vector Machine for classification
+        implemented using liblinear. Check the See Also section of
+        LinearSVC for more comparison element.
+
+    References
+    ----------
+    .. [1] `LIBSVM: A Library for Support Vector Machines
+        `_
+
+    .. [2] `Platt, John (1999). "Probabilistic Outputs for Support Vector
+        Machines and Comparisons to Regularized Likelihood Methods"
+        `_
+
     Examples
     --------
     >>> import numpy as np
@@ -626,50 +843,49 @@ class SVC(BaseSVC):
     >>> print(clf.predict([[-0.8, -1]]))
     [1]
 
-    See Also
-    --------
-    SVR : Support Vector Machine for Regression implemented using libsvm.
-
-    LinearSVC : Scalable Linear Support Vector Machine for classification
-        implemented using liblinear. Check the See Also section of
-        LinearSVC for more comparison element.
-
-    References
-    ----------
-    .. [1] `LIBSVM: A Library for Support Vector Machines
-        `_
-
-    .. [2] `Platt, John (1999). "Probabilistic outputs for support vector
-        machines and comparison to regularizedlikelihood methods."
-        `_
+    For a comparison of the SVC with other classifiers see:
+    :ref:`sphx_glr_auto_examples_classification_plot_classification_probability.py`.
     """
 
-    _impl = 'c_svc'
-
-    @_deprecate_positional_args
-    def __init__(self, *, C=1.0, kernel='rbf', degree=3, gamma='scale',
-                 coef0=0.0, shrinking=True, probability=False,
-                 tol=1e-3, cache_size=200, class_weight=None,
-                 verbose=False, max_iter=-1, decision_function_shape='ovr',
-                 break_ties=False,
-                 random_state=None):
-
+    _impl = "c_svc"
+
+    def __init__(
+        self,
+        *,
+        C=1.0,
+        kernel="rbf",
+        degree=3,
+        gamma="scale",
+        coef0=0.0,
+        shrinking=True,
+        probability=False,
+        tol=1e-3,
+        cache_size=200,
+        class_weight=None,
+        verbose=False,
+        max_iter=-1,
+        decision_function_shape="ovr",
+        break_ties=False,
+        random_state=None,
+    ):
         super().__init__(
-            kernel=kernel, degree=degree, gamma=gamma,
-            coef0=coef0, tol=tol, C=C, nu=0., shrinking=shrinking,
-            probability=probability, cache_size=cache_size,
-            class_weight=class_weight, verbose=verbose, max_iter=max_iter,
+            kernel=kernel,
+            degree=degree,
+            gamma=gamma,
+            coef0=coef0,
+            tol=tol,
+            C=C,
+            nu=0.0,
+            shrinking=shrinking,
+            probability=probability,
+            cache_size=cache_size,
+            class_weight=class_weight,
+            verbose=verbose,
+            max_iter=max_iter,
             decision_function_shape=decision_function_shape,
             break_ties=break_ties,
-            random_state=random_state)
-
-    def _more_tags(self):
-        return {
-            '_xfail_checks': {
-                'check_sample_weights_invariance':
-                'zero sample_weight is not equivalent to removing samples',
-            }
-        }
+            random_state=random_state,
+        )
 
 
 class NuSVC(BaseSVC):
@@ -689,23 +905,25 @@ class NuSVC(BaseSVC):
         `) and a lower bound of the fraction of support vectors.
         Should be in the interval (0, 1].
 
-    kernel : {'linear', 'poly', 'rbf', 'sigmoid', 'precomputed'}, default='rbf'
-         Specifies the kernel type to be used in the algorithm.
-         It must be one of 'linear', 'poly', 'rbf', 'sigmoid', 'precomputed' or
-         a callable.
-         If none is given, 'rbf' will be used. If a callable is given it is
-         used to precompute the kernel matrix.
+    kernel : {'linear', 'poly', 'rbf', 'sigmoid', 'precomputed'} or callable,  \
+        default='rbf'
+        Specifies the kernel type to be used in the algorithm.
+        If none is given, 'rbf' will be used. If a callable is given it is
+        used to precompute the kernel matrix. For an intuitive
+        visualization of different kernel types see
+        :ref:`sphx_glr_auto_examples_svm_plot_svm_kernels.py`.
 
     degree : int, default=3
         Degree of the polynomial kernel function ('poly').
-        Ignored by all other kernels.
+        Must be non-negative. Ignored by all other kernels.
 
     gamma : {'scale', 'auto'} or float, default='scale'
         Kernel coefficient for 'rbf', 'poly' and 'sigmoid'.
 
         - if ``gamma='scale'`` (default) is passed then it uses
           1 / (n_features * X.var()) as value of gamma,
-        - if 'auto', uses 1 / n_features.
+        - if 'auto', uses 1 / n_features
+        - if float, must be non-negative.
 
         .. versionchanged:: 0.22
            The default value of ``gamma`` changed from 'auto' to 'scale'.
@@ -735,7 +953,7 @@ class NuSVC(BaseSVC):
         SVC. If not given, all classes are supposed to have
         weight one. The "balanced" mode uses the values of y to automatically
         adjust weights inversely proportional to class frequencies as
-        ``n_samples / (n_classes * np.bincount(y))``
+        ``n_samples / (n_classes * np.bincount(y))``.
 
     verbose : bool, default=False
         Enable verbose output. Note that this setting takes advantage of a
@@ -768,6 +986,8 @@ class NuSVC(BaseSVC):
         :term:`decision_function`; otherwise the first class among the tied
         classes is returned. Please note that breaking ties comes at a
         relatively high computational cost compared to a simple predict.
+        See :ref:`sphx_glr_auto_examples_svm_plot_svm_tie_breaking.py` for an
+        example of its usage with ``decision_function_shape='ovr'``.
 
         .. versionadded:: 0.22
 
@@ -808,6 +1028,24 @@ class NuSVC(BaseSVC):
     intercept_ : ndarray of shape (n_classes * (n_classes - 1) / 2,)
         Constants in decision function.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    n_iter_ : ndarray of shape (n_classes * (n_classes - 1) // 2,)
+        Number of iterations run by the optimization routine to fit the model.
+        The shape of this attribute depends on the number of models optimized
+        which in turn depends on the number of classes.
+
+        .. versionadded:: 1.1
+
     support_ : ndarray of shape (n_SV,)
         Indices of support vectors.
 
@@ -821,6 +1059,7 @@ class NuSVC(BaseSVC):
         0 if correctly fitted, 1 if the algorithm did not converge.
 
     probA_ : ndarray of shape (n_classes * (n_classes - 1) / 2,)
+
     probB_ : ndarray of shape (n_classes * (n_classes - 1) / 2,)
         If `probability=True`, it corresponds to the parameters learned in
         Platt scaling to produce probability estimates from decision values.
@@ -834,6 +1073,22 @@ class NuSVC(BaseSVC):
     shape_fit_ : tuple of int of shape (n_dimensions_of_X,)
         Array dimensions of training vector ``X``.
 
+    See Also
+    --------
+    SVC : Support Vector Machine for classification using libsvm.
+
+    LinearSVC : Scalable linear Support Vector Machine for classification using
+        liblinear.
+
+    References
+    ----------
+    .. [1] `LIBSVM: A Library for Support Vector Machines
+        `_
+
+    .. [2] `Platt, John (1999). "Probabilistic Outputs for Support Vector
+        Machines and Comparisons to Regularized Likelihood Methods"
+        `_
+
     Examples
     --------
     >>> import numpy as np
@@ -847,52 +1102,53 @@ class NuSVC(BaseSVC):
     Pipeline(steps=[('standardscaler', StandardScaler()), ('nusvc', NuSVC())])
     >>> print(clf.predict([[-0.8, -1]]))
     [1]
-
-    See Also
-    --------
-    SVC : Support Vector Machine for classification using libsvm.
-
-    LinearSVC : Scalable linear Support Vector Machine for classification using
-        liblinear.
-
-    References
-    ----------
-    .. [1] `LIBSVM: A Library for Support Vector Machines
-        `_
-
-    .. [2] `Platt, John (1999). "Probabilistic outputs for support vector
-        machines and comparison to regularizedlikelihood methods."
-        `_
     """
 
-    _impl = 'nu_svc'
-
-    @_deprecate_positional_args
-    def __init__(self, *, nu=0.5, kernel='rbf', degree=3, gamma='scale',
-                 coef0=0.0, shrinking=True, probability=False, tol=1e-3,
-                 cache_size=200, class_weight=None, verbose=False, max_iter=-1,
-                 decision_function_shape='ovr', break_ties=False,
-                 random_state=None):
-
+    _impl = "nu_svc"
+
+    _parameter_constraints: dict = {
+        **BaseSVC._parameter_constraints,
+        "nu": [Interval(Real, 0.0, 1.0, closed="right")],
+    }
+    _parameter_constraints.pop("C")
+
+    def __init__(
+        self,
+        *,
+        nu=0.5,
+        kernel="rbf",
+        degree=3,
+        gamma="scale",
+        coef0=0.0,
+        shrinking=True,
+        probability=False,
+        tol=1e-3,
+        cache_size=200,
+        class_weight=None,
+        verbose=False,
+        max_iter=-1,
+        decision_function_shape="ovr",
+        break_ties=False,
+        random_state=None,
+    ):
         super().__init__(
-            kernel=kernel, degree=degree, gamma=gamma,
-            coef0=coef0, tol=tol, C=0., nu=nu, shrinking=shrinking,
-            probability=probability, cache_size=cache_size,
-            class_weight=class_weight, verbose=verbose, max_iter=max_iter,
+            kernel=kernel,
+            degree=degree,
+            gamma=gamma,
+            coef0=coef0,
+            tol=tol,
+            C=0.0,
+            nu=nu,
+            shrinking=shrinking,
+            probability=probability,
+            cache_size=cache_size,
+            class_weight=class_weight,
+            verbose=verbose,
+            max_iter=max_iter,
             decision_function_shape=decision_function_shape,
             break_ties=break_ties,
-            random_state=random_state)
-
-    def _more_tags(self):
-        return {
-            '_xfail_checks': {
-                'check_methods_subset_invariance':
-                'fails for the decision_function method',
-                'check_class_weight_classifiers': 'class_weight is ignored.',
-                'check_sample_weights_invariance':
-                'zero sample_weight is not equivalent to removing samples',
-            }
-        }
+            random_state=random_state,
+        )
 
 
 class SVR(RegressorMixin, BaseLibSVM):
@@ -905,29 +1161,32 @@ class SVR(RegressorMixin, BaseLibSVM):
     to scale to datasets with more than a couple of 10000 samples. For large
     datasets consider using :class:`~sklearn.svm.LinearSVR` or
     :class:`~sklearn.linear_model.SGDRegressor` instead, possibly after a
-    :class:`~sklearn.kernel_approximation.Nystroem` transformer.
+    :class:`~sklearn.kernel_approximation.Nystroem` transformer or
+    other :ref:`kernel_approximation`.
 
     Read more in the :ref:`User Guide `.
 
     Parameters
     ----------
-    kernel : {'linear', 'poly', 'rbf', 'sigmoid', 'precomputed'}, default='rbf'
+    kernel : {'linear', 'poly', 'rbf', 'sigmoid', 'precomputed'} or callable,  \
+        default='rbf'
          Specifies the kernel type to be used in the algorithm.
-         It must be one of 'linear', 'poly', 'rbf', 'sigmoid', 'precomputed' or
-         a callable.
          If none is given, 'rbf' will be used. If a callable is given it is
          used to precompute the kernel matrix.
+         For an intuitive visualization of different kernel types
+         see :ref:`sphx_glr_auto_examples_svm_plot_svm_regression.py`
 
     degree : int, default=3
         Degree of the polynomial kernel function ('poly').
-        Ignored by all other kernels.
+        Must be non-negative. Ignored by all other kernels.
 
     gamma : {'scale', 'auto'} or float, default='scale'
         Kernel coefficient for 'rbf', 'poly' and 'sigmoid'.
 
         - if ``gamma='scale'`` (default) is passed then it uses
           1 / (n_features * X.var()) as value of gamma,
-        - if 'auto', uses 1 / n_features.
+        - if 'auto', uses 1 / n_features
+        - if float, must be non-negative.
 
         .. versionchanged:: 0.22
            The default value of ``gamma`` changed from 'auto' to 'scale'.
@@ -942,13 +1201,15 @@ class SVR(RegressorMixin, BaseLibSVM):
     C : float, default=1.0
         Regularization parameter. The strength of the regularization is
         inversely proportional to C. Must be strictly positive.
-        The penalty is a squared l2 penalty.
+        The penalty is a squared l2. For an intuitive visualization of the
+        effects of scaling the regularization parameter C, see
+        :ref:`sphx_glr_auto_examples_svm_plot_svm_scale_c.py`.
 
     epsilon : float, default=0.1
          Epsilon in the epsilon-SVR model. It specifies the epsilon-tube
          within which no penalty is associated in the training loss function
          with points predicted within a distance epsilon from the actual
-         value.
+         value. Must be non-negative.
 
     shrinking : bool, default=True
         Whether to use the shrinking heuristic.
@@ -967,10 +1228,6 @@ class SVR(RegressorMixin, BaseLibSVM):
 
     Attributes
     ----------
-    class_weight_ : ndarray of shape (n_classes,)
-        Multipliers of parameter C for each class.
-        Computed based on the ``class_weight`` parameter.
-
     coef_ : ndarray of shape (1, n_features)
         Weights assigned to the features (coefficients in the primal
         problem). This is only available in the case of a linear kernel.
@@ -987,8 +1244,24 @@ class SVR(RegressorMixin, BaseLibSVM):
     intercept_ : ndarray of shape (1,)
         Constants in decision function.
 
-    n_support_ : ndarray of shape (n_classes,), dtype=int32
-        Number of support vectors for each class.
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    n_iter_ : int
+        Number of iterations run by the optimization routine to fit the model.
+
+        .. versionadded:: 1.1
+
+    n_support_ : ndarray of shape (1,), dtype=int32
+        Number of support vectors.
 
     shape_fit_ : tuple of int of shape (n_dimensions_of_X,)
         Array dimensions of training vector ``X``.
@@ -999,6 +1272,23 @@ class SVR(RegressorMixin, BaseLibSVM):
     support_vectors_ : ndarray of shape (n_SV, n_features)
         Support vectors.
 
+    See Also
+    --------
+    NuSVR : Support Vector Machine for regression implemented using libsvm
+        using a parameter to control the number of support vectors.
+
+    LinearSVR : Scalable Linear Support Vector Machine for regression
+        implemented using liblinear.
+
+    References
+    ----------
+    .. [1] `LIBSVM: A Library for Support Vector Machines
+        `_
+
+    .. [2] `Platt, John (1999). "Probabilistic Outputs for Support Vector
+        Machines and Comparisons to Regularized Likelihood Methods"
+        `_
+
     Examples
     --------
     >>> from sklearn.svm import SVR
@@ -1013,61 +1303,46 @@ class SVR(RegressorMixin, BaseLibSVM):
     >>> regr.fit(X, y)
     Pipeline(steps=[('standardscaler', StandardScaler()),
                     ('svr', SVR(epsilon=0.2))])
-
-    See Also
-    --------
-    NuSVR : Support Vector Machine for regression implemented using libsvm
-        using a parameter to control the number of support vectors.
-
-    LinearSVR : Scalable Linear Support Vector Machine for regression
-        implemented using liblinear.
-
-    References
-    ----------
-    .. [1] `LIBSVM: A Library for Support Vector Machines
-        `_
-
-    .. [2] `Platt, John (1999). "Probabilistic outputs for support vector
-        machines and comparison to regularizedlikelihood methods."
-        `_
     """
 
-    _impl = 'epsilon_svr'
-
-    @_deprecate_positional_args
-    def __init__(self, *, kernel='rbf', degree=3, gamma='scale',
-                 coef0=0.0, tol=1e-3, C=1.0, epsilon=0.1, shrinking=True,
-                 cache_size=200, verbose=False, max_iter=-1):
-
+    _impl = "epsilon_svr"
+
+    _parameter_constraints: dict = {**BaseLibSVM._parameter_constraints}
+    for unused_param in ["class_weight", "nu", "probability", "random_state"]:
+        _parameter_constraints.pop(unused_param)
+
+    def __init__(
+        self,
+        *,
+        kernel="rbf",
+        degree=3,
+        gamma="scale",
+        coef0=0.0,
+        tol=1e-3,
+        C=1.0,
+        epsilon=0.1,
+        shrinking=True,
+        cache_size=200,
+        verbose=False,
+        max_iter=-1,
+    ):
         super().__init__(
-            kernel=kernel, degree=degree, gamma=gamma,
-            coef0=coef0, tol=tol, C=C, nu=0., epsilon=epsilon, verbose=verbose,
-            shrinking=shrinking, probability=False, cache_size=cache_size,
-            class_weight=None, max_iter=max_iter, random_state=None)
-
-    # mypy error: Decorated property not supported
-    @deprecated(  # type: ignore
-        "The probA_ attribute is deprecated in version 0.23 and will be "
-        "removed in version 1.0 (renaming of 0.25).")
-    @property
-    def probA_(self):
-        return self._probA
-
-    # mypy error: Decorated property not supported
-    @deprecated(  # type: ignore
-        "The probB_ attribute is deprecated in version 0.23 and will be "
-        "removed in version 1.0 (renaming of 0.25).")
-    @property
-    def probB_(self):
-        return self._probB
-
-    def _more_tags(self):
-        return {
-            '_xfail_checks': {
-                'check_sample_weights_invariance':
-                'zero sample_weight is not equivalent to removing samples',
-            }
-        }
+            kernel=kernel,
+            degree=degree,
+            gamma=gamma,
+            coef0=coef0,
+            tol=tol,
+            C=C,
+            nu=0.0,
+            epsilon=epsilon,
+            verbose=verbose,
+            shrinking=shrinking,
+            probability=False,
+            cache_size=cache_size,
+            class_weight=None,
+            max_iter=max_iter,
+            random_state=None,
+        )
 
 
 class NuSVR(RegressorMixin, BaseLibSVM):
@@ -1089,25 +1364,29 @@ class NuSVR(RegressorMixin, BaseLibSVM):
         default 0.5 will be taken.
 
     C : float, default=1.0
-        Penalty parameter C of the error term.
+        Penalty parameter C of the error term. For an intuitive visualization
+        of the effects of scaling the regularization parameter C, see
+        :ref:`sphx_glr_auto_examples_svm_plot_svm_scale_c.py`.
 
-    kernel : {'linear', 'poly', 'rbf', 'sigmoid', 'precomputed'}, default='rbf'
+    kernel : {'linear', 'poly', 'rbf', 'sigmoid', 'precomputed'} or callable,  \
+        default='rbf'
          Specifies the kernel type to be used in the algorithm.
-         It must be one of 'linear', 'poly', 'rbf', 'sigmoid', 'precomputed' or
-         a callable.
          If none is given, 'rbf' will be used. If a callable is given it is
          used to precompute the kernel matrix.
+         For an intuitive visualization of different kernel types see
+         See :ref:`sphx_glr_auto_examples_svm_plot_svm_regression.py`
 
     degree : int, default=3
         Degree of the polynomial kernel function ('poly').
-        Ignored by all other kernels.
+        Must be non-negative. Ignored by all other kernels.
 
     gamma : {'scale', 'auto'} or float, default='scale'
         Kernel coefficient for 'rbf', 'poly' and 'sigmoid'.
 
         - if ``gamma='scale'`` (default) is passed then it uses
           1 / (n_features * X.var()) as value of gamma,
-        - if 'auto', uses 1 / n_features.
+        - if 'auto', uses 1 / n_features
+        - if float, must be non-negative.
 
         .. versionchanged:: 0.22
            The default value of ``gamma`` changed from 'auto' to 'scale'.
@@ -1136,10 +1415,6 @@ class NuSVR(RegressorMixin, BaseLibSVM):
 
     Attributes
     ----------
-    class_weight_ : ndarray of shape (n_classes,)
-        Multipliers of parameter C for each class.
-        Computed based on the ``class_weight`` parameter.
-
     coef_ : ndarray of shape (1, n_features)
         Weights assigned to the features (coefficients in the primal
         problem). This is only available in the case of a linear kernel.
@@ -1156,8 +1431,24 @@ class NuSVR(RegressorMixin, BaseLibSVM):
     intercept_ : ndarray of shape (1,)
         Constants in decision function.
 
-    n_support_ : ndarray of shape (n_classes,), dtype=int32
-        Number of support vectors for each class.
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    n_iter_ : int
+        Number of iterations run by the optimization routine to fit the model.
+
+        .. versionadded:: 1.1
+
+    n_support_ : ndarray of shape (1,), dtype=int32
+        Number of support vectors.
 
     shape_fit_ : tuple of int of shape (n_dimensions_of_X,)
         Array dimensions of training vector ``X``.
@@ -1168,6 +1459,23 @@ class NuSVR(RegressorMixin, BaseLibSVM):
     support_vectors_ : ndarray of shape (n_SV, n_features)
         Support vectors.
 
+    See Also
+    --------
+    NuSVC : Support Vector Machine for classification implemented with libsvm
+        with a parameter to control the number of support vectors.
+
+    SVR : Epsilon Support Vector Machine for regression implemented with
+        libsvm.
+
+    References
+    ----------
+    .. [1] `LIBSVM: A Library for Support Vector Machines
+        `_
+
+    .. [2] `Platt, John (1999). "Probabilistic Outputs for Support Vector
+        Machines and Comparisons to Regularized Likelihood Methods"
+        `_
+
     Examples
     --------
     >>> from sklearn.svm import NuSVR
@@ -1182,45 +1490,46 @@ class NuSVR(RegressorMixin, BaseLibSVM):
     >>> regr.fit(X, y)
     Pipeline(steps=[('standardscaler', StandardScaler()),
                     ('nusvr', NuSVR(nu=0.1))])
-
-    See Also
-    --------
-    NuSVC : Support Vector Machine for classification implemented with libsvm
-        with a parameter to control the number of support vectors.
-
-    SVR : Epsilon Support Vector Machine for regression implemented with
-        libsvm.
-
-    References
-    ----------
-    .. [1] `LIBSVM: A Library for Support Vector Machines
-        `_
-
-    .. [2] `Platt, John (1999). "Probabilistic outputs for support vector
-        machines and comparison to regularizedlikelihood methods."
-        `_
     """
 
-    _impl = 'nu_svr'
-
-    @_deprecate_positional_args
-    def __init__(self, *, nu=0.5, C=1.0, kernel='rbf', degree=3,
-                 gamma='scale', coef0=0.0, shrinking=True,
-                 tol=1e-3, cache_size=200, verbose=False, max_iter=-1):
-
+    _impl = "nu_svr"
+
+    _parameter_constraints: dict = {**BaseLibSVM._parameter_constraints}
+    for unused_param in ["class_weight", "epsilon", "probability", "random_state"]:
+        _parameter_constraints.pop(unused_param)
+
+    def __init__(
+        self,
+        *,
+        nu=0.5,
+        C=1.0,
+        kernel="rbf",
+        degree=3,
+        gamma="scale",
+        coef0=0.0,
+        shrinking=True,
+        tol=1e-3,
+        cache_size=200,
+        verbose=False,
+        max_iter=-1,
+    ):
         super().__init__(
-            kernel=kernel, degree=degree, gamma=gamma, coef0=coef0,
-            tol=tol, C=C, nu=nu, epsilon=0., shrinking=shrinking,
-            probability=False, cache_size=cache_size, class_weight=None,
-            verbose=verbose, max_iter=max_iter, random_state=None)
-
-    def _more_tags(self):
-        return {
-            '_xfail_checks': {
-                'check_sample_weights_invariance':
-                'zero sample_weight is not equivalent to removing samples',
-            }
-        }
+            kernel=kernel,
+            degree=degree,
+            gamma=gamma,
+            coef0=coef0,
+            tol=tol,
+            C=C,
+            nu=nu,
+            epsilon=0.0,
+            shrinking=shrinking,
+            probability=False,
+            cache_size=cache_size,
+            class_weight=None,
+            verbose=verbose,
+            max_iter=max_iter,
+            random_state=None,
+        )
 
 
 class OneClassSVM(OutlierMixin, BaseLibSVM):
@@ -1234,23 +1543,23 @@ class OneClassSVM(OutlierMixin, BaseLibSVM):
 
     Parameters
     ----------
-    kernel : {'linear', 'poly', 'rbf', 'sigmoid', 'precomputed'}, default='rbf'
+    kernel : {'linear', 'poly', 'rbf', 'sigmoid', 'precomputed'} or callable,  \
+        default='rbf'
          Specifies the kernel type to be used in the algorithm.
-         It must be one of 'linear', 'poly', 'rbf', 'sigmoid', 'precomputed' or
-         a callable.
          If none is given, 'rbf' will be used. If a callable is given it is
          used to precompute the kernel matrix.
 
     degree : int, default=3
         Degree of the polynomial kernel function ('poly').
-        Ignored by all other kernels.
+        Must be non-negative. Ignored by all other kernels.
 
     gamma : {'scale', 'auto'} or float, default='scale'
         Kernel coefficient for 'rbf', 'poly' and 'sigmoid'.
 
         - if ``gamma='scale'`` (default) is passed then it uses
           1 / (n_features * X.var()) as value of gamma,
-        - if 'auto', uses 1 / n_features.
+        - if 'auto', uses 1 / n_features
+        - if float, must be non-negative.
 
         .. versionchanged:: 0.22
            The default value of ``gamma`` changed from 'auto' to 'scale'.
@@ -1285,10 +1594,6 @@ class OneClassSVM(OutlierMixin, BaseLibSVM):
 
     Attributes
     ----------
-    class_weight_ : ndarray of shape (n_classes,)
-        Multipliers of parameter C for each class.
-        Computed based on the ``class_weight`` parameter.
-
     coef_ : ndarray of shape (1, n_features)
         Weights assigned to the features (coefficients in the primal
         problem). This is only available in the case of a linear kernel.
@@ -1305,6 +1610,22 @@ class OneClassSVM(OutlierMixin, BaseLibSVM):
     intercept_ : ndarray of shape (1,)
         Constant in the decision function.
 
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+        .. versionadded:: 0.24
+
+    feature_names_in_ : ndarray of shape (`n_features_in_`,)
+        Names of features seen during :term:`fit`. Defined only when `X`
+        has feature names that are all strings.
+
+        .. versionadded:: 1.0
+
+    n_iter_ : int
+        Number of iterations run by the optimization routine to fit the model.
+
+        .. versionadded:: 1.1
+
     n_support_ : ndarray of shape (n_classes,), dtype=int32
         Number of support vectors for each class.
 
@@ -1325,6 +1646,14 @@ class OneClassSVM(OutlierMixin, BaseLibSVM):
     support_vectors_ : ndarray of shape (n_SV, n_features)
         Support vectors.
 
+    See Also
+    --------
+    sklearn.linear_model.SGDOneClassSVM : Solves linear One-Class SVM using
+        Stochastic Gradient Descent.
+    sklearn.neighbors.LocalOutlierFactor : Unsupervised Outlier Detection using
+        Local Outlier Factor (LOF).
+    sklearn.ensemble.IsolationForest : Isolation Forest Algorithm.
+
     Examples
     --------
     >>> from sklearn.svm import OneClassSVM
@@ -1333,48 +1662,76 @@ class OneClassSVM(OutlierMixin, BaseLibSVM):
     >>> clf.predict(X)
     array([-1,  1,  1,  1, -1])
     >>> clf.score_samples(X)
-    array([1.7798..., 2.0547..., 2.0556..., 2.0561..., 1.7332...])
-    """
+    array([1.7798, 2.0547, 2.0556, 2.0561, 1.7332])
 
-    _impl = 'one_class'
-
-    @_deprecate_positional_args
-    def __init__(self, *, kernel='rbf', degree=3, gamma='scale',
-                 coef0=0.0, tol=1e-3, nu=0.5, shrinking=True, cache_size=200,
-                 verbose=False, max_iter=-1):
+    For a more extended example,
+    see :ref:`sphx_glr_auto_examples_applications_plot_species_distribution_modeling.py`
+    """
 
+    _impl = "one_class"
+
+    _parameter_constraints: dict = {**BaseLibSVM._parameter_constraints}
+    for unused_param in ["C", "class_weight", "epsilon", "probability", "random_state"]:
+        _parameter_constraints.pop(unused_param)
+
+    def __init__(
+        self,
+        *,
+        kernel="rbf",
+        degree=3,
+        gamma="scale",
+        coef0=0.0,
+        tol=1e-3,
+        nu=0.5,
+        shrinking=True,
+        cache_size=200,
+        verbose=False,
+        max_iter=-1,
+    ):
         super().__init__(
-            kernel, degree, gamma, coef0, tol, 0., nu, 0.,
-            shrinking, False, cache_size, None, verbose, max_iter,
-            random_state=None)
-
-    def fit(self, X, y=None, sample_weight=None, **params):
-        """Detects the soft boundary of the set of samples X.
+            kernel,
+            degree,
+            gamma,
+            coef0,
+            tol,
+            0.0,
+            nu,
+            0.0,
+            shrinking,
+            False,
+            cache_size,
+            None,
+            verbose,
+            max_iter,
+            random_state=None,
+        )
+
+    def fit(self, X, y=None, sample_weight=None):
+        """Detect the soft boundary of the set of samples X.
 
         Parameters
         ----------
         X : {array-like, sparse matrix} of shape (n_samples, n_features)
-            Set of samples, where n_samples is the number of samples and
-            n_features is the number of features.
+            Set of samples, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
+
+        y : Ignored
+            Not used, present for API consistency by convention.
 
         sample_weight : array-like of shape (n_samples,), default=None
             Per-sample weights. Rescale C per sample. Higher weights
             force the classifier to put more emphasis on these points.
 
-        y : Ignored
-            not used, present for API consistency by convention.
-
         Returns
         -------
         self : object
+            Fitted estimator.
 
         Notes
         -----
         If X is not a C-ordered contiguous array it is copied.
-
         """
-        super().fit(X, np.ones(_num_samples(X)),
-                    sample_weight=sample_weight, **params)
+        super().fit(X, np.ones(_num_samples(X)), sample_weight=sample_weight)
         self.offset_ = -self._intercept_
         return self
 
@@ -1430,27 +1787,3 @@ def predict(self, X):
         """
         y = super().predict(X)
         return np.asarray(y, dtype=np.intp)
-
-    # mypy error: Decorated property not supported
-    @deprecated(  # type: ignore
-        "The probA_ attribute is deprecated in version 0.23 and will be "
-        "removed in version 1.0.")
-    @property
-    def probA_(self):
-        return self._probA
-
-    # mypy error: Decorated property not supported
-    @deprecated(  # type: ignore
-        "The probB_ attribute is deprecated in version 0.23 and will be "
-        "removed in version 1.0.")
-    @property
-    def probB_(self):
-        return self._probB
-
-    def _more_tags(self):
-        return {
-            '_xfail_checks': {
-                'check_sample_weights_invariance':
-                'zero sample_weight is not equivalent to removing samples',
-            }
-        }
diff --git a/sklearn/svm/_liblinear.pxi b/sklearn/svm/_liblinear.pxi
index 148bf694dab4f..0df269b070f5c 100644
--- a/sklearn/svm/_liblinear.pxi
+++ b/sklearn/svm/_liblinear.pxi
@@ -1,8 +1,10 @@
+from ..utils._typedefs cimport intp_t
+
 cdef extern from "_cython_blas_helpers.h":
-    ctypedef double (*dot_func)(int, double*, int, double*, int)
-    ctypedef void (*axpy_func)(int, double, double*, int, double*, int)
-    ctypedef void (*scal_func)(int, double, double*, int)
-    ctypedef double (*nrm2_func)(int, double*, int)
+    ctypedef double (*dot_func)(int, const double*, int, const double*, int)
+    ctypedef void (*axpy_func)(int, double, const double*, int, double*, int)
+    ctypedef void (*scal_func)(int, double, const double*, int)
+    ctypedef double (*nrm2_func)(int, const double*, int)
     cdef struct BlasFunctions:
         dot_func dot
         axpy_func axpy
@@ -33,7 +35,7 @@ cdef extern from "liblinear_helper.c":
     problem *set_problem (char *, int, int, int, int, double, char *, char *)
     problem *csr_set_problem (char *, int, char *, char *, int, int, int, double, char *, char *)
 
-    model *set_model(parameter *, char *, np.npy_intp *, char *, double)
+    model *set_model(parameter *, char *, intp_t *, char *, double)
 
     double get_bias(model *)
     void free_problem (problem *)
diff --git a/sklearn/svm/_liblinear.pyx b/sklearn/svm/_liblinear.pyx
index 9dd15e0716c7f..6d5347e746384 100644
--- a/sklearn/svm/_liblinear.pyx
+++ b/sklearn/svm/_liblinear.pyx
@@ -5,52 +5,100 @@ Author: fabian.pedregosa@inria.fr
 """
 
 import  numpy as np
-cimport numpy as np
 
 from ..utils._cython_blas cimport _dot, _axpy, _scal, _nrm2
+from ..utils._typedefs cimport float32_t, float64_t, int32_t
 
 include "_liblinear.pxi"
 
-np.import_array()
 
-
-def train_wrap(X, np.ndarray[np.float64_t, ndim=1, mode='c'] Y,
-               bint is_sparse, int solver_type, double eps, double bias,
-               double C, np.ndarray[np.float64_t, ndim=1] class_weight,
-               int max_iter, unsigned random_seed, double epsilon,
-               np.ndarray[np.float64_t, ndim=1, mode='c'] sample_weight):
+def train_wrap(
+    object X,
+    const float64_t[::1] Y,
+    bint is_sparse,
+    int solver_type,
+    double eps,
+    double bias,
+    double C,
+    const float64_t[:] class_weight,
+    int max_iter,
+    unsigned random_seed,
+    double epsilon,
+    const float64_t[::1] sample_weight
+):
     cdef parameter *param
     cdef problem *problem
     cdef model *model
     cdef char_const_ptr error_msg
     cdef int len_w
+    cdef bint X_has_type_float64 = X.dtype == np.float64
+    cdef char * X_data_bytes_ptr
+    cdef const float64_t[::1] X_data_64
+    cdef const float32_t[::1] X_data_32
+    cdef const int32_t[::1] X_indices
+    cdef const int32_t[::1] X_indptr
 
     if is_sparse:
+        X_indices = X.indices
+        X_indptr = X.indptr
+        if X_has_type_float64:
+            X_data_64 = X.data
+            X_data_bytes_ptr =  &X_data_64[0]
+        else:
+            X_data_32 = X.data
+            X_data_bytes_ptr =  &X_data_32[0]
+
         problem = csr_set_problem(
-                (X.data).data, X.dtype == np.float64,
-                (X.indices).data,
-                (X.indptr).data,
-                (X.shape[0]), (X.shape[1]),
-                (X.nnz), bias, sample_weight.data, Y.data)
+            X_data_bytes_ptr,
+            X_has_type_float64,
+             &X_indices[0],
+             &X_indptr[0],
+            (X.shape[0]),
+            (X.shape[1]),
+            (X.nnz),
+            bias,
+             &sample_weight[0],
+             &Y[0]
+        )
     else:
-        problem = set_problem(
-                (X).data, X.dtype == np.float64,
-                (X.shape[0]), (X.shape[1]),
-                (np.count_nonzero(X)), bias, sample_weight.data,
-                Y.data)
+        X_as_1d_array = X.reshape(-1)
+        if X_has_type_float64:
+            X_data_64 = X_as_1d_array
+            X_data_bytes_ptr =  &X_data_64[0]
+        else:
+            X_data_32 = X_as_1d_array
+            X_data_bytes_ptr =  &X_data_32[0]
 
-    cdef np.ndarray[np.int32_t, ndim=1, mode='c'] \
-        class_weight_label = np.arange(class_weight.shape[0], dtype=np.intc)
-    param = set_parameter(solver_type, eps, C, class_weight.shape[0],
-                          class_weight_label.data, class_weight.data,
-                          max_iter, random_seed, epsilon)
+        problem = set_problem(
+            X_data_bytes_ptr,
+            X_has_type_float64,
+            (X.shape[0]),
+            (X.shape[1]),
+            (np.count_nonzero(X)),
+            bias,
+             &sample_weight[0],
+             &Y[0]
+        )
+
+    cdef int32_t[::1] class_weight_label = np.arange(class_weight.shape[0], dtype=np.intc)
+    param = set_parameter(
+        solver_type,
+        eps,
+        C,
+        class_weight.shape[0],
+         &class_weight_label[0] if class_weight_label.size > 0 else NULL,
+         &class_weight[0] if class_weight.size > 0 else NULL,
+        max_iter,
+        random_seed,
+        epsilon
+    )
 
     error_msg = check_parameter(problem, param)
     if error_msg:
         free_problem(problem)
         free_parameter(param)
         raise ValueError(error_msg)
-    
+
     cdef BlasFunctions blas_functions
     blas_functions.dot = _dot[double]
     blas_functions.axpy = _axpy[double]
@@ -61,34 +109,35 @@ def train_wrap(X, np.ndarray[np.float64_t, ndim=1, mode='c'] Y,
     with nogil:
         model = train(problem, param, &blas_functions)
 
-    ### FREE
+    # FREE
     free_problem(problem)
     free_parameter(param)
     # destroy_param(param)  don't call this or it will destroy class_weight_label and class_weight
 
     # coef matrix holder created as fortran since that's what's used in liblinear
-    cdef np.ndarray[np.float64_t, ndim=2, mode='fortran'] w
+    cdef float64_t[::1, :] w
     cdef int nr_class = get_nr_class(model)
 
     cdef int labels_ = nr_class
     if nr_class == 2:
         labels_ = 1
-    cdef np.ndarray[np.int32_t, ndim=1, mode='c'] n_iter = np.zeros(labels_, dtype=np.intc)
-    get_n_iter(model, n_iter.data)
+    cdef int32_t[::1] n_iter = np.zeros(labels_, dtype=np.intc)
+    get_n_iter(model,  &n_iter[0])
 
     cdef int nr_feature = get_nr_feature(model)
-    if bias > 0: nr_feature = nr_feature + 1
+    if bias > 0:
+        nr_feature = nr_feature + 1
     if nr_class == 2 and solver_type != 4:  # solver is not Crammer-Singer
-        w = np.empty((1, nr_feature),order='F')
-        copy_w(w.data, model, nr_feature)
+        w = np.empty((1, nr_feature), order='F')
+        copy_w(&w[0, 0], model, nr_feature)
     else:
         len_w = (nr_class) * nr_feature
-        w = np.empty((nr_class, nr_feature),order='F')
-        copy_w(w.data, model, len_w)
+        w = np.empty((nr_class, nr_feature), order='F')
+        copy_w(&w[0, 0], model, len_w)
 
     free_and_destroy_model(&model)
 
-    return w, n_iter
+    return w.base, n_iter.base
 
 
 def set_verbosity_wrap(int verbosity):
diff --git a/sklearn/svm/_libsvm.pxi b/sklearn/svm/_libsvm.pxi
index ab7d212e3ba1a..74ddfd66c538e 100644
--- a/sklearn/svm/_libsvm.pxi
+++ b/sklearn/svm/_libsvm.pxi
@@ -1,7 +1,9 @@
 ################################################################################
 # Includes
+from ..utils._typedefs cimport intp_t
+
 cdef extern from "_svm_cython_blas_helpers.h":
-    ctypedef double (*dot_func)(int, double*, int, double*, int)
+    ctypedef double (*dot_func)(int, const double*, int, const double*, int)
     cdef struct BlasFunctions:
         dot_func dot
 
@@ -12,29 +14,29 @@ cdef extern from "svm.h":
     cdef struct svm_parameter:
         int svm_type
         int kernel_type
-        int degree	# for poly
-        double gamma	# for poly/rbf/sigmoid
-        double coef0	# for poly/sigmoid
+        int degree    # for poly
+        double gamma  # for poly/rbf/sigmoid
+        double coef0  # for poly/sigmoid
 
         # these are for training only
-        double cache_size # in MB
-        double eps	# stopping criteria
-        double C	# for C_SVC, EPSILON_SVR and NU_SVR
-        int nr_weight		# for C_SVC
-        int *weight_label	# for C_SVC
-        double* weight		# for C_SVC
-        double nu	# for NU_SVC, ONE_CLASS, and NU_SVR
-        double p	# for EPSILON_SVR
-        int shrinking	# use the shrinking heuristics
-        int probability # do probability estimates
-        int max_iter  # ceiling on Solver runtime
-        int random_seed  # seed for random generator in probability estimation
+        double cache_size  # in MB
+        double eps         # stopping criteria
+        double C           # for C_SVC, EPSILON_SVR and NU_SVR
+        int nr_weight      # for C_SVC
+        int *weight_label  # for C_SVC
+        double* weight     # for C_SVC
+        double nu          # for NU_SVC, ONE_CLASS, and NU_SVR
+        double p           # for EPSILON_SVR
+        int shrinking      # use the shrinking heuristics
+        int probability    # do probability estimates
+        int max_iter       # ceiling on Solver runtime
+        int random_seed    # seed for random generator in probability estimation
 
     cdef struct svm_problem:
         int l
         double *y
         svm_node *x
-        double *W # instance weights
+        double *W  # instance weights
 
     char *svm_check_parameter(svm_problem *, svm_parameter *)
     svm_model *svm_train(svm_problem *, svm_parameter *, int *, BlasFunctions *) nogil
@@ -44,29 +46,30 @@ cdef extern from "svm.h":
 
 cdef extern from "libsvm_helper.c":
     # this file contains methods for accessing libsvm 'hidden' fields
-    svm_node **dense_to_sparse (char *, np.npy_intp *)
+    svm_node **dense_to_sparse (char *, intp_t *)
     void set_parameter (svm_parameter *, int , int , int , double, double ,
-                                  double , double , double , double,
-                                  double, int, int, int, char *, char *, int,
-                                  int)
-    void set_problem (svm_problem *, char *, char *, char *, np.npy_intp *, int)
+                        double , double , double , double,
+                        double, int, int, int, char *, char *, int,
+                        int)
+    void set_problem (svm_problem *, char *, char *, char *, intp_t *, int)
 
-    svm_model *set_model (svm_parameter *, int, char *, np.npy_intp *,
-                         char *, np.npy_intp *, np.npy_intp *, char *,
-                         char *, char *, char *, char *)
+    svm_model *set_model (svm_parameter *, int, char *, intp_t *,
+                          char *, intp_t *, intp_t *, char *,
+                          char *, char *, char *, char *)
 
     void copy_sv_coef   (char *, svm_model *)
-    void copy_intercept (char *, svm_model *, np.npy_intp *)
-    void copy_SV        (char *, svm_model *, np.npy_intp *)
+    void copy_n_iter  (char *, svm_model *)
+    void copy_intercept (char *, svm_model *, intp_t *)
+    void copy_SV        (char *, svm_model *, intp_t *)
     int copy_support (char *data, svm_model *model)
-    int copy_predict (char *, svm_model *, np.npy_intp *, char *, BlasFunctions *) nogil
-    int copy_predict_proba (char *, svm_model *, np.npy_intp *, char *, BlasFunctions *) nogil
-    int copy_predict_values(char *, svm_model *, np.npy_intp *, char *, int, BlasFunctions *) nogil
+    int copy_predict (char *, svm_model *, intp_t *, char *, BlasFunctions *) nogil
+    int copy_predict_proba (char *, svm_model *, intp_t *, char *, BlasFunctions *) nogil
+    int copy_predict_values(char *, svm_model *, intp_t *, char *, int, BlasFunctions *) nogil
     void copy_nSV     (char *, svm_model *)
-    void copy_probA   (char *, svm_model *, np.npy_intp *)
-    void copy_probB   (char *, svm_model *, np.npy_intp *)
-    np.npy_intp  get_l  (svm_model *)
-    np.npy_intp  get_nr (svm_model *)
+    void copy_probA   (char *, svm_model *, intp_t *)
+    void copy_probB   (char *, svm_model *, intp_t *)
+    intp_t  get_l  (svm_model *)
+    intp_t  get_nr (svm_model *)
     int  free_problem   (svm_problem *)
     int  free_model     (svm_model *)
     void set_verbosity(int)
diff --git a/sklearn/svm/_libsvm.pyx b/sklearn/svm/_libsvm.pyx
index 9488bda4ccf58..be0a0826c3736 100644
--- a/sklearn/svm/_libsvm.pyx
+++ b/sklearn/svm/_libsvm.pyx
@@ -18,10 +18,7 @@ where no sort of memory checks are done.
 
 Notes
 -----
-Maybe we could speed it a bit further by decorating functions with
-@cython.boundscheck(False), but probably it is not worth since all
-work is done in lisvm_helper.c
-Also, the signature mode='c' is somewhat superficial, since we already
+The signature mode='c' is somewhat superficial, since we already
 check that arrays are C-contiguous in svm.py
 
 Authors
@@ -30,11 +27,10 @@ Authors
       Gael Varoquaux 
 """
 
-import warnings
 import  numpy as np
-cimport numpy as np
 from libc.stdlib cimport free
 from ..utils._cython_blas cimport _dot
+from ..utils._typedefs cimport float64_t, int32_t, intp_t
 
 include "_libsvm.pxi"
 
@@ -42,8 +38,6 @@ cdef extern from *:
     ctypedef struct svm_parameter:
         pass
 
-np.import_array()
-
 
 ################################################################################
 # Internal variables
@@ -54,19 +48,25 @@ LIBSVM_KERNEL_TYPES = ['linear', 'poly', 'rbf', 'sigmoid', 'precomputed']
 # Wrapper functions
 
 def fit(
-    np.ndarray[np.float64_t, ndim=2, mode='c'] X,
-    np.ndarray[np.float64_t, ndim=1, mode='c'] Y,
-    int svm_type=0, kernel='rbf', int degree=3,
-    double gamma=0.1, double coef0=0., double tol=1e-3,
-    double C=1., double nu=0.5, double epsilon=0.1,
-    np.ndarray[np.float64_t, ndim=1, mode='c']
-        class_weight=np.empty(0),
-    np.ndarray[np.float64_t, ndim=1, mode='c']
-        sample_weight=np.empty(0),
-    int shrinking=1, int probability=0,
+    const float64_t[:, ::1] X,
+    const float64_t[::1] Y,
+    int svm_type=0,
+    kernel='rbf',
+    int degree=3,
+    double gamma=0.1,
+    double coef0=0.0,
+    double tol=1e-3,
+    double C=1.0,
+    double nu=0.5,
+    double epsilon=0.1,
+    const float64_t[::1] class_weight=np.empty(0),
+    const float64_t[::1] sample_weight=np.empty(0),
+    int shrinking=1,
+    int probability=0,
     double cache_size=100.,
     int max_iter=-1,
-    int random_seed=0):
+    int random_seed=0,
+):
     """
     Train the model using libsvm (low-level method)
 
@@ -155,35 +155,59 @@ def fit(
 
     probA, probB : array of shape (n_class*(n_class-1)/2,)
         Probability estimates, empty array for probability=False.
+
+    n_iter : ndarray of shape (max(1, (n_class * (n_class - 1) // 2)),)
+        Number of iterations run by the optimization routine to fit the model.
     """
 
     cdef svm_parameter param
     cdef svm_problem problem
     cdef svm_model *model
     cdef const char *error_msg
-    cdef np.npy_intp SV_len
-    cdef np.npy_intp nr
-
+    cdef intp_t SV_len
 
     if len(sample_weight) == 0:
         sample_weight = np.ones(X.shape[0], dtype=np.float64)
     else:
-        assert sample_weight.shape[0] == X.shape[0], \
-               "sample_weight and X have incompatible shapes: " + \
-               "sample_weight has %s samples while X has %s" % \
-               (sample_weight.shape[0], X.shape[0])
+        assert sample_weight.shape[0] == X.shape[0], (
+            f"sample_weight and X have incompatible shapes: sample_weight has "
+            f"{sample_weight.shape[0]} samples while X has {X.shape[0]}"
+        )
 
     kernel_index = LIBSVM_KERNEL_TYPES.index(kernel)
     set_problem(
-        &problem, X.data, Y.data, sample_weight.data, X.shape, kernel_index)
+        &problem,
+         &X[0, 0],
+         &Y[0],
+         &sample_weight[0],
+         X.shape,
+        kernel_index,
+    )
     if problem.x == NULL:
         raise MemoryError("Seems we've run out of memory")
-    cdef np.ndarray[np.int32_t, ndim=1, mode='c'] \
-        class_weight_label = np.arange(class_weight.shape[0], dtype=np.int32)
+    cdef int32_t[::1] class_weight_label = np.arange(
+        class_weight.shape[0], dtype=np.int32
+    )
     set_parameter(
-        ¶m, svm_type, kernel_index, degree, gamma, coef0, nu, cache_size,
-        C, tol, epsilon, shrinking, probability,  class_weight.shape[0],
-        class_weight_label.data, class_weight.data, max_iter, random_seed)
+        ¶m,
+        svm_type,
+        kernel_index,
+        degree,
+        gamma,
+        coef0,
+        nu,
+        cache_size,
+        C,
+        tol,
+        epsilon,
+        shrinking,
+        probability,
+         class_weight.shape[0],
+         &class_weight_label[0] if class_weight_label.size > 0 else NULL,
+         &class_weight[0] if class_weight.size > 0 else NULL,
+        max_iter,
+        random_seed,
+    )
 
     error_msg = svm_check_parameter(&problem, ¶m)
     if error_msg:
@@ -199,50 +223,56 @@ def fit(
 
     # from here until the end, we just copy the data returned by
     # svm_train
-    SV_len  = get_l(model)
+    SV_len = get_l(model)
     n_class = get_nr(model)
 
-    cdef np.ndarray[np.float64_t, ndim=2, mode='c'] sv_coef
-    sv_coef = np.empty((n_class-1, SV_len), dtype=np.float64)
-    copy_sv_coef (sv_coef.data, model)
+    cdef int[::1] n_iter = np.empty(max(1, n_class * (n_class - 1) // 2), dtype=np.intc)
+    copy_n_iter( &n_iter[0], model)
+
+    cdef float64_t[:, ::1] sv_coef = np.empty((n_class-1, SV_len), dtype=np.float64)
+    copy_sv_coef( &sv_coef[0, 0] if sv_coef.size > 0 else NULL, model)
 
     # the intercept is just model.rho but with sign changed
-    cdef np.ndarray[np.float64_t, ndim=1, mode='c'] intercept
-    intercept = np.empty(int((n_class*(n_class-1))/2), dtype=np.float64)
-    copy_intercept (intercept.data, model, intercept.shape)
+    cdef float64_t[::1] intercept = np.empty(
+        int((n_class*(n_class-1))/2), dtype=np.float64
+    )
+    copy_intercept( &intercept[0], model,  intercept.shape)
 
-    cdef np.ndarray[np.int32_t, ndim=1, mode='c'] support
-    support = np.empty (SV_len, dtype=np.int32)
-    copy_support (support.data, model)
+    cdef int32_t[::1] support = np.empty(SV_len, dtype=np.int32)
+    copy_support( &support[0] if support.size > 0 else NULL, model)
 
     # copy model.SV
-    cdef np.ndarray[np.float64_t, ndim=2, mode='c'] support_vectors
+    cdef float64_t[:, ::1] support_vectors
     if kernel_index == 4:
         # precomputed kernel
         support_vectors = np.empty((0, 0), dtype=np.float64)
     else:
         support_vectors = np.empty((SV_len, X.shape[1]), dtype=np.float64)
-        copy_SV(support_vectors.data, model, support_vectors.shape)
+        copy_SV(
+             &support_vectors[0, 0] if support_vectors.size > 0 else NULL,
+            model,
+             support_vectors.shape,
+        )
 
-    cdef np.ndarray[np.int32_t, ndim=1, mode='c'] n_class_SV
+    cdef int32_t[::1] n_class_SV
     if svm_type == 0 or svm_type == 1:
         n_class_SV = np.empty(n_class, dtype=np.int32)
-        copy_nSV(n_class_SV.data, model)
+        copy_nSV( &n_class_SV[0] if n_class_SV.size > 0 else NULL, model)
     else:
         # OneClass and SVR are considered to have 2 classes
         n_class_SV = np.array([SV_len, SV_len], dtype=np.int32)
 
-    cdef np.ndarray[np.float64_t, ndim=1, mode='c'] probA
-    cdef np.ndarray[np.float64_t, ndim=1, mode='c'] probB
+    cdef float64_t[::1] probA
+    cdef float64_t[::1] probB
     if probability != 0:
-        if svm_type < 2: # SVC and NuSVC
+        if svm_type < 2:  # SVC and NuSVC
             probA = np.empty(int(n_class*(n_class-1)/2), dtype=np.float64)
             probB = np.empty(int(n_class*(n_class-1)/2), dtype=np.float64)
-            copy_probB(probB.data, model, probB.shape)
+            copy_probB( &probB[0], model,  probB.shape)
         else:
             probA = np.empty(1, dtype=np.float64)
             probB = np.empty(0, dtype=np.float64)
-        copy_probA(probA.data, model, probA.shape)
+        copy_probA( &probA[0], model,  probA.shape)
     else:
         probA = np.empty(0, dtype=np.float64)
         probB = np.empty(0, dtype=np.float64)
@@ -250,47 +280,85 @@ def fit(
     svm_free_and_destroy_model(&model)
     free(problem.x)
 
-    return (support, support_vectors, n_class_SV, sv_coef, intercept,
-           probA, probB, fit_status)
+    return (
+        support.base,
+        support_vectors.base,
+        n_class_SV.base,
+        sv_coef.base,
+        intercept.base,
+        probA.base,
+        probB.base,
+        fit_status,
+        n_iter.base,
+    )
 
 
 cdef void set_predict_params(
-    svm_parameter *param, int svm_type, kernel, int degree, double gamma,
-    double coef0, double cache_size, int probability, int nr_weight,
-    char *weight_label, char *weight) except *:
+    svm_parameter *param,
+    int svm_type,
+    kernel,
+    int degree,
+    double gamma,
+    double coef0,
+    double cache_size,
+    int probability,
+    int nr_weight,
+    char *weight_label,
+    char *weight,
+) except *:
     """Fill param with prediction time-only parameters."""
 
     # training-time only parameters
-    cdef double C = .0
-    cdef double epsilon = .1
+    cdef double C = 0.0
+    cdef double epsilon = 0.1
     cdef int max_iter = 0
-    cdef double nu = .5
+    cdef double nu = 0.5
     cdef int shrinking = 0
-    cdef double tol = .1
+    cdef double tol = 0.1
     cdef int random_seed = -1
 
     kernel_index = LIBSVM_KERNEL_TYPES.index(kernel)
 
-    set_parameter(param, svm_type, kernel_index, degree, gamma, coef0, nu,
-                         cache_size, C, tol, epsilon, shrinking, probability,
-                         nr_weight, weight_label, weight, max_iter, random_seed)
-
-
-def predict(np.ndarray[np.float64_t, ndim=2, mode='c'] X,
-            np.ndarray[np.int32_t, ndim=1, mode='c'] support,
-            np.ndarray[np.float64_t, ndim=2, mode='c'] SV,
-            np.ndarray[np.int32_t, ndim=1, mode='c'] nSV,
-            np.ndarray[np.float64_t, ndim=2, mode='c'] sv_coef,
-            np.ndarray[np.float64_t, ndim=1, mode='c'] intercept,
-            np.ndarray[np.float64_t, ndim=1, mode='c'] probA=np.empty(0),
-            np.ndarray[np.float64_t, ndim=1, mode='c'] probB=np.empty(0),
-            int svm_type=0, kernel='rbf', int degree=3,
-            double gamma=0.1, double coef0=0.,
-            np.ndarray[np.float64_t, ndim=1, mode='c']
-                class_weight=np.empty(0),
-            np.ndarray[np.float64_t, ndim=1, mode='c']
-                sample_weight=np.empty(0),
-            double cache_size=100.):
+    set_parameter(
+        param,
+        svm_type,
+        kernel_index,
+        degree,
+        gamma,
+        coef0,
+        nu,
+        cache_size,
+        C,
+        tol,
+        epsilon,
+        shrinking,
+        probability,
+        nr_weight,
+        weight_label,
+        weight,
+        max_iter,
+        random_seed,
+    )
+
+
+def predict(
+    const float64_t[:, ::1] X,
+    const int32_t[::1] support,
+    const float64_t[:, ::1] SV,
+    const int32_t[::1] nSV,
+    const float64_t[:, ::1] sv_coef,
+    const float64_t[::1] intercept,
+    const float64_t[::1] probA=np.empty(0),
+    const float64_t[::1] probB=np.empty(0),
+    int svm_type=0,
+    kernel='rbf',
+    int degree=3,
+    double gamma=0.1,
+    double coef0=0.0,
+    const float64_t[::1] class_weight=np.empty(0),
+    const float64_t[::1] sample_weight=np.empty(0),
+    double cache_size=100.0,
+):
     """
     Predict target values of X given a model (low-level method)
 
@@ -340,51 +408,81 @@ def predict(np.ndarray[np.float64_t, ndim=2, mode='c'] X,
     dec_values : array
         Predicted values.
     """
-    cdef np.ndarray[np.float64_t, ndim=1, mode='c'] dec_values
+    cdef float64_t[::1] dec_values
     cdef svm_parameter param
     cdef svm_model *model
     cdef int rv
 
-    cdef np.ndarray[np.int32_t, ndim=1, mode='c'] \
-        class_weight_label = np.arange(class_weight.shape[0], dtype=np.int32)
-
-    set_predict_params(¶m, svm_type, kernel, degree, gamma, coef0,
-                       cache_size, 0, class_weight.shape[0],
-                       class_weight_label.data, class_weight.data)
-    model = set_model(¶m,  nSV.shape[0], SV.data, SV.shape,
-                      support.data, support.shape, sv_coef.strides,
-                      sv_coef.data, intercept.data, nSV.data, probA.data, probB.data)
+    cdef int32_t[::1] class_weight_label = np.arange(
+        class_weight.shape[0], dtype=np.int32
+    )
+
+    set_predict_params(
+        ¶m,
+        svm_type,
+        kernel,
+        degree,
+        gamma,
+        coef0,
+        cache_size,
+        0,
+        class_weight.shape[0],
+         &class_weight_label[0] if class_weight_label.size > 0 else NULL,
+         &class_weight[0] if class_weight.size > 0 else NULL,
+    )
+    model = set_model(
+        ¶m,
+         nSV.shape[0],
+         &SV[0, 0] if SV.size > 0 else NULL,
+         SV.shape,
+         &support[0] if support.size > 0 else NULL,
+         support.shape,
+         sv_coef.strides,
+         &sv_coef[0, 0] if sv_coef.size > 0 else NULL,
+         &intercept[0],
+         &nSV[0],
+         &probA[0] if probA.size > 0 else NULL,
+         &probB[0] if probB.size > 0 else NULL,
+    )
     cdef BlasFunctions blas_functions
     blas_functions.dot = _dot[double]
-    #TODO: use check_model
+    # TODO: use check_model
     try:
         dec_values = np.empty(X.shape[0])
         with nogil:
-            rv = copy_predict(X.data, model, X.shape, dec_values.data, &blas_functions)
+            rv = copy_predict(
+                 &X[0, 0],
+                model,
+                 X.shape,
+                 &dec_values[0],
+                &blas_functions,
+            )
         if rv < 0:
             raise MemoryError("We've run out of memory")
     finally:
         free_model(model)
 
-    return dec_values
+    return dec_values.base
 
 
 def predict_proba(
-    np.ndarray[np.float64_t, ndim=2, mode='c'] X,
-    np.ndarray[np.int32_t, ndim=1, mode='c'] support,
-    np.ndarray[np.float64_t, ndim=2, mode='c'] SV,
-    np.ndarray[np.int32_t, ndim=1, mode='c'] nSV,
-    np.ndarray[np.float64_t, ndim=2, mode='c'] sv_coef,
-    np.ndarray[np.float64_t, ndim=1, mode='c'] intercept,
-    np.ndarray[np.float64_t, ndim=1, mode='c'] probA=np.empty(0),
-    np.ndarray[np.float64_t, ndim=1, mode='c'] probB=np.empty(0),
-    int svm_type=0, kernel='rbf', int degree=3,
-    double gamma=0.1, double coef0=0.,
-    np.ndarray[np.float64_t, ndim=1, mode='c']
-        class_weight=np.empty(0),
-    np.ndarray[np.float64_t, ndim=1, mode='c']
-        sample_weight=np.empty(0),
-    double cache_size=100.):
+    const float64_t[:, ::1] X,
+    const int32_t[::1] support,
+    const float64_t[:, ::1] SV,
+    const int32_t[::1] nSV,
+    float64_t[:, ::1] sv_coef,
+    float64_t[::1] intercept,
+    float64_t[::1] probA=np.empty(0),
+    float64_t[::1] probB=np.empty(0),
+    int svm_type=0,
+    kernel='rbf',
+    int degree=3,
+    double gamma=0.1,
+    double coef0=0.0,
+    float64_t[::1] class_weight=np.empty(0),
+    float64_t[::1] sample_weight=np.empty(0),
+    double cache_size=100.0,
+):
     """
     Predict probabilities
 
@@ -444,52 +542,81 @@ def predict_proba(
     dec_values : array
         Predicted values.
     """
-    cdef np.ndarray[np.float64_t, ndim=2, mode='c'] dec_values
+    cdef float64_t[:, ::1] dec_values
     cdef svm_parameter param
     cdef svm_model *model
-    cdef np.ndarray[np.int32_t, ndim=1, mode='c'] \
-        class_weight_label = np.arange(class_weight.shape[0], dtype=np.int32)
+    cdef int32_t[::1] class_weight_label = np.arange(
+        class_weight.shape[0], dtype=np.int32
+    )
     cdef int rv
 
-    set_predict_params(¶m, svm_type, kernel, degree, gamma, coef0,
-                       cache_size, 1, class_weight.shape[0],
-                       class_weight_label.data, class_weight.data)
-    model = set_model(¶m,  nSV.shape[0], SV.data, SV.shape,
-                      support.data, support.shape, sv_coef.strides,
-                      sv_coef.data, intercept.data, nSV.data,
-                      probA.data, probB.data)
-
-    cdef np.npy_intp n_class = get_nr(model)
+    set_predict_params(
+        ¶m,
+        svm_type,
+        kernel,
+        degree,
+        gamma,
+        coef0,
+        cache_size,
+        1,
+         class_weight.shape[0],
+         &class_weight_label[0] if class_weight_label.size > 0 else NULL,
+         &class_weight[0] if class_weight.size > 0 else NULL,
+    )
+    model = set_model(
+        ¶m,
+         nSV.shape[0],
+         &SV[0, 0] if SV.size > 0 else NULL,
+         SV.shape,
+         &support[0],
+         support.shape,
+         sv_coef.strides,
+         &sv_coef[0, 0],
+         &intercept[0],
+         &nSV[0],
+         &probA[0] if probA.size > 0 else NULL,
+         &probB[0] if probB.size > 0 else NULL,
+    )
+
+    cdef intp_t n_class = get_nr(model)
     cdef BlasFunctions blas_functions
     blas_functions.dot = _dot[double]
     try:
         dec_values = np.empty((X.shape[0], n_class), dtype=np.float64)
         with nogil:
-            rv = copy_predict_proba(X.data, model, X.shape, dec_values.data, &blas_functions)
+            rv = copy_predict_proba(
+                 &X[0, 0],
+                model,
+                 X.shape,
+                 &dec_values[0, 0],
+                &blas_functions,
+            )
         if rv < 0:
             raise MemoryError("We've run out of memory")
     finally:
         free_model(model)
 
-    return dec_values
+    return dec_values.base
 
 
 def decision_function(
-    np.ndarray[np.float64_t, ndim=2, mode='c'] X,
-    np.ndarray[np.int32_t, ndim=1, mode='c'] support,
-    np.ndarray[np.float64_t, ndim=2, mode='c'] SV,
-    np.ndarray[np.int32_t, ndim=1, mode='c'] nSV,
-    np.ndarray[np.float64_t, ndim=2, mode='c'] sv_coef,
-    np.ndarray[np.float64_t, ndim=1, mode='c'] intercept,
-    np.ndarray[np.float64_t, ndim=1, mode='c'] probA=np.empty(0),
-    np.ndarray[np.float64_t, ndim=1, mode='c'] probB=np.empty(0),
-    int svm_type=0, kernel='rbf', int degree=3,
-    double gamma=0.1, double coef0=0.,
-    np.ndarray[np.float64_t, ndim=1, mode='c']
-        class_weight=np.empty(0),
-    np.ndarray[np.float64_t, ndim=1, mode='c']
-         sample_weight=np.empty(0),
-    double cache_size=100.):
+    const float64_t[:, ::1] X,
+    const int32_t[::1] support,
+    const float64_t[:, ::1] SV,
+    const int32_t[::1] nSV,
+    const float64_t[:, ::1] sv_coef,
+    const float64_t[::1] intercept,
+    const float64_t[::1] probA=np.empty(0),
+    const float64_t[::1] probB=np.empty(0),
+    int svm_type=0,
+    kernel='rbf',
+    int degree=3,
+    double gamma=0.1,
+    double coef0=0.0,
+    const float64_t[::1] class_weight=np.empty(0),
+    const float64_t[::1] sample_weight=np.empty(0),
+    double cache_size=100.0,
+):
     """
     Predict margin (libsvm name for this is predict_values)
 
@@ -542,24 +669,45 @@ def decision_function(
     dec_values : array
         Predicted values.
     """
-    cdef np.ndarray[np.float64_t, ndim=2, mode='c'] dec_values
+    cdef float64_t[:, ::1] dec_values
     cdef svm_parameter param
     cdef svm_model *model
-    cdef np.npy_intp n_class
+    cdef intp_t n_class
 
-    cdef np.ndarray[np.int32_t, ndim=1, mode='c'] \
-        class_weight_label = np.arange(class_weight.shape[0], dtype=np.int32)
+    cdef int32_t[::1] class_weight_label = np.arange(
+        class_weight.shape[0], dtype=np.int32
+    )
 
     cdef int rv
 
-    set_predict_params(¶m, svm_type, kernel, degree, gamma, coef0,
-                       cache_size, 0, class_weight.shape[0],
-                       class_weight_label.data, class_weight.data)
-
-    model = set_model(¶m,  nSV.shape[0], SV.data, SV.shape,
-                      support.data, support.shape, sv_coef.strides,
-                      sv_coef.data, intercept.data, nSV.data,
-                      probA.data, probB.data)
+    set_predict_params(
+        ¶m,
+        svm_type,
+        kernel,
+        degree,
+        gamma,
+        coef0,
+        cache_size,
+        0,
+         class_weight.shape[0],
+         &class_weight_label[0] if class_weight_label.size > 0 else NULL,
+         &class_weight[0] if class_weight.size > 0 else NULL,
+    )
+
+    model = set_model(
+        ¶m,
+         nSV.shape[0],
+         &SV[0, 0] if SV.size > 0 else NULL,
+         SV.shape,
+         &support[0],
+         support.shape,
+         sv_coef.strides,
+         &sv_coef[0, 0],
+         &intercept[0],
+         &nSV[0],
+         &probA[0] if probA.size > 0 else NULL,
+         &probB[0] if probB.size > 0 else NULL,
+    )
 
     if svm_type > 1:
         n_class = 1
@@ -571,28 +719,43 @@ def decision_function(
     try:
         dec_values = np.empty((X.shape[0], n_class), dtype=np.float64)
         with nogil:
-            rv = copy_predict_values(X.data, model, X.shape, dec_values.data, n_class, &blas_functions)
+            rv = copy_predict_values(
+                 &X[0, 0],
+                model,
+                 X.shape,
+                 &dec_values[0, 0],
+                n_class,
+                &blas_functions,
+            )
         if rv < 0:
             raise MemoryError("We've run out of memory")
     finally:
         free_model(model)
 
-    return dec_values
+    return dec_values.base
 
 
 def cross_validation(
-    np.ndarray[np.float64_t, ndim=2, mode='c'] X,
-    np.ndarray[np.float64_t, ndim=1, mode='c'] Y,
-    int n_fold, svm_type=0, kernel='rbf', int degree=3,
-    double gamma=0.1, double coef0=0., double tol=1e-3,
-    double C=1., double nu=0.5, double epsilon=0.1,
-    np.ndarray[np.float64_t, ndim=1, mode='c']
-        class_weight=np.empty(0),
-    np.ndarray[np.float64_t, ndim=1, mode='c']
-        sample_weight=np.empty(0),
-    int shrinking=0, int probability=0, double cache_size=100.,
+    const float64_t[:, ::1] X,
+    const float64_t[::1] Y,
+    int n_fold,
+    int svm_type=0,
+    kernel='rbf',
+    int degree=3,
+    double gamma=0.1,
+    double coef0=0.0,
+    double tol=1e-3,
+    double C=1.0,
+    double nu=0.5,
+    double epsilon=0.1,
+    float64_t[::1] class_weight=np.empty(0),
+    float64_t[::1] sample_weight=np.empty(0),
+    int shrinking=0,
+    int probability=0,
+    double cache_size=100.0,
     int max_iter=-1,
-    int random_seed=0):
+    int random_seed=0,
+):
     """
     Binding of the cross-validation routine (low-level routine)
 
@@ -673,18 +836,15 @@ def cross_validation(
 
     cdef svm_parameter param
     cdef svm_problem problem
-    cdef svm_model *model
     cdef const char *error_msg
-    cdef np.npy_intp SV_len
-    cdef np.npy_intp nr
 
     if len(sample_weight) == 0:
         sample_weight = np.ones(X.shape[0], dtype=np.float64)
     else:
-        assert sample_weight.shape[0] == X.shape[0], \
-               "sample_weight and X have incompatible shapes: " + \
-               "sample_weight has %s samples while X has %s" % \
-               (sample_weight.shape[0], X.shape[0])
+        assert sample_weight.shape[0] == X.shape[0], (
+            f"sample_weight and X have incompatible shapes: sample_weight has "
+            f"{sample_weight.shape[0]} samples while X has {X.shape[0]}"
+        )
 
     if X.shape[0] < n_fold:
         raise ValueError("Number of samples is less than number of folds")
@@ -692,34 +852,62 @@ def cross_validation(
     # set problem
     kernel_index = LIBSVM_KERNEL_TYPES.index(kernel)
     set_problem(
-        &problem, X.data, Y.data, sample_weight.data, X.shape, kernel_index)
+        &problem,
+         &X[0, 0],
+         &Y[0],
+         &sample_weight[0] if sample_weight.size > 0 else NULL,
+         X.shape,
+        kernel_index,
+    )
     if problem.x == NULL:
         raise MemoryError("Seems we've run out of memory")
-    cdef np.ndarray[np.int32_t, ndim=1, mode='c'] \
-        class_weight_label = np.arange(class_weight.shape[0], dtype=np.int32)
+    cdef int32_t[::1] class_weight_label = np.arange(
+        class_weight.shape[0], dtype=np.int32
+    )
 
     # set parameters
     set_parameter(
-        ¶m, svm_type, kernel_index, degree, gamma, coef0, nu, cache_size,
-        C, tol, tol, shrinking, probability, 
-        class_weight.shape[0], class_weight_label.data,
-        class_weight.data, max_iter, random_seed)
+        ¶m,
+        svm_type,
+        kernel_index,
+        degree,
+        gamma,
+        coef0,
+        nu,
+        cache_size,
+        C,
+        tol,
+        tol,
+        shrinking,
+        probability,
+         class_weight.shape[0],
+         &class_weight_label[0] if class_weight_label.size > 0 else NULL,
+         &class_weight[0] if class_weight.size > 0 else NULL,
+        max_iter,
+        random_seed,
+    )
 
-    error_msg = svm_check_parameter(&problem, ¶m);
+    error_msg = svm_check_parameter(&problem, ¶m)
     if error_msg:
         raise ValueError(error_msg)
 
-    cdef np.ndarray[np.float64_t, ndim=1, mode='c'] target
+    cdef float64_t[::1] target
     cdef BlasFunctions blas_functions
     blas_functions.dot = _dot[double]
     try:
         target = np.empty((X.shape[0]), dtype=np.float64)
         with nogil:
-            svm_cross_validation(&problem, ¶m, n_fold,  target.data, &blas_functions)
+            svm_cross_validation(
+                &problem,
+                ¶m,
+                n_fold,
+                 &target[0],
+                &blas_functions,
+            )
     finally:
         free(problem.x)
 
-    return target
+    return target.base
 
 
 def set_verbosity_wrap(int verbosity):
diff --git a/sklearn/svm/_libsvm_sparse.pyx b/sklearn/svm/_libsvm_sparse.pyx
index 92d94b0c685a5..529758061d299 100644
--- a/sklearn/svm/_libsvm_sparse.pyx
+++ b/sklearn/svm/_libsvm_sparse.pyx
@@ -1,10 +1,7 @@
-import warnings
 import  numpy as np
-cimport numpy as np
 from scipy import sparse
-from ..exceptions import ConvergenceWarning
 from ..utils._cython_blas cimport _dot
-np.import_array()
+from ..utils._typedefs cimport float64_t, int32_t, intp_t
 
 cdef extern from *:
     ctypedef char* const_char_p "const char*"
@@ -13,7 +10,7 @@ cdef extern from *:
 # Includes
 
 cdef extern from "_svm_cython_blas_helpers.h":
-    ctypedef double (*dot_func)(int, double*, int, double*, int)
+    ctypedef double (*dot_func)(int, const double*, int, const double*, int)
     cdef struct BlasFunctions:
         dot_func dot
 
@@ -28,42 +25,43 @@ cdef extern from "svm.h":
 
 cdef extern from "libsvm_sparse_helper.c":
     # this file contains methods for accessing libsvm 'hidden' fields
-    svm_csr_problem * csr_set_problem (char *, np.npy_intp *,
-         char *, np.npy_intp *, char *, char *, char *, int )
+    svm_csr_problem * csr_set_problem (
+        char *, intp_t *, char *, intp_t *, char *, char *, char *, int)
     svm_csr_model *csr_set_model(svm_parameter *param, int nr_class,
-                            char *SV_data, np.npy_intp *SV_indices_dims,
-                            char *SV_indices, np.npy_intp *SV_intptr_dims,
-                            char *SV_intptr,
-                            char *sv_coef, char *rho, char *nSV,
-                            char *probA, char *probB)
+                                 char *SV_data, intp_t *SV_indices_dims,
+                                 char *SV_indices, intp_t *SV_intptr_dims,
+                                 char *SV_intptr,
+                                 char *sv_coef, char *rho, char *nSV,
+                                 char *probA, char *probB)
     svm_parameter *set_parameter (int , int , int , double, double ,
                                   double , double , double , double,
                                   double, int, int, int, char *, char *, int,
                                   int)
     void copy_sv_coef   (char *, svm_csr_model *)
+    void copy_n_iter  (char *, svm_csr_model *)
     void copy_support   (char *, svm_csr_model *)
-    void copy_intercept (char *, svm_csr_model *, np.npy_intp *)
-    int copy_predict (char *, svm_csr_model *, np.npy_intp *, char *, BlasFunctions *)
-    int csr_copy_predict_values (np.npy_intp *data_size, char *data, np.npy_intp *index_size,
-        	char *index, np.npy_intp *intptr_size, char *size,
-                svm_csr_model *model, char *dec_values, int nr_class, BlasFunctions *)
-    int csr_copy_predict (np.npy_intp *data_size, char *data, np.npy_intp *index_size,
-        	char *index, np.npy_intp *intptr_size, char *size,
-                svm_csr_model *model, char *dec_values, BlasFunctions *) nogil
-    int csr_copy_predict_proba (np.npy_intp *data_size, char *data, np.npy_intp *index_size,
-        	char *index, np.npy_intp *intptr_size, char *size,
-                svm_csr_model *model, char *dec_values, BlasFunctions *) nogil
-
-    int  copy_predict_values(char *, svm_csr_model *, np.npy_intp *, char *, int, BlasFunctions *)
-    int  csr_copy_SV (char *values, np.npy_intp *n_indices,
-        	char *indices, np.npy_intp *n_indptr, char *indptr,
-                svm_csr_model *model, int n_features)
-    np.npy_intp get_nonzero_SV ( svm_csr_model *)
+    void copy_intercept (char *, svm_csr_model *, intp_t *)
+    int copy_predict (char *, svm_csr_model *, intp_t *, char *, BlasFunctions *)
+    int csr_copy_predict_values (intp_t *data_size, char *data, intp_t *index_size,
+                                 char *index, intp_t *intptr_size, char *size,
+                                 svm_csr_model *model, char *dec_values, int nr_class, BlasFunctions *)
+    int csr_copy_predict (intp_t *data_size, char *data, intp_t *index_size,
+                          char *index, intp_t *intptr_size, char *size,
+                          svm_csr_model *model, char *dec_values, BlasFunctions *) nogil
+    int csr_copy_predict_proba (intp_t *data_size, char *data, intp_t *index_size,
+                                char *index, intp_t *intptr_size, char *size,
+                                svm_csr_model *model, char *dec_values, BlasFunctions *) nogil
+
+    int  copy_predict_values(char *, svm_csr_model *, intp_t *, char *, int, BlasFunctions *)
+    int  csr_copy_SV (char *values, intp_t *n_indices,
+                      char *indices, intp_t *n_indptr, char *indptr,
+                      svm_csr_model *model, int n_features)
+    intp_t get_nonzero_SV (svm_csr_model *)
     void copy_nSV     (char *, svm_csr_model *)
-    void copy_probA   (char *, svm_csr_model *, np.npy_intp *)
-    void copy_probB   (char *, svm_csr_model *, np.npy_intp *)
-    np.npy_intp  get_l  (svm_csr_model *)
-    np.npy_intp  get_nr (svm_csr_model *)
+    void copy_probA   (char *, svm_csr_model *, intp_t *)
+    void copy_probB   (char *, svm_csr_model *, intp_t *)
+    intp_t  get_l  (svm_csr_model *)
+    intp_t  get_nr (svm_csr_model *)
     int  free_problem   (svm_csr_problem *)
     int  free_model     (svm_csr_model *)
     int  free_param     (svm_parameter *)
@@ -71,21 +69,18 @@ cdef extern from "libsvm_sparse_helper.c":
     void set_verbosity(int)
 
 
-np.import_array()
-
-
-def libsvm_sparse_train ( int n_features,
-                     np.ndarray[np.float64_t, ndim=1, mode='c'] values,
-                     np.ndarray[np.int32_t,   ndim=1, mode='c'] indices,
-                     np.ndarray[np.int32_t,   ndim=1, mode='c'] indptr,
-                     np.ndarray[np.float64_t, ndim=1, mode='c'] Y,
-                     int svm_type, int kernel_type, int degree, double gamma,
-                     double coef0, double eps, double C,
-                     np.ndarray[np.float64_t, ndim=1, mode='c'] class_weight,
-                     np.ndarray[np.float64_t, ndim=1, mode='c'] sample_weight,
-                     double nu, double cache_size, double p, int
-                     shrinking, int probability, int max_iter,
-                     int random_seed):
+def libsvm_sparse_train (int n_features,
+                         const float64_t[::1] values,
+                         const int32_t[::1] indices,
+                         const int32_t[::1] indptr,
+                         const float64_t[::1] Y,
+                         int svm_type, int kernel_type, int degree, double gamma,
+                         double coef0, double eps, double C,
+                         const float64_t[::1] class_weight,
+                         const float64_t[::1] sample_weight,
+                         double nu, double cache_size, double p, int
+                         shrinking, int probability, int max_iter,
+                         int random_seed):
     """
     Wrap svm_train from libsvm using a scipy.sparse.csr matrix
 
@@ -127,24 +122,44 @@ def libsvm_sparse_train ( int n_features,
     assert(kernel_type != 4)
 
     # set libsvm problem
-    problem = csr_set_problem(values.data, indices.shape, indices.data,
-                              indptr.shape, indptr.data, Y.data,
-                              sample_weight.data, kernel_type)
-
-    cdef np.ndarray[np.int32_t, ndim=1, mode='c'] \
+    problem = csr_set_problem(
+         &values[0],
+         indices.shape,
+         &indices[0],
+         indptr.shape,
+         &indptr[0],
+         &Y[0],
+         &sample_weight[0],
+        kernel_type,
+    )
+
+    cdef int32_t[::1] \
         class_weight_label = np.arange(class_weight.shape[0], dtype=np.int32)
 
     # set parameters
-    param = set_parameter(svm_type, kernel_type, degree, gamma, coef0,
-                          nu, cache_size, C, eps, p, shrinking,
-                          probability,  class_weight.shape[0],
-                          class_weight_label.data, class_weight.data, max_iter,
-                          random_seed)
+    param = set_parameter(
+        svm_type,
+        kernel_type,
+        degree,
+        gamma,
+        coef0,
+        nu,
+        cache_size,
+        C,
+        eps,
+        p,
+        shrinking,
+        probability,
+         class_weight.shape[0],
+         &class_weight_label[0] if class_weight_label.size > 0 else NULL,
+         &class_weight[0] if class_weight.size > 0 else NULL, max_iter,
+        random_seed,
+    )
 
     # check parameters
     if (param == NULL or problem == NULL):
         raise MemoryError("Seems we've run out of memory")
-    error_msg = svm_csr_check_parameter(problem, param);
+    error_msg = svm_csr_check_parameter(problem, param)
     if error_msg:
         free_problem(problem)
         free_param(param)
@@ -156,58 +171,71 @@ def libsvm_sparse_train ( int n_features,
     with nogil:
         model = svm_csr_train(problem, param, &fit_status, &blas_functions)
 
-    cdef np.npy_intp SV_len = get_l(model)
-    cdef np.npy_intp n_class = get_nr(model)
+    cdef intp_t SV_len = get_l(model)
+    cdef intp_t n_class = get_nr(model)
+
+    cdef int[::1] n_iter
+    n_iter = np.empty(max(1, n_class * (n_class - 1) // 2), dtype=np.intc)
+    copy_n_iter( &n_iter[0], model)
 
     # copy model.sv_coef
     # we create a new array instead of resizing, otherwise
     # it would not erase previous information
-    cdef np.ndarray sv_coef_data
+    cdef float64_t[::1] sv_coef_data
     sv_coef_data = np.empty((n_class-1)*SV_len, dtype=np.float64)
-    copy_sv_coef (sv_coef_data.data, model)
+    copy_sv_coef ( &sv_coef_data[0] if sv_coef_data.size > 0 else NULL, model)
 
-    cdef np.ndarray[np.int32_t, ndim=1, mode='c'] support
+    cdef int32_t[::1] support
     support = np.empty(SV_len, dtype=np.int32)
-    copy_support(support.data, model)
+    copy_support( &support[0] if support.size > 0 else NULL, model)
 
     # copy model.rho into the intercept
     # the intercept is just model.rho but with sign changed
-    cdef np.ndarray intercept
+    cdef float64_t[::1]intercept
     intercept = np.empty(n_class*(n_class-1)//2, dtype=np.float64)
-    copy_intercept (intercept.data, model, intercept.shape)
+    copy_intercept ( &intercept[0], model,  intercept.shape)
 
     # copy model.SV
     # we erase any previous information in SV
     # TODO: custom kernel
-    cdef np.npy_intp nonzero_SV
+    cdef intp_t nonzero_SV
     nonzero_SV = get_nonzero_SV (model)
 
-    cdef np.ndarray SV_data, SV_indices, SV_indptr
+    cdef float64_t[::1] SV_data
+    cdef int32_t[::1] SV_indices, SV_indptr
     SV_data = np.empty(nonzero_SV, dtype=np.float64)
     SV_indices = np.empty(nonzero_SV, dtype=np.int32)
-    SV_indptr = np.empty(SV_len + 1, dtype=np.int32)
-    csr_copy_SV(SV_data.data, SV_indices.shape, SV_indices.data,
-                SV_indptr.shape, SV_indptr.data, model, n_features)
+    SV_indptr = np.empty(SV_len + 1, dtype=np.int32)
+    csr_copy_SV(
+         &SV_data[0] if SV_data.size > 0 else NULL,
+         SV_indices.shape,
+         &SV_indices[0] if SV_indices.size > 0 else NULL,
+         SV_indptr.shape,
+         &SV_indptr[0] if SV_indptr.size > 0 else NULL,
+        model,
+        n_features,
+    )
     support_vectors_ = sparse.csr_matrix(
-	(SV_data, SV_indices, SV_indptr), (SV_len, n_features))
+        (SV_data, SV_indices, SV_indptr), (SV_len, n_features)
+    )
 
     # copy model.nSV
     # TODO: do only in classification
-    cdef np.ndarray n_class_SV
+    cdef int32_t[::1]n_class_SV
     n_class_SV = np.empty(n_class, dtype=np.int32)
-    copy_nSV(n_class_SV.data, model)
+    copy_nSV( &n_class_SV[0], model)
 
     # # copy probabilities
-    cdef np.ndarray probA, probB
+    cdef float64_t[::1] probA, probB
     if probability != 0:
-        if svm_type < 2: # SVC and NuSVC
+        if svm_type < 2:  # SVC and NuSVC
             probA = np.empty(n_class*(n_class-1)//2, dtype=np.float64)
             probB = np.empty(n_class*(n_class-1)//2, dtype=np.float64)
-            copy_probB(probB.data, model, probB.shape)
+            copy_probB( &probB[0], model,  probB.shape)
         else:
             probA = np.empty(1, dtype=np.float64)
             probB = np.empty(0, dtype=np.float64)
-        copy_probA(probA.data, model, probA.shape)
+        copy_probA( &probA[0], model,  probA.shape)
     else:
         probA = np.empty(0, dtype=np.float64)
         probB = np.empty(0, dtype=np.float64)
@@ -216,27 +244,36 @@ def libsvm_sparse_train ( int n_features,
     free_problem(problem)
     free_param(param)
 
-    return (support, support_vectors_, sv_coef_data, intercept, n_class_SV,
-            probA, probB, fit_status)
-
-
-def libsvm_sparse_predict (np.ndarray[np.float64_t, ndim=1, mode='c'] T_data,
-                            np.ndarray[np.int32_t,   ndim=1, mode='c'] T_indices,
-                            np.ndarray[np.int32_t,   ndim=1, mode='c'] T_indptr,
-                            np.ndarray[np.float64_t, ndim=1, mode='c'] SV_data,
-                            np.ndarray[np.int32_t,   ndim=1, mode='c'] SV_indices,
-                            np.ndarray[np.int32_t,   ndim=1, mode='c'] SV_indptr,
-                            np.ndarray[np.float64_t, ndim=1, mode='c'] sv_coef,
-                            np.ndarray[np.float64_t, ndim=1, mode='c']
-                            intercept, int svm_type, int kernel_type, int
-                            degree, double gamma, double coef0, double
-                            eps, double C,
-                            np.ndarray[np.float64_t, ndim=1] class_weight,
-                            double nu, double p, int
-                            shrinking, int probability,
-                            np.ndarray[np.int32_t, ndim=1, mode='c'] nSV,
-                            np.ndarray[np.float64_t, ndim=1, mode='c'] probA,
-                            np.ndarray[np.float64_t, ndim=1, mode='c'] probB):
+    return (
+        support.base,
+        support_vectors_,
+        sv_coef_data.base,
+        intercept.base,
+        n_class_SV.base,
+        probA.base,
+        probB.base,
+        fit_status,
+        n_iter.base,
+    )
+
+
+def libsvm_sparse_predict (const float64_t[::1] T_data,
+                           const int32_t[::1] T_indices,
+                           const int32_t[::1] T_indptr,
+                           const float64_t[::1] SV_data,
+                           const int32_t[::1] SV_indices,
+                           const int32_t[::1] SV_indptr,
+                           const float64_t[::1] sv_coef,
+                           const float64_t[::1]
+                           intercept, int svm_type, int kernel_type, int
+                           degree, double gamma, double coef0, double
+                           eps, double C,
+                           const float64_t[:] class_weight,
+                           double nu, double p, int
+                           shrinking, int probability,
+                           const int32_t[::1] nSV,
+                           const float64_t[::1] probA,
+                           const float64_t[::1] probB):
     """
     Predict values T given a model.
 
@@ -259,147 +296,222 @@ def libsvm_sparse_predict (np.ndarray[np.float64_t, ndim=1, mode='c'] T_data,
     dec_values : array
         predicted values.
     """
-    cdef np.ndarray[np.float64_t, ndim=1, mode='c'] dec_values
+    cdef float64_t[::1] dec_values
     cdef svm_parameter *param
     cdef svm_csr_model *model
-    cdef np.ndarray[np.int32_t, ndim=1, mode='c'] \
+    cdef int32_t[::1] \
         class_weight_label = np.arange(class_weight.shape[0], dtype=np.int32)
     cdef int rv
-    param = set_parameter(svm_type, kernel_type, degree, gamma,
-                          coef0, nu,
-                          100., # cache size has no effect on predict
-                          C, eps, p, shrinking,
-                          probability,  class_weight.shape[0], class_weight_label.data,
-                          class_weight.data, -1,
-                          -1) # random seed has no effect on predict either
-
-    model = csr_set_model(param,  nSV.shape[0], SV_data.data,
-                          SV_indices.shape, SV_indices.data,
-                          SV_indptr.shape, SV_indptr.data,
-                          sv_coef.data, intercept.data,
-                          nSV.data, probA.data, probB.data)
-    #TODO: use check_model
+    param = set_parameter(
+        svm_type,
+        kernel_type,
+        degree,
+        gamma,
+        coef0,
+        nu,
+        100.0,  # cache size has no effect on predict
+        C,
+        eps,
+        p,
+        shrinking,
+        probability,
+         class_weight.shape[0],
+         &class_weight_label[0] if class_weight_label.size > 0 else NULL,
+         &class_weight[0] if class_weight.size > 0 else NULL,
+        -1,
+        -1,  # random seed has no effect on predict either
+    )
+
+    model = csr_set_model(
+        param,  nSV.shape[0],
+         &SV_data[0] if SV_data.size > 0 else NULL,
+        SV_indices.shape,
+         &SV_indices[0] if SV_indices.size > 0 else NULL,
+         SV_indptr.shape,
+         &SV_indptr[0] if SV_indptr.size > 0 else NULL,
+         &sv_coef[0] if sv_coef.size > 0 else NULL,
+         &intercept[0],
+         &nSV[0],
+         &probA[0] if probA.size > 0 else NULL,
+         &probB[0] if probB.size > 0 else NULL,
+    )
+    # TODO: use check_model
     dec_values = np.empty(T_indptr.shape[0]-1)
     cdef BlasFunctions blas_functions
     blas_functions.dot = _dot[double]
     with nogil:
-        rv = csr_copy_predict(T_data.shape, T_data.data,
-                              T_indices.shape, T_indices.data,
-                              T_indptr.shape, T_indptr.data,
-                              model, dec_values.data,
-                              &blas_functions)
+        rv = csr_copy_predict(
+             T_data.shape,
+             &T_data[0],
+             T_indices.shape,
+             &T_indices[0],
+             T_indptr.shape,
+             &T_indptr[0],
+            model,
+             &dec_values[0],
+            &blas_functions,
+        )
     if rv < 0:
         raise MemoryError("We've run out of memory")
     # free model and param
     free_model_SV(model)
     free_model(model)
     free_param(param)
-    return dec_values
+    return dec_values.base
 
 
 def libsvm_sparse_predict_proba(
-    np.ndarray[np.float64_t, ndim=1, mode='c'] T_data,
-    np.ndarray[np.int32_t,   ndim=1, mode='c'] T_indices,
-    np.ndarray[np.int32_t,   ndim=1, mode='c'] T_indptr,
-    np.ndarray[np.float64_t, ndim=1, mode='c'] SV_data,
-    np.ndarray[np.int32_t,   ndim=1, mode='c'] SV_indices,
-    np.ndarray[np.int32_t,   ndim=1, mode='c'] SV_indptr,
-    np.ndarray[np.float64_t, ndim=1, mode='c'] sv_coef,
-    np.ndarray[np.float64_t, ndim=1, mode='c']
+    const float64_t[::1] T_data,
+    const int32_t[::1] T_indices,
+    const int32_t[::1] T_indptr,
+    const float64_t[::1] SV_data,
+    const int32_t[::1] SV_indices,
+    const int32_t[::1] SV_indptr,
+    const float64_t[::1] sv_coef,
+    const float64_t[::1]
     intercept, int svm_type, int kernel_type, int
     degree, double gamma, double coef0, double
     eps, double C,
-    np.ndarray[np.float64_t, ndim=1] class_weight,
+    const float64_t[:] class_weight,
     double nu, double p, int shrinking, int probability,
-    np.ndarray[np.int32_t, ndim=1, mode='c'] nSV,
-    np.ndarray[np.float64_t, ndim=1, mode='c'] probA,
-    np.ndarray[np.float64_t, ndim=1, mode='c'] probB):
+    const int32_t[::1] nSV,
+    const float64_t[::1] probA,
+    const float64_t[::1] probB,
+):
     """
     Predict values T given a model.
     """
-    cdef np.ndarray[np.float64_t, ndim=2, mode='c'] dec_values
+    cdef float64_t[:, ::1] dec_values
     cdef svm_parameter *param
     cdef svm_csr_model *model
-    cdef np.ndarray[np.int32_t, ndim=1, mode='c'] \
+    cdef int32_t[::1] \
         class_weight_label = np.arange(class_weight.shape[0], dtype=np.int32)
-    param = set_parameter(svm_type, kernel_type, degree, gamma,
-                          coef0, nu,
-                          100., # cache size has no effect on predict
-                          C, eps, p, shrinking,
-                          probability,  class_weight.shape[0], class_weight_label.data,
-                          class_weight.data, -1,
-                          -1) # random seed has no effect on predict either
-
-    model = csr_set_model(param,  nSV.shape[0], SV_data.data,
-                          SV_indices.shape, SV_indices.data,
-                          SV_indptr.shape, SV_indptr.data,
-                          sv_coef.data, intercept.data,
-                          nSV.data, probA.data, probB.data)
-    #TODO: use check_model
-    cdef np.npy_intp n_class = get_nr(model)
+    param = set_parameter(
+        svm_type,
+        kernel_type,
+        degree,
+        gamma,
+        coef0,
+        nu,
+        100.0,  # cache size has no effect on predict
+        C,
+        eps,
+        p,
+        shrinking,
+        probability,
+         class_weight.shape[0],
+         &class_weight_label[0] if class_weight_label.size > 0 else NULL,
+         &class_weight[0] if class_weight.size > 0 else NULL,
+        -1,
+        -1,  # random seed has no effect on predict either
+    )
+
+    model = csr_set_model(
+        param,
+         nSV.shape[0],
+         &SV_data[0] if SV_data.size > 0 else NULL,
+         SV_indices.shape,
+         &SV_indices[0] if SV_indices.size > 0 else NULL,
+         SV_indptr.shape,
+         &SV_indptr[0] if SV_indptr.size > 0 else NULL,
+         &sv_coef[0] if sv_coef.size > 0 else NULL,
+         &intercept[0],
+         &nSV[0],
+         &probA[0] if probA.size > 0 else NULL,
+         &probB[0] if probB.size > 0 else NULL,
+    )
+    # TODO: use check_model
+    cdef intp_t n_class = get_nr(model)
     cdef int rv
     dec_values = np.empty((T_indptr.shape[0]-1, n_class), dtype=np.float64)
     cdef BlasFunctions blas_functions
     blas_functions.dot = _dot[double]
     with nogil:
-        rv = csr_copy_predict_proba(T_data.shape, T_data.data,
-                                    T_indices.shape, T_indices.data,
-                                    T_indptr.shape, T_indptr.data,
-                                    model, dec_values.data,
-                                    &blas_functions)
+        rv = csr_copy_predict_proba(
+             T_data.shape,
+             &T_data[0],
+             T_indices.shape,
+             &T_indices[0],
+             T_indptr.shape,
+             &T_indptr[0],
+            model,
+             &dec_values[0, 0],
+            &blas_functions,
+        )
     if rv < 0:
         raise MemoryError("We've run out of memory")
     # free model and param
     free_model_SV(model)
     free_model(model)
     free_param(param)
-    return dec_values
-
-
+    return dec_values.base
 
 
 def libsvm_sparse_decision_function(
-    np.ndarray[np.float64_t, ndim=1, mode='c'] T_data,
-    np.ndarray[np.int32_t,   ndim=1, mode='c'] T_indices,
-    np.ndarray[np.int32_t,   ndim=1, mode='c'] T_indptr,
-    np.ndarray[np.float64_t, ndim=1, mode='c'] SV_data,
-    np.ndarray[np.int32_t,   ndim=1, mode='c'] SV_indices,
-    np.ndarray[np.int32_t,   ndim=1, mode='c'] SV_indptr,
-    np.ndarray[np.float64_t, ndim=1, mode='c'] sv_coef,
-    np.ndarray[np.float64_t, ndim=1, mode='c']
+    const float64_t[::1] T_data,
+    const int32_t[::1] T_indices,
+    const int32_t[::1] T_indptr,
+    const float64_t[::1] SV_data,
+    const int32_t[::1] SV_indices,
+    const int32_t[::1] SV_indptr,
+    const float64_t[::1] sv_coef,
+    const float64_t[::1]
     intercept, int svm_type, int kernel_type, int
     degree, double gamma, double coef0, double
     eps, double C,
-    np.ndarray[np.float64_t, ndim=1] class_weight,
+    const float64_t[:] class_weight,
     double nu, double p, int shrinking, int probability,
-    np.ndarray[np.int32_t, ndim=1, mode='c'] nSV,
-    np.ndarray[np.float64_t, ndim=1, mode='c'] probA,
-    np.ndarray[np.float64_t, ndim=1, mode='c'] probB):
+    const int32_t[::1] nSV,
+    const float64_t[::1] probA,
+    const float64_t[::1] probB,
+):
     """
     Predict margin (libsvm name for this is predict_values)
 
     We have to reconstruct model and parameters to make sure we stay
     in sync with the python object.
     """
-    cdef np.ndarray[np.float64_t, ndim=2, mode='c'] dec_values
+    cdef float64_t[:, ::1] dec_values
     cdef svm_parameter *param
-    cdef np.npy_intp n_class
+    cdef intp_t n_class
 
     cdef svm_csr_model *model
-    cdef np.ndarray[np.int32_t, ndim=1, mode='c'] \
+    cdef int32_t[::1] \
         class_weight_label = np.arange(class_weight.shape[0], dtype=np.int32)
-    param = set_parameter(svm_type, kernel_type, degree, gamma,
-                          coef0, nu,
-                          100., # cache size has no effect on predict
-                          C, eps, p, shrinking,
-                          probability,  class_weight.shape[0],
-                          class_weight_label.data, class_weight.data, -1, -1)
-
-    model = csr_set_model(param,  nSV.shape[0], SV_data.data,
-                          SV_indices.shape, SV_indices.data,
-                          SV_indptr.shape, SV_indptr.data,
-                          sv_coef.data, intercept.data,
-                          nSV.data, probA.data, probB.data)
+    param = set_parameter(
+        svm_type,
+        kernel_type,
+        degree,
+        gamma,
+        coef0,
+        nu,
+        100.0,  # cache size has no effect on predict
+        C,
+        eps,
+        p,
+        shrinking,
+        probability,
+         class_weight.shape[0],
+         &class_weight_label[0] if class_weight_label.size > 0 else NULL,
+         &class_weight[0] if class_weight.size > 0 else NULL,
+        -1,
+        -1,
+    )
+
+    model = csr_set_model(
+        param,
+         nSV.shape[0],
+         &SV_data[0] if SV_data.size > 0 else NULL,
+         SV_indices.shape,
+         &SV_indices[0] if SV_indices.size > 0 else NULL,
+         SV_indptr.shape,
+         &SV_indptr[0] if SV_indptr.size > 0 else NULL,
+         &sv_coef[0] if sv_coef.size > 0 else NULL,
+         &intercept[0],
+         &nSV[0],
+         &probA[0] if probA.size > 0 else NULL,
+         &probB[0] if probB.size > 0 else NULL,
+    )
 
     if svm_type > 1:
         n_class = 1
@@ -410,18 +522,25 @@ def libsvm_sparse_decision_function(
     dec_values = np.empty((T_indptr.shape[0] - 1, n_class), dtype=np.float64)
     cdef BlasFunctions blas_functions
     blas_functions.dot = _dot[double]
-    if csr_copy_predict_values(T_data.shape, T_data.data,
-                        T_indices.shape, T_indices.data,
-                        T_indptr.shape, T_indptr.data,
-                        model, dec_values.data, n_class,
-                        &blas_functions) < 0:
+    if csr_copy_predict_values(
+             T_data.shape,
+             &T_data[0],
+             T_indices.shape,
+             &T_indices[0],
+             T_indptr.shape,
+             &T_indptr[0],
+            model,
+             &dec_values[0, 0],
+            n_class,
+            &blas_functions,
+    ) < 0:
         raise MemoryError("We've run out of memory")
     # free model and param
     free_model_SV(model)
     free_model(model)
     free_param(param)
 
-    return dec_values
+    return dec_values.base
 
 
 def set_verbosity_wrap(int verbosity):
diff --git a/sklearn/svm/_newrand.pyx b/sklearn/svm/_newrand.pyx
index 0f82cbcbfdd6f..af543ed73286a 100644
--- a/sklearn/svm/_newrand.pyx
+++ b/sklearn/svm/_newrand.pyx
@@ -1,14 +1,13 @@
-"""
-Wrapper for newrand.h
-
-"""
-
-cdef extern from "newrand.h":
-	void set_seed(unsigned int)
-	unsigned int bounded_rand_int(unsigned int)
-
-def set_seed_wrap(unsigned int custom_seed):
-	set_seed(custom_seed)
-
-def bounded_rand_int_wrap(unsigned int range_):
-	return bounded_rand_int(range_)
+"""Wrapper for newrand.h"""
+
+cdef extern from "newrand.h":
+    void set_seed(unsigned int)
+    unsigned int bounded_rand_int(unsigned int)
+
+
+def set_seed_wrap(unsigned int custom_seed):
+    set_seed(custom_seed)
+
+
+def bounded_rand_int_wrap(unsigned int range_):
+    return bounded_rand_int(range_)
diff --git a/sklearn/svm/meson.build b/sklearn/svm/meson.build
new file mode 100644
index 0000000000000..6232d747d1feb
--- /dev/null
+++ b/sklearn/svm/meson.build
@@ -0,0 +1,48 @@
+newrand_include = include_directories('src/newrand')
+libsvm_include = include_directories('src/libsvm')
+liblinear_include = include_directories('src/liblinear')
+
+_newrand = py.extension_module(
+  '_newrand',
+  cython_gen_cpp.process('_newrand.pyx'),
+  include_directories: [newrand_include],
+  subdir: 'sklearn/svm',
+  install: true
+)
+
+libsvm_skl = static_library(
+  'libsvm-skl',
+  ['src/libsvm/libsvm_template.cpp'],
+)
+
+py.extension_module(
+  '_libsvm',
+  [cython_gen.process('_libsvm.pyx'), utils_cython_tree],
+  include_directories: [newrand_include, libsvm_include],
+  link_with: libsvm_skl,
+  subdir: 'sklearn/svm',
+  install: true
+)
+
+py.extension_module(
+  '_libsvm_sparse',
+  [cython_gen.process('_libsvm_sparse.pyx'), utils_cython_tree],
+  include_directories: [newrand_include, libsvm_include],
+  link_with: libsvm_skl,
+  subdir: 'sklearn/svm',
+  install: true
+)
+
+liblinear_skl = static_library(
+  'liblinear-skl',
+  ['src/liblinear/linear.cpp', 'src/liblinear/tron.cpp'],
+)
+
+py.extension_module(
+  '_liblinear',
+  [cython_gen.process('_liblinear.pyx'), utils_cython_tree],
+  include_directories: [newrand_include, liblinear_include],
+  link_with: [liblinear_skl],
+  subdir: 'sklearn/svm',
+  install: true
+)
diff --git a/sklearn/svm/setup.py b/sklearn/svm/setup.py
deleted file mode 100644
index dffcff8eb203d..0000000000000
--- a/sklearn/svm/setup.py
+++ /dev/null
@@ -1,109 +0,0 @@
-import os
-from os.path import join
-import numpy
-
-
-def configuration(parent_package='', top_path=None):
-    from numpy.distutils.misc_util import Configuration
-
-    config = Configuration('svm', parent_package, top_path)
-
-    config.add_subpackage('tests')
-
-    # newrand wrappers
-    config.add_extension('_newrand',
-                         sources=['_newrand.pyx'],
-                         include_dirs=[numpy.get_include(),
-                                       join('src', 'newrand')],
-                         depends=[join('src', 'newrand', 'newrand.h')],
-                         language='c++',
-                         # Use C++11 random number generator fix
-                         extra_compile_args=['-std=c++11']
-                         )
-
-    # Section LibSVM
-
-    # we compile both libsvm and libsvm_sparse
-    config.add_library('libsvm-skl',
-                       sources=[join('src', 'libsvm', 'libsvm_template.cpp')],
-                       depends=[join('src', 'libsvm', 'svm.cpp'),
-                                join('src', 'libsvm', 'svm.h'),
-                                join('src', 'newrand', 'newrand.h')],
-                       # Force C++ linking in case gcc is picked up instead
-                       # of g++ under windows with some versions of MinGW
-                       extra_link_args=['-lstdc++'],
-                       # Use C++11 to use the random number generator fix
-                       extra_compiler_args=['-std=c++11'],
-                       )
-
-    libsvm_sources = ['_libsvm.pyx']
-    libsvm_depends = [join('src', 'libsvm', 'libsvm_helper.c'),
-                      join('src', 'libsvm', 'libsvm_template.cpp'),
-                      join('src', 'libsvm', 'svm.cpp'),
-                      join('src', 'libsvm', 'svm.h'),
-                      join('src', 'newrand', 'newrand.h')]
-
-    config.add_extension('_libsvm',
-                         sources=libsvm_sources,
-                         include_dirs=[numpy.get_include(),
-                                       join('src', 'libsvm'),
-                                       join('src', 'newrand')],
-                         libraries=['libsvm-skl'],
-                         depends=libsvm_depends,
-                         )
-
-    # liblinear module
-    libraries = []
-    if os.name == 'posix':
-        libraries.append('m')
-
-    # precompile liblinear to use C++11 flag
-    config.add_library('liblinear-skl',
-                       sources=[join('src', 'liblinear', 'linear.cpp'),
-                                join('src', 'liblinear', 'tron.cpp')],
-                       depends=[join('src', 'liblinear', 'linear.h'),
-                                join('src', 'liblinear', 'tron.h'),
-                                join('src', 'newrand', 'newrand.h')],
-                       # Force C++ linking in case gcc is picked up instead
-                       # of g++ under windows with some versions of MinGW
-                       extra_link_args=['-lstdc++'],
-                       # Use C++11 to use the random number generator fix
-                       extra_compiler_args=['-std=c++11'],
-                       )
-
-    liblinear_sources = ['_liblinear.pyx']
-    liblinear_depends = [join('src', 'liblinear', '*.h'),
-                         join('src', 'newrand', 'newrand.h'),
-                         join('src', 'liblinear', 'liblinear_helper.c')]
-
-    config.add_extension('_liblinear',
-                         sources=liblinear_sources,
-                         libraries=['liblinear-skl'] + libraries,
-                         include_dirs=[join('.', 'src', 'liblinear'),
-                                       join('.', 'src', 'newrand'),
-                                       join('..', 'utils'),
-                                       numpy.get_include()],
-                         depends=liblinear_depends,
-                         # extra_compile_args=['-O0 -fno-inline'],
-                         )
-
-    # end liblinear module
-
-    # this should go *after* libsvm-skl
-    libsvm_sparse_sources = ['_libsvm_sparse.pyx']
-    config.add_extension('_libsvm_sparse', libraries=['libsvm-skl'],
-                         sources=libsvm_sparse_sources,
-                         include_dirs=[numpy.get_include(),
-                                       join("src", "libsvm"),
-                                       join("src", "newrand")],
-                         depends=[join("src", "libsvm", "svm.h"),
-                                  join('src', 'newrand', 'newrand.h'),
-                                  join("src", "libsvm",
-                                       "libsvm_sparse_helper.c")])
-
-    return config
-
-
-if __name__ == '__main__':
-    from numpy.distutils.core import setup
-    setup(**configuration(top_path='').todict())
diff --git a/sklearn/svm/src/liblinear/_cython_blas_helpers.h b/sklearn/svm/src/liblinear/_cython_blas_helpers.h
index 6b2475e9d56cf..bdec1a2f99eb9 100644
--- a/sklearn/svm/src/liblinear/_cython_blas_helpers.h
+++ b/sklearn/svm/src/liblinear/_cython_blas_helpers.h
@@ -1,10 +1,10 @@
 #ifndef _CYTHON_BLAS_HELPERS_H
 #define _CYTHON_BLAS_HELPERS_H
 
-typedef double (*dot_func)(int, double*, int, double*, int);
-typedef void (*axpy_func)(int, double, double*, int, double*, int);
-typedef void (*scal_func)(int, double, double*, int);
-typedef double (*nrm2_func)(int, double*, int);
+typedef double (*dot_func)(int, const double*, int, const double*, int);
+typedef void (*axpy_func)(int, double, const double*, int, double*, int);
+typedef void (*scal_func)(int, double, const double*, int);
+typedef double (*nrm2_func)(int, const double*, int);
 
 typedef struct BlasFunctions{
     dot_func dot;
diff --git a/sklearn/svm/src/liblinear/liblinear_helper.c b/sklearn/svm/src/liblinear/liblinear_helper.c
index 7433a0086f682..b66f08413e11b 100644
--- a/sklearn/svm/src/liblinear/liblinear_helper.c
+++ b/sklearn/svm/src/liblinear/liblinear_helper.c
@@ -1,7 +1,9 @@
 #include 
-#include 
+#define PY_SSIZE_T_CLEAN
+#include 
 #include "linear.h"
 
+
 /*
  * Convert matrix to sparse representation suitable for liblinear. x is
  * expected to be an array of length n_samples*n_features.
@@ -140,7 +142,7 @@ struct problem * set_problem(char *X, int double_precision_X, int n_samples,
                         n_nonzero, bias);
     problem->bias = bias;
 
-    if (problem->x == NULL) { 
+    if (problem->x == NULL) {
         free(problem);
         return NULL;
     }
@@ -174,8 +176,8 @@ struct problem * csr_set_problem (char *X, int double_precision_X,
 
 /* Create a parameter struct with and return it */
 struct parameter *set_parameter(int solver_type, double eps, double C,
-                                npy_intp nr_weight, char *weight_label,
-                                char *weight, int max_iter, unsigned seed, 
+                                Py_ssize_t nr_weight, char *weight_label,
+                                char *weight, int max_iter, unsigned seed,
                                 double epsilon)
 {
     struct parameter *param = malloc(sizeof(struct parameter));
@@ -196,7 +198,7 @@ struct parameter *set_parameter(int solver_type, double eps, double C,
 
 void copy_w(void *data, struct model *model, int len)
 {
-    memcpy(data, model->w, len * sizeof(double)); 
+    memcpy(data, model->w, len * sizeof(double));
 }
 
 double get_bias(struct model *model)
diff --git a/sklearn/svm/src/liblinear/linear.cpp b/sklearn/svm/src/liblinear/linear.cpp
index 29a5581c280dc..63648adbe2947 100644
--- a/sklearn/svm/src/liblinear/linear.cpp
+++ b/sklearn/svm/src/liblinear/linear.cpp
@@ -1769,6 +1769,7 @@ static int solve_l1r_lr(
 	int max_num_linesearch = 20;
 	int active_size;
 	int QP_active_size;
+	int QP_no_change = 0;
 
 	double nu = 1e-12;
 	double inner_eps = 1;
@@ -1896,9 +1897,13 @@ static int solve_l1r_lr(
 		if(newton_iter == 0)
 			Gnorm1_init = Gnorm1_new;
 
-		if(Gnorm1_new <= eps*Gnorm1_init)
+		// Break outer-loop if the accumulated violation is small.
+		// Also break if no update in QP inner-loop ten times in a row.
+		if(Gnorm1_new <= eps*Gnorm1_init || QP_no_change >= 10)
 			break;
 
+		QP_no_change++;
+
 		iter = 0;
 		QP_Gmax_old = INF;
 		QP_active_size = active_size;
@@ -1955,9 +1960,6 @@ static int solve_l1r_lr(
 				else
 					violation = fabs(Gn);
 
-				QP_Gmax_new = max(QP_Gmax_new, violation);
-				QP_Gnorm1_new += violation;
-
 				// obtain solution of one-variable problem
 				if(Gp < H*wpd[j])
 					z = -Gp/H;
@@ -1970,6 +1972,10 @@ static int solve_l1r_lr(
 					continue;
 				z = min(max(z,-10.0),10.0);
 
+				QP_no_change = 0;
+				QP_Gmax_new = max(QP_Gmax_new, violation);
+				QP_Gnorm1_new += violation;
+
 				wpd[j] += z;
 
 				x = prob_col->x[j];
@@ -2453,7 +2459,6 @@ model* train(const problem *prob, const parameter *param, BlasFunctions *blas_fu
 	int l = prob->l;
 	int n = prob->n;
 	int w_size = prob->n;
-	int n_iter;
 	model *model_ = Malloc(model,1);
 
 	if(prob->bias>=0)
diff --git a/sklearn/svm/src/liblinear/linear.h b/sklearn/svm/src/liblinear/linear.h
index 1e4952b184d97..1dfc1c0ed0149 100644
--- a/sklearn/svm/src/liblinear/linear.h
+++ b/sklearn/svm/src/liblinear/linear.h
@@ -84,4 +84,3 @@ void set_print_string_function(void (*print_func) (const char*));
 #endif
 
 #endif /* _LIBLINEAR_H */
-
diff --git a/sklearn/svm/src/libsvm/LIBSVM_CHANGES b/sklearn/svm/src/libsvm/LIBSVM_CHANGES
index bde6beaca2694..663550b8ddd6f 100644
--- a/sklearn/svm/src/libsvm/LIBSVM_CHANGES
+++ b/sklearn/svm/src/libsvm/LIBSVM_CHANGES
@@ -7,4 +7,5 @@ This is here mainly as checklist for incorporation of new versions of libsvm.
   * Improved random number generator (fix on windows, enhancement on other
     platforms). See 
   * invoke scipy blas api for svm kernel function to improve performance with speedup rate of 1.5X to 2X for dense data only. See 
+  * Expose the number of iterations run in optimization. See 
 The changes made with respect to upstream are detailed in the heading of svm.cpp
diff --git a/sklearn/svm/src/libsvm/_svm_cython_blas_helpers.h b/sklearn/svm/src/libsvm/_svm_cython_blas_helpers.h
index 057e08195e9a5..2548c7844d267 100644
--- a/sklearn/svm/src/libsvm/_svm_cython_blas_helpers.h
+++ b/sklearn/svm/src/libsvm/_svm_cython_blas_helpers.h
@@ -1,7 +1,7 @@
 #ifndef _SVM_CYTHON_BLAS_HELPERS_H
 #define _SVM_CYTHON_BLAS_HELPERS_H
 
-typedef double (*dot_func)(int, double*, int, double*, int);
+typedef double (*dot_func)(int, const double*, int, const double*, int);
 typedef struct BlasFunctions{
     dot_func dot;
 } BlasFunctions;
diff --git a/sklearn/svm/src/libsvm/libsvm_helper.c b/sklearn/svm/src/libsvm/libsvm_helper.c
index 17f328f9e7c4c..b87b52a6fbdc2 100644
--- a/sklearn/svm/src/libsvm/libsvm_helper.c
+++ b/sklearn/svm/src/libsvm/libsvm_helper.c
@@ -1,7 +1,15 @@
 #include 
-#include 
+#define PY_SSIZE_T_CLEAN
+#include 
 #include "svm.h"
 #include "_svm_cython_blas_helpers.h"
+
+
+#ifndef MAX
+    #define MAX(x, y) (((x) > (y)) ? (x) : (y))
+#endif
+
+
 /*
  * Some helper methods for libsvm bindings.
  *
@@ -9,9 +17,9 @@
  * but libsvm does not expose this structure, so we define it here
  * along some utilities to convert from numpy arrays.
  *
- * License: BSD 3 clause
+ * Authors: The scikit-learn developers
+ * SPDX-License-Identifier: BSD-3-Clause
  *
- * Author: 2010 Fabian Pedregosa 
  */
 
 
@@ -30,10 +38,10 @@
  * contiguous, but in practice its a reasonable assumption.
  *
  */
-struct svm_node *dense_to_libsvm (double *x, npy_intp *dims)
+struct svm_node *dense_to_libsvm (double *x, Py_ssize_t *dims)
 {
     struct svm_node *node;
-    npy_intp len_row = dims[1];
+    Py_ssize_t len_row = dims[1];
     double *tx = x;
     int i;
 
@@ -82,7 +90,7 @@ void set_parameter(struct svm_parameter *param, int svm_type, int kernel_type, i
 /*
  * Fill an svm_problem struct. problem->x will be malloc'd.
  */
-void set_problem(struct svm_problem *problem, char *X, char *Y, char *sample_weight, npy_intp *dims, int kernel_type)
+void set_problem(struct svm_problem *problem, char *X, char *Y, char *sample_weight, Py_ssize_t *dims, int kernel_type)
 {
     if (problem == NULL) return;
     problem->l = (int) dims[0]; /* number of samples */
@@ -105,9 +113,9 @@ void set_problem(struct svm_problem *problem, char *X, char *Y, char *sample_wei
  *
  */
 struct svm_model *set_model(struct svm_parameter *param, int nr_class,
-                            char *SV, npy_intp *SV_dims,
-                            char *support, npy_intp *support_dims,
-                            npy_intp *sv_coef_strides,
+                            char *SV, Py_ssize_t *SV_dims,
+                            char *support, Py_ssize_t *support_dims,
+                            Py_ssize_t *sv_coef_strides,
                             char *sv_coef, char *rho, char *nSV,
                             char *probA, char *probB)
 {
@@ -128,6 +136,9 @@ struct svm_model *set_model(struct svm_parameter *param, int nr_class,
     if ((model->rho = malloc( m * sizeof(double))) == NULL)
         goto rho_error;
 
+    // This is only allocated in dynamic memory while training.
+    model->n_iter = NULL;
+
     model->nr_class = nr_class;
     model->param = *param;
     model->l = (int) support_dims[0];
@@ -204,18 +215,27 @@ struct svm_model *set_model(struct svm_parameter *param, int nr_class,
 /*
  * Get the number of support vectors in a model.
  */
-npy_intp get_l(struct svm_model *model)
+Py_ssize_t get_l(struct svm_model *model)
 {
-    return (npy_intp) model->l;
+    return (Py_ssize_t) model->l;
 }
 
 /*
  * Get the number of classes in a model, = 2 in regression/one class
  * svm.
  */
-npy_intp get_nr(struct svm_model *model)
+Py_ssize_t get_nr(struct svm_model *model)
 {
-    return (npy_intp) model->nr_class;
+    return (Py_ssize_t) model->nr_class;
+}
+
+/*
+ * Get the number of iterations run in optimization
+ */
+void copy_n_iter(char *data, struct svm_model *model)
+{
+    const int n_models = MAX(1, model->nr_class * (model->nr_class-1) / 2);
+    memcpy(data, model->n_iter, n_models * sizeof(int));
 }
 
 /*
@@ -233,10 +253,10 @@ void copy_sv_coef(char *data, struct svm_model *model)
     }
 }
 
-void copy_intercept(char *data, struct svm_model *model, npy_intp *dims)
+void copy_intercept(char *data, struct svm_model *model, Py_ssize_t *dims)
 {
     /* intercept = -rho */
-    npy_intp i, n = dims[0];
+    Py_ssize_t i, n = dims[0];
     double t, *ddata = (double *) data;
     for (i=0; irho[i];
@@ -251,7 +271,7 @@ void copy_intercept(char *data, struct svm_model *model, npy_intp *dims)
  * structures, so we have to do the conversion on the fly and also
  * iterate fast over data.
  */
-void copy_SV(char *data, struct svm_model *model, npy_intp *dims)
+void copy_SV(char *data, struct svm_model *model, Py_ssize_t *dims)
 {
     int i, n = model->l;
     double *tdata = (double *) data;
@@ -277,12 +297,12 @@ void copy_nSV(char *data, struct svm_model *model)
     memcpy(data, model->nSV, model->nr_class * sizeof(int));
 }
 
-void copy_probA(char *data, struct svm_model *model, npy_intp * dims)
+void copy_probA(char *data, struct svm_model *model, Py_ssize_t * dims)
 {
     memcpy(data, model->probA, dims[0] * sizeof(double));
 }
 
-void copy_probB(char *data, struct svm_model *model, npy_intp * dims)
+void copy_probB(char *data, struct svm_model *model, Py_ssize_t * dims)
 {
     memcpy(data, model->probB, dims[0] * sizeof(double));
 }
@@ -292,12 +312,12 @@ void copy_probB(char *data, struct svm_model *model, npy_intp * dims)
  *
  *  It will return -1 if we run out of memory.
  */
-int copy_predict(char *predict, struct svm_model *model, npy_intp *predict_dims,
+int copy_predict(char *predict, struct svm_model *model, Py_ssize_t *predict_dims,
                  char *dec_values, BlasFunctions *blas_functions)
 {
     double *t = (double *) dec_values;
     struct svm_node *predict_nodes;
-    npy_intp i;
+    Py_ssize_t i;
 
     predict_nodes = dense_to_libsvm((double *) predict, predict_dims);
 
@@ -312,9 +332,9 @@ int copy_predict(char *predict, struct svm_model *model, npy_intp *predict_dims,
 }
 
 int copy_predict_values(char *predict, struct svm_model *model,
-                        npy_intp *predict_dims, char *dec_values, int nr_class, BlasFunctions *blas_functions)
+                        Py_ssize_t *predict_dims, char *dec_values, int nr_class, BlasFunctions *blas_functions)
 {
-    npy_intp i;
+    Py_ssize_t i;
     struct svm_node *predict_nodes;
     predict_nodes = dense_to_libsvm((double *) predict, predict_dims);
     if (predict_nodes == NULL)
@@ -331,13 +351,13 @@ int copy_predict_values(char *predict, struct svm_model *model,
 
 
 
-int copy_predict_proba(char *predict, struct svm_model *model, npy_intp *predict_dims,
+int copy_predict_proba(char *predict, struct svm_model *model, Py_ssize_t *predict_dims,
                  char *dec_values, BlasFunctions *blas_functions)
 {
-    npy_intp i, n, m;
+    Py_ssize_t i, n, m;
     struct svm_node *predict_nodes;
     n = predict_dims[0];
-    m = (npy_intp) model->nr_class;
+    m = (Py_ssize_t) model->nr_class;
     predict_nodes = dense_to_libsvm((double *) predict, predict_dims);
     if (predict_nodes == NULL)
         return -1;
@@ -363,9 +383,11 @@ int free_model(struct svm_model *model)
     if (model == NULL) return -1;
     free(model->SV);
 
-    /* We don't free sv_ind, since we did not create them in
+    /* We don't free sv_ind and n_iter, since we did not create them in
        set_model */
-    /* free(model->sv_ind); */
+    /* free(model->sv_ind);
+     * free(model->n_iter);
+     */
     free(model->sv_coef);
     free(model->rho);
     free(model->label);
diff --git a/sklearn/svm/src/libsvm/libsvm_sparse_helper.c b/sklearn/svm/src/libsvm/libsvm_sparse_helper.c
index 9018000a1d47b..0ba153647cb8c 100644
--- a/sklearn/svm/src/libsvm/libsvm_sparse_helper.c
+++ b/sklearn/svm/src/libsvm/libsvm_sparse_helper.c
@@ -1,12 +1,19 @@
 #include 
-#include 
+#define PY_SSIZE_T_CLEAN
+#include 
 #include "svm.h"
 #include "_svm_cython_blas_helpers.h"
 
+
+#ifndef MAX
+    #define MAX(x, y) (((x) > (y)) ? (x) : (y))
+#endif
+
+
 /*
  * Convert scipy.sparse.csr to libsvm's sparse data structure
  */
-struct svm_csr_node **csr_to_libsvm (double *values, int* indices, int* indptr, npy_int n_samples)
+struct svm_csr_node **csr_to_libsvm (double *values, int* indices, int* indptr, int n_samples)
 {
     struct svm_csr_node **sparse, *temp;
     int i, j=0, k=0, n;
@@ -76,8 +83,8 @@ struct svm_parameter * set_parameter(int svm_type, int kernel_type, int degree,
  *
  * TODO: precomputed kernel.
  */
-struct svm_csr_problem * csr_set_problem (char *values, npy_intp *n_indices,
-		char *indices, npy_intp *n_indptr, char *indptr, char *Y,
+struct svm_csr_problem * csr_set_problem (char *values, Py_ssize_t *n_indices,
+		char *indices, Py_ssize_t *n_indptr, char *indptr, char *Y,
                 char *sample_weight, int kernel_type) {
 
     struct svm_csr_problem *problem;
@@ -99,8 +106,8 @@ struct svm_csr_problem * csr_set_problem (char *values, npy_intp *n_indices,
 
 
 struct svm_csr_model *csr_set_model(struct svm_parameter *param, int nr_class,
-                            char *SV_data, npy_intp *SV_indices_dims,
-                            char *SV_indices, npy_intp *SV_indptr_dims,
+                            char *SV_data, Py_ssize_t *SV_indices_dims,
+                            char *SV_indices, Py_ssize_t *SV_indptr_dims,
                             char *SV_intptr,
                             char *sv_coef, char *rho, char *nSV,
                             char *probA, char *probB)
@@ -122,6 +129,9 @@ struct svm_csr_model *csr_set_model(struct svm_parameter *param, int nr_class,
     if ((model->rho = malloc( m * sizeof(double))) == NULL)
         goto rho_error;
 
+    // This is only allocated in dynamic memory while training.
+    model->n_iter = NULL;
+
     /* in the case of precomputed kernels we do not use
        dense_to_precomputed because we don't want the leading 0. As
        indices start at 1 (not at 0) this will work */
@@ -203,8 +213,8 @@ struct svm_csr_model *csr_set_model(struct svm_parameter *param, int nr_class,
 /*
  * Copy support vectors into a scipy.sparse.csr matrix
  */
-int csr_copy_SV (char *data, npy_intp *n_indices,
-		char *indices, npy_intp *n_indptr, char *indptr,
+int csr_copy_SV (char *data, Py_ssize_t *n_indices,
+		char *indices, Py_ssize_t *n_indptr, char *indptr,
 		struct svm_csr_model *model, int n_features)
 {
 	int i, j, k=0, index;
@@ -227,9 +237,9 @@ int csr_copy_SV (char *data, npy_intp *n_indices,
 }
 
 /* get number of nonzero coefficients in support vectors */
-npy_intp get_nonzero_SV (struct svm_csr_model *model) {
+Py_ssize_t get_nonzero_SV (struct svm_csr_model *model) {
 	int i, j;
-	npy_intp count=0;
+	Py_ssize_t count=0;
 	for (i=0; il; ++i) {
 		j = 0;
 		while (model->SV[i][j].index != -1) {
@@ -242,14 +252,14 @@ npy_intp get_nonzero_SV (struct svm_csr_model *model) {
 
 
 /*
- * Predict using a model, where data is expected to be enconded into a csr matrix.
+ * Predict using a model, where data is expected to be encoded into a csr matrix.
  */
-int csr_copy_predict (npy_intp *data_size, char *data, npy_intp *index_size,
-		char *index, npy_intp *intptr_size, char *intptr, struct svm_csr_model *model,
+int csr_copy_predict (Py_ssize_t *data_size, char *data, Py_ssize_t *index_size,
+		char *index, Py_ssize_t *intptr_size, char *intptr, struct svm_csr_model *model,
 		char *dec_values, BlasFunctions *blas_functions) {
     double *t = (double *) dec_values;
     struct svm_csr_node **predict_nodes;
-    npy_intp i;
+    Py_ssize_t i;
 
     predict_nodes = csr_to_libsvm((double *) data, (int *) index,
                                   (int *) intptr, intptr_size[0]-1);
@@ -265,11 +275,11 @@ int csr_copy_predict (npy_intp *data_size, char *data, npy_intp *index_size,
     return 0;
 }
 
-int csr_copy_predict_values (npy_intp *data_size, char *data, npy_intp *index_size,
-                char *index, npy_intp *intptr_size, char *intptr, struct svm_csr_model *model,
+int csr_copy_predict_values (Py_ssize_t *data_size, char *data, Py_ssize_t *index_size,
+                char *index, Py_ssize_t *intptr_size, char *intptr, struct svm_csr_model *model,
                 char *dec_values, int nr_class, BlasFunctions *blas_functions) {
     struct svm_csr_node **predict_nodes;
-    npy_intp i;
+    Py_ssize_t i;
 
     predict_nodes = csr_to_libsvm((double *) data, (int *) index,
                                   (int *) intptr, intptr_size[0]-1);
@@ -287,12 +297,12 @@ int csr_copy_predict_values (npy_intp *data_size, char *data, npy_intp *index_si
     return 0;
 }
 
-int csr_copy_predict_proba (npy_intp *data_size, char *data, npy_intp *index_size,
-		char *index, npy_intp *intptr_size, char *intptr, struct svm_csr_model *model,
+int csr_copy_predict_proba (Py_ssize_t *data_size, char *data, Py_ssize_t *index_size,
+		char *index, Py_ssize_t *intptr_size, char *intptr, struct svm_csr_model *model,
 		char *dec_values, BlasFunctions *blas_functions) {
 
     struct svm_csr_node **predict_nodes;
-    npy_intp i;
+    Py_ssize_t i;
     int m = model->nr_class;
 
     predict_nodes = csr_to_libsvm((double *) data, (int *) index,
@@ -310,15 +320,15 @@ int csr_copy_predict_proba (npy_intp *data_size, char *data, npy_intp *index_siz
 }
 
 
-npy_intp get_nr(struct svm_csr_model *model)
+Py_ssize_t get_nr(struct svm_csr_model *model)
 {
-    return (npy_intp) model->nr_class;
+    return (Py_ssize_t) model->nr_class;
 }
 
-void copy_intercept(char *data, struct svm_csr_model *model, npy_intp *dims)
+void copy_intercept(char *data, struct svm_csr_model *model, Py_ssize_t *dims)
 {
     /* intercept = -rho */
-    npy_intp i, n = dims[0];
+    Py_ssize_t i, n = dims[0];
     double t, *ddata = (double *) data;
     for (i=0; irho[i];
@@ -348,12 +358,21 @@ void copy_sv_coef(char *data, struct svm_csr_model *model)
     }
 }
 
+/*
+ * Get the number of iterations run in optimization
+ */
+void copy_n_iter(char *data, struct svm_csr_model *model)
+{
+    const int n_models = MAX(1, model->nr_class * (model->nr_class-1) / 2);
+    memcpy(data, model->n_iter, n_models * sizeof(int));
+}
+
 /*
  * Get the number of support vectors in a model.
  */
-npy_intp get_l(struct svm_csr_model *model)
+Py_ssize_t get_l(struct svm_csr_model *model)
 {
-    return (npy_intp) model->l;
+    return (Py_ssize_t) model->l;
 }
 
 void copy_nSV(char *data, struct svm_csr_model *model)
@@ -372,12 +391,12 @@ void copy_label(char *data, struct svm_csr_model *model)
     memcpy(data, model->label, model->nr_class * sizeof(int));
 }
 
-void copy_probA(char *data, struct svm_csr_model *model, npy_intp * dims)
+void copy_probA(char *data, struct svm_csr_model *model, Py_ssize_t * dims)
 {
     memcpy(data, model->probA, dims[0] * sizeof(double));
 }
 
-void copy_probB(char *data, struct svm_csr_model *model, npy_intp * dims)
+void copy_probB(char *data, struct svm_csr_model *model, Py_ssize_t * dims)
 {
     memcpy(data, model->probB, dims[0] * sizeof(double));
 }
@@ -402,6 +421,7 @@ int free_problem(struct svm_csr_problem *problem)
 int free_model(struct svm_csr_model *model)
 {
     /* like svm_free_and_destroy_model, but does not free sv_coef[i] */
+    /* We don't free n_iter, since we did not create them in set_model. */
     if (model == NULL) return -1;
     free(model->SV);
     free(model->sv_coef);
diff --git a/sklearn/svm/src/libsvm/svm.cpp b/sklearn/svm/src/libsvm/svm.cpp
index d209e35fc0a35..a6f191d6616c9 100644
--- a/sklearn/svm/src/libsvm/svm.cpp
+++ b/sklearn/svm/src/libsvm/svm.cpp
@@ -31,7 +31,7 @@ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 
-/* 
+/*
    Modified 2010:
 
    - Support for dense data by Ming-Fang Weng
@@ -55,6 +55,10 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
      Sylvain Marie, Schneider Electric
      see 
 
+   Modified 2021:
+
+   - Exposed number of iterations run in optimization, Juan Martín Loyola.
+     See 
  */
 
 #include 
@@ -125,7 +129,7 @@ static void info(const char *fmt,...)
 and dense versions of this library */
 #ifdef _DENSE_REP
   #ifdef PREFIX
-    #undef PREFIX  
+    #undef PREFIX
   #endif
   #ifdef NAMESPACE
     #undef NAMESPACE
@@ -136,7 +140,7 @@ and dense versions of this library */
 #else
   /* sparse representation */
   #ifdef PREFIX
-    #undef PREFIX  
+    #undef PREFIX
   #endif
   #ifdef NAMESPACE
     #undef NAMESPACE
@@ -163,7 +167,7 @@ class Cache
 	// return some position p where [p,len) need to be filled
 	// (p >= len if nothing needs to be filled)
 	int get_data(const int index, Qfloat **data, int len);
-	void swap_index(int i, int j);	
+	void swap_index(int i, int j);
 private:
 	int l;
 	long int size;
@@ -439,7 +443,7 @@ double Kernel::dot(const PREFIX(node) *px, const PREFIX(node) *py, BlasFunctions
 				++py;
 			else
 				++px;
-		}			
+		}
 	}
 	return sum;
 }
@@ -483,7 +487,7 @@ double Kernel::k_function(const PREFIX(node) *x, const PREFIX(node) *y,
 				else
 				{
 					if(x->index > y->index)
-					{	
+					{
 						sum += y->value * y->value;
 						++y;
 					}
@@ -520,7 +524,7 @@ double Kernel::k_function(const PREFIX(node) *x, const PREFIX(node) *y,
 #endif
                     }
 		default:
-			return 0;  // Unreachable 
+			return 0;  // Unreachable
 	}
 }
 // An SMO algorithm in Fan et al., JMLR 6(2005), p. 1889--1918
@@ -553,6 +557,7 @@ class Solver {
                 double *upper_bound;
 		double r;	// for Solver_NU
                 bool solve_timed_out;
+		int n_iter;
 	};
 
 	void Solve(int l, const QMatrix& Q, const double *p_, const schar *y_,
@@ -597,7 +602,7 @@ class Solver {
 	virtual double calculate_rho();
 	virtual void do_shrinking();
 private:
-	bool be_shrunk(int i, double Gmax1, double Gmax2);	
+	bool be_shrunk(int i, double Gmax1, double Gmax2);
 };
 
 void Solver::swap_index(int i, int j)
@@ -745,11 +750,11 @@ void Solver::Solve(int l, const QMatrix& Q, const double *p_, const schar *y_,
 			else
 				counter = 1;	// do shrinking next iteration
 		}
-		
+
 		++iter;
 
 		// update alpha[i] and alpha[j], handle bounds carefully
-		
+
 		const Qfloat *Q_i = Q.get_Q(i,active_size);
 		const Qfloat *Q_j = Q.get_Q(j,active_size);
 
@@ -768,7 +773,7 @@ void Solver::Solve(int l, const QMatrix& Q, const double *p_, const schar *y_,
 			double diff = alpha[i] - alpha[j];
 			alpha[i] += delta;
 			alpha[j] += delta;
-			
+
 			if(diff > 0)
 			{
 				if(alpha[j] < 0)
@@ -850,7 +855,7 @@ void Solver::Solve(int l, const QMatrix& Q, const double *p_, const schar *y_,
 
 		double delta_alpha_i = alpha[i] - old_alpha_i;
 		double delta_alpha_j = alpha[j] - old_alpha_j;
-		
+
 		for(int k=0;kupper_bound[i] = C[i];
 
+	// store number of iterations
+	si->n_iter = iter;
+
 	info("\noptimization finished, #iter = %d\n",iter);
 
 	delete[] p;
@@ -939,7 +947,7 @@ int Solver::select_working_set(int &out_i, int &out_j)
 	// j: minimizes the decrease of obj value
 	//    (if quadratic coefficient <= 0, replace it with tau)
 	//    -y_j*grad(f)_j < -y_i*grad(f)_i, j in I_low(\alpha)
-	
+
 	double Gmax = -INF;
 	double Gmax2 = -INF;
 	int Gmax_idx = -1;
@@ -947,7 +955,7 @@ int Solver::select_working_set(int &out_i, int &out_j)
 	double obj_diff_min = INF;
 
 	for(int t=0;t= Gmax)
@@ -982,7 +990,7 @@ int Solver::select_working_set(int &out_i, int &out_j)
 					Gmax2 = G[j];
 				if (grad_diff > 0)
 				{
-					double obj_diff; 
+					double obj_diff;
 					double quad_coef = QD[i]+QD[j]-2.0*y[i]*Q_i[j];
 					if (quad_coef > 0)
 						obj_diff = -(grad_diff*grad_diff)/quad_coef;
@@ -1006,7 +1014,7 @@ int Solver::select_working_set(int &out_i, int &out_j)
 					Gmax2 = -G[j];
 				if (grad_diff > 0)
 				{
-					double obj_diff; 
+					double obj_diff;
 					double quad_coef = QD[i]+QD[j]+2.0*y[i]*Q_i[j];
 					if (quad_coef > 0)
 						obj_diff = -(grad_diff*grad_diff)/quad_coef;
@@ -1044,7 +1052,7 @@ bool Solver::be_shrunk(int i, double Gmax1, double Gmax2)
 	{
 		if(y[i]==+1)
 			return(G[i] > Gmax2);
-		else	
+		else
 			return(G[i] > Gmax1);
 	}
 	else
@@ -1060,27 +1068,27 @@ void Solver::do_shrinking()
 	// find maximal violating pair first
 	for(i=0;i= Gmax1)
 					Gmax1 = -G[i];
 			}
-			if(!is_lower_bound(i))	
+			if(!is_lower_bound(i))
 			{
 				if(G[i] >= Gmax2)
 					Gmax2 = G[i];
 			}
 		}
-		else	
+		else
 		{
-			if(!is_upper_bound(i))	
+			if(!is_upper_bound(i))
 			{
 				if(-G[i] >= Gmax2)
 					Gmax2 = -G[i];
 			}
-			if(!is_lower_bound(i))	
+			if(!is_lower_bound(i))
 			{
 				if(G[i] >= Gmax1)
 					Gmax1 = G[i];
@@ -1088,7 +1096,7 @@ void Solver::do_shrinking()
 		}
 	}
 
-	if(unshrink == false && Gmax1 + Gmax2 <= eps*10) 
+	if(unshrink == false && Gmax1 + Gmax2 <= eps*10)
 	{
 		unshrink = true;
 		reconstruct_gradient();
@@ -1227,14 +1235,14 @@ int Solver_NU::select_working_set(int &out_i, int &out_j)
 	{
 		if(y[j]==+1)
 		{
-			if (!is_lower_bound(j))	
+			if (!is_lower_bound(j))
 			{
 				double grad_diff=Gmaxp+G[j];
 				if (G[j] >= Gmaxp2)
 					Gmaxp2 = G[j];
 				if (grad_diff > 0)
 				{
-					double obj_diff; 
+					double obj_diff;
 					double quad_coef = QD[ip]+QD[j]-2*Q_ip[j];
 					if (quad_coef > 0)
 						obj_diff = -(grad_diff*grad_diff)/quad_coef;
@@ -1258,7 +1266,7 @@ int Solver_NU::select_working_set(int &out_i, int &out_j)
 					Gmaxn2 = -G[j];
 				if (grad_diff > 0)
 				{
-					double obj_diff; 
+					double obj_diff;
 					double quad_coef = QD[in]+QD[j]-2*Q_in[j];
 					if (quad_coef > 0)
 						obj_diff = -(grad_diff*grad_diff)/quad_coef;
@@ -1293,14 +1301,14 @@ bool Solver_NU::be_shrunk(int i, double Gmax1, double Gmax2, double Gmax3, doubl
 	{
 		if(y[i]==+1)
 			return(-G[i] > Gmax1);
-		else	
+		else
 			return(-G[i] > Gmax4);
 	}
 	else if(is_lower_bound(i))
 	{
 		if(y[i]==+1)
 			return(G[i] > Gmax2);
-		else	
+		else
 			return(G[i] > Gmax3);
 	}
 	else
@@ -1329,14 +1337,14 @@ void Solver_NU::do_shrinking()
 		if(!is_lower_bound(i))
 		{
 			if(y[i]==+1)
-			{	
+			{
 				if(G[i] > Gmax2) Gmax2 = G[i];
 			}
 			else	if(G[i] > Gmax3) Gmax3 = G[i];
 		}
 	}
 
-	if(unshrink == false && max(Gmax1+Gmax2,Gmax3+Gmax4) <= eps*10) 
+	if(unshrink == false && max(Gmax1+Gmax2,Gmax3+Gmax4) <= eps*10)
 	{
 		unshrink = true;
 		reconstruct_gradient();
@@ -1399,12 +1407,12 @@ double Solver_NU::calculate_rho()
 		r1 = sum_free1/nr_free1;
 	else
 		r1 = (ub1+lb1)/2;
-	
+
 	if(nr_free2 > 0)
 		r2 = sum_free2/nr_free2;
 	else
 		r2 = (ub2+lb2)/2;
-	
+
 	si->r = (r1+r2)/2;
 	return (r1-r2)/2;
 }
@@ -1413,7 +1421,7 @@ double Solver_NU::calculate_rho()
 // Q matrices for various formulations
 //
 class SVC_Q: public Kernel
-{ 
+{
 public:
 	SVC_Q(const PREFIX(problem)& prob, const svm_parameter& param, const schar *y_, BlasFunctions *blas_functions)
 	:Kernel(prob.l, prob.x, param, blas_functions)
@@ -1424,7 +1432,7 @@ class SVC_Q: public Kernel
 		for(int i=0;i*kernel_function)(i,i);
 	}
-	
+
 	Qfloat *get_Q(int i, int len) const
 	{
 		Qfloat *data;
@@ -1473,7 +1481,7 @@ class ONE_CLASS_Q: public Kernel
 		for(int i=0;i*kernel_function)(i,i);
 	}
-	
+
 	Qfloat *get_Q(int i, int len) const
 	{
 		Qfloat *data;
@@ -1509,7 +1517,7 @@ class ONE_CLASS_Q: public Kernel
 };
 
 class SVR_Q: public Kernel
-{ 
+{
 public:
 	SVR_Q(const PREFIX(problem)& prob, const svm_parameter& param, BlasFunctions *blas_functions)
 	:Kernel(prob.l, prob.x, param, blas_functions)
@@ -1539,7 +1547,7 @@ class SVR_Q: public Kernel
 		swap(index[i],index[j]);
 		swap(QD[i],QD[j]);
 	}
-	
+
 	Qfloat *get_Q(int i, int len) const
 	{
 		Qfloat *data;
@@ -1655,7 +1663,7 @@ static void solve_nu_svc(
 
 		C[i] = prob->W[i];
 	}
-	
+
 	double nu_l = 0;
 	for(i=0;iupper_bound[i] /= r;                
+		si->upper_bound[i] /= r;
         }
 
 	si->rho /= r;
@@ -1836,7 +1844,8 @@ static void solve_nu_svr(
 struct decision_function
 {
 	double *alpha;
-	double rho;	
+	double rho;
+	int n_iter;
 };
 
 static decision_function svm_train_one(
@@ -1848,23 +1857,23 @@ static decision_function svm_train_one(
 	switch(param->svm_type)
 	{
  		case C_SVC:
-			si.upper_bound = Malloc(double,prob->l); 
+			si.upper_bound = Malloc(double,prob->l);
  			solve_c_svc(prob,param,alpha,&si,Cp,Cn,blas_functions);
  			break;
  		case NU_SVC:
-			si.upper_bound = Malloc(double,prob->l); 
+			si.upper_bound = Malloc(double,prob->l);
  			solve_nu_svc(prob,param,alpha,&si,blas_functions);
  			break;
  		case ONE_CLASS:
-			si.upper_bound = Malloc(double,prob->l); 
+			si.upper_bound = Malloc(double,prob->l);
  			solve_one_class(prob,param,alpha,&si,blas_functions);
  			break;
  		case EPSILON_SVR:
-			si.upper_bound = Malloc(double,2*prob->l); 
+			si.upper_bound = Malloc(double,2*prob->l);
  			solve_epsilon_svr(prob,param,alpha,&si,blas_functions);
  			break;
  		case NU_SVR:
-			si.upper_bound = Malloc(double,2*prob->l); 
+			si.upper_bound = Malloc(double,2*prob->l);
  			solve_nu_svr(prob,param,alpha,&si,blas_functions);
  			break;
 	}
@@ -1902,12 +1911,13 @@ static decision_function svm_train_one(
 	decision_function f;
 	f.alpha = alpha;
 	f.rho = si.rho;
+	f.n_iter = si.n_iter;
 	return f;
 }
 
 // Platt's binary SVM Probabilistic Output: an improvement from Lin et al.
 static void sigmoid_train(
-	int l, const double *dec_values, const double *labels, 
+	int l, const double *dec_values, const double *labels,
 	double& A, double& B)
 {
 	double prior1=0, prior0 = 0;
@@ -1916,7 +1926,7 @@ static void sigmoid_train(
 	for (i=0;i 0) prior1+=1;
 		else prior0+=1;
-	
+
 	int max_iter=100;	// Maximal number of iterations
 	double min_step=1e-10;	// Minimal step taken in line search
 	double sigma=1e-12;	// For numerically strict PD of Hessian
@@ -1926,8 +1936,8 @@ static void sigmoid_train(
 	double *t=Malloc(double,l);
 	double fApB,p,q,h11,h22,h21,g1,g2,det,dA,dB,gd,stepsize;
 	double newA,newB,newf,d1,d2;
-	int iter; 
-	
+	int iter;
+
 	// Initial Point and Initial Fun Value
 	A=0.0; B=log((prior0+1.0)/(prior1+1.0));
 	double fval = 0.0;
@@ -2037,7 +2047,7 @@ static void multiclass_probability(int k, double **r, double *p)
 	double **Q=Malloc(double *,k);
 	double *Qp=Malloc(double,k);
 	double pQp, eps=0.005/k;
-	
+
 	for (t=0;tx+perm[j]),&(dec_values[perm[j]]), blas_functions); 
+                                PREFIX(predict_values)(submodel,(prob->x+perm[j]),&(dec_values[perm[j]]), blas_functions);
 #else
-				PREFIX(predict_values)(submodel,prob->x[perm[j]],&(dec_values[perm[j]]), blas_functions); 
+				PREFIX(predict_values)(submodel,prob->x[perm[j]],&(dec_values[perm[j]]), blas_functions);
 #endif
 				// ensure +1 -1 order; reason not using CV subroutine
 				dec_values[perm[j]] *= submodel->label[0];
-			}		
+			}
 			PREFIX(free_and_destroy_model)(&submodel);
 			PREFIX(destroy_param)(&subparam);
 		}
 		free(subprob.x);
 		free(subprob.y);
                 free(subprob.W);
-	}		
+	}
 	sigmoid_train(prob->l,dec_values,prob->y,probA,probB);
 	free(dec_values);
 	free(perm);
 }
 
-// Return parameter of a Laplace distribution 
+// Return parameter of a Laplace distribution
 static double svm_svr_probability(
 	const PREFIX(problem) *prob, const svm_parameter *param, BlasFunctions *blas_functions)
 {
@@ -2210,15 +2220,15 @@ static double svm_svr_probability(
 	{
 		ymv[i]=prob->y[i]-ymv[i];
 		mae += fabs(ymv[i]);
-	}		
+	}
 	mae /= prob->l;
 	double std=sqrt(2*mae*mae);
 	int count=0;
 	mae=0;
 	for(i=0;il;i++)
-		if (fabs(ymv[i]) > 5*std) 
+		if (fabs(ymv[i]) > 5*std)
 			count=count+1;
-		else 
+		else
 			mae+=fabs(ymv[i]);
 	mae /= (prob->l-count);
 	info("Prob. model for test data: target value = predicted value + z,\nz: Laplace distribution e^(-|z|/sigma)/(2sigma),sigma= %g\n",mae);
@@ -2237,7 +2247,7 @@ static void svm_group_classes(const PREFIX(problem) *prob, int *nr_class_ret, in
 	int nr_class = 0;
 	int *label = Malloc(int,max_nr_class);
 	int *count = Malloc(int,max_nr_class);
-	int *data_label = Malloc(int,l);	
+	int *data_label = Malloc(int,l);
 	int i, j, this_label, this_count;
 
 	for(i=0;i 0.
 //
-static void remove_zero_weight(PREFIX(problem) *newprob, const PREFIX(problem) *prob) 
+static void remove_zero_weight(PREFIX(problem) *newprob, const PREFIX(problem) *prob)
 {
 	int i;
 	int l = 0;
@@ -2376,7 +2386,7 @@ PREFIX(model) *PREFIX(train)(const PREFIX(problem) *prob, const svm_parameter *p
 		model->probA = NULL; model->probB = NULL;
 		model->sv_coef = Malloc(double *,1);
 
-		if(param->probability && 
+		if(param->probability &&
 		   (param->svm_type == EPSILON_SVR ||
 		    param->svm_type == NU_SVR))
 		{
@@ -2387,6 +2397,8 @@ PREFIX(model) *PREFIX(train)(const PREFIX(problem) *prob, const svm_parameter *p
                 NAMESPACE::decision_function f = NAMESPACE::svm_train_one(prob,param,0,0, status,blas_functions);
 		model->rho = Malloc(double,1);
 		model->rho[0] = f.rho;
+		model->n_iter = Malloc(int,1);
+		model->n_iter[0] = f.n_iter;
 
 		int nSV = 0;
 		int i;
@@ -2408,7 +2420,7 @@ PREFIX(model) *PREFIX(train)(const PREFIX(problem) *prob, const svm_parameter *p
                                 model->sv_ind[j] = i;
 				model->sv_coef[0][j] = f.alpha[i];
 				++j;
-			}		
+			}
 
 		free(f.alpha);
 	}
@@ -2423,7 +2435,7 @@ PREFIX(model) *PREFIX(train)(const PREFIX(problem) *prob, const svm_parameter *p
 		int *perm = Malloc(int,l);
 
 		// group training data of the same class
-                NAMESPACE::svm_group_classes(prob,&nr_class,&label,&start,&count,perm);		
+                NAMESPACE::svm_group_classes(prob,&nr_class,&label,&start,&count,perm);
 #ifdef _DENSE_REP
 		PREFIX(node) *x = Malloc(PREFIX(node),l);
 #else
@@ -2444,7 +2456,7 @@ PREFIX(model) *PREFIX(train)(const PREFIX(problem) *prob, const svm_parameter *p
 		for(i=0;iC;
 		for(i=0;inr_weight;i++)
-		{	
+		{
 			int j;
 			for(j=0;jweight_label[i] == label[j])
@@ -2456,7 +2468,7 @@ PREFIX(model) *PREFIX(train)(const PREFIX(problem) *prob, const svm_parameter *p
 		}
 
 		// train k*(k-1)/2 models
-		
+
 		bool *nonzero = Malloc(bool,l);
 		for(i=0;inr_class = nr_class;
-		
+
 		model->label = Malloc(int,nr_class);
 		for(i=0;ilabel[i] = label[i];
-		
+
 		model->rho = Malloc(double,nr_class*(nr_class-1)/2);
+		model->n_iter = Malloc(int,nr_class*(nr_class-1)/2);
 		for(i=0;irho[i] = f[i].rho;
+			model->n_iter[i] = f[i].n_iter;
+		}
 
 		if(param->probability)
 		{
@@ -2550,7 +2566,7 @@ PREFIX(model) *PREFIX(train)(const PREFIX(problem) *prob, const svm_parameter *p
 			int nSV = 0;
 			for(int j=0;jSV[p] = x[i];
                                 model->sv_ind[p] = perm[i];
                                 ++p;
@@ -2597,7 +2613,7 @@ PREFIX(model) *PREFIX(train)(const PREFIX(problem) *prob, const svm_parameter *p
 				int sj = start[j];
 				int ci = count[i];
 				int cj = count[j];
-				
+
 				int q = nz_start[i];
 				int k;
 				for(k=0;ksv_coef[i][q++] = f[p].alpha[ci+k];
 				++p;
 			}
-		
+
 		free(label);
 		free(probA);
 		free(probB);
@@ -2661,7 +2677,7 @@ void PREFIX(cross_validation)(const PREFIX(problem) *prob, const svm_parameter *
 		int *index = Malloc(int,l);
 		for(i=0;iprobability && 
+		if(param->probability &&
 		   (param->svm_type == C_SVC || param->svm_type == NU_SVC))
 		{
 			double *prob_estimates=Malloc(double, PREFIX(get_nr_class)(submodel));
@@ -2751,7 +2767,7 @@ void PREFIX(cross_validation)(const PREFIX(problem) *prob, const svm_parameter *
 #else
                                 target[perm[j]] = PREFIX(predict_probability)(submodel,prob->x[perm[j]],prob_estimates, blas_functions);
 #endif
-			free(prob_estimates);			
+			free(prob_estimates);
 		}
 		else
 			for(j=begin;jsv_coef[0];
 		double sum = 0;
-		
+
 		for(i=0;il;i++)
 #ifdef _DENSE_REP
                     sum += sv_coef[i] * NAMESPACE::Kernel::k_function(x,model->SV+i,model->param,blas_functions);
@@ -2827,7 +2843,7 @@ double PREFIX(predict_values)(const PREFIX(model) *model, const PREFIX(node) *x,
 	{
 		int nr_class = model->nr_class;
 		int l = model->l;
-		
+
 		double *kvalue = Malloc(double,l);
 		for(i=0;inSV[i];
 				int cj = model->nSV[j];
-				
+
 				int k;
 				double *coef1 = model->sv_coef[j-1];
 				double *coef2 = model->sv_coef[i];
@@ -2892,7 +2908,7 @@ double PREFIX(predict)(const PREFIX(model) *model, const PREFIX(node) *x, BlasFu
 	   model->param.svm_type == EPSILON_SVR ||
 	   model->param.svm_type == NU_SVR)
 		dec_values = Malloc(double, 1);
-	else 
+	else
 		dec_values = Malloc(double, nr_class*(nr_class-1)/2);
 	double pred_result = PREFIX(predict_values)(model, x, dec_values, blas_functions);
 	free(dec_values);
@@ -2931,10 +2947,10 @@ double PREFIX(predict_probability)(
 		for(i=0;ilabel[prob_max_idx];
 	}
-	else 
+	else
 		return PREFIX(predict)(model, x, blas_functions);
 }
 
@@ -2978,6 +2994,9 @@ void PREFIX(free_model_content)(PREFIX(model)* model_ptr)
 
 	free(model_ptr->nSV);
 	model_ptr->nSV = NULL;
+
+	free(model_ptr->n_iter);
+	model_ptr->n_iter = NULL;
 }
 
 void PREFIX(free_and_destroy_model)(PREFIX(model)** model_ptr_ptr)
@@ -3007,9 +3026,9 @@ const char *PREFIX(check_parameter)(const PREFIX(problem) *prob, const svm_param
 	   svm_type != EPSILON_SVR &&
 	   svm_type != NU_SVR)
 		return "unknown svm type";
-	
+
 	// kernel_type, degree
-	
+
 	int kernel_type = param->kernel_type;
 	if(kernel_type != LINEAR &&
 	   kernel_type != POLY &&
@@ -3062,7 +3081,7 @@ const char *PREFIX(check_parameter)(const PREFIX(problem) *prob, const svm_param
 
 
 	// check whether nu-svc is feasible
-	
+
 	if(svm_type == NU_SVC)
 	{
 		int l = prob->l;
@@ -3096,7 +3115,7 @@ const char *PREFIX(check_parameter)(const PREFIX(problem) *prob, const svm_param
 				++nr_class;
 			}
 		}
-	
+
 		for(i=0;il != newprob.l && 
+		if(newprob.l == 0) {
+			free(newprob.x);
+			free(newprob.y);
+			free(newprob.W);
+			return "Invalid input - all samples have zero or negative weights.";
+		}
+		else if(prob->l != newprob.l &&
 		        svm_type == C_SVC)
 		{
 			bool only_one_label = true;
@@ -3141,15 +3163,17 @@ const char *PREFIX(check_parameter)(const PREFIX(problem) *prob, const svm_param
 					break;
 				}
 			}
-			if(only_one_label == true)
-				msg = "Invalid input - all samples with positive weights have the same label.";
+			if(only_one_label) {
+				free(newprob.x);
+				free(newprob.y);
+				free(newprob.W);
+				return "Invalid input - all samples with positive weights belong to the same class.";
+			}
 		}
 
 		free(newprob.x);
 		free(newprob.y);
 		free(newprob.W);
-		if(msg != NULL)
-			return msg;
 	}
 	return NULL;
 }
diff --git a/sklearn/svm/src/libsvm/svm.h b/sklearn/svm/src/libsvm/svm.h
index 0e509c61c37ed..518872c67bc5c 100644
--- a/sklearn/svm/src/libsvm/svm.h
+++ b/sklearn/svm/src/libsvm/svm.h
@@ -76,6 +76,7 @@ struct svm_model
 	int l;			/* total #SV */
 	struct svm_node *SV;		/* SVs (SV[l]) */
 	double **sv_coef;	/* coefficients for SVs in decision functions (sv_coef[k-1][l]) */
+	int *n_iter;		/* number of iterations run by the optimization routine to fit the model */
 
 	int *sv_ind;            /* index of support vectors */
 
@@ -101,6 +102,7 @@ struct svm_csr_model
 	int l;			/* total #SV */
 	struct svm_csr_node **SV;		/* SVs (SV[l]) */
 	double **sv_coef;	/* coefficients for SVs in decision functions (sv_coef[k-1][l]) */
+	int *n_iter;		/* number of iterations run by the optimization routine to fit the model */
 
         int *sv_ind;            /* index of support vectors */
 
@@ -118,7 +120,7 @@ struct svm_csr_model
 				/* 0 if svm_model is created by svm_train */
 };
 
-
+/* svm_ functions are defined by libsvm_template.cpp from generic versions in svm.cpp */
 struct svm_model *svm_train(const struct svm_problem *prob, const struct svm_parameter *param, int *status, BlasFunctions *blas_functions);
 void svm_cross_validation(const struct svm_problem *prob, const struct svm_parameter *param, int nr_fold, double *target, BlasFunctions *blas_functions);
 
@@ -145,6 +147,7 @@ void svm_set_print_string_function(void (*print_func)(const char *));
 
 /* sparse version */
 
+/* svm_csr_ functions are defined by libsvm_template.cpp from generic versions in svm.cpp */
 struct svm_csr_model *svm_csr_train(const struct svm_csr_problem *prob, const struct svm_parameter *param, int *status, BlasFunctions *blas_functions);
 void svm_csr_cross_validation(const struct svm_csr_problem *prob, const struct svm_parameter *param, int nr_fold, double *target, BlasFunctions *blas_functions);
 
diff --git a/sklearn/svm/tests/test_bounds.py b/sklearn/svm/tests/test_bounds.py
index 423d5ed7a7fba..af7e8cfb1159d 100644
--- a/sklearn/svm/tests/test_bounds.py
+++ b/sklearn/svm/tests/test_bounds.py
@@ -1,55 +1,53 @@
 import numpy as np
-from scipy import sparse as sp
-from scipy import stats
-
 import pytest
+from scipy import stats
 
-from sklearn.svm._bounds import l1_min_c
-from sklearn.svm import LinearSVC
 from sklearn.linear_model import LogisticRegression
-from sklearn.svm._newrand import set_seed_wrap, bounded_rand_int_wrap
-
-from sklearn.utils._testing import assert_raise_message
-
+from sklearn.svm import LinearSVC
+from sklearn.svm._bounds import l1_min_c
+from sklearn.svm._newrand import bounded_rand_int_wrap, set_seed_wrap
+from sklearn.utils.fixes import CSR_CONTAINERS
 
 dense_X = [[-1, 0], [0, 1], [1, 1], [1, 1]]
-sparse_X = sp.csr_matrix(dense_X)
 
 Y1 = [0, 1, 1, 1]
 Y2 = [2, 1, 0, 0]
 
 
-@pytest.mark.parametrize('loss', ['squared_hinge', 'log'])
-@pytest.mark.parametrize('X_label', ['sparse', 'dense'])
-@pytest.mark.parametrize('Y_label', ['two-classes', 'multi-class'])
-@pytest.mark.parametrize('intercept_label', ['no-intercept', 'fit-intercept'])
-def test_l1_min_c(loss, X_label, Y_label, intercept_label):
-    Xs = {'sparse': sparse_X, 'dense': dense_X}
-    Ys = {'two-classes': Y1, 'multi-class': Y2}
-    intercepts = {'no-intercept': {'fit_intercept': False},
-                  'fit-intercept': {'fit_intercept': True,
-                                    'intercept_scaling': 10}}
-
-    X = Xs[X_label]
+# TODO(1.8): remove filterwarnings after the deprecation of liblinear multiclass
+#            and maybe remove LogisticRegression from this test
+@pytest.mark.filterwarnings(
+    "ignore:.*'liblinear' solver for multiclass classification is deprecated.*"
+)
+@pytest.mark.parametrize("X_container", CSR_CONTAINERS + [np.array])
+@pytest.mark.parametrize("loss", ["squared_hinge", "log"])
+@pytest.mark.parametrize("Y_label", ["two-classes", "multi-class"])
+@pytest.mark.parametrize("intercept_label", ["no-intercept", "fit-intercept"])
+def test_l1_min_c(X_container, loss, Y_label, intercept_label):
+    Ys = {"two-classes": Y1, "multi-class": Y2}
+    intercepts = {
+        "no-intercept": {"fit_intercept": False},
+        "fit-intercept": {"fit_intercept": True, "intercept_scaling": 10},
+    }
+
+    X = X_container(dense_X)
     Y = Ys[Y_label]
     intercept_params = intercepts[intercept_label]
     check_l1_min_c(X, Y, loss, **intercept_params)
 
 
-def test_l1_min_c_l2_loss():
-    # loss='l2' should raise ValueError
-    assert_raise_message(ValueError, "loss type not in",
-                         l1_min_c, dense_X, Y1, loss="l2")
-
-
-def check_l1_min_c(X, y, loss, fit_intercept=True, intercept_scaling=None):
-    min_c = l1_min_c(X, y, loss=loss, fit_intercept=fit_intercept,
-                     intercept_scaling=intercept_scaling)
+def check_l1_min_c(X, y, loss, fit_intercept=True, intercept_scaling=1.0):
+    min_c = l1_min_c(
+        X,
+        y,
+        loss=loss,
+        fit_intercept=fit_intercept,
+        intercept_scaling=intercept_scaling,
+    )
 
     clf = {
-        'log': LogisticRegression(penalty='l1', solver='liblinear'),
-        'squared_hinge': LinearSVC(loss='squared_hinge',
-                                   penalty='l1', dual=False),
+        "log": LogisticRegression(penalty="l1", solver="liblinear"),
+        "squared_hinge": LinearSVC(loss="squared_hinge", penalty="l1", dual=False),
     }[loss]
 
     clf.fit_intercept = fit_intercept
@@ -62,8 +60,7 @@ def check_l1_min_c(X, y, loss, fit_intercept=True, intercept_scaling=None):
 
     clf.C = min_c * 1.01
     clf.fit(X, y)
-    assert ((np.asarray(clf.coef_) != 0).any() or
-            (np.asarray(clf.intercept_) != 0).any())
+    assert (np.asarray(clf.coef_) != 0).any() or (np.asarray(clf.intercept_) != 0).any()
 
 
 def test_ill_posed_min_c():
@@ -73,38 +70,42 @@ def test_ill_posed_min_c():
         l1_min_c(X, y)
 
 
-def test_unsupported_loss():
-    with pytest.raises(ValueError):
-        l1_min_c(dense_X, Y1, loss='l1')
+_MAX_UNSIGNED_INT = 4294967295
 
 
-_MAX_UNSIGNED_INT = 4294967295
+def test_newrand_default():
+    """Test that bounded_rand_int_wrap without seeding respects the range
 
+    Note this test should pass either if executed alone, or in conjunctions
+    with other tests that call set_seed explicit in any order: it checks
+    invariants on the RNG instead of specific values.
+    """
+    generated = [bounded_rand_int_wrap(100) for _ in range(10)]
+    assert all(0 <= x < 100 for x in generated)
+    assert not all(x == generated[0] for x in generated)
 
-@pytest.mark.parametrize('seed, val',
-                         [(None, 81),
-                          (0, 54),
-                          (_MAX_UNSIGNED_INT, 9)])
-def test_newrand_set_seed(seed, val):
+
+@pytest.mark.parametrize("seed, expected", [(0, 54), (_MAX_UNSIGNED_INT, 9)])
+def test_newrand_set_seed(seed, expected):
     """Test that `set_seed` produces deterministic results"""
-    if seed is not None:
-        set_seed_wrap(seed)
-    x = bounded_rand_int_wrap(100)
-    assert x == val, f'Expected {val} but got {x} instead'
+    set_seed_wrap(seed)
+    generated = bounded_rand_int_wrap(100)
+    assert generated == expected
 
 
-@pytest.mark.parametrize('seed',
-                         [-1, _MAX_UNSIGNED_INT + 1])
+@pytest.mark.parametrize("seed", [-1, _MAX_UNSIGNED_INT + 1])
 def test_newrand_set_seed_overflow(seed):
     """Test that `set_seed_wrap` is defined for unsigned 32bits ints"""
     with pytest.raises(OverflowError):
         set_seed_wrap(seed)
 
 
-@pytest.mark.parametrize('range_, n_pts',
-                         [(_MAX_UNSIGNED_INT, 10000), (100, 25)])
+@pytest.mark.parametrize("range_, n_pts", [(_MAX_UNSIGNED_INT, 10000), (100, 25)])
 def test_newrand_bounded_rand_int(range_, n_pts):
     """Test that `bounded_rand_int` follows a uniform distribution"""
+    # XXX: this test is very seed sensitive: either it is wrong (too strict?)
+    # or the wrapped RNG is not uniform enough, at least on some platforms.
+    set_seed_wrap(42)
     n_iter = 100
     ks_pvals = []
     uniform_dist = stats.uniform(loc=0, scale=range_)
@@ -126,7 +127,8 @@ def test_newrand_bounded_rand_int(range_, n_pts):
     assert res_pvals.pvalue > 0.05, (
         "Null hypothesis rejected: generated random numbers are not uniform."
         " Details: the (meta) p-value of the test of uniform distribution"
-        f" of p-values is {res_pvals.pvalue} which is not > 0.05")
+        f" of p-values is {res_pvals.pvalue} which is not > 0.05"
+    )
 
     # (2) (safety belt) check that 90% of p-values are above 0.05
     min_10pct_pval = np.percentile(ks_pvals, q=10)
@@ -135,11 +137,10 @@ def test_newrand_bounded_rand_int(range_, n_pts):
     assert min_10pct_pval > 0.05, (
         "Null hypothesis rejected: generated random numbers are not uniform. "
         f"Details: lower 10th quantile p-value of {min_10pct_pval} not > 0.05."
-        )
+    )
 
 
-@pytest.mark.parametrize('range_',
-                         [-1, _MAX_UNSIGNED_INT + 1])
+@pytest.mark.parametrize("range_", [-1, _MAX_UNSIGNED_INT + 1])
 def test_newrand_bounded_rand_int_limits(range_):
     """Test that `bounded_rand_int_wrap` is defined for unsigned 32bits ints"""
     with pytest.raises(OverflowError):
diff --git a/sklearn/svm/tests/test_sparse.py b/sklearn/svm/tests/test_sparse.py
index bb935e55e1912..59fede29f359c 100644
--- a/sklearn/svm/tests/test_sparse.py
+++ b/sklearn/svm/tests/test_sparse.py
@@ -1,124 +1,151 @@
-import pytest
-
 import numpy as np
-from numpy.testing import assert_array_almost_equal, assert_array_equal
+import pytest
 from scipy import sparse
 
-from sklearn import datasets, svm, linear_model, base
-from sklearn.datasets import make_classification, load_digits, make_blobs
-from sklearn.svm.tests import test_svm
+from sklearn import base, datasets, linear_model, svm
+from sklearn.datasets import load_digits, make_blobs, make_classification
 from sklearn.exceptions import ConvergenceWarning
+from sklearn.svm.tests import test_svm
+from sklearn.utils._testing import (
+    assert_allclose,
+    assert_array_almost_equal,
+    assert_array_equal,
+    ignore_warnings,
+    skip_if_32bit,
+)
 from sklearn.utils.extmath import safe_sparse_dot
-from sklearn.utils._testing import (assert_raise_message, ignore_warnings,
-                                    skip_if_32bit)
-
+from sklearn.utils.fixes import (
+    CSR_CONTAINERS,
+    DOK_CONTAINERS,
+    LIL_CONTAINERS,
+)
 
 # test sample 1
 X = np.array([[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]])
-X_sp = sparse.lil_matrix(X)
 Y = [1, 1, 1, 2, 2, 2]
 T = np.array([[-1, -1], [2, 2], [3, 2]])
 true_result = [1, 2, 2]
 
 # test sample 2
-X2 = np.array([[0, 0, 0], [1, 1, 1], [2, 0, 0, ],
-               [0, 0, 2], [3, 3, 3]])
-X2_sp = sparse.dok_matrix(X2)
+X2 = np.array(
+    [
+        [0, 0, 0],
+        [1, 1, 1],
+        [2, 0, 0],
+        [0, 0, 2],
+        [3, 3, 3],
+    ]
+)
 Y2 = [1, 2, 2, 2, 3]
 T2 = np.array([[-1, -1, -1], [1, 1, 1], [2, 2, 2]])
 true_result2 = [1, 2, 3]
 
-
 iris = datasets.load_iris()
-# permute
 rng = np.random.RandomState(0)
 perm = rng.permutation(iris.target.size)
 iris.data = iris.data[perm]
 iris.target = iris.target[perm]
-# sparsify
-iris.data = sparse.csr_matrix(iris.data)
 
+X_blobs, y_blobs = make_blobs(n_samples=100, centers=10, random_state=0)
+
+
+def check_svm_model_equal(dense_svm, X_train, y_train, X_test):
+    # Use the original svm model for dense fit and clone an exactly same
+    # svm model for sparse fit
+    sparse_svm = base.clone(dense_svm)
 
-def check_svm_model_equal(dense_svm, sparse_svm, X_train, y_train, X_test):
     dense_svm.fit(X_train.toarray(), y_train)
-    if sparse.isspmatrix(X_test):
+    if sparse.issparse(X_test):
         X_test_dense = X_test.toarray()
     else:
         X_test_dense = X_test
     sparse_svm.fit(X_train, y_train)
     assert sparse.issparse(sparse_svm.support_vectors_)
     assert sparse.issparse(sparse_svm.dual_coef_)
-    assert_array_almost_equal(dense_svm.support_vectors_,
-                              sparse_svm.support_vectors_.toarray())
-    assert_array_almost_equal(dense_svm.dual_coef_,
-                              sparse_svm.dual_coef_.toarray())
+    assert_allclose(dense_svm.support_vectors_, sparse_svm.support_vectors_.toarray())
+    assert_allclose(dense_svm.dual_coef_, sparse_svm.dual_coef_.toarray())
     if dense_svm.kernel == "linear":
         assert sparse.issparse(sparse_svm.coef_)
         assert_array_almost_equal(dense_svm.coef_, sparse_svm.coef_.toarray())
-    assert_array_almost_equal(dense_svm.support_, sparse_svm.support_)
-    assert_array_almost_equal(dense_svm.predict(X_test_dense),
-                              sparse_svm.predict(X_test))
-    assert_array_almost_equal(dense_svm.decision_function(X_test_dense),
-                              sparse_svm.decision_function(X_test))
-    assert_array_almost_equal(dense_svm.decision_function(X_test_dense),
-                              sparse_svm.decision_function(X_test_dense))
+    assert_allclose(dense_svm.support_, sparse_svm.support_)
+    assert_allclose(dense_svm.predict(X_test_dense), sparse_svm.predict(X_test))
+
+    assert_array_almost_equal(
+        dense_svm.decision_function(X_test_dense), sparse_svm.decision_function(X_test)
+    )
+    assert_array_almost_equal(
+        dense_svm.decision_function(X_test_dense),
+        sparse_svm.decision_function(X_test_dense),
+    )
     if isinstance(dense_svm, svm.OneClassSVM):
         msg = "cannot use sparse input in 'OneClassSVM' trained on dense data"
     else:
-        assert_array_almost_equal(dense_svm.predict_proba(X_test_dense),
-                                  sparse_svm.predict_proba(X_test), 4)
+        assert_array_almost_equal(
+            dense_svm.predict_proba(X_test_dense),
+            sparse_svm.predict_proba(X_test),
+            decimal=4,
+        )
         msg = "cannot use sparse input in 'SVC' trained on dense data"
-    if sparse.isspmatrix(X_test):
-        assert_raise_message(ValueError, msg, dense_svm.predict, X_test)
+    if sparse.issparse(X_test):
+        with pytest.raises(ValueError, match=msg):
+            dense_svm.predict(X_test)
 
 
 @skip_if_32bit
-def test_svc():
-    """Check that sparse SVC gives the same result as SVC"""
-    # many class dataset:
-    X_blobs, y_blobs = make_blobs(n_samples=100, centers=10, random_state=0)
-    X_blobs = sparse.csr_matrix(X_blobs)
-
-    datasets = [[X_sp, Y, T], [X2_sp, Y2, T2],
-                [X_blobs[:80], y_blobs[:80], X_blobs[80:]],
-                [iris.data, iris.target, iris.data]]
-    kernels = ["linear", "poly", "rbf", "sigmoid"]
-    for dataset in datasets:
-        for kernel in kernels:
-            clf = svm.SVC(gamma=1, kernel=kernel, probability=True,
-                          random_state=0, decision_function_shape='ovo')
-            sp_clf = svm.SVC(gamma=1, kernel=kernel, probability=True,
-                             random_state=0, decision_function_shape='ovo')
-            check_svm_model_equal(clf, sp_clf, *dataset)
-
-
-def test_unsorted_indices():
+@pytest.mark.parametrize(
+    "X_train, y_train, X_test",
+    [
+        [X, Y, T],
+        [X2, Y2, T2],
+        [X_blobs[:80], y_blobs[:80], X_blobs[80:]],
+        [iris.data, iris.target, iris.data],
+    ],
+)
+@pytest.mark.parametrize("kernel", ["linear", "poly", "rbf", "sigmoid"])
+@pytest.mark.parametrize("sparse_container", CSR_CONTAINERS + LIL_CONTAINERS)
+def test_svc(X_train, y_train, X_test, kernel, sparse_container):
+    """Check that sparse SVC gives the same result as SVC."""
+    X_train = sparse_container(X_train)
+
+    clf = svm.SVC(
+        gamma=1,
+        kernel=kernel,
+        probability=True,
+        random_state=0,
+        decision_function_shape="ovo",
+    )
+    check_svm_model_equal(clf, X_train, y_train, X_test)
+
+
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_unsorted_indices(csr_container):
     # test that the result with sorted and unsorted indices in csr is the same
     # we use a subset of digits as iris, blobs or make_classification didn't
     # show the problem
     X, y = load_digits(return_X_y=True)
-    X_test = sparse.csr_matrix(X[50:100])
+    X_test = csr_container(X[50:100])
     X, y = X[:50], y[:50]
 
-    X_sparse = sparse.csr_matrix(X)
-    coef_dense = svm.SVC(kernel='linear', probability=True,
-                         random_state=0).fit(X, y).coef_
-    sparse_svc = svm.SVC(kernel='linear', probability=True,
-                         random_state=0).fit(X_sparse, y)
+    X_sparse = csr_container(X)
+    coef_dense = (
+        svm.SVC(kernel="linear", probability=True, random_state=0).fit(X, y).coef_
+    )
+    sparse_svc = svm.SVC(kernel="linear", probability=True, random_state=0).fit(
+        X_sparse, y
+    )
     coef_sorted = sparse_svc.coef_
     # make sure dense and sparse SVM give the same result
-    assert_array_almost_equal(coef_dense, coef_sorted.toarray())
+    assert_allclose(coef_dense, coef_sorted.toarray())
 
     # reverse each row's indices
     def scramble_indices(X):
         new_data = []
         new_indices = []
         for i in range(1, len(X.indptr)):
-            row_slice = slice(*X.indptr[i - 1: i + 1])
+            row_slice = slice(*X.indptr[i - 1 : i + 1])
             new_data.extend(X.data[row_slice][::-1])
             new_indices.extend(X.indices[row_slice][::-1])
-        return sparse.csr_matrix((new_data, new_indices, X.indptr),
-                                 shape=X.shape)
+        return csr_container((new_data, new_indices, X.indptr), shape=X.shape)
 
     X_sparse_unsorted = scramble_indices(X_sparse)
     X_test_unsorted = scramble_indices(X_test)
@@ -126,87 +153,95 @@ def scramble_indices(X):
     assert not X_sparse_unsorted.has_sorted_indices
     assert not X_test_unsorted.has_sorted_indices
 
-    unsorted_svc = svm.SVC(kernel='linear', probability=True,
-                           random_state=0).fit(X_sparse_unsorted, y)
+    unsorted_svc = svm.SVC(kernel="linear", probability=True, random_state=0).fit(
+        X_sparse_unsorted, y
+    )
     coef_unsorted = unsorted_svc.coef_
     # make sure unsorted indices give same result
-    assert_array_almost_equal(coef_unsorted.toarray(), coef_sorted.toarray())
-    assert_array_almost_equal(sparse_svc.predict_proba(X_test_unsorted),
-                              sparse_svc.predict_proba(X_test))
+    assert_allclose(coef_unsorted.toarray(), coef_sorted.toarray())
+    assert_allclose(
+        sparse_svc.predict_proba(X_test_unsorted), sparse_svc.predict_proba(X_test)
+    )
 
 
-def test_svc_with_custom_kernel():
+@pytest.mark.parametrize("lil_container", LIL_CONTAINERS)
+def test_svc_with_custom_kernel(lil_container):
     def kfunc(x, y):
         return safe_sparse_dot(x, y.T)
-    clf_lin = svm.SVC(kernel='linear').fit(X_sp, Y)
+
+    X_sp = lil_container(X)
+    clf_lin = svm.SVC(kernel="linear").fit(X_sp, Y)
     clf_mylin = svm.SVC(kernel=kfunc).fit(X_sp, Y)
     assert_array_equal(clf_lin.predict(X_sp), clf_mylin.predict(X_sp))
 
 
-def test_svc_iris():
+@skip_if_32bit
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+@pytest.mark.parametrize("kernel", ["linear", "poly", "rbf"])
+def test_svc_iris(csr_container, kernel):
     # Test the sparse SVC with the iris dataset
-    for k in ('linear', 'poly', 'rbf'):
-        sp_clf = svm.SVC(kernel=k).fit(iris.data, iris.target)
-        clf = svm.SVC(kernel=k).fit(iris.data.toarray(),
-                                                   iris.target)
-
-        assert_array_almost_equal(clf.support_vectors_,
-                                  sp_clf.support_vectors_.toarray())
-        assert_array_almost_equal(clf.dual_coef_, sp_clf.dual_coef_.toarray())
-        assert_array_almost_equal(
-            clf.predict(iris.data.toarray()), sp_clf.predict(iris.data))
-        if k == 'linear':
-            assert_array_almost_equal(clf.coef_, sp_clf.coef_.toarray())
+    iris_data_sp = csr_container(iris.data)
 
+    sp_clf = svm.SVC(kernel=kernel).fit(iris_data_sp, iris.target)
+    clf = svm.SVC(kernel=kernel).fit(iris.data, iris.target)
 
-def test_sparse_decision_function():
+    assert_allclose(clf.support_vectors_, sp_clf.support_vectors_.toarray())
+    assert_allclose(clf.dual_coef_, sp_clf.dual_coef_.toarray())
+    assert_allclose(clf.predict(iris.data), sp_clf.predict(iris_data_sp))
+    if kernel == "linear":
+        assert_allclose(clf.coef_, sp_clf.coef_.toarray())
+
+
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sparse_decision_function(csr_container):
     # Test decision_function
 
     # Sanity check, test that decision_function implemented in python
     # returns the same as the one in libsvm
 
     # multi class:
-    svc = svm.SVC(kernel='linear', C=0.1, decision_function_shape='ovo')
-    clf = svc.fit(iris.data, iris.target)
+    iris_data_sp = csr_container(iris.data)
+    svc = svm.SVC(kernel="linear", C=0.1, decision_function_shape="ovo")
+    clf = svc.fit(iris_data_sp, iris.target)
 
-    dec = safe_sparse_dot(iris.data, clf.coef_.T) + clf.intercept_
+    dec = safe_sparse_dot(iris_data_sp, clf.coef_.T) + clf.intercept_
 
-    assert_array_almost_equal(dec, clf.decision_function(iris.data))
+    assert_allclose(dec, clf.decision_function(iris_data_sp))
 
     # binary:
     clf.fit(X, Y)
     dec = np.dot(X, clf.coef_.T) + clf.intercept_
     prediction = clf.predict(X)
-    assert_array_almost_equal(dec.ravel(), clf.decision_function(X))
-    assert_array_almost_equal(
-        prediction,
-        clf.classes_[(clf.decision_function(X) > 0).astype(int).ravel()])
-    expected = np.array([-1., -0.66, -1., 0.66, 1., 1.])
-    assert_array_almost_equal(clf.decision_function(X), expected, 2)
+    assert_allclose(dec.ravel(), clf.decision_function(X))
+    assert_allclose(
+        prediction, clf.classes_[(clf.decision_function(X) > 0).astype(int).ravel()]
+    )
+    expected = np.array([-1.0, -0.66, -1.0, 0.66, 1.0, 1.0])
+    assert_array_almost_equal(clf.decision_function(X), expected, decimal=2)
 
 
-def test_error():
+@pytest.mark.parametrize("lil_container", LIL_CONTAINERS)
+def test_error(lil_container):
     # Test that it gives proper exception on deficient input
-    # impossible value of C
-    with pytest.raises(ValueError):
-        svm.SVC(C=-1).fit(X, Y)
-
-    # impossible value of nu
-    clf = svm.NuSVC(nu=0.0)
-    with pytest.raises(ValueError):
-        clf.fit(X_sp, Y)
+    clf = svm.SVC()
+    X_sp = lil_container(X)
 
     Y2 = Y[:-1]  # wrong dimensions for labels
     with pytest.raises(ValueError):
         clf.fit(X_sp, Y2)
 
-    clf = svm.SVC()
     clf.fit(X_sp, Y)
     assert_array_equal(clf.predict(T), true_result)
 
 
-def test_linearsvc():
+@pytest.mark.parametrize(
+    "lil_container, dok_container", zip(LIL_CONTAINERS, DOK_CONTAINERS)
+)
+def test_linearsvc(lil_container, dok_container):
     # Similar to test_SVC
+    X_sp = lil_container(X)
+    X2_sp = dok_container(X2)
+
     clf = svm.LinearSVC(random_state=0).fit(X, Y)
     sp_clf = svm.LinearSVC(random_state=0).fit(X_sp, Y)
 
@@ -215,7 +250,7 @@ def test_linearsvc():
     assert_array_almost_equal(clf.coef_, sp_clf.coef_, decimal=4)
     assert_array_almost_equal(clf.intercept_, sp_clf.intercept_, decimal=4)
 
-    assert_array_almost_equal(clf.predict(X), sp_clf.predict(X_sp))
+    assert_allclose(clf.predict(X), sp_clf.predict(X_sp))
 
     clf.fit(X2, Y2)
     sp_clf.fit(X2_sp, Y2)
@@ -224,55 +259,63 @@ def test_linearsvc():
     assert_array_almost_equal(clf.intercept_, sp_clf.intercept_, decimal=4)
 
 
-def test_linearsvc_iris():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_linearsvc_iris(csr_container):
     # Test the sparse LinearSVC with the iris dataset
+    iris_data_sp = csr_container(iris.data)
 
-    sp_clf = svm.LinearSVC(random_state=0).fit(iris.data, iris.target)
-    clf = svm.LinearSVC(random_state=0).fit(iris.data.toarray(), iris.target)
+    sp_clf = svm.LinearSVC(random_state=0).fit(iris_data_sp, iris.target)
+    clf = svm.LinearSVC(random_state=0).fit(iris.data, iris.target)
 
     assert clf.fit_intercept == sp_clf.fit_intercept
 
     assert_array_almost_equal(clf.coef_, sp_clf.coef_, decimal=1)
     assert_array_almost_equal(clf.intercept_, sp_clf.intercept_, decimal=1)
-    assert_array_almost_equal(
-        clf.predict(iris.data.toarray()), sp_clf.predict(iris.data))
+    assert_allclose(clf.predict(iris.data), sp_clf.predict(iris_data_sp))
 
     # check decision_function
-    pred = np.argmax(sp_clf.decision_function(iris.data), 1)
-    assert_array_almost_equal(pred, clf.predict(iris.data.toarray()))
+    pred = np.argmax(sp_clf.decision_function(iris_data_sp), axis=1)
+    assert_allclose(pred, clf.predict(iris.data))
 
     # sparsify the coefficients on both models and check that they still
     # produce the same results
     clf.sparsify()
-    assert_array_equal(pred, clf.predict(iris.data))
+    assert_array_equal(pred, clf.predict(iris_data_sp))
     sp_clf.sparsify()
-    assert_array_equal(pred, sp_clf.predict(iris.data))
+    assert_array_equal(pred, sp_clf.predict(iris_data_sp))
 
 
-def test_weight():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_weight(csr_container):
     # Test class weights
-    X_, y_ = make_classification(n_samples=200, n_features=100,
-                                 weights=[0.833, 0.167], random_state=0)
+    X_, y_ = make_classification(
+        n_samples=200, n_features=100, weights=[0.833, 0.167], random_state=0
+    )
 
-    X_ = sparse.csr_matrix(X_)
-    for clf in (linear_model.LogisticRegression(),
-                svm.LinearSVC(random_state=0),
-                svm.SVC()):
+    X_ = csr_container(X_)
+    for clf in (
+        linear_model.LogisticRegression(),
+        svm.LinearSVC(random_state=0),
+        svm.SVC(),
+    ):
         clf.set_params(class_weight={0: 5})
         clf.fit(X_[:180], y_[:180])
         y_pred = clf.predict(X_[180:])
         assert np.sum(y_pred == y_[180:]) >= 11
 
 
-def test_sample_weights():
+@pytest.mark.parametrize("lil_container", LIL_CONTAINERS)
+def test_sample_weights(lil_container):
     # Test weights on individual samples
+    X_sp = lil_container(X)
+
     clf = svm.SVC()
     clf.fit(X_sp, Y)
-    assert_array_equal(clf.predict([X[2]]), [1.])
+    assert_array_equal(clf.predict([X[2]]), [1.0])
 
-    sample_weight = [.1] * 3 + [10] * 3
+    sample_weight = [0.1] * 3 + [10] * 3
     clf.fit(X_sp, Y, sample_weight=sample_weight)
-    assert_array_equal(clf.predict([X[2]]), [2.])
+    assert_array_equal(clf.predict([X[2]]), [2.0])
 
 
 def test_sparse_liblinear_intercept_handling():
@@ -280,79 +323,164 @@ def test_sparse_liblinear_intercept_handling():
     test_svm.test_dense_liblinear_intercept_handling(svm.LinearSVC)
 
 
-@pytest.mark.parametrize("datasets_index", range(4))
+@pytest.mark.parametrize(
+    "X_train, y_train, X_test",
+    [
+        [X, None, T],
+        [X2, None, T2],
+        [X_blobs[:80], None, X_blobs[80:]],
+        [iris.data, None, iris.data],
+    ],
+)
 @pytest.mark.parametrize("kernel", ["linear", "poly", "rbf", "sigmoid"])
+@pytest.mark.parametrize("sparse_container", CSR_CONTAINERS + LIL_CONTAINERS)
 @skip_if_32bit
-def test_sparse_oneclasssvm(datasets_index, kernel):
+def test_sparse_oneclasssvm(X_train, y_train, X_test, kernel, sparse_container):
     # Check that sparse OneClassSVM gives the same result as dense OneClassSVM
-    # many class dataset:
-    X_blobs, _ = make_blobs(n_samples=100, centers=10, random_state=0)
-    X_blobs = sparse.csr_matrix(X_blobs)
-    datasets = [[X_sp, None, T], [X2_sp, None, T2],
-                [X_blobs[:80], None, X_blobs[80:]],
-                [iris.data, None, iris.data]]
-    dataset = datasets[datasets_index]
+    X_train = sparse_container(X_train)
+
     clf = svm.OneClassSVM(gamma=1, kernel=kernel)
-    sp_clf = svm.OneClassSVM(gamma=1, kernel=kernel)
-    check_svm_model_equal(clf, sp_clf, *dataset)
+    check_svm_model_equal(clf, X_train, y_train, X_test)
 
 
-def test_sparse_realdata():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sparse_realdata(csr_container):
     # Test on a subset from the 20newsgroups dataset.
     # This catches some bugs if input is not correctly converted into
     # sparse format or weights are not correctly initialized.
-
     data = np.array([0.03771744, 0.1003567, 0.01174647, 0.027069])
-    indices = np.array([6, 5, 35, 31])
-    indptr = np.array(
-        [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2,
-         2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-         2, 2, 2, 2, 2, 2, 2, 2, 2, 4, 4, 4])
-    X = sparse.csr_matrix((data, indices, indptr))
+
+    # SVC does not support large sparse, so we specify int32 indices
+    # In this case, `csr_matrix` automatically uses int32 regardless of the dtypes of
+    # `indices` and `indptr` but `csr_array` may or may not use the same dtype as
+    # `indices` and `indptr`, which would be int64 if not specified
+    indices = np.array([6, 5, 35, 31], dtype=np.int32)
+    indptr = np.array([0] * 8 + [1] * 32 + [2] * 38 + [4] * 3, dtype=np.int32)
+
+    X = csr_container((data, indices, indptr))
     y = np.array(
-        [1., 0., 2., 2., 1., 1., 1., 2., 2., 0., 1., 2., 2.,
-         0., 2., 0., 3., 0., 3., 0., 1., 1., 3., 2., 3., 2.,
-         0., 3., 1., 0., 2., 1., 2., 0., 1., 0., 2., 3., 1.,
-         3., 0., 1., 0., 0., 2., 0., 1., 2., 2., 2., 3., 2.,
-         0., 3., 2., 1., 2., 3., 2., 2., 0., 1., 0., 1., 2.,
-         3., 0., 0., 2., 2., 1., 3., 1., 1., 0., 1., 2., 1.,
-         1., 3.])
+        [
+            1.0,
+            0.0,
+            2.0,
+            2.0,
+            1.0,
+            1.0,
+            1.0,
+            2.0,
+            2.0,
+            0.0,
+            1.0,
+            2.0,
+            2.0,
+            0.0,
+            2.0,
+            0.0,
+            3.0,
+            0.0,
+            3.0,
+            0.0,
+            1.0,
+            1.0,
+            3.0,
+            2.0,
+            3.0,
+            2.0,
+            0.0,
+            3.0,
+            1.0,
+            0.0,
+            2.0,
+            1.0,
+            2.0,
+            0.0,
+            1.0,
+            0.0,
+            2.0,
+            3.0,
+            1.0,
+            3.0,
+            0.0,
+            1.0,
+            0.0,
+            0.0,
+            2.0,
+            0.0,
+            1.0,
+            2.0,
+            2.0,
+            2.0,
+            3.0,
+            2.0,
+            0.0,
+            3.0,
+            2.0,
+            1.0,
+            2.0,
+            3.0,
+            2.0,
+            2.0,
+            0.0,
+            1.0,
+            0.0,
+            1.0,
+            2.0,
+            3.0,
+            0.0,
+            0.0,
+            2.0,
+            2.0,
+            1.0,
+            3.0,
+            1.0,
+            1.0,
+            0.0,
+            1.0,
+            2.0,
+            1.0,
+            1.0,
+            3.0,
+        ]
+    )
 
-    clf = svm.SVC(kernel='linear').fit(X.toarray(), y)
-    sp_clf = svm.SVC(kernel='linear').fit(sparse.coo_matrix(X), y)
+    clf = svm.SVC(kernel="linear").fit(X.toarray(), y)
+    sp_clf = svm.SVC(kernel="linear").fit(X.tocoo(), y)
 
     assert_array_equal(clf.support_vectors_, sp_clf.support_vectors_.toarray())
     assert_array_equal(clf.dual_coef_, sp_clf.dual_coef_.toarray())
 
 
-def test_sparse_svc_clone_with_callable_kernel():
+@pytest.mark.parametrize("lil_container", LIL_CONTAINERS)
+def test_sparse_svc_clone_with_callable_kernel(lil_container):
     # Test that the "dense_fit" is called even though we use sparse input
     # meaning that everything works fine.
-    a = svm.SVC(C=1, kernel=lambda x, y: x * y.T,
-                probability=True, random_state=0)
+    a = svm.SVC(C=1, kernel=lambda x, y: x @ y.T, probability=True, random_state=0)
     b = base.clone(a)
 
+    X_sp = lil_container(X)
     b.fit(X_sp, Y)
     pred = b.predict(X_sp)
     b.predict_proba(X_sp)
 
-    dense_svm = svm.SVC(C=1, kernel=lambda x, y: np.dot(x, y.T),
-                        probability=True, random_state=0)
+    dense_svm = svm.SVC(
+        C=1, kernel=lambda x, y: np.dot(x, y.T), probability=True, random_state=0
+    )
     pred_dense = dense_svm.fit(X, Y).predict(X)
     assert_array_equal(pred_dense, pred)
     # b.decision_function(X_sp)  # XXX : should be supported
 
 
-def test_timeout():
-    sp = svm.SVC(C=1, kernel=lambda x, y: x * y.T,
-                 probability=True, random_state=0, max_iter=1)
+@pytest.mark.parametrize("lil_container", LIL_CONTAINERS)
+def test_timeout(lil_container):
+    sp = svm.SVC(
+        C=1, kernel=lambda x, y: x @ y.T, probability=True, random_state=0, max_iter=1
+    )
     warning_msg = (
-        r'Solver terminated early \(max_iter=1\).  Consider pre-processing '
-        r'your data with StandardScaler or MinMaxScaler.'
+        r"Solver terminated early \(max_iter=1\).  Consider pre-processing "
+        r"your data with StandardScaler or MinMaxScaler."
     )
     with pytest.warns(ConvergenceWarning, match=warning_msg):
-        sp.fit(X_sp, Y)
+        sp.fit(lil_container(X), Y)
 
 
 def test_consistent_proba():
@@ -362,4 +490,4 @@ def test_consistent_proba():
     a = svm.SVC(probability=True, max_iter=1, random_state=0)
     with ignore_warnings(category=ConvergenceWarning):
         proba_2 = a.fit(X, Y).predict_proba(X)
-    assert_array_almost_equal(proba_1, proba_2)
+    assert_allclose(proba_1, proba_2)
diff --git a/sklearn/svm/tests/test_svm.py b/sklearn/svm/tests/test_svm.py
index f1e2cea4be2dc..62396451e736d 100644
--- a/sklearn/svm/tests/test_svm.py
+++ b/sklearn/svm/tests/test_svm.py
@@ -3,31 +3,40 @@
 
 TODO: remove hard coded numerical results when possible
 """
+
 import numpy as np
-import itertools
 import pytest
+from numpy.testing import (
+    assert_allclose,
+    assert_almost_equal,
+    assert_array_almost_equal,
+    assert_array_equal,
+)
 
-from numpy.testing import assert_array_equal, assert_array_almost_equal
-from numpy.testing import assert_almost_equal
-from numpy.testing import assert_allclose
-from scipy import sparse
-from sklearn import svm, linear_model, datasets, metrics, base
-from sklearn.svm import LinearSVC
-from sklearn.svm import LinearSVR
-from sklearn.model_selection import train_test_split
-from sklearn.datasets import make_classification, make_blobs
+from sklearn import base, datasets, linear_model, metrics, svm
+from sklearn.datasets import make_blobs, make_classification, make_regression
+from sklearn.exceptions import (
+    ConvergenceWarning,
+    NotFittedError,
+)
 from sklearn.metrics import f1_score
 from sklearn.metrics.pairwise import rbf_kernel
-from sklearn.utils import check_random_state
-from sklearn.utils._testing import assert_raise_message
-from sklearn.utils._testing import ignore_warnings
-from sklearn.utils.validation import _num_samples
-from sklearn.utils import shuffle
-from sklearn.exceptions import ConvergenceWarning
-from sklearn.exceptions import NotFittedError, UndefinedMetricWarning
+from sklearn.model_selection import train_test_split
 from sklearn.multiclass import OneVsRestClassifier
+
 # mypy error: Module 'sklearn.svm' has no attribute '_libsvm'
-from sklearn.svm import _libsvm  # type: ignore
+from sklearn.svm import (  # type: ignore[attr-defined]
+    SVR,
+    LinearSVC,
+    LinearSVR,
+    NuSVR,
+    OneClassSVM,
+    _libsvm,
+)
+from sklearn.svm._classes import _validate_dual_parameter
+from sklearn.utils import check_random_state, shuffle
+from sklearn.utils.fixes import _IS_32BIT, CSR_CONTAINERS, LIL_CONTAINERS
+from sklearn.utils.validation import _num_samples
 
 # toy sample
 X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]]
@@ -45,11 +54,11 @@
 
 def test_libsvm_parameters():
     # Test parameters on classes that make use of libsvm.
-    clf = svm.SVC(kernel='linear').fit(X, Y)
-    assert_array_equal(clf.dual_coef_, [[-0.25, .25]])
+    clf = svm.SVC(kernel="linear").fit(X, Y)
+    assert_array_equal(clf.dual_coef_, [[-0.25, 0.25]])
     assert_array_equal(clf.support_, [1, 3])
     assert_array_equal(clf.support_vectors_, (X[1], X[3]))
-    assert_array_equal(clf.intercept_, [0.])
+    assert_array_equal(clf.intercept_, [0.0])
     assert_array_equal(clf.predict(X), Y)
 
 
@@ -57,43 +66,86 @@ def test_libsvm_iris():
     # Check consistency on dataset iris.
 
     # shuffle the dataset so that labels are not ordered
-    for k in ('linear', 'rbf'):
+    for k in ("linear", "rbf"):
         clf = svm.SVC(kernel=k).fit(iris.data, iris.target)
         assert np.mean(clf.predict(iris.data) == iris.target) > 0.9
-        assert hasattr(clf, "coef_") == (k == 'linear')
+        assert hasattr(clf, "coef_") == (k == "linear")
 
     assert_array_equal(clf.classes_, np.sort(clf.classes_))
 
     # check also the low-level API
-    model = _libsvm.fit(iris.data, iris.target.astype(np.float64))
-    pred = _libsvm.predict(iris.data, *model)
-    assert np.mean(pred == iris.target) > .95
-
-    model = _libsvm.fit(iris.data, iris.target.astype(np.float64),
-                        kernel='linear')
-    pred = _libsvm.predict(iris.data, *model, kernel='linear')
-    assert np.mean(pred == iris.target) > .95
-
-    pred = _libsvm.cross_validation(iris.data,
-                                    iris.target.astype(np.float64), 5,
-                                    kernel='linear',
-                                    random_seed=0)
-    assert np.mean(pred == iris.target) > .95
+    # We unpack the values to create a dictionary with some of the return values
+    # from Libsvm's fit.
+    (
+        libsvm_support,
+        libsvm_support_vectors,
+        libsvm_n_class_SV,
+        libsvm_sv_coef,
+        libsvm_intercept,
+        libsvm_probA,
+        libsvm_probB,
+        # libsvm_fit_status and libsvm_n_iter won't be used below.
+        libsvm_fit_status,
+        libsvm_n_iter,
+    ) = _libsvm.fit(iris.data, iris.target.astype(np.float64))
+
+    model_params = {
+        "support": libsvm_support,
+        "SV": libsvm_support_vectors,
+        "nSV": libsvm_n_class_SV,
+        "sv_coef": libsvm_sv_coef,
+        "intercept": libsvm_intercept,
+        "probA": libsvm_probA,
+        "probB": libsvm_probB,
+    }
+    pred = _libsvm.predict(iris.data, **model_params)
+    assert np.mean(pred == iris.target) > 0.95
+
+    # We unpack the values to create a dictionary with some of the return values
+    # from Libsvm's fit.
+    (
+        libsvm_support,
+        libsvm_support_vectors,
+        libsvm_n_class_SV,
+        libsvm_sv_coef,
+        libsvm_intercept,
+        libsvm_probA,
+        libsvm_probB,
+        # libsvm_fit_status and libsvm_n_iter won't be used below.
+        libsvm_fit_status,
+        libsvm_n_iter,
+    ) = _libsvm.fit(iris.data, iris.target.astype(np.float64), kernel="linear")
+
+    model_params = {
+        "support": libsvm_support,
+        "SV": libsvm_support_vectors,
+        "nSV": libsvm_n_class_SV,
+        "sv_coef": libsvm_sv_coef,
+        "intercept": libsvm_intercept,
+        "probA": libsvm_probA,
+        "probB": libsvm_probB,
+    }
+    pred = _libsvm.predict(iris.data, **model_params, kernel="linear")
+    assert np.mean(pred == iris.target) > 0.95
+
+    pred = _libsvm.cross_validation(
+        iris.data, iris.target.astype(np.float64), 5, kernel="linear", random_seed=0
+    )
+    assert np.mean(pred == iris.target) > 0.95
 
     # If random_seed >= 0, the libsvm rng is seeded (by calling `srand`), hence
     # we should get deterministic results (assuming that there is no other
     # thread calling this wrapper calling `srand` concurrently).
-    pred2 = _libsvm.cross_validation(iris.data,
-                                     iris.target.astype(np.float64), 5,
-                                     kernel='linear',
-                                     random_seed=0)
+    pred2 = _libsvm.cross_validation(
+        iris.data, iris.target.astype(np.float64), 5, kernel="linear", random_seed=0
+    )
     assert_array_equal(pred, pred2)
 
 
 def test_precomputed():
     # SVC with a precomputed kernel.
     # We test it with a toy dataset and with iris.
-    clf = svm.SVC(kernel='precomputed')
+    clf = svm.SVC(kernel="precomputed")
     # Gram matrix for train data (square matrix)
     # (we use just a linear kernel)
     K = np.dot(X, np.array(X).T)
@@ -104,7 +156,7 @@ def test_precomputed():
     with pytest.raises(ValueError):
         clf.predict(KT.T)
 
-    assert_array_equal(clf.dual_coef_, [[-0.25, .25]])
+    assert_array_equal(clf.dual_coef_, [[-0.25, 0.25]])
     assert_array_equal(clf.support_, [1, 3])
     assert_array_equal(clf.intercept_, [0])
     assert_array_almost_equal(clf.support_, [1, 3])
@@ -123,20 +175,22 @@ def test_precomputed():
     # same as before, but using a callable function instead of the kernel
     # matrix. kernel is just a linear kernel
 
-    kfunc = lambda x, y: np.dot(x, y.T)
+    def kfunc(x, y):
+        return np.dot(x, y.T)
+
     clf = svm.SVC(kernel=kfunc)
     clf.fit(np.array(X), Y)
     pred = clf.predict(T)
 
-    assert_array_equal(clf.dual_coef_, [[-0.25, .25]])
+    assert_array_equal(clf.dual_coef_, [[-0.25, 0.25]])
     assert_array_equal(clf.intercept_, [0])
     assert_array_almost_equal(clf.support_, [1, 3])
     assert_array_equal(pred, true_result)
 
     # test a precomputed kernel with the iris dataset
     # and check parameters against a linear SVC
-    clf = svm.SVC(kernel='precomputed')
-    clf2 = svm.SVC(kernel='linear')
+    clf = svm.SVC(kernel="precomputed")
+    clf2 = svm.SVC(kernel="linear")
     K = np.dot(iris.data, iris.data.T)
     clf.fit(K, iris.target)
     clf2.fit(iris.data, iris.target)
@@ -144,7 +198,7 @@ def test_precomputed():
     assert_array_almost_equal(clf.support_, clf2.support_)
     assert_array_almost_equal(clf.dual_coef_, clf2.dual_coef_)
     assert_array_almost_equal(clf.intercept_, clf2.intercept_)
-    assert_almost_equal(np.mean(pred == iris.target), .99, decimal=2)
+    assert_almost_equal(np.mean(pred == iris.target), 0.99, decimal=2)
 
     # Gram matrix for test data but compute KT[i,j]
     # for support vectors j only.
@@ -154,22 +208,24 @@ def test_precomputed():
             K[i, j] = np.dot(iris.data[i], iris.data[j])
 
     pred = clf.predict(K)
-    assert_almost_equal(np.mean(pred == iris.target), .99, decimal=2)
+    assert_almost_equal(np.mean(pred == iris.target), 0.99, decimal=2)
 
     clf = svm.SVC(kernel=kfunc)
     clf.fit(iris.data, iris.target)
-    assert_almost_equal(np.mean(pred == iris.target), .99, decimal=2)
+    assert_almost_equal(np.mean(pred == iris.target), 0.99, decimal=2)
 
 
 def test_svr():
     # Test Support Vector Regression
 
     diabetes = datasets.load_diabetes()
-    for clf in (svm.NuSVR(kernel='linear', nu=.4, C=1.0),
-                svm.NuSVR(kernel='linear', nu=.4, C=10.),
-                svm.SVR(kernel='linear', C=10.),
-                svm.LinearSVR(C=10.),
-                svm.LinearSVR(C=10.)):
+    for clf in (
+        svm.NuSVR(kernel="linear", nu=0.4, C=1.0),
+        svm.NuSVR(kernel="linear", nu=0.4, C=10.0),
+        svm.SVR(kernel="linear", C=10.0),
+        svm.LinearSVR(C=10.0),
+        svm.LinearSVR(C=10.0),
+    ):
         clf.fit(diabetes.data, diabetes.target)
         assert clf.score(diabetes.data, diabetes.target) > 0.02
 
@@ -186,11 +242,10 @@ def test_linearsvr():
     lsvr = svm.LinearSVR(C=1e3).fit(diabetes.data, diabetes.target)
     score1 = lsvr.score(diabetes.data, diabetes.target)
 
-    svr = svm.SVR(kernel='linear', C=1e3).fit(diabetes.data, diabetes.target)
+    svr = svm.SVR(kernel="linear", C=1e3).fit(diabetes.data, diabetes.target)
     score2 = svr.score(diabetes.data, diabetes.target)
 
-    assert_allclose(np.linalg.norm(lsvr.coef_),
-                    np.linalg.norm(svr.coef_), 1, 0.0001)
+    assert_allclose(np.linalg.norm(lsvr.coef_), np.linalg.norm(svr.coef_), 1, 0.0001)
     assert_almost_equal(score1, score2, 2)
 
 
@@ -202,30 +257,34 @@ def test_linearsvr_fit_sampleweight():
     n_samples = len(diabetes.target)
     unit_weight = np.ones(n_samples)
     lsvr = svm.LinearSVR(C=1e3, tol=1e-12, max_iter=10000).fit(
-        diabetes.data, diabetes.target, sample_weight=unit_weight)
+        diabetes.data, diabetes.target, sample_weight=unit_weight
+    )
     score1 = lsvr.score(diabetes.data, diabetes.target)
 
     lsvr_no_weight = svm.LinearSVR(C=1e3, tol=1e-12, max_iter=10000).fit(
-        diabetes.data, diabetes.target)
+        diabetes.data, diabetes.target
+    )
     score2 = lsvr_no_weight.score(diabetes.data, diabetes.target)
 
-    assert_allclose(np.linalg.norm(lsvr.coef_),
-                    np.linalg.norm(lsvr_no_weight.coef_), 1, 0.0001)
+    assert_allclose(
+        np.linalg.norm(lsvr.coef_), np.linalg.norm(lsvr_no_weight.coef_), 1, 0.0001
+    )
     assert_almost_equal(score1, score2, 2)
 
-    # check that fit(X)  = fit([X1, X2, X3],sample_weight = [n1, n2, n3]) where
+    # check that fit(X)  = fit([X1, X2, X3], sample_weight = [n1, n2, n3]) where
     # X = X1 repeated n1 times, X2 repeated n2 times and so forth
     random_state = check_random_state(0)
     random_weight = random_state.randint(0, 10, n_samples)
     lsvr_unflat = svm.LinearSVR(C=1e3, tol=1e-12, max_iter=10000).fit(
-        diabetes.data, diabetes.target, sample_weight=random_weight)
-    score3 = lsvr_unflat.score(diabetes.data, diabetes.target,
-                               sample_weight=random_weight)
+        diabetes.data, diabetes.target, sample_weight=random_weight
+    )
+    score3 = lsvr_unflat.score(
+        diabetes.data, diabetes.target, sample_weight=random_weight
+    )
 
     X_flat = np.repeat(diabetes.data, random_weight, axis=0)
     y_flat = np.repeat(diabetes.target, random_weight, axis=0)
-    lsvr_flat = svm.LinearSVR(C=1e3, tol=1e-12, max_iter=10000).fit(
-        X_flat, y_flat)
+    lsvr_flat = svm.LinearSVR(C=1e3, tol=1e-12, max_iter=10000).fit(X_flat, y_flat)
     score4 = lsvr_flat.score(X_flat, y_flat)
 
     assert_almost_equal(score3, score4, 2)
@@ -249,11 +308,9 @@ def test_oneclass():
     pred = clf.predict(T)
 
     assert_array_equal(pred, [1, -1, -1])
-    assert pred.dtype == np.dtype('intp')
+    assert pred.dtype == np.dtype("intp")
     assert_array_almost_equal(clf.intercept_, [-1.218], decimal=3)
-    assert_array_almost_equal(clf.dual_coef_,
-                              [[0.750, 0.750, 0.750, 0.750]],
-                              decimal=3)
+    assert_array_almost_equal(clf.dual_coef_, [[0.750, 0.750, 0.750, 0.750]], decimal=3)
     with pytest.raises(AttributeError):
         (lambda: clf.coef_)()
 
@@ -279,9 +336,9 @@ def test_oneclass_decision_function():
 
     # predict things
     y_pred_test = clf.predict(X_test)
-    assert np.mean(y_pred_test == 1) > .9
+    assert np.mean(y_pred_test == 1) > 0.9
     y_pred_outliers = clf.predict(X_outliers)
-    assert np.mean(y_pred_outliers == -1) > .9
+    assert np.mean(y_pred_outliers == -1) > 0.9
     dec_func_test = clf.decision_function(X_test)
     assert_array_equal((dec_func_test > 0).ravel(), y_pred_test == 1)
     dec_func_outliers = clf.decision_function(X_outliers)
@@ -291,8 +348,10 @@ def test_oneclass_decision_function():
 def test_oneclass_score_samples():
     X_train = [[1, 1], [1, 2], [2, 1]]
     clf = svm.OneClassSVM(gamma=1).fit(X_train)
-    assert_array_equal(clf.score_samples([[2., 2.]]),
-                       clf.decision_function([[2., 2.]]) + clf.offset_)
+    assert_array_equal(
+        clf.score_samples([[2.0, 2.0]]),
+        clf.decision_function([[2.0, 2.0]]) + clf.offset_,
+    )
 
 
 def test_tweak_params():
@@ -302,30 +361,31 @@ def test_tweak_params():
     # of C/Python copying in the libsvm bindings.
     # The success of this test ensures that the mapping between libsvm and
     # the python classifier is complete.
-    clf = svm.SVC(kernel='linear', C=1.0)
+    clf = svm.SVC(kernel="linear", C=1.0)
     clf.fit(X, Y)
-    assert_array_equal(clf.dual_coef_, [[-.25, .25]])
-    assert_array_equal(clf.predict([[-.1, -.1]]), [1])
-    clf._dual_coef_ = np.array([[.0, 1.]])
-    assert_array_equal(clf.predict([[-.1, -.1]]), [2])
+    assert_array_equal(clf.dual_coef_, [[-0.25, 0.25]])
+    assert_array_equal(clf.predict([[-0.1, -0.1]]), [1])
+    clf._dual_coef_ = np.array([[0.0, 1.0]])
+    assert_array_equal(clf.predict([[-0.1, -0.1]]), [2])
 
 
 def test_probability():
     # Predict probabilities using SVC
     # This uses cross validation, so we use a slightly bigger testing set.
 
-    for clf in (svm.SVC(probability=True, random_state=0, C=1.0),
-                svm.NuSVC(probability=True, random_state=0)):
+    for clf in (
+        svm.SVC(probability=True, random_state=0, C=1.0),
+        svm.NuSVC(probability=True, random_state=0),
+    ):
         clf.fit(iris.data, iris.target)
 
         prob_predict = clf.predict_proba(iris.data)
-        assert_array_almost_equal(
-            np.sum(prob_predict, 1), np.ones(iris.data.shape[0]))
-        assert np.mean(np.argmax(prob_predict, 1)
-                       == clf.predict(iris.data)) > 0.9
+        assert_array_almost_equal(np.sum(prob_predict, 1), np.ones(iris.data.shape[0]))
+        assert np.mean(np.argmax(prob_predict, 1) == clf.predict(iris.data)) > 0.9
 
-        assert_almost_equal(clf.predict_proba(iris.data),
-                            np.exp(clf.predict_log_proba(iris.data)), 8)
+        assert_almost_equal(
+            clf.predict_proba(iris.data), np.exp(clf.predict_log_proba(iris.data)), 8
+        )
 
 
 def test_decision_function():
@@ -333,8 +393,9 @@ def test_decision_function():
     # Sanity check, test that decision_function implemented in python
     # returns the same as the one in libsvm
     # multi class:
-    clf = svm.SVC(kernel='linear', C=0.1,
-                  decision_function_shape='ovo').fit(iris.data, iris.target)
+    clf = svm.SVC(kernel="linear", C=0.1, decision_function_shape="ovo").fit(
+        iris.data, iris.target
+    )
 
     dec = np.dot(iris.data, clf.coef_.T) + clf.intercept_
 
@@ -346,13 +407,13 @@ def test_decision_function():
     prediction = clf.predict(X)
     assert_array_almost_equal(dec.ravel(), clf.decision_function(X))
     assert_array_almost_equal(
-        prediction,
-        clf.classes_[(clf.decision_function(X) > 0).astype(int)])
-    expected = np.array([-1., -0.66, -1., 0.66, 1., 1.])
+        prediction, clf.classes_[(clf.decision_function(X) > 0).astype(int)]
+    )
+    expected = np.array([-1.0, -0.66, -1.0, 0.66, 1.0, 1.0])
     assert_array_almost_equal(clf.decision_function(X), expected, 2)
 
     # kernel binary:
-    clf = svm.SVC(kernel='rbf', gamma=1, decision_function_shape='ovo')
+    clf = svm.SVC(kernel="rbf", gamma=1, decision_function_shape="ovo")
     clf.fit(X, Y)
 
     rbfs = rbf_kernel(X, clf.support_vectors_, gamma=clf.gamma)
@@ -360,13 +421,14 @@ def test_decision_function():
     assert_array_almost_equal(dec.ravel(), clf.decision_function(X))
 
 
-@pytest.mark.parametrize('SVM', (svm.SVC, svm.NuSVC))
+@pytest.mark.parametrize("SVM", (svm.SVC, svm.NuSVC))
 def test_decision_function_shape(SVM):
     # check that decision_function_shape='ovr' or 'ovo' gives
     # correct shape and is consistent with predict
 
-    clf = SVM(kernel='linear',
-              decision_function_shape='ovr').fit(iris.data, iris.target)
+    clf = SVM(kernel="linear", decision_function_shape="ovr").fit(
+        iris.data, iris.target
+    )
     dec = clf.decision_function(iris.data)
     assert dec.shape == (len(iris.data), 3)
     assert_array_equal(clf.predict(iris.data), np.argmax(dec, axis=1))
@@ -375,21 +437,16 @@ def test_decision_function_shape(SVM):
     X, y = make_blobs(n_samples=80, centers=5, random_state=0)
     X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
 
-    clf = SVM(kernel='linear',
-              decision_function_shape='ovr').fit(X_train, y_train)
+    clf = SVM(kernel="linear", decision_function_shape="ovr").fit(X_train, y_train)
     dec = clf.decision_function(X_test)
     assert dec.shape == (len(X_test), 5)
     assert_array_equal(clf.predict(X_test), np.argmax(dec, axis=1))
 
     # check shape of ovo_decition_function=True
-    clf = SVM(kernel='linear',
-              decision_function_shape='ovo').fit(X_train, y_train)
+    clf = SVM(kernel="linear", decision_function_shape="ovo").fit(X_train, y_train)
     dec = clf.decision_function(X_train)
     assert dec.shape == (len(X_train), 10)
 
-    with pytest.raises(ValueError, match="must be either 'ovr' or 'ovo'"):
-        SVM(decision_function_shape='bad').fit(X_train, y_train)
-
 
 def test_svr_predict():
     # Test SVR's decision_function
@@ -400,13 +457,13 @@ def test_svr_predict():
     y = iris.target
 
     # linear kernel
-    reg = svm.SVR(kernel='linear', C=0.1).fit(X, y)
+    reg = svm.SVR(kernel="linear", C=0.1).fit(X, y)
 
     dec = np.dot(X, reg.coef_.T) + reg.intercept_
     assert_array_almost_equal(dec.ravel(), reg.predict(X).ravel())
 
     # rbf kernel
-    reg = svm.SVR(kernel='rbf', gamma=1).fit(X, y)
+    reg = svm.SVR(kernel="rbf", gamma=1).fit(X, y)
 
     rbfs = rbf_kernel(X, reg.support_vectors_, gamma=reg.gamma)
     dec = np.dot(rbfs, reg.dual_coef_.T) + reg.intercept_
@@ -421,15 +478,19 @@ def test_weight():
     # so all predicted values belong to class 2
     assert_array_almost_equal(clf.predict(X), [2] * 6)
 
-    X_, y_ = make_classification(n_samples=200, n_features=10,
-                                 weights=[0.833, 0.167], random_state=2)
+    X_, y_ = make_classification(
+        n_samples=200, n_features=10, weights=[0.833, 0.167], random_state=2
+    )
 
-    for clf in (linear_model.LogisticRegression(),
-                svm.LinearSVC(random_state=0), svm.SVC()):
-        clf.set_params(class_weight={0: .1, 1: 10})
+    for clf in (
+        linear_model.LogisticRegression(),
+        svm.LinearSVC(random_state=0),
+        svm.SVC(),
+    ):
+        clf.set_params(class_weight={0: 0.1, 1: 10})
         clf.fit(X_[:100], y_[:100])
         y_pred = clf.predict(X_[100:])
-        assert f1_score(y_[100:], y_pred) > .3
+        assert f1_score(y_[100:], y_pred) > 0.3
 
 
 @pytest.mark.parametrize("estimator", [svm.SVC(C=1e-2), svm.NuSVC()])
@@ -437,53 +498,50 @@ def test_svm_classifier_sided_sample_weight(estimator):
     # fit a linear SVM and check that giving more weight to opposed samples
     # in the space will flip the decision toward these samples.
     X = [[-2, 0], [-1, -1], [0, -2], [0, 2], [1, 1], [2, 0]]
-    estimator.set_params(kernel='linear')
+    estimator.set_params(kernel="linear")
 
     # check that with unit weights, a sample is supposed to be predicted on
     # the boundary
     sample_weight = [1] * 6
     estimator.fit(X, Y, sample_weight=sample_weight)
-    y_pred = estimator.decision_function([[-1., 1.]])
+    y_pred = estimator.decision_function([[-1.0, 1.0]])
     assert y_pred == pytest.approx(0)
 
     # give more weights to opposed samples
-    sample_weight = [10., .1, .1, .1, .1, 10]
+    sample_weight = [10.0, 0.1, 0.1, 0.1, 0.1, 10]
     estimator.fit(X, Y, sample_weight=sample_weight)
-    y_pred = estimator.decision_function([[-1., 1.]])
+    y_pred = estimator.decision_function([[-1.0, 1.0]])
     assert y_pred < 0
 
-    sample_weight = [1., .1, 10., 10., .1, .1]
+    sample_weight = [1.0, 0.1, 10.0, 10.0, 0.1, 0.1]
     estimator.fit(X, Y, sample_weight=sample_weight)
-    y_pred = estimator.decision_function([[-1., 1.]])
+    y_pred = estimator.decision_function([[-1.0, 1.0]])
     assert y_pred > 0
 
 
-@pytest.mark.parametrize(
-    "estimator",
-    [svm.SVR(C=1e-2), svm.NuSVR(C=1e-2)]
-)
+@pytest.mark.parametrize("estimator", [svm.SVR(C=1e-2), svm.NuSVR(C=1e-2)])
 def test_svm_regressor_sided_sample_weight(estimator):
     # similar test to test_svm_classifier_sided_sample_weight but for
     # SVM regressors
     X = [[-2, 0], [-1, -1], [0, -2], [0, 2], [1, 1], [2, 0]]
-    estimator.set_params(kernel='linear')
+    estimator.set_params(kernel="linear")
 
     # check that with unit weights, a sample is supposed to be predicted on
     # the boundary
     sample_weight = [1] * 6
     estimator.fit(X, Y, sample_weight=sample_weight)
-    y_pred = estimator.predict([[-1., 1.]])
+    y_pred = estimator.predict([[-1.0, 1.0]])
     assert y_pred == pytest.approx(1.5)
 
     # give more weights to opposed samples
-    sample_weight = [10., .1, .1, .1, .1, 10]
+    sample_weight = [10.0, 0.1, 0.1, 0.1, 0.1, 10]
     estimator.fit(X, Y, sample_weight=sample_weight)
-    y_pred = estimator.predict([[-1., 1.]])
+    y_pred = estimator.predict([[-1.0, 1.0]])
     assert y_pred < 1.5
 
-    sample_weight = [1., .1, 10., 10., .1, .1]
+    sample_weight = [1.0, 0.1, 10.0, 10.0, 0.1, 0.1]
     estimator.fit(X, Y, sample_weight=sample_weight)
-    y_pred = estimator.predict([[-1., 1.]])
+    y_pred = estimator.predict([[-1.0, 1.0]])
     assert y_pred > 1.5
 
 
@@ -499,93 +557,92 @@ def test_svm_equivalence_sample_weight_C():
 
 @pytest.mark.parametrize(
     "Estimator, err_msg",
-    [(svm.SVC,
-      'Invalid input - all samples have zero or negative weights.'),
-     (svm.NuSVC, '(negative dimensions are not allowed|nu is infeasible)'),
-     (svm.SVR,
-      'Invalid input - all samples have zero or negative weights.'),
-     (svm.NuSVR,
-      'Invalid input - all samples have zero or negative weights.'),
-     (svm.OneClassSVM,
-      'Invalid input - all samples have zero or negative weights.')
-     ],
-    ids=['SVC', 'NuSVC', 'SVR', 'NuSVR', 'OneClassSVM']
+    [
+        (svm.SVC, "Invalid input - all samples have zero or negative weights."),
+        (svm.NuSVC, "(negative dimensions are not allowed|nu is infeasible)"),
+        (svm.SVR, "Invalid input - all samples have zero or negative weights."),
+        (svm.NuSVR, "Invalid input - all samples have zero or negative weights."),
+        (svm.OneClassSVM, "Invalid input - all samples have zero or negative weights."),
+    ],
+    ids=["SVC", "NuSVC", "SVR", "NuSVR", "OneClassSVM"],
 )
 @pytest.mark.parametrize(
     "sample_weight",
     [[0] * len(Y), [-0.3] * len(Y)],
-    ids=['weights-are-zero', 'weights-are-negative']
+    ids=["weights-are-zero", "weights-are-negative"],
 )
-def test_negative_sample_weights_mask_all_samples(Estimator,
-                                                  err_msg, sample_weight):
-    est = Estimator(kernel='linear')
+def test_negative_sample_weights_mask_all_samples(Estimator, err_msg, sample_weight):
+    est = Estimator(kernel="linear")
     with pytest.raises(ValueError, match=err_msg):
         est.fit(X, Y, sample_weight=sample_weight)
 
 
 @pytest.mark.parametrize(
     "Classifier, err_msg",
-    [(svm.SVC,
-     'Invalid input - all samples with positive weights have the same label'),
-     (svm.NuSVC, 'specified nu is infeasible')],
-    ids=['SVC', 'NuSVC']
+    [
+        (
+            svm.SVC,
+            (
+                "Invalid input - all samples with positive weights belong to the same"
+                " class"
+            ),
+        ),
+        (svm.NuSVC, "specified nu is infeasible"),
+    ],
+    ids=["SVC", "NuSVC"],
 )
 @pytest.mark.parametrize(
     "sample_weight",
-    [[0, -0.5, 0, 1, 1, 1],
-     [1, 1, 1, 0, -0.1, -0.3]],
-    ids=['mask-label-1', 'mask-label-2']
+    [[0, -0.5, 0, 1, 1, 1], [1, 1, 1, 0, -0.1, -0.3]],
+    ids=["mask-label-1", "mask-label-2"],
 )
-def test_negative_weights_svc_leave_just_one_label(Classifier,
-                                                   err_msg,
-                                                   sample_weight):
-    clf = Classifier(kernel='linear')
+def test_negative_weights_svc_leave_just_one_label(Classifier, err_msg, sample_weight):
+    clf = Classifier(kernel="linear")
     with pytest.raises(ValueError, match=err_msg):
         clf.fit(X, Y, sample_weight=sample_weight)
 
 
 @pytest.mark.parametrize(
     "Classifier, model",
-    [(svm.SVC, {'when-left': [0.3998, 0.4], 'when-right': [0.4, 0.3999]}),
-     (svm.NuSVC, {'when-left': [0.3333, 0.3333],
-      'when-right': [0.3333, 0.3333]})],
-    ids=['SVC', 'NuSVC']
+    [
+        (svm.SVC, {"when-left": [0.3998, 0.4], "when-right": [0.4, 0.3999]}),
+        (svm.NuSVC, {"when-left": [0.3333, 0.3333], "when-right": [0.3333, 0.3333]}),
+    ],
+    ids=["SVC", "NuSVC"],
 )
 @pytest.mark.parametrize(
     "sample_weight, mask_side",
-    [([1, -0.5, 1, 1, 1, 1], 'when-left'),
-     ([1, 1, 1, 0, 1, 1], 'when-right')],
-    ids=['partial-mask-label-1', 'partial-mask-label-2']
+    [([1, -0.5, 1, 1, 1, 1], "when-left"), ([1, 1, 1, 0, 1, 1], "when-right")],
+    ids=["partial-mask-label-1", "partial-mask-label-2"],
 )
-def test_negative_weights_svc_leave_two_labels(Classifier, model,
-                                               sample_weight, mask_side):
-    clf = Classifier(kernel='linear')
+def test_negative_weights_svc_leave_two_labels(
+    Classifier, model, sample_weight, mask_side
+):
+    clf = Classifier(kernel="linear")
     clf.fit(X, Y, sample_weight=sample_weight)
     assert_allclose(clf.coef_, [model[mask_side]], rtol=1e-3)
 
 
 @pytest.mark.parametrize(
-    "Estimator",
-    [svm.SVC, svm.NuSVC, svm.NuSVR],
-    ids=['SVC', 'NuSVC', 'NuSVR']
+    "Estimator", [svm.SVC, svm.NuSVC, svm.NuSVR], ids=["SVC", "NuSVC", "NuSVR"]
 )
 @pytest.mark.parametrize(
     "sample_weight",
     [[1, -0.5, 1, 1, 1, 1], [1, 1, 1, 0, 1, 1]],
-    ids=['partial-mask-label-1', 'partial-mask-label-2']
+    ids=["partial-mask-label-1", "partial-mask-label-2"],
 )
 def test_negative_weight_equal_coeffs(Estimator, sample_weight):
     # model generates equal coefficients
-    est = Estimator(kernel='linear')
+    est = Estimator(kernel="linear")
     est.fit(X, Y, sample_weight=sample_weight)
     coef = np.abs(est.coef_).ravel()
     assert coef[0] == pytest.approx(coef[1], rel=1e-3)
 
 
-@ignore_warnings(category=UndefinedMetricWarning)
 def test_auto_weight():
     # Test class weights for imbalanced data
     from sklearn.linear_model import LogisticRegression
+
     # We take as dataset the two-dimensional projection of iris so
     # that it is not separable and remove half of predictors from
     # class 1.
@@ -593,60 +650,58 @@ def test_auto_weight():
     # class_weight="balanced"
     # used to work only when the labels where a range [0..K).
     from sklearn.utils import compute_class_weight
+
     X, y = iris.data[:, :2], iris.target + 1
     unbalanced = np.delete(np.arange(y.size), np.where(y > 2)[0][::2])
 
     classes = np.unique(y[unbalanced])
-    class_weights = compute_class_weight('balanced', classes=classes,
-                                         y=y[unbalanced])
+    class_weights = compute_class_weight("balanced", classes=classes, y=y[unbalanced])
     assert np.argmax(class_weights) == 2
 
-    for clf in (svm.SVC(kernel='linear'), svm.LinearSVC(random_state=0),
-                LogisticRegression()):
+    for clf in (
+        svm.SVC(kernel="linear"),
+        svm.LinearSVC(random_state=0),
+        LogisticRegression(),
+    ):
         # check that score is better when class='balanced' is set.
         y_pred = clf.fit(X[unbalanced], y[unbalanced]).predict(X)
-        clf.set_params(class_weight='balanced')
-        y_pred_balanced = clf.fit(X[unbalanced], y[unbalanced],).predict(X)
-        assert (metrics.f1_score(y, y_pred, average='macro')
-                <= metrics.f1_score(y, y_pred_balanced,
-                                    average='macro'))
-
+        clf.set_params(class_weight="balanced")
+        y_pred_balanced = clf.fit(
+            X[unbalanced],
+            y[unbalanced],
+        ).predict(X)
+        assert metrics.f1_score(y, y_pred, average="macro") <= metrics.f1_score(
+            y, y_pred_balanced, average="macro"
+        )
 
-def test_bad_input():
-    # Test that it gives proper exception on deficient input
-    # impossible value of C
-    with pytest.raises(ValueError):
-        svm.SVC(C=-1).fit(X, Y)
-
-    # impossible value of nu
-    clf = svm.NuSVC(nu=0.0)
-    with pytest.raises(ValueError):
-        clf.fit(X, Y)
 
+@pytest.mark.parametrize("lil_container", LIL_CONTAINERS)
+def test_bad_input(lil_container):
+    # Test dimensions for labels
     Y2 = Y[:-1]  # wrong dimensions for labels
     with pytest.raises(ValueError):
-        clf.fit(X, Y2)
+        svm.SVC().fit(X, Y2)
 
     # Test with arrays that are non-contiguous.
     for clf in (svm.SVC(), svm.LinearSVC(random_state=0)):
         Xf = np.asfortranarray(X)
-        assert not Xf.flags['C_CONTIGUOUS']
+        assert not Xf.flags["C_CONTIGUOUS"]
         yf = np.ascontiguousarray(np.tile(Y, (2, 1)).T)
         yf = yf[:, -1]
-        assert not yf.flags['F_CONTIGUOUS']
-        assert not yf.flags['C_CONTIGUOUS']
+        assert not yf.flags["F_CONTIGUOUS"]
+        assert not yf.flags["C_CONTIGUOUS"]
         clf.fit(Xf, yf)
         assert_array_equal(clf.predict(T), true_result)
 
     # error for precomputed kernelsx
-    clf = svm.SVC(kernel='precomputed')
+    clf = svm.SVC(kernel="precomputed")
     with pytest.raises(ValueError):
         clf.fit(X, Y)
 
     # predict with sparse input when trained with dense
     clf = svm.SVC().fit(X, Y)
     with pytest.raises(ValueError):
-        clf.predict(sparse.lil_matrix(X))
+        clf.predict(lil_container(X))
 
     Xt = np.array(X).T
     clf.fit(np.dot(X, Xt), Y)
@@ -659,93 +714,71 @@ def test_bad_input():
         clf.predict(Xt)
 
 
-@pytest.mark.parametrize(
-    'Estimator, data',
-    [(svm.SVC, datasets.load_iris(return_X_y=True)),
-     (svm.NuSVC, datasets.load_iris(return_X_y=True)),
-     (svm.SVR, datasets.load_diabetes(return_X_y=True)),
-     (svm.NuSVR, datasets.load_diabetes(return_X_y=True)),
-     (svm.OneClassSVM, datasets.load_iris(return_X_y=True))]
-)
-def test_svm_gamma_error(Estimator, data):
-    X, y = data
-    est = Estimator(gamma='auto_deprecated')
-    err_msg = "When 'gamma' is a string, it should be either 'scale' or 'auto'"
-    with pytest.raises(ValueError, match=err_msg):
-        est.fit(X, y)
+def test_svc_nonfinite_params():
+    # Check SVC throws ValueError when dealing with non-finite parameter values
+    rng = np.random.RandomState(0)
+    n_samples = 10
+    fmax = np.finfo(np.float64).max
+    X = fmax * rng.uniform(size=(n_samples, 2))
+    y = rng.randint(0, 2, size=n_samples)
+
+    clf = svm.SVC()
+    msg = "The dual coefficients or intercepts are not finite"
+    with pytest.raises(ValueError, match=msg):
+        clf.fit(X, y)
 
 
 def test_unicode_kernel():
     # Test that a unicode kernel name does not cause a TypeError
-    clf = svm.SVC(kernel='linear', probability=True)
+    clf = svm.SVC(kernel="linear", probability=True)
     clf.fit(X, Y)
     clf.predict_proba(T)
-    _libsvm.cross_validation(iris.data,
-                             iris.target.astype(np.float64), 5,
-                             kernel='linear',
-                             random_seed=0)
+    _libsvm.cross_validation(
+        iris.data, iris.target.astype(np.float64), 5, kernel="linear", random_seed=0
+    )
 
 
-def test_sparse_precomputed():
-    clf = svm.SVC(kernel='precomputed')
-    sparse_gram = sparse.csr_matrix([[1, 0], [0, 1]])
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sparse_precomputed(csr_container):
+    clf = svm.SVC(kernel="precomputed")
+    sparse_gram = csr_container([[1, 0], [0, 1]])
     with pytest.raises(TypeError, match="Sparse precomputed"):
         clf.fit(sparse_gram, [0, 1])
 
 
-def test_sparse_fit_support_vectors_empty():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sparse_fit_support_vectors_empty(csr_container):
     # Regression test for #14893
-    X_train = sparse.csr_matrix([[0, 1, 0, 0],
-                                 [0, 0, 0, 1],
-                                 [0, 0, 1, 0],
-                                 [0, 0, 0, 1]])
+    X_train = csr_container([[0, 1, 0, 0], [0, 0, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]])
     y_train = np.array([0.04, 0.04, 0.10, 0.16])
-    model = svm.SVR(kernel='linear')
+    model = svm.SVR(kernel="linear")
     model.fit(X_train, y_train)
     assert not model.support_vectors_.data.size
     assert not model.dual_coef_.data.size
 
 
-def test_linearsvc_parameters():
+@pytest.mark.parametrize("loss", ["hinge", "squared_hinge"])
+@pytest.mark.parametrize("penalty", ["l1", "l2"])
+@pytest.mark.parametrize("dual", [True, False])
+def test_linearsvc_parameters(loss, penalty, dual):
     # Test possible parameter combinations in LinearSVC
     # Generate list of possible parameter combinations
-    losses = ['hinge', 'squared_hinge', 'logistic_regression', 'foo']
-    penalties, duals = ['l1', 'l2', 'bar'], [True, False]
-
-    X, y = make_classification(n_samples=5, n_features=5)
-
-    for loss, penalty, dual in itertools.product(losses, penalties, duals):
-        clf = svm.LinearSVC(penalty=penalty, loss=loss, dual=dual)
-        if ((loss, penalty) == ('hinge', 'l1') or
-                (loss, penalty, dual) == ('hinge', 'l2', False) or
-                (penalty, dual) == ('l1', True) or
-                loss == 'foo' or penalty == 'bar'):
-
-            with pytest.raises(ValueError, match="Unsupported set of "
-                               "arguments.*penalty='%s.*loss='%s.*dual=%s"
-                               % (penalty, loss, dual)):
-                clf.fit(X, y)
-        else:
+    X, y = make_classification(n_samples=5, n_features=5, random_state=0)
+
+    clf = svm.LinearSVC(penalty=penalty, loss=loss, dual=dual, random_state=0)
+    if (
+        (loss, penalty) == ("hinge", "l1")
+        or (loss, penalty, dual) == ("hinge", "l2", False)
+        or (penalty, dual) == ("l1", True)
+    ):
+        with pytest.raises(
+            ValueError,
+            match="Unsupported set of arguments.*penalty='%s.*loss='%s.*dual=%s"
+            % (penalty, loss, dual),
+        ):
             clf.fit(X, y)
-
-    # Incorrect loss value - test if explicit error message is raised
-    with pytest.raises(ValueError, match=".*loss='l3' is not supported.*"):
-        svm.LinearSVC(loss="l3").fit(X, y)
-
-
-def test_linear_svx_uppercase_loss_penality_raises_error():
-    # Check if Upper case notation raises error at _fit_liblinear
-    # which is called by fit
-
-    X, y = [[0.0], [1.0]], [0, 1]
-
-    assert_raise_message(ValueError, "loss='SQuared_hinge' is not supported",
-                         svm.LinearSVC(loss="SQuared_hinge").fit, X, y)
-
-    assert_raise_message(ValueError,
-                         ("The combination of penalty='L2'"
-                          " and loss='squared_hinge' is not supported"),
-                         svm.LinearSVC(penalty="L2").fit, X, y)
+    else:
+        clf.fit(X, y)
 
 
 def test_linearsvc():
@@ -759,16 +792,17 @@ def test_linearsvc():
     assert_array_almost_equal(clf.intercept_, [0], decimal=3)
 
     # the same with l1 penalty
-    clf = svm.LinearSVC(penalty='l1', loss='squared_hinge', dual=False,
-                        random_state=0).fit(X, Y)
+    clf = svm.LinearSVC(
+        penalty="l1", loss="squared_hinge", dual=False, random_state=0
+    ).fit(X, Y)
     assert_array_equal(clf.predict(T), true_result)
 
     # l2 penalty with dual formulation
-    clf = svm.LinearSVC(penalty='l2', dual=True, random_state=0).fit(X, Y)
+    clf = svm.LinearSVC(penalty="l2", dual=True, random_state=0).fit(X, Y)
     assert_array_equal(clf.predict(T), true_result)
 
     # l2 penalty, l1 loss
-    clf = svm.LinearSVC(penalty='l2', loss='hinge', dual=True, random_state=0)
+    clf = svm.LinearSVC(penalty="l2", loss="hinge", dual=True, random_state=0)
     clf.fit(X, Y)
     assert_array_equal(clf.predict(T), true_result)
 
@@ -781,19 +815,20 @@ def test_linearsvc():
 def test_linearsvc_crammer_singer():
     # Test LinearSVC with crammer_singer multi-class svm
     ovr_clf = svm.LinearSVC(random_state=0).fit(iris.data, iris.target)
-    cs_clf = svm.LinearSVC(multi_class='crammer_singer', random_state=0)
+    cs_clf = svm.LinearSVC(multi_class="crammer_singer", random_state=0)
     cs_clf.fit(iris.data, iris.target)
 
     # similar prediction for ovr and crammer-singer:
-    assert (ovr_clf.predict(iris.data) ==
-            cs_clf.predict(iris.data)).mean() > .9
+    assert (ovr_clf.predict(iris.data) == cs_clf.predict(iris.data)).mean() > 0.9
 
     # classifiers shouldn't be the same
     assert (ovr_clf.coef_ != cs_clf.coef_).all()
 
     # test decision function
-    assert_array_equal(cs_clf.predict(iris.data),
-                       np.argmax(cs_clf.decision_function(iris.data), axis=1))
+    assert_array_equal(
+        cs_clf.predict(iris.data),
+        np.argmax(cs_clf.decision_function(iris.data), axis=1),
+    )
     dec_func = np.dot(iris.data, cs_clf.coef_.T) + cs_clf.intercept_
     assert_array_almost_equal(dec_func, cs_clf.decision_function(iris.data))
 
@@ -803,8 +838,9 @@ def test_linearsvc_fit_sampleweight():
     n_samples = len(X)
     unit_weight = np.ones(n_samples)
     clf = svm.LinearSVC(random_state=0).fit(X, Y)
-    clf_unitweight = svm.LinearSVC(random_state=0, tol=1e-12, max_iter=1000).\
-        fit(X, Y, sample_weight=unit_weight)
+    clf_unitweight = svm.LinearSVC(random_state=0, tol=1e-12, max_iter=1000).fit(
+        X, Y, sample_weight=unit_weight
+    )
 
     # check if same as sample_weight=None
     assert_array_equal(clf_unitweight.predict(T), clf.predict(T))
@@ -815,14 +851,17 @@ def test_linearsvc_fit_sampleweight():
 
     random_state = check_random_state(0)
     random_weight = random_state.randint(0, 10, n_samples)
-    lsvc_unflat = svm.LinearSVC(random_state=0, tol=1e-12, max_iter=1000).\
-        fit(X, Y, sample_weight=random_weight)
+    lsvc_unflat = svm.LinearSVC(random_state=0, tol=1e-12, max_iter=1000).fit(
+        X, Y, sample_weight=random_weight
+    )
+
     pred1 = lsvc_unflat.predict(T)
 
     X_flat = np.repeat(X, random_weight, axis=0)
     y_flat = np.repeat(Y, random_weight, axis=0)
     lsvc_flat = svm.LinearSVC(random_state=0, tol=1e-12, max_iter=1000).fit(
-        X_flat, y_flat)
+        X_flat, y_flat
+    )
     pred2 = lsvc_flat.predict(T)
 
     assert_array_equal(pred1, pred2)
@@ -834,9 +873,15 @@ def test_crammer_singer_binary():
     X, y = make_classification(n_classes=2, random_state=0)
 
     for fit_intercept in (True, False):
-        acc = svm.LinearSVC(fit_intercept=fit_intercept,
-                            multi_class="crammer_singer",
-                            random_state=0).fit(X, y).score(X, y)
+        acc = (
+            svm.LinearSVC(
+                fit_intercept=fit_intercept,
+                multi_class="crammer_singer",
+                random_state=0,
+            )
+            .fit(X, y)
+            .score(X, y)
+        )
         assert acc > 0.9
 
 
@@ -855,13 +900,17 @@ def test_linearsvc_iris():
 
 def test_dense_liblinear_intercept_handling(classifier=svm.LinearSVC):
     # Test that dense liblinear honours intercept_scaling param
-    X = [[2, 1],
-         [3, 1],
-         [1, 3],
-         [2, 3]]
+    X = [[2, 1], [3, 1], [1, 3], [2, 3]]
     y = [0, 0, 1, 1]
-    clf = classifier(fit_intercept=True, penalty='l1', loss='squared_hinge',
-                     dual=False, C=4, tol=1e-7, random_state=0)
+    clf = classifier(
+        fit_intercept=True,
+        penalty="l1",
+        loss="squared_hinge",
+        dual=False,
+        C=4,
+        tol=1e-7,
+        random_state=0,
+    )
     assert clf.intercept_scaling == 1, clf.intercept_scaling
     assert clf.fit_intercept
 
@@ -896,10 +945,7 @@ def test_liblinear_set_coef():
     assert_array_almost_equal(values, values2)
 
     # binary-class case
-    X = [[2, 1],
-         [3, 1],
-         [1, 3],
-         [2, 3]]
+    X = [[2, 1], [3, 1], [1, 3], [2, 3]]
     y = [0, 0, 1, 1]
 
     clf = svm.LinearSVC().fit(X, y)
@@ -913,15 +959,15 @@ def test_liblinear_set_coef():
 def test_immutable_coef_property():
     # Check that primal coef modification are not silently ignored
     svms = [
-        svm.SVC(kernel='linear').fit(iris.data, iris.target),
-        svm.NuSVC(kernel='linear').fit(iris.data, iris.target),
-        svm.SVR(kernel='linear').fit(iris.data, iris.target),
-        svm.NuSVR(kernel='linear').fit(iris.data, iris.target),
-        svm.OneClassSVM(kernel='linear').fit(iris.data),
+        svm.SVC(kernel="linear").fit(iris.data, iris.target),
+        svm.NuSVC(kernel="linear").fit(iris.data, iris.target),
+        svm.SVR(kernel="linear").fit(iris.data, iris.target),
+        svm.NuSVR(kernel="linear").fit(iris.data, iris.target),
+        svm.OneClassSVM(kernel="linear").fit(iris.data),
     ]
     for clf in svms:
         with pytest.raises(AttributeError):
-            clf.__setattr__('coef_', np.arange(3))
+            clf.__setattr__("coef_", np.arange(3))
         with pytest.raises((RuntimeError, ValueError)):
             clf.coef_.__setitem__((0, 0), 0)
 
@@ -929,6 +975,7 @@ def test_immutable_coef_property():
 def test_linearsvc_verbose():
     # stdout: redirect
     import os
+
     stdout = os.dup(1)  # save original stdout
     os.dup2(os.pipe()[1], 1)  # replace it
 
@@ -943,29 +990,34 @@ def test_linearsvc_verbose():
 def test_svc_clone_with_callable_kernel():
     # create SVM with callable linear kernel, check that results are the same
     # as with built-in linear kernel
-    svm_callable = svm.SVC(kernel=lambda x, y: np.dot(x, y.T),
-                           probability=True, random_state=0,
-                           decision_function_shape='ovr')
+    svm_callable = svm.SVC(
+        kernel=lambda x, y: np.dot(x, y.T),
+        probability=True,
+        random_state=0,
+        decision_function_shape="ovr",
+    )
     # clone for checking clonability with lambda functions..
     svm_cloned = base.clone(svm_callable)
     svm_cloned.fit(iris.data, iris.target)
 
-    svm_builtin = svm.SVC(kernel='linear', probability=True, random_state=0,
-                          decision_function_shape='ovr')
+    svm_builtin = svm.SVC(
+        kernel="linear", probability=True, random_state=0, decision_function_shape="ovr"
+    )
     svm_builtin.fit(iris.data, iris.target)
 
-    assert_array_almost_equal(svm_cloned.dual_coef_,
-                              svm_builtin.dual_coef_)
-    assert_array_almost_equal(svm_cloned.intercept_,
-                              svm_builtin.intercept_)
-    assert_array_equal(svm_cloned.predict(iris.data),
-                       svm_builtin.predict(iris.data))
+    assert_array_almost_equal(svm_cloned.dual_coef_, svm_builtin.dual_coef_)
+    assert_array_almost_equal(svm_cloned.intercept_, svm_builtin.intercept_)
+    assert_array_equal(svm_cloned.predict(iris.data), svm_builtin.predict(iris.data))
 
-    assert_array_almost_equal(svm_cloned.predict_proba(iris.data),
-                              svm_builtin.predict_proba(iris.data),
-                              decimal=4)
-    assert_array_almost_equal(svm_cloned.decision_function(iris.data),
-                              svm_builtin.decision_function(iris.data))
+    assert_array_almost_equal(
+        svm_cloned.predict_proba(iris.data),
+        svm_builtin.predict_proba(iris.data),
+        decimal=4,
+    )
+    assert_array_almost_equal(
+        svm_cloned.decision_function(iris.data),
+        svm_builtin.decision_function(iris.data),
+    )
 
 
 def test_svc_bad_kernel():
@@ -974,15 +1026,17 @@ def test_svc_bad_kernel():
         svc.fit(X, Y)
 
 
-def test_timeout():
-    a = svm.SVC(kernel=lambda x, y: np.dot(x, y.T), probability=True,
-                random_state=0, max_iter=1)
+def test_libsvm_convergence_warnings():
+    a = svm.SVC(
+        kernel=lambda x, y: np.dot(x, y.T), probability=True, random_state=0, max_iter=2
+    )
     warning_msg = (
-        r'Solver terminated early \(max_iter=1\).  Consider pre-processing '
-        r'your data with StandardScaler or MinMaxScaler.'
+        r"Solver terminated early \(max_iter=2\).  Consider pre-processing "
+        r"your data with StandardScaler or MinMaxScaler."
     )
     with pytest.warns(ConvergenceWarning, match=warning_msg):
         a.fit(np.array(X), Y)
+    assert np.all(a.n_iter_ == 2)
 
 
 def test_unfitted():
@@ -998,7 +1052,7 @@ def test_unfitted():
 
 
 # ignore convergence warnings from max_iter=1
-@ignore_warnings
+@pytest.mark.filterwarnings("ignore::sklearn.exceptions.ConvergenceWarning")
 def test_consistent_proba():
     a = svm.SVC(probability=True, max_iter=1, random_state=0)
     proba_1 = a.fit(X, Y).predict_proba(X)
@@ -1011,16 +1065,18 @@ def test_linear_svm_convergence_warnings():
     # Test that warnings are raised if model does not converge
 
     lsvc = svm.LinearSVC(random_state=0, max_iter=2)
-    warning_msg = (
-        "Liblinear failed to converge, increase the number of iterations."
-    )
+    warning_msg = "Liblinear failed to converge, increase the number of iterations."
     with pytest.warns(ConvergenceWarning, match=warning_msg):
         lsvc.fit(X, Y)
+    # Check that we have an n_iter_ attribute with int type as opposed to a
+    # numpy array or an np.int32 so as to match the docstring.
+    assert isinstance(lsvc.n_iter_, int)
     assert lsvc.n_iter_ == 2
 
     lsvr = svm.LinearSVR(random_state=0, max_iter=2)
     with pytest.warns(ConvergenceWarning, match=warning_msg):
         lsvr.fit(iris.data, iris.target)
+    assert isinstance(lsvr.n_iter_, int)
     assert lsvr.n_iter_ == 2
 
 
@@ -1030,31 +1086,23 @@ def test_svr_coef_sign():
     X = np.random.RandomState(21).randn(10, 3)
     y = np.random.RandomState(12).randn(10)
 
-    for svr in [svm.SVR(kernel='linear'), svm.NuSVR(kernel='linear'),
-                svm.LinearSVR()]:
+    for svr in [
+        svm.SVR(kernel="linear"),
+        svm.NuSVR(kernel="linear"),
+        svm.LinearSVR(),
+    ]:
         svr.fit(X, y)
         assert_array_almost_equal(
             svr.predict(X), np.dot(X, svr.coef_.ravel()) + svr.intercept_
         )
 
 
-def test_linear_svc_intercept_scaling():
-    # Test that the right error message is thrown when intercept_scaling <= 0
-
-    for i in [-1, 0]:
-        lsvc = svm.LinearSVC(intercept_scaling=i)
-        msg = ('Intercept scaling is %r but needs to be greater than 0.'
-               ' To disable fitting an intercept,'
-               ' set fit_intercept=False.' % lsvc.intercept_scaling)
-        assert_raise_message(ValueError, msg, lsvc.fit, X, Y)
-
-
 def test_lsvc_intercept_scaling_zero():
     # Test that intercept_scaling is ignored when fit_intercept is False
 
     lsvc = svm.LinearSVC(fit_intercept=False)
     lsvc.fit(X, Y)
-    assert lsvc.intercept_ == 0.
+    assert lsvc.intercept_ == 0.0
 
 
 def test_hasattr_predict_proba():
@@ -1062,29 +1110,32 @@ def test_hasattr_predict_proba():
     # `probability` param
 
     G = svm.SVC(probability=True)
-    assert hasattr(G, 'predict_proba')
+    assert hasattr(G, "predict_proba")
     G.fit(iris.data, iris.target)
-    assert hasattr(G, 'predict_proba')
+    assert hasattr(G, "predict_proba")
 
     G = svm.SVC(probability=False)
-    assert not hasattr(G, 'predict_proba')
+    assert not hasattr(G, "predict_proba")
     G.fit(iris.data, iris.target)
-    assert not hasattr(G, 'predict_proba')
+    assert not hasattr(G, "predict_proba")
 
     # Switching to `probability=True` after fitting should make
     # predict_proba available, but calling it must not work:
     G.probability = True
-    assert hasattr(G, 'predict_proba')
+    assert hasattr(G, "predict_proba")
     msg = "predict_proba is not available when fitted with probability=False"
-    assert_raise_message(NotFittedError, msg, G.predict_proba, iris.data)
+
+    with pytest.raises(NotFittedError, match=msg):
+        G.predict_proba(iris.data)
 
 
 def test_decision_function_shape_two_class():
     for n_classes in [2, 3]:
         X, y = make_blobs(centers=n_classes, random_state=0)
         for estimator in [svm.SVC, svm.NuSVC]:
-            clf = OneVsRestClassifier(
-                estimator(decision_function_shape="ovr")).fit(X, y)
+            clf = OneVsRestClassifier(estimator(decision_function_shape="ovr")).fit(
+                X, y
+            )
             assert len(clf.predict(X)) == len(y)
 
 
@@ -1097,16 +1148,18 @@ def test_ovr_decision_function():
     base_points = np.array([[5, 5], [10, 10]])
 
     # For all the quadrants (classes)
-    X_test = np.vstack((
-        base_points * [1, 1],    # Q1
-        base_points * [-1, 1],   # Q2
-        base_points * [-1, -1],  # Q3
-        base_points * [1, -1]    # Q4
-    ))
+    X_test = np.vstack(
+        (
+            base_points * [1, 1],  # Q1
+            base_points * [-1, 1],  # Q2
+            base_points * [-1, -1],  # Q3
+            base_points * [1, -1],  # Q4
+        )
+    )
 
     y_test = [0] * 2 + [1] * 2 + [2] * 2 + [3] * 2
 
-    clf = svm.SVC(kernel='linear', decision_function_shape='ovr')
+    clf = svm.SVC(kernel="linear", decision_function_shape="ovr")
     clf.fit(X_train, y_train)
 
     y_pred = clf.predict(X_test)
@@ -1134,8 +1187,9 @@ def test_ovr_decision_function():
 def test_svc_invalid_break_ties_param(SVCClass):
     X, y = make_blobs(random_state=42)
 
-    svm = SVCClass(kernel="linear", decision_function_shape='ovo',
-                   break_ties=True, random_state=42).fit(X, y)
+    svm = SVCClass(
+        kernel="linear", decision_function_shape="ovo", break_ties=True, random_state=42
+    ).fit(X, y)
 
     with pytest.raises(ValueError, match="break_ties must be False"):
         svm.predict(y)
@@ -1146,76 +1200,89 @@ def test_svc_ovr_tie_breaking(SVCClass):
     """Test if predict breaks ties in OVR mode.
     Related issue: https://github.com/scikit-learn/scikit-learn/issues/8277
     """
-    X, y = make_blobs(random_state=27)
+    if SVCClass.__name__ == "NuSVC" and _IS_32BIT:
+        # XXX: known failure to be investigated. Either the code needs to be
+        # fixed or the test itself might need to be made less sensitive to
+        # random changes in test data and rounding errors more generally.
+        # https://github.com/scikit-learn/scikit-learn/issues/29633
+        pytest.xfail("Failing test on 32bit OS")
+
+    X, y = make_blobs(random_state=0, n_samples=20, n_features=2)
 
-    xs = np.linspace(X[:, 0].min(), X[:, 0].max(), 1000)
-    ys = np.linspace(X[:, 1].min(), X[:, 1].max(), 1000)
+    xs = np.linspace(X[:, 0].min(), X[:, 0].max(), 100)
+    ys = np.linspace(X[:, 1].min(), X[:, 1].max(), 100)
     xx, yy = np.meshgrid(xs, ys)
 
-    svm = SVCClass(kernel="linear", decision_function_shape='ovr',
-                   break_ties=False, random_state=42).fit(X, y)
+    common_params = dict(
+        kernel="rbf", gamma=1e6, random_state=42, decision_function_shape="ovr"
+    )
+    svm = SVCClass(
+        break_ties=False,
+        **common_params,
+    ).fit(X, y)
     pred = svm.predict(np.c_[xx.ravel(), yy.ravel()])
     dv = svm.decision_function(np.c_[xx.ravel(), yy.ravel()])
     assert not np.all(pred == np.argmax(dv, axis=1))
 
-    svm = SVCClass(kernel="linear", decision_function_shape='ovr',
-                   break_ties=True, random_state=42).fit(X, y)
+    svm = SVCClass(
+        break_ties=True,
+        **common_params,
+    ).fit(X, y)
     pred = svm.predict(np.c_[xx.ravel(), yy.ravel()])
     dv = svm.decision_function(np.c_[xx.ravel(), yy.ravel()])
     assert np.all(pred == np.argmax(dv, axis=1))
 
 
-def test_gamma_auto():
-    X, y = [[0.0, 1.2], [1.0, 1.3]], [0, 1]
-
-    with pytest.warns(None) as record:
-        svm.SVC(kernel='linear').fit(X, y)
-    assert not len(record)
-
-    with pytest.warns(None) as record:
-        svm.SVC(kernel='precomputed').fit(X, y)
-    assert not len(record)
-
-
 def test_gamma_scale():
-    X, y = [[0.], [1.]], [0, 1]
+    X, y = [[0.0], [1.0]], [0, 1]
 
     clf = svm.SVC()
-    with pytest.warns(None) as record:
-        clf.fit(X, y)
-    assert not len(record)
+    clf.fit(X, y)
     assert_almost_equal(clf._gamma, 4)
 
-    # X_var ~= 1 shouldn't raise warning, for when
-    # gamma is not explicitly set.
-    X, y = [[1, 2], [3, 2 * np.sqrt(6) / 3 + 2]], [0, 1]
-    with pytest.warns(None) as record:
-        clf.fit(X, y)
-    assert not len(record)
-
 
 @pytest.mark.parametrize(
     "SVM, params",
-    [(LinearSVC, {'penalty': 'l1', 'loss': 'squared_hinge', 'dual': False}),
-     (LinearSVC, {'penalty': 'l2', 'loss': 'squared_hinge', 'dual': True}),
-     (LinearSVC, {'penalty': 'l2', 'loss': 'squared_hinge', 'dual': False}),
-     (LinearSVC, {'penalty': 'l2', 'loss': 'hinge', 'dual': True}),
-     (LinearSVR, {'loss': 'epsilon_insensitive', 'dual': True}),
-     (LinearSVR, {'loss': 'squared_epsilon_insensitive', 'dual': True}),
-     (LinearSVR, {'loss': 'squared_epsilon_insensitive', 'dual': True})]
+    [
+        (LinearSVC, {"penalty": "l1", "loss": "squared_hinge", "dual": False}),
+        (LinearSVC, {"penalty": "l2", "loss": "squared_hinge", "dual": True}),
+        (LinearSVC, {"penalty": "l2", "loss": "squared_hinge", "dual": False}),
+        (LinearSVC, {"penalty": "l2", "loss": "hinge", "dual": True}),
+        (LinearSVR, {"loss": "epsilon_insensitive", "dual": True}),
+        (LinearSVR, {"loss": "squared_epsilon_insensitive", "dual": True}),
+        (LinearSVR, {"loss": "squared_epsilon_insensitive", "dual": True}),
+    ],
 )
 def test_linearsvm_liblinear_sample_weight(SVM, params):
-    X = np.array([[1, 3], [1, 3], [1, 3], [1, 3],
-                  [2, 1], [2, 1], [2, 1], [2, 1],
-                  [3, 3], [3, 3], [3, 3], [3, 3],
-                  [4, 1], [4, 1], [4, 1], [4, 1]], dtype=np.dtype('float'))
-    y = np.array([1, 1, 1, 1, 2, 2, 2, 2,
-                  1, 1, 1, 1, 2, 2, 2, 2], dtype=np.dtype('int'))
+    X = np.array(
+        [
+            [1, 3],
+            [1, 3],
+            [1, 3],
+            [1, 3],
+            [2, 1],
+            [2, 1],
+            [2, 1],
+            [2, 1],
+            [3, 3],
+            [3, 3],
+            [3, 3],
+            [3, 3],
+            [4, 1],
+            [4, 1],
+            [4, 1],
+            [4, 1],
+        ],
+        dtype=np.dtype("float"),
+    )
+    y = np.array(
+        [1, 1, 1, 1, 2, 2, 2, 2, 1, 1, 1, 1, 2, 2, 2, 2], dtype=np.dtype("int")
+    )
 
     X2 = np.vstack([X, X])
     y2 = np.hstack([y, 3 - y])
     sample_weight = np.ones(shape=len(y) * 2)
-    sample_weight[len(y):] = 0
+    sample_weight[len(y) :] = 0
     X2, y2, sample_weight = shuffle(X2, y2, sample_weight, random_state=0)
 
     base_estimator = SVM(random_state=42)
@@ -1233,38 +1300,18 @@ def test_linearsvm_liblinear_sample_weight(SVM, params):
             assert_allclose(X_est_no_weight, X_est_with_weight)
 
 
-def test_n_support_oneclass_svr():
+@pytest.mark.parametrize("Klass", (OneClassSVM, SVR, NuSVR))
+def test_n_support(Klass):
     # Make n_support is correct for oneclass and SVR (used to be
     # non-initialized)
     # this is a non regression test for issue #14774
     X = np.array([[0], [0.44], [0.45], [0.46], [1]])
-    clf = svm.OneClassSVM()
-    assert not hasattr(clf, 'n_support_')
-    clf.fit(X)
-    assert clf.n_support_ == clf.support_vectors_.shape[0]
-    assert clf.n_support_.size == 1
-    assert clf.n_support_ == 3
-
     y = np.arange(X.shape[0])
-    reg = svm.SVR().fit(X, y)
-    assert reg.n_support_ == reg.support_vectors_.shape[0]
-    assert reg.n_support_.size == 1
-    assert reg.n_support_ == 4
-
-
-# TODO: Remove in 1.0 when probA_ and probB_ are deprecated
-@pytest.mark.parametrize("SVMClass, data", [
-    (svm.OneClassSVM, (X, )),
-    (svm.SVR, (X, Y))
-])
-@pytest.mark.parametrize("deprecated_prob", ["probA_", "probB_"])
-def test_svm_probA_proB_deprecated(SVMClass, data, deprecated_prob):
-    clf = SVMClass().fit(*data)
-
-    msg = ("The {} attribute is deprecated in version 0.23 and will be "
-           "removed in version 1.0").format(deprecated_prob)
-    with pytest.warns(FutureWarning, match=msg):
-        getattr(clf, deprecated_prob)
+    est = Klass()
+    assert not hasattr(est, "n_support_")
+    est.fit(X, y)
+    assert est.n_support_[0] == est.support_vectors_.shape[0]
+    assert est.n_support_.size == 1
 
 
 @pytest.mark.parametrize("Estimator", [svm.SVC, svm.SVR])
@@ -1281,8 +1328,8 @@ def string_kernel(X1, X2):
         K = np.zeros((n_samples1, n_samples2))
         for ii in range(n_samples1):
             for jj in range(ii, n_samples2):
-                K[ii, jj] = X1[ii].count('A') * X2[jj].count('A')
-                K[ii, jj] += X1[ii].count('B') * X2[jj].count('B')
+                K[ii, jj] = X1[ii].count("A") * X2[jj].count("A")
+                K[ii, jj] += X1[ii].count("B") * X2[jj].count("B")
                 K[jj, ii] = K[ii, jj]
         return K
 
@@ -1290,18 +1337,104 @@ def string_kernel(X1, X2):
     assert_array_equal(np.dot(X, X.T), K)
 
     svc1 = Estimator(kernel=string_kernel).fit(data, y)
-    svc2 = Estimator(kernel='linear').fit(X, y)
-    svc3 = Estimator(kernel='precomputed').fit(K, y)
+    svc2 = Estimator(kernel="linear").fit(X, y)
+    svc3 = Estimator(kernel="precomputed").fit(K, y)
 
     assert svc1.score(data, y) == svc3.score(K, y)
     assert svc1.score(data, y) == svc2.score(X, y)
-    if hasattr(svc1, 'decision_function'):  # classifier
-        assert_allclose(svc1.decision_function(data),
-                        svc2.decision_function(X))
-        assert_allclose(svc1.decision_function(data),
-                        svc3.decision_function(K))
+    if hasattr(svc1, "decision_function"):  # classifier
+        assert_allclose(svc1.decision_function(data), svc2.decision_function(X))
+        assert_allclose(svc1.decision_function(data), svc3.decision_function(K))
         assert_array_equal(svc1.predict(data), svc2.predict(X))
         assert_array_equal(svc1.predict(data), svc3.predict(K))
     else:  # regressor
         assert_allclose(svc1.predict(data), svc2.predict(X))
         assert_allclose(svc1.predict(data), svc3.predict(K))
+
+
+def test_svc_raises_error_internal_representation():
+    """Check that SVC raises error when internal representation is altered.
+
+    Non-regression test for #18891 and https://nvd.nist.gov/vuln/detail/CVE-2020-28975
+    """
+    clf = svm.SVC(kernel="linear").fit(X, Y)
+    clf._n_support[0] = 1000000
+
+    msg = "The internal representation of SVC was altered"
+    with pytest.raises(ValueError, match=msg):
+        clf.predict(X)
+
+
+@pytest.mark.parametrize(
+    "estimator, expected_n_iter_type",
+    [
+        (svm.SVC, np.ndarray),
+        (svm.NuSVC, np.ndarray),
+        (svm.SVR, int),
+        (svm.NuSVR, int),
+        (svm.OneClassSVM, int),
+    ],
+)
+@pytest.mark.parametrize(
+    "dataset",
+    [
+        make_classification(n_classes=2, n_informative=2, random_state=0),
+        make_classification(n_classes=3, n_informative=3, random_state=0),
+        make_classification(n_classes=4, n_informative=4, random_state=0),
+    ],
+)
+def test_n_iter_libsvm(estimator, expected_n_iter_type, dataset):
+    # Check that the type of n_iter_ is correct for the classes that inherit
+    # from BaseSVC.
+    # Note that for SVC, and NuSVC this is an ndarray; while for SVR, NuSVR, and
+    # OneClassSVM, it is an int.
+    # For SVC and NuSVC also check the shape of n_iter_.
+    X, y = dataset
+    n_iter = estimator(kernel="linear").fit(X, y).n_iter_
+    assert type(n_iter) == expected_n_iter_type
+    if estimator in [svm.SVC, svm.NuSVC]:
+        n_classes = len(np.unique(y))
+        assert n_iter.shape == (n_classes * (n_classes - 1) // 2,)
+
+
+@pytest.mark.parametrize("loss", ["squared_hinge", "squared_epsilon_insensitive"])
+def test_dual_auto(loss):
+    # OvR, L2, N > M (6,2)
+    dual = _validate_dual_parameter("auto", loss, "l2", "ovr", np.asarray(X))
+    assert dual is False
+    # OvR, L2, N < M (2,6)
+    dual = _validate_dual_parameter("auto", loss, "l2", "ovr", np.asarray(X).T)
+    assert dual is True
+
+
+def test_dual_auto_edge_cases():
+    # Hinge, OvR, L2, N > M (6,2)
+    dual = _validate_dual_parameter("auto", "hinge", "l2", "ovr", np.asarray(X))
+    assert dual is True  # only supports True
+    dual = _validate_dual_parameter(
+        "auto", "epsilon_insensitive", "l2", "ovr", np.asarray(X)
+    )
+    assert dual is True  # only supports True
+    # SqHinge, OvR, L1, N < M (2,6)
+    dual = _validate_dual_parameter(
+        "auto", "squared_hinge", "l1", "ovr", np.asarray(X).T
+    )
+    assert dual is False  # only supports False
+
+
+@pytest.mark.parametrize(
+    "Estimator, make_dataset",
+    [(svm.SVC, make_classification), (svm.SVR, make_regression)],
+)
+@pytest.mark.parametrize("C_inf", [np.inf, float("inf")])
+def test_svm_with_infinite_C(Estimator, make_dataset, C_inf, global_random_seed):
+    """Check that we can pass `C=inf` that is equivalent to a very large C value.
+
+    Non-regression test for
+    https://github.com/scikit-learn/scikit-learn/issues/29772
+    """
+    X, y = make_dataset(random_state=global_random_seed)
+    estimator_C_inf = Estimator(C=C_inf).fit(X, y)
+    estimator_C_large = Estimator(C=1e10).fit(X, y)
+
+    assert_allclose(estimator_C_large.predict(X), estimator_C_inf.predict(X))
diff --git a/sklearn/tests/metadata_routing_common.py b/sklearn/tests/metadata_routing_common.py
new file mode 100644
index 0000000000000..f4dd79581db90
--- /dev/null
+++ b/sklearn/tests/metadata_routing_common.py
@@ -0,0 +1,584 @@
+import inspect
+from collections import defaultdict
+from functools import partial
+
+import numpy as np
+from numpy.testing import assert_array_equal
+
+from sklearn.base import (
+    BaseEstimator,
+    ClassifierMixin,
+    MetaEstimatorMixin,
+    RegressorMixin,
+    TransformerMixin,
+    clone,
+)
+from sklearn.metrics._scorer import _Scorer, mean_squared_error
+from sklearn.model_selection import BaseCrossValidator
+from sklearn.model_selection._split import GroupsConsumerMixin
+from sklearn.utils._metadata_requests import (
+    SIMPLE_METHODS,
+)
+from sklearn.utils.metadata_routing import (
+    MetadataRouter,
+    MethodMapping,
+    process_routing,
+)
+from sklearn.utils.multiclass import _check_partial_fit_first_call
+
+
+def record_metadata(obj, record_default=True, **kwargs):
+    """Utility function to store passed metadata to a method of obj.
+
+    If record_default is False, kwargs whose values are "default" are skipped.
+    This is so that checks on keyword arguments whose default was not changed
+    are skipped.
+
+    """
+    stack = inspect.stack()
+    callee = stack[1].function
+    caller = stack[2].function
+    if not hasattr(obj, "_records"):
+        obj._records = defaultdict(lambda: defaultdict(list))
+    if not record_default:
+        kwargs = {
+            key: val
+            for key, val in kwargs.items()
+            if not isinstance(val, str) or (val != "default")
+        }
+    obj._records[callee][caller].append(kwargs)
+
+
+def check_recorded_metadata(obj, method, parent, split_params=tuple(), **kwargs):
+    """Check whether the expected metadata is passed to the object's method.
+
+    Parameters
+    ----------
+    obj : estimator object
+        sub-estimator to check routed params for
+    method : str
+        sub-estimator's method where metadata is routed to, or otherwise in
+        the context of metadata routing referred to as 'callee'
+    parent : str
+        the parent method which should have called `method`, or otherwise in
+        the context of metadata routing referred to as 'caller'
+    split_params : tuple, default=empty
+        specifies any parameters which are to be checked as being a subset
+        of the original values
+    **kwargs : dict
+        passed metadata
+    """
+    all_records = (
+        getattr(obj, "_records", dict()).get(method, dict()).get(parent, list())
+    )
+    for record in all_records:
+        # first check that the names of the metadata passed are the same as
+        # expected. The names are stored as keys in `record`.
+        assert set(kwargs.keys()) == set(record.keys()), (
+            f"Expected {kwargs.keys()} vs {record.keys()}"
+        )
+        for key, value in kwargs.items():
+            recorded_value = record[key]
+            # The following condition is used to check for any specified parameters
+            # being a subset of the original values
+            if key in split_params and recorded_value is not None:
+                assert np.isin(recorded_value, value).all()
+            else:
+                if isinstance(recorded_value, np.ndarray):
+                    assert_array_equal(recorded_value, value)
+                else:
+                    assert recorded_value is value, (
+                        f"Expected {recorded_value} vs {value}. Method: {method}"
+                    )
+
+
+record_metadata_not_default = partial(record_metadata, record_default=False)
+
+
+def assert_request_is_empty(metadata_request, exclude=None):
+    """Check if a metadata request dict is empty.
+
+    One can exclude a method or a list of methods from the check using the
+    ``exclude`` parameter. If metadata_request is a MetadataRouter, then
+    ``exclude`` can be of the form ``{"object" : [method, ...]}``.
+    """
+    if isinstance(metadata_request, MetadataRouter):
+        for name, route_mapping in metadata_request:
+            if exclude is not None and name in exclude:
+                _exclude = exclude[name]
+            else:
+                _exclude = None
+            assert_request_is_empty(route_mapping.router, exclude=_exclude)
+        return
+
+    exclude = [] if exclude is None else exclude
+    for method in SIMPLE_METHODS:
+        if method in exclude:
+            continue
+        mmr = getattr(metadata_request, method)
+        props = [
+            prop
+            for prop, alias in mmr.requests.items()
+            if isinstance(alias, str) or alias is not None
+        ]
+        assert not props
+
+
+def assert_request_equal(request, dictionary):
+    for method, requests in dictionary.items():
+        mmr = getattr(request, method)
+        assert mmr.requests == requests
+
+    empty_methods = [method for method in SIMPLE_METHODS if method not in dictionary]
+    for method in empty_methods:
+        assert not len(getattr(request, method).requests)
+
+
+class _Registry(list):
+    # This list is used to get a reference to the sub-estimators, which are not
+    # necessarily stored on the metaestimator. We need to override __deepcopy__
+    # because the sub-estimators are probably cloned, which would result in a
+    # new copy of the list, but we need copy and deep copy both to return the
+    # same instance.
+    def __deepcopy__(self, memo):
+        return self
+
+    def __copy__(self):
+        return self
+
+
+class ConsumingRegressor(RegressorMixin, BaseEstimator):
+    """A regressor consuming metadata.
+
+    Parameters
+    ----------
+    registry : list, default=None
+        If a list, the estimator will append itself to the list in order to have
+        a reference to the estimator later on. Since that reference is not
+        required in all tests, registration can be skipped by leaving this value
+        as None.
+    """
+
+    def __init__(self, registry=None):
+        self.registry = registry
+
+    def partial_fit(self, X, y, sample_weight="default", metadata="default"):
+        if self.registry is not None:
+            self.registry.append(self)
+
+        record_metadata_not_default(
+            self, sample_weight=sample_weight, metadata=metadata
+        )
+        return self
+
+    def fit(self, X, y, sample_weight="default", metadata="default"):
+        if self.registry is not None:
+            self.registry.append(self)
+
+        record_metadata_not_default(
+            self, sample_weight=sample_weight, metadata=metadata
+        )
+        return self
+
+    def predict(self, X, y=None, sample_weight="default", metadata="default"):
+        record_metadata_not_default(
+            self, sample_weight=sample_weight, metadata=metadata
+        )
+        return np.zeros(shape=(len(X),))
+
+    def score(self, X, y, sample_weight="default", metadata="default"):
+        record_metadata_not_default(
+            self, sample_weight=sample_weight, metadata=metadata
+        )
+        return 1
+
+
+class NonConsumingClassifier(ClassifierMixin, BaseEstimator):
+    """A classifier which accepts no metadata on any method."""
+
+    def __init__(self, alpha=0.0):
+        self.alpha = alpha
+
+    def fit(self, X, y):
+        self.classes_ = np.unique(y)
+        self.coef_ = np.ones_like(X)
+        return self
+
+    def partial_fit(self, X, y, classes=None):
+        return self
+
+    def decision_function(self, X):
+        return self.predict(X)
+
+    def predict(self, X):
+        y_pred = np.empty(shape=(len(X),))
+        y_pred[: len(X) // 2] = 0
+        y_pred[len(X) // 2 :] = 1
+        return y_pred
+
+    def predict_proba(self, X):
+        # dummy probabilities to support predict_proba
+        y_proba = np.empty(shape=(len(X), len(self.classes_)), dtype=np.float32)
+        # each row sums up to 1.0:
+        y_proba[:] = np.random.dirichlet(alpha=np.ones(len(self.classes_)), size=len(X))
+        return y_proba
+
+    def predict_log_proba(self, X):
+        # dummy probabilities to support predict_log_proba
+        return self.predict_proba(X)
+
+
+class NonConsumingRegressor(RegressorMixin, BaseEstimator):
+    """A classifier which accepts no metadata on any method."""
+
+    def fit(self, X, y):
+        return self
+
+    def partial_fit(self, X, y):
+        return self
+
+    def predict(self, X):
+        return np.ones(len(X))  # pragma: no cover
+
+
+class ConsumingClassifier(ClassifierMixin, BaseEstimator):
+    """A classifier consuming metadata.
+
+    Parameters
+    ----------
+    registry : list, default=None
+        If a list, the estimator will append itself to the list in order to have
+        a reference to the estimator later on. Since that reference is not
+        required in all tests, registration can be skipped by leaving this value
+        as None.
+
+    alpha : float, default=0
+        This parameter is only used to test the ``*SearchCV`` objects, and
+        doesn't do anything.
+    """
+
+    def __init__(self, registry=None, alpha=0.0):
+        self.alpha = alpha
+        self.registry = registry
+
+    def partial_fit(
+        self, X, y, classes=None, sample_weight="default", metadata="default"
+    ):
+        if self.registry is not None:
+            self.registry.append(self)
+
+        record_metadata_not_default(
+            self, sample_weight=sample_weight, metadata=metadata
+        )
+        _check_partial_fit_first_call(self, classes)
+        return self
+
+    def fit(self, X, y, sample_weight="default", metadata="default"):
+        if self.registry is not None:
+            self.registry.append(self)
+
+        record_metadata_not_default(
+            self, sample_weight=sample_weight, metadata=metadata
+        )
+
+        self.classes_ = np.unique(y)
+        self.coef_ = np.ones_like(X)
+        return self
+
+    def predict(self, X, sample_weight="default", metadata="default"):
+        record_metadata_not_default(
+            self, sample_weight=sample_weight, metadata=metadata
+        )
+        y_score = np.empty(shape=(len(X),), dtype="int8")
+        y_score[len(X) // 2 :] = 0
+        y_score[: len(X) // 2] = 1
+        return y_score
+
+    def predict_proba(self, X, sample_weight="default", metadata="default"):
+        record_metadata_not_default(
+            self, sample_weight=sample_weight, metadata=metadata
+        )
+        y_proba = np.empty(shape=(len(X), len(self.classes_)), dtype=np.float32)
+        # each row sums up to 1.0:
+        y_proba[:] = np.random.dirichlet(alpha=np.ones(len(self.classes_)), size=len(X))
+        return y_proba
+
+    def predict_log_proba(self, X, sample_weight="default", metadata="default"):
+        record_metadata_not_default(
+            self, sample_weight=sample_weight, metadata=metadata
+        )
+        return self.predict_proba(X)
+
+    def decision_function(self, X, sample_weight="default", metadata="default"):
+        record_metadata_not_default(
+            self, sample_weight=sample_weight, metadata=metadata
+        )
+        y_score = np.empty(shape=(len(X),))
+        y_score[len(X) // 2 :] = 0
+        y_score[: len(X) // 2] = 1
+        return y_score
+
+    def score(self, X, y, sample_weight="default", metadata="default"):
+        record_metadata_not_default(
+            self, sample_weight=sample_weight, metadata=metadata
+        )
+        return 1
+
+
+class ConsumingClassifierWithoutPredictProba(ConsumingClassifier):
+    """ConsumingClassifier without a predict_proba method, but with predict_log_proba.
+
+    Used to mimic dynamic method selection such as in the `_parallel_predict_proba()`
+    function called by `BaggingClassifier`.
+    """
+
+    @property
+    def predict_proba(self):
+        raise AttributeError("This estimator does not support predict_proba")
+
+
+class ConsumingClassifierWithoutPredictLogProba(ConsumingClassifier):
+    """ConsumingClassifier without a predict_log_proba method, but with predict_proba.
+
+    Used to mimic dynamic method selection such as in
+    `BaggingClassifier.predict_log_proba()`.
+    """
+
+    @property
+    def predict_log_proba(self):
+        raise AttributeError("This estimator does not support predict_log_proba")
+
+
+class ConsumingClassifierWithOnlyPredict(ConsumingClassifier):
+    """ConsumingClassifier with only a predict method.
+
+    Used to mimic dynamic method selection such as in
+    `BaggingClassifier.predict_log_proba()`.
+    """
+
+    @property
+    def predict_proba(self):
+        raise AttributeError("This estimator does not support predict_proba")
+
+    @property
+    def predict_log_proba(self):
+        raise AttributeError("This estimator does not support predict_log_proba")
+
+
+class ConsumingTransformer(TransformerMixin, BaseEstimator):
+    """A transformer which accepts metadata on fit and transform.
+
+    Parameters
+    ----------
+    registry : list, default=None
+        If a list, the estimator will append itself to the list in order to have
+        a reference to the estimator later on. Since that reference is not
+        required in all tests, registration can be skipped by leaving this value
+        as None.
+    """
+
+    def __init__(self, registry=None):
+        self.registry = registry
+
+    def fit(self, X, y=None, sample_weight="default", metadata="default"):
+        if self.registry is not None:
+            self.registry.append(self)
+
+        record_metadata_not_default(
+            self, sample_weight=sample_weight, metadata=metadata
+        )
+        self.fitted_ = True
+        return self
+
+    def transform(self, X, sample_weight="default", metadata="default"):
+        record_metadata_not_default(
+            self, sample_weight=sample_weight, metadata=metadata
+        )
+        return X + 1
+
+    def fit_transform(self, X, y, sample_weight="default", metadata="default"):
+        # implementing ``fit_transform`` is necessary since
+        # ``TransformerMixin.fit_transform`` doesn't route any metadata to
+        # ``transform``, while here we want ``transform`` to receive
+        # ``sample_weight`` and ``metadata``.
+        record_metadata_not_default(
+            self, sample_weight=sample_weight, metadata=metadata
+        )
+        return self.fit(X, y, sample_weight=sample_weight, metadata=metadata).transform(
+            X, sample_weight=sample_weight, metadata=metadata
+        )
+
+    def inverse_transform(self, X, sample_weight=None, metadata=None):
+        record_metadata_not_default(
+            self, sample_weight=sample_weight, metadata=metadata
+        )
+        return X - 1
+
+
+class ConsumingNoFitTransformTransformer(BaseEstimator):
+    """A metadata consuming transformer that doesn't inherit from
+    TransformerMixin, and thus doesn't implement `fit_transform`. Note that
+    TransformerMixin's `fit_transform` doesn't route metadata to `transform`."""
+
+    def __init__(self, registry=None):
+        self.registry = registry
+
+    def fit(self, X, y=None, sample_weight=None, metadata=None):
+        if self.registry is not None:
+            self.registry.append(self)
+
+        record_metadata(self, sample_weight=sample_weight, metadata=metadata)
+
+        return self
+
+    def transform(self, X, sample_weight=None, metadata=None):
+        record_metadata(self, sample_weight=sample_weight, metadata=metadata)
+        return X
+
+
+class ConsumingScorer(_Scorer):
+    def __init__(self, registry=None):
+        super().__init__(
+            score_func=mean_squared_error, sign=1, kwargs={}, response_method="predict"
+        )
+        self.registry = registry
+
+    def _score(self, method_caller, clf, X, y, **kwargs):
+        if self.registry is not None:
+            self.registry.append(self)
+
+        record_metadata_not_default(self, **kwargs)
+
+        sample_weight = kwargs.get("sample_weight", None)
+        return super()._score(method_caller, clf, X, y, sample_weight=sample_weight)
+
+
+class ConsumingSplitter(GroupsConsumerMixin, BaseCrossValidator):
+    def __init__(self, registry=None):
+        self.registry = registry
+
+    def split(self, X, y=None, groups="default", metadata="default"):
+        if self.registry is not None:
+            self.registry.append(self)
+
+        record_metadata_not_default(self, groups=groups, metadata=metadata)
+
+        split_index = len(X) // 2
+        train_indices = list(range(0, split_index))
+        test_indices = list(range(split_index, len(X)))
+        yield test_indices, train_indices
+        yield train_indices, test_indices
+
+    def get_n_splits(self, X=None, y=None, groups=None, metadata=None):
+        return 2
+
+    def _iter_test_indices(self, X=None, y=None, groups=None):
+        split_index = len(X) // 2
+        train_indices = list(range(0, split_index))
+        test_indices = list(range(split_index, len(X)))
+        yield test_indices
+        yield train_indices
+
+
+class MetaRegressor(MetaEstimatorMixin, RegressorMixin, BaseEstimator):
+    """A meta-regressor which is only a router."""
+
+    def __init__(self, estimator):
+        self.estimator = estimator
+
+    def fit(self, X, y, **fit_params):
+        params = process_routing(self, "fit", **fit_params)
+        self.estimator_ = clone(self.estimator).fit(X, y, **params.estimator.fit)
+
+    def get_metadata_routing(self):
+        router = MetadataRouter(owner=self.__class__.__name__).add(
+            estimator=self.estimator,
+            method_mapping=MethodMapping().add(caller="fit", callee="fit"),
+        )
+        return router
+
+
+class WeightedMetaRegressor(MetaEstimatorMixin, RegressorMixin, BaseEstimator):
+    """A meta-regressor which is also a consumer."""
+
+    def __init__(self, estimator, registry=None):
+        self.estimator = estimator
+        self.registry = registry
+
+    def fit(self, X, y, sample_weight=None, **fit_params):
+        if self.registry is not None:
+            self.registry.append(self)
+
+        record_metadata(self, sample_weight=sample_weight)
+        params = process_routing(self, "fit", sample_weight=sample_weight, **fit_params)
+        self.estimator_ = clone(self.estimator).fit(X, y, **params.estimator.fit)
+        return self
+
+    def predict(self, X, **predict_params):
+        params = process_routing(self, "predict", **predict_params)
+        return self.estimator_.predict(X, **params.estimator.predict)
+
+    def get_metadata_routing(self):
+        router = (
+            MetadataRouter(owner=self.__class__.__name__)
+            .add_self_request(self)
+            .add(
+                estimator=self.estimator,
+                method_mapping=MethodMapping()
+                .add(caller="fit", callee="fit")
+                .add(caller="predict", callee="predict"),
+            )
+        )
+        return router
+
+
+class WeightedMetaClassifier(MetaEstimatorMixin, ClassifierMixin, BaseEstimator):
+    """A meta-estimator which also consumes sample_weight itself in ``fit``."""
+
+    def __init__(self, estimator, registry=None):
+        self.estimator = estimator
+        self.registry = registry
+
+    def fit(self, X, y, sample_weight=None, **kwargs):
+        if self.registry is not None:
+            self.registry.append(self)
+
+        record_metadata(self, sample_weight=sample_weight)
+        params = process_routing(self, "fit", sample_weight=sample_weight, **kwargs)
+        self.estimator_ = clone(self.estimator).fit(X, y, **params.estimator.fit)
+        return self
+
+    def get_metadata_routing(self):
+        router = (
+            MetadataRouter(owner=self.__class__.__name__)
+            .add_self_request(self)
+            .add(
+                estimator=self.estimator,
+                method_mapping=MethodMapping().add(caller="fit", callee="fit"),
+            )
+        )
+        return router
+
+
+class MetaTransformer(MetaEstimatorMixin, TransformerMixin, BaseEstimator):
+    """A simple meta-transformer."""
+
+    def __init__(self, transformer):
+        self.transformer = transformer
+
+    def fit(self, X, y=None, **fit_params):
+        params = process_routing(self, "fit", **fit_params)
+        self.transformer_ = clone(self.transformer).fit(X, y, **params.transformer.fit)
+        return self
+
+    def transform(self, X, y=None, **transform_params):
+        params = process_routing(self, "transform", **transform_params)
+        return self.transformer_.transform(X, **params.transformer.transform)
+
+    def get_metadata_routing(self):
+        return MetadataRouter(owner=self.__class__.__name__).add(
+            transformer=self.transformer,
+            method_mapping=MethodMapping()
+            .add(caller="fit", callee="fit")
+            .add(caller="transform", callee="transform"),
+        )
diff --git a/sklearn/tests/test_base.py b/sklearn/tests/test_base.py
index 666df1499d7dc..e57d36351f0d4 100644
--- a/sklearn/tests/test_base.py
+++ b/sklearn/tests/test_base.py
@@ -1,36 +1,48 @@
-# Author: Gael Varoquaux
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+import pickle
+import re
+import warnings
 
 import numpy as np
-import scipy.sparse as sp
 import pytest
+import scipy.sparse as sp
+from numpy.testing import assert_allclose
 
 import sklearn
-from sklearn.utils._testing import assert_array_equal
-from sklearn.utils._testing import assert_no_warnings
-from sklearn.utils._testing import assert_warns_message
-from sklearn.utils._testing import ignore_warnings
-
-from sklearn.base import BaseEstimator, clone, is_classifier, _is_pairwise
-from sklearn.svm import SVC
-from sklearn.pipeline import Pipeline
+from sklearn import config_context, datasets
+from sklearn.base import (
+    BaseEstimator,
+    OutlierMixin,
+    TransformerMixin,
+    clone,
+    is_classifier,
+    is_clusterer,
+    is_outlier_detector,
+    is_regressor,
+)
+from sklearn.cluster import KMeans
+from sklearn.decomposition import PCA
+from sklearn.ensemble import IsolationForest
+from sklearn.exceptions import InconsistentVersionWarning
 from sklearn.model_selection import GridSearchCV
-from sklearn.decomposition import KernelPCA
-
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.tree import DecisionTreeRegressor
-from sklearn import datasets
-
-from sklearn.base import TransformerMixin
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import StandardScaler
+from sklearn.svm import SVC, SVR
+from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
 from sklearn.utils._mocking import MockDataFrame
-from sklearn import config_context
-import pickle
+from sklearn.utils._set_output import _get_output_config
+from sklearn.utils._testing import (
+    _convert_container,
+    assert_array_equal,
+)
+from sklearn.utils.validation import _check_n_features, validate_data
 
 
 #############################################################################
 # A few test classes
 class MyEstimator(BaseEstimator):
-
     def __init__(self, l1=0, empty=None):
         self.l1 = l1
         self.empty = empty
@@ -49,23 +61,28 @@ def __init__(self, a=None, b=None):
 
 
 class NaNTag(BaseEstimator):
-    def _more_tags(self):
-        return {'allow_nan': True}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.allow_nan = True
+        return tags
 
 
 class NoNaNTag(BaseEstimator):
-    def _more_tags(self):
-        return {'allow_nan': False}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.allow_nan = False
+        return tags
 
 
 class OverrideTag(NaNTag):
-    def _more_tags(self):
-        return {'allow_nan': False}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags.input_tags.allow_nan = False
+        return tags
 
 
 class DiamondOverwriteTag(NaNTag, NoNaNTag):
-    def _more_tags(self):
-        return dict()
+    pass
 
 
 class InheritDiamondOverwriteTag(DiamondOverwriteTag):
@@ -77,12 +94,13 @@ class ModifyInitParams(BaseEstimator):
     Equal parameters but with a type cast.
     Doesn't fulfill a is a
     """
+
     def __init__(self, a=np.array([0])):
         self.a = a.copy()
 
 
 class Buggy(BaseEstimator):
-    " A buggy estimator that does not set its parameters right. "
+    "A buggy estimator that does not set its parameters right."
 
     def __init__(self, a=None):
         self.a = 1
@@ -101,6 +119,7 @@ def predict(self, X=None):
 
 class VargEstimator(BaseEstimator):
     """scikit-learn estimators shouldn't have vargs."""
+
     def __init__(self, *vargs):
         pass
 
@@ -108,6 +127,7 @@ def __init__(self, *vargs):
 #############################################################################
 # The tests
 
+
 def test_clone():
     # Tests that clone creates a correct deep copy.
     # We create an estimator, make a copy of its original state
@@ -179,10 +199,19 @@ def test_clone_nan():
     assert clf.empty is clf2.empty
 
 
+def test_clone_dict():
+    # test that clone creates a clone of a dict
+    orig = {"a": MyEstimator()}
+    cloned = clone(orig)
+    assert orig["a"] is not cloned["a"]
+
+
 def test_clone_sparse_matrices():
     sparse_matrix_classes = [
-        getattr(sp, name)
-        for name in dir(sp) if name.endswith('_matrix')]
+        cls
+        for name in dir(sp)
+        if name.endswith("_matrix") and type(cls := getattr(sp, name)) is type
+    ]
 
     for cls in sparse_matrix_classes:
         sparse_matrix = cls(np.eye(5))
@@ -214,9 +243,7 @@ def test_repr():
     my_estimator = MyEstimator()
     repr(my_estimator)
     test = T(K(), K())
-    assert (
-        repr(test) ==
-        "T(a=K(), b=K())")
+    assert repr(test) == "T(a=K(), b=K())"
 
     some_est = T(a=["long_params"] * 1000)
     assert len(repr(some_est)) == 485
@@ -229,10 +256,10 @@ def test_str():
 
 
 def test_get_params():
-    test = T(K(), K())
+    test = T(K(), K)
 
-    assert 'a__d' in test.get_params(deep=True)
-    assert 'a__d' not in test.get_params(deep=False)
+    assert "a__d" in test.get_params(deep=True)
+    assert "a__d" not in test.get_params(deep=False)
 
     test.set_params(a__d=2)
     assert test.a.d == 2
@@ -241,13 +268,70 @@ def test_get_params():
         test.set_params(a__a=2)
 
 
-def test_is_classifier():
-    svc = SVC()
-    assert is_classifier(svc)
-    assert is_classifier(GridSearchCV(svc, {'C': [0.1, 1]}))
-    assert is_classifier(Pipeline([('svc', svc)]))
-    assert is_classifier(Pipeline(
-        [('svc_cv', GridSearchCV(svc, {'C': [0.1, 1]}))]))
+# TODO(1.8): Remove this test when the deprecation is removed
+def test_is_estimator_type_class():
+    with pytest.warns(FutureWarning, match="passing a class to.*is deprecated"):
+        assert is_classifier(SVC)
+
+    with pytest.warns(FutureWarning, match="passing a class to.*is deprecated"):
+        assert is_regressor(SVR)
+
+    with pytest.warns(FutureWarning, match="passing a class to.*is deprecated"):
+        assert is_clusterer(KMeans)
+
+    with pytest.warns(FutureWarning, match="passing a class to.*is deprecated"):
+        assert is_outlier_detector(IsolationForest)
+
+
+@pytest.mark.parametrize(
+    "estimator, expected_result",
+    [
+        (SVC(), True),
+        (GridSearchCV(SVC(), {"C": [0.1, 1]}), True),
+        (Pipeline([("svc", SVC())]), True),
+        (Pipeline([("svc_cv", GridSearchCV(SVC(), {"C": [0.1, 1]}))]), True),
+        (SVR(), False),
+        (GridSearchCV(SVR(), {"C": [0.1, 1]}), False),
+        (Pipeline([("svr", SVR())]), False),
+        (Pipeline([("svr_cv", GridSearchCV(SVR(), {"C": [0.1, 1]}))]), False),
+    ],
+)
+def test_is_classifier(estimator, expected_result):
+    assert is_classifier(estimator) == expected_result
+
+
+@pytest.mark.parametrize(
+    "estimator, expected_result",
+    [
+        (SVR(), True),
+        (GridSearchCV(SVR(), {"C": [0.1, 1]}), True),
+        (Pipeline([("svr", SVR())]), True),
+        (Pipeline([("svr_cv", GridSearchCV(SVR(), {"C": [0.1, 1]}))]), True),
+        (SVC(), False),
+        (GridSearchCV(SVC(), {"C": [0.1, 1]}), False),
+        (Pipeline([("svc", SVC())]), False),
+        (Pipeline([("svc_cv", GridSearchCV(SVC(), {"C": [0.1, 1]}))]), False),
+    ],
+)
+def test_is_regressor(estimator, expected_result):
+    assert is_regressor(estimator) == expected_result
+
+
+@pytest.mark.parametrize(
+    "estimator, expected_result",
+    [
+        (KMeans(), True),
+        (GridSearchCV(KMeans(), {"n_clusters": [3, 8]}), True),
+        (Pipeline([("km", KMeans())]), True),
+        (Pipeline([("km_cv", GridSearchCV(KMeans(), {"n_clusters": [3, 8]}))]), True),
+        (SVC(), False),
+        (GridSearchCV(SVC(), {"C": [0.1, 1]}), False),
+        (Pipeline([("svc", SVC())]), False),
+        (Pipeline([("svc_cv", GridSearchCV(SVC(), {"C": [0.1, 1]}))]), False),
+    ],
+)
+def test_is_clusterer(estimator, expected_result):
+    assert is_clusterer(estimator) == expected_result
 
 
 def test_set_params():
@@ -264,8 +348,8 @@ def test_set_params():
 
     # we don't currently catch if the things in pipeline are estimators
     # bad_pipeline = Pipeline([("bad", NoEstimator())])
-    # assert_raises(AttributeError, bad_pipeline.set_params,
-    #               bad__stupid_param=True)
+    # with pytest.raises(AttributeError):
+    #    bad_pipeline.set_params(bad__stupid_param=True)
 
 
 def test_set_params_passes_all_parameters():
@@ -279,11 +363,12 @@ def set_params(self, **kwargs):
             assert kwargs == expected_kwargs
             return self
 
-    expected_kwargs = {'max_depth': 5, 'min_samples_leaf': 2}
-    for est in [Pipeline([('estimator', TestDecisionTree())]),
-                GridSearchCV(TestDecisionTree(), {})]:
-        est.set_params(estimator__max_depth=5,
-                       estimator__min_samples_leaf=2)
+    expected_kwargs = {"max_depth": 5, "min_samples_leaf": 2}
+    for est in [
+        Pipeline([("estimator", TestDecisionTree())]),
+        GridSearchCV(TestDecisionTree(), {}),
+    ]:
+        est.set_params(estimator__max_depth=5, estimator__min_samples_leaf=2)
 
 
 def test_set_params_updates_valid_params():
@@ -294,30 +379,34 @@ def test_set_params_updates_valid_params():
     assert gscv.estimator.C == 42.0
 
 
-def test_score_sample_weight():
-
+@pytest.mark.parametrize(
+    "tree,dataset",
+    [
+        (
+            DecisionTreeClassifier(max_depth=2, random_state=0),
+            datasets.make_classification(random_state=0),
+        ),
+        (
+            DecisionTreeRegressor(max_depth=2, random_state=0),
+            datasets.make_regression(random_state=0),
+        ),
+    ],
+)
+def test_score_sample_weight(tree, dataset):
     rng = np.random.RandomState(0)
+    # check that the score with and without sample weights are different
+    X, y = dataset
 
-    # test both ClassifierMixin and RegressorMixin
-    estimators = [DecisionTreeClassifier(max_depth=2),
-                  DecisionTreeRegressor(max_depth=2)]
-    sets = [datasets.load_iris(),
-            datasets.load_boston()]
-
-    for est, ds in zip(estimators, sets):
-        est.fit(ds.data, ds.target)
-        # generate random sample weights
-        sample_weight = rng.randint(1, 10, size=len(ds.target))
-        # check that the score with and without sample weights are different
-        assert (est.score(ds.data, ds.target) !=
-                est.score(ds.data, ds.target,
-                          sample_weight=sample_weight)), (
-                              "Unweighted and weighted scores "
-                              "are unexpectedly equal")
+    tree.fit(X, y)
+    # generate random sample weights
+    sample_weight = rng.randint(1, 10, size=len(y))
+    score_unweighted = tree.score(X, y)
+    score_weighted = tree.score(X, y, sample_weight=sample_weight)
+    msg = "Unweighted and weighted scores are unexpectedly equal"
+    assert score_unweighted != score_weighted, msg
 
 
 def test_clone_pandas_dataframe():
-
     class DummyEstimator(TransformerMixin, BaseEstimator):
         """This is a dummy class for generating numerical features
 
@@ -333,6 +422,7 @@ class DummyEstimator(TransformerMixin, BaseEstimator):
         Notes
         -----
         """
+
         def __init__(self, df=None, scalar_param=1):
             self.df = df
             self.scalar_param = scalar_param
@@ -354,12 +444,59 @@ def transform(self, X):
     assert e.scalar_param == cloned_e.scalar_param
 
 
+def test_clone_protocol():
+    """Checks that clone works with `__sklearn_clone__` protocol."""
+
+    class FrozenEstimator(BaseEstimator):
+        def __init__(self, fitted_estimator):
+            self.fitted_estimator = fitted_estimator
+
+        def __getattr__(self, name):
+            return getattr(self.fitted_estimator, name)
+
+        def __sklearn_clone__(self):
+            return self
+
+        def fit(self, *args, **kwargs):
+            return self
+
+        def fit_transform(self, *args, **kwargs):
+            return self.fitted_estimator.transform(*args, **kwargs)
+
+    X = np.array([[-1, -1], [-2, -1], [-3, -2]])
+    pca = PCA().fit(X)
+    components = pca.components_
+
+    frozen_pca = FrozenEstimator(pca)
+    assert_allclose(frozen_pca.components_, components)
+
+    # Calling PCA methods such as `get_feature_names_out` still works
+    assert_array_equal(frozen_pca.get_feature_names_out(), pca.get_feature_names_out())
+
+    # Fitting on a new data does not alter `components_`
+    X_new = np.asarray([[-1, 2], [3, 4], [1, 2]])
+    frozen_pca.fit(X_new)
+    assert_allclose(frozen_pca.components_, components)
+
+    # `fit_transform` does not alter state
+    frozen_pca.fit_transform(X_new)
+    assert_allclose(frozen_pca.components_, components)
+
+    # Cloning estimator is a no-op
+    clone_frozen_pca = clone(frozen_pca)
+    assert clone_frozen_pca is frozen_pca
+    assert_allclose(clone_frozen_pca.components_, components)
+
+
 def test_pickle_version_warning_is_not_raised_with_matching_version():
     iris = datasets.load_iris()
     tree = DecisionTreeClassifier().fit(iris.data, iris.target)
     tree_pickle = pickle.dumps(tree)
-    assert b"version" in tree_pickle
-    tree_restored = assert_no_warnings(pickle.loads, tree_pickle)
+    assert b"_sklearn_version" in tree_pickle
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        tree_restored = pickle.loads(tree_pickle)
 
     # test that we can predict with the restored decision tree classifier
     score_of_original = tree.score(iris.data, iris.target)
@@ -377,17 +514,27 @@ def __getstate__(self):
     "version {old_version} when using version "
     "{current_version}. This might "
     "lead to breaking code or invalid results. "
-    "Use at your own risk.")
+    "Use at your own risk."
+)
 
 
 def test_pickle_version_warning_is_issued_upon_different_version():
     iris = datasets.load_iris()
     tree = TreeBadVersion().fit(iris.data, iris.target)
     tree_pickle_other = pickle.dumps(tree)
-    message = pickle_error_message.format(estimator="TreeBadVersion",
-                                          old_version="something",
-                                          current_version=sklearn.__version__)
-    assert_warns_message(UserWarning, message, pickle.loads, tree_pickle_other)
+    message = pickle_error_message.format(
+        estimator="TreeBadVersion",
+        old_version="something",
+        current_version=sklearn.__version__,
+    )
+    with pytest.warns(UserWarning, match=message) as warning_record:
+        pickle.loads(tree_pickle_other)
+
+    message = warning_record.list[0].message
+    assert isinstance(message, InconsistentVersionWarning)
+    assert message.estimator_name == "TreeBadVersion"
+    assert message.original_sklearn_version == "something"
+    assert message.current_sklearn_version == sklearn.__version__
 
 
 class TreeNoVersion(DecisionTreeClassifier):
@@ -401,13 +548,15 @@ def test_pickle_version_warning_is_issued_when_no_version_info_in_pickle():
     tree = TreeNoVersion().fit(iris.data, iris.target)
 
     tree_pickle_noversion = pickle.dumps(tree)
-    assert b"version" not in tree_pickle_noversion
-    message = pickle_error_message.format(estimator="TreeNoVersion",
-                                          old_version="pre-0.18",
-                                          current_version=sklearn.__version__)
+    assert b"_sklearn_version" not in tree_pickle_noversion
+    message = pickle_error_message.format(
+        estimator="TreeNoVersion",
+        old_version="pre-0.18",
+        current_version=sklearn.__version__,
+    )
     # check we got the warning about using pre-0.18 pickle
-    assert_warns_message(UserWarning, message, pickle.loads,
-                         tree_pickle_noversion)
+    with pytest.warns(UserWarning, match=message):
+        pickle.loads(tree_pickle_noversion)
 
 
 def test_pickle_version_no_warning_is_issued_with_non_sklearn_estimator():
@@ -417,7 +566,11 @@ def test_pickle_version_no_warning_is_issued_with_non_sklearn_estimator():
     try:
         module_backup = TreeNoVersion.__module__
         TreeNoVersion.__module__ = "notsklearn"
-        assert_no_warnings(pickle.loads, tree_pickle_noversion)
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("error")
+
+            pickle.loads(tree_pickle_noversion)
     finally:
         TreeNoVersion.__module__ = module_backup
 
@@ -459,10 +612,9 @@ def test_pickling_when_getstate_is_overwritten_by_mixin_outside_of_sklearn():
         type(estimator).__module__ = "notsklearn"
 
         serialized = estimator.__getstate__()
-        assert serialized == {'_attribute_not_pickled': None,
-                              'attribute_pickled': 5}
+        assert serialized == {"_attribute_not_pickled": None, "attribute_pickled": 5}
 
-        serialized['attribute_pickled'] = 4
+        serialized["attribute_pickled"] = 4
         estimator.__setstate__(serialized)
         assert estimator.attribute_pickled == 4
         assert estimator._restored
@@ -476,12 +628,11 @@ def __init__(self, attribute_pickled=5):
         self._attribute_not_pickled = None
 
     def __getstate__(self):
-        data = self.__dict__.copy()
-        data["_attribute_not_pickled"] = None
-        return data
+        state = super().__getstate__()
+        state["_attribute_not_pickled"] = None
+        return state
 
 
-@ignore_warnings(category=(UserWarning))
 def test_pickling_works_when_getstate_is_overwritten_in_the_child_class():
     estimator = SingleInheritanceEstimator()
     estimator._attribute_not_pickled = "this attribute should not be pickled"
@@ -497,17 +648,17 @@ def test_tag_inheritance():
 
     nan_tag_est = NaNTag()
     no_nan_tag_est = NoNaNTag()
-    assert nan_tag_est._get_tags()['allow_nan']
-    assert not no_nan_tag_est._get_tags()['allow_nan']
+    assert nan_tag_est.__sklearn_tags__().input_tags.allow_nan
+    assert not no_nan_tag_est.__sklearn_tags__().input_tags.allow_nan
 
     redefine_tags_est = OverrideTag()
-    assert not redefine_tags_est._get_tags()['allow_nan']
+    assert not redefine_tags_est.__sklearn_tags__().input_tags.allow_nan
 
     diamond_tag_est = DiamondOverwriteTag()
-    assert diamond_tag_est._get_tags()['allow_nan']
+    assert diamond_tag_est.__sklearn_tags__().input_tags.allow_nan
 
     inherit_diamond_tag_est = InheritDiamondOverwriteTag()
-    assert inherit_diamond_tag_est._get_tags()['allow_nan']
+    assert inherit_diamond_tag_est.__sklearn_tags__().input_tags.allow_nan
 
 
 def test_raises_on_get_params_non_attribute():
@@ -530,54 +681,322 @@ def test_repr_mimebundle_():
     tree = DecisionTreeClassifier()
     output = tree._repr_mimebundle_()
     assert "text/plain" in output
-    assert "text/html" not in output
+    assert "text/html" in output
 
-    with config_context(display='diagram'):
+    with config_context(display="text"):
         output = tree._repr_mimebundle_()
         assert "text/plain" in output
-        assert "text/html" in output
+        assert "text/html" not in output
 
 
 def test_repr_html_wraps():
     # Checks the display configuration flag controls the html output
     tree = DecisionTreeClassifier()
-    msg = "_repr_html_ is only defined when"
-    with pytest.raises(AttributeError, match=msg):
-        output = tree._repr_html_()
-
-    with config_context(display='diagram'):
-        output = tree._repr_html_()
-        assert "'
-                  f'
'
-                  '
')
-        _write_estimator_html(out, estimator, estimator.__class__.__name__,
-                              str(estimator), first_call=True)
-        out.write('
')
-
-        html_output = out.getvalue()
-        return html_output
diff --git a/sklearn/utils/_fast_dict.pxd b/sklearn/utils/_fast_dict.pxd
index 9a1bde475ee62..e37f254661ce6 100644
--- a/sklearn/utils/_fast_dict.pxd
+++ b/sklearn/utils/_fast_dict.pxd
@@ -1,5 +1,6 @@
-# Author: Gael Varoquaux
-# License: BSD
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
 """
 Uses C++ map containers for fast dict-like behavior with keys being
 integers, and values float.
@@ -7,16 +8,12 @@ integers, and values float.
 
 from libcpp.map cimport map as cpp_map
 
-# Import the C-level symbols of numpy
-cimport numpy as np
-
-ctypedef np.float64_t DTYPE_t
+from ._typedefs cimport float64_t, intp_t
 
-ctypedef np.intp_t ITYPE_t
 
 ###############################################################################
 # An object to be used in Python
 
 cdef class IntFloatDict:
-    cdef cpp_map[ITYPE_t, DTYPE_t] my_map
-    cdef _to_arrays(self, ITYPE_t [:] keys, DTYPE_t [:] values)
+    cdef cpp_map[intp_t, float64_t] my_map
+    cdef _to_arrays(self, intp_t [:] keys, float64_t [:] values)
diff --git a/sklearn/utils/_fast_dict.pyx b/sklearn/utils/_fast_dict.pyx
index 719cafc3cc8c1..cdf84d9b592e1 100644
--- a/sklearn/utils/_fast_dict.pyx
+++ b/sklearn/utils/_fast_dict.pyx
@@ -2,31 +2,18 @@
 Uses C++ map containers for fast dict-like behavior with keys being
 integers, and values float.
 """
-# Author: Gael Varoquaux
-# License: BSD
-
-cimport cython
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
 # C++
-from cython.operator cimport dereference as deref, preincrement as inc, \
-    predecrement as dec
+from cython.operator cimport dereference as deref, preincrement as inc
 from libcpp.utility cimport pair
 from libcpp.map cimport map as cpp_map
 
 import numpy as np
 
-# Import the C-level symbols of numpy
-cimport numpy as np
-
-# Numpy must be initialized. When using numpy from C or Cython you must
-# _always_ do that, or you will have segfaults
-np.import_array()
-
-#DTYPE = np.float64
-#ctypedef np.float64_t DTYPE_t
+from ._typedefs cimport float64_t, intp_t
 
-#ITYPE = np.intp
-#ctypedef np.intp_t ITYPE_t
 
 ###############################################################################
 # An object to be used in Python
@@ -38,10 +25,11 @@ np.import_array()
 
 cdef class IntFloatDict:
 
-    @cython.boundscheck(False)
-    @cython.wraparound(False)
-    def __init__(self, np.ndarray[ITYPE_t, ndim=1] keys,
-                       np.ndarray[DTYPE_t, ndim=1] values):
+    def __init__(
+        self,
+        intp_t[:] keys,
+        float64_t[:] values,
+    ):
         cdef int i
         cdef int size = values.size
         # Should check that sizes for keys and values are equal, and
@@ -53,7 +41,7 @@ cdef class IntFloatDict:
         return self.my_map.size()
 
     def __getitem__(self, int key):
-        cdef cpp_map[ITYPE_t, DTYPE_t].iterator it = self.my_map.find(key)
+        cdef cpp_map[intp_t, float64_t].iterator it = self.my_map.find(key)
         if it == self.my_map.end():
             # The key is not in the dict
             raise KeyError('%i' % key)
@@ -64,21 +52,21 @@ cdef class IntFloatDict:
 
     # Cython 0.20 generates buggy code below. Commenting this out for now
     # and relying on the to_arrays method
-    #def __iter__(self):
-    #    cdef cpp_map[ITYPE_t, DTYPE_t].iterator it = self.my_map.begin()
-    #    cdef cpp_map[ITYPE_t, DTYPE_t].iterator end = self.my_map.end()
-    #    while it != end:
-    #        yield deref(it).first, deref(it).second
-    #        inc(it)
-    
+    # def __iter__(self):
+    #     cdef cpp_map[intp_t, float64_t].iterator it = self.my_map.begin()
+    #     cdef cpp_map[intp_t, float64_t].iterator end = self.my_map.end()
+    #     while it != end:
+    #         yield deref(it).first, deref(it).second
+    #         inc(it)
+
     def __iter__(self):
         cdef int size = self.my_map.size()
-        cdef ITYPE_t [:] keys = np.empty(size, dtype=np.intp)
-        cdef DTYPE_t [:] values = np.empty(size, dtype=np.float64)
+        cdef intp_t [:] keys = np.empty(size, dtype=np.intp)
+        cdef float64_t [:] values = np.empty(size, dtype=np.float64)
         self._to_arrays(keys, values)
         cdef int idx
-        cdef ITYPE_t key
-        cdef DTYPE_t value
+        cdef intp_t key
+        cdef float64_t value
         for idx in range(size):
             key = keys[idx]
             value = values[idx]
@@ -96,17 +84,15 @@ cdef class IntFloatDict:
                 The values of the data points
         """
         cdef int size = self.my_map.size()
-        cdef np.ndarray[ITYPE_t, ndim=1] keys = np.empty(size,
-                                                         dtype=np.intp)
-        cdef np.ndarray[DTYPE_t, ndim=1] values = np.empty(size,
-                                                           dtype=np.float64)
+        keys = np.empty(size, dtype=np.intp)
+        values = np.empty(size, dtype=np.float64)
         self._to_arrays(keys, values)
         return keys, values
 
-    cdef _to_arrays(self, ITYPE_t [:] keys, DTYPE_t [:] values):
+    cdef _to_arrays(self, intp_t [:] keys, float64_t [:] values):
         # Internal version of to_arrays that takes already-initialized arrays
-        cdef cpp_map[ITYPE_t, DTYPE_t].iterator it = self.my_map.begin()
-        cdef cpp_map[ITYPE_t, DTYPE_t].iterator end = self.my_map.end()
+        cdef cpp_map[intp_t, float64_t].iterator it = self.my_map.begin()
+        cdef cpp_map[intp_t, float64_t].iterator end = self.my_map.end()
         cdef int index = 0
         while it != end:
             keys[index] = deref(it).first
@@ -115,8 +101,8 @@ cdef class IntFloatDict:
             index += 1
 
     def update(self, IntFloatDict other):
-        cdef cpp_map[ITYPE_t, DTYPE_t].iterator it = other.my_map.begin()
-        cdef cpp_map[ITYPE_t, DTYPE_t].iterator end = other.my_map.end()
+        cdef cpp_map[intp_t, float64_t].iterator it = other.my_map.begin()
+        cdef cpp_map[intp_t, float64_t].iterator end = other.my_map.end()
         while it != end:
             self.my_map[deref(it).first] = deref(it).second
             inc(it)
@@ -127,29 +113,25 @@ cdef class IntFloatDict:
         out_obj.my_map = self.my_map
         return out_obj
 
-    def append(self, ITYPE_t key, DTYPE_t value):
-        cdef cpp_map[ITYPE_t, DTYPE_t].iterator end = self.my_map.end()
-        # Decrement the iterator
-        dec(end)
+    def append(self, intp_t key, float64_t value):
         # Construct our arguments
-        cdef pair[ITYPE_t, DTYPE_t] args
+        cdef pair[intp_t, float64_t] args
         args.first = key
         args.second = value
-        self.my_map.insert(end, args)
+        self.my_map.insert(args)
 
 
 ###############################################################################
 # operation on dict
 
 def argmin(IntFloatDict d):
-    cdef cpp_map[ITYPE_t, DTYPE_t].iterator it = d.my_map.begin()
-    cdef cpp_map[ITYPE_t, DTYPE_t].iterator end = d.my_map.end()
-    cdef ITYPE_t min_key
-    cdef DTYPE_t min_value = np.inf
+    cdef cpp_map[intp_t, float64_t].iterator it = d.my_map.begin()
+    cdef cpp_map[intp_t, float64_t].iterator end = d.my_map.end()
+    cdef intp_t min_key = -1
+    cdef float64_t min_value = np.inf
     while it != end:
         if deref(it).second < min_value:
             min_value = deref(it).second
             min_key = deref(it).first
         inc(it)
     return min_key, min_value
-
diff --git a/sklearn/utils/_heap.pxd b/sklearn/utils/_heap.pxd
new file mode 100644
index 0000000000000..39de4dc02d315
--- /dev/null
+++ b/sklearn/utils/_heap.pxd
@@ -0,0 +1,14 @@
+# Heap routines, used in various Cython implementations.
+
+from cython cimport floating
+
+from ._typedefs cimport intp_t
+
+
+cdef int heap_push(
+    floating* values,
+    intp_t* indices,
+    intp_t size,
+    floating val,
+    intp_t val_idx,
+) noexcept nogil
diff --git a/sklearn/utils/_heap.pyx b/sklearn/utils/_heap.pyx
new file mode 100644
index 0000000000000..98bc3046a0798
--- /dev/null
+++ b/sklearn/utils/_heap.pyx
@@ -0,0 +1,85 @@
+from cython cimport floating
+
+from ._typedefs cimport intp_t
+
+
+cdef inline int heap_push(
+    floating* values,
+    intp_t* indices,
+    intp_t size,
+    floating val,
+    intp_t val_idx,
+) noexcept nogil:
+    """Push a tuple (val, val_idx) onto a fixed-size max-heap.
+
+    The max-heap is represented as a Structure of Arrays where:
+     - values is the array containing the data to construct the heap with
+     - indices is the array containing the indices (meta-data) of each value
+
+    Notes
+    -----
+    Arrays are manipulated via a pointer to there first element and their size
+    as to ease the processing of dynamically allocated buffers.
+
+    For instance, in pseudo-code:
+
+        values = [1.2, 0.4, 0.1],
+        indices = [42, 1, 5],
+        heap_push(
+            values=values,
+            indices=indices,
+            size=3,
+            val=0.2,
+            val_idx=4,
+        )
+
+    will modify values and indices inplace, giving at the end of the call:
+
+        values  == [0.4, 0.2, 0.1]
+        indices == [1, 4, 5]
+
+    """
+    cdef:
+        intp_t current_idx, left_child_idx, right_child_idx, swap_idx
+
+    # Check if val should be in heap
+    if val >= values[0]:
+        return 0
+
+    # Insert val at position zero
+    values[0] = val
+    indices[0] = val_idx
+
+    # Descend the heap, swapping values until the max heap criterion is met
+    current_idx = 0
+    while True:
+        left_child_idx = 2 * current_idx + 1
+        right_child_idx = left_child_idx + 1
+
+        if left_child_idx >= size:
+            break
+        elif right_child_idx >= size:
+            if values[left_child_idx] > val:
+                swap_idx = left_child_idx
+            else:
+                break
+        elif values[left_child_idx] >= values[right_child_idx]:
+            if val < values[left_child_idx]:
+                swap_idx = left_child_idx
+            else:
+                break
+        else:
+            if val < values[right_child_idx]:
+                swap_idx = right_child_idx
+            else:
+                break
+
+        values[current_idx] = values[swap_idx]
+        indices[current_idx] = indices[swap_idx]
+
+        current_idx = swap_idx
+
+    values[current_idx] = val
+    indices[current_idx] = val_idx
+
+    return 0
diff --git a/sklearn/utils/_indexing.py b/sklearn/utils/_indexing.py
new file mode 100644
index 0000000000000..09427376a4059
--- /dev/null
+++ b/sklearn/utils/_indexing.py
@@ -0,0 +1,743 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+import numbers
+import sys
+import warnings
+from collections import UserList
+from itertools import compress, islice
+
+import numpy as np
+from scipy.sparse import issparse
+
+from ._array_api import _is_numpy_namespace, get_namespace
+from ._param_validation import Interval, validate_params
+from .extmath import _approximate_mode
+from .validation import (
+    _check_sample_weight,
+    _is_arraylike_not_scalar,
+    _is_pandas_df,
+    _is_polars_df_or_series,
+    _is_pyarrow_data,
+    _use_interchange_protocol,
+    check_array,
+    check_consistent_length,
+    check_random_state,
+)
+
+
+def _array_indexing(array, key, key_dtype, axis):
+    """Index an array or scipy.sparse consistently across NumPy version."""
+    xp, is_array_api = get_namespace(array)
+    if is_array_api:
+        return xp.take(array, key, axis=axis)
+    if issparse(array) and key_dtype == "bool":
+        key = np.asarray(key)
+    if isinstance(key, tuple):
+        key = list(key)
+    return array[key, ...] if axis == 0 else array[:, key]
+
+
+def _pandas_indexing(X, key, key_dtype, axis):
+    """Index a pandas dataframe or a series."""
+    if _is_arraylike_not_scalar(key):
+        key = np.asarray(key)
+
+    if key_dtype == "int" and not (isinstance(key, slice) or np.isscalar(key)):
+        # using take() instead of iloc[] ensures the return value is a "proper"
+        # copy that will not raise SettingWithCopyWarning
+        return X.take(key, axis=axis)
+    else:
+        # check whether we should index with loc or iloc
+        indexer = X.iloc if key_dtype == "int" else X.loc
+        return indexer[:, key] if axis else indexer[key]
+
+
+def _list_indexing(X, key, key_dtype):
+    """Index a Python list."""
+    if np.isscalar(key) or isinstance(key, slice):
+        # key is a slice or a scalar
+        return X[key]
+    if key_dtype == "bool":
+        # key is a boolean array-like
+        return list(compress(X, key))
+    # key is a integer array-like of key
+    return [X[idx] for idx in key]
+
+
+def _polars_indexing(X, key, key_dtype, axis):
+    """Index a polars dataframe or series."""
+    # Polars behavior is more consistent with lists
+    if isinstance(key, np.ndarray):
+        # Convert each element of the array to a Python scalar
+        key = key.tolist()
+    elif not (np.isscalar(key) or isinstance(key, slice)):
+        key = list(key)
+
+    if axis == 1:
+        # Here we are certain to have a polars DataFrame; which can be indexed with
+        # integer and string scalar, and list of integer, string and boolean
+        return X[:, key]
+
+    if key_dtype == "bool":
+        # Boolean mask can be indexed in the same way for Series and DataFrame (axis=0)
+        return X.filter(key)
+
+    # Integer scalar and list of integer can be indexed in the same way for Series and
+    # DataFrame (axis=0)
+    X_indexed = X[key]
+    if np.isscalar(key) and len(X.shape) == 2:
+        # `X_indexed` is a DataFrame with a single row; we return a Series to be
+        # consistent with pandas
+        pl = sys.modules["polars"]
+        return pl.Series(X_indexed.row(0))
+    return X_indexed
+
+
+def _pyarrow_indexing(X, key, key_dtype, axis):
+    """Index a pyarrow data."""
+    scalar_key = np.isscalar(key)
+    if isinstance(key, slice):
+        if isinstance(key.stop, str):
+            start = X.column_names.index(key.start)
+            stop = X.column_names.index(key.stop) + 1
+        else:
+            start = 0 if not key.start else key.start
+            stop = key.stop
+        step = 1 if not key.step else key.step
+        key = list(range(start, stop, step))
+
+    if axis == 1:
+        # Here we are certain that X is a pyarrow Table or RecordBatch.
+        if key_dtype == "int" and not isinstance(key, list):
+            # pyarrow's X.select behavior is more consistent with integer lists.
+            key = np.asarray(key).tolist()
+        if key_dtype == "bool":
+            key = np.asarray(key).nonzero()[0].tolist()
+
+        if scalar_key:
+            return X.column(key)
+
+        return X.select(key)
+
+    # axis == 0 from here on
+    if scalar_key:
+        if hasattr(X, "shape"):
+            # X is a Table or RecordBatch
+            key = [key]
+        else:
+            return X[key].as_py()
+    elif not isinstance(key, list):
+        key = np.asarray(key)
+
+    if key_dtype == "bool":
+        X_indexed = X.filter(key)
+    else:
+        X_indexed = X.take(key)
+
+    if scalar_key and len(getattr(X, "shape", [0])) == 2:
+        # X_indexed is a dataframe-like with a single row; we return a Series to be
+        # consistent with pandas
+        pa = sys.modules["pyarrow"]
+        return pa.array(X_indexed.to_pylist()[0].values())
+    return X_indexed
+
+
+def _determine_key_type(key, accept_slice=True):
+    """Determine the data type of key.
+
+    Parameters
+    ----------
+    key : scalar, slice or array-like
+        The key from which we want to infer the data type.
+
+    accept_slice : bool, default=True
+        Whether or not to raise an error if the key is a slice.
+
+    Returns
+    -------
+    dtype : {'int', 'str', 'bool', None}
+        Returns the data type of key.
+    """
+    err_msg = (
+        "No valid specification of the columns. Only a scalar, list or "
+        "slice of all integers or all strings, or boolean mask is "
+        "allowed"
+    )
+
+    dtype_to_str = {int: "int", str: "str", bool: "bool", np.bool_: "bool"}
+    array_dtype_to_str = {
+        "i": "int",
+        "u": "int",
+        "b": "bool",
+        "O": "str",
+        "U": "str",
+        "S": "str",
+    }
+
+    if key is None:
+        return None
+    if isinstance(key, tuple(dtype_to_str.keys())):
+        try:
+            return dtype_to_str[type(key)]
+        except KeyError:
+            raise ValueError(err_msg)
+    if isinstance(key, slice):
+        if not accept_slice:
+            raise TypeError(
+                "Only array-like or scalar are supported. A Python slice was given."
+            )
+        if key.start is None and key.stop is None:
+            return None
+        key_start_type = _determine_key_type(key.start)
+        key_stop_type = _determine_key_type(key.stop)
+        if key_start_type is not None and key_stop_type is not None:
+            if key_start_type != key_stop_type:
+                raise ValueError(err_msg)
+        if key_start_type is not None:
+            return key_start_type
+        return key_stop_type
+    # TODO(1.9) remove UserList when the force_int_remainder_cols param
+    # of ColumnTransformer is removed
+    if isinstance(key, (list, tuple, UserList)):
+        unique_key = set(key)
+        key_type = {_determine_key_type(elt) for elt in unique_key}
+        if not key_type:
+            return None
+        if len(key_type) != 1:
+            raise ValueError(err_msg)
+        return key_type.pop()
+    if hasattr(key, "dtype"):
+        xp, is_array_api = get_namespace(key)
+        # NumPy arrays are special-cased in their own branch because the Array API
+        # cannot handle object/string-based dtypes that are often used to index
+        # columns of dataframes by names.
+        if is_array_api and not _is_numpy_namespace(xp):
+            if xp.isdtype(key.dtype, "bool"):
+                return "bool"
+            elif xp.isdtype(key.dtype, "integral"):
+                return "int"
+            else:
+                raise ValueError(err_msg)
+        else:
+            try:
+                return array_dtype_to_str[key.dtype.kind]
+            except KeyError:
+                raise ValueError(err_msg)
+    raise ValueError(err_msg)
+
+
+def _safe_indexing(X, indices, *, axis=0):
+    """Return rows, items or columns of X using indices.
+
+    .. warning::
+
+        This utility is documented, but **private**. This means that
+        backward compatibility might be broken without any deprecation
+        cycle.
+
+    Parameters
+    ----------
+    X : array-like, sparse-matrix, list, pandas.DataFrame, pandas.Series
+        Data from which to sample rows, items or columns. `list` are only
+        supported when `axis=0`.
+    indices : bool, int, str, slice, array-like
+        - If `axis=0`, boolean and integer array-like, integer slice,
+          and scalar integer are supported.
+        - If `axis=1`:
+            - to select a single column, `indices` can be of `int` type for
+              all `X` types and `str` only for dataframe. The selected subset
+              will be 1D, unless `X` is a sparse matrix in which case it will
+              be 2D.
+            - to select multiples columns, `indices` can be one of the
+              following: `list`, `array`, `slice`. The type used in
+              these containers can be one of the following: `int`, 'bool' and
+              `str`. However, `str` is only supported when `X` is a dataframe.
+              The selected subset will be 2D.
+    axis : int, default=0
+        The axis along which `X` will be subsampled. `axis=0` will select
+        rows while `axis=1` will select columns.
+
+    Returns
+    -------
+    subset
+        Subset of X on axis 0 or 1.
+
+    Notes
+    -----
+    CSR, CSC, and LIL sparse matrices are supported. COO sparse matrices are
+    not supported.
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from sklearn.utils import _safe_indexing
+    >>> data = np.array([[1, 2], [3, 4], [5, 6]])
+    >>> _safe_indexing(data, 0, axis=0)  # select the first row
+    array([1, 2])
+    >>> _safe_indexing(data, 0, axis=1)  # select the first column
+    array([1, 3, 5])
+    """
+    if indices is None:
+        return X
+
+    if axis not in (0, 1):
+        raise ValueError(
+            "'axis' should be either 0 (to index rows) or 1 (to index "
+            " column). Got {} instead.".format(axis)
+        )
+
+    indices_dtype = _determine_key_type(indices)
+
+    if axis == 0 and indices_dtype == "str":
+        raise ValueError("String indexing is not supported with 'axis=0'")
+
+    if axis == 1 and isinstance(X, list):
+        raise ValueError("axis=1 is not supported for lists")
+
+    if axis == 1 and (ndim := len(getattr(X, "shape", [0]))) != 2:
+        raise ValueError(
+            "'X' should be a 2D NumPy array, 2D sparse matrix or "
+            "dataframe when indexing the columns (i.e. 'axis=1'). "
+            f"Got {type(X)} instead with {ndim} dimension(s)."
+        )
+
+    if (
+        axis == 1
+        and indices_dtype == "str"
+        and not (_is_pandas_df(X) or _use_interchange_protocol(X))
+    ):
+        raise ValueError(
+            "Specifying the columns using strings is only supported for dataframes."
+        )
+
+    if hasattr(X, "iloc"):
+        # TODO: we should probably use _is_pandas_df_or_series(X) instead but:
+        # 1) Currently, it (probably) works for dataframes compliant to pandas' API.
+        # 2) Updating would require updating some tests such as
+        #    test_train_test_split_mock_pandas.
+        return _pandas_indexing(X, indices, indices_dtype, axis=axis)
+    elif _is_polars_df_or_series(X):
+        return _polars_indexing(X, indices, indices_dtype, axis=axis)
+    elif _is_pyarrow_data(X):
+        return _pyarrow_indexing(X, indices, indices_dtype, axis=axis)
+    elif _use_interchange_protocol(X):  # pragma: no cover
+        # Once the dataframe X is converted into its dataframe interchange protocol
+        # version by calling X.__dataframe__(), it becomes very hard to turn it back
+        # into its original type, e.g., a pyarrow.Table, see
+        # https://github.com/data-apis/dataframe-api/issues/85.
+        raise warnings.warn(
+            message="A data object with support for the dataframe interchange protocol"
+            "was passed, but scikit-learn does currently not know how to handle this "
+            "kind of data. Some array/list indexing will be tried.",
+            category=UserWarning,
+        )
+
+    if hasattr(X, "shape"):
+        return _array_indexing(X, indices, indices_dtype, axis=axis)
+    else:
+        return _list_indexing(X, indices, indices_dtype)
+
+
+def _safe_assign(X, values, *, row_indexer=None, column_indexer=None):
+    """Safe assignment to a numpy array, sparse matrix, or pandas dataframe.
+
+    Parameters
+    ----------
+    X : {ndarray, sparse-matrix, dataframe}
+        Array to be modified. It is expected to be 2-dimensional.
+
+    values : ndarray
+        The values to be assigned to `X`.
+
+    row_indexer : array-like, dtype={int, bool}, default=None
+        A 1-dimensional array to select the rows of interest. If `None`, all
+        rows are selected.
+
+    column_indexer : array-like, dtype={int, bool}, default=None
+        A 1-dimensional array to select the columns of interest. If `None`, all
+        columns are selected.
+    """
+    row_indexer = slice(None, None, None) if row_indexer is None else row_indexer
+    column_indexer = (
+        slice(None, None, None) if column_indexer is None else column_indexer
+    )
+
+    if hasattr(X, "iloc"):  # pandas dataframe
+        with warnings.catch_warnings():
+            # pandas >= 1.5 raises a warning when using iloc to set values in a column
+            # that does not have the same type as the column being set. It happens
+            # for instance when setting a categorical column with a string.
+            # In the future the behavior won't change and the warning should disappear.
+            # TODO(1.3): check if the warning is still raised or remove the filter.
+            warnings.simplefilter("ignore", FutureWarning)
+            X.iloc[row_indexer, column_indexer] = values
+    else:  # numpy array or sparse matrix
+        X[row_indexer, column_indexer] = values
+
+
+def _get_column_indices_for_bool_or_int(key, n_columns):
+    # Convert key into list of positive integer indexes
+    try:
+        idx = _safe_indexing(np.arange(n_columns), key)
+    except IndexError as e:
+        raise ValueError(
+            f"all features must be in [0, {n_columns - 1}] or [-{n_columns}, 0]"
+        ) from e
+    return np.atleast_1d(idx).tolist()
+
+
+def _get_column_indices(X, key):
+    """Get feature column indices for input data X and key.
+
+    For accepted values of `key`, see the docstring of
+    :func:`_safe_indexing`.
+    """
+    key_dtype = _determine_key_type(key)
+    if _use_interchange_protocol(X):
+        return _get_column_indices_interchange(X.__dataframe__(), key, key_dtype)
+
+    n_columns = X.shape[1]
+    if isinstance(key, (list, tuple)) and not key:
+        # we get an empty list
+        return []
+    elif key_dtype in ("bool", "int"):
+        return _get_column_indices_for_bool_or_int(key, n_columns)
+    else:
+        try:
+            all_columns = X.columns
+        except AttributeError:
+            raise ValueError(
+                "Specifying the columns using strings is only supported for dataframes."
+            )
+        if isinstance(key, str):
+            columns = [key]
+        elif isinstance(key, slice):
+            start, stop = key.start, key.stop
+            if start is not None:
+                start = all_columns.get_loc(start)
+            if stop is not None:
+                # pandas indexing with strings is endpoint included
+                stop = all_columns.get_loc(stop) + 1
+            else:
+                stop = n_columns + 1
+            return list(islice(range(n_columns), start, stop))
+        else:
+            columns = list(key)
+
+        try:
+            column_indices = []
+            for col in columns:
+                col_idx = all_columns.get_loc(col)
+                if not isinstance(col_idx, numbers.Integral):
+                    raise ValueError(
+                        f"Selected columns, {columns}, are not unique in dataframe"
+                    )
+                column_indices.append(col_idx)
+
+        except KeyError as e:
+            raise ValueError("A given column is not a column of the dataframe") from e
+
+        return column_indices
+
+
+def _get_column_indices_interchange(X_interchange, key, key_dtype):
+    """Same as _get_column_indices but for X with __dataframe__ protocol."""
+
+    n_columns = X_interchange.num_columns()
+
+    if isinstance(key, (list, tuple)) and not key:
+        # we get an empty list
+        return []
+    elif key_dtype in ("bool", "int"):
+        return _get_column_indices_for_bool_or_int(key, n_columns)
+    else:
+        column_names = list(X_interchange.column_names())
+
+        if isinstance(key, slice):
+            if key.step not in [1, None]:
+                raise NotImplementedError("key.step must be 1 or None")
+            start, stop = key.start, key.stop
+            if start is not None:
+                start = column_names.index(start)
+
+            if stop is not None:
+                stop = column_names.index(stop) + 1
+            else:
+                stop = n_columns + 1
+            return list(islice(range(n_columns), start, stop))
+
+        selected_columns = [key] if np.isscalar(key) else key
+
+        try:
+            return [column_names.index(col) for col in selected_columns]
+        except ValueError as e:
+            raise ValueError("A given column is not a column of the dataframe") from e
+
+
+@validate_params(
+    {
+        "replace": ["boolean"],
+        "n_samples": [Interval(numbers.Integral, 1, None, closed="left"), None],
+        "random_state": ["random_state"],
+        "stratify": ["array-like", "sparse matrix", None],
+        "sample_weight": ["array-like", None],
+    },
+    prefer_skip_nested_validation=True,
+)
+def resample(
+    *arrays,
+    replace=True,
+    n_samples=None,
+    random_state=None,
+    stratify=None,
+    sample_weight=None,
+):
+    """Resample arrays or sparse matrices in a consistent way.
+
+    The default strategy implements one step of the bootstrapping
+    procedure.
+
+    Parameters
+    ----------
+    *arrays : sequence of array-like of shape (n_samples,) or \
+            (n_samples, n_outputs)
+        Indexable data-structures can be arrays, lists, dataframes or scipy
+        sparse matrices with consistent first dimension.
+
+    replace : bool, default=True
+        Implements resampling with replacement. It must be set to True
+        whenever sampling with non-uniform weights: a few data points with very large
+        weights are expected to be sampled several times with probability to preserve
+        the distribution induced by the weights. If False, this will implement
+        (sliced) random permutations.
+
+    n_samples : int, default=None
+        Number of samples to generate. If left to None this is
+        automatically set to the first dimension of the arrays.
+        If replace is False it should not be larger than the length of
+        arrays.
+
+    random_state : int, RandomState instance or None, default=None
+        Determines random number generation for shuffling
+        the data.
+        Pass an int for reproducible results across multiple function calls.
+        See :term:`Glossary `.
+
+    stratify : {array-like, sparse matrix} of shape (n_samples,) or \
+            (n_samples, n_outputs), default=None
+        If not None, data is split in a stratified fashion, using this as
+        the class labels.
+
+    sample_weight : array-like of shape (n_samples,), default=None
+        Contains weight values to be associated with each sample. Values are
+        normalized to sum to one and interpreted as probability for sampling
+        each data point.
+
+        .. versionadded:: 1.7
+
+    Returns
+    -------
+    resampled_arrays : sequence of array-like of shape (n_samples,) or \
+            (n_samples, n_outputs)
+        Sequence of resampled copies of the collections. The original arrays
+        are not impacted.
+
+    See Also
+    --------
+    shuffle : Shuffle arrays or sparse matrices in a consistent way.
+
+    Examples
+    --------
+    It is possible to mix sparse and dense arrays in the same run::
+
+      >>> import numpy as np
+      >>> X = np.array([[1., 0.], [2., 1.], [0., 0.]])
+      >>> y = np.array([0, 1, 2])
+
+      >>> from scipy.sparse import coo_matrix
+      >>> X_sparse = coo_matrix(X)
+
+      >>> from sklearn.utils import resample
+      >>> X, X_sparse, y = resample(X, X_sparse, y, random_state=0)
+      >>> X
+      array([[1., 0.],
+             [2., 1.],
+             [1., 0.]])
+
+      >>> X_sparse
+      
+
+      >>> X_sparse.toarray()
+      array([[1., 0.],
+             [2., 1.],
+             [1., 0.]])
+
+      >>> y
+      array([0, 1, 0])
+
+      >>> resample(y, n_samples=2, random_state=0)
+      array([0, 1])
+
+    Example using stratification::
+
+      >>> y = [0, 0, 1, 1, 1, 1, 1, 1, 1]
+      >>> resample(y, n_samples=5, replace=False, stratify=y,
+      ...          random_state=0)
+      [1, 1, 1, 0, 1]
+    """
+    max_n_samples = n_samples
+    random_state = check_random_state(random_state)
+
+    if len(arrays) == 0:
+        return None
+
+    first = arrays[0]
+    n_samples = first.shape[0] if hasattr(first, "shape") else len(first)
+
+    if max_n_samples is None:
+        max_n_samples = n_samples
+    elif (max_n_samples > n_samples) and (not replace):
+        raise ValueError(
+            "Cannot sample %d out of arrays with dim %d when replace is False"
+            % (max_n_samples, n_samples)
+        )
+
+    check_consistent_length(*arrays)
+
+    if sample_weight is not None and not replace:
+        raise NotImplementedError(
+            "Resampling with sample_weight is only implemented for replace=True."
+        )
+    if sample_weight is not None and stratify is not None:
+        raise NotImplementedError(
+            "Resampling with sample_weight is only implemented for stratify=None."
+        )
+    if stratify is None:
+        if replace:
+            if sample_weight is not None:
+                sample_weight = _check_sample_weight(
+                    sample_weight, first, dtype=np.float64
+                )
+                p = sample_weight / sample_weight.sum()
+            else:
+                p = None
+            indices = random_state.choice(
+                n_samples,
+                size=max_n_samples,
+                p=p,
+                replace=True,
+            )
+        else:
+            indices = np.arange(n_samples)
+            random_state.shuffle(indices)
+            indices = indices[:max_n_samples]
+    else:
+        # Code adapted from StratifiedShuffleSplit()
+        y = check_array(stratify, ensure_2d=False, dtype=None)
+        if y.ndim == 2:
+            # for multi-label y, map each distinct row to a string repr
+            # using join because str(row) uses an ellipsis if len(row) > 1000
+            y = np.array([" ".join(row.astype("str")) for row in y])
+
+        classes, y_indices = np.unique(y, return_inverse=True)
+        n_classes = classes.shape[0]
+
+        class_counts = np.bincount(y_indices)
+
+        # Find the sorted list of instances for each class:
+        # (np.unique above performs a sort, so code is O(n logn) already)
+        class_indices = np.split(
+            np.argsort(y_indices, kind="mergesort"), np.cumsum(class_counts)[:-1]
+        )
+
+        n_i = _approximate_mode(class_counts, max_n_samples, random_state)
+
+        indices = []
+
+        for i in range(n_classes):
+            indices_i = random_state.choice(class_indices[i], n_i[i], replace=replace)
+            indices.extend(indices_i)
+
+        indices = random_state.permutation(indices)
+
+    # convert sparse matrices to CSR for row-based indexing
+    arrays = [a.tocsr() if issparse(a) else a for a in arrays]
+    resampled_arrays = [_safe_indexing(a, indices) for a in arrays]
+    if len(resampled_arrays) == 1:
+        # syntactic sugar for the unit argument case
+        return resampled_arrays[0]
+    else:
+        return resampled_arrays
+
+
+def shuffle(*arrays, random_state=None, n_samples=None):
+    """Shuffle arrays or sparse matrices in a consistent way.
+
+    This is a convenience alias to ``resample(*arrays, replace=False)`` to do
+    random permutations of the collections.
+
+    Parameters
+    ----------
+    *arrays : sequence of indexable data-structures
+        Indexable data-structures can be arrays, lists, dataframes or scipy
+        sparse matrices with consistent first dimension.
+
+    random_state : int, RandomState instance or None, default=None
+        Determines random number generation for shuffling
+        the data.
+        Pass an int for reproducible results across multiple function calls.
+        See :term:`Glossary `.
+
+    n_samples : int, default=None
+        Number of samples to generate. If left to None this is
+        automatically set to the first dimension of the arrays.  It should
+        not be larger than the length of arrays.
+
+    Returns
+    -------
+    shuffled_arrays : sequence of indexable data-structures
+        Sequence of shuffled copies of the collections. The original arrays
+        are not impacted.
+
+    See Also
+    --------
+    resample : Resample arrays or sparse matrices in a consistent way.
+
+    Examples
+    --------
+    It is possible to mix sparse and dense arrays in the same run::
+
+      >>> import numpy as np
+      >>> X = np.array([[1., 0.], [2., 1.], [0., 0.]])
+      >>> y = np.array([0, 1, 2])
+
+      >>> from scipy.sparse import coo_matrix
+      >>> X_sparse = coo_matrix(X)
+
+      >>> from sklearn.utils import shuffle
+      >>> X, X_sparse, y = shuffle(X, X_sparse, y, random_state=0)
+      >>> X
+      array([[0., 0.],
+             [2., 1.],
+             [1., 0.]])
+
+      >>> X_sparse
+      
+
+      >>> X_sparse.toarray()
+      array([[0., 0.],
+             [2., 1.],
+             [1., 0.]])
+
+      >>> y
+      array([2, 1, 0])
+
+      >>> shuffle(y, n_samples=2, random_state=0)
+      array([0, 1])
+    """
+    return resample(
+        *arrays, replace=False, n_samples=n_samples, random_state=random_state
+    )
diff --git a/sklearn/utils/_isfinite.pyx b/sklearn/utils/_isfinite.pyx
new file mode 100644
index 0000000000000..f3918eeacb5c4
--- /dev/null
+++ b/sklearn/utils/_isfinite.pyx
@@ -0,0 +1,51 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+from libc.math cimport isnan, isinf
+from cython cimport floating
+
+
+cpdef enum FiniteStatus:
+    all_finite = 0
+    has_nan = 1
+    has_infinite = 2
+
+
+def cy_isfinite(floating[::1] a, bint allow_nan=False):
+    cdef FiniteStatus result
+    with nogil:
+        result = _isfinite(a, allow_nan)
+    return result
+
+
+cdef inline FiniteStatus _isfinite(floating[::1] a, bint allow_nan) noexcept nogil:
+    cdef floating* a_ptr = &a[0]
+    cdef Py_ssize_t length = len(a)
+    if allow_nan:
+        return _isfinite_allow_nan(a_ptr, length)
+    else:
+        return _isfinite_disable_nan(a_ptr, length)
+
+
+cdef inline FiniteStatus _isfinite_allow_nan(floating* a_ptr,
+                                             Py_ssize_t length) noexcept nogil:
+    cdef Py_ssize_t i
+    cdef floating v
+    for i in range(length):
+        v = a_ptr[i]
+        if isinf(v):
+            return FiniteStatus.has_infinite
+    return FiniteStatus.all_finite
+
+
+cdef inline FiniteStatus _isfinite_disable_nan(floating* a_ptr,
+                                               Py_ssize_t length) noexcept nogil:
+    cdef Py_ssize_t i
+    cdef floating v
+    for i in range(length):
+        v = a_ptr[i]
+        if isnan(v):
+            return FiniteStatus.has_nan
+        elif isinf(v):
+            return FiniteStatus.has_infinite
+    return FiniteStatus.all_finite
diff --git a/sklearn/utils/_joblib.py b/sklearn/utils/_joblib.py
deleted file mode 100644
index 3cd7e7fe074fe..0000000000000
--- a/sklearn/utils/_joblib.py
+++ /dev/null
@@ -1,19 +0,0 @@
-import warnings as _warnings
-
-with _warnings.catch_warnings():
-    _warnings.simplefilter("ignore")
-    # joblib imports may raise DeprecationWarning on certain Python
-    # versions
-    import joblib
-    from joblib import logger
-    from joblib import dump, load
-    from joblib import __version__
-    from joblib import effective_n_jobs
-    from joblib import hash
-    from joblib import cpu_count, Parallel, Memory, delayed
-    from joblib import parallel_backend, register_parallel_backend
-
-
-__all__ = ["parallel_backend", "register_parallel_backend", "cpu_count",
-           "Parallel", "Memory", "delayed", "effective_n_jobs", "hash",
-           "logger", "dump", "load", "joblib", "__version__"]
diff --git a/sklearn/utils/_logistic_sigmoid.pyx b/sklearn/utils/_logistic_sigmoid.pyx
deleted file mode 100644
index 3531d99bc4f44..0000000000000
--- a/sklearn/utils/_logistic_sigmoid.pyx
+++ /dev/null
@@ -1,33 +0,0 @@
-#cython: boundscheck=False
-#cython: cdivision=True
-#cython: wraparound=False
-
-from libc.math cimport log, exp
-
-import numpy as np
-cimport numpy as np
-
-np.import_array()
-ctypedef np.float64_t DTYPE_t
-
-
-cdef inline DTYPE_t _inner_log_logistic_sigmoid(const DTYPE_t x):
-    """Log of the logistic sigmoid function log(1 / (1 + e ** -x))"""
-    if x > 0:
-        return -log(1. + exp(-x))
-    else:
-        return x - log(1. + exp(x))
-
-
-def _log_logistic_sigmoid(unsigned int n_samples,
-                          unsigned int n_features,
-                          DTYPE_t[:, :] X,
-                          DTYPE_t[:, :] out):
-    cdef:
-        unsigned int i
-        unsigned int j
-
-    for i in range(n_samples):
-        for j in range(n_features):
-            out[i, j] = _inner_log_logistic_sigmoid(X[i, j])
-    return out
diff --git a/sklearn/utils/_mask.py b/sklearn/utils/_mask.py
index 2bdbad5342fbd..da21c8e68b72d 100644
--- a/sklearn/utils/_mask.py
+++ b/sklearn/utils/_mask.py
@@ -1,11 +1,25 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+from contextlib import suppress
+
 import numpy as np
 from scipy import sparse as sp
 
-from . import is_scalar_nan
+from ._missing import is_scalar_nan
+from ._param_validation import validate_params
 from .fixes import _object_dtype_isnan
 
 
 def _get_dense_mask(X, value_to_mask):
+    with suppress(ImportError, AttributeError):
+        # We also suppress `AttributeError` because older versions of pandas do
+        # not have `NA`.
+        import pandas
+
+        if value_to_mask is pandas.NA:
+            return pandas.isna(X)
+
     if is_scalar_nan(value_to_mask):
         if X.dtype.kind == "f":
             Xt = np.isnan(X)
@@ -45,10 +59,123 @@ def _get_mask(X, value_to_mask):
 
     Xt = _get_dense_mask(X.data, value_to_mask)
 
-    sparse_constructor = (sp.csr_matrix if X.format == 'csr'
-                          else sp.csc_matrix)
+    sparse_constructor = sp.csr_matrix if X.format == "csr" else sp.csc_matrix
     Xt_sparse = sparse_constructor(
         (Xt, X.indices.copy(), X.indptr.copy()), shape=X.shape, dtype=bool
     )
 
     return Xt_sparse
+
+
+@validate_params(
+    {
+        "X": ["array-like", "sparse matrix"],
+        "mask": ["array-like"],
+    },
+    prefer_skip_nested_validation=True,
+)
+def safe_mask(X, mask):
+    """Return a mask which is safe to use on X.
+
+    Parameters
+    ----------
+    X : {array-like, sparse matrix}
+        Data on which to apply mask.
+
+    mask : array-like
+        Mask to be used on X.
+
+    Returns
+    -------
+    mask : ndarray
+        Array that is safe to use on X.
+
+    Examples
+    --------
+    >>> from sklearn.utils import safe_mask
+    >>> from scipy.sparse import csr_matrix
+    >>> data = csr_matrix([[1], [2], [3], [4], [5]])
+    >>> condition = [False, True, True, False, True]
+    >>> mask = safe_mask(data, condition)
+    >>> data[mask].toarray()
+    array([[2],
+           [3],
+           [5]])
+    """
+    mask = np.asarray(mask)
+    if np.issubdtype(mask.dtype, np.signedinteger):
+        return mask
+
+    if hasattr(X, "toarray"):
+        ind = np.arange(mask.shape[0])
+        mask = ind[mask]
+    return mask
+
+
+def axis0_safe_slice(X, mask, len_mask):
+    """Return a mask which is safer to use on X than safe_mask.
+
+    This mask is safer than safe_mask since it returns an
+    empty array, when a sparse matrix is sliced with a boolean mask
+    with all False, instead of raising an unhelpful error in older
+    versions of SciPy.
+
+    See: https://github.com/scipy/scipy/issues/5361
+
+    Also note that we can avoid doing the dot product by checking if
+    the len_mask is not zero in _huber_loss_and_gradient but this
+    is not going to be the bottleneck, since the number of outliers
+    and non_outliers are typically non-zero and it makes the code
+    tougher to follow.
+
+    Parameters
+    ----------
+    X : {array-like, sparse matrix}
+        Data on which to apply mask.
+
+    mask : ndarray
+        Mask to be used on X.
+
+    len_mask : int
+        The length of the mask.
+
+    Returns
+    -------
+    mask : ndarray
+        Array that is safe to use on X.
+    """
+    if len_mask != 0:
+        return X[safe_mask(X, mask), :]
+    return np.zeros(shape=(0, X.shape[1]))
+
+
+def indices_to_mask(indices, mask_length):
+    """Convert list of indices to boolean mask.
+
+    Parameters
+    ----------
+    indices : list-like
+        List of integers treated as indices.
+    mask_length : int
+        Length of boolean mask to be generated.
+        This parameter must be greater than max(indices).
+
+    Returns
+    -------
+    mask : 1d boolean nd-array
+        Boolean array that is True where indices are present, else False.
+
+    Examples
+    --------
+    >>> from sklearn.utils._mask import indices_to_mask
+    >>> indices = [1, 2 , 3, 4]
+    >>> indices_to_mask(indices, 5)
+    array([False,  True,  True,  True,  True])
+    """
+    if mask_length <= np.max(indices):
+        raise ValueError("mask_length must be greater than max(indices)")
+
+    mask = np.zeros(mask_length, dtype=bool)
+    mask[indices] = True
+
+    return mask
diff --git a/sklearn/utils/_metadata_requests.py b/sklearn/utils/_metadata_requests.py
new file mode 100644
index 0000000000000..2c7e650b133d6
--- /dev/null
+++ b/sklearn/utils/_metadata_requests.py
@@ -0,0 +1,1604 @@
+"""
+Metadata Routing Utility
+
+In order to better understand the components implemented in this file, one
+needs to understand their relationship to one another.
+
+The only relevant public API for end users are the ``set_{method}_request`` methods,
+e.g. ``estimator.set_fit_request(sample_weight=True)``. However, third-party
+developers and users who implement custom meta-estimators, need to deal with
+the objects implemented in this file.
+
+All estimators (should) implement a ``get_metadata_routing`` method, returning
+the routing requests set for the estimator. This method is automatically
+implemented via ``BaseEstimator`` for all simple estimators, but needs a custom
+implementation for meta-estimators.
+
+In non-routing consumers, i.e. the simplest case, e.g. ``SVM``,
+``get_metadata_routing`` returns a ``MetadataRequest`` object.
+
+In routers, e.g. meta-estimators and a multi metric scorer,
+``get_metadata_routing`` returns a ``MetadataRouter`` object.
+
+An object which is both a router and a consumer, e.g. a meta-estimator which
+consumes ``sample_weight`` and routes ``sample_weight`` to its sub-estimators,
+routing information includes both information about the object itself (added
+via ``MetadataRouter.add_self_request``), as well as the routing information
+for its sub-estimators.
+
+A ``MetadataRequest`` instance includes one ``MethodMetadataRequest`` per
+method in ``METHODS``, which includes ``fit``, ``score``, etc.
+
+Request values are added to the routing mechanism by adding them to
+``MethodMetadataRequest`` instances, e.g.
+``metadatarequest.fit.add(param="sample_weight", alias="my_weights")``. This is
+used in ``set_{method}_request`` which are automatically generated, so users
+and developers almost never need to directly call methods on a
+``MethodMetadataRequest``.
+
+The ``alias`` above in the ``add`` method has to be either a string (an alias),
+or a {True (requested), False (unrequested), None (error if passed)}``. There
+are some other special values such as ``UNUSED`` and ``WARN`` which are used
+for purposes such as warning of removing a metadata in a child class, but not
+used by the end users.
+
+``MetadataRouter`` includes information about sub-objects' routing and how
+methods are mapped together. For instance, the information about which methods
+of a sub-estimator are called in which methods of the meta-estimator are all
+stored here. Conceptually, this information looks like:
+
+```
+{
+    "sub_estimator1": (
+        mapping=[(caller="fit", callee="transform"), ...],
+        router=MetadataRequest(...),  # or another MetadataRouter
+    ),
+    ...
+}
+```
+
+To give the above representation some structure, we use the following objects:
+
+- ``(caller=..., callee=...)`` is a namedtuple called ``MethodPair``
+
+- The list of ``MethodPair`` stored in the ``mapping`` field of a `RouterMappingPair` is
+  a ``MethodMapping`` object
+
+- ``(mapping=..., router=...)`` is a namedtuple called ``RouterMappingPair``
+
+The ``set_{method}_request`` methods are dynamically generated for estimators
+which inherit from the ``BaseEstimator``. This is done by attaching instances
+of the ``RequestMethod`` descriptor to classes, which is done in the
+``_MetadataRequester`` class, and ``BaseEstimator`` inherits from this mixin.
+This mixin also implements the ``get_metadata_routing``, which meta-estimators
+need to override, but it works for simple consumers as is.
+"""
+
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+import inspect
+from collections import namedtuple
+from copy import deepcopy
+from typing import TYPE_CHECKING, Optional, Union
+from warnings import warn
+
+from .. import get_config
+from ..exceptions import UnsetMetadataPassedError
+from ._bunch import Bunch
+
+# Only the following methods are supported in the routing mechanism. Adding new
+# methods at the moment involves monkeypatching this list.
+# Note that if this list is changed or monkeypatched, the corresponding method
+# needs to be added under a TYPE_CHECKING condition like the one done here in
+# _MetadataRequester
+SIMPLE_METHODS = [
+    "fit",
+    "partial_fit",
+    "predict",
+    "predict_proba",
+    "predict_log_proba",
+    "decision_function",
+    "score",
+    "split",
+    "transform",
+    "inverse_transform",
+]
+
+# These methods are a composite of other methods and one cannot set their
+# requests directly. Instead they should be set by setting the requests of the
+# simple methods which make the composite ones.
+COMPOSITE_METHODS = {
+    "fit_transform": ["fit", "transform"],
+    "fit_predict": ["fit", "predict"],
+}
+
+METHODS = SIMPLE_METHODS + list(COMPOSITE_METHODS.keys())
+
+
+def _routing_enabled():
+    """Return whether metadata routing is enabled.
+
+    .. versionadded:: 1.3
+
+    Returns
+    -------
+    enabled : bool
+        Whether metadata routing is enabled. If the config is not set, it
+        defaults to False.
+    """
+    return get_config().get("enable_metadata_routing", False)
+
+
+def _raise_for_params(params, owner, method, allow=None):
+    """Raise an error if metadata routing is not enabled and params are passed.
+
+    .. versionadded:: 1.4
+
+    Parameters
+    ----------
+    params : dict
+        The metadata passed to a method.
+
+    owner : object
+        The object to which the method belongs.
+
+    method : str
+        The name of the method, e.g. "fit".
+
+    allow : list of str, default=None
+        A list of parameters which are allowed to be passed even if metadata
+        routing is not enabled.
+
+    Raises
+    ------
+    ValueError
+        If metadata routing is not enabled and params are passed.
+    """
+    caller = (
+        f"{owner.__class__.__name__}.{method}" if method else owner.__class__.__name__
+    )
+
+    allow = allow if allow is not None else {}
+
+    if not _routing_enabled() and (params.keys() - allow):
+        raise ValueError(
+            f"Passing extra keyword arguments to {caller} is only supported if"
+            " enable_metadata_routing=True, which you can set using"
+            " `sklearn.set_config`. See the User Guide"
+            "  for more"
+            f" details. Extra parameters passed are: {set(params)}"
+        )
+
+
+def _raise_for_unsupported_routing(obj, method, **kwargs):
+    """Raise when metadata routing is enabled and metadata is passed.
+
+    This is used in meta-estimators which have not implemented metadata routing
+    to prevent silent bugs. There is no need to use this function if the
+    meta-estimator is not accepting any metadata, especially in `fit`, since
+    if a meta-estimator accepts any metadata, they would do that in `fit` as
+    well.
+
+    Parameters
+    ----------
+    obj : estimator
+        The estimator for which we're raising the error.
+
+    method : str
+        The method where the error is raised.
+
+    **kwargs : dict
+        The metadata passed to the method.
+    """
+    kwargs = {key: value for key, value in kwargs.items() if value is not None}
+    if _routing_enabled() and kwargs:
+        cls_name = obj.__class__.__name__
+        raise NotImplementedError(
+            f"{cls_name}.{method} cannot accept given metadata ({set(kwargs.keys())})"
+            f" since metadata routing is not yet implemented for {cls_name}."
+        )
+
+
+class _RoutingNotSupportedMixin:
+    """A mixin to be used to remove the default `get_metadata_routing`.
+
+    This is used in meta-estimators where metadata routing is not yet
+    implemented.
+
+    This also makes it clear in our rendered documentation that this method
+    cannot be used.
+    """
+
+    def get_metadata_routing(self):
+        """Raise `NotImplementedError`.
+
+        This estimator does not support metadata routing yet."""
+        raise NotImplementedError(
+            f"{self.__class__.__name__} has not implemented metadata routing yet."
+        )
+
+
+# Request values
+# ==============
+# Each request value needs to be one of the following values, or an alias.
+
+# this is used in `__metadata_request__*` attributes to indicate that a
+# metadata is not present even though it may be present in the
+# corresponding method's signature.
+UNUSED = "$UNUSED$"
+
+# this is used whenever a default value is changed, and therefore the user
+# should explicitly set the value, otherwise a warning is shown. An example
+# is when a meta-estimator is only a router, but then becomes also a
+# consumer in a new release.
+WARN = "$WARN$"
+
+# this is the default used in `set_{method}_request` methods to indicate no
+# change requested by the user.
+UNCHANGED = "$UNCHANGED$"
+
+VALID_REQUEST_VALUES = [False, True, None, UNUSED, WARN]
+
+
+def request_is_alias(item):
+    """Check if an item is a valid alias.
+
+    Values in ``VALID_REQUEST_VALUES`` are not considered aliases in this
+    context. Only a string which is a valid identifier is.
+
+    Parameters
+    ----------
+    item : object
+        The given item to be checked if it can be an alias.
+
+    Returns
+    -------
+    result : bool
+        Whether the given item is a valid alias.
+    """
+    if item in VALID_REQUEST_VALUES:
+        return False
+
+    # item is only an alias if it's a valid identifier
+    return isinstance(item, str) and item.isidentifier()
+
+
+def request_is_valid(item):
+    """Check if an item is a valid request value (and not an alias).
+
+    Parameters
+    ----------
+    item : object
+        The given item to be checked.
+
+    Returns
+    -------
+    result : bool
+        Whether the given item is valid.
+    """
+    return item in VALID_REQUEST_VALUES
+
+
+# Metadata Request for Simple Consumers
+# =====================================
+# This section includes MethodMetadataRequest and MetadataRequest which are
+# used in simple consumers.
+
+
+class MethodMetadataRequest:
+    """A prescription of how metadata is to be passed to a single method.
+
+    Refer to :class:`MetadataRequest` for how this class is used.
+
+    .. versionadded:: 1.3
+
+    Parameters
+    ----------
+    owner : str
+        A display name for the object owning these requests.
+
+    method : str
+        The name of the method to which these requests belong.
+
+    requests : dict of {str: bool, None or str}, default=None
+        The initial requests for this method.
+    """
+
+    def __init__(self, owner, method, requests=None):
+        self._requests = requests or dict()
+        self.owner = owner
+        self.method = method
+
+    @property
+    def requests(self):
+        """Dictionary of the form: ``{key: alias}``."""
+        return self._requests
+
+    def add_request(
+        self,
+        *,
+        param,
+        alias,
+    ):
+        """Add request info for a metadata.
+
+        Parameters
+        ----------
+        param : str
+            The property for which a request is set.
+
+        alias : str, or {True, False, None}
+            Specifies which metadata should be routed to `param`
+
+            - str: the name (or alias) of metadata given to a meta-estimator that
+              should be routed to this parameter.
+
+            - True: requested
+
+            - False: not requested
+
+            - None: error if passed
+        """
+        if not request_is_alias(alias) and not request_is_valid(alias):
+            raise ValueError(
+                f"The alias you're setting for `{param}` should be either a "
+                "valid identifier or one of {None, True, False}, but given "
+                f"value is: `{alias}`"
+            )
+
+        if alias == param:
+            alias = True
+
+        if alias == UNUSED:
+            if param in self._requests:
+                del self._requests[param]
+            else:
+                raise ValueError(
+                    f"Trying to remove parameter {param} with UNUSED which doesn't"
+                    " exist."
+                )
+        else:
+            self._requests[param] = alias
+
+        return self
+
+    def _get_param_names(self, return_alias):
+        """Get names of all metadata that can be consumed or routed by this method.
+
+        This method returns the names of all metadata, even the ``False``
+        ones.
+
+        Parameters
+        ----------
+        return_alias : bool
+            Controls whether original or aliased names should be returned. If
+            ``False``, aliases are ignored and original names are returned.
+
+        Returns
+        -------
+        names : set of str
+            A set of strings with the names of all parameters.
+        """
+        return set(
+            alias if return_alias and not request_is_valid(alias) else prop
+            for prop, alias in self._requests.items()
+            if not request_is_valid(alias) or alias is not False
+        )
+
+    def _check_warnings(self, *, params):
+        """Check whether metadata is passed which is marked as WARN.
+
+        If any metadata is passed which is marked as WARN, a warning is raised.
+
+        Parameters
+        ----------
+        params : dict
+            The metadata passed to a method.
+        """
+        params = {} if params is None else params
+        warn_params = {
+            prop
+            for prop, alias in self._requests.items()
+            if alias == WARN and prop in params
+        }
+        for param in warn_params:
+            warn(
+                f"Support for {param} has recently been added to this class. "
+                "To maintain backward compatibility, it is ignored now. "
+                f"Using `set_{self.method}_request({param}={{True, False}})` "
+                "on this method of the class, you can set the request value "
+                "to False to silence this warning, or to True to consume and "
+                "use the metadata."
+            )
+
+    def _route_params(self, params, parent, caller):
+        """Prepare the given parameters to be passed to the method.
+
+        The output of this method can be used directly as the input to the
+        corresponding method as extra props.
+
+        Parameters
+        ----------
+        params : dict
+            A dictionary of provided metadata.
+
+        parent : object
+            Parent class object, that routes the metadata.
+
+        caller : str
+            Method from the parent class object, where the metadata is routed from.
+
+        Returns
+        -------
+        params : Bunch
+            A :class:`~sklearn.utils.Bunch` of {prop: value} which can be given to the
+            corresponding method.
+        """
+        self._check_warnings(params=params)
+        unrequested = dict()
+        args = {arg: value for arg, value in params.items() if value is not None}
+        res = Bunch()
+        for prop, alias in self._requests.items():
+            if alias is False or alias == WARN:
+                continue
+            elif alias is True and prop in args:
+                res[prop] = args[prop]
+            elif alias is None and prop in args:
+                unrequested[prop] = args[prop]
+            elif alias in args:
+                res[prop] = args[alias]
+        if unrequested:
+            if self.method in COMPOSITE_METHODS:
+                callee_methods = COMPOSITE_METHODS[self.method]
+            else:
+                callee_methods = [self.method]
+            set_requests_on = "".join(
+                [
+                    f".set_{method}_request({{metadata}}=True/False)"
+                    for method in callee_methods
+                ]
+            )
+            message = (
+                f"[{', '.join([key for key in unrequested])}] are passed but are not"
+                " explicitly set as requested or not requested for"
+                f" {self.owner}.{self.method}, which is used within"
+                f" {parent}.{caller}. Call `{self.owner}"
+                + set_requests_on
+                + "` for each metadata you want to request/ignore. See the"
+                " Metadata Routing User guide"
+                "  for more"
+                " information."
+            )
+            raise UnsetMetadataPassedError(
+                message=message,
+                unrequested_params=unrequested,
+                routed_params=res,
+            )
+        return res
+
+    def _consumes(self, params):
+        """Check whether the given parameters are consumed by this method.
+
+        Parameters
+        ----------
+        params : iterable of str
+            An iterable of parameters to check.
+
+        Returns
+        -------
+        consumed : set of str
+            A set of parameters which are consumed by this method.
+        """
+        params = set(params)
+        res = set()
+        for prop, alias in self._requests.items():
+            if alias is True and prop in params:
+                res.add(prop)
+            elif isinstance(alias, str) and alias in params:
+                res.add(alias)
+        return res
+
+    def _serialize(self):
+        """Serialize the object.
+
+        Returns
+        -------
+        obj : dict
+            A serialized version of the instance in the form of a dictionary.
+        """
+        return self._requests
+
+    def __repr__(self):
+        return str(self._serialize())
+
+    def __str__(self):
+        return str(repr(self))
+
+
+class MetadataRequest:
+    """Contains the metadata request info of a consumer.
+
+    Instances of `MethodMetadataRequest` are used in this class for each
+    available method under `metadatarequest.{method}`.
+
+    Consumer-only classes such as simple estimators return a serialized
+    version of this class as the output of `get_metadata_routing()`.
+
+    .. versionadded:: 1.3
+
+    Parameters
+    ----------
+    owner : str
+        The name of the object to which these requests belong.
+    """
+
+    # this is here for us to use this attribute's value instead of doing
+    # `isinstance` in our checks, so that we avoid issues when people vendor
+    # this file instead of using it directly from scikit-learn.
+    _type = "metadata_request"
+
+    def __init__(self, owner):
+        self.owner = owner
+        for method in SIMPLE_METHODS:
+            setattr(
+                self,
+                method,
+                MethodMetadataRequest(owner=owner, method=method),
+            )
+
+    def consumes(self, method, params):
+        """Check whether the given parameters are consumed by the given method.
+
+        .. versionadded:: 1.4
+
+        Parameters
+        ----------
+        method : str
+            The name of the method to check.
+
+        params : iterable of str
+            An iterable of parameters to check.
+
+        Returns
+        -------
+        consumed : set of str
+            A set of parameters which are consumed by the given method.
+        """
+        return getattr(self, method)._consumes(params=params)
+
+    def __getattr__(self, name):
+        # Called when the default attribute access fails with an AttributeError
+        # (either __getattribute__() raises an AttributeError because name is
+        # not an instance attribute or an attribute in the class tree for self;
+        # or __get__() of a name property raises AttributeError). This method
+        # should either return the (computed) attribute value or raise an
+        # AttributeError exception.
+        # https://docs.python.org/3/reference/datamodel.html#object.__getattr__
+        if name not in COMPOSITE_METHODS:
+            raise AttributeError(
+                f"'{self.__class__.__name__}' object has no attribute '{name}'"
+            )
+
+        requests = {}
+        for method in COMPOSITE_METHODS[name]:
+            mmr = getattr(self, method)
+            existing = set(requests.keys())
+            upcoming = set(mmr.requests.keys())
+            common = existing & upcoming
+            conflicts = [key for key in common if requests[key] != mmr._requests[key]]
+            if conflicts:
+                raise ValueError(
+                    f"Conflicting metadata requests for {', '.join(conflicts)} while"
+                    f" composing the requests for {name}. Metadata with the same name"
+                    f" for methods {', '.join(COMPOSITE_METHODS[name])} should have the"
+                    " same request value."
+                )
+            requests.update(mmr._requests)
+        return MethodMetadataRequest(owner=self.owner, method=name, requests=requests)
+
+    def _get_param_names(self, method, return_alias, ignore_self_request=None):
+        """Get names of all metadata that can be consumed or routed by specified \
+            method.
+
+        This method returns the names of all metadata, even the ``False``
+        ones.
+
+        Parameters
+        ----------
+        method : str
+            The name of the method for which metadata names are requested.
+
+        return_alias : bool
+            Controls whether original or aliased names should be returned. If
+            ``False``, aliases are ignored and original names are returned.
+
+        ignore_self_request : bool
+            Ignored. Present for API compatibility.
+
+        Returns
+        -------
+        names : set of str
+            A set of strings with the names of all parameters.
+        """
+        return getattr(self, method)._get_param_names(return_alias=return_alias)
+
+    def _route_params(self, *, params, method, parent, caller):
+        """Prepare the given parameters to be passed to the method.
+
+        The output of this method can be used directly as the input to the
+        corresponding method as extra keyword arguments to pass metadata.
+
+        Parameters
+        ----------
+        params : dict
+            A dictionary of provided metadata.
+
+        method : str
+            The name of the method for which the parameters are requested and
+            routed.
+
+        parent : object
+            Parent class object, that routes the metadata.
+
+        caller : str
+            Method from the parent class object, where the metadata is routed from.
+
+        Returns
+        -------
+        params : Bunch
+            A :class:`~sklearn.utils.Bunch` of {prop: value} which can be given to the
+            corresponding method.
+        """
+        return getattr(self, method)._route_params(
+            params=params, parent=parent, caller=caller
+        )
+
+    def _check_warnings(self, *, method, params):
+        """Check whether metadata is passed which is marked as WARN.
+
+        If any metadata is passed which is marked as WARN, a warning is raised.
+
+        Parameters
+        ----------
+        method : str
+            The name of the method for which the warnings should be checked.
+
+        params : dict
+            The metadata passed to a method.
+        """
+        getattr(self, method)._check_warnings(params=params)
+
+    def _serialize(self):
+        """Serialize the object.
+
+        Returns
+        -------
+        obj : dict
+            A serialized version of the instance in the form of a dictionary.
+        """
+        output = dict()
+        for method in SIMPLE_METHODS:
+            mmr = getattr(self, method)
+            if len(mmr.requests):
+                output[method] = mmr._serialize()
+        return output
+
+    def __repr__(self):
+        return str(self._serialize())
+
+    def __str__(self):
+        return str(repr(self))
+
+
+# Metadata Request for Routers
+# ============================
+# This section includes all objects required for MetadataRouter which is used
+# in routers, returned by their ``get_metadata_routing``.
+
+# `RouterMappingPair` is used to store a (mapping, router) tuple where `mapping` is a
+# `MethodMapping` object and `router` is the output of `get_metadata_routing`.
+# `MetadataRouter` stores a collection of `RouterMappingPair` objects in its
+# `_route_mappings` attribute.
+RouterMappingPair = namedtuple("RouterMappingPair", ["mapping", "router"])
+
+# `MethodPair` is used to store a single method routing. `MethodMapping` stores a list
+# of `MethodPair` objects in its `_routes` attribute.
+MethodPair = namedtuple("MethodPair", ["caller", "callee"])
+
+
+class MethodMapping:
+    """Stores the mapping between caller and callee methods for a router.
+
+    This class is primarily used in a ``get_metadata_routing()`` of a router
+    object when defining the mapping between the router's methods and a sub-object (a
+    sub-estimator or a scorer).
+
+    Iterating through an instance of this class yields
+    ``MethodPair(caller, callee)`` instances.
+
+    .. versionadded:: 1.3
+    """
+
+    def __init__(self):
+        self._routes = []
+
+    def __iter__(self):
+        return iter(self._routes)
+
+    def add(self, *, caller, callee):
+        """Add a method mapping.
+
+        Parameters
+        ----------
+
+        caller : str
+            Parent estimator's method name in which the ``callee`` is called.
+
+        callee : str
+            Child object's method name. This method is called in ``caller``.
+
+        Returns
+        -------
+        self : MethodMapping
+            Returns self.
+        """
+        if caller not in METHODS:
+            raise ValueError(
+                f"Given caller:{caller} is not a valid method. Valid methods are:"
+                f" {METHODS}"
+            )
+        if callee not in METHODS:
+            raise ValueError(
+                f"Given callee:{callee} is not a valid method. Valid methods are:"
+                f" {METHODS}"
+            )
+        self._routes.append(MethodPair(caller=caller, callee=callee))
+        return self
+
+    def _serialize(self):
+        """Serialize the object.
+
+        Returns
+        -------
+        obj : list
+            A serialized version of the instance in the form of a list.
+        """
+        result = list()
+        for route in self._routes:
+            result.append({"caller": route.caller, "callee": route.callee})
+        return result
+
+    def __repr__(self):
+        return str(self._serialize())
+
+    def __str__(self):
+        return str(repr(self))
+
+
+class MetadataRouter:
+    """Stores and handles metadata routing for a router object.
+
+    This class is used by router objects to store and handle metadata routing.
+    Routing information is stored as a dictionary of the form ``{"object_name":
+    RouteMappingPair(method_mapping, routing_info)}``, where ``method_mapping``
+    is an instance of :class:`~sklearn.utils.metadata_routing.MethodMapping` and
+    ``routing_info`` is either a
+    :class:`~sklearn.utils.metadata_routing.MetadataRequest` or a
+    :class:`~sklearn.utils.metadata_routing.MetadataRouter` instance.
+
+    .. versionadded:: 1.3
+
+    Parameters
+    ----------
+    owner : str
+        The name of the object to which these requests belong.
+    """
+
+    # this is here for us to use this attribute's value instead of doing
+    # `isinstance`` in our checks, so that we avoid issues when people vendor
+    # this file instead of using it directly from scikit-learn.
+    _type = "metadata_router"
+
+    def __init__(self, owner):
+        self._route_mappings = dict()
+        # `_self_request` is used if the router is also a consumer.
+        # _self_request, (added using `add_self_request()`) is treated
+        # differently from the other objects which are stored in
+        # _route_mappings.
+        self._self_request = None
+        self.owner = owner
+
+    def add_self_request(self, obj):
+        """Add `self` (as a consumer) to the routing.
+
+        This method is used if the router is also a consumer, and hence the
+        router itself needs to be included in the routing. The passed object
+        can be an estimator or a
+        :class:`~sklearn.utils.metadata_routing.MetadataRequest`.
+
+        A router should add itself using this method instead of `add` since it
+        should be treated differently than the other objects to which metadata
+        is routed by the router.
+
+        Parameters
+        ----------
+        obj : object
+            This is typically the router instance, i.e. `self` in a
+            ``get_metadata_routing()`` implementation. It can also be a
+            ``MetadataRequest`` instance.
+
+        Returns
+        -------
+        self : MetadataRouter
+            Returns `self`.
+        """
+        if getattr(obj, "_type", None) == "metadata_request":
+            self._self_request = deepcopy(obj)
+        elif hasattr(obj, "_get_metadata_request"):
+            self._self_request = deepcopy(obj._get_metadata_request())
+        else:
+            raise ValueError(
+                "Given `obj` is neither a `MetadataRequest` nor does it implement the"
+                " required API. Inheriting from `BaseEstimator` implements the required"
+                " API."
+            )
+        return self
+
+    def add(self, *, method_mapping, **objs):
+        """Add named objects with their corresponding method mapping.
+
+        Parameters
+        ----------
+        method_mapping : MethodMapping
+            The mapping between the child and the parent's methods.
+
+        **objs : dict
+            A dictionary of objects from which metadata is extracted by calling
+            :func:`~sklearn.utils.metadata_routing.get_routing_for_object` on them.
+
+        Returns
+        -------
+        self : MetadataRouter
+            Returns `self`.
+        """
+        method_mapping = deepcopy(method_mapping)
+
+        for name, obj in objs.items():
+            self._route_mappings[name] = RouterMappingPair(
+                mapping=method_mapping, router=get_routing_for_object(obj)
+            )
+        return self
+
+    def consumes(self, method, params):
+        """Check whether the given parameters are consumed by the given method.
+
+        .. versionadded:: 1.4
+
+        Parameters
+        ----------
+        method : str
+            The name of the method to check.
+
+        params : iterable of str
+            An iterable of parameters to check.
+
+        Returns
+        -------
+        consumed : set of str
+            A set of parameters which are consumed by the given method.
+        """
+        res = set()
+        if self._self_request:
+            res = res | self._self_request.consumes(method=method, params=params)
+
+        for _, route_mapping in self._route_mappings.items():
+            for caller, callee in route_mapping.mapping:
+                if caller == method:
+                    res = res | route_mapping.router.consumes(
+                        method=callee, params=params
+                    )
+
+        return res
+
+    def _get_param_names(self, *, method, return_alias, ignore_self_request):
+        """Get names of all metadata that can be consumed or routed by specified \
+            method.
+
+        This method returns the names of all metadata, even the ``False``
+        ones.
+
+        Parameters
+        ----------
+        method : str
+            The name of the method for which metadata names are requested.
+
+        return_alias : bool
+            Controls whether original or aliased names should be returned,
+            which only applies to the stored `self`. If no `self` routing
+            object is stored, this parameter has no effect.
+
+        ignore_self_request : bool
+            If `self._self_request` should be ignored. This is used in `_route_params`.
+            If ``True``, ``return_alias`` has no effect.
+
+        Returns
+        -------
+        names : set of str
+            A set of strings with the names of all parameters.
+        """
+        res = set()
+        if self._self_request and not ignore_self_request:
+            res = res.union(
+                self._self_request._get_param_names(
+                    method=method, return_alias=return_alias
+                )
+            )
+
+        for name, route_mapping in self._route_mappings.items():
+            for caller, callee in route_mapping.mapping:
+                if caller == method:
+                    res = res.union(
+                        route_mapping.router._get_param_names(
+                            method=callee, return_alias=True, ignore_self_request=False
+                        )
+                    )
+        return res
+
+    def _route_params(self, *, params, method, parent, caller):
+        """Prepare the given parameters to be passed to the method.
+
+        This is used when a router is used as a child object of another router.
+        The parent router then passes all parameters understood by the child
+        object to it and delegates their validation to the child.
+
+        The output of this method can be used directly as the input to the
+        corresponding method as extra props.
+
+        Parameters
+        ----------
+        params : dict
+            A dictionary of provided metadata.
+
+        method : str
+            The name of the method for which the parameters are requested and
+            routed.
+
+        parent : object
+            Parent class object, that routes the metadata.
+
+        caller : str
+            Method from the parent class object, where the metadata is routed from.
+
+        Returns
+        -------
+        params : Bunch
+            A :class:`~sklearn.utils.Bunch` of {prop: value} which can be given to the
+            corresponding method.
+        """
+        res = Bunch()
+        if self._self_request:
+            res.update(
+                self._self_request._route_params(
+                    params=params,
+                    method=method,
+                    parent=parent,
+                    caller=caller,
+                )
+            )
+
+        param_names = self._get_param_names(
+            method=method, return_alias=True, ignore_self_request=True
+        )
+        child_params = {
+            key: value for key, value in params.items() if key in param_names
+        }
+        for key in set(res.keys()).intersection(child_params.keys()):
+            # conflicts are okay if the passed objects are the same, but it's
+            # an issue if they're different objects.
+            if child_params[key] is not res[key]:
+                raise ValueError(
+                    f"In {self.owner}, there is a conflict on {key} between what is"
+                    " requested for this estimator and what is requested by its"
+                    " children. You can resolve this conflict by using an alias for"
+                    " the child estimator(s) requested metadata."
+                )
+
+        res.update(child_params)
+        return res
+
+    def route_params(self, *, caller, params):
+        """Return the input parameters requested by child objects.
+
+        The output of this method is a :class:`~sklearn.utils.Bunch`, which includes the
+        metadata for all methods of each child object that is used in the router's
+        `caller` method.
+
+        If the router is also a consumer, it also checks for warnings of
+        `self`'s/consumer's requested metadata.
+
+        Parameters
+        ----------
+        caller : str
+            The name of the method for which the parameters are requested and
+            routed. If called inside the :term:`fit` method of a router, it
+            would be `"fit"`.
+
+        params : dict
+            A dictionary of provided metadata.
+
+        Returns
+        -------
+        params : Bunch
+            A :class:`~sklearn.utils.Bunch` of the form
+            ``{"object_name": {"method_name": {params: value}}}`` which can be
+            used to pass the required metadata to corresponding methods or
+            corresponding child objects.
+        """
+        if self._self_request:
+            self._self_request._check_warnings(params=params, method=caller)
+
+        res = Bunch()
+        for name, route_mapping in self._route_mappings.items():
+            router, mapping = route_mapping.router, route_mapping.mapping
+
+            res[name] = Bunch()
+            for _caller, _callee in mapping:
+                if _caller == caller:
+                    res[name][_callee] = router._route_params(
+                        params=params,
+                        method=_callee,
+                        parent=self.owner,
+                        caller=caller,
+                    )
+        return res
+
+    def validate_metadata(self, *, method, params):
+        """Validate given metadata for a method.
+
+        This raises a ``TypeError`` if some of the passed metadata are not
+        understood by child objects.
+
+        Parameters
+        ----------
+        method : str
+            The name of the method for which the parameters are requested and
+            routed. If called inside the :term:`fit` method of a router, it
+            would be `"fit"`.
+
+        params : dict
+            A dictionary of provided metadata.
+        """
+        param_names = self._get_param_names(
+            method=method, return_alias=False, ignore_self_request=False
+        )
+        if self._self_request:
+            self_params = self._self_request._get_param_names(
+                method=method, return_alias=False
+            )
+        else:
+            self_params = set()
+        extra_keys = set(params.keys()) - param_names - self_params
+        if extra_keys:
+            raise TypeError(
+                f"{self.owner}.{method} got unexpected argument(s) {extra_keys}, which"
+                " are not routed to any object."
+            )
+
+    def _serialize(self):
+        """Serialize the object.
+
+        Returns
+        -------
+        obj : dict
+            A serialized version of the instance in the form of a dictionary.
+        """
+        res = dict()
+        if self._self_request:
+            res["$self_request"] = self._self_request._serialize()
+        for name, route_mapping in self._route_mappings.items():
+            res[name] = dict()
+            res[name]["mapping"] = route_mapping.mapping._serialize()
+            res[name]["router"] = route_mapping.router._serialize()
+
+        return res
+
+    def __iter__(self):
+        if self._self_request:
+            method_mapping = MethodMapping()
+            for method in METHODS:
+                method_mapping.add(caller=method, callee=method)
+            yield (
+                "$self_request",
+                RouterMappingPair(mapping=method_mapping, router=self._self_request),
+            )
+        for name, route_mapping in self._route_mappings.items():
+            yield (name, route_mapping)
+
+    def __repr__(self):
+        return str(self._serialize())
+
+    def __str__(self):
+        return str(repr(self))
+
+
+def get_routing_for_object(obj=None):
+    """Get a ``Metadata{Router, Request}`` instance from the given object.
+
+    This function returns a
+    :class:`~sklearn.utils.metadata_routing.MetadataRouter` or a
+    :class:`~sklearn.utils.metadata_routing.MetadataRequest` from the given input.
+
+    This function always returns a copy or an instance constructed from the
+    input, such that changing the output of this function will not change the
+    original object.
+
+    .. versionadded:: 1.3
+
+    Parameters
+    ----------
+    obj : object
+        - If the object provides a `get_metadata_routing` method, return a copy
+            of the output of that method.
+        - If the object is already a
+            :class:`~sklearn.utils.metadata_routing.MetadataRequest` or a
+            :class:`~sklearn.utils.metadata_routing.MetadataRouter`, return a copy
+            of that.
+        - Returns an empty :class:`~sklearn.utils.metadata_routing.MetadataRequest`
+            otherwise.
+
+    Returns
+    -------
+    obj : MetadataRequest or MetadataRouting
+        A ``MetadataRequest`` or a ``MetadataRouting`` taken or created from
+        the given object.
+    """
+    # doing this instead of a try/except since an AttributeError could be raised
+    # for other reasons.
+    if hasattr(obj, "get_metadata_routing"):
+        return deepcopy(obj.get_metadata_routing())
+
+    elif getattr(obj, "_type", None) in ["metadata_request", "metadata_router"]:
+        return deepcopy(obj)
+
+    return MetadataRequest(owner=None)
+
+
+# Request method
+# ==============
+# This section includes what's needed for the request method descriptor and
+# their dynamic generation in a meta class.
+
+# These strings are used to dynamically generate the docstrings for
+# set_{method}_request methods.
+REQUESTER_DOC = """        Request metadata passed to the ``{method}`` method.
+
+        Note that this method is only relevant if
+        ``enable_metadata_routing=True`` (see :func:`sklearn.set_config`).
+        Please see :ref:`User Guide ` on how the routing
+        mechanism works.
+
+        The options for each parameter are:
+
+        - ``True``: metadata is requested, and \
+passed to ``{method}`` if provided. The request is ignored if \
+metadata is not provided.
+
+        - ``False``: metadata is not requested and the meta-estimator \
+will not pass it to ``{method}``.
+
+        - ``None``: metadata is not requested, and the meta-estimator \
+will raise an error if the user provides it.
+
+        - ``str``: metadata should be passed to the meta-estimator with \
+this given alias instead of the original name.
+
+        The default (``sklearn.utils.metadata_routing.UNCHANGED``) retains the
+        existing request. This allows you to change the request for some
+        parameters and not others.
+
+        .. versionadded:: 1.3
+
+        .. note::
+            This method is only relevant if this estimator is used as a
+            sub-estimator of a meta-estimator, e.g. used inside a
+            :class:`~sklearn.pipeline.Pipeline`. Otherwise it has no effect.
+
+        Parameters
+        ----------
+"""
+REQUESTER_DOC_PARAM = """        {metadata} : str, True, False, or None, \
+                    default=sklearn.utils.metadata_routing.UNCHANGED
+            Metadata routing for ``{metadata}`` parameter in ``{method}``.
+
+"""
+REQUESTER_DOC_RETURN = """        Returns
+        -------
+        self : object
+            The updated object.
+"""
+
+
+class RequestMethod:
+    """
+    A descriptor for request methods.
+
+    .. versionadded:: 1.3
+
+    Parameters
+    ----------
+    name : str
+        The name of the method for which the request function should be
+        created, e.g. ``"fit"`` would create a ``set_fit_request`` function.
+
+    keys : list of str
+        A list of strings which are accepted parameters by the created
+        function, e.g. ``["sample_weight"]`` if the corresponding method
+        accepts it as a metadata.
+
+    validate_keys : bool, default=True
+        Whether to check if the requested parameters fit the actual parameters
+        of the method.
+
+    Notes
+    -----
+    This class is a descriptor [1]_ and uses PEP-362 to set the signature of
+    the returned function [2]_.
+
+    References
+    ----------
+    .. [1] https://docs.python.org/3/howto/descriptor.html
+
+    .. [2] https://www.python.org/dev/peps/pep-0362/
+    """
+
+    def __init__(self, name, keys, validate_keys=True):
+        self.name = name
+        self.keys = keys
+        self.validate_keys = validate_keys
+
+    def __get__(self, instance, owner):
+        # we would want to have a method which accepts only the expected args
+        def func(*args, **kw):
+            """Updates the request for provided parameters
+
+            This docstring is overwritten below.
+            See REQUESTER_DOC for expected functionality
+            """
+            if not _routing_enabled():
+                raise RuntimeError(
+                    "This method is only available when metadata routing is enabled."
+                    " You can enable it using"
+                    " sklearn.set_config(enable_metadata_routing=True)."
+                )
+
+            if self.validate_keys and (set(kw) - set(self.keys)):
+                raise TypeError(
+                    f"Unexpected args: {set(kw) - set(self.keys)} in {self.name}. "
+                    f"Accepted arguments are: {set(self.keys)}"
+                )
+
+            # This makes it possible to use the decorated method as an unbound method,
+            # for instance when monkeypatching.
+            # https://github.com/scikit-learn/scikit-learn/issues/28632
+            if instance is None:
+                _instance = args[0]
+                args = args[1:]
+            else:
+                _instance = instance
+
+            # Replicating python's behavior when positional args are given other than
+            # `self`, and `self` is only allowed if this method is unbound.
+            if args:
+                raise TypeError(
+                    f"set_{self.name}_request() takes 0 positional argument but"
+                    f" {len(args)} were given"
+                )
+
+            requests = _instance._get_metadata_request()
+            method_metadata_request = getattr(requests, self.name)
+
+            for prop, alias in kw.items():
+                if alias is not UNCHANGED:
+                    method_metadata_request.add_request(param=prop, alias=alias)
+            _instance._metadata_request = requests
+
+            return _instance
+
+        # Now we set the relevant attributes of the function so that it seems
+        # like a normal method to the end user, with known expected arguments.
+        func.__name__ = f"set_{self.name}_request"
+        params = [
+            inspect.Parameter(
+                name="self",
+                kind=inspect.Parameter.POSITIONAL_OR_KEYWORD,
+                annotation=owner,
+            )
+        ]
+        params.extend(
+            [
+                inspect.Parameter(
+                    k,
+                    inspect.Parameter.KEYWORD_ONLY,
+                    default=UNCHANGED,
+                    annotation=Optional[Union[bool, None, str]],
+                )
+                for k in self.keys
+            ]
+        )
+        func.__signature__ = inspect.Signature(
+            params,
+            return_annotation=owner,
+        )
+        doc = REQUESTER_DOC.format(method=self.name)
+        for metadata in self.keys:
+            doc += REQUESTER_DOC_PARAM.format(metadata=metadata, method=self.name)
+        doc += REQUESTER_DOC_RETURN
+        func.__doc__ = doc
+        return func
+
+
+class _MetadataRequester:
+    """Mixin class for adding metadata request functionality.
+
+    ``BaseEstimator`` inherits from this Mixin.
+
+    .. versionadded:: 1.3
+    """
+
+    if TYPE_CHECKING:  # pragma: no cover
+        # This code is never run in runtime, but it's here for type checking.
+        # Type checkers fail to understand that the `set_{method}_request`
+        # methods are dynamically generated, and they complain that they are
+        # not defined. We define them here to make type checkers happy.
+        # During type checking analyzers assume this to be True.
+        # The following list of defined methods mirrors the list of methods
+        # in SIMPLE_METHODS.
+        # fmt: off
+        def set_fit_request(self, **kwargs): pass
+        def set_partial_fit_request(self, **kwargs): pass
+        def set_predict_request(self, **kwargs): pass
+        def set_predict_proba_request(self, **kwargs): pass
+        def set_predict_log_proba_request(self, **kwargs): pass
+        def set_decision_function_request(self, **kwargs): pass
+        def set_score_request(self, **kwargs): pass
+        def set_split_request(self, **kwargs): pass
+        def set_transform_request(self, **kwargs): pass
+        def set_inverse_transform_request(self, **kwargs): pass
+        # fmt: on
+
+    def __init_subclass__(cls, **kwargs):
+        """Set the ``set_{method}_request`` methods.
+
+        This uses PEP-487 [1]_ to set the ``set_{method}_request`` methods. It
+        looks for the information available in the set default values which are
+        set using ``__metadata_request__*`` class attributes, or inferred
+        from method signatures.
+
+        The ``__metadata_request__*`` class attributes are used when a method
+        does not explicitly accept a metadata through its arguments or if the
+        developer would like to specify a request value for those metadata
+        which are different from the default ``None``.
+
+        References
+        ----------
+        .. [1] https://www.python.org/dev/peps/pep-0487
+        """
+        try:
+            requests = cls._get_default_requests()
+        except Exception:
+            # if there are any issues in the default values, it will be raised
+            # when ``get_metadata_routing`` is called. Here we are going to
+            # ignore all the issues such as bad defaults etc.
+            super().__init_subclass__(**kwargs)
+            return
+
+        for method in SIMPLE_METHODS:
+            mmr = getattr(requests, method)
+            # set ``set_{method}_request`` methods
+            if not len(mmr.requests):
+                continue
+            setattr(
+                cls,
+                f"set_{method}_request",
+                RequestMethod(method, sorted(mmr.requests.keys())),
+            )
+        super().__init_subclass__(**kwargs)
+
+    @classmethod
+    def _build_request_for_signature(cls, router, method):
+        """Build the `MethodMetadataRequest` for a method using its signature.
+
+        This method takes all arguments from the method signature and uses
+        ``None`` as their default request value, except ``X``, ``y``, ``Y``,
+        ``Xt``, ``yt``, ``*args``, and ``**kwargs``.
+
+        Parameters
+        ----------
+        router : MetadataRequest
+            The parent object for the created `MethodMetadataRequest`.
+        method : str
+            The name of the method.
+
+        Returns
+        -------
+        method_request : MethodMetadataRequest
+            The prepared request using the method's signature.
+        """
+        mmr = MethodMetadataRequest(owner=cls.__name__, method=method)
+        # Here we use `isfunction` instead of `ismethod` because calling `getattr`
+        # on a class instead of an instance returns an unbound function.
+        if not hasattr(cls, method) or not inspect.isfunction(getattr(cls, method)):
+            return mmr
+        # ignore the first parameter of the method, which is usually "self"
+        params = list(inspect.signature(getattr(cls, method)).parameters.items())[1:]
+        for pname, param in params:
+            if pname in {"X", "y", "Y", "Xt", "yt"}:
+                continue
+            if param.kind in {param.VAR_POSITIONAL, param.VAR_KEYWORD}:
+                continue
+            mmr.add_request(
+                param=pname,
+                alias=None,
+            )
+        return mmr
+
+    @classmethod
+    def _get_default_requests(cls):
+        """Collect default request values.
+
+        This method combines the information present in ``__metadata_request__*``
+        class attributes, as well as determining request keys from method
+        signatures.
+        """
+        requests = MetadataRequest(owner=cls.__name__)
+
+        for method in SIMPLE_METHODS:
+            setattr(
+                requests,
+                method,
+                cls._build_request_for_signature(router=requests, method=method),
+            )
+
+        # Then overwrite those defaults with the ones provided in
+        # __metadata_request__* attributes. Defaults set in
+        # __metadata_request__* attributes take precedence over signature
+        # sniffing.
+
+        # need to go through the MRO since this is a class attribute and
+        # ``vars`` doesn't report the parent class attributes. We go through
+        # the reverse of the MRO so that child classes have precedence over
+        # their parents.
+        substr = "__metadata_request__"
+        for base_class in reversed(inspect.getmro(cls)):
+            for attr, value in vars(base_class).items():
+                if substr not in attr:
+                    continue
+                # we don't check for attr.startswith() since python prefixes attrs
+                # starting with __ with the `_ClassName`.
+                method = attr[attr.index(substr) + len(substr) :]
+                for prop, alias in value.items():
+                    # Here we add request values specified via those class attributes
+                    # to the `MetadataRequest` object. Adding a request which already
+                    # exists will override the previous one. Since we go through the
+                    # MRO in reverse order, the one specified by the lowest most classes
+                    # in the inheritance tree are the ones which take effect.
+                    getattr(requests, method).add_request(param=prop, alias=alias)
+
+        return requests
+
+    def _get_metadata_request(self):
+        """Get requested data properties.
+
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
+
+        Returns
+        -------
+        request : MetadataRequest
+            A :class:`~sklearn.utils.metadata_routing.MetadataRequest` instance.
+        """
+        if hasattr(self, "_metadata_request"):
+            requests = get_routing_for_object(self._metadata_request)
+        else:
+            requests = self._get_default_requests()
+
+        return requests
+
+    def get_metadata_routing(self):
+        """Get metadata routing of this object.
+
+        Please check :ref:`User Guide ` on how the routing
+        mechanism works.
+
+        Returns
+        -------
+        routing : MetadataRequest
+            A :class:`~sklearn.utils.metadata_routing.MetadataRequest` encapsulating
+            routing information.
+        """
+        return self._get_metadata_request()
+
+
+# Process Routing in Routers
+# ==========================
+# This is almost always the only method used in routers to process and route
+# given metadata. This is to minimize the boilerplate required in routers.
+
+
+# Here the first two arguments are positional only which makes everything
+# passed as keyword argument a metadata. The first two args also have an `_`
+# prefix to reduce the chances of name collisions with the passed metadata, and
+# since they're positional only, users will never type those underscores.
+def process_routing(_obj, _method, /, **kwargs):
+    """Validate and route input parameters.
+
+    This function is used inside a router's method, e.g. :term:`fit`,
+    to validate the metadata and handle the routing.
+
+    Assuming this signature of a router's fit method:
+    ``fit(self, X, y, sample_weight=None, **fit_params)``,
+    a call to this function would be:
+    ``process_routing(self, "fit", sample_weight=sample_weight, **fit_params)``.
+
+    Note that if routing is not enabled and ``kwargs`` is empty, then it
+    returns an empty routing where ``process_routing(...).ANYTHING.ANY_METHOD``
+    is always an empty dictionary.
+
+    .. versionadded:: 1.3
+
+    Parameters
+    ----------
+    _obj : object
+        An object implementing ``get_metadata_routing``. Typically a
+        meta-estimator.
+
+    _method : str
+        The name of the router's method in which this function is called.
+
+    **kwargs : dict
+        Metadata to be routed.
+
+    Returns
+    -------
+    routed_params : Bunch
+        A :class:`~utils.Bunch` of the form ``{"object_name": {"method_name":
+        {params: value}}}`` which can be used to pass the required metadata to
+        A :class:`~sklearn.utils.Bunch` of the form ``{"object_name": {"method_name":
+        {params: value}}}`` which can be used to pass the required metadata to
+        corresponding methods or corresponding child objects. The object names
+        are those defined in `obj.get_metadata_routing()`.
+    """
+    if not kwargs:
+        # If routing is not enabled and kwargs are empty, then we don't have to
+        # try doing any routing, we can simply return a structure which returns
+        # an empty dict on routed_params.ANYTHING.ANY_METHOD.
+        class EmptyRequest:
+            def get(self, name, default=None):
+                return Bunch(**{method: dict() for method in METHODS})
+
+            def __getitem__(self, name):
+                return Bunch(**{method: dict() for method in METHODS})
+
+            def __getattr__(self, name):
+                return Bunch(**{method: dict() for method in METHODS})
+
+        return EmptyRequest()
+
+    if not (hasattr(_obj, "get_metadata_routing") or isinstance(_obj, MetadataRouter)):
+        raise AttributeError(
+            f"The given object ({_obj.__class__.__name__!r}) needs to either"
+            " implement the routing method `get_metadata_routing` or be a"
+            " `MetadataRouter` instance."
+        )
+    if _method not in METHODS:
+        raise TypeError(
+            f"Can only route and process input on these methods: {METHODS}, "
+            f"while the passed method is: {_method}."
+        )
+
+    request_routing = get_routing_for_object(_obj)
+    request_routing.validate_metadata(params=kwargs, method=_method)
+    routed_params = request_routing.route_params(params=kwargs, caller=_method)
+
+    return routed_params
diff --git a/sklearn/utils/_missing.py b/sklearn/utils/_missing.py
new file mode 100644
index 0000000000000..daeb9ba68cc1c
--- /dev/null
+++ b/sklearn/utils/_missing.py
@@ -0,0 +1,68 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+import math
+import numbers
+from contextlib import suppress
+
+
+def is_scalar_nan(x):
+    """Test if x is NaN.
+
+    This function is meant to overcome the issue that np.isnan does not allow
+    non-numerical types as input, and that np.nan is not float('nan').
+
+    Parameters
+    ----------
+    x : any type
+        Any scalar value.
+
+    Returns
+    -------
+    bool
+        Returns true if x is NaN, and false otherwise.
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from sklearn.utils._missing import is_scalar_nan
+    >>> is_scalar_nan(np.nan)
+    True
+    >>> is_scalar_nan(float("nan"))
+    True
+    >>> is_scalar_nan(None)
+    False
+    >>> is_scalar_nan("")
+    False
+    >>> is_scalar_nan([np.nan])
+    False
+    """
+    return (
+        not isinstance(x, numbers.Integral)
+        and isinstance(x, numbers.Real)
+        and math.isnan(x)
+    )
+
+
+def is_pandas_na(x):
+    """Test if x is pandas.NA.
+
+    We intentionally do not use this function to return `True` for `pd.NA` in
+    `is_scalar_nan`, because estimators that support `pd.NA` are the exception
+    rather than the rule at the moment. When `pd.NA` is more universally
+    supported, we may reconsider this decision.
+
+    Parameters
+    ----------
+    x : any type
+
+    Returns
+    -------
+    boolean
+    """
+    with suppress(ImportError):
+        from pandas import NA
+
+        return x is NA
+
+    return False
diff --git a/sklearn/utils/_mocking.py b/sklearn/utils/_mocking.py
index 00109051d035e..87fb4106f3b59 100644
--- a/sklearn/utils/_mocking.py
+++ b/sklearn/utils/_mocking.py
@@ -1,7 +1,18 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
 import numpy as np
 
 from ..base import BaseEstimator, ClassifierMixin
-from .validation import _num_samples, check_array, check_is_fitted
+from ..utils._metadata_requests import RequestMethod
+from .metaestimators import available_if
+from .validation import (
+    _check_sample_weight,
+    _num_samples,
+    check_array,
+    check_is_fitted,
+    check_random_state,
+)
 
 
 class ArraySlicingWrapper:
@@ -24,6 +35,7 @@ class MockDataFrame:
     ----------
     array
     """
+
     # have shape and length but don't support indexing.
 
     def __init__(self, array):
@@ -49,6 +61,9 @@ def __eq__(self, other):
     def __ne__(self, other):
         return not self == other
 
+    def take(self, indices, axis=0):
+        return MockDataFrame(self.array.take(indices, axis=axis))
+
 
 class CheckingClassifier(ClassifierMixin, BaseEstimator):
     """Dummy classifier to test pipelining and meta-estimators.
@@ -64,10 +79,12 @@ class CheckingClassifier(ClassifierMixin, BaseEstimator):
     ----------
     check_y, check_X : callable, default=None
         The callable used to validate `X` and `y`. These callable should return
-        a bool where `False` will trigger an `AssertionError`.
+        a bool where `False` will trigger an `AssertionError`. If `None`, the
+        data is not validated. Default is `None`.
 
     check_y_params, check_X_params : dict, default=None
-        The optional parameters to pass to `check_X` and `check_y`.
+        The optional parameters to pass to `check_X` and `check_y`. If `None`,
+        then no parameters are passed in.
 
     methods_to_check : "all" or list of str, default="all"
         The methods in which the checks should be applied. By default,
@@ -78,6 +95,9 @@ class CheckingClassifier(ClassifierMixin, BaseEstimator):
         A `foo` param. When `foo > 1`, the output of :meth:`score` will be 1
         otherwise it is 0.
 
+    expected_sample_weight : bool, default=False
+        Whether to check if a valid `sample_weight` was passed to `fit`.
+
     expected_fit_params : list of str, default=None
         A list of the expected parameters given when calling `fit`.
 
@@ -111,16 +131,28 @@ class CheckingClassifier(ClassifierMixin, BaseEstimator):
     CheckingClassifier(...)
     """
 
-    def __init__(self, *, check_y=None, check_y_params=None,
-                 check_X=None, check_X_params=None, methods_to_check="all",
-                 foo_param=0, expected_fit_params=None):
+    def __init__(
+        self,
+        *,
+        check_y=None,
+        check_y_params=None,
+        check_X=None,
+        check_X_params=None,
+        methods_to_check="all",
+        foo_param=0,
+        expected_sample_weight=None,
+        expected_fit_params=None,
+        random_state=None,
+    ):
         self.check_y = check_y
         self.check_y_params = check_y_params
         self.check_X = check_X
         self.check_X_params = check_X_params
         self.methods_to_check = methods_to_check
         self.foo_param = foo_param
+        self.expected_sample_weight = expected_sample_weight
         self.expected_fit_params = expected_fit_params
+        self.random_state = random_state
 
     def _check_X_y(self, X, y=None, should_be_fitted=True):
         """Validate X and y and make extra check.
@@ -129,8 +161,10 @@ def _check_X_y(self, X, y=None, should_be_fitted=True):
         ----------
         X : array-like of shape (n_samples, n_features)
             The data set.
+            `X` is checked only if `check_X` is not `None` (default is None).
         y : array-like of shape (n_samples), default=None
-            The corresponding target, by default None.
+            The corresponding target, by default `None`.
+            `y` is checked only if `check_y` is not `None` (default is None).
         should_be_fitted : bool, default=True
             Whether or not the classifier should be already fitted.
             By default True.
@@ -157,20 +191,23 @@ def _check_X_y(self, X, y=None, should_be_fitted=True):
                 y = checked_y
         return X, y
 
-    def fit(self, X, y, **fit_params):
+    def fit(self, X, y, sample_weight=None, **fit_params):
         """Fit classifier.
 
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
-            Training vector, where n_samples is the number of samples and
-            n_features is the number of features.
+            Training vector, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
 
         y : array-like of shape (n_samples, n_outputs) or (n_samples,), \
                 default=None
             Target relative to X for classification or regression;
             None for unsupervised learning.
 
+        sample_weight : array-like of shape (n_samples,), default=None
+            Sample weights. If None, then samples are equally weighted.
+
         **fit_params : dict of string -> object
             Parameters passed to the ``fit`` method of the estimator
 
@@ -182,21 +219,23 @@ def fit(self, X, y, **fit_params):
         if self.methods_to_check == "all" or "fit" in self.methods_to_check:
             X, y = self._check_X_y(X, y, should_be_fitted=False)
         self.n_features_in_ = np.shape(X)[1]
-        self.classes_ = np.unique(
-            check_array(y, ensure_2d=False, allow_nd=True)
-        )
+        self.classes_ = np.unique(check_array(y, ensure_2d=False, allow_nd=True))
         if self.expected_fit_params:
             missing = set(self.expected_fit_params) - set(fit_params)
             if missing:
                 raise AssertionError(
-                    f'Expected fit parameter(s) {list(missing)} not seen.'
+                    f"Expected fit parameter(s) {list(missing)} not seen."
                 )
             for key, value in fit_params.items():
                 if _num_samples(value) != _num_samples(X):
                     raise AssertionError(
-                        f'Fit parameter {key} has length {_num_samples(value)}'
-                        f'; expected {_num_samples(X)}.'
+                        f"Fit parameter {key} has length {_num_samples(value)}"
+                        f"; expected {_num_samples(X)}."
                     )
+        if self.expected_sample_weight:
+            if sample_weight is None:
+                raise AssertionError("Expected sample_weight to be passed")
+            _check_sample_weight(sample_weight, X)
 
         return self
 
@@ -211,12 +250,12 @@ def predict(self, X):
         Returns
         -------
         preds : ndarray of shape (n_samples,)
-            Predictions of the first class seens in `classes_`.
+            Predictions of the first class seen in `classes_`.
         """
-        if (self.methods_to_check == "all" or
-                "predict" in self.methods_to_check):
+        if self.methods_to_check == "all" or "predict" in self.methods_to_check:
             X, y = self._check_X_y(X)
-        return self.classes_[np.zeros(_num_samples(X), dtype=int)]
+        rng = check_random_state(self.random_state)
+        return rng.choice(self.classes_, size=_num_samples(X))
 
     def predict_proba(self, X):
         """Predict probabilities for each class.
@@ -234,11 +273,12 @@ def predict_proba(self, X):
         proba : ndarray of shape (n_samples, n_classes)
             The probabilities for each sample and class.
         """
-        if (self.methods_to_check == "all" or
-                "predict_proba" in self.methods_to_check):
+        if self.methods_to_check == "all" or "predict_proba" in self.methods_to_check:
             X, y = self._check_X_y(X)
-        proba = np.zeros((_num_samples(X), len(self.classes_)))
-        proba[:, 0] = 1
+        rng = check_random_state(self.random_state)
+        proba = rng.randn(_num_samples(X), len(self.classes_))
+        proba = np.abs(proba, out=proba)
+        proba /= np.sum(proba, axis=1)[:, np.newaxis]
         return proba
 
     def decision_function(self, X):
@@ -255,17 +295,18 @@ def decision_function(self, X):
                 else (n_samples, n_classes)
             Confidence score.
         """
-        if (self.methods_to_check == "all" or
-                "decision_function" in self.methods_to_check):
+        if (
+            self.methods_to_check == "all"
+            or "decision_function" in self.methods_to_check
+        ):
             X, y = self._check_X_y(X)
+        rng = check_random_state(self.random_state)
         if len(self.classes_) == 2:
             # for binary classifier, the confidence score is related to
             # classes_[1] and therefore should be null.
-            return np.zeros(_num_samples(X))
+            return rng.randn(_num_samples(X))
         else:
-            decision = np.zeros((_num_samples(X), len(self.classes_)))
-            decision[:, 0] = 1
-            return decision
+            return rng.randn(_num_samples(X), len(self.classes_))
 
     def score(self, X=None, Y=None):
         """Fake score.
@@ -273,8 +314,8 @@ def score(self, X=None, Y=None):
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
-            Input data, where n_samples is the number of samples and
-            n_features is the number of features.
+            Input data, where `n_samples` is the number of samples and
+            `n_features` is the number of features.
 
         Y : array-like of shape (n_samples, n_output) or (n_samples,)
             Target relative to X for classification or regression;
@@ -289,13 +330,25 @@ def score(self, X=None, Y=None):
         if self.methods_to_check == "all" or "score" in self.methods_to_check:
             self._check_X_y(X, Y)
         if self.foo_param > 1:
-            score = 1.
+            score = 1.0
         else:
-            score = 0.
+            score = 0.0
         return score
 
-    def _more_tags(self):
-        return {'_skip_test': True, 'X_types': ['1dlabel']}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags._skip_test = True
+        tags.input_tags.two_d_array = False
+        tags.target_tags.one_d_labels = True
+        return tags
+
+
+# Deactivate key validation for CheckingClassifier because we want to be able to
+# call fit with arbitrary fit_params and record them. Without this change, we
+# would get an error because those arbitrary params are not expected.
+CheckingClassifier.set_fit_request = RequestMethod(  # type: ignore[assignment,method-assign]
+    name="fit", keys=[], validate_keys=False
+)
 
 
 class NoSampleWeightWrapper(BaseEstimator):
@@ -319,5 +372,48 @@ def predict(self, X):
     def predict_proba(self, X):
         return self.est.predict_proba(X)
 
-    def _more_tags(self):
-        return {'_skip_test': True}
+    def __sklearn_tags__(self):
+        tags = super().__sklearn_tags__()
+        tags._skip_test = True
+        return tags
+
+
+def _check_response(method):
+    def check(self):
+        return self.response_methods is not None and method in self.response_methods
+
+    return check
+
+
+class _MockEstimatorOnOffPrediction(BaseEstimator):
+    """Estimator for which we can turn on/off the prediction methods.
+
+    Parameters
+    ----------
+    response_methods: list of \
+            {"predict", "predict_proba", "decision_function"}, default=None
+        List containing the response implemented by the estimator. When, the
+        response is in the list, it will return the name of the response method
+        when called. Otherwise, an `AttributeError` is raised. It allows to
+        use `getattr` as any conventional estimator. By default, no response
+        methods are mocked.
+    """
+
+    def __init__(self, response_methods=None):
+        self.response_methods = response_methods
+
+    def fit(self, X, y):
+        self.classes_ = np.unique(y)
+        return self
+
+    @available_if(_check_response("predict"))
+    def predict(self, X):
+        return "predict"
+
+    @available_if(_check_response("predict_proba"))
+    def predict_proba(self, X):
+        return "predict_proba"
+
+    @available_if(_check_response("decision_function"))
+    def decision_function(self, X):
+        return "decision_function"
diff --git a/sklearn/utils/_openmp_helpers.pxd b/sklearn/utils/_openmp_helpers.pxd
new file mode 100644
index 0000000000000..a7694d0be2d93
--- /dev/null
+++ b/sklearn/utils/_openmp_helpers.pxd
@@ -0,0 +1,33 @@
+# Helpers to safely access OpenMP routines
+#
+# no-op implementations are provided for the case where OpenMP is not available.
+#
+# All calls to OpenMP routines should be cimported from this module.
+
+cdef extern from *:
+    """
+    #ifdef _OPENMP
+        #include 
+        #define SKLEARN_OPENMP_PARALLELISM_ENABLED 1
+    #else
+        #define SKLEARN_OPENMP_PARALLELISM_ENABLED 0
+        #define omp_lock_t int
+        #define omp_init_lock(l) (void)0
+        #define omp_destroy_lock(l) (void)0
+        #define omp_set_lock(l) (void)0
+        #define omp_unset_lock(l) (void)0
+        #define omp_get_thread_num() 0
+        #define omp_get_max_threads() 1
+    #endif
+    """
+    bint SKLEARN_OPENMP_PARALLELISM_ENABLED
+
+    ctypedef struct omp_lock_t:
+        pass
+
+    void omp_init_lock(omp_lock_t*) noexcept nogil
+    void omp_destroy_lock(omp_lock_t*) noexcept nogil
+    void omp_set_lock(omp_lock_t*) noexcept nogil
+    void omp_unset_lock(omp_lock_t*) noexcept nogil
+    int omp_get_thread_num() noexcept nogil
+    int omp_get_max_threads() noexcept nogil
diff --git a/sklearn/utils/_openmp_helpers.pyx b/sklearn/utils/_openmp_helpers.pyx
index fb8920074a84e..88dca51089c56 100644
--- a/sklearn/utils/_openmp_helpers.pyx
+++ b/sklearn/utils/_openmp_helpers.pyx
@@ -1,28 +1,31 @@
-IF SKLEARN_OPENMP_PARALLELISM_ENABLED:
-    import os
-    cimport openmp
-    from joblib import cpu_count
+import os
+from joblib import cpu_count
+
+
+# Module level cache for cpu_count as we do not expect this to change during
+# the lifecycle of a Python program. This dictionary is keyed by
+# only_physical_cores.
+_CPU_COUNTS = {}
 
 
 def _openmp_parallelism_enabled():
     """Determines whether scikit-learn has been built with OpenMP
-    
+
     It allows to retrieve at runtime the information gathered at compile time.
     """
-    # SKLEARN_OPENMP_PARALLELISM_ENABLED is resolved at compile time during
-    # cythonization. It is defined via the `compile_time_env` kwarg of the
-    # `cythonize` call and behaves like the `-D` option of the C preprocessor.
+    # SKLEARN_OPENMP_PARALLELISM_ENABLED is resolved at compile time and defined
+    # in _openmp_helpers.pxd as a boolean. This function exposes it to Python.
     return SKLEARN_OPENMP_PARALLELISM_ENABLED
 
 
-cpdef _openmp_effective_n_threads(n_threads=None):
+cpdef _openmp_effective_n_threads(n_threads=None, only_physical_cores=True):
     """Determine the effective number of threads to be used for OpenMP calls
 
     - For ``n_threads = None``,
       - if the ``OMP_NUM_THREADS`` environment variable is set, return
         ``openmp.omp_get_max_threads()``
       - otherwise, return the minimum between ``openmp.omp_get_max_threads()``
-        and the number of cpus, taking cgroups quotas into account. Cgroups 
+        and the number of cpus, taking cgroups quotas into account. Cgroups
         quotas can typically be set by tools such as Docker.
       The result of ``omp_get_max_threads`` can be influenced by environment
       variable ``OMP_NUM_THREADS`` or at runtime by ``omp_set_num_threads``.
@@ -36,27 +39,39 @@ cpdef _openmp_effective_n_threads(n_threads=None):
 
     - Raise a ValueError for ``n_threads = 0``.
 
+    Passing the `only_physical_cores=False` flag makes it possible to use extra
+    threads for SMT/HyperThreading logical cores. It has been empirically
+    observed that using as many threads as available SMT cores can slightly
+    improve the performance in some cases, but can severely degrade
+    performance other times. Therefore it is recommended to use
+    `only_physical_cores=True` unless an empirical study has been conducted to
+    assess the impact of SMT on a case-by-case basis (using various input data
+    shapes, in particular small data shapes).
+
     If scikit-learn is built without OpenMP support, always return 1.
     """
     if n_threads == 0:
         raise ValueError("n_threads = 0 is invalid")
 
-    IF SKLEARN_OPENMP_PARALLELISM_ENABLED:
-        if os.getenv("OMP_NUM_THREADS"):
-            # Fall back to user provided number of threads making it possible
-            # to exceed the number of cpus.
-            max_n_threads = openmp.omp_get_max_threads()
-        else:
-            max_n_threads = min(openmp.omp_get_max_threads(), cpu_count())
-
-        if n_threads is None:
-            return max_n_threads
-        elif n_threads < 0:
-            return max(1, max_n_threads + n_threads + 1)
-
-        return n_threads
-    ELSE:
+    if not SKLEARN_OPENMP_PARALLELISM_ENABLED:
         # OpenMP disabled at build-time => sequential mode
         return 1
 
-    
+    if os.getenv("OMP_NUM_THREADS"):
+        # Fall back to user provided number of threads making it possible
+        # to exceed the number of cpus.
+        max_n_threads = omp_get_max_threads()
+    else:
+        try:
+            n_cpus = _CPU_COUNTS[only_physical_cores]
+        except KeyError:
+            n_cpus = cpu_count(only_physical_cores=only_physical_cores)
+            _CPU_COUNTS[only_physical_cores] = n_cpus
+        max_n_threads = min(omp_get_max_threads(), n_cpus)
+
+    if n_threads is None:
+        return max_n_threads
+    elif n_threads < 0:
+        return max(1, max_n_threads + n_threads + 1)
+
+    return n_threads
diff --git a/sklearn/utils/_optional_dependencies.py b/sklearn/utils/_optional_dependencies.py
new file mode 100644
index 0000000000000..5f0041285090a
--- /dev/null
+++ b/sklearn/utils/_optional_dependencies.py
@@ -0,0 +1,46 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+
+def check_matplotlib_support(caller_name):
+    """Raise ImportError with detailed error message if mpl is not installed.
+
+    Plot utilities like any of the Display's plotting functions should lazily import
+    matplotlib and call this helper before any computation.
+
+    Parameters
+    ----------
+    caller_name : str
+        The name of the caller that requires matplotlib.
+    """
+    try:
+        import matplotlib  # noqa: F401
+    except ImportError as e:
+        raise ImportError(
+            "{} requires matplotlib. You can install matplotlib with "
+            "`pip install matplotlib`".format(caller_name)
+        ) from e
+
+
+def check_pandas_support(caller_name):
+    """Raise ImportError with detailed error message if pandas is not installed.
+
+    Plot utilities like :func:`fetch_openml` should lazily import
+    pandas and call this helper before any computation.
+
+    Parameters
+    ----------
+    caller_name : str
+        The name of the caller that requires pandas.
+
+    Returns
+    -------
+    pandas
+        The pandas package.
+    """
+    try:
+        import pandas
+
+        return pandas
+    except ImportError as e:
+        raise ImportError("{} requires pandas.".format(caller_name)) from e
diff --git a/sklearn/utils/_param_validation.py b/sklearn/utils/_param_validation.py
new file mode 100644
index 0000000000000..27df9f4526d5c
--- /dev/null
+++ b/sklearn/utils/_param_validation.py
@@ -0,0 +1,910 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+import functools
+import math
+import operator
+import re
+from abc import ABC, abstractmethod
+from collections.abc import Iterable
+from inspect import signature
+from numbers import Integral, Real
+
+import numpy as np
+from scipy.sparse import csr_matrix, issparse
+
+from .._config import config_context, get_config
+from .validation import _is_arraylike_not_scalar
+
+
+class InvalidParameterError(ValueError, TypeError):
+    """Custom exception to be raised when the parameter of a class/method/function
+    does not have a valid type or value.
+    """
+
+    # Inherits from ValueError and TypeError to keep backward compatibility.
+
+
+def validate_parameter_constraints(parameter_constraints, params, caller_name):
+    """Validate types and values of given parameters.
+
+    Parameters
+    ----------
+    parameter_constraints : dict or {"no_validation"}
+        If "no_validation", validation is skipped for this parameter.
+
+        If a dict, it must be a dictionary `param_name: list of constraints`.
+        A parameter is valid if it satisfies one of the constraints from the list.
+        Constraints can be:
+        - an Interval object, representing a continuous or discrete range of numbers
+        - the string "array-like"
+        - the string "sparse matrix"
+        - the string "random_state"
+        - callable
+        - None, meaning that None is a valid value for the parameter
+        - any type, meaning that any instance of this type is valid
+        - an Options object, representing a set of elements of a given type
+        - a StrOptions object, representing a set of strings
+        - the string "boolean"
+        - the string "verbose"
+        - the string "cv_object"
+        - the string "nan"
+        - a MissingValues object representing markers for missing values
+        - a HasMethods object, representing method(s) an object must have
+        - a Hidden object, representing a constraint not meant to be exposed to the user
+
+    params : dict
+        A dictionary `param_name: param_value`. The parameters to validate against the
+        constraints.
+
+    caller_name : str
+        The name of the estimator or function or method that called this function.
+    """
+    for param_name, param_val in params.items():
+        # We allow parameters to not have a constraint so that third party estimators
+        # can inherit from sklearn estimators without having to necessarily use the
+        # validation tools.
+        if param_name not in parameter_constraints:
+            continue
+
+        constraints = parameter_constraints[param_name]
+
+        if constraints == "no_validation":
+            continue
+
+        constraints = [make_constraint(constraint) for constraint in constraints]
+
+        for constraint in constraints:
+            if constraint.is_satisfied_by(param_val):
+                # this constraint is satisfied, no need to check further.
+                break
+        else:
+            # No constraint is satisfied, raise with an informative message.
+
+            # Ignore constraints that we don't want to expose in the error message,
+            # i.e. options that are for internal purpose or not officially supported.
+            constraints = [
+                constraint for constraint in constraints if not constraint.hidden
+            ]
+
+            if len(constraints) == 1:
+                constraints_str = f"{constraints[0]}"
+            else:
+                constraints_str = (
+                    f"{', '.join([str(c) for c in constraints[:-1]])} or"
+                    f" {constraints[-1]}"
+                )
+
+            raise InvalidParameterError(
+                f"The {param_name!r} parameter of {caller_name} must be"
+                f" {constraints_str}. Got {param_val!r} instead."
+            )
+
+
+def make_constraint(constraint):
+    """Convert the constraint into the appropriate Constraint object.
+
+    Parameters
+    ----------
+    constraint : object
+        The constraint to convert.
+
+    Returns
+    -------
+    constraint : instance of _Constraint
+        The converted constraint.
+    """
+    if isinstance(constraint, str) and constraint == "array-like":
+        return _ArrayLikes()
+    if isinstance(constraint, str) and constraint == "sparse matrix":
+        return _SparseMatrices()
+    if isinstance(constraint, str) and constraint == "random_state":
+        return _RandomStates()
+    if constraint is callable:
+        return _Callables()
+    if constraint is None:
+        return _NoneConstraint()
+    if isinstance(constraint, type):
+        return _InstancesOf(constraint)
+    if isinstance(
+        constraint, (Interval, StrOptions, Options, HasMethods, MissingValues)
+    ):
+        return constraint
+    if isinstance(constraint, str) and constraint == "boolean":
+        return _Booleans()
+    if isinstance(constraint, str) and constraint == "verbose":
+        return _VerboseHelper()
+    if isinstance(constraint, str) and constraint == "cv_object":
+        return _CVObjects()
+    if isinstance(constraint, Hidden):
+        constraint = make_constraint(constraint.constraint)
+        constraint.hidden = True
+        return constraint
+    if (isinstance(constraint, str) and constraint == "nan") or (
+        isinstance(constraint, float) and np.isnan(constraint)
+    ):
+        return _NanConstraint()
+    raise ValueError(f"Unknown constraint type: {constraint}")
+
+
+def validate_params(parameter_constraints, *, prefer_skip_nested_validation):
+    """Decorator to validate types and values of functions and methods.
+
+    Parameters
+    ----------
+    parameter_constraints : dict
+        A dictionary `param_name: list of constraints`. See the docstring of
+        `validate_parameter_constraints` for a description of the accepted constraints.
+
+        Note that the *args and **kwargs parameters are not validated and must not be
+        present in the parameter_constraints dictionary.
+
+    prefer_skip_nested_validation : bool
+        If True, the validation of parameters of inner estimators or functions
+        called by the decorated function will be skipped.
+
+        This is useful to avoid validating many times the parameters passed by the
+        user from the public facing API. It's also useful to avoid validating
+        parameters that we pass internally to inner functions that are guaranteed to
+        be valid by the test suite.
+
+        It should be set to True for most functions, except for those that receive
+        non-validated objects as parameters or that are just wrappers around classes
+        because they only perform a partial validation.
+
+    Returns
+    -------
+    decorated_function : function or method
+        The decorated function.
+    """
+
+    def decorator(func):
+        # The dict of parameter constraints is set as an attribute of the function
+        # to make it possible to dynamically introspect the constraints for
+        # automatic testing.
+        setattr(func, "_skl_parameter_constraints", parameter_constraints)
+
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            global_skip_validation = get_config()["skip_parameter_validation"]
+            if global_skip_validation:
+                return func(*args, **kwargs)
+
+            func_sig = signature(func)
+
+            # Map *args/**kwargs to the function signature
+            params = func_sig.bind(*args, **kwargs)
+            params.apply_defaults()
+
+            # ignore self/cls and positional/keyword markers
+            to_ignore = [
+                p.name
+                for p in func_sig.parameters.values()
+                if p.kind in (p.VAR_POSITIONAL, p.VAR_KEYWORD)
+            ]
+            to_ignore += ["self", "cls"]
+            params = {k: v for k, v in params.arguments.items() if k not in to_ignore}
+
+            validate_parameter_constraints(
+                parameter_constraints, params, caller_name=func.__qualname__
+            )
+
+            try:
+                with config_context(
+                    skip_parameter_validation=(
+                        prefer_skip_nested_validation or global_skip_validation
+                    )
+                ):
+                    return func(*args, **kwargs)
+            except InvalidParameterError as e:
+                # When the function is just a wrapper around an estimator, we allow
+                # the function to delegate validation to the estimator, but we replace
+                # the name of the estimator by the name of the function in the error
+                # message to avoid confusion.
+                msg = re.sub(
+                    r"parameter of \w+ must be",
+                    f"parameter of {func.__qualname__} must be",
+                    str(e),
+                )
+                raise InvalidParameterError(msg) from e
+
+        return wrapper
+
+    return decorator
+
+
+class RealNotInt(Real):
+    """A type that represents reals that are not instances of int.
+
+    Behaves like float, but also works with values extracted from numpy arrays.
+    isintance(1, RealNotInt) -> False
+    isinstance(1.0, RealNotInt) -> True
+    """
+
+
+RealNotInt.register(float)
+
+
+def _type_name(t):
+    """Convert type into human readable string."""
+    module = t.__module__
+    qualname = t.__qualname__
+    if module == "builtins":
+        return qualname
+    elif t == Real:
+        return "float"
+    elif t == Integral:
+        return "int"
+    return f"{module}.{qualname}"
+
+
+class _Constraint(ABC):
+    """Base class for the constraint objects."""
+
+    def __init__(self):
+        self.hidden = False
+
+    @abstractmethod
+    def is_satisfied_by(self, val):
+        """Whether or not a value satisfies the constraint.
+
+        Parameters
+        ----------
+        val : object
+            The value to check.
+
+        Returns
+        -------
+        is_satisfied : bool
+            Whether or not the constraint is satisfied by this value.
+        """
+
+    @abstractmethod
+    def __str__(self):
+        """A human readable representational string of the constraint."""
+
+
+class _InstancesOf(_Constraint):
+    """Constraint representing instances of a given type.
+
+    Parameters
+    ----------
+    type : type
+        The valid type.
+    """
+
+    def __init__(self, type):
+        super().__init__()
+        self.type = type
+
+    def is_satisfied_by(self, val):
+        return isinstance(val, self.type)
+
+    def __str__(self):
+        return f"an instance of {_type_name(self.type)!r}"
+
+
+class _NoneConstraint(_Constraint):
+    """Constraint representing the None singleton."""
+
+    def is_satisfied_by(self, val):
+        return val is None
+
+    def __str__(self):
+        return "None"
+
+
+class _NanConstraint(_Constraint):
+    """Constraint representing the indicator `np.nan`."""
+
+    def is_satisfied_by(self, val):
+        return (
+            not isinstance(val, Integral) and isinstance(val, Real) and math.isnan(val)
+        )
+
+    def __str__(self):
+        return "numpy.nan"
+
+
+class _PandasNAConstraint(_Constraint):
+    """Constraint representing the indicator `pd.NA`."""
+
+    def is_satisfied_by(self, val):
+        try:
+            import pandas as pd
+
+            return isinstance(val, type(pd.NA)) and pd.isna(val)
+        except ImportError:
+            return False
+
+    def __str__(self):
+        return "pandas.NA"
+
+
+class Options(_Constraint):
+    """Constraint representing a finite set of instances of a given type.
+
+    Parameters
+    ----------
+    type : type
+
+    options : set
+        The set of valid scalars.
+
+    deprecated : set or None, default=None
+        A subset of the `options` to mark as deprecated in the string
+        representation of the constraint.
+    """
+
+    def __init__(self, type, options, *, deprecated=None):
+        super().__init__()
+        self.type = type
+        self.options = options
+        self.deprecated = deprecated or set()
+
+        if self.deprecated - self.options:
+            raise ValueError("The deprecated options must be a subset of the options.")
+
+    def is_satisfied_by(self, val):
+        return isinstance(val, self.type) and val in self.options
+
+    def _mark_if_deprecated(self, option):
+        """Add a deprecated mark to an option if needed."""
+        option_str = f"{option!r}"
+        if option in self.deprecated:
+            option_str = f"{option_str} (deprecated)"
+        return option_str
+
+    def __str__(self):
+        options_str = (
+            f"{', '.join([self._mark_if_deprecated(o) for o in self.options])}"
+        )
+        return f"a {_type_name(self.type)} among {{{options_str}}}"
+
+
+class StrOptions(Options):
+    """Constraint representing a finite set of strings.
+
+    Parameters
+    ----------
+    options : set of str
+        The set of valid strings.
+
+    deprecated : set of str or None, default=None
+        A subset of the `options` to mark as deprecated in the string
+        representation of the constraint.
+    """
+
+    def __init__(self, options, *, deprecated=None):
+        super().__init__(type=str, options=options, deprecated=deprecated)
+
+
+class Interval(_Constraint):
+    """Constraint representing a typed interval.
+
+    Parameters
+    ----------
+    type : {numbers.Integral, numbers.Real, RealNotInt}
+        The set of numbers in which to set the interval.
+
+        If RealNotInt, only reals that don't have the integer type
+        are allowed. For example 1.0 is allowed but 1 is not.
+
+    left : float or int or None
+        The left bound of the interval. None means left bound is -∞.
+
+    right : float, int or None
+        The right bound of the interval. None means right bound is +∞.
+
+    closed : {"left", "right", "both", "neither"}
+        Whether the interval is open or closed. Possible choices are:
+
+        - `"left"`: the interval is closed on the left and open on the right.
+          It is equivalent to the interval `[ left, right )`.
+        - `"right"`: the interval is closed on the right and open on the left.
+          It is equivalent to the interval `( left, right ]`.
+        - `"both"`: the interval is closed.
+          It is equivalent to the interval `[ left, right ]`.
+        - `"neither"`: the interval is open.
+          It is equivalent to the interval `( left, right )`.
+
+    Notes
+    -----
+    Setting a bound to `None` and setting the interval closed is valid. For instance,
+    strictly speaking, `Interval(Real, 0, None, closed="both")` corresponds to
+    `[0, +∞) U {+∞}`.
+    """
+
+    def __init__(self, type, left, right, *, closed):
+        super().__init__()
+        self.type = type
+        self.left = left
+        self.right = right
+        self.closed = closed
+
+        self._check_params()
+
+    def _check_params(self):
+        if self.type not in (Integral, Real, RealNotInt):
+            raise ValueError(
+                "type must be either numbers.Integral, numbers.Real or RealNotInt."
+                f" Got {self.type} instead."
+            )
+
+        if self.closed not in ("left", "right", "both", "neither"):
+            raise ValueError(
+                "closed must be either 'left', 'right', 'both' or 'neither'. "
+                f"Got {self.closed} instead."
+            )
+
+        if self.type is Integral:
+            suffix = "for an interval over the integers."
+            if self.left is not None and not isinstance(self.left, Integral):
+                raise TypeError(f"Expecting left to be an int {suffix}")
+            if self.right is not None and not isinstance(self.right, Integral):
+                raise TypeError(f"Expecting right to be an int {suffix}")
+            if self.left is None and self.closed in ("left", "both"):
+                raise ValueError(
+                    f"left can't be None when closed == {self.closed} {suffix}"
+                )
+            if self.right is None and self.closed in ("right", "both"):
+                raise ValueError(
+                    f"right can't be None when closed == {self.closed} {suffix}"
+                )
+        else:
+            if self.left is not None and not isinstance(self.left, Real):
+                raise TypeError("Expecting left to be a real number.")
+            if self.right is not None and not isinstance(self.right, Real):
+                raise TypeError("Expecting right to be a real number.")
+
+        if self.right is not None and self.left is not None and self.right <= self.left:
+            raise ValueError(
+                f"right can't be less than left. Got left={self.left} and "
+                f"right={self.right}"
+            )
+
+    def __contains__(self, val):
+        if not isinstance(val, Integral) and np.isnan(val):
+            return False
+
+        left_cmp = operator.lt if self.closed in ("left", "both") else operator.le
+        right_cmp = operator.gt if self.closed in ("right", "both") else operator.ge
+
+        left = -np.inf if self.left is None else self.left
+        right = np.inf if self.right is None else self.right
+
+        if left_cmp(val, left):
+            return False
+        if right_cmp(val, right):
+            return False
+        return True
+
+    def is_satisfied_by(self, val):
+        if not isinstance(val, self.type):
+            return False
+
+        return val in self
+
+    def __str__(self):
+        type_str = "an int" if self.type is Integral else "a float"
+        left_bracket = "[" if self.closed in ("left", "both") else "("
+        left_bound = "-inf" if self.left is None else self.left
+        right_bound = "inf" if self.right is None else self.right
+        right_bracket = "]" if self.closed in ("right", "both") else ")"
+
+        # better repr if the bounds were given as integers
+        if not self.type == Integral and isinstance(self.left, Real):
+            left_bound = float(left_bound)
+        if not self.type == Integral and isinstance(self.right, Real):
+            right_bound = float(right_bound)
+
+        return (
+            f"{type_str} in the range "
+            f"{left_bracket}{left_bound}, {right_bound}{right_bracket}"
+        )
+
+
+class _ArrayLikes(_Constraint):
+    """Constraint representing array-likes"""
+
+    def is_satisfied_by(self, val):
+        return _is_arraylike_not_scalar(val)
+
+    def __str__(self):
+        return "an array-like"
+
+
+class _SparseMatrices(_Constraint):
+    """Constraint representing sparse matrices."""
+
+    def is_satisfied_by(self, val):
+        return issparse(val)
+
+    def __str__(self):
+        return "a sparse matrix"
+
+
+class _Callables(_Constraint):
+    """Constraint representing callables."""
+
+    def is_satisfied_by(self, val):
+        return callable(val)
+
+    def __str__(self):
+        return "a callable"
+
+
+class _RandomStates(_Constraint):
+    """Constraint representing random states.
+
+    Convenience class for
+    [Interval(Integral, 0, 2**32 - 1, closed="both"), np.random.RandomState, None]
+    """
+
+    def __init__(self):
+        super().__init__()
+        self._constraints = [
+            Interval(Integral, 0, 2**32 - 1, closed="both"),
+            _InstancesOf(np.random.RandomState),
+            _NoneConstraint(),
+        ]
+
+    def is_satisfied_by(self, val):
+        return any(c.is_satisfied_by(val) for c in self._constraints)
+
+    def __str__(self):
+        return (
+            f"{', '.join([str(c) for c in self._constraints[:-1]])} or"
+            f" {self._constraints[-1]}"
+        )
+
+
+class _Booleans(_Constraint):
+    """Constraint representing boolean likes.
+
+    Convenience class for
+    [bool, np.bool_]
+    """
+
+    def __init__(self):
+        super().__init__()
+        self._constraints = [
+            _InstancesOf(bool),
+            _InstancesOf(np.bool_),
+        ]
+
+    def is_satisfied_by(self, val):
+        return any(c.is_satisfied_by(val) for c in self._constraints)
+
+    def __str__(self):
+        return (
+            f"{', '.join([str(c) for c in self._constraints[:-1]])} or"
+            f" {self._constraints[-1]}"
+        )
+
+
+class _VerboseHelper(_Constraint):
+    """Helper constraint for the verbose parameter.
+
+    Convenience class for
+    [Interval(Integral, 0, None, closed="left"), bool, numpy.bool_]
+    """
+
+    def __init__(self):
+        super().__init__()
+        self._constraints = [
+            Interval(Integral, 0, None, closed="left"),
+            _InstancesOf(bool),
+            _InstancesOf(np.bool_),
+        ]
+
+    def is_satisfied_by(self, val):
+        return any(c.is_satisfied_by(val) for c in self._constraints)
+
+    def __str__(self):
+        return (
+            f"{', '.join([str(c) for c in self._constraints[:-1]])} or"
+            f" {self._constraints[-1]}"
+        )
+
+
+class MissingValues(_Constraint):
+    """Helper constraint for the `missing_values` parameters.
+
+    Convenience for
+    [
+        Integral,
+        Interval(Real, None, None, closed="both"),
+        str,   # when numeric_only is False
+        None,  # when numeric_only is False
+        _NanConstraint(),
+        _PandasNAConstraint(),
+    ]
+
+    Parameters
+    ----------
+    numeric_only : bool, default=False
+        Whether to consider only numeric missing value markers.
+
+    """
+
+    def __init__(self, numeric_only=False):
+        super().__init__()
+
+        self.numeric_only = numeric_only
+
+        self._constraints = [
+            _InstancesOf(Integral),
+            # we use an interval of Real to ignore np.nan that has its own constraint
+            Interval(Real, None, None, closed="both"),
+            _NanConstraint(),
+            _PandasNAConstraint(),
+        ]
+        if not self.numeric_only:
+            self._constraints.extend([_InstancesOf(str), _NoneConstraint()])
+
+    def is_satisfied_by(self, val):
+        return any(c.is_satisfied_by(val) for c in self._constraints)
+
+    def __str__(self):
+        return (
+            f"{', '.join([str(c) for c in self._constraints[:-1]])} or"
+            f" {self._constraints[-1]}"
+        )
+
+
+class HasMethods(_Constraint):
+    """Constraint representing objects that expose specific methods.
+
+    It is useful for parameters following a protocol and where we don't want to impose
+    an affiliation to a specific module or class.
+
+    Parameters
+    ----------
+    methods : str or list of str
+        The method(s) that the object is expected to expose.
+    """
+
+    @validate_params(
+        {"methods": [str, list]},
+        prefer_skip_nested_validation=True,
+    )
+    def __init__(self, methods):
+        super().__init__()
+        if isinstance(methods, str):
+            methods = [methods]
+        self.methods = methods
+
+    def is_satisfied_by(self, val):
+        return all(callable(getattr(val, method, None)) for method in self.methods)
+
+    def __str__(self):
+        if len(self.methods) == 1:
+            methods = f"{self.methods[0]!r}"
+        else:
+            methods = (
+                f"{', '.join([repr(m) for m in self.methods[:-1]])} and"
+                f" {self.methods[-1]!r}"
+            )
+        return f"an object implementing {methods}"
+
+
+class _IterablesNotString(_Constraint):
+    """Constraint representing iterables that are not strings."""
+
+    def is_satisfied_by(self, val):
+        return isinstance(val, Iterable) and not isinstance(val, str)
+
+    def __str__(self):
+        return "an iterable"
+
+
+class _CVObjects(_Constraint):
+    """Constraint representing cv objects.
+
+    Convenient class for
+    [
+        Interval(Integral, 2, None, closed="left"),
+        HasMethods(["split", "get_n_splits"]),
+        _IterablesNotString(),
+        None,
+    ]
+    """
+
+    def __init__(self):
+        super().__init__()
+        self._constraints = [
+            Interval(Integral, 2, None, closed="left"),
+            HasMethods(["split", "get_n_splits"]),
+            _IterablesNotString(),
+            _NoneConstraint(),
+        ]
+
+    def is_satisfied_by(self, val):
+        return any(c.is_satisfied_by(val) for c in self._constraints)
+
+    def __str__(self):
+        return (
+            f"{', '.join([str(c) for c in self._constraints[:-1]])} or"
+            f" {self._constraints[-1]}"
+        )
+
+
+class Hidden:
+    """Class encapsulating a constraint not meant to be exposed to the user.
+
+    Parameters
+    ----------
+    constraint : str or _Constraint instance
+        The constraint to be used internally.
+    """
+
+    def __init__(self, constraint):
+        self.constraint = constraint
+
+
+def generate_invalid_param_val(constraint):
+    """Return a value that does not satisfy the constraint.
+
+    Raises a NotImplementedError if there exists no invalid value for this constraint.
+
+    This is only useful for testing purpose.
+
+    Parameters
+    ----------
+    constraint : _Constraint instance
+        The constraint to generate a value for.
+
+    Returns
+    -------
+    val : object
+        A value that does not satisfy the constraint.
+    """
+    if isinstance(constraint, StrOptions):
+        return f"not {' or '.join(constraint.options)}"
+
+    if isinstance(constraint, MissingValues):
+        return np.array([1, 2, 3])
+
+    if isinstance(constraint, _VerboseHelper):
+        return -1
+
+    if isinstance(constraint, HasMethods):
+        return type("HasNotMethods", (), {})()
+
+    if isinstance(constraint, _IterablesNotString):
+        return "a string"
+
+    if isinstance(constraint, _CVObjects):
+        return "not a cv object"
+
+    if isinstance(constraint, Interval) and constraint.type is Integral:
+        if constraint.left is not None:
+            return constraint.left - 1
+        if constraint.right is not None:
+            return constraint.right + 1
+
+        # There's no integer outside (-inf, +inf)
+        raise NotImplementedError
+
+    if isinstance(constraint, Interval) and constraint.type in (Real, RealNotInt):
+        if constraint.left is not None:
+            return constraint.left - 1e-6
+        if constraint.right is not None:
+            return constraint.right + 1e-6
+
+        # bounds are -inf, +inf
+        if constraint.closed in ("right", "neither"):
+            return -np.inf
+        if constraint.closed in ("left", "neither"):
+            return np.inf
+
+        # interval is [-inf, +inf]
+        return np.nan
+
+    raise NotImplementedError
+
+
+def generate_valid_param(constraint):
+    """Return a value that does satisfy a constraint.
+
+    This is only useful for testing purpose.
+
+    Parameters
+    ----------
+    constraint : Constraint instance
+        The constraint to generate a value for.
+
+    Returns
+    -------
+    val : object
+        A value that does satisfy the constraint.
+    """
+    if isinstance(constraint, _ArrayLikes):
+        return np.array([1, 2, 3])
+
+    if isinstance(constraint, _SparseMatrices):
+        return csr_matrix([[0, 1], [1, 0]])
+
+    if isinstance(constraint, _RandomStates):
+        return np.random.RandomState(42)
+
+    if isinstance(constraint, _Callables):
+        return lambda x: x
+
+    if isinstance(constraint, _NoneConstraint):
+        return None
+
+    if isinstance(constraint, _InstancesOf):
+        if constraint.type is np.ndarray:
+            # special case for ndarray since it can't be instantiated without arguments
+            return np.array([1, 2, 3])
+
+        if constraint.type in (Integral, Real):
+            # special case for Integral and Real since they are abstract classes
+            return 1
+
+        return constraint.type()
+
+    if isinstance(constraint, _Booleans):
+        return True
+
+    if isinstance(constraint, _VerboseHelper):
+        return 1
+
+    if isinstance(constraint, MissingValues) and constraint.numeric_only:
+        return np.nan
+
+    if isinstance(constraint, MissingValues) and not constraint.numeric_only:
+        return "missing"
+
+    if isinstance(constraint, HasMethods):
+        return type(
+            "ValidHasMethods", (), {m: lambda self: None for m in constraint.methods}
+        )()
+
+    if isinstance(constraint, _IterablesNotString):
+        return [1, 2, 3]
+
+    if isinstance(constraint, _CVObjects):
+        return 5
+
+    if isinstance(constraint, Options):  # includes StrOptions
+        for option in constraint.options:
+            return option
+
+    if isinstance(constraint, Interval):
+        interval = constraint
+        if interval.left is None and interval.right is None:
+            return 0
+        elif interval.left is None:
+            return interval.right - 1
+        elif interval.right is None:
+            return interval.left + 1
+        else:
+            if interval.type is Real:
+                return (interval.left + interval.right) / 2
+            else:
+                return interval.left + 1
+
+    raise ValueError(f"Unknown constraint type: {constraint}")
diff --git a/sklearn/utils/_plotting.py b/sklearn/utils/_plotting.py
new file mode 100644
index 0000000000000..1a3883b7db7f5
--- /dev/null
+++ b/sklearn/utils/_plotting.py
@@ -0,0 +1,419 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+import warnings
+from collections.abc import Mapping
+
+import numpy as np
+
+from . import check_consistent_length
+from ._optional_dependencies import check_matplotlib_support
+from ._response import _get_response_values_binary
+from .fixes import parse_version
+from .multiclass import type_of_target
+from .validation import _check_pos_label_consistency, _num_samples
+
+
+class _BinaryClassifierCurveDisplayMixin:
+    """Mixin class to be used in Displays requiring a binary classifier.
+
+    The aim of this class is to centralize some validations regarding the estimator and
+    the target and gather the response of the estimator.
+    """
+
+    def _validate_plot_params(self, *, ax=None, name=None):
+        check_matplotlib_support(f"{self.__class__.__name__}.plot")
+        import matplotlib.pyplot as plt
+
+        if ax is None:
+            _, ax = plt.subplots()
+
+        # Display classes are in process of changing from `estimator_name` to `name`.
+        # Try old attr name: `estimator_name` first.
+        if name is None:
+            name = getattr(self, "estimator_name", getattr(self, "name", None))
+        return ax, ax.figure, name
+
+    @classmethod
+    def _validate_and_get_response_values(
+        cls, estimator, X, y, *, response_method="auto", pos_label=None, name=None
+    ):
+        check_matplotlib_support(f"{cls.__name__}.from_estimator")
+
+        name = estimator.__class__.__name__ if name is None else name
+
+        y_pred, pos_label = _get_response_values_binary(
+            estimator,
+            X,
+            response_method=response_method,
+            pos_label=pos_label,
+        )
+
+        return y_pred, pos_label, name
+
+    @classmethod
+    def _validate_from_predictions_params(
+        cls, y_true, y_pred, *, sample_weight=None, pos_label=None, name=None
+    ):
+        check_matplotlib_support(f"{cls.__name__}.from_predictions")
+
+        if type_of_target(y_true) != "binary":
+            raise ValueError(
+                f"The target y is not binary. Got {type_of_target(y_true)} type of"
+                " target."
+            )
+
+        check_consistent_length(y_true, y_pred, sample_weight)
+        pos_label = _check_pos_label_consistency(pos_label, y_true)
+
+        name = name if name is not None else "Classifier"
+
+        return pos_label, name
+
+    @classmethod
+    def _validate_from_cv_results_params(
+        cls,
+        cv_results,
+        X,
+        y,
+        *,
+        sample_weight,
+        pos_label,
+    ):
+        check_matplotlib_support(f"{cls.__name__}.from_cv_results")
+
+        required_keys = {"estimator", "indices"}
+        if not all(key in cv_results for key in required_keys):
+            raise ValueError(
+                "`cv_results` does not contain one of the following required keys: "
+                f"{required_keys}. Set explicitly the parameters "
+                "`return_estimator=True` and `return_indices=True` to the function"
+                "`cross_validate`."
+            )
+
+        train_size, test_size = (
+            len(cv_results["indices"]["train"][0]),
+            len(cv_results["indices"]["test"][0]),
+        )
+
+        if _num_samples(X) != train_size + test_size:
+            raise ValueError(
+                "`X` does not contain the correct number of samples. "
+                f"Expected {train_size + test_size}, got {_num_samples(X)}."
+            )
+
+        if type_of_target(y) != "binary":
+            raise ValueError(
+                f"The target `y` is not binary. Got {type_of_target(y)} type of target."
+            )
+        check_consistent_length(X, y, sample_weight)
+
+        try:
+            pos_label = _check_pos_label_consistency(pos_label, y)
+        except ValueError as e:
+            # Adapt error message
+            raise ValueError(str(e).replace("y_true", "y"))
+
+        return pos_label
+
+    @staticmethod
+    def _get_legend_label(curve_legend_metric, curve_name, legend_metric_name):
+        """Helper to get legend label using `name` and `legend_metric`"""
+        if curve_legend_metric is not None and curve_name is not None:
+            label = f"{curve_name} ({legend_metric_name} = {curve_legend_metric:0.2f})"
+        elif curve_legend_metric is not None:
+            label = f"{legend_metric_name} = {curve_legend_metric:0.2f}"
+        elif curve_name is not None:
+            label = curve_name
+        else:
+            label = None
+        return label
+
+    @staticmethod
+    def _validate_curve_kwargs(
+        n_curves,
+        name,
+        legend_metric,
+        legend_metric_name,
+        curve_kwargs,
+        **kwargs,
+    ):
+        """Get validated line kwargs for each curve.
+
+        Parameters
+        ----------
+        n_curves : int
+            Number of curves.
+
+        name : list of str or None
+            Name for labeling legend entries.
+
+        legend_metric : dict
+            Dictionary with "mean" and "std" keys, or "metric" key of metric
+            values for each curve. If None, "label" will not contain metric values.
+
+        legend_metric_name : str
+            Name of the summary value provided in `legend_metrics`.
+
+        curve_kwargs : dict or list of dict or None
+            Dictionary with keywords passed to the matplotlib's `plot` function
+            to draw the individual curves. If a list is provided, the
+            parameters are applied to the curves sequentially. If a single
+            dictionary is provided, the same parameters are applied to all
+            curves.
+
+        **kwargs : dict
+            Deprecated. Keyword arguments to be passed to matplotlib's `plot`.
+        """
+        # TODO(1.9): Remove deprecated **kwargs
+        if curve_kwargs and kwargs:
+            raise ValueError(
+                "Cannot provide both `curve_kwargs` and `kwargs`. `**kwargs` is "
+                "deprecated in 1.7 and will be removed in 1.9. Pass all matplotlib "
+                "arguments to `curve_kwargs` as a dictionary."
+            )
+        if kwargs:
+            warnings.warn(
+                "`**kwargs` is deprecated and will be removed in 1.9. Pass all "
+                "matplotlib arguments to `curve_kwargs` as a dictionary instead.",
+                FutureWarning,
+            )
+            curve_kwargs = kwargs
+
+        if isinstance(curve_kwargs, list) and len(curve_kwargs) != n_curves:
+            raise ValueError(
+                f"`curve_kwargs` must be None, a dictionary or a list of length "
+                f"{n_curves}. Got: {curve_kwargs}."
+            )
+
+        # Ensure valid `name` and `curve_kwargs` combination.
+        if (
+            isinstance(name, list)
+            and len(name) != 1
+            and not isinstance(curve_kwargs, list)
+        ):
+            raise ValueError(
+                "To avoid labeling individual curves that have the same appearance, "
+                f"`curve_kwargs` should be a list of {n_curves} dictionaries. "
+                "Alternatively, set `name` to `None` or a single string to label "
+                "a single legend entry with mean ROC AUC score of all curves."
+            )
+
+        # Ensure `name` is of the correct length
+        if isinstance(name, str):
+            name = [name]
+        if isinstance(name, list) and len(name) == 1:
+            name = name * n_curves
+        name = [None] * n_curves if name is None else name
+
+        # Ensure `curve_kwargs` is of correct length
+        if isinstance(curve_kwargs, Mapping):
+            curve_kwargs = [curve_kwargs] * n_curves
+
+        default_multi_curve_kwargs = {"alpha": 0.5, "linestyle": "--", "color": "blue"}
+        if curve_kwargs is None:
+            if n_curves > 1:
+                curve_kwargs = [default_multi_curve_kwargs] * n_curves
+            else:
+                curve_kwargs = [{}]
+
+        labels = []
+        if "mean" in legend_metric:
+            label_aggregate = _BinaryClassifierCurveDisplayMixin._get_legend_label(
+                legend_metric["mean"], name[0], legend_metric_name
+            )
+            # Note: "std" always `None` when "mean" is `None` - no metric value added
+            # to label in this case
+            if legend_metric["std"] is not None:
+                # Add the "+/- std" to the end (in brackets if name provided)
+                if name[0] is not None:
+                    label_aggregate = (
+                        label_aggregate[:-1] + f" +/- {legend_metric['std']:0.2f})"
+                    )
+                else:
+                    label_aggregate = (
+                        label_aggregate + f" +/- {legend_metric['std']:0.2f}"
+                    )
+            # Add `label` for first curve only, set to `None` for remaining curves
+            labels.extend([label_aggregate] + [None] * (n_curves - 1))
+        else:
+            for curve_legend_metric, curve_name in zip(legend_metric["metric"], name):
+                labels.append(
+                    _BinaryClassifierCurveDisplayMixin._get_legend_label(
+                        curve_legend_metric, curve_name, legend_metric_name
+                    )
+                )
+
+        curve_kwargs_ = [
+            _validate_style_kwargs({"label": label}, curve_kwargs[fold_idx])
+            for fold_idx, label in enumerate(labels)
+        ]
+        return curve_kwargs_
+
+
+def _validate_score_name(score_name, scoring, negate_score):
+    """Validate the `score_name` parameter.
+
+    If `score_name` is provided, we just return it as-is.
+    If `score_name` is `None`, we use `Score` if `negate_score` is `False` and
+    `Negative score` otherwise.
+    If `score_name` is a string or a callable, we infer the name. We replace `_` by
+    spaces and capitalize the first letter. We remove `neg_` and replace it by
+    `"Negative"` if `negate_score` is `False` or just remove it otherwise.
+    """
+    if score_name is not None:
+        return score_name
+    elif scoring is None:
+        return "Negative score" if negate_score else "Score"
+    else:
+        score_name = scoring.__name__ if callable(scoring) else scoring
+        if negate_score:
+            if score_name.startswith("neg_"):
+                score_name = score_name[4:]
+            else:
+                score_name = f"Negative {score_name}"
+        elif score_name.startswith("neg_"):
+            score_name = f"Negative {score_name[4:]}"
+        score_name = score_name.replace("_", " ")
+        return score_name.capitalize()
+
+
+def _interval_max_min_ratio(data):
+    """Compute the ratio between the largest and smallest inter-point distances.
+
+    A value larger than 5 typically indicates that the parameter range would
+    better be displayed with a log scale while a linear scale would be more
+    suitable otherwise.
+    """
+    diff = np.diff(np.sort(data))
+    return diff.max() / diff.min()
+
+
+def _validate_style_kwargs(default_style_kwargs, user_style_kwargs):
+    """Create valid style kwargs by avoiding Matplotlib alias errors.
+
+    Matplotlib raises an error when, for example, 'color' and 'c', or 'linestyle' and
+    'ls', are specified together. To avoid this, we automatically keep only the one
+    specified by the user and raise an error if the user specifies both.
+
+    Parameters
+    ----------
+    default_style_kwargs : dict
+        The Matplotlib style kwargs used by default in the scikit-learn display.
+    user_style_kwargs : dict
+        The user-defined Matplotlib style kwargs.
+
+    Returns
+    -------
+    valid_style_kwargs : dict
+        The validated style kwargs taking into account both default and user-defined
+        Matplotlib style kwargs.
+    """
+
+    invalid_to_valid_kw = {
+        "ls": "linestyle",
+        "c": "color",
+        "ec": "edgecolor",
+        "fc": "facecolor",
+        "lw": "linewidth",
+        "mec": "markeredgecolor",
+        "mfcalt": "markerfacecoloralt",
+        "ms": "markersize",
+        "mew": "markeredgewidth",
+        "mfc": "markerfacecolor",
+        "aa": "antialiased",
+        "ds": "drawstyle",
+        "font": "fontproperties",
+        "family": "fontfamily",
+        "name": "fontname",
+        "size": "fontsize",
+        "stretch": "fontstretch",
+        "style": "fontstyle",
+        "variant": "fontvariant",
+        "weight": "fontweight",
+        "ha": "horizontalalignment",
+        "va": "verticalalignment",
+        "ma": "multialignment",
+    }
+    for invalid_key, valid_key in invalid_to_valid_kw.items():
+        if invalid_key in user_style_kwargs and valid_key in user_style_kwargs:
+            raise TypeError(
+                f"Got both {invalid_key} and {valid_key}, which are aliases of one "
+                "another"
+            )
+    valid_style_kwargs = default_style_kwargs.copy()
+
+    for key in user_style_kwargs.keys():
+        if key in invalid_to_valid_kw:
+            valid_style_kwargs[invalid_to_valid_kw[key]] = user_style_kwargs[key]
+        else:
+            valid_style_kwargs[key] = user_style_kwargs[key]
+
+    return valid_style_kwargs
+
+
+def _despine(ax):
+    """Remove the top and right spines of the plot.
+
+    Parameters
+    ----------
+    ax : matplotlib.axes.Axes
+        The axes of the plot to despine.
+    """
+    for s in ["top", "right"]:
+        ax.spines[s].set_visible(False)
+    for s in ["bottom", "left"]:
+        ax.spines[s].set_bounds(0, 1)
+
+
+def _deprecate_estimator_name(estimator_name, name, version):
+    """Deprecate `estimator_name` in favour of `name`."""
+    version = parse_version(version)
+    version_remove = f"{version.major}.{version.minor + 2}"
+    if estimator_name != "deprecated":
+        if name:
+            raise ValueError(
+                "Cannot provide both `estimator_name` and `name`. `estimator_name` "
+                f"is deprecated in {version} and will be removed in {version_remove}. "
+                "Use `name` only."
+            )
+        warnings.warn(
+            f"`estimator_name` is deprecated in {version} and will be removed in "
+            f"{version_remove}. Use `name` instead.",
+            FutureWarning,
+        )
+        return estimator_name
+    return name
+
+
+def _convert_to_list_leaving_none(param):
+    """Convert parameters to a list, leaving `None` as is."""
+    if param is None:
+        return None
+    if isinstance(param, list):
+        return param
+    return [param]
+
+
+def _check_param_lengths(required, optional, class_name):
+    """Check required and optional parameters are of the same length."""
+    optional_provided = {}
+    for name, param in optional.items():
+        if isinstance(param, list):
+            optional_provided[name] = param
+
+    all_params = {**required, **optional_provided}
+    if len({len(param) for param in all_params.values()}) > 1:
+        param_keys = [key for key in all_params.keys()]
+        # Note: below code requires `len(param_keys) >= 2`, which is the case for all
+        # display classes
+        params_formatted = " and ".join([", ".join(param_keys[:-1]), param_keys[-1]])
+        or_plot = ""
+        if "'name' (or self.name)" in param_keys:
+            or_plot = " (or `plot`)"
+        lengths_formatted = ", ".join(
+            f"{key}: {len(value)}" for key, value in all_params.items()
+        )
+        raise ValueError(
+            f"{params_formatted} from `{class_name}` initialization{or_plot}, "
+            f"should all be lists of the same length. Got: {lengths_formatted}"
+        )
diff --git a/sklearn/utils/_pprint.py b/sklearn/utils/_pprint.py
index 8a7e53311d2af..527843fe42f0b 100644
--- a/sklearn/utils/_pprint.py
+++ b/sklearn/utils/_pprint.py
@@ -1,6 +1,9 @@
 """This module contains the _EstimatorPrettyPrinter class used in
 BaseEstimator.__repr__ for pretty-printing estimators"""
 
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
 # Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
 # 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Python Software Foundation;
 # All Rights Reserved
@@ -65,15 +68,15 @@
 
 import inspect
 import pprint
-from collections import OrderedDict
 
-from ..base import BaseEstimator
 from .._config import get_config
-from . import is_scalar_nan
+from ..base import BaseEstimator
+from ._missing import is_scalar_nan
 
 
 class KeyValTuple(tuple):
     """Dummy class for correctly rendering key-value tuples from dicts."""
+
     def __repr__(self):
         # needed for _dispatch[tuple.__repr__] not to be overridden
         return super().__repr__()
@@ -81,6 +84,7 @@ def __repr__(self):
 
 class KeyValTupleParam(KeyValTuple):
     """Dummy class for correctly rendering key-value tuples from parameters."""
+
     pass
 
 
@@ -89,8 +93,7 @@ def _changed_params(estimator):
     estimator with non-default values."""
 
     params = estimator.get_params(deep=False)
-    init_func = getattr(estimator.__init__, 'deprecated_original',
-                        estimator.__init__)
+    init_func = getattr(estimator.__init__, "deprecated_original", estimator.__init__)
     init_params = inspect.signature(init_func).parameters
     init_params = {name: param.default for name, param in init_params.items()}
 
@@ -100,12 +103,12 @@ def has_changed(k, v):
         if init_params[k] == inspect._empty:  # k has no default value
             return True
         # try to avoid calling repr on nested estimators
-        if (isinstance(v, BaseEstimator) and
-           v.__class__ != init_params[k].__class__):
+        if isinstance(v, BaseEstimator) and v.__class__ != init_params[k].__class__:
             return True
         # Use repr as a last resort. It may be expensive.
-        if (repr(v) != repr(init_params[k]) and
-           not (is_scalar_nan(init_params[k]) and is_scalar_nan(v))):
+        if repr(v) != repr(init_params[k]) and not (
+            is_scalar_nan(init_params[k]) and is_scalar_nan(v)
+        ):
             return True
         return False
 
@@ -128,7 +131,7 @@ class _EstimatorPrettyPrinter(pprint.PrettyPrinter):
       here)
     - format() directly calls _safe_repr() for a first try at rendering the
       object
-    - _safe_repr formats the whole object reccursively, only calling itself,
+    - _safe_repr formats the whole object recursively, only calling itself,
       not caring about line length or anything
     - back to _format(), if the output string is too long, _format() then calls
       the appropriate _pprint_TYPE() method (e.g. _pprint_list()) depending on
@@ -163,26 +166,34 @@ class _EstimatorPrettyPrinter(pprint.PrettyPrinter):
     KeyValTupleParam for this.
     """
 
-    def __init__(self, indent=1, width=80, depth=None, stream=None, *,
-                 compact=False, indent_at_name=True,
-                 n_max_elements_to_show=None):
+    def __init__(
+        self,
+        indent=1,
+        width=80,
+        depth=None,
+        stream=None,
+        *,
+        compact=False,
+        indent_at_name=True,
+        n_max_elements_to_show=None,
+    ):
         super().__init__(indent, width, depth, stream, compact=compact)
         self._indent_at_name = indent_at_name
         if self._indent_at_name:
             self._indent_per_level = 1  # ignore indent param
-        self._changed_only = get_config()['print_changed_only']
+        self._changed_only = get_config()["print_changed_only"]
         # Max number of elements in a list, dict, tuple until we start using
         # ellipsis. This also affects the number of arguments of an estimators
         # (they are treated as dicts)
         self.n_max_elements_to_show = n_max_elements_to_show
 
     def format(self, object, context, maxlevels, level):
-        return _safe_repr(object, context, maxlevels, level,
-                          changed_only=self._changed_only)
+        return _safe_repr(
+            object, context, maxlevels, level, changed_only=self._changed_only
+        )
 
-    def _pprint_estimator(self, object, stream, indent, allowance, context,
-                          level):
-        stream.write(object.__class__.__name__ + '(')
+    def _pprint_estimator(self, object, stream, indent, allowance, context, level):
+        stream.write(object.__class__.__name__ + "(")
         if self._indent_at_name:
             indent += len(object.__class__.__name__)
 
@@ -191,24 +202,24 @@ def _pprint_estimator(self, object, stream, indent, allowance, context,
         else:
             params = object.get_params(deep=False)
 
-        params = OrderedDict((name, val)
-                             for (name, val) in sorted(params.items()))
-
-        self._format_params(params.items(), stream, indent, allowance + 1,
-                            context, level)
-        stream.write(')')
+        self._format_params(
+            sorted(params.items()), stream, indent, allowance + 1, context, level
+        )
+        stream.write(")")
 
-    def _format_dict_items(self, items, stream, indent, allowance, context,
-                           level):
+    def _format_dict_items(self, items, stream, indent, allowance, context, level):
         return self._format_params_or_dict_items(
-            items, stream, indent, allowance, context, level, is_dict=True)
+            items, stream, indent, allowance, context, level, is_dict=True
+        )
 
     def _format_params(self, items, stream, indent, allowance, context, level):
         return self._format_params_or_dict_items(
-            items, stream, indent, allowance, context, level, is_dict=False)
+            items, stream, indent, allowance, context, level, is_dict=False
+        )
 
-    def _format_params_or_dict_items(self, object, stream, indent, allowance,
-                                     context, level, is_dict):
+    def _format_params_or_dict_items(
+        self, object, stream, indent, allowance, context, level, is_dict
+    ):
         """Format dict items or parameters respecting the compact=True
         parameter. For some reason, the builtin rendering of dict items doesn't
         respect compact=True and will use one line per key-value if all cannot
@@ -221,8 +232,8 @@ def _format_params_or_dict_items(self, object, stream, indent, allowance,
         """
         write = stream.write
         indent += self._indent_per_level
-        delimnl = ',\n' + ' ' * indent
-        delim = ''
+        delimnl = ",\n" + " " * indent
+        delim = ""
         width = max_width = self._width - indent + 1
         it = iter(object)
         try:
@@ -233,7 +244,7 @@ def _format_params_or_dict_items(self, object, stream, indent, allowance,
         n_items = 0
         while not last:
             if n_items == self.n_max_elements_to_show:
-                write(', ...')
+                write(", ...")
                 break
             n_items += 1
             ent = next_ent
@@ -249,7 +260,7 @@ def _format_params_or_dict_items(self, object, stream, indent, allowance,
                 vrepr = self._repr(v, context, level)
                 if not is_dict:
                     krepr = krepr.strip("'")
-                middle = ': ' if is_dict else '='
+                middle = ": " if is_dict else "="
                 rep = krepr + middle + vrepr
                 w = len(rep) + 2
                 if width < w:
@@ -259,14 +270,15 @@ def _format_params_or_dict_items(self, object, stream, indent, allowance,
                 if width >= w:
                     width -= w
                     write(delim)
-                    delim = ', '
+                    delim = ", "
                     write(rep)
                     continue
             write(delim)
             delim = delimnl
             class_ = KeyValTuple if is_dict else KeyValTupleParam
-            self._format(class_(ent), stream, indent,
-                         allowance if last else 1, context, level)
+            self._format(
+                class_(ent), stream, indent, allowance if last else 1, context, level
+            )
 
     def _format_items(self, items, stream, indent, allowance, context, level):
         """Format the items of an iterable (list, tuple...). Same as the
@@ -276,9 +288,9 @@ def _format_items(self, items, stream, indent, allowance, context, level):
         write = stream.write
         indent += self._indent_per_level
         if self._indent_per_level > 1:
-            write((self._indent_per_level - 1) * ' ')
-        delimnl = ',\n' + ' ' * indent
-        delim = ''
+            write((self._indent_per_level - 1) * " ")
+        delimnl = ",\n" + " " * indent
+        delim = ""
         width = max_width = self._width - indent + 1
         it = iter(items)
         try:
@@ -289,7 +301,7 @@ def _format_items(self, items, stream, indent, allowance, context, level):
         n_items = 0
         while not last:
             if n_items == self.n_max_elements_to_show:
-                write(', ...')
+                write(", ...")
                 break
             n_items += 1
             ent = next_ent
@@ -309,34 +321,33 @@ def _format_items(self, items, stream, indent, allowance, context, level):
                 if width >= w:
                     width -= w
                     write(delim)
-                    delim = ', '
+                    delim = ", "
                     write(rep)
                     continue
             write(delim)
             delim = delimnl
-            self._format(ent, stream, indent,
-                         allowance if last else 1, context, level)
+            self._format(ent, stream, indent, allowance if last else 1, context, level)
 
-    def _pprint_key_val_tuple(self, object, stream, indent, allowance, context,
-                              level):
+    def _pprint_key_val_tuple(self, object, stream, indent, allowance, context, level):
         """Pretty printing for key-value tuples from dict or parameters."""
         k, v = object
         rep = self._repr(k, context, level)
         if isinstance(object, KeyValTupleParam):
             rep = rep.strip("'")
-            middle = '='
+            middle = "="
         else:
-            middle = ': '
+            middle = ": "
         stream.write(rep)
         stream.write(middle)
-        self._format(v, stream, indent + len(rep) + len(middle), allowance,
-                     context, level)
+        self._format(
+            v, stream, indent + len(rep) + len(middle), allowance, context, level
+        )
 
     # Note: need to copy _dispatch to prevent instances of the builtin
     # PrettyPrinter class to call methods of _EstimatorPrettyPrinter (see issue
     # 12906)
     # mypy error: "Type[PrettyPrinter]" has no attribute "_dispatch"
-    _dispatch = pprint.PrettyPrinter._dispatch.copy()  # type: ignore
+    _dispatch = pprint.PrettyPrinter._dispatch.copy()  # type: ignore[attr-defined]
     _dispatch[BaseEstimator.__repr__] = _pprint_estimator
     _dispatch[KeyValTuple.__repr__] = _pprint_key_val_tuple
 
@@ -368,9 +379,11 @@ def _safe_repr(object, context, maxlevels, level, changed_only=False):
         items = sorted(object.items(), key=pprint._safe_tuple)
         for k, v in items:
             krepr, kreadable, krecur = saferepr(
-                k, context, maxlevels, level, changed_only=changed_only)
+                k, context, maxlevels, level, changed_only=changed_only
+            )
             vrepr, vreadable, vrecur = saferepr(
-                v, context, maxlevels, level, changed_only=changed_only)
+                v, context, maxlevels, level, changed_only=changed_only
+            )
             append("%s: %s" % (krepr, vrepr))
             readable = readable and kreadable and vreadable
             if krecur or vrecur:
@@ -378,8 +391,9 @@ def _safe_repr(object, context, maxlevels, level, changed_only=False):
         del context[objid]
         return "{%s}" % ", ".join(components), readable, recursive
 
-    if (issubclass(typ, list) and r is list.__repr__) or \
-       (issubclass(typ, tuple) and r is tuple.__repr__):
+    if (issubclass(typ, list) and r is list.__repr__) or (
+        issubclass(typ, tuple) and r is tuple.__repr__
+    ):
         if issubclass(typ, list):
             if not object:
                 return "[]", True, False
@@ -403,7 +417,8 @@ def _safe_repr(object, context, maxlevels, level, changed_only=False):
         level += 1
         for o in object:
             orepr, oreadable, orecur = _safe_repr(
-                o, context, maxlevels, level, changed_only=changed_only)
+                o, context, maxlevels, level, changed_only=changed_only
+            )
             append(orepr)
             if not oreadable:
                 readable = False
@@ -415,7 +430,7 @@ def _safe_repr(object, context, maxlevels, level, changed_only=False):
     if issubclass(typ, BaseEstimator):
         objid = id(object)
         if maxlevels and level >= maxlevels:
-            return "{...}", False, objid in context
+            return f"{typ.__name__}(...)", False, objid in context
         if objid in context:
             return pprint._recursion(object), False, True
         context[objid] = 1
@@ -432,16 +447,17 @@ def _safe_repr(object, context, maxlevels, level, changed_only=False):
         items = sorted(params.items(), key=pprint._safe_tuple)
         for k, v in items:
             krepr, kreadable, krecur = saferepr(
-                k, context, maxlevels, level, changed_only=changed_only)
+                k, context, maxlevels, level, changed_only=changed_only
+            )
             vrepr, vreadable, vrecur = saferepr(
-                v, context, maxlevels, level, changed_only=changed_only)
+                v, context, maxlevels, level, changed_only=changed_only
+            )
             append("%s=%s" % (krepr.strip("'"), vrepr))
             readable = readable and kreadable and vreadable
             if krecur or vrecur:
                 recursive = True
         del context[objid]
-        return ("%s(%s)" % (typ.__name__, ", ".join(components)), readable,
-                recursive)
+        return ("%s(%s)" % (typ.__name__, ", ".join(components)), readable, recursive)
 
     rep = repr(object)
-    return rep, (rep and not rep.startswith('<')), False
+    return rep, (rep and not rep.startswith("<")), False
diff --git a/sklearn/utils/_random.pxd b/sklearn/utils/_random.pxd
index 276ab50518d40..7ac4f9774cfa4 100644
--- a/sklearn/utils/_random.pxd
+++ b/sklearn/utils/_random.pxd
@@ -1,44 +1,34 @@
-# Authors: Arnaud Joly
-#
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
 
+from ._typedefs cimport uint32_t
 
-import numpy as np
-cimport numpy as np
-ctypedef np.npy_uint32 UINT32_t
 
-cdef inline UINT32_t DEFAULT_SEED = 1
+cdef inline uint32_t DEFAULT_SEED = 1
 
 cdef enum:
     # Max value for our rand_r replacement (near the bottom).
     # We don't use RAND_MAX because it's different across platforms and
     # particularly tiny on Windows/MSVC.
-    RAND_R_MAX = 0x7FFFFFFF
+    # It corresponds to the maximum representable value for
+    # 32-bit signed integers (i.e. 2^31 - 1).
+    RAND_R_MAX = 2147483647
 
-cpdef sample_without_replacement(np.int_t n_population,
-                                 np.int_t n_samples,
-                                 method=*,
-                                 random_state=*)
 
 # rand_r replacement using a 32bit XorShift generator
 # See http://www.jstatsoft.org/v08/i14/paper for details
-cdef inline UINT32_t our_rand_r(UINT32_t* seed) nogil:
+cdef inline uint32_t our_rand_r(uint32_t* seed) nogil:
     """Generate a pseudo-random np.uint32 from a np.uint32 seed"""
     # seed shouldn't ever be 0.
-    if (seed[0] == 0): seed[0] = DEFAULT_SEED
-
-    seed[0] ^= (seed[0] << 13)
-    seed[0] ^= (seed[0] >> 17)
-    seed[0] ^= (seed[0] << 5)
-
-    # Note: we must be careful with the final line cast to np.uint32 so that
-    # the function behaves consistently across platforms.
-    #
-    # The following cast might yield different results on different platforms:
-    # wrong_cast =  RAND_R_MAX + 1
-    #
-    # We can use:
-    # good_cast = (RAND_R_MAX + 1)
-    # or:
-    # cdef np.uint32_t another_good_cast = RAND_R_MAX + 1
-    return seed[0] % (RAND_R_MAX + 1)
+    if (seed[0] == 0):
+        seed[0] = DEFAULT_SEED
+
+    seed[0] ^= (seed[0] << 13)
+    seed[0] ^= (seed[0] >> 17)
+    seed[0] ^= (seed[0] << 5)
+
+    # Use the modulo to make sure that we don't return a values greater than the
+    # maximum representable value for signed 32bit integers (i.e. 2^31 - 1).
+    # Note that the parenthesis are needed to avoid overflow: here
+    # RAND_R_MAX is cast to uint32_t before 1 is added.
+    return seed[0] % ((RAND_R_MAX) + 1)
diff --git a/sklearn/utils/_random.pyx b/sklearn/utils/_random.pyx
index 3caf062079211..f0e649e60fe7c 100644
--- a/sklearn/utils/_random.pyx
+++ b/sklearn/utils/_random.pyx
@@ -1,9 +1,6 @@
-# cython: boundscheck=False
-# cython: wraparound=False
-#
-# Author: Arnaud Joly
-#
-# License: BSD 3 clause
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
 """
 Random utility function
 =======================
@@ -13,19 +10,26 @@ The module contains:
     * Several algorithms to sample integers without replacement.
     * Fast rand_r alternative based on xor shifts
 """
-cimport cython
-
 import numpy as np
-cimport numpy as np
-np.import_array()
-
 from . import check_random_state
 
-cdef UINT32_t DEFAULT_SEED = 1
+from ._typedefs cimport intp_t
+
+
+cdef uint32_t DEFAULT_SEED = 1
 
 
-cpdef _sample_without_replacement_check_input(np.int_t n_population,
-                                              np.int_t n_samples):
+# Compatibility type to always accept the default int type used by NumPy, both
+# before and after NumPy 2. On Windows, `long` does not always match `inp_t`.
+# See the comments in the `sample_without_replacement` Python function for more
+# details.
+ctypedef fused default_int:
+    intp_t
+    long
+
+
+cpdef _sample_without_replacement_check_input(default_int n_population,
+                                              default_int n_samples):
     """ Check that input are consistent for sample_without_replacement"""
     if n_population < 0:
         raise ValueError('n_population should be greater than 0, got %s.'
@@ -38,8 +42,8 @@ cpdef _sample_without_replacement_check_input(np.int_t n_population,
 
 
 cpdef _sample_without_replacement_with_tracking_selection(
-        np.int_t n_population,
-        np.int_t n_samples,
+        default_int n_population,
+        default_int n_samples,
         random_state=None):
     r"""Sample integers without replacement.
 
@@ -81,9 +85,9 @@ cpdef _sample_without_replacement_with_tracking_selection(
     """
     _sample_without_replacement_check_input(n_population, n_samples)
 
-    cdef np.int_t i
-    cdef np.int_t j
-    cdef np.ndarray[np.int_t, ndim=1] out = np.empty((n_samples, ), dtype=int)
+    cdef default_int i
+    cdef default_int j
+    cdef default_int[::1] out = np.empty((n_samples, ), dtype=int)
 
     rng = check_random_state(random_state)
     rng_randint = rng.randint
@@ -99,11 +103,11 @@ cpdef _sample_without_replacement_with_tracking_selection(
         selected.add(j)
         out[i] = j
 
-    return out
+    return np.asarray(out)
 
 
-cpdef _sample_without_replacement_with_pool(np.int_t n_population,
-                                            np.int_t n_samples,
+cpdef _sample_without_replacement_with_pool(default_int n_population,
+                                            default_int n_samples,
                                             random_state=None):
     """Sample integers without replacement.
 
@@ -136,12 +140,10 @@ cpdef _sample_without_replacement_with_pool(np.int_t n_population,
     """
     _sample_without_replacement_check_input(n_population, n_samples)
 
-    cdef np.int_t i
-    cdef np.int_t j
-    cdef np.ndarray[np.int_t, ndim=1] out = np.empty((n_samples, ), dtype=int)
-
-    cdef np.ndarray[np.int_t, ndim=1] pool = np.empty((n_population, ),
-                                                      dtype=int)
+    cdef default_int i
+    cdef default_int j
+    cdef default_int[::1] out = np.empty((n_samples,), dtype=int)
+    cdef default_int[::1] pool = np.empty((n_population,), dtype=int)
 
     rng = check_random_state(random_state)
     rng_randint = rng.randint
@@ -155,16 +157,16 @@ cpdef _sample_without_replacement_with_pool(np.int_t n_population,
     for i in range(n_samples):
         j = rng_randint(n_population - i)  # invariant: non-selected at [0,n-i)
         out[i] = pool[j]
-        pool[j] = pool[n_population - i - 1]  # move non-selected item into
-                                              # vacancy
+        pool[j] = pool[n_population - i - 1]  # move non-selected item into vacancy
 
-    return out
+    return np.asarray(out)
 
 
 cpdef _sample_without_replacement_with_reservoir_sampling(
-    np.int_t n_population,
-    np.int_t n_samples,
-    random_state=None):
+    default_int n_population,
+    default_int n_samples,
+    random_state=None
+):
     """Sample integers without replacement.
 
     Select n_samples integers from the set [0, n_population) without
@@ -198,9 +200,9 @@ cpdef _sample_without_replacement_with_reservoir_sampling(
     """
     _sample_without_replacement_check_input(n_population, n_samples)
 
-    cdef np.int_t i
-    cdef np.int_t j
-    cdef np.ndarray[np.int_t, ndim=1] out = np.empty((n_samples, ), dtype=int)
+    cdef default_int i
+    cdef default_int j
+    cdef default_int[::1] out = np.empty((n_samples, ), dtype=int)
 
     rng = check_random_state(random_state)
     rng_randint = rng.randint
@@ -217,15 +219,58 @@ cpdef _sample_without_replacement_with_reservoir_sampling(
         if j < n_samples:
             out[j] = i
 
-    return out
+    return np.asarray(out)
 
 
-cpdef sample_without_replacement(np.int_t n_population,
-                                 np.int_t n_samples,
+cdef _sample_without_replacement(default_int n_population,
+                                 default_int n_samples,
                                  method="auto",
                                  random_state=None):
     """Sample integers without replacement.
 
+    Private function for the implementation, see sample_without_replacement
+    documentation for more details.
+    """
+    _sample_without_replacement_check_input(n_population, n_samples)
+
+    all_methods = ("auto", "tracking_selection", "reservoir_sampling", "pool")
+
+    ratio =  n_samples / n_population if n_population != 0.0 else 1.0
+
+    # Check ratio and use permutation unless ratio < 0.01 or ratio > 0.99
+    if method == "auto" and ratio > 0.01 and ratio < 0.99:
+        rng = check_random_state(random_state)
+        return rng.permutation(n_population)[:n_samples]
+
+    if method == "auto" or method == "tracking_selection":
+        # TODO the pool based method can also be used.
+        #      however, it requires special benchmark to take into account
+        #      the memory requirement of the array vs the set.
+
+        # The value 0.2 has been determined through benchmarking.
+        if ratio < 0.2:
+            return _sample_without_replacement_with_tracking_selection(
+                n_population, n_samples, random_state)
+        else:
+            return _sample_without_replacement_with_reservoir_sampling(
+                n_population, n_samples, random_state)
+
+    elif method == "reservoir_sampling":
+        return _sample_without_replacement_with_reservoir_sampling(
+            n_population, n_samples, random_state)
+
+    elif method == "pool":
+        return _sample_without_replacement_with_pool(n_population, n_samples,
+                                                     random_state)
+    else:
+        raise ValueError('Expected a method name in %s, got %s. '
+                         % (all_methods, method))
+
+
+def sample_without_replacement(
+        object n_population, object n_samples, method="auto", random_state=None):
+    """Sample integers without replacement.
+
     Select n_samples integers from the set [0, n_population) without
     replacement.
 
@@ -273,44 +318,38 @@ cpdef sample_without_replacement(np.int_t n_population,
     out : ndarray of shape (n_samples,)
         The sampled subsets of integer. The subset of selected integer might
         not be randomized, see the method argument.
-    """
-    _sample_without_replacement_check_input(n_population, n_samples)
-
-    all_methods = ("auto", "tracking_selection", "reservoir_sampling", "pool")
 
-    ratio = n_samples / n_population if n_population != 0.0 else 1.0
-
-    # Check ratio and use permutation unless ratio < 0.01 or ratio > 0.99
-    if method == "auto" and ratio > 0.01 and ratio < 0.99:
-        rng = check_random_state(random_state)
-        return rng.permutation(n_population)[:n_samples]
-
-    if method == "auto" or method == "tracking_selection":
-        # TODO the pool based method can also be used.
-        #      however, it requires special benchmark to take into account
-        #      the memory requirement of the array vs the set.
-
-        # The value 0.2 has been determined through benchmarking.
-        if ratio < 0.2:
-            return _sample_without_replacement_with_tracking_selection(
-                n_population, n_samples, random_state)
-        else:
-            return _sample_without_replacement_with_reservoir_sampling(
-                n_population, n_samples, random_state)
-
-    elif method == "reservoir_sampling":
-        return _sample_without_replacement_with_reservoir_sampling(
-            n_population, n_samples, random_state)
-
-    elif method == "pool":
-        return _sample_without_replacement_with_pool(n_population, n_samples,
-                                                     random_state)
+    Examples
+    --------
+    >>> from sklearn.utils.random import sample_without_replacement
+    >>> sample_without_replacement(10, 5, random_state=42)
+    array([8, 1, 5, 0, 7])
+    """
+    cdef:
+        intp_t n_pop_intp, n_samples_intp
+        long n_pop_long, n_samples_long
+
+    # On most platforms `np.int_ is np.intp`.  However, before NumPy 2 the
+    # default integer `np.int_` was a long which is 32bit on 64bit windows
+    # while `intp` is 64bit on 64bit platforms and 32bit on 32bit ones.
+    if np.int_ is np.intp:
+        # Branch always taken on NumPy >=2 (or when not on 64bit windows).
+        # Cython has different rules for conversion of values to integers.
+        # For NumPy <1.26.2 AND Cython 3, this first branch requires `int()`
+        # called explicitly to allow e.g. floats.
+        n_pop_intp = int(n_population)
+        n_samples_intp = int(n_samples)
+        return _sample_without_replacement(
+                n_pop_intp, n_samples_intp, method, random_state)
     else:
-        raise ValueError('Expected a method name in %s, got %s. '
-                         % (all_methods, method))
+        # Branch taken on 64bit windows with Numpy<2.0 where `long` is 32bit
+        n_pop_long = n_population
+        n_samples_long = n_samples
+        return _sample_without_replacement(
+                n_pop_long, n_samples_long, method, random_state)
 
 
 def _our_rand_r_py(seed):
     """Python utils to test the our_rand_r function"""
-    cdef UINT32_t my_seed = seed
+    cdef uint32_t my_seed = seed
     return our_rand_r(&my_seed)
diff --git a/sklearn/utils/_repr_html/__init__.py b/sklearn/utils/_repr_html/__init__.py
new file mode 100644
index 0000000000000..67dd18fb94b59
--- /dev/null
+++ b/sklearn/utils/_repr_html/__init__.py
@@ -0,0 +1,2 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
diff --git a/sklearn/utils/_repr_html/base.py b/sklearn/utils/_repr_html/base.py
new file mode 100644
index 0000000000000..28020a2a74698
--- /dev/null
+++ b/sklearn/utils/_repr_html/base.py
@@ -0,0 +1,152 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+import itertools
+
+from ... import __version__
+from ..._config import get_config
+from ..fixes import parse_version
+
+
+class _HTMLDocumentationLinkMixin:
+    """Mixin class allowing to generate a link to the API documentation.
+
+    This mixin relies on three attributes:
+    - `_doc_link_module`: it corresponds to the root module (e.g. `sklearn`). Using this
+      mixin, the default value is `sklearn`.
+    - `_doc_link_template`: it corresponds to the template used to generate the
+      link to the API documentation. Using this mixin, the default value is
+      `"https://scikit-learn.org/{version_url}/modules/generated/
+      {estimator_module}.{estimator_name}.html"`.
+    - `_doc_link_url_param_generator`: it corresponds to a function that generates the
+      parameters to be used in the template when the estimator module and name are not
+      sufficient.
+
+    The method :meth:`_get_doc_link` generates the link to the API documentation for a
+    given estimator.
+
+    This useful provides all the necessary states for
+    :func:`sklearn.utils.estimator_html_repr` to generate a link to the API
+    documentation for the estimator HTML diagram.
+
+    Examples
+    --------
+    If the default values for `_doc_link_module`, `_doc_link_template` are not suitable,
+    then you can override them and provide a method to generate the URL parameters:
+    >>> from sklearn.base import BaseEstimator
+    >>> doc_link_template = "https://address.local/{single_param}.html"
+    >>> def url_param_generator(estimator):
+    ...     return {"single_param": estimator.__class__.__name__}
+    >>> class MyEstimator(BaseEstimator):
+    ...     # use "builtins" since it is the associated module when declaring
+    ...     # the class in a docstring
+    ...     _doc_link_module = "builtins"
+    ...     _doc_link_template = doc_link_template
+    ...     _doc_link_url_param_generator = url_param_generator
+    >>> estimator = MyEstimator()
+    >>> estimator._get_doc_link()
+    'https://address.local/MyEstimator.html'
+
+    If instead of overriding the attributes inside the class definition, you want to
+    override a class instance, you can use `types.MethodType` to bind the method to the
+    instance:
+    >>> import types
+    >>> estimator = BaseEstimator()
+    >>> estimator._doc_link_template = doc_link_template
+    >>> estimator._doc_link_url_param_generator = types.MethodType(
+    ...     url_param_generator, estimator)
+    >>> estimator._get_doc_link()
+    'https://address.local/BaseEstimator.html'
+    """
+
+    _doc_link_module = "sklearn"
+    _doc_link_url_param_generator = None
+
+    @property
+    def _doc_link_template(self):
+        sklearn_version = parse_version(__version__)
+        if sklearn_version.dev is None:
+            version_url = f"{sklearn_version.major}.{sklearn_version.minor}"
+        else:
+            version_url = "dev"
+        return getattr(
+            self,
+            "__doc_link_template",
+            (
+                f"https://scikit-learn.org/{version_url}/modules/generated/"
+                "{estimator_module}.{estimator_name}.html"
+            ),
+        )
+
+    @_doc_link_template.setter
+    def _doc_link_template(self, value):
+        setattr(self, "__doc_link_template", value)
+
+    def _get_doc_link(self):
+        """Generates a link to the API documentation for a given estimator.
+
+        This method generates the link to the estimator's documentation page
+        by using the template defined by the attribute `_doc_link_template`.
+
+        Returns
+        -------
+        url : str
+            The URL to the API documentation for this estimator. If the estimator does
+            not belong to module `_doc_link_module`, the empty string (i.e. `""`) is
+            returned.
+        """
+        if self.__class__.__module__.split(".")[0] != self._doc_link_module:
+            return ""
+
+        if self._doc_link_url_param_generator is None:
+            estimator_name = self.__class__.__name__
+            # Construct the estimator's module name, up to the first private submodule.
+            # This works because in scikit-learn all public estimators are exposed at
+            # that level, even if they actually live in a private sub-module.
+            estimator_module = ".".join(
+                itertools.takewhile(
+                    lambda part: not part.startswith("_"),
+                    self.__class__.__module__.split("."),
+                )
+            )
+            return self._doc_link_template.format(
+                estimator_module=estimator_module, estimator_name=estimator_name
+            )
+        return self._doc_link_template.format(**self._doc_link_url_param_generator())
+
+
+class ReprHTMLMixin:
+    """Mixin to handle consistently the HTML representation.
+
+    When inheriting from this class, you need to define an attribute `_html_repr`
+    which is a callable that returns the HTML representation to be shown.
+    """
+
+    @property
+    def _repr_html_(self):
+        """HTML representation of estimator.
+        This is redundant with the logic of `_repr_mimebundle_`. The latter
+        should be favored in the long term, `_repr_html_` is only
+        implemented for consumers who do not interpret `_repr_mimbundle_`.
+        """
+        if get_config()["display"] != "diagram":
+            raise AttributeError(
+                "_repr_html_ is only defined when the "
+                "'display' configuration option is set to "
+                "'diagram'"
+            )
+        return self._repr_html_inner
+
+    def _repr_html_inner(self):
+        """This function is returned by the @property `_repr_html_` to make
+        `hasattr(estimator, "_repr_html_") return `True` or `False` depending
+        on `get_config()["display"]`.
+        """
+        return self._html_repr()
+
+    def _repr_mimebundle_(self, **kwargs):
+        """Mime bundle used by jupyter kernels to display estimator"""
+        output = {"text/plain": repr(self)}
+        if get_config()["display"] == "diagram":
+            output["text/html"] = self._html_repr()
+        return output
diff --git a/sklearn/utils/_repr_html/estimator.css b/sklearn/utils/_repr_html/estimator.css
new file mode 100644
index 0000000000000..ece8781c6bd76
--- /dev/null
+++ b/sklearn/utils/_repr_html/estimator.css
@@ -0,0 +1,413 @@
+#$id {
+  /* Definition of color scheme common for light and dark mode */
+  --sklearn-color-text: #000;
+  --sklearn-color-text-muted: #666;
+  --sklearn-color-line: gray;
+  /* Definition of color scheme for unfitted estimators */
+  --sklearn-color-unfitted-level-0: #fff5e6;
+  --sklearn-color-unfitted-level-1: #f6e4d2;
+  --sklearn-color-unfitted-level-2: #ffe0b3;
+  --sklearn-color-unfitted-level-3: chocolate;
+  /* Definition of color scheme for fitted estimators */
+  --sklearn-color-fitted-level-0: #f0f8ff;
+  --sklearn-color-fitted-level-1: #d4ebff;
+  --sklearn-color-fitted-level-2: #b3dbfd;
+  --sklearn-color-fitted-level-3: cornflowerblue;
+
+  /* Specific color for light theme */
+  --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));
+  --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, white)));
+  --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));
+  --sklearn-color-icon: #696969;
+
+  @media (prefers-color-scheme: dark) {
+    /* Redefinition of color scheme for dark theme */
+    --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));
+    --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, #111)));
+    --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));
+    --sklearn-color-icon: #878787;
+  }
+}
+
+#$id {
+  color: var(--sklearn-color-text);
+}
+
+#$id pre {
+  padding: 0;
+}
+
+#$id input.sk-hidden--visually {
+  border: 0;
+  clip: rect(1px 1px 1px 1px);
+  clip: rect(1px, 1px, 1px, 1px);
+  height: 1px;
+  margin: -1px;
+  overflow: hidden;
+  padding: 0;
+  position: absolute;
+  width: 1px;
+}
+
+#$id div.sk-dashed-wrapped {
+  border: 1px dashed var(--sklearn-color-line);
+  margin: 0 0.4em 0.5em 0.4em;
+  box-sizing: border-box;
+  padding-bottom: 0.4em;
+  background-color: var(--sklearn-color-background);
+}
+
+#$id div.sk-container {
+  /* jupyter's `normalize.less` sets `[hidden] { display: none; }`
+     but bootstrap.min.css set `[hidden] { display: none !important; }`
+     so we also need the `!important` here to be able to override the
+     default hidden behavior on the sphinx rendered scikit-learn.org.
+     See: https://github.com/scikit-learn/scikit-learn/issues/21755 */
+  display: inline-block !important;
+  position: relative;
+}
+
+#$id div.sk-text-repr-fallback {
+  display: none;
+}
+
+div.sk-parallel-item,
+div.sk-serial,
+div.sk-item {
+  /* draw centered vertical line to link estimators */
+  background-image: linear-gradient(var(--sklearn-color-text-on-default-background), var(--sklearn-color-text-on-default-background));
+  background-size: 2px 100%;
+  background-repeat: no-repeat;
+  background-position: center center;
+}
+
+/* Parallel-specific style estimator block */
+
+#$id div.sk-parallel-item::after {
+  content: "";
+  width: 100%;
+  border-bottom: 2px solid var(--sklearn-color-text-on-default-background);
+  flex-grow: 1;
+}
+
+#$id div.sk-parallel {
+  display: flex;
+  align-items: stretch;
+  justify-content: center;
+  background-color: var(--sklearn-color-background);
+  position: relative;
+}
+
+#$id div.sk-parallel-item {
+  display: flex;
+  flex-direction: column;
+}
+
+#$id div.sk-parallel-item:first-child::after {
+  align-self: flex-end;
+  width: 50%;
+}
+
+#$id div.sk-parallel-item:last-child::after {
+  align-self: flex-start;
+  width: 50%;
+}
+
+#$id div.sk-parallel-item:only-child::after {
+  width: 0;
+}
+
+/* Serial-specific style estimator block */
+
+#$id div.sk-serial {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  background-color: var(--sklearn-color-background);
+  padding-right: 1em;
+  padding-left: 1em;
+}
+
+
+/* Toggleable style: style used for estimator/Pipeline/ColumnTransformer box that is
+clickable and can be expanded/collapsed.
+- Pipeline and ColumnTransformer use this feature and define the default style
+- Estimators will overwrite some part of the style using the `sk-estimator` class
+*/
+
+/* Pipeline and ColumnTransformer style (default) */
+
+#$id div.sk-toggleable {
+  /* Default theme specific background. It is overwritten whether we have a
+  specific estimator or a Pipeline/ColumnTransformer */
+  background-color: var(--sklearn-color-background);
+}
+
+/* Toggleable label */
+#$id label.sk-toggleable__label {
+  cursor: pointer;
+  display: flex;
+  width: 100%;
+  margin-bottom: 0;
+  padding: 0.5em;
+  box-sizing: border-box;
+  text-align: center;
+  align-items: start;
+  justify-content: space-between;
+  gap: 0.5em;
+}
+
+#$id label.sk-toggleable__label .caption {
+  font-size: 0.6rem;
+  font-weight: lighter;
+  color: var(--sklearn-color-text-muted);
+}
+
+#$id label.sk-toggleable__label-arrow:before {
+  /* Arrow on the left of the label */
+  content: "▸";
+  float: left;
+  margin-right: 0.25em;
+  color: var(--sklearn-color-icon);
+}
+
+#$id label.sk-toggleable__label-arrow:hover:before {
+  color: var(--sklearn-color-text);
+}
+
+/* Toggleable content - dropdown */
+
+#$id div.sk-toggleable__content {
+  display: none;
+  text-align: left;
+  /* unfitted */
+  background-color: var(--sklearn-color-unfitted-level-0);
+}
+
+#$id div.sk-toggleable__content.fitted {
+  /* fitted */
+  background-color: var(--sklearn-color-fitted-level-0);
+}
+
+#$id div.sk-toggleable__content pre {
+  margin: 0.2em;
+  border-radius: 0.25em;
+  color: var(--sklearn-color-text);
+  /* unfitted */
+  background-color: var(--sklearn-color-unfitted-level-0);
+}
+
+#$id div.sk-toggleable__content.fitted pre {
+  /* unfitted */
+  background-color: var(--sklearn-color-fitted-level-0);
+}
+
+#$id input.sk-toggleable__control:checked~div.sk-toggleable__content {
+  /* Expand drop-down */
+  display: block;
+  width: 100%;
+  overflow: visible;
+}
+
+#$id input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {
+  content: "▾";
+}
+
+/* Pipeline/ColumnTransformer-specific style */
+
+#$id div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {
+  color: var(--sklearn-color-text);
+  background-color: var(--sklearn-color-unfitted-level-2);
+}
+
+#$id div.sk-label.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {
+  background-color: var(--sklearn-color-fitted-level-2);
+}
+
+/* Estimator-specific style */
+
+/* Colorize estimator box */
+#$id div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {
+  /* unfitted */
+  background-color: var(--sklearn-color-unfitted-level-2);
+}
+
+#$id div.sk-estimator.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {
+  /* fitted */
+  background-color: var(--sklearn-color-fitted-level-2);
+}
+
+#$id div.sk-label label.sk-toggleable__label,
+#$id div.sk-label label {
+  /* The background is the default theme color */
+  color: var(--sklearn-color-text-on-default-background);
+}
+
+/* On hover, darken the color of the background */
+#$id div.sk-label:hover label.sk-toggleable__label {
+  color: var(--sklearn-color-text);
+  background-color: var(--sklearn-color-unfitted-level-2);
+}
+
+/* Label box, darken color on hover, fitted */
+#$id div.sk-label.fitted:hover label.sk-toggleable__label.fitted {
+  color: var(--sklearn-color-text);
+  background-color: var(--sklearn-color-fitted-level-2);
+}
+
+/* Estimator label */
+
+#$id div.sk-label label {
+  font-family: monospace;
+  font-weight: bold;
+  display: inline-block;
+  line-height: 1.2em;
+}
+
+#$id div.sk-label-container {
+  text-align: center;
+}
+
+/* Estimator-specific */
+#$id div.sk-estimator {
+  font-family: monospace;
+  border: 1px dotted var(--sklearn-color-border-box);
+  border-radius: 0.25em;
+  box-sizing: border-box;
+  margin-bottom: 0.5em;
+  /* unfitted */
+  background-color: var(--sklearn-color-unfitted-level-0);
+}
+
+#$id div.sk-estimator.fitted {
+  /* fitted */
+  background-color: var(--sklearn-color-fitted-level-0);
+}
+
+/* on hover */
+#$id div.sk-estimator:hover {
+  /* unfitted */
+  background-color: var(--sklearn-color-unfitted-level-2);
+}
+
+#$id div.sk-estimator.fitted:hover {
+  /* fitted */
+  background-color: var(--sklearn-color-fitted-level-2);
+}
+
+/* Specification for estimator info (e.g. "i" and "?") */
+
+/* Common style for "i" and "?" */
+
+.sk-estimator-doc-link,
+a:link.sk-estimator-doc-link,
+a:visited.sk-estimator-doc-link {
+  float: right;
+  font-size: smaller;
+  line-height: 1em;
+  font-family: monospace;
+  background-color: var(--sklearn-color-background);
+  border-radius: 1em;
+  height: 1em;
+  width: 1em;
+  text-decoration: none !important;
+  margin-left: 0.5em;
+  text-align: center;
+  /* unfitted */
+  border: var(--sklearn-color-unfitted-level-1) 1pt solid;
+  color: var(--sklearn-color-unfitted-level-1);
+}
+
+.sk-estimator-doc-link.fitted,
+a:link.sk-estimator-doc-link.fitted,
+a:visited.sk-estimator-doc-link.fitted {
+  /* fitted */
+  border: var(--sklearn-color-fitted-level-1) 1pt solid;
+  color: var(--sklearn-color-fitted-level-1);
+}
+
+/* On hover */
+div.sk-estimator:hover .sk-estimator-doc-link:hover,
+.sk-estimator-doc-link:hover,
+div.sk-label-container:hover .sk-estimator-doc-link:hover,
+.sk-estimator-doc-link:hover {
+  /* unfitted */
+  background-color: var(--sklearn-color-unfitted-level-3);
+  color: var(--sklearn-color-background);
+  text-decoration: none;
+}
+
+div.sk-estimator.fitted:hover .sk-estimator-doc-link.fitted:hover,
+.sk-estimator-doc-link.fitted:hover,
+div.sk-label-container:hover .sk-estimator-doc-link.fitted:hover,
+.sk-estimator-doc-link.fitted:hover {
+  /* fitted */
+  background-color: var(--sklearn-color-fitted-level-3);
+  color: var(--sklearn-color-background);
+  text-decoration: none;
+}
+
+/* Span, style for the box shown on hovering the info icon */
+.sk-estimator-doc-link span {
+  display: none;
+  z-index: 9999;
+  position: relative;
+  font-weight: normal;
+  right: .2ex;
+  padding: .5ex;
+  margin: .5ex;
+  width: min-content;
+  min-width: 20ex;
+  max-width: 50ex;
+  color: var(--sklearn-color-text);
+  box-shadow: 2pt 2pt 4pt #999;
+  /* unfitted */
+  background: var(--sklearn-color-unfitted-level-0);
+  border: .5pt solid var(--sklearn-color-unfitted-level-3);
+}
+
+.sk-estimator-doc-link.fitted span {
+  /* fitted */
+  background: var(--sklearn-color-fitted-level-0);
+  border: var(--sklearn-color-fitted-level-3);
+}
+
+.sk-estimator-doc-link:hover span {
+  display: block;
+}
+
+/* "?"-specific style due to the `` HTML tag */
+
+#$id a.estimator_doc_link {
+  float: right;
+  font-size: 1rem;
+  line-height: 1em;
+  font-family: monospace;
+  background-color: var(--sklearn-color-background);
+  border-radius: 1rem;
+  height: 1rem;
+  width: 1rem;
+  text-decoration: none;
+  /* unfitted */
+  color: var(--sklearn-color-unfitted-level-1);
+  border: var(--sklearn-color-unfitted-level-1) 1pt solid;
+}
+
+#$id a.estimator_doc_link.fitted {
+  /* fitted */
+  border: var(--sklearn-color-fitted-level-1) 1pt solid;
+  color: var(--sklearn-color-fitted-level-1);
+}
+
+/* On hover */
+#$id a.estimator_doc_link:hover {
+  /* unfitted */
+  background-color: var(--sklearn-color-unfitted-level-3);
+  color: var(--sklearn-color-background);
+  text-decoration: none;
+}
+
+#$id a.estimator_doc_link.fitted:hover {
+  /* fitted */
+  background-color: var(--sklearn-color-fitted-level-3);
+}
diff --git a/sklearn/utils/_repr_html/estimator.js b/sklearn/utils/_repr_html/estimator.js
new file mode 100644
index 0000000000000..5de0a021c63bb
--- /dev/null
+++ b/sklearn/utils/_repr_html/estimator.js
@@ -0,0 +1,42 @@
+function copyToClipboard(text, element) {
+    // Get the parameter prefix from the closest toggleable content
+    const toggleableContent = element.closest('.sk-toggleable__content');
+    const paramPrefix = toggleableContent ? toggleableContent.dataset.paramPrefix : '';
+    const fullParamName = paramPrefix ? `${paramPrefix}${text}` : text;
+
+    const originalStyle = element.style;
+    const computedStyle = window.getComputedStyle(element);
+    const originalWidth = computedStyle.width;
+    const originalHTML = element.innerHTML.replace('Copied!', '');
+
+    navigator.clipboard.writeText(fullParamName)
+        .then(() => {
+            element.style.width = originalWidth;
+            element.style.color = 'green';
+            element.innerHTML = "Copied!";
+
+            setTimeout(() => {
+                element.innerHTML = originalHTML;
+                element.style = originalStyle;
+            }, 2000);
+        })
+        .catch(err => {
+            console.error('Failed to copy:', err);
+            element.style.color = 'red';
+            element.innerHTML = "Failed!";
+            setTimeout(() => {
+                element.innerHTML = originalHTML;
+                element.style = originalStyle;
+            }, 2000);
+        });
+    return false;
+}
+
+document.querySelectorAll('.fa-regular.fa-copy').forEach(function(element) {
+    const toggleableContent = element.closest('.sk-toggleable__content');
+    const paramPrefix = toggleableContent ? toggleableContent.dataset.paramPrefix : '';
+    const paramName = element.parentElement.nextElementSibling.textContent.trim();
+    const fullParamName = paramPrefix ? `${paramPrefix}${paramName}` : paramName;
+
+    element.setAttribute('title', fullParamName);
+});
diff --git a/sklearn/utils/_repr_html/estimator.py b/sklearn/utils/_repr_html/estimator.py
new file mode 100644
index 0000000000000..7d101dde58d74
--- /dev/null
+++ b/sklearn/utils/_repr_html/estimator.py
@@ -0,0 +1,497 @@
+# Authors: The scikit-learn developers
+# SPDX-License-Identifier: BSD-3-Clause
+
+import html
+from contextlib import closing
+from inspect import isclass
+from io import StringIO
+from pathlib import Path
+from string import Template
+
+from ... import config_context
+
+
+class _IDCounter:
+    """Generate sequential ids with a prefix."""
+
+    def __init__(self, prefix):
+        self.prefix = prefix
+        self.count = 0
+
+    def get_id(self):
+        self.count += 1
+        return f"{self.prefix}-{self.count}"
+
+
+def _get_css_style():
+    estimator_css_file = Path(__file__).parent / "estimator.css"
+    params_css_file = Path(__file__).parent / "params.css"
+
+    estimator_css = estimator_css_file.read_text(encoding="utf-8")
+    params_css = params_css_file.read_text(encoding="utf-8")
+
+    return f"{estimator_css}\n{params_css}"
+
+
+_CONTAINER_ID_COUNTER = _IDCounter("sk-container-id")
+_ESTIMATOR_ID_COUNTER = _IDCounter("sk-estimator-id")
+_CSS_STYLE = _get_css_style()
+
+
+class _VisualBlock:
+    """HTML Representation of Estimator
+
+    Parameters
+    ----------
+    kind : {'serial', 'parallel', 'single'}
+        kind of HTML block
+
+    estimators : list of estimators or `_VisualBlock`s or a single estimator
+        If kind != 'single', then `estimators` is a list of
+        estimators.
+        If kind == 'single', then `estimators` is a single estimator.
+
+    names : list of str, default=None
+        If kind != 'single', then `names` corresponds to estimators.
+        If kind == 'single', then `names` is a single string corresponding to
+        the single estimator.
+
+    name_details : list of str, str, or None, default=None
+        If kind != 'single', then `name_details` corresponds to `names`.
+        If kind == 'single', then `name_details` is a single string
+        corresponding to the single estimator.
+
+    name_caption : str, default=None
+        The caption below the name. `None` stands for no caption.
+        Only active when kind == 'single'.
+
+    doc_link_label : str, default=None
+        The label for the documentation link. If provided, the label would be
+        "Documentation for {doc_link_label}". Otherwise it will look for `names`.
+        Only active when kind == 'single'.
+
+    dash_wrapped : bool, default=True
+        If true, wrapped HTML element will be wrapped with a dashed border.
+        Only active when kind != 'single'.
+    """
+
+    def __init__(
+        self,
+        kind,
+        estimators,
+        *,
+        names=None,
+        name_details=None,
+        name_caption=None,
+        doc_link_label=None,
+        dash_wrapped=True,
+    ):
+        self.kind = kind
+        self.estimators = estimators
+        self.dash_wrapped = dash_wrapped
+        self.name_caption = name_caption
+        self.doc_link_label = doc_link_label
+
+        if self.kind in ("parallel", "serial"):
+            if names is None:
+                names = (None,) * len(estimators)
+            if name_details is None:
+                name_details = (None,) * len(estimators)
+
+        self.names = names
+        self.name_details = name_details
+
+    def _sk_visual_block_(self):
+        return self
+
+
+def _write_label_html(
+    out,
+    params,
+    name,
+    name_details,
+    name_caption=None,
+    doc_link_label=None,
+    outer_class="sk-label-container",
+    inner_class="sk-label",
+    checked=False,
+    doc_link="",
+    is_fitted_css_class="",
+    is_fitted_icon="",
+    param_prefix="",
+):
+    """Write labeled html with or without a dropdown with named details.
+
+    Parameters
+    ----------
+    out : file-like object
+        The file to write the HTML representation to.
+    params: str
+        If estimator has `get_params` method, this is the HTML representation
+        of the estimator's parameters and their values. When the estimator
+        does not have `get_params`, it is an empty string.
+    name : str
+        The label for the estimator. It corresponds either to the estimator class name
+        for a simple estimator or in the case of a `Pipeline` and `ColumnTransformer`,
+        it corresponds to the name of the step.
+    name_details : str
+        The details to show as content in the dropdown part of the toggleable label. It
+        can contain information such as non-default parameters or column information for
+        `ColumnTransformer`.
+    name_caption : str, default=None
+        The caption below the name. If `None`, no caption will be created.
+    doc_link_label : str, default=None
+        The label for the documentation link. If provided, the label would be
+        "Documentation for {doc_link_label}". Otherwise it will look for `name`.
+    outer_class : {"sk-label-container", "sk-item"}, default="sk-label-container"
+        The CSS class for the outer container.
+    inner_class : {"sk-label", "sk-estimator"}, default="sk-label"
+        The CSS class for the inner container.
+    checked : bool, default=False
+        Whether the dropdown is folded or not. With a single estimator, we intend to
+        unfold the content.
+    doc_link : str, default=""
+        The link to the documentation for the estimator. If an empty string, no link is
+        added to the diagram. This can be generated for an estimator if it uses the
+        `_HTMLDocumentationLinkMixin`.
+    is_fitted_css_class : {"", "fitted"}
+        The CSS class to indicate whether or not the estimator is fitted. The
+        empty string means that the estimator is not fitted and "fitted" means that the
+        estimator is fitted.
+    is_fitted_icon : str, default=""
+        The HTML representation to show the fitted information in the diagram. An empty
+        string means that no information is shown.
+    param_prefix : str, default=""
+        The prefix to prepend to parameter names for nested estimators.
+    """
+    out.write(
+        f'
'
+    )
+    name = html.escape(name)
+    if name_details is not None:
+        name_details = html.escape(str(name_details))
+        checked_str = "checked" if checked else ""
+        est_id = _ESTIMATOR_ID_COUNTER.get_id()
+
+        if doc_link:
+            doc_label = "Online documentation"
+            if doc_link_label is not None:
+                doc_label = f"Documentation for {doc_link_label}"
+            elif name is not None:
+                doc_label = f"Documentation for {name}"
+            doc_link = (
+                f'?{doc_label}'
+            )
+
+        name_caption_div = (
+            ""
+            if name_caption is None
+            else f'
{html.escape(name_caption)}
'
+        )
+        name_caption_div = f"
{name}
{name_caption_div}
"
+        links_div = (
+            f"
{doc_link}{is_fitted_icon}
"
+            if doc_link or is_fitted_icon
+            else ""
+        )
+
+        label_html = (
+            f''
+        )
+
+        fmt_str = (
+            f'{label_html}
'
+        )
+
+        if params:
+            fmt_str = "".join([fmt_str, f"{params}
"])
+        elif name_details and ("Pipeline" not in name):
+            fmt_str = "".join([fmt_str, f"
{name_details}
"])
+
+        out.write(fmt_str)
+    else:
+        out.write(f"")
+    out.write("
") # outer_class inner_class + + +def _get_visual_block(estimator): + """Generate information about how to display an estimator.""" + if hasattr(estimator, "_sk_visual_block_"): + try: + return estimator._sk_visual_block_() + except Exception: + return _VisualBlock( + "single", + estimator, + names=estimator.__class__.__name__, + name_details=str(estimator), + ) + + if isinstance(estimator, str): + return _VisualBlock( + "single", estimator, names=estimator, name_details=estimator + ) + elif estimator is None: + return _VisualBlock("single", estimator, names="None", name_details="None") + + # check if estimator looks like a meta estimator (wraps estimators) + if hasattr(estimator, "get_params") and not isclass(estimator): + estimators = [ + (key, est) + for key, est in estimator.get_params(deep=False).items() + if hasattr(est, "get_params") and hasattr(est, "fit") and not isclass(est) + ] + if estimators: + return _VisualBlock( + "parallel", + [est for _, est in estimators], + names=[f"{key}: {est.__class__.__name__}" for key, est in estimators], + name_details=[str(est) for _, est in estimators], + ) + + return _VisualBlock( + "single", + estimator, + names=estimator.__class__.__name__, + name_details=str(estimator), + ) + + +def _write_estimator_html( + out, + estimator, + estimator_label, + estimator_label_details, + is_fitted_css_class, + is_fitted_icon="", + first_call=False, + param_prefix="", +): + """Write estimator to html in serial, parallel, or by itself (single). + + For multiple estimators, this function is called recursively. + + Parameters + ---------- + out : file-like object + The file to write the HTML representation to. + estimator : estimator object + The estimator to visualize. + estimator_label : str + The label for the estimator. It corresponds either to the estimator class name + for simple estimator or in the case of `Pipeline` and `ColumnTransformer`, it + corresponds to the name of the step. + estimator_label_details : str + The details to show as content in the dropdown part of the toggleable label. + It can contain information as non-default parameters or column information for + `ColumnTransformer`. + is_fitted_css_class : {"", "fitted"} + The CSS class to indicate whether or not the estimator is fitted or not. The + empty string means that the estimator is not fitted and "fitted" means that the + estimator is fitted. + is_fitted_icon : str, default="" + The HTML representation to show the fitted information in the diagram. An empty + string means that no information is shown. If the estimator to be shown is not + the first estimator (i.e. `first_call=False`), `is_fitted_icon` is always an + empty string. + first_call : bool, default=False + Whether this is the first time this function is called. + param_prefix : str, default="" + The prefix to prepend to parameter names for nested estimators. + For example, in a pipeline this might be "pipeline__stepname__". + """ + if first_call: + est_block = _get_visual_block(estimator) + else: + is_fitted_icon = "" + with config_context(print_changed_only=True): + est_block = _get_visual_block(estimator) + # `estimator` can also be an instance of `_VisualBlock` + if hasattr(estimator, "_get_doc_link"): + doc_link = estimator._get_doc_link() + else: + doc_link = "" + if est_block.kind in ("serial", "parallel"): + dashed_wrapped = first_call or est_block.dash_wrapped + dash_cls = " sk-dashed-wrapped" if dashed_wrapped else "" + out.write(f'
') + + if estimator_label: + if hasattr(estimator, "get_params") and hasattr( + estimator, "_get_params_html" + ): + params = estimator._get_params_html(deep=False)._repr_html_inner() + else: + params = "" + + _write_label_html( + out, + params, + estimator_label, + estimator_label_details, + doc_link=doc_link, + is_fitted_css_class=is_fitted_css_class, + is_fitted_icon=is_fitted_icon, + param_prefix=param_prefix, + ) + + kind = est_block.kind + out.write(f'
') + est_infos = zip(est_block.estimators, est_block.names, est_block.name_details) + + for est, name, name_details in est_infos: + # Build the parameter prefix for nested estimators + + if param_prefix and hasattr(name, "split"): + # If we already have a prefix, append the new component + new_prefix = f"{param_prefix}{name.split(':')[0]}__" + elif hasattr(name, "split"): + # If this is the first level, start the prefix + new_prefix = f"{name.split(':')[0]}__" if name else "" + else: + new_prefix = param_prefix + + if kind == "serial": + _write_estimator_html( + out, + est, + name, + name_details, + is_fitted_css_class=is_fitted_css_class, + param_prefix=new_prefix, + ) + else: # parallel + out.write('
') + # wrap element in a serial visualblock + serial_block = _VisualBlock("serial", [est], dash_wrapped=False) + _write_estimator_html( + out, + serial_block, + name, + name_details, + is_fitted_css_class=is_fitted_css_class, + param_prefix=new_prefix, + ) + out.write("
") # sk-parallel-item + + out.write("
") + elif est_block.kind == "single": + if hasattr(estimator, "_get_params_html"): + params = estimator._get_params_html()._repr_html_inner() + else: + params = "" + + _write_label_html( + out, + params, + est_block.names, + est_block.name_details, + est_block.name_caption, + est_block.doc_link_label, + outer_class="sk-item", + inner_class="sk-estimator", + checked=first_call, + doc_link=doc_link, + is_fitted_css_class=is_fitted_css_class, + is_fitted_icon=is_fitted_icon, + param_prefix=param_prefix, + ) + + +def estimator_html_repr(estimator): + """Build a HTML representation of an estimator. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + estimator : estimator object + The estimator to visualize. + + Returns + ------- + html: str + HTML representation of estimator. + + Examples + -------- + >>> from sklearn.utils._repr_html.estimator import estimator_html_repr + >>> from sklearn.linear_model import LogisticRegression + >>> estimator_html_repr(LogisticRegression()) + '" + f"" + f'
' + '
' + f"
{html.escape(estimator_str)}
{fallback_msg}" + "
" + '
" + + out.write(html_end) + + html_output = out.getvalue() + return html_output diff --git a/sklearn/utils/_repr_html/params.css b/sklearn/utils/_repr_html/params.css new file mode 100644 index 0000000000000..df815f966ffcf --- /dev/null +++ b/sklearn/utils/_repr_html/params.css @@ -0,0 +1,63 @@ +.estimator-table summary { + padding: .5rem; + font-family: monospace; + cursor: pointer; +} + +.estimator-table details[open] { + padding-left: 0.1rem; + padding-right: 0.1rem; + padding-bottom: 0.3rem; +} + +.estimator-table .parameters-table { + margin-left: auto !important; + margin-right: auto !important; +} + +.estimator-table .parameters-table tr:nth-child(odd) { + background-color: #fff; +} + +.estimator-table .parameters-table tr:nth-child(even) { + background-color: #f6f6f6; +} + +.estimator-table .parameters-table tr:hover { + background-color: #e0e0e0; +} + +.estimator-table table td { + border: 1px solid rgba(106, 105, 104, 0.232); +} + +.user-set td { + color:rgb(255, 94, 0); + text-align: left; +} + +.user-set td.value pre { + color:rgb(255, 94, 0) !important; + background-color: transparent !important; +} + +.default td { + color: black; + text-align: left; +} + +.user-set td i, +.default td i { + color: black; +} + +.copy-paste-icon { + background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0NDggNTEyIj48IS0tIUZvbnQgQXdlc29tZSBGcmVlIDYuNy4yIGJ5IEBmb250YXdlc29tZSAtIGh0dHBzOi8vZm9udGF3ZXNvbWUuY29tIExpY2Vuc2UgLSBodHRwczovL2ZvbnRhd2Vzb21lLmNvbS9saWNlbnNlL2ZyZWUgQ29weXJpZ2h0IDIwMjUgRm9udGljb25zLCBJbmMuLS0+PHBhdGggZD0iTTIwOCAwTDMzMi4xIDBjMTIuNyAwIDI0LjkgNS4xIDMzLjkgMTQuMWw2Ny45IDY3LjljOSA5IDE0LjEgMjEuMiAxNC4xIDMzLjlMNDQ4IDMzNmMwIDI2LjUtMjEuNSA0OC00OCA0OGwtMTkyIDBjLTI2LjUgMC00OC0yMS41LTQ4LTQ4bDAtMjg4YzAtMjYuNSAyMS41LTQ4IDQ4LTQ4ek00OCAxMjhsODAgMCAwIDY0LTY0IDAgMCAyNTYgMTkyIDAgMC0zMiA2NCAwIDAgNDhjMCAyNi41LTIxLjUgNDgtNDggNDhMNDggNTEyYy0yNi41IDAtNDgtMjEuNS00OC00OEwwIDE3NmMwLTI2LjUgMjEuNS00OCA0OC00OHoiLz48L3N2Zz4=); + background-repeat: no-repeat; + background-size: 14px 14px; + background-position: 0; + display: inline-block; + width: 14px; + height: 14px; + cursor: pointer; +} diff --git a/sklearn/utils/_repr_html/params.py b/sklearn/utils/_repr_html/params.py new file mode 100644 index 0000000000000..d85bf1280a8fc --- /dev/null +++ b/sklearn/utils/_repr_html/params.py @@ -0,0 +1,83 @@ +# Authors: The scikit-learn developers +# SPDX-License-Identifier: BSD-3-Clause + +import html +import reprlib +from collections import UserDict + +from sklearn.utils._repr_html.base import ReprHTMLMixin + + +def _read_params(name, value, non_default_params): + """Categorizes parameters as 'default' or 'user-set' and formats their values. + Escapes or truncates parameter values for display safety and readability. + """ + r = reprlib.Repr() + r.maxlist = 2 # Show only first 2 items of lists + r.maxtuple = 1 # Show only first item of tuples + r.maxstring = 50 # Limit string length + cleaned_value = html.escape(r.repr(value)) + + param_type = "user-set" if name in non_default_params else "default" + + return {"param_type": param_type, "param_name": name, "param_value": cleaned_value} + + +def _params_html_repr(params): + """Generate HTML representation of estimator parameters. + + Creates an HTML table with parameter names and values, wrapped in a + collapsible details element. Parameters are styled differently based + on whether they are default or user-set values. + """ + HTML_TEMPLATE = """ +
+
+ Parameters + + + {rows} + +
+
+
+ """ + ROW_TEMPLATE = """ + + + {param_name}  + {param_value} + + """ + + rows = [ + ROW_TEMPLATE.format(**_read_params(name, value, params.non_default)) + for name, value in params.items() + ] + + return HTML_TEMPLATE.format(rows="\n".join(rows)) + + +class ParamsDict(ReprHTMLMixin, UserDict): + """Dictionary-like class to store and provide an HTML representation. + + It builds an HTML structure to be used with Jupyter notebooks or similar + environments. It allows storing metadata to track non-default parameters. + + Parameters + ---------- + params : dict, default=None + The original dictionary of parameters and their values. + + non_default : tuple + The list of non-default parameters. + """ + + _html_repr = _params_html_repr + + def __init__(self, params=None, non_default=tuple()): + super().__init__(params or {}) + self.non_default = non_default diff --git a/sklearn/utils/_repr_html/tests/__init__.py b/sklearn/utils/_repr_html/tests/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/sklearn/utils/_repr_html/tests/test_estimator.py b/sklearn/utils/_repr_html/tests/test_estimator.py new file mode 100644 index 0000000000000..cc975d854ed8f --- /dev/null +++ b/sklearn/utils/_repr_html/tests/test_estimator.py @@ -0,0 +1,616 @@ +import html +import locale +import re +import types +from contextlib import closing +from functools import partial +from io import StringIO +from unittest.mock import patch + +import numpy as np +import pytest + +from sklearn import config_context +from sklearn.base import BaseEstimator +from sklearn.cluster import AgglomerativeClustering, Birch +from sklearn.compose import ColumnTransformer, make_column_transformer +from sklearn.datasets import load_iris +from sklearn.decomposition import PCA, TruncatedSVD +from sklearn.ensemble import StackingClassifier, StackingRegressor, VotingClassifier +from sklearn.feature_selection import SelectPercentile +from sklearn.gaussian_process.kernels import ExpSineSquared +from sklearn.impute import SimpleImputer +from sklearn.kernel_ridge import KernelRidge +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import RandomizedSearchCV +from sklearn.multiclass import OneVsOneClassifier +from sklearn.neural_network import MLPClassifier +from sklearn.pipeline import FeatureUnion, Pipeline, make_pipeline +from sklearn.preprocessing import FunctionTransformer, OneHotEncoder, StandardScaler +from sklearn.svm import LinearSVC, LinearSVR +from sklearn.tree import DecisionTreeClassifier +from sklearn.utils._repr_html.base import _HTMLDocumentationLinkMixin +from sklearn.utils._repr_html.estimator import ( + _get_css_style, + _get_visual_block, + _write_label_html, + estimator_html_repr, +) +from sklearn.utils.fixes import parse_version + + +def dummy_function(x, y): + return x + y # pragma: nocover + + +@pytest.mark.parametrize("checked", [True, False]) +def test_write_label_html(checked): + # Test checking logic and labeling + name = "LogisticRegression" + params = "" + tool_tip = "hello-world" + + with closing(StringIO()) as out: + _write_label_html(out, params, name, tool_tip, checked=checked) + html_label = out.getvalue() + + p = ( + r'